diff --git a/.clang-format b/.clang-format
index 6a3de86ab27a4..5ef5743b77c95 100644
--- a/.clang-format
+++ b/.clang-format
@@ -167,7 +167,7 @@ ForEachMacros:
   - 'drm_connector_for_each_possible_encoder'
   - 'drm_exec_for_each_locked_object'
   - 'drm_exec_for_each_locked_object_reverse'
-  - 'drm_for_each_bridge_in_chain_scoped'
+  - 'drm_for_each_bridge_in_chain'
   - 'drm_for_each_connector_iter'
   - 'drm_for_each_crtc'
   - 'drm_for_each_crtc_reverse'
diff --git a/.get_maintainer.ignore b/.get_maintainer.ignore
index 60b6b2a374cd6..5ad082b4dd036 100644
--- a/.get_maintainer.ignore
+++ b/.get_maintainer.ignore
@@ -5,4 +5,5 @@ Askar Safin <safinaskar@gmail.com>
 Christoph Hellwig <hch@lst.de>
 Jeff Kirsher <jeffrey.t.kirsher@intel.com>
 Marc Gonzalez <marc.w.gonzalez@free.fr>
+Nathan Chancellor <nathan@kernel.org>
 Ralf Baechle <ralf@linux-mips.org>
diff --git a/.mailmap b/.mailmap
index 12f3acdebd722..22e5f70bbf6a0 100644
--- a/.mailmap
+++ b/.mailmap
@@ -185,6 +185,7 @@ Carlos Bilbao <carlos.bilbao@kernel.org> <bilbao@vt.edu>
 Changbin Du <changbin.du@intel.com> <changbin.du@gmail.com>
 Chao Yu <chao@kernel.org> <chao2.yu@samsung.com>
 Chao Yu <chao@kernel.org> <yuchao0@huawei.com>
+Charlie Jenkins <thecharlesjenkins@gmail.com> <charlie@rivosinc.com>
 Chen-Yu Tsai <wens@kernel.org> <wens@csie.org>
 Chester Lin <chester62515@gmail.com> <clin@suse.com>
 Chris Chiu <chris.chiu@canonical.com> <chiu@endlessm.com>
@@ -399,6 +400,7 @@ Jens Axboe <axboe@kernel.dk> <jens.axboe@oracle.com>
 Jens Axboe <axboe@kernel.dk> <axboe@fb.com>
 Jens Axboe <axboe@kernel.dk> <axboe@meta.com>
 Jens Osterkamp <Jens.Osterkamp@de.ibm.com>
+Jens Wiklander <jenswi@kernel.org> <jens.wiklander@linaro.org>
 Jernej Skrabec <jernej.skrabec@gmail.com> <jernej.skrabec@siol.net>
 Jesper Dangaard Brouer <hawk@kernel.org> <brouer@redhat.com>
 Jesper Dangaard Brouer <hawk@kernel.org> <hawk@comx.dk>
@@ -559,6 +561,7 @@ Martyna Szapar-Mudlaw <martyna.szapar-mudlaw@linux.intel.com> <martyna.szapar-mu
 Mathieu Othacehe <othacehe@gnu.org> <m.othacehe@gmail.com>
 Mat Martineau <martineau@kernel.org> <mathew.j.martineau@linux.intel.com>
 Mat Martineau <martineau@kernel.org> <mathewm@codeaurora.org>
+Matt Coster <opensource@mtcoster.net> <matt.coster@imgtec.com>
 Matthew Wilcox <willy@infradead.org> <matthew.r.wilcox@intel.com>
 Matthew Wilcox <willy@infradead.org> <matthew@wil.cx>
 Matthew Wilcox <willy@infradead.org> <mawilcox@linuxonhyperv.com>
@@ -816,6 +819,7 @@ Simon Wunderlich <sw@simonwunderlich.de> <simon.wunderlich@s2003.tu-chemnitz.de>
 Simon Wunderlich <sw@simonwunderlich.de> <simon.wunderlich@saxnet.de>
 Simon Wunderlich <sw@simonwunderlich.de> <simon@open-mesh.com>
 Simon Wunderlich <sw@simonwunderlich.de> <siwu@hrz.tu-chemnitz.de>
+SJ Park <sj@kernel.org>
 Sricharan Ramabadhran <quic_srichara@quicinc.com> <sricharan@codeaurora.org>
 Srinivas Kandagatla <srini@kernel.org> <srinivas.kandagatla@st.com>
 Srinivas Kandagatla <srini@kernel.org> <srinivas.kandagatla@linaro.org>
@@ -825,8 +829,8 @@ Sriram Yagnaraman <sriram.yagnaraman@ericsson.com> <sriram.yagnaraman@est.tech>
 Stanislav Fomichev <sdf@fomichev.me> <sdf@google.com>
 Stanislav Fomichev <sdf@fomichev.me> <stfomichev@gmail.com>
 Stefan Wahren <wahrenst@gmx.net> <stefan.wahren@i2se.com>
-Stéphane Grosjean <stephane.grosjean@hms-networks.com> <s.grosjean@peak-system.com>
-Stéphane Grosjean <stephane.grosjean@hms-networks.com> <stephane.grosjean@free.fr>
+Stéphane Grosjean <s.grosjean@peak-system.fr> <s.grosjean@peak-system.com>
+Stéphane Grosjean <s.grosjean@peak-system.fr> <stephane.grosjean@free.fr>
 Stéphane Witzmann <stephane.witzmann@ubpmes.univ-bpclermont.fr>
 Stephen Hemminger <stephen@networkplumber.org> <shemminger@linux-foundation.org>
 Stephen Hemminger <stephen@networkplumber.org> <shemminger@osdl.org>
@@ -872,6 +876,7 @@ Tobias Klauser <tklauser@distanz.ch> <klto@zhaw.ch>
 Tobias Klauser <tklauser@distanz.ch> <tklauser@nuerscht.ch>
 Tobias Klauser <tklauser@distanz.ch> <tklauser@xenon.tklauser.home>
 Todor Tomov <todor.too@gmail.com> <todor.tomov@linaro.org>
+Tomasz Duszynski <tduszyns@gmail.com> <tomasz.duszynski@octakon.com>
 Tomasz Jeznach <tomasz.jeznach@linux.dev> <tjeznach@rivosinc.com>
 Tony Luck <tony.luck@intel.com>
 Trilok Soni <quic_tsoni@quicinc.com> <tsoni@codeaurora.org>
diff --git a/CREDITS b/CREDITS
index 84793a967a0b2..0806ab21c1d37 100644
--- a/CREDITS
+++ b/CREDITS
@@ -640,6 +640,10 @@ S: 25-29 St Giles
 S: Oxford
 S: United Kingdom
 
+N: Bingbu Cao
+E: bingbu.cao@amd.com
+D: Ipu6, ipu7 and camera sensor drivers
+
 N: Luiz Fernando N. Capitulino
 E: lcapitulino@mandriva.com.br
 E: lcapitulino@gmail.com
@@ -1503,6 +1507,10 @@ N: Andy Gross
 E: agross@kernel.org
 D: Qualcomm SoC subsystem and drivers
 
+N: Mark Gross
+E: markgross@kernel.org
+D: x86/mellanox platform maintenance and various x86 specific drivers
+
 N: Grant Grundler
 E: grantgrundler@gmail.com
 W: http://obmouse.sourceforge.net/
@@ -3096,6 +3104,10 @@ N: Jens Osterkamp
 E: jens@de.ibm.com
 D: Maintainer of Spidernet network driver for Cell
 
+N: Kent Overstreet
+E: kent.overstreet@linux.dev
+D: Co-authored and contributed to Memory Allocation Profiling
+
 N: Gadi Oxman
 E: gadio@netvision.net.il
 D: Original author and maintainer of IDE/ATAPI floppy/tape drivers
@@ -3334,6 +3346,9 @@ E: rpurdie@rpsys.net
 D: Backlight subsystem maintainer
 S: United Kingdom
 
+N: Tian Shu Qiu
+D: Ipu6, ipu7 and camera sensor drivers
+
 N: Daniel Quinlan
 E: quinlan@pathname.com
 W: https://www.pathname.com/~quinlan/
@@ -3626,6 +3641,13 @@ S: 69 rue Dunois
 S: 75013 Paris
 S: France
 
+N: Wolfram Sang
+E: wsa@kernel.org
+W: sang-engineering.com
+P: rsa4096/140DE4CC14A029B6 3991 B1EA B9E2 6751 A4F7 645D 140D E4CC 14A0 29B6
+D: I2C Maintainer 2012 - 2026
+S: Berlin, Germany
+
 N: Aleksa Sarai
 E: cyphar@cyphar.com
 W: https://www.cyphar.com/
diff --git a/Documentation/ABI/README b/Documentation/ABI/README
index 315fffe1f831f..27c962e6a872c 100644
--- a/Documentation/ABI/README
+++ b/Documentation/ABI/README
@@ -62,7 +62,7 @@ Users:		All users of this interface who wish to be notified when
 
 
 Note:
-   The fields should be use a simple notation, compatible with ReST markup.
+   The fields should use a simple notation, compatible with ReST markup.
    Also, the file **should not** have a top-level index, like::
 
 	===
diff --git a/Documentation/ABI/obsolete/sysfs-bus-usb b/Documentation/ABI/obsolete/sysfs-bus-usb
index bd096d33fbc73..067016e62e11b 100644
--- a/Documentation/ABI/obsolete/sysfs-bus-usb
+++ b/Documentation/ABI/obsolete/sysfs-bus-usb
@@ -26,6 +26,6 @@ Description:
 		initializes all non-hub devices in the "on" level.  Some
 		drivers may change this setting when they are bound.
 
-		This file is deprecated and will be removed after 2010.
+		This file is deprecated.
 		Use the power/control file instead; it does exactly the
 		same thing.
diff --git a/Documentation/ABI/testing/amdpmf-interface b/Documentation/ABI/testing/amdpmf-interface
new file mode 100644
index 0000000000000..625f90a013d04
--- /dev/null
+++ b/Documentation/ABI/testing/amdpmf-interface
@@ -0,0 +1,73 @@
+What:		/dev/amdpmf_interface
+Date:		June 2026
+KernelVersion:	7.2
+Contact:	Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
+Description:
+		The AMD Platform Management Framework (PMF) util layer exposes a
+		minimal user-space interface via a character device for feature
+		discovery and metrics monitoring.
+
+		When CONFIG_AMD_PMF_UTIL_SUPPORT is enabled, the driver creates
+		a character device:
+
+		======================
+		/dev/amdpmf_interface
+		======================
+
+		The interface supports a single ioctl:
+
+		============================ =======================================
+		IOCTL			     Usage
+		IOCTL_AMD_PMF_POPULATE_DATA  User passes a struct amd_pmf_info with
+					     the size field set to sizeof(struct
+					     amd_pmf_info). The driver returns all
+					     available metrics and feature status
+					     in the structure.
+		============================ =======================================
+
+		struct amd_pmf_info layout:
+
+		======================= ========== ================================
+		Field			Type	   Description
+		size			__u64	   Structure size for versioning
+		features_supported	__u32	   Bitmask of supported features
+		platform_type		__u32	   Platform form factor orientation
+		power_source		__u32	   AC/DC power source
+		laptop_placement	__u32	   Device placement state
+		lid_state		__u32	   Lid open/closed status
+		user_presence		__u32	   User presence detection
+		slider_position		__u32	   Current power slider position
+		skin_temp		__s32	   Skin temperature (centidegrees)
+		gfx_busy		__u32	   Graphics workload percentage
+		ambient_light		__s32	   Ambient light sensor reading
+		avg_c0_residency	__u32	   Average C0 state residency
+		max_c0_residency	__u32	   Maximum C0 state residency
+		socket_power		__u32	   Socket power consumption
+		bios_input[10]		__u32	   Custom BIOS input parameters
+		bios_output[10]		__u32	   Custom BIOS output parameters
+		======================= ========== ================================
+
+		Feature Support Flags (features_supported bitmask):
+
+		=====================================    ====  =============================
+		Flag				         Bit   Description
+		AMD_PMF_FEAT_AUTO_MODE		         0     Auto Mode feature support
+		AMD_PMF_FEAT_STATIC_POWER_SLIDER         1     Static Power Slider support
+		AMD_PMF_FEAT_POLICY_BUILDER	         2     Policy Builder (Smart PC)
+		AMD_PMF_FEAT_DYNAMIC_POWER_SLIDER_AC     3     Dynamic slider on AC
+		AMD_PMF_FEAT_DYNAMIC_POWER_SLIDER_DC     4     Dynamic slider on DC
+		=====================================    ====  =============================
+
+		Return codes:
+
+		============= ============================================================
+		Return code   Description
+		0	      Success
+		EINVAL	      Invalid size or parameter
+		EFAULT	      copy_to_user/copy_from_user failures
+		ENODEV	      PMF device not available
+		ENOTTY	      Unknown ioctl command
+		============= ============================================================
+
+		User-space tools integrating with AMD PMF to discover capabilities and
+		monitor real-time metrics for thermal and power management validation.
diff --git a/Documentation/ABI/testing/sysfs-block-zram b/Documentation/ABI/testing/sysfs-block-zram
index 64c03010e9511..1ea9018a627f2 100644
--- a/Documentation/ABI/testing/sysfs-block-zram
+++ b/Documentation/ABI/testing/sysfs-block-zram
@@ -152,15 +152,15 @@ Description:
 		compression algorithm parameters.
 
 What:		/sys/block/zram<id>/compressed_writeback
-Date:		Decemeber 2025
+Date:		December 2025
 Contact:	Richard Chang <richardycc@google.com>
 Description:
-		The compressed_writeback device atrribute toggles compressed
+		The compressed_writeback device attribute toggles compressed
 		writeback feature.
 
 What:		/sys/block/zram<id>/writeback_batch_size
 Date:		November 2025
 Contact:	Sergey Senozhatsky <senozhatsky@chromium.org>
 Description:
-		The writeback_batch_size device atrribute sets the maximum
+		The writeback_batch_size device attribute sets the maximum
 		number of in-flight writeback operations.
diff --git a/Documentation/ABI/testing/sysfs-bus-dax b/Documentation/ABI/testing/sysfs-bus-dax
index b34266bfae49a..60703fffa99e0 100644
--- a/Documentation/ABI/testing/sysfs-bus-dax
+++ b/Documentation/ABI/testing/sysfs-bus-dax
@@ -151,3 +151,27 @@ Description:
 		memmap_on_memory parameter for memory_hotplug. This is
 		typically set on the kernel command line -
 		memory_hotplug.memmap_on_memory set to 'true' or 'force'."
+
+What:		/sys/bus/dax/devices/daxX.Y/state
+Contact:	nvdimm@lists.linux.dev
+Description:
+		(RW) Controls the state of the memory region.
+		Applies to all memory blocks associated with the device.
+		Only applies to dax_kmem devices.
+
+		Reading returns the current state; the writable states mirror
+		the per-block /sys/devices/system/memory/memoryX/state ABI::
+
+		  "unplugged": memory blocks are not present
+		  "online": memory is online, zone chosen by the kernel
+		  "online_kernel": memory is online in ZONE_NORMAL
+		  "online_movable": memory is online in ZONE_MOVABLE
+
+		"offline" (memory blocks are present but offline) may also be
+		reported - this happens when the device is bound while the
+		auto_online_blocks policy is "offline".  It cannot be written,
+		as it's not useful and creates device destruction races.
+
+		A device can only be onlined from the "unplugged" state, so a
+		device must be returned to "unplugged" before it can be onlined
+		into a different state.
diff --git a/Documentation/ABI/testing/sysfs-bus-iio b/Documentation/ABI/testing/sysfs-bus-iio
index d8d6d85235b01..5cc2e82c4997e 100644
--- a/Documentation/ABI/testing/sysfs-bus-iio
+++ b/Documentation/ABI/testing/sysfs-bus-iio
@@ -770,6 +770,7 @@ Description:
 		3.85kohm_to_gnd: connected to ground via a 3.85kOhm resistor,
 		6kohm_to_gnd: connected to ground via a 6kOhm resistor,
 		7.7kohm_to_gnd: connected to ground via a 7.7kOhm resistor,
+		10kohm_to_gnd: connected to ground via a 10kOhm resistor,
 		16kohm_to_gnd: connected to ground via a 16kOhm resistor,
 		20kohm_to_gnd: connected to ground via a 20kOhm resistor,
 		32kohm_to_gnd: connected to ground via a 32kOhm resistor,
@@ -917,7 +918,7 @@ Description:
 
 		Note the driver will assume the last p events requested are
 		to be enabled where p is how many it supports (which may vary
-		depending on the exact set requested. So if you want to be
+		depending on the exact set requested). So if you want to be
 		sure you have set what you think you have, check the contents of
 		these attributes after everything is configured. Drivers may
 		have to buffer any parameters so that they are consistent when
@@ -973,7 +974,7 @@ Description:
 
 		Note the driver will assume the last p events requested are
 		to be enabled where p is however many it supports (which may
-		vary depending on the exact set requested. So if you want to be
+		vary depending on the exact set requested). So if you want to be
 		sure you have set what you think you have, check the contents of
 		these attributes after everything is configured. Drivers may
 		have to buffer any parameters so that they are consistent when
@@ -1528,9 +1529,9 @@ Description:
 		the unused bits, so to get a clean value the bits value must be
 		used to mask the buffer output value appropriately. The storagebits
 		value also specifies the data alignment. So s48/64>>2 will be a
-		signed 48 bit integer stored in a 64 bit location aligned to a 64
-		bit boundary. To obtain the clean value, shift right 2 and apply a
-		mask to zero the top 16 bits of the result.
+		signed 48-bit integer stored in a 64-bit location aligned to a
+		64-bit boundary. To obtain the clean value, shift right 2 and apply
+		a mask to zero the top 16 bits of the result.
 		For other storage combinations this attribute will be extended
 		appropriately.
 
diff --git a/Documentation/ABI/testing/sysfs-bus-iio-adc-ad7192 b/Documentation/ABI/testing/sysfs-bus-iio-adc-ad7192
index 28be1cabf1124..d44eb5f8728b6 100644
--- a/Documentation/ABI/testing/sysfs-bus-iio-adc-ad7192
+++ b/Documentation/ABI/testing/sysfs-bus-iio-adc-ad7192
@@ -16,7 +16,7 @@ Description:
 		In bridge applications, such as strain gauges and load cells,
 		the bridge itself consumes the majority of the current in the
 		system. To minimize the current consumption of the system,
-		the bridge can be disconnected (when it is not being used
+		the bridge can be disconnected (when it is not being used)
 		using the bridge_switch_en attribute.
 
 What:		/sys/bus/iio/devices/iio:deviceX/in_voltage2-voltage2_shorted_raw
diff --git a/Documentation/ABI/testing/sysfs-bus-iio-frequency b/Documentation/ABI/testing/sysfs-bus-iio-frequency
new file mode 100644
index 0000000000000..46075aee7804f
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-iio-frequency
@@ -0,0 +1,11 @@
+What:		/sys/bus/iio/devices/iio:deviceX/out_altvoltageY_frequency_resolution
+KernelVersion:	3.4.0
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Stores channel Y frequency resolution/channel spacing in Hz for PLL
+		devices. The given value directly influences the operating mode when
+		fractional-N synthesis is required, as it derives values for
+		configurable modulus parameters used in the calculation of the output
+		frequency. It is assumed that the algorithm used to compute the various
+		dividers is able to generate proper values for multiples of the channel
+		spacing.
diff --git a/Documentation/ABI/testing/sysfs-bus-iio-frequency-adf4350 b/Documentation/ABI/testing/sysfs-bus-iio-frequency-adf4350
index 1254457a726e8..76987a119feb1 100644
--- a/Documentation/ABI/testing/sysfs-bus-iio-frequency-adf4350
+++ b/Documentation/ABI/testing/sysfs-bus-iio-frequency-adf4350
@@ -1,13 +1,3 @@
-What:		/sys/bus/iio/devices/iio:deviceX/out_altvoltageY_frequency_resolution
-KernelVersion:	3.4.0
-Contact:	linux-iio@vger.kernel.org
-Description:
-		Stores channel Y frequency resolution/channel spacing in Hz.
-		The value given directly influences the MODULUS used by
-		the fractional-N PLL. It is assumed that the algorithm
-		that is used to compute the various dividers, is able to
-		generate proper values for multiples of channel spacing.
-
 What:		/sys/bus/iio/devices/iio:deviceX/out_altvoltageY_refin_frequency
 KernelVersion:	3.4.0
 Contact:	linux-iio@vger.kernel.org
diff --git a/Documentation/ABI/testing/sysfs-bus-pci b/Documentation/ABI/testing/sysfs-bus-pci
index b767db2c52cb7..55ea1db749a18 100644
--- a/Documentation/ABI/testing/sysfs-bus-pci
+++ b/Documentation/ABI/testing/sysfs-bus-pci
@@ -171,7 +171,7 @@ Description:
 		all devices attached through the subordinate bus of a specific
 		bridge device, writing 1 to this will try to do it.  This will
 		affect all devices attached to the system through this bridge
-		similiar to writing 1 to their individual "reset" file, so use
+		similar to writing 1 to their individual "reset" file, so use
 		with caution.
 
 What:		/sys/bus/pci/devices/.../vpd
diff --git a/Documentation/ABI/testing/sysfs-bus-usb b/Documentation/ABI/testing/sysfs-bus-usb
index af9b653422f16..baf4bf6d32ee3 100644
--- a/Documentation/ABI/testing/sysfs-bus-usb
+++ b/Documentation/ABI/testing/sysfs-bus-usb
@@ -293,7 +293,7 @@ Description:
 
 		Supported values are 0 - 15.
 		More information on how besl values map to microseconds can be found in
-		USB 2.0 ECN Errata for Link Power Management, section 4.10)
+		USB 2.0 ECN Errata for Link Power Management, section 4.10
 
 What:		/sys/bus/usb/devices/.../rx_lanes
 Date:		March 2018
diff --git a/Documentation/ABI/testing/sysfs-driver-intel-xe-hwmon b/Documentation/ABI/testing/sysfs-driver-intel-xe-hwmon
index 55ab45f669acf..0da739d9a8160 100644
--- a/Documentation/ABI/testing/sysfs-driver-intel-xe-hwmon
+++ b/Documentation/ABI/testing/sysfs-driver-intel-xe-hwmon
@@ -251,6 +251,13 @@ Description:	RO. Fan 2 speed in RPM.
 
 		Only supported for particular Intel Xe graphics platforms.
 
+		On DG2 the driver always shows two fan channels, because the
+		FSC_READ_NUM_FANS command does not work on some cards. OEMs
+		decide how the fans map to tach channels, so two fans can share
+		one tach line. When that happens, the second channel
+		reads 0 RPM even though the fan is spinning. This is normal, not
+		a bug.
+
 What:		/sys/bus/pci/drivers/xe/.../hwmon/hwmon<i>/fan3_input
 Date:		March 2025
 KernelVersion:	6.16
diff --git a/Documentation/ABI/testing/sysfs-driver-intel-xe-ras b/Documentation/ABI/testing/sysfs-driver-intel-xe-ras
new file mode 100644
index 0000000000000..3870e5a03a92a
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-driver-intel-xe-ras
@@ -0,0 +1,30 @@
+What:		/sys/bus/pci/drivers/xe/.../gpu_health
+Date:		July 2026
+KernelVersion:	7.3
+Contact:	intel-xe@lists.freedesktop.org
+Description:
+		This file exposes the current gpu health state and allows the gpu
+		health state to be updated.
+
+		This sysfs file is present only on Intel Xe platforms that support
+		the gpu health indicator interface for RAS. Reading the current
+		health state is available to all users, while updating the health
+		state is restricted to administrative users only.
+
+		Read returns a single line containing one of the valid values for
+		the current gpu health state. Writing one of the valid values
+		updates the current gpu health state.
+
+		The valid values for the gpu health state are:
+
+			ok
+				The gpu is healthy and operating within normal
+				parameters.
+
+			warning
+				The gpu is experiencing minor issues but remains
+				operational.
+
+			critical
+				The gpu is in a critical state and may not be
+				operational.
diff --git a/Documentation/ABI/testing/sysfs-driver-uniwill-laptop b/Documentation/ABI/testing/sysfs-driver-uniwill-laptop
index 2397c65c969a6..943f92c6b5618 100644
--- a/Documentation/ABI/testing/sysfs-driver-uniwill-laptop
+++ b/Documentation/ABI/testing/sysfs-driver-uniwill-laptop
@@ -78,3 +78,28 @@ Description:
 
 		Reading this file returns the profile names with the currently active one in
 		brackets.
+
+What:		/sys/bus/platform/devices/INOU0000:XX/ac_auto_boot
+Date:		March 2026
+KernelVersion:	7.1
+Contact:	Armin Wolf <W_Armin@gmx.de>
+Description:
+		Allows userspace applications to configure if the device should boot automatically
+		when being connected to a power source. Writing "1"/"0" into this file
+		enables/disables this functionality. Enabling both AC auto boot and USB powershare
+		at the same time is not supported.
+
+		Reading this file returns the current status of the AC auto boot functionality.
+
+What:		/sys/bus/platform/devices/INOU0000:XX/usb_powershare_high
+Date:		March 2026
+KernelVersion:	7.1
+Contact:	Armin Wolf <W_Armin@gmx.de>
+Description:
+		Allows userspace applications to configure if the device should continue to provide
+		power via the USB ports when hibernating or powered off. Might also increase the
+		power budget available to USB ports on some devices. Writing "1"/"0" into this
+		file enables/disables this functionality. Enabling both USB powershare and AC auto
+		boot at the same time is not supported.
+
+		Reading this file returns the current status of the USB powershare functionality.
diff --git a/Documentation/ABI/testing/sysfs-firmware-gsmi b/Documentation/ABI/testing/sysfs-firmware-gsmi
index 7a558354c1ee0..3b3c3ae6f458d 100644
--- a/Documentation/ABI/testing/sysfs-firmware-gsmi
+++ b/Documentation/ABI/testing/sysfs-firmware-gsmi
@@ -19,7 +19,7 @@ Description:
 		/sys/firmware/gsmi/vars:
 
 			This directory has the same layout (and
-			underlying implementation as /sys/firmware/efi/vars.
+			underlying implementation) as /sys/firmware/efi/vars.
 			See `Documentation/ABI/*/sysfs-firmware-efi-vars`
 			for more information on how to interact with
 			this structure.
diff --git a/Documentation/ABI/testing/sysfs-firmware-sev-vulnerabilities b/Documentation/ABI/testing/sysfs-firmware-sev-vulnerabilities
new file mode 100644
index 0000000000000..964362558bb2c
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-firmware-sev-vulnerabilities
@@ -0,0 +1,19 @@
+What:		/sys/firmware/sev/vulnerabilities/supported_mitigations
+Date:		June 2026
+Contact:	linux-crypto@vger.kernel.org
+Description:
+		Read-only interface that reports the vector of SEV-SNP
+		firmware vulnerability mitigations supported by the firmware.
+
+What:		/sys/firmware/sev/vulnerabilities/verified_mitigations
+Date:		June 2026
+Contact:	linux-crypto@vger.kernel.org
+Description:
+		Read/write interface that reports the vector of SEV-SNP
+		firmware vulnerability mitigations already verified by the
+		firmware. Writing a vector value requests the firmware to
+		VERIFY the corresponding mitigation bit(s).
+
+		The list of supported mitigations and the meaning of each
+		vector bit are both platform- and bug-specific and are
+		published as part of the AMD Security Bulletin.
diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon
index b73e6bc28ea5f..e675a57145e36 100644
--- a/Documentation/ABI/testing/sysfs-kernel-mm-damon
+++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon
@@ -1,26 +1,26 @@
 what:		/sys/kernel/mm/damon/
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Interface for Data Access MONitoring (DAMON).  Contains files
 		for controlling DAMON.  For more details on DAMON itself,
 		please refer to Documentation/admin-guide/mm/damon/index.rst.
 
 What:		/sys/kernel/mm/damon/admin/
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Interface for privileged users of DAMON.  Contains files for
 		controlling DAMON that aimed to be used by privileged users.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/nr_kdamonds
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a number 'N' to this file creates the number of
 		directories for controlling each DAMON worker thread (kdamond)
 		named '0' to 'N-1' under the kdamonds/ directory.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/state
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing 'on' or 'off' to this file makes the kdamond starts or
 		stops, respectively.  Reading the file returns the keywords
 		based on the current status.  Writing 'commit' to this file
@@ -36,37 +36,40 @@ Description:	Writing 'on' or 'off' to this file makes the kdamond starts or
 		kdamond.  Writing 'clear_schemes_tried_regions' to the file
 		removes contents of the 'tried_regions' directory.  Writing
 		'update_schemes_effective_quotas' to the file updates
-		'.../quotas/effective_bytes' files of this kdamond.
+		'.../quotas/effective_bytes' files of this kdamond.  Writing
+		'update_tuned_intervals' to the file updates 'sample_us' and
+		'aggr_us' files under 'monitoring_attrs/intervals' with the
+		auto-tuned sampling and aggregation intervals.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/pid
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Reading this file returns the pid of the kdamond if it is
 		running.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/refresh_ms
 Date:		Jul 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a value to this file sets the time interval for
 		automatic DAMON status file contents update.  Writing '0'
 		disables the update.  Reading this file returns the value.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/nr_contexts
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a number 'N' to this file creates the number of
 		directories for controlling each DAMON context named '0' to
 		'N-1' under the contexts/ directory.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/avail_operations
 Date:		Apr 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Reading this file returns the available monitoring operations
 		sets on the currently running kernel.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/operations
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a keyword for a monitoring operations set ('vaddr' for
 		virtual address spaces monitoring, 'fvaddr' for fixed virtual
 		address ranges monitoring, and 'paddr' for the physical address
@@ -79,108 +82,154 @@ Description:	Writing a keyword for a monitoring operations set ('vaddr' for
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/addr_unit
 Date:		Aug 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing an integer to this file sets the 'address unit'
 		parameter of the given operations set of the context.  Reading
 		the file returns the last-written 'address unit' value.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/pause
 Date:		Mar 2026
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a boolean keyword to this file sets the 'pause' request
 		parameter for the context.  Reading the file returns the
 		last-written 'pause' value.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/sample_us
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a value to this file sets the sampling interval of the
 		DAMON context in microseconds as the value.  Reading this file
 		returns the value.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/aggr_us
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a value to this file sets the aggregation interval of
 		the DAMON context in microseconds as the value.  Reading this
 		file returns the value.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/update_us
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a value to this file sets the update interval of the
 		DAMON context in microseconds as the value.  Reading this file
 		returns the value.
 
-What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/intrvals_goal/access_bp
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/intervals_goal/access_bp
 Date:		Feb 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a value to this file sets the monitoring intervals
 		auto-tuning target DAMON-observed access events ratio within
 		the given time interval (aggrs in same directory), in bp
 		(1/10,000).  Reading this file returns the value.
 
-What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/intrvals_goal/aggrs
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/intervals_goal/aggrs
 Date:		Feb 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a value to this file sets the time interval to achieve
 		the monitoring intervals auto-tuning target DAMON-observed
 		access events ratio (access_bp in same directory) within.
 		Reading this file returns the value.
 
-What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/intrvals_goal/min_sample_us
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/intervals_goal/min_sample_us
 Date:		Feb 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a value to this file sets the minimum value of
 		auto-tuned sampling interval in microseconds.  Reading this
 		file returns the value.
 
-What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/intrvals_goal/max_sample_us
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/intervals_goal/max_sample_us
 Date:		Feb 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a value to this file sets the maximum value of
 		auto-tuned sampling interval in microseconds.  Reading this
 		file returns the value.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/nr_regions/min
 
-WDate:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Date:		Mar 2022
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a value to this file sets the minimum number of
 		monitoring regions of the DAMON context as the value.  Reading
 		this file returns the value.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/nr_regions/max
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a value to this file sets the maximum number of
 		monitoring regions of the DAMON context as the value.  Reading
 		this file returns the value.
 
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/nr_probes
+Date:		May 2026
+Contact:	SJ Park <sj@kernel.org>
+Description:	Writing a number 'N' to this file creates the number of
+		directories for each DAMON probe named '0' to 'N-1' under the
+		probes/ directory.
+
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/weight
+Date:		Jul 2026
+Contact:	SJ Park <sj@kernel.org>
+Description:	Writing to and reading from this file sets and gets the
+		per-probe attribute weight.
+
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/filters/nr_filters
+Date:		May 2026
+Contact:	SJ Park <sj@kernel.org>
+Description:	Writing a number 'N' to this file creates the number of
+		directories for each DAMON probe filter named '0' to 'N-1'
+		under the filters/ directory.
+
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/filters/<F>/type
+Date:		May 2026
+Contact:	SJ Park <sj@kernel.org>
+Description:	Writing to and reading from this file sets and gets the type of
+		the memory of the interest.
+
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/filters/<F>/path
+Date:		May 2026
+Contact:	SJ Park <sj@kernel.org>
+Description:	If 'memcg' is written to the 'type' file, writing to and
+		reading from this file sets and gets the path to the memory
+		cgroup of the interest.
+
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/filters/<F>/matching
+Date:		May 2026
+Contact:	SJ Park <sj@kernel.org>
+Description:	Writing 'Y' or 'N' to this file sets whether the filter is for
+		the memory of the 'type', or all except the 'type'.
+
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/filters/<F>/allow
+Date:		May 2026
+Contact:	SJ Park <sj@kernel.org>
+Description:	Writing 'Y' or 'N' to this file sets whether to allow or reject
+		hitting the probe for the memory that satisfies the 'type' and
+		the 'matching' of the directory.
+
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/targets/nr_targets
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a number 'N' to this file creates the number of
 		directories for controlling each DAMON target of the context
 		named '0' to 'N-1' under the contexts/ directory.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/targets/<T>/pid_target
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the pid of
 		the target process if the context is for virtual address spaces
 		monitoring, respectively.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/targets/<T>/obsolete_target
 Date:		Oct 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the
 		obsoleteness of the matching parameters commit destination
 		target.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/targets/<T>/regions/nr_regions
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a number 'N' to this file creates the number of
 		directories for setting each DAMON target memory region of the
 		context named '0' to 'N-1' under the regions/ directory.  In
@@ -190,181 +239,181 @@ Description:	Writing a number 'N' to this file creates the number of
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/targets/<T>/regions/<R>/start
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the start
 		address of the monitoring region.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/targets/<T>/regions/<R>/end
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the end
 		address of the monitoring region.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/nr_schemes
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a number 'N' to this file creates the number of
 		directories for controlling each DAMON-based operation scheme
 		of the context named '0' to 'N-1' under the schemes/ directory.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/action
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the action
 		of the scheme.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/target_nid
 Date:		Jun 2024
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Action's target NUMA node id.  Supported by only relevant
 		actions.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/apply_interval_us
 Date:		Sep 2023
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a value to this file sets the action apply interval of
 		the scheme in microseconds.  Reading this file returns the
 		value.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/access_pattern/sz/min
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the minimum
 		size of the scheme's target regions in bytes.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/access_pattern/sz/max
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the maximum
 		size of the scheme's target regions in bytes.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/access_pattern/nr_accesses/min
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
-Description:	Writing to and reading from this file sets and gets the manimum
+Contact:	SJ Park <sj@kernel.org>
+Description:	Writing to and reading from this file sets and gets the minimum
 		'nr_accesses' of the scheme's target regions.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/access_pattern/nr_accesses/max
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the maximum
 		'nr_accesses' of the scheme's target regions.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/access_pattern/age/min
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the minimum
 		'age' of the scheme's target regions.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/access_pattern/age/max
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the maximum
 		'age' of the scheme's target regions.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/ms
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the time
 		quota of the scheme in milliseconds.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/bytes
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the size
 		quota of the scheme in bytes.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/effective_bytes
 Date:		Feb 2024
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Reading from this file gets the effective size quota of the
 		scheme in bytes, which adjusted for the time quota and goals.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/reset_interval_ms
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the quotas
 		charge reset interval of the scheme in milliseconds.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/goals/nr_goals
 Date:		Nov 2023
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a number 'N' to this file creates the number of
 		directories for setting automatic tuning of the scheme's
 		aggressiveness named '0' to 'N-1' under the goals/ directory.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/goals/<G>/target_metric
 Date:		Feb 2024
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the quota
 		auto-tuning goal metric.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/goals/<G>/target_value
 Date:		Nov 2023
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the target
 		value of the goal metric.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/goals/<G>/current_value
 Date:		Nov 2023
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the current
 		value of the goal metric.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/goals/<G>/nid
 Date:		Apr 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the nid
 		parameter of the goal.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/goals/<G>/path
 Date:		Oct 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the path
 		parameter of the goal.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/goal_tuner
 Date:		Mar 2026
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the
 		goal-based effective quota auto-tuning algorithm to use.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/fail_charge_num
 Date:		Mar 2026
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the
 		action-failed memory quota charging ratio numerator.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/fail_charge_denom
 Date:		Mar 2026
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the
 		action-failed memory quota charging ratio denominator.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/weights/sz_permil
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the
 		under-quota limit regions prioritization weight for 'size' in
 		permil.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/weights/nr_accesses_permil
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the
 		under-quota limit regions prioritization weight for
 		'nr_accesses' in permil.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/weights/age_permil
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the
 		under-quota limit regions prioritization weight for 'age' in
 		permil.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/watermarks/metric
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the metric
 		of the watermarks for the scheme.  The writable/readable
 		keywords for this file are 'none' for disabling the watermarks
@@ -373,44 +422,44 @@ Description:	Writing to and reading from this file sets and gets the metric
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/watermarks/interval_us
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the metric
 		check interval of the watermarks for the scheme in
 		microseconds.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/watermarks/high
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the high
 		watermark of the scheme in permil.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/watermarks/mid
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the mid
 		watermark of the scheme in permil.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/watermarks/low
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the low
 		watermark of the scheme in permil.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters
 Date:		Feb 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Directory for DAMON core layer-handled DAMOS filters.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/nr_filters
 Date:		Feb 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a number 'N' to this file creates the number of
 		directories for setting filters of the scheme named '0' to
 		'N-1' under the core_filters/ directory.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/type
 Date:		Feb 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the type of
 		the memory of the interest.  'anon' for anonymous pages,
 		'memcg' for specific memory cgroup, 'young' for young pages,
@@ -419,62 +468,62 @@ Description:	Writing to and reading from this file sets and gets the type of
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/memcg_path
 Date:		Feb 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	If 'memcg' is written to the 'type' file, writing to and
 		reading from this file sets and gets the path to the memory
 		cgroup of the interest.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/addr_start
 Date:		Feb 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	If 'addr' is written to the 'type' file, writing to or reading
 		from this file sets or gets the start address of the address
 		range for the filter.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/addr_end
 Date:		Feb 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	If 'addr' is written to the 'type' file, writing to or reading
 		from this file sets or gets the end address of the address
 		range for the filter.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/min
 Date:		Feb 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	If 'hugepage_size' is written to the 'type' file, writing to
 		or reading from this file sets or gets the minimum size of the
 		hugepage for the filter.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/max
 Date:		Feb 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	If 'hugepage_size' is written to the 'type' file, writing to
 		or reading from this file sets or gets the maximum size of the
 		hugepage for the filter.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/damon_target_idx
 Date:		Feb 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	If 'target' is written to the 'type' file, writing to or
 		reading from this file sets or gets the index of the DAMON
 		monitoring target of the interest.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/matching
 Date:		Feb 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing 'Y' or 'N' to this file sets whether the filter is for
 		the memory of the 'type', or all except the 'type'.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/allow
 Date:		Feb 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing 'Y' or 'N' to this file sets whether to allow or reject
 		applying the scheme's action to the memory that satisfies the
 		'type' and the 'matching' of the directory.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/ops_filters
 Date:		Feb 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Directory for DAMON operations set layer-handled DAMOS filters.
 		Files under this directory works same to those of
 		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters
@@ -482,7 +531,7 @@ Description:	Directory for DAMON operations set layer-handled DAMOS filters.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/filters
 Date:		Dec 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Directory for DAMOS filters.  Files under this directory works
 		same to those of
 		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/{core,ops}_filters
@@ -491,14 +540,14 @@ Description:	Directory for DAMOS filters.  Files under this directory works
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/dests/nr_dests
 Date:		Jul 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a number 'N' to this file creates the number of
 		directories for setting action destinations of the scheme named
 		'0' to 'N-1' under the dests/ directory.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/dests/<D>/id
 Date:		Jul 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the id of
 		the DAMOS action destination.  For DAMOS_MIGRATE_{HOT,COLD}
 		actions, the destination node's node id can be written and
@@ -506,98 +555,110 @@ Description:	Writing to and reading from this file sets and gets the id of
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/dests/<D>/weight
 Date:		Jul 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the weight
 		of the DAMOS action destination to select as the destination of
 		each action among the destinations.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/stats/nr_tried
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Reading this file returns the number of regions that the action
 		of the scheme has tried to be applied.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/stats/sz_tried
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Reading this file returns the total size of regions that the
 		action of the scheme has tried to be applied in bytes.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/stats/nr_applied
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Reading this file returns the number of regions that the action
 		of the scheme has successfully applied.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/stats/sz_applied
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Reading this file returns the total size of regions that the
 		action of the scheme has successfully applied in bytes.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/stats/sz_ops_filter_passed
 Date:		Dec 2024
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Reading this file returns the total size of memory that passed
 		DAMON operations layer-handled filters of the scheme in bytes.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/stats/qt_exceeds
 Date:		Mar 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Reading this file returns the number of the exceed events of
 		the scheme's quotas.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/stats/nr_snapshots
 Date:		Dec 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Reading this file returns the total number of DAMON snapshots
 		that the scheme has tried to be applied.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/stats/max_nr_snapshots
 Date:		Dec 2025
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Writing a number to this file sets the upper limit of
 		nr_snapshots that deactivates the scheme when the limit is
 		reached or exceeded.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/tried_regions/total_bytes
 Date:		Jul 2023
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Reading this file returns the total amount of memory that
 		corresponding DAMON-based Operation Scheme's action has tried
 		to be applied.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/tried_regions/<R>/start
 Date:		Oct 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Reading this file returns the start address of a memory region
 		that corresponding DAMON-based Operation Scheme's action has
 		tried to be applied.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/tried_regions/<R>/end
 Date:		Oct 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Reading this file returns the end address of a memory region
 		that corresponding DAMON-based Operation Scheme's action has
 		tried to be applied.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/tried_regions/<R>/nr_accesses
 Date:		Oct 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Reading this file returns the 'nr_accesses' of a memory region
 		that corresponding DAMON-based Operation Scheme's action has
 		tried to be applied.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/tried_regions/<R>/age
 Date:		Oct 2022
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Reading this file returns the 'age' of a memory region that
 		corresponding DAMON-based Operation Scheme's action has tried
 		to be applied.
 
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/tried_regions/<R>/sz_filter_passed
 Date:		Dec 2024
-Contact:	SeongJae Park <sj@kernel.org>
+Contact:	SJ Park <sj@kernel.org>
 Description:	Reading this file returns the size of the memory in the region
 		that passed DAMON operations layer-handled filters of the
 		scheme in bytes.
+
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/tried_regions/<R>/probes/
+Date:		May 2026
+Contact:	SJ Park <sj@kernel.org>
+Description:	Directory for DAMON data-attribute monitoring probes of the
+		region.
+
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/tried_regions/<R>/probes/<P>/hits
+Date:		May 2026
+Contact:	SJ Park <sj@kernel.org>
+Description:	Reading this file returns the number of data-attribute
+		monitoring probe-hit positive samples of the region.
diff --git a/Documentation/ABI/testing/sysfs-kernel-slab b/Documentation/ABI/testing/sysfs-kernel-slab
index b26e4299f8226..c52034e3e7948 100644
--- a/Documentation/ABI/testing/sysfs-kernel-slab
+++ b/Documentation/ABI/testing/sysfs-kernel-slab
@@ -113,8 +113,10 @@ KernelVersion:	2.6.22
 Contact:	Pekka Enberg <penberg@cs.helsinki.fi>,
 		Christoph Lameter <cl@gentwo.org>
 Description:
-		The cpu_slabs file is read-only and displays how many cpu slabs
-		are active and their NUMA locality.
+		The cpu_slabs file is read-only. It is deprecated and always
+		reads "0" since the removal of per-cpu slabs in Linux 7.0. It
+		previously displayed how many cpu slabs were active and their
+		NUMA locality. The file is kept for backwards compatibility.
 
 What:		/sys/kernel/slab/<cache>/cpuslab_flush
 Date:		April 2009
@@ -509,12 +511,16 @@ What:		/sys/kernel/slab/<cache>/slabs_cpu_partial
 Date:		Aug 2011
 Contact:	Christoph Lameter <cl@gentwo.org>
 Description:
-		This read-only file shows the number of partialli allocated
-		frozen slabs.
+		This read-only file is deprecated and always reads "0(0)" since
+		the removal of per-cpu partial slabs in Linux 7.0. It previously
+		showed the number of partially allocated frozen slabs. The file
+		is kept for backwards compatibility.
 
 What:		/sys/kernel/slab/<cache>/cpu_partial
 Date:		Aug 2011
 Contact:	Christoph Lameter <cl@gentwo.org>
 Description:
-		This read-only file shows the number of per cpu partial
-		pages to keep around.
+		This file is deprecated and always reads "0" since the removal of
+		per-cpu partial slabs in Linux 7.0. It previously showed the
+		number of per-cpu partial pages to keep around. The file is kept
+		for backwards compatibility.
diff --git a/Documentation/accounting/delay-accounting.rst b/Documentation/accounting/delay-accounting.rst
index e209c46241b09..0f1d8023338bd 100644
--- a/Documentation/accounting/delay-accounting.rst
+++ b/Documentation/accounting/delay-accounting.rst
@@ -181,7 +181,7 @@ Interactive keyboard controls during runtime::
 	M - Toggle display mode (Default/Memory Verbose)
 	q - Quit
 
-Available sort fields(use -s/--sort or interactive command)::
+Available sort fields (use -s/--sort or interactive command)::
 
 	cpu(c)       - CPU delay
 	blkio(i)     - I/O delay
@@ -212,3 +212,46 @@ Advanced usage examples::
 
 	# ./delaytop -d secs
 	Specify refresh interval as secs
+
+	# ./delaytop -t type
+	Display only specified delay type with avg/max/timestamp
+    (rows sorted by MAX for that type, largest first)
+
+
+
+delaytop add delay_max fields to track the maximum delay value for each delay
+  type (cpu, blkio, irq, swapin, freepages, thrashing, compact, wpcopy) per task::
+
+	bash# ./delaytop -t cpu
+	System Pressure Information: (avg10/avg60/avg300/total)
+	CPU some:       0.4%/   0.2%/   0.1%/     220(ms)
+	CPU full:       0.0%/   0.0%/   0.0%/       0(ms)
+	Memory full:    0.0%/   0.0%/   0.0%/       0(ms)
+	Memory some:    0.0%/   0.0%/   0.0%/       0(ms)
+	IO full:        0.0%/   0.0%/   0.0%/      12(ms)
+	IO some:        0.0%/   0.0%/   0.0%/      13(ms)
+	IRQ full:       0.0%/   0.0%/   0.0%/       0(ms)
+	[q]quit
+	Top 20 processes (sorted by cpu MAX delay, largest first):
+	     PID      TGID  COMMAND          AVG(ms)  MAX(ms)   MAX_TIMESTAMP
+	------------------------------------------------------------------------
+	       9         9  kworker/0:0-eve   0.59   16.87  2026-05-27T13:32:39
+	      30        30  kworker/2:0H-kb   2.87   11.36  2026-05-27T13:32:36
+	      27        27  migration/2       1.05    9.51  2026-05-27T13:32:37
+	      50        50  kworker/2:1-eve   0.50    9.13  2026-05-27T13:32:37
+	      15        15  rcu_preempt       0.11    8.98  2026-05-27T13:32:37
+	       1         1  init              0.17    7.12  2026-05-27T13:32:38
+	      67        67  scsi_eh_0         1.20    4.23  2026-05-27T13:32:37
+	      23        23  ksoftirqd/1       1.12    3.77  2026-05-27T13:32:36
+	       3         3  pool_workqueue_   0.72    3.55  2026-05-27T13:32:38
+	      62        62  kworker/u20:2-a   0.49    3.03  2026-05-27T13:32:37
+	       2         2  kthreadd          0.18    2.82  2026-05-27T13:32:37
+	      11        11  kworker/0:1       1.42    2.76  2026-05-27T13:32:36
+	      39        39  kworker/u20:0-a   0.10    2.71  2026-05-27T13:32:38
+	      17        17  rcu_exp_gp_kthr   0.25    2.65  2026-05-27T13:32:37
+	      66        66  kworker/u20:3-e   0.38    2.55  2026-05-27T13:32:37
+	      20        20  cpuhp/0           0.53    2.51  2026-05-27T13:32:37
+	      28        28  ksoftirqd/2       0.59    2.48  2026-05-27T13:32:37
+	      55        55  kworker/u19:1     0.88    2.42  2026-05-27T13:32:37
+	      13        13  kworker/R-mm_pe   1.18    2.35  2026-05-27T13:32:36
+	      54        54  kworker/3:1-eve   0.14    2.20  2026-05-27T13:32:38
diff --git a/Documentation/admin-guide/LSM/landlock.rst b/Documentation/admin-guide/LSM/landlock.rst
index 314052bbeb0a9..8eb85c9381ffa 100644
--- a/Documentation/admin-guide/LSM/landlock.rst
+++ b/Documentation/admin-guide/LSM/landlock.rst
@@ -52,6 +52,7 @@ AUDIT_LANDLOCK_ACCESS
         - fs.refer (ABI 2+)
         - fs.truncate (ABI 3+)
         - fs.ioctl_dev (ABI 5+)
+        - fs.resolve_unix (ABI 9+)
 
     **net.*** - Network access rights (ABI 4+):
         - net.bind_tcp - TCP port binding was denied
diff --git a/Documentation/admin-guide/bootconfig.rst b/Documentation/admin-guide/bootconfig.rst
index f712758472d5c..3d6412458c8b6 100644
--- a/Documentation/admin-guide/bootconfig.rst
+++ b/Documentation/admin-guide/bootconfig.rst
@@ -234,6 +234,87 @@ Kconfig option selected.
 Note that even if you set this option, you can override the embedded
 bootconfig by another bootconfig which attached to the initrd.
 
+Rendering Embedded kernel.* Keys at Build Time
+----------------------------------------------
+
+By default, the embedded bootconfig (``CONFIG_BOOT_CONFIG_EMBED=y``) is
+parsed at runtime, after ``parse_early_param()`` has already run. Early
+parameter handlers (``mem=``, ``earlycon=``, ``loglevel=``, ...) therefore
+cannot see values supplied via the embedded ``kernel`` subtree.
+
+``CONFIG_CMDLINE_FROM_BOOTCONFIG`` resolves this by rendering the
+``kernel`` subtree of ``CONFIG_BOOT_CONFIG_EMBED_FILE`` into a flat cmdline
+string at kernel build time (via ``tools/bootconfig -C``) and prepending
+it to ``boot_command_line`` during early architecture setup, so the keys
+are visible to ``parse_early_param()``.
+
+The option requires ``CONFIG_BOOT_CONFIG_EMBED=y``, a non-empty
+``CONFIG_BOOT_CONFIG_EMBED_FILE``, ``CONFIG_CMDLINE`` to be empty, and
+an architecture that selects ``CONFIG_ARCH_SUPPORTS_CMDLINE_FROM_BOOTCONFIG``.
+Currently only x86 selects it; on other architectures the embedded
+bootconfig still works, but only through the late runtime parser.
+
+The same ``bootconfig`` opt-in applies as elsewhere: the rendered keys
+are prepended only when ``bootconfig`` (in any form) appears on the
+kernel command line, or when ``CONFIG_BOOT_CONFIG_FORCE`` is set, which
+defaults to ``y`` when ``CONFIG_BOOT_CONFIG_EMBED`` is set.
+
+For example, given::
+
+ kernel {
+   loglevel = 7
+   mem = 4G
+ }
+
+the kernel boots as if ``loglevel=7 mem=4G`` had been prepended to the
+bootloader command line, with the values visible to early-parsed
+handlers. Comma-separated values are still expanded into multiple
+cmdline entries per the bootconfig array convention -- the embedded
+``kernel.earlycon = "uart8250,io,0x3f8"`` must be quoted to land as a
+single ``earlycon=`` entry, exactly as for the runtime parser.
+
+If the rendered string would not fit in ``COMMAND_LINE_SIZE`` together
+with the existing command line, the prepend is skipped and an error is
+logged, so an oversized embedded bootconfig cannot brick a boot.
+
+Interaction with other command line and bootconfig sources
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+With ``CONFIG_CMDLINE_FROM_BOOTCONFIG=y`` the rendered ``kernel``
+subtree behaves like a build-time command line (similar to
+``CONFIG_CMDLINE``), not like a bootconfig source. It is prepended to
+``boot_command_line`` in ``setup_arch()``, before ``parse_early_param()``
+and long before the runtime parser looks at an initrd. Options can reach
+the kernel from up to four places:
+
+- Bootloader command line: the arguments the boot loader passes. The
+  embedded cmdline is prepended in front of them, so for last-one-wins
+  parameters a bootloader option still overrides the embedded value.
+  Visible in /proc/cmdline.
+- Embedded cmdline (this option): the rendered ``kernel`` subtree,
+  prepended early so it is seen by ``parse_early_param()``. Visible in
+  /proc/cmdline.
+- Initrd bootconfig: parsed late in ``setup_boot_config()``; its
+  ``kernel`` keys are placed ahead of ``boot_command_line``, i.e. before
+  the embedded cmdline, so last-wins favors the embedded values. As a
+  bootconfig source, an initrd bootconfig still replaces the embedded
+  bootconfig. Visible in /proc/cmdline and /proc/bootconfig.
+- Embedded bootconfig (runtime): parsed late, only when no initrd
+  bootconfig is present. Visible in /proc/cmdline and /proc/bootconfig.
+
+So with this option the embedded ``kernel.*`` values take precedence
+over an initrd bootconfig's ``kernel.*`` values: for early parameters
+the initrd is not parsed yet, and for ordinary parameters the embedded
+keys land later in the command line. If you need an initrd bootconfig to
+override the embedded ``kernel.*`` keys, leave this option off and rely
+on the runtime parser.
+
+The rendered string is part of the command line, so it appears in
+/proc/cmdline. It is deliberately not shown in /proc/bootconfig: that
+file keeps reporting the parsed bootconfig tree -- the initrd bootconfig
+if present, otherwise the embedded bootconfig -- independent of whether
+build-time cmdline rendering is enabled.
+
 Kernel parameters via Boot Config
 =================================
 
diff --git a/Documentation/admin-guide/cgroup-v1/cpusets.rst b/Documentation/admin-guide/cgroup-v1/cpusets.rst
index c7909e5ac1361..52a213aff04e5 100644
--- a/Documentation/admin-guide/cgroup-v1/cpusets.rst
+++ b/Documentation/admin-guide/cgroup-v1/cpusets.rst
@@ -284,7 +284,7 @@ take action.
 ==>
     Unless this feature is enabled by writing "1" to the special file
     /dev/cpuset/memory_pressure_enabled, the hook in the rebalance
-    code of __alloc_pages() for this metric reduces to simply noticing
+    code of the page allocator for this metric reduces to simply noticing
     that the cpuset_memory_pressure_enabled flag is zero.  So only
     systems that enable this feature will compute the metric.
 
diff --git a/Documentation/admin-guide/cgroup-v1/memcg_test.rst b/Documentation/admin-guide/cgroup-v1/memcg_test.rst
index ebedbc3c3f9cc..d9951c319ef5b 100644
--- a/Documentation/admin-guide/cgroup-v1/memcg_test.rst
+++ b/Documentation/admin-guide/cgroup-v1/memcg_test.rst
@@ -10,7 +10,7 @@ Because VM is getting complex (one of reasons is memcg...), memcg's behavior
 is complex. This is a document for memcg's internal behavior.
 Please note that implementation details can be changed.
 
-(*) Topics on API should be in Documentation/admin-guide/cgroup-v1/memory.rst)
+(*) Topics on API should be in Documentation/admin-guide/cgroup-v1/memory.rst
 
 0. How to record usage ?
 ========================
diff --git a/Documentation/admin-guide/cgroup-v1/rdma.rst b/Documentation/admin-guide/cgroup-v1/rdma.rst
index e69369b7252ea..abddf34d26675 100644
--- a/Documentation/admin-guide/cgroup-v1/rdma.rst
+++ b/Documentation/admin-guide/cgroup-v1/rdma.rst
@@ -9,6 +9,7 @@ RDMA Controller
      1-2. Why RDMA controller needed?
      1-3. How is RDMA controller implemented?
    2. Usage Examples
+   3. RDMA Interface Files
 
 1. Overview
 ===========
@@ -115,3 +116,68 @@ Following resources can be accounted by rdma controller.
 (d) Delete resource limit::
 
 	echo mlx4_0 hca_handle=max hca_object=max > /sys/fs/cgroup/rdma/1/rdma.max
+
+3. RDMA Interface Files
+========================
+
+The following interface files are available in each non-root RDMA cgroup.
+
+  rdma.max
+	A read-write file which describes the configured resource limit
+	for an RDMA/IB device.  See the Usage Examples above.
+
+  rdma.current
+	A read-only file which describes the current resource usage.
+
+  rdma.peak
+	A read-only nested-keyed file which shows the historical high
+	watermark of resource usage per device since the cgroup was created.
+
+	An example for mlx4 and ocrdma device follows::
+
+	  mlx4_0 hca_handle=1 hca_object=20
+	  ocrdma1 hca_handle=0 hca_object=23
+
+  rdma.events
+	A read-only nested-keyed file which exists on non-root cgroups
+	and contains the following keys:
+
+	  max
+		The number of times a process in this cgroup or its
+		descendants attempted an RDMA resource allocation that
+		was rejected because a rdma.max limit in the subtree
+		was reached.  This is a hierarchical counter propagated
+		upward to all ancestor cgroups.  A value change in this
+		file generates a file modified event.
+
+	  alloc_fail
+		The number of RDMA resource allocation attempts that
+		originated in this cgroup or its descendants and failed
+		due to a rdma.max limit being reached.  This is a
+		hierarchical counter propagated upward.
+
+	An example for mlx4 device follows::
+
+	  mlx4_0 hca_handle.max=5 hca_handle.alloc_fail=3 hca_object.max=0 hca_object.alloc_fail=0
+
+  rdma.events.local
+	Similar to rdma.events but the fields are local to the cgroup,
+	i.e. not hierarchical.  The file modified event generated on this
+	file reflects only the local events.
+
+	The following nested keys are defined.
+
+	  max
+		The number of times a process in this cgroup or its
+		descendants attempted an RDMA resource allocation that
+		was rejected because this cgroup's own rdma.max limit
+		was reached.
+
+	  alloc_fail
+		The number of RDMA resource allocation attempts
+		originating from this cgroup that failed due to this
+		cgroup's or an ancestor's rdma.max limit.
+
+	An example for mlx4 device follows::
+
+	  mlx4_0 hca_handle.max=5 hca_handle.alloc_fail=0 hca_object.max=0 hca_object.alloc_fail=0
diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst
index 993446ab66d0f..0df15a672cf34 100644
--- a/Documentation/admin-guide/cgroup-v2.rst
+++ b/Documentation/admin-guide/cgroup-v2.rst
@@ -1145,7 +1145,7 @@ will be referred to. All time durations are in microseconds.
 	This file exists whether the controller is enabled or not.
 
 	It always reports the following three stats, which account for all the
-	processes in the cgroup:
+	processes in the cgroup (including those in descendant cgroups):
 
 	- usage_usec
 	- user_usec
@@ -1160,6 +1160,27 @@ will be referred to. All time durations are in microseconds.
 	- nr_bursts
 	- burst_usec
 
+	Note that the above five CFS bandwidth stats are non-hierarchical;
+	they only account for throttling caused by this cgroup's own bandwidth
+	limit, not including throttling inherited from ancestor cgroups.
+
+  cpu.stat.local
+	A read-only flat-keyed file.
+	This file exists whether the controller is enabled or not.
+
+	It reports the following stat when the controller is enabled:
+
+	- throttled_usec
+
+	Unlike the ``throttled_usec`` reported by ``cpu.stat`` which
+	accounts for throttling caused by this cgroup's own CFS
+	bandwidth limit, ``cpu.stat.local`` reports the actual
+	throttling time incurred by this cgroup's own runqueues,
+	which may include throttling inherited from ancestor
+	cgroup bandwidth limits.
+
+	When the controller is not enabled, this stat is not reported.
+
   cpu.weight
 	A read-write single value file which exists on non-root
 	cgroups.  The default is "100".
@@ -1570,7 +1591,7 @@ The following nested keys are defined.
 	  sock (npn)
 		Amount of memory used in network transmission buffers
 
-	  vmalloc (npn)
+	  vmalloc
 		Amount of memory used for vmap backed memory.
 
 	  shmem
@@ -1735,7 +1756,7 @@ The following nested keys are defined.
 		Number of pages written from zswap to swap.
 
 	  zswap_incomp
-		Number of incompressible pages currently stored in zswap
+		Amount of memory used by incompressible pages currently stored in zswap
 		without compression. These pages could not be compressed to
 		a size smaller than PAGE_SIZE, so they are stored as-is.
 
@@ -2257,10 +2278,11 @@ groups D and F will influence each other.  Group G will influence nobody::
 So the ideal way to configure this is to set io.latency in groups A, B, and C.
 Generally you do not want to set a value lower than the latency your device
 supports.  Experiment to find the value that works best for your workload.
-Start at higher than the expected latency for your device and watch the
-avg_lat value in io.stat for your workload group to get an idea of the
-latency you see during normal operation.  Use the avg_lat value as a basis for
-your real setting, setting at 10-15% higher than the value in io.stat.
+Start at higher than the expected latency for your device and, with
+blkcg_debug_stats enabled, watch the avg_lat value in io.stat for your
+workload group to get an idea of the latency you see during normal operation.
+Use the avg_lat value as a basis for your real setting, setting at 10-15%
+higher than the value in io.stat.
 
 How IO Latency Throttling Works
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -2298,7 +2320,9 @@ IO Latency Interface Files
 
   io.stat
 	If the controller is enabled you will see extra stats in io.stat in
-	addition to the normal ones.
+	addition to the normal ones.  These debug stats are only emitted when
+	the blkcg_debug_stats module parameter is enabled (it is disabled by
+	default).
 
 	  depth
 		This is the current queue depth for the group.
@@ -2527,6 +2551,13 @@ Cpuset Interface Files
 	a need to change "cpuset.mems" with active tasks, it shouldn't
 	be done frequently.
 
+	For a multithreaded process, the threadgroup leader is
+	considered the owner of the group's memory. Memory policy
+	rebinding and migration will only happen with respect to the
+	threadgroup leader. To avoid unexpected results, non-leading
+	threads shouldn't be put into another cgroup whose "cpuset.mems"
+	doesn't fully overlap that of the threadgroup leader.
+
   cpuset.mems.effective
 	A read-only multiple values file which exists on all
 	cpuset-enabled cgroups.
@@ -2861,6 +2892,12 @@ DMEM Interface Files
 	The semantics are the same as for the memory cgroup controller, and are
 	calculated in the same way.
 
+  dmem.peak
+	A read-only nested-keyed file that exists on non-root cgroups.
+
+	The max device memory usage recorded for the cgroup and its
+	descendants since the creation of the cgroup for each region.
+
   dmem.capacity
 	A read-only file that describes maximum region capacity.
 	It only exists on the root cgroup. Not all memory can be
@@ -2934,7 +2971,8 @@ include/linux/misc_cgroup.h.
 Misc Interface Files
 ~~~~~~~~~~~~~~~~~~~~
 
-Miscellaneous controller provides 3 interface files. If two misc resources (res_a and res_b) are registered then:
+Miscellaneous controller provides the following interface files. If two misc
+resources (res_a and res_b) are registered then:
 
   misc.capacity
         A read-only flat-keyed file shown only in the root cgroup.  It shows
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index b5493a7f8f228..eb4e0d28f74b8 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -1089,14 +1089,14 @@ Kernel parameters
 			It will be ignored when crashkernel=X,high is not used
 			or memory reserved is below 4G.
 	crashkernel=size[KMG],cma
-			[KNL, X86, ppc] Reserve additional crash kernel memory from
-			CMA. This reservation is usable by the first system's
-			userspace memory and kernel movable allocations (memory
-			balloon, zswap). Pages allocated from this memory range
-			will not be included in the vmcore so this should not
-			be used if dumping of userspace memory is intended and
-			it has to be expected that some movable kernel pages
-			may be missing from the dump.
+			[KNL, X86, ARM64, RISCV, PPC] Reserve additional crash
+			kernel memory from CMA. This reservation is usable by
+			the first system's userspace memory and kernel movable
+			allocations (memory balloon, zswap). Pages allocated
+			from this memory range will not be included in the vmcore
+			so this should not be used if dumping of userspace memory
+			is intended and it has to be expected that some movable
+			kernel pages may be missing from the dump.
 
 			A standard crashkernel reservation, as described above,
 			is still needed to hold the crash kernel and initrd.
@@ -1614,10 +1614,17 @@ Kernel parameters
 			on all PCI bridges while in the EFI boot stub
 
 	efi_no_storage_paranoia [EFI,X86,EARLY]
-			Using this parameter you can use more than 50% of
-			your efi variable storage. Use this parameter only if
-			you are really sure that your UEFI does sane gc and
-			fulfills the spec otherwise your board may brick.
+			The kernel reserves 5KB of EFI variable storage for
+			safety, because some UEFI implementation may fail to
+			boot if there's insufficient space in the EFI variable
+			storage.
+
+			Using this parameter, you can use the 5KB reservation
+			in the EFI variable storage.
+
+			However, Use this parameter only if you are really
+			sure that your UEFI does sane gc and fulfills the spec
+			otherwise your board may brick.
 
 	efivar_ssdt=	[EFI; X86] Name of an EFI variable that contains an SSDT
 			that is to be dynamically loaded by Linux. If there are
@@ -2064,8 +2071,11 @@ Kernel parameters
 	hugetlb_cma=	[HW,CMA,EARLY] The size of a CMA area used for allocation
 			of gigantic hugepages. Or using node format, the size
 			of a CMA area per node can be specified.
-			Format: nn[KMGTPE] or (node format)
-				<node>:nn[KMGTPE][,<node>:nn[KMGTPE]]
+			The size can be an absolute value (e.g., 2G) or a
+			percentage of the total memory or node memory (e.g., 20%).
+			Format: nn[KMGTPE] or nn% or (node format)
+				<node>:nn[KMGTPE][,<node>:nn[KMGTPE]] or
+				<node>:nn%[,<node>:nn%]
 
 			The size must be a multiple of the gigantic page size.
 			When using node format, this applies to each per-node size.
@@ -3977,6 +3987,11 @@ Kernel parameters
 			Note that even when enabled, there are a few cases where
 			the feature is not effective.
 
+	mempool_debug	[MM]
+			Enable mempool debugging. This enables element
+			poison checking when freeing elements back to the
+			pool. Useful for debugging mempool corruption.
+
 	memtest=	[KNL,X86,ARM,M68K,PPC,RISCV,EARLY] Enable memtest
 			Format: <integer>
 			default : 0 <disable>
@@ -4587,9 +4602,6 @@ Kernel parameters
 	norandmaps	Don't use address space randomization.  Equivalent to
 			echo 0 > /proc/sys/kernel/randomize_va_space
 
-	noreplace-smp	[X86-32,SMP] Don't replace SMP instructions
-			with UP alternatives
-
 	noresume	[SWSUSP] Disables resume and restores original swap
 			space.
 
@@ -6413,9 +6425,9 @@ Kernel parameters
 	reboot=		[KNL]
 			Format (x86 or x86_64):
 				[w[arm] | c[old] | h[ard] | s[oft] | g[pio]] | d[efault] \
-				[[,]s[mp]#### \
+				[[,]s[mp]####] \
 				[[,]b[ios] | a[cpi] | k[bd] | t[riple] | e[fi] | p[ci]] \
-				[[,]f[orce]
+				[[,]f[orce]]
 			Where reboot_mode is one of warm (soft) or cold (hard) or gpio
 					(prefix with 'panic_' to set mode for panic
 					reboot only),
@@ -6875,7 +6887,7 @@ Kernel parameters
 			xtime_lock contention on larger systems, and/or RCU lock
 			contention on all systems with CONFIG_MAXSMP set.
 			Format: { "0" | "1" }
-			0 -- disable. (may be 1 via CONFIG_CMDLINE="skew_tick=1"
+			0 -- disable. (may be 1 via CONFIG_CMDLINE="skew_tick=1")
 			1 -- enable.
 			Note: increases power consumption, thus should only be
 			enabled if running jitter sensitive (HPC/RT) workloads.
@@ -6916,7 +6928,7 @@ Kernel parameters
 			apic=verbose is specified.
 			Example: apic=debug show_lapic=all
 
-	slab_debug[=options[,slabs][;[options[,slabs]]...]	[MM]
+	slab_debug[=options[,slabs][;[options[,slabs]]...]]	[MM]
 			Enabling slab_debug allows one to determine the
 			culprit if slab objects become corrupted. Enabling
 			slab_debug can create guard zones around objects and
diff --git a/Documentation/admin-guide/laptops/thinkpad-acpi.rst b/Documentation/admin-guide/laptops/thinkpad-acpi.rst
index f874db31801dc..db4588af02781 100644
--- a/Documentation/admin-guide/laptops/thinkpad-acpi.rst
+++ b/Documentation/admin-guide/laptops/thinkpad-acpi.rst
@@ -1543,6 +1543,30 @@ Values:
 
 	This setting can also be toggled via the Fn+doubletap hotkey.
 
+USB-C Security
+--------------
+
+sysfs: usb_c_security
+
+Reports the current state of the USB-C Security (Restricted Mode) feature
+on supported ThinkPad systems. When enabled, USB-C data connections are
+disabled while power delivery is preserved.
+
+The available command is::
+
+        cat /sys/devices/platform/thinkpad_acpi/usb_c_security
+
+Values:
+
+	* ``enabled``  - USB-C data connections are currently blocked
+	* ``disabled`` - USB-C data connections are currently allowed
+
+The attribute is read-only. The USB-C Security state can only be toggled
+via the Fn+U followed by Fn+S hotkey chord.
+
+The sysfs attribute is not created on platforms that do not support this
+feature.
+
 Auxmac
 ------
 
diff --git a/Documentation/admin-guide/laptops/uniwill-laptop.rst b/Documentation/admin-guide/laptops/uniwill-laptop.rst
index 24b41dbab886b..be50b45b82efa 100644
--- a/Documentation/admin-guide/laptops/uniwill-laptop.rst
+++ b/Documentation/admin-guide/laptops/uniwill-laptop.rst
@@ -77,6 +77,19 @@ LED class device. The default name of this LED class device is ``uniwill:multico
 See Documentation/ABI/testing/sysfs-driver-uniwill-laptop for details on how to control the various
 animation modes of the lightbar.
 
+Keyboard Backlight
+------------------
+
+The ``uniwill-laptop`` driver supports controlling the keyboard backlight using the standard
+LED class interface. The default name of this LED class device is ``uniwill:white:kbd_backlight``
+when the keyboard backlight supports only a single color, or ``uniwill:multicolor:kbd_backlight``
+when the keyboard backlight supports RGB colors. The maximum intensity for each color channel
+in RGB mode is 50.
+
+Keep in mind that due to hardware design choices, the driver does not support the RGB value
+``0x000000`` (black), instead it will fall back to ``0x010101`` (faint white). In order to
+disable the keyboard backlight, the standard LED brightness setting has to be used instead.
+
 Configurable TGP
 ----------------
 
@@ -85,6 +98,20 @@ allow it.
 
 See Documentation/ABI/testing/sysfs-driver-uniwill-laptop for details.
 
+AC Auto Boot
+------------
+
+The ``uniwill-laptop`` driver allows the user to configure if the system should automatically
+boot when being connected to a power source, see
+Documentation/ABI/testing/sysfs-driver-uniwill-laptop for details.
+
+USB Powershare
+--------------
+
+The ``uniwill-laptop`` driver allows the user to configure if the system should continue to
+provide power via the USB ports when hibernating or powered off, see
+Documentation/ABI/testing/sysfs-driver-uniwill-laptop for details.
+
 References
 ==========
 
diff --git a/Documentation/admin-guide/media/bttv.rst b/Documentation/admin-guide/media/bttv.rst
index 58cbaf6df694e..78c3e560b806f 100644
--- a/Documentation/admin-guide/media/bttv.rst
+++ b/Documentation/admin-guide/media/bttv.rst
@@ -1239,7 +1239,7 @@ Models:
 - Galaxis DVB Card C CI
 - Galaxis DVB Card S
 - Galaxis DVB Card C
-- Galaxis plug.in S [neuer Name: Galaxis DVB Card S CI
+- Galaxis plug.in S [new Name: Galaxis DVB Card S CI]
 
 Hauppauge
 ~~~~~~~~~
diff --git a/Documentation/admin-guide/media/em28xx-cardlist.rst b/Documentation/admin-guide/media/em28xx-cardlist.rst
index 7dac07986d917..741aa7f7c8822 100644
--- a/Documentation/admin-guide/media/em28xx-cardlist.rst
+++ b/Documentation/admin-guide/media/em28xx-cardlist.rst
@@ -446,3 +446,7 @@ EM28xx cards list
      - MyGica UTV3 Analog USB2.0 TV Box
      - em2860
      - eb1a:2860
+   * - 113
+     - StarTech SVID2USB232
+     - em28281
+     - eb1a:8286
diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst
index 011296f1e7c21..da5f9afd08aef 100644
--- a/Documentation/admin-guide/mm/damon/usage.rst
+++ b/Documentation/admin-guide/mm/damon/usage.rst
@@ -73,9 +73,10 @@ comma (",").
     │ │ │ │ │ │ │ intervals_goal/access_bp,aggrs,min_sample_us,max_sample_us
     │ │ │ │ │ │ nr_regions/min,max
     │ │ │ │ │ │ :ref:`probes <damon_usage_sysfs_probes>`/nr_probes
-    │ │ │ │ │ │ │ 0/filters/nr_filters
-    │ │ │ │ │ │ │ │ 0/type,matching,allow,path
-    │ │ │ │ │ │ │ │ ...
+    │ │ │ │ │ │ │ 0/weight
+    │ │ │ │ │ │ │ │ filters/nr_filters
+    │ │ │ │ │ │ │ │ │ 0/type,matching,allow,path
+    │ │ │ │ │ │ │ │ │ ...
     │ │ │ │ │ │ │ ...
     │ │ │ │ │ :ref:`targets <sysfs_targets>`/nr_targets
     │ │ │ │ │ │ :ref:`0 <sysfs_target>`/pid_target,obsolete_target
@@ -246,7 +247,7 @@ writing to and reading from the files.
 Under ``nr_regions`` directory, two files for the lower-bound and upper-bound
 of DAMON's monitoring regions (``min`` and ``max``, respectively), which
 controls the monitoring overhead, exist.  You can set and get the values by
-writing to and rading from the files.
+writing to and reading from the files.
 
 For more details about the intervals and monitoring regions range, please refer
 to the Design document (:doc:`/mm/damon/design`).
@@ -264,7 +265,7 @@ Please refer to  the :ref:`design document of the feature
 <damon_design_monitoring_intervals_autotuning>` for the internal of the tuning
 mechanism.  Reading and writing the four files under ``intervals_goal``
 directory shows and updates the tuning parameters that described in the
-:ref:design doc <damon_design_monitoring_intervals_autotuning>` with the same
+:ref:`design doc <damon_design_monitoring_intervals_autotuning>` with the same
 names.  The tuning starts with the user-set ``sample_us`` and ``aggr_us``.  The
 tuning-applied current values of the two intervals can be read from the
 ``sample_us`` and ``aggr_us`` files after writing ``update_tuned_intervals`` to
@@ -286,6 +287,10 @@ In each probe directory, one directory, ``filters`` exists.  The directory
 contains files for installing filters for the probe, that is used to determine
 the data attribute for the probe.
 
+Each probe directory also contains ``weight`` file.  Reading from and writing
+to the file gets and sets the :ref:`attributes-only monitoring
+<damon_design_attrs_only_monitoring>` weight for the attribute of the probe.
+
 In the beginning, ``filters`` directory has only one file, ``nr_filters``.
 Writing a number (``N``) to the file creates the number of child directories
 named ``0`` to ``N-1``.  Each directory represents each filter and works in a
@@ -377,7 +382,7 @@ schemes/<N>/
 In each scheme directory, nine directories (``access_pattern``, ``quotas``,
 ``watermarks``, ``core_filters``, ``ops_filters``, ``filters``, ``dests``,
 ``stats``, and ``tried_regions``) and three files (``action``, ``target_nid``
-and ``apply_interval``) exist.
+and ``apply_interval_us``) exist.
 
 The ``action`` file is for setting and getting the scheme's :ref:`action
 <damon_design_damos_action>`.  The keywords that can be written to and read
@@ -743,7 +748,7 @@ counter).  Finally the tenth field (``X``) shows the ``age`` of the region
 (refer to :ref:`design <damon_design_age_tracking>` for more details of the
 counter).
 
-If the event was ``damon:damos_beofre_apply``, the ``perf script`` output would
+If the event was ``damon:damos_before_apply``, the ``perf script`` output would
 be somewhat like below::
 
     kdamond.0 47293 [000] 80801.060214: damon:damos_before_apply: ctx_idx=0 scheme_idx=0 target_idx=0 nr_regions=11 121932607488-135128711168: 0 136
diff --git a/Documentation/admin-guide/mm/pagemap.rst b/Documentation/admin-guide/mm/pagemap.rst
index c57e61b5d8aa8..77447c4546321 100644
--- a/Documentation/admin-guide/mm/pagemap.rst
+++ b/Documentation/admin-guide/mm/pagemap.rst
@@ -19,8 +19,11 @@ There are four components to pagemap:
     * Bit  55    pte is soft-dirty (see
       Documentation/admin-guide/mm/soft-dirty.rst)
     * Bit  56    page exclusively mapped (since 4.2)
-    * Bit  57    pte is uffd-wp write-protected (since 5.13) (see
-      Documentation/admin-guide/mm/userfaultfd.rst)
+    * Bit  57    pte is tracked by userfaultfd (since 5.13) — in a
+      ``VM_UFFD_WP`` VMA this indicates a write-protected PTE; in a
+      ``VM_UFFD_RWP`` VMA it indicates an RWP-protected PTE. WP and
+      RWP are mutually exclusive per VMA, so the meaning is
+      unambiguous. See Documentation/admin-guide/mm/userfaultfd.rst.
     * Bit  58    pte is a guard region (since 6.15) (see madvise (2) man page)
     * Bits 59-60 zero
     * Bit  61    page is file-page or shared-anon (since 3.5)
@@ -67,7 +70,7 @@ number of times a page is mapped.
  * ``/proc/kpageflags``.  This file contains a 64-bit set of flags for each
    page, indexed by PFN.
 
-   The flags are (from ``fs/proc/page.c``, above kpageflags_read):
+   The flags are (from ``include/uapi/linux/kernel-page-flags.h``):
 
     0. LOCKED
     1. ERROR
@@ -244,7 +247,8 @@ in this IOCTL:
 Following flags about pages are currently supported:
 
 - ``PAGE_IS_WPALLOWED`` - Page has async-write-protection enabled
-- ``PAGE_IS_WRITTEN`` - Page has been written to from the time it was write protected
+- ``PAGE_IS_WRITTEN`` - Page in a ``UFFDIO_REGISTER_MODE_WP`` VMA has been
+  written to since it was write-protected. Only reported inside such VMAs.
 - ``PAGE_IS_FILE`` - Page is file backed
 - ``PAGE_IS_PRESENT`` - Page is present in the memory
 - ``PAGE_IS_SWAPPED`` - Page is in swapped
@@ -252,6 +256,9 @@ Following flags about pages are currently supported:
 - ``PAGE_IS_HUGE`` - Page is PMD-mapped THP or Hugetlb backed
 - ``PAGE_IS_SOFT_DIRTY`` - Page is soft-dirty
 - ``PAGE_IS_GUARD`` - Page is a part of a guard region
+- ``PAGE_IS_ACCESSED`` - Page in a ``UFFDIO_REGISTER_MODE_RWP`` VMA has been
+  accessed since RWP was applied. Only reported inside such VMAs. See
+  Documentation/admin-guide/mm/userfaultfd.rst for the RWP workflow.
 
 The ``struct pm_scan_arg`` is used as the argument of the IOCTL.
 
@@ -264,7 +271,7 @@ The ``struct pm_scan_arg`` is used as the argument of the IOCTL.
     provided or not.
  3. The range is specified through ``start`` and ``end``.
  4. The walk can abort before visiting the complete range such as the user buffer
-    can get full etc. The walk ending address is specified in``end_walk``.
+    can get full etc. The walk ending address is specified in ``walk_end``.
  5. The output buffer of ``struct page_region`` array and size is specified in
     ``vec`` and ``vec_len``.
  6. The optional maximum requested pages are specified in the ``max_pages``.
@@ -275,7 +282,7 @@ Find pages which have been written and WP them as well::
 
    struct pm_scan_arg arg = {
    .size = sizeof(arg),
-   .flags = PM_SCAN_CHECK_WPASYNC | PM_SCAN_CHECK_WPASYNC,
+   .flags = PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC,
    ..
    .category_mask = PAGE_IS_WRITTEN,
    .return_mask = PAGE_IS_WRITTEN,
@@ -288,7 +295,7 @@ present or huge::
    .size = sizeof(arg),
    .flags = 0,
    ..
-   .category_mask = PAGE_IS_WRITTEN | PAGE_IS_SWAPPED,
+   .category_mask = PAGE_IS_WRITTEN | PAGE_IS_FILE,
    .category_inverted = PAGE_IS_SWAPPED,
    .category_anyof_mask = PAGE_IS_PRESENT | PAGE_IS_HUGE,
    .return_mask = PAGE_IS_WRITTEN | PAGE_IS_SWAPPED |
diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst
index 23f8d13c2629d..16f37135ed80d 100644
--- a/Documentation/admin-guide/mm/transhuge.rst
+++ b/Documentation/admin-guide/mm/transhuge.rst
@@ -761,7 +761,7 @@ compact_fail
 	but failed.
 
 It is possible to establish how long the stalls were using the function
-tracer to record how long was spent in __alloc_pages() and
+tracer to record how long was spent in the page allocator and
 using the mm_page_alloc tracepoint to identify which allocations were
 for huge pages.
 
diff --git a/Documentation/admin-guide/mm/userfaultfd.rst b/Documentation/admin-guide/mm/userfaultfd.rst
index e5cc8848dcb39..783d969f0e28f 100644
--- a/Documentation/admin-guide/mm/userfaultfd.rst
+++ b/Documentation/admin-guide/mm/userfaultfd.rst
@@ -131,6 +131,16 @@ userfaults on the range registered. Not all ioctls will necessarily be
 supported for all memory types (e.g. anonymous memory vs. shmem vs.
 hugetlbfs), or all types of intercepted faults.
 
+.. note::
+
+   Re-registering an already-registered range must not drop any of the
+   modes that install per-PTE markers — currently
+   ``UFFDIO_REGISTER_MODE_WP`` and ``UFFDIO_REGISTER_MODE_RWP``. Doing
+   so would strand markers with no flag to describe them, so the call
+   is rejected with ``-EBUSY``; userspace must issue
+   ``UFFDIO_UNREGISTER`` first. This differs from older kernels, which
+   silently replaced the mode bits on re-registration.
+
 Userland can use the ``uffdio_register.ioctls`` to manage the virtual
 address space in the background (to add or potentially also remove
 memory from the ``userfaultfd`` registered range). This means a userfault
@@ -265,16 +275,16 @@ tracking and it can be different in a few ways:
   - Dirty information will not get lost if the pte was zapped due to
     various reasons (e.g. during split of a shmem transparent huge page).
 
-  - Due to a reverted meaning of soft-dirty (page clean when uffd-wp bit
-    set; dirty when uffd-wp bit cleared), it has different semantics on
-    some of the memory operations.  For example: ``MADV_DONTNEED`` on
+  - Due to a reverted meaning of soft-dirty (page clean when the uffd bit
+    is set; dirty when the uffd bit is cleared), it has different semantics
+    on some of the memory operations.  For example: ``MADV_DONTNEED`` on
     anonymous (or ``MADV_REMOVE`` on a file mapping) will be treated as
-    dirtying of memory by dropping uffd-wp bit during the procedure.
+    dirtying of memory by dropping the uffd bit during the procedure.
 
 The user app can collect the "written/dirty" status by looking up the
-uffd-wp bit for the pages being interested in /proc/pagemap.
+uffd bit for the pages being interested in /proc/pagemap.
 
-The page will not be under track of uffd-wp async mode until the page is
+The page will not be under track of userfaultfd-wp async mode until the page is
 explicitly write-protected by ``ioctl(UFFDIO_WRITEPROTECT)`` with the mode
 flag ``UFFDIO_WRITEPROTECT_MODE_WP`` set.  Trying to resolve a page fault
 that was tracked by async mode userfaultfd-wp is invalid.
@@ -297,6 +307,253 @@ transparent to the guest, we want that same address range to act as if it was
 still poisoned, even though it's on a new physical host which ostensibly
 doesn't have a memory error in the exact same spot.
 
+Read-Write Protection
+---------------------
+
+``UFFDIO_REGISTER_MODE_RWP`` enables read-write protection tracking on a
+memory range. It is similar to (but faster than) ``mprotect(PROT_NONE)``
+combined with a signal handler; unlike ``mprotect(PROT_NONE)``, RWP only
+traps accesses to *present* PTEs, so accesses to unpopulated addresses in a
+protected range fall through to the normal missing-page path. It uses the
+PROT_NONE hinting mechanism (same as NUMA balancing) to make pages
+inaccessible while keeping them resident in memory. Works on anonymous,
+shmem, and hugetlbfs memory.
+
+RWP is designed for VM memory managers that need to track the working set
+of guest memory for cold page eviction to tiered or remote storage.
+
+**Setup:**
+
+1. Open a userfaultfd and enable ``UFFD_FEATURE_RWP`` via ``UFFDIO_API``.
+   Optionally request ``UFFD_FEATURE_RWP_ASYNC`` as well — it requires
+   ``UFFD_FEATURE_RWP`` to be set in the same ``UFFDIO_API`` call.
+
+2. Register the guest memory range with ``UFFDIO_REGISTER_MODE_RWP``
+   (and ``UFFDIO_REGISTER_MODE_MISSING`` if evicted pages will need to be
+   fetched back from storage).
+
+**Feature availability:**
+
+RWP is built on top of two kernel primitives: a spare PTE bit owned by
+userfaultfd (``CONFIG_HAVE_ARCH_USERFAULTFD_WP``) and architecture support
+for present-but-inaccessible PTEs (``CONFIG_ARCH_HAS_PTE_PROTNONE``). When both
+are available on a 64-bit kernel, the build selects
+``CONFIG_USERFAULTFD_RWP=y`` and the ``VM_UFFD_RWP`` VMA flag becomes
+available.
+
+``UFFD_FEATURE_RWP`` and ``UFFD_FEATURE_RWP_ASYNC`` are unavailable when
+the running kernel or architecture does not support them — for example
+32-bit kernels (where ``VM_UFFD_RWP`` is unavailable), kernels built
+without ``CONFIG_USERFAULTFD_RWP``, and architectures whose ptes cannot
+carry the uffd bit at runtime (e.g. riscv without the ``SVRSW60T59B``
+extension). Requesting an unsupported feature in
+``uffdio_api.features`` makes ``UFFDIO_API`` fail with ``EINVAL`` and
+leaves the userfaultfd context uninitialized; the structure is returned
+zeroed, so the error path cannot be used to discover what the kernel
+supports. The recommended probe sequence is therefore to open a
+throwaway userfaultfd, call ``UFFDIO_API`` once with ``features = 0``,
+inspect the returned bitmask, close that fd, then open the real one
+and call ``UFFDIO_API`` again with only the supported features set.
+
+**Protecting and Unprotecting:**
+
+Use ``UFFDIO_RWPROTECT`` to protect or unprotect a range, mirroring the
+``UFFDIO_WRITEPROTECT`` interface::
+
+    struct uffdio_rwprotect rwp = {
+        .range = { .start = addr, .len = len },
+        .mode = UFFDIO_RWPROTECT_MODE_RWP,  /* protect */
+    };
+    ioctl(uffd, UFFDIO_RWPROTECT, &rwp);
+
+Setting ``UFFDIO_RWPROTECT_MODE_RWP`` sets PROT_NONE on present PTEs in the
+range. Pages stay resident and their physical frames are preserved — only
+access permissions are removed.
+
+Clearing ``UFFDIO_RWPROTECT_MODE_RWP`` restores normal VMA permissions and
+wakes any faulting threads (unless ``UFFDIO_RWPROTECT_MODE_DONTWAKE`` is set).
+
+**Scope of protection:**
+
+RWP protection is a property of *present* PTEs. ``UFFDIO_RWPROTECT`` only
+affects entries that are already populated. Unpopulated addresses within
+the range remain unpopulated; when first accessed they fault through the
+normal missing path (``do_anonymous_page()``, ``do_swap_page()``,
+``finish_fault()``) and the resulting PTE is not RWP-protected. To observe
+the population itself, co-register the range with
+``UFFDIO_REGISTER_MODE_MISSING``.
+
+Protection is preserved across page reclaim: a page swapped out while
+RWP-protected carries the marker on its swap entry, and swap-in restores
+the PROT_NONE state so the first access after swap-in still faults. The
+same applies to pages temporarily replaced by migration entries.
+
+Operations that drop the PTE entirely — ``MADV_DONTNEED`` on anonymous
+memory, hole-punch on shmem, truncation of a file mapping — also drop the
+RWP marker: the next access re-populates the range without protection.
+Unlike WP (which persists via ``PTE_MARKER_UFFD_WP``), there is no
+persistent RWP marker today. The user needs to re-arm the range with
+``UFFDIO_RWPROTECT`` after any operation that explicitly frees PTEs.
+
+**Fault Handling:**
+
+When a protected page is accessed:
+
+- **Sync mode** (default): The faulting thread blocks and a
+  ``UFFD_PAGEFAULT_FLAG_RWP`` message is delivered to the userfaultfd
+  handler. The handler resolves the fault with ``UFFDIO_RWPROTECT``
+  (clearing ``MODE_RWP``), which restores the PTE permissions and wakes
+  the faulting thread.
+
+- **Async mode** (``UFFD_FEATURE_RWP_ASYNC``): The kernel automatically
+  restores PTE permissions and the thread continues without blocking. No
+  message is delivered to the handler.
+
+**Runtime Mode Switching:**
+
+``UFFDIO_SET_MODE`` toggles ``UFFD_FEATURE_RWP_ASYNC`` at runtime, allowing
+the VMM to switch between lightweight async detection and safe sync
+eviction without re-registering. The toggle takes ``mmap_write_lock()``
+and calls ``vma_start_write()`` on each UFFD-armed VMA, draining
+in-flight per-VMA-locked faults before the new mode takes effect.
+
+**Working-set detection with PAGEMAP_SCAN:**
+
+RWP-protected PTEs carry the uffd PTE bit; an access (and, in async mode, its
+auto-resolution) clears it. ``PAGEMAP_SCAN`` reports ``PAGE_IS_ACCESSED`` once
+the bit is clear on a ``VM_UFFD_RWP`` VMA, so a *non-inverted* scan reports the
+pages that were touched during the interval -- the hot set::
+
+    struct pm_scan_arg arg = {
+        .size = sizeof(arg),
+        .start = guest_mem_start,
+        .end = guest_mem_end,
+        .vec = (uint64_t)regions,
+        .vec_len = regions_len,
+        .category_mask = PAGE_IS_ACCESSED,
+        .return_mask = PAGE_IS_ACCESSED,
+    };
+    long n = ioctl(pagemap_fd, PAGEMAP_SCAN, &arg);
+
+The returned ``page_region`` array lists the hot ranges. ``PAGE_IS_ACCESSED``
+is set on an accessed page whether it is still present or has since been
+swapped out, so the hot scan needs no ``PAGE_IS_PRESENT`` filter -- unpopulated
+holes carry neither bit and are excluded on their own.
+
+Track the hot set and reclaim everything else from the backing file (see the
+workflow below). Do **not** invert the scan to enumerate "cold" pages
+directly: an inverted scan reports only the ``VM_UFFD_RWP`` PTEs that are still
+protected, i.e. the resident portion of *this* VMA. For a file mapping the
+working set spans the whole file -- pages that live in the page cache but are
+not mapped into this VMA (a pre-populated tmpfs file, or memory populated
+through another mapping) are ``pte_none`` here, never appear in the scan, and
+would never be considered for eviction even though they occupy memory. Driving
+eviction from "file offsets minus the hot set" avoids that blind spot; a cold
+PTE scan cannot. To additionally record the *first* access to a cached but
+unmapped page (e.g. pre-populated content) as hot, co-register the range with
+``UFFDIO_REGISTER_MODE_MINOR``: such accesses then fault as minor faults
+instead of mapping the page silently.
+
+**Cleanup:**
+
+When the userfaultfd is closed or the range is unregistered, all PROT_NONE
+PTEs are automatically restored to their normal VMA permissions. This
+prevents pages from becoming permanently inaccessible.
+
+**VMM Working Set Tracking Workflow:**
+
+A typical VMM lifecycle for cold page eviction to tiered storage. Two
+mappings of the same shmem (or hugetlbfs) file are used: ``guest_mem`` is
+the RWP-registered mapping that vCPUs access through, and ``io_mem`` is a
+private mapping for VMM-side I/O. Reading ``io_mem`` does not go through
+the RWP-protected PTEs of ``guest_mem``, so the VMM's own ``pwrite()``
+never traps on its own ::
+
+    /* One-time setup */
+    fd = memfd_create("guest", MFD_CLOEXEC);
+    ftruncate(fd, guest_size);
+    guest_mem = mmap(NULL, guest_size, PROT_READ | PROT_WRITE,
+                     MAP_SHARED, fd, 0);  /* vCPU view, RWP-registered */
+    io_mem    = mmap(NULL, guest_size, PROT_READ | PROT_WRITE,
+                     MAP_SHARED, fd, 0);  /* VMM I/O view, unprotected */
+
+    uffd = userfaultfd(O_CLOEXEC | O_NONBLOCK);
+    struct uffdio_api api = {
+        .api = UFFD_API,
+        .features = UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC,
+    };
+    ioctl(uffd, UFFDIO_API, &api);
+    if (!(api.features & UFFD_FEATURE_RWP))
+        /* RWP unavailable on this kernel/arch -- fall back. */
+    ioctl(uffd, UFFDIO_REGISTER, &(struct uffdio_register){
+        .range = { guest_mem, guest_size },
+        .mode = UFFDIO_REGISTER_MODE_RWP |
+                UFFDIO_REGISTER_MODE_MISSING,
+    });
+
+    /* Tracking loop */
+    while (vm_running) {
+        /* 1. Detection phase (async -- no vCPU stalls) */
+        ioctl(uffd, UFFDIO_RWPROTECT, &(struct uffdio_rwprotect){
+            .range = full_range,
+            .mode = UFFDIO_RWPROTECT_MODE_RWP });
+        sleep(tracking_interval);
+
+        /*
+         * 2. Switch to sync BEFORE scanning. In async mode a vCPU
+         * access races eviction: it would auto-resolve and mark the
+         * page hot just as the VMM writes it out and punches it,
+         * losing the update. Sync mode makes such accesses block and
+         * be delivered, freezing the hot snapshot for the rest of the
+         * iteration.
+         */
+        ioctl(uffd, UFFDIO_SET_MODE,
+              &(struct uffdio_set_mode){
+                  .disable = UFFD_FEATURE_RWP_ASYNC });
+
+        /* 3. Read the hot set: pages touched this interval. */
+        ioctl(pagemap_fd, PAGEMAP_SCAN, &(struct pm_scan_arg){
+            .category_mask = PAGE_IS_ACCESSED,
+            .return_mask = PAGE_IS_ACCESSED,
+            ...
+        });
+
+        /*
+         * 4. Reclaim the file offsets that are NOT in the hot set.
+         * Driving this from the file's offset space (rather than from a
+         * cold PTE scan) also reclaims pages that are cached but not
+         * mapped into guest_mem, e.g. pre-populated content.
+         */
+        for each non-hot offset range:
+            /* Read from io_mem -- bypasses RWP, no fault. */
+            pwrite(storage_fd, (char *)io_mem + off, len, off);
+            /* Drop the page from the shared file. */
+            fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
+                      off, len);
+            /*
+             * Wake any vCPU blocked on the RWP fault for this range:
+             * fallocate() does not iterate ctx->fault_pending_wqh.
+             */
+            ioctl(uffd, UFFDIO_WAKE, &(struct uffdio_range){
+                .start = (uintptr_t)guest_mem + off, .len = len });
+
+        /* 5. Resume async tracking */
+        ioctl(uffd, UFFDIO_SET_MODE,
+              &(struct uffdio_set_mode){
+                  .enable = UFFD_FEATURE_RWP_ASYNC });
+    }
+
+During step 4, a vCPU that accesses a ``guest_mem`` offset being evicted
+blocks with a ``UFFD_PAGEFAULT_FLAG_RWP`` fault while the eviction is in
+progress. After ``fallocate()`` punches the page out and ``UFFDIO_WAKE``
+fires, the vCPU retries the access, faults as ``MISSING``, and the
+handler resolves it with ``UFFDIO_COPY`` from storage.
+
+This workflow targets shmem and hugetlbfs (both support a private
+``io_mem`` mapping over the same fd). Anonymous-memory backings need a
+different inner-loop strategy because the VMM has no way to read the
+page without going through the RWP-protected mapping.
+
 QEMU/KVM
 ========
 
diff --git a/Documentation/admin-guide/pm/cpufreq.rst b/Documentation/admin-guide/pm/cpufreq.rst
index 8831cface5855..34baf20cc2025 100644
--- a/Documentation/admin-guide/pm/cpufreq.rst
+++ b/Documentation/admin-guide/pm/cpufreq.rst
@@ -497,7 +497,7 @@ This governor exposes the following tunables:
 	represented by it to be 1.5 times as high as the transition latency
 	(the default)::
 
-	# echo `$(($(cat cpuinfo_transition_latency) * 3 / 2))` > ondemand/sampling_rate
+	# echo $(($(cat cpuinfo_transition_latency) * 3 / 2)) > ondemand/sampling_rate
 
 ``up_threshold``
 	If the estimated CPU load is above this value (in percent), the governor
diff --git a/Documentation/admin-guide/sysctl/crypto.rst b/Documentation/admin-guide/sysctl/crypto.rst
index b707bd314a64b..9a1bd53287f47 100644
--- a/Documentation/admin-guide/sysctl/crypto.rst
+++ b/Documentation/admin-guide/sysctl/crypto.rst
@@ -7,6 +7,42 @@ kernel configuration:
 
 .. contents:: :local:
 
+.. _af_alg_restrict:
+
+af_alg_restrict
+===============
+
+Controls the level of restriction of AF_ALG.
+
+AF_ALG is a deprecated and rarely-used userspace interface that is a
+frequent source of vulnerabilities. It also unnecessarily exposes a
+large number of kernel implementation details. For more information
+about AF_ALG, see :ref:`Documentation/crypto/userspace-if.rst
+<crypto_userspace_interface>`.
+
+Starting in Linux v7.3, AF_ALG supports only a limited set of
+algorithms by default. This sysctl allows the system administrator to
+remove this restriction when needed for compatibility reasons, or to
+go further and disable AF_ALG entirely. The default value is 1.
+
+===  ==================================================================
+0    AF_ALG is unrestricted.
+
+1    AF_ALG is supported with a limited list of algorithms. The list
+     is designed for compatibility with known users such as iwd and
+     bluez that haven't yet been fixed to use userspace crypto code.
+
+     Specifically, there is an allowlist for unprivileged processes
+     and a somewhat longer allowlist for processes that hold
+     CAP_SYS_ADMIN or CAP_NET_ADMIN in the initial user namespace.
+
+     Attempts to bind() an AF_ALG socket with a disallowed algorithm
+     fail with ENOENT.
+
+2    AF_ALG is completely disabled. Attempts to create an AF_ALG
+     socket fail with EAFNOSUPPORT.
+===  ==================================================================
+
 fips_enabled
 ============
 
diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst
index c6994e55d1411..b6328cd0f43e9 100644
--- a/Documentation/admin-guide/sysctl/kernel.rst
+++ b/Documentation/admin-guide/sysctl/kernel.rst
@@ -1402,6 +1402,23 @@ Note that if you change this from 0 to 1, already created segments
 without users and with a dead originative process will be destroyed.
 
 
+syscall_user_dispatch
+=====================
+
+Controls whether userspace may arm Syscall User Dispatch via
+``prctl(PR_SET_SYSCALL_USER_DISPATCH, ...)`` or the
+``PTRACE_SET_SYSCALL_USER_DISPATCH_CONFIG`` ptrace request:
+
+  ==  ===================================================================
+  0   Arming syscall user dispatch is denied with ``-EPERM``.  Tasks that
+      already armed it keep it, and disabling it is always permitted.
+  1   (default) Arming syscall user dispatch is permitted.
+  ==  ===================================================================
+
+Only present when the kernel is built with ``CONFIG_SYSCALL_USER_DISPATCH``
+and ``CONFIG_PROC_SYSCTL``.
+
+
 sysctl_writes_strict
 ====================
 
diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst
index b9b0c218bfb44..5b318d17aa4b4 100644
--- a/Documentation/admin-guide/sysctl/vm.rst
+++ b/Documentation/admin-guide/sysctl/vm.rst
@@ -19,9 +19,6 @@ The files in this directory can be used to tune the operation
 of the virtual memory (VM) subsystem of the Linux kernel and
 the writeout of dirty data to disk.
 
-Default values and initialization routines for most of these
-files can be found in mm/swap.c.
-
 Currently, these files are in /proc/sys/vm:
 
 - admin_reserve_kbytes
@@ -67,6 +64,7 @@ Currently, these files are in /proc/sys/vm:
 - page-cluster
 - page_lock_unfairness
 - panic_on_oom
+- panic_on_unrecoverable_memory_failure
 - percpu_pagelist_high_fraction
 - stat_interval
 - stat_refresh
@@ -925,6 +923,85 @@ panic_on_oom=2+kdump gives you very strong tool to investigate
 why oom happens. You can get snapshot.
 
 
+panic_on_unrecoverable_memory_failure
+======================================
+
+When a hardware memory error (e.g. multi-bit ECC) hits a kernel page
+that cannot be recovered by the memory failure handler, the default
+behaviour is to ignore the error and continue operation.  This is
+dangerous because the corrupted data remains accessible to the kernel,
+risking silent data corruption or a delayed crash when the poisoned
+memory is next accessed.
+
+When enabled, this sysctl triggers a panic on memory failure events
+hitting kernel-owned pages that the handler cannot recover:
+``PageReserved`` (firmware reservations, kernel image, vDSO, zero
+page, and similar memblock-reserved regions), ``PageSlab``,
+``PageTable``, and ``PageLargeKmalloc``.  These are owned by the
+kernel and the memory failure handler cannot reliably evict their
+contents.
+
+Other unrecoverable kernel-owned populations (vmalloc allocations,
+kernel stack pages, ...) are not currently covered because the
+handler has no page-type signal that distinguishes them from a
+userspace folio temporarily off the LRU during migration or
+compaction.  Such pages still go through the standard
+MF_MSG_GET_HWPOISON path: ``PG_hwpoison`` is set on them and a
+delayed crash on the next access remains possible.  Coverage may
+grow as the handler gains stronger kernel-ownership signals.
+
+Recoverable failure paths are also intentionally left out: in-flight
+buddy allocations and other transient races with the page allocator
+can reach the same diagnostic, and panicking on them would risk
+killing the box for a page destined for userspace where the standard
+SIGBUS recovery path applies.  Pages whose state could not be
+classified at all are not covered either, since an unknown state is
+not a sound basis for a panic decision.
+
+For many environments it is preferable to panic immediately with a clean
+crash dump that captures the original error context, rather than to
+continue and face a random crash later whose cause is difficult to
+diagnose.
+
+Use cases
+---------
+
+This option is most useful in environments where unattributed crashes
+are expensive to debug or where data integrity must take precedence
+over availability:
+
+* Large fleets, where multi-bit ECC errors on kernel pages are observed
+  regularly and post-mortem analysis of an unrelated downstream crash
+  (often seconds to minutes after the original error) consumes
+  significant engineering effort.
+
+* Systems configured with kdump, where panicking at the moment of the
+  hardware error produces a vmcore that still contains the faulting
+  address, the affected page state, and the originating MCE/GHES
+  record — context that is typically lost by the time a delayed crash
+  occurs.
+
+* High-availability clusters that rely on fast, deterministic node
+  failure for failover, and prefer an immediate panic over silent data
+  corruption propagating to replicas or persistent storage.
+
+* Kernel and platform developers reproducing hwpoison issues with
+  tools such as ``mce-inject`` or error-injection debugfs interfaces,
+  where panicking on the unrecoverable path makes regressions
+  immediately visible instead of surfacing as later, unrelated
+  failures.
+
+= =====================================================================
+0 Try to continue operation (default).
+1 Panic immediately.  If the ``panic`` sysctl is also non-zero then the
+  machine will be rebooted.
+= =====================================================================
+
+Example::
+
+     echo 1 > /proc/sys/vm/panic_on_unrecoverable_memory_failure
+
+
 percpu_pagelist_high_fraction
 =============================
 
diff --git a/Documentation/admin-guide/thunderbolt.rst b/Documentation/admin-guide/thunderbolt.rst
index 91a6cb1099889..ff25fe8537061 100644
--- a/Documentation/admin-guide/thunderbolt.rst
+++ b/Documentation/admin-guide/thunderbolt.rst
@@ -294,8 +294,8 @@ for the retimers::
 
 This enumerates and adds the on-board retimers. Now retimer NVM can be
 upgraded in the same way than with cable connected (see previous
-section). However, the retimer is not disconnected as we are offline
-mode) so after writing ``1`` to ``nvm_authenticate`` one should wait for
+section). However, the retimer is not disconnected as we are in offline
+mode, so after writing ``1`` to ``nvm_authenticate`` one should wait for
 5 or more seconds before running rescan again::
 
   # echo 1 > /sys/bus/thunderbolt/devices/0-0/usb4_port1/rescan
diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst
index 014aa1c215a16..b0c68b64f5ac2 100644
--- a/Documentation/arch/arm64/silicon-errata.rst
+++ b/Documentation/arch/arm64/silicon-errata.rst
@@ -352,6 +352,7 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | Qualcomm Tech. | Kryo4xx Gold    | N/A             | ARM64_ERRATUM_1286807       |
 +----------------+-----------------+-----------------+-----------------------------+
+| Renesas        | S4/V4H/V4M      | N/A             | RENESAS_ERRATUM_GEN4GICITS1 |
 +----------------+-----------------+-----------------+-----------------------------+
 | Rockchip       | RK3588          | #3588001        | ROCKCHIP_ERRATUM_3588001    |
 +----------------+-----------------+-----------------+-----------------------------+
diff --git a/Documentation/arch/x86/amd-memory-encryption.rst b/Documentation/arch/x86/amd-memory-encryption.rst
index bd840df708ea4..92edb26a50d28 100644
--- a/Documentation/arch/x86/amd-memory-encryption.rst
+++ b/Documentation/arch/x86/amd-memory-encryption.rst
@@ -53,7 +53,7 @@ CPUID function 0x8000001f reports information related to SME::
 			   system physical addresses, not guest physical
 			   addresses)
 
-If support for SME is present, MSR 0xc00100010 (MSR_AMD64_SYSCFG) can be used to
+If support for SME is present, MSR 0xc0010010 (MSR_AMD64_SYSCFG) can be used to
 determine if SME is enabled and/or to enable memory encryption::
 
 	0xc0010010:
diff --git a/Documentation/block/biovecs.rst b/Documentation/block/biovecs.rst
index 11126ed6f40f7..b9dc0c9dbee44 100644
--- a/Documentation/block/biovecs.rst
+++ b/Documentation/block/biovecs.rst
@@ -135,6 +135,7 @@ Usage of helpers:
 	bio_first_bvec_all()
 	bio_first_page_all()
 	bio_first_folio_all()
+	bio_last_bvec_all()
 
 * The following helpers iterate over single-page segment. The passed 'struct
   bio_vec' will contain a single-page IO vector during the iteration::
diff --git a/Documentation/bpf/bpf_devel_QA.rst b/Documentation/bpf/bpf_devel_QA.rst
index 45bc5c5cd7937..edf8107a7beba 100644
--- a/Documentation/bpf/bpf_devel_QA.rst
+++ b/Documentation/bpf/bpf_devel_QA.rst
@@ -479,7 +479,10 @@ for details.
 
 To maximize the number of tests passing, the .config of the kernel
 under test should match the config file fragment in
-tools/testing/selftests/bpf as closely as possible.
+tools/testing/selftests/bpf as closely as possible. If not possible,
+however, you can set ``BPF_STRICT_BUILD=0`` when invoking ``make``
+to tolerate individual compilation failures and continue building
+the remaining tests rather than treating each failure as fatal.
 
 Finally to ensure support for latest BPF Type Format features -
 discussed in Documentation/bpf/btf.rst - pahole version 1.16
diff --git a/Documentation/bpf/index.rst b/Documentation/bpf/index.rst
index 0d5c6f6592667..638a00d42bc2a 100644
--- a/Documentation/bpf/index.rst
+++ b/Documentation/bpf/index.rst
@@ -28,6 +28,7 @@ that goes into great technical depth about the BPF Architecture.
    classic_vs_extended.rst
    bpf_iterators
    bpf_licensing
+   signing
    test_debug
    clang-notes
    linux-notes
diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst
index 4c814ff6061e0..c801a330aece0 100644
--- a/Documentation/bpf/kfuncs.rst
+++ b/Documentation/bpf/kfuncs.rst
@@ -273,22 +273,29 @@ flags on a set of kfuncs as follows::
         BTF_KFUNCS_END(bpf_task_set)
 
 This set encodes the BTF ID of each kfunc listed above, and encodes the flags
-along with it. Ofcourse, it is also allowed to specify no flags.
+along with it. It is also allowed to specify no flags.
 
 kfunc definitions should also always be annotated with the ``__bpf_kfunc``
-macro. This prevents issues such as the compiler inlining the kfunc if it's a
-static kernel function, or the function being elided in an LTO build as it's
-not used in the rest of the kernel. Developers should not manually add
-annotations to their kfunc to prevent these issues. If an annotation is
-required to prevent such an issue with your kfunc, it is a bug and should be
-added to the definition of the macro so that other kfuncs are similarly
-protected. An example is given below::
+macro. This prevents issues such as the compiler inlining the kfunc, or the
+function being elided in an LTO build as it's not used in the rest of the
+kernel. Developers should not manually add annotations to their kfunc to prevent
+these issues. If an annotation is required to prevent such an issue with your
+kfunc, it is a bug and should be added to the definition of the macro so that
+other kfuncs are similarly protected. An example is given below::
 
         __bpf_kfunc struct task_struct *bpf_get_task_pid(s32 pid)
         {
         ...
         }
 
+Note that kfuncs must not be declared ``static``. A kfunc can be called from a
+BPF program ``*.c`` file outside the compilation unit that defines it, so its
+externally visible name must remain available for BTF ID lookup. ``static``
+linkage allows the compiler to rename the function, which can break this
+BTF-based kfunc resolution. Further note that sparse may warn that an otherwise
+unreferenced kfunc should be static. Such warnings should be ignored for kfunc
+definitions.
+
 2.5.1 KF_ACQUIRE flag
 ---------------------
 
diff --git a/Documentation/bpf/signing.rst b/Documentation/bpf/signing.rst
new file mode 100644
index 0000000000000..e73eaaebd8b15
--- /dev/null
+++ b/Documentation/bpf/signing.rst
@@ -0,0 +1,497 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+============
+BPF signing
+============
+
+This document describes how BPF programs are cryptographically signed, how the
+kernel verifies them at load time, and how Linux Security Modules (LSMs) -
+including the BPF LSM - use the resulting verdict to enforce policy. It is
+written for developers who want to produce signed BPF objects, understand what
+the signature actually guarantees, or build a policy on top of it.
+
+Motivation
+==========
+
+A signed BPF program lets the kernel establish that the bytecode being loaded
+originates from a trusted producer and was not modified in transit. On its own
+the kernel does not *require* signatures - an unsigned program loads exactly as
+before - but it records a verdict (see `The verdict`_) that an LSM can gate on.
+This is the building block for policies such as "only run BPF that was signed by
+a key in the trusted keyring", as could in the future be enforced by an LSM
+such as IPE.
+
+Signing is orthogonal to the existing permission model: it does not replace the
+capability checks or the verifier. A signed load still requires the usual
+privileges (``CAP_BPF`` and any program-type-specific capability, subject to
+``kernel.unprivileged_bpf_disabled``), and the loader's instructions are still
+checked by the verifier like any other program. A valid signature establishes
+*origin and integrity*, not safety - it lets a policy trust where the bytecode
+came from, it does not let a load skip any check it would otherwise face.
+
+The hard part is *what* gets signed. A naive scheme would sign a program's
+instruction buffer at build time and verify that signature at
+``BPF_PROG_LOAD``. That does not survive contact with real BPF objects, because
+the bytes the kernel finally loads are not the bytes the developer built and
+signed. Between the two, libbpf and the kernel rewrite the program:
+
+- **map file descriptors** are patched into ``ld_imm64`` instructions
+  (``BPF_PSEUDO_MAP_FD``), and a map's fd is assigned at load time, so it
+  differs on every run;
+- **CO-RE relocations** rewrite field offsets, sizes and existence flags against
+  the *running* kernel's BTF, so the result differs from one kernel to the next;
+- **kfunc and ksym references** are resolved to ids/addresses in the running
+  kernel;
+- **global data** (``.rodata``/``.data``/``.bss``) is created and seeded as maps
+  at load.
+
+So a signature over the original instructions cannot match the relocated
+instructions the verifier ends up checking, and the relocated form cannot be
+produced ahead of time because it depends on the target kernel. There is no
+fixed byte string that is both signable at build time and what the kernel
+actually loads - which is why a program cannot simply be signed and loaded
+directly.
+
+The trusted loader
+==================
+
+The solution is to move that setup work *into* a small BPF program - the
+**loader** - and sign the loader instead of the individual programs. libbpf's
+``gen_loader`` machinery (``bpftool gen skeleton -L``, the "light skeleton")
+emits a ``BPF_PROG_TYPE_SYSCALL`` program whose body performs the bpf() syscalls
+that create maps, apply relocations, and load the real programs. The payload it
+installs - the serialized programs, map descriptions, relocation data and
+initial values - lives in a separate array map, the **metadata map**
+(``__loader.map``).
+
+So the unit of trust is the loader, and the signing contract is::
+
+    Sig(I_loader || D_meta)
+
+where ``I_loader`` is the loader's instruction stream and ``D_meta`` is the
+content of the metadata map. Verifying the loader's signature establishes that
+both the loader *and* the payload it is about to install are authentic. The
+loader is reproducible: ``gen_loader`` builds it from primitives so the same
+object yields the same bytes on any build host.
+
+Why the loader is signable when the program is not
+--------------------------------------------------
+
+The loader sidesteps every rewrite listed above, because the bytes that are
+signed are *relocation-invariant*:
+
+- The loader's own instructions are a fixed sequence of bpf() syscalls emitted
+  by ``gen_loader``; they carry no CO-RE relocations and resolve no ksyms, so
+  they are identical on every kernel. The metadata map is referenced by *index*
+  into ``fd_array`` (``BPF_PSEUDO_MAP_IDX_VALUE``), not by a baked-in file
+  descriptor, so even that reference does not change between build and load.
+  The loader instruction bytes the kernel verifies are exactly the bytes that
+  were signed.
+- The metadata map is opaque, frozen data - the serialized target programs,
+  their relocation records, map descriptions and initial values. Its bytes are
+  identical at build time and at load time, so they are simply appended to the
+  instructions and covered by the same signature (there is no separate metadata
+  hash to compute or compare).
+
+All the host-specific rewriting - creating maps, patching their fds into the
+target programs, applying CO-RE, resolving ksyms, seeding global data - still
+happens, but it happens *inside the loader at runtime*, on the verified
+metadata, **after** the kernel has verified the ``insns || metadata`` signature.
+The kernel never has to verify the relocated target programs: it verifies the
+loader and its inputs once, and trust transfers to whatever that now-trusted,
+deterministic loader installs. The relocation step is moved from "before the
+signature can be checked" to "after a trusted program runs" - which is exactly
+what makes it signable.
+
+Because the metadata map is the loader's only untrusted input, two existing map
+properties are reused to keep it trustworthy across the load:
+
+Exclusive maps
+    A map created with ``excl_prog_hash`` (see ``BPF_MAP_CREATE``) may only be
+    accessed by a program whose digest matches that hash. The verifier enforces
+    ``map->excl_prog_sha == prog->digest`` for every map a program uses, so the
+    metadata map is bound to exactly the signed loader and cannot be shared with
+    or mutated by another program.
+
+Frozen maps
+    The metadata map is frozen (``BPF_MAP_FREEZE``) before the loader is loaded.
+    Freezing blocks further userspace writes, so the bytes folded into the
+    signature cannot change before the loader runs. (Freezing does not make the
+    map read-only to the loader program itself, which still writes created file
+    descriptors back into the blob's scratch area.)
+
+Load-time verification
+=======================
+
+Rather than have the loader check its own metadata from within BPF, the kernel
+verifies it directly at ``BPF_PROG_LOAD``, with no new UAPI. The mechanism
+reuses the existing ``fd_array``:
+
+#. Userspace creates the metadata map with ``excl_prog_hash`` set to the
+   loader's digest, populates it, and freezes it.
+#. The loader is loaded with ``signature``/``signature_size``/``keyring_id``
+   set, the metadata map referenced through ``fd_array``, and ``fd_array_cnt``
+   set so the kernel knows the array's length.
+#. Signature verification runs inside the verifier (``bpf_check()``), once it
+   has resolved the ``fd_array`` entries into the program's ``used_maps``. The
+   maps folded into the signature are therefore the very objects the program
+   binds - a single resolution of ``fd_array``, not a separate read, so the
+   verified bytes cannot be swapped for a different map after the check (no
+   time-of-check/time-of-use window). Each folded map must be exclusive (carry
+   ``excl_prog_sha``) and a plain array map (``BPF_MAP_TYPE_ARRAY``); only an
+   array map exposes its value buffer through ``map_direct_value_addr()`` as a
+   kernel address spanning ``value_size`` bytes. A map that is not exclusive, not
+   frozen, or not a plain array is rejected, with a verifier log message naming
+   the offending map. The kernel appends each map's frozen
+   contents to the instruction buffer and verifies the PKCS#7 signature over the
+   concatenation ``insns || metadata_0 || metadata_1 || ...`` in ``used_maps``
+   order, before it rewrites the (signed) instructions.
+
+A signed program therefore takes one of exactly two shapes, both fully
+supported:
+
+- **No bound maps** (``fd_array_cnt == 0``): there is nothing to append, so the
+  kernel verifies the signature over the instructions alone. A valid signature
+  yields ``BPF_SIG_VERIFIED`` and the program loads. This is the ordinary case
+  for a directly-loaded signed program with no separate payload; it is *not*
+  rejected for "missing" metadata, because it has none to cover.
+- **Exclusive bound maps** (``fd_array_cnt > 0``): every entry is exclusive and
+  folded, so the signature covers ``insns || metadata``.
+
+There is no third shape: a non-exclusive map in a signed program's ``fd_array``
+is rejected rather than silently left out of the signature, so a signed loader
+never binds a map its signature does not cover.
+
+The digest binding (``excl_prog_sha == prog->digest``) is enforced by the
+verifier as usual; because that check runs while ``fd_array`` is resolved -
+before the verifier would otherwise compute the tag - ``prog->digest`` is
+computed up front in the verifier, over the unmodified (signature-covered)
+instructions, for any signed load.
+
+Coverage is then enforced as the verifier resolves instructions, at the point
+each object is bound rather than by a count taken afterwards. Once the signature
+has been verified, binding any further map is refused: a map reached by a
+directly-referenced fd, or a map swapped into an ``fd_array`` slot the loader
+reads, is not among those already folded, so it is rejected the moment the
+verifier tries to bind it. A BTF is refused outright for a signed program - a
+ksym or a BTF fd in ``fd_array``, whether resolved up front or lazily for a
+module kfunc, is rejected when it would be bound. Together with the fold rule
+above this keeps the verdict binary: a signed program cannot use a map its
+signature does not cover, and a different but equally digest-bound map cannot be
+substituted at an ``fd_array`` slot. Non-exclusive maps are never folded, so a
+signed program cannot use one at all.
+
+The verdict
+===========
+
+A program is either unsigned or fully verified - there is no intermediate
+state. The outcome is recorded in ``prog->aux->sig.verdict``:
+
+.. code-block:: c
+
+    enum bpf_sig_verdict {
+            BPF_SIG_UNSIGNED = 0,
+            BPF_SIG_VERIFIED,
+    };
+
+``BPF_SIG_VERIFIED`` means the signature is valid and covers the instructions
+*and* the frozen contents of every exclusive map the program uses:
+
+- For an ordinary, directly-loaded signed program the instructions are the whole
+  artifact and it uses no exclusive maps, so a valid instruction signature is
+  the complete verification.
+- For a signed loader the metadata map is exclusive, so its contents are folded
+  in and the signature covers ``insns || metadata``.
+
+There is deliberately no "instructions verified but metadata not" verdict: a
+signed loader that fails to cover its metadata is *rejected* (see above), not
+recorded with a weaker verdict. ``BPF_SIG_VERIFIED`` therefore always means the
+program and everything the signature is responsible for are authentic, which is
+what a policy can rely on.
+
+Alongside the verdict the kernel records which keyring validated the signature;
+see `Keyrings`_.
+
+Enforcement via LSMs
+====================
+
+Signing only *records* a verdict; an LSM turns it into policy. The verdict and
+keyring fields live in ``struct bpf_prog_aux``, so a BPF LSM program can read
+them directly (see Documentation/bpf/prog_lsm.rst for writing and attaching BPF
+LSM programs); the same fields are equally available to in-tree LSMs. Two hooks
+are useful at different points of the load: the dedicated
+``security_bpf_prog_load()`` gates admission before the main verification work,
+and the existing ``security_bpf_prog()`` observes a program that has fully
+loaded.
+
+Admission: ``security_bpf_prog_load()``
+---------------------------------------
+
+This hook gates admission **for every load**, from a single call site inside the
+verifier (``bpf_check()``), before the main verification work. It runs after the
+optional signature verification, so the verdict and keyring fields are final - the
+hook can see whether, and how strongly, the program was signed, which keyring
+validated it, the load ``attr``, the BPF token and whether the load came from the
+kernel. For a signed load the verdict is ``BPF_SIG_VERIFIED`` here (the signature
+has just been checked); for an unsigned load it is ``BPF_SIG_UNSIGNED``.
+
+This is the place for *coarse admission* that must also see unsigned and
+not-yet-verified loads: require a signature at all, restrict the acceptable
+keyring, restrict which token/credentials may load BPF, apply per-program-type
+rules, or audit every load attempt that makes it past signature verification -
+attempts failing the signature or the metadata binding abort before this hook
+fires. It is the primary deny point.
+
+One subtlety: this hook runs *before* the verifier finishes its work, so
+``BPF_SIG_VERIFIED`` *here* means only "validly signed" - not "loaded". Allowing
+a load at this point lets it *proceed*; it does not guarantee the program will
+load. A validly signed program can still be rejected afterwards on two
+independent grounds: the verifier may reject it like any other program (unsafe
+memory access, bad control flow, resource limits, ...), and the kernel separately
+refuses - as the verifier resolves instructions and binds each object - any map
+the signature does not cover or any BTF at all, regardless of what this hook
+returned. Only after the program has fully loaded, at the next hook
+(``security_bpf_prog()``), does ``BPF_SIG_VERIFIED`` carry its full meaning:
+validly signed *and* fully verified.
+
+A more realistic admission policy than "is it signed at all": accept programs
+signed by a system keyring, accept a user-keyring signature only if the
+key/keyring it was verified against is on an explicit allowlist, and emit a
+tamper-evident record of every decision so that even denied attempts are
+auditable. (Illustrative - error checking elided.)
+
+.. code-block:: c
+
+    /* Serials of user keys/keyrings we additionally trust. */
+    struct {
+            __uint(type, BPF_MAP_TYPE_HASH);
+            __type(key, __s32);             /* keyring_serial */
+            __type(value, __u8);
+            __uint(max_entries, 64);
+    } trusted_user_keys SEC(".maps");
+
+    /* Audit stream consumed by a userspace logger. */
+    struct {
+            __uint(type, BPF_MAP_TYPE_RINGBUF);
+            __uint(max_entries, 1 << 16);
+    } audit SEC(".maps");
+
+    struct decision { __u32 prog_type, verdict, ktype; __s32 serial, ret; };
+
+    SEC("lsm/bpf_prog_load")
+    int BPF_PROG(admit, struct bpf_prog *prog, union bpf_attr *attr,
+                 struct bpf_token *token, bool kernel)
+    {
+            __u32 verdict = prog->aux->sig.verdict;
+            __u32 ktype   = prog->aux->sig.keyring_type;
+            __s32 serial  = prog->aux->sig.keyring_serial;
+            struct decision *d;
+            int ret = 0;
+
+            if (kernel)
+                    return 0;                       /* trust in-kernel loads */
+
+            if (verdict != BPF_SIG_VERIFIED)
+                    ret = -EPERM;                   /* must be validly signed */
+            else if (ktype == BPF_SIG_KEYRING_USER &&
+                     !bpf_map_lookup_elem(&trusted_user_keys, &serial))
+                    ret = -EPERM;                   /* key/keyring not allowlisted */
+
+            d = bpf_ringbuf_reserve(&audit, sizeof(*d), 0);
+            if (d) {
+                    d->prog_type = attr->prog_type;
+                    d->verdict = verdict;
+                    d->ktype = ktype;
+                    d->serial = serial;
+                    d->ret = ret;
+                    bpf_ringbuf_submit(d, 0);       /* record allow *and* deny */
+            }
+            return ret;
+    }
+
+Observing a verified load: ``security_bpf_prog()``
+--------------------------------------------------
+
+There is deliberately no separate "metadata attested" hook. The coverage check
+above is enforced by the kernel unconditionally, so a signed loader that fails
+to cover its metadata never loads and an LSM never has to re-establish that
+fact. To *act on* a program that has successfully and fully loaded, use the
+existing ``security_bpf_prog()`` hook (``lsm/bpf_prog``), which fires from
+``bpf_prog_new_fd()`` - after the verifier, after the coverage check, and after
+``bpf_prog_alloc_id()``. Relative to the admission hook this point is strictly
+later and stronger:
+
+- the program has an id (``prog->aux->id``), so it can be recorded or correlated
+  with later events;
+- ``verdict == BPF_SIG_VERIFIED`` *here* means **fully** verified - a program
+  that used a map the signature does not cover was already rejected, so it cannot
+  reach this point;
+- it observes only programs that actually loaded; a failed load never mints an
+  fd, so it never reaches this hook.
+
+It takes only the ``prog`` and a non-zero return still aborts (the fd is not
+handed out), so it can veto as well as observe. One wrinkle: it also fires on
+other paths that mint a new program fd - notably ``bpf_prog_get_fd_by_id()`` -
+not just on a fresh load. Because the program already has its id here, an LSM
+can tell the two apart with a small hash map: the *first* time an id is seen is
+the load; a later sighting of the same id is just another fd to a program that
+already exists.
+
+To bound the map and let a reused id read as a fresh load, this can be paired
+with ``security_bpf_prog_free()`` (``lsm/bpf_prog_free``), which deletes the
+entry on teardown - keyed by the same ``prog`` pointer, since
+``bpf_prog_free_id()`` has already cleared ``prog->aux->id`` to ``0`` by the time
+that hook runs. (Illustrative - privileged LSM, error checking elided.)
+
+.. code-block:: c
+
+    struct rec { __u32 id, ktype; __s32 serial; };
+
+    struct {
+            __uint(type, BPF_MAP_TYPE_HASH);
+            __type(key, __u64);             /* struct bpf_prog * -- stable id */
+            __type(value, struct rec);
+            __uint(max_entries, 4096);
+    } live SEC(".maps");
+
+    SEC("lsm/bpf_prog")            /* fires after load and on every later fd */
+    int BPF_PROG(observe, struct bpf_prog *prog)
+    {
+            __u64 key = (__u64)(unsigned long)prog;
+            struct rec r;
+
+            if (prog->aux->sig.verdict != BPF_SIG_VERIFIED)
+                    return 0;
+            if (bpf_map_lookup_elem(&live, &key))
+                    return 0;               /* seen before: a later fd, not a load */
+
+            /* First sighting == this program just loaded; id is valid here. */
+            r.id     = prog->aux->id;
+            r.ktype  = prog->aux->sig.keyring_type;
+            r.serial = prog->aux->sig.keyring_serial;
+            bpf_map_update_elem(&live, &key, &r, BPF_NOEXIST);
+            /* ... newly-loaded verified-program action, e.g. record r.id ... */
+            return 0;
+    }
+
+Putting them together: to *require* verified BPF, deny at the admission hook
+unless the verdict is ``BPF_SIG_VERIFIED`` (and, if desired, restrict the
+keyring). The kernel then guarantees that any program which actually loads with
+that verdict covered all of its exclusive maps, rejecting any that did not - so
+a deny-by-default admission policy needs no second enforcement point. Use
+``security_bpf_prog()`` to record or finally gate the verified programs once
+they carry an id. The ``verdict``, ``keyring_type`` and ``keyring_serial`` fields
+let a policy distinguish, for example, "verified and signed by a builtin key"
+from "verified by a user key". A policy LSM such as IPE could consume the same
+hooks to enforce system policy without writing any BPF, though none implements
+this today.
+
+Keyrings
+========
+
+``keyring_id`` selects the trusted keyring the PKCS#7 signature is verified
+against. The well-known ids ``0`` (builtin), ``VERIFY_USE_SECONDARY_KEYRING``
+and ``VERIFY_USE_PLATFORM_KEYRING`` select the corresponding system keyrings;
+any other value is treated as the serial of a user/session key or keyring.
+The keyring is looked up first, before the signature bytes are examined, so a
+signature naming a non-existent keyring is rejected up front, and a failed
+verification aborts the load - so a program that loads successfully with a
+signature always has consistent keyring fields recorded.
+
+Two fields are recorded in ``prog->aux->sig`` for an LSM to inspect:
+
+``keyring_type`` (``enum bpf_sig_keyring``)
+    Classified purely from ``keyring_id`` whenever the program is signed:
+    ``BPF_SIG_KEYRING_BUILTIN``, ``_SECONDARY``, ``_PLATFORM`` for the system
+    keyrings, or ``_USER`` for a user/session keyring. It is
+    ``BPF_SIG_KEYRING_NONE`` for an unsigned program.
+
+``keyring_serial`` (``s32``)
+    Set **only** on a successful verification, to the serial of the
+    **user/session key or keyring** that ``keyring_id`` resolved to - the
+    object the signature was verified against, not the individual asymmetric
+    key inside it that matched the signer. Passing
+    ``KEY_SPEC_SESSION_KEYRING``, for example, records the session keyring's
+    serial. The system keyrings are trusted as a whole and expose no serial
+    here, so the serial is ``0`` for builtin, secondary and platform
+    signatures, and ``0`` for unsigned programs. In other words, a non-zero
+    ``keyring_serial`` is exactly "verified against the user key/keyring with
+    this serial".
+
+.. list-table::
+   :header-rows: 1
+
+   * - ``keyring_id``
+     - ``keyring_type``
+     - ``keyring_serial``
+   * - (no signature)
+     - ``BPF_SIG_KEYRING_NONE``
+     - ``0``
+   * - ``0``
+     - ``BPF_SIG_KEYRING_BUILTIN``
+     - ``0``
+   * - ``VERIFY_USE_SECONDARY_KEYRING``
+     - ``BPF_SIG_KEYRING_SECONDARY``
+     - ``0``
+   * - ``VERIFY_USE_PLATFORM_KEYRING``
+     - ``BPF_SIG_KEYRING_PLATFORM``
+     - ``0``
+   * - other (a user/session key serial)
+     - ``BPF_SIG_KEYRING_USER``
+     - serial of the resolved key/keyring
+
+Producing a signed object
+==========================
+
+``bpftool`` generates and signs a light skeleton in one step::
+
+    bpftool gen skeleton -L -S -k <private_key.pem> -i <certificate.x509> \
+            obj.bpf.o > obj.lskel.h
+
+``-L`` selects the light-skeleton (``gen_loader``) backend and ``-S`` enables
+signing; ``-k`` and ``-i`` supply the signing key and its X.509 certificate.
+``bpftool`` signs ``insns || metadata`` - the exact bytes the kernel
+reconstructs - and also computes ``excl_prog_hash`` as the digest of the loader
+instructions so the metadata map can be bound to the loader. The signature and
+hash are embedded in the generated header; the certificate is used only for
+signing and is not included. Loading the skeleton performs the
+create/populate/freeze/load sequence described above.
+
+At runtime the trusted public key must be present in the chosen keyring (for
+example added to the session keyring, or built into the kernel's builtin trusted
+keyring) for verification to succeed.
+
+UAPI reference
+==============
+
+``BPF_PROG_LOAD`` (``union bpf_attr``):
+
+``signature``, ``signature_size``
+    Pointer to and length of the PKCS#7 signature blob.
+
+``keyring_id``
+    Trusted keyring selector (see `Keyrings`_).
+
+``fd_array``, ``fd_array_cnt``
+    Array of map (and module BTF) file descriptors bound to the program.
+    ``fd_array_cnt`` must be set for the kernel to scan the array. When a
+    signature is present, a BTF entry is rejected outright, and every map must
+    be exclusive; its frozen contents are folded into the verified buffer, and
+    a non-exclusive entry is rejected.
+
+``BPF_MAP_CREATE`` (``union bpf_attr``):
+
+``excl_prog_hash``, ``excl_prog_hash_size``
+    SHA-256 digest of the program permitted to access this (exclusive) map. This
+    binds the metadata map to the loader; it is not a hash of the map *content*.
+    The map content is not hashed separately at all - it is covered, as bytes,
+    by the program signature.
+
+Notes and limitations
+======================
+
+- The instructions plus folded metadata are verified as one ``bpf_dynptr``,
+  which bounds the combined size (currently ~16 MiB); very large objects can
+  exceed it.
+- The metadata container is a single-element array map, accessed through
+  ``map_direct_value_addr``.
diff --git a/Documentation/conf.py b/Documentation/conf.py
index 9b822ab470d9d..3f82cfbff778e 100644
--- a/Documentation/conf.py
+++ b/Documentation/conf.py
@@ -61,12 +61,12 @@ manpages_url = 'https://man7.org/linux/man-pages/man{section}/{page}.{section}.h
 
 def config_init(app, config):
     """
-    Initialize path-dependent variabled
+    Initialize path-dependent variables
 
     On Sphinx, all directories are relative to what it is passed as
     SOURCEDIR parameter for sphinx-build. Due to that, all patterns
     that have directory names on it need to be dynamically set, after
-    converting them to a relative patch.
+    converting them to a relative path.
 
     As Sphinx doesn't include any patterns outside SOURCEDIR, we should
     exclude relative patterns that start with "../".
diff --git a/Documentation/core-api/SMP.rst b/Documentation/core-api/SMP.rst
new file mode 100644
index 0000000000000..0265a9835f23b
--- /dev/null
+++ b/Documentation/core-api/SMP.rst
@@ -0,0 +1,11 @@
+.. SPDX-License-Identifier: GPL-2.0+
+
+==============
+SMP primitives
+==============
+
+.. kernel-doc:: include/linux/smp.h
+   :internal:
+
+.. kernel-doc:: kernel/smp.c
+   :export:
diff --git a/Documentation/core-api/dma-api.rst b/Documentation/core-api/dma-api.rst
index ca75b35416792..ba23a472f7948 100644
--- a/Documentation/core-api/dma-api.rst
+++ b/Documentation/core-api/dma-api.rst
@@ -508,7 +508,7 @@ call to dma_iova_try_alloc.  This can be useful in the unmap path.
 
 Is used to link ranges to the IOVA previously allocated.  The start of all
 but the first call to dma_iova_link for a given state must be aligned
-to the DMA merge boundary returned by ``dma_get_merge_boundary())``, and
+to the DMA merge boundary returned by ``dma_get_merge_boundary()``, and
 the size of all but the last range must be aligned to the DMA merge boundary
 as well.
 
diff --git a/Documentation/core-api/entry.rst b/Documentation/core-api/entry.rst
index 71d8eedc05493..79fdaed954d9d 100644
--- a/Documentation/core-api/entry.rst
+++ b/Documentation/core-api/entry.rst
@@ -58,32 +58,59 @@ state transitions must run with interrupts disabled.
 Syscalls
 --------
 
-Syscall-entry code starts in assembly code and calls out into low-level C code
-after establishing low-level architecture-specific state and stack frames. This
-low-level C code must not be instrumented. A typical syscall handling function
-invoked from low-level assembly code looks like this:
+Syscall-entry code starts in assembly code and calls out into low-level C
+code after establishing low-level architecture-specific state and stack
+frames. This low-level C code must not be instrumented. The recommended
+syscall handling function invoked from low-level assembly code looks like
+this:
 
 .. code-block:: c
 
-  noinstr void syscall(struct pt_regs *regs, int nr)
+  noinstr void syscall(struct pt_regs *regs, long nr)
   {
 	arch_syscall_enter(regs);
-	nr = syscall_enter_from_user_mode(regs, nr);
+	result_reg(regs) = -ENOSYS;
+	if (syscall_enter_from_user_mode_randomize_stack(regs, &nr)) {
+		instrumentation_begin();
+		if (valid(nr)
+			result_reg(regs) = invoke_syscall(regs, nr);
+		instrumentation_end();
+	}
+	syscall_exit_to_user_mode(regs);
+  }
 
-	instrumentation_begin();
-	if (!invoke_syscall(regs, nr) && nr != -1)
-	 	result_reg(regs) = __sys_ni_syscall(regs);
-	instrumentation_end();
+This is the most resilent variant as it has always a guaranteed valid
+return code. The alternative variant is:
 
+.. code-block:: c
+
+  noinstr void syscall(struct pt_regs *regs, long nr)
+  {
+	arch_syscall_enter(regs);
+	if (syscall_enter_from_user_mode_randomize_stack(regs, &nr)) {
+		instrumentation_begin();
+		if (valid(nr)
+			result_reg(regs) = invoke_syscall(regs, nr);
+		else
+			result_reg(regs) = -ENOSYS;
+		instrumentation_end();
+	}
 	syscall_exit_to_user_mode(regs);
   }
 
-syscall_enter_from_user_mode() first invokes enter_from_user_mode() which
-establishes state in the following order:
+That works for most situations except when a probe/BPF attached to the
+syscall tracepoint sets an invalid syscall number e.g. -1 and also modifies
+the result register. So this variant will obviously overwrite the modified
+result with -ENOSYS.
+
+syscall_enter_from_user_mode_randomize_stack() first invokes
+enter_from_user_mode_randomize_stack() which establishes state in the
+following order:
 
   * Lockdep
   * RCU / Context tracking
   * Tracing
+  * Apply stack randomization
 
 and then invokes the various entry work functions like ptrace, seccomp, audit,
 syscall tracing, etc. After all that is done, the instrumentable invoke_syscall
@@ -99,10 +126,11 @@ transition in the reverse order:
   * RCU / Context tracking
   * Lockdep
 
-syscall_enter_from_user_mode() and syscall_exit_to_user_mode() are also
-available as fine grained subfunctions in cases where the architecture code
-has to do extra work between the various steps. In such cases it has to
-ensure that enter_from_user_mode() is called first on entry and
+syscall_enter_from_user_mode_randomize_stack() and
+syscall_exit_to_user_mode() are also available as fine grained subfunctions
+in cases where the architecture code has to do extra work between the
+various steps. In such cases it has to ensure that
+enter_from_user_mode_randomize_stack() is called first on entry and
 exit_to_user_mode() is called last on exit.
 
 Do not nest syscalls. Nested syscalls will cause RCU and/or context tracking
diff --git a/Documentation/core-api/errseq.rst b/Documentation/core-api/errseq.rst
index ff332e272405e..d298d4cd2f60d 100644
--- a/Documentation/core-api/errseq.rst
+++ b/Documentation/core-api/errseq.rst
@@ -143,7 +143,7 @@ Because of this, it's often advantageous to first do an errseq_check to
 see if anything has changed, and only later do an
 errseq_check_and_advance after taking the lock. e.g.::
 
-        if (errseq_check(&wd.wd_err, READ_ONCE(su.s_wd_err)) {
+        if (errseq_check(&wd.wd_err, READ_ONCE(su.s_wd_err))) {
                 /* su.s_wd_err is protected by s_wd_err_lock */
                 spin_lock(&su.s_wd_err_lock);
                 err = errseq_check_and_advance(&wd.wd_err, &su.s_wd_err);
diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst
index 13769d5c40bf2..92f91c6a0d79d 100644
--- a/Documentation/core-api/index.rst
+++ b/Documentation/core-api/index.rst
@@ -81,6 +81,7 @@ Documentation/locking/index.rst for more related documentation.
    padata
    ../RCU/index
    wrappers/memory-barriers.rst
+   SMP
 
 Low-level hardware management
 =============================
diff --git a/Documentation/core-api/list.rst b/Documentation/core-api/list.rst
index 479aa91cc3958..df8b078bb366a 100644
--- a/Documentation/core-api/list.rst
+++ b/Documentation/core-api/list.rst
@@ -458,7 +458,7 @@ The list_move() and list_move_tail() functions can be used to move an entry
 from one list to another, to either the start or end respectively.
 
 In the following example, we'll assume we start with two lists ("clowns" and
-"sidewalk" in the following initial state "State 0"::
+"sidewalk") in the following initial state "State 0"::
 
          .----------------------------------------------------------------.
          v                                                                |
diff --git a/Documentation/core-api/maple_tree.rst b/Documentation/core-api/maple_tree.rst
index ccdd1615cf974..1eae24a9c18b4 100644
--- a/Documentation/core-api/maple_tree.rst
+++ b/Documentation/core-api/maple_tree.rst
@@ -17,7 +17,8 @@ supports iterating over a range of entries and going to the previous or next
 entry in a cache-efficient manner.  The tree can also be put into an RCU-safe
 mode of operation which allows reading and writing concurrently.  Writers must
 synchronize on a lock, which can be the default spinlock, or the user can set
-the lock to an external lock of a different type.
+the lock to an external lock of a different type.  Note that external locks may
+interfere with allocations in a low memory situation.
 
 The Maple Tree maintains a small memory footprint and was designed to use
 modern processor cache efficiently.  The majority of the users will be able to
@@ -42,6 +43,15 @@ successful store operation within a given
 code segment when allocating cannot be done.  Allocations of nodes are
 relatively small at around 256 bytes.
 
+Since the maple tree uses internal nodes that are allocated and has rules on
+data density, erasing an entry may cause allocations to occur.  That is,
+erasing an entry may consume memory.  Users must take care to ensure that they
+do not violate the larger system constraints on when and how memory is
+allocated.  Most situations are fine to allocate, but the pre-allocation
+support is provided as a mechanism to avoid trickier situations.  There is also
+the possibility of using special entries and clean up the tree later, in
+extreme circumstances.
+
 .. _maple-tree-normal-api:
 
 Normal API
@@ -63,7 +73,8 @@ success or an error code otherwise.  mtree_store_range() works in the same way
 but takes a range.  mtree_load() is used to retrieve the entry stored at a
 given index.  You can use mtree_erase() to erase an entire range by only
 knowing one value within that range, or mtree_store() call with an entry of
-NULL may be used to partially erase a range or many ranges at once.
+NULL may be used to partially erase a range or many ranges at once.  Note that
+mtree_erase() may use GFP_KERNEL on allocations.
 
 If you want to only store a new entry to a range (or index) if that range is
 currently ``NULL``, you can use mtree_insert_range() or mtree_insert() which
@@ -163,7 +174,9 @@ You can use mas_erase() to erase an entire range by setting index and
 last of the maple state to the desired range to erase.  This will erase
 the first range that is found in that range, set the maple state index
 and last as the range that was erased and return the entry that existed
-at that location.
+at that location.  Note that mas_erase() may allocate with the GFP_KERNEL flag.
+If this is not okay, consider using mas_store_gfp() and pass it a ``NULL``,
+after setting up the correct range by walking to the entry.
 
 You can walk each entry within a range by using mas_for_each().  If you want
 to walk each element of the tree then ``0`` and ``ULONG_MAX`` may be used as
@@ -211,7 +224,7 @@ Advanced Locking
 
 The maple tree uses a spinlock by default, but external locks can be used for
 tree updates as well.  To use an external lock, the tree must be initialized
-with the ``MT_FLAGS_LOCK_EXTERN flag``, this is usually done with the
+with the ``MT_FLAGS_LOCK_EXTERN`` flag, this is usually done with the
 MTREE_INIT_EXT() #define, which takes an external lock as an argument.
 
 Functions and structures
diff --git a/Documentation/core-api/mm-api.rst b/Documentation/core-api/mm-api.rst
index aabdd3cba58e8..c1d03a5a2a192 100644
--- a/Documentation/core-api/mm-api.rst
+++ b/Documentation/core-api/mm-api.rst
@@ -73,6 +73,7 @@ Readahead
 Writeback
 ---------
 
+.. kernel-doc:: include/linux/writeback.h
 .. kernel-doc:: mm/page-writeback.c
    :export:
 
@@ -117,7 +118,7 @@ More Memory Management Functions
 .. #kernel-doc:: mm/hmm.c (build warnings)
 .. kernel-doc:: mm/memremap.c
 .. kernel-doc:: mm/hugetlb.c
-.. kernel-doc:: mm/swap.c
+.. kernel-doc:: mm/folio.c
 .. kernel-doc:: mm/memcontrol.c
 .. #kernel-doc:: mm/memory-tiers.c (build warnings)
 .. kernel-doc:: mm/shmem.c
diff --git a/Documentation/core-api/packing.rst b/Documentation/core-api/packing.rst
index f68f1e08fef91..cff1a262efce1 100644
--- a/Documentation/core-api/packing.rst
+++ b/Documentation/core-api/packing.rst
@@ -330,7 +330,7 @@ Here is an example of how to use the fields APIs:
 
    void unpack_your_data(const packed_buf_t *buf, struct data *unpacked)
    {
-           BUILD_BUG_ON(sizeof(*buf) != SIZE;
+           BUILD_BUG_ON(sizeof(*buf) != SIZE);
 
            unpack_fields(buf, sizeof(*buf), unpacked, fields,
                          QUIRK_LITTLE_ENDIAN);
@@ -338,7 +338,7 @@ Here is an example of how to use the fields APIs:
 
    void pack_your_data(const struct data *unpacked, packed_buf_t *buf)
    {
-           BUILD_BUG_ON(sizeof(*buf) != SIZE;
+           BUILD_BUG_ON(sizeof(*buf) != SIZE);
 
            pack_fields(buf, sizeof(*buf), unpacked, fields,
                        QUIRK_LITTLE_ENDIAN);
diff --git a/Documentation/core-api/workqueue.rst b/Documentation/core-api/workqueue.rst
index 411e1b28b8dec..bb770f556568e 100644
--- a/Documentation/core-api/workqueue.rst
+++ b/Documentation/core-api/workqueue.rst
@@ -356,7 +356,7 @@ Guidelines
   well under the default limit.
 
 * A wq serves as a domain for forward progress guarantee
-  (``WQ_MEM_RECLAIM``, flush and work item attributes.  Work items
+  (``WQ_MEM_RECLAIM``), flush and work item attributes.  Work items
   which are not involved in memory reclaim and don't need to be
   flushed as a part of a group of work items, and don't require any
   special attribute, can use one of the system wq.  There is no
diff --git a/Documentation/crypto/architecture.rst b/Documentation/crypto/architecture.rst
index 249b54d0849f5..ec2e99d99aff5 100644
--- a/Documentation/crypto/architecture.rst
+++ b/Documentation/crypto/architecture.rst
@@ -95,7 +95,7 @@ additional templates may enclose other templates, such as
 
 ::
 
-        template1(template2(single block cipher)))
+        template1(template2(single block cipher))
 
 
 The kernel crypto API may provide multiple implementations of a template
diff --git a/Documentation/crypto/libcrypto-blockcipher.rst b/Documentation/crypto/libcrypto-blockcipher.rst
index dd5ce2f8b5151..fd85e27fab8d3 100644
--- a/Documentation/crypto/libcrypto-blockcipher.rst
+++ b/Documentation/crypto/libcrypto-blockcipher.rst
@@ -6,14 +6,14 @@ Block ciphers
 AES
 ---
 
-Support for the AES block cipher.
+This API provides support for the AES block cipher.
 
 .. kernel-doc:: include/crypto/aes.h
 
 DES
 ---
 
-Support for the DES block cipher.  This algorithm is obsolete and is supported
-only for backwards compatibility.
+This API provides support for the DES block cipher.  This algorithm is obsolete
+and is supported only for backwards compatibility.
 
 .. kernel-doc:: include/crypto/des.h
diff --git a/Documentation/crypto/libcrypto-hash.rst b/Documentation/crypto/libcrypto-hash.rst
index 4248e6fdc9527..fa4c54236af6e 100644
--- a/Documentation/crypto/libcrypto-hash.rst
+++ b/Documentation/crypto/libcrypto-hash.rst
@@ -6,81 +6,83 @@ Hash functions, MACs, and XOFs
 AES-CMAC and AES-XCBC-MAC
 -------------------------
 
-Support for the AES-CMAC and AES-XCBC-MAC message authentication codes.
+This API provides support for the AES-CMAC and AES-XCBC-MAC message
+authentication codes.
 
 .. kernel-doc:: include/crypto/aes-cbc-macs.h
 
 BLAKE2b
 -------
 
-Support for the BLAKE2b cryptographic hash function.
+This API provides support for the BLAKE2b cryptographic hash function.
 
 .. kernel-doc:: include/crypto/blake2b.h
 
 BLAKE2s
 -------
 
-Support for the BLAKE2s cryptographic hash function.
+This API provides support for the BLAKE2s cryptographic hash function.
 
 .. kernel-doc:: include/crypto/blake2s.h
 
 GHASH and POLYVAL
 -----------------
 
-Support for the GHASH and POLYVAL universal hash functions.  These algorithms
-are used only as internal components of other algorithms.
+This API provides support for the GHASH and POLYVAL universal hash functions.
+These algorithms are used only as internal components of other algorithms.
 
 .. kernel-doc:: include/crypto/gf128hash.h
 
 MD5
 ---
 
-Support for the MD5 cryptographic hash function and HMAC-MD5.  This algorithm is
-obsolete and is supported only for backwards compatibility.
+This API provides support for the MD5 cryptographic hash function and HMAC-MD5.
+This algorithm is obsolete and is supported only for backwards compatibility.
 
 .. kernel-doc:: include/crypto/md5.h
 
 NH
 --
 
-Support for the NH universal hash function.  This algorithm is used only as an
-internal component of other algorithms.
+This API provides support for the NH universal hash function.  This algorithm is
+used only as an internal component of other algorithms.
 
 .. kernel-doc:: include/crypto/nh.h
 
 Poly1305
 --------
 
-Support for the Poly1305 universal hash function.  This algorithm is used only
-as an internal component of other algorithms.
+This API provides support for the Poly1305 universal hash function.  This
+algorithm is used only as an internal component of other algorithms.
 
 .. kernel-doc:: include/crypto/poly1305.h
 
 SHA-1
 -----
 
-Support for the SHA-1 cryptographic hash function and HMAC-SHA1.  This algorithm
-is obsolete and is supported only for backwards compatibility.
+This API provides support for the SHA-1 cryptographic hash function and
+HMAC-SHA1.  This algorithm is obsolete and is supported only for backwards
+compatibility.
 
 .. kernel-doc:: include/crypto/sha1.h
 
 SHA-2
 -----
 
-Support for the SHA-2 family of cryptographic hash functions, including SHA-224,
-SHA-256, SHA-384, and SHA-512.  This also includes their corresponding HMACs:
-HMAC-SHA224, HMAC-SHA256, HMAC-SHA384, and HMAC-SHA512.
+This API provides support for the SHA-2 family of cryptographic hash functions,
+including SHA-224, SHA-256, SHA-384, and SHA-512.  This also includes their
+corresponding HMACs: HMAC-SHA224, HMAC-SHA256, HMAC-SHA384, and HMAC-SHA512.
 
 .. kernel-doc:: include/crypto/sha2.h
 
 SHA-3
 -----
 
-The SHA-3 functions are documented in :ref:`sha3`.
+The SHA-3 API is documented in :ref:`sha3`.
 
 SM3
 ---
 
-Support for the SM3 cryptographic hash function.
+This API provides support for the SM3 cryptographic hash function.
 
 .. kernel-doc:: include/crypto/sm3.h
diff --git a/Documentation/crypto/libcrypto-signature.rst b/Documentation/crypto/libcrypto-signature.rst
index e80d59fa51b6a..2a6dc793f0de8 100644
--- a/Documentation/crypto/libcrypto-signature.rst
+++ b/Documentation/crypto/libcrypto-signature.rst
@@ -6,6 +6,6 @@ Digital signature algorithms
 ML-DSA
 ------
 
-Support for the ML-DSA digital signature algorithm.
+This API provides support for the ML-DSA digital signature algorithm.
 
 .. kernel-doc:: include/crypto/mldsa.h
diff --git a/Documentation/crypto/libcrypto.rst b/Documentation/crypto/libcrypto.rst
index a1557d45b0e5a..0733e603d2295 100644
--- a/Documentation/crypto/libcrypto.rst
+++ b/Documentation/crypto/libcrypto.rst
@@ -4,8 +4,9 @@
 Crypto library
 ==============
 
-``lib/crypto/`` provides faster and easier access to cryptographic algorithms
-than the traditional crypto API.
+The Linux kernel's crypto library (``lib/crypto/``) provides kernel-internal
+users of cryptographic algorithms with faster and easier access to those
+algorithms than the traditional kernel crypto API.
 
 Each cryptographic algorithm is supported via a set of dedicated functions.
 "Crypto agility", where needed, is left to calling code.
diff --git a/Documentation/crypto/userspace-if.rst b/Documentation/crypto/userspace-if.rst
index ab93300c8e045..d6194346e3664 100644
--- a/Documentation/crypto/userspace-if.rst
+++ b/Documentation/crypto/userspace-if.rst
@@ -1,3 +1,5 @@
+.. _crypto_userspace_interface:
+
 User Space Interface
 ====================
 
@@ -12,9 +14,14 @@ AF_ALG is insecure and is deprecated. Originally added to the kernel in 2010,
 most kernel developers now consider it to be a mistake. Support for hardware
 accelerators, which was the original purpose of AF_ALG, has been removed.
 
-AF_ALG continues to be supported only for backwards compatibility. On systems
-where no programs using AF_ALG remain, the support for it should be disabled by
-disabling ``CONFIG_CRYPTO_USER_API_*``.
+AF_ALG continues to be supported only for backwards compatibility.
+
+Starting in Linux v7.3, the set of algorithms supported by AF_ALG is limited by
+default. See :ref:`/proc/sys/crypto/af_alg_restrict <af_alg_restrict>`.
+
+On systems where no programs using AF_ALG remain, the support for it should be
+disabled entirely by setting ``/proc/sys/crypto/af_alg_restrict`` to 2 or by
+disabling ``CONFIG_CRYPTO_USER_API_*`` in the kernel configuration.
 
 Deprecation
 -----------
diff --git a/Documentation/dev-tools/container.rst b/Documentation/dev-tools/container.rst
index 452415b646623..9e23f79d5ae13 100644
--- a/Documentation/dev-tools/container.rst
+++ b/Documentation/dev-tools/container.rst
@@ -40,7 +40,7 @@ Available options:
 
 ``-r, --runtime RUNTIME``
 
-    Container runtime name.  Supported runtimes: ``docker``, ``podman``.
+    Container runtime name.  Supported runtimes: ``podman``, ``docker``.
 
     If not specified, the first one found on the system will be used
     i.e. Podman if present, otherwise Docker.
@@ -75,8 +75,8 @@ working directory and adjust the user and group id as needed.
 
 The container image which would typically include a compiler toolchain is
 provided by the user and selected via the ``-i`` option.  The container runtime
-can be selected with the ``-r`` option, which can be either ``docker`` or
-``podman``.  If none is specified, the first one found on the system will be
+can be selected with the ``-r`` option, which can be either ``podman`` or
+``docker``.  If none is specified, the first one found on the system will be
 used while giving priority to Podman.  Support for other runtimes may be added
 later depending on their popularity among users.
 
diff --git a/Documentation/dev-tools/kmemleak.rst b/Documentation/dev-tools/kmemleak.rst
index 7d784e03f3f9d..a8a83bc69ceb8 100644
--- a/Documentation/dev-tools/kmemleak.rst
+++ b/Documentation/dev-tools/kmemleak.rst
@@ -198,6 +198,14 @@ systems, because of pointers temporarily stored in CPU registers or
 stacks. Kmemleak defines MSECS_MIN_AGE (defaulting to 1000) representing
 the minimum age of an object to be reported as a memory leak.
 
+The ``min_unref_scans`` module parameter (default 1) requires an object to
+be seen unreferenced in that many consecutive scans before it is reported.
+Keeping it at 1 preserves the historical behaviour; higher values filter
+the transient false positives described above, at the cost of delaying
+genuine reports by up to that many scans. It can be set at boot with
+``kmemleak.min_unref_scans=<n>`` or at run-time via
+``/sys/module/kmemleak/parameters/min_unref_scans``.
+
 Limitations and Drawbacks
 -------------------------
 
diff --git a/Documentation/dev-tools/kunit/faq.rst b/Documentation/dev-tools/kunit/faq.rst
index fae426f2634a1..b1341c1a62d95 100644
--- a/Documentation/dev-tools/kunit/faq.rst
+++ b/Documentation/dev-tools/kunit/faq.rst
@@ -25,19 +25,21 @@ disqualifying any of them from being considered unit testing frameworks.
 Does KUnit support running on architectures other than UML?
 ===========================================================
 
-Yes, mostly.
+Yes. KUnit can run on any architecture, though the kunit.py tool can only
+build and run kernels for some architectures (of which UML is the default).
 
-For the most part, the KUnit core framework (what we use to write the tests)
-can compile to any architecture. It compiles like just another part of the
-kernel and runs when the kernel boots, or when built as a module, when the
-module is loaded.  However, there is infrastructure, like the KUnit Wrapper
-(``tools/testing/kunit/kunit.py``) that might not support some architectures
-(see :ref:`kunit-on-qemu`).
+You can build and run tests without kunit.py at all on any architecture by
+enabling ``CONFIG_KUNIT=y`` and booting the kernel.
+See Documentation/dev-tools/kunit/run_manual.rst for more details.
 
-In short, yes, you can run KUnit on other architectures, but it might require
-more work than using KUnit on UML.
+Alternatively, kunit.py supports many common architectures using
+cross-compilers and the qemu emulator. This can be done using the ``--arch``
+parameter when running the tests, and the ``--cross_compile`` parameter
+when building (if the architecture is not supported by the host compiler).
+See :ref:`kunit-on-qemu` for more details.
 
-For more information, see :ref:`kunit-on-non-uml`.
+When writing tests targeting other architectures, it's worth keeping the tips
+on the :ref:`kunit-on-non-uml` page in mind.
 
 .. _kinds-of-tests:
 
@@ -78,27 +80,30 @@ things to try.
    down where an issue is occurring. (If you think the parser is at fault, you
    can run it manually against ``stdin`` or a file with ``kunit.py parse``.)
 3. Running the UML kernel directly can often reveal issues or error messages,
-   ``kunit_tool`` ignores. This should be as simple as running ``./vmlinux``
-   after building the UML kernel (for example, by using ``kunit.py build``).
+   ``kunit_tool`` ignores. This should be as simple as runningi the ``vmlinux``
+   binary in the output directory (by default ``./.kunit/vmlinux``) after
+   building the UML kernel (for example, by using ``kunit.py build``).
    Note that UML has some unusual requirements (such as the host having a tmpfs
    filesystem mounted), and has had issues in the past when built statically and
    the host has KASLR enabled. (On older host kernels, you may need to run
    ``setarch `uname -m` -R ./vmlinux`` to disable KASLR.)
-4. Make sure the kernel .config has ``CONFIG_KUNIT=y`` and at least one test
+4. Try running KUnit on a different architecture by using the ``--arch``
+   option. On an x86_64 host, using ``--arch=x86_64`` is a good first step.
+5. Make sure the kernel .config has ``CONFIG_KUNIT=y`` and at least one test
    (e.g. ``CONFIG_KUNIT_EXAMPLE_TEST=y``). kunit_tool will keep its .config
    around, so you can see what config was used after running ``kunit.py run``.
    It also preserves any config changes you might make, so you can
    enable/disable things with ``make ARCH=um menuconfig`` or similar, and then
    re-run kunit_tool.
-5. Try to run ``make ARCH=um defconfig`` before running ``kunit.py run``. This
+6. Try to run ``make ARCH=um defconfig`` before running ``kunit.py run``. This
    may help clean up any residual config items which could be causing problems.
-6. Finally, try running KUnit outside UML. KUnit and KUnit tests can be
-   built into any kernel, or can be built as a module and loaded at runtime.
-   Doing so should allow you to determine if UML is causing the issue you're
-   seeing. When tests are built-in, they will execute when the kernel boots, and
+7. Finally, try running KUnit manually, instead of via ``kunit.py``. KUnit can
+   be built into any kernel, or can be built as a module and loaded at runtime.
+   When tests are built-in, they will execute when the kernel boots, and
    modules will automatically execute associated tests when loaded. Test results
    can be collected from ``/sys/kernel/debug/kunit/<test suite>/results``, and
-   can be parsed with ``kunit.py parse``. For more details, see :ref:`kunit-on-qemu`.
+   can be parsed with ``kunit.py parse``. For more details, see
+   Documentation/dev-tools/kunit/run_manual.rst
 
 If none of the above tricks help, you are always welcome to email any issues to
 kunit-dev@googlegroups.com.
diff --git a/Documentation/dev-tools/kunit/style.rst b/Documentation/dev-tools/kunit/style.rst
index eac81a714a290..449f9f816fc71 100644
--- a/Documentation/dev-tools/kunit/style.rst
+++ b/Documentation/dev-tools/kunit/style.rst
@@ -164,9 +164,10 @@ This Kconfig entry must:
 * be visible only if ``CONFIG_KUNIT_ALL_TESTS`` is not enabled.
 * have a default value of ``CONFIG_KUNIT_ALL_TESTS``.
 * have a brief description of KUnit in the help text.
-
-If we are not able to meet above conditions (for example, the test is unable to
-be built as a module), Kconfig entries for tests should be tristate.
+* depend on the feature being tested, rather than selecting it (so that
+  enabling ``CONFIG_KUNIT_ALL_TESTS`` does not enable unrelated functionality).
+* be ``tristate``, unless there is a specific reason that the test cannot be
+  built as a module.
 
 For example, a Kconfig entry might look like:
 
diff --git a/Documentation/devicetree/bindings/Makefile b/Documentation/devicetree/bindings/Makefile
index 00149e8242611..6b4b4f51c371d 100644
--- a/Documentation/devicetree/bindings/Makefile
+++ b/Documentation/devicetree/bindings/Makefile
@@ -6,7 +6,7 @@ DT_MK_SCHEMA ?= dt-mk-schema
 DT_SCHEMA_LINT = $(shell which yamllint || \
   echo "warning: python package 'yamllint' not installed, skipping" >&2)
 
-DT_SCHEMA_MIN_VERSION = 2023.9
+DT_SCHEMA_MIN_VERSION = 2024.4
 
 PHONY += check_dtschema_version
 check_dtschema_version:
diff --git a/Documentation/devicetree/bindings/arm/altera.yaml b/Documentation/devicetree/bindings/arm/altera.yaml
index 0b418d9d60dbd..3030cf46fe74c 100644
--- a/Documentation/devicetree/bindings/arm/altera.yaml
+++ b/Documentation/devicetree/bindings/arm/altera.yaml
@@ -116,6 +116,12 @@ properties:
               - intel,socfpga-agilex5-socdk-nand
           - const: intel,socfpga-agilex5
 
+      - description: Agilex72 boards
+        items:
+          - enum:
+              - intel,socfpga-agilex72-socdk
+          - const: intel,socfpga-agilex72
+
       - description: Agilex7m boards
         items:
           - enum:
diff --git a/Documentation/devicetree/bindings/arm/fsl.yaml b/Documentation/devicetree/bindings/arm/fsl.yaml
index 86876311ec59a..7b57a6ee1b50d 100644
--- a/Documentation/devicetree/bindings/arm/fsl.yaml
+++ b/Documentation/devicetree/bindings/arm/fsl.yaml
@@ -1064,6 +1064,12 @@ properties:
           - const: solidrun,imx8mm-sr-som
           - const: fsl,imx8mm
 
+      - description: Variscite DART-MX8MM based boards
+        items:
+          - const: variscite,var-dart-mx8mm-sonata # Variscite DART-MX8MM on Sonata Development Board
+          - const: variscite,var-dart-mx8mm # Variscite DART-MX8M-MINI SOM
+          - const: fsl,imx8mm
+
       - description: Variscite VAR-SOM-MX8MM based boards
         items:
           - const: variscite,var-som-mx8mm-symphony
@@ -1310,6 +1316,24 @@ properties:
           - const: tq,imx8mp-tqma8mpql            # TQ-Systems GmbH i.MX8MP TQMa8MPQL SOM
           - const: fsl,imx8mp
 
+      - description:
+          TQMa8MPxS is a series of SOM featuring NXP i.MX8MP system-on-chip
+          variants. It has the SMARC-2.0 form factor and is designed to be
+          placed on different carrier boards. All CPU variants use the same
+          device tree hence only one compatible is needed. MB-SMARC-2 is a
+          carrier reference design.
+        items:
+          - enum:
+              - tq,imx8mp-tqma8mpqs-mb-smarc-2    # TQ-Systems GmbH i.MX8MP TQMa8MPQS SOM on MB-SMARC-2
+          - const: tq,imx8mp-tqma8mpqs            # TQ-Systems GmbH i.MX8MP TQMa8MPQS SOM
+          - const: fsl,imx8mp
+
+      - description: Variscite DART-MX8M Plus based boards
+        items:
+          - const: variscite,var-dart-mx8mp-sonata # Variscite DART-MX8MP on Sonata Development Board
+          - const: variscite,var-dart-mx8mp # Variscite DART-MX8MP SOM
+          - const: fsl,imx8mp
+
       - description: Variscite VAR-SOM-MX8M Plus based boards
         items:
           - const: variscite,var-som-mx8mp-symphony
diff --git a/Documentation/devicetree/bindings/arm/qcom,coresight-ctcu.yaml b/Documentation/devicetree/bindings/arm/qcom,coresight-ctcu.yaml
index e002f87361ad3..f3f3feac4ce28 100644
--- a/Documentation/devicetree/bindings/arm/qcom,coresight-ctcu.yaml
+++ b/Documentation/devicetree/bindings/arm/qcom,coresight-ctcu.yaml
@@ -29,7 +29,11 @@ properties:
     oneOf:
       - items:
           - enum:
+              - qcom,glymur-ctcu
+              - qcom,kaanapali-ctcu
               - qcom,qcs8300-ctcu
+              - qcom,sm8750-ctcu
+              - qcom,x1e80100-ctcu
           - const: qcom,sa8775p-ctcu
       - enum:
           - qcom,sa8775p-ctcu
diff --git a/Documentation/devicetree/bindings/arm/qcom-soc.yaml b/Documentation/devicetree/bindings/arm/qcom-soc.yaml
index 27261039d56f6..37fdd5a080b75 100644
--- a/Documentation/devicetree/bindings/arm/qcom-soc.yaml
+++ b/Documentation/devicetree/bindings/arm/qcom-soc.yaml
@@ -11,10 +11,10 @@ maintainers:
 
 description: |
   Guidelines for new compatibles for SoC blocks/components.
-  When adding new compatibles in new bindings, use the format::
+  When adding new compatibles in new bindings, use the format:
     qcom,SoC-IP
 
-  For example::
+  For example:
    qcom,sdm845-llcc-bwmon
 
   When adding new compatibles to existing bindings, use the format in the
diff --git a/Documentation/devicetree/bindings/arm/qcom.yaml b/Documentation/devicetree/bindings/arm/qcom.yaml
index 50cc18a6ec5ed..c7c9a92796846 100644
--- a/Documentation/devicetree/bindings/arm/qcom.yaml
+++ b/Documentation/devicetree/bindings/arm/qcom.yaml
@@ -49,18 +49,36 @@ properties:
               - qcom,apq8074-dragonboard
           - const: qcom,apq8074
 
-      - items:
-          - enum:
-              - qcom,apq8060-dragonboard
-              - qcom,msm8660-surf
-          - const: qcom,msm8660
-
       - items:
           - enum:
               - qcom,apq8084-mtp
               - qcom,apq8084-sbc
           - const: qcom,apq8084
 
+      - items:
+          - const: qcom,eliza-cqs-evk
+          - const: qcom,eliza-cqs-som
+          - const: qcom,eliza
+
+      - items:
+          - enum:
+              - sony,karin_windy
+          - const: qcom,apq8094
+
+      - items:
+          - enum:
+              - arrow,apq8096-db820c
+              - inforce,ifc6640
+          - const: qcom,apq8096-sbc
+          - const: qcom,apq8096
+
+      - items:
+          - const: arrow,apq8096sg-db820c
+          - const: arrow,apq8096-db820c
+          - const: qcom,apq8096-sbc
+          - const: qcom,apq8096sg
+          - const: qcom,apq8096
+
       - items:
           - enum:
               - qcom,eliza-mtp
@@ -73,102 +91,133 @@ properties:
 
       - items:
           - enum:
-              - qcom,mahua-crd
-          - const: qcom,mahua
+              - 8dev,jalapeno
+              - alfa-network,ap120c-ac
+          - const: qcom,ipq4018
 
       - items:
           - enum:
-              - fairphone,fp6
-              - nothing,asteroids
-          - const: qcom,milos
+              - qcom,ipq4019-ap-dk01.1-c1
+              - qcom,ipq4019-ap-dk04.1-c3
+              - qcom,ipq4019-ap-dk07.1-c1
+              - qcom,ipq4019-ap-dk07.1-c2
+              - qcom,ipq4019-dk04.1-c1
+          - const: qcom,ipq4019
 
       - items:
           - enum:
-              - microsoft,dempsey
-              - microsoft,makepeace
-              - microsoft,moneypenny
-              - motorola,falcon
-              - samsung,ms013g
-              - samsung,s3ve3g
-          - const: qcom,msm8226
+              - qcom,ipq5018-rdp432-c2
+              - tplink,archer-ax55-v1
+          - const: qcom,ipq5018
 
       - items:
           - enum:
-              - htc,memul
-              - microsoft,superman-lte
-              - microsoft,tesla
-              - motorola,peregrine
-              - samsung,matisselte
-          - const: qcom,msm8926
-          - const: qcom,msm8226
+              - qcom,ipq5210-rdp504
+          - const: qcom,ipq5210
 
       - items:
           - enum:
-              - wingtech,wt82918hd
-          - const: qcom,msm8929
+              - qcom,ipq5332-ap-mi01.2
+              - qcom,ipq5332-ap-mi01.3
+              - qcom,ipq5332-ap-mi01.6
+              - qcom,ipq5332-ap-mi01.9
+          - const: qcom,ipq5332
 
       - items:
           - enum:
-              - asus,z00t
-              - huawei,kiwi
-              - longcheer,l9100
-              - samsung,a7
-              - sony,kanuti-tulip
-              - square,apq8039-t2
-              - wingtech,wt82918
-              - wingtech,wt82918hdhw39
-          - const: qcom,msm8939
+              - qcom,ipq5424-rdp466
+          - const: qcom,ipq5424
 
       - items:
           - enum:
-              - sony,kugo-row
-              - sony,suzu-row
-          - const: qcom,msm8956
+              - qcom,ipq6018-cp01
+              - qcom,ipq6018-cp01-c1
+          - const: qcom,ipq6018
 
       - items:
           - enum:
-              - qcom,msm8960-cdp
-              - samsung,expressatt
-          - const: qcom,msm8960
+              - mikrotik,rb3011
+              - qcom,ipq8064-ap148
+          - const: qcom,ipq8064
 
       - items:
           - enum:
-              - sony,huashan
-          - const: qcom,msm8960t
-          - const: qcom,msm8960
+              - qcom,ipq8074-hk01
+              - qcom,ipq8074-hk10-c1
+              - qcom,ipq8074-hk10-c2
+          - const: qcom,ipq8074
 
       - items:
           - enum:
-              - lge,hammerhead
-              - samsung,hlte
-              - sony,xperia-amami
-              - sony,xperia-honami
-              - sony,xperia-togari
-          - const: qcom,msm8974
+              - qcom,ipq9574-ap-al02-c2
+              - qcom,ipq9574-ap-al02-c2-emmc
+              - qcom,ipq9574-ap-al02-c6
+              - qcom,ipq9574-ap-al02-c7
+              - qcom,ipq9574-ap-al02-c7-emmc
+              - qcom,ipq9574-ap-al02-c8
+              - qcom,ipq9574-ap-al02-c9
+          - const: qcom,ipq9574
 
       - items:
           - enum:
-              - fairphone,fp2
-              - htc,m8
-              - oneplus,bacon
-              - samsung,klte
-              - sony,xperia-aries
-              - sony,xperia-castor
-              - sony,xperia-leo
-          - const: qcom,msm8974pro
-          - const: qcom,msm8974
+              - qcom,ipq9650-rdp488
+          - const: qcom,ipq9650
 
       - items:
           - enum:
-              - samsung,kltechn
-          - const: samsung,klte
-          - const: qcom,msm8974pro
-          - const: qcom,msm8974
+              - qcom,hawi-mtp
+          - const: qcom,hawi
 
       - items:
           - enum:
-              - longcheer,l9360
-          - const: qcom,msm8976
+              - qcom,kaanapali-mtp
+              - qcom,kaanapali-qrd
+          - const: qcom,kaanapali
+
+      - items:
+          - enum:
+              - qcom,mahua-crd
+          - const: qcom,mahua
+
+      - description: Sierra Wireless MangOH Green with WP8548 Module
+        items:
+          - const: swir,mangoh-green-wp8548
+          - const: swir,wp8548
+          - const: qcom,mdm9615
+
+      - items:
+          - enum:
+              - fairphone,fp6
+              - nothing,asteroids
+          - const: qcom,milos
+
+      - items:
+          - enum:
+              - microsoft,dempsey
+              - microsoft,makepeace
+              - microsoft,moneypenny
+              - motorola,falcon
+              - motorola,titan
+              - samsung,ms013g
+              - samsung,s3ve3g
+          - const: qcom,msm8226
+
+      - items:
+          - enum:
+              - htc,memul
+              - microsoft,superman-lte
+              - microsoft,tesla
+              - motorola,peregrine
+              - samsung,matisselte
+              - sony,eagle
+          - const: qcom,msm8926
+          - const: qcom,msm8226
+
+      - items:
+          - enum:
+              - qcom,apq8060-dragonboard
+              - qcom,msm8660-surf
+          - const: qcom,msm8660
 
       - items:
           - enum:
@@ -217,11 +266,28 @@ properties:
               - xiaomi,tiare
           - const: qcom,msm8917
 
+      - items:
+          - enum:
+              - wingtech,wt82918hd
+          - const: qcom,msm8929
+
       - items:
           - enum:
               - xiaomi,land
           - const: qcom,msm8937
 
+      - items:
+          - enum:
+              - asus,z00t
+              - huawei,kiwi
+              - longcheer,l9100
+              - samsung,a7
+              - sony,kanuti-tulip
+              - square,apq8039-t2
+              - wingtech,wt82918
+              - wingtech,wt82918hdhw39
+          - const: qcom,msm8939
+
       - items:
           - enum:
               - flipkart,rimob
@@ -232,6 +298,57 @@ properties:
               - xiaomi,vince
           - const: qcom,msm8953
 
+      - items:
+          - enum:
+              - sony,kugo-row
+              - sony,suzu-row
+          - const: qcom,msm8956
+
+      - items:
+          - enum:
+              - qcom,msm8960-cdp
+              - samsung,expressatt
+          - const: qcom,msm8960
+
+      - items:
+          - enum:
+              - sony,huashan
+          - const: qcom,msm8960t
+          - const: qcom,msm8960
+
+      - items:
+          - enum:
+              - lge,hammerhead
+              - samsung,hlte
+              - sony,xperia-amami
+              - sony,xperia-honami
+              - sony,xperia-togari
+          - const: qcom,msm8974
+
+      - items:
+          - enum:
+              - fairphone,fp2
+              - htc,m8
+              - oneplus,bacon
+              - samsung,klte
+              - sony,xperia-aries
+              - sony,xperia-castor
+              - sony,xperia-leo
+          - const: qcom,msm8974pro
+          - const: qcom,msm8974
+
+      - items:
+          - enum:
+              - samsung,kltechn
+          - const: samsung,klte
+          - const: qcom,msm8974pro
+          - const: qcom,msm8974
+
+      - items:
+          - enum:
+              - longcheer,l9360
+          - const: qcom,msm8976
+
       - items:
           - enum:
               - lg,bullhead
@@ -240,11 +357,6 @@ properties:
               - xiaomi,libra
           - const: qcom,msm8992
 
-      - items:
-          - enum:
-              - sony,karin_windy
-          - const: qcom,apq8094
-
       - items:
           - enum:
               - huawei,angler
@@ -256,20 +368,6 @@ properties:
               - sony,suzuran-row
           - const: qcom,msm8994
 
-      - items:
-          - enum:
-              - arrow,apq8096-db820c
-              - inforce,ifc6640
-          - const: qcom,apq8096-sbc
-          - const: qcom,apq8096
-
-      - items:
-          - const: arrow,apq8096sg-db820c
-          - const: arrow,apq8096-db820c
-          - const: qcom,apq8096-sbc
-          - const: qcom,apq8096sg
-          - const: qcom,apq8096
-
       - items:
           - enum:
               - oneplus,oneplus3
@@ -303,86 +401,6 @@ properties:
               - xiaomi,sagit
           - const: qcom,msm8998
 
-      - items:
-          - enum:
-              - 8dev,jalapeno
-              - alfa-network,ap120c-ac
-          - const: qcom,ipq4018
-
-      - items:
-          - enum:
-              - qcom,ipq4019-ap-dk01.1-c1
-              - qcom,ipq4019-ap-dk04.1-c3
-              - qcom,ipq4019-ap-dk07.1-c1
-              - qcom,ipq4019-ap-dk07.1-c2
-              - qcom,ipq4019-dk04.1-c1
-          - const: qcom,ipq4019
-
-      - items:
-          - enum:
-              - qcom,ipq5018-rdp432-c2
-              - tplink,archer-ax55-v1
-          - const: qcom,ipq5018
-
-      - items:
-          - enum:
-              - qcom,ipq5210-rdp504
-          - const: qcom,ipq5210
-
-      - items:
-          - enum:
-              - qcom,ipq5332-ap-mi01.2
-              - qcom,ipq5332-ap-mi01.3
-              - qcom,ipq5332-ap-mi01.6
-              - qcom,ipq5332-ap-mi01.9
-          - const: qcom,ipq5332
-
-      - items:
-          - enum:
-              - qcom,ipq5424-rdp466
-          - const: qcom,ipq5424
-
-      - items:
-          - enum:
-              - mikrotik,rb3011
-              - qcom,ipq8064-ap148
-          - const: qcom,ipq8064
-
-      - items:
-          - enum:
-              - qcom,ipq8074-hk01
-              - qcom,ipq8074-hk10-c1
-              - qcom,ipq8074-hk10-c2
-          - const: qcom,ipq8074
-
-      - items:
-          - enum:
-              - qcom,ipq9574-ap-al02-c2
-              - qcom,ipq9574-ap-al02-c2-emmc
-              - qcom,ipq9574-ap-al02-c6
-              - qcom,ipq9574-ap-al02-c7
-              - qcom,ipq9574-ap-al02-c7-emmc
-              - qcom,ipq9574-ap-al02-c8
-              - qcom,ipq9574-ap-al02-c9
-          - const: qcom,ipq9574
-
-      - items:
-          - enum:
-              - qcom,ipq9650-rdp488
-          - const: qcom,ipq9650
-
-      - items:
-          - enum:
-              - qcom,kaanapali-mtp
-              - qcom,kaanapali-qrd
-          - const: qcom,kaanapali
-
-      - description: Sierra Wireless MangOH Green with WP8548 Module
-        items:
-          - const: swir,mangoh-green-wp8548
-          - const: swir,wp8548
-          - const: qcom,mdm9615
-
       - description: Qualcomm Technologies, Inc. Robotics RB1
         items:
           - enum:
@@ -401,8 +419,23 @@ properties:
               - shift,otter
               - thundercomm,minipc-g1iot
               - thundercomm,rubikpi3
+              - vicharak,axon-mini
           - const: qcom,qcm6490
 
+      - items:
+          - enum:
+              - qcom,qcs404-evb-1000
+              - qcom,qcs404-evb-4000
+          - const: qcom,qcs404-evb
+          - const: qcom,qcs404
+
+      - items:
+          - enum:
+              - arduino,monza
+              - qcom,monaco-evk
+              - qcom,qcs8300-ride
+          - const: qcom,qcs8300
+
       - description: Qualcomm Technologies, Inc. Distributed Unit 1000 platform
         items:
           - enum:
@@ -416,6 +449,31 @@ properties:
               - qcom,qru1000-idp
           - const: qcom,qru1000
 
+      - items:
+          - enum:
+              - qcom,sa8155p-adp
+          - const: qcom,sa8155p
+
+      - items:
+          - enum:
+              - qcom,sa8295p-adp
+              - qcom,sa8540p-ride
+          - const: qcom,sa8540p
+
+      - items:
+          - enum:
+              - qcom,sa8775p-ride
+              - qcom,sa8775p-ride-r3
+          - const: qcom,sa8775p
+
+      - items:
+          - enum:
+              - qcom,lemans-evk
+              - qcom,qcs9100-ride
+              - qcom,qcs9100-ride-r3
+          - const: qcom,qcs9100
+          - const: qcom,sa8775p
+
       - description: Qualcomm AR2 Gen1 platform
         items:
           - enum:
@@ -828,14 +886,6 @@ properties:
           - const: google,zombie-sku514
           - const: qcom,sc7280
 
-      - description: Xiaomi Poco F1
-        items:
-          - enum:
-              - xiaomi,beryllium-ebbg
-              - xiaomi,beryllium-tianma
-          - const: xiaomi,beryllium
-          - const: qcom,sdm845
-
       - items:
           - enum:
               - lenovo,flex-5g
@@ -853,6 +903,11 @@ properties:
               - qcom,sc8280xp-qrd
           - const: qcom,sc8280xp
 
+      - items:
+          - enum:
+              - inforce,ifc6560
+          - const: qcom,sda660
+
       - items:
           - enum:
               - lenovo,tbx605f
@@ -867,11 +922,6 @@ properties:
               - sony,voyager-row
           - const: qcom,sdm630
 
-      - items:
-          - enum:
-              - inforce,ifc6560
-          - const: qcom,sda660
-
       - items:
           - enum:
               - fairphone,fp3
@@ -899,74 +949,13 @@ properties:
           - const: google,bonito
           - const: qcom,sdm670
 
-      - items:
-          - enum:
-              - qcom,sdx55-mtp
-              - qcom,sdx55-telit-fn980-tlb
-              - qcom,sdx55-t55
-          - const: qcom,sdx55
-
-      - items:
-          - enum:
-              - qcom,sdx65-mtp
-          - const: qcom,sdx65
-
-      - items:
-          - enum:
-              - qcom,sdx75-idp
-          - const: qcom,sdx75
-
-      - items:
-          - enum:
-              - qcom,ipq6018-cp01
-              - qcom,ipq6018-cp01-c1
-          - const: qcom,ipq6018
-
-      - items:
-          - enum:
-              - qcom,qcs404-evb-1000
-              - qcom,qcs404-evb-4000
-          - const: qcom,qcs404-evb
-          - const: qcom,qcs404
-
-      - items:
-          - enum:
-              - arduino,monza
-              - qcom,monaco-evk
-              - qcom,qcs8300-ride
-          - const: qcom,qcs8300
-
-      - items:
-          - enum:
-              - qcom,qcs615-ride
-              - qcom,talos-evk
-          - const: qcom,qcs615
-          - const: qcom,sm6150
-
-      - items:
-          - enum:
-              - qcom,sa8155p-adp
-          - const: qcom,sa8155p
-
-      - items:
-          - enum:
-              - qcom,sa8295p-adp
-              - qcom,sa8540p-ride
-          - const: qcom,sa8540p
-
-      - items:
-          - enum:
-              - qcom,sa8775p-ride
-              - qcom,sa8775p-ride-r3
-          - const: qcom,sa8775p
-
-      - items:
+      - description: Xiaomi Poco F1
+        items:
           - enum:
-              - qcom,lemans-evk
-              - qcom,qcs9100-ride
-              - qcom,qcs9100-ride-r3
-          - const: qcom,qcs9100
-          - const: qcom,sa8775p
+              - xiaomi,beryllium-ebbg
+              - xiaomi,beryllium-tianma
+          - const: xiaomi,beryllium
+          - const: qcom,sdm845
 
       - items:
           - enum:
@@ -989,6 +978,41 @@ properties:
               - xiaomi,polaris
           - const: qcom,sdm845
 
+      - items:
+          - enum:
+              - qcom,shikra-cqm-evk
+          - const: qcom,shikra-cqm-som
+          - const: qcom,shikra
+
+      - items:
+          - enum:
+              - qcom,shikra-cqs-evk
+          - const: qcom,shikra-cqs-som
+          - const: qcom,shikra
+
+      - items:
+          - enum:
+              - qcom,shikra-iqs-evk
+          - const: qcom,shikra-iqs-som
+          - const: qcom,shikra
+
+      - items:
+          - enum:
+              - qcom,sdx55-mtp
+              - qcom,sdx55-telit-fn980-tlb
+              - qcom,sdx55-t55
+          - const: qcom,sdx55
+
+      - items:
+          - enum:
+              - qcom,sdx65-mtp
+          - const: qcom,sdx65
+
+      - items:
+          - enum:
+              - qcom,sdx75-idp
+          - const: qcom,sdx75
+
       - items:
           - enum:
               - oneplus,billie2
@@ -1024,6 +1048,13 @@ properties:
               - xiaomi,willow
           - const: qcom,sm6125
 
+      - items:
+          - enum:
+              - qcom,qcs615-ride
+              - qcom,talos-evk
+          - const: qcom,qcs615
+          - const: qcom,sm6150
+
       - items:
           - enum:
               - sony,pdx213
@@ -1054,6 +1085,7 @@ properties:
           - enum:
               - motorola,dubai
               - nothing,spacewar
+              - xiaomi,taoyao
           - const: qcom,sm7325
 
       - items:
@@ -1112,6 +1144,12 @@ properties:
           - const: qcom,qcs8550
           - const: qcom,sm8550
 
+      - items:
+          - enum:
+              - qcom,qcs8550-rb5gen2
+          - const: qcom,qcs8550
+          - const: qcom,sm8550
+
       - items:
           - enum:
               - ayaneo,pocket-s2
@@ -1160,6 +1198,7 @@ properties:
               - dell,inspiron-14-plus-7441
               - dell,latitude-7455
               - dell,xps13-9345
+              - honor,magicbook-art-14-snapdragon
               - hp,elitebook-ultra-g1q
               - hp,omnibook-x14
               - lenovo,ideacentre-mini-01q8x10
@@ -1215,7 +1254,7 @@ properties:
     items:
       items:
         - description: |
-            MSM chipset ID - an exact match value consisting of two bitfields::
+            MSM chipset ID - an exact match value consisting of two bitfields:
              - bits 0-15  - The unique MSM chipset ID
              - bits 16-31 - Reserved; should be 0
         - description: |
@@ -1241,7 +1280,7 @@ properties:
       - items:
           - items:
               - description: |
-                  Board ID consisting of three bitfields::
+                  Board ID consisting of three bitfields:
                     - bits 31-24 - Unused
                     - bits 23-16 - Platform Version Major
                     - bits 15-8  - Platform Version Minor
diff --git a/Documentation/devicetree/bindings/arm/rockchip.yaml b/Documentation/devicetree/bindings/arm/rockchip.yaml
index 1a9dde18626d0..d9409b150701c 100644
--- a/Documentation/devicetree/bindings/arm/rockchip.yaml
+++ b/Documentation/devicetree/bindings/arm/rockchip.yaml
@@ -162,6 +162,13 @@ properties:
           - const: coolpi,pi-4b
           - const: rockchip,rk3588s
 
+      - description: Alientek CLRV1126F SoM based boards
+        items:
+          - enum:
+              - alientek,dlrv1126
+          - const: alientek,clrv1126f
+          - const: rockchip,rv1126
+
       - description: Edgeble Neural Compute Module 2(Neu2) SoM based boards
         items:
           - const: edgeble,neural-compute-module-2-io   # Edgeble Neural Compute Module 2 IO Board
@@ -684,6 +691,12 @@ properties:
           - const: google,veyron
           - const: rockchip,rk3288
 
+      - description: Graperain G3568 series board
+        items:
+          - enum:
+              - graperain,g3568-v2
+          - const: rockchip,rk3568
+
       - description: H96 Max V58 TV Box
         items:
           - const: haochuangyi,h96-max-v58
@@ -724,6 +737,11 @@ properties:
           - const: hardkernel,odroid-m2
           - const: rockchip,rk3588s
 
+      - description: HINLINK H28K
+        items:
+          - const: hinlink,h28k
+          - const: rockchip,rk3528
+
       - description: HINLINK H66K / H68K
         items:
           - enum:
@@ -779,6 +797,11 @@ properties:
           - const: lckfb,tspi-rk3566
           - const: rockchip,rk3566
 
+      - description: LCKFB Taishan Pi 3M RK3576
+        items:
+          - const: lckfb,tspi-3m-rk3576
+          - const: rockchip,rk3576
+
       - description: LinkEase EasePi R1
         items:
           - const: linkease,easepi-r1
@@ -1338,6 +1361,7 @@ properties:
         items:
           - enum:
               - xunlong,orangepi-5
+              - xunlong,orangepi-5-pro
               - xunlong,orangepi-5b
           - const: rockchip,rk3588s
 
@@ -1347,6 +1371,11 @@ properties:
           - const: xunlong,orangepi-cm5
           - const: rockchip,rk3588s
 
+      - description: Youyeetoo YY3588
+        items:
+          - const: youyeetoo,yy3588
+          - const: rockchip,rk3588
+
       - description: Zkmagic A95X Z2
         items:
           - const: zkmagic,a95x-z2
diff --git a/Documentation/devicetree/bindings/arm/rockchip/pmu.yaml b/Documentation/devicetree/bindings/arm/rockchip/pmu.yaml
index 55b2200d6e75b..1cff2328cd01e 100644
--- a/Documentation/devicetree/bindings/arm/rockchip/pmu.yaml
+++ b/Documentation/devicetree/bindings/arm/rockchip/pmu.yaml
@@ -61,7 +61,13 @@ properties:
     type: object
 
   reboot-mode:
-    type: object
+    $ref: /schemas/power/reset/syscon-reboot-mode.yaml
+    unevaluatedProperties: false
+
+    patternProperties:
+      # Negative look-ahead to disallow unsupported modes. The '$' has to be
+      # part of lookahead group to work, instead of trailing outside of ().
+      "^mode-(?!(bootloader$|loader$|normal$|recovery$))": false
 
 required:
   - compatible
diff --git a/Documentation/devicetree/bindings/arm/samsung/samsung-boards.yaml b/Documentation/devicetree/bindings/arm/samsung/samsung-boards.yaml
index 753b3ba1b607d..273464400477c 100644
--- a/Documentation/devicetree/bindings/arm/samsung/samsung-boards.yaml
+++ b/Documentation/devicetree/bindings/arm/samsung/samsung-boards.yaml
@@ -235,6 +235,12 @@ properties:
               - winlink,e850-96                 # WinLink E850-96
           - const: samsung,exynos850
 
+      - description: Exynos8855 based boards
+        items:
+          - enum:
+              - samsung,exynos8855-smdk         # Samsung SMDK
+          - const: samsung,exynos8855
+
       - description: Exynos8895 based boards
         items:
           - enum:
diff --git a/Documentation/devicetree/bindings/arm/samsung/samsung-soc.yaml b/Documentation/devicetree/bindings/arm/samsung/samsung-soc.yaml
index 653f859976434..ab000befe76da 100644
--- a/Documentation/devicetree/bindings/arm/samsung/samsung-soc.yaml
+++ b/Documentation/devicetree/bindings/arm/samsung/samsung-soc.yaml
@@ -11,10 +11,10 @@ maintainers:
 
 description: |
   Guidelines for new compatibles for SoC blocks/components.
-  When adding new compatibles in new bindings, use the format::
+  When adding new compatibles in new bindings, use the format:
     samsung,SoC-IP
 
-  For example::
+  For example:
     samsung,exynos5433-cmu-isp
 
 select:
diff --git a/Documentation/devicetree/bindings/arm/sunxi.yaml b/Documentation/devicetree/bindings/arm/sunxi.yaml
index e6443c266fa1e..f04db28c630ac 100644
--- a/Documentation/devicetree/bindings/arm/sunxi.yaml
+++ b/Documentation/devicetree/bindings/arm/sunxi.yaml
@@ -96,6 +96,12 @@ properties:
           - const: allwinner,ba10-tvbox
           - const: allwinner,sun4i-a10
 
+      - description: Baijie A133 HelperBoard
+        items:
+          - const: baijie,helperboard-a133
+          - const: baijie,helperboard-a133-core
+          - const: allwinner,sun50i-a100
+
       - description: BananaPi
         items:
           - const: lemaker,bananapi
@@ -598,6 +604,7 @@ properties:
       - description: NetCube Systems Nagami SoM based boards
         items:
           - enum:
+              - netcube,dobermann
               - netcube,nagami-basic-carrier
               - netcube,nagami-keypad-carrier
           - const: netcube,nagami
diff --git a/Documentation/devicetree/bindings/arm/xen.txt b/Documentation/devicetree/bindings/arm/xen.txt
deleted file mode 100644
index f925290d46415..0000000000000
--- a/Documentation/devicetree/bindings/arm/xen.txt
+++ /dev/null
@@ -1,62 +0,0 @@
-* Xen hypervisor device tree bindings
-
-Xen ARM virtual platforms shall have a top-level "hypervisor" node with
-the following properties:
-
-- compatible:
-	compatible = "xen,xen-<version>", "xen,xen";
-  where <version> is the version of the Xen ABI of the platform.
-
-- reg: specifies the base physical address and size of the regions in memory
-  where the special resources should be mapped to, using an HYPERVISOR_memory_op
-  hypercall.
-  Region 0 is reserved for mapping grant table, it must be always present.
-  The memory region is large enough to map the whole grant table (it is larger
-  or equal to gnttab_max_grant_frames()).
-  Regions 1...N are extended regions (unused address space) for mapping foreign
-  GFNs and grants, they might be absent if there is nothing to expose.
-
-- interrupts: the interrupt used by Xen to inject event notifications.
-  A GIC node is also required.
-
-To support UEFI on Xen ARM virtual platforms, Xen populates the FDT "uefi" node
-under /hypervisor with following parameters:
-
-________________________________________________________________________________
-Name                      | Size   | Description
-================================================================================
-xen,uefi-system-table     | 64-bit | Guest physical address of the UEFI System
-			  |	   | Table.
---------------------------------------------------------------------------------
-xen,uefi-mmap-start       | 64-bit | Guest physical address of the UEFI memory
-			  |	   | map.
---------------------------------------------------------------------------------
-xen,uefi-mmap-size        | 32-bit | Size in bytes of the UEFI memory map
-                          |        | pointed to in previous entry.
---------------------------------------------------------------------------------
-xen,uefi-mmap-desc-size   | 32-bit | Size in bytes of each entry in the UEFI
-                          |        | memory map.
---------------------------------------------------------------------------------
-xen,uefi-mmap-desc-ver    | 32-bit | Version of the mmap descriptor format.
---------------------------------------------------------------------------------
-
-Example (assuming #address-cells = <2> and #size-cells = <2>):
-
-hypervisor {
-	compatible = "xen,xen-4.3", "xen,xen";
-	reg = <0 0xb0000000 0 0x20000>;
-	interrupts = <1 15 0xf08>;
-	uefi {
-		xen,uefi-system-table = <0xXXXXXXXX>;
-		xen,uefi-mmap-start = <0xXXXXXXXX>;
-		xen,uefi-mmap-size = <0xXXXXXXXX>;
-		xen,uefi-mmap-desc-size = <0xXXXXXXXX>;
-		xen,uefi-mmap-desc-ver = <0xXXXXXXXX>;
-        };
-};
-
-The format and meaning of the "xen,uefi-*" parameters are similar to those in
-Documentation/arch/arm/uefi.rst, which are provided by the regular UEFI stub. However
-they differ because they are provided by the Xen hypervisor, together with a set
-of UEFI runtime services implemented via hypercalls, see
-http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,platform.h.html.
diff --git a/Documentation/devicetree/bindings/arm/xen.yaml b/Documentation/devicetree/bindings/arm/xen.yaml
new file mode 100644
index 0000000000000..bad077a112c91
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/xen.yaml
@@ -0,0 +1,110 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/arm/xen.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Xen hypervisor
+
+maintainers:
+  - Stefano Stabellini <sstabellini@kernel.org>
+
+description:
+  Xen ARM virtual platforms shall have a top-level "hypervisor" node with
+  the properties defined below.
+
+properties:
+  $nodename:
+    pattern: '^hypervisor(@[0-9a-f]+)?$'
+
+  compatible:
+    description:
+      Specifies the Xen hypervisor. The version of the Xen ABI is encoded
+      in the first item as "xen,xen-<version>", followed by the generic
+      "xen,xen" string.
+    items:
+      - pattern: '^xen,xen-[0-9]+\.[0-9]+$'
+      - const: xen,xen
+
+  reg:
+    description: |
+      Base physical address and size of the regions in memory where special
+      resources should be mapped to, using a HYPERVISOR_memory_op hypercall.
+
+      Region 0 is reserved for mapping the grant table and must always be
+      present. The memory region must be large enough to map the whole grant
+      table (it is larger or equal to gnttab_max_grant_frames()).
+
+      Regions 1...N are extended regions (unused address space) for mapping
+      foreign GFNs and grants. They might be absent if there is nothing to
+      expose.
+    minItems: 1
+
+  interrupts:
+    description:
+      The interrupt used by Xen to inject event notifications.
+      A GIC node is also required.
+    maxItems: 1
+
+  uefi:
+    type: object
+    description:
+      Node populated by Xen to support UEFI on Xen ARM virtual platforms.
+      The format and meaning of the "xen,uefi-*" parameters are similar to
+      those in Documentation/arch/arm/uefi.rst, but are provided by the Xen
+      hypervisor together with a set of UEFI runtime services implemented via
+      hypercalls.
+
+    properties:
+      xen,uefi-system-table:
+        description: Guest physical address of the UEFI System Table.
+        $ref: /schemas/types.yaml#/definitions/uint64
+
+      xen,uefi-mmap-start:
+        description: Guest physical address of the UEFI memory map.
+        $ref: /schemas/types.yaml#/definitions/uint64
+
+      xen,uefi-mmap-size:
+        description: Size in bytes of the UEFI memory map pointed to by xen,uefi-mmap-start.
+        $ref: /schemas/types.yaml#/definitions/uint32
+
+      xen,uefi-mmap-desc-size:
+        description: Size in bytes of each entry in the UEFI memory map.
+        $ref: /schemas/types.yaml#/definitions/uint32
+
+      xen,uefi-mmap-desc-ver:
+        description: Version of the mmap descriptor format.
+        $ref: /schemas/types.yaml#/definitions/uint32
+
+    required:
+      - xen,uefi-system-table
+      - xen,uefi-mmap-start
+      - xen,uefi-mmap-size
+      - xen,uefi-mmap-desc-size
+      - xen,uefi-mmap-desc-ver
+
+    additionalProperties: false
+
+required:
+  - compatible
+  - reg
+  - interrupts
+
+additionalProperties: false
+
+examples:
+  - |
+    hypervisor@b0000000 {
+        compatible = "xen,xen-4.3", "xen,xen";
+        reg = <0xb0000000 0x20000>;
+        interrupts = <1 15 0xf08>;
+
+        uefi {
+            xen,uefi-system-table = /bits/ 64 <0x1301415>;
+            xen,uefi-mmap-start = /bits/ 64 <0x7591400>;
+            xen,uefi-mmap-size = <0x1800>;
+            xen,uefi-mmap-desc-size = <0x30>;
+            xen,uefi-mmap-desc-ver = <1>;
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/bus/ti,da850-mstpri.txt b/Documentation/devicetree/bindings/bus/ti,da850-mstpri.txt
deleted file mode 100644
index 72daefc6b4a1f..0000000000000
--- a/Documentation/devicetree/bindings/bus/ti,da850-mstpri.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-* Device tree bindings for Texas Instruments da8xx master peripheral
-  priority driver
-
-DA8XX SoCs feature a set of registers allowing to change the priority of all
-peripherals classified as masters.
-
-Documentation:
-OMAP-L138 (DA850) - http://www.ti.com/lit/ug/spruh82c/spruh82c.pdf
-
-Required properties:
-
-- compatible:		"ti,da850-mstpri" - for da850 based boards
-- reg:			offset and length of the mstpri registers
-
-Example for da850-lcdk is shown below.
-
-mstpri {
-	compatible = "ti,da850-mstpri";
-	reg = <0x14110 0x0c>;
-};
diff --git a/Documentation/devicetree/bindings/bus/ti,da850-mstpri.yaml b/Documentation/devicetree/bindings/bus/ti,da850-mstpri.yaml
new file mode 100644
index 0000000000000..5b03abd3fc3be
--- /dev/null
+++ b/Documentation/devicetree/bindings/bus/ti,da850-mstpri.yaml
@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/bus/ti,da850-mstpri.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments DA8XX Master Peripheral Priority Registers
+
+maintainers:
+  - Eduard Bostina <egbostina@gmail.com>
+
+description: |
+  DA8XX SoCs feature a set of registers allowing to change the priority of all
+  peripherals classified as masters.
+
+  Documentation:
+  OMAP-L138 (DA850) - http://www.ti.com/lit/ug/spruh82c/spruh82c.pdf
+
+properties:
+  compatible:
+    const: ti,da850-mstpri
+
+  reg:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    mstpri@14110 {
+        compatible = "ti,da850-mstpri";
+        reg = <0x14110 0x0c>;
+    };
diff --git a/Documentation/devicetree/bindings/cache/qcom,llcc.yaml b/Documentation/devicetree/bindings/cache/qcom,llcc.yaml
index 34e3a2d785927..a1340226a0d03 100644
--- a/Documentation/devicetree/bindings/cache/qcom,llcc.yaml
+++ b/Documentation/devicetree/bindings/cache/qcom,llcc.yaml
@@ -19,35 +19,40 @@ description: |
 
 properties:
   compatible:
-    enum:
-      - qcom,eliza-llcc
-      - qcom,glymur-llcc
-      - qcom,hawi-llcc
-      - qcom,ipq5424-llcc
-      - qcom,kaanapali-llcc
-      - qcom,qcs615-llcc
-      - qcom,qcs8300-llcc
-      - qcom,qdu1000-llcc
-      - qcom,sa8775p-llcc
-      - qcom,sar1130p-llcc
-      - qcom,sar2130p-llcc
-      - qcom,sc7180-llcc
-      - qcom,sc7280-llcc
-      - qcom,sc8180x-llcc
-      - qcom,sc8280xp-llcc
-      - qcom,sdm670-llcc
-      - qcom,sdm845-llcc
-      - qcom,shikra-llcc
-      - qcom,sm6350-llcc
-      - qcom,sm7150-llcc
-      - qcom,sm8150-llcc
-      - qcom,sm8250-llcc
-      - qcom,sm8350-llcc
-      - qcom,sm8450-llcc
-      - qcom,sm8550-llcc
-      - qcom,sm8650-llcc
-      - qcom,sm8750-llcc
-      - qcom,x1e80100-llcc
+    oneOf:
+      - enum:
+          - qcom,eliza-llcc
+          - qcom,glymur-llcc
+          - qcom,hawi-llcc
+          - qcom,ipq5424-llcc
+          - qcom,kaanapali-llcc
+          - qcom,qcs615-llcc
+          - qcom,qcs8300-llcc
+          - qcom,qdu1000-llcc
+          - qcom,sa8775p-llcc
+          - qcom,sar1130p-llcc
+          - qcom,sar2130p-llcc
+          - qcom,sc7180-llcc
+          - qcom,sc7280-llcc
+          - qcom,sc8180x-llcc
+          - qcom,sc8280xp-llcc
+          - qcom,sdm670-llcc
+          - qcom,sdm845-llcc
+          - qcom,shikra-llcc
+          - qcom,sm6350-llcc
+          - qcom,sm7150-llcc
+          - qcom,sm8150-llcc
+          - qcom,sm8250-llcc
+          - qcom,sm8350-llcc
+          - qcom,sm8450-llcc
+          - qcom,sm8550-llcc
+          - qcom,sm8650-llcc
+          - qcom,sm8750-llcc
+          - qcom,x1e80100-llcc
+      - items:
+          - enum:
+              - qcom,maili-llcc
+          - const: qcom,hawi-llcc
 
   reg:
     minItems: 1
diff --git a/Documentation/devicetree/bindings/clock/qcom,dispcc-sm8x50.yaml b/Documentation/devicetree/bindings/clock/qcom,dispcc-sm8x50.yaml
index 53a5ab3191591..6863db9bd0928 100644
--- a/Documentation/devicetree/bindings/clock/qcom,dispcc-sm8x50.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,dispcc-sm8x50.yaml
@@ -13,7 +13,7 @@ description: |
   Qualcomm display clock control module provides the clocks, resets and power
   domains on SM8150/SM8250/SM8350.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,dispcc-sm8150.h
     include/dt-bindings/clock/qcom,dispcc-sm8250.h
     include/dt-bindings/clock/qcom,dispcc-sm8350.h
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-apq8064.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-apq8064.yaml
index 27df7e3e5bf36..68532244901e1 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-apq8064.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-apq8064.yaml
@@ -14,7 +14,7 @@ description: |
   Qualcomm global clock control module provides the clocks, resets and power
   domains on APQ8064.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,gcc-msm8960.h
     include/dt-bindings/reset/qcom,gcc-msm8960.h
 
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-apq8084.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-apq8084.yaml
index 0a0a26d9beab1..1c022e75fd719 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-apq8084.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-apq8084.yaml
@@ -14,7 +14,7 @@ description: |
   Qualcomm global clock control module provides the clocks, resets and power
   domains on APQ8084.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,gcc-apq8084.h
     include/dt-bindings/reset/qcom,gcc-apq8084.h
 
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-ipq6018.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-ipq6018.yaml
index 4d2614d4f3686..c7fb84438db72 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-ipq6018.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-ipq6018.yaml
@@ -15,7 +15,7 @@ description: |
   Qualcomm global clock control module provides the clocks, resets and power
   domains on IPQ6018.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,gcc-ipq6018.h
     include/dt-bindings/reset/qcom,gcc-ipq6018.h
 
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-ipq8064.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-ipq8064.yaml
index a71557395c014..b4d3175780bc8 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-ipq8064.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-ipq8064.yaml
@@ -13,7 +13,7 @@ description: |
   Qualcomm global clock control module provides the clocks, resets and power
   domains on IPQ8064.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,gcc-ipq806x.h (qcom,gcc-ipq8064)
     include/dt-bindings/reset/qcom,gcc-ipq806x.h (qcom,gcc-ipq8064)
 
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-mdm9607.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-mdm9607.yaml
index d7da30b0e7ee6..0a7be7583bdd9 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-mdm9607.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-mdm9607.yaml
@@ -14,7 +14,7 @@ description: |
   Qualcomm global clock control module provides the clocks, resets and power
   domains.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,gcc-mdm9607.h
 
 allOf:
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-mdm9615.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-mdm9615.yaml
index 418dea31eb62d..0656d5ee448d4 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-mdm9615.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-mdm9615.yaml
@@ -14,7 +14,7 @@ description: |
   Qualcomm global clock control module provides the clocks, resets and power
   domains.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,gcc-mdm9615.h
 
 allOf:
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8660.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8660.yaml
index e03b6d0acdb6e..70c9da1f35c20 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8660.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8660.yaml
@@ -14,7 +14,7 @@ description: |
   Qualcomm global clock control module provides the clocks and resets on
   MSM8660
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,gcc-msm8660.h
     include/dt-bindings/reset/qcom,gcc-msm8660.h
 
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8909.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8909.yaml
index ce1f5a60bd8c9..2edb6c251d999 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8909.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8909.yaml
@@ -13,7 +13,7 @@ description: |
   Qualcomm global clock control module provides the clocks, resets and power
   domains on MSM8909, MSM8917 or QM215.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,gcc-msm8909.h
     include/dt-bindings/clock/qcom,gcc-msm8917.h
 
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8916.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8916.yaml
index 258b6b93deca7..af4b639ea8c34 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8916.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8916.yaml
@@ -14,7 +14,7 @@ description: |
   Qualcomm global clock control module provides the clocks, resets and power
   domains on MSM8916 or MSM8939.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,gcc-msm8916.h
     include/dt-bindings/clock/qcom,gcc-msm8939.h
     include/dt-bindings/reset/qcom,gcc-msm8916.h
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8953.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8953.yaml
index ced3118c85800..9f2b970bfb483 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8953.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8953.yaml
@@ -7,15 +7,13 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Qualcomm Global Clock & Reset Controller on MSM8937, MSM8940, MSM8953 and SDM439
 
 maintainers:
-  - Adam Skladowski <a_skl39@protonmail.com>
-  - Sireesh Kodali <sireeshkodali@protonmail.com>
   - Barnabas Czeman <barnabas.czeman@mainlining.org>
 
 description: |
   Qualcomm global clock control module provides the clocks, resets and power
   domains on MSM8937, MSM8940, MSM8953 or SDM439.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,gcc-msm8917.h
     include/dt-bindings/clock/qcom,gcc-msm8953.h
 
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8974.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8974.yaml
index 929fafc84c195..378dfe7854ac7 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8974.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8974.yaml
@@ -15,7 +15,7 @@ description: |
   Qualcomm global clock control module provides the clocks, resets and power
   domains on MSM8974 (all variants) and MSM8226.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,gcc-msm8974.h (qcom,gcc-msm8226 and qcom,gcc-msm8974)
     include/dt-bindings/reset/qcom,gcc-msm8974.h (qcom,gcc-msm8226 and qcom,gcc-msm8974)
 
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sdm660.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sdm660.yaml
index 724ce0491118a..72aaf699cf705 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-sdm660.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sdm660.yaml
@@ -14,7 +14,7 @@ description: |
   Qualcomm global clock control module provides the clocks, resets and power
   domains on SDM630, SDM636 and SDM660
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,gcc-sdm660.h  (qcom,gcc-sdm630 and qcom,gcc-sdm660)
 
 $ref: qcom,gcc.yaml#
diff --git a/Documentation/devicetree/bindings/clock/qcom,glymur-evacc.yaml b/Documentation/devicetree/bindings/clock/qcom,glymur-evacc.yaml
new file mode 100644
index 0000000000000..fb0bc1acc920e
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/qcom,glymur-evacc.yaml
@@ -0,0 +1,72 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/qcom,glymur-evacc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm EVA Clock & Reset Controller on Glymur SoC
+
+maintainers:
+  - Taniya Das <taniya.das@oss.qualcomm.com>
+
+description: |
+  Qualcomm EVA clock control module which supports the clocks, resets and
+  power domains for the EVA instances on Glymur SoC.
+
+  See also:
+  - include/dt-bindings/clock/qcom,glymur-evacc.h
+
+properties:
+  compatible:
+    const: qcom,glymur-evacc
+
+  clocks:
+    items:
+      - description: Interface clock from GCC
+      - description: Board XO source
+      - description: Sleep clock source
+
+  power-domains:
+    items:
+      - description: MMCX power domain
+      - description: MXC power domain
+
+  required-opps:
+    description:
+      Required OPP nodes for the MMCX and MXC power domains.
+    items:
+      - description: MMCX performance point
+      - description: MXC performance point
+
+required:
+  - compatible
+  - clocks
+  - power-domains
+  - required-opps
+  - '#power-domain-cells'
+
+allOf:
+  - $ref: qcom,gcc.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/qcom,glymur-gcc.h>
+    #include <dt-bindings/clock/qcom,rpmh.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
+    clock-controller@ab00000 {
+        compatible = "qcom,glymur-evacc";
+        reg = <0x0ab00000 0x10000>;
+        clocks = <&gcc GCC_EVA_AHB_CLK>,
+                 <&rpmhcc RPMH_CXO_CLK>,
+                 <&sleep_clk>;
+        power-domains = <&rpmhpd RPMHPD_MMCX>,
+                        <&rpmhpd RPMHPD_MXC>;
+        required-opps = <&rpmhpd_opp_low_svs>,
+                        <&rpmhpd_opp_low_svs>;
+        #clock-cells = <1>;
+        #reset-cells = <1>;
+        #power-domain-cells = <1>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/clock/qcom,gpucc.yaml b/Documentation/devicetree/bindings/clock/qcom,gpucc.yaml
index 4cdff6161bf0b..3ac4419009a9e 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gpucc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gpucc.yaml
@@ -14,7 +14,7 @@ description: |
   Qualcomm graphics clock control module provides the clocks, resets and power
   domains on Qualcomm SoCs.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,gpucc-sdm845.h
     include/dt-bindings/clock/qcom,gpucc-sa8775p.h
     include/dt-bindings/clock/qcom,gpucc-sc7180.h
diff --git a/Documentation/devicetree/bindings/clock/qcom,ipq5018-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,ipq5018-gcc.yaml
index 489d0fc5607c0..9925b931ecad6 100644
--- a/Documentation/devicetree/bindings/clock/qcom,ipq5018-gcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,ipq5018-gcc.yaml
@@ -13,7 +13,7 @@ description: |
   Qualcomm global clock control module provides the clocks, resets and power
   domains on IPQ5018
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,ipq5018-gcc.h
     include/dt-bindings/reset/qcom,ipq5018-gcc.h
 
diff --git a/Documentation/devicetree/bindings/clock/qcom,ipq9574-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,ipq9574-gcc.yaml
index 27ae9938febc0..1337bd87fdddb 100644
--- a/Documentation/devicetree/bindings/clock/qcom,ipq9574-gcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,ipq9574-gcc.yaml
@@ -8,13 +8,12 @@ title: Qualcomm Global Clock & Reset Controller on IPQ9574
 
 maintainers:
   - Bjorn Andersson <andersson@kernel.org>
-  - Anusha Rao <quic_anusha@quicinc.com>
 
 description: |
   Qualcomm global clock control module provides the clocks, resets and power
   domains on IPQ9574
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,ipq9574-gcc.h
     include/dt-bindings/reset/qcom,ipq9574-gcc.h
 
diff --git a/Documentation/devicetree/bindings/clock/qcom,ipq9574-nsscc.yaml b/Documentation/devicetree/bindings/clock/qcom,ipq9574-nsscc.yaml
index 7ff4ff3587cad..a35bbc7b19247 100644
--- a/Documentation/devicetree/bindings/clock/qcom,ipq9574-nsscc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,ipq9574-nsscc.yaml
@@ -8,7 +8,6 @@ title: Qualcomm Networking Sub System Clock & Reset Controller on IPQ9574 and IP
 
 maintainers:
   - Bjorn Andersson <andersson@kernel.org>
-  - Anusha Rao <quic_anusha@quicinc.com>
 
 description: |
   Qualcomm networking sub system clock control module provides the clocks,
diff --git a/Documentation/devicetree/bindings/clock/qcom,qca8k-nsscc.yaml b/Documentation/devicetree/bindings/clock/qcom,qca8k-nsscc.yaml
index 480745349a5d1..8d7a586093725 100644
--- a/Documentation/devicetree/bindings/clock/qcom,qca8k-nsscc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,qca8k-nsscc.yaml
@@ -14,7 +14,7 @@ description: |
   Qualcomm NSS clock control module provides the clocks and resets
   on QCA8386(switch mode)/QCA8084(PHY mode)
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,qca8k-nsscc.h
     include/dt-bindings/reset/qcom,qca8k-nsscc.h
 
diff --git a/Documentation/devicetree/bindings/clock/qcom,qcm2290-gpucc.yaml b/Documentation/devicetree/bindings/clock/qcom,qcm2290-gpucc.yaml
index 734880805c1b9..bedbdabef6728 100644
--- a/Documentation/devicetree/bindings/clock/qcom,qcm2290-gpucc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,qcm2290-gpucc.yaml
@@ -13,7 +13,7 @@ description: |
   Qualcomm graphics clock control module provides the clocks, resets and power
   domains on Qualcomm SoCs.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,qcm2290-gpucc.h
 
 properties:
diff --git a/Documentation/devicetree/bindings/clock/qcom,rpmcc.yaml b/Documentation/devicetree/bindings/clock/qcom,rpmcc.yaml
index ab97d4b7dba8b..17332aa558862 100644
--- a/Documentation/devicetree/bindings/clock/qcom,rpmcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,rpmcc.yaml
@@ -12,7 +12,7 @@ maintainers:
 
 description: |
   The clock enumerators are defined in <dt-bindings/clock/qcom,rpmcc.h> and
-  come in pairs:: FOO_CLK followed by FOO_A_CLK. The latter clock is
+  come in pairs: FOO_CLK followed by FOO_A_CLK. The latter clock is
   an "active" clock, which means that the consumer only care that the clock is
   available when the apps CPU subsystem is active, i.e. not suspended or in
   deep idle. If it is important that the clock keeps running during system
@@ -21,35 +21,41 @@ description: |
 
 properties:
   compatible:
-    items:
-      - enum:
-          - qcom,rpmcc-apq8060
-          - qcom,rpmcc-apq8064
-          - qcom,rpmcc-ipq806x
-          - qcom,rpmcc-mdm9607
-          - qcom,rpmcc-msm8226
-          - qcom,rpmcc-msm8660
-          - qcom,rpmcc-msm8909
-          - qcom,rpmcc-msm8916
-          - qcom,rpmcc-msm8917
-          - qcom,rpmcc-msm8936
-          - qcom,rpmcc-msm8937
-          - qcom,rpmcc-msm8940
-          - qcom,rpmcc-msm8953
-          - qcom,rpmcc-msm8974
-          - qcom,rpmcc-msm8976
-          - qcom,rpmcc-msm8992
-          - qcom,rpmcc-msm8994
-          - qcom,rpmcc-msm8996
-          - qcom,rpmcc-msm8998
-          - qcom,rpmcc-qcm2290
-          - qcom,rpmcc-qcs404
-          - qcom,rpmcc-sdm429
-          - qcom,rpmcc-sdm660
-          - qcom,rpmcc-sm6115
-          - qcom,rpmcc-sm6125
-          - qcom,rpmcc-sm6375
-      - const: qcom,rpmcc
+    oneOf:
+      - items:
+          - enum:
+              - qcom,rpmcc-apq8060
+              - qcom,rpmcc-apq8064
+              - qcom,rpmcc-ipq806x
+              - qcom,rpmcc-mdm9607
+              - qcom,rpmcc-msm8226
+              - qcom,rpmcc-msm8660
+              - qcom,rpmcc-msm8909
+              - qcom,rpmcc-msm8916
+              - qcom,rpmcc-msm8917
+              - qcom,rpmcc-msm8936
+              - qcom,rpmcc-msm8937
+              - qcom,rpmcc-msm8940
+              - qcom,rpmcc-msm8953
+              - qcom,rpmcc-msm8974
+              - qcom,rpmcc-msm8976
+              - qcom,rpmcc-msm8992
+              - qcom,rpmcc-msm8994
+              - qcom,rpmcc-msm8996
+              - qcom,rpmcc-msm8998
+              - qcom,rpmcc-qcm2290
+              - qcom,rpmcc-qcs404
+              - qcom,rpmcc-sdm429
+              - qcom,rpmcc-sdm660
+              - qcom,rpmcc-sm6115
+              - qcom,rpmcc-sm6125
+              - qcom,rpmcc-sm6375
+          - const: qcom,rpmcc
+      - items:
+          - enum:
+              - qcom,rpmcc-shikra
+          - const: qcom,rpmcc-qcm2290
+          - const: qcom,rpmcc
 
   '#clock-cells':
     const: 1
@@ -126,6 +132,7 @@ allOf:
               - qcom,rpmcc-qcs404
               - qcom,rpmcc-sdm429
               - qcom,rpmcc-sdm660
+              - qcom,rpmcc-shikra
               - qcom,rpmcc-sm6115
               - qcom,rpmcc-sm6125
 
diff --git a/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscorecc.yaml b/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscorecc.yaml
index 99ab9106009f8..fd06ac9bceb91 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscorecc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscorecc.yaml
@@ -13,7 +13,7 @@ description: |
   Qualcomm LPASS core and audio clock control module provides the clocks and
   power domains on SC7280.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,lpasscorecc-sc7280.h
     include/dt-bindings/clock/qcom,lpassaudiocc-sc7280.h
 
diff --git a/Documentation/devicetree/bindings/clock/qcom,sc8280xp-lpasscc.yaml b/Documentation/devicetree/bindings/clock/qcom,sc8280xp-lpasscc.yaml
index 273d66e245c5f..f235b4e24cc70 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sc8280xp-lpasscc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sc8280xp-lpasscc.yaml
@@ -13,7 +13,7 @@ description: |
   Qualcomm LPASS core and audio clock control module provides the clocks,
   and reset on SC8280XP.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,lpasscc-sc8280xp.h
 
 properties:
diff --git a/Documentation/devicetree/bindings/clock/qcom,shikra-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,shikra-gcc.yaml
new file mode 100644
index 0000000000000..da6eebfa84c22
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/qcom,shikra-gcc.yaml
@@ -0,0 +1,70 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/qcom,shikra-gcc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Global Clock & Reset Controller on Qualcomm Shikra SoC
+
+maintainers:
+  - Imran Shaik <imran.shaik@oss.qualcomm.com>
+  - Taniya Das <taniya.das@oss.qualcomm.com>
+
+description: |
+  Global clock control module provides the clocks, resets and power
+  domains on Qualcomm Shikra SoC platform.
+
+  See also: include/dt-bindings/clock/qcom,shikra-gcc.h
+
+properties:
+  compatible:
+    const: qcom,shikra-gcc
+
+  clocks:
+    items:
+      - description: Board XO source
+      - description: Sleep clock source
+      - description: EMAC0 sgmiiphy mac rclk source
+      - description: EMAC0 sgmiiphy mac tclk source
+      - description: EMAC1 sgmiiphy mac rclk source
+      - description: EMAC1 sgmiiphy mac tclk source
+      - description: PCIE Pipe clock source
+      - description: USB3 phy wrapper pipe clock source
+
+  power-domains:
+    items:
+      - description: CX domain
+
+required:
+  - compatible
+  - clocks
+  - power-domains
+  - '#power-domain-cells'
+
+allOf:
+  - $ref: qcom,gcc.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/qcom,rpmcc.h>
+    #include <dt-bindings/power/qcom-rpmpd.h>
+    clock-controller@1400000 {
+        compatible = "qcom,shikra-gcc";
+        reg = <0x01400000 0x1f0000>;
+        clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>,
+                 <&sleep_clk>,
+                 <&emac0_sgmiiphy_rclk>,
+                 <&emac0_sgmiiphy_tclk>,
+                 <&emac1_sgmiiphy_rclk>,
+                 <&emac1_sgmiiphy_tclk>,
+                 <&pcie_pipe_clk>,
+                 <&usb3_phy_wrapper_gcc_usb30_pipe_clk>;
+        power-domains = <&rpmpd RPMPD_VDDCX>;
+        #clock-cells = <1>;
+        #power-domain-cells = <1>;
+        #reset-cells = <1>;
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm6115-lpasscc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm6115-lpasscc.yaml
index 8cbab3fbb6601..d7e1938b5e1b3 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sm6115-lpasscc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sm6115-lpasscc.yaml
@@ -14,7 +14,7 @@ description: |
   Qualcomm LPASS core and audio clock controllers provide audio-related resets
   on SM6115 and its derivatives.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,sm6115-lpasscc.h
 
 properties:
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8350-videocc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8350-videocc.yaml
index 5c2ecec0624e3..a986ab4ce7c74 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sm8350-videocc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sm8350-videocc.yaml
@@ -13,7 +13,7 @@ description: |
   Qualcomm video clock control module provides the clocks, resets and power
   domains on Qualcomm SoCs.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,videocc-sm8350.h
     include/dt-bindings/reset/qcom,videocc-sm8350.h
 
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8450-camcc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8450-camcc.yaml
index 8492a7ef73245..b41247b257b19 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sm8450-camcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sm8450-camcc.yaml
@@ -70,13 +70,11 @@ allOf:
   - if:
       properties:
         compatible:
-          contains:
-            enum:
-              - qcom,kaanapali-cambistmclkcc
-              - qcom,kaanapali-camcc
-              - qcom,sc8280xp-camcc
-              - qcom,sm8450-camcc
-              - qcom,sm8550-camcc
+          not:
+            contains:
+              enum:
+                - qcom,sm8475-camcc
+                - qcom,sm8650-camcc
     then:
       required:
         - required-opps
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml
index 5d77029bfaf88..1e1fec1cd7ce9 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml
@@ -16,6 +16,7 @@ description: |
 
   See also:
     include/dt-bindings/clock/qcom,glymur-videocc.h
+    include/dt-bindings/clock/qcom,hawi-videocc.h
     include/dt-bindings/clock/qcom,kaanapali-videocc.h
     include/dt-bindings/clock/qcom,sm8450-videocc.h
     include/dt-bindings/clock/qcom,sm8650-videocc.h
@@ -26,6 +27,7 @@ properties:
   compatible:
     enum:
       - qcom,glymur-videocc
+      - qcom,hawi-videocc
       - qcom,kaanapali-videocc
       - qcom,sm8450-videocc
       - qcom,sm8475-videocc
@@ -65,14 +67,11 @@ allOf:
   - if:
       properties:
         compatible:
-          contains:
-            enum:
-              - qcom,glymur-videocc
-              - qcom,kaanapali-videocc
-              - qcom,sm8450-videocc
-              - qcom,sm8550-videocc
-              - qcom,sm8750-videocc
-              - qcom,x1p42100-videocc
+          not:
+            contains:
+              enum:
+                - qcom,sm8475-videocc
+                - qcom,sm8650-videocc
     then:
       required:
         - required-opps
diff --git a/Documentation/devicetree/bindings/clock/qcom,videocc.yaml b/Documentation/devicetree/bindings/clock/qcom,videocc.yaml
index f4ff9acef9d5f..124d259fc85e8 100644
--- a/Documentation/devicetree/bindings/clock/qcom,videocc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,videocc.yaml
@@ -13,7 +13,7 @@ description: |
   Qualcomm video clock control module provides the clocks, resets and power
   domains on Qualcomm SoCs.
 
-  See also::
+  See also:
     include/dt-bindings/clock/qcom,sm6350-videocc.h
     include/dt-bindings/clock/qcom,videocc-sc7180.h
     include/dt-bindings/clock/qcom,videocc-sc7280.h
diff --git a/Documentation/devicetree/bindings/clock/qcom,x1e80100-camcc.yaml b/Documentation/devicetree/bindings/clock/qcom,x1e80100-camcc.yaml
index b28614186cc09..e7cb8cfa957a4 100644
--- a/Documentation/devicetree/bindings/clock/qcom,x1e80100-camcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,x1e80100-camcc.yaml
@@ -8,12 +8,14 @@ title: Qualcomm Camera Clock & Reset Controller on x1e80100
 
 maintainers:
   - Bryan O'Donoghue <bryan.odonoghue@linaro.org>
+  - Jagadeesh Kona <jagadeesh.kona@oss.qualcomm.com>
 
 description: |
   Qualcomm camera clock control module provides the clocks, resets and power
   domains on x1e80100.
 
   See also:
+    include/dt-bindings/clock/qcom,glymur-camcc.h
     include/dt-bindings/clock/qcom,x1e80100-camcc.h
 
 allOf:
@@ -22,6 +24,7 @@ allOf:
 properties:
   compatible:
     enum:
+      - qcom,glymur-camcc
       - qcom,x1e80100-camcc
       - qcom,x1p42100-camcc
 
diff --git a/Documentation/devicetree/bindings/clock/renesas,rzv2h-cpg.yaml b/Documentation/devicetree/bindings/clock/renesas,rzv2h-cpg.yaml
index f261445bf341c..dd3e66a4559bc 100644
--- a/Documentation/devicetree/bindings/clock/renesas,rzv2h-cpg.yaml
+++ b/Documentation/devicetree/bindings/clock/renesas,rzv2h-cpg.yaml
@@ -26,16 +26,22 @@ properties:
     maxItems: 1
 
   clocks:
+    minItems: 3
     items:
       - description: AUDIO_EXTAL clock input
       - description: RTXIN clock input
       - description: QEXTAL clock input
+      - description: AUDIO_CLKB clock input
+      - description: AUDIO_CLKC clock input
 
   clock-names:
+    minItems: 3
     items:
       - const: audio_extal
       - const: rtxin
       - const: qextal
+      - const: audio_clkb
+      - const: audio_clkc
 
   '#clock-cells':
     description: |
diff --git a/Documentation/devicetree/bindings/clock/renesas,versaclock7.yaml b/Documentation/devicetree/bindings/clock/renesas,versaclock7.yaml
index b339f1f9f072c..990d287d0a90f 100644
--- a/Documentation/devicetree/bindings/clock/renesas,versaclock7.yaml
+++ b/Documentation/devicetree/bindings/clock/renesas,versaclock7.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Renesas Versaclock7 Programmable Clock
 
 maintainers:
-  - Alex Helms <alexander.helms.jy@renesas.com>
+  - Biju Das <biju.das.jz@bp.renesas.com>
 
 description: |
   Renesas Versaclock7 is a family of configurable clock generator and
diff --git a/Documentation/devicetree/bindings/clock/samsung,exynos5260-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynos5260-clock.yaml
index b05f83533e3de..56ab972c3da5f 100644
--- a/Documentation/devicetree/bindings/clock/samsung,exynos5260-clock.yaml
+++ b/Documentation/devicetree/bindings/clock/samsung,exynos5260-clock.yaml
@@ -14,17 +14,17 @@ maintainers:
 
 description: |
   Expected external clocks, defined in DTS as fixed-rate clocks with a matching
-  name::
+  name:
     - "fin_pll" - PLL input clock from XXTI
     - "xrtcxti" - input clock from XRTCXTI
     - "ioclk_pcm_extclk" - pcm external operation clock
     - "ioclk_spdif_extclk" - spdif external operation clock
     - "ioclk_i2s_cdclk" - i2s0 codec clock
 
-  Phy clocks::
+  Phy clocks:
   There are several clocks which are generated by specific PHYs.  These clocks
   are fed into the clock controller and then routed to the hardware blocks.
-  These clocks are defined as fixed clocks in the driver with following names::
+  These clocks are defined as fixed clocks in the driver with following names:
     - "phyclk_dptx_phy_ch3_txd_clk" - dp phy clock for channel 3
     - "phyclk_dptx_phy_ch2_txd_clk" - dp phy clock for channel 2
     - "phyclk_dptx_phy_ch1_txd_clk" - dp phy clock for channel 1
diff --git a/Documentation/devicetree/bindings/clock/samsung,exynos5410-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynos5410-clock.yaml
index b737c9d35a1c2..1d907dd8fbf10 100644
--- a/Documentation/devicetree/bindings/clock/samsung,exynos5410-clock.yaml
+++ b/Documentation/devicetree/bindings/clock/samsung,exynos5410-clock.yaml
@@ -14,7 +14,7 @@ maintainers:
 
 description: |
   Expected external clocks, defined in DTS as fixed-rate clocks with a matching
-  name::
+  name:
     - "fin_pll" - PLL input clock from XXTI
 
   All available clocks are defined as preprocessor macros in
diff --git a/Documentation/devicetree/bindings/clock/samsung,exynos5433-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynos5433-clock.yaml
index 3f9326e09f79e..8a289f1e2ace0 100644
--- a/Documentation/devicetree/bindings/clock/samsung,exynos5433-clock.yaml
+++ b/Documentation/devicetree/bindings/clock/samsung,exynos5433-clock.yaml
@@ -14,7 +14,7 @@ maintainers:
 
 description: |
   Expected external clocks, defined in DTS as fixed-rate clocks with a matching
-  name::
+  name:
     - "oscclk" - PLL input clock from XXTI
 
   All available clocks are defined as preprocessor macros in
diff --git a/Documentation/devicetree/bindings/clock/samsung,exynos7-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynos7-clock.yaml
index c137c6744ef91..a51cd4fafb41e 100644
--- a/Documentation/devicetree/bindings/clock/samsung,exynos7-clock.yaml
+++ b/Documentation/devicetree/bindings/clock/samsung,exynos7-clock.yaml
@@ -14,7 +14,7 @@ maintainers:
 
 description: |
   Expected external clocks, defined in DTS as fixed-rate clocks with a matching
-  name::
+  name:
     - "fin_pll" - PLL input clock from XXTI
 
   All available clocks are defined as preprocessor macros in
diff --git a/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml
index cdc5ded59fe5e..68c2fd3187652 100644
--- a/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml
+++ b/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml
@@ -17,7 +17,7 @@ description: |
   Exynos850 clock controller is comprised of several CMU units, generating
   clocks for different domains. Those CMU units are modeled as separate device
   tree nodes, and might depend on each other. Root clocks in that clock tree are
-  two external clocks:: OSCCLK (26 MHz) and RTCCLK (32768 Hz). Those external
+  two external clocks: OSCCLK (26 MHz) and RTCCLK (32768 Hz). Those external
   clocks must be defined as fixed-rate clocks in dts.
 
   CMU_TOP is a top-level CMU, where all base clocks are prepared using PLLs and
diff --git a/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml
index 32f39e543b362..3dcdfa7a87929 100644
--- a/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml
+++ b/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Samsung Exynos Auto v9 SoC clock controller
 
 maintainers:
-  - Chanho Park <chanho61.park@samsung.com>
+  - Alim Akhtar <alim.akhtar@samsung.com>
   - Chanwoo Choi <cw00.choi@samsung.com>
   - Krzysztof Kozlowski <krzk@kernel.org>
   - Sylwester Nawrocki <s.nawrocki@samsung.com>
@@ -17,7 +17,7 @@ description: |
   Exynos Auto v9 clock controller is comprised of several CMU units, generating
   clocks for different domains. Those CMU units are modeled as separate device
   tree nodes, and might depend on each other. Root clocks in that clock tree are
-  two external clocks:: OSCCLK/XTCXO (26 MHz) and RTCCLK/XrtcXTI (32768 Hz).
+  two external clocks: OSCCLK/XTCXO (26 MHz) and RTCCLK/XrtcXTI (32768 Hz).
   The external OSCCLK must be defined as fixed-rate clock in dts.
 
   CMU_TOP is a top-level CMU, where all base clocks are prepared using PLLs and
diff --git a/Documentation/devicetree/bindings/clock/samsung,exynosautov920-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynosautov920-clock.yaml
index 6b1fc61a2ff94..475db824d4d33 100644
--- a/Documentation/devicetree/bindings/clock/samsung,exynosautov920-clock.yaml
+++ b/Documentation/devicetree/bindings/clock/samsung,exynosautov920-clock.yaml
@@ -17,7 +17,7 @@ description: |
   ExynosAuto v920 clock controller is comprised of several CMU units, generating
   clocks for different domains. Those CMU units are modeled as separate device
   tree nodes, and might depend on each other. Root clocks in that clock tree are
-  two external clocks:: OSCCLK/XTCXO (38.4 MHz) and RTCCLK/XrtcXTI (32768 Hz).
+  two external clocks: OSCCLK/XTCXO (38.4 MHz) and RTCCLK/XrtcXTI (32768 Hz).
   The external OSCCLK must be defined as fixed-rate clock in dts.
 
   CMU_TOP is a top-level CMU, where all base clocks are prepared using PLLs and
diff --git a/Documentation/devicetree/bindings/clock/samsung,s5pv210-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,s5pv210-clock.yaml
index 67a33665cf00b..b1617d96d3fbe 100644
--- a/Documentation/devicetree/bindings/clock/samsung,s5pv210-clock.yaml
+++ b/Documentation/devicetree/bindings/clock/samsung,s5pv210-clock.yaml
@@ -14,7 +14,7 @@ maintainers:
 
 description: |
   Expected external clocks, defined in DTS as fixed-rate clocks with a matching
-  name::
+  name:
     - "xxti" - external crystal oscillator connected to XXTI and XXTO pins of
       the SoC,
     - "xusbxti" - external crystal oscillator connected to XUSBXTI and XUSBXTO
diff --git a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml
index db895c50e2d25..7be14e99be281 100644
--- a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml
+++ b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml
@@ -16,7 +16,9 @@ properties:
           - qcom,eliza-inline-crypto-engine
           - qcom,hawi-inline-crypto-engine
           - qcom,kaanapali-inline-crypto-engine
+          - qcom,maili-inline-crypto-engine
           - qcom,milos-inline-crypto-engine
+          - qcom,nord-inline-crypto-engine
           - qcom,qcs8300-inline-crypto-engine
           - qcom,sa8775p-inline-crypto-engine
           - qcom,sc7180-inline-crypto-engine
@@ -62,7 +64,10 @@ allOf:
           contains:
             enum:
               - qcom,eliza-inline-crypto-engine
+              - qcom,hawi-inline-crypto-engine
+              - qcom,maili-inline-crypto-engine
               - qcom,milos-inline-crypto-engine
+              - qcom,nord-inline-crypto-engine
 
     then:
       required:
diff --git a/Documentation/devicetree/bindings/crypto/qcom,prng.yaml b/Documentation/devicetree/bindings/crypto/qcom,prng.yaml
index dc270c8aedf3a..6116289ec413a 100644
--- a/Documentation/devicetree/bindings/crypto/qcom,prng.yaml
+++ b/Documentation/devicetree/bindings/crypto/qcom,prng.yaml
@@ -23,6 +23,7 @@ properties:
               - qcom,ipq5424-trng
               - qcom,ipq9574-trng
               - qcom,kaanapali-trng
+              - qcom,maili-trng
               - qcom,milos-trng
               - qcom,nord-trng
               - qcom,qcs615-trng
diff --git a/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml b/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml
index dbc0613e427ed..395425a70db8f 100644
--- a/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml
+++ b/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml
@@ -73,16 +73,16 @@ properties:
 
   clocks:
     description: |
-      Several clocks are used, depending on the variant. Typical ones are::
-       - bus:: Display AHB clock.
-       - byte:: Display byte clock.
-       - byte_intf:: Display byte interface clock.
-       - core:: Display core clock.
-       - core_mss:: Core MultiMedia SubSystem clock.
-       - iface:: Display AXI clock.
-       - mdp_core:: MDP Core clock.
-       - mnoc:: MNOC clock
-       - pixel:: Display pixel clock.
+      Several clocks are used, depending on the variant. Typical ones are:
+       - bus: Display AHB clock.
+       - byte: Display byte clock.
+       - byte_intf: Display byte interface clock.
+       - core: Display core clock.
+       - core_mss: Core MultiMedia SubSystem clock.
+       - iface: Display AXI clock.
+       - mdp_core: MDP Core clock.
+       - mnoc: MNOC clock
+       - pixel: Display pixel clock.
     minItems: 3
     maxItems: 12
 
diff --git a/Documentation/devicetree/bindings/display/panel/anbernic,td4310.yaml b/Documentation/devicetree/bindings/display/panel/anbernic,td4310.yaml
new file mode 100644
index 0000000000000..f820dc2f732bf
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/anbernic,td4310.yaml
@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/panel/anbernic,td4310.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Anbernic TD4310 Based Panels
+
+maintainers:
+  - Chris Morgan <macromorgan@hotmail.com>
+
+description:
+  Anbernic TD4310 Based Panels, such as the RG-Vita-Pro panel
+  (a 1080x1920 5.5 inch panel).
+
+allOf:
+  - $ref: panel-common.yaml#
+
+properties:
+  compatible:
+    items:
+      - enum:
+          - anbernic,panel-vita-pro
+      - const: anbernic,td4310
+
+  reg:
+    maxItems: 1
+
+  vdd-supply:
+    description: Panel power supply
+
+required:
+  - compatible
+  - port
+  - reg
+  - reset-gpios
+  - vdd-supply
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    dsi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        panel@0 {
+            compatible = "anbernic,panel-vita-pro", "anbernic,td4310";
+            reg = <0>;
+            backlight = <&backlight>;
+            enable-gpios = <&gpio0 9 GPIO_ACTIVE_HIGH>;
+            reset-gpios = <&gpio0 13 GPIO_ACTIVE_LOW>;
+            rotation = <270>;
+            vdd-supply = <&vdd_lcd>;
+
+            port {
+                endpoint {
+                    remote-endpoint = <&dsi_out>;
+                };
+            };
+        };
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/display/panel/chipone,icna3512.yaml b/Documentation/devicetree/bindings/display/panel/chipone,icna3512.yaml
new file mode 100644
index 0000000000000..90e69f30cd91c
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/chipone,icna3512.yaml
@@ -0,0 +1,79 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/panel/chipone,icna3512.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Chipone ICNA3512 and ICNA3520 display drivers
+
+maintainers:
+  - Neil Armstrong <neil.armstrong@linaro.org>
+
+description:
+  The Chipone ICNA3512 and ICNA3520 are DDICs connected
+  using a MIPI-DSI video interface.
+
+allOf:
+  - $ref: panel-common.yaml#
+
+properties:
+  compatible:
+    oneOf:
+      - items:
+          - enum:
+              - ayaneo,pocketds-panel-top
+              - ayntec,odin2portal-panel
+          - const: chipone,icna3512
+
+      - items:
+          - enum:
+              - ayntec,odin3-panel
+              - ayntec,thor-panel-top
+          - const: chipone,icna3520
+
+  reg:
+    maxItems: 1
+    description: DSI virtual channel
+
+  vdd-supply: true
+  vddio-supply: true
+  vci-supply: true
+  disp-supply: true
+  blvdd-supply: true
+
+  port: true
+  reset-gpios: true
+  rotation: true
+
+required:
+  - compatible
+  - reg
+  - vdd-supply
+  - vddio-supply
+  - vci-supply
+  - disp-supply
+  - blvdd-supply
+  - reset-gpios
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    dsi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+        panel@0 {
+            compatible = "ayntec,odin2portal-panel", "chipone,icna3512";
+            reg = <0>;
+            vdd-supply = <&vreg_l11b_1p2>;
+            vddio-supply = <&vreg_l12b_1p8>;
+            vci-supply = <&vreg_l13b_3p0>;
+            disp-supply = <&vdd_disp_2v8>;
+            blvdd-supply = <&vdd_bl_5v0>;
+            reset-gpios = <&tlmm 133 GPIO_ACTIVE_LOW>;
+        };
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/display/panel/himax,hx83121a.yaml b/Documentation/devicetree/bindings/display/panel/himax,hx83121a.yaml
index e067a2f6d0b22..aeca3c9a599c2 100644
--- a/Documentation/devicetree/bindings/display/panel/himax,hx83121a.yaml
+++ b/Documentation/devicetree/bindings/display/panel/himax,hx83121a.yaml
@@ -40,6 +40,9 @@ properties:
   vddi-supply:
     description: power supply for IC
 
+  bl-supply:
+    description: power supply for backlight, in case it's managed via DSC
+
   backlight: true
   ports: true
 
diff --git a/Documentation/devicetree/bindings/display/panel/ilitek,ili7807s.yaml b/Documentation/devicetree/bindings/display/panel/ilitek,ili7807s.yaml
new file mode 100644
index 0000000000000..ba8c5bbf8ffc4
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/ilitek,ili7807s.yaml
@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/panel/ilitek,ili7807s.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Ilitek ILI7807S-based DSI panels
+
+maintainers:
+  - Arpit Saini <arpit.saini@oss.qualcomm.com>
+
+allOf:
+  - $ref: panel-common.yaml#
+
+properties:
+  compatible:
+    items:
+      - enum:
+          - dlc,dlc0697
+      - const: ilitek,ili7807s
+
+  reg:
+    maxItems: 1
+    description: DSI virtual channel
+
+  vddi-supply:
+    description: I/O voltage supply (1.8V)
+
+  avdd-supply:
+    description: Positive LCD bias supply (AVDD), typically +5.5V
+      (range 4.5V to 6.3V)
+
+  avee-supply:
+    description: Negative LCD bias supply (AVEE), typically -5.5V
+      (range -6.3V to -4.5V)
+
+required:
+  - compatible
+  - reg
+  - reset-gpios
+  - vddi-supply
+  - avdd-supply
+  - avee-supply
+  - port
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    dsi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        panel@0 {
+            compatible = "dlc,dlc0697", "ilitek,ili7807s";
+            reg = <0>;
+
+            reset-gpios = <&tlmm 3 GPIO_ACTIVE_LOW>;
+            vddi-supply = <&pm4125_l15>;
+            avdd-supply = <&avdd>;
+            avee-supply = <&avee>;
+
+            port {
+                panel_in: endpoint {
+                    remote-endpoint = <&dsi0_out>;
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/display/panel/ilitek,ili9488.yaml b/Documentation/devicetree/bindings/display/panel/ilitek,ili9488.yaml
new file mode 100644
index 0000000000000..ea74492730228
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/ilitek,ili9488.yaml
@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/panel/ilitek,ili9488.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Ilitek ILI9488 based MIPI-DSI panels
+
+maintainers:
+  - Igor Reznichenko <igor@reznichenko.net>
+
+allOf:
+  - $ref: panel-common.yaml#
+
+properties:
+  compatible:
+    items:
+      - enum:
+          - focuslcds,e35gh-i-mw800cb
+      - const: ilitek,ili9488
+
+  reg:
+    maxItems: 1
+
+  vci-supply: true
+  iovcc-supply: true
+
+required:
+  - compatible
+  - reg
+  - vci-supply
+  - iovcc-supply
+  - reset-gpios
+  - backlight
+  - port
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    dsi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        panel@0 {
+            compatible = "focuslcds,e35gh-i-mw800cb", "ilitek,ili9488";
+            reg = <0>;
+            vci-supply = <&reg_vci_panel>;
+            iovcc-supply = <&reg_iovcc_panel>;
+            reset-gpios = <&gpio3 6 GPIO_ACTIVE_LOW>;
+            backlight = <&pwm_bl>;
+
+            port {
+                panel_in: endpoint {
+                    remote-endpoint = <&dsi_out>;
+                };
+            };
+        };
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/display/panel/renesas,r63419.yaml b/Documentation/devicetree/bindings/display/panel/renesas,r63419.yaml
new file mode 100644
index 0000000000000..adfdd2c300a35
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/renesas,r63419.yaml
@@ -0,0 +1,98 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/panel/renesas,r63419.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Renesas R63419 based dual-DSI video mode Display Panel
+
+maintainers:
+  - Neil Armstrong <neil.armstrong@linaro.org>
+
+description:
+  The Renesas R63419 is a generic DDIC used to control dual-DSI LCD panels.
+
+allOf:
+  - $ref: panel-common-dual.yaml#
+
+properties:
+  compatible:
+    items:
+      - enum:
+          - ayaneo,wt0600-2k
+          - ayaneo,wt0630-2k
+      - const: renesas,r63419
+
+  reg:
+    maxItems: 1
+
+  vdd-supply: true
+  vddio-supply: true
+  vsp-supply: true
+  vsn-supply: true
+  vci-supply: true
+
+  backlight: true
+  reset-gpios: true
+  rotation: true
+  ports: true
+
+required:
+  - compatible
+  - reg
+  - vdd-supply
+  - vddio-supply
+  - vsp-supply
+  - vsn-supply
+  - vci-supply
+  - backlight
+  - reset-gpios
+  - ports
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    dsi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        panel@0 {
+            compatible = "ayaneo,wt0600-2k", "renesas,r63419";
+            reg = <0>;
+
+            reset-gpios = <&gpio 176 GPIO_ACTIVE_LOW>;
+
+            vdd-supply = <&vdd_3v0_lcd>;
+            vddio-supply = <&vdd_1v8_io>;
+            vsn-supply = <&vdd_5v0_neg>;
+            vsp-supply = <&vdd_5v0_pos>;
+            vci-supply = <&vdd_3v0_vci>;
+
+            backlight = <&backlight>;
+
+            rotation = <90>;
+
+            ports {
+                #address-cells = <1>;
+                #size-cells = <0>;
+
+                port@0 {
+                    reg = <0>;
+                    panel_in0: endpoint {
+                        remote-endpoint = <&dsi0_out>;
+                    };
+                };
+
+                port@1 {
+                    reg = <1>;
+                    panel_in1: endpoint {
+                        remote-endpoint = <&dsi1_out>;
+                    };
+                };
+            };
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/display/panel/samsung,atna33xc20.yaml b/Documentation/devicetree/bindings/display/panel/samsung,atna33xc20.yaml
index 1bbe0da3997c3..27536eeaed859 100644
--- a/Documentation/devicetree/bindings/display/panel/samsung,atna33xc20.yaml
+++ b/Documentation/devicetree/bindings/display/panel/samsung,atna33xc20.yaml
@@ -25,6 +25,8 @@ properties:
               - samsung,atna40ct06
               # Samsung 14" WQXGA+ (2880x1800 pixels) eDP AMOLED panel
               - samsung,atna40cu11
+              # Samsung 14" WQXGA+ (2880x1800 pixels) eDP AMOLED panel
+              - samsung,atna40hq08
               # Samsung 14" WQXGA+ (2880×1800 pixels) eDP AMOLED panel
               - samsung,atna40yk20
               # Samsung 14.5" WQXGA+ (2880x1800 pixels) eDP AMOLED panel
diff --git a/Documentation/devicetree/bindings/display/rockchip/rockchip,analogix-dp.yaml b/Documentation/devicetree/bindings/display/rockchip/rockchip,analogix-dp.yaml
index d99b23b88cc59..6548f157fd966 100644
--- a/Documentation/devicetree/bindings/display/rockchip/rockchip,analogix-dp.yaml
+++ b/Documentation/devicetree/bindings/display/rockchip/rockchip,analogix-dp.yaml
@@ -15,18 +15,24 @@ properties:
     enum:
       - rockchip,rk3288-dp
       - rockchip,rk3399-edp
+      - rockchip,rk3576-edp
       - rockchip,rk3588-edp
 
   clocks:
     minItems: 2
-    maxItems: 3
+    items:
+      - description: Reference clock
+      - description: APB bus clock
+      - description: GRF or AHB bus clock
 
   clock-names:
     minItems: 2
     items:
       - const: dp
       - const: pclk
-      - const: grf
+      - enum:
+          - grf
+          - hclk
 
   power-domains:
     maxItems: 1
@@ -65,9 +71,46 @@ allOf:
         compatible:
           contains:
             enum:
+              - rockchip,rk3288-dp
+    then:
+      properties:
+        clocks:
+          maxItems: 2
+        clock-names:
+          maxItems: 2
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - rockchip,rk3399-edp
+    then:
+      properties:
+        clocks:
+          minItems: 3
+        clock-names:
+          items:
+            - const: dp
+            - const: pclk
+            - const: grf
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - rockchip,rk3576-edp
               - rockchip,rk3588-edp
     then:
       properties:
+        clocks:
+          minItems: 3
+        clock-names:
+          items:
+            - const: dp
+            - const: pclk
+            - const: hclk
         resets:
           minItems: 2
         reset-names:
diff --git a/Documentation/devicetree/bindings/display/samsung/samsung,fimd.yaml b/Documentation/devicetree/bindings/display/samsung/samsung,fimd.yaml
index ff685031bb2cf..729705f419bb8 100644
--- a/Documentation/devicetree/bindings/display/samsung/samsung,fimd.yaml
+++ b/Documentation/devicetree/bindings/display/samsung/samsung,fimd.yaml
@@ -41,7 +41,7 @@ properties:
     additionalProperties: false
     description: |
       Timing configuration for lcd i80 interface support.
-      The parameters are defined as::
+      The parameters are defined as:
       VCLK(internal)  __|??????|_____|??????|_____|??????|_____|??????|_____|??
                         :            :            :            :            :
       Address Output  --:<XXXXXXXXXXX:XXXXXXXXXXXX:XXXXXXXXXXXX:XXXXXXXXXXXX:XX
@@ -132,7 +132,7 @@ patternProperties:
   "^port@[0-4]+$":
     $ref: /schemas/graph.yaml#/properties/port
     description: |
-      Contains ports with port with index::
+      Contains ports with port with index:
        0 - for CAMIF0 input,
        1 - for CAMIF1 input,
        2 - for CAMIF2 input,
diff --git a/Documentation/devicetree/bindings/display/tegra/nvidia,tegra124-vic.yaml b/Documentation/devicetree/bindings/display/tegra/nvidia,tegra124-vic.yaml
index 7200095ef19e3..bdf981781bd5f 100644
--- a/Documentation/devicetree/bindings/display/tegra/nvidia,tegra124-vic.yaml
+++ b/Documentation/devicetree/bindings/display/tegra/nvidia,tegra124-vic.yaml
@@ -22,6 +22,7 @@ properties:
           - nvidia,tegra186-vic
           - nvidia,tegra194-vic
           - nvidia,tegra234-vic
+          - nvidia,tegra264-vic
 
       - items:
           - const: nvidia,tegra132-vic
diff --git a/Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-host1x.yaml b/Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-host1x.yaml
index 3563378a01af4..8312b7699cbed 100644
--- a/Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-host1x.yaml
+++ b/Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-host1x.yaml
@@ -25,6 +25,7 @@ properties:
           - nvidia,tegra186-host1x
           - nvidia,tegra194-host1x
           - nvidia,tegra234-host1x
+          - nvidia,tegra264-host1x
 
       - items:
           - const: nvidia,tegra132-host1x
@@ -57,7 +58,8 @@ properties:
     enum: [1, 2]
 
   ranges:
-    maxItems: 1
+    minItems: 1
+    maxItems: 2
 
   clocks:
     description: Must contain one entry, for the module clock. See
@@ -192,6 +194,7 @@ allOf:
           contains:
             enum:
               - nvidia,tegra234-host1x
+              - nvidia,tegra264-host1x
     then:
       properties:
         reg-names:
@@ -239,6 +242,21 @@ allOf:
 
       required:
         - reg-names
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - nvidia,tegra264-host1x
+    then:
+      properties:
+        ranges:
+          minItems: 2
+          maxItems: 2
+    else:
+      properties:
+        ranges:
+          maxItems: 1
 
 examples:
   - |
diff --git a/Documentation/devicetree/bindings/display/verisilicon,dc.yaml b/Documentation/devicetree/bindings/display/verisilicon,dc.yaml
index 9dc35ab973f20..919a900122012 100644
--- a/Documentation/devicetree/bindings/display/verisilicon,dc.yaml
+++ b/Documentation/devicetree/bindings/display/verisilicon,dc.yaml
@@ -75,6 +75,8 @@ required:
   - interrupts
   - clocks
   - clock-names
+  - resets
+  - reset-names
   - ports
 
 additionalProperties: false
diff --git a/Documentation/devicetree/bindings/dma/altr,msgdma.yaml b/Documentation/devicetree/bindings/dma/altr,msgdma.yaml
index 391bf58386025..bea302b894534 100644
--- a/Documentation/devicetree/bindings/dma/altr,msgdma.yaml
+++ b/Documentation/devicetree/bindings/dma/altr,msgdma.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Altera mSGDMA IP core
 
 maintainers:
-  - Olivier Dautricourt <olivierdautricourt@gmail.com>
+  - Adrian Ng Ho Yin <adrian.ho.yin.ng@altera.com>
 
 description: |
   Altera / Intel modular Scatter-Gather Direct Memory Access (mSGDMA)
diff --git a/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml b/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml
index 3708518fe7fc1..bba20e88a6dca 100644
--- a/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml
+++ b/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml
@@ -40,6 +40,7 @@ properties:
       - items:
           - enum:
               - mediatek,mt6991-uart-dma
+              - mediatek,mt8189-uart-dma
               - mediatek,mt8196-uart-dma
           - const: mediatek,mt6985-uart-dma
       - enum:
diff --git a/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml b/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml
index 0923fb189ada9..e72adc172af10 100644
--- a/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml
+++ b/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml
@@ -48,7 +48,7 @@ properties:
 
   iommus:
     minItems: 1
-    maxItems: 6
+    maxItems: 7
 
   num-channels:
     $ref: /schemas/types.yaml#/definitions/uint32
diff --git a/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml b/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml
index 340ae9e91cb01..2c88e927d9d3b 100644
--- a/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml
+++ b/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml
@@ -81,8 +81,13 @@ properties:
     description: Should be the number of framebuffers as configured in h/w.
 
   xlnx,flush-fsync:
-    type: boolean
-    description: Tells which channel to Flush on Frame sync.
+    $ref: /schemas/types.yaml#/definitions/uint32
+    enum: [1, 2, 3]
+    description:
+      Tells which channel to flush on frame sync.
+      1 - flush both channels
+      2 - flush mm2s channel
+      3 - flush s2mm channel
 
   xlnx,sg-length-width:
     $ref: /schemas/types.yaml#/definitions/uint32
@@ -93,7 +98,7 @@ properties:
       Width in bits of the length register as configured in hardware.
 
   xlnx,irq-delay:
-    $ref: /schemas/types.yaml#/definitions/uint32
+    $ref: /schemas/types.yaml#/definitions/uint8
     minimum: 0
     maximum: 255
     description:
@@ -251,7 +256,7 @@ examples:
                       "m_axi_s2mm_aclk", "m_axis_mm2s_aclk",
                       "s_axis_s2mm_aclk";
         xlnx,num-fstores = <8>;
-        xlnx,flush-fsync;
+        xlnx,flush-fsync = <1>;
         xlnx,addrwidth = <32>;
 
         dma-channel-mm2s {
diff --git a/Documentation/devicetree/bindings/firmware/qcom,scm.yaml b/Documentation/devicetree/bindings/firmware/qcom,scm.yaml
index 25f62bacbc919..037d4ad8f718d 100644
--- a/Documentation/devicetree/bindings/firmware/qcom,scm.yaml
+++ b/Documentation/devicetree/bindings/firmware/qcom,scm.yaml
@@ -37,6 +37,7 @@ properties:
           - qcom,scm-ipq9574
           - qcom,scm-ipq9650
           - qcom,scm-kaanapali
+          - qcom,scm-maili
           - qcom,scm-mdm9607
           - qcom,scm-milos
           - qcom,scm-msm8226
@@ -67,6 +68,7 @@ properties:
           - qcom,scm-sdx55
           - qcom,scm-sdx65
           - qcom,scm-sdx75
+          - qcom,scm-shikra
           - qcom,scm-sm6115
           - qcom,scm-sm6125
           - qcom,scm-sm6350
@@ -129,6 +131,13 @@ properties:
           - description: offset of the download mode control register
     description: TCSR hardware block
 
+  sram:
+    description:
+      Phandle to a region in always-on SRAM used to store the download
+      mode value for boot firmware to read before DDR is initialised on
+      the next warm reset.
+    maxItems: 1
+
 allOf:
   # Clocks
   - if:
@@ -148,6 +157,7 @@ allOf:
               - qcom,scm-msm8974
               - qcom,scm-msm8976
               - qcom,scm-qcm2290
+              - qcom,scm-shikra
               - qcom,scm-sm6375
     then:
       required:
@@ -167,6 +177,7 @@ allOf:
               - qcom,scm-msm8660
               - qcom,scm-msm8960
               - qcom,scm-qcm2290
+              - qcom,scm-shikra
               - qcom,scm-sm6375
     then:
       properties:
@@ -212,6 +223,7 @@ allOf:
                 - qcom,scm-eliza
                 - qcom,scm-hawi
                 - qcom,scm-kaanapali
+                - qcom,scm-maili
                 - qcom,scm-milos
                 - qcom,scm-sm8450
                 - qcom,scm-sm8550
diff --git a/Documentation/devicetree/bindings/gpio/cdns,gpio.yaml b/Documentation/devicetree/bindings/gpio/cdns,gpio.yaml
index a84d60b394595..7eacf5af6554d 100644
--- a/Documentation/devicetree/bindings/gpio/cdns,gpio.yaml
+++ b/Documentation/devicetree/bindings/gpio/cdns,gpio.yaml
@@ -14,8 +14,11 @@ properties:
     oneOf:
       - const: cdns,gpio-r1p02
       - items:
-          - enum:
-              - axiado,ax3000-gpio
+          - const: axiado,ax3000-gpio
+          - const: cdns,gpio-r1p02
+      - items:
+          - const: axiado,ax3005-gpio
+          - const: axiado,ax3000-gpio
           - const: cdns,gpio-r1p02
 
   reg:
diff --git a/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml b/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml
index 4f955f855e1ab..4631388a7d914 100644
--- a/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml
+++ b/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml
@@ -22,6 +22,9 @@ properties:
       - items:
           - const: diodes,pi4ioe5v6534q
           - const: nxp,pcal6534
+      - items:
+          - const: kinetic,kts1622
+          - const: ti,tcal6416
       - items:
           - enum:
               - exar,xra1202
diff --git a/Documentation/devicetree/bindings/gpio/realtek,rtd1625-gpio.yaml b/Documentation/devicetree/bindings/gpio/realtek,rtd1625-gpio.yaml
new file mode 100644
index 0000000000000..f13c910b73c69
--- /dev/null
+++ b/Documentation/devicetree/bindings/gpio/realtek,rtd1625-gpio.yaml
@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+# Copyright 2023 Realtek Semiconductor Corporation
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/gpio/realtek,rtd1625-gpio.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Realtek DHC RTD1625 GPIO controller
+
+maintainers:
+  - Tzuyi Chang <tychang@realtek.com>
+
+description: |
+  GPIO controller for the Realtek RTD1625 SoC, featuring a per-pin register
+  architecture that differs significantly from earlier RTD series controllers.
+  Each GPIO has dedicated registers for configuration (direction, input/output
+  values, debounce), and interrupt control supporting edge and level detection
+  modes.
+
+properties:
+  compatible:
+    enum:
+      - realtek,rtd1625-iso-gpio
+      - realtek,rtd1625-isom-gpio
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    items:
+      - description: Interrupt number of the assert GPIO interrupt, which is
+                     triggered when there is a rising edge.
+      - description: Interrupt number of the deassert GPIO interrupt, which is
+                     triggered when there is a falling edge.
+      - description: Interrupt number of the level-sensitive GPIO interrupt,
+                     triggered by a configured logic level.
+
+  interrupt-controller: true
+
+  "#interrupt-cells":
+    const: 2
+
+  gpio-ranges: true
+
+  gpio-controller: true
+
+  "#gpio-cells":
+    const: 2
+
+required:
+  - compatible
+  - reg
+  - gpio-ranges
+  - gpio-controller
+  - "#gpio-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    gpio@89100 {
+      compatible = "realtek,rtd1625-isom-gpio";
+      reg = <0x89100 0x30>;
+      interrupt-parent = <&iso_m_irq_mux>;
+      interrupts = <0>, <1>, <2>;
+      interrupt-controller;
+      #interrupt-cells = <2>;
+      gpio-ranges = <&isom_pinctrl 0 0 4>;
+      gpio-controller;
+      #gpio-cells = <2>;
+    };
diff --git a/Documentation/devicetree/bindings/gpu/host1x/nvidia,tegra210-nvdec.yaml b/Documentation/devicetree/bindings/gpu/host1x/nvidia,tegra210-nvdec.yaml
index ba4c6473ff929..84a84275432ae 100644
--- a/Documentation/devicetree/bindings/gpu/host1x/nvidia,tegra210-nvdec.yaml
+++ b/Documentation/devicetree/bindings/gpu/host1x/nvidia,tegra210-nvdec.yaml
@@ -12,7 +12,7 @@ description: |
   programmed through Host1x channels.
 
 maintainers:
-  - Thierry Reding <treding@gmail.com>
+  - Thierry Reding <thierry.reding@kernel.org>
   - Mikko Perttunen <mperttunen@nvidia.com>
 
 properties:
diff --git a/Documentation/devicetree/bindings/gpu/host1x/nvidia,tegra210-nvenc.yaml b/Documentation/devicetree/bindings/gpu/host1x/nvidia,tegra210-nvenc.yaml
index c23dae713eb89..8916a868dafb0 100644
--- a/Documentation/devicetree/bindings/gpu/host1x/nvidia,tegra210-nvenc.yaml
+++ b/Documentation/devicetree/bindings/gpu/host1x/nvidia,tegra210-nvenc.yaml
@@ -12,7 +12,7 @@ description: |
   programmed through Host1x channels.
 
 maintainers:
-  - Thierry Reding <treding@gmail.com>
+  - Thierry Reding <thierry.reding@kernel.org>
   - Mikko Perttunen <mperttunen@nvidia.com>
 
 properties:
diff --git a/Documentation/devicetree/bindings/gpu/host1x/nvidia,tegra210-nvjpg.yaml b/Documentation/devicetree/bindings/gpu/host1x/nvidia,tegra210-nvjpg.yaml
index 99a33a5eac3fc..582eb44eb1577 100644
--- a/Documentation/devicetree/bindings/gpu/host1x/nvidia,tegra210-nvjpg.yaml
+++ b/Documentation/devicetree/bindings/gpu/host1x/nvidia,tegra210-nvjpg.yaml
@@ -12,7 +12,7 @@ description: |
   through Host1x channels.
 
 maintainers:
-  - Thierry Reding <treding@gmail.com>
+  - Thierry Reding <thierry.reding@kernel.org>
   - Mikko Perttunen <mperttunen@nvidia.com>
 
 properties:
diff --git a/Documentation/devicetree/bindings/gpu/host1x/nvidia,tegra234-nvdec.yaml b/Documentation/devicetree/bindings/gpu/host1x/nvidia,tegra234-nvdec.yaml
index 0b7561c8b9bb2..4eb325cfd2961 100644
--- a/Documentation/devicetree/bindings/gpu/host1x/nvidia,tegra234-nvdec.yaml
+++ b/Documentation/devicetree/bindings/gpu/host1x/nvidia,tegra234-nvdec.yaml
@@ -12,7 +12,7 @@ description: |
   programmed through Host1x channels.
 
 maintainers:
-  - Thierry Reding <treding@gmail.com>
+  - Thierry Reding <thierry.reding@kernel.org>
   - Mikko Perttunen <mperttunen@nvidia.com>
 
 properties:
diff --git a/Documentation/devicetree/bindings/gpu/img,powervr-rogue.yaml b/Documentation/devicetree/bindings/gpu/img,powervr-rogue.yaml
index a1f54dbae3f31..a6bccb600a57c 100644
--- a/Documentation/devicetree/bindings/gpu/img,powervr-rogue.yaml
+++ b/Documentation/devicetree/bindings/gpu/img,powervr-rogue.yaml
@@ -8,7 +8,8 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Imagination Technologies PowerVR and IMG Rogue GPUs
 
 maintainers:
-  - Frank Binns <frank.binns@imgtec.com>
+  - Alessio Belle <alessio.belle@imgtec.com>
+  - Luigi Santivetti <luigi.santivetti@imgtec.com>
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/gpu/img,powervr-sgx.yaml b/Documentation/devicetree/bindings/gpu/img,powervr-sgx.yaml
index f5898b04381cb..58b799b8e96e2 100644
--- a/Documentation/devicetree/bindings/gpu/img,powervr-sgx.yaml
+++ b/Documentation/devicetree/bindings/gpu/img,powervr-sgx.yaml
@@ -9,7 +9,8 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Imagination Technologies PowerVR SGX GPUs
 
 maintainers:
-  - Frank Binns <frank.binns@imgtec.com>
+  - Alessio Belle <alessio.belle@imgtec.com>
+  - Luigi Santivetti <luigi.santivetti@imgtec.com>
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/hwmon/adi,adm1275.yaml b/Documentation/devicetree/bindings/hwmon/adi,adm1275.yaml
index d6a7517f2a50c..283cd16626892 100644
--- a/Documentation/devicetree/bindings/hwmon/adi,adm1275.yaml
+++ b/Documentation/devicetree/bindings/hwmon/adi,adm1275.yaml
@@ -25,19 +25,38 @@ description: |
     https://www.silergy.com/
     download/downloadFile?id=5669&type=product&ftype=note
 
+  The BD12780 and BD12780A are hot-swap controllers from ROHM. They are
+  functionally compatible with the ADM1278. The main difference between
+  the BD12780A and the BD12780 is amount of configurable I2C addresses.
+
+  Datasheets:
+    https://fscdn.rohm.com/en/products/databook/datasheet/ic/power/power_switch/bd12780muv-lb-e.pdf
+    https://fscdn.rohm.com/en/products/databook/datasheet/ic/power/power_switch/bd12780amuv-lb-e.pdf
+
+  The BD12790 is a ROHM hot-swap controller, functionally similar to the
+  ADM1272.
+
 properties:
   compatible:
-    enum:
-      - adi,adm1075
-      - adi,adm1272
-      - adi,adm1273
-      - adi,adm1275
-      - adi,adm1276
-      - adi,adm1278
-      - adi,adm1281
-      - adi,adm1293
-      - adi,adm1294
-      - silergy,mc09c
+    oneOf:
+      - enum:
+          - adi,adm1075
+          - adi,adm1272
+          - adi,adm1273
+          - adi,adm1275
+          - adi,adm1276
+          - adi,adm1278
+          - adi,adm1281
+          - adi,adm1293
+          - adi,adm1294
+          - rohm,bd12780
+          - rohm,bd12790
+          - silergy,mc09c
+
+      # Require BD12780 as a fall-back for BD12780A.
+      - items:
+          - const: rohm,bd12780a
+          - const: rohm,bd12780
 
   reg:
     maxItems: 1
@@ -88,6 +107,7 @@ allOf:
             enum:
               - adi,adm1272
               - adi,adm1273
+              - rohm,bd12790
     then:
       properties:
         adi,volt-curr-sample-average:
@@ -104,6 +124,7 @@ allOf:
               - adi,adm1281
               - adi,adm1293
               - adi,adm1294
+              - rohm,bd12780
               - silergy,mc09c
     then:
       properties:
diff --git a/Documentation/devicetree/bindings/hwmon/amphenol,chipcap2.yaml b/Documentation/devicetree/bindings/hwmon/amphenol,chipcap2.yaml
index 17351fdbefce6..56b0cecfca5fb 100644
--- a/Documentation/devicetree/bindings/hwmon/amphenol,chipcap2.yaml
+++ b/Documentation/devicetree/bindings/hwmon/amphenol,chipcap2.yaml
@@ -45,6 +45,8 @@ properties:
       - const: low
       - const: high
 
+  label: true
+
   vdd-supply:
     description:
       Dedicated, controllable supply-regulator to reset the device and
@@ -55,6 +57,9 @@ required:
   - reg
   - vdd-supply
 
+allOf:
+  - $ref: hwmon-common.yaml#
+
 additionalProperties: false
 
 examples:
@@ -72,6 +77,7 @@ examples:
                          <5 IRQ_TYPE_EDGE_RISING>,
                          <6 IRQ_TYPE_EDGE_RISING>;
             interrupt-names = "ready", "low", "high";
+            label = "Room";
             vdd-supply = <&reg_vdd>;
         };
     };
diff --git a/Documentation/devicetree/bindings/hwmon/eswin,eic7700-pvt.yaml b/Documentation/devicetree/bindings/hwmon/eswin,eic7700-pvt.yaml
new file mode 100644
index 0000000000000..58ec8635dce3d
--- /dev/null
+++ b/Documentation/devicetree/bindings/hwmon/eswin,eic7700-pvt.yaml
@@ -0,0 +1,72 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/hwmon/eswin,eic7700-pvt.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ESWIN EIC7700 PVT Sensor
+
+maintainers:
+  - Yulin Lu <luyulin@eswincomputing.com>
+  - Huan He <hehuan1@eswincomputing.com>
+
+description:
+  ESWIN EIC7700 SoC integrates embedded voltage and temperature sensors to
+  monitor the internal SoC environment. The system includes two PVT sensor
+  instances. The PVT0 monitors the main SoC power domain. The PVT1 sensor
+  monitors the DDR core power domain.
+
+allOf:
+  - $ref: /schemas/hwmon/hwmon-common.yaml#
+
+properties:
+  compatible:
+    const: eswin,eic7700-pvt
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: PVT enable clock
+      - description: APB bus clock
+
+  clock-names:
+    items:
+      - const: enable
+      - const: apb
+
+  interrupts:
+    maxItems: 1
+
+  resets:
+    maxItems: 1
+
+  '#thermal-sensor-cells':
+    const: 0
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+  - interrupts
+  - resets
+  - '#thermal-sensor-cells'
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    sensor@50b00000 {
+      compatible = "eswin,eic7700-pvt";
+      reg = <0x50b00000 0x10000>;
+      clocks = <&clocks 244>, <&clocks 234>;
+      clock-names = "enable", "apb";
+      interrupts = <349>;
+      interrupt-parent = <&plic>;
+      label = "pvt0";
+      resets = <&reset 111>;
+      #thermal-sensor-cells = <0>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/hwmon/pmbus/silergy,sq24860.yaml b/Documentation/devicetree/bindings/hwmon/pmbus/silergy,sq24860.yaml
new file mode 100644
index 0000000000000..03ef82c11e1a2
--- /dev/null
+++ b/Documentation/devicetree/bindings/hwmon/pmbus/silergy,sq24860.yaml
@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+
+$id: http://devicetree.org/schemas/hwmon/pmbus/silergy,sq24860.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Silergy SQ24860 eFuse
+
+maintainers:
+  - Ziming Zhu <ziming.zhu@silergycorp.com>
+
+description:
+  The Silergy SQ24860 is an integrated, high-current circuit protection and
+  power management device with PMBus interface.
+
+properties:
+  compatible:
+    const: silergy,sq24860
+
+  reg:
+    maxItems: 1
+
+  silergy,rimon-micro-ohms:
+    description:
+      Micro-ohms value of the resistance installed between the IMON pin and
+      the ground reference.
+
+  interrupts:
+    description: PMBus SMBAlert interrupt.
+    maxItems: 1
+
+  regulators:
+    type: object
+    description:
+      List of regulators provided by this controller.
+
+    properties:
+      vout:
+        $ref: /schemas/regulator/regulator.yaml#
+        type: object
+        unevaluatedProperties: false
+
+    additionalProperties: false
+
+required:
+  - compatible
+  - reg
+  - silergy,rimon-micro-ohms
+
+additionalProperties: false
+
+examples:
+  - |
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        hw-monitor@40 {
+            compatible = "silergy,sq24860";
+            reg = <0x40>;
+
+            interrupt-parent = <&gpio>;
+            interrupts = <42 8>;
+            silergy,rimon-micro-ohms = <1600000000>;
+
+            regulators {
+                cpu0_vout: vout {
+                    regulator-name = "main_cpu0";
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml b/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml
index a20f140dc79a6..fe42daabaaa84 100644
--- a/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml
+++ b/Documentation/devicetree/bindings/hwmon/pmbus/ti,lm25066.yaml
@@ -46,12 +46,32 @@ properties:
 
     additionalProperties: false
 
+  ti,current-range:
+    description: |
+      Configure the current limit setting. When present, this property
+      overrides the hardware setting of the physical CL pin by configuring
+      the DEVICE_SETUP register.
+      - "low": maps to 25 mV (LM25066) or 26 mV (LM5064, LM5066, LM5066i)
+      - "high": maps to 46 mV (LM25066) or 50 mV (LM5064, LM5066, LM5066i)
+    $ref: /schemas/types.yaml#/definitions/string
+    enum:
+      - low
+      - high
+
 required:
   - compatible
   - reg
 
 allOf:
   - $ref: /schemas/hwmon/hwmon-common.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: ti,lm25056
+    then:
+      properties:
+        ti,current-range: false
 
 unevaluatedProperties: false
 
diff --git a/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml b/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml
index 009d78b308596..a79ade3142fa4 100644
--- a/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml
+++ b/Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml
@@ -28,6 +28,7 @@ properties:
       - ti,ina228
       - ti,ina230
       - ti,ina231
+      - ti,ina232
       - ti,ina233
       - ti,ina234
       - ti,ina237
@@ -114,6 +115,7 @@ allOf:
               - ti,ina228
               - ti,ina230
               - ti,ina231
+              - ti,ina232
               - ti,ina234
               - ti,ina237
               - ti,ina238
@@ -136,6 +138,7 @@ allOf:
               - ti,ina226
               - ti,ina230
               - ti,ina231
+              - ti,ina232
               - ti,ina234
               - ti,ina260
               - ti,ina700
diff --git a/Documentation/devicetree/bindings/i2c/altr,softip-i2c-v1.0.yaml b/Documentation/devicetree/bindings/i2c/altr,softip-i2c-v1.0.yaml
new file mode 100644
index 0000000000000..d04570680c05f
--- /dev/null
+++ b/Documentation/devicetree/bindings/i2c/altr,softip-i2c-v1.0.yaml
@@ -0,0 +1,62 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/i2c/altr,softip-i2c-v1.0.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Altera I2C Controller
+
+maintainers:
+  - Thor Thayer <thor.thayer@linux.intel.com>
+  - Chaitanya Sabnis <chaitanya.msabnis@gmail.com>
+
+description:
+  Altera's synthesizable logic block I2C Controller for use in Altera's FPGAs.
+
+allOf:
+  - $ref: /schemas/i2c/i2c-controller.yaml#
+
+properties:
+  compatible:
+    const: altr,softip-i2c-v1.0
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  fifo-size:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: Size of the RX and TX FIFOs in bytes.
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    i2c@80000 {
+        compatible = "altr,softip-i2c-v1.0";
+        reg = <0x80000 0x40>;
+        interrupt-parent = <&intc>;
+        interrupts = <0 43 4>;
+        clocks = <&clk_0>;
+        clock-frequency = <100000>;
+        #address-cells = <1>;
+        #size-cells = <0>;
+        fifo-size = <4>;
+
+        eeprom@51 {
+            compatible = "atmel,24c32";
+            reg = <0x51>;
+            pagesize = <32>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/i2c/i2c-altera.txt b/Documentation/devicetree/bindings/i2c/i2c-altera.txt
deleted file mode 100644
index 767664f448ece..0000000000000
--- a/Documentation/devicetree/bindings/i2c/i2c-altera.txt
+++ /dev/null
@@ -1,39 +0,0 @@
-* Altera I2C Controller
-* This is Altera's synthesizable logic block I2C Controller for use
-* in Altera's FPGAs.
-
-Required properties :
- - compatible : should be "altr,softip-i2c-v1.0"
- - reg        : Offset and length of the register set for the device
- - interrupts : <IRQ> where IRQ is the interrupt number.
- - clocks     : phandle to input clock.
- - #address-cells = <1>;
- - #size-cells = <0>;
-
-Recommended properties :
- - clock-frequency : desired I2C bus clock frequency in Hz.
-
-Optional properties :
- - fifo-size : Size of the RX and TX FIFOs in bytes.
- - Child nodes conforming to i2c bus binding
-
-Example :
-
-	i2c@100080000 {
-		compatible = "altr,softip-i2c-v1.0";
-		reg = <0x00000001 0x00080000 0x00000040>;
-		interrupt-parent = <&intc>;
-		interrupts = <0 43 4>;
-		clocks = <&clk_0>;
-		clock-frequency = <100000>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		fifo-size = <4>;
-
-		eeprom@51 {
-			compatible = "atmel,24c32";
-			reg = <0x51>;
-			pagesize = <32>;
-		};
-	};
-
diff --git a/Documentation/devicetree/bindings/i2c/i2c-axxia.txt b/Documentation/devicetree/bindings/i2c/i2c-axxia.txt
deleted file mode 100644
index 7d53a2b795530..0000000000000
--- a/Documentation/devicetree/bindings/i2c/i2c-axxia.txt
+++ /dev/null
@@ -1,30 +0,0 @@
-LSI Axxia I2C
-
-Required properties :
-- compatible : Must be "lsi,api2c"
-- reg : Offset and length of the register set for the device
-- interrupts : the interrupt specifier
-- #address-cells : Must be <1>;
-- #size-cells : Must be <0>;
-- clock-names : Must contain "i2c".
-- clocks: Must contain an entry for each name in clock-names. See the common
-  clock bindings.
-
-Optional properties :
-- clock-frequency : Desired I2C bus clock frequency in Hz. If not specified,
-  the default 100 kHz frequency will be used. As only Normal and Fast modes
-  are supported, possible values are 100000 and 400000.
-
-Example :
-
-i2c@2010084000 {
-	compatible = "lsi,api2c";
-	device_type = "i2c";
-	#address-cells = <1>;
-	#size-cells = <0>;
-	reg = <0x20 0x10084000 0x00 0x1000>;
-	interrupts = <0 19 4>;
-	clocks = <&clk_per>;
-	clock-names = "i2c";
-	clock-frequency = <400000>;
-};
diff --git a/Documentation/devicetree/bindings/i2c/lsi,api2c.yaml b/Documentation/devicetree/bindings/i2c/lsi,api2c.yaml
new file mode 100644
index 0000000000000..2d1c3069c3a3f
--- /dev/null
+++ b/Documentation/devicetree/bindings/i2c/lsi,api2c.yaml
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/i2c/lsi,api2c.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: LSI Axxia I2C Controller
+
+maintainers:
+  - Anders Berg <anders.berg@lsi.com>
+  - Chaitanya Sabnis <chaitanya.msabnis@gmail.com>
+
+allOf:
+  - $ref: /schemas/i2c/i2c-controller.yaml#
+
+properties:
+  compatible:
+    const: lsi,api2c
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  clock-names:
+    const: i2c
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+  - clock-names
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    i2c@10084000 {
+        compatible = "lsi,api2c";
+        reg = <0x10084000 0x1000>;
+        interrupts = <0 19 4>;
+        clocks = <&clk_per>;
+        clock-names = "i2c";
+        clock-frequency = <400000>;
+        #address-cells = <1>;
+        #size-cells = <0>;
+    };
diff --git a/Documentation/devicetree/bindings/i2c/qcom,sa8255p-geni-i2c.yaml b/Documentation/devicetree/bindings/i2c/qcom,sa8255p-geni-i2c.yaml
new file mode 100644
index 0000000000000..a61e40b5cbc16
--- /dev/null
+++ b/Documentation/devicetree/bindings/i2c/qcom,sa8255p-geni-i2c.yaml
@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/i2c/qcom,sa8255p-geni-i2c.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm SA8255p QUP GENI I2C Controller
+
+maintainers:
+  - Praveen Talari <praveen.talari@oss.qualcomm.com>
+
+properties:
+  compatible:
+    const: qcom,sa8255p-geni-i2c
+
+  reg:
+    maxItems: 1
+
+  dmas:
+    maxItems: 2
+
+  dma-names:
+    items:
+      - const: tx
+      - const: rx
+
+  interrupts:
+    maxItems: 1
+
+  power-domains:
+    maxItems: 2
+
+  power-domain-names:
+    items:
+      - const: power
+      - const: perf
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - power-domains
+
+allOf:
+  - $ref: /schemas/i2c/i2c-controller.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/dma/qcom-gpi.h>
+
+    i2c@a90000 {
+        compatible = "qcom,sa8255p-geni-i2c";
+        reg = <0xa90000 0x4000>;
+        interrupts = <GIC_SPI 357 IRQ_TYPE_LEVEL_HIGH>;
+        dmas = <&gpi_dma0 0 0 QCOM_GPI_I2C>,
+               <&gpi_dma0 1 0 QCOM_GPI_I2C>;
+        dma-names = "tx", "rx";
+        power-domains = <&scmi0_pd 0>, <&scmi0_dvfs 0>;
+        power-domain-names = "power", "perf";
+    };
+...
diff --git a/Documentation/devicetree/bindings/i2c/samsung,s3c2410-i2c.yaml b/Documentation/devicetree/bindings/i2c/samsung,s3c2410-i2c.yaml
index a2ddc68036176..07600b49f2f9c 100644
--- a/Documentation/devicetree/bindings/i2c/samsung,s3c2410-i2c.yaml
+++ b/Documentation/devicetree/bindings/i2c/samsung,s3c2410-i2c.yaml
@@ -35,7 +35,7 @@ properties:
 
   gpios:
     description: |
-      The order of the GPIOs should be the following:: <SDA, SCL>.  The GPIO
+      The order of the GPIOs should be the following: <SDA, SCL>.  The GPIO
       specifier depends on the gpio controller. Required in all cases except
       for "samsung,s3c2440-hdmiphy-i2c" whose input/output lines are
       permanently wired to the respective client.
diff --git a/Documentation/devicetree/bindings/iio/adc/avia-hx711.yaml b/Documentation/devicetree/bindings/iio/adc/avia-hx711.yaml
index 1ea60dff98d59..099235b5fd548 100644
--- a/Documentation/devicetree/bindings/iio/adc/avia-hx711.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/avia-hx711.yaml
@@ -33,6 +33,26 @@ properties:
     description:
       Analog supply voltage (AVDD).
 
+  vsup-supply:
+    description:
+      Supply voltage for the on-chip regulator (VSUP).
+
+  dvdd-supply:
+    description:
+      Digital supply voltage (DVDD). On the HX711, DVDD is a
+      board-supplied digital rail; it was missing from the original
+      binding. On the HX710B, DVDD must be greater than or equal to
+      AVDD; when DVDD is a battery rail and AVDD is a regulated supply,
+      one channel monitors the DVDD-AVDD difference for battery level
+      detection.
+
+  rate-gpios:
+    description:
+      GPIO connected to the RATE pin. When driven low the output data
+      rate is 10 SPS; when driven high it is 80 SPS. If omitted the
+      RATE pin state is determined by the board wiring.
+    maxItems: 1
+
   clock-frequency:
     description:
       Controls the SCK bit-bang timing. The value is used to derive the
@@ -57,6 +77,7 @@ examples:
         compatible = "avia,hx711";
         sck-gpios = <&gpio3 10 GPIO_ACTIVE_HIGH>;
         dout-gpios = <&gpio0 7 GPIO_ACTIVE_HIGH>;
+        rate-gpios = <&gpio0 9 GPIO_ACTIVE_HIGH>;
         avdd-supply = <&avdd>;
         clock-frequency = <100000>;
     };
diff --git a/Documentation/devicetree/bindings/iio/adc/mediatek,mt6359-auxadc.yaml b/Documentation/devicetree/bindings/iio/adc/mediatek,mt6359-auxadc.yaml
index 9936aa605c7ba..c2f7387e4bfcf 100644
--- a/Documentation/devicetree/bindings/iio/adc/mediatek,mt6359-auxadc.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/mediatek,mt6359-auxadc.yaml
@@ -4,7 +4,7 @@
 $id: http://devicetree.org/schemas/iio/adc/mediatek,mt6359-auxadc.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: MediaTek MT6350 series PMIC AUXADC
+title: MediaTek MT6350 series and similar PMIC AUXADC
 
 maintainers:
   - AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
@@ -20,6 +20,7 @@ properties:
   compatible:
     oneOf:
       - enum:
+          - mediatek,mt6323-auxadc
           - mediatek,mt6357-auxadc
           - mediatek,mt6358-auxadc
           - mediatek,mt6359-auxadc
diff --git a/Documentation/devicetree/bindings/iio/adc/xlnx,versal-sysmon.yaml b/Documentation/devicetree/bindings/iio/adc/xlnx,versal-sysmon.yaml
new file mode 100644
index 0000000000000..1ad58e3d61631
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/adc/xlnx,versal-sysmon.yaml
@@ -0,0 +1,154 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+# Copyright (C) 2022 - 2026, Advanced Micro Devices, Inc.
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/adc/xlnx,versal-sysmon.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: AMD/Xilinx Versal System Monitor
+
+maintainers:
+  - Salih Erim <salih.erim@amd.com>
+
+description:
+  The AMD/Xilinx Versal System Monitor (SysMon) is the successor to the
+  Zynq UltraScale+ AMS block. It provides on-chip voltage and temperature
+  monitoring with up to 160 voltage measurement points and up to
+  64 temperature satellites distributed across the SoC. The hardware
+  supports configurable threshold alarms and oversampling. The device
+  can be accessed via memory-mapped I/O or via an I2C interface.
+
+properties:
+  compatible:
+    const: xlnx,versal-sysmon
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  '#io-channel-cells':
+    const: 1
+
+  voltage-channels:
+    type: object
+    description:
+      Container for voltage measurement channels.
+
+    properties:
+      '#address-cells':
+        const: 1
+
+      '#size-cells':
+        const: 0
+
+    patternProperties:
+      '^channel@([0-9a-f]|[1-9][0-9a-f])$':
+        $ref: adc.yaml
+
+        description:
+          Measures a voltage rail. The register index and rail
+          name are assigned by the hardware design tool (Vivado).
+
+        properties:
+          reg:
+            minimum: 0
+            maximum: 159
+            description:
+              Voltage measurement register index assigned by the hardware
+              design tool.
+
+        required:
+          - reg
+          - label
+
+        unevaluatedProperties: false
+
+    required:
+      - '#address-cells'
+      - '#size-cells'
+
+    additionalProperties: false
+
+  temperature-channels:
+    type: object
+    description:
+      Container for temperature satellite measurement channels.
+
+    properties:
+      '#address-cells':
+        const: 1
+
+      '#size-cells':
+        const: 0
+
+    patternProperties:
+      '^channel@([1-9a-f]|[1-3][0-9a-f]|40)$':
+        $ref: adc.yaml
+
+        description:
+          Reads a temperature satellite sensor. Each satellite monitors
+          a specific region of the SoC die.
+
+        properties:
+          reg:
+            minimum: 1
+            maximum: 64
+            description:
+              Temperature satellite number (1-based hardware index).
+
+        required:
+          - reg
+          - label
+
+        unevaluatedProperties: false
+
+    required:
+      - '#address-cells'
+      - '#size-cells'
+
+    additionalProperties: false
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    sysmon@f1270000 {
+        compatible = "xlnx,versal-sysmon";
+        reg = <0xf1270000 0x4000>;
+        interrupts = <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>;
+        #io-channel-cells = <1>;
+
+        voltage-channels {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            channel@0 {
+                reg = <0>;
+                label = "vccaux";
+            };
+
+            channel@3 {
+                reg = <3>;
+                label = "vcc_ram";
+                bipolar;
+            };
+        };
+
+        temperature-channels {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            channel@a {
+                reg = <10>;
+                label = "aie-temp-ch1";
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/iio/chemical/sensirion,scd30.yaml b/Documentation/devicetree/bindings/iio/chemical/sensirion,scd30.yaml
index a5b0debe85b1a..25291bda1a8be 100644
--- a/Documentation/devicetree/bindings/iio/chemical/sensirion,scd30.yaml
+++ b/Documentation/devicetree/bindings/iio/chemical/sensirion,scd30.yaml
@@ -8,6 +8,7 @@ title: Sensirion SCD30 carbon dioxide sensor
 
 maintainers:
   - Maxwell Doose <m32285159@gmail.com>
+  - Tomasz Duszynski <tduszyns@gmail.com>
 
 description: |
   Air quality sensor capable of measuring co2 concentration, temperature
diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad3530r.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad3530r.yaml
index a355d52a9d641..2fe098619772f 100644
--- a/Documentation/devicetree/bindings/iio/dac/adi,ad3530r.yaml
+++ b/Documentation/devicetree/bindings/iio/dac/adi,ad3530r.yaml
@@ -10,15 +10,17 @@ maintainers:
   - Kim Seer Paller <kimseer.paller@analog.com>
 
 description: |
-  The AD3530/AD3530R (8-channel) and AD3531/AD3531R (4-channel) are low-power,
-  16-bit, buffered voltage output digital-to-analog converters (DACs) with
-  software-programmable gain controls, providing full-scale output spans of 2.5V
-  or 5V for reference voltages of 2.5V. These devices operate from a single 2.7V
-  to 5.5V supply and are guaranteed monotonic by design. The "R" variants
-  include a 2.5V, 5ppm/°C internal reference, which is disabled by default.
+  The AD3530/AD3530R (8-channel), AD3531/AD3531R (4-channel), and AD3532/AD3532R
+  (16-channel) are low-power, 16-bit, buffered voltage output digital-to-analog
+  converters (DACs) with software-programmable gain controls, providing
+  full-scale output spans of 2.5V or 5V for reference voltages of 2.5V. These
+  devices operate from a single 2.7V to 5.5V supply and are guaranteed monotonic
+  by design. The "R" variants include a 2.5V, 5ppm/°C internal reference, which
+  is disabled by default.
   Datasheet can be found here:
   https://www.analog.com/media/en/technical-documentation/data-sheets/ad3530_ad530r.pdf
   https://www.analog.com/media/en/technical-documentation/data-sheets/ad3531-ad3531r.pdf
+  https://www.analog.com/media/en/technical-documentation/data-sheets/ad3532r.pdf
 
 properties:
   compatible:
@@ -27,6 +29,8 @@ properties:
       - adi,ad3530r
       - adi,ad3531
       - adi,ad3531r
+      - adi,ad3532
+      - adi,ad3532r
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml
index 713f535bb33ae..02e8c78e36d3d 100644
--- a/Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml
+++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml
@@ -32,27 +32,95 @@ properties:
   reg:
     maxItems: 1
 
+  vdd-supply:
+    description: Input power supply.
+
+  vlogic-supply:
+    description:
+      Digital power supply. On some tiny package variants for single-channel
+      devices, this supply is internally connected to vdd; in that case, specify
+      this property with the same regulator as vdd.
+
+  vref-supply:
+    description:
+      Reference voltage supply. If not supplied the internal reference is used.
+
   vcc-supply:
-    description: If not supplied the internal reference is used.
+    deprecated: true
+    description: Use vref-supply instead.
+
+  reset-gpios:
+    description: Active-low RESET pin to reset the device.
+    maxItems: 1
+
+  ldac-gpios:
+    description:
+      Active-low LDAC pin used to asynchronously update the DAC channels.
+    maxItems: 1
+
+  gain-gpios:
+    description:
+      GAIN pin that sets a multiplier for the DAC output voltage. When high,
+      the DAC output voltage is multiplied by 2, otherwise it is unchanged.
+    maxItems: 1
+
+  adi,range-double:
+    description:
+      Sets the initial voltage output range from 0 to 2xVREF. On devices that
+      have a GAIN pin and no gain-gpios property is set, this indicates the pin
+      is hardwired high.
+    type: boolean
 
 required:
   - compatible
   - reg
+  - vdd-supply
+  - vlogic-supply
 
 allOf:
   - $ref: /schemas/spi/spi-peripheral-props.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - adi,ad5676
+              - adi,ad5683
+              - adi,ad5684
+              - adi,ad5686
+    then:
+      required:
+        - vref-supply
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - adi,ad5310r
+              - adi,ad5681r
+              - adi,ad5682r
+              - adi,ad5683
+              - adi,ad5683r
+    then:
+      properties:
+        gain-gpios: false
 
 unevaluatedProperties: false
 
 examples:
   - |
+    #include <dt-bindings/gpio/gpio.h>
     spi {
         #address-cells = <1>;
         #size-cells = <0>;
         dac@0 {
             reg = <0>;
             compatible = "adi,ad5310r";
-            vcc-supply = <&dac_vref0>;
+            vdd-supply = <&dac_vdd>;
+            vlogic-supply = <&dac_vlogic>;
+            vref-supply = <&dac_vref>;
+            reset-gpios = <&gpio0 0 GPIO_ACTIVE_LOW>;
+            ldac-gpios = <&gpio0 1 GPIO_ACTIVE_LOW>;
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5696.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5696.yaml
index b5a88b03dc2f0..e10f8596f9d3f 100644
--- a/Documentation/devicetree/bindings/iio/dac/adi,ad5696.yaml
+++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5696.yaml
@@ -33,18 +33,82 @@ properties:
   reg:
     maxItems: 1
 
+  vdd-supply:
+    description: Input power supply.
+
+  vlogic-supply:
+    description:
+      Digital power supply. On some tiny package variants for single-channel
+      devices, this supply is internally connected to vdd; in that case, specify
+      this property with the same regulator as vdd.
+
+  vref-supply:
+    description:
+      Reference voltage supply. If not supplied the internal reference is used.
+
   vcc-supply:
-    description: |
-      The regulator supply for DAC reference voltage.
+    deprecated: true
+    description: Use vref-supply instead.
+
+  reset-gpios:
+    description: Active-low RESET pin to reset the device.
+    maxItems: 1
+
+  ldac-gpios:
+    description:
+      Active-low LDAC pin used to asynchronously update the DAC channels.
+    maxItems: 1
+
+  gain-gpios:
+    description:
+      GAIN pin that sets a multiplier for the DAC output voltage. When high,
+      the DAC output voltage is multiplied by 2, otherwise it is unchanged.
+    maxItems: 1
+
+  adi,range-double:
+    description:
+      Sets the initial voltage output range from 0 to 2xVREF. On devices that
+      have a GAIN pin and no gain-gpios property is set, this indicates the pin
+      is hardwired high.
+    type: boolean
 
 required:
   - compatible
   - reg
+  - vdd-supply
+  - vlogic-supply
+
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - adi,ad5693
+              - adi,ad5694
+              - adi,ad5696
+    then:
+      required:
+        - vref-supply
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - adi,ad5311r
+              - adi,ad5691r
+              - adi,ad5692r
+              - adi,ad5693
+              - adi,ad5693r
+    then:
+      properties:
+        gain-gpios: false
 
 additionalProperties: false
 
 examples:
   - |
+    #include <dt-bindings/gpio/gpio.h>
     i2c {
       #address-cells = <1>;
       #size-cells = <0>;
@@ -52,7 +116,10 @@ examples:
       ad5696: dac@0 {
         compatible = "adi,ad5696";
         reg = <0>;
-        vcc-supply = <&dac_vref>;
+        vdd-supply = <&dac_vdd>;
+        vlogic-supply = <&dac_vlogic>;
+        vref-supply = <&dac_vref>;
+        ldac-gpios = <&gpio0 1 GPIO_ACTIVE_LOW>;
       };
     };
 ...
diff --git a/Documentation/devicetree/bindings/iio/frequency/adi,adf41513.yaml b/Documentation/devicetree/bindings/iio/frequency/adi,adf41513.yaml
new file mode 100644
index 0000000000000..f4fae9210382a
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/frequency/adi,adf41513.yaml
@@ -0,0 +1,227 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/frequency/adi,adf41513.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Analog Devices ADF41513 PLL Frequency Synthesizer
+
+maintainers:
+  - Rodrigo Alencar <rodrigo.alencar@analog.com>
+
+description:
+  The ADF41513 is an ultralow noise frequency synthesizer that can be used to
+  implement local oscillators (LOs) as high as 26.5 GHz in the upconversion and
+  downconversion sections of wireless receivers and transmitters. The ADF41510
+  supports frequencies up to 10 GHz.
+
+  https://www.analog.com/en/products/adf41510.html
+  https://www.analog.com/en/products/adf41513.html
+
+properties:
+  compatible:
+    enum:
+      - adi,adf41510
+      - adi,adf41513
+
+  reg:
+    maxItems: 1
+
+  spi-max-frequency:
+    maximum: 25000000
+
+  clocks:
+    maxItems: 1
+    description: Clock that provides the reference input frequency.
+
+  avdd1-supply:
+    description: PFD and Up and Down Digital Driver Power Supply (3.3 V)
+
+  avdd2-supply:
+    description: RF Buffer and Prescaler Power Supply (3.3 V)
+
+  avdd3-supply:
+    description: N Divider Power Supply (3.3 V)
+
+  avdd4-supply:
+    description: R Divider and Lock Detector Power Supply (3.3 V)
+
+  avdd5-supply:
+    description: Sigma-Delta Modulator and SPI Power Supply (3.3 V)
+
+  vp-supply:
+    description: Charge Pump Power Supply (3.3 V)
+
+  enable-gpios:
+    description:
+      GPIO that controls the chip enable pin. A logic low on this pin
+      powers down the device and puts the charge pump output into
+      three-state mode.
+    maxItems: 1
+
+  lock-detect-gpios:
+    description:
+      GPIO for lock detect functionality. When configured for digital lock
+      detect, this pin will output a logic high when the PLL is locked.
+    maxItems: 1
+
+  adi,power-up-frequency-mhz:
+    minimum: 1000
+    maximum: 26500
+    default: 10000
+    description:
+      The PLL tunes to this frequency during the initialization sequence.
+      This property should be set to a frequency supported by the loop filter
+      and VCO used in the design. Range is 1 GHz to 26.5 GHz for ADF41513,
+      and 1 GHz to 10 GHz for ADF41510.
+
+  adi,reference-div-factor:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    minimum: 1
+    maximum: 32
+    default: 1
+    description:
+      Value for the reference division factor (R Counter). The driver will
+      increment R Counter as needed to achieve a PFD frequency within the
+      allowed range. High R counter values will reduce the PFD frequency, which
+      lowers the frequency resolution, and affects phase noise performance.
+      As it affects the PFD frequency, this value depends on the loop filter
+      design.
+
+  adi,reference-doubler-enable:
+    description:
+      Enables the reference doubler when deriving the PFD frequency.
+      The maximum reference frequency when the doubler is enabled is 225 MHz.
+      As it affects the PFD frequency, this value depends on the loop filter
+      design.
+    type: boolean
+
+  adi,reference-div2-enable:
+    description:
+      Enables the reference divide-by-2 function when deriving the PFD
+      frequency. As it affects the PFD frequency, this value depends on the
+      loop filter design.
+    type: boolean
+
+  adi,charge-pump-resistor-ohms:
+    minimum: 1800
+    maximum: 10000
+    default: 2700
+    description:
+      External charge pump resistor (R_SET) value in ohms. This sets the maximum
+      charge pump current along with the charge pump current setting.
+
+  adi,charge-pump-current-microamp:
+    minimum: 81
+    maximum: 7200
+    description:
+      Charge pump current (I_CP) in microamps. The value will be rounded to the
+      nearest supported value. Range of acceptable values depends on the
+      charge pump resistor value, such that 810 mV <= I_CP * R_SET <= 12960 mV.
+      This value depends on the loop filter and the VCO design.
+
+  adi,logic-level-1v8-enable:
+    description:
+      Set MUXOUT and DLD logic levels to 1.8V. Default is 3.3V.
+    type: boolean
+
+  adi,phase-detector-polarity-positive-enable:
+    description:
+      Set phase detector polarity to positive. Default is negative.
+      Use positive polarity with non-inverting loop filter and VCO with
+      positive tuning slope, or with inverting loop filter and VCO with
+      negative tuning slope.
+    type: boolean
+
+  adi,lock-detector-count:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    default: 64
+    description:
+      Sets the value for Lock Detector count of the PLL, which determines the
+      number of consecutive phase detector cycles that must be within the lock
+      detector window before lock is declared. Lower values increase the lock
+      detection sensitivity, while higher values provides a more stable lock
+      detection. Applications that consume the lock detect signal may require
+      different settings based on system requirements.
+    enum: [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
+
+  adi,phase-resync-period-ns:
+    default: 0
+    description:
+      When this value is non-zero, enable phase resync functionality, which
+      produces a consistent output phase offset with respect to the input
+      reference. The value specifies the resync period in nanoseconds, used
+      to configure clock dividers with respect to the PFD frequency. This value
+      should be set to a value that is at least as long as the worst case lock
+      time, i.e., it depends mostly on the loop filter design.
+
+  adi,le-sync-enable:
+    description:
+      Synchronizes Load Enable (LE) transitions with the reference signal to
+      avoid asynchronous glitches in the output. This is recommended when using
+      the PLL as a frequency synthesizer, where the reference signal will always
+      be present while the device is being configured. When using the PLL as a
+      frequency tracker, where the reference signal may be absent, LE sync
+      should be left disabled.
+    type: boolean
+
+dependencies:
+  adi,charge-pump-resistor-ohms: [ 'adi,charge-pump-current-microamp' ]
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - avdd1-supply
+  - avdd2-supply
+  - avdd3-supply
+  - avdd4-supply
+  - avdd5-supply
+  - vp-supply
+
+allOf:
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: adi,adf41510
+    then:
+      properties:
+        adi,power-up-frequency-mhz:
+          maximum: 10000
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        pll@0 {
+            compatible = "adi,adf41513";
+            reg = <0>;
+            spi-max-frequency = <25000000>;
+            clocks = <&ref_clk>;
+            avdd1-supply = <&avdd1_3v3>;
+            avdd2-supply = <&avdd2_3v3>;
+            avdd3-supply = <&avdd3_3v3>;
+            avdd4-supply = <&avdd4_3v3>;
+            avdd5-supply = <&avdd5_3v3>;
+            vp-supply = <&vp_3v3>;
+            enable-gpios = <&gpio0 10 GPIO_ACTIVE_HIGH>;
+            lock-detect-gpios = <&gpio0 11 GPIO_ACTIVE_HIGH>;
+
+            adi,power-up-frequency-mhz = <15500>;
+            adi,charge-pump-current-microamp = <3600>;
+            adi,charge-pump-resistor-ohms = <2700>;
+            adi,reference-doubler-enable;
+            adi,lock-detector-count = <64>;
+            adi,phase-resync-period-ns = <0>;
+            adi,phase-detector-polarity-positive-enable;
+            adi,le-sync-enable;
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/iio/magnetometer/qstcorp,qmc5883l.yaml b/Documentation/devicetree/bindings/iio/magnetometer/qstcorp,qmc5883l.yaml
new file mode 100644
index 0000000000000..38abd083a4fbe
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/magnetometer/qstcorp,qmc5883l.yaml
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/magnetometer/qstcorp,qmc5883l.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: QST QMC5883L 3-Axis Magnetic Sensor
+
+maintainers:
+  - Siratul Islam <siratul.islam@linux.dev>
+
+description: |
+  QST QMC5883L 3-Axis Magnetic Sensor on I2C bus.
+  https://www.qstcorp.com/upload/pdf/202512/13-52-04%20QMC5883L%20Datasheet%20Rev.%20B.pdf
+
+properties:
+  compatible:
+    enum:
+      - qstcorp,qmc5883l
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  vdd-supply: true
+
+  vddio-supply: true
+
+additionalProperties: false
+
+required:
+  - compatible
+  - reg
+  - vdd-supply
+  - vddio-supply
+
+examples:
+  - |
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        magnetometer@d {
+            compatible = "qstcorp,qmc5883l";
+            reg = <0x0d>;
+            vdd-supply = <&vdd_3v3_reg>;
+            vddio-supply = <&vdd_3v3_reg>;
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/iio/temperature/ti,tmp117.yaml b/Documentation/devicetree/bindings/iio/temperature/ti,tmp117.yaml
index fbba5e934861c..7fb3e68207919 100644
--- a/Documentation/devicetree/bindings/iio/temperature/ti,tmp117.yaml
+++ b/Documentation/devicetree/bindings/iio/temperature/ti,tmp117.yaml
@@ -7,19 +7,25 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: TI TMP117 - Digital temperature sensor with integrated NV memory
 
 description: |
-    TI TMP116/117 - Digital temperature sensor with integrated NV memory that
-    supports I2C interface.
+    TI TMP116/117/119 - Digital temperature sensor with integrated NV memory
+    that supports I2C interface.
       https://www.ti.com/lit/gpn/tmp116
       https://www.ti.com/lit/gpn/tmp117
+      https://www.ti.com/lit/gpn/tmp119
 
 maintainers:
   - Puranjay Mohan <puranjay12@gmail.com>
 
 properties:
   compatible:
-    enum:
-      - ti,tmp116
-      - ti,tmp117
+    oneOf:
+      - enum:
+          - ti,tmp116
+          - ti,tmp117
+      - items:
+          - enum:
+              - ti,tmp119
+          - const: ti,tmp117
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/input/allwinner,sun4i-a10-lradc-keys.yaml b/Documentation/devicetree/bindings/input/allwinner,sun4i-a10-lradc-keys.yaml
index 6bdb8040be656..524c8b51f53fe 100644
--- a/Documentation/devicetree/bindings/input/allwinner,sun4i-a10-lradc-keys.yaml
+++ b/Documentation/devicetree/bindings/input/allwinner,sun4i-a10-lradc-keys.yaml
@@ -23,6 +23,7 @@ properties:
       - const: allwinner,sun50i-r329-lradc
       - items:
           - enum:
+              - allwinner,sun50i-a100-lradc
               - allwinner,sun50i-h616-lradc
               - allwinner,sun20i-d1-lradc
           - const: allwinner,sun50i-r329-lradc
diff --git a/Documentation/devicetree/bindings/input/fsl,scu-key.yaml b/Documentation/devicetree/bindings/input/fsl,scu-key.yaml
index 29921aab9d971..2566cffc252d7 100644
--- a/Documentation/devicetree/bindings/input/fsl,scu-key.yaml
+++ b/Documentation/devicetree/bindings/input/fsl,scu-key.yaml
@@ -18,7 +18,9 @@ allOf:
 properties:
   compatible:
     items:
-      - const: fsl,imx8qxp-sc-key
+      - enum:
+          - fsl,imx8qm-sc-key
+          - fsl,imx8qxp-sc-key
       - const: fsl,imx-sc-key
 
   linux,keycodes:
diff --git a/Documentation/devicetree/bindings/input/gpio-charlieplex-keypad.yaml b/Documentation/devicetree/bindings/input/gpio-charlieplex-keypad.yaml
index c085de6dab854..c6842c0179341 100644
--- a/Documentation/devicetree/bindings/input/gpio-charlieplex-keypad.yaml
+++ b/Documentation/devicetree/bindings/input/gpio-charlieplex-keypad.yaml
@@ -11,7 +11,7 @@ maintainers:
   - Hugo Villeneuve <hvilleneuve@dimonoff.com>
 
 description: |
-  The charlieplex keypad supports N^2)-N different key combinations (where N is
+  The charlieplex keypad supports (N^2)-N different key combinations (where N is
   the number of I/O lines). Key presses and releases are detected by configuring
   only one line as output at a time, and reading other line states. This process
   is repeated for each line. Diodes are required to ensure current flows in only
diff --git a/Documentation/devicetree/bindings/input/imagis,isa1200.yaml b/Documentation/devicetree/bindings/input/imagis,isa1200.yaml
new file mode 100644
index 0000000000000..4bc8630edcdd0
--- /dev/null
+++ b/Documentation/devicetree/bindings/input/imagis,isa1200.yaml
@@ -0,0 +1,141 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/input/imagis,isa1200.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Imagis ISA1200 haptic motor driver
+
+maintainers:
+  - Svyatoslav Ryhel <clamor95@gmail.com>
+  - Linus Walleij <linusw@kernel.org>
+
+description:
+  The ISA1200 is a high-performance enhanced haptic motor driver designed
+  for mobile hand-held devices. It supports various voltages for both ERM
+  (Eccentric Rotating Mass) and LRA (Linear Resonant Actuator) type
+  actuators. Thanks to an embedded LDO, battery power can be used directly
+  in handheld applications.
+
+properties:
+  compatible:
+    const: imagis,isa1200
+
+  reg:
+    maxItems: 1
+
+  control-gpios:
+    description:
+      One or two GPIOs flagged as active high linked to HEN and LEN pins
+    minItems: 1
+    maxItems: 2
+
+  clocks:
+    maxItems: 1
+
+  pwms:
+    maxItems: 1
+
+  vdd-supply:
+    description:
+      Regulator for 2.4V - 5.5V power supply
+
+  vddp-supply:
+    description:
+      Regulator for 2.4V - 3.6V IO power supply
+
+  imagis,clk-div:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      Divider for the external input clock/PWM
+    enum: [128, 256, 512, 1024]
+    default: 128
+
+  imagis,pll-div:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      Divider for the internal PLL clock
+    minimum: 1
+    maximum: 15
+    default: 1
+
+  imagis,mode:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: |
+      Defines the motor type isa1200 drives
+      0 - LRA (Linear Resonant Actuator)
+      1 - ERM (Eccentric Rotating Mass)
+    enum: [0, 1]
+    default: 0
+
+  imagis,period-ns:
+    description:
+      Period of the internal PWM channel in nanoseconds.
+    minimum: 10000
+    maximum: 30000
+
+  imagis,duty-cycle-ns:
+    description:
+      Duty cycle of the external/internal PWM channel in nanoseconds,
+      defaults to 50% of the channel's period
+
+  ldo:
+    $ref: /schemas/regulator/regulator.yaml#
+    type: object
+    description:
+      Embedded LDO regulator with voltage range 2.3V - 3.8V
+    unevaluatedProperties: false
+
+    required:
+      - regulator-min-microvolt
+      - regulator-max-microvolt
+
+required:
+  - compatible
+  - reg
+  - ldo
+
+oneOf:
+  - required:
+      - clocks
+      - imagis,period-ns
+  - required:
+      - pwms
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        haptic-engine@49 {
+            compatible = "imagis,isa1200";
+            reg = <0x49>;
+
+            clocks = <&isa1200_refclk>;
+
+            control-gpios = <&gpio 22 GPIO_ACTIVE_HIGH>,
+                            <&gpio 23 GPIO_ACTIVE_HIGH>;
+
+            vdd-supply = <&vdd_3v3_vbat>;
+            vddp-supply = <&vdd_2v8_vvib>;
+
+            imagis,clk-div = <256>;
+            imagis,pll-div = <2>;
+
+            imagis,mode = <0>; /* LRA_MODE */
+
+            imagis,period-ns = <13400>;
+            imagis,duty-cycle-ns = <100>;
+
+            ldo {
+                regulator-name = "vdd_vib";
+                regulator-min-microvolt = <2300000>;
+                regulator-max-microvolt = <2300000>;
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/input/microchip,cap11xx.yaml b/Documentation/devicetree/bindings/input/microchip,cap11xx.yaml
index 2d762193f1c07..983f4701b51e4 100644
--- a/Documentation/devicetree/bindings/input/microchip,cap11xx.yaml
+++ b/Documentation/devicetree/bindings/input/microchip,cap11xx.yaml
@@ -10,6 +10,16 @@ description: |
   The Microchip CAP1xxx Family of RightTouchTM multiple-channel capacitive
   touch controllers and LED drivers. The device communication via I2C only.
 
+  For more product information please see the links below:
+    CAP1106: https://ww1.microchip.com/downloads/en/DeviceDoc/00001624B.pdf
+    CAP1114: https://ww1.microchip.com/downloads/en/DeviceDoc/00002444A.pdf
+    CAP1126: https://ww1.microchip.com/downloads/en/DeviceDoc/00001623B.pdf
+    CAP1188: https://ww1.microchip.com/downloads/en/DeviceDoc/00001620C.pdf
+    CAP1203: https://ww1.microchip.com/downloads/en/DeviceDoc/00001572B.pdf
+    CAP1206: https://ww1.microchip.com/downloads/en/DeviceDoc/00001567B.pdf
+    CAP1293: https://ww1.microchip.com/downloads/en/DeviceDoc/00001566B.pdf
+    CAP1298: https://ww1.microchip.com/downloads/en/DeviceDoc/00001571B.pdf
+
 maintainers:
   - Rob Herring <robh@kernel.org>
 
@@ -17,6 +27,7 @@ properties:
   compatible:
     enum:
       - microchip,cap1106
+      - microchip,cap1114
       - microchip,cap1126
       - microchip,cap1188
       - microchip,cap1203
@@ -40,13 +51,20 @@ properties:
       device's ALERT#/CM_IRQ# pin is connected to.
       The device only has one interrupt source.
 
+  reset-gpios:
+    description: |
+      GPIO connected to the active-high RESET pin of the chip;
+      driving it high asserts reset and deep sleep, while driving
+      it low releases reset for normal operation.
+    maxItems: 1
+
   autorepeat:
     description: |
       Enables the Linux input system's autorepeat feature on the input device.
 
   linux,keycodes:
     minItems: 3
-    maxItems: 8
+    maxItems: 14
     description: |
       Specifies an array of numeric keycode values to
       be used for the channels. If this property is
@@ -106,6 +124,8 @@ properties:
       is required for a touch to be registered, making the touch sensor less
       sensitive.
       The number of entries must correspond to the number of channels.
+      CAP1114 is an exception where channels 8~14 reuse the eighth entry's
+      threshold, so counts differ.
 
   microchip,calib-sensitivity:
     $ref: /schemas/types.yaml#/definitions/uint32-array
@@ -124,14 +144,16 @@ properties:
       The number of entries must correspond to the number of channels.
 
 patternProperties:
-  "^led@[0-7]$":
+  "^led@[0-9a]$":
     type: object
     description: CAP11xx LEDs
     $ref: /schemas/leds/common.yaml#
 
     properties:
       reg:
-        enum: [0, 1, 2, 3, 4, 5, 6, 7]
+        description: LED channel number
+        minimum: 0
+        maximum: 10
 
       label: true
 
@@ -156,9 +178,64 @@ allOf:
               - microchip,cap1206
               - microchip,cap1293
               - microchip,cap1298
+    then:
+      properties:
+        reset-gpios: false
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - microchip,cap1114
+    then:
+      properties:
+        linux,keycodes:
+          minItems: 14
+    else:
+      properties:
+        linux,keycodes:
+          maxItems: 8
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - microchip,cap1106
+              - microchip,cap1203
+              - microchip,cap1206
+              - microchip,cap1293
+              - microchip,cap1298
+    then:
+      patternProperties:
+        "^led@": false
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - microchip,cap1126
     then:
       patternProperties:
-        "^led@[0-7]$": false
+        "^led@":
+          properties:
+            reg:
+              maximum: 1
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - microchip,cap1188
+    then:
+      patternProperties:
+        "^led@":
+          properties:
+            reg:
+              maximum: 7
 
   - if:
       properties:
@@ -166,6 +243,7 @@ allOf:
           contains:
             enum:
               - microchip,cap1106
+              - microchip,cap1114
               - microchip,cap1126
               - microchip,cap1188
               - microchip,cap1203
@@ -183,6 +261,8 @@ additionalProperties: false
 
 examples:
   - |
+    #include <dt-bindings/gpio/gpio.h>
+
     i2c {
       #address-cells = <1>;
       #size-cells = <0>;
@@ -204,6 +284,8 @@ examples:
                          <109>, /* KEY_PAGEDOWN */
                          <104>; /* KEY_PAGEUP */
 
+        reset-gpios = <&gpio 17 GPIO_ACTIVE_HIGH>;
+
         #address-cells = <1>;
         #size-cells = <0>;
 
diff --git a/Documentation/devicetree/bindings/input/syna,rmi4.yaml b/Documentation/devicetree/bindings/input/syna,rmi4.yaml
index 8685ef4481f4a..fb4804ac3544d 100644
--- a/Documentation/devicetree/bindings/input/syna,rmi4.yaml
+++ b/Documentation/devicetree/bindings/input/syna,rmi4.yaml
@@ -18,9 +18,14 @@ description: |
 
 properties:
   compatible:
-    enum:
-      - syna,rmi4-i2c
-      - syna,rmi4-spi
+    oneOf:
+      - enum:
+          - syna,rmi4-i2c
+          - syna,rmi4-spi
+      - items:
+          - enum:
+              - syna,rmi4-s3706b  # OnePlus 6/6T
+          - const: syna,rmi4-i2c
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,maili-rpmh.yaml b/Documentation/devicetree/bindings/interconnect/qcom,maili-rpmh.yaml
new file mode 100644
index 0000000000000..3db8d8b23219f
--- /dev/null
+++ b/Documentation/devicetree/bindings/interconnect/qcom,maili-rpmh.yaml
@@ -0,0 +1,127 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interconnect/qcom,maili-rpmh.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm RPMh Network-On-Chip Interconnect on Maili SoC
+
+maintainers:
+  - Raviteja Laggyshetty <raviteja.laggyshetty@oss.qualcomm.com>
+
+description: |
+  RPMh interconnect providers support system bandwidth requirements through
+  RPMh hardware accelerators known as Bus Clock Manager (BCM). The provider is
+  able to communicate with the BCM through the Resource State Coordinator (RSC)
+  associated with each execution environment. Provider nodes must point to at
+  least one RPMh device child node pertaining to their RSC and each provider
+  can map to multiple RPMh resources.
+
+  See also: include/dt-bindings/interconnect/qcom,maili-rpmh.h
+
+properties:
+  compatible:
+    enum:
+      - qcom,maili-aggre-noc
+      - qcom,maili-clk-virt
+      - qcom,maili-cnoc-main
+      - qcom,maili-gem-noc
+      - qcom,maili-llclpi-noc
+      - qcom,maili-lpass-ag-noc
+      - qcom,maili-lpass-lpiaon-noc
+      - qcom,maili-lpass-lpicx-noc
+      - qcom,maili-mc-virt
+      - qcom,maili-mmss-noc
+      - qcom,maili-nsp-noc
+      - qcom,maili-pcie-anoc
+      - qcom,maili-stdst-cfg
+      - qcom,maili-stdst-main
+      - qcom,maili-system-noc
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    minItems: 2
+    maxItems: 3
+
+required:
+  - compatible
+
+allOf:
+  - $ref: qcom,rpmh-common.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,maili-clk-virt
+              - qcom,maili-mc-virt
+    then:
+      properties:
+        reg: false
+    else:
+      required:
+        - reg
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,maili-aggre-noc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: aggre UFS PHY AXI clock
+            - description: aggre USB3 PRIM AXI clock
+            - description: RPMH CC IPA clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,maili-pcie-anoc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: aggre-NOC PCIe AXI clock
+            - description: cfg-NOC PCIe a-NOC AHB clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,maili-aggre-noc
+              - qcom,maili-pcie-anoc
+    then:
+      required:
+        - clocks
+    else:
+      properties:
+        clocks: false
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    gem_noc: interconnect@31100000 {
+      compatible = "qcom,maili-gem-noc";
+      reg = <0x31100000 0x160200>;
+      #interconnect-cells = <2>;
+      qcom,bcm-voters = <&apps_bcm_voter>;
+    };
+
+    aggre_noc: interconnect@f00000 {
+      compatible = "qcom,maili-aggre-noc";
+      reg = <0x00f00000 0x56200>;
+      #interconnect-cells = <2>;
+      clocks = <&gcc_phy_axi_clk>,
+               <&gcc_prim_axi_clk>,
+               <&rpmhcc_ipa_clk>;
+      qcom,bcm-voters = <&apps_bcm_voter>;
+    };
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml b/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml
index ff64225e82817..c0a5e925ff954 100644
--- a/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml
+++ b/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml
@@ -13,7 +13,7 @@ description: |
   Bandwidth Monitor measures current throughput on buses between various NoC
   fabrics and provides information when it crosses configured thresholds.
 
-  Certain SoCs might have more than one Bandwidth Monitors, for example on SDM845::
+  Certain SoCs might have more than one Bandwidth Monitors, for example on SDM845:
    - Measuring the bandwidth between CPUs and Last Level Cache Controller -
      called just BWMON,
    - Measuring the bandwidth between Last Level Cache Controller and memory
@@ -25,6 +25,7 @@ properties:
       - const: qcom,msm8998-bwmon       # BWMON v4
       - items:
           - enum:
+              - qcom,eliza-cpu-bwmon
               - qcom,glymur-cpu-bwmon
               - qcom,hawi-cpu-bwmon
               - qcom,kaanapali-cpu-bwmon
@@ -46,12 +47,14 @@ properties:
           - const: qcom,sdm845-bwmon    # BWMON v4, unified register space
       - items:
           - enum:
+              - qcom,eliza-llcc-bwmon
               - qcom,hawi-llcc-bwmon
               - qcom,qcs615-llcc-bwmon
               - qcom,qcs8300-llcc-bwmon
               - qcom,sa8775p-llcc-bwmon
               - qcom,sc7180-llcc-bwmon
               - qcom,sc8280xp-llcc-bwmon
+              - qcom,shikra-cpu-bwmon
               - qcom,sm6350-cpu-bwmon
               - qcom,sm8250-llcc-bwmon
               - qcom,sm8550-llcc-bwmon
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,osm-l3.yaml b/Documentation/devicetree/bindings/interconnect/qcom,osm-l3.yaml
index 41b9f758bf8b8..3b8ebe17a9760 100644
--- a/Documentation/devicetree/bindings/interconnect/qcom,osm-l3.yaml
+++ b/Documentation/devicetree/bindings/interconnect/qcom,osm-l3.yaml
@@ -17,6 +17,8 @@ description:
 properties:
   compatible:
     oneOf:
+      - enum:
+          - qcom,shikra-epss-l3
       - items:
           - enum:
               - qcom,sc7180-osm-l3
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,sc8280xp-rpmh.yaml b/Documentation/devicetree/bindings/interconnect/qcom,sc8280xp-rpmh.yaml
index 2a5a7594bafdc..cd327a3bf3b9d 100644
--- a/Documentation/devicetree/bindings/interconnect/qcom,sc8280xp-rpmh.yaml
+++ b/Documentation/devicetree/bindings/interconnect/qcom,sc8280xp-rpmh.yaml
@@ -32,18 +32,115 @@ properties:
       - qcom,sc8280xp-nspb-noc
       - qcom,sc8280xp-system-noc
 
+  reg:
+    maxItems: 1
+
+  clocks:
+    minItems: 7
+    maxItems: 8
+
 required:
   - compatible
 
 allOf:
   - $ref: qcom,rpmh-common.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,sc8280xp-clk-virt
+              - qcom,sc8280xp-mc-virt
+    then:
+      properties:
+        reg: false
+        clocks: false
+    else:
+      required:
+        - reg
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,sc8280xp-aggre1-noc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: aggre UFS PHY AXI clock
+            - description: aggre USB3 PRIM AXI clock
+            - description: aggre USB3 SEC AXI clock
+            - description: aggre USB3 MP AXI clock
+            - description: aggre USB4 AXI clock
+            - description: aggre USB4 1 AXI clock
+            - description: aggre USB NOC SOUTH AXI clock
+            - description: RPMH CC IPA clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,sc8280xp-aggre2-noc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: aggre NOC PCIE0 tunnel AXI clock
+            - description: aggre NOC PCIE1 tunnel AXI clock
+            - description: aggre NOC PCIE 4 AXI clock
+            - description: aggre NOC PCIE SOUTH SF AXI clock
+            - description: aggre UFS CARD AXI clock
+            - description: DDRSS GPU AXI clock
+            - description: DDRSS PCIE SF TBU clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,sc8280xp-aggre1-noc
+              - qcom,sc8280xp-aggre2-noc
+    then:
+      required:
+        - clocks
+    else:
+      properties:
+        clocks: false
 
 unevaluatedProperties: false
 
 examples:
   - |
-    interconnect-0 {
+    #include <dt-bindings/clock/qcom,gcc-sc8280xp.h>
+    #include <dt-bindings/clock/qcom,rpmh.h>
+
+    interconnect {
+        compatible = "qcom,sc8280xp-clk-virt";
+        #interconnect-cells = <2>;
+        qcom,bcm-voters = <&apps_bcm_voter>;
+    };
+
+    interconnect@9100000 {
+        compatible = "qcom,sc8280xp-gem-noc";
+        reg = <0x9100000 0xb8400>;
+        #interconnect-cells = <2>;
+        qcom,bcm-voters = <&apps_bcm_voter>;
+    };
+
+    interconnect@16c0000 {
         compatible = "qcom,sc8280xp-aggre1-noc";
+        reg = <0x16c0000 0x3af80>;
         #interconnect-cells = <2>;
         qcom,bcm-voters = <&apps_bcm_voter>;
+        clocks = <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>,
+                 <&gcc GCC_AGGRE_USB3_PRIM_AXI_CLK>,
+                 <&gcc GCC_AGGRE_USB3_SEC_AXI_CLK>,
+                 <&gcc GCC_AGGRE_USB3_MP_AXI_CLK>,
+                 <&gcc GCC_AGGRE_USB4_AXI_CLK>,
+                 <&gcc GCC_AGGRE_USB4_1_AXI_CLK>,
+                 <&gcc GCC_AGGRE_USB_NOC_SOUTH_AXI_CLK>,
+                 <&rpmhcc RPMH_IPA_CLK>;
     };
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,x1e80100-rpmh.yaml b/Documentation/devicetree/bindings/interconnect/qcom,x1e80100-rpmh.yaml
index 0840b0ec6e276..d863cddb21ac5 100644
--- a/Documentation/devicetree/bindings/interconnect/qcom,x1e80100-rpmh.yaml
+++ b/Documentation/devicetree/bindings/interconnect/qcom,x1e80100-rpmh.yaml
@@ -46,6 +46,10 @@ properties:
   reg:
     maxItems: 1
 
+  clocks:
+    minItems: 1
+    maxItems: 6
+
 required:
   - compatible
 
@@ -65,6 +69,73 @@ allOf:
       required:
         - reg
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,x1e80100-aggre1-noc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: aggre UFS PHY AXI clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,x1e80100-usb-north-anoc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: aggre USB2 PRIM AXI clock
+            - description: aggre USB3 MP AXI clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,x1e80100-usb-south-anoc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: aggre USB3 PRIM AXI clock
+            - description: aggre USB3 SEC AXI clock
+            - description: aggre USB3 TERT AXI clock
+            - description: aggre USB4_0 AXI clock
+            - description: aggre USB4_1 AXI clock
+            - description: aggre USB4_2 AXI clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,x1e80100-aggre2-noc
+              - qcom,x1e80100-clk-virt
+              - qcom,x1e80100-cnoc-cfg
+              - qcom,x1e80100-cnoc-main
+              - qcom,x1e80100-gem-noc
+              - qcom,x1e80100-lpass-ag-noc
+              - qcom,x1e80100-lpass-lpiaon-noc
+              - qcom,x1e80100-lpass-lpicx-noc
+              - qcom,x1e80100-mc-virt
+              - qcom,x1e80100-mmss-noc
+              - qcom,x1e80100-nsp-noc
+              - qcom,x1e80100-pcie-center-anoc
+              - qcom,x1e80100-pcie-north-anoc
+              - qcom,x1e80100-pcie-south-anoc
+              - qcom,x1e80100-system-noc
+              - qcom,x1e80100-usb-center-anoc
+    then:
+      properties:
+        clocks: false
+
 unevaluatedProperties: false
 
 examples:
@@ -80,4 +151,5 @@ examples:
       reg = <0x016e0000 0x14400>;
       #interconnect-cells = <2>;
       qcom,bcm-voters = <&apps_bcm_voter>;
+      clocks = <&gcc_aggre_ufs_phy_axi_clk>;
     };
diff --git a/Documentation/devicetree/bindings/interconnect/samsung,exynos-bus.yaml b/Documentation/devicetree/bindings/interconnect/samsung,exynos-bus.yaml
index 5e26e48c7217d..0203959c89958 100644
--- a/Documentation/devicetree/bindings/interconnect/samsung,exynos-bus.yaml
+++ b/Documentation/devicetree/bindings/interconnect/samsung,exynos-bus.yaml
@@ -23,7 +23,7 @@ description: |
   The each AXI bus has the owned source clock but, has not the only owned power
   line. The power line might be shared among one more sub-blocks.  So, we can
   divide into two type of device as the role of each sub-block.  There are two
-  type of bus devices as following::
+  type of bus devices as following:
    - parent bus device
    - passive bus device
 
@@ -44,8 +44,8 @@ description: |
   able to support the bus frequency for all Exynos SoCs.
 
   Detailed correlation between sub-blocks and power line according
-  to Exynos SoC::
-   - In case of Exynos3250, there are two power line as following::
+  to Exynos SoC:
+   - In case of Exynos3250, there are two power line as following:
      VDD_MIF |--- DMC (Dynamic Memory Controller)
 
      VDD_INT |--- LEFTBUS (parent device)
@@ -89,7 +89,7 @@ description: |
        |L5   |200000 |200000  |400000 |300000 |       ||1000000 |
        ----------------------------------------------------------
 
-   - In case of Exynos4210, there is one power line as following::
+   - In case of Exynos4210, there is one power line as following:
      VDD_INT |--- DMC (parent device, Dynamic Memory Controller)
        |--- LEFTBUS
        |--- PERIL
@@ -106,7 +106,7 @@ description: |
        |--- LCD0
        |--- LCD1
 
-   - In case of Exynos4x12, there are two power line as following::
+   - In case of Exynos4x12, there are two power line as following:
      VDD_MIF |--- DMC (Dynamic Memory Controller)
 
      VDD_INT |--- LEFTBUS (parent device)
@@ -124,7 +124,7 @@ description: |
        |--- LCD0
        |--- ISP
 
-   - In case of Exynos5422, there are two power line as following::
+   - In case of Exynos5422, there are two power line as following:
      VDD_MIF |--- DREX 0 (parent device, DRAM EXpress controller)
              |--- DREX 1
 
@@ -143,7 +143,7 @@ description: |
        |--- FSYS
        |--- FSYS2
 
-   - In case of Exynos5433, there is VDD_INT power line as following::
+   - In case of Exynos5433, there is VDD_INT power line as following:
      VDD_INT |--- G2D (parent device)
        |--- MSCL
        |--- GSCL
diff --git a/Documentation/devicetree/bindings/interrupt-controller/realtek,rtl-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/realtek,rtl-intc.yaml
index 833a01cdd1b17..5c52e57c6647e 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/realtek,rtl-intc.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/realtek,rtl-intc.yaml
@@ -32,8 +32,9 @@ properties:
 
   "#interrupt-cells":
     description:
-      SoC interrupt line index.
-    const: 1
+      First cell is the SoC interrupt line index. Optional second cell
+      specifies the parent interrupt index to route to.
+    enum: [1, 2]
 
   reg:
     minItems: 1
diff --git a/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml b/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml
index decc43df3c839..78d8cd77b4ecf 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml
@@ -71,6 +71,7 @@ properties:
       - items:
           - enum:
               - allwinner,sun20i-d1-plic
+              - canaan,k230-plic
               - sophgo,cv1800b-plic
               - sophgo,cv1812h-plic
               - sophgo,sg2000-plic
diff --git a/Documentation/devicetree/bindings/iommu/nvidia,tegra264-cmdqv.yaml b/Documentation/devicetree/bindings/iommu/nvidia,tegra264-cmdqv.yaml
index 3f5006a59805b..76ef34fe5c723 100644
--- a/Documentation/devicetree/bindings/iommu/nvidia,tegra264-cmdqv.yaml
+++ b/Documentation/devicetree/bindings/iommu/nvidia,tegra264-cmdqv.yaml
@@ -38,5 +38,5 @@ examples:
     cmdqv@5200000 {
             compatible = "nvidia,tegra264-cmdqv";
             reg = <0x5200000 0x830000>;
-            interrupts = <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>;
+            interrupts = <GIC_SPI 19 IRQ_TYPE_EDGE_RISING>;
     };
diff --git a/Documentation/devicetree/bindings/leds/backlight/awinic,aw99706.yaml b/Documentation/devicetree/bindings/leds/backlight/awinic,aw99706.yaml
index f48ce7a3434da..60eae50ff742f 100644
--- a/Documentation/devicetree/bindings/leds/backlight/awinic,aw99706.yaml
+++ b/Documentation/devicetree/bindings/leds/backlight/awinic,aw99706.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Awinic AW99706 6-channel WLED Backlight Driver
 
 maintainers:
-  - Junjie Cao <caojunjie650@gmail.com>
+  - Junjie Cao <junjie.cao@linux.dev>
 
 allOf:
   - $ref: common.yaml#
diff --git a/Documentation/devicetree/bindings/leds/backlight/silergy,sy7758.yaml b/Documentation/devicetree/bindings/leds/backlight/silergy,sy7758.yaml
new file mode 100644
index 0000000000000..64af6b34641ac
--- /dev/null
+++ b/Documentation/devicetree/bindings/leds/backlight/silergy,sy7758.yaml
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/leds/backlight/silergy,sy7758.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Silergy SY7758 6-channel High Efficiency LED Driver
+
+maintainers:
+  - Neil Armstrong <neil.armstrong@linaro.org>
+
+description:
+  Silergy SY7758 is a high efficiency 6-channels LED backlight
+  driver with I2C brightness control.
+
+allOf:
+  - $ref: common.yaml#
+
+properties:
+  compatible:
+    const: silergy,sy7758
+
+  reg:
+    maxItems: 1
+
+  vdd-supply: true
+
+  enable-gpios:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - enable-gpios
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        backlight@11 {
+            compatible = "silergy,sy7758";
+            reg = <0x11>;
+            vdd-supply = <&bl_vdd>;
+            enable-gpios = <&gpio 16 GPIO_ACTIVE_HIGH>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/leds/qcom,pm8058-led.yaml b/Documentation/devicetree/bindings/leds/qcom,pm8058-led.yaml
index b409b2a8b5c5b..5165bfddcd543 100644
--- a/Documentation/devicetree/bindings/leds/qcom,pm8058-led.yaml
+++ b/Documentation/devicetree/bindings/leds/qcom,pm8058-led.yaml
@@ -10,10 +10,10 @@ maintainers:
   - Krzysztof Kozlowski <krzk@kernel.org>
 
 description: |
-  The Qualcomm PM8058 contains an LED block for up to six LEDs:: three normal
+  The Qualcomm PM8058 contains an LED block for up to six LEDs: three normal
   LEDs, two "flash" LEDs and one "keypad backlight" LED. The names are quoted
   because sometimes these LED drivers are used for wildly different things than
-  flash or keypad backlight:: their names are more of a suggestion than a
+  flash or keypad backlight: their names are more of a suggestion than a
   hard-wired usecase.
 
   Hardware-wise the different LEDs support slightly different output currents.
diff --git a/Documentation/devicetree/bindings/leds/skyworks,aat1290.yaml b/Documentation/devicetree/bindings/leds/skyworks,aat1290.yaml
index a6aaa92dbccd9..65576dfdca110 100644
--- a/Documentation/devicetree/bindings/leds/skyworks,aat1290.yaml
+++ b/Documentation/devicetree/bindings/leds/skyworks,aat1290.yaml
@@ -11,7 +11,7 @@ maintainers:
   - Krzysztof Kozlowski <krzk@kernel.org>
 
 description: |
-  The device is controlled through two pins:: FL_EN and EN_SET. The pins when,
+  The device is controlled through two pins: FL_EN and EN_SET. The pins when,
   asserted high, enable flash strobe and movie mode (max 1/2 of flash current)
   respectively. In order to add a capability of selecting the strobe signal
   source (e.g. CPU or camera sensor) there is an additional switch required,
@@ -39,11 +39,11 @@ properties:
       flash-max-microamp:
         description: |
           Maximum flash LED supply current can be calculated using following
-          formula:: I = 1A * 162 kOhm / Rset.
+          formula: I = 1A * 162 kOhm / Rset.
 
       flash-max-timeout-us:
         description: |
-          Maximum flash timeout can be calculated using following formula::
+          Maximum flash timeout can be calculated using following formula:
             T = 8.82 * 10^9 * Ct.
 
     required:
diff --git a/Documentation/devicetree/bindings/media/cec/cec-gpio.yaml b/Documentation/devicetree/bindings/media/cec/cec-gpio.yaml
index 582c6c9cae48d..21118e4bae0f1 100644
--- a/Documentation/devicetree/bindings/media/cec/cec-gpio.yaml
+++ b/Documentation/devicetree/bindings/media/cec/cec-gpio.yaml
@@ -14,7 +14,7 @@ description: |
   hooked up to a pull-up GPIO line and - optionally - the HPD line is hooked up
   to another GPIO line.
 
-  Please note:: the maximum voltage for the CEC line is 3.63V, for the HPD and
+  Please note: the maximum voltage for the CEC line is 3.63V, for the HPD and
   5V lines it is 5.3V. So you may need some sort of level conversion
   circuitry when connecting them to a GPIO line.
 
diff --git a/Documentation/devicetree/bindings/media/i2c/sony,imx355.yaml b/Documentation/devicetree/bindings/media/i2c/sony,imx355.yaml
index 6050d7e7dcfea..d9cdfda699bfe 100644
--- a/Documentation/devicetree/bindings/media/i2c/sony,imx355.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/sony,imx355.yaml
@@ -50,8 +50,15 @@ properties:
 
         properties:
           data-lanes:
-            minItems: 4
-            maxItems: 4
+            oneOf:
+              - items:
+                  - const: 1
+                  - const: 2
+                  - const: 3
+                  - const: 4
+              - items:
+                  - const: 1
+                  - const: 2
 
         required:
           - link-frequencies
diff --git a/Documentation/devicetree/bindings/media/i2c/sony,imx678.yaml b/Documentation/devicetree/bindings/media/i2c/sony,imx678.yaml
new file mode 100644
index 0000000000000..098f72544216a
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/i2c/sony,imx678.yaml
@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+# Copyright (C) 2026 Ideas on Board Oy
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/media/i2c/sony,imx678.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Sony IMX678 Sensor
+
+maintainers:
+  - Jai Luthra <jai.luthra@ideasonboard.com>
+
+description:
+  Sony IMX678 diagonal 8.86 mm (Type 1/1.8) CMOS active pixel type solid-state
+  image sensor with a square pixel array and 8.40M (3856x2180) effective pixels.
+
+allOf:
+  - $ref: /schemas/media/video-interface-devices.yaml#
+
+properties:
+  compatible:
+    items:
+      - enum:
+          - sony,imx678-aamr
+          - sony,imx678-aaqr
+      - const: sony,imx678
+    description:
+      The IMX678 sensor exists in a colour variant (IMX678-AAQR) and a mono
+      variant (IMX678-AAMR). The variant-specific compatible describes the
+      exact sensor variant, with the generic IMX678 compatible as a fallback.
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    description: |
+      Input clock (INCK). Only the rates listed in the datasheet are supported:
+      13.5, 18, 24, 27, 36, 37.125, 72, and 74.25 MHz.
+    maxItems: 1
+
+  avdd-supply:
+    description: Analog power supply (3.3V)
+
+  ovdd-supply:
+    description: Interface power supply (1.8V)
+
+  dvdd-supply:
+    description: Digital power supply (1.1V)
+
+  reset-gpios:
+    description: Sensor RESETN (XCLR) GPIO
+    maxItems: 1
+
+  port:
+    $ref: /schemas/graph.yaml#/$defs/port-base
+    additionalProperties: false
+
+    properties:
+      endpoint:
+        $ref: /schemas/media/video-interfaces.yaml#
+        unevaluatedProperties: false
+
+        properties:
+          data-lanes:
+            oneOf:
+              - items:
+                  - const: 1
+                  - const: 2
+              - items:
+                  - const: 1
+                  - const: 2
+                  - const: 3
+                  - const: 4
+              - items:
+                  - const: 1
+                  - const: 2
+                  - const: 3
+                  - const: 4
+                  - const: 5
+                  - const: 6
+                  - const: 7
+                  - const: 8
+
+        required:
+          - data-lanes
+          - link-frequencies
+
+    required:
+      - endpoint
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - avdd-supply
+  - ovdd-supply
+  - dvdd-supply
+  - port
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        camera@1a {
+            compatible = "sony,imx678-aaqr", "sony,imx678";
+            reg = <0x1a>;
+            clocks = <&imx678_clk>;
+
+            avdd-supply = <&camera_avdd_3v3>;
+            ovdd-supply = <&camera_ovdd_1v8>;
+            dvdd-supply = <&camera_dvdd_1v1>;
+
+            reset-gpios = <&gpio 0 GPIO_ACTIVE_LOW>;
+
+            port {
+                imx678: endpoint {
+                    remote-endpoint = <&cam>;
+                    data-lanes = <1 2 3 4>;
+                    link-frequencies = /bits/ 64 <720000000>;
+                };
+            };
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/media/i2c/st,vd55g1.yaml b/Documentation/devicetree/bindings/media/i2c/st,vd55g1.yaml
index 060ac6829b661..58b1f9e85a9d1 100644
--- a/Documentation/devicetree/bindings/media/i2c/st,vd55g1.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/st,vd55g1.yaml
@@ -27,9 +27,10 @@ properties:
   compatible:
     enum:
       - st,vd55g1
+      - st,vd55g4
       - st,vd65g4
     description:
-      VD55G1 is the monochrome variant, while VD65G4 is the color one.
+      VD55G1 and VD55G4 are monochrome variants, while VD65G4 is a color one.
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/media/rc.yaml b/Documentation/devicetree/bindings/media/rc.yaml
index dedc5a4b81ecb..43d84296e70db 100644
--- a/Documentation/devicetree/bindings/media/rc.yaml
+++ b/Documentation/devicetree/bindings/media/rc.yaml
@@ -122,10 +122,10 @@ properties:
       - rc-proteus-2309
       - rc-purpletv
       - rc-pv951
-      - rc-rc5-tv
       - rc-rc6-mce
       - rc-real-audio-220-32-keys
       - rc-reddo
+      - rc-siemens-gigaset-rc20
       - rc-snapstream-firefly
       - rc-streamzap
       - rc-su3000
@@ -147,11 +147,11 @@ properties:
       - rc-tt-1500
       - rc-twinhan-dtv-cab-ci
       - rc-twinhan1027
+      - rc-vega-s9x
       - rc-videomate-k100
       - rc-videomate-s350
       - rc-videomate-tv-pvr
       - rc-videostrong-kii-pro
-      - rc-vega-s9x
       - rc-wetek-hub
       - rc-wetek-play2
       - rc-winfast
diff --git a/Documentation/devicetree/bindings/mfd/marvell,88pm886-a1.yaml b/Documentation/devicetree/bindings/mfd/marvell,88pm886-a1.yaml
index 92a72a99fd790..9402628983533 100644
--- a/Documentation/devicetree/bindings/mfd/marvell,88pm886-a1.yaml
+++ b/Documentation/devicetree/bindings/mfd/marvell,88pm886-a1.yaml
@@ -29,7 +29,7 @@ properties:
     type: object
     additionalProperties: false
     patternProperties:
-      "^(ldo(1[0-6]|[1-9])|buck[1-5])$":
+      "^(ldo(1[0-6]|[1-9])|buck[1-5]|vbus)$":
         type: object
         $ref: /schemas/regulator/regulator.yaml#
         description: LDO or buck regulator.
diff --git a/Documentation/devicetree/bindings/mfd/mediatek,mt6397.yaml b/Documentation/devicetree/bindings/mfd/mediatek,mt6397.yaml
index 3cbc0dc12c319..d2de227ae7a6d 100644
--- a/Documentation/devicetree/bindings/mfd/mediatek,mt6397.yaml
+++ b/Documentation/devicetree/bindings/mfd/mediatek,mt6397.yaml
@@ -159,6 +159,23 @@ properties:
     required:
       - compatible
 
+  efuse:
+    type: object
+    unevaluatedProperties: false
+    description:
+      The efuse is responsible for storing calibration data, such as thermal
+      sensor calibration.
+
+    properties:
+      compatible:
+        const: mediatek,mt6323-efuse
+
+      nvmem-layout:
+        $ref: /schemas/nvmem/layouts/nvmem-layout.yaml#
+
+    required:
+      - compatible
+
   leds:
     type: object
     additionalProperties: false
@@ -307,6 +324,10 @@ examples:
         interrupt-controller;
         #interrupt-cells = <2>;
 
+        efuse {
+          compatible = "mediatek,mt6323-efuse";
+        };
+
         leds {
             compatible = "mediatek,mt6323-led";
             #address-cells = <1>;
diff --git a/Documentation/devicetree/bindings/mfd/mediatek,mt8195-scpsys.yaml b/Documentation/devicetree/bindings/mfd/mediatek,mt8195-scpsys.yaml
index 4cafa381979bf..9f073d0c28ef9 100644
--- a/Documentation/devicetree/bindings/mfd/mediatek,mt8195-scpsys.yaml
+++ b/Documentation/devicetree/bindings/mfd/mediatek,mt8195-scpsys.yaml
@@ -26,6 +26,7 @@ properties:
           - mediatek,mt8183-scpsys
           - mediatek,mt8186-scpsys
           - mediatek,mt8188-scpsys
+          - mediatek,mt8189-scpsys
           - mediatek,mt8192-scpsys
           - mediatek,mt8195-scpsys
           - mediatek,mt8365-scpsys
diff --git a/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml b/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml
index 644c42b5e2e57..809be2756a0c0 100644
--- a/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml
+++ b/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml
@@ -80,6 +80,7 @@ properties:
           - qcom,pmcx0102
           - qcom,pmd8028
           - qcom,pmd9635
+          - qcom,pmg1110
           - qcom,pmh0101
           - qcom,pmh0104
           - qcom,pmh0110
diff --git a/Documentation/devicetree/bindings/mfd/syscon-common.yaml b/Documentation/devicetree/bindings/mfd/syscon-common.yaml
index 14a08e7bc8bdd..2d5eef5add544 100644
--- a/Documentation/devicetree/bindings/mfd/syscon-common.yaml
+++ b/Documentation/devicetree/bindings/mfd/syscon-common.yaml
@@ -32,6 +32,7 @@ properties:
   compatible:
     contains:
       const: syscon
+    # Always require a specific compatible for syscon
     minItems: 2
     maxItems: 5  # Should be enough
 
@@ -52,11 +53,44 @@ allOf:
           contains:
             const: simple-mfd
     then:
+      # Always require a specific compatible for syscon with simple-mfd
       properties:
         compatible:
           minItems: 3
           maxItems: 5
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: simple-bus
+    then:
+      # simple-bus conflicts with syscon - if a device is a system controller
+      # with miscellaneous registers, then it has at least one dedicated
+      # function thus it is not a simple bus. Allow existing exceptions.
+      if:
+        properties:
+          compatible:
+            not:
+              contains:
+                # This list CANNOT grow
+                enum:
+                  - cznic,turris1x-cpld
+                  - img,pistachio-cr-periph
+                  - ti,am3352-scm-conf
+                  - ti,am4372-scm-conf
+                  - ti,dm814-scm-conf
+                  - ti,dm8168-scm-conf
+                  - ti,dra7-scm-conf
+                  - ti,omap2-scm-conf
+                  - ti,omap3-scm-conf
+                  - ti,omap4-sysc-padconf-global
+                  - ti,omap5-scm-wkup-conf
+                  - ti,omap5-sysc-padconf-global
+      then:
+        required:
+          - incorrect-usage-of-simple-bus-and-syscon
+
 additionalProperties: true
 
 examples:
diff --git a/Documentation/devicetree/bindings/mfd/syscon.yaml b/Documentation/devicetree/bindings/mfd/syscon.yaml
index e228670880631..fe882d3f828b1 100644
--- a/Documentation/devicetree/bindings/mfd/syscon.yaml
+++ b/Documentation/devicetree/bindings/mfd/syscon.yaml
@@ -19,122 +19,6 @@ description: |
 maintainers:
   - Lee Jones <lee@kernel.org>
 
-# Need a select with all compatibles listed for compatibility with older
-# dtschema (<2024.02), so this will not be selected for other schemas having
-# syscon fallback.
-select:
-  properties:
-    compatible:
-      contains:
-        enum:
-          - airoha,en7581-pbus-csr
-          - al,alpine-sysfabric-service
-          - allwinner,sun8i-a83t-system-controller
-          - allwinner,sun8i-h3-system-controller
-          - allwinner,sun8i-v3s-system-controller
-          - allwinner,sun50i-a64-system-controller
-          - altr,l3regs
-          - altr,sdr-ctl
-          - amd,pensando-elba-syscon
-          - amlogic,meson-mx-assist
-          - amlogic,meson-mx-bootrom
-          - amlogic,meson8-analog-top
-          - amlogic,meson8b-analog-top
-          - amlogic,meson8-pmu
-          - amlogic,meson8b-pmu
-          - apm,merlin-poweroff-mailbox
-          - apm,mustang-poweroff-mailbox
-          - apm,xgene-csw
-          - apm,xgene-efuse
-          - apm,xgene-mcb
-          - apm,xgene-rb
-          - apm,xgene-scu
-          - atmel,sama5d2-sfrbu
-          - atmel,sama5d3-nfc-io
-          - atmel,sama5d3-sfrbu
-          - atmel,sama5d4-sfrbu
-          - axis,artpec6-syscon
-          - brcm,cru-clkset
-          - brcm,sr-cdru
-          - brcm,sr-mhb
-          - cirrus,ep7209-syscon1
-          - cirrus,ep7209-syscon2
-          - cirrus,ep7209-syscon3
-          - cnxt,cx92755-uc
-          - econet,en751221-chip-scu
-          - freecom,fsg-cs2-system-controller
-          - fsl,imx93-aonmix-ns-syscfg
-          - fsl,imx93-wakeupmix-syscfg
-          - fsl,ls1088a-reset
-          - fsl,vf610-anatop
-          - fsl,vf610-mscm-cpucfg
-          - hisilicon,dsa-subctrl
-          - hisilicon,hi6220-sramctrl
-          - hisilicon,hip04-ppe
-          - hisilicon,pcie-sas-subctrl
-          - hisilicon,peri-subctrl
-          - hpe,gxp-sysreg
-          - loongson,ls1b-syscon
-          - loongson,ls1c-syscon
-          - lsi,axxia-syscon
-          - marvell,armada-3700-cpu-misc
-          - marvell,armada-3700-nb-pm
-          - marvell,armada-3700-avs
-          - marvell,armada-3700-usb2-host-device-misc
-          - marvell,armada-3700-usb2-host-misc
-          - marvell,dove-global-config
-          - mediatek,mt2701-pctl-a-syscfg
-          - mediatek,mt2712-pctl-a-syscfg
-          - mediatek,mt6397-pctl-pmic-syscfg
-          - mediatek,mt7981-topmisc
-          - mediatek,mt7988-topmisc
-          - mediatek,mt8135-pctl-a-syscfg
-          - mediatek,mt8135-pctl-b-syscfg
-          - mediatek,mt8173-pctl-a-syscfg
-          - mediatek,mt8365-syscfg
-          - microchip,lan966x-cpu-syscon
-          - microchip,mpfs-control-scb
-          - microchip,mpfs-sysreg-scb
-          - microchip,sam9x60-sfr
-          - microchip,sama7d65-ddr3phy
-          - microchip,sama7d65-sfrbu
-          - microchip,sama7g5-ddr3phy
-          - mscc,ocelot-cpu-syscon
-          - mstar,msc313-pmsleep
-          - nuvoton,ma35d1-sys
-          - nuvoton,wpcm450-shm
-          - nxp,s32g2-gpr
-          - nxp,s32g3-gpr
-          - qcom,apq8064-mmss-sfpb
-          - qcom,apq8064-sps-sic
-          - rockchip,px30-qos
-          - rockchip,rk3036-qos
-          - rockchip,rk3066-qos
-          - rockchip,rk3128-qos
-          - rockchip,rk3228-qos
-          - rockchip,rk3288-qos
-          - rockchip,rk3368-qos
-          - rockchip,rk3399-qos
-          - rockchip,rk3528-qos
-          - rockchip,rk3562-qos
-          - rockchip,rk3568-qos
-          - rockchip,rk3576-qos
-          - rockchip,rk3588-qos
-          - rockchip,rv1126-qos
-          - st,spear1340-misc
-          - stericsson,nomadik-pmu
-          - starfive,jh7100-sysmain
-          - ti,am62-opp-efuse-table
-          - ti,am62-usb-phy-ctrl
-          - ti,am625-dss-oldi-io-ctrl
-          - ti,am62p-cpsw-mac-efuse
-          - ti,am654-dss-oldi-io-ctrl
-          - ti,j784s4-acspcie-proxy-ctrl
-          - ti,j784s4-pcie-ctrl
-          - ti,keystone-pllctrl
-  required:
-    - compatible
-
 properties:
   compatible:
     oneOf:
diff --git a/Documentation/devicetree/bindings/mmc/mtk-sd.yaml b/Documentation/devicetree/bindings/mmc/mtk-sd.yaml
index eb3755bdfdf7e..a4d032224dcee 100644
--- a/Documentation/devicetree/bindings/mmc/mtk-sd.yaml
+++ b/Documentation/devicetree/bindings/mmc/mtk-sd.yaml
@@ -193,7 +193,6 @@ allOf:
             - mediatek,mt8183-mmc
             - mediatek,mt8186-mmc
             - mediatek,mt8188-mmc
-            - mediatek,mt8189-mmc
             - mediatek,mt8195-mmc
             - mediatek,mt8196-mmc
             - mediatek,mt8516-mmc
@@ -348,6 +347,34 @@ allOf:
             - const: axi_cg
             - const: ahb_cg
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: mediatek,mt8189-mmc
+    then:
+      properties:
+        clocks:
+          minItems: 6
+          items:
+            - description: source clock
+            - description: HCLK which used for host
+            - description: independent source clock gate
+            - description: bus clock used for internal register access
+            - description: peripheral bus clock gate
+            - description: AXI bus clock gate
+            - description: crypto clock used for data encrypt/decrypt (optional)
+        clock-names:
+          minItems: 6
+          items:
+            - const: source
+            - const: hclk
+            - const: source_cg
+            - const: bus_clk
+            - const: pclk_cg
+            - const: axi_cg
+            - const: crypto
+
 unevaluatedProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/mmc/qcom,sdhci-msm.yaml b/Documentation/devicetree/bindings/mmc/qcom,sdhci-msm.yaml
index bd558a11b7929..b3fcc1673c106 100644
--- a/Documentation/devicetree/bindings/mmc/qcom,sdhci-msm.yaml
+++ b/Documentation/devicetree/bindings/mmc/qcom,sdhci-msm.yaml
@@ -145,6 +145,11 @@ properties:
     $ref: /schemas/types.yaml#/definitions/uint32
     description: platform specific settings for DLL_CONFIG reg.
 
+  qcom,ice:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description:
+      phandle to the Inline Crypto Engine (ICE) hardware block for this controller.
+
   iommus:
     minItems: 1
     maxItems: 8
@@ -198,35 +203,69 @@ allOf:
             enum:
               - qcom,sdhci-msm-v4
     then:
-      properties:
-        reg:
-          minItems: 2
-          items:
-            - description: Host controller register map
-            - description: SD Core register map
-            - description: CQE register map
-            - description: Inline Crypto Engine register map
-        reg-names:
-          minItems: 2
-          items:
-            - const: hc
-            - const: core
-            - const: cqhci
-            - const: ice
+      if:
+        required:
+          - qcom,ice
+      then:
+        properties:
+          reg:
+            minItems: 2
+            items:
+              - description: Host controller register map
+              - description: SD Core register map
+              - description: CQE register map
+          reg-names:
+            minItems: 2
+            items:
+              - const: hc
+              - const: core
+              - const: cqhci
+      else:
+        properties:
+          reg:
+            minItems: 2
+            items:
+              - description: Host controller register map
+              - description: SD Core register map
+              - description: CQE register map
+              - description: Inline Crypto Engine register map
+          reg-names:
+            minItems: 2
+            items:
+              - const: hc
+              - const: core
+              - const: cqhci
+              - const: ice
     else:
-      properties:
-        reg:
-          minItems: 1
-          items:
-            - description: Host controller register map
-            - description: CQE register map
-            - description: Inline Crypto Engine register map
-        reg-names:
-          minItems: 1
-          items:
-            - const: hc
-            - const: cqhci
-            - const: ice
+      if:
+        required:
+          - qcom,ice
+      then:
+        properties:
+          reg:
+            minItems: 1
+            items:
+              - description: Host controller register map
+              - description: CQE register map
+          reg-names:
+            minItems: 1
+            items:
+              - const: hc
+              - const: cqhci
+      else:
+        properties:
+          reg:
+            minItems: 1
+            items:
+              - description: Host controller register map
+              - description: CQE register map
+              - description: Inline Crypto Engine register map
+          reg-names:
+            minItems: 1
+            items:
+              - const: hc
+              - const: cqhci
+              - const: ice
 
 unevaluatedProperties: false
 
diff --git a/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.yaml b/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.yaml
index 4965bb518c54a..21ecfaa454f45 100644
--- a/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.yaml
+++ b/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.yaml
@@ -43,6 +43,7 @@ properties:
               - rockchip,rk3562-dw-mshc
               - rockchip,rk3568-dw-mshc
               - rockchip,rk3588-dw-mshc
+              - rockchip,rv1106-dw-mshc
               - rockchip,rv1108-dw-mshc
               - rockchip,rv1126-dw-mshc
           - const: rockchip,rk3288-dw-mshc
diff --git a/Documentation/devicetree/bindings/mmc/samsung,exynos-dw-mshc.yaml b/Documentation/devicetree/bindings/mmc/samsung,exynos-dw-mshc.yaml
index 27c4060f2f917..223fcc9f651fb 100644
--- a/Documentation/devicetree/bindings/mmc/samsung,exynos-dw-mshc.yaml
+++ b/Documentation/devicetree/bindings/mmc/samsung,exynos-dw-mshc.yaml
@@ -85,7 +85,7 @@ properties:
     description: |
       The value of CIU TX and RX clock phase shift value for HS400 mode
       operation.
-      Valid values for SDR and DDR CIU clock timing::
+      Valid values for SDR and DDR CIU clock timing:
         - valid value for tx phase shift and rx phase shift is 0 to 7.
         - when CIU clock divider value is set to 3, all possible 8 phase shift
           values can be used.
diff --git a/Documentation/devicetree/bindings/mmc/sdhci-st.txt b/Documentation/devicetree/bindings/mmc/sdhci-st.txt
deleted file mode 100644
index ccf82b4ee838e..0000000000000
--- a/Documentation/devicetree/bindings/mmc/sdhci-st.txt
+++ /dev/null
@@ -1,110 +0,0 @@
-* STMicroelectronics sdhci-st MMC/SD controller
-
-This file documents the differences between the core properties in
-Documentation/devicetree/bindings/mmc/mmc.txt and the properties
-used by the sdhci-st driver.
-
-Required properties:
-- compatible:		Must be "st,sdhci" and it can be compatible to "st,sdhci-stih407"
-			to set the internal glue logic used for configuring the MMC
-			subsystem (mmcss) inside the FlashSS (available in STiH407 SoC
-			family).
-
-- clock-names:		Should be "mmc" and "icn".  (NB: The latter is not compulsory)
-			See: Documentation/devicetree/bindings/resource-names.txt
-- clocks:		Phandle to the clock.
-			See: Documentation/devicetree/bindings/clock/clock-bindings.txt
-
-- interrupts:		One mmc interrupt should be described here.
-- interrupt-names:	Should be "mmcirq".
-
-- pinctrl-names:	A pinctrl state names "default" must be defined.
-- pinctrl-0:		Phandle referencing pin configuration of the sd/emmc controller.
-			See: Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt
-
-- reg:			This must provide the host controller base address and it can also
-			contain the FlashSS Top register for TX/RX delay used by the driver
-			to configure DLL inside the flashSS, if so reg-names must also be
-			specified.
-
-Optional properties:
-- reg-names:		Should be "mmc" and "top-mmc-delay". "top-mmc-delay" is optional
-			for eMMC on stih407 family silicon to configure DLL inside FlashSS.
-
-- non-removable:	Non-removable slot. Also used for configuring mmcss in STiH407 SoC
-			family.
-			See:  Documentation/devicetree/bindings/mmc/mmc.txt.
-
-- bus-width:		Number of data lines.
-			See:  Documentation/devicetree/bindings/mmc/mmc.txt.
-
-- max-frequency:	Can be 200MHz, 100MHz or 50MHz (default) and used for
-			configuring the CCONFIG3 in the mmcss.
-			See:  Documentation/devicetree/bindings/mmc/mmc.txt.
-
-- resets:		Phandle and reset specifier pair to softreset line of HC IP.
-			See: Documentation/devicetree/bindings/reset/reset.txt
-
-- vqmmc-supply:		Phandle to the regulator dt node, mentioned as the vcc/vdd
-			supply in eMMC/SD specs.
-
-- sd-uhs-sdr50:	To enable the SDR50 in the mmcss.
-			See:  Documentation/devicetree/bindings/mmc/mmc.txt.
-
-- sd-uhs-sdr104:	To enable the SDR104 in the mmcss.
-			See:  Documentation/devicetree/bindings/mmc/mmc.txt.
-
-- sd-uhs-ddr50:		To enable the DDR50 in the mmcss.
-			See:  Documentation/devicetree/bindings/mmc/mmc.txt.
-
-Example:
-
-/* Example stih416e eMMC configuration */
-
-mmc0: sdhci@fe81e000 {
-	compatible	= "st,sdhci";
-	reg		= <0xfe81e000 0x1000>;
-	interrupts	= <GIC_SPI 127 IRQ_TYPE_NONE>;
-	interrupt-names	= "mmcirq";
-	pinctrl-names	= "default";
-	pinctrl-0	= <&pinctrl_mmc0>;
-	clock-names	= "mmc";
-	clocks		= <&clk_s_a1_ls 1>;
-	bus-width	= <8>
-
-/* Example SD stih407 family configuration */
-
-mmc1: sdhci@9080000 {
-	compatible	= "st,sdhci-stih407", "st,sdhci";
-	reg		= <0x09080000 0x7ff>;
-	reg-names	= "mmc";
-	interrupts	= <GIC_SPI 90 IRQ_TYPE_NONE>;
-	interrupt-names	= "mmcirq";
-	pinctrl-names	= "default";
-	pinctrl-0	= <&pinctrl_sd1>;
-	clock-names	= "mmc";
-	clocks		= <&clk_s_c0_flexgen CLK_MMC_1>;
-	resets		= <&softreset STIH407_MMC1_SOFTRESET>;
-	bus-width	= <4>;
-};
-
-/* Example eMMC stih407 family configuration */
-
-mmc0: sdhci@9060000 {
-	compatible	= "st,sdhci-stih407", "st,sdhci";
-	reg		= <0x09060000 0x7ff>, <0x9061008 0x20>;
-	reg-names	= "mmc", "top-mmc-delay";
-	interrupts	= <GIC_SPI 92 IRQ_TYPE_NONE>;
-	interrupt-names	= "mmcirq";
-	pinctrl-names	= "default";
-	pinctrl-0	= <&pinctrl_mmc0>;
-	clock-names	= "mmc";
-	clocks		= <&clk_s_c0_flexgen CLK_MMC_0>;
-	vqmmc-supply	= <&vmmc_reg>;
-	max-frequency	= <200000000>;
-	bus-width	= <8>;
-	non-removable;
-	sd-uhs-sdr50;
-	sd-uhs-sdr104;
-	sd-uhs-ddr50;
-};
diff --git a/Documentation/devicetree/bindings/mmc/st,sdhci.yaml b/Documentation/devicetree/bindings/mmc/st,sdhci.yaml
new file mode 100644
index 0000000000000..798af599d3749
--- /dev/null
+++ b/Documentation/devicetree/bindings/mmc/st,sdhci.yaml
@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/mmc/st,sdhci.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: STMicroelectronics SDHCI-ST MMC/SD Controller
+
+maintainers:
+  - Peter Griffin <peter.griffin@linaro.org>
+
+description:
+  The STMicroelectronics SDHCI-ST MMC/SD host controller is compliant with
+  the SD Host Controller Interface (SDHCI) specification and is used to
+  interface with MMC, SD and SDIO cards. The ST SDHCI controller extends the
+  standard SDHCI capabilities with platform-specific configurations such as
+  additional register regions, clock inputs, and delay control mechanisms
+  required for signal timing adjustments to support high-speed modes across
+  different ST SoCs.
+
+allOf:
+  - $ref: mmc-controller.yaml#
+  - if:
+      properties:
+        reg:
+          minItems: 2
+      required:
+        - reg
+    then:
+      required:
+        - reg-names
+
+properties:
+  compatible:
+    oneOf:
+      - const: st,sdhci
+      - items:
+          - const: st,sdhci-stih407
+          - const: st,sdhci
+
+  reg:
+    minItems: 1
+    items:
+      - description: MMC controller base registers
+      - description: FlashSS Top registers for TX/RX DLL delay configuration
+
+  reg-names:
+    minItems: 1
+    items:
+      - const: mmc
+      - const: top-mmc-delay
+
+  clocks:
+    minItems: 1
+    items:
+      - description: Clock for the MMC controller
+      - description: Interconnect (ICN) clock
+
+  clock-names:
+    minItems: 1
+    items:
+      - const: mmc
+      - const: icn
+
+  interrupts:
+    maxItems: 1
+
+  interrupt-names:
+    const: mmcirq
+
+  resets:
+    maxItems: 1
+
+  max-frequency:
+    enum: [200000000, 100000000, 50000000]
+    default: 50000000
+
+required:
+  - reg
+  - compatible
+  - clocks
+  - clock-names
+  - interrupts
+  - interrupt-names
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/clock/stih407-clks.h>
+    mmc@9060000 {
+        compatible = "st,sdhci-stih407", "st,sdhci";
+        reg = <0x09060000 0x7ff>, <0x9061008 0x20>;
+        reg-names = "mmc", "top-mmc-delay";
+        interrupts = <GIC_SPI 92 IRQ_TYPE_LEVEL_HIGH>;
+        interrupt-names = "mmcirq";
+        pinctrl-names = "default";
+        pinctrl-0 = <&pinctrl_mmc0>;
+        clock-names = "mmc", "icn";
+        clocks = <&clk_s_c0_flexgen CLK_MMC_0>,
+                 <&clk_s_c0_flexgen CLK_RX_ICN_HVA>;
+        bus-width = <8>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/mmc/ti,omap2420-mmc.yaml b/Documentation/devicetree/bindings/mmc/ti,omap2420-mmc.yaml
new file mode 100644
index 0000000000000..794d5ec0b58e6
--- /dev/null
+++ b/Documentation/devicetree/bindings/mmc/ti,omap2420-mmc.yaml
@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/mmc/ti,omap2420-mmc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments MMC host controller for OMAP1 and 2420
+
+maintainers:
+  - Eduard Bostina <egbostina@gmail.com>
+
+description: |
+  The MMC Host Controller on TI OMAP1 and 2420 family provides
+  an interface for MMC, SD, and SDIO types of memory cards.
+
+allOf:
+  - $ref: /schemas/mmc/mmc-controller.yaml#
+
+properties:
+  compatible:
+    const: ti,omap2420-mmc
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  ti,hwmods:
+    description: |
+      Name of the hwmod associated to the MMC controller. Must be
+      "msdi<n>" where n is the controller instance starting from 1.
+    $ref: /schemas/types.yaml#/definitions/string
+    pattern: "^msdi([1-9]|[1-9][0-9]+)$"
+
+  dmas:
+    maxItems: 2
+
+  dma-names:
+    items:
+      - const: tx
+      - const: rx
+
+required:
+  - compatible
+  - reg
+  - interrupts
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    mmc@4809c000 {
+        compatible = "ti,omap2420-mmc";
+        ti,hwmods = "msdi1";
+        reg = <0x4809c000 0x80>;
+        interrupts = <83>;
+        dmas = <&sdma 61 &sdma 62>;
+        dma-names = "tx", "rx";
+    };
diff --git a/Documentation/devicetree/bindings/mmc/ti-omap.txt b/Documentation/devicetree/bindings/mmc/ti-omap.txt
deleted file mode 100644
index 02fd31cf361d6..0000000000000
--- a/Documentation/devicetree/bindings/mmc/ti-omap.txt
+++ /dev/null
@@ -1,26 +0,0 @@
-* TI MMC host controller for OMAP1 and 2420
-
-The MMC Host Controller on TI OMAP1 and 2420 family provides
-an interface for MMC, SD, and SDIO types of memory cards.
-
-This file documents differences between the core properties described
-by mmc.txt and the properties used by the omap mmc driver.
-
-Note that this driver will not work with omap2430 or later omaps,
-please see the omap hsmmc driver for the current omaps.
-
-Required properties:
-- compatible: Must be "ti,omap2420-mmc", for OMAP2420 controllers
-- ti,hwmods: For 2420, must be "msdi<n>", where n is controller
-  instance starting 1
-
-Examples:
-
-	msdi1: mmc@4809c000 {
-		compatible = "ti,omap2420-mmc";
-		ti,hwmods = "msdi1";
-		reg = <0x4809c000 0x80>;
-		interrupts = <83>;
-		dmas = <&sdma 61 &sdma 62>;
-		dma-names = "tx", "rx";
-	};
diff --git a/Documentation/devicetree/bindings/net/altr,socfpga-stmmac.yaml b/Documentation/devicetree/bindings/net/altr,socfpga-stmmac.yaml
index fc445ad5a1f1a..63084f7623738 100644
--- a/Documentation/devicetree/bindings/net/altr,socfpga-stmmac.yaml
+++ b/Documentation/devicetree/bindings/net/altr,socfpga-stmmac.yaml
@@ -63,11 +63,29 @@ properties:
       - const: ptp_ref
 
   interrupts:
-    maxItems: 1
+    minItems: 1
+    maxItems: 17
 
   interrupt-names:
+    minItems: 1
     items:
       - const: macirq
+      - const: tx-queue-0
+      - const: tx-queue-1
+      - const: tx-queue-2
+      - const: tx-queue-3
+      - const: tx-queue-4
+      - const: tx-queue-5
+      - const: tx-queue-6
+      - const: tx-queue-7
+      - const: rx-queue-0
+      - const: rx-queue-1
+      - const: rx-queue-2
+      - const: rx-queue-3
+      - const: rx-queue-4
+      - const: rx-queue-5
+      - const: rx-queue-6
+      - const: rx-queue-7
 
   iommus:
     minItems: 1
@@ -149,10 +167,36 @@ required:
   - clocks
   - clock-names
   - altr,sysmgr-syscon
+  - interrupts
+  - interrupt-names
 
 allOf:
   - $ref: snps,dwmac.yaml#
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: altr,socfpga-stmmac-agilex5
+    then:
+      properties:
+        interrupts:
+          minItems: 17
+
+        interrupt-names:
+          minItems: 17
+
+        dma-coherent: true
+    else:
+      properties:
+        interrupts:
+          maxItems: 1
+
+        interrupt-names:
+          maxItems: 1
+
+        dma-coherent: false
+
 unevaluatedProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/nvmem/airoha,smc-efuses.yaml b/Documentation/devicetree/bindings/nvmem/airoha,smc-efuses.yaml
new file mode 100644
index 0000000000000..c52f8d4bec392
--- /dev/null
+++ b/Documentation/devicetree/bindings/nvmem/airoha,smc-efuses.yaml
@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/nvmem/airoha,smc-efuses.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Airoha SMC eFuses
+
+description: |
+  Airoha new SoC AN7581 expose banks of eFuse accessible
+  via specific SMC commands.
+
+  2 different bank of eFuse or 64 cells of 32 bit are exposed
+  read-only used to give information on HW Revision, PHY Calibration,
+  Device Model, Private Key...
+
+maintainers:
+  - Christian Marangi <ansuelsmth@gmail.com>
+
+properties:
+  compatible:
+    enum:
+      - airoha,an7581-efuses
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 0
+
+patternProperties:
+  '^efuse-bank@[0-1]$':
+    type: object
+
+    allOf:
+      - $ref: nvmem.yaml#
+
+    properties:
+      reg:
+        description: Identify the eFuse bank.
+        enum: [0, 1]
+
+    required:
+      - reg
+
+    unevaluatedProperties: false
+
+required:
+  - compatible
+  - '#address-cells'
+  - '#size-cells'
+
+additionalProperties: false
+
+examples:
+  - |
+    efuse {
+        compatible = "airoha,an7581-efuses";
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        efuse-bank@0 {
+           reg = <0>;
+        };
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/nvmem/microchip,lan9662-otpc.yaml b/Documentation/devicetree/bindings/nvmem/microchip,lan9662-otpc.yaml
index f97c6beb4766a..c03e96afe5641 100644
--- a/Documentation/devicetree/bindings/nvmem/microchip,lan9662-otpc.yaml
+++ b/Documentation/devicetree/bindings/nvmem/microchip,lan9662-otpc.yaml
@@ -25,6 +25,7 @@ properties:
           - const: microchip,lan9662-otpc
       - enum:
           - microchip,lan9662-otpc
+          - microchip,lan9691-otpc
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml b/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml
index 2ab047f2bb69d..bc0a0898b9e47 100644
--- a/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml
+++ b/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml
@@ -19,14 +19,19 @@ properties:
       - enum:
           - qcom,apq8064-qfprom
           - qcom,apq8084-qfprom
+          - qcom,glymur-qfprom
+          - qcom,hawi-qfprom
           - qcom,ipq5018-qfprom
+          - qcom,ipq5210-qfprom
           - qcom,ipq5332-qfprom
           - qcom,ipq5424-qfprom
           - qcom,ipq6018-qfprom
           - qcom,ipq8064-qfprom
           - qcom,ipq8074-qfprom
           - qcom,ipq9574-qfprom
+          - qcom,ipq9650-qfprom
           - qcom,kaanapali-qfprom
+          - qcom,milos-qfprom
           - qcom,msm8226-qfprom
           - qcom,msm8916-qfprom
           - qcom,msm8917-qfprom
@@ -48,6 +53,7 @@ properties:
           - qcom,sdm630-qfprom
           - qcom,sdm670-qfprom
           - qcom,sdm845-qfprom
+          - qcom,shikra-qfprom
           - qcom,sm6115-qfprom
           - qcom,sm6350-qfprom
           - qcom,sm6375-qfprom
diff --git a/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml b/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml
index e8b8131f5f23b..4f56e8e4f1008 100644
--- a/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml
+++ b/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml
@@ -58,12 +58,18 @@ properties:
     items:
       - description: builtin MSI controller.
       - description: builtin DMA controller.
+      - description: PCIe event interrupt.
+      - description: builtin AER SPI standalone interrupt line.
+      - description: builtin PME SPI standalone interrupt line.
 
   interrupt-names:
     minItems: 1
     items:
       - const: msi
       - const: dma
+      - const: intr
+      - const: aer
+      - const: pme
 
   reset-gpio:
     deprecated: true
@@ -249,6 +255,25 @@ allOf:
             - const: ref
             - const: extref  # Optional
 
+  - if:
+      properties:
+        compatible:
+          enum:
+            - fsl,imx6q-pcie
+            - fsl,imx6sx-pcie
+            - fsl,imx6qp-pcie
+            - fsl,imx7d-pcie
+            - fsl,imx8mm-pcie
+            - fsl,imx8mp-pcie
+            - fsl,imx8mq-pcie
+            - fsl,imx8q-pcie
+    then:
+      properties:
+        interrupts:
+          maxItems: 2
+        interrupt-names:
+          maxItems: 2
+
 unevaluatedProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/pci/mediatek-pcie-gen3.yaml b/Documentation/devicetree/bindings/pci/mediatek-pcie-gen3.yaml
index 4db700fc36ba7..fbc0378392d4a 100644
--- a/Documentation/devicetree/bindings/pci/mediatek-pcie-gen3.yaml
+++ b/Documentation/devicetree/bindings/pci/mediatek-pcie-gen3.yaml
@@ -51,6 +51,7 @@ properties:
               - mediatek,mt7981-pcie
               - mediatek,mt7986-pcie
               - mediatek,mt8188-pcie
+              - mediatek,mt8189-pcie
               - mediatek,mt8195-pcie
           - const: mediatek,mt8192-pcie
       - items:
@@ -193,6 +194,7 @@ allOf:
           contains:
             enum:
               - mediatek,mt8188-pcie
+              - mediatek,mt8189-pcie
               - mediatek,mt8195-pcie
     then:
       properties:
diff --git a/Documentation/devicetree/bindings/pci/qcom,hawi-pcie.yaml b/Documentation/devicetree/bindings/pci/qcom,hawi-pcie.yaml
new file mode 100644
index 0000000000000..aba89b5991552
--- /dev/null
+++ b/Documentation/devicetree/bindings/pci/qcom,hawi-pcie.yaml
@@ -0,0 +1,201 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pci/qcom,hawi-pcie.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Hawi PCI Express Root Complex
+
+maintainers:
+  - Bjorn Andersson <andersson@kernel.org>
+  - Manivannan Sadhasivam <mani@kernel.org>
+
+description:
+  Qualcomm Hawi SoC (and compatible) PCIe root complex controller is based on
+  the Synopsys DesignWare PCIe IP.
+
+properties:
+  compatible:
+    oneOf:
+      - const: qcom,hawi-pcie
+      - items:
+          - enum:
+              - qcom,maili-pcie
+          - const: qcom,hawi-pcie
+
+  reg:
+    items:
+      - description: Qualcomm specific registers
+      - description: DesignWare PCIe registers
+      - description: External local bus interface registers
+      - description: ATU address space
+      - description: PCIe configuration space
+      - description: MHI registers
+
+  reg-names:
+    items:
+      - const: parf
+      - const: dbi
+      - const: elbi
+      - const: atu
+      - const: config
+      - const: mhi
+
+  clocks:
+    items:
+      - description: PCIe Auxiliary clock
+      - description: PCIe Configuration clock
+      - description: PCIe Master AXI clock
+      - description: PCIe Slave AXI clock
+      - description: PCIe Slave Q2A AXI clock
+      - description: PCIe Aggre NoC AXI clock
+      - description: PCIe Config NoC AXI clock
+
+  clock-names:
+    items:
+      - const: aux
+      - const: cfg
+      - const: bus_master
+      - const: bus_slave
+      - const: slave_q2a
+      - const: noc_aggr
+      - const: cnoc_sf_axi
+
+  interrupts:
+    minItems: 9
+    maxItems: 9
+
+  interrupt-names:
+    items:
+      - const: msi0
+      - const: msi1
+      - const: msi2
+      - const: msi3
+      - const: msi4
+      - const: msi5
+      - const: msi6
+      - const: msi7
+      - const: global
+
+  resets:
+    items:
+      - description: PCIe core reset
+      - description: PCIe link down reset
+
+  reset-names:
+    items:
+      - const: pci
+      - const: link_down
+
+required:
+  - power-domains
+  - resets
+  - reset-names
+
+allOf:
+  - $ref: qcom,pcie-common.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/interconnect/qcom,icc.h>
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    soc {
+        #address-cells = <2>;
+        #size-cells = <2>;
+
+        pcie@1c00000 {
+            compatible = "qcom,hawi-pcie";
+            reg = <0 0x01c00000 0 0x3000>,
+                  <0 0x40000000 0 0xf1d>,
+                  <0 0x40000f20 0 0xa8>,
+                  <0 0x40001000 0 0x1000>,
+                  <0 0x40100000 0 0x100000>,
+                  <0 0x01c03000 0 0x1000>;
+            reg-names = "parf", "dbi", "elbi", "atu", "config", "mhi";
+            ranges = <0x01000000 0x0 0x00000000 0x0 0x40200000 0x0 0x100000>,
+                     <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x3d00000>;
+
+            bus-range = <0x00 0xff>;
+            device_type = "pci";
+            linux,pci-domain = <0>;
+            num-lanes = <2>;
+
+            #address-cells = <3>;
+            #size-cells = <2>;
+
+            clocks = <&gcc_pcie_0_aux_clk>,
+                     <&gcc_pcie_0_cfg_ahb_clk>,
+                     <&gcc_pcie_0_mstr_axi_clk>,
+                     <&gcc_pcie_0_slv_axi_clk>,
+                     <&gcc_pcie_0_slv_q2a_axi_clk>,
+                     <&gcc_aggre_noc_pcie_axi_clk>,
+                     <&gcc_cnoc_pcie_sf_axi_clk>;
+            clock-names = "aux",
+                          "cfg",
+                          "bus_master",
+                          "bus_slave",
+                          "slave_q2a",
+                          "noc_aggr",
+                          "cnoc_sf_axi";
+
+            dma-coherent;
+
+            interrupts = <GIC_ESPI 205 IRQ_TYPE_LEVEL_HIGH>,
+                         <GIC_ESPI 206 IRQ_TYPE_LEVEL_HIGH>,
+                         <GIC_ESPI 207 IRQ_TYPE_LEVEL_HIGH>,
+                         <GIC_ESPI 208 IRQ_TYPE_LEVEL_HIGH>,
+                         <GIC_ESPI 209 IRQ_TYPE_LEVEL_HIGH>,
+                         <GIC_ESPI 210 IRQ_TYPE_LEVEL_HIGH>,
+                         <GIC_ESPI 211 IRQ_TYPE_LEVEL_HIGH>,
+                         <GIC_ESPI 212 IRQ_TYPE_LEVEL_HIGH>,
+                         <GIC_ESPI 204 IRQ_TYPE_LEVEL_HIGH>;
+            interrupt-names = "msi0", "msi1", "msi2", "msi3",
+                              "msi4", "msi5", "msi6", "msi7", "global";
+            #interrupt-cells = <1>;
+            interrupt-map-mask = <0 0 0 0x7>;
+            interrupt-map = <0 0 0 1 &intc 0 0 GIC_ESPI 213 IRQ_TYPE_LEVEL_HIGH>, /* int_a */
+                            <0 0 0 2 &intc 0 0 GIC_ESPI 214 IRQ_TYPE_LEVEL_HIGH>, /* int_b */
+                            <0 0 0 3 &intc 0 0 GIC_ESPI 215 IRQ_TYPE_LEVEL_HIGH>, /* int_c */
+                            <0 0 0 4 &intc 0 0 GIC_ESPI 216 IRQ_TYPE_LEVEL_HIGH>; /* int_d */
+
+            interconnects = <&pcie_anoc_master_pcie_0 QCOM_ICC_TAG_ALWAYS
+                             &mc_virt_slave_ebi1 QCOM_ICC_TAG_ALWAYS>,
+                            <&gem_noc_master_appss_proc QCOM_ICC_TAG_ACTIVE_ONLY
+                             &cnoc_main_slave_pcie_0 QCOM_ICC_TAG_ACTIVE_ONLY>;
+            interconnect-names = "pcie-mem", "cpu-pcie";
+
+            iommu-map = <0x0 &apps_smmu 0x1000 0x1>,
+                        <0x100 &apps_smmu 0x1001 0x1>;
+
+            pinctrl-0 = <&pcie0_default_state>;
+            pinctrl-names = "default";
+
+            power-domains = <&gcc_pcie_0_phy_gdsc>;
+
+            resets = <&gcc_pcie_0_bcr>,
+                     <&gcc_pcie_0_link_down_bcr>;
+            reset-names = "pci", "link_down";
+
+            msi-map = <0x0 &gic_its 0x1000 0x1>,
+                      <0x100 &gic_its 0x1001 0x1>;
+            msi-map-mask = <0xff00>;
+
+            pcie@0 {
+                device_type = "pci";
+                reg = <0x0 0x0 0x0 0x0 0x0>;
+                bus-range = <0x01 0xff>;
+
+                #address-cells = <3>;
+                #size-cells = <2>;
+                ranges;
+
+                phys = <&pcie0_phy>;
+                wake-gpios = <&tlmm 104 GPIO_ACTIVE_HIGH>;
+                reset-gpios = <&tlmm 102 GPIO_ACTIVE_LOW>;
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-ipq9574.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-ipq9574.yaml
index 4be342cc04e12..5330798118811 100644
--- a/Documentation/devicetree/bindings/pci/qcom,pcie-ipq9574.yaml
+++ b/Documentation/devicetree/bindings/pci/qcom,pcie-ipq9574.yaml
@@ -17,8 +17,10 @@ properties:
           - qcom,pcie-ipq9574
       - items:
           - enum:
+              - qcom,pcie-ipq5210
               - qcom,pcie-ipq5332
               - qcom,pcie-ipq5424
+              - qcom,pcie-ipq9650
           - const: qcom,pcie-ipq9574
 
   reg:
@@ -82,6 +84,19 @@ required:
 
 allOf:
   - $ref: qcom,pcie-common.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,pcie-ipq5210
+              - qcom,pcie-ipq9650
+    then:
+      properties:
+        interrupts:
+          minItems: 9
+        interrupt-names:
+          minItems: 9
 
 unevaluatedProperties: false
 
diff --git a/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml b/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml
index 90086909e9210..09d78bd1a5770 100644
--- a/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml
+++ b/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml
@@ -14,7 +14,7 @@ description: |
   with PCIe Base Specification 4.0 and supports different link speeds
   depending on the SoC variant:
     - Gen2 (5 GT/s): RZ/G3S
-    - Gen3 (8 GT/s): RZ/G3E, RZ/V2N
+    - Gen3 (8 GT/s): RZ/G3E, RZ/V2H(P), RZ/V2N
 
 properties:
   compatible:
@@ -22,6 +22,7 @@ properties:
       - enum:
           - renesas,r9a08g045-pcie # RZ/G3S
           - renesas,r9a09g047-pcie # RZ/G3E
+          - renesas,r9a09g057-pcie # RZ/V2H(P)
       - items:
           - const: renesas,r9a09g056-pcie # RZ/V2N
           - const: renesas,r9a09g047-pcie
@@ -139,7 +140,13 @@ properties:
       - clkl1pm clock request state
       - power off information in L2 state
       - errors (fatal, non-fatal, correctable)
-    $ref: /schemas/types.yaml#/definitions/phandle
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+    items:
+      - items:
+          - description: Phandle to system controller
+          - description: PCIe controller index
+            enum: [0, 1]
+        minItems: 1
 
 patternProperties:
   "^pcie@0,[0-0]$":
@@ -220,7 +227,9 @@ allOf:
       properties:
         compatible:
           contains:
-            const: renesas,r9a09g047-pcie
+            enum:
+              - renesas,r9a09g047-pcie
+              - renesas,r9a09g057-pcie
     then:
       properties:
         interrupts:
@@ -235,6 +244,25 @@ allOf:
           maxItems: 1
         reset-names:
           maxItems: 1
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: renesas,r9a09g057-pcie
+    then:
+      properties:
+        num-lanes:
+          enum: [2, 4]
+        renesas,sysc:
+          items:
+            - minItems: 2
+      required:
+        - num-lanes
+    else:
+      properties:
+        renesas,sysc:
+          items:
+            - maxItems: 1
 
 unevaluatedProperties: false
 
diff --git a/Documentation/devicetree/bindings/phy/samsung,mipi-video-phy.yaml b/Documentation/devicetree/bindings/phy/samsung,mipi-video-phy.yaml
index 16967ef8e9ecc..87b6a35b2626a 100644
--- a/Documentation/devicetree/bindings/phy/samsung,mipi-video-phy.yaml
+++ b/Documentation/devicetree/bindings/phy/samsung,mipi-video-phy.yaml
@@ -13,14 +13,14 @@ maintainers:
 
 description: |
   For samsung,s5pv210-mipi-video-phy compatible PHYs the second cell in the
-  PHY specifier identifies the PHY and its meaning is as follows::
+  PHY specifier identifies the PHY and its meaning is as follows:
     0 - MIPI CSIS 0,
     1 - MIPI DSIM 0,
     2 - MIPI CSIS 1,
     3 - MIPI DSIM 1.
 
   samsung,exynos5420-mipi-video-phy and samsung,exynos5433-mipi-video-phy
-  support additional fifth PHY::
+  support additional fifth PHY:
     4 - MIPI CSIS 2.
 
 properties:
diff --git a/Documentation/devicetree/bindings/phy/samsung,usb2-phy.yaml b/Documentation/devicetree/bindings/phy/samsung,usb2-phy.yaml
index d9f22a801cbf7..7db7605a82e21 100644
--- a/Documentation/devicetree/bindings/phy/samsung,usb2-phy.yaml
+++ b/Documentation/devicetree/bindings/phy/samsung,usb2-phy.yaml
@@ -14,7 +14,7 @@ maintainers:
 description: |
   The first phandle argument in the PHY specifier identifies the PHY, its
   meaning is compatible dependent. For the currently supported SoCs (Exynos4210
-  and Exynos4212) it is as follows::
+  and Exynos4212) it is as follows:
     0 - USB device ("device"),
     1 - USB host ("host"),
     2 - HSIC0 ("hsic0"),
diff --git a/Documentation/devicetree/bindings/phy/samsung,usb3-drd-phy.yaml b/Documentation/devicetree/bindings/phy/samsung,usb3-drd-phy.yaml
index 4562e0468f4f4..a1b3d9e6a094f 100644
--- a/Documentation/devicetree/bindings/phy/samsung,usb3-drd-phy.yaml
+++ b/Documentation/devicetree/bindings/phy/samsung,usb3-drd-phy.yaml
@@ -14,7 +14,7 @@ maintainers:
 description: |
   For samsung,exynos5250-usbdrd-phy and samsung,exynos5420-usbdrd-phy
   compatible PHYs, the second cell in the PHY specifier identifies the
-  PHY id, which is interpreted as follows::
+  PHY id, which is interpreted as follows:
     0 - UTMI+ type phy,
     1 - PIPE3 type phy.
 
diff --git a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml
index 41073176bc691..f08d2c4f57841 100644
--- a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml
@@ -20,6 +20,8 @@ properties:
       - items:
           - enum:
               - apple,t6020-pinctrl
+              - apple,t6030-pinctrl
+              - apple,t6031-pinctrl
               - apple,t8122-pinctrl
           - const: apple,t8103-pinctrl
       - items:
diff --git a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2700-soc1-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2700-soc1-pinctrl.yaml
index 76944fd14e2c7..fe7cef4fef6ae 100644
--- a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2700-soc1-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2700-soc1-pinctrl.yaml
@@ -356,6 +356,7 @@ patternProperties:
           - I3C8
           - I3C9
           - JTAGM1
+          - JTAGM1TRST
           - LPC0
           - LPC1
           - LTPI
diff --git a/Documentation/devicetree/bindings/pinctrl/nxp,s32g2-siul2-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/nxp,s32g2-siul2-pinctrl.yaml
index a24286e4def62..36f2393fa4065 100644
--- a/Documentation/devicetree/bindings/pinctrl/nxp,s32g2-siul2-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/nxp,s32g2-siul2-pinctrl.yaml
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-# Copyright 2022 NXP
+# Copyright 2022, 2026 NXP
 %YAML 1.2
 ---
 $id: http://devicetree.org/schemas/pinctrl/nxp,s32g2-siul2-pinctrl.yaml#
@@ -17,8 +17,10 @@ description: |
     SIUL2_0 @ 0x4009c000
     SIUL2_1 @ 0x44010000
 
-  Every SIUL2 region has multiple register types, and here only MSCR and
-  IMCR registers need to be revealed for kernel to configure pinmux.
+  Every SIUL2 region has multiple register types. MSCR and IMCR registers
+  need to be revealed for kernel to configure pinmux. PGPDO and PGPDI
+  registers are used for GPIO output/input operations. EIRQ registers
+  are used for external interrupt configuration.
 
   Please note that some register indexes are reserved in S32G2, such as
   MSCR102-MSCR111, MSCR123-MSCR143, IMCR84-IMCR118 and IMCR398-IMCR429.
@@ -29,14 +31,22 @@ properties:
       - nxp,s32g2-siul2-pinctrl
 
   reg:
+    minItems: 6
     description: |
-      A list of MSCR/IMCR register regions to be reserved.
+      A list of MSCR/IMCR/PGPDO/PGPDI/EIRQ register regions to be reserved.
       - MSCR (Multiplexed Signal Configuration Register)
         An MSCR register can configure the associated pin as either a GPIO pin
         or a function output pin depends on the selected signal source.
       - IMCR (Input Multiplexed Signal Configuration Register)
         An IMCR register can configure the associated pin as function input
         pin depends on the selected signal source.
+      - PGPDO (Parallel GPIO Pad Data Out Register)
+        A PGPDO register is used to set the output value of a GPIO pin.
+      - PGPDI (Parallel GPIO Pad Data In Register)
+        A PGPDI register is used to read the input value of a GPIO pin.
+      - EIRQ (External Interrupt Request)
+        EIRQ registers are used to configure and manage external interrupts.
+
     items:
       - description: MSCR registers group 0 in SIUL2_0
       - description: MSCR registers group 1 in SIUL2_1
@@ -44,6 +54,28 @@ properties:
       - description: IMCR registers group 0 in SIUL2_0
       - description: IMCR registers group 1 in SIUL2_1
       - description: IMCR registers group 2 in SIUL2_1
+      - description: PGPDO registers in SIUL2_0
+      - description: PGPDI registers in SIUL2_0
+      - description: PGPDO registers in SIUL2_1
+      - description: PGPDI registers in SIUL2_1
+      - description: EIRQ registers in SIUL2_1
+
+  gpio-controller: true
+
+  "#gpio-cells":
+    const: 2
+
+  gpio-ranges:
+    minItems: 1
+    maxItems: 4
+
+  interrupt-controller: true
+
+  "#interrupt-cells":
+    const: 2
+
+  interrupts:
+    maxItems: 1
 
 patternProperties:
   '-pins$':
@@ -86,11 +118,38 @@ required:
   - compatible
   - reg
 
+oneOf:
+  - description: Legacy pinctrl-only node
+    properties:
+      reg:
+        maxItems: 6
+
+      gpio-controller: false
+      "#gpio-cells": false
+      gpio-ranges: false
+      interrupt-controller: false
+      "#interrupt-cells": false
+      interrupts: false
+
+  - description: Pinctrl node with GPIO and external interrupt support
+    required:
+      - gpio-controller
+      - "#gpio-cells"
+      - gpio-ranges
+      - interrupt-controller
+      - "#interrupt-cells"
+      - interrupts
+    properties:
+      reg:
+        minItems: 11
+
 additionalProperties: false
 
 examples:
   - |
-    pinctrl@4009c240 {
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    pinctrl: pinctrl@4009c240 {
         compatible = "nxp,s32g2-siul2-pinctrl";
 
               /* MSCR0-MSCR101 registers on siul2_0 */
@@ -104,7 +163,26 @@ examples:
               /* IMCR119-IMCR397 registers on siul2_1 */
               <0x44010c1c 0x45c>,
               /* IMCR430-IMCR495 registers on siul2_1 */
-              <0x440110f8 0x108>;
+              <0x440110f8 0x108>,
+              /* PGPDO registers on siul2_0 */
+              <0x4009d700 0x10>,
+              /* PGPDI registers on siul2_0 */
+              <0x4009d740 0x10>,
+              /* PGPDO registers on siul2_1 */
+              <0x44011700 0x18>,
+              /* PGPDI registers on siul2_1 */
+              <0x44011740 0x18>,
+              /* EIRQ registers on siul2_1 */
+              <0x44010010 0x34>;
+
+        gpio-controller;
+        #gpio-cells = <2>;
+        gpio-ranges = <&pinctrl 0 0 102>,
+                      <&pinctrl 112 112 79>;
+
+        interrupt-controller;
+        #interrupt-cells = <2>;
+        interrupts = <GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH>;
 
         llce-can0-pins {
             llce-can0-grp0 {
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,maili-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,maili-tlmm.yaml
new file mode 100644
index 0000000000000..64fe90b2391bd
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,maili-tlmm.yaml
@@ -0,0 +1,120 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/qcom,maili-tlmm.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Technologies, Inc. Maili TLMM block
+
+maintainers:
+  - Jingyi Wang <jingyi.wang@oss.qualcomm.com>
+
+description:
+  Top Level Mode Multiplexer pin controller in Qualcomm Maili SoC.
+
+allOf:
+  - $ref: /schemas/pinctrl/qcom,tlmm-common.yaml#
+
+properties:
+  compatible:
+    const: qcom,maili-tlmm
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  gpio-reserved-ranges:
+    minItems: 1
+    maxItems: 113
+
+  gpio-line-names:
+    maxItems: 226
+
+patternProperties:
+  "-state$":
+    oneOf:
+      - $ref: "#/$defs/qcom-maili-tlmm-state"
+      - patternProperties:
+          "-pins$":
+            $ref: "#/$defs/qcom-maili-tlmm-state"
+        additionalProperties: false
+
+$defs:
+  qcom-maili-tlmm-state:
+    type: object
+    description:
+      Pinctrl node's client devices use subnodes for desired pin configuration.
+      Client device subnodes use below standard properties.
+    $ref: qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state
+    unevaluatedProperties: false
+
+    properties:
+      pins:
+        description:
+          List of gpio pins affected by the properties specified in this
+          subnode.
+        items:
+          oneOf:
+            - pattern: "^gpio([0-9]|[1-9][0-9]|1[0-9][0-9]|20[0-9]|21[0-9]|22[0-5])$"
+            - enum: [ ufs_reset, sdc2_clk, sdc2_cmd, sdc2_data ]
+        minItems: 1
+        maxItems: 36
+
+      function:
+        description:
+          Specify the alternative function to be configured for the specified
+          pins.
+        enum: [ gpio, aoss_cti, atest_char, atest_usb, audio_ext_mclk,
+                audio_ref_clk, cam_mclk, cci_async_in, cci_i2c0, cci_i2c1,
+                cci_i2c2, cci_i2c3, cci_timer, coex_espmi, coex_uart1_rx,
+                coex_uart1_tx, dbg_out_clk, ddr_bist, ddr_pxi, dp_hot, egpio,
+                gcc_gp, gnss_adc, host2wlan_sol, host_rst, i2chub0_se0,
+                i2chub0_se1, i2chub0_se2, i2chub0_se3, i2chub0_se4, i2s0, i2s1,
+                ibi_i3c, ibi_i3c_qup5_se0, jitter_bist, mdp_esync0, mdp_esync1,
+                mdp_esync2, mdp_vsync, mdp_vsync_e, mdp_vsync_p, mdp_vsync0_out,
+                mdp_vsync1_out, mdp_vsync2_out, mdp_vsync3_out, mdp_vsync5_out,
+                modem_pps_in, modem_pps_out, nav_gpio, nav_gpio0, nav_gpio3,
+                nav_rffe, pcie0_clk_req_n, pcie1_clk_req_n, pcie1_rst_n,
+                phase_flag, pll_bist_sync, pll_clk_aux, qdss_cti, qlink, qspi,
+                qspi_clk, qspi_cs, qup1_se0, qup1_se1, qup1_se2, qup1_se3,
+                qup1_se4, qup1_se5, qup1_se6, qup1_se7, qup2_se0, qup2_se1,
+                qup2_se2, qup2_se3, qup2_se4_01, qup2_se4_23, qup3_se0,
+                qup3_se1, qup3_se2, qup3_se3, qup3_se4, qup3_se5, qup4_se0,
+                qup4_se1, qup4_se2, qup4_se3_01, qup4_se3_23, qup4_se3_l3,
+                qup4_se4_01, qup4_se4_23, qup4_se4_l3, qup5_se0, rng_rosc,
+                sd_write_protect, sdc2_clk, sdc2_cmd, sdc2_data, sdc2_rclk,
+                sdc4_clk, sdc4_cmd, sdc4_data, sys_throttle, tb_trig_sdc,
+                tmess_rng, tsense_clm, tsense_pwm, uim0, uim1, usb0_hs, usb_phy,
+                vfr, vsense_trigger_mirnat, wcn_sw ]
+
+    required:
+      - pins
+
+required:
+  - compatible
+  - reg
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    tlmm: pinctrl@f100000 {
+        compatible = "qcom,maili-tlmm";
+        reg = <0x0f100000 0x300000>;
+        interrupts = <GIC_ESPI 272 IRQ_TYPE_LEVEL_HIGH>;
+        gpio-controller;
+        #gpio-cells = <2>;
+        gpio-ranges = <&tlmm 0 0 227>;
+        interrupt-controller;
+        #interrupt-cells = <2>;
+
+        qup-uart7-state {
+          pins = "gpio62", "gpio63";
+          function = "qup1_se7";
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,milos-lpass-lpi-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,milos-lpass-lpi-pinctrl.yaml
index 73e84f1885911..86c1da0f577c9 100644
--- a/Documentation/devicetree/bindings/pinctrl/qcom,milos-lpass-lpi-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,milos-lpass-lpi-pinctrl.yaml
@@ -15,9 +15,12 @@ description:
 
 properties:
   compatible:
-    const: qcom,milos-lpass-lpi-pinctrl
+    enum:
+      - qcom,eliza-lpass-lpi-pinctrl
+      - qcom,milos-lpass-lpi-pinctrl
 
   reg:
+    minItems: 1
     items:
       - description: LPASS LPI TLMM Control and Status registers
       - description: LPASS LPI MCC registers
@@ -74,6 +77,19 @@ $defs:
 allOf:
   - $ref: qcom,lpass-lpi-common.yaml#
 
+  - if:
+      properties:
+        compatible:
+          const: qcom,eliza-lpass-lpi-pinctrl
+    then:
+      properties:
+        reg:
+          maxItems: 1
+    else:
+      properties:
+        reg:
+          minItems: 2
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml
index b8109e6c2a109..c27e24cb206fc 100644
--- a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml
@@ -62,6 +62,7 @@ properties:
           - qcom,pmc8380-gpio
           - qcom,pmcx0102-gpio
           - qcom,pmd8028-gpio
+          - qcom,pmg1110-gpio
           - qcom,pmh0101-gpio
           - qcom,pmh0104-gpio
           - qcom,pmh0110-gpio
@@ -155,6 +156,7 @@ allOf:
               - qcom,pm8450-gpio
               - qcom,pm8916-gpio
               - qcom,pmd8028-gpio
+              - qcom,pmg1110-gpio
               - qcom,pmk8350-gpio
               - qcom,pmr735a-gpio
               - qcom,pmr735b-gpio
@@ -496,6 +498,7 @@ $defs:
                  - gpio1-gpio22 for pma8084
                  - gpio1-gpio14 for pmcx0102
                  - gpio1-gpio4 for pmd8028
+                 - gpio1-gpio4 for pmg1110
                  - gpio1-gpio18 for pmh0101
                  - gpio1-gpio8 for pmh0104
                  - gpio1-gpio14 for pmh0110
diff --git a/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl-wakeup-interrupt.yaml b/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl-wakeup-interrupt.yaml
index 2b88f25e80a6d..802911e23aff5 100644
--- a/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl-wakeup-interrupt.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl-wakeup-interrupt.yaml
@@ -41,6 +41,7 @@ properties:
               - samsung,exynos7870-wakeup-eint
               - samsung,exynos7885-wakeup-eint
               - samsung,exynos850-wakeup-eint
+              - samsung,exynos8855-wakeup-eint
               - samsung,exynos8890-wakeup-eint
               - samsung,exynos8895-wakeup-eint
           - const: samsung,exynos7-wakeup-eint
diff --git a/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml
index 7b006009ca0e7..1feb6a6fc6908 100644
--- a/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml
@@ -18,7 +18,7 @@ description: |
   All the pin controller nodes should be represented in the aliases node using
   the following format 'pinctrl{n}' where n is a unique number for the alias.
 
-  The controller supports three types of interrupts::
+  The controller supports three types of interrupts:
    - External GPIO interrupts (see interrupts property in pin controller node);
 
    - External wake-up interrupts - multiplexed (capable of waking up the system
@@ -53,6 +53,7 @@ properties:
       - samsung,exynos7870-pinctrl
       - samsung,exynos7885-pinctrl
       - samsung,exynos850-pinctrl
+      - samsung,exynos8855-pinctrl
       - samsung,exynos8890-pinctrl
       - samsung,exynos8895-pinctrl
       - samsung,exynos9610-pinctrl
diff --git a/Documentation/devicetree/bindings/power/mediatek,power-controller.yaml b/Documentation/devicetree/bindings/power/mediatek,power-controller.yaml
index 07f046277f8a6..070c6e5666dc0 100644
--- a/Documentation/devicetree/bindings/power/mediatek,power-controller.yaml
+++ b/Documentation/devicetree/bindings/power/mediatek,power-controller.yaml
@@ -34,6 +34,7 @@ properties:
       - mediatek,mt8189-power-controller
       - mediatek,mt8192-power-controller
       - mediatek,mt8195-power-controller
+      - mediatek,mt8196-hfrp-power-controller
       - mediatek,mt8196-hwv-hfrp-power-controller
       - mediatek,mt8196-hwv-scp-power-controller
       - mediatek,mt8196-power-controller
diff --git a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml
index 0744a867c79eb..0a45e1f550621 100644
--- a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml
+++ b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml
@@ -75,6 +75,10 @@ properties:
           - enum:
               - qcom,msm8937-rpmpd
           - const: qcom,msm8917-rpmpd
+      - items:
+          - enum:
+              - qcom,maili-rpmhpd
+          - const: qcom,hawi-rpmhpd
 
   '#power-domain-cells':
     const: 1
diff --git a/Documentation/devicetree/bindings/power/renesas,rcar-sysc.yaml b/Documentation/devicetree/bindings/power/renesas,rcar-sysc.yaml
index 347571e2545a8..b67aa170b2c1e 100644
--- a/Documentation/devicetree/bindings/power/renesas,rcar-sysc.yaml
+++ b/Documentation/devicetree/bindings/power/renesas,rcar-sysc.yaml
@@ -13,7 +13,7 @@ maintainers:
 description: |
   The R-Car (RZ/G) System Controller provides power management for the CPU
   cores and various coprocessors.
-  The power domain IDs for consumers are defined in header files::
+  The power domain IDs for consumers are defined in header files:
   include/dt-bindings/power/r8*-sysc.h
 
 properties:
diff --git a/Documentation/devicetree/bindings/power/reset/restart-handler.yaml b/Documentation/devicetree/bindings/power/reset/restart-handler.yaml
index 965a834a3dbe8..00c00ec5ec814 100644
--- a/Documentation/devicetree/bindings/power/reset/restart-handler.yaml
+++ b/Documentation/devicetree/bindings/power/reset/restart-handler.yaml
@@ -18,12 +18,12 @@ properties:
   priority:
     $ref: /schemas/types.yaml#/definitions/uint32
     description: |
-      A priority ranging from 0 to 255 according to the following guidelines::
-        0::   Restart handler of last resort, with limited restart capabilities.
-        128:: Typical, default restart handler; use if no other restart handler
+      A priority ranging from 0 to 255 according to the following guidelines:
+        0:   Restart handler of last resort, with limited restart capabilities.
+        128: Typical, default restart handler; use if no other restart handler
               is expected to be available, and/or if restart functionality is
               sufficient to restart the entire system.
-        255:: Highest priority restart handler, will preempt all other restart handlers.
+        255: Highest priority restart handler, will preempt all other restart handlers.
     minimum: 0
     maximum: 255
 
diff --git a/Documentation/devicetree/bindings/pwm/nvidia,tegra20-pwm.yaml b/Documentation/devicetree/bindings/pwm/nvidia,tegra20-pwm.yaml
index 41cea4979132f..cb2f36e7b5d67 100644
--- a/Documentation/devicetree/bindings/pwm/nvidia,tegra20-pwm.yaml
+++ b/Documentation/devicetree/bindings/pwm/nvidia,tegra20-pwm.yaml
@@ -16,6 +16,7 @@ properties:
       - enum:
           - nvidia,tegra20-pwm
           - nvidia,tegra186-pwm
+          - nvidia,tegra264-pwm
 
       - items:
           - enum:
diff --git a/Documentation/devicetree/bindings/regulator/maxim,max77802.yaml b/Documentation/devicetree/bindings/regulator/maxim,max77802.yaml
index b704f05ea4542..b886495c1396e 100644
--- a/Documentation/devicetree/bindings/regulator/maxim,max77802.yaml
+++ b/Documentation/devicetree/bindings/regulator/maxim,max77802.yaml
@@ -22,13 +22,13 @@ description: |
 
   Certain regulators support "regulator-initial-mode" and "regulator-mode".
   The valid modes list is defined in the dt-bindings/regulator/maxim,max77802.h
-  and their meaning is::
+  and their meaning is:
     1 - Normal regulator voltage output mode.
     3 - Low Power which reduces the quiescent current down to only 1uA
 
   The standard "regulator-mode" property can only be used for regulators that
   support changing their mode to Low Power Mode during suspend. These
-  regulators are:: bucks 2-4 and LDOs 1-35. Also, it only takes effect if the
+  regulators are: bucks 2-4 and LDOs 1-35. Also, it only takes effect if the
   regulator has been enabled for the given suspend state using
   "regulator-on-in-suspend" and has not been disabled for that state using
   "regulator-off-in-suspend".
diff --git a/Documentation/devicetree/bindings/regulator/qcom,sdm845-refgen-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,sdm845-refgen-regulator.yaml
index 40f9223d4c272..0bbf7c806fbc4 100644
--- a/Documentation/devicetree/bindings/regulator/qcom,sdm845-refgen-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/qcom,sdm845-refgen-regulator.yaml
@@ -13,9 +13,6 @@ description:
   The REFGEN (reference voltage generator) regulator provides reference
   voltage for on-chip IPs (like PHYs) on some Qualcomm SoCs.
 
-allOf:
-  - $ref: regulator.yaml#
-
 properties:
   compatible:
     oneOf:
@@ -39,16 +36,44 @@ properties:
           - const: qcom,sm8250-refgen-regulator
 
       - enum:
+          - qcom,ipq9650-refgen-regulator
           - qcom,sdm845-refgen-regulator
           - qcom,sm8250-refgen-regulator
 
   reg:
     maxItems: 1
 
+  clocks:
+    items:
+      - description: Core reference clock
+      - description: AHB interface clock
+
+  clock-names:
+    items:
+      - const: core
+      - const: hclk
+
 required:
   - compatible
   - reg
 
+allOf:
+  - $ref: regulator.yaml#
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: qcom,ipq9650-refgen-regulator
+    then:
+      required:
+        - clocks
+        - clock-names
+    else:
+      properties:
+        clocks: false
+        clock-names: false
+
 unevaluatedProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml
index fcefc722ee2a4..024b34d0eb1a6 100644
--- a/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/qcom,usb-vbus-regulator.yaml
@@ -14,17 +14,21 @@ description: |
   regulator will be enabled in situations where the device is required to
   provide power to the connected peripheral.
 
-allOf:
-  - $ref: regulator.yaml#
+  The pm8150b variant uses an OTG current-limit selector, supporting limits
+  of 500 mA, 1000 mA, 1500 mA, 2000 mA, 2500 mA and 3000 mA.
+
+  The pm4125 variant uses a different register layout with a 2-bit VBOOST
+  voltage selector supporting output voltages of 4.25 V, 4.5 V, 4.75 V
+  and 5.0 V.
 
 properties:
   compatible:
     oneOf:
       - enum:
           - qcom,pm8150b-vbus-reg
+          - qcom,pm4125-vbus-reg
       - items:
           - enum:
-              - qcom,pm4125-vbus-reg
               - qcom,pm6150-vbus-reg
               - qcom,pm7250b-vbus-reg
               - qcom,pmi632-vbus-reg
@@ -34,11 +38,35 @@ properties:
     maxItems: 1
     description: VBUS output base address
 
+allOf:
+  - $ref: regulator.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,pm8150b-vbus-reg
+              - qcom,pm6150-vbus-reg
+              - qcom,pm7250b-vbus-reg
+              - qcom,pmi632-vbus-reg
+    then:
+      required:
+        - regulator-min-microamp
+        - regulator-max-microamp
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: qcom,pm4125-vbus-reg
+    then:
+      required:
+        - regulator-min-microvolt
+        - regulator-max-microvolt
+
 required:
   - compatible
   - reg
-  - regulator-min-microamp
-  - regulator-max-microamp
 
 unevaluatedProperties: false
 
@@ -55,4 +83,16 @@ examples:
             regulator-max-microamp = <3000000>;
         };
      };
+  - |
+     pmic {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        usb-vbus-regulator@1100 {
+            compatible = "qcom,pm4125-vbus-reg";
+            reg = <0x1100>;
+            regulator-min-microvolt = <4250000>;
+            regulator-max-microvolt = <5000000>;
+        };
+     };
 ...
diff --git a/Documentation/devicetree/bindings/regulator/richtek,rtq2208.yaml b/Documentation/devicetree/bindings/regulator/richtek,rtq2208.yaml
index 022c1f1973646..4c3236be87094 100644
--- a/Documentation/devicetree/bindings/regulator/richtek,rtq2208.yaml
+++ b/Documentation/devicetree/bindings/regulator/richtek,rtq2208.yaml
@@ -21,7 +21,7 @@ description: |
   conduction mode (FCCM).
 
   The definition of modes is in the datasheet which is available in below link
-  and their meaning is::
+  and their meaning is:
     0 - Auto mode for power saving, which reducing the switching frequency at light load condition
     to maintain high frequency.
     1 - FCCM to meet the strict voltage regulation accuracy, which keeping constant switching frequency.
@@ -42,11 +42,14 @@ properties:
 
   richtek,mtp-sel-high:
     type: boolean
+    deprecated: true
     description:
       vout register selection based on this boolean value.
       false - Using DVS0 register setting to adjust vout
       true - Using DVS1 register setting to adjust vout
 
+      The property is now deprecated. Will be identified by RG HW register.
+
   regulators:
     type: object
     additionalProperties: false
@@ -100,7 +103,6 @@ examples:
         compatible = "richtek,rtq2208";
         reg = <0x10>;
         interrupts-extended = <&gpio26 0 IRQ_TYPE_LEVEL_LOW>;
-        richtek,mtp-sel-high;
 
         regulators {
           buck-a {
diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd71815-regulator.yaml b/Documentation/devicetree/bindings/regulator/rohm,bd71815-regulator.yaml
index cc4ceb32e9d69..68b5e579e2d3e 100644
--- a/Documentation/devicetree/bindings/regulator/rohm,bd71815-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/rohm,bd71815-regulator.yaml
@@ -38,7 +38,9 @@ patternProperties:
     type: object
     description:
       Properties for single LDO/BUCK regulator.
-    $ref: regulator.yaml#
+    allOf:
+      - $ref: regulator.yaml#
+      - $ref: rohm,pmic-states.yaml#
 
     properties:
       regulator-name:
@@ -51,15 +53,6 @@ patternProperties:
         description:
           GPIO used to control ldo4 state (when ldo4 is controlled by GPIO).
 
-      rohm,dvs-run-voltage:
-        description:
-          PMIC "RUN" state voltage in uV when PMIC HW states are used. See
-          comments below for bucks/LDOs which support this. 0 means
-          regulator should be disabled at RUN state.
-        $ref: /schemas/types.yaml#/definitions/uint32
-        minimum: 0
-        maximum: 3300000
-
       rohm,dvs-snvs-voltage:
         description:
           Whether to keep regulator enabled at "SNVS" state or not.
@@ -71,21 +64,15 @@ patternProperties:
         minimum: 0
         maximum: 3300000
 
+      rohm,dvs-run-voltage:
+        minimum: 0
+        maximum: 3300000
+
       rohm,dvs-suspend-voltage:
-        description:
-          PMIC "SUSPEND" state voltage in uV when PMIC HW states are used. See
-          comments below for bucks/LDOs which support this. 0 means
-          regulator should be disabled at SUSPEND state.
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 3300000
 
       rohm,dvs-lpsr-voltage:
-        description:
-          PMIC "LPSR" state voltage in uV when PMIC HW states are used. See
-          comments below for bucks/LDOs which support this. 0 means
-          regulator should be disabled at LPSR state.
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 3300000
 
diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd71828-regulator.yaml b/Documentation/devicetree/bindings/regulator/rohm,bd71828-regulator.yaml
index d898800d6bcae..8cba397678a6f 100644
--- a/Documentation/devicetree/bindings/regulator/rohm,bd71828-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/rohm,bd71828-regulator.yaml
@@ -40,7 +40,9 @@ patternProperties:
     type: object
     description:
       Properties for single BUCK regulator.
-    $ref: regulator.yaml#
+    allOf:
+      - $ref: regulator.yaml#
+      - $ref: rohm,pmic-states.yaml#
 
     properties:
       regulator-name:
@@ -49,34 +51,18 @@ patternProperties:
           should be "buck1", ..., "buck7"
 
       rohm,dvs-run-voltage:
-        description:
-          PMIC default "RUN" state voltage in uV. See below table for
-          bucks which support this. 0 means disabled.
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 3300000
 
       rohm,dvs-idle-voltage:
-        description:
-          PMIC default "IDLE" state voltage in uV. See below table for
-          bucks which support this. 0 means disabled.
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 3300000
 
       rohm,dvs-suspend-voltage:
-        description:
-          PMIC default "SUSPEND" state voltage in uV. See below table for
-          bucks which support this. 0 means disabled.
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 3300000
 
       rohm,dvs-lpsr-voltage:
-        description:
-          PMIC default "LPSR" state voltage in uV. See below table for
-          bucks which support this. 0 means disabled.
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 3300000
 
diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.yaml b/Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.yaml
index 29b350a4f88a0..efb38dd15145e 100644
--- a/Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/rohm,bd71837-regulator.yaml
@@ -44,7 +44,9 @@ patternProperties:
 
   "^BUCK[1-8]$":
     type: object
-    $ref: regulator.yaml#
+    allOf:
+      - $ref: regulator.yaml#
+      - $ref: rohm,pmic-states.yaml#
     description:
       Properties for single BUCK regulator.
 
@@ -55,28 +57,16 @@ patternProperties:
           should be "buck1", ..., "buck8"
 
       rohm,dvs-run-voltage:
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 1300000
-        description:
-          PMIC default "RUN" state voltage in uV. See below table for
-          bucks which support this. 0 means disabled.
 
       rohm,dvs-idle-voltage:
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 1300000
-        description:
-          PMIC default "IDLE" state voltage in uV. See below table for
-          bucks which support this. 0 means disabled.
 
       rohm,dvs-suspend-voltage:
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 1300000
-        description:
-          PMIC default "SUSPEND" state voltage in uV. See below table for
-          bucks which support this. 0 means disabled.
 
         # Supported default DVS states:
         #
diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd71847-regulator.yaml b/Documentation/devicetree/bindings/regulator/rohm,bd71847-regulator.yaml
index 7ba4ccf723d82..ef6f69be24ae3 100644
--- a/Documentation/devicetree/bindings/regulator/rohm,bd71847-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/rohm,bd71847-regulator.yaml
@@ -43,7 +43,9 @@ patternProperties:
 
   "^BUCK[1-6]$":
     type: object
-    $ref: regulator.yaml#
+    allOf:
+      - $ref: regulator.yaml#
+      - $ref: rohm,pmic-states.yaml#
     description:
       Properties for single BUCK regulator.
 
@@ -54,28 +56,16 @@ patternProperties:
           should be "buck1", ..., "buck6"
 
       rohm,dvs-run-voltage:
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 1300000
-        description:
-          PMIC default "RUN" state voltage in uV. See below table for
-          bucks which support this. 0 means disabled.
 
       rohm,dvs-idle-voltage:
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 1300000
-        description:
-          PMIC default "IDLE" state voltage in uV. See below table for
-          bucks which support this. 0 means disabled.
 
       rohm,dvs-suspend-voltage:
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 1300000
-        description:
-          PMIC default "SUSPEND" state voltage in uV. See below table for
-          bucks which support this. 0 means disabled.
 
         # Supported default DVS states:
         #
diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd72720-regulator.yaml b/Documentation/devicetree/bindings/regulator/rohm,bd72720-regulator.yaml
index 5518082129bdb..76747bf2bbdd5 100644
--- a/Documentation/devicetree/bindings/regulator/rohm,bd72720-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/rohm,bd72720-regulator.yaml
@@ -26,41 +26,27 @@ patternProperties:
     type: object
     description:
       Properties for single LDO regulator.
-    $ref: regulator.yaml#
+    allOf:
+      - $ref: regulator.yaml#
+      - $ref: rohm,pmic-states.yaml#
 
     properties:
       regulator-name:
         pattern: "^ldo([1-9]|1[0-1])$"
 
       rohm,dvs-run-voltage:
-        description:
-          PMIC default "RUN" state voltage in uV. See below table for
-          LDOs which support this. 0 means disabled.
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 3300000
 
       rohm,dvs-idle-voltage:
-        description:
-          PMIC default "IDLE" state voltage in uV. See below table for
-          LDOs which support this. 0 means disabled.
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 3300000
 
       rohm,dvs-suspend-voltage:
-        description:
-          PMIC default "SUSPEND" state voltage in uV. See below table for
-          LDOs which support this. 0 means disabled.
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 3300000
 
       rohm,dvs-lpsr-voltage:
-        description:
-          PMIC default "deep-idle" state voltage in uV. See below table for
-          LDOs which support this. 0 means disabled.
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 3300000
 
@@ -82,7 +68,9 @@ patternProperties:
     type: object
     description:
       Properties for single BUCK regulator.
-    $ref: regulator.yaml#
+    allOf:
+      - $ref: regulator.yaml#
+      - $ref: rohm,pmic-states.yaml#
 
     properties:
       regulator-name:
@@ -97,34 +85,18 @@ patternProperties:
         maximum: 300000
 
       rohm,dvs-run-voltage:
-        description:
-          PMIC default "RUN" state voltage in uV. See below table for
-          bucks which support this. 0 means disabled.
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 3300000
 
       rohm,dvs-idle-voltage:
-        description:
-          PMIC default "IDLE" state voltage in uV. See below table for
-          bucks which support this. 0 means disabled.
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 3300000
 
       rohm,dvs-suspend-voltage:
-        description:
-          PMIC default "SUSPEND" state voltage in uV. See below table for
-          bucks which support this. 0 means disabled.
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 3300000
 
       rohm,dvs-lpsr-voltage:
-        description:
-          PMIC default "deep-idle" state voltage in uV. See below table for
-          bucks which support this. 0 means disabled.
-        $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
         maximum: 3300000
 
diff --git a/Documentation/devicetree/bindings/regulator/rohm,pmic-states.yaml b/Documentation/devicetree/bindings/regulator/rohm,pmic-states.yaml
new file mode 100644
index 0000000000000..f09f021bdde46
--- /dev/null
+++ b/Documentation/devicetree/bindings/regulator/rohm,pmic-states.yaml
@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/rohm,pmic-states.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ROHM PMICs' hardware state machine descriptions
+
+maintainers:
+  - Matti Vaittinen <mazziesaccount@gmail.com>
+
+description:
+  Many of the ROHM PMICs have internal state-machine designed to provide
+  correct voltages for different SOC states. In many cases the states and
+  transitions are SOC specific, but ones described here can be found from a
+  few of the different ROHM PMICs.
+
+properties:
+  rohm,dvs-run-voltage:
+    description:
+      PMIC "RUN" state voltage in uV when PMIC HW states are used. 0 means the
+      regulator should be disabled at RUN state. Some regulators do not support
+      setting HW state specific voltage but do support enable/disable control.
+      For them any positive value means the regulator should be enabled without
+      touching the voltage.
+    $ref: /schemas/types.yaml#/definitions/uint32
+
+  rohm,dvs-suspend-voltage:
+    description:
+      PMIC "SUSPEND" state voltage in uV when PMIC HW states are used. 0 means
+      the regulator should be disabled at SUSPEND state. Some regulators do not
+      support setting HW state specific voltage but do support enable/disable
+      control. For them any positive value means the regulator should be
+      enabled without touching the voltage.
+    $ref: /schemas/types.yaml#/definitions/uint32
+
+  rohm,dvs-idle-voltage:
+    description:
+      PMIC "IDLE" state voltage in uV. 0 means the regulator should be disabled
+      at IDLE state. Some regulators do not support setting HW state specific
+      voltage but do support enable/disable control. For them any positive
+      value means the regulator should be enabled without touching the voltage.
+    $ref: /schemas/types.yaml#/definitions/uint32
+
+  rohm,dvs-lpsr-voltage:
+    description:
+      PMIC "LPSR" state voltage in uV when PMIC HW states are used. 0 means the
+      regulator should be disabled at LPSR state. Some regulators do not
+      support setting HW state specific voltage but do support enable/disable
+      control. For them any positive value means the regulator should be
+      enabled without touching the voltage. Some PMICs' documentation may refer
+      to this state as a "deep-idle state".
+    $ref: /schemas/types.yaml#/definitions/uint32
+
+# The schema is expected to be reference by PMIC schemas, which may introduce
+# other properties that must be allowed.
+additionalProperties: true
diff --git a/Documentation/devicetree/bindings/remoteproc/amlogic,meson-mx-ao-arc.yaml b/Documentation/devicetree/bindings/remoteproc/amlogic,meson-mx-ao-arc.yaml
index 3f710433e9377..4cd5121b9a4e8 100644
--- a/Documentation/devicetree/bindings/remoteproc/amlogic,meson-mx-ao-arc.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/amlogic,meson-mx-ao-arc.yaml
@@ -25,7 +25,7 @@ properties:
       - const: amlogic,meson-mx-ao-arc
 
   firmware-name:
-    $ref: /schemas/types.yaml#/definitions/string
+    maxItems: 1
     description:
       The name of the firmware which should be loaded for this remote
       processor.
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.yaml
index 16a245fe2738d..7e8ecae8e6cb6 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.yaml
@@ -32,6 +32,14 @@ properties:
   reg:
     maxItems: 1
 
+  clocks:
+    items:
+      - description: XO clock
+
+  clock-names:
+    items:
+      - const: xo
+
   cx-supply: true
 
   px-supply:
@@ -49,6 +57,30 @@ properties:
     maxItems: 1
     description: Firmware name for the Hexagon core
 
+  interrupts:
+    items:
+      - description: Watchdog interrupt
+      - description: Fatal interrupt
+      - description: Ready interrupt
+      - description: Handover interrupt
+      - description: Stop acknowledge interrupt
+
+  interrupt-names:
+    items:
+      - const: wdog
+      - const: fatal
+      - const: ready
+      - const: handover
+      - const: stop-ack
+
+  qcom,smem-states:
+    maxItems: 1
+    description: States used by the AP to signal the Hexagon core
+
+  qcom,smem-state-names:
+    maxItems: 1
+    description: The names of the state bits used for SMP2P output
+
 required:
   - compatible
   - memory-region
@@ -57,56 +89,6 @@ unevaluatedProperties: false
 
 allOf:
   - $ref: /schemas/remoteproc/qcom,pas-common.yaml#
-  - if:
-      properties:
-        compatible:
-          contains:
-            enum:
-              - qcom,msm8226-adsp-pil
-              - qcom,msm8953-adsp-pil
-              - qcom,msm8974-adsp-pil
-              - qcom,msm8996-adsp-pil
-              - qcom,msm8996-slpi-pil
-              - qcom,msm8998-adsp-pas
-              - qcom,msm8998-slpi-pas
-              - qcom,sdm660-adsp-pas
-              - qcom,sdm660-cdsp-pas
-              - qcom,sdm845-adsp-pas
-              - qcom,sdm845-cdsp-pas
-              - qcom,sdm845-slpi-pas
-    then:
-      properties:
-        clocks:
-          items:
-            - description: XO clock
-        clock-names:
-          items:
-            - const: xo
-
-  - if:
-      properties:
-        compatible:
-          contains:
-            enum:
-              - qcom,msm8226-adsp-pil
-              - qcom,msm8953-adsp-pil
-              - qcom,msm8974-adsp-pil
-              - qcom,msm8996-adsp-pil
-              - qcom,msm8996-slpi-pil
-              - qcom,msm8998-adsp-pas
-              - qcom,msm8998-slpi-pas
-              - qcom,sdm660-adsp-pas
-              - qcom,sdm660-cdsp-pas
-              - qcom,sdm845-adsp-pas
-              - qcom,sdm845-cdsp-pas
-              - qcom,sdm845-slpi-pas
-    then:
-      properties:
-        interrupts:
-          maxItems: 5
-        interrupt-names:
-          maxItems: 5
-
   - if:
       properties:
         compatible:
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,kaanapali-soccp-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,kaanapali-soccp-pas.yaml
new file mode 100644
index 0000000000000..8fd6913e414d5
--- /dev/null
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,kaanapali-soccp-pas.yaml
@@ -0,0 +1,156 @@
+# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/remoteproc/qcom,kaanapali-soccp-pas.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Kaanapali SoCCP Peripheral Authentication Service
+
+maintainers:
+  - Jingyi Wang <jingyi.wang@oss.qualcomm.com>
+
+description:
+  The SoC Control Processor (SoCCP) is a small RISC-V MCU that controls USB
+  Type-C, battery charging and various other functions on Qualcomm SoCs, somewhat
+  analogous to traditional PC Embedded Controllers. This document describes
+  the Peripheral Authentication Service that loads and boots firmware for SoCCP.
+
+properties:
+  compatible:
+    oneOf:
+      - items:
+          - enum:
+              - qcom,glymur-soccp-pas
+              - qcom,hawi-soccp-pas
+              - qcom,maili-soccp-pas
+          - const: qcom,kaanapali-soccp-pas
+      - enum:
+          - qcom,kaanapali-soccp-pas
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: XO clock
+
+  clock-names:
+    items:
+      - const: xo
+
+  power-domains:
+    items:
+      - description: CX power domain
+      - description: MX power domain
+
+  power-domain-names:
+    items:
+      - const: cx
+      - const: mx
+
+  firmware-name:
+    items:
+      - description: Firmware name of the SoC Control Processor
+      - description: Firmware name of the SoCCP Devicetree
+
+  memory-region:
+    items:
+      - description: Memory region for main Firmware authentication
+      - description: Memory region for Devicetree Firmware authentication
+
+  interrupts:
+    items:
+      - description: Watchdog interrupt
+      - description: Fatal interrupt
+      - description: Ready interrupt
+      - description: Handover interrupt
+      - description: Stop acknowledge interrupt
+      - description: Pong interrupt
+
+  interrupt-names:
+    items:
+      - const: wdog
+      - const: fatal
+      - const: ready
+      - const: handover
+      - const: stop-ack
+      - const: pong
+
+  qcom,smem-states:
+    minItems: 2
+    description: States used by the AP to signal the SoC Control Processor
+
+  qcom,smem-state-names:
+    minItems: 2
+    description: The names of the state bits used for SMP2P output
+
+required:
+  - compatible
+  - reg
+  - memory-region
+  - power-domains
+  - power-domain-names
+
+allOf:
+  - $ref: /schemas/remoteproc/qcom,pas-common.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/qcom,rpmh.h>
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+    #include <dt-bindings/mailbox/qcom-ipcc.h>
+    #include <dt-bindings/power/qcom-rpmpd.h>
+    #define IPCC_MPROC_SOCCP
+
+    remoteproc@d00000 {
+        compatible = "qcom,kaanapali-soccp-pas";
+        reg = <0x00d00000 0x200000>;
+
+        clocks = <&rpmhcc RPMH_CXO_CLK>;
+        clock-names = "xo";
+
+        interrupts-extended = <&intc GIC_SPI 167 IRQ_TYPE_EDGE_RISING>,
+                              <&soccp_smp2p_in 0 IRQ_TYPE_EDGE_RISING>,
+                              <&soccp_smp2p_in 1 IRQ_TYPE_EDGE_RISING>,
+                              <&soccp_smp2p_in 2 IRQ_TYPE_EDGE_RISING>,
+                              <&soccp_smp2p_in 3 IRQ_TYPE_EDGE_RISING>,
+                              <&soccp_smp2p_in 9 IRQ_TYPE_EDGE_RISING>;
+        interrupt-names = "wdog",
+                          "fatal",
+                          "ready",
+                          "handover",
+                          "stop-ack",
+                          "pong";
+
+        memory-region = <&soccp_mem>,
+                        <&soccp_dtb_mem_mem>;
+
+        firmware-name = "qcom/kaanapali/soccp.mbn",
+                        "qcom/kaanapali/soccp_dtb.mbn";
+
+        power-domains = <&rpmhpd RPMHPD_CX>,
+                        <&rpmhpd RPMHPD_MX>;
+        power-domain-names = "cx",
+                             "mx";
+
+        qcom,smem-states = <&soccp_smp2p_out 0>,
+                           <&soccp_smp2p_out 8>;
+        qcom,smem-state-names = "stop",
+                                "ping";
+
+        glink-edge {
+            interrupts-extended = <&ipcc IPCC_MPROC_SOCCP
+                                         IPCC_MPROC_SIGNAL_GLINK_QMP
+                                         IRQ_TYPE_EDGE_RISING>;
+            mboxes = <&ipcc IPCC_MPROC_SOCCP
+                            IPCC_MPROC_SIGNAL_GLINK_QMP>;
+
+            label = "soccp";
+            qcom,remote-pid = <19>;
+
+            /* ... */
+        };
+    };
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,milos-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,milos-pas.yaml
index e5cce0d05fc69..99d7337e58ec5 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,milos-pas.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,milos-pas.yaml
@@ -34,12 +34,22 @@ properties:
       - const: xo
 
   interrupts:
-    minItems: 6
-    maxItems: 6
+    items:
+      - description: Watchdog interrupt
+      - description: Fatal interrupt
+      - description: Ready interrupt
+      - description: Handover interrupt
+      - description: Stop acknowledge interrupt
+      - description: Shutdown acknowledge interrupt
 
   interrupt-names:
-    minItems: 6
-    maxItems: 6
+    items:
+      - const: wdog
+      - const: fatal
+      - const: ready
+      - const: handover
+      - const: stop-ack
+      - const: shutdown-ack
 
   qcom,qmp:
     $ref: /schemas/types.yaml#/definitions/phandle
@@ -59,6 +69,14 @@ properties:
       - description: Memory region for core Firmware authentication
       - description: Memory region for Devicetree Firmware authentication
 
+  qcom,smem-states:
+    maxItems: 1
+    description: States used by the AP to signal the Hexagon core
+
+  qcom,smem-state-names:
+    maxItems: 1
+    description: The names of the state bits used for SMP2P output
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml
index 4607b459131b4..11faf655f530f 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml
@@ -26,23 +26,11 @@ properties:
 
   interrupts:
     minItems: 5
-    items:
-      - description: Watchdog interrupt
-      - description: Fatal interrupt
-      - description: Ready interrupt
-      - description: Handover interrupt
-      - description: Stop acknowledge interrupt
-      - description: Shutdown acknowledge interrupt
+    maxItems: 6
 
   interrupt-names:
     minItems: 5
-    items:
-      - const: wdog
-      - const: fatal
-      - const: ready
-      - const: handover
-      - const: stop-ack
-      - const: shutdown-ack
+    maxItems: 6
 
   iommus:
     maxItems: 1
@@ -58,8 +46,16 @@ properties:
   qcom,smem-states:
     $ref: /schemas/types.yaml#/definitions/phandle-array
     description: States used by the AP to signal the Hexagon core
+    minItems: 1
     items:
-      - description: Stop the modem
+      - description: Stop the remoteproc
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
+      - description: Ping the remoteproc
         items:
           - description: Phandle to the Shared Memory Point 2 Point device
               handling the communication with a remote processor
@@ -69,8 +65,10 @@ properties:
 
   qcom,smem-state-names:
     description: The names of the state bits used for SMP2P output
+    minItems: 1
     items:
       - const: stop
+      - const: ping
 
   smd-edge:
     $ref: /schemas/remoteproc/qcom,smd-edge.yaml#
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,qcs404-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,qcs404-pas.yaml
index ad45fd00ae346..bf9bf1af9ff14 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,qcs404-pas.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,qcs404-pas.yaml
@@ -32,10 +32,20 @@ properties:
       - const: xo
 
   interrupts:
-    maxItems: 5
+    items:
+      - description: Watchdog interrupt
+      - description: Fatal interrupt
+      - description: Ready interrupt
+      - description: Handover interrupt
+      - description: Stop acknowledge interrupt
 
   interrupt-names:
-    maxItems: 5
+    items:
+      - const: wdog
+      - const: fatal
+      - const: ready
+      - const: handover
+      - const: stop-ack
 
   power-domains: false
   power-domain-names: false
@@ -49,6 +59,14 @@ properties:
     maxItems: 1
     description: Firmware name for the Hexagon core
 
+  qcom,smem-states:
+    maxItems: 1
+    description: States used by the AP to signal the Hexagon core
+
+  qcom,smem-state-names:
+    maxItems: 1
+    description: The names of the state bits used for SMP2P output
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sa8775p-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sa8775p-pas.yaml
index bcd2bcf96e246..dda2d144b7206 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,sa8775p-pas.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,sa8775p-pas.yaml
@@ -59,10 +59,28 @@ properties:
       - description: Memory region for main Firmware authentication
 
   interrupts:
-    maxItems: 5
+    items:
+      - description: Watchdog interrupt
+      - description: Fatal interrupt
+      - description: Ready interrupt
+      - description: Handover interrupt
+      - description: Stop acknowledge interrupt
 
   interrupt-names:
-    maxItems: 5
+    items:
+      - const: wdog
+      - const: fatal
+      - const: ready
+      - const: handover
+      - const: stop-ack
+
+  qcom,smem-states:
+    maxItems: 1
+    description: States used by the AP to signal the Hexagon core
+
+  qcom,smem-state-names:
+    maxItems: 1
+    description: The names of the state bits used for SMP2P output
 
 required:
   - compatible
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-pas.yaml
index 66b455d0a8e32..b20780e5e26bd 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-pas.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-pas.yaml
@@ -48,6 +48,34 @@ properties:
     maxItems: 1
     description: Firmware name for the Hexagon core
 
+  interrupts:
+    minItems: 5
+    items:
+      - description: Watchdog interrupt
+      - description: Fatal interrupt
+      - description: Ready interrupt
+      - description: Handover interrupt
+      - description: Stop acknowledge interrupt
+      - description: Shutdown acknowledge interrupt
+
+  interrupt-names:
+    minItems: 5
+    items:
+      - const: wdog
+      - const: fatal
+      - const: ready
+      - const: handover
+      - const: stop-ack
+      - const: shutdown-ack
+
+  qcom,smem-states:
+    maxItems: 1
+    description: States used by the AP to signal the Hexagon core
+
+  qcom,smem-state-names:
+    maxItems: 1
+    description: The names of the state bits used for SMP2P output
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sc8280xp-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sc8280xp-pas.yaml
index 8227527c1d770..4bbe4a986c7c5 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,sc8280xp-pas.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,sc8280xp-pas.yaml
@@ -45,6 +45,34 @@ properties:
     maxItems: 1
     description: Firmware name for the Hexagon core
 
+  interrupts:
+    minItems: 5
+    items:
+      - description: Watchdog interrupt
+      - description: Fatal interrupt
+      - description: Ready interrupt
+      - description: Handover interrupt
+      - description: Stop acknowledge interrupt
+      - description: Shutdown acknowledge interrupt
+
+  interrupt-names:
+    minItems: 5
+    items:
+      - const: wdog
+      - const: fatal
+      - const: ready
+      - const: handover
+      - const: stop-ack
+      - const: shutdown-ack
+
+  qcom,smem-states:
+    maxItems: 1
+    description: States used by the AP to signal the Hexagon core
+
+  qcom,smem-state-names:
+    maxItems: 1
+    description: The names of the state bits used for SMP2P output
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sdx55-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sdx55-pas.yaml
index 8c4abde749150..8c16b01c53e42 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,sdx55-pas.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,sdx55-pas.yaml
@@ -30,10 +30,22 @@ properties:
       - const: xo
 
   interrupts:
-    minItems: 6
+    items:
+      - description: Watchdog interrupt
+      - description: Fatal interrupt
+      - description: Ready interrupt
+      - description: Handover interrupt
+      - description: Stop acknowledge interrupt
+      - description: Shutdown acknowledge interrupt
 
   interrupt-names:
-    minItems: 6
+    items:
+      - const: wdog
+      - const: fatal
+      - const: ready
+      - const: handover
+      - const: stop-ack
+      - const: shutdown-ack
 
   power-domains:
     items:
@@ -59,6 +71,14 @@ properties:
     maxItems: 1
     description: Firmware name for the Hexagon core
 
+  qcom,smem-states:
+    maxItems: 1
+    description: States used by the AP to signal the Hexagon core
+
+  qcom,smem-state-names:
+    maxItems: 1
+    description: The names of the state bits used for SMP2P output
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,shikra-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,shikra-pas.yaml
index 253b14eb2b598..34a2f15e93616 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,shikra-pas.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,shikra-pas.yaml
@@ -51,6 +51,26 @@ properties:
       and devices related to the remoteproc core.
     unevaluatedProperties: false
 
+  interrupts:
+    minItems: 5
+    items:
+      - description: Watchdog interrupt
+      - description: Fatal interrupt
+      - description: Ready interrupt
+      - description: Handover interrupt
+      - description: Stop acknowledge interrupt
+      - description: Shutdown acknowledge interrupt
+
+  interrupt-names:
+    minItems: 5
+    items:
+      - const: wdog
+      - const: fatal
+      - const: ready
+      - const: handover
+      - const: stop-ack
+      - const: shutdown-ack
+
   qcom,smem-states:
     $ref: /schemas/types.yaml#/definitions/phandle-array
     description: States used by the AP to signal the Hexagon core
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sm6115-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sm6115-pas.yaml
index eeb6a8aafeb92..454ba82bd6f18 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,sm6115-pas.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,sm6115-pas.yaml
@@ -51,6 +51,34 @@ properties:
     maxItems: 1
     description: Firmware name for the Hexagon core
 
+  interrupts:
+    minItems: 5
+    items:
+      - description: Watchdog interrupt
+      - description: Fatal interrupt
+      - description: Ready interrupt
+      - description: Handover interrupt
+      - description: Stop acknowledge interrupt
+      - description: Shutdown acknowledge interrupt
+
+  interrupt-names:
+    minItems: 5
+    items:
+      - const: wdog
+      - const: fatal
+      - const: ready
+      - const: handover
+      - const: stop-ack
+      - const: shutdown-ack
+
+  qcom,smem-states:
+    maxItems: 1
+    description: States used by the AP to signal the Hexagon core
+
+  qcom,smem-state-names:
+    maxItems: 1
+    description: The names of the state bits used for SMP2P output
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sm6350-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sm6350-pas.yaml
index c1a3cc308bdb2..42e02c64347a0 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,sm6350-pas.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,sm6350-pas.yaml
@@ -45,6 +45,34 @@ properties:
     maxItems: 1
     description: Firmware name for the Hexagon core
 
+  interrupts:
+    minItems: 5
+    items:
+      - description: Watchdog interrupt
+      - description: Fatal interrupt
+      - description: Ready interrupt
+      - description: Handover interrupt
+      - description: Stop acknowledge interrupt
+      - description: Shutdown acknowledge interrupt
+
+  interrupt-names:
+    minItems: 5
+    items:
+      - const: wdog
+      - const: fatal
+      - const: ready
+      - const: handover
+      - const: stop-ack
+      - const: shutdown-ack
+
+  qcom,smem-states:
+    maxItems: 1
+    description: States used by the AP to signal the Hexagon core
+
+  qcom,smem-state-names:
+    maxItems: 1
+    description: The names of the state bits used for SMP2P output
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sm6375-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sm6375-pas.yaml
index 7286b2baa19f2..274f87880e2e6 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,sm6375-pas.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,sm6375-pas.yaml
@@ -39,8 +39,36 @@ properties:
     maxItems: 1
     description: Firmware name for the Hexagon core
 
+  interrupts:
+    minItems: 5
+    items:
+      - description: Watchdog interrupt
+      - description: Fatal interrupt
+      - description: Ready interrupt
+      - description: Handover interrupt
+      - description: Stop acknowledge interrupt
+      - description: Shutdown acknowledge interrupt
+
+  interrupt-names:
+    minItems: 5
+    items:
+      - const: wdog
+      - const: fatal
+      - const: ready
+      - const: handover
+      - const: stop-ack
+      - const: shutdown-ack
+
   smd-edge: false
 
+  qcom,smem-states:
+    maxItems: 1
+    description: States used by the AP to signal the Hexagon core
+
+  qcom,smem-state-names:
+    maxItems: 1
+    description: The names of the state bits used for SMP2P output
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sm8150-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sm8150-pas.yaml
index a8cddf7e2fe1a..5a7c5f8c92d17 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,sm8150-pas.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,sm8150-pas.yaml
@@ -61,6 +61,34 @@ properties:
     maxItems: 1
     description: Firmware name for the Hexagon core
 
+  interrupts:
+    minItems: 5
+    items:
+      - description: Watchdog interrupt
+      - description: Fatal interrupt
+      - description: Ready interrupt
+      - description: Handover interrupt
+      - description: Stop acknowledge interrupt
+      - description: Shutdown acknowledge interrupt
+
+  interrupt-names:
+    minItems: 5
+    items:
+      - const: wdog
+      - const: fatal
+      - const: ready
+      - const: handover
+      - const: stop-ack
+      - const: shutdown-ack
+
+  qcom,smem-states:
+    maxItems: 1
+    description: States used by the AP to signal the Hexagon core
+
+  qcom,smem-state-names:
+    maxItems: 1
+    description: The names of the state bits used for SMP2P output
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sm8350-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sm8350-pas.yaml
index 6d09823153fc8..72d0db5698c5c 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,sm8350-pas.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,sm8350-pas.yaml
@@ -55,6 +55,34 @@ properties:
     maxItems: 1
     description: Firmware name for the Hexagon core
 
+  interrupts:
+    minItems: 5
+    items:
+      - description: Watchdog interrupt
+      - description: Fatal interrupt
+      - description: Ready interrupt
+      - description: Handover interrupt
+      - description: Stop acknowledge interrupt
+      - description: Shutdown acknowledge interrupt
+
+  interrupt-names:
+    minItems: 5
+    items:
+      - const: wdog
+      - const: fatal
+      - const: ready
+      - const: handover
+      - const: stop-ack
+      - const: shutdown-ack
+
+  qcom,smem-states:
+    maxItems: 1
+    description: States used by the AP to signal the Hexagon core
+
+  qcom,smem-state-names:
+    maxItems: 1
+    description: The names of the state bits used for SMP2P output
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml
index 9f30a38152a38..2a4c527552e69 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml
@@ -32,6 +32,7 @@ properties:
               - qcom,glymur-adsp-pas
               - qcom,hawi-adsp-pas
               - qcom,kaanapali-adsp-pas
+              - qcom,maili-adsp-pas
               - qcom,sm8750-adsp-pas
           - const: qcom,sm8550-adsp-pas
       - items:
@@ -39,6 +40,7 @@ properties:
               - qcom,glymur-cdsp-pas
               - qcom,hawi-cdsp-pas
               - qcom,kaanapali-cdsp-pas
+              - qcom,maili-cdsp-pas
           - const: qcom,sm8550-cdsp-pas
       - items:
           - const: qcom,sm8750-cdsp-pas
@@ -75,6 +77,34 @@ properties:
       - description: DSM Memory region 2
       - description: Memory region for Qlink Logging
 
+  interrupts:
+    minItems: 5
+    items:
+      - description: Watchdog interrupt
+      - description: Fatal interrupt
+      - description: Ready interrupt
+      - description: Handover interrupt
+      - description: Stop acknowledge interrupt
+      - description: Shutdown acknowledge interrupt
+
+  interrupt-names:
+    minItems: 5
+    items:
+      - const: wdog
+      - const: fatal
+      - const: ready
+      - const: handover
+      - const: stop-ack
+      - const: shutdown-ack
+
+  qcom,smem-states:
+    maxItems: 1
+    description: States used by the AP to signal the Hexagon core
+
+  qcom,smem-state-names:
+    maxItems: 1
+    description: The names of the state bits used for SMP2P output
+
 required:
   - compatible
   - reg
@@ -110,6 +140,8 @@ allOf:
               - qcom,hawi-cdsp-pas
               - qcom,kaanapali-adsp-pas
               - qcom,kaanapali-cdsp-pas
+              - qcom,maili-adsp-pas
+              - qcom,maili-cdsp-pas
               - qcom,sm8750-adsp-pas
     then:
       properties:
diff --git a/Documentation/devicetree/bindings/remoteproc/ti,pru-consumer.yaml b/Documentation/devicetree/bindings/remoteproc/ti,pru-consumer.yaml
index 2811334515d16..591471e5f93af 100644
--- a/Documentation/devicetree/bindings/remoteproc/ti,pru-consumer.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/ti,pru-consumer.yaml
@@ -30,7 +30,6 @@ properties:
       maxItems: 1
 
   firmware-name:
-    $ref: /schemas/types.yaml#/definitions/string-array
     minItems: 1
     maxItems: 6
     description: |
diff --git a/Documentation/devicetree/bindings/riscv/canaan.yaml b/Documentation/devicetree/bindings/riscv/canaan.yaml
index 41fd11f70a49b..f9854ff43ac68 100644
--- a/Documentation/devicetree/bindings/riscv/canaan.yaml
+++ b/Documentation/devicetree/bindings/riscv/canaan.yaml
@@ -10,7 +10,7 @@ maintainers:
   - Damien Le Moal <dlemoal@kernel.org>
 
 description:
-  Canaan Kendryte K210 SoC-based boards
+  Canaan Kendryte SoC-based boards
 
 properties:
   $nodename:
@@ -42,6 +42,12 @@ properties:
       - items:
           - const: canaan,kendryte-k210
 
+      - items:
+          - enum:
+              - canaan,canmv-k230
+              - canaan,k230-usip-lp3-evb
+          - const: canaan,kendryte-k230
+
 additionalProperties: true
 
 ...
diff --git a/Documentation/devicetree/bindings/rng/inside-secure,safexcel-eip76.yaml b/Documentation/devicetree/bindings/rng/inside-secure,safexcel-eip76.yaml
index f501fc7691c6e..92d9069982115 100644
--- a/Documentation/devicetree/bindings/rng/inside-secure,safexcel-eip76.yaml
+++ b/Documentation/devicetree/bindings/rng/inside-secure,safexcel-eip76.yaml
@@ -4,7 +4,7 @@
 $id: http://devicetree.org/schemas/rng/inside-secure,safexcel-eip76.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Inside-Secure HWRNG Module
+title: Inside-Secure SafeXcel EIP-76 HWRNG Module
 
 maintainers:
   - Jayesh Choudhary <j-choudhary@ti.com>
diff --git a/Documentation/devicetree/bindings/rtc/allwinner,sun6i-a31-rtc.yaml b/Documentation/devicetree/bindings/rtc/allwinner,sun6i-a31-rtc.yaml
index 9df5cdb6f63f2..f2b91186ed377 100644
--- a/Documentation/devicetree/bindings/rtc/allwinner,sun6i-a31-rtc.yaml
+++ b/Documentation/devicetree/bindings/rtc/allwinner,sun6i-a31-rtc.yaml
@@ -33,6 +33,7 @@ properties:
           - enum:
               - allwinner,sun20i-d1-rtc
               - allwinner,sun55i-a523-rtc
+              - allwinner,sun60i-a733-rtc
           - const: allwinner,sun50i-r329-rtc
 
   reg:
@@ -175,6 +176,18 @@ allOf:
         interrupts:
           minItems: 2
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - allwinner,sun50i-h616-rtc
+              - allwinner,sun50i-r329-rtc
+
+    then:
+      properties:
+        clock-output-names: false
+
 required:
   - "#clock-cells"
   - compatible
diff --git a/Documentation/devicetree/bindings/serial/maxim,max310x.yaml b/Documentation/devicetree/bindings/serial/maxim,max310x.yaml
index 889eeaca64a02..e598dda4d13f5 100644
--- a/Documentation/devicetree/bindings/serial/maxim,max310x.yaml
+++ b/Documentation/devicetree/bindings/serial/maxim,max310x.yaml
@@ -9,6 +9,13 @@ title: Maxim MAX310X Advanced Universal Asynchronous Receiver-Transmitter (UART)
 maintainers:
   - Hugo Villeneuve <hvilleneuve@dimonoff.com>
 
+description:
+  The MAX310X is a family of SPI/I2C UARTs with one (max3107, max3108),
+  two (max3109) or four (max14830) channels. Single-channel parts are
+  described as a serial node with RS-485 properties on the chip node;
+  multi-channel parts use one "serial@N" child node per channel, each
+  carrying its own serial/RS-485 properties.
+
 properties:
   compatible:
     enum:
@@ -49,8 +56,55 @@ required:
 
 allOf:
   - $ref: /schemas/spi/spi-peripheral-props.yaml#
-  - $ref: /schemas/serial/serial.yaml#
-  - $ref: /schemas/serial/rs485.yaml#
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - maxim,max3107
+              - maxim,max3108
+    then:
+      allOf:
+        - $ref: /schemas/serial/serial.yaml#
+        - $ref: /schemas/serial/rs485.yaml#
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - maxim,max3109
+              - maxim,max14830
+    then:
+      properties:
+        "#address-cells":
+          const: 1
+        "#size-cells":
+          const: 0
+      patternProperties:
+        "^serial@[0-3]$":
+          type: object
+          description: A single UART channel of the chip.
+          allOf:
+            - $ref: /schemas/serial/serial.yaml#
+            - $ref: /schemas/serial/rs485.yaml#
+          properties:
+            reg:
+              description: UART channel number on the chip.
+              maximum: 3
+          required:
+            - reg
+          unevaluatedProperties: false
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: maxim,max3109
+    then:
+      patternProperties:
+        "^serial@[23]$": false
 
 unevaluatedProperties: false
 
@@ -70,5 +124,39 @@ examples:
             interrupts = <7 IRQ_TYPE_LEVEL_LOW>;
             gpio-controller;
             #gpio-cells = <2>;
+            rs485-rts-active-low;
+            linux,rs485-enabled-at-boot-time;
+        };
+    };
+
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        serial@0 {
+            compatible = "maxim,max14830";
+            reg = <0>;
+            spi-max-frequency = <26000000>;
+            clocks = <&xtal4m>;
+            clock-names = "xtal";
+            interrupt-parent = <&gpio3>;
+            interrupts = <7 IRQ_TYPE_LEVEL_LOW>;
+            gpio-controller;
+            #gpio-cells = <2>;
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            serial@0 {
+                reg = <0>;
+                rs485-rts-active-low;
+                linux,rs485-enabled-at-boot-time;
+            };
+
+            serial@2 {
+                reg = <2>;
+                rs485-rts-active-low;
+            };
         };
     };
diff --git a/Documentation/devicetree/bindings/serial/qcom,msm-uartdm.yaml b/Documentation/devicetree/bindings/serial/qcom,msm-uartdm.yaml
index 788ef5c1c4469..bc967ead2350c 100644
--- a/Documentation/devicetree/bindings/serial/qcom,msm-uartdm.yaml
+++ b/Documentation/devicetree/bindings/serial/qcom,msm-uartdm.yaml
@@ -17,7 +17,7 @@ description: |
   software perspective it's mostly compatible with the MSM serial UART except
   that it supports reading and writing multiple characters at a time.
 
-  Note:: Aliases may be defined to ensure the correct ordering of the UARTs.
+  Note: Aliases may be defined to ensure the correct ordering of the UARTs.
   The alias serialN will result in the UART being assigned port N.  If any
   serialN alias exists, then an alias must exist for each enabled UART.  The
   serialN aliases should be in a .dts file instead of in a .dtsi file.
diff --git a/Documentation/devicetree/bindings/slimbus/slimbus.yaml b/Documentation/devicetree/bindings/slimbus/slimbus.yaml
index 5a941610ce4ed..3910327c8ded1 100644
--- a/Documentation/devicetree/bindings/slimbus/slimbus.yaml
+++ b/Documentation/devicetree/bindings/slimbus/slimbus.yaml
@@ -29,7 +29,7 @@ patternProperties:
     description: |
       Every SLIMbus controller node can contain zero or more child nodes
       representing slave devices on the bus. Every SLIMbus slave device is
-      uniquely determined by the enumeration address containing 4 fields::
+      uniquely determined by the enumeration address containing 4 fields:
       Manufacturer ID, Product code, Device index, and Instance value for the
       device.
 
@@ -48,7 +48,7 @@ patternProperties:
       reg:
         maxItems: 1
         description: |
-          Pair of (device index, instande ID), where::
+          Pair of (device index, instande ID), where:
            - Device index, which uniquely identifies multiple devices within a
              single component.
            - Instance ID, can be used for the cases where multiple devices of
diff --git a/Documentation/devicetree/bindings/soc/amlogic/amlogic,meson-gx-clk-measure.yaml b/Documentation/devicetree/bindings/soc/amlogic/amlogic,meson-gx-clk-measure.yaml
index 39d4637c2d088..b1200e6940acd 100644
--- a/Documentation/devicetree/bindings/soc/amlogic/amlogic,meson-gx-clk-measure.yaml
+++ b/Documentation/devicetree/bindings/soc/amlogic/amlogic,meson-gx-clk-measure.yaml
@@ -24,6 +24,8 @@ properties:
       - amlogic,meson-sm1-clk-measure
       - amlogic,c3-clk-measure
       - amlogic,s4-clk-measure
+      - amlogic,a1-clk-measure
+      - amlogic,t7-clk-measure
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml
new file mode 100644
index 0000000000000..588100222f892
--- /dev/null
+++ b/Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml
@@ -0,0 +1,45 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/soc/apple/apple,t6000-pmgr-misc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Apple SoC PMGR Misc Power States
+
+maintainers:
+  - Sasha Finkelstein <k@chaosmail.tech>
+
+description: |
+  Certain Apple SoCs include additional PMGR power states that are controlled
+  via a different "misc" control block. This includes the fabric and memory
+  controller states.
+
+properties:
+  compatible:
+    enum:
+      - apple,t6000-pmgr-misc
+      - apple,t6020-pmgr-misc
+
+  reg:
+    maxItems: 2
+
+  reg-names:
+    items:
+      - const: fabric-ps
+      - const: dcs-ps
+
+required:
+  - compatible
+  - reg
+  - reg-names
+
+additionalProperties: false
+
+examples:
+  - |
+    power-management@8e20c000 {
+        compatible = "apple,t6000-pmgr-misc";
+        reg = <0x8e20c000 0x400>,
+              <0x8e20c800 0x400>;
+        reg-names = "fabric-ps", "dcs-ps";
+    };
diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml
index d828c2e82965c..124f5c206ee3d 100644
--- a/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml
+++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml
@@ -26,6 +26,12 @@ properties:
   reg:
     maxItems: 1
 
+  '#address-cells':
+    const: 1
+
+  '#size-cells':
+    const: 1
+
   '#power-domain-cells':
     const: 1
 
@@ -92,6 +98,11 @@ properties:
       - compatible
       - ports
 
+  bridge@20:
+    type: object
+    $ref: /schemas/display/bridge/fsl,ldb.yaml#
+    unevaluatedProperties: false
+
 allOf:
   - if:
       properties:
@@ -112,6 +123,7 @@ allOf:
             - const: lcdif
             - const: isi
             - const: csi
+        bridge@20: false
   - if:
       properties:
         compatible:
@@ -163,6 +175,8 @@ examples:
                <&clk IMX93_CLK_MIPI_DSI_GATE>;
                clock-names = "apb", "axi", "nic", "disp", "cam",
                              "pxp", "lcdif", "isi", "csi", "dsi";
+      #address-cells = <1>;
+      #size-cells = <1>;
       #power-domain-cells = <1>;
 
       dpi-bridge {
@@ -190,4 +204,29 @@ examples:
           };
         };
       };
+
+      bridge@20 {
+        compatible = "fsl,imx93-ldb";
+        reg = <0x20 0x4>, <0x24 0x4>;
+        reg-names = "ldb", "lvds";
+        clocks = <&clk IMX93_CLK_LVDS_GATE>;
+        clock-names = "ldb";
+
+        ports {
+          #address-cells = <1>;
+          #size-cells = <0>;
+
+          port@0 {
+            reg = <0>;
+
+            endpoint {
+              remote-endpoint = <&lcdif_to_ldb>;
+            };
+          };
+
+          port@1 {
+            reg = <1>;
+          };
+        };
+      };
     };
diff --git a/Documentation/devicetree/bindings/soc/mediatek/mediatek,mutex.yaml b/Documentation/devicetree/bindings/soc/mediatek/mediatek,mutex.yaml
index 5267cfe92572d..1ba086ad749dd 100644
--- a/Documentation/devicetree/bindings/soc/mediatek/mediatek,mutex.yaml
+++ b/Documentation/devicetree/bindings/soc/mediatek/mediatek,mutex.yaml
@@ -4,18 +4,21 @@
 $id: http://devicetree.org/schemas/soc/mediatek/mediatek,mutex.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Mediatek mutex
+title: MediaTek Mute-X
 
 maintainers:
+  - AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
   - Chun-Kuang Hu <chunkuang.hu@kernel.org>
   - Philipp Zabel <p.zabel@pengutronix.de>
 
-description: |
-  Mediatek mutex, namely MUTEX, is used to send the triggers signals called
-  Start Of Frame (SOF) / End Of Frame (EOF) to each sub-modules on the display
-  data path or MDP data path.
-  In some SoC, such as mt2701, MUTEX could be a hardware mutex which protects
-  the shadow register.
+description:
+  MediaTek Mute-X, namely MUTEX, is used to "mute" or "unmute" trigger signals
+  like Start Of Frame (SOF), End Of Frame (EOF), Tearing Effect (TE / VSYNC)
+  and others to each hardware sub-modules in the Display Controller IP or in
+  the Media Data Path (MDP) IP.
+  In some SoCs like MT2701, this hardware module may feature functionality
+  to, for example, protect shadow registers by blocking auto write triggers
+  upon operation (usually frame push) completion.
   MUTEX device node must be siblings to the central MMSYS_CONFIG node.
   For a description of the MMSYS_CONFIG binding, see
   Documentation/devicetree/bindings/arm/mediatek/mediatek,mmsys.yaml
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml
index 8eaa04431d745..9f9d9bcb66c41 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml
@@ -29,6 +29,7 @@ properties:
           - qcom,glymur-aoss-qmp
           - qcom,hawi-aoss-qmp
           - qcom,kaanapali-aoss-qmp
+          - qcom,maili-aoss-qmp
           - qcom,milos-aoss-qmp
           - qcom,qcs615-aoss-qmp
           - qcom,qcs8300-aoss-qmp
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,apr-services.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,apr-services.yaml
index bdf482db32aac..b663be3ea5a17 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom,apr-services.yaml
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,apr-services.yaml
@@ -40,7 +40,7 @@ properties:
     $ref: /schemas/types.yaml#/definitions/string-array
     description: |
       Protection domain service name and path for APR service (if supported).
-      Possible values are::
+      Possible values are:
       "avs/audio", "msm/adsp/audio_pd".
       "kernel/elf_loader", "msm/modem/wlan_pd".
       "tms/servreg", "msm/adsp/audio_pd".
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,pmic-glink.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,pmic-glink.yaml
index 2db4288a8a54a..16ad4f5698fc3 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom,pmic-glink.yaml
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,pmic-glink.yaml
@@ -35,6 +35,7 @@ properties:
       - items:
           - enum:
               - qcom,hawi-pmic-glink
+              - qcom,maili-pmic-glink
           - const: qcom,kaanapali-pmic-glink
           - const: qcom,pmic-glink
       - items:
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,rpmh-rsc.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,rpmh-rsc.yaml
index 26d9bc773ec5c..1889139a3f7af 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom,rpmh-rsc.yaml
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,rpmh-rsc.yaml
@@ -23,7 +23,7 @@ description: |
   with a few variations that are captured by the properties here.
 
   A TCS may be triggered from Linux or triggered by the F/W after all the CPUs
-  have powered off to facilitate idle power saving. TCS could be classified as::
+  have powered off to facilitate idle power saving. TCS could be classified as:
     ACTIVE  - Triggered by Linux
     SLEEP   - Triggered by F/W
     WAKE    - Triggered by F/W
@@ -76,7 +76,7 @@ properties:
     items:
       items:
         - description: |
-            TCS type::
+            TCS type:
              - ACTIVE_TCS
              - SLEEP_TCS
              - WAKE_TCS
@@ -152,7 +152,7 @@ examples:
   - |
     // For a TCS whose RSC base address is 0x179C0000 and is at a DRV id of
     // 2, the register offsets for DRV2 start at 0D00, the register
-    // calculations are like this::
+    // calculations are like this:
     // DRV0: 0x179C0000
     // DRV2: 0x179C0000 + 0x10000 = 0x179D0000
     // DRV2: 0x179C0000 + 0x10000 * 2 = 0x179E0000
@@ -182,7 +182,7 @@ examples:
   - |
     // For a TCS whose RSC base address is 0xAF20000 and is at DRV id of 0, the
     // register offsets for DRV0 start at 01C00, the register calculations are
-    // like this::
+    // like this:
     // DRV0: 0xAF20000
     // TCS-OFFSET: 0x1C00
     #include <dt-bindings/interrupt-controller/arm-gic.h>
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,smd-rpm.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,smd-rpm.yaml
index 270bcd079f886..bd1d32898461c 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom,smd-rpm.yaml
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,smd-rpm.yaml
@@ -57,6 +57,7 @@ properties:
               - qcom,rpm-qcm2290
               - qcom,rpm-qcs404
               - qcom,rpm-sdm660
+              - qcom,rpm-shikra
               - qcom,rpm-sm6115
               - qcom,rpm-sm6125
               - qcom,rpm-sm6375
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,wcnss.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,wcnss.yaml
index 4fcae6bedfffa..72a7f8cb09bad 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom,wcnss.yaml
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,wcnss.yaml
@@ -28,7 +28,7 @@ properties:
     $ref: /schemas/types.yaml#/definitions/phandle
     description: |
       Reference to a node specifying the wcnss "ccu" and "dxe" register blocks.
-      The node must be compatible with one of the following::
+      The node must be compatible with one of the following:
            - qcom,riva"
            - qcom,pronto"
 
diff --git a/Documentation/devicetree/bindings/soc/renesas/renesas-soc.yaml b/Documentation/devicetree/bindings/soc/renesas/renesas-soc.yaml
index 5ddd31f30f267..57c9d3c570212 100644
--- a/Documentation/devicetree/bindings/soc/renesas/renesas-soc.yaml
+++ b/Documentation/devicetree/bindings/soc/renesas/renesas-soc.yaml
@@ -12,10 +12,10 @@ maintainers:
 
 description: |
   Guidelines for new compatibles for SoC blocks/components.
-  When adding new compatibles in new bindings, use the format::
+  When adding new compatibles in new bindings, use the format:
     renesas,SoC-IP
 
-  For example::
+  For example:
    renesas,r8a77965-csi2
 
   When adding new compatibles to existing bindings, use the format in the
diff --git a/Documentation/devicetree/bindings/soc/rockchip/grf.yaml b/Documentation/devicetree/bindings/soc/rockchip/grf.yaml
index 2cc43742b8e3b..7bcb4e2f47ec5 100644
--- a/Documentation/devicetree/bindings/soc/rockchip/grf.yaml
+++ b/Documentation/devicetree/bindings/soc/rockchip/grf.yaml
@@ -239,11 +239,14 @@ allOf:
       properties:
         reboot-mode:
           type: object
-
           $ref: /schemas/power/reset/syscon-reboot-mode.yaml#
-
           unevaluatedProperties: false
 
+          patternProperties:
+            # Negative look-ahead to disallow unsupported modes. The '$' has to be
+            # part of lookahead group to work, instead of trailing outside of ().
+            "^mode-(?!(bootloader$|fastboot$|loader$|normal$|recovery$))": false
+
   - if:
       properties:
         compatible:
diff --git a/Documentation/devicetree/bindings/soc/samsung/exynos-pmu.yaml b/Documentation/devicetree/bindings/soc/samsung/exynos-pmu.yaml
index 76ce7e98c10f3..5bd39876e3b36 100644
--- a/Documentation/devicetree/bindings/soc/samsung/exynos-pmu.yaml
+++ b/Documentation/devicetree/bindings/soc/samsung/exynos-pmu.yaml
@@ -94,19 +94,19 @@ properties:
 
   reboot-mode:
     $ref: /schemas/power/reset/syscon-reboot-mode.yaml
-    type: object
+    unevaluatedProperties: false
     description:
       Reboot mode to alter bootloader behavior for the next boot
 
   syscon-poweroff:
     $ref: /schemas/power/reset/syscon-poweroff.yaml#
-    type: object
+    unevaluatedProperties: false
     description:
       Node for power off method
 
   syscon-reboot:
     $ref: /schemas/power/reset/syscon-reboot.yaml#
-    type: object
+    unevaluatedProperties: false
     description:
       Node for reboot method
 
@@ -176,6 +176,20 @@ allOf:
       properties:
         dp-phy: false
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - samsung,exynos7-pmu
+    then:
+      properties:
+        reboot-mode:
+          patternProperties:
+            # Negative look-ahead to disallow unsupported modes. The '$' has to be
+            # part of lookahead group to work, instead of trailing outside of ().
+            "^mode-(?!(bootloader$|download$|fastboot$|recovery$))": false
+
   - if:
       properties:
         compatible:
@@ -183,6 +197,13 @@ allOf:
             enum:
               - google,gs101-pmu
     then:
+      properties:
+        reboot-mode:
+          patternProperties:
+            # Negative look-ahead to disallow unsupported modes. The '$' has to be
+            # part of lookahead group to work, instead of trailing outside of ().
+            "^mode-(?!(bootloader$|charge$|dm-verity-device-corrupted$|fastboot$|reboot-ab-update$|recovery$)|rescue$|shutdown-thermal$|shutdown-thermal-battery$)": false
+
       required:
         - google,pmu-intr-gen-syscon
     else:
diff --git a/Documentation/devicetree/bindings/soc/samsung/samsung,exynos-sysreg.yaml b/Documentation/devicetree/bindings/soc/samsung/samsung,exynos-sysreg.yaml
index 9c63dbcd4d77f..fcdb9ffd26448 100644
--- a/Documentation/devicetree/bindings/soc/samsung/samsung,exynos-sysreg.yaml
+++ b/Documentation/devicetree/bindings/soc/samsung/samsung,exynos-sysreg.yaml
@@ -37,6 +37,7 @@ properties:
               - samsung,exynos8895-peric1-sysreg
               - samsung,exynos990-peric0-sysreg
               - samsung,exynos990-peric1-sysreg
+              - samsung,exynosautov920-hsi0-sysreg
               - samsung,exynosautov920-hsi2-sysreg
               - samsung,exynosautov920-peric0-sysreg
               - samsung,exynosautov920-peric1-sysreg
diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs35l36.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs35l36.yaml
new file mode 100644
index 0000000000000..2a142b32acf50
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/cirrus,cs35l36.yaml
@@ -0,0 +1,240 @@
+# SPDX-License-Identifier: GPL-2.0-only
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/cirrus,cs35l36.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Cirrus Logic CS35L36 Speaker Amplifier
+
+maintainers:
+  - David Rhodes <drhodes@opensource.cirrus.com>
+  - patches@opensource.cirrus.com
+
+description:
+  CS35L36 is a boosted mono Class D amplifier
+
+allOf:
+  - $ref: dai-common.yaml#
+
+properties:
+  compatible:
+    enum:
+      - cirrus,cs35l36
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  VA-supply:
+    description: Voltage regulator of analog internal section
+
+  VP-supply:
+    description: Voltage regulator of boost converter
+
+  reset-gpios:
+    maxItems: 1
+
+  cirrus,boost-ctl-millivolt:
+    description: Boost converter output voltage (step 50)
+    $ref: /schemas/types.yaml#/definitions/uint32
+    minimum: 2550
+    maximum: 12000
+
+  cirrus,boost-peak-milliamp:
+    description: Boost-converter peak current limit (step 50)
+    $ref: /schemas/types.yaml#/definitions/uint32
+    default: 4500
+    minimum: 1600
+    maximum: 4500
+
+  cirrus,boost-ind-nanohenry:
+    description: Initial inductor estimation reference value (1000=1μH, 1200=1.2μH)
+    $ref: /schemas/types.yaml#/definitions/uint32
+    default: 1000
+
+  cirrus,multi-amp-mode:
+    description: Hi-Z ASP port when more than one amplifier in system
+    type: boolean
+
+  cirrus,boost-ctl-select:
+    description: Boost converter control source selection
+    $ref: /schemas/types.yaml#/definitions/uint32
+    default: 1
+    enum:
+      - 0 # Control Port
+      - 1 # Class
+      - 2 # Sync
+
+  cirrus,amp-pcm-inv:
+    description: Invert incoming PCM data when true
+    type: boolean
+
+  cirrus,imon-pol-inv:
+    description: Invert polarity of outbound IMON feedback when true
+    type: boolean
+
+  cirrus,vmon-pol-inv:
+    description: Invert polarity of outbound VMON feedback when true
+    type: boolean
+
+  cirrus,dcm-mode-enable:
+    description: Enable boost converter automatic Discontinuous Conduction Mode
+    type: boolean
+
+  cirrus,weak-fet-disable:
+    description: Reduce output driver strength in Weak-FET Drive Mode when true
+    type: boolean
+
+  cirrus,classh-wk-fet-delay-ms:
+    description: Weak-FET entry delay
+    default: 100
+    enum: [0, 5, 10, 50, 100, 200, 500, 1000]
+
+  cirrus,classh-weak-fet-thld-millivolt:
+    description: Weak-FET drive threshold
+    $ref: /schemas/types.yaml#/definitions/uint32
+    enum: [50, 100, 150, 200, 250, 300, 350, 400, 450, 500, 550, 600, 650, 700]
+
+  cirrus,temp-warn-threshold:
+    description: Overtemperature warning threshold
+    $ref: /schemas/types.yaml#/definitions/uint32
+    default: 2
+    enum:
+      - 0 # 105°C
+      - 1 # 115°C
+      - 2 # 125°C
+      - 3 # 135°C
+
+  cirrus,irq-drive-select:
+    description: Interrupt output driver type
+    $ref: /schemas/types.yaml#/definitions/uint32
+    default: 1
+    enum:
+      - 0 # open-drain
+      - 1 # push-pull
+
+  cirrus,irq-gpio-select:
+    description: Programmable IRQ pin selection
+    $ref: /schemas/types.yaml#/definitions/uint32
+    enum:
+      - 0 # PDM_DATA/SWIRE_SD/INT
+      - 1 # GPIO
+
+  cirrus,vpbr-config:
+    $ref: "#/$defs/vpbr-config"
+
+$defs:
+  vpbr-config:
+    description: Brownout prevention configuration sub-node
+    type: object
+    additionalProperties: false
+
+    properties:
+      cirrus,vpbr-en:
+        description: VBST brownout prevention enable
+        $ref: /schemas/types.yaml#/definitions/uint32
+        default: 0
+        enum:
+          - 0 # disabled
+          - 1 # enabled
+
+      cirrus,vpbr-thld:
+        description: Initial VPBR threshold voltage
+        $ref: /schemas/types.yaml#/definitions/uint32
+        minimum: 2
+        maximum: 31
+
+      cirrus,vpbr-atk-rate:
+        description: Attenuation attack step rate
+        $ref: /schemas/types.yaml#/definitions/uint32
+        minimum: 0
+        maximum: 7
+
+      cirrus,vpbr-atk-vol:
+        description: VP brownout prevention step size
+        $ref: /schemas/types.yaml#/definitions/uint32
+        minimum: 0
+        maximum: 7
+
+      cirrus,vpbr-max-attn:
+        description: Maximum attenuation during VP brownout prevention (dB)
+        $ref: /schemas/types.yaml#/definitions/uint32
+        minimum: 0
+        maximum: 15
+
+      cirrus,vpbr-wait:
+        description: Delay between brownout clearance and attenuation release (ms)
+        $ref: /schemas/types.yaml#/definitions/uint32
+        default: 1
+        enum:
+          - 0 # 10
+          - 1 # 100
+          - 2 # 250
+          - 3 # 500
+
+      cirrus,vpbr-rel-rate:
+        description: Attenuation release step rate
+        $ref: /schemas/types.yaml#/definitions/uint32
+        minimum: 0
+        maximum: 7
+
+      cirrus,vpbr-mute-en:
+        description: Mute audio if maximum attenuation reached
+        $ref: /schemas/types.yaml#/definitions/uint32
+        minimum: 0
+        maximum: 1
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - VA-supply
+  - cirrus,boost-ctl-millivolt
+  - cirrus,boost-peak-milliamp
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        codec@40 {
+            compatible = "cirrus,cs35l36";
+            reg = <0x40>;
+            VA-supply = <&dummy_vreg>;
+            VP-supply = <&dummy_vreg>;
+            reset-gpios = <&gpio0 54 GPIO_ACTIVE_HIGH>;
+            interrupts = <3 IRQ_TYPE_LEVEL_LOW>;
+
+            cirrus,boost-ind-nanohenry = <1000>;
+            cirrus,boost-ctl-millivolt = <10000>;
+            cirrus,boost-peak-milliamp = <4500>;
+            cirrus,boost-ctl-select = <0>;
+            cirrus,classh-wk-fet-delay-ms = <100>;
+            cirrus,classh-weak-fet-thld-millivolt = <100>;
+            cirrus,temp-warn-threshold = <1>;
+            cirrus,multi-amp-mode;
+            cirrus,irq-drive-select = <1>;
+            cirrus,irq-gpio-select = <1>;
+
+            cirrus,vpbr-config {
+                cirrus,vpbr-en = <0>;
+                cirrus,vpbr-thld = <5>;
+                cirrus,vpbr-atk-rate = <2>;
+                cirrus,vpbr-atk-vol = <1>;
+                cirrus,vpbr-max-attn = <9>;
+                cirrus,vpbr-wait = <1>;
+                cirrus,vpbr-rel-rate = <5>;
+                cirrus,vpbr-mute-en = <0>;
+            };
+        };
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/sound/cs35l36.txt b/Documentation/devicetree/bindings/sound/cs35l36.txt
deleted file mode 100644
index d34117b8558e5..0000000000000
--- a/Documentation/devicetree/bindings/sound/cs35l36.txt
+++ /dev/null
@@ -1,168 +0,0 @@
-CS35L36 Speaker Amplifier
-
-Required properties:
-
-  - compatible : "cirrus,cs35l36"
-
-  - reg : the I2C address of the device for I2C
-
-  - VA-supply, VP-supply : power supplies for the device,
-  as covered in
-  Documentation/devicetree/bindings/regulator/regulator.txt.
-
-  - cirrus,boost-ctl-millivolt : Boost Voltage Value.  Configures the boost
-  converter's output voltage in mV. The range is from 2550mV to 12000mV with
-  increments of 50mV.
-  (Default) VP
-
-  - cirrus,boost-peak-milliamp : Boost-converter peak current limit in mA.
-  Configures the peak current by monitoring the current through the boost FET.
-  Range starts at 1600mA and goes to a maximum of 4500mA with increments of
-  50mA.
-  (Default) 4.50 Amps
-
-  - cirrus,boost-ind-nanohenry : Inductor estimation LBST reference value.
-  Seeds the digital boost converter's inductor estimation block with the initial
-  inductance value to reference.
-
-  1000 = 1uH (Default)
-  1200 = 1.2uH
-
-Optional properties:
-  - cirrus,multi-amp-mode : Boolean to determine if there are more than
-  one amplifier in the system. If more than one it is best to Hi-Z the ASP
-  port to prevent bus contention on the output signal
-
-  - cirrus,boost-ctl-select : Boost converter control source selection.
-  Selects the source of the BST_CTL target VBST voltage for the boost
-  converter to generate.
-  0x00 - Control Port Value
-  0x01 - Class H Tracking (Default)
-  0x10 - MultiDevice Sync Value
-
-  - cirrus,amp-pcm-inv : Boolean to determine Amplifier will invert incoming
-  PCM data
-
-  - cirrus,imon-pol-inv : Boolean to determine Amplifier will invert the
-  polarity of outbound IMON feedback data
-
-  - cirrus,vmon-pol-inv : Boolean to determine Amplifier will invert the
-  polarity of outbound VMON feedback data
-
-  - cirrus,dcm-mode-enable : Boost converter automatic DCM Mode enable.
-  This enables the digital boost converter to operate in a low power
-  (Discontinuous Conduction) mode during low loading conditions.
-
-  - cirrus,weak-fet-disable : Boolean : The strength of the output drivers is
-  reduced when operating in a Weak-FET Drive Mode and must not be used to drive
-  a large load.
-
-  - cirrus,classh-wk-fet-delay :  Weak-FET entry delay. Controls the delay
-  (in ms) before the Class H algorithm switches to the weak-FET voltage
-  (after the audio falls and remains below the value specified in WKFET_AMP_THLD).
-
-  0 = 0ms
-  1 = 5ms
-  2 = 10ms
-  3 = 50ms
-  4 = 100ms (Default)
-  5 = 200ms
-  6 = 500ms
-  7 = 1000ms
-
-  - cirrus,classh-weak-fet-thld-millivolt : Weak-FET amplifier drive threshold.
-  Configures the signal threshold at which the PWM output stage enters
-  weak-FET operation. The range is 50mV to 700mV in 50mV increments.
-
-  - cirrus,temp-warn-threshold :  Amplifier overtemperature warning threshold.
-  Configures the threshold at which the overtemperature warning condition occurs.
-  When the threshold is met, the overtemperature warning attenuation is applied
-  and the TEMP_WARN_EINT interrupt status bit is set.
-  If TEMP_WARN_MASK = 0, INTb is asserted.
-
-  0 = 105C
-  1 = 115C
-  2 = 125C (Default)
-  3 = 135C
-
-  - cirrus,irq-drive-select : Selects the driver type of the selected interrupt
-  output.
-
-  0 = Open-drain
-  1 = Push-pull (Default)
-
-  - cirrus,irq-gpio-select : Selects the pin to serve as the programmable
-  interrupt output.
-
-  0 = PDM_DATA / SWIRE_SD / INT (Default)
-  1 = GPIO
-
-Optional properties for the "cirrus,vpbr-config" Sub-node
-
-  - cirrus,vpbr-en : VBST brownout prevention enable. Configures whether the
-  VBST brownout prevention algorithm is enabled or disabled.
-
-  0 = VBST brownout prevention disabled (default)
-  1 = VBST brownout prevention enabled
-
-  See Section 7.31.1 VPBR Config for configuration options & further details
-
-  - cirrus,vpbr-thld : Initial VPBR threshold. Configures the VP brownout
-  threshold voltage
-
-  - cirrus,cirrus,vpbr-atk-rate : Attenuation attack step rate. Configures the
-  amount delay between consecutive volume attenuation steps when a brownout
-  condition is present and the VP brownout condition is in an attacking state.
-
-  - cirrus,vpbr-atk-vol : VP brownout prevention step size. Configures the VP
-  brownout prevention attacking attenuation step size when operating in either
-  digital volume or analog gain modes.
-
-  - cirrus,vpbr-max-attn : Maximum attenuation that the VP brownout prevention
-  can apply to the audio signal.
-
-  - cirrus,vpbr-wait : Configures the delay time between a brownout condition
-  no longer being present and the VP brownout prevention entering an attenuation
-  release state.
-
-  - cirrus,vpbr-rel-rate : Attenuation release step rate. Configures the delay
-  between consecutive volume attenuation release steps when a brownout condition
-  is not longer present and the VP brownout is in an attenuation release state.
-
-  - cirrus,vpbr-mute-en : During the attack state, if the vpbr-max-attn value
-  is reached, the error condition still remains, and this bit is set, the audio
-  is muted.
-
-Example:
-
-cs35l36: cs35l36@40 {
-	compatible = "cirrus,cs35l36";
-	reg = <0x40>;
-	VA-supply = <&dummy_vreg>;
-	VP-supply = <&dummy_vreg>;
-	reset-gpios = <&gpio0 54 0>;
-	interrupt-parent = <&gpio8>;
-	interrupts = <3 IRQ_TYPE_LEVEL_LOW>;
-
-	cirrus,boost-ind-nanohenry = <1000>;
-	cirrus,boost-ctl-millivolt = <10000>;
-	cirrus,boost-peak-milliamp = <4500>;
-	cirrus,boost-ctl-select = <0x00>;
-	cirrus,weak-fet-delay = <0x04>;
-	cirrus,weak-fet-thld = <0x01>;
-	cirrus,temp-warn-threshold = <0x01>;
-	cirrus,multi-amp-mode;
-	cirrus,irq-drive-select = <0x01>;
-	cirrus,irq-gpio-select = <0x01>;
-
-	cirrus,vpbr-config {
-		cirrus,vpbr-en = <0x00>;
-		cirrus,vpbr-thld = <0x05>;
-		cirrus,vpbr-atk-rate = <0x02>;
-		cirrus,vpbr-atk-vol = <0x01>;
-		cirrus,vpbr-max-attn = <0x09>;
-		cirrus,vpbr-wait = <0x01>;
-		cirrus,vpbr-rel-rate = <0x05>;
-		cirrus,vpbr-mute-en = <0x00>;
-	};
-};
diff --git a/Documentation/devicetree/bindings/sound/dmic-codec.yaml b/Documentation/devicetree/bindings/sound/dmic-codec.yaml
index cc3c84dd4c26f..83c23e029ea48 100644
--- a/Documentation/devicetree/bindings/sound/dmic-codec.yaml
+++ b/Documentation/devicetree/bindings/sound/dmic-codec.yaml
@@ -39,6 +39,10 @@ properties:
   wakeup-delay-ms:
     description: Delay (in ms) after enabling the DMIC
 
+  port:
+    $ref: audio-graph-port.yaml#
+    unevaluatedProperties: false
+
 required:
   - compatible
 
diff --git a/Documentation/devicetree/bindings/sound/fsl,micfil.yaml b/Documentation/devicetree/bindings/sound/fsl,micfil.yaml
index c47b7a0974906..4c7dadb310de3 100644
--- a/Documentation/devicetree/bindings/sound/fsl,micfil.yaml
+++ b/Documentation/devicetree/bindings/sound/fsl,micfil.yaml
@@ -66,6 +66,10 @@ properties:
   "#sound-dai-cells":
     const: 0
 
+  port:
+    $ref: audio-graph-port.yaml#
+    unevaluatedProperties: false
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/sound/loongson,ls-audio-card.yaml b/Documentation/devicetree/bindings/sound/loongson,ls-audio-card.yaml
index 61e8babed402f..dc7f4afbb7775 100644
--- a/Documentation/devicetree/bindings/sound/loongson,ls-audio-card.yaml
+++ b/Documentation/devicetree/bindings/sound/loongson,ls-audio-card.yaml
@@ -8,19 +8,22 @@ title: Loongson 7axxx/2kxxx ASoC audio sound card driver
 
 maintainers:
   - Yingkun Meng <mengyingkun@loongson.cn>
+  - Binbin Zhou <zhoubinbin@loongson.cn>
 
 description:
   The binding describes the sound card present in loongson
   7axxx/2kxxx platform. The sound card is an ASoC component
   which uses Loongson I2S controller to transfer the audio data.
 
+allOf:
+  - $ref: sound-card-common.yaml#
+
 properties:
   compatible:
-    const: loongson,ls-audio-card
-
-  model:
-    $ref: /schemas/types.yaml#/definitions/string
-    description: User specified audio sound card name
+    enum:
+      - loongson,ls-audio-card  # Loongson-2K1000/Loongson-2K2000/LS7A
+      - loongson,ls2k0300-forever-pi-audio-card # CTCISZ Forever Pi
+      - loongson,ls2k0300-dl2k0300b-audio-card # ATK-DL2K0300B
 
   mclk-fs:
     $ref: simple-card.yaml#/definitions/mclk-fs
@@ -45,14 +48,25 @@ properties:
     required:
       - sound-dai
 
+  spkr-en-gpios:
+    maxItems: 1
+    description: The GPIO that enables the speakers
+
+  hp-ctl-gpios:
+    maxItems: 1
+    description: The GPIO that control the headphones
+
+  hp-det-gpios:
+    maxItems: 1
+    description: The GPIO that detect headphones are plugged in
+
 required:
   - compatible
-  - model
   - mclk-fs
   - cpu
   - codec
 
-additionalProperties: false
+unevaluatedProperties: false
 
 examples:
   - |
@@ -68,3 +82,28 @@ examples:
              sound-dai = <&es8323>;
         };
     };
+
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    sound {
+        compatible = "loongson,ls2k0300-dl2k0300b-audio-card";
+        model = "loongson-audio";
+        mclk-fs = <512>;
+        hp-det-gpios = <&gpio 81 GPIO_ACTIVE_HIGH>;
+        spkr-en-gpios = <&gpio 86 GPIO_ACTIVE_HIGH>;
+        hp-ctl-gpios = <&gpio 87 GPIO_ACTIVE_HIGH>;
+        audio-routing =
+                      "Headphone", "LOUT1",
+                      "Headphone", "ROUT1",
+                      "Speaker", "LOUT2",
+                      "Speaker", "ROUT2";
+
+        cpu {
+            sound-dai = <&i2s>;
+        };
+
+        codec {
+             sound-dai = <&es8388>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/sound/loongson,ls2k1000-i2s.yaml b/Documentation/devicetree/bindings/sound/loongson,ls2k1000-i2s.yaml
index da79510bb2d91..51e23c189f7a1 100644
--- a/Documentation/devicetree/bindings/sound/loongson,ls2k1000-i2s.yaml
+++ b/Documentation/devicetree/bindings/sound/loongson,ls2k1000-i2s.yaml
@@ -14,9 +14,12 @@ allOf:
 
 properties:
   compatible:
-    const: loongson,ls2k1000-i2s
+    enum:
+      - loongson,ls2k0300-i2s
+      - loongson,ls2k1000-i2s
 
   reg:
+    minItems: 1
     items:
       - description: Loongson I2S controller Registers.
       - description: APB DMA config register for Loongson I2S controller.
@@ -49,6 +52,23 @@ required:
 
 unevaluatedProperties: false
 
+if:
+  properties:
+    compatible:
+      contains:
+        enum:
+          - loongson,ls2k1000-i2s
+
+then:
+  properties:
+    reg:
+      minItems: 2
+
+else:
+  properties:
+    reg:
+      maxItems: 1
+
 examples:
   - |
     #include <dt-bindings/clock/loongson,ls2k-clk.h>
diff --git a/Documentation/devicetree/bindings/sound/mediatek,mtk-btcvsd-snd.yaml b/Documentation/devicetree/bindings/sound/mediatek,mtk-btcvsd-snd.yaml
new file mode 100644
index 0000000000000..1b74516554767
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/mediatek,mtk-btcvsd-snd.yaml
@@ -0,0 +1,59 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/mediatek,mtk-btcvsd-snd.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Mediatek ALSA BT SCO CVSD/MSBC
+
+maintainers:
+  - Luca Leonardo Scorcia <l.scorcia@gmail.com>
+
+properties:
+  compatible:
+    const: mediatek,mtk-btcvsd-snd
+
+  reg:
+    items:
+      - description: PKV region
+      - description: SRAM_BANK2 region
+
+  interrupts:
+    items:
+      - description: BT-SCO interrupt
+
+  mediatek,infracfg:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description: The phandle of the infracfg controller
+
+  mediatek,offset:
+    $ref: /schemas/types.yaml#/definitions/uint32-array
+    description: Array of register offsets and masks
+    items:
+      - description: infra_misc_offset
+      - description: infra_conn_bt_cvsd_mask
+      - description: cvsd_mcu_read_offset
+      - description: cvsd_mcu_write_offset
+      - description: cvsd_packet_indicator_offset
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - mediatek,infracfg
+  - mediatek,offset
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    mtk-btcvsd-snd@18000000 {
+      compatible = "mediatek,mtk-btcvsd-snd";
+      reg = <0x18000000 0x1000>,
+            <0x18080000 0x8000>;
+      interrupts = <GIC_SPI 286 IRQ_TYPE_LEVEL_LOW>;
+      mediatek,infracfg = <&infrasys>;
+      mediatek,offset = <0xf00 0x800 0xfd0 0xfd4 0xfd8>;
+    };
diff --git a/Documentation/devicetree/bindings/sound/mtk-btcvsd-snd.txt b/Documentation/devicetree/bindings/sound/mtk-btcvsd-snd.txt
deleted file mode 100644
index 679e44839b48f..0000000000000
--- a/Documentation/devicetree/bindings/sound/mtk-btcvsd-snd.txt
+++ /dev/null
@@ -1,24 +0,0 @@
-Mediatek ALSA BT SCO CVSD/MSBC Driver
-
-Required properties:
-- compatible = "mediatek,mtk-btcvsd-snd";
-- reg: register location and size of PKV and SRAM_BANK2
-- interrupts: should contain BTSCO interrupt
-- mediatek,infracfg: the phandles of INFRASYS
-- mediatek,offset: Array contains of register offset and mask
-    infra_misc_offset,
-    infra_conn_bt_cvsd_mask,
-    cvsd_mcu_read_offset,
-    cvsd_mcu_write_offset,
-    cvsd_packet_indicator_offset
-
-Example:
-
-	mtk-btcvsd-snd@18000000 {
-		compatible = "mediatek,mtk-btcvsd-snd";
-		reg=<0 0x18000000 0 0x1000>,
-		    <0 0x18080000 0 0x8000>;
-		interrupts = <GIC_SPI 286 IRQ_TYPE_LEVEL_LOW>;
-		mediatek,infracfg = <&infrasys>;
-		mediatek,offset = <0xf00 0x800 0xfd0 0xfd4 0xfd8>;
-	};
diff --git a/Documentation/devicetree/bindings/sound/qcom,lpass-rx-macro.yaml b/Documentation/devicetree/bindings/sound/qcom,lpass-rx-macro.yaml
index 2eed2277511f8..4988e7ed6e34b 100644
--- a/Documentation/devicetree/bindings/sound/qcom,lpass-rx-macro.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,lpass-rx-macro.yaml
@@ -21,6 +21,7 @@ properties:
           - qcom,sc8280xp-lpass-rx-macro
       - items:
           - enum:
+              - qcom,eliza-lpass-rx-macro
               - qcom,kaanapali-lpass-rx-macro
               - qcom,sm8650-lpass-rx-macro
               - qcom,sm8750-lpass-rx-macro
diff --git a/Documentation/devicetree/bindings/sound/qcom,lpass-tx-macro.yaml b/Documentation/devicetree/bindings/sound/qcom,lpass-tx-macro.yaml
index e5e65e226a02d..d8682ff2e82c5 100644
--- a/Documentation/devicetree/bindings/sound/qcom,lpass-tx-macro.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,lpass-tx-macro.yaml
@@ -21,6 +21,7 @@ properties:
           - qcom,sc8280xp-lpass-tx-macro
       - items:
           - enum:
+              - qcom,eliza-lpass-tx-macro
               - qcom,kaanapali-lpass-tx-macro
               - qcom,sm8650-lpass-tx-macro
               - qcom,sm8750-lpass-tx-macro
diff --git a/Documentation/devicetree/bindings/sound/qcom,lpass-va-macro.yaml b/Documentation/devicetree/bindings/sound/qcom,lpass-va-macro.yaml
index 5c42b2b323ee4..aea31fbdad376 100644
--- a/Documentation/devicetree/bindings/sound/qcom,lpass-va-macro.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,lpass-va-macro.yaml
@@ -21,6 +21,7 @@ properties:
           - qcom,sc8280xp-lpass-va-macro
       - items:
           - enum:
+              - qcom,eliza-lpass-va-macro
               - qcom,glymur-lpass-va-macro
               - qcom,kaanapali-lpass-va-macro
               - qcom,sm8650-lpass-va-macro
diff --git a/Documentation/devicetree/bindings/sound/qcom,lpass-wsa-macro.yaml b/Documentation/devicetree/bindings/sound/qcom,lpass-wsa-macro.yaml
index d5f22b5cf0210..9fedd80532e27 100644
--- a/Documentation/devicetree/bindings/sound/qcom,lpass-wsa-macro.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,lpass-wsa-macro.yaml
@@ -20,6 +20,7 @@ properties:
           - qcom,sc8280xp-lpass-wsa-macro
       - items:
           - enum:
+              - qcom,eliza-lpass-wsa-macro
               - qcom,glymur-lpass-wsa-macro
               - qcom,kaanapali-lpass-wsa-macro
               - qcom,sm8650-lpass-wsa-macro
diff --git a/Documentation/devicetree/bindings/sound/qcom,q6asm-dais.yaml b/Documentation/devicetree/bindings/sound/qcom,q6asm-dais.yaml
index 47a105a97ecf9..bc8c8ba24f9cc 100644
--- a/Documentation/devicetree/bindings/sound/qcom,q6asm-dais.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,q6asm-dais.yaml
@@ -45,7 +45,7 @@ patternProperties:
         $ref: /schemas/types.yaml#/definitions/uint32
         enum: [0, 1, 2]
         description: |
-          The direction of the dai stream::
+          The direction of the dai stream:
            - Q6ASM_DAI_TX_RX (0) for both tx and rx
            - Q6ASM_DAI_TX (1) for only tx (Capture/Encode)
            - Q6ASM_DAI_RX (2) for only rx (Playback/Decode)
diff --git a/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml b/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml
index 15f38622b98b9..dae440ecab590 100644
--- a/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml
@@ -23,6 +23,7 @@ properties:
           - const: qcom,sdm845-sndcard
       - items:
           - enum:
+              - qcom,eliza-sndcard
               - qcom,kaanapali-sndcard
               - qcom,sm8550-sndcard
               - qcom,sm8650-sndcard
diff --git a/Documentation/devicetree/bindings/sound/wlf,wm8524.yaml b/Documentation/devicetree/bindings/sound/wlf,wm8524.yaml
index 4d951ece394e9..dffc41d5f7de1 100644
--- a/Documentation/devicetree/bindings/sound/wlf,wm8524.yaml
+++ b/Documentation/devicetree/bindings/sound/wlf,wm8524.yaml
@@ -24,6 +24,10 @@ properties:
     description:
       a GPIO spec for the MUTE pin.
 
+  port:
+    $ref: audio-graph-port.yaml#
+    unevaluatedProperties: false
+
 required:
   - compatible
   - wlf,mute-gpios
diff --git a/Documentation/devicetree/bindings/soundwire/qcom,soundwire.yaml b/Documentation/devicetree/bindings/soundwire/qcom,soundwire.yaml
index 9447a2f371b56..8e6973fa229c9 100644
--- a/Documentation/devicetree/bindings/soundwire/qcom,soundwire.yaml
+++ b/Documentation/devicetree/bindings/soundwire/qcom,soundwire.yaml
@@ -90,7 +90,7 @@ properties:
       or applicable for the respective data port.
       More info in MIPI Alliance SoundWire 1.0 Specifications.
     minItems: 3
-    maxItems: 16
+    maxItems: 17
 
   qcom,ports-sinterval-low:
     $ref: /schemas/types.yaml#/definitions/uint8-array
@@ -101,7 +101,7 @@ properties:
       or applicable for the respective data port.
       More info in MIPI Alliance SoundWire 1.0 Specifications.
     minItems: 3
-    maxItems: 16
+    maxItems: 17
 
   qcom,ports-sinterval:
     $ref: /schemas/types.yaml#/definitions/uint16-array
@@ -112,7 +112,7 @@ properties:
       or applicable for the respective data port.
       More info in MIPI Alliance SoundWire 1.0 Specifications.
     minItems: 3
-    maxItems: 16
+    maxItems: 17
 
   qcom,ports-offset1:
     $ref: /schemas/types.yaml#/definitions/uint8-array
@@ -123,7 +123,7 @@ properties:
       or applicable for the respective data port.
       More info in MIPI Alliance SoundWire 1.0 Specifications.
     minItems: 3
-    maxItems: 16
+    maxItems: 17
 
   qcom,ports-offset2:
     $ref: /schemas/types.yaml#/definitions/uint8-array
@@ -134,7 +134,7 @@ properties:
       or applicable for the respective data port.
       More info in MIPI Alliance SoundWire 1.0 Specifications.
     minItems: 3
-    maxItems: 16
+    maxItems: 17
 
   qcom,ports-lane-control:
     $ref: /schemas/types.yaml#/definitions/uint8-array
@@ -145,7 +145,7 @@ properties:
       or applicable for the respective data port.
       More info in MIPI Alliance SoundWire 1.0 Specifications.
     minItems: 3
-    maxItems: 16
+    maxItems: 17
 
   qcom,ports-block-pack-mode:
     $ref: /schemas/types.yaml#/definitions/uint8-array
@@ -158,7 +158,7 @@ properties:
       or applicable for the respective data port.
       More info in MIPI Alliance SoundWire 1.0 Specifications.
     minItems: 3
-    maxItems: 16
+    maxItems: 17
     items:
       oneOf:
         - minimum: 0
@@ -175,7 +175,7 @@ properties:
       or applicable for the respective data port.
       More info in MIPI Alliance SoundWire 1.0 Specifications.
     minItems: 3
-    maxItems: 16
+    maxItems: 17
     items:
       oneOf:
         - minimum: 0
@@ -192,7 +192,7 @@ properties:
       or applicable for the respective data port.
       More info in MIPI Alliance SoundWire 1.0 Specifications.
     minItems: 3
-    maxItems: 16
+    maxItems: 17
     items:
       oneOf:
         - minimum: 0
@@ -208,7 +208,7 @@ properties:
       or applicable for the respective data port.
       More info in MIPI Alliance SoundWire 1.0 Specifications.
     minItems: 3
-    maxItems: 16
+    maxItems: 17
     items:
       oneOf:
         - minimum: 0
diff --git a/Documentation/devicetree/bindings/spi/cavium,octeon-3010-spi.yaml b/Documentation/devicetree/bindings/spi/cavium,octeon-3010-spi.yaml
new file mode 100644
index 0000000000000..f0b708e1ccbbf
--- /dev/null
+++ b/Documentation/devicetree/bindings/spi/cavium,octeon-3010-spi.yaml
@@ -0,0 +1,61 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/spi/cavium,octeon-3010-spi.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Cavium, Inc. OCTEON SoC SPI master controller
+
+description:
+  The Cavium OCTEON SPI controller is an SPI master controller found in
+  OCTEON SoCs.
+
+maintainers:
+  - Rob Herring <robh@kernel.org>
+
+allOf:
+  - $ref: spi-controller.yaml#
+
+properties:
+  compatible:
+    const: cavium,octeon-3010-spi
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - interrupts
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    soc {
+        #address-cells = <2>;
+        #size-cells = <2>;
+
+        spi@1070000001000 {
+            compatible = "cavium,octeon-3010-spi";
+            reg = <0x10700 0x00001000 0x0 0x100>;
+            interrupts = <0 58>;
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            eeprom@0 {
+                compatible = "st,m95256", "atmel,at25";
+                reg = <0>;
+                spi-max-frequency = <5000000>;
+                spi-cpha;
+                spi-cpol;
+                pagesize = <64>;
+                size = <32768>;
+                address-width = <16>;
+            };
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/spi/qcom,sa8255p-geni-spi.yaml b/Documentation/devicetree/bindings/spi/qcom,sa8255p-geni-spi.yaml
new file mode 100644
index 0000000000000..d9347d780ca41
--- /dev/null
+++ b/Documentation/devicetree/bindings/spi/qcom,sa8255p-geni-spi.yaml
@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/spi/qcom,sa8255p-geni-spi.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: GENI based Qualcomm Universal Peripheral (QUP) Serial Peripheral Interface (SPI)
+
+maintainers:
+  - Praveen Talari <praveen.talari@oss.qualcomm.com>
+
+properties:
+  compatible:
+    const: qcom,sa8255p-geni-spi
+
+  reg:
+    maxItems: 1
+
+  dmas:
+    maxItems: 2
+
+  dma-names:
+    items:
+      - const: tx
+      - const: rx
+
+  interrupts:
+    maxItems: 1
+
+  power-domains:
+    maxItems: 2
+
+  power-domain-names:
+    items:
+      - const: power
+      - const: perf
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - power-domains
+  - power-domain-names
+
+allOf:
+  - $ref: /schemas/spi/spi-controller.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/dma/qcom-gpi.h>
+
+    spi@888000 {
+        compatible = "qcom,sa8255p-geni-spi";
+        reg = <0x888000 0x4000>;
+        interrupts = <GIC_SPI 584 IRQ_TYPE_LEVEL_HIGH>;
+        dmas = <&gpi_dma0 0 0 QCOM_GPI_SPI>,
+               <&gpi_dma0 1 0 QCOM_GPI_SPI>;
+        dma-names = "tx", "rx";
+        power-domains = <&scmi0_pd 0>, <&scmi0_dvfs 0>;
+        power-domain-names = "power", "perf";
+    };
diff --git a/Documentation/devicetree/bindings/spi/spi-octeon.txt b/Documentation/devicetree/bindings/spi/spi-octeon.txt
deleted file mode 100644
index 431add192342e..0000000000000
--- a/Documentation/devicetree/bindings/spi/spi-octeon.txt
+++ /dev/null
@@ -1,33 +0,0 @@
-Cavium, Inc. OCTEON SOC SPI master controller.
-
-Required properties:
-- compatible : "cavium,octeon-3010-spi"
-- reg : The register base for the controller.
-- interrupts : One interrupt, used by the controller.
-- #address-cells : <1>, as required by generic SPI binding.
-- #size-cells : <0>, also as required by generic SPI binding.
-
-Child nodes as per the generic SPI binding.
-
-Example:
-
-	spi@1070000001000 {
-		compatible = "cavium,octeon-3010-spi";
-		reg = <0x10700 0x00001000 0x0 0x100>;
-		interrupts = <0 58>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		eeprom@0 {
-			compatible = "st,m95256", "atmel,at25";
-			reg = <0>;
-			spi-max-frequency = <5000000>;
-			spi-cpha;
-			spi-cpol;
-
-			pagesize = <64>;
-			size = <32768>;
-			address-width = <16>;
-		};
-	};
-
diff --git a/Documentation/devicetree/bindings/spi/st,stm32-qspi.yaml b/Documentation/devicetree/bindings/spi/st,stm32-qspi.yaml
index 3f1a27efff80e..ee57739b73b8c 100644
--- a/Documentation/devicetree/bindings/spi/st,stm32-qspi.yaml
+++ b/Documentation/devicetree/bindings/spi/st,stm32-qspi.yaml
@@ -50,6 +50,9 @@ properties:
     minItems: 1
     maxItems: 2
 
+  power-domains:
+    maxItems: 1
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/sram/qcom,imem.yaml b/Documentation/devicetree/bindings/sram/qcom,imem.yaml
index c63026904061d..eaa8ef367ff48 100644
--- a/Documentation/devicetree/bindings/sram/qcom,imem.yaml
+++ b/Documentation/devicetree/bindings/sram/qcom,imem.yaml
@@ -89,6 +89,26 @@ required:
   - compatible
   - reg
 
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,apq8064-imem
+              - qcom,msm8226-imem
+              - qcom,msm8974-imem
+    then:
+      properties:
+        reboot-mode:
+          patternProperties:
+            # Negative look-ahead to disallow unsupported modes. The '$' has to be
+            # part of lookahead group to work, instead of trailing outside of ().
+            "^mode-(?!(bootloader$|normal$|recovery$))": false
+    else:
+      properties:
+        reboot-mode: false
+
 additionalProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/sram/sram.yaml b/Documentation/devicetree/bindings/sram/sram.yaml
index eb695698a03e4..8f9b335c5a9df 100644
--- a/Documentation/devicetree/bindings/sram/sram.yaml
+++ b/Documentation/devicetree/bindings/sram/sram.yaml
@@ -35,8 +35,10 @@ properties:
         - nvidia,tegra194-sysram
         - nvidia,tegra234-sysram
         - qcom,eliza-imem
+        - qcom,glymur-imem
         - qcom,hawi-imem
         - qcom,kaanapali-imem
+        - qcom,maili-imem
         - qcom,milos-imem
         - qcom,rpm-msg-ram
         - rockchip,rk3288-pmu-sram
diff --git a/Documentation/devicetree/bindings/thermal/samsung,exynos-thermal.yaml b/Documentation/devicetree/bindings/thermal/samsung,exynos-thermal.yaml
index 29a08b0729eee..3f333db72a711 100644
--- a/Documentation/devicetree/bindings/thermal/samsung,exynos-thermal.yaml
+++ b/Documentation/devicetree/bindings/thermal/samsung,exynos-thermal.yaml
@@ -40,7 +40,7 @@ properties:
     description: |
       The Exynos TMU supports generating interrupts when reaching given
       temperature thresholds. Number of supported thermal trip points depends
-      on the SoC (only first trip points defined in DT will be configured)::
+      on the SoC (only first trip points defined in DT will be configured):
        - most of SoC: 4
        - samsung,exynos5433-tmu: 8
        - samsung,exynos7-tmu: 8
@@ -52,7 +52,7 @@ properties:
       - description: |
           Shared TMU registers.
 
-          Note:: On Exynos5420, the TRIMINFO register is misplaced for TMU
+          Note: On Exynos5420, the TRIMINFO register is misplaced for TMU
           channels 2, 3 and 4 Use "samsung,exynos5420-tmu-ext-triminfo" in
           cases, there is a misplaced register, also provide clock to access
           that base.
diff --git a/Documentation/devicetree/bindings/timer/sifive,clint.yaml b/Documentation/devicetree/bindings/timer/sifive,clint.yaml
index 105e3ef539e6f..3673836e14dee 100644
--- a/Documentation/devicetree/bindings/timer/sifive,clint.yaml
+++ b/Documentation/devicetree/bindings/timer/sifive,clint.yaml
@@ -50,6 +50,7 @@ properties:
       - items:
           - enum:
               - allwinner,sun20i-d1-clint
+              - canaan,k230-clint
               - sophgo,cv1800b-clint
               - sophgo,cv1812h-clint
               - sophgo,sg2000-clint
diff --git a/Documentation/devicetree/bindings/ufs/ufs-common.yaml b/Documentation/devicetree/bindings/ufs/ufs-common.yaml
index ed97f56825093..cc32e1189d508 100644
--- a/Documentation/devicetree/bindings/ufs/ufs-common.yaml
+++ b/Documentation/devicetree/bindings/ufs/ufs-common.yaml
@@ -105,6 +105,64 @@ properties:
       Restricts the UFS controller to rate-a or rate-b for both TX and
       RX directions.
 
+  tx-precode-enable-g6:
+    $ref: /schemas/types.yaml#/definitions/uint32-matrix
+    minItems: 1
+    items:
+      - items:
+          - description: Host_Lane0 precode
+            enum: [0, 1]
+          - description: Device_Lane0 precode
+            enum: [0, 1]
+      - items:
+          - description: Host_Lane1 precode
+            enum: [0, 1]
+          - description: Device_Lane1 precode
+            enum: [0, 1]
+    description:
+      Static TX Precode enable values for HS-G6 only.
+
+patternProperties:
+  "^txeq-preshoot-g[1-6]$":
+    $ref: /schemas/types.yaml#/definitions/uint32-matrix
+    minItems: 1
+    items:
+      - items:
+          - description: Host_Lane0 Preshoot value
+            enum: [0, 1, 2, 3, 4, 5, 6, 7]
+          - description: Device_Lane0 Preshoot value
+            enum: [0, 1, 2, 3, 4, 5, 6, 7]
+      - items:
+          - description: Host_Lane1 Preshoot value
+            enum: [0, 1, 2, 3, 4, 5, 6, 7]
+          - description: Device_Lane1 Preshoot value
+            enum: [0, 1, 2, 3, 4, 5, 6, 7]
+    description: |
+      Static TX Equalization PreShoot settings for High Speed Gears. These
+      values are programmed to the corresponding UniPro PA layer attribute
+      PA_TxEQG[1-6]Setting. Each value selects a Pre-Shoot level as defined
+      by the MIPI M-PHY specification (TX_HS_PreShoot_Setting).
+
+  "^txeq-deemphasis-g[1-6]$":
+    $ref: /schemas/types.yaml#/definitions/uint32-matrix
+    minItems: 1
+    items:
+      - items:
+          - description: Host_Lane0 DeEmphasis value
+            enum: [0, 1, 2, 3, 4, 5, 6, 7]
+          - description: Device_Lane0 DeEmphasis value
+            enum: [0, 1, 2, 3, 4, 5, 6, 7]
+      - items:
+          - description: Host_Lane1 DeEmphasis value
+            enum: [0, 1, 2, 3, 4, 5, 6, 7]
+          - description: Device_Lane1 DeEmphasis value
+            enum: [0, 1, 2, 3, 4, 5, 6, 7]
+    description: |
+      Static TX Equalization DeEmphasis settings for High Speed Gears. These
+      values are programmed to the corresponding UniPro PA layer attribute
+      PA_TxEQG[1-6]Setting. Each value selects a De-Emphasis level as defined
+      by the MIPI M-PHY specification (TX_HS_DeEmphasis_Setting).
+
 dependencies:
   freq-table-hz: [ clocks ]
   operating-points-v2: [ clocks, clock-names ]
diff --git a/Documentation/devicetree/bindings/usb/generic-ehci.yaml b/Documentation/devicetree/bindings/usb/generic-ehci.yaml
index 55a5aa7d7a54e..ae9fb70d0212c 100644
--- a/Documentation/devicetree/bindings/usb/generic-ehci.yaml
+++ b/Documentation/devicetree/bindings/usb/generic-ehci.yaml
@@ -98,6 +98,10 @@ properties:
         - if a USB DRD channel: first clock should be host and second
           one should be peripheral
 
+  clock-names:
+    minItems: 1
+    maxItems: 4
+
   power-domains:
     maxItems: 1
 
@@ -186,6 +190,9 @@ allOf:
       required:
         - clocks
         - clock-names
+    else:
+      properties:
+        clock-names: false
 
 unevaluatedProperties: false
 
diff --git a/Documentation/devicetree/bindings/usb/generic-ohci.yaml b/Documentation/devicetree/bindings/usb/generic-ohci.yaml
index d42f448fa2044..322808aaa2839 100644
--- a/Documentation/devicetree/bindings/usb/generic-ohci.yaml
+++ b/Documentation/devicetree/bindings/usb/generic-ohci.yaml
@@ -83,6 +83,10 @@ properties:
         - if a USB DRD channel: first clock should be host and second
           one should be peripheral
 
+  clock-names:
+    minItems: 1
+    maxItems: 4
+
   power-domains:
     maxItems: 1
 
@@ -182,6 +186,7 @@ allOf:
 
     else:
       properties:
+        clock-names: false
         atmel,vbus-gpio: false
         atmel,oc-gpio: false
 
diff --git a/Documentation/devicetree/bindings/usb/mediatek,mtu3.yaml b/Documentation/devicetree/bindings/usb/mediatek,mtu3.yaml
index 21fc6bbe954f9..d148e938d6472 100644
--- a/Documentation/devicetree/bindings/usb/mediatek,mtu3.yaml
+++ b/Documentation/devicetree/bindings/usb/mediatek,mtu3.yaml
@@ -28,6 +28,7 @@ properties:
           - mediatek,mt8188-mtu3
           - mediatek,mt8192-mtu3
           - mediatek,mt8195-mtu3
+          - mediatek,mt8196-mtu3
           - mediatek,mt8365-mtu3
       - const: mediatek,mtu3
 
@@ -200,7 +201,10 @@ properties:
             103 - used by mt8195, IP0, specific 1.03;
             105 - used by mt8195, IP2, specific 1.05;
             106 - used by mt8195, IP3, specific 1.06;
-          enum: [1, 2, 101, 102, 103, 105, 106]
+            107 - used by mt8196, IP0, specific 1.07;
+            108 - used by mt8196, IP1, specific 1.08;
+            109 - used by mt8196, IP2, specific 1.09;
+            enum: [1, 2, 101, 102, 103, 105, 106, 107, 108, 109]
 
   mediatek,u3p-dis-msk:
     $ref: /schemas/types.yaml#/definitions/uint32
diff --git a/Documentation/devicetree/bindings/usb/qcom,dwc3.yaml b/Documentation/devicetree/bindings/usb/qcom,dwc3.yaml
index a7f58114c02e8..90daee6168809 100644
--- a/Documentation/devicetree/bindings/usb/qcom,dwc3.yaml
+++ b/Documentation/devicetree/bindings/usb/qcom,dwc3.yaml
@@ -92,14 +92,14 @@ properties:
 
   clocks:
     description: |
-      Several clocks are used, depending on the variant. Typical ones are::
-       - cfg_noc:: System Config NOC clock.
-       - core:: Master/Core clock, has to be >= 125 MHz for SS operation and >=
+      Several clocks are used, depending on the variant. Typical ones are:
+       - cfg_noc: System Config NOC clock.
+       - core: Master/Core clock, has to be >= 125 MHz for SS operation and >=
                 60MHz for HS operation.
-       - iface:: System bus AXI clock.
-       - sleep:: Sleep clock, used for wakeup when USB3 core goes into low
+       - iface: System bus AXI clock.
+       - sleep: Sleep clock, used for wakeup when USB3 core goes into low
                  power mode (U3).
-       - mock_utmi:: Mock utmi clock needed for ITP/SOF generation in host
+       - mock_utmi: Mock utmi clock needed for ITP/SOF generation in host
                      mode. Its frequency should be 19.2MHz.
     minItems: 1
     maxItems: 9
diff --git a/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml b/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml
index 8201656b41ed7..c9dc5f302c4c9 100644
--- a/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml
+++ b/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml
@@ -29,6 +29,7 @@ properties:
           - qcom,glymur-dwc3-mp
           - qcom,ipq4019-dwc3
           - qcom,ipq5018-dwc3
+          - qcom,ipq5210-dwc3
           - qcom,ipq5332-dwc3
           - qcom,ipq5424-dwc3
           - qcom,ipq6018-dwc3
@@ -87,14 +88,14 @@ properties:
 
   clocks:
     description: |
-      Several clocks are used, depending on the variant. Typical ones are::
-       - cfg_noc:: System Config NOC clock.
-       - core:: Master/Core clock, has to be >= 125 MHz for SS operation and >=
+      Several clocks are used, depending on the variant. Typical ones are:
+       - cfg_noc: System Config NOC clock.
+       - core: Master/Core clock, has to be >= 125 MHz for SS operation and >=
                 60MHz for HS operation.
-       - iface:: System bus AXI clock.
-       - sleep:: Sleep clock, used for wakeup when USB3 core goes into low
+       - iface: System bus AXI clock.
+       - sleep: Sleep clock, used for wakeup when USB3 core goes into low
                  power mode (U3).
-       - mock_utmi:: Mock utmi clock needed for ITP/SOF generation in host
+       - mock_utmi: Mock utmi clock needed for ITP/SOF generation in host
                      mode. Its frequency should be 19.2MHz.
     minItems: 1
     maxItems: 9
@@ -203,6 +204,7 @@ allOf:
         compatible:
           contains:
             enum:
+              - qcom,ipq5210-dwc3
               - qcom,ipq5424-dwc3
               - qcom,ipq9574-dwc3
               - qcom,kaanapali-dwc3
@@ -497,6 +499,7 @@ allOf:
         compatible:
           contains:
             enum:
+              - qcom,ipq5210-dwc3
               - qcom,ipq5424-dwc3
               - qcom,ipq9574-dwc3
     then:
diff --git a/Documentation/devicetree/bindings/usb/renesas,usbhs.yaml b/Documentation/devicetree/bindings/usb/renesas,usbhs.yaml
index dc74e70f1b925..13715b7c94c87 100644
--- a/Documentation/devicetree/bindings/usb/renesas,usbhs.yaml
+++ b/Documentation/devicetree/bindings/usb/renesas,usbhs.yaml
@@ -27,6 +27,7 @@ properties:
               - renesas,usbhs-r9a07g044 # RZ/G2{L,LC}
               - renesas,usbhs-r9a07g054 # RZ/V2L
               - renesas,usbhs-r9a08g045 # RZ/G3S
+              - renesas,usbhs-r9a08g046 # RZ/G3L
               - renesas,usbhs-r9a09g047 # RZ/G3E
               - renesas,usbhs-r9a09g056 # RZ/V2N
               - renesas,usbhs-r9a09g057 # RZ/V2H(P)
diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml
index 396044f368e7c..66bf58a93910d 100644
--- a/Documentation/devicetree/bindings/vendor-prefixes.yaml
+++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml
@@ -88,6 +88,8 @@ patternProperties:
     description: ALFA Network Inc.
   "^algoltek,.*":
     description: AlgolTek, Inc.
+  "^alientek,.*":
+    description: Guangzhou Xingyi Intelligent Technology Co., Ltd.
   "^allegro,.*":
     description: Allegro DVT
   "^allegromicro,.*":
@@ -225,10 +227,14 @@ patternProperties:
     description: Axis Communications AB
   "^ayaneo,.*":
     description: Anyun Intelligent Technology (Hong Kong) Co., Ltd
+  "^ayntec,.*":
+    description: AYN Technologies Co., Ltd.
   "^azoteq,.*":
     description: Azoteq (Pty) Ltd
   "^azw,.*":
     description: Shenzhen AZW Technology Co., Ltd.
+  "^baijie,.*":
+    description: Shenzhen Baijie Technology Co., Ltd.
   "^baikal,.*":
     description: BAIKAL ELECTRONICS, JSC
   "^bananapi,.*":
@@ -604,6 +610,8 @@ patternProperties:
     description: Flipkart Inc.
   "^focaltech,.*":
     description: FocalTech Systems Co.,Ltd
+  "^focuslcds,.*":
+    description: Focus Display Solutions, Inc.
   "^forlinx,.*":
     description: Baoding Forlinx Embedded Technology Co., Ltd.
   "^foursemi,.*":
@@ -682,6 +690,8 @@ patternProperties:
     description: Goramo Gorecki
   "^gplus,.*":
     description: GPLUS
+  "^graperain,.*":
+    description: Shenzhen Graperain Technology Co., Ltd.
   "^grinn,.*":
     description: Grinn
   "^grmn,.*":
@@ -735,6 +745,8 @@ patternProperties:
     description: Honestar Technologies Co., Ltd.
   "^honeywell,.*":
     description: Honeywell
+  "^honor,.*":
+    description: Honor Device Co., Ltd.
   "^hoperf,.*":
     description: Shenzhen Hope Microelectronics Co., Ltd.
   "^hoperun,.*":
@@ -1363,6 +1375,8 @@ patternProperties:
     description: Shenzhen QiShenglong Industrialist Co., Ltd.
   "^qnap,.*":
     description: QNAP Systems, Inc.
+  "^qstcorp,.*":
+    description: QST Corporation Limited
   "^quanta,.*":
     description: Quanta Computer Inc.
   "^radxa,.*":
@@ -1411,6 +1425,8 @@ patternProperties:
     description: Embest RIoT
   "^riscv,.*":
     description: RISC-V Foundation
+  "^riverdi,.*":
+    description: Riverdi Sp. z o.o
   "^rockchip,.*":
     description: Rockchip Electronics Co., Ltd.
   "^rocktech,.*":
@@ -1801,6 +1817,8 @@ patternProperties:
     description: VIA Technologies, Inc.
   "^vialab,.*":
     description: VIA Labs, Inc.
+  "^vicharak,.*":
+    description: Vicharak Computers Pvt. Ltd.
   "^vicor,.*":
     description: Vicor Corporation
   "^videostrong,.*":
@@ -1925,6 +1943,8 @@ patternProperties:
     description: YSH & ATIL
   "^yones-toptech,.*":
     description: Yones Toptech Co., Ltd.
+  "^youyeetoo,.*":
+    description: Hong Kong Cybodev Tech Limited (youyeetoo)
   "^ys,.*":
     description: Shenzhen Yashi Changhua Intelligent Technology Co., Ltd.
   "^ysoft,.*":
diff --git a/Documentation/devicetree/bindings/w1/omap-hdq.txt b/Documentation/devicetree/bindings/w1/omap-hdq.txt
deleted file mode 100644
index 913c5f91a0f9f..0000000000000
--- a/Documentation/devicetree/bindings/w1/omap-hdq.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-* OMAP HDQ One wire bus master controller
-
-Required properties:
-- compatible : should be "ti,omap3-1w" or "ti,am4372-hdq"
-- reg : Address and length of the register set for the device
-- interrupts : interrupt line.
-- ti,hwmods : "hdq1w"
-
-Optional properties:
-- ti,mode: should be "hdq": HDQ mode  "1w": one-wire mode.
-	   If not specified HDQ mode is implied.
-
-Example:
-
-- From omap3.dtsi
-  hdqw1w: 1w@480b2000 {
-	compatible = "ti,omap3-1w";
-	reg = <0x480b2000 0x1000>;
-	interrupts = <58>;
-	ti,hwmods = "hdq1w";
-	ti,mode = "hdq";
-  };
diff --git a/Documentation/devicetree/bindings/w1/ti,hdq.yaml b/Documentation/devicetree/bindings/w1/ti,hdq.yaml
new file mode 100644
index 0000000000000..fd402c0510db3
--- /dev/null
+++ b/Documentation/devicetree/bindings/w1/ti,hdq.yaml
@@ -0,0 +1,70 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/w1/ti,hdq.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments OMAP HDQ One Wire Bus Master Controller
+
+maintainers:
+  - Eduard Bostina <egbostina@gmail.com>
+
+properties:
+  compatible:
+    enum:
+      - ti,omap3-1w
+      - ti,am4372-hdq
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  clock-names:
+    items:
+      - const: fck
+
+  ti,hwmods:
+    description: Name of the hwmod associated to the hdq
+    $ref: /schemas/types.yaml#/definitions/string
+    const: hdq1w
+
+  ti,mode:
+    description: |
+      Operation mode. "hdq" for HDQ mode, "1w" for One-Wire mode.
+      If not specified, HDQ mode is implied.
+    $ref: /schemas/types.yaml#/definitions/string
+    enum:
+      - hdq
+      - 1w
+    default: hdq
+
+required:
+  - compatible
+  - reg
+  - interrupts
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    onewire@0 {
+        compatible = "ti,am4372-hdq";
+        reg = <0x0 0x1000>;
+        interrupts = <GIC_SPI 139 IRQ_TYPE_LEVEL_HIGH>;
+        clocks = <&func_12m_clk>;
+        clock-names = "fck";
+    };
+  - |
+    onewire@480b2000 {
+        compatible = "ti,omap3-1w";
+        reg = <0x480b2000 0x1000>;
+        interrupts = <58>;
+        ti,hwmods = "hdq1w";
+        ti,mode = "hdq";
+    };
diff --git a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml
index 845b5e8b5abc3..ca4cf01db59f7 100644
--- a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml
+++ b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml
@@ -18,7 +18,10 @@ properties:
       - items:
           - enum:
               - apple,t6020-wdt
+              - apple,t6030-wdt
+              - apple,t6031-wdt
               - apple,t8122-wdt
+              - apple,t8132-wdt
           - const: apple,t8103-wdt
       - items:
           - enum:
diff --git a/Documentation/devicetree/bindings/watchdog/microchip,pic32-dmt.txt b/Documentation/devicetree/bindings/watchdog/microchip,pic32-dmt.txt
deleted file mode 100644
index 49485f8313739..0000000000000
--- a/Documentation/devicetree/bindings/watchdog/microchip,pic32-dmt.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-* Microchip PIC32 Deadman Timer
-
-The deadman timer is used to reset the processor in the event of a software
-malfunction. It is a free-running instruction fetch timer, which is clocked
-whenever an instruction fetch occurs until a count match occurs.
-
-Required properties:
-- compatible: must be "microchip,pic32mzda-dmt".
-- reg: physical base address of the controller and length of memory mapped
-  region.
-- clocks: phandle of source clk. Should be <&rootclk PB7CLK>.
-
-Example:
-
-	watchdog@1f800a00 {
-		compatible = "microchip,pic32mzda-dmt";
-		reg = <0x1f800a00 0x80>;
-		clocks = <&rootclk PB7CLK>;
-	};
diff --git a/Documentation/devicetree/bindings/watchdog/microchip,pic32-wdt.txt b/Documentation/devicetree/bindings/watchdog/microchip,pic32-wdt.txt
deleted file mode 100644
index f03a29a1b3239..0000000000000
--- a/Documentation/devicetree/bindings/watchdog/microchip,pic32-wdt.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-* Microchip PIC32 Watchdog Timer
-
-When enabled, the watchdog peripheral can be used to reset the device if the
-WDT is not cleared periodically in software.
-
-Required properties:
-- compatible: must be "microchip,pic32mzda-wdt".
-- reg: physical base address of the controller and length of memory mapped
-  region.
-- clocks: phandle of source clk. Should be <&rootclk LPRCCLK>.
-
-Example:
-
-	watchdog@1f800800 {
-		compatible = "microchip,pic32mzda-wdt";
-		reg = <0x1f800800 0x200>;
-		clocks = <&rootclk LPRCCLK>;
-	};
diff --git a/Documentation/devicetree/bindings/watchdog/microchip,pic32mzda-dmt.yaml b/Documentation/devicetree/bindings/watchdog/microchip,pic32mzda-dmt.yaml
new file mode 100644
index 0000000000000..01638624e3340
--- /dev/null
+++ b/Documentation/devicetree/bindings/watchdog/microchip,pic32mzda-dmt.yaml
@@ -0,0 +1,45 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/watchdog/microchip,pic32mzda-dmt.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Microchip PIC32MZDA Deadman Timer
+
+maintainers:
+  - Thomas Bogendoerfer <tsbogend@alpha.franken.de>
+
+description: |
+  The deadman timer is used to reset the processor in the event of a software
+  malfunction. It is a free-running instruction fetch timer, which is clocked
+  whenever an instruction fetch occurs until a count match occurs.
+
+allOf:
+  - $ref: watchdog.yaml#
+
+properties:
+  compatible:
+    const: microchip,pic32mzda-dmt
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - clocks
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/microchip,pic32-clock.h>
+
+    watchdog@1f800a00 {
+        compatible = "microchip,pic32mzda-dmt";
+        reg = <0x1f800a00 0x80>;
+        clocks = <&rootclk PB7CLK>;
+       };
diff --git a/Documentation/devicetree/bindings/watchdog/microchip,pic32mzda-wdt.yaml b/Documentation/devicetree/bindings/watchdog/microchip,pic32mzda-wdt.yaml
new file mode 100644
index 0000000000000..a5dd633c3e782
--- /dev/null
+++ b/Documentation/devicetree/bindings/watchdog/microchip,pic32mzda-wdt.yaml
@@ -0,0 +1,44 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/watchdog/microchip,pic32mzda-wdt.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Microchip PIC32MZDA Watchdog Timer
+
+maintainers:
+  - Thomas Bogendoerfer <tsbogend@alpha.franken.de>
+
+description:
+  The PIC32 watchdog timer can be used to reset the device if software fails
+  to periodically service the watchdog.
+
+allOf:
+  - $ref: watchdog.yaml#
+
+properties:
+  compatible:
+    const: microchip,pic32mzda-wdt
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - clocks
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/microchip,pic32-clock.h>
+
+    watchdog@1f800800 {
+        compatible = "microchip,pic32mzda-wdt";
+        reg = <0x1f800800 0x200>;
+        clocks = <&rootclk LPRCCLK>;
+    };
diff --git a/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml b/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml
index 74117f5726a77..4ff61102e407e 100644
--- a/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml
+++ b/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml
@@ -27,6 +27,7 @@ properties:
               - qcom,apss-wdt-ipq9574
               - qcom,apss-wdt-ipq9650
               - qcom,apss-wdt-kaanapali
+              - qcom,apss-wdt-maili
               - qcom,apss-wdt-msm8226
               - qcom,apss-wdt-msm8974
               - qcom,apss-wdt-msm8994
diff --git a/Documentation/devicetree/bindings/xilinx.txt b/Documentation/devicetree/bindings/xilinx.txt
index 0ee9de99b3aed..a1a2cde7b7d8c 100644
--- a/Documentation/devicetree/bindings/xilinx.txt
+++ b/Documentation/devicetree/bindings/xilinx.txt
@@ -1,93 +1,4 @@
-   d) Xilinx IP cores
-
-   The Xilinx EDK toolchain ships with a set of IP cores (devices) for use
-   in Xilinx Spartan and Virtex FPGAs.  The devices cover the whole range
-   of standard device types (network, serial, etc.) and miscellaneous
-   devices (gpio, LCD, spi, etc).  Also, since these devices are
-   implemented within the fpga fabric every instance of the device can be
-   synthesised with different options that change the behaviour.
-
-   Each IP-core has a set of parameters which the FPGA designer can use to
-   control how the core is synthesized.  Historically, the EDK tool would
-   extract the device parameters relevant to device drivers and copy them
-   into an 'xparameters.h' in the form of #define symbols.  This tells the
-   device drivers how the IP cores are configured, but it requires the kernel
-   to be recompiled every time the FPGA bitstream is resynthesized.
-
-   The new approach is to export the parameters into the device tree and
-   generate a new device tree each time the FPGA bitstream changes.  The
-   parameters which used to be exported as #defines will now become
-   properties of the device node.  In general, device nodes for IP-cores
-   will take the following form:
-
-	(name): (generic-name)@(base-address) {
-		compatible = "xlnx,(ip-core-name)-(HW_VER)"
-			     [, (list of compatible devices), ...];
-		reg = <(baseaddr) (size)>;
-		interrupt-parent = <&interrupt-controller-phandle>;
-		interrupts = < ... >;
-		xlnx,(parameter1) = "(string-value)";
-		xlnx,(parameter2) = <(int-value)>;
-	};
-
-	(generic-name):   an open firmware-style name that describes the
-			generic class of device.  Preferably, this is one word, such
-			as 'serial' or 'ethernet'.
-	(ip-core-name):	the name of the ip block (given after the BEGIN
-			directive in system.mhs).  Should be in lowercase
-			and all underscores '_' converted to dashes '-'.
-	(name):		is derived from the "PARAMETER INSTANCE" value.
-	(parameter#):	C_* parameters from system.mhs.  The C_ prefix is
-			dropped from the parameter name, the name is converted
-			to lowercase and all underscore '_' characters are
-			converted to dashes '-'.
-	(baseaddr):	the baseaddr parameter value (often named C_BASEADDR).
-	(HW_VER):	from the HW_VER parameter.
-	(size):		the address range size (often C_HIGHADDR - C_BASEADDR + 1).
-
-   Typically, the compatible list will include the exact IP core version
-   followed by an older IP core version which implements the same
-   interface or any other device with the same interface.
-
-   'reg' and 'interrupts' are all optional properties.
-
-   For example, the following block from system.mhs:
-
-	BEGIN opb_uartlite
-		PARAMETER INSTANCE = opb_uartlite_0
-		PARAMETER HW_VER = 1.00.b
-		PARAMETER C_BAUDRATE = 115200
-		PARAMETER C_DATA_BITS = 8
-		PARAMETER C_ODD_PARITY = 0
-		PARAMETER C_USE_PARITY = 0
-		PARAMETER C_CLK_FREQ = 50000000
-		PARAMETER C_BASEADDR = 0xEC100000
-		PARAMETER C_HIGHADDR = 0xEC10FFFF
-		BUS_INTERFACE SOPB = opb_7
-		PORT OPB_Clk = CLK_50MHz
-		PORT Interrupt = opb_uartlite_0_Interrupt
-		PORT RX = opb_uartlite_0_RX
-		PORT TX = opb_uartlite_0_TX
-		PORT OPB_Rst = sys_bus_reset_0
-	END
-
-   becomes the following device tree node:
-
-	opb_uartlite_0: serial@ec100000 {
-		device_type = "serial";
-		compatible = "xlnx,opb-uartlite-1.00.b";
-		reg = <ec100000 10000>;
-		interrupt-parent = <&opb_intc_0>;
-		interrupts = <1 0>; // got this from the opb_intc parameters
-		current-speed = <d#115200>;	// standard serial device prop
-		clock-frequency = <d#50000000>;	// standard serial device prop
-		xlnx,data-bits = <8>;
-		xlnx,odd-parity = <0>;
-		xlnx,use-parity = <0>;
-	};
-
-   That covers the general approach to binding xilinx IP cores into the
-   device tree.  The following are bindings for specific devices:
+   Bindings for specific devices:
 
       i) Xilinx ML300 Framebuffer
 
@@ -102,13 +13,6 @@
                                            Default is <d#1024 d#480>.
        - rotate-display (empty) : rotate display 180 degrees.
 
-      iii) Xilinx EMAC and Xilinx TEMAC
-
-      Xilinx Ethernet devices.  In addition to general xilinx properties
-      listed above, nodes for these devices should include a phy-handle
-      property, and may include other common network device properties
-      like local-mac-address.
-
       v) Xilinx hwicap
 
 		Xilinx hwicap devices provide access to the configuration logic
diff --git a/Documentation/driver-api/acpi/acpi-drivers.rst b/Documentation/driver-api/acpi/acpi-drivers.rst
deleted file mode 100644
index 376b6d8a678cf..0000000000000
--- a/Documentation/driver-api/acpi/acpi-drivers.rst
+++ /dev/null
@@ -1,80 +0,0 @@
-.. SPDX-License-Identifier: GPL-2.0
-.. include:: <isonum.txt>
-
-=========================================
-Why using ACPI drivers is not a good idea
-=========================================
-
-:Copyright: |copy| 2026, Intel Corporation
-
-:Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
-
-Even though binding drivers directly to struct acpi_device objects, also
-referred to as "ACPI device nodes", allows basic functionality to be provided
-at least in some cases, there are problems with it, related to general
-consistency, sysfs layout, power management operation ordering, and code
-cleanliness.
-
-First of all, ACPI device nodes represent firmware entities rather than
-hardware and in many cases they provide auxiliary information on devices
-enumerated independently (like PCI devices or CPUs).  It is therefore generally
-questionable to assign resources to them because the entities represented by
-them do not decode addresses in the memory or I/O address spaces and do not
-generate interrupts or similar (all of that is done by hardware).
-
-Second, as a general rule, a struct acpi_device can only be a parent of another
-struct acpi_device.  If that is not the case, the location of the child device
-in the device hierarchy is at least confusing and it may not be straightforward
-to identify the piece of hardware providing functionality represented by it.
-However, binding a driver directly to an ACPI device node may cause that to
-happen if the given driver registers input devices or wakeup sources under it,
-for example.
-
-Next, using system suspend and resume callbacks directly on ACPI device nodes
-is also questionable because it may cause ordering problems to appear.  Namely,
-ACPI device nodes are registered before enumerating hardware corresponding to
-them and they land on the PM list in front of the majority of other device
-objects.  Consequently, the execution ordering of their PM callbacks may be
-different from what is generally expected.  Also, in general, dependencies
-returned by _DEP objects do not affect ACPI device nodes themselves, but the
-"physical" devices associated with them, which potentially is one more source
-of inconsistency related to treating ACPI device nodes as "real" device
-representation.
-
-All of the above means that binding drivers to ACPI device nodes should
-generally be avoided and so struct acpi_driver objects should not be used.
-
-Moreover, a device ID is necessary to bind a driver directly to an ACPI device
-node, but device IDs are not generally associated with all of them.  Some of
-them contain alternative information allowing the corresponding pieces of
-hardware to be identified, for example represented by an _ADR object return
-value, and device IDs are not used in those cases.  In consequence, confusingly
-enough, binding an ACPI driver to an ACPI device node may even be impossible.
-
-When that happens, the piece of hardware corresponding to the given ACPI device
-node is represented by another device object, like a struct pci_dev, and the
-ACPI device node is the "ACPI companion" of that device, accessible through its
-fwnode pointer used by the ACPI_COMPANION() macro.  The ACPI companion holds
-additional information on the device configuration and possibly some "recipes"
-on device manipulation in the form of AML (ACPI Machine Language) bytecode
-provided by the platform firmware.  Thus the role of the ACPI device node is
-similar to the role of a struct device_node on a system where Device Tree is
-used for platform description.
-
-For consistency, this approach has been extended to the cases in which ACPI
-device IDs are used.  Namely, in those cases, an additional device object is
-created to represent the piece of hardware corresponding to a given ACPI device
-node.  By default, it is a platform device, but it may also be a PNP device, a
-CPU device, or another type of device, depending on what the given piece of
-hardware actually is.  There are even cases in which multiple devices are
-"backed" or "accompanied" by one ACPI device node (e.g. ACPI device nodes
-corresponding to GPUs that may provide firmware interfaces for backlight
-brightness control in addition to GPU configuration information).
-
-This means that it really should never be necessary to bind a driver directly to
-an ACPI device node because there is a "proper" device object representing the
-corresponding piece of hardware that can be bound to by a "proper" driver using
-the given ACPI device node as the device's ACPI companion.  Thus, in principle,
-there is no reason to use ACPI drivers and if they all were replaced with other
-driver types (for example, platform drivers), some code could be dropped and
-some complexity would go away.
diff --git a/Documentation/driver-api/acpi/index.rst b/Documentation/driver-api/acpi/index.rst
index 2b10d83f9994b..ace0008e54c2f 100644
--- a/Documentation/driver-api/acpi/index.rst
+++ b/Documentation/driver-api/acpi/index.rst
@@ -7,4 +7,3 @@ ACPI Support
 
    linuxized-acpica
    scan_handlers
-   acpi-drivers
diff --git a/Documentation/driver-api/basics.rst b/Documentation/driver-api/basics.rst
index 8b6a5888cb11d..3b182cfdf1350 100644
--- a/Documentation/driver-api/basics.rst
+++ b/Documentation/driver-api/basics.rst
@@ -90,6 +90,9 @@ Reference counting
 .. kernel-doc:: lib/refcount.c
    :export:
 
+.. kernel-doc:: include/linux/percpu-refcount.h
+.. kernel-doc:: lib/percpu-refcount.c
+
 Atomics
 -------
 
diff --git a/Documentation/driver-api/cxl/linux/dax-driver.rst b/Documentation/driver-api/cxl/linux/dax-driver.rst
index 10d953a2167b0..72c85a0f8606f 100644
--- a/Documentation/driver-api/cxl/linux/dax-driver.rst
+++ b/Documentation/driver-api/cxl/linux/dax-driver.rst
@@ -35,9 +35,9 @@ will be exposed to the kernel page allocator in the user-selected memory
 zone.
 
 The :code:`memmap_on_memory` setting (both global and DAX device local)
-dictates where the kernell will allocate the :code:`struct folio` descriptors
+dictates where the kernel will allocate the :code:`struct folio` descriptors
 for this memory will come from.  If :code:`memmap_on_memory` is set, memory
 hotplug will set aside a portion of the memory block capacity to allocate
 folios. If unset, the memory is allocated via a normal :code:`GFP_KERNEL`
-allocation - and as a result will most likely land on the local NUM node of the
+allocation - and as a result will most likely land on the local NUMA node of the
 CPU executing the hotplug operation.
diff --git a/Documentation/driver-api/dmaengine/client.rst b/Documentation/driver-api/dmaengine/client.rst
index d491e385d61a9..5ee5d4a3596dd 100644
--- a/Documentation/driver-api/dmaengine/client.rst
+++ b/Documentation/driver-api/dmaengine/client.rst
@@ -80,6 +80,10 @@ The details of these operations are:
 
   - slave_sg: DMA a list of scatter gather buffers from/to a peripheral
 
+  - config_sg: Similar with slave_sg, just pass down dma_slave_config
+    struct to avoid calling dmaengine_slave_config() every time adjusting the
+    burst length or the FIFO address is needed.
+
   - peripheral_dma_vec: DMA an array of scatter gather buffers from/to a
     peripheral. Similar to slave_sg, but uses an array of dma_vec
     structures instead of a scatterlist.
@@ -106,6 +110,11 @@ The details of these operations are:
 		unsigned int sg_len, enum dma_data_direction direction,
 		unsigned long flags);
 
+     struct dma_async_tx_descriptor *dmaengine_prep_config_sg(
+		struct dma_chan *chan, struct scatterlist *sgl,
+		unsigned int sg_len, enum dma_transfer_direction dir,
+		unsigned long flags, struct dma_slave_config *config);
+
      struct dma_async_tx_descriptor *dmaengine_prep_peripheral_dma_vec(
 		struct dma_chan *chan, const struct dma_vec *vecs,
 		size_t nents, enum dma_data_direction direction,
diff --git a/Documentation/driver-api/dpll.rst b/Documentation/driver-api/dpll.rst
index bae14766d4f7b..f83150917814e 100644
--- a/Documentation/driver-api/dpll.rst
+++ b/Documentation/driver-api/dpll.rst
@@ -91,6 +91,13 @@ following pin states:
 - ``DPLL_PIN_STATE_DISCONNECTED`` - the pin shall be not considered as
   a valid input for automatic selection algorithm
 
+Pins that have the ``DPLL_PIN_CAPABILITIES_STATE_CONNECTED_OVERRIDE``
+capability can additionally be set to ``DPLL_PIN_STATE_CONNECTED`` in
+automatic mode, overriding the active input selection. This is useful
+for automatic-only DPLL devices where mode cannot be switched to manual.
+When such a pin is disconnected, the device returns to automatic input
+selection.
+
 The actual hardware status of a pin is reported via the operational
 state (``DPLL_A_PIN_OPERSTATE``) attribute nested under the parent
 device:
diff --git a/Documentation/driver-api/media/maintainer-entry-profile.rst b/Documentation/driver-api/media/maintainer-entry-profile.rst
index c5c00c66d85c7..7806d83128d4b 100644
--- a/Documentation/driver-api/media/maintainer-entry-profile.rst
+++ b/Documentation/driver-api/media/maintainer-entry-profile.rst
@@ -432,6 +432,10 @@ The Media Driver Maintainers responsible for specific areas are:
 
     - Qualcomm drivers
 
+  - Ricardo Ribalda <ribalda@chromium.org>
+
+    - Media-CI
+
 Submit Checklist Addendum
 -------------------------
 
diff --git a/Documentation/driver-api/pin-control.rst b/Documentation/driver-api/pin-control.rst
index 1f585ecca63c8..80106e44a368e 100644
--- a/Documentation/driver-api/pin-control.rst
+++ b/Documentation/driver-api/pin-control.rst
@@ -1175,7 +1175,7 @@ Possible standard state names are: "default", "init", "sleep" and "idle".
   selected after the driver probe.
 
 - the ``sleep`` and ``idle`` states are for power management and can only
-  be selected with the PM API bellow.
+  be selected with the PM API below.
 
 PM interfaces
 =================
diff --git a/Documentation/driver-api/usb/writing_usb_driver.rst b/Documentation/driver-api/usb/writing_usb_driver.rst
index 95c4f5d14052a..6f024e1647cdb 100644
--- a/Documentation/driver-api/usb/writing_usb_driver.rst
+++ b/Documentation/driver-api/usb/writing_usb_driver.rst
@@ -322,7 +322,4 @@ http://linux-hotplug.sourceforge.net/
 linux-usb Mailing List Archives:
 https://lore.kernel.org/linux-usb/
 
-Programming Guide for Linux USB Device Drivers:
-https://lmu.web.psi.ch/docu/manuals/software_manuals/linux_sl/usb_linux_programming_guide.pdf
-
 USB Home Page: https://www.usb.org
diff --git a/Documentation/driver-api/vfio.rst b/Documentation/driver-api/vfio.rst
index 2a21a42c93868..07192386f7a64 100644
--- a/Documentation/driver-api/vfio.rst
+++ b/Documentation/driver-api/vfio.rst
@@ -275,8 +275,6 @@ in a VFIO group.
 With CONFIG_VFIO_DEVICE_CDEV=y the user can now acquire a device fd
 by directly opening a character device /dev/vfio/devices/vfioX where
 "X" is the number allocated uniquely by VFIO for registered devices.
-cdev interface does not support noiommu devices, so user should use
-the legacy group interface if noiommu is wanted.
 
 The cdev only works with IOMMUFD.  Both VFIO drivers and applications
 must adapt to the new cdev security model which requires using
@@ -370,6 +368,93 @@ IOMMUFD IOAS/HWPT to enable userspace DMA::
 
 	/* Other device operations as stated in "VFIO Usage Example" */
 
+VFIO NOIOMMU mode
+-------------------------------------------------------------------------------
+VFIO also supports a no-IOMMU mode, intended for usages where unsafe DMA can
+be performed by userspace drivers w/o physical IOMMU protection. This mode
+is controlled by the parameter:
+
+/sys/module/vfio/parameters/enable_unsafe_noiommu_mode
+
+Upon enabling this mode, with an assigned device, the user will be presented
+with a VFIO group and device file, e.g.::
+
+  /dev/vfio/
+  |-- devices
+  |   `-- noiommu-vfioX	/* VFIO device cdev */
+  |-- noiommu-Y		/* VFIO group */
+  `-- vfio
+
+The capabilities vary depending on the device programming interface and kernel
+configuration used. The following table summarizes the differences ("Yes" means
+the UAPI is accessible and functional in noiommu mode, "No" means the UAPI is
+not supported):
+
++-------------------+---------------------+----------------------+
+| Feature           | VFIO group          | VFIO device cdev     |
++===================+=====================+======================+
+| VFIO device UAPI  | Yes                 | Yes                  |
++-------------------+---------------------+----------------------+
+| VFIO container    | No                  | No                   |
++-------------------+---------------------+----------------------+
+| IOMMUFD IOAS      | No                  | Yes*                 |
++-------------------+---------------------+----------------------+
+
+Note that the VFIO container case includes IOMMUFD provided VFIO compatibility
+interfaces when either CONFIG_VFIO_CONTAINER or CONFIG_IOMMUFD_VFIO_CONTAINER is
+enabled.
+
+* IOMMUFD UAPI is available for VFIO device cdev to pin and map user memory with
+  the ability to retrieve physical addresses for DMA command submission.
+
+Kconfig Support Matrix
+^^^^^^^^^^^^^^^^^^^^^^
+
+The visibility of CONFIG_VFIO_NOIOMMU depends on the combination of
+CONFIG_VFIO_GROUP, CONFIG_VFIO_DEVICE_CDEV, and whether a container backend
+(CONFIG_VFIO_CONTAINER or CONFIG_IOMMUFD_VFIO_CONTAINER) is configured.  The
+Kconfig dependencies enforce the following constraints:
+
+- At least one access path (group or cdev) must be available.
+- If VFIO_GROUP is enabled, a container backend is required; otherwise the
+  group node would be unusable in noiommu mode.
+- The cdev noiommu path requires CONFIG_GENERIC_ATOMIC64=n because it depends
+  on CONFIG_IOMMUFD_NOIOMMU.
+
+The resulting support matrix:
+
++------+-------+-----------+------+---------+---------------------------+
+| Case | GROUP | Container | CDEV | NOIOMMU | Notes                     |
++======+=======+===========+======+=========+===========================+
+|  1   |   y   |     y     |  n   |   yes   | Group noiommu works       |
++------+-------+-----------+------+---------+---------------------------+
+|  2   |   y   |     n     |  n   |   no    | Blocked - no container    |
++------+-------+-----------+------+---------+---------------------------+
+|  3   |   y   |     y     |  y   |   yes   | Both paths work [#cdev]_  |
++------+-------+-----------+------+---------+---------------------------+
+|  4   |   y   |     n     |  y   |   no    | Blocked - no container    |
++------+-------+-----------+------+---------+---------------------------+
+|  5   |   n   |     -     |  y   |   yes   | Cdev-only works [#cdev]_  |
++------+-------+-----------+------+---------+---------------------------+
+|  6   |   n   |     -     |  n   |   no    | No access path            |
++------+-------+-----------+------+---------+---------------------------+
+
+Container = CONFIG_VFIO_CONTAINER or CONFIG_IOMMUFD_VFIO_CONTAINER (either
+suffices).  Case 4 is intentionally blocked: allowing NOIOMMU with GROUP
+enabled but no container would create unusable group nodes.  Users who want
+cdev-only noiommu should set CONFIG_VFIO_GROUP=n (case 5).
+
+.. [#cdev] The cdev noiommu path requires CONFIG_GENERIC_ATOMIC64=n.  When
+   CONFIG_VFIO_GROUP=y, CONFIG_VFIO_DEVICE_CDEV=y, and
+   CONFIG_GENERIC_ATOMIC64=y, CONFIG_VFIO_NOIOMMU remains selectable for the
+   group path, but no noiommu device cdev is registered.  Cdev-only noiommu is
+   not selectable on those platforms.
+
+A new IOMMUFD ioctl IOMMU_IOAS_NOIOMMU_GET_PA is added to retrieve the physical
+address for a given IOVA. Although there is no physical DMA remapping hardware,
+IOMMU_IOAS_MAP_FIXED_IOVA is still used to establish IOVA-to-PA mappings in the
+software page table for later IOMMU_IOAS_NOIOMMU_GET_PA lookups.
+
 VFIO User API
 -------------------------------------------------------------------------------
 
diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst
index 08d01bc62c319..5e808c1ebc681 100644
--- a/Documentation/filesystems/locking.rst
+++ b/Documentation/filesystems/locking.rst
@@ -61,7 +61,7 @@ inode_operations
 
 prototypes::
 
-	int (*create) (struct mnt_idmap *, struct inode *,struct dentry *,umode_t, bool);
+	int (*create) (struct mnt_idmap *, struct inode *,struct dentry *,umode_t);
 	struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int);
 	int (*link) (struct dentry *,struct inode *,struct dentry *);
 	int (*unlink) (struct inode *,struct dentry *);
@@ -355,13 +355,14 @@ should perform any validation and preparation necessary to ensure that
 writes can be performed with minimal memory allocation.  It should call
 add_swap_extent(), or the helper iomap_swapfile_activate(), and return
 the number of extents added.  If IO should be submitted through
-->swap_rw(), it should set SWP_FS_OPS, otherwise IO will be submitted
+->swap_rw(), it should call swap_fs_activate, otherwise IO will be submitted
 directly to the block device ``sis->bdev``.
 
 ->swap_deactivate() will be called in the sys_swapoff()
 path after ->swap_activate() returned success.
 
-->swap_rw will be called for swap IO if SWP_FS_OPS was set by ->swap_activate().
+->swap_rw will be called for swap IO if swap_fs_activate was called by
+->swap_activate().
 
 file_lock_operations
 ====================
diff --git a/Documentation/filesystems/overlayfs.rst b/Documentation/filesystems/overlayfs.rst
index eb846518e6acb..16c35b491dad4 100644
--- a/Documentation/filesystems/overlayfs.rst
+++ b/Documentation/filesystems/overlayfs.rst
@@ -123,8 +123,7 @@ At mount time, the two directories given as mount options "lowerdir" and
   mount -t overlay overlay -olowerdir=/lower,upperdir=/upper,\
   workdir=/work /merged
 
-The "workdir" needs to be an empty directory on the same filesystem
-as upperdir.
+The "workdir" needs to be a directory on the same filesystem as upperdir.
 
 Then whenever a lookup is requested in such a merged directory, the
 lookup is performed in each actual directory and the combined result
@@ -347,6 +346,22 @@ The resulting access permissions should be the same.  The difference is in
 the time of copy (on-demand vs. up-front).
 
 
+Idmapped mounts
+---------------
+
+The overlay mount itself can be turned into an idmapped mount by applying an
+idmapping to it with mount_setattr(2) and MOUNT_ATTR_IDMAP, just like for
+other filesystems that support idmapped mounts.
+
+The mount idmapping only changes how ownership and permissions of the overlay
+inodes are presented to and interpreted for the caller.  It does not change
+how overlayfs accesses the underlying layers: those are still accessed with
+the stashed mounter's credentials through their own mounts, which may
+themselves be idmapped.  The overlay mount idmapping and any layer idmapping
+compose, an underlying id is first mapped according to the relevant layer
+idmapping and then according to the overlay mount idmapping.
+
+
 Multiple lower layers
 ---------------------
 
diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst
index d13f0a23c882f..60880eb0c49d1 100644
--- a/Documentation/filesystems/porting.rst
+++ b/Documentation/filesystems/porting.rst
@@ -1173,7 +1173,7 @@ these conditions don't require explicit checks:
  - if LOOKUP_CREATE is NOT given, then the dentry won't be negative,
    ERR_PTR(-ENOENT) is returned instead
  - if LOOKUP_EXCL IS given, then the dentry won't be positive,
-   ERR_PTR(-EEXIST) is rreturned instread
+   ERR_PTR(-EEXIST) is returned instead
 
 LOOKUP_EXCL now means "target must not exist".  It can be combined with
 LOOK_CREATE or LOOKUP_RENAME_TARGET.
@@ -1401,3 +1401,11 @@ as with d_dispose_if_unused() these are not trivial; with this variant
 of API it's more explicit, since grabbing ->d_lock is caller-side, but
 d_dispose_if_unused() had all the same issues.  It's a low-level primitive;
 use only if you have no alternative.
+
+---
+
+**mandatory**
+
+The .create inode_operation no longer receives the 'excl' arg.  It must
+always assume the file does not already exist.  If the filesystem needs
+to be involved in non-exclusive create, it should provide atomic_open.
diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst
index 2ccd5b2dfdd33..27189f6e004ee 100644
--- a/Documentation/filesystems/proc.rst
+++ b/Documentation/filesystems/proc.rst
@@ -608,6 +608,7 @@ encoded manner. The codes are the following:
     um    userfaultfd missing tracking
     uw    userfaultfd wr-protect tracking
     ui    userfaultfd minor fault
+    ur    userfaultfd read-write-protect tracking
     ss    shadow/guarded control stack page
     sl    sealed
     lf    lock on fault pages
diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst
index 7c753148af888..c2886fa50e510 100644
--- a/Documentation/filesystems/vfs.rst
+++ b/Documentation/filesystems/vfs.rst
@@ -415,7 +415,7 @@ As of kernel 2.6.22, the following members are defined:
 .. code-block:: c
 
 	struct inode_operations {
-		int (*create) (struct mnt_idmap *, struct inode *,struct dentry *, umode_t, bool);
+		int (*create) (struct mnt_idmap *, struct inode *,struct dentry *, umode_t);
 		struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int);
 		int (*link) (struct dentry *,struct inode *,struct dentry *);
 		int (*unlink) (struct inode *,struct dentry *);
@@ -977,7 +977,7 @@ cache in your filesystem.  The following members are defined:
 	can be performed with minimal memory allocation.  It should call
 	add_swap_extent(), or the helper iomap_swapfile_activate(), and
 	return the number of extents added.  If IO should be submitted
-	through ->swap_rw(), it should set SWP_FS_OPS, otherwise IO will
+	through ->swap_rw(), it should call swap_fs_activate, otherwise IO will
 	be submitted directly to the block device ``sis->bdev``.
 
 ``swap_deactivate``
@@ -985,7 +985,7 @@ cache in your filesystem.  The following members are defined:
 	successful.
 
 ``swap_rw``
-	Called to read or write swap pages when SWP_FS_OPS is set.
+	Called to read or write swap pages when swap_fs_activate was called.
 
 The File Object
 ===============
diff --git a/Documentation/gpu/amdgpu/display/display-manager.rst b/Documentation/gpu/amdgpu/display/display-manager.rst
index b269ff3f7a545..f6de9e7779e29 100644
--- a/Documentation/gpu/amdgpu/display/display-manager.rst
+++ b/Documentation/gpu/amdgpu/display/display-manager.rst
@@ -32,6 +32,9 @@ Interrupts
 .. kernel-doc:: drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
    :functions: register_hpd_handlers dm_crtc_high_irq dm_pflip_high_irq
 
+.. kernel-doc:: drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+   :functions: amdgpu_display_hotplug_work_func
+
 Atomic Implementation
 =====================
 
@@ -178,3 +181,9 @@ following path:
 2. On DC interface, :c:type:`struct mpcc_blnd_cfg <mpcc_blnd_cfg>` programs the
    MPCC blend configuration considering the :c:type:`dc_plane_info
    <dc_plane_info>` input from DPP.
+
+Display Properties
+==================
+
+.. kernel-doc:: drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+   :doc: property for adaptive backlight modulation
diff --git a/Documentation/gpu/automated_testing.rst b/Documentation/gpu/automated_testing.rst
index 62aa3ede02a5d..8a7328aef10ef 100644
--- a/Documentation/gpu/automated_testing.rst
+++ b/Documentation/gpu/automated_testing.rst
@@ -99,7 +99,8 @@ How to enable automated testing on your tree
 ============================================
 
 1. Create a Linux tree in https://gitlab.freedesktop.org/ if you don't have one
-yet
+yet, by forking https://gitlab.freedesktop.org/drm/kernel (this allows GitLab
+to internally track that these are the same git objects).
 
 2. In your kernel repo's configuration (eg.
 https://gitlab.freedesktop.org/janedoe/linux/-/settings/ci_cd), change the
diff --git a/Documentation/gpu/drm-kms-helpers.rst b/Documentation/gpu/drm-kms-helpers.rst
index 80453dda33b84..94cfc26aceccc 100644
--- a/Documentation/gpu/drm-kms-helpers.rst
+++ b/Documentation/gpu/drm-kms-helpers.rst
@@ -171,6 +171,12 @@ Bridge Operations
 .. kernel-doc:: drivers/gpu/drm/drm_bridge.c
    :doc: bridge operations
 
+Bridge Chain Format Selection
+-----------------------------
+
+.. kernel-doc:: drivers/gpu/drm/drm_bridge.c
+   :doc: bridge chain format selection
+
 Bridge Connector Helper
 -----------------------
 
diff --git a/Documentation/gpu/drm-kms.rst b/Documentation/gpu/drm-kms.rst
index fa69a5450f967..0dd440a149461 100644
--- a/Documentation/gpu/drm-kms.rst
+++ b/Documentation/gpu/drm-kms.rst
@@ -287,6 +287,12 @@ structure, ordering of committing state changes to hardware is sequenced using
 Read on in this chapter, and also in :ref:`drm_atomic_helper` for more detailed
 coverage of specific topics.
 
+Atomic State Lifetime
+---------------------
+
+.. kernel-doc:: drivers/gpu/drm/drm_atomic.c
+   :doc: state lifetime
+
 Handling Driver Private State
 -----------------------------
 
@@ -604,6 +610,12 @@ Color Management Properties
 .. kernel-doc:: drivers/gpu/drm/drm_color_mgmt.c
    :doc: overview
 
+Color Format Property
+---------------------
+
+.. kernel-doc:: drivers/gpu/drm/drm_connector.c
+   :doc: Color format
+
 Tile Group Property
 -------------------
 
diff --git a/Documentation/gpu/intel-display/dp-link-capabilities.rst b/Documentation/gpu/intel-display/dp-link-capabilities.rst
new file mode 100644
index 0000000000000..331cc69d13a04
--- /dev/null
+++ b/Documentation/gpu/intel-display/dp-link-capabilities.rst
@@ -0,0 +1,11 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+DisplayPort Link Capabilities
+=============================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dp_link_caps.c
+   :doc: DisplayPort link capabilities
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dp_link_caps.h
+   :internal:
diff --git a/Documentation/gpu/intel-display/dp-link-training.rst b/Documentation/gpu/intel-display/dp-link-training.rst
new file mode 100644
index 0000000000000..d0bde965021d9
--- /dev/null
+++ b/Documentation/gpu/intel-display/dp-link-training.rst
@@ -0,0 +1,8 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+DisplayPort Link Training
+=========================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dp_link_training.c
+   :doc: DisplayPort link training
diff --git a/Documentation/gpu/intel-display/index.rst b/Documentation/gpu/intel-display/index.rst
index 01c3d1e576b71..e81f49bf20df6 100644
--- a/Documentation/gpu/intel-display/index.rst
+++ b/Documentation/gpu/intel-display/index.rst
@@ -38,6 +38,8 @@ driver. The display driver isn't an independent driver in that sense.
    fifo-underrun
    frontbuffer
    hotplug
+   dp-link-training
+   dp-link-capabilities
    plane
    psr
    snps-phy
diff --git a/Documentation/gpu/nova/core/fsp.rst b/Documentation/gpu/nova/core/fsp.rst
new file mode 100644
index 0000000000000..52d618d22bb89
--- /dev/null
+++ b/Documentation/gpu/nova/core/fsp.rst
@@ -0,0 +1,142 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+===================================================
+FSP (Foundation Security Processor) and Secure Boot
+===================================================
+This document describes the role of the FSP in the GPU boot sequence on
+Hopper and Blackwell GPUs, and how it differs from the earlier Ampere boot
+flow. It also provides a brief overview of the PRC (Product Reconfiguration
+Control) protocol used to query device configuration through FSP. As with
+other documents in this directory, the information is subject to change and
+is intended to help developers understand the corresponding kernel code.
+
+What is FSP?
+============
+The Foundation Security Processor (FSP) is the GPU's Internal Root of Trust
+(IROT). It is a dedicated security processor that boots from immutable ROM
+(Boot ROM) inside the GPU and is responsible for establishing the Chain of
+Trust before any other firmware is allowed to run.
+
+FSP runs independently of the host CPU and starts executing as soon as the
+GPU is powered on. By the time the nova-core driver is loaded, FSP has
+already completed its own secure boot and is ready to accept commands from
+the driver.
+
+Simplified boot flow (Hopper/Blackwell)
+=======================================
+Starting with Hopper, the boot flow is significantly simplified compared to
+earlier GPU generations like Ampere.
+
+On an **Ampere** GPU, the boot verification chain involves multiple Falcon
+engines and multiple ucode stages (see falcon.rst for details)::
+
+     Hardware BROM (SEC2)
+          -> HS Booter (SEC2)
+               -> LS GSP-RM (GSP)
+
+The driver must extract ucode from VBIOS, manage SEC2 and GSP, and
+orchestrate the Booter to load GSP-RM. This involves FWSEC-FRTS, devinit,
+and the Booter stages.
+
+On **Hopper/Blackwell** GPUs, FSP replaces this multi-stage process with a
+single message-driven interface::
+
+     FSP (hardware root of trust, boots from ROM)
+          -> FMC (Falcon Microcontroller, verified by FSP)
+               -> GSP-RM (verified and loaded by FMC)
+
+The driver only needs to:
+
+1. Wait for FSP to complete its own secure boot (polling a scratch register).
+2. Send a Chain of Trust (COT) message to FSP with the FMC firmware location,
+   cryptographic signatures, and GSP boot parameters.
+3. FSP authenticates the FMC firmware and boots it, FMC in turn loads GSP-RM.
+
+There is no SEC2 involvement, no Booter ucode, and no FWSEC-FRTS stage. The
+entire secure boot is driven by a single FSP message exchange.
+
+Chain of Trust (COT) protocol
+=============================
+The Chain of Trust establishes a cryptographically enforced boot sequence,
+ensuring the GPU reaches a known, trusted state.
+
+The driver communicates with FSP using a message queue (Falcon MSGQ
+interface). Each message consists of an MCTP (Management Component Transport
+Protocol) transport header and an NVDM (NVIDIA Vendor Defined Message) header,
+followed by a protocol-specific payload.
+
+For Chain of Trust, the payload includes:
+
+- The system memory address of the FMC firmware image.
+- Cryptographic material: a SHA-384 hash, RSA-3K public key, and RSA-3K
+  signature extracted from the FMC ELF firmware.
+- FRTS (Firmware Runtime Services) region information (vidmem offset and size).
+- The system memory address of the GSP boot arguments structure.
+
+FSP verifies the signature against the provided public key and hash, and if
+verification succeeds, boots the FMC. The FMC then authenticates and launches
+GSP-RM.
+
+The message flow is::
+
+     nova-core                          FSP
+        |                                |
+        |  1. Poll scratch register      |
+        |  (wait for FSP boot complete)  |
+        |                                |
+        |  2. COT message  ------------> |
+        |     (FMC addr, signatures,     |
+        |      boot params)              |
+        |                                |
+        |                                |--- Verify FMC signature
+        |                                |--- Boot FMC
+        |                                |--- FMC loads GSP-RM
+        |                                |
+        |  3. COT response <------------ |
+        |     (success/error)            |
+        |                                |
+
+FSP message format
+==================
+All FSP messages share a common header format consisting of two 32-bit words:
+
+**MCTP header** (Management Component Transport Protocol):
+
+- Bit 31: SOM (Start of Message)
+- Bit 30: EOM (End of Message)
+- Bits 29:28: Packet sequence number
+- Bits 23:16: Source Endpoint ID
+
+**NVDM header** (NVIDIA Vendor Defined Message):
+
+- Bits 6:0: MCTP message type (0x7e = vendor-defined PCI)
+- Bits 23:8: PCI vendor ID (0x10de = NVIDIA)
+- Bits 31:24: NVDM type (0x14 = COT, 0x13 = PRC, 0x15 = FSP response)
+
+PRC (Product Reconfiguration Control) protocol
+===============================================
+PRC is an API system exposed through FSP's Management Partition that allows
+querying and modifying device configuration without firmware updates.
+
+Configuration parameters are called "knobs". Each knob has a unique object
+ID and controls a specific device behavior. Examples include vGPU mode, ECC
+enable, confidential computing mode, and NVLINK configuration.
+
+Each knob has two values:
+
+- **Active**: the currently effective value for this boot cycle.
+- **Persistent**: the value stored in InfoROM, applied on subsequent boots.
+
+The nova-core driver uses PRC to read the vGPU mode knob (object ID 0x29)
+during early boot, before firmware loading, to determine whether the GPU
+should operate in vGPU mode.
+
+The PRC message format follows the same MCTP/NVDM header structure as COT,
+with NVDM type 0x13. The payload contains:
+
+- A sub-command (e.g., 0x0c for read).
+- Flags indicating which value to read (bit 0 = persistent, bit 1 = active).
+- The knob object ID.
+
+The response includes the common FSP response header (with error status)
+followed by the knob's 16-bit state value.
diff --git a/Documentation/gpu/nova/index.rst b/Documentation/gpu/nova/index.rst
index e39cb3163581e..1783513cbd05a 100644
--- a/Documentation/gpu/nova/index.rst
+++ b/Documentation/gpu/nova/index.rst
@@ -30,5 +30,6 @@ vGPU manager VFIO driver and the nova-drm driver.
    core/todo
    core/vbios
    core/devinit
+   core/fsp
    core/fwsec
    core/falcon
diff --git a/Documentation/gpu/todo.rst b/Documentation/gpu/todo.rst
index cdddf8db35f53..14cf37590fc7e 100644
--- a/Documentation/gpu/todo.rst
+++ b/Documentation/gpu/todo.rst
@@ -55,7 +55,7 @@ There are still drivers that use drm_simple_display_pipe. The task here is to
 convert them to use regular atomic helpers. Search for a driver that calls
 drm_simple_display_pipe_init() and inline all helpers from drm_simple_kms_helper.c
 into the driver, such that no simple-KMS interfaces are required. Please also
-rename all inlined fucntions according to driver conventions.
+rename all inlined functions according to driver conventions.
 
 Contact: Thomas Zimmermann, respective driver maintainer
 
@@ -896,10 +896,11 @@ complection of submission.
 One minor feature still missing is a generic DRM IOCTL to query the error
 status of binary and timeline drm_syncobj.
 
-This should probably be improved by implementing the necessary kernel interface
-and adding support for that in the userspace stack.
+This was already improved by implementing the necessary kernel interface
+(patches are on the dri-devel mailing list) but adding support for that in the
+userspace stack is still missing.
 
-Contact: Christian König
+Contact: Christian König, Michel Dänzer
 
 Level: Starter
 
@@ -948,6 +949,47 @@ Contact: Philipp Stanner <phasta@kernel.org>
 
 Level: Intermediate
 
+Replace the lockless queue with a locked list
+---------------------------------------------
+
+drm_sched is the only user in the entire kernel of a special lockless queue, the
+spsc_queue. This queue utilizes:
+
+- preempt_disable()
+- atomic instructions
+- memory barriers
+- ACCESS_ONCE()
+
+whereas a conventional spinlock utilizes:
+
+- preempt_disable()
+- 1 atomic instruction for taking / releasing the lock
+- memory barriers
+
+Moreover, drm_sched_entity_push_job(), the only user of spsc_queue_push(), has
+to take a lock in some situations anyways and calls to it are often serialized
+with a driver lock.
+
+It is, thus, highly questionable whether the lockless queue grants any advantage
+at all. Considering that its internals are not well documented and its correctness
+is not formally proven, it seems desirable to replace the queue with a mere list
+or hlist that is protected by a spinlock.
+
+Tasks:
+
+- Replace the spsc_queue in drm/sched (and those who might access the scheduler's
+  internal queue) with a spinlock + (h)list.
+- Ideally, check with some micro benchmarks and real world tests (preferably
+  with amdgpu) for relevant performance regressions.
+- Remove the spsc_queue from the kernel altogether.
+
+Contact:
+
+- Philipp Stanner <phasta@kernel.org>
+- Christian König <christian.koenig@amd.com>
+
+Level: Beginner
+
 Outside DRM
 ===========
 
diff --git a/Documentation/gpu/xe/xe_device.rst b/Documentation/gpu/xe/xe_device.rst
index 39a937b97cd35..d3a022362ade1 100644
--- a/Documentation/gpu/xe/xe_device.rst
+++ b/Documentation/gpu/xe/xe_device.rst
@@ -8,3 +8,10 @@ Xe Device Wedging
 
 .. kernel-doc:: drivers/gpu/drm/xe/xe_device.c
    :doc: Xe Device Wedging
+
+====================
+GPU Health Indicator
+====================
+
+.. kernel-doc:: drivers/gpu/drm/xe/xe_ras.c
+   :doc: GPU Health Indicator
diff --git a/Documentation/hwmon/adm1275.rst b/Documentation/hwmon/adm1275.rst
index cf923f20fa523..d8495be313b84 100644
--- a/Documentation/hwmon/adm1275.rst
+++ b/Documentation/hwmon/adm1275.rst
@@ -67,6 +67,30 @@ Supported chips:
 
     Datasheet: https://www.analog.com/media/en/technical-documentation/data-sheets/ADM1293_1294.pdf
 
+  * ROHM Semiconductor BD12780
+
+    Prefix: 'bd12780'
+
+    Addresses scanned: -
+
+    Datasheet: https://fscdn.rohm.com/en/products/databook/datasheet/ic/power/power_switch/bd12780muv-lb-e.pdf
+
+  * ROHM Semiconductor BD12780A
+
+    Prefix: 'bd12780'
+
+    Addresses scanned: -
+
+    Datasheet: https://fscdn.rohm.com/en/products/databook/datasheet/ic/power/power_switch/bd12780amuv-lb-e.pdf
+
+  * ROHM Semiconductor BD12790
+
+    Prefix: 'bd12790'
+
+    Addresses scanned: -
+
+    Datasheet: -
+
   * Silergy SQ24905C
 
     Prefix: 'mc09c'
diff --git a/Documentation/hwmon/asus_ec_sensors.rst b/Documentation/hwmon/asus_ec_sensors.rst
index 77a709517437a..a5db48d5a72e7 100644
--- a/Documentation/hwmon/asus_ec_sensors.rst
+++ b/Documentation/hwmon/asus_ec_sensors.rst
@@ -17,6 +17,7 @@ Supported boards:
  * ProArt X670E-CREATOR WIFI
  * ProArt X870E-CREATOR WIFI
  * ProArt B550-CREATOR
+ * ProArt Z690-CREATOR WIFI
  * ROG CROSSHAIR VIII DARK HERO
  * ROG CROSSHAIR VIII HERO (WI-FI)
  * ROG CROSSHAIR VIII FORMULA
@@ -25,11 +26,13 @@ Supported boards:
  * ROG CROSSHAIR X670E EXTREME
  * ROG CROSSHAIR X670E HERO
  * ROG CROSSHAIR X670E GENE
+ * ROG CROSSHAIR X870E HERO
  * ROG MAXIMUS X HERO
  * ROG MAXIMUS XI HERO
  * ROG MAXIMUS XI HERO (WI-FI)
  * ROG MAXIMUS Z690 FORMULA
  * ROG MAXIMUS Z790 EXTREME
+ * ROG MAXIMUS Z790 HERO
  * ROG STRIX B550-E GAMING
  * ROG STRIX B550-I GAMING
  * ROG STRIX B650E-E GAMING WIFI
@@ -47,6 +50,7 @@ Supported boards:
  * ROG STRIX X870-F GAMING WIFI
  * ROG STRIX X870-I GAMING WIFI
  * ROG STRIX X870E-E GAMING WIFI
+ * ROG STRIX X870E-E GAMING WIFI7 R2
  * ROG STRIX X870E-H GAMING WIFI7
  * ROG STRIX Z390-F GAMING
  * ROG STRIX Z490-F GAMING
diff --git a/Documentation/hwmon/chipcap2.rst b/Documentation/hwmon/chipcap2.rst
index dc165becc64cf..c38d87b91b697 100644
--- a/Documentation/hwmon/chipcap2.rst
+++ b/Documentation/hwmon/chipcap2.rst
@@ -70,4 +70,6 @@ humidity1_min_hyst:             RW      humidity low hystersis
 humidity1_max_hyst:             RW      humidity high hystersis
 humidity1_min_alarm:            RO      humidity low alarm indicator
 humidity1_max_alarm:            RO      humidity high alarm indicator
+humidity1_label:                RO      descriptive name for the sensor
+temp1_label:                    RO      descriptive name for the sensor
 =============================== ======= ========================================
diff --git a/Documentation/hwmon/coretemp.rst b/Documentation/hwmon/coretemp.rst
index f63b21f24d42c..3493016833814 100644
--- a/Documentation/hwmon/coretemp.rst
+++ b/Documentation/hwmon/coretemp.rst
@@ -44,9 +44,9 @@ Temperature known as TjMax is the maximum junction temperature of processor,
 which depends on the CPU model. See table below. At this temperature, protection
 mechanism will perform actions to forcibly cool down the processor. Alarm
 may be raised, if the temperature grows enough (more than TjMax) to trigger
-the Out-Of-Spec bit. Following table summarizes the exported sysfs files:
+the Out-Of-Spec bit. The following table summarizes the exported sysfs files:
 
-All Sysfs entries are named with their core_id (represented here by 'X').
+All sysfs entries are named with their core_id (represented here by 'X').
 
 ================= ========================================================
 tempX_input	  Core temperature (in millidegrees Celsius).
diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst
index 29130df44d12f..2267893762173 100644
--- a/Documentation/hwmon/index.rst
+++ b/Documentation/hwmon/index.rst
@@ -253,6 +253,7 @@ Hardware Monitoring Kernel Drivers
    smsc47m1
    sparx5-temp
    spd5118
+   sq24860
    stpddc60
    surface_fan
    sy7636a-hwmon
@@ -272,6 +273,7 @@ Hardware Monitoring Kernel Drivers
    tps53679
    tps546d24
    tsc1641
+   tvs-mpfs
    twl4030-madc-hwmon
    ucd9000
    ucd9200
diff --git a/Documentation/hwmon/sq24860.rst b/Documentation/hwmon/sq24860.rst
new file mode 100644
index 0000000000000..f0182b955d8a7
--- /dev/null
+++ b/Documentation/hwmon/sq24860.rst
@@ -0,0 +1,96 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Kernel driver sq24860
+=====================
+
+Supported chips:
+
+  * Silergy SQ24860
+
+    Prefix: 'sq24860'
+
+Author:
+
+	Ziming Zhu <ziming.zhu@silergycorp.com>
+
+Description
+------------
+
+This driver implements support for the Silergy SQ24860 eFuse. The device is an
+integrated circuit protection and power management device with a PMBus
+interface.
+
+The device supports direct format for reading input voltage, output voltage,
+auxiliary voltage, input current, input power, and temperature.
+
+The current and power measurement scale depends on the resistor connected
+between the IMON pin and ground. The resistor value can be configured with the
+``silergy,rimon-micro-ohms`` device tree property. See
+``Documentation/devicetree/bindings/hwmon/pmbus/silergy,sq24860.yaml`` for details.
+
+Due to the specificities of the chip, all history reset attributes are tied
+together. Resetting the history of one sensor resets the history of all sensors.
+
+Sysfs entries
+-------------
+
+The following attributes are supported. Limits are read-write; all other
+attributes are read-only.
+
+======================= ======================================================
+in1_label               "vin"
+in1_input               Measured input voltage.
+in1_average             Average measured input voltage.
+in1_min                 Minimum input voltage limit.
+in1_lcrit               Critical low input voltage limit.
+in1_max                 Maximum input voltage limit.
+in1_crit                Critical high input voltage limit.
+in1_min_alarm           Input voltage low warning alarm.
+in1_lcrit_alarm         Input voltage low fault alarm.
+in1_max_alarm           Input voltage high warning alarm.
+in1_crit_alarm          Input voltage high fault alarm.
+in1_highest             Historical maximum input voltage.
+in1_lowest              Historical minimum input voltage.
+in1_reset_history       Write any value to reset history.
+
+in2_label               "vmon"
+in2_input               Measured auxiliary input voltage.
+
+in3_label               "vout1"
+in3_input               Measured output voltage.
+in3_average             Average measured output voltage.
+in3_min                 Minimum output voltage limit.
+in3_min_alarm           Output voltage low alarm.
+in3_lowest              Historical minimum output voltage.
+in3_reset_history       Write any value to reset history.
+
+curr1_label             "iin"
+curr1_input             Measured input current.
+curr1_average           Average measured input current.
+curr1_max               Maximum input current warning limit.
+curr1_crit              Critical input over-current fault limit.
+curr1_max_alarm         Input current warning alarm.
+curr1_crit_alarm        Input over-current fault alarm.
+curr1_highest           Historical maximum input current.
+curr1_reset_history     Write any value to reset history.
+
+power1_label            "pin"
+power1_input            Measured input power.
+power1_average          Average measured input power.
+power1_max              Maximum input power warning limit.
+power1_alarm            Input power warning alarm.
+power1_input_highest    Historical maximum input power.
+power1_reset_history    Write any value to reset history.
+
+temp1_input             Measured temperature.
+temp1_average           Average measured temperature.
+temp1_max               Maximum temperature warning limit.
+temp1_crit              Critical temperature fault limit.
+temp1_max_alarm         Temperature warning alarm.
+temp1_crit_alarm        Temperature fault alarm.
+temp1_highest           Historical maximum temperature.
+temp1_reset_history     Write any value to reset history.
+
+samples                 Number of samples used for average values.
+======================= ======================================================
+
diff --git a/Documentation/hwmon/tvs-mpfs.rst b/Documentation/hwmon/tvs-mpfs.rst
new file mode 100644
index 0000000000000..1035812f363ac
--- /dev/null
+++ b/Documentation/hwmon/tvs-mpfs.rst
@@ -0,0 +1,53 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Kernel driver tvs-mpfs
+======================
+
+Supported chips:
+
+  * PolarFire SoC
+
+Authors:
+
+   - Conor Dooley <conor.dooley@microchip.com>
+   - Lars Randers <lranders@mail.dk>
+
+Description
+-----------
+
+This driver implements support for the temperature and voltage sensors on
+PolarFire SoC. The temperature reports how hot the die is, and the voltages are
+the SoC's 1.05, 1.8 and 2.5 volt rails respectively.
+
+
+Usage Notes
+-----------
+
+update_interval has a permitted range of 0 to 8 milliseconds.
+
+Temperatures are read in millidegrees Celsius, but the hardware measures in
+degrees Kelvin, storing the result as 11.4 fixed point data, for a maximum
+value of 2047.9375 degrees Kelvin.
+
+Voltages are read in millivolts. The hardware measures in millivolts, storing
+the value as 12.3 fixed point data, for a maximum of 4095.875 millivolts.
+The minimum value reportable by the driver is 0 volts, although the hardware
+is capable of measuring negative values.
+
+Sysfs entries
+-------------
+
+The following attributes are supported. update_interval is read-write, as are
+the enables. All other attributes are read only.
+
+======================= ====================================================
+temp1_label		Fixed name for channel.
+temp1_input		Measured temperature for channel.
+temp1_enable		Enable/disable for channel.
+
+in[0-2]_label		Fixed name for channel.
+in[0-2]_input		Measured voltage for channel.
+in[0-2]_enable		Enable/disable for channel.
+
+update_interval		The interval at which the chip will update readings.
+======================= ====================================================
diff --git a/Documentation/iio/adf41513.rst b/Documentation/iio/adf41513.rst
new file mode 100644
index 0000000000000..244453cce6f68
--- /dev/null
+++ b/Documentation/iio/adf41513.rst
@@ -0,0 +1,199 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+===============
+ADF41513 driver
+===============
+
+This driver supports Analog Devices' ADF41513 and similar SPI PLL frequency
+synthesizers.
+
+1. Supported devices
+====================
+
+* `ADF41510 <https://www.analog.com/ADF41510>`_
+* `ADF41513 <https://www.analog.com/ADF41513>`_
+
+The ADF41513 is an ultralow noise frequency synthesizer that can be used to
+implement local oscillators (LOs) as high as 26.5 GHz in the upconversion and
+downconversion sections of wireless receivers and transmitters. The ADF41510
+is a similar device that supports frequencies up to 10 GHz.
+
+Both devices support integer-N and fractional-N operation modes, providing
+excellent phase noise performance and flexible frequency generation
+capabilities.
+
+Key Features:
+
+- **ADF41510**: 1 GHz to 10 GHz frequency range
+- **ADF41513**: 1 GHz to 26.5 GHz frequency range
+- Integer-N and fractional-N operation modes
+- Ultra-low phase noise (-235 dBc/Hz integer-N, -231 dBc/Hz fractional-N)
+- High maximum PFD frequency (250 MHz integer-N, 125 MHz fractional-N)
+- 25-bit fixed modulus or 49-bit variable modulus fractional modes
+- Programmable charge pump currents with 16x range
+- Digital lock detect functionality
+- Phase resync capability for consistent output phase
+
+2. Device attributes
+====================
+
+The ADF41513 driver provides the following IIO extended attributes for
+frequency control and monitoring:
+
+Each IIO device has a device folder under ``/sys/bus/iio/devices/iio:deviceX``,
+where X is the IIO index of the device. Under these folders reside a set of
+device files that provide access to the synthesizer's functionality.
+
+The following table shows the ADF41513 related device files:
+
++--------------------------------------+-------------------------------------------------------+
+| Device file                          | Description                                           |
++======================================+=======================================================+
+| out_altvoltage0_frequency            | RF output frequency control and readback (Hz)         |
++--------------------------------------+-------------------------------------------------------+
+| out_altvoltage0_frequency_resolution | Target frequency resolution control (Hz)              |
++--------------------------------------+-------------------------------------------------------+
+| out_altvoltage0_powerdown            | Power management control (0=active, 1=power down)     |
++--------------------------------------+-------------------------------------------------------+
+| out_altvoltage0_phase                | RF output phase adjustment and readback (radians)     |
++--------------------------------------+-------------------------------------------------------+
+
+2.1 Frequency Control
+----------------------
+
+The ``out_altvoltage0_frequency`` attribute controls the RF output frequency
+with sub-Hz precision. The driver automatically selects between integer-N and
+fractional-N modes to achieve the requested frequency with the best possible
+phase noise performance.
+
+**Supported ranges:**
+
+- **ADF41510**: 1,000,000,000 Hz to 10,000,000,000 Hz (1 GHz to 10 GHz)
+- **ADF41513**: 1,000,000,000 Hz to 26,500,000,000 Hz (1 GHz to 26.5 GHz)
+
+The frequency is specified in Hz, for sub-Hz precision use decimal notation.
+For example, 12.102 GHz would be written as "12102000000.000000".
+
+2.2 Frequency Resolution Control
+--------------------------------
+
+The ``out_altvoltage0_frequency_resolution`` attribute controls the target
+frequency resolution that the driver attempts to achieve. This affects the
+choice between integer-N and fractional-N modes, including fixed modulus
+(25-bit) and variable modulus (49-bit) fractional-N modes:
+
+- **Integer-N**: Resolution = :math:`f_{PFD}` (same as PFD frequency)
+- **Fixed modulus**: Resolution = :math:`f_{PFD} / 2^{25}` (~3 Hz with 100 MHz PFD)
+- **Variable modulus**: Resolution = :math:`f_{PFD} / 2^{49}` (µHz resolution possible)
+
+Default resolution is 1 Hz (1,000,000 µHz).
+
+2.3 Phase adjustment
+--------------------
+
+The ``out_altvoltage0_phase`` attribute allows adjustment of the output phase
+in radians. Setting this attribute enables phase adjustment. It can be set
+from 0 to :math:`2\pi` radians. Reading this attribute returns the current
+phase offset of the output signal. To create a consistent phase relationship
+with the reference signal, the phase resync feature needs to be enabled by
+setting a non-zero value to the ``adi,phase-resync-period-ns`` device property,
+which triggers a phase resynchronization after locking is achieved.
+
+3. Operating modes
+==================
+
+3.1 Integer-N Mode
+------------------
+
+When the requested frequency can be achieved as an integer multiple of the PFD
+frequency (within the specified resolution tolerance), the driver automatically
+selects integer-N mode for optimal phase noise performance.
+
+In integer-N mode:
+
+- Phase noise: -235 dBc/Hz normalized floor
+- Frequency resolution: :math:`f_{PFD}` (same as PFD frequency)
+- Maximum PFD frequency: 250 MHz
+- Bleed current: Disabled
+
+3.2 Fractional-N Mode
+---------------------
+
+When sub-integer frequency steps are required, the driver automatically selects
+fractional-N mode using either fixed or variable modulus.
+
+**Fixed Modulus (25-bit)**:
+
+- Used when variable modulus is not required
+- Resolution: :math:`f_{PFD} / 2^{25}`
+- Simpler implementation, faster settling
+
+**Variable Modulus (49-bit)**:
+
+- Used for maximum resolution requirements
+- Resolution: :math:`f_{PFD} / 2^{49}` (theoretical)
+- Exact frequency synthesis capability
+
+In fractional-N mode:
+
+- Phase noise: -231 dBc/Hz normalized floor
+- Maximum PFD frequency: 125 MHz
+- Bleed current: Automatically enabled and optimized
+- Dithering: Enabled to reduce fractional spurs
+
+3.3 Automatic Mode Selection
+----------------------------
+
+The driver automatically selects the optimal operating mode based on:
+
+1. **Frequency accuracy requirements**: Determined by ``frequency_resolution`` setting
+2. **Phase noise optimization**: Integer-N preferred when possible
+3. **PFD frequency constraints**: Different limits for integer vs fractional modes
+4. **Prescaler selection**: Automatic 4/5 vs 8/9 prescaler selection based on frequency
+
+4. Usage examples
+=================
+
+4.1 Basic Frequency Setting
+----------------------------
+
+Set output frequency to 12.102 GHz:
+
+.. code-block:: bash
+
+    root:/sys/bus/iio/devices/iio:device0> echo 12102000000 > out_altvoltage0_frequency
+
+Read current frequency:
+
+.. code-block:: bash
+
+    root:/sys/bus/iio/devices/iio:device0> cat out_altvoltage0_frequency
+    12101999999.582767
+
+4.2 High Resolution Frequency Control
+-------------------------------------
+
+Configure for sub-Hz resolution and set a precise frequency:
+
+.. code-block:: bash
+
+    # Set resolution to 0.1 Hz (100,000 µHz)
+    root:/sys/bus/iio/devices/iio:device0> echo 0.1 > out_altvoltage0_frequency_resolution
+
+    # Set frequency to 12.102 GHz (1 µHz precision)
+    root:/sys/bus/iio/devices/iio:device0> echo 12102000000 > out_altvoltage0_frequency
+    root:/sys/bus/iio/devices/iio:device0> cat out_altvoltage0_frequency
+    12101999999.980131
+
+4.3 Monitor Lock Status
+-----------------------
+
+When lock detect GPIO is configured, check if PLL is locked:
+
+.. code-block:: bash
+
+    # Read frequency - will return error if not locked
+    root:/sys/bus/iio/devices/iio:device0> cat out_altvoltage0_frequency
+
+If the PLL is not locked, the frequency read will return ``-EBUSY`` (Device or
+resource busy).
diff --git a/Documentation/iio/index.rst b/Documentation/iio/index.rst
index 007e0a1fcc5a2..b02b879b053a1 100644
--- a/Documentation/iio/index.rst
+++ b/Documentation/iio/index.rst
@@ -31,6 +31,7 @@ Industrial I/O Kernel Drivers
    ad7625
    ad7944
    ade9000
+   adf41513
    adis16475
    adis16480
    adis16550
diff --git a/Documentation/infiniband/user_mad.rst b/Documentation/infiniband/user_mad.rst
index d88abfc0e3700..cd66e7623d669 100644
--- a/Documentation/infiniband/user_mad.rst
+++ b/Documentation/infiniband/user_mad.rst
@@ -62,7 +62,7 @@ Receiving MADs
 	struct ib_user_mad *mad;
 	mad = malloc(sizeof *mad + 256);
 	ret = read(fd, mad, sizeof *mad + 256);
-	if (ret == -ENOSPC)) {
+	if (ret == -ENOSPC) {
 		length = mad.length;
 		free(mad);
 		mad = malloc(sizeof *mad + length);
diff --git a/Documentation/kernel-hacking/locking.rst b/Documentation/kernel-hacking/locking.rst
index c969c76ef7cb4..23fd393c81932 100644
--- a/Documentation/kernel-hacking/locking.rst
+++ b/Documentation/kernel-hacking/locking.rst
@@ -471,7 +471,7 @@ to protect the cache and all the objects within it. Here's the code::
             obj = __cache_find(id);
             if (obj) {
                     ret = 0;
-                    strcpy(name, obj->name);
+                    strscpy(name, obj->name);
             }
             mutex_unlock(&cache_lock);
             return ret;
@@ -553,7 +553,7 @@ which are taken away, and the ``+`` are lines which are added.
              obj = __cache_find(id);
              if (obj) {
                      ret = 0;
-                     strcpy(name, obj->name);
+                     strscpy(name, obj->name);
              }
     -        mutex_unlock(&cache_lock);
     +        spin_unlock_irqrestore(&cache_lock, flags);
@@ -676,7 +676,7 @@ Here is the code::
              obj = __cache_find(id);
     -        if (obj) {
     -                ret = 0;
-    -                strcpy(name, obj->name);
+    -                strscpy(name, obj->name);
     -        }
     +        if (obj)
     +                __object_get(obj);
@@ -1317,7 +1317,7 @@ from user context, and can sleep.
 
    -  put_user()
 
--  kmalloc(GP_KERNEL) <kmalloc>`
+-  kmalloc(GFP_KERNEL) <kmalloc>
 
 -  mutex_lock_interruptible() and
    mutex_lock()
diff --git a/Documentation/leds/leds-st1202.rst b/Documentation/leds/leds-st1202.rst
index 1a09fbfcedcff..ed32eb3a27d45 100644
--- a/Documentation/leds/leds-st1202.rst
+++ b/Documentation/leds/leds-st1202.rst
@@ -16,8 +16,9 @@ in terms of PWM duty-cycle and duration (ms).
 To be compatible with the hardware pattern format, maximum 8 tuples of
 brightness (PWM) and duration must be written to hw_pattern.
 
+- Brightness range: 0-255
 - Min pattern duration: 22 ms
-- Max pattern duration: 5660 ms
+- Max pattern duration: 5610 ms
 
 The format of the hardware pattern values should be:
 "brightness duration brightness duration ..."
@@ -26,9 +27,7 @@ The format of the hardware pattern values should be:
 ----------------------------
 
 Specify a pattern repeat number, which is common for all channels.
-Default is 1; negative numbers and 0 are invalid.
+Default is 1. Writing 0 is invalid. Writing -1 or 255 repeats the
+pattern indefinitely.
 
 This file will always return the originally written repeat number.
-
-When the 255 value is written to it, all patterns will repeat
-indefinitely.
diff --git a/Documentation/mm/allocation-profiling.rst b/Documentation/mm/allocation-profiling.rst
index 5389d241176a8..c3a28467955f3 100644
--- a/Documentation/mm/allocation-profiling.rst
+++ b/Documentation/mm/allocation-profiling.rst
@@ -46,6 +46,11 @@ sysctl:
 Runtime info:
   /proc/allocinfo
 
+  Profiling data can be retrieved either by reading `/proc/allocinfo` directly as
+  text or programmatically via `ioctl()` calls defined in `<uapi/linux/alloc_tag.h>`.
+  The ioctl interface supports structured binary data extraction as well as filtering
+  by module name, function, file, line number, accuracy, or allocation size limits.
+
 Example output::
 
   root@moria-kvm:~# sort -g /proc/allocinfo|tail|numfmt --to=iec
diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst
index 2da7ca0d3d17a..aed6cb1cf4831 100644
--- a/Documentation/mm/damon/design.rst
+++ b/Documentation/mm/damon/design.rst
@@ -86,7 +86,7 @@ To know how user-space can do the configuration via :ref:`DAMON sysfs interface
 documentation.
 
 
- .. _damon_design_vaddr_target_regions_construction:
+.. _damon_design_vaddr_target_regions_construction:
 
 VMA-based Target Address Range Construction
 -------------------------------------------
@@ -316,6 +316,31 @@ Another way to do this for higher accuracy is using :ref:`DAMOS filter
 information in page level.  But, because it is operated in page level, the
 overhead is proportional to the size of the memory.
 
+.. _damon_design_attrs_only_monitoring:
+
+Data Attributes-only Monitoring
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Data access is the primary monitoring information for DAMON.  Hence it
+:ref:`adjusts <damon_design_data_attrs_monitoring>` regions using the access
+:ref:`counter <damon_design_region_based_sampling>` (``nr_accesses``).  In some
+use cases, however, users may want some of :ref:`attributes
+<damon_design_data_attrs_monitoring>` to be the primary information.
+
+Data attributes-only monitoring mode supports this use case.  For the mode,
+each attribute probe has their priority weight value.  Users can describe by
+what combination of the attributes the primary information is decided, by
+setting the priority weight value.  If the total sum of the weights is not
+zero, the mode is enabled.  The regions adjustment mechanism uses the weighted
+sum of the :ref:`probe hit counts <damon_design_data_attrs_monitoring>` instead
+of ``nr_accesses`` in the case.  When the mode is enabled, access monitoring is
+automatically turned off.  The access counter (``nr_accesses``) will always be
+zero and not updated.  Hence the mode is called Data Attributes "only"
+monitoring.
+
+Refer to the :ref:`admin guide <damon_usage_sysfs_probes>` to know how users
+can use the mode.
+
 Dynamic Target Space Updates Handling
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -686,9 +711,11 @@ mechanism tries to make ``current_value`` of ``target_metric`` be same to
   (1/10,000).
 - ``inactive_mem_bp``: Inactive to active + inactive (LRU) memory size ratio in
   bp (1/10,000).
+- ``node_eligible_mem_bp``: Scheme target access pattern-eligible memory ratio
+  of a node in bp (1/10,000).
 
-``nid`` is optionally required for only ``node_mem_used_bp``,
-``node_mem_free_bp``, ``node_memcg_used_bp`` and ``node_memcg_free_bp`` to
+``nid`` is optionally required for ``node_mem_used_bp``, ``node_mem_free_bp``,
+``node_memcg_used_bp``, ``node_memcg_free_bp`` and ``node_eligible_mem_bp`` to
 point the specific NUMA node.
 
 ``path`` is optionally required for only ``node_memcg_used_bp`` and
@@ -930,11 +957,11 @@ control parameters for the usage would also need to be optimized for the
 purpose.
 
 To support such cases, yet more DAMON API user kernel modules that provide more
-simple and optimized user space interfaces are available.  Currently, two
-modules for proactive reclamation and LRU lists manipulation are provided.  For
-more detail, please read the usage documents for those
-(:doc:`/admin-guide/mm/damon/stat`, :doc:`/admin-guide/mm/damon/reclaim` and
-:doc:`/admin-guide/mm/damon/lru_sort`).
+simple and optimized user space interfaces are available.  Currently, three
+modules for access monitoring statistics, proactive reclamation, and LRU lists
+manipulation are provided.  For more detail, please read the usage documents for
+those (:doc:`/admin-guide/mm/damon/stat`, :doc:`/admin-guide/mm/damon/reclaim`
+and :doc:`/admin-guide/mm/damon/lru_sort`).
 
 .. _damon_design_special_purpose_modules_exclusivity:
 
diff --git a/Documentation/mm/hmm.rst b/Documentation/mm/hmm.rst
index 7d61b7a8b65b7..e021218ada58f 100644
--- a/Documentation/mm/hmm.rst
+++ b/Documentation/mm/hmm.rst
@@ -156,42 +156,57 @@ During the ops->invalidate() callback the device driver must perform the
 update action to the range (mark range read only, or fully unmap, etc.). The
 device must complete the update before the driver callback returns.
 
-When the device driver wants to populate a range of virtual addresses, it can
-use::
+When the device driver wants to populate a range of virtual addresses, the
+normal interface is::
 
-  int hmm_range_fault(struct hmm_range *range);
+  int hmm_range_fault_unlocked_timeout(struct hmm_range *range,
+                                       unsigned long timeout);
 
 It will trigger a page fault on missing or read-only entries if write access is
 requested (see below). Page faults use the generic mm page fault code path just
-like a CPU page fault. The usage pattern is::
+like a CPU page fault.
+
+The caller must not hold ``mmap_read_lock`` before the call.
+``hmm_range_fault_unlocked_timeout()`` takes the mmap read lock internally and
+allows ``handle_mm_fault()`` to drop it during fault handling. This is required
+for VMAs whose fault handlers may release the mmap lock, for example regions
+managed by ``userfaultfd``.
+
+If the mmap lock is dropped or the range is invalidated, the function refreshes
+``range->notifier_seq`` and restarts the walk internally. ``-EINTR`` is returned
+if mmap lock acquisition is interrupted or a fatal signal is pending during
+retry handling.
+
+The timeout is specified in jiffies; passing ``0`` means retry indefinitely. The
+timeout exists to preserve caller policy for repeated mmu-notifier invalidation
+and is checked between retry attempts. HMM does not interrupt page fault
+handling when the timeout expires, but returns ``-EBUSY`` if the retry budget is
+exhausted before a stable range is obtained.
+
+The usage pattern is::
 
  int driver_populate_range(...)
  {
       struct hmm_range range;
+      unsigned long timeout;
       ...
 
+      timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
       range.notifier = &interval_sub;
       range.start = ...;
       range.end = ...;
       range.hmm_pfns = ...;
 
-      if (!mmget_not_zero(interval_sub->notifier.mm))
+      if (!mmget_not_zero(interval_sub.mm))
           return -EFAULT;
 
  again:
-      range.notifier_seq = mmu_interval_read_begin(&interval_sub);
-      mmap_read_lock(mm);
-      ret = hmm_range_fault(&range);
-      if (ret) {
-          mmap_read_unlock(mm);
-          if (ret == -EBUSY)
-                 goto again;
-          return ret;
-      }
-      mmap_read_unlock(mm);
+      ret = hmm_range_fault_unlocked_timeout(&range, timeout);
+      if (ret)
+          goto out_put;
 
       take_lock(driver->update);
-      if (mmu_interval_read_retry(&ni, range.notifier_seq) {
+      if (mmu_interval_read_retry(range.notifier, range.notifier_seq)) {
           release_lock(driver->update);
           goto again;
       }
@@ -200,13 +215,31 @@ like a CPU page fault. The usage pattern is::
        * under the update lock */
 
       release_lock(driver->update);
-      return 0;
+      ret = 0;
+
+ out_put:
+      mmput(interval_sub.mm);
+      return ret;
  }
 
 The driver->update lock is the same lock that the driver takes inside its
 invalidate() callback. That lock must be held before calling
 mmu_interval_read_retry() to avoid any race with a concurrent CPU page table
-update.
+update. The retry check must use the same notifier and sequence number stored
+in ``range`` by ``hmm_range_fault_unlocked_timeout()``.
+
+Holding the mmap lock across HMM faults
+=======================================
+
+Most callers should use ``hmm_range_fault_unlocked_timeout()``. If a driver
+really needs to hold the mmap lock across work outside HMM, it can use::
+
+  int hmm_range_fault(struct hmm_range *range);
+
+The mmap lock must be held by the caller and will remain held on return. This
+interface cannot support VMAs whose fault handlers need to drop the mmap lock.
+New callers should prefer ``hmm_range_fault_unlocked_timeout()`` unless they
+have a specific requirement to keep the mmap lock held across the call.
 
 Leverage default_flags and pfn_flags_mask
 =========================================
@@ -221,8 +254,8 @@ permission, it sets::
     range->default_flags = HMM_PFN_REQ_FAULT;
     range->pfn_flags_mask = 0;
 
-and calls hmm_range_fault() as described above. This will fill fault all pages
-in the range with at least read permission.
+and calls the HMM range fault helper as described above. This will fault
+all pages in the range with at least read permission.
 
 Now let's say the driver wants to do the same except for one page in the range for
 which it wants to have write permission. Now driver set::
@@ -236,9 +269,9 @@ address == range->start + (index_of_write << PAGE_SHIFT) it will fault with
 write permission i.e., if the CPU pte does not have write permission set then HMM
 will call handle_mm_fault().
 
-After hmm_range_fault completes the flag bits are set to the current state of
-the page tables, ie HMM_PFN_VALID | HMM_PFN_WRITE will be set if the page is
-writable.
+After the HMM range fault helper completes the flag bits are set to the
+current state of the page tables, ie HMM_PFN_VALID | HMM_PFN_WRITE will be
+set if the page is writable.
 
 
 Represent and manage device memory from core kernel point of view
diff --git a/Documentation/mm/page_owner.rst b/Documentation/mm/page_owner.rst
index 6b12f3b007ec9..a6bd3fe6423ad 100644
--- a/Documentation/mm/page_owner.rst
+++ b/Documentation/mm/page_owner.rst
@@ -65,7 +65,15 @@ un-tracking state.
 Usage
 =====
 
-1) Build user-space helper::
+1) Build user-space helpers:
+::
+
+   To filter page_owner output:
+
+	cd tools/mm
+	make page_owner_filter
+
+   To sort and analyze page_owner output:
 
 	cd tools/mm
 	make page_owner_sort
@@ -74,7 +82,11 @@ Usage
 
 3) Do the job that you want to debug.
 
-4) Analyze information from page owner::
+4) (Optional) Filter page_owner output::
+
+	./page_owner_filter -m handle -n 0,1,2 > filtered_page_owner.txt
+
+5) Analyze information from page owner::
 
 	cat /sys/kernel/debug/page_owner_stacks/show_stacks > stacks.txt
 	cat stacks.txt
@@ -263,3 +275,65 @@ STANDARD FORMAT SPECIFIERS
 	f		free		whether the page has been released or not
 	st		stacktrace	stack trace of the page allocation
 	ator		allocator	memory allocator for pages
+
+Filtering page_owner output
+============================
+
+page_owner supports filtering output at the kernel level before reading,
+which reduces the amount of data that needs to be processed in userspace.
+
+The page_owner_filter tool provides a convenient interface for this filtering
+capability. It supports two types of filters:
+
+1. **print_mode filter**: Control what information is printed for each page
+	- ``stack``: Print full stack traces (default, compatible with existing usage)
+	- ``handle``: Print only stack handle numbers (much faster, smaller output)
+	- ``stack_handle``: Print both stack traces and handle numbers
+
+	The ``handle`` mode uses numeric identifiers instead of full stack traces.
+	The mapping from handles to actual stack traces can be obtained via the
+	show_stacks_handles interface.
+
+2. **NUMA node filter**: Filter pages by NUMA node ID
+	- Supports single node: ``-n 0``
+	- Multiple nodes: ``-n 0,1,2``
+	- Ranges: ``-n 0-3``
+	- Mixed format: ``-n 0,2-3,5``
+
+Usage examples::
+
+	# Filter by print mode
+	./page_owner_filter -m handle
+	./page_owner_filter -m stack_handle
+
+	# Filter by NUMA node
+	./page_owner_filter -n 0
+	./page_owner_filter -n 0-3
+
+	# Combined filters
+	./page_owner_filter -m stack -n 0,1,2
+	./page_owner_filter -m handle -n 0,2-3
+
+	# Save to file
+	./page_owner_filter -m handle -o filtered_output.txt
+
+The handle mode is particularly useful for monitoring and performance-critical
+scenarios as it dramatically reduces output size. Testing shows handle mode can
+reduce output size by ~66% (84MB vs 244MB) and improve read performance by ~4.4x
+compared to full stack output.
+
+The NUMA node filter is useful for NUMA-aware memory allocation analysis and debugging.
+
+Behind the scenes, page_owner_filter opens /sys/kernel/debug/page_owner and
+writes filter commands before reading the filtered output. The filtering uses
+per-file-descriptor state, allowing each open() to have independent filter settings.
+
+Each file descriptor maintains its own filter state, so you can have multiple
+independent filtering operations running concurrently. For example, in different
+terminals you can run different filters simultaneously::
+
+	# Terminal 1: Filter node 0
+	./page_owner_filter -n 0 > node0_output.txt
+
+	# Terminal 2: Filter node 1 (runs concurrently)
+	./page_owner_filter -n 1 > node1_output.txt
diff --git a/Documentation/netlink/specs/devlink.yaml b/Documentation/netlink/specs/devlink.yaml
index 52ad1e7805d19..38b1190f3d269 100644
--- a/Documentation/netlink/specs/devlink.yaml
+++ b/Documentation/netlink/specs/devlink.yaml
@@ -895,6 +895,16 @@ attribute-sets:
           resource-dump response. Bit 0 (dev) selects device-level
           resources; bit 1 (port) selects port-level resources.
           When absent all classes are returned.
+      -
+        name: parent-dev
+        type: nest
+        nested-attributes: dl-parent-dev
+        doc: |
+          Identifies the devlink instance which owns the parent rate node.
+          Used with rate-set and rate-new to parent a rate object to a node on
+          a different devlink instance, enabling cross-device rate scheduling.
+          When absent, the parent node is resolved on the same instance.
+
   -
     name: dl-dev-stats
     subset-of: devlink
@@ -1317,6 +1327,16 @@ attribute-sets:
              Specifies the bandwidth share assigned to the Traffic Class.
              The bandwidth for the traffic class is determined
              in proportion to the sum of the shares of all configured classes.
+  -
+    name: dl-parent-dev
+    subset-of: devlink
+    attributes:
+      -
+        name: bus-name
+      -
+        name: dev-name
+      -
+        name: index
 
 operations:
   enum-model: directional
@@ -2289,8 +2309,8 @@ operations:
       dont-validate: [strict]
       flags: [admin-perm]
       do:
-        pre: devlink-nl-pre-doit
-        post: devlink-nl-post-doit
+        pre: devlink-nl-pre-doit-parent-dev-optional
+        post: devlink-nl-post-doit-parent-dev-optional
         request:
           attributes:
             - bus-name
@@ -2303,6 +2323,7 @@ operations:
             - rate-tx-weight
             - rate-parent-node-name
             - rate-tc-bws
+            - parent-dev
 
     -
       name: rate-new
@@ -2311,8 +2332,8 @@ operations:
       dont-validate: [strict]
       flags: [admin-perm]
       do:
-        pre: devlink-nl-pre-doit
-        post: devlink-nl-post-doit
+        pre: devlink-nl-pre-doit-parent-dev-optional
+        post: devlink-nl-post-doit-parent-dev-optional
         request:
           attributes:
             - bus-name
@@ -2325,6 +2346,7 @@ operations:
             - rate-tx-weight
             - rate-parent-node-name
             - rate-tc-bws
+            - parent-dev
 
     -
       name: rate-del
diff --git a/Documentation/netlink/specs/dpll.yaml b/Documentation/netlink/specs/dpll.yaml
index 2bf83f6732ab0..cdc8c7b456df8 100644
--- a/Documentation/netlink/specs/dpll.yaml
+++ b/Documentation/netlink/specs/dpll.yaml
@@ -165,6 +165,13 @@ definitions:
       -
         name: gnss
         doc: GNSS recovered clock
+      -
+        name: int-nco
+        doc: |
+          Device internal numerically controlled oscillator.
+          When connected as a DPLL input, the DPLL enters NCO mode
+          where the output frequency is adjusted by the host via
+          the PTP clock interface.
     render-max: true
   -
     type: enum
@@ -252,6 +259,12 @@ definitions:
       -
         name: state-can-change
         doc: pin state can be changed
+      -
+        name: state-connected-override
+        doc: |
+          pin state can be set to connected regardless of current
+          DPLL device mode, overriding the active input selection.
+          Requires state-can-change to be set as well.
   -
     type: const
     name: phase-offset-divider
@@ -456,6 +469,9 @@ attribute-sets:
           offset on the media associated with the pin. Inside
           the pin-parent-device nest it represents the frequency
           offset between the pin and its parent DPLL device.
+          For pins of type PIN_TYPE_INT_NCO this represents
+          the DPLL's current output frequency offset from its
+          nominal frequency.
           Value is in PPM (parts per million).
           This is a lower-precision version of
           fractional-frequency-offset-ppt.
@@ -502,6 +518,9 @@ attribute-sets:
           offset on the media associated with the pin. Inside
           the pin-parent-device nest it represents the frequency
           offset between the pin and its parent DPLL device.
+          For pins of type PIN_TYPE_INT_NCO this represents
+          the DPLL's current output frequency offset from its
+          nominal frequency.
           Value is in PPT (parts per trillion, 10^-12).
           This is a higher-precision version of
           fractional-frequency-offset.
diff --git a/Documentation/netlink/specs/rt-addr.yaml b/Documentation/netlink/specs/rt-addr.yaml
index 163a106c41bb4..0ecbd24c890c6 100644
--- a/Documentation/netlink/specs/rt-addr.yaml
+++ b/Documentation/netlink/specs/rt-addr.yaml
@@ -123,6 +123,9 @@ attribute-sets:
       -
         name: proto
         type: u8
+      -
+        name: mc-users
+        type: u32
 
 
 operations:
@@ -176,6 +179,7 @@ operations:
           value: 58
           attributes: &mcaddr-attrs
             - multicast
+            - mc-users
             - cacheinfo
       dump:
         request:
diff --git a/Documentation/networking/devlink/devlink-port.rst b/Documentation/networking/devlink/devlink-port.rst
index 9374ebe70f485..fe2cfee3e2a6d 100644
--- a/Documentation/networking/devlink/devlink-port.rst
+++ b/Documentation/networking/devlink/devlink-port.rst
@@ -107,6 +107,15 @@ doesn't have the eswitch. Local controller (identified by controller number = 0)
 has the eswitch. The Devlink instance on the local controller has eswitch
 devlink ports for both the controllers.
 
+A non-zero controller number may also be used for ports that are not external.
+For example, a SmartNIC may have additional local PCI physical functions
+that are managed by the eswitch but are not on an external host. These
+ports use a non-zero controller number to distinguish them from the eswitch
+manager's own functions, while the external flag remains unset.
+
+The ``phys_port_name`` includes the controller prefix (``c<controller_num>``)
+whenever the controller number is non-zero, regardless of the external flag.
+
 Function configuration
 ======================
 
@@ -420,6 +429,8 @@ API allows to configure following rate object's parameters:
   Parent node name. Parent node rate limits are considered as additional limits
   to all node children limits. ``tx_max`` is an upper limit for children.
   ``tx_share`` is a total bandwidth distributed among children.
+  If the device supports cross-function scheduling, the parent can be from a
+  different function of the same underlying device.
 
 ``tc_bw``
   Allow users to set the bandwidth allocation per traffic class on rate
diff --git a/Documentation/networking/devlink/index.rst b/Documentation/networking/devlink/index.rst
index 32f70879ddd06..4745148fecf4f 100644
--- a/Documentation/networking/devlink/index.rst
+++ b/Documentation/networking/devlink/index.rst
@@ -31,10 +31,10 @@ sure to respect following rules:
 
  - Lock ordering should be maintained. If driver needs to take instance
    lock of both nested and parent instances at the same time, devlink
-   instance lock of the parent instance should be taken first, only then
-   instance lock of the nested instance could be taken.
- - Driver should use object-specific helpers to setup the nested relationship
-   before registering the nested devlink instance:
+   instance lock of the nested instance should be taken first, only then
+   instance lock of the parent instance could be taken.
+ - Driver should use object-specific helpers to setup the
+   nested relationship:
 
    - ``devl_nested_devlink_set()`` - called to setup devlink -> nested
      devlink relationship (could be used for multiple nested instances).
diff --git a/Documentation/networking/devlink/mlx5.rst b/Documentation/networking/devlink/mlx5.rst
index 4bba4d780a4ae..cf1dffa676696 100644
--- a/Documentation/networking/devlink/mlx5.rst
+++ b/Documentation/networking/devlink/mlx5.rst
@@ -419,3 +419,36 @@ User commands examples:
 
 .. note::
    This command can run over all interfaces such as PF/VF and representor ports.
+
+Rates
+=====
+
+mlx5 devices can limit transmission of individual VFs or a group of them via
+the devlink-rate API in switchdev mode.
+
+User commands examples:
+
+- Print the existing rates::
+
+    $ devlink port function rate show
+
+- Set a max tx limit on traffic from VF0::
+
+    $ devlink port function rate set pci/0000:82:00.0/1 tx_max 10Gbit
+
+- Create a rate group with a max tx limit and add two VFs to it::
+
+    $ devlink port function rate add pci/0000:82:00.0/group1 tx_max 10Gbit
+    $ devlink port function rate set pci/0000:82:00.0/1 parent group1
+    $ devlink port function rate set pci/0000:82:00.0/2 parent group1
+
+- Same scenario, with a min guarantee of 20% of the bandwidth for the first VF::
+
+    $ devlink port function rate add pci/0000:82:00.0/group1 tx_max 10Gbit
+    $ devlink port function rate set pci/0000:82:00.0/1 parent group1 tx_share 2Gbit
+    $ devlink port function rate set pci/0000:82:00.0/2 parent group1
+
+- Cross-device scheduling::
+
+    $ devlink port function rate add pci/0000:82:00.0/group1 tx_max 10Gbit
+    $ devlink port function rate set pci/0000:82:00.1/32769 parent pci/0000:82:00.0/group1
diff --git a/Documentation/networking/net_cachelines/netns_ipv4_sysctl.rst b/Documentation/networking/net_cachelines/netns_ipv4_sysctl.rst
index 6dbd97d435e9f..3dc03bff739e4 100644
--- a/Documentation/networking/net_cachelines/netns_ipv4_sysctl.rst
+++ b/Documentation/networking/net_cachelines/netns_ipv4_sysctl.rst
@@ -22,6 +22,7 @@ struct_mutex                    ra_mutex
 struct_fib_rules_ops*           rules_ops
 struct_fib_table                fib_main
 struct_fib_table                fib_default
+spinlock_t                      fib_table_hash_lock
 unsigned_int                    fib_rules_require_fldissect
 bool                            fib_has_custom_rules
 bool                            fib_has_custom_local_routes
diff --git a/Documentation/networking/netdevices.rst b/Documentation/networking/netdevices.rst
index d2a238f8cc8b9..db71d4283032a 100644
--- a/Documentation/networking/netdevices.rst
+++ b/Documentation/networking/netdevices.rst
@@ -421,10 +421,17 @@ running under the lock:
 * ``NETDEV_CHANGENAME``
 * ``NETDEV_REGISTER``
 * ``NETDEV_UP``
+* ``NETDEV_DOWN``
+* ``NETDEV_GOING_DOWN``
 
 The following notifiers are running without the lock:
 * ``NETDEV_UNREGISTER``
 
+Many SW devices (uppers) catch their lower's ``NETDEV_UNREGISTER``
+events and may interact with them via ``dev_*()`` handlers, which take
+the instance lock. Until we convert these devices to ``netif_*()`` variants,
+``NETDEV_UNREGISTER`` stays unlocked.
+
 There are no clear expectations for the remaining notifiers. Notifiers not on
 the list may run with or without the instance lock, potentially even invoking
 the same notifier type with and without the lock from different code paths.
diff --git a/Documentation/process/submitting-patches.rst b/Documentation/process/submitting-patches.rst
index cc6a1f73d7f20..7ae79452e1b46 100644
--- a/Documentation/process/submitting-patches.rst
+++ b/Documentation/process/submitting-patches.rst
@@ -404,12 +404,11 @@ patches that are being emailed around.
 The sign-off is a simple line at the end of the explanation for the
 patch, which certifies that you wrote it or otherwise have the right to
 pass it on as an open-source patch.  The rules are pretty simple: if you
-can certify the below:
+can certify the below::
 
-Developer's Certificate of Origin 1.1
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+        Developer's Certificate of Origin 1.1
 
-By making a contribution to this project, I certify that:
+        By making a contribution to this project, I certify that:
 
         (a) The contribution was created in whole or in part by me and I
             have the right to submit it under the open source license
@@ -554,12 +553,11 @@ some testing has been performed, provides a means to locate testers for
 future patches, and ensures credit for the testers.
 
 Reviewed-by:, instead, indicates that the patch has been reviewed and found
-acceptable according to the Reviewer's Statement:
+acceptable according to the Reviewer's Statement::
 
-Reviewer's statement of oversight
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+	Reviewer's statement of oversight
 
-By offering my Reviewed-by: tag, I state that:
+	By offering my Reviewed-by: tag, I state that:
 
 	 (a) I have carried out a technical review of this patch to
 	     evaluate its appropriateness and readiness for inclusion into
diff --git a/Documentation/scheduler/sched-energy.rst b/Documentation/scheduler/sched-energy.rst
index 4e47aaf103eb7..83bac0da4869c 100644
--- a/Documentation/scheduler/sched-energy.rst
+++ b/Documentation/scheduler/sched-energy.rst
@@ -141,7 +141,7 @@ in its previous activation.
 find_energy_efficient_cpu() uses compute_energy() to estimate what will be the
 energy consumed by the system if the waking task was migrated. compute_energy()
 looks at the current utilization landscape of the CPUs and adjusts it to
-'simulate' the task migration. The EM framework provides the em_pd_energy() API
+'simulate' the task migration. The EM framework provides the em_cpu_energy() API
 which computes the expected energy consumption of each performance domain for
 the given utilization landscape.
 
diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst
index 4b1ffd03f5165..2771ea4cc14af 100644
--- a/Documentation/scheduler/sched-ext.rst
+++ b/Documentation/scheduler/sched-ext.rst
@@ -493,8 +493,9 @@ a freshly woken up task gets on a CPU.
 Where to Look
 =============
 
-* ``include/linux/sched/ext.h`` defines the core data structures, ops table
-  and constants.
+* ``include/linux/sched/ext.h`` defines the core data structures and
+  constants, while the ops table (``struct sched_ext_ops``) is defined in
+  ``kernel/sched/ext/internal.h``.
 
 * ``kernel/sched/ext/ext.c`` contains sched_ext core implementation and helpers.
   The functions prefixed with ``scx_bpf_`` can be called from the BPF
@@ -555,7 +556,8 @@ ABI Instability
 ===============
 
 The APIs provided by sched_ext to BPF schedulers programs have no stability
-guarantees. This includes the ops table callbacks and constants defined in
+guarantees. This includes the ops table callbacks defined in
+``kernel/sched/ext/internal.h`` and the constants defined in
 ``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in
 ``kernel/sched/ext/ext.c`` and ``kernel/sched/ext/idle.c``.
 
diff --git a/Documentation/scheduler/sched-pelt.c b/Documentation/scheduler/sched-pelt.c
index 7238b355919c7..38ff53dcf3526 100644
--- a/Documentation/scheduler/sched-pelt.c
+++ b/Documentation/scheduler/sched-pelt.c
@@ -25,7 +25,8 @@ void calc_runnable_avg_yN_inv(void)
 	for (i = 0; i < HALFLIFE; i++) {
 		x = ((1UL<<32)-1)*pow(y, i);
 
-		if (i % 6 == 0) printf("\n\t");
+		if (i % 6 == 0)
+			printf("\n\t");
 		printf("0x%8x, ", x);
 	}
 	printf("\n};\n\n");
diff --git a/Documentation/security/credentials.rst b/Documentation/security/credentials.rst
index 4996838491b10..a39a2a2f67aa5 100644
--- a/Documentation/security/credentials.rst
+++ b/Documentation/security/credentials.rst
@@ -393,16 +393,14 @@ the credentials so obtained when they're finished with.
    The result of ``__task_cred()`` should not be passed directly to
    ``get_cred()`` as this may race with ``commit_cred()``.
 
-There are a couple of convenience functions to access bits of another task's
-credentials, hiding the RCU magic from the caller::
+There is a convenience function to access bits of another task's credentials,
+hiding the RCU magic from the caller::
 
 	uid_t task_uid(task)		Task's real UID
-	uid_t task_euid(task)		Task's effective UID
 
 If the caller is holding the RCU read lock at the time anyway, then::
 
 	__task_cred(task)->uid
-	__task_cred(task)->euid
 
 should be used instead.  Similarly, if multiple aspects of a task's credentials
 need to be accessed, RCU read lock should be used, ``__task_cred()`` called,
diff --git a/Documentation/sphinx-static/custom.css b/Documentation/sphinx-static/custom.css
index 5aa0a1ed9864c..be33d9ed12800 100644
--- a/Documentation/sphinx-static/custom.css
+++ b/Documentation/sphinx-static/custom.css
@@ -3,6 +3,8 @@
  * CSS tweaks for the Alabaster theme
  */
 
+div.body {  max-width: 120em; }
+
 /* Shrink the headers a bit */
 div.body h1 { font-size: 180%; }
 div.body h2 { font-size: 150%; }
@@ -156,15 +158,6 @@ div.language-selection ul li:hover {
     background: #dddddd;
 }
 
-/*
- * Let long inline literals in paragraph text wrap as needed to prevent
- * overflow.
- */
-code.docutils.literal span.pre {
-    white-space: normal;
-    overflow-wrap: anywhere;
-}
-
 /* Let rendered reference links in tables wrap when needed. */
 div.body table.docutils a.reference {
     overflow-wrap: anywhere;
diff --git a/Documentation/sphinx/maintainers_include.py b/Documentation/sphinx/maintainers_include.py
index dc9f9e188ffa8..7ffe19b5ed58d 100755
--- a/Documentation/sphinx/maintainers_include.py
+++ b/Documentation/sphinx/maintainers_include.py
@@ -161,7 +161,7 @@ class MaintainersParser:
                     html = KERNELDOC_URL + ename + ".html"
                     entries[entry] = f'`{ename} <{html}>`_'
                 else:
-                    entries[entry] = f':doc:`{ename} </{entry}>`'
+                    entries[entry] = f'/{entry}'
 
         return entries
 
@@ -345,7 +345,10 @@ class MaintainersProfile(Include):
                 output += f"- {name}: {entry}\n"
                 self.warning(f"{profile}: Invalid 'P' tag: {entry}\n")
             else:
-                output += f"- {entry}\n"
+                if not name:
+                    name = entry
+
+                output += f"- :doc:`{name} <{entry}>`\n"
 
         #
         # Create a hidden TOC table with all profiles. That allows adding
diff --git a/Documentation/staging/rpmsg.rst b/Documentation/staging/rpmsg.rst
index 42bac1149d9d6..63612b7ee120a 100644
--- a/Documentation/staging/rpmsg.rst
+++ b/Documentation/staging/rpmsg.rst
@@ -212,7 +212,6 @@ be probed with.
 
 unregisters an rpmsg driver from the rpmsg bus. user should provide
 a pointer to a previously-registered rpmsg_driver struct.
-Returns 0 on success, and an appropriate error value on failure.
 
 
 Typical usage
diff --git a/Documentation/sunrpc/xdr/nfs4_1.x b/Documentation/sunrpc/xdr/nfs4_1.x
index 5b45547b2ebc8..e66f396ae6592 100644
--- a/Documentation/sunrpc/xdr/nfs4_1.x
+++ b/Documentation/sunrpc/xdr/nfs4_1.x
@@ -45,19 +45,162 @@ pragma header nfs4;
 /*
  * Basic typedefs for RFC 1832 data type definitions
  */
-typedef hyper		int64_t;
-typedef unsigned int	uint32_t;
+typedef int                  int32_t;
+typedef unsigned int         uint32_t;
+typedef hyper                int64_t;
+typedef unsigned hyper       uint64_t;
+
+const NFS4_VERIFIER_SIZE        = 8;
+const NFS4_FHSIZE               = 128;
+
+enum nfsstat4 {
+ NFS4_OK                = 0,    /* everything is okay      */
+ NFS4ERR_PERM           = 1,    /* caller not privileged   */
+ NFS4ERR_NOENT          = 2,    /* no such file/directory  */
+ NFS4ERR_IO             = 5,    /* hard I/O error          */
+ NFS4ERR_NXIO           = 6,    /* no such device          */
+ NFS4ERR_ACCESS         = 13,   /* access denied           */
+ NFS4ERR_EXIST          = 17,   /* file already exists     */
+ NFS4ERR_XDEV           = 18,   /* different filesystems   */
+
+ /*
+  * Please do not allocate value 19; it was used in NFSv3
+  * and we do not want a value in NFSv3 to have a different
+  * meaning in NFSv4.x.
+  */
+
+ NFS4ERR_NOTDIR         = 20,   /* should be a directory   */
+ NFS4ERR_ISDIR          = 21,   /* should not be directory */
+ NFS4ERR_INVAL          = 22,   /* invalid argument        */
+ NFS4ERR_FBIG           = 27,   /* file exceeds server max */
+ NFS4ERR_NOSPC          = 28,   /* no space on filesystem  */
+ NFS4ERR_ROFS           = 30,   /* read-only filesystem    */
+ NFS4ERR_MLINK          = 31,   /* too many hard links     */
+ NFS4ERR_NAMETOOLONG    = 63,   /* name exceeds server max */
+ NFS4ERR_NOTEMPTY       = 66,   /* directory not empty     */
+ NFS4ERR_DQUOT          = 69,   /* hard quota limit reached*/
+ NFS4ERR_STALE          = 70,   /* file no longer exists   */
+ NFS4ERR_BADHANDLE      = 10001,/* Illegal filehandle      */
+ NFS4ERR_BAD_COOKIE     = 10003,/* READDIR cookie is stale */
+ NFS4ERR_NOTSUPP        = 10004,/* operation not supported */
+ NFS4ERR_TOOSMALL       = 10005,/* response limit exceeded */
+ NFS4ERR_SERVERFAULT    = 10006,/* undefined server error  */
+ NFS4ERR_BADTYPE        = 10007,/* type invalid for CREATE */
+ NFS4ERR_DELAY          = 10008,/* file "busy" - retry     */
+ NFS4ERR_SAME           = 10009,/* nverify says attrs same */
+ NFS4ERR_DENIED         = 10010,/* lock unavailable        */
+ NFS4ERR_EXPIRED        = 10011,/* lock lease expired      */
+ NFS4ERR_LOCKED         = 10012,/* I/O failed due to lock  */
+ NFS4ERR_GRACE          = 10013,/* in grace period         */
+ NFS4ERR_FHEXPIRED      = 10014,/* filehandle expired      */
+ NFS4ERR_SHARE_DENIED   = 10015,/* share reserve denied    */
+ NFS4ERR_WRONGSEC       = 10016,/* wrong security flavor   */
+ NFS4ERR_CLID_INUSE     = 10017,/* clientid in use         */
+
+ /* NFS4ERR_RESOURCE is not a valid error in NFSv4.1 */
+ NFS4ERR_RESOURCE       = 10018,/* resource exhaustion     */
+
+ NFS4ERR_MOVED          = 10019,/* filesystem relocated    */
+ NFS4ERR_NOFILEHANDLE   = 10020,/* current FH is not set   */
+ NFS4ERR_MINOR_VERS_MISMATCH= 10021,/* minor vers not supp */
+ NFS4ERR_STALE_CLIENTID = 10022,/* server has rebooted     */
+ NFS4ERR_STALE_STATEID  = 10023,/* server has rebooted     */
+ NFS4ERR_OLD_STATEID    = 10024,/* state is out of sync    */
+ NFS4ERR_BAD_STATEID    = 10025,/* incorrect stateid       */
+ NFS4ERR_BAD_SEQID      = 10026,/* request is out of seq.  */
+ NFS4ERR_NOT_SAME       = 10027,/* verify - attrs not same */
+ NFS4ERR_LOCK_RANGE     = 10028,/* overlapping lock range  */
+ NFS4ERR_SYMLINK        = 10029,/* should be file/directory*/
+ NFS4ERR_RESTOREFH      = 10030,/* no saved filehandle     */
+ NFS4ERR_LEASE_MOVED    = 10031,/* some filesystem moved   */
+ NFS4ERR_ATTRNOTSUPP    = 10032,/* recommended attr not sup*/
+ NFS4ERR_NO_GRACE       = 10033,/* reclaim outside of grace*/
+ NFS4ERR_RECLAIM_BAD    = 10034,/* reclaim error at server */
+ NFS4ERR_RECLAIM_CONFLICT= 10035,/* conflict on reclaim    */
+ NFS4ERR_BADXDR         = 10036,/* XDR decode failed       */
+ NFS4ERR_LOCKS_HELD     = 10037,/* file locks held at CLOSE*/
+ NFS4ERR_OPENMODE       = 10038,/* conflict in OPEN and I/O*/
+ NFS4ERR_BADOWNER       = 10039,/* owner translation bad   */
+ NFS4ERR_BADCHAR        = 10040,/* utf-8 char not supported*/
+ NFS4ERR_BADNAME        = 10041,/* name not supported      */
+ NFS4ERR_BAD_RANGE      = 10042,/* lock range not supported*/
+ NFS4ERR_LOCK_NOTSUPP   = 10043,/* no atomic up/downgrade  */
+ NFS4ERR_OP_ILLEGAL     = 10044,/* undefined operation     */
+ NFS4ERR_DEADLOCK       = 10045,/* file locking deadlock   */
+ NFS4ERR_FILE_OPEN      = 10046,/* open file blocks op.    */
+ NFS4ERR_ADMIN_REVOKED  = 10047,/* lockowner state revoked */
+ NFS4ERR_CB_PATH_DOWN   = 10048,/* callback path down      */
+
+ /* NFSv4.1 errors start here. */
+
+ NFS4ERR_BADIOMODE      = 10049,
+ NFS4ERR_BADLAYOUT      = 10050,
+ NFS4ERR_BAD_SESSION_DIGEST = 10051,
+ NFS4ERR_BADSESSION     = 10052,
+ NFS4ERR_BADSLOT        = 10053,
+ NFS4ERR_COMPLETE_ALREADY = 10054,
+ NFS4ERR_CONN_NOT_BOUND_TO_SESSION = 10055,
+ NFS4ERR_DELEG_ALREADY_WANTED = 10056,
+ NFS4ERR_BACK_CHAN_BUSY = 10057,/*backchan reqs outstanding*/
+ NFS4ERR_LAYOUTTRYLATER = 10058,
+ NFS4ERR_LAYOUTUNAVAILABLE = 10059,
+ NFS4ERR_NOMATCHING_LAYOUT = 10060,
+ NFS4ERR_RECALLCONFLICT = 10061,
+ NFS4ERR_UNKNOWN_LAYOUTTYPE = 10062,
+ NFS4ERR_SEQ_MISORDERED = 10063,/* unexpected seq.ID in req*/
+ NFS4ERR_SEQUENCE_POS   = 10064,/* [CB_]SEQ. op not 1st op */
+ NFS4ERR_REQ_TOO_BIG    = 10065,/* request too big         */
+ NFS4ERR_REP_TOO_BIG    = 10066,/* reply too big           */
+ NFS4ERR_REP_TOO_BIG_TO_CACHE =10067,/* rep. not all cached*/
+ NFS4ERR_RETRY_UNCACHED_REP =10068,/* retry & rep. uncached*/
+ NFS4ERR_UNSAFE_COMPOUND =10069,/* retry/recovery too hard */
+ NFS4ERR_TOO_MANY_OPS   = 10070,/*too many ops in [CB_]COMP*/
+ NFS4ERR_OP_NOT_IN_SESSION =10071,/* op needs [CB_]SEQ. op */
+ NFS4ERR_HASH_ALG_UNSUPP = 10072, /* hash alg. not supp.   */
+                                /* Error 10073 is unused.  */
+ NFS4ERR_CLIENTID_BUSY  = 10074,/* clientid has state      */
+ NFS4ERR_PNFS_IO_HOLE   = 10075,/* IO to _SPARSE file hole */
+ NFS4ERR_SEQ_FALSE_RETRY= 10076,/* Retry != original req.  */
+ NFS4ERR_BAD_HIGH_SLOT  = 10077,/* req has bad highest_slot*/
+ NFS4ERR_DEADSESSION    = 10078,/*new req sent to dead sess*/
+ NFS4ERR_ENCR_ALG_UNSUPP= 10079,/* encr alg. not supp.     */
+ NFS4ERR_PNFS_NO_LAYOUT = 10080,/* I/O without a layout    */
+ NFS4ERR_NOT_ONLY_OP    = 10081,/* addl ops not allowed    */
+ NFS4ERR_WRONG_CRED     = 10082,/* op done by wrong cred   */
+ NFS4ERR_WRONG_TYPE     = 10083,/* op on wrong type object */
+ NFS4ERR_DIRDELEG_UNAVAIL=10084,/* delegation not avail.   */
+ NFS4ERR_REJECT_DELEG   = 10085,/* cb rejected delegation  */
+ NFS4ERR_RETURNCONFLICT = 10086,/* layout get before return*/
+ NFS4ERR_DELEG_REVOKED  = 10087, /* deleg./layout revoked   */
+ NFS4ERR_PARTNER_NOTSUPP = 10088,
+ NFS4ERR_PARTNER_NO_AUTH = 10089,
+ NFS4ERR_UNION_NOTSUPP = 10090,
+ NFS4ERR_OFFLOAD_DENIED = 10091,
+ NFS4ERR_WRONG_LFS = 10092,
+ NFS4ERR_BADLABEL = 10093,
+ NFS4ERR_OFFLOAD_NO_REQS = 10094,
+ NFS4ERR_NOXATTR = 10095,
+ NFS4ERR_XATTR2BIG = 10096
+};
 
 /*
  * Basic data types
  */
+typedef opaque		attrlist4<>;
 typedef uint32_t	bitmap4<>;
+typedef opaque		verifier4[NFS4_VERIFIER_SIZE];
+typedef uint64_t        nfs_cookie4;
+typedef opaque		nfs_fh4<NFS4_FHSIZE>;
 
 typedef opaque		utf8string<>;
 typedef utf8string	utf8str_cis;
 typedef utf8string	utf8str_cs;
 typedef utf8string	utf8str_mixed;
 
+typedef utf8str_cs      component4;
+typedef utf8str_cs      linktext4;
+typedef component4      pathname4<>;
+
 /*
  * Timeval
  */
@@ -66,6 +209,21 @@ struct nfstime4 {
 	uint32_t	nseconds;
 };
 
+/*
+ * File attribute container
+ */
+struct fattr4 {
+        bitmap4         attrmask;
+        attrlist4       attr_vals;
+};
+
+/*
+ * Stateid
+ */
+struct stateid4 {
+        uint32_t        seqid;
+        opaque          other[12];
+};
 
 /*
  * The following content was extracted from draft-ietf-nfsv4-delstid
@@ -245,3 +403,99 @@ const FATTR4_ACL_TRUEFORM	= 89;
 const FATTR4_ACL_TRUEFORM_SCOPE	= 90;
 const FATTR4_POSIX_DEFAULT_ACL	= 91;
 const FATTR4_POSIX_ACCESS_ACL	= 92;
+
+/*
+ * Directory notification types.
+ */
+enum notify_type4 {
+        NOTIFY4_CHANGE_CHILD_ATTRS = 0,
+        NOTIFY4_CHANGE_DIR_ATTRS = 1,
+        NOTIFY4_REMOVE_ENTRY = 2,
+        NOTIFY4_ADD_ENTRY = 3,
+        NOTIFY4_RENAME_ENTRY = 4,
+        NOTIFY4_CHANGE_COOKIE_VERIFIER = 5,
+        /* Proposed in RFC8881bis */
+        NOTIFY4_GFLAG_EXTEND = 6,
+        NOTIFY4_AUFLAG_VALID = 7,
+        NOTIFY4_AUFLAG_USER = 8,
+        NOTIFY4_AUFLAG_GROUP = 9,
+        NOTIFY4_AUFLAG_OTHER = 10,
+        NOTIFY4_CHANGE_AUTH = 11,
+        NOTIFY4_CFLAG_ORDER = 12,
+        NOTIFY4_AUFLAG_GANOW = 13,
+        NOTIFY4_AUFLAG_GALATER = 14,
+        NOTIFY4_CHANGE_GA = 15,
+        NOTIFY4_CHANGE_AMASK = 16
+};
+
+/* Changed entry information.  */
+struct notify_entry4 {
+        component4      ne_file;
+        fattr4          ne_attrs;
+};
+
+/* Previous entry information */
+struct prev_entry4 {
+        notify_entry4   pe_prev_entry;
+        /* what READDIR returned for this entry */
+        nfs_cookie4     pe_prev_entry_cookie;
+};
+
+struct notify_remove4 {
+        notify_entry4   nrm_old_entry;
+        nfs_cookie4     nrm_old_entry_cookie;
+};
+pragma public notify_remove4;
+
+struct notify_add4 {
+        /*
+         * Information on object
+         * possibly renamed over.
+         */
+        notify_remove4      nad_old_entry<1>;
+        notify_entry4       nad_new_entry;
+        /* what READDIR would have returned for this entry */
+        nfs_cookie4         nad_new_entry_cookie<1>;
+        prev_entry4         nad_prev_entry<1>;
+        bool                nad_last_entry;
+};
+pragma public notify_add4;
+
+struct notify_attr4 {
+        notify_entry4   na_changed_entry;
+};
+pragma public notify_attr4;
+
+struct notify_rename4 {
+        notify_remove4  nrn_old_entry;
+        notify_add4     nrn_new_entry;
+};
+pragma public notify_rename4;
+
+struct notify_verifier4 {
+        verifier4       nv_old_cookieverf;
+        verifier4       nv_new_cookieverf;
+};
+
+/*
+ * Objects of type notify_<>4 and
+ * notify_device_<>4 are encoded in this.
+ */
+typedef opaque notifylist4<>;
+
+struct notify4 {
+        /* composed from notify_type4 or notify_deviceid_type4 */
+        bitmap4         notify_mask;
+        notifylist4     notify_vals;
+};
+
+struct CB_NOTIFY4args {
+        stateid4    cna_stateid;
+        nfs_fh4     cna_fh;
+        notify4     cna_changes<>;
+};
+pragma public CB_NOTIFY4args;
+
+struct CB_NOTIFY4res {
+        nfsstat4    cnr_status;
+};
diff --git a/Documentation/trace/eprobetrace.rst b/Documentation/trace/eprobetrace.rst
index fe3602540569a..680e0af43d5df 100644
--- a/Documentation/trace/eprobetrace.rst
+++ b/Documentation/trace/eprobetrace.rst
@@ -49,7 +49,12 @@ Synopsis of eprobe_events
   (STRUCT)FIELD->MEMBER[->MEMBER] : If BTF is supported, typecast FIELD to
                   a pointer to STRUCT and then derference the pointer defined by
                   ->MEMBER. Note that when this is used, the FIELD name does not
-                  need to be prefixed with a '$'.
+                  need to be prefixed with a '$'. ASGN can be specified optionally.
+		  If ASGN is specified, FIELD will be cast to the same offset
+		  position as the ASGN member, rather than to the beginning of
+		  the STRUCT.
+  (STRUCT)(FETCHARG)->MEMBER[->MEMBER] : typecast can nest, so the above can
+		  also be used with another FETCHARG instead of FIELD.
 
 Types
 -----
diff --git a/Documentation/trace/fprobetrace.rst b/Documentation/trace/fprobetrace.rst
index b4c2ca3d02c16..3439bc9bd351e 100644
--- a/Documentation/trace/fprobetrace.rst
+++ b/Documentation/trace/fprobetrace.rst
@@ -50,13 +50,23 @@ Synopsis of fprobe-events
   $argN         : Fetch the Nth function argument. (N >= 1) (\*2)
   $retval       : Fetch return value.(\*3)
   $comm         : Fetch current task comm.
+  $current      : Fetch the address of the current task_struct.
   +|-[u]OFFS(FETCHARG) : Fetch memory at FETCHARG +|- OFFS address.(\*4)(\*5)
+  this_cpu_read(FETCHARG) : Read the value of the per-CPU variable FETCHARG on the current CPU.
+  this_cpu_ptr(FETCHARG) : Get the address of the per-CPU variable FETCHARG on the current CPU.
   \IMM          : Store an immediate value to the argument.
   NAME=FETCHARG : Set NAME as the argument name of FETCHARG.
   FETCHARG:TYPE : Set TYPE as the type of FETCHARG. Currently, basic types
                   (u8/u16/u32/u64/s8/s16/s32/s64), hexadecimal types
                   (x8/x16/x32/x64), "char", "string", "ustring", "symbol", "symstr"
                   and bitfield are supported.
+  (STRUCT[,ASGN])FIELD->MEMBER[->MEMBER] : If BTF is supported, typecast FIELD to
+                  a pointer to STRUCT and then derference the pointer defined by
+                  ->MEMBER. ASGN can be specified optionally. If ASGN is specified,
+		  FIELD will be cast to the same offset position as the ASGN member,
+		  rather than to the beginning of the STRUCT.
+  (STRUCT[,ASGN])(FETCHARG)->MEMBER[->MEMBER] : typecast can nest, so the above can
+                 also be used with another FETCHARG instead of FIELD.
 
   (\*1) This is available only when BTF is enabled.
   (\*2) only for the probe on function entry (offs == 0). Note, this argument access
diff --git a/Documentation/trace/kprobetrace.rst b/Documentation/trace/kprobetrace.rst
index 3b6791c17e9b6..9ae330eb0a524 100644
--- a/Documentation/trace/kprobetrace.rst
+++ b/Documentation/trace/kprobetrace.rst
@@ -53,7 +53,10 @@ Synopsis of kprobe_events
   $argN		: Fetch the Nth function argument. (N >= 1) (\*1)
   $retval	: Fetch return value.(\*2)
   $comm		: Fetch current task comm.
+  $current      : Fetch the address of the current task_struct.
   +|-[u]OFFS(FETCHARG) : Fetch memory at FETCHARG +|- OFFS address.(\*3)(\*4)
+  this_cpu_read(FETCHARG) : Read the value of the per-CPU variable FETCHARG on the current CPU.
+  this_cpu_ptr(FETCHARG) : Get the address of the per-CPU variable FETCHARG on the current CPU.
   \IMM		: Store an immediate value to the argument.
   NAME=FETCHARG : Set NAME as the argument name of FETCHARG.
   FETCHARG:TYPE : Set TYPE as the type of FETCHARG. Currently, basic types
@@ -61,6 +64,14 @@ Synopsis of kprobe_events
 		  (x8/x16/x32/x64), VFS layer common type(%pd/%pD), "char",
                   "string", "ustring", "symbol", "symstr" and bitfield are
                   supported.
+  (STRUCT[,ASGN])FIELD->MEMBER[->MEMBER] : If BTF is supported, typecast FIELD to
+                  a pointer to STRUCT and then derference the pointer defined by
+                  ->MEMBER. Note that this is available only when the probe is
+		   on function entry. ASGN can be specified optionally. If ASGN
+		   is specified, FIELD will be cast to the same offset position
+		   as the ASGN member, rather than to the beginning of the STRUCT.
+  (STRUCT[,ASGN])(FETCHARG)->MEMBER[->MEMBER] : typecast can nest, so the above can
+                 also be used with another FETCHARG instead of FIELD.
 
   (\*1) only for the probe on function entry (offs == 0). Note, this argument access
         is best effort, because depending on the argument type, it may be passed on
diff --git a/Documentation/translations/ja_JP/process/submitting-patches.rst b/Documentation/translations/ja_JP/process/submitting-patches.rst
index d31d469909e4e..d8ee82ba790bf 100644
--- a/Documentation/translations/ja_JP/process/submitting-patches.rst
+++ b/Documentation/translations/ja_JP/process/submitting-patches.rst
@@ -182,7 +182,7 @@ URL は禁止です。
 変更を分割する
 --------------
 
-各 **論理的な変更** は、個別のパッチに分けてください。
+それぞれの\ **論理的な変更**\ は、個別のパッチに分けてください。
 
 たとえば、単一のドライバに対する変更にバグ修正と性能改善の
 両方が含まれるなら、それらは 2 つ以上のパッチに分けてください。
@@ -208,7 +208,7 @@ URL は禁止です。
 ことがあります。途中でバグを持ち込めば、彼らに感謝されることは
 ないでしょう。
 
-パッチセットをこれ以上小さくできないなら、一度に投稿するのは
+パッチセットをそれ以上小さくできないなら、一度に投稿するのは
 15 個程度までにして、レビューと統合を待ってください。
 
 
@@ -220,18 +220,17 @@ Documentation/process/coding-style.rst を参照してください。
 これを怠ると、単にレビューアの時間を無駄にするだけでなく、
 パッチはおそらく読まれもせずに却下されます。
 
-大きな例外が 1 つあります。コードをあるファイルから別の
-ファイルへ移動する場合です。このときは、コードを移動する
-その同じパッチの中で、移動したコードを一切変更してはいけません。
-そうすることで、コードの移動という行為と、あなたの変更とを
-明確に区別できます。これは実際の差分のレビューを大いに助け、
-ツールがコード自体の履歴をより適切に追跡できるようにします。
+一つの重要な例外は、コードをあるファイルから別のファイルへ移動する場合です。
+その際は、コードを移動するその同じパッチの中で、一切コードを変更しては
+いけません。これにより、コードの移動という行為と、コードの変更とが
+明確に区別されます。これは実際の差分のレビューを大いに助け、また、ツールを
+使ったコード変更の履歴の追跡を容易にします。
 
 提出前に、パッチスタイルチェッカー
 (``scripts/checkpatch.pl``) でパッチを確認してください。
-ただし、スタイルチェッカーは指針として見るべきであり、
+ただし、スタイルチェッカーは指針にすぎず、
 人間の判断に取って代わるものではないことに注意してください。
-違反があっても、その方がコードの見栄えがよいなら、
+違反が指摘されるままのコードの方が見栄えがよいなら、おそらく
 そのままにしておくのが最善でしょう。
 
 チェッカーは 3 つのレベルで報告します:
@@ -240,8 +239,7 @@ Documentation/process/coding-style.rst を参照してください。
  - WARNING: 慎重なレビューを要するもの
  - CHECK: 検討を要するもの
 
-パッチに残した違反については、すべて理由を説明できなければ
-なりません。
+パッチに違反を残す場合は、そのすべてを正当化できなければなりません。
 
 
 パッチの宛先を選択する
@@ -256,8 +254,8 @@ Documentation/process/coding-style.rst を参照してください。
 サブシステムのメンテナが見つからない場合は、Andrew Morton
 (akpm@linux-foundation.org) が最後の手段となるメンテナです。
 
-すべてのパッチでは、デフォルトで linux-kernel@vger.kernel.org を
-使うべきですが、このリストの流量が多いため、目を通さなくなった
+すべてのパッチは、デフォルトで linux-kernel@vger.kernel.org にも
+送られるべきですが、このリストは流量が多く、目を通さなくなった
 開発者も少なくありません。とはいえ、無関係なメーリングリストや
 無関係な人々にスパムを送らないでください。
 
@@ -268,103 +266,104 @@ Documentation/process/coding-style.rst を参照してください。
 Linux カーネルに採用されるすべての変更の最終的な裁定者は
 Linus Torvalds です。彼のメールアドレスは
 <torvalds@linux-foundation.org> です。Linus は大量のメールを
-受け取っており、現時点では彼に直接届くパッチはごくわずかなので、
-通常は彼にメールを送ることを極力避けてください。
+受け取っており、現時点では直接彼を経由するパッチはごくわずかなので、
+通常は彼にメールを送ることを極力\ **避けて**\ ください。
 
-悪用可能なセキュリティバグを修正するパッチがあるなら、
-そのパッチを security@kernel.org に送ってください。深刻なバグに
+悪用可能なセキュリティバグを修正するパッチの場合は、
+それを security@kernel.org に送ってください。深刻なバグに
 ついては、ディストリビュータがユーザーにパッチを配布できるよう、
-短期間の embargo が検討される場合があります。そのような場合、
-そのパッチを公開メーリングリストに送るべきではありません。
+短期間の秘匿措置 (訳註: embargo) が検討される可能性があります。
+ですので、その種のパッチを公開メーリングリストに送らないでください。
 Documentation/process/security-bugs.rst も参照してください。
 
 リリース済みカーネルの深刻なバグを修正するパッチは、次のような行を
-パッチの sign-off 欄に入れることで、stable メンテナへ向けてください::
+パッチの sign-off 欄に入れることで、stable メンテナに知らせてください。
+(メールの宛先ではないことに注意。) ::
 
   Cc: stable@vger.kernel.org
 
-これはメールの受信者ではないことに注意してください。また、
-この文書に加えて Documentation/process/stable-kernel-rules.rst も
-読んでください。
+また、この文書に加えて Documentation/process/stable-kernel-rules.rst
+も読んでください。
 
 変更がユーザーランドとカーネルのインターフェースに影響する場合は、
 MAINTAINERS ファイルに記載されている MAN-PAGES メンテナに
-man-pages パッチ、少なくとも変更の通知を送って、情報が
-マニュアルページに反映されるようにしてください。ユーザー空間 API の
+マニュアルページのパッチ、もしくは少なくとも変更の通知を送って、情報が
+そちらにも反映されるようにしてください。ユーザー空間 API の
 変更は、linux-api@vger.kernel.org にも Cc してください。
 
 MIME・リンク・圧縮・添付なし、プレーンテキストのみ
 ----------------------------------------------------
 
 Linus や他のカーネル開発者は、あなたが投稿する変更を読み、
-コメントできる必要があります。カーネル開発者が標準的な
-メールツールを使ってあなたの変更を「引用」し、コードの特定の
-箇所についてコメントできることが重要です。
+コメントできる必要があります。カーネル開発者にとって、コードの特定の
+箇所について、標準的なメールツールを使ってあなたの変更を「引用」し、
+コメントできることが重要です。
 
-このため、すべてのパッチはメール本文中に ``inline`` で投稿すべきです。
+このため、すべてのパッチはメール本文中に「インライン」で投稿すべきです。
 これを行う最も簡単な方法は ``git send-email`` を使うことであり、
 強く推奨されます。``git send-email`` の対話型チュートリアルは
-https://git-send-email.io で利用できます。
+https://git-send-email.io にあります。
 
-``git send-email`` を使わないことを選ぶ場合:
+``git send-email`` を使わない場合:
 
 .. warning::
 
-  パッチをコピー＆ペーストする場合は、エディタの word-wrap によって
-  パッチが壊れないよう注意してください。
+  パッチをコピー＆ペーストする際に、エディタによる自動改行で
+  パッチが壊されないよう注意してください。
 
 圧縮の有無にかかわらず、パッチを MIME 添付ファイルとして添付しては
-いけません。多くの一般的なメールアプリケーションは、MIME 添付
-ファイルを常にプレーンテキストとして送信するとは限らず、あなたの
-コードにコメントできなくなります。MIME 添付ファイルは Linus が
-処理するのにも少し余分な時間がかかるため、MIME 添付された変更が
-受け入れられる可能性を下げます。
+いけません。よく使われるメールアプリケーションの多くは、MIME 添付
+ファイルをプレーンテキストとして送信するとは限らず、あなたのコードに
+対するコメントを妨げます。MIME 添付ファイルは Linus (訳補: をはじめ
+とする開発者）が処理するのに余分な手間がかかるため、MIME 添付すると
+その変更が受け入れられる可能性を下げることになります。
 
-例外:  メーラがパッチを壊してしまう場合は、誰かから MIME を使って
-再送するよう求められることがあります。
+例外:  パッチがメーラーによって壊されている場合に、MIME による再送
+を求められることがあります。
 
-パッチを変更せずに送信するようメールクライアントを設定するための
-ヒントについては、Documentation/process/email-clients.rst を参照してください。
+改変なしにパッチを送信するためのメールクライアント設定のヒントは、
+Documentation/process/email-clients.rst を参照してください。
 
 
-レビューコメントに返答する
+レビューコメントに応答する
 --------------------------
 
-あなたのパッチには、ほぼ確実に、パッチを改善する方法について
-レビューアからコメントが付きます。それは、あなたのメールへの返信という
-形で届きます。それらのコメントには必ず返答してください。レビューアを
-無視することは、こちらも無視されるためのよい方法です。コメントに
-答えるには、単にそのメールへ返信すれば構いません。コード変更に
+あなたのパッチには、ほぼ確実に、その改善に向けてレビューアから
+コメントが付きます。それは、あなたのメールへの返信という
+形で届きます。それらのコメントには必ず応答してください。レビューアを
+無視することは、あなたが無視されることにつながります。コメントに
+答えるには、単にそのメールへ返信すればよいです。コード変更に
 つながらないレビューコメントや質問であっても、次のレビューアが状況を
-よりよく理解できるように、ほぼ確実にコメントまたは changelog エントリに
-反映すべきです。
-
-どのような変更を行うのかをレビューアに必ず伝え、時間を割いてくれた
-ことに感謝してください。コードレビューは疲れる、時間のかかる作業であり、
-レビューアが不機嫌になることもあります。そのような場合であっても、
-丁寧に返答し、指摘された問題に対応してください。次の版を送るときは、
-cover letter または個々のパッチに ``patch changelog`` を追加し、前回の
+よりよく理解できるよう、多くの場合、コメントまたは changelog エントリ
+として残すべきです。
+
+どのような変更を行うのかを忘れずにレビューアに伝えてください。そして
+時間を割いてくれることへの感謝を忘れないでください。
+コードレビューは疲れる、時間のかかる作業であり、
+ときにはレビューアが機嫌を損ねることもあります。そのような場合でも、
+丁寧に応答し、指摘された問題に対応してください。次の版を送る際には、
+カバーレターまたは個々のパッチに ``patch changelog`` を追加し、前回の
 投稿との差分を説明してください。詳細は原文の該当節
 ("The canonical patch format") を参照してください。
 
 .. TODO: Convert to file-local cross-reference when the destination is
    translated.
 
-あなたのパッチにコメントした人には、パッチの Cc リストに追加して、
-新しい版を知らせてください。
+あなたのパッチにコメントしてくれた人たちは、パッチの Cc リストに追加して
+新しい版について知らせてください。
 
 メールクライアントとメーリングリストでの作法についての推奨事項は、
 Documentation/process/email-clients.rst を参照してください。
 
-メール議論では不要な引用を削った interleaved replies を使う
-------------------------------------------------------------
+要点に絞ったインライン返信での議論
+---------------------------------------
 
-Linux カーネル開発の議論では、top-posting は強く非推奨とされています。
-Interleaved replies、または ``inline`` replies を使うと、会話の流れを
-ずっと追いやすくなります。詳細は次を参照してください:
+Linux カーネル開発の議論では、全文引用 (訳註: top-posting) は強く非推奨です。
+インライン返信 (訳註: interleaved reples or "inline" replies) を使うと、
+会話の流れをずっと追いやすくなります。詳細は次を参照してください:
 https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
 
-メーリングリストでは、よく次のように引用されます::
+これについて、メーリングリストでは、次の引用をしばしば目にします::
 
   A: http://en.wikipedia.org/wiki/Top_post
   Q: Where do I find info about this thing called top-posting?
@@ -381,24 +380,25 @@ https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
   Q: Should I include quotations after my reply?
 
 
-落胆しない、そして急がない
---------------------------
+落胆しない - いらいらしない
+---------------------------
 
 変更を投稿した後は、辛抱強く待ってください。レビューアは忙しい人たちであり、
-あなたのパッチをすぐに見られるとは限りません。
+あなたのパッチにすぐに取りかかれるとは限りません。
 
 かつては、パッチが何のコメントもなく虚空へ消えていくこともありましたが、
 現在の開発プロセスはそれよりも円滑に機能しています。数週間以内、
 通常は 2〜3 週間以内にコメントを受け取るはずです。そうならない場合は、
 パッチを正しい場所へ送ったか確認してください。再投稿したりレビューアに
-ping したりする前に、少なくとも 1 週間は待ってください。merge window の
-ような忙しい時期には、さらに長く待つ方がよい場合もあります。
+ping したりする前に、少なくとも 1 週間は待ってください。マージ期間
+(訳註: merge window) のような忙しい時期には、さらに長く待ちましょう。
 
-数週間後に、subject line に "RESEND" を追加して、パッチまたは
-パッチシリーズを再送しても構いません::
+数週間後に、件名に "RESEND" を追加したパッチまたはパッチシリーズを
+再送することは構いません::
 
    [PATCH Vx RESEND] sub/sys: Condensed patch summary
 
-パッチまたはパッチシリーズの修正版を投稿する場合は、"RESEND" を
-追加しないでください。"RESEND" は、前回の投稿から一切変更していない
-パッチまたはパッチシリーズを再送する場合にのみ使います。
+ただし、パッチまたはパッチシリーズの修正版を投稿する際には "RESEND"
+を追加しないでください。
+"RESEND" は、前回の投稿から一切変更のないパッチまたはパッチシリーズの
+再送だけに当てはまります。
diff --git a/Documentation/translations/pt_BR/index.rst b/Documentation/translations/pt_BR/index.rst
index 7a488f662a1e3..232d18f7cfce9 100644
--- a/Documentation/translations/pt_BR/index.rst
+++ b/Documentation/translations/pt_BR/index.rst
@@ -68,13 +68,24 @@ kernel e sobre como ver seu trabalho integrado.
 
    Introdução <process/1.Intro>
    Guia do Processo de Desenvolvimento <process/development-process>
+   Como aplicar patches <process/applying-patches>
+   Backporting e resolução de conflitos <process/backporting>
+   Como não Deixar as ioctls malfeitas <process/botching-up-ioctls>
    Index de documentos do Kernel <process/kernel-docs>
    Regras de licenciamento <process/license-rules>
    Como começar <process/howto>
    Requisitos mínimos <process/changes>
    Conclave (Continuidade do projeto) <process/conclave>
+   Modelos de Maturidade para Contribuição no Kernel Linux <process/contribution-maturity-model.rst>
    Manuais dos mantenedores <process/maintainer-handbooks>
    Processo do subsistema de rede (netdev) <process/maintainer-netdev>
    Processo do subsistema SoC <process/maintainer-soc>
    Conformidade de DTS para SoC <process/maintainer-soc-clean-dts>
    Processo do subsistema KVM x86 <process/maintainer-kvm-x86>
+   Adicionando uma nova chamada de Sistema <process/adding-syscalls>
+   Declaração sobre Drivers do Kernel <process/kernel-driver-statement>
+   Lista de verificação para submissão de patches do kernel Linux <process/submit-checklist>
+   Interpretação do Código de Conduta do Kernel Linux <process/code-of-conduct-interpretation>
+   Código de Conduta de Compromisso do Colaborador <process/code-of-conduct>
+   Interfaces, recursos de linguagem, atributos e convenções obsoletos <process/deprecated>
+
diff --git a/Documentation/translations/pt_BR/process/4.Coding.rst b/Documentation/translations/pt_BR/process/4.Coding.rst
new file mode 100644
index 0000000000000..ca4c74774a910
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/4.Coding.rst
@@ -0,0 +1,440 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Escrever o código corretamente
+==============================
+
+Embora haja muito o que se dizer sobre um processo de design sólido e orientado
+à comunidade, a prova de qualquer projeto de desenvolvimento de kernel está no
+código resultante. É o código que será examinado por outros desenvolvedores e
+mesclado (ou não) na árvore principal (*mainline*). Portanto, é a qualidade
+deste código que determinará o sucesso final do projeto.
+
+Esta seção examinará o processo de codificação. Começaremos analisando uma série
+de maneiras pelas quais os desenvolvedores de kernel podem errar. Em seguida, o
+foco mudará para como fazer as coisas do jeito certo e as ferramentas que podem
+ajudar nessa busca.
+
+
+Armadilhas
+----------
+
+Estilo de Codificação
+*********************
+
+O kernel há muito possui um estilo de codificação padrão, descrito em
+:ref:`Documentation/process/coding-style.rst <codingstyle>`. Por grande parte
+desse tempo, as políticas descritas naquele arquivo eram consideradas, no
+máximo, como recomendações. Como resultado, há uma quantidade substancial
+de código no kernel que não cumpre as diretrizes de estilo de codificação.
+A presença desse código leva a dois riscos independentes para os
+desenvolvedores do kernel.
+
+O primeiro deles é acreditar que os padrões de codificação do kernel não importam
+e não são exigidos. A verdade é que adicionar novo código ao kernel é muito
+difícil se esse código não estiver escrito de acordo com o padrão; muitos
+desenvolvedores solicitarão que o código seja reformatado antes mesmo de
+revisá-lo. Uma base de código tão grande quanto a do kernel exige certa
+uniformidade para tornar possível que os desenvolvedores entendam rapidamente
+qualquer parte dela. Portanto, não há mais espaço para códigos com formatações
+estranhas.
+
+Ocasionalmente, o estilo de codificação do kernel entrará em conflito com o
+estilo exigido por um empregador. Nesses casos, o estilo do kernel terá que
+vencer para que o código possa ser mesclado. Colocar código no kernel significa
+abrir mão de um certo grau de controle de várias maneiras — incluindo o controle
+sobre como o código é formatado.
+
+A outra armadilha é presumir que o código já presente no kernel necessita
+urgentemente de correções de estilo de codificação. Os desenvolvedores podem
+começar a gerar patches de reformatação como uma forma de ganhar familiaridade
+com o processo, ou como um meio de incluir seus nomes nos logs de alterações
+(*changelogs*) do kernel  ou ambos. No entanto, patches puramente de estilo de
+codificação são vistos como ruído pela comunidade de desenvolvimento; eles tendem
+a receber uma recepção fria. Portanto, é melhor evitar esse tipo de patch. É
+natural corrigir o estilo de um trecho de código ao trabalhar nele por outros
+motivos, mas mudanças de estilo de codificação não devem ser feitas apenas por
+fazer.
+
+O documento de estilo de codificação também não deve ser lido como uma lei
+absoluta que nunca pode ser transgredida. Se houver um bom motivo para ir contra
+o estilo (uma linha que se torna muito menos legível se for dividida para caber
+no limite de 80 colunas, por exemplo), simplesmente faça isso.
+
+Note que você também pode usar a ferramenta ``clang-format`` para ajudá-lo com
+essas regras, para reformatar rapidamente partes do seu código de forma automática
+e para revisar arquivos completos a fim de identificar erros de estilo de
+codificação, erros de digitação e possíveis melhorias. Ela também é útil para
+ordenar ``#includes``, alinhar variáveis/macros, reajustar o fluxo de textos e
+outras tarefas semelhantes. Veja o arquivo
+:ref:`Documentation/dev-tools/clang-format.rst <clangformat>` para mais detalhes.
+
+Algumas configurações básicas do editor, como indentação e fins de linha,
+serão definidas automaticamente se você estiver usando um editor compatível
+com o EditorConfig. Consulte o site oficial do EditorConfig para obter mais
+informações: https://editorconfig.org/
+
+Camadas de Abstração
+********************
+
+Os professores de Ciência da Computação ensinam os alunos a fazerem uso
+extensivo de camadas de abstração em nome da flexibilidade e da ocultação de
+informações. Certamente o kernel faz uso extensivo de abstração; nenhum
+projeto que envolva vários milhões de linhas de código poderia fazer o
+contrário e sobreviver. No entanto, a experiência tem mostrado que a
+abstração excessiva ou prematura pode ser tão prejudicial quanto a otimização
+prematura. A abstração deve ser usada até o nível necessário e não além.
+
+Em um nível simples, considere uma função que possui um argumento que é
+sempre passado como zero por todos os chamadores. Alguém poderia manter esse
+argumento caso alguém eventualmente precise usar a flexibilidade extra que ele
+oferece. A essa altura, no entanto, as chances são grandes de que o código que
+implementa esse argumento extra tenha sido quebrado de alguma forma sutil que
+nunca foi percebida — porque ele nunca foi usado. Ou, quando surge a
+necessidade de flexibilidade extra, ela não ocorre de uma forma que corresponda
+à expectativa inicial do programador. Os desenvolvedores do kernel enviam
+patches rotineiramente para remover argumentos não utilizados; eles não devem,
+em geral, ser adicionados em primeiro lugar.
+
+Camadas de abstração que ocultam o acesso ao hardware — frequentemente para
+permitir que a maior parte de um driver seja usada com múltiplos sistemas
+operacionais — são especialmente malvistas. Essas camadas obscurecem o código
+e podem impor uma penalidade de desempenho; elas não pertencem ao kernel
+Linux.
+
+Por outro lado, se você se pegar copiando quantidades significativas de código
+de outro subsistema do kernel, é hora de perguntar se faria sentido, de fato,
+extrair parte desse código em uma biblioteca separada ou implementar essa
+funcionalidade em um nível superior. Não há valor em duplicar o mesmo código
+por todo o kernel.
+
+
+Uso de #ifdef e do pré-processador em geral
+*******************************************
+
+O pré-processador C parece apresentar uma forte tentação para alguns
+programadores C, que o veem como uma forma de codificar eficientemente uma grande
+quantidade de flexibilidade em um arquivo-fonte. No entanto, o pré-processador
+não é C, e o uso pesado dele resulta em um código muito mais difícil de ser lido
+por outros e mais difícil para o compilador verificar a correção. O uso pesado
+do pré-processador é quase sempre um sinal de código que precisa de algum
+trabalho de limpeza.
+
+A compilação condicional com #ifdef é, de fato, um recurso poderoso, e é
+utilizada dentro do kernel. Mas há pouco desejo de ver um código que seja
+salpicado liberalmente com blocos #ifdef. Como regra geral, o uso de #ifdef
+deve ser confinado a arquivos de cabeçalho (headers) sempre que possível. O
+código compilado condicionalmente pode ser confinado a funções que, se o código
+não estiver presente, simplesmente se tornam vazias. O compilador irá então,
+silenciosamente, otimizar e remover a chamada para a função vazia. O resultado
+é um código muito mais limpo e fácil de acompanhar.
+
+As macros do pré-processador C apresentam uma série de riscos, incluindo a
+possível avaliação múltipla de expressões com efeitos colaterais e a falta de
+segurança de tipos. Se você se sentir tentado a definir uma macro, considere a
+criação de uma função inline em seu lugar. O código resultante será o mesmo,
+mas as funções inline são mais fáceis de ler, não avaliam seus argumentos
+múltiplas vezes e permitem que o compilador realize a checagem de tipos nos
+argumentos e no valor de retorno.
+
+
+Funções Inline
+**************
+
+No entanto, as funções inline apresentam um perigo próprio. Os programadores
+podem ficar encantados com a eficiência percebida inerente a evitar uma chamada
+de função e encher um arquivo de código-fonte com funções inline. Essas
+funções, contudo, podem na verdade reduzir o desempenho. Como seu código é
+replicado em cada local de chamada, elas acabam inflando o tamanho do kernel
+compilado. Isso, por sua vez, cria pressão nos caches de memória do
+processador, o que pode desacelerar a execução drasticamente. As funções
+inline, como regra, devem ser bastante pequenas e relativamente raras. O custo
+de uma chamada de função, afinal de contas, não é tão alto; a criação de um
+grande número de funções inline é um exemplo clássico de otimização prematura.
+
+Em geral, os programadores de kernel ignoram os efeitos de cache por sua própria
+conta e risco. O clássico compromisso entre tempo e espaço (tradeoff) ensinado
+nas aulas introdutórias de estruturas de dados frequentemente não se aplica ao
+hardware contemporâneo. Espaço *é* tempo, no sentido de que um programa maior
+será executado mais lentamente do que um que seja mais compacto.
+
+Compiladores mais recentes desempenham um papel cada vez mais ativo em decidir
+se uma determinada função deve ou não ser realmente inline. Portanto, a inserção
+liberal da palavra-chave "inline" pode não apenas ser excessiva; ela também pode
+ser irrelevante.
+
+
+Mecanismo de Trava
+******************
+
+Em maio de 2006, a pilha de rede "Devicescape" foi, com grande alarde, lançada
+sob a GPL e disponibilizada para inclusão no kernel mainline. Essa doação foi uma
+notícia bem-vinda; o suporte para redes sem fio no Linux era considerado abaixo do
+padrão, na melhor das hipóteses, e a pilha da Devicescape oferecia a promessa de
+corrigir essa situação. No entanto, esse código só entrou de fato no mainline em
+junho de 2007 (2.6.22). O que aconteceu?
+
+Esse código mostrava vários sinais de ter sido desenvolvido a portas fechadas em
+ambiente corporativo. Mas um grande problema em particular era que ele não havia
+sido projetado para funcionar em sistemas multiprocessados. Antes que essa pilha
+de rede (agora chamada de mac80211) pudesse ser integrada, um esquema de locking
+(bloqueio) precisou ser adaptado a ela.
+
+Era uma vez uma época em que o código do kernel Linux podia ser desenvolvido sem
+pensar nos problemas de concorrência apresentados por sistemas multiprocessados.
+Hoje, no entanto, este documento está sendo escrito em um laptop dual-core.
+Mesmo em sistemas com um único processador, o trabalho feito para melhorar a
+capacidade de resposta aumentará o nível de concorrência dentro do kernel. Os
+dias em que o código do kernel podia ser escrito sem pensar em locking ficaram
+há muito tempo no passado.
+
+Qualquer recurso (estruturas de dados, registradores de hardware, etc.) que
+possa ser acessado concorrentemente por mais de uma linha de execução deve ser
+protegido por uma trava (lock). O novo código deve ser escrito com esse
+requisito em mente; adaptar o locking após o fato é uma tarefa consideravelmente
+mais difícil. Os desenvolvedores do kernel devem dedicar um tempo para
+compreender as primitivas de locking disponíveis bem o suficiente para escolher
+a ferramenta certa para o trabalho. Códigos que mostrem falta de atenção à
+concorrência terão um caminho difícil para entrar no mainline.
+
+
+Regressions
+***********
+
+Um perigo final que vale a pena mencionar é este: pode ser tentador fazer uma
+alteração (que pode trazer grandes melhorias) que faça algo quebrar para os
+usuários existentes. Esse tipo de alteração é chamado de "regressão", e as
+regressões tornaram-se totalmente indesejadas no kernel mainline. Com poucas
+exceções, as alterações que causarem regressões serão revertidas se a regressão
+não puder ser corrigida em tempo hábil. É muito melhor evitar a regressão em
+primeiro lugar.
+
+Muitas vezes argumenta-se que uma regressão pode ser justificada se ela fizer as
+coisas funcionarem para mais pessoas do que os problemas que ela cria. Por que
+não fazer uma alteração se ela trouxer uma nova funcionalidade para dez sistemas
+para cada um que ela quebrar? A melhor resposta para essa pergunta foi expressa
+por Linus em julho de 2007:
+
+::
+
+  Portanto, nós não corrigimos bugs introduzindo novos problemas. Esse caminho
+  leva à loucura, e ninguém nunca sabe se você está realmente fazendo algum
+  progresso real. São dois passos para frente, um passo para trás, ou um passo
+  para frente e dois passos para trás?
+
+(https://lwn.net/Articles/243460/).
+
+Um tipo de regressão especialmente indesejado é qualquer tipo de alteração na
+ABI do espaço do usuário (user-space ABI). Uma vez que uma interface tenha sido
+exportada para o espaço do usuário, ela deve receber suporte indefinidamente.
+Esse fato torna a criação de interfaces de espaço do usuário particularmente
+desafiadora: já que elas não podem ser alteradas de maneiras incompatíveis, elas
+devem ser feitas corretamente na primeira vez. Por essa razão, exige-se sempre
+muita reflexão, documentação clara e uma ampla revisão para as interfaces do
+espaço do usuário.
+
+
+Ferramentas de verificação de código
+------------------------------------
+
+Por enquanto, pelo menos, a escrita de código livre de erros continua sendo um
+ideal que poucos de nós conseguem alcançar. O que podemos esperar fazer, no
+entanto, é capturar e corrigir o máximo possível desses erros antes que nosso
+código entre no kernel mainline. Para esse fim, os desenvolvedores do kernel
+reuniram um conjunto impressionante de ferramentas que podem capturar uma ampla
+variedade de problemas obscuros de forma automatizada. Qualquer problema
+capturado pelo computador é um problema que não afligirá um usuário mais tarde,
+portanto, é lógico que as ferramentas automatizadas devem ser usadas sempre que
+possível.
+
+O primeiro passo é simplesmente prestar atenção aos avisos (warnings) produzidos
+com o compilador. As versões contemporâneas do gcc podem detectar (e alertar
+sobre) um grande número de erros potenciais. Com bastante frequência, esses
+avisos apontam para problemas reais. O código enviado para revisão deve, como
+regra, não produzir nenhum aviso do compilador. Ao silenciar os avisos, tome o
+cuidado de entender a real causa e tente evitar "correções" que façam o aviso
+desaparecer sem resolver a sua origem.
+
+Note que nem todos os avisos do compilador ficam ativados por padrão. Compile o
+kernel com "make KCFLAGS=-W" para obter o conjunto completo.
+
+O kernel fornece várias opções de configuração que ativam recursos de
+depuração; a maioria delas é encontrada no submanu "kernel hacking". Várias
+dessas opções devem ser ativadas para qualquer kernel usado para fins de
+desenvolvimento ou teste. Em particular, você deve ativar:
+
+ - FRAME_WARN para obter avisos sobre quadros de pilha (stack frames) maiores
+   que um determinado valor. A saída gerada pode ser volumosa, mas não é
+   necessário se preocupar com os avisos de outras partes do kernel.
+
+ - DEBUG_OBJECTS adicionará código para rastrear o tempo de vida de vários
+   objetos criados pelo kernel e alertará quando as ações forem feitas fora de
+   ordem. Se você estiver adicionando um subsistema que cria (e exporta) seus
+   próprios objetos complexos, considere adicionar suporte à infraestrutura de
+   depuração de objetos.
+
+ - DEBUG_SLAB pode encontrar uma variedade de erros de alocação e uso de
+   memória; ele deve ser usado na maioria dos kernels de desenvolvimento.
+
+ - DEBUG_SPINLOCK, DEBUG_ATOMIC_SLEEP e DEBUG_MUTEXES encontrarão uma série de
+   erros comuns de locking (bloqueio).
+
+Existem várias outras opções de depuração, algumas das quais serão discutidas
+abaixo. Algumas delas têm um impacto significativo no desempenho e não devem ser
+usadas o tempo todo. Mas um tempo gasto aprendendo as opções disponíveis
+provavelmente se pagará muitas vezes em pouco tempo.
+
+Uma das ferramentas de depuração mais pesadas é o verificador de locking, ou
+"lockdep". Esta ferramenta rastreará a aquisição e a liberação de cada trava
+(spinlock ou mutex) no sistema, a ordem em que as travas são adquiridas umas em
+relação às outras, o ambiente de interrupção atual e muito mais. Ela pode,
+então, garantir que as travas sejam sempre adquiridas na mesma ordem, que as
+mesmas suposições de interrupção se apliquem em todas as situações e assim por
+diante. Em outras palavras, o lockdep pode encontrar uma série de cenários nos
+quais o sistema poderia, em raras ocasiões, entrar em deadlock. Esse tipo de
+problema pode ser doloroso (tanto para desenvolvedores quanto para usuários) em
+um sistema implantado; o lockdep permite que eles sejam encontrados de maneira
+automatizada e antecipada. Códigos com qualquer tipo de locking não trivial
+devem ser executados com o lockdep ativado antes de serem enviados para inclusão.
+
+Como um programador de kernel diligente, você irá, sem dúvida, verificar o
+status de retorno de qualquer operação (como uma alocação de memória) que possa
+falhar. O fato, porém, é que os caminhos de recuperação de falha resultantes
+estão, provavelmente, completamente não testados. Código não testado tende a ser
+código quebrado; você poderia estar muito mais confiante em seu código se todos
+esses caminhos de tratamento de erros tivessem sido exercitados algumas vezes.
+
+O kernel fornece um framework de injeção de falhas (fault injection) que pode
+fazer exatamente isso, especialmente onde alocações de memória estão
+envolvidas. Com a injeção de falhas ativada, uma porcentagem configurável das
+alocações de memória será forçada a falhar; essas falhas podem ser restritas a
+um intervalo específico de código. Executar o código com a injeção de falhas
+ativada permite ao programador ver como o código responde quando as coisas vão
+mal. Veja Documentation/fault-injection/fault-injection.rst para mais
+informações sobre como usar esse recurso.
+
+Outros tipos de erros podem ser encontrados com a ferramenta de análise estática
+"sparse". Com o sparse, o programador pode ser alertado sobre confusões entre
+endereços do espaço do usuário e do espaço do kernel, mistura de quantidades
+big-endian e small-endian, a passagem de valores inteiros onde um conjunto de
+sinalizadores de bits (bit flags) é esperado, e assim por diante. O sparse deve
+ser instalado separadamente (ele pode ser encontrado em
+https://sparse.wiki.kernel.org/index.php/Main_Page se a sua distribuição não o
+incluir como pacote); ele pode então ser executado no código adicionando "C=1"
+ao seu comando make.
+
+A ferramenta "Coccinelle" (http://coccinelle.lip6.fr/) é capaz de encontrar uma
+ampla variedade de potenciais problemas de codificação; ela também pode propor
+correções para esses problemas. Uma quantidade considerável de "patches
+semânticos" para o kernel foi empacotada sob o diretório scripts/coccinelle;
+executar "make coccicheck" passará por esses patches semânticos e relatará
+quaisquer problemas encontrados. Veja
+:ref:`Documentation/dev-tools/coccinelle.rst <devtools_coccinelle>`
+para mais informações.
+
+Outros tipos de erros de portabilidade são encontrados mais facilmente ao
+compilar seu código para outras arquiteturas. Se você por acaso não tiver um
+sistema S/390 ou uma placa de desenvolvimento Blackfin à mão, ainda assim poderá
+realizar a etapa de compilação. Um grande conjunto de compiladores cruzados
+(cross-compilers) para sistemas x86 pode ser encontrado em:
+
+  https://www.kernel.org/pub/tools/crosstool/
+
+Um tempo gasto instalando e usando esses compiladores ajudará a evitar
+constrangimentos mais tarde.
+
+
+Documentação
+-------------
+
+A documentação frequentemente tem sido mais a exceção do que a regra no
+desenvolvimento do kernel. Mesmo assim, uma documentação adequada ajudará a
+facilitar a integração de novos códigos ao kernel, tornará a vida mais fácil para
+outros desenvolvedores e será útil para os seus usuários. Em muitos casos, a
+adição de documentação tornou-se essencialmente obrigatória.
+
+A primeira parte da documentação de qualquer patch é o seu log de alterações
+(changelog) associado. As entradas do log devem descrever o problema que está
+sendo resolvido, a forma da solução, as pessoas que trabalharam no patch,
+quaisquer efeitos relevantes no desempenho e qualquer outra coisa que possa ser
+necessária para entender o patch. Certifique-se de que o changelog diga o
+*porquê* de o patch valer a pena ser aplicado; um número surpreendente de
+desenvolvedores falha em fornecer essa informação.
+
+Qualquer código que adicione uma nova interface de espaço do usuário — incluindo
+novos arquivos sysfs ou /proc — deve incluir a documentação dessa interface, de
+modo a permitir que os desenvolvedores do espaço do usuário saibam com o que
+estão trabalhando. Veja Documentation/ABI/README para uma descrição de como essa
+documentação deve ser formatada e quais informações precisam ser fornecidas.
+
+O arquivo :ref:`Documentation/admin-guide/kernel-parameters.rst
+<kernelparameters>` descreve todos os parâmetros de boot do kernel. Qualquer
+patch que adicione novos parâmetros deve adicionar as entradas apropriadas a
+este arquivo.
+
+Quaisquer novas opções de configuração devem ser acompanhadas por um texto de
+ajuda que explique claramente as opções e quando o usuário pode querer
+selecioná-las.
+
+As informações de API interna de muitos subsistemas são documentadas por meio de
+comentários com formatação especial; esses comentários podem ser extraídos e
+formatados de várias maneiras pelo script "kernel-doc". Se você estiver
+trabalhando em um subsistema que possui comentários kerneldoc, você deve
+mantê-los e adicioná-los, conforme apropriado, para funções disponíveis
+externamente. Mesmo em áreas que não tenham sido documentadas dessa forma, não há
+mal nenhum em adicionar comentários kerneldoc para o futuro; de fato, esta pode
+ser uma atividade útil para desenvolvedores iniciantes de kernel. O formato
+desses comentários, junto com algumas informações sobre como criar modelos de
+kerneldoc, pode ser encontrado em :ref:`Documentation/doc-guide/ <doc_guide>`.
+
+Qualquer pessoa que leia uma quantidade significativa de código existente do
+kernel notará que, frequentemente, os comentários chamam a atenção por sua
+ausência. Mais uma vez, as expectativas para códigos novos são mais altas do que
+eram no passado; integrar código sem comentários será mais difícil. Dito isso,
+há pouco interesse em códigos comentados de forma prolixa. O código deve, por si
+só, ser legível, com os comentários explicando os aspectos mais sutis.
+
+Certas coisas devem sempre ser comentadas. O uso de barreiras de memória
+(memory barriers) deve ser acompanhado por uma linha explicando por que a
+barreira é necessária. As regras de locking (bloqueio) para estruturas de dados
+geralmente precisam ser explicadas em algum lugar. Grandes estruturas de dados
+precisam de uma documentação abrangente em geral. Dependências não óbvias entre
+trechos distintos de código devem ser apontadas. Qualquer coisa que possa tentar
+um "faxineiro de código" (code janitor) a fazer uma "limpeza" incorreta precisa
+de um comentário dizendo por que foi feita daquela maneira. E assim por diante.
+
+
+Alterações de API interna
+-------------------------
+
+A interface binária fornecida pelo kernel para o espaço do usuário não pode ser
+quebrada, exceto sob as circunstâncias mais graves. Por outro lado, as
+interfaces de programação internas do kernel são altamente fluidas e podem ser
+alteradas quando surgir a necessidade. Se você se encontrar tendo que criar uma
+gambiarra para contornar uma API do kernel, ou simplesmente deixando de usar uma
+funcionalidade específica porque ela não atende às suas necessidades, isso pode
+ser um sinal de que a API precisa mudar. Como desenvolvedor de kernel, você tem
+o poder de fazer tais alterações.
+
+Existem, é claro, algumas pegadinhas. Alterações de API podem ser feitas, mas
+precisam ser bem justificadas. Portanto, qualquer patch que faça uma alteração de
+API interna deve ser acompanhado por uma descrição do que é a mudança e do porquê
+ela é necessária. Esse tipo de alteração também deve ser separado em um patch
+independente, em vez de ser enterrado dentro de um patch maior.
+
+A outra pegadinha é que o desenvolvedor que altera uma API interna é geralmente
+encarregado da tarefa de corrigir qualquer código dentro da árvore do kernel que
+tenha sido quebrado pela mudança. Para uma função amplamente utilizada, esse
+dever pode levar a literalmente centenas ou milhares de alterações — muitas das
+quais provavelmente entrarão em conflito com o trabalho que está sendo feito por
+outros desenvolvedores. Desnecessário dizer que isso pode ser um grande
+trabalho, então é melhor ter certeza de que a justificativa é sólida. Note que
+a ferramenta Coccinelle pode ajudar com alterações de API de amplo alcance.
+
+Ao fazer uma alteração incompatível de API, deve-se, sempre que possível,
+garantir que o código que não foi atualizado seja capturado pelo compilador.
+Isso ajudará você a ter certeza de que encontrou todos os usos dessa interface
+dentro da árvore (in-tree). Isso também alertará os desenvolvedores de códigos
+fora da árvore (out-of-tree) de que há uma mudança à qual eles precisam
+responder. Dar suporte a código fora da árvore não é algo com que os
+desenvolvedores do kernel precisem se preocupar, mas também não temos que
+tornar a vida dos desenvolvedores fora da árvore mais difícil do que precisa ser.
diff --git a/Documentation/translations/pt_BR/process/5.Posting.rst b/Documentation/translations/pt_BR/process/5.Posting.rst
new file mode 100644
index 0000000000000..820a56b661dbb
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/5.Posting.rst
@@ -0,0 +1,376 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Enviando patches
+================
+
+Cedo ou tarde, chega o momento em que seu trabalho está pronto para ser
+apresentado à comunidade para revisão e, eventualmente, inclusão no kernel
+mainline. Sem surpresa, a comunidade de desenvolvimento do kernel evoluiu um
+conjunto de convenções e procedimentos que são usados no envio de patches;
+segui-los tornará a vida muito mais fácil para todos os envolvidos. Este
+documento tentará cobrir essas expectativas em detalhes razoáveis; mais
+informações também podem ser encontradas nos arquivos
+:ref:`Documentation/process/submitting-patches.rst <submittingpatches>`
+e :ref:`Documentation/process/submit-checklist.rst <submitchecklist>`.
+
+
+Quando enviar
+-------------
+
+Existe uma tentação constante de evitar o envio de patches antes que eles
+estejam completamente "prontos". Para patches simples, isso não é um problema.
+No entanto, se o trabalho que está sendo feito for complexo, há muito a se
+ganhar obtendo feedback da comunidade antes que o trabalho esteja concluído.
+Portanto, você deve considerar o envio de trabalhos em andamento, ou até mesmo
+disponibilizar uma árvore git para que os desenvolvedores interessados possam
+acompanhar o seu trabalho a qualquer momento.
+
+Ao enviar um código que ainda não é considerado pronto para inclusão, é uma boa
+ideia dizer isso no próprio envio. Mencione também qualquer trabalho importante
+que ainda precise ser feito e quaisquer problemas conhecidos. Menos pessoas vão
+olhar para patches que sabidamente estão "meio cozidos" (half-baked), mas aqueles
+que o fizerem virão com a ideia de que podem ajudá-lo a conduzir o trabalho na
+direção certa.
+
+
+Antes de criar patches
+----------------------
+
+Há uma série de coisas que devem ser feitas antes de você considerar o envio
+de patches para la comunidade de desenvolvimento. Elas incluem:
+
+ - Teste o código tanto quanto puder. Faça uso das ferramentas de depuração
+   do kernel, garanta que o kernel seja compilado com todas as combinações
+   razoáveis de opções de configuração, use compiladores cruzados (cross-
+   compilers) para compilar para diferentes arquiteturas, etc. Adicione testes,
+   provavelmente usando um framework de testes existente como o KUnit, e
+   inclua-os como um membro separado da sua série (veja a próxima seção para
+   mais informações sobre séries de patches). Note que isso pode ser
+   obrigatório ao afetar alguns subsistemas. Por exemplo, funções de biblioteca
+   (localizadas sob lib/) são amplamente utilizadas em quase todos os lugares e
+   espera-se que sejam testadas adequadamente.
+
+ - Certifique-se de que seu código esteja em conformidade com as diretrizes de
+   estilo de codificação do kernel.
+
+ - Sua alteração tem implicações no desempenho? Se sim, você deve executar
+   benchmarks mostrando qual é o impacto (ou benefício) da sua mudança; um
+   resumo dos resultados deve ser incluído junto ao patch.
+
+ - Tenha certeza de que você tem o direito de enviar o código. Se este
+   trabalho foi feito para um empregador, o empregador provavelmente tem direito
+   sobre o trabalho e deve estar de acordo com a sua liberação sob a GPL.
+
+Como regra geral, dedicar um pouco de reflexão extra antes de enviar o código
+quase sempre compensa o esforço em pouco tempo.
+
+
+Preparação de patches
+---------------------
+
+A preparação de patches para envio pode dar uma quantidade surpreendente de
+trabalho, mas, mais uma vez, tentar economizar tempo aqui geralmente não é
+aconselhável, mesmo a curto prazo.
+
+Os patches devem ser preparados contra uma versão específica do kernel. Como
+regra geral, um patch deve ser baseado no mainline atual encontrado na árvore
+git do Linus. Ao basear-se no mainline, comece a partir de um ponto de
+lançamento bem conhecido — um release estável ou -rc —, em vez de criar uma
+bifurcação (branch) a partir do mainline em um ponto arbitrário.
+
+No entanto, pode tornar-se necessário criar versões contra a árvore -mm,
+linux-next ou a árvore de um subsistema, para facilitar testes e revisões mais
+amplos. Dependendo da área do seu patch e do que está acontecendo em outros
+lugares, basear um patch contra essas outras árvores pode exigir uma quantidade
+significativa de trabalho para resolver conflitos e lidar com mudanças de API.
+
+Apenas as alterações mais simples devem ser formatadas como um único patch; tudo
+o mais deve ser feito como uma série lógica de mudanças. Dividir patches é uma
+arte; alguns desenvolvedores passam muito tempo descobrindo como fazer isso da
+maneira que a comunidade espera. Existem algumas regras práticas, no entanto,
+que podem ajudar consideravelmente:
+
+ - A série de patches que você envia quase certamente não será a série de
+   alterações encontrada no seu sistema de controle de versão de trabalho. Em
+   vez disso, as mudanças que você fez precisam ser consideradas em sua forma
+   final e, então, divididas de maneiras que façam sentido. Os desenvolvedores
+   estão interessados em alterações discretas e autocontidas, não no caminho
+   que você percorreu para chegar a essas alterações.
+
+ - Cada alteração logicamente independente deve ser formatada como um patch separado.
+   Essas alterações podem ser pequenas ("adicionar um campo a esta estrutura") ou
+   grandes (adicionar um driver totalmente novo, por exemplo), mas devem ser
+   conceitualmente pequenas e passíveis de uma descrição de uma única linha. Cada
+   patch deve fazer uma alteração específica que possa ser revisada por si só e
+   verificada para garantir que faz o que diz fazer.
+
+ - Como uma forma de reafirmar a diretriz acima: não misture diferentes tipos de
+   alterações no mesmo patch. Se um único patch corrige uma falha crítica de
+   segurança, reorganiza algumas estruturas e reformatará o código, há uma grande
+   chance de que ele seja ignorado e a correção importante seja perdida.
+
+ - Cada patch deve resultar em um kernel que compile e funcione corretamente; se
+   sua série de patches for interrompida no meio, o resultado ainda deve ser um
+   kernel funcional. A aplicação parcial de uma série de patches é um cenário
+   comum quando a ferramenta "git bisect" é usada para encontrar regressões; se o
+   resultado for um kernel quebrado, você tornará a vida mais difícil para os
+   desenvolvedores e usuários que estão engajados no nobre trabalho de rastrear
+   problemas.
+
+ - No entanto, não exagere. Certa vez, um desenvolvedor enviou um conjunto de
+   edições em um único arquivo como 500 patches separados — um ato que não o
+   tornou a pessoa mais popular na lista de discussão do kernel. Um único patch
+   pode ser razoavelmente grande, desde que ainda contenha uma única alteração
+   *lógica*.
+
+ - Pode ser tentador adicionar toda uma nova infraestrutura com uma série de
+   patches, mas deixar essa infraestrutura sem uso até que o patch final da série
+   ative tudo. Essa tentação deve ser evitada, se possível; se essa série
+   adicionar regressões, a bisseção (bisection) apontará o último patch como aquele
+   que causou o problema, mesmo que o bug real esteja em outro lugar. Sempre que
+   possível, um patch que adiciona código novo deve tornar esse código ativo
+   imediatamente.
+
+Trabalhar para criar a série de patches perfeita pode ser um processo
+frustrante, que exige bastante tempo e reflexão após o "trabalho real" ter sido
+concluído. Quando feito corretamente, no entanto, é um tempo bem gasto.
+
+
+Formatação de patches e logs de alterações
+------------------------------------------
+
+Então agora você tem uma série perfeita de patches para enviar, mas o trabalho
+ainda não terminou. Cada patch precisa ser formatado em uma mensagem que comunique
+de forma rápida e clara o seu propósito para o resto do mundo. Para esse fim,
+cada patch será composto pelo seguinte:
+
+ - Uma linha "From" opcional que nomeia o autor do patch. Esta linha só é
+   necessária se você estiver repassando o patch de outra pessoa via e-mail,
+   mas nunca é demais adicioná-la em caso de dúvida.
+
+ - Uma descrição de uma única linha sobre o que o patch faz. Esta mensagem deve
+   ser suficiente para que um leitor que a veja sem outro contexto consiga
+   compreender o escopo do patch; esta é a linha que aparecerá nos logs de
+   alterações (changelogs) de "forma curta". Esta mensagem geralmente é formatada
+   com o nome do subsistema relevante primeiro, seguido pelo propósito do patch.
+   Por exemplo:
+
+   ::
+
+     gpio: fix build on CONFIG_GPIO_SYSFS=n
+
+ - Uma linha em branco seguida por uma descrição detalhada do conteúdo do
+   patch. Esta descrição pode ser tão longa quanto necessário; ela deve dizer
+   o que o patch faz e por que ele deve ser aplicado ao kernel.
+
+ - Uma ou mais linhas de marcadores (tags) com, no mínimo, uma linha
+   "Signed-off-by:" do autor do patch. Os marcadores serão descritos em mais
+   detalhes abaixo.
+
+Os itens acima, juntos, formam o log de alterações (changelog) do patch. Escrever
+bons changelogs é uma arte crucial, mas frequentemente negligenciada; vale a
+pena dedicar mais um momento para discutir esse assunto. Ao escrever um
+changelog, você deve ter em mente que várias pessoas diferentes lerão suas
+palavras. Elas incluem mantenedores de subsistemas e revisores que precisam
+decidir se o patch deve ser incluído, distribuidores e outros mantenedores
+tentando decidir se um patch deve ser retroportado (backported) para outros
+kernels, caçadores de bugs se perguntando se o patch é responsável por um
+problema que estão perseguindo, usuários que querem saber como o kernel mudou e
+muito mais. Um bom changelog transmite a informação necessária para todas essas
+pessoas da maneira mais direta e concisa possível.
+
+Para esse fim, a linha de resumo deve descrever os efeitos e a motivação da
+alteração o melhor possível, dada a restrição de uma única linha. A descrição
+detalhada pode então ampliar esses tópicos e fornecer qualquer informação
+adicional necessária. Se o patch corrige um bug, cite o commit que introduziu o
+bug, se possível (e, por favor, forneça tanto o ID do commit quanto o título ao
+citar commits). Se um problema estiver associado a uma saída específica de log
+ou do compilador, inclua essa saída para ajudar outras pessoas que buscam uma
+solução para o mesmo problema. Se a mudança tem o objetivo de dar suporte a
+outras alterações que virão em um patch posterior, informe isso. Se as APIs
+internas forem alteradas, detalhe essas mudanças e como outros desenvolvedores
+devem reagir. Em geral, quanto mais você puder se colocar no lugar de todos que
+lerão seu changelog, melhor será esse changelog (e o kernel como um todo).
+
+Desnecessário dizer que o changelog deve ser o texto usado ao submeter (commit)
+a alteração em um sistema de controle de versão. Ele será seguido por:
+
+ - O patch em si, no formato de patch unificado ("-u"). O uso da opção "-p" no
+   diff associará os nomes das funções às alterações, tornando o patch resultante
+   mais fácil de ser lido por outras pessoas.
+
+As tags  já mencionadas brevemente acima são usados para fornecer
+informações sobre como o patch surgiu. Eles são descritos em detalhes no
+documento :ref:`Documentation/process/submitting-patches.rst <submittingpatches>`;
+o que se segue aqui é um breve resumo.
+
+Um marcador é usado para se referir a commits anteriores que introduziram os
+problemas corrigidos pelo patch::
+
+	Fixes: 1f2e3d4c5b6a ("The first line of the commit specified by the first 12 characters of its SHA-1 ID")
+
+Outro marcador é usado para vincular páginas da web com contextos ou detalhes
+adicionais, por exemplo, uma discussão anterior que levou ao patch ou um
+documento com uma especificação implementada pelo patch::
+
+  Link: https://example.com/somewhere.html  optional-other-stuff
+
+De acordo com as orientações do Pinguim-Chefe, um marcador Link
+só deve ser adicionado a um commit se ele levar a informações úteis que não
+são encontradas no próprio commit.
+
+Se a URL apontar para um relatório de bug público que está sendo corrigido pelo
+patch, use o marcador "Closes:" em seu lugar::
+
+	Closes: https://example.com/issues/1234  optional-other-stuff
+
+Alguns rastreadores de bugs têm a capacidade de fechar problemas de forma
+automática quando um commit com tal marcador é aplicado. Alguns bots que
+monitoram listas de discussão também podem rastrear esses marcadores e tomar certas
+ações. Rastreadores de bugs privados e URLs inválidas são proibidos.
+
+Outro tipo de marcador é usado para documentar quem esteve envolvido no
+desenvolvimento do patch. Cada um deles usa este formato::
+
+  tag: Full Name <email address>  optional-other-stuff
+
+Os marcadores de uso comum são:
+
+ - Signed-off-by: esta é uma certificação do desenvolvedor de que ele ou ela
+   tem o direito de enviar o patch para inclusão no kernel. É um acordo com o
+   Developer's Certificate of Origin (Certificado de Origem do Desenvolvedor),
+   cujo texto completo pode ser encontrado em
+   :ref:`Documentation/process/submitting-patches.rst <submittingpatches>`.
+   Códigos sem um signoff adequado não podem ser mesclados (merged) no mainline.
+
+ - Co-developed-by: afirma que o patch foi criado em coautoria por vários
+   desenvolvedores; é usado para dar atribuição aos coautores (além do autor
+   atribuído pelo marcador From:) quando várias pessoas trabalham em um único
+   patch. Cada Co-developed-by: deve ser imediatamente seguido por um
+   Signed-off-by: do coautor associado. Detalhes e exemplos podem ser encontrados
+   em :ref:`Documentation/process/submitting-patches.rst <submittingpatches>`.
+
+ - Acked-by: indica o acordo de outro desenvolvedor (frequentemente um
+   mantenedor do código relevante) de que o patch é apropriado para inclusão
+   no kernel.
+
+ - Tested-by: afirma que a pessoa nomeada testou o patch e verificou que ele
+   funciona.
+
+ - Reviewed-by: o desenvolvedor nomeado revisou o patch para verificar sua
+   correção; veja a declaração do revisor em
+   :ref:`Documentation/process/submitting-patches.rst <submittingpatches>`
+   para mais detalhes.
+
+ - Reported-by: nomeia um usuário que relatou o problema que é corrigido por este
+   patch; este marcador é usado para dar crédito às pessoas (frequentemente sub-
+   valorizadas) que testam nosso código e nos informam quando as coisas não
+   funcionam corretamente. Nota: este marcador deve ser seguido por um marcador
+   Closes: apontando para o relato, a menos que o relato não esteja disponível na
+   web. O marcador Link: pode ser usado em vez de Closes: se o patch corrigir
+   apenas uma parte do(s) problema(s) relatado(s).
+
+ - A Suggested-by: este marcador indica que a ideia do patch foi sugerida pela
+   pessoa nomeada e garante o crédito a ela pela ideia. Isso, espera-se, irá
+   inspirá-la a nos ajudar novamente no futuro.
+
+ - Cc: a pessoa nomeada recebeu uma cópia do patch e teve a oportunidade de
+   comentar sobre ele.
+
+Tenha cuidado ao adicionar os marcadores mencionados acima aos seus patches, pois
+todos, exceto Cc:, Reported-by: e Suggested-by:, precisam de permissão explícita
+fontes da pessoa nomeada. Para esses três, a permissão implícita é suficiente se
+a pessoa contribuiu para o kernel Linux usando esse nome e endereço de e-mail de
+acordo com os arquivos do lore ou o histórico de commits — e, no caso de
+Reported-by: e Suggested-by:, se fizeram o relato ou a sugestão publicamente.
+Nota: o bugzilla.kernel.org é um local público nesse sentido, mas os endereços
+de e-mail usados lá são privados; portanto, não os exponha em marcadores, a menos
+que a pessoa os tenha usado em contribuições anteriores.
+
+
+Enviando o patch
+-----------------
+
+Antes de enviar seus patches por e-mail, há algumas outras coisas com as quais
+você deve se preocupar:
+
+ - Você tem certeza de que seu cliente de e-mail não vai corromper os patches?
+   Patches que sofreram alterações desnecessárias de espaço em branco ou quebra
+   de linha causadas pelo cliente de e-mail não serão aplicados na outra ponta
+   e, frequentemente, não serão examinados em detalhes. Se houver qualquer
+   dúvida, envie o patch para você mesmo e certifique-se de que ele chegue intacto.
+
+   O documento :ref:`Documentation/process/email-clients.rst <email_clients>`
+   possui algumas dicas úteis sobre como fazer clientes de e-mail específicos
+   funcionarem para o envio de patches.
+
+ - Você tem certeza de que seu patch está livre de erros bobos? Você deve sempre
+   passar os patches pelo scripts/checkpatch.pl e corrigir as reclamações que
+   ele apresentar. Por favor, tenha em mente que o checkpatch.pl, embora seja a
+   personificação de uma quantidade razoável de reflexão sobre como os patches do
+   kernel devem parecer, não é mais inteligente que você. Se corrigir uma
+   reclamação do checkpatch.pl piorar o código, não o faça.
+
+Os patches devem sempre ser enviados como texto simples (plain text). Por favor,
+não os envie como anexos; isso torna muito mais difícil para os revisores citarem
+trechos do patch em suas respostas. Em vez disso, coloque o patch diretamente no
+corpo da sua mensagem.
+
+Ao enviar patches por e-mail, é importante enviar cópias para qualquer pessoa
+que possa estar interessada neles. Ao contrário de alguns outros projetos, o
+kernel incentiva as pessoas a pecarem pelo excesso, enviando cópias demais; não
+assuma que as pessoas relevantes verão sua publicação nas listas de discussão. Em
+particular, as cópias devem ir para:
+
+- O(s) mantenedor(es) do(s) subsistema(s) afetado(s). Como descrito antes, o
+   arquivo MAINTAINERS é o primeiro lugar para procurar por essas pessoas.
+
+ - Outros desenvolvedores que estiveram trabalhando na mesma área — especialmente
+   aqueles que possam estar trabalhando lá agora. Usar o git para ver quem mais
+   modificou os arquivos nos quais você está trabalhando pode ser útil.
+
+ - Se você estiver respondendo a um relato de bug ou a uma solicitação de recurso
+   (feature request), envie uma cópia também para o autor original.
+
+ - Envie uma cópia para a lista de discussão relevante ou, se nada mais se
+   aplicar, para a lista linux-kernel.
+
+ - Se você estiver corrigindo um bug, pense se a correção deve ir para a próxima
+   atualização estável (stable update). Se sim, stable@vger.kernel.org deve
+   receber uma cópia do patch. Adicione também um "Cc: stable@vger.kernel.org"
+   aos marcadores (tags) dentro do próprio patch; isso fará com que a equipe do
+   stable receba uma notificação quando sua correção for integrada ao mainline.
+
+Ao selecionar os destinatários para um patch, é bom ter uma ideia de quem você
+acha que eventualmente aceitará o patch e fará a mesclagem (merge). Embora seja
+possível enviar patches diretamente para Linus Torvalds e fazer com que ele os
+mescle, as coisas normalmente não são feitas dessa forma. Linus está ocupado, e
+existem mantenedores de subsistemas que vigiam partes específicas do kernel. Em
+geral, você desejará que esse mantenedor mescle seus patches. Se não houver um
+mantenedor óbvio, Andrew Morton costuma ser o destino de patch de último recurso.
+
+Os patches precisam de boas linhas de assunto (subject lines). O formato canônico
+para a linha de um patch é algo como:
+
+::
+
+
+  [PATCH nn/mm] subsys: descrição de uma linha do patch
+
+onde "nn" é o número ordinal do patch, "mm" é o número total de patches na
+série, e "subsys" é o nome do subsistema afetado. Claramente, nn/mm pode ser
+omitido no caso de um patch único e isolado (standalone).
+
+Se você tiver uma série significativa de patches, é costumeiro enviar uma
+descrição introdutória como a parte zero. Essa convenção não é seguida
+universalmente, no entanto; se você a utilizar, lembre-se de que as informações
+da introdução não entram nos changelogs do kernel. Portanto, certifique-se de
+que os patches, em si, possuam informações completas em seus changelogs.
+
+Em geral, a segunda parte e as subsequentes de um patch de múltiplas partes devem
+ser enviadas como uma resposta à primeira parte, de modo que todas formem uma
+única linha de discussão (thread) na ponta receptora. Ferramentas como o git e o
+quilt possuem comandos para enviar por e-mail um conjunto de patches com o
+encadeamento correto. Se você tiver uma série longa, contudo, e estiver usando o
+git, por favor, evite a opção --chain-reply-to para não criar um aninhamento
+excepcionalmente profundo.
diff --git a/Documentation/translations/pt_BR/process/6.Followthrough.rst b/Documentation/translations/pt_BR/process/6.Followthrough.rst
new file mode 100644
index 0000000000000..d6bdaa2cb8a4d
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/6.Followthrough.rst
@@ -0,0 +1,220 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Acompanhamento
+==============
+
+Neste ponto, você seguiu as diretrizes apresentadas até aqui e, com a
+adição de suas próprias habilidades de engenharia, enviou uma série perfeita
+de patches. Um dos maiores erros que até mesmo desenvolvedores experientes
+do kernel podem cometer é concluir que o seu trabalho agora está concluído.
+Na verdade, o envio de patches indica uma transição para a próxima etapa
+do processo, possivelmente com uma quantidade considerável de trabalho
+ainda por fazer.
+
+É raro um patch ser tão bom em seu primeiro envio que não haja margem para
+melhorias. O processo de desenvolvimento do kernel reconhece esse fato e,
+como resultado, é fortemente orientado para o aprimoramento do código
+enviado. Espera-se que você, como autor desse código, trabalhe junto à
+comunidade do kernel para garantir que seu código esteja de acordo com os
+padrões de qualidade do kernel. A falha em participar desse processo muito
+provavelmente impedirá a inclusão de seus patches na árvore principal
+(*mainline*).
+
+
+Trabalhando com revisores
+-------------------------
+
+Um patch de qualquer relevância resultará em uma série de comentários de outros
+desenvolvedores à medida que eles revisam o código. Trabalhar com revisores
+pode ser, para muitos desenvolvedores, a parte mais intimidadora do processo
+de desenvolvimento do kernel. No entanto, a vida pode se tornar muito mais
+fácil se você mantiver algumas coisas em mente:
+
+* Se você explicou bem o seu patch, os revisores entenderão o seu valor
+  e o porquê de você ter tido o trabalho de escrevê-lo. Contudo, esse valor
+  não os impedirá de fazer uma pergunta fundamental: como será manter um
+  kernel com este código inserido nele daqui a cinco ou dez anos? Muitas das
+  mudanças que podem lhe pedir para fazer — desde ajustes de estilo de código
+  até reescritas substanciais — vêm do entendimento de que o Linux ainda estará
+  por aqui e sob desenvolvimento daqui a uma década.
+
+* A revisão de código é um trabalho árduo e uma ocupação relativamente
+  ingrata; as pessoas lembram quem escreveu o código do kernel, mas há pouca
+  fama duradoura para aqueles que o revisaram. Portanto, os revisores podem
+  ficar ranzinzas, especialmente quando veem os mesmos erros sendo cometidos
+  repetidamente. Se você receber uma revisão que pareça irritada, insultuosa
+  ou abertamente ofensiva, resista ao impulso de responder à altura. A revisão
+  de código diz respeito ao código, não às pessoas, e os revisores de código
+  não estão atacando você pessoalmente.
+
+* Da mesma forma, os revisores de código não estão tentando promover os
+  interesses de seus empregadores em detrimento dos seus. Os desenvolvedores
+  do kernel geralmente esperam continuar trabalhando no kernel daqui a muitos
+  anos, mas entendem que seu empregador pode mudar. Quase sem exceção, eles
+  estão verdadeiramente trabalhando em prol da criação do melhor kernel possível;
+  eles não estão tentando causar desconforto aos concorrentes de seus empregadores.
+
+* Esteja preparado para solicitações aparentemente tolas de mudanças no estilo
+  de codificação e pedidos para refatorar parte do seu código em seções
+  compartilhadas do kernel. Uma das funções dos mantenedores é manter as coisas
+  com a mesma aparência. Às vezes, isso significa que aquele truque inteligente
+  (*clever hack*) em seu driver para contornar um problema
+
+Note que você não precisa concordar com todas as mudanças sugeridas pelos
+revisores. Se você acredita que o revisor entendeu mal o seu código, explique
+o que realmente está acontecendo. Se tiver uma objeção técnica a uma mudança
+sugerida, descreva-a e justifique a sua solução para o problema. Se as suas
+explicações fizerem sentido, o revisor as aceitará. Contudo, caso a sua
+explicação não seja persuasiva — especialmente se outros começarem a concordar
+com o revisor —, reserve um tempo para repensar as coisas. Pode ser fácil ficar
+ceguificado por sua própria solução para um problema, a ponto de não perceber
+que algo está fundamentalmente errado ou que, talvez, você não esteja sequer
+resolvendo o problema certo.
+
+Andrew Morton sugeriu que todo comentário de revisão que não resulte em uma
+alteração de código deveria, em vez disso, resultar em um comentário adicional
+no próprio código; isso pode ajudar os futuros revisores a evitar as dúvidas
+que surgiram da primeira vez.
+
+Um erro fatal é ignorar os comentários de revisão na esperança de que eles
+desapareçam. Eles não vão desaparecer. Se você reenviar o código sem ter
+respondido aos comentários que recebeu da vez anterior, é provável que descubra
+que os seus patches não vão a lugar nenhum.
+
+Por falar em reenviar código: tenha em mente que os revisores não vão se
+lembrar de todos os detalhes do código que você enviou da última vez. Portanto,
+é sempre uma boa ideia lembrar os revisores dos problemas levantados
+anteriormente e de como você lidou com eles; o registro de alterações
+(*changelog*) do patch é um bom lugar para esse tipo de informação. Os revisores
+não deveriam ter que vasculhar os arquivos das listas de discussão para se
+familiarizarem com o que foi dito na última vez; se você ajudá-los a começar
+com o pé direito, eles estarão de melhor humor quando revisitarem o seu código.
+
+E se você tentou fazer tudo certo e as coisas ainda não estão avançando? A
+maioria das divergências técnicas pode ser resolvida por meio de discussão,
+mas há momentos em que alguém simplesmente precisa tomar uma decisão. Se você
+acredita genuinamente que essa decisão está indo contra você de forma errada,
+você sempre pode tentar recorrer a uma instância superior. Até o momento em
+que este texto foi escrito, essa instância superior costuma ser Andrew Morton.
+Andrew goza de um enorme respeito na comunidade de desenvolvimento do kernel;
+ele frequentemente consegue destravar uma situação que parece desesperadoramente
+bloqueada. Recorrer a Andrew, no entanto, não deve ser feito de ânimo leve e nem
+antes que todas as outras alternativas tenham sido esgotadas. E tenha em mente,
+é claro, que ele também pode não concordar com você.
+
+O que acontece a seguir
+-----------------------
+
+Se um patch for considerado algo bom para ser adicionado ao kernel, e assim
+que a maioria dos problemas de revisão tiver sido resolvida, o próximo passo
+geralmente é a entrada na árvore de um mantenedor de subsistema. Como isso
+funciona varia de um subsistema para o outro; cada mantenedor tem sua própria
+maneira de fazer as coisas. Em particular, pode haver mais de uma árvore — uma,
+talvez, dedicada a patches planejados para a próxima janela de mesclagem
+(*merge window*), e outra para trabalhos de longo prazo.
+
+Para patches que se aplicam a áreas para quais não há uma árvore de subsistema
+óbvia (patches de gerenciamento de memória, por exemplo), a árvore padrão
+geralmente acaba sendo a *-mm*. Patches que afetam múltiplos subsistemas
+também podem acabar passando pela árvore *-mm*.
+
+A inclusão em uma árvore de subsistema pode trazer um nível mais alto de
+visibilidade para um patch. Agora, outros desenvolvedores que trabalham com
+aquela árvore receberão o patch por padrão. As árvores de subsistemas tipicamente
+alimentam a *linux-next* também, tornando seus conteúdos visíveis para a
+comunidade de desenvolvimento como um todo. Neste ponto, há uma boa chance de
+você receber mais comentários de um novo conjunto de revisores; esses
+comentários precisam ser respondidos da mesma forma que na rodada anterior.
+
+O que também pode acontecer neste ponto, dependendo da natureza do seu patch,
+é surgirem conflitos com o trabalho que está sendo feito por outros. No pior
+dos casos, conflitos pesados de patches podem fazer com que alguns trabalhos
+sejam deixados em segundo plano, para que os patches restantes possam ser
+ajustados e mesclados. Outras vezes, a resolução de conflitos envolverá trabalhar
+junto a outros desenvolvedores e, possivelmente, mover alguns patches entre
+árvores para garantir que tudo se aplique de forma limpa. Este trabalho pode ser
+árduo, mas console-se com uma vantagem: antes do surgimento da árvore *linux-next*,
+esses conflitos frequentemente só apareciam durante a janela de mesclagem e
+tinham que ser resolvidos às pressas. Agora eles podem ser resolvidos com calma,
+antes que a janela de mesclagem se abra.
+
+Um belo dia, se tudo correr bem, você fará login e verá que o seu patch foi
+mesclado ao kernel principal (*mainline*). Parabéns! No entanto, assim que a
+comemoração terminar (e você tiver se adicionado ao arquivo MAINTAINERS), vale
+a pena lembrar de um pequeno fato importante: o trabalho ainda não acabou. A
+mesclagem na árvore principal traz os seus próprios desafios.
+
+Para começar, a visibilidade do seu patch aumentou ainda mais. Pode haver
+uma nova rodada de comentários de desenvolvedores que não estavam cientes do
+patch antes. Pode ser tentador ignorá-los, já que não há mais nenhuma dúvida
+sobre a mesclagem do seu código. No entanto, resista a essa tentação; você
+ainda precisa ser receptivo aos desenvolvedores que tiverem dúvidas ou
+sugestões.
+
+Mais importante ainda: a inclusão na árvore principal coloca o seu código
+nas mãos de um grupo muito maior de testadores. Mesmo que você tenha contribuído
+com um driver para um hardware que ainda não está disponível, você se
+surpreenderá com a quantidade de pessoas que compilarão seu código em seus
+próprios kernels. E, logicamente, onde há testadores, haverá relatórios de
+erros (*bug reports*).
+
+O pior tipo de relatório de erro são as regressões (*regressions*). Se o seu
+patch causar uma regressão, você descobrirá uma quantidade desconfortável de
+olhos voltados para você; as regressões precisam ser corrigidas o mais rápido
+possível. Se você não estiver disposto ou for incapaz de corrigir a regressão
+(e ninguém mais fizer isso por você), seu patch quase certamente será removido
+durante o período de estabilização. Além de anular todo o trabalho que você teve
+para colocar seu patch na árvore principal, ter um patch removido como resultado
+da falha em corrigir uma regressão pode muito bem tornar mais difícil para você
+mesclar trabalhos no futuro.
+
+Depois que todas as regressões tiverem sido tratadas, pode haver outros erros
+comuns com os quais lidar. O período de estabilização é a sua melhor oportunidade
+para corrigir esses problemas e garantir que a estreia do seu código em um
+lançamento do kernel principal seja o mais sólida possível. Portanto, por favor,
+responda aos relatórios de erros e corrija os problemas, se for viável. É para
+isso que serve o período de estabilização; você pode começar a criar novos
+patches fantásticos assim que quaisquer problemas com os antigos tiverem sido
+resolvidos.
+
+E não se esqueça de que existem outros marcos que também podem gerar relatórios
+de erros: o próximo lançamento estável da árvore principal, o momento em que
+distribuidores proeminentes adotarem uma versão do kernel que contenha o seu
+patch, etc. Continuar respondendo a esses relatórios é uma questão de orgulho
+básico pelo seu trabalho. Se isso não for motivação suficiente, contudo, também
+vale a pena considerar que a comunidade de desenvolvimento se lembra dos
+desenvolvedores que perdem o interesse em seu próprio código após a mesclagem.
+A próxima vez que você enviar um patch, eles o avaliarão sob a suposição de
+que você não estará por perto para mantê-lo depois.
+
+
+Outras coisas que podem acontecer
+---------------------------------
+
+Um dia, você poderá abrir o seu cliente de e-mail e ver que alguém lhe enviou
+um patch para o seu código. Afinal, essa é uma das vantagens de ter o seu
+código disponível publicamente. Se você concordar com o patch, poderá encaminhá-lo
+para o mantenedor do subsistema (certifique-se de incluir uma linha ``From:``
+adequada para que a atribuição de autoria esteja correta e adicione a sua
+própria assinatura — *signoff*) ou enviar uma resposta com um ``Acked-by:``
+e deixar que o remetente original o envie para cima.
+
+Se você não concordar com o patch, envie uma resposta educada explicando o
+motivo. Se possível, diga ao autor quais alterações precisam ser feitas para
+que o patch seja aceitável para você. Existe uma certa resistência em mesclar
+patches que sofrem oposição do autor e mantenedor do código, mas isso tem limite.
+Se você for visto como alguém que está bloqueando um bom trabalho sem necessidade,
+esses patches eventualmente seguirão outro fluxo ao seu redor e entrarão na
+árvore principal de qualquer maneira. No kernel do Linux, ninguém tem poder de
+veto absoluto sobre nenhum código. Exceto, talvez, o Linus.
+
+Em ocasiões muito raras, você poderá ver algo completamente diferente: outro
+desenvolvedor envia uma solução diferente para o seu problema. Nesse ponto,
+as chances são de que um dos dois patches não seja mesclado, e o argumento
+"o meu chegou primeiro" não é considerado um argumento técnico convincente.
+Se o patch de outra pessoa deslocar o seu e entrar na árvore principal, existe
+realmente apenas uma maneira de responder: fique satisfeito pelo fato de o seu
+problema ter sido resolvido e siga adiante com o seu trabalho. Ter o próprio
+trabalho deixado de lado dessa maneira pode ser doloroso e desanimador, mas a
+comunidade se lembrará da sua reação muito depois de terem esquecido de quem
+foi o patch que realmente foi mesclado.
diff --git a/Documentation/translations/pt_BR/process/7.AdvancedTopics.rst b/Documentation/translations/pt_BR/process/7.AdvancedTopics.rst
new file mode 100644
index 0000000000000..97466fad1994c
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/7.AdvancedTopics.rst
@@ -0,0 +1,201 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Tópicos avançados
+=================
+
+Neste ponto, esperamos que você já tenha uma boa noção de como funciona o
+processo de desenvolvimento. No entanto, ainda há mais a aprender! Esta seção
+cobrirá uma série de tópicos que podem ser úteis para desenvolvedores que
+desejam se tornar parte regular do processo de desenvolvimento do kernel Linux.
+
+Gerenciamento de patches com o git
+----------------------------------
+
+O uso de controle de versão distribuído para o kernel começou no início de
+2002, quando Linus começou a testar o aplicativo proprietário BitKeeper.
+Embora o BitKeeper fosse controverso, a abordagem de gerenciamento de versão
+de software que ele incorporava certamente não era. O controle de versão
+distribuído permitiu uma aceleração imediata do projeto de desenvolvimento do
+kernel. Atualmente, existem várias alternativas gratuitas ao BitKeeper. Para o
+bem ou para o mal, o projeto do kernel adotou o git como sua ferramenta de
+escolha.
+
+Gerenciar patches com o git pode facilitar muito a vida do desenvolvedor,
+especialmente à medida que o volume desses patches cresce. O git também tem suas
+pontas soltas e apresenta certos riscos; é uma ferramenta jovem e poderosa que
+ainda está sendo refinada por seus desenvolvedores. Este documento não tentará
+ensinar o leitor a usar o git; isso seria material suficiente para um documento
+longo por si só. Em vez disso, o foco aqui será em como o git se encaixa
+especificamente no processo de desenvolvimento do kernel. Os desenvolvedores
+que desejam se atualizar com o git encontrarão mais informações em:
+
+	https://git-scm.com/
+
+	https://www.kernel.org/pub/software/scm/git/docs/user-manual.html
+
+e em vários tutoriais encontrados na web.
+
+A primeira ordem do dia é ler os sites acima e obter uma compreensão sólida de
+como o git funciona antes de tentar usá-lo para disponibilizar patches para
+outros. Um desenvolvedor que utiliza o git deve ser capaz de obter uma cópia do
+repositório principal, explorar o histórico de revisões, comitar alterações na
+árvore, usar branches, etc. A compreensão das ferramentas do git para a
+reescrita de histórico (como o rebase) também é útil. O git vem com sua própria
+terminologia e conceitos; um novo usuário do git deve saber sobre refs, remote
+branches, o index, fast-forward merges, pushes e pulls, detached HEADs, etc.
+Tudo isso pode ser um pouco intimidante no início, mas os conceitos não são tão
+difíceis de entender com um pouco de estudo.
+
+Usar o git para gerar patches para submissão por e-mail pode ser um bom exercício
+enquanto você se atualiza.
+
+Quando estiver pronto para começar a disponibilizar árvores git para que outros
+possam examinar, você, logicamente, precisará de um servidor a partir do qual um
+pull possa ser feito. Configurar um servidor desse tipo com o git-daemon é
+relativamente simples se você tiver um sistema acessível à internet. Caso
+contrário, sites de hospedagem públicos e gratuitos (o GitHub, por exemplo)
+estão começando a surgir na rede. Desenvolvedores estabelecidos podem obter uma
+conta no kernel.org, mas estas não são fáceis de conseguir; consulte
+https://kernel.org/faq/ para mais informações.
+
+O fluxo de trabalho normal do git envolve o uso de muitas branches. Cada linha
+de desenvolvimento pode ser separada em uma "topic branch" distinta e mantida de
+forma independente. Branches no git são baratas, não há razão para não fazer um
+uso livre delas. E, em qualquer caso, você não deve fazer o seu desenvolvimento
+em nenhuma branch a partir da qual pretenda pedir para que outros deem pull.
+Branches disponíveis publicamente devem ser criadas com cuidado; mescle patches
+de branches de desenvolvimento quando eles estiverem em sua forma final e prontos
+para seguir em frente — não antes.
+
+O git fornece algumas ferramentas poderosas que podem permitir que você
+reescreva o seu histórico de desenvolvimento. Um patch inconveniente (um que
+quebre o bisection, por exemplo, ou que tenha algum outro tipo de bug óbvio)
+pode ser corrigido localmente ou feito desaparecer completamente do histórico.
+Uma série de patches pode ser reescrita como se tivesse sido escrita no topo da
+linha principal de hoje, mesmo que você esteja trabalhando nela há meses. As
+alterações podem ser movidas de forma transparente de uma branch para outra. E
+assim por diante. O uso criterioso da capacidade do git de revisar o histórico
+pode ajudar na criação de conjuntos de patches limpos e com menos problemas.
+
+O uso excessivo dessa capacidade pode levar a outros problemas, no entanto, além
+de uma simples obsessão pela criação do histórico de projeto perfeito. Reescrever
+o histórico reescreverá as alterações contidas nele, transformando uma árvore do
+kernel testada (assim se espera) em uma não testada. Mas, além disso, os
+desenvolvedores não podem colaborar facilmente se não tiverem uma visão
+compartilhada do histórico do projeto; se você reescrever o histórico que outros
+desenvolvedores já deram pull em seus repositórios, tornará a vida deles muito
+mais difícil. Portanto, uma regra prática simples se aplica aqui: o histórico
+que foi exportado para terceiros deve ser visto geralmente como imutável dali em
+diante.
+
+Sendo assim, uma vez que você faz o push de um conjunto de alterações para o seu
+servidor disponível publicamente, essas alterações não devem ser reescritas. O
+git tentará aplicar essa regra se você tentar dar push em alterações que não
+resultem em um fast-forward merge (ou seja, alterações que não compartilham o
+mesmo histórico). É possível anular essa verificação, e pode haver momentos em
+que seja necessário reescrever uma árvore exportada. Mover changesets entre
+árvores para evitar conflitos na linux-next é um exemplo. No entanto, tais ações
+devem ser raras. Esta é uma das razões pelas quais o desenvolvimento deve ser
+feito em branches privadas (que podem ser reescritas, se necessário) e apenas
+movido para branches públicas quando estiver em um estado razoavelmente avançado.
+
+À medida que a linha principal (ou outra árvore na qual um conjunto de
+alterações se baseia) avança, é tentador fazer o merge com essa árvore para
+permanecer na vanguarda. Para uma branch privada, o rebasing pode ser uma maneira
+fácil de acompanhar outra árvore, mas o rebasing não é uma opção uma vez que uma
+árvore é exportada para o mundo. Quando isso acontece, um merge completo deve
+ser feito. Fazer merges ocasionalmente faz todo o sentido, mas merges excessivamente
+frequentes podem poluir o histórico desnecessariamente. A técnica sugerida neste
+caso é fazer merges raramente, e geralmente apenas em release points específicos
+(como um lançamento -rc da linha principal). Se você estiver inseguro sobre
+mudanças específicas, sempre poderá realizar merges de teste em uma branch
+privada. A ferramenta "rerere" do git pode ser útil nessas situações; ela se
+lembra de como os conflitos de merge foram resolvidos para que você não precise
+fazer o mesmo trabalho duas vezes.
+
+Uma das maiores reclamações recorrentes sobre ferramentas como o git é esta: o
+movimento em massa de patches de um repositório para outro torna fácil a
+inclusão de mudanças desaconselháveis que entram na linha principal abaixo do
+radar de revisão. Os desenvolvedores do kernel costumam ficar descontentes quando
+veem esse tipo de coisa acontecer; disponibilizar uma árvore git com patches não
+revisados ou fora do tópico pode afetar a sua capacidade de ter suas árvores
+puxadas no futuro. Citando Linus:
+
+::
+
+    Você pode me enviar patches, mas para eu puxar um patch git de você, eu
+    preciso saber que você sabe o que está fazendo, e preciso ser capaz de
+    confiar nas coisas *sem* ter que ir lá e verificar cada mudança
+    individualmente à mão.
+
+(https://lwn.net/Articles/224135/).
+
+Para evitar esse tipo de situação, certifique-se de que todos os patches
+dentro de uma determinada branch permaneçam estritamente alinhados ao tópico
+associado; uma branch de "correções de drivers" não deveria fazer alterações no
+código central de gerenciamento de memória. E, acima de tudo, não use uma árvore
+git para burlar o processo de revisão. Publique ocasionalmente um resumo da
+árvore na lista de discussão relevante e, quando for o momento certo, solicite
+que a árvore seja incluída na linux-next.
+
+Se e quando outros começarem a enviar patches para inclusão em sua árvore, não
+se esqueça de revisá-los. Certifique-se também de manter as informações corretas
+de autoria; a ferramenta "am" do git faz o melhor que pode a esse respeito, mas
+você pode ter que adicionar uma linha "From:" ao patch se ele tiver sido
+retransmitido a você por terceiros.
+
+Ao solicitar um pull, certifique-se de fornecer todas as informações
+relevantes: onde está a sua árvore, qual branch deve ser puxada e quais
+alterações resultarão do pull. O comando git request-pull pode ser útil a esse
+respeito; ele formatará a solicitação da maneira que outros desenvolvedores
+esperam e também verificará se você se lembrou de dar push nessas alterações
+para o servidor público.
+
+
+Revisão de patches
+------------------
+
+Alguns leitores certamente objetarão a inclusão desta seção em "tópicos
+avançados" sob o argumento de que mesmo desenvolvedores iniciantes do kernel
+deveriam estar revisando patches. É certamente verdade que não há melhor maneira
+de aprender a programar no ambiente do kernel do que examinando o código
+postado por outros. Além disso, revisores estão sempre em falta; ao examinar o
+código, você pode fazer uma contribuição significativa para o processo como um
+todo.
+
+Revisar código pode ser uma perspectiva intimidadora, especialmente para um novo
+desenvolvedor do kernel que pode se sentir nervoso em questionar — em público —
+um código que foi postado por aqueles com mais experiência. No entanto, mesmo o
+código escrito pelos desenvolvedores mais experientes pode ser aprimorado. Talvez
+o melhor conselho para revisores (todos os revisores) seja este: formule os
+comentários de revisão como perguntas em vez de críticas. Perguntar "como o lock
+é liberado neste caminho?" sempre funcionará melhor do que afirmar "o bloqueio
+aqui está errado."
+
+Outra técnica útil em caso de desacordo é pedir que outros se manifestem. Se uma
+discussão chegar a um impasse após algumas trocas de mensagens, peça a opinião
+de outros revisores ou mantenedores. Frequentemente, aqueles que concordam com
+um revisor permanecem em silêncio, a menos que sejam solicitados. A opinião de
+múltiplas pessoas carrega exponencialmente mais peso.
+
+Diferentes desenvolvedores revisarão o código sob diferentes pontos de vista.
+Alguns estão preocupados principalmente com o estilo de codificação e se as
+linhas de código possuem espaços em branco no final (trailing white space).
+Outros se concentrarão principalmente em saber se a alteração implementada pelo
+patch como um todo é algo bom para o kernel ou não. Ainda assim, outros buscarão
+por bloqueios problemáticos, uso excessivo de pilha (stack usage), possíveis
+problemas de segurança, duplicação de código encontrado em outros lugares,
+documentação adequada, efeitos adversos no desempenho, alterações na ABI do
+espaço do usuário (user-space ABI), etc. Todos os tipos de revisão, se levarem a
+um código melhor entrando no kernel, são bem-vindos e valem a pena.
+
+Não há exigência estrita para o uso de tags específicas como ``Reviewed-by``. Na
+verdade, revisões em texto simples são mais informativas e incentivadas mesmo
+quando uma tag é fornecida, por exemplo: "Analisei os aspectos A, B e C deste
+envio e tudo me parece correto." Alguma forma de mensagem de revisão ou resposta
+é obviamente necessária, caso contrário, os mantenedores não saberão que o
+revisor sequer examinou o patch!
+
+Por último, mas não menos importante, a revisão de patches pode se tornar um
+processo negativo, focado em apontar problemas. Por favor, reserve um elogio de
+vez em quando, particularmente para os novatos!
diff --git a/Documentation/translations/pt_BR/process/8.Conclusion.rst b/Documentation/translations/pt_BR/process/8.Conclusion.rst
new file mode 100644
index 0000000000000..d5af31e7c4d9b
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/8.Conclusion.rst
@@ -0,0 +1,73 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Para mais informações
+=====================
+
+Há inúmeras fontes de informação sobre o desenvolvimento do kernel Linux e
+tópicos relacionados. A primeira delas sempre será o diretório Documentation
+encontrado na distribuição do código-fonte do kernel. Comece com o arquivo de
+nível superior :ref:`process/howto.rst <process_howto>`; leia também
+:ref:`process/submitting-patches.rst <submittingpatches>`. Muitas APIs internas
+do kernel são documentadas usando o mecanismo kerneldoc; "make htmldocs" ou
+"make pdfdocs" podem ser usados para gerar esses documentos em formato HTML ou
+PDF (embora a versão do TeX fornecida por algumas distribuições esbarre em
+limites internos e falhe em processar os documentos corretamente).
+
+Vários sites discutem o desenvolvimento do kernel em todos os níveis de
+detalhes. O autor gostaria de sugerir humildemente o https://lwn.net/ como uma
+fonte; informações sobre muitos tópicos específicos do kernel podem ser
+encontradas através do índice do kernel do LWN em:
+
+	https://lwn.net/Kernel/Index/
+
+Além disso, um recurso valioso para os desenvolvedores do kernel é:
+	https://kernelnewbies.org/
+
+E, claro, não se deve esquecer o https://kernel.org/, o local definitivo
+para informações sobre os lançamentos do kernel.
+
+Há uma série de livros sobre o desenvolvimento do kernel:
+
+	Linux Device Drivers, 3rd Edition (Jonathan Corbet, Alessandro
+	Rubini, and Greg Kroah-Hartman).  Online at
+	https://lwn.net/Kernel/LDD3/.
+
+	Linux Kernel Development (Robert Love).
+
+    Understanding the Linux Kernel (Daniel Bovet and Marco Cesati).
+
+Todos esses livros, no entanto, sofrem de um defeito comum: eles tendem a estar
+um pouco obsoletos quando chegam às prateleiras, e já estão nelas há algum
+tempo. Ainda assim, há uma boa quantidade de informações úteis a serem
+encontradas ali.
+
+A documentação para o git pode ser encontrada em:
+
+	https://www.kernel.org/pub/software/scm/git/docs/
+
+	https://www.kernel.org/pub/software/scm/git/docs/user-manual.html
+
+
+Conclusão
+=========
+
+Parabéns a qualquer pessoa que tenha chegado ao fim deste documento longo e
+detalhado. Esperamos que ele tenha fornecido uma compreensão útil de como o
+kernel Linux é desenvolvido e de como você pode participar desse processo.
+
+No fim das contas, é a participação que importa. Qualquer projeto de software
+de código aberto não é nada mais do que a soma do que seus colaboradores
+dedicam a ele. O kernel Linux progrediu tão rápido e tão bem porque foi ajudado
+por um grupo impressionantemente grande de desenvolvedores, todos trabalhando
+para torná-lo melhor. O kernel é um exemplo primordial do que pode ser feito
+quando milhares de pessoas trabalham juntas em direção a um objetivo comum.
+
+O kernel, no entanto, sempre pode se beneficiar de uma base maior de
+desenvolvedores. Há sempre mais trabalho a fazer. Mas, de forma igualmente
+importante, a maioria dos outros participantes do ecossistema Linux pode se
+beneficiar ao contribuir para o kernel. Colocar o código na linha principal
+(mainline) é a chave para uma maior qualidade de código, menores custos de
+manutenção e distribuição, um nível mais alto de influência sobre a direção do
+desenvolvimento do kernel e muito mais. É uma situação em que todos os
+envolvidos ganham. Abra o seu editor e venha se juntar a nós; você será mais do
+que bem-vindo.
diff --git a/Documentation/translations/pt_BR/process/adding-syscalls.rst b/Documentation/translations/pt_BR/process/adding-syscalls.rst
new file mode 100644
index 0000000000000..cdf8b5033765a
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/adding-syscalls.rst
@@ -0,0 +1,700 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=======================================
+Adicionando uma Nova Chamada de Sistema
+=======================================
+
+Este documento descreve o que está envolvido na adição de uma nova chamada de
+sistema (system call) ao kernel Linux, indo além dos conselhos normais de
+submissão em
+:ref:`Documentation/process/submitting-patches.rst <submittingpatches>`.
+
+
+Alternativas às Chamadas de Sistema
+-----------------------------------
+
+A primeira coisa a se considerar ao adicionar uma nova chamada de sistema é se
+uma das alternativas poderia ser mais adequada. Embora as chamadas de sistema
+sejam os pontos de interação mais tradicionais e óbvios entre o espaço do
+usuário (userspace) e o kernel, existem outras possibilidades -- escolha o que
+melhor se adapta à sua interface.
+
+ - Se as operações envolvidas puderem ser moldadas para se parecerem com um
+   objeto do tipo arquivo, pode fazer mais sentido criar um novo sistema de
+   arquivos ou dispositivo. Isso também torna mais fácil encapsular a nova
+   funcionalidade em um módulo de kernel, em vez de exigir que ela seja
+   incorporada ao kernel principal.
+
+     - Se a nova funcionalidade envolver operações em que o kernel notifica o
+       espaço do usuário de que algo aconteceu, retornar um novo descritor de
+       arquivo (file descriptor) para o objeto relevante permite que o espaço
+       do usuário use ``poll``/``select``/``epoll`` para receber essa
+       notificação.
+     - No entanto, as operações que não se mapeiam para operações do tipo
+       :manpage:`read(2)`/:manpage:`write(2)` precisam ser implementadas como
+       requisições :manpage:`ioctl(2)`, o que pode levar a uma API um tanto
+       quanto opaca.
+
+ - Se você estiver apenas expondo informações do sistema em tempo de execução,
+   um novo nó no sysfs (veja ``Documentation/filesystems/sysfs.rst``) ou no
+   sistema de arquivos ``/proc`` pode ser mais apropriado. No entanto, o acesso
+   a esses mecanismos exige que o sistema de arquivos relevante esteja montado,
+   o que pode não ser sempre o caso (por exemplo, em um ambiente com namespaces,
+   sandboxed ou chrooted). Evite adicionar qualquer API ao debugfs, pois este
+   não é considerado uma interface de "produção" para o espaço do usuário.
+ - Se a operação for específica para um arquivo ou descritor de arquivo de um
+   determinado objeto, então uma opção de comando adicional para :manpage:`fcntl(2)`
+   pode ser mais adequada. Contudo, o :manpage:`fcntl(2)` é uma chamada de sistema
+   de multiplexação que oculta muita complexidade, portanto, esta opção é melhor
+   para quando a nova função for intimamente análoga à funcionalidade existente
+   do :manpage:`fcntl(2)`, ou se a nova funcionalidade for muito simples (por
+   exemplo, obter/definir uma flag simples relacionada a um descritor de arquivo).
+ - Se a operação for específica para uma tarefa (task) ou processo específico,
+   então uma opção de comando adicional para :manpage:`prctl(2)` pode ser mais
+   apropriada. Assim como no caso do :manpage:`fcntl(2)`, esta chamada de sistema
+   é um multiplexador complicado, sendo melhor reservá-la para análogos próximos
+   de comandos ``prctl()`` existentes ou para obter/definir uma flag simples
+   relacionada a um processo.
+
+
+Projetando a API: Planejando a Extensibilidade
+----------------------------------------------
+
+Uma nova chamada de sistema faz parte da API do kernel e deve ser suportada
+indefinidamente. Sendo assim, é uma excelente ideia discutir explicitamente a
+interface na lista de discussão do kernel (LKML), e é crucial planejar extensões
+futuras para essa interface.
+
+(A tabela de chamadas de sistema está repleta de exemplos históricos onde isso
+não foi feito, juntamente com as respectivas chamadas de sistema de acompanhamento
+-- ``eventfd``/``eventfd2``, ``dup2``/``dup3``, ``inotify_init``/``inotify_init1``,
+``pipe``/``pipe2``, ``renameat``/``renameat2`` -- portanto, aprenda com a história
+do kernel e planeje as extensões desde o início.)
+
+Para chamadas de sistema mais simples que recebem apenas alguns argumentos, a
+maneira preferencial de permitir extensibilidade futura é incluir um argumento de
+flags na chamada de sistema. Para garantir que os programas do espaço do usuário
+possam usar flags de forma segura entre diferentes versões do kernel, verifique
+se o valor de flags contém qualquer flag desconhecida e rejeite a chamada de
+sistema (com ``EINVAL``) se contiver::
+
+    if (flags & ~(THING_FLAG1 | THING_FLAG2 | THING_FLAG3))
+        return -EINVAL;
+
+(Se nenhum valor de flag for utilizado ainda, verifique se o argumento de flags
+é zero.)
+
+Para chamadas de sistema mais sofisticadas que envolvem um número maior de
+argumentos, prefere-se encapsular a maioria dos argumentos em uma estrutura
+(struct) que é passada por meio de um ponteiro. Esse tipo de estrutura pode
+lidar com extensões futuras incluindo um argumento de tamanho (size) na própria
+estrutura::
+
+    struct xyzzy_params {
+        u32 size; /* o espaço do usuário define p->size = sizeof(struct xyzzy_params) */
+        u32 param_1;
+        u64 param_2;
+        u64 param_3;
+    };
+
+Desde que qualquer campo adicionado subsequentemente, digamos ``param_4``, seja
+projetado de forma que um valor zero mantenha o comportamento anterior, isso
+permitirá lidar com a divergência de versões em ambas as direções:
+
+ - Para lidar com um programa de espaço do usuário mais novo chamando um kernel
+   mais antigo, o código do kernel deve verificar se qualquer memória além do
+   tamanho da estrutura que ele espera está zerada (efetivamente verificando
+   se ``param_4 == 0``).
+ - Para lidar com um programa de espaço do usuário mais antigo chamando um kernel
+   mais novo, o código do kernel pode preencher com zero (zero-extend) a
+   instância menor da estrutura (efetivamente definindo ``param_4 = 0``).
+
+Veja :manpage:`perf_event_open(2)` e a função ``perf_copy_attr()`` (em
+``kernel/events/core.c``) para um exemplo desta abordagem.
+
+
+Projetando a API: Outras Considerações
+--------------------------------------
+
+Se a sua nova chamada de sistema permitir que o espaço do usuário se refira a
+um objeto do kernel, ela deve usar um descritor de arquivo (file descriptor)
+como o handle (identificador) para esse objeto -- não invente um novo tipo de
+handle de objeto para o espaço do usuário quando o kernel já possui mecanismos
+e semânticas bem definidas para o uso de descritores de arquivo.
+
+Se a sua nova chamada de sistema (2) de fato retornar un novo descritor de
+arquivo, então o argumento de flags deve incluir um valor que seja equivalente
+a definir ``O_CLOEXEC`` no novo FD. Isso torna possível para o espaço do usuário
+fechar a janela de tempo entre a chamada ``()`` e a execução de
+``fcntl(fd, F_SETFD, FD_CLOEXEC)``, onde um ``fork()`` e ``execve()`` inesperados
+em outra thread poderiam vazar um descritor para o programa executado. (Contudo,
+resista à tentação de reutilizar o valor real da constante ``O_CLOEXEC``, pois
+ela é específica de cada arquitetura e faz parte de um espaço de numeração de
+flags ``O_*`` que está bastante cheio.)
+
+Se a sua chamada de sistema retornar um novo descritor de arquivo, você também
+deve considerar o que significa usar a família de chamadas de sistema
+:manpage:`poll(2)` nesse descritor de arquivo. Tornar um descritor de arquivo
+pronto para leitura ou escrita é a maneira normal de o kernel indicar ao espaço
+do usuário que um evento ocorreu no objeto correspondente do kernel.
+
+Se a sua nova chamada de sistema (2) envolver um argumento de nome de arquivo
+(filename)::
+
+    int sys_xyzzy(const char __user *path, ..., unsigned int flags);
+
+você também deve considerar se uma versão xyzzyat(2) seria mais apropriada::
+
+    int sys_xyzzyat(int dfd, const char __user *path, ..., unsigned int flags);
+
+Isso permite maior flexibilidade para a forma como o espaço do usuário especifica
+o arquivo em questão; em particular, permite que o espaço do usuário solicite a
+funcionalidade para um descritor de arquivo já aberto usando a flag
+``AT_EMPTY_PATH``, fornecendo efetivamente uma operação fxyzzy(3) de graça::
+
+ - xyzzyat(AT_FDCWD, path, ..., 0) é equivalente a (path,...)
+ - xyzzyat(fd, "", ..., AT_EMPTY_PATH) é equivalente a fxyzzy(fd, ...)
+
+(Para mais detalhes sobre a justificativa das chamadas \*at(), veja a página de
+manual :manpage:`openat(2)`; para um exemplo de AT_EMPTY_PATH, veja a página de
+manual :manpage:`fstatat(2)`.)
+
+Se a sua nova chamada de sistema (2) envolver um parâmetro que descreve um
+deslocamento (offset) dentro de um arquivo, mude o seu tipo para ``loff_t`` para
+que offsets de 64 bits possam ser suportados mesmo em arquiteturas de 32 bits.
+
+Se a sua nova chamada de sistema (2) envolver funcionalidades privilegiadas,
+ela precisa ser governada pelo bit de capacidade (capability) do Linux apropriado
+(verificado com uma chamada a ``capable()``), conforme descrito na página de
+manual :manpage:`capabilities(7)`. Escolha um bit de capacidade existente que governe
+funcionalidades relacionadas, mas tente evitar combinar muitas funções que tenham
+apenas uma vaga relação sob o mesmo bit, pois isso vai contra o propósito das
+capabilities de dividir o poder do root. Em particular, evite adicionar novos
+usos para a capacidade ``CAP_SYS_ADMIN``, que já é excessivamente generalista.
+
+Se a sua nova chamada de sistema (2) manipular um processo diferente do
+processo que a chamou, ela deve ser restrita (usando uma chamada a
+``ptrace_may_access()``) para que apenas um processo chamador com as mesmas
+permissões do processo alvo, ou com as capacidades necessárias, possa manipular
+o processo alvo.
+
+Finalmente, esteja ciente de que algumas arquiteturas não-x86 lidam melhor se os
+parâmetros da chamada de sistema que são explicitamente de 64 bits caírem em
+argumentos de numeração ímpar (ou seja, parâmetro 1, 3, 5), para permitir o uso
+de pares contíguos de registradores de 32 bits. (Esta preocupação não se aplica
+se os argumentos fizerem parte de uma estrutura que é passada por meio de um
+ponteiro.)
+
+
+Propondo a API
+--------------
+
+Para tornar as novas chamadas de sistema fáceis de revisar, é melhor dividir o
+conjunto de patches (patchset) em blocos separados. Estes devem incluir, pelo
+menos, os seguintes itens como commits distintos (cada um dos quais é descrito
+mais adiante):
+
+ - A implementação central da chamada de sistema, juntamente com protótipos,
+   numeração genérica, alterações no Kconfig e a implementação de stub de realinhamento (fallback stub).
+ - A fiação (wiring up) da nova chamada de sistema para uma arquitetura em
+   particular, geralmente x86 (incluindo todas as variantes x86_64, x86_32 e x32).
+ - Uma demonstração do uso da nova chamada de sistema no espaço do usuário por
+   meio de um selftest em ``tools/testing/selftests/``.
+ - Um rascunho da página de manual (man-page) para a nova chamada de sistema,
+   seja como texto simples na carta de apresentação (cover letter) ou como um
+   patch para o repositório (separado) de man-pages.
+
+Novas propostas de chamadas de sistema, como qualquer alteração na API do
+kernel, devem sempre ser enviadas com cópia (cc'ed) para linux-api@vger.kernel.org.
+
+
+Implementação Genérica de Chamadas de Sistema
+---------------------------------------------
+
+O ponto de entrada principal para a sua nova chamada de sistema (2) será chamado
+de ``sys_xyzzy()``, mas você deve adicionar esse ponto de entrada com a macro
+``SYSCALL_DEFINEn()`` apropriada, em vez de fazer isso explicitamente. O 'n'
+indica o número de argumentos da chamada de sistema, e a macro recebe o nome da
+chamada de sistema seguido pelos pares (tipo, nome) para os parâmetros como
+argumentos. O uso dessa macro permite que os metadados sobre a nova chamada de
+sistema fiquem disponíveis para outras ferramentas.
+
+O novo ponto de entrada também precisa de um protótipo de função correspondente
+em ``include/linux/syscalls.h``, marcado como asmlinkage para corresponder à
+maneira como as chamadas de sistema são invocadas::
+
+    asmlinkage long sys_xyzzy(...);
+
+Algumas arquiteturas (por exemplo, x86) possuem suas próprias tabelas de syscall
+específicas da arquitetura, mas várias outras arquiteturas compartilham uma tabela
+de syscall genérica. Adicione a sua nova chamada de sistema à lista genérica
+adicionando uma entrada na lista em ``include/uapi/asm-generic/unistd.h``::
+
+    #define __NR_xyzzy 292
+    __SYSCALL(__NR_xyzzy, sys_xyzzy)
+
+Atualize também a contagem de __NR_syscalls para refletir a chamada de sistema
+adicional, e observe que se múltiplas novas chamadas de sistema forem adicionadas
+na mesma janela de mesclagem (merge window), o número da sua nova syscall poderá
+ser ajustado para resolver conflitos.
+
+O arquivo ``kernel/sys_ni.c`` fornece uma implementação de stub de fallback para
+cada chamada de sistema, retornando ``-ENOSYS``. Adicione a sua nova chamada de
+sistema aqui também::
+
+    COND_SYSCALL(sys_xyzzy);
+
+A sua nova funcionalidade de kernel, e a chamada de sistema que a controla, deve
+normalmente ser opcional, portanto adicione uma opção ``CONFIG`` (tipicamente em
+``init/Kconfig``) para ela. Como de costume para novas opções ``CONFIG``:
+
+ - Inclua uma descrição da nova funcionalidade e da chamada de sistema controlada
+   pela opção.
+ - Faça a opção depender de EXPERT se ela deve ser ocultada dos usuários normais.
+ - Faça com que quaisquer novos arquivos de código-fonte que implementem a função
+   sejam dependentes da opção CONFIG no Makefile (por exemplo,
+   ``obj-$(CONFIG_XYZZY_SYSCALL) += xyzzy.o``).
+ - Verifique duas vezes se o kernel ainda compila com a nova opção CONFIG desativada.
+
+Para resumir, você precisa de um commit que inclua:
+
+ - Opção ``CONFIG`` para a nova função, normalmente em ``init/Kconfig``
+ - ``SYSCALL_DEFINEn(, ...)`` para o ponto de entrada
+ - Protótipo correspondente em ``include/linux/syscalls.h``
+ - Entrada na tabela genérica em ``include/uapi/asm-generic/unistd.h``
+ - Stub de fallback em ``kernel/sys_ni.c``
+
+
+.. _pt_BR_syscall_generic_6_11:
+
+Desde a versão 6.11
+~~~~~~~~~~~~~~~~~~~
+
+A partir da versão 6.11 do kernel, a implementação de chamadas de sistema
+genéricas para as seguintes arquiteturas não requer mais modificações em
+``include/uapi/asm-generic/unistd.h``:
+
+ - arc
+ - arm64
+ - csky
+ - hexagon
+ - loongarch
+ - nios2
+ - openrisc
+ - riscv
+
+Em vez disso, você precisa atualizar ``scripts/syscall.tbl`` e, se aplicável,
+ajustar ``arch/*/kernel/Makefile.syscalls``.
+
+Como o ``scripts/syscall.tbl`` serve como uma tabela de syscall comum para
+múltiplas arquiteturas, uma nova entrada é necessária nesta tabela::
+
+    468   common        sys_xyzzy
+
+Note que adicionar uma entrada ao ``scripts/syscall.tbl`` com a ABI "common"
+também afeta todas as arquiteturas que compartilham essa tabela. Para alterações
+mais limitadas ou específicas de uma arquitetura, considere usar uma ABI
+específica da arquitetura ou definir uma nova.
+
+Se uma nova ABI, digamos ``xyz``, for introduzida, as atualizações
+correspondentes também devem ser feitas em ``arch/*/kernel/Makefile.syscalls``::
+
+    syscall_abis_{32,64} += xyz (...)
+
+Para resumir, você precisa de um commit que inclua:
+
+ - Opção ``CONFIG`` para a nova função, normalmente em ``init/Kconfig``
+ - ``SYSCALL_DEFINEn(, ...)`` para o ponto de entrada
+ - Protótipo correspondente em ``include/linux/syscalls.h``
+ - Nova entrada em ``scripts/syscall.tbl``
+ - (Se necessário) Atualizações de Makefile em ``arch/*/kernel/Makefile.syscalls``
+ - Stub de fallback em ``kernel/sys_ni.c``
+
+
+Implementação de Chamadas de Sistema em x86
+-------------------------------------------
+
+Para interligar (wire up) a sua nova chamada de sistema nas plataformas x86, você
+precisa atualizar as tabelas mestras de syscall. Assumindo que a sua nova chamada
+de sistema não seja especial de alguma forma (veja abaixo), isso envolve uma
+entrada "common" (para x86_64 e x32) em
+``arch/x86/entry/syscalls/syscall_64.tbl``::
+
+    333   common        sys_xyzzy
+
+e uma entrada "i386" em ``arch/x86/entry/syscalls/syscall_32.tbl``::
+
+    380   i386          sys_xyzzy
+
+Novamente, esses números estão sujeitos a alterações caso ocorram conflitos na
+janela de mesclagem (merge window) relevante.
+
+Chamadas de Sistema de Compatibilidade (Genéricas)
+--------------------------------------------------
+
+Para a maioria das chamadas de sistema, a mesma implementação de 64 bits pode
+ser invocada mesmo quando o programa do espaço do usuário é, ele próprio, de 32
+bits; mesmo se os parâmetros da chamada de sistema incluírem um ponteiro
+explícito, isso é tratado de forma transparente.
+
+No entanto, existem algumas situações em que uma camada de compatibilidade
+(compatibility layer) é necessária para lidar com as diferenças de tamanho entre
+32 bits e 64 bits.
+
+A primeira é se o kernel de 64 bits também suportar programas de espaço do
+usuário de 32 bits e, portanto, precisar analisar áreas de memória
+(``__user``) que poderiam conter valores de 32 bits ou 64 bits. Em particular,
+isso é necessário sempre que um argumento de chamada de sistema for:
+
+ - um ponteiro para um ponteiro
+ - um ponteiro para uma struct que contém um ponteiro (por exemplo,
+   ``struct iovec __user *``)
+ - um ponteiro para um tipo integral de tamanho variável (``time_t``,
+   ``off_t``, ``long``, ...)
+ - um ponteiro para uma struct que contém um tipo integral de tamanho variável.
+
+A segunda situação que requer uma camada de compatibilidade é se um dos
+argumentos da chamada de sistema tiver um tipo que é explicitamente de 64 bits,
+mesmo em uma arquitetura de 32 bits, por exemplo, ``loff_t`` ou ``__u64``. Neste
+caso, um valor que chega ao kernel de 64 bits vindo de uma aplicação de 32 bits
+será dividido em dois valores de 32 bits, que precisarão ser remontados na
+camada de compatibilidade.
+
+(Note que um argumento de chamada de sistema que seja um ponteiro para um tipo
+explícito de 64 bits **não** precisa de uma camada de compatibilidade; por
+exemplo, os argumentos do :manpage:`splice(2)` do tipo ``loff_t __user *`` não
+disparam a necessidade de uma chamada de sistema ``compat_``.)
+
+A versão de compatibilidade da chamada de sistema é chamada de
+``compat_sys_xyzzy()`` e é adicionada com a macro ``COMPAT_SYSCALL_DEFINEn()``,
+de forma análoga à macro SYSCALL_DEFINEn. Esta versão da implementação roda como
+parte de um kernel de 64 bits, mas espera receber valores de parâmetros de 32
+bits e faz o que for necessário para lidar com eles. (Tipicamente, a versão
+``compat_sys_`` converte os valores para versões de 64 bits e chama a versão
+``sys_``, ou ambas chamam uma função interna comum de implementação).
+
+O ponto de entrada compat também precisa de um protótipo de função
+correspondente em ``include/linux/compat.h``, marcado como asmlinkage para
+corresponder à maneira como as chamadas de sistema são invocadas::
+
+    asmlinkage long compat_sys_xyzzy(...);
+
+Se a chamada de sistema envolver uma estrutura cujo layout seja diferente em
+sistemas de 32 bits e 64 bits, digamos ``struct xyzzy_args``, então o arquivo de
+cabeçalho ``include/linux/compat.h`` também deve incluir uma versão compat da
+estrutura (``struct compat_xyzzy_args``), onde cada campo de tamanho variável
+tenha o tipo ``compat_`` correspondente ao tipo na ``struct xyzzy_args``. A
+rotina ``compat_sys_xyzzy()`` pode então usar essa estrutura ``compat_`` para
+analisar os argumentos vindos de uma invocação de 32 bits.
+
+Por exemplo, se existirem os campos::
+
+    struct xyzzy_args {
+        const char __user *ptr;
+        __kernel_long_t varying_val;
+        u64 fixed_val;
+        /* ... */
+    };
+
+na struct xyzzy_args, então a struct compat_xyzzy_args teria::
+
+    struct compat_xyzzy_args {
+        compat_uptr_t ptr;
+        compat_long_t varying_val;
+        u64 fixed_val;
+        /* ... */
+    };
+
+A lista genérica de chamadas de sistema também precisa de ajustes para permitir
+a versão compat; a entrada em ``include/uapi/asm-generic/unistd.h`` deve usar
+``__SC_COMP`` em vez de ``__SYSCALL``::
+
+    #define __NR_xyzzy 292
+    __SC_COMP(__NR_xyzzy, sys_xyzzy, compat_sys_xyzzy)
+
+Para resumir, você precisa de:
+
+ - uma macro ``COMPAT_SYSCALL_DEFINEn(, ...)`` para o ponto de entrada compat
+ - protótipo correspondente em ``include/linux/compat.h``
+ - (se necessário) struct de mapeamento de 32 bits em ``include/linux/compat.h``
+ - instância de ``__SC_COMP``, e não de ``__SYSCALL``, em
+   ``include/uapi/asm-generic/unistd.h``
+
+Desde a versão 6.11
+~~~~~~~~~~~~~~~~~~~
+
+Isso se aplica a todas as arquiteturas listadas em
+:ref:`Desde a versão 6.11<pt_BR_syscall_generic_6_11>` sob "Implementação Genérica de
+Chamadas de Sistema", exceto arm64. Veja
+:ref:`Chamadas de Sistema de Compatibilidade (arm64)<pt_BR_compat_arm64>` para mais
+informações.
+
+Você precisa estender a entrada em ``scripts/syscall.tbl`` com uma coluna extra
+para indicar que um programa de espaço do usuário de 32 bits rodando em um
+kernel de 64 bits deve atingir o ponto de entrada compat::
+
+    468   common          sys_xyzzy    compat_sys_xyzzy
+
+Para resumir, você precisa de:
+
+ - ``COMPAT_SYSCALL_DEFINEn(, ...)`` para o ponto de entrada compat
+ - Protótipo correspondente em ``include/linux/compat.h``
+ - Modificação da entrada em ``scripts/syscall.tbl`` para incluir uma coluna
+   "compat" extra
+ - (Se necessário) Struct de mapeamento de 32 bits em ``include/linux/compat.h``
+
+
+.. _pt_BR_compat_arm64:
+
+Chamadas de Sistema de Compatibilidade (arm64)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+No arm64, existe uma tabela de syscall dedicada para chamadas de sistema de
+compatibilidade voltadas para o espaço do usuário de 32 bits (AArch32):
+``arch/arm64/tools/syscall_32.tbl``. Você precisa adicionar uma linha adicional
+a esta tabela especificando o ponto de entrada compat::
+
+    468   common          sys_xyzzy    compat_sys_xyzzy
+
+
+Chamadas de Sistema de Compatibilidade (x86)
+--------------------------------------------
+
+Para interligar a arquitetura x86 de uma chamada de sistema com uma versão de
+compatibilidade, as entradas nas tabelas de syscall precisam ser ajustadas.
+
+Primeiro, a entrada em ``arch/x86/entry/syscalls/syscall_32.tbl`` ganha uma
+coluna extra para indicar que um programa de espaço do usuário de 32 bits rodando
+em um kernel de 64 bits deve atingir o ponto de entrada compat::
+
+    380   i386          sys_xyzzy    __ia32_compat_sys_xyzzy
+
+Segundo, você precisa definir o que deve acontecer para a versão da ABI x32 da
+nova chamada de sistema. Há uma escolha aqui: o layout dos argumentos deve
+corresponder à versão de 64 bits ou à versão de 32 bits.
+
+Se houver um ponteiro para um ponteiro envolvido, a decisão é fácil: x32 é
+ILP32 (inteiro, long e ponteiro possuem 32 bits), portanto o layout deve
+corresponder à versão de 32 bits, e a entrada em
+``arch/x86/entry/syscalls/syscall_64.tbl`` é dividida para que os programas x32
+atinjam o wrapper de compatibilidade::
+
+    333   64            sys_xyzzy
+    ...
+    555   x32           __x32_compat_sys_xyzzy
+
+Se não houver ponteiros envolvidos, então é preferível reutilizar a chamada de
+sistema de 64 bits para a ABI x32 (e, consequentemente, a entrada em
+``arch/x86/entry/syscalls/syscall_64.tbl`` permanece inalterada).
+
+Em qualquer um dos casos, você deve verificar se os tipos envolvidos no layout
+dos seus argumentos de fato se mapeiam exatamente do x32 (-mx32) para os seus
+equivalentes de 32 bits (-m32) ou 64 bits (-m64).
+
+
+Chamadas de Sistema com Retorno para Outro Local
+------------------------------------------------
+
+Para a maioria das chamadas de sistema (syscalls), assim que a execução é
+concluída, o programa do usuário continua exatamente de onde parou -- na
+próxima instrução, com a pilha idêntica e a maior parte dos registradores no
+mesmo estado de antes da chamada, além do mesmo espaço de memória virtual.
+
+No entanto, algumas poucas chamadas de sistema agem de forma diferente. Elas
+podem retornar para um local distinto (``rt_sigreturn``), alterar o espaço de
+memória (``fork``/``vfork``/``clone``) ou até mesmo modificar a arquitetura
+(``execve``/``execveat``) do programa.
+
+Para permitir isso, a implementação da chamada de sistema no kernel pode
+precisar salvar e restaurar registradores adicionais na pilha do kernel,
+garantindo controle total de onde e como a execução continuará após a syscall.
+
+Isso é específico de cada arquitetura (arch-specific), mas tipicamente envolve
+a definição de pontos de entrada em assembly que salvam/restauram esses
+registradores adicionais e invocam o ponto de entrada real da chamada de
+sistema.
+
+Para x86_64, isso é implementado como um ponto de entrada ``stub_xyzzy`` em
+``arch/x86/entry/entry_64.S``, e a entrada correspondente na tabela de syscalls
+(``arch/x86/entry/syscalls/syscall_64.tbl``) é ajustada para refletir::
+
+    333   common        stub_xyzzy
+
+O equivalente para programas de 32 bits executados em um kernel de 64 bits é
+normalmente chamado de ``stub32_xyzzy`` e implementado em
+``arch/x86/entry/entry_64_compat.S``, com o respectivo ajuste na tabela de
+syscalls em ``arch/x86/entry/syscalls/syscall_32.tbl``::
+
+    380   i386          sys_xyzzy    stub32_xyzzy
+
+Se a chamada de sistema precisar de uma camada de compatibilidade (como na
+seção anterior), a versão ``stub32_`` precisará chamar a versão
+``compat_sys_`` da chamada de sistema em vez da versão nativa de 64 bits. Além
+disso, se a implementação da ABI x32 não for compartilhada com a versão
+x86_64, sua tabela de syscalls também precisará invocar um stub que direcione
+para a versão ``compat_sys_``.
+
+Por questões de integridade, também é recomendado configurar um mapeamento para
+que o User-Mode Linux (UML) continue funcionando -- sua tabela de syscalls fará
+referência a ``stub_xyzzy``, mas o build do UML não inclui a implementação de
+``arch/x86/entry/entry_64.S`` (já que o UML simula registradores, etc.). Corrigir
+isso é tão simples quanto adicionar um #define em
+``arch/x86/um/sys_call_table_64.c``::
+
+    #define stub_xyzzy sys_xyzzy
+
+
+Outros Detalhes
+---------------
+
+A maior parte do kernel trata as chamadas de sistema de maneira genérica, mas
+há exceções ocasionais que podem precisar de atualização para a sua chamada
+de sistema específica.
+
+O subsistema de auditoria (audit) é um desses casos especiais; ele inclui
+funções (específicas de cada arquitetura) que classificam alguns tipos
+especiais de chamada de sistema -- especificamente operações de abertura de
+arquivo (``open``/``openat``), execução de programa (``execve``/``exeveat``) ou
+multiplexador de socket (``socketcall``). Se a sua nova chamada de sistema for
+análoga a uma dessas, o sistema de auditoria deverá ser atualizado.
+
+De forma mais geral, se existir uma chamada de sistema atual que seja análoga
+à sua nova chamada de sistema, vale a pena fazer um grep em todo o kernel pela
+chamada existente para verificar se não há outros casos especiais.
+
+
+Testes
+------
+
+Uma nova chamada de sistema deve, obviamente, ser testada; também é útil
+fornecer aos revisores uma demonstração de como os programas do espaço do
+usuário (user space) usarão a chamada de sistema. Uma boa maneira de combinar
+esses objetivos é incluir um programa simples de autoteste em um novo diretório
+sob ``tools/testing/selftests/``.
+
+Para uma nova chamada de sistema, obviamente não haverá uma função de wrapper
+na libc e, portanto, o teste precisará invocá-la usando ``syscall()``; além
+disso, se a chamada de sistema envolver uma nova estrutura visível para o
+espaço do usuário, o cabeçalho correspondente precisará ser instalado para
+compilar o teste.
+
+Certifique-se de que o autoteste seja executado com sucesso em todas as
+arquiteturas suportadas. Por exemplo, verifique se ele funciona quando compitado
+como um programa ABI x86_64 (-m64), x86_32 (-m32) e x32 (-mx32).
+
+Para testes mais extensos e minuciosos de novas funcionalidades, você também
+deve considerar a adição de testes ao Linux Test Project ou ao projeto
+xfstests para alterações relacionadas
+
+Página de Manual (Man Page)
+---------------------------
+
+Todas as novas chamadas de sistema devem vir acompanhadas de uma página de
+manual completa, idealmente usando a marcação groff, mas texto simples também
+é aceitável. Se o groff for utilizado, é útil incluir uma versão ASCII pré-
+renderizada da página de manual no e-mail de apresentação (cover letter) do
+conjunto de patches (patchset), para a conveniência dos revisores.
+
+A página de manual deve ser enviada com cópia (cc) para
+linux-man@vger.kernel.org. Para mais detalhes, consulte
+https://www.kernel.org/doc/man-pages/patches.html
+
+
+Não invoque Chamadas de Sistema dentro do Kernel
+------------------------------------------------
+
+As chamadas de sistema são, como mencionado acima, pontos de interação entre o
+espaço do usuário (userspace) e o kernel. Portanto, funções de chamada de
+sistema como ``sys_xyzzy()`` ou ``compat_sys_xyzzy()`` só devem ser chamadas a
+partir do espaço do usuário por meio da tabela de syscalls, e não de outros
+lugares do kernel. Se a funcionalidade da syscall for útil para ser utilizada
+dentro do kernel, precisar ser compartilhada entre uma syscall antiga e uma
+nova, ou precisar ser compartilhada entre uma syscall e sua variante de
+compatibilidade, ela deve ser implementada por meio de uma função auxiliadora
+("helper", como ``ksys_xyzzy()``). Essa função do kernel poderá então ser
+chamada dentro do stub da syscall (``sys_xyzzy()``), do stub da syscall de
+compatibilidade (``compat_sys_xyzzy()``) e/ou de outro código do kernel.
+
+Pelo menos em x86 de 64 bits, será um requisito rígido a partir da versão v4.17
+em diante não chamar funções de chamadas de sistema no kernel. Essa arquitetura
+utiliza uma convenção de chamada diferente para chamadas de sistema na qual a
+``struct pt_regs`` é decodificada dinamicamente em um wrapper de syscall, que
+então repassa o processamento para a função real da syscall. Isso significa que
+apenas os parâmetros realmente necessários para uma syscall específica são
+passados durante a entrada da syscall, em vez de preencher seis registradores da
+CPU com conteúdos aleatórios do espaço do usuário o tempo todo (o que poderia
+causar problemas sérios no decorrer da cadeia de chamadas).
+
+Além disso, as regras sobre como os dados podem ser acessados diferem entre os
+dados do kernel e os dados do usuário. Essa é outra razão pela qual chamar
+``sys_xyzzy()`` geralmente é uma má ideia.
+
+Exceções a essa regra são permitidas apenas em substituições (overrides)
+específicas de cada arquitetura, wrappers de compatibilidade específicos de cada
+arquitetura ou outros códigos dentro do diretório arch/.
+
+Referências e Fontes
+--------------------
+
+ - Artigo da LWN por Michael Kerrisk sobre o uso do argumento flags em chamadas
+   de sistema:
+   https://lwn.net/Articles/585415/
+ - Artigo da LWN por Michael Kerrisk sobre como lidar com flags desconhecidas
+   em uma chamada de sistema: https://lwn.net/Articles/588444/
+ - Artigo da LWN por Jake Edge descrevendo restrições em argumentos de chamadas
+   de sistema de 64 bits: https://lwn.net/Articles/311630/
+ - Par de artigos da LWN por David Drysdale que descrevem detalhadamente os
+   caminhos de implementação de chamadas de sistema para a v3.14:
+
+    - https://lwn.net/Articles/604287/
+    - https://lwn.net/Articles/604515/
+
+ - Os requisitos específicos de arquitetura para chamadas de sistema são
+   discutidos na página de manual :manpage:`syscall(2)`:
+   http://man7.org/linux/man-pages/man2/syscall.2.html#NOTES
+ - E-mails compilados de Linus Torvalds discutindo os problemas com ``ioctl()``:
+   https://yarchive.net/comp/linux/ioctl.html
+ - "How to not invent kernel interfaces", Arnd Bergmann,
+   https://www.ukuug.org/events/linux2007/2007/papers/Bergmann.pdf
+ - Artigo da LWN por Michael Kerrisk sobre evitar novos usos de CAP_SYS_ADMIN:
+   https://lwn.net/Articles/486306/
+ - Recomendação de Andrew Morton para que todas as informações relacionadas a
+   uma nova chamada de sistema venham na mesma thread de e-mail:
+   https://lore.kernel.org/r/20140724144747.3041b208832bbdf9fbce5d96@linux-foundation.org
+ - Recomendação de Michael Kerrisk para que uma nova chamada de sistema venha
+   acompanhada de uma página de manual:
+   https://lore.kernel.org/r/CAKgNAkgMA39AfoSoA5Pe1r9N+ZzfYQNvNPvcRN7tOvRb8+v06Q@mail.gmail.com
+ - Sugestão de Thomas Gleixner para que a vinculação (wire-up) do x86 esteja em
+   um commit separado:
+   https://lore.kernel.org/r/alpine.DEB.2.11.1411191249560.3909@nanos
+ - Sugestão de Greg Kroah-Hartman de que é bom que novas chamadas de sistema
+   venham acompanhadas de uma página de manual e um autoteste:
+   https://lore.kernel.org/r/20140320025530.GA25469@kroah.com
+ - Discussão de Michael Kerrisk sobre uma nova chamada de sistema versus a
+   extensão de :manpage:`prctl(2)`:
+   https://lore.kernel.org/r/CAHO5Pa3F2MjfTtfNxa8LbnkeeU8=YJ+9tDqxZpw7Gz59E-4AUg@mail.gmail.com
+ - Sugestão de Ingo Molnar de que as chamadas de sistema que envolvem múltiplos
+   argumentos devem encapsular esses argumentos em uma struct, a qual inclua um
+   campo de tamanho (size) para fins de extensibilidade futura:
+   https://lore.kernel.org/r/20150730083831.GA22182@gmail.com
+ - Excentricidades de numeração decorrentes do uso (e reuso) de flags do espaço
+   de numeração O_*:
+
+    - commit 75069f2b5bfb ("vfs: renumber FMODE_NONOTIFY and add to uniqueness
+      check")
+    - commit 12ed2e36c98a ("fanotify: FMODE_NONOTIFY and __O_SYNC in sparc
+      conflict")
+    - commit bb458c644a59 ("Safer ABI for O_TMPFILE")
+
+ - Discussão de Matthew Wilcox sobre restrições em argumentos de 64 bits:
+   https://lore.kernel.org/r/20081212152929.GM26095@parisc-linux.org
+ - Recomendação de Greg Kroah-Hartman de que flags desconhecidas devem ser
+   fiscalizadas/policiadas:
+   https://lore.kernel.org/r/20140717193330.GB4703@kroah.com
+ - Recomendação de Linus Torvalds de que as chamadas de sistema x32 devem
+   preferir a compatibilidade com as versões de 64 bits em vez das versões de
+   32 bits:
+   https://lore.kernel.org/r/CA+55aFxfmwfB7jbbrXxa=K7VBYPfAvmu3XOkGrLbB1UFjX1+Ew@mail.gmail.com
+ - Série de patches revisando a infraestrutura da tabela de chamadas de sistema
+   para utilizar scripts/syscall.tbl em múltiplas arquiteturas:
+   https://lore.kernel.org/lkml/20240704143611.2979589-1-arnd@kernel.org
diff --git a/Documentation/translations/pt_BR/process/applying-patches.rst b/Documentation/translations/pt_BR/process/applying-patches.rst
new file mode 100644
index 0000000000000..313401bc23357
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/applying-patches.rst
@@ -0,0 +1,447 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Aplicando Patches ao Kernel Linux
++++++++++++++++++++++++++++++++++
+
+Autor Original:
+    Jesper Juhl, Agosto de 2005
+
+.. note::
+
+   Este documento está obsoleto. Na maioria dos casos, em vez de usar ``patch``
+   manualmente, você quase certamente desejará considerar o uso do Git.
+
+Uma pergunta feita com frequência na Linux Kernel Mailing List é como aplicar
+an patch ao kernel ou, mais especificamente, a qual kernel base um patch para
+uma das muitas árvores/branches deve ser aplicado. Esperamos que este documento
+explique isso a você.
+
+Além de explicar como aplicar e reverter patches, uma breve descrição das
+diferentes árvores do kernel (e exemplos de como aplicar seus patches
+específicos) também é fornecida.
+
+
+O que é um Patch?
+=================
+
+Um patch é um pequeno documento de texto que contém uma diferença (delta) de
+alterações entre duas versões diferentes de uma árvore de código-fonte. Os
+patches são criados com o programa ``diff``.
+
+Para aplicar um patch corretamente, você precisa saber de qual base ele foi
+gerado e em qual nova versão o patch transformará a árvore de código-fonte.
+Ambas as informações devem estar presentes nos metadados do arquivo de patch
+ou ser possíveis de deduzir a partir do nome do arquivo.
+
+
+Como eu aplico ou reverto um patch?
+===================================
+
+Você aplica um patch com o programa ``patch``. O programa patch lê um arquivo
+de diff (ou patch) e faz as alterações descritas nele na árvore de
+código-fonte.
+
+Os patches para o kernel Linux são gerados relativamente ao diretório pai que
+contém o diretório do código-fonte do kernel.
+
+Isso significa que os caminhos para os arquivos dentro do arquivo de patch
+contêm o nome dos diretórios do código-fonte do kernel contra os quais ele foi
+gerado (ou alguns outros nomes de diretório como "a/" e "b/").
+
+Como é improvável que isso corresponda ao nome do diretório do código-fonte do
+kernel na sua máquina local (mas frequentemente é uma informação útil para ver
+contra qual versão um patch sem identificação foi gerado), você deve entrar no
+seu diretório de código-fonte do kernel e, em seguida, remover o primeiro
+elemento do caminho dos nomes de arquivos no arquivo de patch ao aplicá-lo (o
+argumento ``-p1`` para o ``patch`` faz isso).
+
+Para reverter um patch aplicado anteriormente, use o argumento -R para o patch.
+Portanto, se você aplicou um patch desta forma::
+
+    patch -p1 < ../patch-x.y.z
+
+Você pode revertê-lo (desfazê-lo) assim::
+
+    patch -R -p1 < ../patch-x.y.z
+
+
+Como eu passo um arquivo de patch/diff para o ``patch``?
+========================================================
+
+Isso (como de costume no Linux e em outros sistemas operacionais do tipo UNIX)
+pode ser feito de várias maneiras diferentes.
+
+Em todos os exemplos abaixo, eu passo o arquivo (em formato não compactado) para
+o patch via stdin usando a seguinte sintaxe::
+
+    patch -p1 < path/to/patch-x.y.z
+
+Se você quer apenas ser capaz de seguir os exemplos abaixo e não deseja
+conhecer mais do que uma maneira de usar o patch, então você pode parar a
+leitura desta seção aqui.
+
+O patch também pode receber o nome do arquivo a ser usado através do argumento
+-i, desta forma::
+
+    patch -p1 -i path/to/patch-x.y.z
+
+Se o seu arquivo de patch estiver compactado com gzip ou xz e você não quiser
+descompactá-lo antes de aplicá-lo, você pode passá-lo para o patch desta outra
+forma::
+
+    xzcat path/to/patch-x.y.z.xz | patch -p1
+    bzcat path/to/patch-x.y.z.gz | patch -p1
+
+Se você deseja descompactar o arquivo de patch manualmente primeiro antes de
+aplicá-lo (o que presumo que você tenha feito nos exemplos abaixo), basta
+executar gunzip ou xz no arquivo -- desta forma::
+
+    gunzip patch-x.y.z.gz
+    xz -d patch-x.y.z.xz
+
+O que deixará você com um arquivo patch-x.y.z em texto puro que você pode
+passar para o patch via stdin ou pelo argumento ``-i``, conforme sua preferência.
+
+Alguns outros argumentos úteis para o patch são ``-s``, que faz com que o patch
+seja silencioso (exceto por erros), o que é bom para evitar que erros sumam da
+tela rolando rápido demais; e ``--dry-run``, que faz com que o patch apenas
+imprima uma lista do que aconteceria, mas sem realizar nenhuma alteração de
+fato. Por fim, ``--verbose`` diz ao patch para imprimir mais informações sobre o
+trabalho que está sendo realizado.
+
+
+Erros comuns ao aplicar patches
+===============================
+
+Quando o patch aplica um arquivo de patch, ele tenta verificar a integridade do
+arquivo de diferentes maneiras.
+
+Verificar se o arquivo parece um arquivo de patch válido e checar se o código ao
+redor dos trechos sendo modificados corresponde ao contexto fornecido no patch
+são apenas duas das verificações básicas de integridade que o patch faz.
+
+Se o patch encontrar algo que não pareça totalmente correto, ele tem duas
+opções. Ele pode se recusar a aplicar as alterações e abortar, ou pode tentar
+encontrar uma maneira de fazer o patch ser aplicado com algumas pequenas
+alterações.
+
+Um exemplo de algo que não está "totalmente correto" e que o patch tentará
+corrigir é se todo o contexto coincidir, as linhas sendo alteradas coincidirem,
+mas os números das linhas forem diferentes. Isso pode acontecer, por exemplo, se
+o patch fizer uma alteração no meio do arquivo, mas, por algum motivo, algumas
+linhas tiverem sido adicionadas ou removidas perto do início do arquivo. Nesse
+caso, tudo parece correto, apenas mudou um pouco para cima ou para baixo, e o
+patch geralmente ajustará os números das linhas e aplicará o patch.
+
+Sempre que o patch aplicar um patch que ele teve de modificar um pouco para
+fazer caber, ele avisará você dizendo que o patch foi aplicado com **fuzz**.
+Você deve ser cauteloso com tais alterações porque, embora o patch
+provavelmente tenha acertado, ele nem /sempre/ acerta, e o resultado às vezes
+será incorreto.
+
+Quando o patch encontra uma alteração que não consegue corrigir com fuzz, ele a
+rejeita imediatamente e deixa um arquivo com a extensão ``.rej`` (um arquivo de
+rejeição). Você pode ler esse arquivo para ver exatamente qual alteração não
+pôde ser aplicada, para que possa corrigi-la manualmente, se desejar.
+
+Se você não tem nenhum patch de terceiros aplicado ao seu código-fonte do
+kernel, mas apenas patches do kernel.org, e você aplica os patches na ordem
+correta, e não fez nenhuma modificação por conta própria nos arquivos de
+origem, então você nunca deveria ver uma mensagem de fuzz ou de rejeição (reject)
+do patch. Se você ainda assim vir tais mensagens, então há um alto risco de que
+sua árvore de código-fonte local ou o arquivo de patch estejam corrompidos de
+alguma forma. Nesse caso, você provavelmente deveria tentar baixar o patch
+novamente e, se as coisas ainda não estiverem certas, aconselha-se começar com
+uma árvore limpa baixada na íntegra do kernel.org.
+
+Vamos examinar um pouco mais algumas das mensagens que o patch pode produzir.
+
+Se o patch parar e apresentar um prompt ``File to patch:``, então o patch não
+conseguiu encontrar um arquivo para ser modificado. O mais provável é que você
+tenha esquecido de especificar -p1 ou esteja no diretório errado. Com menos
+frequência, você encontrará patches que precisam ser aplicados com ``-p0`` em
+vez de ``-p1`` (a leitura do arquivo de patch deve revelar se este é o caso -- se
+for, isso é um erro da pessoa que criou o patch, mas não é fatal).
+
+Se você receber ``Hunk #2 succeeded at 1887 with fuzz 2 (offset 7 lines).`` ou
+uma mensagem semelhante a essa, significa que o patch teve que ajustar o local
+da alteração (neste exemplo, ele precisou se mover 7 linhas de onde esperava
+fazer a alteração para fazê-la caber).
+
+O arquivo resultante pode ou não estar correto, dependendo do motivo pelo qual o
+arquivo estava diferente do esperado.
+
+Isso geralmente acontece se você tentar aplicar un patch que foi gerado contra uma
+versão de kernel diferente daquela que você está tentando modificar.
+
+Se você receber uma mensagem como ``Hunk #3 FAILED at 2387.``, significa que o
+patch não pôde ser aplicado corretamente e o programa patch não foi capaz de
+encontrar um caminho usando o fuzz. Isso gerará um arquivo ``.rej`` com a
+alteração que fez o patch falhar e também um arquivo ``.orig`` mostrando o
+conteúdo original que não pôde ser alterado.
+
+Se você receber ``Reversed (or previously applied) patch detected!  Assume -R? [n]``
+então o patch detectou que a alteração contida no patch parece já ter sido feita.
+
+Se você realmente aplicou este patch anteriormente e apenas o reaplicou por erro,
+basta dizer [n]ão (n) e abortar este patch. Se você aplicou este patch
+anteriormente e realmente pretendia revertê-lo, mas esqueceu de especificar -R,
+você pode dizer [**y**]es (sim) aqui para fazer o patch revertê-lo para você.
+
+Isso também pode acontecer se o criador do patch inverteu os diretórios de
+origem e destino ao criar o patch e, nesse caso, reverter o patch irá, na
+verdade, aplicá-lo.
+
+Uma mensagem semelhante a ``patch: **** unexpected end of file in patch`` ou
+``patch unexpectedly ends in middle of line`` significa que o patch não conseguiu
+fazer sentido do arquivo que você passou para ele. Ou o seu download está
+quebrado, ou você tentou passar para o patch um arquivo de patch compactado sem
+descompactá-lo primeiro, ou o arquivo de patch que você está usando foi alterado
+por um cliente de e-mail ou agente de transferência de e-mail em algum lugar pelo
+caminho, por exemplo, dividindo uma linha longa em duas linhas. Frequentemente,
+esses avisos podem ser corrigidos facilmente juntando (concatenando) as duas
+linhas que foram divididas.
+
+Como já mencionei acima, esses erros nunca deveriam acontecer se você aplicar um
+patch do kernel.org na versão correta de uma árvore de código-fonte não
+modificada. Portanto, se você obtiver esses erros com patches do kernel.org,
+você provavelmente deve assumir que o seu arquivo de patch ou a sua árvore está
+quebrada, e eu o aconselharia a recomeçar com um download limpo de uma árvore
+completa do kernel e do patch que deseja aplicar.
+
+Existem alternativas ao ``patch``?
+==================================
+
+Sim, existem alternativas.
+
+Você pode usar o programa ``interdiff`` (http://cyberelk.net/tim/patchutils/) para
+gerar um patch que represente as diferenças entre dois patches e, em seguida,
+aplicar o resultado.
+
+Isso permitirá que você passe de algo como 5.7.2 para 5.7.3 em um único
+passo. A flag -z do interdiff permite até mesmo passar patches em formato
+compactado com gzip ou bzip2 diretamente, sem o uso de zcat, bzcat ou
+descompactação manual.
+
+Aqui está como você passaria de 5.7.2 para 5.7.3 em um único passo::
+
+    interdiff -z ../patch-5.7.2.gz ../patch-5.7.3.gz | patch -p1
+
+Embora o interdiff possa economizar um ou dois passos, geralmente recomenda-se
+realizar os passos adicionais, já que o interdiff pode errar em alguns casos.
+
+Outra alternativa é o ``ketchup``, que é um script em python para download e
+aplicação automática de patches (https://www.selenic.com/ketchup/).
+
+Outras ferramentas úteis são o diffstat, que mostra um resumo das alterações
+feitas por um patch; o lsdiff, que exibe uma lista curta dos arquivos afetados
+em um arquivo de patch, junto com (opcionalmente) os números das linhas de
+início de cada patch; e o grepdiff, que exibe uma lista dos arquivos modificados
+por um patch onde o patch contém uma determinada expressão regular.
+
+
+Onde posso baixar os patches?
+=============================
+
+Os patches estão disponíveis em https://kernel.org/
+Os patches mais recentes estão vinculados na página principal, mas eles também
+possuem locais específicos.
+
+Os patches 5.x.y (-stable) e 5.x residem em
+
+    https://www.kernel.org/pub/linux/kernel/v5.x/
+
+Os patches incrementais 5.x.y residem em
+
+    https://www.kernel.org/pub/linux/kernel/v5.x/incr/
+
+Os patches -rc não são armazenados no servidor web, mas são gerados sob
+demanda a partir de tags do git, tais como
+
+    https://git.kernel.org/torvalds/p/v5.1-rc1/v5.0
+
+Os patches estáveis -rc residem em
+
+    https://www.kernel.org/pub/linux/kernel/v5.x/stable-review/
+
+
+Os kernels 5.x
+==============
+
+Estes são os lançamentos estáveis base publicados por Linus. O lançamento com o
+número mais alto é o mais recente.
+
+Se regressões ou outras falhas graves forem encontradas, um patch de correção
+-stable será lançado (veja abaixo) sobre esta base. Assim que um novo kernel
+base 5.x é lançado, um patch é disponibilizado contendo o delta entre o kernel
+5.x anterior e o novo.
+
+Para aplicar um patch mudando da versão 5.6 para a 5.7, você faria o seguinte
+(note que tais patches **NÃO** se aplicam sobre kernels 5.x.y, mas sim sobre o
+kernel base 5.x -- se você precisar mudar de 5.x.y para 5.x+1, você deve
+primeiro reverter o patch do 5.x.y).
+
+Aqui estão alguns exemplos::
+
+    # mudando de 5.6 para 5.7
+
+    $ cd ~/linux-5.6            # muda para o dir do fonte do kernel
+    $ patch -p1 < ../patch-5.7      # aplica o patch do 5.7
+    $ cd ..
+    $ mv linux-5.6 linux-5.7        # renomeia o dir do fonte
+
+    # mudando de 5.6.1 para 5.7
+
+    $ cd ~/linux-5.6.1          # muda para o dir do fonte do kernel
+    $ patch -p1 -R < ../patch-5.6.1     # reverte o patch do 5.6.1
+                        # o dir do fonte agora é o 5.6
+    $ patch -p1 < ../patch-5.7      # aplica o novo patch do 5.7
+    $ cd ..
+    $ mv linux-5.6.1 linux-5.7      # renomeia o dir do fonte
+
+Os kernels 5.x.y
+================
+
+Kernels com versões de 3 dígitos são kernels -stable (estáveis). Eles contêm
+correções críticas relativamente pequenas para problemas de segurança ou
+regressões significativas descobertas em um determinado kernel 5.x.
+
+Esta é a ramificação recomendada para usuários que desejam o kernel estável mais
+recente e não estão interessados em ajudar a testar versões de desenvolvimento
+ou experimentais.
+
+Se nenhum kernel 5.x.y estiver disponível, então o kernel 5.x com o número mais
+alto será o atual kernel estável.
+
+A equipe -stable fornece patches normais, bem como incrementais. Abaixo está
+como aplicar esses patches.
+
+Patches normais
+~~~~~~~~~~~~~~~
+
+Estes patches não são incrementais, o que significa que, por exemplo, o patch
+5.7.3 não se aplica sobre o código-fonte do kernel 5.7.2, mas sim sobre o
+código-fonte do kernel base 5.7.
+
+Portanto, para aplicar o patch 5.7.3 ao seu código-fonte existente do kernel
+5.7.2, você deve primeiro remover o patch 5.7.2 (de modo que reste apenas o
+código-fonte do kernel base 5.7) e então aplicar o novo patch 5.7.3.
+
+Aqui está um pequeno exemplo::
+
+    $ cd ~/linux-5.7.2          # muda para o dir do fonte do kernel
+    $ patch -p1 -R < ../patch-5.7.2     # reverte o patch do 5.7.2
+    $ patch -p1 < ../patch-5.7.3        # aplica o novo patch do 5.7.3
+    $ cd ..
+    $ mv linux-5.7.2 linux-5.7.3        # renomeia o dir do fonte do kernel
+
+Patches incrementais
+~~~~~~~~~~~~~~~~~~~~
+
+Os patches incrementais são diferentes: em vez de serem aplicados sobre o kernel
+base 5.x, eles são aplicados sobre o kernel estável anterior (5.x.y-1).
+
+Aqui está o exemplo para aplicar estes::
+
+    $ cd ~/linux-5.7.2          # muda para o dir do fonte do kernel
+    $ patch -p1 < ../patch-5.7.2-3      # aplica o novo patch do 5.7.3
+    $ cd ..
+    $ mv linux-5.7.2 linux-5.7.3        # renomeia o dir do fonte do kernel
+
+
+Os kernels -rc
+==============
+
+Estes são os kernels candidatos a lançamento (release-candidate). São kernels
+de desenvolvimento publicados por Linus sempre que ele considera que a árvore
+atual do git (a ferramenta de gerenciamento de código-fonte do kernel) está em
+um estado razoavelmente íntegro e adequado para testes.
+
+Estes kernels não são estáveis e você deve esperar quebras ocasionais se pretender
+executá-los. Esta é, no entanto, a mais estável das principais ramificações de
+desenvolvimento e é também o que eventualmente se tornará o próximo kernel
+estável, por isso é importante que seja testado pelo maior número possível de
+pessoas.
+
+Esta é uma boa ramificação para pessoas que querem ajudar a testar kernels de
+desenvolvimento, mas não querem executar algumas das coisas realmente
+experimentais (essas pessoas devem ver as seções sobre os kernels -next e -mm
+abaixo).
+
+Os patches -rc não são incrementais; eles se aplicam a um kernel base 5.x, assim
+como os patches 5.x.y descritos acima. A versão do kernel antes do sufixo -rcN
+indica a versão do kernel na qual este kernel -rc eventualmente se tornará.
+
+Portanto, 5.8-rc5 significa que este é o quinto candidato a lançamento para o
+kernel 5.8 e o patch deve ser aplicado sobre o código-fonte do kernel 5.7.
+
+Aqui estão 3 exemplos de como aplicar esses patches::
+
+    # primeiro, um exemplo de mudança do 5.7 para o 5.8-rc3
+
+    $ cd ~/linux-5.7            # muda para o dir do fonte do 5.7
+    $ patch -p1 < ../patch-5.8-rc3      # aplica o patch do 5.8-rc3
+    $ cd ..
+    $ mv linux-5.7 linux-5.8-rc3        # renomeia o dir do fonte
+
+    # agora vamos mudar do 5.8-rc3 para o 5.8-rc5
+
+    $ cd ~/linux-5.8-rc3            # muda para o dir do 5.8-rc3
+    $ patch -p1 -R < ../patch-5.8-rc3   # reverte o patch do 5.8-rc3
+    $ patch -p1 < ../patch-5.8-rc5      # aplica o novo patch do 5.8-rc5
+    $ cd ..
+    $ mv linux-5.8-rc3 linux-5.8-rc5    # renomeia o dir do fonte
+
+    # por fim, vamos tentar mudar do 5.7.3 para o 5.8-rc5
+
+    $ cd ~/linux-5.7.3          # muda para o dir do fonte do kernel
+    $ patch -p1 -R < ../patch-5.7.3     # reverte o patch do 5.7.3
+    $ patch -p1 < ../patch-5.8-rc5      # aplica o novo patch do 5.8-rc5
+    $ cd ..
+    $ mv linux-5.7.3 linux-5.8-rc5      # renomeia o dir do fonte do kernel
+
+
+Os patches -mm e a árvore linux-next
+====================================
+
+Os patches -mm são patches experimentais publicados por Andrew Morton.
+
+No passado, a árvore -mm também era usada para testar patches de subsistemas,
+mas essa função agora é realizada por meio da árvore
+`linux-next` (https://www.kernel.org/doc/man-pages/linux-next.html).
+Os mantenedores de subsistemas enviam seus patches primeiro para a linux-next e,
+durante a janela de mesclagem (merge window), enviam-nos diretamente para Linus.
+
+Os patches -mm servem como uma espécie de campo de testes para novos recursos e
+outros patches experimentais que não são mesclados por meio de uma árvore de
+subsistema. Assim que tais patches provam seu valor na -mm por um tempo, Andrew
+os envia para Linus para inclusão na linha principal (mainline).
+
+A árvore linux-next é atualizada diariamente e inclui os patches -mm. Ambas
+estão em constante fluxo e contêm muitos recursos experimentais, uma grande
+quantidade de patches de depuração (debugging) não apropriados para a linha
+principal etc., sendo as mais experimentais das ramificações descritas neste
+documento.
+
+Estes patches não são apropriados para uso em sistemas que devem ser estáveis e
+são mais arriscados de executar do que qualquer uma das outras ramificações
+(certifique-se de ter backups atualizados -- isso vale para qualquer kernel
+experimental, mas ainda mais para patches -mm ou ao usar um kernel da árvore
+linux-next).
+
+O teste dos patches -mm e da linux-next é imensamente apreciado, pois todo o
+objetivo deles é eliminar regressões, travamentos (crashes), bugs de corrupção
+de dados, quebras de compilação (e qualquer outro bug em geral) antes que as
+alterações sejam mescladas na árvore principal do Linus, que é mais estável.
+
+Mas os testadores da -mm e da linux-next devem estar cientes de que quebras são
+mais comuns do que em qualquer outra árvore.
+
+
+Isso conclui esta lista de explicações sobre as várias árvores do kernel.
+Espero que agora você tenha clareza sobre como aplicar os vários patches e
+ajudar a testar o kernel.
+
+Agradecimentos a Randy Dunlap, Rolf Eike Beer, Linus Torvalds, Bodo Eggert,
+Johannes Stezenbach, Grant Coady, Pavel Machek e outros que posso ter esquecido
+por suas revisões e contribuições para este documento.
\ No newline at end of file
diff --git a/Documentation/translations/pt_BR/process/backporting.rst b/Documentation/translations/pt_BR/process/backporting.rst
new file mode 100644
index 0000000000000..ce3f9fb4fc5ba
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/backporting.rst
@@ -0,0 +1,598 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+====================================
+Backporting e resolução de conflitos
+====================================
+
+:Autor: Vegard Nossum <vegard.nossum@oracle.com>
+
+.. contents::
+    :local:
+    :depth: 3
+    :backlinks: none
+
+Introdução
+==========
+
+Alguns desenvolvedores podem nunca precisar lidar de fato com backporting de
+patches, mesclagem de ramificações (branches) ou resolução de conflitos em seu
+trabalho diário, portanto, quando um conflito de mesclagem aparece, pode ser
+assustador. Felizmente, resolver conflitos é uma habilidade como qualquer outra,
+e existem muitas técnicas úteis que você pode usar para tornar o processo mais
+suave e aumentar sua confiança no resultado.
+
+Este documento tem como objetivo ser um guia abrangente e passo a passo para
+backporting e resolução de conflitos.
+
+Aplicando o patch a uma árvore
+==============================
+
+Às vezes, o patch que você está fazendo backport já existe como um commit do
+git, caso em que você apenas faz o cherry-pick dele diretamente usando
+``git cherry-pick``. No entanto, se o patch vier de um e-mail, como costuma
+acontecer no caso do kernel Linux, você precisará aplicá-lo a uma árvore usando
+``git am``.
+
+Se você já usou o ``git am``, provavelmente já sabe que ele é bastante exigente
+sobre o patch ser aplicado perfeitamente à sua árvore de código-fonte. Na
+verdade, você provavelmente já teve pesadelos com arquivos ``.rej`` e tentando
+editar o patch para fazê-lo ser aplicado.
+
+Recomenda-se fortemente, em vez disso, encontrar uma versão base apropriada onde
+o patch se aplique de forma limpa e *então* fazer o cherry-pick dele para a sua
+árvore de destino, pois isso fará com que o git exiba marcadores de conflito e
+permitirá que você resolva os conflitos com a ajuda do git e de quaisquer outras
+ferramentas de resolução de conflitos que preferir usar. Por exemplo, se você
+quiser aplicar um patch que acabou de chegar na LKML a um kernel estável mais
+antigo, você pode aplicá-lo ao kernel principal (mainline) mais recente e, em
+seguida, fazer o cherry-pick dele para a sua ramificação estável mais antiga.
+
+Geralmente é melhor usar exatamente a mesma base a partir da qual o patch foi
+gerado, mas isso não importa tanto, desde que ele se aplique de forma limpa e
+não esteja muito longe da base original. O único problema ao aplicar o patch na
+base "errada" é que isso pode trazer mais alterações não relacionadas no
+contexto do diff ao fazer o cherry-pick dele para a ramificação mais antiga.
+
+Um bom motivo para preferir o ``git cherry-pick`` em vez do ``git am`` é que o
+git conhece o histórico preciso de um commit existente, de modo que ele saberá
+quando o código foi movido de lugar e teve seus números de linha alterados; isso,
+por sua vez, torna menos provável que o patch seja aplicado no lugar errado (o
+que pode resultar em erros silenciosos ou conflitos confusos).
+
+Se você estiver usando o `b4`_. e estiver aplicando o patch diretamente de um
+e-mail, você pode usar o ``b4 am`` com as opções ``-g``/``--guess-base`` e
+``-3``/``--prep-3way`` para fazer parte disso automaticamente (veja a
+`apresentação do b4`_ para mais informações). No entanto, o restante deste
+artigo assumirá que você está fazendo um ``git cherry-pick`` simples.
+
+.. _b4: https://people.kernel.org/monsieuricon/introducing-b4-and-patch-attestation
+.. _apresentação do b4: https://youtu.be/mF10hgVIx9o?t=2996
+
+Assim que tiver o patch no git, você pode prosseguir e fazer o cherry-pick dele
+em sua árvore de código-fonte. Não se esqueça de fazer o cherry-pick com ``-x``
+se quiser um registro por escrito de onde o patch veio!
+
+Note que, se você estiver enviando um patch para a ramificação estável (stable),
+o formato é ligeiramente diferente; a primeira linha após a linha de assunto
+precisa ser::
+
+    commit <upstream commit> upstream
+
+ou::
+
+    [ Upstream commit <upstream commit> ]
+
+Resolvendo conflitos
+====================
+
+Ih, rapaz; o cherry-pick falhou com uma mensagem vagamente ameaçadora::
+
+    CONFLICT (content): Merge conflict
+
+O que fazer agora?
+
+Em geral, os conflitos aparecem quando o contexto do patch (ou seja, as linhas
+que estão sendo alteradas e/ou as linhas que cercam as alterações) não
+corresponde ao que está na árvore à qual você está tentando aplicar o patch.
+
+No caso de backports, o que provavelmente aconteceu foi que a ramificação
+(branch) a partir da qual você está fazendo o backport contém patches que não
+estão na ramificação para a qual você está fazendo o backport. No entanto, o
+inverso também é possível. Em qualquer caso, o resultado é um conflito que
+precisa ser resolvido.
+
+Se a sua tentativa de cherry-pick falhar com um conflito, o git edita os
+arquivos automaticamente para incluir os chamados marcadores de conflito,
+mostrando onde está o conflito e como as duas ramificações divergiram. Resolver
+o conflito normalmente significa editar o resultado final de forma que ele leve
+em consideração esses outros commits.
+
+A resolução do conflito pode ser feita manualmente em um editor de texto comum
+ou usando uma ferramenta dedicada de resolução de conflitos.
+
+Muitas pessoas preferem usar seu editor de texto comum e editar o conflito
+diretamente, pois pode ser mais fácil entender o que você está fazendo e
+controlar o resultado final. Definitivamente, existem prós e contras em cada
+método, e às vezes há valor em usar ambos.
+
+Não abordaremos o uso de ferramentas de mesclagem (merge tools) dedicadas aqui,
+além de fornecer algumas indicações de várias ferramentas que você poderia usar:
+
+-  `Modo Emacs Ediff <https://www.emacswiki.org/emacs/EdiffMode>`__
+-  `vimdiff/gvimdiff <https://linux.die.net/man/1/vimdiff>`__
+-  `KDiff3 <http://kdiff3.sourceforge.net/>`__
+-  `TortoiseMerge <https://tortoisesvn.net/TortoiseMerge.html>`__
+-  `Meld <https://meldmerge.org/help/>`__
+-  `P4Merge <https://www.perforce.com/products/helix-core-apps/merge-diff-tool-p4merge>`__
+-  `Beyond Compare <https://www.scootersoftware.com/>`__
+-  `IntelliJ <https://www.jetbrains.com/help/idea/resolve-conflicts.html>`__
+-  `VSCode <https://code.visualstudio.com/docs/editor/versioncontrol>`__
+
+Para configurar o git para funcionar com elas, veja ``git mergetool --help`` ou
+a `documentação oficial do git-mergetool`_.
+
+.. _documentação oficial do git-mergetool: https://git-scm.com/docs/git-mergetool
+
+Patches pré-requisitos
+----------------------
+
+A maioria dos conflitos acontece porque a ramificação para a qual você está
+fazendo o backport não possui alguns patches em comparação com a ramificação a
+partir da qual você está fazendo o backport. No caso mais geral (como a
+mesclagem de duas ramificações independentes), o desenvolvimento poderia ter
+ocorrido em qualquer uma das ramificações, ou as ramificações simplesmente
+divergiram -- talvez a sua ramificação mais antiga tenha recebido alguns outros
+backports que, por si só, precisaram de resoluções de conflitos, causando uma
+divergência.
+
+É importante sempre identificar o commit ou os commits que causaram o conflito,
+pois, caso contrário, você não poderá ter confiança na correção da sua
+resolução. Como um bônus adicional, especialmente se o patch for em uma área com
+a qual você não está muito familiarizado, os registros de alterações (changelogs)
+desses commits frequentemente lhe darão o contexto para entender o código e os
+problemas ou armadilhas potenciais com a sua resolução de conflito.
+
+git log
+~~~~~~~
+
+Um bom primeiro passo é olhar o ``git log`` para o arquivo que possui o
+conflito -- isso geralmente é suficiente quando não há muitos patches no
+arquivo, mas pode ficar confuso se o arquivo for grande e frequentemente
+modificado por patches. Você deve executar o ``git log`` no intervalo de commits
+entre a sua ramificação atualmente ativa (``HEAD``) e o pai do patch que você está
+escolhendo (``<commit>``), ou seja::
+
+    git log HEAD..<commit>^ -- <path>
+
+Melhor ainda, se você quiser restringir essa saída a uma única função (porque é
+onde o conflito aparece), você pode usar a seguinte sintaxe::
+
+    git log -L:'\<function\>':<path> HEAD..<commit>^
+
+.. note::
+     O ``\<`` e o ``\>`` ao redor do nome da função garantem que as
+     correspondências fiquem ancoradas em um limite de palavra. Isso é
+     importante, pois essa parte é na verdade uma regex e o git segue apenas a
+     primeira correspondência; portanto, se você usar
+     ``-L:thread_stack:kernel/fork.c``, ele poderá fornecer apenas resultados
+     para a função ``try_release_thread_stack_to_cache``, embora existam muitas
+     outras funções naquele arquivo contendo a string ``thread_stack`` em seus
+     nomes.
+
+Outra opção útil para o ``git log`` é a ``-G``, que permite filtrar por certas
+strings que aparecem nos diffs dos commits que você está listando::
+
+    git log -G'regex' HEAD..<commit>^ -- <path>
+
+Esta também pode ser uma maneira prática de encontrar rapidamente quando algo
+(por exemplo, uma chamada de função ou uma variável) foi alterado, adicionado
+ou removido. A string de busca é uma expressão regular, o que significa que você
+pode potencialmente buscar por coisas mais específicas, como atribuições a um
+membro específico de uma struct::
+
+    git log -G'\->index\>.*='
+
+git blame
+~~~~~~~~~
+
+Outra maneira de encontrar commits pré-requisitos (embora apenas o mais recente
+para um determinado conflito) é executar o ``git blame``. Neste caso, você
+precisa executá-lo no commit pai do patch para o qual está fazendo o
+cherry-pick e no arquivo onde o conflito apareceu, ou seja::
+
+    git blame <commit>^ -- <path>
+
+Este comando também aceita o argumento ``-L`` (para restringir a saída a uma
+única função), mas, neste caso, você especifica o nome do arquivo no final do
+comando, como de costume::
+
+    git blame -L:'\<function\>' <commit>^ -- <path>
+
+Navegue até o local onde o conflito ocorreu. A primeira coluna da saída do
+blame é o ID do commit do patch que adicionou uma determinada linha de código.
+
+Pode ser uma boa ideia dar um ``git show`` nesses commits e ver se eles se
+parecem com a possível origem do conflito. Às vezes, haverá mais de um desses
+commits, seja porque múltiplos commits alteraram linhas diferentes da mesma área
+de conflito *ou* porque múltiplos patches subsequentes alteraram a mesma linha
+(ou linhas) várias vezes. Neste último caso, você pode ter que executar o
+``git blame`` novamente e especificar a versão mais antiga do arquivo para
+analisar, a fim de cavar mais fundo no histórico do arquivo.
+
+Patches pré-requisitos vs. incidentais
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Tendo encontrado o patch que causou o conflito, você precisa determinar se ele
+é um pré-requisito para o patch que você está fazendo o backport ou se é apenas
+incidental e pode ser pulado. Um patch incidental seria aquele que toca no mesmo
+código que o patch para o qual você está fazendo o backport, mas não altera a
+semântica do código de nenhuma forma relevante. Por exemplo, um patch de limpeza
+de espaços em branco é completamente incidental -- da mesma forma, um patch que
+simplesmente renomeia uma função ou uma variável também seria incidental. Por
+outro lado, se a função que está sendo alterada sequer existe na sua ramificação
+atual, então isso não seria nada incidental e você precisa considerar com
+cuidado se o patch que adiciona a função deve ser aplicado via cherry-pick
+primeiro.
+
+Se você descobrir que há um patch pré-requisito necessário, então você precisa
+parar e fazer o cherry-pick dele em vez disso. Se você já resolveu alguns
+conflitos em um arquivo diferente e não quer fazer isso de novo, você pode
+criar uma cópia temporária daquele arquivo.
+
+Para abortar o cherry-pick atual, vá em frente e execute
+``git cherry-pick --abort`` e, em seguida, reinicie o processo de cherry-pick
+com o ID do commit do patch pré-requisito.
+
+Entendendo os marcadores de conflito
+------------------------------------
+
+Diffs combinados
+~~~~~~~~~~~~~~~~
+
+Digamos que você tenha decidido não fazer o cherry-pick (ou o revert) de patches
+adicionais e quer apenas resolver o conflito. O Git terá inserido marcadores de
+conflito no seu arquivo. Por padrão, isso se parecerá com algo como::
+
+    <<<<<<< HEAD
+    this is what's in your current tree before cherry-picking
+    =======
+    this is what the patch wants it to be after cherry-picking
+    >>>>>>> <commit>... title
+
+Isso é o que você veria se abrisse o arquivo no seu editor. No entanto, se você
+executasse o ``git diff`` sem nenhum argumento, a saída seria algo assim::
+
+    $ git diff
+    [...]
+    ++<<<<<<<< HEAD
+     +this is what's in your current tree before cherry-picking
+    ++========
+    + this is what the patch wants it to be after cherry-picking
+    ++>>>>>>>> <commit>... title
+
+Quando você está resolvendo um conflito, o comportamento do ``git diff`` difere
+do seu comportamento normal. Note as duas colunas de marcadores de diff em vez
+da coluna única usual; este é o chamado "`diff combinado`_", aqui mostrando o
+diff de 3 vias (ou diff-de-diffs) entre:
+
+#. a ramificação atual (antes do cherry-pick) e o diretório de trabalho atual, e
+#. a ramificação atual (antes do cherry-pick) e o arquivo como ele fica após o
+   patch original ter sido aplicado.
+
+.. _diff combinado: https://git-scm.com/docs/diff-format#_combined_diff_format
+
+Diffs melhores
+~~~~~~~~~~~~~~
+
+Diffs combinados de 3 vias incluem todas as outras alterações que aconteceram
+no arquivo entre a sua ramificação atual e a ramificação a partir da qual você
+está fazendo o cherry-pick. Embora isso seja útil para detectar outras
+alterações que você precisa levar em consideração, também torna a saída do
+``git diff`` um tanto intimidadora e difícil de ler. Em vez disso, você pode
+preferir executar ``git diff HEAD`` (ou ``git diff --ours``), que mostra apenas
+o diff entre a ramificação atual antes do cherry-pick e o diretório de trabalho
+atual. Ele se parece com isso::
+
+    $ git diff HEAD
+    [...]
+    +<<<<<<<< HEAD
+     this is what's in your current tree before cherry-picking
+    +========
+    +this is what the patch wants it to be after cherry-picking
+    +>>>>>>>> <commit>... title
+
+Como você pode ver, isso é lido exatamente como qualquer outro diff e deixa claro
+quais linhas estão na ramificação atual e quais linhas estão sendo adicionadas
+porque fazem parte do conflito de mesclagem ou do patch que está sendo aplicado
+via cherry-pick.
+
+Estilos de mesclagem e diff3
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+O estilo padrão de marcador de conflito mostrado acima é conhecido como o estilo
+``merge``. Também está disponível um outro estilo, conhecido como o estilo
+``diff3``, que se parece com isso::
+
+    <<<<<<< HEAD
+    this is what is in your current tree before cherry-picking
+    ||||||| parent of <commit> (title)
+    this is what the patch expected to find there
+    =======
+    this is what the patch wants it to be after being applied
+    >>>>>>> <commit> (title)
+
+Como você pode ver, isso tem 3 partes em vez de 2, e inclui o que o git
+esperava encontrar lá, mas não encontrou. É *altamente recomendável* usar este
+estilo de conflito, pois deixa muito mais claro o que o patch realmente alterou;
+ou seja, ele permite que você compare as versões de antes e depois do arquivo
+para o commit do qual está fazendo o cherry-pick. Isso permite que você tome
+melhores decisões sobre como resolver o conflito.
+
+Para alterar os estilos de marcadores de conflito, você pode usar o seguinte
+comando::
+
+    git config merge.conflictStyle diff3
+
+Existe uma terceira opção, ``zdiff3``, introduzida no `Git 2.35`_, que possui as
+mesmas 3 seções do ``diff3``, mas onde as linhas comuns foram cortadas, tornando
+a área de conflito menor em alguns casos.
+
+.. _Git 2.35: https://github.blog/2022-01-24-highlights-from-git-2-35/
+
+Iterando em resoluções de conflito
+----------------------------------
+
+O primeiro passo em qualquer processo de resolução de conflito é entender o
+patch para o qual você está fazendo o backport. Para o kernel Linux, isso é
+especialmente importante, pois uma alteração incorreta pode levar ao travamento
+de todo o sistema -- ou pior, a uma vulnerabilidade de segurança não detectada.
+
+Entender o patch pode ser fácil ou difícil, dependendo do próprio patch, do
+registro de alterações (changelog) e da sua familiaridade com o código que está
+sendo alterado. No entanto, uma boa pergunta para cada alteração (ou cada bloco/
+hunk do patch) seria: "Por que este hunk está no patch?" As respostas a essas
+perguntas orientarão a sua resolução de conflito.
+
+Processo de resolução
+~~~~~~~~~~~~~~~~~~~~~
+
+Às vezes, a coisa mais fácil a fazer é apenas remover tudo, exceto a primeira
+parteda do conflito, deixando o arquivo essencialmente inalterado, e aplicar
+as alterações manualmente. Talvez o patch esteja alterando um argumento de
+chamada de função de ``0`` para ``1``, enquanto uma alteração conflitante
+adicionou um parâmetro totalmente novo (e insignificante) ao final da lista de
+parâmetros; nesse caso, é bastante fácil alterar o argumento de ``0`` para ``1``
+manualmente e deixar o restante dos argumentos como estão. Esta técnica de
+aplicar alterações manualmente é mais útil se o conflito tiver trazido muito
+contexto não relacionado com o qual você não precisa realmente se preocupar.
+
+Para conflitos particularmente difíceis com muitos marcadores de conflito, você
+pode usar ``git add`` ou ``git add -i`` para indexar (stage) seletivamente as
+suas resoluções para tirá-las do caminho; isso também permite que você use
+``git diff HEAD`` para ver sempre o que ainda resta a ser resolvido ou
+``git diff --cached`` para ver como está o seu patch até o momento.
+
+Lidando com arquivos renomeados
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Uma das coisas mais irritantes que podem acontecer ao fazer o backport de um
+patch é descobrir que um dos arquivos modificados foi renomeado, pois isso
+geralmente significa que o git sequer colocará marcadores de conflito, mas
+apenas lavará as mãos e dirá (parafraseando): "Caminho não mesclado! Faça você o
+trabalho..."
+
+Geralmente existem algumas maneiras de lidar com isso. Se o patch para o
+arquivo renomeado for pequeno, como uma alteração de uma única linha, a coisa
+mais fácil é prosseguir, aplicar a alteração manualmente e dar o caso por
+encerrado. Por outro lado, se a alteração for grande ou complicada, você
+definitivamente não vai querer fazê-la manualmente.
+
+Como uma primeira tentativa, você pode tentar algo assim, que reduzirá o limite
+(threshold) de detecção de renomeação para 30% (por padrão, o git usa 50%, o que
+significa que dois arquivos precisam ter pelo menos 50% em comum para que ele
+considere um par de adição/remoção como uma renomeação potencial)::
+
+  git cherry-pick -strategy=recursive -Xrename-threshold=30
+
+Às vezes, a coisa certa a fazer será fazer o backport também do patch que
+realizou a renomeação, mas esse definitivamente não é o caso mais comum. Em vez
+disso, o que você pode fazer é renomear temporariamente o arquivo na
+ramificação para a qual está fazendo o backport (usando ``git mv`` e commitando
+o resultado), reiniciar a tentativa de cherry-pick do patch, renomear o arquivo
+de volta (``git mv`` e commitando novamente) e, finalmente, esmagar (squash) o
+resultado usando ``git rebase -i`` (veja o `tutorial de rebase`_) para que ele
+apareça como um único commit quando você terminar.
+
+.. _tutorial de rebase: [https://medium.com/@slamflipstrom/a-beginners-guide-to-squashing-commits-with-git-rebase-8185cf6e62ec](https://medium.com/@slamflipstrom/a-beginners-guide-to-squashing-commits-with-git-rebase-8185cf6e62ec)
+
+Pegadinhas
+----------
+
+Argumentos de função
+~~~~~~~~~~~~~~~~~~~~
+
+Preste atenção às alterações em argumentos de função! É fácil deixar passar
+detalhes e pensar que duas linhas são iguais quando, na verdade, elas diferem em
+algum pequeno detalhe, como qual variável foi passada como argumento
+(especialmente se as duas variáveis forem de apenas um caractere e parecerem
+iguais, como i e j).
+
+Tratamento de erros
+~~~~~~~~~~~~~~~~~~~
+
+Se você fizer o cherry-pick de um patch que inclua uma instrução ``goto``
+(geralmente para tratamento de erros), é absolutamente imperativo verificar em
+dobro se o rótulo (label) de destino ainda está correto na ramificação para a
+qual você está fazendo o backport. O mesmo vale para instruções ``return``,
+``break`` e ``continue`` adicionadas.
+
+O tratamento de erros geralmente fica localizado no final da função, portanto,
+pode não fazer parte do conflito, mesmo que possa ter sido alterado por outros
+patches.
+
+Uma boa maneira de garantir que você revise os caminhos de erro é sempre usar
+``git diff -W`` e ``git show -W`` (também conhecido como ``--function-context``)
+ao inspecionar suas alterações. Para código em C, isso mostrará toda a função
+que está sendo alterada em um patch. Uma das coisas que frequentemente dão
+errado durante backports é que algo mais na função mudou em qualquer uma das
+ramificações a partir da qual ou para a qual você está fazendo o backport. Ao
+incluir a função inteira no diff, você obtém mais contexto e pode identificar
+mais facilmente problemas que de outra forma poderiam passar despercebidos.
+
+Código refatorado
+~~~~~~~~~~~~~~~~~
+
+Algo que acontece com bastante frequência é o código ser refatorado ao "isolar"
+uma sequência ou padrão de código comum em uma função auxiliar. Ao fazer o
+backport de patches para uma área onde tal refatoração ocorreu, você efetivamente
+precisa fazer o inverso ao realizar o backport: um patch para um único local pode
+precisar ser aplicado a múltiplos locais na versão que recebeu o backport. (Um
+indicativo para este cenário é que uma função foi renomeada -- mas nem sempre é o
+caso.)
+
+Para evitar backports incompletos, vale a pena tentar descobrir se o patch
+corrige um bug que aparece em mais de um lugar. Uma maneira de fazer isso seria
+usar o ``git grep``. (Isso, na verdade, é uma boa ideia de se fazer em geral, não
+apenas para backports.) Se você descobrir que o mesmo tipo de correção se
+aplicaria a outros lugares, também vale a pena ver se esses lugares existem no
+upstream -- se não existirem, é provável que o patch precise ser ajustado. O
+``git log`` é seu amigo para descobrir o que aconteceu com essas áreas, já que o
+``git blame`` não mostrará código que foi removido.
+
+Se você encontrar outras instâncias do mesmo padrão na árvore do upstream e não
+tiver certeza se isso também é um bug, pode valer a pena perguntar ao autor do
+patch. Não é incomum encontrar novos bugs durante o processo de backport!
+
+Verificando o resultado
+=======================
+
+colordiff
+---------
+
+Tendo commitado um novo patch sem conflitos, você pode agora comparar o seu
+patch com o patch original. É altamente recomendável que você use uma
+ferramenta como o `colordiff`_ que possa mostrar dois arquivos lado a lado e
+colori-los de acordo com as alterações entre eles::
+
+    colordiff -yw -W 200 <(git diff -W <upstream commit>^-) <(git diff -W HEAD^-) | less -SR
+
+.. _colordiff: https://www.colordiff.org/
+
+Aqui, ``-y`` significa fazer uma comparação lado a lado; ``-w`` ignora
+espaços em branco e ``-W 200`` define a largura da saída (caso contrário, ele
+usará 130 por padrão, o que costuma ser um pouco pouco).
+
+A sintaxe ``rev^-`` é um atalho prático para ``rev^..rev``, essencialmente
+fornecendo apenas o diff para aquele único commit; veja também a
+`documentação oficial do git rev-parse`_.
+
+.. _documentação oficial do git rev-parse: https://git-scm.com/docs/git-rev-parse#_other_rev_parent_shorthand_notations
+
+Novamente, note a inclusão de ``-W`` para o ``git diff``; isso garante que você
+verá a função completa para qualquer função que tenha mudado.
+
+Uma coisa incrivelmente importante que o colordiff faz é destacar as linhas que
+são diferentes. Por exemplo, se um ``goto`` de tratamento de erros teve seus
+rótulos alterados entre o patch original e o que sofreu o backport, o colordiff
+irá mostrá-los lado a lado, mas destacados em uma cor diferente. Assim, é fácil
+ver que as duas instruções ``goto`` estão saltando para rótulos diferentes. Da
+mesma forma, linhas que não foram modificadas por nenhum dos patches, mas que
+diferem no contexto, também serão destacadas e, portanto, se destacarão durante
+uma inspeção manual.
+
+Claro, esta é apenas uma inspeção visual; o teste real é compilar e executar o
+kernel (ou programa) com o patch aplicado.
+
+Testes de compilação (Build testing)
+------------------------------------
+
+Não abordaremos os testes em tempo de execução aqui, mas pode ser uma boa ideia
+compilar apenas os arquivos tocados pelo patch como uma verificação rápida de
+sanidade. Para o kernel Linux, você pode compilar arquivos únicos assim,
+assumindo que você tenha o ``.config`` e o ambiente de compilação configurados
+corretamente::
+
+    make caminho/para/o/arquivo.o
+
+Note que isso não descobrirá erros de ligação (linker errors), então você ainda
+deve fazer uma compilação completa após verificar que o arquivo único compila.
+Ao compilar o arquivo único primeiro, você pode evitar ter que esperar por uma
+compilação completa *caso* haja erros de compilador em qualquer um dos arquivos
+que você alterou.
+
+Testes em tempo de execução
+---------------------------
+
+Mesmo um teste de compilação ou de boot bem-sucedido não é necessariamente o
+suficiente para descartar uma dependência ausente em algum lugar. Embora as
+chances sejam pequenas, pode haver alterações de código onde duas modificações
+independentes no mesmo arquivo resultem em nenhum conflito, nenhum erro em tempo
+de compilação e erros em tempo de execução apenas em casos excepcionais.
+
+Um exemplo concreto disso foi um par de patches para o código de entrada de
+chamada de sistema (system call entry code), onde o primeiro patch salvava/
+restaurava um registrador e um patch posterior fazia uso do mesmo registrador
+em algum lugar no meio dessa sequência. Como não havia sobreposição entre as
+alterações, era possível fazer o cherry-pick do segundo patch, não ter conflitos
+e acreditar que tudo estava bem, quando na verdade o código estava agora
+sobrescrevendo (scribbling over) um registrador não salvo.
+
+Embora a vasta maioria dos erros seja capturada durante a compilação ou ao
+exercitar o código superficialmente, a única maneira de *realmente* verificar um
+backport é revisar o patch final com o mesmo nível de escrutínio que você daria
+(ou deveria dar) a qualquer outro patch. Ter testes unitários e testes de
+regressão ou outros tipos de testes automáticos pode ajudar a aumentar a
+confiança na correção de um backport.
+
+Enviando backports para a árvore estável (stable)
+=================================================
+
+À medida que os mantenedores da árvore estável tentam aplicar correções da linha
+principal (mainline) em seus kernels estáveis via cherry-pick, eles podem enviar
+e-mails solicitando backports quando encontram conflitos; veja, por exemplo,
+<https://lore.kernel.org/stable/2023101528-jawed-shelving-071a@gregkh/>.
+Esses e-mails normalmente incluem os passos exatos que você precisa seguir para
+fazer o cherry-pick do patch para a árvore correta e enviá-lo.
+
+Uma coisa a se certificar é que o seu registro de alterações (changelog) esteja
+em conformidade com o formato esperado::
+
+  <original patch title>
+
+  [ Upstream commit <mainline rev> ]
+
+  <rest of the original changelog>
+  [ <summary of the conflicts and their resolutions> ]
+  Signed-off-by: <your name and email>
+
+A linha "Upstream commit" às vezes é ligeiramente diferente dependendo da versão
+estável. Versões mais antigas usavam este formato::
+
+  commit <mainline rev> upstream.
+
+O mais comum é indicar a versão do kernel à qual o patch se aplica na linha de
+assunto do e-mail (usando, por exemplo,
+``git send-email --subject-prefix='PATCH 6.1.y'``), mas você também pode
+colocá-la na área do Signed-off-by: ou abaixo da linha ``---``.
+
+Os mantenedores da árvore estável esperam envios separados para cada versão
+estável ativa, e cada envio também deve ser testado separadamente.
+
+Algumas palavras finais de conselho
+===================================
+
+1) Aborde o processo de backport com humildade.
+2) Entenda o patch para o qual você está fazendo o backport; isso significa ler
+   tanto o registro de alterações (changelog) quanto o código.
+3) Seja honesto sobre a sua confiança no resultado ao enviar o patch.
+4) Peça aprovações explícitas (acks) aos mantenedores relevantes.
+
+Exemplos
+========
+
+O texto acima mostra, de forma geral, o processo idealizado de backport de um
+patch. Para um exemplo mais concreto, veja este tutorial em vídeo onde dois
+patches são portados da linha principal (mainline) para a estável (stable):
+`Backporting Linux Kernel Patches`_.
+
+.. _Backporting Linux Kernel Patches: https://youtu.be/sBR7R1V2FeA
\ No newline at end of file
diff --git a/Documentation/translations/pt_BR/process/botching-up-ioctls.rst b/Documentation/translations/pt_BR/process/botching-up-ioctls.rst
new file mode 100644
index 0000000000000..193297619ac22
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/botching-up-ioctls.rst
@@ -0,0 +1,256 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+============================================
+(Como evitar) Deixar as ioctls malfeitas
+============================================
+
+De: https://blog.ffwll.ch/2013/11/botching-up-ioctls.html
+
+Por: Daniel Vetter, Copyright © 2013 Intel Corporation
+
+Uma percepção clara que os hackers de gráficos do kernel tiveram nos últimos
+anos é que tentar criar uma interface unificada para gerenciar as unidades de
+execução e a memória em GPUs completamente diferentes é um esforço inútil.
+Portanto, hoje em dia, cada driver tem seu próprio conjunto de ioctls para
+alocar memória e enviar trabalho para a GPU. O que é bom, já que não há mais a
+insanidade na forma de interfaces falsamente genéricas, mas que na verdade só
+são usadas uma vez. No entanto, a desvantagem clara é que há muito mais
+potencial para estragar as coisas.
+
+Para evitar repetir todos os mesmos erros novamente, escrevi algumas das lições
+aprendidas enquanto fazia um trabalho malfeito para o driver drm/i915. A maioria
+delas aborda apenas tecnicalidades e não os problemas macro (big-picture), como
+deveria ser exatamente a aparência da ioctl de envio de comando. Aprender essas
+lições é provavelmente algo que cada driver de GPU tem que fazer por conta
+própria.
+
+
+Pré-requisitos
+--------------
+
+Primeiro, os pré-requisitos. Sem estes você já falhou, porque precisará
+adicionar uma camada de compatibilidade de 32 bits (compat layer):
+
+ * Use apenas inteiros de tamanho fixo. Para evitar conflitos com typedefs no
+   espaço de usuário (userspace), o kernel possui tipos especiais como __u32 e
+   __s64. Use-os.
+
+ * Alinhe tudo ao tamanho natural e use preenchimento (padding) explícito.
+   Plataformas de 32 bits não alinham necessariamente valores de 64 bits a
+   limites (boundaries) de 64 bits, mas plataformas de 64 bits o fazem. Portanto,
+   sempre precisamos de padding para o tamanho natural para acertar isso.
+
+ * Preencha a struct inteira para um múltiplo de 64 bits se a estrutura contiver
+   tipos de 64 bits -- caso contrário, o tamanho da estrutura diferirá entre
+   32 bits e 64 bits. Ter um tamanho de estrutura diferente prejudica ao passar
+   matrizes (arrays) de estruturas para o kernel, ou se o kernel verificar o
+   tamanho da estrutura, o que o core do drm, por exemplo, faz.
+
+ * Ponteiros são __u64, convertidos de/para um uintptr_t no lado do espaço de
+   usuário e de/para um void __user * no kernel. Tente de verdade não atrasar
+   essa conversão ou, pior ainda, manipular o __u64 bruto pelo seu código, pois
+   isso diminui a verificação que ferramentas como o sparse podem fornecer. A
+   macro u64_to_user_ptr pode ser usada no kernel para evitar avisos sobre
+   inteiros e ponteiros de tamanhos diferentes.
+
+
+Conceitos básicos
+-----------------
+
+Evitadas as alegrias de escrever uma camada de compatibilidade (compat layer),
+podemos dar uma olhada nos deslizes básicos. Negligenciar estes pontos tornará a
+compatibilidade retroativa e futura uma verdadeira dor de cabeça. E, como errar
+na primeira tentativa é garantido, você certamente terá uma segunda iteração ou,
+pelo menos, uma extensão para qualquer interface fornecida.
+
+ * Tenha uma maneira clara para o espaço de usuário descobrir se a sua nova
+   ioctl ou extensão de ioctl é suportada em um determinado kernel. Se você não
+   puder confiar que os kernels antigos rejeitarão as novas flags/modos ou
+   ioctls (já que fazer isso foi deixado de lado no passado), então você
+   precisará de uma flag de recurso (feature flag) do driver ou de um número de
+   revisão em algum lugar.
+
+ * Tenha um plano para estender as ioctls com novas flags ou novos campos no
+   final da estrutura. O core do drm verifica o tamanho passado para cada
+   chamada de ioctl e preenche com zero (zero-extends) quaisquer divergências
+   entre o kernel e o espaço de usuário. Isso ajuda, mas não é uma solução
+   completa, já que um espaço de usuário mais novo em um kernel mais antigo não
+   notará que os campos recém-adicionados no final estão sendo ignorados.
+   Portanto, isso ainda exige novas flags de recurso do driver.
+
+ * Verifique todos os campos e flags não utilizados, além de todo o preenchimento
+   (padding), para garantir que estejam em 0, e rejeite a ioctl se esse não for
+   o caso. Caso contrário, seu excelente plano para extensões futuras irá por
+   água abaixo, pois alguém enviará uma struct de ioctl com lixo de pilha
+   (stack garbage) aleatório nas partes ainda não utilizadas. O que, então,
+   consolida na ABI que esses campos nunca poderão ser usados para nada além de
+   lixo. Esta também é a razão pela qual você deve preencher explicitamente todas
+   as estruturas, mesmo que nunca as use em uma matriz (array) -- o padding que
+   o compilador possa inserir poderia conter lixo.
+
+ * Tenha casos de teste simples para tudo o que foi mencionado acima.
+
+
+Diversão com caminhos de erro (Error Paths)
+-------------------------------------------
+
+Hoje em dia, não temos mais nenhuma desculpa para que os drivers drm sejam pequenos
+exploits de root disfarçados. Isso significa que precisamos tanto de uma
+validação completa de entrada quanto de caminhos sólidos de tratamento de erros
+-- as GPUs eventualmente vão parar de funcionar (die) nos casos mais bizarros
+de qualquer maneira:
+
+ * A ioctl deve verificar se há estouros de matriz (array overflows). Ela também
+   precisa verificar estouros superiores/inferiores (over/underflows) e problemas
+   de limitação (clamping) de valores inteiros em geral. O exemplo usual são os
+   valores de posicionamento de sprite alimentados diretamente no hardware, onde
+   o hardware possui apenas 12 bits ou algo assim. Funciona perfeitamente até que
+   algum servidor de exibição bizarro não se preocupe em fazer o clamping por si
+   mesmo e o cursor dê a volta (wrap around) na tela.
+
+ * Tenha casos de teste simples para cada caso de falha de validação de entrada
+   na sua ioctl. Verifique se o código de erro corresponde às suas expectativas.
+   E, finalmente, certifique-se de testar apenas um único caminho de erro em
+   cada subteste, enviando dados que, de outra forma, seriam perfeitamente
+   válidos. Sem isso, uma verificação anterior já poderia rejeitar a ioctl e
+   ofuscar (shadow) o caminho de código que você realmente deseja testar,
+   ocultando bugs e regressões.
+
+ * Torne todas as suas ioctls reiniciáveis (restartable). Primeiro, o X (X11)
+   realmente ama sinais (signals) e, segundo, isso permitirá que você teste 90%
+   de todos os caminhos de tratamento de erro apenas interrompendo sua suíte de
+   testes principal constantemente com sinais. Graças ao amor do X por sinais,
+   você obterá uma excelente cobertura de base de todos os seus caminhos de erro
+   praticamente de graça para drivers de gráficos. Além disso, seja consistente
+   na forma como você lida com a reinicialização de ioctls -- por exemplo, o drm
+   possui um pequeno helper drmIoctl em sua biblioteca de espaço de usuário. O
+   driver i915 estragou isso com a ioctl set_tiling; agora estamos presos para
+   sempre com algumas semânticas arcanas tanto no kernel quanto no espaço de
+   usuário.
+
+ * Se você não puder tornar um determinado caminho de código reiniciável, torne
+   uma tarefa travada pelo menos finalizável (killable). As GPUs simplesmente
+   morrem, e seus usuários não vão gostar mais de você se você travar a máquina
+   inteira deles (por meio de um processo do X impossível de matar). Se a
+   recuperação de estado ainda for muito complicada, tenha um timeout ou uma
+   rede de segurança de verificação de travamento (hangcheck) como um esforço de
+   última hora (last-ditch) caso o hardware enlouqueça (gone bananas).
+
+ * Tenha casos de teste para os cenários mais complexos (corner cases) no seu
+   código de recuperação de erros -- é fácil demais criar um deadlock entre seu
+   código de hangcheck e os processos que estão aguardando (waiters).
+
+
+Tempo, Espera e a Perda de Prazos
+---------------------------------
+
+As GPUs fazem quase tudo de forma assíncrona, portanto, temos a necessidade de
+cronometrar operações e aguardar pelas que estão pendentes. Esse é um negócio
+realmente complicado; no momento, nenhuma das ioctls suportadas pelo drm/i915
+acerta isso completamente, o que significa que ainda há toneladas de lições para
+aprender aqui.
+
+ * Use CLOCK_MONOTONIC como seu tempo de referência, sempre. É o que o alsa, o
+   drm e o v4l usam por padrão hoje em dia. Mas informe ao espaço de usuário
+   quais carimbos de data/hora (timestamps) são derivados de domínios de relógio
+   diferentes, como o relógio principal do seu sistema (fornecido pelo kernel)
+   ou algum contador de hardware independente em outro lugar. Os relógios vão
+   divergir se você olhar de perto o suficiente, mas se as ferramentas de
+   medição de desempenho tiverem essa informação, elas poderão ao menos compensar.
+   Se o seu espaço de usuário puder obter os valores brutos de alguns relógios
+   (por exemplo, por meio de instruções de amostragem de contador de desempenho
+   no fluxo de comandos), considere expor esses também.
+
+ * Use __s64 para segundos mais __u64 para nanossegundos para especificar o
+   tempo. Não é a especificação de tempo mais conveniente, mas é praticamente o
+   padrão.
+
+ * Verifique se os valores de tempo de entrada estão normalizados e rejeite-os
+   caso contrário. Note que a struct nativa do kernel, ktime, possui um inteiro
+   sinalizado tanto para segundos quanto para nanossegundos, portanto, cuidado
+   aqui.
+
+ * Para timeouts, use tempos absolutos. Se você for um bom sujeito e tiver
+   tornado a sua ioctl reiniciável, os timeouts relativos tendem a ser muito
+   imprecisos (coarse) e podem estender indefinidamente o seu tempo de espera
+   devido ao arredondamento a cada reinicialização. Especialmente se o seu relógio
+   de referência for algo realmente lento, como o contador de quadros da tela
+   (display frame counter). Vestindo o chapéu de advogado de especificações, isso
+   não é um bug, já que os timeouts sempre podem ser estendidos -- mas os usuários
+   com certeza vão odiar você se as belas animações deles começarem a gaguejar
+   (stutter) devido a isso.
+
+ * Considere descartar quaisquer ioctls de espera síncrona com timeouts e apenas
+   entregue um evento assíncrono em um descritor de arquivo passível de poll
+   (pollable file descriptor). Isso se encaixa muito melhor no loop principal de
+   aplicações orientadas a eventos.
+
+ * Tenha casos de teste para cenários complexos (corner-cases), especialmente se
+   os valores de retorno para eventos já concluídos, esperas bem-sucedidas e
+   esperas que estouraram o tempo (timed-out) são todos sãos e adequados às suas
+   necessidades.
+
+
+Evitando o vazamento de recursos (Leaking Resources, Not)
+---------------------------------------------------------
+
+Um driver drm completo essencialmente implementa um pequeno SO, mas especializado
+para as plataformas de GPU fornecidas. Isso significa que um driver precisa
+expor toneladas de handles (identificadores) para diferentes objetos e outros
+recursos para o espaço de usuário. Fazer isso corretamente traz seu próprio
+pequeno conjunto de armadilhas:
+
+ * Sempre vincule o tempo de vida (lifetime) de seus recursos criados
+   dinamicamente ao tempo de vida de um descritor de arquivo (file descriptor -
+   fd). Considere usar um mapeamento 1:1 se o seu recurso precisar ser
+   compartilhado entre processos -- a passagem de fds sobre unix domain sockets
+   também simplifica o gerenciamento do tempo de vida para o espaço de usuário.
+
+ * Sempre tenha suporte a O_CLOEXEC.
+
+ * Certifique-se de que você tem isolamento suficiente entre os diferentes
+   clientes. Por padrão, escolha um namespace privado por fd, o que força
+   qualquer compartilhamento a ser feito de forma explícita. Só adote um
+   namespace mais global por dispositivo se os objetos forem verdadeiramente
+   únicos do dispositivo. Um contraexemplo nas interfaces de modeset do drm é
+   que os objetos de modeset por dispositivo, como conectores, compartilham um
+   namespace com objetos de framebuffer, que na maioria das vezes não são
+   compartilhados de forma alguma. Um namespace separado, privado por padrão,
+   para os framebuffers teria sido mais adequado.
+
+ * Pense sobre os requisitos de unicidade para os handles do espaço de usuário.
+   Por exemplo, para a maioria dos drivers drm, é um bug do espaço de usuário
+   enviar o mesmo objeto duas vezes na mesma ioctl de envio de comando. Mas,
+   se os objetos forem compartilháveis, o espaço de usuário precisa saber se
+   já viu um objeto importado de outro processo ou não. Eu ainda não tentei isso
+   sozinho devido à falta de uma nova classe de objetos, mas considere usar
+   números de inode em seus descritores de arquivo compartilhados como
+   identificadores únicos -- é assim que arquivos reais também são diferenciados.
+   Infelizmente, isso requer um sistema de arquivos virtual completo no kernel.
+
+
+Por último, mas não menos importante
+------------------------------------
+
+Nem todo problema precisa de uma nova ioctl:
+
+ * Pense bem se você realmente quer uma interface privada do driver. Claro que
+   é muito mais rápido aprovar uma interface privada do driver do que se envolver
+   em discussões longas por uma solução mais genérica. E, ocasionalmente, criar
+   uma interface privada para liderar um novo conceito é o que se exige. Mas,
+   no final, assim que a interface genérica surgir, você acabará mantendo duas
+   interfaces. Indefinidamente.
+
+ * Considere outras interfaces além de ioctls. Um atributo sysfs é muito melhor
+   para configurações por dispositivo ou para objetos filhos com tempos de vida
+   razoavelmente estáticos (como conectores de saída no drm com todos os seus
+   atributos de sobreposição de detecção). Ou talvez apenas a sua suíte de
+   testes precise dessa interface e, nesse caso, o debugfs, com seu aviso de
+   isenção de responsabilidade por não ter uma ABI estável, seria melhor.
+
+Finalmente, o objetivo principal é acertar na primeira tentativa, pois se o seu
+driver se provar popular e suas plataformas de hardware forem duradouras, você
+ficará preso a uma determinada ioctl essencialmente para sempre. Você pode
+tentar depreciar ioctls horríveis em iterações mais novas do seu hardware, mas
+geralmente leva anos para conseguir isso. E depois mais anos até que o último
+usuário capaz de reclamar sobre regressões desapareça também.
\ No newline at end of file
diff --git a/Documentation/translations/pt_BR/process/code-of-conduct-interpretation.rst b/Documentation/translations/pt_BR/process/code-of-conduct-interpretation.rst
new file mode 100644
index 0000000000000..866c9f7e7a12d
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/code-of-conduct-interpretation.rst
@@ -0,0 +1,257 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Interpretação do Código de Conduta do Kernel Linux
+==================================================
+
+O :ref:`pt_BR_code_of_conduct` é um documento geral que tem como objetivo
+fornecer um conjunto de regras para quase todas as comunidades de código
+aberto. Toda comunidade de código aberto é única e o kernel Linux não é
+exceção. Por causa disso, este documento descreve como nós, na comunidade
+do kernel Linux, o interpretaremos. Nós também não esperamos que esta
+interpretação seja estática ao longo do tempo, e a ajustaremos conforme
+necessário.
+
+O esforço de desenvolvimento do kernel Linux é um processo muito pessoal
+em comparação com as formas "tradicionais" de desenvolvimento de software.
+Suas contribuições e as ideias por trás delas serão cuidadosamente
+revisadas, frequentemente resultando em críticas e apontamentos. A
+revisão quase sempre exigirá melhorias antes que o material possa ser
+incluído no kernel. Saiba que isso acontece porque todos os envolvidos
+querem ver a melhor solução possível para o sucesso geral do Linux. Este
+processo de desenvolvimento provou criar o kernel de sistema operacional
+mais robusto de todos os tempos, e nós não queremos fazer nada que cause
+a diminuição da qualidade do envio e do resultado final.
+
+Mantenedores
+------------
+
+O Código de Conduta usa o termo "mantenedores" várias vezes. Na
+comunidade do kernel, um "mantenedor" é qualquer pessoa responsável por
+um subsistema, driver ou arquivo, e que esteja listada no arquivo
+MAINTAINERS na árvore de código-fonte do kernel.
+
+Responsabilidades
+-----------------
+
+O Código de Conduta menciona direitos e responsabilidades para os
+mantenedores, e isso precisa de alguns esclarecimentos adicionais.
+
+Em primeiro lugar e acima de tudo, é uma expectativa razoável que os
+mantenedores liderem pelo exemplo.
+
+Dito isto, nossa comunidade é vasta e ampla, e não há um novo requisito
+para que os mantenedores lidem unilateralmente com o comportamento de
+outras pessoas nas partes da comunidade onde atuam. Essa responsabilidade
+é de todos nós e, em última análise, o Código de Conduta documenta os
+caminhos finais de escalonamento em caso de preocupações não resolvidas
+em relação a questões de conduta.
+
+Os mantenedores devem estar dispostos a ajudar quando ocorrerem problemas
+e trabalhar com outras pessoas na comunidade quando necessário. Não tenha
+medo de entrar em contato com o Conselho Consultivo Técnico (TAB) ou
+outros mantenedores se não tiver certeza de como lidar com as situações
+que surgirem. Isso não será considerado um relato de violação, a menos
+que você queira que seja. Se não tiver certeza sobre como abordar o TAB
+ou quaisquer outros mantenedores, entre em contato com nossa mediadora
+de conflitos, Joanna Lee <jlee@linuxfoundation.org>.
+
+No final, "sejam gentis uns com os outros" é realmente o objetivo final
+para todos. Sabemos que todos são humanos e que todos falhamos às vezes,
+mas o objetivo principal para todos nós deve ser trabalhar em direção a
+resoluções amigáveis dos problemas. A aplicação do código de conduta será
+apenas uma opção de último recurso.
+
+Nosso objetivo de criar um sistema operacional robusto e tecnicamente
+avançado e a complexidade técnica envolvida exigem naturalmente
+conhecimento técnico e tomada de decisões.
+
+O conhecimento técnico exigido varia dependendo da área de contribuição.
+Ele é determinado principalmente pelo contexto e pela complexidade técnica
+e apenas secundariamente pelas expectativas de contribuidores e
+mantenedores.
+
+Tanto as expectativas de conhecimento técnico quanto a tomada de decisões
+estão sujeitas a discussão, mas no final das contas há uma necessidade
+básica de sermos capazes de tomar decisões para progredir. Esta
+prerrogativa está nas mãos dos mantenedores e da liderança do projeto e
+espera-se que seja usada de boa fé.
+
+Como consequência, definir expectativas de conhecimento técnico, tomar
+decisões e rejeitar contribuições inadequadas não são vistos como uma
+violação do Código de Conduta.
+
+Embora os mantenedores sejam em geral receptivos aos novatos, sua
+capacidade de ajudar os contribuidores a superar os obstáculos de entrada
+é limitada, portanto, eles devem definir prioridades. Isso, também, não
+deve ser visto como uma violação do Código de Conduta. A comunidade do
+kernel está ciente disso e fornece programas de nível de entrada de
+várias formas, como o kernelnewbies.org.
+
+Escopo
+------
+
+A comunidade do kernel Linux interage primariamente em um conjunto de listas
+de e-mail públicas distribuídas por vários servidores diferentes controlados
+por várias empresas ou indivíduos diferentes. Todas essas listas estão
+definidas no arquivo MAINTAINERS na árvore de código-fonte do kernel.
+Quaisquer e-mails enviados para essas listas de e-mail são considerados
+cobertos pelo Código de Conduta.
+
+Desenvolvedores que usam o bugzilla do kernel.org e outras ferramentas de
+rastreamento de bugs ou bugzilla de subsistemas devem seguir as diretrizes
+do Código de Conduta. A comunidade do kernel Linux não possui um endereço
+de e-mail de projeto "oficial" ou endereço "oficial" de mídia social.
+Qualquer atividade realizada usando uma conta de e-mail do kernel.org deve
+seguir o Código de Conduta conforme publicado para o kernel.org, assim como
+qualquer indivíduo usando uma conta de e-mail corporativa deve seguir as
+regras específicas daquela corporação.
+
+O Código de Conduta não proíbe que se continue a incluir nomes, endereços de
+e-mail e comentários associados em mensagens de listas de discussão,
+mensagens de log de alterações (changelog) do kernel ou comentários no
+código.
+
+A interação em outros fóruns é coberta pelas regras que se aplicam a tais
+fóruns e, em geral, não é coberta pelo Código de Conduta. Exceções podem
+ser consideradas para circunstâncias extremas.
+
+As contribuições enviadas para o kernel devem usar linguagem apropriada.
+O conteúdo já existente que precede o Código de Conduta não será tratado
+agora como uma violação. No entanto, a linguagem inapropriada pode ser vista
+como um bug; tais bugs serão corrigidos mais rapidamente se quaisquer partes
+interessadas enviarem patches com esse propósito. Expressões que atualmente
+fazem parte da API de usuário/kernel, ou que refletem a terminologia usada
+em padrões ou especificações publicadas, não são consideradas bugs.
+
+Aplicação
+---------
+
+O endereço listado no Código de Conduta vai para o Comitê do Código de
+Conduta. Os membros exatos que recebem esses e-mails a qualquer momento
+estão listados em https://kernel.org/code-of-conduct.html. Os membros não
+podem acessar relatos feitos antes de se juntarem ou após terem deixado o
+comitê.
+
+O Comitê do Código de Conduta consiste em membros voluntários da comunidade
+nomeados pelo TAB, bem como um mediador profissional agindo como um terceiro
+neutro. Os processos que o comitê do Código de Conduta usará para lidar com
+os relatos variam e dependerão das circunstâncias individuais; no entanto,
+este arquivo serve como documentação para o processo geral utilizado.
+
+Qualquer membro do comitê, incluindo o mediador, pode ser contatado
+diretamente se o relator não desejar incluir todo o comitê em uma
+reclamação ou preocupação.
+
+O Comitê do Código de Conduta revisa os casos de acordo com os processos
+(veja acima) e consulta o TAB conforme a necessidade e a conveniência,
+por exemplo, para solicitar e receber informações sobre a comunidade do
+kernel.
+
+Quaisquer decisões a respeito de recomendações de aplicação (enforcement)
+serão levadas ao TAB para implementação junto aos mantenedores relevantes,
+se necessário. Uma vez que o TAB aprove uma ou mais das medidas descritas
+no escopo do banimento pelo voto de dois terços dos membros, o Comitê do
+Código de Conduta aplicará as medidas aprovadas pelo TAB. Quaisquer membros
+do Comitê do Código de Conduta que sirvam no TAB não votarão nas medidas.
+
+Em intervalos trimestrais, o Comitê do Código de Conduta e o TAB fornecerão
+um relatório resumindo os relatos anonimizados que o comitê do Código de
+Conduta recebeu e o status deles, bem como os detalhes de quaisquer
+decisões aprovadas pelo TAB, incluindo dados completos e identificáveis da
+votação.
+
+Como a maneira pela qual interpretamos e aplicamos o Código de Conduta
+evoluirá com o tempo, este documento será atualizado quando necessário para
+refletir quaisquer mudanças.
+
+Aplicação para Violações de Comportamento Inaceitável do Código de Conduta
+----------------------------------------------------------------------------
+
+O comitê do Código de Conduta trabalha para garantir que nossa comunidade
+continue a ser inclusiva e promova discussões e pontos de vista diversos, e
+trabalha para melhorar essas características ao longo do tempo. A maioria
+dos relatos que o Comitê do Código de Conduta recebe decorre da
+compreensão incorreta sobre o processo de desenvolvimento e os papéis,
+responsabilidades e o direito dos mantenedores de tomar decisões sobre a
+aceitação de código. Estes são resolvidos através do esclarecimento do
+processo de desenvolvimento e do escopo do Código de Conduta.
+
+Comportamentos inaceitáveis podem interromper a colaboração respeitosa por um
+curto período de tempo e impactar negativamente a saúde da comunidade a longo
+prazo. Comportamentos inaceitáveis frequentemente são resolvidos quando os
+indivíduos reconhecem seu comportamento e se retratam por ele no ambiente em
+que a violação ocorreu.
+
+O Comitê do Código de Conduta recebe relatos sobre comportamentos
+inaceitáveis quando eles não são resolvidos através de discussões na
+comunidade. O comitê do Código de Conduta toma medidas para restaurar a
+colaboração produtiva e respeitosa quando um comportamento inaceitável
+impactou negativamente esse relacionamento.
+
+O Comitê do Código de Conduta tem a obrigação de manter os relatos e as
+informações dos relatores em sigilo. Os relatos podem vir de partes lesadas
+e membros da comunidade que são observadores de comportamentos
+inaceitáveis. O Comitê do Código de Conduta tem a responsabilidade de
+investigar e resolver esses relatos, trabalhando com todas as partes
+envolvidas.
+
+O Comitê do Código de Conduta trabalha com o indivíduo para promover uma
+mudança em sua compreensão sobre a importância de reparar os danos causados
+por seu comportamento à parte lesada e o impacto negativo a longo prazo na
+comunidade.
+
+O objetivo é alcançar uma resolução que seja aceitável para todas as partes.
+Se trabalhar com o indivíduo não trouxer o resultado desejado, o Comitê do
+Código de Conduta avaliará outras medidas, como buscar um pedido de
+desculpas público para reparar o dano.
+
+Buscar retratação pública pela violação
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+O Comitê do Código de Conduta chama a atenção publicamente para o comportamento
+no ambiente em que a violação ocorreu, buscando uma retratação pública
+pela violação.
+
+Uma retratação pública pela violação é o primeiro passo para reconstruir a
+confiança. A confiança é essencial para o sucesso contínuo e a saúde da
+comunidade, que opera com base na confiança e no respeito.
+
+Medidas corretivas se não houver uma retratação pública pela violação
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+O Comitê do Código de Conduta determina o próximo curso de ação para restaurar
+a colaboração saudável, recomendando medida(s) corretiva(s) ao TAB para
+aprovação.
+
+- Banir o infrator de participar do processo de desenvolvimento do kernel por
+  um período de até um ciclo completo de desenvolvimento do kernel. O Comitê
+  do Código de Conduta pode exigir uma retratação pública como condição
+  para suspender o banimento.
+
+O escopo do banimento por um período de tempo pode incluir:
+
+    a. negar contribuições de patch e pull requests
+    b. pausar a colaboração com o infrator ignorando suas contribuições e/ou
+       bloqueando sua(s) conta(s) de e-mail
+    c. restringir sua capacidade de se comunicar pelas plataformas kernel.org,
+       como listas de discussão e sites de mídia social
+
+Uma vez que o TAB aprove uma ou mais das medidas descritas no escopo do
+banimento por dois terços dos membros votando a favor das medidas, o Comitê do
+Código de Conduta aplicará a(s) medida(s) aprovada(s) pelo TAB em colaboração
+com a comunidade, mantenedores, submantenedores e administradores do
+kernel.org. Quaisquer membros do Comitê do Código de Conduta atuando no TAB
+não votarão nas medidas.
+
+O Comitê do Código de Conduta está ciente do impacto negativo que buscar uma
+retratação pública e instituir um banimento pode ter sobre os indivíduos.
+Também está ciente dos danos a longo prazo para a comunidade que podem resultar
+da falta de ação quando tais violações públicas graves ocorrem.
+
+A eficácia da(s) medida(s) corretiva(s) aprovada(s) pelo TAB depende da
+confiança e cooperação da comunidade, mantenedores, submantenedores e
+administradores do kernel.org na sua aplicação.
+
+O Comitê do Código de Conduta espera sinceramente que comportamentos
+inaceitáveis que exijam a busca de retratações públicas continuem a ser
+ocorrências extremamente raras no futuro.
\ No newline at end of file
diff --git a/Documentation/translations/pt_BR/process/code-of-conduct.rst b/Documentation/translations/pt_BR/process/code-of-conduct.rst
new file mode 100644
index 0000000000000..1ab171bf21cff
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/code-of-conduct.rst
@@ -0,0 +1,88 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. _pt_BR_code_of_conduct:
+
+Código de Conduta de Compromisso do Colaborador
++++++++++++++++++++++++++++++++++++++++++++++++
+
+Nosso Compromisso
+=================
+
+No interesse de promover um ambiente aberto e acolhedor, nós, como colaboradores
+e mantenedores, nos comprometemos a tornar a participação em nosso projeto e em
+nossa comunidade uma experiência livre de assédio para todos, independentemente
+da idade, tamanho corporal, deficiência, etnia, características sexuais,
+identidade e expressão de gênero, nível de experiência, educação, status
+socioeconômico, nacionalidade, aparência pessoal, raça, religião ou identidade
+e orientação sexual.
+
+Nossos Padrões
+==============
+
+Exemplos de comportamentos que contribuem para a criação de um ambiente positivo
+incluem:
+
+* Utilizar linguagem acolhedora e inclusiva
+* Respeitar pontos de vista e experiências divergentes
+* Aceitar críticas construtivas de forma cortês
+* Focar no que é melhor para a comunidade
+* Demonstrar empatia para com outros membros da comunidade
+
+Exemplos de comportamentos inaceitáveis por parte dos participantes incluem:
+
+* O uso de linguagem ou imagens de teor sexual, bem como atenção ou avanços
+  sexuais indesejados
+* Provocações (*trolling*), comentários insultuosos/depreciativos e ataques
+  pessoais ou políticos
+* Assédio público ou privado
+* Publicar informações privadas de terceiros, como endereço físico ou eletrônico,
+  sem permissão explícita
+* Qualquer outra conduta que possa ser razoavelmente considerada inadequada em um
+  ambiente profissional
+
+Nossas Responsabilidades
+========================
+
+Os mantenedores são responsáveis por esclarecer os padrões de comportamento
+aceitável e devem tomar medidas corretivas apropriadas e justas em resposta a
+quaisquer casos de comportamento inaceitável.
+
+Os mantenedores têm o direito e a responsabilidade de remover, editar ou rejeitar
+comentários, commits, código, edições em wiki, issues e outras contribuições que
+não estejam alinhadas a este Código de Conduta, ou de banir temporária ou
+permanentemente qualquer colaborador por outros comportamentos que considerem
+inadequados, ameaçadores, ofensivos ou prejudiciais.
+
+Escopo
+======
+
+Este Código de Conduta se aplica tanto dentro dos espaços do projeto quanto em
+espaços públicos quando um indivíduo estiver representando o projeto ou sua
+comunidade. Exemplos de representação do projeto ou comunidade incluem o uso de um
+endereço de e-mail oficial do projeto, publicações por meio de uma conta oficial
+em redes sociais ou atuar como um representante nomeado em um evento online ou
+presencial. A representação de um projeto pode ser detalhada e esclarecida de forma
+adicional pelos mantenedores do projeto.
+
+Aplicação
+=========
+
+Casos de comportamento abusivo, de assédio ou inaceitável sob qualquer outro aspecto
+podem ser relatados entrando em contato com o Comitê do Código de Conduta pelo e-mail
+<conduct@kernel.org>. Todas as reclamações serão revisadas e investigadas, resultando
+em uma resposta considerada necessária e adequada às circunstâncias. O Comitê do
+Código de Conduta é obrigado a manter a confidencialidade em relação ao denunciante
+de um incidente. Detalhes adicionais sobre políticas de aplicação específicas podem
+ser publicados separadamente.
+
+Atribuição
+==========
+
+Este Código de Conduta foi adaptado do Contributor Covenant, versão 1.4,
+disponível em https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+Interpretação
+=============
+
+Consulte o documento :ref:`code_of_conduct_interpretation` para entender como a
+comunidade do kernel Linux interpretará este documento.
diff --git a/Documentation/translations/pt_BR/process/contribution-maturity-model.rst b/Documentation/translations/pt_BR/process/contribution-maturity-model.rst
new file mode 100644
index 0000000000000..bb003c4cd0d96
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/contribution-maturity-model.rst
@@ -0,0 +1,111 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=======================================================
+Modelos de Maturidade para Contribuição no Kernel Linux
+=======================================================
+
+
+Contexto
+========
+
+Como parte do Linux Kernel Maintainers’ Summit de 2021, houve uma
+`discussão <https://lwn.net/Articles/870581/>`_ sobre os desafios na
+contratação de mantenedores do kernel, bem como a sucessão de mantenedores.
+Algumas das conclusões daquela discussão incluíram que as empresas que fazem
+parte da comunidade do Kernel Linux precisam permitir que os engenheiros atuem
+como mantenedores como parte de seu trabalho, para que possam crescer e se
+tornar líderes respeitados e, eventualmente, mantenedores do kernel. Para
+apoiar um fluxo forte de talentos, os desenvolvedores devem ser autorizados e
+incentivados a assumir contribuições no upstream, como revisar os patches de
+outras pessoas, refatorar a infraestrutura do kernel e escrever documentação.
+
+Para tanto, o Conselho Técnico Consultivo (Technical Advisory Board - TAB) da
+Linux Foundation propõe este Modelo de Maturidade para Contribuição no Kernel
+Linux. Essas expectativas comuns para o engajamento da comunidade upstream visam
+aumentar a influência de desenvolvedores individuais, aumentar a colaboração
+de organizações e melhorar a saúde geral do ecossistema do Kernel Linux.
+
+O TAB insta as organizações a avaliarem continuamente seu modelo de maturidade
+em Open Source e a se comprometerem com melhorias para se alinharem a este
+modelo. Para ser eficaz, essa avaliação deve incorporar o feedback de toda a
+organização, incluindo a gerência e os desenvolvedores de todos os níveis de
+senioridade. No espírito do Open Source, incentivamos as organizações a
+publicarem suas avaliações e planos para melhorar seu engajamento com a
+comunidade upstream.
+
+Nível 0
+=======
+
+* Engenheiros de Software não têm permissão para contribuir com patches para o
+  kernel Linux.
+
+
+Nível 1
+=======
+
+* Engenheiros de Software têm permissão para contribuir com patches para o
+  kernel Linux, seja como parte de suas responsabilidades de trabalho ou em seu
+  próprio tempo.
+
+Nível 2
+=======
+
+* Espera-se que os Engenheiros de Software contribuam para o Kernel Linux como
+  parte de suas responsabilidades de trabalho.
+* Os Engenheiros de Software receberão apoio para participar de conferências
+  relacionadas ao Linux como parte de seu trabalho.
+* As contribuições de código no upstream de um Engenheiro de Software serão
+  consideradas em promoções e avaliações de desempenho.
+
+Nível 3
+=======
+
+* Espera-se que os Engenheiros de Software revisem patches (incluindo patches
+  escritos por engenheiros de outras empresas) como parte de suas
+  responsabilidades de trabalho.
+* A contribuição com apresentações ou artigos para conferências acadêmicas ou
+  relacionadas ao Linux (como as organizadas pela Linux Foundation, Usenix,
+  ACM, etc.) é considerada parte do trabalho do engenheiro.
+* As contribuições comunitárias de um Engenheiro de Software serão consideradas
+  em promoções e avaliações de desempenho.
+* As organizações relatarão regularmente as métricas de suas contribuições em
+  open source e acompanharão essas métricas ao longo do tempo. Essas métricas
+  podem ser publicadas apenas internamente na organização ou, a critério da
+  organização, algumas ou todas podem ser publicadas externamente. As métricas
+  fortemente sugeridas incluem:
+
+  * O número de contribuições ao kernel no upstream por equipe ou organização
+    (por exemplo, todas as pessoas que se reportam a um gerente, diretor ou
+    vice-presidente).
+  * A porcentagem de desenvolvedores de kernel que fizeram contribuições no
+    upstream em relação ao total de desenvolvedores de kernel na organização.
+  * O intervalo de tempo entre os kernels usados nos servidores e/ou produtos
+    da organização e a data de publicação do kernel upstream no qual o kernel
+    interno se baseia.
+  * O número de commits fora da árvore (out-of-tree) presentes nos kernels
+    internos.
+
+Nível 4
+=======
+
+* Os Engenheiros de Software são incentivados a dedicar uma parte do seu tempo
+  de trabalho focados no Trabalho no Upstream, o qual é definido como a revisão
+  de patches, atuação em comitês de programa, melhoria da infraestrutura central
+  do projeto -- como escrita ou manutenção de testes, redução de dívida técnica
+  no upstream, escrita de documentação, etc.
+* Os Engenheiros de Software recebem apoio para ajudar a organizar conferências
+  relacionadas ao Linux.
+* As organizações considerarão o feedback dos membros da comunidade em
+  avaliações de desempenho oficiais.
+
+Nível 5
+=======
+
+* O desenvolvimento de kernel no upstream é considerado um cargo formal, com
+  pelo menos um terço do tempo do engenheiro dedicado à realização de Trabalho
+  no Upstream.
+* As organizações buscarão ativamente o feedback dos membros da comunidade como
+  um fator nas avaliações de desempenho oficiais.
+* As organizações relatarão internamente e de forma regular a proporção entre o
+  Trabalho no Upstream e o trabalho focado em atingir diretamente os objetivos
+  de negócios.
\ No newline at end of file
diff --git a/Documentation/translations/pt_BR/process/deprecated.rst b/Documentation/translations/pt_BR/process/deprecated.rst
new file mode 100644
index 0000000000000..2793a1fc92fe3
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/deprecated.rst
@@ -0,0 +1,422 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+========================================================================
+Interfaces, recursos de linguagem, atributos e convenções obsoletos
+========================================================================
+
+Em um mundo perfeito, seria possível converter todas as instâncias de alguma
+API obsoleta para a nova API e remover completamente a API antiga em um único
+ciclo de desenvolvimento. No entanto, devido ao tamanho do kernel, à hierarquia
+de manutenção e ao cronograma, nem sempre é viável realizar esse tipo de
+conversão de uma só vez. Isso significa que novas instâncias podem acabar
+entrando no kernel enquanto as antigas estão sendo removidas, apenas aumentando
+o volume de trabalho para remover a API. A fim de instruir os desenvolvedores
+sobre o que se tornou obsoleto e o porquê, esta lista foi criada para servir de
+referência quando o uso de elementos obsoletos for proposto para inclusão no
+kernel.
+
+__deprecated
+------------
+Embora este atributo marque visualmente uma interface como obsoleta, ele `não
+gera mais avisos durante as compilações
+<https://git.kernel.org/linus/771c035372a036f83353eef46dbb829780330234>`_ porque
+um dos objetivos permanentes do kernel é compilar sem avisos (*warnings*), e
+ninguém estava de fato agindo para remover essas interfaces obsoletas. Embora o
+uso de `__deprecated` seja útil para sinalizar uma API antiga em um arquivo de
+cabeçalho (*header file*), não é a solução completa. Tais interfaces devem ser
+totalmente removidas do kernel ou adicionadas a este arquivo para desestimular
+outros desenvolvedores de usá-las no futuro.
+
+BBUG() e BUG_ON()
+-----------------
+Em vez disso, use WARN() e WARN_ON() e trate a condição de erro "impossível"
+da forma mais amigável possível. Embora a família de APIs BUG() tenha sido
+originalmente projetada para agir como uma asserção de "situação impossível" e
+eliminar uma thread do kernel de forma "segura", ela se mostrou arriscada
+demais. (Por exemplo: "Em que ordem os bloqueios precisam ser liberados? Os
+diversos estados foram restaurados?") Muito frequentemente, o uso de BUG() vai
+desestabilizar o sistema ou travá-lo por completo, o que torna impossível
+depurar ou até mesmo obter relatórios de travamento (*crash reports*) viáveis.
+Linus tem opiniões `muito fortes
+<https://lore.kernel.org/lkml/CA+55aFy6jNLsywVYdGp83AMrXBo_P-pkjkphPGrO=82SPKCpLQ@mail.gmail.com/>`_
+`sobre isso
+<https://lore.kernel.org/lkml/CAHk-=whDHsbK3HTOpTF=ue_o04onRwTEaK_ZoJp_fjbqq4+=Jw@mail.gmail.com/>`_.
+
+Note que a família WARN() só deve ser usada para situações que "espera-se que
+sejam inacessíveis". Se você quiser alertar sobre situações que são
+"acessíveis, mas indesejáveis", use a família de funções pr_warn(). Os
+administradores do sistema podem ter configurado o sysctl *panic_on_warn* para
+garantir que seus sistemas não continuem executando diante de condições
+"inacessíveis". (Para exemplos, veja commits como `este aqui
+<https://git.kernel.org/linus/d4689846881d160a4d12a514e991a740bcb5d65a>`_.)
+
+Aritmética explícita em argumentos do alocador
+----------------------------------------------
+Cálculos dinâmicos de tamanho (especialmente multiplicação) não devem ser
+realizados em argumentos de funções de alocação de memória (ou similares)
+devido ao risco de estouro de capacidade (*overflow*). Isso poderia fazer com
+que os valores dessem a volta (*wrap around*), resultando em uma alocação menor
+do que o esperado pelo chamador. O uso dessas alocações pode levar a estouros
+lineares na memória heap e a outros comportamentos incorretos. (Uma exceção a
+isso são valores literais onde o compilador pode emitir um aviso se houver
+risco de estouro. No entanto, a maneira preferível nesses casos é refatorar o
+código conforme sugerido abaixo para evitar a aritmética explícita.)
+
+Por exemplo, não use ``count * size`` como argumento, como em::
+
+        foo = kmalloc(count * size, GFP_KERNEL);
+
+Em vez disso, a forma de dois fatores do alocador deve ser utilizada::
+
+        foo = kmalloc_array(count, size, GFP_KERNEL);
+
+Especificamente, kmalloc() pode ser substituído por kmalloc_array(), e
+kzalloc() pode ser substituído por kcalloc().
+
+Se nenhuma forma de dois fatores estiver disponível, os auxiliares de
+saturação em estouro (*saturate-on-overflow*) devem ser usados::
+
+        bar = dma_alloc_coherent(dev, array_size(count, size), &dma, GFP_KERNEL);
+
+Outro caso comum a ser evitado é calcular o tamanho de uma estrutura com uma
+matriz final de outras estruturas, como em::
+
+        header = kzalloc(sizeof(*header) + count * sizeof(*header->item),
+                         GFP_KERNEL);
+
+Em vez disso, use o auxiliar::
+
+        header = kzalloc(struct_size(header, item, count), GFP_KERNEL);
+
+.. note:: Se você estiver usando struct_size() em uma estrutura que contém uma
+        matriz de comprimento zero ou de um único elemento como membro final,
+        refatore o uso dessa matriz e mude para um `membro de matriz flexível
+        <#zero-length-and-one-element-arrays>`_ em seu lugar.
+
+Para outros cálculos, faça a composição usando os auxiliares size_mul(),
+size_add() e size_sub(). Por exemplo, no caso de::
+
+        foo = krealloc(current_size + chunk_size * (count - 3), GFP_KERNEL);
+
+Em vez disso, use os auxiliares::
+
+        foo = krealloc(size_add(current_size,
+                                size_mul(chunk_size,
+                                         size_sub(count, 3))), GFP_KERNEL);
+
+Para mais detalhes, veja também array3_size() e flex_array_size(), bem como as
+funções relacionadas das famílias check_mul_overflow(), check_add_overflow(),
+check_sub_overflow() e check_shl_overflow().\
+
+simple_strtol(), simple_strtoll(), simple_strtoul(), simple_strtoull()
+----------------------------------------------------------------------
+As funções simple_strtol(), simple_strtoll(), simple_strtoul() e
+simple_strtoull() ignoram explicitamente estouros de capacidade (*overflows*),
+o que pode levar a resultados inesperados nos chamadores. As respectivas
+funções kstrtol(), kstrtoll(), kstrtoul() e kstrtoull() tendem a ser as
+substitutas corretas, embora se deva notar que estas exigem que a string seja
+terminada em NUL ou em nova linha (*newline*).
+
+strcpy()
+--------
+A função strcpy() não realiza verificação de limites no buffer de destino.
+Isso pode resultar em estouros lineares além do final do buffer, levando a todo
+tipo de comportamentos incorretos. Embora ``CONFIG_FORTIFY_SOURCE=y`` e várias
+opções do compilador ajudem a reduzir o risco de usar esta função, não há uma
+boa razão para adicionar novos usos dela. A substituta segura é strscpy(),
+embora se deva ter cuidado nos casos em que o valor de retorno de strcpy() era
+utilizado, já que strscpy() não retorna um ponteiro para o destino, mas sim a
+quantidade de bytes não-NUL copiados (ou um código de erro errno negativo quando
+ocorre truncamento).
+
+strncpy()
+---------
+A função strncpy() foi removida do kernel. Todos os chamadores antigos foram
+migrados para alternativas mais seguras.
+
+A função strncpy() não garantia a terminação em NUL do buffer de destino,
+levando a estouros de leitura linear e outros comportamentos incorretos. Ela
+também preenchia incondicionalmente o destino com NUL, o que representava uma
+penalidade de desempenho desnecessária para chamadores que usavam apenas
+strings terminadas em NUL. Devido aos seus diversos comportamentos, ela era uma
+API ambígua para determinar qual era a real intenção do autor ao realizar a
+cópia.
+
+As substitutas para strncpy() são:
+
+- strscpy() quando o destino deve ser terminado em NUL.
+- strscpy_pad() quando o destino deve ser terminado em NUL e preenchido com
+  zeros (por exemplo, estruturas que cruzam fronteiras de privilégio).
+- memtostr() para destinos terminados em NUL a partir de origens de largura
+  fixa não terminadas em NUL (com o atributo ``__nonstring`` na origem).
+- memtostr_pad() para o mesmo caso anterior, mas com preenchimento de zeros.
+- strtomem() para destinos de largura fixa não terminados em NUL, com o
+  atributo ``__nonstring`` no destino.
+- strtomem_pad() para destinos não terminados em NUL que também precisam de
+  preenchimento com zeros.
+- memcpy_and_pad() para cópias limitadas a partir de origens potencialmente não
+  terminadas, onde o tamanho do destino é um valor definido em tempo de
+  execução (*runtime*).
+
+strlcpy()
+---------
+A função strlcpy() lê primeiro todo o buffer de origem (já que o valor de
+retorno deve corresponder ao de strlen()). Essa leitura pode exceder o limite
+de tamanho do destino. Isso é ineficiente e pode levar a estouros de leitura
+linear se a string de origem não for terminada em NUL. A substituta segura é
+strscpy(), embora se deva ter cuidado nos casos em que o valor de retorno de
+strlcpy() é utilizado, já que strscpy() retornará valores negativos de errno
+quando houver truncamento.
+
+Especificador de formato %p
+----------------------------
+Tradicionalmente, o uso de "%p" em strings de formatação causava falhas de
+exposição de endereços reais no dmesg, proc, sysfs, etc. Em vez de deixar esses
+endereços expostos a explorações, todos os usos de "%p" no kernel agora são
+exibidos como um valor hash, tornando-os inúteis para fins de endereçamento.
+Novos usos de "%p" não devem ser adicionados ao kernel. Para endereços de texto,
+usar "%pS" costuma ser melhor, pois exibe o nome do símbolo, que é muito mais
+útil. Para quase todo o restante, simplesmente não adicione "%p" de forma
+alguma.
+
+Parafraseando as diretrizes atuais do Linus `guidance
+<https://lore.kernel.org/lkml/CA+55aFwQEd_d40g4mUCSsVRZzrFPUJt74vc6PPpb675hYNXcKw@mail.gmail.com/>`_:
+
+- Se o valor hash de "%p" é inútil, pergunte a si mesmo se o ponteiro em si é
+  importante. Talvez ele deva ser removido por completo?
+- Se você realmente acredita que o valor real do ponteiro é importante, por que
+  algum estado do sistema ou nível de privilégio do usuário seria considerado
+  "especial"? Se você acha que pode justificar isso (em comentários e no log de
+  commit) de forma sólida o suficiente para resistir ao escrutínio do Linus,
+  talvez possa usar "%px", certificando-se de aplicar permissões adequadas.
+
+Se você estiver depurando algo em que a geração de hash de "%p" esteja causando
+problemas, é possível inicializar o sistema temporariamente com a opção de
+depuração "`no_hash_pointers
+<https://git.kernel.org/linus/5ead723a20e0447bc7db33dc3070b420e5f80aa6>`_".
+
+Matrizes de Tamanho Variável (VLAs)
+-----------------------------------
+O uso de VLAs (Variable Length Arrays) na pilha de execução gera um código de
+máquina muito pior do que matrizes de tamanho estático na pilha. Embora esses
+problemas significativos de `desempenho
+<https://git.kernel.org/linus/02361bc77888>`_ sejam motivo suficiente para
+eliminar as VLAs, elas também representam um risco de segurança. O crescimento
+dinâmico de uma matriz na pilha pode exceder a memória restante no segmento da
+pilha. Isso pode levar a um travamento, à possível sobrescrita de dados
+sensíveis no final da pilha (quando compilado sem ``CONFIG_THREAD_INFO_IN_TASK=y``)
+ou à sobrescrita de posições de memória adjacentes à pilha (quando compilado sem
+``CONFIG_VMAP_STACK=y``).
+
+Passagem direta implícita no switch case (fall-through)
+-------------------------------------------------------
+A linguagem C permite que o fluxo de execução de um bloco switch passe
+diretamente para o próximo caso (fall-through) quando uma instrução "break"
+está ausente ao final de um caso. No entanto, isso introduz ambiguidade no
+código, pois nem sempre fica claro se o "break" ausente é intencional ou um bug.
+Por exemplo, não é óbvio apenas olhando para o código se o ``STATE_ONE`` foi
+intencionalmente projetado para passar diretamente para o ``STATE_TWO``::
+
+        switch (value) {
+        case STATE_ONE:
+                do_something();
+        case STATE_TWO:
+                do_other();
+                break;
+        default:
+                WARN("unknown state");
+        }
+
+Como há uma longa lista de falhas `causadas pela ausência de instruções "break"
+<https://cwe.mitre.org/data/definitions/484.html>`_, não permitimos mais a
+passagem direta implícita. Para identificar os casos de passagem direta
+intencionais, adotamos a macro pseudo-palavra-chave "fallthrough", que se
+expande para a extensão do gcc `__attribute__((__fallthrough__))
+<https://gcc.gnu.org/onlinedocs/gcc/Statement-Attributes.html>`_. (Quando a
+sintaxe ``[[fallthrough]]`` do C17/C18 for suportada de forma mais ampla por
+compiladores C, analisadores estáticos e IDEs, poderemos mudar para o uso dessa
+sintaxe para a pseudo-palavra-chave da macro.)
+
+Todos os blocos de switch/case devem terminar com um dos seguintes elementos:
+
+* break;
+* fallthrough;
+* continue;
+* goto <rótulo>;
+* return [expressão];
+
+Matrizes de comprimento zero e de um único elemento
+----------------------------------------------------
+Há uma necessidade frequente no kernel de fornecer uma maneira de declarar uma
+estrutura com um conjunto de elementos finais de tamanho dinâmico. O código do
+kernel deve sempre usar `"membros de matriz flexível"
+<https://en.wikipedia.org/wiki/Flexible_array_member>`_ para esses casos. O
+estilo antigo de matrizes de um único elemento ou de comprimento zero não deve
+mais ser utilizado.
+
+No código C mais antigo, elementos finais de tamanho dinâmico eram declarados
+especificando uma matriz de um elemento ao final de uma estrutura::
+
+        struct something {
+                size_t count;
+                struct foo items[1];
+        };
+
+Isso levava a cálculos de tamanho frágeis via sizeof() (que exigiam a subtração
+do tamanho do elemento final único para obter o tamanho correto do "cabeçalho").
+Uma `extensão GNU C <https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html>`_ foi
+introduzida para permitir matrizes de comprimento zero, a fim de evitar esses
+problemas de cálculo de tamanho::
+
+        struct something {
+                size_t count;
+                struct foo items[0];
+        };
+
+No entanto, isso trouxe outros problemas e não resolveu algumas limitações de
+ambos os estilos, como a incapacidade de detectar quando tal matriz é usada
+acidentalmente *fora* do final de uma estrutura (o que poderia ocorrer
+diretamente, ou quando tal estrutura estava contida em unions, estruturas de
+estruturas, etc.).
+
+O padrão C99 introduziu os "membros de matriz flexível", nos quais a declaração
+da matriz simplesmente não possui um tamanho numérico::
+
+        struct something {
+                size_t count;
+                struct foo items[];
+        };
+
+Esta é a maneira como o kernel espera que elementos finais de tamanho dinâmico
+sejam declarados. Isso permite que o compilador gere erros quando a matriz
+flexível não for o último elemento da estrutura, o que ajuda a evitar que bugs
+de `comportamento indefinido
+<https://git.kernel.org/linus/76497732932f15e7323dc805e8ea8dc11bb587cf>`_ sejam
+introduzidos inadvertidamente na base de código. Também permite que o
+compilador analise corretamente os tamanhos das matrizes (via sizeof(),
+``CONFIG_FORTIFY_SOURCE`` e ``CONFIG_UBSAN_BOUNDS``). Por exemplo, não existe um
+mecanismo que nos alerte de que a seguinte aplicação do operador sizeof() a uma
+matriz de comprimento zero sempre resulta em zero::
+
+        struct something {
+                size_t count;
+                struct foo items[0];
+        };
+
+        struct something *instance;
+
+        instance = kmalloc(struct_size(instance, items, count), GFP_KERNEL);
+        instance->count = count;
+
+        size = sizeof(instance->items) * instance->count;
+        memcpy(instance->items, source, size);
+
+Na última linha do código acima, ``size`` acaba sendo ``zero``, quando se
+poderia pensar que ele representaria o tamanho total em bytes da memória
+dinâmica recentemente alocada para a matriz final ``items``. Aqui estão alguns
+exemplos deste problema: `link 1
+<https://git.kernel.org/linus/f2cd32a443da694ac4e28fbf4ac6f9d5cc63a539>`_,
+`link 2
+<https://git.kernel.org/linus/ab91c2a89f86be2898cee208d492816ec238b2cf>`_.
+Em vez disso, `membros de matriz flexível têm tipo incompleto e, portanto, o
+operador sizeof() não pode ser aplicado
+<https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html>`_, de modo que qualquer
+uso incorreto de tais operadores será imediatamente percebido em tempo de
+compilação.
+
+Em relação às matrizes de um único elemento, é preciso estar muito ciente de que
+`tais matrizes ocupam pelo menos o mesmo espaço que um único objeto daquele tipo
+<https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html>`_ e, portanto, contribuem
+para o tamanho da estrutura que as contém. Isso é propício a erros sempre que se
+deseja calcular o tamanho total da memória dinâmica a ser alocada para uma
+estrutura que contém uma matriz desse tipo como membro::
+
+        struct something {
+                size_t count;
+                struct foo items[1];
+        };
+
+        struct something *instance;
+
+        instance = kmalloc(struct_size(instance, items, count - 1), GFP_KERNEL);
+        instance->count = count;
+
+        size = sizeof(instance->items) * instance->count;
+        memcpy(instance->items, source, size);
+
+No exemplo acima, foi necessário lembrar de calcular ``count - 1`` ao usar o
+auxiliar struct_size(); caso contrário, teríamos alocado memória --de forma não
+intencional-- para um objeto ``items`` a mais. A maneira mais limpa e menos
+sujeita a erros de implementar isso é através do uso de um `membro de matriz
+flexível`, em conjunto com os auxiliares struct_size() e flex_array_size()::
+
+        struct something {
+                size_t count;
+                struct foo items[];
+        };
+
+        struct something *instance;
+
+        instance = kmalloc(struct_size(instance, items, count), GFP_KERNEL);
+        instance->count = count;
+
+        memcpy(instance->items, source, flex_array_size(instance, items, instance->count));
+
+Existem dois casos especiais de substituição nos quais o auxiliar
+DECLARE_FLEX_ARRAY() precisa ser utilizado. (Note que ele é nomeado
+__DECLARE_FLEX_ARRAY() para uso em cabeçalhos de UAPI.) Esses casos ocorrem
+quando a matriz flexível está sozinha em uma estrutura ou faz parte de uma
+union. Isso não é permitido pela especificação C99, mas sem justificativa
+técnica (como pode ser visto tanto pelo uso existente de tais matrizes nesses
+locais quanto pela solução alternativa que DECLARE_FLEX_ARRAY() adota). Por
+exemplo, para converter isto::
+
+        struct something {
+                ...
+                union {
+                        struct type1 one[0];
+                        struct type2 two[0];
+                };
+        };
+
+O auxiliar deve ser utilizado::
+
+        struct something {
+                ...
+                union {
+                        DECLARE_FLEX_ARRAY(struct type1, one);
+                        DECLARE_FLEX_ARRAY(struct type2, two);
+                };
+        };
+
+Atribuições diretas de kmalloc para objetos struct
+--------------------------------------------------
+Realizar atribuições diretas (*open-coded*) de alocações da família kmalloc()
+impede que o kernel (e o compilador) consigam examinar o tipo da variável que
+está recebendo a atribuição, o que limita qualquer introspecção relacionada que
+possa ajudar com alinhamento, estouros de capacidade (*wrap-around*) ou
+proteções adicionais (*hardening*). A família de macros kmalloc_obj() fornece
+essa introspecção, que pode ser usada para os padrões de código comuns de
+alocações de objetos únicos, de matrizes ou de objetos flexíveis. Por exemplo,
+estas atribuições diretas::
+
+        ptr = kmalloc(sizeof(*ptr), gfp);
+        ptr = kzalloc(sizeof(*ptr), gfp);
+        ptr = kmalloc_array(count, sizeof(*ptr), gfp);
+        ptr = kcalloc(count, sizeof(*ptr), gfp);
+        ptr = kmalloc(struct_size(ptr, flex_member, count), gfp);
+        ptr = kmalloc(sizeof(struct foo), gfp);
+
+tornam-se, respectivamente::
+
+        ptr = kmalloc_obj(*ptr [, gfp] );
+        ptr = kzalloc_obj(*ptr [, gfp] );
+        ptr = kmalloc_objs(*ptr, count [, gfp] );
+        ptr = kzalloc_objs(*ptr, count [, gfp] );
+        ptr = kmalloc_flex(*ptr, flex_member, count [, gfp] );
+        __auto_type ptr = kmalloc_obj(struct foo [, gfp] );
+
+O argumento gfp é opcional, sendo o valor padrão GFP_KERNEL. Se
+``ptr->flex_member`` estiver anotado com __counted_by(), a alocação falhará
+automaticamente caso ``count`` seja maior do que o valor máximo representável
+que pode ser armazenado no membro contador associado a ``flex_member``.
diff --git a/Documentation/translations/pt_BR/process/development-process.rst b/Documentation/translations/pt_BR/process/development-process.rst
index 599c34c859be6..d303ab92b2d35 100644
--- a/Documentation/translations/pt_BR/process/development-process.rst
+++ b/Documentation/translations/pt_BR/process/development-process.rst
@@ -20,3 +20,8 @@ conhecimento profundo de programação de kernel para ser compreendida.
    1.Intro
    2.Process
    3.Early-stage
+   4.Coding
+   5.Posting
+   6.Followthrough
+   7.AdvancedTopics
+   8.Conclusion
diff --git a/Documentation/translations/pt_BR/process/kernel-driver-statement.rst b/Documentation/translations/pt_BR/process/kernel-driver-statement.rst
new file mode 100644
index 0000000000000..5a8a8f41d566d
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/kernel-driver-statement.rst
@@ -0,0 +1,205 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Declaração sobre Drivers do Kernel
+----------------------------------
+
+Posicionamento sobre os Módulos do Kernel Linux
+===============================================
+
+
+Nós, os desenvolvedores do kernel Linux abaixo assinados, consideramos
+qualquer módulo ou driver de código fechado para o kernel Linux
+prejudicial e indesejável. Repetidamente, constatamos que eles são
+nocivos aos usuários do Linux, às empresas e ao ecossistema Linux como
+um todo. Tais módulos negam a abertura, a estabilidade, a flexibilidade
+e a manutenibilidade do modelo de desenvolvimento do Linux e privam
+seus usuários do conhecimento da comunidade Linux. Fornecedores que
+oferecem módulos de kernel de código fechado forçam seus clientes a
+abrir mão de vantagens fundamentais do Linux ou a escolher novos
+fornecedores. Portanto, para aproveitar plenamente a economia de custos
+e os benefícios de suporte compartilhado que o código aberto tem a
+oferecer, incentivamos fortemente para que os fornecedores adotem uma
+política de dar suporte a seus clientes no Linux com código de kernel
+de código aberto.
+
+Falamos apenas por nós mesmos, e não por qualquer empresa para a qual
+possamos trabalhar hoje, tenhamos trabalhado no passado ou venhamos a
+trabalhar no futuro.
+
+ - Dave Airlie
+ - Nick Andrew
+ - Jens Axboe
+ - Ralf Baechle
+ - Felipe Balbi
+ - Ohad Ben-Cohen
+ - Muli Ben-Yehuda
+ - Jiri Benc
+ - Arnd Bergmann
+ - Thomas Bogendoerfer
+ - Vitaly Bordug
+ - James Bottomley
+ - Josh Boyer
+ - Neil Brown
+ - Mark Brown
+ - David Brownell
+ - Michael Buesch
+ - Franck Bui-Huu
+ - Adrian Bunk
+ - François Cami
+ - Ralph Campbell
+ - Luiz Fernando N. Capitulino
+ - Mauro Carvalho Chehab
+ - Denis Cheng
+ - Jonathan Corbet
+ - Glauber Costa
+ - Alan Cox
+ - Magnus Damm
+ - Ahmed S. Darwish
+ - Robert P. J. Day
+ - Hans de Goede
+ - Arnaldo Carvalho de Melo
+ - Helge Deller
+ - Jean Delvare
+ - Mathieu Desnoyers
+ - Sven-Thorsten Dietrich
+ - Alexey Dobriyan
+ - Daniel Drake
+ - Alex Dubov
+ - Randy Dunlap
+ - Michael Ellerman
+ - Pekka Enberg
+ - Jan Engelhardt
+ - Mark Fasheh
+ - J. Bruce Fields
+ - Larry Finger
+ - Jeremy Fitzhardinge
+ - Mike Frysinger
+ - Kumar Gala
+ - Robin Getz
+ - Liam Girdwood
+ - Jan-Benedict Glaw
+ - Thomas Gleixner
+ - Brice Goglin
+ - Cyrill Gorcunov
+ - Andy Gospodarek
+ - Thomas Graf
+ - Krzysztof Halasa
+ - Harvey Harrison
+ - Stephen Hemminger
+ - Michael Hennerich
+ - Tejun Heo
+ - Benjamin Herrenschmidt
+ - Kristian Høgsberg
+ - Henrique de Moraes Holschuh
+ - Marcel Holtmann
+ - Mike Isely
+ - Takashi Iwai
+ - Olof Johansson
+ - Dave Jones
+ - Jesper Juhl
+ - Matthias Kaehlcke
+ - Kenji Kaneshige
+ - Jan Kara
+ - Jeremy Kerr
+ - Russell King
+ - Olaf Kirch
+ - Roel Kluin
+ - Hans-Jürgen Koch
+ - Auke Kok
+ - Peter Korsgaard
+ - Jiri Kosina
+ - Aaro Koskinen
+ - Mariusz Kozlowski
+ - Greg Kroah-Hartman
+ - Michael Krufky
+ - Aneesh Kumar
+ - Clemens Ladisch
+ - Christoph Lameter
+ - Gunnar Larisch
+ - Anders Larsen
+ - Grant Likely
+ - John W. Linville
+ - Yinghai Lu
+ - Tony Luck
+ - Pavel Machek
+ - Matt Mackall
+ - Paul Mackerras
+ - Roland McGrath
+ - Patrick McHardy
+ - Kyle McMartin
+ - Paul Menage
+ - Thierry Merle
+ - Eric Miao
+ - Akinobu Mita
+ - Ingo Molnar
+ - James Morris
+ - Andrew Morton
+ - Paul Mundt
+ - Oleg Nesterov
+ - Luca Olivetti
+ - S.Çağlar Onur
+ - Pierre Ossman
+ - Keith Owens
+ - Venkatesh Pallipadi
+ - Nick Piggin
+ - Nicolas Pitre
+ - Evgeniy Polyakov
+ - Richard Purdie
+ - Mike Rapoport
+ - Sam Ravnborg
+ - Gerrit Renker
+ - Stefan Richter
+ - David Rientjes
+ - Luis R. Rodriguez
+ - Stefan Roese
+ - Francois Romieu
+ - Rami Rosen
+ - Stephen Rothwell
+ - Maciej W. Rozycki
+ - Mark Salyzyn
+ - Yoshinori Sato
+ - Deepak Saxena
+ - Holger Schurig
+ - Amit Shah
+ - Yoshihiro Shimoda
+ - Sergei Shtylyov
+ - Kay Sievers
+ - Sebastian Siewior
+ - Rik Snel
+ - Jes Sorensen
+ - Alexey Starikovskiy
+ - Alan Stern
+ - Timur Tabi
+ - Hirokazu Takata
+ - Eliezer Tamir
+ - Eugene Teo
+ - Doug Thompson
+ - FUJITA Tomonori
+ - Dmitry Torokhov
+ - Marcelo Tosatti
+ - Steven Toth
+ - Theodore Tso
+ - Matthias Urlichs
+ - Geert Uytterhoeven
+ - Arjan van de Ven
+ - Ivo van Doorn
+ - Rik van Riel
+ - Wim Van Sebroeck
+ - Hans Verkuil
+ - Horst H. von Brand
+ - Dmitri Vorobiev
+ - Anton Vorontsov
+ - Daniel Walker
+ - Johannes Weiner
+ - Harald Welte
+ - Matthew Wilcox
+ - Dan J. Williams
+ - Darrick J. Wong
+ - David Woodhouse
+ - Chris Wright
+ - Bryan Wu
+ - Rafael J. Wysocki
+ - Herbert Xu
+ - Vlad Yasevich
+ - Peter Zijlstra
+ - Bartlomiej Zolnierkiewicz
diff --git a/Documentation/translations/pt_BR/process/maintainer-netdev.rst b/Documentation/translations/pt_BR/process/maintainer-netdev.rst
index 5de2828041b98..e9bb998c7bfd0 100644
--- a/Documentation/translations/pt_BR/process/maintainer-netdev.rst
+++ b/Documentation/translations/pt_BR/process/maintainer-netdev.rst
@@ -22,11 +22,11 @@ netdev
 ------
 A **netdev** é a lista de discussão para todos os assuntos do Linux relacionados
 a rede. Isso inclui qualquer item encontrado em ``net/`` (ex: código principal
-como IPv6) e  em ``drivers/net`` (ex: drivers específicos de hardware) na árvore
+como IPv6) e em ``drivers/net`` (ex: drivers específicos de hardware) na árvore
 de diretórios do Linux.
 
 Note que alguns subsistemas (ex: drivers de rede sem fio/wireless), que possuem
-um  alto volume de tráfego, possuem suas próprias listas de discussão e árvores
+um alto volume de tráfego, possuem suas próprias listas de discussão e árvores
 específicas.
 
 Como muitas outras listas de discussão do Linux, a lista netdev é hospedada no
@@ -34,7 +34,7 @@ Como muitas outras listas de discussão do Linux, a lista netdev é hospedada no
 https://lore.kernel.org/netdev/.
 
 À exceção dos subsistemas mencionados anteriormente, todo o desenvolvimento de
-rede  do Linux (ex: RFCs, revisões, comentários, etc.) ocorre na **netdev**.
+rede do Linux (ex: RFCs, revisões, comentários, etc.) ocorre na **netdev**.
 
 Ciclo de Desenvolvimento
 ------------------------
@@ -506,8 +506,14 @@ netdevsim
 
 O ``netdevsim`` é um driver de teste que pode ser usado para exercitar APIs de
 configuração de driver sem a necessidade de hardware compatível. Mock-ups e
-testes baseados no ``netdevsim`` são fortemente encorajados ao adicionar novas
-APIs, mas o ``netdevsim`` em si **não** é considerado um caso de uso/usuário.
+testes baseados no ``netdevsim`` são encorajados ao adicionar novas APIs com
+lógica complexa na pilha. Os testes devem ser escritos de forma que possam ser
+executados tanto contra o ``netdevsim`` quanto contra um dispositivo real
+(veja ``tools/testing/selftests/drivers/net/README.rst``). Testes exclusivos
+para o ``netdevsim`` devem se concentrar em testar casos extremos e caminhos de
+falha no núcleo que são difíceis de exercitar com um driver real.
+
+``netdevsim`` em si **não** é considerado um caso de uso/usuário.
 Você também deve implementar as novas APIs em um driver real.
 
 Não damos garantias de que o ``netdevsim`` mudará no futuro de uma forma que
@@ -577,8 +583,11 @@ independentemente do nível de experiência. Para orientações gerais e dicas
 
 É seguro assumir que os mantenedores da netdev conhecem a comunidade e o nível
 de experiência dos revisores. Os revisores não devem se preocupar com o fato de
-seus comentários impedirem ou desviarem o fluxo de patches. Revisores menos
-experientes são fortemente incentivados a fazer uma revisão mais aprofundada das
+seus comentários impedirem ou desviarem o fluxo de patches. Uma tag Reviewed-by
+é entendida como "Eu revisei este código da melhor maneira possível" em vez de
+"Posso atestar que este código está correto".
+
+Revisores são fortemente incentivados a fazer uma revisão mais aprofundada das
 submissões e não focar exclusivamente em questões triviais ou subjetivas, como
 formatação de código, tags, etc.
 
diff --git a/Documentation/translations/pt_BR/process/submit-checklist.rst b/Documentation/translations/pt_BR/process/submit-checklist.rst
new file mode 100644
index 0000000000000..003fc956832df
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/submit-checklist.rst
@@ -0,0 +1,145 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+==============================================================
+Lista de verificação para submissão de patches do kernel Linux
+==============================================================
+
+Aqui estão algumas coisas básicas que os desenvolvedores devem fazer se
+quiserem ver suas submissões de patches de kernel aceitas mais rapidamente.
+
+Estas diretrizes vão além da documentação fornecida em
+:ref:`Documentation/process/submitting-patches.rst <submittingpatches>`
+e em outros locais sobre o envio de patches para o kernel Linux.
+
+Revise seu código
+=================
+
+1) Se você usar um recurso, faça o #include do arquivo que define/declara
+   esse recurso. Não dependa de outros arquivos de cabeçalho que incluam
+   os que você usa de forma indireta.
+
+2) Verifique o estilo geral do seu patch conforme detalhado em
+   :ref:`Documentation//process/coding-style.rst <codingstyle>`.
+
+3) Todas as barreiras de memória {por exemplo, ``barrier()``, ``rmb()``,
+   ``wmb()``} precisam de um comentário no código-fonte que explique a
+   lógica do que estão fazendo e o porquê.
+
+Revise as alterações do Kconfig
+===============================
+
+1) Quaisquer novas ou modificadas opções de ``CONFIG`` não bagunçam o
+   menu de configuração e têm 'desativado' (off) como padrão, a menos que
+   atendam aos critérios de exceção documentados em
+   ``Documentation/kbuild/kconfig-language.rst``, atributos de menu: valor
+   padrão.
+
+2) Todas as novas opções de ``Kconfig`` possuem texto de ajuda.
+
+3) Foram cuidadosamente revisadas com relação às combinações relevantes de
+   ``Kconfig``. Isso é muito difícil de acertar apenas com testes --- exige
+   capacidade de raciocínio.
+   pays off here.
+
+Forneça documentação
+====================
+
+1) Inclua :ref:`kernel-doc <kernel_doc>` para documentar as APIs globais
+   do kernel. (Não é obrigatório para funções estáticas, mas também é
+   aceitável nelas.)
+
+2) Todas as novas entradas em ``/proc`` devem ser documentadas sob
+   ``Documentation/``.
+
+3) Todos os novos parâmetros de inicialização (boot) do kernel devem ser
+   documentados em ``Documentation/admin-guide/kernel-parameters.rst``.
+
+4) Todos os novos parâmetros de módulo devem ser documentados com
+   ``MODULE_PARM_DESC()``.
+
+5) Todas as novas interfaces com o espaço de usuário (userspace) devem ser
+   documentadas em ``Documentation/ABI/``. Consulte
+   ``Documentation/admin-guide/abi.rst`` (ou ``Documentation/ABI/README``)
+   para obter mais informações. Patches que alteram interfaces de espaço
+   de usuário devem incluir em cópia (CC) linux-api@vger.kernel.org.
+
+6) Se quaisquer ioctls forem adicionados pelo patch, atualize também
+   ``Documentation/userspace-api/ioctl/ioctl-number.rst``.
+
+Verifique seu código com ferramentas
+====================================
+
+1) Verifique se há violações triviais com o verificador de estilo de patch
+   antes do envio (``scripts/checkpatch.pl``). Você deve ser capaz de
+   justificar todas as violações que permanecerem no seu patch.
+
+2) Faça uma verificação limpa com o sparse.
+
+3) Use ``make checkstack`` e corrija quaisquer problemas encontrados por ele.
+   Observe que o ``checkstack`` não aponta problemas explicitamente, mas
+   qualquer função individual que utilize mais de 512 bytes na pilha é uma
+   candidata a alteração.
+
+Compile seu código
+==================
+
+1) Compila de forma limpa:
+
+  a) com as opções de ``CONFIG`` aplicáveis ou modificadas definidas como
+     ``=y``, ``=m`` e ``=n``. Sem avisos/erros do ``gcc``, sem avisos/erros do
+     vinculador (linker).
+
+  b) Passa em ``allnoconfig``, ``allmodconfig``
+
+  c) Compila com sucesso ao usar ``O=builddir``
+
+  d) Quaisquer alterações em Documentation/ compilam com sucesso sem novos
+     avisos/erros. Use ``make htmldocs`` ou ``make pdfdocs`` para verificar
+     a compilação e corrigir quaisquer problemas.
+
+2) Compila em múltiplas arquiteturas de CPU usando ferramentas locais de
+   compilação cruzada (cross-compile) ou alguma outra fazenda de compilação
+   (build farm).
+   Observe que testar em arquiteturas de diferentes tamanhos de palavra
+   (32 e 64 bits) e diferentes endianness (big- e little-endian) é eficaz
+   para capturar vários problemas de portabilidade decorrentes de falsas
+   suposições sobre o intervalo de quantidade representável, alinhamento
+   de dados ou endianness, entre outros.
+
+3) O novo código adicionado foi compilado com ``gcc -W`` (use
+   ``make KCFLAGS=-W``). Isso gerará muito ruído, mas é bom para encontrar
+   bugs como "warning: comparison between signed and unsigned".
+
+4) Se o seu código-fonte modificado depender ou usar quaisquer APIs ou
+   recursos do kernel relacionados aos seguintes símbolos do ``Kconfig``,
+   teste múltiplas compilações com os símbolos relacionados do ``Kconfig``
+   desativados e/ou definidos como ``=m`` (se essa opção estiver disponível)
+   [não todos ao mesmo tempo, apenas combinações variadas/aleatórias deles]:
+
+   ``CONFIG_SMP``, ``CONFIG_SYSFS``, ``CONFIG_PROC_FS``, ``CONFIG_INPUT``,
+   ``CONFIG_PCI``, ``CONFIG_BLOCK``, ``CONFIG_PM``, ``CONFIG_MAGIC_SYSRQ``,
+   ``CONFIG_NET``, ``CONFIG_INET=n`` (mas este último com ``CONFIG_NET=y``).
+
+Teste seu código
+================
+
+1) Foi testado com ``CONFIG_PREEMPT``, ``CONFIG_DEBUG_PREEMPT``,
+   ``CONFIG_SLUB_DEBUG``, ``CONFIG_DEBUG_PAGEALLOC``,
+   ``CONFIG_DEBUG_MUTEXES``, ``CONFIG_DEBUG_SPINLOCK``,
+   ``CONFIG_DEBUG_ATOMIC_SLEEP``, ``CONFIG_PROVE_RCU`` e
+   ``CONFIG_DEBUG_OBJECTS_RCU_HEAD`` todos habilitados simultaneamente.
+
+2) Foi testado em tempo de compilação e de execução com e sem ``CONFIG_SMP``
+   e ``CONFIG_PREEMPT``.
+
+3) Todos os caminhos de código foram executados com todos os recursos de
+   lockdep ativados.
+
+4) Foi verificado com a injeção de falhas de pelo menos slab e alocação de
+   páginas. Consulte ``Documentation/fault-injection/``.
+   Se o novo código for substancial, a adição de injeção de falhas específica
+   do subsistema pode ser apropriada.
+
+5) Testado com a tag mais recente do linux-next para garantir que ele ainda
+   funcione com todos os outros patches enfileirados e com várias alterações
+   na VM, VFS e outros subsistemas.
diff --git a/Documentation/translations/zh_CN/scheduler/sched-energy.rst b/Documentation/translations/zh_CN/scheduler/sched-energy.rst
index fdbf6cfeea933..03dedc69839a5 100644
--- a/Documentation/translations/zh_CN/scheduler/sched-energy.rst
+++ b/Documentation/translations/zh_CN/scheduler/sched-energy.rst
@@ -119,7 +119,7 @@ EAS覆盖了CFS的任务唤醒平衡代码。在唤醒平衡时，它使用平
 
 如果唤醒的任务被迁移，find_energy_efficient_cpu()使用compute_energy()来估算
 系统将消耗多少能量。compute_energy()检查各CPU当前的利用率情况，并尝试调整来
-“模拟”任务迁移。EM框架提供了API em_pd_energy()计算每个性能域在给定的利用率条件
+“模拟”任务迁移。EM框架提供了API em_cpu_energy()计算每个性能域在给定的利用率条件
 下的预期能量消耗。
 
 下面详细介绍一个优化能量消耗的任务放置决策的例子。
diff --git a/Documentation/translations/zh_CN/security/credentials.rst b/Documentation/translations/zh_CN/security/credentials.rst
index 88fcd9152ffe9..20c8696f8198e 100644
--- a/Documentation/translations/zh_CN/security/credentials.rst
+++ b/Documentation/translations/zh_CN/security/credentials.rst
@@ -337,15 +337,13 @@ const指针上操作，因此不需要进行类型转换，但需要临时放弃
    ``__task_cred()`` 的结果不应直接传递给 ``get_cred()`` ，
    因为这可能与 ``commit_cred()`` 发生竞争条件。
 
-还有一些方便的函数可以访问另一个任务凭据的特定部分，将RCU操作对调用方隐藏起来::
+有一个方便的函数可用于访问另一个任务凭据的特定部分，从而对调用方隐藏RCU机制::
 
 	uid_t task_uid(task)		Task's real UID
-	uid_t task_euid(task)		Task's effective UID
 
 如果调用方在此时已经持有RCU读锁，则应使用::
 
 	__task_cred(task)->uid
-	__task_cred(task)->euid
 
 类似地，如果需要访问任务凭据的多个方面，应使用RCU读锁，调用 ``__task_cred()``
 函数，将结果存储在临时指针中，然后从临时指针中调用凭据的各个方面，最后释放锁。
diff --git a/Documentation/translations/zh_CN/usb/CREDITS b/Documentation/translations/zh_CN/usb/CREDITS
index c133b1a5daff4..ccfeced03ea61 100644
--- a/Documentation/translations/zh_CN/usb/CREDITS
+++ b/Documentation/translations/zh_CN/usb/CREDITS
@@ -10,12 +10,10 @@
 :校译:
 
 
-简易 Linux USB 驱动的致谢名单：
+Simple Linux USB 驱动项目致谢名单：
 
-以下人员都为 Linux USB 驱动代码作出了贡献
-（按姓氏字母顺序排列）。我相信这份名单本应
-更长一些，但确实不容易维护。
-如需将自己加入名单，请提交补丁。
+以下人员都为 Linux USB 驱动代码作出过贡献（按姓氏字母顺序排列）。这份名
+单本该更长，只是确实不易维护；如果你也应列名其中，欢迎提交补丁把自己加上。
 
   Georg Acher <acher@informatik.tu-muenchen.de>
   David Brownell <dbrownell@users.sourceforge.net>
@@ -41,123 +39,124 @@
 特别感谢：
 
   Inaky Perez Gonzalez <inaky@peloncho.fis.ucm.es>
-  感谢他发起了 Linux USB 驱动开发工作，并编写了体量较大的 uusbd
-  驱动中的大部分代码。我们从那项工作中学到了很多。
+  感谢他牵头开发 Linux USB 驱动，并编写了 uusbd 驱动的大部分代码，我们
+  从中学到了很多。
 
   NetBSD 和 FreeBSD 的 USB 开发者们
   感谢他们加入 Linux USB 邮件列表，提供建议并分享实现经验。
 
-附加感谢：
-  还要感谢以下公司与个人在硬件、支持、时间投入和开发方面提供的捐赠与帮助
-  （摘自 Inaky 驱动原始的 THANKS 文件）：
+另外还要感谢：
 
-    以下公司曾帮助我们开发 Linux USB / UUSBD：
+  以下公司和个人在硬件、支持、时间和开发工作上给予了帮助（摘自 Inaky
+  驱动原始的 THANKS 文件）：
 
-        - 3Com GmbH 捐赠了一台 ISDN Pro TA，并在技术问题和测试设备方面为我
-          提供支持。没想到能得到这么大的帮助。
+    以下公司曾为 Linux USB / UUSBD 的开发提供帮助：
 
-        - USAR Systems 向我们提供了他们出色的 USB 评估套件，
-          使我们能够测试 Linux USB 驱动对最新 USB 规范的符合性。
-          USAR Systems 认识到保持开放操作系统与时俱进的重要性，
-          并以硬件支持这个项目。感谢！
+        - 3Com GmbH 捐赠了一台 ISDN Pro TA，并在技术问题和测试设备方面提供
+          了大力支持。
+
+        - USAR Systems 向我们提供了出色的 USB 评估套件，使我们得以测试
+          Linux USB 驱动对最新 USB 规范的符合性。USAR Systems 也认识到，
+          让开放操作系统跟上时代很重要，因此以硬件支持了这个项目，在此
+          致谢。
 
         - 感谢英特尔提供的宝贵帮助。
 
         - 我们与 Cherry 合作，使 Linux 成为首个内置 USB 支持的操作系统。
           Cherry 是全球最大的键盘制造商之一。
 
-        - CMD Technology, Inc. 慷慨捐赠了一块 CSA-6700 PCI-to-USB
-          控制卡，用于测试 OHCI 实现。
+        - CMD Technology, Inc. 慷慨捐赠了一块 CSA-6700 PCI 转 USB 控制卡，
+          用于测试 OHCI 实现。
 
-        - 由于他们对我们的支持，Keytronic 可以放心，
-          他们的键盘能卖给至少 300 万 Linux 用户中的一部分。
+        - 有了他们的支持，Keytronic 可以确信，其键盘能够销售给至少 300 万
+          Linux 用户中的一部分。
 
-        - ing büro h doran [http://www.ibhdoran.com]!
-          在欧洲，想给主板买一个 PC 背板 USB 连接器几乎是不可能的
-          （我自己做的那个相当糟糕 :)）。现在我知道该去哪里买漂亮的 USB
-          配件了！
+        - 特别感谢 ing büro h doran [http://www.ibhdoran.com]。
+          在欧洲，想给主板配一个 PC 背板 USB 连接器几乎是不可能的（我自己
+          做的那个效果并不好）。现在我知道该去哪里购买合适的 USB 配件了。
 
         - Genius Germany 捐赠了一只 USB 鼠标，用于测试鼠标启动协议；
-          他们还捐赠了 F-23 数字摇杆和 NetMouse Pro。感谢！
+          他们还捐赠了 F-23 数字摇杆和 NetMouse Pro，在此致谢。
 
-        - AVM GmbH Berlin 支持我们开发 Linux 下的 AVM ISDN Controller B1 USB 驱动。
-          AVM 是领先的 ISDN 控制器制造商，其主动式设计对包括 Linux 在内的
-          所有操作系统平台开放。
+        - AVM GmbH Berlin 支持我们开发 Linux 下的 AVM ISDN Controller B1 USB
+          驱动。AVM 是主动式和被动式 ISDN 控制器及基于 CAPI 2.0 软件的领先
+          制造商。AVM B1 的主动式设计对包括 Linux 在内的所有操作系统平台
+          开放。
 
-        - 非常感谢 Y-E Data, Inc 捐赠的 FlashBuster-U USB 软驱，
-          使我们能够测试批量传输代码。
+        - 非常感谢 Y-E Data, Inc 捐赠的 FlashBuster-U USB 软驱，使我们能够测试
+          批量传输代码。
 
         - 感谢 Logitech 捐赠了一只三轴 USB 鼠标。
 
-          Logitech 负责设计、制造并销售各种人机接口设备，
-          在键盘、鼠标、轨迹球、摄像头、扬声器，以及面向游戏和专业用途的
-          控制设备方面拥有悠久历史和丰富经验。
+          Logitech 负责设计、制造并销售各种人机接口设备，在键盘、鼠标、轨迹球、
+          摄像头、扬声器，以及面向游戏和专业用途的控制设备方面拥有悠久历史和
+          丰富经验。
 
-          作为这些设备广为人知的供应商和销售商，他们捐赠了 USB 鼠标、
-          摇杆和扫描仪，以表明 Linux 的重要性，也让 Logitech 的客户
-          能在自己喜欢的操作系统上获得支持，并让所有 Linux 用户都能使用
-          Logitech 以及其他 USB 硬件。
+          作为这些设备广为人知的供应商和销售商，他们捐赠了 USB 鼠标、摇杆和
+          扫描仪，以表明 Linux 的重要性，也让 Logitech 的客户能在自己偏爱的
+          操作系统上获得支持，并让所有 Linux 用户都能使用 Logitech 及其他
+          USB 硬件。
 
           Logitech 也是 1999 年 2 月 11 日维也纳 Linux 大会的官方赞助商，
           我们将在会上展示 Linux USB 工作的最新进展。
 
-        - 感谢 CATC 提供 USB Inspector，帮助我们揭开 UHCI 内部实现中
-          那些不为人知的角落。
+        - 感谢 CATC 提供 USB Inspector，帮助我们看到 UHCI 内部实现中的那些
+          隐秘角落。
 
         - 感谢 Entrega 为开发工作提供 PCI 转 USB 卡、集线器和转换器产品。
 
-        - 感谢 ConnectTech 提供 WhiteHEAT USB 转串口转换器以及相关文档，
-          让这个驱动得以写成。
+        - 感谢 ConnectTech 提供 WhiteHEAT USB 转串口转换器以及相关文档，让
+          这个驱动得以写成。
 
-        - 感谢 ADMtek 提供 Pegasus 和 Pegasus II 评估板、规格说明，
-          以及驱动开发过程中的宝贵建议。
+        - 感谢 ADMtek 提供 Pegasus 和 Pegasus II 评估板、规格说明，以及驱动
+          开发过程中的宝贵建议。
 
-    另外还要感谢以下个人（嘿，顺序不分先后 :)）
+    另外还要感谢以下个人（排名不分先后）：
 
-        - Oren Tirosh <orenti@hishome.net>,
-          他非常耐心地听我唠叨各种 USB 疑问，还给了很多很酷的想法。
+        - Oren Tirosh <orenti@hishome.net>
+          他非常耐心地解答我反复提出的各种 USB 问题，并提供了许多有价值的
+          想法。
 
-        - Jochen Karrer <karrer@wpfd25.physik.uni-wuerzburg.de>,
-          指出了致命 bug，并给出了宝贵建议。
+        - Jochen Karrer <karrer@wpfd25.physik.uni-wuerzburg.de>
+          指出了严重问题，并给出了宝贵建议。
 
-        - Edmund Humemberger <ed@atnet.at>，他在公共关系与项目管理方面
-          为 Linux-USB 项目付出了巨大的努力。
+        - Edmund Humemberger <ed@atnet.at>，他在公共关系与项目管理方面为
+          Linux-USB 项目付出了巨大的努力。
 
-        - Alberto Menegazzi <flash@flash.iol.it> 正在着手编写 UUSBD 文档，加油！
+        - Alberto Menegazzi <flash@flash.iol.it> 正在着手编写 UUSBD 文档。
 
-        - Ric Klaren <ia_ric@cs.utwente.nl> 编写了很好的入门文档，
-          与 Alberto 的作品形成良性竞争：）。
+        - Ric Klaren <ia_ric@cs.utwente.nl> 编写了很好的入门文档，与
+          Alberto 的作品形成了良性互补。
 
-        - Christian Groessler <cpg@aladdin.de>，感谢他在那些棘手细节上的帮助。
+        - Christian Groessler <cpg@aladdin.de>，感谢他在诸多复杂细节上的帮助。
 
-        - Paul MacKerras 改进了 OHCI 实现，推动了对 iMac 的支持，
-          并提供了大量的改进意见。
+        - Paul MacKerras 改进了 OHCI 实现，推动了对 iMac 的支持，并提供了
+          大量的改进意见。
 
-        - Fernando Herrera <fherrera@eurielec.etsit.upm.es>
-          负责撰写、维护并不断补充那份期待已久、独一无二又精彩的
-          UUSBD FAQ！太棒了！
+        - Fernando Herrera <fherrera@eurielec.etsit.upm.es> 负责撰写、维护并
+          持续补充那份期待已久、内容翔实的 UUSBD FAQ。
 
-        - Rasca Gmelch <thron@gmx.de> 重新启用了 raw 驱动，
-          指出了一些错误，并启动了 uusbd-utils 软件包。
+        - Rasca Gmelch <thron@gmx.de> 重新启用了 raw 驱动，指出了一些错误，并
+          启动了 uusbd-utils 软件包。
 
-        - Peter Dettori <dettori@ozy.dec.com>，像疯了一样挖掘 bug，
-          还提出了很多很酷的建议，太棒了！
+        - Peter Dettori <dettori@ozy.dec.com>，持续发现问题，并提出了许多
+          有价值的建议。
 
-        - 自由软件与 Linux 社区的所有成员，包括 FSF、GNU 项目、
-          MIT X 联盟、TeX 社区等等，谢谢你们！
+        - 自由软件与 Linux 社区的所有成员，包括 FSF、GNU 项目、MIT X 联盟、
+          TeX 社区等，在此一并致谢。
 
-        - 特别感谢 Richard Stallman 创造了 Emacs！
+        - 特别感谢 Richard Stallman 创造了 Emacs。
 
-        - 感谢 linux-usb 邮件列表的所有成员，读了那么多邮件——不开玩笑了，
-          感谢你们提出的所有建议！
+        - 感谢 linux-usb 邮件列表的所有成员阅读了大量邮件，并提出了诸多
+          建议。
 
         - 感谢 USB Implementers Forum 成员们的帮助与支持。
 
-        - Nathan Myers <ncm@cantrip.org>，感谢他的建议！
-          （希望你喜欢 Cibeles 的派对。）
+        - Nathan Myers <ncm@cantrip.org>，感谢他的建议。（也希望你喜欢
+          Cibeles 的派对。）
 
-        - 感谢 Linus Torvalds 创建、开发并管理 Linux。
+        - 感谢 Linus Torvalds 创立、开发并维护 Linux。
 
         - Mike Smith、Craig Keithley、Thierry Giron 和 Janet Schank
-          感谢他们让我认识到标准 USB 集线器其实也没那么“标准”，
-          这有助于我们在标准集线器驱动中加入厂商特定的特殊处理。
+          感谢他们让我认识到，标准 USB 集线器其实一点也不“标准”；也正因
+          如此，我们才能在标准集线器驱动中加入厂商特定的处理。
diff --git a/Documentation/translations/zh_CN/usb/acm.rst b/Documentation/translations/zh_CN/usb/acm.rst
index 51d6eb8f5660e..b2e35787af452 100644
--- a/Documentation/translations/zh_CN/usb/acm.rst
+++ b/Documentation/translations/zh_CN/usb/acm.rst
@@ -20,33 +20,26 @@ Linux ACM 驱动 v0.16
 
 0. 免责声明
 ~~~~~~~~~~~
-本程序是自由软件；你可以在自由软件基金会发布的
-GNU 通用公共许可证第 2 版，或者（按你的选择）
-任何后续版本的条款下重新发布和/或修改它。
+本程序是自由软件；你可以在自由软件基金会发布的 GNU 通用公共许可证第 2 版，
+或者（按你的选择）任何后续版本的条款下重新发布和/或修改它。
 
-发布本程序是希望它能发挥作用，但它不附带任何担保；
-甚至不包括对适销性或特定用途适用性的默示担保。
-详情见 GNU 通用公共许可证。
+发布本程序是希望它能发挥作用，但它不附带任何担保；甚至不包括对适销性或
+特定用途适用性的默示担保。详情见 GNU 通用公共许可证。
 
-你应该已经随本程序收到了 GNU 通用公共许可证的副本；
-如果没有，请致信：Free Software Foundation, Inc., 59
-Temple Place, Suite 330, Boston, MA 02111-1307 USA。
+你应该已经随本程序收到了 GNU 通用公共许可证的副本；如果没有，请参见
+COPYING 文件。
 
-如需联系作者，可发送电子邮件至 vojtech@suse.cz，
-或邮寄至：
-Vojtech Pavlik, Ucitelska 1576, Prague 8,
-182 00, Czech Republic。
+如需联系作者，可发送电子邮件至 vojtech@suse.cz，或邮寄至：
+Vojtech Pavlik, Ucitelska 1576, Prague 8, 182 00, Czech Republic。
 
-为方便起见，软件包中已附带 GNU 通用公共许可证
-第 2 版：见 COPYING 文件。
+为方便起见，软件包中已附带 GNU 通用公共许可证第 2 版：见 COPYING 文件。
 
 1. 使用方法
 ~~~~~~~~~~~
-``drivers/usb/class/cdc-acm.c`` 驱动可用于符合 USB
-通信设备类抽象控制模型（USB CDC ACM）规范的
-USB 调制解调器和 USB ISDN 终端适配器。
+``drivers/usb/class/cdc-acm.c`` 驱动适用于符合 USB 通信设备类抽象控制模型
+（USB CDC ACM）规范的 USB 调制解调器和 USB ISDN 终端适配器。
 
-许多调制解调器支持此驱动，以下是我所知道的一些型号：
+已知支持该驱动的调制解调器包括：
 
 	- 3Com OfficeConnect 56k
 	- 3Com Voice FaxModem Pro
@@ -56,17 +49,16 @@ USB 调制解调器和 USB ISDN 终端适配器。
 	- Compaq 56k FaxModem
 	- ELSA Microlink 56k
 
-我知道有一款 ISDN 终端适配器可以与 ACM 驱动一起使用：
+已知有一款 ISDN 终端适配器可以配合 ACM 驱动使用：
 
 	- 3Com USR ISDN Pro TA
 
-一些手机也可以通过 USB 连接。
-我知道以下机型可以正常工作：
+一些手机也可以通过 USB 连接，已知可用的机型有：
 
 	- SonyEricsson K800i
 
-遗憾的是，许多调制解调器和大多数 ISDN TA
-都使用专有接口，因此无法与此驱动配合工作。
+遗憾的是，很多调制解调器和大多数 ISDN TA 都使用专有接口，因此无法配合该
+驱动工作。
 购买前请先确认设备是否符合 ACM 规范。
 
 要使用这些调制解调器，需要加载以下模块::
@@ -75,15 +67,13 @@ USB 调制解调器和 USB ISDN 终端适配器。
 	uhci-hcd.ko ohci-hcd.ko or ehci-hcd.ko
 	cdc-acm.ko
 
-之后就应该可以访问这些调制解调器了。
-应当可以使用 ``minicom``、``ppp`` 和 ``mgetty``
-与它们通信。
+之后就应该能访问这些调制解调器，并用 ``minicom``、``ppp`` 和
+``mgetty`` 与它们通信。
 
 2. 验证驱动是否正常工作
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-第一步是检查 ``/sys/kernel/debug/usb/devices``，
-其内容应该类似如下::
+第一步是查看 ``/sys/kernel/debug/usb/devices``，其内容应当类似下面这样::
 
   T:  Bus=01 Lev=00 Prnt=00 Port=00 Cnt=00 Dev#=  1 Spd=12  MxCh= 2
   B:  Alloc=  0/900 us ( 0%), #Int=  0, #Iso=  0
@@ -112,11 +102,10 @@ USB 调制解调器和 USB ISDN 终端适配器。
   E:  Ad=85(I) Atr=02(Bulk) MxPS=  64 Ivl=  0ms
   E:  Ad=04(O) Atr=02(Bulk) MxPS=  64 Ivl=  0ms
 
-这三行的存在很关键（以及 ``Cls=`` 字段里出现的
-``comm`` 和 ``data`` 类）；它说明这是一个 ACM
-设备。``Driver=acm`` 表示该设备正在使用 acm 驱动。
-如果只看到 ``Cls=ff(vend.)``，那就无能为力了：
-这说明你手上的设备使用的是厂商专有接口::
+关键是看这三行，再结合 ``Cls=`` 字段里出现的 ``comm`` 和 ``data`` 类，就
+能判断这是一台 ACM 设备。``Driver=acm`` 表示该设备正在使用 acm 驱动。如果
+只看到 ``Cls=ff(vend.)``，那就说明这台设备使用的是厂商专有接口，ACM 驱动
+无法处理::
 
     D:  Ver= 1.00 Cls=02(comm.) Sub=00 Prot=00 MxPS= 8 #Cfgs=  2
     I:  If#= 0 Alt= 0 #EPs= 1 Cls=02(comm.) Sub=02 Prot=01 Driver=acm
@@ -142,6 +131,5 @@ USB 调制解调器和 USB ISDN 终端适配器。
   usb.c: acm driver claimed interface c7b5f3f8
   usb.c: acm driver claimed interface c7691fa0
 
-如果以上都正常，请启动 ``minicom``，
-把它配置为连接 ``ttyACM`` 设备，然后
-尝试输入 ``at``。如果返回 ``OK``，说明一切工作正常。
+如果这些都正常，请启动 ``minicom``，把它配置为连接到 ``ttyACM`` 设备，然后
+尝试输入 ``at``。如果返回 ``OK``，说明驱动工作正常。
diff --git a/Documentation/translations/zh_CN/usb/authorization.rst b/Documentation/translations/zh_CN/usb/authorization.rst
index 2aa311f6b9675..e2ff2282bd038 100644
--- a/Documentation/translations/zh_CN/usb/authorization.rst
+++ b/Documentation/translations/zh_CN/usb/authorization.rst
@@ -10,34 +10,32 @@
 :校译:
 
 
-=============================
-授权或禁止 USB 设备连接到系统
-=============================
+===========================
+允许或禁止 USB 设备接入系统
+===========================
 
 版权 (C) 2007 Inaky Perez-Gonzalez
 <inaky@linux.intel.com> 英特尔公司
 
-此功能允许你控制 USB 设备是否可以在系统中使用。
-借助它，你可以完全通过用户空间实现对 USB 设备的锁定。
+有了这项功能，你就可以控制 USB 设备是否允许在系统中使用，并把 USB 设备锁
+定机制完全放在用户空间实现。
 
-目前，当插入一个 USB 设备时，系统会对其进行配置，
-其接口会立即向用户开放。
-有了这项改动，只有在 root 授权设备完成配置后，
-设备才可被使用。
+目前，USB 设备一接入系统就会被立即配置，其接口也会立刻向用户开放。引入
+这项机制后，只有在 root 明确授权后，设备才会完成配置并允许使用。
 
 
 用法
 ====
 
-授权设备接入::
+授权设备接入系统::
 
 	$ echo 1 > /sys/bus/usb/devices/DEVICE/authorized
 
-取消对设备的授权::
+取消设备授权::
 
 	$ echo 0 > /sys/bus/usb/devices/DEVICE/authorized
 
-将新连接到 ``hostX`` 的设备默认设为未授权（即锁定）::
+将连接到 ``hostX`` 的新设备默认设为未授权（即锁定）::
 
 	$ echo 0 > /sys/bus/usb/devices/usbX/authorized_default
 
@@ -45,15 +43,14 @@
 
 	$ echo 1 > /sys/bus/usb/devices/usbX/authorized_default
 
-默认情况下，所有 USB 设备都是授权的。
-向 ``authorized_default`` 属性写入 ``2`` 会使内核
-默认只授权连接到内部 USB 端口的设备。
+默认情况下，所有 USB 设备都是授权的。向 ``authorized_default`` 属性写入
+``2`` 会使内核默认只授权连接到内部 USB 端口的设备。
 
-系统锁定示例（比较粗糙）
-------------------------
+系统锁定示例（简化版）
+----------------------
 
-假设你想实现一个锁定功能，只允许类型为 XYZ 的设备接入
-（例如某台带有外露 USB 端口的自助服务终端）::
+假设你想做一个锁定机制，只允许 XYZ 类型的设备接入（例如一台带有外露 USB
+端口的自助终端）::
 
   启动系统
   rc.local ->
@@ -63,21 +60,18 @@
       echo 0 > $host/authorized_default
    done
 
-给 udev 挂一个脚本，用于处理新插入的 USB 设备::
+为 udev 配置一个脚本，用于处理新插入的 USB 设备::
 
  if device_is_my_type $DEV
  then
    echo 1 > $device_path/authorized
- done
+ fi
 
 
-``device_is_my_type()`` 才是锁定方案真正见功夫的
-地方。仅仅检查 class、type 和 protocol 是否匹配
-某个值，是你能做出的最糟糕的安全验证之一；
-对想绕过它的人来说，这反而是最容易利用的方案。
-如果你需要真正安全的办法，那就该使用加密、
-证书认证之类的机制。把 USB 存储设备当作
-“钥匙”的一个简单例子可以是::
+锁定方案是否可靠，关键全在 ``device_is_my_type()`` 的实现。仅仅检查
+class、type 和 protocol 是否匹配，几乎是最差的一种安全校验方式；对想绕过
+它的人来说，这种做法反而最容易伪造。如果你真要做安全控制，就该使用加密、
+证书认证之类的机制。把 USB 存储设备当作“钥匙”的一个简单示例可以写成::
 
  function device_is_my_type()
  {
@@ -87,7 +81,7 @@
    sum=$(md5sum /mntpoint/.signature)
    if [ $sum = $(cat /etc/lockdown/keysum) ]
    then
-        echo "We are good, connected"
+        echo "验证通过，已连接"
         umount /mntpoint
         # 再做一些额外处理，让其他人也能使用它
    else
@@ -96,17 +90,16 @@
  }
 
 
-当然，这个例子很粗糙；真正要做的话，
-你会想用基于 PKI 的证书校验，这样就不必依赖
-共享密钥之类的东西。不过你应该已经明白意思了。
-任何拿到设备仿真工具包的人都能伪造描述符和设备信息。
-别信这个。
+当然，这个例子仍然比较简化。真正落地时，更合适的做法是使用基于 PKI 的证
+书校验，这样就不必依赖共享密钥之类的机制了。不过意思已经很清楚：任何拿到
+设备仿真工具包的人，都能伪造描述符和设备信息，所以别把这类检查当成真正
+的安全保障。
 
 接口授权
 --------
 
-也有类似的方法用于允许或拒绝特定 USB 接口。
-这使得你可以只阻止某个 USB 设备中的部分接口。
+也可以用类似的方法允许或拒绝特定的 USB 接口。这样一来，你只需要阻止某个
+USB 设备中的部分接口。
 
 授权接口::
 
@@ -126,14 +119,12 @@
 
 	$ echo 0 > /sys/bus/usb/devices/usbX/interface_authorized_default
 
-默认情况下，
-``interface_authorized_default`` 位为 ``1``，
-因此所有接口默认都处于已授权状态。
+默认情况下，``interface_authorized_default`` 位为 ``1``，因此所有接口默认
+都会处于授权状态。
 
 注意：
-  如果把一个先前未授权的接口改为已授权，
-  则必须通过将 ``INTERFACE`` 写入 ``/sys/bus/usb/drivers_probe``
-  来手动触发驱动探测。
+  如果把一个先前未授权的接口改为已授权，则必须通过将 ``INTERFACE`` 写入
+  ``/sys/bus/usb/drivers_probe`` 来手动触发驱动探测。
 
-对于需要多个接口的驱动程序，应先授权所有必需接口，
-然后再触发驱动探测。这样做可以避免副作用。
+对于需要多个接口的驱动程序，应先授权所有必需接口，然后再触发驱动探测。
+这样做可以避免副作用。
diff --git a/Documentation/translations/zh_CN/usb/chipidea.rst b/Documentation/translations/zh_CN/usb/chipidea.rst
index ea0dc3043189f..ee5407a4ce448 100644
--- a/Documentation/translations/zh_CN/usb/chipidea.rst
+++ b/Documentation/translations/zh_CN/usb/chipidea.rst
@@ -17,18 +17,17 @@ ChipIdea 高速双角色控制器驱动
 1. 如何测试 OTG FSM（HNP 和 SRP）
 ---------------------------------
 
-下面以两块 Freescale i.MX6Q Sabre SD 开发板为例，
-说明如何通过 sysfs 输入文件演示 OTG 的 HNP 和 SRP 功能。
+下面以两块 Freescale i.MX6Q Sabre SD 开发板为例，演示如何通过 sysfs 属性
+测试 OTG 的 HNP 和 SRP 功能。
 
-1.1 如何使能 OTG FSM
+1.1 如何启用 OTG FSM
 --------------------
 
 1.1.1 在 ``menuconfig`` 中选择 ``CONFIG_USB_OTG_FSM``，并重新编译内核
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-重新构建内核镜像和模块。如果想查看 OTG FSM 的
-一些内部变量，可以挂载 ``debugfs``；其中有两个文件
-可以显示 OTG FSM 变量以及部分控制器寄存器的值::
+重新构建内核镜像和模块。如果想查看 OTG FSM 的内部变量，可以挂载
+``debugfs``；其中有两个文件，分别显示 OTG FSM 的变量和部分控制器寄存器值::
 
 	cat /sys/kernel/debug/ci_hdrc.0/otg
 	cat /sys/kernel/debug/ci_hdrc.0/registers
@@ -44,11 +43,10 @@ ChipIdea 高速双角色控制器驱动
 1.2 测试步骤
 ------------
 
-1) 给两块 Freescale i.MX6Q Sabre SD 开发板上电，
-   并加载 gadget 类驱动（例如 ``g_mass_storage``）。
+1) 给两块 Freescale i.MX6Q Sabre SD 开发板上电，并加载 gadget 类驱动（例如
+   ``g_mass_storage``）。
 
-2) 用 USB 线连接两块开发板：
-   一端是 micro A 插头，另一端是 micro B 插头。
+2) 用 USB 线连接两块开发板：一端是 micro A 插头，另一端是 micro B 插头。
 
    插入 micro A 插头的一端是 A 设备，它应枚举另一端的 B 设备。
 
@@ -66,32 +64,28 @@ ChipIdea 高速双角色控制器驱动
 
 	echo 0 > /sys/bus/platform/devices/ci_hdrc.0/inputs/b_bus_req
 
-   或者，通过引入 HNP 轮询，B 端主机可以知道
-   A 端外设希望切换为主机角色，因此这次角色切换
-   也可以通过 A 端外设响应 B 端主机的轮询，
-   在 A 侧触发。
-   这可以通过在 A 设备上执行下面的命令来完成::
+   或者，也可以借助 HNP 轮询，让 B 端主机知道 A 端外设希望切回主机角色。
+   因此，这次切换也可以由 A 侧触发，也就是由 A 端外设响应 B 端主机的轮询
+   来完成。可在 A 设备上执行下面的命令::
 
 	echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/a_bus_req
 
    A 设备应切回主机角色并枚举 B 设备。
 
-5) 拔掉 B 设备（拔掉 micro B 插头），
-   并在 10 秒内重新插入；
+5) 拔掉 B 设备（拔掉 micro B 插头），并在 10 秒内重新插入。
    A 设备应重新枚举 B 设备。
 
-6) 拔掉 B 设备（拔掉 micro B 插头），
-   并在 10 秒后重新插入；
+6) 拔掉 B 设备（拔掉 micro B 插头），并在 10 秒后重新插入。
    A 设备不应重新枚举 B 设备。
 
-   如果 A 设备希望使用总线：
+   如果 A 设备还想继续使用总线：
 
    在 A 设备上执行::
 
 	echo 0 > /sys/bus/platform/devices/ci_hdrc.0/inputs/a_bus_drop
 	echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/a_bus_req
 
-   如果 B 设备希望使用总线：
+   如果 B 设备想使用总线：
 
    在 B 设备上执行::
 
@@ -111,40 +105,41 @@ ChipIdea 高速双角色控制器驱动
 
 	echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/b_bus_req
 
-   A 设备应恢复 USB 总线并枚举 B 设备。
+   A 设备应恢复 USB 总线，并枚举 B 设备。
 
 1.3 参考文档
 ------------
-《On-The-Go and Embedded Host Supplement
-to the USB Revision 2.0 Specification
+《On-The-Go and Embedded Host Supplement to the USB Revision 2.0 Specification
 July 27, 2012 Revision 2.0 version 1.1a》
 
 2. 如何将 USB 用作系统唤醒源
 ----------------------------
-下面是在 i.MX6 平台上把 USB 用作系统唤醒源的示例。
+下面给出在 i.MX6 平台上将 USB 用作系统唤醒源的示例。
 
-2.1 使能核心控制器的唤醒功能::
+2.1 启用核心控制器的唤醒功能::
 
 	echo enabled > /sys/bus/platform/devices/ci_hdrc.0/power/wakeup
 
-2.2 使能 glue 层的唤醒功能::
+2.2 启用 glue 层的唤醒功能::
 
 	echo enabled > /sys/bus/platform/devices/2184000.usb/power/wakeup
 
-2.3 使能 PHY 的唤醒功能（可选）::
+2.3 启用 PHY 的唤醒功能（可选）::
 
 	echo enabled > /sys/bus/platform/devices/20c9000.usbphy/power/wakeup
 
-2.4 使能根集线器的唤醒功能::
+2.4 启用根集线器的唤醒功能::
 
 	echo enabled > /sys/bus/usb/devices/usb1/power/wakeup
 
-2.5 使能相关设备的唤醒功能::
+2.5 启用相关设备的唤醒功能::
 
 	echo enabled > /sys/bus/usb/devices/1-1/power/wakeup
 
-如果系统只有一个 USB 端口，
-而你希望在该端口上启用 USB 唤醒功能，
-可以使用下面的脚本::
+如果系统只有一个 USB 端口，而你希望在该端口上启用 USB 唤醒功能，可以使用
+下面的脚本::
 
-	for i in $(find /sys -name wakeup | grep usb);do echo enabled > $i;done;
+	for i in $(find /sys -name wakeup | grep usb)
+	do
+		echo enabled > $i
+	done
diff --git a/Documentation/translations/zh_CN/usb/dwc3.rst b/Documentation/translations/zh_CN/usb/dwc3.rst
index 3468ce50c5baa..9584cbbf6d030 100644
--- a/Documentation/translations/zh_CN/usb/dwc3.rst
+++ b/Documentation/translations/zh_CN/usb/dwc3.rst
@@ -18,46 +18,43 @@ DWC3 驱动
 待办
 ~~~~
 
-阅读时如果想顺手认领点任务，可以从下面挑一项 :)
+如果你愿意接手其中一项任务，可以从下面选择：
 
 - 将中断处理程序改为每个端点各自使用线程化 IRQ
 
-  事实证明，有些 DWC3 命令大约需要 ``~1 ms`` 才能完成。
-  当前代码会一直自旋等待命令完成，这种设计并不好。
+  实践表明，某些 DWC3 命令大约需要 ``~1 ms`` 才能完成。当前代码会一直自旋
+  等待命令完成，这并不是好办法。
 
   实现思路：
 
-  - DWC 核心实现了一个按端点对中断进行解复用的 IRQ 控制器。
-    中断号在探测（``probe``）阶段分配，并归属于该设备。
-    如果硬件通过 ``MSI`` 为每个端点提供独立中断，
-    那么这个“虚拟”IRQ 控制器就可以被真实的端点中断取代。
+  - DWC 核心实现了一个按端点分发中断的 IRQ 控制器。中断号在探测
+    （``probe``）阶段分配，并归属于该设备。如果硬件通过 ``MSI`` 为每个
+    端点提供独立中断，那么这个“虚拟”IRQ 控制器就可以被真实的端点中断
+    取代。
 
-  - 在调用 ``usb_ep_enable()`` 时请求并分配中断资源，
-    在调用 ``usb_ep_disable()`` 时释放中断资源。
-    最坏情况下需要 32 个中断，最少是 ``ep0/1`` 的两个中断。
+  - 在调用 ``usb_ep_enable()`` 时请求并分配中断资源，在调用
+    ``usb_ep_disable()`` 时释放中断资源。最坏情况下需要 32 个中断，最少是
+    ``ep0/1`` 的两个中断。
   - ``dwc3_send_gadget_ep_cmd()`` 将在 ``wait_for_completion_timeout()``
     中休眠，直到命令完成。
   - 中断处理程序分为以下几个部分：
 
     - 设备级主中断处理程序
-      遍历每个事件，并对其调用 ``generic_handle_irq()``。
-      从 ``generic_handle_irq()`` 返回后，确认事件计数器，使中断最终消失。
+      遍历每个事件，并调用 ``generic_handle_irq()`` 处理。返回后再确认
+      事件计数器，让中断最终消失。
 
     - 设备级线程化处理程序
       无。
 
     - 端点中断的主处理程序
-      读取事件并尽量处理它。凡是需要睡眠的操作都交给线程处理。
-      事件保存在每个端点的数据结构中。
-      还要注意，一旦把某项工作交给线程处理，
-      就不要再在主处理程序里处理它，
-      以免出现优先级反转之类的问题。
+      读取事件并尽量处理；凡是需要睡眠的操作都交给线程处理。事件保存在
+      每个端点的数据结构中。一旦某项工作已经交给线程处理，主处理程序里就
+      不要再碰它，以免出现优先级反转之类的问题。
 
     - 端点中断的线程化处理程序
       处理剩余的端点工作，这些工作可能会睡眠，例如等待命令完成。
 
-  延迟:
+  延迟：
 
-   不应增加延迟，因为中断线程具有较高优先级，
-   会在普通用户态任务之前运行
+   不应增加额外延迟，因为中断线程优先级较高，会在普通用户任务之前运行
    （除非用户更改了调度优先级）。
diff --git a/Documentation/translations/zh_CN/usb/ehci.rst b/Documentation/translations/zh_CN/usb/ehci.rst
index e05e493a30d38..c4c52303b13e5 100644
--- a/Documentation/translations/zh_CN/usb/ehci.rst
+++ b/Documentation/translations/zh_CN/usb/ehci.rst
@@ -14,45 +14,37 @@
 EHCI 驱动
 =========
 
-2002年12月27日
+2002 年 12 月 27 日
 
-EHCI 驱动用于通过支持 USB 2.0 的主机控制器
-硬件与高速 USB 2.0 设备通信。USB 2.0 兼容
-USB 1.1 标准，它定义了三种传输速率：
+EHCI 驱动用于借助支持 USB 2.0 的主机控制器，与高速 USB 2.0 设备通信。USB
+2.0 向下兼容 USB 1.1，并定义了三种传输速率：
 
     - “高速”（High Speed）480 Mbit/sec（60 MByte/sec）
     - “全速”（Full Speed）12 Mbit/sec（1.5 MByte/sec）
     - “低速”（Low Speed）1.5 Mbit/sec
 
-USB 1.1 仅支持全速与低速。
-高速设备可以在 USB 1.1 系统上使用，
-但速度会降到 USB 1.1 的速率。
+USB 1.1 仅支持全速与低速。高速设备可以在 USB 1.1 系统上使用，但速度会
+降到 USB 1.1 的速率。
 
-USB 1.1 设备也可以在 USB 2.0 系统上使用。当它们
-插入 EHCI 控制器时，会被交由 USB 1.1 的伴随
-（companion）控制器处理，该控制器通常是 OHCI 或 UHCI。
+USB 1.1 设备也可以在 USB 2.0 系统上使用。当它们插入 EHCI 控制器时，会交给
+USB 1.1 的伴随（companion）控制器处理，该控制器通常是 OHCI 或 UHCI。
 
-当 USB 1.1 设备插入 USB 2.0 集线器时，它们通过
-集线器中的事务转换器（Transaction Translator，TT）
-与 EHCI 控制器交互，该转换器将低速或全速事务转换为
-高速分割事务，从而避免浪费传输带宽。
+当 USB 1.1 设备插入 USB 2.0 集线器时，它们会通过集线器里的事务转换器
+（Transaction Translator，TT）与 EHCI 控制器通信。该转换器会把低速或全
+速事务转换为高速分割事务，从而避免浪费传输带宽。
 
-截至本文撰写时，该驱动已在以下 EHCI 实现上成功运行
-（按字母顺序）：Intel、NEC、Philips 和 VIA。
-其他供应商的 EHCI 实现正在陆续问世；
-预计该驱动在这些实现上也可正常运行。
+截至本文撰写时，该驱动已在以下 EHCI 实现上成功运行（按字母顺序）：
+Intel、NEC、Philips 和 VIA。随着其他供应商的 EHCI 实现陆续问世，预计该
+驱动在那些实现上也能正常运行。
 
-自 2001 年年中起，usb-storage 设备就已可用
-（在 2.4 版该驱动上速度相当不错），
-集线器则直到 2001 年底才开始可用，而其他类型的高速设备
-似乎要等到更多系统内置 USB 2.0 后才会出现。
-这类新系统从 2002 年初开始上市，
-并在 2002 年下半年变得更加常见。
+自 2001 年年中起，usb-storage 设备就已可用（在 2.4 版该驱动上速度表现相当
+不错），集线器则直到 2001 年底才开始可用。其他类型的高速设备似乎要等到
+更多系统内置 USB 2.0 后才会出现。这类新系统从 2002 年初开始上市，并在
+2002 年下半年变得更加常见。
 
-注意，USB 2.0 支持并不只是 EHCI 本身。
-它还需要对 Linux-USB 核心 API 作出其他修改，
-包括 hub 驱动；不过这些修改并不需要真正改变
-暴露给 USB 设备驱动的基本 ``usbcore`` API。
+注意，USB 2.0 的支持并不只靠 EHCI 本身。它还需要对 Linux-USB 核心 API
+做其他修改，包括 hub 驱动；不过这些修改并不需要真正改变向 USB 设备驱动
+暴露的基本 ``usbcore`` API。
 
 - David Brownell
   <dbrownell@users.sourceforge.net>
@@ -61,58 +53,46 @@ USB 1.1 设备也可以在 USB 2.0 系统上使用。当它们
 功能
 ====
 
-该驱动会定期在 x86 硬件上进行测试，
-也已在 PPC 硬件上使用，因此大小端问题应当已经解决。
-因此可以认为，它已经处理好了所有必要的 PCI 细节，
-所以即便在 DMA 映射有些特殊的系统上，
-I/O 也应能正常运行。
+该驱动长期在 x86 硬件上接受测试，也在 PPC 平台上使用过，因此大小端问题应
+该都已解决。再加上各种必要的 PCI 细节都已处理妥当，即便在 DMA 映射较特
+殊的系统上，I/O 也应能正常工作。
 
 传输类型
 --------
 
-截至本文撰写时，该驱动应当已经能够很好地处理
-所有控制传输、批量传输和中断传输，
-包括通过 USB 2.0 集线器中的事务转换器
-与 USB 1.1 设备通信；但仍可能存在 bug。
+截至本文撰写时，该驱动应当已经能够稳定处理所有控制传输、批量传输和中断传
+输，包括经由 USB 2.0 集线器里的事务转换器访问 USB 1.1 设备；不过仍可能
+存在 bug。
 
-高速等时（ISO）传输支持也已可用，但截至本文撰写时，
-还没有 Linux 驱动使用这项支持。
+高速等时（ISO）传输支持也已可用，不过截至本文撰写时，还没有 Linux 驱动真
+正使用它。
 
-目前尚不支持通过事务转换器实现全速等时传输。
-需要注意，ISO 传输的 split transaction 支持
-与高速 ISO 传输几乎无法共用代码，
-因为 EHCI 用不同的数据结构表示它们。
-因此，目前大多数 USB 音频和视频设备
-还不能通过高速总线连接使用。
+目前尚不支持通过事务转换器实现全速等时传输。需要注意，ISO 传输的分割
+事务支持与高速 ISO 传输几乎无法共用代码，因为 EHCI 用不同的数据结构表示
+它们。因此，目前大多数 USB 音频和视频设备还无法在高速总线上使用。
 
 驱动行为
 --------
 
-所有类型的传输都可以排队。
-这意味着来自一个接口驱动的控制传输
-（或通过 usbfs 发出的控制传输）不会干扰
-另一个驱动的控制传输，而且中断传输可以使用 1 帧的周期，
-而不必担心中断处理开销导致的数据丢失。
+所有类型的传输都可以排队提交。这意味着某个接口驱动发出的控制传输（或经由
+usbfs 提交的控制传输）不会干扰其他驱动的控制传输，而中断传输可以按 1 帧
+周期运行，不必担心中断处理开销导致数据丢失。
 
 
-EHCI 根集线器代码会将 USB 1.1 设备移交给其伴随控制器。
-该驱动不需要了解那些驱动的任何细节；
-一个原本就能正常工作的 OHCI 或 UHCI 驱动，
-并不会因为 EHCI 驱动也存在而需要更改。
+EHCI 根集线器代码会将 USB 1.1 设备交给其伴随控制器处理。该驱动无需了解
+那些驱动的任何细节；一个原本就能正常工作的 OHCI 或 UHCI 驱动，也不会因为
+EHCI 驱动存在而需要修改。
 
-电源管理方面还有一些问题；
-当前挂起/恢复的行为还不完全正确。
+电源管理方面仍有一些问题；当前挂起/恢复行为还不完全正确。
 
-此外，在调度周期性事务
-（中断和等时传输）时还采取了一些简化处理。
-这些简化会限制可调度的周期性事务数量，
-并且无法使用小于一帧的轮询间隔。
+此外，在调度周期性事务（中断和等时传输）时还采取了一些简化处理。这些
+简化会限制可调度的周期性事务数量，并且无法使用小于一帧的轮询间隔。
 
 使用方式
 ========
 
-假设有一个 EHCI 控制器（位于 PCI 卡或主板上），
-并且已将此驱动编译为模块，可这样加载::
+假设系统中有一个 EHCI 控制器（位于 PCI 卡或主板上），并且此驱动是以模块形
+式编译的，那么可以这样加载::
 
     # modprobe ehci-hcd
 
@@ -120,27 +100,24 @@ EHCI 根集线器代码会将 USB 1.1 设备移交给其伴随控制器。
 
     # rmmod ehci-hcd
 
-还应加载一个伴随控制器驱动，
-例如 ``ohci-hcd`` 或 ``uhci-hcd``。
-如果 EHCI 驱动出现任何问题，只需卸载它的模块，
-随后该伴随控制器驱动就会接手
-此前由 EHCI 驱动处理的所有设备
-（但速度会降低）。
+还应加载一个伴随控制器驱动，例如 ``ohci-hcd`` 或 ``uhci-hcd``。如果 EHCI
+驱动出了问题，只要卸载它的模块，伴随控制器驱动就会接管此前由 EHCI 驱动处
+理的全部设备（只是速度会降低）。
 
 模块参数（传给 ``modprobe``）包括：
 
     log2_irq_thresh（默认值 0）：
-        默认中断延迟的 log2 值，单位是微帧。默认值 0 表示 1 个微帧
-        （125 微秒）。最大值 6 表示 2^6 = 64 个微帧。
+        默认中断延迟的 log2 值，单位为微帧。默认值 0 表示 1 个微帧
+        （125 微秒），最大值 6 表示 2^6 = 64 个微帧。
         该值控制 EHCI 控制器发出中断的频率。
 
-如果在 2.5 内核上使用此驱动，并且启用了 USB 调试支持，
-则会在任一 EHCI 控制器的 ``sysfs`` 目录中看到三个文件：
+如果你在 2.5 内核上使用此驱动，并且启用了 USB 调试支持，那么任一 EHCI 控
+制器对应的 ``sysfs`` 目录下都会看到三个文件：
 
     ``async``
         转储异步调度，用于控制传输和批量传输。它会显示每个活动的 ``qh``
         以及待处理的 ``qtd``，通常每个 ``urb`` 对应一个 ``qtd``。
-        （可以在 ``usb-storage`` 做磁盘 I/O 时看它；顺便观察请求队列！）
+        （可以在 ``usb-storage`` 执行磁盘 I/O 时查看；也可顺便观察请求队列。）
 
     ``periodic``
         转储周期性调度，用于中断传输和等时传输。不显示 ``qtd``。
@@ -151,111 +128,81 @@ EHCI 根集线器代码会将 USB 1.1 设备移交给其伴随控制器。
 这些文件的内容有助于定位驱动问题。
 
 
-设备驱动通常不需要关心自己是否运行在 EHCI 之上，
-但它们可能想检查
-``usb_device->speed == USB_SPEED_HIGH``。
-高速设备能做到全速（或低速）设备做不到的事，
-例如高带宽的周期性传输（中断或 ISO 传输）。
-另外，设备描述符中的某些值
-（例如周期性传输的轮询间隔）
-在高速模式下使用不同的编码方式。
+设备驱动通常不需要关心自己是否运行在 EHCI 之上，但有时可能会想检查
+``usb_device->speed == USB_SPEED_HIGH``。高速设备能做到全速（或低速）设备
+做不到的事，例如高带宽的周期性传输（中断或 ISO 传输）。另外，设备描述符
+中的某些值（例如周期性传输的轮询间隔）在高速模式下使用不同的编码方式。
 
-不过，一定要让设备驱动经过 USB 2.0 集线器的测试。
-当使用事务转换器时，这些集线器报告某些故障
-（例如断开连接）的方式会不同；
-已经见过一些驱动在遇到与 OHCI 或 UHCI
-所报告的不同故障时表现不佳。
+不过，设备驱动一定要在 USB 2.0 集线器后面测一遍。使用事务转换器时，这些
+集线器报告某些故障（例如断开连接）的方式会有所不同；已经见过一些驱动在
+遇到与 OHCI 或 UHCI 不同的故障时表现不佳。
 
 性能
 ====
 
-USB 2.0 吞吐量主要受两个因素制约：
-主机控制器处理请求的速度，以及设备响应这些请求的速度。
-480 Mbit/sec 的“原始传输率”对所有设备都成立，
-但总吞吐量还会受到诸如单个高速包之间的延迟、
-驱动是否足够聪明，以及系统整体负载等因素的影响。
-延迟也是性能考量因素。
+USB 2.0 的吞吐量主要受两个因素制约：主机控制器处理请求的速度，以及设备响
+应这些请求的速度。480 Mbit/sec 的“原始传输率”对所有设备都一样，但整体吞
+吐量还会受到诸如高速包之间的间隔、驱动实现是否足够高效以及系统总体负载等
+因素影响。延迟同样是需要考虑的性能指标。
 
-批量传输最常用于关注吞吐量的场景。
-需要记住的是，批量传输总是以 512 字节包为单位，
-而一个 USB 2.0 微帧中最多只能容纳 13 个这样的包。
-8 个 USB 2.0 微帧构成一个 USB 1.1 帧；
-一个微帧的时长是 1 毫秒 / 8 = 125 微秒。
+批量传输通常用于看重吞吐量的场景。需要记住的是，批量传输总是以 512 字节包
+为单位，而一个 USB 2.0 微帧中最多只能容纳 13 个这样的包。8 个 USB 2.0 微
+帧构成一个 USB 1.1 帧，因此一个微帧的时长就是 125 微秒。
 
-因此，只要硬件和设备驱动软件都允许，
-批量传输可提供超过 50 MByte/sec 的带宽。
-周期性传输模式（等时和中断）允许使用更大的包大小，
-从而可以逼近所宣称的 480 Mbit/sec 传输率。
+因此，只要硬件和驱动实现都足够成熟，批量传输就可以提供 50 MByte/sec 以上
+的带宽。周期性传输模式（等时和中断）允许使用更大的包大小，从而可以逼近所
+宣称的 480 Mbit/sec 传输率。
 
 硬件性能
 --------
 
-截至本文撰写时，单个 USB 2.0 设备的最大传输速率
-通常约为 20 MByte/sec。
-这当然会随着时间改变：一些设备现在更快，一些更慢。
+截至本文撰写时，单个 USB 2.0 设备的最大传输速率通常约为 20 MByte/sec。
+这种情况当然会随时间变化：有些设备现在更快，有些则更慢。
 
-第一代 NEC EHCI 实现似乎存在
-大约 28 MByte/sec 的硬件瓶颈。
-虽然这对单个 20 MByte/sec 的设备显然已经够用，
-但把三个这样的设备挂到同一总线上，
-并不能得到 60 MByte/sec。
-问题似乎在于控制器硬件无法并发进行 USB 与 PCI 访问，
-因此它每个微帧只会尝试 6 次（也许是 7 次）
-USB 事务，而不是 13 次。
-（对一个比其他产品早上市一年的芯片来说，
-这是个合理的妥协！）
+第一代 NEC EHCI 实现似乎存在大约 28 MByte/sec 的硬件瓶颈。虽然这对单个
+20 MByte/sec 的设备显然已经够用，但把三个这样的设备挂到同一总线上，并不
+能得到 60 MByte/sec。问题似乎在于控制器硬件无法并发进行 USB 与 PCI 访问，
+因此它每个微帧只会尝试 6 次（也许是 7 次）USB 事务，而不是 13 次。
+（对一款比其他产品早上市一年的芯片来说，这样的取舍也算合理。）
 
 
-预计较新的实现会在这方面做得更好，
-通过投入更多芯片面积来解决这个问题，
-使新的主板芯片组更接近 60 MByte/sec 的目标。
-这既包括 NEC 的更新实现，也包括其他厂商的芯片。
+预计较新的实现会在这方面做得更好，通过投入更多芯片面积来解决这个问题，
+使新的主板芯片组更接近 60 MByte/sec 的目标。这既包括 NEC 的更新实现，也
+包括其他厂商的芯片。
 
 
-主机从 EHCI 控制器收到“请求已完成”中断的最小延迟
-为一个微帧（125 微秒）。该延迟可以调节；
-驱动提供了一个模块选项。默认情况下，
-``ehci-hcd`` 使用最小延迟，这意味着当发出一个控制
-或批量请求时，通常可以在不到 250 微秒内得知它已完成
-（具体取决于传输大小）。
+主机从 EHCI 控制器收到“请求已完成”中断的最小延迟为一个微帧
+（125 微秒）。该延迟可以调节；驱动提供了一个模块选项。
+默认情况下，``ehci-hcd`` 使用最小延迟，这意味着发出控制或批量请求后，通
+常不到 250 微秒就能得知它已经完成（具体取决于传输大小）。
 
 软件性能
 --------
 
-即便只是要达到 20 MByte/sec 的传输速率，
-Linux-USB 设备驱动也必须让 EHCI 队列始终保持满载。
-这意味着要发出较大的请求，
-或者在需要发出一连串小请求时使用批量请求排队。
-如果驱动未做到这一点，那么会直接从性能结果上表现出来。
+即便只是要达到 20 MByte/sec 的传输速率，Linux-USB 设备驱动也必须让 EHCI
+队列始终保持满载。这意味着要发出较大的请求，或者在需要发出一连串小请求
+时使用批量请求排队。如果驱动做不到这一点，性能就会明显受影响。
 
 
-在典型情况下，使用 ``usb_bulk_msg()``
-以 4 KB 块循环写出，
-会浪费超过一半的 USB 2.0 带宽。
-I/O 完成与驱动发出下一次请求之间的延迟，
-通常会比一次 I/O 本身耗时更长。
-如果同样的循环改用 16 KB 块，会好一些；
-若使用一连串 128 KB 块，则浪费会少得多。
+在典型场景下，如果使用 ``usb_bulk_msg()`` 以 4 KB 块循环写出，会浪费超过
+一半的 USB 2.0 带宽。I/O 完成与驱动发出下一次请求之间的空档，往往比一次
+I/O 本身耗时还长。如果同样的循环改用 16 KB 块，情况会好一些；若使用一连串
+128 KB 块，则浪费会少得多。
 
+但与其依赖这么大的 I/O 缓冲区来提升同步 I/O 的效率，不如直接向主机控制器
+排队提交多个（批量）请求，然后等待它们全部完成（或在出错时取消）。这种
+URB 排队方式对所有 USB 1.1 主机控制器驱动同样适用。
 
-但与其依赖这么大的 I/O 缓冲区来让同步 I/O 高效，
-不如直接向主机控制器排入多个（批量）请求，
-然后等待它们全部完成（或在出错时取消）。
-这种 URB 排队方式对所有 USB 1.1
-主机控制器驱动也同样适用。
 
-
-在 Linux 2.5 内核中，定义了新的 ``usb_sg_*()`` API；
-它们会把 scatterlist 中的所有缓冲区都排入队列。
-它们还使用 scatterlist 的 DMA 映射
-（其中可能应用 IOMMU）并减少中断次数，
-这些都有助于让高速传输尽可能快地运行。
+在 Linux 2.5 内核中，定义了新的 ``usb_sg_*()`` API；它们会把 scatterlist
+中的所有缓冲区都排入队列。它们还使用 scatterlist 的 DMA 映射（其中可能
+应用 IOMMU）并减少中断次数，这些都有助于让高速传输尽可能快地运行。
 
 待办：
    中断传输和等时（ISO）传输的性能问题。
-   这些周期性传输都是完全调度的，因此，主要问题可能在于如何触发高带宽模式。
+   这些周期性传输都是完全调度的，因此主要问题可能在于如何触发高带宽模式。
 
 待办：
-   通过 ``sysfs`` 中的 ``uframe_periodic_max`` 参数，
-   可以分配超过标准 80% 的周期性带宽。
+   通过 ``sysfs`` 中的 ``uframe_periodic_max`` 参数，可以分配超过标准
+   80% 的周期性带宽。
    后续将对此进行说明。
diff --git a/Documentation/translations/zh_CN/usb/index.rst b/Documentation/translations/zh_CN/usb/index.rst
index eb5aca0c13ecc..df99814c6497e 100644
--- a/Documentation/translations/zh_CN/usb/index.rst
+++ b/Documentation/translations/zh_CN/usb/index.rst
@@ -1,4 +1,14 @@
 .. SPDX-License-Identifier: GPL-2.0
+
+.. only:: subproject and latex
+
+   .. raw:: latex
+
+      \renewcommand{\thesection}{}
+      \renewcommand{\thesubsection}{}
+      \kerneldocCJKon
+      \kerneldocBeginSC{
+
 .. include:: ../disclaimer-zh_CN.rst
 
 :Original: Documentation/usb/index.rst
@@ -24,7 +34,7 @@ USB 支持
     ehci
     usbmon
 
-Todolist:
+待翻译文档：
 
 * functionfs
 * functionfs-desc
@@ -52,3 +62,9 @@ Todolist:
    ====
 
    * :ref:`genindex`
+
+.. only:: subproject and latex
+
+   .. raw:: latex
+
+      }\kerneldocEndSC
diff --git a/Documentation/translations/zh_CN/usb/usbmon.rst b/Documentation/translations/zh_CN/usb/usbmon.rst
index 11b6d5b59dce2..db6030cd64a1c 100644
--- a/Documentation/translations/zh_CN/usb/usbmon.rst
+++ b/Documentation/translations/zh_CN/usb/usbmon.rst
@@ -16,67 +16,56 @@ usbmon
 
 简介
 ====
-小写形式的 ``usbmon`` 指的是内核中的一项功能，
-用于收集 USB 总线上的 I/O 跟踪信息。它类似于网络监控工具
-``tcpdump(1)`` 或 Ethereal 所使用的数据包套接字。
-类似地，人们希望使用 usbdump 或 USBMon
-（首字母大写）之类的工具来检查
-usbmon 生成的原始跟踪数据。
-
-usbmon 报告的是各个外设驱动
-向主机控制器驱动（HCD）发出的请求。
-因此，如果 HCD 本身有 bug，那么 usbmon 报告的跟踪信息
-可能无法精确对应实际的总线事务。
-这和 tcpdump 的情况是一样的。
-
-目前实现了两种 API: ``text`` 和 ``binary``。
-二进制 API 通过 ``/dev`` 命名空间中的字符设备提供，
-并且属于 ABI。文本 API 自内核 2.6.35 起已废弃，
-但为了方便仍然可用。
+小写的 ``usbmon`` 指的是内核中的一项功能，用于收集 USB 总线上的 I/O 跟踪
+信息。它类似于网络监控工具 ``tcpdump(1)`` 或 Ethereal 使用的数据包套接
+字。通常会用 usbdump 或 USBMon（首字母大写）之类的工具来查看 usbmon 生成
+的原始跟踪数据。
+
+usbmon 记录的是各个设备驱动向主机控制器驱动（HCD）发出的请求。因此，如果
+HCD 自身有 bug，usbmon 输出的跟踪信息就未必能和真实的总线事务一一对应。
+这和 tcpdump 的情况类似。
+
+目前实现了两种 API：``text`` 和 ``binary``。二进制 API 通过 ``/dev`` 下的
+字符设备提供，是 ABI 的一部分。文本 API 自内核 2.6.35 起已废弃，但为了
+兼容和使用方便，至今仍然保留。
 
 如何使用 usbmon 收集原始文本跟踪信息
 ====================================
 
-与数据包套接字不同，usbmon 提供了一种接口，
-可以输出文本格式的跟踪信息。这样做有两个目的：
-第一，在更完善的格式最终确定之前，
-它作为工具间通用的跟踪交换格式；
-第二，在不使用工具的情况下，人们也可以直接阅读这些信息。
+与数据包套接字不同，usbmon 还提供了一个输出文本格式跟踪信息的接口。这样
+做主要有两个目的：一是在更完善的格式最终确定之前，将其作为工具间通用的跟
+踪交换格式；二是在没有工具时也能直接阅读这些信息。
 
-要收集原始文本跟踪信息，请按以下步骤进行操作。
+要收集原始文本跟踪信息，按下面的步骤做即可。
 
 1. 准备
 -------
 
-挂载 debugfs（内核配置中必须启用它），并加载 usbmon 模块
-（如果它是作为模块构建的）。如果 usbmon 已经编入内核，
-那么第二步可以省略。
+挂载 debugfs（内核配置里必须启用它），并加载 usbmon 模块（如果它是以模块
+方式构建的）。如果 usbmon 已经编译进内核，这一步就可以省略。
 
 命令示例::
 
-    # mount -t debugfs none_debugs /sys/kernel/debug
+    # mount -t debugfs none /sys/kernel/debug
     # modprobe usbmon
     #
 
-确认总线套接字是否存在::
+确认 ``usbmon`` 目录下是否有这些条目::
 
     # ls /sys/kernel/debug/usb/usbmon
     0s  0u  1s  1t  1u  2s  2t  2u  3s  3t  3u  4s  4t  4u
     #
 
-现在，你可以选择使用 ``0u`` 捕获所有总线上的数据包，
-并跳到第 3 步；
-也可以先按第 2 步找到目标设备所在的总线。
-这样可以过滤掉那些持续输出数据的烦人设备。
+现在，你可以直接用 ``0u`` 捕获所有总线上的数据包，然后跳到第 3 步；也可
+以先按第 2 步找出目标设备所在的总线。这样可以把那些持续产生流量的设备过
+滤掉。
 
 2. 查找目标设备连接的是哪条总线
 -------------------------------
 
-运行 ``cat /sys/kernel/debug/usb/devices``，
-找到对应设备的 T 行。通常可以通过厂商字符串来查找。
-如果有许多类似设备，可以拔掉其中一个，
-再比较前后两次 ``/sys/kernel/debug/usb/devices``
-的输出。T 行里会包含总线编号。
+运行 ``cat /sys/kernel/debug/usb/devices``，找到对应设备的 T 行。通常可以通过
+厂商字符串来查找。如果有很多相似设备，可以拔掉其中一个，再比较前后两次
+``/sys/kernel/debug/usb/devices`` 的输出。T 行里会包含总线编号。
 
 示例::
 
@@ -86,8 +75,8 @@ usbmon 报告的是各个外设驱动
   S:  Manufacturer=ATEN
   S:  Product=UC100KM V2.00
 
-``Bus=03`` 表示它位于 3 号总线上。或者，
-也可以查看 ``lsusb`` 的输出，并从对应行得到总线编号。
+``Bus=03`` 表示它位于 3 号总线上。或者，也可以查看 ``lsusb`` 的输出，并从
+对应条目里找到总线编号。
 
 示例如下::
 
@@ -97,133 +86,110 @@ usbmon 报告的是各个外设驱动
 3. 启动 cat 命令
 ----------------
 
-如果只监听单条总线，可执行::
+如果只监听单条总线，执行::
 
     # cat /sys/kernel/debug/usb/usbmon/3u > /tmp/1.mon.out
 
-否则，如果要监听所有总线，则执行::
+否则，如果要监听所有总线，执行::
 
     # cat /sys/kernel/debug/usb/usbmon/0u > /tmp/1.mon.out
 
-此进程会一直读取，直到被终止。
-由于输出通常会很长，因此更推荐将输出重定向到某个位置。
+这个进程会一直运行到被终止为止。由于输出通常会很长，最好把它重定向到文件
+或其他位置。
 
 
 4. 在 USB 总线上执行期望的操作
 ------------------------------
 
-此处需要执行一些会产生 USB 流量的动作，
-比如插入 U 盘、拷贝文件、操作摄像头等。
+这里做一些会产生 USB 流量的操作即可，比如插入 U 盘、拷贝文件、操作摄像头
+等。
 
 
 5. 停止 cat
 -----------
 
-这一步通常通过键盘中断（Control-C）完成。
+这一步通常按下键盘中断（Control-C）即可完成。
 
-此时输出文件（本例中为 ``/tmp/1.mon.out``）
-可以保存、通过电子邮件发送，或使用文本编辑器查看。
-如果使用最后一种方式，请确保文件不会大到编辑器无法打开。
+此时，输出文件（本例中为 ``/tmp/1.mon.out``）可以保存下来，通过电子邮件发
+送，也可以用文本编辑器查看。如果要用文本编辑器查看，请确保文件大小不会
+大到编辑器无法处理。
 
 
 原始文本数据格式
 ================
 
-目前支持两种格式：原始格式，也就是 ``1t`` 格式，
-以及 ``1u`` 格式。``1t`` 格式在内核 2.6.21 中已被废弃。
-``1u`` 格式增加了一些字段，例如 ISO 帧描述符、
-``interval`` 等。它生成的行会稍长一些，
-但在其他方面是 ``1t`` 格式的完整超集。
-
-如果程序需要区分上述两种格式，
-可以查看 ``address`` 字段（见下文）。
-如果其中有两个冒号，就是 ``1t`` 格式；
-否则是 ``1u`` 格式。
-
-任何文本格式的数据由一系列事件组成，
-如 URB 提交、URB 回调、提交错误等。
-每个事件对应单独的一行文本，
-由使用空白符间隔的若干字段组成。
-字段的数量与位置可能取决于事件类型，
-但以下字段对所有类型都通用：
-
-下面按从左到右的顺序列出这些共有字段：
-
-- URB Tag。用于标识 URB，通常是 URB 结构体在内核中的地址
-  （以十六进制表示），
-  但也可能是序号或其他合理的唯一字符串。
-
-- 时间戳（微秒），十进制数字。
-  时间戳的精度取决于可用时钟，
-  因此可能远差于
-  1 微秒（例如实现使用的是 jiffies）。
-
-- 事件类型。它表示的是事件的格式，而不是 URB 的类型。
-  可用值为：``S`` 表示提交，``C`` 表示回调，``E`` 表示提交错误。
-
-- ``Address`` 字段（以前称作 ``pipe``）。
-  它包含四个由冒号分隔的字段：
-  URB 类型及方向、总线号、设备地址和端点号。类型与方向的编码如下：
-
-    == ==   ==========================
-    Ci Co   控制输入和输出
-    Zi Zo   等时输入和输出
-    Ii Io   中断输入和输出
-    Bi Bo   批量输入和输出
-    == ==   ==========================
-
-  总线号、设备地址和端点号使用十进制，但可能有前导零。
-
-- URB 状态字段。这个字段要么是一个字母，
-  要么是几个由冒号分隔的数字：
-  URB 状态、``interval``、``start frame`` 和 ``error count``。
-  与 ``address`` 字段不同，除了状态外，其余字段都是可选的。
-  ``interval`` 只会为中断和等时 URB 打印；``start frame`` 只会为
-  等时 URB 打印；错误计数只会在等时回调事件中打印。
-
-  状态字段是一个十进制数字，有时为负数，
-  对应 URB 的 ``status`` 字段。
-  对于提交事件，这个字段本身没有实际意义，
-  但为了便于脚本解析，它仍然存在。
-  当发生错误时，该字段包含错误码。
-
-  在提交控制包时，这个字段包含的是 ``Setup Tag``，
-  而不是一组数字。
-  判断 ``Setup Tag`` 是否存在很容易，因为它从来不是数字。
-  因此，如果脚本在这个字段里发现的是一组数字，
-  就会继续读取数据长度（等时 URB 除外）。
-  如果发现的是其他内容，比如一个字母，
-  那么脚本会先读取 ``Setup`` 包，再读取数据长度或等时描述符。
-
-- ``Setup`` 包由 5 个字段组成：
-  ``bmRequestType``、``bRequest``、``wValue``、
-  ``wIndex`` 和 ``wLength``。这些字段由 USB 2.0 规范定义。
-  如果 ``Setup Tag`` 为 ``s``，就可以安全地解码这些字段。
-  否则，说明 Setup 包虽然存在，但并未被捕获，此时各字段中会填入占位内容。
+目前支持两种格式：原始的 ``1t`` 格式和 ``1u`` 格式。``1t`` 格式在内核
+2.6.21 中已被废弃。``1u`` 格式增加了一些字段，例如 ISO 帧描述符和
+``interval``。它生成的行会稍长一些，但除此之外，它是 ``1t`` 格式的完整
+超集。
+
+如果程序需要区分上述两种格式，可以查看 ``address`` 字段（见下文）。如果
+其中有两个冒号，就是 ``1t`` 格式；否则是 ``1u`` 格式。
+
+任何文本格式的数据都由一系列事件构成，例如 URB 提交、URB 回调和提交错
+误。每个事件占一行，由若干以空白符分隔的字段组成。字段数量和位置会随事件
+类型变化，但下面这些字段对所有类型都通用：
+
+下面按从左到右的顺序说明这些通用字段：
+
+- URB 标识（URB Tag）。用于标识 URB，通常是 URB 结构体在内核中的地址
+  （十六进制），也可能是序号或其他足以唯一标识 URB 的字符串。
+
+- 时间戳（微秒），十进制数字。时间戳的精度取决于可用时钟，所以可能远低于
+  1 微秒（例如实现使用 jiffies 时）。
+
+- 事件类型。它表示的是这一行事件的格式，而不是 URB 的类型。可用值为：
+  ``S`` 表示提交，``C`` 表示回调，``E`` 表示提交错误。
+
+- ``Address`` 字段（以前称为 ``pipe``）。它包含四个由冒号分隔的字段：URB
+  类型及方向、总线号、设备地址和端点号。类型与方向按下面的方式编码：
+
+  == ==   ====================
+  Ci Co   控制输入与输出
+  Zi Zo   等时输入与输出
+  Ii Io   中断输入与输出
+  Bi Bo   批量输入与输出
+  == ==   ====================
+
+  总线号、设备地址和端点号都是十进制数，但可能有前导零，方便人阅读。
+
+- URB 状态字段。这个字段要么是一个字母，要么是几个用冒号分隔的数字，依次
+  表示 URB 状态、``interval``、``start frame`` 和 ``error count``。与
+  ``address`` 字段不同，除状态外，其余字段都可能省略。``interval`` 只会在
+  中断和等时 URB 中打印；``start frame`` 只会在等时 URB 中打印；错误计数只
+  会在等时回调事件中打印。
+
+  状态字段是一个十进制数，有时为负数，对应 URB 的 ``status`` 字段。对于提
+  交事件，这个字段本身并无实际语义，但为了便于脚本解析仍会保留。发生错误
+  时，这里填的是错误码。
+
+  如果是控制包的提交事件，这个字段里放的不是一组数字，而是 ``Setup Tag``。
+  这很容易分辨，因为 ``Setup Tag`` 永远不是数字。所以脚本如果在这里读到一
+  组数字，就会继续读取数据长度（等时 URB 除外）；如果读到的是字母之类的内
+  容，就要先读取 ``Setup`` 包，再读取数据长度或等时描述符。
+
+- ``Setup`` 包由 5 个字段组成：``bmRequestType``、``bRequest``、``wValue``、
+  ``wIndex`` 和 ``wLength``。这些字段由 USB 2.0 规范定义。如果 ``Setup Tag``
+  是 ``s``，就可以安全解码这些字段。否则，说明 Setup 包虽然存在，但并未被
+  捕获，此时各字段中会填入占位内容。
 
 - 等时传输帧描述符的数量及其内容：
-  如果一个等时传输事件带有一组描述符，首先打印该 URB 中描述符的总数，
-  然后为每个描述符打印一个字段，最多打印 5 个字段。
-  每个字段由三个用冒号分隔的十进制数字组成，
-  分别表示状态（status）、偏移（offset）和长度（length）。
-  对于提交（submission），报告的是初始长度；
-  对于回调（callback），报告的是实际长度。
+  如果某个等时传输事件带有描述符，会先打印该 URB 的描述符总数，再为每个描
+  述符打印一个字段，最多 5 个。每个字段由三个用冒号分隔的十进制数组成，依
+  次表示状态（status）、偏移（offset）和长度（length）。对于提交事件，报
+  告的是初始长度；对于回调事件，报告的是实际长度。
 
-- 数据长度：
-  对于提交，表示请求的长度；对于回调，表示实际传输的长度。
+- 数据长度：对于提交，表示请求的长度；对于回调，表示实际传输的长度。
 
-- 数据标签：
-  即使数据长度非零，usbmon 也不一定会捕获数据。
-  仅当标签为 ``=`` 时，才会有数据字段。
+- 数据标签：即使数据长度非零，usbmon 也不一定会捕获数据。只有标签为
+  ``=`` 时，才会有数据字段。
 
-- 数据字段：
-  以大端十六进制格式显示。注意，这些并不是真正的机器字，
-  而只是把字节流拆成若干“字”以便阅读。因此最后一个字可能只包含
-  1 到 4 个字节。
-  收集的数据长度是有限的，可能小于数据长度字段中报告的值。
-  因为数据长度字段只统计实际接收到的字节，而数据字段包含整个传输缓冲区，
-  所以，在等时输入（Zi）完成且缓冲区中接收到的数据稀疏的情况下，
-  收集的数据长度可能大于数据长度字段的值。
+- 数据字段：以大端十六进制格式显示。注意，这些并不是真正的机器字，只是为
+  了便于阅读，把字节流按“字”分组显示。因此最后一个字可能只包含 1 到 4 个
+  字节。捕获的数据长度是有限的，可能小于数据长度字段中报告的值。对于等时
+  输入（Zi）完成事件，如果缓冲区里的接收数据比较稀疏，捕获数据的长度甚至
+  可能大于数据长度字段，因为后者只统计实际接收到的字节，而数据字段展示的
+  是整个传输缓冲区。
 
 
 
@@ -234,18 +200,19 @@ usbmon 报告的是各个外设驱动
     d5ea89a0 3575914555 S Ci:1:001:0 s a3 00 0000 0003 0004 4 <
     d5ea89a0 3575914560 C Ci:1:001:0 0 4 = 01050000
 
-向地址为 5 的存储设备发送
-31 字节 Bulk 包装的 SCSI 命令 ``0x28``
-（``READ_10``）的输出批量传输::
+向地址为 5 的存储设备发送一个输出批量传输，其中 31 字节的 Bulk 封装用于承
+载 SCSI 命令 ``0x28``（``READ_10``）。为便于排版，下面的第一条记录按两行
+显示，但实际 usbmon 输出仍是一行::
 
-    dd65f0e8 4128379752 S Bo:1:005:2 -115 31 = 55534243 ad000000 00800000 80010a28 20000000 20000040 00000000 000000
+    dd65f0e8 4128379752 S Bo:1:005:2 -115 31 =
+      55534243 ad000000 00800000 80010a28 20000000 20000040
+      00000000 000000
     dd65f0e8 4128379808 C Bo:1:005:2 0 31 >
 
 原始二进制格式与 API
 ====================
-API 的整体架构与前文大体相同，只是事件以二进制格式传递。
-每个事件都通过下面的结构发送
-（这个名字是为了叙述方便而虚构的）::
+API 的整体架构与前文大体相同，只是事件以二进制格式传递。每个事件都通过
+下面的结构发送（这个结构名只是为了叙述方便而虚构的）::
 
 
   struct usbmon_packet {
@@ -275,29 +242,22 @@ API 的整体架构与前文大体相同，只是事件以二进制格式传递
 	unsigned int ndesc;	/* 60: 实际 ISO 描述符数量 */
   };				/* 64 总长度 */
 
-可以用 ``read(2)``、``ioctl(2)``，
-或者通过 ``mmap`` 访问缓冲区，
-从字符设备接收这些事件。
-不过，出于兼容性原因，``read(2)``
-只返回前 48 个字节。
+可以用 ``read(2)``、``ioctl(2)``，或者通过 ``mmap`` 访问缓冲区，从字符设
+备接收这些事件。不过，出于兼容性原因，``read(2)`` 只返回前 48 个字节。
 
-字符设备通常命名为 ``/dev/usbmonN``，
-其中 ``N`` 是 USB 总线号。
-编号为零的设备（``/dev/usbmon0``）比较特殊，
-表示“所有总线”。
-请注意，具体命名策略由 Linux 发行版决定。
+字符设备通常命名为 ``/dev/usbmonN``，其中 ``N`` 是 USB 总线号。编号为零的
+设备（``/dev/usbmon0``）比较特殊，表示“所有总线”。具体命名策略由 Linux
+发行版决定。
 
-如果你手动创建 ``/dev/usbmon0``，
-请确保它归 root 所有，并且权限为 ``0600``。
-否则，非特权用户将能够窃听键盘流量。
+如果你手动创建 ``/dev/usbmon0``，请确保它归 root 所有，并且权限为 ``0600``。
+否则，非特权用户就能窃听键盘输入流量。
 
 以下 ``MON_IOC_MAGIC`` 为 ``0x92`` 的 ioctl 调用可用：
 
 ``MON_IOCQ_URB_LEN``，定义为 ``_IO(MON_IOC_MAGIC, 1)``
 
-该调用返回下一个事件的数据长度。
-注意大多数事件不包含数据，
-因此如果该调用返回零，并不意味着没有事件。
+该调用返回下一个事件的数据长度。注意大多数事件不包含数据，因此如果它返回
+零，并不意味着没有事件。
 
 ``MON_IOCG_STATS``，定义为
 ``_IOR(MON_IOC_MAGIC, 3, struct mon_bin_stats)``
@@ -309,18 +269,16 @@ API 的整体架构与前文大体相同，只是事件以二进制格式传递
 	u32 dropped;
   };
 
-成员 ``queued`` 表示当前缓冲区中已经排队的事件数量，
-而不是自上次重置以来处理过的事件数量。
+成员 ``queued`` 表示当前缓冲区中已经排队的事件数量，而不是自上次重置以来
+处理过的事件数量。
 
-成员 ``dropped`` 表示自上次调用
-``MON_IOCG_STATS`` 以来丢失的事件数量。
+成员 ``dropped`` 表示自上次调用 ``MON_IOCG_STATS`` 以来丢失的事件数量。
 
 ``MON_IOCT_RING_SIZE``，定义为 ``_IO(MON_IOC_MAGIC, 4)``
 
-此调用设置缓冲区大小。参数为以字节为单位的缓冲区大小。
-大小可能会向下取整到下一个块（或页）。
-如果请求的大小超出该内核的 [未指定] 范围，
-则调用会失败并返回 ``-EINVAL``。
+此调用设置缓冲区大小。参数是以字节为单位的缓冲区大小。大小可能会向下取整
+到下一个块（或页）。如果请求的大小超出当前内核允许的范围，则调用会失败并
+返回 ``-EINVAL``。
 
 ``MON_IOCQ_RING_SIZE``，定义为 ``_IO(MON_IOC_MAGIC, 5)``
 
@@ -331,9 +289,8 @@ API 的整体架构与前文大体相同，只是事件以二进制格式传递
 ``MON_IOCX_GETX``，定义为
 ``_IOW(MON_IOC_MAGIC, 10, struct mon_get_arg)``
 
-如果内核缓冲区中没有事件，
-这些调用就会一直等待，直到有事件到达，
-然后返回第一个事件。
+如果内核缓冲区中没有事件，这些调用就会一直等待，直到有事件到达，然后返回
+第一个事件。
 参数是指向以下结构的指针::
 
   struct mon_get_arg {
@@ -343,20 +300,18 @@ API 的整体架构与前文大体相同，只是事件以二进制格式传递
   };
 
 
-调用前，应填好 ``hdr``、``data`` 和 ``alloc``。
-调用返回后，``hdr`` 指向的区域中包含下一个事件的结构；
-如果存在数据，那么数据缓冲区中也会包含相应数据。
-该事件会从内核缓冲区中移除。
+调用前，应填好 ``hdr``、``data`` 和 ``alloc``。调用返回后，``hdr`` 指向的
+内存区域中会写入下一个事件的结构；如果存在数据，数据缓冲区中也会填入相应
+内容。该事件会从内核缓冲区中移除。
 
-``MON_IOCX_GET`` 会将 48 字节的数据复制到 ``hdr`` 区域，
-``MON_IOCX_GETX`` 会复制 64 字节。
+``MON_IOCX_GET`` 会将 48 字节的数据复制到 ``hdr`` 区域，``MON_IOCX_GETX``
+会复制 64 字节。
 
 ``MON_IOCX_MFETCH``，定义为
 ``_IOWR(MON_IOC_MAGIC, 7, struct mon_mfetch_arg)``
 
-当应用程序通过 ``mmap(2)`` 访问缓冲区时，
-主要使用这个 ioctl。
-其参数是指向以下结构的指针::
+应用程序通过 ``mmap(2)`` 访问缓冲区时，主要使用这个 ioctl。其参数是指向
+以下结构的指针::
 
   struct mon_mfetch_arg {
 	uint32_t *offvec;	/* 获取的事件偏移向量 */
@@ -365,41 +320,36 @@ API 的整体架构与前文大体相同，只是事件以二进制格式传递
   };
 
 
-该 ioctl 的操作分为三个阶段：
+这个 ioctl 的流程分为三个阶段：
 
-首先，从内核缓冲区移除并丢弃最多 ``nflush`` 个事件。
-实际丢弃的事件数量会写回 ``nflush``。
+首先，从内核缓冲区移除并丢弃最多 ``nflush`` 个事件。实际丢弃的事件数量会
+写回 ``nflush``。
 
-其次，除非伪设备以 ``O_NONBLOCK`` 打开，否则会一直等待，
-直到缓冲区中出现事件。
+其次，除非设备以 ``O_NONBLOCK`` 打开，否则会一直等待，直到缓冲区中出现
+事件。
 
-第三，将最多 ``nfetch`` 个偏移量提取到 mmap
-缓冲区，并存入 ``offvec`` 中。
-实际提取到的事件偏移数量会存回 ``nfetch``。
+第三，将最多 ``nfetch`` 个偏移量提取到 mmap 缓冲区，并存入 ``offvec`` 中。
+实际提取到的事件偏移数量会写回 ``nfetch``。
 
 ``MON_IOCH_MFLUSH``，定义为 ``_IO(MON_IOC_MAGIC, 8)``
 
-此调用从内核缓冲区移除若干事件。
-其参数为要移除的事件数量。
-如果缓冲区中的事件少于请求数量，
-则移除所有事件，且不报告错误。
-当没有事件时也可使用。
+此调用从内核缓冲区移除若干事件。其参数是要移除的事件数量。如果缓冲区中的
+事件少于请求数量，则移除全部现有事件，且不报告错误。即使当前没有事件，也
+可以调用。
 
 ``FIONBIO``
 
 如果有需要，将来可能会实现 ``FIONBIO`` ioctl。
 
-除了 ``ioctl(2)`` 和 ``read(2)`` 之外，
-二进制 API 的特殊文件也可以用 ``select(2)`` 和
-``poll(2)`` 轮询。
-但 ``lseek(2)`` 不起作用。
+除了 ``ioctl(2)`` 和 ``read(2)`` 之外，二进制 API 对应的特殊文件还可以用
+``select(2)`` 和 ``poll(2)`` 轮询，但 ``lseek(2)`` 不可用。
 
 * 二进制 API 的内核缓冲区内存映射访问
 
-基本思想很简单：
+基本思路很简单：
 
-准备时，先获取当前大小，再用 ``mmap(2)`` 映射缓冲区。
-然后执行类似下面伪代码的循环::
+准备时，先查询当前大小，再用 ``mmap(2)`` 映射缓冲区。之后运行与下面伪代码
+类似的循环::
 
    struct mon_mfetch_arg fetch;
    struct usbmon_packet *hdr;
@@ -411,7 +361,7 @@ API 的整体架构与前文大体相同，只是事件以二进制格式传递
       ioctl(fd, MON_IOCX_MFETCH, &fetch);   // 同时处理错误
       nflush = fetch.nfetch;       // 完成后要刷新这么多包
       for (i = 0; i < nflush; i++) {
-         hdr = (struct ubsmon_packet *) &mmap_area[vec[i]];
+         hdr = (struct usbmon_packet *) &mmap_area[vec[i]];
          if (hdr->type == '@')     // 填充包
             continue;
          caddr_t data = &mmap_area[vec[i]] + 64;
@@ -421,7 +371,7 @@ API 的整体架构与前文大体相同，只是事件以二进制格式传递
 
 
 
-因此，主要思想是每 N 个事件只执行一次 ioctl。
+因此，这里的核心思路就是每 N 个事件只执行一次 ioctl。
 
-虽然缓冲区是环形的，但返回的头和数据不会跨越缓冲区末端，
-因此上面的伪代码无需任何合并操作。
+虽然缓冲区是环形的，但返回的头部和数据不会跨越缓冲区末端，因此上面的伪代
+码无需做任何拼接。
diff --git a/Documentation/usb/functionfs.rst b/Documentation/usb/functionfs.rst
index f7487b0d80578..b189cf5626ba1 100644
--- a/Documentation/usb/functionfs.rst
+++ b/Documentation/usb/functionfs.rst
@@ -72,6 +72,82 @@ have been written to their ep0's.
 Conversely, the gadget is unregistered after the first USB function
 closes its endpoints.
 
+Endpoint IOCTLs
+===============
+
+FunctionFS supports additional IOCTLs that can be performed on data endpoints
+(ie. not ep0). For a full list of these IOCTLs, please refer to the documentation
+in ``include/uapi/linux/usb/functionfs.h``.
+
+One such IOCTL is:
+
+  ``FUNCTIONFS_ENDPOINT_ENABLE_ZLP(__u32 *)``
+    Enable or disable automatic zero-length packet (ZLP) appending for the
+    endpoint. The argument is a pointer to a __u32: 0 to disable, non-zero to
+    enable. When enabled, the kernel will automatically append a ZLP at the end
+    of a transfer if the payload length is an exact multiple of the endpoint's
+    max packet size. This is useful for compatibility with legacy protocols
+    which require automatic ZLP appending to data written from userspace. This
+    IOCTL can only be used on IN endpoints. It can be called at any time after
+    the FunctionFS instance is active, even before the host has connected or
+    enabled the endpoint. Returns zero on success, or a negative errno value on
+    error:
+
+    * ``-ENODEV``: The FunctionFS instance is not active.
+    * ``-EINVAL``: The endpoint is not an IN endpoint.
+    * ``-EFAULT``: Invalid user space pointer for the argument.
+
+RW Proxy Endpoints
+==================
+
+If the ``FUNCTIONFS_RW_PROXY_EPS`` flag is passed in the descriptor header
+(requires ``FUNCTIONFS_DESCRIPTORS_MAGIC_V2``), FunctionFS will provision a
+bidirectional rw_proxy file descriptor (e.g., "ep1_rw") alongside each pair
+of IN and OUT endpoints. The rw_proxy file aliases the underlying hardware
+endpoints, allowing userspace to use a single file descriptor for both reading
+(OUT) and writing (IN).
+
+This flag requires the total number of hardware endpoints to be an even number.
+FunctionFS will automatically walk the provided endpoints and group them into
+adjacent pairs (e.g., ep1 and ep2 form the first pair, ep3 and ep4 form the
+second pair). Each pair must consist of exactly one IN endpoint and one OUT
+endpoint.
+
+For each valid pair, a rw_proxy file is created and named after the first
+endpoint in the pair with a "_rw" suffix. For example, if ep1 and ep2 are
+paired, a rw_proxy file named "ep1_rw" is created. If ep3 and ep4 are paired,
+"ep3_rw" is created.
+
+If the ``FUNCTIONFS_VIRTUAL_ADDR`` flag is also enabled, the endpoints will be
+named using their physical endpoint address in hexadecimal instead of their
+index. RW proxy files will inherit this naming convention. For example, if the
+first endpoint of a pair maps to address 0x02, the rw_proxy file will be
+named "ep02_rw".
+
+When this flag is enabled, userspace has the choice of performing data transfers
+via the single rw_proxy file descriptor or the two base file descriptors. The
+rw_proxy file descriptor acts as a pure VFS alias that proxies all operations
+directly to the underlying base file descriptors.
+
+Because it is a pure proxy, there are no data races or buffer corruptions if
+userspace uses both the rw_proxy endpoint and the base endpoints concurrently.
+The native mutexes of the base endpoints perfectly serialize all concurrent
+transfers. However, userspace should generally pick one method and stick to it
+to avoid interleaving its own data stream.
+
+- **IOCTLs (Clear Halt, etc.):** RW proxy endpoints do not support IOCTLs and
+  will return ``-ENOTTY``. To clear a host-initiated halt, userspace must issue
+  the ``FUNCTIONFS_CLEAR_HALT`` ioctl directly on the corresponding base
+  endpoint file descriptor.
+- **Intentional Stalls:** The traditional mechanism for intentionally halting an
+  endpoint by issuing a reverse-direction data operation (e.g., attempting to
+  read from an IN endpoint) continues to work, but it must be issued on the
+  base endpoint. RW proxy endpoints cannot be used to trigger a stall because
+  they are fully bidirectional.
+
+Note that DMABUF data transfers (``FUNCTIONFS_DMABUF_TRANSFER``) are unsupported
+via the rw_proxy endpoint because it does not support IOCTLs. If DMABUF
+transfers are required, users must use the standard base endpoints.
 DMABUF interface
 ================
 
@@ -79,6 +155,10 @@ FunctionFS additionally supports a DMABUF based interface, where the
 userspace can attach DMABUF objects (externally created) to an endpoint,
 and subsequently use them for data transfers.
 
+Note: The DMABUF interface is unsupported on rw_proxy endpoints. See
+the RW Proxy Endpoints section for details on using DMABUF alongside
+the ``FUNCTIONFS_RW_PROXY_EPS`` flag.
+
 A userspace application can then use this interface to share DMABUF
 objects between several interfaces, allowing it to transfer data in a
 zero-copy fashion, for instance between IIO and the USB stack.
diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst
index 3f0ef1e27eb07..2fc53093752d1 100644
--- a/Documentation/userspace-api/ioctl/ioctl-number.rst
+++ b/Documentation/userspace-api/ioctl/ioctl-number.rst
@@ -346,6 +346,8 @@ Code  Seq#    Include File                                             Comments
                                                                        <mailto:luzmaximilian@gmail.com>
 0xA5  20-2F  linux/surface_aggregator/dtx.h                            Microsoft Surface DTX driver
                                                                        <mailto:luzmaximilian@gmail.com>
+0xA6  00-0F  uapi/linux/alloc_tag.h                                    Memory allocation profiling
+                                                                       <mailto:surenb@google.com>
 0xAA  00-3F  linux/uapi/linux/userfaultfd.h
 0xAB  00-1F  linux/nbd.h
 0xAC  00-1F  linux/raw.h
diff --git a/Documentation/userspace-api/media/cec/cec-ioc-adap-g-log-addrs.rst b/Documentation/userspace-api/media/cec/cec-ioc-adap-g-log-addrs.rst
index f3293a589dd6b..8397bf5737988 100644
--- a/Documentation/userspace-api/media/cec/cec-ioc-adap-g-log-addrs.rst
+++ b/Documentation/userspace-api/media/cec/cec-ioc-adap-g-log-addrs.rst
@@ -185,6 +185,13 @@ logical address types are already defined will return with error ``EBUSY``.
 	are CEC devices that can only handle CDC messages.
 
 	All other messages are ignored.
+    * .. _`CEC-LOG-ADDRS-FL-CONFIG-FAILED`:
+
+      - ``CEC_LOG_ADDRS_FL_CONFIG_FAILED``
+      - 8
+      - If this flag is set, then the CEC device failed to claim a free logical
+        address and is in the unconfigured state. This can never happen if
+	``CEC_LOG_ADDRS_FL_ALLOW_UNREG_FALLBACK`` was set as well.
 
 .. tabularcolumns:: |p{7.8cm}|p{1.0cm}|p{8.5cm}|
 
diff --git a/Documentation/userspace-api/media/v4l/vidioc-g-ext-ctrls.rst b/Documentation/userspace-api/media/v4l/vidioc-g-ext-ctrls.rst
index b8698b85bd808..d78328152b755 100644
--- a/Documentation/userspace-api/media/v4l/vidioc-g-ext-ctrls.rst
+++ b/Documentation/userspace-api/media/v4l/vidioc-g-ext-ctrls.rst
@@ -72,12 +72,6 @@ partial array, all elements have to be set or retrieved. The total size
 is calculated as ``elems`` * ``elem_size``. These values can be obtained
 by calling :ref:`VIDIOC_QUERY_EXT_CTRL <VIDIOC_QUERYCTRL>`.
 
-To change the value of a set of controls applications initialize the
-``id``, ``size``, ``reserved2`` and ``value/value64/string/ptr`` fields
-of each struct :c:type:`v4l2_ext_control` and call
-the :ref:`VIDIOC_S_EXT_CTRLS <VIDIOC_G_EXT_CTRLS>` ioctl. The controls will only be set if *all*
-control values are valid.
-
 To check if a set of controls have correct values applications
 initialize the ``id``, ``size``, ``reserved2`` and
 ``value/value64/string/ptr`` fields of each struct
@@ -86,6 +80,15 @@ initialize the ``id``, ``size``, ``reserved2`` and
 values are automatically adjusted to a valid value or if an error is
 returned.
 
+To change the value of a set of controls applications initialize the
+``id``, ``size``, ``reserved2`` and ``value/value64/string/ptr`` fields
+of each struct :c:type:`v4l2_ext_control` and call
+the :ref:`VIDIOC_S_EXT_CTRLS <VIDIOC_G_EXT_CTRLS>` ioctl.
+:ref:`VIDIOC_S_EXT_CTRLS <VIDIOC_G_EXT_CTRLS>` will implicitly call
+:ref:`VIDIOC_TRY_EXT_CTRLS <VIDIOC_G_EXT_CTRLS>` first and return an
+error if that fails. So the controls will only be set if *all*
+control values are valid.
+
 When the ``id`` or ``which`` is invalid drivers return an ``EINVAL`` error
 code. When the value is out of bounds drivers can choose to take the
 closest valid value or return an ``ERANGE`` error code, whatever seems more
diff --git a/Documentation/w1/masters/omap-hdq.rst b/Documentation/w1/masters/omap-hdq.rst
index 5347b5d9e90aa..2ecf572be1a45 100644
--- a/Documentation/w1/masters/omap-hdq.rst
+++ b/Documentation/w1/masters/omap-hdq.rst
@@ -50,5 +50,5 @@ The driver also supports 1-wire mode. In this mode, there is no need to
 pass slave ID as parameter. The driver will auto-detect slaves connected
 to the bus using SEARCH_ROM procedure. 1-wire mode can be selected by
 setting "ti,mode" property to "1w" in DT (see
-Documentation/devicetree/bindings/w1/omap-hdq.txt for more details).
+Documentation/devicetree/bindings/w1/ti,hdq.yaml for more details).
 By default driver is in HDQ mode.
diff --git a/Documentation/watchdog/watchdog-parameters.rst b/Documentation/watchdog/watchdog-parameters.rst
index ec5f67c060cbd..502cb6adbeda7 100644
--- a/Documentation/watchdog/watchdog-parameters.rst
+++ b/Documentation/watchdog/watchdog-parameters.rst
@@ -59,7 +59,7 @@ advantechwdt:
 
 alim1535_wdt:
     timeout:
-	Watchdog timeout in seconds. (0 < timeout < 18000, default=60
+	Watchdog timeout in seconds. (0 < timeout < 18000, default=60)
     nowayout:
 	Watchdog cannot be stopped once started
 	(default=kernel config parameter)
@@ -68,7 +68,7 @@ alim1535_wdt:
 
 alim7101_wdt:
     timeout:
-	Watchdog timeout in seconds. (1<=timeout<=3600, default=30
+	Watchdog timeout in seconds. (1<=timeout<=3600, default=30)
     use_gpio:
 	Use the gpio watchdog (required by old cobalt boards).
 	default=0/off/no
diff --git a/MAINTAINERS b/MAINTAINERS
index 806bd2d80d153..52e2c22d533a2 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -952,10 +952,10 @@ S:	Maintained
 F:	drivers/mailbox/mailbox-altera.c
 
 ALTERA MSGDMA IP CORE DRIVER
-M:	Olivier Dautricourt <olivierdautricourt@gmail.com>
+M:	Adrian Ng Ho Yin <adrian.ho.yin.ng@altera.com>
 R:	Stefan Roese <sr@denx.de>
 L:	dmaengine@vger.kernel.org
-S:	Odd Fixes
+S:	Maintained
 F:	Documentation/devicetree/bindings/dma/altr,msgdma.yaml
 F:	drivers/dma/altera-msgdma.c
 
@@ -1138,6 +1138,13 @@ F:	drivers/char/hw_random/geode-rng.c
 F:	drivers/crypto/geode*
 F:	drivers/video/fbdev/geode/
 
+AMD HALO BOX RGB LED DRIVER
+M:	Mario Limonciello (AMD) <superm1@kernel.org>
+R:	Yo-Jung Leo Lin (AMD) <Leo.Lin@amd.com>
+L:	platform-driver-x86@vger.kernel.org
+S:	Supported
+F:	drivers/platform/x86/amd/amd_halo_led.c
+
 AMD HSMP DRIVER
 M:	Naveen Krishna Chatradhi <naveenkrishna.chatradhi@amd.com>
 R:	Carlos Bilbao <carlos.bilbao@kernel.org>
@@ -1256,6 +1263,7 @@ L:	platform-driver-x86@vger.kernel.org
 S:	Supported
 F:	Documentation/ABI/testing/sysfs-amd-pmf
 F:	drivers/platform/x86/amd/pmf/
+F:	include/uapi/linux/amd-pmf.h
 
 AMD POWERPLAY AND SWSMU
 M:	Kenneth Feng <kenneth.feng@amd.com>
@@ -1702,6 +1710,15 @@ W:	https://ez.analog.com/linux-software-drivers
 F:	Documentation/devicetree/bindings/iio/adc/adi,ade9000.yaml
 F:	drivers/iio/adc/ade9000.c
 
+ANALOG DEVICES INC ADF41513 DRIVER
+M:	Rodrigo Alencar <rodrigo.alencar@analog.com>
+L:	linux-iio@vger.kernel.org
+S:	Supported
+W:	https://ez.analog.com/linux-software-drivers
+F:	Documentation/devicetree/bindings/iio/frequency/adi,adf41513.yaml
+F:	Documentation/iio/adf41513.rst
+F:	drivers/iio/frequency/adf41513.c
+
 ANALOG DEVICES INC ADF4377 DRIVER
 M:	Antoniu Miclaus <antoniu.miclaus@analog.com>
 L:	linux-iio@vger.kernel.org
@@ -2607,6 +2624,7 @@ F:	Documentation/devicetree/bindings/power/apple*
 F:	Documentation/devicetree/bindings/power/reset/apple,smc-reboot.yaml
 F:	Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml
 F:	Documentation/devicetree/bindings/rtc/apple,smc-rtc.yaml
+F:	Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml
 F:	Documentation/devicetree/bindings/spi/apple,spi.yaml
 F:	Documentation/devicetree/bindings/spmi/apple,spmi.yaml
 F:	Documentation/devicetree/bindings/usb/apple,dwc3.yaml
@@ -2767,12 +2785,12 @@ F:	arch/arm/mach-ep93xx/
 F:	drivers/iio/adc/ep93xx_adc.c
 
 ARM/CIX SOC SUPPORT
-M:	Peter Chen <peter.chen@cixtech.com>
+M:	Gary Yang <gary.yang@cixtech.com>
 M:	Fugang Duan <fugang.duan@cixtech.com>
 R:	CIX Linux Kernel Upstream Group <cix-kernel-upstream@cixtech.com>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/peter.chen/cix.git
+T:	git https://github.com/cixtech/linux-mainline.git
 F:	Documentation/devicetree/bindings/arm/cix.yaml
 F:	Documentation/devicetree/bindings/mailbox/cix,sky1-mbox.yaml
 F:	arch/arm64/boot/dts/cix/
@@ -3757,6 +3775,7 @@ M:	Vignesh Raghavendra <vigneshr@ti.com>
 M:	Tero Kristo <kristo@kernel.org>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Supported
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/ti/linux.git
 F:	Documentation/devicetree/bindings/arm/ti/k3.yaml
 F:	Documentation/devicetree/bindings/hwinfo/ti,k3-socinfo.yaml
 F:	arch/arm64/boot/dts/ti/Makefile
@@ -3946,7 +3965,7 @@ F:	Documentation/devicetree/bindings/leds/ams,as3668.yaml
 F:	drivers/leds/leds-as3668.c
 
 ASAHI KASEI AK7375 LENS VOICE COIL DRIVER
-M:	Tianshu Qiu <tian.shu.qiu@intel.com>
+M:	Sakari Ailus <sakari.ailus@linux.intel.com>
 L:	linux-media@vger.kernel.org
 S:	Maintained
 T:	git git://linuxtv.org/media.git
@@ -4168,11 +4187,11 @@ F:	include/linux/async_tx.h
 
 AT24 EEPROM DRIVER
 M:	Bartosz Golaszewski <brgl@kernel.org>
-L:	linux-i2c@vger.kernel.org
+L:	linux-kernel@vger.kernel.org
 S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git
 F:	Documentation/devicetree/bindings/eeprom/at24.yaml
-F:	drivers/misc/eeprom/at24.c
+F:	drivers/nvmem/at24.c
 
 ATA OVER ETHERNET (AOE) DRIVER
 M:	"Justin Sanders" <justin@coraid.com>
@@ -4388,7 +4407,7 @@ S:	Maintained
 F:	drivers/media/pci/hws/
 
 AWINIC AW99706 WLED BACKLIGHT DRIVER
-M:	Junjie Cao <caojunjie650@gmail.com>
+M:	Junjie Cao <junjie.cao@linux.dev>
 S:	Maintained
 F:	Documentation/devicetree/bindings/leds/backlight/awinic,aw99706.yaml
 F:	drivers/video/backlight/aw99706.c
@@ -5997,6 +6016,7 @@ B:	https://tracker.ceph.com/
 T:	git https://github.com/ceph/ceph-client.git
 F:	Documentation/filesystems/ceph.rst
 F:	fs/ceph/
+F:	tools/testing/selftests/filesystems/ceph/
 
 CERTIFICATE HANDLING
 M:	David Howells <dhowells@redhat.com>
@@ -6209,6 +6229,7 @@ F:	include/dt-bindings/sound/cs*
 F:	include/linux/mfd/cs42l43*
 F:	include/sound/cs*
 F:	sound/hda/codecs/cirrus*
+F:	sound/hda/codecs/side-codecs/cirrus*
 F:	sound/hda/codecs/side-codecs/cs*
 F:	sound/hda/codecs/side-codecs/hda_component*
 F:	sound/soc/codecs/cs*
@@ -6851,6 +6872,13 @@ L:	linux-riscv@lists.infradead.org
 S:	Maintained
 F:	drivers/cpuidle/cpuidle-riscv-sbi.c
 
+CPUIDLE DRIVER - S390
+M:	Mete Durlu <meted@linux.ibm.com>
+L:	linux-pm@vger.kernel.org
+L:	linux-s390@vger.kernel.org
+S:	Maintained
+F:	drivers/cpuidle/cpuidle-s390.c
+
 CPUMASK API [RUST]
 M:	Viresh Kumar <viresh.kumar@linaro.org>
 R:	Yury Norov <yury.norov@gmail.com>
@@ -7124,7 +7152,7 @@ W:	https://docs.dasharo.com/
 F:	drivers/platform/x86/dasharo-acpi.c
 
 DAMON
-M:	SeongJae Park <sj@kernel.org>
+M:	SJ Park <sj@kernel.org>
 L:	damon@lists.linux.dev
 L:	linux-mm@kvack.org
 S:	Maintained
@@ -8120,6 +8148,12 @@ T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
 F:	Documentation/devicetree/bindings/display/ilitek,ili9486.yaml
 F:	drivers/gpu/drm/tiny/ili9486.c
 
+DRM DRIVER FOR ILITEK ILI9488 PANELS
+M:	Igor Reznichenko <igor@reznichenko.net>
+S:	Maintained
+F:	Documentation/devicetree/bindings/display/panel/ilitek,ili9488.yaml
+F:	drivers/gpu/drm/panel/panel-ilitek-ili9488.c
+
 DRM DRIVER FOR ILITEK ILI9805 PANELS
 M:	Michael Trimarchi <michael@amarulasolutions.com>
 S:	Maintained
@@ -8852,11 +8886,14 @@ M:	Sandy Huang <hjc@rock-chips.com>
 M:	Heiko Stübner <heiko@sntech.de>
 M:	Andy Yan <andy.yan@rock-chips.com>
 L:	dri-devel@lists.freedesktop.org
+L:	linux-rockchip@lists.infradead.org
 S:	Maintained
 T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
 F:	Documentation/devicetree/bindings/display/rockchip/
+F:	drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.*
 F:	drivers/gpu/drm/ci/xfails/rockchip*
 F:	drivers/gpu/drm/rockchip/
+F:	include/drm/bridge/dw_hdmi_qp.h
 
 DRM DRIVERS FOR STI
 M:	Alain Volmat <alain.volmat@foss.st.com>
@@ -9337,9 +9374,8 @@ S:	Supported
 F:	drivers/edac/dmc520_edac.c
 
 EDAC-E752X
-M:	Mark Gross <markgross@kernel.org>
 L:	linux-edac@vger.kernel.org
-S:	Maintained
+S:	Orphan
 F:	drivers/edac/e752x_edac.c
 
 EDAC-E7XXX
@@ -9414,9 +9450,8 @@ S:	Maintained
 F:	drivers/edac/igen6_edac.c
 
 EDAC-MPC85XX
-M:	Johannes Thumshirn <morbidrsa@gmail.com>
 L:	linux-edac@vger.kernel.org
-S:	Maintained
+S:	Orphan
 F:	drivers/edac/mpc85xx_edac.[ch]
 
 EDAC-NPCM
@@ -9493,16 +9528,6 @@ L:	linux-fbdev@vger.kernel.org
 S:	Maintained
 F:	drivers/video/fbdev/efifb.c
 
-EFS FILESYSTEM
-S:	Orphan
-W:	http://aeschi.ch.eu.org/efs/
-F:	fs/efs/
-
-EHEA (IBM pSeries eHEA 10Gb ethernet adapter) DRIVER
-L:	netdev@vger.kernel.org
-S:	Orphan
-F:	drivers/net/ethernet/ibm/ehea/
-
 ELM327 CAN NETWORK DRIVER
 M:	Max Staudt <max@enpas.org>
 L:	linux-can@vger.kernel.org
@@ -9758,7 +9783,7 @@ EXFAT FILE SYSTEM
 M:	Namjae Jeon <linkinjeon@kernel.org>
 M:	Sungjong Seo <sj1557.seo@samsung.com>
 R:	Yuezhang Mo <yuezhang.mo@sony.com>
-L:	linux-fsdevel@vger.kernel.org
+L:	exfat@lists.linux.dev
 S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/exfat.git
 F:	fs/exfat/
@@ -9846,6 +9871,7 @@ F:	fs/proc/bootconfig.c
 F:	include/linux/bootconfig.h
 F:	lib/bootconfig-data.S
 F:	lib/bootconfig.c
+F:	lib/embedded-cmdline.S
 F:	tools/bootconfig/*
 F:	tools/bootconfig/scripts/*
 
@@ -10133,6 +10159,11 @@ F:	drivers/base/firmware_loader/
 F:	rust/kernel/firmware.rs
 F:	include/linux/firmware.h
 
+FIXED-LAYOUT NVMEM LAYOUT DRIVER
+M:	Mathieu Dubois-Briand <mathieu.dubois-briand@bootlin.com>
+S:	Maintained
+F:	drivers/nvmem/layouts/fixed-layout.c
+
 FLEXTIMER FTM-QUADDEC DRIVER
 M:	Patrick Havelange <patrick.havelange@essensium.com>
 L:	linux-iio@vger.kernel.org
@@ -11430,7 +11461,7 @@ M:	Bjorn Andersson <andersson@kernel.org>
 R:	Baolin Wang <baolin.wang7@gmail.com>
 L:	linux-remoteproc@vger.kernel.org
 S:	Maintained
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/andersson/remoteproc.git hwspinlock-next
+T:	git https://git.kernel.org/pub/scm/linux/kernel/git/remoteproc/linux.git hwspinlock-next
 F:	Documentation/devicetree/bindings/hwlock/
 F:	Documentation/locking/hwspinlock.rst
 F:	drivers/hwspinlock/
@@ -11788,7 +11819,7 @@ F:	Documentation/devicetree/bindings/infiniband/hisilicon-hns-roce.txt
 F:	drivers/infiniband/hw/hns/
 
 HISILICON SAS Controller
-M:	Yihang Li <liyihang9@h-partners.com>
+M:	Xingui Yang <yangxingui@huawei.com>
 S:	Supported
 W:	http://www.hisilicon.com
 F:	Documentation/devicetree/bindings/scsi/hisilicon-sas.txt
@@ -11837,7 +11868,7 @@ W:	http://www.hisilicon.com
 F:	drivers/spi/spi-hisi-sfc-v3xx.c
 
 HISILICON ZIP Controller DRIVER
-M:	Yang Shen <shenyang39@huawei.com>
+M:	Chenghai Huang<huangchenghai2@huawei.com>
 M:	Zhou Wang <wangzhou1@hisilicon.com>
 L:	linux-crypto@vger.kernel.org
 S:	Maintained
@@ -12664,12 +12695,15 @@ S:	Orphan
 F:	drivers/media/rc/img-ir/
 
 IMGTEC POWERVR DRM DRIVER
-M:	Frank Binns <frank.binns@imgtec.com>
-M:	Matt Coster <matt.coster@imgtec.com>
+M:	Alessio Belle <alessio.belle@imgtec.com>
+M:	Luigi Santivetti <luigi.santivetti@imgtec.com>
+L:	imagination@lists.freedesktop.org
 S:	Supported
+Q:	https://patchwork.freedesktop.org/project/imagination/list/
+B:	https://gitlab.freedesktop.org/imagination/linux/-/issues
+C:	irc://irc.oftc.net/powervr
 T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
-F:	Documentation/devicetree/bindings/gpu/img,powervr-rogue.yaml
-F:	Documentation/devicetree/bindings/gpu/img,powervr-sgx.yaml
+F:	Documentation/devicetree/bindings/gpu/img,powervr-*.yaml
 F:	Documentation/gpu/imagination/
 F:	drivers/gpu/drm/ci/xfails/powervr*
 F:	drivers/gpu/drm/imagination/
@@ -13135,9 +13169,7 @@ F:	drivers/iommu/intel/
 INTEL IPU3 CSI-2 CIO2 DRIVER
 M:	Yong Zhi <yong.zhi@intel.com>
 M:	Sakari Ailus <sakari.ailus@linux.intel.com>
-M:	Bingbu Cao <bingbu.cao@intel.com>
 M:	Dan Scally <dan.scally@ideasonboard.com>
-R:	Tianshu Qiu <tian.shu.qiu@intel.com>
 L:	linux-media@vger.kernel.org
 S:	Maintained
 T:	git git://linuxtv.org/media.git
@@ -13146,8 +13178,6 @@ F:	drivers/media/pci/intel/ipu3/
 
 INTEL IPU3 CSI-2 IMGU DRIVER
 M:	Sakari Ailus <sakari.ailus@linux.intel.com>
-R:	Bingbu Cao <bingbu.cao@intel.com>
-R:	Tianshu Qiu <tian.shu.qiu@intel.com>
 L:	linux-media@vger.kernel.org
 S:	Maintained
 F:	Documentation/admin-guide/media/ipu3.rst
@@ -13157,8 +13187,6 @@ F:	drivers/staging/media/ipu3/
 
 INTEL IPU6 INPUT SYSTEM DRIVER
 M:	Sakari Ailus <sakari.ailus@linux.intel.com>
-M:	Bingbu Cao <bingbu.cao@intel.com>
-R:	Tianshu Qiu <tian.shu.qiu@intel.com>
 L:	linux-media@vger.kernel.org
 S:	Maintained
 T:	git git://linuxtv.org/media.git
@@ -13167,7 +13195,6 @@ F:	drivers/media/pci/intel/ipu6/
 
 INTEL IPU7 INPUT SYSTEM DRIVER
 M:	Sakari Ailus <sakari.ailus@linux.intel.com>
-R:	Bingbu Cao <bingbu.cao@intel.com>
 L:	linux-media@vger.kernel.org
 S:	Maintained
 T:	git git://linuxtv.org/media.git
@@ -13202,7 +13229,6 @@ F:	drivers/crypto/intel/keembay/ocs-aes.h
 
 INTEL KEEM BAY OCS ECC CRYPTO DRIVER
 M:	Prabhjot Khurana <prabhjot.khurana@intel.com>
-M:	Mark Gross <mgross@linux.intel.com>
 S:	Maintained
 F:	Documentation/devicetree/bindings/crypto/intel,keembay-ocs-ecc.yaml
 F:	drivers/crypto/intel/keembay/Kconfig
@@ -13229,6 +13255,17 @@ F:	drivers/spi/spi-ljca.c
 F:	drivers/usb/misc/usb-ljca.c
 F:	include/linux/usb/ljca.h
 
+INTEL LPSS (Low Power SubSystem) DRIVERS
+R:	Andy Shevchenko <andriy.shevchenko@linux.intel.com>
+S:	Supported
+F:	drivers/dma/idma64*
+F:	drivers/i2c/busses/i2c-designware-*
+F:	drivers/mfd/intel-lpss*
+F:	drivers/pwm/pwm-lpss*
+F:	drivers/tty/serial/8250/8250_dw.c
+F:	drivers/tty/serial/8250/8250_dwlib.*
+F:	drivers/spi/spi-pxa2xx*
+
 INTEL MANAGEMENT ENGINE (mei)
 M:	Alexander Usyskin <alexander.usyskin@intel.com>
 L:	linux-kernel@vger.kernel.org
@@ -13379,6 +13416,7 @@ INTEL SKYLAKE INT3472 ACPI DEVICE DRIVER
 M:	Daniel Scally <dan.scally@ideasonboard.com>
 M:	Sakari Ailus <sakari.ailus@linux.intel.com>
 S:	Maintained
+T:	git git://linuxtv.org/media.git
 F:	drivers/platform/x86/intel/int3472/
 F:	include/linux/platform_data/x86/int3472.h
 
@@ -13469,7 +13507,6 @@ F:	drivers/net/wireless/intel/iwlwifi/
 
 INTEL VISION SENSING CONTROLLER DRIVER
 M:	Sakari Ailus <sakari.ailus@linux.intel.com>
-R:	Bingbu Cao <bingbu.cao@intel.com>
 R:	Lixu Zhang <lixu.zhang@intel.com>
 L:	linux-media@vger.kernel.org
 S:	Maintained
@@ -14191,6 +14228,7 @@ F:	virt/kvm/*
 KERNEL VIRTUAL MACHINE FOR ARM64 (KVM/arm64)
 M:	Marc Zyngier <maz@kernel.org>
 M:	Oliver Upton <oupton@kernel.org>
+R:	Fuad Tabba <tabba@google.com>
 R:	Joey Gouly <joey.gouly@arm.com>
 R:	Steffen Eiden <seiden@linux.ibm.com>
 R:	Suzuki K Poulose <suzuki.poulose@arm.com>
@@ -14334,6 +14372,7 @@ S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git
 F:	Documentation/admin-guide/mm/kho.rst
 F:	Documentation/core-api/kho/*
+F:	include/asm-generic/kexec_handover.h
 F:	include/linux/kexec_handover.h
 F:	include/linux/kho/
 F:	include/linux/kho_block.h
@@ -16554,6 +16593,18 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/mmc/mtk-sd.yaml
 F:	drivers/mmc/host/mtk-sd.c
 
+MEDIATEK MT6323 PMIC AUXADC DRIVER
+M:	Roman Vivchar <rva333@protonmail.com>
+L:	linux-iio@vger.kernel.org
+S:	Maintained
+F:	drivers/iio/adc/mt6323-auxadc.c
+F:	include/dt-bindings/iio/adc/mediatek,mt6323-auxadc.h
+
+MEDIATEK MT6323 PMIC NVMEM DRIVER
+M:	Roman Vivchar <rva333@protonmail.com>
+S:	Maintained
+F:	drivers/nvmem/mt6323-efuse.c
+
 MEDIATEK MT6735 CLOCK & RESET DRIVERS
 M:	Yassine Oudjana <y.oudjana@protonmail.com>
 L:	linux-clk@vger.kernel.org
@@ -16877,24 +16928,13 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git fixes
 F:	Documentation/core-api/boot-time-mm.rst
 F:	include/linux/kho/abi/memblock.h
 F:	include/linux/memblock.h
-F:	mm/bootmem_info.c
 F:	mm/memblock.c
 F:	mm/memtest.c
 F:	mm/mm_init.c
+F:	mm/mm_init.h
 F:	mm/rodata_test.c
 F:	tools/testing/memblock/
 
-MEMORY ALLOCATION PROFILING
-M:	Suren Baghdasaryan <surenb@google.com>
-M:	Kent Overstreet <kent.overstreet@linux.dev>
-R:	Hao Ge <hao.ge@linux.dev>
-L:	linux-mm@kvack.org
-S:	Maintained
-F:	Documentation/mm/allocation-profiling.rst
-F:	include/linux/alloc_tag.h
-F:	include/linux/pgalloc_tag.h
-F:	lib/alloc_tag.c
-
 MEMORY CONTROLLER DRIVERS
 M:	Krzysztof Kozlowski <krzk@kernel.org>
 L:	linux-kernel@vger.kernel.org
@@ -16939,6 +16979,18 @@ T:	quilt git://git.kernel.org/pub/scm/linux/kernel/git/akpm/25-new
 F:	mm/
 F:	tools/mm/
 
+MEMORY MANAGEMENT - ALLOCATION PROFILING (ALLOC TAG)
+M:	Suren Baghdasaryan <surenb@google.com>
+R:	Hao Ge <hao.ge@linux.dev>
+L:	linux-mm@kvack.org
+S:	Maintained
+F:	Documentation/mm/allocation-profiling.rst
+F:	include/linux/alloc_tag.h
+F:	include/linux/pgalloc_tag.h
+F:	include/uapi/linux/alloc_tag.h
+F:	mm/alloc_tag.c
+F:	tools/testing/selftests/alloc_tag/
+
 MEMORY MANAGEMENT - BALLOON
 M:	Andrew Morton <akpm@linux-foundation.org>
 M:	David Hildenbrand <david@kernel.org>
@@ -16998,6 +17050,7 @@ F:	mm/pgtable-generic.c
 F:	mm/ptdump.c
 F:	mm/sparse-vmemmap.c
 F:	mm/sparse.c
+F:	mm/sparse.h
 F:	mm/util.c
 F:	mm/vmpressure.c
 F:	mm/vmstat.c
@@ -17086,7 +17139,7 @@ F:	Documentation/admin-guide/mm/multigen_lru.rst
 F:	Documentation/mm/multigen_lru.rst
 F:	include/linux/mm_inline.h
 F:	include/linux/mmzone.h
-F:	mm/swap.c
+F:	mm/folio.c
 F:	mm/vmscan.c
 F:	mm/workingset.c
 
@@ -17171,6 +17224,7 @@ F:	mm/debug_page_alloc.c
 F:	mm/debug_page_ref.c
 F:	mm/fail_page_alloc.c
 F:	mm/page_alloc.c
+F:	mm/page_alloc.h
 F:	mm/page_ext.c
 F:	mm/page_frag_cache.c
 F:	mm/page_isolation.c
@@ -17208,6 +17262,7 @@ R:	Lance Yang <lance.yang@linux.dev>
 L:	linux-mm@kvack.org
 S:	Maintained
 F:	include/linux/rmap.h
+F:	mm/interval_tree.c
 F:	mm/page_vma_mapped.c
 F:	mm/rmap.c
 F:	tools/testing/selftests/mm/rmap.c
@@ -17237,7 +17292,6 @@ F:	include/linux/swap.h
 F:	include/linux/swapfile.h
 F:	include/linux/swapops.h
 F:	mm/page_io.c
-F:	mm/swap.c
 F:	mm/swap.h
 F:	mm/swap_table.h
 F:	mm/swap_state.c
@@ -17255,6 +17309,7 @@ R:	Ryan Roberts <ryan.roberts@arm.com>
 R:	Dev Jain <dev.jain@arm.com>
 R:	Barry Song <baohua@kernel.org>
 R:	Lance Yang <lance.yang@linux.dev>
+R:	Usama Arif <usama.arif@linux.dev>
 L:	linux-mm@kvack.org
 S:	Maintained
 W:	http://www.linux-mm.org
@@ -17313,7 +17368,6 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
 F:	include/trace/events/mmap.h
 F:	fs/proc/task_mmu.c
 F:	fs/proc/task_nommu.c
-F:	mm/interval_tree.c
 F:	mm/mincore.c
 F:	mm/mlock.c
 F:	mm/mmap.c
@@ -17463,6 +17517,7 @@ F:	arch/arm64/boot/dts/marvell/armada-3720-uDPU.*
 
 MHI BUS
 M:	Manivannan Sadhasivam <mani@kernel.org>
+R:	Jeff Hugo <jeff.hugo@oss.qualcomm.com>
 L:	mhi@lists.linux.dev
 L:	linux-arm-msm@vger.kernel.org
 S:	Maintained
@@ -19280,7 +19335,7 @@ F:	drivers/ntb/hw/intel/
 NTFS FILESYSTEM
 M:	Namjae Jeon <linkinjeon@kernel.org>
 M:	Hyunchul Lee <hyc.lee@gmail.com>
-L:	linux-fsdevel@vger.kernel.org
+L:	ntfs@lists.linux.dev
 S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/ntfs.git
 F:	Documentation/filesystems/ntfs.rst
@@ -19903,7 +19958,8 @@ F:	Documentation/devicetree/bindings/media/i2c/ovti,os05b10.yaml
 F:	drivers/media/i2c/os05b10.c
 
 OMNIVISION OV01A10 SENSOR DRIVER
-M:	Bingbu Cao <bingbu.cao@intel.com>
+M:	Bingbu Cao <bingbu.cao@amd.com>
+M:	Sakari Ailus <sakari.ailus@linux.intel.com>
 L:	linux-media@vger.kernel.org
 S:	Maintained
 T:	git git://linuxtv.org/media.git
@@ -19996,9 +20052,8 @@ F:	Documentation/devicetree/bindings/media/i2c/ovti,ov2735.yaml
 F:	drivers/media/i2c/ov2735.c
 
 OMNIVISION OV2740 SENSOR DRIVER
-M:	Tianshu Qiu <tian.shu.qiu@intel.com>
-R:	Sakari Ailus <sakari.ailus@linux.intel.com>
-R:	Bingbu Cao <bingbu.cao@intel.com>
+M:	Sakari Ailus <sakari.ailus@linux.intel.com>
+R:	Bingbu Cao <bingbu.cao@amd.com>
 L:	linux-media@vger.kernel.org
 S:	Maintained
 T:	git git://linuxtv.org/media.git
@@ -20140,8 +20195,8 @@ F:	Documentation/devicetree/bindings/media/i2c/ovti,ov9650.txt
 F:	drivers/media/i2c/ov9650.c
 
 OMNIVISION OV9734 SENSOR DRIVER
-M:	Tianshu Qiu <tian.shu.qiu@intel.com>
-R:	Bingbu Cao <bingbu.cao@intel.com>
+M:	Sakari Ailus <sakari.ailus@linux.intel.com>
+R:	Bingbu Cao <bingbu.cao@amd.com>
 L:	linux-media@vger.kernel.org
 S:	Maintained
 T:	git git://linuxtv.org/media.git
@@ -20195,7 +20250,7 @@ W:	http://www.onsemi.com
 F:	drivers/net/phy/ncn*
 
 OP-TEE DRIVER
-M:	Jens Wiklander <jens.wiklander@linaro.org>
+M:	Jens Wiklander <jenswi@kernel.org>
 L:	op-tee@lists.trustedfirmware.org (moderated for non-subscribers)
 S:	Maintained
 F:	Documentation/ABI/testing/sysfs-bus-optee-devices
@@ -20687,7 +20742,6 @@ F:	include/linux/switchtec.h
 F:	include/uapi/linux/switchtec_ioctl.h
 
 PCI DRIVER FOR MOBIVEIL PCIE IP
-M:	Karthikeyan Mitran <m.karthikeyan@mobiveil.co.in>
 M:	Hou Zhiqiang <Zhiqiang.Hou@nxp.com>
 L:	linux-pci@vger.kernel.org
 S:	Supported
@@ -21012,6 +21066,7 @@ M:	Manivannan Sadhasivam <mani@kernel.org>
 L:	linux-pci@vger.kernel.org
 L:	linux-arm-msm@vger.kernel.org
 S:	Maintained
+F:	Documentation/devicetree/bindings/pci/qcom,*.yaml
 F:	drivers/pci/controller/dwc/pcie-qcom-common.c
 F:	drivers/pci/controller/dwc/pcie-qcom.c
 
@@ -22039,6 +22094,13 @@ F:	Documentation/networking/device_drivers/ethernet/freescale/dpaa2/overview.rst
 F:	drivers/bus/fsl-mc/
 F:	include/uapi/linux/fsl_mc.h
 
+QST QMC5883L 3-Axis Magnetic Sensor
+M:	Siratul Islam <siratul.islam@linux.dev>
+L:	linux-iio@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/iio/magnetometer/qstcorp,qmc5883l.yaml
+F:	drivers/iio/magnetometer/qmc5883l.c
+
 QT1010 MEDIA DRIVER
 L:	linux-media@vger.kernel.org
 S:	Orphan
@@ -22189,7 +22251,7 @@ F:	drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c
 
 QUALCOMM FASTRPC DRIVER
 M:	Srinivas Kandagatla <srini@kernel.org>
-M:	Amol Maheshwari <amahesh@qti.qualcomm.com>
+R:	Ekansh Gupta <ekansh.gupta@oss.qualcomm.com>
 L:	linux-arm-msm@vger.kernel.org
 L:	dri-devel@lists.freedesktop.org
 S:	Maintained
@@ -22316,6 +22378,15 @@ F:	Documentation/devicetree/bindings/media/*qcom*
 F:	drivers/media/platform/qcom
 F:	include/dt-bindings/media/*qcom*
 
+QUALCOMM PAS TZ SERVICE
+M:	Sumit Garg <sumit.garg@oss.qualcomm.com>
+L:	linux-arm-msm@vger.kernel.org
+S:	Maintained
+F:	drivers/firmware/qcom/qcom_pas.c
+F:	drivers/firmware/qcom/qcom_pas.h
+F:	drivers/firmware/qcom/qcom_pas_tee.c
+F:	include/linux/firmware/qcom/qcom_pas.h
+
 QUALCOMM SMB CHARGER DRIVER
 M:	Casey Connolly <casey.connolly@linaro.org>
 L:	linux-arm-msm@vger.kernel.org
@@ -22343,6 +22414,13 @@ L:	linux-arm-msm@vger.kernel.org
 S:	Maintained
 F:	drivers/firmware/qcom/qcom_qseecom_uefisecapp.c
 
+QUALCOMM PINCTRL DRIVERS
+M:	Bartosz Golaszewski <brgl@kernel.org>
+L:	linux-arm-msm@vger.kernel.org
+S:	Supported
+F:	Documentation/devicetree/bindings/pinctrl/qcom,*.yaml
+F:	drivers/pinctrl/qcom/
+
 QUALCOMM RMNET DRIVER
 M:	Subash Abhinov Kasiviswanathan <subash.a.kasiviswanathan@oss.qualcomm.com>
 M:	Sean Tranchetti <sean.tranchetti@oss.qualcomm.com>
@@ -23091,10 +23169,12 @@ L:	linux-renesas-soc@vger.kernel.org
 S:	Maintained
 F:	drivers/phy/renesas/phy-rcar-gen3-usb*.c
 
-RENESAS VERSACLOCK 7 CLOCK DRIVER
-M:	Alex Helms <alexander.helms.jy@renesas.com>
+RENESAS VERSACLOCK 3 and VERSACLOCK 7 CLOCK DRIVER
+M:	Biju Das <biju.das.jz@bp.renesas.com>
 S:	Maintained
+F:	Documentation/devicetree/bindings/clock/renesas,5p35023.yaml
 F:	Documentation/devicetree/bindings/clock/renesas,versaclock7.yaml
+F:	drivers/clk/clk-versaclock3.c
 F:	drivers/clk/clk-versaclock7.c
 
 RENESAS X9250 DIGITAL POTENTIOMETERS DRIVER
@@ -23257,6 +23337,7 @@ F:	drivers/char/hw_random/mpfs-rng.c
 F:	drivers/clk/microchip/clk-mpfs*.c
 F:	drivers/firmware/microchip/mpfs-auto-update.c
 F:	drivers/gpio/gpio-mpfs.c
+F:	drivers/hwmon/tvs-mpfs.c
 F:	drivers/i2c/busses/i2c-microchip-corei2c.c
 F:	drivers/mailbox/mailbox-mpfs.c
 F:	drivers/pci/controller/plda/pcie-microchip-host.c
@@ -23313,7 +23394,7 @@ L:	spacemit@lists.linux.dev
 S:	Maintained
 W:	https://github.com/spacemit-com/linux/wiki
 C:	irc://irc.libera.chat/spacemit
-T:	git https://github.com/spacemit-com/linux
+T:	https://git.kernel.org/pub/scm/linux/kernel/git/spacemit/linux.git
 F:	arch/riscv/boot/dts/spacemit/
 N:	spacemit
 K:	spacemit
@@ -23572,7 +23653,7 @@ F:	Documentation/devicetree/bindings/media/allwinner,sun8i-a83t-de2-rotate.yaml
 F:	drivers/media/platform/sunxi/sun8i-rotate/
 
 RPMB SUBSYSTEM
-M:	Jens Wiklander <jens.wiklander@linaro.org>
+M:	Jens Wiklander <jenswi@kernel.org>
 L:	linux-kernel@vger.kernel.org
 S:	Supported
 F:	drivers/misc/rpmb-core.c
@@ -23791,6 +23872,7 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git
 F:	Documentation/driver-api/s390-drivers.rst
 F:	Documentation/arch/s390/
 F:	arch/s390/
+F:	drivers/cpuidle/cpuidle-s390.c
 F:	drivers/s390/
 F:	drivers/watchdog/diag288_wdt.c
 
@@ -23994,6 +24076,13 @@ F:	arch/arm64/boot/dts/exynos/exynos850*
 F:	drivers/clk/samsung/clk-exynos850.c
 F:	include/dt-bindings/clock/exynos850.h
 
+SAMSUNG EXYNOS8855 SoC SUPPORT
+M:	Alim Akhtar <alim.akhtar@samsung.com>
+L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
+L:	linux-samsung-soc@vger.kernel.org
+S:	Maintained
+F:	arch/arm64/boot/dts/exynos/exynos8855*
+
 SAMSUNG EXYNOS ACPM MAILBOX PROTOCOL
 M:	Tudor Ambarus <tudor.ambarus@linaro.org>
 L:	linux-kernel@vger.kernel.org
@@ -24528,6 +24617,7 @@ F:	drivers/iio/chemical/sunrise_co2.c
 
 SENSIRION SCD30 CARBON DIOXIDE SENSOR DRIVER
 M:	Maxwell Doose <m32285159@gmail.com>
+M:	Tomasz Duszynski <tduszyns@gmail.com>
 S:	Maintained
 F:	Documentation/devicetree/bindings/iio/chemical/sensirion,scd30.yaml
 F:	drivers/iio/chemical/scd30.h
@@ -25102,10 +25192,12 @@ R:	Andy Shevchenko <andriy.shevchenko@linux.intel.com>
 R:	Daniel Scally <djrscally@gmail.com>
 R:	Heikki Krogerus <heikki.krogerus@linux.intel.com>
 R:	Sakari Ailus <sakari.ailus@linux.intel.com>
+R:	Bartosz Golaszewski <brgl@kernel.org>
 L:	linux-acpi@vger.kernel.org
 S:	Maintained
 F:	drivers/base/property.c
 F:	drivers/base/swnode.c
+F:	drivers/base/test/swnode-devlink-test.c
 F:	include/linux/fwnode.h
 F:	include/linux/property.h
 
@@ -25218,7 +25310,8 @@ F:	Documentation/devicetree/bindings/media/i2c/sony,imx296.yaml
 F:	drivers/media/i2c/imx296.c
 
 SONY IMX319 SENSOR DRIVER
-M:	Bingbu Cao <bingbu.cao@intel.com>
+M:	Bingbu Cao <bingbu.cao@amd.com>
+M:	Sakari Ailus <sakari.ailus@linux.intel.com>
 L:	linux-media@vger.kernel.org
 S:	Maintained
 T:	git git://linuxtv.org/media.git
@@ -25240,7 +25333,9 @@ F:	Documentation/devicetree/bindings/media/i2c/sony,imx335.yaml
 F:	drivers/media/i2c/imx335.c
 
 SONY IMX355 SENSOR DRIVER
-M:	Tianshu Qiu <tian.shu.qiu@intel.com>
+M:	Sakari Ailus <sakari.ailus@linux.intel.com>
+M:	Dave Stevenson <dave.stevenson@raspberrypi.com>
+R:	David Heidelberg <david@ixit.cz>
 L:	linux-media@vger.kernel.org
 S:	Maintained
 T:	git git://linuxtv.org/media.git
@@ -25261,6 +25356,19 @@ T:	git git://linuxtv.org/media.git
 F:	Documentation/devicetree/bindings/media/i2c/sony,imx415.yaml
 F:	drivers/media/i2c/imx415.c
 
+SONY IMX471 SENSOR DRIVER
+M:	Kate Hsuan <hpa@redhat.com>
+L:	linux-media@vger.kernel.org
+S:	Maintained
+F:	drivers/media/i2c/imx471.c
+
+SONY IMX678 SENSOR DRIVER
+M:	Jai Luthra <jai.luthra@ideasonboard.com>
+L:	linux-media@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/media/i2c/sony,imx678.yaml
+F:	drivers/media/i2c/imx678.c
+
 SONY MEMORYSTICK SUBSYSTEM
 M:	Maxim Levitsky <maximlevitsky@gmail.com>
 M:	Alex Dubov <oakad@yahoo.com>
@@ -26526,7 +26634,7 @@ F:	drivers/media/i2c/tw9910.c
 F:	include/media/i2c/tw9910.h
 
 TEE SUBSYSTEM
-M:	Jens Wiklander <jens.wiklander@linaro.org>
+M:	Jens Wiklander <jenswi@kernel.org>
 R:	Sumit Garg <sumit.garg@kernel.org>
 L:	op-tee@lists.trustedfirmware.org (moderated for non-subscribers)
 S:	Maintained
@@ -26645,8 +26753,7 @@ S:	Maintained
 F:	drivers/net/ethernet/tehuti/tn40*
 
 TELECOM CLOCK DRIVER FOR MCPL0010
-M:	Mark Gross <markgross@kernel.org>
-S:	Supported
+S:	Orphan
 F:	drivers/char/tlclk.c
 
 TEMPO SEMICONDUCTOR DRIVERS
@@ -26958,6 +27065,13 @@ Q:	http://patchwork.linuxtv.org/project/linux-media/list/
 T:	git git://linuxtv.org/mhadli/v4l-dvb-davinci_devices.git
 F:	drivers/media/platform/ti/am437x/
 
+TI AM654 SDHCI DRIVER
+M:	Judith Mendez <jm@ti.com>
+L:	linux-mmc@vger.kernel.org
+S:	Supported
+F:	Documentation/devicetree/bindings/mmc/sdhci-am654.yaml
+F:	drivers/mmc/host/sdhci_am654.c
+
 TI BANDGAP AND THERMAL DRIVER
 M:	Eduardo Valentin <edubezval@gmail.com>
 M:	Keerthy <j-keerthy@ti.com>
@@ -27201,6 +27315,7 @@ TIMEKEEPING, CLOCKSOURCE CORE, NTP, ALARMTIMER
 M:	John Stultz <jstultz@google.com>
 M:	Thomas Gleixner <tglx@kernel.org>
 R:	Stephen Boyd <sboyd@kernel.org>
+R:	Miroslav Lichvar <mlichvar@redhat.com>
 L:	linux-kernel@vger.kernel.org
 S:	Supported
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git timers/core
@@ -28813,6 +28928,7 @@ W:	http://www.linux-mm.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
 F:	include/linux/vmalloc.h
 F:	mm/vmalloc.c
+F:	mm/vmalloc.h
 F:	lib/test_vmalloc.c
 
 VME SUBSYSTEM
@@ -29508,7 +29624,7 @@ F:	include/uapi/linux/dqblk_xfs.h
 F:	include/uapi/linux/fsmap.h
 
 XILINX AMS DRIVER
-M:	Salih Erim <salih.erim@amd.com>
+M:	Sai Krishna Potthuri <sai.krishna.potthuri@amd.com>
 M:	Conall O'Griofa <conall.ogriofa@amd.com>
 L:	linux-iio@vger.kernel.org
 S:	Maintained
@@ -29596,17 +29712,27 @@ F:	include/uapi/linux/xilinx-v4l2-controls.h
 XILINX VERSAL EDAC DRIVER
 M:	Shubhrajyoti Datta <shubhrajyoti.datta@amd.com>
 M:	Sai Krishna Potthuri <sai.krishna.potthuri@amd.com>
+R:	Radhey Shyam Pandey <radhey.shyam.pandey@amd.com>
 S:	Maintained
 F:	Documentation/devicetree/bindings/memory-controllers/xlnx,versal-ddrmc-edac.yaml
 F:	drivers/edac/versal_edac.c
 
 XILINX VERSALNET EDAC DRIVER
 M:	Shubhrajyoti Datta <shubhrajyoti.datta@amd.com>
+R:	Radhey Shyam Pandey <radhey.shyam.pandey@amd.com>
 S:	Maintained
 F:	Documentation/devicetree/bindings/memory-controllers/xlnx,versal-net-ddrmc5.yaml
 F:	drivers/edac/versalnet_edac.c
 F:	include/linux/cdx/edac_cdx_pcol.h
 
+XILINX VERSAL SYSMON DRIVER
+M:	Salih Erim <salih.erim@amd.com>
+M:	Sai Krishna Potthuri <sai.krishna.potthuri@amd.com>
+L:	linux-iio@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/iio/adc/xlnx,versal-sysmon.yaml
+F:	drivers/iio/adc/versal-sysmon*
+
 XILINX WATCHDOG DRIVER
 M:	Srinivas Neeli <srinivas.neeli@amd.com>
 R:	Shubhrajyoti Datta <shubhrajyoti.datta@amd.com>
@@ -29639,6 +29765,7 @@ F:	include/dt-bindings/dma/xlnx-zynqmp-dpdma.h
 XILINX ZYNQMP OCM EDAC DRIVER
 M:	Shubhrajyoti Datta <shubhrajyoti.datta@amd.com>
 M:	Sai Krishna Potthuri <sai.krishna.potthuri@amd.com>
+R:	Radhey Shyam Pandey <radhey.shyam.pandey@amd.com>
 S:	Maintained
 F:	Documentation/devicetree/bindings/memory-controllers/xlnx,zynqmp-ocmc-1.0.yaml
 F:	drivers/edac/zynqmp_edac.c
@@ -29782,8 +29909,11 @@ M:	Minchan Kim <minchan@kernel.org>
 M:	Sergey Senozhatsky <senozhatsky@chromium.org>
 L:	linux-kernel@vger.kernel.org
 S:	Maintained
+F:	Documentation/ABI/testing/sysfs-block-zram
+F:	Documentation/ABI/testing/sysfs-class-zram
 F:	Documentation/admin-guide/blockdev/zram.rst
 F:	drivers/block/zram/
+F:	tools/testing/selftests/zram/
 
 ZS DECSTATION Z85C30 SERIAL DRIVER
 M:	"Maciej W. Rozycki" <macro@orcam.me.uk>
diff --git a/Makefile b/Makefile
index bfb47ad8cb9a7..ddac7a3d20db7 100644
--- a/Makefile
+++ b/Makefile
@@ -475,6 +475,10 @@ KBUILD_USERLDFLAGS := $(USERLDFLAGS)
 
 # These flags apply to all Rust code in the tree, including the kernel and
 # host programs.
+#
+# `-Aclippy::unwrap_or_default`: the lint is buggy [1] and ignores our
+# MSRV. It can trigger depending on the optimization level.
+# [1] https://github.com/rust-lang/rust-clippy/issues/17379
 export rust_common_flags := --edition=2021 \
 			    -Zbinary_dep_depinfo=y \
 			    -Astable_features \
@@ -503,6 +507,7 @@ export rust_common_flags := --edition=2021 \
 			    -Aclippy::uninlined_format_args \
 			    -Wclippy::unnecessary_safety_comment \
 			    -Wclippy::unnecessary_safety_doc \
+			    -Aclippy::unwrap_or_default \
 			    -Wrustdoc::missing_crate_level_docs \
 			    -Wrustdoc::unescaped_backticks
 
@@ -1222,6 +1227,12 @@ KBUILD_RUSTFLAGS += $(KRUSTFLAGS)
 KBUILD_LDFLAGS_MODULE += --build-id=sha1
 LDFLAGS_vmlinux += --build-id=sha1
 
+# Specific code, such as outlined KASAN checks, may be placed in
+# COMDAT-deduplicated sections. Use --force-group-allocation to resolve these
+# groups when linking modules. The option is available from ld.bfd 2.29 and
+# ld.lld 19.1.0.
+KBUILD_LDFLAGS_MODULE += $(call ld-option,--force-group-allocation)
+
 KBUILD_LDFLAGS	+= -z noexecstack
 ifeq ($(CONFIG_LD_IS_BFD),y)
 KBUILD_LDFLAGS	+= $(call ld-option,--no-warn-rwx-segments)
@@ -1552,6 +1563,22 @@ prepare: tools/bpf/resolve_btfids
 endif
 endif
 
+# tools/bootconfig renders the embedded bootconfig into a cmdline at build time.
+ifdef CONFIG_CMDLINE_FROM_BOOTCONFIG
+prepare: tools/bootconfig
+endif
+
+# tools/bootconfig is run on the build host during prepare, so force a host
+# binary here; its own Makefile keeps $(CC) for standalone and cross builds.
+# CROSS_COMPILE= is cleared so tools/scripts/Makefile.include does not inject
+# the target's --target=/--sysroot= flags into the host clang invocation under
+# LLVM=1 cross builds (which would produce a target binary that fails to exec).
+tools/bootconfig: export CC := $(HOSTCC)
+tools/bootconfig: FORCE
+	$(Q)mkdir -p $(objtree)/tools
+	$(Q)$(MAKE) O=$(abspath $(objtree)) subdir=tools -C $(srctree)/tools/ \
+		bootconfig CROSS_COMPILE=
+
 # The tools build system is not a part of Kbuild and tends to introduce
 # its own unique issues. If you need to integrate a new tool into Kbuild,
 # please consider locating that tool outside the tools/ tree and using the
@@ -1578,6 +1605,15 @@ ifneq ($(wildcard $(objtool_O)),)
 	$(Q)$(MAKE) -sC $(abs_srctree)/tools/objtool O=$(objtool_O) srctree=$(abs_srctree) $(patsubst objtool_%,%,$@)
 endif
 
+PHONY += bootconfig_clean
+
+bootconfig_O = $(abspath $(objtree))/tools/bootconfig
+
+bootconfig_clean:
+ifneq ($(wildcard $(bootconfig_O)),)
+	$(Q)$(MAKE) -sC $(srctree)/tools/bootconfig O=$(bootconfig_O) clean
+endif
+
 tools/: FORCE
 	$(Q)mkdir -p $(objtree)/tools
 	$(Q)$(MAKE) O=$(abspath $(objtree)) subdir=tools -C $(srctree)/tools/
@@ -1753,7 +1789,7 @@ vmlinuxclean:
 	$(Q)$(CONFIG_SHELL) $(srctree)/scripts/link-vmlinux.sh clean
 	$(Q)$(if $(ARCH_POSTLINK), $(MAKE) -f $(ARCH_POSTLINK) clean)
 
-clean: archclean vmlinuxclean resolve_btfids_clean objtool_clean
+clean: archclean vmlinuxclean resolve_btfids_clean objtool_clean bootconfig_clean
 
 # mrproper - Delete all generated files, including .config
 #
diff --git a/Next/SHA1s b/Next/SHA1s
new file mode 100644
index 0000000000000..7342af145389e
--- /dev/null
+++ b/Next/SHA1s
@@ -0,0 +1,430 @@
+Name		SHA1
+----		----
+origin		fce2dfa773ced15f27dd27cd0b482a7473cdcf2a
+ext4-fixes	981fcc5674e67158d24d23e841523eccba19d0e7
+vfs-brauner-fixes	24dddc384fb9aec2d7eea5463ca6dac98a3b3854
+fscrypt-current	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+fsverity-current	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+btrfs-fixes	6c893b948351d42cfc3761cc746ab5b3d03ee7f3
+vfs-fixes	49c5d168a3a8f4eb27d44a2a22b7e8a856ca601f
+erofs-fixes	1572282de6d3377ca8605d48c50df3a8c08468e9
+nfsd-fixes	92ea163c773cb4d0d5eaf103ed80c49d6758b96f
+v9fs-fixes	028ef9c96e96197026887c0f092424679298aae8
+overlayfs-fixes	4549871118cf616eecdd2d939f78e3b9e1dddc48
+fscrypt		2d0afaac9137e95e504bd3ad3512a9044745536b
+btrfs		2e35e224edd823bf477b5ea05deeaeb3bdbe0081
+ceph		1397040896280b7a6a6213e874ee4f6146bc9cd9
+cifs		fce2dfa773ced15f27dd27cd0b482a7473cdcf2a
+configfs	6363844fdbbb76afe1d44d678fe0746390204a5f
+ecryptfs	95ce5ffd54cf66098f91892f98606c3bd33846fe
+dlm		e61113cfcaf71cbdf4c17e3d086d8fb7f92c62bf
+erofs		dc59e4fea9d83f03bad6bddf3fa2e52491777482
+exfat		13a315afaeb67e38f1ec76deeac19fb5f9e4b402
+ext3		04267552014fe275493c5347913456a8a09470b5
+ext4		c143957520c6c9b5cd72e0de8b52b814f0c576fe
+f2fs		6429f3bf6dc33cba42055260f905559c43976eeb
+fsverity	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+fuse		7d87a5a284bb34edb3f4e7e312ef403b3385a7b7
+gfs2		5f80d9113360c08111ae7471f662f3f89f23ce32
+jfs		dad98c5b2a05ef744af4c884c97066a3c8cdad61
+ksmbd		39ccf3cdc6790ee29b4c5411dfff94ebdf5dbafc
+nfs		a13c140cc289c0b7b3770bce5b3ad42ab35074aa
+nfs-anna	27934d02cbeb8a957dd11c985a579e58d30c5270
+nfsd		c83cc1e5e8d2b1a57015946f53e4fe6af10cf227
+ntfs		9e3ca21f5d941783c871b4fbad02ba7662654950
+ntfs3		349a7959ffd7a2592830ead72e098d0e909df814
+orangefs	e61bc5e4d87433c8759e7dc92bb640ef71a8970c
+overlayfs	1f6ee9be92f8df85a8c9a5a78c20fd39c0c21a95
+ubifs		11efa98bcc0d00271e8f66c7c940c284f973f39d
+v9fs		aa88278693cbfaf7a2acf961379973fbb63b165c
+v9fs-ericvh	028ef9c96e96197026887c0f092424679298aae8
+xfs		59c462b0f5cfa107794228051724b34ae9334168
+zonefs		3a8389d42bdf4213730f4067f8bfa78bae6564ef
+vfs-brauner	b148f6ef8f3b0306ccff2644529a65ff19a11f03
+vfs		4dda01b67c8662c5d0c53034974cb8280c575a5d
+mm-hotfixes	3566356c5b2ea34cf7551c59e97a03d02bb93624
+fs-current	7f6f78211933ee1c47d0c236bc0d2d8b2a590691
+kbuild-current	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+clang-fixes	175db11786bde9061db526bf1ac5107d915f5163
+arc-current	76f38ad1b39cf0321f7ecb7ea37b46f61a0f3d75
+arm-current	009b6c6486b94a3aff566b017256b598dc96bf18
+arm64-fixes	f797f51a185ffdc1e3f915afed55f308b376842f
+arm-soc-fixes	add11ea55f2d26d798c058fe4d67fb2941e77523
+davinci-current	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+realtek-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+drivers-memory-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+sophgo-fixes	19272b37aa4f83ca52bdf9c16d5d81bdd1354494
+sophgo-soc-fixes	0af2f6be1b4281385b618cb86ad946eded089ac8
+m68k-current	dc5200f6b1ada318463dd141b041ec9f044b2bf5
+powerpc-fixes	a2c02aa0c6ca3ec9fab6f1c99912a440c7b8bfdb
+s390-fixes	49145bce539117db4b6e9e83c0e5ef528e361050
+net		2c72eb6286347d05a885412fb076993bd5286b53
+bpf		3d84d674e3a5a33ce5eaa14b92bc839c3821a348
+ipsec		21697720ff43b8dfa25b8e8d9ca7f56f4597fc80
+netfilter	2c7c88a412aa6d09cd04b414211b4ef8553b5309
+ipvs		2c7c88a412aa6d09cd04b414211b4ef8553b5309
+wireless	a0d82fb8505326cbc53dc9a0c08f97d11197bb30
+ath		a1a21995c2e1cc2ca6b2226cfe4f5f018370182a
+iwlwifi		a426d3227c669ef0ad9855b311a68129f10ac8bf
+wpan		8ce4f287524c74a118b0af1eebd4b24a8efca57a
+rdma-fixes	3cda0dfe8c651dcbb9e38977905d3d3b1750c4ab
+sound-current	ac1e8d01f4ea0d4a5739c6f40b07484c6385cfbb
+sound-asoc-fixes	0b604e886ece11b71c4daaeccc512c784b89b014
+regmap-fixes	63d441e937b7c5bfadb878203ef5045e29cfff39
+regulator-fixes	d38f8bd771c4999b797d7074b348cf201414bd34
+spi-fixes	e919ca35a6e84b5adf085da9ffa1544c01d4ce1e
+pci-current	34db32102439d948e0c9aea060e48f979aae827d
+driver-core.current	667d0fb32149f023b8b34a1f6f3d384556eafb5a
+tty.current	af071d9e07e57cfff239e8d09d2f3b05ebc9c667
+usb.current	71132cedd1ecbc4032d76e9928c18a10f7e39b80
+usb-serial-fixes	fad0fd120e29041b3e6cdf41bb12e3184fb524a2
+phy		dc59e4fea9d83f03bad6bddf3fa2e52491777482
+staging.current	a13c140cc289c0b7b3770bce5b3ad42ab35074aa
+iio-fixes	a41000ba3a230bed1e422f283486ff8f77fe0d30
+watchdog-fixes	bcfcd7619f277842430d197556463b401b839ee9
+counter-current	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+char-misc.current	a13c140cc289c0b7b3770bce5b3ad42ab35074aa
+soundwire-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+thunderbolt-fixes	db79679595326fd3f6bd1e6fd0cefc3ba016039a
+input-current	adea84ee6cdea611146c4251d3c1616f5a09feca
+crypto-current	8173f7e2ce67e6ca1d4763f3da14e5b01ce77456
+libcrypto-fixes	e073f1238ecaea366f53e98724c40b31856da56a
+vfio-fixes	e242e974e812e7a47e3088860c80d9492fac314f
+kselftest-fixes	c5c413534d40eb4c982f1794e62813f926aba747
+dmaengine-fixes	867621ba203027338b525af6729719c544135336
+backlight-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+mtd-fixes	2b533e775aec580cf60074417f4ca00ac9cf3580
+mfd-fixes	d5d2d7a8d8be18681a0864f58e3875f1c639e11c
+v4l-dvb-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+reset-fixes	71827776667f4e4677a4fa806bcfb24d4b8dd9d7
+mips-fixes	a13c140cc289c0b7b3770bce5b3ad42ab35074aa
+at91-fixes	765aaba18413a66f6c8fe8416336ca9b3dd98a79
+omap-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+kvm-fixes	25f744ffa0c8e799e06250ce2e618367b166b0d4
+kvms390-fixes	3e3aa6da87d30a0064a17b836685cd43c90a3572
+kvm-arm-fixes	8d187d4b33c262c0f3e44842553521151d8629e8
+hwmon-fixes	f151d0143ac4e086f92f52328ebdbdc50933d8ef
+nvdimm-fixes	a8aec14230322ed8f1e8042b6d656c1631d41163
+cxl-fixes	d90f236f8b9e354848bd226f581db27755ab901d
+dma-mapping-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+drivers-x86-fixes	78bf392ba77dd8b2a25656e489449d2f91cfd1eb
+samsung-krzk-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+pinctrl-samsung-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+pinctrl-qcom-fixes	437a8d2aa1aa442c4a176fdf4700a9b3bb0c8794
+devicetree-fixes	b39a6b2e9d5bd6a3153aed4c7440172b8f6a739e
+dt-krzk-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+scsi-fixes	e166bafc483e927150cb9b5f286c9191ea0df84e
+drm-fixes	5d1238c7ac366e2d7d02a665d92867310449ac94
+drm-intel-fixes	612978b83f45bf7018815209db5395d759db6f26
+mmc-fixes	e27c946b589c53520409a0956b33d52ef7a0898f
+rtc-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+gnss-fixes	a13c140cc289c0b7b3770bce5b3ad42ab35074aa
+hyperv-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+risc-v-fixes	5caae1deee89a6582c761d5dcd4b924b744426cc
+riscv-dt-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+riscv-soc-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+fpga-fixes	19272b37aa4f83ca52bdf9c16d5d81bdd1354494
+spdx		dc59e4fea9d83f03bad6bddf3fa2e52491777482
+gpio-brgl-fixes	9dc325327babe7f159e84cbe9380a45342da0585
+gpio-intel-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+pinctrl-intel-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+auxdisplay-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+kunit-fixes	5142c56651578abc346d6c17f3fb919b9ffbb317
+memblock-fixes	2ebce860bdd7ae5e13002811bc9bbbf33fcfc221
+renesas-fixes	6fa6ee724d8dadf392139e242ac936b5da730c4b
+perf-current	558ef39aeb9a089a6be9dda8413b0b9d42e843ea
+efi-fixes	d8809f6931065cbbf3554647a50a65a471ab5983
+battery-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+iommufd-fixes	c3b8ee84a965058b41275069d4696f37a8b14bf6
+rust-fixes	a19bda861b3a79e25417462539df8b0d77c6b322
+w1-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+pmdomain-fixes	a4876f11aa1d076802676e23f8af500706e780e3
+i2c-andi-fixes	deb35336b5bfed5db9231b5348bc1514db930797
+i2c-rust-fixes	4eb422482ca5d924d7212ad2ca1cb7ea6f5b524d
+sparc-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+clk-fixes	8cdeaa50eae8dad34885515f62559ee83e7e8dda
+thead-clk-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+tenstorrent-clk-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+fustini-config-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+pwrseq-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+thead-dt-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+ftrace-fixes	1650a1b6cb1ae6cb99bb4fce21b30ebdf9fc238e
+ring-buffer-fixes	057caace5214da3b457bbd295e1a2ad34d3685ea
+trace-fixes	601ddaceb861be7eb557278109966320a6f3478c
+tracefs-fixes	07004a8c4b572171934390148ee48c4175c77eed
+spacemit-fixes	eb525edd48907795c0d4e498ff57ad168070b289
+tip-fixes	3d2378ed6c8ffdae7d5f852eb9559b09266f78a3
+slab-fixes	d9e6a7623938968e3752b67e37eaff097e559a54
+kexec-fixes	ada2e5a44e99113e08ad9b7b71396c6c572204da
+liveupdate-fixes	0b27b476044118b39e9869a9c34f525320026b52
+drm-msm-fixes	e2332abed2a4d3caa59052095dc16e4ce44791ea
+uml-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+fwctl-fixes	4549871118cf616eecdd2d939f78e3b9e1dddc48
+devsec-tsm-fixes	c3fd16c3b98ed726294feab2f94f876290bf7b61
+drm-rust-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+tenstorrent-dt-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+nfc-fixes	e87faad807329d1348595dbcea3444acd7bb0ca6
+drm-misc-fixes	faebb7ba1ac65fa5810b640df02ce04e509fdc11
+rust		7059bdf4f04a3e14f4fafb3ac35fdca913e3e21a
+rust-interop	05f7e89ab9731565d8a62e3b5d1ec206485eeb0b
+rust-alloc	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+rust-io		86731a2a651e58953fc949573895f2fa6d456841
+rust-pin-init	0c20f77a26b89bc911d31cd79f1abe1a7ae57f60
+rust-timekeeping	ddb1444d3335129ae87d9796ab1debf41c0ee51b
+rust-xarray	c455f19bbe6104debd980bb15515faf716bd81b8
+rust-analyzer	5f45afb8ab04d934fc0601a202f95267ebc20059
+mm-stable	0e35b9b6ec0ffcc5e23cbdec09f5c622ad532b53
+mm-nonmm-stable	0e35b9b6ec0ffcc5e23cbdec09f5c622ad532b53
+mm-unstable	e5492213654050379e78ec6f9acfd6c9fe00f334
+mm-nonmm-unstable	8ce79abc8a1252636f6f7ac0e89fa3bca309db34
+kbuild		4859c0d5918e28f6d4844e33022e791c55dbf373
+clang-format	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+perf		3f7909fa921e38b7aba8566b03ca5e0754a8fa8b
+compiler-attributes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+dma-mapping	0188616c057d3eba8d8a4af62a59b5e7f2362571
+asm-generic	adbbd9714f8058730f93c8df5c5bf1679456424b
+alpha		d58041d2c63e09a1c9083e0e9f4151e487c4e16a
+arm		75ca78c7d982a0b7c658722fa9eb2ce70e180307
+arm64		a13c140cc289c0b7b3770bce5b3ad42ab35074aa
+arm-perf	5936245125f78d896fdb1bbc2ae79213e28a6579
+arm-soc		900d72f1052c1028d6e8c1f182529a403ba50a72
+amlogic		4336e970ec6890fbd424128c352564a9c1dd514a
+asahi-soc	297f9bd201933d2414976c069722b68b95476fcf
+at91		6d892f8d5cd9f20cfef5e9d59b0cde53c30daec6
+bmc		40ae2e16214adcd59481a8d799af03be7b0f5e08
+broadcom	f705b0c80bab155e9ffc6b796408ab16c9abf206
+cix		a0cffbd8878c55b12dc4555f883adf3372cecd91
+davinci		dc59e4fea9d83f03bad6bddf3fa2e52491777482
+drivers-memory	8111c7f3723f414b7f655a3880775cbbd0e9de8e
+fsl		3300a285c28529ca1a178f01b4cc046fd3547dbd
+imx-mxs		d089adef10f380cbdf65cbb87a59a9aab86cb0d2
+mediatek	b1ccec7cca55821b3eed9b471bcea7da97247bad
+mvebu		3ea0dabfa4500104c5c6eebecfa4d23aeb847611
+omap		ba251bb5ce2db9228a4c1ca9aef787c368121af7
+qcom		64a28053710d050b72f8f8aaa9e75654fac0f2d5
+realtek		3c778f0c9fa36b6b9a26bd2146df24ed4e830ba4
+renesas		f9592c355314cb76e76078dfc58ea747e382ea15
+reset		d373605cd514837d8a6de3d00c786d4bae6dbaf8
+rockchip	70d6721bd79c8e38064967cab3dc744e77d3cd05
+samsung-krzk	4a7924bcff476f8eab2960432c6fd2b8597fb17c
+scmi		f6ce093c482b9cf18e3ff25f9e0268c6eda4a955
+sophgo		f7337210bede62fc7c6230ef58013dddf7e0a921
+sophgo-soc	c8754c7deab4cbfa947fa2d656cbaf83771828ef
+spacemit	ab092591e4d9d6f51282d3815372350b861e5744
+stm32		fba4a31a7f3b6b29b01c83180f83e7ed4c398738
+sunxi		36078904c9908a8ad674ef5b3d31cf8d90828e6a
+tee		a3067938fd192b116b6dfb325b654300a6d1b461
+tegra		f98a7877ab4a8b76a61b781d97966281c108571b
+tenstorrent-dt	33583baeb1ba7d328e6a9775d889036900b74cdb
+fustini-config	fd52109054693750f38d6fae134cffd1ac8c9c74
+thead-dt	3a5791956edbfa84d7256224167941931cbc46e7
+ti		1b98f483adfd99bf7008dc334ae72fc608d47a77
+xilinx		50a8e96a1513760418e7f2e5bc51a7e48fc2b82e
+socfpga		39ea7cb4019245ce488ea547b3ec68a3d7d57aef
+clk		8cdeaa50eae8dad34885515f62559ee83e7e8dda
+clk-imx		b3dcc8c608fbb6352bd94932ba935f2078c9090d
+clk-renesas	2df26b8663b6f315d0e9364927c8503280fe3dd3
+thead-clk	b2ee00d0bf4cca5dd02702fc339d2b523191966f
+tenstorrent-clk	23c8ebc952849b3ba47d04d0ec95daf5cc136061
+csky		abb81e5ce7d995baa41556b8125fa59e28ba3be8
+loongarch	8227805754a68791baadc90280d1610353c4b698
+m68k		dc5200f6b1ada318463dd141b041ec9f044b2bf5
+m68knommu	d13e97dad566448c38b82ffd6807f61346b7536e
+microblaze	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+mips		27c61f8565a438b2d1dea6f6a47af0e6dc12c112
+openrisc	aca063c9024522e4e5b9a9d1927433f6a01785a3
+parisc-hd	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+powerpc		8cdeaa50eae8dad34885515f62559ee83e7e8dda
+risc-v		1d8304bdd65f78a061e1f57be596c2776faf083a
+riscv-dt	82ab962ef6c2a3571e3d590b30c023118c92b776
+riscv-soc	8cdeaa50eae8dad34885515f62559ee83e7e8dda
+s390		8b313d717bcf4f944fb46fe4ed4d8680bdbc5648
+sh		b0aa5e4b087b686575f1b31ce54048b4d059b7b8
+sparc		5b2a3b1a98fb47c593144c2770e012d463952b70
+uml		254f49634ee16a731174d2ae34bc50bd5f45e731
+xtensa		a7c958e8721eb2724ee2e2ef13129bc67e1c8a75
+fs-next		070f41f072722437b7edda13037a400459630a5e
+printk		703a9d76a45dd97b19b2432fc1ba4beb4d67a94f
+pci		41d034327abe154679c927a9efd8744af6f290a1
+pstore		24b8f8dcb9a139a36cf48bfbe935e8dc1f33ed79
+hid		194a48576843858afcd9c00ed4f105ec2b398e6c
+i2c		8cd9520d35a6c38db6567e97dd93b1f11f185dc6
+i2c-andi	c844ed7266c9d9d2663f34cfa32d4f66ea3a9119
+i2c-rust	c0260c740d20751c8498865a9c5dc18b69c3d267
+i3c		eeedacd0866eec6e9dad2f082a5b828b895a5ed0
+dmi		1afafbaf749d8e8ec53f8e38efdc731131902b5b
+hwmon-staging	ca078d004cf58137bcf8cb24a8b271397431ba58
+jc_docs		be828e6819ed139b699bf7feb00a546cb7806266
+v4l-dvb		a52e6f7923c17a672135b485ffd96fbd72f46267
+v4l-dvb-next	adc218676eef25575469234709c2d87185ca223a
+pm		ca2b8f92568b7c34b3a12371c32a2df3702cf55a
+cpufreq-arm	6a9e0e0f7592313ace66303cf5eca68e04c10f30
+cpupower	5100bd356cd315112c4e27e66e4f0129800eabd2
+devfreq		c096be11c2a9d4acca8e75f1f8edeb744aceb521
+pmdomain	8247e08d2196cf2c8ce1c87c0ada726bce831a14
+opp		ce46fede7792cedd247e34b48bc8a02eb90c7848
+thermal		968098b4ca5219b0d2e0a981aed1dacfbd5adc69
+rdma		e952e079d226209a132110acc1de96a099701292
+net-next	ce6b4d3216b63f902bb8e9695ee6c10c83415f65
+bpf-next	9a3a07d06e7d74f4aecc51396c771149336ac55d
+ipsec-next	355f808d8a11fa69b19dfd8811bc87d97830f5d6
+mlx5-next	bee40a7d0bd1263934f99054db037cdd4a33fd86
+netfilter-next	f6f3b36c15ed44de1fbb44e645e4fae8c4a4453e
+ipvs-next	f6f3b36c15ed44de1fbb44e645e4fae8c4a4453e
+bluetooth	bd8bee79e1fa8db2d587733f8b6fd9597b04d6e3
+wireless-next	ac798f757d6475dc6fee2ec899980d6740714596
+ath-next	f78703024c71ac60033139b42d040981efe083bb
+iwlwifi-next	905f57aefde4f4092a411c8a55856182fb1c7598
+wpan-next	a6bfdfcc6711d1d5a92e98644359dedc67c0c858
+wpan-staging	a6bfdfcc6711d1d5a92e98644359dedc67c0c858
+mtd		d276783e490d73536135f90f96c5875f263481ab
+nand		adfc275b317c02cd043b0cf28b8cfb7459b041f0
+spi-nor		df415c5e1de0f1aeefacb4e6252ff98d38c04437
+crypto		df373d39c6f038d176af303ae72f02c7c70b953d
+libcrypto	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+drm		a1c81cb05273077219fa809424330fdaa4346d5b
+drm-exynos	3a8660878839faadb4f1a6dd72c3179c1df56787
+drm-misc	6143359440d41d4c571435c9cd2fbaf9f8dc2788
+amdgpu		8e1f8214cbba2ccda3f3c625df0a67dcfd9b1356
+drm-intel	67e955e073c6ff9167828444913a864a14275463
+drm-msm		e2332abed2a4d3caa59052095dc16e4ce44791ea
+drm-msm-lumag	8de061ca149c94871e41d91a5b2e442ecc26a572
+drm-xe		53a7115f9862f30ee748c1e1c5e6398ffa092672
+drm-rust	ce2d97f714c8a37cac506a41e52da45b09555b81
+drm-nova	93296e9d9528f0d87f2cf3fee494599060a0f14a
+etnaviv		6bde14ba5f7ef59e103ac317df6cc5ac4291ff4a
+fbdev		dc59e4fea9d83f03bad6bddf3fa2e52491777482
+regmap		8979c222ebebb34066a2cf390c7746f69d4ebea7
+sound		4392569d95e6a794780fc33ce168744c67cb162e
+ieee1394	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+sound-asoc	d721340cd338e2f6307c26fe664cfe9e967f78d5
+modules		2600d43b235b029d0ba4f2d3cc71e2137114fde3
+input		3ed4ba919eb35860ee1867e2851ee3e6f9fe37f5
+block		1a5c200b36e4fb7275fd5f51dc52c0502c235e36
+device-mapper	22eb919f1b7900e2af0e952b1f1bee522f612b4a
+libata		1393f2c98fb42495269d14b65b1f68b047d65fb7
+pcmcia		b3c26ea81ccc522e77ed0b1707add61fc9206216
+mmc		5608386899d9de8030980bce3d05bded28f9bfb8
+mfd		6384e5c5e0bcfbc9a915d70cb3925e0022d7b337
+backlight	4b3f5baf67dea91452b5b114dcb19139feba71dd
+battery		a888754e51e915731c8974c4d6d62709facb35d3
+regulator	e370243ea76704d90e3f79ecef9041425f3edac1
+security	0cee720cfd51402cfcb14d96cb326a36c13b823a
+apparmor	c431011ab65fd4ef205e3ce8cfc4b031c67768d9
+integrity	1da739feb31b4fecae465ebf87ba44a97e44101b
+selinux		f47f45c7f1ca2fe66a3f7277acf0738ad715aec3
+smack		a7c44fd9f80e37763acf9cd3c87a58058d206427
+tomoyo		58717b2a1365d06c8c64b72aa948541b53fe31eb
+tpmdd-tpm	4ece5759c1a29fd711b7b1d1a3b81078547697fd
+tpmdd-keys	51cb1aa1250c36269474b8b6ca6b6319e170f5a5
+watchdog	94794a448352339042085d98f82f33941c63d0d8
+iommu		750356af88d6178156ce776df17c10a9035fcc9c
+audit		1715cd1303c703e73115e2f7bac7e01b9ca84049
+devicetree	bcef80df6d6f4d616ee0bfbfe8e9acd44517a15e
+dt-krzk		dc59e4fea9d83f03bad6bddf3fa2e52491777482
+mailbox		36cac4b5101f8ecbc851356df175b99543c84ec6
+spi		97a74f7b0a6a08e3279b9868280cb0b7d2c82e1e
+tip		b94f4faa5968ae9eff8bec1af0d53c4ea2a058ba
+kexec		af78cec42d1b0113993e427139f0ab2f34b047ee
+liveupdate	fc1b1fca6c3097b94d15e7dbe786f9095e0487fb
+clockevents	a9ac745bc320cbdc2ed3c851eb78f91f22ff975b
+edac		5adf9141e58007e7939349c215f92717ad39f01d
+ftrace		030bb1d8bb0773053df6518ed635f27005fd5e43
+rcu		e853c1b28580ea93fda3cd729e440a3fc16fa647
+paulmck		c2d939be90724b234a55e633ba0aff60f78f352f
+kvm		a204badd8432f93b7e862e7dac6db0fe3d65f370
+kvm-arm		cb9360ae6549120975f20887840fb33d47c13898
+kvms390		babe08404e1993697a523e60bc0f9d096ffe1ef8
+kvm-ppc		8cdeaa50eae8dad34885515f62559ee83e7e8dda
+kvm-riscv	8d9c9b135b5c23de9811a8426257cbd2fa024a99
+kvm-x86		6bc96b971766fbbbbdd9fb2642cedacaf02da957
+xen-tip		fcd245ea7528d50fddffc0fd1308941a9180f5b3
+percpu		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+workqueues	c5a3a03816ddba2ebf73fd7cbec3c5169424ea04
+sched-ext	1ed374f61d7ce64a8e35def02e223304cd4a52cb
+drivers-x86	2b3a5dabe89e330413af403246b648c1890f368f
+chrome-platform	9a3f43b30373c61477d0d3ab52946c05f9492bf9
+chrome-platform-firmware	dd06eb60e86a1ab6d3bd4f641100f47b6f7342b6
+hsi		254f49634ee16a731174d2ae34bc50bd5f45e731
+leds-lj		96fa4d69958cf90740d756d1e45cdb02cface41b
+ipmi		6d920a75df9a83ab096b3cde7a643b656e4fdfeb
+driver-core	2b74f1f2024486c7a81915528b6aa7f8f5daa021
+usb		b60af0b9e752aa77f29daac026dd2314cc2a0bb0
+thunderbolt	d49d71ebec8f03eee2514304279a05a4952fadf7
+usb-serial	091738d09786ae4da789b5297cb4dae3024d1c13
+tty		facd005eee65df088cd73733741fbcf8d989fc5e
+char-misc	775553d19b163446c38c5ff24dd0a01065376932
+coresight	98495b5a4d77dd22e106f462b76e1093a55b29a7
+fastrpc		17ec912dc87bf6b9aba3a85f6a94e99f999d6511
+fpga		b5ba63e247075087ab8a6a087622c762dc4172e9
+icc		e11872288b4b33c0e90da6987d6006bb49fecbc9
+iio		2e2f2de7532cbbc2269de8be20ec709606c6e79b
+nfc		ed85d4cbbfaa4e630c5aa0d607348b42620d976b
+phy-next	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+soundwire	6409292ab5f7d1a60a6f8948bd6dcd8633c36dfe
+extcon		254f49634ee16a731174d2ae34bc50bd5f45e731
+gnss		a13c140cc289c0b7b3770bce5b3ad42ab35074aa
+vfio		785562e31dbcd85ca583cf58c446e63aa8a5af08
+w1		169ae5e65e5aaf213b6a578f6478a9fd2e523606
+spmi		8cdeaa50eae8dad34885515f62559ee83e7e8dda
+staging		64276d9bfe4d1fdd5cf2636f1065f7ea55c2defb
+counter-next	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+mux		59b723cd2adbac2a34fc8e12c74ae26ae45bf230
+dmaengine	338c853b7c3b422fefda195b9b8010e40611c96f
+cgroup		6486f6d8f5455bc7fa898d5ee432f8c0f7f9ec20
+scsi		cf1af0ccca54d5547cc31bf476c6a07cf658ad29
+scsi-mkp	aec9f1f18ffdda04b4d68ddadbc3d893eb3a9cbb
+vhost		1f673033c02735bb38ee2f16f1c86f042f4de9d4
+rpmsg		f041a428881dc576d8f2cd5f1ae4dc671624369f
+gpio-brgl	fbd35c8cc6c6e817385f2bfa07fa4227370a3010
+gpio-intel	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+pinctrl		a7c4e636e400e3d919b078ece1a6101dd9ebfe8c
+pinctrl-intel	ed1608d5e3c4432c1dcba709668b90f44228c10c
+pinctrl-renesas	5a653cedec948423fc8a0180c90b8d05c2239d9d
+pinctrl-samsung	d441836c9d704d0f3db5dbf44df87af2ba7a9d26
+pinctrl-qcom	82c3edf8701154e2768973ebd4c0d2f3bd29621e
+pwm		7f1565db53367d302aa06b802a96205b9a0ce923
+ktest		932cdaf3e273a2727e77af97f79f12577174c5a0
+kselftest	20f01a5565ce79923b75a2ac8e8ef2a1d7adf517
+kunit		dc59e4fea9d83f03bad6bddf3fa2e52491777482
+kunit-next	34b5c0132952c3d176bf5a169c8f7093ecb8c4e8
+livepatching	0839c8963b7b28d25350bd5ea69bacde794124ab
+rtc		dc59e4fea9d83f03bad6bddf3fa2e52491777482
+nvdimm		6191eeb6c70b41f7bc71967055adab5ef93274dc
+at24		dc59e4fea9d83f03bad6bddf3fa2e52491777482
+ntb		dc59e4fea9d83f03bad6bddf3fa2e52491777482
+seccomp		41fa04327384148b0e2e828c9be9862c5240e9fa
+slimbus		c922423ce66bcee6c61d1b5aadea09b29ca81400
+nvmem		2a3ed29b053137a91af7963038d437657065e57e
+hyperv		a4ffc59238be84dd1c26bf1c001543e832674fc6
+auxdisplay	95545ddcb3cfb390c59429b1c18aaf187d6768ba
+kgdb		fdbdd0ccb30af18d3b29e714ac8d5ab6163279e0
+hmm		19272b37aa4f83ca52bdf9c16d5d81bdd1354494
+cfi		dc59e4fea9d83f03bad6bddf3fa2e52491777482
+mhi		34c66fc4c223ae693ce550068ab5f48f3e3fe604
+memblock	1a6e6a79b8df153f9ec702aa4d5407333eeff917
+cxl		5ca04f3ba91f1773bbd5da6d9c654ccc1ba7831d
+zstd		65d1f5507ed2c78c64fce40e44e5574a9419eb09
+efi		4953e0a8a149d68e3e02b2e7713513bffdb7a746
+unicode		bcfee135d584714c2130031c7e28aafa91057b9a
+slab		56aa9b819edce4acfa879728af56534c951d1818
+random		bb9ff576fdff48c242876f55098a3ee20a29df5d
+landlock	55f82176ef8dde632ea3eb94a6224950ed809d7c
+sysctl		dccde188b7fc11155f3a8d5c3aa19abc4954a0aa
+execve		9bf092c97b86af63694d9902b9e14047214ba76d
+bitmap		fc6926b0ead4b46fee932c7839f616ef8d2d5565
+hte		005b25ad11717139ca5c14c9f06c2a60855c2afd
+kspp		3421b9b056a6576d0ebac1030eafb48ad0544092
+nolibc		070cf03f5ec454eec9c42827ee9f3c0b3fad09f1
+iommufd		b737ad0b96a6ef19e1bdb680973f479386c11f7c
+turbostat	1c996a37fd244d19e5dbb715328c1676e28ef607
+pwrseq		162ea02941a936f8899f3dbe10607b1d5af1b07b
+capabilities-next	071588136007482d70fd2667b827036bc60b1f8f
+ipe		028ef9c96e96197026887c0f092424679298aae8
+kcsan		07a1a6562ce29e2e0c134a57882d6e52e8758492
+crc		dc59e4fea9d83f03bad6bddf3fa2e52491777482
+keys-next	965e9a2cf23b066d8bdeb690dff9cd7089c5f667
+fwctl		dc59e4fea9d83f03bad6bddf3fa2e52491777482
+devsec-tsm	3177779ae17db4c66c851f799505fb95c7530c03
+hisilicon	a5db65458a911daa6f8927264366dab501715dc0
+device-id	995832b2cebe6969d1b42635db698803ee31294d
diff --git a/Next/Trees b/Next/Trees
new file mode 100644
index 0000000000000..5cfae71682b54
--- /dev/null
+++ b/Next/Trees
@@ -0,0 +1,431 @@
+Trees included into this release:
+
+Name		Url
+----		---
+origin		https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git#master
+ext4-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git#fixes
+vfs-brauner-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git#vfs.fixes
+fscrypt-current	https://git.kernel.org/pub/scm/fs/fscrypt/linux.git#for-current
+fsverity-current	https://git.kernel.org/pub/scm/fs/fsverity/linux.git#for-current
+btrfs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git#next-fixes
+vfs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git#fixes
+erofs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git#fixes
+nfsd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux#nfsd-fixes
+v9fs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git#fixes/next
+overlayfs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git#ovl-fixes
+fscrypt		https://git.kernel.org/pub/scm/fs/fscrypt/linux.git#for-next
+btrfs		https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git#for-next
+ceph		https://github.com/ceph/ceph-client.git#master
+cifs		git://git.samba.org/sfrench/cifs-2.6.git#for-next
+configfs	https://git.kernel.org/pub/scm/linux/kernel/git/a.hindborg/linux.git#configfs-next
+ecryptfs	https://git.kernel.org/pub/scm/linux/kernel/git/tyhicks/ecryptfs.git#next
+dlm		https://git.kernel.org/pub/scm/linux/kernel/git/teigland/linux-dlm.git#next
+erofs		https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git#dev
+exfat		https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/exfat.git#dev
+ext3		https://git.kernel.org/pub/scm/linux/kernel/git/jack/linux-fs.git#for_next
+ext4		https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git#dev
+f2fs		https://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git#dev
+fsverity	https://git.kernel.org/pub/scm/fs/fsverity/linux.git#for-next
+fuse		https://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git#for-next
+gfs2		https://git.kernel.org/pub/scm/linux/kernel/git/gfs2/linux-gfs2.git#for-next
+jfs		https://github.com/kleikamp/linux-shaggy.git#jfs-next
+ksmbd		https://github.com/smfrench/smb3-kernel.git#ksmbd-for-next
+nfs		git://git.linux-nfs.org/projects/trondmy/nfs-2.6.git#linux-next
+nfs-anna	git://git.linux-nfs.org/projects/anna/linux-nfs.git#linux-next
+nfsd		https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux#nfsd-next
+ntfs		https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/ntfs.git#ntfs-next
+ntfs3		https://github.com/Paragon-Software-Group/linux-ntfs3.git#master
+orangefs	https://git.kernel.org/pub/scm/linux/kernel/git/hubcap/linux.git#for-next
+overlayfs	https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git#overlayfs-next
+ubifs		https://git.kernel.org/pub/scm/linux/kernel/git/rw/ubifs.git#next
+v9fs		https://github.com/martinetd/linux#9p-next
+v9fs-ericvh	https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git#ericvh/for-next
+xfs		https://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git#for-next
+zonefs		https://git.kernel.org/pub/scm/linux/kernel/git/dlemoal/zonefs.git#for-next
+vfs-brauner	https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git#vfs.all
+vfs		https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git#for-next
+mm-hotfixes	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-hotfixes-unstable
+kbuild-current	https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git#kbuild-fixes-for-next
+clang-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/nathan/linux.git#clang-fixes-for-next
+arc-current	https://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc.git#for-curr
+arm-current	https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git#fixes
+arm64-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux#for-next/fixes
+arm-soc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git#arm/fixes
+davinci-current	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#davinci/for-current
+realtek-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/yu_chun/linux.git#fixes
+drivers-memory-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git#fixes
+sophgo-fixes	https://github.com/sophgo/linux.git#fixes
+sophgo-soc-fixes	https://github.com/sophgo/linux.git#soc-fixes
+m68k-current	https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git#for-linus
+powerpc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#fixes
+s390-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git#fixes
+net		https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git#main
+bpf		https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git/#master
+ipsec		https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec.git#master
+netfilter	https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf.git#main
+ipvs		https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs.git#main
+wireless	https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless.git#for-next
+ath		https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git#for-current
+iwlwifi		https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git#fixes
+wpan		https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan.git#master
+rdma-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#for-rc
+sound-current	https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git#for-linus
+sound-asoc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git#for-linus
+regmap-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git#for-linus
+regulator-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git#for-linus
+spi-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git#for-linus
+pci-current	https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git#for-linus
+driver-core.current	https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git#driver-core-linus
+tty.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git#tty-linus
+usb.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git#usb-linus
+usb-serial-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git#usb-linus
+phy		https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git#fixes
+staging.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git#staging-linus
+iio-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git#fixes-togreg
+watchdog-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#watchdog
+counter-current	https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git#counter-current
+char-misc.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git#char-misc-linus
+soundwire-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git#fixes
+thunderbolt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git#fixes
+input-current	https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git#for-linus
+crypto-current	https://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git#master
+libcrypto-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git#libcrypto-fixes
+vfio-fixes	https://github.com/awilliam/linux-vfio.git#for-linus
+kselftest-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#fixes
+dmaengine-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git#fixes
+backlight-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git#for-backlight-fixes
+mtd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#mtd/fixes
+mfd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git#for-mfd-fixes
+v4l-dvb-fixes	git://linuxtv.org/media-ci/media-pending.git#fixes
+reset-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/pza/linux#reset/fixes
+mips-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git#mips-fixes
+at91-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git#at91-fixes
+omap-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git#fixes
+kvm-fixes	git://git.kernel.org/pub/scm/virt/kvm/kvm.git#master
+kvms390-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git#master
+kvm-arm-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git#fixes
+hwmon-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#hwmon
+nvdimm-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git#libnvdimm-fixes
+cxl-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git#fixes
+dma-mapping-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git#dma-mapping-fixes
+drivers-x86-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git#fixes
+samsung-krzk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git#fixes
+pinctrl-samsung-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git#fixes
+pinctrl-qcom-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#pinctrl-qcom/for-current
+devicetree-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git#dt/linus
+dt-krzk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git#fixes
+scsi-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git#fixes
+drm-fixes	https://gitlab.freedesktop.org/drm/kernel.git#drm-fixes
+drm-intel-fixes	https://gitlab.freedesktop.org/drm/i915/kernel.git#for-linux-next-fixes
+mmc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git#fixes
+rtc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git#rtc-fixes
+gnss-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git#gnss-linus
+hyperv-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git#hyperv-fixes
+risc-v-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git#fixes
+riscv-dt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-dt-fixes
+riscv-soc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-soc-fixes
+fpga-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git#fixes
+spdx		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/spdx.git#spdx-linus
+gpio-brgl-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#gpio/for-current
+gpio-intel-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git#fixes
+pinctrl-intel-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git#fixes
+auxdisplay-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git#fixes
+kunit-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#kunit-fixes
+memblock-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git#fixes
+renesas-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git#fixes
+perf-current	https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools.git#perf-tools
+efi-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git#urgent
+battery-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git#fixes
+iommufd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git#for-rc
+rust-fixes	https://github.com/Rust-for-Linux/linux.git#rust-fixes
+w1-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git#fixes
+pmdomain-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git#fixes
+i2c-andi-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git#i2c/i2c-fixes
+i2c-rust-fixes	https://github.com/ikrtn/rust-for-linux#rust-i2c-fixes
+sparc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git#for-linus
+clk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git#clk-fixes
+thead-clk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-clk-fixes
+tenstorrent-clk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git#tenstorrent-clk-fixes
+fustini-config-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#riscv-config-fixes
+pwrseq-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#pwrseq/for-current
+thead-dt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-dt-fixes
+ftrace-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#ftrace/fixes
+ring-buffer-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#ring-buffer/fixes
+trace-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#trace/fixes
+tracefs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#tracefs/fixes
+spacemit-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/spacemit/linux#fixes
+tip-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git#tip/urgent
+slab-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git#slab/for-next-fixes
+kexec-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git#kexec-fixes
+liveupdate-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git#fixes
+drm-msm-fixes	https://gitlab.freedesktop.org/drm/msm.git#msm-fixes
+uml-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git#fixes
+fwctl-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git#for-rc
+devsec-tsm-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git#fixes
+drm-rust-fixes	https://gitlab.freedesktop.org/drm/rust/kernel.git#for-linux-next-fixes
+tenstorrent-dt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git#tenstorrent-dt-fixes
+nfc-fixes	https://codeberg.org/linux-nfc/linux.git#for-linus
+drm-misc-fixes	https://gitlab.freedesktop.org/drm/misc/kernel.git#for-linux-next-fixes
+rust		https://github.com/Rust-for-Linux/linux.git#rust-next
+rust-interop	https://github.com/Rust-for-Linux/linux.git#interop-next
+rust-alloc	https://github.com/Rust-for-Linux/linux.git#alloc-next
+rust-io		https://github.com/Rust-for-Linux/linux.git#io-next
+rust-pin-init	https://github.com/Rust-for-Linux/linux.git#pin-init-next
+rust-timekeeping	https://github.com/Rust-for-Linux/linux.git#timekeeping-next
+rust-xarray	https://github.com/Rust-for-Linux/linux.git#xarray-next
+rust-analyzer	https://github.com/Rust-for-Linux/linux.git#rust-analyzer-next
+mm-stable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-stable
+mm-nonmm-stable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-nonmm-stable
+mm-unstable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-unstable
+mm-nonmm-unstable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-nonmm-unstable
+kbuild		https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git#kbuild-for-next
+clang-format	https://github.com/ojeda/linux.git#clang-format
+perf		https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git#perf-tools-next
+compiler-attributes	https://github.com/ojeda/linux.git#compiler-attributes
+dma-mapping	https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git#dma-mapping-for-next
+asm-generic	https://git.kernel.org/pub/scm/linux/kernel/git/arnd/asm-generic#master
+alpha		https://git.kernel.org/pub/scm/linux/kernel/git/mattst88/alpha.git#alpha-next
+arm		https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git#for-next
+arm64		https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux#for-next/core
+arm-perf	https://git.kernel.org/pub/scm/linux/kernel/git/will/linux.git#for-next/perf
+arm-soc		https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git#for-next
+amlogic		https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux.git#for-next
+asahi-soc	https://github.com/AsahiLinux/linux.git#asahi-soc/for-next
+at91		https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git#at91-next
+bmc		https://git.kernel.org/pub/scm/linux/kernel/git/bmc/linux.git#for-next
+broadcom	https://github.com/Broadcom/stblinux.git#next
+cix		https://git.kernel.org/pub/scm/linux/kernel/git/peter.chen/cix.git#for-next
+davinci		https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#davinci/for-next
+drivers-memory	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git#for-next
+fsl		https://git.kernel.org/pub/scm/linux/kernel/git/chleroy/linux.git#soc_fsl
+imx-mxs		https://git.kernel.org/pub/scm/linux/kernel/git/frank.li/linux.git#for-next
+mediatek	https://git.kernel.org/pub/scm/linux/kernel/git/mediatek/linux.git#for-next
+mvebu		https://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu.git#for-next
+omap		https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git#for-next
+qcom		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git#for-next
+realtek		https://git.kernel.org/pub/scm/linux/kernel/git/yu_chun/linux.git#for-next
+renesas		https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git#next
+reset		https://git.kernel.org/pub/scm/linux/kernel/git/pza/linux#reset/next
+rockchip	https://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip.git#for-next
+samsung-krzk	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git#for-next
+scmi		https://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux.git#for-linux-next
+sophgo		https://github.com/sophgo/linux.git#for-next
+sophgo-soc	https://github.com/sophgo/linux.git#soc-for-next
+spacemit	https://git.kernel.org/pub/scm/linux/kernel/git/spacemit/linux#for-next
+stm32		https://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32.git#stm32-next
+sunxi		https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux.git#sunxi/for-next
+tee		https://git.kernel.org/pub/scm/linux/kernel/git/jenswi/linux-tee.git#next
+tegra		https://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux.git#for-next
+tenstorrent-dt	https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git#tenstorrent-dt-for-next
+fustini-config	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#riscv-config-for-next
+thead-dt	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-dt-for-next
+ti		https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux.git#ti-next
+xilinx		https://github.com/Xilinx/linux-xlnx.git#for-next
+socfpga		https://git.kernel.org/pub/scm/linux/kernel/git/dinguyen/linux.git#for-next
+clk		https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git#clk-next
+clk-imx		https://git.kernel.org/pub/scm/linux/kernel/git/abelvesa/linux.git#for-next
+clk-renesas	https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git#renesas-clk
+thead-clk	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-clk-for-next
+tenstorrent-clk	https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git#tenstorrent-clk-for-next
+csky		https://github.com/c-sky/csky-linux.git#linux-next
+loongarch	https://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson.git#loongarch-next
+m68k		https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git#for-next
+m68knommu	https://git.kernel.org/pub/scm/linux/kernel/git/gerg/m68knommu.git#for-next
+microblaze	git://git.monstr.eu/linux-2.6-microblaze.git#next
+mips		https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git#mips-next
+openrisc	https://github.com/openrisc/linux.git#for-next
+parisc-hd	https://git.kernel.org/pub/scm/linux/kernel/git/deller/parisc-linux.git#for-next
+powerpc		https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#next
+risc-v		https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git#for-next
+riscv-dt	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-dt-for-next
+riscv-soc	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-soc-for-next
+s390		https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git#for-next
+sh		https://git.kernel.org/pub/scm/linux/kernel/git/glaubitz/sh-linux.git#for-next
+sparc		https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git#for-next
+uml		https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git#next
+xtensa		https://github.com/jcmvbkbc/linux-xtensa.git#xtensa-for-next
+printk		https://git.kernel.org/pub/scm/linux/kernel/git/printk/linux.git#for-next
+pci		https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git#next
+pstore		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/pstore
+hid		https://git.kernel.org/pub/scm/linux/kernel/git/hid/hid.git#for-next
+i2c		https://git.kernel.org/pub/scm/linux/kernel/git/wsa/linux.git#i2c/for-next
+i2c-andi	https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git#i2c/i2c-next
+i2c-rust	https://github.com/ikrtn/rust-for-linux#rust-i2c-next
+i3c		https://git.kernel.org/pub/scm/linux/kernel/git/i3c/linux.git#i3c/next
+dmi		https://git.kernel.org/pub/scm/linux/kernel/git/jdelvare/staging.git#dmi-for-next
+hwmon-staging	https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#hwmon-next
+jc_docs		git://git.lwn.net/linux.git#docs-next
+v4l-dvb		git://linuxtv.org/media-ci/media-pending.git#next
+v4l-dvb-next	git://linuxtv.org/mchehab/media-next.git#master
+pm		https://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git#linux-next
+cpufreq-arm	https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git#cpufreq/arm/linux-next
+cpupower	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux.git#cpupower
+devfreq		https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/linux.git#devfreq-next
+pmdomain	https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git#next
+opp		https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git#opp/linux-next
+thermal		https://git.kernel.org/pub/scm/linux/kernel/git/thermal/linux.git#thermal/linux-next
+rdma		https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#for-next
+net-next	https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git#main
+bpf-next	https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git#for-next
+ipsec-next	https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec-next.git#master
+mlx5-next	https://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux.git#mlx5-next
+netfilter-next	https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf-next.git#main
+ipvs-next	https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs-next.git#main
+bluetooth	https://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next.git#master
+wireless-next	https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next.git#for-next
+ath-next	https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git#for-next
+iwlwifi-next	https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git#next
+wpan-next	https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git#master
+wpan-staging	https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git#staging
+mtd		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#mtd/next
+nand		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#nand/next
+spi-nor		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#spi-nor/next
+crypto		https://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6.git#master
+libcrypto	https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git#libcrypto-next
+drm		https://gitlab.freedesktop.org/drm/kernel.git#drm-next
+drm-exynos	https://git.kernel.org/pub/scm/linux/kernel/git/daeinki/drm-exynos.git#for-linux-next
+drm-misc	https://gitlab.freedesktop.org/drm/misc/kernel.git#for-linux-next
+amdgpu		https://gitlab.freedesktop.org/agd5f/linux.git#drm-next
+drm-intel	https://gitlab.freedesktop.org/drm/i915/kernel.git#for-linux-next
+drm-msm		https://gitlab.freedesktop.org/drm/msm.git#msm-next
+drm-msm-lumag	https://gitlab.freedesktop.org/lumag/msm.git#msm-next-lumag
+drm-xe		https://gitlab.freedesktop.org/drm/xe/kernel.git#drm-xe-next
+drm-rust	https://gitlab.freedesktop.org/drm/rust/kernel.git#for-linux-next
+drm-nova	https://gitlab.freedesktop.org/drm/nova.git#nova-next
+etnaviv		https://git.pengutronix.de/git/lst/linux#etnaviv/next
+fbdev		https://git.kernel.org/pub/scm/linux/kernel/git/deller/linux-fbdev.git#for-next
+regmap		https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git#for-next
+sound		https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git#for-next
+ieee1394	https://git.kernel.org/pub/scm/linux/kernel/git/ieee1394/linux1394.git#for-next
+sound-asoc	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git#for-next
+modules		https://git.kernel.org/pub/scm/linux/kernel/git/modules/linux.git#modules-next
+input		https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git#next
+block		https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git#for-next
+device-mapper	https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git#for-next
+libata		https://git.kernel.org/pub/scm/linux/kernel/git/libata/linux#for-next
+pcmcia		https://git.kernel.org/pub/scm/linux/kernel/git/brodo/linux.git#pcmcia-next
+mmc		https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git#next
+mfd		https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git#for-mfd-next
+backlight	https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git#for-backlight-next
+battery		https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git#for-next
+regulator	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git#for-next
+security	https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git#next
+apparmor	https://git.kernel.org/pub/scm/linux/kernel/git/jj/linux-apparmor#apparmor-next
+integrity	https://git.kernel.org/pub/scm/linux/kernel/git/zohar/linux-integrity#next-integrity
+selinux		https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/selinux.git#next
+smack		https://github.com/cschaufler/smack-next#next
+tomoyo		git://git.code.sf.net/p/tomoyo/tomoyo.git#master
+tpmdd-tpm	https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git#for-next-tpm
+tpmdd-keys	https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git#for-next-keys
+watchdog	https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#watchdog-next
+iommu		https://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git#next
+audit		https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git#next
+devicetree	https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git#for-next
+dt-krzk		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git#for-next
+mailbox		https://git.kernel.org/pub/scm/linux/kernel/git/jassibrar/mailbox.git#for-next
+spi		https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git#for-next
+tip		https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git#master
+kexec		https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git#kexec-next
+liveupdate	https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git#next
+clockevents	https://git.kernel.org/pub/scm/linux/kernel/git/daniel.lezcano/linux.git#timers/drivers/next
+edac		https://git.kernel.org/pub/scm/linux/kernel/git/ras/ras.git#edac-for-next
+ftrace		https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#for-next
+rcu		https://git.kernel.org/pub/scm/linux/kernel/git/rcu/linux#next
+paulmck		https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git#non-rcu/next
+kvm		git://git.kernel.org/pub/scm/virt/kvm/kvm.git#next
+kvm-arm		https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git#next
+kvms390		https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git#next
+kvm-ppc		https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#topic/ppc-kvm
+kvm-riscv	https://github.com/kvm-riscv/linux.git#riscv_kvm_next
+kvm-x86		https://github.com/kvm-x86/linux.git#next
+xen-tip		https://git.kernel.org/pub/scm/linux/kernel/git/xen/tip.git#linux-next
+percpu		https://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu.git#for-next
+workqueues	https://git.kernel.org/pub/scm/linux/kernel/git/tj/wq.git#for-next
+sched-ext	https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git#for-next
+drivers-x86	https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git#for-next
+chrome-platform	https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git#for-next
+chrome-platform-firmware	https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git#for-firmware-next
+hsi		https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-hsi.git#for-next
+leds-lj		https://git.kernel.org/pub/scm/linux/kernel/git/lee/leds.git#for-leds-next
+ipmi		https://github.com/cminyard/linux-ipmi.git#for-next
+driver-core	https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git#driver-core-next
+usb		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git#usb-next
+thunderbolt	https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git#next
+usb-serial	https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git#usb-next
+tty		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git#tty-next
+char-misc	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git#char-misc-next
+coresight	https://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git#next
+fastrpc		https://git.kernel.org/pub/scm/linux/kernel/git/srini/fastrpc.git#for-next
+fpga		https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git#for-next
+icc		https://git.kernel.org/pub/scm/linux/kernel/git/djakov/icc.git#icc-next
+iio		https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git#togreg
+nfc		https://codeberg.org/linux-nfc/linux.git#for-next
+phy-next	https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git#next
+soundwire	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git#next
+extcon		https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/extcon.git#extcon-next
+gnss		https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git#gnss-next
+vfio		https://github.com/awilliam/linux-vfio.git#next
+w1		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git#for-next
+spmi		https://git.kernel.org/pub/scm/linux/kernel/git/sboyd/spmi.git#spmi-next
+staging		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git#staging-next
+counter-next	https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git#counter-next
+mux		https://gitlab.com/peda-linux/mux.git#for-next
+dmaengine	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git#next
+cgroup		https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git#for-next
+scsi		https://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi.git#for-next
+scsi-mkp	https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git#for-next
+vhost		https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git#linux-next
+rpmsg		https://git.kernel.org/pub/scm/linux/kernel/git/remoteproc/linux.git#for-next
+gpio-brgl	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#gpio/for-next
+gpio-intel	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git#for-next
+pinctrl		https://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl.git#for-next
+pinctrl-intel	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git#for-next
+pinctrl-renesas	https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git#renesas-pinctrl
+pinctrl-samsung	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git#for-next
+pinctrl-qcom	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#pinctrl-qcom/for-next
+pwm		https://git.kernel.org/pub/scm/linux/kernel/git/ukleinek/linux.git#pwm/for-next
+ktest		https://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-ktest.git#for-next
+kselftest	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#next
+kunit		https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#test
+kunit-next	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#kunit
+livepatching	https://git.kernel.org/pub/scm/linux/kernel/git/livepatching/livepatching.git#for-next
+rtc		https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git#rtc-next
+nvdimm		https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git#libnvdimm-for-next
+at24		https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#at24/for-next
+ntb		https://github.com/jonmason/ntb.git#ntb-next
+seccomp		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/seccomp
+slimbus		https://git.kernel.org/pub/scm/linux/kernel/git/srini/slimbus.git#for-next
+nvmem		https://git.kernel.org/pub/scm/linux/kernel/git/srini/nvmem.git#for-next
+hyperv		https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git#hyperv-next
+auxdisplay	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git#for-next
+kgdb		https://git.kernel.org/pub/scm/linux/kernel/git/danielt/linux.git#kgdb/for-next
+hmm		https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#hmm
+cfi		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#cfi/next
+mhi		https://git.kernel.org/pub/scm/linux/kernel/git/mani/mhi.git#mhi-next
+memblock	https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git#for-next
+cxl		https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git#next
+zstd		https://github.com/terrelln/linux.git#zstd-next
+efi		https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git#next
+unicode		https://git.kernel.org/pub/scm/linux/kernel/git/krisman/unicode.git#for-next
+slab		https://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git#slab/for-next
+random		https://git.kernel.org/pub/scm/linux/kernel/git/crng/random.git#master
+landlock	https://git.kernel.org/pub/scm/linux/kernel/git/mic/linux.git#next
+sysctl		https://git.kernel.org/pub/scm/linux/kernel/git/sysctl/sysctl.git#sysctl-next
+execve		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/execve
+bitmap		https://github.com/norov/linux.git#bitmap-for-next
+hte		https://git.kernel.org/pub/scm/linux/kernel/git/pateldipen1984/linux.git#for-next
+kspp		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/kspp
+nolibc		https://git.kernel.org/pub/scm/linux/kernel/git/nolibc/linux-nolibc.git#for-next
+iommufd		https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git#for-next
+turbostat	https://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux.git#next
+pwrseq		https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#pwrseq/for-next
+capabilities-next	https://git.kernel.org/pub/scm/linux/kernel/git/sergeh/linux.git#caps-next
+ipe		https://git.kernel.org/pub/scm/linux/kernel/git/wufan/ipe.git#next
+kcsan		https://git.kernel.org/pub/scm/linux/kernel/git/melver/linux.git#next
+crc		https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git#crc-next
+keys-next	https://git.kernel.org/pub/scm/linux/kernel/git/dhowells/linux-fs.git#keys-next
+fwctl		https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git#for-next
+devsec-tsm	https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git#next
+hisilicon	https://github.com/hisilicon/linux-hisi.git#for-next
+device-id	https://git.kernel.org/pub/scm/linux/kernel/git/ukleinek/linux.git#device-id-rework
+kthread		https://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git#for-next
diff --git a/Next/merge.log b/Next/merge.log
new file mode 100644
index 0000000000000..893463f081044
--- /dev/null
+++ b/Next/merge.log
@@ -0,0 +1,10167 @@
+$ date -R
+Fri, 17 Jul 2026 13:00:40 +0100
+$ git checkout master
+Switched to branch 'master'
+$ git reset --hard stable
+HEAD is now at 37e2f878a7a66 Merge tag 'linux_kselftest-kunit-fixes-7.2-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest
+Merging origin/master (fce2dfa773ced Merge tag 'v7.2-rc3-smb3-client-fixes' of git://git.samba.org/sfrench/cifs-2.6)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git origin/master
+Updating 37e2f878a7a66..fce2dfa773ced
+Fast-forward (no commit created; -m option ignored)
+ MAINTAINERS                                        |   1 +
+ arch/arm64/include/asm/kvm_nested.h                |   8 +
+ arch/arm64/kvm/at.c                                |   8 -
+ arch/arm64/kvm/emulate-nested.c                    |  41 +++-
+ arch/arm64/kvm/hyp/include/hyp/switch.h            |  11 +-
+ arch/arm64/kvm/hyp/nvhe/ffa.c                      |  44 ++--
+ arch/arm64/kvm/hyp/nvhe/pkvm.c                     |   3 +-
+ arch/arm64/kvm/hyp/nvhe/sys_regs.c                 |   3 +-
+ arch/arm64/kvm/hyp/pgtable.c                       |  15 +-
+ arch/arm64/kvm/inject_fault.c                      |  18 +-
+ arch/arm64/kvm/mmio.c                              |   7 +-
+ arch/arm64/kvm/nested.c                            | 164 ++++++-------
+ arch/arm64/kvm/pkvm.c                              |   2 +-
+ arch/arm64/kvm/vgic/vgic-its.c                     |   2 +
+ arch/arm64/kvm/vgic/vgic.c                         |  20 +-
+ arch/s390/kvm/dat.c                                |   2 +
+ arch/s390/kvm/gmap.c                               |   7 +-
+ arch/s390/kvm/kvm-s390.c                           |   6 +-
+ arch/s390/kvm/pci.c                                |  12 +-
+ arch/x86/kvm/irq.c                                 |   4 +-
+ arch/x86/kvm/lapic.c                               |   8 +-
+ arch/x86/kvm/svm/sev.c                             |   6 +-
+ arch/x86/kvm/svm/svm.c                             |   7 +-
+ arch/x86/kvm/trace.h                               |   2 +-
+ arch/x86/kvm/vmx/nested.c                          |  89 ++++---
+ arch/x86/kvm/vmx/tdx.c                             |   6 +-
+ arch/x86/kvm/vmx/vmx.h                             |   7 +
+ drivers/cpufreq/cpufreq.c                          |   3 +
+ drivers/cpufreq/intel_pstate.c                     |   6 +-
+ drivers/firmware/arm_ffa/driver.c                  |  25 +-
+ drivers/pmdomain/imx/imx8m-blk-ctrl.c              |  46 +++-
+ drivers/pmdomain/imx/imx93-blk-ctrl.c              |  60 ++++-
+ drivers/pmdomain/mediatek/mtk-pm-domains.c         |  40 +++-
+ fs/erofs/ishare.c                                  |   8 +-
+ fs/erofs/super.c                                   |  19 +-
+ fs/erofs/zmap.c                                    |   3 +-
+ fs/smb/client/cifsfs.c                             |  10 +-
+ fs/smb/client/smb1maperror.c                       |  24 +-
+ fs/smb/client/smb1maperror_test.c                  |   1 +
+ fs/smb/client/smb2maperror.c                       |  12 +-
+ fs/smb/client/smb2maperror_test.c                  |   1 +
+ fs/smb/client/smb2misc.c                           |  34 ++-
+ fs/smb/client/smb2ops.c                            | 183 +++++++++++++--
+ fs/smb/client/smb2pdu.c                            |  19 ++
+ fs/smb/client/smb2proto.h                          |   3 +
+ fs/smb/common/fscc.h                               |   5 +
+ fs/smb/server/smb2pdu.h                            |   4 -
+ fs/xfs/libxfs/xfs_defer.c                          |   2 +-
+ fs/xfs/scrub/cow_repair.c                          | 179 +++++++--------
+ fs/xfs/scrub/dirtree.c                             |  16 +-
+ fs/xfs/scrub/dqiterate.c                           |   2 +-
+ fs/xfs/scrub/inode_repair.c                        |   4 +-
+ fs/xfs/scrub/rgsuper.c                             |  14 +-
+ fs/xfs/scrub/rtbitmap.c                            |   2 +-
+ fs/xfs/scrub/rtrefcount.c                          |   2 +-
+ fs/xfs/scrub/rtrmap.c                              |   8 +-
+ fs/xfs/scrub/scrub.h                               |   6 +-
+ fs/xfs/scrub/trace.h                               |  35 ---
+ fs/xfs/scrub/xfarray.c                             |   3 +-
+ fs/xfs/xfs_bmap_util.c                             |  89 +++++++
+ fs/xfs/xfs_bmap_util.h                             |   4 +
+ fs/xfs/xfs_log_cil.c                               |   2 +-
+ fs/xfs/xfs_log_recover.c                           |  26 ++-
+ fs/xfs/xfs_reflink.c                               |  36 +++
+ fs/xfs/xfs_sysfs.c                                 |  28 ++-
+ fs/xfs/xfs_sysfs.h                                 |   2 +
+ fs/xfs/xfs_trace.h                                 |  41 ++++
+ fs/xfs/xfs_zone_alloc.c                            |   8 +
+ include/dt-bindings/power/fsl,imx93-power.h        |   1 +
+ include/linux/arm_ffa.h                            |   9 +-
+ security/landlock/net.c                            |   8 +
+ security/landlock/ruleset.h                        |   4 +-
+ security/landlock/task.c                           |   9 +-
+ tools/testing/selftests/kvm/Makefile.kvm           |   1 +
+ tools/testing/selftests/kvm/arm64/mmio_sign_ext.c  | 255 +++++++++++++++++++++
+ .../testing/selftests/kvm/x86/sev_migrate_tests.c  |  47 ++++
+ tools/testing/selftests/landlock/net_test.c        |  97 ++++++++
+ .../selftests/landlock/scoped_signal_test.c        |  67 ++++--
+ virt/kvm/kvm_main.c                                |  16 +-
+ 79 files changed, 1506 insertions(+), 526 deletions(-)
+ create mode 100644 tools/testing/selftests/kvm/arm64/mmio_sign_ext.c
+Merging ext4-fixes/fixes (981fcc5674e67 jbd2: fix deadlock in jbd2_journal_cancel_revoke())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git ext4-fixes/fixes
+Already up to date.
+Merging vfs-brauner-fixes/vfs.fixes (24dddc384fb9a Merge patch series "iomap: consolidate bio submission")
+$ git merge -m Merge branch 'vfs.fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git vfs-brauner-fixes/vfs.fixes
+Already up to date.
+Merging fscrypt-current/for-current (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'for-current' of https://git.kernel.org/pub/scm/fs/fscrypt/linux.git fscrypt-current/for-current
+Already up to date.
+Merging fsverity-current/for-current (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'for-current' of https://git.kernel.org/pub/scm/fs/fsverity/linux.git fsverity-current/for-current
+Already up to date.
+Merging btrfs-fixes/next-fixes (6c893b948351d Merge branch 'misc-7.2' into next-fixes)
+$ git merge -m Merge branch 'next-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git btrfs-fixes/next-fixes
+Merge made by the 'ort' strategy.
+Merging vfs-fixes/fixes (49c5d168a3a8f udf: fix nls leak on udf_fill_super() failure)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git vfs-fixes/fixes
+Auto-merging fs/udf/super.c
+Merge made by the 'ort' strategy.
+Merging erofs-fixes/fixes (1572282de6d33 erofs: hide "cache_strategy=" for plain filesystems)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git erofs-fixes/fixes
+Already up to date.
+Merging nfsd-fixes/nfsd-fixes (92ea163c773cb NFSD: Prevent post-shutdown use-after-free in NFSD_CMD_UNLOCK_FILESYSTEM)
+$ git merge -m Merge branch 'nfsd-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux nfsd-fixes/nfsd-fixes
+Already up to date.
+Merging v9fs-fixes/fixes/next (028ef9c96e961 Linux 7.0)
+$ git merge -m Merge branch 'fixes/next' of https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git v9fs-fixes/fixes/next
+Already up to date.
+Merging overlayfs-fixes/ovl-fixes (4549871118cf6 Linux 7.1-rc7)
+$ git merge -m Merge branch 'ovl-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git overlayfs-fixes/ovl-fixes
+Already up to date.
+Merging fscrypt/for-next (2d0afaac9137e fscrypt: Remove workaround for bug in gcc 7 and earlier)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/fs/fscrypt/linux.git fscrypt/for-next
+Merge made by the 'ort' strategy.
+ fs/crypto/crypto.c                 | 40 ++++++++------------------------------
+ fs/crypto/fscrypt_private.h        |  5 +----
+ fs/crypto/keyring.c                | 21 +++++++++-----------
+ fs/crypto/keysetup.c               | 26 ++++++++++---------------
+ fs/crypto/policy.c                 | 17 ++--------------
+ include/uapi/linux/fscrypt.h       |  1 -
+ tools/include/uapi/linux/fscrypt.h |  1 -
+ 7 files changed, 30 insertions(+), 81 deletions(-)
+Merging btrfs/for-next (2e35e224edd82 Merge branch 'for-next-current-v7.1-20260714' into for-next-20260714)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git btrfs/for-next
+Auto-merging fs/btrfs/inode.c
+Merge made by the 'ort' strategy.
+ fs/btrfs/backref.c         |  5 ++++-
+ fs/btrfs/btrfs_inode.h     |  2 ++
+ fs/btrfs/extent_io.c       |  4 ++--
+ fs/btrfs/extent_map.c      |  4 ++--
+ fs/btrfs/inode.c           |  3 ++-
+ fs/btrfs/ioctl.c           |  1 +
+ fs/btrfs/relocation.c      | 55 ++++++++++++++++++++++++++++++++++++----------
+ fs/btrfs/subpage.c         | 26 +++++++++++++++++++---
+ include/uapi/linux/btrfs.h |  2 +-
+ 9 files changed, 80 insertions(+), 22 deletions(-)
+Merging ceph/master (1397040896280 selftests: ceph: wire up Ceph reset kselftests and documentation)
+$ git merge -m Merge branch 'master' of https://github.com/ceph/ceph-client.git ceph/master
+Auto-merging MAINTAINERS
+Auto-merging tools/testing/selftests/Makefile
+Merge made by the 'ort' strategy.
+ MAINTAINERS                                        |   1 +
+ tools/testing/selftests/Makefile                   |   1 +
+ tools/testing/selftests/filesystems/ceph/Makefile  |   7 +
+ tools/testing/selftests/filesystems/ceph/README    |  84 +++
+ .../filesystems/ceph/reset_corner_cases.sh         | 646 +++++++++++++++++++
+ .../selftests/filesystems/ceph/reset_stress.sh     | 694 +++++++++++++++++++++
+ .../selftests/filesystems/ceph/run_validation.sh   | 350 +++++++++++
+ tools/testing/selftests/filesystems/ceph/settings  |   1 +
+ .../filesystems/ceph/validate_consistency.py       | 297 +++++++++
+ 9 files changed, 2081 insertions(+)
+ create mode 100644 tools/testing/selftests/filesystems/ceph/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/ceph/README
+ create mode 100755 tools/testing/selftests/filesystems/ceph/reset_corner_cases.sh
+ create mode 100755 tools/testing/selftests/filesystems/ceph/reset_stress.sh
+ create mode 100755 tools/testing/selftests/filesystems/ceph/run_validation.sh
+ create mode 100644 tools/testing/selftests/filesystems/ceph/settings
+ create mode 100755 tools/testing/selftests/filesystems/ceph/validate_consistency.py
+Merging cifs/for-next (fce2dfa773ced Merge tag 'v7.2-rc3-smb3-client-fixes' of git://git.samba.org/sfrench/cifs-2.6)
+$ git merge -m Merge branch 'for-next' of git://git.samba.org/sfrench/cifs-2.6.git cifs/for-next
+Already up to date.
+Merging configfs/configfs-next (6363844fdbbb7 samples: configfs: Constify struct configfs_item_operations and configfs_group_operations)
+$ git merge -m Merge branch 'configfs-next' of https://git.kernel.org/pub/scm/linux/kernel/git/a.hindborg/linux.git configfs/configfs-next
+Already up to date.
+Merging ecryptfs/next (95ce5ffd54cf6 ecryptfs: use kasprintf in ecryptfs_crypto_api_algify_cipher_name)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/tyhicks/ecryptfs.git ecryptfs/next
+Already up to date.
+Merging dlm/next (e61113cfcaf71 dlm: init per node debugfs before add to node hash)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/teigland/linux-dlm.git dlm/next
+Already up to date.
+Merging erofs/dev (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git erofs/dev
+Already up to date.
+Merging exfat/dev (13a315afaeb67 MAINTAINERS: update mailing list address for exfat)
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/exfat.git exfat/dev
+Auto-merging MAINTAINERS
+Auto-merging fs/exfat/iomap.c
+Merge made by the 'ort' strategy.
+ MAINTAINERS         |   2 +-
+ fs/exfat/exfat_fs.h |   2 +-
+ fs/exfat/file.c     | 222 +++++++++++++++++++++++++++++++++++++++++++---------
+ fs/exfat/iomap.c    |  22 +++++-
+ 4 files changed, 209 insertions(+), 39 deletions(-)
+Merging ext3/for_next (04267552014fe Pull isofs compressed block offset fix.)
+$ git merge -m Merge branch 'for_next' of https://git.kernel.org/pub/scm/linux/kernel/git/jack/linux-fs.git ext3/for_next
+Merge made by the 'ort' strategy.
+ fs/isofs/compress.c           |  6 ++++--
+ fs/notify/fanotify/fanotify.c |  1 +
+ fs/udf/balloc.c               | 12 ++++++++++--
+ fs/udf/inode.c                | 26 +++++++++++++++++++++++---
+ fs/udf/partition.c            |  2 +-
+ fs/udf/super.c                | 23 ++++++++++++++---------
+ fs/udf/truncate.c             |  2 +-
+ fs/udf/udfdecl.h              |  3 ++-
+ 8 files changed, 56 insertions(+), 19 deletions(-)
+Merging ext4/dev (c143957520c6c ext4: validate donor file superblock early in EXT4_IOC_MOVE_EXT)
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git ext4/dev
+Already up to date.
+Merging f2fs/dev (6429f3bf6dc33 f2fs: fix folio_nr_pages() race after put in large folio invalidate)
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git f2fs/dev
+Merge made by the 'ort' strategy.
+ fs/f2fs/data.c    |  5 +++--
+ fs/f2fs/debug.c   |  4 ----
+ fs/f2fs/f2fs.h    | 29 ++++++++++++++++++++++++++++-
+ fs/f2fs/file.c    | 28 +++++++++++++++++-----------
+ fs/f2fs/gc.c      | 36 +++++++++++++++---------------------
+ fs/f2fs/gc.h      | 27 +--------------------------
+ fs/f2fs/segment.c |  9 ++++-----
+ fs/f2fs/super.c   |  6 +++---
+ fs/f2fs/sysfs.c   | 22 +++++++++++-----------
+ 9 files changed, 82 insertions(+), 84 deletions(-)
+Merging fsverity/for-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/fs/fsverity/linux.git fsverity/for-next
+Already up to date.
+Merging fuse/for-next (7d87a5a284bb3 fuse-uring: clear ent->fuse_req in commit_fetch error path)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git fuse/for-next
+Already up to date.
+Merging gfs2/for-next (5f80d9113360c Merge tag 'gfs2-for-7.2' of git://git.kernel.org/pub/scm/linux/kernel/git/gfs2/linux-gfs2)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gfs2/linux-gfs2.git gfs2/for-next
+Already up to date.
+Merging jfs/jfs-next (dad98c5b2a05e jfs: avoid -Wtautological-constant-out-of-range-compare warning again)
+$ git merge -m Merge branch 'jfs-next' of https://github.com/kleikamp/linux-shaggy.git jfs/jfs-next
+Already up to date.
+Merging ksmbd/ksmbd-for-next (39ccf3cdc6790 ksmbd: Do not skip lock checks for single-byte ranges)
+$ git merge -m Merge branch 'ksmbd-for-next' of https://github.com/smfrench/smb3-kernel.git ksmbd/ksmbd-for-next
+Merge made by the 'ort' strategy.
+ fs/smb/server/compress.c          |   3 +-
+ fs/smb/server/connection.c        |  16 ++-
+ fs/smb/server/ksmbd_netlink.h     |   1 +
+ fs/smb/server/mgmt/user_session.c |  25 ++---
+ fs/smb/server/mgmt/user_session.h |   2 +-
+ fs/smb/server/oplock.c            |   6 +-
+ fs/smb/server/smb2misc.c          |   5 +
+ fs/smb/server/smb2pdu.c           | 224 ++++++++++++++++++++++++++++++++------
+ fs/smb/server/smbacl.c            | 147 +++++++++++++++++--------
+ fs/smb/server/smbacl.h            |   3 +-
+ fs/smb/server/vfs.c               |   8 +-
+ 11 files changed, 334 insertions(+), 106 deletions(-)
+$ git am -3 ../patches/0001-ntfs3-Fix-up-merge-with-Linus.patch
+Applying: ntfs3: Fix up merge with Linus
+Using index info to reconstruct a base tree...
+M	fs/ntfs3/file.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+Merging nfs/linux-next (a13c140cc289c Linux 7.2-rc3)
+$ git merge -m Merge branch 'linux-next' of git://git.linux-nfs.org/projects/trondmy/nfs-2.6.git nfs/linux-next
+Already up to date.
+Merging nfs-anna/linux-next (27934d02cbeb8 NFS: Charge unstable writes by request size, not folio size)
+$ git merge -m Merge branch 'linux-next' of git://git.linux-nfs.org/projects/anna/linux-nfs.git nfs-anna/linux-next
+Already up to date.
+Merging nfsd/nfsd-next (c83cc1e5e8d2b nfsd: add support to CB_NOTIFY for dir attribute changes)
+$ git merge -m Merge branch 'nfsd-next' of https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux nfsd/nfsd-next
+Auto-merging fs/nfsd/nfsctl.c
+Merge made by the 'ort' strategy.
+ Documentation/sunrpc/xdr/nfs4_1.x        | 258 ++++++++-
+ fs/lockd/lockd.h                         |   2 +-
+ fs/lockd/svc.c                           |   4 +-
+ fs/lockd/svc4proc.c                      |   4 +-
+ fs/lockd/svclock.c                       |  42 +-
+ fs/lockd/svcproc.c                       |   4 +-
+ fs/lockd/svcsubs.c                       | 107 +++-
+ fs/nfs/callback.c                        |   4 +-
+ fs/nfs_common/nfslocalio.c               |  16 +-
+ fs/nfsd/filecache.c                      | 265 ++++++---
+ fs/nfsd/filecache.h                      |   3 +
+ fs/nfsd/flexfilelayoutxdr.c              |  20 +-
+ fs/nfsd/localio.c                        |   8 +-
+ fs/nfsd/lockd.c                          |   6 +-
+ fs/nfsd/netns.h                          |  29 +-
+ fs/nfsd/nfs2acl.c                        |  21 +-
+ fs/nfsd/nfs3acl.c                        |  17 +-
+ fs/nfsd/nfs3proc.c                       |  40 ++
+ fs/nfsd/nfs4callback.c                   | 207 ++++++-
+ fs/nfsd/nfs4layouts.c                    |  45 +-
+ fs/nfsd/nfs4proc.c                       | 114 +++-
+ fs/nfsd/nfs4recover.c                    |  48 +-
+ fs/nfsd/nfs4state.c                      | 944 ++++++++++++++++++++++++++-----
+ fs/nfsd/nfs4xdr.c                        | 392 +++++++++++--
+ fs/nfsd/nfs4xdr_gen.c                    | 600 +++++++++++++++++++-
+ fs/nfsd/nfs4xdr_gen.h                    |  17 +-
+ fs/nfsd/nfscache.c                       |   4 +-
+ fs/nfsd/nfsctl.c                         | 156 +++--
+ fs/nfsd/nfsd.h                           |   9 +-
+ fs/nfsd/nfsfh.c                          |  42 +-
+ fs/nfsd/nfsfh.h                          |   3 +
+ fs/nfsd/nfsproc.c                        |   7 +
+ fs/nfsd/nfssvc.c                         |  61 +-
+ fs/nfsd/nfsxdr.c                         |  32 +-
+ fs/nfsd/state.h                          |  88 ++-
+ fs/nfsd/trace.h                          |  42 +-
+ fs/nfsd/vfs.c                            |  48 +-
+ fs/nfsd/xdr4.h                           |   5 +
+ fs/nfsd/xdr4cb.h                         |  12 +
+ include/linux/lockd/bind.h               |  12 +-
+ include/linux/nfs4.h                     | 127 -----
+ include/linux/sunrpc/bc_xprt.h           |   5 +
+ include/linux/sunrpc/svc.h               |   1 +
+ include/linux/sunrpc/svc_rdma_pcl.h      |   4 +-
+ include/linux/sunrpc/xdrgen/nfs4_1.h     | 290 +++++++++-
+ include/uapi/linux/nfs4.h                |   2 -
+ net/sunrpc/auth_gss/auth_gss.c           |   6 +-
+ net/sunrpc/auth_gss/gss_krb5_unseal.c    |   3 +
+ net/sunrpc/auth_gss/gss_krb5_wrap.c      |  13 +-
+ net/sunrpc/auth_gss/gss_rpc_upcall.c     |   6 -
+ net/sunrpc/auth_gss/gss_rpc_upcall.h     |   1 -
+ net/sunrpc/auth_gss/gss_rpc_xdr.c        |  15 +-
+ net/sunrpc/auth_gss/svcauth_gss.c        |   8 +-
+ net/sunrpc/backchannel_rqst.c            |  38 +-
+ net/sunrpc/cache.c                       |   7 +-
+ net/sunrpc/sunrpc_syms.c                 |   1 +
+ net/sunrpc/svc.c                         |  81 ++-
+ net/sunrpc/svcauth_unix.c                |   4 +-
+ net/sunrpc/xdr.c                         |   2 +-
+ net/sunrpc/xprtrdma/ib_client.c          |  42 +-
+ net/sunrpc/xprtrdma/svc_rdma_pcl.c       |  63 ++-
+ net/sunrpc/xprtrdma/svc_rdma_recvfrom.c  |  24 +-
+ net/sunrpc/xprtrdma/svc_rdma_rw.c        |  52 +-
+ net/sunrpc/xprtrdma/svc_rdma_transport.c |  69 ++-
+ 64 files changed, 3787 insertions(+), 815 deletions(-)
+$ git am -3 ../patches/0001-Revert-smb-client-implement-fileattr_get-to-support-.patch
+Applying: Revert "smb: client: implement fileattr_get to support FS_IOC_GETFLAGS"
+Using index info to reconstruct a base tree...
+M	fs/smb/client/cifsfs.c
+M	fs/smb/client/cifsfs.h
+M	fs/smb/client/inode.c
+Falling back to patching base and 3-way merge...
+Auto-merging fs/smb/client/inode.c
+Auto-merging fs/smb/client/cifsfs.h
+Auto-merging fs/smb/client/cifsfs.c
+No changes -- Patch already applied.
+Merging ntfs/ntfs-next (9e3ca21f5d941 ntfs: use pagecache_isize_extended() on size extension)
+$ git merge -m Merge branch 'ntfs-next' of https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/ntfs.git ntfs/ntfs-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS        |   2 +-
+ fs/ntfs/attrib.c   |  15 ++++++++
+ fs/ntfs/compress.c | 102 ++++++++++++++++++++---------------------------------
+ fs/ntfs/dir.c      |   5 +--
+ fs/ntfs/ea.c       |  85 ++++++++++++++++++++++++++++++++++----------
+ fs/ntfs/file.c     |  71 ++++++++++++++++++++++++++-----------
+ fs/ntfs/inode.c    |   6 +---
+ fs/ntfs/inode.h    |   2 +-
+ fs/ntfs/iomap.c    |  34 +-----------------
+ fs/ntfs/mft.c      |   3 +-
+ fs/ntfs/namei.c    |   3 +-
+ fs/ntfs/reparse.c  |   6 ++--
+ fs/ntfs/runlist.c  |  50 ++++++++++++++------------
+ 13 files changed, 210 insertions(+), 174 deletions(-)
+Merging ntfs3/master (349a7959ffd7a fs/ntfs3: load ATTR_BITMAP run extents from $MFT extension records)
+$ git merge -m Merge branch 'master' of https://github.com/Paragon-Software-Group/linux-ntfs3.git ntfs3/master
+Auto-merging fs/ntfs3/inode.c
+Merge made by the 'ort' strategy.
+ fs/ntfs3/dir.c     | 36 +++++++++++++++++++++++++++---------
+ fs/ntfs3/file.c    |  2 +-
+ fs/ntfs3/frecord.c | 31 +++++++++++++++++++++----------
+ fs/ntfs3/fslog.c   | 21 ++++++++-------------
+ fs/ntfs3/fsntfs.c  |  8 ++++----
+ fs/ntfs3/index.c   |  7 +++----
+ fs/ntfs3/inode.c   | 18 +++++++++++++-----
+ fs/ntfs3/lznt.c    |  4 +++-
+ fs/ntfs3/namei.c   |  9 +++++++--
+ fs/ntfs3/ntfs_fs.h |  6 +++---
+ fs/ntfs3/run.c     |  9 +++++----
+ fs/ntfs3/super.c   |  9 +++++----
+ fs/ntfs3/xattr.c   |  1 -
+ 13 files changed, 100 insertions(+), 61 deletions(-)
+Merging orangefs/for-next (e61bc5e4d8743    bufmap: manage as folios, V2.)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/hubcap/linux.git orangefs/for-next
+Already up to date.
+Merging overlayfs/overlayfs-next (1f6ee9be92f8d ovl: make fsync after metadata copy-up opt-in mount option)
+$ git merge -m Merge branch 'overlayfs-next' of https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git overlayfs/overlayfs-next
+Already up to date.
+Merging ubifs/next (11efa98bcc0d0 ubi: ubi.h: fix kernel-doc warnings)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/rw/ubifs.git ubifs/next
+Auto-merging drivers/mtd/ubi/ubi.h
+Merge made by the 'ort' strategy.
+ drivers/mtd/ubi/ubi.h | 9 ++++++---
+ 1 file changed, 6 insertions(+), 3 deletions(-)
+Merging v9fs/9p-next (aa88278693cbf 9p: Add missing read barrier in virtio zero-copy path)
+$ git merge -m Merge branch '9p-next' of https://github.com/martinetd/linux v9fs/9p-next
+Already up to date.
+Merging v9fs-ericvh/ericvh/for-next (028ef9c96e961 Linux 7.0)
+$ git merge -m Merge branch 'ericvh/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git v9fs-ericvh/ericvh/for-next
+Already up to date.
+Merging xfs/for-next (59c462b0f5cfa xfs: don't zap bmbt forks if they are MAXLEVELS tall)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git xfs/for-next
+Already up to date.
+Merging zonefs/for-next (3a8389d42bdf4 zonefs: handle integer overflow in zonefs_fname_to_fno)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dlemoal/zonefs.git zonefs/for-next
+Already up to date.
+Merging vfs-brauner/vfs.all (b148f6ef8f3b0 Merge branch 'vfs-7.3.kthread' into vfs.all)
+$ git merge -m Merge branch 'vfs.all' of https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git vfs-brauner/vfs.all
+Auto-merging MAINTAINERS
+Auto-merging fs/afs/dir.c
+Auto-merging fs/bpf_fs_kfuncs.c
+CONFLICT (content): Merge conflict in fs/bpf_fs_kfuncs.c
+Auto-merging fs/btrfs/inode.c
+Auto-merging fs/btrfs/ioctl.c
+Auto-merging fs/erofs/super.c
+Auto-merging fs/f2fs/super.c
+Auto-merging fs/iomap/buffered-io.c
+Auto-merging fs/iomap/direct-io.c
+Auto-merging fs/iomap/ioend.c
+Auto-merging fs/namei.c
+Auto-merging fs/nfs/dir.c
+Auto-merging fs/nfs/internal.h
+Auto-merging fs/ntfs/namei.c
+Auto-merging fs/ntfs3/namei.c
+Auto-merging fs/overlayfs/inode.c
+Auto-merging fs/smb/client/cifsfs.h
+Auto-merging fs/smb/client/inode.c
+Auto-merging fs/smb/server/smb2pdu.c
+Auto-merging fs/smb/server/vfs.c
+Auto-merging fs/xfs/xfs_buf.c
+Auto-merging fs/xfs/xfs_super.c
+Auto-merging include/linux/fs.h
+Auto-merging include/linux/netfs.h
+Auto-merging kernel/exit.c
+Auto-merging kernel/fork.c
+Auto-merging tools/testing/selftests/filesystems/.gitignore
+CONFLICT (content): Merge conflict in tools/testing/selftests/filesystems/.gitignore
+Auto-merging tools/testing/selftests/filesystems/Makefile
+CONFLICT (content): Merge conflict in tools/testing/selftests/filesystems/Makefile
+Resolved 'fs/bpf_fs_kfuncs.c' using previous resolution.
+Resolved 'tools/testing/selftests/filesystems/.gitignore' using previous resolution.
+Resolved 'tools/testing/selftests/filesystems/Makefile' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[fs-next 6aad3f0fd71d1] Merge branch 'vfs.all' of https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/filesystems/locking.rst              |   2 +-
+ Documentation/filesystems/overlayfs.rst            |  16 +
+ Documentation/filesystems/porting.rst              |  10 +-
+ Documentation/filesystems/vfs.rst                  |   2 +-
+ MAINTAINERS                                        |   5 -
+ block/bdev.c                                       | 125 +++-
+ drivers/base/devtmpfs.c                            |   2 +-
+ drivers/block/rnbd/rnbd-srv.c                      |   4 +-
+ drivers/char/misc_minor_kunit.c                    |  25 +-
+ drivers/crypto/ccp/sev-dev.c                       |  12 +-
+ drivers/target/target_core_alua.c                  |  11 +-
+ drivers/target/target_core_pr.c                    |   4 +-
+ fs/9p/vfs_inode.c                                  |   5 +-
+ fs/9p/vfs_inode_dotl.c                             |   4 +-
+ fs/Kconfig                                         |   1 -
+ fs/Makefile                                        |   1 -
+ fs/affs/affs.h                                     |   2 +-
+ fs/affs/namei.c                                    |   4 +-
+ fs/afs/dir.c                                       |   6 +-
+ fs/autofs/root.c                                   |   2 +-
+ fs/bad_inode.c                                     |   2 +-
+ fs/bfs/dir.c                                       |   2 +-
+ fs/bpf_fs_kfuncs.c                                 |  37 ++
+ fs/btrfs/dev-replace.c                             |  65 ++-
+ fs/btrfs/inode.c                                   |   4 +-
+ fs/btrfs/ioctl.c                                   |   4 +-
+ fs/btrfs/volumes.c                                 | 116 +++-
+ fs/btrfs/volumes.h                                 |   6 +-
+ fs/ceph/dir.c                                      |   3 +-
+ fs/coda/dir.c                                      |   9 +-
+ fs/coredump.c                                      |  11 +-
+ fs/cramfs/inode.c                                  |   2 +-
+ fs/ecryptfs/inode.c                                |   2 +-
+ fs/efivarfs/inode.c                                |   2 +-
+ fs/efs/Kconfig                                     |  16 -
+ fs/efs/Makefile                                    |   8 -
+ fs/efs/dir.c                                       | 105 ----
+ fs/efs/efs.h                                       | 144 -----
+ fs/efs/file.c                                      |  42 --
+ fs/efs/inode.c                                     | 315 ----------
+ fs/efs/namei.c                                     | 120 ----
+ fs/efs/super.c                                     | 368 ------------
+ fs/efs/symlink.c                                   |  50 --
+ fs/erofs/super.c                                   |  35 +-
+ fs/exfat/namei.c                                   |   2 +-
+ fs/ext2/namei.c                                    |   4 +-
+ fs/ext4/extents-test.c                             |   9 +-
+ fs/ext4/mballoc-test.c                             |   9 +-
+ fs/ext4/namei.c                                    |   4 +-
+ fs/ext4/super.c                                    |  12 +-
+ fs/f2fs/namei.c                                    |   4 +-
+ fs/f2fs/super.c                                    |   6 +-
+ fs/fat/namei_msdos.c                               |   5 +-
+ fs/fat/namei_vfat.c                                |   2 +-
+ fs/fs_struct.c                                     | 103 +++-
+ fs/fuse/dir.c                                      |  10 +-
+ fs/gfs2/inode.c                                    |   7 +-
+ fs/hfs/dir.c                                       |   4 +-
+ fs/hfsplus/dir.c                                   |   4 +-
+ fs/hostfs/hostfs_kern.c                            |   2 +-
+ fs/hpfs/namei.c                                    |   6 +-
+ fs/hugetlbfs/inode.c                               |   4 +-
+ fs/inode.c                                         |  23 +-
+ fs/internal.h                                      |   1 +
+ fs/iomap/buffered-io.c                             |   6 +-
+ fs/iomap/direct-io.c                               | 293 +++++++++-
+ fs/iomap/ioend.c                                   |  21 +-
+ fs/jffs2/dir.c                                     |   6 +-
+ fs/jfs/namei.c                                     |   4 +-
+ fs/kernel_read_file.c                              |   9 +-
+ fs/minix/namei.c                                   |   4 +-
+ fs/namei.c                                         |  30 +-
+ fs/namespace.c                                     |  25 +-
+ fs/nfs/blocklayout/dev.c                           |  15 +-
+ fs/nfs/dir.c                                       |   6 +-
+ fs/nfs/internal.h                                  |   2 +-
+ fs/nilfs2/namei.c                                  |   4 +-
+ fs/ntfs/namei.c                                    |   4 +-
+ fs/ntfs3/namei.c                                   |   4 +-
+ fs/nullfs.c                                        |  14 +-
+ fs/ocfs2/dlmfs/dlmfs.c                             |   5 +-
+ fs/ocfs2/namei.c                                   |   5 +-
+ fs/ocfs2/super.c                                   |   1 -
+ fs/omfs/dir.c                                      |   4 +-
+ fs/orangefs/namei.c                                |   5 +-
+ fs/overlayfs/dir.c                                 |  18 +-
+ fs/overlayfs/inode.c                               |  26 +-
+ fs/overlayfs/overlayfs.h                           |   1 +
+ fs/overlayfs/super.c                               |   2 +-
+ fs/overlayfs/xattrs.c                              |   1 +
+ fs/posix_acl.c                                     |   2 -
+ fs/proc/array.c                                    |   4 +-
+ fs/proc/base.c                                     |   8 +-
+ fs/proc_namespace.c                                |   4 +-
+ fs/ramfs/inode.c                                   |   4 +-
+ fs/romfs/super.c                                   |  10 +-
+ fs/smb/client/cifsfs.h                             |   2 +-
+ fs/smb/client/dir.c                                |   2 +-
+ fs/smb/client/inode.c                              |   7 +
+ fs/smb/server/mgmt/share_config.c                  |   4 +-
+ fs/smb/server/smb2pdu.c                            |   4 +-
+ fs/smb/server/vfs.c                                |   9 +-
+ fs/super.c                                         | 639 ++++++++++++++-------
+ fs/ubifs/dir.c                                     |   4 +-
+ fs/udf/namei.c                                     |   4 +-
+ fs/ufs/namei.c                                     |   5 +-
+ fs/vboxsf/dir.c                                    |   4 +-
+ fs/xfs/xfs_buf.c                                   |   2 +-
+ fs/xfs/xfs_iops.c                                  |   7 +-
+ fs/xfs/xfs_super.c                                 |  10 +-
+ include/linux/blk_types.h                          |   2 +-
+ include/linux/blkdev.h                             |  12 +-
+ include/linux/efs_vh.h                             |  54 --
+ include/linux/fs.h                                 |   4 +-
+ include/linux/fs/super.h                           |   8 +
+ include/linux/fs/super_types.h                     |   4 +-
+ include/linux/fs_struct.h                          |  34 ++
+ include/linux/init_task.h                          |   1 +
+ include/linux/net.h                                |   1 +
+ include/linux/sched.h                              |   1 +
+ include/linux/sched/task.h                         |   1 +
+ include/linux/types.h                              |   2 +
+ include/uapi/asm-generic/errno.h                   |   2 +-
+ include/uapi/linux/efs_fs_sb.h                     |  63 --
+ init/init_task.c                                   |   1 +
+ init/initramfs.c                                   |  14 +-
+ init/initramfs_test.c                              | 220 +++----
+ init/main.c                                        |  10 +-
+ ipc/mqueue.c                                       |   7 +-
+ kernel/exit.c                                      |   2 +-
+ kernel/fork.c                                      |  53 +-
+ kernel/kcmp.c                                      |   2 +-
+ kernel/umh.c                                       |   6 +-
+ mm/shmem.c                                         |   2 +-
+ net/socket.c                                       |  25 +
+ net/unix/af_unix.c                                 |  17 +-
+ tools/testing/selftests/bpf/bpf_experimental.h     |   3 +
+ .../testing/selftests/bpf/prog_tests/sock_xattr.c  |  67 +++
+ .../testing/selftests/bpf/progs/sock_read_xattr.c  |  54 ++
+ tools/testing/selftests/filesystems/.gitignore     |   1 +
+ tools/testing/selftests/filesystems/Makefile       |   2 +-
+ .../selftests/filesystems/overlayfs/.gitignore     |   1 +
+ .../selftests/filesystems/overlayfs/Makefile       |   2 +
+ .../filesystems/overlayfs/idmapped_mounts.c        | 501 ++++++++++++++++
+ .../filesystems/overlayfs/set_layers_via_fds.c     |  16 +-
+ .../filesystems/statmount/statmount_test.c         |   5 +-
+ tools/testing/selftests/filesystems/ustat_test.c   | 135 +++++
+ tools/testing/selftests/proc/proc-pidns.c          |   1 +
+ 148 files changed, 2536 insertions(+), 1982 deletions(-)
+ delete mode 100644 fs/efs/Kconfig
+ delete mode 100644 fs/efs/Makefile
+ delete mode 100644 fs/efs/dir.c
+ delete mode 100644 fs/efs/efs.h
+ delete mode 100644 fs/efs/file.c
+ delete mode 100644 fs/efs/inode.c
+ delete mode 100644 fs/efs/namei.c
+ delete mode 100644 fs/efs/super.c
+ delete mode 100644 fs/efs/symlink.c
+ delete mode 100644 include/linux/efs_vh.h
+ delete mode 100644 include/uapi/linux/efs_fs_sb.h
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/sock_xattr.c
+ create mode 100644 tools/testing/selftests/bpf/progs/sock_read_xattr.c
+ create mode 100644 tools/testing/selftests/filesystems/overlayfs/idmapped_mounts.c
+ create mode 100644 tools/testing/selftests/filesystems/ustat_test.c
+$ git am -3 ../patches/0001-ksmbd-Fix-removal-of-type-parameter-from-vfs_path_pa.patch
+Applying: ksmbd: Fix removal of type parameter from vfs_path_parent_lookup()
+Using index info to reconstruct a base tree...
+M	fs/smb/server/vfs.c
+Falling back to patching base and 3-way merge...
+Auto-merging fs/smb/server/vfs.c
+No changes -- Patch already applied.
+Merging vfs/for-next (4dda01b67c866 Merge branches 'work.dcache', 'work.dcache-d_add' and 'work.configfs' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git vfs/for-next
+Merge made by the 'ort' strategy.
+Merging mm-hotfixes/mm-hotfixes-unstable (3566356c5b2ea mm: memcg: initialize *locked in memcg1_oom_prepare() stub)
+$ git merge -m Merge branch 'mm-hotfixes-unstable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-hotfixes/mm-hotfixes-unstable
+Auto-merging MAINTAINERS
+Auto-merging arch/arm64/mm/mmu.c
+Merge made by the 'ort' strategy.
+ .mailmap                                        |   1 +
+ Documentation/ABI/testing/sysfs-kernel-mm-damon | 174 ++++++++++++------------
+ MAINTAINERS                                     |   3 +-
+ arch/arm64/include/asm/ptdump.h                 |   2 -
+ arch/arm64/mm/mmu.c                             |  43 +-----
+ arch/arm64/mm/ptdump.c                          |  11 +-
+ arch/m68k/include/asm/page_mm.h                 |   2 +
+ arch/x86/mm/pat/set_memory.c                    |  15 +-
+ drivers/virtio/virtio_balloon.c                 |   5 +-
+ fs/fat/fat.h                                    |   4 +-
+ fs/fat/fat_test.c                               |   4 +-
+ fs/ocfs2/dir.c                                  |   5 +-
+ fs/proc/task_mmu.c                              |  34 ++++-
+ include/linux/damon.h                           |   2 -
+ include/linux/memcontrol.h                      |  25 ++++
+ include/linux/mmap_lock.h                       |   3 +
+ include/linux/page_reporting.h                  |   4 +-
+ include/trace/events/memory-failure.h           |   6 +-
+ lib/test_hmm.c                                  |   2 +-
+ mm/damon/core.c                                 |  15 +-
+ mm/damon/lru_sort.c                             |   2 -
+ mm/damon/modules-common.c                       |   2 -
+ mm/damon/modules-common.h                       |   2 -
+ mm/damon/ops-common.c                           |   2 -
+ mm/damon/ops-common.h                           |   2 -
+ mm/damon/paddr.c                                |   2 -
+ mm/damon/reclaim.c                              |   2 -
+ mm/damon/sysfs-common.c                         |   2 -
+ mm/damon/sysfs-common.h                         |   2 -
+ mm/damon/sysfs-schemes.c                        |   2 -
+ mm/damon/sysfs.c                                |   2 -
+ mm/damon/tests/core-kunit.h                     |   4 -
+ mm/damon/tests/sysfs-kunit.h                    |   2 -
+ mm/damon/tests/vaddr-kunit.h                    |   4 -
+ mm/damon/vaddr.c                                |   2 -
+ mm/huge_memory.c                                |  14 +-
+ mm/hugetlb.c                                    |  24 +++-
+ mm/hugetlb_cma.c                                |   2 +-
+ mm/kmemleak.c                                   |   7 +-
+ mm/memcontrol-v1.h                              |   6 +-
+ mm/memremap.c                                   |   1 +
+ mm/migrate.c                                    |   6 +-
+ mm/migrate_device.c                             |   5 +-
+ mm/mm_init.c                                    |  15 +-
+ mm/page_reporting.c                             |  24 ++--
+ mm/page_vma_mapped.c                            |  33 +++--
+ mm/pagewalk.c                                   |  36 +++--
+ mm/percpu-km.c                                  |   2 +-
+ mm/ptdump.c                                     |   7 +
+ mm/userfaultfd.c                                |   5 +-
+ mm/util.c                                       |   2 +-
+ mm/vmalloc.c                                    |  40 ++++--
+ mm/vmscan.c                                     |  11 +-
+ tools/testing/selftests/filesystems/fclog.c     |   4 +-
+ tools/testing/selftests/mm/pagemap_ioctl.c      |  56 +++++++-
+ 55 files changed, 413 insertions(+), 281 deletions(-)
+Merging fs-current (7f6f78211933e Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git)
+$ git merge -m Merge branch 'fs-current' of linux-next fs-current
+Merge made by the 'ort' strategy.
+Merging kbuild-current/kbuild-fixes-for-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'kbuild-fixes-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git kbuild-current/kbuild-fixes-for-next
+Already up to date.
+Merging clang-fixes/clang-fixes-for-next (175db11786bde Disable -Wattribute-alias for clang-23 and newer)
+$ git merge -m Merge branch 'clang-fixes-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/nathan/linux.git clang-fixes/clang-fixes-for-next
+Already up to date.
+Merging arc-current/for-curr (76f38ad1b39cf ARC: configs: Drop redundant I2C_DESIGNWARE_PLATFORM)
+$ git merge -m Merge branch 'for-curr' of https://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc.git arc-current/for-curr
+Merge made by the 'ort' strategy.
+ arch/arc/configs/axs101_defconfig     | 1 -
+ arch/arc/configs/axs103_defconfig     | 1 -
+ arch/arc/configs/axs103_smp_defconfig | 1 -
+ arch/arc/configs/tb10x_defconfig      | 1 -
+ arch/arc/kernel/smp.c                 | 6 ++++--
+ 5 files changed, 4 insertions(+), 6 deletions(-)
+Merging arm-current/fixes (009b6c6486b94 ARM: 9476/1: mm: fix kexec and hibernation with CONFIG_CPU_TTBR0_PAN)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git arm-current/fixes
+Already up to date.
+Merging arm64-fixes/for-next/fixes (f797f51a185ff arm64: mm: When logging data aborts only decode Xs when ISV=1)
+$ git merge -m Merge branch 'for-next/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux arm64-fixes/for-next/fixes
+Merge made by the 'ort' strategy.
+ arch/arm64/include/asm/fixmap.h |  6 +++++-
+ arch/arm64/kernel/ptrace.c      | 29 +++++++++++++++++++++++++++++
+ arch/arm64/mm/fault.c           |  7 ++++---
+ drivers/resctrl/mpam_devices.c  | 36 ++++++++++++++++++++++--------------
+ 4 files changed, 60 insertions(+), 18 deletions(-)
+Merging arm-soc-fixes/arm/fixes (add11ea55f2d2 Merge tag 'scmi-ffa-fixes-7.2' of https://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux into arm/fixes)
+$ git merge -m Merge branch 'arm/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git arm-soc-fixes/arm/fixes
+Auto-merging .mailmap
+Auto-merging MAINTAINERS
+Auto-merging drivers/firmware/arm_ffa/driver.c
+Merge made by the 'ort' strategy.
+ .mailmap                                     |  1 +
+ MAINTAINERS                                  | 12 ++++++------
+ arch/arm/mach-ixp4xx/Kconfig                 |  1 -
+ arch/arm/mach-pxa/Kconfig                    |  3 ---
+ arch/arm64/boot/dts/nvidia/tegra234.dtsi     | 24 +++++++++++------------
+ arch/arm64/boot/dts/nvidia/tegra264.dtsi     |  4 ++--
+ drivers/firmware/arm_ffa/driver.c            | 29 ++++++++++++++++++++--------
+ drivers/firmware/arm_scmi/Kconfig            |  4 ++--
+ drivers/firmware/arm_scmi/clock.c            | 13 ++++++++-----
+ drivers/firmware/arm_scmi/notify.c           |  6 +++---
+ drivers/gpu/drm/tegra/sor.c                  |  6 ++----
+ drivers/reset/reset-imx7.c                   |  6 ++++++
+ drivers/reset/reset-sunxi.c                  |  4 +++-
+ drivers/reset/spacemit/reset-spacemit-k3.c   |  2 +-
+ drivers/soc/tegra/fuse/tegra-apbmisc.c       |  2 +-
+ include/dt-bindings/reset/altr,rst-mgr-s10.h |  2 +-
+ include/soc/tegra/pmc.h                      |  5 ++++-
+ 17 files changed, 73 insertions(+), 51 deletions(-)
+Merging davinci-current/davinci/for-current (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'davinci/for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git davinci-current/davinci/for-current
+Already up to date.
+Merging realtek-fixes/fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/yu_chun/linux.git realtek-fixes/fixes
+Already up to date.
+Merging drivers-memory-fixes/fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git drivers-memory-fixes/fixes
+Already up to date.
+Merging sophgo-fixes/fixes (19272b37aa4f8 Linux 6.16-rc1)
+$ git merge -m Merge branch 'fixes' of https://github.com/sophgo/linux.git sophgo-fixes/fixes
+Already up to date.
+Merging sophgo-soc-fixes/soc-fixes (0af2f6be1b428 Linux 6.15-rc1)
+$ git merge -m Merge branch 'soc-fixes' of https://github.com/sophgo/linux.git sophgo-soc-fixes/soc-fixes
+Already up to date.
+Merging m68k-current/for-linus (dc5200f6b1ada m68k: Correct CONFIG_MVME16x macro name in #endif comment)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git m68k-current/for-linus
+Already up to date.
+Merging powerpc-fixes/fixes (a2c02aa0c6ca3 powerpc: Remove dead non-preemption code)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git powerpc-fixes/fixes
+Merge made by the 'ort' strategy.
+ arch/powerpc/include/asm/preempt.h                       | 16 ----------------
+ arch/powerpc/include/asm/uaccess.h                       |  2 +-
+ arch/powerpc/kernel/dt_cpu_ftrs.c                        |  9 +++++++++
+ arch/powerpc/kernel/time.c                               |  4 +++-
+ arch/powerpc/lib/vmx-helper.c                            |  2 +-
+ arch/powerpc/platforms/85xx/common.c                     |  2 ++
+ arch/powerpc/platforms/cell/spufs/file.c                 |  6 ++++--
+ arch/powerpc/platforms/pseries/Kconfig                   |  1 +
+ .../powerpc/platforms/pseries/papr_platform_attributes.c |  8 +++-----
+ 9 files changed, 24 insertions(+), 26 deletions(-)
+ delete mode 100644 arch/powerpc/include/asm/preempt.h
+Merging s390-fixes/fixes (49145bce53911 s390/perf_cpum_cf: Add missing array_index_nospec() to __hw_perf_event_init())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git s390-fixes/fixes
+Merge made by the 'ort' strategy.
+ arch/s390/kernel/perf_cpum_cf.c | 3 +++
+ arch/s390/lib/csum-partial.c    | 2 +-
+ 2 files changed, 4 insertions(+), 1 deletion(-)
+Merging net/main (2c72eb6286347 llc: fix SAP refcount leak when creating incoming sockets)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git net/main
+Auto-merging .mailmap
+Merge made by the 'ort' strategy.
+ .mailmap                                           |   4 +-
+ drivers/bluetooth/btqca.c                          |   2 +-
+ drivers/bluetooth/btrtl.c                          |   5 +-
+ drivers/bluetooth/hci_qca.c                        |   4 +
+ drivers/dpll/dpll_netlink.c                        |   3 +
+ drivers/net/can/Kconfig                            |   7 +-
+ drivers/net/can/peak_canfd/peak_canfd.c            |   2 +-
+ drivers/net/can/peak_canfd/peak_canfd_user.h       |   2 +-
+ drivers/net/can/peak_canfd/peak_pciefd_main.c      |   4 +-
+ drivers/net/can/sja1000/peak_pci.c                 |   4 +-
+ drivers/net/can/sja1000/peak_pcmcia.c              |   4 +-
+ drivers/net/can/usb/esd_usb.c                      |   5 +-
+ drivers/net/can/usb/peak_usb/pcan_usb.c            |   2 +-
+ drivers/net/can/usb/peak_usb/pcan_usb_core.c       |   4 +-
+ drivers/net/can/usb/peak_usb/pcan_usb_core.h       |   2 +-
+ drivers/net/can/usb/peak_usb/pcan_usb_fd.c         |   2 +-
+ drivers/net/can/usb/peak_usb/pcan_usb_pro.c        |   2 +-
+ drivers/net/can/usb/peak_usb/pcan_usb_pro.h        |   2 +-
+ drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c      |  39 +-
+ drivers/net/ethernet/mellanox/mlx5/core/lib/st.c   |   1 +
+ drivers/net/macsec.c                               |   7 +-
+ drivers/net/ppp/ppp_generic.c                      |  18 +-
+ .../broadcom/brcm80211/brcmfmac/cyw/core.c         |   6 +
+ .../wireless/broadcom/brcm80211/brcmfmac/sdio.c    |   2 +-
+ drivers/net/wireless/intel/ipw2x00/ipw2100.c       |   8 +-
+ drivers/net/wireless/intel/ipw2x00/libipw_rx.c     |   4 +-
+ drivers/net/wireless/intersil/p54/txrx.c           |   8 +
+ drivers/net/wireless/marvell/libertas/firmware.c   |   1 +
+ drivers/net/wireless/marvell/libertas/tx.c         |   7 +
+ drivers/net/wireless/marvell/libertas_tf/main.c    |   2 +-
+ drivers/net/wireless/marvell/mwifiex/cfg80211.c    |   2 +-
+ drivers/net/wireless/marvell/mwifiex/join.c        |   1 -
+ drivers/net/wireless/ralink/rt2x00/rt2400pci.c     |   2 +-
+ drivers/net/wireless/ralink/rt2x00/rt2500pci.c     |   2 +-
+ drivers/net/wireless/ralink/rt2x00/rt2800pci.c     |   2 +-
+ drivers/net/wireless/ralink/rt2x00/rt2x00dev.c     |  12 +-
+ drivers/net/wireless/ralink/rt2x00/rt61pci.c       |   2 +-
+ drivers/net/wireless/rsi/rsi_91x_hal.c             |   8 +
+ drivers/net/wireless/rsi/rsi_91x_mgmt.c            |  12 +-
+ drivers/net/wireless/virtual/mac80211_hwsim_main.c |  16 +-
+ include/linux/can/dev/peak_canfd.h                 |   2 +-
+ include/linux/ieee80211-eht.h                      |  12 +-
+ include/net/bluetooth/hci_core.h                   |   1 +
+ include/net/cfg80211.h                             |   4 +-
+ include/net/sctp/structs.h                         |   3 +-
+ include/net/xfrm.h                                 |   2 +
+ net/bluetooth/hci_event.c                          |  18 +-
+ net/bluetooth/hci_sync.c                           |  68 ++-
+ net/bluetooth/mgmt.c                               | 105 +++-
+ net/can/bcm.c                                      | 654 ++++++++++++++++-----
+ net/can/isotp.c                                    | 298 +++++++---
+ net/can/j1939/transport.c                          |  18 +-
+ net/can/raw.c                                      |  66 +--
+ net/core/dev.c                                     |  10 +-
+ net/ipv4/fib_trie.c                                |   2 +-
+ net/ipv4/tcp_ipv4.c                                |   6 +-
+ net/ipv6/netfilter.c                               |   4 +-
+ net/ipv6/tcp_ipv6.c                                |   6 +-
+ net/ipv6/xfrm6_policy.c                            |   1 +
+ net/iucv/af_iucv.c                                 |   4 +
+ net/llc/llc_conn.c                                 |   1 -
+ net/mac80211/cfg.c                                 |  11 +-
+ net/mac80211/ibss.c                                |  13 +-
+ net/mac80211/iface.c                               |   8 +-
+ net/mac80211/main.c                                |   3 +-
+ net/mac80211/mlme.c                                |  12 +-
+ net/mac80211/nan.c                                 |  35 +-
+ net/mac80211/rx.c                                  |  34 +-
+ net/mac80211/sta_info.c                            |  15 +-
+ net/mac80211/tx.c                                  |  17 +-
+ net/mac80211/util.c                                |   3 +
+ net/netfilter/ipvs/ip_vs_app.c                     |  10 +-
+ net/netfilter/ipvs/ip_vs_core.c                    |   3 +-
+ net/netfilter/ipvs/ip_vs_xmit.c                    |   6 +-
+ net/netfilter/nf_conncount.c                       |   6 +-
+ net/netfilter/nf_conntrack_ecache.c                |   2 +-
+ net/netfilter/nf_flow_table_core.c                 |   6 +-
+ net/netfilter/xt_nat.c                             |   9 +
+ net/netfilter/xt_physdev.c                         |   5 -
+ net/openvswitch/flow_netlink.c                     | 201 +++++--
+ net/sched/act_tunnel_key.c                         |  14 +-
+ net/sctp/sm_statefuns.c                            |  23 +-
+ net/wireless/core.c                                |  14 +-
+ net/wireless/core.h                                |   4 +-
+ net/wireless/mlme.c                                | 105 +++-
+ net/wireless/nl80211.c                             |  25 +-
+ net/wireless/pmsr.c                                |  34 +-
+ net/wireless/scan.c                                |  18 +-
+ net/wireless/sme.c                                 |   6 +-
+ net/xfrm/xfrm_device.c                             |  13 +-
+ net/xfrm/xfrm_iptfs.c                              |   1 +
+ net/xfrm/xfrm_nat_keepalive.c                      |  15 +-
+ net/xfrm/xfrm_policy.c                             |   4 +-
+ net/xfrm/xfrm_state.c                              |  12 +-
+ net/xfrm/xfrm_user.c                               |  41 +-
+ .../drivers/net/netconsole/netcons_resume.sh       |   3 +-
+ .../selftests/net/netfilter/nft_flowtable.sh       |  55 ++
+ 97 files changed, 1685 insertions(+), 608 deletions(-)
+Merging bpf/master (3d84d674e3a5a Merge branch 'bpf-fix-tracing-of-kfuncs-with-implicit-args')
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git/ bpf/master
+Merge made by the 'ort' strategy.
+ include/linux/btf.h                                |  1 +
+ include/linux/skmsg.h                              | 14 ----
+ kernel/bpf/btf.c                                   | 29 +++++++
+ kernel/bpf/verifier.c                              | 97 ++++++++++++++++------
+ net/core/bpf_sk_storage.c                          |  2 -
+ net/core/sock.c                                    |  3 +
+ net/core/sock_map.c                                |  2 +
+ net/ipv4/tcp_bpf.c                                 | 17 +++-
+ .../bpf/prog_tests/kfunc_implicit_args_tracing.c   | 36 ++++++++
+ .../prog_tests/raw_tp_writable_reject_bad_access.c | 57 +++++++++++++
+ .../raw_tp_writable_reject_nbd_invalid.c           | 43 ----------
+ .../selftests/bpf/prog_tests/sockmap_basic.c       | 45 +++++++++-
+ .../selftests/bpf/prog_tests/sockmap_listen.c      | 21 ++---
+ tools/testing/selftests/bpf/prog_tests/verifier.c  |  2 +
+ .../bpf/progs/kfunc_implicit_args_tracing.c        | 77 +++++++++++++++++
+ .../selftests/bpf/progs/verifier_ptr_to_buf.c      | 27 ++++++
+ .../selftests/bpf/progs/verifier_raw_tp_writable.c | 16 ++++
+ tools/testing/selftests/bpf/test_maps.c            | 13 ++-
+ 18 files changed, 397 insertions(+), 105 deletions(-)
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/kfunc_implicit_args_tracing.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_bad_access.c
+ delete mode 100644 tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_nbd_invalid.c
+ create mode 100644 tools/testing/selftests/bpf/progs/kfunc_implicit_args_tracing.c
+ create mode 100644 tools/testing/selftests/bpf/progs/verifier_ptr_to_buf.c
+Merging ipsec/master (21697720ff43b esp: do not unref managed frag pages in esp_ssg_unref())
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec.git ipsec/master
+Merge made by the 'ort' strategy.
+ net/ipv4/esp4.c        | 7 +++++++
+ net/ipv6/esp6.c        | 7 +++++++
+ net/ipv6/xfrm6_input.c | 2 +-
+ 3 files changed, 15 insertions(+), 1 deletion(-)
+Merging netfilter/main (2c7c88a412aa6 Merge tag 'net-7.2-rc3' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf.git netfilter/main
+Already up to date.
+Merging ipvs/main (2c7c88a412aa6 Merge tag 'net-7.2-rc3' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs.git ipvs/main
+Already up to date.
+Merging wireless/for-next (a0d82fb850532 Merge tag 'wireless-2026-07-09' of https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless.git wireless/for-next
+Already up to date.
+Merging ath/for-current (a1a21995c2e1c wifi: carl9170: fix buffer overflow in rx_stream failover path)
+$ git merge -m Merge branch 'for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git ath/for-current
+Merge made by the 'ort' strategy.
+ drivers/net/wireless/ath/ath10k/htt_rx.c      |  2 ++
+ drivers/net/wireless/ath/ath11k/ahb.c         |  1 +
+ drivers/net/wireless/ath/ath11k/dp_rx.c       |  3 +++
+ drivers/net/wireless/ath/ath11k/pci.c         |  4 ++++
+ drivers/net/wireless/ath/ath11k/qmi.c         | 11 ++++++---
+ drivers/net/wireless/ath/ath12k/dp_peer.c     | 11 +++++++--
+ drivers/net/wireless/ath/ath12k/dp_peer.h     |  1 +
+ drivers/net/wireless/ath/ath12k/dp_rx.c       |  7 +++---
+ drivers/net/wireless/ath/ath12k/hw.h          | 16 +++++++++++++
+ drivers/net/wireless/ath/ath12k/mac.c         | 10 +++++---
+ drivers/net/wireless/ath/ath12k/pci.c         |  4 ++++
+ drivers/net/wireless/ath/ath12k/peer.c        |  3 +++
+ drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c | 33 +++++++++++++++++++++++++++
+ drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h |  6 +++++
+ drivers/net/wireless/ath/ath12k/wifi7/hw.c    |  3 +++
+ drivers/net/wireless/ath/ath6kl/txrx.c        | 12 ++++++----
+ drivers/net/wireless/ath/ath6kl/wmi.c         | 20 ++++++++++++++++
+ drivers/net/wireless/ath/ath9k/hif_usb.c      |  7 +-----
+ drivers/net/wireless/ath/carl9170/rx.c        |  7 ++++--
+ drivers/net/wireless/ath/carl9170/tx.c        |  2 +-
+ 20 files changed, 137 insertions(+), 26 deletions(-)
+Merging iwlwifi/fixes (a426d3227c669 wifi: iwlwifi: adapt ND match notif sizing to fixed matches array)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git iwlwifi/fixes
+Merge made by the 'ort' strategy.
+ drivers/net/wireless/intel/iwlwifi/cfg/rf-pe.c     |  1 +
+ .../net/wireless/intel/iwlwifi/fw/api/nvm-reg.h    | 33 ++++++++++-
+ drivers/net/wireless/intel/iwlwifi/fw/api/scan.h   |  4 +-
+ drivers/net/wireless/intel/iwlwifi/fw/pnvm.c       |  9 ++-
+ drivers/net/wireless/intel/iwlwifi/fw/regulatory.c |  2 +-
+ drivers/net/wireless/intel/iwlwifi/fw/smem.c       | 28 ++++++++--
+ drivers/net/wireless/intel/iwlwifi/iwl-config.h    |  2 +
+ drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c | 64 ++++++++++++++--------
+ drivers/net/wireless/intel/iwlwifi/mld/agg.c       |  7 ++-
+ drivers/net/wireless/intel/iwlwifi/mld/mac80211.c  |  2 +-
+ drivers/net/wireless/intel/iwlwifi/mld/mcc.c       | 10 +++-
+ drivers/net/wireless/intel/iwlwifi/mld/thermal.c   |  9 ++-
+ drivers/net/wireless/intel/iwlwifi/mld/tx.c        |  4 ++
+ drivers/net/wireless/intel/iwlwifi/mvm/d3.c        | 46 +++++++++++-----
+ .../net/wireless/intel/iwlwifi/mvm/ftm-initiator.c | 55 +++++++++++++------
+ drivers/net/wireless/intel/iwlwifi/mvm/fw.c        | 10 ++++
+ drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c  | 25 +++++----
+ drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c  |  7 +--
+ drivers/net/wireless/intel/iwlwifi/mvm/mvm.h       |  3 -
+ drivers/net/wireless/intel/iwlwifi/mvm/nvm.c       | 43 ++++++++++++---
+ drivers/net/wireless/intel/iwlwifi/mvm/ptp.c       |  4 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c     |  9 ++-
+ drivers/net/wireless/intel/iwlwifi/mvm/rx.c        |  5 ++
+ drivers/net/wireless/intel/iwlwifi/mvm/scan.c      | 10 +++-
+ .../net/wireless/intel/iwlwifi/mvm/time-event.c    |  5 ++
+ drivers/net/wireless/intel/iwlwifi/mvm/tx.c        | 45 ++++++++++++++-
+ .../net/wireless/intel/iwlwifi/pcie/ctxt-info.c    |  7 ++-
+ .../net/wireless/intel/iwlwifi/pcie/gen1_2/rx.c    |  9 ++-
+ 28 files changed, 349 insertions(+), 109 deletions(-)
+Merging wpan/master (8ce4f287524c7 net: libwx: fix firmware mailbox abnormal return)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan.git wpan/master
+Already up to date.
+Merging rdma-fixes/for-rc (3cda0dfe8c651 RDMA/irdma: Prevent overflows in memory contiguity checks)
+$ git merge -m Merge branch 'for-rc' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git rdma-fixes/for-rc
+Already up to date.
+Merging sound-current/for-linus (ac1e8d01f4ea0 ALSA: hda/realtek: Add inverted LED quirk for HP ZBook 8 G2a)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git sound-current/for-linus
+Merge made by the 'ort' strategy.
+ sound/core/seq/seq_timer.c                 | 13 ++++++++-
+ sound/hda/codecs/hdmi/hdmi.c               |  3 ++
+ sound/hda/codecs/hdmi/hdmi_local.h         |  9 ++++++
+ sound/hda/codecs/hdmi/intelhdmi.c          | 36 ++++++++++++++++++++----
+ sound/hda/codecs/realtek/alc269.c          | 45 ++++++++++++++++++++++++++++++
+ sound/hda/codecs/side-codecs/cs35l41_hda.c | 11 +++++++-
+ 6 files changed, 109 insertions(+), 8 deletions(-)
+Merging sound-asoc-fixes/for-linus (0b604e886ece1 ASoC: bt-sco: fix duplicate DAPM widget names for wideband DAI)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git sound-asoc-fixes/for-linus
+Merge made by the 'ort' strategy.
+ sound/soc/amd/acp-config.c        | 12 ++++++++
+ sound/soc/amd/yc/acp6x-mach.c     |  7 +++++
+ sound/soc/codecs/bt-sco.c         | 10 +++++--
+ sound/soc/codecs/tas2781-fmwlib.c | 63 +++++++++++++++++++++++++++++++++++----
+ sound/soc/fsl/fsl_sai.c           | 29 +++++++++++++++---
+ sound/soc/fsl/imx-card.c          | 12 +++++---
+ sound/soc/sunxi/sun4i-codec.c     |  1 +
+ 7 files changed, 118 insertions(+), 16 deletions(-)
+Merging regmap-fixes/for-linus (63d441e937b7c Merge remote-tracking branch 'regmap/for-7.1' into regmap-linus)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git regmap-fixes/for-linus
+Merge made by the 'ort' strategy.
+Merging regulator-fixes/for-linus (d38f8bd771c49 regulator: core: regulator_lock_two() should test for EDEADLK not EDEADLOCK)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git regulator-fixes/for-linus
+Already up to date.
+Merging spi-fixes/for-linus (e919ca35a6e84 spi: cadence-quadspi: Fix indirect write timeout when DMA read mode is enabled)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git spi-fixes/for-linus
+Merge made by the 'ort' strategy.
+ drivers/spi/spi-cadence-quadspi.c | 12 ++++++++----
+ drivers/spi/spi-dw-dma.c          |  3 ++-
+ 2 files changed, 10 insertions(+), 5 deletions(-)
+Merging pci-current/for-linus (34db32102439d MAINTAINERS: Drop Karthikeyan Mitran from Mobiveil PCIe entry)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git pci-current/for-linus
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS | 1 -
+ 1 file changed, 1 deletion(-)
+Merging driver-core.current/driver-core-linus (667d0fb32149f driver core: add missing kernel-doc for union members)
+$ git merge -m Merge branch 'driver-core-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git driver-core.current/driver-core-linus
+Merge made by the 'ort' strategy.
+ include/linux/device.h | 2 ++
+ 1 file changed, 2 insertions(+)
+Merging tty.current/tty-linus (af071d9e07e57 serial: sc16is7xx: implement gpio get_direction() callback)
+$ git merge -m Merge branch 'tty-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git tty.current/tty-linus
+Merge made by the 'ort' strategy.
+ drivers/tty/serial/8250/8250_mid.c | 14 +++++++++-----
+ drivers/tty/serial/sc16is7xx.c     | 12 ++++++++++++
+ 2 files changed, 21 insertions(+), 5 deletions(-)
+Merging usb.current/usb-linus (71132cedd1ecb usb: atm: ueagle-atm: reject descriptors that confuse probe and disconnect)
+$ git merge -m Merge branch 'usb-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git usb.current/usb-linus
+Merge made by the 'ort' strategy.
+ drivers/usb/atm/ueagle-atm.c            |  17 +++
+ drivers/usb/chipidea/core.c             |   1 +
+ drivers/usb/core/port.c                 |   2 +
+ drivers/usb/core/sysfs.c                |   8 +-
+ drivers/usb/gadget/function/f_midi.c    |   1 +
+ drivers/usb/gadget/function/f_ncm.c     |  17 ++-
+ drivers/usb/gadget/function/f_printer.c |  23 ++-
+ drivers/usb/gadget/function/f_tcm.c     | 192 +++++++++++++++++++++----
+ drivers/usb/gadget/function/tcm.h       |  13 ++
+ drivers/usb/gadget/function/uvc_v4l2.c  |   2 +
+ drivers/usb/gadget/udc/bdc/bdc_core.c   |  20 +++
+ drivers/usb/gadget/udc/bdc/bdc_udc.c    |   7 +-
+ drivers/usb/gadget/udc/dummy_hcd.c      |  40 ++++--
+ drivers/usb/gadget/udc/fsl_udc_core.c   |  99 ++++++-------
+ drivers/usb/gadget/udc/fsl_usb2_udc.h   |   1 +
+ drivers/usb/gadget/udc/pxa27x_udc.c     |  10 +-
+ drivers/usb/gadget/udc/snps_udc_core.c  |   1 -
+ drivers/usb/host/xhci-pci.c             |   1 +
+ drivers/usb/host/xhci.c                 |  15 +-
+ drivers/usb/host/xhci.h                 |   1 +
+ drivers/usb/musb/omap2430.c             |   2 -
+ drivers/usb/storage/unusual_devs.h      |   7 +
+ drivers/usb/typec/ucsi/ucsi.c           | 240 +++++++++++++++++++++++++-------
+ drivers/usb/typec/ucsi/ucsi_yoga_c630.c |  23 ---
+ 24 files changed, 541 insertions(+), 202 deletions(-)
+Merging usb-serial-fixes/usb-linus (fad0fd120e290 USB: serial: ftdi_sio: add support for E+H FXA291)
+$ git merge -m Merge branch 'usb-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git usb-serial-fixes/usb-linus
+Merge made by the 'ort' strategy.
+ drivers/usb/serial/ftdi_sio.c     |  2 ++
+ drivers/usb/serial/ftdi_sio_ids.h |  5 +++++
+ drivers/usb/serial/keyspan_pda.c  | 44 +++++++++++++++++++++++++++++++--------
+ 3 files changed, 42 insertions(+), 9 deletions(-)
+Merging phy/fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git phy/fixes
+Already up to date.
+Merging staging.current/staging-linus (a13c140cc289c Linux 7.2-rc3)
+$ git merge -m Merge branch 'staging-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git staging.current/staging-linus
+Already up to date.
+Merging iio-fixes/fixes-togreg (a41000ba3a230 iio: light: gp2ap002: Disable regulators on resume failure)
+$ git merge -m Merge branch 'fixes-togreg' of https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git iio-fixes/fixes-togreg
+Auto-merging drivers/iio/temperature/hid-sensor-temperature.c
+Merge made by the 'ort' strategy.
+ drivers/iio/adc/Kconfig                          |  2 ++
+ drivers/iio/dac/m62332.c                         | 17 ++++++++++++-----
+ drivers/iio/light/gp2ap002.c                     | 10 ++++++++--
+ drivers/iio/temperature/hid-sensor-temperature.c |  3 ++-
+ 4 files changed, 24 insertions(+), 8 deletions(-)
+Merging watchdog-fixes/watchdog (bcfcd7619f277 watchdog: airoha: Prevent division by zero when clock frequency is zero)
+$ git merge -m Merge branch 'watchdog' of https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git watchdog-fixes/watchdog
+Merge made by the 'ort' strategy.
+ Documentation/watchdog/watchdog-parameters.rst | 4 ++--
+ drivers/watchdog/airoha_wdt.c                  | 3 +++
+ drivers/watchdog/ni903x_wdt.c                  | 7 ++++++-
+ drivers/watchdog/s32g_wdt.c                    | 3 +--
+ drivers/watchdog/watchdog_pretimeout.c         | 2 ++
+ 5 files changed, 14 insertions(+), 5 deletions(-)
+Merging counter-current/counter-current (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'counter-current' of https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git counter-current/counter-current
+Already up to date.
+Merging char-misc.current/char-misc-linus (a13c140cc289c Linux 7.2-rc3)
+$ git merge -m Merge branch 'char-misc-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git char-misc.current/char-misc-linus
+Already up to date.
+Merging soundwire-fixes/fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git soundwire-fixes/fixes
+Already up to date.
+Merging thunderbolt-fixes/fixes (db79679595326 thunderbolt: Bound the DROM dual link port number before indexing sw->ports)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git thunderbolt-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/thunderbolt/eeprom.c | 9 ++++++++-
+ drivers/thunderbolt/stream.c | 2 +-
+ drivers/thunderbolt/tb.c     | 2 +-
+ 3 files changed, 10 insertions(+), 3 deletions(-)
+Merging input-current/for-linus (adea84ee6cdea Input: mms114 - fix multi-touch slot corruption)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git input-current/for-linus
+Already up to date.
+Merging crypto-current/master (8173f7e2ce67e rhashtable: clear stale iter->p on table restart)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git crypto-current/master
+Merge made by the 'ort' strategy.
+ lib/rhashtable.c | 1 +
+ 1 file changed, 1 insertion(+)
+Merging libcrypto-fixes/libcrypto-fixes (e073f1238ecae crypto: aes - Fix conditions for selecting MAC dependencies)
+$ git merge -m Merge branch 'libcrypto-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git libcrypto-fixes/libcrypto-fixes
+Merge made by the 'ort' strategy.
+ Documentation/crypto/libcrypto-blockcipher.rst |  6 ++--
+ Documentation/crypto/libcrypto-hash.rst        | 38 ++++++++++++++------------
+ Documentation/crypto/libcrypto-signature.rst   |  2 +-
+ Documentation/crypto/libcrypto.rst             |  5 ++--
+ crypto/Kconfig                                 |  4 +--
+ lib/crypto/Kconfig                             |  3 +-
+ lib/crypto/md5.c                               | 14 ----------
+ 7 files changed, 30 insertions(+), 42 deletions(-)
+Merging vfio-fixes/for-linus (e242e974e812e vfio: selftests: Add luuid to libvfio.mk's list of libraries, not to the Makefile)
+$ git merge -m Merge branch 'for-linus' of https://github.com/awilliam/linux-vfio.git vfio-fixes/for-linus
+Already up to date.
+Merging kselftest-fixes/fixes (c5c413534d40e selftests/tracing: Have trigger-hist-poll.tc use sched_process_exit)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kselftest-fixes/fixes
+Already up to date.
+Merging dmaengine-fixes/fixes (867621ba20302 dmaengine: qcom: bam_dma: Fix command element mask field for BAM v1.6.0+)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git dmaengine-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/dma/idxd/cdev.c          |  4 +++-
+ drivers/dma/idxd/init.c          | 36 +++++-------------------------------
+ drivers/dma/sun6i-dma.c          | 11 ++++-------
+ drivers/dma/switchtec_dma.c      |  2 +-
+ include/linux/dma/qcom_bam_dma.h | 21 ++++++++++++++++-----
+ 5 files changed, 29 insertions(+), 45 deletions(-)
+Merging backlight-fixes/for-backlight-fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'for-backlight-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git backlight-fixes/for-backlight-fixes
+Already up to date.
+Merging mtd-fixes/mtd/fixes (2b533e775aec5 Revert "mtd: maps: remove uclinux map driver")
+$ git merge -m Merge branch 'mtd/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git mtd-fixes/mtd/fixes
+Merge made by the 'ort' strategy.
+ drivers/mtd/devices/mchp23k256.c           |   2 +-
+ drivers/mtd/maps/Kconfig                   |   6 ++
+ drivers/mtd/maps/Makefile                  |   1 +
+ drivers/mtd/maps/uclinux.c                 | 118 +++++++++++++++++++++++++++++
+ drivers/mtd/mtd_virt_concat.c              |   8 +-
+ drivers/mtd/mtdcore.c                      |  23 +++++-
+ drivers/mtd/mtdpart.c                      |   8 ++
+ drivers/mtd/mtdswap.c                      |   5 +-
+ drivers/mtd/nand/ecc-mtk.c                 |  24 ++++--
+ drivers/mtd/nand/onenand/onenand_samsung.c |   7 +-
+ drivers/mtd/nand/raw/Kconfig               |   1 +
+ drivers/mtd/nand/raw/fsl_ifc_nand.c        |   9 ++-
+ drivers/mtd/nand/raw/ingenic/ingenic_ecc.c |   7 +-
+ drivers/mtd/nand/raw/lpc32xx_mlc.c         |  12 ++-
+ drivers/mtd/nand/raw/lpc32xx_slc.c         |  12 ++-
+ drivers/mtd/nand/raw/ndfc.c                |   2 +-
+ drivers/mtd/nand/spi/core.c                |   2 +-
+ 17 files changed, 224 insertions(+), 23 deletions(-)
+ create mode 100644 drivers/mtd/maps/uclinux.c
+Merging mfd-fixes/for-mfd-fixes (d5d2d7a8d8be1 MAINTAINERS: Add a mailing list entry to MFD)
+$ git merge -m Merge branch 'for-mfd-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git mfd-fixes/for-mfd-fixes
+Already up to date.
+Merging v4l-dvb-fixes/fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'fixes' of git://linuxtv.org/media-ci/media-pending.git v4l-dvb-fixes/fixes
+Already up to date.
+Merging reset-fixes/reset/fixes (71827776667f4 reset: imx7: Correct polarity of MIPI CSI resets on i.MX8MQ)
+$ git merge -m Merge branch 'reset/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/pza/linux reset-fixes/reset/fixes
+Already up to date.
+Merging mips-fixes/mips-fixes (a13c140cc289c Linux 7.2-rc3)
+$ git merge -m Merge branch 'mips-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git mips-fixes/mips-fixes
+Already up to date.
+Merging at91-fixes/at91-fixes (765aaba18413a ARM: dts: microchip: sam9x7: fix GMAC clock configuration)
+$ git merge -m Merge branch 'at91-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git at91-fixes/at91-fixes
+Already up to date.
+Merging omap-fixes/fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git omap-fixes/fixes
+Already up to date.
+Merging kvm-fixes/master (25f744ffa0c8e KVM: SVM: Bump asid_generation on CPU online to avoid ASID collision after hotplug)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/virt/kvm/kvm.git kvm-fixes/master
+Already up to date.
+Merging kvms390-fixes/master (3e3aa6da87d30 KVM: s390: pci: Fix handling of AIF enable without AISB)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git kvms390-fixes/master
+Already up to date.
+Merging kvm-arm-fixes/fixes (8d187d4b33c26 KVM: arm64: Fix propagation of TLBI level in kvm_pgtable_stage2_relax_perms())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git kvm-arm-fixes/fixes
+Already up to date.
+Merging hwmon-fixes/hwmon (f151d0143ac4e hwmon: (nzxt-kraken3) Stop device IO before calling hid_hw_stop)
+$ git merge -m Merge branch 'hwmon' of https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git hwmon-fixes/hwmon
+Merge made by the 'ort' strategy.
+ drivers/hwmon/corsair-cpro.c        | 1 +
+ drivers/hwmon/corsair-psu.c         | 1 +
+ drivers/hwmon/gigabyte_waterforce.c | 4 +++-
+ drivers/hwmon/nzxt-kraken3.c        | 6 ++++--
+ drivers/hwmon/nzxt-smart2.c         | 2 +-
+ 5 files changed, 10 insertions(+), 4 deletions(-)
+Merging nvdimm-fixes/libnvdimm-fixes (a8aec14230322 nvdimm/bus: Fix potential use after free in asynchronous initialization)
+$ git merge -m Merge branch 'libnvdimm-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git nvdimm-fixes/libnvdimm-fixes
+Already up to date.
+Merging cxl-fixes/fixes (d90f236f8b9e3 cxl/test: Update mock dev array before calling platform_device_add())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git cxl-fixes/fixes
+Already up to date.
+Merging dma-mapping-fixes/dma-mapping-fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'dma-mapping-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git dma-mapping-fixes/dma-mapping-fixes
+Already up to date.
+Merging drivers-x86-fixes/fixes (78bf392ba77dd platform/x86: asus-wmi: temporarily revert to setting a charge limit)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git drivers-x86-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/platform/x86/asus-wmi.c   | 26 +++++++++++++++++++++++---
+ drivers/platform/x86/intel/vsec.c | 17 ++++++++++++-----
+ 2 files changed, 35 insertions(+), 8 deletions(-)
+Merging samsung-krzk-fixes/fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git samsung-krzk-fixes/fixes
+Already up to date.
+Merging pinctrl-samsung-fixes/fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git pinctrl-samsung-fixes/fixes
+Already up to date.
+Merging pinctrl-qcom-fixes/pinctrl-qcom/for-current (437a8d2aa1aa4 pinctrl: qcom: sc8280xp: Add missing wakeup entries for GPIO143/151)
+$ git merge -m Merge branch 'pinctrl-qcom/for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git pinctrl-qcom-fixes/pinctrl-qcom/for-current
+Merge made by the 'ort' strategy.
+ drivers/pinctrl/qcom/pinctrl-msm.c      |  8 ++++----
+ drivers/pinctrl/qcom/pinctrl-sc8280xp.c | 21 +++++++++++----------
+ 2 files changed, 15 insertions(+), 14 deletions(-)
+Merging devicetree-fixes/dt/linus (b39a6b2e9d5bd dt-bindings: mfd: khadas,mcu: Drop type reference from "fan-supply")
+$ git merge -m Merge branch 'dt/linus' of https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git devicetree-fixes/dt/linus
+Already up to date.
+Merging dt-krzk-fixes/fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git dt-krzk-fixes/fixes
+Already up to date.
+Merging scsi-fixes/fixes (e166bafc483e9 scsi: hpsa: Fix DMA mapping leak on IOACCEL2 reset path)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git scsi-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/scsi/bfa/bfa_fcs_lport.c        |  2 +-
+ drivers/scsi/elx/efct/efct_hw.c         |  2 +
+ drivers/scsi/elx/efct/efct_unsol.c      |  1 +
+ drivers/scsi/hosts.c                    |  2 +
+ drivers/scsi/hpsa.c                     |  4 ++
+ drivers/scsi/lpfc/lpfc_init.c           |  3 +-
+ drivers/scsi/scsi_error.c               | 22 +++++++-
+ drivers/scsi/scsi_lib.c                 |  8 ---
+ drivers/scsi/scsi_priv.h                |  2 +
+ drivers/scsi/sg.c                       |  7 ++-
+ drivers/target/target_core_fabric_lib.c | 89 +++++++++++++++++++++++++--------
+ drivers/target/target_core_internal.h   |  3 +-
+ drivers/target/target_core_pr.c         | 12 +++--
+ drivers/xen/xen-scsiback.c              | 30 ++++++++---
+ include/scsi/scsi_device.h              |  1 -
+ include/scsi/scsi_host.h                |  3 ++
+ 16 files changed, 141 insertions(+), 50 deletions(-)
+Merging drm-fixes/drm-fixes (5d1238c7ac366 Merge tag 'amd-drm-fixes-7.2-2026-07-16' of https://gitlab.freedesktop.org/agd5f/linux into drm-fixes)
+$ git merge -m Merge branch 'drm-fixes' of https://gitlab.freedesktop.org/drm/kernel.git drm-fixes/drm-fixes
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c           |  62 ++-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c   |  10 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.h   |   1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c         |   9 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c      |   4 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_object.c         |  13 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c            |  20 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c          |  21 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c             |  10 +-
+ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.c  |   4 +
+ drivers/gpu/drm/amd/amdkfd/kfd_queue.c             |  23 +-
+ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c  | 551 +++++++++++++--------
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c |  70 +--
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c  |  20 +-
+ drivers/gpu/drm/amd/display/dc/core/dc.c           |  45 ++
+ drivers/gpu/drm/amd/display/dc/dc.h                |   3 +
+ .../gpu/drm/amd/display/dc/dce/dce_clock_source.c  |  11 +-
+ .../gpu/drm/amd/display/dc/link/link_detection.c   |   2 +-
+ .../display/dc/link/protocols/link_dp_capability.c |   6 +-
+ .../display/dc/resource/dce100/dce100_resource.c   |   7 +-
+ .../amd/display/dc/resource/dcn42/dcn42_resource.c |   1 +
+ .../display/dc/resource/dcn42b/dcn42b_resource.c   |  23 +-
+ .../display/dc/resource/dcn42b/dcn42b_resource.h   | 116 +----
+ drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c         |  17 +-
+ drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c     |   7 +-
+ .../gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c    |  16 +-
+ drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c          |  12 +
+ drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h      |  12 +-
+ .../gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c   |  45 ++
+ drivers/gpu/drm/drm_panel_backlight_quirks.c       |   9 +
+ include/drm/drm_utils.h                            |   1 +
+ 31 files changed, 720 insertions(+), 431 deletions(-)
+Merging drm-intel-fixes/for-linux-next-fixes (612978b83f45b drm/i915/selftests: Fix GT PM sort comparators)
+$ git merge -m Merge branch 'for-linux-next-fixes' of https://gitlab.freedesktop.org/drm/i915/kernel.git drm-intel-fixes/for-linux-next-fixes
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/i915/display/skl_universal_plane.c | 33 +++++++++-------------
+ drivers/gpu/drm/i915/display/skl_watermark.c       |  2 +-
+ drivers/gpu/drm/i915/gt/intel_engine_user.c        |  2 +-
+ drivers/gpu/drm/i915/gt/selftest_gt_pm.c           |  8 +++---
+ 4 files changed, 19 insertions(+), 26 deletions(-)
+Merging mmc-fixes/fixes (e27c946b589c5 mmc: sdhci-esdhc-imx: fix resume error handling)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git mmc-fixes/fixes
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/mmc/mtk-sd.yaml | 29 +++++++-
+ drivers/memstick/core/ms_block.c                  |  4 +
+ drivers/mmc/core/block.c                          |  3 +-
+ drivers/mmc/core/mmc_test.c                       | 18 +++--
+ drivers/mmc/host/sdhci-esdhc-imx.c                | 91 +++++++++++++++--------
+ drivers/mmc/host/sdhci-of-dwcmshc.c               | 14 ++--
+ drivers/mmc/host/vub300.c                         | 36 ++++++---
+ 7 files changed, 135 insertions(+), 60 deletions(-)
+Merging rtc-fixes/rtc-fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'rtc-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git rtc-fixes/rtc-fixes
+Already up to date.
+Merging gnss-fixes/gnss-linus (a13c140cc289c Linux 7.2-rc3)
+$ git merge -m Merge branch 'gnss-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git gnss-fixes/gnss-linus
+Already up to date.
+Merging hyperv-fixes/hyperv-fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'hyperv-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git hyperv-fixes/hyperv-fixes
+Already up to date.
+Merging risc-v-fixes/fixes (5caae1deee89a riscv: hwprobe: Avoid uninitialized read in hwprobe_get_cpus())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git risc-v-fixes/fixes
+Merge made by the 'ort' strategy.
+ arch/riscv/Kconfig                    |  2 +-
+ arch/riscv/include/asm/cacheflush.h   |  5 +++--
+ arch/riscv/include/asm/io.h           |  8 ++++++++
+ arch/riscv/kernel/entry.S             |  2 +-
+ arch/riscv/kernel/sys_hwprobe.c       |  1 +
+ arch/riscv/kernel/vdso/Makefile       |  6 +++---
+ arch/riscv/kernel/vdso/rt_sigreturn.S | 10 +++++++++-
+ arch/riscv/mm/init.c                  |  2 ++
+ mm/sparse-vmemmap.c                   |  2 ++
+ 9 files changed, 30 insertions(+), 8 deletions(-)
+Merging riscv-dt-fixes/riscv-dt-fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'riscv-dt-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-dt-fixes/riscv-dt-fixes
+Already up to date.
+Merging riscv-soc-fixes/riscv-soc-fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'riscv-soc-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-soc-fixes/riscv-soc-fixes
+Already up to date.
+Merging fpga-fixes/fixes (19272b37aa4f8 Linux 6.16-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git fpga-fixes/fixes
+Already up to date.
+Merging spdx/spdx-linus (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'spdx-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/spdx.git spdx/spdx-linus
+Already up to date.
+Merging gpio-brgl-fixes/gpio/for-current (9dc325327babe gpio: pca953x: fix pca953x_irq_bus_sync_unlock regmap lock)
+$ git merge -m Merge branch 'gpio/for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git gpio-brgl-fixes/gpio/for-current
+Merge made by the 'ort' strategy.
+ drivers/gpio/gpio-pca953x.c               | 19 ++++++++++++++-----
+ drivers/gpio/gpio-sloppy-logic-analyzer.c | 10 ++++++++++
+ drivers/gpio/gpiolib.c                    | 13 ++++++++-----
+ 3 files changed, 32 insertions(+), 10 deletions(-)
+Merging gpio-intel-fixes/fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git gpio-intel-fixes/fixes
+Already up to date.
+Merging pinctrl-intel-fixes/fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git pinctrl-intel-fixes/fixes
+Already up to date.
+Merging auxdisplay-fixes/fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git auxdisplay-fixes/fixes
+Already up to date.
+Merging kunit-fixes/kunit-fixes (5142c56651578 bug: fix warning suppressions with kunit built as module)
+$ git merge -m Merge branch 'kunit-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit-fixes/kunit-fixes
+Already up to date.
+Merging memblock-fixes/fixes (2ebce860bdd7a mm/mm_init: handle alloc_percpu failure in free_area_init_core_hotplug)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git memblock-fixes/fixes
+Auto-merging mm/mm_init.c
+Merge made by the 'ort' strategy.
+ include/linux/memory_hotplug.h |  2 +-
+ mm/memory_hotplug.c            |  3 ++-
+ mm/mm_init.c                   | 19 +++++++++++++------
+ 3 files changed, 16 insertions(+), 8 deletions(-)
+Merging renesas-fixes/fixes (6fa6ee724d8da arm64: dts: renesas: ironhide: Describe inline ECC carveouts)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git renesas-fixes/fixes
+Merge made by the 'ort' strategy.
+ arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts | 41 +++++++++++++++++++++++
+ 1 file changed, 41 insertions(+)
+Merging perf-current/perf-tools (558ef39aeb9a0 Merge tag 'dmaengine-7.2-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine)
+$ git merge -m Merge branch 'perf-tools' of https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools.git perf-current/perf-tools
+Already up to date.
+Merging efi-fixes/urgent (d8809f6931065 efi: sysfb_efi: Extend quirk to cover IdeaPad Duet 3 10IGL5-LTE)
+$ git merge -m Merge branch 'urgent' of https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git efi-fixes/urgent
+Already up to date.
+Merging battery-fixes/fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git battery-fixes/fixes
+Already up to date.
+Merging iommufd-fixes/for-rc (c3b8ee84a9650 iommu/arm-smmu-v3-iommufd: Require exactly one Stream ID for a vDEVICE)
+$ git merge -m Merge branch 'for-rc' of https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git iommufd-fixes/for-rc
+Merge made by the 'ort' strategy.
+ .../iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c    | 15 +++++++++++++++
+ drivers/iommu/iommufd/device.c                     |  2 +-
+ drivers/iommu/iommufd/pages.c                      |  3 +++
+ drivers/iommu/iommufd/viommu.c                     | 22 ++++++++++++++++------
+ 4 files changed, 35 insertions(+), 7 deletions(-)
+Merging rust-fixes/rust-fixes (a19bda861b3a7 rust: device: avoid trailing ; in printing macros)
+$ git merge -m Merge branch 'rust-fixes' of https://github.com/Rust-for-Linux/linux.git rust-fixes/rust-fixes
+Auto-merging Makefile
+Merge made by the 'ort' strategy.
+ Makefile                                      |  5 +++
+ init/Kconfig                                  |  3 ++
+ rust/Makefile                                 | 16 +++++++--
+ rust/bindings/lib.rs                          |  4 +++
+ rust/kernel/device.rs                         | 20 +++++------
+ rust/uapi/lib.rs                              |  4 +++
+ rust/zerocopy-derive/README.md                | 14 ++++----
+ rust/zerocopy-derive/derive/from_bytes.rs     |  2 +-
+ rust/zerocopy-derive/derive/into_bytes.rs     |  2 +-
+ rust/zerocopy-derive/derive/known_layout.rs   |  3 +-
+ rust/zerocopy-derive/derive/mod.rs            | 20 ++++++-----
+ rust/zerocopy-derive/derive/try_from_bytes.rs |  6 ++--
+ rust/zerocopy-derive/derive/unaligned.rs      |  2 +-
+ rust/zerocopy-derive/lib.rs                   | 36 ++++++++++++++++++-
+ rust/zerocopy-derive/repr.rs                  |  2 +-
+ rust/zerocopy-derive/util.rs                  | 34 +++++++++++++-----
+ rust/zerocopy/README.md                       | 15 ++++----
+ rust/zerocopy/rustdoc/style.css               |  1 -
+ rust/zerocopy/src/byte_slice.rs               |  2 +-
+ rust/zerocopy/src/byteorder.rs                | 50 ++++++++++++++++++++++-----
+ rust/zerocopy/src/deprecated.rs               |  2 +-
+ rust/zerocopy/src/error.rs                    |  2 +-
+ rust/zerocopy/src/impls.rs                    |  4 +--
+ rust/zerocopy/src/layout.rs                   |  4 ++-
+ rust/zerocopy/src/lib.rs                      | 12 ++++---
+ rust/zerocopy/src/macros.rs                   |  2 +-
+ rust/zerocopy/src/pointer/inner.rs            |  2 +-
+ rust/zerocopy/src/pointer/invariant.rs        |  2 +-
+ rust/zerocopy/src/pointer/mod.rs              |  2 +-
+ rust/zerocopy/src/pointer/ptr.rs              |  2 +-
+ rust/zerocopy/src/pointer/transmute.rs        |  2 +-
+ rust/zerocopy/src/ref.rs                      |  2 +-
+ rust/zerocopy/src/split_at.rs                 |  2 +-
+ rust/zerocopy/src/util/macro_util.rs          |  2 +-
+ rust/zerocopy/src/util/macros.rs              |  2 +-
+ rust/zerocopy/src/util/mod.rs                 |  8 ++++-
+ rust/zerocopy/src/wrappers.rs                 |  2 +-
+ tools/objtool/check.c                         |  1 +
+ 38 files changed, 208 insertions(+), 88 deletions(-)
+Merging w1-fixes/fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git w1-fixes/fixes
+Already up to date.
+Merging pmdomain-fixes/fixes (a4876f11aa1d0 pmdomain: mediatek: Fix possible nullptr KP in HWV cleanup/on-check)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git pmdomain-fixes/fixes
+Already up to date.
+Merging i2c-andi-fixes/i2c/i2c-fixes (deb35336b5bfe i2c: mediatek: fix WRRD for SoCs without auto_restart option)
+$ git merge -m Merge branch 'i2c/i2c-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git i2c-andi-fixes/i2c/i2c-fixes
+Merge made by the 'ort' strategy.
+ CREDITS                         |  7 +++++++
+ drivers/i2c/busses/i2c-imx.c    | 36 +++++++++++++++++++++++++++++++++---
+ drivers/i2c/busses/i2c-k1.c     |  2 +-
+ drivers/i2c/busses/i2c-mlxbf.c  |  4 +++-
+ drivers/i2c/busses/i2c-mt65xx.c |  2 +-
+ 5 files changed, 45 insertions(+), 6 deletions(-)
+Merging i2c-rust-fixes/rust-i2c-fixes (4eb422482ca5d rust: i2c: fix I2cAdapter refcounts double increment)
+$ git merge -m Merge branch 'rust-i2c-fixes' of https://github.com/ikrtn/rust-for-linux i2c-rust-fixes/rust-i2c-fixes
+Already up to date.
+Merging sparc-fixes/for-linus (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git sparc-fixes/for-linus
+Already up to date.
+Merging clk-fixes/clk-fixes (8cdeaa50eae8d Linux 7.2-rc2)
+$ git merge -m Merge branch 'clk-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git clk-fixes/clk-fixes
+Already up to date.
+Merging thead-clk-fixes/thead-clk-fixes (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'thead-clk-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-clk-fixes/thead-clk-fixes
+Already up to date.
+Merging tenstorrent-clk-fixes/tenstorrent-clk-fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'tenstorrent-clk-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git tenstorrent-clk-fixes/tenstorrent-clk-fixes
+Already up to date.
+Merging fustini-config-fixes/riscv-config-fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'riscv-config-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git fustini-config-fixes/riscv-config-fixes
+Already up to date.
+Merging pwrseq-fixes/pwrseq/for-current (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'pwrseq/for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git pwrseq-fixes/pwrseq/for-current
+Already up to date.
+Merging thead-dt-fixes/thead-dt-fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'thead-dt-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-dt-fixes/thead-dt-fixes
+Already up to date.
+Merging ftrace-fixes/ftrace/fixes (1650a1b6cb1ae fgraph: Check ftrace_pids_enabled on registration for early filtering)
+$ git merge -m Merge branch 'ftrace/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git ftrace-fixes/ftrace/fixes
+Already up to date.
+Merging ring-buffer-fixes/ring-buffer/fixes (057caace5214d tracing: Create output file from cmd_check_undefined)
+$ git merge -m Merge branch 'ring-buffer/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git ring-buffer-fixes/ring-buffer/fixes
+Already up to date.
+Merging trace-fixes/trace/fixes (601ddaceb861b ring-buffer: Allow sparse CPU masks in ring_buffer_desc())
+$ git merge -m Merge branch 'trace/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git trace-fixes/trace/fixes
+Already up to date.
+Merging tracefs-fixes/tracefs/fixes (07004a8c4b572 eventfs: Hold eventfs_mutex and SRCU when remount walks events)
+$ git merge -m Merge branch 'tracefs/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git tracefs-fixes/tracefs/fixes
+Already up to date.
+Merging spacemit-fixes/fixes (eb525edd48907 clk: spacemit: k3: set hdma clock as critical)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/spacemit/linux spacemit-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/clk/spacemit/ccu-k3.c | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+Merging tip-fixes/tip/urgent (3d2378ed6c8ff Merge branch into tip/master: 'x86/urgent')
+$ git merge -m Merge branch 'tip/urgent' of https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git tip-fixes/tip/urgent
+Merge made by the 'ort' strategy.
+ arch/x86/boot/compressed/acpi.c      |  7 ++++++-
+ arch/x86/boot/early_serial_console.c | 11 +++++++----
+ arch/x86/kernel/cpu/Makefile         |  1 -
+ 3 files changed, 13 insertions(+), 6 deletions(-)
+Merging slab-fixes/slab/for-next-fixes (d9e6a76239389 mm/slab: prevent unbounded recursion in free path with new kmalloc type)
+$ git merge -m Merge branch 'slab/for-next-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git slab-fixes/slab/for-next-fixes
+Merge made by the 'ort' strategy.
+ include/linux/alloc_tag.h |   3 ++
+ include/linux/slab.h      |  19 +++++++-
+ lib/alloc_tag.c           |   9 ++++
+ mm/slab.h                 |  32 ++++++++++--
+ mm/slab_common.c          |  19 +++++++-
+ mm/slub.c                 | 121 +++++++++++++++++++---------------------------
+ 6 files changed, 126 insertions(+), 77 deletions(-)
+Merging kexec-fixes/kexec-fixes (ada2e5a44e991 crash_dump: release keyring reference at the correct time)
+$ git merge -m Merge branch 'kexec-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git kexec-fixes/kexec-fixes
+Merge made by the 'ort' strategy.
+ kernel/crash_dump_dm_crypt.c | 15 ++++++++++-----
+ 1 file changed, 10 insertions(+), 5 deletions(-)
+Merging liveupdate-fixes/fixes (0b27b47604411 liveupdate: reject nonzero reserved value for SESSION_FINISH)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git liveupdate-fixes/fixes
+Merge made by the 'ort' strategy.
+ kernel/liveupdate/luo_session.c | 8 ++++++--
+ 1 file changed, 6 insertions(+), 2 deletions(-)
+Merging drm-msm-fixes/msm-fixes (e2332abed2a4d drm/msm: Only fini scheduler after successful init)
+$ git merge -m Merge branch 'msm-fixes' of https://gitlab.freedesktop.org/drm/msm.git drm-msm-fixes/msm-fixes
+Auto-merging drivers/gpu/drm/msm/msm_ringbuffer.c
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/msm/adreno/a3xx_gpu.c         | 36 ++++++++++++++++++++++++++-
+ drivers/gpu/drm/msm/adreno/a5xx_gpu.c         |  5 ++++
+ drivers/gpu/drm/msm/adreno/a6xx_gmu.c         |  5 +++-
+ drivers/gpu/drm/msm/adreno/a6xx_gpu.c         |  8 ++++--
+ drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c   |  5 +++-
+ drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h   | 12 +++++++++
+ drivers/gpu/drm/msm/adreno/a8xx_gpu.c         |  5 +++-
+ drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c |  2 +-
+ drivers/gpu/drm/msm/dp/dp_panel.c             |  4 ++-
+ drivers/gpu/drm/msm/dsi/dsi_host.c            | 12 +++++++++
+ drivers/gpu/drm/msm/dsi/phy/dsi_phy.h         |  1 +
+ drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c     | 18 ++++++++++++--
+ drivers/gpu/drm/msm/msm_gpu.c                 |  8 ++++--
+ drivers/gpu/drm/msm/msm_ringbuffer.c          |  7 +++---
+ drivers/gpu/drm/msm/msm_ringbuffer.h          |  1 +
+ drivers/gpu/drm/msm/registers/adreno/a3xx.xml |  2 ++
+ 16 files changed, 116 insertions(+), 15 deletions(-)
+Merging uml-fixes/fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git uml-fixes/fixes
+Already up to date.
+Merging fwctl-fixes/for-rc (4549871118cf6 Linux 7.1-rc7)
+$ git merge -m Merge branch 'for-rc' of https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git fwctl-fixes/for-rc
+Already up to date.
+Merging devsec-tsm-fixes/fixes (c3fd16c3b98ed virt: tdx-guest: Fix handling of host controlled 'quote' buffer length)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git devsec-tsm-fixes/fixes
+Already up to date.
+Merging drm-rust-fixes/for-linux-next-fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'for-linux-next-fixes' of https://gitlab.freedesktop.org/drm/rust/kernel.git drm-rust-fixes/for-linux-next-fixes
+Already up to date.
+Merging tenstorrent-dt-fixes/tenstorrent-dt-fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'tenstorrent-dt-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git tenstorrent-dt-fixes/tenstorrent-dt-fixes
+Already up to date.
+Merging nfc-fixes/for-linus (e87faad807329 nfc: nci: fix out-of-bounds write in nci_target_auto_activated())
+$ git merge -m Merge branch 'for-linus' of https://codeberg.org/linux-nfc/linux.git nfc-fixes/for-linus
+Merge made by the 'ort' strategy.
+ net/nfc/digital_technology.c |  2 ++
+ net/nfc/llcp_commands.c      | 18 ++++++++++++++++--
+ net/nfc/llcp_core.c          | 29 +++++++++++++++++++++--------
+ net/nfc/llcp_sock.c          | 14 +++++++++++---
+ net/nfc/nci/data.c           | 10 +++++-----
+ net/nfc/nci/ntf.c            | 32 ++++++++++++++++++++++++++++----
+ net/nfc/nci/rsp.c            | 28 ++++++++++++++++++++++++++--
+ 7 files changed, 109 insertions(+), 24 deletions(-)
+Merging drm-misc-fixes/for-linux-next-fixes (faebb7ba1ac65 accel/amdxdna: Fix use-after-free of mm_struct in job scheduler)
+$ git merge -m Merge branch 'for-linux-next-fixes' of https://gitlab.freedesktop.org/drm/misc/kernel.git drm-misc-fixes/for-linux-next-fixes
+Merge made by the 'ort' strategy.
+ drivers/accel/amdxdna/amdxdna_ctx.c      | 18 +++++++++++
+ drivers/accel/ivpu/ivpu_fw_log.c         |  4 +++
+ drivers/accel/ivpu/ivpu_hw_btrs.c        |  2 +-
+ drivers/dma-buf/udmabuf.c                |  9 +++---
+ drivers/gpu/drm/drm_gpusvm.c             | 53 ++++++++++++++++++++++----------
+ drivers/gpu/drm/panthor/panthor_device.c |  5 ++-
+ drivers/gpu/drm/panthor/panthor_fw.c     |  1 +
+ drivers/gpu/drm/ttm/ttm_pool.c           | 34 +++++++++++++++++---
+ drivers/gpu/drm/virtio/virtgpu_gem.c     | 14 +++++----
+ drivers/gpu/drm/virtio/virtgpu_kms.c     |  5 ++-
+ drivers/gpu/drm/virtio/virtgpu_vq.c      |  3 --
+ drivers/gpu/host1x/bus.c                 |  4 +--
+ 12 files changed, 114 insertions(+), 38 deletions(-)
+Merging rust/rust-next (7059bdf4f04a3 Merge tag 'for-7.2-rc3-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux)
+$ git merge -m Merge branch 'rust-next' of https://github.com/Rust-for-Linux/linux.git rust/rust-next
+Already up to date.
+Merging rust-interop/interop-next (05f7e89ab9731 Linux 6.19)
+$ git merge -m Merge branch 'interop-next' of https://github.com/Rust-for-Linux/linux.git rust-interop/interop-next
+Already up to date.
+Merging rust-alloc/alloc-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'alloc-next' of https://github.com/Rust-for-Linux/linux.git rust-alloc/alloc-next
+Already up to date.
+Merging rust-io/io-next (86731a2a651e5 Linux 6.16-rc3)
+$ git merge -m Merge branch 'io-next' of https://github.com/Rust-for-Linux/linux.git rust-io/io-next
+Already up to date.
+Merging rust-pin-init/pin-init-next (0c20f77a26b89 rust: pin-init: make `[pin_]chain` unwind safe)
+$ git merge -m Merge branch 'pin-init-next' of https://github.com/Rust-for-Linux/linux.git rust-pin-init/pin-init-next
+Merge made by the 'ort' strategy.
+ rust/pin-init/examples/mutex.rs        |   5 +-
+ rust/pin-init/internal/src/init.rs     |   8 +-
+ rust/pin-init/internal/src/pin_data.rs |  10 +--
+ rust/pin-init/src/lib.rs               | 151 ++++++++++++++++++++-------------
+ 4 files changed, 101 insertions(+), 73 deletions(-)
+Merging rust-timekeeping/timekeeping-next (ddb1444d33351 hrtimer: add usage examples to documentation)
+$ git merge -m Merge branch 'timekeeping-next' of https://github.com/Rust-for-Linux/linux.git rust-timekeeping/timekeeping-next
+Already up to date.
+Merging rust-xarray/xarray-next (c455f19bbe610 rust: xarray: add __rust_helper to helpers)
+$ git merge -m Merge branch 'xarray-next' of https://github.com/Rust-for-Linux/linux.git rust-xarray/xarray-next
+Already up to date.
+Merging rust-analyzer/rust-analyzer-next (5f45afb8ab04d scripts: generate_rust_analyzer.py: pass cfg to macros crate)
+$ git merge -m Merge branch 'rust-analyzer-next' of https://github.com/Rust-for-Linux/linux.git rust-analyzer/rust-analyzer-next
+Auto-merging scripts/generate_rust_analyzer.py
+Merge made by the 'ort' strategy.
+ scripts/generate_rust_analyzer.py | 1 +
+ 1 file changed, 1 insertion(+)
+Merging mm-stable/mm-stable (0e35b9b6ec0ff Merge tag 'mm-hotfixes-stable-2026-07-06-17-49' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm)
+$ git merge -m Merge branch 'mm-stable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-stable/mm-stable
+Already up to date.
+Merging mm-nonmm-stable/mm-nonmm-stable (0e35b9b6ec0ff Merge tag 'mm-hotfixes-stable-2026-07-06-17-49' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm)
+$ git merge -m Merge branch 'mm-nonmm-stable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-nonmm-stable/mm-nonmm-stable
+Already up to date.
+Merging mm-unstable/mm-unstable (e549221365405 mm/damon/sysfs: read ops_id only once in damon_sysfs_apply_inputs())
+  28a5d16195e4d ("mm/mm_init: handle alloc_percpu failure in free_area_init_core_hotplug")
+$ git merge -m Merge branch 'mm-unstable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-unstable/mm-unstable
+Auto-merging CREDITS
+Auto-merging MAINTAINERS
+Auto-merging arch/riscv/Kconfig
+Auto-merging drivers/accel/amdxdna/aie2_ctx.c
+Auto-merging drivers/gpu/drm/drm_gpusvm.c
+Auto-merging fs/smb/client/file.c
+Auto-merging include/linux/alloc_tag.h
+Auto-merging include/linux/memory_hotplug.h
+Auto-merging init/Kconfig
+Auto-merging kernel/cgroup/cpuset.c
+Auto-merging kernel/events/core.c
+Auto-merging kernel/trace/ring_buffer.c
+Auto-merging mm/alloc_tag.c
+Auto-merging mm/memory_hotplug.c
+Auto-merging mm/mm_init.c
+Auto-merging mm/slab.h
+Auto-merging mm/slub.c
+Auto-merging mm/sparse-vmemmap.c
+Merge made by the 'ort' strategy.
+ CREDITS                                            |   4 +
+ Documentation/ABI/testing/sysfs-block-zram         |   6 +-
+ Documentation/ABI/testing/sysfs-bus-dax            |  24 +
+ Documentation/ABI/testing/sysfs-kernel-mm-damon    |  75 +-
+ Documentation/admin-guide/cgroup-v1/cpusets.rst    |   2 +-
+ Documentation/admin-guide/kernel-parameters.txt    |   7 +-
+ Documentation/admin-guide/mm/damon/usage.rst       |  19 +-
+ Documentation/admin-guide/mm/pagemap.rst           |  13 +-
+ Documentation/admin-guide/mm/transhuge.rst         |   2 +-
+ Documentation/admin-guide/mm/userfaultfd.rst       | 269 +++++-
+ Documentation/admin-guide/sysctl/vm.rst            |  83 +-
+ Documentation/core-api/maple_tree.rst              |  21 +-
+ Documentation/core-api/mm-api.rst                  |   3 +-
+ Documentation/dev-tools/kmemleak.rst               |   8 +
+ Documentation/driver-api/basics.rst                |   3 +
+ Documentation/filesystems/locking.rst              |   5 +-
+ Documentation/filesystems/proc.rst                 |   1 +
+ Documentation/filesystems/vfs.rst                  |   4 +-
+ Documentation/mm/allocation-profiling.rst          |   5 +
+ Documentation/mm/damon/design.rst                  |  43 +-
+ Documentation/mm/hmm.rst                           |  79 +-
+ Documentation/mm/page_owner.rst                    |  78 +-
+ Documentation/userspace-api/ioctl/ioctl-number.rst |   2 +
+ MAINTAINERS                                        |  36 +-
+ arch/arm/mm/fault-armv.c                           |   4 +-
+ arch/arm/mm/flush.c                                |   2 +-
+ arch/arm64/Kconfig                                 |   3 +-
+ arch/arm64/include/asm/pgtable-prot.h              |   8 +-
+ arch/arm64/include/asm/pgtable.h                   |  51 +-
+ arch/arm64/include/asm/vmalloc.h                   |   8 +-
+ arch/arm64/mm/hugetlbpage.c                        |  17 +-
+ arch/csky/abiv1/inc/abi/cacheflush.h               |   7 +-
+ arch/loongarch/Kconfig                             |   3 +-
+ arch/loongarch/include/asm/pgtable.h               |   4 +-
+ arch/mips/kernel/vdso.c                            |   4 +-
+ arch/nios2/mm/cacheflush.c                         |   2 +-
+ arch/parisc/kernel/cache.c                         |   2 +-
+ arch/powerpc/include/asm/book3s/64/pgtable.h       |  10 +-
+ arch/powerpc/mm/book3s64/radix_pgtable.c           |   7 +-
+ arch/powerpc/mm/book3s64/slice.c                   |   6 +-
+ arch/powerpc/mm/hugetlbpage.c                      |  13 +-
+ arch/powerpc/platforms/Kconfig.cputype             |   3 +-
+ arch/riscv/Kconfig                                 |   3 +-
+ arch/riscv/include/asm/pgtable-bits.h              |  12 +-
+ arch/riscv/include/asm/pgtable.h                   |  67 +-
+ arch/s390/Kconfig                                  |   3 +-
+ arch/s390/include/asm/hugetlb.h                    |  12 +-
+ arch/s390/include/asm/pgtable.h                    |   6 +-
+ arch/s390/mm/vmem.c                                |   4 +-
+ arch/sparc/include/asm/pgtable_64.h                |   4 -
+ arch/x86/Kconfig                                   |   3 +-
+ arch/x86/events/intel/ds.c                         |   6 +-
+ arch/x86/include/asm/pgtable.h                     |  58 +-
+ arch/x86/include/asm/pgtable_types.h               |  16 +-
+ arch/x86/kernel/cpu/sgx/virt.c                     |   3 +-
+ arch/x86/kvm/vmx/vmx.c                             |   2 +-
+ arch/x86/mm/init_64.c                              | 134 +--
+ arch/x86/virt/hw.c                                 |   2 +-
+ drivers/accel/amdxdna/aie2_ctx.c                   |  23 +-
+ drivers/base/arch_numa.c                           |   4 -
+ drivers/base/memory.c                              |   9 +
+ drivers/dax/bus.h                                  |   2 +
+ drivers/dax/kmem.c                                 | 506 +++++++++---
+ drivers/gpu/drm/drm_gem.c                          |   2 +-
+ drivers/gpu/drm/drm_gem_dma_helper.c               |   2 +-
+ drivers/gpu/drm/drm_gem_shmem_helper.c             |   2 +-
+ drivers/gpu/drm/drm_gpusvm.c                       |  61 +-
+ drivers/gpu/drm/etnaviv/etnaviv_gem.c              |   5 +-
+ drivers/gpu/drm/exynos/exynos_drm_gem.c            |   6 +-
+ drivers/gpu/drm/gma500/gem.c                       |   2 +-
+ drivers/gpu/drm/i915/gem/i915_gem_mman.c           |  12 +-
+ drivers/gpu/drm/i915/gem/i915_gem_shmem.c          |   2 +-
+ drivers/gpu/drm/msm/msm_gem.c                      |   5 +-
+ drivers/gpu/drm/nouveau/nouveau_gem.c              |   2 +-
+ drivers/gpu/drm/nouveau/nouveau_svm.c              |  20 +-
+ drivers/gpu/drm/omapdrm/omap_fbdev.c               |   2 +-
+ drivers/gpu/drm/omapdrm/omap_gem.c                 |  11 +-
+ drivers/gpu/drm/panthor/panthor_gem.c              |   2 +-
+ drivers/gpu/drm/rockchip/rockchip_drm_gem.c        |   2 +-
+ drivers/gpu/drm/tegra/gem.c                        |   5 +-
+ drivers/gpu/drm/ttm/ttm_backup.c                   |   2 +-
+ drivers/gpu/drm/ttm/ttm_bo_vm.c                    |   7 +-
+ drivers/gpu/drm/virtio/virtgpu_vram.c              |   2 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c         |   2 +-
+ drivers/gpu/drm/xe/xe_device.c                     |   2 +-
+ drivers/gpu/drm/xe/xe_mmio_gem.c                   |   2 +-
+ drivers/gpu/drm/xen/xen_drm_front_gem.c            |   2 +-
+ drivers/hv/mshv_regions.c                          |  54 +-
+ drivers/infiniband/core/umem_odp.c                 |  18 +-
+ drivers/misc/sgi-xp/xpc_uv.c                       |   5 +-
+ drivers/net/ethernet/fungible/funeth/funeth_rx.c   |   2 +-
+ drivers/vfio/pci/nvgrace-gpu/main.c                |   3 +-
+ drivers/vfio/pci/vfio_pci_core.c                   |   3 +-
+ drivers/video/fbdev/core/fb_io_fops.c              |   2 +-
+ fs/aio.c                                           |   2 +-
+ fs/dax.c                                           |   2 +-
+ fs/hugetlbfs/inode.c                               |  18 +-
+ fs/nfs/file.c                                      |   4 +-
+ fs/proc/task_mmu.c                                 | 119 ++-
+ fs/smb/client/file.c                               |   4 +-
+ include/asm-generic/fixmap.h                       |   4 +-
+ include/asm-generic/getorder.h                     |   4 +-
+ include/asm-generic/hugetlb.h                      |  18 +-
+ include/asm-generic/memory_model.h                 |   6 +-
+ include/asm-generic/mmu.h                          |   2 +-
+ include/asm-generic/pgtable-nop4d.h                |   4 +-
+ include/asm-generic/pgtable-nopmd.h                |   4 +-
+ include/asm-generic/pgtable-nopud.h                |   4 +-
+ include/asm-generic/pgtable_uffd.h                 |  32 +-
+ include/linux/alloc_tag.h                          |   4 +-
+ include/linux/backing-dev.h                        |   2 -
+ include/linux/bootmem_info.h                       |  89 --
+ include/linux/codetag.h                            |   2 +
+ include/linux/cpuset.h                             |   4 +-
+ include/linux/damon.h                              | 160 ++--
+ include/linux/gfp.h                                |  54 +-
+ include/linux/gfp_types.h                          |  12 +-
+ include/linux/hmm.h                                |   2 +
+ include/linux/huge_mm.h                            |  20 +-
+ include/linux/hugetlb.h                            |  57 +-
+ include/linux/hugetlb_cgroup.h                     |   2 +-
+ include/linux/leafops.h                            |  32 +-
+ include/linux/lockdep.h                            |   3 +
+ include/linux/lockdep_types.h                      |   3 +-
+ include/linux/maple_tree.h                         |  14 +-
+ include/linux/memfd.h                              |   6 +-
+ include/linux/memory.h                             |  27 +
+ include/linux/memory_hotplug.h                     |  14 +
+ include/linux/migrate.h                            |   8 +-
+ include/linux/migrate_mode.h                       |   1 +
+ include/linux/mm.h                                 | 269 ++++--
+ include/linux/mm_inline.h                          |   8 +-
+ include/linux/mm_types.h                           |   3 +
+ include/linux/mmap_lock.h                          |   8 +
+ include/linux/mmzone.h                             |  43 +-
+ include/linux/oom.h                                |   2 +-
+ include/linux/page-flags.h                         |  14 -
+ include/linux/page_owner.h                         |   7 +-
+ include/linux/pagemap.h                            |  35 +-
+ include/linux/percpu-defs.h                        |   4 +-
+ include/linux/percpu-refcount.h                    |   5 +-
+ include/linux/pfn.h                                |   2 +-
+ include/linux/pgtable.h                            |  71 +-
+ include/linux/radix-tree.h                         |   5 +-
+ include/linux/sched.h                              |   1 +
+ include/linux/sched/mm.h                           |  13 +-
+ include/linux/shmem_fs.h                           |   5 +-
+ include/linux/skbuff.h                             |   2 +-
+ include/linux/swap.h                               |  90 +-
+ include/linux/swapops.h                            |  10 +-
+ include/linux/userfaultfd_k.h                      | 100 ++-
+ include/linux/vm_event_item.h                      |   4 +
+ include/linux/vmalloc.h                            |   4 +-
+ include/linux/vmpressure.h                         |  48 +-
+ include/linux/writeback.h                          |   2 +-
+ include/trace/events/damon.h                       |   8 +-
+ include/trace/events/huge_memory.h                 |   2 +-
+ include/trace/events/migrate.h                     |  11 +-
+ include/trace/events/mmflags.h                     |  17 +-
+ include/trace/events/pagemap.h                     |   8 +
+ include/uapi/linux/alloc_tag.h                     |  99 +++
+ include/uapi/linux/fs.h                            |   1 +
+ include/uapi/linux/mempolicy.h                     |   2 +-
+ include/uapi/linux/userfaultfd.h                   |  54 +-
+ init/Kconfig                                       |   8 +
+ ipc/shm.c                                          |   3 +-
+ kernel/cgroup/cpuset.c                             |   2 +-
+ kernel/dma/coherent.c                              |   7 +-
+ kernel/dma/direct.c                                |   6 +-
+ kernel/dma/mapping.c                               |   8 +-
+ kernel/dma/ops_helpers.c                           |   4 +-
+ kernel/events/core.c                               |  20 +-
+ kernel/events/uprobes.c                            |  13 +-
+ kernel/kcov.c                                      |   2 +-
+ kernel/liveupdate/kexec_handover.c                 |   3 +-
+ kernel/locking/lockdep.c                           |  58 +-
+ kernel/trace/ring_buffer.c                         |   3 +-
+ lib/Kconfig.debug                                  |  28 -
+ lib/Makefile                                       |   1 -
+ lib/codetag.c                                      |  18 +
+ lib/maple_tree.c                                   | 361 +++++---
+ lib/percpu-refcount.c                              |   2 +-
+ lib/radix-tree.c                                   |   7 +-
+ lib/test_hmm.c                                     | 109 ++-
+ lib/test_hmm_uapi.h                                |   1 +
+ mm/Kconfig                                         |  33 +-
+ mm/Kconfig.debug                                   |  28 +
+ mm/Makefile                                        |   4 +-
+ {lib => mm}/alloc_tag.c                            | 368 ++++++++-
+ mm/bootmem_info.c                                  |  72 --
+ mm/cma.c                                           |   4 +-
+ mm/compaction.c                                    |   8 +-
+ mm/damon/core.c                                    | 646 ++++++++++-----
+ mm/damon/lru_sort.c                                |  13 +-
+ mm/damon/ops-common.c                              |   2 +-
+ mm/damon/paddr.c                                   |  36 +-
+ mm/damon/reclaim.c                                 |  11 +-
+ mm/damon/stat.c                                    |   2 +-
+ mm/damon/sysfs-schemes.c                           |  31 +-
+ mm/damon/sysfs.c                                   | 200 +++--
+ mm/damon/tests/core-kunit.h                        | 245 ++++--
+ mm/damon/vaddr.c                                   |  42 +-
+ mm/debug.c                                         |   7 +-
+ mm/debug_vm_pgtable.c                              |  16 +-
+ mm/execmem.c                                       |   1 +
+ mm/filemap.c                                       |  50 +-
+ mm/{swap.c => folio.c}                             |  68 +-
+ mm/gup.c                                           |  13 +-
+ mm/hmm.c                                           | 277 +++++--
+ mm/huge_memory.c                                   | 227 +++--
+ mm/hugetlb.c                                       | 728 +++++++++-------
+ mm/hugetlb_cma.c                                   | 140 +++-
+ mm/hugetlb_cma.h                                   |   8 +-
+ mm/hugetlb_vmemmap.c                               | 141 +---
+ mm/hugetlb_vmemmap.h                               |   5 -
+ mm/internal.h                                      | 598 +++-----------
+ mm/interval_tree.c                                 | 111 ++-
+ mm/kasan/hw_tags.c                                 |   2 +-
+ mm/khugepaged.c                                    |  54 +-
+ mm/kmemleak.c                                      | 181 +++-
+ mm/kmsan/hooks.c                                   |   1 +
+ mm/kmsan/init.c                                    |   2 +-
+ mm/kmsan/shadow.c                                  |   1 +
+ mm/ksm.c                                           |  95 ++-
+ mm/madvise.c                                       |  32 +-
+ mm/mapping_dirty_helpers.c                         |   2 +-
+ mm/memblock.c                                      |   1 +
+ mm/memcontrol-v1.c                                 | 309 ++++++-
+ mm/memcontrol-v1.h                                 |   6 -
+ mm/memcontrol.c                                    |  48 +-
+ mm/memfd.c                                         |  16 +-
+ mm/memory-failure.c                                | 119 ++-
+ mm/memory.c                                        | 345 +++++---
+ mm/memory_hotplug.c                                | 166 +++-
+ mm/mempolicy.c                                     |  53 +-
+ mm/migrate.c                                       |  95 ++-
+ mm/migrate_device.c                                |  42 +-
+ mm/mincore.c                                       |  11 +-
+ mm/mlock.c                                         | 101 +--
+ mm/mm_init.c                                       |  90 +-
+ mm/mm_init.h                                       | 120 +++
+ mm/mm_slot.h                                       |  11 +
+ mm/mmap.c                                          | 205 +++--
+ mm/mmu_notifier.c                                  |   2 +-
+ mm/mmzone.c                                        |   5 +-
+ mm/mprotect.c                                      | 117 ++-
+ mm/mremap.c                                        |  67 +-
+ mm/msync.c                                         |   4 +-
+ mm/nommu.c                                         |  68 +-
+ mm/page-writeback.c                                |  10 -
+ mm/page_alloc.c                                    | 407 +++++----
+ mm/page_alloc.h                                    | 312 +++++++
+ mm/page_frag_cache.c                               |   6 +-
+ mm/page_io.c                                       | 588 ++++++-------
+ mm/page_isolation.c                                |   1 +
+ mm/page_owner.c                                    | 303 +++++--
+ mm/page_reporting.c                                |   1 +
+ mm/page_table_check.c                              |   8 +-
+ mm/page_vma_mapped.c                               |   8 +-
+ mm/pagewalk.c                                      |   4 +-
+ mm/percpu-km.c                                     |   9 +-
+ mm/percpu-vm.c                                     |  49 +-
+ mm/percpu.c                                        |  20 +-
+ mm/rmap.c                                          | 477 ++++++-----
+ mm/shmem.c                                         |  43 +-
+ mm/show_mem.c                                      |  10 +-
+ mm/shuffle.c                                       |   1 +
+ mm/slab.h                                          |   2 +-
+ mm/slub.c                                          |  17 +-
+ mm/sparse-vmemmap.c                                |  28 +-
+ mm/sparse.c                                        |  53 +-
+ mm/sparse.h                                        |  63 ++
+ mm/swap.h                                          |  75 +-
+ mm/swap_state.c                                    | 105 ++-
+ mm/swapfile.c                                      |  42 +-
+ mm/userfaultfd.c                                   | 403 ++++++++-
+ mm/util.c                                          |  14 +-
+ mm/vma.c                                           | 309 ++++---
+ mm/vma.h                                           |  88 +-
+ mm/vma_exec.c                                      |  32 +-
+ mm/vma_init.c                                      |   6 +-
+ mm/vma_internal.h                                  |   4 +-
+ mm/vmalloc.c                                       | 296 +++++--
+ mm/vmalloc.h                                       |  46 ++
+ mm/vmpressure.c                                    | 301 +------
+ mm/vmscan.c                                        |  66 +-
+ mm/vmstat.c                                        |   9 +-
+ mm/zsmalloc.c                                      | 230 +++++-
+ mm/zswap.c                                         |   6 +-
+ samples/damon/Kconfig                              |   2 +-
+ samples/damon/mtier.c                              |   9 +-
+ samples/damon/prcl.c                               |   9 +-
+ samples/damon/wsse.c                               |   9 +-
+ scripts/gdb/linux/page_owner.py                    |   4 +-
+ sound/core/memalloc.c                              |   2 +-
+ tools/include/linux/gfp_types.h                    |   7 -
+ tools/include/uapi/linux/fs.h                      |   1 +
+ tools/include/uapi/linux/userfaultfd.h             |  54 +-
+ tools/mm/.gitignore                                |   1 +
+ tools/mm/Makefile                                  |   4 +-
+ tools/mm/page_owner_filter.c                       | 310 +++++++
+ tools/mm/page_owner_sort.c                         |  87 +-
+ tools/testing/memblock/internal.h                  |  20 -
+ tools/testing/memblock/mm_init.h                   |  24 +
+ tools/testing/radix-tree/maple.c                   |   4 +-
+ tools/testing/selftests/Makefile                   |   2 +
+ tools/testing/selftests/alloc_tag/Makefile         |   9 +
+ .../selftests/alloc_tag/allocinfo_ioctl_test.c     | 548 ++++++++++++
+ tools/testing/selftests/damon/Makefile             |   1 +
+ tools/testing/selftests/damon/_damon_sysfs.py      |  23 +-
+ .../selftests/damon/damos_apply_interval.py        |   2 +-
+ tools/testing/selftests/damon/damos_quota_goal.py  |   2 +-
+ .../testing/selftests/damon/damos_tried_regions.py |   4 +-
+ .../selftests/damon/drgn_dump_damon_status.py      |   3 +-
+ tools/testing/selftests/damon/sysfs.py             |  35 +-
+ tools/testing/selftests/damon/sysfs.sh             |  79 +-
+ tools/testing/selftests/damon/sysfs_refresh.py     |  75 ++
+ ..._update_schemes_tried_regions_wss_estimation.py |   2 +-
+ tools/testing/selftests/dax/Makefile               |   6 +
+ tools/testing/selftests/dax/config                 |   4 +
+ tools/testing/selftests/dax/dax-kmem-hotplug.sh    | 317 +++++++
+ tools/testing/selftests/dax/settings               |   1 +
+ tools/testing/selftests/mincore/mincore_selftest.c |  12 +-
+ tools/testing/selftests/mm/Makefile                |   6 +-
+ tools/testing/selftests/mm/hmm-tests.c             | 792 ++++++++----------
+ tools/testing/selftests/mm/hugepage_settings.c     |   2 +-
+ tools/testing/selftests/mm/hugetlb-vmemmap.c       | 132 ---
+ tools/testing/selftests/mm/hwpoison-panic.sh       | 255 ++++++
+ .../testing/selftests/mm/ksft_kmemleak_confirm.sh  | 132 +++
+ tools/testing/selftests/mm/ksm_tests.c             |  28 +-
+ tools/testing/selftests/mm/pkey-helpers.h          |   4 +-
+ tools/testing/selftests/mm/pkey-powerpc.h          |   2 +-
+ tools/testing/selftests/mm/pkey_sighandler_tests.c | 115 +--
+ tools/testing/selftests/mm/pkey_util.c             |  90 ++
+ tools/testing/selftests/mm/protection_keys.c       |  99 +--
+ tools/testing/selftests/mm/rmap.c                  |  81 ++
+ tools/testing/selftests/mm/run_vmtests.sh          |   3 +-
+ tools/testing/selftests/mm/uffd-common.c           |   9 +-
+ tools/testing/selftests/mm/uffd-unit-tests.c       | 919 +++++++++++++++++++++
+ tools/testing/selftests/mm/vm_util.c               |  26 +-
+ tools/testing/selftests/mm/vm_util.h               |   1 +
+ tools/testing/shared/linux/kmemleak.h              |   1 +
+ tools/testing/vma/Makefile                         |   2 +-
+ tools/testing/vma/include/dup.h                    |  52 +-
+ tools/testing/vma/include/stubs.h                  |  16 +-
+ tools/testing/vma/shared.c                         |   9 -
+ tools/testing/vma/shared.h                         |  43 +-
+ tools/testing/vma/tests/merge.c                    |  40 +-
+ tools/testing/vma/vma_internal.h                   |   4 +-
+ virt/kvm/guest_memfd.c                             |   2 +-
+ 350 files changed, 13483 insertions(+), 6450 deletions(-)
+ delete mode 100644 include/linux/bootmem_info.h
+ create mode 100644 include/uapi/linux/alloc_tag.h
+ rename {lib => mm}/alloc_tag.c (73%)
+ delete mode 100644 mm/bootmem_info.c
+ rename mm/{swap.c => folio.c} (95%)
+ create mode 100644 mm/mm_init.h
+ create mode 100644 mm/page_alloc.h
+ create mode 100644 mm/sparse.h
+ create mode 100644 mm/vmalloc.h
+ create mode 100644 tools/mm/page_owner_filter.c
+ create mode 100644 tools/testing/memblock/mm_init.h
+ create mode 100644 tools/testing/selftests/alloc_tag/Makefile
+ create mode 100644 tools/testing/selftests/alloc_tag/allocinfo_ioctl_test.c
+ create mode 100755 tools/testing/selftests/damon/sysfs_refresh.py
+ create mode 100644 tools/testing/selftests/dax/Makefile
+ create mode 100644 tools/testing/selftests/dax/config
+ create mode 100755 tools/testing/selftests/dax/dax-kmem-hotplug.sh
+ create mode 100644 tools/testing/selftests/dax/settings
+ delete mode 100644 tools/testing/selftests/mm/hugetlb-vmemmap.c
+ create mode 100755 tools/testing/selftests/mm/hwpoison-panic.sh
+ create mode 100755 tools/testing/selftests/mm/ksft_kmemleak_confirm.sh
+$ git am -3 ../patches/0001-mm-Fix-up-build-for-uninitialised-pte.patch
+Applying: mm: Fix up build for uninitialised pte
+Using index info to reconstruct a base tree...
+M	mm/khugepaged.c
+Falling back to patching base and 3-way merge...
+Auto-merging mm/khugepaged.c
+No changes -- Patch already applied.
+Merging mm-nonmm-unstable/mm-nonmm-unstable (8ce79abc8a125 tools/accounting: fix macro typos in getdelays)
+$ git merge -m Merge branch 'mm-nonmm-unstable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-nonmm-unstable/mm-nonmm-unstable
+Auto-merging .mailmap
+Auto-merging arch/riscv/include/asm/cacheflush.h
+Auto-merging arch/riscv/include/asm/pgtable.h
+Auto-merging arch/riscv/kernel/entry.S
+Auto-merging kernel/signal.c
+Auto-merging lib/Kconfig.debug
+Merge made by the 'ort' strategy.
+ .get_maintainer.ignore                             |   1 +
+ .mailmap                                           |   1 +
+ Documentation/accounting/delay-accounting.rst      |  45 +++-
+ arch/riscv/include/asm/cacheflush.h                |  15 +-
+ arch/riscv/include/asm/pgtable.h                   |   9 +-
+ arch/riscv/kernel/entry.S                          |  11 +-
+ drivers/media/v4l2-core/v4l2-vp9.c                 |  30 +--
+ drivers/pps/clients/pps-gpio.c                     |  66 ++---
+ drivers/pps/kc.c                                   |  10 +
+ drivers/pps/pps.c                                  |  10 +-
+ drivers/rapidio/rio-scan.c                         |   1 +
+ fs/fat/inode.c                                     |   5 +-
+ fs/fat/misc.c                                      |   4 +
+ fs/fat/nfs.c                                       |   4 +-
+ fs/ocfs2/cluster/heartbeat.c                       | 134 ++++++----
+ fs/ocfs2/cluster/nodemanager.c                     |  19 +-
+ fs/ocfs2/cluster/nodemanager.h                     |   2 +
+ fs/ocfs2/dir.c                                     |  20 +-
+ fs/ocfs2/dlm/dlmmaster.c                           |   6 +
+ fs/ocfs2/dlm/dlmrecovery.c                         |   9 +
+ fs/ocfs2/inode.c                                   |   6 +-
+ fs/ocfs2/namei.c                                   |  17 +-
+ fs/ocfs2/refcounttree.c                            |  27 ++
+ fs/ocfs2/xattr.c                                   | 215 +++++++++++++++-
+ fs/ocfs2/xattr.h                                   |   2 +
+ fs/ramfs/file-nommu.c                              |   3 +
+ include/linux/nmi.h                                |   4 +-
+ include/linux/pps_kernel.h                         |   1 +
+ include/linux/taskstats_kern.h                     |   1 -
+ ipc/ipc_sysctl.c                                   |   2 +-
+ kernel/fork.c                                      |   2 +-
+ kernel/params.c                                    |   4 +-
+ kernel/reboot.c                                    | 169 +++++-------
+ kernel/resource.c                                  |   2 +-
+ kernel/signal.c                                    |  51 ++--
+ kernel/taskstats.c                                 |   2 +-
+ kernel/watchdog.c                                  |  16 +-
+ lib/Kconfig                                        |   6 -
+ lib/Kconfig.debug                                  |  26 ++
+ lib/idr.c                                          |  55 ++--
+ lib/math/tests/Makefile                            |   1 +
+ lib/math/tests/polynomial_kunit.c                  | 270 +++++++++++++++++++
+ lib/raid/Kconfig                                   |  12 +
+ lib/raid/raid6/Makefile                            |   2 +
+ lib/raid/raid6/algos.c                             | 102 ++++----
+ lib/raid/raid6/tests/raid6_kunit.c                 |  66 ++++-
+ lib/raid/xor/Makefile                              |   2 +
+ lib/raid/xor/tests/Makefile                        |   2 +
+ lib/raid/xor/tests/xor_kunit.c                     |  64 ++++-
+ lib/raid/xor/xor-core.c                            |  55 ++--
+ lib/random32.c                                     | 184 +------------
+ lib/string.c                                       |   3 +-
+ lib/test_ida.c                                     |  14 +
+ lib/tests/Makefile                                 |   3 +-
+ lib/tests/random32_kunit.c                         | 182 +++++++++++++
+ lib/xz/xz_dec_bcj.c                                |   2 +-
+ lib/xz/xz_dec_lzma2.c                              |  63 +++--
+ lib/xz/xz_dec_stream.c                             |   4 +-
+ lib/xz/xz_lzma2.h                                  |   2 +-
+ scripts/checkpatch.pl                              |   6 +
+ tools/accounting/delaytop.c                        | 286 ++++++++++++++++++---
+ tools/accounting/getdelays.c                       |  37 +--
+ tools/include/linux/compiler.h                     |   2 +-
+ tools/testing/selftests/acct/.gitignore            |   1 +
+ tools/testing/selftests/acct/Makefile              |  11 +
+ tools/testing/selftests/acct/cgroupstats.c         | 231 +++++++++++++++++
+ tools/testing/selftests/acct/netlink_helper.c      | 116 +++++++++
+ tools/testing/selftests/acct/netlink_helper.h      |  44 ++++
+ .../selftests/acct/taskstats_fill_stats_tgid.c     | 134 +---------
+ 69 files changed, 2105 insertions(+), 809 deletions(-)
+ create mode 100644 lib/math/tests/polynomial_kunit.c
+ create mode 100644 lib/tests/random32_kunit.c
+ create mode 100644 tools/testing/selftests/acct/cgroupstats.c
+ create mode 100644 tools/testing/selftests/acct/netlink_helper.c
+ create mode 100644 tools/testing/selftests/acct/netlink_helper.h
+Merging kbuild/kbuild-for-next (4859c0d5918e2 scripts/tags.sh: Add support for rust source files)
+$ git merge -m Merge branch 'kbuild-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git kbuild/kbuild-for-next
+Auto-merging Makefile
+Merge made by the 'ort' strategy.
+ Makefile                            |  6 ++++++
+ arch/arm64/include/asm/module.lds.h | 13 ------------
+ scripts/Makefile.build              |  4 ++--
+ scripts/headers_install.sh          |  1 +
+ scripts/kconfig/merge_config.sh     |  2 +-
+ scripts/tags.sh                     | 42 ++++++++++++++++++++++++++++---------
+ 6 files changed, 42 insertions(+), 26 deletions(-)
+Merging clang-format/clang-format (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'clang-format' of https://github.com/ojeda/linux.git clang-format/clang-format
+Already up to date.
+Merging perf/perf-tools-next (3f7909fa921e3 perf stat: Do not open cgroups for BPF counters)
+$ git merge -m Merge branch 'perf-tools-next' of https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git perf/perf-tools-next
+Merge made by the 'ort' strategy.
+ tools/lib/api/fd/array.c                           |   17 +-
+ tools/perf/.clang-format                           |    9 +-
+ tools/perf/Documentation/perf-test.txt             |    6 +-
+ tools/perf/Documentation/topdown.txt               |    4 +-
+ tools/perf/Makefile.perf                           |    9 +-
+ tools/perf/arch/arm/util/cs-etm.c                  |   10 +-
+ tools/perf/arch/arm64/util/arm-spe.c               |    8 +-
+ tools/perf/arch/arm64/util/hisi-ptt.c              |    2 +-
+ tools/perf/arch/x86/tests/amd-ibs-period.c         |   20 +-
+ tools/perf/arch/x86/tests/hybrid.c                 |   22 +-
+ tools/perf/arch/x86/tests/topdown.c                |    4 +-
+ tools/perf/arch/x86/util/auxtrace.c                |    2 +-
+ tools/perf/arch/x86/util/intel-bts.c               |    6 +-
+ tools/perf/arch/x86/util/intel-pt.c                |    9 +-
+ tools/perf/arch/x86/util/iostat.c                  |   14 +-
+ tools/perf/bench/evlist-open-close.c               |   29 +-
+ tools/perf/builtin-annotate.c                      |    7 +-
+ tools/perf/builtin-ftrace.c                        |   14 +-
+ tools/perf/builtin-inject.c                        |    9 +-
+ tools/perf/builtin-kvm.c                           |  123 +-
+ tools/perf/builtin-kwork.c                         |    8 +-
+ tools/perf/builtin-lock.c                          |    4 +-
+ tools/perf/builtin-record.c                        |  146 +-
+ tools/perf/builtin-report.c                        |    6 +-
+ tools/perf/builtin-sched.c                         |   61 +-
+ tools/perf/builtin-script.c                        |   16 +-
+ tools/perf/builtin-stat.c                          |  113 +-
+ tools/perf/builtin-timechart.c                     |  212 +-
+ tools/perf/builtin-top.c                           |  104 +-
+ tools/perf/builtin-trace.c                         |   65 +-
+ .../pmu-events/arch/x86/alderlake/adl-metrics.json |   80 +-
+ .../perf/pmu-events/arch/x86/alderlake/cache.json  |   36 +
+ .../arch/x86/alderlake/metricgroups.json           |    2 +-
+ .../pmu-events/arch/x86/alderlake/pipeline.json    |    6 +-
+ .../perf/pmu-events/arch/x86/alderlaken/cache.json |   32 +
+ .../arch/x86/amdzen5/floating-point.json           |  160 +-
+ .../pmu-events/arch/x86/amdzen5/load-store.json    |    8 +-
+ .../arch/x86/amdzen6/floating-point.json           |   34 +-
+ .../pmu-events/arch/x86/arrowlake/arl-metrics.json |  152 +-
+ .../perf/pmu-events/arch/x86/arrowlake/cache.json  |   30 +-
+ .../arch/x86/arrowlake/floating-point.json         |   45 +
+ .../perf/pmu-events/arch/x86/arrowlake/memory.json |   18 +
+ .../arch/x86/arrowlake/metricgroups.json           |    2 +-
+ .../pmu-events/arch/x86/arrowlake/pipeline.json    |  131 +-
+ .../arch/x86/arrowlake/uncore-memory.json          |   84 +-
+ .../arch/x86/clearwaterforest/memory.json          |   33 +
+ .../arch/x86/clearwaterforest/pipeline.json        |   19 +
+ .../x86/clearwaterforest/uncore-interconnect.json  |   10 +
+ .../arch/x86/clearwaterforest/uncore-memory.json   |    2 +-
+ .../pmu-events/arch/x86/emeraldrapids/cache.json   |    9 +
+ .../arch/x86/emeraldrapids/emr-metrics.json        |  135 +-
+ .../arch/x86/emeraldrapids/metricgroups.json       |    2 +-
+ .../perf/pmu-events/arch/x86/grandridge/cache.json |   25 +
+ .../pmu-events/arch/x86/grandridge/counter.json    |   42 -
+ .../arch/x86/grandridge/floating-point.json        |   56 +
+ .../pmu-events/arch/x86/grandridge/memory.json     |   57 +
+ .../pmu-events/arch/x86/grandridge/pipeline.json   |  140 ++
+ .../arch/x86/grandridge/uncore-memory.json         |    2 +-
+ .../arch/x86/grandridge/virtual-memory.json        |    8 +
+ .../arch/x86/graniterapids/gnr-metrics.json        |  102 +-
+ .../arch/x86/graniterapids/metricgroups.json       |    2 +-
+ .../arch/x86/graniterapids/pipeline.json           |    2 +-
+ .../x86/graniterapids/uncore-interconnect.json     |   10 +
+ .../arch/x86/graniterapids/uncore-memory.json      |    2 +-
+ .../perf/pmu-events/arch/x86/lunarlake/cache.json  |    2 +-
+ .../pmu-events/arch/x86/lunarlake/lnl-metrics.json |  142 +-
+ .../arch/x86/lunarlake/metricgroups.json           |    2 +-
+ .../pmu-events/arch/x86/lunarlake/pipeline.json    |   29 +-
+ .../arch/x86/lunarlake/uncore-memory.json          |  208 +-
+ tools/perf/pmu-events/arch/x86/mapfile.csv         |   25 +-
+ .../perf/pmu-events/arch/x86/meteorlake/cache.json |   11 +-
+ .../arch/x86/meteorlake/metricgroups.json          |    2 +-
+ .../arch/x86/meteorlake/mtl-metrics.json           |   88 +-
+ .../pmu-events/arch/x86/meteorlake/pipeline.json   |    4 +-
+ .../arch/x86/meteorlake/uncore-memory.json         |   84 +-
+ tools/perf/pmu-events/arch/x86/novalake/cache.json |  749 ++++++
+ .../perf/pmu-events/arch/x86/novalake/counter.json |   17 +
+ .../arch/x86/novalake/floating-point.json          |  452 ++++
+ .../pmu-events/arch/x86/novalake/frontend.json     |  600 +++++
+ .../perf/pmu-events/arch/x86/novalake/memory.json  |  299 +++
+ tools/perf/pmu-events/arch/x86/novalake/other.json |  144 ++
+ .../pmu-events/arch/x86/novalake/pipeline.json     | 1976 +++++++++++++++
+ .../arch/x86/novalake/uncore-memory.json           |   42 +
+ .../arch/x86/novalake/virtual-memory.json          |  338 +++
+ .../pmu-events/arch/x86/pantherlake/counter.json   |    5 +
+ .../pmu-events/arch/x86/pantherlake/pipeline.json  |   31 +-
+ .../arch/x86/pantherlake/uncore-interconnect.json  |   10 +
+ .../arch/x86/pantherlake/uncore-memory.json        |  221 +-
+ .../pmu-events/arch/x86/sierraforest/counter.json  |   77 -
+ .../arch/x86/sierraforest/uncore-interconnect.json |   10 +
+ .../arch/x86/sierraforest/uncore-memory.json       |    2 +-
+ tools/perf/pmu-events/jevents.py                   |    9 +-
+ tools/perf/pmu-events/metric.py                    |    6 +-
+ tools/perf/python/perf.pyi                         |  672 ++++++
+ tools/perf/python/perf_live.py                     |   59 +
+ tools/perf/scripts/python/arm-cs-trace-disasm.py   |    9 +-
+ tools/perf/scripts/python/powerpc-hcalls.py        |  137 +-
+ tools/perf/tests/backward-ring-buffer.c            |   26 +-
+ tools/perf/tests/builtin-test.c                    |  183 +-
+ tools/perf/tests/code-reading.c                    |   14 +-
+ tools/perf/tests/event-times.c                     |    6 +-
+ tools/perf/tests/event_update.c                    |    4 +-
+ tools/perf/tests/evsel-roundtrip-name.c            |    8 +-
+ tools/perf/tests/evsel-tp-sched.c                  |    4 +-
+ tools/perf/tests/expand-cgroup.c                   |   15 +-
+ tools/perf/tests/hists_cumulate.c                  |    2 +-
+ tools/perf/tests/hists_filter.c                    |    2 +-
+ tools/perf/tests/hists_link.c                      |    2 +-
+ tools/perf/tests/hists_output.c                    |    2 +-
+ tools/perf/tests/hwmon_pmu.c                       |    7 +-
+ tools/perf/tests/keep-tracking.c                   |   10 +-
+ tools/perf/tests/mmap-basic.c                      |   24 +-
+ tools/perf/tests/openat-syscall-all-cpus.c         |    6 +-
+ tools/perf/tests/openat-syscall-tp-fields.c        |   26 +-
+ tools/perf/tests/openat-syscall.c                  |    6 +-
+ tools/perf/tests/parse-events.c                    |  150 +-
+ tools/perf/tests/parse-metric.c                    |   11 +-
+ tools/perf/tests/parse-no-sample-id-all.c          |    2 +-
+ tools/perf/tests/perf-record.c                     |   38 +-
+ tools/perf/tests/perf-time-to-tsc.c                |   12 +-
+ tools/perf/tests/pfm.c                             |   12 +-
+ tools/perf/tests/pmu-events.c                      |   21 +-
+ tools/perf/tests/pmu.c                             |    4 +-
+ tools/perf/tests/sample-parsing.c                  |   45 +-
+ tools/perf/tests/shell/coresight/callchain.sh      |  172 ++
+ .../shell/coresight/test_arm_coresight_disasm.sh   |    4 +-
+ tools/perf/tests/shell/inject_aslr.sh              |   10 +-
+ tools/perf/tests/shell/jitdump-python.sh           |   69 +-
+ tools/perf/tests/shell/kvm.sh                      |   71 +-
+ .../perf/tests/shell/lib/perf_metric_validation.py |   11 +-
+ tools/perf/tests/shell/lib/perf_record.sh          |   58 +
+ tools/perf/tests/shell/lib/setup_python.sh         |   13 +
+ tools/perf/tests/shell/lock_contention.sh          |   33 +-
+ tools/perf/tests/shell/pipe_test.sh                |    4 +-
+ .../shell/record+zstd_comp_decomp_multi_record.sh  |   63 +
+ tools/perf/tests/shell/record.sh                   |  216 +-
+ tools/perf/tests/shell/record_lbr.sh               |   50 +-
+ tools/perf/tests/shell/record_offcpu.sh            |   14 +-
+ tools/perf/tests/shell/stat.sh                     |   12 +
+ tools/perf/tests/shell/stat_all_metrics.sh         |   77 +-
+ tools/perf/tests/shell/stat_bpf_counters.sh        |   28 +-
+ tools/perf/tests/shell/stat_metrics_values.sh      |    9 +-
+ tools/perf/tests/shell/test_brstack.sh             |   99 +-
+ tools/perf/tests/shell/timechart.sh                |   24 +-
+ tools/perf/tests/shell/trace_record_replay.sh      |   38 +-
+ tools/perf/tests/sw-clock.c                        |   20 +-
+ tools/perf/tests/switch-tracking.c                 |   11 +-
+ tools/perf/tests/task-exit.c                       |   20 +-
+ tools/perf/tests/tests.h                           |    1 +
+ tools/perf/tests/time-utils-test.c                 |   14 +-
+ tools/perf/tests/tool_pmu.c                        |    7 +-
+ tools/perf/tests/topology.c                        |    4 +-
+ tools/perf/tests/uncore-event-sorting.c            |    6 +-
+ tools/perf/tests/workloads/Build                   |    2 +
+ tools/perf/tests/workloads/callchain.c             |   33 +
+ tools/perf/tests/workloads/noploop.c               |   17 +-
+ tools/perf/tests/workloads/thloop.c                |   16 +-
+ tools/perf/ui/browsers/annotate.c                  |    2 +-
+ tools/perf/ui/browsers/hists.c                     |   22 +-
+ tools/perf/util/Build                              |    1 -
+ tools/perf/util/amd-sample-raw.c                   |    2 +-
+ tools/perf/util/annotate-data.c                    |    2 +-
+ tools/perf/util/annotate.c                         |   10 +-
+ tools/perf/util/aslr.c                             |    1 +
+ tools/perf/util/auxtrace.c                         |   14 +-
+ tools/perf/util/block-info.c                       |    4 +-
+ tools/perf/util/bpf_counter.c                      |    2 +-
+ tools/perf/util/bpf_counter_cgroup.c               |   14 +-
+ tools/perf/util/bpf_ftrace.c                       |    9 +-
+ tools/perf/util/bpf_lock_contention.c              |   12 +-
+ tools/perf/util/bpf_off_cpu.c                      |   44 +-
+ tools/perf/util/bpf_trace_augment.c                |    8 +-
+ tools/perf/util/capstone.c                         |   13 +-
+ tools/perf/util/cgroup.c                           |   26 +-
+ tools/perf/util/compress.h                         |    6 +-
+ tools/perf/util/cs-etm.c                           |  382 +--
+ tools/perf/util/data-convert-bt.c                  |    2 +-
+ tools/perf/util/data-convert-json.c                |    1 +
+ tools/perf/util/data.c                             |   27 +-
+ tools/perf/util/data.h                             |    4 +-
+ tools/perf/util/dso.h                              |   57 +-
+ tools/perf/util/evlist.c                           |  494 ++--
+ tools/perf/util/evlist.h                           |  273 ++-
+ tools/perf/util/evsel.c                            |   41 +-
+ tools/perf/util/evsel.h                            |   42 +-
+ tools/perf/util/expr.c                             |    2 +-
+ tools/perf/util/header.c                           |   69 +-
+ tools/perf/util/header.h                           |    2 +-
+ tools/perf/util/intel-pt.c                         |    8 +-
+ tools/perf/util/intel-tpebs.c                      |    7 +-
+ tools/perf/util/iostat.c                           |    2 +-
+ tools/perf/util/iostat.h                           |    2 +-
+ tools/perf/util/kvm-stat-arch/book3s_hv_exits.h    |    4 +-
+ tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c   |   31 +-
+ tools/perf/util/kvm-stat-arch/kvm-stat-x86.c       |   35 +-
+ tools/perf/util/kvm-stat.c                         |   41 +
+ tools/perf/util/kvm-stat.h                         |   18 +-
+ tools/perf/util/map.h                              |    9 +-
+ tools/perf/util/metricgroup.c                      |   45 +-
+ tools/perf/util/metricgroup.h                      |    4 +-
+ tools/perf/util/parse-events.c                     |   40 +-
+ tools/perf/util/parse-events.h                     |   17 +-
+ tools/perf/util/parse-events.y                     |    2 +-
+ tools/perf/util/perf_api_probe.c                   |   20 +-
+ tools/perf/util/pfm.c                              |    4 +-
+ tools/perf/util/pmu.c                              |    6 +-
+ tools/perf/util/pmus.c                             |    2 +
+ tools/perf/util/print-events.c                     |    2 +-
+ tools/perf/util/probe-file.c                       |    2 +
+ tools/perf/util/python.c                           | 2503 +++++++++++++++++---
+ tools/perf/util/record.c                           |   11 +-
+ tools/perf/util/s390-sample-raw.c                  |   22 +-
+ tools/perf/util/sample-raw.c                       |    4 +-
+ tools/perf/util/sample.c                           |   19 +-
+ tools/perf/util/session.c                          |   69 +-
+ tools/perf/util/session.h                          |    2 +
+ tools/perf/util/setup.py                           |    5 +
+ tools/perf/util/sideband_evlist.c                  |   54 +-
+ tools/perf/util/sort.c                             |    2 +-
+ tools/perf/util/stat-display.c                     |    6 +-
+ tools/perf/util/stat-shadow.c                      |   24 +-
+ tools/perf/util/stat.c                             |   22 +-
+ tools/perf/util/stream.c                           |    4 +-
+ tools/perf/util/symbol-elf.c                       |    4 +
+ tools/perf/util/symbol.c                           |    4 +-
+ tools/perf/util/symbol.h                           |   12 +
+ tools/perf/util/synthetic-events.c                 |   11 +-
+ tools/perf/util/time-utils.c                       |   12 +-
+ tools/perf/util/top.c                              |    4 +-
+ tools/perf/util/zstd.c                             |   25 +-
+ 230 files changed, 12863 insertions(+), 2712 deletions(-)
+ delete mode 100644 tools/perf/pmu-events/arch/x86/grandridge/counter.json
+ create mode 100644 tools/perf/pmu-events/arch/x86/novalake/cache.json
+ create mode 100644 tools/perf/pmu-events/arch/x86/novalake/counter.json
+ create mode 100644 tools/perf/pmu-events/arch/x86/novalake/floating-point.json
+ create mode 100644 tools/perf/pmu-events/arch/x86/novalake/frontend.json
+ create mode 100644 tools/perf/pmu-events/arch/x86/novalake/memory.json
+ create mode 100644 tools/perf/pmu-events/arch/x86/novalake/other.json
+ create mode 100644 tools/perf/pmu-events/arch/x86/novalake/pipeline.json
+ create mode 100644 tools/perf/pmu-events/arch/x86/novalake/uncore-memory.json
+ create mode 100644 tools/perf/pmu-events/arch/x86/novalake/virtual-memory.json
+ create mode 100644 tools/perf/pmu-events/arch/x86/pantherlake/uncore-interconnect.json
+ delete mode 100644 tools/perf/pmu-events/arch/x86/sierraforest/counter.json
+ create mode 100644 tools/perf/python/perf.pyi
+ create mode 100755 tools/perf/python/perf_live.py
+ create mode 100755 tools/perf/tests/shell/coresight/callchain.sh
+ create mode 100644 tools/perf/tests/shell/lib/perf_record.sh
+ create mode 100755 tools/perf/tests/shell/record+zstd_comp_decomp_multi_record.sh
+ create mode 100644 tools/perf/tests/workloads/callchain.c
+Merging compiler-attributes/compiler-attributes (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'compiler-attributes' of https://github.com/ojeda/linux.git compiler-attributes/compiler-attributes
+Already up to date.
+Merging dma-mapping/dma-mapping-for-next (0188616c057d3 dma-direct: Improve readability of the dma_direct_map_sg() for P2PDMA case)
+$ git merge -m Merge branch 'dma-mapping-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git dma-mapping/dma-mapping-for-next
+Auto-merging kernel/dma/coherent.c
+Auto-merging kernel/dma/direct.c
+Merge made by the 'ort' strategy.
+ drivers/iommu/dma-iommu.c | 69 +++++++++++++++++++----------------------------
+ kernel/dma/coherent.c     | 10 +++----
+ kernel/dma/direct.c       |  3 +--
+ 3 files changed, 34 insertions(+), 48 deletions(-)
+Merging asm-generic/master (adbbd9714f805 scripts: headers_install.sh: Remove config leak ignore machinery)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/arnd/asm-generic asm-generic/master
+Already up to date.
+Merging alpha/alpha-next (d58041d2c63e0 MAINTAINERS: Add Magnus Lindholm as maintainer for alpha port)
+$ git merge -m Merge branch 'alpha-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mattst88/alpha.git alpha/alpha-next
+Already up to date.
+Merging arm/for-next (75ca78c7d982a Merge branches 'fixes' and 'misc' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git arm/for-next
+Merge made by the 'ort' strategy.
+ arch/arm/probes/kprobes/test-core.c | 14 +++-----------
+ 1 file changed, 3 insertions(+), 11 deletions(-)
+Merging arm64/for-next/core (a13c140cc289c Linux 7.2-rc3)
+$ git merge -m Merge branch 'for-next/core' of https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux arm64/for-next/core
+Already up to date.
+Merging arm-perf/for-next/perf (5936245125f78 perf/arm-cmn: Fix DVM node events)
+$ git merge -m Merge branch 'for-next/perf' of https://git.kernel.org/pub/scm/linux/kernel/git/will/linux.git arm-perf/for-next/perf
+Already up to date.
+Merging arm-soc/for-next (900d72f1052c1 soc: document merges)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git arm-soc/for-next
+Merge made by the 'ort' strategy.
+ arch/arm/Kconfig                                  | 80 +++++++++++++----------
+ arch/arm/Kconfig.platforms                        |  2 +-
+ arch/arm/arm-soc-for-next-contents.txt            | 36 ++++++++++
+ arch/arm/boot/dts/st/spear1340-evb.dts            |  2 +-
+ arch/arm/boot/dts/st/spear13xx.dtsi               |  4 --
+ arch/arm/boot/dts/st/spear320.dtsi                |  2 +-
+ arch/arm/boot/dts/st/ste-dbx5x0.dtsi              |  2 +-
+ arch/arm/boot/dts/st/ste-href.dtsi                |  2 +-
+ arch/arm/boot/dts/st/ste-snowball.dts             |  2 +-
+ arch/arm/boot/dts/st/ste-ux500-samsung-gavini.dts |  2 +-
+ arch/arm/boot/dts/st/ste-ux500-samsung-janice.dts |  2 +-
+ arch/arm/boot/dts/st/ste-ux500-samsung-kyle.dts   |  2 +-
+ arch/arm/boot/dts/st/ste-ux500-samsung-skomer.dts |  2 +-
+ arch/arm/configs/am200epdkit_defconfig            |  3 +-
+ arch/arm/configs/aspeed_g4_defconfig              |  2 -
+ arch/arm/configs/aspeed_g5_defconfig              |  1 -
+ arch/arm/configs/assabet_defconfig                |  3 +
+ arch/arm/configs/at91_dt_defconfig                |  2 -
+ arch/arm/configs/axm55xx_defconfig                |  2 -
+ arch/arm/configs/bcm2835_defconfig                |  1 -
+ arch/arm/configs/clps711x_defconfig               |  1 -
+ arch/arm/configs/collie_defconfig                 |  2 +-
+ arch/arm/configs/davinci_all_defconfig            |  1 -
+ arch/arm/configs/dove_defconfig                   |  2 +-
+ arch/arm/configs/ep93xx_defconfig                 |  1 -
+ arch/arm/configs/footbridge_defconfig             |  2 +
+ arch/arm/configs/gemini_defconfig                 |  1 -
+ arch/arm/configs/h3600_defconfig                  |  3 +-
+ arch/arm/configs/hisi_defconfig                   |  1 -
+ arch/arm/configs/imx_v4_v5_defconfig              |  1 -
+ arch/arm/configs/integrator_defconfig             |  2 -
+ arch/arm/configs/ixp4xx_defconfig                 |  1 -
+ arch/arm/configs/jornada720_defconfig             |  3 +-
+ arch/arm/configs/keystone_defconfig               |  1 -
+ arch/arm/configs/lpc32xx_defconfig                |  1 -
+ arch/arm/configs/mmp2_defconfig                   |  1 -
+ arch/arm/configs/moxart_defconfig                 |  2 -
+ arch/arm/configs/mps2_defconfig                   |  1 -
+ arch/arm/configs/multi_v4t_defconfig              |  2 -
+ arch/arm/configs/multi_v5_defconfig               | 10 ---
+ arch/arm/configs/mv78xx0_defconfig                |  3 +-
+ arch/arm/configs/mvebu_v5_defconfig               | 10 ---
+ arch/arm/configs/mxs_defconfig                    |  1 -
+ arch/arm/configs/neponset_defconfig               |  3 +
+ arch/arm/configs/netwinder_defconfig              |  3 +
+ arch/arm/configs/nhk8815_defconfig                |  1 -
+ arch/arm/configs/omap1_defconfig                  |  3 +-
+ arch/arm/configs/orion5x_defconfig                | 11 ----
+ arch/arm/configs/pxa168_defconfig                 |  2 -
+ arch/arm/configs/pxa3xx_defconfig                 |  2 -
+ arch/arm/configs/pxa910_defconfig                 |  2 -
+ arch/arm/configs/pxa_defconfig                    |  5 --
+ arch/arm/configs/rpc_defconfig                    |  3 +
+ arch/arm/configs/s3c6400_defconfig                |  2 +
+ arch/arm/configs/sama5_defconfig                  |  1 -
+ arch/arm/configs/sama7_defconfig                  |  1 -
+ arch/arm/configs/spear13xx_defconfig              |  1 -
+ arch/arm/configs/spitz_defconfig                  |  2 +-
+ arch/arm/configs/stm32_defconfig                  |  1 -
+ arch/arm/configs/versatile_defconfig              |  2 -
+ arch/arm/configs/vt8500_v6_v7_defconfig           |  1 -
+ arch/arm/configs/wpcm450_defconfig                |  2 -
+ arch/arm/mach-at91/Kconfig                        |  4 +-
+ arch/arm/mach-axxia/Kconfig                       |  4 +-
+ arch/arm/mach-dove/Kconfig                        |  8 ++-
+ arch/arm/mach-footbridge/Kconfig                  |  7 +-
+ arch/arm/mach-imx/Kconfig                         | 13 +++-
+ arch/arm/mach-mv78xx0/Kconfig                     |  4 +-
+ arch/arm/mach-omap2/Kconfig                       | 18 +++--
+ arch/arm/mach-orion5x/Kconfig                     | 47 ++++++++++---
+ arch/arm/mach-pxa/Kconfig                         | 10 ++-
+ arch/arm/mach-rpc/Kconfig                         |  5 +-
+ arch/arm/mach-s3c/Kconfig.s3c64xx                 | 15 +++--
+ arch/arm/mach-sa1100/Kconfig                      |  5 +-
+ arch/arm/mach-stm32/Kconfig                       | 14 ++--
+ arch/arm/mach-versatile/Kconfig                   | 15 +++--
+ arch/arm/mm/Kconfig                               | 29 +++++++-
+ 77 files changed, 270 insertions(+), 183 deletions(-)
+ create mode 100644 arch/arm/arm-soc-for-next-contents.txt
+Merging amlogic/for-next (4336e970ec689 Merge branch 'v7.3/arm64-dt' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux.git amlogic/for-next
+Merge made by the 'ort' strategy.
+ .../soc/amlogic/amlogic,meson-gx-clk-measure.yaml  |   2 +
+ arch/arm64/boot/dts/amlogic/amlogic-a9.dtsi        | 140 +++++++++++
+ arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi        |   5 +
+ arch/arm64/boot/dts/amlogic/meson-a1.dtsi          |   5 +
+ drivers/soc/amlogic/meson-clk-measure.c            | 272 +++++++++++++++++++++
+ 5 files changed, 424 insertions(+)
+Merging asahi-soc/asahi-soc/for-next (297f9bd201933 Merge branch 'apple-soc/drivers-7.3' into asahi-soc/for-next)
+$ git merge -m Merge branch 'asahi-soc/for-next' of https://github.com/AsahiLinux/linux.git asahi-soc/asahi-soc/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../bindings/soc/apple/apple,t6000-pmgr-misc.yaml  |  45 +++++
+ MAINTAINERS                                        |   1 +
+ arch/arm64/boot/dts/apple/t600x-die0.dtsi          |   7 +
+ arch/arm64/boot/dts/apple/t602x-die0.dtsi          |   7 +
+ arch/arm64/boot/dts/apple/t8122.dtsi               |  30 ++--
+ drivers/soc/apple/Kconfig                          |  10 ++
+ drivers/soc/apple/Makefile                         |   2 +
+ drivers/soc/apple/apple-pmgr-misc.c                | 181 +++++++++++++++++++++
+ drivers/soc/apple/sart.c                           |   9 +-
+ 9 files changed, 275 insertions(+), 17 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/soc/apple/apple,t6000-pmgr-misc.yaml
+ create mode 100644 drivers/soc/apple/apple-pmgr-misc.c
+Merging at91/at91-next (6d892f8d5cd9f Merge branch 'clk-microchip-fixes' into at91-next)
+$ git merge -m Merge branch 'at91-next' of https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git at91/at91-next
+Merge made by the 'ort' strategy.
+Merging bmc/for-next (40ae2e16214ad Merge branch 'nuvoton/arm/dt' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/bmc/linux.git bmc/for-next
+Merge made by the 'ort' strategy.
+ arch/arm/boot/dts/nuvoton/nuvoton-common-npcm7xx.dtsi | 9 +++------
+ 1 file changed, 3 insertions(+), 6 deletions(-)
+Merging broadcom/next (f705b0c80bab1 arm: dts: bcm2711: Fix typo in gpio-line-names)
+$ git merge -m Merge branch 'next' of https://github.com/Broadcom/stblinux.git broadcom/next
+Merge made by the 'ort' strategy.
+Merging cix/for-next (a0cffbd8878c5 Merge remote-tracking branch 'cix/dt' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/peter.chen/cix.git cix/for-next
+Merge made by the 'ort' strategy.
+Merging davinci/davinci/for-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'davinci/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git davinci/davinci/for-next
+Already up to date.
+Merging drivers-memory/for-next (8111c7f3723f4 memory: tegra: Guard against NULL mc_regs in IRQ handler)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git drivers-memory/for-next
+Auto-merging drivers/memory/stm32_omm.c
+Auto-merging drivers/memory/tegra/tegra186-emc.c
+CONFLICT (content): Merge conflict in drivers/memory/tegra/tegra186-emc.c
+Resolved 'drivers/memory/tegra/tegra186-emc.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 0dffd3ba933bc] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git
+$ git diff -M --stat --summary HEAD^..
+ drivers/memory/stm32_omm.c          |  2 +-
+ drivers/memory/tegra/mc.c           | 48 ++++++++++++++++++++-----
+ drivers/memory/tegra/mc.h           | 37 +++++++++++++++++++
+ drivers/memory/tegra/tegra186-emc.c | 71 ++++++++++++++++++++++++++++---------
+ 4 files changed, 132 insertions(+), 26 deletions(-)
+Merging fsl/soc_fsl (3300a285c2852 soc: fsl: qe_ports_ic: Use generic I/O helper instead of specific powerPC ones)
+$ git merge -m Merge branch 'soc_fsl' of https://git.kernel.org/pub/scm/linux/kernel/git/chleroy/linux.git fsl/soc_fsl
+Merge made by the 'ort' strategy.
+ drivers/soc/fsl/qe/qe_ports_ic.c | 15 +++++++++------
+ 1 file changed, 9 insertions(+), 6 deletions(-)
+Merging imx-mxs/for-next (d089adef10f38 Merge branch 'imx/dt64' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/frank.li/linux.git imx-mxs/for-next
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/fsl.yaml     |   24 +
+ .../devicetree/bindings/input/fsl,scu-key.yaml     |    4 +-
+ .../bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml |   39 +
+ arch/arm/boot/dts/nxp/imx/imx6qdl-sabresd.dtsi     |    2 +
+ arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6a.dtsi      |   10 +
+ arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6b.dtsi      |   10 +
+ arch/arm/boot/dts/nxp/imx/imx6sx-sdb.dtsi          |    2 +
+ .../boot/dts/nxp/imx/imx6ul-tqma6ul-common.dtsi    |   10 +
+ arch/arm/boot/dts/nxp/imx/imx7d-pico.dtsi          |   23 +
+ arch/arm/boot/dts/nxp/ls/ls1021a-twr.dts           |    2 +-
+ arch/arm/mach-imx/avic.c                           |    2 +
+ arch/arm/mach-imx/common.h                         |    7 -
+ arch/arm/mach-imx/src.c                            |    3 +
+ arch/arm/mach-imx/system.c                         |    8 -
+ arch/arm64/boot/dts/freescale/Makefile             |   24 +
+ .../arm64/boot/dts/freescale/imx8-apalis-v1.1.dtsi |    2 +-
+ arch/arm64/boot/dts/freescale/imx8dxl-evk.dts      |    3 +
+ .../boot/dts/freescale/imx8mm-emtop-baseboard.dts  |  301 +++--
+ arch/arm64/boot/dts/freescale/imx8mm-evk.dtsi      |    2 +
+ .../boot/dts/freescale/imx8mm-var-dart-sonata.dts  |  516 +++++++++
+ arch/arm64/boot/dts/freescale/imx8mm-var-dart.dtsi |  558 ++++++++++
+ .../boot/dts/freescale/imx8mm-var-som-symphony.dts |   12 +
+ arch/arm64/boot/dts/freescale/imx8mp-ab2.dts       |    4 +
+ .../boot/dts/freescale/imx8mp-evk-flexcan2.dtso    |   15 +
+ arch/arm64/boot/dts/freescale/imx8mp-evk.dts       |   97 +-
+ arch/arm64/boot/dts/freescale/imx8mp-frdm.dts      |    2 +
+ .../dts/freescale/imx8mp-nitrogen-smarc-som.dtsi   |   82 +-
+ ...imx8mp-tqma8mpqs-mb-smarc-2-lvds-g133han01.dtso |   81 ++
+ ...8mp-tqma8mpqs-mb-smarc-2-lvds0-tm070jvhg33.dtso |   43 +
+ ...8mp-tqma8mpqs-mb-smarc-2-lvds1-tm070jvhg33.dtso |   43 +
+ .../dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2.dts  |  380 +++++++
+ .../arm64/boot/dts/freescale/imx8mp-tqma8mpqs.dtsi | 1178 ++++++++++++++++++++
+ .../boot/dts/freescale/imx8mp-var-dart-sonata.dts  |  733 ++++++++++++
+ arch/arm64/boot/dts/freescale/imx8mp-var-dart.dtsi |  475 ++++++++
+ .../boot/dts/freescale/imx8mp-var-som-symphony.dts |  389 ++++++-
+ arch/arm64/boot/dts/freescale/imx8mp-var-som.dtsi  |   12 +-
+ arch/arm64/boot/dts/freescale/imx8mp.dtsi          |    1 +
+ arch/arm64/boot/dts/freescale/imx8mq-evk.dts       |   84 ++
+ .../boot/dts/freescale/imx8mq-librem5-devkit.dts   |    4 +-
+ arch/arm64/boot/dts/freescale/imx8mq.dtsi          |   23 +
+ arch/arm64/boot/dts/freescale/imx8qm-mek.dts       |    3 +
+ arch/arm64/boot/dts/freescale/imx8qm.dtsi          |    7 +
+ arch/arm64/boot/dts/freescale/imx8qxp-mek.dts      |    3 +
+ arch/arm64/boot/dts/freescale/imx91-9x9-qsb.dts    |  160 +++
+ .../dts/freescale/imx91-tqma9131-mba91xxca.dts     |    4 +
+ arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi  |    4 +-
+ .../boot/dts/freescale/imx91-var-dart-sonata.dts   |   14 +-
+ .../freescale/imx93-11x11-evk-dy1212w-4856.dtso    |   81 ++
+ .../boot/dts/freescale/imx93-kontron-osm-s.dtsi    |    4 +-
+ .../dts/freescale/imx93-tqma9352-mba93xxca.dts     |   21 +
+ .../imx93-tqma9352-mba93xxla-lvds-tm070jvhg33.dtso |   38 +
+ .../freescale/imx93-tqma9352-mba93xxla-mini.dts    |   45 +
+ .../dts/freescale/imx93-tqma9352-mba93xxla.dts     |   21 +
+ .../boot/dts/freescale/imx93-var-dart-sonata.dts   |   14 +-
+ arch/arm64/boot/dts/freescale/imx93.dtsi           |   39 +-
+ arch/arm64/boot/dts/freescale/imx94.dtsi           |   16 +-
+ arch/arm64/boot/dts/freescale/imx943-evk.dts       |   14 +-
+ arch/arm64/boot/dts/freescale/imx943.dtsi          |   16 +-
+ arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts  |    4 +-
+ arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts  |    8 +-
+ .../boot/dts/freescale/imx95-toradex-smarc.dtsi    |    1 +
+ .../boot/dts/freescale/imx95-var-dart-sonata.dts   |    3 -
+ arch/arm64/boot/dts/freescale/imx95-var-dart.dtsi  |   10 +
+ arch/arm64/boot/dts/freescale/imx95.dtsi           |   62 +-
+ arch/arm64/boot/dts/freescale/s32g2.dtsi           |   21 +-
+ arch/arm64/boot/dts/freescale/s32g3.dtsi           |   53 +-
+ drivers/firmware/imx/sm-misc.c                     |    3 +
+ drivers/soc/imx/soc-imx9.c                         |    9 +
+ 68 files changed, 5595 insertions(+), 293 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-var-dart-sonata.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-var-dart.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-evk-flexcan2.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds-g133han01.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds0-tm070jvhg33.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds1-tm070jvhg33.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-var-dart-sonata.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-var-dart.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-11x11-evk-dy1212w-4856.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxla-lvds-tm070jvhg33.dtso
+Merging mediatek/for-next (b1ccec7cca558 Merge branch 'v7.2-next/soc' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mediatek/linux.git mediatek/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/soc/mediatek/mediatek,mutex.yaml      |  17 +-
+ arch/arm/boot/dts/mediatek/mt8127.dtsi             |   2 +-
+ arch/arm64/boot/dts/mediatek/mt6795.dtsi           |   4 +-
+ .../boot/dts/mediatek/mt8183-kukui-kodama.dtsi     |   6 +-
+ .../dts/mediatek/mt8183-kukui-krane-sku176.dts     |   4 +-
+ arch/arm64/boot/dts/mediatek/mt8183-kukui.dtsi     |   2 +
+ arch/arm64/boot/dts/mediatek/mt8183.dtsi           |  22 ++
+ arch/arm64/boot/dts/mediatek/mt8186.dtsi           | 132 +++++----
+ arch/arm64/boot/dts/mediatek/mt8188-geralt.dtsi    |   1 +
+ arch/arm64/boot/dts/mediatek/mt8188.dtsi           |  33 +++
+ arch/arm64/boot/dts/mediatek/mt8192.dtsi           |  17 ++
+ arch/arm64/boot/dts/mediatek/mt8195.dtsi           |  37 +++
+ .../boot/dts/mediatek/mt8390-genio-common.dtsi     |   2 +-
+ .../boot/dts/mediatek/mt8390-tungsten-smarc.dtsi   |   4 -
+ drivers/soc/mediatek/mt8167-mmsys.h                |  21 +-
+ drivers/soc/mediatek/mt8173-mmsys.h                |  28 +-
+ drivers/soc/mediatek/mt8183-mmsys.h                |  14 +-
+ drivers/soc/mediatek/mt8186-mmsys.h                |  22 +-
+ drivers/soc/mediatek/mt8188-mmsys.h                |  78 +++---
+ drivers/soc/mediatek/mt8192-mmsys.h                |  20 +-
+ drivers/soc/mediatek/mt8195-mmsys.h                | 181 ++++++-------
+ drivers/soc/mediatek/mt8365-mmsys.h                |  20 +-
+ drivers/soc/mediatek/mtk-dvfsrc.c                  |   1 +
+ drivers/soc/mediatek/mtk-mmsys.h                   | 300 +++++++++------------
+ drivers/soc/mediatek/mtk-socinfo.c                 |   7 +-
+ 25 files changed, 531 insertions(+), 444 deletions(-)
+Merging mvebu/for-next (3ea0dabfa4500 Merge branch 'mvebu/dt64' into mvebu/for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu.git mvebu/for-next
+Merge made by the 'ort' strategy.
+ arch/arm64/boot/dts/marvell/armada-37xx.dtsi | 1 +
+ 1 file changed, 1 insertion(+)
+Merging omap/for-next (ba251bb5ce2db Merge branch 'omap-for-v7.3/drivers' into tmp/omap-next-20260706.114259)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git omap/for-next
+Merge made by the 'ort' strategy.
+ arch/arm/boot/dts/ti/davinci/da850-lcdk.dts        |   2 +-
+ arch/arm/boot/dts/ti/omap/Makefile                 |   4 +
+ .../dts/ti/omap/am335x-icev2-prueth-overlay.dtso   | 156 +++++++++++++++++++++
+ arch/arm/boot/dts/ti/omap/am33xx-l4.dtsi           |  15 +-
+ arch/arm/boot/dts/ti/omap/am3517-evm.dts           |   2 +-
+ arch/arm/boot/dts/ti/omap/am4372.dtsi              |  11 ++
+ arch/arm/boot/dts/ti/omap/am437x-idk-evm.dts       | 103 +++++++++++++-
+ arch/arm/boot/dts/ti/omap/am437x-l4.dtsi           |   2 +-
+ arch/arm/boot/dts/ti/omap/am57-pruss.dtsi          |  11 ++
+ arch/arm/boot/dts/ti/omap/am571x-idk.dts           |   8 +-
+ arch/arm/boot/dts/ti/omap/am5729-beagleboneai.dts  |   2 +-
+ arch/arm/boot/dts/ti/omap/am572x-idk.dts           |  10 +-
+ arch/arm/boot/dts/ti/omap/am574x-idk.dts           |  10 +-
+ arch/arm/boot/dts/ti/omap/am57xx-idk-common.dtsi   |  61 ++++++++
+ arch/arm/boot/dts/ti/omap/dm8148-evm.dts           |   4 +-
+ arch/arm/boot/dts/ti/omap/dm814x.dtsi              |   2 +-
+ arch/arm/boot/dts/ti/omap/dm816x-clocks.dtsi       |   2 +-
+ arch/arm/boot/dts/ti/omap/dm816x.dtsi              |   2 +-
+ arch/arm/boot/dts/ti/omap/dra7-l4.dtsi             |   4 +-
+ .../boot/dts/ti/omap/motorola-mapphone-common.dtsi |  10 +-
+ arch/arm/boot/dts/ti/omap/omap2430.dtsi            |   2 +-
+ arch/arm/boot/dts/ti/omap/omap3-cm-t3x30.dtsi      |   2 +-
+ arch/arm/boot/dts/ti/omap/omap3-n950-n9.dtsi       |   2 +-
+ .../boot/dts/ti/omap/omap3-overo-common-lcd35.dtsi |   2 +-
+ .../boot/dts/ti/omap/omap3-overo-common-lcd43.dtsi |   2 +-
+ arch/arm/boot/dts/ti/omap/omap3.dtsi               |   2 +-
+ arch/arm/boot/dts/ti/omap/omap4-epson-embt2ws.dts  |  13 +-
+ arch/arm/boot/dts/ti/omap/omap4-l4.dtsi            |   2 +-
+ arch/arm/boot/dts/ti/omap/omap4-panda-es.dts       |   2 +-
+ arch/arm/boot/dts/ti/omap/omap4-var-som-om44.dtsi  |  14 ++
+ arch/arm/boot/dts/ti/omap/omap5-l4.dtsi            |   6 +-
+ arch/arm/configs/multi_v7_defconfig                |   2 +
+ arch/arm/configs/omap2plus_defconfig               |  13 ++
+ arch/arm/mach-omap2/omap_hwmod.c                   |  26 ++--
+ arch/arm/mach-omap2/sleep44xx.S                    |   5 +-
+ drivers/bus/ti-sysc.c                              |  12 +-
+ 36 files changed, 472 insertions(+), 56 deletions(-)
+ create mode 100644 arch/arm/boot/dts/ti/omap/am335x-icev2-prueth-overlay.dtso
+Merging qcom/for-next (5bdb10dafd3fd Merge branches 'arm32-for-7.3', 'arm64-defconfig-for-7.3', 'arm64-fixes-for-7.2', 'arm64-for-7.3', 'clk-fixes-for-7.2', 'clk-for-7.3' and 'drivers-for-7.3' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git qcom/for-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/clk/qcom/camcc-sc8280xp.c
+Auto-merging drivers/clk/qcom/camcc-x1p42100.c
+Auto-merging drivers/clk/qcom/dispcc-eliza.c
+Auto-merging drivers/clk/qcom/gcc-glymur.c
+Auto-merging drivers/clk/qcom/gcc-qcs8300.c
+Auto-merging drivers/clk/qcom/gxclkctl-kaanapali.c
+Auto-merging drivers/clk/qcom/videocc-x1p42100.c
+Auto-merging drivers/gpu/drm/msm/adreno/a5xx_gpu.c
+Auto-merging drivers/soc/qcom/qcom_pd_mapper.c
+Merge made by the 'ort' strategy.
+ .../bindings/arm/qcom,coresight-ctcu.yaml          |    4 +
+ Documentation/devicetree/bindings/arm/qcom.yaml    |  533 +--
+ .../devicetree/bindings/cache/qcom,llcc.yaml       |   63 +-
+ .../bindings/clock/qcom,gcc-msm8953.yaml           |    2 -
+ .../bindings/clock/qcom,glymur-evacc.yaml          |   72 +
+ .../devicetree/bindings/clock/qcom,glymur-gcc.yaml |    8 +
+ .../bindings/clock/qcom,ipq9574-gcc.yaml           |    1 -
+ .../bindings/clock/qcom,ipq9574-nsscc.yaml         |    1 -
+ .../devicetree/bindings/clock/qcom,rpmcc.yaml      |   65 +-
+ .../devicetree/bindings/clock/qcom,shikra-gcc.yaml |   70 +
+ .../bindings/clock/qcom,sm8450-camcc.yaml          |   12 +-
+ .../bindings/clock/qcom,sm8450-videocc.yaml        |   15 +-
+ .../bindings/clock/qcom,x1e80100-camcc.yaml        |    3 +
+ .../bindings/clock/samsung,exynosautov9-clock.yaml |    2 +-
+ .../devicetree/bindings/firmware/qcom,scm.yaml     |    9 +
+ .../bindings/soc/qcom/qcom,aoss-qmp.yaml           |    1 +
+ .../bindings/soc/qcom/qcom,pmic-glink.yaml         |    1 +
+ .../devicetree/bindings/soc/qcom/qcom,smd-rpm.yaml |    1 +
+ Documentation/devicetree/bindings/sram/sram.yaml   |    3 +
+ .../devicetree/bindings/vendor-prefixes.yaml       |    4 +
+ MAINTAINERS                                        |    9 +
+ arch/arm/boot/dts/qcom/Makefile                    |    2 +
+ arch/arm/boot/dts/qcom/msm8226-motorola-titan.dts  |  350 ++
+ .../dts/qcom/msm8926-sony-xperia-yukon-eagle.dts   |  407 ++
+ arch/arm/boot/dts/qcom/pm8921.dtsi                 |   12 +
+ arch/arm/boot/dts/qcom/qcom-apq8064-pins.dtsi      |    2 +-
+ arch/arm/boot/dts/qcom/qcom-ipq8064.dtsi           |    4 +-
+ arch/arm/boot/dts/qcom/qcom-mdm9615-wp8548.dtsi    |    2 +-
+ .../dts/qcom/qcom-msm8960-samsung-expressatt.dts   |   54 +
+ arch/arm/boot/dts/qcom/qcom-msm8974.dtsi           |    2 +-
+ arch/arm/boot/dts/qcom/qcom-msm8974pro-htc-m8.dts  |   94 +-
+ arch/arm/boot/dts/qcom/qcom-sdx55-t55.dts          |    2 -
+ .../boot/dts/qcom/qcom-sdx55-telit-fn980-tlb.dts   |    2 -
+ arch/arm/boot/dts/qcom/qcom-sdx55.dtsi             |    3 +
+ arch/arm/boot/dts/qcom/qcom-sdx65-mtp.dts          |    2 +-
+ arch/arm64/boot/dts/qcom/Makefile                  |   19 +-
+ arch/arm64/boot/dts/qcom/apq8016-sbc-usb-host.dtso |    2 +-
+ arch/arm64/boot/dts/qcom/eliza-cqs-evk.dts         |   14 +
+ arch/arm64/boot/dts/qcom/eliza-cqs-som.dtsi        |  399 ++
+ arch/arm64/boot/dts/qcom/eliza-evk.dtsi            |   20 +
+ arch/arm64/boot/dts/qcom/eliza-mtp.dts             |   44 +
+ arch/arm64/boot/dts/qcom/eliza.dtsi                |  100 +-
+ arch/arm64/boot/dts/qcom/glymur-crd.dts            |  399 --
+ arch/arm64/boot/dts/qcom/glymur-crd.dtsi           |  546 ++-
+ arch/arm64/boot/dts/qcom/glymur.dtsi               |  871 +++-
+ arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts         |   16 +
+ arch/arm64/boot/dts/qcom/hamoa.dtsi                |  267 +-
+ arch/arm64/boot/dts/qcom/ipq5018.dtsi              |   15 +-
+ arch/arm64/boot/dts/qcom/ipq5332-rdp-common.dtsi   |   24 +-
+ arch/arm64/boot/dts/qcom/ipq5332-rdp441.dts        |   16 +-
+ arch/arm64/boot/dts/qcom/ipq5332.dtsi              |   53 +-
+ arch/arm64/boot/dts/qcom/ipq5424-rdp466.dts        |   12 +-
+ arch/arm64/boot/dts/qcom/ipq5424.dtsi              |   24 +-
+ arch/arm64/boot/dts/qcom/ipq6018.dtsi              |   10 +
+ arch/arm64/boot/dts/qcom/ipq9574.dtsi              |   34 +-
+ arch/arm64/boot/dts/qcom/kaanapali-mtp.dts         |   58 +-
+ arch/arm64/boot/dts/qcom/kaanapali-qrd.dts         |   58 +-
+ arch/arm64/boot/dts/qcom/kaanapali.dtsi            |  190 +-
+ arch/arm64/boot/dts/qcom/kodiak.dtsi               |   36 +-
+ arch/arm64/boot/dts/qcom/lemans-auto.dtsi          |  104 -
+ arch/arm64/boot/dts/qcom/lemans-evk-camera.dtso    |    2 +-
+ arch/arm64/boot/dts/qcom/lemans-evk.dts            |  251 +-
+ arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi   |    4 +-
+ arch/arm64/boot/dts/qcom/lemans.dtsi               |   55 +-
+ arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts   |   29 +-
+ .../boot/dts/qcom/milos-nothing-asteroids.dts      |   38 -
+ arch/arm64/boot/dts/qcom/milos.dtsi                |  282 +-
+ arch/arm64/boot/dts/qcom/monaco-arduino-monza.dts  |  120 +-
+ arch/arm64/boot/dts/qcom/monaco.dtsi               |   86 +-
+ .../arm64/boot/dts/qcom/msm8953-flipkart-rimob.dts |  138 +-
+ arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts |    2 +-
+ arch/arm64/boot/dts/qcom/msm8996.dtsi              |   41 +
+ arch/arm64/boot/dts/qcom/pm4125.dtsi               |    2 +-
+ arch/arm64/boot/dts/qcom/pm7550-eliza.dtsi         |   61 +
+ arch/arm64/boot/dts/qcom/pm8010-kaanapali.dtsi     |   20 +
+ arch/arm64/boot/dts/qcom/pm8010.dtsi               |   20 +
+ arch/arm64/boot/dts/qcom/pm8550vs-d-eliza.dtsi     |   63 +
+ arch/arm64/boot/dts/qcom/pm8550vs-g-eliza.dtsi     |   63 +
+ arch/arm64/boot/dts/qcom/pmk8550-eliza.dtsi        |   87 +
+ arch/arm64/boot/dts/qcom/pmr735d-eliza.dtsi        |   63 +
+ arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi        |    4 +
+ arch/arm64/boot/dts/qcom/purwa.dtsi                |   52 +
+ arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts |   64 +-
+ .../boot/dts/qcom/qcm6490-particle-tachyon.dts     |   16 +-
+ arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts   |    7 +-
+ arch/arm64/boot/dts/qcom/qcs615-ride.dts           |    6 +-
+ .../boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts     |   20 +-
+ arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts       |    7 +-
+ .../dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts  |   18 +-
+ .../boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts |   20 +-
+ .../boot/dts/qcom/qcs6490-vicharak-axon-mini.dts   | 1014 +++++
+ arch/arm64/boot/dts/qcom/qcs8550-rb5gen2.dts       | 1573 +++++++
+ arch/arm64/boot/dts/qcom/qdu1000.dtsi              |    2 +-
+ arch/arm64/boot/dts/qcom/qrb2210-arduino-imola.dts |    4 +-
+ .../dts/qcom/qrb2210-rb1-vision-mezzanine.dtso     |    4 +-
+ arch/arm64/boot/dts/qcom/qrb2210-rb1.dts           |    4 +-
+ arch/arm64/boot/dts/qcom/qrb4210-rb2.dts           |    6 +
+ arch/arm64/boot/dts/qcom/sa8295p-adp.dts           |   32 +-
+ arch/arm64/boot/dts/qcom/sa8540p-ride.dts          |   16 +-
+ arch/arm64/boot/dts/qcom/sa8775p-ride-r3.dts       |   17 -
+ arch/arm64/boot/dts/qcom/sa8775p-ride.dts          |   17 -
+ arch/arm64/boot/dts/qcom/sar2130p.dtsi             |    2 +-
+ arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi       |    5 +
+ arch/arm64/boot/dts/qcom/sc7180.dtsi               |    2 +-
+ .../boot/dts/qcom/sc7280-herobrine-lte-sku.dtsi    |    2 +
+ arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi     |    5 +
+ arch/arm64/boot/dts/qcom/sc7280-idp.dtsi           |    5 +-
+ .../arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts |   57 +-
+ arch/arm64/boot/dts/qcom/sc8180x-primus.dts        |   56 +-
+ arch/arm64/boot/dts/qcom/sc8180x.dtsi              |    2 +-
+ arch/arm64/boot/dts/qcom/sc8280xp-crd.dts          |   22 +-
+ .../boot/dts/qcom/sc8280xp-huawei-gaokun3.dts      |   16 +-
+ .../dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts     |   38 +-
+ .../boot/dts/qcom/sc8280xp-microsoft-arcata.dts    |  301 +-
+ .../boot/dts/qcom/sc8280xp-microsoft-blackrock.dts |   25 +-
+ arch/arm64/boot/dts/qcom/sc8280xp.dtsi             |  347 +-
+ arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi |    5 +
+ arch/arm64/boot/dts/qcom/sdm670.dtsi               |    2 +-
+ arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi |    8 +-
+ arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi     |    3 -
+ arch/arm64/boot/dts/qcom/sdm845-mtp.dts            |    2 -
+ .../arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi |    4 +-
+ .../boot/dts/qcom/sdm845-samsung-starqltechn.dts   |    2 -
+ arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts  |    2 -
+ .../dts/qcom/sdm845-xiaomi-beryllium-common.dtsi   |    2 -
+ arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts |    2 -
+ arch/arm64/boot/dts/qcom/sdm845.dtsi               |    5 +-
+ .../dts/qcom/sdm850-huawei-matebook-e-2019.dts     |    2 -
+ .../boot/dts/qcom/sdm850-lenovo-yoga-c630.dts      |    6 +-
+ arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts   |    3 +-
+ arch/arm64/boot/dts/qcom/sdx75.dtsi                |    2 +-
+ arch/arm64/boot/dts/qcom/shikra-cqm-evk.dts        |   40 +
+ arch/arm64/boot/dts/qcom/shikra-cqm-som.dtsi       |  156 +
+ arch/arm64/boot/dts/qcom/shikra-cqs-evk.dts        |   40 +
+ arch/arm64/boot/dts/qcom/shikra-evk.dtsi           |   14 +
+ arch/arm64/boot/dts/qcom/shikra-iqs-evk.dts        |   40 +
+ arch/arm64/boot/dts/qcom/shikra-iqs-som.dtsi       |  170 +
+ arch/arm64/boot/dts/qcom/shikra.dtsi               |  842 ++++
+ arch/arm64/boot/dts/qcom/sm4450.dtsi               |    2 +-
+ arch/arm64/boot/dts/qcom/sm6115-fxtec-pro1x.dts    |    2 +-
+ arch/arm64/boot/dts/qcom/sm6115.dtsi               |   65 +
+ .../boot/dts/qcom/sm6125-xiaomi-ginkgo-common.dtsi |   64 +-
+ .../boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts  |    2 +-
+ arch/arm64/boot/dts/qcom/sm6125.dtsi               |  168 +-
+ arch/arm64/boot/dts/qcom/sm6350.dtsi               |   19 +-
+ arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts  |    5 +-
+ .../dts/qcom/sm7325-motorola-dubai-common.dtsi     | 1454 +++++++
+ .../boot/dts/qcom/sm7325-motorola-dubai-csot.dts   |   13 +
+ .../boot/dts/qcom/sm7325-motorola-dubai-tianma.dts |   13 +
+ arch/arm64/boot/dts/qcom/sm7325-motorola-dubai.dts |    3 -
+ .../boot/dts/qcom/sm7325-nothing-spacewar.dts      |    2 -
+ arch/arm64/boot/dts/qcom/sm7325-xiaomi-taoyao.dts  |  913 ++++
+ arch/arm64/boot/dts/qcom/sm8150.dtsi               |    2 +-
+ .../boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi  |  153 +
+ arch/arm64/boot/dts/qcom/sm8250.dtsi               |   57 +-
+ arch/arm64/boot/dts/qcom/sm8350-hdk.dts            |   11 +-
+ .../dts/qcom/sm8350-microsoft-surface-duo2.dts     |    2 -
+ arch/arm64/boot/dts/qcom/sm8350-mtp.dts            |    2 -
+ .../boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi   |    3 +-
+ arch/arm64/boot/dts/qcom/sm8350.dtsi               |   86 +-
+ arch/arm64/boot/dts/qcom/sm8450.dtsi               |   38 +-
+ arch/arm64/boot/dts/qcom/sm8550-hdk.dts            |    4 +-
+ arch/arm64/boot/dts/qcom/sm8550-qrd.dts            |    4 +-
+ arch/arm64/boot/dts/qcom/sm8550.dtsi               |   28 +-
+ arch/arm64/boot/dts/qcom/sm8650-hdk.dts            |    3 +-
+ arch/arm64/boot/dts/qcom/sm8650-qrd.dts            |    3 +-
+ arch/arm64/boot/dts/qcom/sm8650.dtsi               |  131 +-
+ arch/arm64/boot/dts/qcom/sm8750.dtsi               |  266 +-
+ arch/arm64/boot/dts/qcom/talos-evk-som.dtsi        |    8 +
+ arch/arm64/boot/dts/qcom/talos.dtsi                |  140 +-
+ arch/arm64/boot/dts/qcom/x1-crd.dtsi               |   16 +
+ arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi        |    2 +
+ .../dts/qcom/x1e80100-honor-magicbook-art-14.dts   | 1341 ++++++
+ .../boot/dts/qcom/x1p42100-microsoft-sp12in.dts    | 1194 ++++++
+ arch/arm64/configs/defconfig                       |    3 +
+ drivers/bus/qcom-ebi2.c                            |   44 +-
+ drivers/clk/qcom/Kconfig                           |  232 +-
+ drivers/clk/qcom/Makefile                          |    4 +
+ drivers/clk/qcom/camcc-glymur.c                    | 2280 ++++++++++
+ drivers/clk/qcom/camcc-sc8280xp.c                  |   19 -
+ drivers/clk/qcom/camcc-x1p42100.c                  |    6 +-
+ drivers/clk/qcom/clk-regmap-phy-mux.c              |   58 +-
+ drivers/clk/qcom/clk-smd-rpm.c                     |    5 +
+ drivers/clk/qcom/common.c                          |    2 +-
+ drivers/clk/qcom/dispcc-eliza.c                    |    2 +-
+ drivers/clk/qcom/evacc-glymur.c                    |  455 ++
+ drivers/clk/qcom/gcc-glymur.c                      |   33 +-
+ drivers/clk/qcom/gcc-ipq5424.c                     |   11 -
+ drivers/clk/qcom/gcc-ipq9650.c                     |   89 +
+ drivers/clk/qcom/gcc-qcs8300.c                     |    8 +-
+ drivers/clk/qcom/gcc-shikra.c                      | 4431 ++++++++++++++++++++
+ drivers/clk/qcom/gcc-sm6115.c                      |    1 +
+ drivers/clk/qcom/gdsc.c                            |   47 +-
+ drivers/clk/qcom/gdsc.h                            |    1 +
+ drivers/clk/qcom/gxclkctl-kaanapali.c              |   12 +-
+ drivers/clk/qcom/videocc-hawi.c                    |  840 ++++
+ drivers/clk/qcom/videocc-x1p42100.c                |    4 +-
+ drivers/cpuidle/Kconfig.arm                        |    4 +-
+ drivers/firmware/qcom/Kconfig                      |   22 +-
+ drivers/firmware/qcom/Makefile                     |    3 +
+ drivers/firmware/qcom/qcom_pas.c                   |  298 ++
+ drivers/firmware/qcom/qcom_pas.h                   |   50 +
+ drivers/firmware/qcom/qcom_pas_tee.c               |  479 +++
+ drivers/firmware/qcom/qcom_scm-smc.c               |   10 +
+ drivers/firmware/qcom/qcom_scm.c                   |  456 +-
+ drivers/firmware/qcom/qcom_scm_trace.h             |  143 +
+ drivers/gpu/drm/msm/Kconfig                        |    1 +
+ drivers/gpu/drm/msm/adreno/a5xx_gpu.c              |    4 +-
+ drivers/gpu/drm/msm/adreno/adreno_gpu.c            |   11 +-
+ drivers/net/ipa/Kconfig                            |    2 +-
+ drivers/net/ipa/ipa_main.c                         |   13 +-
+ drivers/soc/qcom/Kconfig                           |  120 +-
+ drivers/soc/qcom/cmd-db.c                          |    2 +-
+ drivers/soc/qcom/ice.c                             |    4 +-
+ drivers/soc/qcom/llcc-qcom.c                       |   29 +
+ drivers/soc/qcom/qcom_pd_mapper.c                  |   38 +
+ drivers/soc/qcom/qcom_stats.c                      |    2 +
+ drivers/soc/qcom/rpmh-rsc.c                        |   37 +-
+ drivers/soc/qcom/socinfo.c                         |    1 +
+ drivers/soc/qcom/ubwc_config.c                     |  370 +-
+ include/dt-bindings/arm/qcom,ids.h                 |    1 +
+ include/dt-bindings/clock/qcom,glymur-camcc.h      |  122 +
+ include/dt-bindings/clock/qcom,glymur-evacc.h      |   38 +
+ include/dt-bindings/clock/qcom,hawi-videocc.h      |   64 +
+ include/dt-bindings/clock/qcom,ipq9650-gcc.h       |    5 +
+ include/dt-bindings/clock/qcom,shikra-gcc.h        |  263 ++
+ include/linux/firmware/qcom/qcom_pas.h             |   43 +
+ include/linux/soc/qcom/ubwc.h                      |   83 +-
+ 228 files changed, 26761 insertions(+), 2478 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/clock/qcom,glymur-evacc.yaml
+ create mode 100644 Documentation/devicetree/bindings/clock/qcom,shikra-gcc.yaml
+ create mode 100644 arch/arm/boot/dts/qcom/msm8226-motorola-titan.dts
+ create mode 100644 arch/arm/boot/dts/qcom/msm8926-sony-xperia-yukon-eagle.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/eliza-cqs-evk.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/eliza-cqs-som.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/eliza-evk.dtsi
+ delete mode 100644 arch/arm64/boot/dts/qcom/lemans-auto.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/pm7550-eliza.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/pm8550vs-d-eliza.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/pm8550vs-g-eliza.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/pmk8550-eliza.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/pmr735d-eliza.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/qcs6490-vicharak-axon-mini.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/qcs8550-rb5gen2.dts
+ delete mode 100644 arch/arm64/boot/dts/qcom/sa8775p-ride-r3.dts
+ delete mode 100644 arch/arm64/boot/dts/qcom/sa8775p-ride.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/shikra-cqm-evk.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/shikra-cqm-som.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/shikra-cqs-evk.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/shikra-evk.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/shikra-iqs-evk.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/shikra-iqs-som.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/shikra.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/sm7325-motorola-dubai-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/sm7325-motorola-dubai-csot.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/sm7325-motorola-dubai-tianma.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/sm7325-xiaomi-taoyao.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/x1e80100-honor-magicbook-art-14.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/x1p42100-microsoft-sp12in.dts
+ create mode 100644 drivers/clk/qcom/camcc-glymur.c
+ create mode 100644 drivers/clk/qcom/evacc-glymur.c
+ create mode 100644 drivers/clk/qcom/gcc-shikra.c
+ create mode 100644 drivers/clk/qcom/videocc-hawi.c
+ create mode 100644 drivers/firmware/qcom/qcom_pas.c
+ create mode 100644 drivers/firmware/qcom/qcom_pas.h
+ create mode 100644 drivers/firmware/qcom/qcom_pas_tee.c
+ create mode 100644 drivers/firmware/qcom/qcom_scm_trace.h
+ create mode 100644 include/dt-bindings/clock/qcom,glymur-camcc.h
+ create mode 100644 include/dt-bindings/clock/qcom,glymur-evacc.h
+ create mode 100644 include/dt-bindings/clock/qcom,hawi-videocc.h
+ create mode 100644 include/dt-bindings/clock/qcom,shikra-gcc.h
+ create mode 100644 include/linux/firmware/qcom/qcom_pas.h
+$ git reset --hard HEAD^
+HEAD is now at e2e42cfb943a1 Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git
+Merging next-20260716 version of qcom
+$ git merge -m next-20260716/qcom 64a28053710d050b72f8f8aaa9e75654fac0f2d5
+Auto-merging MAINTAINERS
+Auto-merging drivers/clk/qcom/camcc-sc8280xp.c
+Auto-merging drivers/clk/qcom/camcc-x1p42100.c
+Auto-merging drivers/clk/qcom/dispcc-eliza.c
+Auto-merging drivers/clk/qcom/gcc-glymur.c
+Auto-merging drivers/clk/qcom/gcc-qcs8300.c
+Auto-merging drivers/clk/qcom/gxclkctl-kaanapali.c
+Auto-merging drivers/clk/qcom/videocc-x1p42100.c
+Auto-merging drivers/gpu/drm/msm/adreno/a5xx_gpu.c
+Auto-merging drivers/soc/qcom/qcom_pd_mapper.c
+Merge made by the 'ort' strategy.
+ .../bindings/arm/qcom,coresight-ctcu.yaml          |    4 +
+ Documentation/devicetree/bindings/arm/qcom.yaml    |  533 +--
+ .../devicetree/bindings/cache/qcom,llcc.yaml       |   63 +-
+ .../bindings/clock/qcom,gcc-msm8953.yaml           |    2 -
+ .../bindings/clock/qcom,glymur-evacc.yaml          |   72 +
+ .../bindings/clock/qcom,ipq9574-gcc.yaml           |    1 -
+ .../bindings/clock/qcom,ipq9574-nsscc.yaml         |    1 -
+ .../devicetree/bindings/clock/qcom,rpmcc.yaml      |   65 +-
+ .../devicetree/bindings/clock/qcom,shikra-gcc.yaml |   70 +
+ .../bindings/clock/qcom,sm8450-camcc.yaml          |   12 +-
+ .../bindings/clock/qcom,sm8450-videocc.yaml        |   15 +-
+ .../bindings/clock/qcom,x1e80100-camcc.yaml        |    3 +
+ .../bindings/clock/samsung,exynosautov9-clock.yaml |    2 +-
+ .../devicetree/bindings/firmware/qcom,scm.yaml     |    9 +
+ .../bindings/soc/qcom/qcom,aoss-qmp.yaml           |    1 +
+ .../bindings/soc/qcom/qcom,pmic-glink.yaml         |    1 +
+ .../devicetree/bindings/soc/qcom/qcom,smd-rpm.yaml |    1 +
+ Documentation/devicetree/bindings/sram/sram.yaml   |    2 +
+ .../devicetree/bindings/vendor-prefixes.yaml       |    4 +
+ MAINTAINERS                                        |    9 +
+ arch/arm/boot/dts/qcom/Makefile                    |    2 +
+ arch/arm/boot/dts/qcom/msm8226-motorola-titan.dts  |  350 ++
+ .../dts/qcom/msm8926-sony-xperia-yukon-eagle.dts   |  407 ++
+ arch/arm/boot/dts/qcom/pm8921.dtsi                 |   12 +
+ arch/arm/boot/dts/qcom/qcom-apq8064-pins.dtsi      |    2 +-
+ arch/arm/boot/dts/qcom/qcom-ipq8064.dtsi           |    4 +-
+ arch/arm/boot/dts/qcom/qcom-mdm9615-wp8548.dtsi    |    2 +-
+ .../dts/qcom/qcom-msm8960-samsung-expressatt.dts   |   54 +
+ arch/arm/boot/dts/qcom/qcom-msm8974.dtsi           |    2 +-
+ arch/arm/boot/dts/qcom/qcom-msm8974pro-htc-m8.dts  |   94 +-
+ arch/arm/boot/dts/qcom/qcom-sdx55-t55.dts          |    2 -
+ .../boot/dts/qcom/qcom-sdx55-telit-fn980-tlb.dts   |    2 -
+ arch/arm/boot/dts/qcom/qcom-sdx55.dtsi             |    3 +
+ arch/arm/boot/dts/qcom/qcom-sdx65-mtp.dts          |    2 +-
+ arch/arm64/boot/dts/qcom/Makefile                  |   19 +-
+ arch/arm64/boot/dts/qcom/apq8016-sbc-usb-host.dtso |    2 +-
+ arch/arm64/boot/dts/qcom/eliza-cqs-evk.dts         |   14 +
+ arch/arm64/boot/dts/qcom/eliza-cqs-som.dtsi        |  399 ++
+ arch/arm64/boot/dts/qcom/eliza-evk.dtsi            |   20 +
+ arch/arm64/boot/dts/qcom/eliza-mtp.dts             |   44 +
+ arch/arm64/boot/dts/qcom/eliza.dtsi                |  100 +-
+ arch/arm64/boot/dts/qcom/glymur-crd.dts            |  399 --
+ arch/arm64/boot/dts/qcom/glymur-crd.dtsi           |  546 ++-
+ arch/arm64/boot/dts/qcom/glymur.dtsi               |  871 +++-
+ arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts         |   16 +
+ arch/arm64/boot/dts/qcom/hamoa.dtsi                |  284 +-
+ arch/arm64/boot/dts/qcom/ipq5018.dtsi              |   15 +-
+ arch/arm64/boot/dts/qcom/ipq5332-rdp-common.dtsi   |   24 +-
+ arch/arm64/boot/dts/qcom/ipq5332-rdp441.dts        |   16 +-
+ arch/arm64/boot/dts/qcom/ipq5332.dtsi              |   53 +-
+ arch/arm64/boot/dts/qcom/ipq5424-rdp466.dts        |   12 +-
+ arch/arm64/boot/dts/qcom/ipq5424.dtsi              |   24 +-
+ arch/arm64/boot/dts/qcom/ipq6018.dtsi              |   10 +
+ arch/arm64/boot/dts/qcom/ipq9574.dtsi              |   34 +-
+ arch/arm64/boot/dts/qcom/kaanapali-mtp.dts         |   58 +-
+ arch/arm64/boot/dts/qcom/kaanapali-qrd.dts         |   58 +-
+ arch/arm64/boot/dts/qcom/kaanapali.dtsi            |  190 +-
+ arch/arm64/boot/dts/qcom/kodiak.dtsi               |   36 +-
+ arch/arm64/boot/dts/qcom/lemans-auto.dtsi          |  104 -
+ arch/arm64/boot/dts/qcom/lemans-evk-camera.dtso    |    2 +-
+ arch/arm64/boot/dts/qcom/lemans-evk.dts            |  251 +-
+ arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi   |    4 +-
+ arch/arm64/boot/dts/qcom/lemans.dtsi               |   55 +-
+ arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts   |   29 +-
+ .../boot/dts/qcom/milos-nothing-asteroids.dts      |   38 -
+ arch/arm64/boot/dts/qcom/milos.dtsi                |  282 +-
+ arch/arm64/boot/dts/qcom/monaco-arduino-monza.dts  |  120 +-
+ arch/arm64/boot/dts/qcom/monaco.dtsi               |   86 +-
+ .../arm64/boot/dts/qcom/msm8953-flipkart-rimob.dts |  138 +-
+ arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts |    2 +-
+ arch/arm64/boot/dts/qcom/msm8996.dtsi              |   41 +
+ arch/arm64/boot/dts/qcom/pm4125.dtsi               |    2 +-
+ arch/arm64/boot/dts/qcom/pm7550-eliza.dtsi         |   61 +
+ arch/arm64/boot/dts/qcom/pm8010-kaanapali.dtsi     |   20 +
+ arch/arm64/boot/dts/qcom/pm8010.dtsi               |   20 +
+ arch/arm64/boot/dts/qcom/pm8550vs-d-eliza.dtsi     |   63 +
+ arch/arm64/boot/dts/qcom/pm8550vs-g-eliza.dtsi     |   63 +
+ arch/arm64/boot/dts/qcom/pmk8550-eliza.dtsi        |   87 +
+ arch/arm64/boot/dts/qcom/pmr735d-eliza.dtsi        |   63 +
+ arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi        |    4 +
+ arch/arm64/boot/dts/qcom/purwa.dtsi                |   62 +
+ arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts |   64 +-
+ .../boot/dts/qcom/qcm6490-particle-tachyon.dts     |   16 +-
+ arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts   |    7 +-
+ arch/arm64/boot/dts/qcom/qcs615-ride.dts           |    6 +-
+ .../boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts     |   20 +-
+ arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts       |    7 +-
+ .../dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts  |   18 +-
+ .../boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts |   20 +-
+ .../boot/dts/qcom/qcs6490-vicharak-axon-mini.dts   | 1014 +++++
+ arch/arm64/boot/dts/qcom/qcs8550-rb5gen2.dts       | 1573 +++++++
+ arch/arm64/boot/dts/qcom/qdu1000.dtsi              |    2 +-
+ arch/arm64/boot/dts/qcom/qrb2210-arduino-imola.dts |    4 +-
+ .../dts/qcom/qrb2210-rb1-vision-mezzanine.dtso     |    4 +-
+ arch/arm64/boot/dts/qcom/qrb2210-rb1.dts           |    4 +-
+ arch/arm64/boot/dts/qcom/qrb4210-rb2.dts           |    6 +
+ arch/arm64/boot/dts/qcom/sa8295p-adp.dts           |   32 +-
+ arch/arm64/boot/dts/qcom/sa8540p-ride.dts          |   16 +-
+ arch/arm64/boot/dts/qcom/sa8775p-ride-r3.dts       |   17 -
+ arch/arm64/boot/dts/qcom/sa8775p-ride.dts          |   17 -
+ arch/arm64/boot/dts/qcom/sar2130p.dtsi             |    2 +-
+ arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi       |    5 +
+ arch/arm64/boot/dts/qcom/sc7180.dtsi               |    2 +-
+ .../boot/dts/qcom/sc7280-herobrine-lte-sku.dtsi    |    2 +
+ arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi     |    5 +
+ arch/arm64/boot/dts/qcom/sc7280-idp.dtsi           |    5 +-
+ .../arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts |   57 +-
+ arch/arm64/boot/dts/qcom/sc8180x-primus.dts        |   56 +-
+ arch/arm64/boot/dts/qcom/sc8180x.dtsi              |    2 +-
+ arch/arm64/boot/dts/qcom/sc8280xp-crd.dts          |   22 +-
+ .../boot/dts/qcom/sc8280xp-huawei-gaokun3.dts      |   16 +-
+ .../dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts     |   38 +-
+ .../boot/dts/qcom/sc8280xp-microsoft-arcata.dts    |  301 +-
+ .../boot/dts/qcom/sc8280xp-microsoft-blackrock.dts |   25 +-
+ arch/arm64/boot/dts/qcom/sc8280xp.dtsi             |  347 +-
+ arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi |    5 +
+ arch/arm64/boot/dts/qcom/sdm670.dtsi               |    2 +-
+ arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi |    8 +-
+ arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi     |    3 -
+ arch/arm64/boot/dts/qcom/sdm845-mtp.dts            |    2 -
+ .../arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi |    4 +-
+ .../boot/dts/qcom/sdm845-samsung-starqltechn.dts   |    2 -
+ arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts  |    2 -
+ .../dts/qcom/sdm845-xiaomi-beryllium-common.dtsi   |    2 -
+ arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts |    2 -
+ arch/arm64/boot/dts/qcom/sdm845.dtsi               |    5 +-
+ .../dts/qcom/sdm850-huawei-matebook-e-2019.dts     |    2 -
+ .../boot/dts/qcom/sdm850-lenovo-yoga-c630.dts      |    6 +-
+ arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts   |    3 +-
+ arch/arm64/boot/dts/qcom/sdx75.dtsi                |    2 +-
+ arch/arm64/boot/dts/qcom/shikra-cqm-evk.dts        |   40 +
+ arch/arm64/boot/dts/qcom/shikra-cqm-som.dtsi       |  156 +
+ arch/arm64/boot/dts/qcom/shikra-cqs-evk.dts        |   40 +
+ arch/arm64/boot/dts/qcom/shikra-evk.dtsi           |   14 +
+ arch/arm64/boot/dts/qcom/shikra-iqs-evk.dts        |   40 +
+ arch/arm64/boot/dts/qcom/shikra-iqs-som.dtsi       |  170 +
+ arch/arm64/boot/dts/qcom/shikra.dtsi               |  842 ++++
+ arch/arm64/boot/dts/qcom/sm4450.dtsi               |    2 +-
+ arch/arm64/boot/dts/qcom/sm6115-fxtec-pro1x.dts    |    2 +-
+ arch/arm64/boot/dts/qcom/sm6115.dtsi               |   65 +
+ .../boot/dts/qcom/sm6125-xiaomi-ginkgo-common.dtsi |   64 +-
+ .../boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts  |    2 +-
+ arch/arm64/boot/dts/qcom/sm6125.dtsi               |  168 +-
+ arch/arm64/boot/dts/qcom/sm6350.dtsi               |   19 +-
+ arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts  |    5 +-
+ .../dts/qcom/sm7325-motorola-dubai-common.dtsi     | 1454 +++++++
+ .../boot/dts/qcom/sm7325-motorola-dubai-csot.dts   |   13 +
+ .../boot/dts/qcom/sm7325-motorola-dubai-tianma.dts |   13 +
+ arch/arm64/boot/dts/qcom/sm7325-motorola-dubai.dts |    3 -
+ .../boot/dts/qcom/sm7325-nothing-spacewar.dts      |    2 -
+ arch/arm64/boot/dts/qcom/sm7325-xiaomi-taoyao.dts  |  913 ++++
+ arch/arm64/boot/dts/qcom/sm8150.dtsi               |    2 +-
+ .../boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi  |  153 +
+ arch/arm64/boot/dts/qcom/sm8250.dtsi               |   57 +-
+ arch/arm64/boot/dts/qcom/sm8350-hdk.dts            |   11 +-
+ .../dts/qcom/sm8350-microsoft-surface-duo2.dts     |    2 -
+ arch/arm64/boot/dts/qcom/sm8350-mtp.dts            |    2 -
+ .../boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi   |    3 +-
+ arch/arm64/boot/dts/qcom/sm8350.dtsi               |   86 +-
+ arch/arm64/boot/dts/qcom/sm8450.dtsi               |   38 +-
+ arch/arm64/boot/dts/qcom/sm8550-hdk.dts            |    4 +-
+ arch/arm64/boot/dts/qcom/sm8550-qrd.dts            |    4 +-
+ arch/arm64/boot/dts/qcom/sm8550.dtsi               |   28 +-
+ arch/arm64/boot/dts/qcom/sm8650-hdk.dts            |    3 +-
+ arch/arm64/boot/dts/qcom/sm8650-qrd.dts            |    3 +-
+ arch/arm64/boot/dts/qcom/sm8650.dtsi               |  131 +-
+ arch/arm64/boot/dts/qcom/sm8750.dtsi               |  266 +-
+ arch/arm64/boot/dts/qcom/talos-evk-som.dtsi        |    8 +
+ arch/arm64/boot/dts/qcom/talos.dtsi                |  140 +-
+ arch/arm64/boot/dts/qcom/x1-crd.dtsi               |   16 +
+ arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi        |    2 +
+ .../dts/qcom/x1e80100-honor-magicbook-art-14.dts   | 1341 ++++++
+ .../boot/dts/qcom/x1p42100-microsoft-sp12in.dts    | 1194 ++++++
+ arch/arm64/configs/defconfig                       |    3 +
+ drivers/bus/qcom-ebi2.c                            |   44 +-
+ drivers/clk/qcom/Kconfig                           |  232 +-
+ drivers/clk/qcom/Makefile                          |    4 +
+ drivers/clk/qcom/camcc-glymur.c                    | 2280 ++++++++++
+ drivers/clk/qcom/camcc-sc8280xp.c                  |   19 -
+ drivers/clk/qcom/camcc-x1p42100.c                  |    6 +-
+ drivers/clk/qcom/clk-regmap-phy-mux.c              |   58 +-
+ drivers/clk/qcom/clk-smd-rpm.c                     |    5 +
+ drivers/clk/qcom/common.c                          |    2 +-
+ drivers/clk/qcom/dispcc-eliza.c                    |    2 +-
+ drivers/clk/qcom/evacc-glymur.c                    |  455 ++
+ drivers/clk/qcom/gcc-glymur.c                      |   32 +-
+ drivers/clk/qcom/gcc-ipq5424.c                     |   11 -
+ drivers/clk/qcom/gcc-ipq9650.c                     |   89 +
+ drivers/clk/qcom/gcc-qcs8300.c                     |    8 +-
+ drivers/clk/qcom/gcc-shikra.c                      | 4431 ++++++++++++++++++++
+ drivers/clk/qcom/gcc-sm6115.c                      |    1 +
+ drivers/clk/qcom/gdsc.c                            |   47 +-
+ drivers/clk/qcom/gdsc.h                            |    1 +
+ drivers/clk/qcom/gxclkctl-kaanapali.c              |   12 +-
+ drivers/clk/qcom/videocc-hawi.c                    |  840 ++++
+ drivers/clk/qcom/videocc-x1p42100.c                |    4 +-
+ drivers/cpuidle/Kconfig.arm                        |    4 +-
+ drivers/firmware/qcom/Kconfig                      |   22 +-
+ drivers/firmware/qcom/Makefile                     |    3 +
+ drivers/firmware/qcom/qcom_pas.c                   |  298 ++
+ drivers/firmware/qcom/qcom_pas.h                   |   50 +
+ drivers/firmware/qcom/qcom_pas_tee.c               |  479 +++
+ drivers/firmware/qcom/qcom_scm-smc.c               |   10 +
+ drivers/firmware/qcom/qcom_scm.c                   |  456 +-
+ drivers/firmware/qcom/qcom_scm_trace.h             |  143 +
+ drivers/gpu/drm/msm/Kconfig                        |    1 +
+ drivers/gpu/drm/msm/adreno/a5xx_gpu.c              |    4 +-
+ drivers/gpu/drm/msm/adreno/adreno_gpu.c            |   11 +-
+ drivers/soc/qcom/Kconfig                           |  120 +-
+ drivers/soc/qcom/cmd-db.c                          |    2 +-
+ drivers/soc/qcom/ice.c                             |    4 +-
+ drivers/soc/qcom/llcc-qcom.c                       |   29 +
+ drivers/soc/qcom/qcom_pd_mapper.c                  |   38 +
+ drivers/soc/qcom/qcom_stats.c                      |    2 +
+ drivers/soc/qcom/rpmh-rsc.c                        |   37 +-
+ drivers/soc/qcom/socinfo.c                         |    1 +
+ drivers/soc/qcom/ubwc_config.c                     |  370 +-
+ include/dt-bindings/arm/qcom,ids.h                 |    1 +
+ include/dt-bindings/clock/qcom,glymur-camcc.h      |  122 +
+ include/dt-bindings/clock/qcom,glymur-evacc.h      |   38 +
+ include/dt-bindings/clock/qcom,hawi-videocc.h      |   64 +
+ include/dt-bindings/clock/qcom,ipq9650-gcc.h       |    5 +
+ include/dt-bindings/clock/qcom,shikra-gcc.h        |  263 ++
+ include/linux/firmware/qcom/qcom_pas.h             |   43 +
+ include/linux/soc/qcom/ubwc.h                      |   83 +-
+ 225 files changed, 26769 insertions(+), 2472 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/clock/qcom,glymur-evacc.yaml
+ create mode 100644 Documentation/devicetree/bindings/clock/qcom,shikra-gcc.yaml
+ create mode 100644 arch/arm/boot/dts/qcom/msm8226-motorola-titan.dts
+ create mode 100644 arch/arm/boot/dts/qcom/msm8926-sony-xperia-yukon-eagle.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/eliza-cqs-evk.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/eliza-cqs-som.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/eliza-evk.dtsi
+ delete mode 100644 arch/arm64/boot/dts/qcom/lemans-auto.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/pm7550-eliza.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/pm8550vs-d-eliza.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/pm8550vs-g-eliza.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/pmk8550-eliza.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/pmr735d-eliza.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/qcs6490-vicharak-axon-mini.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/qcs8550-rb5gen2.dts
+ delete mode 100644 arch/arm64/boot/dts/qcom/sa8775p-ride-r3.dts
+ delete mode 100644 arch/arm64/boot/dts/qcom/sa8775p-ride.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/shikra-cqm-evk.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/shikra-cqm-som.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/shikra-cqs-evk.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/shikra-evk.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/shikra-iqs-evk.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/shikra-iqs-som.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/shikra.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/sm7325-motorola-dubai-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/sm7325-motorola-dubai-csot.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/sm7325-motorola-dubai-tianma.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/sm7325-xiaomi-taoyao.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/x1e80100-honor-magicbook-art-14.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/x1p42100-microsoft-sp12in.dts
+ create mode 100644 drivers/clk/qcom/camcc-glymur.c
+ create mode 100644 drivers/clk/qcom/evacc-glymur.c
+ create mode 100644 drivers/clk/qcom/gcc-shikra.c
+ create mode 100644 drivers/clk/qcom/videocc-hawi.c
+ create mode 100644 drivers/firmware/qcom/qcom_pas.c
+ create mode 100644 drivers/firmware/qcom/qcom_pas.h
+ create mode 100644 drivers/firmware/qcom/qcom_pas_tee.c
+ create mode 100644 drivers/firmware/qcom/qcom_scm_trace.h
+ create mode 100644 include/dt-bindings/clock/qcom,glymur-camcc.h
+ create mode 100644 include/dt-bindings/clock/qcom,glymur-evacc.h
+ create mode 100644 include/dt-bindings/clock/qcom,hawi-videocc.h
+ create mode 100644 include/dt-bindings/clock/qcom,shikra-gcc.h
+ create mode 100644 include/linux/firmware/qcom/qcom_pas.h
+Merging realtek/for-next (3c778f0c9fa36 arm64: dts: realtek: Add GPIO support for RTD1625)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/yu_chun/linux.git realtek/for-next
+Merge made by the 'ort' strategy.
+ arch/arm64/boot/dts/realtek/kent.dtsi    | 41 +++++++++++++++++++++++++++++++-
+ arch/arm64/boot/dts/realtek/rtd16xx.dtsi |  3 ++-
+ 2 files changed, 42 insertions(+), 2 deletions(-)
+Merging renesas/next (f9592c355314c Merge branches 'renesas-arm-soc-for-v7.3', 'renesas-drivers-for-v7.3' and 'renesas-dts-for-v7.3' into renesas-next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git renesas/next
+Merge made by the 'ort' strategy.
+ arch/arm/boot/dts/renesas/r8a7740.dtsi             |  12 +-
+ .../arm/boot/dts/renesas/r9a06g032-rzn1d400-eb.dts |  25 +
+ arch/arm/boot/dts/renesas/r9a06g032.dtsi           |  84 ++++
+ arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c |  10 +-
+ arch/arm/mach-shmobile/setup-rcar-gen2.c           |   9 +-
+ arch/arm64/boot/dts/renesas/r8a774a1.dtsi          |   2 +-
+ arch/arm64/boot/dts/renesas/r8a774b1.dtsi          |   2 +-
+ arch/arm64/boot/dts/renesas/r8a774e1.dtsi          |   2 +-
+ arch/arm64/boot/dts/renesas/r8a77960.dtsi          |   2 +-
+ arch/arm64/boot/dts/renesas/r8a77961.dtsi          |   2 +-
+ .../arm64/boot/dts/renesas/r8a77965-salvator-x.dts |   4 +
+ .../boot/dts/renesas/r8a77965-salvator-xs.dts      |   4 +
+ arch/arm64/boot/dts/renesas/r8a77965-ulcb.dts      |   4 +
+ arch/arm64/boot/dts/renesas/r8a77965.dtsi          |   2 +-
+ arch/arm64/boot/dts/renesas/r8a77970.dtsi          |   2 +-
+ arch/arm64/boot/dts/renesas/r8a77980.dtsi          |   2 +-
+ arch/arm64/boot/dts/renesas/r8a77995.dtsi          |   2 +-
+ arch/arm64/boot/dts/renesas/r8a779g0.dtsi          |  17 +
+ arch/arm64/boot/dts/renesas/r8a78000.dtsi          |  32 ++
+ arch/arm64/boot/dts/renesas/r9a08g046.dtsi         |  69 +++
+ arch/arm64/boot/dts/renesas/r9a08g046l48-smarc.dts |  78 +++
+ arch/arm64/boot/dts/renesas/r9a09g047.dtsi         | 525 ++++++++++++++++++++-
+ arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts | 114 ++++-
+ arch/arm64/boot/dts/renesas/r9a09g056.dtsi         |  22 +-
+ arch/arm64/boot/dts/renesas/r9a09g057.dtsi         | 160 ++++++-
+ .../boot/dts/renesas/r9a09g057h44-rzv2h-evk.dts    |  18 +
+ .../boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts    | 144 ++++--
+ .../boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts    | 144 ++++--
+ arch/arm64/boot/dts/renesas/rzg3e-smarc-som.dtsi   |  44 ++
+ .../boot/dts/renesas/rzt2h-n2h-evk-common.dtsi     |  14 +
+ drivers/soc/renesas/Kconfig                        |   1 -
+ 31 files changed, 1430 insertions(+), 122 deletions(-)
+Merging reset/reset/next (d373605cd5148 Merge tag 'reset-fixes-for-v7.0-2' into reset/next)
+$ git merge -m Merge branch 'reset/next' of https://git.kernel.org/pub/scm/linux/kernel/git/pza/linux reset/reset/next
+Already up to date.
+Merging rockchip/for-next (70d6721bd79c8 Merge branch 'v7.3-armsoc/dts64' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip.git rockchip/for-next
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.yaml
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/arm/rockchip.yaml          |   29 +
+ .../devicetree/bindings/arm/rockchip/pmu.yaml      |    8 +-
+ .../devicetree/bindings/soc/rockchip/grf.yaml      |    7 +-
+ .../devicetree/bindings/vendor-prefixes.yaml       |    6 +
+ arch/arm/boot/dts/rockchip/Makefile                |    1 +
+ arch/arm/boot/dts/rockchip/rk3288-tinker.dtsi      |    5 +
+ .../dts/rockchip/rv1126-alientek-clrv1126f.dtsi    |  291 +++++
+ .../boot/dts/rockchip/rv1126-alientek-dlrv1126.dts |  256 +++++
+ arch/arm/boot/dts/rockchip/rv1126-pinctrl.dtsi     |   10 +
+ arch/arm/boot/dts/rockchip/rv1126.dtsi             |   15 +
+ arch/arm64/boot/dts/rockchip/Makefile              |    5 +
+ arch/arm64/boot/dts/rockchip/px30-cobra.dtsi       |    2 +-
+ arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi      |    2 +-
+ .../rockchip/px30-ringneck-haikou-video-demo.dtso  |   10 +-
+ arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi    |    2 +-
+ .../boot/dts/rockchip/rk3399-pinephone-pro.dts     |   18 -
+ arch/arm64/boot/dts/rockchip/rk3399-rock960.dtsi   |    2 +-
+ .../boot/dts/rockchip/rk3528-hinlink-h28k.dts      |  318 ++++++
+ .../boot/dts/rockchip/rk3568-9tripod-x3568-v4.dts  |   60 +-
+ .../dts/rockchip/rk3568-graperain-g3568-v2.dts     |  943 ++++++++++++++++
+ .../boot/dts/rockchip/rk3576-lckfb-tspi-3m.dts     | 1003 +++++++++++++++++
+ arch/arm64/boot/dts/rockchip/rk3576.dtsi           |   28 +
+ arch/arm64/boot/dts/rockchip/rk3588-base.dtsi      |    4 +-
+ arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi     |    4 +-
+ .../rockchip/rk3588-friendlyelec-cm3588-nas.dts    |   31 +
+ arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi |   14 +-
+ .../boot/dts/rockchip/rk3588-youyeetoo-yy3588.dts  | 1190 ++++++++++++++++++++
+ .../boot/dts/rockchip/rk3588s-orangepi-5-5b.dtsi   |  256 +++++
+ .../boot/dts/rockchip/rk3588s-orangepi-5-pro.dts   |  358 ++++++
+ .../arm64/boot/dts/rockchip/rk3588s-orangepi-5.dts |    6 +-
+ .../boot/dts/rockchip/rk3588s-orangepi-5.dtsi      |  263 +----
+ .../boot/dts/rockchip/rk3588s-orangepi-5b.dts      |    2 +-
+ .../boot/dts/rockchip/rk3588s-radxa-cm5-io.dts     |   19 +
+ arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts    |   29 +-
+ drivers/clk/rockchip/clk-rk3588.c                  |   10 +-
+ 35 files changed, 4898 insertions(+), 309 deletions(-)
+ create mode 100644 arch/arm/boot/dts/rockchip/rv1126-alientek-clrv1126f.dtsi
+ create mode 100644 arch/arm/boot/dts/rockchip/rv1126-alientek-dlrv1126.dts
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3528-hinlink-h28k.dts
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3568-graperain-g3568-v2.dts
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3576-lckfb-tspi-3m.dts
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3588-youyeetoo-yy3588.dts
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5-5b.dtsi
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5-pro.dts
+Merging samsung-krzk/for-next (4a7924bcff476 Merge branches 'next/clk' and 'next/dt64' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git samsung-krzk/for-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/clk/samsung/clk-exynos990.c
+Merge made by the 'ort' strategy.
+ .../bindings/arm/samsung/samsung-boards.yaml       |   6 +
+ .../bindings/soc/samsung/exynos-pmu.yaml           |  27 +-
+ .../soc/samsung/samsung,exynos-sysreg.yaml         |   1 +
+ MAINTAINERS                                        |   7 +
+ arch/arm/boot/dts/samsung/exynos5250-manta.dts     |  41 +-
+ arch/arm/mach-s3c/map-base.h                       |   2 +-
+ arch/arm64/boot/dts/exynos/Makefile                |   1 +
+ arch/arm64/boot/dts/exynos/exynos8855-pinctrl.dtsi | 574 +++++++++++++++++++++
+ arch/arm64/boot/dts/exynos/exynos8855-smdk.dts     |  31 ++
+ arch/arm64/boot/dts/exynos/exynos8855.dtsi         | 204 ++++++++
+ arch/arm64/boot/dts/exynos/exynosautov920.dtsi     |   6 +
+ drivers/clk/samsung/clk-exynos990.c                |  24 +-
+ drivers/soc/samsung/exynos-pmu.c                   |   4 +-
+ include/dt-bindings/clock/samsung,exynos990.h      |   1 +
+ include/linux/serial_s3c.h                         |   4 +-
+ 15 files changed, 914 insertions(+), 19 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/exynos/exynos8855-pinctrl.dtsi
+ create mode 100644 arch/arm64/boot/dts/exynos/exynos8855-smdk.dts
+ create mode 100644 arch/arm64/boot/dts/exynos/exynos8855.dtsi
+Merging scmi/for-linux-next (f6ce093c482b9 Merge branch 'for-next/scmi/updates' of git://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux)
+$ git merge -m Merge branch 'for-linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux.git scmi/for-linux-next
+Merge made by the 'ort' strategy.
+ drivers/firmware/arm_scmi/bus.c                | 115 +++++++++++++++----------
+ drivers/firmware/arm_scmi/common.h             |   2 +
+ drivers/firmware/arm_scmi/driver.c             |  81 ++++++++++-------
+ drivers/firmware/arm_scmi/notify.c             |  64 ++++++++------
+ drivers/firmware/arm_scmi/notify.h             |   1 +
+ drivers/firmware/arm_scmi/transports/mailbox.c |  26 ++++--
+ drivers/firmware/arm_scmi/transports/smc.c     |  15 ++--
+ 7 files changed, 190 insertions(+), 114 deletions(-)
+Merging sophgo/for-next (f7337210bede6 Merge branch 'dt/riscv' into for-next)
+$ git merge -m Merge branch 'for-next' of https://github.com/sophgo/linux.git sophgo/for-next
+Merge made by the 'ort' strategy.
+Merging sophgo-soc/soc-for-next (c8754c7deab4c soc: sophgo: cv1800: rtcsys: New driver (handling RTC only))
+$ git merge -m Merge branch 'soc-for-next' of https://github.com/sophgo/linux.git sophgo-soc/soc-for-next
+Already up to date.
+Merging spacemit/for-next (ab092591e4d9d Merge branch 'spacemit-dt-for-next' into spacemit-for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/spacemit/linux spacemit/for-next
+Merge made by the 'ort' strategy.
+ arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts |  35 +++++++-
+ arch/riscv/boot/dts/spacemit/k1-opp.dtsi        | 105 ++++++++++++++++++++++++
+ arch/riscv/boot/dts/spacemit/k1-pdma.h          |  56 +++++++++++++
+ arch/riscv/boot/dts/spacemit/k1.dtsi            |  12 ++-
+ arch/riscv/boot/dts/spacemit/k3-com260-ifx.dts  |  18 ++++
+ arch/riscv/boot/dts/spacemit/k3-pico-itx.dts    |  99 +++++++++++++++++++++-
+ arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi    |  76 +++++++++++++++++
+ arch/riscv/boot/dts/spacemit/k3.dtsi            |  30 +++++++
+ drivers/clk/spacemit/ccu-k3.c                   |  10 ++-
+ drivers/cpufreq/cpufreq-dt-platdev.c            |   2 +
+ include/dt-bindings/clock/spacemit,k3-clocks.h  |   1 +
+ 11 files changed, 437 insertions(+), 7 deletions(-)
+ create mode 100644 arch/riscv/boot/dts/spacemit/k1-opp.dtsi
+ create mode 100644 arch/riscv/boot/dts/spacemit/k1-pdma.h
+Merging stm32/stm32-next (fba4a31a7f3b6 arm64: dts: st: Fix SAI addresses on stm32mp251)
+$ git merge -m Merge branch 'stm32-next' of https://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32.git stm32/stm32-next
+Already up to date.
+Merging sunxi/sunxi/for-next (36078904c9908 Merge branches 'sunxi/clk-for-7.3', 'sunxi/config-for-7.3' and 'sunxi/dt-for-7.3' into sunxi/for-next)
+$ git merge -m Merge branch 'sunxi/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux.git sunxi/sunxi/for-next
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.yaml
+Auto-merging arch/arm64/configs/defconfig
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/sunxi.yaml   |   7 +
+ .../input/allwinner,sun4i-a10-lradc-keys.yaml      |   1 +
+ .../bindings/rtc/allwinner,sun6i-a31-rtc.yaml      |  13 ++
+ .../devicetree/bindings/vendor-prefixes.yaml       |   2 +
+ arch/arm/boot/dts/allwinner/Makefile               |   5 +
+ .../boot/dts/allwinner/sun8i-h3-nanopi-duo2.dts    |   4 +-
+ .../allwinner/sun8i-t113s-netcube-dobermann.dts    | 149 ++++++++++++++++
+ arch/arm64/boot/dts/allwinner/Makefile             |   1 +
+ arch/arm64/boot/dts/allwinner/sun50i-a100.dtsi     |  10 ++
+ .../allwinner/sun50i-a133-helperboard-core.dtsi    | 197 +++++++++++++++++++++
+ .../boot/dts/allwinner/sun50i-a133-helperboard.dts | 148 ++++++++++++++++
+ arch/arm64/configs/defconfig                       |   1 +
+ arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi  |   6 +
+ drivers/clk/sunxi-ng/ccu-sun6i-rtc.c               |  38 ++--
+ drivers/clk/sunxi-ng/ccu_common.c                  |  12 +-
+ drivers/clk/sunxi-ng/ccu_common.h                  |   1 +
+ drivers/clk/sunxi-ng/ccu_mux.c                     |   1 -
+ include/dt-bindings/clock/sun6i-rtc.h              |   4 +
+ 18 files changed, 564 insertions(+), 36 deletions(-)
+ create mode 100644 arch/arm/boot/dts/allwinner/sun8i-t113s-netcube-dobermann.dts
+ create mode 100644 arch/arm64/boot/dts/allwinner/sun50i-a133-helperboard-core.dtsi
+ create mode 100644 arch/arm64/boot/dts/allwinner/sun50i-a133-helperboard.dts
+Merging tee/next (a3067938fd192 Merge branches 'qcomtee_for_v7.3', 'optee_fix_for_v7.2' and 'jw-korg' into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/jenswi/linux-tee.git tee/next
+Merge made by the 'ort' strategy.
+ drivers/tee/optee/ffa_abi.c | 3 +++
+ drivers/tee/qcomtee/call.c  | 7 ++++++-
+ 2 files changed, 9 insertions(+), 1 deletion(-)
+Merging tegra/for-next (f98a7877ab4a8 Merge branch for-7.3/arm64/dt into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux.git tegra/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/gpu/host1x/nvidia,tegra210-nvdec.yaml |   2 +-
+ .../bindings/gpu/host1x/nvidia,tegra210-nvenc.yaml |   2 +-
+ .../bindings/gpu/host1x/nvidia,tegra210-nvjpg.yaml |   2 +-
+ .../bindings/gpu/host1x/nvidia,tegra234-nvdec.yaml |   2 +-
+ .../bindings/iommu/nvidia,tegra264-cmdqv.yaml      |   2 +-
+ arch/arm64/boot/dts/nvidia/tegra264-p3834.dtsi     |   4 +
+ arch/arm64/boot/dts/nvidia/tegra264.dtsi           | 613 +++++++++++++++++----
+ drivers/soc/tegra/pmc.c                            |  21 +-
+ 8 files changed, 537 insertions(+), 111 deletions(-)
+Merging tenstorrent-dt/tenstorrent-dt-for-next (33583baeb1ba7 dt-bindings: iommu: riscv: Add bindings for Tenstorrent RISC-V IOMMU)
+$ git merge -m Merge branch 'tenstorrent-dt-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git tenstorrent-dt/tenstorrent-dt-for-next
+Already up to date.
+Merging fustini-config/riscv-config-for-next (fd52109054693 riscv: defconfig: thead: enable PCA953X GPIO driver)
+$ git merge -m Merge branch 'riscv-config-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git fustini-config/riscv-config-for-next
+Auto-merging arch/riscv/configs/defconfig
+Merge made by the 'ort' strategy.
+ arch/riscv/configs/defconfig | 2 ++
+ 1 file changed, 2 insertions(+)
+Merging thead-dt/thead-dt-for-next (3a5791956edbf riscv: dts: thead: Enable wifi on the BeagleV-Ahead)
+$ git merge -m Merge branch 'thead-dt-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-dt/thead-dt-for-next
+Already up to date.
+Merging ti/ti-next (1b98f483adfd9 Merge branches 'ti-drivers-soc-next' and 'ti-k3-maintainer-next' into ti-next)
+$ git merge -m Merge branch 'ti-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux.git ti/ti-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS               | 1 +
+ drivers/firmware/ti_sci.c | 4 ++++
+ 2 files changed, 5 insertions(+)
+Merging xilinx/for-next (50a8e96a15137 dt-bindings: xilinx: Remove EDK/Ethernet references)
+$ git merge -m Merge branch 'for-next' of https://github.com/Xilinx/linux-xlnx.git xilinx/for-next
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/xilinx.txt       | 98 +---------------------
+ arch/arm/boot/dts/xilinx/zynq-ebaz4205.dts         |  1 -
+ arch/arm/boot/dts/xilinx/zynq-zc702.dts            |  2 -
+ arch/arm/boot/dts/xilinx/zynq-zc706.dts            |  2 -
+ .../boot/dts/xilinx/zynqmp-sck-kv-g-revA.dtso      | 37 ++++----
+ .../boot/dts/xilinx/zynqmp-sck-kv-g-revB.dtso      |  1 -
+ .../boot/dts/xilinx/zynqmp-zc1751-xm015-dc1.dts    |  4 -
+ .../boot/dts/xilinx/zynqmp-zc1751-xm019-dc5.dts    |  2 -
+ arch/arm64/boot/dts/xilinx/zynqmp-zcu100-revC.dts  |  1 -
+ arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revA.dts  |  2 -
+ arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revA.dts  |  1 -
+ arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revC.dts  |  1 -
+ arch/arm64/boot/dts/xilinx/zynqmp-zcu106-revA.dts  |  2 -
+ arch/arm64/boot/dts/xilinx/zynqmp-zcu111-revA.dts  |  1 -
+ 14 files changed, 19 insertions(+), 136 deletions(-)
+Merging socfpga/for-next (39ea7cb401924 ARM: dts: intel: arria10: Correct indentation)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dinguyen/linux.git socfpga/for-next
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/altera.yaml  |   6 +
+ .../bindings/net/altr,socfpga-stmmac.yaml          |  46 +++++-
+ .../boot/dts/intel/socfpga/socfpga_arria10.dtsi    |   2 +-
+ arch/arm64/boot/dts/intel/Makefile                 |   1 +
+ arch/arm64/boot/dts/intel/keembay-soc.dtsi         |   2 +-
+ arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi     | 112 ++++++++++++++-
+ arch/arm64/boot/dts/intel/socfpga_agilex72.dtsi    | 156 +++++++++++++++++++++
+ .../boot/dts/intel/socfpga_agilex72_socdk.dts      |  27 ++++
+ 8 files changed, 342 insertions(+), 10 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/intel/socfpga_agilex72.dtsi
+ create mode 100644 arch/arm64/boot/dts/intel/socfpga_agilex72_socdk.dts
+Merging clk/clk-next (8cdeaa50eae8d Linux 7.2-rc2)
+$ git merge -m Merge branch 'clk-next' of https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git clk/clk-next
+Already up to date.
+Merging clk-imx/for-next (b3dcc8c608fbb clk: imx: Add audio PLL debugfs for K-divider control)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/abelvesa/linux.git clk-imx/for-next
+Merge made by the 'ort' strategy.
+ drivers/clk/imx/clk-imx8mm.c  |   6 +++
+ drivers/clk/imx/clk-pll14xx.c | 119 +++++++++++++++++++++++++++++++++++++++++-
+ drivers/clk/imx/clk.h         |   1 +
+ 3 files changed, 125 insertions(+), 1 deletion(-)
+Merging clk-renesas/renesas-clk (2df26b8663b6f clk: r9a08g045: Add clocks and resets for CAN-FD)
+$ git merge -m Merge branch 'renesas-clk' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git clk-renesas/renesas-clk
+Auto-merging MAINTAINERS
+Auto-merging drivers/clk/renesas/renesas-cpg-mssr.c
+Auto-merging drivers/clk/renesas/rzv2h-cpg.c
+Merge made by the 'ort' strategy.
+ .../bindings/clock/renesas,rzv2h-cpg.yaml          |   6 +
+ .../bindings/clock/renesas,versaclock7.yaml        |   2 +-
+ MAINTAINERS                                        |   6 +-
+ drivers/clk/renesas/Kconfig                        |   6 +
+ drivers/clk/renesas/Makefile                       |   1 +
+ drivers/clk/renesas/r9a08g045-cpg.c                |   9 +
+ drivers/clk/renesas/r9a08g046-cpg.c                | 130 +++++++
+ drivers/clk/renesas/r9a09g047-cpg.c                | 123 ++++++-
+ drivers/clk/renesas/r9a09g077-cpg.c                | 375 ++++++++++++++++++++-
+ drivers/clk/renesas/renesas-cpg-mssr.c             |  20 +-
+ drivers/clk/renesas/rzv2h-cpg-lib.c                | 217 ++++++++++++
+ drivers/clk/renesas/rzv2h-cpg.c                    | 203 -----------
+ .../dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h |   2 +
+ .../dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h |   2 +
+ include/linux/clk/renesas.h                        |  34 +-
+ 15 files changed, 913 insertions(+), 223 deletions(-)
+ create mode 100644 drivers/clk/renesas/rzv2h-cpg-lib.c
+Merging thead-clk/thead-clk-for-next (b2ee00d0bf4cc clk: thead: allow COMPILE_TEST builds)
+$ git merge -m Merge branch 'thead-clk-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-clk/thead-clk-for-next
+Merge made by the 'ort' strategy.
+ drivers/clk/Makefile | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging tenstorrent-clk/tenstorrent-clk-for-next (23c8ebc952849 clk: tenstorrent: Add Atlantis clock controller driver)
+$ git merge -m Merge branch 'tenstorrent-clk-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git tenstorrent-clk/tenstorrent-clk-for-next
+Already up to date.
+Merging csky/linux-next (abb81e5ce7d99 csky: Fix a4/a5 restoration in syscall trace path)
+$ git merge -m Merge branch 'linux-next' of https://github.com/c-sky/csky-linux.git csky/linux-next
+Merge made by the 'ort' strategy.
+ arch/csky/kernel/entry.S | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+Merging loongarch/loongarch-next (8227805754a68 platform/loongarch: laptop: Stop setting acpi_device_class())
+$ git merge -m Merge branch 'loongarch-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson.git loongarch/loongarch-next
+Merge made by the 'ort' strategy.
+ arch/loongarch/include/asm/asmmacro.h        |  2 +-
+ arch/loongarch/kernel/acpi.c                 | 27 ++++++++++++++++++++++++++-
+ arch/loongarch/kernel/machine_kexec.c        |  5 ++++-
+ arch/loongarch/kernel/rethook_trampoline.S   | 18 +++++++++---------
+ arch/loongarch/kernel/smp.c                  |  4 ++--
+ arch/loongarch/net/bpf_jit.c                 |  8 ++++----
+ drivers/platform/loongarch/loongson-laptop.c |  3 ---
+ 7 files changed, 46 insertions(+), 21 deletions(-)
+Merging m68k/for-next (dc5200f6b1ada m68k: Correct CONFIG_MVME16x macro name in #endif comment)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git m68k/for-next
+Already up to date.
+Merging m68knommu/for-next (d13e97dad5664 m68k/coldfire: replace linux/gpio.h inclusions)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gerg/m68knommu.git m68knommu/for-next
+Merge made by the 'ort' strategy.
+ arch/m68k/coldfire/device.c          |  2 +-
+ arch/m68k/configs/m5235evb_defconfig | 52 ++++++++++++++++++++++++++++++++++++
+ arch/m68k/configs/m5282evb_defconfig | 51 +++++++++++++++++++++++++++++++++++
+ arch/m68k/include/asm/mcfgpio.h      |  2 +-
+ 4 files changed, 105 insertions(+), 2 deletions(-)
+ create mode 100644 arch/m68k/configs/m5235evb_defconfig
+ create mode 100644 arch/m68k/configs/m5282evb_defconfig
+Merging microblaze/next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'next' of git://git.monstr.eu/linux-2.6-microblaze.git microblaze/next
+Already up to date.
+Merging mips/mips-next (27c61f8565a43 MIPS: BCM47XX: Convert buttons to software nodes)
+$ git merge -m Merge branch 'mips-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git mips/mips-next
+Merge made by the 'ort' strategy.
+ arch/mips/bcm47xx/buttons.c           | 444 ++++++++++++++++++++--------------
+ arch/mips/include/asm/gio_device.h    |   1 -
+ arch/mips/include/asm/txx9/generic.h  |   9 +-
+ arch/mips/include/asm/txx9/tx4938.h   |   1 -
+ arch/mips/include/asm/txx9pio.h       |   3 +-
+ arch/mips/kernel/gpio_txx9.c          |   5 +-
+ arch/mips/kernel/proc.c               |   4 +-
+ arch/mips/txx9/Kconfig                |   1 -
+ arch/mips/txx9/generic/setup.c        |  45 ++--
+ arch/mips/txx9/generic/setup_tx4938.c |   6 -
+ arch/mips/txx9/rbtx4927/setup.c       |  41 +++-
+ drivers/bcma/driver_gpio.c            |  43 +++-
+ drivers/ssb/driver_gpio.c             |  48 +++-
+ include/linux/bcma/bcma.h             |   2 +
+ include/linux/ssb/ssb.h               |   2 +
+ 15 files changed, 393 insertions(+), 262 deletions(-)
+Merging openrisc/for-next (aca063c902452 openrisc: Fix jump_label smp syncing)
+$ git merge -m Merge branch 'for-next' of https://github.com/openrisc/linux.git openrisc/for-next
+Already up to date.
+Merging parisc-hd/for-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/deller/parisc-linux.git parisc-hd/for-next
+Already up to date.
+Merging powerpc/next (8cdeaa50eae8d Linux 7.2-rc2)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git powerpc/next
+Already up to date.
+Merging risc-v/for-next (1d8304bdd65f7 riscv: cfi: reduce shadow stack size limit from 2GB to 512MB)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git risc-v/for-next
+Auto-merging arch/riscv/Kconfig
+CONFLICT (content): Merge conflict in arch/riscv/Kconfig
+Auto-merging arch/riscv/include/asm/pgtable.h
+Resolved 'arch/riscv/Kconfig' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 6fef84fd898e3] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git
+$ git diff -M --stat --summary HEAD^..
+ arch/riscv/Kconfig                                 |  1 +
+ arch/riscv/include/asm/acpi.h                      |  2 ++
+ arch/riscv/include/asm/pgtable.h                   | 15 ++++++++++++--
+ arch/riscv/include/asm/tlbflush.h                  | 23 +++++++++++++++++++++
+ arch/riscv/kernel/acpi.c                           |  5 +++++
+ arch/riscv/kernel/efi.c                            |  5 +++++
+ arch/riscv/kernel/probes/decode-insn.c             |  2 +-
+ arch/riscv/kernel/probes/simulate-insn.c           |  7 +++++++
+ arch/riscv/kernel/probes/simulate-insn.h           |  1 +
+ arch/riscv/kernel/reset.c                          |  7 +++++++
+ arch/riscv/kernel/setup.c                          |  2 ++
+ arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S | 24 ++++++++++++++++++++++
+ arch/riscv/kernel/usercfi.c                        |  4 ++--
+ arch/riscv/kernel/vdso/Makefile                    |  7 ++-----
+ arch/riscv/mm/tlbflush.c                           | 23 ---------------------
+ drivers/acpi/tables.c                              |  1 +
+ 16 files changed, 96 insertions(+), 33 deletions(-)
+Merging riscv-dt/riscv-dt-for-next (82ab962ef6c2a Merge branch 'k230-basic' into riscv-dt-for-next)
+$ git merge -m Merge branch 'riscv-dt-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-dt/riscv-dt-for-next
+Auto-merging Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml
+Auto-merging Documentation/devicetree/bindings/timer/sifive,clint.yaml
+Merge made by the 'ort' strategy.
+ .../interrupt-controller/sifive,plic-1.0.0.yaml    |   1 +
+ .../devicetree/bindings/riscv/canaan.yaml          |   8 +-
+ .../devicetree/bindings/timer/sifive,clint.yaml    |   1 +
+ arch/riscv/boot/dts/canaan/Makefile                |   2 +
+ arch/riscv/boot/dts/canaan/k230-canmv.dts          | 343 +++++++++++++++++++++
+ arch/riscv/boot/dts/canaan/k230-evb.dts            |  39 +++
+ arch/riscv/boot/dts/canaan/k230-pinctrl.h          |  18 ++
+ arch/riscv/boot/dts/canaan/k230.dtsi               | 167 ++++++++++
+ 8 files changed, 578 insertions(+), 1 deletion(-)
+ create mode 100644 arch/riscv/boot/dts/canaan/k230-canmv.dts
+ create mode 100644 arch/riscv/boot/dts/canaan/k230-evb.dts
+ create mode 100644 arch/riscv/boot/dts/canaan/k230-pinctrl.h
+ create mode 100644 arch/riscv/boot/dts/canaan/k230.dtsi
+Merging riscv-soc/riscv-soc-for-next (8cdeaa50eae8d Linux 7.2-rc2)
+$ git merge -m Merge branch 'riscv-soc-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-soc/riscv-soc-for-next
+Already up to date.
+Merging s390/for-next (8b313d717bcf4 Merge branch 'features' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git s390/for-next
+Auto-merging MAINTAINERS
+Auto-merging arch/s390/Kconfig
+Auto-merging arch/s390/include/asm/hugetlb.h
+Merge made by the 'ort' strategy.
+ MAINTAINERS                         |   8 +++
+ arch/s390/Kconfig                   |   5 ++
+ arch/s390/configs/debug_defconfig   |   2 +
+ arch/s390/configs/defconfig         |   2 +
+ arch/s390/include/asm/hugetlb.h     |   4 +-
+ arch/s390/include/asm/processor.h   |   4 --
+ arch/s390/include/asm/thread_info.h |   1 +
+ arch/s390/kernel/idle.c             |   1 -
+ arch/s390/kernel/irq.c              |   4 --
+ arch/s390/kernel/smp.c              |   1 -
+ arch/s390/kernel/traps.c            |  41 ++++++++++++-
+ arch/s390/mm/hugetlbpage.c          |  10 ++--
+ drivers/cpuidle/Kconfig             |   5 ++
+ drivers/cpuidle/Kconfig.s390        |  11 ++++
+ drivers/cpuidle/Makefile            |   4 ++
+ drivers/cpuidle/cpuidle-s390.c      | 115 ++++++++++++++++++++++++++++++++++++
+ include/linux/tick.h                |   3 -
+ kernel/time/tick-sched.c            |   7 +--
+ 18 files changed, 204 insertions(+), 24 deletions(-)
+ create mode 100644 drivers/cpuidle/Kconfig.s390
+ create mode 100644 drivers/cpuidle/cpuidle-s390.c
+Merging sh/for-next (b0aa5e4b087b6 sh: Fix fallout from ZERO_PAGE consolidation)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/glaubitz/sh-linux.git sh/for-next
+Already up to date.
+Merging sparc/for-next (5b2a3b1a98fb4 sparc: Remove remaining defconfig references to the pktcdvd driver)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git sparc/for-next
+Already up to date.
+Merging uml/next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git uml/next
+Already up to date.
+Merging xtensa/xtensa-for-next (a7c958e8721eb xtensa: correct CONFIG_XTENSA_CALIBRATE_CCOUNT macro name in comment)
+$ git merge -m Merge branch 'xtensa-for-next' of https://github.com/jcmvbkbc/linux-xtensa.git xtensa/xtensa-for-next
+Merge made by the 'ort' strategy.
+ arch/xtensa/include/asm/platform.h  | 2 +-
+ arch/xtensa/platforms/iss/console.c | 5 +++--
+ 2 files changed, 4 insertions(+), 3 deletions(-)
+Merging fs-next (070f41f072722 Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git)
+$ git merge -m Merge branch 'fs-next' of linux-next fs-next
+Auto-merging Documentation/filesystems/locking.rst
+Auto-merging Documentation/filesystems/vfs.rst
+Auto-merging MAINTAINERS
+Auto-merging drivers/target/target_core_pr.c
+Auto-merging fs/hugetlbfs/inode.c
+Auto-merging fs/ocfs2/namei.c
+Auto-merging include/linux/sched.h
+Auto-merging kernel/fork.c
+Auto-merging mm/shmem.c
+Auto-merging tools/testing/selftests/Makefile
+Merge made by the 'ort' strategy.
+ Documentation/filesystems/locking.rst              |   2 +-
+ Documentation/filesystems/overlayfs.rst            |  16 +
+ Documentation/filesystems/porting.rst              |  10 +-
+ Documentation/filesystems/vfs.rst                  |   2 +-
+ Documentation/sunrpc/xdr/nfs4_1.x                  | 258 +++++-
+ MAINTAINERS                                        |  10 +-
+ block/bdev.c                                       | 125 ++-
+ drivers/base/devtmpfs.c                            |   2 +-
+ drivers/block/rnbd/rnbd-srv.c                      |   4 +-
+ drivers/char/misc_minor_kunit.c                    |  25 +-
+ drivers/crypto/ccp/sev-dev.c                       |  12 +-
+ drivers/mtd/ubi/ubi.h                              |   9 +-
+ drivers/target/target_core_alua.c                  |  11 +-
+ drivers/target/target_core_pr.c                    |   4 +-
+ fs/9p/vfs_inode.c                                  |   5 +-
+ fs/9p/vfs_inode_dotl.c                             |   4 +-
+ fs/Kconfig                                         |   1 -
+ fs/Makefile                                        |   1 -
+ fs/affs/affs.h                                     |   2 +-
+ fs/affs/namei.c                                    |   4 +-
+ fs/afs/dir.c                                       |   6 +-
+ fs/autofs/root.c                                   |   2 +-
+ fs/bad_inode.c                                     |   2 +-
+ fs/bfs/dir.c                                       |   2 +-
+ fs/bpf_fs_kfuncs.c                                 |  37 +
+ fs/btrfs/backref.c                                 |   5 +-
+ fs/btrfs/btrfs_inode.h                             |   2 +
+ fs/btrfs/dev-replace.c                             |  65 +-
+ fs/btrfs/extent_io.c                               |   4 +-
+ fs/btrfs/extent_map.c                              |   4 +-
+ fs/btrfs/inode.c                                   |   7 +-
+ fs/btrfs/ioctl.c                                   |   5 +-
+ fs/btrfs/relocation.c                              |  55 +-
+ fs/btrfs/subpage.c                                 |  26 +-
+ fs/btrfs/volumes.c                                 | 116 ++-
+ fs/btrfs/volumes.h                                 |   6 +-
+ fs/ceph/dir.c                                      |   3 +-
+ fs/coda/dir.c                                      |   9 +-
+ fs/coredump.c                                      |  11 +-
+ fs/cramfs/inode.c                                  |   2 +-
+ fs/crypto/crypto.c                                 |  40 +-
+ fs/crypto/fscrypt_private.h                        |   5 +-
+ fs/crypto/keyring.c                                |  21 +-
+ fs/crypto/keysetup.c                               |  26 +-
+ fs/crypto/policy.c                                 |  17 +-
+ fs/ecryptfs/inode.c                                |   2 +-
+ fs/efivarfs/inode.c                                |   2 +-
+ fs/efs/Kconfig                                     |  16 -
+ fs/efs/Makefile                                    |   8 -
+ fs/efs/dir.c                                       | 105 ---
+ fs/efs/efs.h                                       | 144 ----
+ fs/efs/file.c                                      |  42 -
+ fs/efs/inode.c                                     | 315 -------
+ fs/efs/namei.c                                     | 120 ---
+ fs/efs/super.c                                     | 368 --------
+ fs/efs/symlink.c                                   |  50 --
+ fs/erofs/super.c                                   |  35 +-
+ fs/exfat/exfat_fs.h                                |   2 +-
+ fs/exfat/file.c                                    | 222 ++++-
+ fs/exfat/iomap.c                                   |  22 +-
+ fs/exfat/namei.c                                   |   2 +-
+ fs/ext2/namei.c                                    |   4 +-
+ fs/ext4/extents-test.c                             |   9 +-
+ fs/ext4/mballoc-test.c                             |   9 +-
+ fs/ext4/namei.c                                    |   4 +-
+ fs/ext4/super.c                                    |  12 +-
+ fs/f2fs/data.c                                     |   5 +-
+ fs/f2fs/debug.c                                    |   4 -
+ fs/f2fs/f2fs.h                                     |  29 +-
+ fs/f2fs/file.c                                     |  28 +-
+ fs/f2fs/gc.c                                       |  36 +-
+ fs/f2fs/gc.h                                       |  27 +-
+ fs/f2fs/namei.c                                    |   4 +-
+ fs/f2fs/segment.c                                  |   9 +-
+ fs/f2fs/super.c                                    |  12 +-
+ fs/f2fs/sysfs.c                                    |  22 +-
+ fs/fat/namei_msdos.c                               |   5 +-
+ fs/fat/namei_vfat.c                                |   2 +-
+ fs/fs_struct.c                                     | 103 ++-
+ fs/fuse/dir.c                                      |  10 +-
+ fs/gfs2/inode.c                                    |   7 +-
+ fs/hfs/dir.c                                       |   4 +-
+ fs/hfsplus/dir.c                                   |   4 +-
+ fs/hostfs/hostfs_kern.c                            |   2 +-
+ fs/hpfs/namei.c                                    |   6 +-
+ fs/hugetlbfs/inode.c                               |   4 +-
+ fs/inode.c                                         |  23 +-
+ fs/internal.h                                      |   1 +
+ fs/iomap/buffered-io.c                             |   6 +-
+ fs/iomap/direct-io.c                               | 293 ++++++-
+ fs/iomap/ioend.c                                   |  21 +-
+ fs/isofs/compress.c                                |   6 +-
+ fs/jffs2/dir.c                                     |   6 +-
+ fs/jfs/namei.c                                     |   4 +-
+ fs/kernel_read_file.c                              |   9 +-
+ fs/lockd/lockd.h                                   |   2 +-
+ fs/lockd/svc.c                                     |   4 +-
+ fs/lockd/svc4proc.c                                |   4 +-
+ fs/lockd/svclock.c                                 |  42 +-
+ fs/lockd/svcproc.c                                 |   4 +-
+ fs/lockd/svcsubs.c                                 | 107 ++-
+ fs/minix/namei.c                                   |   4 +-
+ fs/namei.c                                         |  30 +-
+ fs/namespace.c                                     |  25 +-
+ fs/nfs/blocklayout/dev.c                           |  15 +-
+ fs/nfs/callback.c                                  |   4 +-
+ fs/nfs/dir.c                                       |   6 +-
+ fs/nfs/internal.h                                  |   2 +-
+ fs/nfs_common/nfslocalio.c                         |  16 +-
+ fs/nfsd/filecache.c                                | 265 ++++--
+ fs/nfsd/filecache.h                                |   3 +
+ fs/nfsd/flexfilelayoutxdr.c                        |  20 +-
+ fs/nfsd/localio.c                                  |   8 +-
+ fs/nfsd/lockd.c                                    |   6 +-
+ fs/nfsd/netns.h                                    |  29 +-
+ fs/nfsd/nfs2acl.c                                  |  21 +-
+ fs/nfsd/nfs3acl.c                                  |  17 +-
+ fs/nfsd/nfs3proc.c                                 |  40 +
+ fs/nfsd/nfs4callback.c                             | 207 ++++-
+ fs/nfsd/nfs4layouts.c                              |  45 +-
+ fs/nfsd/nfs4proc.c                                 | 114 ++-
+ fs/nfsd/nfs4recover.c                              |  48 +-
+ fs/nfsd/nfs4state.c                                | 944 +++++++++++++++++----
+ fs/nfsd/nfs4xdr.c                                  | 392 ++++++++-
+ fs/nfsd/nfs4xdr_gen.c                              | 600 ++++++++++++-
+ fs/nfsd/nfs4xdr_gen.h                              |  17 +-
+ fs/nfsd/nfscache.c                                 |   4 +-
+ fs/nfsd/nfsctl.c                                   | 156 +++-
+ fs/nfsd/nfsd.h                                     |   9 +-
+ fs/nfsd/nfsfh.c                                    |  42 +-
+ fs/nfsd/nfsfh.h                                    |   3 +
+ fs/nfsd/nfsproc.c                                  |   7 +
+ fs/nfsd/nfssvc.c                                   |  61 +-
+ fs/nfsd/nfsxdr.c                                   |  32 +-
+ fs/nfsd/state.h                                    |  88 +-
+ fs/nfsd/trace.h                                    |  42 +-
+ fs/nfsd/vfs.c                                      |  48 +-
+ fs/nfsd/xdr4.h                                     |   5 +
+ fs/nfsd/xdr4cb.h                                   |  12 +
+ fs/nilfs2/namei.c                                  |   4 +-
+ fs/notify/fanotify/fanotify.c                      |   1 +
+ fs/ntfs/attrib.c                                   |  15 +
+ fs/ntfs/compress.c                                 | 102 +--
+ fs/ntfs/dir.c                                      |   5 +-
+ fs/ntfs/ea.c                                       |  85 +-
+ fs/ntfs/file.c                                     |  71 +-
+ fs/ntfs/inode.c                                    |   6 +-
+ fs/ntfs/inode.h                                    |   2 +-
+ fs/ntfs/iomap.c                                    |  34 +-
+ fs/ntfs/mft.c                                      |   3 +-
+ fs/ntfs/namei.c                                    |   7 +-
+ fs/ntfs/reparse.c                                  |   6 +-
+ fs/ntfs/runlist.c                                  |  50 +-
+ fs/ntfs3/dir.c                                     |  36 +-
+ fs/ntfs3/file.c                                    |   2 +-
+ fs/ntfs3/frecord.c                                 |  31 +-
+ fs/ntfs3/fslog.c                                   |  21 +-
+ fs/ntfs3/fsntfs.c                                  |   8 +-
+ fs/ntfs3/index.c                                   |   7 +-
+ fs/ntfs3/inode.c                                   |  18 +-
+ fs/ntfs3/lznt.c                                    |   4 +-
+ fs/ntfs3/namei.c                                   |  13 +-
+ fs/ntfs3/ntfs_fs.h                                 |   6 +-
+ fs/ntfs3/run.c                                     |   9 +-
+ fs/ntfs3/super.c                                   |   9 +-
+ fs/ntfs3/xattr.c                                   |   1 -
+ fs/nullfs.c                                        |  14 +-
+ fs/ocfs2/dlmfs/dlmfs.c                             |   5 +-
+ fs/ocfs2/namei.c                                   |   5 +-
+ fs/ocfs2/super.c                                   |   1 -
+ fs/omfs/dir.c                                      |   4 +-
+ fs/orangefs/namei.c                                |   5 +-
+ fs/overlayfs/dir.c                                 |  18 +-
+ fs/overlayfs/inode.c                               |  26 +-
+ fs/overlayfs/overlayfs.h                           |   1 +
+ fs/overlayfs/super.c                               |   2 +-
+ fs/overlayfs/xattrs.c                              |   1 +
+ fs/posix_acl.c                                     |   2 -
+ fs/proc/array.c                                    |   4 +-
+ fs/proc/base.c                                     |   8 +-
+ fs/proc_namespace.c                                |   4 +-
+ fs/ramfs/inode.c                                   |   4 +-
+ fs/romfs/super.c                                   |  10 +-
+ fs/smb/client/cifsfs.h                             |   2 +-
+ fs/smb/client/dir.c                                |   2 +-
+ fs/smb/client/inode.c                              |   7 +
+ fs/smb/server/compress.c                           |   3 +-
+ fs/smb/server/connection.c                         |  16 +-
+ fs/smb/server/ksmbd_netlink.h                      |   1 +
+ fs/smb/server/mgmt/share_config.c                  |   4 +-
+ fs/smb/server/mgmt/user_session.c                  |  25 +-
+ fs/smb/server/mgmt/user_session.h                  |   2 +-
+ fs/smb/server/oplock.c                             |   6 +-
+ fs/smb/server/smb2misc.c                           |   5 +
+ fs/smb/server/smb2pdu.c                            | 228 ++++-
+ fs/smb/server/smbacl.c                             | 147 +++-
+ fs/smb/server/smbacl.h                             |   3 +-
+ fs/smb/server/vfs.c                                |  17 +-
+ fs/super.c                                         | 639 +++++++++-----
+ fs/ubifs/dir.c                                     |   4 +-
+ fs/udf/balloc.c                                    |  12 +-
+ fs/udf/inode.c                                     |  26 +-
+ fs/udf/namei.c                                     |   4 +-
+ fs/udf/partition.c                                 |   2 +-
+ fs/udf/super.c                                     |  23 +-
+ fs/udf/truncate.c                                  |   2 +-
+ fs/udf/udfdecl.h                                   |   3 +-
+ fs/ufs/namei.c                                     |   5 +-
+ fs/vboxsf/dir.c                                    |   4 +-
+ fs/xfs/xfs_buf.c                                   |   2 +-
+ fs/xfs/xfs_iops.c                                  |   7 +-
+ fs/xfs/xfs_super.c                                 |  10 +-
+ include/linux/blk_types.h                          |   2 +-
+ include/linux/blkdev.h                             |  12 +-
+ include/linux/efs_vh.h                             |  54 --
+ include/linux/fs.h                                 |   4 +-
+ include/linux/fs/super.h                           |   8 +
+ include/linux/fs/super_types.h                     |   4 +-
+ include/linux/fs_struct.h                          |  34 +
+ include/linux/init_task.h                          |   1 +
+ include/linux/lockd/bind.h                         |  12 +-
+ include/linux/net.h                                |   1 +
+ include/linux/nfs4.h                               | 127 ---
+ include/linux/sched.h                              |   1 +
+ include/linux/sched/task.h                         |   1 +
+ include/linux/sunrpc/bc_xprt.h                     |   5 +
+ include/linux/sunrpc/svc.h                         |   1 +
+ include/linux/sunrpc/svc_rdma_pcl.h                |   4 +-
+ include/linux/sunrpc/xdrgen/nfs4_1.h               | 290 ++++++-
+ include/linux/types.h                              |   2 +
+ include/uapi/asm-generic/errno.h                   |   2 +-
+ include/uapi/linux/btrfs.h                         |   2 +-
+ include/uapi/linux/efs_fs_sb.h                     |  63 --
+ include/uapi/linux/fscrypt.h                       |   1 -
+ include/uapi/linux/nfs4.h                          |   2 -
+ init/init_task.c                                   |   1 +
+ init/initramfs.c                                   |  14 +-
+ init/initramfs_test.c                              | 220 ++---
+ init/main.c                                        |  10 +-
+ ipc/mqueue.c                                       |   7 +-
+ kernel/exit.c                                      |   2 +-
+ kernel/fork.c                                      |  53 +-
+ kernel/kcmp.c                                      |   2 +-
+ kernel/umh.c                                       |   6 +-
+ mm/shmem.c                                         |   2 +-
+ net/socket.c                                       |  25 +
+ net/sunrpc/auth_gss/auth_gss.c                     |   6 +-
+ net/sunrpc/auth_gss/gss_krb5_unseal.c              |   3 +
+ net/sunrpc/auth_gss/gss_krb5_wrap.c                |  13 +-
+ net/sunrpc/auth_gss/gss_rpc_upcall.c               |   6 -
+ net/sunrpc/auth_gss/gss_rpc_upcall.h               |   1 -
+ net/sunrpc/auth_gss/gss_rpc_xdr.c                  |  15 +-
+ net/sunrpc/auth_gss/svcauth_gss.c                  |   8 +-
+ net/sunrpc/backchannel_rqst.c                      |  38 +-
+ net/sunrpc/cache.c                                 |   7 +-
+ net/sunrpc/sunrpc_syms.c                           |   1 +
+ net/sunrpc/svc.c                                   |  81 +-
+ net/sunrpc/svcauth_unix.c                          |   4 +-
+ net/sunrpc/xdr.c                                   |   2 +-
+ net/sunrpc/xprtrdma/ib_client.c                    |  42 +-
+ net/sunrpc/xprtrdma/svc_rdma_pcl.c                 |  63 +-
+ net/sunrpc/xprtrdma/svc_rdma_recvfrom.c            |  24 +-
+ net/sunrpc/xprtrdma/svc_rdma_rw.c                  |  52 +-
+ net/sunrpc/xprtrdma/svc_rdma_transport.c           |  69 +-
+ net/unix/af_unix.c                                 |  17 +-
+ tools/include/uapi/linux/fscrypt.h                 |   1 -
+ tools/testing/selftests/Makefile                   |   1 +
+ tools/testing/selftests/bpf/bpf_experimental.h     |   3 +
+ .../testing/selftests/bpf/prog_tests/sock_xattr.c  |  67 ++
+ .../testing/selftests/bpf/progs/sock_read_xattr.c  |  54 ++
+ tools/testing/selftests/filesystems/.gitignore     |   1 +
+ tools/testing/selftests/filesystems/Makefile       |   2 +-
+ tools/testing/selftests/filesystems/ceph/Makefile  |   7 +
+ tools/testing/selftests/filesystems/ceph/README    |  84 ++
+ .../filesystems/ceph/reset_corner_cases.sh         | 646 ++++++++++++++
+ .../selftests/filesystems/ceph/reset_stress.sh     | 694 +++++++++++++++
+ .../selftests/filesystems/ceph/run_validation.sh   | 350 ++++++++
+ tools/testing/selftests/filesystems/ceph/settings  |   1 +
+ .../filesystems/ceph/validate_consistency.py       | 297 +++++++
+ .../selftests/filesystems/overlayfs/.gitignore     |   1 +
+ .../selftests/filesystems/overlayfs/Makefile       |   2 +
+ .../filesystems/overlayfs/idmapped_mounts.c        | 501 +++++++++++
+ .../filesystems/overlayfs/set_layers_via_fds.c     |  16 +-
+ .../filesystems/statmount/statmount_test.c         |   5 +-
+ tools/testing/selftests/filesystems/ustat_test.c   | 135 +++
+ tools/testing/selftests/proc/proc-pidns.c          |   1 +
+ 286 files changed, 9511 insertions(+), 3386 deletions(-)
+ delete mode 100644 fs/efs/Kconfig
+ delete mode 100644 fs/efs/Makefile
+ delete mode 100644 fs/efs/dir.c
+ delete mode 100644 fs/efs/efs.h
+ delete mode 100644 fs/efs/file.c
+ delete mode 100644 fs/efs/inode.c
+ delete mode 100644 fs/efs/namei.c
+ delete mode 100644 fs/efs/super.c
+ delete mode 100644 fs/efs/symlink.c
+ delete mode 100644 include/linux/efs_vh.h
+ delete mode 100644 include/uapi/linux/efs_fs_sb.h
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/sock_xattr.c
+ create mode 100644 tools/testing/selftests/bpf/progs/sock_read_xattr.c
+ create mode 100644 tools/testing/selftests/filesystems/ceph/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/ceph/README
+ create mode 100755 tools/testing/selftests/filesystems/ceph/reset_corner_cases.sh
+ create mode 100755 tools/testing/selftests/filesystems/ceph/reset_stress.sh
+ create mode 100755 tools/testing/selftests/filesystems/ceph/run_validation.sh
+ create mode 100644 tools/testing/selftests/filesystems/ceph/settings
+ create mode 100755 tools/testing/selftests/filesystems/ceph/validate_consistency.py
+ create mode 100644 tools/testing/selftests/filesystems/overlayfs/idmapped_mounts.c
+ create mode 100644 tools/testing/selftests/filesystems/ustat_test.c
+Merging printk/for-next (703a9d76a45dd Merge branch 'for-7.3-console-registration-cleanup' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/printk/linux.git printk/for-next
+Merge made by the 'ort' strategy.
+ kernel/printk/printk.c     | 30 +++++++++++++++++-------------
+ lib/tests/test_ratelimit.c | 29 +++++++++++++++++++----------
+ lib/vsprintf.c             |  4 ++++
+ 3 files changed, 40 insertions(+), 23 deletions(-)
+Merging pci/next (41d034327abe1 Merge branch 'pci/misc')
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git pci/next
+Auto-merging MAINTAINERS
+Auto-merging arch/arm64/Kconfig
+Auto-merging drivers/irqchip/irq-gic-v3-its.c
+Auto-merging drivers/pci/controller/dwc/pci-meson.c
+Auto-merging drivers/pci/controller/dwc/pcie-spacemit-k1.c
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-bus-pci            |   2 +-
+ Documentation/arch/arm64/silicon-errata.rst        |   1 +
+ .../devicetree/bindings/pci/fsl,imx6q-pcie.yaml    |  25 +++
+ .../bindings/pci/mediatek-pcie-gen3.yaml           |   2 +
+ .../devicetree/bindings/pci/qcom,hawi-pcie.yaml    | 201 ++++++++++++++++++
+ .../devicetree/bindings/pci/qcom,pcie-ipq9574.yaml |  15 ++
+ .../bindings/pci/renesas,r9a08g045-pcie.yaml       |  34 ++-
+ MAINTAINERS                                        |   1 +
+ arch/alpha/kernel/pci-sysfs.c                      |   6 +-
+ arch/arm64/Kconfig                                 |   9 +
+ arch/powerpc/kernel/pci-common.c                   |   9 +-
+ drivers/irqchip/irq-gic-v3-its.c                   |  24 ++-
+ drivers/pci/controller/Kconfig                     |   2 +-
+ drivers/pci/controller/cadence/pcie-cadence-plat.c |   1 +
+ drivers/pci/controller/dwc/Kconfig                 |   6 +-
+ drivers/pci/controller/dwc/pci-imx6.c              |  63 +++---
+ drivers/pci/controller/dwc/pci-keystone.c          |   6 +-
+ drivers/pci/controller/dwc/pci-meson.c             |   2 +-
+ drivers/pci/controller/dwc/pcie-designware-host.c  |  10 +-
+ drivers/pci/controller/dwc/pcie-qcom.c             |  38 ++++
+ drivers/pci/controller/dwc/pcie-rcar-gen4.c        | 118 ++++++++++-
+ drivers/pci/controller/dwc/pcie-spacemit-k1.c      |   1 +
+ drivers/pci/controller/pci-host-common.c           |  29 +--
+ drivers/pci/controller/pci-xgene.c                 |   5 +-
+ drivers/pci/controller/pcie-mediatek.c             | 154 ++++++++++++++
+ drivers/pci/controller/pcie-rzg3s-host.c           | 233 +++++++++++++++++++--
+ drivers/pci/hotplug/acpiphp_ibm.c                  |   3 +-
+ drivers/pci/pci-sysfs.c                            |  38 +++-
+ drivers/pci/pcie/portdrv.c                         |   2 +-
+ 29 files changed, 928 insertions(+), 112 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/pci/qcom,hawi-pcie.yaml
+Merging pstore/for-next/pstore (24b8f8dcb9a13 pstore/ftrace: Factor KASLR offset in the core kernel instruction addresses)
+$ git merge -m Merge branch 'for-next/pstore' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git pstore/for-next/pstore
+Already up to date.
+Merging hid/for-next (194a485768438 Merge branch 'for-7.2/upstream-fixes' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/hid/hid.git hid/for-next
+Merge made by the 'ort' strategy.
+ drivers/hid/Kconfig           |   9 ++
+ drivers/hid/hid-core.c        |   7 +
+ drivers/hid/hid-ids.h         |   2 +
+ drivers/hid/hid-nintendo.c    |   1 -
+ drivers/hid/hid-roccat-kone.c |  65 ++++++++-
+ drivers/hid/hid-sony.c        | 245 +++++++++++++++++----------------
+ drivers/hid/hid-steelseries.c | 313 ++++++++++++++++++++++++++++++++++++++++--
+ drivers/hid/wacom_wac.c       |   8 +-
+ 8 files changed, 516 insertions(+), 134 deletions(-)
+Merging i2c/i2c/for-next (8cd9520d35a6c Linux 7.1)
+$ git merge -m Merge branch 'i2c/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wsa/linux.git i2c/i2c/for-next
+Already up to date.
+$ git am -3 ../patches/0001-i2c-Fix-up-the-rest-of-the-merge.patch
+Applying: i2c: Fix up the rest of the merge
+Using index info to reconstruct a base tree...
+M	drivers/i2c/i2c-core-base.c
+Falling back to patching base and 3-way merge...
+Auto-merging drivers/i2c/i2c-core-base.c
+No changes -- Patch already applied.
+Merging i2c-andi/i2c/i2c-next (c844ed7266c9d Merge branch 'i2c/i2c-fixes' into i2c/i2c-next)
+$ git merge -m Merge branch 'i2c/i2c-next' of https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git i2c-andi/i2c/i2c-next
+Merge made by the 'ort' strategy.
+ .../bindings/i2c/altr,softip-i2c-v1.0.yaml         |  62 ++++
+ .../devicetree/bindings/i2c/i2c-altera.txt         |  39 ---
+ .../devicetree/bindings/i2c/i2c-axxia.txt          |  30 --
+ .../devicetree/bindings/i2c/lsi,api2c.yaml         |  52 ++++
+ .../bindings/i2c/qcom,sa8255p-geni-i2c.yaml        |  64 ++++
+ drivers/i2c/busses/Kconfig                         |   2 +-
+ drivers/i2c/busses/i2c-amd-asf-plat.c              |   4 +
+ drivers/i2c/busses/i2c-davinci.c                   |   4 +-
+ drivers/i2c/busses/i2c-k1.c                        | 183 +++++++++++-
+ drivers/i2c/busses/i2c-microchip-corei2c.c         |   4 +-
+ drivers/i2c/busses/i2c-nomadik.c                   |   4 +-
+ drivers/i2c/busses/i2c-octeon-core.h               |   2 +-
+ drivers/i2c/busses/i2c-pnx.c                       |   3 +-
+ drivers/i2c/busses/i2c-qcom-geni.c                 | 328 ++++++++++-----------
+ drivers/i2c/i2c-core-acpi.c                        |   6 +-
+ 15 files changed, 522 insertions(+), 265 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/i2c/altr,softip-i2c-v1.0.yaml
+ delete mode 100644 Documentation/devicetree/bindings/i2c/i2c-altera.txt
+ delete mode 100644 Documentation/devicetree/bindings/i2c/i2c-axxia.txt
+ create mode 100644 Documentation/devicetree/bindings/i2c/lsi,api2c.yaml
+ create mode 100644 Documentation/devicetree/bindings/i2c/qcom,sa8255p-geni-i2c.yaml
+Merging i2c-rust/rust-i2c-next (c0260c740d207 i2c: rust: mark I2cAdapter methods as inline)
+$ git merge -m Merge branch 'rust-i2c-next' of https://github.com/ikrtn/rust-for-linux i2c-rust/rust-i2c-next
+Auto-merging rust/kernel/i2c.rs
+Merge made by the 'ort' strategy.
+ rust/kernel/i2c.rs | 3 +++
+ 1 file changed, 3 insertions(+)
+Merging i3c/i3c/next (eeedacd0866ee i3c: dw: avoid shift-out-of-bounds when DAA assigns no devices)
+$ git merge -m Merge branch 'i3c/next' of https://git.kernel.org/pub/scm/linux/kernel/git/i3c/linux.git i3c/i3c/next
+Merge made by the 'ort' strategy.
+ drivers/i3c/master/adi-i3c-master.c | 15 +++++++--------
+ drivers/i3c/master/dw-i3c-master.c  | 10 +++++++++-
+ drivers/i3c/master/svc-i3c-master.c | 18 +++++++++++++++---
+ 3 files changed, 31 insertions(+), 12 deletions(-)
+Merging dmi/dmi-for-next (1afafbaf749d8 firmware/dmi: Include product_family info to modalias)
+$ git merge -m Merge branch 'dmi-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jdelvare/staging.git dmi/dmi-for-next
+Already up to date.
+Merging hwmon-staging/hwmon-next (ca078d004cf58 hwmon: (asus_ec_sensors) add ProArt Z690-CREATOR WIFI)
+$ git merge -m Merge branch 'hwmon-next' of https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git hwmon-staging/hwmon-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/hwmon/cros_ec_hwmon.c
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/hwmon/adi,adm1275.yaml     |  43 +-
+ .../bindings/hwmon/amphenol,chipcap2.yaml          |   6 +
+ .../bindings/hwmon/eswin,eic7700-pvt.yaml          |  72 +++
+ .../bindings/hwmon/pmbus/silergy,sq24860.yaml      |  74 +++
+ .../bindings/hwmon/pmbus/ti,lm25066.yaml           |  20 +
+ .../devicetree/bindings/hwmon/ti,ina2xx.yaml       |   3 +
+ Documentation/hwmon/adm1275.rst                    |  24 +
+ Documentation/hwmon/asus_ec_sensors.rst            |   4 +
+ Documentation/hwmon/chipcap2.rst                   |   2 +
+ Documentation/hwmon/coretemp.rst                   |   4 +-
+ Documentation/hwmon/index.rst                      |   2 +
+ Documentation/hwmon/sq24860.rst                    |  96 ++++
+ Documentation/hwmon/tvs-mpfs.rst                   |  53 +++
+ MAINTAINERS                                        |   1 +
+ drivers/hwmon/Kconfig                              |  24 +
+ drivers/hwmon/Makefile                             |   2 +
+ drivers/hwmon/acpi_power_meter.c                   |   6 +-
+ drivers/hwmon/asus-ec-sensors.c                    |  33 ++
+ drivers/hwmon/chipcap2.c                           |  25 +-
+ drivers/hwmon/coretemp.c                           |   2 +-
+ drivers/hwmon/cros_ec_hwmon.c                      |  12 +-
+ drivers/hwmon/eic7700-pvt.c                        | 509 +++++++++++++++++++++
+ drivers/hwmon/eic7700-pvt.h                        |  99 ++++
+ drivers/hwmon/ina2xx.c                             |  20 +
+ drivers/hwmon/pmbus/Kconfig                        |  33 +-
+ drivers/hwmon/pmbus/Makefile                       |   1 +
+ drivers/hwmon/pmbus/adm1275.c                      | 142 +++++-
+ drivers/hwmon/pmbus/lm25066.c                      |  37 ++
+ drivers/hwmon/pmbus/sq24860.c                      | 430 +++++++++++++++++
+ drivers/hwmon/pmbus/xdpe1a2g7b.c                   |  11 +
+ drivers/hwmon/tvs-mpfs.c                           | 390 ++++++++++++++++
+ drivers/hwmon/xgene-hwmon.c                        |   4 -
+ drivers/hwmon/yogafan.c                            |  14 +
+ 33 files changed, 2162 insertions(+), 36 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/hwmon/eswin,eic7700-pvt.yaml
+ create mode 100644 Documentation/devicetree/bindings/hwmon/pmbus/silergy,sq24860.yaml
+ create mode 100644 Documentation/hwmon/sq24860.rst
+ create mode 100644 Documentation/hwmon/tvs-mpfs.rst
+ create mode 100644 drivers/hwmon/eic7700-pvt.c
+ create mode 100644 drivers/hwmon/eic7700-pvt.h
+ create mode 100644 drivers/hwmon/pmbus/sq24860.c
+ create mode 100644 drivers/hwmon/tvs-mpfs.c
+Merging jc_docs/docs-next (be828e6819ed1 docs: pt_BR: Translate deprecated interfaces guide to Portuguese)
+$ git merge -m Merge branch 'docs-next' of git://git.lwn.net/linux.git jc_docs/docs-next
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging Documentation/admin-guide/mm/pagemap.rst
+Auto-merging Documentation/filesystems/overlayfs.rst
+Merge made by the 'ort' strategy.
+ Documentation/ABI/README                           |   2 +-
+ Documentation/admin-guide/kernel-parameters.txt    |  23 +-
+ Documentation/admin-guide/media/bttv.rst           |   2 +-
+ Documentation/admin-guide/mm/pagemap.rst           |   8 +-
+ Documentation/conf.py                              |   4 +-
+ Documentation/core-api/SMP.rst                     |  11 +
+ Documentation/core-api/dma-api.rst                 |   2 +-
+ Documentation/core-api/errseq.rst                  |   2 +-
+ Documentation/core-api/index.rst                   |   1 +
+ Documentation/core-api/list.rst                    |   2 +-
+ Documentation/core-api/packing.rst                 |   4 +-
+ Documentation/dev-tools/container.rst              |   6 +-
+ .../driver-api/usb/writing_usb_driver.rst          |   3 -
+ Documentation/filesystems/overlayfs.rst            |   3 +-
+ Documentation/kernel-hacking/locking.rst           |   8 +-
+ Documentation/process/submitting-patches.rst       |  14 +-
+ Documentation/scheduler/sched-energy.rst           |   2 +-
+ Documentation/scheduler/sched-pelt.c               |   3 +-
+ Documentation/sphinx-static/custom.css             |  11 +-
+ Documentation/sphinx/maintainers_include.py        |   7 +-
+ .../ja_JP/process/submitting-patches.rst           | 148 ++---
+ Documentation/translations/pt_BR/index.rst         |  11 +
+ .../translations/pt_BR/process/4.Coding.rst        | 440 +++++++++++++
+ .../translations/pt_BR/process/5.Posting.rst       | 376 +++++++++++
+ .../translations/pt_BR/process/6.Followthrough.rst | 220 +++++++
+ .../pt_BR/process/7.AdvancedTopics.rst             | 201 ++++++
+ .../translations/pt_BR/process/8.Conclusion.rst    |  73 +++
+ .../translations/pt_BR/process/adding-syscalls.rst | 700 +++++++++++++++++++++
+ .../pt_BR/process/applying-patches.rst             | 447 +++++++++++++
+ .../translations/pt_BR/process/backporting.rst     | 598 ++++++++++++++++++
+ .../pt_BR/process/botching-up-ioctls.rst           | 256 ++++++++
+ .../process/code-of-conduct-interpretation.rst     | 257 ++++++++
+ .../translations/pt_BR/process/code-of-conduct.rst |  88 +++
+ .../pt_BR/process/contribution-maturity-model.rst  | 111 ++++
+ .../translations/pt_BR/process/deprecated.rst      | 422 +++++++++++++
+ .../pt_BR/process/development-process.rst          |   5 +
+ .../pt_BR/process/kernel-driver-statement.rst      | 205 ++++++
+ .../pt_BR/process/maintainer-netdev.rst            |  23 +-
+ .../pt_BR/process/submit-checklist.rst             | 145 +++++
+ .../translations/zh_CN/scheduler/sched-energy.rst  |   2 +-
+ README                                             |  18 +-
+ tools/lib/python/kdoc/kdoc_output.py               |   2 +-
+ 42 files changed, 4720 insertions(+), 146 deletions(-)
+ create mode 100644 Documentation/core-api/SMP.rst
+ create mode 100644 Documentation/translations/pt_BR/process/4.Coding.rst
+ create mode 100644 Documentation/translations/pt_BR/process/5.Posting.rst
+ create mode 100644 Documentation/translations/pt_BR/process/6.Followthrough.rst
+ create mode 100644 Documentation/translations/pt_BR/process/7.AdvancedTopics.rst
+ create mode 100644 Documentation/translations/pt_BR/process/8.Conclusion.rst
+ create mode 100644 Documentation/translations/pt_BR/process/adding-syscalls.rst
+ create mode 100644 Documentation/translations/pt_BR/process/applying-patches.rst
+ create mode 100644 Documentation/translations/pt_BR/process/backporting.rst
+ create mode 100644 Documentation/translations/pt_BR/process/botching-up-ioctls.rst
+ create mode 100644 Documentation/translations/pt_BR/process/code-of-conduct-interpretation.rst
+ create mode 100644 Documentation/translations/pt_BR/process/code-of-conduct.rst
+ create mode 100644 Documentation/translations/pt_BR/process/contribution-maturity-model.rst
+ create mode 100644 Documentation/translations/pt_BR/process/deprecated.rst
+ create mode 100644 Documentation/translations/pt_BR/process/kernel-driver-statement.rst
+ create mode 100644 Documentation/translations/pt_BR/process/submit-checklist.rst
+Merging v4l-dvb/next (a52e6f7923c17 media: chips-media: wave5: Resume device before setting EOS flag)
+$ git merge -m Merge branch 'next' of git://linuxtv.org/media-ci/media-pending.git v4l-dvb/next
+Auto-merging CREDITS
+Auto-merging MAINTAINERS
+Auto-merging drivers/media/i2c/cvs/core.c
+Auto-merging drivers/media/i2c/ov8865.c
+Auto-merging drivers/media/i2c/video-i2c.c
+Auto-merging drivers/media/usb/em28xx/em28xx.h
+Merge made by the 'ort' strategy.
+ CREDITS                                            |    7 +
+ .../admin-guide/media/em28xx-cardlist.rst          |    4 +
+ .../devicetree/bindings/media/i2c/sony,imx355.yaml |   11 +-
+ .../devicetree/bindings/media/i2c/sony,imx678.yaml |  130 ++
+ .../devicetree/bindings/media/i2c/st,vd55g1.yaml   |    3 +-
+ Documentation/devicetree/bindings/media/rc.yaml    |    4 +-
+ .../driver-api/media/maintainer-entry-profile.rst  |    4 +
+ .../media/cec/cec-ioc-adap-g-log-addrs.rst         |    7 +
+ .../userspace-api/media/v4l/vidioc-g-ext-ctrls.rst |   15 +-
+ MAINTAINERS                                        |   43 +-
+ drivers/acpi/scan.c                                |    1 +
+ drivers/media/cec/core/cec-adap.c                  |   95 +-
+ drivers/media/cec/core/cec-api.c                   |    5 +-
+ drivers/media/cec/core/cec-core.c                  |  117 +-
+ drivers/media/cec/core/cec-pin.c                   |   18 +-
+ drivers/media/cec/platform/meson/ao-cec-g12a.c     |    1 +
+ drivers/media/cec/platform/seco/seco-cec.c         |    5 +-
+ drivers/media/cec/platform/stm32/stm32-cec.c       |   19 +-
+ drivers/media/cec/platform/tegra/tegra_cec.c       |   33 +-
+ .../extron-da-hd-4k-plus/extron-da-hd-4k-plus.c    |    3 +-
+ drivers/media/common/v4l2-tpg/v4l2-tpg-core.c      |   14 +-
+ drivers/media/common/videobuf2/videobuf2-v4l2.c    |   13 -
+ drivers/media/dvb-core/dvb_frontend.c              |  157 +-
+ drivers/media/dvb-frontends/helene.c               |   56 +-
+ drivers/media/dvb-frontends/rtl2832_sdr.c          |   23 +-
+ drivers/media/i2c/Kconfig                          |   23 +
+ drivers/media/i2c/Makefile                         |    2 +
+ drivers/media/i2c/alvium-csi2.c                    |    2 +-
+ drivers/media/i2c/cvs/core.c                       |    3 +
+ drivers/media/i2c/gc0310.c                         |   30 +-
+ drivers/media/i2c/imx355.c                         | 1524 +++++----------
+ drivers/media/i2c/imx412.c                         |  751 ++++----
+ drivers/media/i2c/imx471.c                         |  957 ++++++++++
+ drivers/media/i2c/imx678.c                         | 1447 +++++++++++++++
+ drivers/media/i2c/ov02a10.c                        |   12 +-
+ drivers/media/i2c/ov7740.c                         |   10 +-
+ drivers/media/i2c/ov8865.c                         |   48 +-
+ drivers/media/i2c/ov9282.c                         |    1 -
+ drivers/media/i2c/rdacm21.c                        |    5 +-
+ drivers/media/i2c/vd55g1.c                         |  160 +-
+ drivers/media/i2c/video-i2c.c                      |    8 +-
+ drivers/media/pci/bt8xx/bttv-driver.c              |   12 +-
+ drivers/media/pci/cobalt/cobalt-alsa-main.c        |    1 -
+ drivers/media/pci/cobalt/cobalt-driver.c           |    5 +-
+ drivers/media/pci/cx18/cx18-driver.c               |    5 +-
+ drivers/media/pci/cx18/cx18-vbi.c                  |    2 +-
+ drivers/media/pci/cx23885/cx23885-core.c           |   10 +-
+ drivers/media/pci/cx25821/cx25821-alsa.c           |    4 +-
+ drivers/media/pci/cx25821/cx25821-core.c           |   10 +-
+ drivers/media/pci/cx88/cx88-alsa.c                 |    6 +-
+ drivers/media/pci/cx88/cx88-mpeg.c                 |    5 +-
+ drivers/media/pci/cx88/cx88-video.c                |    5 +-
+ drivers/media/pci/ddbridge/ddbridge-main.c         |    2 +-
+ drivers/media/pci/ddbridge/ddbridge.h              |    2 +-
+ drivers/media/pci/dm1105/dm1105.c                  |   10 +-
+ drivers/media/pci/dt3155/dt3155.c                  |    4 +-
+ drivers/media/pci/hws/hws_v4l2_ioctl.h             |    4 -
+ drivers/media/pci/intel/ipu-bridge.c               |   51 +-
+ drivers/media/pci/intel/ipu3/ipu3-cio2.c           |    2 +-
+ drivers/media/pci/intel/ipu6/ipu6.c                |   10 +-
+ drivers/media/pci/ivtv/ivtv-driver.c               |    8 +-
+ drivers/media/pci/ivtv/ivtv-vbi.c                  |    2 +-
+ drivers/media/pci/mantis/mantis_common.h           |    8 +-
+ drivers/media/pci/mgb4/mgb4_core.c                 |    6 +-
+ drivers/media/pci/netup_unidvb/netup_unidvb_core.c |    2 +-
+ drivers/media/pci/ngene/ngene-cards.c              |    5 +-
+ drivers/media/pci/pluto2/pluto2.c                  |    5 +-
+ drivers/media/pci/saa7134/saa7134-cards.c          | 1949 +++++++-------------
+ drivers/media/pci/saa7146/hexium_gemini.c          |   22 +-
+ drivers/media/pci/saa7146/hexium_orion.c           |   33 +-
+ drivers/media/pci/saa7146/mxb.c                    |   10 +-
+ drivers/media/pci/saa7164/saa7164-core.c           |    5 +-
+ drivers/media/pci/smipcie/smipcie-main.c           |    5 +-
+ drivers/media/pci/solo6x10/solo6x10-core.c         |    2 +-
+ drivers/media/pci/ttpci/budget-ci.c                |    4 +-
+ drivers/media/pci/ttpci/budget.c                   |    4 +-
+ drivers/media/pci/tw5864/tw5864-core.c             |    4 +-
+ drivers/media/pci/tw68/tw68-core.c                 |   16 +-
+ drivers/media/pci/tw686x/tw686x-core.c             |   23 +-
+ drivers/media/pci/zoran/zoran_card.c               |    7 +-
+ drivers/media/platform/amd/isp4/Kconfig            |    1 +
+ drivers/media/platform/amd/isp4/isp4_interface.c   |    4 +-
+ drivers/media/platform/amd/isp4/isp4_interface.h   |    2 +-
+ drivers/media/platform/amd/isp4/isp4_subdev.c      |    2 +-
+ drivers/media/platform/amphion/venc.c              |    2 -
+ drivers/media/platform/broadcom/bcm2835-unicam.c   |    2 +
+ .../platform/chips-media/wave5/wave5-vpu-dec.c     |   64 +-
+ .../platform/chips-media/wave5/wave5-vpu-enc.c     |    6 +-
+ .../platform/chips-media/wave5/wave5-vpuconfig.h   |    2 +-
+ .../media/platform/mediatek/mdp3/mtk-mdp3-cmdq.c   |    6 +-
+ .../media/platform/mediatek/mdp3/mtk-mdp3-comp.h   |    6 +-
+ .../media/platform/mediatek/mdp3/mtk-mdp3-core.c   |    6 +-
+ .../vcodec/decoder/vdec/vdec_vp9_req_lat_if.c      |   29 +-
+ drivers/media/platform/qcom/iris/iris_vpu_buffer.c |    4 +-
+ .../media/platform/renesas/rzg2l-cru/rzg2l-cru.h   |    2 -
+ drivers/media/platform/rockchip/rga/rga3-hw.c      |   12 +-
+ .../platform/rockchip/rkvdec/rkvdec-hevc-common.c  |    3 +
+ .../media/platform/rockchip/rkvdec/rkvdec-hevc.c   |    4 +-
+ .../platform/rockchip/rkvdec/rkvdec-vdpu381-hevc.c |    8 +-
+ .../platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c |    8 +-
+ drivers/media/platform/rockchip/rkvdec/rkvdec.c    |    8 +-
+ .../media/platform/samsung/s3c-camif/camif-core.c  |   10 +-
+ drivers/media/platform/st/stm32/stm32-dcmi.c       |    4 +-
+ .../platform/st/stm32/stm32-dcmipp/dcmipp-core.c   |    4 +-
+ drivers/media/platform/ti/Kconfig                  |    1 +
+ drivers/media/platform/verisilicon/hantro_drv.c    |   12 +-
+ drivers/media/radio/radio-cadet.c                  |    4 +-
+ drivers/media/radio/radio-gemtek.c                 |    4 +-
+ drivers/media/rc/fintek-cir.c                      |    4 +-
+ drivers/media/rc/ite-cir.c                         |   29 +-
+ drivers/media/rc/keymaps/Makefile                  |    2 +-
+ .../{rc-videomate-m1f.c => rc-videomate-k100.c}    |    2 +-
+ drivers/media/rc/nuvoton-cir.c                     |    6 +-
+ drivers/media/rc/sunxi-cir.c                       |    9 +-
+ drivers/media/rc/winbond-cir.c                     |    4 +-
+ drivers/media/test-drivers/vicodec/vicodec-core.c  |    4 +-
+ drivers/media/test-drivers/vimc/vimc-sensor.c      |   12 +-
+ drivers/media/test-drivers/vivid/vivid-meta-out.c  |   23 +-
+ drivers/media/usb/airspy/airspy.c                  |   19 +-
+ drivers/media/usb/au0828/au0828-cards.c            |   14 +
+ drivers/media/usb/au0828/au0828-cards.h            |    1 +
+ drivers/media/usb/au0828/au0828-dvb.c              |    1 +
+ drivers/media/usb/cx231xx/cx231xx-audio.c          |    5 +
+ drivers/media/usb/cx231xx/cx231xx-video.c          |    4 +-
+ drivers/media/usb/em28xx/em28xx-cards.c            |   33 +-
+ drivers/media/usb/em28xx/em28xx-core.c             |    4 +-
+ drivers/media/usb/em28xx/em28xx-dvb.c              |    2 +-
+ drivers/media/usb/em28xx/em28xx-reg.h              |    1 +
+ drivers/media/usb/em28xx/em28xx-video.c            |  114 +-
+ drivers/media/usb/em28xx/em28xx.h                  |    3 +-
+ drivers/media/usb/go7007/go7007-driver.c           |   14 +-
+ drivers/media/usb/go7007/go7007-fw.c               |   28 +-
+ drivers/media/usb/go7007/snd-go7007.c              |   10 +-
+ drivers/media/usb/s2255/s2255drv.c                 |    5 +
+ drivers/media/usb/stk1160/stk1160-core.c           |   15 +-
+ drivers/media/usb/stk1160/stk1160-video.c          |    3 +
+ drivers/media/usb/usbtv/usbtv-audio.c              |   11 +
+ drivers/media/v4l2-core/v4l2-async.c               |   15 +-
+ drivers/media/v4l2-core/v4l2-ctrls-core.c          |   15 +
+ drivers/media/v4l2-core/v4l2-fwnode.c              |    2 +
+ drivers/platform/x86/intel/int3472/discrete.c      |   18 +
+ .../pci/base/circbuf/interface/ia_css_circbuf.h    |   40 +-
+ .../base/circbuf/interface/ia_css_circbuf_comm.h   |    4 +-
+ .../base/circbuf/interface/ia_css_circbuf_desc.h   |   14 +-
+ .../pci/camera/util/interface/ia_css_util.h        |    2 +-
+ .../pci/css_2401_system/host/csi_rx_private.h      |   18 +-
+ .../pci/css_2401_system/host/isys_irq_private.h    |    4 +-
+ .../host/isys_stream2mmio_private.h                |   12 +-
+ .../pci/css_2401_system/host/pixelgen_private.h    |    8 +-
+ .../atomisp/pci/css_2401_system/pixelgen_global.h  |    2 +-
+ .../pci/hive_isp_css_common/timed_ctrl_global.h    |    2 +-
+ .../device_access/device_access.h                  |    2 +-
+ .../pci/hive_isp_css_include/host/csi_rx_public.h  |   18 +-
+ .../host/isys_stream2mmio_public.h                 |   12 +-
+ .../hive_isp_css_include/host/pixelgen_public.h    |    8 +-
+ .../pci/hive_isp_css_include/host/tag_public.h     |    4 +-
+ .../pci/hive_isp_css_include/platform_support.h    |    2 +-
+ .../pci/hive_isp_css_include/type_support.h        |    2 +-
+ .../media/atomisp/pci/ia_css_frame_public.h        |    2 +-
+ .../staging/media/atomisp/pci/ia_css_host_data.h   |    4 +-
+ .../staging/media/atomisp/pci/ia_css_pipe_public.h |    8 +-
+ drivers/staging/media/atomisp/pci/ia_css_prbs.h    |    4 +-
+ drivers/staging/media/atomisp/pci/ia_css_stream.h  |    2 +-
+ drivers/staging/media/atomisp/pci/ia_css_timer.h   |    7 +-
+ drivers/staging/media/atomisp/pci/ia_css_types.h   |    6 +-
+ .../conversion_1.0/ia_css_conversion_types.h       |    2 +-
+ .../pci/isp/kernels/eed1_8/ia_css_eed1_8_types.h   |    2 +-
+ .../atomisp/pci/isp/kernels/hdr/ia_css_hdr_types.h |   42 +-
+ .../ipu2_io_ls/common/ia_css_common_io_param.h     |    2 +-
+ .../ipu2_io_ls/common/ia_css_common_io_types.h     |    2 +-
+ .../yuv444_io_ls/ia_css_yuv444_io.host.h           |    2 +-
+ .../yuv444_io_ls/ia_css_yuv444_io_param.h          |    2 +-
+ .../yuv444_io_ls/ia_css_yuv444_io_types.h          |    2 +-
+ .../pci/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h   |    4 +-
+ .../isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h    |   14 +-
+ .../pci/isp/modes/interface/input_buf.isp.h        |    2 +-
+ .../pci/runtime/binary/interface/ia_css_binary.h   |    2 +-
+ .../pci/runtime/bufq/interface/ia_css_bufq.h       |   26 +-
+ .../pci/runtime/debug/interface/ia_css_debug.h     |   10 +-
+ .../runtime/debug/interface/ia_css_debug_pipe.h    |   10 +-
+ .../pci/runtime/eventq/interface/ia_css_eventq.h   |    4 +-
+ .../isp_param/interface/ia_css_isp_param_types.h   |    2 +-
+ .../pci/runtime/isys/interface/ia_css_isys.h       |    8 +-
+ .../runtime/pipeline/interface/ia_css_pipeline.h   |   10 +-
+ .../pci/runtime/rmgr/interface/ia_css_rmgr.h       |    4 +-
+ .../pci/runtime/rmgr/interface/ia_css_rmgr_vbuf.h  |   18 +-
+ .../tagger/interface/ia_css_tagger_common.h        |    4 +-
+ .../staging/media/atomisp/pci/sh_css_internal.h    |    4 +-
+ drivers/staging/media/atomisp/pci/sh_css_sp.h      |   14 +-
+ drivers/staging/media/atomisp/pci/system_global.h  |    4 +-
+ drivers/staging/media/av7110/av7110.c              |   20 +-
+ drivers/staging/media/ipu7/ipu7.c                  |    4 +-
+ drivers/staging/media/meson/vdec/vdec.c            |    4 +-
+ drivers/staging/media/sunxi/cedrus/cedrus.c        |    4 +-
+ drivers/staging/media/tegra-video/vi.c             |   87 +-
+ drivers/staging/media/tegra-video/vip.c            |   10 +-
+ include/media/cec.h                                |    6 +-
+ include/media/drv-intf/s3c_camif.h                 |    2 -
+ include/media/drv-intf/saa7146.h                   |   11 +-
+ include/media/media-entity.h                       |    1 +
+ include/media/rc-map.h                             |    2 -
+ include/media/v4l2-ioctl.h                         |    4 +-
+ include/media/videobuf2-v4l2.h                     |   13 +
+ include/uapi/linux/cec.h                           |    2 +
+ 204 files changed, 5526 insertions(+), 3720 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/media/i2c/sony,imx678.yaml
+ create mode 100644 drivers/media/i2c/imx471.c
+ create mode 100644 drivers/media/i2c/imx678.c
+ rename drivers/media/rc/keymaps/{rc-videomate-m1f.c => rc-videomate-k100.c} (97%)
+Merging v4l-dvb-next/master (adc218676eef2 Linux 6.12)
+$ git merge -m Merge branch 'master' of git://linuxtv.org/mchehab/media-next.git v4l-dvb-next/master
+Already up to date.
+Merging pm/linux-next (ca2b8f92568b7 Merge branch 'intel-idle-lpi' into linux-next)
+$ git merge -m Merge branch 'linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git pm/linux-next
+Auto-merging arch/riscv/include/asm/acpi.h
+Auto-merging drivers/acpi/scan.c
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/pm/cpufreq.rst       |   2 +-
+ Documentation/driver-api/acpi/acpi-drivers.rst |  80 ------
+ Documentation/driver-api/acpi/index.rst        |   1 -
+ arch/riscv/include/asm/acpi.h                  |   1 +
+ arch/x86/include/asm/cpufeatures.h             |   2 +
+ arch/x86/include/asm/msr-index.h               |   2 +
+ drivers/acpi/acpi_processor.c                  | 354 +++++++++++++++++++++++++
+ drivers/acpi/apei/ghes.c                       |   4 +-
+ drivers/acpi/arm64/iort.c                      |  22 +-
+ drivers/acpi/bus.c                             | 138 +---------
+ drivers/acpi/fan_core.c                        |  25 +-
+ drivers/acpi/irq.c                             | 172 +++++++++++-
+ drivers/acpi/numa/srat.c                       |   2 -
+ drivers/acpi/power.c                           |   1 -
+ drivers/acpi/processor_idle.c                  | 269 +------------------
+ drivers/acpi/riscv/irq.c                       | 141 +---------
+ drivers/acpi/scan.c                            |  18 +-
+ drivers/base/power/runtime.c                   |   6 +-
+ drivers/cpufreq/amd-pstate.c                   |   1 +
+ drivers/cpufreq/cpufreq.c                      |  14 +-
+ drivers/cpufreq/intel_pstate.c                 | 131 ++++-----
+ drivers/idle/intel_idle.c                      | 199 +++++++++++---
+ drivers/irqchip/irq-gic-v3.c                   |   2 +-
+ drivers/irqchip/irq-gic-v5-iwb.c               |   2 +
+ drivers/irqchip/irq-gic-v5.c                   |  13 +-
+ drivers/irqchip/irq-gic.c                      |   2 +-
+ drivers/irqchip/irq-loongarch-cpu.c            |   2 +-
+ drivers/irqchip/irq-riscv-intc.c               |   3 +-
+ drivers/pnp/card.c                             |  11 +-
+ drivers/pnp/pnpacpi/core.c                     |   5 +-
+ drivers/pnp/system.c                           |   6 +-
+ drivers/thermal/intel/therm_throt.c            | 277 ++++++++++++++++++-
+ drivers/xen/xen-acpi-pad.c                     |   5 -
+ include/acpi/acpi_bus.h                        |  50 +---
+ include/linux/acpi.h                           |  26 +-
+ include/linux/acpi_iort.h                      |   3 +-
+ include/linux/cpufreq.h                        |   2 +
+ kernel/power/wakelock.c                        |   2 +-
+ kernel/sched/cpufreq_schedutil.c               |   6 +-
+ rust/kernel/cpufreq.rs                         |  11 +-
+ 40 files changed, 1153 insertions(+), 860 deletions(-)
+ delete mode 100644 Documentation/driver-api/acpi/acpi-drivers.rst
+Merging cpufreq-arm/cpufreq/arm/linux-next (6a9e0e0f75923 cpufreq: spear: Fix an IS_ERR() vs NULL bug in spear1340_set_cpu_rate())
+$ git merge -m Merge branch 'cpufreq/arm/linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git cpufreq-arm/cpufreq/arm/linux-next
+Auto-merging drivers/cpufreq/cpufreq-dt-platdev.c
+Merge made by the 'ort' strategy.
+ drivers/cpufreq/apple-soc-cpufreq.c   | 36 ++++++++++++++---------------------
+ drivers/cpufreq/brcmstb-avs-cpufreq.c |  5 +----
+ drivers/cpufreq/cpufreq-dt-platdev.c  |  1 +
+ drivers/cpufreq/qcom-cpufreq-nvmem.c  |  8 ++++++++
+ drivers/cpufreq/rcpufreq_dt.rs        | 13 ++++++++++---
+ drivers/cpufreq/spear-cpufreq.c       |  4 ++--
+ 6 files changed, 36 insertions(+), 31 deletions(-)
+Merging cpupower/cpupower (5100bd356cd31 cpupower: Print kernel and hardware frequency information)
+$ git merge -m Merge branch 'cpupower' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux.git cpupower/cpupower
+Merge made by the 'ort' strategy.
+ tools/power/cpupower/Makefile                |  2 +-
+ tools/power/cpupower/utils/cpufreq-info.c    | 11 +++---
+ tools/power/cpupower/utils/helpers/cppc.c    | 56 ++++++++++++++++++++++++++++
+ tools/power/cpupower/utils/helpers/helpers.h |  2 +
+ 4 files changed, 65 insertions(+), 6 deletions(-)
+ create mode 100644 tools/power/cpupower/utils/helpers/cppc.c
+Merging devfreq/devfreq-next (c096be11c2a9d PM / devfreq: Fix governor_store() failing when device has no current governor)
+$ git merge -m Merge branch 'devfreq-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/linux.git devfreq/devfreq-next
+Merge made by the 'ort' strategy.
+ drivers/devfreq/devfreq.c            | 50 +++++++-----------------------------
+ drivers/devfreq/event/rockchip-dfi.c |  4 ++-
+ 2 files changed, 12 insertions(+), 42 deletions(-)
+Merging pmdomain/next (8247e08d2196c pmdomain: Merge branch dt into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git pmdomain/next
+Merge made by the 'ort' strategy.
+ .../bindings/power/mediatek,power-controller.yaml  |   1 +
+ .../devicetree/bindings/power/qcom,rpmpd.yaml      |   4 +
+ drivers/pmdomain/arm/Kconfig                       |   2 +-
+ drivers/pmdomain/bcm/bcm2835-power.c               |   9 +-
+ drivers/pmdomain/imx/Kconfig                       |  11 +-
+ drivers/pmdomain/mediatek/mt8196-pm-domains.h      |  27 ++++
+ drivers/pmdomain/mediatek/mtk-pm-domains.c         | 146 +++++++++++++++++----
+ drivers/pmdomain/mediatek/mtk-pm-domains.h         |   1 +
+ drivers/pmdomain/qcom/rpmhpd.c                     |  38 +++---
+ drivers/pmdomain/tegra/powergate-bpmp.c            |   9 +-
+ include/dt-bindings/power/mediatek,mt8196-power.h  |   4 +
+ 11 files changed, 200 insertions(+), 52 deletions(-)
+Merging opp/opp/linux-next (ce46fede7792c OPP: Fix cleanup ordering)
+$ git merge -m Merge branch 'opp/linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git opp/opp/linux-next
+Merge made by the 'ort' strategy.
+ drivers/opp/core.c | 13 +++++--------
+ 1 file changed, 5 insertions(+), 8 deletions(-)
+Merging thermal/thermal/linux-next (968098b4ca521 thermal/drivers/qcom/tsens: Disable wakeup interrupt setup on automotive targets)
+$ git merge -m Merge branch 'thermal/linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/thermal/linux.git thermal/thermal/linux-next
+Already up to date.
+Merging rdma/for-next (e952e079d2262 RDMA/umem: Constify struct dma_buf_attach_ops)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git rdma/for-next
+Auto-merging drivers/infiniband/core/cma.c
+Auto-merging drivers/infiniband/hw/irdma/verbs.c
+Auto-merging drivers/infiniband/sw/siw/siw_verbs.c
+Auto-merging net/sunrpc/xprtrdma/svc_rdma_transport.c
+Merge made by the 'ort' strategy.
+ Documentation/infiniband/user_mad.rst           |   2 +-
+ drivers/infiniband/core/addr.c                  |   2 +-
+ drivers/infiniband/core/cma.c                   |   2 +-
+ drivers/infiniband/core/cq.c                    |   3 +-
+ drivers/infiniband/core/iwpm_msg.c              |   2 +-
+ drivers/infiniband/core/nldev.c                 |  30 +++-
+ drivers/infiniband/core/restrack.c              |   2 +-
+ drivers/infiniband/core/umem.c                  |   5 +-
+ drivers/infiniband/core/umem_dmabuf.c           |   4 +-
+ drivers/infiniband/core/uverbs_std_types_mr.c   |   6 +-
+ drivers/infiniband/hw/bng_re/bng_fw.c           |  10 +-
+ drivers/infiniband/hw/bnxt_re/ib_verbs.c        |   1 -
+ drivers/infiniband/hw/efa/Makefile              |   4 +-
+ drivers/infiniband/hw/efa/efa_admin_cmds_defs.h |   6 +-
+ drivers/infiniband/hw/efa/efa_ah_cache.c        | 135 ++++++++++++++++
+ drivers/infiniband/hw/efa/efa_ah_cache.h        |  39 +++++
+ drivers/infiniband/hw/efa/efa_com.c             |  12 +-
+ drivers/infiniband/hw/efa/efa_com.h             |   3 +
+ drivers/infiniband/hw/efa/efa_com_cmd.c         |  41 ++++-
+ drivers/infiniband/hw/efa/efa_com_cmd.h         |   1 +
+ drivers/infiniband/hw/efa/efa_main.c            |   2 +
+ drivers/infiniband/hw/efa/efa_verbs.c           |  10 +-
+ drivers/infiniband/hw/erdma/erdma_verbs.c       |   2 -
+ drivers/infiniband/hw/hfi1/chip.c               |  21 ++-
+ drivers/infiniband/hw/hfi1/chip.h               |   2 +-
+ drivers/infiniband/hw/hfi1/device.c             |  32 +---
+ drivers/infiniband/hw/hfi1/device.h             |   1 -
+ drivers/infiniband/hw/hfi1/file_ops.c           |   2 +-
+ drivers/infiniband/hw/hfi1/hfi.h                |   4 +-
+ drivers/infiniband/hw/hfi1/init.c               | 196 ++++++++++--------------
+ drivers/infiniband/hw/hfi1/pcie.c               |  32 ++--
+ drivers/infiniband/hw/hns/hns_roce_hw_v2.c      |   2 +-
+ drivers/infiniband/hw/hns/hns_roce_main.c       |   2 -
+ drivers/infiniband/hw/ionic/ionic_ibdev.h       |   1 -
+ drivers/infiniband/hw/irdma/i40iw_hw.c          |   4 +-
+ drivers/infiniband/hw/irdma/icrdma_hw.c         |   2 +
+ drivers/infiniband/hw/irdma/icrdma_hw.h         |   2 +
+ drivers/infiniband/hw/irdma/icrdma_if.c         |   8 +-
+ drivers/infiniband/hw/irdma/irdma.h             |   2 +
+ drivers/infiniband/hw/irdma/utils.c             |   6 +
+ drivers/infiniband/hw/irdma/verbs.c             | 117 ++++++++++----
+ drivers/infiniband/hw/irdma/verbs.h             |   3 +
+ drivers/infiniband/hw/mana/main.c               |   1 -
+ drivers/infiniband/hw/mlx4/cm.c                 |  13 +-
+ drivers/infiniband/hw/mlx4/main.c               |   2 -
+ drivers/infiniband/hw/mlx5/main.c               |   1 -
+ drivers/infiniband/hw/mlx5/mr.c                 |   6 +-
+ drivers/infiniband/hw/mlx5/odp.c                |   6 +-
+ drivers/infiniband/hw/mlx5/umr.c                |   2 +-
+ drivers/infiniband/hw/mthca/mthca_allocator.c   |   6 +-
+ drivers/infiniband/hw/mthca/mthca_provider.c    |   7 +-
+ drivers/infiniband/hw/ocrdma/ocrdma_verbs.c     |   1 -
+ drivers/infiniband/hw/qedr/verbs.c              |   4 +-
+ drivers/infiniband/hw/usnic/usnic_ib_verbs.c    |   1 -
+ drivers/infiniband/sw/rdmavt/qp.c               |   8 +-
+ drivers/infiniband/sw/rdmavt/vt.c               |   4 +-
+ drivers/infiniband/sw/rxe/rxe_mw.c              |  20 ++-
+ drivers/infiniband/sw/rxe/rxe_odp.c             |  11 +-
+ drivers/infiniband/sw/rxe/rxe_qp.c              |  22 +--
+ drivers/infiniband/sw/rxe/rxe_resp.c            |   9 +-
+ drivers/infiniband/sw/rxe/rxe_srq.c             |  16 +-
+ drivers/infiniband/sw/siw/siw_verbs.c           |   2 -
+ drivers/infiniband/ulp/ipoib/ipoib_cm.c         |  10 +-
+ drivers/infiniband/ulp/ipoib/ipoib_verbs.c      |   3 +-
+ drivers/infiniband/ulp/iser/iser_verbs.c        |   5 +-
+ drivers/infiniband/ulp/isert/ib_isert.c         |   7 +-
+ drivers/infiniband/ulp/rtrs/rtrs-clt.c          |  11 +-
+ drivers/infiniband/ulp/rtrs/rtrs-srv.c          |  11 +-
+ drivers/infiniband/ulp/srp/ib_srp.c             |   2 +-
+ drivers/infiniband/ulp/srpt/ib_srpt.c           |  21 ++-
+ drivers/nvme/host/rdma.c                        |   8 +-
+ drivers/nvme/target/rdma.c                      |  26 ++--
+ fs/smb/smbdirect/accept.c                       |   5 +-
+ fs/smb/smbdirect/connect.c                      |   5 +-
+ fs/smb/smbdirect/connection.c                   |   8 +-
+ include/linux/sunrpc/svc_rdma.h                 |   4 +-
+ include/rdma/ib_verbs.h                         |  52 +++----
+ include/rdma/restrack.h                         |   2 +-
+ include/rdma/uverbs_ioctl.h                     |  20 +++
+ include/uapi/rdma/rdma_netlink.h                |   5 +
+ net/rds/ib.c                                    |  10 +-
+ net/rds/ib_cm.c                                 |  10 +-
+ net/sunrpc/xprtrdma/frwr_ops.c                  |   7 +-
+ net/sunrpc/xprtrdma/svc_rdma_transport.c        |   5 +-
+ net/sunrpc/xprtrdma/verbs.c                     |   2 +-
+ 85 files changed, 709 insertions(+), 435 deletions(-)
+ create mode 100644 drivers/infiniband/hw/efa/efa_ah_cache.c
+ create mode 100644 drivers/infiniband/hw/efa/efa_ah_cache.h
+Merging net-next/main (ce6b4d3216b63 net: mana: Add handler for sriov configure)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git net-next/main
+Auto-merging MAINTAINERS
+CONFLICT (content): Merge conflict in MAINTAINERS
+Auto-merging drivers/net/macsec.c
+Auto-merging include/linux/net.h
+Auto-merging net/core/dev.c
+Auto-merging net/ipv4/fib_trie.c
+Resolved 'MAINTAINERS' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 0e044f7b7ddca] Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/driver-api/dpll.rst                  |    7 +
+ Documentation/netlink/specs/devlink.yaml           |   30 +-
+ Documentation/netlink/specs/dpll.yaml              |   19 +
+ Documentation/netlink/specs/rt-addr.yaml           |    4 +
+ Documentation/networking/devlink/devlink-port.rst  |   11 +
+ Documentation/networking/devlink/index.rst         |    8 +-
+ Documentation/networking/devlink/mlx5.rst          |   33 +
+ .../net_cachelines/netns_ipv4_sysctl.rst           |    1 +
+ Documentation/networking/netdevices.rst            |    7 +
+ MAINTAINERS                                        |    5 -
+ arch/powerpc/configs/ppc64_defconfig               |    1 -
+ arch/powerpc/mm/mem.c                              |    6 -
+ drivers/cxl/core/core.h                            |    2 +
+ drivers/cxl/core/mbox.c                            |   51 +-
+ drivers/cxl/core/memdev.c                          |   67 +
+ drivers/cxl/core/pci.c                             |    1 +
+ drivers/cxl/core/port.c                            |    1 +
+ drivers/cxl/core/regs.c                            |    1 +
+ drivers/cxl/cxlpci.h                               |   12 -
+ drivers/cxl/pci.c                                  |    1 +
+ drivers/dpll/dpll_core.c                           |    6 +-
+ drivers/dpll/dpll_nl.c                             |    2 +-
+ drivers/dpll/zl3073x/chan.c                        |  128 +-
+ drivers/dpll/zl3073x/chan.h                        |   48 +
+ drivers/dpll/zl3073x/core.c                        |   48 +-
+ drivers/dpll/zl3073x/core.h                        |   12 +-
+ drivers/dpll/zl3073x/dpll.c                        |  491 ++-
+ drivers/dpll/zl3073x/dpll.h                        |    4 +
+ drivers/dpll/zl3073x/regs.h                        |   11 +
+ drivers/net/amt.c                                  |   43 +-
+ drivers/net/bareudp.c                              |   45 +-
+ drivers/net/bonding/bond_3ad.c                     |   24 +-
+ drivers/net/bonding/bond_main.c                    |    6 -
+ drivers/net/bonding/bond_netlink.c                 |  109 +-
+ drivers/net/bonding/bond_options.c                 |    8 +-
+ drivers/net/dsa/b53/b53_priv.h                     |    3 +-
+ drivers/net/dsa/microchip/ksz8.c                   |  431 ++-
+ drivers/net/dsa/microchip/ksz9477.c                |  164 +-
+ drivers/net/dsa/microchip/ksz9477.h                |    4 +-
+ drivers/net/dsa/microchip/ksz_common.c             |  594 +---
+ drivers/net/dsa/microchip/ksz_common.h             |   85 +-
+ drivers/net/dsa/microchip/lan937x_main.c           |    6 +-
+ drivers/net/dsa/qca/qca8k-leds.c                   |    3 +
+ drivers/net/dsa/realtek/rtl8366rb.c                |  268 +-
+ drivers/net/dsa/realtek/rtl83xx.c                  |   26 +-
+ drivers/net/ethernet/airoha/airoha_ppe.c           |   10 +-
+ drivers/net/ethernet/allwinner/sun4i-emac.c        |    2 +-
+ drivers/net/ethernet/apm/xgene/xgene_enet_main.c   |    2 +-
+ drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.c     |    6 +-
+ drivers/net/ethernet/chelsio/cxgb4/t4_hw.c         |    3 +-
+ .../net/ethernet/freescale/dpaa2/dpaa2-switch.c    |  931 ++++-
+ .../net/ethernet/freescale/dpaa2/dpaa2-switch.h    |   42 +-
+ drivers/net/ethernet/freescale/dpaa2/dpsw-cmd.h    |   18 +-
+ drivers/net/ethernet/freescale/dpaa2/dpsw.c        |   60 +
+ drivers/net/ethernet/freescale/dpaa2/dpsw.h        |   30 +
+ drivers/net/ethernet/hisilicon/hns3/hnae3.h        |   14 +
+ .../hisilicon/hns3/hns3_common/hclge_comm_cmd.h    |    3 +
+ drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c |   12 +
+ .../net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h |    9 +
+ .../ethernet/hisilicon/hns3/hns3pf/hclge_main.c    |  172 +
+ .../ethernet/hisilicon/hns3/hns3pf/hclge_main.h    |    7 +
+ drivers/net/ethernet/ibm/Kconfig                   |    9 -
+ drivers/net/ethernet/ibm/Makefile                  |    1 -
+ drivers/net/ethernet/ibm/ehea/Makefile             |    7 -
+ drivers/net/ethernet/ibm/ehea/ehea.h               |  477 ---
+ drivers/net/ethernet/ibm/ehea/ehea_ethtool.c       |  277 --
+ drivers/net/ethernet/ibm/ehea/ehea_hw.h            |  253 --
+ drivers/net/ethernet/ibm/ehea/ehea_main.c          | 3581 --------------------
+ drivers/net/ethernet/ibm/ehea/ehea_phyp.c          |  612 ----
+ drivers/net/ethernet/ibm/ehea/ehea_phyp.h          |  433 ---
+ drivers/net/ethernet/ibm/ehea/ehea_qmr.c           |  999 ------
+ drivers/net/ethernet/ibm/ehea/ehea_qmr.h           |  390 ---
+ drivers/net/ethernet/intel/ice/ice_gnss.c          |    5 +-
+ drivers/net/ethernet/intel/igb/e1000_82575.h       |   21 +
+ drivers/net/ethernet/intel/igb/igb.h               |    3 +
+ drivers/net/ethernet/intel/igb/igb_ethtool.c       |   89 +-
+ drivers/net/ethernet/intel/igb/igb_main.c          |   25 +-
+ drivers/net/ethernet/intel/igc/igc.h               |    3 +
+ drivers/net/ethernet/intel/igc/igc_base.c          |   33 +-
+ drivers/net/ethernet/intel/igc/igc_defines.h       |    9 +-
+ drivers/net/ethernet/intel/igc/igc_ethtool.c       |  279 +-
+ drivers/net/ethernet/intel/igc/igc_hw.h            |   10 +-
+ drivers/net/ethernet/intel/igc/igc_mac.c           |   35 +-
+ drivers/net/ethernet/intel/igc/igc_main.c          |   10 +-
+ drivers/net/ethernet/intel/igc/igc_phy.c           |   65 +-
+ drivers/net/ethernet/intel/igc/igc_phy.h           |    1 +
+ drivers/net/ethernet/marvell/mvneta.c              |   18 +
+ drivers/net/ethernet/marvell/mvneta_bm.c           |   72 +
+ drivers/net/ethernet/marvell/octeontx2/af/mbox.h   |    2 +
+ drivers/net/ethernet/marvell/octeontx2/af/npc.h    |    4 +-
+ drivers/net/ethernet/marvell/octeontx2/af/rvu.h    |    2 +-
+ .../net/ethernet/marvell/octeontx2/af/rvu_nix.c    |    9 +-
+ .../net/ethernet/marvell/octeontx2/af/rvu_npc.c    |   43 +-
+ drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c |    1 +
+ .../ethernet/marvell/octeontx2/nic/otx2_common.c   |    2 +-
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c        |    5 +
+ drivers/net/ethernet/mellanox/mlx5/core/devlink.c  |    1 +
+ .../ethernet/mellanox/mlx5/core/esw/devlink_port.c |    8 +-
+ drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c  |  605 ++--
+ drivers/net/ethernet/mellanox/mlx5/core/esw/qos.h  |    3 -
+ drivers/net/ethernet/mellanox/mlx5/core/eswitch.c  |   27 +-
+ drivers/net/ethernet/mellanox/mlx5/core/eswitch.h  |   18 +-
+ .../net/ethernet/mellanox/mlxsw/spectrum_acl_erp.c |    2 +-
+ .../ethernet/mellanox/mlxsw/spectrum_acl_tcam.h    |    2 +-
+ drivers/net/ethernet/microsoft/Kconfig             |    1 +
+ drivers/net/ethernet/microsoft/mana/gdma_main.c    |   53 +
+ drivers/net/ethernet/microsoft/mana/mana_en.c      |  189 ++
+ drivers/net/ethernet/microsoft/mana/mana_ethtool.c |  167 +-
+ .../net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c   |    2 +-
+ .../net/ethernet/pensando/ionic/ionic_rx_filter.c  |    7 +-
+ drivers/net/ethernet/qlogic/qed/qed_dcbx.c         |    2 +-
+ drivers/net/ethernet/qualcomm/ppe/ppe.c            |    1 -
+ drivers/net/ethernet/qualcomm/ppe/ppe_config.c     |    7 +-
+ drivers/net/ethernet/qualcomm/ppe/ppe_config.h     |    2 +-
+ drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c |    3 +-
+ drivers/net/ethernet/sfc/Kconfig                   |    9 +
+ drivers/net/ethernet/sfc/Makefile                  |    1 +
+ drivers/net/ethernet/sfc/ef10.c                    |   41 +-
+ drivers/net/ethernet/sfc/efx.c                     |   18 +-
+ drivers/net/ethernet/sfc/efx_cxl.c                 |  105 +
+ drivers/net/ethernet/sfc/efx_cxl.h                 |   32 +
+ drivers/net/ethernet/sfc/mcdi.c                    |    7 +-
+ drivers/net/ethernet/sfc/net_driver.h              |   10 +
+ drivers/net/ethernet/sfc/nic.h                     |    3 +
+ drivers/net/ethernet/sfc/siena/mcdi.c              |    7 +-
+ drivers/net/ethernet/sgi/ioc3-eth.c                |    9 +-
+ drivers/net/gtp.c                                  |   12 +-
+ drivers/net/ipvlan/ipvlan.h                        |   12 +-
+ drivers/net/ipvlan/ipvlan_main.c                   |  147 +-
+ drivers/net/ipvlan/ipvtap.c                        |   25 +-
+ drivers/net/macsec.c                               |   78 +-
+ drivers/net/macvlan.c                              |  109 +-
+ drivers/net/netconsole.c                           |  201 +-
+ drivers/net/phy/mdio_device.c                      |    2 +-
+ drivers/net/phy/phylink.c                          |   15 +-
+ drivers/net/ppp/ppp_async.c                        |    2 -
+ drivers/net/ppp/ppp_synctty.c                      |    2 -
+ drivers/net/tun.c                                  |   60 +-
+ drivers/net/tun_vnet.h                             |    8 +-
+ drivers/net/usb/rtl8150.c                          |    4 +-
+ drivers/net/veth.c                                 |   34 +-
+ include/cxl/cxl.h                                  |    2 +
+ include/cxl/pci.h                                  |   22 +
+ include/linux/mdio.h                               |    1 -
+ include/linux/net.h                                |   23 +
+ include/linux/netdevice.h                          |   24 +
+ include/linux/netfilter.h                          |    7 +
+ include/linux/netfilter/nf_conntrack_h323.h        |    2 -
+ include/linux/netfilter/nf_conntrack_pptp.h        |    2 -
+ include/linux/netfilter/nf_conntrack_sane.h        |    2 -
+ include/linux/netfilter/nf_conntrack_tftp.h        |    2 -
+ include/linux/netpoll.h                            |   18 -
+ include/linux/rtnetlink.h                          |    8 +
+ include/net/bond_3ad.h                             |    4 +-
+ include/net/bonding.h                              |    8 +-
+ include/net/devlink.h                              |   15 +-
+ include/net/fib_rules.h                            |    4 +-
+ include/net/ip_fib.h                               |    3 +-
+ include/net/ip_vs.h                                |    2 +-
+ include/net/mana/gdma.h                            |   24 +-
+ include/net/mana/mana.h                            |   57 +
+ include/net/net_namespace.h                        |    3 +
+ include/net/netfilter/nf_conntrack_helper.h        |   10 +-
+ include/net/netns/ipv4.h                           |    1 +
+ include/net/udp.h                                  |    2 +-
+ include/uapi/linux/devlink.h                       |    2 +
+ include/uapi/linux/dpll.h                          |    8 +
+ include/uapi/linux/ethtool.h                       |    6 +
+ include/uapi/linux/if_addr.h                       |    1 +
+ net/batman-adv/bat_iv_ogm.c                        |   12 +-
+ net/batman-adv/bat_v_elp.c                         |    9 +-
+ net/batman-adv/bat_v_ogm.c                         |   33 +-
+ net/batman-adv/bridge_loop_avoidance.c             |    9 +-
+ net/batman-adv/hard-interface.c                    |  165 +-
+ net/batman-adv/hard-interface.h                    |   10 +-
+ net/batman-adv/main.c                              |    9 -
+ net/batman-adv/main.h                              |    3 -
+ net/batman-adv/mesh-interface.c                    |   13 +-
+ net/batman-adv/netlink.c                           |    4 +-
+ net/batman-adv/originator.c                        |    4 -
+ net/batman-adv/tp_meter.c                          |  215 +-
+ net/batman-adv/tvlv.c                              |   86 +-
+ net/batman-adv/types.h                             |   28 +-
+ net/bridge/netfilter/ebtables.c                    |   12 +-
+ net/core/dev.c                                     |  140 +-
+ net/core/fib_rules.c                               |   83 +-
+ net/core/lock_debug.c                              |    4 +-
+ net/core/net_namespace.c                           |    4 +
+ net/core/netpoll.c                                 |   51 +-
+ net/core/rtnetlink.c                               |   74 +-
+ net/devlink/core.c                                 |   20 +-
+ net/devlink/dev.c                                  |   16 +-
+ net/devlink/devl_internal.h                        |   20 +
+ net/devlink/dpipe.c                                |   14 +-
+ net/devlink/health.c                               |   12 +-
+ net/devlink/linecard.c                             |    4 +-
+ net/devlink/netlink.c                              |   82 +-
+ net/devlink/netlink_gen.c                          |   24 +-
+ net/devlink/netlink_gen.h                          |    8 +
+ net/devlink/param.c                                |    4 +-
+ net/devlink/port.c                                 |   24 +-
+ net/devlink/rate.c                                 |  329 +-
+ net/devlink/region.c                               |    6 +-
+ net/devlink/resource.c                             |   14 +-
+ net/devlink/sb.c                                   |   22 +-
+ net/devlink/trap.c                                 |   12 +-
+ net/dsa/dsa.c                                      |   20 +-
+ net/dsa/user.c                                     |   19 +-
+ net/ipv4/fib_frontend.c                            |   69 +-
+ net/ipv4/fib_rules.c                               |   20 +-
+ net/ipv4/fib_trie.c                                |    3 +-
+ net/ipv4/igmp.c                                    |    2 +
+ net/ipv4/netfilter/nf_nat_snmp_basic_main.c        |    2 +-
+ net/ipv4/raw.c                                     |   41 +-
+ net/ipv4/udp.c                                     |   39 +-
+ net/ipv6/addrconf.c                                |    1 +
+ net/ipv6/fib6_rules.c                              |   17 +-
+ net/ipv6/mcast.c                                   |    1 +
+ net/ipv6/udp.c                                     |   19 +-
+ net/netfilter/ipvs/ip_vs_nfct.c                    |    2 +-
+ net/netfilter/nf_conntrack_amanda.c                |    6 +-
+ net/netfilter/nf_conntrack_broadcast.c             |    2 -
+ net/netfilter/nf_conntrack_ftp.c                   |   32 +-
+ net/netfilter/nf_conntrack_h323_main.c             |   12 +-
+ net/netfilter/nf_conntrack_helper.c                |   77 +-
+ net/netfilter/nf_conntrack_irc.c                   |   27 +-
+ net/netfilter/nf_conntrack_netbios_ns.c            |    2 -
+ net/netfilter/nf_conntrack_ovs.c                   |    6 +-
+ net/netfilter/nf_conntrack_pptp.c                  |    2 +-
+ net/netfilter/nf_conntrack_sane.c                  |   34 +-
+ net/netfilter/nf_conntrack_sip.c                   |   45 +-
+ net/netfilter/nf_conntrack_snmp.c                  |    4 +-
+ net/netfilter/nf_conntrack_tftp.c                  |   33 +-
+ net/netfilter/nf_nat_core.c                        |    6 -
+ net/netfilter/nf_nat_proto.c                       |    8 +
+ net/netfilter/nfnetlink_cthelper.c                 |   21 +-
+ net/netfilter/nfnetlink_cttimeout.c                |    2 +-
+ net/netfilter/nfnetlink_hook.c                     |   37 +-
+ net/netfilter/nft_ct.c                             |   35 +
+ net/netfilter/nft_set_rbtree.c                     |    3 +-
+ net/netfilter/x_tables.c                           |   30 +-
+ net/netfilter/xt_TCPOPTSTRIP.c                     |    8 +-
+ net/netfilter/xt_dscp.c                            |   12 +
+ net/netfilter/xt_recent.c                          |    2 +-
+ net/netfilter/xt_tcpmss.c                          |   13 +
+ net/sched/act_ct.c                                 |    4 +-
+ net/vmw_vsock/virtio_transport_common.c            |   49 +-
+ tools/net/ynl/pyynl/__init__.py                    |    9 +
+ tools/net/ynl/pyynl/cli.py                         |   56 +-
+ tools/net/ynl/pyynl/lib/__init__.py                |    3 +-
+ tools/net/ynl/pyynl/lib/nlspec.py                  |   22 +-
+ tools/net/ynl/pyynl/lib/specdir.py                 |   51 +
+ tools/net/ynl/pyynl/lib/ynl.py                     |   19 +-
+ tools/net/ynl/tests/ethtool.py                     |    7 +-
+ tools/testing/selftests/bpf/prog_tests/test_xsk.c  |    4 +-
+ tools/testing/selftests/drivers/net/hw/Makefile    |    1 +
+ .../drivers/net/hw/devlink_rate_cross_esw.py       |  296 ++
+ tools/testing/selftests/drivers/net/hw/rss_ctx.py  |   11 +-
+ tools/testing/selftests/drivers/net/hw/toeplitz.py |   22 +
+ tools/testing/selftests/drivers/net/lib/py/env.py  |   27 +-
+ tools/testing/selftests/drivers/net/xdp.py         |   94 +-
+ tools/testing/selftests/net/getsockopt_iter.c      |   97 +
+ .../selftests/net/openvswitch/openvswitch.sh       |   87 +
+ tools/testing/selftests/net/rtnetlink.py           |  101 +-
+ .../tc-testing/tc-tests/infra/qdiscs.json          |  164 +
+ 265 files changed, 7593 insertions(+), 10083 deletions(-)
+ delete mode 100644 drivers/net/ethernet/ibm/ehea/Makefile
+ delete mode 100644 drivers/net/ethernet/ibm/ehea/ehea.h
+ delete mode 100644 drivers/net/ethernet/ibm/ehea/ehea_ethtool.c
+ delete mode 100644 drivers/net/ethernet/ibm/ehea/ehea_hw.h
+ delete mode 100644 drivers/net/ethernet/ibm/ehea/ehea_main.c
+ delete mode 100644 drivers/net/ethernet/ibm/ehea/ehea_phyp.c
+ delete mode 100644 drivers/net/ethernet/ibm/ehea/ehea_phyp.h
+ delete mode 100644 drivers/net/ethernet/ibm/ehea/ehea_qmr.c
+ delete mode 100644 drivers/net/ethernet/ibm/ehea/ehea_qmr.h
+ create mode 100644 drivers/net/ethernet/sfc/efx_cxl.c
+ create mode 100644 drivers/net/ethernet/sfc/efx_cxl.h
+ create mode 100644 include/cxl/pci.h
+ create mode 100644 tools/net/ynl/pyynl/lib/specdir.py
+ create mode 100755 tools/testing/selftests/drivers/net/hw/devlink_rate_cross_esw.py
+Merging bpf-next/for-next (9a3a07d06e7d7 Merge branch 'bpf-fix-warning-in-bpf_trampoline_multi_detach')
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git bpf-next/for-next
+Auto-merging kernel/bpf/btf.c
+Auto-merging kernel/bpf/verifier.c
+Auto-merging tools/testing/selftests/bpf/prog_tests/sockmap_basic.c
+Merge made by the 'ort' strategy.
+ Documentation/bpf/bpf_devel_QA.rst                 |    5 +-
+ Documentation/bpf/index.rst                        |    1 +
+ Documentation/bpf/kfuncs.rst                       |   23 +-
+ Documentation/bpf/signing.rst                      |  497 +++++++++
+ arch/s390/net/bpf_jit_comp.c                       |    4 +-
+ include/linux/bpf.h                                |   16 +-
+ include/linux/bpf_verifier.h                       |   28 +-
+ include/uapi/linux/bpf.h                           |   28 +-
+ kernel/bpf/bpf_lsm.c                               |    1 -
+ kernel/bpf/btf.c                                   |   19 +-
+ kernel/bpf/core.c                                  |    9 +-
+ kernel/bpf/cpumask.c                               |    6 +-
+ kernel/bpf/fixups.c                                |   11 +-
+ kernel/bpf/syscall.c                               |   85 +-
+ kernel/bpf/task_iter.c                             |   36 +-
+ kernel/bpf/trampoline.c                            |    1 +
+ kernel/bpf/verifier.c                              |  627 +++++++++---
+ kernel/trace/bpf_trace.c                           |   55 +
+ net/core/filter.c                                  |  106 +-
+ tools/bpf/bpftool/Makefile                         |    2 +-
+ tools/bpf/bpftool/btf.c                            |   13 +-
+ tools/bpf/bpftool/btf_dumper.c                     |   14 +-
+ tools/bpf/bpftool/gen.c                            |    2 +
+ tools/bpf/bpftool/link.c                           |  133 +++
+ tools/bpf/bpftool/map.c                            |   12 +-
+ tools/bpf/bpftool/sign.c                           |   24 +-
+ tools/bpf/bpftool/struct_ops.c                     |    4 +
+ tools/bpf/resolve_btfids/Makefile                  |    5 +-
+ tools/include/linux/btf_ids.h                      |   78 +-
+ tools/include/uapi/linux/bpf.h                     |   28 +-
+ tools/lib/bpf/bpf_gen_internal.h                   |    1 -
+ tools/lib/bpf/btf.c                                |   20 +-
+ tools/lib/bpf/btf_dump.c                           |    4 +-
+ tools/lib/bpf/btf_relocate.c                       |    8 +-
+ tools/lib/bpf/elf.c                                |    2 +-
+ tools/lib/bpf/gen_loader.c                         |   92 +-
+ tools/lib/bpf/libbpf.c                             |   88 +-
+ tools/lib/bpf/libbpf_internal.h                    |    1 -
+ tools/lib/bpf/nlattr.c                             |    2 +-
+ tools/lib/bpf/relo_core.c                          |   22 +-
+ tools/lib/bpf/skel_internal.h                      |   31 +-
+ tools/lib/bpf/usdt.c                               |   33 +-
+ tools/sched_ext/include/scx/compat.bpf.h           |    2 +-
+ .../bpf/libarena/include/libarena/bitmap.h         |   34 +
+ .../bpf/libarena/include/libarena/common.h         |    2 +-
+ .../{st_asan_buddy.bpf.c => test_asan_buddy.bpf.c} |   10 +-
+ .../{st_asan_common.h => test_asan_common.h}       |    0
+ .../bpf/libarena/selftests/test_bitmap.bpf.c       |  394 +++++++
+ .../selftests/{st_buddy.bpf.c => test_buddy.bpf.c} |    8 +-
+ .../libarena/selftests/test_parallel_bitmap.bpf.c  |  190 ++++
+ .../libarena/selftests/test_parallel_spmc.bpf.c    |    9 +-
+ .../selftests/bpf/libarena/src/bitmap.bpf.c        |  245 +++++
+ .../selftests/bpf/libarena/src/common.bpf.c        |    9 +-
+ tools/testing/selftests/bpf/network_helpers.c      |    4 +-
+ tools/testing/selftests/bpf/network_helpers.h      |    5 +
+ .../testing/selftests/bpf/prog_tests/bpf_tcp_ca.c  |   12 +
+ .../selftests/bpf/prog_tests/fexit_bpf2bpf.c       |   14 -
+ .../selftests/bpf/prog_tests/fill_link_info.c      |  242 +++++
+ .../bpf/prog_tests/get_smp_processor_id.c          |   45 +
+ .../selftests/bpf/prog_tests/icmp_send_kfunc.c     |  292 ++++++
+ .../testing/selftests/bpf/prog_tests/kfunc_call.c  |    1 +
+ tools/testing/selftests/bpf/prog_tests/libarena.c  |    8 +-
+ .../selftests/bpf/prog_tests/libarena_asan.c       |    8 +-
+ .../testing/selftests/bpf/prog_tests/linked_list.c |    1 +
+ tools/testing/selftests/bpf/prog_tests/mptcp.c     |   13 +-
+ .../selftests/bpf/prog_tests/res_spin_lock.c       |   20 +
+ .../selftests/bpf/prog_tests/resolve_btfids.c      |  169 ++-
+ .../selftests/bpf/prog_tests/signed_loader.c       | 1083 ++++++++++++++++----
+ .../selftests/bpf/prog_tests/sockmap_basic.c       |   52 -
+ .../testing/selftests/bpf/prog_tests/tc_redirect.c |   68 ++
+ .../selftests/bpf/prog_tests/test_map_uninit.c     |   68 ++
+ .../selftests/bpf/prog_tests/tracing_multi.c       |   69 ++
+ .../testing/selftests/bpf/progs/bpf_tracing_net.h  |    3 +
+ tools/testing/selftests/bpf/progs/btf_data.c       |   10 +
+ tools/testing/selftests/bpf/progs/cpumask_common.h |    2 +-
+ .../testing/selftests/bpf/progs/cpumask_failure.c  |   23 +-
+ .../testing/selftests/bpf/progs/cpumask_success.c  |    6 +-
+ .../selftests/bpf/progs/freplace_cls_redirect.c    |   34 -
+ .../selftests/bpf/progs/get_smp_processor_id.c     |   20 +
+ tools/testing/selftests/bpf/progs/icmp_send.c      |  123 +++
+ .../testing/selftests/bpf/progs/kfunc_call_fail.c  |   33 +
+ .../testing/selftests/bpf/progs/linked_list_fail.c |   23 +
+ tools/testing/selftests/bpf/progs/map_kptr.c       |   12 +
+ tools/testing/selftests/bpf/progs/mptcpify.c       |    2 +-
+ .../selftests/bpf/progs/sockmap_verdict_prog.c     |   14 +-
+ .../bpf/progs/tcp_ca_untrusted_btf_write.c         |   26 +
+ .../selftests/bpf/progs/test_fill_link_info.c      |    6 +
+ .../selftests/bpf/progs/test_signed_loader.c       |    9 +-
+ .../selftests/bpf/progs/test_sockmap_update.c      |   48 -
+ tools/testing/selftests/bpf/progs/test_tc_peer.c   |   22 +
+ .../bpf/progs/tracing_multi_intersect_attach.c     |    8 +
+ tools/testing/selftests/bpf/progs/verifier_gotox.c |   25 +
+ .../testing/selftests/bpf/progs/verifier_map_ptr.c |   23 +-
+ .../selftests/bpf/progs/verifier_scalar_ids.c      |   56 +
+ .../selftests/bpf/progs/verifier_sockmap_mutate.c  |   12 +-
+ .../selftests/bpf/progs/verifier_spill_fill.c      |   68 ++
+ .../testing/selftests/bpf/progs/verifier_var_off.c |  110 ++
+ tools/testing/selftests/bpf/test_sockmap.c         |    3 +-
+ tools/testing/selftests/bpf/veristat.c             |   23 +
+ 99 files changed, 5037 insertions(+), 961 deletions(-)
+ create mode 100644 Documentation/bpf/signing.rst
+ create mode 100644 tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h
+ rename tools/testing/selftests/bpf/libarena/selftests/{st_asan_buddy.bpf.c => test_asan_buddy.bpf.c} (98%)
+ rename tools/testing/selftests/bpf/libarena/selftests/{st_asan_common.h => test_asan_common.h} (100%)
+ create mode 100644 tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c
+ rename tools/testing/selftests/bpf/libarena/selftests/{st_buddy.bpf.c => test_buddy.bpf.c} (97%)
+ create mode 100644 tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c
+ create mode 100644 tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/get_smp_processor_id.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/icmp_send_kfunc.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/test_map_uninit.c
+ delete mode 100644 tools/testing/selftests/bpf/progs/freplace_cls_redirect.c
+ create mode 100644 tools/testing/selftests/bpf/progs/get_smp_processor_id.c
+ create mode 100644 tools/testing/selftests/bpf/progs/icmp_send.c
+ create mode 100644 tools/testing/selftests/bpf/progs/tcp_ca_untrusted_btf_write.c
+ delete mode 100644 tools/testing/selftests/bpf/progs/test_sockmap_update.c
+$ git am -3 ../patches/0001-perf-Fixup-for-btf_vlan-API-change.patch
+Applying: perf: Fixup for btf_vlan() API change
+Using index info to reconstruct a base tree...
+M	tools/perf/builtin-trace.c
+M	tools/perf/util/btf.c
+Falling back to patching base and 3-way merge...
+Auto-merging tools/perf/builtin-trace.c
+No changes -- Patch already applied.
+Merging ipsec-next/master (355f808d8a11f Merge branch 'xfrm: XFRM_MSG_MIGRATE_STATE new netlink message')
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec-next.git ipsec-next/master
+Already up to date.
+Merging mlx5-next/mlx5-next (bee40a7d0bd12 net/mlx5: Add PSP related fields to the mlx5_ifc)
+$ git merge -m Merge branch 'mlx5-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux.git mlx5-next/mlx5-next
+Merge made by the 'ort' strategy.
+ drivers/net/ethernet/mellanox/mlx5/core/fs_core.h  | 12 +------
+ .../ethernet/mellanox/mlx5/core/steering/hws/cmd.c |  6 ++--
+ include/linux/mlx5/device.h                        |  1 +
+ include/linux/mlx5/mlx5_ifc.h                      | 38 ++++++++++++----------
+ 4 files changed, 25 insertions(+), 32 deletions(-)
+Merging netfilter-next/main (f6f3b36c15ed4 net: ethernet: qualcomm: remove unneeded 'fast_io' parameter in regmap_config)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf-next.git netfilter-next/main
+Already up to date.
+Merging ipvs-next/main (f6f3b36c15ed4 net: ethernet: qualcomm: remove unneeded 'fast_io' parameter in regmap_config)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs-next.git ipvs-next/main
+Already up to date.
+Merging bluetooth/master (bd8bee79e1fa8 Bluetooth: btintel_pcie: serialize reset_type with RECOVERY_IN_PROGRESS)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next.git bluetooth/master
+Auto-merging drivers/bluetooth/btintel_pcie.c
+CONFLICT (content): Merge conflict in drivers/bluetooth/btintel_pcie.c
+Auto-merging drivers/bluetooth/btnxpuart.c
+Auto-merging drivers/bluetooth/btqca.c
+Auto-merging drivers/bluetooth/hci_qca.c
+Auto-merging net/bluetooth/af_bluetooth.c
+Auto-merging net/bluetooth/hci_event.c
+Auto-merging net/bluetooth/hci_sync.c
+Auto-merging net/bluetooth/l2cap_sock.c
+Recorded preimage for 'drivers/bluetooth/btintel_pcie.c'
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+Recorded resolution for 'drivers/bluetooth/btintel_pcie.c'.
+[master 8d77a46db4d58] Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next.git
+$ git diff -M --stat --summary HEAD^..
+ drivers/bluetooth/Makefile       |   2 +
+ drivers/bluetooth/btintel_pcie.c | 260 ++++++++++++++++++++++++++-------------
+ drivers/bluetooth/btintel_pcie.h |  12 +-
+ drivers/bluetooth/btnxpuart.c    |   4 +-
+ drivers/bluetooth/btqca.c        |  25 +---
+ drivers/bluetooth/btusb.c        | 100 +++++++++------
+ drivers/bluetooth/hci_nokia.c    |  20 +++
+ include/net/bluetooth/hci_sync.h |   4 +-
+ net/bluetooth/Makefile           |   2 +
+ net/bluetooth/af_bluetooth.c     |   7 +-
+ net/bluetooth/bnep/Makefile      |   2 +
+ net/bluetooth/coredump.c         |   2 -
+ net/bluetooth/hci_core.c         |   3 +
+ net/bluetooth/hci_debugfs.c      |  12 +-
+ net/bluetooth/hci_event.c        |   5 +-
+ net/bluetooth/hci_sync.c         |  44 ++-----
+ net/bluetooth/hidp/Makefile      |   2 +
+ net/bluetooth/l2cap_sock.c       |   1 +
+ net/bluetooth/rfcomm/Makefile    |   2 +
+ net/bluetooth/rfcomm/sock.c      |   1 +
+ 20 files changed, 309 insertions(+), 201 deletions(-)
+Merging wireless-next/for-next (ac798f757d647 wifi: mac80211: Route (Re)association req/response to per-STA queue)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next.git wireless-next/for-next
+Auto-merging drivers/net/wireless/ath/ath12k/mac.c
+Auto-merging drivers/net/wireless/marvell/mwifiex/cfg80211.c
+Auto-merging drivers/net/wireless/virtual/mac80211_hwsim_main.c
+Auto-merging include/net/cfg80211.h
+Auto-merging net/mac80211/cfg.c
+Auto-merging net/mac80211/tx.c
+Auto-merging net/wireless/nl80211.c
+Merge made by the 'ort' strategy.
+ drivers/net/wireless/ath/ath12k/mac.c              |   6 -
+ drivers/net/wireless/ath/wil6210/cfg80211.c        |  10 +-
+ drivers/net/wireless/broadcom/b43/debugfs.c        |  12 +-
+ drivers/net/wireless/broadcom/b43legacy/debugfs.c  |  12 +-
+ drivers/net/wireless/intersil/p54/Kconfig          |   6 +-
+ drivers/net/wireless/intersil/p54/fwio.c           |   4 +-
+ drivers/net/wireless/intersil/p54/p54usb.c         |   2 +-
+ drivers/net/wireless/marvell/libertas/debugfs.c    |  39 +-
+ drivers/net/wireless/marvell/mwifiex/cfg80211.c    |   8 +-
+ drivers/net/wireless/marvell/mwifiex/debugfs.c     |  62 ++-
+ drivers/net/wireless/realtek/rtw89/core.c          |   3 -
+ drivers/net/wireless/ti/wlcore/main.c              |  14 +-
+ drivers/net/wireless/virtual/mac80211_hwsim_main.c |   7 +-
+ include/linux/ieee80211.h                          |  12 +
+ include/net/cfg80211.h                             |  41 +-
+ include/net/ieee80211_radiotap.h                   | 190 ++++++++
+ include/uapi/linux/nl80211.h                       |  46 +-
+ net/mac80211/cfg.c                                 |   6 +-
+ net/mac80211/ieee80211_i.h                         |   4 +-
+ net/mac80211/status.c                              |   2 +-
+ net/mac80211/tx.c                                  |   7 +
+ net/wireless/nl80211.c                             | 476 ++++++++++++++-------
+ net/wireless/rdev-ops.h                            |  10 +-
+ net/wireless/trace.h                               |   2 +-
+ net/wireless/wext-core.c                           |   6 +-
+ 25 files changed, 693 insertions(+), 294 deletions(-)
+Merging ath-next/for-next (f78703024c71a wifi: ath6kl: avoid buffer overreads in WMI event handlers)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git ath-next/for-next
+Auto-merging drivers/net/wireless/ath/ath10k/htt_rx.c
+Auto-merging drivers/net/wireless/ath/ath11k/dp_rx.c
+Auto-merging drivers/net/wireless/ath/ath12k/mac.c
+Auto-merging drivers/net/wireless/ath/ath12k/pci.c
+Auto-merging drivers/net/wireless/ath/ath12k/peer.c
+Auto-merging drivers/net/wireless/ath/ath12k/wifi7/hw.c
+Auto-merging drivers/net/wireless/ath/ath6kl/wmi.c
+Merge made by the 'ort' strategy.
+ drivers/net/wireless/ath/ath10k/ahb.c              |  12 +-
+ drivers/net/wireless/ath/ath10k/htt_rx.c           |   6 +-
+ drivers/net/wireless/ath/ath11k/dp_rx.c            |  32 +--
+ drivers/net/wireless/ath/ath11k/wmi.c              |   6 +
+ drivers/net/wireless/ath/ath12k/Kconfig            |   2 +-
+ drivers/net/wireless/ath/ath12k/ahb.c              |  11 +-
+ drivers/net/wireless/ath/ath12k/ahb.h              |   2 +-
+ drivers/net/wireless/ath/ath12k/core.c             |   9 +-
+ drivers/net/wireless/ath/ath12k/core.h             |   5 +-
+ drivers/net/wireless/ath/ath12k/debugfs.c          |  14 +-
+ drivers/net/wireless/ath/ath12k/dp.c               |  14 +-
+ drivers/net/wireless/ath/ath12k/dp.h               |   3 +-
+ drivers/net/wireless/ath/ath12k/dp_mon.c           |  70 +-----
+ drivers/net/wireless/ath/ath12k/dp_mon.h           |   4 +-
+ drivers/net/wireless/ath/ath12k/hal.c              |  42 +++-
+ drivers/net/wireless/ath/ath12k/hal.h              |  15 +-
+ drivers/net/wireless/ath/ath12k/mac.c              |  39 ++-
+ drivers/net/wireless/ath/ath12k/pci.c              |  12 +-
+ drivers/net/wireless/ath/ath12k/peer.c             | 132 ++++++++---
+ drivers/net/wireless/ath/ath12k/peer.h             |  14 +-
+ drivers/net/wireless/ath/ath12k/qmi.c              | 261 ++++++++++++---------
+ drivers/net/wireless/ath/ath12k/qmi.h              |  34 +--
+ drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c     |  66 +++---
+ .../net/wireless/ath/ath12k/wifi7/hal_qcc2072.c    |   4 +-
+ .../net/wireless/ath/ath12k/wifi7/hal_qcn9274.c    |  13 +-
+ .../net/wireless/ath/ath12k/wifi7/hal_rx_desc.h    |  34 ++-
+ .../net/wireless/ath/ath12k/wifi7/hal_wcn7850.c    |  13 +-
+ drivers/net/wireless/ath/ath12k/wifi7/hw.c         |  67 ++++--
+ drivers/net/wireless/ath/ath12k/wmi.c              |  65 +++--
+ drivers/net/wireless/ath/ath12k/wmi.h              |   8 +
+ drivers/net/wireless/ath/ath6kl/wmi.c              |  17 +-
+ kernel/irq/manage.c                                |   1 +
+ 32 files changed, 635 insertions(+), 392 deletions(-)
+Merging iwlwifi-next/next (905f57aefde4f wifi: iwlwifi: mei: pass correct argument to function)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git iwlwifi-next/next
+Auto-merging drivers/net/wireless/intel/iwlwifi/fw/api/nvm-reg.h
+Auto-merging drivers/net/wireless/intel/iwlwifi/fw/pnvm.c
+Auto-merging drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
+Auto-merging drivers/net/wireless/intel/iwlwifi/mld/agg.c
+Auto-merging drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
+Auto-merging drivers/net/wireless/intel/iwlwifi/mld/mcc.c
+Auto-merging drivers/net/wireless/intel/iwlwifi/mvm/d3.c
+Auto-merging drivers/net/wireless/intel/iwlwifi/mvm/fw.c
+Auto-merging drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
+Auto-merging drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
+Merge made by the 'ort' strategy.
+ drivers/net/wireless/intel/iwlwifi/dvm/main.c      |  4 +-
+ drivers/net/wireless/intel/iwlwifi/dvm/tx.c        |  4 +-
+ drivers/net/wireless/intel/iwlwifi/fw/acpi.c       |  4 +-
+ .../net/wireless/intel/iwlwifi/fw/api/nvm-reg.h    | 41 +++++++++++-
+ drivers/net/wireless/intel/iwlwifi/fw/dump.c       |  2 +-
+ drivers/net/wireless/intel/iwlwifi/fw/error-dump.h |  2 +-
+ drivers/net/wireless/intel/iwlwifi/fw/file.h       |  5 +-
+ drivers/net/wireless/intel/iwlwifi/fw/pnvm.c       |  6 ++
+ drivers/net/wireless/intel/iwlwifi/fw/runtime.h    | 13 +++-
+ drivers/net/wireless/intel/iwlwifi/fw/uefi.c       | 26 +++++---
+ drivers/net/wireless/intel/iwlwifi/fw/uefi.h       |  4 +-
+ drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c   |  8 ++-
+ drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c | 33 +++++++++-
+ drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h | 22 ++++++-
+ drivers/net/wireless/intel/iwlwifi/mei/main.c      | 28 ++++++--
+ drivers/net/wireless/intel/iwlwifi/mld/agg.c       |  9 ++-
+ drivers/net/wireless/intel/iwlwifi/mld/d3.c        | 70 ++++++++++++++------
+ .../net/wireless/intel/iwlwifi/mld/ftm-initiator.c | 21 ++++++
+ .../net/wireless/intel/iwlwifi/mld/ftm-initiator.h |  1 +
+ drivers/net/wireless/intel/iwlwifi/mld/hcmd.h      |  9 ++-
+ drivers/net/wireless/intel/iwlwifi/mld/mac80211.c  | 19 ++++--
+ drivers/net/wireless/intel/iwlwifi/mld/mac80211.h  |  2 +
+ drivers/net/wireless/intel/iwlwifi/mld/mcc.c       | 26 +++++---
+ drivers/net/wireless/intel/iwlwifi/mld/mld.c       |  7 +-
+ drivers/net/wireless/intel/iwlwifi/mld/mld.h       |  2 -
+ drivers/net/wireless/intel/iwlwifi/mld/notif.c     |  2 +-
+ .../net/wireless/intel/iwlwifi/mld/regulatory.c    | 74 ++++++++++++++++------
+ drivers/net/wireless/intel/iwlwifi/mld/sta.c       |  5 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/fw.c        | 36 +++++++++++
+ drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c  |  9 ++-
+ drivers/net/wireless/intel/iwlwifi/mvm/mvm.h       | 23 +++++--
+ drivers/net/wireless/intel/iwlwifi/mvm/ops.c       | 10 +--
+ drivers/net/wireless/intel/iwlwifi/mvm/utils.c     | 54 +++-------------
+ drivers/net/wireless/intel/iwlwifi/pcie/drv.c      |  1 +
+ .../wireless/intel/iwlwifi/pcie/gen1_2/tx-gen2.c   |  2 +
+ .../net/wireless/intel/iwlwifi/pcie/gen1_2/tx.c    | 11 +++-
+ 36 files changed, 434 insertions(+), 161 deletions(-)
+Merging wpan-next/master (a6bfdfcc6711d ieee802154: allow legacy LLSEC ADD/DEL ops to pass strict validation)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git wpan-next/master
+Already up to date.
+Merging wpan-staging/staging (a6bfdfcc6711d ieee802154: allow legacy LLSEC ADD/DEL ops to pass strict validation)
+$ git merge -m Merge branch 'staging' of https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git wpan-staging/staging
+Already up to date.
+Merging mtd/mtd/next (d276783e490d7 mtd: maps: correct CONFIG_MTD_COMPLEX_MAPPINGS macro name in comment)
+$ git merge -m Merge branch 'mtd/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git mtd/mtd/next
+Merge made by the 'ort' strategy.
+ drivers/mtd/chips/cfi_cmdset_0001.c | 6 ++++++
+ drivers/mtd/maps/map_funcs.c        | 2 +-
+ 2 files changed, 7 insertions(+), 1 deletion(-)
+Merging nand/nand/next (adfc275b317c0 mtd: parsers: redboot: reject unterminated FIS names)
+$ git merge -m Merge branch 'nand/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git nand/nand/next
+Auto-merging drivers/mtd/nand/spi/core.c
+Merge made by the 'ort' strategy.
+ drivers/mtd/nand/raw/atmel/nand-controller.c |   3 +-
+ drivers/mtd/nand/raw/pl35x-nand-controller.c |  20 ++--
+ drivers/mtd/nand/spi/Makefile                |   2 +-
+ drivers/mtd/nand/spi/core.c                  |   1 +
+ drivers/mtd/nand/spi/fmsh.c                  |  95 +++++++++++++++++++
+ drivers/mtd/nand/spi/heyangtek.c             | 132 +++++++++++++++++++++++++++
+ drivers/mtd/parsers/redboot.c                |   9 +-
+ include/linux/mtd/spinand.h                  |   1 +
+ 8 files changed, 252 insertions(+), 11 deletions(-)
+ create mode 100644 drivers/mtd/nand/spi/heyangtek.c
+Merging spi-nor/spi-nor/next (df415c5e1de0f mtd: spi-nor: spansion: add die erase support in s28hx-t)
+$ git merge -m Merge branch 'spi-nor/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git spi-nor/spi-nor/next
+Already up to date.
+Merging crypto/master (df373d39c6f03 crypto: qce - Mark QCE as BROKEN)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6.git crypto/master
+Auto-merging MAINTAINERS
+Auto-merging arch/arm/configs/multi_v7_defconfig
+Auto-merging arch/arm64/configs/defconfig
+Auto-merging drivers/char/hw_random/airoha-trng.c
+Auto-merging drivers/char/hw_random/atmel-rng.c
+Auto-merging drivers/char/hw_random/imx-rngc.c
+Auto-merging drivers/char/hw_random/xgene-rng.c
+Auto-merging drivers/char/hw_random/xilinx-trng.c
+Auto-merging drivers/crypto/atmel-tdes.c
+Auto-merging drivers/crypto/ccp/sev-dev.c
+Auto-merging lib/rhashtable.c
+Merge made by the 'ort' strategy.
+ .../ABI/testing/sysfs-firmware-sev-vulnerabilities |  19 ++
+ Documentation/admin-guide/sysctl/crypto.rst        |  36 +++
+ Documentation/crypto/architecture.rst              |   2 +-
+ Documentation/crypto/userspace-if.rst              |  13 +-
+ .../bindings/crypto/qcom,inline-crypto-engine.yaml |   5 +
+ .../devicetree/bindings/crypto/qcom,prng.yaml      |   1 +
+ .../bindings/rng/inside-secure,safexcel-eip76.yaml |   2 +-
+ MAINTAINERS                                        |   2 +-
+ arch/arm/configs/multi_v7_defconfig                |   3 +-
+ arch/arm/configs/qcom_defconfig                    |   2 +-
+ arch/arm64/configs/defconfig                       |   3 +-
+ arch/powerpc/crypto/aes_cbc.c                      |   6 +-
+ crypto/af_alg.c                                    |  72 +++++-
+ crypto/algif_aead.c                                |  11 +
+ crypto/algif_hash.c                                |  24 ++
+ crypto/algif_rng.c                                 |   9 +
+ crypto/algif_skcipher.c                            |  23 ++
+ crypto/api.c                                       |   2 +-
+ crypto/crypto_user.c                               |   9 +-
+ crypto/ecc.c                                       |  98 +++++----
+ crypto/hctr2.c                                     |   3 +-
+ crypto/lrw.c                                       |   2 +-
+ crypto/lskcipher.c                                 |   3 +-
+ crypto/xts.c                                       |   3 +-
+ drivers/char/hw_random/Kconfig                     |  13 +-
+ drivers/char/hw_random/Makefile                    |   1 +
+ drivers/char/hw_random/airoha-trng.c               |   4 +-
+ drivers/char/hw_random/atmel-rng.c                 |   2 +-
+ drivers/char/hw_random/cctrng.c                    |   2 +-
+ drivers/char/hw_random/core.c                      |   4 +-
+ drivers/char/hw_random/imx-rngc.c                  |   2 +-
+ drivers/char/hw_random/jh7110-trng.c               |   3 +-
+ drivers/char/hw_random/ks-sa-rng.c                 |   9 +-
+ drivers/char/hw_random/omap-rng.c                  |  35 +--
+ drivers/{crypto => char/hw_random}/qcom-rng.c      | 156 +++----------
+ drivers/char/hw_random/xgene-rng.c                 |   2 +-
+ drivers/char/hw_random/xilinx-trng.c               |  32 ++-
+ drivers/crypto/Kconfig                             |  18 +-
+ drivers/crypto/Makefile                            |   1 -
+ drivers/crypto/allwinner/Kconfig                   |  16 --
+ drivers/crypto/allwinner/sun8i-ce/Makefile         |   1 -
+ drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c  |  63 ------
+ drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c  | 159 --------------
+ drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h       |  29 ---
+ drivers/crypto/allwinner/sun8i-ss/Makefile         |   1 -
+ drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c  |  45 ----
+ drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c  | 177 ---------------
+ drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h       |  23 --
+ drivers/crypto/amcc/crypto4xx_core.c               |  11 +-
+ drivers/crypto/aspeed/aspeed-acry.c                |   2 +-
+ drivers/crypto/aspeed/aspeed-hace.c                |   2 +-
+ drivers/crypto/atmel-ecc.c                         |  50 ++---
+ drivers/crypto/atmel-i2c.c                         |   5 +-
+ drivers/crypto/atmel-sha204a.c                     |  16 +-
+ drivers/crypto/atmel-tdes.c                        |  66 +++---
+ drivers/crypto/caam/Kconfig                        |   9 -
+ drivers/crypto/caam/Makefile                       |   1 -
+ drivers/crypto/caam/caamprng.c                     | 241 ---------------------
+ drivers/crypto/caam/intern.h                       |  15 --
+ drivers/crypto/caam/jr.c                           |   2 -
+ drivers/crypto/cavium/nitrox/nitrox_hal.c          |   3 +-
+ drivers/crypto/ccp/ccp-crypto-sha.c                |   2 +-
+ drivers/crypto/ccp/sev-dev.c                       | 196 ++++++++++++++++-
+ drivers/crypto/ccp/sev-dev.h                       |   3 +
+ drivers/crypto/gemini/sl3516-ce-cipher.c           |   8 -
+ drivers/crypto/hisilicon/qm.c                      |   5 +-
+ drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c        |   1 +
+ .../crypto/intel/keembay/keembay-ocs-aes-core.c    |   4 +-
+ drivers/crypto/intel/keembay/keembay-ocs-ecc.c     |   1 +
+ drivers/crypto/intel/qat/qat_common/adf_aer.c      |   2 +
+ drivers/crypto/intel/qat/qat_common/adf_cfg.c      |   7 +-
+ .../crypto/intel/qat/qat_common/adf_cfg_services.c |  15 +-
+ .../crypto/intel/qat/qat_common/adf_cfg_services.h |   2 +-
+ .../crypto/intel/qat/qat_common/adf_mstate_mgr.c   |   3 +-
+ drivers/crypto/intel/qat/qat_common/adf_sysfs.c    |   2 +-
+ .../intel/qat/qat_common/adf_transport_debug.c     |   3 +-
+ drivers/crypto/intel/qat/qat_common/qat_algs.c     |   1 +
+ .../crypto/intel/qat/qat_common/qat_compression.c  |   3 +-
+ drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c  |   4 +-
+ .../crypto/marvell/octeontx2/otx2_cptpf_ucode.c    |   4 +-
+ drivers/crypto/mxs-dcp.c                           |   2 +-
+ drivers/crypto/qce/aead.c                          |  56 +----
+ drivers/crypto/qce/common.c                        |  55 +----
+ drivers/crypto/qce/common.h                        |  16 +-
+ drivers/crypto/qce/regs-v5.h                       |   4 -
+ drivers/crypto/qce/sha.c                           |  61 +-----
+ drivers/crypto/qce/sha.h                           |   1 -
+ drivers/crypto/qce/skcipher.c                      |  97 +--------
+ drivers/crypto/rockchip/rk3288_crypto_ahash.c      |   7 +-
+ drivers/crypto/sa2ul.c                             |   6 +-
+ drivers/gpu/drm/ci/arm64.config                    |   2 +-
+ include/crypto/aes.h                               |   2 +-
+ include/crypto/if_alg.h                            |   8 +
+ include/linux/psp-sev.h                            |  51 +++++
+ include/linux/rhashtable.h                         |   1 -
+ lib/rhashtable.c                                   |   2 +
+ 96 files changed, 783 insertions(+), 1428 deletions(-)
+ create mode 100644 Documentation/ABI/testing/sysfs-firmware-sev-vulnerabilities
+ rename drivers/{crypto => char/hw_random}/qcom-rng.c (53%)
+ delete mode 100644 drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c
+ delete mode 100644 drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c
+ delete mode 100644 drivers/crypto/caam/caamprng.c
+Merging libcrypto/libcrypto-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'libcrypto-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git libcrypto/libcrypto-next
+Already up to date.
+Merging drm/drm-next (a1c81cb052730 Merge tag 'drm-intel-gt-next-2026-07-16' of https://gitlab.freedesktop.org/drm/i915/kernel into drm-next)
+  0421fc6ab3a85 ("drm/amd/display: fix __udivdi3 link error")
+  11a90eaf5c808 ("drm/amd/display: Fix 8K Mode Not Parsed by EDID")
+  11c141672045f ("drm/amdgpu: Fix VFCT bus number matching with soft filter")
+  14a2012a49025 ("drm/i915/gt: use correct selftest config symbol")
+  183bbded999a7 ("drm/amd/display: fix dcn42b det allocation order")
+  198663d035cc4 ("drm/amd/display: fix dcn42 det allocation order")
+  1fff2e07b6670 ("drm/amdkfd: free MQD managers on DQM init failures")
+  28ec64943e3ee ("drm/amd/display: dce100: skip non-DP stream encoders for DP MST")
+  2f79f0130f828 ("drm/amd/display: Set native cursor mode for disabled CRTCs")
+  319f7e13423ae ("drm/amdkfd: fix 32-bit overflow in CWSR total size calculation")
+  358dd0a9ce66d ("drm/amd/pm/si: Fix AC/DC switch notification")
+  3ddc0ae76202c ("drm/amdgpu: fix bo->pin leaking in amdgpu_bo_create_reserved")
+  4e2c0821509fe ("drm/amdgpu: Reserve space for IB contents in devcoredumps")
+  516f8fc30a1b5 ("drm/amd/pm/smu7: Fix AC/DC switch notification")
+  682ea2d28d18b ("drm/i915/selftests: Fix GT PM sort comparators")
+  85453fb4ff726 ("drm/amd/display: wire DCN42B mcache programming callback")
+  871ceb853841b ("drm/amd/display: Add dp_skip_rbr flag for NUTMEG")
+  99f3af19073b3 ("drm/amd/display: set new_stream to NULL after release")
+  9ceb4e034a327 ("drm/amdgpu: Disable PCIe dynamic speed switching on Ryzen Pinnacle Ridge")
+  a8171229bc836 ("drm/amdgpu/ttm: Consider concurrent VM flushes for buffer entities")
+  b7d69145907cd ("drm/amd/display: Fix DCN42B null registers & register masks")
+  bc639a9eadc75 ("drm/amdgpu: always emit the job vm fence")
+  bd018d36171a6 ("drm/amd/pm/si: Don't schedule thermal work when queue isn't initialized")
+  ca5988682b4cb ("drm/amdgpu: Release VFCT ACPI table reference")
+  d34acad064ee7 ("drm/amd/pm/ci: Don't disable MCLK DPM on Bonaire 0x6658 (R7 260X)")
+  e78b0a367f869 ("drm/amd/display: Fix preferred link rate for NUTMEG")
+  ee186ae23ab0f ("Revert "drm/amd/display: Restore 5s vbl offdelay for NV3x+ DGPUs"")
+  f8ee6447e7ec1 ("drm/amdgpu/discovery: Fix device family for DCN42")
+  fed7aa36d7980 ("drm/amdgpu: Print vmid, pasid and more task info in devcoredump")
+$ git merge -m Merge branch 'drm-next' of https://gitlab.freedesktop.org/drm/kernel.git drm/drm-next
+Auto-merging .mailmap
+Auto-merging Documentation/admin-guide/cgroup-v2.rst
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.yaml
+Auto-merging MAINTAINERS
+Auto-merging drivers/accel/amdxdna/amdxdna_gem.c
+Auto-merging drivers/gpu/buddy.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/soc21.c
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+CONFLICT (content): Merge conflict in drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_events.c
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_process.c
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+Auto-merging drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+Auto-merging drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c
+Auto-merging drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+Auto-merging drivers/gpu/drm/amd/display/dc/core/dc.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/amd/display/dc/core/dc.c
+Auto-merging drivers/gpu/drm/amd/display/dc/dc.h
+Auto-merging drivers/gpu/drm/amd/display/dc/link/link_detection.c
+Auto-merging drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
+Auto-merging drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c
+Auto-merging drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c
+Auto-merging drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.h
+Auto-merging drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c
+Auto-merging drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
+Auto-merging drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
+CONFLICT (content): Merge conflict in drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
+Auto-merging drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c
+Auto-merging drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c
+Auto-merging drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
+Auto-merging drivers/gpu/drm/drm_fb_helper.c
+Auto-merging drivers/gpu/drm/drm_gem.c
+Auto-merging drivers/gpu/drm/drm_gem_shmem_helper.c
+Auto-merging drivers/gpu/drm/drm_gpusvm.c
+Auto-merging drivers/gpu/drm/i915/display/intel_dp.c
+Auto-merging drivers/gpu/drm/i915/display/intel_lt_phy.c
+Auto-merging drivers/gpu/drm/i915/display/intel_psr.c
+Auto-merging drivers/gpu/drm/msm/dsi/dsi_host.c
+Auto-merging drivers/gpu/drm/nouveau/nouveau_gem.c
+Auto-merging drivers/gpu/drm/panthor/panthor_gem.c
+Auto-merging drivers/gpu/drm/ttm/ttm_pool.c
+Auto-merging drivers/gpu/drm/v3d/v3d_submit.c
+Auto-merging drivers/gpu/drm/virtio/virtgpu_kms.c
+Auto-merging drivers/gpu/drm/virtio/virtgpu_vq.c
+Auto-merging drivers/gpu/drm/virtio/virtgpu_vram.c
+Auto-merging drivers/gpu/drm/xe/tests/xe_pci.c
+Auto-merging drivers/gpu/drm/xe/xe_device.c
+Auto-merging drivers/gpu/drm/xe/xe_pt.c
+Auto-merging drivers/gpu/drm/xe/xe_vm_madvise.c
+Auto-merging kernel/events/core.c
+Recorded preimage for 'drivers/gpu/drm/amd/amdgpu/amdgpu_device.c'
+Recorded preimage for 'drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c'
+Resolved 'drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h' using previous resolution.
+Resolved 'drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c' using previous resolution.
+Recorded preimage for 'drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c'
+Recorded preimage for 'drivers/gpu/drm/amd/display/dc/core/dc.c'
+Recorded preimage for 'drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c'
+Recorded preimage for 'drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h'
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+Recorded resolution for 'drivers/gpu/drm/amd/amdgpu/amdgpu_device.c'.
+Recorded resolution for 'drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c'.
+Recorded resolution for 'drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c'.
+Recorded resolution for 'drivers/gpu/drm/amd/display/dc/core/dc.c'.
+Recorded resolution for 'drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c'.
+Recorded resolution for 'drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h'.
+[master 71105826f28dd] Merge branch 'drm-next' of https://gitlab.freedesktop.org/drm/kernel.git
+$ git diff -M --stat --summary HEAD^..
+ .clang-format                                      |    2 +-
+ .mailmap                                           |    1 +
+ .../ABI/testing/sysfs-driver-intel-xe-hwmon        |    7 +
+ Documentation/admin-guide/cgroup-v2.rst            |    6 +
+ .../bindings/display/panel/anbernic,td4310.yaml    |   66 +
+ .../bindings/display/panel/chipone,icna3512.yaml   |   79 +
+ .../bindings/display/panel/himax,hx83121a.yaml     |    3 +
+ .../bindings/display/panel/ilitek,ili7807s.yaml    |   71 +
+ .../bindings/display/panel/ilitek,ili9488.yaml     |   63 +
+ .../bindings/display/panel/renesas,r63419.yaml     |   98 +
+ .../bindings/display/panel/samsung,atna33xc20.yaml |    2 +
+ .../display/rockchip/rockchip,analogix-dp.yaml     |   47 +-
+ .../devicetree/bindings/gpu/img,powervr-rogue.yaml |    3 +-
+ .../devicetree/bindings/gpu/img,powervr-sgx.yaml   |    3 +-
+ .../devicetree/bindings/vendor-prefixes.yaml       |    6 +
+ .../gpu/amdgpu/display/display-manager.rst         |    9 +
+ Documentation/gpu/automated_testing.rst            |    3 +-
+ Documentation/gpu/drm-kms-helpers.rst              |    6 +
+ Documentation/gpu/drm-kms.rst                      |   12 +
+ .../gpu/intel-display/dp-link-training.rst         |    8 +
+ Documentation/gpu/intel-display/index.rst          |    1 +
+ Documentation/gpu/todo.rst                         |   50 +-
+ MAINTAINERS                                        |   20 +-
+ drivers/accel/amdxdna/amdxdna_gem.c                |    6 +
+ drivers/accel/ethosu/Makefile                      |    2 +-
+ drivers/accel/ethosu/ethosu_device.h               |   35 +-
+ drivers/accel/ethosu/ethosu_drv.c                  |   23 +-
+ drivers/accel/ethosu/ethosu_drv.h                  |   61 +-
+ drivers/accel/ethosu/ethosu_job.c                  |   61 +-
+ drivers/accel/ethosu/ethosu_job.h                  |    2 +
+ drivers/accel/ethosu/ethosu_perfmon.c              |  301 +
+ drivers/accel/ivpu/ivpu_drv.c                      |   28 +-
+ drivers/accel/ivpu/ivpu_drv.h                      |    5 +-
+ drivers/accel/ivpu/ivpu_hw.c                       |    4 +
+ drivers/accel/ivpu/ivpu_ipc.c                      |   25 +-
+ drivers/accel/ivpu/ivpu_ipc.h                      |    3 +-
+ drivers/accel/ivpu/ivpu_job.c                      |   59 +-
+ drivers/accel/ivpu/ivpu_job.h                      |    7 +-
+ drivers/accel/rocket/rocket_job.c                  |   32 +-
+ drivers/dma-buf/dma-fence-unwrap.c                 |    8 +-
+ drivers/dma-buf/dma-fence.c                        |   22 +-
+ drivers/dma-buf/dma-heap.c                         |    3 +-
+ drivers/dma-buf/st-dma-fence-chain.c               |    4 +-
+ drivers/dma-buf/st-dma-fence-unwrap.c              |   42 +-
+ drivers/dma-buf/st-dma-fence.c                     |   16 +-
+ drivers/dma-buf/st-dma-resv.c                      |   10 +-
+ drivers/firmware/Kconfig                           |   28 +-
+ drivers/gpu/buddy.c                                |  105 +-
+ drivers/gpu/drm/adp/adp-mipi.c                     |    4 +
+ drivers/gpu/drm/amd/amdgpu/Makefile                |    5 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu.h                |  320 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c            |  985 ---
+ drivers/gpu/drm/amd/amdgpu/amdgpu_aca.h            |  232 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c           |   68 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.h           |  151 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c         |   14 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h         |   28 +-
+ .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c    |   63 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c |    3 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12.c |    3 +-
+ .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c   |    3 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c   |   20 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c       |    2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c   |    2 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c   |   33 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c           |   10 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c        |  184 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h        |   22 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c            |   62 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c           |  135 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_cper.h           |    9 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c             |   70 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c            |  215 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h            |   14 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c        |    9 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c   |    1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c         |  349 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c      |  312 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h      |    6 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c            |   33 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c          |   53 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c            |  290 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h            |   24 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c            |    9 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h            |    1 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c             |   14 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ip.c             |  157 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ip.h             |    8 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c            |    2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_job.c            |   13 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c           |    9 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.h           |    3 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c            |   39 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c            |  486 --
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mca.h            |  108 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c            |  293 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h            |   89 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h          |   23 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_object.c         |    3 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_object.h         |   40 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_preempt_mgr.c    |   14 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c         |    5 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c            | 1008 +--
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h            |   70 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c     |  514 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.h     |   26 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h     |    8 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c           |  206 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h           |    8 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c            |   39 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h            |   10 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h             |   77 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c          |    8 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c           |   19 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h           |   28 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h          |  150 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c            |    2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h            |    1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c          |    3 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c            |  266 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h            |   44 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_umsch_mm.c       |    6 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c          |  165 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h          |   19 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c    |   10 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c            |   76 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h            |    3 +
+ .../amdgpu_video_codecs.h}                         |   42 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c           |    1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c           |   10 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c            |    8 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_wb.c             |  129 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_wb.h             |  102 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c            |   42 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c           |  302 +-
+ drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c         |   61 +-
+ drivers/gpu/drm/amd/amdgpu/atom.c                  |   66 +-
+ drivers/gpu/drm/amd/amdgpu/atom.h                  |    3 +-
+ drivers/gpu/drm/amd/amdgpu/cik.c                   |    8 +-
+ drivers/gpu/drm/amd/amdgpu/cik_sdma.c              |   11 +-
+ drivers/gpu/drm/amd/amdgpu/dce_v10_0.c             |   66 -
+ drivers/gpu/drm/amd/amdgpu/dce_v6_0.c              |   57 -
+ drivers/gpu/drm/amd/amdgpu/dce_v8_0.c              |   57 -
+ drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c             |   10 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c             |  373 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c             |  268 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c             |  285 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c              |  236 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c              |   16 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c            | 1097 +--
+ drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c              |    2 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c              |  125 -
+ drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c              |   19 +-
+ drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c             |    2 +-
+ drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.h             |    4 +
+ drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c             |   21 +-
+ drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c           |   88 +-
+ drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c           |    2 +-
+ drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.h           |    4 +
+ drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c           |   94 +-
+ drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.h           |    8 +
+ drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c           |   66 +-
+ drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c           |   22 +-
+ drivers/gpu/drm/amd/amdgpu/mes_userqueue.c         |  165 +-
+ drivers/gpu/drm/amd/amdgpu/mes_userqueue.h         |    9 +
+ drivers/gpu/drm/amd/amdgpu/mes_v11_0.c             |  333 +-
+ drivers/gpu/drm/amd/amdgpu/mes_v12_0.c             |  273 +-
+ drivers/gpu/drm/amd/amdgpu/mes_v12_1.c             |  363 +-
+ drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c            |  231 +-
+ drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c          |   65 +-
+ drivers/gpu/drm/amd/amdgpu/nv.c                    |    7 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c             |   10 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c             |   73 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c             |    8 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c           |  204 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c             |   10 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c             |   10 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c             |   28 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c             |   28 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c             |   28 +-
+ drivers/gpu/drm/amd/amdgpu/si.c                    |   12 +-
+ drivers/gpu/drm/amd/amdgpu/si_dma.c                |    8 +-
+ drivers/gpu/drm/amd/amdgpu/soc15.c                 |   10 +-
+ drivers/gpu/drm/amd/amdgpu/soc15_common.h          |   61 +-
+ drivers/gpu/drm/amd/amdgpu/soc21.c                 |   13 +-
+ drivers/gpu/drm/amd/amdgpu/soc24.c                 |   12 +-
+ drivers/gpu/drm/amd/amdgpu/soc_v1_0.c              |   15 +-
+ drivers/gpu/drm/amd/amdgpu/tonga_ih.c              |   40 -
+ drivers/gpu/drm/amd/amdgpu/umc_v12_0.c             |  736 +-
+ drivers/gpu/drm/amd/amdgpu/umc_v12_0.h             |   31 -
+ drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c              |   45 -
+ drivers/gpu/drm/amd/amdgpu/vce_v3_0.c              |   69 -
+ drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c              |   12 +-
+ drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c              |   45 +-
+ drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c            |   87 +-
+ drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c            |    6 +-
+ drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c            |   90 +-
+ drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c            |    3 -
+ drivers/gpu/drm/amd/amdgpu/vi.c                    |   23 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_chardev.c           |   17 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c           |  115 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_device.c            |   24 +
+ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.c  |  223 +-
+ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.h  |    2 +
+ drivers/gpu/drm/amd/amdkfd/kfd_events.c            |  224 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_events.h            |    4 -
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c       |   16 +
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h       |    2 +
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c    |   14 -
+ drivers/gpu/drm/amd/amdkfd/kfd_priv.h              |   32 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_process.c           |  147 +-
+ .../gpu/drm/amd/amdkfd/kfd_process_queue_manager.c |    2 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c        |    4 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_svm.c               |    8 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_topology.c          |   11 +-
+ drivers/gpu/drm/amd/amdxcp/amdgpu_xcp_drv.c        |   18 +-
+ drivers/gpu/drm/amd/display/amdgpu_dm/Makefile     |    6 +-
+ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c  | 7259 +-------------------
+ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h  |   90 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_audio.c    |  337 +
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_audio.h    |   58 +
+ .../amd/display/amdgpu_dm/amdgpu_dm_backlight.c    |  737 ++
+ .../amd/display/amdgpu_dm/amdgpu_dm_backlight.h    |   90 +
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_color.c    |  183 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_color.h    |    8 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c  |    1 +
+ .../amd/display/amdgpu_dm/amdgpu_dm_connector.c    | 3715 ++++++++++
+ .../amd/display/amdgpu_dm/amdgpu_dm_connector.h    |  177 +
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c  |  102 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.h  |    6 +
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c |   22 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h |    6 +
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c  |   78 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.c |  949 +++
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.h |   68 +
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c |  191 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h |   17 +
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c  |  136 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.h  |   61 +
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c  | 1762 ++++-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.h  |   84 +
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c  |    4 +
+ .../display/amdgpu_dm/amdgpu_dm_kunit_helpers.h    |    1 +
+ .../amd/display/amdgpu_dm/amdgpu_dm_mst_types.c    |  193 +-
+ .../amd/display/amdgpu_dm/amdgpu_dm_mst_types.h    |   26 +
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c    |  182 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_plane.h    |   60 +
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.c   |  327 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.h   |   61 +
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c  |   51 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.h  |    5 +
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_quirks.c   |    2 +
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_replay.c   |   10 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_services.c |    7 +
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.c   |   34 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.h   |   17 +
+ .../drm/amd/display/amdgpu_dm/tests/.kunitconfig   |    6 +
+ .../gpu/drm/amd/display/amdgpu_dm/tests/Makefile   |   18 +
+ .../display/amdgpu_dm/tests/amdgpu_dm_audio_test.c |  608 ++
+ .../amdgpu_dm/tests/amdgpu_dm_backlight_test.c     | 1950 ++++++
+ .../display/amdgpu_dm/tests/amdgpu_dm_color_test.c |   64 +-
+ .../amdgpu_dm/tests/amdgpu_dm_colorop_test.c       |  140 +-
+ .../amdgpu_dm/tests/amdgpu_dm_connector_test.c     | 3813 ++++++++++
+ .../display/amdgpu_dm/tests/amdgpu_dm_crc_test.c   |  122 +
+ .../display/amdgpu_dm/tests/amdgpu_dm_crtc_test.c  |  523 ++
+ .../display/amdgpu_dm/tests/amdgpu_dm_dmub_test.c  |  582 ++
+ .../display/amdgpu_dm/tests/amdgpu_dm_hdcp_test.c  |  537 +-
+ .../amdgpu_dm/tests/amdgpu_dm_helpers_test.c       | 3907 +++++++++++
+ .../display/amdgpu_dm/tests/amdgpu_dm_irq_test.c   | 4096 +++++++++++
+ .../display/amdgpu_dm/tests/amdgpu_dm_ism_test.c   |  392 +-
+ .../amdgpu_dm/tests/amdgpu_dm_kunit_helpers.c      |  142 +
+ .../amdgpu_dm/tests/amdgpu_dm_kunit_test_helpers.h |   32 +
+ .../amdgpu_dm/tests/amdgpu_dm_mst_types_test.c     | 1429 ++++
+ .../display/amdgpu_dm/tests/amdgpu_dm_plane_test.c | 3284 +++++++++
+ .../amdgpu_dm/tests/amdgpu_dm_pp_smu_test.c        | 2454 +++++++
+ .../display/amdgpu_dm/tests/amdgpu_dm_psr_test.c   |  518 +-
+ .../amdgpu_dm/tests/amdgpu_dm_quirks_test.c        |  103 +
+ .../amdgpu_dm/tests/amdgpu_dm_replay_test.c        |  432 +-
+ .../amdgpu_dm/tests/amdgpu_dm_services_test.c      |  313 +
+ .../amd/display/amdgpu_dm/tests/amdgpu_dm_test.c   |  949 +++
+ .../display/amdgpu_dm/tests/amdgpu_dm_wb_test.c    |  436 ++
+ .../amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.c |   57 +-
+ .../amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.h |    9 +
+ .../amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c   |   16 +-
+ drivers/gpu/drm/amd/display/dc/core/dc.c           |  355 +-
+ .../gpu/drm/amd/display/dc/core/dc_hw_sequencer.c  |  174 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_resource.c  |   11 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_stream.c    |   21 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_surface.c   |   44 +-
+ drivers/gpu/drm/amd/display/dc/dc.h                |  243 +-
+ drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c       |   45 +-
+ drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h       |   43 +-
+ drivers/gpu/drm/amd/display/dc/dc_dp_types.h       |    1 +
+ drivers/gpu/drm/amd/display/dc/dc_helper.c         |  226 -
+ drivers/gpu/drm/amd/display/dc/dc_stream.h         |   29 +
+ drivers/gpu/drm/amd/display/dc/dc_types.h          |   75 +-
+ .../gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c |   11 +-
+ .../gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.h |    5 +-
+ .../gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c |   61 +-
+ .../gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h |    1 +
+ drivers/gpu/drm/amd/display/dc/dce/Makefile        |    4 +-
+ drivers/gpu/drm/amd/display/dc/dce/dce_aux.c       |    9 +-
+ drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c      |   73 +-
+ drivers/gpu/drm/amd/display/dc/dce/dmub_abm_cacp.c |  213 +
+ drivers/gpu/drm/amd/display/dc/dce/dmub_abm_cacp.h |   20 +
+ drivers/gpu/drm/amd/display/dc/dce/dmub_abm_lcd.c  |    4 +-
+ .../drm/amd/display/dc/dce110/dce110_compressor.c  |    2 -
+ .../drm/amd/display/dc/dce110/dce110_mem_input_v.c |    2 -
+ .../drm/amd/display/dc/dce112/dce112_compressor.c  |    8 +-
+ .../gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c |  184 +
+ .../amd/display/dc/dio/dcn10/dcn10_link_encoder.c  |   10 +-
+ .../display/dc/dio/dcn31/dcn31_dio_link_encoder.c  |    4 +
+ .../display/dc/dio/dcn31/dcn31_dio_link_encoder.h  |    2 +
+ drivers/gpu/drm/amd/display/dc/dm_services.h       |    4 -
+ .../gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c   |    2 +-
+ .../amd/display/dc/dml/dcn31/display_mode_vba_31.c |    2 +-
+ .../display/dc/dml/dcn314/display_mode_vba_314.c   |    2 +-
+ .../display/dc/dml2_0/dml21/dml21_wrapper_fpu.c    |    8 +-
+ .../dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h |    6 -
+ .../dml21/inc/bounding_boxes/dcn42b_soc_bb.h       |   40 +-
+ .../dml2_0/dml21/inc/dml_top_soc_parameter_types.h |   13 +
+ .../dml21/src/dml2_core/dml2_core_dcn4_calcs.c     |   41 +-
+ .../dml2_0/dml21/src/dml2_core/dml2_core_factory.c |    2 -
+ .../dml21/src/dml2_core/dml2_core_shared_types.h   |    2 -
+ .../dml21/src/inc/dml2_internal_shared_types.h     |    1 -
+ .../drm/amd/display/dc/dpp/dcn10/dcn10_dpp_cm.c    |    5 -
+ .../amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c    |   25 +-
+ .../drm/amd/display/dc/dwb/dcn30/dcn30_cm_common.h |    4 +
+ .../amd/display/dc/gpio/dce80/hw_translate_dce80.c |    5 +-
+ .../amd/display/dc/gpio/dcn10/hw_translate_dcn10.c |  484 +-
+ .../amd/display/dc/gpio/dcn20/hw_translate_dcn20.c |  434 +-
+ .../amd/display/dc/gpio/dcn21/hw_translate_dcn21.c |  419 +-
+ .../amd/display/dc/gpio/dcn30/hw_translate_dcn30.c |  434 +-
+ .../display/dc/gpio/dcn315/hw_translate_dcn315.c   |  420 +-
+ .../amd/display/dc/gpio/dcn32/hw_translate_dcn32.c |  386 +-
+ .../display/dc/gpio/dcn401/hw_translate_dcn401.c   |  394 +-
+ .../amd/display/dc/gpio/dcn42/hw_translate_dcn42.c |  191 +-
+ drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c |   86 +
+ drivers/gpu/drm/amd/display/dc/gpio/hw_translate.h |   21 +
+ .../drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c |    8 +-
+ .../drm/amd/display/dc/hubbub/dcn35/dcn35_hubbub.c |    2 +-
+ .../amd/display/dc/hubbub/dcn401/dcn401_hubbub.c   |    3 +-
+ .../amd/display/dc/hubbub/dcn401/dcn401_hubbub.h   |    4 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c |    3 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.h |    3 +-
+ .../drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c   |    2 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c |   57 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h |    5 +
+ .../drm/amd/display/dc/hwss/dce110/dce110_hwseq.c  |   20 +-
+ .../drm/amd/display/dc/hwss/dce60/dce60_hwseq.c    |    8 +-
+ .../drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c    |   62 +-
+ .../drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c    |   76 +-
+ .../drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.c  |    2 +-
+ .../drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.c    |   40 +-
+ .../drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.h    |    2 +
+ .../drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c    |   10 +-
+ .../drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c    |   70 +-
+ .../drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c    |   36 +-
+ .../drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c    |   18 +-
+ .../drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c  |  364 +-
+ .../drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.h  |   12 +-
+ .../drm/amd/display/dc/hwss/dcn401/dcn401_init.c   |    2 +
+ .../drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c    |  329 +-
+ .../drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h    |    5 +-
+ drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h |   82 +
+ .../drm/amd/display/dc/hwss/hw_sequencer_private.h |    3 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/abm.h        |    3 +
+ drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h    |   12 +
+ drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h       |    1 +
+ drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h       |    2 +-
+ drivers/gpu/drm/amd/display/dc/inc/link_service.h  |    1 +
+ drivers/gpu/drm/amd/display/dc/inc/reg_helper.h    |   19 -
+ .../drm/amd/display/dc/inc/soc_and_ip_translator.h |   14 -
+ .../gpu/drm/amd/display/dc/link/link_detection.c   |   27 +-
+ drivers/gpu/drm/amd/display/dc/link/link_dpms.c    |  194 +-
+ drivers/gpu/drm/amd/display/dc/link/link_factory.c |    7 +-
+ .../drm/amd/display/dc/link/protocols/link_ddc.c   |   20 +-
+ .../drm/amd/display/dc/link/protocols/link_ddc.h   |    2 +
+ .../display/dc/link/protocols/link_dp_capability.c |   14 +-
+ .../dc/link/protocols/link_dp_irq_handler.c        |    1 -
+ .../dc/link/protocols/link_dp_panel_replay.c       |    3 +-
+ .../dc/link/protocols/link_edp_panel_control.c     |   21 +-
+ .../gpu/drm/amd/display/dc/mpc/dcn20/dcn20_mpc.c   |    4 -
+ .../gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.c   |   51 +-
+ .../gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.c |    5 -
+ .../gpu/drm/amd/display/dc/optc/dcn20/dcn20_optc.c |    5 -
+ .../gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.c |    5 -
+ .../drm/amd/display/dc/optc/dcn314/dcn314_optc.c   |    5 -
+ .../gpu/drm/amd/display/dc/optc/dcn32/dcn32_optc.c |    5 -
+ .../gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c |    5 -
+ .../drm/amd/display/dc/optc/dcn401/dcn401_optc.c   |    5 -
+ .../drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c    |  258 +-
+ .../drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.h    |   22 +-
+ .../amd/display/dc/resource/dcn30/dcn30_resource.c |    1 -
+ .../display/dc/resource/dcn302/dcn302_resource.c   |    1 -
+ .../display/dc/resource/dcn303/dcn303_resource.c   |    1 -
+ .../amd/display/dc/resource/dcn31/dcn31_resource.c |    5 +-
+ .../display/dc/resource/dcn314/dcn314_resource.c   |    4 +-
+ .../display/dc/resource/dcn315/dcn315_resource.c   |    5 +-
+ .../display/dc/resource/dcn316/dcn316_resource.c   |    5 +-
+ .../amd/display/dc/resource/dcn32/dcn32_resource.c |    1 -
+ .../display/dc/resource/dcn321/dcn321_resource.c   |    1 -
+ .../amd/display/dc/resource/dcn35/dcn35_resource.c |    6 +-
+ .../display/dc/resource/dcn351/dcn351_resource.c   |    6 +-
+ .../amd/display/dc/resource/dcn36/dcn36_resource.c |    6 +-
+ .../display/dc/resource/dcn401/dcn401_resource.h   |    3 +-
+ .../amd/display/dc/resource/dcn42/dcn42_resource.c |   11 +-
+ .../amd/display/dc/resource/dcn42/dcn42_resource.h |    1 +
+ .../display/dc/resource/dcn42b/dcn42b_resource.c   |   25 +-
+ .../display/dc/resource/dcn42b/dcn42b_resource.h   |   43 +-
+ .../amd/display/dc/soc_and_ip_translator/Makefile  |    3 +
+ .../dcn401/dcn401_soc_and_ip_translator.c          |   16 +
+ .../dcn42/dcn42_soc_and_ip_translator.c            |   18 +-
+ .../dcn42/dcn42_soc_and_ip_translator.h            |    1 +
+ .../dcn42b/dcn42b_soc_and_ip_translator.c          |   42 +
+ .../dcn42b/dcn42b_soc_and_ip_translator.h          |   17 +
+ .../soc_and_ip_translator/soc_and_ip_translator.c  |    5 +-
+ drivers/gpu/drm/amd/display/dc/sspl/spl_debug.h    |    4 +-
+ drivers/gpu/drm/amd/display/dmub/dmub_srv.h        |    9 +-
+ drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h    |  289 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c    |   55 +-
+ .../drm/amd/display/include/ddc_service_types.h    |    1 +
+ drivers/gpu/drm/amd/display/include/gpio_types.h   |   48 +
+ .../gpu/drm/amd/display/modules/inc/mod_power.h    |   11 +
+ drivers/gpu/drm/amd/display/modules/power/Makefile |    2 +-
+ drivers/gpu/drm/amd/display/modules/power/power.c  |   28 +-
+ .../gpu/drm/amd/display/modules/power/power_abm.c  |  150 +-
+ .../drm/amd/display/modules/power/power_helpers.h  |   10 +
+ .../gpu/drm/amd/display/modules/power/power_psr.c  |    7 +
+ .../drm/amd/display/modules/power/power_replay.c   |   16 +-
+ drivers/gpu/drm/amd/include/amd_shared.h           |    6 -
+ .../amd/include/asic_reg/sdma/sdma_4_4_2_offset.h  |    4 +
+ drivers/gpu/drm/amd/include/kgd_kfd_interface.h    |    3 +
+ drivers/gpu/drm/amd/include/mes_v11_api_def.h      |   52 +-
+ drivers/gpu/drm/amd/include/mes_v12_api_def.h      |    5 +-
+ drivers/gpu/drm/amd/include/soc15_hw_ip.h          |    1 +
+ drivers/gpu/drm/amd/include/v9_structs.h           |    4 +-
+ drivers/gpu/drm/amd/pm/amdgpu_dpm.c                |    7 -
+ drivers/gpu/drm/amd/pm/amdgpu_pm.c                 |  116 +-
+ drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h            |    1 -
+ drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c   |   21 +-
+ .../gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c    |   38 +-
+ .../gpu/drm/amd/pm/powerplay/hwmgr/ppatomfwctrl.c  |   11 +-
+ .../amd/pm/powerplay/hwmgr/process_pptables_v1_0.c |  753 +-
+ .../drm/amd/pm/powerplay/hwmgr/processpptables.c   |  450 +-
+ .../gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c    |   60 +-
+ .../gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c  |   83 +-
+ .../pm/powerplay/hwmgr/vega10_processpptables.c    |  761 +-
+ .../pm/powerplay/hwmgr/vega12_processpptables.c    |    7 +
+ .../pm/powerplay/hwmgr/vega20_processpptables.c    |    7 +
+ drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h       |   20 +
+ .../gpu/drm/amd/pm/powerplay/smumgr/smu7_smumgr.c  |   25 +-
+ drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c          |   45 +-
+ drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h      |   44 +-
+ drivers/gpu/drm/amd/pm/swsmu/inc/smu_v11_0.h       |    2 -
+ drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h       |    6 -
+ drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h       |    4 -
+ drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h       |    4 -
+ drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c  |    8 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c    |    8 +-
+ .../drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c    |    8 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c     |   82 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c    |    1 -
+ drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c |    1 -
+ drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c     |   94 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c   |    7 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c  |  136 -
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c   |  674 --
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h   |   31 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c   |    7 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c     |   85 +-
+ .../gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c   |    9 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c     |   85 +-
+ .../gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c   |    7 +-
+ .../gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h   |    2 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c             |  140 +
+ drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h             |   13 +
+ drivers/gpu/drm/amd/pm/swsmu/smu_internal.h        |    1 -
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c   |   84 +-
+ .../drm/amd/ras/ras_mgr/amdgpu_ras_eeprom_i2c.c    |    2 +-
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c   |   79 +-
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h   |    2 +
+ .../gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.c |   14 +
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_sys.c   |   19 +
+ drivers/gpu/drm/amd/ras/ras_mgr/ras_sys.h          |    3 +
+ drivers/gpu/drm/amd/ras/rascore/ras.h              |    5 +
+ drivers/gpu/drm/amd/ras/rascore/ras_aca_v1_0.c     |   13 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_core.c         |   15 +
+ drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c       |    3 +
+ drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c    |    2 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_mp1.c          |   19 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_mp1.h          |    3 +
+ drivers/gpu/drm/amd/ras/rascore/ras_mp1_v13_0.c    |   13 +
+ drivers/gpu/drm/amd/ras/rascore/ras_process.c      |    5 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_umc.c          |   32 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_umc.h          |    6 +
+ drivers/gpu/drm/amd/ras/rascore/ras_umc_v12_0.h    |    2 +
+ drivers/gpu/drm/bridge/adv7511/adv7511_drv.c       |    2 +-
+ drivers/gpu/drm/bridge/analogix/analogix-anx6345.c |   13 +-
+ drivers/gpu/drm/bridge/analogix/analogix-anx78xx.c |   13 +-
+ drivers/gpu/drm/bridge/analogix/analogix_dp_core.c |   19 +-
+ drivers/gpu/drm/bridge/analogix/analogix_dp_core.h |    6 +-
+ drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c  |   32 +-
+ drivers/gpu/drm/bridge/analogix/anx7625.c          |    2 +-
+ drivers/gpu/drm/bridge/aux-bridge.c                |    4 +
+ drivers/gpu/drm/bridge/aux-hpd-bridge.c            |    4 +
+ drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c     |    9 +-
+ .../gpu/drm/bridge/cadence/cdns-mhdp8546-core.c    |    8 +-
+ drivers/gpu/drm/bridge/chipone-icn6211.c           |    2 +-
+ drivers/gpu/drm/bridge/chrontel-ch7033.c           |   13 +-
+ drivers/gpu/drm/bridge/cros-ec-anx7688.c           |    4 +
+ drivers/gpu/drm/bridge/display-connector.c         |   38 +-
+ drivers/gpu/drm/bridge/fsl-ldb.c                   |    2 +-
+ drivers/gpu/drm/bridge/imx/imx8mp-hdmi-pvi.c       |    2 +-
+ drivers/gpu/drm/bridge/imx/imx8qm-ldb.c            |    2 +-
+ drivers/gpu/drm/bridge/imx/imx8qxp-ldb.c           |    2 +-
+ .../gpu/drm/bridge/imx/imx8qxp-pixel-combiner.c    |    2 +-
+ drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c    |    2 +-
+ drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c       |    2 +-
+ drivers/gpu/drm/bridge/inno-hdmi.c                 |    2 +-
+ drivers/gpu/drm/bridge/ite-it6263.c                |    2 +-
+ drivers/gpu/drm/bridge/ite-it6505.c                |    2 +-
+ drivers/gpu/drm/bridge/ite-it66121.c               |    2 +-
+ drivers/gpu/drm/bridge/lontium-lt8713sx.c          |    4 +
+ drivers/gpu/drm/bridge/lontium-lt8912b.c           |    8 +-
+ drivers/gpu/drm/bridge/lontium-lt9211.c            |    2 +-
+ drivers/gpu/drm/bridge/lontium-lt9611.c            |    2 +-
+ drivers/gpu/drm/bridge/lontium-lt9611uxc.c         |    3 +
+ drivers/gpu/drm/bridge/lvds-codec.c                |   12 +-
+ .../drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c   |    3 +
+ drivers/gpu/drm/bridge/microchip-lvds.c            |    3 +
+ drivers/gpu/drm/bridge/nwl-dsi.c                   |    2 +-
+ drivers/gpu/drm/bridge/nxp-ptn3460.c               |   17 +-
+ drivers/gpu/drm/bridge/of-display-mode-bridge.c    |    6 +-
+ drivers/gpu/drm/bridge/panel.c                     |    2 +-
+ drivers/gpu/drm/bridge/parade-ps8622.c             |   18 +-
+ drivers/gpu/drm/bridge/parade-ps8640.c             |    2 +-
+ drivers/gpu/drm/bridge/samsung-dsim.c              |    4 +-
+ drivers/gpu/drm/bridge/sii902x.c                   |    6 +-
+ drivers/gpu/drm/bridge/sii9234.c                   |   39 +-
+ drivers/gpu/drm/bridge/sil-sii8620.c               |    4 +
+ drivers/gpu/drm/bridge/simple-bridge.c             |   13 +-
+ drivers/gpu/drm/bridge/ssd2825.c                   |    2 +-
+ drivers/gpu/drm/bridge/synopsys/dw-dp.c            |    8 +-
+ drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c       |    3 +-
+ drivers/gpu/drm/bridge/synopsys/dw-hdmi.c          |    2 +-
+ drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c      |    2 +-
+ drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi2.c     |    2 +-
+ drivers/gpu/drm/bridge/tc358762.c                  |    2 +-
+ drivers/gpu/drm/bridge/tc358764.c                  |   13 +-
+ drivers/gpu/drm/bridge/tc358767.c                  |    6 +-
+ drivers/gpu/drm/bridge/tc358768.c                  |    2 +-
+ drivers/gpu/drm/bridge/tc358775.c                  |    2 +-
+ drivers/gpu/drm/bridge/tda998x_drv.c               |   13 +-
+ drivers/gpu/drm/bridge/thc63lvd1024.c              |   14 +-
+ drivers/gpu/drm/bridge/ti-dlpc3433.c               |    2 +-
+ drivers/gpu/drm/bridge/ti-sn65dsi83.c              |    3 +-
+ drivers/gpu/drm/bridge/ti-sn65dsi86.c              |    2 +-
+ drivers/gpu/drm/bridge/ti-tdp158.c                 |    2 +-
+ drivers/gpu/drm/bridge/ti-tfp410.c                 |   12 +-
+ drivers/gpu/drm/bridge/ti-tpd12s015.c              |    4 +
+ drivers/gpu/drm/bridge/waveshare-dsi.c             |   14 +-
+ drivers/gpu/drm/clients/drm_log.c                  |   10 +-
+ drivers/gpu/drm/display/drm_bridge_connector.c     |   45 +-
+ drivers/gpu/drm/display/drm_hdmi_state_helper.c    |   68 +-
+ drivers/gpu/drm/drm_atomic.c                       |   82 +-
+ drivers/gpu/drm/drm_atomic_helper.c                |   86 +
+ drivers/gpu/drm/drm_atomic_state_helper.c          |  154 +-
+ drivers/gpu/drm/drm_atomic_uapi.c                  |    4 +
+ drivers/gpu/drm/drm_blend.c                        |   20 +-
+ drivers/gpu/drm/drm_bridge.c                       |  187 +-
+ drivers/gpu/drm/drm_buddy.c                        |   30 +-
+ drivers/gpu/drm/drm_colorop.c                      |   41 +-
+ drivers/gpu/drm/drm_connector.c                    |  192 +-
+ drivers/gpu/drm/drm_displayid_internal.h           |   24 +
+ drivers/gpu/drm/drm_draw_internal.h                |    7 -
+ drivers/gpu/drm/drm_drv.c                          |    4 +-
+ drivers/gpu/drm/drm_dumb_buffers.c                 |    1 -
+ drivers/gpu/drm/drm_edid.c                         |   80 +-
+ drivers/gpu/drm/drm_exec.c                         |    2 +-
+ drivers/gpu/drm/drm_fb_helper.c                    |   11 +-
+ drivers/gpu/drm/drm_gem.c                          |    3 +-
+ drivers/gpu/drm/drm_gem_shmem_helper.c             |   22 +-
+ drivers/gpu/drm/drm_gpusvm.c                       |  243 +-
+ drivers/gpu/drm/drm_gpuvm.c                        |   44 -
+ drivers/gpu/drm/drm_mode_config.c                  |  210 +-
+ drivers/gpu/drm/drm_modes.c                        |   23 +
+ drivers/gpu/drm/drm_of.c                           |   46 +-
+ drivers/gpu/drm/drm_panel.c                        |   44 +-
+ drivers/gpu/drm/drm_panic.c                        |    6 +-
+ drivers/gpu/drm/drm_print.c                        |    4 +-
+ drivers/gpu/drm/drm_syncobj.c                      |    2 +-
+ drivers/gpu/drm/drm_writeback.c                    |    6 +
+ drivers/gpu/drm/etnaviv/etnaviv_gpu.c              |   23 +-
+ drivers/gpu/drm/exynos/exynos_drm_mic.c            |   14 +-
+ drivers/gpu/drm/gma500/psb_drv.c                   |   56 +-
+ drivers/gpu/drm/hisilicon/hibmc/Kconfig            |    4 +-
+ drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c     |  114 +-
+ drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c    |   78 +-
+ drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h    |    5 +
+ drivers/gpu/drm/hyperv/hyperv_drm.h                |   16 +-
+ drivers/gpu/drm/hyperv/hyperv_drm_drv.c            |   92 +-
+ drivers/gpu/drm/hyperv/hyperv_drm_modeset.c        |  110 +-
+ drivers/gpu/drm/hyperv/hyperv_drm_proto.c          |   70 +-
+ drivers/gpu/drm/i915/Makefile                      |    5 +-
+ drivers/gpu/drm/i915/display/g4x_dp.c              |   15 +-
+ drivers/gpu/drm/i915/display/g4x_hdmi.c            |    4 +-
+ drivers/gpu/drm/i915/display/icl_dsi.c             |    5 +-
+ drivers/gpu/drm/i915/display/intel_alpm.c          |   15 +-
+ drivers/gpu/drm/i915/display/intel_bios.c          |   62 +-
+ drivers/gpu/drm/i915/display/intel_bw.c            |  168 +-
+ drivers/gpu/drm/i915/display/intel_cdclk.c         |   54 +-
+ drivers/gpu/drm/i915/display/intel_cmtg.c          |  280 +-
+ drivers/gpu/drm/i915/display/intel_cmtg.h          |   19 +-
+ drivers/gpu/drm/i915/display/intel_cmtg_regs.h     |   24 +-
+ drivers/gpu/drm/i915/display/intel_color.c         |  115 +-
+ drivers/gpu/drm/i915/display/intel_crt.c           |    9 +-
+ drivers/gpu/drm/i915/display/intel_crtc.c          |    2 +-
+ drivers/gpu/drm/i915/display/intel_cursor.c        |    2 +-
+ drivers/gpu/drm/i915/display/intel_cx0_phy.c       |   11 +
+ drivers/gpu/drm/i915/display/intel_cx0_phy.h       |    1 -
+ drivers/gpu/drm/i915/display/intel_ddi.c           |   29 +-
+ drivers/gpu/drm/i915/display/intel_de.c            |   40 +-
+ drivers/gpu/drm/i915/display/intel_display.c       |  170 +-
+ drivers/gpu/drm/i915/display/intel_display.h       |    7 +
+ drivers/gpu/drm/i915/display/intel_display_core.h  |   14 +-
+ .../gpu/drm/i915/display/intel_display_debugfs.c   |    2 +
+ .../gpu/drm/i915/display/intel_display_device.c    |   14 +
+ .../gpu/drm/i915/display/intel_display_device.h    |    3 +-
+ .../gpu/drm/i915/display/intel_display_driver.c    |  131 +-
+ .../gpu/drm/i915/display/intel_display_driver.h    |    9 +-
+ drivers/gpu/drm/i915/display/intel_display_irq.c   |   19 +
+ drivers/gpu/drm/i915/display/intel_display_irq.h   |    2 +
+ .../gpu/drm/i915/display/intel_display_limits.h    |    2 +
+ drivers/gpu/drm/i915/display/intel_display_power.c |  185 +-
+ drivers/gpu/drm/i915/display/intel_display_power.h |   40 +
+ .../drm/i915/display/intel_display_power_well.c    |   76 +-
+ .../drm/i915/display/intel_display_power_well.h    |    1 +
+ drivers/gpu/drm/i915/display/intel_display_regs.h  |   46 +-
+ drivers/gpu/drm/i915/display/intel_display_types.h |   57 +-
+ drivers/gpu/drm/i915/display/intel_dmc.c           |   15 +-
+ drivers/gpu/drm/i915/display/intel_dmc_regs.h      |    2 +
+ drivers/gpu/drm/i915/display/intel_dmc_wl.c        |    2 +-
+ drivers/gpu/drm/i915/display/intel_dp.c            |  539 +-
+ drivers/gpu/drm/i915/display/intel_dp.h            |   16 +-
+ drivers/gpu/drm/i915/display/intel_dp_link_caps.c  |  675 ++
+ drivers/gpu/drm/i915/display/intel_dp_link_caps.h  |   47 +
+ .../gpu/drm/i915/display/intel_dp_link_training.c  | 1093 ++-
+ .../gpu/drm/i915/display/intel_dp_link_training.h  |   15 +
+ drivers/gpu/drm/i915/display/intel_dp_mst.c        |    8 +-
+ drivers/gpu/drm/i915/display/intel_dp_test.c       |    7 +-
+ drivers/gpu/drm/i915/display/intel_dp_tunnel.c     |    6 +-
+ drivers/gpu/drm/i915/display/intel_dsb.c           |   32 +-
+ drivers/gpu/drm/i915/display/intel_dvo.c           |    5 +-
+ drivers/gpu/drm/i915/display/intel_gmbus.c         |  148 +-
+ drivers/gpu/drm/i915/display/intel_gmbus.h         |   22 +-
+ drivers/gpu/drm/i915/display/intel_hdmi.c          |   71 +-
+ drivers/gpu/drm/i915/display/intel_hdmi.h          |    2 +
+ drivers/gpu/drm/i915/display/intel_initial_plane.c |    2 +-
+ drivers/gpu/drm/i915/display/intel_lpe_audio.c     |    4 +-
+ drivers/gpu/drm/i915/display/intel_lt_phy.c        |   14 +-
+ drivers/gpu/drm/i915/display/intel_lvds.c          |    5 +-
+ drivers/gpu/drm/i915/display/intel_panel.c         |  170 +-
+ drivers/gpu/drm/i915/display/intel_panel.h         |    6 +-
+ drivers/gpu/drm/i915/display/intel_parent.c        |    6 +-
+ drivers/gpu/drm/i915/display/intel_parent.h        |    4 +-
+ drivers/gpu/drm/i915/display/intel_plane.c         |   54 +-
+ drivers/gpu/drm/i915/display/intel_plane.h         |    5 +-
+ drivers/gpu/drm/i915/display/intel_psr.c           |  264 +-
+ drivers/gpu/drm/i915/display/intel_psr.h           |    1 +
+ drivers/gpu/drm/i915/display/intel_psr_regs.h      |    1 +
+ drivers/gpu/drm/i915/display/intel_sdvo.c          |    7 +-
+ drivers/gpu/drm/i915/display/intel_tv.c            |    5 +-
+ drivers/gpu/drm/i915/display/intel_vrr.c           |   29 +-
+ drivers/gpu/drm/i915/display/intel_vrr.h           |    4 +-
+ drivers/gpu/drm/i915/display/vlv_dsi.c             |    5 +-
+ drivers/gpu/drm/i915/gem/i915_gem_object.h         |    7 -
+ drivers/gpu/drm/i915/gem/i915_gem_pages.c          |  128 -
+ drivers/gpu/drm/i915/gem/i915_gem_panic.c          |  147 +
+ drivers/gpu/drm/i915/gem/i915_gem_panic.h          |   11 +
+ drivers/gpu/drm/i915/gt/intel_gsc.c                |    2 +-
+ drivers/gpu/drm/i915/gt/intel_reset.c              |    2 +-
+ .../gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c    |    2 +-
+ drivers/gpu/drm/i915/gvt/edid.c                    |   14 +-
+ drivers/gpu/drm/i915/i915_active.c                 |    2 +-
+ drivers/gpu/drm/i915/i915_driver.c                 |  123 +-
+ drivers/gpu/drm/i915/i915_panic.c                  |   35 -
+ drivers/gpu/drm/i915/i915_panic.h                  |    9 -
+ drivers/gpu/drm/imagination/pvr_drv.c              |    2 +-
+ drivers/gpu/drm/imx/dc/dc-ic.c                     |    6 +-
+ drivers/gpu/drm/imx/ipuv3/parallel-display.c       |    2 +-
+ drivers/gpu/drm/ingenic/ingenic-drm-drv.c          |    2 +-
+ drivers/gpu/drm/lima/lima_device.c                 |   12 +-
+ drivers/gpu/drm/loongson/lsdc_drv.c                |    4 +-
+ drivers/gpu/drm/mcde/mcde_dsi.c                    |    3 +
+ drivers/gpu/drm/mediatek/mtk_dp.c                  |    2 +-
+ drivers/gpu/drm/mediatek/mtk_dpi.c                 |   12 +-
+ drivers/gpu/drm/mediatek/mtk_dsi.c                 |    2 +-
+ drivers/gpu/drm/mediatek/mtk_hdmi.c                |    2 +-
+ drivers/gpu/drm/mediatek/mtk_hdmi_v2.c             |    2 +-
+ drivers/gpu/drm/meson/meson_encoder_cvbs.c         |    2 +-
+ drivers/gpu/drm/meson/meson_encoder_dsi.c          |    2 +-
+ drivers/gpu/drm/meson/meson_encoder_hdmi.c         |    2 +-
+ drivers/gpu/drm/mgag200/mgag200_drv.c              |   24 +-
+ drivers/gpu/drm/msm/dp/dp_drm.c                    |    4 +-
+ drivers/gpu/drm/msm/dsi/dsi_host.c                 |   25 +-
+ drivers/gpu/drm/msm/dsi/dsi_manager.c              |   13 +-
+ drivers/gpu/drm/msm/hdmi/hdmi_bridge.c             |    2 +-
+ drivers/gpu/drm/msm/msm_drv.c                      |    2 -
+ drivers/gpu/drm/mxsfb/lcdif_drv.c                  |    2 +-
+ drivers/gpu/drm/mxsfb/lcdif_kms.c                  |   18 +-
+ drivers/gpu/drm/mxsfb/lcdif_regs.h                 |    1 +
+ drivers/gpu/drm/nouveau/nouveau_drm.c              |   35 +-
+ drivers/gpu/drm/nouveau/nouveau_gem.c              |   11 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c   |    4 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h    |    2 +
+ .../gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c |    8 +-
+ .../gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/fbsr.c |    1 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c |   39 +-
+ drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h |    1 +
+ drivers/gpu/drm/omapdrm/dss/dpi.c                  |   14 +-
+ drivers/gpu/drm/omapdrm/dss/dsi.c                  |   14 +-
+ drivers/gpu/drm/omapdrm/dss/hdmi4.c                |    2 +-
+ drivers/gpu/drm/omapdrm/dss/hdmi5.c                |    2 +-
+ drivers/gpu/drm/omapdrm/dss/sdi.c                  |   14 +-
+ drivers/gpu/drm/omapdrm/dss/venc.c                 |   14 +-
+ drivers/gpu/drm/panel/Kconfig                      |   56 +
+ drivers/gpu/drm/panel/Makefile                     |    5 +
+ drivers/gpu/drm/panel/panel-anbernic-td4310.c      |  258 +
+ .../gpu/drm/panel/panel-boe-th101mb31ig002-28a.c   |    3 +-
+ drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c     |    3 +-
+ drivers/gpu/drm/panel/panel-chipone-icna35xx.c     |  423 ++
+ drivers/gpu/drm/panel/panel-chipwealth-ch13726a.c  |    3 +-
+ drivers/gpu/drm/panel/panel-edp.c                  |   24 +-
+ drivers/gpu/drm/panel/panel-elida-kd35t133.c       |    3 +-
+ drivers/gpu/drm/panel/panel-focaltech-ota7290b.c   |    3 +-
+ drivers/gpu/drm/panel/panel-himax-hx83102.c        |    3 +-
+ drivers/gpu/drm/panel/panel-himax-hx83121a.c       |   39 +-
+ drivers/gpu/drm/panel/panel-himax-hx8394.c         |    3 +-
+ drivers/gpu/drm/panel/panel-ilitek-ili7807s.c      |  285 +
+ drivers/gpu/drm/panel/panel-ilitek-ili9488.c       |  290 +
+ drivers/gpu/drm/panel/panel-ilitek-ili9806e-dsi.c  |    3 +-
+ drivers/gpu/drm/panel/panel-ilitek-ili9881c.c      |    3 +-
+ drivers/gpu/drm/panel/panel-ilitek-ili9882t.c      |    3 +-
+ drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c   |    2 +-
+ drivers/gpu/drm/panel/panel-lvds.c                 |    2 +-
+ drivers/gpu/drm/panel/panel-novatek-nt36523.c      |    3 +-
+ drivers/gpu/drm/panel/panel-novatek-nt36672a.c     |  522 +-
+ drivers/gpu/drm/panel/panel-novatek-nt37801.c      |  106 +-
+ drivers/gpu/drm/panel/panel-renesas-r63419.c       |  351 +
+ drivers/gpu/drm/panel/panel-samsung-s6d16d0.c      |   10 +-
+ drivers/gpu/drm/panel/panel-simple.c               |    2 +-
+ drivers/gpu/drm/panel/panel-sitronix-st7701.c      |    3 +-
+ drivers/gpu/drm/panel/panel-sitronix-st7703.c      |    3 +-
+ drivers/gpu/drm/panel/panel-sitronix-st7789v.c     |    3 +-
+ drivers/gpu/drm/panel/panel-visionox-vtdr6130.c    |   42 +-
+ drivers/gpu/drm/panfrost/panfrost_drv.c            |    1 +
+ drivers/gpu/drm/panthor/panthor_device.h           |    3 +
+ drivers/gpu/drm/panthor/panthor_drv.c              |   14 +-
+ drivers/gpu/drm/panthor/panthor_gem.c              |   18 +
+ drivers/gpu/drm/panthor/panthor_gem.h              |    2 +
+ drivers/gpu/drm/panthor/panthor_mmu.c              |  259 +-
+ drivers/gpu/drm/panthor/panthor_sched.c            |    1 -
+ drivers/gpu/drm/qxl/qxl_display.c                  |   12 +-
+ drivers/gpu/drm/qxl/qxl_drv.c                      |   15 +-
+ drivers/gpu/drm/radeon/r600_dpm.c                  |   21 +-
+ drivers/gpu/drm/radeon/radeon_bios.c               |   64 +-
+ drivers/gpu/drm/renesas/rcar-du/rcar_du_drv.c      |   20 +-
+ drivers/gpu/drm/renesas/rcar-du/rcar_lvds.c        |    2 +-
+ drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c    |    2 +-
+ drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c     |    2 +-
+ drivers/gpu/drm/rockchip/analogix_dp-rockchip.c    |   40 +-
+ drivers/gpu/drm/rockchip/cdn-dp-core.c             |   11 +-
+ drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c    |    9 +-
+ drivers/gpu/drm/rockchip/dw-mipi-dsi2-rockchip.c   |    8 +-
+ drivers/gpu/drm/rockchip/dw_dp-rockchip.c          |   16 +-
+ drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c        |  228 +-
+ drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c     |  112 +-
+ drivers/gpu/drm/rockchip/rk3066_hdmi.c             |   10 +-
+ drivers/gpu/drm/rockchip/rockchip_drm_drv.h        |    4 +
+ drivers/gpu/drm/rockchip/rockchip_drm_vop2.c       |  176 +-
+ drivers/gpu/drm/rockchip/rockchip_drm_vop2.h       |    2 +-
+ drivers/gpu/drm/rockchip/rockchip_lvds.c           |   11 +-
+ drivers/gpu/drm/rockchip/rockchip_rgb.c            |    8 +-
+ drivers/gpu/drm/rockchip/rockchip_vop2_reg.c       |   46 +-
+ drivers/gpu/drm/scheduler/sched_entity.c           |   11 +-
+ drivers/gpu/drm/scheduler/sched_main.c             |    9 -
+ drivers/gpu/drm/scheduler/tests/tests_basic.c      |  359 +
+ drivers/gpu/drm/scheduler/tests/tests_scheduler.c  |    2 +-
+ drivers/gpu/drm/solomon/ssd130x.c                  |  360 +-
+ drivers/gpu/drm/sti/sti_dvo.c                      |   20 +-
+ drivers/gpu/drm/sti/sti_hda.c                      |   20 +-
+ drivers/gpu/drm/sti/sti_hdmi.c                     |   20 +-
+ drivers/gpu/drm/stm/lvds.c                         |    2 +-
+ drivers/gpu/drm/sun4i/sun4i_backend.c              |    2 +-
+ drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c             |   11 +-
+ drivers/gpu/drm/sun4i/sun4i_lvds.c                 |   10 +-
+ drivers/gpu/drm/sun4i/sun4i_rgb.c                  |   10 +-
+ drivers/gpu/drm/sun4i/sun4i_tv.c                   |   10 +-
+ drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c             |   10 +-
+ drivers/gpu/drm/sun4i/sun8i_dw_hdmi.c              |    8 +-
+ drivers/gpu/drm/sysfb/Kconfig                      |    7 +-
+ drivers/gpu/drm/sysfb/simpledrm.c                  |  120 +-
+ drivers/gpu/drm/tegra/drm.c                        |    2 +
+ drivers/gpu/drm/tegra/dsi.c                        |  126 +-
+ drivers/gpu/drm/tegra/dsi.h                        |   10 +
+ drivers/gpu/drm/tests/Makefile                     |    3 +-
+ drivers/gpu/drm/tests/drm_bridge_test.c            | 1017 ++-
+ drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c |  347 +-
+ drivers/gpu/drm/tests/drm_kunit_edid.c             |  995 +++
+ drivers/gpu/drm/tests/drm_kunit_edid.h             |  985 +--
+ drivers/gpu/drm/tidss/tidss_crtc.c                 |   17 +-
+ drivers/gpu/drm/tidss/tidss_encoder.c              |    2 +-
+ drivers/gpu/drm/tidss/tidss_oldi.c                 |    2 +-
+ drivers/gpu/drm/tidss/tidss_plane.c                |    2 +-
+ drivers/gpu/drm/tiny/gm12u320.c                    |    5 +-
+ drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c   |    2 +-
+ drivers/gpu/drm/ttm/ttm_bo.c                       |    2 +-
+ drivers/gpu/drm/ttm/ttm_pool.c                     |   37 +-
+ drivers/gpu/drm/v3d/Kconfig                        |    1 +
+ drivers/gpu/drm/v3d/v3d_drv.c                      |   20 +-
+ drivers/gpu/drm/v3d/v3d_drv.h                      |   83 +-
+ drivers/gpu/drm/v3d/v3d_gem.c                      |    7 +-
+ drivers/gpu/drm/v3d/v3d_irq.c                      |    7 +-
+ drivers/gpu/drm/v3d/v3d_perfmon.c                  |  189 +-
+ drivers/gpu/drm/v3d/v3d_power.c                    |    4 +
+ drivers/gpu/drm/v3d/v3d_sched.c                    |  103 +-
+ drivers/gpu/drm/v3d/v3d_submit.c                   |  966 +--
+ drivers/gpu/drm/vc4/vc4_dsi.c                      |    2 +-
+ drivers/gpu/drm/vc4/vc4_hdmi.c                     |    2 +-
+ drivers/gpu/drm/verisilicon/vs_bridge.c            |    4 +-
+ drivers/gpu/drm/virtio/virtgpu_drv.c               |  101 +-
+ drivers/gpu/drm/virtio/virtgpu_drv.h               |   26 +-
+ drivers/gpu/drm/virtio/virtgpu_kms.c               |   71 +-
+ drivers/gpu/drm/virtio/virtgpu_object.c            |   90 +-
+ drivers/gpu/drm/virtio/virtgpu_prime.c             |   45 +-
+ drivers/gpu/drm/virtio/virtgpu_vq.c                |   50 +-
+ drivers/gpu/drm/virtio/virtgpu_vram.c              |    5 +-
+ drivers/gpu/drm/xe/Kconfig.profile                 |   71 +-
+ drivers/gpu/drm/xe/Makefile                        |    2 +
+ drivers/gpu/drm/xe/display/xe_display.c            |  244 +-
+ drivers/gpu/drm/xe/display/xe_display.h            |   10 +-
+ drivers/gpu/drm/xe/display/xe_panic.c              |   26 +-
+ drivers/gpu/drm/xe/tests/Makefile                  |    1 +
+ drivers/gpu/drm/xe/tests/xe_pci.c                  |   53 +-
+ drivers/gpu/drm/xe/tests/xe_rtp_tables_test.c      |  112 +
+ drivers/gpu/drm/xe/tests/xe_rtp_test.c             |   52 +
+ drivers/gpu/drm/xe/tests/xe_wa_test.c              |    3 -
+ drivers/gpu/drm/xe/xe_bo.c                         |    2 +-
+ drivers/gpu/drm/xe/xe_debugfs.c                    |    1 -
+ drivers/gpu/drm/xe/xe_device.c                     |   38 +-
+ drivers/gpu/drm/xe/xe_device.h                     |   17 +-
+ drivers/gpu/drm/xe/xe_device_types.h               |   10 +-
+ drivers/gpu/drm/xe/xe_drm_client.c                 |    5 +-
+ drivers/gpu/drm/xe/xe_drm_ras.c                    |   41 +-
+ drivers/gpu/drm/xe/xe_eu_stall.c                   |    5 +-
+ drivers/gpu/drm/xe/xe_exec_queue_types.h           |    2 -
+ drivers/gpu/drm/xe/xe_execlist.c                   |    7 -
+ drivers/gpu/drm/xe/xe_ggtt.c                       |    8 +-
+ drivers/gpu/drm/xe/xe_gt.c                         |   14 +-
+ drivers/gpu/drm/xe/xe_gt_mcr.c                     |   19 +-
+ drivers/gpu/drm/xe/xe_guc_submit.c                 |   18 +-
+ drivers/gpu/drm/xe/xe_guc_tlb_inval.c              |    6 -
+ drivers/gpu/drm/xe/xe_hw_engine.c                  |  218 +-
+ drivers/gpu/drm/xe/xe_hw_error.c                   |   23 +-
+ drivers/gpu/drm/xe/xe_i2c.c                        |    3 -
+ drivers/gpu/drm/xe/xe_lrc.c                        |    8 +-
+ drivers/gpu/drm/xe/xe_mmio.c                       |  100 +-
+ drivers/gpu/drm/xe/xe_module.c                     |    3 -
+ drivers/gpu/drm/xe/xe_module.h                     |    1 -
+ drivers/gpu/drm/xe/xe_oa.c                         |   25 +-
+ drivers/gpu/drm/xe/xe_observation.c                |   32 +-
+ drivers/gpu/drm/xe/xe_observation.h                |    3 +-
+ drivers/gpu/drm/xe/xe_pci.c                        |  176 +-
+ drivers/gpu/drm/xe/xe_pci_error.c                  |  122 +
+ drivers/gpu/drm/xe/xe_pci_error.h                  |   13 +
+ drivers/gpu/drm/xe/xe_pci_types.h                  |    1 +
+ drivers/gpu/drm/xe/xe_pcode.c                      |    8 +-
+ drivers/gpu/drm/xe/xe_pcode.h                      |    2 +-
+ drivers/gpu/drm/xe/xe_pt.c                         |    8 +-
+ drivers/gpu/drm/xe/xe_pxp.c                        |   35 +-
+ drivers/gpu/drm/xe/xe_query.c                      |    7 +-
+ drivers/gpu/drm/xe/xe_ras.c                        |  260 +
+ drivers/gpu/drm/xe/xe_ras.h                        |    5 +
+ drivers/gpu/drm/xe/xe_ras_types.h                  |   51 +
+ drivers/gpu/drm/xe/xe_reg_whitelist.c              |    5 +-
+ drivers/gpu/drm/xe/xe_reg_whitelist.h              |    4 +
+ drivers/gpu/drm/xe/xe_rtp.c                        |   10 +-
+ drivers/gpu/drm/xe/xe_rtp.h                        |   19 +
+ drivers/gpu/drm/xe/xe_rtp_types.h                  |   33 +-
+ drivers/gpu/drm/xe/xe_sched_job.c                  |    2 +-
+ drivers/gpu/drm/xe/xe_step.c                       |   38 +-
+ drivers/gpu/drm/xe/xe_step.h                       |    8 +-
+ drivers/gpu/drm/xe/xe_svm.c                        |   68 +-
+ drivers/gpu/drm/xe/xe_svm.h                        |   14 +-
+ drivers/gpu/drm/xe/xe_sysctrl_mailbox.c            |   28 +
+ drivers/gpu/drm/xe/xe_sysctrl_mailbox.h            |    3 +
+ drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h      |    4 +
+ drivers/gpu/drm/xe/xe_tile.c                       |    4 +-
+ drivers/gpu/drm/xe/xe_tuning.c                     |    3 +-
+ drivers/gpu/drm/xe/xe_tuning.h                     |    6 +
+ drivers/gpu/drm/xe/xe_userptr.c                    |    7 +-
+ drivers/gpu/drm/xe/xe_vm.c                         |    4 +-
+ drivers/gpu/drm/xe/xe_vm_madvise.c                 |   19 +-
+ drivers/gpu/drm/xe/xe_wa.c                         |   15 +-
+ drivers/gpu/drm/xe/xe_wa.h                         |    7 +
+ drivers/gpu/drm/xe/xe_wa_oob.rules                 |    1 +
+ drivers/gpu/drm/xlnx/zynqmp_dp.c                   |    2 +-
+ include/drm/bridge/analogix_dp.h                   |   13 +-
+ include/drm/bridge/dw_dp.h                         |    1 +
+ include/drm/display/drm_dp.h                       |   13 +-
+ include/drm/display/drm_hdmi_state_helper.h        |    4 +-
+ include/drm/drm_atomic.h                           |    5 +-
+ include/drm/drm_atomic_helper.h                    |    7 +
+ include/drm/drm_atomic_state_helper.h              |   18 +-
+ include/drm/drm_bridge.h                           |  149 +-
+ include/drm/drm_colorop.h                          |    2 +
+ include/drm/drm_connector.h                        |  118 +
+ include/drm/drm_crtc.h                             |   16 +
+ include/drm/drm_debugfs.h                          |    2 +-
+ include/drm/drm_drv.h                              |    6 -
+ include/drm/drm_gem.h                              |    3 -
+ include/drm/drm_gem_shmem_helper.h                 |    4 +
+ include/drm/drm_gpusvm.h                           |   67 +-
+ include/drm/drm_gpuvm.h                            |    4 +-
+ include/drm/drm_managed.h                          |    2 +-
+ include/drm/drm_mipi_dsi.h                         |    2 +
+ include/drm/drm_mode_config.h                      |    1 +
+ include/drm/drm_modes.h                            |    1 +
+ include/drm/drm_of.h                               |   11 +
+ include/drm/drm_panel.h                            |    8 -
+ include/drm/drm_plane.h                            |   16 +
+ include/drm/drm_print.h                            |    2 +-
+ include/drm/gpu_scheduler.h                        |    3 +-
+ include/drm/intel/display_parent_interface.h       |    4 +-
+ include/drm/intel/mchbar_regs.h                    |    2 -
+ include/drm/intel/pciids.h                         |    5 +-
+ include/linux/dma-fence-unwrap.h                   |    6 +-
+ include/linux/dma-fence.h                          |    4 +-
+ include/linux/font.h                               |    3 +
+ include/linux/gpu_buddy.h                          |   15 +
+ include/linux/perf_event.h                         |   31 +-
+ include/linux/sysfb.h                              |    4 +-
+ include/uapi/drm/amdgpu_drm.h                      |   21 +
+ include/uapi/drm/drm_mode.h                        |    1 +
+ include/uapi/drm/ethosu_accel.h                    |   60 +-
+ include/uapi/drm/panthor_drm.h                     |   26 +-
+ include/uapi/drm/xe_drm.h                          |   11 +-
+ kernel/cgroup/dmem.c                               |   15 +
+ kernel/events/core.c                               |   18 +
+ lib/fonts/fonts.c                                  |   31 +
+ 951 files changed, 64703 insertions(+), 29987 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/display/panel/anbernic,td4310.yaml
+ create mode 100644 Documentation/devicetree/bindings/display/panel/chipone,icna3512.yaml
+ create mode 100644 Documentation/devicetree/bindings/display/panel/ilitek,ili7807s.yaml
+ create mode 100644 Documentation/devicetree/bindings/display/panel/ilitek,ili9488.yaml
+ create mode 100644 Documentation/devicetree/bindings/display/panel/renesas,r63419.yaml
+ create mode 100644 Documentation/gpu/intel-display/dp-link-training.rst
+ create mode 100644 drivers/accel/ethosu/ethosu_perfmon.c
+ delete mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c
+ delete mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_aca.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
+ rename drivers/gpu/drm/amd/{display/modules/power/power_helpers.c => amdgpu/amdgpu_video_codecs.h} (63%)
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_wb.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_wb.h
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_audio.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_audio.h
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.h
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.h
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.h
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.h
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.h
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_audio_test.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_backlight_test.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_connector_test.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_crtc_test.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_dmub_test.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_helpers_test.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_irq_test.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_kunit_helpers.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_kunit_test_helpers.h
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_mst_types_test.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_plane_test.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_pp_smu_test.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_quirks_test.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_services_test.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_test.c
+ create mode 100644 drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_wb_test.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dce/dmub_abm_cacp.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dce/dmub_abm_cacp.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42b/dcn42b_soc_and_ip_translator.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42b/dcn42b_soc_and_ip_translator.h
+ create mode 100644 drivers/gpu/drm/i915/display/intel_dp_link_caps.c
+ create mode 100644 drivers/gpu/drm/i915/display/intel_dp_link_caps.h
+ create mode 100644 drivers/gpu/drm/i915/gem/i915_gem_panic.c
+ create mode 100644 drivers/gpu/drm/i915/gem/i915_gem_panic.h
+ delete mode 100644 drivers/gpu/drm/i915/i915_panic.c
+ delete mode 100644 drivers/gpu/drm/i915/i915_panic.h
+ create mode 100644 drivers/gpu/drm/panel/panel-anbernic-td4310.c
+ create mode 100644 drivers/gpu/drm/panel/panel-chipone-icna35xx.c
+ create mode 100644 drivers/gpu/drm/panel/panel-ilitek-ili7807s.c
+ create mode 100644 drivers/gpu/drm/panel/panel-ilitek-ili9488.c
+ create mode 100644 drivers/gpu/drm/panel/panel-renesas-r63419.c
+ create mode 100644 drivers/gpu/drm/tests/drm_kunit_edid.c
+ create mode 100644 drivers/gpu/drm/xe/tests/xe_rtp_tables_test.c
+ create mode 100644 drivers/gpu/drm/xe/xe_pci_error.c
+ create mode 100644 drivers/gpu/drm/xe/xe_pci_error.h
+$ git am -3 ../patches/0001-drm-Fix-up-lut3d-mismerge.patch
+Applying: drm: Fix up lut3d mismerge
+Using index info to reconstruct a base tree...
+M	drivers/gpu/drm/drm_atomic.c
+M	include/drm/drm_colorop.h
+Falling back to patching base and 3-way merge...
+Auto-merging include/drm/drm_colorop.h
+Auto-merging drivers/gpu/drm/drm_atomic.c
+No changes -- Patch already applied.
+Merging drm-exynos/for-linux-next (3a8660878839f Linux 6.18-rc1)
+$ git merge -m Merge branch 'for-linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/daeinki/drm-exynos.git drm-exynos/for-linux-next
+Already up to date.
+Merging drm-misc/for-linux-next (6143359440d41 drm/gma500: Remove dependency on DRM simple helpers)
+$ git merge -m Merge branch 'for-linux-next' of https://gitlab.freedesktop.org/drm/misc/kernel.git drm-misc/for-linux-next
+Auto-merging drivers/gpu/host1x/bus.c
+Merge made by the 'ort' strategy.
+ .../display/tegra/nvidia,tegra124-vic.yaml         |   1 +
+ .../display/tegra/nvidia,tegra20-host1x.yaml       |  20 ++-
+ drivers/gpu/drm/gma500/cdv_intel_crt.c             |   9 +-
+ drivers/gpu/drm/gma500/cdv_intel_dp.c              | 120 +++++++-------
+ drivers/gpu/drm/gma500/cdv_intel_hdmi.c            |  10 +-
+ drivers/gpu/drm/gma500/cdv_intel_lvds.c            |   9 +-
+ drivers/gpu/drm/gma500/oaktrail_hdmi.c             |   9 +-
+ drivers/gpu/drm/gma500/oaktrail_lvds.c             |   9 +-
+ drivers/gpu/drm/gma500/psb_intel_lvds.c            |   9 +-
+ drivers/gpu/drm/solomon/ssd130x.c                  |   9 +-
+ drivers/gpu/drm/tegra/drm.c                        |   1 +
+ drivers/gpu/drm/tegra/falcon.c                     |  66 ++++++--
+ drivers/gpu/drm/tegra/falcon.h                     |  23 +++
+ drivers/gpu/drm/tegra/vic.c                        |  92 ++++++++---
+ drivers/gpu/drm/tegra/vic.h                        |   9 +-
+ drivers/gpu/host1x/Makefile                        |   3 +-
+ drivers/gpu/host1x/bus.c                           |   2 +-
+ drivers/gpu/host1x/cdma.c                          |   3 +-
+ drivers/gpu/host1x/context.c                       |  13 +-
+ drivers/gpu/host1x/debug.c                         |   4 +-
+ drivers/gpu/host1x/dev.c                           |  41 +++++
+ drivers/gpu/host1x/hw/cdma_hw.c                    |  14 +-
+ drivers/gpu/host1x/hw/channel_hw.c                 |  15 +-
+ drivers/gpu/host1x/hw/host1x10.c                   |  33 ++++
+ drivers/gpu/host1x/hw/host1x10.h                   |  15 ++
+ drivers/gpu/host1x/hw/host1x10_hardware.h          |  21 +++
+ drivers/gpu/host1x/hw/hw_host1x10_common.h         |   6 +
+ drivers/gpu/host1x/hw/hw_host1x10_hypervisor.h     |  10 ++
+ drivers/gpu/host1x/hw/hw_host1x10_uclass.h         | 181 +++++++++++++++++++++
+ drivers/gpu/host1x/hw/hw_host1x10_vm.h             |  36 ++++
+ drivers/gpu/host1x/intr.c                          |   5 +-
+ drivers/gpu/host1x/job.c                           |   2 +-
+ drivers/gpu/host1x/syncpt.c                        |   7 +-
+ drivers/gpu/host1x/syncpt.h                        |   3 +-
+ 34 files changed, 672 insertions(+), 138 deletions(-)
+ create mode 100644 drivers/gpu/host1x/hw/host1x10.c
+ create mode 100644 drivers/gpu/host1x/hw/host1x10.h
+ create mode 100644 drivers/gpu/host1x/hw/host1x10_hardware.h
+ create mode 100644 drivers/gpu/host1x/hw/hw_host1x10_common.h
+ create mode 100644 drivers/gpu/host1x/hw/hw_host1x10_hypervisor.h
+ create mode 100644 drivers/gpu/host1x/hw/hw_host1x10_uclass.h
+ create mode 100644 drivers/gpu/host1x/hw/hw_host1x10_vm.h
+$ git am -3 ../patches/0001-drm-bridge-microchip-lvds-Rename-drm_atomic_state-to.patch
+Applying: drm/bridge: microchip-lvds: Rename drm_atomic_state to drm_atomic_commit
+Using index info to reconstruct a base tree...
+M	drivers/gpu/drm/bridge/microchip-lvds.c
+Falling back to patching base and 3-way merge...
+Auto-merging drivers/gpu/drm/bridge/microchip-lvds.c
+No changes -- Patch already applied.
+Merging amdgpu/drm-next (8e1f8214cbba2 drm/radeon: Only define radeon_acpi_vfct_match when actually used)
+$ git merge -m Merge branch 'drm-next' of https://gitlab.freedesktop.org/agd5f/linux.git amdgpu/drm-next
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c         |  3 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c             |  4 +--
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c            |  5 +--
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c            |  3 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c             | 38 ++++++++++++++++++--
+ drivers/gpu/drm/amd/amdgpu/amdgpu_job.c            | 17 ++++-----
+ drivers/gpu/drm/amd/amdgpu/amdgpu_job.h            |  8 ++---
+ drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c           |  5 +--
+ drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c            | 41 +++++++++++++++++++---
+ drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h            |  2 ++
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h           |  4 +--
+ drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c             | 18 ++++++----
+ drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h             |  3 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c            | 11 +++---
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c          | 37 ++++++++++++++++++-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c            |  5 +--
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c            |  8 +++--
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c            | 12 ++++---
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c        |  2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c           |  6 ++--
+ drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c            |  2 +-
+ drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c           | 24 +++++++++++++
+ drivers/gpu/drm/amd/amdgpu/mes_userqueue.c         | 19 +++-------
+ drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h            |  1 +
+ drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c              | 10 +++---
+ drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c              | 10 +++---
+ drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c            | 26 ++++++++++++++
+ drivers/gpu/drm/amd/amdkfd/kfd_crat.c              |  9 +++++
+ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.c  | 19 +++++++++-
+ drivers/gpu/drm/amd/amdkfd/kfd_migrate.c           |  4 +--
+ drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c    | 16 +++++++--
+ drivers/gpu/drm/radeon/radeon_bios.c               |  2 +-
+ 32 files changed, 289 insertions(+), 85 deletions(-)
+$ git am -3 ../patches/0001-amdgpu-Fix-up-drm_atomic_commit-rename.patch
+Applying: amdgpu: Fix up drm_atomic_commit rename
+Using index info to reconstruct a base tree...
+M	drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+Falling back to patching base and 3-way merge...
+Auto-merging drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+No changes -- Patch already applied.
+Merging drm-intel/for-linux-next (67e955e073c6f drm/i915/display: expose blend mode on alpha-capable planes)
+$ git merge -m Merge branch 'for-linux-next' of https://gitlab.freedesktop.org/drm/i915/kernel.git drm-intel/for-linux-next
+Auto-merging drivers/gpu/drm/i915/display/intel_dp.c
+Auto-merging drivers/gpu/drm/i915/display/skl_watermark.c
+Auto-merging drivers/gpu/drm/xe/Makefile
+Auto-merging drivers/gpu/drm/xe/display/xe_display_bo.c
+Auto-merging drivers/gpu/drm/xe/display/xe_fb_pin.c
+Merge made by the 'ort' strategy.
+ .../gpu/intel-display/dp-link-capabilities.rst     |   11 +
+ Documentation/gpu/intel-display/index.rst          |    1 +
+ drivers/gpu/drm/i915/.kunitconfig                  |    9 +
+ drivers/gpu/drm/i915/Kconfig.debug                 |   12 +
+ drivers/gpu/drm/i915/Makefile                      |    2 +
+ drivers/gpu/drm/i915/display/i9xx_plane.c          |    4 +
+ drivers/gpu/drm/i915/display/intel_audio.c         |   55 +-
+ drivers/gpu/drm/i915/display/intel_bw.c            |   92 +-
+ drivers/gpu/drm/i915/display/intel_cursor.c        |    3 +
+ drivers/gpu/drm/i915/display/intel_ddi.c           |    6 +-
+ drivers/gpu/drm/i915/display/intel_display.c       |   31 +-
+ drivers/gpu/drm/i915/display/intel_display_core.h  |    2 +
+ .../gpu/drm/i915/display/intel_display_device.h    |    2 +
+ .../gpu/drm/i915/display/intel_display_driver.c    |   71 +
+ .../gpu/drm/i915/display/intel_display_driver.h    |    8 +
+ drivers/gpu/drm/i915/display/intel_display_irq.c   |    2 +-
+ drivers/gpu/drm/i915/display/intel_display_power.c |    4 +-
+ drivers/gpu/drm/i915/display/intel_display_regs.h  |    3 +
+ drivers/gpu/drm/i915/display/intel_display_types.h |    2 +-
+ drivers/gpu/drm/i915/display/intel_display_wa.c    |    2 +
+ drivers/gpu/drm/i915/display/intel_display_wa.h    |    1 +
+ drivers/gpu/drm/i915/display/intel_dmc.c           |   10 +-
+ drivers/gpu/drm/i915/display/intel_dp.c            |  295 ++--
+ drivers/gpu/drm/i915/display/intel_dp.h            |   11 +-
+ drivers/gpu/drm/i915/display/intel_dp_link_caps.c  |  759 ++++++++++-
+ drivers/gpu/drm/i915/display/intel_dp_link_caps.h  |  172 ++-
+ .../gpu/drm/i915/display/intel_dp_link_training.c  |  237 ++--
+ .../gpu/drm/i915/display/intel_dp_link_training.h  |   31 +
+ drivers/gpu/drm/i915/display/intel_dp_mst.c        |   74 +-
+ drivers/gpu/drm/i915/display/intel_dp_test.c       |  107 +-
+ drivers/gpu/drm/i915/display/intel_dp_test.h       |    3 +-
+ drivers/gpu/drm/i915/display/intel_dp_tunnel.c     |    8 +-
+ drivers/gpu/drm/i915/display/intel_fbdev.c         |   35 +-
+ drivers/gpu/drm/i915/display/intel_initial_plane.c |    6 +-
+ drivers/gpu/drm/i915/display/intel_modeset_setup.c |   18 +-
+ drivers/gpu/drm/i915/display/intel_pmdemand.c      |   12 +-
+ drivers/gpu/drm/i915/display/intel_psr.c           |   16 +
+ drivers/gpu/drm/i915/display/intel_sprite.c        |    4 +
+ drivers/gpu/drm/i915/display/intel_vrr.c           |    6 +
+ drivers/gpu/drm/i915/display/skl_watermark.c       |    6 +
+ drivers/gpu/drm/i915/display/tests/Makefile        |    7 +
+ .../drm/i915/display/tests/intel_dp_link_test.c    | 1440 ++++++++++++++++++++
+ drivers/gpu/drm/i915/i915_driver.c                 |   58 +-
+ drivers/gpu/drm/i915/i915_initial_plane.c          |   12 +-
+ drivers/gpu/drm/xe/.gitignore                      |    1 +
+ drivers/gpu/drm/xe/.kunitconfig-display            |   11 +
+ drivers/gpu/drm/xe/Makefile                        |    3 +
+ drivers/gpu/drm/xe/display/tests/Makefile          |   13 +
+ drivers/gpu/drm/xe/display/xe_display.c            |   44 +-
+ drivers/gpu/drm/xe/display/xe_display.h            |    2 +
+ drivers/gpu/drm/xe/display/xe_display_bo.c         |   33 +-
+ drivers/gpu/drm/xe/display/xe_fb_pin.c             |   33 +-
+ drivers/gpu/drm/xe/display/xe_initial_plane.c      |   58 +-
+ drivers/gpu/drm/xe/xe_pm.c                         |    2 +
+ drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c             |   16 +-
+ 55 files changed, 3243 insertions(+), 623 deletions(-)
+ create mode 100644 Documentation/gpu/intel-display/dp-link-capabilities.rst
+ create mode 100644 drivers/gpu/drm/i915/.kunitconfig
+ create mode 100644 drivers/gpu/drm/i915/display/tests/Makefile
+ create mode 100644 drivers/gpu/drm/i915/display/tests/intel_dp_link_test.c
+ create mode 100644 drivers/gpu/drm/xe/.kunitconfig-display
+ create mode 100644 drivers/gpu/drm/xe/display/tests/Makefile
+Merging drm-msm/msm-next (e2332abed2a4d drm/msm: Only fini scheduler after successful init)
+$ git merge -m Merge branch 'msm-next' of https://gitlab.freedesktop.org/drm/msm.git drm-msm/msm-next
+Already up to date.
+Merging drm-msm-lumag/msm-next-lumag (8de061ca149c9 drm/msm: Restore second parameter name in purge() and evict())
+$ git merge -m Merge branch 'msm-next-lumag' of https://gitlab.freedesktop.org/lumag/msm.git drm-msm-lumag/msm-next-lumag
+Already up to date.
+Merging drm-xe/drm-xe-next (53a7115f9862f drm/xe/xe_ras: Add RAS GPU health indicator)
+$ git merge -m Merge branch 'drm-xe-next' of https://gitlab.freedesktop.org/drm/xe/kernel.git drm-xe/drm-xe-next
+Auto-merging drivers/gpu/drm/xe/xe_device.c
+Auto-merging drivers/gpu/drm/xe/xe_pt.c
+Merge made by the 'ort' strategy.
+ .../ABI/testing/sysfs-driver-intel-xe-ras          |  30 ++
+ Documentation/gpu/xe/xe_device.rst                 |   7 +
+ drivers/gpu/drm/xe/abi/guc_actions_abi.h           |   1 +
+ drivers/gpu/drm/xe/abi/guc_capture_abi.h           |   3 +-
+ drivers/gpu/drm/xe/abi/guc_klvs_abi.h              |  39 +-
+ drivers/gpu/drm/xe/abi/guc_scheduler_abi.h         |   3 +-
+ drivers/gpu/drm/xe/abi/xe_driver_klvs_abi.h        |  27 ++
+ drivers/gpu/drm/xe/regs/xe_pmt.h                   |   2 +
+ .../gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c    | 429 ++++++++++++++++++++
+ drivers/gpu/drm/xe/tests/xe_sriov_packet_kunit.c   |  89 +++++
+ drivers/gpu/drm/xe/xe_bo.c                         |  58 ++-
+ drivers/gpu/drm/xe/xe_bo.h                         |   3 +-
+ drivers/gpu/drm/xe/xe_bo_types.h                   |   2 +
+ drivers/gpu/drm/xe/xe_configfs.c                   |  65 +++
+ drivers/gpu/drm/xe/xe_configfs.h                   |   2 +
+ drivers/gpu/drm/xe/xe_debugfs.c                    |  87 ++++-
+ drivers/gpu/drm/xe/xe_debugfs.h                    |   6 +
+ drivers/gpu/drm/xe/xe_device.c                     |   4 +-
+ drivers/gpu/drm/xe/xe_device_types.h               |   2 +-
+ drivers/gpu/drm/xe/xe_dma_buf.c                    |   2 +-
+ drivers/gpu/drm/xe/xe_exec_queue.c                 |  17 +-
+ drivers/gpu/drm/xe/xe_exec_queue_types.h           |  27 ++
+ drivers/gpu/drm/xe/xe_execlist.c                   |   1 +
+ drivers/gpu/drm/xe/xe_gt.c                         |   5 +-
+ drivers/gpu/drm/xe/xe_gt.h                         |  14 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c        |   5 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c         |   5 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_vf.c                |  47 +++
+ drivers/gpu/drm/xe/xe_gt_sriov_vf.h                |   1 +
+ drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h          |   4 +
+ drivers/gpu/drm/xe/xe_gt_types.h                   |  17 +-
+ drivers/gpu/drm/xe/xe_guc.c                        |  70 ++++
+ drivers/gpu/drm/xe/xe_guc.h                        |  23 +-
+ drivers/gpu/drm/xe/xe_guc_ads.c                    | 134 +++++--
+ drivers/gpu/drm/xe/xe_guc_capture.c                |  21 +-
+ drivers/gpu/drm/xe/xe_guc_capture.h                |   6 +-
+ drivers/gpu/drm/xe/xe_guc_ct.c                     |  43 +-
+ drivers/gpu/drm/xe/xe_guc_engine_activity.c        |  12 +-
+ drivers/gpu/drm/xe/xe_guc_exec_queue_types.h       |  12 +
+ drivers/gpu/drm/xe/xe_guc_fwif.h                   |   1 +
+ drivers/gpu/drm/xe/xe_guc_klv_helpers.c            | 260 +++++++++++-
+ drivers/gpu/drm/xe/xe_guc_klv_helpers.h            |  10 +
+ drivers/gpu/drm/xe/xe_guc_submit.c                 | 434 ++++++++++++++++++---
+ drivers/gpu/drm/xe/xe_guc_submit.h                 |   1 +
+ drivers/gpu/drm/xe/xe_guc_types.h                  |   2 -
+ drivers/gpu/drm/xe/xe_hw_engine.c                  |  82 +++-
+ drivers/gpu/drm/xe/xe_hw_engine_group.c            |  85 +++-
+ drivers/gpu/drm/xe/xe_hw_engine_types.h            |   7 +-
+ drivers/gpu/drm/xe/xe_hw_error.c                   |  11 +-
+ drivers/gpu/drm/xe/xe_migrate.c                    |  37 +-
+ drivers/gpu/drm/xe/xe_migrate.h                    |   1 +
+ drivers/gpu/drm/xe/xe_module.c                     |  49 +++
+ drivers/gpu/drm/xe/xe_module.h                     |   5 +
+ drivers/gpu/drm/xe/xe_nvm.c                        |   9 +-
+ drivers/gpu/drm/xe/xe_pci.c                        |  34 +-
+ drivers/gpu/drm/xe/xe_pci_error.c                  |  21 +-
+ drivers/gpu/drm/xe/xe_pci_types.h                  |   2 +
+ drivers/gpu/drm/xe/xe_pcode.c                      |  32 ++
+ drivers/gpu/drm/xe/xe_pcode.h                      |   7 +
+ drivers/gpu/drm/xe/xe_preempt_fence.c              |   7 +
+ drivers/gpu/drm/xe/xe_pt.c                         |   3 +
+ drivers/gpu/drm/xe/xe_ras.c                        | 276 ++++++++++++-
+ drivers/gpu/drm/xe/xe_ras.h                        |   2 +
+ drivers/gpu/drm/xe/xe_ras_types.h                  |  97 +++++
+ drivers/gpu/drm/xe/xe_sriov_packet.c               | 115 +++---
+ drivers/gpu/drm/xe/xe_sriov_vf_ccs.c               |  20 +-
+ drivers/gpu/drm/xe/xe_sriov_vf_ccs.h               |   3 +-
+ drivers/gpu/drm/xe/xe_sysctrl_event.c              |   2 +-
+ drivers/gpu/drm/xe/xe_sysctrl_event_types.h        |   3 -
+ drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h      |   9 +
+ drivers/gpu/drm/xe/xe_trace.h                      |   5 +
+ drivers/gpu/drm/xe/xe_uc_fw.c                      |   1 +
+ drivers/gpu/drm/xe/xe_vm.c                         |  43 +-
+ drivers/gpu/drm/xe/xe_wa_oob.rules                 |   5 +
+ drivers/gpu/drm/xe/xe_wopcm.c                      |  15 +-
+ include/uapi/drm/xe_drm.h                          |   6 +-
+ 76 files changed, 2692 insertions(+), 335 deletions(-)
+ create mode 100644 Documentation/ABI/testing/sysfs-driver-intel-xe-ras
+ create mode 100644 drivers/gpu/drm/xe/abi/xe_driver_klvs_abi.h
+ create mode 100644 drivers/gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c
+ create mode 100644 drivers/gpu/drm/xe/tests/xe_sriov_packet_kunit.c
+$ git am -3 ../patches/0001-drm-xe-Fix-up-merge-issue.patch
+Applying: drm: xe: Fix up merge issue
+Using index info to reconstruct a base tree...
+M	drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+Falling back to patching base and 3-way merge...
+Auto-merging drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+No changes -- Patch already applied.
+Merging drm-rust/for-linux-next (ce2d97f714c8a gpu: nova-core: store Fsp instance in Gpu)
+$ git merge -m Merge branch 'for-linux-next' of https://gitlab.freedesktop.org/drm/rust/kernel.git drm-rust/for-linux-next
+Auto-merging drivers/gpu/drm/drm_drv.c
+Merge made by the 'ort' strategy.
+ Documentation/gpu/nova/core/fsp.rst                |  142 ++
+ Documentation/gpu/nova/index.rst                   |    1 +
+ drivers/gpu/Makefile                               |   58 +-
+ drivers/gpu/drm/Makefile                           |    2 +-
+ drivers/gpu/drm/drm_drv.c                          |   34 +-
+ drivers/gpu/drm/nova/Makefile                      |    5 +-
+ drivers/gpu/drm/nova/driver.rs                     |   36 +-
+ drivers/gpu/drm/nova/file.rs                       |   22 +-
+ drivers/gpu/drm/nova/gem.rs                        |   18 +-
+ drivers/gpu/drm/tyr/driver.rs                      |   25 +-
+ drivers/gpu/drm/tyr/file.rs                        |    8 +-
+ drivers/gpu/drm/tyr/gem.rs                         |   11 +-
+ drivers/gpu/drm/tyr/regs.rs                        |   14 +-
+ drivers/gpu/nova-core/.gitignore                   |    1 +
+ drivers/gpu/nova-core/Makefile                     |    5 +-
+ drivers/gpu/nova-core/bitfield.rs                  |  329 -----
+ drivers/gpu/nova-core/falcon.rs                    |  195 ++-
+ drivers/gpu/nova-core/falcon/fsp.rs                |   44 +-
+ drivers/gpu/nova-core/falcon/gsp.rs                |   32 +-
+ drivers/gpu/nova-core/falcon/hal.rs                |   14 +-
+ drivers/gpu/nova-core/falcon/hal/ga102.rs          |   29 +-
+ drivers/gpu/nova-core/falcon/hal/tu102.rs          |   24 +-
+ drivers/gpu/nova-core/fb.rs                        |    4 +-
+ drivers/gpu/nova-core/fb/hal/gb202.rs              |   28 +-
+ drivers/gpu/nova-core/fb/hal/gh100.rs              |   31 +-
+ drivers/gpu/nova-core/fb/regs.rs                   |   25 +
+ drivers/gpu/nova-core/firmware.rs                  |   34 +-
+ drivers/gpu/nova-core/firmware/booter.rs           |   25 +-
+ drivers/gpu/nova-core/firmware/fwsec.rs            |   24 +-
+ drivers/gpu/nova-core/firmware/fwsec/bootloader.rs |   16 +-
+ drivers/gpu/nova-core/fsp.rs                       |  137 +-
+ drivers/gpu/nova-core/gpu.rs                       |  184 ++-
+ drivers/gpu/nova-core/gsp.rs                       |   99 +-
+ drivers/gpu/nova-core/gsp/boot.rs                  |  137 +-
+ drivers/gpu/nova-core/gsp/cmdq.rs                  |   69 +-
+ drivers/gpu/nova-core/gsp/commands.rs              |   89 +-
+ drivers/gpu/nova-core/gsp/fw.rs                    |  101 +-
+ drivers/gpu/nova-core/gsp/fw/commands.rs           |   42 +-
+ drivers/gpu/nova-core/gsp/hal.rs                   |   68 +-
+ drivers/gpu/nova-core/gsp/hal/ga102.rs             |   14 +
+ drivers/gpu/nova-core/gsp/hal/gh100.rs             |   89 +-
+ drivers/gpu/nova-core/gsp/hal/tu102.rs             |  396 +++--
+ drivers/gpu/nova-core/gsp/regs.rs                  |   11 +
+ drivers/gpu/nova-core/gsp/sequencer.rs             |   98 +-
+ drivers/gpu/nova-core/mctp.rs                      |   86 +-
+ drivers/gpu/nova-core/nova_core.rs                 |    5 +-
+ drivers/gpu/nova-core/nova_core_exports.c          |   15 +
+ drivers/gpu/nova-core/regs.rs                      |   70 +-
+ drivers/gpu/nova-core/vbios.rs                     |   64 +-
+ drivers/pwm/pwm_th1520.rs                          |    7 +-
+ rust/helpers/io.c                                  |   13 +
+ rust/kernel/devres.rs                              |   24 +-
+ rust/kernel/dma.rs                                 |  281 ++--
+ rust/kernel/drm/device.rs                          |  291 +++-
+ rust/kernel/drm/driver.rs                          |  113 +-
+ rust/kernel/drm/gem/mod.rs                         |  111 +-
+ rust/kernel/drm/gem/shmem.rs                       |  594 +++++++-
+ rust/kernel/drm/gpuvm/mod.rs                       |   36 +-
+ rust/kernel/drm/gpuvm/sm_ops.rs                    |    4 +-
+ rust/kernel/drm/gpuvm/va.rs                        |    8 +
+ rust/kernel/drm/gpuvm/vm_bo.rs                     |    9 +
+ rust/kernel/drm/ioctl.rs                           |   56 +-
+ rust/kernel/drm/mod.rs                             |    4 +-
+ rust/kernel/faux.rs                                |   81 +-
+ rust/kernel/io.rs                                  | 1526 +++++++++++++++-----
+ rust/kernel/io/mem.rs                              |   29 +-
+ rust/kernel/io/poll.rs                             |    6 +-
+ rust/kernel/io/register.rs                         |   44 +-
+ rust/kernel/lib.rs                                 |    3 +
+ rust/kernel/pci.rs                                 |    1 -
+ rust/kernel/pci/io.rs                              |  170 ++-
+ rust/kernel/ptr.rs                                 |   12 +
+ samples/rust/rust_dma.rs                           |   12 +-
+ samples/rust/rust_driver_faux.rs                   |    3 +-
+ 74 files changed, 4067 insertions(+), 2381 deletions(-)
+ create mode 100644 Documentation/gpu/nova/core/fsp.rst
+ create mode 100644 drivers/gpu/nova-core/.gitignore
+ delete mode 100644 drivers/gpu/nova-core/bitfield.rs
+ create mode 100644 drivers/gpu/nova-core/fb/regs.rs
+ create mode 100644 drivers/gpu/nova-core/gsp/hal/ga102.rs
+ create mode 100644 drivers/gpu/nova-core/gsp/regs.rs
+ create mode 100644 drivers/gpu/nova-core/nova_core_exports.c
+Merging drm-nova/nova-next (93296e9d9528f gpu: nova-core: vbios: store reference to Device where relevant)
+$ git merge -m Merge branch 'nova-next' of https://gitlab.freedesktop.org/drm/nova.git drm-nova/nova-next
+Already up to date.
+Merging etnaviv/etnaviv/next (6bde14ba5f7ef drm/etnaviv: add optional reset support)
+$ git merge -m Merge branch 'etnaviv/next' of https://git.pengutronix.de/git/lst/linux etnaviv/etnaviv/next
+Already up to date.
+Merging fbdev/for-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/deller/linux-fbdev.git fbdev/for-next
+Already up to date.
+$ git am -3 ../patches/0001-fix-up-for-drm-hyperv-Remove-reference-to-hyperv_fb-.patch
+Applying: fix up for "drm/hyperv: Remove reference to hyperv_fb driver"
+$ git reset HEAD^
+Unstaged changes after reset:
+M	drivers/gpu/drm/hyperv/Kconfig
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master d860d0dea2534] Merge branch 'for-linux-next' of https://gitlab.freedesktop.org/drm/rust/kernel.git
+ Date: Fri Jul 17 16:50:35 2026 +0100
+Merging regmap/for-next (8979c222ebebb Merge remote-tracking branch 'regmap/for-7.3' into regmap-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git regmap/for-next
+Merge made by the 'ort' strategy.
+ drivers/base/regmap/internal.h        |  2 +-
+ drivers/base/regmap/regcache-flat.c   |  4 +---
+ drivers/base/regmap/regcache-maple.c  |  8 +++-----
+ drivers/base/regmap/regcache-rbtree.c |  8 +++-----
+ drivers/base/regmap/regcache.c        | 24 +++++++++++++-----------
+ drivers/base/regmap/regmap-irq.c      | 22 ++++++++++++++++++++++
+ include/linux/regmap.h                |  2 ++
+ 7 files changed, 45 insertions(+), 25 deletions(-)
+Merging sound/for-next (4392569d95e6a ALSA: aloop: make read-only array texts static const)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git sound/for-next
+Auto-merging sound/hda/codecs/hdmi/hdmi.c
+Auto-merging sound/hda/codecs/realtek/alc269.c
+Merge made by the 'ort' strategy.
+ include/sound/ac97/codec.h                    |   4 +-
+ include/sound/hda_codec.h                     |   1 +
+ include/sound/hda_regmap.h                    |  42 +-
+ include/uapi/sound/firewire.h                 |  24 +-
+ sound/core/compress_offload.c                 |  18 +-
+ sound/core/control.c                          |  29 +-
+ sound/core/hwdep.c                            |  25 +-
+ sound/core/info.c                             |   2 +-
+ sound/core/init.c                             |  23 +-
+ sound/core/jack.c                             |  38 +-
+ sound/core/oss/mixer_oss.c                    |  19 +-
+ sound/core/oss/pcm_oss.c                      |  27 +-
+ sound/core/pcm_native.c                       |  62 +-
+ sound/core/rawmidi.c                          |  20 +-
+ sound/core/seq/oss/seq_oss.c                  |  21 +-
+ sound/core/seq/seq_clientmgr.c                |  24 +-
+ sound/core/sound.c                            |  25 +-
+ sound/core/timer.c                            |  30 +-
+ sound/drivers/aloop.c                         | 120 +++-
+ sound/hda/codecs/cm9825.c                     | 148 ++++-
+ sound/hda/codecs/hdmi/hdmi.c                  |   4 +-
+ sound/hda/codecs/helpers/razer_blade16_2025.c | 820 ++++++++++++++++++++++++++
+ sound/hda/codecs/realtek/alc269.c             |  21 +
+ sound/hda/common/codec.c                      |   5 +-
+ sound/hda/controllers/intel.c                 |  11 +
+ sound/hda/core/controller.c                   |  28 +-
+ sound/hda/core/stream.c                       |  40 +-
+ sound/pci/asihpi/hpi6000.c                    |   5 +
+ sound/pci/via82xx_modem.c                     |  26 +-
+ sound/usb/card.c                              |  70 ++-
+ sound/usb/mixer_quirks.c                      |  20 +
+ sound/usb/quirks-table.h                      |  67 +++
+ sound/usb/usx2y/us144mkii_pcm.h               |   6 +-
+ 33 files changed, 1532 insertions(+), 293 deletions(-)
+ create mode 100644 sound/hda/codecs/helpers/razer_blade16_2025.c
+Merging ieee1394/for-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ieee1394/linux1394.git ieee1394/for-next
+Already up to date.
+Merging sound-asoc/for-next (d721340cd338e Merge remote-tracking branch 'asoc/for-7.3' into asoc-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git sound-asoc/for-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/sound/cirrus,cs35l36.yaml  | 240 ++++++
+ .../devicetree/bindings/sound/cs35l36.txt          | 168 -----
+ .../devicetree/bindings/sound/dmic-codec.yaml      |   4 +
+ .../devicetree/bindings/sound/fsl,micfil.yaml      |   4 +
+ .../bindings/sound/loongson,ls-audio-card.yaml     |  53 +-
+ .../bindings/sound/loongson,ls2k1000-i2s.yaml      |  22 +-
+ .../bindings/sound/mediatek,mtk-btcvsd-snd.yaml    |  59 ++
+ .../devicetree/bindings/sound/mtk-btcvsd-snd.txt   |  24 -
+ .../devicetree/bindings/sound/wlf,wm8524.yaml      |   4 +
+ include/sound/simple_card_utils.h                  |  37 +-
+ include/sound/soc_sdw_utils.h                      |  31 +-
+ include/sound/sof/dai-amd.h                        |   1 +
+ include/sound/sof/dai.h                            |   3 +
+ include/uapi/sound/sof/tokens.h                    |   1 +
+ sound/soc/amd/acp-config.c                         |  12 +
+ sound/soc/amd/acp/acp-i2s.c                        |   6 +
+ sound/soc/amd/acp/acp-sdw-legacy-mach.c            |   2 +-
+ sound/soc/amd/acp/acp-sdw-sof-mach.c               |   2 +-
+ sound/soc/amd/mach-config.h                        |   1 +
+ sound/soc/amd/raven/acp3x-i2s.c                    |   6 +
+ sound/soc/amd/vangogh/acp5x-i2s.c                  |   6 +
+ sound/soc/amd/yc/acp6x-mach.c                      |  12 +-
+ sound/soc/apple/mca.c                              | 308 +++++---
+ sound/soc/au1x/psc-ac97.c                          |   4 +-
+ sound/soc/codecs/Kconfig                           |  34 +-
+ sound/soc/codecs/ab8500-codec.c                    |  12 +-
+ sound/soc/codecs/ak4613.c                          |  77 +-
+ sound/soc/codecs/arizona-jack.c                    | 121 ++-
+ sound/soc/codecs/arizona.c                         |  26 +-
+ sound/soc/codecs/aw87390.c                         |  10 +-
+ sound/soc/codecs/aw88081.c                         |  18 +-
+ sound/soc/codecs/aw88166.c                         |  24 +-
+ sound/soc/codecs/aw88261.c                         |  27 +-
+ sound/soc/codecs/aw88261.h                         |   6 -
+ sound/soc/codecs/aw88395/aw88395.c                 |  24 +-
+ sound/soc/codecs/aw88395/aw88395_device.c          |  40 +-
+ sound/soc/codecs/aw88399.c                         |  15 +-
+ sound/soc/codecs/cros_ec_codec.c                   |  18 +-
+ sound/soc/codecs/cs-amp-lib.c                      |  11 +-
+ sound/soc/codecs/cs35l32.c                         |  10 +-
+ sound/soc/codecs/cs35l33.c                         |  14 +-
+ sound/soc/codecs/cs35l34.c                         |  14 +-
+ sound/soc/codecs/cs35l56-i2c.c                     |   4 +-
+ sound/soc/codecs/cs35l56-sdw.c                     |   2 +
+ sound/soc/codecs/cs35l56-shared.c                  |  26 +-
+ sound/soc/codecs/cs35l56-spi.c                     |   4 +-
+ sound/soc/codecs/cs35l56.c                         |  18 +-
+ sound/soc/codecs/cs42l42-sdw.c                     |  14 +-
+ sound/soc/codecs/cs42l42.c                         |  56 +-
+ sound/soc/codecs/cs42l43-jack.c                    | 138 ++--
+ sound/soc/codecs/cs42l43.c                         |  16 +-
+ sound/soc/codecs/cs42l84.c                         |  10 +-
+ sound/soc/codecs/cs43130.c                         |  72 +-
+ sound/soc/codecs/cs4349.c                          |   8 +-
+ sound/soc/codecs/cs47l15.c                         |  11 +-
+ sound/soc/codecs/cs47l35.c                         |  11 +-
+ sound/soc/codecs/cs47l85.c                         |  11 +-
+ sound/soc/codecs/cs47l90.c                         |  11 +-
+ sound/soc/codecs/cs47l92.c                         |  11 +-
+ sound/soc/codecs/cs48l32.c                         |  26 +-
+ sound/soc/codecs/cx2072x.c                         |   5 +-
+ sound/soc/codecs/es8328.c                          |   5 +
+ sound/soc/codecs/es8389.c                          | 236 +++++-
+ sound/soc/codecs/es8389.h                          |  11 +-
+ sound/soc/codecs/es9356.c                          |   2 +-
+ sound/soc/codecs/fs210x.c                          |   1 +
+ sound/soc/codecs/hdac_hda.c                        |   4 +-
+ sound/soc/codecs/hdac_hdmi.c                       |  10 +-
+ sound/soc/codecs/lpass-rx-macro.c                  | 141 ++--
+ sound/soc/codecs/lpass-tx-macro.c                  | 130 ++--
+ sound/soc/codecs/lpass-va-macro.c                  | 139 ++--
+ sound/soc/codecs/lpass-wsa-macro.c                 | 144 ++--
+ sound/soc/codecs/max98090.c                        |   9 +-
+ sound/soc/codecs/max98363.c                        |   7 +-
+ sound/soc/codecs/max98373-i2c.c                    |   9 +-
+ sound/soc/codecs/max98373-sdw.c                    |   7 +-
+ sound/soc/codecs/max98388.c                        |   8 +-
+ sound/soc/codecs/max98390.c                        |   8 +-
+ sound/soc/codecs/max98396.c                        |  24 +-
+ sound/soc/codecs/max98927.c                        |   8 +-
+ sound/soc/codecs/pcm1681.c                         |   7 +
+ sound/soc/codecs/pcm1754.c                         |   6 +
+ sound/soc/codecs/pcm1789.c                         |   7 +
+ sound/soc/codecs/pcm179x.c                         |   6 +
+ sound/soc/codecs/pcm186x.c                         |   9 +
+ sound/soc/codecs/pcm3060.c                         |   8 +
+ sound/soc/codecs/pcm512x.c                         |   9 +
+ sound/soc/codecs/pm4125-sdw.c                      |   8 +-
+ sound/soc/codecs/pm4125.c                          |   3 +-
+ sound/soc/codecs/rt1017-sdca-sdw.c                 |   7 +-
+ sound/soc/codecs/rt1316-sdw.c                      |   7 +-
+ sound/soc/codecs/rt1318-sdw.c                      |   7 +-
+ sound/soc/codecs/rt1320-sdw.c                      | 833 +++++++++++++++++----
+ sound/soc/codecs/rt1320-sdw.h                      |  12 +
+ sound/soc/codecs/rt5514.c                          |  16 +-
+ sound/soc/codecs/rt5645.c                          |  16 +-
+ sound/soc/codecs/rt5677.c                          |  17 +
+ sound/soc/codecs/rt5682-sdw.c                      |  20 +-
+ sound/soc/codecs/rt700-sdw.c                       |   6 +-
+ sound/soc/codecs/rt712-sdca-dmic.c                 |  17 +-
+ sound/soc/codecs/rt712-sdca-sdw.c                  |  17 +-
+ sound/soc/codecs/rt721-sdca-sdw.c                  |  17 +-
+ sound/soc/codecs/rt722-sdca-sdw.c                  |   8 +-
+ sound/soc/codecs/rt9120.c                          |   9 +-
+ sound/soc/codecs/sma1303.c                         |   2 +
+ sound/soc/codecs/tas2552.c                         |  10 +-
+ sound/soc/codecs/tas2783-sdw.c                     |  16 +-
+ sound/soc/codecs/tlv320aic26.c                     |   6 +
+ sound/soc/codecs/uda1342.c                         |   9 +-
+ sound/soc/codecs/wcd937x-sdw.c                     |   8 +-
+ sound/soc/codecs/wcd938x-sdw.c                     |   8 +-
+ sound/soc/codecs/wcd939x-sdw.c                     |   8 +-
+ sound/soc/codecs/wm2200.c                          |  10 +-
+ sound/soc/codecs/wm5100.c                          |  10 +-
+ sound/soc/codecs/wm8962.c                          |   9 +-
+ sound/soc/codecs/wsa881x.c                         |   8 +-
+ sound/soc/codecs/wsa883x.c                         |   8 +-
+ sound/soc/codecs/wsa884x.c                         |   8 +-
+ sound/soc/fsl/fsl_asrc.c                           |  10 +-
+ sound/soc/fsl/fsl_audmix.c                         |  35 +-
+ sound/soc/fsl/fsl_easrc.c                          |  36 +-
+ sound/soc/fsl/fsl_esai.c                           |  16 +-
+ sound/soc/fsl/fsl_spdif.c                          |   8 +-
+ sound/soc/fsl/fsl_ssi.c                            |  13 +-
+ sound/soc/fsl/fsl_xcvr.c                           |  29 +-
+ sound/soc/fsl/imx-audio-rpmsg.c                    |  25 +-
+ sound/soc/fsl/imx-pcm-rpmsg.c                      |  69 +-
+ sound/soc/fsl/mpc5200_dma.c                        |  56 +-
+ sound/soc/fsl/mpc5200_psc_ac97.c                   |  34 +-
+ sound/soc/fsl/mpc5200_psc_i2s.c                    |   1 +
+ sound/soc/generic/audio-graph-card.c               |  16 +-
+ sound/soc/generic/audio-graph-card2.c              |  35 +-
+ sound/soc/generic/simple-card-utils.c              |  82 +-
+ sound/soc/generic/simple-card.c                    | 127 ++--
+ sound/soc/intel/atom/sst-atom-controls.c           |  67 +-
+ sound/soc/intel/atom/sst-mfld-platform-pcm.c       |  59 +-
+ sound/soc/intel/atom/sst/sst_ipc.c                 |   5 +-
+ sound/soc/intel/atom/sst/sst_pci.c                 |   4 +-
+ sound/soc/intel/atom/sst/sst_pvt.c                 |   9 +-
+ sound/soc/intel/avs/apl.c                          |   8 +-
+ sound/soc/intel/avs/control.c                      |   7 +-
+ sound/soc/intel/avs/core.c                         |   4 +-
+ sound/soc/intel/avs/debug.h                        |  10 +-
+ sound/soc/intel/avs/debugfs.c                      |  19 +-
+ sound/soc/intel/avs/ipc.c                          |  59 +-
+ sound/soc/intel/avs/loader.c                       |  15 +-
+ sound/soc/intel/avs/path.c                         |  30 +-
+ sound/soc/intel/avs/utils.c                        |  45 +-
+ sound/soc/intel/boards/sof_sdw.c                   |   4 +-
+ sound/soc/loongson/loongson_card.c                 | 171 ++++-
+ sound/soc/loongson/loongson_i2s_plat.c             |  42 +-
+ sound/soc/mediatek/Kconfig                         |   2 +-
+ sound/soc/mediatek/mt8173/mt8173-rt5650.c          |   2 +-
+ sound/soc/mediatek/mt8365/mt8365-mt6357.c          |   9 +-
+ sound/soc/meson/Makefile                           |   2 +
+ sound/soc/meson/aiu-encoder-i2s.c                  | 281 +++++--
+ sound/soc/meson/aiu-formatter-i2s.c                | 103 +++
+ sound/soc/meson/aiu.c                              |  32 +-
+ sound/soc/meson/aiu.h                              |   4 +
+ sound/soc/meson/g12a-toacodec.c                    |   2 +-
+ sound/soc/meson/g12a-tohdmitx.c                    |   2 +-
+ sound/soc/meson/gx-formatter.c                     | 282 +++++++
+ sound/soc/meson/gx-formatter.h                     |  56 ++
+ sound/soc/meson/gx-interface.h                     |  48 ++
+ sound/soc/meson/t9015.c                            |  12 +-
+ sound/soc/renesas/fsi.c                            |   9 +-
+ sound/soc/renesas/rcar/adg.c                       |   8 +-
+ sound/soc/samsung/aries_wm8994.c                   |   1 +
+ sound/soc/samsung/i2s.c                            |  33 +-
+ sound/soc/samsung/spdif.c                          |   8 +-
+ sound/soc/sdw_utils/soc_sdw_bridge_cs35l56.c       |   4 +-
+ sound/soc/sdw_utils/soc_sdw_utils.c                |  36 +-
+ sound/soc/sof/amd/Kconfig                          |  11 +
+ sound/soc/sof/amd/Makefile                         |   2 +
+ sound/soc/sof/amd/acp-dsp-offset.h                 |  17 +
+ sound/soc/sof/amd/acp-ipc.c                        |  13 +-
+ sound/soc/sof/amd/acp-loader.c                     |  37 +-
+ sound/soc/sof/amd/acp.c                            | 267 ++++++-
+ sound/soc/sof/amd/acp.h                            |  20 +
+ sound/soc/sof/amd/acp7x.c                          | 185 +++++
+ sound/soc/sof/amd/pci-acp7x.c                      | 116 +++
+ sound/soc/sof/intel/hda-loader.c                   |  11 +-
+ sound/soc/sof/intel/hda-sdw-bpt.c                  |   2 +-
+ sound/soc/sof/intel/hda-stream.c                   |  15 +-
+ sound/soc/sof/intel/hda.h                          |   5 +-
+ sound/soc/sof/ipc3-pcm.c                           |   6 +
+ sound/soc/sof/ipc3-topology.c                      |  39 +-
+ sound/soc/sof/ipc4-topology.c                      |   2 +
+ sound/soc/sof/nocodec.c                            |  16 +-
+ sound/soc/sof/topology.c                           |  15 +-
+ sound/soc/spacemit/k1_i2s.c                        |   7 +
+ sound/soc/sunxi/sun4i-codec.c                      |   4 +-
+ sound/soc/tegra/tegra20_das.c                      |   7 +-
+ sound/soc/tegra/tegra210_adx.c                     |  12 +-
+ sound/soc/tegra/tegra210_amx.c                     |  12 +-
+ sound/soc/tegra/tegra210_i2s.c                     |  10 +-
+ sound/soc/ti/j721e-evm.c                           |   4 +-
+ sound/soc/xilinx/xlnx_i2s.c                        |  18 +-
+ sound/soc/xilinx/xlnx_spdif.c                      |  31 +-
+ sound/soc/xtensa/xtfpga-i2s.c                      |  60 +-
+ 200 files changed, 4926 insertions(+), 2164 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/sound/cirrus,cs35l36.yaml
+ delete mode 100644 Documentation/devicetree/bindings/sound/cs35l36.txt
+ create mode 100644 Documentation/devicetree/bindings/sound/mediatek,mtk-btcvsd-snd.yaml
+ delete mode 100644 Documentation/devicetree/bindings/sound/mtk-btcvsd-snd.txt
+ create mode 100644 sound/soc/meson/aiu-formatter-i2s.c
+ create mode 100644 sound/soc/meson/gx-formatter.c
+ create mode 100644 sound/soc/meson/gx-formatter.h
+ create mode 100644 sound/soc/meson/gx-interface.h
+ create mode 100644 sound/soc/sof/amd/acp7x.c
+ create mode 100644 sound/soc/sof/amd/pci-acp7x.c
+Merging modules/modules-next (2600d43b235b0 rust: module_param: support bool parameters)
+$ git merge -m Merge branch 'modules-next' of https://git.kernel.org/pub/scm/linux/kernel/git/modules/linux.git modules/modules-next
+Auto-merging kernel/params.c
+Merge made by the 'ort' strategy.
+ arch/xtensa/platforms/iss/simdisk.c | 38 ++++++++++++++++++++++++++++++++-----
+ include/linux/module.h              |  4 ----
+ kernel/module/dups.c                |  2 +-
+ kernel/module/kallsyms.c            | 24 +++++++++++++++--------
+ kernel/module/main.c                | 15 ++++++++-------
+ kernel/module/procfs.c              |  2 +-
+ kernel/module/stats.c               |  2 +-
+ kernel/params.c                     |  2 +-
+ rust/kernel/module_param.rs         | 27 ++++++++++++++++++++++++--
+ rust/macros/lib.rs                  |  1 +
+ rust/macros/module.rs               |  1 +
+ samples/rust/rust_minimal.rs        | 10 +++++++++-
+ scripts/module.lds.S                |  5 -----
+ 13 files changed, 97 insertions(+), 36 deletions(-)
+Merging input/next (3ed4ba919eb35 Input: wacom_w8001 - replace strlcat() with a strscpy() helper)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git input/next
+Merge made by the 'ort' strategy.
+ .../bindings/input/gpio-charlieplex-keypad.yaml    |   2 +-
+ .../devicetree/bindings/input/imagis,isa1200.yaml  | 141 ++++++
+ .../bindings/input/microchip,cap11xx.yaml          |  92 +++-
+ .../devicetree/bindings/input/syna,rmi4.yaml       |  11 +-
+ drivers/input/keyboard/cap11xx.c                   | 275 +++++++----
+ drivers/input/keyboard/matrix_keypad.c             |   1 -
+ drivers/input/keyboard/tca8418_keypad.c            |   3 +-
+ drivers/input/misc/Kconfig                         |  12 +
+ drivers/input/misc/Makefile                        |   1 +
+ drivers/input/misc/ims-pcu.c                       |   1 +
+ drivers/input/misc/isa1200.c                       | 533 +++++++++++++++++++++
+ drivers/input/mouse/synaptics.c                    |   1 +
+ drivers/input/serio/i8042-acpipnpio.h              |  27 +-
+ drivers/input/touchscreen/mms114.c                 | 299 ++++++------
+ drivers/input/touchscreen/sur40.c                  |   4 +-
+ drivers/input/touchscreen/wacom_w8001.c            |  17 +-
+ 16 files changed, 1149 insertions(+), 271 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/input/imagis,isa1200.yaml
+ create mode 100644 drivers/input/misc/isa1200.c
+Merging block/for-next (1a5c200b36e4f Merge branch 'block-7.2' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git block/for-next
+Merge made by the 'ort' strategy.
+ block/bio.c                    | 64 +++++++++++++++++++++++++-----------------
+ block/blk-cgroup.c             |  4 +--
+ block/blk-map.c                |  9 ++++--
+ block/blk.h                    | 13 +++++++--
+ block/elevator.c               |  9 ++++--
+ block/error-injection.c        |  7 +++--
+ block/partitions/aix.c         |  9 +++++-
+ drivers/block/loop.c           | 32 +++++++++------------
+ include/linux/io_uring_types.h |  2 ++
+ io_uring/fs.c                  |  3 +-
+ io_uring/io-wq.c               |  5 ++++
+ io_uring/io_uring.c            |  2 ++
+ io_uring/kbuf.c                |  4 +--
+ io_uring/sqpoll.c              |  7 ++++-
+ 14 files changed, 110 insertions(+), 60 deletions(-)
+$ git am -3 ../patches/0001-Revert-block-remove-bio_last_bvec_all.patch
+Applying: Revert "block: remove bio_last_bvec_all"
+$ git reset HEAD^
+Unstaged changes after reset:
+M	Documentation/block/biovecs.rst
+M	include/linux/bio.h
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master bd4ddc05ec9b4] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git
+ Date: Fri Jul 17 16:50:48 2026 +0100
+Merging device-mapper/for-next (22eb919f1b790 dm-inlinecrypt: don't overwrite the error with -EINVAL)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git device-mapper/for-next
+Merge made by the 'ort' strategy.
+ drivers/md/dm-cache-target.c   |  56 +++++------
+ drivers/md/dm-inlinecrypt.c    |   1 -
+ drivers/md/dm-integrity.c      |   6 +-
+ drivers/md/dm-ioctl.c          | 208 ++++++++++++++++++-----------------------
+ drivers/md/dm-snap.c           |   4 +-
+ drivers/md/dm-switch.c         |   2 +-
+ drivers/md/dm-table.c          |   2 +-
+ drivers/md/dm-verity-target.c  |   2 -
+ drivers/md/dm-zoned-metadata.c |   8 +-
+ drivers/md/dm-zoned-reclaim.c  |   4 +-
+ 10 files changed, 127 insertions(+), 166 deletions(-)
+Merging libata/for-next (1393f2c98fb42 ata: pata_mpc52xx: Remove redundant dev_err())
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/libata/linux libata/for-next
+Auto-merging drivers/ata/libata-core.c
+Merge made by the 'ort' strategy.
+ drivers/ata/libata-core.c  |  79 ++++-
+ drivers/ata/libata-eh.c    |   6 +-
+ drivers/ata/libata-scsi.c  | 716 ++++++++++++++++++++++++++++++++++++++-------
+ drivers/ata/libata.h       |   2 +-
+ drivers/ata/pata_mpc52xx.c |   4 +-
+ drivers/scsi/scsi_debug.c  |  46 ---
+ include/linux/ata.h        |  17 ++
+ include/linux/libata.h     |  93 +++---
+ include/scsi/scsi_proto.h  |  74 +++++
+ 9 files changed, 832 insertions(+), 205 deletions(-)
+Merging pcmcia/pcmcia-next (b3c26ea81ccc5 pcmcia: remove obsolete host controller drivers)
+$ git merge -m Merge branch 'pcmcia-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brodo/linux.git pcmcia/pcmcia-next
+Already up to date.
+Merging mmc/next (5608386899d9d mmc: bcm2835: DMA mapping improvements)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git mmc/next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/mmc/qcom,sdhci-msm.yaml    |  95 ++++++++++-----
+ .../devicetree/bindings/mmc/rockchip-dw-mshc.yaml  |   1 +
+ Documentation/devicetree/bindings/mmc/sdhci-st.txt | 110 -----------------
+ .../devicetree/bindings/mmc/st,sdhci.yaml          | 105 +++++++++++++++++
+ .../devicetree/bindings/mmc/ti,omap2420-mmc.yaml   |  60 ++++++++++
+ Documentation/devicetree/bindings/mmc/ti-omap.txt  |  26 ----
+ MAINTAINERS                                        |   7 ++
+ arch/arm/boot/dts/st/stih407-family.dtsi           |   4 +-
+ arch/arm/boot/dts/st/stih410-b2260.dts             |   4 +-
+ arch/arm/boot/dts/st/stih418-b2199.dts             |   4 +-
+ arch/arm/boot/dts/st/stih418.dtsi                  |   2 +-
+ drivers/misc/cardreader/rtsx_usb.c                 |  26 +++-
+ drivers/mmc/core/block.c                           |   5 +
+ drivers/mmc/core/sd.c                              |   9 +-
+ drivers/mmc/host/bcm2835.c                         |  37 +++---
+ drivers/mmc/host/cqhci-core.c                      |  17 ++-
+ drivers/mmc/host/cqhci.h                           |   3 +-
+ drivers/mmc/host/dw_mmc-pltfm.h                    |   1 -
+ drivers/mmc/host/dw_mmc.c                          |   1 +
+ drivers/mmc/host/dw_mmc.h                          |   2 +
+ drivers/mmc/host/litex_mmc.c                       | 131 +++++----------------
+ drivers/mmc/host/moxart-mmc.c                      |  11 +-
+ drivers/mmc/host/mtk-sd.c                          |   2 +-
+ drivers/mmc/host/pxamci.c                          |   3 +-
+ drivers/mmc/host/rtsx_usb_sdmmc.c                  |  13 +-
+ drivers/mmc/host/sdhci-brcmstb.c                   |   2 +-
+ drivers/mmc/host/sdhci-esdhc-imx.c                 |   2 +-
+ drivers/mmc/host/sdhci-esdhc-mcf.c                 |  24 ++--
+ drivers/mmc/host/sdhci-msm.c                       |   2 +-
+ drivers/mmc/host/sdhci-of-arasan.c                 |   2 +-
+ drivers/mmc/host/sdhci-of-dwcmshc.c                |   3 +-
+ drivers/mmc/host/sdhci-of-ma35d1.c                 |   1 +
+ drivers/mmc/host/sdhci-pci-core.c                  |   2 +-
+ drivers/mmc/host/sdhci-pci-gli.c                   |   2 +-
+ drivers/mmc/host/sdhci-tegra.c                     |   2 +-
+ drivers/mmc/host/sdhci_am654.c                     |   2 +-
+ drivers/mmc/host/wbsd.c                            |   6 +-
+ include/linux/rtsx_usb.h                           |   3 +
+ include/linux/soc/pxa/cpu.h                        |  56 ---------
+ 39 files changed, 388 insertions(+), 400 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/mmc/sdhci-st.txt
+ create mode 100644 Documentation/devicetree/bindings/mmc/st,sdhci.yaml
+ create mode 100644 Documentation/devicetree/bindings/mmc/ti,omap2420-mmc.yaml
+ delete mode 100644 Documentation/devicetree/bindings/mmc/ti-omap.txt
+Merging mfd/for-mfd-next (6384e5c5e0bcf mfd: Unify style of spi_device_id arrays)
+$ git merge -m Merge branch 'for-mfd-next' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git mfd/for-mfd-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/mfd/cs42l43-i2c.c
+Auto-merging drivers/mfd/cs42l43-sdw.c
+Auto-merging drivers/mfd/motorola-cpcap.c
+Auto-merging drivers/mfd/ocelot-spi.c
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/Makefile         |   2 +-
+ .../bindings/mfd/marvell,88pm886-a1.yaml           |   2 +-
+ .../bindings/mfd/mediatek,mt8195-scpsys.yaml       |   1 +
+ .../devicetree/bindings/mfd/qcom,spmi-pmic.yaml    |   1 +
+ .../devicetree/bindings/mfd/syscon-common.yaml     |  34 ++
+ Documentation/devicetree/bindings/mfd/syscon.yaml  | 116 -----
+ MAINTAINERS                                        |  11 +
+ arch/arm/mach-ux500/cpu-db8500.c                   |   6 +-
+ drivers/clk/ux500/clk-prcmu.c                      |  20 +-
+ drivers/clk/ux500/u8500_of_clk.c                   |   2 +-
+ drivers/cpuidle/cpuidle-ux500.c                    |   6 +-
+ drivers/mfd/88pm886.c                              |  21 +-
+ drivers/mfd/Kconfig                                |   1 -
+ drivers/mfd/ab8500-core.c                          |   2 +-
+ drivers/mfd/altera-a10sr.c                         |   2 +-
+ drivers/mfd/arizona-spi.c                          |  12 +-
+ drivers/mfd/axp20x.c                               |   2 +-
+ drivers/mfd/cs40l50-spi.c                          |   4 +-
+ drivers/mfd/cs42l43-i2c.c                          |   2 -
+ drivers/mfd/cs42l43-sdw.c                          |   7 -
+ drivers/mfd/cs42l43.c                              |  15 +-
+ drivers/mfd/da9052-spi.c                           |  12 +-
+ drivers/mfd/db8500-prcmu.c                         |   6 +-
+ drivers/mfd/intel-m10-bmc-spi.c                    |   6 +-
+ drivers/mfd/ipaq-micro.c                           |   2 +-
+ drivers/mfd/madera-spi.c                           |  18 +-
+ drivers/mfd/motorola-cpcap.c                       |   6 +-
+ drivers/mfd/mt6397-core.c                          |   3 +
+ drivers/mfd/ocelot-spi.c                           |   2 +-
+ drivers/mfd/rk8xx-spi.c                            |   2 +-
+ drivers/mfd/rohm-bd71828.c                         | 125 +++--
+ drivers/mfd/rohm-bd718x7.c                         | 123 +++--
+ drivers/mfd/rsmu_spi.c                             |  12 +-
+ drivers/mfd/si476x-cmd.c                           |   1 -
+ drivers/mfd/si476x-i2c.c                           |  46 +-
+ drivers/mfd/sprd-sc27xx-spi.c                      |   3 +-
+ drivers/mfd/stmpe-spi.c                            |  12 +-
+ drivers/mfd/tps65912-spi.c                         |   2 +-
+ drivers/mfd/viperboard.c                           |   3 +-
+ drivers/mfd/wm831x-spi.c                           |  16 +-
+ drivers/regulator/db8500-prcmu.c                   |  12 +-
+ drivers/thermal/db8500_thermal.c                   |  10 +-
+ drivers/watchdog/db8500_wdt.c                      |  22 +-
+ include/linux/mfd/88pm886.h                        |   5 +
+ include/linux/mfd/cs42l43.h                        |   2 -
+ include/linux/mfd/db8500-prcmu.h                   | 252 ++++++++-
+ include/linux/mfd/dbx500-prcmu.h                   | 575 ---------------------
+ include/linux/mfd/si476x-core.h                    |   5 +-
+ include/linux/mfd/si476x-platform.h                |   2 -
+ sound/soc/ux500/ux500_msp_dai.c                    |   2 +-
+ 50 files changed, 637 insertions(+), 919 deletions(-)
+ delete mode 100644 include/linux/mfd/dbx500-prcmu.h
+Merging backlight/for-backlight-next (4b3f5baf67dea MAINTAINERS: Update my email address for the AW99706 backlight driver)
+$ git merge -m Merge branch 'for-backlight-next' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git backlight/for-backlight-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../bindings/leds/backlight/awinic,aw99706.yaml    |   2 +-
+ .../bindings/leds/backlight/silergy,sy7758.yaml    |  52 +++++
+ MAINTAINERS                                        |   2 +-
+ drivers/video/backlight/Kconfig                    |   8 +
+ drivers/video/backlight/Makefile                   |   1 +
+ drivers/video/backlight/sy7758.c                   | 259 +++++++++++++++++++++
+ 6 files changed, 322 insertions(+), 2 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/leds/backlight/silergy,sy7758.yaml
+ create mode 100644 drivers/video/backlight/sy7758.c
+Merging battery/for-next (a888754e51e91 Documentation: ABI: sysfs-class-reboot-mode-reboot_modes: fix doc warnings)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git battery/for-next
+Already up to date.
+Merging regulator/for-next (e370243ea7670 Merge remote-tracking branch 'regulator/for-7.3' into regulator-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git regulator/for-next
+Merge made by the 'ort' strategy.
+ .../regulator/qcom,sdm845-refgen-regulator.yaml    |  31 +-
+ .../regulator/qcom,usb-vbus-regulator.yaml         |  50 ++-
+ .../bindings/regulator/richtek,rtq2208.yaml        |   4 +-
+ .../bindings/regulator/rohm,bd71815-regulator.yaml |  27 +-
+ .../bindings/regulator/rohm,bd71828-regulator.yaml |  20 +-
+ .../bindings/regulator/rohm,bd71837-regulator.yaml |  16 +-
+ .../bindings/regulator/rohm,bd71847-regulator.yaml |  16 +-
+ .../bindings/regulator/rohm,bd72720-regulator.yaml |  40 +--
+ .../bindings/regulator/rohm,pmic-states.yaml       |  57 ++++
+ drivers/regulator/88pg86x.c                        |   4 +-
+ drivers/regulator/Kconfig                          |   7 +
+ drivers/regulator/Makefile                         |   1 +
+ drivers/regulator/act8865-regulator.c              |   2 +-
+ drivers/regulator/ad5398.c                         |   4 +-
+ drivers/regulator/adp5055-regulator.c              |   6 +-
+ drivers/regulator/as3722-regulator.c               |   1 -
+ drivers/regulator/aw37503-regulator.c              |   4 +-
+ drivers/regulator/da9121-regulator.c               |  20 +-
+ drivers/regulator/da9210-regulator.c               |   4 +-
+ drivers/regulator/da9211-regulator.c               |  18 +-
+ drivers/regulator/fan53880.c                       |   4 +-
+ drivers/regulator/isl6271a-regulator.c             |   2 +-
+ drivers/regulator/isl9305.c                        |   4 +-
+ drivers/regulator/lp3971.c                         |   2 +-
+ drivers/regulator/lp3972.c                         |   2 +-
+ drivers/regulator/lp872x.c                         |   4 +-
+ drivers/regulator/lp8755.c                         |   4 +-
+ drivers/regulator/ltc3589.c                        |   6 +-
+ drivers/regulator/ltc3676.c                        |  12 +-
+ drivers/regulator/max14577-regulator.c             | 103 +++++-
+ drivers/regulator/max1586.c                        |   2 +-
+ drivers/regulator/max20086-regulator.c             |   8 +-
+ drivers/regulator/max20411-regulator.c             |   2 +-
+ drivers/regulator/max77503-regulator.c             |   2 +-
+ drivers/regulator/max77675-regulator.c             |   2 +-
+ drivers/regulator/max77826-regulator.c             |   2 +-
+ drivers/regulator/max77838-regulator.c             |   2 +-
+ drivers/regulator/max77857-regulator.c             |   8 +-
+ drivers/regulator/max8649.c                        |   2 +-
+ drivers/regulator/max8893.c                        |   2 +-
+ drivers/regulator/max8952.c                        |   2 +-
+ drivers/regulator/max8973-regulator.c              |   6 +-
+ drivers/regulator/max8998.c                        |   1 -
+ drivers/regulator/mcp16502.c                       |  27 +-
+ drivers/regulator/mp5416.c                         |   6 +-
+ drivers/regulator/mp8859.c                         |   4 +-
+ drivers/regulator/mp886x.c                         |   6 +-
+ drivers/regulator/mpq7920.c                        |   4 +-
+ drivers/regulator/mt6311-regulator.c               |   4 +-
+ drivers/regulator/pf530x-regulator.c               |   8 +-
+ drivers/regulator/pf8x00-regulator.c               |   8 +-
+ drivers/regulator/pv88060-regulator.c              |   4 +-
+ drivers/regulator/pv88080-regulator.c              |   8 +-
+ drivers/regulator/pv88090-regulator.c              |   4 +-
+ drivers/regulator/qcom-refgen-regulator.c          | 113 ++++++-
+ drivers/regulator/qcom_usb_vbus-regulator.c        | 102 +++++-
+ drivers/regulator/rtq2208-regulator.c              |   7 +-
+ drivers/regulator/sc2730-regulator.c               | 375 +++++++++++++++++++++
+ drivers/regulator/sgm3804-regulator.c              |   2 +-
+ drivers/regulator/slg51000-regulator.c             |   4 +-
+ drivers/regulator/sy8106a-regulator.c              |   2 +-
+ drivers/regulator/sy8824x.c                        |   8 +-
+ drivers/regulator/sy8827n.c                        |   4 +-
+ drivers/regulator/tps51632-regulator.c             |   4 +-
+ drivers/regulator/tps62360-regulator.c             |  10 +-
+ drivers/regulator/tps6286x-regulator.c             |  10 +-
+ drivers/regulator/tps6287x-regulator.c             |  10 +-
+ drivers/regulator/tps65023-regulator.c             |   6 +-
+ drivers/regulator/tps65132-regulator.c             |   4 +-
+ drivers/regulator/tps6594-regulator.c              |  19 +-
+ include/linux/mfd/max14577-private.h               |   3 +
+ 71 files changed, 997 insertions(+), 285 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/regulator/rohm,pmic-states.yaml
+ create mode 100644 drivers/regulator/sc2730-regulator.c
+Merging security/next (0cee720cfd514 selftests/lsm: Fix memory leak in attr_lsm_count)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git security/next
+Merge made by the 'ort' strategy.
+ Documentation/security/credentials.rst                    |  6 ++----
+ Documentation/translations/zh_CN/security/credentials.rst |  4 +---
+ include/linux/cred.h                                      |  1 -
+ rust/helpers/task.c                                       |  5 -----
+ rust/kernel/task.rs                                       | 11 ++---------
+ security/security.c                                       |  5 ++---
+ tools/testing/selftests/lsm/common.c                      |  4 +++-
+ 7 files changed, 10 insertions(+), 26 deletions(-)
+Merging apparmor/apparmor-next (c431011ab65fd apparmor: replace decompress_zstd() prototype with its entity)
+$ git merge -m Merge branch 'apparmor-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jj/linux-apparmor apparmor/apparmor-next
+Merge made by the 'ort' strategy.
+ security/apparmor/af_unix.c               |   1 +
+ security/apparmor/apparmorfs.c            | 199 ++++++++++++++++++++++++------
+ security/apparmor/domain.c                |   1 +
+ security/apparmor/include/apparmorfs.h    |   3 +
+ security/apparmor/include/capability.h    |   1 +
+ security/apparmor/include/path.h          |   3 +
+ security/apparmor/include/policy.h        |   4 +-
+ security/apparmor/include/policy_unpack.h |   4 +-
+ security/apparmor/include/procattr.h      |   2 +
+ security/apparmor/include/task.h          |   5 +
+ security/apparmor/policy.c                |   8 +-
+ security/apparmor/policy_unpack.c         |  24 +++-
+ 12 files changed, 209 insertions(+), 46 deletions(-)
+Merging integrity/next-integrity (1da739feb31b4 ima: measure userspace policy writes before parsing)
+$ git merge -m Merge branch 'next-integrity' of https://git.kernel.org/pub/scm/linux/kernel/git/zohar/linux-integrity integrity/next-integrity
+Merge made by the 'ort' strategy.
+ include/linux/lsm_hooks.h                 |  2 +
+ security/integrity/ima/Kconfig            | 10 ++++
+ security/integrity/ima/ima.h              |  5 ++
+ security/integrity/ima/ima_efi.c          |  2 +
+ security/integrity/ima/ima_fs.c           |  9 ++-
+ security/integrity/ima/ima_init.c         | 15 +++--
+ security/integrity/ima/ima_main.c         | 26 +++++++++
+ security/integrity/ima/ima_policy.c       | 95 ++++++++++++++++++++++++++++++-
+ security/integrity/ima/ima_template_lib.c |  3 +-
+ security/lsm_init.c                       | 13 ++++-
+ 10 files changed, 170 insertions(+), 10 deletions(-)
+Merging selinux/next (f47f45c7f1ca2 Automated merge of 'dev' into 'next')
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/selinux.git selinux/next
+Merge made by the 'ort' strategy.
+ security/selinux/avc.c         |  3 +-
+ security/selinux/hooks.c       | 69 +++++++++++++++++++++++++-----------------
+ security/selinux/ima.c         | 65 +++++++++++++++++++--------------------
+ security/selinux/include/ima.h |  4 +++
+ security/selinux/selinuxfs.c   | 19 ++++++++----
+ security/selinux/ss/policydb.c | 11 +++++--
+ security/selinux/ss/services.c | 18 +++++------
+ 7 files changed, 110 insertions(+), 79 deletions(-)
+Merging smack/next (a7c44fd9f80e3 smack: restrict smackfs/{direct,mapped} values to 0-255)
+$ git merge -m Merge branch 'next' of https://github.com/cschaufler/smack-next smack/next
+Auto-merging security/smack/smack_lsm.c
+Merge made by the 'ort' strategy.
+ security/smack/smack.h        |   6 +-
+ security/smack/smack_access.c |   9 ++
+ security/smack/smack_lsm.c    |  69 +++++++++----
+ security/smack/smackfs.c      | 224 +++++++++++++-----------------------------
+ 4 files changed, 130 insertions(+), 178 deletions(-)
+Merging tomoyo/master (58717b2a1365d Merge tag 'sound-7.2-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound)
+$ git merge -m Merge branch 'master' of git://git.code.sf.net/p/tomoyo/tomoyo.git tomoyo/master
+Already up to date.
+Merging tpmdd-tpm/for-next-tpm (4ece5759c1a29 tpm: tpm_i2c_nuvoton: disable IRQ on wait timeout)
+$ git merge -m Merge branch 'for-next-tpm' of https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git tpmdd-tpm/for-next-tpm
+Merge made by the 'ort' strategy.
+ drivers/char/tpm/tpm_i2c_nuvoton.c | 6 ++++--
+ 1 file changed, 4 insertions(+), 2 deletions(-)
+Merging tpmdd-keys/for-next-keys (51cb1aa1250c3 Merge tag 'loongarch-7.2' of git://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson)
+$ git merge -m Merge branch 'for-next-keys' of https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git tpmdd-keys/for-next-keys
+Already up to date.
+Merging watchdog/watchdog-next (94794a4483523 dt-bindings: watchdog: apple,wdt: Add t6030 and t6031 compatibles)
+$ git merge -m Merge branch 'watchdog-next' of https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git watchdog/watchdog-next
+Auto-merging drivers/watchdog/cros_ec_wdt.c
+Auto-merging drivers/watchdog/max63xx_wdt.c
+Auto-merging drivers/watchdog/max77620_wdt.c
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/watchdog/apple,wdt.yaml    |   3 +
+ .../devicetree/bindings/watchdog/qcom-wdt.yaml     |   1 +
+ drivers/watchdog/Kconfig                           |  31 ++-
+ drivers/watchdog/Makefile                          |   2 +
+ drivers/watchdog/cros_ec_wdt.c                     |   4 +-
+ drivers/watchdog/lenovo_se30g2_se60_wdt.c          | 289 +++++++++++++++++++++
+ drivers/watchdog/max63xx_wdt.c                     |  14 +-
+ drivers/watchdog/max77620_wdt.c                    |   6 +-
+ drivers/watchdog/pretimeout_dump.c                 |  45 ++++
+ drivers/watchdog/w83627hf_wdt.c                    |  22 +-
+ drivers/watchdog/watchdog_pretimeout.h             |   4 +-
+ 11 files changed, 402 insertions(+), 19 deletions(-)
+ create mode 100644 drivers/watchdog/lenovo_se30g2_se60_wdt.c
+ create mode 100644 drivers/watchdog/pretimeout_dump.c
+Merging iommu/next (750356af88d61 Merge branches 'mediatek', 'qualcomm/msm', 'riscv', 'amd/amd-vi' and 'core' into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git iommu/next
+Merge made by the 'ort' strategy.
+ drivers/iommu/amd/init.c                    | 57 ++++++++++++++++++++++-------
+ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c |  9 ++++-
+ drivers/iommu/intel/iommu.c                 | 15 ++++++--
+ drivers/iommu/msm_iommu.c                   |  2 +-
+ drivers/iommu/mtk_iommu_v1.c                |  2 +-
+ drivers/iommu/riscv/iommu-pci.c             |  6 +--
+ drivers/pci/ats.c                           | 13 +++++--
+ rust/kernel/iommu/pgtable.rs                |  3 +-
+ 8 files changed, 78 insertions(+), 29 deletions(-)
+Merging audit/next (1715cd1303c70 Automated merge of 'dev' into 'next')
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git audit/next
+Merge made by the 'ort' strategy.
+ include/asm-generic/audit_change_attr.h |  3 +++
+ include/asm-generic/audit_read.h        | 31 +++++++++++++++++++++++++++++++
+ include/asm-generic/audit_write.h       |  3 +++
+ 3 files changed, 37 insertions(+)
+Merging devicetree/for-next (bcef80df6d6f4 dt-bindings: interconnect: qcom-bwmon: Add Shikra cpu-bwmon compatible)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git devicetree/for-next
+Auto-merging Documentation/devicetree/bindings/arm/qcom.yaml
+Auto-merging Documentation/devicetree/bindings/clock/qcom,gcc-msm8953.yaml
+Auto-merging Documentation/devicetree/bindings/clock/qcom,ipq9574-gcc.yaml
+Auto-merging Documentation/devicetree/bindings/clock/qcom,rpmcc.yaml
+Auto-merging Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml
+Auto-merging Documentation/devicetree/bindings/firmware/qcom,scm.yaml
+Auto-merging Documentation/devicetree/bindings/regulator/richtek,rtq2208.yaml
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/arm/qcom-soc.yaml          |   4 +-
+ Documentation/devicetree/bindings/arm/qcom.yaml    |   4 +-
+ .../bindings/arm/samsung/samsung-soc.yaml          |   4 +-
+ Documentation/devicetree/bindings/arm/xen.txt      |  62 ------------
+ Documentation/devicetree/bindings/arm/xen.yaml     | 110 +++++++++++++++++++++
+ .../devicetree/bindings/bus/ti,da850-mstpri.txt    |  20 ----
+ .../devicetree/bindings/bus/ti,da850-mstpri.yaml   |  37 +++++++
+ .../bindings/clock/qcom,dispcc-sm8x50.yaml         |   2 +-
+ .../bindings/clock/qcom,gcc-apq8064.yaml           |   2 +-
+ .../bindings/clock/qcom,gcc-apq8084.yaml           |   2 +-
+ .../bindings/clock/qcom,gcc-ipq6018.yaml           |   2 +-
+ .../bindings/clock/qcom,gcc-ipq8064.yaml           |   2 +-
+ .../bindings/clock/qcom,gcc-mdm9607.yaml           |   2 +-
+ .../bindings/clock/qcom,gcc-mdm9615.yaml           |   2 +-
+ .../bindings/clock/qcom,gcc-msm8660.yaml           |   2 +-
+ .../bindings/clock/qcom,gcc-msm8909.yaml           |   2 +-
+ .../bindings/clock/qcom,gcc-msm8916.yaml           |   2 +-
+ .../bindings/clock/qcom,gcc-msm8953.yaml           |   2 +-
+ .../bindings/clock/qcom,gcc-msm8974.yaml           |   2 +-
+ .../devicetree/bindings/clock/qcom,gcc-sdm660.yaml |   2 +-
+ .../devicetree/bindings/clock/qcom,gpucc.yaml      |   2 +-
+ .../bindings/clock/qcom,ipq5018-gcc.yaml           |   2 +-
+ .../bindings/clock/qcom,ipq9574-gcc.yaml           |   2 +-
+ .../bindings/clock/qcom,qca8k-nsscc.yaml           |   2 +-
+ .../bindings/clock/qcom,qcm2290-gpucc.yaml         |   2 +-
+ .../devicetree/bindings/clock/qcom,rpmcc.yaml      |   2 +-
+ .../bindings/clock/qcom,sc7280-lpasscorecc.yaml    |   2 +-
+ .../bindings/clock/qcom,sc8280xp-lpasscc.yaml      |   2 +-
+ .../bindings/clock/qcom,sm6115-lpasscc.yaml        |   2 +-
+ .../bindings/clock/qcom,sm8350-videocc.yaml        |   2 +-
+ .../devicetree/bindings/clock/qcom,videocc.yaml    |   2 +-
+ .../bindings/clock/samsung,exynos5260-clock.yaml   |   6 +-
+ .../bindings/clock/samsung,exynos5410-clock.yaml   |   2 +-
+ .../bindings/clock/samsung,exynos5433-clock.yaml   |   2 +-
+ .../bindings/clock/samsung,exynos7-clock.yaml      |   2 +-
+ .../bindings/clock/samsung,exynos850-clock.yaml    |   2 +-
+ .../bindings/clock/samsung,exynosautov9-clock.yaml |   2 +-
+ .../clock/samsung,exynosautov920-clock.yaml        |   2 +-
+ .../bindings/clock/samsung,s5pv210-clock.yaml      |   2 +-
+ .../bindings/display/msm/dsi-controller-main.yaml  |  20 ++--
+ .../bindings/display/samsung/samsung,fimd.yaml     |   4 +-
+ .../bindings/display/verisilicon,dc.yaml           |   2 +
+ .../devicetree/bindings/firmware/qcom,scm.yaml     |   3 +
+ .../bindings/i2c/samsung,s3c2410-i2c.yaml          |   2 +-
+ .../bindings/interconnect/qcom,msm8998-bwmon.yaml  |   5 +-
+ .../bindings/interconnect/samsung,exynos-bus.yaml  |  14 +--
+ .../devicetree/bindings/leds/qcom,pm8058-led.yaml  |   4 +-
+ .../devicetree/bindings/leds/skyworks,aat1290.yaml |   6 +-
+ .../devicetree/bindings/media/cec/cec-gpio.yaml    |   2 +-
+ .../bindings/mmc/samsung,exynos-dw-mshc.yaml       |   2 +-
+ .../bindings/phy/samsung,mipi-video-phy.yaml       |   4 +-
+ .../devicetree/bindings/phy/samsung,usb2-phy.yaml  |   2 +-
+ .../bindings/phy/samsung,usb3-drd-phy.yaml         |   2 +-
+ .../bindings/pinctrl/samsung,pinctrl.yaml          |   2 +-
+ .../bindings/power/renesas,rcar-sysc.yaml          |   2 +-
+ .../bindings/power/reset/restart-handler.yaml      |   8 +-
+ .../bindings/regulator/maxim,max77802.yaml         |   4 +-
+ .../bindings/regulator/richtek,rtq2208.yaml        |   2 +-
+ .../bindings/serial/qcom,msm-uartdm.yaml           |   2 +-
+ .../devicetree/bindings/slimbus/slimbus.yaml       |   4 +-
+ .../bindings/soc/qcom/qcom,apr-services.yaml       |   2 +-
+ .../bindings/soc/qcom/qcom,rpmh-rsc.yaml           |   8 +-
+ .../devicetree/bindings/soc/qcom/qcom,wcnss.yaml   |   2 +-
+ .../bindings/soc/renesas/renesas-soc.yaml          |   4 +-
+ .../devicetree/bindings/sound/qcom,q6asm-dais.yaml |   2 +-
+ .../devicetree/bindings/sram/qcom,imem.yaml        |  20 ++++
+ .../bindings/thermal/samsung,exynos-thermal.yaml   |   4 +-
+ .../devicetree/bindings/usb/qcom,dwc3.yaml         |  12 +--
+ .../devicetree/bindings/usb/qcom,snps-dwc3.yaml    |  12 +--
+ .../bindings/watchdog/microchip,pic32-dmt.txt      |  19 ----
+ .../bindings/watchdog/microchip,pic32-wdt.txt      |  18 ----
+ .../bindings/watchdog/microchip,pic32mzda-dmt.yaml |  45 +++++++++
+ .../bindings/watchdog/microchip,pic32mzda-wdt.yaml |  44 +++++++++
+ 73 files changed, 371 insertions(+), 226 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/arm/xen.txt
+ create mode 100644 Documentation/devicetree/bindings/arm/xen.yaml
+ delete mode 100644 Documentation/devicetree/bindings/bus/ti,da850-mstpri.txt
+ create mode 100644 Documentation/devicetree/bindings/bus/ti,da850-mstpri.yaml
+ delete mode 100644 Documentation/devicetree/bindings/watchdog/microchip,pic32-dmt.txt
+ delete mode 100644 Documentation/devicetree/bindings/watchdog/microchip,pic32-wdt.txt
+ create mode 100644 Documentation/devicetree/bindings/watchdog/microchip,pic32mzda-dmt.yaml
+ create mode 100644 Documentation/devicetree/bindings/watchdog/microchip,pic32mzda-wdt.yaml
+Merging dt-krzk/for-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git dt-krzk/for-next
+Already up to date.
+Merging mailbox/for-next (36cac4b5101f8 mailbox: imx: Don't force-thread the primary handler)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jassibrar/mailbox.git mailbox/for-next
+Already up to date.
+Merging spi/for-next (97a74f7b0a6a0 Merge remote-tracking branch 'spi/for-7.3' into spi-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git spi/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/spi/cavium,octeon-3010-spi.yaml       |  61 ++++++++++++
+ .../bindings/spi/qcom,sa8255p-geni-spi.yaml        |  64 +++++++++++++
+ .../devicetree/bindings/spi/spi-octeon.txt         |  33 -------
+ .../devicetree/bindings/spi/st,stm32-qspi.yaml     |   3 +
+ drivers/spi/atmel-quadspi.c                        |  54 +++++------
+ drivers/spi/spi-altera-platform.c                  |  13 +--
+ drivers/spi/spi-atcspi200.c                        |   5 +-
+ drivers/spi/spi-bcm-qspi.c                         |   6 +-
+ drivers/spi/spi-bcm2835.c                          |   2 +-
+ drivers/spi/spi-bcm63xx-hsspi.c                    |   6 +-
+ drivers/spi/spi-bcm63xx.c                          |   5 +-
+ drivers/spi/spi-bcmbca-hsspi.c                     |   6 +-
+ drivers/spi/spi-fsl-dspi.c                         |  21 ++++-
+ drivers/spi/spi-geni-qcom.c                        | 105 ++++++++++-----------
+ drivers/spi/spi-npcm-fiu.c                         |  15 +--
+ drivers/spi/spi-nxp-fspi.c                         |  31 ++++--
+ drivers/spi/spi-qpic-snand.c                       |   6 +-
+ drivers/spi/spi-realtek-rtl.c                      |  26 ++---
+ 18 files changed, 300 insertions(+), 162 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/spi/cavium,octeon-3010-spi.yaml
+ create mode 100644 Documentation/devicetree/bindings/spi/qcom,sa8255p-geni-spi.yaml
+ delete mode 100644 Documentation/devicetree/bindings/spi/spi-octeon.txt
+Merging tip/master (b94f4faa5968a Merge branch into tip/master: 'x86/tdx')
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git tip/master
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging MAINTAINERS
+Auto-merging arch/arm/mm/Kconfig
+Auto-merging arch/arm64/Kconfig
+Auto-merging arch/arm64/kernel/ptrace.c
+Auto-merging arch/loongarch/Kconfig
+Auto-merging arch/mips/kernel/vdso.c
+Auto-merging arch/riscv/Kconfig
+Auto-merging arch/s390/Kconfig
+Auto-merging arch/x86/Kconfig
+Auto-merging arch/x86/events/intel/ds.c
+Auto-merging arch/x86/include/asm/cpufeatures.h
+Auto-merging arch/x86/include/asm/pgtable.h
+Auto-merging arch/x86/include/asm/pgtable_types.h
+Auto-merging arch/x86/kvm/vmx/vmx.c
+Auto-merging arch/x86/mm/init_64.c
+Auto-merging arch/x86/mm/pat/set_memory.c
+Auto-merging drivers/irqchip/irq-gic-v3-its.c
+Auto-merging drivers/irqchip/irq-gic-v5-iwb.c
+Auto-merging drivers/thermal/intel/therm_throt.c
+Auto-merging include/linux/cpuset.h
+Auto-merging include/linux/sched.h
+Auto-merging init/Kconfig
+Auto-merging kernel/cgroup/cpuset.c
+Auto-merging kernel/events/core.c
+Auto-merging kernel/events/uprobes.c
+Auto-merging kernel/fork.c
+Auto-merging kernel/irq/manage.c
+Auto-merging kernel/locking/lockdep.c
+Auto-merging tools/lib/bpf/usdt.c
+Auto-merging tools/objtool/check.c
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/kernel-parameters.txt    |    3 -
+ Documentation/admin-guide/sysctl/kernel.rst        |   17 +
+ Documentation/arch/x86/amd-memory-encryption.rst   |    2 +-
+ Documentation/core-api/entry.rst                   |   64 +-
+ .../interrupt-controller/realtek,rtl-intc.yaml     |    5 +-
+ MAINTAINERS                                        |    1 +
+ arch/Kconfig                                       |   17 +-
+ arch/alpha/kernel/ptrace.c                         |    4 +-
+ arch/arc/kernel/ptrace.c                           |    2 +-
+ arch/arm/kernel/ptrace.c                           |    4 +-
+ arch/arm/mm/Kconfig                                |    1 -
+ arch/arm/vdso/vdso.lds.S                           |    2 +
+ arch/arm/vdso/vgettimeofday.c                      |   14 +-
+ arch/arm64/Kconfig                                 |    1 -
+ arch/arm64/kernel/ptrace.c                         |    4 +-
+ arch/arm64/kernel/vdso32/vdso.lds.S                |    2 +
+ arch/arm64/kernel/vdso32/vgettimeofday.c           |   14 +-
+ arch/csky/kernel/ptrace.c                          |    4 +-
+ arch/hexagon/kernel/traps.c                        |    2 +-
+ arch/loongarch/Kconfig                             |    2 +-
+ arch/loongarch/include/asm/syscall.h               |    6 -
+ arch/loongarch/kernel/syscall.c                    |   17 +-
+ arch/m68k/kernel/ptrace.c                          |    4 +-
+ arch/microblaze/kernel/ptrace.c                    |    2 +-
+ arch/mips/Kconfig                                  |    1 -
+ arch/mips/kernel/ptrace.c                          |    4 +-
+ arch/mips/kernel/vdso.c                            |    4 +-
+ arch/mips/vdso/vdso.lds.S                          |    2 +
+ arch/mips/vdso/vgettimeofday.c                     |    3 +
+ arch/nios2/kernel/ptrace.c                         |    2 +-
+ arch/openrisc/kernel/ptrace.c                      |    2 +-
+ arch/parisc/kernel/ptrace.c                        |   12 +-
+ arch/parisc/kernel/smp.c                           |    2 +-
+ arch/powerpc/Kconfig                               |    2 +-
+ arch/powerpc/include/asm/syscall.h                 |    5 -
+ arch/powerpc/kernel/syscall.c                      |    5 +-
+ arch/powerpc/kernel/vdso/gettimeofday.S            |    8 +
+ arch/powerpc/kernel/vdso/vdso32.lds.S              |    8 +-
+ arch/powerpc/kernel/vdso/vgettimeofday.c           |   16 +-
+ arch/riscv/Kconfig                                 |    8 +-
+ arch/riscv/include/asm/syscall.h                   |    5 -
+ arch/riscv/kernel/traps.c                          |   14 +-
+ arch/s390/Kconfig                                  |    1 -
+ arch/s390/include/asm/syscall.h                    |    5 -
+ arch/s390/kernel/syscall.c                         |   11 +-
+ arch/sh/kernel/ptrace_32.c                         |    4 +-
+ arch/sparc/Kconfig                                 |    1 -
+ arch/sparc/kernel/ptrace_32.c                      |    2 +-
+ arch/sparc/kernel/ptrace_64.c                      |    2 +-
+ arch/sparc/vdso/vclock_gettime.c                   |    4 +
+ arch/sparc/vdso/vdso32/vdso32.lds.S                |    6 +-
+ arch/um/include/asm/cpufeature.h                   |   16 +-
+ arch/um/kernel/ptrace.c                            |    2 +-
+ arch/um/kernel/skas/syscall.c                      |    2 +-
+ arch/um/kernel/um_arch.c                           |   12 -
+ arch/x86/Kconfig                                   |    2 +-
+ arch/x86/boot/compressed/head_64.S                 |   18 +-
+ arch/x86/coco/tdx/tdx.c                            |   10 +-
+ arch/x86/entry/entry_64_fred.S                     |    2 +
+ arch/x86/entry/syscall_32.c                        |   70 +-
+ arch/x86/entry/syscall_64.c                        |   59 +-
+ arch/x86/entry/vdso/common/vclock_gettime.c        |   20 +-
+ arch/x86/entry/vdso/vdso32/vdso32.lds.S            |    2 +
+ arch/x86/entry/vsyscall/vsyscall_64.c              |   14 +-
+ arch/x86/events/amd/core.c                         |    2 +-
+ arch/x86/events/amd/uncore.c                       |   31 +
+ arch/x86/events/core.c                             |    3 +-
+ arch/x86/events/intel/core.c                       |   62 +-
+ arch/x86/events/intel/ds.c                         |   13 -
+ arch/x86/events/intel/lbr.c                        |   38 +-
+ arch/x86/events/intel/uncore.c                     |  227 ++--
+ arch/x86/events/intel/uncore.h                     |   39 +-
+ arch/x86/events/intel/uncore_discovery.c           |   21 +-
+ arch/x86/events/intel/uncore_discovery.h           |    6 +-
+ arch/x86/events/intel/uncore_nhmex.c               |    3 +-
+ arch/x86/events/intel/uncore_snb.c                 |   82 +-
+ arch/x86/events/intel/uncore_snbep.c               |   77 +-
+ arch/x86/events/perf_event.h                       |    6 +-
+ arch/x86/hyperv/hv_apic.c                          |   12 +-
+ arch/x86/include/asm/alternative.h                 |   68 +-
+ arch/x86/include/asm/archrandom.h                  |    4 +-
+ arch/x86/include/asm/cmpxchg_32.h                  |    4 +-
+ arch/x86/include/asm/cpufeature.h                  |   10 +-
+ arch/x86/include/asm/cpufeatures.h                 |    2 +-
+ arch/x86/include/asm/debugreg.h                    |    2 +-
+ arch/x86/include/asm/entry-common.h                |    1 -
+ arch/x86/include/asm/insn-eval.h                   |   36 +
+ arch/x86/include/asm/linkage.h                     |    2 +-
+ arch/x86/include/asm/microcode.h                   |    2 -
+ arch/x86/include/asm/nospec-branch.h               |    1 +
+ arch/x86/include/asm/pgtable.h                     |    6 +-
+ arch/x86/include/asm/pgtable_types.h               |    2 +-
+ arch/x86/include/asm/processor.h                   |    1 +
+ arch/x86/include/asm/resctrl.h                     |    5 +-
+ arch/x86/include/asm/sync_core.h                   |    4 +-
+ arch/x86/include/asm/syscall.h                     |   10 +-
+ arch/x86/kernel/acpi/sleep.c                       |   20 +-
+ arch/x86/kernel/alternative.c                      |  202 +---
+ arch/x86/kernel/apic/apic.c                        |   39 +-
+ arch/x86/kernel/apic/apic_numachip.c               |    2 +-
+ arch/x86/kernel/cpu/amd.c                          |   34 +-
+ arch/x86/kernel/cpu/bugs.c                         |   10 +-
+ arch/x86/kernel/cpu/cacheinfo.c                    |    2 +-
+ arch/x86/kernel/cpu/centaur.c                      |   35 +-
+ arch/x86/kernel/cpu/common.c                       |   12 +-
+ arch/x86/kernel/cpu/feat_ctl.c                     |   25 +-
+ arch/x86/kernel/cpu/hygon.c                        |    5 +-
+ arch/x86/kernel/cpu/intel.c                        |   43 +-
+ arch/x86/kernel/cpu/mce/amd.c                      |   87 +-
+ arch/x86/kernel/cpu/mce/core.c                     |    2 +-
+ arch/x86/kernel/cpu/mce/p5.c                       |   16 +-
+ arch/x86/kernel/cpu/mce/winchip.c                  |   10 +-
+ arch/x86/kernel/cpu/microcode/intel.c              |   36 -
+ arch/x86/kernel/cpu/mtrr/amd.c                     |   31 +-
+ arch/x86/kernel/cpu/mtrr/centaur.c                 |   18 +-
+ arch/x86/kernel/cpu/mtrr/cleanup.c                 |   18 +-
+ arch/x86/kernel/cpu/mtrr/generic.c                 |   97 +-
+ arch/x86/kernel/cpu/mtrr/mtrr.c                    |    4 +-
+ arch/x86/kernel/cpu/resctrl/core.c                 |   25 +-
+ arch/x86/kernel/cpu/resctrl/ctrlmondata.c          |    4 +-
+ arch/x86/kernel/cpu/resctrl/intel_aet.c            |    5 +-
+ arch/x86/kernel/cpu/resctrl/monitor.c              |   29 +-
+ arch/x86/kernel/cpu/resctrl/pseudo_lock.c          |   12 +-
+ arch/x86/kernel/cpu/resctrl/rdtgroup.c             |    4 +-
+ arch/x86/kernel/cpu/transmeta.c                    |    9 +-
+ arch/x86/kernel/cpu/zhaoxin.c                      |   12 +-
+ arch/x86/kernel/fpu/core.c                         |   20 +-
+ arch/x86/kernel/fpu/signal.c                       |    2 +-
+ arch/x86/kernel/kprobes/core.c                     |    3 -
+ arch/x86/kernel/kprobes/opt.c                      |    3 +-
+ arch/x86/kernel/ldt.c                              |    2 +-
+ arch/x86/kernel/module.c                           |   13 +-
+ arch/x86/kernel/process.c                          |   10 +-
+ arch/x86/kernel/process_64.c                       |    6 +-
+ arch/x86/kernel/signal_32.c                        |    2 +-
+ arch/x86/kernel/smpboot.c                          |    3 -
+ arch/x86/kernel/time.c                             |    2 +-
+ arch/x86/kernel/tsc.c                              |    6 +-
+ arch/x86/kernel/tsc_msr.c                          |   15 +-
+ arch/x86/kernel/uprobes.c                          |  408 ++++---
+ arch/x86/kernel/vmlinux.lds.S                      |   20 +-
+ arch/x86/kvm/cpuid.c                               |    4 +-
+ arch/x86/kvm/emulate.c                             |   26 +-
+ arch/x86/kvm/svm/svm.c                             |   22 +-
+ arch/x86/kvm/vmx/vmx.c                             |   30 +-
+ arch/x86/kvm/x86.c                                 |   10 +-
+ arch/x86/kvm/xen.c                                 |    2 +-
+ arch/x86/lib/copy_mc.c                             |    4 +-
+ arch/x86/lib/msr-smp.c                             |    8 +-
+ arch/x86/lib/usercopy_32.c                         |    2 +-
+ arch/x86/mm/extable.c                              |    2 +-
+ arch/x86/mm/fault.c                                |    8 +-
+ arch/x86/mm/ident_map.c                            |    2 +-
+ arch/x86/mm/init_64.c                              |   16 +-
+ arch/x86/mm/pat/set_memory.c                       |   46 +-
+ arch/x86/mm/pgtable.c                              |    4 +-
+ arch/x86/mm/tlb.c                                  |   14 +-
+ arch/x86/pci/mmconfig-shared.c                     |    8 +-
+ arch/x86/platform/intel-quark/imr.c                |    4 +-
+ arch/x86/platform/olpc/olpc-xo1-sci.c              |   11 +-
+ arch/x86/ras/Kconfig                               |   23 -
+ arch/x86/tools/relocs.c                            |    2 +-
+ arch/x86/virt/vmx/tdx/tdx.c                        |    6 +-
+ arch/x86/xen/mmu_pv.c                              |   22 +-
+ arch/xtensa/kernel/ptrace.c                        |    5 +-
+ drivers/acpi/processor_perflib.c                   |   11 +-
+ drivers/acpi/processor_throttling.c                |   14 +-
+ drivers/cpufreq/acpi-cpufreq.c                     |   22 +-
+ drivers/cpufreq/e_powersaver.c                     |   49 +-
+ drivers/cpufreq/longhaul.c                         |   15 +-
+ drivers/cpufreq/longrun.c                          |   77 +-
+ drivers/cpufreq/powernow-k6.c                      |   12 +-
+ drivers/cpufreq/powernow-k8.c                      |   67 +-
+ drivers/cpufreq/speedstep-centrino.c               |   16 +-
+ drivers/cpufreq/speedstep-lib.c                    |   63 +-
+ drivers/edac/ie31200_edac.c                        |   10 +-
+ drivers/edac/mce_amd.c                             |    8 +-
+ drivers/firewire/ohci.c                            |    2 +-
+ drivers/gpu/drm/drm_cache.c                        |    8 +-
+ drivers/gpu/drm/i915/gem/i915_gem_domain.c         |    4 +-
+ drivers/gpu/drm/i915/i915_memcpy.c                 |    2 +-
+ drivers/hwmon/hwmon-vid.c                          |   11 +-
+ drivers/irqchip/Kconfig                            |    5 -
+ drivers/irqchip/Makefile                           |    1 -
+ drivers/irqchip/irq-gic-its-msi-parent.c           |    3 +-
+ drivers/irqchip/irq-gic-v3-its.c                   |   12 +-
+ drivers/irqchip/irq-gic-v5-iwb.c                   |    2 +-
+ drivers/irqchip/irq-imgpdc.c                       |  495 --------
+ drivers/irqchip/irq-qcom-mpm.c                     |    4 +-
+ drivers/irqchip/irq-realtek-rtl.c                  |  152 ++-
+ drivers/irqchip/irq-renesas-irqc.c                 |    1 +
+ drivers/irqchip/irq-renesas-rzt2h.c                |    2 +-
+ drivers/irqchip/irq-renesas-rzv2h.c                |    4 +-
+ drivers/irqchip/qcom-pdc.c                         |  548 +++++++--
+ drivers/md/dm-writecache.c                         |    2 +-
+ .../x86/intel/speed_select_if/isst_tpmi_core.c     |    2 +-
+ drivers/powercap/intel_rapl_common.c               |   20 +-
+ drivers/ras/Kconfig                                |   24 +-
+ drivers/thermal/intel/intel_tcc.c                  |   10 +-
+ drivers/thermal/intel/therm_throt.c                |   78 +-
+ drivers/thermal/intel/x86_pkg_temp_thermal.c       |   32 +-
+ fs/resctrl/ctrlmondata.c                           |   50 +-
+ fs/resctrl/internal.h                              |    3 +-
+ fs/resctrl/monitor.c                               |  131 ++-
+ fs/resctrl/pseudo_lock.c                           |    2 +-
+ fs/resctrl/rdtgroup.c                              |  732 +++++++-----
+ include/asm-generic/syscall.h                      |    4 +-
+ include/asm-generic/vdso/vsyscall.h                |    4 +-
+ include/linux/cpuhotplug.h                         |    1 -
+ include/linux/cpuset.h                             |    6 +
+ include/linux/entry-common.h                       |  164 +--
+ include/linux/hrtimer.h                            |   34 +-
+ include/linux/{hrtimer_defs.h => hrtimer_bases.h}  |   14 +-
+ include/linux/hrtimer_rearm.h                      |    5 +
+ include/linux/irq-entry-common.h                   |    6 +-
+ include/linux/irqdesc.h                            |    2 +-
+ include/linux/ptrace.h                             |   13 +-
+ include/linux/randomize_kstack.h                   |   19 +
+ include/linux/sched.h                              |    4 +-
+ include/linux/seccomp.h                            |   12 +-
+ include/linux/syscall_user_dispatch.h              |   28 +-
+ include/linux/syscall_user_dispatch_types.h        |    2 +-
+ include/linux/timekeeper_internal.h                |   30 +-
+ include/linux/timekeeping.h                        |   10 +-
+ include/linux/uprobes.h                            |    5 -
+ include/linux/vdso_datastore.h                     |    6 +-
+ include/uapi/linux/time_types.h                    |    2 +-
+ include/vdso/datapage.h                            |    6 +-
+ include/vdso/helpers.h                             |    4 +-
+ include/vdso/processor.h                           |    4 +-
+ include/vdso/vsyscall.h                            |    4 +-
+ init/Kconfig                                       |    2 +-
+ kernel/cgroup/cpuset.c                             |   22 +
+ kernel/entry/Makefile                              |    3 +-
+ kernel/entry/syscall-common.c                      |   21 +-
+ kernel/entry/syscall_user_dispatch.c               |   38 +-
+ kernel/events/core.c                               |   41 +-
+ kernel/events/uprobes.c                            |   10 -
+ kernel/fork.c                                      |    1 -
+ kernel/futex/core.c                                |    3 +-
+ kernel/irq/irqdesc.c                               |   11 +-
+ kernel/irq/irqdomain.c                             |    8 +-
+ kernel/irq/manage.c                                |    6 +-
+ kernel/locking/lockdep.c                           |   34 +-
+ kernel/sched/core.c                                |    5 +-
+ kernel/sched/deadline.c                            |    6 +-
+ kernel/sched/debug.c                               |   92 +-
+ kernel/sched/fair.c                                | 1226 +++++++++++---------
+ kernel/sched/pelt.c                                |    6 +-
+ kernel/sched/psi.c                                 |    2 +-
+ kernel/sched/rt.c                                  |    3 +-
+ kernel/sched/sched.h                               |   49 +-
+ kernel/seccomp.c                                   |   36 +-
+ kernel/smp.c                                       |   26 +-
+ kernel/sys_ni.c                                    |    4 +
+ kernel/time/clocksource.c                          |    9 +-
+ kernel/time/hrtimer.c                              |   56 +-
+ kernel/time/namespace.c                            |    6 +-
+ kernel/time/namespace_vdso.c                       |    7 +-
+ kernel/time/ntp.c                                  |  312 ++++-
+ kernel/time/ntp_internal.h                         |    4 +-
+ kernel/time/posix-cpu-timers.c                     |    9 +-
+ kernel/time/tick-internal.h                        |    1 +
+ kernel/time/time.c                                 |   25 +-
+ kernel/time/timekeeping.c                          |  141 ++-
+ kernel/time/timekeeping_internal.h                 |   21 +
+ kernel/time/timer_list.c                           |   17 +-
+ kernel/time/timer_migration.c                      |    4 +-
+ lib/vdso/Kconfig                                   |    8 +-
+ lib/vdso/Makefile                                  |    2 +-
+ lib/vdso/datastore.c                               |   63 +-
+ lib/vdso/gettimeofday.c                            |   20 +
+ sound/drivers/dummy.c                              |    1 +
+ tools/arch/x86/include/asm/cpufeatures.h           |    2 +-
+ tools/lib/bpf/features.c                           |    4 +-
+ tools/lib/bpf/usdt.c                               |   16 +-
+ tools/objtool/check.c                              |    1 -
+ tools/objtool/klp-diff.c                           |    1 -
+ tools/testing/selftests/bpf/bench.c                |   20 +-
+ tools/testing/selftests/bpf/benchs/bench_trigger.c |   38 +-
+ .../selftests/bpf/benchs/run_bench_uprobes.sh      |    2 +-
+ .../selftests/bpf/prog_tests/uprobe_syscall.c      |  325 +++++-
+ tools/testing/selftests/bpf/prog_tests/usdt.c      |   74 +-
+ tools/testing/selftests/bpf/progs/test_usdt.c      |   25 +
+ tools/testing/selftests/bpf/usdt.h                 |    2 +-
+ tools/testing/selftests/bpf/usdt_2.c               |   15 +-
+ .../testing/selftests/futex/functional/.gitignore  |    1 +
+ tools/testing/selftests/futex/functional/Makefile  |    6 +-
+ .../futex/functional/futex_lock_pi_exiting.c       |  263 +++++
+ .../selftests/futex/functional/futex_numa_mpol.c   |  127 +-
+ .../selftests/futex/functional/futex_priv_hash.c   |  163 +--
+ .../selftests/futex/functional/futex_requeue.c     |   60 +-
+ .../selftests/futex/functional/futex_requeue_pi.c  |  157 +--
+ .../functional/futex_requeue_pi_mismatched_ops.c   |   41 +-
+ .../functional/futex_requeue_pi_signal_restart.c   |   74 +-
+ .../selftests/futex/functional/futex_wait.c        |  129 +-
+ .../functional/futex_wait_private_mapped_file.c    |   36 +-
+ .../futex/functional/futex_wait_timeout.c          |  106 +-
+ .../functional/futex_wait_uninitialized_heap.c     |   35 +-
+ .../futex/functional/futex_wait_wouldblock.c       |   34 +-
+ .../selftests/futex/functional/futex_waitv.c       |  146 +--
+ .../selftests/futex/functional/robust_list.c       |  183 +--
+ tools/testing/selftests/futex/functional/run.sh    |    3 +
+ tools/testing/selftests/futex/include/futex2test.h |   10 +
+ .../testing/selftests/futex/include/futex_thread.h |  115 ++
+ .../selftests/rseq/rseq-x86-thread-pointer.h       |    4 +-
+ tools/testing/selftests/rseq/rseq.c                |    2 +-
+ .../testing/selftests/syscall_user_dispatch/config |    2 +-
+ tools/testing/selftests/timers/Makefile            |    2 +-
+ tools/testing/selftests/timers/adjtick.c           |    5 +-
+ .../testing/selftests/timers/alarmtimer-suspend.c  |    3 +-
+ .../testing/selftests/timers/inconsistency-check.c |    3 +-
+ tools/testing/selftests/timers/leap-a-day.c        |   22 +-
+ tools/testing/selftests/timers/mqueue-lat.c        |    3 +-
+ tools/testing/selftests/timers/nanosleep.c         |    3 +-
+ tools/testing/selftests/timers/nsleep-lat.c        |    3 +-
+ tools/testing/selftests/timers/posix_timers.c      |    4 +-
+ tools/testing/selftests/timers/raw_skew.c          |    3 +-
+ tools/testing/selftests/timers/set-2038.c          |    3 +-
+ tools/testing/selftests/timers/set-timer-lat.c     |    3 +-
+ tools/testing/selftests/timers/valid-adjtimex.c    |    4 +-
+ 321 files changed, 6216 insertions(+), 4511 deletions(-)
+ delete mode 100644 arch/x86/ras/Kconfig
+ delete mode 100644 drivers/irqchip/irq-imgpdc.c
+ rename include/linux/{hrtimer_defs.h => hrtimer_bases.h} (93%)
+ create mode 100644 tools/testing/selftests/futex/functional/futex_lock_pi_exiting.c
+ create mode 100644 tools/testing/selftests/futex/include/futex_thread.h
+$ git am -3 ../patches/0001-vdso-Merge-fixup-with-mm.patch
+Applying: vdso: Merge fixup with mm
+Using index info to reconstruct a base tree...
+M	kernel/time/namespace_vdso.c
+Falling back to patching base and 3-way merge...
+Auto-merging kernel/time/namespace_vdso.c
+No changes -- Patch already applied.
+Merging kexec/kexec-next (af78cec42d1b0 Merge branch 'crashkernel-cma' into kexec-next)
+$ git merge -m Merge branch 'kexec-next' of https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git kexec/kexec-next
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging arch/riscv/mm/init.c
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/kernel-parameters.txt |  16 ++--
+ arch/arm64/kernel/machine_kexec_file.c          |  40 +++-------
+ arch/arm64/mm/init.c                            |   5 +-
+ arch/loongarch/kernel/machine_kexec_file.c      |  40 +++-------
+ arch/powerpc/include/asm/kexec_ranges.h         |   1 -
+ arch/powerpc/kexec/crash.c                      |   5 +-
+ arch/powerpc/kexec/ranges.c                     | 101 +-----------------------
+ arch/riscv/kernel/machine_kexec_file.c          |  39 +++------
+ arch/riscv/mm/init.c                            |   5 +-
+ arch/x86/kernel/crash.c                         |  89 +++------------------
+ drivers/of/fdt.c                                |   9 ++-
+ drivers/of/kexec.c                              |   9 +++
+ include/linux/crash_core.h                      |   9 +++
+ include/linux/crash_reserve.h                   |   4 +-
+ include/linux/kexec.h                           |   2 +-
+ kernel/crash_core.c                             |  89 ++++++++++++++++++++-
+ kernel/kexec_file.c                             |  27 +++++++
+ 17 files changed, 208 insertions(+), 282 deletions(-)
+Merging liveupdate/next (fc1b1fca6c309 Merge branch 'kho-scratch' into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git liveupdate/next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                                        |   1 +
+ arch/x86/boot/compressed/kaslr.c                   |   2 +
+ arch/x86/include/asm/Kbuild                        |   1 +
+ arch/x86/include/asm/setup.h                       |   2 -
+ arch/x86/kernel/setup.c                            |   1 +
+ include/asm-generic/kexec_handover.h               |  12 +
+ include/linux/kexec_handover.h                     |   6 +-
+ kernel/liveupdate/luo_session.c                    |   1 -
+ tools/testing/selftests/liveupdate/.gitignore      |   1 +
+ tools/testing/selftests/liveupdate/config          |   1 +
+ tools/testing/selftests/liveupdate/config.aarch64  |   2 +
+ tools/testing/selftests/liveupdate/config.x86_64   |   2 +
+ tools/testing/selftests/liveupdate/init.c          | 179 ++++++++++++++
+ .../testing/selftests/liveupdate/luo_test_utils.c  |  15 +-
+ tools/testing/selftests/liveupdate/run-vmtests.sh  |  97 ++++++++
+ tools/testing/selftests/liveupdate/vmtest.sh       | 263 +++++++++++++++++++++
+ 16 files changed, 569 insertions(+), 17 deletions(-)
+ create mode 100644 include/asm-generic/kexec_handover.h
+ create mode 100644 tools/testing/selftests/liveupdate/config.aarch64
+ create mode 100644 tools/testing/selftests/liveupdate/config.x86_64
+ create mode 100644 tools/testing/selftests/liveupdate/init.c
+ create mode 100755 tools/testing/selftests/liveupdate/run-vmtests.sh
+ create mode 100755 tools/testing/selftests/liveupdate/vmtest.sh
+Merging clockevents/timers/drivers/next (a9ac745bc320c clocksource: move NXP timer selection to drivers/clocksource)
+$ git merge -m Merge branch 'timers/drivers/next' of https://git.kernel.org/pub/scm/linux/kernel/git/daniel.lezcano/linux.git clockevents/timers/drivers/next
+Already up to date.
+Merging edac/edac-for-next (5adf9141e5800 Merge ras/edac-misc into for-next)
+$ git merge -m Merge branch 'edac-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ras/ras.git edac/edac-for-next
+Auto-merging CREDITS
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ CREDITS                      |   4 ++
+ MAINTAINERS                  |  13 ++---
+ drivers/edac/altera_edac.c   | 135 +++++++++++++++++++++----------------------
+ drivers/edac/altera_edac.h   |   1 +
+ drivers/edac/amd64_edac.c    |   2 +
+ drivers/edac/debugfs.c       |  65 +--------------------
+ drivers/edac/edac_mc_sysfs.c |   2 +-
+ drivers/ras/amd/atl/core.c   |   1 -
+ include/linux/edac.h         |   3 -
+ 9 files changed, 81 insertions(+), 145 deletions(-)
+Merging ftrace/for-next (030bb1d8bb077 Merge probes/for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git ftrace/for-next
+Auto-merging MAINTAINERS
+Auto-merging Makefile
+Auto-merging arch/x86/Kconfig
+Auto-merging arch/x86/kernel/setup.c
+Auto-merging init/Kconfig
+Auto-merging init/main.c
+Auto-merging kernel/trace/trace.c
+Auto-merging lib/Makefile
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/bootconfig.rst           |  81 +++
+ Documentation/trace/eprobetrace.rst                |   7 +-
+ Documentation/trace/fprobetrace.rst                |  10 +
+ Documentation/trace/kprobetrace.rst                |  11 +
+ MAINTAINERS                                        |   1 +
+ Makefile                                           |  27 +-
+ arch/x86/Kconfig                                   |   1 +
+ arch/x86/kernel/setup.c                            |  14 +-
+ include/asm-generic/kprobes.h                      |   4 +-
+ include/linux/bootconfig.h                         |  14 +
+ init/Kconfig                                       |  36 ++
+ init/main.c                                        |  52 +-
+ kernel/trace/Kconfig                               |  12 +
+ kernel/trace/trace.c                               |   8 +-
+ kernel/trace/trace_eprobe.c                        |   2 +
+ kernel/trace/trace_fprobe.c                        |   6 +-
+ kernel/trace/trace_kprobe.c                        |   2 +
+ kernel/trace/trace_probe.c                         | 612 +++++++++++++++++----
+ kernel/trace/trace_probe.h                         | 102 ++--
+ kernel/trace/trace_probe_tmpl.h                    |  27 +-
+ kernel/trace/trace_uprobe.c                        |   3 +
+ lib/Makefile                                       |  16 +
+ lib/bootconfig.c                                   | 139 ++++-
+ lib/embedded-cmdline.S                             |  16 +
+ samples/trace_events/trace-events-sample.c         |  40 +-
+ samples/trace_events/trace-events-sample.h         |  34 +-
+ tools/bootconfig/Makefile                          |   4 +-
+ tools/bootconfig/scripts/ftrace2bconf.sh           |   2 +
+ .../ftrace/test.d/dynevent/btf_probe_event.tc      |  51 ++
+ .../test.d/dynevent/btf_typecast_accepted.tc       | 103 ++++
+ .../test.d/dynevent/eprobes_syntax_errors.tc       |  12 +-
+ .../ftrace/test.d/dynevent/fprobe_syntax_errors.tc |  12 +
+ .../ftrace/test.d/kprobe/kprobe_syntax_errors.tc   |  12 +
+ .../ftrace/test.d/kprobe/uprobe_syntax_errors.tc   |   5 +
+ 34 files changed, 1274 insertions(+), 204 deletions(-)
+ create mode 100644 lib/embedded-cmdline.S
+ create mode 100644 tools/testing/selftests/ftrace/test.d/dynevent/btf_probe_event.tc
+ create mode 100644 tools/testing/selftests/ftrace/test.d/dynevent/btf_typecast_accepted.tc
+$ git am -3 ../patches/0001-ftrace-Fix-semantic-conflict-with-mm-tree.patch
+Applying: ftrace: Fix semantic conflict with mm tree
+Using index info to reconstruct a base tree...
+M	kernel/trace/trace_printk.c
+Falling back to patching base and 3-way merge...
+Auto-merging kernel/trace/trace_printk.c
+No changes -- Patch already applied.
+Merging rcu/next (e853c1b28580e Merge branches 'rcutorture.2026.05.24' and 'misc.2026.05.24' into rcu-merge.2026.05.24)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/rcu/linux rcu/next
+Already up to date.
+Merging paulmck/non-rcu/next (c2d939be90724 smp: Make CSD lock acquisition atomic for debug mode)
+$ git merge -m Merge branch 'non-rcu/next' of https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git paulmck/non-rcu/next
+Auto-merging kernel/smp.c
+Merge made by the 'ort' strategy.
+ kernel/smp.c | 29 ++++++++++++++++++++++++-----
+ 1 file changed, 24 insertions(+), 5 deletions(-)
+Merging kvm/next (a204badd8432f Merge branch 'kvm-chainsaw' into HEAD)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/virt/kvm/kvm.git kvm/next
+Auto-merging arch/x86/kvm/cpuid.c
+Auto-merging arch/x86/kvm/irq.c
+Auto-merging arch/x86/kvm/mmu/mmu.c
+Auto-merging arch/x86/kvm/mmu/spte.c
+Auto-merging arch/x86/kvm/pmu.c
+Auto-merging arch/x86/kvm/svm/svm.c
+Auto-merging arch/x86/kvm/vmx/nested.c
+Auto-merging arch/x86/kvm/vmx/vmx.c
+Auto-merging arch/x86/kvm/x86.c
+CONFLICT (content): Merge conflict in arch/x86/kvm/x86.c
+Auto-merging tools/testing/selftests/kvm/Makefile.kvm
+Recorded preimage for 'arch/x86/kvm/x86.c'
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+Recorded resolution for 'arch/x86/kvm/x86.c'.
+[master 138d5eef7e606] Merge branch 'next' of git://git.kernel.org/pub/scm/virt/kvm/kvm.git
+$ git diff -M --stat --summary HEAD^..
+ arch/x86/include/asm/kvm_host.h                    |  655 +---
+ arch/x86/kvm/Makefile                              |    4 +-
+ arch/x86/kvm/cpuid.c                               |    1 +
+ arch/x86/kvm/fpu.h                                 |   26 +
+ arch/x86/kvm/hyperv.c                              |    7 +-
+ arch/x86/kvm/hyperv.h                              |    3 +-
+ arch/x86/kvm/ioapic.c                              |    1 +
+ arch/x86/kvm/ioapic.h                              |   12 +
+ arch/x86/kvm/irq.c                                 |    7 +
+ arch/x86/kvm/irq.h                                 |    6 +
+ arch/x86/kvm/lapic.h                               |    8 +
+ arch/x86/kvm/mmu.h                                 |  121 +-
+ arch/x86/kvm/mmu/mmu.c                             |  541 +--
+ arch/x86/kvm/mmu/mmu_internal.h                    |   66 -
+ arch/x86/kvm/mmu/paging_tmpl.h                     |   88 +-
+ arch/x86/kvm/mmu/spte.c                            |    4 +-
+ arch/x86/kvm/mmu/spte.h                            |   69 +-
+ arch/x86/kvm/mmu/tdp_mmu.c                         |    3 +-
+ arch/x86/kvm/msrs.c                                | 2745 ++++++++++++++
+ arch/x86/kvm/msrs.h                                |  156 +
+ arch/x86/kvm/mtrr.c                                |    2 +-
+ arch/x86/kvm/pmu.c                                 |   12 +
+ arch/x86/kvm/regs.c                                |  874 +++++
+ arch/x86/kvm/regs.h                                |   60 +
+ arch/x86/kvm/svm/nested.c                          |   14 +-
+ arch/x86/kvm/svm/svm.c                             |    6 +-
+ arch/x86/kvm/tss.h                                 |    7 +
+ arch/x86/kvm/vmx/nested.c                          |   16 +-
+ arch/x86/kvm/vmx/vmx.c                             |   26 +-
+ arch/x86/kvm/x86.c                                 | 3762 +-------------------
+ arch/x86/kvm/x86.h                                 |  462 ++-
+ tools/testing/selftests/kvm/Makefile.kvm           |    1 +
+ tools/testing/selftests/kvm/include/x86/pmu.h      |    6 +
+ .../testing/selftests/kvm/include/x86/processor.h  |   10 +
+ tools/testing/selftests/kvm/include/x86/svm_util.h |    5 +-
+ tools/testing/selftests/kvm/include/x86/vmx.h      |    4 +-
+ tools/testing/selftests/kvm/lib/x86/memstress.c    |   19 +-
+ tools/testing/selftests/kvm/lib/x86/processor.c    |   45 +-
+ tools/testing/selftests/kvm/lib/x86/svm.c          |    6 +-
+ tools/testing/selftests/kvm/lib/x86/vmx.c          |    6 +-
+ tools/testing/selftests/kvm/x86/aperfmperf_test.c  |    9 +-
+ .../selftests/kvm/x86/evmcs_smm_controls_test.c    |    5 +-
+ tools/testing/selftests/kvm/x86/hyperv_evmcs.c     |    6 +-
+ tools/testing/selftests/kvm/x86/hyperv_svm_test.c  |    6 +-
+ tools/testing/selftests/kvm/x86/kvm_buslock_test.c |    9 +-
+ .../selftests/kvm/x86/nested_close_kvm_test.c      |   12 +-
+ .../selftests/kvm/x86/nested_dirty_log_test.c      |    8 +-
+ .../selftests/kvm/x86/nested_emulation_test.c      |    4 +-
+ .../selftests/kvm/x86/nested_exceptions_test.c     |    9 +-
+ .../selftests/kvm/x86/nested_invalid_cr3_test.c    |   10 +-
+ .../selftests/kvm/x86/nested_tdp_fault_test.c      |    9 +-
+ .../selftests/kvm/x86/nested_tsc_adjust_test.c     |   10 +-
+ .../selftests/kvm/x86/nested_tsc_scaling_test.c    |   10 +-
+ .../selftests/kvm/x86/nested_vmsave_vmload_test.c  |    6 +-
+ tools/testing/selftests/kvm/x86/smm_test.c         |    8 +-
+ tools/testing/selftests/kvm/x86/state_test.c       |   11 +-
+ tools/testing/selftests/kvm/x86/svm_int_ctl_test.c |    5 +-
+ .../selftests/kvm/x86/svm_lbr_nested_state.c       |    6 +-
+ .../selftests/kvm/x86/svm_nested_clear_efer_svme.c |    7 +-
+ .../selftests/kvm/x86/svm_nested_pat_test.c        |    8 +-
+ .../selftests/kvm/x86/svm_nested_shutdown_test.c   |    5 +-
+ .../kvm/x86/svm_nested_soft_inject_test.c          |    6 +-
+ .../selftests/kvm/x86/svm_nested_vmcb12_gpa.c      |   13 +-
+ .../selftests/kvm/x86/svm_pmu_host_guest_test.c    |  215 ++
+ tools/testing/selftests/kvm/x86/svm_vmcall_test.c  |    5 +-
+ .../selftests/kvm/x86/triple_fault_event_test.c    |    9 +-
+ .../selftests/kvm/x86/vmx_apic_access_test.c       |    5 +-
+ .../selftests/kvm/x86/vmx_apicv_updates_test.c     |    4 +-
+ .../kvm/x86/vmx_invalid_nested_guest_state.c       |    6 +-
+ .../selftests/kvm/x86/vmx_nested_la57_state_test.c |    5 +-
+ .../selftests/kvm/x86/vmx_preemption_timer_test.c  |    5 +-
+ 71 files changed, 5273 insertions(+), 5029 deletions(-)
+ create mode 100644 arch/x86/kvm/msrs.c
+ create mode 100644 arch/x86/kvm/msrs.h
+ create mode 100644 arch/x86/kvm/regs.c
+ create mode 100644 tools/testing/selftests/kvm/x86/svm_pmu_host_guest_test.c
+$ git am -3 ../patches/kvm-x86-static-cpu-has
+Applying: Signed-off-by: Mark Brown <broonie@kernel.org>
+Using index info to reconstruct a base tree...
+M	arch/x86/kvm/msrs.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+Merging kvm-arm/next (cb9360ae65491 Merge branch 'kvm-arm64/pkvm-7.3' into kvmarm/next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git kvm-arm/next
+Auto-merging arch/arm64/kvm/hyp/nvhe/pkvm.c
+Auto-merging arch/arm64/kvm/pkvm.c
+Merge made by the 'ort' strategy.
+ arch/arm64/kvm/hyp/include/nvhe/clock.h |  8 ++++----
+ arch/arm64/kvm/hyp/nvhe/clock.c         |  4 ++--
+ arch/arm64/kvm/hyp/nvhe/pkvm.c          |  1 -
+ arch/arm64/kvm/hyp/nvhe/trace.c         |  4 ++--
+ arch/arm64/kvm/pkvm.c                   | 13 +++++--------
+ 5 files changed, 13 insertions(+), 17 deletions(-)
+Merging kvms390/next (babe08404e199 KVM: s390: Return failure in case of failure in kvm_s390_set_cmma_bits())
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git kvms390/next
+Already up to date.
+Merging kvm-ppc/topic/ppc-kvm (8cdeaa50eae8d Linux 7.2-rc2)
+$ git merge -m Merge branch 'topic/ppc-kvm' of https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git kvm-ppc/topic/ppc-kvm
+Already up to date.
+Merging kvm-riscv/riscv_kvm_next (8d9c9b135b5c2 KVM: riscv: Fix Spectre-v1 in vector register access)
+$ git merge -m Merge branch 'riscv_kvm_next' of https://github.com/kvm-riscv/linux.git kvm-riscv/riscv_kvm_next
+Merge made by the 'ort' strategy.
+ arch/riscv/include/asm/kvm_host.h | 10 +++---
+ arch/riscv/kvm/aia.c              | 35 ++++++++++++++++----
+ arch/riscv/kvm/gstage.c           |  6 +++-
+ arch/riscv/kvm/mmu.c              |  3 +-
+ arch/riscv/kvm/vcpu.c             | 68 +++++++++++++++++++++++++++------------
+ arch/riscv/kvm/vcpu_exit.c        | 19 +++++++++++
+ arch/riscv/kvm/vcpu_onereg.c      | 46 +++++++-------------------
+ arch/riscv/kvm/vcpu_pmu.c         |  4 +--
+ arch/riscv/kvm/vcpu_sbi_fwft.c    |  6 ++--
+ arch/riscv/kvm/vcpu_vector.c      | 14 ++++++--
+ 10 files changed, 137 insertions(+), 74 deletions(-)
+Merging kvm-x86/next (6bc96b971766f Merge branches 'coco', 'fixes', 'generic', 'irq_test', 'misc', 'selftests', 'svm' and 'vmx')
+$ git merge -m Merge branch 'next' of https://github.com/kvm-x86/linux.git kvm-x86/next
+Auto-merging arch/x86/include/asm/cpufeatures.h
+Auto-merging arch/x86/kvm/cpuid.c
+Auto-merging arch/x86/kvm/emulate.c
+Auto-merging arch/x86/kvm/svm/svm.c
+Auto-merging arch/x86/kvm/vmx/nested.c
+Auto-merging arch/x86/kvm/vmx/vmx.c
+Auto-merging arch/x86/kvm/x86.c
+Auto-merging arch/x86/kvm/xen.c
+Auto-merging tools/testing/selftests/kvm/Makefile.kvm
+Auto-merging virt/kvm/guest_memfd.c
+Auto-merging virt/kvm/kvm_main.c
+Merge made by the 'ort' strategy.
+ arch/x86/include/asm/cpufeatures.h                 |  10 +
+ arch/x86/include/asm/kvm-x86-nested-ops.h          |  36 ++
+ arch/x86/include/asm/kvm-x86-ops.h                 |   8 +-
+ arch/x86/include/asm/kvm_host.h                    | 214 ++----------
+ arch/x86/kvm/cpuid.c                               |  22 +-
+ arch/x86/kvm/emulate.c                             |  49 ++-
+ arch/x86/kvm/hyperv.c                              |  70 ++--
+ arch/x86/kvm/hyperv.h                              | 117 ++++++-
+ arch/x86/kvm/lapic.c                               |  10 +
+ arch/x86/kvm/lapic.h                               |  33 ++
+ arch/x86/kvm/mmu.h                                 |  11 +-
+ arch/x86/kvm/mmu/mmu.c                             |  76 +++--
+ arch/x86/kvm/mmu/paging_tmpl.h                     |   2 +-
+ arch/x86/kvm/msrs.h                                |   4 +-
+ arch/x86/kvm/regs.h                                |  38 +++
+ arch/x86/kvm/svm/avic.c                            |  96 ++++--
+ arch/x86/kvm/svm/nested.c                          |   3 +-
+ arch/x86/kvm/svm/sev.c                             | 173 +++++++---
+ arch/x86/kvm/svm/svm.c                             |  30 +-
+ arch/x86/kvm/svm/svm.h                             |  19 +-
+ arch/x86/kvm/tss.h                                 |   7 +
+ arch/x86/kvm/vmx/main.c                            |   3 +-
+ arch/x86/kvm/vmx/nested.c                          |  64 ++--
+ arch/x86/kvm/vmx/nested.h                          |   6 +-
+ arch/x86/kvm/vmx/tdx.c                             |   7 +-
+ arch/x86/kvm/vmx/vmx.c                             |  20 +-
+ arch/x86/kvm/x86.c                                 |  89 +++--
+ arch/x86/kvm/x86.h                                 |  10 +-
+ arch/x86/kvm/xen.c                                 |  43 ++-
+ include/linux/kvm_host.h                           |   8 +
+ tools/testing/selftests/kvm/Makefile.kvm           |   7 +-
+ tools/testing/selftests/kvm/arch_timer.c           |   2 +-
+ .../selftests/kvm/arm64/arch_timer_edge_cases.c    |   2 +-
+ tools/testing/selftests/kvm/demand_paging_test.c   |   2 +-
+ tools/testing/selftests/kvm/dirty_log_perf_test.c  |   4 +-
+ tools/testing/selftests/kvm/dirty_log_test.c       |  11 +-
+ tools/testing/selftests/kvm/include/kvm_syscalls.h |   7 +
+ tools/testing/selftests/kvm/include/kvm_util.h     |  12 +
+ tools/testing/selftests/kvm/include/proc_util.h    |  11 +
+ tools/testing/selftests/kvm/include/test_util.h    |  25 +-
+ .../selftests/kvm/include/x86/kvm_util_arch.h      |   4 +-
+ tools/testing/selftests/kvm/irq_test.c             | 362 +++++++++++++++++++++
+ tools/testing/selftests/kvm/lib/assert.c           |   8 +-
+ tools/testing/selftests/kvm/lib/kvm_util.c         |  54 ++-
+ tools/testing/selftests/kvm/lib/memstress.c        |   8 +-
+ tools/testing/selftests/kvm/lib/proc_util.c        |  54 +++
+ tools/testing/selftests/kvm/lib/test_util.c        |  27 +-
+ tools/testing/selftests/kvm/mmu_stress_test.c      |  15 +-
+ tools/testing/selftests/kvm/rseq_test.c            |   6 +-
+ tools/testing/selftests/kvm/steal_time.c           |  37 +--
+ tools/testing/selftests/kvm/x86/sev_dbg_test.c     |   2 +-
+ tools/testing/selftests/kvm/x86/xapic_ipi_test.c   |   2 +-
+ virt/kvm/guest_memfd.c                             |   4 +
+ virt/kvm/kvm_main.c                                |   9 +
+ 54 files changed, 1366 insertions(+), 587 deletions(-)
+ create mode 100644 arch/x86/include/asm/kvm-x86-nested-ops.h
+ create mode 100644 tools/testing/selftests/kvm/include/proc_util.h
+ create mode 100644 tools/testing/selftests/kvm/irq_test.c
+ create mode 100644 tools/testing/selftests/kvm/lib/proc_util.c
+$ git am -3 ../patches/0001-KVM-selftests-Fix-up-semantic-changes.patch
+Applying: KVM: selftests: Fix up semantic changes
+Using index info to reconstruct a base tree...
+M	tools/testing/selftests/kvm/lib/kvm_util.c
+Falling back to patching base and 3-way merge...
+Auto-merging tools/testing/selftests/kvm/lib/kvm_util.c
+No changes -- Patch already applied.
+Merging xen-tip/linux-next (fcd245ea7528d x86/Xen: correct commentary and parameter naming of xen_exchange_memory())
+$ git merge -m Merge branch 'linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/xen/tip.git xen-tip/linux-next
+Already up to date.
+Merging percpu/for-next (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu.git percpu/for-next
+Already up to date.
+Merging workqueues/for-next (c5a3a03816ddb Merge branch 'for-7.1-fixes' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tj/wq.git workqueues/for-next
+Merge made by the 'ort' strategy.
+ Documentation/core-api/workqueue.rst |   2 +-
+ kernel/workqueue.c                   | 144 ++++++++++++++++++++++++++++++-----
+ tools/workqueue/wq_dump.py           |  10 +--
+ tools/workqueue/wq_monitor.py        |   6 +-
+ 4 files changed, 135 insertions(+), 27 deletions(-)
+Merging sched-ext/for-next (1ed374f61d7ce Merge branch 'for-7.3' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git sched-ext/for-next
+Auto-merging init/init_task.c
+Auto-merging kernel/sched/sched.h
+Auto-merging tools/sched_ext/include/scx/compat.bpf.h
+Merge made by the 'ort' strategy.
+ include/linux/sched/ext.h                    |   37 +-
+ include/trace/events/sched_ext.h             |   28 +
+ init/init_task.c                             |    1 +
+ kernel/rcu/tree.c                            |    3 +
+ kernel/rcu/tree_exp.h                        |    5 +-
+ kernel/rcu/tree_stall.h                      |   13 +-
+ kernel/sched/build_policy.c                  |    3 +
+ kernel/sched/ext/arena.c                     |   13 +-
+ kernel/sched/ext/cid.c                       |  563 ++++++-
+ kernel/sched/ext/cid.h                       |   19 +-
+ kernel/sched/ext/ext.c                       | 2263 +++++++++++---------------
+ kernel/sched/ext/idle.c                      |   96 +-
+ kernel/sched/ext/inlines.h                   |  128 ++
+ kernel/sched/ext/internal.h                  |  554 ++++++-
+ kernel/sched/ext/sub.c                       | 1791 ++++++++++++++++++++
+ kernel/sched/ext/sub.h                       |  189 +++
+ kernel/sched/ext/types.h                     |   74 +-
+ kernel/sched/idle.c                          |    8 +-
+ kernel/sched/sched.h                         |   19 +-
+ tools/sched_ext/include/scx/cid.bpf.h        |   77 +
+ tools/sched_ext/include/scx/common.bpf.h     |   31 +-
+ tools/sched_ext/include/scx/compat.bpf.h     |   28 +-
+ tools/sched_ext/include/scx/compat.h         |   10 +-
+ tools/sched_ext/scx_cpu0.c                   |    2 +-
+ tools/sched_ext/scx_flatcg.bpf.c             |    5 +-
+ tools/sched_ext/scx_flatcg.c                 |    4 +-
+ tools/sched_ext/scx_qmap.bpf.c               |  858 +++++++++-
+ tools/sched_ext/scx_qmap.c                   |  376 ++++-
+ tools/sched_ext/scx_qmap.h                   |  147 +-
+ tools/sched_ext/scx_sdt.c                    |    2 +-
+ tools/sched_ext/scx_simple.c                 |    2 +-
+ tools/sched_ext/scx_userland.c               |    2 +-
+ tools/testing/selftests/sched_ext/prog_run.c |   36 +-
+ 33 files changed, 5740 insertions(+), 1647 deletions(-)
+ create mode 100644 kernel/sched/ext/inlines.h
+ create mode 100644 kernel/sched/ext/sub.c
+ create mode 100644 kernel/sched/ext/sub.h
+Merging drivers-x86/for-next (2b3a5dabe89e3 platform/surface: acpi-notify: Check ACPI companion before use)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git drivers-x86/for-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/platform/x86/asus-armoury.h
+Auto-merging drivers/platform/x86/asus-wmi.c
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/amdpmf-interface         |  73 +++
+ .../ABI/testing/sysfs-driver-uniwill-laptop        |  25 +
+ .../admin-guide/laptops/thinkpad-acpi.rst          |  24 +
+ .../admin-guide/laptops/uniwill-laptop.rst         |  27 +
+ MAINTAINERS                                        |   8 +
+ arch/x86/platform/olpc/olpc-xo15-sci.c             |   5 -
+ drivers/char/sonypi.c                              |   2 -
+ drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.c |  13 +-
+ drivers/platform/mellanox/mlxbf-pmc.c              |  10 +-
+ drivers/platform/surface/surface_acpi_notify.c     |   6 +-
+ drivers/platform/surface/surface_aggregator_hub.c  |   9 +-
+ .../platform/surface/surface_aggregator_tabletsw.c |   9 +-
+ drivers/platform/surface/surfacepro3_button.c      |   8 +-
+ drivers/platform/x86/Kconfig                       |   1 -
+ drivers/platform/x86/amd/Kconfig                   |  11 +
+ drivers/platform/x86/amd/Makefile                  |   1 +
+ drivers/platform/x86/amd/amd_halo_led.c            | 315 ++++++++++
+ drivers/platform/x86/amd/hsmp/acpi.c               |  53 +-
+ drivers/platform/x86/amd/hsmp/hsmp.c               |  13 +
+ drivers/platform/x86/amd/pmf/Kconfig               |  10 +
+ drivers/platform/x86/amd/pmf/Makefile              |   2 +
+ drivers/platform/x86/amd/pmf/core.c                |   5 +
+ drivers/platform/x86/amd/pmf/pmf.h                 |  34 +-
+ drivers/platform/x86/amd/pmf/spc.c                 |  72 +--
+ drivers/platform/x86/amd/pmf/tee-if.c              |  13 +-
+ drivers/platform/x86/amd/pmf/util.c                | 153 +++++
+ drivers/platform/x86/asus-armoury.c                |  66 ++-
+ drivers/platform/x86/asus-armoury.h                | 124 ++++
+ drivers/platform/x86/asus-laptop.c                 |   9 +-
+ drivers/platform/x86/asus-wireless.c               |  11 +-
+ drivers/platform/x86/asus-wmi.c                    |  30 +-
+ drivers/platform/x86/dell/dell-wmi-base.c          |  19 +-
+ drivers/platform/x86/dell/dell-wmi-ddv.c           |  22 +-
+ drivers/platform/x86/dell/dell-wmi-privacy.c       |   4 +-
+ .../x86/dell/dell-wmi-sysman/biosattr-interface.c  |   1 -
+ drivers/platform/x86/eeepc-laptop.c                |   7 +-
+ drivers/platform/x86/fujitsu-laptop.c              |  18 +-
+ drivers/platform/x86/fujitsu-tablet.c              |  13 +-
+ drivers/platform/x86/hp/hp-bioscfg/bioscfg.c       |  16 +-
+ drivers/platform/x86/hp/hp-bioscfg/bioscfg.h       |   8 +
+ .../platform/x86/hp/hp-bioscfg/enum-attributes.c   |  11 +-
+ .../platform/x86/hp/hp-bioscfg/int-attributes.c    |   4 +-
+ .../x86/hp/hp-bioscfg/order-list-attributes.c      |   8 +-
+ .../x86/hp/hp-bioscfg/passwdobj-attributes.c       |   6 +-
+ .../platform/x86/hp/hp-bioscfg/string-attributes.c |   4 +-
+ drivers/platform/x86/hp/hp-wmi.c                   | 270 ++++++---
+ drivers/platform/x86/huawei-wmi.c                  |  99 ++++
+ drivers/platform/x86/intel/ishtp_eclite.c          |   5 +-
+ drivers/platform/x86/lenovo/thinkpad_acpi.c        | 127 +++-
+ drivers/platform/x86/lenovo/ymc.c                  |   8 +-
+ drivers/platform/x86/lg-laptop.c                   | 652 +++++++++++++--------
+ drivers/platform/x86/msi-ec.c                      |  81 +++
+ drivers/platform/x86/msi-wmi.c                     |  97 ++-
+ drivers/platform/x86/panasonic-laptop.c            |   4 -
+ drivers/platform/x86/sony-laptop.c                 |   5 +-
+ drivers/platform/x86/topstar-laptop.c              |   2 -
+ drivers/platform/x86/toshiba_acpi.c                |   6 +-
+ drivers/platform/x86/toshiba_bluetooth.c           |   6 +-
+ drivers/platform/x86/toshiba_haps.c                |   4 +-
+ drivers/platform/x86/uniwill/uniwill-acpi.c        | 609 ++++++++++++++++++-
+ drivers/platform/x86/uniwill/uniwill-wmi.h         |   2 +
+ drivers/platform/x86/xo15-ebook.c                  |   8 +-
+ drivers/power/supply/surface_battery.c             |  11 +-
+ drivers/power/supply/surface_charger.c             |   7 +-
+ include/linux/amd-pmf-io.h                         |   9 -
+ include/linux/platform_data/x86/asus-wmi.h         |   5 +
+ include/uapi/linux/amd-pmf.h                       | 204 +++++++
+ tools/platform/x86/amd/Makefile                    |  60 ++
+ tools/platform/x86/amd/test-pmf.c                  | 142 +++++
+ 69 files changed, 3065 insertions(+), 641 deletions(-)
+ create mode 100644 Documentation/ABI/testing/amdpmf-interface
+ create mode 100644 drivers/platform/x86/amd/amd_halo_led.c
+ create mode 100644 drivers/platform/x86/amd/pmf/util.c
+ create mode 100644 include/uapi/linux/amd-pmf.h
+ create mode 100644 tools/platform/x86/amd/Makefile
+ create mode 100644 tools/platform/x86/amd/test-pmf.c
+$ git am -3 ../patches/0001-hid-Fix-up-mismerge.patch
+Applying: hid: Fix up mismerge
+$ git reset HEAD^
+Unstaged changes after reset:
+M	drivers/hid/hid-asus.c
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master d5fe85c92364f] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git
+ Date: Fri Jul 17 18:16:32 2026 +0100
+Merging chrome-platform/for-next (9a3f43b30373c platform/chrome: sensorhub: Fix dropped timestamp events and log spam)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git chrome-platform/for-next
+Auto-merging drivers/platform/chrome/cros_ec_chardev.c
+Auto-merging drivers/platform/chrome/cros_ec_debugfs.c
+Auto-merging drivers/platform/chrome/cros_ec_lightbar.c
+Auto-merging drivers/platform/chrome/cros_ec_sensorhub.c
+Auto-merging drivers/platform/chrome/cros_ec_sysfs.c
+Auto-merging drivers/platform/chrome/cros_ec_vbc.c
+Auto-merging drivers/platform/chrome/cros_kbd_led_backlight.c
+Auto-merging drivers/platform/chrome/cros_usbpd_logger.c
+Auto-merging drivers/platform/chrome/cros_usbpd_notify.c
+Auto-merging drivers/platform/chrome/wilco_ec/core.c
+Auto-merging drivers/platform/chrome/wilco_ec/debugfs.c
+Auto-merging drivers/platform/chrome/wilco_ec/telemetry.c
+Merge made by the 'ort' strategy.
+ drivers/platform/chrome/chromeos_of_hw_prober.c  |  2 +-
+ drivers/platform/chrome/cros_ec_chardev.c        |  4 ++--
+ drivers/platform/chrome/cros_ec_debugfs.c        |  4 ++--
+ drivers/platform/chrome/cros_ec_lightbar.c       |  4 ++--
+ drivers/platform/chrome/cros_ec_sensorhub.c      |  4 ++--
+ drivers/platform/chrome/cros_ec_sensorhub_ring.c | 28 +++++++++++++++++++-----
+ drivers/platform/chrome/cros_ec_sysfs.c          |  4 ++--
+ drivers/platform/chrome/cros_ec_typec.c          |  6 +++++
+ drivers/platform/chrome/cros_ec_vbc.c            |  4 ++--
+ drivers/platform/chrome/cros_kbd_led_backlight.c |  4 ++--
+ drivers/platform/chrome/cros_usbpd_logger.c      |  4 ++--
+ drivers/platform/chrome/cros_usbpd_notify.c      |  4 ++--
+ drivers/platform/chrome/wilco_ec/core.c          |  4 ++--
+ drivers/platform/chrome/wilco_ec/debugfs.c       |  4 ++--
+ drivers/platform/chrome/wilco_ec/telemetry.c     |  4 ++--
+ 15 files changed, 54 insertions(+), 30 deletions(-)
+Merging chrome-platform-firmware/for-firmware-next (dd06eb60e86a1 docs: ABI: testing: Fix typo)
+$ git merge -m Merge branch 'for-firmware-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git chrome-platform-firmware/for-firmware-next
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-firmware-gsmi | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging hsi/for-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-hsi.git hsi/for-next
+Already up to date.
+Merging leds-lj/for-leds-next (96fa4d69958cf leds: st1202: Correct and extend hw_pattern documentation)
+$ git merge -m Merge branch 'for-leds-next' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/leds.git leds-lj/for-leds-next
+Auto-merging drivers/leds/rgb/leds-pwm-multicolor.c
+Merge made by the 'ort' strategy.
+ Documentation/leds/leds-st1202.rst     |   9 ++-
+ drivers/leds/leds-pca9532.c            |   6 +-
+ drivers/leds/leds-st1202.c             | 102 ++++++++++++++++++++-------------
+ drivers/leds/rgb/leds-lp5860-core.c    |   3 +-
+ drivers/leds/rgb/leds-lp5860-spi.c     |  10 ++--
+ drivers/leds/rgb/leds-pwm-multicolor.c |   2 +
+ drivers/leds/trigger/ledtrig-netdev.c  |  46 ++++++++++++++-
+ include/linux/leds.h                   |   4 ++
+ 8 files changed, 127 insertions(+), 55 deletions(-)
+Merging ipmi/for-next (6d920a75df9a8 ipmi: si: Fix NULL pointer dereference after failed registration)
+$ git merge -m Merge branch 'for-next' of https://github.com/cminyard/linux-ipmi.git ipmi/for-next
+Merge made by the 'ort' strategy.
+ drivers/char/ipmi/ipmb_dev_int.c    | 5 +++--
+ drivers/char/ipmi/ipmi_msghandler.c | 1 +
+ 2 files changed, 4 insertions(+), 2 deletions(-)
+Merging driver-core/driver-core-next (2b74f1f202448 Merge patch series "software node: provide support for fw_devlink")
+$ git merge -m Merge branch 'driver-core-next' of https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git driver-core/driver-core-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/gpu/drm/xe/xe_i2c.c
+Auto-merging drivers/usb/chipidea/core.c
+Auto-merging include/linux/platform_device.h
+CONFLICT (content): Merge conflict in include/linux/platform_device.h
+Auto-merging rust/macros/lib.rs
+Resolved 'include/linux/platform_device.h' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 5e5c11ee29922] Merge branch 'driver-core-next' of https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git
+$ git diff -M --stat --summary HEAD^..
+ MAINTAINERS                                   |   2 +
+ arch/powerpc/platforms/powermac/low_i2c.c     |   2 +-
+ drivers/base/dd.c                             |   4 +-
+ drivers/base/firmware_loader/builtin/Makefile |  32 +--
+ drivers/base/firmware_loader/builtin/main.c   |   2 +
+ drivers/base/isa.c                            |  16 +-
+ drivers/base/module.c                         |   2 +-
+ drivers/base/platform.c                       |  61 ++++-
+ drivers/base/swnode.c                         |  79 ++++++
+ drivers/base/test/Kconfig                     |   5 +
+ drivers/base/test/Makefile                    |   2 +
+ drivers/base/test/swnode-devlink-test.c       | 337 ++++++++++++++++++++++++++
+ drivers/gpu/drm/xe/xe_i2c.c                   |   2 +-
+ drivers/gpu/nova-core/driver.rs               |   4 +-
+ drivers/i2c/busses/i2c-pxa-pci.c              |   3 +-
+ drivers/iommu/fsl_pamu.c                      |  16 +-
+ drivers/mfd/tps6586x.c                        |   2 +-
+ drivers/net/ethernet/broadcom/genet/bcmmii.c  |   3 +-
+ drivers/net/ethernet/marvell/mv643xx_eth.c    |   2 +-
+ drivers/of/platform.c                         |   2 +-
+ drivers/platform/surface/surface_gpe.c        |   2 +-
+ drivers/pmdomain/imx/gpc.c                    |   3 +-
+ drivers/pwm/pwm_th1520.rs                     |   5 +-
+ drivers/reset/reset-rzg2l-usbphy-ctrl.c       |   2 +-
+ drivers/slimbus/qcom-ngd-ctrl.c               |   2 +-
+ drivers/usb/chipidea/core.c                   |   2 +-
+ drivers/usb/musb/jz4740.c                     |   2 +-
+ fs/debugfs/file.c                             |   3 +-
+ include/kunit/fwnode.h                        |  29 +++
+ include/linux/container_of.h                  |   5 +-
+ include/linux/kobject.h                       |  75 +++++-
+ include/linux/platform_device.h               |   8 +
+ lib/kobject.c                                 |   8 +-
+ lib/kunit/Makefile                            |   1 +
+ lib/kunit/fwnode.c                            | 146 +++++++++++
+ rust/kernel/auxiliary.rs                      |  78 ++++--
+ rust/kernel/devres.rs                         | 131 +++++++++-
+ rust/kernel/io/mem.rs                         |  65 +++--
+ rust/kernel/io/register.rs                    |   2 +-
+ rust/kernel/pci.rs                            |   1 +
+ rust/kernel/pci/io.rs                         |  37 ++-
+ rust/kernel/types.rs                          |   5 +-
+ rust/kernel/types/for_lt.rs                   | 103 +++++---
+ rust/macros/for_lt.rs                         |  41 +++-
+ rust/macros/lib.rs                            |  18 +-
+ samples/kobject/kobject-example.c             |  24 +-
+ samples/rust/rust_driver_auxiliary.rs         |  92 +++++--
+ 47 files changed, 1271 insertions(+), 197 deletions(-)
+ create mode 100644 drivers/base/test/swnode-devlink-test.c
+ create mode 100644 include/kunit/fwnode.h
+ create mode 100644 lib/kunit/fwnode.c
+Merging usb/usb-next (b60af0b9e752a Merge 7.2-rc3 into usb-next)
+$ git merge -m Merge branch 'usb-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git usb/usb-next
+Auto-merging Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml
+Auto-merging drivers/net/wireless/ath/ath9k/hif_usb.c
+Auto-merging drivers/usb/typec/ucsi/ucsi.c
+Merge made by the 'ort' strategy.
+ Documentation/ABI/obsolete/sysfs-bus-usb           |   2 +-
+ Documentation/ABI/testing/sysfs-bus-usb            |   2 +-
+ Documentation/admin-guide/thunderbolt.rst          |   4 +-
+ .../devicetree/bindings/usb/generic-ehci.yaml      |   7 +
+ .../devicetree/bindings/usb/generic-ohci.yaml      |   5 +
+ .../devicetree/bindings/usb/mediatek,mtu3.yaml     |   6 +-
+ .../devicetree/bindings/usb/qcom,snps-dwc3.yaml    |   3 +
+ .../devicetree/bindings/usb/renesas,usbhs.yaml     |   1 +
+ Documentation/translations/zh_CN/usb/CREDITS       | 147 +++++----
+ Documentation/translations/zh_CN/usb/acm.rst       |  62 ++--
+ .../translations/zh_CN/usb/authorization.rst       |  79 +++--
+ Documentation/translations/zh_CN/usb/chipidea.rst  |  63 ++--
+ Documentation/translations/zh_CN/usb/dwc3.rst      |  37 ++-
+ Documentation/translations/zh_CN/usb/ehci.rst      | 249 ++++++----------
+ Documentation/translations/zh_CN/usb/index.rst     |  18 +-
+ Documentation/translations/zh_CN/usb/usbmon.rst    | 332 +++++++++------------
+ Documentation/usb/functionfs.rst                   |  80 +++++
+ drivers/media/usb/as102/as102_usb_drv.c            |  75 ++---
+ drivers/net/usb/pegasus.c                          |  54 ++--
+ drivers/net/usb/pegasus.h                          |   3 -
+ drivers/net/wireless/ath/ath9k/hif_usb.c           |  12 +-
+ drivers/net/wireless/ath/ath9k/hif_usb.h           |   2 +-
+ drivers/usb/atm/xusbatm.c                          |   6 +-
+ drivers/usb/cdns3/cdns3-plat.c                     |   4 +-
+ drivers/usb/chipidea/ci_hdrc_imx.c                 |   2 +-
+ drivers/usb/class/usbtmc.c                         |   2 -
+ drivers/usb/core/config.c                          |   2 +-
+ drivers/usb/core/devices.c                         |   7 +-
+ drivers/usb/core/devio.c                           |  13 +-
+ drivers/usb/core/driver.c                          |  12 +-
+ drivers/usb/dwc2/core_intr.c                       |   3 +-
+ drivers/usb/dwc3/core.c                            |  99 +++++-
+ drivers/usb/dwc3/core.h                            |   7 +-
+ drivers/usb/dwc3/dwc3-am62.c                       |  13 +-
+ drivers/usb/dwc3/dwc3-qcom.c                       |   2 +-
+ drivers/usb/dwc3/dwc3-xilinx.c                     |  18 +-
+ drivers/usb/dwc3/gadget.c                          |  15 +-
+ drivers/usb/gadget/configfs.c                      |   8 +-
+ drivers/usb/gadget/function/f_fs.c                 | 143 +++++++--
+ drivers/usb/gadget/function/f_uac1.c               |  13 +-
+ drivers/usb/gadget/function/f_uac2.c               |  13 +-
+ drivers/usb/gadget/function/u_audio.c              |  12 +-
+ drivers/usb/gadget/function/u_fs.h                 |   8 +-
+ drivers/usb/gadget/legacy/inode.c                  |  17 +-
+ drivers/usb/gadget/legacy/nokia.c                  |   2 +-
+ drivers/usb/gadget/udc/Kconfig                     |   2 +-
+ drivers/usb/gadget/udc/aspeed_udc.c                |  50 ++--
+ drivers/usb/gadget/udc/fsl_qe_udc.c                |   8 +-
+ drivers/usb/gadget/udc/pch_udc.c                   |   1 -
+ drivers/usb/gadget/udc/r8a66597-udc.c              |   1 -
+ drivers/usb/host/fsl-mph-dr-of.c                   |   7 +-
+ drivers/usb/host/ohci-dbg.c                        |   9 +-
+ drivers/usb/misc/usb-ljca.c                        |   3 +
+ drivers/usb/mtu3/mtu3_core.c                       |  11 +-
+ drivers/usb/mtu3/mtu3_host.c                       |  26 ++
+ drivers/usb/mtu3/mtu3_plat.c                       |   3 +-
+ drivers/usb/renesas_usbhs/common.c                 |   6 +-
+ drivers/usb/serial/spcp8x5.c                       |   6 +-
+ drivers/usb/typec/altmodes/displayport.c           |  18 +-
+ drivers/usb/typec/altmodes/thunderbolt.c           |   4 +
+ drivers/usb/typec/class.c                          |  71 +++++
+ drivers/usb/typec/mux/tusb1046.c                   |   1 +
+ drivers/usb/typec/tcpm/tcpm.c                      |  11 +-
+ drivers/usb/typec/tipd/core.c                      |   9 +-
+ drivers/usb/typec/tipd/tps6598x.h                  |  10 +-
+ drivers/usb/typec/ucsi/debugfs.c                   |   5 +-
+ drivers/usb/typec/ucsi/ucsi.c                      |  18 +-
+ drivers/usb/typec/ucsi/ucsi.h                      |   1 +
+ drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c        |  15 +-
+ include/linux/usb.h                                |   3 +-
+ include/linux/usb/pd.h                             |   2 +-
+ include/linux/usb/typec.h                          |   1 +
+ include/uapi/linux/usb/functionfs.h                |  24 ++
+ rust/kernel/usb.rs                                 |  31 +-
+ 74 files changed, 1190 insertions(+), 831 deletions(-)
+Merging thunderbolt/next (d49d71ebec8f0 thunderbolt: xdomain: Notify peers after enumeration)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git thunderbolt/next
+Auto-merging drivers/net/thunderbolt/main.c
+Auto-merging drivers/thunderbolt/stream.c
+Auto-merging drivers/thunderbolt/tb.c
+Auto-merging include/linux/thunderbolt.h
+Merge made by the 'ort' strategy.
+ drivers/net/thunderbolt/main.c |  4 ++--
+ drivers/thunderbolt/dma_test.c |  4 ++--
+ drivers/thunderbolt/domain.c   |  4 +---
+ drivers/thunderbolt/nhi.c      |  2 ++
+ drivers/thunderbolt/pci.c      | 28 +++++++++++++++++++++++-----
+ drivers/thunderbolt/stream.c   |  4 ++--
+ drivers/thunderbolt/switch.c   | 11 ++++++++++-
+ drivers/thunderbolt/tb.c       | 21 +++++++++++++++++++++
+ drivers/thunderbolt/tb.h       |  1 +
+ drivers/thunderbolt/xdomain.c  |  4 ++++
+ include/linux/thunderbolt.h    |  8 +++++++-
+ 11 files changed, 75 insertions(+), 16 deletions(-)
+Merging usb-serial/usb-next (091738d09786a USB: serial: keyspan_pda: drop unused driver data usb-serial pointer)
+$ git merge -m Merge branch 'usb-next' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git usb-serial/usb-next
+Auto-merging drivers/usb/serial/keyspan_pda.c
+Merge made by the 'ort' strategy.
+ drivers/usb/serial/digi_acceleport.c | 325 ++++++++++++++---------------------
+ drivers/usb/serial/keyspan_pda.c     |   1 -
+ drivers/usb/serial/metro-usb.c       |  31 ++--
+ 3 files changed, 150 insertions(+), 207 deletions(-)
+Merging tty/tty-next (facd005eee65d Revert "serial: qcom-geni: remove .pm callback, use runtime PM in startup/shutdown")
+$ git merge -m Merge branch 'tty-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git tty/tty-next
+Auto-merging drivers/tty/serial/sc16is7xx.c
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/serial/maxim,max310x.yaml  |  92 +++++-
+ drivers/tty/amiserial.c                            |  14 +-
+ drivers/tty/goldfish.c                             |  23 +-
+ drivers/tty/serdev/core.c                          |   4 +-
+ drivers/tty/serial/8250/8250.h                     |   7 +
+ drivers/tty/serial/8250/8250_core.c                |   6 +-
+ drivers/tty/serial/8250/8250_early.c               |  19 +-
+ drivers/tty/serial/8250/8250_exar.c                |  30 +-
+ drivers/tty/serial/8250/8250_hub6.c                |   8 +
+ drivers/tty/serial/8250/8250_of.c                  |  12 +
+ drivers/tty/serial/8250/8250_platform.c            |   1 +
+ drivers/tty/serial/8250/8250_pnp.c                 | 309 +++++++++++----------
+ drivers/tty/serial/8250/8250_port.c                |  43 +--
+ drivers/tty/serial/8250/8250_rsa.c                 |  40 ++-
+ drivers/tty/serial/cpm_uart.c                      |   9 +-
+ drivers/tty/serial/earlycon.c                      |  17 +-
+ drivers/tty/serial/kgdboc.c                        |   2 +-
+ drivers/tty/serial/ma35d1_serial.c                 |  12 +-
+ drivers/tty/serial/max3100.c                       |   1 +
+ drivers/tty/serial/max310x.c                       | 100 +++++--
+ drivers/tty/serial/men_z135_uart.c                 |   7 +-
+ drivers/tty/serial/mpc52xx_uart.c                  |   8 +
+ drivers/tty/serial/pch_uart.c                      |  11 +-
+ drivers/tty/serial/qcom_geni_serial.c              |  92 ++++--
+ drivers/tty/serial/rp2.c                           |   2 +-
+ drivers/tty/serial/sc16is7xx.c                     |   9 +-
+ drivers/tty/serial/serial_core.c                   | 135 +++++----
+ drivers/tty/tty_jobctrl.c                          |  41 +--
+ drivers/tty/vt/vc_screen.c                         |   6 +-
+ include/linux/goldfish.h                           |  24 --
+ include/linux/serial_core.h                        |   6 +
+ include/trace/events/qcom_geni_serial.h            |   7 +-
+ include/uapi/linux/serial.h                        |   1 +
+ 33 files changed, 649 insertions(+), 449 deletions(-)
+Merging char-misc/char-misc-next (775553d19b163 Merge 7.2-rc3 into char-misc-next)
+$ git merge -m Merge branch 'char-misc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git char-misc/char-misc-next
+Merge made by the 'ort' strategy.
+ drivers/android/binder/allocation.rs       |  4 +-
+ drivers/android/binder/defs.rs             |  8 ++--
+ drivers/android/binder/freeze.rs           | 67 +++++++++++++++++++---------
+ drivers/android/binder/node.rs             | 55 +++++++++++------------
+ drivers/android/binder/node/wrapper.rs     |  2 +-
+ drivers/android/binder/page_range.rs       |  8 ++--
+ drivers/android/binder/process.rs          | 71 +++++++++++++++++-------------
+ drivers/android/binder/rust_binder_main.rs | 10 +----
+ drivers/android/binder/thread.rs           | 24 +++++-----
+ drivers/android/binder/trace.rs            |  4 +-
+ drivers/android/binder/transaction.rs      | 14 +++---
+ 11 files changed, 152 insertions(+), 115 deletions(-)
+$ git am -3 ../patches/0001-bus-mhi-Fix-up-interaction-with-net-next-tree.patch
+Applying: bus: mhi: Fix up interaction with net-next tree
+Using index info to reconstruct a base tree...
+M	drivers/net/wireless/ath/ath12k/wifi7/mhi.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+$ git am -3 ../patches/char-misc-binder
+Applying: linux-next: manual merge of the char-misc tree with the origin tree
+Using index info to reconstruct a base tree...
+M	drivers/android/binder/process.rs
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+Merging coresight/next (98495b5a4d77d coresight: ultrasoc-smb: Fix OOB write in smb_sync_perf_buffer())
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git coresight/next
+Already up to date.
+Merging fastrpc/for-next (17ec912dc87bf Merge branches 'fastrpc-fixes' and 'fastrpc-for-7.3' into fastrpc-for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/srini/fastrpc.git fastrpc/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS            |  2 +-
+ drivers/misc/fastrpc.c | 46 ++++++++++++++++++++++++++++------------------
+ 2 files changed, 29 insertions(+), 19 deletions(-)
+Merging fpga/for-next (b5ba63e247075 fpga: dfl: fme: add error handling)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git fpga/for-next
+Merge made by the 'ort' strategy.
+ drivers/fpga/dfl-fme-perf.c        |  2 ++
+ drivers/fpga/stratix10-soc.c       | 21 ++++++++++++---------
+ drivers/fpga/xilinx-pr-decoupler.c | 20 ++------------------
+ 3 files changed, 16 insertions(+), 27 deletions(-)
+Merging icc/icc-next (e11872288b4b3 Merge branch 'icc-maili' into icc-next)
+$ git merge -m Merge branch 'icc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/djakov/icc.git icc/icc-next
+Auto-merging drivers/interconnect/qcom/sc8280xp.c
+Merge made by the 'ort' strategy.
+ .../bindings/interconnect/qcom,maili-rpmh.yaml     |  127 ++
+ .../bindings/interconnect/qcom,osm-l3.yaml         |    2 +
+ .../bindings/interconnect/qcom,sc8280xp-rpmh.yaml  |  101 +-
+ .../bindings/interconnect/qcom,x1e80100-rpmh.yaml  |   72 +
+ drivers/interconnect/qcom/Kconfig                  |   11 +
+ drivers/interconnect/qcom/Makefile                 |    2 +
+ drivers/interconnect/qcom/maili.c                  | 2091 ++++++++++++++++++++
+ drivers/interconnect/qcom/osm-l3.c                 |   16 +-
+ drivers/interconnect/qcom/sc8280xp.c               |  356 ++++
+ drivers/interconnect/qcom/x1e80100.c               |  485 +++++
+ include/dt-bindings/interconnect/qcom,maili-rpmh.h |  171 ++
+ 11 files changed, 3431 insertions(+), 3 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/interconnect/qcom,maili-rpmh.yaml
+ create mode 100644 drivers/interconnect/qcom/maili.c
+ create mode 100644 include/dt-bindings/interconnect/qcom,maili-rpmh.h
+Merging iio/togreg (2e2f2de7532cb iio: hid-sensors: Use implicit NULL pointer checks)
+$ git merge -m Merge branch 'togreg' of https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git iio/togreg
+Auto-merging .mailmap
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.yaml
+Auto-merging MAINTAINERS
+Auto-merging drivers/iio/accel/bmc150-accel-core.c
+Auto-merging drivers/iio/adc/Kconfig
+Auto-merging drivers/iio/adc/ti-ads124s08.c
+Auto-merging drivers/iio/imu/bmi160/bmi160_core.c
+Auto-merging drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c
+Auto-merging drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c
+Auto-merging drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.h
+Auto-merging drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c
+Auto-merging drivers/iio/light/gp2ap002.c
+Auto-merging drivers/iio/orientation/hid-sensor-rotation.c
+Auto-merging drivers/iio/temperature/hid-sensor-temperature.c
+Auto-merging lib/vsprintf.c
+Auto-merging sound/soc/samsung/aries_wm8994.c
+Merge made by the 'ort' strategy.
+ .mailmap                                           |    1 +
+ Documentation/ABI/testing/sysfs-bus-iio            |   11 +-
+ Documentation/ABI/testing/sysfs-bus-iio-adc-ad7192 |    2 +-
+ Documentation/ABI/testing/sysfs-bus-iio-frequency  |   11 +
+ .../ABI/testing/sysfs-bus-iio-frequency-adf4350    |   10 -
+ .../devicetree/bindings/iio/adc/avia-hx711.yaml    |   21 +
+ .../bindings/iio/adc/mediatek,mt6359-auxadc.yaml   |    3 +-
+ .../bindings/iio/adc/xlnx,versal-sysmon.yaml       |  154 +++
+ .../bindings/iio/chemical/sensirion,scd30.yaml     |    1 +
+ .../devicetree/bindings/iio/dac/adi,ad3530r.yaml   |   16 +-
+ .../devicetree/bindings/iio/dac/adi,ad5686.yaml    |   72 +-
+ .../devicetree/bindings/iio/dac/adi,ad5696.yaml    |   73 +-
+ .../bindings/iio/frequency/adi,adf41513.yaml       |  227 ++++
+ .../iio/magnetometer/qstcorp,qmc5883l.yaml         |   52 +
+ .../bindings/iio/temperature/ti,tmp117.yaml        |   16 +-
+ .../devicetree/bindings/vendor-prefixes.yaml       |    2 +
+ Documentation/iio/adf41513.rst                     |  199 ++++
+ Documentation/iio/index.rst                        |    1 +
+ MAINTAINERS                                        |   34 +-
+ drivers/extcon/extcon-adc-jack.c                   |    1 +
+ drivers/hwmon/iio_hwmon.c                          |    1 +
+ drivers/hwmon/ntc_thermistor.c                     |    1 +
+ drivers/iio/TODO                                   |   13 +-
+ drivers/iio/accel/adxl345_spi.c                    |    4 +-
+ drivers/iio/accel/adxl355_spi.c                    |    4 +-
+ drivers/iio/accel/adxl367_spi.c                    |    2 +-
+ drivers/iio/accel/adxl372_spi.c                    |    4 +-
+ drivers/iio/accel/adxl380_spi.c                    |    8 +-
+ drivers/iio/accel/bma220_spi.c                     |    2 +-
+ drivers/iio/accel/bma400_spi.c                     |    2 +-
+ drivers/iio/accel/bmc150-accel-core.c              |    6 +-
+ drivers/iio/accel/bmc150-accel-spi.c               |   18 +-
+ drivers/iio/accel/bmi088-accel-spi.c               |    6 +-
+ drivers/iio/accel/fxls8962af-core.c                |    2 +
+ drivers/iio/accel/fxls8962af-spi.c                 |    4 +-
+ drivers/iio/accel/hid-sensor-accel-3d.c            |   68 +-
+ drivers/iio/accel/kxsd9-spi.c                      |    2 +-
+ drivers/iio/accel/mma7455_spi.c                    |    4 +-
+ drivers/iio/accel/sca3000.c                        |    8 +-
+ drivers/iio/accel/sca3300.c                        |    4 +-
+ drivers/iio/accel/st_accel_spi.c                   |   40 +-
+ drivers/iio/accel/stk8312.c                        |   12 +-
+ drivers/iio/accel/stk8ba50.c                       |    9 +-
+ drivers/iio/adc/Kconfig                            |   47 +-
+ drivers/iio/adc/Makefile                           |    4 +
+ drivers/iio/adc/ad4000.c                           |   62 +-
+ drivers/iio/adc/ad4030.c                           |   14 +-
+ drivers/iio/adc/ad4080.c                           |   22 +-
+ drivers/iio/adc/ad4130.c                           |   12 +-
+ drivers/iio/adc/ad4134.c                           |    2 +-
+ drivers/iio/adc/ad4170-4.c                         |    6 +-
+ drivers/iio/adc/ad4851.c                           |   18 +-
+ drivers/iio/adc/ad7091r8.c                         |    6 +-
+ drivers/iio/adc/ad7124.c                           |    4 +-
+ drivers/iio/adc/ad7173.c                           |   26 +-
+ drivers/iio/adc/ad7191.c                           |    2 +-
+ drivers/iio/adc/ad7192.c                           |   10 +-
+ drivers/iio/adc/ad7266.c                           |    4 +-
+ drivers/iio/adc/ad7280a.c                          |    2 +-
+ drivers/iio/adc/ad7292.c                           |    2 +-
+ drivers/iio/adc/ad7298.c                           |    2 +-
+ drivers/iio/adc/ad7380.c                           |   36 +-
+ drivers/iio/adc/ad7476.c                           |   60 +-
+ drivers/iio/adc/ad7606_spi.c                       |   22 +-
+ drivers/iio/adc/ad7766.c                           |   12 +-
+ drivers/iio/adc/ad7768-1.c                         |    8 +-
+ drivers/iio/adc/ad7779.c                           |    6 +-
+ drivers/iio/adc/ad7780.c                           |    8 +-
+ drivers/iio/adc/ad7791.c                           |   10 +-
+ drivers/iio/adc/ad7793.c                           |   18 +-
+ drivers/iio/adc/ad7887.c                           |    2 +-
+ drivers/iio/adc/ad7923.c                           |   14 +-
+ drivers/iio/adc/ad7944.c                           |    6 +-
+ drivers/iio/adc/ad7949.c                           |    6 +-
+ drivers/iio/adc/ad9467.c                           |   14 +-
+ drivers/iio/adc/ade9000.c                          |    4 +-
+ drivers/iio/adc/envelope-detector.c                |    1 +
+ drivers/iio/adc/hi8435.c                           |    2 +-
+ drivers/iio/adc/hx711.c                            |   98 +-
+ drivers/iio/adc/max1027.c                          |   12 +-
+ drivers/iio/adc/max1118.c                          |    6 +-
+ drivers/iio/adc/max11205.c                         |    4 +-
+ drivers/iio/adc/max11410.c                         |    2 +-
+ drivers/iio/adc/max1241.c                          |    9 +-
+ drivers/iio/adc/max14001.c                         |    4 +-
+ drivers/iio/adc/mcp320x.c                          |   26 +-
+ drivers/iio/adc/mcp3564.c                          |   24 +-
+ drivers/iio/adc/mcp3911.c                          |   14 +-
+ drivers/iio/adc/mt6323-auxadc.c                    |  314 +++++
+ drivers/iio/adc/rohm-bd79112.c                     |    2 +-
+ drivers/iio/adc/rtq6056.c                          |    4 +-
+ drivers/iio/adc/stm32-dfsdm-adc.c                  |   12 +-
+ drivers/iio/adc/ti-adc081c.c                       |    6 +-
+ drivers/iio/adc/ti-adc0832.c                       |    8 +-
+ drivers/iio/adc/ti-adc084s021.c                    |    8 +-
+ drivers/iio/adc/ti-adc108s102.c                    |    2 +-
+ drivers/iio/adc/ti-adc12138.c                      |    6 +-
+ drivers/iio/adc/ti-adc128s052.c                    |   24 +-
+ drivers/iio/adc/ti-adc161s626.c                    |    4 +-
+ drivers/iio/adc/ti-ads1015.c                       |   46 +-
+ drivers/iio/adc/ti-ads1018.c                       |    4 +-
+ drivers/iio/adc/ti-ads124s08.c                     |    4 +-
+ drivers/iio/adc/ti-ads1298.c                       |    2 +-
+ drivers/iio/adc/ti-ads131e08.c                     |    6 +-
+ drivers/iio/adc/ti-ads131m02.c                     |   10 +-
+ drivers/iio/adc/ti-ads7950.c                       |   24 +-
+ drivers/iio/adc/ti-ads8688.c                       |    4 +-
+ drivers/iio/adc/ti-lmp92064.c                      |    2 +-
+ drivers/iio/adc/ti-tlc4541.c                       |    4 +-
+ drivers/iio/adc/ti-tsc2046.c                       |   25 +-
+ drivers/iio/adc/versal-sysmon-core.c               | 1052 +++++++++++++++++
+ drivers/iio/adc/versal-sysmon-i2c.c                |  134 +++
+ drivers/iio/adc/versal-sysmon.c                    |   92 ++
+ drivers/iio/adc/versal-sysmon.h                    |  120 ++
+ drivers/iio/addac/ad74115.c                        |    2 +-
+ drivers/iio/afe/iio-rescale.c                      |    1 +
+ drivers/iio/amplifiers/ad8366.c                    |   26 +-
+ drivers/iio/amplifiers/ada4250.c                   |    2 +-
+ drivers/iio/buffer/industrialio-buffer-cb.c        |    1 +
+ drivers/iio/buffer/industrialio-hw-consumer.c      |    1 +
+ drivers/iio/chemical/bme680_spi.c                  |    2 +-
+ drivers/iio/chemical/ens160_spi.c                  |    2 +-
+ drivers/iio/chemical/scd30_core.c                  |   10 +-
+ drivers/iio/chemical/scd30_i2c.c                   |    4 +-
+ drivers/iio/chemical/scd30_serial.c                |    4 +-
+ drivers/iio/chemical/sps30_i2c.c                   |    4 +-
+ drivers/iio/chemical/sps30_serial.c                |    4 +-
+ .../iio/common/hid-sensors/hid-sensor-attributes.c |   81 +-
+ .../iio/common/hid-sensors/hid-sensor-trigger.c    |    7 +-
+ .../iio/common/hid-sensors/hid-sensor-trigger.h    |    2 +-
+ drivers/iio/dac/Kconfig                            |   18 +-
+ drivers/iio/dac/Makefile                           |    1 +
+ drivers/iio/dac/ad3530r.c                          |  359 +++++-
+ drivers/iio/dac/ad5064.c                           |   32 +-
+ drivers/iio/dac/ad5360.c                           |   16 +-
+ drivers/iio/dac/ad5380.c                           |   32 +-
+ drivers/iio/dac/ad5446-spi.c                       |   62 +-
+ drivers/iio/dac/ad5449.c                           |   14 +-
+ drivers/iio/dac/ad5504.c                           |    4 +-
+ drivers/iio/dac/ad5624r_spi.c                      |   12 +-
+ drivers/iio/dac/ad5686.c                           |   40 +-
+ drivers/iio/dac/ad5686.h                           |    4 +
+ drivers/iio/dac/ad5706r.c                          |    2 +-
+ drivers/iio/dac/ad5755.c                           |   10 +-
+ drivers/iio/dac/ad5758.c                           |    2 +-
+ drivers/iio/dac/ad5761.c                           |    8 +-
+ drivers/iio/dac/ad5764.c                           |    8 +-
+ drivers/iio/dac/ad5766.c                           |    4 +-
+ drivers/iio/dac/ad5770r.c                          |    2 +-
+ drivers/iio/dac/ad5791.c                           |   10 +-
+ drivers/iio/dac/ad7293.c                           |    2 +-
+ drivers/iio/dac/ad7303.c                           |    2 +-
+ drivers/iio/dac/ad8460.c                           |    1 +
+ drivers/iio/dac/ad8801.c                           |    4 +-
+ drivers/iio/dac/ad9739a.c                          |    2 +-
+ drivers/iio/dac/dpot-dac.c                         |    1 +
+ drivers/iio/dac/ltc1660.c                          |    4 +-
+ drivers/iio/dac/ltc2632.c                          |   44 +-
+ drivers/iio/dac/ltc2664.c                          |    4 +-
+ drivers/iio/dac/ltc2688.c                          |    2 +-
+ drivers/iio/dac/max22007.c                         |    2 +-
+ drivers/iio/dac/max5522.c                          |   31 +-
+ drivers/iio/dac/mcf54415_dac.c                     |  180 +++
+ drivers/iio/dac/mcp4821.c                          |   12 +-
+ drivers/iio/dac/mcp4922.c                          |    8 +-
+ drivers/iio/dac/rohm-bd79703.c                     |    8 +-
+ drivers/iio/dac/ti-dac082s085.c                    |   12 +-
+ drivers/iio/dac/ti-dac7311.c                       |    6 +-
+ drivers/iio/dac/ti-dac7612.c                       |    2 +-
+ drivers/iio/filter/admv8818.c                      |    2 +-
+ drivers/iio/frequency/Kconfig                      |   10 +
+ drivers/iio/frequency/Makefile                     |    1 +
+ drivers/iio/frequency/ad9523.c                     |    3 +-
+ drivers/iio/frequency/adf41513.c                   | 1246 ++++++++++++++++++++
+ drivers/iio/frequency/adf4350.c                    |    4 +-
+ drivers/iio/frequency/adf4371.c                    |    4 +-
+ drivers/iio/frequency/adf4377.c                    |   85 +-
+ drivers/iio/frequency/admv1013.c                   |    2 +-
+ drivers/iio/frequency/admv1014.c                   |    2 +-
+ drivers/iio/frequency/adrf6780.c                   |    2 +-
+ drivers/iio/gyro/adis16080.c                       |    4 +-
+ drivers/iio/gyro/adis16136.c                       |    8 +-
+ drivers/iio/gyro/adis16260.c                       |   12 +-
+ drivers/iio/gyro/adxrs450.c                        |    4 +-
+ drivers/iio/gyro/bmg160_spi.c                      |    7 +-
+ drivers/iio/gyro/fxas21002c_spi.c                  |    2 +-
+ drivers/iio/gyro/hid-sensor-gyro-3d.c              |   57 +-
+ drivers/iio/gyro/st_gyro_spi.c                     |   18 +-
+ drivers/iio/health/afe4403.c                       |    2 +-
+ drivers/iio/humidity/hid-sensor-humidity.c         |   35 +-
+ drivers/iio/humidity/hts221_spi.c                  |    2 +-
+ drivers/iio/iio_core_trigger.h                     |    2 +-
+ drivers/iio/imu/adis16400.c                        |   30 +-
+ drivers/iio/imu/adis16460.c                        |    2 +-
+ drivers/iio/imu/adis16475.c                        |   54 +-
+ drivers/iio/imu/adis16480.c                        |   44 +-
+ drivers/iio/imu/adis16550.c                        |   89 +-
+ drivers/iio/imu/bmi160/bmi160_core.c               |    2 +-
+ drivers/iio/imu/bmi160/bmi160_spi.c                |    4 +-
+ drivers/iio/imu/bmi270/bmi270_spi.c                |    5 +-
+ drivers/iio/imu/bmi323/bmi323_core.c               |    2 +-
+ drivers/iio/imu/bmi323/bmi323_spi.c                |    2 +-
+ drivers/iio/imu/fxos8700_core.c                    |    2 +-
+ drivers/iio/imu/fxos8700_spi.c                     |    2 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600.h        |   10 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c  |   18 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c |   45 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.h |    3 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600_core.c   |   12 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c   |   16 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600_i2c.c    |    6 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600_spi.c    |   20 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600_temp.c   |    3 +-
+ drivers/iio/imu/inv_icm45600/inv_icm45600.h        |    2 +-
+ drivers/iio/imu/inv_icm45600/inv_icm45600_buffer.c |    7 +-
+ drivers/iio/imu/inv_icm45600/inv_icm45600_core.c   |    2 +-
+ drivers/iio/imu/inv_icm45600/inv_icm45600_i2c.c    |    2 +-
+ drivers/iio/imu/inv_icm45600/inv_icm45600_spi.c    |   16 +-
+ drivers/iio/imu/inv_mpu6050/inv_mpu_magn.c         |    4 +-
+ drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c          |   34 +-
+ drivers/iio/imu/smi240.c                           |    2 +-
+ drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c        |   48 +-
+ drivers/iio/imu/st_lsm9ds0/st_lsm9ds0_spi.c        |    4 +-
+ drivers/iio/industrialio-backend.c                 |    8 +-
+ drivers/iio/industrialio-core.c                    |   49 +-
+ drivers/iio/inkern.c                               |   54 +-
+ drivers/iio/light/cm3605.c                         |    1 +
+ drivers/iio/light/gp2ap002.c                       |    1 +
+ drivers/iio/light/hid-sensor-als.c                 |   52 +-
+ drivers/iio/light/hid-sensor-prox.c                |   49 +-
+ drivers/iio/light/opt3001.c                        |  381 +++---
+ drivers/iio/light/st_uvis25_spi.c                  |    2 +-
+ drivers/iio/light/tcs3472.c                        |  317 ++++-
+ drivers/iio/magnetometer/Kconfig                   |   11 +
+ drivers/iio/magnetometer/Makefile                  |    2 +
+ drivers/iio/magnetometer/bmc150_magn.c             |   50 +-
+ drivers/iio/magnetometer/bmc150_magn_spi.c         |    6 +-
+ drivers/iio/magnetometer/hid-sensor-magn-3d.c      |   73 +-
+ drivers/iio/magnetometer/hmc5843_spi.c             |    6 +-
+ drivers/iio/magnetometer/mmc5983.c                 |    2 +-
+ drivers/iio/magnetometer/qmc5883l.c                |  516 ++++++++
+ drivers/iio/magnetometer/st_magn_spi.c             |   12 +-
+ drivers/iio/multiplexer/iio-mux.c                  |    1 +
+ drivers/iio/orientation/hid-sensor-incl-3d.c       |   55 +-
+ drivers/iio/orientation/hid-sensor-rotation.c      |   48 +-
+ .../iio/position/hid-sensor-custom-intel-hinge.c   |   19 +-
+ drivers/iio/potentiometer/max5481.c                |    8 +-
+ drivers/iio/potentiometer/max5487.c                |    6 +-
+ drivers/iio/potentiometer/mcp41010.c               |   12 +-
+ drivers/iio/potentiometer/mcp4131.c                |  128 +-
+ drivers/iio/potentiometer/x9250.c                  |    4 +-
+ drivers/iio/potentiostat/lmp91000.c                |    1 +
+ drivers/iio/pressure/abp2030pa_spi.c               |    2 +-
+ drivers/iio/pressure/bmp280-core.c                 |   32 +-
+ drivers/iio/pressure/bmp280-spi.c                  |   14 +-
+ drivers/iio/pressure/hid-sensor-press.c            |   57 +-
+ drivers/iio/pressure/hsc030pa_spi.c                |    2 +-
+ drivers/iio/pressure/mpl115_spi.c                  |    2 +-
+ drivers/iio/pressure/mprls0025pa_spi.c             |    2 +-
+ drivers/iio/pressure/ms5611_spi.c                  |    4 +-
+ drivers/iio/pressure/rohm-bm1390.c                 |   23 +-
+ drivers/iio/pressure/st_pressure_spi.c             |   24 +-
+ drivers/iio/pressure/zpa2326_spi.c                 |    2 +-
+ drivers/iio/proximity/as3935.c                     |    2 +-
+ drivers/iio/proximity/vl53l1x-i2c.c                |   39 +-
+ drivers/iio/resolver/ad2s1200.c                    |    4 +-
+ drivers/iio/resolver/ad2s1210.c                    |   32 +-
+ drivers/iio/resolver/ad2s90.c                      |    2 +-
+ drivers/iio/temperature/hid-sensor-temperature.c   |   34 +-
+ drivers/iio/temperature/ltc2983.c                  |   10 +-
+ drivers/iio/temperature/max31856.c                 |    2 +-
+ drivers/iio/temperature/max31865.c                 |    2 +-
+ drivers/iio/temperature/maxim_thermocouple.c       |   18 +-
+ drivers/iio/temperature/tmp117.c                   |   13 +
+ drivers/iio/test/iio-test-format.c                 |   97 +-
+ drivers/iio/test/iio-test-rescale.c                |    2 +-
+ drivers/input/joystick/adc-joystick.c              |    1 +
+ drivers/input/keyboard/adc-keys.c                  |    1 +
+ drivers/input/touchscreen/colibri-vf50-ts.c        |    1 +
+ drivers/input/touchscreen/resistive-adc-touch.c    |    1 +
+ drivers/phy/motorola/phy-cpcap-usb.c               |    1 +
+ drivers/power/supply/ab8500_btemp.c                |    1 +
+ drivers/power/supply/ab8500_charger.c              |    1 +
+ drivers/power/supply/ab8500_fg.c                   |    1 +
+ drivers/power/supply/axp20x_ac_power.c             |    1 +
+ drivers/power/supply/axp20x_battery.c              |    1 +
+ drivers/power/supply/axp20x_usb_power.c            |    1 +
+ drivers/power/supply/axp288_fuel_gauge.c           |    1 +
+ drivers/power/supply/cpcap-battery.c               |    1 +
+ drivers/power/supply/cpcap-charger.c               |    1 +
+ drivers/power/supply/da9150-charger.c              |    1 +
+ drivers/power/supply/generic-adc-battery.c         |    1 +
+ drivers/power/supply/ingenic-battery.c             |    1 +
+ drivers/power/supply/intel_dc_ti_battery.c         |    1 +
+ drivers/power/supply/lego_ev3_battery.c            |    1 +
+ drivers/power/supply/lp8788-charger.c              |    1 +
+ drivers/power/supply/max17040_battery.c            |    1 +
+ drivers/power/supply/mp2629_charger.c              |    1 +
+ drivers/power/supply/mt6370-charger.c              |    1 +
+ drivers/power/supply/qcom_smbx.c                   |    1 +
+ drivers/power/supply/rn5t618_power.c               |    1 +
+ drivers/power/supply/rx51_battery.c                |    1 +
+ drivers/power/supply/sc27xx_fuel_gauge.c           |    1 +
+ drivers/power/supply/twl4030_charger.c             |    1 +
+ drivers/power/supply/twl4030_madc_battery.c        |    1 +
+ drivers/power/supply/twl6030_charger.c             |    1 +
+ drivers/staging/iio/adc/ad7816.c                   |    6 +-
+ drivers/staging/iio/addac/adt7316-spi.c            |   13 +-
+ drivers/staging/iio/frequency/ad9832.c             |   24 +-
+ drivers/staging/iio/frequency/ad9834.c             |   30 +-
+ drivers/staging/iio/frequency/dds.h                |   24 +-
+ drivers/thermal/qcom/qcom-spmi-adc-tm5.c           |    1 +
+ drivers/thermal/qcom/qcom-spmi-temp-alarm.c        |    1 +
+ drivers/thermal/renesas/rzg3s_thermal.c            |    1 +
+ drivers/thermal/thermal-generic-adc.c              |    1 +
+ .../dt-bindings/iio/adc/mediatek,mt6323-auxadc.h   |   24 +
+ include/linux/iio/types.h                          |   20 +
+ include/linux/kstrtox.h                            |    3 +
+ include/linux/math64.h                             |   18 +
+ lib/kstrtox.c                                      |  115 +-
+ lib/kstrtox.h                                      |   17 +-
+ lib/math/div64.c                                   |   15 +
+ lib/math/test_mul_u64_u64_div_u64.c                |    1 +
+ lib/test-kstrtox.c                                 |  182 +++
+ lib/vsprintf.c                                     |    2 +-
+ sound/soc/codecs/audio-iio-aux.c                   |    1 +
+ sound/soc/samsung/aries_wm8994.c                   |    1 +
+ sound/soc/samsung/midas_wm1811.c                   |    1 +
+ sound/soc/stm/stm32_adfsdm.c                       |    1 +
+ 329 files changed, 7677 insertions(+), 1999 deletions(-)
+ create mode 100644 Documentation/ABI/testing/sysfs-bus-iio-frequency
+ create mode 100644 Documentation/devicetree/bindings/iio/adc/xlnx,versal-sysmon.yaml
+ create mode 100644 Documentation/devicetree/bindings/iio/frequency/adi,adf41513.yaml
+ create mode 100644 Documentation/devicetree/bindings/iio/magnetometer/qstcorp,qmc5883l.yaml
+ create mode 100644 Documentation/iio/adf41513.rst
+ create mode 100644 drivers/iio/adc/mt6323-auxadc.c
+ create mode 100644 drivers/iio/adc/versal-sysmon-core.c
+ create mode 100644 drivers/iio/adc/versal-sysmon-i2c.c
+ create mode 100644 drivers/iio/adc/versal-sysmon.c
+ create mode 100644 drivers/iio/adc/versal-sysmon.h
+ create mode 100644 drivers/iio/dac/mcf54415_dac.c
+ create mode 100644 drivers/iio/frequency/adf41513.c
+ create mode 100644 drivers/iio/magnetometer/qmc5883l.c
+ create mode 100644 include/dt-bindings/iio/adc/mediatek,mt6323-auxadc.h
+Merging nfc/for-next (ed85d4cbbfaa4 nfc: llcp: bound SNL TLV parsing to the skb and add length checks)
+$ git merge -m Merge branch 'for-next' of https://codeberg.org/linux-nfc/linux.git nfc/for-next
+Auto-merging MAINTAINERS
+Auto-merging net/nfc/nci/rsp.c
+CONFLICT (content): Merge conflict in net/nfc/nci/rsp.c
+Resolved 'net/nfc/nci/rsp.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 3020a85b2113e] Merge branch 'for-next' of https://codeberg.org/linux-nfc/linux.git
+$ git diff -M --stat --summary HEAD^..
+ net/nfc/nci/rsp.c | 9 +++++++++
+ 1 file changed, 9 insertions(+)
+Merging phy-next/next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git phy-next/next
+Already up to date.
+Merging soundwire/next (6409292ab5f7d soundwire: bus.h: repair kernel-doc comments)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git soundwire/next
+Merge made by the 'ort' strategy.
+ .../bindings/soundwire/qcom,soundwire.yaml         | 20 ++++++++--------
+ drivers/soundwire/bus.h                            | 10 ++++----
+ drivers/soundwire/dmi-quirks.c                     | 28 +++++++++++++++++++++-
+ drivers/soundwire/intel_auxdevice.c                |  1 +
+ drivers/soundwire/qcom.c                           | 23 +++++++++++++++---
+ 5 files changed, 63 insertions(+), 19 deletions(-)
+Merging extcon/extcon-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'extcon-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/extcon.git extcon/extcon-next
+Already up to date.
+Merging gnss/gnss-next (a13c140cc289c Linux 7.2-rc3)
+$ git merge -m Merge branch 'gnss-next' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git gnss/gnss-next
+Already up to date.
+Merging vfio/next (785562e31dbcd vfio: selftests: Ensure libvfio output dirs are always created)
+$ git merge -m Merge branch 'next' of https://github.com/awilliam/linux-vfio.git vfio/next
+Already up to date.
+Merging w1/for-next (169ae5e65e5aa w1: ds28e17: reject an oversize length on an I2C block read)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git w1/for-next
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/w1/omap-hdq.txt | 22 -------
+ Documentation/devicetree/bindings/w1/ti,hdq.yaml  | 70 +++++++++++++++++++++++
+ Documentation/w1/masters/omap-hdq.rst             |  2 +-
+ drivers/w1/masters/ds2482.c                       |  8 +++
+ drivers/w1/slaves/w1_ds28e17.c                    |  8 +++
+ 5 files changed, 87 insertions(+), 23 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/w1/omap-hdq.txt
+ create mode 100644 Documentation/devicetree/bindings/w1/ti,hdq.yaml
+Merging spmi/spmi-next (8cdeaa50eae8d Linux 7.2-rc2)
+$ git merge -m Merge branch 'spmi-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sboyd/spmi.git spmi/spmi-next
+Already up to date.
+Merging staging/staging-next (64276d9bfe4d1 staging: rtl8723bs: Remove space after cast)
+$ git merge -m Merge branch 'staging-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git staging/staging-next
+Merge made by the 'ort' strategy.
+ drivers/staging/axis-fifo/axis-fifo.c              |   3 +-
+ drivers/staging/fbtft/fb_upd161704.c               |   2 +-
+ drivers/staging/fbtft/fbtft-sysfs.c                |   2 +-
+ drivers/staging/greybus/audio_codec.c              |   5 +-
+ drivers/staging/greybus/audio_gb.c                 |  13 +
+ drivers/staging/greybus/audio_manager_sysfs.c      |   2 +-
+ drivers/staging/greybus/authentication.c           |   2 -
+ drivers/staging/greybus/hid.c                      |   2 +-
+ drivers/staging/greybus/loopback.c                 |  26 --
+ drivers/staging/greybus/power_supply.c             |   4 +-
+ drivers/staging/greybus/vibrator.c                 |   2 +-
+ drivers/staging/most/dim2/dim2.c                   |  14 +-
+ drivers/staging/nvec/nvec.c                        |   4 +-
+ drivers/staging/octeon/ethernet-rx.c               |  20 +-
+ drivers/staging/octeon/ethernet-rx.h               |   2 +-
+ drivers/staging/octeon/ethernet-tx.c               |  19 +-
+ drivers/staging/octeon/ethernet-tx.h               |   2 +-
+ drivers/staging/octeon/ethernet.c                  |  66 +++-
+ drivers/staging/rtl8723bs/core/rtw_ap.c            | 174 ++-------
+ drivers/staging/rtl8723bs/core/rtw_cmd.c           | 219 ++++++-----
+ drivers/staging/rtl8723bs/core/rtw_efuse.c         |   6 +-
+ drivers/staging/rtl8723bs/core/rtw_ieee80211.c     |  12 +-
+ drivers/staging/rtl8723bs/core/rtw_io.c            |   7 -
+ drivers/staging/rtl8723bs/core/rtw_ioctl_set.c     |  30 +-
+ drivers/staging/rtl8723bs/core/rtw_mlme.c          |  76 ++--
+ drivers/staging/rtl8723bs/core/rtw_mlme_ext.c      | 429 +++++++++++----------
+ drivers/staging/rtl8723bs/core/rtw_pwrctrl.c       |  28 +-
+ drivers/staging/rtl8723bs/core/rtw_recv.c          |  37 +-
+ drivers/staging/rtl8723bs/core/rtw_security.c      |  34 +-
+ drivers/staging/rtl8723bs/core/rtw_sta_mgt.c       |   4 +-
+ drivers/staging/rtl8723bs/core/rtw_wlan_util.c     |  71 ++--
+ drivers/staging/rtl8723bs/core/rtw_xmit.c          |  71 ++--
+ drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.c    |  26 +-
+ drivers/staging/rtl8723bs/hal/HalPhyRf.c           |  12 +-
+ drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c     |  58 +--
+ drivers/staging/rtl8723bs/hal/HalPwrSeqCmd.c       |  38 +-
+ drivers/staging/rtl8723bs/hal/hal_btcoex.c         |  10 +-
+ drivers/staging/rtl8723bs/hal/hal_com.c            |   8 +-
+ drivers/staging/rtl8723bs/hal/hal_com_phycfg.c     |   2 +-
+ drivers/staging/rtl8723bs/hal/hal_intf.c           |  14 +-
+ drivers/staging/rtl8723bs/hal/hal_sdio.c           |   2 +-
+ drivers/staging/rtl8723bs/hal/odm_DIG.c            |   2 +-
+ drivers/staging/rtl8723bs/hal/odm_EdcaTurboCheck.c |   2 +-
+ drivers/staging/rtl8723bs/hal/odm_HWConfig.c       |   3 +-
+ drivers/staging/rtl8723bs/hal/rtl8723b_cmd.c       | 240 +++++-------
+ drivers/staging/rtl8723bs/hal/rtl8723b_dm.c        |   6 +-
+ drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c  |  58 +--
+ drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c    |   2 +-
+ drivers/staging/rtl8723bs/hal/rtl8723b_rf6052.c    |  67 +---
+ drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c     |  10 +-
+ drivers/staging/rtl8723bs/hal/sdio_halinit.c       |  58 ++-
+ drivers/staging/rtl8723bs/include/drv_types.h      |   2 +-
+ drivers/staging/rtl8723bs/include/hal_com_reg.h    |  10 +-
+ drivers/staging/rtl8723bs/include/hal_data.h       |  99 -----
+ drivers/staging/rtl8723bs/include/hal_intf.h       |   2 +-
+ drivers/staging/rtl8723bs/include/ieee80211.h      |  30 +-
+ drivers/staging/rtl8723bs/include/ioctl_cfg80211.h |   4 +-
+ .../rtl8723bs/include/osdep_service_linux.h        |   3 -
+ drivers/staging/rtl8723bs/include/rtl8192c_recv.h  |   3 -
+ drivers/staging/rtl8723bs/include/rtl8723b_hal.h   |  18 +-
+ drivers/staging/rtl8723bs/include/rtw_ap.h         |   2 +-
+ drivers/staging/rtl8723bs/include/rtw_cmd.h        | 144 ++++---
+ drivers/staging/rtl8723bs/include/rtw_event.h      |   4 -
+ drivers/staging/rtl8723bs/include/rtw_ioctl_set.h  |   2 +-
+ drivers/staging/rtl8723bs/include/rtw_mlme.h       |  61 ++-
+ drivers/staging/rtl8723bs/include/rtw_mlme_ext.h   |  56 +--
+ drivers/staging/rtl8723bs/include/rtw_pwrctrl.h    |   6 +-
+ drivers/staging/rtl8723bs/include/rtw_security.h   |  10 +-
+ drivers/staging/rtl8723bs/include/sta_info.h       |   2 +-
+ drivers/staging/rtl8723bs/include/wifi.h           |  34 +-
+ drivers/staging/rtl8723bs/include/wlan_bssdef.h    |  38 +-
+ drivers/staging/rtl8723bs/include/xmit_osdep.h     |   2 +-
+ drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c  | 146 ++++---
+ drivers/staging/rtl8723bs/os_dep/os_intfs.c        |  18 +-
+ drivers/staging/rtl8723bs/os_dep/sdio_intf.c       |  18 +-
+ drivers/staging/rtl8723bs/os_dep/wifi_regd.c       |   4 +-
+ drivers/staging/rtl8723bs/os_dep/xmit_linux.c      |   6 +-
+ drivers/staging/sm750fb/Kconfig                    |   1 +
+ drivers/staging/sm750fb/sm750.c                    |  27 +-
+ drivers/staging/sm750fb/sm750.h                    |  17 +-
+ drivers/staging/sm750fb/sm750_hw.c                 |  28 +-
+ 81 files changed, 1218 insertions(+), 1585 deletions(-)
+Merging counter-next/counter-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'counter-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git counter-next/counter-next
+Already up to date.
+Merging mux/for-next (59b723cd2adba Linux 6.12-rc6)
+$ git merge -m Merge branch 'for-next' of https://gitlab.com/peda-linux/mux.git mux/for-next
+Already up to date.
+Merging dmaengine/next (338c853b7c3b4 dmaengine: Constify struct dma_descriptor_metadata_ops)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git dmaengine/next
+Auto-merging MAINTAINERS
+Auto-merging drivers/crypto/atmel-aes.c
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/dma/altr,msgdma.yaml       |   2 +-
+ .../devicetree/bindings/dma/mediatek,uart-dma.yaml |   1 +
+ .../devicetree/bindings/dma/qcom,bam-dma.yaml      |   2 +-
+ .../bindings/dma/xilinx/xlnx,axi-dma.yaml          |  13 +-
+ Documentation/driver-api/dmaengine/client.rst      |   9 +
+ MAINTAINERS                                        |   4 +-
+ drivers/crypto/atmel-aes.c                         |  10 +-
+ drivers/dma/arm-dma350.c                           |   9 +
+ drivers/dma/dmaengine.c                            |   9 +-
+ drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c     |   2 +-
+ drivers/dma/dw-edma/dw-edma-core.c                 | 298 ++++++++-------------
+ drivers/dma/dw-edma/dw-edma-core.h                 |  74 +++--
+ drivers/dma/dw-edma/dw-edma-pcie.c                 |   8 +-
+ drivers/dma/dw-edma/dw-edma-v0-core.c              | 240 ++++++++---------
+ drivers/dma/dw-edma/dw-hdma-v0-core.c              | 196 +++++++-------
+ drivers/dma/dw-edma/dw-hdma-v0-regs.h              |   2 +-
+ drivers/dma/fsl-edma-main.c                        |   2 +
+ drivers/dma/fsl_raid.c                             |  58 ++--
+ drivers/dma/fsl_raid.h                             |   4 +-
+ drivers/dma/hisi_dma.c                             |   2 +-
+ drivers/dma/mediatek/mtk-uart-apdma.c              |   2 +-
+ drivers/dma/pl330.c                                |  19 +-
+ drivers/dma/tegra210-adma.c                        |  12 +-
+ drivers/dma/ti/k3-udma.c                           |   2 +-
+ drivers/dma/xilinx/xilinx_dma.c                    |  66 +++--
+ drivers/dma/xilinx/zynqmp_dma.c                    |   6 +-
+ drivers/nvme/target/pci-epf.c                      |  33 +--
+ drivers/pci/endpoint/functions/pci-epf-mhi.c       |  52 ++--
+ drivers/pci/endpoint/functions/pci-epf-test.c      |   8 +-
+ include/linux/dma/edma.h                           |  10 +-
+ include/linux/dmaengine.h                          | 151 ++++++++++-
+ 31 files changed, 696 insertions(+), 610 deletions(-)
+Merging cgroup/for-next (6486f6d8f5455 Merge branch 'for-7.3' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git cgroup/for-next
+Auto-merging Documentation/admin-guide/cgroup-v2.rst
+Auto-merging kernel/cgroup/cpuset.c
+Auto-merging kernel/sched/psi.c
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/cgroup-v1/memcg_test.rst |   2 +-
+ Documentation/admin-guide/cgroup-v2.rst            |  30 +-
+ include/linux/cgroup-defs.h                        |   2 +-
+ include/linux/cgroup.h                             |   2 +-
+ include/linux/psi.h                                |   4 +-
+ kernel/cgroup/cgroup.c                             |  23 +-
+ kernel/cgroup/cpuset-internal.h                    |  12 +-
+ kernel/cgroup/cpuset.c                             | 530 +++++++++++++--------
+ kernel/sched/psi.c                                 |  75 ++-
+ tools/cgroup/iocost_monitor.py                     |  10 +-
+ .../selftests/cgroup/lib/include/cgroup_util.h     |   1 +
+ tools/testing/selftests/cgroup/test_cpu.c          |  43 +-
+ tools/testing/selftests/cgroup/test_cpuset.c       | 243 ++++++++++
+ tools/testing/selftests/cgroup/test_cpuset_prs.sh  |  11 +-
+ .../testing/selftests/cgroup/test_hugetlb_memcg.c  |  10 +-
+ 15 files changed, 761 insertions(+), 237 deletions(-)
+Merging scsi/for-next (cf1af0ccca54d Merge branch 'misc' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi.git scsi/for-next
+Merge made by the 'ort' strategy.
+Merging scsi-mkp/for-next (aec9f1f18ffdd scsi: parisc: superio: Spelling s/Peterson/Petersen/)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git scsi-mkp/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/ufs/ufs-common.yaml        |  58 +++++++
+ MAINTAINERS                                        |   2 +-
+ drivers/parisc/superio.c                           |   2 +-
+ drivers/scsi/mpt3sas/Makefile                      |   2 +
+ drivers/scsi/mpt3sas/mpt3sas_base.h                |  20 +++
+ drivers/scsi/mpt3sas/mpt3sas_config.c              |  36 ++++
+ drivers/scsi/mpt3sas/mpt3sas_hwmon.c               | 191 +++++++++++++++++++++
+ drivers/scsi/mpt3sas/mpt3sas_scsih.c               |   6 +
+ drivers/scsi/qla2xxx/qla_os.c                      |  15 +-
+ drivers/scsi/virtio_scsi.c                         |  18 +-
+ drivers/ufs/core/ufs-sysfs.c                       |   1 +
+ drivers/ufs/core/ufs-txeq.c                        |  73 ++++++--
+ drivers/ufs/core/ufshcd.c                          |  23 ++-
+ drivers/ufs/host/ufs-mediatek.c                    |  12 +-
+ drivers/ufs/host/ufs-mediatek.h                    |   4 +-
+ drivers/ufs/host/ufs-qcom.c                        |   9 +-
+ drivers/ufs/host/ufshcd-pltfrm.c                   | 154 +++++++++++++++++
+ include/ufs/ufshcd.h                               |   7 +-
+ 18 files changed, 579 insertions(+), 54 deletions(-)
+ create mode 100644 drivers/scsi/mpt3sas/mpt3sas_hwmon.c
+$ git am -3 ../patches/device-id-zorro
+Applying: foofof
+$ git reset HEAD^
+Unstaged changes after reset:
+M	include/linux/device-id/zorro.h
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master 18176263e22b4] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git
+ Date: Fri Jul 17 18:31:52 2026 +0100
+Merging vhost/linux-next (1f673033c0273 vduse: Add suspend)
+$ git merge -m Merge branch 'linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git vhost/linux-next
+Auto-merging drivers/vdpa/vdpa_user/vduse_dev.c
+Merge made by the 'ort' strategy.
+ drivers/vdpa/vdpa_user/vduse_dev.c | 136 +++++++++++++++++++++++++++++++++----
+ include/uapi/linux/vduse.h         |  29 +++++++-
+ 2 files changed, 150 insertions(+), 15 deletions(-)
+Merging rpmsg/for-next (f041a428881dc Merge branches 'hwspinlock-next', 'rpmsg-next' and 'rproc-next' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/remoteproc/linux.git rpmsg/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../remoteproc/amlogic,meson-mx-ao-arc.yaml        |   2 +-
+ .../devicetree/bindings/remoteproc/qcom,adsp.yaml  |  82 ++++-----
+ .../remoteproc/qcom,kaanapali-soccp-pas.yaml       | 156 ++++++++++++++++
+ .../bindings/remoteproc/qcom,milos-pas.yaml        |  26 ++-
+ .../bindings/remoteproc/qcom,pas-common.yaml       |  28 ++-
+ .../bindings/remoteproc/qcom,qcs404-pas.yaml       |  22 ++-
+ .../bindings/remoteproc/qcom,sa8775p-pas.yaml      |  22 ++-
+ .../bindings/remoteproc/qcom,sc7180-pas.yaml       |  28 +++
+ .../bindings/remoteproc/qcom,sc8280xp-pas.yaml     |  28 +++
+ .../bindings/remoteproc/qcom,sdx55-pas.yaml        |  24 ++-
+ .../bindings/remoteproc/qcom,shikra-pas.yaml       |  20 ++
+ .../bindings/remoteproc/qcom,sm6115-pas.yaml       |  28 +++
+ .../bindings/remoteproc/qcom,sm6350-pas.yaml       |  28 +++
+ .../bindings/remoteproc/qcom,sm6375-pas.yaml       |  28 +++
+ .../bindings/remoteproc/qcom,sm8150-pas.yaml       |  28 +++
+ .../bindings/remoteproc/qcom,sm8350-pas.yaml       |  28 +++
+ .../bindings/remoteproc/qcom,sm8550-pas.yaml       |  32 ++++
+ .../bindings/remoteproc/ti,pru-consumer.yaml       |   1 -
+ Documentation/staging/rpmsg.rst                    |   1 -
+ MAINTAINERS                                        |   2 +-
+ drivers/firmware/xilinx/zynqmp.c                   |  93 ++++++++++
+ drivers/hwspinlock/hwspinlock_core.c               |   2 +-
+ drivers/remoteproc/Kconfig                         |   4 +-
+ drivers/remoteproc/imx_dsp_rproc.c                 |  41 +----
+ drivers/remoteproc/imx_rproc.c                     |  40 +---
+ drivers/remoteproc/keystone_remoteproc.c           |   2 +-
+ drivers/remoteproc/mtk_scp.c                       |   9 +-
+ drivers/remoteproc/omap_remoteproc.c               |   2 +-
+ drivers/remoteproc/qcom_common.c                   |   4 +-
+ drivers/remoteproc/qcom_common.h                   |   6 +
+ drivers/remoteproc/qcom_q6v5.c                     |  57 ++++--
+ drivers/remoteproc/qcom_q6v5.h                     |   4 +
+ drivers/remoteproc/qcom_q6v5_adsp.c                |   1 +
+ drivers/remoteproc/qcom_q6v5_mss.c                 |  24 ++-
+ drivers/remoteproc/qcom_q6v5_pas.c                 | 171 ++++++++++++++---
+ drivers/remoteproc/qcom_sysmon.c                   |  19 ++
+ drivers/remoteproc/qcom_wcnss.c                    |  35 ++--
+ drivers/remoteproc/rcar_rproc.c                    |  41 +----
+ drivers/remoteproc/remoteproc_core.c               |  43 ++++-
+ drivers/remoteproc/remoteproc_internal.h           |  54 ++++++
+ drivers/remoteproc/st_remoteproc.c                 |  31 +---
+ drivers/remoteproc/stm32_rproc.c                   |  39 +---
+ drivers/remoteproc/ti_k3_r5_remoteproc.c           |   8 +-
+ drivers/remoteproc/xlnx_r5_remoteproc.c            | 204 +++++++++------------
+ drivers/rpmsg/qcom_glink_native.c                  |   5 +-
+ drivers/rpmsg/qcom_glink_smem.c                    |   7 +
+ drivers/rpmsg/qcom_smd.c                           |   2 +-
+ drivers/rpmsg/rpmsg_char.c                         |  11 +-
+ drivers/rpmsg/rpmsg_core.c                         |   6 +-
+ drivers/soc/qcom/mdt_loader.c                      |  12 +-
+ include/linux/firmware/xlnx-zynqmp.h               |  12 ++
+ include/linux/remoteproc.h                         |   4 +
+ include/linux/soc/qcom/mdt_loader.h                |   6 +-
+ 53 files changed, 1148 insertions(+), 465 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/remoteproc/qcom,kaanapali-soccp-pas.yaml
+Merging gpio-brgl/gpio/for-next (fbd35c8cc6c6e gpio: pcf857x: Use le16_to_cpu() and cpu_to_le16() in IO accessors)
+$ git merge -m Merge branch 'gpio/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git gpio-brgl/gpio/for-next
+Auto-merging drivers/gpio/gpio-mvebu.c
+Auto-merging drivers/gpio/gpio-sloppy-logic-analyzer.c
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/gpio/cdns,gpio.yaml        |   7 +-
+ .../devicetree/bindings/gpio/gpio-pca95xx.yaml     |   3 +
+ .../bindings/gpio/realtek,rtd1625-gpio.yaml        |  71 +++
+ drivers/gpio/Kconfig                               |  14 +-
+ drivers/gpio/Makefile                              |   1 +
+ drivers/gpio/gpio-amd-fch.c                        |  22 +-
+ drivers/gpio/gpio-bd72720.c                        |  24 +-
+ drivers/gpio/gpio-bt8xx.c                          |  21 +-
+ drivers/gpio/gpio-mt7621.c                         |  40 +-
+ drivers/gpio/gpio-mvebu.c                          |  40 +-
+ drivers/gpio/gpio-nomadik.c                        |  64 +--
+ drivers/gpio/gpio-pca9570.c                        |   2 +-
+ drivers/gpio/gpio-pcf857x.c                        |  13 +-
+ drivers/gpio/gpio-rcar.c                           |   1 -
+ drivers/gpio/gpio-rockchip.c                       |  16 +-
+ drivers/gpio/gpio-rtd1625.c                        | 611 +++++++++++++++++++++
+ drivers/gpio/gpio-sifive.c                         |   1 +
+ drivers/gpio/gpio-sloppy-logic-analyzer.c          |   1 +
+ drivers/gpio/gpio-tb10x.c                          |   5 +-
+ drivers/gpio/gpio-vf610.c                          |   1 +
+ drivers/gpio/gpiolib-cdev.c                        |   9 +-
+ drivers/gpio/gpiolib-kunit.c                       | 280 +++++++++-
+ drivers/gpio/gpiolib-swnode.c                      |  23 +-
+ include/linux/gpio/machine.h                       |   2 +-
+ 24 files changed, 1104 insertions(+), 168 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/gpio/realtek,rtd1625-gpio.yaml
+ create mode 100644 drivers/gpio/gpio-rtd1625.c
+Merging gpio-intel/for-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git gpio-intel/for-next
+Already up to date.
+Merging pinctrl/for-next (a7c4e636e400e Merge branch 'devel' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl.git pinctrl/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/pinctrl/apple,pinctrl.yaml |   2 +
+ .../pinctrl/aspeed,ast2700-soc1-pinctrl.yaml       |   1 +
+ .../bindings/pinctrl/nxp,s32g2-siul2-pinctrl.yaml  |  90 ++-
+ Documentation/driver-api/pin-control.rst           |   2 +-
+ MAINTAINERS                                        |   7 +
+ arch/sh/boards/mach-rsk/devices-rsk7203.c          | 300 +++++---
+ arch/sh/include/cpu-common/cpu/pfc.h               |   3 +
+ arch/sh/kernel/cpu/pfc.c                           |  20 +-
+ drivers/pinctrl/aspeed/pinctrl-aspeed-g7-soc1.c    |   6 +-
+ drivers/pinctrl/bcm/pinctrl-bcm2835.c              |   1 -
+ drivers/pinctrl/freescale/pinctrl-imx1-core.c      |   7 +-
+ drivers/pinctrl/nxp/Kconfig                        |   3 +-
+ drivers/pinctrl/nxp/pinctrl-s32.h                  |  35 +-
+ drivers/pinctrl/nxp/pinctrl-s32cc.c                | 768 ++++++++++++++++++---
+ drivers/pinctrl/nxp/pinctrl-s32g2.c                |  47 +-
+ drivers/pinctrl/pinctrl-rp1.c                      |   2 +-
+ drivers/pinctrl/pinctrl-sx150x.c                   |   1 +
+ drivers/pinctrl/pinctrl-tb10x.c                    |   2 +-
+ drivers/pinctrl/renesas/gpio.c                     |  28 +
+ 19 files changed, 1124 insertions(+), 201 deletions(-)
+Merging pinctrl-intel/for-next (ed1608d5e3c44 Merge patch series "upboard pinctrl support for device id INTC1055")
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git pinctrl-intel/for-next
+Auto-merging drivers/pinctrl/intel/pinctrl-tigerlake.c
+Merge made by the 'ort' strategy.
+ drivers/pinctrl/intel/pinctrl-tigerlake.c | 32 +++++++++++++++++++++++++++++++
+ drivers/pinctrl/pinctrl-upboard.c         | 21 ++++++++++++++++++++
+ 2 files changed, 53 insertions(+)
+Merging pinctrl-renesas/renesas-pinctrl (5a653cedec948 pinctrl: renesas: rza2: Embed pins in the priv struct)
+$ git merge -m Merge branch 'renesas-pinctrl' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git pinctrl-renesas/renesas-pinctrl
+Merge made by the 'ort' strategy.
+ drivers/pinctrl/renesas/pinctrl-rza2.c | 26 +++++++++++---------------
+ 1 file changed, 11 insertions(+), 15 deletions(-)
+Merging pinctrl-samsung/for-next (d441836c9d704 pinctrl: samsung: Add Exynos8855 pinctrl configuration)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git pinctrl-samsung/for-next
+Auto-merging Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml
+Merge made by the 'ort' strategy.
+ .../pinctrl/samsung,pinctrl-wakeup-interrupt.yaml  |   1 +
+ .../bindings/pinctrl/samsung,pinctrl.yaml          |   1 +
+ drivers/pinctrl/samsung/pinctrl-exynos-arm64.c     | 132 +++++++++++++++++++++
+ drivers/pinctrl/samsung/pinctrl-samsung.c          |   2 +
+ drivers/pinctrl/samsung/pinctrl-samsung.h          |   1 +
+ 5 files changed, 137 insertions(+)
+Merging pinctrl-qcom/pinctrl-qcom/for-next (82c3edf870115 pinctrl: qcom: milos-lpass-lpi: Add Eliza LPASS LPI TLMM)
+$ git merge -m Merge branch 'pinctrl-qcom/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git pinctrl-qcom/pinctrl-qcom/for-next
+Auto-merging drivers/pinctrl/qcom/pinctrl-msm.c
+Merge made by the 'ort' strategy.
+ .../bindings/pinctrl/qcom,maili-tlmm.yaml          |  120 ++
+ .../pinctrl/qcom,milos-lpass-lpi-pinctrl.yaml      |   18 +-
+ .../bindings/pinctrl/qcom,pmic-gpio.yaml           |    3 +
+ drivers/pinctrl/qcom/Kconfig.msm                   |   10 +
+ drivers/pinctrl/qcom/Makefile                      |    1 +
+ drivers/pinctrl/qcom/pinctrl-maili.c               | 1625 ++++++++++++++++++++
+ drivers/pinctrl/qcom/pinctrl-milos-lpass-lpi.c     |   40 +
+ drivers/pinctrl/qcom/pinctrl-msm.c                 |   24 +-
+ drivers/pinctrl/qcom/pinctrl-spmi-gpio.c           |    1 +
+ drivers/pinctrl/qcom/pinctrl-x1e80100.c            |    4 +-
+ drivers/pinctrl/qcom/tlmm-test.c                   |    3 -
+ 11 files changed, 1835 insertions(+), 14 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/pinctrl/qcom,maili-tlmm.yaml
+ create mode 100644 drivers/pinctrl/qcom/pinctrl-maili.c
+Merging pwm/pwm/for-next (7f1565db53367 pwm: rzg2l-gpt: Drop unused rzg2l_gpt_chip parameter from rzg2l_gpt_calculate_prescale())
+$ git merge -m Merge branch 'pwm/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ukleinek/linux.git pwm/pwm/for-next
+Auto-merging drivers/pwm/pwm-adp5585.c
+Auto-merging drivers/pwm/pwm-apple.c
+Auto-merging drivers/pwm/pwm-berlin.c
+Auto-merging drivers/pwm/pwm-lpc18xx-sct.c
+Auto-merging drivers/pwm/pwm-lpss-platform.c
+Auto-merging drivers/pwm/pwm-pxa.c
+Auto-merging drivers/pwm/pwm-sifive.c
+Auto-merging drivers/pwm/pwm-sl28cpld.c
+Auto-merging drivers/pwm/pwm-sprd.c
+Auto-merging drivers/pwm/pwm-sunplus.c
+Auto-merging drivers/pwm/pwm-vt8500.c
+Merge made by the 'ort' strategy.
+ .../bindings/pwm/nvidia,tegra20-pwm.yaml           |   1 +
+ drivers/pwm/Kconfig                                |   2 +-
+ drivers/pwm/core.c                                 |   4 +-
+ drivers/pwm/pwm-adp5585.c                          |   4 +-
+ drivers/pwm/pwm-apple.c                            |   2 +-
+ drivers/pwm/pwm-argon-fan-hat.c                    |   2 +-
+ drivers/pwm/pwm-atmel-hlcdc.c                      |  15 +-
+ drivers/pwm/pwm-atmel-tcb.c                        |   6 +-
+ drivers/pwm/pwm-atmel.c                            |   3 +-
+ drivers/pwm/pwm-bcm-iproc.c                        |   2 +-
+ drivers/pwm/pwm-bcm-kona.c                         |   2 +-
+ drivers/pwm/pwm-bcm2835.c                          |   2 +-
+ drivers/pwm/pwm-berlin.c                           |   2 +-
+ drivers/pwm/pwm-brcmstb.c                          |   2 +-
+ drivers/pwm/pwm-clk.c                              |   2 +-
+ drivers/pwm/pwm-clps711x.c                         |   2 +-
+ drivers/pwm/pwm-cros-ec.c                          |   2 +-
+ drivers/pwm/pwm-hibvt.c                            |   2 +-
+ drivers/pwm/pwm-imx-tpm.c                          |   2 +-
+ drivers/pwm/pwm-imx1.c                             |   2 +-
+ drivers/pwm/pwm-imx27.c                            |   2 +-
+ drivers/pwm/pwm-ipq.c                              |   4 +-
+ drivers/pwm/pwm-jz4740.c                           |   2 +-
+ drivers/pwm/pwm-loongson.c                         |   2 +-
+ drivers/pwm/pwm-lp3943.c                           |   2 +-
+ drivers/pwm/pwm-lpc18xx-sct.c                      |   2 +-
+ drivers/pwm/pwm-lpc32xx.c                          |   2 +-
+ drivers/pwm/pwm-lpss-pci.c                         |   2 +-
+ drivers/pwm/pwm-lpss-platform.c                    |  10 +-
+ drivers/pwm/pwm-mc33xs2410.c                       |   2 +-
+ drivers/pwm/pwm-mediatek.c                         |   2 +-
+ drivers/pwm/pwm-meson.c                            |   9 +-
+ drivers/pwm/pwm-mtk-disp.c                         |   8 +-
+ drivers/pwm/pwm-mxs.c                              |   2 +-
+ drivers/pwm/pwm-omap-dmtimer.c                     |   4 +-
+ drivers/pwm/pwm-pca9685.c                          |   6 +-
+ drivers/pwm/pwm-pxa.c                              |  30 ++--
+ drivers/pwm/pwm-raspberrypi-poe.c                  |   2 +-
+ drivers/pwm/pwm-rcar.c                             |   4 +-
+ drivers/pwm/pwm-renesas-tpu.c                      |  10 +-
+ drivers/pwm/pwm-rockchip.c                         |   8 +-
+ drivers/pwm/pwm-rzg2l-gpt.c                        |   7 +-
+ drivers/pwm/pwm-samsung.c                          |   2 +-
+ drivers/pwm/pwm-sifive.c                           |   2 +-
+ drivers/pwm/pwm-sl28cpld.c                         |   2 +-
+ drivers/pwm/pwm-sophgo-sg2042.c                    |   7 +-
+ drivers/pwm/pwm-sprd.c                             |   4 +-
+ drivers/pwm/pwm-sti.c                              |   2 +-
+ drivers/pwm/pwm-stm32-lp.c                         |   4 +-
+ drivers/pwm/pwm-stm32.c                            |   6 +-
+ drivers/pwm/pwm-sun4i.c                            |   2 +-
+ drivers/pwm/pwm-sunplus.c                          |   4 +-
+ drivers/pwm/pwm-tegra.c                            | 160 ++++++++++++++-------
+ drivers/pwm/pwm-tiecap.c                           |   6 +-
+ drivers/pwm/pwm-tiehrpwm.c                         |   2 +-
+ drivers/pwm/pwm-twl-led.c                          |   2 +-
+ drivers/pwm/pwm-twl.c                              |   2 +-
+ drivers/pwm/pwm-visconti.c                         |   2 +-
+ drivers/pwm/pwm-vt8500.c                           |   2 +-
+ drivers/pwm/pwm-xilinx.c                           |   4 +-
+ 60 files changed, 221 insertions(+), 177 deletions(-)
+Merging ktest/for-next (932cdaf3e273a ktest: Add logfile to failure directory)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-ktest.git ktest/for-next
+Already up to date.
+Merging kselftest/next (20f01a5565ce7 selftests/ftrace: fix spelling error in poll test comment)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kselftest/next
+Merge made by the 'ort' strategy.
+ tools/testing/selftests/ftrace/poll.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging kunit/test (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'test' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit/test
+Already up to date.
+Merging kunit-next/kunit (34b5c0132952c kunit: configs: enable GPIO kunit test cases in all_tests.config)
+$ git merge -m Merge branch 'kunit' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit-next/kunit
+Merge made by the 'ort' strategy.
+ Documentation/dev-tools/kunit/faq.rst        | 43 ++++++++++++++++------------
+ Documentation/dev-tools/kunit/style.rst      |  7 +++--
+ include/kunit/test.h                         |  1 +
+ lib/kunit/debugfs.c                          | 28 ++++++++++++------
+ lib/kunit/kunit-example-test.c               | 29 +++++++++++++++++++
+ lib/kunit/string-stream.c                    |  9 ++++--
+ lib/kunit/test.c                             | 17 ++++++++++-
+ rust/kernel/kunit.rs                         |  1 +
+ tools/testing/kunit/configs/all_tests.config |  1 +
+ 9 files changed, 103 insertions(+), 33 deletions(-)
+Merging livepatching/for-next (0839c8963b7b2 Merge tag 'livepatching-for-7.2' of git://git.kernel.org/pub/scm/linux/kernel/git/livepatching/livepatching)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/livepatching/livepatching.git livepatching/for-next
+Already up to date.
+Merging rtc/rtc-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'rtc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git rtc/rtc-next
+Already up to date.
+Merging nvdimm/libnvdimm-for-next (6191eeb6c70b4 dax: fsdev.c minor formatting cleanup)
+$ git merge -m Merge branch 'libnvdimm-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git nvdimm/libnvdimm-for-next
+Auto-merging fs/dax.c
+Merge made by the 'ort' strategy.
+ drivers/dax/dax-private.h          |   2 -
+ drivers/dax/fsdev.c                | 154 ++++++++++++++++++++++++++-----------
+ drivers/dax/super.c                |  54 +++++++++++--
+ drivers/nvdimm/btt.c               |   8 ++
+ drivers/nvdimm/label.c             |  17 +++-
+ fs/dax.c                           |  12 +--
+ tools/testing/nvdimm/test/ndtest.c |   3 +-
+ tools/testing/nvdimm/test/nfit.c   |   3 +-
+ 8 files changed, 186 insertions(+), 67 deletions(-)
+Merging at24/at24/for-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'at24/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git at24/at24/for-next
+Already up to date.
+Merging ntb/ntb-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'ntb-next' of https://github.com/jonmason/ntb.git ntb/ntb-next
+Already up to date.
+Merging seccomp/for-next/seccomp (41fa043273841 selftests/seccomp: Add hard-coded __NR_uprobe for x86_64)
+$ git merge -m Merge branch 'for-next/seccomp' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git seccomp/for-next/seccomp
+Already up to date.
+Merging slimbus/for-next (c922423ce66bc slimbus: qcom-ngd-ctrl: Use the unified QMI service ID instead of defining it locally)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/srini/slimbus.git slimbus/for-next
+Auto-merging drivers/slimbus/qcom-ngd-ctrl.c
+Merge made by the 'ort' strategy.
+ drivers/slimbus/qcom-ngd-ctrl.c | 5 ++---
+ 1 file changed, 2 insertions(+), 3 deletions(-)
+Merging nvmem/for-next (2a3ed29b05313 Merge branch 'nvmem-for-7.3' into nvmem-for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/srini/nvmem.git nvmem/for-next
+Auto-merging MAINTAINERS
+CONFLICT (content): Merge conflict in MAINTAINERS
+Auto-merging drivers/nvmem/brcm_nvram.c
+Auto-merging drivers/nvmem/nintendo-otp.c
+Resolved 'MAINTAINERS' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 911b7d5309145] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/srini/nvmem.git
+$ git diff -M --stat --summary HEAD^..
+ .../devicetree/bindings/mfd/mediatek,mt6397.yaml   |  21 ++
+ .../bindings/nvmem/airoha,smc-efuses.yaml          |  67 ++++
+ .../bindings/nvmem/microchip,lan9662-otpc.yaml     |   1 +
+ .../devicetree/bindings/nvmem/qcom,qfprom.yaml     |   6 +
+ MAINTAINERS                                        |  14 +-
+ drivers/misc/eeprom/Kconfig                        |  93 ------
+ drivers/misc/eeprom/Makefile                       |   5 -
+ drivers/nvmem/Kconfig                              | 121 +++++++-
+ drivers/nvmem/Makefile                             |   9 +
+ drivers/nvmem/airoha-smc-efuses.c                  | 125 ++++++++
+ drivers/nvmem/apple-spmi-nvmem.c                   |  20 +-
+ drivers/{misc/eeprom => nvmem}/at24.c              |   0
+ drivers/{misc/eeprom => nvmem}/at25.c              |   0
+ drivers/nvmem/brcm_nvram.c                         |  15 +-
+ drivers/nvmem/core.c                               | 341 +++++++++++----------
+ drivers/{misc/eeprom => nvmem}/ee1004.c            |   0
+ drivers/{misc/eeprom => nvmem}/eeprom_93xx46.c     |   0
+ drivers/nvmem/internals.h                          |  15 +-
+ drivers/nvmem/lan9662-otpc.c                       |  12 +-
+ drivers/nvmem/layouts.c                            |  11 -
+ drivers/nvmem/layouts/Kconfig                      |   9 +
+ drivers/nvmem/layouts/Makefile                     |   1 +
+ drivers/nvmem/layouts/fixed-layout.c               |  58 ++++
+ drivers/nvmem/layouts/u-boot-env.c                 |   5 +-
+ drivers/{misc/eeprom => nvmem}/m24lr.c             |   0
+ drivers/nvmem/mt6323-efuse.c                       |  85 +++++
+ drivers/nvmem/nintendo-otp.c                       |   9 +-
+ drivers/nvmem/rockchip-otp.c                       |   9 +-
+ include/linux/nvmem-provider.h                     |   6 +
+ 29 files changed, 746 insertions(+), 312 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/nvmem/airoha,smc-efuses.yaml
+ create mode 100644 drivers/nvmem/airoha-smc-efuses.c
+ rename drivers/{misc/eeprom => nvmem}/at24.c (100%)
+ rename drivers/{misc/eeprom => nvmem}/at25.c (100%)
+ rename drivers/{misc/eeprom => nvmem}/ee1004.c (100%)
+ rename drivers/{misc/eeprom => nvmem}/eeprom_93xx46.c (100%)
+ create mode 100644 drivers/nvmem/layouts/fixed-layout.c
+ rename drivers/{misc/eeprom => nvmem}/m24lr.c (100%)
+ create mode 100644 drivers/nvmem/mt6323-efuse.c
+Merging hyperv/hyperv-next (a4ffc59238be8 mshv: add bounds check on vp_index in mshv_intercept_isr())
+$ git merge -m Merge branch 'hyperv-next' of https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git hyperv/hyperv-next
+Already up to date.
+Merging auxdisplay/for-next (95545ddcb3cfb auxdisplay: panel: Remove unused callback binding code)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git auxdisplay/for-next
+Merge made by the 'ort' strategy.
+ drivers/auxdisplay/ht16k33.c |  5 +----
+ drivers/auxdisplay/panel.c   | 37 -------------------------------------
+ 2 files changed, 1 insertion(+), 41 deletions(-)
+Merging kgdb/kgdb/for-next (fdbdd0ccb30af kdb: remove redundant check for scancode 0xe0)
+$ git merge -m Merge branch 'kgdb/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/danielt/linux.git kgdb/kgdb/for-next
+Already up to date.
+Merging hmm/hmm (19272b37aa4f8 Linux 6.16-rc1)
+$ git merge -m Merge branch 'hmm' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git hmm/hmm
+Already up to date.
+Merging cfi/cfi/next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'cfi/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git cfi/cfi/next
+Already up to date.
+Merging mhi/mhi-next (34c66fc4c223a bus: mhi: host: Flush the posted write after writing to MHI_SOC_RESET_REQ_OFFSET)
+$ git merge -m Merge branch 'mhi-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mani/mhi.git mhi/mhi-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/bus/mhi/ep/main.c
+Merge made by the 'ort' strategy.
+ MAINTAINERS                 |  1 +
+ drivers/bus/mhi/ep/main.c   | 15 ++++++++++-----
+ drivers/bus/mhi/host/main.c |  6 ++++++
+ drivers/bus/mhi/host/pm.c   | 14 +++-----------
+ 4 files changed, 20 insertions(+), 16 deletions(-)
+$ git am -3 ../patches/0001-fix-up-for-net-qrtr-Drop-the-MHI-auto_queue-feature-.patch
+Applying: fix up for "net: qrtr: Drop the MHI auto_queue feature for IPCR DL channels"
+Using index info to reconstruct a base tree...
+M	drivers/net/wireless/ath/ath12k/wifi7/mhi.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+Merging memblock/for-next (1a6e6a79b8df1 Merge branch 'misc' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git memblock/for-next
+Auto-merging arch/riscv/mm/init.c
+Auto-merging drivers/base/arch_numa.c
+Auto-merging mm/hugetlb.c
+Auto-merging mm/memblock.c
+Auto-merging mm/mm_init.c
+Merge made by the 'ort' strategy.
+ arch/arm/mm/kasan_init.c               |  6 ----
+ arch/arm64/mm/kasan_init.c             |  3 --
+ arch/loongarch/kernel/numa.c           |  1 -
+ arch/loongarch/mm/kasan_init.c         |  3 --
+ arch/powerpc/mm/kasan/init_book3e_64.c |  3 --
+ arch/powerpc/mm/kasan/init_book3s_64.c |  3 --
+ arch/riscv/mm/init.c                   |  2 --
+ arch/riscv/mm/kasan_init.c             |  3 --
+ arch/x86/mm/amdtopology.c              |  1 -
+ arch/x86/mm/numa.c                     |  1 -
+ drivers/base/arch_numa.c               |  4 ---
+ drivers/of/of_numa.c                   |  5 +--
+ mm/hugetlb.c                           |  7 ++--
+ mm/memblock.c                          |  4 +--
+ mm/mm_init.c                           | 63 +++-------------------------------
+ mm/numa_memblks.c                      | 41 ++++++++++------------
+ 16 files changed, 27 insertions(+), 123 deletions(-)
+Merging cxl/next (5ca04f3ba91f1 Bounds-check the fwctl feature commands)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git cxl/next
+Merge made by the 'ort' strategy.
+ Documentation/driver-api/cxl/linux/dax-driver.rst |   4 +-
+ drivers/cxl/core/features.c                       |  23 +-
+ drivers/cxl/core/mbox.c                           |  20 +-
+ drivers/cxl/core/mce.c                            |  27 +-
+ drivers/cxl/core/memdev.c                         |   2 +-
+ drivers/cxl/core/port.c                           |   4 +-
+ drivers/cxl/core/region.c                         |  43 +-
+ drivers/cxl/cxl.h                                 |   8 +-
+ drivers/cxl/cxlmem.h                              |   4 +-
+ drivers/cxl/pci.c                                 |  13 +-
+ include/cxl/mailbox.h                             |   2 +
+ tools/testing/cxl/test/Kbuild                     |   2 +
+ tools/testing/cxl/test/accel.c                    |  66 ++
+ tools/testing/cxl/test/cxl.c                      | 833 ++++++++++++++++------
+ tools/testing/cxl/test/hmem_test.c                |   3 +-
+ tools/testing/cxl/test/mock.h                     |   2 +
+ 16 files changed, 775 insertions(+), 281 deletions(-)
+ create mode 100644 tools/testing/cxl/test/accel.c
+Merging zstd/zstd-next (65d1f5507ed2c zstd: Import upstream v1.5.7)
+$ git merge -m Merge branch 'zstd-next' of https://github.com/terrelln/linux.git zstd/zstd-next
+Already up to date.
+Merging efi/next (4953e0a8a149d efi/runtime-wrappers: mark efi_runtime_lock_owner as __used)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git efi/next
+Merge made by the 'ort' strategy.
+ arch/x86/platform/efi/quirks.c          |  9 +----
+ drivers/firmware/efi/runtime-wrappers.c | 70 ++++++++++++++++++++++++++++-----
+ include/linux/efi.h                     |  6 ++-
+ tools/objtool/noreturns.h               |  1 +
+ 4 files changed, 66 insertions(+), 20 deletions(-)
+Merging unicode/for-next (bcfee135d5847 utf8: Remove unused utf8_normalize)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krisman/unicode.git unicode/for-next
+Auto-merging fs/unicode/utf8-core.c
+Merge made by the 'ort' strategy.
+ fs/unicode/utf8-core.c  | 22 ----------------------
+ include/linux/unicode.h |  3 ---
+ 2 files changed, 25 deletions(-)
+Merging slab/slab/for-next (56aa9b819edce mm/slub: add comment explaining intentional kobject handling in sysfs_slab_add)
+$ git merge -m Merge branch 'slab/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git slab/slab/for-next
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging mm/slub.c
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-kernel-slab     |  18 ++-
+ Documentation/admin-guide/kernel-parameters.txt |   5 +
+ mm/mempool.c                                    |  35 +++--
+ mm/slub.c                                       | 179 ++++++++++++------------
+ 4 files changed, 127 insertions(+), 110 deletions(-)
+$ git am -3 ../patches/0001-drm-panthor-I-love-Werror.patch
+Applying: drm/panthor: I love -Werror
+Using index info to reconstruct a base tree...
+M	drivers/gpu/drm/panthor/panthor_mmu.c
+Falling back to patching base and 3-way merge...
+Auto-merging drivers/gpu/drm/panthor/panthor_mmu.c
+No changes -- Patch already applied.
+Merging random/master (bb9ff576fdff4 virt: vmgenid: remap memory as decrypted)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/crng/random.git random/master
+Merge made by the 'ort' strategy.
+ drivers/virt/vmgenid.c | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+Merging landlock/next (55f82176ef8dd selftests/landlock: Fix spelling error in fs_test comment)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/mic/linux.git landlock/next
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/LSM/landlock.rst |  1 +
+ include/uapi/linux/landlock.h              |  8 ++++----
+ security/landlock/task.c                   | 11 +++++++++++
+ tools/testing/selftests/landlock/fs_test.c |  2 +-
+ 4 files changed, 17 insertions(+), 5 deletions(-)
+Merging sysctl/sysctl-next (dccde188b7fc1 sysctl: add Returns: kernel-doc for all functions)
+$ git merge -m Merge branch 'sysctl-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sysctl/sysctl.git sysctl/sysctl-next
+Merge made by the 'ort' strategy.
+ fs/pipe.c              |   2 +-
+ include/linux/sysctl.h | 125 ++++++++----
+ kernel/sysctl.c        | 530 ++++++++++++++++++++++++++-----------------------
+ kernel/time/jiffies.c  |  28 ++-
+ 4 files changed, 395 insertions(+), 290 deletions(-)
+Merging execve/for-next/execve (9bf092c97b86a sched: update task_struct->comm comment)
+$ git merge -m Merge branch 'for-next/execve' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git execve/for-next/execve
+Already up to date.
+Merging bitmap/bitmap-for-next (fc6926b0ead4b nodemask: reduce bitmap width to nr_node_ids in __nodemask_pr_numnodes())
+$ git merge -m Merge branch 'bitmap-for-next' of https://github.com/norov/linux.git bitmap/bitmap-for-next
+Auto-merging arch/x86/events/amd/uncore.c
+Auto-merging arch/x86/events/intel/core.c
+Auto-merging arch/x86/events/intel/uncore.c
+Auto-merging drivers/devfreq/event/rockchip-dfi.c
+Auto-merging drivers/fpga/dfl-fme-perf.c
+Auto-merging drivers/pci/pci-sysfs.c
+Auto-merging kernel/events/core.c
+Merge made by the 'ort' strategy.
+ arch/arm/mach-imx/mmdc.c                    |  2 +-
+ arch/arm/mm/cache-l2x0-pmu.c                |  2 +-
+ arch/powerpc/kernel/cacheinfo.c             |  3 ++-
+ arch/powerpc/perf/hv-24x7.c                 |  2 +-
+ arch/powerpc/perf/hv-gpci.c                 |  2 +-
+ arch/powerpc/perf/imc-pmu.c                 |  2 +-
+ arch/x86/events/amd/iommu.c                 |  2 +-
+ arch/x86/events/amd/power.c                 |  2 +-
+ arch/x86/events/amd/uncore.c                |  2 +-
+ arch/x86/events/intel/core.c                |  2 +-
+ arch/x86/events/intel/uncore.c              |  2 +-
+ drivers/base/cpu.c                          |  2 +-
+ drivers/devfreq/event/rockchip-dfi.c        |  2 +-
+ drivers/devfreq/hisi_uncore_freq.c          |  2 +-
+ drivers/firmware/psci/psci_checker.c        | 15 ++---------
+ drivers/fpga/dfl-fme-perf.c                 |  2 +-
+ drivers/hwtracing/ptt/hisi_ptt.c            |  2 +-
+ drivers/infiniband/hw/hfi1/sdma.c           |  3 ++-
+ drivers/pci/pci-sysfs.c                     |  7 +++---
+ drivers/perf/alibaba_uncore_drw_pmu.c       |  2 +-
+ drivers/perf/amlogic/meson_ddr_pmu_core.c   |  2 +-
+ drivers/perf/arm-cci.c                      |  2 +-
+ drivers/perf/arm-ccn.c                      |  2 +-
+ drivers/perf/arm-cmn.c                      |  2 +-
+ drivers/perf/arm-ni.c                       |  2 +-
+ drivers/perf/arm_cspmu/arm_cspmu.c          |  2 +-
+ drivers/perf/arm_dmc620_pmu.c               |  4 +--
+ drivers/perf/arm_dsu_pmu.c                  |  2 +-
+ drivers/perf/arm_pmu.c                      |  2 +-
+ drivers/perf/arm_smmuv3_pmu.c               |  2 +-
+ drivers/perf/arm_spe_pmu.c                  |  2 +-
+ drivers/perf/cxl_pmu.c                      |  2 +-
+ drivers/perf/dwc_pcie_pmu.c                 |  2 +-
+ drivers/perf/fsl_imx8_ddr_perf.c            |  2 +-
+ drivers/perf/fsl_imx9_ddr_perf.c            |  2 +-
+ drivers/perf/fujitsu_uncore_pmu.c           |  2 +-
+ drivers/perf/hisilicon/hisi_pcie_pmu.c      |  2 +-
+ drivers/perf/hisilicon/hisi_uncore_pmu.c    |  2 +-
+ drivers/perf/marvell_cn10k_ddr_pmu.c        |  2 +-
+ drivers/perf/marvell_cn10k_tad_pmu.c        |  2 +-
+ drivers/perf/marvell_pem_pmu.c              |  2 +-
+ drivers/perf/nvidia_t410_c2c_pmu.c          |  2 +-
+ drivers/perf/nvidia_t410_cmem_latency_pmu.c |  2 +-
+ drivers/perf/qcom_l2_pmu.c                  |  2 +-
+ drivers/perf/qcom_l3_pmu.c                  |  2 +-
+ drivers/perf/starfive_starlink_pmu.c        |  2 +-
+ drivers/perf/thunderx2_pmu.c                |  2 +-
+ drivers/perf/xgene_pmu.c                    |  2 +-
+ include/asm-generic/bitops/le.h             | 18 ++++++-------
+ include/linux/bitmap.h                      |  4 +--
+ include/linux/bits.h                        |  6 ++---
+ include/linux/cpumask.h                     | 19 --------------
+ include/linux/nodemask.h                    | 15 ++++++-----
+ kernel/events/core.c                        |  2 +-
+ lib/bitmap-str.c                            |  9 +++----
+ lib/bitmap.c                                | 32 +++++++++++++----------
+ lib/find_bit_benchmark.c                    | 17 +++++++++++++
+ lib/test_bitmap.c                           | 39 +++++++++++++++++++++++++++++
+ tools/include/linux/bits.h                  |  6 ++---
+ 59 files changed, 161 insertions(+), 124 deletions(-)
+Merging hte/for-next (005b25ad11717 hte: tegra194: Add Tegra264 GTE support)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pateldipen1984/linux.git hte/for-next
+Already up to date.
+Merging kspp/for-next/kspp (3421b9b056a65 selftests/seccomp: Fix pointer type mismatch build error)
+$ git merge -m Merge branch 'for-next/kspp' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git kspp/for-next/kspp
+Merge made by the 'ort' strategy.
+ lib/test_fortify/Makefile                     | 1 +
+ tools/testing/selftests/lkdtm/tests.txt       | 2 +-
+ tools/testing/selftests/seccomp/seccomp_bpf.c | 3 ++-
+ 3 files changed, 4 insertions(+), 2 deletions(-)
+Merging nolibc/for-next (070cf03f5ec45 selftests/nolibc: add debug information)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/nolibc/linux-nolibc.git nolibc/for-next
+Merge made by the 'ort' strategy.
+ tools/include/nolibc/arch-sparc.h               |  2 +-
+ tools/include/nolibc/unistd.h                   | 58 +++++++++++++++++++++++++
+ tools/testing/selftests/nolibc/Makefile.include |  2 +-
+ tools/testing/selftests/nolibc/nolibc-test.c    | 53 ++++++++++++++++++++++
+ 4 files changed, 113 insertions(+), 2 deletions(-)
+Merging iommufd/for-next (b737ad0b96a6e Documentation: Update VFIO NOIOMMU mode)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git iommufd/for-next
+Auto-merging drivers/iommu/iommufd/device.c
+Auto-merging drivers/iommu/iommufd/viommu.c
+Merge made by the 'ort' strategy.
+ Documentation/driver-api/vfio.rst       |  89 ++++++++++++-
+ drivers/iommu/iommu.c                   |   2 +-
+ drivers/iommu/iommufd/Kconfig           |  12 ++
+ drivers/iommu/iommufd/Makefile          |   1 +
+ drivers/iommu/iommufd/device.c          | 228 ++++++++++++++++++++++----------
+ drivers/iommu/iommufd/hw_pagetable.c    |  19 ++-
+ drivers/iommu/iommufd/hwpt_noiommu.c    | 105 +++++++++++++++
+ drivers/iommu/iommufd/io_pagetable.c    |  80 ++++++++++-
+ drivers/iommu/iommufd/ioas.c            |  36 +++++
+ drivers/iommu/iommufd/iommufd_private.h |  39 ++++--
+ drivers/iommu/iommufd/main.c            |  24 ++++
+ drivers/iommu/iommufd/selftest.c        |  12 +-
+ drivers/iommu/iommufd/viommu.c          |  14 +-
+ drivers/vfio/Kconfig                    |   7 +-
+ drivers/vfio/device_cdev.c              |   9 ++
+ drivers/vfio/iommufd.c                  |  12 +-
+ drivers/vfio/vfio.h                     |  23 ++--
+ drivers/vfio/vfio_main.c                |  26 +++-
+ include/linux/vfio.h                    |   1 +
+ include/uapi/linux/iommufd.h            |  28 ++++
+ 20 files changed, 652 insertions(+), 115 deletions(-)
+ create mode 100644 drivers/iommu/iommufd/hwpt_noiommu.c
+Merging turbostat/next (1c996a37fd244 tools/power turbostat: pmt: Improve sscanf() hygiene)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux.git turbostat/next
+Merge made by the 'ort' strategy.
+ tools/power/x86/turbostat/turbostat.8 |   6 +-
+ tools/power/x86/turbostat/turbostat.c | 567 ++++++++++++++++------------------
+ 2 files changed, 266 insertions(+), 307 deletions(-)
+Merging pwrseq/pwrseq/for-next (162ea02941a93 power: sequencing: pwrseq-pcie-m2: Add QCC2072 BT PCI device ID)
+$ git merge -m Merge branch 'pwrseq/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git pwrseq/pwrseq/for-next
+Auto-merging drivers/power/sequencing/pwrseq-pcie-m2.c
+Merge made by the 'ort' strategy.
+ drivers/power/sequencing/pwrseq-pcie-m2.c | 8 ++++++++
+ 1 file changed, 8 insertions(+)
+Merging capabilities-next/caps-next (0715881360074 ipc: don't audit capability check in ipc_permissions())
+$ git merge -m Merge branch 'caps-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sergeh/linux.git capabilities-next/caps-next
+Already up to date.
+$ git am -3 ../patches/0001-sign-file-Fix-up-merge-issue.patch
+Applying: sign-file: Fix up merge issue
+Using index info to reconstruct a base tree...
+M	scripts/sign-file.c
+Falling back to patching base and 3-way merge...
+Auto-merging scripts/sign-file.c
+No changes -- Patch already applied.
+Merging ipe/next (028ef9c96e961 Linux 7.0)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/wufan/ipe.git ipe/next
+Already up to date.
+Merging kcsan/next (07a1a6562ce29 kcsan: Silence -Wmaybe-uninitialized when calling __kcsan_check_access())
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/melver/linux.git kcsan/next
+Already up to date.
+Merging crc/crc-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'crc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git crc/crc-next
+Already up to date.
+Merging keys-next/keys-next (965e9a2cf23b0 pkcs7: Change a pr_warn() to pr_warn_once())
+$ git merge -m Merge branch 'keys-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dhowells/linux-fs.git keys-next/keys-next
+Already up to date.
+Merging fwctl/for-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git fwctl/for-next
+Already up to date.
+Merging devsec-tsm/next (3177779ae17db virt: coco: change tsm_class to a const struct)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git devsec-tsm/next
+Already up to date.
+Merging hisilicon/for-next (a5db65458a911 Merge branch 'next/dt64' into for-next)
+$ git merge -m Merge branch 'for-next' of https://github.com/hisilicon/linux-hisi.git hisilicon/for-next
+Merge made by the 'ort' strategy.
+ arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi |  4 ----
+ arch/arm64/boot/dts/hisilicon/hip05.dtsi       | 14 +++++++-------
+ 2 files changed, 7 insertions(+), 11 deletions(-)
+Merging device-id/device-id-rework (995832b2cebe6 Replace <linux/mod_devicetable.h> by more specific <linux/device-id/*.h> (c files))
+$ git merge -m Merge branch 'device-id-rework' of https://git.kernel.org/pub/scm/linux/kernel/git/ukleinek/linux.git device-id/device-id-rework
+Already up to date.
diff --git a/README b/README
index e6549b2a7aac3..010c587224224 100644
--- a/README
+++ b/README
@@ -30,15 +30,15 @@ Who Are You?
 
 Find your role below:
 
-* New Kernel Developer - Getting started with kernel development
-* Academic Researcher - Studying kernel internals and architecture
-* Security Expert - Hardening and vulnerability analysis
-* Backport/Maintenance Engineer - Maintaining stable kernels
-* System Administrator - Configuring and troubleshooting
-* Maintainer - Leading subsystems and reviewing patches
-* Hardware Vendor - Writing drivers for new hardware
-* Distribution Maintainer - Packaging kernels for distros
-* AI Coding Assistant - LLMs and AI-powered development tools
+* New Kernel Developer: Getting started with kernel development
+* Academic Researcher: Studying kernel internals and architecture
+* Security Expert: Hardening and vulnerability analysis
+* Backport/Maintenance Engineer: Maintaining stable kernels
+* System Administrator: Configuring and troubleshooting
+* Maintainer: Leading subsystems and reviewing patches
+* Hardware Vendor: Writing drivers for new hardware
+* Distribution Maintainer: Packaging kernels for distros
+* AI Coding Assistant: LLMs and AI-powered development tools
 
 
 For Specific Users
diff --git a/arch/Kconfig b/arch/Kconfig
index fa7507ac8e13e..89ee0d1a37759 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -114,6 +114,20 @@ config GENERIC_ENTRY
 	select GENERIC_IRQ_ENTRY
 	select GENERIC_SYSCALL
 
+config ARCH_SUPPORTS_SYSCALL_USER_DISPATCH
+	bool
+
+config SYSCALL_USER_DISPATCH
+	bool "Syscall User Dispatch"
+	depends on ARCH_SUPPORTS_SYSCALL_USER_DISPATCH
+	depends on GENERIC_ENTRY
+	default y
+	help
+	  Syscall User Dispatch lets a thread have its own system calls
+	  intercepted and redirected to a userspace signal handler based
+	  on a prctl() configured instruction pointer range.
+	  If unsure, say Y.
+
 config KPROBES
 	bool "Kprobes"
 	depends on HAVE_KPROBES
@@ -637,7 +651,7 @@ config HAVE_ARCH_SECCOMP_FILTER
 	  - syscall_set_return_value()
 	  - SIGSYS siginfo_t support
 	  - secure_computing is called from a ptrace_event()-safe context
-	  - secure_computing return value is checked and a return value of -1
+	  - secure_computing return value is checked and if false it
 	    results in the system call being skipped immediately.
 	  - seccomp syscall wired up
 	  - if !HAVE_SPARSE_SYSCALL_NR, have SECCOMP_ARCH_NATIVE,
@@ -1682,7 +1696,6 @@ config HAVE_SPARSE_SYSCALL_NR
 	  related optimizations for a given architecture.
 
 config ARCH_HAS_VDSO_ARCH_DATA
-	depends on HAVE_GENERIC_VDSO
 	bool
 
 config ARCH_HAS_VDSO_TIME_DATA
diff --git a/arch/alpha/kernel/pci-sysfs.c b/arch/alpha/kernel/pci-sysfs.c
index 94dbc470cd6c0..7050f0f7fe3d7 100644
--- a/arch/alpha/kernel/pci-sysfs.c
+++ b/arch/alpha/kernel/pci-sysfs.c
@@ -224,17 +224,17 @@ int pci_legacy_write(struct pci_bus *bus, loff_t port, u32 val, size_t size)
 
 	switch(size) {
 	case 1:
-		outb(port, val);
+		outb(val, port);
 		return 1;
 	case 2:
 		if (port & 1)
 			return -EINVAL;
-		outw(port, val);
+		outw(val, port);
 		return 2;
 	case 4:
 		if (port & 3)
 			return -EINVAL;
-		outl(port, val);
+		outl(val, port);
 		return 4;
 	}
 	return -EINVAL;
diff --git a/arch/alpha/kernel/ptrace.c b/arch/alpha/kernel/ptrace.c
index 0687760ea4669..768b0920a9de4 100644
--- a/arch/alpha/kernel/ptrace.c
+++ b/arch/alpha/kernel/ptrace.c
@@ -375,7 +375,7 @@ asmlinkage unsigned long syscall_trace_enter(void)
 	struct pt_regs *regs = current_pt_regs();
 
 	if (test_thread_flag(TIF_SYSCALL_TRACE) &&
-		ptrace_report_syscall_entry(regs)) {
+		!ptrace_report_syscall_permit_entry(regs)) {
 		syscall_set_nr(current, regs, -1);
 		if (regs->r19 == 0 && regs->r0 == (unsigned long)-1)
 			syscall_set_return_value(current, regs, -ENOSYS, 0);
@@ -387,7 +387,7 @@ asmlinkage unsigned long syscall_trace_enter(void)
 	 * If this fails, seccomp may already have set up the return value
 	 * (e.g. SECCOMP_RET_ERRNO / TRACE).
 	 */
-	if (secure_computing() == -1) {
+	if (!seccomp_permit_syscall()) {
 		if (regs->r19 == 0 && regs->r0 == (unsigned long)-1)
 			syscall_set_return_value(current, regs, -ENOSYS, 0);
 		syscall_set_nr(current, regs, -1);
diff --git a/arch/arc/configs/axs101_defconfig b/arch/arc/configs/axs101_defconfig
index f930396d9daef..870e5291b7dbc 100644
--- a/arch/arc/configs/axs101_defconfig
+++ b/arch/arc/configs/axs101_defconfig
@@ -67,7 +67,6 @@ CONFIG_SERIAL_OF_PLATFORM=y
 CONFIG_I2C=y
 CONFIG_I2C_CHARDEV=y
 CONFIG_I2C_DESIGNWARE_CORE=y
-CONFIG_I2C_DESIGNWARE_PLATFORM=y
 # CONFIG_HWMON is not set
 CONFIG_DRM=m
 CONFIG_DRM_I2C_ADV7511=m
diff --git a/arch/arc/configs/axs103_defconfig b/arch/arc/configs/axs103_defconfig
index 6b779dee5ea04..d45e4d3359986 100644
--- a/arch/arc/configs/axs103_defconfig
+++ b/arch/arc/configs/axs103_defconfig
@@ -67,7 +67,6 @@ CONFIG_SERIAL_OF_PLATFORM=y
 CONFIG_I2C=y
 CONFIG_I2C_CHARDEV=y
 CONFIG_I2C_DESIGNWARE_CORE=y
-CONFIG_I2C_DESIGNWARE_PLATFORM=y
 # CONFIG_HWMON is not set
 CONFIG_FB=y
 CONFIG_FRAMEBUFFER_CONSOLE=y
diff --git a/arch/arc/configs/axs103_smp_defconfig b/arch/arc/configs/axs103_smp_defconfig
index a89b50d5369d3..f986c0205f132 100644
--- a/arch/arc/configs/axs103_smp_defconfig
+++ b/arch/arc/configs/axs103_smp_defconfig
@@ -67,7 +67,6 @@ CONFIG_SERIAL_OF_PLATFORM=y
 CONFIG_I2C=y
 CONFIG_I2C_CHARDEV=y
 CONFIG_I2C_DESIGNWARE_CORE=y
-CONFIG_I2C_DESIGNWARE_PLATFORM=y
 # CONFIG_HWMON is not set
 CONFIG_DRM=m
 CONFIG_DRM_I2C_ADV7511=m
diff --git a/arch/arc/configs/tb10x_defconfig b/arch/arc/configs/tb10x_defconfig
index 865fbc19ef031..6e396a9ddb8b2 100644
--- a/arch/arc/configs/tb10x_defconfig
+++ b/arch/arc/configs/tb10x_defconfig
@@ -61,7 +61,6 @@ CONFIG_SERIAL_8250_DW=y
 CONFIG_I2C=y
 # CONFIG_I2C_COMPAT is not set
 CONFIG_I2C_DESIGNWARE_CORE=y
-CONFIG_I2C_DESIGNWARE_PLATFORM=y
 CONFIG_GPIO_SYSFS=y
 # CONFIG_HWMON is not set
 # CONFIG_USB_SUPPORT is not set
diff --git a/arch/arc/kernel/ptrace.c b/arch/arc/kernel/ptrace.c
index cad5367b7c372..fe2b6572e9199 100644
--- a/arch/arc/kernel/ptrace.c
+++ b/arch/arc/kernel/ptrace.c
@@ -342,7 +342,7 @@ long arch_ptrace(struct task_struct *child, long request,
 asmlinkage int syscall_trace_enter(struct pt_regs *regs)
 {
 	if (test_thread_flag(TIF_SYSCALL_TRACE))
-		if (ptrace_report_syscall_entry(regs))
+		if (!ptrace_report_syscall_permit_entry(regs))
 			return ULONG_MAX;
 
 #ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
diff --git a/arch/arc/kernel/smp.c b/arch/arc/kernel/smp.c
index b2f2c59279a67..2d99dffed0ce5 100644
--- a/arch/arc/kernel/smp.c
+++ b/arch/arc/kernel/smp.c
@@ -22,6 +22,7 @@
 #include <linux/irqdomain.h>
 #include <linux/export.h>
 #include <linux/of_fdt.h>
+#include <linux/string.h>
 
 #include <asm/mach_desc.h>
 #include <asm/setup.h>
@@ -43,9 +44,10 @@ static int __init arc_get_cpu_map(const char *name, struct cpumask *cpumask)
 {
 	unsigned long dt_root = of_get_flat_dt_root();
 	const char *buf;
+	int len;
 
-	buf = of_get_flat_dt_prop(dt_root, name, NULL);
-	if (!buf)
+	buf = of_get_flat_dt_prop(dt_root, name, &len);
+	if (!buf || !memchr(buf, '\0', len))
 		return -EINVAL;
 
 	if (cpulist_parse(buf, cpumask))
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 9187240a02db5..5adccaed7d01c 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -471,7 +471,7 @@ source "arch/arm/mach-zynq/Kconfig"
 
 # ARMv7-M architecture
 config ARCH_LPC18XX
-	bool "NXP LPC18xx/LPC43xx"
+	bool "NXP LPC18xx/LPC43xx (DEPRECATED)"
 	depends on ARM_SINGLE_ARMV7M
 	select ARCH_HAS_RESET_CONTROLLER
 	select ARM_AMBA
@@ -481,8 +481,10 @@ config ARCH_LPC18XX
 	  Support for NXP's LPC18xx Cortex-M3 and LPC43xx Cortex-M4
 	  high performance microcontrollers.
 
+	  This platform is scheduled for removal in early 2027
+
 config ARCH_MPS2
-	bool "ARM MPS2 platform"
+	bool "ARM MPS2 platform (DEPRECATED)"
 	depends on ARM_SINGLE_ARMV7M
 	select ARM_AMBA
 	select CLKSRC_MPS2
@@ -493,6 +495,8 @@ config ARCH_MPS2
 	  Please, note that depends which Application Note is used memory map
 	  for the platform may vary, so adjustment of RAM base might be needed.
 
+	  This platform is scheduled for removal in early 2027
+
 # Definitions to make life easier
 config ARCH_ACORN
 	bool
@@ -513,13 +517,21 @@ config PLAT_VERSATILE
 source "arch/arm/mm/Kconfig"
 
 config IWMMXT
-	bool "Enable iWMMXt support"
-	depends on CPU_XSCALE || CPU_XSC3 || CPU_MOHAWK
-	default y if PXA27x || PXA3xx || ARCH_MMP
+	bool "Enable iWMMXt support (DEPRECATED)"
+	depends on PXA27x || PXA3xx || ARCH_MMP
 	help
 	  Enable support for iWMMXt context switching at run time if
 	  running on a CPU that supports it.
 
+	  Machines that actually support this feature are very rare,
+	  and support is deprecated in new gcc-14.  While there were a
+	  few applications that used this in the past, none are known
+	  to still do so.
+
+	  If you use this, please send a patch to remove the
+	  deprecation, otherwise this will be removed in early
+	  2027.
+
 if !MMU
 source "arch/arm/Kconfig-nommu"
 endif
@@ -1151,23 +1163,23 @@ config ARM_PATCH_IDIV
 	  code to do integer division.
 
 config AEABI
-	bool "Use the ARM EABI to compile the kernel" if !CPU_V7 && \
-		!CPU_V7M && !CPU_V6 && !CPU_V6K && !CC_IS_CLANG
-	default CPU_V7 || CPU_V7M || CPU_V6 || CPU_V6K || CC_IS_CLANG
+	bool "Use the ARM EABI to compile the kernel" if CPU_SA110 || CPU_SA1100
+	default y
 	help
-	  This option allows for the kernel to be compiled using the latest
-	  ARM ABI (aka EABI).  This is only useful if you are using a user
-	  space environment that is also compiled with EABI.
+	  The Arm EABI is the default ABI on all modern Linux
+	  distributions, replacing the obsolete and "OABI" that was
+	  commonly used on ARMv4 distributions before ca. 2013.
 
-	  Since there are major incompatibilities between the legacy ABI and
-	  EABI, especially with regard to structure member alignment, this
-	  option also changes the kernel syscall calling convention to
-	  disambiguate both ABIs and allow for backward compatibility support
-	  (selected with CONFIG_OABI_COMPAT).
+	  Everyone should enable this, as support for OABI user space
+	  was dropped in gcc-4.8 and most distributions after ca. 2013.
+
+	  Support for OABI mode will be removed from the kernel
+	  once Intel StrongARM CPUs are phased out.
 
 config OABI_COMPAT
 	bool "Allow old ABI binaries to run with this kernel (EXPERIMENTAL)"
 	depends on AEABI && !THUMB2_KERNEL
+	depends on CPU_SA110 || CPU_SA1100
 	help
 	  This option preserves the old syscall interface along with the
 	  new (ARM EABI) one. It also provides a compatibility layer to
@@ -1180,11 +1192,8 @@ config OABI_COMPAT
 	  selected, since there is no way yet to sensibly distinguish
 	  between calling conventions during filtering.
 
-	  If you know you'll be using only pure EABI user space then you
-	  can say N here. If this option is not selected and you attempt
-	  to execute a legacy ABI binary then the result will be
-	  UNPREDICTABLE (in fact it can be predicted that it won't work
-	  at all). If in doubt say N.
+	  Support for OABI_COMPAT will be removed from the kernel
+	  once Intel StrongARM CPUs are phased out.
 
 config ARCH_SELECT_MEMORY_MODEL
 	def_bool y
@@ -1393,19 +1402,22 @@ config ARCH_WANT_FLAT_DTB_INSTALL
 
 config ATAGS
 	bool "Support for the traditional ATAGS boot data passing"
-	default y
+	depends on EXPERT
 	help
-	  This is the traditional way of passing data to the kernel at boot
-	  time. If you are solely relying on the flattened device tree (or
-	  the ARM_ATAG_DTB_COMPAT option) then you may unselect this option
-	  to remove ATAGS support from your kernel binary.
+	  This is the traditional way of passing data to the kernel at
+	  boot but is now only used by a few of the oldest ARMv4 and
+	  ARMv5 machines.  Users that still rely on ATAGS should plan
+	  on migrating to devicetree based booting.
+
+	  Support for ATAGS will be removed when the last machine using
+	  it is either converted to DT or removed.
 
 config DEPRECATED_PARAM_STRUCT
-	bool "Provide old way to pass kernel parameters"
+	bool "Provide old way to pass kernel parameters (DEPRECATED)"
 	depends on ATAGS
 	help
 	  This was deprecated in 2001 and announced to live on for 5 years.
-	  Some old boot loaders still use this way.
+	  It is now scheduled for removal in early 2027.
 
 # Compressed boot loader in ROM.  Yes, we really want to ask about
 # TEXT and BSS so we preserve their values in the config files.
@@ -1651,16 +1663,16 @@ menu "Floating point emulation"
 comment "At least one emulation must be selected"
 
 config FPE_NWFPE
-	bool "NWFPE math emulation"
+	bool "NWFPE math emulation (DEPRECATED)"
 	depends on (!AEABI || OABI_COMPAT) && !THUMB2_KERNEL
 	help
 	  Say Y to include the NWFPE floating point emulator in the kernel.
-	  This is necessary to run most binaries. Linux does not currently
-	  support floating point hardware so you need to say Y here even if
-	  your machine has an FPA or floating point co-processor podule.
 
-	  You may say N here if you are going to load the Acorn FPEmulator
-	  early in the bootup.
+	  This is only used on OABI userspace binaries, either using a
+	  pure OABI (!CONFIG_AEABI) kernel, or the OABI emulation.
+
+	  Support for NWFPE will be removed in the future when OABI
+	  support is removed.
 
 config FPE_NWFPE_XP
 	bool "Support extended precision"
diff --git a/arch/arm/Kconfig.platforms b/arch/arm/Kconfig.platforms
index 386eccc818683..2e118b65f93b4 100644
--- a/arch/arm/Kconfig.platforms
+++ b/arch/arm/Kconfig.platforms
@@ -33,7 +33,7 @@ config ARCH_MULTI_V4_V5
 config ARCH_MULTI_V6
 	bool "ARMv6 based platforms (ARM11)"
 	select ARCH_MULTI_V6_V7
-	select CPU_V6K
+	select CPU_ARM1176
 
 config ARCH_MULTI_V7
 	bool "ARMv7 based platforms (Cortex-A, PJ4, Scorpion, Krait)"
diff --git a/arch/arm/arm-soc-for-next-contents.txt b/arch/arm/arm-soc-for-next-contents.txt
new file mode 100644
index 0000000000000..bf41208df0013
--- /dev/null
+++ b/arch/arm/arm-soc-for-next-contents.txt
@@ -0,0 +1,36 @@
+soc/arm
+	arm/deprecation
+		https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc tags/arm-feature-deprecation-for-7.3
+
+soc/dt
+	patch
+		ARM: dts: st: spear: Correct indentation
+		ARM: dts: st: ste: Correct indentation
+		ARM: dts: st: spear13xx: Drop unused/incorrect usbh0_id and usbh1_id
+
+soc/drivers
+
+soc/defconfig
+
+soc/late
+
+arm/fixes
+	patch
+		MAINTAINERS: Update SpacemiT SoC git tree repository
+	<no branch> (71827776667f4e4677a4fa806bcfb24d4b8dd9d7)
+		git://git.kernel.org/pub/scm/linux/kernel/git/pza/linux tags/reset-fixes-for-v7.2
+	<no branch> (813e034925814858cc52e7de321ec4848314e15d)
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-7.2-pmc-fixes
+	<no branch> (265d5d4032c5f6eb089a6e6241d37fdbde7da180)
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-7.2-soc-fixes
+	<no branch> (806a66f926c2b6652aeb88983d01f25081b41a73)
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-7.2-arm64-dt-fixes
+	patch
+		ARM: Don't let ARMv5 platforms select USE_OF
+	<no branch> (314c243b201b678fa89226b1eaea51a71340454e)
+		git://git.kernel.org/pub/scm/linux/kernel/git/jenswi/linux-tee tags/tee-update-for-v7.2
+	patch
+		MAINTAINERS: Update maintainer and git tree for CIX SoC
+	<no branch> (980a8bfe7baec9b9ee0d5443b0b204552e41c407)
+		https://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux tags/scmi-ffa-fixes-7.2
+
diff --git a/arch/arm/boot/dts/allwinner/Makefile b/arch/arm/boot/dts/allwinner/Makefile
index f71392a55df87..75b2b6a2f7a67 100644
--- a/arch/arm/boot/dts/allwinner/Makefile
+++ b/arch/arm/boot/dts/allwinner/Makefile
@@ -182,6 +182,9 @@ dtb-$(CONFIG_MACH_SUN7I) += \
 	sun7i-a20-wits-pro-a20-dkt.dtb
 
 # Enables support for device-tree overlays for all pis
+DTC_FLAGS_sun8i-h2-plus-bananapi-m2-zero := -@
+DTC_FLAGS_sun8i-h2-plus-libretech-all-h3-cc := -@
+DTC_FLAGS_sun8i-h2-plus-orangepi-r1 := -@
 DTC_FLAGS_sun8i-h2-plus-orangepi-zero := -@
 DTC_FLAGS_sun8i-h3-orangepi-lite := -@
 DTC_FLAGS_sun8i-h3-bananapi-m2-plus := -@
@@ -201,6 +204,7 @@ DTC_FLAGS_sun8i-h3-orangepi-pc := -@
 DTC_FLAGS_sun8i-h3-bananapi-m2-plus-v1.2 := -@
 DTC_FLAGS_sun8i-h3-orangepi-pc-plus := -@
 DTC_FLAGS_sun8i-t113s-netcube-nagami-basic-carrier := -@
+DTC_FLAGS_sun8i-t113s-netcube-dobermann := -@
 DTC_FLAGS_sun8i-v3s-netcube-kumquat := -@
 dtb-$(CONFIG_MACH_SUN8I) += \
 	sun8i-a23-evb.dtb \
@@ -261,6 +265,7 @@ dtb-$(CONFIG_MACH_SUN8I) += \
 	sun8i-s3-lichee-zero-plus.dtb \
 	sun8i-s3-pinecube.dtb \
 	sun8i-t113s-mangopi-mq-r-t113.dtb \
+	sun8i-t113s-netcube-dobermann.dtb \
 	sun8i-t113s-netcube-nagami-basic-carrier.dtb \
 	sun8i-t113s-netcube-nagami-keypad-carrier.dtb \
 	sun8i-t3-cqa3t-bv3.dtb \
diff --git a/arch/arm/boot/dts/allwinner/sun8i-h3-nanopi-duo2.dts b/arch/arm/boot/dts/allwinner/sun8i-h3-nanopi-duo2.dts
index 2b0566d4b3867..88550da5af626 100644
--- a/arch/arm/boot/dts/allwinner/sun8i-h3-nanopi-duo2.dts
+++ b/arch/arm/boot/dts/allwinner/sun8i-h3-nanopi-duo2.dts
@@ -73,7 +73,7 @@ reg_vcc_dram: vcc-dram {
 		enable-active-high;
 		gpio = <&r_pio 0 9 GPIO_ACTIVE_HIGH>; /* PL9 */
 		vin-supply = <&reg_vcc5v0>;
-        };
+	};
 
 	reg_vdd_sys: vdd-sys {
 		compatible = "regulator-fixed";
@@ -85,7 +85,7 @@ reg_vdd_sys: vdd-sys {
 		enable-active-high;
 		gpio = <&r_pio 0 8 GPIO_ACTIVE_HIGH>; /* PL8 */
 		vin-supply = <&reg_vcc5v0>;
-        };
+	};
 
 	wifi_pwrseq: pwrseq {
 		compatible = "mmc-pwrseq-simple";
diff --git a/arch/arm/boot/dts/allwinner/sun8i-t113s-netcube-dobermann.dts b/arch/arm/boot/dts/allwinner/sun8i-t113s-netcube-dobermann.dts
new file mode 100644
index 0000000000000..d7765caffe2a2
--- /dev/null
+++ b/arch/arm/boot/dts/allwinner/sun8i-t113s-netcube-dobermann.dts
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (C) 2026 Lukas Schmid <lukas.schmid@netcube.li>
+ */
+
+/dts-v1/;
+#include "sun8i-t113s-netcube-nagami.dtsi"
+
+#include <dt-bindings/leds/common.h>
+
+/ {
+	model = "NetCube Systems OpenNMC (dobermann)";
+	compatible = "netcube,dobermann", "netcube,nagami",
+		     "allwinner,sun8i-t113s";
+
+	aliases {
+		serial2 = &uart4; // UART on SmartSlot
+		rtc0 = &ds3232;
+		rtc1 = &rtc; // not battery backed
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led_heartbeat_green: led-heartbeat-green {
+			gpios = <&pio 6 14 GPIO_ACTIVE_HIGH>; /* PG14 */
+			linux,default-trigger = "heartbeat";
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_HEARTBEAT;
+		};
+	};
+};
+
+&ehci0 {
+	status = "okay";
+};
+
+&ehci1 {
+	status = "okay";
+};
+
+&i2c2 {
+	status = "okay";
+
+	tusb320: typec@60 {
+		compatible = "ti,tusb320";
+		reg = <0x60>;
+		interrupts-extended = <&pio 3 22 IRQ_TYPE_LEVEL_LOW>;  /* PD22 */
+	};
+
+	ds3232: rtc@68 {
+		compatible = "dallas,ds3232";
+		reg = <0x68>;
+	};
+};
+
+/* microSD Card Slot on the board */
+&mmc0 {
+	vmmc-supply = <&reg_vcc3v3>;
+	disable-wp;
+	bus-width = <4>;
+	cd-gpios = <&pio 6 15 GPIO_ACTIVE_LOW>; /* PG15 */
+	status = "okay";
+};
+
+&ohci0 {
+	status = "okay";
+};
+
+&ohci1 {
+	status = "okay";
+};
+
+&pio {
+	gpio-line-names = "", "", "", "", // PA
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "SMART_TX", "SMART_RX", // PB
+			  "EXT_IO3", "EXT_IO2", "CONSOLE_TX", "CONSOLE_RX",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "eMMC_CLK", "eMMC_CMD", // PC
+			  "eMMC_D2", "eMMC_D1", "eMMC_D0", "eMMC_D3",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "", // PD
+			  "", "", "", "",
+			  "", "USB_SEC_EN", "EXT_SPI_nCS", "EXT_SPI_SCK",
+			  "EXT_SPI_MOSI", "EXT_SPI_MISO", "EXT_IO5", "EXT_IO4",
+			  "SMART_SEL", "", "", "",
+			  "I2C2_SCL", "I2C2_SDA", "TUSB320_nINT", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "ETH_CRSDV", "ETH_RXD0", "ETH_RXD1", "ETH_TXCK", // PE
+			  "ETH_TXD0", "ETH_TXD1", "ETH_TXEN", "",
+			  "ETH_MDC", "ETH_MDIO", "I2C3_nINT", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "uSD_D1", "uSD_D0", "uSD_CLK", "uSD_CMD", // PF
+			  "uSD_D3", "uSD_D2", "TUSB320_ID", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "ESP_CLK", "ESP_CMD", "ESP_D0", "ESP_D1", // PG
+			  "ESP_D2", "ESP_D3", "ESP_TXD", "ESP_RXD",
+			  "ESP_nBOOT", "ESP_nRST", "I2C3_SCL", "I2C3_SDA",
+			  "EXT_IO1", "EXT_IO0", "LED_HEARTBEAT", "SD_DETECT",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "";
+};
+
+/* SmartSlot serial */
+&uart4 {
+	pinctrl-0 = <&uart4_pb_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&usb_otg {
+	extcon = <&tusb320 0>;
+	dr_mode = "otg";
+	status = "okay";
+};
+
+&usbphy {
+	usb0_id_det-gpios = <&pio 5 6 GPIO_ACTIVE_HIGH>; /* PF6 */
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/intel/socfpga/socfpga_arria10.dtsi b/arch/arm/boot/dts/intel/socfpga/socfpga_arria10.dtsi
index ba4c0ca897883..3c684f1278bff 100644
--- a/arch/arm/boot/dts/intel/socfpga/socfpga_arria10.dtsi
+++ b/arch/arm/boot/dts/intel/socfpga/socfpga_arria10.dtsi
@@ -447,7 +447,7 @@ gmac0: ethernet@ff800000 {
 		gmac1: ethernet@ff802000 {
 			compatible = "altr,socfpga-stmmac-a10-s10", "snps,dwmac-3.72a", "snps,dwmac";
 			altr,sysmgr-syscon = <&sysmgr 0x48 8>;
-		        reg = <0xff802000 0x2000>;
+			reg = <0xff802000 0x2000>;
 			interrupts = <0 93 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "macirq";
 			/* Filled in by bootloader */
diff --git a/arch/arm/boot/dts/mediatek/mt8127.dtsi b/arch/arm/boot/dts/mediatek/mt8127.dtsi
index aced173c2a52e..bd61ec7e70c0c 100644
--- a/arch/arm/boot/dts/mediatek/mt8127.dtsi
+++ b/arch/arm/boot/dts/mediatek/mt8127.dtsi
@@ -75,7 +75,7 @@ uart_clk: dummy26m {
 			compatible = "fixed-clock";
 			clock-frequency = <26000000>;
 			#clock-cells = <0>;
-                };
+		};
 	};
 
 	timer {
diff --git a/arch/arm/boot/dts/nuvoton/nuvoton-common-npcm7xx.dtsi b/arch/arm/boot/dts/nuvoton/nuvoton-common-npcm7xx.dtsi
index ab3c3c5713aea..83cd10b472733 100644
--- a/arch/arm/boot/dts/nuvoton/nuvoton-common-npcm7xx.dtsi
+++ b/arch/arm/boot/dts/nuvoton/nuvoton-common-npcm7xx.dtsi
@@ -191,9 +191,8 @@ fiu0: spi@fb000000 {
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0xfb000000 0x1000>;
-			reg-names = "control", "memory";
+			reg-names = "control";
 			clocks = <&clk NPCM7XX_CLK_SPI0>;
-			clock-names = "clk_spi0";
 			status = "disabled";
 		};
 
@@ -202,9 +201,8 @@ fiu3: spi@c0000000 {
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0xc0000000 0x1000>;
-			reg-names = "control", "memory";
+			reg-names = "control";
 			clocks = <&clk NPCM7XX_CLK_SPI3>;
-			clock-names = "clk_spi3";
 			pinctrl-names = "default";
 			pinctrl-0 = <&spi3_pins>;
 			status = "disabled";
@@ -215,9 +213,8 @@ fiux: spi@fb001000 {
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0xfb001000 0x1000>;
-			reg-names = "control", "memory";
+			reg-names = "control";
 			clocks = <&clk NPCM7XX_CLK_SPIX>;
-			clock-names = "clk_spix";
 			status = "disabled";
 		};
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-sabresd.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-sabresd.dtsi
index fe9046c03ddd6..60fe0f973668b 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-sabresd.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-sabresd.dtsi
@@ -756,12 +756,14 @@ &pcie {
 	pinctrl-0 = <&pinctrl_pcie>;
 	/* This property is deprecated, use reset-gpios from the Root Port node. */
 	reset-gpio = <&gpio7 12 GPIO_ACTIVE_LOW>;
+	/* This property is deprecated, use power supply from the Root Port node. */
 	vpcie-supply = <&reg_pcie>;
 	status = "okay";
 };
 
 &pcie_port0 {
 	reset-gpios = <&gpio7 12 GPIO_ACTIVE_LOW>;
+	vpcie3v3-supply = <&reg_pcie>;
 };
 
 &pwm1 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6a.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6a.dtsi
index 67f8f59aff5ae..2fa116e1ca9a6 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6a.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6a.dtsi
@@ -39,6 +39,16 @@ m24c64_50: eeprom@50 {
 		reg = <0x50>;
 		pagesize = <32>;
 		vcc-supply = <&sw4_reg>;
+
+		nvmem-layout {
+			compatible = "fixed-layout";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			module_info: module-info@20 {
+				reg = <0x20 0x60>;
+			};
+		};
 	};
 };
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6b.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6b.dtsi
index db552802554d4..4361c7efb9d73 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6b.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-tqma6b.dtsi
@@ -32,6 +32,16 @@ m24c64_50: eeprom@50 {
 		reg = <0x50>;
 		pagesize = <32>;
 		vcc-supply = <&sw4_reg>;
+
+		nvmem-layout {
+			compatible = "fixed-layout";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			module_info: module-info@20 {
+				reg = <0x20 0x60>;
+			};
+		};
 	};
 };
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx6sx-sdb.dtsi b/arch/arm/boot/dts/nxp/imx/imx6sx-sdb.dtsi
index 338de4d144b2a..2d3521ea0caa6 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6sx-sdb.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6sx-sdb.dtsi
@@ -284,12 +284,14 @@ &pcie {
 	pinctrl-0 = <&pinctrl_pcie>;
 	/* This property is deprecated, use reset-gpios from the Root Port node. */
 	reset-gpio = <&gpio2 0 GPIO_ACTIVE_LOW>;
+	/* This property is deprecated, use power supply from the Root Port node. */
 	vpcie-supply = <&reg_pcie_gpio>;
 	status = "okay";
 };
 
 &pcie_port0 {
 	reset-gpios = <&gpio2 0 GPIO_ACTIVE_LOW>;
+	vpcie3v3-supply = <&reg_pcie_gpio>;
 };
 
 &lcdif1 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul-common.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul-common.dtsi
index 4fa98e6a66d7c..9b70e58868b08 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul-common.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-tqma6ul-common.dtsi
@@ -126,6 +126,16 @@ m24c64_50: eeprom@50 {
 		compatible = "atmel,24c64";
 		reg = <0x50>;
 		pagesize = <32>;
+
+		nvmem-layout {
+			compatible = "fixed-layout";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			module_info: module-info@20 {
+				reg = <0x20 0x60>;
+			};
+		};
 	};
 
 	m24c02_52: eeprom@52 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-pico.dtsi b/arch/arm/boot/dts/nxp/imx/imx7d-pico.dtsi
index a1574ccec89cb..6d4e8087192c6 100644
--- a/arch/arm/boot/dts/nxp/imx/imx7d-pico.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-pico.dtsi
@@ -339,6 +339,20 @@ &uart7 { /* Bluetooth */
 	assigned-clock-parents = <&clks IMX7D_PLL_SYS_MAIN_240M_CLK>;
 	uart-has-rtscts;
 	status = "okay";
+
+	bluetooth {
+		compatible = "brcm,bcm4329-bt";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_bluetooth>;
+
+		max-speed = <3000000>;
+
+		reset-gpios = <&gpio4 18 GPIO_ACTIVE_LOW>;
+		device-wakeup-gpios = <&gpio4 17 GPIO_ACTIVE_HIGH>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <11 IRQ_TYPE_EDGE_FALLING>;
+		interrupt-names = "host-wakeup";
+	};
 };
 
 &usbotg1 {
@@ -401,6 +415,15 @@ &wdog1 {
 };
 
 &iomuxc {
+	pinctrl_bluetooth: bluetoothgrp {
+		fsl,pins = <
+			MX7D_PAD_ECSPI1_MOSI__GPIO4_IO17	0x14
+			MX7D_PAD_ECSPI1_MISO__GPIO4_IO18	0x14
+			MX7D_PAD_SD2_RESET_B__GPIO5_IO11	0x14
+		>;
+
+	};
+
 	pinctrl_ecspi3: ecspi3grp {
 		fsl,pins = <
 			MX7D_PAD_I2C1_SCL__ECSPI3_MISO		0x2
diff --git a/arch/arm/boot/dts/nxp/ls/ls1021a-twr.dts b/arch/arm/boot/dts/nxp/ls/ls1021a-twr.dts
index 38281b904301d..7341945de3ce6 100644
--- a/arch/arm/boot/dts/nxp/ls/ls1021a-twr.dts
+++ b/arch/arm/boot/dts/nxp/ls/ls1021a-twr.dts
@@ -190,7 +190,7 @@ tbi1: tbi-phy@1f {
 };
 
 &esdhc {
-        status = "okay";
+	status = "okay";
 };
 
 &qspi {
diff --git a/arch/arm/boot/dts/qcom/Makefile b/arch/arm/boot/dts/qcom/Makefile
index 32a44b02d2fa6..026d5b682b2b3 100644
--- a/arch/arm/boot/dts/qcom/Makefile
+++ b/arch/arm/boot/dts/qcom/Makefile
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
 dtb-$(CONFIG_ARCH_QCOM) += \
 	msm8226-motorola-falcon.dtb \
+	msm8226-motorola-titan.dtb \
+	msm8926-sony-xperia-yukon-eagle.dtb \
 	qcom-apq8016-sbc.dtb \
 	qcom-apq8026-asus-sparrow.dtb \
 	qcom-apq8026-huawei-sturgeon.dtb \
diff --git a/arch/arm/boot/dts/qcom/msm8226-motorola-titan.dts b/arch/arm/boot/dts/qcom/msm8226-motorola-titan.dts
new file mode 100644
index 0000000000000..1609fb21c5448
--- /dev/null
+++ b/arch/arm/boot/dts/qcom/msm8226-motorola-titan.dts
@@ -0,0 +1,350 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2026, David Wales <daviewales@disroot.org>
+ */
+
+/dts-v1/;
+
+#include "qcom-msm8226.dtsi"
+#include "pm8226.dtsi"
+
+/delete-node/ &smem_region;
+
+/ {
+	model = "Motorola Moto G2 (2014)";
+	compatible = "motorola,titan", "qcom,msm8226";
+	chassis-type = "handset";
+
+	aliases {
+		mmc0 = &sdhc_1; /* eMMC */
+	};
+
+	chosen {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		framebuffer@3200000 {
+			compatible = "simple-framebuffer";
+			reg = <0x03200000 0x800000>;
+			width = <720>;
+			height = <1280>;
+			stride = <(720 * 3)>;
+			format = "r8g8b8";
+			vsp-supply = <&reg_lcd_pos>;
+			vsn-supply = <&reg_lcd_neg>;
+			vddio-supply = <&vddio_disp_vreg>;
+
+			clocks = <&mmcc MDSS_AHB_CLK>,
+				 <&mmcc MDSS_AXI_CLK>,
+				 <&mmcc MDSS_BYTE0_CLK>,
+				 <&mmcc MDSS_ESC0_CLK>,
+				 <&mmcc MDSS_MDP_CLK>,
+				 <&mmcc MMSS_MISC_AHB_CLK>,
+				 <&mmcc MDSS_PCLK0_CLK>,
+				 <&mmcc MDSS_VSYNC_CLK>;
+			power-domains = <&mmcc MDSS_GDSC>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		event-hall-sensor {
+			label = "Hall Effect Sensor";
+			gpios = <&tlmm 109 GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_SW>;
+			linux,code = <SW_LID>;
+			linux,can-disable;
+		};
+
+		key-volume-down {
+			label = "Volume Down";
+			gpios = <&tlmm 107 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEDOWN>;
+			debounce-interval = <15>;
+		};
+
+		key-volume-up {
+			label = "Volume Up";
+			gpios = <&tlmm 106 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
+			debounce-interval = <15>;
+		};
+	};
+
+	vddio_disp_vreg: regulator-vddio-disp {
+		compatible = "regulator-fixed";
+		regulator-name = "vddio_disp";
+
+		gpio = <&tlmm 10 GPIO_ACTIVE_HIGH>;
+		startup-delay-us = <300>;
+		enable-active-high;
+		regulator-boot-on;
+	};
+
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		framebuffer@3200000 {
+			reg = <0x03200000 0xfa0000>;
+			no-map;
+		};
+
+		dhob@f500000 {
+			reg = <0x0f500000 0x40000>;
+			no-map;
+		};
+
+		shob@f540000 {
+			reg = <0x0f540000 0x2000>;
+			no-map;
+		};
+
+		smem_region: smem@fa00000 {
+			reg = <0x0fa00000 0x100000>;
+			no-map;
+		};
+
+		reserved@fb00000 {
+			reg = <0x0fb00000 0x400000>;
+			no-map;
+		};
+	};
+};
+
+&blsp1_i2c4 {
+	status = "okay";
+
+	regulator@3e {
+		compatible = "ti,tps65132";
+		reg = <0x3e>;
+
+		pinctrl-0 = <&reg_lcd_default>;
+		pinctrl-names = "default";
+
+		reg_lcd_pos: outp {
+			regulator-name = "outp";
+			regulator-min-microvolt = <5400000>;
+			regulator-max-microvolt = <5600000>;
+			regulator-active-discharge = <1>;
+			regulator-boot-on;
+			enable-gpios = <&tlmm 12 GPIO_ACTIVE_HIGH>;
+		};
+
+		reg_lcd_neg: outn {
+			regulator-name = "outn";
+			regulator-min-microvolt = <5400000>;
+			regulator-max-microvolt = <5600000>;
+			regulator-active-discharge = <1>;
+			regulator-boot-on;
+			enable-gpios = <&tlmm 13 GPIO_ACTIVE_HIGH>;
+		};
+	};
+};
+
+&pm8226_vib {
+	status = "okay";
+};
+
+&rpm_requests {
+	regulators {
+		compatible = "qcom,rpm-pm8226-regulators";
+
+		pm8226_s3: s3 {
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1300000>;
+		};
+
+		pm8226_s4: s4 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2200000>;
+		};
+
+		pm8226_s5: s5 {
+			regulator-min-microvolt = <1150000>;
+			regulator-max-microvolt = <1150000>;
+		};
+
+		pm8226_l1: l1 {
+			regulator-min-microvolt = <1225000>;
+			regulator-max-microvolt = <1225000>;
+		};
+
+		pm8226_l2: l2 {
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+		};
+
+		pm8226_l3: l3 {
+			regulator-min-microvolt = <750000>;
+			regulator-max-microvolt = <1337500>;
+		};
+
+		pm8226_l4: l4 {
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+		};
+
+		pm8226_l5: l5 {
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+		};
+
+		pm8226_l6: l6 {
+			/* Hall effect sensor */
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-always-on;
+		};
+
+		pm8226_l7: l7 {
+			regulator-min-microvolt = <1850000>;
+			regulator-max-microvolt = <1850000>;
+		};
+
+		pm8226_l8: l8 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		pm8226_l9: l9 {
+			regulator-min-microvolt = <2050000>;
+			regulator-max-microvolt = <2050000>;
+		};
+
+		pm8226_l10: l10 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		pm8226_l12: l12 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		pm8226_l14: l14 {
+			regulator-min-microvolt = <2750000>;
+			regulator-max-microvolt = <2750000>;
+		};
+
+		pm8226_l15: l15 {
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+		};
+
+		pm8226_l16: l16 {
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3350000>;
+		};
+
+		pm8226_l17: l17 {
+			regulator-min-microvolt = <2950000>;
+			regulator-max-microvolt = <2950000>;
+		};
+
+		pm8226_l18: l18 {
+			regulator-min-microvolt = <2950000>;
+			regulator-max-microvolt = <2950000>;
+		};
+
+		pm8226_l19: l19 {
+			regulator-min-microvolt = <2850000>;
+			regulator-max-microvolt = <2850000>;
+		};
+
+		pm8226_l20: l20 {
+			regulator-min-microvolt = <3075000>;
+			regulator-max-microvolt = <3075000>;
+		};
+
+		pm8226_l21: l21 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2950000>;
+			regulator-allow-set-load;
+		};
+
+		pm8226_l22: l22 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2950000>;
+		};
+
+		pm8226_l23: l23 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2950000>;
+		};
+
+		pm8226_l24: l24 {
+			regulator-min-microvolt = <1300000>;
+			regulator-max-microvolt = <1350000>;
+		};
+
+		pm8226_l25: l25 {
+			regulator-min-microvolt = <1775000>;
+			regulator-max-microvolt = <2125000>;
+		};
+
+		pm8226_l26: l26 {
+			regulator-min-microvolt = <1225000>;
+			regulator-max-microvolt = <1225000>;
+		};
+
+		pm8226_l27: l27 {
+			regulator-min-microvolt = <2050000>;
+			regulator-max-microvolt = <2050000>;
+		};
+
+		pm8226_l28: l28 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <3400000>;
+			regulator-boot-on;
+		};
+
+		pm8226_lvs1: lvs1 {
+			/* Pull-up for I2C lines */
+			regulator-always-on;
+		};
+	};
+};
+
+&sdhc_1 {
+	vmmc-supply = <&pm8226_l17>;
+	vqmmc-supply = <&pm8226_l6>;
+	bus-width = <8>;
+	non-removable;
+
+	status = "okay";
+};
+
+&smbb {
+	qcom,fast-charge-safe-current = <2000000>;
+	qcom,fast-charge-current-limit = <1900000>;
+	qcom,fast-charge-safe-voltage = <4400000>;
+	qcom,minimum-input-voltage = <4300000>;
+
+	status = "okay";
+};
+
+&usb {
+	extcon = <&smbb>;
+	dr_mode = "peripheral";
+
+	status = "okay";
+};
+
+&usb_hs_phy {
+	extcon = <&smbb>;
+	v1p8-supply = <&pm8226_l10>;
+	v3p3-supply = <&pm8226_l20>;
+};
+
+&tlmm {
+	reg_lcd_default: reg-lcd-default-state {
+		pins = "gpio12", "gpio13";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+};
diff --git a/arch/arm/boot/dts/qcom/msm8926-sony-xperia-yukon-eagle.dts b/arch/arm/boot/dts/qcom/msm8926-sony-xperia-yukon-eagle.dts
new file mode 100644
index 0000000000000..b3ef39726b867
--- /dev/null
+++ b/arch/arm/boot/dts/qcom/msm8926-sony-xperia-yukon-eagle.dts
@@ -0,0 +1,407 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2026 Erikas Bitovtas <xerikasxx@gmail.com>
+ */
+
+/dts-v1/;
+
+#include "msm8926.dtsi"
+#include "pm8226.dtsi"
+
+/delete-node/ &adsp_region;
+/delete-node/ &mba_region;
+/delete-node/ &mpss_region;
+/delete-node/ &smem_region;
+
+/ {
+	model = "Sony Xperia M2";
+	compatible = "sony,eagle", "qcom,msm8926", "qcom,msm8226";
+	chassis-type = "handset";
+
+	aliases {
+		mmc0 = &sdhc_1;
+		mmc1 = &sdhc_2;
+		display0 = &framebuffer0;
+	};
+
+	chosen {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		stdout-path = "display0";
+
+		framebuffer0: framebuffer@3201000 {
+			compatible = "simple-framebuffer";
+			reg = <0x03201000 0x800000>;
+			width = <540>;
+			height = <960>;
+			stride = <(540 * 3)>;
+			format = "r8g8b8";
+
+			clocks = <&mmcc MDSS_AHB_CLK>,
+				 <&mmcc MDSS_AXI_CLK>,
+				 <&mmcc MDSS_BYTE0_CLK>,
+				 <&mmcc MDSS_MDP_CLK>,
+				 <&mmcc MDSS_PCLK0_CLK>,
+				 <&mmcc MDSS_VSYNC_CLK>;
+			power-domains = <&mmcc MDSS_GDSC>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&gpio_keys_default>;
+		pinctrl-names = "default";
+
+		button-camera-focus {
+			label = "Camera Focus";
+			gpios = <&tlmm 108 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_CAMERA_FOCUS>;
+			debounce-interval = <15>;
+		};
+
+		button-camera-snapshot {
+			label = "Camera Snapshot";
+			gpios = <&tlmm 107 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_CAMERA>;
+			debounce-interval = <15>;
+		};
+
+		button-volume-down {
+			label = "Volume Down";
+			gpios = <&tlmm 106 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEDOWN>;
+			debounce-interval = <15>;
+		};
+	};
+
+	reserved-memory {
+		framebuffer@3201000 {
+			reg = <0x03201000 0x800000>;
+			no-map;
+		};
+
+		mpss_region: mpss@8000000 {
+			reg = <0x08000000 0x4d00000>;
+			no-map;
+		};
+
+		reserved@cd00000 {
+			reg = <0x0cd00000 0x200000>;
+			no-map;
+		};
+
+		mba_region: mba@d100000 {
+			reg = <0x0d100000 0x3a000>;
+			no-map;
+		};
+
+		reserved@d13a000 {
+			reg = <0x0d13a000 0xc6000>;
+			no-map;
+		};
+
+		wcnss_region: wcnss@d200000 {
+			reg = <0x0d200000 0x5ae000>;
+			no-map;
+		};
+
+		reserved@d850000 {
+			reg = <0x0d850000 0x3b0000>;
+			no-map;
+		};
+
+		adsp_region: adsp@dc00000 {
+			reg = <0x0dc00000 0x1400000>;
+			no-map;
+		};
+
+		reserved@f000000 {
+			reg = <0x0f000000 0x500000>;
+			no-map;
+		};
+
+		venus_region: venus@f500000 {
+			reg = <0x0f500000 0x500000>;
+			no-map;
+		};
+
+		smem_region: smem@fa00000 {
+			reg = <0x0fa00000 0x100000>;
+			no-map;
+		};
+
+		reserved@fb00000 {
+			reg = <0x0fb00000 0x280000>;
+			no-map;
+		};
+
+		rmtfs@fd80000 {
+			compatible = "qcom,rmtfs-mem";
+			reg = <0x0fd80000 0x180000>;
+			no-map;
+
+			qcom,client-id = <1>;
+		};
+	};
+};
+
+&blsp1_i2c2 {
+	status = "okay";
+
+	accelerometer@f {
+		compatible = "kionix,kxtj21009";
+		reg = <0x0f>;
+
+		interrupts-extended = <&tlmm 49 IRQ_TYPE_EDGE_RISING>;
+
+		vdd-supply = <&pm8226_l19>;
+		vddio-supply = <&pm8226_lvs1>;
+
+		mount-matrix = "0",  "1", "0",
+			       "-1", "0", "0",
+			       "0",  "0", "1";
+	};
+
+	light-sensor@48 {
+		compatible = "sensortek,stk3310";
+		reg = <0x48>;
+
+		interrupts-extended = <&tlmm 65 IRQ_TYPE_EDGE_FALLING>;
+
+		proximity-near-level = <1700>;
+	};
+};
+
+&blsp1_i2c3 {
+	status = "okay";
+
+	nfc@28 {
+		compatible = "nxp,pn547", "nxp,nxp-nci-i2c";
+		reg = <0x28>;
+
+		interrupts-extended = <&tlmm 21 IRQ_TYPE_EDGE_RISING>;
+
+		enable-gpios = <&tlmm 20 GPIO_ACTIVE_HIGH>;
+		firmware-gpios = <&tlmm 22 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&pm8226_resin {
+	linux,code = <KEY_VOLUMEUP>;
+
+	status = "okay";
+};
+
+&pm8226_vib {
+	status = "okay";
+};
+
+&rpm_requests {
+	regulators {
+		compatible = "qcom,rpm-pm8226-regulators";
+
+		pm8226_s3: s3 {
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1300000>;
+		};
+
+		pm8226_s4: s4 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2200000>;
+		};
+
+		pm8226_s5: s5 {
+			regulator-min-microvolt = <1150000>;
+			regulator-max-microvolt = <1150000>;
+		};
+
+		pm8226_l1: l1 {
+			regulator-min-microvolt = <1225000>;
+			regulator-max-microvolt = <1225000>;
+		};
+
+		pm8226_l2: l2 {
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+		};
+
+		pm8226_l3: l3 {
+			regulator-min-microvolt = <750000>;
+			regulator-max-microvolt = <1287500>;
+		};
+
+		pm8226_l4: l4 {
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+		};
+
+		pm8226_l5: l5 {
+			regulator-min-microvolt = <1050000>;
+			regulator-max-microvolt = <1200000>;
+		};
+
+		pm8226_l6: l6 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		pm8226_l7: l7 {
+			regulator-min-microvolt = <1850000>;
+			regulator-max-microvolt = <1850000>;
+		};
+
+		pm8226_l8: l8 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		pm8226_l9: l9 {
+			regulator-min-microvolt = <2050000>;
+			regulator-max-microvolt = <2050000>;
+		};
+
+		pm8226_l10: l10 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		pm8226_l12: l12 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		pm8226_l14: l14 {
+			regulator-min-microvolt = <2750000>;
+			regulator-max-microvolt = <2750000>;
+		};
+
+		pm8226_l15: l15 {
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+		};
+
+		pm8226_l16: l16 {
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3350000>;
+		};
+
+		pm8226_l17: l17 {
+			regulator-min-microvolt = <2950000>;
+			regulator-max-microvolt = <2950000>;
+		};
+
+		pm8226_l18: l18 {
+			regulator-min-microvolt = <2950000>;
+			regulator-max-microvolt = <2950000>;
+
+			regulator-system-load = <200000>;
+			regulator-allow-set-load;
+		};
+
+		pm8226_l19: l19 {
+			regulator-min-microvolt = <2850000>;
+			regulator-max-microvolt = <2850000>;
+		};
+
+		pm8226_l20: l20 {
+			regulator-min-microvolt = <3075000>;
+			regulator-max-microvolt = <3075000>;
+		};
+
+		pm8226_l21: l21 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2950000>;
+			regulator-allow-set-load;
+		};
+
+		pm8226_l22: l22 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2950000>;
+		};
+
+		pm8226_l23: l23 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2950000>;
+		};
+
+		pm8226_l24: l24 {
+			regulator-min-microvolt = <1300000>;
+			regulator-max-microvolt = <1350000>;
+		};
+
+		pm8226_l25: l25 {
+			regulator-min-microvolt = <1775000>;
+			regulator-max-microvolt = <2125000>;
+		};
+
+		pm8226_l26: l26 {
+			regulator-min-microvolt = <1225000>;
+			regulator-max-microvolt = <1225000>;
+		};
+
+		pm8226_l27: l27 {
+			regulator-min-microvolt = <2100000>;
+			regulator-max-microvolt = <2100000>;
+		};
+
+		pm8226_l28: l28 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2950000>;
+		};
+
+		pm8226_lvs1: lvs1 {};
+	};
+};
+
+&smbb {
+	qcom,fast-charge-current-limit = <1500000>;
+	qcom,fast-charge-safe-voltage = <4250000>;
+	qcom,minimum-input-voltage = <4200000>;
+
+	status = "okay";
+};
+
+&sdhc_1 {
+	vmmc-supply = <&pm8226_l17>;
+	vqmmc-supply = <&pm8226_l6>;
+
+	bus-width = <8>;
+	non-removable;
+
+	status = "okay";
+};
+
+&sdhc_2 {
+	vmmc-supply = <&pm8226_l18>;
+	vqmmc-supply = <&pm8226_l21>;
+
+	bus-width = <4>;
+	cd-gpios = <&tlmm 38 GPIO_ACTIVE_LOW>;
+
+	status = "okay";
+};
+
+&tlmm {
+	gpio_keys_default: gpio-keys-default-state {
+		pins = "gpio106", "gpio107", "gpio108";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+};
+
+&usb {
+	extcon = <&smbb>;
+	dr_mode = "peripheral";
+
+	status = "okay";
+};
+
+&usb_hs_phy {
+	extcon = <&smbb>;
+	v1p8-supply = <&pm8226_l10>;
+	v3p3-supply = <&pm8226_l20>;
+};
diff --git a/arch/arm/boot/dts/qcom/pm8921.dtsi b/arch/arm/boot/dts/qcom/pm8921.dtsi
index 535cb6a2543f6..15246f4bd2672 100644
--- a/arch/arm/boot/dts/qcom/pm8921.dtsi
+++ b/arch/arm/boot/dts/qcom/pm8921.dtsi
@@ -75,50 +75,62 @@ pm8921_xoadc: xoadc@197 {
 
 			vcoin: adc-channel@0 {
 				reg = <0x00 0x00>;
+				label = "vcoin";
 			};
 
 			vbat: adc-channel@1 {
 				reg = <0x00 0x01>;
+				label = "vbat";
 			};
 
 			dcin: adc-channel@2 {
 				reg = <0x00 0x02>;
+				label = "dcin";
 			};
 
 			vph_pwr: adc-channel@4 {
 				reg = <0x00 0x04>;
+				label = "vph_pwr";
 			};
 
 			batt_therm: adc-channel@8 {
 				reg = <0x00 0x08>;
+				label = "batt_therm";
 			};
 
 			batt_id: adc-channel@9 {
 				reg = <0x00 0x09>;
+				label = "batt_id";
 			};
 
 			usb_vbus: adc-channel@a {
 				reg = <0x00 0x0a>;
+				label = "usb_vbus";
 			};
 
 			die_temp: adc-channel@b {
 				reg = <0x00 0x0b>;
+				label = "die_temp";
 			};
 
 			ref_625mv: adc-channel@c {
 				reg = <0x00 0x0c>;
+				label = "ref_625mv";
 			};
 
 			ref_1250mv: adc-channel@d {
 				reg = <0x00 0x0d>;
+				label = "ref_1250mv";
 			};
 
 			chg_temp: adc-channel@e {
 				reg = <0x00 0x0e>;
+				label = "chg_temp";
 			};
 
 			ref_muxoff: adc-channel@f {
 				reg = <0x00 0x0f>;
+				label = "ref_muxoff";
 			};
 		};
 	};
diff --git a/arch/arm/boot/dts/qcom/qcom-apq8064-pins.dtsi b/arch/arm/boot/dts/qcom/qcom-apq8064-pins.dtsi
index e53de709e9d10..e95da761234b1 100644
--- a/arch/arm/boot/dts/qcom/qcom-apq8064-pins.dtsi
+++ b/arch/arm/boot/dts/qcom/qcom-apq8064-pins.dtsi
@@ -87,7 +87,7 @@ gsbi7_uart_2pins: gsbi7-uart-2pins-state {
 		function = "gsbi7";
 	};
 
-	gsbi7_uart_4pins: gsbi7_uart_4pins-state {
+	gsbi7_uart_4pins: gsbi7-uart-4pins-state {
 		pins = "gpio82", "gpio83", "gpio84", "gpio85";
 		function = "gsbi7";
 	};
diff --git a/arch/arm/boot/dts/qcom/qcom-ipq8064.dtsi b/arch/arm/boot/dts/qcom/qcom-ipq8064.dtsi
index adedcc6da1da5..1a278503d3401 100644
--- a/arch/arm/boot/dts/qcom/qcom-ipq8064.dtsi
+++ b/arch/arm/boot/dts/qcom/qcom-ipq8064.dtsi
@@ -591,7 +591,7 @@ saw1: power-manager@2099000 {
 
 		nss_common: syscon@3000000 {
 			compatible = "syscon";
-			reg = <0x03000000 0x0000FFFF>;
+			reg = <0x03000000 0x0000ffff>;
 		};
 
 		usb3_0: usb@100f8800 {
@@ -1241,7 +1241,7 @@ pcie@0 {
 
 		qsgmii_csr: syscon@1bb00000 {
 			compatible = "syscon";
-			reg = <0x1bb00000 0x000001FF>;
+			reg = <0x1bb00000 0x000001ff>;
 		};
 
 		lcc: clock-controller@28000000 {
diff --git a/arch/arm/boot/dts/qcom/qcom-mdm9615-wp8548.dtsi b/arch/arm/boot/dts/qcom/qcom-mdm9615-wp8548.dtsi
index 0dd52cac0e2e2..7206a33da2795 100644
--- a/arch/arm/boot/dts/qcom/qcom-mdm9615-wp8548.dtsi
+++ b/arch/arm/boot/dts/qcom/qcom-mdm9615-wp8548.dtsi
@@ -15,7 +15,7 @@ / {
 
 	memory@48000000 {
 		device_type = "memory";
-		reg = <0x48000000 0x7F00000>;
+		reg = <0x48000000 0x7f00000>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8960-samsung-expressatt.dts b/arch/arm/boot/dts/qcom/qcom-msm8960-samsung-expressatt.dts
index 5a39abd6f3ce9..5a3d88e31cf5b 100644
--- a/arch/arm/boot/dts/qcom/qcom-msm8960-samsung-expressatt.dts
+++ b/arch/arm/boot/dts/qcom/qcom-msm8960-samsung-expressatt.dts
@@ -52,6 +52,40 @@ key-volume-down {
 			linux,code = <KEY_VOLUMEDOWN>;
 		};
 	};
+
+	touchkey_enable: touchkey-enable {
+		compatible = "regulator-fixed";
+		regulator-name = "touchkey_enable";
+		gpio = <&tlmm 51 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-boot-on;
+	};
+
+	i2c-gpio-touchkey {
+		compatible = "i2c-gpio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		sda-gpios = <&tlmm 71 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+		scl-gpios = <&tlmm 72 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+		pinctrl-0 = <&touchkey_i2c_pins>;
+		pinctrl-names = "default";
+		i2c-gpio,delay-us = <2>;
+
+		touchkey@20 {
+			compatible = "coreriver,tc360-touchkey";
+			reg = <0x20>;
+
+			interrupts-extended = <&tlmm 52 IRQ_TYPE_EDGE_FALLING>;
+			pinctrl-0 = <&touchkey_irq_pin>;
+			pinctrl-names = "default";
+
+			vddio-supply = <&touchkey_enable>;
+			vdd-supply = <&pm8921_l29>;
+			vcc-supply = <&pm8921_l29>;
+
+			linux,keycodes = <KEY_MENU KEY_BACK>;
+		};
+	};
 };
 
 &gsbi2 {
@@ -198,6 +232,20 @@ firmware-pins {
 			bias-disable;
 		};
 	};
+
+	touchkey_i2c_pins: touchkey-i2c-state {
+		pins = "gpio71", "gpio72";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	touchkey_irq_pin: touchkey-irq-state {
+		pins = "gpio52";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
 };
 
 &pm8921 {
@@ -420,6 +468,12 @@ pm8921_l25: l25 {
 			bias-pull-down;
 		};
 
+		pm8921_l29: l29 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <3300000>;
+			bias-pull-down;
+		};
+
 		/* Low Voltage Switch */
 		pm8921_lvs1: lvs1 {
 			bias-pull-down;
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8974.dtsi b/arch/arm/boot/dts/qcom/qcom-msm8974.dtsi
index 95be1d2e214f9..06c0af0aa3176 100644
--- a/arch/arm/boot/dts/qcom/qcom-msm8974.dtsi
+++ b/arch/arm/boot/dts/qcom/qcom-msm8974.dtsi
@@ -1866,7 +1866,7 @@ cci_i2c1_sleep: cci-i2c1-sleep-pins {
 				};
 			};
 
-			spi8_default: spi8_default-state {
+			spi8_default: spi8-default-state {
 				mosi-pins {
 					pins = "gpio45";
 					function = "blsp_spi8";
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8974pro-htc-m8.dts b/arch/arm/boot/dts/qcom/qcom-msm8974pro-htc-m8.dts
index 402372834c53d..66ad93e7dd205 100644
--- a/arch/arm/boot/dts/qcom/qcom-msm8974pro-htc-m8.dts
+++ b/arch/arm/boot/dts/qcom/qcom-msm8974pro-htc-m8.dts
@@ -3,6 +3,7 @@
 #include "pm8841.dtsi"
 #include "pm8941.dtsi"
 #include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
 
 / {
 	model = "HTC One (M8)";
@@ -64,6 +65,75 @@ vreg_vph_pwr: vreg-vph-pwr {
 	};
 };
 
+&blsp1_i2c2 {
+	clock-frequency = <384000>;
+
+	status = "okay";
+
+	touch@20 {
+		compatible = "syna,rmi4-i2c";
+		reg = <0x20>;
+
+		interrupts-extended = <&tlmm 18 IRQ_TYPE_LEVEL_LOW>;
+
+		pinctrl-0 = <&ts_int_pin>;
+		pinctrl-names = "default";
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		rmi4-f01@1 {
+			reg = <0x1>;
+			syna,nosleep-mode = <1>;
+		};
+
+		rmi4-f11@11 {
+			reg = <0x11>;
+			syna,sensor-type = <1>;
+		};
+	};
+};
+
+&blsp1_i2c3 {
+	clock-frequency = <384000>;
+
+	status = "okay";
+
+	nfc@28 {
+		compatible = "nxp,pn544-i2c";
+		reg = <0x28>;
+
+		interrupts-extended = <&tlmm 144 IRQ_TYPE_LEVEL_HIGH>;
+
+		enable-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
+		firmware-gpios = <&tlmm 3 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&pm8941_lpg {
+	qcom,power-source = <1>;
+
+	status = "okay";
+
+	multi-led {
+		color = <LED_COLOR_ID_MULTI>;
+		function = LED_FUNCTION_STATUS;
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		led@6 {
+			reg = <6>;
+			color = <LED_COLOR_ID_GREEN>;
+		};
+
+		led@7 {
+			reg = <7>;
+			color = <LED_COLOR_ID_ORANGE>;
+		};
+	};
+};
+
 &pm8941_vib {
 	status = "okay";
 };
@@ -312,11 +382,27 @@ cmd-data-pins {
 		};
 	};
 
+	ts_int_pin: ts-int-pin-state {
+		pins = "gpio18";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
 	wcnss_pin_a: wcnss-pin-active-state {
-		pins = "gpio36", "gpio37", "gpio38", "gpio39", "gpio40";
-		function = "wlan";
-		drive-strength = <6>;
-		bias-pull-down;
+		bt-pins {
+			pins = "gpio35", "gpio43", "gpio44";
+			function = "bt";
+			drive-strength = <2>;
+			bias-pull-down;
+		};
+
+		wlan-pins {
+			pins = "gpio36", "gpio37", "gpio38", "gpio39", "gpio40";
+			function = "wlan";
+			drive-strength = <6>;
+			bias-pull-down;
+		};
 	};
 };
 
diff --git a/arch/arm/boot/dts/qcom/qcom-sdx55-t55.dts b/arch/arm/boot/dts/qcom/qcom-sdx55-t55.dts
index 082f7ed1a01fb..d23b8ec08af43 100644
--- a/arch/arm/boot/dts/qcom/qcom-sdx55-t55.dts
+++ b/arch/arm/boot/dts/qcom/qcom-sdx55-t55.dts
@@ -237,8 +237,6 @@ &blsp1_uart3 {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	status = "okay";
 };
 
diff --git a/arch/arm/boot/dts/qcom/qcom-sdx55-telit-fn980-tlb.dts b/arch/arm/boot/dts/qcom/qcom-sdx55-telit-fn980-tlb.dts
index e336a15b45c4c..1e80ddad591e9 100644
--- a/arch/arm/boot/dts/qcom/qcom-sdx55-telit-fn980-tlb.dts
+++ b/arch/arm/boot/dts/qcom/qcom-sdx55-telit-fn980-tlb.dts
@@ -237,8 +237,6 @@ &blsp1_uart3 {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	status = "okay";
 };
 
diff --git a/arch/arm/boot/dts/qcom/qcom-sdx55.dtsi b/arch/arm/boot/dts/qcom/qcom-sdx55.dtsi
index 05b79281df571..e9982c46bc2f5 100644
--- a/arch/arm/boot/dts/qcom/qcom-sdx55.dtsi
+++ b/arch/arm/boot/dts/qcom/qcom-sdx55.dtsi
@@ -510,6 +510,9 @@ ipa: ipa@1e40000 {
 			qcom,smem-state-names = "ipa-clock-enabled-valid",
 						"ipa-clock-enabled";
 
+			qcom,gsi-loader = "self";
+			memory-region = <&ipa_fw_mem>;
+
 			status = "disabled";
 		};
 
diff --git a/arch/arm/boot/dts/qcom/qcom-sdx65-mtp.dts b/arch/arm/boot/dts/qcom/qcom-sdx65-mtp.dts
index 07c10c84eefa1..2c0590617ac46 100644
--- a/arch/arm/boot/dts/qcom/qcom-sdx65-mtp.dts
+++ b/arch/arm/boot/dts/qcom/qcom-sdx65-mtp.dts
@@ -55,7 +55,7 @@ vph_pwr: vph-pwr-regulator {
 		regulator-max-microvolt = <3700000>;
 	};
 
-	vreg_bob_3p3: pmx65_bob {
+	vreg_bob_3p3: pmx65-bob-regulator {
 		compatible = "regulator-fixed";
 		regulator-name = "vreg_bob_3p3";
 		regulator-min-microvolt = <3300000>;
diff --git a/arch/arm/boot/dts/renesas/r8a7740.dtsi b/arch/arm/boot/dts/renesas/r8a7740.dtsi
index c7056b96ec0b7..d4776a7877ffd 100644
--- a/arch/arm/boot/dts/renesas/r8a7740.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a7740.dtsi
@@ -501,7 +501,11 @@ sh_fsi2: sound@fe1f0000 {
 		compatible = "renesas,fsi2-r8a7740", "renesas,sh_fsi2";
 		reg = <0xfe1f0000 0x400>;
 		interrupts = <GIC_SPI 9 0x4>;
-		clocks = <&mstp3_clks R8A7740_CLK_FSI>;
+		clocks = <&mstp3_clks R8A7740_CLK_FSI>, <&spu_clk>,
+			<&fsia_clk>, <&fsib_clk>, <&fsiack_clk>,
+			<&fsibck_clk>;
+		clock-names = "fck", "spu", "icka", "ickb", "xcka",
+				"xckb";
 		power-domains = <&pd_a4mp>;
 		status = "disabled";
 	};
@@ -722,6 +726,12 @@ vou_clk: vou@e6150088 {
 				 <0>;
 			#clock-cells = <0>;
 		};
+		fsib_clk: fsib@e6150090 {
+			compatible = "renesas,r8a7740-div6-clock", "renesas,cpg-div6-clock";
+			reg = <0xe6150090 4>;
+			clocks = <&pllc1_div2_clk>, <&fsibck_clk>, <0>, <0>;
+			#clock-cells = <0>;
+		};
 		stpro_clk: stpro@e615009c {
 			compatible = "renesas,r8a7740-div6-clock", "renesas,cpg-div6-clock";
 			reg = <0xe615009c 4>;
diff --git a/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-eb.dts b/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-eb.dts
index 97a339b30d764..ead379988fb1c 100644
--- a/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-eb.dts
+++ b/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-eb.dts
@@ -53,6 +53,10 @@ led@1 {
 	};
 };
 
+&gpio2 {
+	status = "okay";
+};
+
 &i2c2 {
 	/* Sensors are different across revisions. All are LM75B compatible */
 	sensor@49 {
@@ -152,6 +156,13 @@ pins_sdio1_clk: pins-sdio1-clk {
 		drive-strength = <12>;
 	};
 
+	pins_spi1: pins-spi1 {
+		pinmux = <RZN1_PINMUX(156, RZN1_FUNC_SPI0_M)>,
+			 <RZN1_PINMUX(157, RZN1_FUNC_SPI0_M)>,
+			 <RZN1_PINMUX(158, RZN1_FUNC_SPI0_M)>,
+			 <RZN1_PINMUX(159, RZN1_FUNC_GPIO)>;
+	};
+
 	pins_uart2: pins-uart2 {
 		pinmux = <RZN1_PINMUX(105, RZN1_FUNC_UART2)>,
 			 <RZN1_PINMUX(106, RZN1_FUNC_UART2)>,
@@ -168,6 +179,20 @@ &sdio1 {
 	status = "okay";
 };
 
+&spi1 {
+	pinctrl-0 = <&pins_spi1>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	cs-gpios = <&gpio2a 31 GPIO_ACTIVE_LOW>;
+
+	fram: fram@0 {
+		compatible = "cypress,fm25", "atmel,at25";
+		reg = <0>;
+		spi-max-frequency = <12500000>;
+	};
+};
+
 &switch {
 	pinctrl-0 = <&pins_eth1>, <&pins_eth2>, <&pins_eth3>, <&pins_eth4>,
 		    <&pins_mdio1>;
diff --git a/arch/arm/boot/dts/renesas/r9a06g032.dtsi b/arch/arm/boot/dts/renesas/r9a06g032.dtsi
index 442ea26b40f53..19c9bce0a26d6 100644
--- a/arch/arm/boot/dts/renesas/r9a06g032.dtsi
+++ b/arch/arm/boot/dts/renesas/r9a06g032.dtsi
@@ -563,6 +563,90 @@ gic: interrupt-controller@44101000 {
 				<GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_HIGH)>;
 		};
 
+		/* Controller only */
+		spi1: spi@50005000 {
+			compatible = "renesas,r9a06g032-spi", "renesas,rzn1-spi";
+			reg = <0x50005000 0x200>;
+			interrupts = <GIC_SPI 80 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&sysctrl R9A06G032_CLK_SPI0>, <&sysctrl R9A06G032_HCLK_SPI0>;
+			clock-names = "ssi_clk", "pclk";
+			power-domains = <&sysctrl>;
+			num-cs = <4>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		/* Controller only */
+		spi2: spi@50006000 {
+			compatible = "renesas,r9a06g032-spi", "renesas,rzn1-spi";
+			reg = <0x50006000 0x200>;
+			interrupts = <GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&sysctrl R9A06G032_CLK_SPI1>, <&sysctrl R9A06G032_HCLK_SPI1>;
+			clock-names = "ssi_clk", "pclk";
+			power-domains = <&sysctrl>;
+			num-cs = <4>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		/* Controller only */
+		spi3: spi@50007000 {
+			compatible = "renesas,r9a06g032-spi", "renesas,rzn1-spi";
+			reg = <0x50007000 0x200>;
+			interrupts = <GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&sysctrl R9A06G032_CLK_SPI2>, <&sysctrl R9A06G032_HCLK_SPI2>;
+			clock-names = "ssi_clk", "pclk";
+			power-domains = <&sysctrl>;
+			num-cs = <4>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		/* Controller only */
+		spi4: spi@50008000 {
+			compatible = "renesas,r9a06g032-spi", "renesas,rzn1-spi";
+			reg = <0x50008000 0x200>;
+			interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&sysctrl R9A06G032_CLK_SPI3>, <&sysctrl R9A06G032_HCLK_SPI3>;
+			clock-names = "ssi_clk", "pclk";
+			power-domains = <&sysctrl>;
+			num-cs = <4>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		/* Target only */
+		spi5: spi@50009000 {
+			compatible = "renesas,r9a06g032-spi", "renesas,rzn1-spi";
+			reg = <0x50009000 0x200>;
+			interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&sysctrl R9A06G032_CLK_SPI4>, <&sysctrl R9A06G032_HCLK_SPI4>;
+			clock-names = "ssi_clk", "pclk";
+			power-domains = <&sysctrl>;
+			spi-slave;
+			#address-cells = <0>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		/* Target only */
+		spi6: spi@5000a000 {
+			compatible = "renesas,r9a06g032-spi", "renesas,rzn1-spi";
+			reg = <0x5000a000 0x200>;
+			interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&sysctrl R9A06G032_CLK_SPI5>, <&sysctrl R9A06G032_HCLK_SPI5>;
+			clock-names = "ssi_clk", "pclk";
+			power-domains = <&sysctrl>;
+			spi-slave;
+			#address-cells = <0>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		/*
 		 * The GPIO mapping to the corresponding pins is not obvious.
 		 * See the hardware documentation for details.
diff --git a/arch/arm/boot/dts/rockchip/Makefile b/arch/arm/boot/dts/rockchip/Makefile
index d0154fd7ff24f..e9f9e0ac3bfd4 100644
--- a/arch/arm/boot/dts/rockchip/Makefile
+++ b/arch/arm/boot/dts/rockchip/Makefile
@@ -5,6 +5,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += \
 	rv1108-evb.dtb \
 	rv1109-relfor-saib.dtb \
 	rv1109-sonoff-ihost.dtb \
+	rv1126-alientek-dlrv1126.dtb \
 	rv1126-edgeble-neu2-io.dtb \
 	rv1126-sonoff-ihost.dtb \
 	rk3036-evb.dtb \
diff --git a/arch/arm/boot/dts/rockchip/rk3288-tinker.dtsi b/arch/arm/boot/dts/rockchip/rk3288-tinker.dtsi
index 8e27a20f28454..7e16022c7d9e3 100644
--- a/arch/arm/boot/dts/rockchip/rk3288-tinker.dtsi
+++ b/arch/arm/boot/dts/rockchip/rk3288-tinker.dtsi
@@ -344,6 +344,11 @@ regulator-state-mem {
 
 &i2c2 {
 	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c08";
+		reg = <0x50>;
+	};
 };
 
 &i2c5 {
diff --git a/arch/arm/boot/dts/rockchip/rv1126-alientek-clrv1126f.dtsi b/arch/arm/boot/dts/rockchip/rv1126-alientek-clrv1126f.dtsi
new file mode 100644
index 0000000000000..d7c4230a06dc5
--- /dev/null
+++ b/arch/arm/boot/dts/rockchip/rv1126-alientek-clrv1126f.dtsi
@@ -0,0 +1,291 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2026 Yanan He <grumpycat921013@gmail.com>
+ */
+
+#include "rv1126.dtsi"
+
+/ {
+	compatible = "alientek,clrv1126f", "rockchip,rv1126";
+
+	aliases {
+		mmc0 = &emmc;
+	};
+};
+
+&cpu0 {
+	cpu-supply = <&vdd_arm>;
+};
+
+&cpu1 {
+	cpu-supply = <&vdd_arm>;
+};
+
+&cpu2 {
+	cpu-supply = <&vdd_arm>;
+};
+
+&cpu3 {
+	cpu-supply = <&vdd_arm>;
+};
+
+&emmc {
+	bus-width = <8>;
+	cap-mmc-highspeed;
+	mmc-hs200-1_8v;
+	non-removable;
+	pinctrl-names = "default";
+	pinctrl-0 = <&emmc_bus8 &emmc_cmd &emmc_clk &emmc_rstnout>;
+	rockchip,default-sample-phase = <90>;
+	vmmc-supply = <&vcc_3v3>;
+	vqmmc-supply = <&vcc_1v8>;
+	status = "okay";
+};
+
+&i2c0 {
+	clock-frequency = <400000>;
+	status = "okay";
+
+	rk809: pmic@20 {
+		compatible = "rockchip,rk809";
+		reg = <0x20>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PB1 IRQ_TYPE_LEVEL_LOW>;
+		#clock-cells = <1>;
+		#sound-dai-cells = <0>;
+		clock-output-names = "rk808-clkout1", "rk808-clkout2";
+		clock-names = "mclk";
+		clocks = <&cru MCLK_I2S0_TX_OUT2IO>;
+		assigned-clocks = <&cru MCLK_I2S0_TX_OUT2IO>;
+		assigned-clock-parents = <&cru MCLK_I2S0_TX>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pmic_int_l>;
+		rockchip,system-power-controller;
+		wakeup-source;
+
+		vcc1-supply = <&vcc5v0_sys>;
+		vcc2-supply = <&vcc5v0_sys>;
+		vcc3-supply = <&vcc5v0_sys>;
+		vcc4-supply = <&vcc5v0_sys>;
+		vcc5-supply = <&vcc_buck5>;
+		vcc6-supply = <&vcc_buck5>;
+		vcc7-supply = <&vcc5v0_sys>;
+		vcc8-supply = <&vcc3v3_sys>;
+		vcc9-supply = <&vcc5v0_sys>;
+
+		regulators {
+			vdd_npu_vepu: DCDC_REG1 {
+				regulator-name = "vdd_npu_vepu";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-initial-mode = <0x2>;
+				regulator-min-microvolt = <650000>;
+				regulator-max-microvolt = <950000>;
+				regulator-ramp-delay = <6001>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_arm: DCDC_REG2 {
+				regulator-name = "vdd_arm";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-initial-mode = <0x2>;
+				regulator-min-microvolt = <725000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-ramp-delay = <6001>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_ddr: DCDC_REG3 {
+				regulator-name = "vcc_ddr";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-initial-mode = <0x2>;
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			vcc3v3_sys: DCDC_REG4 {
+				regulator-name = "vcc3v3_sys";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-initial-mode = <0x2>;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <3300000>;
+				};
+			};
+
+			vcc_buck5: DCDC_REG5 {
+				regulator-name = "vcc_buck5";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <2200000>;
+				regulator-max-microvolt = <2200000>;
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <2200000>;
+				};
+			};
+
+			vcc_0v8: LDO_REG1 {
+				regulator-name = "vcc_0v8";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc1v8_pmu: LDO_REG2 {
+				regulator-name = "vcc1v8_pmu";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vdd0v8_pmu: LDO_REG3 {
+				regulator-name = "vcc0v8_pmu";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <800000>;
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <800000>;
+				};
+			};
+
+			vcc_1v8: LDO_REG4 {
+				regulator-name = "vcc_1v8";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vcc_dovdd: LDO_REG5 {
+				regulator-name = "vcc_dovdd";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_dvdd: LDO_REG6 {
+				regulator-name = "vcc_dvdd";
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_avdd: LDO_REG7 {
+				regulator-name = "vcc_avdd";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vccio_sd: LDO_REG8 {
+				regulator-name = "vccio_sd";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc3v3_sd: LDO_REG9 {
+				regulator-name = "vcc3v3_sd";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_5v0: SWITCH_REG1 {
+				regulator-name = "vcc_5v0";
+			};
+
+			vcc_3v3: SWITCH_REG2 {
+				regulator-name = "vcc_3v3";
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+};
+
+&pinctrl {
+	pmic {
+		pmic_int_l: pmic-int-l {
+			rockchip,pins = <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+};
+
+&pmu_io_domains {
+	pmuio0-supply = <&vcc3v3_sys>;
+	pmuio1-supply = <&vcc3v3_sys>;
+	vccio1-supply = <&vcc_1v8>;
+	vccio2-supply = <&vccio_sd>;
+	vccio3-supply = <&vcc_1v8>;
+	vccio4-supply = <&vcc_3v3>;
+	vccio5-supply = <&vcc_3v3>;
+	vccio6-supply = <&vcc_3v3>;
+	vccio7-supply = <&vcc_1v8>;
+	status = "okay";
+};
+
+&saradc {
+	vref-supply = <&vcc_1v8>;
+	status = "okay";
+};
+
+&wdt {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/rockchip/rv1126-alientek-dlrv1126.dts b/arch/arm/boot/dts/rockchip/rv1126-alientek-dlrv1126.dts
new file mode 100644
index 0000000000000..0234b93ceb59e
--- /dev/null
+++ b/arch/arm/boot/dts/rockchip/rv1126-alientek-dlrv1126.dts
@@ -0,0 +1,256 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2026 Yanan He <grumpycat921013@gmail.com>
+ */
+
+/dts-v1/;
+#include <dt-bindings/input/input.h>
+#include "rv1126-alientek-clrv1126f.dtsi"
+
+/ {
+	model = "Alientek ATK-DLRV1126";
+	compatible = "alientek,dlrv1126", "alientek,clrv1126f", "rockchip,rv1126";
+
+	aliases {
+		ethernet0 = &gmac;
+		mmc1 = &sdio;
+		mmc2 = &sdmmc;
+	};
+
+	chosen {
+		stdout-path = "serial2:1500000n8";
+	};
+
+	adc-keys {
+		compatible = "adc-keys";
+		io-channels = <&saradc 0>;
+		io-channel-names = "buttons";
+		keyup-threshold-microvolt = <1800000>;
+		poll-interval = <100>;
+
+		button-esc {
+			label = "esc";
+			linux,code = <KEY_ESC>;
+			press-threshold-microvolt = <0>;
+		};
+
+		button-left {
+			label = "left";
+			linux,code = <KEY_LEFT>;
+			press-threshold-microvolt = <801562>;
+		};
+
+		button-menu {
+			label = "menu";
+			linux,code = <KEY_MENU>;
+			press-threshold-microvolt = <1198828>;
+		};
+
+		button-right {
+			label = "right";
+			linux,code = <KEY_RIGHT>;
+			press-threshold-microvolt = <400781>;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-0 {
+			label = "sys-led";
+			gpios = <&gpio3 RK_PD4 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+			default-state = "on";
+		};
+
+		led-1 {
+			label = "user-led";
+			gpios = <&gpio3 RK_PD6 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "none";
+			default-state = "on";
+		};
+	};
+
+	sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,name = "Analog RK809";
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,widgets =
+			"Speaker", "Speaker",
+			"Headphone", "Headphones",
+			"Microphone", "Mic Jack";
+		simple-audio-card,routing =
+			"Speaker", "SPKO",
+			"Headphones", "HPOL",
+			"Headphones", "HPOR",
+			"MICL", "Mic Jack";
+
+		simple-audio-card,cpu {
+			sound-dai = <&i2s0>;
+		};
+
+		simple-audio-card,codec {
+			sound-dai = <&rk809>;
+		};
+	};
+
+	vcc5v0_sys: regulator-vcc5v0-sys {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc5v0_sys";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+	};
+
+	sdio_pwrseq: pwrseq-sdio {
+		compatible = "mmc-pwrseq-simple";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wifi_enable_h>;
+		reset-gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_LOW>;
+		post-power-on-delay-ms = <200>;
+		power-off-delay-us = <20000>;
+	};
+};
+
+&gmac {
+	assigned-clocks = <&cru CLK_GMAC_SRC>, <&cru CLK_GMAC_TX_RX>,
+			  <&cru CLK_GMAC_ETHERNET_OUT>;
+	assigned-clock-parents = <&cru CLK_GMAC_SRC_M1>,
+				 <&cru RGMII_MODE_CLK>;
+	assigned-clock-rates = <125000000>, <0>, <25000000>;
+	clock_in_out = "input";
+	phy-handle = <&phy>;
+	phy-mode = "rgmii-id";
+	pinctrl-names = "default";
+	pinctrl-0 = <&rgmiim1_miim &rgmiim1_bus2 &rgmiim1_bus4
+		     &clk_out_ethernetm1_pins>;
+	status = "okay";
+};
+
+&i2c5 {
+	status = "okay";
+	clock-frequency = <400000>;
+
+	pcf8563: rtc@51 {
+		compatible = "nxp,pcf8563";
+		reg = <0x51>;
+		#clock-cells = <0>;
+		clock-output-names = "xin32k";
+		interrupt-parent = <&gpio1>;
+		interrupts = <RK_PD0 IRQ_TYPE_LEVEL_LOW>;
+	};
+};
+
+&i2s0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2s0m0_sclk_tx>,
+		    <&i2s0m0_mclk>,
+		    <&i2s0m0_lrck_tx>,
+		    <&i2s0m0_sdo0>,
+		    <&i2s0m0_sdo1_sdi3>;
+	rockchip,trcm-sync-tx-only;
+	rockchip,i2s-rx-route = <3 1 2 0>;
+	rockchip,i2s-tx-route = <0 1 2 3>;
+	status = "okay";
+};
+
+&mdio {
+	phy: ethernet-phy@1 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <0x1>;
+		clocks = <&cru CLK_GMAC_ETHERNET_OUT>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&eth_phy_rst>;
+		reset-gpios = <&gpio3 RK_PA0 GPIO_ACTIVE_LOW>;
+		reset-assert-us = <20000>;
+		reset-deassert-us = <100000>;
+	};
+};
+
+&pinctrl {
+	bt {
+		bt_enable: bt-enable {
+			rockchip,pins = <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		bt_wake_dev: bt-wake-dev {
+			rockchip,pins = <1 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		bt_wake_host: bt-wake-host {
+			rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	ethernet {
+		eth_phy_rst: eth-phy-rst {
+			rockchip,pins = <3 RK_PA0 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+	};
+
+	wifi {
+		wifi_enable_h: wifi-enable-h {
+			rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+};
+
+&sdio {
+	bus-width = <4>;
+	cap-sdio-irq;
+	keep-power-in-suspend;
+	max-frequency = <25000000>;
+	mmc-pwrseq = <&sdio_pwrseq>;
+	non-removable;
+	pinctrl-names = "default";
+	pinctrl-0 = <&sdmmc1_clk &sdmmc1_cmd &sdmmc1_bus4>;
+	rockchip,default-sample-phase = <90>;
+	vmmc-supply = <&vcc3v3_sd>;
+	vqmmc-supply = <&vcc_1v8>;
+	status = "okay";
+};
+
+&sdmmc {
+	bus-width = <4>;
+	cap-mmc-highspeed;
+	cap-sd-highspeed;
+	card-detect-delay = <200>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&sdmmc0_clk &sdmmc0_cmd &sdmmc0_bus4 &sdmmc0_det>;
+	rockchip,default-sample-phase = <90>;
+	sd-uhs-sdr12;
+	sd-uhs-sdr25;
+	sd-uhs-sdr104;
+	vmmc-supply = <&vcc3v3_sd>;
+	vqmmc-supply = <&vccio_sd>;
+	status = "okay";
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_xfer &uart0_ctsn &uart0_rtsn>;
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "brcm,bcm43430a1-bt";
+		clocks = <&rk809 1>;
+		clock-names = "lpo";
+		device-wakeup-gpios = <&gpio1 RK_PD1 GPIO_ACTIVE_HIGH>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PA5 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "host-wakeup";
+		max-speed = <115200>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&bt_enable>, <&bt_wake_dev>, <&bt_wake_host>;
+		shutdown-gpios = <&gpio0 RK_PA7 GPIO_ACTIVE_HIGH>;
+		vbat-supply = <&vcc_3v3>;
+		vddio-supply = <&vcc_1v8>;
+	};
+};
+
+&uart2 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/rockchip/rv1126-pinctrl.dtsi b/arch/arm/boot/dts/rockchip/rv1126-pinctrl.dtsi
index 35ef6732281fd..1d883b80aed43 100644
--- a/arch/arm/boot/dts/rockchip/rv1126-pinctrl.dtsi
+++ b/arch/arm/boot/dts/rockchip/rv1126-pinctrl.dtsi
@@ -123,6 +123,16 @@ i2c3m2_xfer: i2c3m2-xfer {
 				<1 RK_PD7 3 &pcfg_pull_none>;
 		};
 	};
+	i2c5 {
+		/omit-if-no-ref/
+		i2c5m0_xfer: i2c5m0-xfer {
+			rockchip,pins =
+				/* i2c5_scl_m0 */
+				<2 RK_PA5 7 &pcfg_pull_none_drv_level_0_smt>,
+				/* i2c5_sda_m0 */
+				<2 RK_PB3 7 &pcfg_pull_none_drv_level_0_smt>;
+		};
+	};
 	i2s0 {
 		i2s0m0_lrck_tx: i2s0m0-lrck-tx {
 			rockchip,pins =
diff --git a/arch/arm/boot/dts/rockchip/rv1126.dtsi b/arch/arm/boot/dts/rockchip/rv1126.dtsi
index d6e8b63daa420..d0cdc5f742120 100644
--- a/arch/arm/boot/dts/rockchip/rv1126.dtsi
+++ b/arch/arm/boot/dts/rockchip/rv1126.dtsi
@@ -23,6 +23,7 @@ aliases {
 		i2c0 = &i2c0;
 		i2c2 = &i2c2;
 		i2c3 = &i2c3;
+		i2c5 = &i2c5;
 		serial0 = &uart0;
 		serial1 = &uart1;
 		serial2 = &uart2;
@@ -400,6 +401,20 @@ i2c3: i2c@ff520000 {
 		status = "disabled";
 	};
 
+	i2c5: i2c@ff540000 {
+		compatible = "rockchip,rv1126-i2c", "rockchip,rk3399-i2c";
+		reg = <0xff540000 0x1000>;
+		interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&cru CLK_I2C5>, <&cru PCLK_I2C5>;
+		clock-names = "i2c", "pclk";
+		pinctrl-names = "default";
+		pinctrl-0 = <&i2c5m0_xfer>;
+		rockchip,grf = <&pmugrf>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		status = "disabled";
+	};
+
 	pwm8: pwm@ff550000 {
 		compatible = "rockchip,rv1126-pwm", "rockchip,rk3328-pwm";
 		reg = <0xff550000 0x10>;
diff --git a/arch/arm/boot/dts/samsung/exynos5250-manta.dts b/arch/arm/boot/dts/samsung/exynos5250-manta.dts
index 24a27b342227b..76d3657eb22f0 100644
--- a/arch/arm/boot/dts/samsung/exynos5250-manta.dts
+++ b/arch/arm/boot/dts/samsung/exynos5250-manta.dts
@@ -461,6 +461,13 @@ acc_int: acc-int-pins {
 		samsung,pin-pud = <EXYNOS_PIN_PULL_UP>;
 	};
 
+	bt_host_wakeup: bt-host-wakeup-pins {
+		samsung,pins = "gpx2-6";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_INPUT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS4_PIN_DRV_LV1>;
+	};
+
 	max77686_irq: max77686-irq-pins {
 		samsung,pins = "gpx0-2";
 		samsung,pin-function = <EXYNOS_PIN_FUNC_F>;
@@ -488,6 +495,20 @@ bh1721fvc_reset: bh1721fvc-reset-pins {
 		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
 	};
 
+	bt_reg_on: bt-reg-on-pins {
+		samsung,pins = "gph0-0";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_OUTPUT>;
+		samsung,pin-con-pdn = <EXYNOS_PIN_PDN_PREV>;
+		samsung,pin-pud-pdn = <EXYNOS_PIN_PULL_NONE>;
+	};
+
+	bt_wake: bt-wake-pins {
+		samsung,pins = "gph1-3";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_OUTPUT>;
+		samsung,pin-con-pdn = <EXYNOS_PIN_PDN_PREV>;
+		samsung,pin-pud-pdn = <EXYNOS_PIN_PULL_NONE>;
+	};
+
 	msense_reset: msense-reset-pins {
 		samsung,pins = "gpg2-0";
 		samsung,pin-function = <EXYNOS_PIN_FUNC_OUTPUT>;
@@ -536,7 +557,25 @@ &sd1_cmd {
 
 /* Bluetooth */
 &serial_0 {
-	status = "disabled";
+	pinctrl-0 = <&uart0_data &uart0_fctl>;
+	pinctrl-names = "default";
+
+	bluetooth {
+		compatible = "brcm,bcm4330-bt";
+
+		pinctrl-0 = <&bt_reg_on &bt_wake &bt_host_wakeup>;
+		pinctrl-names = "default";
+
+		shutdown-gpios = <&gph0 0 GPIO_ACTIVE_HIGH>;
+		device-wakeup-gpios = <&gph1 3 GPIO_ACTIVE_HIGH>;
+
+		interrupt-parent = <&gpx2>;
+		interrupts = <6 IRQ_TYPE_EDGE_FALLING>;
+		interrupt-names = "host-wakeup";
+
+		clocks = <&max77686 MAX77686_CLK_PMIC>;
+		clock-names = "lpo";
+	};
 };
 
 /* GPS */
diff --git a/arch/arm/boot/dts/st/spear1340-evb.dts b/arch/arm/boot/dts/st/spear1340-evb.dts
index 9e7c356b1d9e1..133015a5dd463 100644
--- a/arch/arm/boot/dts/st/spear1340-evb.dts
+++ b/arch/arm/boot/dts/st/spear1340-evb.dts
@@ -259,7 +259,7 @@ sound {
 			stream_name = "spdif-cap", "spdif-play", "i2s-play", "i2s-cap";
 			dai_name = "spdifin-pcm", "spdifout-pcm", "i2s0-pcm", "i2s1-pcm";
 			nr_controllers = <4>;
-		        status = "okay";
+			status = "okay";
 		};
 
 		spdif0: spdif-in@d0100000 {
diff --git a/arch/arm/boot/dts/st/spear13xx.dtsi b/arch/arm/boot/dts/st/spear13xx.dtsi
index 0bb88f2d4ef5f..cf98160ba268f 100644
--- a/arch/arm/boot/dts/st/spear13xx.dtsi
+++ b/arch/arm/boot/dts/st/spear13xx.dtsi
@@ -178,7 +178,6 @@ usb@e4800000 {
 			compatible = "st,spear600-ehci", "usb-ehci";
 			reg = <0xe4800000 0x1000>;
 			interrupts = <0 64 0x4>;
-			usbh0_id = <0>;
 			status = "disabled";
 		};
 
@@ -186,7 +185,6 @@ usb@e5800000 {
 			compatible = "st,spear600-ehci", "usb-ehci";
 			reg = <0xe5800000 0x1000>;
 			interrupts = <0 66 0x4>;
-			usbh1_id = <1>;
 			status = "disabled";
 		};
 
@@ -194,7 +192,6 @@ usb@e4000000 {
 			compatible = "st,spear600-ohci", "usb-ohci";
 			reg = <0xe4000000 0x1000>;
 			interrupts = <0 65 0x4>;
-			usbh0_id = <0>;
 			status = "disabled";
 		};
 
@@ -202,7 +199,6 @@ usb@e5000000 {
 			compatible = "st,spear600-ohci", "usb-ohci";
 			reg = <0xe5000000 0x1000>;
 			interrupts = <0 67 0x4>;
-			usbh1_id = <1>;
 			status = "disabled";
 		};
 
diff --git a/arch/arm/boot/dts/st/spear320.dtsi b/arch/arm/boot/dts/st/spear320.dtsi
index 56f141297ea35..cbf494419f43d 100644
--- a/arch/arm/boot/dts/st/spear320.dtsi
+++ b/arch/arm/boot/dts/st/spear320.dtsi
@@ -82,7 +82,7 @@ pwm: pwm@a8000000 {
 			reg = <0xa8000000 0x1000>;
 			#pwm-cells = <2>;
 			status = "disabled";
-                };
+		};
 
 		apb {
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/st/ste-dbx5x0.dtsi b/arch/arm/boot/dts/st/ste-dbx5x0.dtsi
index 0f87abeddc335..25a20fafc6b96 100644
--- a/arch/arm/boot/dts/st/ste-dbx5x0.dtsi
+++ b/arch/arm/boot/dts/st/ste-dbx5x0.dtsi
@@ -1119,7 +1119,7 @@ msp2: msp@80117000 {
 
 			dmas = <&dma 14 0 0x12>, /* Logical  - DevToMem - HighPrio */
 			       <&dma 14 1 0x19>; /* Physical Chan 1 - MemToDev
-                                                    HighPrio - Fixed */
+						    HighPrio - Fixed */
 			dma-names = "rx", "tx";
 
 			clocks = <&prcc_kclk 2 3>, <&prcc_pclk 2 5>;
diff --git a/arch/arm/boot/dts/st/ste-href.dtsi b/arch/arm/boot/dts/st/ste-href.dtsi
index fbf0309e108f0..a85d876656107 100644
--- a/arch/arm/boot/dts/st/ste-href.dtsi
+++ b/arch/arm/boot/dts/st/ste-href.dtsi
@@ -207,7 +207,7 @@ mmc@80005000 {
 		// On-board eMMC
 		mmc@80114000 {
 			arm,primecell-periphid = <0x10480180>;
-		        max-frequency = <100000000>;
+			max-frequency = <100000000>;
 			bus-width = <8>;
 			cap-mmc-highspeed;
 			non-removable;
diff --git a/arch/arm/boot/dts/st/ste-snowball.dts b/arch/arm/boot/dts/st/ste-snowball.dts
index 1322abfc7acfb..14ed1079ff6cb 100644
--- a/arch/arm/boot/dts/st/ste-snowball.dts
+++ b/arch/arm/boot/dts/st/ste-snowball.dts
@@ -295,7 +295,7 @@ mmc@80005000 {
 		// On-board eMMC
 		mmc@80114000 {
 			arm,primecell-periphid = <0x10480180>;
-		        max-frequency = <100000000>;
+			max-frequency = <100000000>;
 			bus-width = <8>;
 			cap-mmc-highspeed;
 			no-sdio;
diff --git a/arch/arm/boot/dts/st/ste-ux500-samsung-gavini.dts b/arch/arm/boot/dts/st/ste-ux500-samsung-gavini.dts
index ad9a20ccaaebe..0c3eef7a294a4 100644
--- a/arch/arm/boot/dts/st/ste-ux500-samsung-gavini.dts
+++ b/arch/arm/boot/dts/st/ste-ux500-samsung-gavini.dts
@@ -402,7 +402,7 @@ wifi@1 {
 		/* eMMC */
 		mmc@80005000 {
 			arm,primecell-periphid = <0x10480180>;
-		        max-frequency = <50000000>;
+			max-frequency = <50000000>;
 			bus-width = <8>;
 			non-removable;
 			cap-mmc-highspeed;
diff --git a/arch/arm/boot/dts/st/ste-ux500-samsung-janice.dts b/arch/arm/boot/dts/st/ste-ux500-samsung-janice.dts
index 64562a3a262c4..18fb7263058ed 100644
--- a/arch/arm/boot/dts/st/ste-ux500-samsung-janice.dts
+++ b/arch/arm/boot/dts/st/ste-ux500-samsung-janice.dts
@@ -452,7 +452,7 @@ wifi@1 {
 		/* eMMC */
 		mmc@80005000 {
 			arm,primecell-periphid = <0x10480180>;
-		        max-frequency = <50000000>;
+			max-frequency = <50000000>;
 			bus-width = <8>;
 			non-removable;
 			cap-mmc-highspeed;
diff --git a/arch/arm/boot/dts/st/ste-ux500-samsung-kyle.dts b/arch/arm/boot/dts/st/ste-ux500-samsung-kyle.dts
index cdb147dcc1dbc..c34d43a2b7506 100644
--- a/arch/arm/boot/dts/st/ste-ux500-samsung-kyle.dts
+++ b/arch/arm/boot/dts/st/ste-ux500-samsung-kyle.dts
@@ -274,7 +274,7 @@ wifi@1 {
 		 */
 		mmc@80005000 {
 			arm,primecell-periphid = <0x10480180>;
-		        max-frequency = <100000000>;
+			max-frequency = <100000000>;
 			bus-width = <8>;
 			non-removable;
 			cap-mmc-highspeed;
diff --git a/arch/arm/boot/dts/st/ste-ux500-samsung-skomer.dts b/arch/arm/boot/dts/st/ste-ux500-samsung-skomer.dts
index 064d6fee88210..7bf4287f6e138 100644
--- a/arch/arm/boot/dts/st/ste-ux500-samsung-skomer.dts
+++ b/arch/arm/boot/dts/st/ste-ux500-samsung-skomer.dts
@@ -253,7 +253,7 @@ wifi@1 {
 		 */
 		mmc@80005000 {
 			arm,primecell-periphid = <0x10480180>;
-		        max-frequency = <100000000>;
+			max-frequency = <100000000>;
 			bus-width = <8>;
 			non-removable;
 			cap-mmc-highspeed;
diff --git a/arch/arm/boot/dts/st/stih407-family.dtsi b/arch/arm/boot/dts/st/stih407-family.dtsi
index 3e6a0542e3aea..08acba209c564 100644
--- a/arch/arm/boot/dts/st/stih407-family.dtsi
+++ b/arch/arm/boot/dts/st/stih407-family.dtsi
@@ -596,7 +596,7 @@ spi@9542000 {
 			status = "disabled";
 		};
 
-		mmc0: sdhci@9060000 {
+		mmc0: mmc@9060000 {
 			compatible = "st,sdhci-stih407", "st,sdhci";
 			status = "disabled";
 			reg = <0x09060000 0x7ff>, <0x9061008 0x20>;
@@ -611,7 +611,7 @@ mmc0: sdhci@9060000 {
 			bus-width = <8>;
 		};
 
-		mmc1: sdhci@9080000 {
+		mmc1: mmc@9080000 {
 			compatible = "st,sdhci-stih407", "st,sdhci";
 			status = "disabled";
 			reg = <0x09080000 0x7ff>;
diff --git a/arch/arm/boot/dts/st/stih410-b2260.dts b/arch/arm/boot/dts/st/stih410-b2260.dts
index 736b1e059b0a8..44048a98648e8 100644
--- a/arch/arm/boot/dts/st/stih410-b2260.dts
+++ b/arch/arm/boot/dts/st/stih410-b2260.dts
@@ -141,14 +141,14 @@ i2c3: i2c@9843000 {
 			status = "okay";
 		};
 
-		mmc0: sdhci@9060000 {
+		mmc0: mmc@9060000 {
 			pinctrl-0 = <&pinctrl_sd0>;
 			bus-width = <4>;
 			status = "okay";
 		};
 
 		/* high speed expansion connector */
-		mmc1: sdhci@9080000 {
+		mmc1: mmc@9080000 {
 			status = "okay";
 		};
 
diff --git a/arch/arm/boot/dts/st/stih418-b2199.dts b/arch/arm/boot/dts/st/stih418-b2199.dts
index 53ac6c2b7b7d5..4ea1745973653 100644
--- a/arch/arm/boot/dts/st/stih418-b2199.dts
+++ b/arch/arm/boot/dts/st/stih418-b2199.dts
@@ -82,11 +82,11 @@ i2c@9541000 {
 			st,i2c-min-sda-pulse-width-us = <5>;
 		};
 
-		mmc1: sdhci@9080000 {
+		mmc1: mmc@9080000 {
 			status = "okay";
 		};
 
-		mmc0: sdhci@9060000 {
+		mmc0: mmc@9060000 {
 			status = "okay";
 			max-frequency = <200000000>;
 			sd-uhs-sdr50;
diff --git a/arch/arm/boot/dts/st/stih418.dtsi b/arch/arm/boot/dts/st/stih418.dtsi
index 8fb8b3af5e495..1a5fbbedc8cdd 100644
--- a/arch/arm/boot/dts/st/stih418.dtsi
+++ b/arch/arm/boot/dts/st/stih418.dtsi
@@ -136,7 +136,7 @@ ehci1: usb@9a83e00 {
 			phy-names = "usb";
 		};
 
-		mmc0: sdhci@9060000 {
+		mmc0: mmc@9060000 {
 			assigned-clocks = <&clk_s_c0_flexgen CLK_MMC_0>;
 			assigned-clock-parents = <&clk_s_c0_pll1 0>;
 			assigned-clock-rates = <200000000>;
diff --git a/arch/arm/boot/dts/ti/davinci/da850-lcdk.dts b/arch/arm/boot/dts/ti/davinci/da850-lcdk.dts
index 8390d71b000ab..e70edd5274fe3 100644
--- a/arch/arm/boot/dts/ti/davinci/da850-lcdk.dts
+++ b/arch/arm/boot/dts/ti/davinci/da850-lcdk.dts
@@ -366,7 +366,7 @@ nand@2000000,0 {
 			 * to NAND block 1 (NAND block 0 is not used by default)".
 			 * The same doc mentions that for ROM "Silicon Revision 2.1",
 			 * "Updated NAND boot mode to offer boot from block 0 or block 1".
-			 * However the limitaion is left here by default for compatibility
+			 * However the limitation is left here by default for compatibility
 			 * with older silicon and because it needs new boot pin settings
 			 * not possible in stock LCDK.
 			 */
diff --git a/arch/arm/boot/dts/ti/omap/Makefile b/arch/arm/boot/dts/ti/omap/Makefile
index 3a4d9204339b9..498c36ccb5ea6 100644
--- a/arch/arm/boot/dts/ti/omap/Makefile
+++ b/arch/arm/boot/dts/ti/omap/Makefile
@@ -88,6 +88,9 @@ dtb-$(CONFIG_ARCH_OMAP4) += \
 am335x-bonegreen-hdmi-00a0-dtbs := am335x-bonegreen-eco.dtb \
 	am335x-bone-hdmi-00a0.dtbo
 
+am335x-icev2-prueth-dtbs := am335x-icev2.dtb \
+	am335x-icev2-prueth-overlay.dtbo
+
 dtb-$(CONFIG_SOC_AM33XX) += \
 	am335x-baltos-ir2110.dtb \
 	am335x-baltos-ir3220.dtb \
@@ -106,6 +109,7 @@ dtb-$(CONFIG_SOC_AM33XX) += \
 	am335x-evmsk.dtb \
 	am335x-guardian.dtb \
 	am335x-icev2.dtb \
+	am335x-icev2-prueth.dtb \
 	am335x-lxm.dtb \
 	am335x-mba335x.dtb \
 	am335x-moxa-uc-2101.dtb \
diff --git a/arch/arm/boot/dts/ti/omap/am335x-icev2-prueth-overlay.dtso b/arch/arm/boot/dts/ti/omap/am335x-icev2-prueth-overlay.dtso
new file mode 100644
index 0000000000000..c0f32c889f69d
--- /dev/null
+++ b/arch/arm/boot/dts/ti/omap/am335x-icev2-prueth-overlay.dtso
@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * DT overlay for IDK AM335x
+ *
+ * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+/*
+ * AM335x ICE V2 board
+ * http://www.ti.com/tool/tmdsice3359
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/bus/ti-sysc.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/pinctrl/am33xx.h>
+#include <dt-bindings/clock/am3.h>
+
+&{/} {
+	/* Dual-MAC Ethernet application node on PRU-ICSS */
+	pruss_eth: pruss-eth {
+		compatible = "ti,am3359-prueth";
+		ti,prus = <&pru0>, <&pru1>;
+		sram = <&ocmcram>;
+		ti,mii-rt = <&pruss_mii_rt>;
+		ti,iep = <&pruss_iep>;
+		ti,ecap = <&pruss_ecap>;
+		interrupts = <20 2 2>, <21 3 3>;
+		interrupt-names = "rx_hp", "rx_lp";
+		interrupt-parent = <&pruss_intc>;
+
+		pinctrl-0 = <&pruss_eth_default>;
+		pinctrl-names = "default";
+
+		ethernet-ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pruss_emac0: ethernet-port@0 {
+				reg = <0>;
+				phy-handle = <&pruss_eth0_phy>;
+				phy-mode = "mii";
+				interrupts = <20 2 2>, <26 6 6>, <23 6 6>;
+				interrupt-names = "rx", "emac_ptp_tx",
+						  "hsr_ptp_tx";
+				/* Filled in by bootloader */
+				local-mac-address = [00 00 00 00 00 00];
+			};
+
+			pruss_emac1: ethernet-port@1 {
+				reg = <1>;
+				phy-handle = <&pruss_eth1_phy>;
+				phy-mode = "mii";
+				interrupts = <21 3 3>, <27 9 7>, <24 9 7>;
+				interrupt-names = "rx", "emac_ptp_tx",
+						  "hsr_ptp_tx";
+				/* Filled in by bootloader */
+				local-mac-address = [00 00 00 00 00 00];
+			};
+		};
+	};
+};
+
+&am33xx_pinmux {
+	/* MDIO node for PRU-ICSS */
+	pruss_mdio_default: pruss-mdio-default-pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x88c, PIN_OUTPUT | MUX_MODE5) /* (V12) gpmc_clk.pr1_mdio_mdclk */
+			AM33XX_IOPAD(0x888, PIN_INPUT | MUX_MODE5) /* (T13) gpmc_csn3.pr1_mdio_data */
+		>;
+	};
+
+	/* Pinmux configuration for PRU-ICSS */
+	pruss_eth_default: pruss-eth-default-pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x8a0, PIN_INPUT | MUX_MODE2) /* (R1) lcd_data0.pr1_mii_mt0_clk */
+			AM33XX_IOPAD(0x8b4, PIN_OUTPUT | MUX_MODE2) /* (T2) lcd_data5.pr1_mii0_txd0 */
+			AM33XX_IOPAD(0x8b0, PIN_OUTPUT | MUX_MODE2) /* (T1) lcd_data4.pr1_mii0_txd1 */
+			AM33XX_IOPAD(0x8ac, PIN_OUTPUT | MUX_MODE2) /* (R4) lcd_data3.pr1_mii0_txd2 */
+			AM33XX_IOPAD(0x8a8, PIN_OUTPUT | MUX_MODE2) /* (R3) lcd_data2.pr1_mii0_txd3 */
+			AM33XX_IOPAD(0x8cc, PIN_INPUT | MUX_MODE5) /* (U4) lcd_data11.pr1_mii0_rxd0 */
+			AM33XX_IOPAD(0x8c8, PIN_INPUT | MUX_MODE5) /* (U3) lcd_data10.pr1_mii0_rxd1 */
+			AM33XX_IOPAD(0x8c4, PIN_INPUT | MUX_MODE5) /* (U2) lcd_data9.pr1_mii0_rxd2 */
+			AM33XX_IOPAD(0x8c0, PIN_INPUT | MUX_MODE5) /* (U1) lcd_data8.pr1_mii0_rxd3 */
+			AM33XX_IOPAD(0x8a4, PIN_OUTPUT | MUX_MODE2) /* (R2) lcd_data1.pr1_mii0_txen */
+			AM33XX_IOPAD(0x8d8, PIN_INPUT | MUX_MODE5) /* (V4) lcd_data14.pr1_mii_mr0_clk */
+			AM33XX_IOPAD(0x8dc, PIN_INPUT | MUX_MODE5) /* (T5) lcd_data15.pr1_mii0_rxdv */
+			AM33XX_IOPAD(0x8d4, PIN_INPUT | MUX_MODE5) /* (V3) lcd_data13.pr1_mii0_rxer */
+			AM33XX_IOPAD(0x8d0, PIN_INPUT | MUX_MODE5) /* (V2) lcd_data12.pr1_mii0_rxlink */
+			AM33XX_IOPAD(0x8e8, PIN_INPUT | MUX_MODE2) /* (V5) lcd_pclk.pr1_mii0_crs */
+
+			AM33XX_IOPAD(0x840, PIN_INPUT | MUX_MODE5) /* (R13) gpmc_a0.pr1_mii_mt1_clk */
+			AM33XX_IOPAD(0x850, PIN_OUTPUT | MUX_MODE5) /* (R14) gpmc_a4.pr1_mii1_txd0 */
+			AM33XX_IOPAD(0x84c, PIN_OUTPUT | MUX_MODE5) /* (T14) gpmc_a3.pr1_mii1_txd1 */
+			AM33XX_IOPAD(0x848, PIN_OUTPUT | MUX_MODE5) /* (U14) gpmc_a2.pr1_mii1_txd2 */
+			AM33XX_IOPAD(0x844, PIN_OUTPUT | MUX_MODE5) /* (V14) gpmc_a1.pr1_mii1_txd3 */
+			AM33XX_IOPAD(0x860, PIN_INPUT | MUX_MODE5) /* (V16) gpmc_a8.pr1_mii1_rxd0 */
+			AM33XX_IOPAD(0x85c, PIN_INPUT | MUX_MODE5) /* (T15) gpmc_a7.pr1_mii1_rxd1 */
+			AM33XX_IOPAD(0x858, PIN_INPUT | MUX_MODE5) /* (U15) gpmc_a6.pr1_mii1_rxd2 */
+			AM33XX_IOPAD(0x854, PIN_INPUT | MUX_MODE5) /* (V15) gpmc_a5.pr1_mii1_rxd3 */
+			AM33XX_IOPAD(0x874, PIN_OUTPUT | MUX_MODE5) /* (U17) gpmc_wpn.pr1_mii1_txen */
+			AM33XX_IOPAD(0x864, PIN_INPUT | MUX_MODE5) /* (U16) gpmc_a9.pr1_mii_mr1_clk */
+			AM33XX_IOPAD(0x868, PIN_INPUT | MUX_MODE5) /* (T16) gpmc_a10.pr1_mii1_rxdv */
+			AM33XX_IOPAD(0x86c, PIN_INPUT | MUX_MODE5) /* (V17) gpmc_a11.pr1_mii1_rxer */
+			AM33XX_IOPAD(0x878, PIN_INPUT | MUX_MODE5) /* (U18) gpmc_be1n.pr1_mii1_rxlink */
+			AM33XX_IOPAD(0x8ec, PIN_INPUT | MUX_MODE2) /* (R6) lcd_ac_bias_en.pr1_mii1_crs */
+		>;
+	};
+};
+
+&gpio3 {
+	mux-mii-hog {
+		status = "disabled";
+	};
+
+	mux-mii-hog-0 {
+		gpio-hog;
+		gpios = <10 GPIO_ACTIVE_HIGH>;
+		/* ETH1 mux: Low for MII-PRU, high for RMII-CPSW */
+		output-low;
+		line-name = "MUX_MII_CTL1";
+	};
+};
+
+/*
+ * Disable CPSW switch node and
+ * MDIO configuration to prevent
+ * conflict with PRU-ICSS
+ */
+&mac_sw {
+	status = "disabled";
+};
+
+&davinci_mdio_sw {
+	status = "disabled";
+};
+
+/* PRU-ICSS MDIO configuration */
+&pruss_mdio {
+	pinctrl-0 = <&pruss_mdio_default>;
+	pinctrl-names = "default";
+	reset-gpios = <&gpio2 5 GPIO_ACTIVE_LOW>;
+	reset-delay-us = <2>; /* PHY datasheet states 1uS min */
+	status = "okay";
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	pruss_eth0_phy: ethernet-phy@1 {
+		reg = <1>;
+	};
+
+	pruss_eth1_phy: ethernet-phy@3 {
+		reg = <3>;
+	};
+};
diff --git a/arch/arm/boot/dts/ti/omap/am33xx-l4.dtsi b/arch/arm/boot/dts/ti/omap/am33xx-l4.dtsi
index 89d16fcc773e0..57a45609f9f5c 100644
--- a/arch/arm/boot/dts/ti/omap/am33xx-l4.dtsi
+++ b/arch/arm/boot/dts/ti/omap/am33xx-l4.dtsi
@@ -308,7 +308,7 @@ am33xx_pinmux: pinmux@800 {
 				};
 
 				scm_conf: scm_conf@0 {
-					compatible = "syscon", "simple-bus";
+					compatible = "ti,am3352-scm-conf", "syscon", "simple-bus";
 					reg = <0x0 0x800>;
 					#address-cells = <1>;
 					#size-cells = <1>;
@@ -896,6 +896,17 @@ pruss_mii_rt: mii-rt@32000 {
 					reg = <0x32000 0x58>;
 				};
 
+				pruss_iep: iep@2e000 {
+					compatible = "ti,am3356-icss-iep";
+					reg = <0x2e000 0x31c>;
+					clocks = <&pruss_iepclk_mux>;
+				};
+
+				pruss_ecap: ecap@30000 {
+					compatible = "ti,pruss-ecap";
+					reg = <0x30000 0x60>;
+				};
+
 				pruss_intc: interrupt-controller@20000 {
 					compatible = "ti,pruss-intc";
 					reg = <0x20000 0x2000>;
@@ -1859,7 +1870,7 @@ SYSC_OMAP2_SOFTRESET |
 
 			gpio2: gpio@0 {
 				compatible = "ti,omap4-gpio";
-                                gpio-ranges =	<&am33xx_pinmux  0 34 18>,
+				gpio-ranges =	<&am33xx_pinmux  0 34 18>,
 						<&am33xx_pinmux 18 77  4>,
 						<&am33xx_pinmux 22 56 10>;
 				gpio-controller;
diff --git a/arch/arm/boot/dts/ti/omap/am3517-evm.dts b/arch/arm/boot/dts/ti/omap/am3517-evm.dts
index 40f15da810435..a4357fa9f006a 100644
--- a/arch/arm/boot/dts/ti/omap/am3517-evm.dts
+++ b/arch/arm/boot/dts/ti/omap/am3517-evm.dts
@@ -213,7 +213,7 @@ &i2c2 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&i2c2_pins>;
 	clock-frequency = <400000>;
-	/* User DIP swithes [1:8] / User LEDS [1:2] */
+	/* User DIP switches [1:8] / User LEDS [1:2] */
 	tca6416: gpio@21 {
 		compatible = "ti,tca6416";
 		reg = <0x21>;
diff --git a/arch/arm/boot/dts/ti/omap/am4372.dtsi b/arch/arm/boot/dts/ti/omap/am4372.dtsi
index 504fa6b57d39b..494f251c8e6af 100644
--- a/arch/arm/boot/dts/ti/omap/am4372.dtsi
+++ b/arch/arm/boot/dts/ti/omap/am4372.dtsi
@@ -476,6 +476,17 @@ pruss1_mii_rt: mii-rt@32000 {
 					reg = <0x32000 0x58>;
 				};
 
+				pruss1_iep: iep@2e000 {
+					compatible = "ti,am4376-icss-iep";
+					reg = <0x2e000 0x31c>;
+					clocks = <&pruss1_iepclk_mux>;
+				};
+
+				pruss1_ecap: ecap@30000 {
+					compatible = "ti,pruss-ecap";
+					reg = <0x30000 0x60>;
+				};
+
 				pruss1_intc: interrupt-controller@20000 {
 					compatible = "ti,pruss-intc";
 					reg = <0x20000 0x2000>;
diff --git a/arch/arm/boot/dts/ti/omap/am437x-idk-evm.dts b/arch/arm/boot/dts/ti/omap/am437x-idk-evm.dts
index 826f687c368ab..2efa303d45be3 100644
--- a/arch/arm/boot/dts/ti/omap/am437x-idk-evm.dts
+++ b/arch/arm/boot/dts/ti/omap/am437x-idk-evm.dts
@@ -168,6 +168,48 @@ led-out7 {
 			default-state = "off";
 		};
 	};
+
+	/* Dual-MAC Ethernet application node on PRU-ICSS1 */
+	pruss1_eth: pruss1-eth {
+		compatible = "ti,am4376-prueth";
+		ti,prus = <&pru1_0>, <&pru1_1>;
+		sram = <&ocmcram>;
+		ti,mii-rt = <&pruss1_mii_rt>;
+		ti,iep = <&pruss1_iep>;
+		ti,ecap = <&pruss1_ecap>;
+		interrupts = <20 2 2>, <21 3 3>;
+		interrupt-names = "rx_hp", "rx_lp";
+		interrupt-parent = <&pruss1_intc>;
+
+		pinctrl-0 = <&pruss1_eth_default>;
+		pinctrl-names = "default";
+
+		ethernet-ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pruss1_emac0: ethernet-port@0 {
+				reg = <0>;
+				phy-handle = <&pruss1_eth0_phy>;
+				phy-mode = "mii";
+				interrupts = <20 2 2>, <26 6 6>, <23 6 6>;
+				interrupt-names = "rx", "emac_ptp_tx",
+						  "hsr_ptp_tx";
+				/* Filled in by bootloader */
+				local-mac-address = [00 00 00 00 00 00];
+			};
+
+			pruss1_emac1: ethernet-port@1 {
+				reg = <1>;
+				phy-handle = <&pruss1_eth1_phy>;
+				phy-mode = "mii";
+				interrupts = <21 3 3>, <27 9 5>, <24 9 5>;
+				interrupt-names = "rx", "emac_ptp_tx",
+						  "hsr_ptp_tx";
+				/* Filled in by bootloader */
+				local-mac-address = [00 00 00 00 00 00];
+			};
+		};
+	};
 };
 
 &am43xx_pinmux {
@@ -303,6 +345,52 @@ AM4372_IOPAD(0x94c, PIN_INPUT_PULLDOWN | MUX_MODE7)
 		>;
 	};
 
+	pruss1_mdio_default: pruss1-mdio-default-pins {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x88c, PIN_OUTPUT | MUX_MODE5) /* (A12) gpmc_clk.pr1_mdio_mdclk */
+			AM4372_IOPAD(0xa70, PIN_INPUT | MUX_MODE8) /* (D24) xdma_event_intr0.pr1_mdio_data */
+			AM4372_IOPAD(0xa00, PIN_INPUT_PULLUP | MUX_MODE7) /* (AD23) cam1_data6.gpio4[20] */
+		>;
+	};
+
+	pruss1_eth_default: pruss1-eth-default-pins {
+		pinctrl-single,pins = <
+			AM4372_IOPAD(0x8a0, PIN_INPUT | MUX_MODE2) /* (B22) dss_data0.pr1_mii_mt0_clk */
+			AM4372_IOPAD(0x8b4, PIN_OUTPUT | MUX_MODE2) /* (B20) dss_data5.pr1_mii0_txd0 */
+			AM4372_IOPAD(0x8b0, PIN_OUTPUT | MUX_MODE2) /* (A20) dss_data4.pr1_mii0_txd1 */
+			AM4372_IOPAD(0x8ac, PIN_OUTPUT | MUX_MODE2) /* (C21) dss_data3.pr1_mii0_txd2 */
+			AM4372_IOPAD(0x8a8, PIN_OUTPUT | MUX_MODE2) /* (B21) dss_data2.pr1_mii0_txd3 */
+			AM4372_IOPAD(0x8cc, PIN_INPUT | MUX_MODE5) /* (B18) dss_data11.pr1_mii0_rxd0 */
+			AM4372_IOPAD(0x8c8, PIN_INPUT | MUX_MODE5) /* (A18) dss_data10.pr1_mii0_rxd1 */
+			AM4372_IOPAD(0x8c4, PIN_INPUT | MUX_MODE5) /* (B19) dss_data9.pr1_mii0_rxd2 */
+			AM4372_IOPAD(0x8c0, PIN_INPUT | MUX_MODE5) /* (A19) dss_data8.pr1_mii0_rxd3 */
+			AM4372_IOPAD(0x8a4, PIN_OUTPUT | MUX_MODE2) /* (A21) dss_data1.pr1_mii0_txen */
+			AM4372_IOPAD(0x8d8, PIN_INPUT | MUX_MODE5) /* (C17) dss_data14.pr1_mii_mr0_clk */
+			AM4372_IOPAD(0x8dc, PIN_INPUT | MUX_MODE5) /* (D17) dss_data15.pr1_mii0_rxdv */
+			AM4372_IOPAD(0x8d4, PIN_INPUT | MUX_MODE5) /* (D19) dss_data13.pr1_mii0_rxer */
+			AM4372_IOPAD(0x8d0, PIN_INPUT | MUX_MODE5) /* (C19) dss_data12.pr1_mii0_rxlink */
+			AM4372_IOPAD(0xa40, PIN_INPUT | MUX_MODE5) /* (G20) gpio5_10.pr1_mii0_crs */
+			AM4372_IOPAD(0xa38, PIN_INPUT | MUX_MODE5) /* (D25) gpio5_8.pr1_mii0_col */
+
+			AM4372_IOPAD(0x858, PIN_INPUT | MUX_MODE5) /* (E8) gpmc_a6.pr1_mii_mt1_clk */
+			AM4372_IOPAD(0x854, PIN_OUTPUT | MUX_MODE5) /* (E7) gpmc_a5.pr1_mii1_txd0 */
+			AM4372_IOPAD(0x850, PIN_OUTPUT | MUX_MODE5) /* (D7) gpmc_a4.pr1_mii1_txd1 */
+			AM4372_IOPAD(0x84c, PIN_OUTPUT | MUX_MODE5) /* (A4) gpmc_a3.pr1_mii1_txd2 */
+			AM4372_IOPAD(0x848, PIN_OUTPUT | MUX_MODE5) /* (C6) gpmc_a2.pr1_mii1_txd3 */
+			AM4372_IOPAD(0x86c, PIN_INPUT | MUX_MODE5) /* (D8) gpmc_a11.pr1_mii1_rxd0 */
+			AM4372_IOPAD(0x868, PIN_INPUT | MUX_MODE5) /* (G8) gpmc_a10.pr1_mii1_rxd1 */
+			AM4372_IOPAD(0x864, PIN_INPUT | MUX_MODE5) /* (B4) gpmc_a9.pr1_mii1_rxd2 */
+			AM4372_IOPAD(0x860, PIN_INPUT | MUX_MODE5) /* (F7) gpmc_a8.pr1_mii1_rxd3 */
+			AM4372_IOPAD(0x840, PIN_OUTPUT | MUX_MODE5) /* (C3) gpmc_a0.pr1_mii1_txen */
+			AM4372_IOPAD(0x85c, PIN_INPUT | MUX_MODE5) /* (F6) gpmc_a7.pr1_mii_mr1_clk */
+			AM4372_IOPAD(0x844, PIN_INPUT | MUX_MODE5) /* (C5) gpmc_a1.pr1_mii1_rxdv */
+			AM4372_IOPAD(0x874, PIN_INPUT | MUX_MODE5) /* (B3) gpmc_wpn.pr1_mii1_rxer */
+			AM4372_IOPAD(0xa4c, PIN_INPUT | MUX_MODE5) /* (E24) gpio5_13.pr1_mii1_rxlink */
+			AM4372_IOPAD(0xa44, PIN_INPUT | MUX_MODE5) /* (F23) gpio5_11.pr1_mii1_crs */
+			AM4372_IOPAD(0x878, PIN_INPUT | MUX_MODE5) /* (A3) gpmc_be1n.pr1_mii1_col */
+		>;
+	};
+
 	qspi_pins_default: qspi-default-pins {
 		pinctrl-single,pins = <
 			AM4372_IOPAD(0x87c, PIN_OUTPUT_PULLUP | MUX_MODE3)	/* gpmc_csn0.qspi_csn */
@@ -539,5 +627,18 @@ opp-100-600000000 {
 };
 
 &pruss1_mdio {
-	status = "disabled";
+	pinctrl-0 = <&pruss1_mdio_default>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	reset-gpios = <&gpio4 20 GPIO_ACTIVE_LOW>;
+	reset-delay-us = <2>;	/* PHY datasheet states 1uS min */
+
+	pruss1_eth0_phy: ethernet-phy@0 {
+		reg = <0>;
+	};
+
+	pruss1_eth1_phy: ethernet-phy@1 {
+		reg = <1>;
+	};
 };
diff --git a/arch/arm/boot/dts/ti/omap/am437x-l4.dtsi b/arch/arm/boot/dts/ti/omap/am437x-l4.dtsi
index e08f356e71cb0..30fcce33f4b7c 100644
--- a/arch/arm/boot/dts/ti/omap/am437x-l4.dtsi
+++ b/arch/arm/boot/dts/ti/omap/am437x-l4.dtsi
@@ -301,7 +301,7 @@ am43xx_pinmux: pinmux@800 {
 				};
 
 				scm_conf: scm_conf@0 {
-					compatible = "syscon", "simple-bus";
+					compatible = "ti,am4372-scm-conf", "syscon", "simple-bus";
 					reg = <0x0 0x800>;
 					#address-cells = <1>;
 					#size-cells = <1>;
diff --git a/arch/arm/boot/dts/ti/omap/am57-pruss.dtsi b/arch/arm/boot/dts/ti/omap/am57-pruss.dtsi
index 46c5383f0eee2..f733166256084 100644
--- a/arch/arm/boot/dts/ti/omap/am57-pruss.dtsi
+++ b/arch/arm/boot/dts/ti/omap/am57-pruss.dtsi
@@ -170,6 +170,17 @@ pruss2_iepclk_mux: iepclk-mux@30 {
 				};
 			};
 
+			pruss2_iep: iep@2e000 {
+				compatible = "ti,am5728-icss-iep";
+				reg = <0x2e000 0x31c>;
+				clocks = <&pruss2_iepclk_mux>;
+			};
+
+			pruss2_ecap: ecap@30000 {
+				compatible = "ti,pruss-ecap";
+				reg = <0x30000 0x60>;
+			};
+
 			pruss2_mii_rt: mii-rt@32000 {
 				compatible = "ti,pruss-mii", "syscon";
 				reg = <0x32000 0x58>;
diff --git a/arch/arm/boot/dts/ti/omap/am571x-idk.dts b/arch/arm/boot/dts/ti/omap/am571x-idk.dts
index 322cf79d22e99..02653b440585d 100644
--- a/arch/arm/boot/dts/ti/omap/am571x-idk.dts
+++ b/arch/arm/boot/dts/ti/omap/am571x-idk.dts
@@ -214,5 +214,11 @@ &pruss1_mdio {
 };
 
 &pruss2_mdio {
-	status = "disabled";
+	reset-gpios = <&gpio5 9 GPIO_ACTIVE_LOW>;
+	reset-delay-us = <2>;   /* PHY datasheet states 1uS min */
+};
+
+&pruss2_eth {
+	ti,pruss-gp-mux-sel = <4>,      /* MII2, needed for PRUSS1_MII0 */
+			      <4>;      /* MII2, needed for PRUSS1_MII1 */
 };
diff --git a/arch/arm/boot/dts/ti/omap/am5729-beagleboneai.dts b/arch/arm/boot/dts/ti/omap/am5729-beagleboneai.dts
index 43cf4ade950b6..76bfb364777e5 100644
--- a/arch/arm/boot/dts/ti/omap/am5729-beagleboneai.dts
+++ b/arch/arm/boot/dts/ti/omap/am5729-beagleboneai.dts
@@ -420,7 +420,7 @@ stmpe811@41 {
 		st,mod-12b = <1>; /* 12-bit ADC */
 		st,ref-sel = <0>; /* internal ADC reference */
 		st,adc-freq = <1>; /* 3.25 MHz ADC clock speed */
-		st,sample-time = <4>; /* ADC converstion time: 80 clocks */
+		st,sample-time = <4>; /* ADC conversion time: 80 clocks */
 
 		stmpe_adc {
 			compatible = "st,stmpe-adc";
diff --git a/arch/arm/boot/dts/ti/omap/am572x-idk.dts b/arch/arm/boot/dts/ti/omap/am572x-idk.dts
index 94a738cb0a4d6..54a8ccb9ca141 100644
--- a/arch/arm/boot/dts/ti/omap/am572x-idk.dts
+++ b/arch/arm/boot/dts/ti/omap/am572x-idk.dts
@@ -28,10 +28,12 @@ &mmc2 {
 	pinctrl-2 = <&mmc2_pins_ddr_rev20>;
 };
 
-&pruss1_mdio {
-	status = "disabled";
+&pruss2_eth0_phy {
+	reset-gpios = <&gpio5 8 GPIO_ACTIVE_LOW>;
+	reset-assert-us = <2>;   /* PHY datasheet states 1uS min */
 };
 
-&pruss2_mdio {
-	status = "disabled";
+&pruss2_eth1_phy {
+	reset-gpios = <&gpio5 9 GPIO_ACTIVE_LOW>;
+	reset-assert-us = <2>;   /* PHY datasheet states 1uS min */
 };
diff --git a/arch/arm/boot/dts/ti/omap/am574x-idk.dts b/arch/arm/boot/dts/ti/omap/am574x-idk.dts
index 47b9174d23530..47b6c6cb210c8 100644
--- a/arch/arm/boot/dts/ti/omap/am574x-idk.dts
+++ b/arch/arm/boot/dts/ti/omap/am574x-idk.dts
@@ -40,10 +40,12 @@ &emif1 {
 	status = "okay";
 };
 
-&pruss1_mdio {
-	status = "disabled";
+&pruss2_eth0_phy {
+	reset-gpios = <&gpio5 8 GPIO_ACTIVE_LOW>;
+	reset-assert-us = <2>;   /* PHY datasheet states 1uS min */
 };
 
-&pruss2_mdio {
-	status = "disabled";
+&pruss2_eth1_phy {
+	reset-gpios = <&gpio5 9 GPIO_ACTIVE_LOW>;
+	reset-assert-us = <2>;   /* PHY datasheet states 1uS min */
 };
diff --git a/arch/arm/boot/dts/ti/omap/am57xx-idk-common.dtsi b/arch/arm/boot/dts/ti/omap/am57xx-idk-common.dtsi
index 43e3623f079c8..5eccff3bb4b67 100644
--- a/arch/arm/boot/dts/ti/omap/am57xx-idk-common.dtsi
+++ b/arch/arm/boot/dts/ti/omap/am57xx-idk-common.dtsi
@@ -155,6 +155,52 @@ src_clk_x1: src_clk_x1 {
 		compatible = "fixed-clock";
 		clock-frequency = <20000000>;
 	};
+
+	/* Dual-MAC Ethernet application node on PRU-ICSS2 */
+	pruss2_eth: pruss2-eth {
+		compatible = "ti,am57-prueth";
+		ti,prus = <&pru2_0>, <&pru2_1>;
+		sram = <&ocmcram1>;
+		ti,mii-rt = <&pruss2_mii_rt>;
+		ti,iep = <&pruss2_iep>;
+		ti,ecap = <&pruss2_ecap>;
+		interrupts = <20 2 2>, <21 3 3>;
+		interrupt-names = "rx_hp", "rx_lp";
+		interrupt-parent = <&pruss2_intc>;
+
+		ethernet-ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pruss2_emac0: ethernet-port@0 {
+				reg = <0>;
+				phy-handle = <&pruss2_eth0_phy>;
+				phy-mode = "mii";
+				interrupts = <20 2 2>, <26 6 6>, <23 6 6>;
+				interrupt-names = "rx", "emac_ptp_tx",
+						  "hsr_ptp_tx";
+				/* Filled in by bootloader */
+				local-mac-address = [00 00 00 00 00 00];
+			};
+
+			pruss2_emac1: ethernet-port@1 {
+				reg = <1>;
+				phy-handle = <&pruss2_eth1_phy>;
+				phy-mode = "mii";
+				interrupts = <21 3 3>, <27 9 7>, <24 9 7>;
+				interrupt-names = "rx", "emac_ptp_tx",
+						  "hsr_ptp_tx";
+				/* Filled in by bootloader */
+				local-mac-address = [00 00 00 00 00 00];
+			};
+		};
+	};
+
+};
+
+&pruss2_iep {
+	interrupt-parent = <&pruss2_intc>;
+	interrupts = <7 7 8>;
+	interrupt-names = "iep_cap_cmp";
 };
 
 &dra7_pmx_core {
@@ -606,3 +652,18 @@ dpi_out: endpoint {
 		};
 	};
 };
+
+&pruss2_mdio {
+	status = "okay";
+	pruss2_eth0_phy: ethernet-phy@0 {
+		reg = <0>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <30 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	pruss2_eth1_phy: ethernet-phy@1 {
+		reg = <1>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <31 IRQ_TYPE_LEVEL_LOW>;
+	};
+};
diff --git a/arch/arm/boot/dts/ti/omap/dm8148-evm.dts b/arch/arm/boot/dts/ti/omap/dm8148-evm.dts
index 57a9eef09f6f0..8236d9eb438a4 100644
--- a/arch/arm/boot/dts/ti/omap/dm8148-evm.dts
+++ b/arch/arm/boot/dts/ti/omap/dm8148-evm.dts
@@ -99,7 +99,7 @@ partition@780000 {
 };
 
 &mmc1 {
-        status = "disabled";
+	status = "disabled";
 };
 
 &mmc2 {
@@ -111,7 +111,7 @@ &mmc2 {
 };
 
 &mmc3 {
-        status = "disabled";
+	status = "disabled";
 };
 
 &pincntl {
diff --git a/arch/arm/boot/dts/ti/omap/dm814x.dtsi b/arch/arm/boot/dts/ti/omap/dm814x.dtsi
index 27d1f35a31fd0..9e02bfa5c3a29 100644
--- a/arch/arm/boot/dts/ti/omap/dm814x.dtsi
+++ b/arch/arm/boot/dts/ti/omap/dm814x.dtsi
@@ -432,7 +432,7 @@ control: control@140000 {
 				ranges = <0 0x140000 0x20000>;
 
 				scm_conf: scm_conf@0 {
-					compatible = "syscon", "simple-bus";
+					compatible = "ti,dm814-scm-conf", "syscon", "simple-bus";
 					reg = <0x0 0x800>;
 					#address-cells = <1>;
 					#size-cells = <1>;
diff --git a/arch/arm/boot/dts/ti/omap/dm816x-clocks.dtsi b/arch/arm/boot/dts/ti/omap/dm816x-clocks.dtsi
index 338449b32a183..76ebc6c3c3e5e 100644
--- a/arch/arm/boot/dts/ti/omap/dm816x-clocks.dtsi
+++ b/arch/arm/boot/dts/ti/omap/dm816x-clocks.dtsi
@@ -177,7 +177,7 @@ mpu_ck: mpu_ck@15dc {
 		compatible = "ti,gate-clock";
 		clocks = <&sysclk2_ck>;
 		ti,bit-shift = <1>;
-                reg = <0x15dc>;
+		reg = <0x15dc>;
 	};
 
 	audio_pll_a_ck: audio_pll_a_ck@35c {
diff --git a/arch/arm/boot/dts/ti/omap/dm816x.dtsi b/arch/arm/boot/dts/ti/omap/dm816x.dtsi
index a1e0e904e0f05..ee0090f7aa64e 100644
--- a/arch/arm/boot/dts/ti/omap/dm816x.dtsi
+++ b/arch/arm/boot/dts/ti/omap/dm816x.dtsi
@@ -100,7 +100,7 @@ dm816x_pinmux: pinmux@800 {
 
 			/* Device Configuration Registers */
 			scm_conf: syscon@600 {
-				compatible = "syscon", "simple-bus";
+				compatible = "ti,dm8168-scm-conf", "syscon", "simple-bus";
 				reg = <0x600 0x110>;
 				#address-cells = <1>;
 				#size-cells = <1>;
diff --git a/arch/arm/boot/dts/ti/omap/dra7-l4.dtsi b/arch/arm/boot/dts/ti/omap/dra7-l4.dtsi
index c8d325b0f57b2..a595745afe595 100644
--- a/arch/arm/boot/dts/ti/omap/dra7-l4.dtsi
+++ b/arch/arm/boot/dts/ti/omap/dra7-l4.dtsi
@@ -64,7 +64,7 @@ scm: scm@0 {
 				ranges = <0 0 0x2000>;
 
 				scm_conf: scm_conf@0 {
-					compatible = "syscon", "simple-bus";
+					compatible = "ti,dra7-scm-conf", "syscon", "simple-bus";
 					reg = <0x0 0x1400>;
 					#address-cells = <1>;
 					#size-cells = <1>;
@@ -1695,9 +1695,9 @@ uart4: serial@0 {
 				reg = <0x0 0x100>;
 				interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
 				clock-frequency = <48000000>;
-			                        status = "disabled";
 				dmas = <&sdma_xbar 55>, <&sdma_xbar 56>;
 				dma-names = "tx", "rx";
+				status = "disabled";
 			};
 		};
 
diff --git a/arch/arm/boot/dts/ti/omap/motorola-mapphone-common.dtsi b/arch/arm/boot/dts/ti/omap/motorola-mapphone-common.dtsi
index a0c53d9c26255..28b4f12b82a73 100644
--- a/arch/arm/boot/dts/ti/omap/motorola-mapphone-common.dtsi
+++ b/arch/arm/boot/dts/ti/omap/motorola-mapphone-common.dtsi
@@ -91,22 +91,22 @@ &cpu_thermal {
 };
 
 &cpu_alert0 {
-        temperature = <80000>; /* millicelsius */
+	temperature = <80000>; /* millicelsius */
 };
 
 &cpu0 {
-        /*
+	/*
 	 * Note that the 1.2GiHz mode is enabled for all SoC variants for
 	 * the Motorola Android Linux v3.0.8 based kernel.
 	 */
-        operating-points = <
-	        /* kHz    uV */
+	operating-points = <
+		/* kHz    uV */
 	        300000  1025000
 	        600000  1200000
 	        800000  1313000
 	        1008000 1375000
 		1200000 1375000
-        >;
+	>;
 };
 
 &dss {
diff --git a/arch/arm/boot/dts/ti/omap/omap2430.dtsi b/arch/arm/boot/dts/ti/omap/omap2430.dtsi
index 222613d2a4d18..01bd471f92231 100644
--- a/arch/arm/boot/dts/ti/omap/omap2430.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap2430.dtsi
@@ -50,7 +50,7 @@ omap2430_pmx: pinmux@30 {
 				};
 
 				scm_conf: scm_conf@270 {
-					compatible = "syscon",
+					compatible = "ti,omap2-scm-conf", "syscon",
 						     "simple-bus";
 					reg = <0x270 0x240>;
 					#address-cells = <1>;
diff --git a/arch/arm/boot/dts/ti/omap/omap3-cm-t3x30.dtsi b/arch/arm/boot/dts/ti/omap/omap3-cm-t3x30.dtsi
index 0e942513560d6..6c4916ec41b40 100644
--- a/arch/arm/boot/dts/ti/omap/omap3-cm-t3x30.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap3-cm-t3x30.dtsi
@@ -29,7 +29,7 @@ OMAP3_CORE1_IOPAD(0x219a, PIN_INPUT_PULLUP | MUX_MODE4)	/* uart3_cts_rctx.gpio_1
 		>;
 	};
 
- 	hsusb0_pins: hsusb0-pins {
+	hsusb0_pins: hsusb0-pins {
 		pinctrl-single,pins = <
 			OMAP3_CORE1_IOPAD(0x21a2, PIN_OUTPUT | MUX_MODE0)		/* hsusb0_clk.hsusb0_clk */
 			OMAP3_CORE1_IOPAD(0x21a4, PIN_OUTPUT | MUX_MODE0)		/* hsusb0_stp.hsusb0_stp */
diff --git a/arch/arm/boot/dts/ti/omap/omap3-n950-n9.dtsi b/arch/arm/boot/dts/ti/omap/omap3-n950-n9.dtsi
index aa4fcdbedd8fb..519f7b42a59a5 100644
--- a/arch/arm/boot/dts/ti/omap/omap3-n950-n9.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap3-n950-n9.dtsi
@@ -287,7 +287,7 @@ lis302: lis302@1d {
 		pinctrl-names = "default";
 		pinctrl-0 = <&accelerator_pins>;
 
-                interrupts-extended = <&gpio6 20 IRQ_TYPE_EDGE_FALLING>, <&gpio6 21 IRQ_TYPE_EDGE_FALLING>; /* 180, 181 */
+		interrupts-extended = <&gpio6 20 IRQ_TYPE_EDGE_FALLING>, <&gpio6 21 IRQ_TYPE_EDGE_FALLING>; /* 180, 181 */
 
 		/* click flags */
 		st,click-single-x;
diff --git a/arch/arm/boot/dts/ti/omap/omap3-overo-common-lcd35.dtsi b/arch/arm/boot/dts/ti/omap/omap3-overo-common-lcd35.dtsi
index 0da561a23f36b..2f7a4a7173591 100644
--- a/arch/arm/boot/dts/ti/omap/omap3-overo-common-lcd35.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap3-overo-common-lcd35.dtsi
@@ -103,7 +103,7 @@ ads7846reg: ads7846-reg {
 
 	backlight {
 		compatible = "gpio-backlight";
-		
+
 		pinctrl-names = "default";
 		pinctrl-0 = <&backlight_pins>;
 		gpios = <&gpio5 17 GPIO_ACTIVE_HIGH>;		/* gpio_145 */
diff --git a/arch/arm/boot/dts/ti/omap/omap3-overo-common-lcd43.dtsi b/arch/arm/boot/dts/ti/omap/omap3-overo-common-lcd43.dtsi
index 981f02f088f85..35701f5969725 100644
--- a/arch/arm/boot/dts/ti/omap/omap3-overo-common-lcd43.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap3-overo-common-lcd43.dtsi
@@ -134,7 +134,7 @@ ads7846reg: ads7846-reg {
 
 	backlight {
 		compatible = "gpio-backlight";
-		
+
 		pinctrl-names = "default";
 		pinctrl-0 = <&backlight_pins>;
 		gpios = <&gpio5 17 GPIO_ACTIVE_HIGH>;		/* gpio_145 */
diff --git a/arch/arm/boot/dts/ti/omap/omap3.dtsi b/arch/arm/boot/dts/ti/omap/omap3.dtsi
index 959069e247300..447736d2e53ca 100644
--- a/arch/arm/boot/dts/ti/omap/omap3.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap3.dtsi
@@ -116,7 +116,7 @@ omap3_pmx_core: pinmux@30 {
 				};
 
 				scm_conf: scm_conf@270 {
-					compatible = "syscon", "simple-bus";
+					compatible = "ti,omap3-scm-conf", "syscon", "simple-bus";
 					reg = <0x270 0x330>;
 					#address-cells = <1>;
 					#size-cells = <1>;
diff --git a/arch/arm/boot/dts/ti/omap/omap4-epson-embt2ws.dts b/arch/arm/boot/dts/ti/omap/omap4-epson-embt2ws.dts
index e253e0775ea95..e11d1931c42a7 100644
--- a/arch/arm/boot/dts/ti/omap/omap4-epson-embt2ws.dts
+++ b/arch/arm/boot/dts/ti/omap/omap4-epson-embt2ws.dts
@@ -64,6 +64,13 @@ key-lock {
 		};
 	};
 
+	multi-led {
+		compatible = "leds-group-multicolor";
+		color = <LED_COLOR_ID_RGB>;
+		function = LED_FUNCTION_STATUS;
+		leds = <&led_r>, <&led_g>, <&led_b>;
+	};
+
 	cb_v18: regulator-cb-v18 {
 		pinctrl-names = "default";
 		pinctrl-0 = <&cb_v18_pins>;
@@ -368,19 +375,19 @@ led-controller@66 {
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		led@0 {
+		led_g: led@0 {
 			reg = <0>;
 			color = <LED_COLOR_ID_GREEN>;
 			function = LED_FUNCTION_STATUS;
 		};
 
-		led@2 {
+		led_b: led@2 {
 			reg = <2>;
 			color = <LED_COLOR_ID_BLUE>;
 			function = LED_FUNCTION_STATUS;
 		};
 
-		led@4 {
+		led_r: led@4 {
 			reg = <4>;
 			color = <LED_COLOR_ID_RED>;
 			function = LED_FUNCTION_STATUS;
diff --git a/arch/arm/boot/dts/ti/omap/omap4-l4.dtsi b/arch/arm/boot/dts/ti/omap/omap4-l4.dtsi
index 4c78a0b28fab3..c1afc49f456cf 100644
--- a/arch/arm/boot/dts/ti/omap/omap4-l4.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap4-l4.dtsi
@@ -681,7 +681,7 @@ omap4_pmx_core: pinmux@40 {
 			};
 
 			omap4_padconf_global: omap4_padconf_global@5a0 {
-				compatible = "syscon",
+				compatible = "ti,omap4-sysc-padconf-global", "syscon",
 					     "simple-bus";
 				reg = <0x5a0 0x170>;
 				#address-cells = <1>;
diff --git a/arch/arm/boot/dts/ti/omap/omap4-panda-es.dts b/arch/arm/boot/dts/ti/omap/omap4-panda-es.dts
index a933fe560834b..70c609811d119 100644
--- a/arch/arm/boot/dts/ti/omap/omap4-panda-es.dts
+++ b/arch/arm/boot/dts/ti/omap/omap4-panda-es.dts
@@ -12,7 +12,7 @@ / {
 	compatible = "ti,omap4-panda-es", "ti,omap4-panda", "ti,omap4460", "ti,omap4430", "ti,omap4";
 };
 
-/* Audio routing is differnet between PandaBoard4430 and PandaBoardES */
+/* Audio routing is different between PandaBoard4430 and PandaBoardES */
 &sound {
 	ti,model = "PandaBoardES";
 
diff --git a/arch/arm/boot/dts/ti/omap/omap4-var-som-om44.dtsi b/arch/arm/boot/dts/ti/omap/omap4-var-som-om44.dtsi
index 37d56b3010cff..9e04aa70f9822 100644
--- a/arch/arm/boot/dts/ti/omap/omap4-var-som-om44.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap4-var-som-om44.dtsi
@@ -322,4 +322,18 @@ &usbhshost {
 
 &usbhsehci {
 	phys = <&hsusb1_phy>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	usbhub: hub@1 {
+		compatible = "usb424,2514";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethernet: ethernet@3 {
+			compatible = "usb424,7500";
+			reg = <3>;
+		};
+	};
 };
diff --git a/arch/arm/boot/dts/ti/omap/omap5-l4.dtsi b/arch/arm/boot/dts/ti/omap/omap5-l4.dtsi
index 915870eb5c990..72849e1c95b06 100644
--- a/arch/arm/boot/dts/ti/omap/omap5-l4.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap5-l4.dtsi
@@ -96,8 +96,7 @@ omap5_pmx_core: pinmux@40 {
 				};
 
 				omap5_padconf_global: omap5_padconf_global@5a0 {
-					compatible = "syscon",
-						     "simple-bus";
+					compatible = "ti,omap5-sysc-padconf-global", "syscon", "simple-bus";
 					reg = <0x5a0 0xec>;
 					#address-cells = <1>;
 					#size-cells = <1>;
@@ -2311,7 +2310,8 @@ omap5_scm_wkup_pad_conf: omap5_scm_wkup_pad_conf@da0 {
 				ranges = <0 0 0x60>;
 
 				scm_wkup_pad_conf: scm_conf@0 {
-					compatible = "syscon", "simple-bus";
+					compatible = "ti,omap5-scm-wkup-conf",
+						     "syscon", "simple-bus";
 					reg = <0x0 0x60>;
 					#address-cells = <1>;
 					#size-cells = <1>;
diff --git a/arch/arm/boot/dts/xilinx/zynq-ebaz4205.dts b/arch/arm/boot/dts/xilinx/zynq-ebaz4205.dts
index 14f644156a6f4..19b564019502e 100644
--- a/arch/arm/boot/dts/xilinx/zynq-ebaz4205.dts
+++ b/arch/arm/boot/dts/xilinx/zynq-ebaz4205.dts
@@ -99,7 +99,6 @@ conf-cd {
 			groups = "gpio0_34_grp";
 			io-standard = <3>;
 			slew-rate = <0>;
-			bias-high-impedance;
 			bias-pull-up;
 		};
 	};
diff --git a/arch/arm/boot/dts/xilinx/zynq-zc702.dts b/arch/arm/boot/dts/xilinx/zynq-zc702.dts
index 6955637c5b1ac..36e919aea84c2 100644
--- a/arch/arm/boot/dts/xilinx/zynq-zc702.dts
+++ b/arch/arm/boot/dts/xilinx/zynq-zc702.dts
@@ -324,7 +324,6 @@ mux-cd {
 
 		conf-cd {
 			groups = "gpio0_0_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <0>;
 			power-source = <1>;
@@ -337,7 +336,6 @@ mux-wp {
 
 		conf-wp {
 			groups = "gpio0_15_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <0>;
 			power-source = <1>;
diff --git a/arch/arm/boot/dts/xilinx/zynq-zc706.dts b/arch/arm/boot/dts/xilinx/zynq-zc706.dts
index 3b803c698473e..1ce35eaa607bb 100644
--- a/arch/arm/boot/dts/xilinx/zynq-zc706.dts
+++ b/arch/arm/boot/dts/xilinx/zynq-zc706.dts
@@ -237,7 +237,6 @@ mux-cd {
 
 		conf-cd {
 			groups = "gpio0_14_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <0>;
 			power-source = <1>;
@@ -250,7 +249,6 @@ mux-wp {
 
 		conf-wp {
 			groups = "gpio0_15_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <0>;
 			power-source = <1>;
diff --git a/arch/arm/configs/am200epdkit_defconfig b/arch/arm/configs/am200epdkit_defconfig
index d8198592fe1b1..61beac88e863c 100644
--- a/arch/arm/configs/am200epdkit_defconfig
+++ b/arch/arm/configs/am200epdkit_defconfig
@@ -7,8 +7,7 @@ CONFIG_EXPERT=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_PXA=y
 CONFIG_ARCH_GUMSTIX=y
-CONFIG_AEABI=y
-# CONFIG_OABI_COMPAT is not set
+CONFIG_ATAGS=y
 CONFIG_CMDLINE="console=ttyS0,115200n8 root=1f01 rootfstype=jffs2"
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
diff --git a/arch/arm/configs/aspeed_g4_defconfig b/arch/arm/configs/aspeed_g4_defconfig
index 45d8738abb750..f86dd4ce7d0de 100644
--- a/arch/arm/configs/aspeed_g4_defconfig
+++ b/arch/arm/configs/aspeed_g4_defconfig
@@ -22,9 +22,7 @@ CONFIG_KEXEC=y
 CONFIG_ARCH_ASPEED=y
 CONFIG_MACH_ASPEED_G4=y
 CONFIG_VMSPLIT_2G=y
-CONFIG_AEABI=y
 CONFIG_UACCESS_WITH_MEMCPY=y
-# CONFIG_ATAGS is not set
 CONFIG_JUMP_LABEL=y
 CONFIG_STRICT_KERNEL_RWX=y
 # CONFIG_BLK_DEBUG_FS is not set
diff --git a/arch/arm/configs/aspeed_g5_defconfig b/arch/arm/configs/aspeed_g5_defconfig
index ec558e57d081b..45b937419dbda 100644
--- a/arch/arm/configs/aspeed_g5_defconfig
+++ b/arch/arm/configs/aspeed_g5_defconfig
@@ -29,7 +29,6 @@ CONFIG_VMSPLIT_2G=y
 CONFIG_NR_CPUS=2
 CONFIG_HIGHMEM=y
 CONFIG_UACCESS_WITH_MEMCPY=y
-# CONFIG_ATAGS is not set
 CONFIG_VFP=y
 CONFIG_NEON=y
 CONFIG_KERNEL_MODE_NEON=y
diff --git a/arch/arm/configs/assabet_defconfig b/arch/arm/configs/assabet_defconfig
index 07ab9eaac4af1..e4264fdac2e57 100644
--- a/arch/arm/configs/assabet_defconfig
+++ b/arch/arm/configs/assabet_defconfig
@@ -1,10 +1,13 @@
 CONFIG_SYSVIPC=y
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_BLK_DEV_INITRD=y
+CONFIG_EXPERT=y
 CONFIG_ARCH_MULTI_V4=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_SA1100=y
 CONFIG_SA1100_ASSABET=y
+# CONFIG_AEABI is not set
+CONFIG_ATAGS=y
 CONFIG_CMDLINE="mem=32M console=ttySA0,38400n8 initrd=0xc0800000,3M root=/dev/ram"
 CONFIG_FPE_NWFPE=y
 CONFIG_PM=y
diff --git a/arch/arm/configs/at91_dt_defconfig b/arch/arm/configs/at91_dt_defconfig
index e331242dece71..830c6c0dbdaf3 100644
--- a/arch/arm/configs/at91_dt_defconfig
+++ b/arch/arm/configs/at91_dt_defconfig
@@ -18,9 +18,7 @@ CONFIG_SOC_AT91SAM9=y
 CONFIG_SOC_SAM9X60=y
 CONFIG_SOC_SAM9X7=y
 # CONFIG_ATMEL_CLOCKSOURCE_PIT is not set
-CONFIG_AEABI=y
 CONFIG_UACCESS_WITH_MEMCPY=y
-# CONFIG_ATAGS is not set
 CONFIG_ARM_APPENDED_DTB=y
 CONFIG_ARM_ATAG_DTB_COMPAT=y
 CONFIG_CMDLINE="console=ttyS0,115200 initrd=0x21100000,25165824 root=/dev/ram0 rw"
diff --git a/arch/arm/configs/axm55xx_defconfig b/arch/arm/configs/axm55xx_defconfig
index 0952e5e94c5e7..541e38e2205b1 100644
--- a/arch/arm/configs/axm55xx_defconfig
+++ b/arch/arm/configs/axm55xx_defconfig
@@ -38,8 +38,6 @@ CONFIG_PCIE_AXXIA=y
 CONFIG_SMP=y
 CONFIG_NR_CPUS=16
 CONFIG_HOTPLUG_CPU=y
-CONFIG_AEABI=y
-CONFIG_OABI_COMPAT=y
 CONFIG_HIGHMEM=y
 CONFIG_ARM_APPENDED_DTB=y
 CONFIG_ARM_ATAG_DTB_COMPAT=y
diff --git a/arch/arm/configs/bcm2835_defconfig b/arch/arm/configs/bcm2835_defconfig
index 4a8ac09843d73..b469ecc36cf58 100644
--- a/arch/arm/configs/bcm2835_defconfig
+++ b/arch/arm/configs/bcm2835_defconfig
@@ -25,7 +25,6 @@ CONFIG_CRASH_DUMP=y
 CONFIG_ARCH_MULTI_V6=y
 CONFIG_ARCH_BCM=y
 CONFIG_ARCH_BCM2835=y
-CONFIG_AEABI=y
 CONFIG_SECCOMP=y
 CONFIG_KEXEC=y
 CONFIG_CPU_FREQ=y
diff --git a/arch/arm/configs/clps711x_defconfig b/arch/arm/configs/clps711x_defconfig
index f66d502ce2efc..7ba344e84c62f 100644
--- a/arch/arm/configs/clps711x_defconfig
+++ b/arch/arm/configs/clps711x_defconfig
@@ -6,7 +6,6 @@ CONFIG_RD_LZMA=y
 CONFIG_EXPERT=y
 CONFIG_JUMP_LABEL=y
 CONFIG_PARTITION_ADVANCED=y
-CONFIG_AEABI=y
 # CONFIG_COREDUMP is not set
 CONFIG_SLUB_TINY=y
 CONFIG_NET=y
diff --git a/arch/arm/configs/collie_defconfig b/arch/arm/configs/collie_defconfig
index 578c6a4af620d..570a5f8bc57be 100644
--- a/arch/arm/configs/collie_defconfig
+++ b/arch/arm/configs/collie_defconfig
@@ -9,8 +9,8 @@ CONFIG_ARCH_MULTI_V4=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_SA1100=y
 CONFIG_SA1100_COLLIE=y
+CONFIG_ATAGS=y
 CONFIG_CMDLINE="noinitrd root=/dev/mtdblock2 rootfstype=jffs2 fbcon=rotate:1"
-CONFIG_FPE_NWFPE=y
 CONFIG_PM=y
 # CONFIG_SWAP is not set
 CONFIG_SLUB_TINY=y
diff --git a/arch/arm/configs/davinci_all_defconfig b/arch/arm/configs/davinci_all_defconfig
index 72703ef0c51cc..e6fbe1f039206 100644
--- a/arch/arm/configs/davinci_all_defconfig
+++ b/arch/arm/configs/davinci_all_defconfig
@@ -17,7 +17,6 @@ CONFIG_ARCH_DAVINCI=y
 CONFIG_ARCH_DAVINCI_DA850=y
 CONFIG_DAVINCI_MUX_DEBUG=y
 CONFIG_DAVINCI_MUX_WARNINGS=y
-CONFIG_AEABI=y
 CONFIG_SECCOMP=y
 CONFIG_ARM_APPENDED_DTB=y
 CONFIG_ARM_ATAG_DTB_COMPAT=y
diff --git a/arch/arm/configs/dove_defconfig b/arch/arm/configs/dove_defconfig
index b6ed01216a622..917a864fec743 100644
--- a/arch/arm/configs/dove_defconfig
+++ b/arch/arm/configs/dove_defconfig
@@ -7,8 +7,8 @@ CONFIG_EXPERT=y
 CONFIG_ARCH_MULTI_V7=y
 CONFIG_ARCH_DOVE=y
 CONFIG_MACH_CM_A510=y
-CONFIG_AEABI=y
 CONFIG_HIGHMEM=y
+CONFIG_ATAGS=y
 CONFIG_ARM_APPENDED_DTB=y
 CONFIG_ARM_ATAG_DTB_COMPAT=y
 CONFIG_VFP=y
diff --git a/arch/arm/configs/ep93xx_defconfig b/arch/arm/configs/ep93xx_defconfig
index ce41dc8c435cb..6863d8fd7713e 100644
--- a/arch/arm/configs/ep93xx_defconfig
+++ b/arch/arm/configs/ep93xx_defconfig
@@ -18,7 +18,6 @@ CONFIG_MACH_EDB9315=y
 CONFIG_MACH_EDB9315A=y
 CONFIG_MACH_TS72XX=y
 CONFIG_MACH_VISION_EP9307=y
-CONFIG_AEABI=y
 CONFIG_CMDLINE="console=ttyAM0,115200 root=/dev/nfs ip=bootp"
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
diff --git a/arch/arm/configs/footbridge_defconfig b/arch/arm/configs/footbridge_defconfig
index 5f6963687ee4f..7fa556fb6cf4e 100644
--- a/arch/arm/configs/footbridge_defconfig
+++ b/arch/arm/configs/footbridge_defconfig
@@ -9,8 +9,10 @@ CONFIG_ARCH_MULTI_V4=y
 CONFIG_ARCH_FOOTBRIDGE=y
 CONFIG_ARCH_EBSA285_HOST=y
 CONFIG_ARCH_NETWINDER=y
+CONFIG_ATAGS=y
 CONFIG_FPE_NWFPE=y
 CONFIG_FPE_NWFPE_XP=y
+# CONFIG_AEABI is not set
 CONFIG_MODULES=y
 CONFIG_PARTITION_ADVANCED=y
 CONFIG_ACORN_PARTITION=y
diff --git a/arch/arm/configs/gemini_defconfig b/arch/arm/configs/gemini_defconfig
index 7b1daec630cbe..5860b2fe7d1fb 100644
--- a/arch/arm/configs/gemini_defconfig
+++ b/arch/arm/configs/gemini_defconfig
@@ -11,7 +11,6 @@ CONFIG_KEXEC=y
 CONFIG_ARCH_MULTI_V4=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_GEMINI=y
-CONFIG_AEABI=y
 CONFIG_HIGHMEM=y
 CONFIG_CMDLINE="console=ttyS0,115200n8"
 CONFIG_PM=y
diff --git a/arch/arm/configs/h3600_defconfig b/arch/arm/configs/h3600_defconfig
index 4e272875c7978..3e8da62d959d6 100644
--- a/arch/arm/configs/h3600_defconfig
+++ b/arch/arm/configs/h3600_defconfig
@@ -4,12 +4,13 @@ CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_BLK_DEV_INITRD=y
+CONFIG_EXPERT=y
 CONFIG_ARCH_MULTI_V4=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_SA1100=y
 CONFIG_SA1100_H3600=y
 # CONFIG_CPU_FREQ_STAT is not set
-CONFIG_FPE_NWFPE=y
+CONFIG_ATAGS=y
 CONFIG_MODULES=y
 CONFIG_NET=y
 CONFIG_UNIX=y
diff --git a/arch/arm/configs/hisi_defconfig b/arch/arm/configs/hisi_defconfig
index dde9cff951d42..c89b743784ca4 100644
--- a/arch/arm/configs/hisi_defconfig
+++ b/arch/arm/configs/hisi_defconfig
@@ -10,7 +10,6 @@ CONFIG_ARCH_HIP04=y
 CONFIG_ARCH_HIX5HD2=y
 CONFIG_SMP=y
 CONFIG_NR_CPUS=16
-CONFIG_AEABI=y
 CONFIG_HIGHMEM=y
 CONFIG_ARM_APPENDED_DTB=y
 CONFIG_ARM_ATAG_DTB_COMPAT=y
diff --git a/arch/arm/configs/imx_v4_v5_defconfig b/arch/arm/configs/imx_v4_v5_defconfig
index 9139d1784c708..871e93dd65b5c 100644
--- a/arch/arm/configs/imx_v4_v5_defconfig
+++ b/arch/arm/configs/imx_v4_v5_defconfig
@@ -15,7 +15,6 @@ CONFIG_ARCH_MXC=y
 CONFIG_SOC_IMX1=y
 CONFIG_SOC_IMX25=y
 CONFIG_SOC_IMX27=y
-CONFIG_AEABI=y
 CONFIG_PM_DEBUG=y
 CONFIG_KPROBES=y
 CONFIG_MODULES=y
diff --git a/arch/arm/configs/integrator_defconfig b/arch/arm/configs/integrator_defconfig
index 61711d4bbf74b..82180381bf3cd 100644
--- a/arch/arm/configs/integrator_defconfig
+++ b/arch/arm/configs/integrator_defconfig
@@ -13,8 +13,6 @@ CONFIG_ARCH_INTEGRATOR=y
 CONFIG_ARCH_INTEGRATOR_AP=y
 CONFIG_INTEGRATOR_IMPD1=y
 CONFIG_ARCH_INTEGRATOR_CP=y
-CONFIG_AEABI=y
-# CONFIG_ATAGS is not set
 CONFIG_CMDLINE="console=ttyAM0,38400n8 root=/dev/nfs ip=bootp"
 CONFIG_CPU_FREQ=y
 CONFIG_CPU_FREQ_GOV_POWERSAVE=y
diff --git a/arch/arm/configs/ixp4xx_defconfig b/arch/arm/configs/ixp4xx_defconfig
index 418ef909572b5..122ca7354986c 100644
--- a/arch/arm/configs/ixp4xx_defconfig
+++ b/arch/arm/configs/ixp4xx_defconfig
@@ -9,7 +9,6 @@ CONFIG_EXPERT=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_IXP4XX=y
 CONFIG_CPU_BIG_ENDIAN=y
-CONFIG_AEABI=y
 CONFIG_CMDLINE="console=ttyS0,115200"
 CONFIG_STRICT_KERNEL_RWX=y
 CONFIG_STRICT_MODULE_RWX=y
diff --git a/arch/arm/configs/jornada720_defconfig b/arch/arm/configs/jornada720_defconfig
index d57285cfefb29..a96cfb81aba54 100644
--- a/arch/arm/configs/jornada720_defconfig
+++ b/arch/arm/configs/jornada720_defconfig
@@ -1,12 +1,13 @@
 CONFIG_SYSVIPC=y
 CONFIG_LOG_BUF_SHIFT=14
+CONFIG_EXPERT=y
 CONFIG_ARCH_MULTI_V4=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_SA1100=y
 CONFIG_SA1100_JORNADA720=y
 CONFIG_SA1100_JORNADA720_SSP=y
-CONFIG_FPE_NWFPE=y
 CONFIG_PM=y
+CONFIG_ATAGS=y
 CONFIG_MODULES=y
 CONFIG_NET=y
 CONFIG_PACKET=y
diff --git a/arch/arm/configs/keystone_defconfig b/arch/arm/configs/keystone_defconfig
index b0cadd8781526..af91b9a420c9e 100644
--- a/arch/arm/configs/keystone_defconfig
+++ b/arch/arm/configs/keystone_defconfig
@@ -22,7 +22,6 @@ CONFIG_PCI_KEYSTONE=y
 CONFIG_SMP=y
 CONFIG_HOTPLUG_CPU=y
 CONFIG_ARM_PSCI=y
-CONFIG_AEABI=y
 CONFIG_HIGHMEM=y
 CONFIG_VFP=y
 CONFIG_NEON=y
diff --git a/arch/arm/configs/lpc32xx_defconfig b/arch/arm/configs/lpc32xx_defconfig
index b9e2e603cd95e..c0b45eea1b0f3 100644
--- a/arch/arm/configs/lpc32xx_defconfig
+++ b/arch/arm/configs/lpc32xx_defconfig
@@ -10,7 +10,6 @@ CONFIG_CC_OPTIMIZE_FOR_SIZE=y
 CONFIG_EXPERT=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_LPC32XX=y
-CONFIG_AEABI=y
 CONFIG_ARM_APPENDED_DTB=y
 CONFIG_ARM_ATAG_DTB_COMPAT=y
 CONFIG_CMDLINE="console=ttyS0,115200n81 root=/dev/ram0"
diff --git a/arch/arm/configs/mmp2_defconfig b/arch/arm/configs/mmp2_defconfig
index 0ea608c75f226..2ca166d4e78f6 100644
--- a/arch/arm/configs/mmp2_defconfig
+++ b/arch/arm/configs/mmp2_defconfig
@@ -4,7 +4,6 @@ CONFIG_PREEMPT=y
 CONFIG_LOG_BUF_SHIFT=14
 # CONFIG_BLK_DEV_BSG is not set
 CONFIG_ARCH_MMP=y
-CONFIG_AEABI=y
 CONFIG_MACH_MMP2_DT=y
 CONFIG_CMDLINE="root=/dev/nfs rootfstype=nfs nfsroot=192.168.1.100:/nfsroot/ ip=192.168.1.101:192.168.1.100::255.255.255.0::eth0:on console=ttyS2,38400 mem=128M user_debug=255 earlyprintk"
 CONFIG_VFP=y
diff --git a/arch/arm/configs/moxart_defconfig b/arch/arm/configs/moxart_defconfig
index e2d9f36100636..699d7ad98dd21 100644
--- a/arch/arm/configs/moxart_defconfig
+++ b/arch/arm/configs/moxart_defconfig
@@ -16,8 +16,6 @@ CONFIG_ARCH_MULTI_V4=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_MOXART=y
 CONFIG_MACH_UC7112LX=y
-CONFIG_AEABI=y
-# CONFIG_ATAGS is not set
 CONFIG_ARM_APPENDED_DTB=y
 # CONFIG_SWAP is not set
 # CONFIG_COMPAT_BRK is not set
diff --git a/arch/arm/configs/mps2_defconfig b/arch/arm/configs/mps2_defconfig
index e995e50537efd..cacf3876f6e4b 100644
--- a/arch/arm/configs/mps2_defconfig
+++ b/arch/arm/configs/mps2_defconfig
@@ -16,7 +16,6 @@ CONFIG_ARCH_MPS2=y
 CONFIG_SET_MEM_PARAM=y
 CONFIG_DRAM_BASE=0x21000000
 CONFIG_DRAM_SIZE=0x1000000
-# CONFIG_ATAGS is not set
 # CONFIG_SUSPEND is not set
 # CONFIG_BLOCK is not set
 CONFIG_BINFMT_FLAT=y
diff --git a/arch/arm/configs/multi_v4t_defconfig b/arch/arm/configs/multi_v4t_defconfig
index 1a86dc3055236..7b592345db215 100644
--- a/arch/arm/configs/multi_v4t_defconfig
+++ b/arch/arm/configs/multi_v4t_defconfig
@@ -17,8 +17,6 @@ CONFIG_INTEGRATOR_IMPD1=y
 CONFIG_INTEGRATOR_CM720T=y
 CONFIG_INTEGRATOR_CM920T=y
 CONFIG_INTEGRATOR_CM922T_XA10=y
-CONFIG_AEABI=y
-# CONFIG_ATAGS is not set
 CONFIG_CPU_IDLE=y
 CONFIG_ARM_CPUIDLE=y
 CONFIG_ARM_CLPS711X_CPUIDLE=y
diff --git a/arch/arm/configs/multi_v5_defconfig b/arch/arm/configs/multi_v5_defconfig
index d237ea8ea3275..a0d2fb2fcf9a8 100644
--- a/arch/arm/configs/multi_v5_defconfig
+++ b/arch/arm/configs/multi_v5_defconfig
@@ -22,21 +22,11 @@ CONFIG_ARCH_NPCM=y
 CONFIG_ARCH_WPCM450=y
 CONFIG_ARCH_ORION5X=y
 CONFIG_MACH_RD88F5182_DT=y
-CONFIG_MACH_KUROBOX_PRO=y
-CONFIG_MACH_DNS323=y
-CONFIG_MACH_TS209=y
-CONFIG_MACH_TERASTATION_PRO2=y
-CONFIG_MACH_LINKSTATION_PRO=y
 CONFIG_MACH_LINKSTATION_MINI=y
-CONFIG_MACH_TS409=y
-CONFIG_MACH_TS78XX=y
-CONFIG_MACH_MV2120=y
 CONFIG_MACH_D2NET_DT=y
-CONFIG_MACH_NET2BIG=y
 CONFIG_MACH_MSS2_DT=y
 CONFIG_ARCH_SUNXI=y
 CONFIG_ARCH_VERSATILE=y
-CONFIG_AEABI=y
 CONFIG_HIGHMEM=y
 CONFIG_ARM_APPENDED_DTB=y
 CONFIG_ARM_ATAG_DTB_COMPAT=y
diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig
index 2de547c8b901d..f9b8dbf9318c9 100644
--- a/arch/arm/configs/multi_v7_defconfig
+++ b/arch/arm/configs/multi_v7_defconfig
@@ -149,6 +149,7 @@ CONFIG_INET6_IPCOMP=m
 CONFIG_IPV6_MIP6=m
 CONFIG_IPV6_TUNNEL=m
 CONFIG_IPV6_MULTIPLE_TABLES=y
+CONFIG_BRIDGE=m
 CONFIG_NET_DSA=m
 CONFIG_QRTR=m
 CONFIG_QRTR_SMD=m
@@ -288,6 +289,7 @@ CONFIG_ICPLUS_PHY=y
 CONFIG_MARVELL_PHY=y
 CONFIG_AT803X_PHY=y
 CONFIG_ROCKCHIP_PHY=y
+CONFIG_DP83848_PHY=m
 CONFIG_DP83867_PHY=y
 CONFIG_CAN_AT91=m
 CONFIG_CAN_FLEXCAN=m
@@ -405,6 +407,7 @@ CONFIG_ASPEED_KCS_IPMI_BMC=m
 CONFIG_ASPEED_BT_IPMI_BMC=m
 CONFIG_HW_RANDOM=y
 CONFIG_HW_RANDOM_ST=y
+CONFIG_HW_RANDOM_QCOM=m
 CONFIG_TCG_TPM=m
 CONFIG_TCG_TIS_I2C_INFINEON=m
 CONFIG_I2C_CHARDEV=y
@@ -1320,8 +1323,6 @@ CONFIG_CRYPTO_DEV_ATMEL_AES=m
 CONFIG_CRYPTO_DEV_ATMEL_TDES=m
 CONFIG_CRYPTO_DEV_ATMEL_SHA=m
 CONFIG_CRYPTO_DEV_MARVELL_CESA=m
-CONFIG_CRYPTO_DEV_QCE=m
-CONFIG_CRYPTO_DEV_QCOM_RNG=m
 CONFIG_CRYPTO_DEV_ROCKCHIP=m
 CONFIG_CRYPTO_DEV_STM32_HASH=m
 CONFIG_CRYPTO_DEV_STM32_CRYP=m
diff --git a/arch/arm/configs/mv78xx0_defconfig b/arch/arm/configs/mv78xx0_defconfig
index 1174893102bd8..e3cc8de7c1c20 100644
--- a/arch/arm/configs/mv78xx0_defconfig
+++ b/arch/arm/configs/mv78xx0_defconfig
@@ -11,9 +11,8 @@ CONFIG_ARCH_MULTI_V5=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_MV78XX0=y
 CONFIG_MACH_TERASTATION_WXL=y
-CONFIG_AEABI=y
 CONFIG_HIGHMEM=y
-CONFIG_FPE_NWFPE=y
+CONFIG_ATAGS=y
 CONFIG_VFP=y
 CONFIG_KPROBES=y
 CONFIG_MODULES=y
diff --git a/arch/arm/configs/mvebu_v5_defconfig b/arch/arm/configs/mvebu_v5_defconfig
index 781f8f72df5f4..dbc6c511e70bc 100644
--- a/arch/arm/configs/mvebu_v5_defconfig
+++ b/arch/arm/configs/mvebu_v5_defconfig
@@ -10,19 +10,9 @@ CONFIG_ARCH_MVEBU=y
 CONFIG_MACH_KIRKWOOD=y
 CONFIG_ARCH_ORION5X=y
 CONFIG_MACH_RD88F5182_DT=y
-CONFIG_MACH_KUROBOX_PRO=y
-CONFIG_MACH_DNS323=y
-CONFIG_MACH_TS209=y
-CONFIG_MACH_TERASTATION_PRO2=y
-CONFIG_MACH_LINKSTATION_PRO=y
 CONFIG_MACH_LINKSTATION_MINI=y
-CONFIG_MACH_TS409=y
-CONFIG_MACH_TS78XX=y
-CONFIG_MACH_MV2120=y
 CONFIG_MACH_D2NET_DT=y
-CONFIG_MACH_NET2BIG=y
 CONFIG_MACH_MSS2_DT=y
-CONFIG_AEABI=y
 CONFIG_HIGHMEM=y
 CONFIG_ARM_APPENDED_DTB=y
 CONFIG_ARM_ATAG_DTB_COMPAT=y
diff --git a/arch/arm/configs/mxs_defconfig b/arch/arm/configs/mxs_defconfig
index 603fb003b2236..b0b5eb33d01e4 100644
--- a/arch/arm/configs/mxs_defconfig
+++ b/arch/arm/configs/mxs_defconfig
@@ -17,7 +17,6 @@ CONFIG_BLK_DEV_INITRD=y
 CONFIG_PERF_EVENTS=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_MXS=y
-CONFIG_AEABI=y
 CONFIG_MODULES=y
 CONFIG_MODULE_FORCE_LOAD=y
 CONFIG_MODULE_UNLOAD=y
diff --git a/arch/arm/configs/neponset_defconfig b/arch/arm/configs/neponset_defconfig
index 8a5dcca743fcc..164643f1b6c0c 100644
--- a/arch/arm/configs/neponset_defconfig
+++ b/arch/arm/configs/neponset_defconfig
@@ -1,16 +1,19 @@
 CONFIG_SYSVIPC=y
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_BLK_DEV_INITRD=y
+CONFIG_EXPERT=y
 CONFIG_ARCH_MULTI_V4=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_SA1100=y
 CONFIG_SA1100_ASSABET=y
 CONFIG_ASSABET_NEPONSET=y
+CONFIG_ATAGS=y
 CONFIG_ZBOOT_ROM_TEXT=0x80000
 CONFIG_ZBOOT_ROM_BSS=0xc1000000
 CONFIG_ZBOOT_ROM=y
 CONFIG_CMDLINE="console=ttySA0,38400n8 cpufreq=221200 rw root=/dev/mtdblock2 mtdparts=sa1100:512K(boot),1M(kernel),2560K(initrd),4M(root) mem=32M noinitrd initrd=0xc0800000,3M"
 CONFIG_FPE_NWFPE=y
+# CONFIG_AEABI is not set
 CONFIG_PM=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
diff --git a/arch/arm/configs/netwinder_defconfig b/arch/arm/configs/netwinder_defconfig
index e639e6ad02cb8..77d4a2b5916bf 100644
--- a/arch/arm/configs/netwinder_defconfig
+++ b/arch/arm/configs/netwinder_defconfig
@@ -1,9 +1,12 @@
 CONFIG_SYSVIPC=y
 CONFIG_LOG_BUF_SHIFT=14
+CONFIG_EXPERT=y
 CONFIG_ARCH_MULTI_V4=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_FOOTBRIDGE=y
 CONFIG_ARCH_NETWINDER=y
+# CONFIG_AEABI is not set
+CONFIG_ATAGS=y
 CONFIG_DEPRECATED_PARAM_STRUCT=y
 CONFIG_CMDLINE="root=0x801"
 CONFIG_FPE_NWFPE=y
diff --git a/arch/arm/configs/nhk8815_defconfig b/arch/arm/configs/nhk8815_defconfig
index 696b4fbc2412d..7a307bd93730f 100644
--- a/arch/arm/configs/nhk8815_defconfig
+++ b/arch/arm/configs/nhk8815_defconfig
@@ -12,7 +12,6 @@ CONFIG_KALLSYMS_ALL=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_NOMADIK=y
 CONFIG_MACH_NOMADIK_8815NHK=y
-CONFIG_AEABI=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 # CONFIG_SWAP is not set
diff --git a/arch/arm/configs/omap1_defconfig b/arch/arm/configs/omap1_defconfig
index 7bf58e8a5ab57..fcbaaf37bc9bf 100644
--- a/arch/arm/configs/omap1_defconfig
+++ b/arch/arm/configs/omap1_defconfig
@@ -28,9 +28,8 @@ CONFIG_MACH_OMAP_PALMTE=y
 CONFIG_MACH_SX1=y
 CONFIG_MACH_NOKIA770=y
 CONFIG_MACH_AMS_DELTA=y
-CONFIG_AEABI=y
+CONFIG_ATAGS=y
 CONFIG_CMDLINE="root=1f03 rootfstype=jffs2"
-CONFIG_FPE_NWFPE=y
 # CONFIG_SUSPEND is not set
 CONFIG_PM=y
 CONFIG_MODULES=y
diff --git a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2plus_defconfig
index ad5ae1636dee4..f827f89567fdf 100644
--- a/arch/arm/configs/omap2plus_defconfig
+++ b/arch/arm/configs/omap2plus_defconfig
@@ -257,6 +257,9 @@ CONFIG_RXKAD=y
 CONFIG_CFG80211=m
 CONFIG_CFG80211_WEXT=y
 CONFIG_MAC80211=m
+CONFIG_RFKILL=m
+CONFIG_RFKILL_INPUT=y
+CONFIG_RFKILL_GPIO=m
 CONFIG_PCI=y
 CONFIG_PCI_MSI=y
 CONFIG_PCI_DRA7XX_EP=y
@@ -602,6 +605,7 @@ CONFIG_MMC_OMAP_HS=y
 CONFIG_MMC_SDHCI_OMAP=y
 CONFIG_NEW_LEDS=y
 CONFIG_LEDS_CLASS=m
+CONFIG_LEDS_CLASS_MULTICOLOR=m
 CONFIG_LEDS_CPCAP=m
 CONFIG_LEDS_LM3532=m
 CONFIG_LEDS_GPIO=m
@@ -611,6 +615,7 @@ CONFIG_LEDS_PCA963X=m
 CONFIG_LEDS_PWM=m
 CONFIG_LEDS_BD2606MVV=m
 CONFIG_LEDS_TCA6507=m
+CONFIG_LEDS_GROUP_MULTICOLOR=m
 CONFIG_LEDS_TRIGGERS=y
 CONFIG_LEDS_TRIGGER_TIMER=m
 CONFIG_LEDS_TRIGGER_ONESHOT=m
@@ -703,7 +708,15 @@ CONFIG_NFS_V4=y
 CONFIG_ROOT_NFS=y
 CONFIG_NLS_CODEPAGE_437=y
 CONFIG_NLS_ISO8859_1=y
+CONFIG_KEY_DH_OPERATIONS=y
 CONFIG_SECURITY=y
+CONFIG_CRYPTO_DH_RFC7919_GROUPS=y
+CONFIG_CRYPTO_DES=m
+CONFIG_CRYPTO_DEFLATE=y
+CONFIG_CRYPTO_LZO=y
+CONFIG_CRYPTO_ZSTD=y
+CONFIG_CRYPTO_USER_API_HASH=m
+CONFIG_CRYPTO_USER_API_SKCIPHER=m
 CONFIG_CRYPTO_GHASH_ARM_CE=m
 CONFIG_CRYPTO_AES=m
 CONFIG_CRYPTO_AES_ARM_BS=m
diff --git a/arch/arm/configs/orion5x_defconfig b/arch/arm/configs/orion5x_defconfig
index f5be2e26d9ae1..f47a84ac872f7 100644
--- a/arch/arm/configs/orion5x_defconfig
+++ b/arch/arm/configs/orion5x_defconfig
@@ -10,20 +10,9 @@ CONFIG_ARCH_MULTI_V5=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_ORION5X=y
 CONFIG_MACH_RD88F5182_DT=y
-CONFIG_MACH_KUROBOX_PRO=y
-CONFIG_MACH_DNS323=y
-CONFIG_MACH_TS209=y
-CONFIG_MACH_TERASTATION_PRO2=y
-CONFIG_MACH_LINKSTATION_PRO=y
 CONFIG_MACH_LINKSTATION_MINI=y
-CONFIG_MACH_TS409=y
-CONFIG_MACH_TS78XX=y
-CONFIG_MACH_MV2120=y
-CONFIG_MACH_NET2BIG=y
-CONFIG_AEABI=y
 CONFIG_ARM_APPENDED_DTB=y
 CONFIG_ARM_ATAG_DTB_COMPAT=y
-CONFIG_FPE_NWFPE=y
 CONFIG_VFP=y
 CONFIG_KPROBES=y
 CONFIG_MODULES=y
diff --git a/arch/arm/configs/pxa168_defconfig b/arch/arm/configs/pxa168_defconfig
index 8cbca84fe33a3..d6af01434f04f 100644
--- a/arch/arm/configs/pxa168_defconfig
+++ b/arch/arm/configs/pxa168_defconfig
@@ -3,11 +3,9 @@ CONFIG_SYSVIPC=y
 CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
-CONFIG_AEABI=y
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_ARCH_MMP=y
 CONFIG_CMDLINE="root=/dev/nfs rootfstype=nfs nfsroot=192.168.2.100:/nfsroot/ ip=192.168.2.101:192.168.2.100::255.255.255.0::eth0:on console=ttyS0,115200 mem=128M"
-CONFIG_FPE_NWFPE=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 CONFIG_MODULE_FORCE_UNLOAD=y
diff --git a/arch/arm/configs/pxa3xx_defconfig b/arch/arm/configs/pxa3xx_defconfig
index fb272e3a23377..2a777698d06f7 100644
--- a/arch/arm/configs/pxa3xx_defconfig
+++ b/arch/arm/configs/pxa3xx_defconfig
@@ -7,9 +7,7 @@ CONFIG_KALLSYMS_ALL=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_PXA=y
 CONFIG_MACH_PXA3XX_DT=y
-CONFIG_AEABI=y
 CONFIG_CMDLINE="root=/dev/nfs rootfstype=nfs nfsroot=192.168.1.100:/nfsroot/ ip=192.168.1.101:192.168.1.100::255.255.255.0::eth0:on console=ttyS0,115200 mem=64M debug"
-CONFIG_FPE_NWFPE=y
 CONFIG_MODULES=y
 CONFIG_NET=y
 CONFIG_PACKET=y
diff --git a/arch/arm/configs/pxa910_defconfig b/arch/arm/configs/pxa910_defconfig
index 71ed0d73f8a94..9e7b11d1e723f 100644
--- a/arch/arm/configs/pxa910_defconfig
+++ b/arch/arm/configs/pxa910_defconfig
@@ -10,8 +10,6 @@ CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 CONFIG_MODULE_FORCE_UNLOAD=y
 # CONFIG_BLK_DEV_BSG is not set
-CONFIG_AEABI=y
-CONFIG_FPE_NWFPE=y
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig
index 66cc149c5ca46..e7a7ed2b7ef7e 100644
--- a/arch/arm/configs/pxa_defconfig
+++ b/arch/arm/configs/pxa_defconfig
@@ -16,11 +16,6 @@ CONFIG_PROFILING=y
 CONFIG_KEXEC=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_PXA=y
-CONFIG_ARCH_GUMSTIX=y
-CONFIG_PXA_SHARPSL=y
-CONFIG_MACH_AKITA=y
-CONFIG_MACH_BORZOI=y
-CONFIG_AEABI=y
 CONFIG_ARCH_FORCE_MAX_ORDER=8
 CONFIG_CMDLINE="root=/dev/ram0 ro"
 CONFIG_CPU_FREQ=y
diff --git a/arch/arm/configs/qcom_defconfig b/arch/arm/configs/qcom_defconfig
index 03309b89ea4cf..df0a0ce5b097b 100644
--- a/arch/arm/configs/qcom_defconfig
+++ b/arch/arm/configs/qcom_defconfig
@@ -117,6 +117,7 @@ CONFIG_SERIAL_MSM=y
 CONFIG_SERIAL_MSM_CONSOLE=y
 CONFIG_SERIAL_DEV_BUS=y
 CONFIG_HW_RANDOM=y
+CONFIG_HW_RANDOM_QCOM=m
 CONFIG_I2C=y
 CONFIG_I2C_CHARDEV=y
 CONFIG_I2C_QUP=y
@@ -296,7 +297,6 @@ CONFIG_CRYPTO_USER_API_HASH=m
 CONFIG_CRYPTO_USER_API_SKCIPHER=m
 CONFIG_CRYPTO_USER_API_RNG=m
 CONFIG_CRYPTO_USER_API_AEAD=m
-CONFIG_CRYPTO_DEV_QCOM_RNG=m
 CONFIG_DMA_CMA=y
 CONFIG_CMA_SIZE_MBYTES=64
 CONFIG_PRINTK_TIME=y
diff --git a/arch/arm/configs/rpc_defconfig b/arch/arm/configs/rpc_defconfig
index 46df453e224eb..edb52d56f679c 100644
--- a/arch/arm/configs/rpc_defconfig
+++ b/arch/arm/configs/rpc_defconfig
@@ -2,11 +2,14 @@
 CONFIG_SYSVIPC=y
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_BLK_DEV_INITRD=y
+CONFIG_EXPERT=y
 CONFIG_ARCH_MULTI_V4=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_RPC=y
 CONFIG_CPU_SA110=y
 CONFIG_FPE_NWFPE=y
+# CONFIG_AEABI is not set
+CONFIG_ATAGS=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 CONFIG_PARTITION_ADVANCED=y
diff --git a/arch/arm/configs/s3c6400_defconfig b/arch/arm/configs/s3c6400_defconfig
index 7bf28a83946af..a0707707d960f 100644
--- a/arch/arm/configs/s3c6400_defconfig
+++ b/arch/arm/configs/s3c6400_defconfig
@@ -2,8 +2,10 @@ CONFIG_BLK_DEV_INITRD=y
 CONFIG_KALLSYMS_ALL=y
 CONFIG_ARCH_MULTI_V6=y
 # CONFIG_ARCH_MULTI_V7 is not set
+CONFIG_EXPERT=y
 CONFIG_ARCH_S3C64XX=y
 CONFIG_MACH_WLF_CRAGG_6410=y
+CONFIG_ATAGS=y
 CONFIG_CMDLINE="console=ttySAC0,115200 root=/dev/ram init=/linuxrc initrd=0x51000000,6M ramdisk_size=6144"
 CONFIG_VFP=y
 CONFIG_MODULES=y
diff --git a/arch/arm/configs/sama5_defconfig b/arch/arm/configs/sama5_defconfig
index bd7f0b5f7d660..1eafad9139f1d 100644
--- a/arch/arm/configs/sama5_defconfig
+++ b/arch/arm/configs/sama5_defconfig
@@ -13,7 +13,6 @@ CONFIG_SOC_SAMA5D3=y
 CONFIG_SOC_SAMA5D4=y
 # CONFIG_ATMEL_CLOCKSOURCE_PIT is not set
 CONFIG_UACCESS_WITH_MEMCPY=y
-# CONFIG_ATAGS is not set
 CONFIG_CMDLINE="console=ttyS0,115200 initrd=0x21100000,25165824 root=/dev/ram0 rw"
 CONFIG_VFP=y
 CONFIG_NEON=y
diff --git a/arch/arm/configs/sama7_defconfig b/arch/arm/configs/sama7_defconfig
index cc7f6daf9fc7a..4af87c57483da 100644
--- a/arch/arm/configs/sama7_defconfig
+++ b/arch/arm/configs/sama7_defconfig
@@ -20,7 +20,6 @@ CONFIG_ATMEL_CLOCKSOURCE_TCB=y
 # CONFIG_CPU_SW_DOMAIN_PAN is not set
 CONFIG_ARCH_FORCE_MAX_ORDER=14
 CONFIG_UACCESS_WITH_MEMCPY=y
-# CONFIG_ATAGS is not set
 CONFIG_CMDLINE="console=ttyS0,115200 earlyprintk ignore_loglevel"
 CONFIG_CPU_FREQ=y
 CONFIG_CPU_FREQ_DEFAULT_GOV_CONSERVATIVE=y
diff --git a/arch/arm/configs/spear13xx_defconfig b/arch/arm/configs/spear13xx_defconfig
index 6712ae741c196..648260f536a11 100644
--- a/arch/arm/configs/spear13xx_defconfig
+++ b/arch/arm/configs/spear13xx_defconfig
@@ -9,7 +9,6 @@ CONFIG_MACH_SPEAR1340=y
 CONFIG_SMP=y
 # CONFIG_SMP_ON_UP is not set
 # CONFIG_ARM_CPU_TOPOLOGY is not set
-CONFIG_AEABI=y
 CONFIG_ARM_APPENDED_DTB=y
 CONFIG_ARM_ATAG_DTB_COMPAT=y
 CONFIG_VFP=y
diff --git a/arch/arm/configs/spitz_defconfig b/arch/arm/configs/spitz_defconfig
index 8f8a058294fb8..c7cf089001bf2 100644
--- a/arch/arm/configs/spitz_defconfig
+++ b/arch/arm/configs/spitz_defconfig
@@ -10,8 +10,8 @@ CONFIG_ARCH_PXA=y
 CONFIG_PXA_SHARPSL=y
 CONFIG_MACH_AKITA=y
 CONFIG_MACH_BORZOI=y
+CONFIG_ATAGS=y
 CONFIG_CMDLINE="console=ttyS0,115200n8 console=tty1 noinitrd root=/dev/mtdblock2 rootfstype=jffs2   debug"
-CONFIG_FPE_NWFPE=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 CONFIG_MODULE_FORCE_UNLOAD=y
diff --git a/arch/arm/configs/stm32_defconfig b/arch/arm/configs/stm32_defconfig
index 82190b155b14c..7366aade19ad4 100644
--- a/arch/arm/configs/stm32_defconfig
+++ b/arch/arm/configs/stm32_defconfig
@@ -20,7 +20,6 @@ CONFIG_ARCH_STM32=y
 CONFIG_CPU_V7M_NUM_IRQ=240
 CONFIG_SET_MEM_PARAM=y
 CONFIG_DRAM_BASE=0x90000000
-# CONFIG_ATAGS is not set
 CONFIG_XIP_KERNEL=y
 CONFIG_XIP_PHYS_ADDR=0x08008000
 # CONFIG_SUSPEND is not set
diff --git a/arch/arm/configs/versatile_defconfig b/arch/arm/configs/versatile_defconfig
index 849118cbbb441..bb9eb9ccbbdea 100644
--- a/arch/arm/configs/versatile_defconfig
+++ b/arch/arm/configs/versatile_defconfig
@@ -6,8 +6,6 @@ CONFIG_LOG_BUF_SHIFT=14
 CONFIG_BLK_DEV_INITRD=y
 # CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_VERSATILE=y
-CONFIG_AEABI=y
-CONFIG_OABI_COMPAT=y
 CONFIG_CMDLINE="root=1f03 mem=32M"
 CONFIG_FPE_NWFPE=y
 CONFIG_VFP=y
diff --git a/arch/arm/configs/vt8500_v6_v7_defconfig b/arch/arm/configs/vt8500_v6_v7_defconfig
index 41607a84abc85..2925a1f1dbb6a 100644
--- a/arch/arm/configs/vt8500_v6_v7_defconfig
+++ b/arch/arm/configs/vt8500_v6_v7_defconfig
@@ -8,7 +8,6 @@ CONFIG_ARM_ERRATA_720789=y
 CONFIG_ARM_ERRATA_754322=y
 CONFIG_ARM_ERRATA_775420=y
 CONFIG_HAVE_ARM_ARCH_TIMER=y
-CONFIG_AEABI=y
 CONFIG_HIGHMEM=y
 CONFIG_HIGHPTE=y
 CONFIG_ARM_APPENDED_DTB=y
diff --git a/arch/arm/configs/wpcm450_defconfig b/arch/arm/configs/wpcm450_defconfig
index 67b64a3781663..7512ec4292a0c 100644
--- a/arch/arm/configs/wpcm450_defconfig
+++ b/arch/arm/configs/wpcm450_defconfig
@@ -13,9 +13,7 @@ CONFIG_PROFILING=y
 CONFIG_ARCH_NPCM=y
 CONFIG_ARCH_WPCM450=y
 CONFIG_CPU_DCACHE_WRITETHROUGH=y
-CONFIG_AEABI=y
 CONFIG_UACCESS_WITH_MEMCPY=y
-# CONFIG_ATAGS is not set
 CONFIG_ARM_APPENDED_DTB=y
 CONFIG_CPU_IDLE=y
 CONFIG_KPROBES=y
diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c
index 7951b2c06fec6..ed7a2a87a6707 100644
--- a/arch/arm/kernel/ptrace.c
+++ b/arch/arm/kernel/ptrace.c
@@ -840,7 +840,7 @@ static void report_syscall(struct pt_regs *regs, enum ptrace_syscall_dir dir)
 
 	if (dir == PTRACE_SYSCALL_EXIT)
 		ptrace_report_syscall_exit(regs, 0);
-	else if (ptrace_report_syscall_entry(regs))
+	else if (!ptrace_report_syscall_permit_entry(regs))
 		current_thread_info()->abi_syscall = -1;
 
 	regs->ARM_ip = ip;
@@ -855,7 +855,7 @@ asmlinkage int syscall_trace_enter(struct pt_regs *regs)
 
 	/* Do seccomp after ptrace; syscall may have changed. */
 #ifdef CONFIG_HAVE_ARCH_SECCOMP_FILTER
-	if (secure_computing() == -1)
+	if (!seccomp_permit_syscall())
 		return -1;
 #else
 	/* XXX: remove this once OABI gets fixed */
diff --git a/arch/arm/mach-at91/Kconfig b/arch/arm/mach-at91/Kconfig
index c5ef27e3cd8f0..cb0e3ff8e0caf 100644
--- a/arch/arm/mach-at91/Kconfig
+++ b/arch/arm/mach-at91/Kconfig
@@ -15,13 +15,15 @@ menuconfig ARCH_AT91
 
 if ARCH_AT91
 config SOC_SAMV7
-	bool "SAM Cortex-M7 family" if ARM_SINGLE_ARMV7M
+	bool "SAM Cortex-M7 family (DEPRECATED)" if ARM_SINGLE_ARMV7M
 	select COMMON_CLK_AT91
 	select PINCTRL_AT91
 	help
 	  Select this if you are using an SoC from Microchip's SAME7, SAMS7 or SAMV7
 	  families.
 
+	  This platform is scheduled for removal in early 2027
+
 config SOC_SAMA5D2
 	bool "SAMA5D2 family"
 	depends on ARCH_MULTI_V7
diff --git a/arch/arm/mach-axxia/Kconfig b/arch/arm/mach-axxia/Kconfig
index d3eae6037913a..f773a66c14028 100644
--- a/arch/arm/mach-axxia/Kconfig
+++ b/arch/arm/mach-axxia/Kconfig
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 config ARCH_AXXIA
-	bool "LSI Axxia platforms"
+	bool "LSI Axxia platforms (DEPRECATED)"
 	depends on ARCH_MULTI_V7 && ARM_LPAE
 	select ARM_AMBA
 	select ARM_GIC
@@ -13,3 +13,5 @@ config ARCH_AXXIA
 
 	  The LSI Axxia platforms require a Flattened Device Tree to be passed
 	  to the kernel.
+
+	  This platform is scheduled for removal in early 2027
diff --git a/arch/arm/mach-dove/Kconfig b/arch/arm/mach-dove/Kconfig
index 996888ffcfe7f..f547120e223ea 100644
--- a/arch/arm/mach-dove/Kconfig
+++ b/arch/arm/mach-dove/Kconfig
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 menuconfig ARCH_DOVE
-	bool "Marvell Dove" if ARCH_MULTI_V7
+	bool "Marvell Dove (DEPRECATED)" if ARCH_MULTI_V7
 	depends on ATAGS
 	select CPU_PJ4
 	select GPIOLIB
@@ -11,7 +11,11 @@ menuconfig ARCH_DOVE
 	select PM_GENERIC_DOMAINS if PM
 	select PCI_QUIRKS if PCI
 	help
-	  Support for the Marvell Dove SoC 88AP510
+	  Support for the Marvell Dove SoC 88AP510 using board files.
+
+	  Support for this machine will be removed in early 2027, unless
+	  there is a strong reason to keep it. Users should migrate to
+	  the devicetree-enabled version (CONFIG_MACH_DOVE) instead.
 
 if ARCH_DOVE
 
diff --git a/arch/arm/mach-footbridge/Kconfig b/arch/arm/mach-footbridge/Kconfig
index 78189997caa1d..96a74d447028d 100644
--- a/arch/arm/mach-footbridge/Kconfig
+++ b/arch/arm/mach-footbridge/Kconfig
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0-only
 menuconfig ARCH_FOOTBRIDGE
-	bool "FootBridge Implementations"
+	bool "FootBridge Implementations (DEPRECATED)"
 	depends on ARCH_MULTI_V4 && !(ARCH_MULTI_V4T || ARCH_MULTI_V5)
 	depends on !(ARCH_MOXART || ARCH_GEMINI || ARCH_SA1100)
 	depends on ATAGS
@@ -12,7 +12,10 @@ menuconfig ARCH_FOOTBRIDGE
 	select NEED_MACH_MEMORY_H
 	help
 	  Support for systems based on the DC21285 companion chip
-	  ("FootBridge"), such as the Simtec CATS and the Rebel NetWinder.
+	  ("FootBridge"), used in the EBSA285 and the Rebel NetWinder.
+
+	  Support for these machines will go away in 2027,
+	  unless there are any remaining users that speak up.
 
 if ARCH_FOOTBRIDGE
 
diff --git a/arch/arm/mach-imx/Kconfig b/arch/arm/mach-imx/Kconfig
index a361840d7a047..3f0521c652cb0 100644
--- a/arch/arm/mach-imx/Kconfig
+++ b/arch/arm/mach-imx/Kconfig
@@ -40,14 +40,19 @@ if ARCH_MULTI_V6
 comment "ARM1136 platforms"
 
 config SOC_IMX31
-	bool "i.MX31 support"
-	select CPU_V6
+	bool "i.MX31 support (DEPRECATED)"
+	depends on !SMP
+	select CPU_ARM1136R0
 	select MXC_AVIC
 	help
 	  This enables support for Freescale i.MX31 processor
 
+	  This SoC is scheduled for removal in early 2027,
+	  since it uses the ARM1136r0 CPU revision.
+
 config SOC_IMX35
 	bool "i.MX35 support"
+	select CPU_ARM1136R1
 	select MXC_AVIC
 	help
 	  This enables support for Freescale i.MX35 processor
@@ -214,12 +219,14 @@ config SOC_IMX7ULP
 	  This enables support for Freescale i.MX7 Ultra Low Power processor.
 
 config SOC_IMXRT
-	bool "i.MXRT support"
+	bool "i.MXRT support (DEPRECATED)"
 	depends on ARM_SINGLE_ARMV7M
 	select ARMV7M_SYSTICK if ARM_SINGLE_ARMV7M
 	help
 	  This enables support for Freescale i.MXRT Crossover processor.
 
+	  This platform is scheduled for removal in early 2027
+
 config SOC_VF610
 	bool "Vybrid Family VF610 support"
 	select ARM_GIC if ARCH_MULTI_V7
diff --git a/arch/arm/mach-imx/avic.c b/arch/arm/mach-imx/avic.c
index 3067c06b4b8eb..6873a50bbe2c0 100644
--- a/arch/arm/mach-imx/avic.c
+++ b/arch/arm/mach-imx/avic.c
@@ -173,6 +173,7 @@ static void __init mxc_init_irq(void __iomem *irqbase)
 
 	np = of_find_compatible_node(NULL, NULL, "fsl,imx25-ccm");
 	mx25_ccm_base = of_iomap(np, 0);
+	of_node_put(np);
 
 	if (mx25_ccm_base) {
 		/*
@@ -203,6 +204,7 @@ static void __init mxc_init_irq(void __iomem *irqbase)
 	np = of_find_compatible_node(NULL, NULL, "fsl,avic");
 	domain = irq_domain_create_legacy(of_fwnode_handle(np), AVIC_NUM_IRQS, irq_base, 0,
 					  &irq_domain_simple_ops, NULL);
+	of_node_put(np);
 	WARN_ON(!domain);
 
 	for (i = 0; i < AVIC_NUM_IRQS / 32; i++, irq_base += 32)
diff --git a/arch/arm/mach-imx/common.h b/arch/arm/mach-imx/common.h
index 45c1a2a7b35f9..d7ecaa822adbd 100644
--- a/arch/arm/mach-imx/common.h
+++ b/arch/arm/mach-imx/common.h
@@ -14,23 +14,16 @@ struct platform_device;
 struct pt_regs;
 struct clk;
 struct device_node;
-enum mxc_cpu_pwr_mode;
-struct of_device_id;
 
 void mx31_map_io(void);
 void mx35_map_io(void);
-void imx21_init_early(void);
 void imx31_init_early(void);
 void imx35_init_early(void);
-void mx31_init_irq(void);
-void mx35_init_irq(void);
 void mxc_set_cpu_type(unsigned int type);
 void mxc_restart(enum reboot_mode, const char *);
 void mxc_arch_reset_init(void __iomem *);
-void imx1_reset_init(void __iomem *);
 void imx_set_aips(void __iomem *);
 void imx_aips_allow_unprivileged_access(const char *compat);
-int mxc_device_init(void);
 void imx_set_soc_revision(unsigned int rev);
 void imx_init_revision_from_anatop(void);
 void imx6_enable_rbc(bool enable);
diff --git a/arch/arm/mach-imx/mmdc.c b/arch/arm/mach-imx/mmdc.c
index b71467c48b877..f6d993b9b1d4a 100644
--- a/arch/arm/mach-imx/mmdc.c
+++ b/arch/arm/mach-imx/mmdc.c
@@ -127,7 +127,7 @@ static ssize_t mmdc_pmu_cpumask_show(struct device *dev,
 {
 	struct mmdc_pmu *pmu_mmdc = dev_get_drvdata(dev);
 
-	return cpumap_print_to_pagebuf(true, buf, &pmu_mmdc->cpu);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&pmu_mmdc->cpu));
 }
 
 static struct device_attribute mmdc_pmu_cpumask_attr =
diff --git a/arch/arm/mach-imx/src.c b/arch/arm/mach-imx/src.c
index 59a8e8cc44693..c3c80b4c3d53b 100644
--- a/arch/arm/mach-imx/src.c
+++ b/arch/arm/mach-imx/src.c
@@ -171,6 +171,7 @@ void __init imx_src_init(void)
 	if (!np)
 		return;
 	src_base = of_iomap(np, 0);
+	of_node_put(np);
 	WARN_ON(!src_base);
 
 	/*
@@ -195,6 +196,7 @@ void __init imx7_src_init(void)
 		return;
 
 	src_base = of_iomap(np, 0);
+	of_node_put(np);
 	if (!src_base)
 		return;
 
@@ -203,6 +205,7 @@ void __init imx7_src_init(void)
 		return;
 
 	gpc_base = of_iomap(np, 0);
+	of_node_put(np);
 	if (!gpc_base)
 		return;
 }
diff --git a/arch/arm/mach-imx/system.c b/arch/arm/mach-imx/system.c
index e88ca027129db..1afae485f203f 100644
--- a/arch/arm/mach-imx/system.c
+++ b/arch/arm/mach-imx/system.c
@@ -74,14 +74,6 @@ void __init mxc_arch_reset_init(void __iomem *base)
 		clk_prepare(wdog_clk);
 }
 
-#ifdef CONFIG_SOC_IMX1
-void __init imx1_reset_init(void __iomem *base)
-{
-	wcr_enable = (1 << 0);
-	mxc_arch_reset_init(base);
-}
-#endif
-
 #ifdef CONFIG_CACHE_L2X0
 void __init imx_init_l2cache(void)
 {
diff --git a/arch/arm/mach-ixp4xx/Kconfig b/arch/arm/mach-ixp4xx/Kconfig
index cb46802f5ce52..7f812020e0820 100644
--- a/arch/arm/mach-ixp4xx/Kconfig
+++ b/arch/arm/mach-ixp4xx/Kconfig
@@ -14,6 +14,5 @@ menuconfig ARCH_IXP4XX
 	select IXP4XX_TIMER
 	select USB_EHCI_BIG_ENDIAN_DESC
 	select USB_EHCI_BIG_ENDIAN_MMIO
-	select USE_OF
 	help
 	  Support for Intel's IXP4XX (XScale) family of processors.
diff --git a/arch/arm/mach-mv78xx0/Kconfig b/arch/arm/mach-mv78xx0/Kconfig
index 670e6587827e4..98b35f5d05bac 100644
--- a/arch/arm/mach-mv78xx0/Kconfig
+++ b/arch/arm/mach-mv78xx0/Kconfig
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0-only
 menuconfig ARCH_MV78XX0
-	bool "Marvell MV78xx0"
+	bool "Marvell MV78xx0 (DEPRECATED)"
 	depends on ARCH_MULTI_V5
 	depends on CPU_LITTLE_ENDIAN
 	depends on ATAGS
@@ -14,6 +14,8 @@ menuconfig ARCH_MV78XX0
 	  Support for the following Marvell MV78xx0 series SoCs:
 	  MV781x0, MV782x0.
 
+	  This platform is scheduled for removal in early 2027.
+
 if ARCH_MV78XX0
 
 config MACH_TERASTATION_WXL
diff --git a/arch/arm/mach-omap2/Kconfig b/arch/arm/mach-omap2/Kconfig
index f3f19bcfca2c2..79f8afe3a6bfa 100644
--- a/arch/arm/mach-omap2/Kconfig
+++ b/arch/arm/mach-omap2/Kconfig
@@ -8,8 +8,9 @@ config OMAP_HWMOD
 config ARCH_OMAP2
 	bool "TI OMAP2"
 	depends on ARCH_MULTI_V6
+	depends on !SMP
 	select ARCH_OMAP2PLUS
-	select CPU_V6
+	select CPU_ARM1136R0
 	select OMAP_HWMOD
 	select SOC_HAS_OMAP2_SDRC
 
@@ -206,18 +207,24 @@ comment "OMAP Core Type"
 	depends on ARCH_OMAP2
 
 config SOC_OMAP2420
-	bool "OMAP2420 support"
+	bool "OMAP2420 support (DEPRECATED)"
 	depends on ARCH_OMAP2
 	default y
 	select OMAP_DM_SYSTIMER
 	select OMAP_DM_TIMER
 	select SOC_HAS_OMAP2_SDRC
+	help
+	  This SoC is scheduled for removal in early 2027,
+	  since it uses the ARM1136r0 CPU revision.
 
 config SOC_OMAP2430
-	bool "OMAP2430 support"
+	bool "OMAP2430 support (DEPRECATED)"
 	depends on ARCH_OMAP2
 	default y
 	select SOC_HAS_OMAP2_SDRC
+	help
+	  This SoC is scheduled for removal in early 2027,
+	  since it uses the ARM1136r0 CPU revision.
 
 config SOC_OMAP3430
 	bool "OMAP3430 support"
@@ -248,11 +255,14 @@ config MACH_NOKIA_N810_WIMAX
 	bool
 
 config MACH_NOKIA_N8X0
-	bool "Nokia N800/N810"
+	bool "Nokia N800/N810 (DEPRECATED)"
 	depends on SOC_OMAP2420
 	default y
 	select MACH_NOKIA_N810
 	select MACH_NOKIA_N810_WIMAX
+	help
+	  This machine is scheduled for removal in early 2027,
+	  since it uses the ARM1136r0 CPU revision.
 
 endmenu
 
diff --git a/arch/arm/mach-omap2/omap_hwmod.c b/arch/arm/mach-omap2/omap_hwmod.c
index 974107ff18b4e..03cd523dff874 100644
--- a/arch/arm/mach-omap2/omap_hwmod.c
+++ b/arch/arm/mach-omap2/omap_hwmod.c
@@ -2331,13 +2331,15 @@ static int __init _init(struct omap_hwmod *oh, void *data)
 	if (r < 0) {
 		WARN(1, "omap_hwmod: %s: doesn't have mpu register target base\n",
 		     oh->name);
-		return 0;
+		r = 0;
+		goto out_put_node;
 	}
 
 	r = _init_clocks(oh, np);
 	if (r < 0) {
 		WARN(1, "omap_hwmod: %s: couldn't init clocks\n", oh->name);
-		return -EINVAL;
+		r = -EINVAL;
+		goto out_put_node;
 	}
 
 	if (np) {
@@ -2345,13 +2347,19 @@ static int __init _init(struct omap_hwmod *oh, void *data)
 
 		parse_module_flags(oh, np);
 		child = of_get_next_child(np, NULL);
-		if (child)
+		if (child) {
 			parse_module_flags(oh, child);
+			of_node_put(child);
+		}
 	}
 
 	oh->_state = _HWMOD_STATE_INITIALIZED;
 
-	return 0;
+	r = 0;
+
+out_put_node:
+	of_node_put(bus);
+	return r;
 }
 
 /**
@@ -3608,13 +3616,13 @@ int omap_hwmod_init_module(struct device *dev,
 #ifdef CONFIG_SERIAL_EARLYCON
 static void __init omap_hwmod_setup_earlycon_flags(void)
 {
-	struct device_node *np;
+	struct device_node *np, *chosen;
 	struct omap_hwmod *oh;
 	const char *uart;
 
-	np = of_find_node_by_path("/chosen");
-	if (np) {
-		uart = of_get_property(np, "stdout-path", NULL);
+	chosen = of_find_node_by_path("/chosen");
+	if (chosen) {
+		uart = of_get_property(chosen, "stdout-path", NULL);
 		if (uart) {
 			np = of_find_node_by_path(uart);
 			if (np) {
@@ -3629,8 +3637,10 @@ static void __init omap_hwmod_setup_earlycon_flags(void)
 				if (oh)
 					oh->flags |= DEBUG_OMAPUART_FLAGS;
 			}
+			of_node_put(np);
 		}
 	}
+	of_node_put(chosen);
 }
 #endif
 
diff --git a/arch/arm/mach-omap2/sleep44xx.S b/arch/arm/mach-omap2/sleep44xx.S
index f09c9197808b1..8b515f30072b0 100644
--- a/arch/arm/mach-omap2/sleep44xx.S
+++ b/arch/arm/mach-omap2/sleep44xx.S
@@ -6,6 +6,7 @@
  * 	Santosh Shilimkar <santosh.shilimkar@ti.com>
  */
 
+#include <linux/cfi_types.h>
 #include <linux/linkage.h>
 #include <asm/assembler.h>
 #include <asm/smp_scu.h>
@@ -58,7 +59,7 @@
  * stack frame and it expects the caller to take care of it. Hence the entire
  * stack frame is saved to avoid possible stack corruption.
  */
-ENTRY(omap4_finish_suspend)
+SYM_TYPED_FUNC_START(omap4_finish_suspend)
 	stmfd	sp!, {r4-r12, lr}
 	cmp	r0, #0x0
 	beq	do_WFI				@ No lowpower state, jump to WFI
@@ -223,7 +224,7 @@ skip_scu_gp_clear:
 	isb
 	dsb
 	ldmfd	sp!, {r4-r12, pc}
-ENDPROC(omap4_finish_suspend)
+SYM_FUNC_END(omap4_finish_suspend)
 
 /*
  * ============================
diff --git a/arch/arm/mach-orion5x/Kconfig b/arch/arm/mach-orion5x/Kconfig
index cef19bea61641..c61999d01c806 100644
--- a/arch/arm/mach-orion5x/Kconfig
+++ b/arch/arm/mach-orion5x/Kconfig
@@ -19,7 +19,7 @@ menuconfig ARCH_ORION5X
 if ARCH_ORION5X
 
 config ARCH_ORION5X_DT
-	bool "Marvell Orion5x Flattened Device Tree"
+	def_bool y
 	select ORION_CLK
 	select ORION_IRQCHIP
 	select ORION_TIMER
@@ -38,37 +38,49 @@ config MACH_RD88F5182_DT
 	  Orion-NAS (88F5182) RD2, Flattened Device Tree.
 
 config MACH_KUROBOX_PRO
-	bool "KuroBox Pro"
+	bool "KuroBox Pro (DEPRECATED)"
 	select I2C_BOARDINFO if I2C
 	depends on ATAGS
 	help
 	  Say 'Y' here if you want your kernel to support the
 	  KuroBox Pro platform.
 
+	  Support for this machine will go away in early 2027,
+	  unless it gets converted to device tree.
+
 config MACH_DNS323
-	bool "D-Link DNS-323"
+	bool "D-Link DNS-323 (DEPRECATED)"
 	select I2C_BOARDINFO if I2C
 	depends on ATAGS
 	help
 	  Say 'Y' here if you want your kernel to support the
 	  D-Link DNS-323 platform.
 
+	  Support for this machine will go away in early 2027,
+	  unless it gets converted to device tree.
+
 config MACH_TS209
-	bool "QNAP TS-109/TS-209"
+	bool "QNAP TS-109/TS-209 (DEPRECATED)"
 	depends on ATAGS
 	help
 	  Say 'Y' here if you want your kernel to support the
 	  QNAP TS-109/TS-209 platform.
 
+	  Support for this machine will go away in early 2027,
+	  unless it gets converted to device tree.
+
 config MACH_TERASTATION_PRO2
-	bool "Buffalo Terastation Pro II/Live"
+	bool "Buffalo Terastation Pro II/Live (DEPRECATED)"
 	depends on ATAGS
 	help
 	  Say 'Y' here if you want your kernel to support the
 	  Buffalo Terastation Pro II/Live platform.
 
+	  Support for this machine will go away in early 2027,
+	  unless it gets converted to device tree.
+
 config MACH_LINKSTATION_PRO
-	bool "Buffalo Linkstation Pro/Live"
+	bool "Buffalo Linkstation Pro/Live (DEPRECATED)"
 	depends on ATAGS
 	select I2C_BOARDINFO if I2C
 	help
@@ -76,6 +88,9 @@ config MACH_LINKSTATION_PRO
 	  Buffalo Linkstation Pro/Live platform. Both v1 and
 	  v2 devices are supported.
 
+	  Support for this machine will go away in early 2027,
+	  unless it gets converted to device tree.
+
 config MACH_LINKSTATION_MINI
 	bool "Buffalo Linkstation Mini (Flattened Device Tree)"
 	select ARCH_ORION5X_DT
@@ -84,26 +99,35 @@ config MACH_LINKSTATION_MINI
 	  Buffalo Linkstation Mini (LS-WSGL) platform.
 
 config MACH_TS409
-	bool "QNAP TS-409"
+	bool "QNAP TS-409 (DEPRECATED)"
 	depends on ATAGS
 	help
 	  Say 'Y' here if you want your kernel to support the
 	  QNAP TS-409 platform.
 
+	  Support for this machine will go away in early 2027,
+	  unless it gets converted to device tree.
+
 config MACH_TS78XX
-	bool "Technologic Systems TS-78xx"
+	bool "Technologic Systems TS-78xx (DEPRECATED)"
 	depends on ATAGS
 	help
 	  Say 'Y' here if you want your kernel to support the
 	  Technologic Systems TS-78xx platform.
 
+	  Support for this machine will go away in early 2027,
+	  unless it gets converted to device tree.
+
 config MACH_MV2120
-	bool "HP Media Vault mv2120"
+	bool "HP Media Vault mv2120 (DEPRECATED)"
 	depends on ATAGS
 	help
 	  Say 'Y' here if you want your kernel to support the
 	  HP Media Vault mv2120 or mv5100.
 
+	  Support for this machine will go away in early 2027,
+	  unless it gets converted to device tree.
+
 config MACH_D2NET_DT
 	bool "LaCie d2 Network / Big Disk Network (Flattened Device Tree)"
 	select ARCH_ORION5X_DT
@@ -112,13 +136,16 @@ config MACH_D2NET_DT
 	  LaCie d2 Network NAS.
 
 config MACH_NET2BIG
-	bool "LaCie 2Big Network"
+	bool "LaCie 2Big Network (DEPRECATED)"
 	depends on ATAGS
 	select I2C_BOARDINFO if I2C
 	help
 	  Say 'Y' here if you want your kernel to support the
 	  LaCie 2Big Network NAS.
 
+	  Support for this machine will go away in early 2027,
+	  unless it gets converted to device tree.
+
 config MACH_MSS2_DT
 	bool "Maxtor Shared Storage II (Flattened Device Tree)"
 	select ARCH_ORION5X_DT
diff --git a/arch/arm/mach-pxa/Kconfig b/arch/arm/mach-pxa/Kconfig
index 66e26990e2c8d..2fb29b9ef7446 100644
--- a/arch/arm/mach-pxa/Kconfig
+++ b/arch/arm/mach-pxa/Kconfig
@@ -22,7 +22,6 @@ config MACH_PXA25X_DT
 	select PINCTRL
 	select POWER_SUPPLY
 	select PXA25x
-	select USE_OF
 	help
 	  Include support for Marvell PXA25x based platforms using
 	  the device tree. Needn't select any other machine while
@@ -33,7 +32,6 @@ config MACH_PXA27X_DT
 	select PINCTRL
 	select POWER_SUPPLY
 	select PXA27x
-	select USE_OF
 	help
 	  Include support for Marvell PXA27x based platforms using
 	  the device tree. Needn't select any other machine while
@@ -47,7 +45,6 @@ config MACH_PXA3XX_DT
 	select PINCTRL
 	select POWER_SUPPLY
 	select PXA3xx
-	select USE_OF
 	help
 	  Include support for Marvell PXA3xx based platforms using
 	  the device tree. Needn't select any other machine while
@@ -58,12 +55,15 @@ if ATAGS
 comment "Legacy board files"
 
 config ARCH_GUMSTIX
-	bool "Gumstix XScale 255 boards"
+	bool "Gumstix XScale 255 boards (DEPRECATED)"
 	select PXA25x
 	help
 	  Say Y here if you intend to run this kernel on
 	  Basix, Connex, ws-200ax, ws-400ax systems
 
+	  Support for this machine will go away in early 2027,
+	  unless it gets converted to device tree.
+
 choice
 	prompt "Gumstix Carrier/Expansion Board"
 	depends on ARCH_GUMSTIX
@@ -77,7 +77,7 @@ config GUMSTIX_AM300EPD
 endchoice
 
 config PXA_SHARPSL
-	bool "SHARP Zaurus SL-5600, SL-C7xx and SL-Cxx00 Models"
+	bool "SHARP Zaurus SL-5600, SL-C7xx and SL-Cxx00 Models (DEPRECATED)"
 	select SHARP_PARAM
 	select SHARP_SCOOP
 	help
@@ -87,6 +87,9 @@ config PXA_SHARPSL
 	  SL-C3000 (Spitz), SL-C3100 (Borzoi) or SL-C6000x (Tosa)
 	  handheld computer.
 
+	  Support for these machines will go away in early 2027,
+	  unless they get converted to device tree.
+
 config PXA_SHARPSL_DETECT_MACH_ID
 	bool "Detect machine ID at run-time in the decompressor"
 	depends on PXA_SHARPSL
diff --git a/arch/arm/mach-rpc/Kconfig b/arch/arm/mach-rpc/Kconfig
index 55f6d829b677a..60b3823b690c8 100644
--- a/arch/arm/mach-rpc/Kconfig
+++ b/arch/arm/mach-rpc/Kconfig
@@ -1,5 +1,5 @@
 config ARCH_RPC
-	bool "RiscPC"
+	bool "RiscPC (DEPRECATED)"
 	depends on ARCH_MULTI_V4 && !(ARCH_MULTI_V4T || ARCH_MULTI_V5)
 	depends on !(ARCH_FOOTBRIDGE || ARCH_SA1100 || ARCH_MOXART || ARCH_GEMINI)
 	depends on !CC_IS_CLANG && GCC_VERSION < 90100 && GCC_VERSION >= 60000
@@ -19,3 +19,6 @@ config ARCH_RPC
 	help
 	  On the Acorn Risc-PC, Linux can support the internal IDE disk and
 	  CD-ROM interface, serial and parallel port, and the floppy drive.
+
+	  Support for these machines will go away in 2027,
+	  unless there are any remaining users that speak up.
diff --git a/arch/arm/mach-s3c/Kconfig.s3c64xx b/arch/arm/mach-s3c/Kconfig.s3c64xx
index 3f97fba8e4f52..cf59147dc9ca9 100644
--- a/arch/arm/mach-s3c/Kconfig.s3c64xx
+++ b/arch/arm/mach-s3c/Kconfig.s3c64xx
@@ -23,10 +23,17 @@ menuconfig ARCH_S3C64XX
 	help
 	  Samsung S3C64XX series based systems
 
-	  The platform is deprecated and scheduled for removal. Please reach to
-	  the maintainers of the platform and linux-samsung-soc@vger.kernel.org if
-	  you still use it.
-	  Without such feedback, the platform will be removed after 2024.
+	  The use of ATAGS based board files is on its way out, and
+	  the only remaining S3C64xx machine in the kernel (Wolfson
+	  Cragganmore 6410) relies on this, without any realistic way
+	  to migrate it to devicetree.
+
+	  Support for S3C64XX will be kept until the 2027 LTS kernel
+	  release and may be removed in early 2028.
+
+	  Please reach out to the maintainers of the platform and
+	  linux-samsung-soc@vger.kernel.org if you use any additional
+	  s3c64xx machines based on devicetree.
 
 if ARCH_S3C64XX
 
diff --git a/arch/arm/mach-s3c/map-base.h b/arch/arm/mach-s3c/map-base.h
index 463a995b399bc..beb58e6f12e16 100644
--- a/arch/arm/mach-s3c/map-base.h
+++ b/arch/arm/mach-s3c/map-base.h
@@ -20,7 +20,7 @@
 
 #define S3C_ADDR_BASE	0xF6000000
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #define S3C_ADDR(x)	((void __iomem __force *)S3C_ADDR_BASE + (x))
 #else
 #define S3C_ADDR(x)	(S3C_ADDR_BASE + (x))
diff --git a/arch/arm/mach-sa1100/Kconfig b/arch/arm/mach-sa1100/Kconfig
index e23700e0d6c88..d50721e396419 100644
--- a/arch/arm/mach-sa1100/Kconfig
+++ b/arch/arm/mach-sa1100/Kconfig
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0-only
 menuconfig ARCH_SA1100
-	bool "SA11x0 Implementations"
+	bool "SA11x0 Implementations (DEPRECATED)"
 	depends on ARCH_MULTI_V4 && !(ARCH_MULTI_V4T || ARCH_MULTI_V5)
 	depends on !(ARCH_MOXART || ARCH_GEMINI)
 	depends on ATAGS
@@ -20,6 +20,9 @@ menuconfig ARCH_SA1100
 	help
 	  Support for StrongARM 11x0 based boards.
 
+	  Support for these machines will go away in 2027,
+	  unless there are any remaining users that speak up.
+
 if ARCH_SA1100
 
 config SA1100_ASSABET
diff --git a/arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c b/arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c
index 747a96c58df15..647e5f9f6aa8b 100644
--- a/arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c
+++ b/arch/arm/mach-shmobile/regulator-quirk-rcar-gen2.c
@@ -140,6 +140,10 @@ static struct notifier_block regulator_quirk_nb = {
 
 static int __init rcar_gen2_regulator_quirk(void)
 {
+	static const char *const boards[] __initconst = {
+		"renesas,koelsch", "renesas,lager", "renesas,porter",
+		"renesas,stout", "renesas,gose", NULL
+	};
 	struct regulator_quirk *quirk, *pos, *tmp;
 	struct of_phandle_args *args;
 	const struct of_device_id *id;
@@ -147,11 +151,7 @@ static int __init rcar_gen2_regulator_quirk(void)
 	u32 mon, addr;
 	int ret;
 
-	if (!of_machine_is_compatible("renesas,koelsch") &&
-	    !of_machine_is_compatible("renesas,lager") &&
-	    !of_machine_is_compatible("renesas,porter") &&
-	    !of_machine_is_compatible("renesas,stout") &&
-	    !of_machine_is_compatible("renesas,gose"))
+	if (!of_machine_compatible_match(boards))
 		return -ENODEV;
 
 	for_each_matching_node_and_match(np, rcar_gen2_quirk_match, &id) {
diff --git a/arch/arm/mach-shmobile/setup-rcar-gen2.c b/arch/arm/mach-shmobile/setup-rcar-gen2.c
index 3cd34a42e39bb..5145589059191 100644
--- a/arch/arm/mach-shmobile/setup-rcar-gen2.c
+++ b/arch/arm/mach-shmobile/setup-rcar-gen2.c
@@ -56,6 +56,10 @@ static unsigned int __init get_extal_freq(void)
 
 static void __init rcar_gen2_timer_init(void)
 {
+	static const char *const fixed_freq_socs[] __initconst = {
+		"renesas,r8a7745", "renesas,r8a77470", "renesas,r8a7792",
+		"renesas,r8a7794", NULL
+	};
 	bool need_update = true;
 	void __iomem *base;
 	u32 freq;
@@ -76,10 +80,7 @@ static void __init rcar_gen2_timer_init(void)
 
 	secure_cntvoff_init();
 
-	if (of_machine_is_compatible("renesas,r8a7745") ||
-	    of_machine_is_compatible("renesas,r8a77470") ||
-	    of_machine_is_compatible("renesas,r8a7792") ||
-	    of_machine_is_compatible("renesas,r8a7794")) {
+	if (of_machine_compatible_match(fixed_freq_socs)) {
 		freq = 260000000 / 8;	/* ZS / 8 */
 	} else {
 		/* At Linux boot time the r8a7790 arch timer comes up
diff --git a/arch/arm/mach-stm32/Kconfig b/arch/arm/mach-stm32/Kconfig
index 630b992f32b16..4ac3384e397aa 100644
--- a/arch/arm/mach-stm32/Kconfig
+++ b/arch/arm/mach-stm32/Kconfig
@@ -14,30 +14,32 @@ menuconfig ARCH_STM32
 	select STM32_EXTI if ARM_SINGLE_ARMV7M
 	select STM32_FIREWALL
 	help
-	  Support for STMicroelectronics STM32 processors.
+	  Support for STMicroelectronics STM32MP1 processors using Cortex-A7
+	  cores. Note that support for the earlier Cortex-M based variants
+	  is deprecated and scheduled for removal in early 2027.
 
 if ARCH_STM32
 
 if ARM_SINGLE_ARMV7M
 
 config MACH_STM32F429
-	bool "STMicroelectronics STM32F429"
+	bool "STMicroelectronics STM32F429 (DEPRECATED)"
 	default y
 
 config MACH_STM32F469
-	bool "STMicroelectronics STM32F469"
+	bool "STMicroelectronics STM32F469 (DEPRECATED)"
 	default y
 
 config MACH_STM32F746
-	bool "STMicroelectronics STM32F746"
+	bool "STMicroelectronics STM32F746 (DEPRECATED)"
 	default y
 
 config MACH_STM32F769
-	bool "STMicroelectronics STM32F769"
+	bool "STMicroelectronics STM32F769 (DEPRECATED)"
 	default y
 
 config MACH_STM32H743
-	bool "STMicroelectronics STM32H743"
+	bool "STMicroelectronics STM32H743 (DEPRECATED)"
 	default y
 
 endif # ARMv7-M
diff --git a/arch/arm/mach-ux500/cpu-db8500.c b/arch/arm/mach-ux500/cpu-db8500.c
index b1a70f203372b..0d7530fb6ad07 100644
--- a/arch/arm/mach-ux500/cpu-db8500.c
+++ b/arch/arm/mach-ux500/cpu-db8500.c
@@ -12,7 +12,7 @@
 #include <linux/irq.h>
 #include <linux/irqchip.h>
 #include <linux/irqchip/arm-gic.h>
-#include <linux/mfd/dbx500-prcmu.h>
+#include <linux/mfd/db8500-prcmu.h>
 #include <linux/platform_data/arm-ux500-pm.h>
 #include <linux/platform_device.h>
 #include <linux/io.h>
@@ -81,7 +81,7 @@ static void __init ux500_init_irq(void)
 	struct resource r;
 
 	irqchip_init();
-	prcmu_early_init();
+	db8500_prcmu_early_init();
 	np = of_find_compatible_node(NULL, NULL, "stericsson,db8500-prcmu");
 	of_address_to_resource(np, 0, &r);
 	of_node_put(np);
@@ -101,7 +101,7 @@ static void ux500_restart(enum reboot_mode mode, const char *cmd)
 	local_irq_disable();
 	local_fiq_disable();
 
-	prcmu_system_reset(0);
+	db8500_prcmu_system_reset(0);
 }
 
 static const struct of_device_id u8500_local_bus_nodes[] = {
diff --git a/arch/arm/mach-versatile/Kconfig b/arch/arm/mach-versatile/Kconfig
index 5136180784401..1ee9b6608d712 100644
--- a/arch/arm/mach-versatile/Kconfig
+++ b/arch/arm/mach-versatile/Kconfig
@@ -110,10 +110,14 @@ config INTEGRATOR_CM1026EJS
 	select CPU_ARM1026
 
 config INTEGRATOR_CM1136JFS
-	bool "Integrator/CM1136JF-S core module"
+	bool "Integrator/CM1136JF-S core module (DEPRECATED)"
 	depends on ARCH_INTEGRATOR_AP
 	depends on ARCH_MULTI_V6
-	select CPU_V6
+	depends on !SMP
+	select CPU_ARM1136R0
+	help
+	  This machine is scheduled for removal in early 2027,
+	  since it uses the ARM1136r0 CPU revision.
 
 config ARCH_INTEGRATOR_CP
 	bool "Support Integrator/CP platform"
@@ -135,7 +139,7 @@ config INTEGRATOR_CTB36
 	bool "Integrator/CTB36 (ARM1136JF-S) core tile"
 	depends on ARCH_INTEGRATOR_CP
 	depends on ARCH_MULTI_V6
-	select CPU_V6
+	select CPU_ARM1136R1
 
 config ARCH_CINTEGRATOR
 	depends on ARCH_INTEGRATOR_CP
@@ -182,7 +186,7 @@ config MACH_REALVIEW_EB
 config REALVIEW_EB_ARM1136
 	bool "Support ARM1136J(F)-S Tile"
 	depends on MACH_REALVIEW_EB && ARCH_MULTI_V6
-	select CPU_V6
+	select CPU_ARM1136R1
 	help
 	  Enable support for the ARM1136 tile fitted to the
 	  Realview(R) Emulation Baseboard platform.
@@ -201,11 +205,10 @@ config REALVIEW_EB_A9MP
 	  Enable support for the Cortex-A9MPCore tile fitted to the
 	  Realview(R) Emulation Baseboard platform.
 
-# ARMv6 CPU without K extensions, but does have the new exclusive ops
 config MACH_REALVIEW_PB1176
 	bool "Support RealView(R) Platform Baseboard for ARM1176JZF-S"
 	depends on ARCH_MULTI_V6
-	select CPU_V6
+	select CPU_ARM1176
 	select HAVE_TCM
 	help
 	  Include support for the ARM(R) RealView(R) Platform Baseboard for
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index 871bd58d2ccc9..6574659dfb61f 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -376,6 +376,7 @@ config CPU_PJ4B
 # ARMv6
 config CPU_V6
 	bool
+	depends on !SMP
 	select CPU_32v6
 	select CPU_ABRT_EV6
 	select CPU_CACHE_V6
@@ -386,7 +387,6 @@ config CPU_V6
 	select CPU_PABRT_V6
 	select CPU_THUMB_CAPABLE
 	select CPU_TLB_V6 if MMU
-	select SMP_ON_UP if SMP
 
 # ARMv6k
 config CPU_V6K
@@ -403,6 +403,30 @@ config CPU_V6K
 	select CPU_THUMB_CAPABLE
 	select CPU_TLB_V6 if MMU
 
+config CPU_ARM1136R0
+	bool
+	select CPU_V6
+	depends on !SMP
+	help
+	  These early revisions of ARM1136 lack support for the
+	  ARMv6k extensions for multiprocessing.
+	  Support for this revision is scheduled for removal
+	  from the kernel in early 2027.
+
+config CPU_ARM1136R1
+	bool
+	select CPU_V6K
+	help
+	  Later revisions of ARM1136 add ARMv6k (atomics, barriers
+	  and TLS register) in addition to the features from r0.
+
+config CPU_ARM1176
+	bool
+	select CPU_V6K
+	help
+	  ARM1176 implements ARMv6k, VMSAv7 and Trustzone in
+	  addition to the ARMv6 baseline.
+
 # ARMv7 and ARMv8 architectures
 config CPU_V7
 	bool
@@ -754,7 +778,8 @@ config CPU_LITTLE_ENDIAN
 	  space builds.
 
 config CPU_BIG_ENDIAN
-	bool "Build big-endian kernel"
+	bool "Build big-endian kernel (DEPRECATED)"
+	depends on ARCH_MULTI_V5 || BROKEN
 	depends on !LD_IS_LLD
 	help
 	  Say Y if you plan on running a kernel in big-endian mode.
@@ -924,7 +949,6 @@ config VDSO
 	bool "Enable VDSO for acceleration of some system calls"
 	depends on AEABI && MMU && CPU_V7
 	default y if ARM_ARCH_TIMER
-	select HAVE_GENERIC_VDSO
 	select GENERIC_GETTIMEOFDAY
 	help
 	  Place in the process address space an ELF shared object
diff --git a/arch/arm/mm/cache-l2x0-pmu.c b/arch/arm/mm/cache-l2x0-pmu.c
index 3d9caf7464bff..4782270788374 100644
--- a/arch/arm/mm/cache-l2x0-pmu.c
+++ b/arch/arm/mm/cache-l2x0-pmu.c
@@ -390,7 +390,7 @@ static struct attribute_group l2x0_pmu_event_attrs_group = {
 static ssize_t l2x0_pmu_cpumask_show(struct device *dev,
 				     struct device_attribute *attr, char *buf)
 {
-	return cpumap_print_to_pagebuf(true, buf, &pmu_cpu);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&pmu_cpu));
 }
 
 static struct device_attribute l2x0_pmu_cpumask_attr =
diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c
index 91e488767783e..2769b0f3325f2 100644
--- a/arch/arm/mm/fault-armv.c
+++ b/arch/arm/mm/fault-armv.c
@@ -132,7 +132,7 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma,
 	pgoff_t pgoff;
 	int aliases = 0;
 
-	pgoff = vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT);
+	pgoff = linear_page_index(vma, addr);
 
 	/*
 	 * If we have any shared mappings that are in the same mm
@@ -140,7 +140,7 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma,
 	 * cache coherency.
 	 */
 	flush_dcache_mmap_lock(mapping);
-	vma_interval_tree_foreach(mpnt, &mapping->i_mmap, pgoff, pgoff) {
+	mapping_rmap_tree_foreach(mpnt, mapping, pgoff, pgoff) {
 		/*
 		 * If we are using split PTE locks, then we need to take the pte
 		 * lock. Otherwise we are using shared mm->page_table_lock which
diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c
index 4d7ef5cc36b66..66270904f874a 100644
--- a/arch/arm/mm/flush.c
+++ b/arch/arm/mm/flush.c
@@ -251,7 +251,7 @@ static void __flush_dcache_aliases(struct address_space *mapping, struct folio *
 	pgoff_end = pgoff + folio_nr_pages(folio) - 1;
 
 	flush_dcache_mmap_lock(mapping);
-	vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff_end) {
+	mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff_end) {
 		unsigned long start, offset, pfn;
 		unsigned int nr;
 
diff --git a/arch/arm/mm/kasan_init.c b/arch/arm/mm/kasan_init.c
index c6625e808bf85..1f7c74c5df9e9 100644
--- a/arch/arm/mm/kasan_init.c
+++ b/arch/arm/mm/kasan_init.c
@@ -262,12 +262,6 @@ void __init kasan_init(void)
 				&pa_start, &pa_end, &arm_lowmem_limit);
 			end = __va(arm_lowmem_limit);
 		}
-		if (start >= end) {
-			pr_info("Skipping invalid memory block %pa-%pa (virtual %p-%p)\n",
-				&pa_start, &pa_end, start, end);
-			continue;
-		}
-
 		create_mapping(start, end);
 	}
 
diff --git a/arch/arm/probes/kprobes/test-core.c b/arch/arm/probes/kprobes/test-core.c
index 7a2baa135f0f1..2de28088dac89 100644
--- a/arch/arm/probes/kprobes/test-core.c
+++ b/arch/arm/probes/kprobes/test-core.c
@@ -1649,24 +1649,16 @@ static int __init run_all_tests(void)
 
 	return ret;
 }
-
+late_initcall(run_all_tests);
 
 /*
  * Module setup
  */
 
-#ifdef MODULE
-
 static void __exit kprobe_test_exit(void)
 {
 }
-
-module_init(run_all_tests)
 module_exit(kprobe_test_exit)
-MODULE_LICENSE("GPL");
 
-#else /* !MODULE */
-
-late_initcall(run_all_tests);
-
-#endif
+MODULE_DESCRIPTION("Test code for ARM kprobes");
+MODULE_LICENSE("GPL");
diff --git a/arch/arm/vdso/vdso.lds.S b/arch/arm/vdso/vdso.lds.S
index 74d8d8bc8a40d..e61038c0195a6 100644
--- a/arch/arm/vdso/vdso.lds.S
+++ b/arch/arm/vdso/vdso.lds.S
@@ -70,9 +70,11 @@ VERSION
 {
 	LINUX_2.6 {
 	global:
+#ifdef CONFIG_COMPAT_32BIT_TIME
 		__vdso_clock_gettime;
 		__vdso_gettimeofday;
 		__vdso_clock_getres;
+#endif /* CONFIG_COMPAT_32BIT_TIME */
 		__vdso_clock_gettime64;
 		__vdso_clock_getres_time64;
 	local: *;
diff --git a/arch/arm/vdso/vgettimeofday.c b/arch/arm/vdso/vgettimeofday.c
index f7a2f5dc2fdcb..3eebeddbfd183 100644
--- a/arch/arm/vdso/vgettimeofday.c
+++ b/arch/arm/vdso/vgettimeofday.c
@@ -10,16 +10,17 @@
 #include <asm/unwind.h>
 #include <vdso/gettime.h>
 
+#ifdef CONFIG_COMPAT_32BIT_TIME
 int __vdso_clock_gettime(clockid_t clock,
 			 struct old_timespec32 *ts)
 {
 	return __cvdso_clock_gettime32(clock, ts);
 }
 
-int __vdso_clock_gettime64(clockid_t clock,
-			   struct __kernel_timespec *ts)
+int __vdso_clock_getres(clockid_t clock_id,
+			struct old_timespec32 *res)
 {
-	return __cvdso_clock_gettime(clock, ts);
+	return __cvdso_clock_getres_time32(clock_id, res);
 }
 
 int __vdso_gettimeofday(struct __kernel_old_timeval *tv,
@@ -27,11 +28,12 @@ int __vdso_gettimeofday(struct __kernel_old_timeval *tv,
 {
 	return __cvdso_gettimeofday(tv, tz);
 }
+#endif /* CONFIG_COMPAT_32BIT_TIME */
 
-int __vdso_clock_getres(clockid_t clock_id,
-			struct old_timespec32 *res)
+int __vdso_clock_gettime64(clockid_t clock,
+			   struct __kernel_timespec *ts)
 {
-	return __cvdso_clock_getres_time32(clock_id, res);
+	return __cvdso_clock_gettime(clock, ts);
 }
 
 int __vdso_clock_getres_time64(clockid_t clock_id, struct __kernel_timespec *res)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index b3afe0688919b..fb15037b5e3c7 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -17,7 +17,7 @@ config ARM64
 	select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION
 	select ARCH_ENABLE_MEMORY_HOTPLUG
 	select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2
-	select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
+	select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE
 	select ARCH_HAS_CACHE_LINE_SIZE
 	select ARCH_HAS_CC_PLATFORM
 	select ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION
@@ -78,6 +78,7 @@ config ARM64
 	select ARCH_SUPPORTS_CFI
 	select ARCH_SUPPORTS_ATOMIC_RMW
 	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128
+	select ARCH_HAS_PTE_PROTNONE
 	select ARCH_SUPPORTS_NUMA_BALANCING
 	select ARCH_SUPPORTS_PAGE_TABLE_CHECK
 	select ARCH_SUPPORTS_PER_VMA_LOCK
@@ -229,7 +230,6 @@ config ARM64
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_KPROBES
 	select HAVE_KRETPROBES
-	select HAVE_GENERIC_VDSO
 	select HOTPLUG_CORE_SYNC_DEAD if HOTPLUG_CPU
 	select HOTPLUG_SMT if HOTPLUG_CPU
 	select IRQ_DOMAIN
@@ -1382,6 +1382,15 @@ config NVIDIA_CARMEL_CNP_ERRATUM
 
 	  If unsure, say Y.
 
+config RENESAS_ERRATUM_GEN4GICITS1
+	bool "Renesas R-Car Gen4: GIC600 can not access physical addresses above 4 GiB"
+	default y
+	help
+	  The Renesas R-Car Gen4 S4/V4H/V4M GIC600 SoC integrations have AXI
+	  addressing limited to the first 32-bit of physical address space.
+
+	  If unsure, say Y.
+
 config ROCKCHIP_ERRATUM_3568002
 	bool "Rockchip 3568002: GIC600 can not access physical addresses higher than 4GB"
 	default y
diff --git a/arch/arm64/boot/dts/allwinner/Makefile b/arch/arm64/boot/dts/allwinner/Makefile
index 53e6b701e7d3a..aa21f58a4be1f 100644
--- a/arch/arm64/boot/dts/allwinner/Makefile
+++ b/arch/arm64/boot/dts/allwinner/Makefile
@@ -24,6 +24,7 @@ dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a64-sopine-baseboard.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a64-teres-i.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h64-remix-mini-pc.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a100-allwinner-perf1.dtb
+dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a133-helperboard.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a133-liontron-h-a133l.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h5-bananapi-m2-plus.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h5-bananapi-m2-plus-v1.2.dtb
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a100.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-a100.dtsi
index b3fb1e0ee796b..ba6020989ce97 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-a100.dtsi
+++ b/arch/arm64/boot/dts/allwinner/sun50i-a100.dtsi
@@ -466,6 +466,16 @@ ths: thermal-sensor@5070400 {
 			#thermal-sensor-cells = <1>;
 		};
 
+		lradc: lradc@5070800 {
+			compatible = "allwinner,sun50i-a100-lradc",
+				     "allwinner,sun50i-r329-lradc";
+			reg = <0x05070800 0x400>;
+			interrupts = <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_BUS_LRADC>;
+			resets = <&ccu RST_BUS_LRADC>;
+			status = "disabled";
+		};
+
 		usb_otg: usb@5100000 {
 			compatible = "allwinner,sun50i-a100-musb",
 				     "allwinner,sun8i-a33-musb";
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a133-helperboard-core.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-a133-helperboard-core.dtsi
new file mode 100644
index 0000000000000..545972d2324a9
--- /dev/null
+++ b/arch/arm64/boot/dts/allwinner/sun50i-a133-helperboard-core.dtsi
@@ -0,0 +1,197 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2025 Arm Ltd.
+ * Copyright (c) 2026 Alexander Sverdlin <alexander.sverdlin@gmail.com>
+ */
+
+/dts-v1/;
+
+#include "sun50i-a100.dtsi"
+#include "sun50i-a100-cpu-opp.dtsi"
+
+/{
+	model = "Baijie A133 HelperBoard Core";
+	compatible = "baijie,helperboard-a133-core",
+		     "allwinner,sun50i-a100";
+
+	aliases {
+		serial1 = &uart1;	/* BT module */
+	};
+};
+
+&cpu0 {
+	cpu-supply = <&reg_dcdc2>;
+};
+
+&lradc {
+	vref-supply = <&reg_aldo1>;
+};
+
+&mmc2 {
+	vmmc-supply = <&reg_dcdc1>;
+	vqmmc-supply = <&reg_eldo1>;
+	cap-mmc-highspeed;
+	cap-mmc-hw-reset;
+	max-frequency = <100000000>;
+	non-removable;
+	bus-width = <8>;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+	status = "okay";
+};
+
+&pio {
+	vcc-pb-supply = <&reg_dcdc1>;
+	vcc-pc-supply = <&reg_eldo1>;
+	vcc-pd-supply = <&reg_dcdc1>;
+	vcc-pe-supply = <&reg_dldo2>;
+	vcc-pf-supply = <&reg_dcdc1>;
+	vcc-pg-supply = <&reg_dldo1>;
+	vcc-ph-supply = <&reg_dcdc1>;
+	/*
+	 * PL0/PL1 are the I2C connection to PMIC, but it would create a
+	 * circular dependency:
+	 * vcc-pl-supply = <&reg_aldo3>;
+	 */
+};
+
+&r_i2c0 {
+	status = "okay";
+
+	axp803: pmic@34 {
+		compatible = "x-powers,axp803";
+		reg = <0x34>;
+		interrupt-parent = <&r_intc>;
+		interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
+		x-powers,drive-vbus-en;		/* set N_VBUSEN as output pin */
+		aldoin-supply = <&reg_vdd5v>;
+		dldoin-supply = <&reg_vdd5v>;
+		eldoin-supply = <&reg_vdd5v>;
+		fldoin-supply = <&reg_dcdc5>;
+		vin1-supply = <&reg_vdd5v>;
+		vin2-supply = <&reg_vdd5v>;
+		vin3-supply = <&reg_vdd5v>;
+		vin4-supply = <&reg_vdd5v>;
+		vin5-supply = <&reg_vdd5v>;
+		vin6-supply = <&reg_vdd5v>;
+		drivevbus-supply = <&reg_vdd5v>;
+	};
+};
+
+#include "axp803.dtsi"
+
+&ac_power_supply {
+	status = "okay";
+};
+
+&reg_aldo1 {
+	/* PLL + LRADC analog reference */
+	regulator-always-on;
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-name = "vcc-pll";
+};
+
+&reg_aldo2 {
+	/* LPDDR */
+	regulator-always-on;
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-name = "vdd18-lpddr";
+};
+
+&reg_aldo3 {
+	/*
+	 * Port L, but linking it to &pio node would create a circular
+	 * dependency because of PL0/PL1 I2C connection to PMIC
+	 */
+	regulator-always-on;
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-name = "vcc-pl";
+};
+
+&reg_dcdc1 {
+	/* Besides Port D it also powers analog part of USB IP and SoC I/O */
+	regulator-always-on;
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-3v3";
+};
+
+&reg_dcdc2 {
+	regulator-always-on;
+	regulator-min-microvolt = <810000>;
+	regulator-max-microvolt = <1200000>;
+	regulator-name = "vdd-cpu";
+};
+
+/* DCDC3 is polyphased with DCDC2 */
+
+&reg_dcdc4 {
+	/* Digital part of USB IP, "System" SoC power rail */
+	regulator-always-on;
+	regulator-min-microvolt = <950000>;
+	regulator-max-microvolt = <950000>;
+	regulator-name = "vdd-sys";
+};
+
+&reg_dcdc5 {
+	regulator-always-on;
+	regulator-min-microvolt = <1100000>;
+	regulator-max-microvolt = <1100000>;
+	regulator-name = "vcc-dram";
+};
+
+/* DCDC6 unused */
+&reg_dcdc6 {
+	status = "disabled";
+};
+
+&reg_dldo1 {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-name = "vcc-pg";
+};
+
+&reg_dldo2 {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-name = "vcc-pe";
+};
+
+&reg_dldo3 {
+	regulator-min-microvolt = <2800000>;
+	regulator-max-microvolt = <2800000>;
+	regulator-name = "avdd-csi";
+};
+
+&reg_dldo4 {
+	regulator-min-microvolt = <2800000>;
+	regulator-max-microvolt = <2800000>;
+	regulator-name = "afvcc-csi";
+};
+
+&reg_eldo1 {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-name = "vcc-pc";
+};
+
+&reg_eldo2 {
+	regulator-min-microvolt = <1200000>;
+	regulator-max-microvolt = <1200000>;
+	regulator-name = "dvdd-csi";
+};
+
+/* ELDO3 unused */
+
+&reg_fldo1 {
+	/* CPUS power rail */
+	regulator-always-on;
+	regulator-min-microvolt = <900000>;
+	regulator-max-microvolt = <900000>;
+	regulator-name = "vdd-cpus";
+};
+
+/* reg_drivevbus unused */
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a133-helperboard.dts b/arch/arm64/boot/dts/allwinner/sun50i-a133-helperboard.dts
new file mode 100644
index 0000000000000..694c0cacf9064
--- /dev/null
+++ b/arch/arm64/boot/dts/allwinner/sun50i-a133-helperboard.dts
@@ -0,0 +1,148 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2025 Arm Ltd.
+ * Copyright (c) 2026 Alexander Sverdlin <alexander.sverdlin@gmail.com>
+ */
+
+/dts-v1/;
+
+#include "sun50i-a133-helperboard-core.dtsi"
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/linux-event-codes.h>
+#include <dt-bindings/leds/common.h>
+
+/{
+	model = "Baijie HelperBoard A133";
+	compatible = "baijie,helperboard-a133",
+		     "baijie,helperboard-a133-core",
+		     "allwinner,sun50i-a100";
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led {
+			function = LED_FUNCTION_INDICATOR;
+			color = <LED_COLOR_ID_GREEN>;
+			gpios = <&pio 7 13 GPIO_ACTIVE_LOW>;	/* PH13 */
+		};
+	};
+
+	reg_vdd5v: vdd5v {
+		/* board wide 5V supply from a 12V->5V regulator */
+		compatible = "regulator-fixed";
+		regulator-name = "vdd-5v";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+	};
+};
+
+&ehci1 {
+	status = "okay";
+};
+
+&emac0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&rgmii0_pins>;
+	phy-handle = <&rgmii_phy>;
+	phy-mode = "rgmii-id";
+	allwinner,rx-delay-ps = <200>;
+	allwinner,tx-delay-ps = <200>;
+	status = "okay";
+};
+
+&lradc {
+	wakeup-source;
+	status = "okay";
+
+	button-115 {
+		label = "K1";
+		linux,code = <KEY_1>;
+		channel = <0>;
+		voltage = <114607>;
+	};
+
+	button-235 {
+		label = "K2";
+		linux,code = <KEY_2>;
+		channel = <0>;
+		voltage = <234783>;
+	};
+
+	button-360 {
+		label = "K3";
+		linux,code = <KEY_3>;
+		channel = <0>;
+		voltage = <360000>;
+	};
+
+	button-476 {
+		label = "K4";
+		linux,code = <KEY_4>;
+		channel = <0>;
+		voltage = <476471>;
+	};
+
+	button-592 {
+		label = "K5";
+		linux,code = <KEY_5>;
+		channel = <0>;
+		voltage = <591946>;
+	};
+};
+
+&mdio0 {
+	reset-gpios = <&pio 7 11 GPIO_ACTIVE_LOW>;	/* PH11 */
+	reset-delay-us = <10000>;
+	reset-post-delay-us = <150000>;
+
+	rgmii_phy: ethernet-phy@1 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <1>;
+	};
+};
+
+&mmc0 {
+	vmmc-supply = <&reg_dcdc1>;
+	cd-gpios = <&pio 5 6 GPIO_ACTIVE_LOW>;	/* PF6 */
+	bus-width = <4>;
+	disable-wp;
+	status = "okay";
+};
+
+&ohci1 {
+	status = "okay";
+};
+
+&rgmii0_pins {
+	drive-strength = <30>;
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_pb_pins>;
+	status = "okay";
+};
+
+&usb_otg {
+	/*
+	 * Connected to a downstream port of an onboard hub, therefore only
+	 * "peripheral" mode will work here.
+	 */
+	dr_mode = "peripheral";
+	status = "okay";
+};
+
+&usbphy {
+	usb1_vbus-supply = <&reg_vdd5v>;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/amlogic/amlogic-a9.dtsi b/arch/arm64/boot/dts/amlogic/amlogic-a9.dtsi
index 660c8556a864d..b0e0fadeed820 100644
--- a/arch/arm64/boot/dts/amlogic/amlogic-a9.dtsi
+++ b/arch/arm64/boot/dts/amlogic/amlogic-a9.dtsi
@@ -6,6 +6,7 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/pinctrl/amlogic,pinctrl.h>
 
 / {
 	interrupt-parent = <&gic>;
@@ -97,6 +98,95 @@ soc {
 		#size-cells = <2>;
 		ranges;
 
+		apb: bus@fe000000 {
+			compatible = "simple-bus";
+			reg = <0x0 0xfe000000 0x0 0x480000>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges = <0x0 0x0 0x0 0xfe000000 0x0 0x480000>;
+
+			periphs_pinctrl: pinctrl@4000 {
+				compatible = "amlogic,pinctrl-a9";
+				#address-cells = <2>;
+				#size-cells = <2>;
+				ranges = <0x0 0x0 0x0 0x4000 0x0 0x340>;
+
+				gpioz: gpio@c0 {
+					reg = <0 0xc0 0 0x20>, <0 0x18 0 0x8>;
+					reg-names = "gpio", "mux";
+					gpio-controller;
+					#gpio-cells = <2>;
+					gpio-ranges = <&periphs_pinctrl 0 (AMLOGIC_GPIO_Z<<8) 16>;
+				};
+
+				gpiox: gpio@100 {
+					reg = <0 0x100 0 0x24>, <0 0xc 0 0x8>;
+					reg-names = "gpio", "mux";
+					gpio-controller;
+					#gpio-cells = <2>;
+					gpio-ranges = <&periphs_pinctrl 0 (AMLOGIC_GPIO_X<<8) 18>;
+				};
+
+				gpioh: gpio@140 {
+					reg = <0 0x140 0 0x20>, <0 0x2c 0 0x4>;
+					reg-names = "gpio", "mux";
+					gpio-controller;
+					#gpio-cells = <2>;
+					gpio-ranges = <&periphs_pinctrl 0 (AMLOGIC_GPIO_H<<8) 8>;
+				};
+
+				gpiom: gpio@1a0 {
+					reg = <0 0x1a0 0 0x20>, <0 0x20 0 0x4>;
+					reg-names = "gpio", "mux";
+					gpio-controller;
+					#gpio-cells = <2>;
+					gpio-ranges = <&periphs_pinctrl 0 (AMLOGIC_GPIO_M<<8) 8>;
+				};
+
+				gpiob: gpio@240 {
+					reg = <0 0x240 0 0x20>, <0 0x0 0 0x8>;
+					reg-names = "gpio", "mux";
+					gpio-controller;
+					#gpio-cells = <2>;
+					gpio-ranges = <&periphs_pinctrl 0 (AMLOGIC_GPIO_B<<8) 14>;
+				};
+
+				gpioa: gpio@280 {
+					reg = <0 0x280 0 0x24>, <0 0x40 0 0xc>;
+					reg-names = "gpio", "mux";
+					gpio-controller;
+					#gpio-cells = <2>;
+					gpio-ranges = <&periphs_pinctrl 0 (AMLOGIC_GPIO_A<<8) 20>;
+				};
+
+				gpioy: gpio@2c0 {
+					reg = <0 0x2c0 0 0x20>, <0 0x30 0 0x8>;
+					reg-names = "gpio", "mux";
+					gpio-controller;
+					#gpio-cells = <2>;
+					gpio-ranges = <&periphs_pinctrl 0 (AMLOGIC_GPIO_Y<<8) 10>;
+				};
+
+				gpiocc: gpio@300 {
+					reg = <0 0x300 0 0x20>, <0 0x14 0 0x4>;
+					reg-names = "gpio", "mux";
+					gpio-controller;
+					#gpio-cells = <2>;
+					gpio-ranges = <&periphs_pinctrl 0 (AMLOGIC_GPIO_CC<<8) 2>;
+				};
+			};
+
+			gpio_intc: interrupt-controller@4080 {
+				compatible = "amlogic,a9-gpio-intc",
+					     "amlogic,meson-gpio-intc";
+				reg = <0x0 0x4080 0x0 0x20>;
+				interrupt-controller;
+				#interrupt-cells = <2>;
+				amlogic,channel-interrupts =
+					<10 11 12 13 14 15 16 17 18 19 20 21>;
+			};
+		};
+
 		gic: interrupt-controller@ff800000 {
 			compatible = "arm,gic-v3";
 			#interrupt-cells = <3>;
@@ -114,6 +204,56 @@ aobus: bus@ffa00000 {
 			#size-cells = <2>;
 			ranges = <0x0 0x0 0x0 0xffa00000 0x0 0x100000>;
 
+			aobus_pinctrl: pinctrl@4000 {
+				compatible = "amlogic,pinctrl-a9";
+				#address-cells = <2>;
+				#size-cells = <2>;
+				ranges = <0x0 0x0 0x0 0x4000 0x0 0x0e0>;
+
+				gpioao: gpio@1c {
+					reg = <0 0x1c 0 0x20>, <0 0x0 0 0x8>;
+					reg-names = "gpio", "mux";
+					gpio-controller;
+					#gpio-cells = <2>;
+					gpio-ranges = <&aobus_pinctrl 0 (AMLOGIC_GPIO_AO<<8) 13>;
+				};
+
+				gpioc: gpio@3c {
+					reg = <0 0x3c 0 0x20>, <0 0x10 0 0x4>;
+					reg-names = "gpio", "mux";
+					gpio-controller;
+					#gpio-cells = <2>;
+					gpio-ranges = <&aobus_pinctrl 0 (AMLOGIC_GPIO_C<<8) 7>;
+				};
+
+				gpiod: gpio@5c {
+					reg = <0 0x5c 0 0x24>, <0 0x8 0 0x8>;
+					reg-names = "gpio", "mux";
+					gpio-controller;
+					#gpio-cells = <2>;
+					gpio-ranges = <&aobus_pinctrl 0 (AMLOGIC_GPIO_D<<8) 18>;
+				};
+
+				test_n: gpio@c0 {
+					reg = <0 0xc0 0 0x20>;
+					reg-names = "gpio";
+					gpio-controller;
+					#gpio-cells = <2>;
+					gpio-ranges = <&aobus_pinctrl 0 (AMLOGIC_GPIO_TEST_N<<8) 1>;
+				};
+			};
+
+			gpio_ao_intc: interrupt-controller@4080 {
+				compatible = "amlogic,a9-gpio-ao-intc",
+					     "amlogic,meson-gpio-intc";
+				reg = <0x0 0x4080 0x0 0x34>;
+				interrupt-controller;
+				#interrupt-cells = <2>;
+				amlogic,channel-interrupts =
+					<384 385 386 387 388 389 390 391 392 393
+					394 395 396 397 398 399 400 401 402 403>;
+			};
+
 			uart_b: serial@1e000 {
 				compatible = "amlogic,a9-uart",
 					     "amlogic,meson-s4-uart";
diff --git a/arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi b/arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi
index cc371fcd18967..c3dc479b137de 100644
--- a/arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi
+++ b/arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi
@@ -674,6 +674,11 @@ pwm_ao_gh: pwm@32000 {
 				status = "disabled";
 			};
 
+			clock-measurer@48000 {
+				compatible = "amlogic,t7-clk-measure";
+				reg = <0x0 0x48000 0x0 0x1c>;
+			};
+
 			pwm_ab: pwm@58000 {
 				compatible = "amlogic,t7-pwm", "amlogic,meson-s4-pwm";
 				reg = <0x0 0x58000 0x0 0x24>;
diff --git a/arch/arm64/boot/dts/amlogic/meson-a1.dtsi b/arch/arm64/boot/dts/amlogic/meson-a1.dtsi
index 348411411f3d1..6f6a6145cba12 100644
--- a/arch/arm64/boot/dts/amlogic/meson-a1.dtsi
+++ b/arch/arm64/boot/dts/amlogic/meson-a1.dtsi
@@ -576,6 +576,11 @@ saradc: adc@2c00 {
 				status = "disabled";
 			};
 
+			clock-measurer@3400 {
+				compatible = "amlogic,a1-clk-measure";
+				reg = <0x0 0x3400 0x0 0x1c>;
+			};
+
 			i2c1: i2c@5c00 {
 				compatible = "amlogic,meson-axg-i2c";
 				status = "disabled";
diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi
index f715b19efd167..3271d234b483a 100644
--- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi
+++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi
@@ -24,6 +24,13 @@ aic: interrupt-controller@28e100000 {
 		power-domains = <&ps_aic>;
 	};
 
+	pmgr_misc: power-management@28e20c000 {
+		compatible = "apple,t6000-pmgr-misc";
+		reg = <0x2 0x8e20c000 0 0x400>,
+			<0x2 0x8e20c800 0 0x400>;
+		reg-names = "fabric-ps", "dcs-ps";
+	};
+
 	smc: smc@290400000 {
 		compatible = "apple,t6000-smc", "apple,smc";
 		reg = <0x2 0x90400000 0x0 0x4000>,
diff --git a/arch/arm64/boot/dts/apple/t602x-die0.dtsi b/arch/arm64/boot/dts/apple/t602x-die0.dtsi
index 8622ddea7b445..121f2cf759152 100644
--- a/arch/arm64/boot/dts/apple/t602x-die0.dtsi
+++ b/arch/arm64/boot/dts/apple/t602x-die0.dtsi
@@ -23,6 +23,13 @@ aic: interrupt-controller@28e100000 {
 		power-domains = <&ps_aic>;
 	};
 
+	pmgr_misc: power-management@28e20c000 {
+		compatible = "apple,t6020-pmgr-misc";
+		reg = <0x2 0x8e20c000 0 0x400>,
+			<0x2 0x8e20c400 0 0x400>;
+		reg-names = "fabric-ps", "dcs-ps";
+	};
+
 	nub_spmi0: spmi@29e114000 {
 		compatible = "apple,t6020-spmi", "apple,t8103-spmi";
 		reg = <0x2 0x9e114000 0x0 0x100>;
diff --git a/arch/arm64/boot/dts/apple/t8122.dtsi b/arch/arm64/boot/dts/apple/t8122.dtsi
index c6196225e96ef..1ee61c5b3409f 100644
--- a/arch/arm64/boot/dts/apple/t8122.dtsi
+++ b/arch/arm64/boot/dts/apple/t8122.dtsi
@@ -186,12 +186,12 @@ soc {
 		/* Required to get >32-bit DMA via DARTs */
 		dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>;
 
-		i2c0: i2c@235010000 {
+		i2c0: i2c@2a1010000 {
 			compatible = "apple,t8122-i2c", "apple,t8103-i2c";
-			reg = <0x2 0x35010000 0x0 0x4000>;
+			reg = <0x2 0xa1010000 0x0 0x4000>;
 			clocks = <&clkref>;
 			interrupt-parent = <&aic>;
-			interrupts = <AIC_IRQ 761 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <AIC_IRQ 780 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&i2c0_pins>;
 			pinctrl-names = "default";
 			#address-cells = <0x1>;
@@ -200,12 +200,12 @@ i2c0: i2c@235010000 {
 			status = "disabled";
 		};
 
-		i2c1: i2c@235014000 {
+		i2c1: i2c@2a1014000 {
 			compatible = "apple,t8122-i2c", "apple,t8103-i2c";
-			reg = <0x2 0x35014000 0x0 0x4000>;
+			reg = <0x2 0xa1014000 0x0 0x4000>;
 			clocks = <&clkref>;
 			interrupt-parent = <&aic>;
-			interrupts = <AIC_IRQ 762 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <AIC_IRQ 781 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&i2c1_pins>;
 			pinctrl-names = "default";
 			#address-cells = <0x1>;
@@ -214,12 +214,12 @@ i2c1: i2c@235014000 {
 			status = "disabled";
 		};
 
-		i2c2: i2c@235018000 {
+		i2c2: i2c@2a1018000 {
 			compatible = "apple,t8122-i2c", "apple,t8103-i2c";
-			reg = <0x2 0x35018000 0x0 0x4000>;
+			reg = <0x2 0xa1018000 0x0 0x4000>;
 			clocks = <&clkref>;
 			interrupt-parent = <&aic>;
-			interrupts = <AIC_IRQ 763 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <AIC_IRQ 782 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&i2c2_pins>;
 			pinctrl-names = "default";
 			#address-cells = <0x1>;
@@ -228,12 +228,12 @@ i2c2: i2c@235018000 {
 			status = "disabled";
 		};
 
-		i2c3: i2c@23501c000 {
+		i2c3: i2c@2a101c000 {
 			compatible = "apple,t8122-i2c", "apple,t8103-i2c";
-			reg = <0x2 0x3501c000 0x0 0x4000>;
+			reg = <0x2 0xa101c000 0x0 0x4000>;
 			clocks = <&clkref>;
 			interrupt-parent = <&aic>;
-			interrupts = <AIC_IRQ 764 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <AIC_IRQ 783 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&i2c3_pins>;
 			pinctrl-names = "default";
 			#address-cells = <0x1>;
@@ -242,12 +242,12 @@ i2c3: i2c@23501c000 {
 			status = "disabled";
 		};
 
-		i2c4: i2c@235020000 {
+		i2c4: i2c@2a1020000 {
 			compatible = "apple,t8122-i2c", "apple,t8103-i2c";
-			reg = <0x2 0x35020000 0x0 0x4000>;
+			reg = <0x2 0xa1020000 0x0 0x4000>;
 			clocks = <&clkref>;
 			interrupt-parent = <&aic>;
-			interrupts = <AIC_IRQ 765 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <AIC_IRQ 784 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&i2c4_pins>;
 			pinctrl-names = "default";
 			#address-cells = <0x1>;
diff --git a/arch/arm64/boot/dts/exynos/Makefile b/arch/arm64/boot/dts/exynos/Makefile
index 76cc23acb9b29..8c48ce2e02e51 100644
--- a/arch/arm64/boot/dts/exynos/Makefile
+++ b/arch/arm64/boot/dts/exynos/Makefile
@@ -14,6 +14,7 @@ dtb-$(CONFIG_ARCH_EXYNOS) += \
 	exynos7870-on7xelte.dtb		\
 	exynos7885-jackpotlte.dtb	\
 	exynos850-e850-96.dtb		\
+	exynos8855-smdk.dtb		\
 	exynos8895-dreamlte.dtb		\
 	exynos9810-starlte.dtb		\
 	exynos990-c1s.dtb		\
diff --git a/arch/arm64/boot/dts/exynos/exynos8855-pinctrl.dtsi b/arch/arm64/boot/dts/exynos/exynos8855-pinctrl.dtsi
new file mode 100644
index 0000000000000..b65f1698cd74a
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/exynos8855-pinctrl.dtsi
@@ -0,0 +1,574 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Samsung's S5E8855 SoC pin-mux and pin-config device tree source
+ *
+ * Copyright (c) 2023 Samsung Electronics Co., Ltd.
+ *		http://www.samsung.com
+ *
+ * Samsung's S5E8855 SoC pin-mux and pin-config options are listed as device
+ * tree nodes are listed in this file.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include "exynos-pinctrl.h"
+
+&pinctrl_alive {
+	gpa0: gpa0-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpa1: gpa1-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpq0: gpq0-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+
+	gpq1: gpq1-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+
+	gpc0: gpc0-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpc1: gpc1-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpc2: gpc2-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpc3: gpc3-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpc4: gpc4-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpc5: gpc5-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpc6: gpc6-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpc7: gpc7-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 20 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpc8: gpc8-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpc9: gpc9-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpc10: gpc10-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpc11: gpc11-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpc12: gpc12-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpc13: gpc13-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpc14: gpc14-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpj0: gpj0-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 28 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpj1: gpj1-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpj2: gpj2-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+	};
+};
+
+&pinctrl_cmgp {
+	gpm0: gpm0-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm1: gpm1-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm2: gpm2-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm3: gpm3-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm4: gpm4-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm5: gpm5-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm6: gpm6-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm7: gpm7-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm8: gpm8-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm9: gpm9-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm10: gpm10-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 124 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm11: gpm11-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm12: gpm12-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 126 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm13: gpm13-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm14: gpm14-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 128 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm15: gpm15-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 129 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm16: gpm16-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 130 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm17: gpm17-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm18: gpm18-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 132 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm19: gpm19-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 133 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm20: gpm20-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 134 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	gpm21: gpm21-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 135 IRQ_TYPE_LEVEL_HIGH>;
+	};
+};
+
+&pinctrl_hsi_ufs {
+	gpf3: gpf3-gpio-bank{
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+};
+
+&pinctrl_peric {
+	gpp0: gpp0-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpp1: gpp1-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpp2: gpp2-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpg0: gpg0-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpg1: gpg1-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpb0: gpb0-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpb1: gpb1-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+};
+
+&pinctrl_pericmmc {
+	gpf2: gpf2-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+};
+
+&pinctrl_usi {
+	gpp3: gpp3-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpp4: gpp4-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpg2: gpg2-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	gpg3: gpg3-gpio-bank {
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+};
diff --git a/arch/arm64/boot/dts/exynos/exynos8855-smdk.dts b/arch/arm64/boot/dts/exynos/exynos8855-smdk.dts
new file mode 100644
index 0000000000000..e74ff8a1bbcf6
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/exynos8855-smdk.dts
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Samsung Exynos8855 SMDK board device tree source
+ *
+ * Copyright (C) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Device tree source file for Samsung SMDK board which is based on
+ * Exynos8855 SoC.
+ */
+
+/dts-v1/;
+
+#include "exynos8855.dtsi"
+
+/ {
+	model = "Samsung Exynos8855 SMDK board";
+	compatible = "samsung,exynos8855-smdk","samsung,exynos8855";
+
+	chosen {
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 0x80000000>;
+	};
+
+};
+
+&oscclk {
+	clock-frequency = <76800000>;
+};
diff --git a/arch/arm64/boot/dts/exynos/exynos8855.dtsi b/arch/arm64/boot/dts/exynos/exynos8855.dtsi
new file mode 100644
index 0000000000000..9d82fceb5130a
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/exynos8855.dtsi
@@ -0,0 +1,204 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Samsung Exynos8855 SoC device tree source
+ *
+ * Copyright (C) 2023 Samsung Electronics Co., Ltd.
+ *
+ * Samsung Exynos8855 SoC device nodes are listed in this file.
+ * Exynos8855 based board files can include this file and provide
+ * values for board specific bindings.
+ */
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+/ {
+	compatible = "samsung,exynos8855";
+	#address-cells = <2>;
+	#size-cells = <1>;
+
+	interrupt-parent = <&gic>;
+
+	aliases {
+		pinctrl0 = &pinctrl_alive;
+		pinctrl1 = &pinctrl_cmgp;
+		pinctrl2 = &pinctrl_hsi_ufs;
+		pinctrl3 = &pinctrl_peric;
+		pinctrl4 = &pinctrl_pericmmc;
+		pinctrl5 = &pinctrl_usi;
+	};
+
+	oscclk: clock-oscclk {
+		compatible = "fixed-clock";
+		clock-output-names = "oscclk";
+		#clock-cells = <0>;
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&cpu0>;
+				};
+				core1 {
+					cpu = <&cpu1>;
+				};
+				core2 {
+					cpu = <&cpu2>;
+				};
+				core3 {
+					cpu = <&cpu3>;
+				};
+			};
+
+			cluster1 {
+				core0 {
+					cpu = <&cpu4>;
+				};
+				core1 {
+					cpu = <&cpu5>;
+				};
+				core2 {
+					cpu = <&cpu6>;
+				};
+			};
+
+			cluster2 {
+				core0 {
+					cpu = <&cpu7>;
+				};
+			};
+		};
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a520";
+			reg = <0x0>;
+			enable-method = "psci";
+		};
+
+		cpu1: cpu@100 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a520";
+			reg = <0x100>;
+			enable-method = "psci";
+		};
+
+		cpu2: cpu@200 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a520";
+			reg = <0x200>;
+			enable-method = "psci";
+		};
+
+		cpu3: cpu@300 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a520";
+			reg = <0x300>;
+			enable-method = "psci";
+		};
+
+		cpu4: cpu@400 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a720";
+			reg = <0x400>;
+			enable-method = "psci";
+		};
+
+		cpu5: cpu@500 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a720";
+			reg = <0x500>;
+			enable-method = "psci";
+		};
+
+		cpu6: cpu@600 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a720";
+			reg = <0x600>;
+			enable-method = "psci";
+		};
+
+		cpu7: cpu@700 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a720";
+			reg = <0x700>;
+			enable-method = "psci";
+		};
+	};
+
+	psci {
+		compatible = "arm,psci-1.0";
+		method = "smc";
+	};
+
+	soc: soc@0 {
+		compatible = "simple-bus";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x0 0x0 0x0 0x20000000>;
+
+		gic: interrupt-controller@10200000 {
+			compatible = "arm,gic-v3";
+			#interrupt-cells = <3>;
+			interrupt-controller;
+			reg = <0x10200000 0x10000>,
+			      <0x10240000 0x140000>;
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pinctrl_alive: pinctrl@11850000 {
+			compatible = "samsung,exynos8855-pinctrl";
+			reg = <0x11850000 0x1000>;
+
+			wakeup-interrupt-controller {
+				compatible = "samsung,exynos8855-wakeup-eint",
+					     "samsung,exynos7-wakeup-eint";
+			};
+		};
+
+		pinctrl_cmgp: pinctrl@12030000 {
+			compatible = "samsung,exynos8855-pinctrl";
+			reg = <0x12030000 0x1000>;
+			interrupts = <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pinctrl_usi: pinctrl@15030000 {
+			compatible = "samsung,exynos8855-pinctrl";
+			reg = <0x15030000 0x1000>;
+			interrupts = <GIC_SPI 440 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pinctrl_peric: pinctrl@15440000 {
+			compatible = "samsung,exynos8855-pinctrl";
+			reg = <0x15440000 0x1000>;
+			interrupts = <GIC_SPI 394 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pinctrl_pericmmc: pinctrl@154f0000 {
+			compatible = "samsung,exynos8855-pinctrl";
+			reg = <0x154f0000 0x1000>;
+			interrupts = <GIC_SPI 395 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pinctrl_hsi_ufs: pinctrl@17040000 {
+			compatible = "samsung,exynos8855-pinctrl";
+			reg = <0x17040000 0x1000>;
+			interrupts = <GIC_SPI 318 IRQ_TYPE_LEVEL_HIGH>;
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		/* Hypervisor Virtual Timer interrupt is not wired to GIC */
+		interrupts =
+		     <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
+		     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
+		     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
+		     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>;
+	};
+};
+
+#include "exynos8855-pinctrl.dtsi"
diff --git a/arch/arm64/boot/dts/exynos/exynosautov920.dtsi b/arch/arm64/boot/dts/exynos/exynosautov920.dtsi
index f1f5efcdb91ea..e2f962dc1b3d1 100644
--- a/arch/arm64/boot/dts/exynos/exynosautov920.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynosautov920.dtsi
@@ -1382,6 +1382,12 @@ cmu_hsi0: clock-controller@16000000 {
 				      "noc";
 		};
 
+		syscon_hsi0: syscon@16030000 {
+			compatible = "samsung,exynosautov920-hsi0-sysreg",
+				     "syscon";
+			reg = <0x16030000 0x1000>;
+		};
+
 		pinctrl_hsi0: pinctrl@16040000 {
 			compatible = "samsung,exynosautov920-pinctrl";
 			reg = <0x16040000 0x10000>;
diff --git a/arch/arm64/boot/dts/freescale/Makefile b/arch/arm64/boot/dts/freescale/Makefile
index 8ddaab127ab9c..be3c858f1eeb1 100644
--- a/arch/arm64/boot/dts/freescale/Makefile
+++ b/arch/arm64/boot/dts/freescale/Makefile
@@ -166,6 +166,7 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mm-prt8mm.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-tqma8mqml-mba8mx.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-tx8m-1610-moduline-iv-306-d.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-tx8m-1610-moduline-mini-111.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8mm-var-dart-sonata.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-var-som-symphony.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-var-som-symphony-legacy.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw71xx-0x.dtb
@@ -361,6 +362,10 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mp-dhcom-picoitx.dtb \
 
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-edm-g-wb.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-evk.dtb
+
+imx8mp-evk-flexcan2-dtbs += imx8mp-evk.dtb imx8mp-evk-flexcan2.dtbo
+dtb-$(CONFIG_ARCH_MXC) += imx8mp-evk-flexcan2.dtb
+
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-frdm.dtb
 DTC_FLAGS_imx8mp-hummingboard-iiot := -@
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-hummingboard-iiot.dtb
@@ -439,6 +444,7 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mp-skov-revc-jutouch-jt101tm023.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-toradex-smarc-dev.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-tqma8mpql-mba8mpxl.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-tqma8mpql-mba8mp-ras314.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8mp-tqma8mpqs-mb-smarc-2.dtb
 
 imx8mp-tx8p-ml81-moduline-display-106-av101hdt-a10-dtbs += imx8mp-tx8p-ml81-moduline-display-106.dtb \
 	imx8mp-tx8p-ml81-moduline-display-106-av101hdt-a10.dtbo
@@ -448,6 +454,7 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mp-tx8p-ml81-moduline-display-106-av101hdt-a10.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-tx8p-ml81-moduline-display-106-av123z7m-n17.dtb
 
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-ultra-mach-sbc.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8mp-var-dart-sonata.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-var-som-symphony.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-venice-gw71xx-2x.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-venice-gw72xx-2x.dtb
@@ -497,6 +504,13 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mp-tqma8mpql-mba8mp-ras314-imx219.dtbo
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-tqma8mpql-mba8mp-ras314-lvds-tm070jvhg33.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-tqma8mpql-mba8mp-ras314-lvds-tm070jvhg33-imx219.dtb
 
+imx8mp-tqma8mpqs-mb-smarc-2-lvds-g133han01-dtbs += imx8mp-tqma8mpqs-mb-smarc-2.dtb imx8mp-tqma8mpqs-mb-smarc-2-lvds-g133han01.dtbo
+imx8mp-tqma8mpqs-mb-smarc-2-lvds0-tm070jvhg33-dtbs += imx8mp-tqma8mpqs-mb-smarc-2.dtb imx8mp-tqma8mpqs-mb-smarc-2-lvds0-tm070jvhg33.dtbo
+imx8mp-tqma8mpqs-mb-smarc-2-lvds1-tm070jvhg33-dtbs += imx8mp-tqma8mpqs-mb-smarc-2.dtb imx8mp-tqma8mpqs-mb-smarc-2-lvds1-tm070jvhg33.dtbo
+dtb-$(CONFIG_ARCH_MXC) += imx8mp-tqma8mpqs-mb-smarc-2-lvds-g133han01.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8mp-tqma8mpqs-mb-smarc-2-lvds0-tm070jvhg33.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8mp-tqma8mpqs-mb-smarc-2-lvds1-tm070jvhg33.dtb
+
 dtb-$(CONFIG_ARCH_MXC) += imx8mq-evk.dtb
 imx8mq-evk-pcie1-ep-dtbs += imx8mq-evk.dtb imx-pcie1-ep.dtbo
 dtb-$(CONFIG_ARCH_MXC) += imx8mq-evk-pcie1-ep.dtb
@@ -588,6 +602,10 @@ dtb-$(CONFIG_ARCH_MXC) += imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-9x9-qsb-tianma-tm050rdh03.dtb
 
 dtb-$(CONFIG_ARCH_MXC) += imx93-11x11-evk.dtb
+
+imx93-11x11-evk-dy1212w-4856-dtbs += imx93-11x11-evk.dtb imx93-11x11-evk-dy1212w-4856.dtbo
+dtb-$(CONFIG_ARCH_MXC) += imx93-11x11-evk-dy1212w-4856.dtb
+
 dtb-$(CONFIG_ARCH_MXC) += imx93-11x11-frdm.dtb
 
 imx93-11x11-frdm-pixpaper-dtbs += imx93-11x11-frdm.dtb imx93-11x11-frdm-pixpaper.dtbo
@@ -625,9 +643,15 @@ imx93-tqma9352-mba91xxca-rgb-cdtech-dc44-dtbs := imx93-tqma9352-mba91xxca.dtb im
 dtb-$(CONFIG_ARCH_MXC) += imx93-tqma9352-mba91xxca-lvds-tm070jvhg33.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-tqma9352-mba91xxca-rgb-cdtech-dc44.dtb
 
+imx93-tqma9352-mba93xxca-lvds-tm070jvhg33-dtbs += imx93-tqma9352-mba93xxca.dtb imx93-tqma9352-mba93xxla-lvds-tm070jvhg33.dtbo
+imx93-tqma9352-mba93xxla-lvds-tm070jvhg33-dtbs += imx93-tqma9352-mba93xxla.dtb imx93-tqma9352-mba93xxla-lvds-tm070jvhg33.dtbo
 imx93-tqma9352-mba93xxla-mini-ezurio-wlan-dtbs += imx93-tqma9352-mba93xxla-mini.dtb imx93-tqma9352-mba93xxla-mini-ezurio-wlan.dtbo
+imx93-tqma9352-mba93xxla-mini-lvds-tm070jvhg33-dtbs += imx93-tqma9352-mba93xxla-mini.dtb imx93-tqma9352-mba93xxla-lvds-tm070jvhg33.dtbo
 
+dtb-$(CONFIG_ARCH_MXC) += imx93-tqma9352-mba93xxca-lvds-tm070jvhg33.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx93-tqma9352-mba93xxla-lvds-tm070jvhg33.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-tqma9352-mba93xxla-mini-ezurio-wlan.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx93-tqma9352-mba93xxla-mini-lvds-tm070jvhg33.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-var-dart-sonata.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-var-som-symphony.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93w-evk.dtb
diff --git a/arch/arm64/boot/dts/freescale/imx8-apalis-v1.1.dtsi b/arch/arm64/boot/dts/freescale/imx8-apalis-v1.1.dtsi
index 6fc82b5eb58c1..681f24810d52e 100644
--- a/arch/arm64/boot/dts/freescale/imx8-apalis-v1.1.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8-apalis-v1.1.dtsi
@@ -423,7 +423,7 @@ usb-hub@8 {
 		refclk-frequency = <25000000>;
 		reset-gpios = <&lsio_gpio1 2 GPIO_ACTIVE_LOW>;
 	};
-	
+
 	/* On Module Audio Codec */
 	sgtl5000: audio-codec@a {
 		compatible = "fsl,sgtl5000";
diff --git a/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts b/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts
index 78e8d41e67916..0ede0c7ed422c 100644
--- a/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts
@@ -677,6 +677,7 @@ &pcie0 {
 	pinctrl-names = "default";
 	/* This property is deprecated, use reset-gpios from the Root Port node. */
 	reset-gpio = <&lsio_gpio4 0 GPIO_ACTIVE_LOW>;
+	/* This property is deprecated, use power supply from the Root Port node. */
 	vpcie-supply = <&reg_pcieb>;
 	vpcie3v3aux-supply = <&reg_pcieb>;
 	status = "okay";
@@ -692,6 +693,8 @@ &pcie0_ep {
 
 &pcieb_port0 {
 	reset-gpios = <&lsio_gpio4 0 GPIO_ACTIVE_LOW>;
+	vpcie3v3-supply = <&reg_pcieb>;
+	vpcie3v3aux-supply = <&reg_pcieb>;
 };
 
 &sai0 {
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-emtop-baseboard.dts b/arch/arm64/boot/dts/freescale/imx8mm-emtop-baseboard.dts
index 87fe3ebedb8d6..b031a9308ddaf 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-emtop-baseboard.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mm-emtop-baseboard.dts
@@ -35,9 +35,9 @@ leds {
 		pinctrl-0 = <&pinctrl_gpio_led>;
 
 		led-1 {
-		        label = "buzzer";
-		        gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>;
-		        default-state = "off";
+			label = "buzzer";
+			gpios = <&gpio4 29 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
 		};
 	};
 
@@ -49,11 +49,11 @@ osc_can: clock-osc-can {
 	};
 
 	reg_audio: regulator-audio {
-	        compatible = "regulator-fixed";
-	        regulator-name = "wm8904_supply";
-	        regulator-min-microvolt = <1800000>;
-	        regulator-max-microvolt = <1800000>;
-	        regulator-always-on;
+		compatible = "regulator-fixed";
+		regulator-name = "wm8904_supply";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
 	};
 
 	reg_wifi_vmmc: regulator-wifi-vmmc {
@@ -68,13 +68,13 @@ reg_wifi_vmmc: regulator-wifi-vmmc {
 	};
 
 	sound-wm8904 {
-	        compatible = "simple-audio-card";
-	        simple-audio-card,bitclock-master = <&dailink_master>;
-	        simple-audio-card,format = "i2s";
-	        simple-audio-card,frame-master = <&dailink_master>;
-	        simple-audio-card,name = "wm8904-audio";
-	        simple-audio-card,mclk-fs = <256>;
-	        simple-audio-card,routing =
+		compatible = "simple-audio-card";
+		simple-audio-card,bitclock-master = <&dailink_master>;
+		simple-audio-card,format = "i2s";
+		simple-audio-card,frame-master = <&dailink_master>;
+		simple-audio-card,name = "wm8904-audio";
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,routing =
 			"Headphone Jack", "HPOUTL",
 			"Headphone Jack", "HPOUTR",
 			"IN2L", "Line In Jack",
@@ -82,45 +82,44 @@ sound-wm8904 {
 			"Headphone Jack", "MICBIAS",
 			"IN1L", "Headphone Jack";
 
-	        simple-audio-card,widgets =
-	                "Microphone","Headphone Jack",
-	                "Headphone", "Headphone Jack",
-	                "Line", "Line In Jack";
+		simple-audio-card,widgets = "Microphone","Headphone Jack",
+					    "Headphone", "Headphone Jack",
+					    "Line", "Line In Jack";
 
-	        dailink_master: simple-audio-card,codec {
-	                sound-dai = <&wm8904>;
-	        };
+		dailink_master: simple-audio-card,codec {
+			sound-dai = <&wm8904>;
+		};
 
-	        simple-audio-card,cpu {
-	                sound-dai = <&sai3>;
-	        };
+		simple-audio-card,cpu {
+			sound-dai = <&sai3>;
+		};
 	};
 
 	sound-spdif {
-	        compatible = "fsl,imx-audio-spdif";
-	        model = "imx-spdif";
-	        spdif-controller = <&spdif1>;
-	        spdif-out;
-	        spdif-in;
+		compatible = "fsl,imx-audio-spdif";
+		model = "imx-spdif";
+		spdif-controller = <&spdif1>;
+		spdif-out;
+		spdif-in;
 	};
 };
 
 /* CAN BUS */
 &ecspi2 {
-        pinctrl-names = "default";
-        pinctrl-0 = <&pinctrl_ecspi2>;
-        status = "okay";
-
-        can: can@0 {
-                compatible = "microchip,mcp2515";
-                reg = <0>;
-                pinctrl-names = "default";
-                pinctrl-0 = <&pinctrl_canbus>;
-                clocks = <&osc_can>;
-                interrupt-parent = <&gpio1>;
-                interrupts = <14 IRQ_TYPE_LEVEL_HIGH>;
-                spi-max-frequency = <10000000>;
-        };
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi2>;
+	status = "okay";
+
+	can: can@0 {
+		compatible = "microchip,mcp2515";
+		reg = <0>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_canbus>;
+		clocks = <&osc_can>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <14 IRQ_TYPE_LEVEL_HIGH>;
+		spi-max-frequency = <10000000>;
+	};
 };
 
 &fec1 {
@@ -177,29 +176,29 @@ rtc@32 {
 
 /* AUDIO */
 &sai3 {
-        pinctrl-names = "default";
-        pinctrl-0 = <&pinctrl_sai3>;
-        assigned-clocks = <&clk IMX8MM_CLK_SAI3>;
-        assigned-clock-parents = <&clk IMX8MM_AUDIO_PLL1_OUT>;
-        assigned-clock-rates = <24576000>;
-        status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sai3>;
+	assigned-clocks = <&clk IMX8MM_CLK_SAI3>;
+	assigned-clock-parents = <&clk IMX8MM_AUDIO_PLL1_OUT>;
+	assigned-clock-rates = <24576000>;
+	status = "okay";
 };
 
 &spdif1 {
-        pinctrl-names = "default";
-        pinctrl-0 = <&pinctrl_spdif1>;
-        assigned-clocks = <&clk IMX8MM_CLK_SPDIF1>;
-        assigned-clock-parents = <&clk IMX8MM_AUDIO_PLL1_OUT>;
-        assigned-clock-rates = <24576000>;
-        clocks = <&clk IMX8MM_CLK_AUDIO_AHB>, <&clk IMX8MM_CLK_24M>,
-                <&clk IMX8MM_CLK_SPDIF1>, <&clk IMX8MM_CLK_DUMMY>,
-                <&clk IMX8MM_CLK_DUMMY>, <&clk IMX8MM_CLK_DUMMY>,
-                <&clk IMX8MM_CLK_AUDIO_AHB>, <&clk IMX8MM_CLK_DUMMY>,
-                <&clk IMX8MM_CLK_DUMMY>, <&clk IMX8MM_CLK_DUMMY>,
-                <&clk IMX8MM_AUDIO_PLL1_OUT>, <&clk IMX8MM_AUDIO_PLL2_OUT>;
-        clock-names = "core", "rxtx0", "rxtx1", "rxtx2", "rxtx3",
-                "rxtx4", "rxtx5", "rxtx6", "rxtx7", "spba", "pll8k", "pll11k";
-        status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spdif1>;
+	assigned-clocks = <&clk IMX8MM_CLK_SPDIF1>;
+	assigned-clock-parents = <&clk IMX8MM_AUDIO_PLL1_OUT>;
+	assigned-clock-rates = <24576000>;
+	clocks = <&clk IMX8MM_CLK_AUDIO_AHB>, <&clk IMX8MM_CLK_24M>,
+		<&clk IMX8MM_CLK_SPDIF1>, <&clk IMX8MM_CLK_DUMMY>,
+		<&clk IMX8MM_CLK_DUMMY>, <&clk IMX8MM_CLK_DUMMY>,
+		<&clk IMX8MM_CLK_AUDIO_AHB>, <&clk IMX8MM_CLK_DUMMY>,
+		<&clk IMX8MM_CLK_DUMMY>, <&clk IMX8MM_CLK_DUMMY>,
+		<&clk IMX8MM_AUDIO_PLL1_OUT>, <&clk IMX8MM_AUDIO_PLL2_OUT>;
+	clock-names = "core", "rxtx0", "rxtx1", "rxtx2", "rxtx3",
+		"rxtx4", "rxtx5", "rxtx6", "rxtx7", "spba", "pll8k", "pll11k";
+	status = "okay";
 };
 
 /* USBOTG */
@@ -216,8 +215,8 @@ usb_hs_ep: endpoint {
 };
 
 &usbotg2 {
-        dr_mode = "host";
-        status = "okay";
+	dr_mode = "host";
+	status = "okay";
 };
 
 /* Wifi */
@@ -246,26 +245,26 @@ wifi: wifi@1 {
 
 /* SD-card */
 &usdhc2 {
-        pinctrl-names = "default";
-        pinctrl-0 = <&pinctrl_usdhc2>;
-        pinctrl-1 = <&pinctrl_usdhc2_100mhz>;
-        pinctrl-2 = <&pinctrl_usdhc2_200mhz>;
-        cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>;
-        bus-width = <4>;
-        status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc2>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>;
+	cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>;
+	bus-width = <4>;
+	status = "okay";
 };
 
 &iomuxc {
 
 	pinctrl_canbus: canbusgrp {
-	        fsl,pins = <
-		        MX8MM_IOMUXC_GPIO1_IO14_GPIO1_IO14              0x14
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO14_GPIO1_IO14		0x14
 		>;
 	};
 
 	pinctrl_ecspi2: ecspi2grp {
 		fsl,pins = <
-			MX8MM_IOMUXC_ECSPI2_SS0_ECSPI2_SS0  		0x82
+			MX8MM_IOMUXC_ECSPI2_SS0_ECSPI2_SS0		0x82
 			MX8MM_IOMUXC_ECSPI2_MOSI_ECSPI2_MOSI		0x82
 			MX8MM_IOMUXC_ECSPI2_MISO_ECSPI2_MISO		0x82
 			MX8MM_IOMUXC_ECSPI2_SCLK_ECSPI2_SCLK		0x82
@@ -274,125 +273,125 @@ MX8MM_IOMUXC_ECSPI2_SCLK_ECSPI2_SCLK		0x82
 
 	pinctrl_usb_otg: usbotggrp {
 		fsl,pins = <
-			MX8MM_IOMUXC_GPIO1_IO10_GPIO1_IO10		0x140   /* otg_id */
-			MX8MM_IOMUXC_GPIO1_IO12_GPIO1_IO12		0x19    /* otg_vbus */
+			MX8MM_IOMUXC_GPIO1_IO10_GPIO1_IO10		0x140	/* otg_id */
+			MX8MM_IOMUXC_GPIO1_IO12_GPIO1_IO12		0x19	/* otg_vbus */
 		>;
 	};
 
 	pinctrl_fec1: fec1grp {
 		fsl,pins = <
-			MX8MM_IOMUXC_ENET_MDC_ENET1_MDC                 0x3
-			MX8MM_IOMUXC_ENET_MDIO_ENET1_MDIO               0x3
-			MX8MM_IOMUXC_ENET_TD3_ENET1_RGMII_TD3           0x1f
-			MX8MM_IOMUXC_ENET_TD2_ENET1_RGMII_TD2           0x1f
-			MX8MM_IOMUXC_ENET_TD1_ENET1_RGMII_TD1           0x1f
-			MX8MM_IOMUXC_ENET_TD0_ENET1_RGMII_TD0           0x1f
-			MX8MM_IOMUXC_ENET_RD3_ENET1_RGMII_RD3           0x91
-			MX8MM_IOMUXC_ENET_RD2_ENET1_RGMII_RD2           0x91
-			MX8MM_IOMUXC_ENET_RD1_ENET1_RGMII_RD1           0x91
-			MX8MM_IOMUXC_ENET_RD0_ENET1_RGMII_RD0           0x91
-			MX8MM_IOMUXC_ENET_TXC_ENET1_RGMII_TXC           0x1f
-			MX8MM_IOMUXC_ENET_RXC_ENET1_RGMII_RXC           0x91
-			MX8MM_IOMUXC_ENET_RX_CTL_ENET1_RGMII_RX_CTL     0x91
-			MX8MM_IOMUXC_ENET_TX_CTL_ENET1_RGMII_TX_CTL     0x1f
-			MX8MM_IOMUXC_SAI2_RXC_GPIO4_IO22                0x19
+			MX8MM_IOMUXC_ENET_MDC_ENET1_MDC			0x3
+			MX8MM_IOMUXC_ENET_MDIO_ENET1_MDIO		0x3
+			MX8MM_IOMUXC_ENET_TD3_ENET1_RGMII_TD3		0x1f
+			MX8MM_IOMUXC_ENET_TD2_ENET1_RGMII_TD2		0x1f
+			MX8MM_IOMUXC_ENET_TD1_ENET1_RGMII_TD1		0x1f
+			MX8MM_IOMUXC_ENET_TD0_ENET1_RGMII_TD0		0x1f
+			MX8MM_IOMUXC_ENET_RD3_ENET1_RGMII_RD3		0x91
+			MX8MM_IOMUXC_ENET_RD2_ENET1_RGMII_RD2		0x91
+			MX8MM_IOMUXC_ENET_RD1_ENET1_RGMII_RD1		0x91
+			MX8MM_IOMUXC_ENET_RD0_ENET1_RGMII_RD0		0x91
+			MX8MM_IOMUXC_ENET_TXC_ENET1_RGMII_TXC		0x1f
+			MX8MM_IOMUXC_ENET_RXC_ENET1_RGMII_RXC		0x91
+			MX8MM_IOMUXC_ENET_RX_CTL_ENET1_RGMII_RX_CTL	0x91
+			MX8MM_IOMUXC_ENET_TX_CTL_ENET1_RGMII_TX_CTL	0x1f
+			MX8MM_IOMUXC_SAI2_RXC_GPIO4_IO22		0x19
 		>;
 	};
 
 	pinctrl_i2c3: i2c3grp {
-	        fsl,pins = <
-	                MX8MM_IOMUXC_I2C3_SCL_I2C3_SCL                  0x400001c3
-	                MX8MM_IOMUXC_I2C3_SDA_I2C3_SDA                  0x400001c3
-	        >;
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C3_SCL_I2C3_SCL		0x400001c3
+			MX8MM_IOMUXC_I2C3_SDA_I2C3_SDA		0x400001c3
+		>;
 	};
 
 	pinctrl_sai3: sai3grp {
-	        fsl,pins = <
-	                MX8MM_IOMUXC_SAI3_TXFS_SAI3_TX_SYNC             0xd6
-	                MX8MM_IOMUXC_SAI3_TXC_SAI3_TX_BCLK              0xd6
-	                MX8MM_IOMUXC_SAI3_MCLK_SAI3_MCLK                0xd6
-	                MX8MM_IOMUXC_SAI3_TXD_SAI3_TX_DATA0             0xd6
-	                MX8MM_IOMUXC_SAI3_RXD_SAI3_RX_DATA0             0xd6
-	        >;
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI3_TXFS_SAI3_TX_SYNC		0xd6
+			MX8MM_IOMUXC_SAI3_TXC_SAI3_TX_BCLK		0xd6
+			MX8MM_IOMUXC_SAI3_MCLK_SAI3_MCLK		0xd6
+			MX8MM_IOMUXC_SAI3_TXD_SAI3_TX_DATA0		0xd6
+			MX8MM_IOMUXC_SAI3_RXD_SAI3_RX_DATA0		0xd6
+		>;
 	};
 
 	pinctrl_spdif1: spdif1grp {
-	        fsl,pins = <
-	                MX8MM_IOMUXC_SPDIF_TX_SPDIF1_OUT                0xd6
-	        >;
+		fsl,pins = <
+			MX8MM_IOMUXC_SPDIF_TX_SPDIF1_OUT		0xd6
+		>;
 	};
 
 	pinctrl_usdhc1: usdhc1grp {
 		fsl,pins = <
-			MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK                 0x190
-			MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD                 0x1d0
-			MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0             0x1d0
-			MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1             0x1d0
-			MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2             0x1d0
-			MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3             0x1d0
+			MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK			0x190
+			MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD			0x1d0
+			MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0		0x1d0
+			MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1		0x1d0
+			MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2		0x1d0
+			MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3		0x1d0
 		>;
 	};
 
 	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
 		fsl,pins = <
-			MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK         	0x194
-			MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD         	0x1d4
-			MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0     	0x1d4
-			MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1     	0x1d4
-			MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2     	0x1d4
-			MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3     	0x1d4
+			MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK			0x194
+			MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD			0x1d4
+			MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0		0x1d4
+			MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1		0x1d4
+			MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2		0x1d4
+			MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3		0x1d4
 		>;
 	};
 
 	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
 		fsl,pins = <
-			MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK         	0x196
-			MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD         	0x1d6
-			MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0     	0x1d6
-			MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1     	0x1d6
-			MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2     	0x1d6
-			MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3     	0x1d6
+			MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK			0x196
+			MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD			0x1d6
+			MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0		0x1d6
+			MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1		0x1d6
+			MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2		0x1d6
+			MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3		0x1d6
 		>;
 	};
 
 	pinctrl_usdhc1_gpio: usdhc1-gpiogrp {
 		fsl,pins = <
-			MX8MM_IOMUXC_SD1_RESET_B_GPIO2_IO10             0x41    /* wl_reg_on */
-			MX8MM_IOMUXC_SD1_DATA7_GPIO2_IO9                0x41    /* wl_host_wake */
-			MX8MM_IOMUXC_GPIO1_IO00_ANAMIX_REF_CLK_32K      0x141   /* LP0: 32KHz */
+			MX8MM_IOMUXC_SD1_RESET_B_GPIO2_IO10		0x41	/* wl_reg_on */
+			MX8MM_IOMUXC_SD1_DATA7_GPIO2_IO9		0x41	/* wl_host_wake */
+			MX8MM_IOMUXC_GPIO1_IO00_ANAMIX_REF_CLK_32K	0x141	/* LP0: 32KHz */
 		>;
 	};
 
 	pinctrl_usdhc2: usdhc2grp {
-	        fsl,pins = <
-	                MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK         	0x190
-	                MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD         	0x1d0
-	                MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0     	0x1d0
-	                MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1     	0x1d0
-	                MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2     	0x1d0
-	                MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3     	0x1d0
-	        >;
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK			0x190
+			MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD			0x1d0
+			MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0		0x1d0
+			MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1		0x1d0
+			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d0
+			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d0
+		>;
 	};
 
 	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
-	        fsl,pins = <
-	                MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK         	0x194
-	                MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD         	0x1d4
-	                MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0     	0x1d4
-	                MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1     	0x1d4
-	                MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2     	0x1d4
-	                MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3     	0x1d4
-	        >;
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK			0x194
+			MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD			0x1d4
+			MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0		0x1d4
+			MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1		0x1d4
+			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d4
+			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d4
+		>;
 	};
 
 	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
-	        fsl,pins = <
-	                MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK         	0x196
-	                MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD         	0x1d6
-	                MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0     	0x1d6
-	                MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1     	0x1d6
-	                MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2     	0x1d6
-	                MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3     	0x1d6
-	        >;
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK			0x196
+			MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD			0x1d6
+			MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0		0x1d6
+			MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1		0x1d6
+			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d6
+			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d6
+		>;
 	};
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-evk.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-evk.dtsi
index e03aba825c18f..ec49cbbfd9cee 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-evk.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mm-evk.dtsi
@@ -542,6 +542,7 @@ &pcie0 {
 	assigned-clock-rates = <10000000>, <250000000>;
 	assigned-clock-parents = <&clk IMX8MM_SYS_PLL2_50M>,
 				 <&clk IMX8MM_SYS_PLL2_250M>;
+	/* This property is deprecated, use power supply from the Root Port node. */
 	vpcie-supply = <&reg_pcie0>;
 	supports-clkreq;
 	status = "okay";
@@ -562,6 +563,7 @@ &pcie0_ep {
 
 &pcie0_port0 {
 	reset-gpios = <&gpio4 21 GPIO_ACTIVE_LOW>;
+	vpcie3v3-supply = <&reg_pcie0>;
 };
 
 &sai2 {
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-var-dart-sonata.dts b/arch/arm64/boot/dts/freescale/imx8mm-var-dart-sonata.dts
new file mode 100644
index 0000000000000..57d29a5c3821a
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-var-dart-sonata.dts
@@ -0,0 +1,516 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Variscite Sonata carrier board for DART-MX8M-MINI
+ *
+ * Link: https://variscite.com/carrier-boards/sonata-board/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/phy/phy-imx8-pcie.h>
+#include "imx8mm-var-dart.dtsi"
+
+/ {
+	model = "Variscite DART-MX8M-MINI on Sonata-Board";
+	compatible = "variscite,var-dart-mx8mm-sonata",
+		     "variscite,var-dart-mx8mm",
+		     "fsl,imx8mm";
+
+	chosen {
+		stdout-path = &uart1;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		button-home {
+			label = "Home";
+			linux,code = <KEY_HOME>;
+			gpios = <&pca6408_1 4 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+
+		button-up {
+			label = "Up";
+			linux,code = <KEY_UP>;
+			gpios = <&pca6408_1 5 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+
+		button-down {
+			label = "Down";
+			linux,code = <KEY_DOWN>;
+			gpios = <&pca6408_1 6 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+
+		button-back {
+			label = "Back";
+			linux,code = <KEY_BACK>;
+			gpios = <&pca6408_1 7 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+	};
+
+	gpio-leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_leds>;
+
+		led-emmc {
+			gpios = <&gpio4 18 GPIO_ACTIVE_HIGH>;
+			label = "eMMC";
+			linux,default-trigger = "mmc2";
+		};
+	};
+
+	clk40m: oscillator {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <40000000>;
+		clock-output-names = "can_osc";
+	};
+
+	pcie0_refclk: pcie0-refclk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <100000000>;
+	};
+
+	reg_usdhc2_vmmc: regulator-vmmc-usdhc2 {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_vmmc_usdhc2>;
+		regulator-name = "VSD_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio2 19 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		startup-delay-us = <100>;
+		off-on-delay-us = <12000>;
+	};
+};
+
+&ecspi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi1>;
+	cs-gpios = <&gpio5  9 GPIO_ACTIVE_LOW>,
+		   <&gpio1 12 GPIO_ACTIVE_LOW>;
+	status = "okay";
+
+	/* Resistive touch controller */
+	ads7846: touchscreen@0 {
+		compatible = "ti,ads7846";
+		reg = <0>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_restouch>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <3 IRQ_TYPE_EDGE_FALLING>;
+		spi-max-frequency = <1500000>;
+		pendown-gpio = <&gpio1 3 GPIO_ACTIVE_LOW>;
+		ti,x-min = /bits/ 16 <125>;
+		ti,x-max = /bits/ 16 <4008>;
+		ti,y-min = /bits/ 16 <282>;
+		ti,y-max = /bits/ 16 <3864>;
+		ti,x-plate-ohms = /bits/ 16 <180>;
+		ti,pressure-max = /bits/ 16 <255>;
+		ti,debounce-max = /bits/ 16 <10>;
+		ti,debounce-tol = /bits/ 16 <3>;
+		ti,debounce-rep = /bits/ 16 <1>;
+		ti,settle-delay-usec = /bits/ 16 <150>;
+		ti,keep-vref-on;
+		wakeup-source;
+	};
+
+	can0: can@1 {
+		compatible = "microchip,mcp251xfd";
+		reg = <1>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_can>;
+		clocks = <&clk40m>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <6 IRQ_TYPE_LEVEL_LOW>;
+		microchip,rx-int-gpios = <&gpio5 4 GPIO_ACTIVE_LOW>;
+		spi-max-frequency = <20000000>;
+	};
+};
+
+&ethphy0 {
+	leds {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		led@0 {
+			reg = <0>;
+			color = <LED_COLOR_ID_YELLOW>;
+			function = LED_FUNCTION_LAN;
+			linux,default-trigger = "netdev";
+		};
+
+		led@1 {
+			reg = <1>;
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_LAN;
+			linux,default-trigger = "netdev";
+		};
+	};
+};
+
+&i2c2 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c2>;
+	pinctrl-1 = <&pinctrl_i2c2_gpio>;
+	scl-gpios = <&gpio5 16 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 17 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	pca9534: gpio@22 {
+		compatible = "nxp,pca9534";
+		reg = <0x22>;
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		/* RGB_SEL */
+		lvds-brg-enable-hog {
+			gpio-hog;
+			gpios = <7 GPIO_ACTIVE_HIGH>;
+			output-low;
+			line-name = "rgb_sel";
+		};
+	};
+
+	/* Capacitive touch controller */
+	ft5x06_ts: touchscreen@38 {
+		compatible = "edt,edt-ft5206";
+		reg = <0x38>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_captouch>;
+		reset-gpios = <&pca6408_2 4 GPIO_ACTIVE_LOW>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <14 IRQ_TYPE_EDGE_FALLING>;
+		touchscreen-size-x = <800>;
+		touchscreen-size-y = <480>;
+		touchscreen-inverted-x;
+		touchscreen-inverted-y;
+		wakeup-source;
+	};
+
+	extcon_ptn5150: typec@3d {
+		compatible = "nxp,ptn5150";
+		reg = <0x3d>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_extcon>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <10 IRQ_TYPE_EDGE_FALLING>;
+
+		port {
+			typec1_dr_sw: endpoint {
+				remote-endpoint = <&usb1_drd_sw>;
+			};
+		};
+	};
+
+	rtc@68 {
+		compatible = "dallas,ds1337";
+		reg = <0x68>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_rtc>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <15 IRQ_TYPE_EDGE_FALLING>;
+		wakeup-source;
+	};
+};
+
+&i2c4 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c4>;
+	pinctrl-1 = <&pinctrl_i2c4_gpio>;
+	scl-gpios = <&gpio5 20 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 21 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	pca6408_1: gpio@20 {
+		compatible = "nxp,pcal6408";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pca6408>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <5 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	pca6408_2: gpio@21 {
+		compatible = "nxp,pcal6408";
+		reg = <0x21>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <5 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	st33ktpm2xi2c: tpm@2e {
+		compatible = "st,st33ktpm2xi2c", "tcg,tpm-tis-i2c";
+		reg = <0x2e>;
+		label = "tpm";
+		reset-gpios = <&pca9534 0 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&pcie_phy {
+	fsl,refclk-pad-mode = <IMX8_PCIE_REFCLK_PAD_INPUT>;
+	fsl,tx-deemph-gen1 = <0x2d>;
+	fsl,tx-deemph-gen2 = <0xf>;
+	clocks = <&pcie0_refclk>;
+	status = "okay";
+};
+
+&pcie0 {
+	reset-gpios = <&pca6408_2 3 GPIO_ACTIVE_LOW>;
+	clocks = <&clk IMX8MM_CLK_PCIE1_ROOT>, <&pcie0_refclk>,
+		 <&clk IMX8MM_CLK_PCIE1_AUX>;
+	assigned-clocks = <&clk IMX8MM_CLK_PCIE1_AUX>,
+			  <&clk IMX8MM_CLK_PCIE1_CTRL>;
+	assigned-clock-rates = <10000000>, <250000000>;
+	assigned-clock-parents = <&clk IMX8MM_SYS_PLL2_50M>,
+				 <&clk IMX8MM_SYS_PLL2_250M>;
+	status = "okay";
+};
+
+&pwm1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm>;
+	status = "okay";
+};
+
+&snvs_pwrkey {
+	status = "okay";
+};
+
+&snvs_rtc {
+	status = "disabled";
+};
+
+/* Console */
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	wakeup-source;
+	status = "okay";
+};
+
+/* Header */
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart2>;
+	status = "okay";
+};
+
+/* Header */
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3>;
+	status = "okay";
+};
+
+&usbotg1 {
+	dr_mode = "otg";
+	hnp-disable;
+	srp-disable;
+	adp-disable;
+	usb-role-switch;
+	disable-over-current;
+	samsung,picophy-pre-emp-curr-control = <3>;
+	samsung,picophy-dc-vol-level-adjust = <7>;
+	status = "okay";
+
+	port {
+		usb1_drd_sw: endpoint {
+			remote-endpoint = <&typec1_dr_sw>;
+		};
+	};
+};
+
+&usbotg2 {
+	dr_mode = "host";
+	samsung,picophy-pre-emp-curr-control = <3>;
+	samsung,picophy-dc-vol-level-adjust = <7>;
+	disable-over-current;
+	status = "okay";
+};
+
+/* SD */
+&usdhc2 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
+	cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>;
+	bus-width = <4>;
+	vmmc-supply = <&reg_usdhc2_vmmc>;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_can: cangrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO06_GPIO1_IO6		0x16
+			MX8MM_IOMUXC_SPDIF_RX_GPIO5_IO4			0x16
+		>;
+	};
+
+	pinctrl_captouch: captouchgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO14_GPIO1_IO14		0x16
+		>;
+	};
+
+	pinctrl_ecspi1: ecspi1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ECSPI1_SCLK_ECSPI1_SCLK		0x13
+			MX8MM_IOMUXC_ECSPI1_MOSI_ECSPI1_MOSI		0x13
+			MX8MM_IOMUXC_ECSPI1_MISO_ECSPI1_MISO		0x13
+			MX8MM_IOMUXC_ECSPI1_SS0_GPIO5_IO9		0x13
+			MX8MM_IOMUXC_GPIO1_IO12_GPIO1_IO12		0x13
+		>;
+	};
+
+	pinctrl_extcon: extcongrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO10_GPIO1_IO10		0x19
+		>;
+	};
+
+	pinctrl_gpio_leds: ledgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI1_TXD6_GPIO4_IO18		0x1c6
+		>;
+	};
+
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C2_SCL_I2C2_SCL			0x400001c3
+			MX8MM_IOMUXC_I2C2_SDA_I2C2_SDA			0x400001c3
+		>;
+	};
+
+	pinctrl_i2c2_gpio: i2c2-gpiogrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C2_SCL_GPIO5_IO16		0x400001c3
+			MX8MM_IOMUXC_I2C2_SDA_GPIO5_IO17		0x400001c3
+		>;
+	};
+
+	pinctrl_i2c4: i2c4grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C4_SCL_I2C4_SCL			0x400001c3
+			MX8MM_IOMUXC_I2C4_SDA_I2C4_SDA			0x400001c3
+		>;
+	};
+
+	pinctrl_i2c4_gpio: i2c4-gpiogrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C4_SCL_GPIO5_IO20		0x400001c3
+			MX8MM_IOMUXC_I2C4_SDA_GPIO5_IO21		0x400001c3
+		>;
+	};
+
+	pinctrl_pca6408: pca6408grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO05_GPIO1_IO5		0x1c6
+		>;
+	};
+
+	pinctrl_pwm: pwmgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO01_PWM1_OUT		0x06
+		>;
+	};
+
+	pinctrl_vmmc_usdhc2: regvmmc-usdhc2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_RESET_B_GPIO2_IO19		0xc1
+		>;
+	};
+
+	pinctrl_restouch: restouchgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO03_GPIO1_IO3		0x1c0
+		>;
+	};
+
+	pinctrl_rtc: rtcgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO15_GPIO1_IO15		0x1c1
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_UART1_RXD_UART1_DCE_RX		0x140
+			MX8MM_IOMUXC_UART1_TXD_UART1_DCE_TX		0x140
+		>;
+	};
+
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_UART2_RXD_UART2_DCE_RX		0x140
+			MX8MM_IOMUXC_UART2_TXD_UART2_DCE_TX		0x140
+		>;
+	};
+
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_UART3_RXD_UART3_DCE_RX		0x140
+			MX8MM_IOMUXC_UART3_TXD_UART3_DCE_TX		0x140
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK			0x190
+			MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD			0x1d0
+			MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0		0x1d0
+			MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1		0x1d0
+			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d0
+			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d0
+			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x1d0
+		>;
+	};
+
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK			0x194
+			MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD			0x1d4
+			MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0		0x1d4
+			MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1		0x1d4
+			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d4
+			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d4
+			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x1d0
+		>;
+	};
+
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK			0x196
+			MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD			0x1d6
+			MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0		0x1d6
+			MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1		0x1d6
+			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d6
+			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d6
+			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x1d0
+		>;
+	};
+
+	pinctrl_usdhc2_gpio: usdhc2-gpiogrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12		0xc1
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-var-dart.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-var-dart.dtsi
new file mode 100644
index 0000000000000..2324063f8fd90
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-var-dart.dtsi
@@ -0,0 +1,558 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Common dtsi for Variscite DART-MX8M-MINI
+ *
+ * Link: https://variscite.com/system-on-module-som/i-mx-8/i-mx-8m-mini/dart-mx8m-mini/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/dts-v1/;
+
+#include "imx8mm.dtsi"
+
+/ {
+	model = "Variscite DART-MX8M-MINI Module";
+	compatible = "variscite,var-dart-mx8mm", "fsl,imx8mm";
+
+	memory@40000000 {
+		device_type = "memory";
+		reg = <0x0 0x40000000 0 0x80000000>;
+	};
+
+	reg_audio_supply: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-name = "wm8904-supply";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+
+	reg_eth_phy: regulator-eth-phy {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_reg_eth_phy>;
+		regulator-name = "eth_phy_pwr";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-enable-ramp-delay = <20000>;
+		gpio = <&gpio1 7 GPIO_ACTIVE_LOW>;
+	};
+
+	reg_phy_vddio: regulator-phy-vddio {
+		compatible = "regulator-fixed";
+		regulator-name = "vddio-1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	sound-wm8904 {
+		compatible = "simple-audio-card";
+		simple-audio-card,bitclock-master = <&codec_dai>;
+		simple-audio-card,format = "i2s";
+		simple-audio-card,frame-master = <&codec_dai>;
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,name = "wm8904-audio";
+		simple-audio-card,routing =
+			"Headphone Jack", "HPOUTL",
+			"Headphone Jack", "HPOUTR",
+			"IN2L", "Line In Jack",
+			"IN2R", "Line In Jack",
+			"IN1L", "Microphone Jack",
+			"IN1R", "Microphone Jack";
+		simple-audio-card,widgets =
+			"Microphone", "Microphone Jack",
+			"Headphone", "Headphone Jack",
+			"Line", "Line In Jack";
+
+		codec_dai: simple-audio-card,codec {
+			sound-dai = <&wm8904>;
+		};
+
+		simple-audio-card,cpu {
+			sound-dai = <&sai3>;
+		};
+	};
+
+	wifi_pwrseq: wifi-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		post-power-on-delay-ms = <100>;
+		power-off-delay-us = <10000>;
+		reset-gpios = <&gpio2 10 GPIO_ACTIVE_LOW>, /* WIFI_RESET */
+			      <&gpio2 20 GPIO_ACTIVE_LOW>; /* WIFI_PWR_EN */
+	};
+};
+
+&A53_0 {
+	cpu-supply = <&buck2_reg>;
+};
+
+&A53_1 {
+	cpu-supply = <&buck2_reg>;
+};
+
+&A53_2 {
+	cpu-supply = <&buck2_reg>;
+};
+
+&A53_3 {
+	cpu-supply = <&buck2_reg>;
+};
+
+&fec1 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_fec1>;
+	pinctrl-1 = <&pinctrl_fec1_sleep>;
+	/*
+	 * The required RGMII TX and RX 2ns delays are implemented directly
+	 * in hardware via passive delay elements on the SOM PCB.
+	 * No delay configuration is needed in software via PHY driver.
+	 */
+	phy-mode = "rgmii";
+	phy-handle = <&ethphy0>;
+	phy-supply = <&reg_eth_phy>;
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy0: ethernet-phy@0 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <0>;
+			reset-gpios = <&gpio1 9 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <10000>;
+			reset-deassert-us = <100000>;
+			vddio-supply = <&reg_phy_vddio>;
+		};
+	};
+};
+
+&i2c1 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c1>;
+	pinctrl-1 = <&pinctrl_i2c1_gpio>;
+	scl-gpios = <&gpio5 14 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 15 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	pmic@4b {
+		compatible = "rohm,bd71847";
+		reg = <0x4b>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pmic>;
+		interrupt-parent = <&gpio2>;
+		interrupts = <8 IRQ_TYPE_LEVEL_LOW>;
+		rohm,reset-snvs-powered;
+
+		regulators {
+			buck1_reg: BUCK1 {
+				regulator-name = "buck1";
+				regulator-min-microvolt = <700000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <1250>;
+			};
+
+			buck2_reg: BUCK2 {
+				regulator-name = "buck2";
+				regulator-min-microvolt = <700000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <1250>;
+				rohm,dvs-run-voltage = <1000000>;
+				rohm,dvs-idle-voltage = <900000>;
+			};
+
+			buck3_reg: BUCK3 {
+				// BUCK5 in datasheet
+				regulator-name = "buck3";
+				regulator-min-microvolt = <700000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			buck4_reg: BUCK4 {
+				// BUCK6 in datasheet
+				regulator-name = "buck4";
+				regulator-min-microvolt = <2600000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			buck5_reg: BUCK5 {
+				// BUCK7 in datasheet
+				regulator-name = "buck5";
+				regulator-min-microvolt = <1605000>;
+				regulator-max-microvolt = <1995000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			buck6_reg: BUCK6 {
+				// BUCK8 in datasheet
+				regulator-name = "buck6";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1400000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo1_reg: LDO1 {
+				regulator-name = "ldo1";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <1900000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo2_reg: LDO2 {
+				regulator-name = "ldo2";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <900000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo3_reg: LDO3 {
+				regulator-name = "ldo3";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo4_reg: LDO4 {
+				regulator-name = "ldo4";
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+			};
+
+			ldo5_reg: LDO5 {
+				regulator-name = "ldo5";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+			};
+
+			ldo6_reg: LDO6 {
+				regulator-name = "ldo6";
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+		};
+	};
+};
+
+&i2c3 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c3>;
+	pinctrl-1 = <&pinctrl_i2c3_gpio>;
+	scl-gpios = <&gpio5 18 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 19 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	wm8904: audio-codec@1a {
+		compatible = "wlf,wm8904";
+		reg = <0x1a>;
+		#sound-dai-cells = <0>;
+		clocks = <&clk IMX8MM_CLK_SAI3_ROOT>;
+		clock-names = "mclk";
+		AVDD-supply = <&ldo5_reg>;
+		CPVDD-supply = <&ldo5_reg>;
+		DBVDD-supply = <&reg_audio_supply>;
+		DCVDD-supply = <&ldo5_reg>;
+		MICVDD-supply = <&ldo5_reg>;
+		wlf,drc-cfg-names = "default", "peaklimiter", "tradition",
+				    "soft", "music";
+		/*
+		 * Config registers per name, respectively:
+		 * KNEE_IP = 0,   KNEE_OP = 0,     HI_COMP = 1,   LO_COMP = 1
+		 * KNEE_IP = -24, KNEE_OP = -6,    HI_COMP = 1/4, LO_COMP = 1
+		 * KNEE_IP = -42, KNEE_OP = -3,    HI_COMP = 0,   LO_COMP = 1
+		 * KNEE_IP = -45, KNEE_OP = -9,    HI_COMP = 1/8, LO_COMP = 1
+		 * KNEE_IP = -30, KNEE_OP = -10.5, HI_COMP = 1/4, LO_COMP = 1
+		 */
+		wlf,drc-cfg-regs = /bits/ 16 <0x01af 0x3248 0x0000 0x0000>,
+				   /bits/ 16 <0x04af 0x324b 0x0010 0x0408>,
+				   /bits/ 16 <0x04af 0x324b 0x0028 0x0704>,
+				   /bits/ 16 <0x04af 0x324b 0x0018 0x078c>,
+				   /bits/ 16 <0x04af 0x324b 0x0010 0x050e>;
+		/* GPIO1 = DMIC_CLK, don't touch others */
+		wlf,gpio-cfg = <0x0018>, <0xffff>, <0xffff>, <0xffff>;
+	};
+};
+
+&mu {
+	status = "okay";
+};
+
+&sai3 {
+	#sound-dai-cells = <0>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sai3>;
+	assigned-clocks = <&clk IMX8MM_CLK_SAI3>;
+	assigned-clock-parents = <&clk IMX8MM_AUDIO_PLL1_OUT>;
+	assigned-clock-rates = <1536000>;
+	fsl,sai-mclk-direction-output;
+	status = "okay";
+};
+
+/* BT module */
+&uart4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart4>, <&pinctrl_bt>;
+	assigned-clocks = <&clk IMX8MM_CLK_UART4>;
+	assigned-clock-parents = <&clk IMX8MM_SYS_PLL1_80M>;
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth_iw61x: bluetooth {
+		compatible = "nxp,88w8987-bt";
+	};
+};
+
+/* WIFI */
+&usdhc1 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc1>, <&pinctrl_wifi>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>, <&pinctrl_wifi>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>, <&pinctrl_wifi>;
+	bus-width = <4>;
+	keep-power-in-suspend;
+	mmc-pwrseq = <&wifi_pwrseq>;
+	non-removable;
+	status = "okay";
+};
+
+/* eMMC */
+&usdhc3 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	pinctrl-1 = <&pinctrl_usdhc3_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc3_200mhz>;
+	bus-width = <8>;
+	non-removable;
+	status = "okay";
+};
+
+&wdog1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_wdog>;
+	fsl,ext-reset-output;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_bt: btgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD1_DATA4_GPIO2_IO6		0xc1
+			MX8MM_IOMUXC_SPDIF_EXT_CLK_GPIO5_IO5		0xc1
+		>;
+	};
+
+	pinctrl_fec1: fec1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ENET_MDC_ENET1_MDC			0x3
+			MX8MM_IOMUXC_ENET_MDIO_ENET1_MDIO		0x3
+			MX8MM_IOMUXC_ENET_TD3_ENET1_RGMII_TD3		0x1f
+			MX8MM_IOMUXC_ENET_TD2_ENET1_RGMII_TD2		0x1f
+			MX8MM_IOMUXC_ENET_TD1_ENET1_RGMII_TD1		0x1f
+			MX8MM_IOMUXC_ENET_TD0_ENET1_RGMII_TD0		0x1f
+			MX8MM_IOMUXC_ENET_RD3_ENET1_RGMII_RD3		0x91
+			MX8MM_IOMUXC_ENET_RD2_ENET1_RGMII_RD2		0x91
+			MX8MM_IOMUXC_ENET_RD1_ENET1_RGMII_RD1		0x91
+			MX8MM_IOMUXC_ENET_RD0_ENET1_RGMII_RD0		0x91
+			MX8MM_IOMUXC_ENET_TXC_ENET1_RGMII_TXC		0x1f
+			MX8MM_IOMUXC_ENET_RXC_ENET1_RGMII_RXC		0x91
+			MX8MM_IOMUXC_ENET_RX_CTL_ENET1_RGMII_RX_CTL	0x91
+			MX8MM_IOMUXC_ENET_TX_CTL_ENET1_RGMII_TX_CTL	0x1f
+			MX8MM_IOMUXC_GPIO1_IO09_GPIO1_IO9		0x19
+		>;
+	};
+
+	pinctrl_fec1_sleep: fec1sleepgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ENET_MDC_GPIO1_IO16		0x120
+			MX8MM_IOMUXC_ENET_MDIO_GPIO1_IO17		0x120
+			MX8MM_IOMUXC_ENET_TD3_GPIO1_IO18		0x120
+			MX8MM_IOMUXC_ENET_TD2_GPIO1_IO19		0x120
+			MX8MM_IOMUXC_ENET_TD1_GPIO1_IO20		0x120
+			MX8MM_IOMUXC_ENET_TD0_GPIO1_IO21		0x120
+			MX8MM_IOMUXC_ENET_RD3_GPIO1_IO29		0x120
+			MX8MM_IOMUXC_ENET_RD2_GPIO1_IO28		0x120
+			MX8MM_IOMUXC_ENET_RD1_GPIO1_IO27		0x120
+			MX8MM_IOMUXC_ENET_RD0_GPIO1_IO26		0x120
+			MX8MM_IOMUXC_ENET_TXC_GPIO1_IO23		0x120
+			MX8MM_IOMUXC_ENET_RXC_GPIO1_IO25		0x120
+			MX8MM_IOMUXC_ENET_RX_CTL_GPIO1_IO24		0x120
+			MX8MM_IOMUXC_ENET_TX_CTL_GPIO1_IO22		0x120
+			MX8MM_IOMUXC_GPIO1_IO09_GPIO1_IO9		0x100
+		>;
+	};
+
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C1_SCL_I2C1_SCL			0x400001c3
+			MX8MM_IOMUXC_I2C1_SDA_I2C1_SDA			0x400001c3
+		>;
+	};
+
+	pinctrl_i2c1_gpio: i2c1-gpiogrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C1_SCL_GPIO5_IO14		0x400001c3
+			MX8MM_IOMUXC_I2C1_SDA_GPIO5_IO15		0x400001c3
+		>;
+	};
+
+	pinctrl_i2c3: i2c3grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C3_SCL_I2C3_SCL			0x400001c3
+			MX8MM_IOMUXC_I2C3_SDA_I2C3_SDA			0x400001c3
+		>;
+	};
+
+	pinctrl_i2c3_gpio: i2c3-gpiogrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C3_SCL_GPIO5_IO18		0x400001c3
+			MX8MM_IOMUXC_I2C3_SDA_GPIO5_IO19		0x400001c3
+		>;
+	};
+
+	pinctrl_pmic: pmicgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD1_DATA6_GPIO2_IO8		0x140
+		>;
+	};
+
+	pinctrl_reg_eth_phy: regeth-phygrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO07_GPIO1_IO7		0x41
+		>;
+	};
+
+	pinctrl_sai3: sai3grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI3_RXFS_SAI3_RX_SYNC		0xd6
+			MX8MM_IOMUXC_SAI3_RXC_SAI3_RX_BCLK		0xd6
+			MX8MM_IOMUXC_SAI3_RXD_SAI3_RX_DATA0		0xd6
+			MX8MM_IOMUXC_SAI3_TXFS_SAI3_TX_SYNC		0xd6
+			MX8MM_IOMUXC_SAI3_TXC_SAI3_TX_BCLK		0xd6
+			MX8MM_IOMUXC_SAI3_TXD_SAI3_TX_DATA0		0xd6
+			MX8MM_IOMUXC_SAI3_MCLK_SAI3_MCLK		0xd6
+		>;
+	};
+
+	pinctrl_uart4: uart4grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ECSPI2_SCLK_UART4_DCE_RX		0x140
+			MX8MM_IOMUXC_ECSPI2_MOSI_UART4_DCE_TX		0x140
+			MX8MM_IOMUXC_ECSPI2_SS0_UART4_DCE_RTS_B		0x140
+			MX8MM_IOMUXC_ECSPI2_MISO_UART4_DCE_CTS_B	0x140
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK			0x190
+			MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD			0x1d0
+			MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0		0x1d0
+			MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1		0x1d0
+			MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2		0x1d0
+			MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3		0x1d0
+		>;
+	};
+
+	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK			0x194
+			MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD			0x1d4
+			MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0		0x1d4
+			MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1		0x1d4
+			MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2		0x1d4
+			MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3		0x1d4
+		>;
+	};
+
+	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK			0x196
+			MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD			0x1d6
+			MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0		0x1d6
+			MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1		0x1d6
+			MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2		0x1d6
+			MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3		0x1d6
+		>;
+	};
+
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_NAND_WE_B_USDHC3_CLK		0x190
+			MX8MM_IOMUXC_NAND_WP_B_USDHC3_CMD		0x1d0
+			MX8MM_IOMUXC_NAND_DATA04_USDHC3_DATA0		0x1d0
+			MX8MM_IOMUXC_NAND_DATA05_USDHC3_DATA1		0x1d0
+			MX8MM_IOMUXC_NAND_DATA06_USDHC3_DATA2		0x1d0
+			MX8MM_IOMUXC_NAND_DATA07_USDHC3_DATA3		0x1d0
+			MX8MM_IOMUXC_NAND_RE_B_USDHC3_DATA4		0x1d0
+			MX8MM_IOMUXC_NAND_CE2_B_USDHC3_DATA5		0x1d0
+			MX8MM_IOMUXC_NAND_CE3_B_USDHC3_DATA6		0x1d0
+			MX8MM_IOMUXC_NAND_CLE_USDHC3_DATA7		0x1d0
+			MX8MM_IOMUXC_NAND_CE1_B_USDHC3_STROBE		0x190
+		>;
+	};
+
+	pinctrl_usdhc3_100mhz: usdhc3-100mhzgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_NAND_WE_B_USDHC3_CLK		0x194
+			MX8MM_IOMUXC_NAND_WP_B_USDHC3_CMD		0x1d4
+			MX8MM_IOMUXC_NAND_DATA04_USDHC3_DATA0		0x1d4
+			MX8MM_IOMUXC_NAND_DATA05_USDHC3_DATA1		0x1d4
+			MX8MM_IOMUXC_NAND_DATA06_USDHC3_DATA2		0x1d4
+			MX8MM_IOMUXC_NAND_DATA07_USDHC3_DATA3		0x1d4
+			MX8MM_IOMUXC_NAND_RE_B_USDHC3_DATA4		0x1d4
+			MX8MM_IOMUXC_NAND_CE2_B_USDHC3_DATA5		0x1d4
+			MX8MM_IOMUXC_NAND_CE3_B_USDHC3_DATA6		0x1d4
+			MX8MM_IOMUXC_NAND_CLE_USDHC3_DATA7		0x1d4
+			MX8MM_IOMUXC_NAND_CE1_B_USDHC3_STROBE		0x194
+		>;
+	};
+
+	pinctrl_usdhc3_200mhz: usdhc3-200mhzgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_NAND_WE_B_USDHC3_CLK		0x196
+			MX8MM_IOMUXC_NAND_WP_B_USDHC3_CMD		0x1d6
+			MX8MM_IOMUXC_NAND_DATA04_USDHC3_DATA0		0x1d6
+			MX8MM_IOMUXC_NAND_DATA05_USDHC3_DATA1		0x1d6
+			MX8MM_IOMUXC_NAND_DATA06_USDHC3_DATA2		0x1d6
+			MX8MM_IOMUXC_NAND_DATA07_USDHC3_DATA3		0x1d6
+			MX8MM_IOMUXC_NAND_RE_B_USDHC3_DATA4		0x1d6
+			MX8MM_IOMUXC_NAND_CE2_B_USDHC3_DATA5		0x1d6
+			MX8MM_IOMUXC_NAND_CE3_B_USDHC3_DATA6		0x1d6
+			MX8MM_IOMUXC_NAND_CLE_USDHC3_DATA7		0x1d6
+			MX8MM_IOMUXC_NAND_CE1_B_USDHC3_STROBE		0x196
+		>;
+	};
+
+	pinctrl_wdog: wdoggrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO02_WDOG1_WDOG_B		0xc6
+		>;
+	};
+
+	pinctrl_wifi: wifigrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_WP_GPIO2_IO20			0x140
+			MX8MM_IOMUXC_SD1_RESET_B_GPIO2_IO10		0xc1
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-var-som-symphony.dts b/arch/arm64/boot/dts/freescale/imx8mm-var-som-symphony.dts
index 857325ef44619..fd0a1862ce90b 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-var-som-symphony.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mm-var-som-symphony.dts
@@ -55,18 +55,21 @@ key-back {
 			label = "Back";
 			gpios = <&pca9534 1 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_BACK>;
+			wakeup-source;
 		};
 
 		key-home {
 			label = "Home";
 			gpios = <&pca9534 2 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_HOME>;
+			wakeup-source;
 		};
 
 		key-menu {
 			label = "Menu";
 			gpios = <&pca9534 3 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_MENU>;
+			wakeup-source;
 		};
 	};
 
@@ -165,6 +168,14 @@ pca6408: gpio@21 {
 		reg = <0x21>;
 		gpio-controller;
 		#gpio-cells = <2>;
+
+		/* RGB_SEL */
+		lvds-brg-enable-hog {
+			gpio-hog;
+			gpios = <7 GPIO_ACTIVE_HIGH>;
+			output-low;
+			line-name = "rgb_sel";
+		};
 	};
 
 	st33ktpm2xi2c: tpm@2e {
@@ -187,6 +198,7 @@ ft5x06_ts: touchscreen@38 {
 		touchscreen-size-y = <480>;
 		touchscreen-inverted-x;
 		touchscreen-inverted-y;
+		wakeup-source;
 	};
 
 	rtc@68 {
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts b/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts
index 443e4fd5b9bfc..285bf79864ebd 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts
@@ -775,6 +775,10 @@ &micfil {
 	status = "okay";
 };
 
+&mu2 {
+	status = "okay";
+};
+
 &pwm1 {
 	pinctrl-0 = <&pinctrl_pwm1>;
 	pinctrl-names = "default";
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-evk-flexcan2.dtso b/arch/arm64/boot/dts/freescale/imx8mp-evk-flexcan2.dtso
new file mode 100644
index 0000000000000..d871198420271
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-evk-flexcan2.dtso
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 NXP
+ */
+
+/dts-v1/;
+/plugin/;
+
+&flexcan2 {
+	status = "okay"; /* can2 pin conflict with pdm */
+};
+
+&micfil {
+	status = "disabled";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts
index a7f3acdc36d1a..7bdd641292fb6 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts
@@ -6,6 +6,7 @@
 /dts-v1/;
 
 #include <dt-bindings/phy/phy-imx8-pcie.h>
+#include <dt-bindings/usb/pd.h>
 #include "imx8mp.dtsi"
 
 / {
@@ -50,6 +51,25 @@ status {
 		};
 	};
 
+	can_mux: mux-controller-0 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		mux-gpios = <&pca6416 3 GPIO_ACTIVE_HIGH>;
+	};
+
+	can_mux_pinctrl: pinctrl-gpiomux {
+		compatible = "pinctrl-multiplexer";
+
+		can_fun: can-grp {
+			mux-states = <&can_mux 1>;
+		};
+
+		pdm_fun: pdm-grp {
+			mux-states = <&can_mux 0>;
+		};
+	};
+
 	memory@40000000 {
 		device_type = "memory";
 		reg = <0x0 0x40000000 0 0xc0000000>,
@@ -453,7 +473,7 @@ &flexcan1 {
 
 &flexcan2 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_flexcan2>;
+	pinctrl-0 = <&pinctrl_flexcan2>, <&can_fun>;
 	phys = <&flexcan_phy 1>;
 	status = "disabled";/* can2 pin conflict with pdm */
 };
@@ -636,6 +656,35 @@ adv7535_out: endpoint {
 
 		};
 	};
+
+	ptn5110: tcpc@50 {
+		compatible = "nxp,ptn5110", "tcpci";
+		reg = <0x50>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_typec>;
+		interrupt-parent = <&gpio4>;
+		interrupts = <19 IRQ_TYPE_LEVEL_LOW>;
+		orientation-gpios = <&gpio4 20 GPIO_ACTIVE_HIGH>;
+
+		usb_con: connector {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			power-role = "dual";
+			data-role = "dual";
+			try-power-role = "sink";
+			source-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
+			sink-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)
+				     PDO_VAR(5000, 20000, 3000)>;
+			op-sink-microwatt = <15000000>;
+			self-powered;
+
+			port {
+				typec_con_hs: endpoint {
+					remote-endpoint = <&usb3_data_hs>;
+				};
+			};
+		};
+	};
 };
 
 &i2c3 {
@@ -720,7 +769,7 @@ &lcdif3 {
 &micfil {
 	#sound-dai-cells = <0>;
 	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_pdm>;
+	pinctrl-0 = <&pinctrl_pdm>, <&pdm_fun>;
 	assigned-clocks = <&clk IMX8MP_CLK_PDM>;
 	assigned-clock-parents = <&clk IMX8MP_AUDIO_PLL1_OUT>;
 	assigned-clock-rates = <196608000>;
@@ -772,6 +821,7 @@ &pcie0 {
 	pinctrl-0 = <&pinctrl_pcie0>;
 	/* This property is deprecated, use reset-gpios from the Root Port node. */
 	reset-gpio = <&gpio2 7 GPIO_ACTIVE_LOW>;
+	/* This property is deprecated, use power supply from the Root Port node. */
 	vpcie-supply = <&reg_pcie0>;
 	vpcie3v3aux-supply = <&reg_pcie0>;
 	supports-clkreq;
@@ -786,6 +836,8 @@ &pcie0_ep {
 
 &pcie0_port0 {
 	reset-gpios = <&gpio2 7 GPIO_ACTIVE_LOW>;
+	vpcie3v3-supply = <&reg_pcie0>;
+	vpcie3v3aux-supply = <&reg_pcie0>;
 };
 
 &pwm1 {
@@ -851,7 +903,41 @@ &uart2 {
 	status = "okay";
 };
 
+&usb3_phy0 {
+	fsl,phy-tx-vref-tune-percent = <122>;
+	fsl,phy-tx-preemp-amp-tune-microamp = <1800>;
+	fsl,phy-tx-vboost-level-microvolt = <1156>;
+	fsl,phy-comp-dis-tune-percent = <115>;
+	fsl,phy-pcs-tx-deemph-3p5db-attenuation-db = <33>;
+	fsl,phy-pcs-tx-swing-full-percent = <100>;
+	status = "okay";
+};
+
+&usb3_0 {
+	status = "okay";
+};
+
+&usb_dwc3_0 {
+	dr_mode = "otg";
+	hnp-disable;
+	srp-disable;
+	adp-disable;
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	snps,dis-u1-entry-quirk;
+	snps,dis-u2-entry-quirk;
+	status = "okay";
+
+	port {
+		usb3_data_hs: endpoint {
+			remote-endpoint = <&typec_con_hs>;
+		};
+	};
+};
+
 &usb3_phy1 {
+	fsl,phy-tx-preemp-amp-tune-microamp = <1800>;
+	fsl,phy-tx-vref-tune-percent = <116>;
 	status = "okay";
 };
 
@@ -1179,6 +1265,13 @@ MX8MP_IOMUXC_UART2_TXD__UART2_DCE_TX	0x140
 		>;
 	};
 
+	pinctrl_typec: typecgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SAI1_TXD7__GPIO4_IO19	0x1c4
+			MX8MP_IOMUXC_SAI1_MCLK__GPIO4_IO20	0x16
+		>;
+	};
+
 	pinctrl_usb1_vbus: usb1grp {
 		fsl,pins = <
 			MX8MP_IOMUXC_GPIO1_IO14__USB2_OTG_PWR	0x10
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-frdm.dts b/arch/arm64/boot/dts/freescale/imx8mp-frdm.dts
index 5fb9714215bfe..f43330d1ff8b6 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-frdm.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-frdm.dts
@@ -562,6 +562,8 @@ MX8MP_IOMUXC_SAI1_RXD0__GPIO4_IO02		0x10
 
 	pinctrl_hdmi: hdmigrp {
 		fsl,pins = <
+			MX8MP_IOMUXC_HDMI_DDC_SCL__HDMIMIX_HDMI_SCL	0x1c2
+			MX8MP_IOMUXC_HDMI_DDC_SDA__HDMIMIX_HDMI_SDA	0x1c2
 			MX8MP_IOMUXC_HDMI_CEC__HDMIMIX_HDMI_CEC		0x10
 		>;
 	};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-nitrogen-smarc-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-nitrogen-smarc-som.dtsi
index 5da0f1b3ed8ab..347e1bc13b3af 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-nitrogen-smarc-som.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp-nitrogen-smarc-som.dtsi
@@ -32,29 +32,29 @@ led-0 {
 	};
 
 	reg_usdhc2_vmmc: regulator-usdhc2-vmmc {
-	        compatible = "regulator-fixed";
-	        regulator-name = "VSD_3V3";
-	        regulator-min-microvolt = <3300000>;
-	        regulator-max-microvolt = <3300000>;
-	        gpios = <&gpio2 19 GPIO_ACTIVE_HIGH>;
-	        enable-active-high;
+		compatible = "regulator-fixed";
+		regulator-name = "VSD_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpios = <&gpio2 19 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
 	};
 };
 
 &A53_0 {
-        cpu-supply = <&buck2>;
+	cpu-supply = <&buck2>;
 };
 
 &A53_1 {
-        cpu-supply = <&buck2>;
+	cpu-supply = <&buck2>;
 };
 
 &A53_2 {
-        cpu-supply = <&buck2>;
+	cpu-supply = <&buck2>;
 };
 
 &A53_3 {
-        cpu-supply = <&buck2>;
+	cpu-supply = <&buck2>;
 };
 
 &i2c1 {
@@ -220,7 +220,7 @@ &wdog1 {
 &iomuxc {
 	pinctrl_gpio_led: gpioledgrp {
 		fsl,pins = <
-		        MX8MP_IOMUXC_GPIO1_IO10__GPIO1_IO10                     	0x19
+			MX8MP_IOMUXC_GPIO1_IO10__GPIO1_IO10				0x19
 		>;
 	};
 
@@ -233,15 +233,15 @@ MX8MP_IOMUXC_I2C1_SDA__I2C1_SDA					0x400001c3
 
 	pinctrl_i2c6: i2c6grp {
 		fsl,pins = <
-		        MX8MP_IOMUXC_SAI5_RXFS__I2C6_SCL            			0x400001c3
-		        MX8MP_IOMUXC_SAI5_RXC__I2C6_SDA                 		0x400001c3
+			MX8MP_IOMUXC_SAI5_RXFS__I2C6_SCL				0x400001c3
+			MX8MP_IOMUXC_SAI5_RXC__I2C6_SDA					0x400001c3
 		>;
 	};
 
 	pinctrl_mcp23018: mcp23018grp {
 		fsl,pins = <
-		        MX8MP_IOMUXC_SAI2_RXC__GPIO4_IO22            			0x1c0
-			MX8MP_IOMUXC_SAI2_MCLK__GPIO4_IO27             			0x100
+			MX8MP_IOMUXC_SAI2_RXC__GPIO4_IO22				0x1c0
+			MX8MP_IOMUXC_SAI2_MCLK__GPIO4_IO27				0x100
 		>;
 	};
 
@@ -253,15 +253,15 @@ MX8MP_IOMUXC_GPIO1_IO14__GPIO1_IO14				0x1c0
 
 	pinctrl_uart2: uart2grp {
 		fsl,pins = <
-			MX8MP_IOMUXC_UART2_RXD__UART2_DCE_RX                            0x40
-			MX8MP_IOMUXC_UART2_TXD__UART2_DCE_TX                            0x40
+			MX8MP_IOMUXC_UART2_RXD__UART2_DCE_RX				0x40
+			MX8MP_IOMUXC_UART2_TXD__UART2_DCE_TX				0x40
 		>;
 	};
 
 	pinctrl_usdhc1: usdhc1grp {
 		fsl,pins = <
-			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK        			0x10
-			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD        			0x150
+			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK				0x10
+			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD				0x150
 			MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0    			0x150
 			MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1    			0x150
 			MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2    			0x150
@@ -277,8 +277,8 @@ MX8MP_IOMUXC_SD1_RESET_B__USDHC1_RESET_B 			0x140
 
 	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
 		fsl,pins = <
-			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK        			0x14
-			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD        			0x154
+			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK				0x14
+			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD				0x154
 			MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0    			0x154
 			MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1    			0x154
 			MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2    			0x154
@@ -293,8 +293,8 @@ MX8MP_IOMUXC_SD1_STROBE__USDHC1_STROBE  			0x14
 
 	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
 		fsl,pins = <
-			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK        			0x12
-			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD        			0x152
+			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK				0x12
+			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD				0x152
 			MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0    			0x152
 			MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1    			0x152
 			MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2    			0x152
@@ -309,34 +309,34 @@ MX8MP_IOMUXC_SD1_STROBE__USDHC1_STROBE  			0x12
 
 	pinctrl_usdhc2: usdhc2grp {
 		fsl,pins = <
-			MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK                                0x190
-			MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD                                0x1d0
-			MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0                            0x1d0
-			MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1                            0x1d0
-			MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2                            0x1d0
-			MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3                            0x1d0
+			MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK				0x190
+			MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD				0x1d0
+			MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0				0x1d0
+			MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1				0x1d0
+			MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2				0x1d0
+			MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3				0x1d0
 		>;
 	};
 
 	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
 		fsl,pins = <
-			MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK                                0x194
-			MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD                                0x1d4
-			MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0                            0x1d4
-			MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1                            0x1d4
-			MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2                            0x1d4
-			MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3                            0x1d4
+			MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK				0x194
+			MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD				0x1d4
+			MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0				0x1d4
+			MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1				0x1d4
+			MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2				0x1d4
+			MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3				0x1d4
 		>;
 	};
 
 	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
 		fsl,pins = <
-			MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK                                0x196
-			MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD                                0x1d6
-			MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0                            0x1d6
-			MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1                            0x1d6
-			MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2                            0x1d6
-			MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3                            0x1d6
+			MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK				0x196
+			MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD				0x1d6
+			MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0				0x1d6
+			MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1				0x1d6
+			MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2				0x1d6
+			MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3				0x1d6
 		>;
 	};
 
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds-g133han01.dtso b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds-g133han01.dtso
new file mode 100644
index 0000000000000..2d2dfda944faa
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds-g133han01.dtso
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR MIT)
+/*
+ * Copyright (c) 2025-2026 TQ-Systems GmbH <linux@ew.tq-group.com>,
+ * D-82229 Seefeld, Germany.
+ * Author: Martin Schmiedel
+ */
+
+/dts-v1/;
+/plugin/;
+
+&backlight_lvds0 {
+	status = "okay";
+};
+
+&panel_lvds0 {
+	compatible = "auo,g133han01";
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			dual-lvds-odd-pixels;
+
+			panel_in_lvds0: endpoint {
+				remote-endpoint = <&ldb_lvds_ch0>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			dual-lvds-even-pixels;
+
+			panel_in_lvds1: endpoint {
+				remote-endpoint = <&ldb_lvds_ch1>;
+			};
+		};
+	};
+};
+
+&lcdif2 {
+	status = "okay";
+};
+
+&lvds_bridge {
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@1 {
+			reg = <1>;
+
+			ldb_lvds_ch0: endpoint {
+				remote-endpoint = <&panel_in_lvds0>;
+			};
+		};
+
+		port@2 {
+			reg = <2>;
+
+			ldb_lvds_ch1: endpoint {
+				remote-endpoint = <&panel_in_lvds1>;
+			};
+		};
+	};
+};
+
+// Update VIDEO_PLL1 frequency
+&media_blk_ctrl {
+	assigned-clock-rates = <500000000>, <200000000>,
+			       <0>, <0>, <500000000>,
+			       <988400000>;
+};
+
+&pwm3 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds0-tm070jvhg33.dtso b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds0-tm070jvhg33.dtso
new file mode 100644
index 0000000000000..2cf1de8c05b49
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds0-tm070jvhg33.dtso
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR MIT)
+/*
+ * Copyright (c) 2025-2026 TQ-Systems GmbH <linux@ew.tq-group.com>,
+ * D-82229 Seefeld, Germany.
+ * Author: Paul Gerber
+ */
+
+/dts-v1/;
+/plugin/;
+
+&backlight_lvds0 {
+	status = "okay";
+};
+
+&panel_lvds0{
+	compatible = "tianma,tm070jvhg33";
+	status = "okay";
+
+	panel-timing {
+		clock-frequency = <74250000>;
+		hactive = <1280>;
+		vactive = <800>;
+		hfront-porch = <64>;
+		hback-porch = <5>;
+		hsync-len = <1>;
+		vfront-porch = <40>;
+		vback-porch = <2>;
+		vsync-len = <1>;
+		de-active = <1>;
+	};
+};
+
+&lcdif2 {
+	status = "okay";
+};
+
+&lvds_bridge {
+	status = "okay";
+};
+
+&pwm3 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds1-tm070jvhg33.dtso b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds1-tm070jvhg33.dtso
new file mode 100644
index 0000000000000..cbad0cfab2925
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds1-tm070jvhg33.dtso
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR MIT)
+/*
+ * Copyright (c) 2025-2026 TQ-Systems GmbH <linux@ew.tq-group.com>,
+ * D-82229 Seefeld, Germany.
+ * Author: Paul Gerber
+ */
+
+/dts-v1/;
+/plugin/;
+
+&backlight_lvds1 {
+	status = "okay";
+};
+
+&panel_lvds1{
+	compatible = "tianma,tm070jvhg33";
+	status = "okay";
+
+	panel-timing {
+		clock-frequency = <74250000>;
+		hactive = <1280>;
+		vactive = <800>;
+		hfront-porch = <64>;
+		hback-porch = <5>;
+		hsync-len = <1>;
+		vfront-porch = <40>;
+		vback-porch = <2>;
+		vsync-len = <1>;
+		de-active = <1>;
+	};
+};
+
+&lcdif2 {
+	status = "okay";
+};
+
+&lvds_bridge {
+	status = "okay";
+};
+
+&pwm2 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2.dts b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2.dts
new file mode 100644
index 0000000000000..f01af630a7c4a
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2.dts
@@ -0,0 +1,380 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright 2025-2026 TQ-Systems GmbH <linux@ew.tq-group.com>,
+ * D-82229 Seefeld, Germany.
+ * Author: Paul Gerber
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/leds/common.h>
+#include "imx8mp-tqma8mpqs.dtsi"
+
+/ {
+	model = "TQ-Systems i.MX8MPlus TQMa8MPxS on MB-SMARC-2";
+	compatible = "tq,imx8mp-tqma8mpqs-mb-smarc-2", "tq,imx8mp-tqma8mpqs", "fsl,imx8mp";
+	chassis-type = "embedded";
+
+	chosen {
+		stdout-path = &uart3;
+	};
+
+	aliases {
+		ethernet0 = &eqos;
+		ethernet1 = &fec;
+		mmc0 = &usdhc3;
+		mmc1 = &usdhc2;
+		rtc0 = &pcf85063;
+		rtc1 = &snvs_rtc;
+		spi0 = &flexspi;
+		spi1 = &ecspi1;
+		spi2 = &ecspi2;
+		spi3 = &ecspi3;
+	};
+
+	backlight_lvds0: backlight-lvds0 {
+		compatible = "pwm-backlight";
+		pwms = <&pwm3 0 100000 0>;
+		brightness-levels = <0 4 8 16 32 64 128 255>;
+		default-brightness-level = <7>;
+		enable-gpios = <&expander0 0 GPIO_ACTIVE_HIGH>;
+		power-supply = <&reg_12v0>;
+		status = "disabled";
+	};
+
+	backlight_lvds1: backlight-lvds1 {
+		compatible = "pwm-backlight";
+		pwms = <&pwm2 0 100000 0>;
+		brightness-levels = <0 4 8 16 32 64 128 255>;
+		default-brightness-level = <7>;
+		enable-gpios = <&expander0 2 GPIO_ACTIVE_HIGH>;
+		power-supply = <&reg_12v0>;
+		status = "disabled";
+	};
+
+	hdmi-connector {
+		compatible = "hdmi-connector";
+		label = "X6";
+		type = "a";
+
+		port {
+			hdmi_connector_in: endpoint {
+				remote-endpoint = <&hdmi_tx_out>;
+			};
+		};
+	};
+
+	panel_lvds0: panel-lvds0 {
+		/*
+		 * Display is not fixed, so compatible has to be added from
+		 * DT overlay
+		 */
+		backlight = <&backlight_lvds0>;
+		power-supply = <&reg_lvds0>;
+		status = "disabled";
+
+		port {
+			panel_in_lvds0: endpoint {
+				remote-endpoint = <&ldb_lvds_ch0>;
+			};
+		};
+	};
+
+	panel_lvds1: panel-lvds1 {
+		/*
+		 * Display is not fixed, so compatible has to be added from
+		 * DT overlay
+		 */
+		backlight = <&backlight_lvds1>;
+		power-supply = <&reg_lvds1>;
+		status = "disabled";
+
+		port {
+			panel_in_lvds1: endpoint {
+				remote-endpoint = <&ldb_lvds_ch1>;
+			};
+		};
+	};
+
+	reg_1v8: regulator-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "1V8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+	};
+
+	reg_3v3: regulator-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+
+	reg_12v0: regulator-12v0 {
+		compatible = "regulator-fixed";
+		regulator-name = "12V0";
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		regulator-always-on;
+	};
+
+	reg_lvds0: regulator-lvds0 {
+		compatible = "regulator-fixed";
+		regulator-name = "LCD0_VDD_EN";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&expander0 1 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_lvds1: regulator-lvds1 {
+		compatible = "regulator-fixed";
+		regulator-name = "LCD1_VDD_EN";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&expander0 3 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		/* global autoconfigured region for contiguous allocations */
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0 0x38000000>;
+			alloc-ranges = <0 0x40000000 0 0x78000000>;
+			linux,cma-default;
+		};
+	};
+
+	sound {
+		compatible = "fsl,imx-audio-tlv320aic32x4";
+		model = "tqm-tlv320aic32";
+		audio-asrc = <&easrc>;
+		audio-cpu = <&sai5>;
+		audio-codec = <&tlv320aic3x04>;
+		audio-routing =
+			"IN3_L", "Mic Jack",
+			"Mic Jack", "Mic Bias",
+			"IN1_L", "Line In Jack",
+			"IN1_R", "Line In Jack",
+			"Line Out Jack", "LOL",
+			"Line Out Jack", "LOR";
+	};
+
+	usb-connector {
+		compatible = "gpio-usb-b-connector", "usb-b-connector";
+		type = "micro";
+		label = "X4";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usbcon0>;
+		id-gpios = <&gpio1 10 GPIO_ACTIVE_HIGH>;
+
+		port {
+			usb_dr_connector: endpoint {
+				remote-endpoint = <&usb3_dwc>;
+			};
+		};
+	};
+};
+
+&easrc {
+	status = "okay";
+};
+
+&ecspi1 {
+	status = "okay";
+};
+
+&eqos {
+	status = "okay";
+};
+
+&ethphy0 {
+	leds {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		led@1 {
+			reg = <1>;
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_LAN;
+			function-enumerator = <0>;
+			default-state = "keep";
+		};
+
+		led@2 {
+			reg = <2>;
+			color = <LED_COLOR_ID_AMBER>;
+			function = LED_FUNCTION_LAN;
+			function-enumerator = <0>;
+			default-state = "keep";
+		};
+	};
+};
+
+&ethphy3 {
+	leds {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		led@1 {
+			reg = <1>;
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_LAN;
+			function-enumerator = <0>;
+			default-state = "keep";
+		};
+
+		led@2 {
+			reg = <2>;
+			color = <LED_COLOR_ID_AMBER>;
+			function = LED_FUNCTION_LAN;
+			function-enumerator = <0>;
+			default-state = "keep";
+		};
+	};
+};
+
+&fec {
+	status = "okay";
+};
+
+&flexcan1 {
+	xceiver-supply = <&reg_3v3>;
+	status = "okay";
+};
+
+&flexcan2 {
+	xceiver-supply = <&reg_3v3>;
+	status = "okay";
+};
+
+&hdmi_pvi {
+	status = "okay";
+};
+
+&hdmi_tx {
+	status = "okay";
+
+	ports {
+		port@1 {
+			hdmi_tx_out: endpoint {
+				remote-endpoint = <&hdmi_connector_in>;
+			};
+		};
+	};
+};
+
+&hdmi_tx_phy {
+	status = "okay";
+};
+
+&i2c1 {
+	tlv320aic3x04: audio-codec@18 {
+		compatible = "ti,tlv320aic32x4";
+		reg = <0x18>;
+		clock-names = "mclk";
+		clocks = <&audio_blk_ctrl IMX8MP_CLK_AUDIOMIX_SAI5_MCLK1>;
+		iov-supply = <&reg_1v8>;
+		ldoin-supply = <&reg_3v3>;
+		assigned-clocks = <&clk IMX8MP_CLK_SAI5>;
+		assigned-clock-parents = <&clk IMX8MP_AUDIO_PLL1_OUT>;
+		assigned-clock-rates = <12288000>;
+	};
+
+	eeprom2: eeprom@57 {
+		compatible = "atmel,24c32";
+		reg = <0x57>;
+		pagesize = <32>;
+		vcc-supply = <&reg_3v3>;
+	};
+};
+
+&ldb_lvds_ch0 {
+	remote-endpoint = <&panel_in_lvds0>;
+};
+
+&ldb_lvds_ch1 {
+	remote-endpoint = <&panel_in_lvds1>;
+};
+
+&lcdif1 {
+	status = "okay";
+};
+
+&lcdif3 {
+	status = "okay";
+};
+
+&pcie_phy {
+	fsl,clkreq-unsupported;
+	status = "okay";
+};
+
+&pcie {
+	status = "okay";
+};
+
+&reg_usdhc2_vmmc {
+	startup-delay-us = <100>;
+	off-on-delay-us = <200000>;
+	status = "okay";
+};
+
+&reg_usdhc2_vqmmc {
+	status = "okay";
+};
+
+&sai3 {
+	status = "okay";
+};
+
+&sai5 {
+	status = "okay";
+};
+
+&snvs_pwrkey {
+	status = "okay";
+};
+
+&uart1 {
+	status = "okay";
+};
+
+&uart2 {
+	status = "okay";
+};
+
+&uart4 {
+	status = "okay";
+};
+
+&usb3_0 {
+	status = "okay";
+};
+
+&usb_dwc3_0 {
+	status = "okay";
+
+	port {
+		usb3_dwc: endpoint {
+			remote-endpoint = <&usb_dr_connector>;
+		};
+	};
+};
+
+&usdhc2 {
+	cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&gpio2 20 GPIO_ACTIVE_HIGH>;
+	vmmc-supply = <&reg_usdhc2_vmmc>;
+	no-mmc;
+	no-sdio;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs.dtsi
new file mode 100644
index 0000000000000..20d55eeead576
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs.dtsi
@@ -0,0 +1,1178 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright 2025-2026 TQ-Systems GmbH <linux@ew.tq-group.com>,
+ * D-82229 Seefeld, Germany.
+ * Author: Paul Gerber
+ */
+
+#include <dt-bindings/net/ti-dp83867.h>
+#include <dt-bindings/phy/phy-imx8-pcie.h>
+#include "imx8mp.dtsi"
+
+/ {
+	model = "TQ-Systems i.MX8MPlus TQMa8MPxS";
+	compatible = "tq,imx8mp-tqma8mpqs", "fsl,imx8mp";
+
+	clk_dp: clk-dp {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <26000000>;
+	};
+
+	clk_xtal25: clk-xtal25 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <25000000>;
+	};
+
+	memory@40000000 {
+		device_type = "memory";
+		reg = <0x0 0x40000000 0 0x80000000>;
+	};
+
+	reg_5v0: regulator-5v0 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_5V0";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+	};
+
+	reg_usdhc2_vmmc: regulator-usdhc2-vmmc {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_reg_usdhc2_vmmc>;
+		regulator-name = "SDIO_PWR_EN";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio2 19 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		status = "disabled";
+	};
+
+	reg_usdhc2_vqmmc: regulator-usdhc2-vqmmc {
+		compatible = "regulator-gpio";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_reg_usdhc2_vqmmc>;
+		regulator-name = "V_SD";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>;
+		states = <1800000 0x1>,
+			 <3300000 0x0>;
+		vin-supply = <&ldo5>;
+		status = "disabled";
+	};
+};
+
+&A53_0 {
+	cpu-supply = <&buck2>;
+};
+
+&audio_blk_ctrl {
+	assigned-clocks = <&clk IMX8MP_AUDIO_PLL1>,
+			  <&clk IMX8MP_AUDIO_PLL2>;
+	assigned-clock-rates = <393216000>, <147333333>;
+};
+
+&ecspi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi1>;
+	cs-gpios = <&gpio5 9 GPIO_ACTIVE_LOW>, <&gpio5 12 GPIO_ACTIVE_LOW>;
+};
+
+/* GBE0 */
+&eqos {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_eqos>;
+	phy-mode = "rgmii-id";
+	phy-handle = <&ethphy0>;
+	snps,force_thresh_dma_mode;
+	snps,mtl-tx-config = <&mtl_tx_setup>;
+	snps,mtl-rx-config = <&mtl_rx_setup>;
+
+	mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy0: ethernet-phy@0 {
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_eqos_phy>;
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <0>;
+			ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
+			ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
+			ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+			ti,dp83867-rxctrl-strap-quirk;
+			ti,clk-output-sel = <DP83867_CLK_O_SEL_OFF>;
+			reset-gpios = <&expander0 6 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <500000>;
+			reset-deassert-us = <50000>;
+			enet-phy-lane-no-swap;
+			interrupt-parent = <&gpio2>;
+			interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
+		};
+
+		ethphy3: ethernet-phy@3 {
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_fec_phy>;
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <3>;
+			ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
+			ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
+			ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+			ti,dp83867-rxctrl-strap-quirk;
+			ti,clk-output-sel = <DP83867_CLK_O_SEL_OFF>;
+			reset-gpios = <&expander0 7 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <500000>;
+			reset-deassert-us = <50000>;
+			enet-phy-lane-no-swap;
+			interrupt-parent = <&gpio2>;
+			interrupts = <1 IRQ_TYPE_LEVEL_LOW>;
+		};
+	};
+
+	mtl_tx_setup: tx-queues-config {
+		snps,tx-queues-to-use = <5>;
+
+		queue0 {
+			snps,dcb-algorithm;
+			snps,priority = <0x1>;
+		};
+
+		queue1 {
+			snps,dcb-algorithm;
+			snps,priority = <0x2>;
+		};
+
+		queue2 {
+			snps,dcb-algorithm;
+			snps,priority = <0x4>;
+		};
+
+		queue3 {
+			snps,dcb-algorithm;
+			snps,priority = <0x8>;
+		};
+
+		queue4 {
+			snps,dcb-algorithm;
+			snps,priority = <0xf0>;
+		};
+	};
+
+	mtl_rx_setup: rx-queues-config {
+		snps,rx-queues-to-use = <5>;
+		snps,rx-sched-sp;
+
+		queue0 {
+			snps,dcb-algorithm;
+			snps,priority = <0x1>;
+			snps,map-to-dma-channel = <0>;
+		};
+
+		queue1 {
+			snps,dcb-algorithm;
+			snps,priority = <0x2>;
+			snps,map-to-dma-channel = <1>;
+		};
+
+		queue2 {
+			snps,dcb-algorithm;
+			snps,priority = <0x4>;
+			snps,map-to-dma-channel = <2>;
+		};
+
+		queue3 {
+			snps,dcb-algorithm;
+			snps,priority = <0x8>;
+			snps,map-to-dma-channel = <3>;
+		};
+
+		queue4 {
+			snps,dcb-algorithm;
+			snps,priority = <0xf0>;
+			snps,map-to-dma-channel = <4>;
+		};
+	};
+};
+
+/* GBE1 */
+&fec {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_fec>;
+	phy-mode = "rgmii-id";
+	phy-handle = <&ethphy3>;
+	fsl,magic-packet;
+};
+
+&flexcan1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan1>;
+};
+
+&flexcan2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan2>;
+};
+
+&flexspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexspi0>;
+	status = "okay";
+
+	flash0: flash@0 {
+		reg = <0>;
+		compatible = "jedec,spi-nor";
+		spi-max-frequency = <66666666>;
+		spi-tx-bus-width = <4>;
+		spi-rx-bus-width = <4>;
+		vcc-supply = <&buck5>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+		};
+	};
+};
+
+&gpio1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio1>;
+
+	gpio-line-names = "SLEEP", "BATLOW#", "", "LID",
+			  "", "GPIO10", "CHARGING#", "CHG_PRSNT#",
+			  "PMIC_IRQ#", "ESPI_CS1_ALERT#", "USB1_OTG_ID", "USB2_OTG_ID",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "";
+
+	pmic_irq_hog: pmic-irq-hog {
+		gpio-hog;
+		gpios = <8 0>;
+		input;
+		line-name = "PMIC_IRQ#";
+	};
+};
+
+&gpio2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hoggpio2>;
+
+	gpio-line-names = "", "", "", "",
+			  "", "", "", "",
+			  "", "", "PERST#", "PEWAKE#",
+			  "SDIO_CD#", "", "", "",
+			  "", "", "", "SDIO_PWR_EN",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "";
+
+	enet0-int-hog {
+		gpio-hog;
+		gpios = <0 0>;
+		input;
+		line-name = "ENET0_INT#";
+	};
+
+	enet1-int-hog {
+		gpio-hog;
+		gpios = <1 0>;
+		input;
+		line-name = "ENET_INT#";
+	};
+
+	pewake-hog {
+		gpio-hog;
+		gpios = <11 0>;
+		input;
+		line-name = "PEWAKE#";
+	};
+};
+
+&gpio3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio3>;
+
+	gpio-line-names = "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "GPIO4",
+			  "GPIO3", "", "", "",
+			  "", "", "", "",
+			  "TEMP_EVENT#", "", "", "";
+
+	temp-event-hog {
+		gpio-hog;
+		gpios = <28 0>;
+		input;
+		line-name = "TEMP_EVENT#";
+	};
+};
+
+&gpio4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio4>, <&pinctrl_sdp>, <&pinctrl_espi_rst>;
+
+	gpio-line-names = "RTC_INT#", "GPIO8", "", "",
+			  "GPIO9", "ESPI_RST#", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "GBE0_SDP_DIR", "",
+			  "GPIO7", "", "", "",
+			  "", "GPIO0", "GPIO1", "",
+			  "GPIO2", "GPIO6", "", "";
+
+	rtc-int-hog {
+		gpio-hog;
+		gpios = <0 0>;
+		input;
+		line-name = "RTC_INT#";
+	};
+
+	espi-reset-hog {
+		gpio-hog;
+		gpios = <5 0>;
+		output-high;
+		line-name = "ESPI_RST#";
+	};
+};
+
+&gpio5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio5>;
+
+	gpio-line-names = "", "", "", "",
+			  "", "GPIO5", "", "",
+			  "", "", "GPIO12", "GPIO11",
+			  "", "GPIO13", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "";
+
+	dp-hpd-int-hog {
+		gpio-hog;
+		gpios = <21 0>;
+		input;
+		line-name = "DP_HPD_INT";
+	};
+};
+
+&hdmi_tx {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hdmi>;
+};
+
+&i2c1 {
+	clock-frequency = <384000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c1>;
+	pinctrl-1 = <&pinctrl_i2c1_gpio>;
+	scl-gpios = <&gpio5 14 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 15 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	eeprom0: eeprom@50 {
+		compatible = "atmel,24c64";
+		reg = <0x50>;
+		pagesize = <32>;
+		read-only;
+		vcc-supply = <&buck5>;
+	};
+
+	pcf85063: rtc@51 {
+		compatible = "nxp,pcf85063a";
+		reg = <0x51>;
+		quartz-load-femtofarads = <7000>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pcf85063>;
+		interrupt-parent = <&gpio4>;
+		interrupts = <0 IRQ_TYPE_EDGE_FALLING>;
+	};
+
+	eeprom1: eeprom@54 {
+		compatible = "atmel,24c64";
+		reg = <0x54>;
+		pagesize = <32>;
+		vcc-supply = <&buck5>;
+	};
+
+	/* protectable identification memory (part of M24C64-D @50) */
+	eeprom@58 {
+		compatible = "atmel,24c64d-wl";
+		reg = <0x58>;
+		size = <32>;
+		pagesize = <32>;
+		vcc-supply = <&buck5>;
+	};
+
+	/* protectable identification memory (part of M24C64-D @54) */
+	eeprom@5c {
+		compatible = "atmel,24c64d-wl";
+		reg = <0x5c>;
+		size = <32>;
+		pagesize = <32>;
+		vcc-supply = <&buck5>;
+	};
+
+	pcieclk: clock-generator@6a {
+		compatible = "renesas,9fgv0241";
+		reg = <0x6a>;
+		clocks = <&clk_xtal25>;
+		#clock-cells = <1>;
+	};
+
+	inertial-sensor@6b {
+		compatible = "st,ism330dhcx";
+		reg = <0x6b>;
+		vdd-supply = <&buck4>;
+		vddio-supply = <&buck4>;
+	};
+};
+
+&i2c2 {
+	clock-frequency = <384000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c2>;
+	pinctrl-1 = <&pinctrl_i2c2_gpio>;
+	scl-gpios = <&gpio5 16 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 17 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	pmic: pmic@25 {
+		reg = <0x25>;
+		compatible = "nxp,pca9450c";
+
+		/* PMIC PCA9450 PMIC_nINT GPIO1_IO08 */
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pmic>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <8 IRQ_TYPE_LEVEL_LOW>;
+
+		regulators {
+			/* V_0V85_SOC: 0.85 .. 0.95 */
+			buck1: BUCK1 {
+				regulator-name = "BUCK1";
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <950000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <3125>;
+			};
+
+			/* VDD_ARM */
+			buck2: BUCK2 {
+				regulator-name = "BUCK2";
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <1000000>;
+				regulator-boot-on;
+				regulator-always-on;
+				nxp,dvs-run-voltage = <950000>;
+				nxp,dvs-standby-voltage = <850000>;
+				regulator-ramp-delay = <3125>;
+			};
+
+			/* VCC3V3 -> VMMC, ... must not be changed */
+			buck4: BUCK4 {
+				regulator-name = "BUCK4";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			/* V_1V8 -> VQMMC, SPI-NOR, ... must not be changed */
+			buck5: BUCK5 {
+				regulator-name = "BUCK5";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			/* V_1V1 -> RAM, ... must not be changed */
+			buck6: BUCK6 {
+				regulator-name = "BUCK6";
+				regulator-min-microvolt = <1100000>;
+				regulator-max-microvolt = <1100000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			/* V_1V8_SNVS */
+			ldo1: LDO1 {
+				regulator-name = "LDO1";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			/* V_1V8_ANA */
+			ldo3: LDO3 {
+				regulator-name = "LDO3";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			/* unused */
+			ldo4: LDO4 {
+				regulator-name = "LDO4";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <3300000>;
+			};
+
+			/* VCC SD IO - switched using SD2 VSELECT */
+			ldo5: LDO5 {
+				regulator-name = "LDO5";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+			};
+		};
+	};
+
+	tmp1075: temperature-sensor@4a {
+		compatible = "ti,tmp1075";
+		reg = <0x4a>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_tmp1075>;
+		vs-supply = <&buck4>;
+	};
+
+	expander0: gpio@73 {
+		compatible = "nxp,pca9538";
+		reg = <0x73>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		vcc-supply = <&buck5>;
+		gpio-line-names = "LCD0_BKLT_EN", "LCD0_VDD_EN",
+				  "LCD1_BKLT_EN", "LCD1_VDD_EN",
+				  "DP_BRIDGE_EN", "HUB_RST#",
+				  "ENET0_RESET#", "ENET1_RESET#";
+	};
+};
+
+&i2c3 {
+	clock-frequency = <384000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c3>;
+	pinctrl-1 = <&pinctrl_i2c3_gpio>;
+	scl-gpios = <&gpio5 18 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 19 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	dp_bridge: dp-bridge@f {
+		compatible = "toshiba,tc9595", "toshiba,tc358767";
+		reg = <0xf>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_tc9595>;
+		clock-names = "ref";
+		clocks = <&clk_dp>;
+		reset-gpios = <&expander0 4 GPIO_ACTIVE_HIGH>;
+		toshiba,hpd-pin = <0>;
+		interrupts-extended = <&gpio5 21 IRQ_TYPE_LEVEL_HIGH>;
+		status = "disabled";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				dsi_in: endpoint {
+					remote-endpoint = <&dsi_out>;
+					data-lanes = <1 2 3 4>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+
+				endpoint {
+					toshiba,pre-emphasis = /bits/ 8 <1 1>;
+				};
+			};
+		};
+	};
+};
+
+&i2c4 {
+	clock-frequency = <384000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c4>;
+	pinctrl-1 = <&pinctrl_i2c4_gpio>;
+	scl-gpios = <&gpio2 4 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio2 5 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+};
+
+&i2c6 {
+	clock-frequency = <384000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c6>;
+	pinctrl-1 = <&pinctrl_i2c6_gpio>;
+	scl-gpios = <&gpio2 2 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio2 3 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+};
+
+// Change parent clock of disp1 pixel clock to audio_pll2
+&media_blk_ctrl {
+	assigned-clocks = <&clk IMX8MP_CLK_MEDIA_AXI>,
+			  <&clk IMX8MP_CLK_MEDIA_APB>,
+			  <&clk IMX8MP_CLK_MEDIA_DISP1_PIX>,
+			  <&clk IMX8MP_CLK_MEDIA_DISP2_PIX>,
+			  <&clk IMX8MP_CLK_MEDIA_ISP>,
+			  <&clk IMX8MP_VIDEO_PLL1>;
+	assigned-clock-parents = <&clk IMX8MP_SYS_PLL2_1000M>,
+				 <&clk IMX8MP_SYS_PLL1_800M>,
+				 <&clk IMX8MP_AUDIO_PLL2_OUT>,
+				 <&clk IMX8MP_VIDEO_PLL1_OUT>,
+				 <&clk IMX8MP_SYS_PLL2_500M>;
+};
+
+&mipi_dsi {
+	samsung,burst-clock-frequency = <1000000000>;
+	samsung,esc-clock-frequency = <10000000>;
+
+	ports {
+		port@1 {
+			reg = <1>;
+
+			dsi_out: endpoint {
+				remote-endpoint = <&dsi_in>;
+				data-lanes = <1 2 3 4>;
+			};
+		};
+	};
+};
+
+&pcie_phy {
+	fsl,refclk-pad-mode = <IMX8_PCIE_REFCLK_PAD_INPUT>;
+	clock-names = "ref";
+	clocks = <&pcieclk 0>;
+};
+
+&pcie {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pcie>;
+	clocks = <&clk IMX8MP_CLK_HSIO_ROOT>,
+		 <&clk IMX8MP_CLK_HSIO_AXI>,
+		 <&clk IMX8MP_CLK_PCIE_ROOT>;
+	clock-names = "pcie", "pcie_bus", "pcie_aux";
+	reset-gpios = <&gpio2 10 GPIO_ACTIVE_LOW>;
+	assigned-clocks = <&clk IMX8MP_CLK_PCIE_AUX>;
+	assigned-clock-rates = <10000000>;
+	assigned-clock-parents = <&clk IMX8MP_SYS_PLL2_50M>;
+};
+
+&pwm2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm2>;
+};
+
+&pwm3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm3>;
+};
+
+&sai3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sai3>;
+	assigned-clocks = <&clk IMX8MP_CLK_SAI3>;
+	assigned-clock-parents = <&clk IMX8MP_AUDIO_PLL1_OUT>;
+	assigned-clock-rates = <12288000>;
+};
+
+&sai5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sai5>;
+	assigned-clocks = <&clk IMX8MP_CLK_SAI5>;
+	assigned-clock-parents = <&clk IMX8MP_AUDIO_PLL1_OUT>;
+	assigned-clock-rates = <12288000>;
+	fsl,sai-mclk-direction-output;
+};
+
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	assigned-clocks = <&clk IMX8MP_CLK_UART1>;
+	assigned-clock-parents = <&clk IMX8MP_SYS_PLL1_80M>;
+	uart-has-rtscts;
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart2>;
+	assigned-clocks = <&clk IMX8MP_CLK_UART2>;
+	assigned-clock-parents = <&clk IMX8MP_SYS_PLL1_80M>;
+	uart-has-rtscts;
+};
+
+&uart3 {
+	/* console */
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3>;
+	status = "okay";
+};
+
+&uart4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart4>;
+	assigned-clocks = <&clk IMX8MP_CLK_UART4>;
+	assigned-clock-parents = <&clk IMX8MP_SYS_PLL1_80M>;
+};
+
+&usb3_phy0 {
+	vbus-supply = <&reg_5v0>;
+	status = "okay";
+};
+
+&usb3_phy1 {
+	vbus-supply = <&reg_5v0>;
+	status = "okay";
+};
+
+&usb3_0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usb0>;
+	fsl,over-current-active-low;
+};
+
+&usb3_1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usb1>;
+	fsl,over-current-active-low;
+	status = "okay";
+};
+
+&usb_dwc3_0 {
+	/* dual role is implemented, but not a full featured OTG */
+	hnp-disable;
+	srp-disable;
+	adp-disable;
+	dr_mode = "otg";
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	/* SMARC-2 USB0 interface only supports USB 2.0 signals */
+	maximum-speed = "high-speed";
+};
+
+&usb_dwc3_1 {
+	dr_mode = "host";
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	hub_2_0: hub@1 {
+		compatible = "usb451,8142";
+		reg = <1>;
+		peer-hub = <&hub_3_0>;
+		reset-gpios = <&expander0 5 GPIO_ACTIVE_LOW>;
+		vdd-supply = <&reg_3v3>;
+	};
+
+	hub_3_0: hub@2 {
+		compatible = "usb451,8140";
+		reg = <2>;
+		peer-hub = <&hub_2_0>;
+		reset-gpios = <&expander0 5 GPIO_ACTIVE_LOW>;
+		vdd-supply = <&reg_3v3>;
+	};
+};
+
+&usdhc2 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
+	bus-width = <4>;
+	vqmmc-supply = <&reg_usdhc2_vqmmc>;
+	/* NOTE: CD / WP and VMMC support depends on mainboard */
+};
+
+&usdhc3 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	pinctrl-1 = <&pinctrl_usdhc3_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc3_200mhz>;
+	bus-width = <8>;
+	non-removable;
+	no-sdio;
+	no-sd;
+	vmmc-supply = <&buck4>;
+	vqmmc-supply = <&buck5>;
+	status = "okay";
+};
+
+&wdog1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_wdog>;
+	fsl,ext-reset-output;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_ecspi1: ecspi1grp {
+		fsl,pins = <MX8MP_IOMUXC_ECSPI1_MISO__ECSPI1_MISO	0x1c0>,
+			   <MX8MP_IOMUXC_ECSPI1_MOSI__ECSPI1_MOSI	0x1c0>,
+			   <MX8MP_IOMUXC_ECSPI1_SCLK__ECSPI1_SCLK	0x1c0>,
+			   <MX8MP_IOMUXC_ECSPI1_SS0__GPIO5_IO09		0x1c0>,
+			   <MX8MP_IOMUXC_ECSPI2_MISO__GPIO5_IO12	0x1c0>;
+	};
+
+	pinctrl_eqos: eqosgrp {
+		fsl,pins = <MX8MP_IOMUXC_ENET_MDC__ENET_QOS_MDC				0x40000044>,
+			   <MX8MP_IOMUXC_ENET_MDIO__ENET_QOS_MDIO			0x40000044>,
+			   <MX8MP_IOMUXC_ENET_RD0__ENET_QOS_RGMII_RD0			0x90>,
+			   <MX8MP_IOMUXC_ENET_RD1__ENET_QOS_RGMII_RD1			0x90>,
+			   <MX8MP_IOMUXC_ENET_RD2__ENET_QOS_RGMII_RD2			0x90>,
+			   <MX8MP_IOMUXC_ENET_RD3__ENET_QOS_RGMII_RD3			0x90>,
+			   <MX8MP_IOMUXC_ENET_RXC__CCM_ENET_QOS_CLOCK_GENERATE_RX_CLK	0x90>,
+			   <MX8MP_IOMUXC_ENET_RX_CTL__ENET_QOS_RGMII_RX_CTL		0x90>,
+			   <MX8MP_IOMUXC_ENET_TD0__ENET_QOS_RGMII_TD0			0x12>,
+			   <MX8MP_IOMUXC_ENET_TD1__ENET_QOS_RGMII_TD1			0x12>,
+			   <MX8MP_IOMUXC_ENET_TD2__ENET_QOS_RGMII_TD2			0x12>,
+			   <MX8MP_IOMUXC_ENET_TD3__ENET_QOS_RGMII_TD3			0x12>,
+			   <MX8MP_IOMUXC_ENET_TX_CTL__ENET_QOS_RGMII_TX_CTL		0x12>,
+			   <MX8MP_IOMUXC_ENET_TXC__CCM_ENET_QOS_CLOCK_GENERATE_TX_CLK	0x14>;
+	};
+
+	pinctrl_eqos_event: eqosevtgrp {
+		fsl,pins = <MX8MP_IOMUXC_SAI2_MCLK__ENET_QOS_1588_EVENT3_IN	0x100>,
+			   <MX8MP_IOMUXC_SAI2_TXFS__ENET_QOS_1588_EVENT3_OUT	0x1c0>;
+	};
+
+	pinctrl_eqos_phy: eqosphygrp {
+		fsl,pins = <MX8MP_IOMUXC_SD1_CLK__GPIO2_IO00	0x1c0>;
+	};
+
+	pinctrl_espi_rst: espirstgrp {
+		fsl,pins = <MX8MP_IOMUXC_SAI1_RXD3__GPIO4_IO05		0x144>;
+	};
+
+	pinctrl_fec: fecgrp {
+		fsl,pins = <MX8MP_IOMUXC_SAI1_RXD4__ENET1_RGMII_RD0	0x90>,
+			   <MX8MP_IOMUXC_SAI1_RXD5__ENET1_RGMII_RD1	0x90>,
+			   <MX8MP_IOMUXC_SAI1_RXD6__ENET1_RGMII_RD2	0x90>,
+			   <MX8MP_IOMUXC_SAI1_RXD7__ENET1_RGMII_RD3	0x90>,
+			   <MX8MP_IOMUXC_SAI1_TXC__ENET1_RGMII_RXC	0x90>,
+			   <MX8MP_IOMUXC_SAI1_TXFS__ENET1_RGMII_RX_CTL	0x90>,
+			   <MX8MP_IOMUXC_SAI1_TXD0__ENET1_RGMII_TD0	0x12>,
+			   <MX8MP_IOMUXC_SAI1_TXD1__ENET1_RGMII_TD1	0x12>,
+			   <MX8MP_IOMUXC_SAI1_TXD2__ENET1_RGMII_TD2	0x12>,
+			   <MX8MP_IOMUXC_SAI1_TXD3__ENET1_RGMII_TD3	0x12>,
+			   <MX8MP_IOMUXC_SAI1_TXD4__ENET1_RGMII_TX_CTL	0x12>,
+			   <MX8MP_IOMUXC_SAI1_TXD5__ENET1_RGMII_TXC	0x14>;
+	};
+
+	pinctrl_fec_event: fecevtgrp {
+		fsl,pins = <MX8MP_IOMUXC_SAI1_RXD1__ENET1_1588_EVENT1_OUT	0x100>,
+			   <MX8MP_IOMUXC_SAI1_RXD0__ENET1_1588_EVENT1_IN	0x1c0>;
+	};
+
+	pinctrl_fec_phy: fecphygrp {
+		fsl,pins = <MX8MP_IOMUXC_SD1_CMD__GPIO2_IO01	0x1c0>;
+	};
+
+	pinctrl_flexcan1: flexcan1grp {
+		fsl,pins = <MX8MP_IOMUXC_SAI5_RXD1__CAN1_TX	0x150>,
+			   <MX8MP_IOMUXC_SAI5_RXD2__CAN1_RX	0x150>;
+	};
+
+	pinctrl_flexcan2: flexcan2grp {
+		fsl,pins = <MX8MP_IOMUXC_SAI5_RXD3__CAN2_TX	0x150>,
+			   <MX8MP_IOMUXC_SAI5_MCLK__CAN2_RX	0x150>;
+	};
+
+	pinctrl_flexspi0: flexspi0grp {
+		fsl,pins = <MX8MP_IOMUXC_NAND_ALE__FLEXSPI_A_SCLK	0x142>,
+			   <MX8MP_IOMUXC_NAND_CE0_B__FLEXSPI_A_SS0_B	0x82>,
+			   <MX8MP_IOMUXC_NAND_DQS__FLEXSPI_A_DQS	0x40000010>,
+			   <MX8MP_IOMUXC_NAND_DATA00__FLEXSPI_A_DATA00	0x82>,
+			   <MX8MP_IOMUXC_NAND_DATA01__FLEXSPI_A_DATA01	0x82>,
+			   <MX8MP_IOMUXC_NAND_DATA02__FLEXSPI_A_DATA02	0x82>,
+			   <MX8MP_IOMUXC_NAND_DATA03__FLEXSPI_A_DATA03	0x82>;
+	};
+
+	pinctrl_sdp: gbegrp {
+		fsl,pins = <MX8MP_IOMUXC_SAI1_TXD6__GPIO4_IO18	0x10>,
+			   <MX8MP_IOMUXC_SAI1_TXD7__GPIO4_IO19	0x10>;
+	};
+
+	pinctrl_gpio1: gpio1grp {
+		fsl,pins = <MX8MP_IOMUXC_GPIO1_IO00__GPIO1_IO00		0x10>,
+			   <MX8MP_IOMUXC_GPIO1_IO01__GPIO1_IO01		0x10>,
+			   <MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03		0x10>,
+			   <MX8MP_IOMUXC_GPIO1_IO05__GPIO1_IO05		0x10>,
+			   <MX8MP_IOMUXC_GPIO1_IO06__GPIO1_IO06		0x10>,
+			   <MX8MP_IOMUXC_GPIO1_IO07__GPIO1_IO07		0x80>,
+			   <MX8MP_IOMUXC_GPIO1_IO09__GPIO1_IO09		0x80>;
+	};
+
+	pinctrl_gpio3: gpio3grp {
+		fsl,pins = <MX8MP_IOMUXC_SAI5_RXFS__GPIO3_IO19		0x10>,
+			   <MX8MP_IOMUXC_SAI5_RXC__GPIO3_IO20		0x10>;
+	};
+
+	pinctrl_gpio4: gpio4grp {
+		fsl,pins = <MX8MP_IOMUXC_SAI1_RXC__GPIO4_IO01		0x10>,
+			   <MX8MP_IOMUXC_SAI1_RXD2__GPIO4_IO04		0x10>,
+			   <MX8MP_IOMUXC_SAI1_MCLK__GPIO4_IO20		0x10>,
+			   <MX8MP_IOMUXC_SAI2_TXC__GPIO4_IO25		0x10>,
+			   <MX8MP_IOMUXC_SAI2_TXD0__GPIO4_IO26		0x10>,
+			   <MX8MP_IOMUXC_SAI3_RXFS__GPIO4_IO28		0x10>,
+			   <MX8MP_IOMUXC_SAI3_RXC__GPIO4_IO29		0x10>;
+	};
+
+	pinctrl_gpio5: gpio5grp {
+		fsl,pins = <MX8MP_IOMUXC_SPDIF_EXT_CLK__GPIO5_IO05	0x10>,
+			   <MX8MP_IOMUXC_ECSPI2_SCLK__GPIO5_IO10	0x10>,
+			   <MX8MP_IOMUXC_ECSPI2_MOSI__GPIO5_IO11	0x10>,
+			   <MX8MP_IOMUXC_ECSPI2_SS0__GPIO5_IO13		0x10>;
+	};
+
+	pinctrl_hdmi: hdmigrp {
+		fsl,pins = <MX8MP_IOMUXC_HDMI_DDC_SCL__HDMIMIX_HDMI_SCL	0x400001c2>,
+			   <MX8MP_IOMUXC_HDMI_DDC_SDA__HDMIMIX_HDMI_SDA	0x400001c2>,
+			   <MX8MP_IOMUXC_HDMI_HPD__HDMIMIX_HDMI_HPD	0x40000010>;
+	};
+
+	pinctrl_hoggpio2: hoggpio2grp {
+		fsl,pins = <MX8MP_IOMUXC_SD1_STROBE__GPIO2_IO11		0x140>;
+	};
+
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <MX8MP_IOMUXC_I2C1_SCL__I2C1_SCL		0x400001e2>,
+			   <MX8MP_IOMUXC_I2C1_SDA__I2C1_SDA		0x400001e2>;
+	};
+
+	pinctrl_i2c1_gpio: i2c1-gpiogrp {
+		fsl,pins = <MX8MP_IOMUXC_I2C1_SCL__GPIO5_IO14		0x400001e2>,
+			   <MX8MP_IOMUXC_I2C1_SDA__GPIO5_IO15		0x400001e2>;
+	};
+
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <MX8MP_IOMUXC_I2C2_SCL__I2C2_SCL		0x400001e2>,
+			   <MX8MP_IOMUXC_I2C2_SDA__I2C2_SDA		0x400001e2>;
+	};
+
+	pinctrl_i2c2_gpio: i2c2-gpiogrp {
+		fsl,pins = <MX8MP_IOMUXC_I2C2_SCL__GPIO5_IO16		0x400001e2>,
+			   <MX8MP_IOMUXC_I2C2_SDA__GPIO5_IO17		0x400001e2>;
+	};
+
+	pinctrl_i2c3: i2c3grp {
+		fsl,pins = <MX8MP_IOMUXC_I2C3_SCL__I2C3_SCL		0x400001e2>,
+			   <MX8MP_IOMUXC_I2C3_SDA__I2C3_SDA		0x400001e2>;
+	};
+
+	pinctrl_i2c3_gpio: i2c3-gpiogrp {
+		fsl,pins = <MX8MP_IOMUXC_I2C3_SCL__GPIO5_IO18		0x400001e2>,
+			   <MX8MP_IOMUXC_I2C3_SDA__GPIO5_IO19		0x400001e2>;
+	};
+
+	pinctrl_i2c4: i2c4grp {
+		fsl,pins = <MX8MP_IOMUXC_SD1_DATA2__I2C4_SCL		0x400001e2>,
+			   <MX8MP_IOMUXC_SD1_DATA3__I2C4_SDA		0x400001e2>;
+	};
+
+	pinctrl_i2c4_gpio: i2c4-gpiogrp {
+		fsl,pins = <MX8MP_IOMUXC_SD1_DATA2__GPIO2_IO04		0x400001e2>,
+			   <MX8MP_IOMUXC_SD1_DATA3__GPIO2_IO05		0x400001e2>;
+	};
+
+	pinctrl_i2c6: i2c6grp {
+		fsl,pins = <MX8MP_IOMUXC_SD1_DATA0__I2C6_SCL		0x400001e2>,
+			   <MX8MP_IOMUXC_SD1_DATA1__I2C6_SDA		0x400001e2>;
+	};
+
+	pinctrl_i2c6_gpio: i2c6-gpiogrp {
+		fsl,pins = <MX8MP_IOMUXC_SD1_DATA0__GPIO2_IO02		0x400001e2>,
+			   <MX8MP_IOMUXC_SD1_DATA1__GPIO2_IO03		0x400001e2>;
+	};
+
+	pinctrl_pcf85063: pcf85063grp {
+		fsl,pins = <MX8MP_IOMUXC_SAI1_RXFS__GPIO4_IO00		0x80>;
+	};
+
+	pinctrl_pcie: pciegrp {
+		fsl,pins = <MX8MP_IOMUXC_I2C4_SCL__PCIE_CLKREQ_B	0x60>,
+			   <MX8MP_IOMUXC_SD1_RESET_B__GPIO2_IO10	0x94>;
+	};
+
+	pinctrl_pmic: pmicirqgrp {
+		fsl,pins = <MX8MP_IOMUXC_GPIO1_IO08__GPIO1_IO08		0x1c0>;
+	};
+
+	pinctrl_pwm2: pwm2grp {
+		fsl,pins = <MX8MP_IOMUXC_SPDIF_RX__PWM2_OUT		0x14>;
+	};
+
+	pinctrl_pwm3: pwm3grp {
+		fsl,pins = <MX8MP_IOMUXC_SPDIF_TX__PWM3_OUT		0x14>;
+	};
+
+	pinctrl_reg_usdhc2_vmmc: regusdhc2vmmcgrp {
+		fsl,pins = <MX8MP_IOMUXC_SD2_RESET_B__GPIO2_IO19	0x10>;
+	};
+
+	pinctrl_reg_usdhc2_vqmmc: regusdhc2vqmmcgrp {
+		fsl,pins = <MX8MP_IOMUXC_GPIO1_IO04__GPIO1_IO04		0xc0>;
+	};
+
+	pinctrl_sai3: sai3grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SAI3_RXD__AUDIOMIX_SAI3_RX_DATA00	0x94
+			MX8MP_IOMUXC_SAI3_TXD__AUDIOMIX_SAI3_TX_DATA00	0x94
+			MX8MP_IOMUXC_SAI3_TXFS__AUDIOMIX_SAI3_TX_SYNC	0x94
+			MX8MP_IOMUXC_SAI3_TXC__AUDIOMIX_SAI3_TX_BCLK	0x94
+		>;
+	};
+
+	pinctrl_sai5: sai5grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SAI3_MCLK__AUDIOMIX_SAI5_MCLK	0x94
+			MX8MP_IOMUXC_SAI5_RXD0__AUDIOMIX_SAI5_RX_DATA00	0x94
+			MX8MP_IOMUXC_SAI2_RXD0__AUDIOMIX_SAI5_TX_DATA00	0x94
+			MX8MP_IOMUXC_SAI2_RXFS__AUDIOMIX_SAI5_TX_SYNC	0x94
+			MX8MP_IOMUXC_SAI2_RXC__AUDIOMIX_SAI5_TX_BCLK	0x94
+		>;
+	};
+
+	pinctrl_tc9595: tc9595grp {
+		fsl,pins = <MX8MP_IOMUXC_I2C4_SDA__GPIO5_IO21		0x10>;
+	};
+
+	pinctrl_tmp1075: tmp1075grp {
+		fsl,pins = <MX8MP_IOMUXC_HDMI_CEC__GPIO3_IO28	0x140>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <MX8MP_IOMUXC_UART1_TXD__UART1_DCE_TX	0x140>,
+			   <MX8MP_IOMUXC_UART1_RXD__UART1_DCE_RX	0x140>,
+			   <MX8MP_IOMUXC_UART3_RXD__UART1_DCE_CTS	0x140>,
+			   <MX8MP_IOMUXC_UART3_TXD__UART1_DCE_RTS	0x140>;
+	};
+
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <MX8MP_IOMUXC_UART2_TXD__UART2_DCE_TX	0x140>,
+			   <MX8MP_IOMUXC_UART2_RXD__UART2_DCE_RX	0x140>,
+			   <MX8MP_IOMUXC_SD1_DATA4__UART2_DCE_RTS	0x140>,
+			   <MX8MP_IOMUXC_SD1_DATA5__UART2_DCE_CTS	0x140>;
+	};
+
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <MX8MP_IOMUXC_SD1_DATA6__UART3_DCE_TX	0x140>,
+			   <MX8MP_IOMUXC_SD1_DATA7__UART3_DCE_RX	0x140>;
+	};
+
+	pinctrl_uart4: uart4grp {
+		fsl,pins = <MX8MP_IOMUXC_UART4_RXD__UART4_DCE_RX	0x140>,
+			   <MX8MP_IOMUXC_UART4_TXD__UART4_DCE_TX	0x140>;
+	};
+
+	pinctrl_usb0: usb0grp {
+		fsl,pins = <MX8MP_IOMUXC_GPIO1_IO13__USB1_OTG_OC	0x1c0>,
+			   <MX8MP_IOMUXC_GPIO1_IO12__USB1_OTG_PWR	0x1c0>;
+	};
+
+	pinctrl_usb1: usb1grp {
+		fsl,pins = <MX8MP_IOMUXC_GPIO1_IO15__USB2_OTG_OC	0x1c0>,
+			   <MX8MP_IOMUXC_GPIO1_IO14__USB2_OTG_PWR	0x1c0>;
+	};
+
+	pinctrl_usbcon0: usb0congrp {
+		fsl,pins = <MX8MP_IOMUXC_GPIO1_IO10__GPIO1_IO10		0x1c0>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK		0x192>,
+			   <MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD		0x1d2>,
+			   <MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0	0x1d2>,
+			   <MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1	0x1d2>,
+			   <MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2	0x1d2>,
+			   <MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3	0x1d2>;
+	};
+
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins = <MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK		0x194>,
+			   <MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD		0x1d4>,
+			   <MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0	0x1d4>,
+			   <MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1	0x1d4>,
+			   <MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2	0x1d4>,
+			   <MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3	0x1d4>;
+	};
+
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK		0x194>,
+			   <MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD		0x1d4>,
+			   <MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0	0x1d4>,
+			   <MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1	0x1d4>,
+			   <MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2	0x1d4>,
+			   <MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3	0x1d4>;
+	};
+
+	pinctrl_usdhc2_gpio: usdhc2-gpiogrp {
+		fsl,pins = <MX8MP_IOMUXC_SD2_CD_B__GPIO2_IO12		0x1c0>,
+			   <MX8MP_IOMUXC_SD2_WP__GPIO2_IO20		0x1c0>;
+	};
+
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK		0x194>,
+			   <MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD		0x1d4>,
+			   <MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7		0x1d4>,
+			   <MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE	0x84>,
+			   <MX8MP_IOMUXC_NAND_READY_B__USDHC3_RESET_B	0x84>;
+	};
+
+	pinctrl_usdhc3_100mhz: usdhc3-100mhzgrp {
+		fsl,pins = <MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK		0x194>,
+			   <MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD		0x1d4>,
+			   <MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7		0x1d4>,
+			   <MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE	0x84>,
+			   <MX8MP_IOMUXC_NAND_READY_B__USDHC3_RESET_B	0x84>;
+	};
+
+	pinctrl_usdhc3_200mhz: usdhc3-200mhzgrp {
+		fsl,pins = <MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK		0x194>,
+			   <MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD		0x1d4>,
+			   <MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6	0x1d4>,
+			   <MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7		0x1d4>,
+			   <MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE	0x84>,
+			   <MX8MP_IOMUXC_NAND_READY_B__USDHC3_RESET_B	0x84>;
+	};
+
+	pinctrl_wdog: wdoggrp {
+		fsl,pins = <MX8MP_IOMUXC_GPIO1_IO02__WDOG1_WDOG_B	0x1c4>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-var-dart-sonata.dts b/arch/arm64/boot/dts/freescale/imx8mp-var-dart-sonata.dts
new file mode 100644
index 0000000000000..983d67a0c1de7
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-var-dart-sonata.dts
@@ -0,0 +1,733 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Variscite Sonata carrier board for DART-MX8M-PLUS
+ *
+ * Link: https://variscite.com/carrier-boards/sonata-board/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/phy/phy-imx8-pcie.h>
+#include "imx8mp-var-dart.dtsi"
+
+/ {
+	model = "Variscite DART-MX8M-PLUS on Sonata-Board";
+	compatible = "variscite,var-dart-mx8mp-sonata",
+		     "variscite,var-dart-mx8mp",
+		     "fsl,imx8mp";
+
+	chosen {
+		stdout-path = &uart1;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		button-home {
+			label = "Home";
+			linux,code = <KEY_HOME>;
+			gpios = <&pca6408_1 4 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+
+		button-up {
+			label = "Up";
+			linux,code = <KEY_UP>;
+			gpios = <&pca6408_1 5 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+
+		button-down {
+			label = "Down";
+			linux,code = <KEY_DOWN>;
+			gpios = <&pca6408_1 6 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+
+		button-back {
+			label = "Back";
+			linux,code = <KEY_BACK>;
+			gpios = <&pca6408_1 7 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+	};
+
+	gpio-leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_leds>;
+
+		led-emmc {
+			gpios = <&gpio4 18 GPIO_ACTIVE_HIGH>;
+			label = "eMMC";
+			linux,default-trigger = "mmc2";
+		};
+	};
+
+	native-hdmi-connector {
+		compatible = "hdmi-connector";
+		label = "HDMI OUT";
+		type = "a";
+
+		port {
+			hdmi_in: endpoint {
+				remote-endpoint = <&hdmi_tx_out>;
+			};
+		};
+	};
+
+	clk40m: oscillator {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <40000000>;
+		clock-output-names = "can_osc";
+	};
+
+	pcie0_refclk: pcie0-refclk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <100000000>;
+	};
+
+	reg_usdhc2_vmmc: regulator-vmmc-usdhc2 {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_vmmc_usdhc2>;
+		regulator-name = "VSD_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio2 19 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		startup-delay-us = <100>;
+		off-on-delay-us = <12000>;
+	};
+
+	sound-hdmi {
+		compatible = "fsl,imx-audio-hdmi";
+		model = "audio-hdmi";
+		audio-cpu = <&aud2htx>;
+		hdmi-out;
+	};
+
+	sound-xcvr {
+		compatible = "fsl,imx-audio-card";
+		model = "imx-audio-xcvr";
+
+		pri-dai-link {
+			link-name = "XCVR PCM";
+
+			cpu {
+				sound-dai = <&xcvr>;
+			};
+		};
+	};
+};
+
+&aud2htx {
+	status = "okay";
+};
+
+&ecspi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi1>;
+	cs-gpios = <&gpio5  9 GPIO_ACTIVE_LOW>,
+		   <&gpio1 12 GPIO_ACTIVE_LOW>;
+	status = "okay";
+
+	ads7846: touchscreen@0 {
+		compatible = "ti,ads7846";
+		reg = <0>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_restouch>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <7 IRQ_TYPE_EDGE_FALLING>;
+		spi-max-frequency = <1500000>;
+		pendown-gpio = <&gpio1 7 GPIO_ACTIVE_LOW>;
+		ti,x-min = /bits/ 16 <125>;
+		ti,x-max = /bits/ 16 <4008>;
+		ti,y-min = /bits/ 16 <282>;
+		ti,y-max = /bits/ 16 <3864>;
+		ti,x-plate-ohms = /bits/ 16 <180>;
+		ti,pressure-max = /bits/ 16 <255>;
+		ti,debounce-max = /bits/ 16 <10>;
+		ti,debounce-tol = /bits/ 16 <3>;
+		ti,debounce-rep = /bits/ 16 <1>;
+		ti,settle-delay-usec = /bits/ 16 <150>;
+		ti,keep-vref-on;
+		wakeup-source;
+	};
+
+	can0: can@1 {
+		compatible = "microchip,mcp251xfd";
+		reg = <1>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_can>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <6 IRQ_TYPE_LEVEL_LOW>;
+		microchip,rx-int-gpios = <&gpio5 4 GPIO_ACTIVE_LOW>;
+		clocks = <&clk40m>;
+		spi-max-frequency = <20000000>;
+	};
+};
+
+&eqos {
+	mdio {
+		ethphy1: ethernet-phy@1 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			reset-gpios = <&pca6408_2 0 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <10000>;
+			reset-deassert-us = <20000>;
+			vddio-supply = <&reg_phy_vddio>;
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@0 {
+					reg = <0>;
+					color = <LED_COLOR_ID_YELLOW>;
+					function = LED_FUNCTION_LAN;
+					linux,default-trigger = "netdev";
+				};
+
+				led@1 {
+					reg = <1>;
+					color = <LED_COLOR_ID_GREEN>;
+					function = LED_FUNCTION_LAN;
+					linux,default-trigger = "netdev";
+				};
+			};
+		};
+	};
+};
+
+&ethphy0 {
+	leds {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		led@0 {
+			reg = <0>;
+			color = <LED_COLOR_ID_YELLOW>;
+			function = LED_FUNCTION_LAN;
+			linux,default-trigger = "netdev";
+		};
+
+		led@1 {
+			reg = <1>;
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_LAN;
+			linux,default-trigger = "netdev";
+		};
+	};
+};
+
+&fec {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_fec>;
+	/*
+	 * The required RGMII TX and RX 2ns delays are implemented directly
+	 * in hardware via passive delay elements on the SOM PCB.
+	 * No delay configuration is needed in software via PHY driver.
+	 */
+	phy-mode = "rgmii";
+	phy-handle = <&ethphy1>;
+	status = "okay";
+};
+
+&flexcan1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan1>;
+	status = "okay";
+};
+
+&flexcan2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan2>;
+	status = "okay";
+};
+
+&hdmi_pai {
+	status = "okay";
+};
+
+&hdmi_pvi {
+	status = "okay";
+};
+
+&hdmi_tx {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hdmi>;
+	status = "okay";
+
+	ports {
+		port@1 {
+			hdmi_tx_out: endpoint {
+				remote-endpoint = <&hdmi_in>;
+			};
+		};
+	};
+};
+
+&hdmi_tx_phy {
+	status = "okay";
+};
+
+&i2c2 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c2>;
+	pinctrl-1 = <&pinctrl_i2c2_gpio>;
+	scl-gpios = <&gpio5 16 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 17 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	pca9534: gpio@22 {
+		compatible = "nxp,pca9534";
+		reg = <0x22>;
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		eth10g-en-hog {
+			gpio-hog;
+			gpios = <5 GPIO_ACTIVE_HIGH>;
+			output-low;
+			line-name = "eth10g_sel";
+		};
+
+		pcie2-en-hog {
+			gpio-hog;
+			gpios = <6 GPIO_ACTIVE_HIGH>;
+			output-high;
+			line-name = "pcie2_sel";
+		};
+
+		/* RGB_SEL
+		 * Keep signal low and not floating
+		 */
+		lvds-brg-enable-hog {
+			gpio-hog;
+			gpios = <7 GPIO_ACTIVE_HIGH>;
+			output-low;
+			line-name = "lvds_brg_en";
+		};
+	};
+
+	/* Capacitive touch controller */
+	ft5x06_ts: touchscreen@38 {
+		compatible = "edt,edt-ft5206";
+		reg = <0x38>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_captouch>;
+		reset-gpios = <&pca6408_2 4 GPIO_ACTIVE_LOW>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <14 IRQ_TYPE_EDGE_FALLING>;
+		touchscreen-size-x = <800>;
+		touchscreen-size-y = <480>;
+		touchscreen-inverted-x;
+		touchscreen-inverted-y;
+		wakeup-source;
+	};
+
+	typec@3d {
+		compatible = "nxp,ptn5150";
+		reg = <0x3d>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_extcon>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <10 IRQ_TYPE_EDGE_FALLING>;
+
+		port {
+			typec_dr_sw: endpoint {
+				remote-endpoint = <&usb3_drd_sw>;
+			};
+		};
+	};
+
+	rtc@68 {
+		compatible = "dallas,ds1337";
+		reg = <0x68>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_rtc>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <15 IRQ_TYPE_EDGE_FALLING>;
+		wakeup-source;
+	};
+};
+
+&i2c3 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c3>;
+	pinctrl-1 = <&pinctrl_i2c3_gpio>;
+	scl-gpios = <&gpio5 18 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 19 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+};
+
+&i2c4 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c4>;
+	pinctrl-1 = <&pinctrl_i2c4_gpio>;
+	scl-gpios = <&gpio5 20 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 21 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	pca6408_1: gpio@20 {
+		compatible = "nxp,pcal6408";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pca6408>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <5 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	pca6408_2: gpio@21 {
+		compatible = "nxp,pcal6408";
+		reg = <0x21>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <5 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	st33ktpm2xi2c: tpm@2e {
+		compatible = "st,st33ktpm2xi2c", "tcg,tpm-tis-i2c";
+		reg = <0x2e>;
+		label = "tpm";
+		reset-gpios = <&pca9534 0 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&lcdif3 {
+	status = "okay";
+};
+
+&pcie {
+	reset-gpios = <&pca6408_2 3 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&pcie_phy {
+	fsl,refclk-pad-mode = <IMX8_PCIE_REFCLK_PAD_INPUT>;
+	clocks = <&pcie0_refclk>;
+	clock-names = "ref";
+	status = "okay";
+};
+
+&pwm1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm1>;
+	status = "okay";
+};
+
+&snvs_pwrkey {
+	status = "okay";
+};
+
+&snvs_rtc {
+	status = "disabled";
+};
+
+/* Console */
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	status = "okay";
+};
+
+/* Header */
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart2>;
+	status = "okay";
+};
+
+/* Header */
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3>;
+	status = "okay";
+};
+
+&usb3_0 {
+	status = "okay";
+};
+
+&usb3_1 {
+	status = "okay";
+};
+
+&usb_dwc3_0 {
+	dr_mode = "otg";
+	hnp-disable;
+	srp-disable;
+	adp-disable;
+	usb-role-switch;
+	snps,dis-u1-entry-quirk;
+	snps,dis-u2-entry-quirk;
+	status = "okay";
+
+	port {
+		usb3_drd_sw: endpoint {
+			remote-endpoint = <&typec_dr_sw>;
+		};
+	};
+};
+
+&usb_dwc3_1 {
+	dr_mode = "host";
+	status = "okay";
+};
+
+&usb3_phy0 {
+	fsl,phy-tx-vref-tune-percent = <122>;
+	fsl,phy-tx-preemp-amp-tune-microamp = <1800>;
+	fsl,phy-tx-vboost-level-microvolt = <1156>;
+	fsl,phy-comp-dis-tune-percent = <115>;
+	fsl,phy-pcs-tx-deemph-3p5db-attenuation-db = <33>;
+	fsl,phy-pcs-tx-swing-full-percent = <100>;
+	status = "okay";
+};
+
+&usb3_phy1 {
+	fsl,phy-tx-preemp-amp-tune-microamp = <1800>;
+	fsl,phy-tx-vref-tune-percent = <116>;
+	status = "okay";
+};
+
+&usdhc2 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
+	cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>;
+	vmmc-supply = <&reg_usdhc2_vmmc>;
+	bus-width = <4>;
+	status = "okay";
+};
+
+&xcvr {
+	#sound-dai-cells = <0>;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_can: cangrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO06__GPIO1_IO06		0x1c6
+			MX8MP_IOMUXC_SPDIF_RX__GPIO5_IO04		0x16
+		>;
+	};
+
+	pinctrl_captouch: captouchgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO14__GPIO1_IO14		0x16
+		>;
+	};
+
+	pinctrl_ecspi1: ecspi1grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_ECSPI1_SCLK__ECSPI1_SCLK		0x12
+			MX8MP_IOMUXC_ECSPI1_MOSI__ECSPI1_MOSI		0x12
+			MX8MP_IOMUXC_ECSPI1_MISO__ECSPI1_MISO		0x12
+			MX8MP_IOMUXC_ECSPI1_SS0__GPIO5_IO09		0x12
+			MX8MP_IOMUXC_GPIO1_IO12__GPIO1_IO12		0x12
+		>;
+	};
+
+	pinctrl_extcon: extcongrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO10__GPIO1_IO10		0x10
+		>;
+	};
+
+	pinctrl_fec: fecgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SAI1_RXD4__ENET1_RGMII_RD0		0x90
+			MX8MP_IOMUXC_SAI1_RXD5__ENET1_RGMII_RD1		0x90
+			MX8MP_IOMUXC_SAI1_RXD6__ENET1_RGMII_RD2		0x90
+			MX8MP_IOMUXC_SAI1_RXD7__ENET1_RGMII_RD3		0x1d0
+			MX8MP_IOMUXC_SAI1_TXC__ENET1_RGMII_RXC		0x90
+			MX8MP_IOMUXC_SAI1_TXFS__ENET1_RGMII_RX_CTL	0x90
+			MX8MP_IOMUXC_SAI1_TXD0__ENET1_RGMII_TD0		0x00
+			MX8MP_IOMUXC_SAI1_TXD1__ENET1_RGMII_TD1		0x00
+			MX8MP_IOMUXC_SAI1_TXD2__ENET1_RGMII_TD2		0x00
+			MX8MP_IOMUXC_SAI1_TXD3__ENET1_RGMII_TD3		0x00
+			MX8MP_IOMUXC_SAI1_TXD4__ENET1_RGMII_TX_CTL	0x00
+			MX8MP_IOMUXC_SAI1_TXD5__ENET1_RGMII_TXC		0x00
+		>;
+	};
+
+	pinctrl_flexcan1: flexcan1grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SAI2_RXC__CAN1_TX			0x154
+			MX8MP_IOMUXC_SAI2_TXC__CAN1_RX			0x154
+		>;
+	};
+
+	pinctrl_flexcan2: flexcan2grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SAI2_MCLK__CAN2_RX			0x154
+			MX8MP_IOMUXC_SAI2_TXD0__CAN2_TX			0x154
+		>;
+	};
+
+	pinctrl_gpio_leds: ledgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SAI1_TXD6__GPIO4_IO18		0xc6
+		>;
+	};
+
+	pinctrl_hdmi: hdmigrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_HDMI_DDC_SCL__HDMIMIX_HDMI_SCL	0x1c2
+			MX8MP_IOMUXC_HDMI_DDC_SDA__HDMIMIX_HDMI_SDA	0x1c2
+			MX8MP_IOMUXC_HDMI_CEC__HDMIMIX_HDMI_CEC		0x10
+			MX8MP_IOMUXC_HDMI_HPD__HDMIMIX_HDMI_HPD		0x10
+		>;
+	};
+
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_I2C2_SCL__I2C2_SCL			0x400001c2
+			MX8MP_IOMUXC_I2C2_SDA__I2C2_SDA			0x400001c2
+		>;
+	};
+
+	pinctrl_i2c2_gpio: i2c2gpiogrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_I2C2_SCL__GPIO5_IO16		0x1c2
+			MX8MP_IOMUXC_I2C2_SDA__GPIO5_IO17		0x1c2
+		>;
+	};
+
+	pinctrl_i2c3: i2c3grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_I2C3_SCL__I2C3_SCL			0x400001c2
+			MX8MP_IOMUXC_I2C3_SDA__I2C3_SDA			0x400001c2
+		>;
+	};
+
+	pinctrl_i2c3_gpio: i2c3gpiogrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_I2C3_SCL__GPIO5_IO18		0x1c2
+			MX8MP_IOMUXC_I2C3_SDA__GPIO5_IO19		0x1c2
+		>;
+	};
+
+	pinctrl_i2c4: i2c4grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_I2C4_SCL__I2C4_SCL			0x400001c2
+			MX8MP_IOMUXC_I2C4_SDA__I2C4_SDA			0x400001c2
+		>;
+	};
+
+	pinctrl_i2c4_gpio: i2c4gpiogrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_I2C4_SCL__GPIO5_IO20		0x1c2
+			MX8MP_IOMUXC_I2C4_SDA__GPIO5_IO21		0x1c2
+		>;
+	};
+
+	pinctrl_pca6408: pca6408grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO05__GPIO1_IO05		0x1c6
+		>;
+	};
+
+	pinctrl_pwm1: pwm1grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO01__PWM1_OUT		0x116
+		>;
+	};
+
+	pinctrl_restouch: restouchgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO07__GPIO1_IO07		0xc0
+		>;
+	};
+
+	pinctrl_rtc: rtcgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO15__GPIO1_IO15		0x1c0
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_UART1_RXD__UART1_DCE_RX		0x40
+			MX8MP_IOMUXC_UART1_TXD__UART1_DCE_TX		0x40
+		>;
+	};
+
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_UART2_RXD__UART2_DCE_RX		0x40
+			MX8MP_IOMUXC_UART2_TXD__UART2_DCE_TX		0x40
+		>;
+	};
+
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_UART3_RXD__UART3_DCE_RX		0x40
+			MX8MP_IOMUXC_UART3_TXD__UART3_DCE_TX		0x40
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK		0x190
+			MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD		0x1d0
+			MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0		0x1d0
+			MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1		0x1d0
+			MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2		0x1d0
+			MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3		0x1d0
+			MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT		0xc0
+		>;
+	};
+
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK		0x194
+			MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD		0x1d4
+			MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0		0x1d4
+			MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1		0x1d4
+			MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2		0x1d4
+			MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3		0x1d4
+			MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT		0xc0
+		>;
+	};
+
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK		0x196
+			MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD		0x1d6
+			MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0		0x1d6
+			MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1		0x1d6
+			MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2		0x1d6
+			MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3		0x1d6
+			MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT		0xc0
+		>;
+	};
+
+	pinctrl_vmmc_usdhc2: regvmmc-usdhc2grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD2_RESET_B__GPIO2_IO19		0x40
+		>;
+	};
+
+	pinctrl_usdhc2_gpio: usdhc2-gpiogrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD2_CD_B__GPIO2_IO12		0x1c4
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-var-dart.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-var-dart.dtsi
new file mode 100644
index 0000000000000..889e41f2af14d
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-var-dart.dtsi
@@ -0,0 +1,475 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Common dtsi for Variscite DART-MX8M-PLUS
+ *
+ * Link: https://variscite.com/system-on-module-som/i-mx-8/i-mx-8m-plus/dart-mx8m-plus/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/dts-v1/;
+
+#include "imx8mp.dtsi"
+
+/ {
+	model = "Variscite DART-MX8M-PLUS Module";
+	compatible = "variscite,var-dart-mx8mp", "fsl,imx8mp";
+
+	reg_audio_supply: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-name = "wm8904-supply";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+
+	reg_phy_vddio: regulator-phy-vddio {
+		compatible = "regulator-fixed";
+		regulator-name = "vddio-1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	sound-wm8904 {
+		compatible = "simple-audio-card";
+		simple-audio-card,bitclock-master = <&codec_dai>;
+		simple-audio-card,format = "i2s";
+		simple-audio-card,frame-master = <&codec_dai>;
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,name = "wm8904-audio";
+		simple-audio-card,routing =
+			"Headphone Jack", "HPOUTL",
+			"Headphone Jack", "HPOUTR",
+			"IN2L", "Line In Jack",
+			"IN2R", "Line In Jack",
+			"IN1L", "Microphone Jack",
+			"IN1R", "Microphone Jack";
+		simple-audio-card,widgets =
+			"Microphone", "Microphone Jack",
+			"Headphone", "Headphone Jack",
+			"Line", "Line In Jack";
+
+		codec_dai: simple-audio-card,codec {
+			sound-dai = <&wm8904>;
+		};
+
+		simple-audio-card,cpu {
+			sound-dai = <&sai3>;
+		};
+	};
+
+	wifi_pwrseq: wifi-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		post-power-on-delay-ms = <100>;
+		power-off-delay-us = <10000>;
+		reset-gpios = <&gpio2 7 GPIO_ACTIVE_LOW>, /* WIFI_RESET */
+			      <&gpio2 8 GPIO_ACTIVE_LOW>; /* WIFI_PWR_EN */
+	};
+};
+
+&A53_0 {
+	cpu-supply = <&buck2>;
+};
+
+&A53_1 {
+	cpu-supply = <&buck2>;
+};
+
+&A53_2 {
+	cpu-supply = <&buck2>;
+};
+
+&A53_3 {
+	cpu-supply = <&buck2>;
+};
+
+&eqos {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_eqos>;
+	/*
+	 * The required RGMII TX and RX 2ns delays are implemented directly
+	 * in hardware via passive delay elements on the SOM PCB.
+	 * No delay configuration is needed in software via PHY driver.
+	 */
+	phy-mode = "rgmii";
+	phy-handle = <&ethphy0>;
+	status = "okay";
+
+	mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy0: ethernet-phy@0 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <0>;
+			reset-gpios = <&gpio2 11 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <10000>;
+			reset-deassert-us = <100000>;
+			vddio-supply = <&reg_phy_vddio>;
+		};
+	};
+};
+
+&i2c1 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c1>;
+	pinctrl-1 = <&pinctrl_i2c1_gpio>;
+	scl-gpios = <&gpio5 14 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 15 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	wm8904: audio-codec@1a {
+		compatible = "wlf,wm8904";
+		reg = <0x1a>;
+		#sound-dai-cells = <0>;
+		clocks = <&audio_blk_ctrl IMX8MP_CLK_AUDIOMIX_SAI3_MCLK1>;
+		clock-names = "mclk";
+		AVDD-supply = <&reg_audio_supply>;
+		CPVDD-supply = <&reg_audio_supply>;
+		DBVDD-supply = <&reg_audio_supply>;
+		DCVDD-supply = <&reg_audio_supply>;
+		MICVDD-supply = <&reg_audio_supply>;
+		wlf,drc-cfg-names = "default", "peaklimiter", "tradition",
+				    "soft", "music";
+		/*
+		 * Config registers per name, respectively:
+		 * KNEE_IP = 0,   KNEE_OP = 0,     HI_COMP = 1,   LO_COMP = 1
+		 * KNEE_IP = -24, KNEE_OP = -6,    HI_COMP = 1/4, LO_COMP = 1
+		 * KNEE_IP = -42, KNEE_OP = -3,    HI_COMP = 0,   LO_COMP = 1
+		 * KNEE_IP = -45, KNEE_OP = -9,    HI_COMP = 1/8, LO_COMP = 1
+		 * KNEE_IP = -30, KNEE_OP = -10.5, HI_COMP = 1/4, LO_COMP = 1
+		 */
+		wlf,drc-cfg-regs = /bits/ 16 <0x01af 0x3248 0x0000 0x0000>,
+				   /bits/ 16 <0x04af 0x324b 0x0010 0x0408>,
+				   /bits/ 16 <0x04af 0x324b 0x0028 0x0704>,
+				   /bits/ 16 <0x04af 0x324b 0x0018 0x078c>,
+				   /bits/ 16 <0x04af 0x324b 0x0010 0x050e>;
+		/* GPIO1 = DMIC_CLK, don't touch others */
+		wlf,gpio-cfg = <0x0018>, <0xffff>, <0xffff>, <0xffff>;
+	};
+
+	pca9450: pmic@25 {
+		compatible = "nxp,pca9450c";
+		reg = <0x25>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pmic>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <3 IRQ_TYPE_LEVEL_LOW>;
+
+		regulators {
+			buck1: BUCK1 {
+				regulator-name = "BUCK1";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <2187500>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <3125>;
+			};
+
+			buck2: BUCK2 {
+				regulator-name = "BUCK2";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <2187500>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <3125>;
+				nxp,dvs-run-voltage = <950000>;
+				nxp,dvs-standby-voltage = <850000>;
+			};
+
+			buck4: BUCK4 {
+				regulator-name = "BUCK4";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			buck5: BUCK5 {
+				regulator-name = "BUCK5";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			buck6: BUCK6 {
+				regulator-name = "BUCK6";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo1: LDO1 {
+				regulator-name = "LDO1";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo2: LDO2 {
+				regulator-name = "LDO2";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1150000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo3: LDO3 {
+				regulator-name = "LDO3";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo4: LDO4 {
+				regulator-name = "LDO4";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+			};
+
+			ldo5: LDO5 {
+				regulator-name = "LDO5";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+			};
+		};
+	};
+};
+
+&mu2 {
+	status = "okay";
+};
+
+&sai3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sai3>;
+	assigned-clocks = <&clk IMX8MP_CLK_SAI3>;
+	assigned-clock-parents = <&clk IMX8MP_AUDIO_PLL1_OUT>;
+	assigned-clock-rates = <11536000>;
+	clocks = <&audio_blk_ctrl IMX8MP_CLK_AUDIOMIX_SAI3_IPG>,
+		<&clk IMX8MP_CLK_DUMMY>,
+		<&audio_blk_ctrl IMX8MP_CLK_AUDIOMIX_SAI3_MCLK1>,
+		<&clk IMX8MP_CLK_DUMMY>,
+		<&clk IMX8MP_CLK_DUMMY>;
+	clock-names = "bus", "mclk0", "mclk1", "mclk2", "mclk3";
+	#sound-dai-cells = <0>;
+	fsl,sai-mclk-direction-output;
+	status = "okay";
+};
+
+/* BT module */
+&uart4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart4>, <&pinctrl_bt>;
+	assigned-clocks = <&clk IMX8MP_CLK_UART4>;
+	assigned-clock-parents = <&clk IMX8MP_SYS_PLL1_80M>;
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth_iw61x: bluetooth {
+		compatible = "nxp,88w8987-bt";
+	};
+};
+
+/* WIFI */
+&usdhc1 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc1>, <&pinctrl_wifi>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>, <&pinctrl_wifi>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>, <&pinctrl_wifi>;
+	mmc-pwrseq = <&wifi_pwrseq>;
+	bus-width = <4>;
+	non-removable;
+	keep-power-in-suspend;
+	status = "okay";
+};
+
+/* eMMC */
+&usdhc3 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	pinctrl-1 = <&pinctrl_usdhc3_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc3_200mhz>;
+	bus-width = <8>;
+	non-removable;
+	status = "okay";
+};
+
+&wdog1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_wdog>;
+	fsl,ext-reset-output;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_bt: btgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_DATA4__GPIO2_IO06				0xc0
+			MX8MP_IOMUXC_SD1_DATA7__GPIO2_IO09				0xc0
+		>;
+	};
+
+	pinctrl_eqos: eqosgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_ENET_MDC__ENET_QOS_MDC				0x2
+			MX8MP_IOMUXC_ENET_MDIO__ENET_QOS_MDIO				0x2
+			MX8MP_IOMUXC_ENET_RD0__ENET_QOS_RGMII_RD0			0x90
+			MX8MP_IOMUXC_ENET_RD1__ENET_QOS_RGMII_RD1			0x90
+			MX8MP_IOMUXC_ENET_RD2__ENET_QOS_RGMII_RD2			0x90
+			MX8MP_IOMUXC_ENET_RD3__ENET_QOS_RGMII_RD3			0x90
+			MX8MP_IOMUXC_ENET_RXC__CCM_ENET_QOS_CLOCK_GENERATE_RX_CLK	0x90
+			MX8MP_IOMUXC_ENET_RX_CTL__ENET_QOS_RGMII_RX_CTL			0x90
+			MX8MP_IOMUXC_ENET_TD0__ENET_QOS_RGMII_TD0			0x16
+			MX8MP_IOMUXC_ENET_TD1__ENET_QOS_RGMII_TD1			0x16
+			MX8MP_IOMUXC_ENET_TD2__ENET_QOS_RGMII_TD2			0x16
+			MX8MP_IOMUXC_ENET_TD3__ENET_QOS_RGMII_TD3			0x16
+			MX8MP_IOMUXC_ENET_TX_CTL__ENET_QOS_RGMII_TX_CTL			0x16
+			MX8MP_IOMUXC_ENET_TXC__CCM_ENET_QOS_CLOCK_GENERATE_TX_CLK	0x16
+			MX8MP_IOMUXC_SD1_STROBE__GPIO2_IO11				0x150
+		>;
+	};
+
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_I2C1_SCL__I2C1_SCL					0x400001c2
+			MX8MP_IOMUXC_I2C1_SDA__I2C1_SDA					0x400001c2
+		>;
+	};
+
+	pinctrl_i2c1_gpio: i2c1gpiogrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_I2C1_SCL__GPIO5_IO14				0x1c2
+			MX8MP_IOMUXC_I2C1_SDA__GPIO5_IO15				0x1c2
+		>;
+	};
+
+	pinctrl_pmic: pmicgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03				0x1c0
+		>;
+	};
+
+	pinctrl_sai3: sai3grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SAI3_TXFS__AUDIOMIX_SAI3_TX_SYNC			0xd6
+			MX8MP_IOMUXC_SAI3_TXC__AUDIOMIX_SAI3_TX_BCLK			0xd6
+			MX8MP_IOMUXC_SAI3_RXD__AUDIOMIX_SAI3_RX_DATA00			0xd6
+			MX8MP_IOMUXC_SAI3_TXD__AUDIOMIX_SAI3_TX_DATA00			0xd6
+			MX8MP_IOMUXC_SAI3_MCLK__AUDIOMIX_SAI3_MCLK			0xd6
+			MX8MP_IOMUXC_SAI3_RXFS__AUDIOMIX_SAI3_RX_SYNC			0xd6
+			MX8MP_IOMUXC_SAI3_RXC__AUDIOMIX_SAI3_RX_BCLK			0xd6
+		>;
+	};
+
+	pinctrl_uart4: uart4grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_ECSPI2_SCLK__UART4_DCE_RX				0x140
+			MX8MP_IOMUXC_ECSPI2_MOSI__UART4_DCE_TX				0x140
+			MX8MP_IOMUXC_ECSPI2_MISO__UART4_DCE_CTS				0x140
+			MX8MP_IOMUXC_ECSPI2_SS0__UART4_DCE_RTS				0x140
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK				0x190
+			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD				0x1d0
+			MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0				0x1d0
+			MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1				0x1d0
+			MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2				0x1d0
+			MX8MP_IOMUXC_SD1_DATA3__USDHC1_DATA3				0x1d0
+		>;
+	};
+
+	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK				0x194
+			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD				0x1d4
+			MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0				0x1d4
+			MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1				0x1d4
+			MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2				0x1d4
+			MX8MP_IOMUXC_SD1_DATA3__USDHC1_DATA3				0x1d4
+		>;
+	};
+
+	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK				0x196
+			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD				0x1d6
+			MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0				0x1d6
+			MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1				0x1d6
+			MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2				0x1d6
+			MX8MP_IOMUXC_SD1_DATA3__USDHC1_DATA3				0x1d6
+		>;
+	};
+
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK				0x190
+			MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD				0x1d0
+			MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0				0x1d0
+			MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1				0x1d0
+			MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2				0x1d0
+			MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3				0x1d0
+			MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4				0x1d0
+			MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5				0x1d0
+			MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6				0x1d0
+			MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7				0x1d0
+			MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE				0x190
+		>;
+	};
+
+	pinctrl_usdhc3_100mhz: usdhc3-100mhzgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK				0x194
+			MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD				0x1d4
+			MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0				0x1d4
+			MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1				0x1d4
+			MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2				0x1d4
+			MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3				0x1d4
+			MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4				0x1d4
+			MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5				0x1d4
+			MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6				0x1d4
+			MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7				0x1d4
+			MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE				0x194
+		>;
+	};
+
+	pinctrl_usdhc3_200mhz: usdhc3-200mhzgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK				0x196
+			MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD				0x1d6
+			MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0				0x1d6
+			MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1				0x1d6
+			MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2				0x1d6
+			MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3				0x1d6
+			MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4				0x1d6
+			MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5				0x1d6
+			MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6				0x1d6
+			MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7				0x1d6
+			MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE				0x196
+		>;
+	};
+
+	pinctrl_wdog: wdoggrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO02__WDOG1_WDOG_B				0xc6
+		>;
+	};
+
+	pinctrl_wifi: wifigrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_DATA5__GPIO2_IO07				0xc0
+			MX8MP_IOMUXC_SD1_DATA6__GPIO2_IO08				0xc0
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-var-som-symphony.dts b/arch/arm64/boot/dts/freescale/imx8mp-var-som-symphony.dts
index 291f65e36865b..e928a3d44b1d5 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-var-som-symphony.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-var-som-symphony.dts
@@ -13,6 +13,31 @@ chosen {
 		stdout-path = &uart2;
 	};
 
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		button-back {
+			label = "Back";
+			linux,code = <KEY_BACK>;
+			gpios = <&pca9534 1 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+
+		button-home {
+			label = "Home";
+			linux,code = <KEY_HOME>;
+			gpios = <&pca9534 2 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+
+		button-menu {
+			label = "Menu";
+			linux,code = <KEY_MENU>;
+			gpios = <&pca9534 3 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+	};
+
 	gpio-leds {
 		compatible = "gpio-leds";
 
@@ -23,6 +48,34 @@ led-0 {
 		};
 	};
 
+	native-hdmi-connector {
+		compatible = "hdmi-connector";
+		label = "HDMI OUT";
+		type = "a";
+
+		port {
+			hdmi_in: endpoint {
+				remote-endpoint = <&hdmi_tx_out>;
+			};
+		};
+	};
+
+	pcie0_refclk: pcie0-refclk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <100000000>;
+	};
+
+	reg_fec_phy: regulator-fec-phy {
+		compatible = "regulator-fixed";
+		regulator-name = "fec-phy";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-enable-ramp-delay = <20000>;
+		gpio = <&pca9534 7 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
 	reg_usdhc2_vmmc: regulator-usdhc2-vmmc {
 		compatible = "regulator-fixed";
 		regulator-name = "VSD_3V3";
@@ -43,12 +96,92 @@ reg_usdhc2_vqmmc: regulator-usdhc2-vqmmc {
 		states = <3300000 0x0 1800000 0x1>;
 		vin-supply = <&ldo5>;
 	};
+
+	sound-hdmi {
+		compatible = "fsl,imx-audio-hdmi";
+		model = "audio-hdmi";
+		audio-cpu = <&aud2htx>;
+		hdmi-out;
+	};
+};
+
+&aud2htx {
+	status = "okay";
+};
+
+&ecspi2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi2>;
+	cs-gpios = <&gpio5 13 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&eqos {
+	mdio {
+		ethphy1: ethernet-phy@5 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <5>;
+			reset-gpios = <&pca9534 5 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <10000>;
+			reset-deassert-us = <20000>;
+			vddio-supply = <&reg_phy_vddio>;
+		};
+	};
+};
+
+&fec {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_fec>;
+	/*
+	 * The required RGMII TX and RX 2ns delays are implemented directly
+	 * in hardware via passive delay elements on the SOM PCB.
+	 * No delay configuration is needed in software via PHY driver.
+	 */
+	phy-mode = "rgmii";
+	phy-handle = <&ethphy1>;
+	phy-supply = <&reg_fec_phy>;
+	status = "okay";
+};
+
+&flexcan2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan2>;
+	status = "okay";
+};
+
+&hdmi_pai {
+	status = "okay";
+};
+
+&hdmi_pvi {
+	status = "okay";
+};
+
+&hdmi_tx {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hdmi>;
+	status = "okay";
+
+	ports {
+		port@1 {
+			hdmi_tx_out: endpoint {
+				remote-endpoint = <&hdmi_in>;
+			};
+		};
+	};
+};
+
+&hdmi_tx_phy {
+	status = "okay";
 };
 
 &i2c3 {
 	clock-frequency = <400000>;
-	pinctrl-names = "default";
+	pinctrl-names = "default", "gpio";
 	pinctrl-0 = <&pinctrl_i2c3>;
+	pinctrl-1 = <&pinctrl_i2c3_gpio>;
+	scl-gpios = <&gpio5 18 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 19 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 	status = "okay";
 
 	/* GPIO expander */
@@ -70,6 +203,112 @@ usb3-sata-sel-hog {
 			line-name = "usb3_sata_sel";
 		};
 	};
+
+	/* USB Type-C Controller */
+	typec@3d {
+		compatible = "nxp,ptn5150";
+		reg = <0x3d>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_ptn5150>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <5 IRQ_TYPE_EDGE_FALLING>;
+
+		port {
+			typec_dr_sw: endpoint {
+				remote-endpoint = <&usb3_drd_sw>;
+			};
+		};
+	};
+};
+
+&i2c4 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default", "gpio", "sleep";
+	pinctrl-0 = <&pinctrl_i2c4>;
+	pinctrl-1 = <&pinctrl_i2c4_gpio>;
+	pinctrl-2 = <&pinctrl_i2c4_gpio>;
+	scl-gpios = <&gpio5 20 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio5 21 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	pcal6408: gpio@21 {
+		compatible = "nxp,pcal6408";
+		reg = <0x21>;
+		#gpio-cells = <2>;
+		gpio-controller;
+
+		/* RGB_SEL */
+		lvds-brg-enable-hog {
+			gpio-hog;
+			gpios = <7 GPIO_ACTIVE_HIGH>;
+			output-low;
+			line-name = "rgb_sel";
+		};
+	};
+
+	st33ktpm2xi2c: tpm@2e {
+		compatible = "st,st33ktpm2xi2c", "tcg,tpm-tis-i2c";
+		reg = <0x2e>;
+		label = "tpm";
+		reset-gpios = <&pcal6408 4 GPIO_ACTIVE_LOW>;
+	};
+
+	/* Capacitive touch controller */
+	ft5x06_ts: touchscreen@38 {
+		compatible = "edt,edt-ft5206";
+		reg = <0x38>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_captouch>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <3 IRQ_TYPE_EDGE_FALLING>;
+		touchscreen-size-x = <800>;
+		touchscreen-size-y = <480>;
+		touchscreen-inverted-x;
+		touchscreen-inverted-y;
+		wakeup-source;
+	};
+
+	rtc@68 {
+		compatible = "dallas,ds1337";
+		reg = <0x68>;
+	};
+};
+
+&lcdif3 {
+	status = "okay";
+};
+
+&pcie {
+	reset-gpios = <&pcal6408 1 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&pcie_phy {
+	clocks = <&pcie0_refclk>;
+	clock-names = "ref";
+	fsl,refclk-pad-mode = <IMX8_PCIE_REFCLK_PAD_INPUT>;
+	status = "okay";
+};
+
+&pwm1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm1>;
+	status = "okay";
+};
+
+&snvs_pwrkey {
+	status = "okay";
+};
+
+&snvs_rtc {
+	status = "disabled";
+};
+
+/* Header UART */
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	status = "okay";
 };
 
 /* Console */
@@ -79,6 +318,59 @@ &uart2 {
 	status = "okay";
 };
 
+/* Header UART */
+&uart4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart4>;
+	status = "okay";
+};
+
+&usb3_0 {
+	status = "okay";
+};
+
+&usb3_1 {
+	status = "okay";
+};
+
+&usb_dwc3_0 {
+	adp-disable;
+	dr_mode = "otg";
+	hnp-disable;
+	snps,dis-u1-entry-quirk;
+	snps,dis-u2-entry-quirk;
+	srp-disable;
+	usb-role-switch;
+	status = "okay";
+
+	port {
+		usb3_drd_sw: endpoint {
+			remote-endpoint = <&typec_dr_sw>;
+		};
+	};
+};
+
+&usb_dwc3_1 {
+	dr_mode = "host";
+	status = "okay";
+};
+
+&usb3_phy0 {
+	fsl,phy-comp-dis-tune-percent = <115>;
+	fsl,phy-pcs-tx-deemph-3p5db-attenuation-db = <33>;
+	fsl,phy-pcs-tx-swing-full-percent = <100>;
+	fsl,phy-tx-preemp-amp-tune-microamp = <1800>;
+	fsl,phy-tx-vboost-level-microvolt = <1156>;
+	fsl,phy-tx-vref-tune-percent = <122>;
+	status = "okay";
+};
+
+&usb3_phy1 {
+	fsl,phy-tx-preemp-amp-tune-microamp = <1800>;
+	fsl,phy-tx-vref-tune-percent = <116>;
+	status = "okay";
+};
+
 /* SD-card */
 &usdhc2 {
 	pinctrl-names = "default", "state_100mhz", "state_200mhz";
@@ -93,6 +385,54 @@ &usdhc2 {
 };
 
 &iomuxc {
+	pinctrl_captouch: captouchgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SPDIF_TX__GPIO5_IO03				0x16
+		>;
+	};
+
+	pinctrl_ecspi2: ecspi2grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_ECSPI2_SCLK__ECSPI2_SCLK				0x12
+			MX8MP_IOMUXC_ECSPI2_MOSI__ECSPI2_MOSI				0x12
+			MX8MP_IOMUXC_ECSPI2_MISO__ECSPI2_MISO				0x12
+			MX8MP_IOMUXC_ECSPI2_SS0__GPIO5_IO13				0x12
+		>;
+	};
+
+	pinctrl_fec: fecgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SAI1_RXD4__ENET1_RGMII_RD0				0x90
+			MX8MP_IOMUXC_SAI1_RXD5__ENET1_RGMII_RD1				0x90
+			MX8MP_IOMUXC_SAI1_RXD6__ENET1_RGMII_RD2				0x90
+			MX8MP_IOMUXC_SAI1_RXD7__ENET1_RGMII_RD3				0x90
+			MX8MP_IOMUXC_SAI1_TXC__ENET1_RGMII_RXC				0x90
+			MX8MP_IOMUXC_SAI1_TXFS__ENET1_RGMII_RX_CTL			0x90
+			MX8MP_IOMUXC_SAI1_TXD0__ENET1_RGMII_TD0				0x00
+			MX8MP_IOMUXC_SAI1_TXD1__ENET1_RGMII_TD1				0x00
+			MX8MP_IOMUXC_SAI1_TXD2__ENET1_RGMII_TD2				0x00
+			MX8MP_IOMUXC_SAI1_TXD3__ENET1_RGMII_TD3				0x00
+			MX8MP_IOMUXC_SAI1_TXD4__ENET1_RGMII_TX_CTL			0x00
+			MX8MP_IOMUXC_SAI1_TXD5__ENET1_RGMII_TXC				0x00
+		>;
+	};
+
+	pinctrl_flexcan2: flexcan2grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_UART3_RXD__CAN2_TX					0x154
+			MX8MP_IOMUXC_UART3_TXD__CAN2_RX					0x154
+		>;
+	};
+
+	pinctrl_hdmi: hdmigrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_HDMI_DDC_SCL__HDMIMIX_HDMI_SCL			0x1c2
+			MX8MP_IOMUXC_HDMI_DDC_SDA__HDMIMIX_HDMI_SDA			0x1c2
+			MX8MP_IOMUXC_HDMI_CEC__HDMIMIX_HDMI_CEC				0x10
+			MX8MP_IOMUXC_HDMI_HPD__HDMIMIX_HDMI_HPD				0x10
+		>;
+	};
+
 	pinctrl_i2c3: i2c3grp {
 		fsl,pins = <
 			MX8MP_IOMUXC_I2C3_SCL__I2C3_SCL                                 0x400001c2
@@ -100,12 +440,52 @@ MX8MP_IOMUXC_I2C3_SDA__I2C3_SDA                                 0x400001c2
 		>;
 	};
 
+	pinctrl_i2c3_gpio: i2c3gpiogrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_I2C3_SCL__GPIO5_IO18				0x400001c2
+			MX8MP_IOMUXC_I2C3_SDA__GPIO5_IO19				0x400001c2
+		>;
+	};
+
+	pinctrl_i2c4: i2c4grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_I2C4_SCL__I2C4_SCL					0x400001c2
+			MX8MP_IOMUXC_I2C4_SDA__I2C4_SDA					0x400001c2
+		>;
+	};
+
+	pinctrl_i2c4_gpio: i2c4gpiogrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_I2C4_SCL__GPIO5_IO20				0x400001c2
+			MX8MP_IOMUXC_I2C4_SDA__GPIO5_IO21				0x400001c2
+		>;
+	};
+
 	pinctrl_pca9534: pca9534grp {
 		fsl,pins = <
 			MX8MP_IOMUXC_GPIO1_IO15__GPIO1_IO15                             0xc0
 		>;
 	};
 
+	pinctrl_ptn5150: ptn5150grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO05__GPIO1_IO05				0x10
+		>;
+	};
+
+	pinctrl_pwm1: pwm1grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SPDIF_EXT_CLK__PWM1_OUT				0x116
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_UART1_RXD__UART1_DCE_RX				0x40
+			MX8MP_IOMUXC_UART1_TXD__UART1_DCE_TX				0x40
+		>;
+	};
+
 	pinctrl_uart2: uart2grp {
 		fsl,pins = <
 			MX8MP_IOMUXC_UART2_RXD__UART2_DCE_RX                            0x40
@@ -113,6 +493,13 @@ MX8MP_IOMUXC_UART2_TXD__UART2_DCE_TX                            0x40
 		>;
 	};
 
+	pinctrl_uart4: uart4grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_UART4_RXD__UART4_DCE_RX				0x40
+			MX8MP_IOMUXC_UART4_TXD__UART4_DCE_TX				0x40
+		>;
+	};
+
 	pinctrl_usdhc2_gpio: usdhc2-gpiogrp {
 		fsl,pins = <
 			MX8MP_IOMUXC_GPIO1_IO14__GPIO1_IO14                             0x1c4
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-var-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-var-som.dtsi
index 49467b48d0b0b..61786eee0e82c 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-var-som.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp-var-som.dtsi
@@ -180,8 +180,11 @@ led@1 {
 
 &i2c1 {
 	clock-frequency = <400000>;
-	pinctrl-names = "default";
+	pinctrl-names = "default", "gpio";
 	pinctrl-0 = <&pinctrl_i2c1>;
+	pinctrl-1 = <&pinctrl_i2c1_gpio>;
+	scl-gpios = <&gpio2 6 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio2 7 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 	status = "okay";
 
 	pmic@25 {
@@ -415,6 +418,13 @@ MX8MP_IOMUXC_SD1_DATA5__I2C1_SDA				0x400001c2
 		>;
 	};
 
+	pinctrl_i2c1_gpio: i2c1gpiogrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_DATA4__GPIO2_IO06				0x1c2
+			MX8MP_IOMUXC_SD1_DATA5__GPIO2_IO07				0x1c2
+		>;
+	};
+
 	pinctrl_pmic: pmicgrp {
 		fsl,pins = <
 			MX8MP_IOMUXC_SPDIF_RX__GPIO5_IO04				0x1c0
diff --git a/arch/arm64/boot/dts/freescale/imx8mp.dtsi b/arch/arm64/boot/dts/freescale/imx8mp.dtsi
index 5ce2825182fdb..9cf9f6f08654d 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp.dtsi
@@ -1599,6 +1599,7 @@ xcvr: xcvr@30cc0000 {
 					      <0x30cc0e00 0x080>;
 					reg-names = "ram", "regs", "rxfifo",
 						    "txfifo";
+					#sound-dai-cells = <0>;
 					interrupts = /* XCVR IRQ 0 */
 						     <GIC_SPI 128 IRQ_TYPE_LEVEL_HIGH>,
 						     /* XCVR IRQ 1 */
diff --git a/arch/arm64/boot/dts/freescale/imx8mq-evk.dts b/arch/arm64/boot/dts/freescale/imx8mq-evk.dts
index e7d87ea81b697..5ea04703ddd7e 100644
--- a/arch/arm64/boot/dts/freescale/imx8mq-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mq-evk.dts
@@ -6,6 +6,7 @@
 
 /dts-v1/;
 
+#include <dt-bindings/usb/pd.h>
 #include "imx8mq.dtsi"
 
 / {
@@ -330,6 +331,35 @@ vgen6_reg: vgen6 {
 			};
 		};
 	};
+
+	ptn5110: tcpc@50 {
+		compatible = "nxp,ptn5110", "tcpci";
+		reg = <0x50>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_typec>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <3 IRQ_TYPE_LEVEL_LOW>;
+		orientation-gpios = <&gpio3 15 GPIO_ACTIVE_LOW>;
+
+		usb_con: connector {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			power-role = "dual";
+			data-role = "dual";
+			try-power-role = "sink";
+			source-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
+			sink-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)
+				     PDO_VAR(5000, 20000, 3000)>;
+			op-sink-microwatt = <15000000>;
+			self-powered;
+
+			port {
+				typec_con_hs: endpoint {
+					remote-endpoint = <&usb3_data_hs>;
+				};
+			};
+		};
+	};
 };
 
 &lcdif {
@@ -403,6 +433,7 @@ &pcie1 {
 		 <&pcie0_refclk>,
 		 <&clk IMX8MQ_CLK_PCIE2_PHY>,
 		 <&clk IMX8MQ_CLK_PCIE2_AUX>;
+	/* This property is deprecated, use power supply from the Root Port node. */
 	vpcie-supply = <&reg_pcie1>;
 	vpcie3v3aux-supply = <&reg_pcie1>;
 	vph-supply = <&vgen5_reg>;
@@ -422,6 +453,8 @@ &pcie1_ep {
 
 &pcie1_port0 {
 	reset-gpios = <&gpio5 12 GPIO_ACTIVE_LOW>;
+	vpcie3v3-supply = <&reg_pcie1>;
+	vpcie3v3aux-supply = <&reg_pcie1>;
 };
 
 &pgc_gpu {
@@ -498,6 +531,41 @@ &uart1 {
 	status = "okay";
 };
 
+&uart3 { /* BT */
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3>;
+	assigned-clocks = <&clk IMX8MQ_CLK_UART3>;
+	assigned-clock-parents = <&clk IMX8MQ_SYS1_PLL_80M>;
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "nxp,88w8987-bt";
+	};
+};
+
+&usb3_phy0 {
+	status = "okay";
+};
+
+&usb_dwc3_0 {
+	dr_mode = "otg";
+	hnp-disable;
+	srp-disable;
+	adp-disable;
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	snps,dis-u1-entry-quirk;
+	snps,dis-u2-entry-quirk;
+	status = "okay";
+
+	port {
+		usb3_data_hs: endpoint {
+			remote-endpoint = <&typec_con_hs>;
+		};
+	};
+};
+
 &usb3_phy1 {
 	status = "okay";
 };
@@ -650,6 +718,13 @@ MX8MQ_IOMUXC_SPDIF_RX_SPDIF1_IN		0xd6
 		>;
 	};
 
+	pinctrl_typec: typecgrp {
+		fsl,pins = <
+			MX8MQ_IOMUXC_NAND_CE2_B_GPIO3_IO3		0x17059
+			MX8MQ_IOMUXC_NAND_RE_B_GPIO3_IO15		0x16
+		>;
+	};
+
 	pinctrl_uart1: uart1grp {
 		fsl,pins = <
 			MX8MQ_IOMUXC_UART1_RXD_UART1_DCE_RX		0x49
@@ -657,6 +732,15 @@ MX8MQ_IOMUXC_UART1_TXD_UART1_DCE_TX		0x49
 		>;
 	};
 
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <
+			MX8MQ_IOMUXC_UART3_TXD_UART3_DCE_TX		0x49
+			MX8MQ_IOMUXC_UART3_RXD_UART3_DCE_RX		0x49
+			MX8MQ_IOMUXC_ECSPI1_MISO_UART3_DCE_CTS_B	0x49
+			MX8MQ_IOMUXC_ECSPI1_SS0_UART3_DCE_RTS_B		0x49
+		>;
+	};
+
 	pinctrl_usdhc1: usdhc1grp {
 		fsl,pins = <
 			MX8MQ_IOMUXC_SD1_CLK_USDHC1_CLK			0x83
diff --git a/arch/arm64/boot/dts/freescale/imx8mq-librem5-devkit.dts b/arch/arm64/boot/dts/freescale/imx8mq-librem5-devkit.dts
index aadaeef928bd9..0c3f1b20b3d74 100644
--- a/arch/arm64/boot/dts/freescale/imx8mq-librem5-devkit.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mq-librem5-devkit.dts
@@ -215,7 +215,7 @@ vibrator {
 		compatible = "gpio-vibrator";
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_haptic>;
-	        enable-gpios = <&gpio5 4 GPIO_ACTIVE_LOW>;
+		enable-gpios = <&gpio5 4 GPIO_ACTIVE_LOW>;
 		vcc-supply = <&reg_3v3_p>;
 	};
 
@@ -856,7 +856,7 @@ MX8MQ_IOMUXC_GPIO1_IO02_WDOG1_WDOG_B	0xc6
 
 	pinctrl_wifi_pwr_en: wifipwrengrp {
 		fsl,pins = <
-			MX8MQ_IOMUXC_NAND_CLE_GPIO3_IO5         0x06
+			MX8MQ_IOMUXC_NAND_CLE_GPIO3_IO5		0x06
 		>;
 	};
 
diff --git a/arch/arm64/boot/dts/freescale/imx8mq.dtsi b/arch/arm64/boot/dts/freescale/imx8mq.dtsi
index e60872aeeb49e..d75710cdb5b41 100644
--- a/arch/arm64/boot/dts/freescale/imx8mq.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mq.dtsi
@@ -1598,6 +1598,29 @@ aips4: bus@32c00000 { /* AIPS4 */
 			#size-cells = <1>;
 			ranges = <0x32c00000 0x32c00000 0x400000>;
 
+			dcss: display-controller@32e00000 {
+				compatible = "nxp,imx8mq-dcss";
+				reg = <0x32e00000 0x2d000>, <0x32e2f000 0x1000>;
+				interrupts = <6>, <8>, <9>;
+				interrupt-names = "ctxld", "ctxld_kick", "vblank";
+				interrupt-parent = <&irqsteer>;
+				clocks = <&clk IMX8MQ_CLK_DISP_APB_ROOT>,
+					 <&clk IMX8MQ_CLK_DISP_AXI_ROOT>,
+					 <&clk IMX8MQ_CLK_DISP_RTRM_ROOT>,
+					 <&clk IMX8MQ_VIDEO2_PLL_OUT>,
+					 <&clk IMX8MQ_CLK_DISP_DTRC>;
+				clock-names = "apb", "axi", "rtrm", "pix", "dtrc";
+				assigned-clocks = <&clk IMX8MQ_CLK_DISP_AXI>,
+						  <&clk IMX8MQ_CLK_DISP_RTRM>,
+						  <&clk IMX8MQ_VIDEO2_PLL1_REF_SEL>;
+				assigned-clock-parents = <&clk IMX8MQ_SYS1_PLL_800M>,
+							 <&clk IMX8MQ_SYS1_PLL_800M>,
+							 <&clk IMX8MQ_CLK_27M>;
+				assigned-clock-rates = <800000000>,
+						       <400000000>;
+				status = "disabled";
+			};
+
 			irqsteer: interrupt-controller@32e2d000 {
 				compatible = "fsl,imx8m-irqsteer", "fsl,imx-irqsteer";
 				reg = <0x32e2d000 0x1000>;
diff --git a/arch/arm64/boot/dts/freescale/imx8qm-mek.dts b/arch/arm64/boot/dts/freescale/imx8qm-mek.dts
index f706c86137c0f..9e49ac377162b 100644
--- a/arch/arm64/boot/dts/freescale/imx8qm-mek.dts
+++ b/arch/arm64/boot/dts/freescale/imx8qm-mek.dts
@@ -812,6 +812,7 @@ &pciea {
 	pinctrl-names = "default";
 	/* This property is deprecated, use reset-gpios from the Root Port node. */
 	reset-gpio = <&lsio_gpio4 29 GPIO_ACTIVE_LOW>;
+	/* This property is deprecated, use power supply from the Root Port node. */
 	vpcie-supply = <&reg_pciea>;
 	vpcie3v3aux-supply = <&reg_pciea>;
 	supports-clkreq;
@@ -820,6 +821,8 @@ &pciea {
 
 &pciea_port0 {
 	reset-gpios = <&lsio_gpio4 29 GPIO_ACTIVE_LOW>;
+	vpcie3v3-supply = <&reg_pciea>;
+	vpcie3v3aux-supply = <&reg_pciea>;
 };
 
 &pcieb {
diff --git a/arch/arm64/boot/dts/freescale/imx8qm.dtsi b/arch/arm64/boot/dts/freescale/imx8qm.dtsi
index ae7de9f990553..9f9b34153ad54 100644
--- a/arch/arm64/boot/dts/freescale/imx8qm.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8qm.dtsi
@@ -7,6 +7,7 @@
 #include <dt-bindings/clock/imx8-lpcg.h>
 #include <dt-bindings/firmware/imx/rsrc.h>
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/pinctrl/pads-imx8qm.h>
 #include <dt-bindings/thermal/thermal.h>
@@ -330,6 +331,12 @@ clk: clock-controller {
 			#clock-cells = <2>;
 		};
 
+		scu_key: keys {
+			compatible = "fsl,imx8qm-sc-key", "fsl,imx-sc-key";
+			linux,keycodes = <KEY_POWER>;
+			wakeup-source;
+		};
+
 		iomuxc: pinctrl {
 			compatible = "fsl,imx8qm-iomuxc";
 		};
diff --git a/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts b/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts
index 2af32eca612aa..857a6e5122283 100644
--- a/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts
+++ b/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts
@@ -732,6 +732,7 @@ &pcie0 {
 	pinctrl-names = "default";
 	/* This property is deprecated, use reset-gpios from the Root Port node. */
 	reset-gpios = <&lsio_gpio4 0 GPIO_ACTIVE_LOW>;
+	/* This property is deprecated, use power supply from the Root Port node. */
 	vpcie-supply = <&reg_pcieb>;
 	vpcie3v3aux-supply = <&reg_pcieb>;
 	supports-clkreq;
@@ -748,6 +749,8 @@ &pcie0_ep {
 
 &pcieb_port0 {
 	reset-gpios = <&lsio_gpio4 0 GPIO_ACTIVE_LOW>;
+	vpcie3v3-supply = <&reg_pcieb>;
+	vpcie3v3aux-supply = <&reg_pcieb>;
 };
 
 &scu_key {
diff --git a/arch/arm64/boot/dts/freescale/imx91-9x9-qsb.dts b/arch/arm64/boot/dts/freescale/imx91-9x9-qsb.dts
index ea8cf14e0bc6a..044b37fcc0ba0 100644
--- a/arch/arm64/boot/dts/freescale/imx91-9x9-qsb.dts
+++ b/arch/arm64/boot/dts/freescale/imx91-9x9-qsb.dts
@@ -30,6 +30,79 @@ chosen {
 		stdout-path = &lpuart1;
 	};
 
+	wm8524: audio-codec {
+		compatible = "wlf,wm8524";
+		#sound-dai-cells = <0>;
+		wlf,mute-gpios = <&pcal6524 2 GPIO_ACTIVE_LOW>;
+
+		port {
+			codec_ep: endpoint {
+				remote-endpoint = <&sai1_ep1>;
+			};
+		};
+	};
+
+	dmic: dmic {
+		compatible = "dmic-codec";
+		#sound-dai-cells = <0>;
+		num-channels = <4>;
+
+		port {
+			dmic_ep: endpoint {
+				remote-endpoint = <&micfil_ep>;
+			};
+		};
+	};
+
+	can_mux: mux-controller-0 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		mux-gpios = <&pcal6524 17 GPIO_ACTIVE_HIGH>;
+	};
+
+	sai1_mux: mux-controller-1 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		mux-gpios = <&pcal6524 18 GPIO_ACTIVE_HIGH>;
+	};
+
+	sai1_en_mux: mux-controller-2 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		mux-gpios = <&pcal6524 9 GPIO_ACTIVE_HIGH>;
+	};
+
+	pinctrl-gpiomux {
+		compatible = "pinctrl-multiplexer";
+
+		can_fun: can-grp {
+			mux-states = <&can_mux 1>;
+		};
+
+		m2_fun: m2-grp {
+			mux-states = <&sai1_mux 1>;
+		};
+
+		pdm_fun: pdm-grp {
+			mux-states = <&can_mux 0>;
+		};
+
+		sai1_disable: sai1-disable-grp {
+			mux-states = <&sai1_en_mux 1>;
+		};
+
+		sai1_enable: sai1-enable-grp {
+			mux-states = <&sai1_en_mux 0>;
+		};
+
+		sai1_fun: sai1-grp {
+			mux-states = <&sai1_mux 0>;
+		};
+	};
+
 	reg_vref_1v8: regulator-adc-vref {
 		compatible = "regulator-fixed";
 		regulator-name = "vref_1v8";
@@ -61,6 +134,22 @@ linux,cma {
 			linux,cma-default;
 		};
 	};
+
+	sound-micfil {
+		compatible = "audio-graph-card2";
+		label = "micfil-audio";
+		links = <&micfil_port>;
+	};
+
+	sound-wm8524 {
+		compatible = "audio-graph-card2";
+		label = "wm8524-audio";
+		links = <&sai1_port1>;
+		widgets = "Line", "Line Out Jack";
+		routing =
+			"Line Out Jack", "LINEVOUTL",
+			"Line Out Jack", "LINEVOUTR";
+	};
 };
 
 &adc1 {
@@ -131,12 +220,29 @@ MX91_PAD_CCM_CLKO1__GPIO3_IO26                          0x31e
 		>;
 	};
 
+	pinctrl_pdm: pdmgrp {
+		fsl,pins = <
+			MX91_PAD_PDM_CLK__PDM_CLK				0x31e
+			MX91_PAD_PDM_BIT_STREAM0__PDM_BIT_STREAM0		0x31e
+			MX91_PAD_PDM_BIT_STREAM1__PDM_BIT_STREAM1		0x31e
+		>;
+	};
+
 	pinctrl_reg_usdhc2_vmmc: regusdhc2vmmcgrp {
 		fsl,pins = <
 			MX91_PAD_SD2_RESET_B__GPIO3_IO7                         0x31e
 		>;
 	};
 
+	pinctrl_sai1: sai1grp {
+		fsl,pins = <
+			MX91_PAD_SAI1_TXC__SAI1_TX_BCLK				0x31e
+			MX91_PAD_SAI1_TXFS__SAI1_TX_SYNC			0x31e
+			MX91_PAD_SAI1_TXD0__SAI1_TX_DATA0			0x31e
+			MX91_PAD_SAI1_RXD0__SAI1_MCLK				0x31e
+		>;
+	};
+
 	pinctrl_uart1: uart1grp {
 		fsl,pins = <
 			MX91_PAD_UART1_RXD__LPUART1_RX                          0x31e
@@ -384,6 +490,60 @@ &lpuart1 {
 	status = "okay";
 };
 
+&micfil {
+	#sound-dai-cells = <0>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pdm>, <&pdm_fun>;
+	assigned-clocks = <&clk IMX93_CLK_PDM>;
+	assigned-clock-parents = <&clk IMX93_CLK_AUDIO_PLL>;
+	assigned-clock-rates = <49152000>;
+	status = "okay";
+
+	micfil_port: port {
+		capture-only;
+
+		micfil_ep: endpoint {
+			remote-endpoint = <&dmic_ep>;
+		};
+	};
+};
+
+&sai1 {
+	#sound-dai-cells = <0>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sai1>, <&sai1_fun>, <&sai1_enable>;
+	assigned-clocks = <&clk IMX93_CLK_SAI1>;
+	assigned-clock-parents = <&clk IMX93_CLK_AUDIO_PLL>;
+	assigned-clock-rates = <24576000>;
+	clocks = <&clk IMX93_CLK_SAI1_IPG>, <&clk IMX93_CLK_DUMMY>,
+		 <&clk IMX93_CLK_SAI1_GATE>, <&clk IMX93_CLK_DUMMY>,
+		 <&clk IMX93_CLK_DUMMY>, <&clk IMX93_CLK_AUDIO_PLL>;
+	clock-names = "bus", "mclk0", "mclk1", "mclk2", "mclk3", "pll8k";
+	fsl,sai-mclk-direction-output;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		sai1_port1: port@1 {
+			reg = <1>;
+			playback-only;
+
+			sai1_ep1: endpoint {
+				dai-format = "i2s";
+				dai-tdm-slot-num = <2>;
+				dai-tdm-slot-width = <32>;
+				bitclock-master;
+				frame-master;
+				mclk-fs = <256>;
+				system-clock-direction-out;
+				remote-endpoint = <&codec_ep>;
+			};
+		};
+	};
+};
+
 &usbotg1 {
 	adp-disable;
 	disable-over-current;
diff --git a/arch/arm64/boot/dts/freescale/imx91-tqma9131-mba91xxca.dts b/arch/arm64/boot/dts/freescale/imx91-tqma9131-mba91xxca.dts
index 5c430e6fca651..60d11a4573e3f 100644
--- a/arch/arm64/boot/dts/freescale/imx91-tqma9131-mba91xxca.dts
+++ b/arch/arm64/boot/dts/freescale/imx91-tqma9131-mba91xxca.dts
@@ -518,6 +518,8 @@ &usbotg1 {
 	adp-disable;
 	usb-role-switch;
 	disable-over-current;
+	samsung,picophy-pre-emp-curr-control = <3>;
+	samsung,picophy-dc-vol-level-adjust = <7>;
 	status = "okay";
 
 	port {
@@ -532,6 +534,8 @@ &usbotg2 {
 	#address-cells = <1>;
 	#size-cells = <0>;
 	disable-over-current;
+	samsung,picophy-pre-emp-curr-control = <3>;
+	samsung,picophy-dc-vol-level-adjust = <7>;
 	status = "okay";
 
 	hub_2_0: hub@1 {
diff --git a/arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi b/arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi
index c99d7bc168483..a8c79731eace5 100644
--- a/arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx91-tqma9131.dtsi
@@ -33,9 +33,9 @@ linux,cma {
 		};
 
 		/* EdgeLock secure enclave  */
-		ele_reserved: ele-reserved@a4120000 {
+		ele_reserved: ele-reserved@94120000 {
 			compatible = "shared-dma-pool";
-			reg = <0 0xa4120000 0 0x100000>;
+			reg = <0 0x94120000 0 0x100000>;
 			no-map;
 		};
 	};
diff --git a/arch/arm64/boot/dts/freescale/imx91-var-dart-sonata.dts b/arch/arm64/boot/dts/freescale/imx91-var-dart-sonata.dts
index 3f0fd321d95f0..f7e32f9375f50 100644
--- a/arch/arm64/boot/dts/freescale/imx91-var-dart-sonata.dts
+++ b/arch/arm64/boot/dts/freescale/imx91-var-dart-sonata.dts
@@ -97,6 +97,17 @@ reg_rgb_sel: regulator-rgb-enable {
 		enable-active-high;
 	};
 
+	reg_phy1_supply: regulator-phy1 {
+		compatible = "regulator-fixed";
+		regulator-name = "SUPPLY_PHY1";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&pca6408_2 0 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		startup-delay-us = <10000>;
+		regulator-always-on;
+	};
+
 	reg_usdhc2_vmmc: regulator-vmmc-usdhc2 {
 		compatible = "regulator-fixed";
 		pinctrl-names = "default";
@@ -139,9 +150,6 @@ mdio {
 		ethphy1: ethernet-phy@1 {
 			compatible = "ethernet-phy-ieee802.3-c22";
 			reg = <1>;
-			reset-gpios = <&pca6408_2 0 GPIO_ACTIVE_LOW>;
-			reset-assert-us = <15000>;
-			reset-deassert-us = <100000>;
 
 			leds {
 				#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/freescale/imx93-11x11-evk-dy1212w-4856.dtso b/arch/arm64/boot/dts/freescale/imx93-11x11-evk-dy1212w-4856.dtso
new file mode 100644
index 0000000000000..35f7c5699e3af
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-11x11-evk-dy1212w-4856.dtso
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 NXP
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/clock/imx93-clock.h>
+
+&{/} {
+	panel-lvds {
+		compatible = "boe,ev121wxm-n10-1850";
+		backlight = <&backlight_lvds>;
+		power-supply = <&buck4>;
+
+		panel-timing {
+			/*
+			 * Set clock frequency to 71142858Hz to accommodate
+			 * IMX93_CLK_VIDEO_PLL rate at 498000000Hz in a rate
+			 * table.
+			 */
+			clock-frequency = <71142858>;
+			hactive = <1280>;
+			vactive = <800>;
+			hfront-porch = <48>;
+			hback-porch = <80>;
+			hsync-len = <32>;
+			vfront-porch = <3>;
+			vback-porch = <14>;
+			vsync-len = <6>;
+		};
+
+		port {
+			panel_lvds_in: endpoint {
+				remote-endpoint = <&ldb_lvds_ch0>;
+			};
+		};
+	};
+};
+
+&backlight_lvds {
+	status = "okay";
+};
+
+&lcdif {
+	status = "okay";
+};
+
+&lvds_bridge {
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@1 {
+			reg = <1>;
+
+			ldb_lvds_ch0: endpoint {
+				remote-endpoint = <&panel_lvds_in>;
+			};
+		};
+	};
+};
+
+&media_blk_ctrl {
+	assigned-clocks = <&clk IMX93_CLK_MEDIA_AXI>,
+			  <&clk IMX93_CLK_MEDIA_APB>,
+			  <&clk IMX93_CLK_MEDIA_DISP_PIX>,
+			  <&clk IMX93_CLK_VIDEO_PLL>;
+	assigned-clock-parents = <&clk IMX93_CLK_SYS_PLL_PFD1>,
+				 <&clk IMX93_CLK_SYS_PLL_PFD1_DIV2>,
+				 <&clk IMX93_CLK_VIDEO_PLL>;
+	/*
+	 * Set IMX93_CLK_MEDIA_DISP_PIX rate to 71142858Hz to accommodate
+	 * IMX93_CLK_VIDEO_PLL rate at 498000000Hz in a rate table.
+	 */
+	assigned-clock-rates = <400000000>, <133333333>, <71142858>, <498000000>;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi b/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi
index c79b1df339db1..f881912cde460 100644
--- a/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi
@@ -15,9 +15,9 @@ aliases {
 		rtc1 = &bbnsm_rtc;
 	};
 
-	memory@40000000 {
+	memory@80000000 {
 		device_type = "memory";
-		reg = <0x0 0x40000000 0 0x80000000>;
+		reg = <0x0 0x80000000 0 0x40000000>;
 	};
 
 	chosen {
diff --git a/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxca.dts b/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxca.dts
index 9108181e6592b..39a8742de1da4 100644
--- a/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxca.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxca.dts
@@ -67,6 +67,23 @@ backlight_lvds: backlight {
 		status = "disabled";
 	};
 
+	display: display {
+		/*
+		 * Display is not fixed, so compatible has to be added from
+		 * DT overlay
+		 */
+		power-supply = <&reg_3v3>;
+		enable-gpios = <&expander2 1 GPIO_ACTIVE_HIGH>;
+		backlight = <&backlight_lvds>;
+		status = "disabled";
+
+		port {
+			panel_in_lvds0: endpoint {
+				remote-endpoint = <&ldb_lvds_ch0>;
+			};
+		};
+	};
+
 	fan0: pwm-fan {
 		compatible = "pwm-fan";
 		pinctrl-names = "default";
@@ -571,6 +588,10 @@ &lpuart8 {
 	status = "okay";
 };
 
+&ldb_lvds_ch0 {
+	remote-endpoint = <&panel_in_lvds0>;
+};
+
 &pcf85063 {
 	/* RTC_EVENT# from SoM is connected on mainboard */
 	pinctrl-names = "default";
diff --git a/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxla-lvds-tm070jvhg33.dtso b/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxla-lvds-tm070jvhg33.dtso
new file mode 100644
index 0000000000000..4cb2d08b671ed
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxla-lvds-tm070jvhg33.dtso
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR MIT)
+/*
+ * Copyright (c) 2023-2026 TQ-Systems GmbH <linux@ew.tq-group.com>,
+ * D-82229 Seefeld, Germany.
+ * Author: Alexander Stein
+ */
+
+#include <dt-bindings/clock/imx93-clock.h>
+
+/dts-v1/;
+/plugin/;
+
+&backlight_lvds {
+	status = "okay";
+};
+
+&display {
+	compatible = "tianma,tm070jvhg33";
+	status = "okay";
+};
+
+&lcdif {
+	assigned-clocks = <&clk IMX93_CLK_VIDEO_PLL>;
+	assigned-clock-rates = <477400000>;
+	status = "okay";
+};
+
+&lvds_bridge {
+	status = "okay";
+};
+
+&media_blk_ctrl {
+	status = "okay";
+};
+
+&tpm5 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxla-mini.dts b/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxla-mini.dts
index 4afd6b650d9d8..0d553a8da7135 100644
--- a/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxla-mini.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxla-mini.dts
@@ -58,6 +58,37 @@ aliases {
 		spi5 = &lpspi6;
 	};
 
+	backlight_lvds: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&tpm5 0 5000000 0>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_backlight>;
+		brightness-levels = <0 4 8 16 32 64 128 255>;
+		default-brightness-level = <7>;
+		power-supply = <&reg_12v0>;
+		enable-gpios = <&gpio2 5 GPIO_ACTIVE_HIGH>;
+		status = "disabled";
+	};
+
+	display: display {
+		/*
+		 * Display is not fixed, so compatible has to be added from
+		 * DT overlay
+		 */
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_display>;
+		power-supply = <&reg_3v3>;
+		enable-gpios = <&gpio2 7 GPIO_ACTIVE_HIGH>;
+		backlight = <&backlight_lvds>;
+		status = "disabled";
+
+		port {
+			panel_in_lvds0: endpoint {
+				remote-endpoint = <&ldb_lvds_ch0>;
+			};
+		};
+	};
+
 	iio-hwmon {
 		compatible = "iio-hwmon";
 		io-channels = <&adc1 0>, <&adc1 1>, <&adc1 2>, <&adc1 3>;
@@ -311,6 +342,10 @@ &lpuart8 {
 	status = "okay";
 };
 
+&ldb_lvds_ch0 {
+	remote-endpoint = <&panel_in_lvds0>;
+};
+
 &pcf85063 {
 	/* RTC_EVENT# from SoM is connected on mainboard */
 	pinctrl-names = "default";
@@ -375,6 +410,16 @@ &usdhc3 {
 };
 
 &iomuxc {
+	pinctrl_backlight: backlightgrp {
+		fsl,pins = /* HYS | PD | FSEL_2 | DSE X4 */
+			   <MX93_PAD_GPIO_IO05__GPIO2_IO05			0x011e>;
+	};
+
+	pinctrl_display: displaygrp {
+		fsl,pins = /* HYS | PD | FSEL_2 | DSE X4 */
+			   <MX93_PAD_GPIO_IO07__GPIO2_IO07			0x011e>;
+	};
+
 	pinctrl_eqos: eqosgrp {
 		fsl,pins = /* PD | FSEL_2 | DSE X4 */
 			   <MX93_PAD_ENET1_MDC__ENET_QOS_MDC				0x51e>,
diff --git a/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxla.dts b/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxla.dts
index a78bbc46c59b2..914efa9b1c22a 100644
--- a/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxla.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-tqma9352-mba93xxla.dts
@@ -73,6 +73,23 @@ clk_dp: clk-dp {
 		clock-frequency = <26000000>;
 	};
 
+	display: display {
+		/*
+		 * Display is not fixed, so compatible has to be added from
+		 * DT overlay
+		 */
+		power-supply = <&reg_3v3>;
+		enable-gpios = <&expander2 1 GPIO_ACTIVE_HIGH>;
+		backlight = <&backlight_lvds>;
+		status = "disabled";
+
+		port {
+			panel_in_lvds0: endpoint {
+				remote-endpoint = <&ldb_lvds_ch0>;
+			};
+		};
+	};
+
 	gpio-keys {
 		compatible = "gpio-keys";
 		autorepeat;
@@ -534,6 +551,10 @@ &lpuart8 {
 	status = "okay";
 };
 
+&ldb_lvds_ch0 {
+	remote-endpoint = <&panel_in_lvds0>;
+};
+
 &pcf85063 {
 	/* RTC_EVENT# from SoM is connected on mainboard */
 	pinctrl-names = "default";
diff --git a/arch/arm64/boot/dts/freescale/imx93-var-dart-sonata.dts b/arch/arm64/boot/dts/freescale/imx93-var-dart-sonata.dts
index 5513d3b148a2f..1ca3ec8330d07 100644
--- a/arch/arm64/boot/dts/freescale/imx93-var-dart-sonata.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-var-dart-sonata.dts
@@ -103,6 +103,17 @@ reg_vref_1v8: regulator-adc-vref {
 		regulator-min-microvolt = <1800000>;
 	};
 
+	reg_phy1_supply: regulator-phy1 {
+		compatible = "regulator-fixed";
+		regulator-name = "SUPPLY_PHY1";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&pca6408_2 0 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		startup-delay-us = <10000>;
+		regulator-always-on;
+	};
+
 	reg_usdhc2_vmmc: regulator-vmmc-usdhc2 {
 		compatible = "regulator-fixed";
 		regulator-name = "VDD_SD2_3V3";
@@ -180,9 +191,6 @@ mdio {
 		ethphy1: ethernet-phy@1 {
 			compatible = "ethernet-phy-ieee802.3-c22";
 			reg = <1>;
-			reset-assert-us = <15000>;
-			reset-deassert-us = <100000>;
-			reset-gpios = <&pca6408_2 0 GPIO_ACTIVE_LOW>;
 
 			leds {
 				#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/freescale/imx93.dtsi b/arch/arm64/boot/dts/freescale/imx93.dtsi
index b9abe143cb567..e926b6360a9d1 100644
--- a/arch/arm64/boot/dts/freescale/imx93.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx93.dtsi
@@ -80,7 +80,7 @@ mu1: mailbox@44230000 {
 	};
 
 	tmu: tmu@44482000 {
-		compatible = "fsl,qoriq-tmu";
+		compatible = "fsl,imx93-tmu";
 		reg = <0x44482000 0x1000>;
 		interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clk IMX93_CLK_TMC_GATE>;
@@ -178,6 +178,7 @@ &lcdif {
 	port {
 		lcdif_to_ldb: endpoint@1 {
 			reg = <1>;
+			remote-endpoint = <&ldb_from_lcdif>;
 		};
 
 		lcdif_to_dsi: endpoint@2 {
@@ -186,6 +187,42 @@ lcdif_to_dsi: endpoint@2 {
 	};
 };
 
+&media_blk_ctrl {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	lvds_bridge: bridge@20 {
+		compatible = "fsl,imx93-ldb";
+		reg = <0x20 0x4>, <0x24 0x4>;
+		reg-names = "ldb", "lvds";
+		clocks = <&clk IMX93_CLK_LVDS_GATE>;
+		clock-names = "ldb";
+		assigned-clocks = <&clk IMX93_CLK_MEDIA_LDB>;
+		assigned-clock-parents = <&clk IMX93_CLK_VIDEO_PLL>;
+		status = "disabled";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				ldb_from_lcdif: endpoint {
+					remote-endpoint = <&lcdif_to_ldb>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				ldb_lvds_ch0: endpoint {
+				};
+			};
+		};
+	};
+};
+
 &src {
 	mlmix: power-domain@44461800 {
 		compatible = "fsl,imx93-src-slice";
diff --git a/arch/arm64/boot/dts/freescale/imx94.dtsi b/arch/arm64/boot/dts/freescale/imx94.dtsi
index a6cb5a6e848b3..dfbb73603cb24 100644
--- a/arch/arm64/boot/dts/freescale/imx94.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx94.dtsi
@@ -1374,8 +1374,9 @@ pcie0: pcie@4c300000 {
 			      <0 0x4c360000 0 0x10000>,
 			      <0 0x4c340000 0 0x4000>;
 			reg-names = "dbi", "config", "atu", "app";
-			ranges = <0x81000000 0x0 0x00000000 0x0 0x6ff00000 0 0x00100000>,
-				 <0x82000000 0x0 0x10000000 0x9 0x10000000 0 0x80000000>;
+			ranges = <0x43000000 0x9 0x00000000 0x9 0x00000000 0x0 0xe0000000>,
+				 <0x82000000 0x0 0xe0000000 0x9 0xe0000000 0x0 0x10000000>,
+				 <0x81000000 0x0 0x00000000 0x9 0xf0000000 0x0 0x00100000>;
 			#address-cells = <3>;
 			#size-cells = <2>;
 			device_type = "pci";
@@ -1410,6 +1411,17 @@ pcie0: pcie@4c300000 {
 			power-domains = <&scmi_devpd IMX94_PD_HSIO_TOP>;
 			fsl,max-link-speed = <3>;
 			status = "disabled";
+
+			pcie0_port0: pcie@0 {
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
 		};
 
 		pcie0_ep: pcie-ep@4c300000 {
diff --git a/arch/arm64/boot/dts/freescale/imx943-evk.dts b/arch/arm64/boot/dts/freescale/imx943-evk.dts
index 7cfd424689507..674410e541cba 100644
--- a/arch/arm64/boot/dts/freescale/imx943-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx943-evk.dts
@@ -1034,12 +1034,15 @@ &pcie0 {
 		 <&pcie_ref_clk>;
 	clock-names = "pcie", "pcie_bus", "pcie_phy", "pcie_aux",
 		      "ref", "extref";
-	reset-gpio = <&pcal6416_i2c3_u46 3 GPIO_ACTIVE_LOW>;
-	vpcie3v3aux-supply = <&reg_m2_wlan>;
 	supports-clkreq;
 	status = "okay";
 };
 
+&pcie0_port0 {
+	reset-gpios = <&pcal6416_i2c3_u46 3 GPIO_ACTIVE_LOW>;
+	vpcie3v3aux-supply = <&reg_m2_wlan>;
+};
+
 &pcie0_ep {
 	pinctrl-0 = <&pinctrl_pcie0>;
 	pinctrl-names = "default";
@@ -1058,12 +1061,15 @@ &pcie1 {
 		 <&pcie_ref_clk>;
 	clock-names = "pcie", "pcie_bus", "pcie_phy", "pcie_aux",
 		      "ref", "extref";
-	reset-gpio = <&pcal6416_i2c3_u46 1 GPIO_ACTIVE_LOW>;
-	vpcie3v3aux-supply = <&reg_slot_pwr>;
 	supports-clkreq;
 	status = "okay";
 };
 
+&pcie1_port0 {
+	reset-gpios = <&pcal6416_i2c3_u46 1 GPIO_ACTIVE_LOW>;
+	vpcie3v3aux-supply = <&reg_slot_pwr>;
+};
+
 &pcie1_ep {
 	pinctrl-0 = <&pinctrl_pcie1>;
 	pinctrl-names = "default";
diff --git a/arch/arm64/boot/dts/freescale/imx943.dtsi b/arch/arm64/boot/dts/freescale/imx943.dtsi
index ed030d4bc7bd9..01152fd0efa5e 100644
--- a/arch/arm64/boot/dts/freescale/imx943.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx943.dtsi
@@ -218,8 +218,9 @@ pcie1: pcie@4c380000 {
 			      <0 0x4c3e0000 0 0x10000>,
 			      <0 0x4c3c0000 0 0x4000>;
 			reg-names = "dbi", "config", "atu", "app";
-			ranges = <0x81000000 0 0x00000000 0x8 0x8ff00000 0 0x00100000>,
-				 <0x82000000 0 0x10000000 0xa 0x10000000 0 0x80000000>;
+			ranges = <0x43000000 0xa 0x00000000 0xa 0x00000000 0x0 0xe0000000>,
+				 <0x82000000 0x0 0xe0000000 0xa 0xe0000000 0x0 0x10000000>,
+				 <0x81000000 0x0 0x00000000 0xa 0xf0000000 0x0 0x00100000>;
 			#address-cells = <3>;
 			#size-cells = <2>;
 			device_type = "pci";
@@ -254,6 +255,17 @@ pcie1: pcie@4c380000 {
 			power-domains = <&scmi_devpd IMX94_PD_HSIO_TOP>;
 			fsl,max-link-speed = <3>;
 			status = "disabled";
+
+			pcie1_port0: pcie@0 {
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
 		};
 
 		pcie1_ep: pcie-ep@4c380000 {
diff --git a/arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts b/arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts
index 7d820a0f80b22..6aedcbbe915a5 100644
--- a/arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts
@@ -555,8 +555,6 @@ &pcie0 {
 	pinctrl-names = "default";
 	/* This property is deprecated, use reset-gpios from the Root Port node. */
 	reset-gpio = <&gpio5 13 GPIO_ACTIVE_LOW>;
-	vpcie-supply = <&reg_m2_pwr>;
-	vpcie3v3aux-supply = <&reg_m2_pwr>;
 	supports-clkreq;
 	status = "disabled";
 };
@@ -570,6 +568,8 @@ &pcie0_ep {
 
 &pcie0_port0 {
 	reset-gpios = <&gpio5 13 GPIO_ACTIVE_LOW>;
+	vpcie3v3-supply = <&reg_m2_pwr>;
+	vpcie3v3aux-supply = <&reg_m2_pwr>;
 };
 
 &sai1 {
diff --git a/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts b/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts
index 2e463bc7c6012..340ab0253ec21 100644
--- a/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts
@@ -542,8 +542,6 @@ &pcie0 {
 	pinctrl-names = "default";
 	/* This property is deprecated, use reset-gpios from the Root Port node. */
 	reset-gpio = <&i2c7_pcal6524 5 GPIO_ACTIVE_LOW>;
-	vpcie-supply = <&reg_pcie0>;
-	vpcie3v3aux-supply = <&reg_pcie0>;
 	supports-clkreq;
 	status = "okay";
 };
@@ -557,6 +555,8 @@ &pcie0_ep {
 
 &pcie0_port0 {
 	reset-gpios = <&i2c7_pcal6524 5 GPIO_ACTIVE_LOW>;
+	vpcie3v3-supply = <&reg_pcie0>;
+	vpcie3v3aux-supply = <&reg_pcie0>;
 };
 
 &pcie1 {
@@ -564,8 +564,6 @@ &pcie1 {
 	pinctrl-names = "default";
 	/* This property is deprecated, use reset-gpios from the Root Port node. */
 	reset-gpio = <&i2c7_pcal6524 16 GPIO_ACTIVE_LOW>;
-	vpcie-supply = <&reg_slot_pwr>;
-	vpcie3v3aux-supply = <&reg_slot_pwr>;
 	status = "okay";
 };
 
@@ -578,6 +576,8 @@ &pcie1_ep {
 
 &pcie1_port0 {
 	reset-gpios = <&i2c7_pcal6524 16 GPIO_ACTIVE_LOW>;
+	vpcie3v3-supply = <&reg_slot_pwr>;
+	vpcie3v3aux-supply = <&reg_slot_pwr>;
 };
 
 &sai1 {
diff --git a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi
index 7d760470201fa..a6c5398a81e3c 100644
--- a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi
@@ -24,6 +24,7 @@ aliases {
 		serial1 = &lpuart1;
 		serial2 = &lpuart6;
 		serial3 = &lpuart3;
+		serial4 = &lpuart5;
 	};
 
 	chosen {
diff --git a/arch/arm64/boot/dts/freescale/imx95-var-dart-sonata.dts b/arch/arm64/boot/dts/freescale/imx95-var-dart-sonata.dts
index 7a58ad38ffb46..ccf7e77016e48 100644
--- a/arch/arm64/boot/dts/freescale/imx95-var-dart-sonata.dts
+++ b/arch/arm64/boot/dts/freescale/imx95-var-dart-sonata.dts
@@ -346,9 +346,6 @@ &netc_emdio {
 	ethphy1: ethernet-phy@1 {
 		compatible = "ethernet-phy-ieee802.3-c22";
 		reg = <1>;
-		reset-gpios = <&pca6408_2 0 GPIO_ACTIVE_LOW>;
-		reset-assert-us = <10000>;
-		reset-deassert-us = <100000>;
 
 		leds {
 			#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/freescale/imx95-var-dart.dtsi b/arch/arm64/boot/dts/freescale/imx95-var-dart.dtsi
index a20fadacaa6d8..b4c55bd338583 100644
--- a/arch/arm64/boot/dts/freescale/imx95-var-dart.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx95-var-dart.dtsi
@@ -158,6 +158,8 @@ wm8904: audio-codec@1a {
 				   /bits/ 16 <0x04af 0x324b 0x0010 0x050e>;
 		/* GPIO1 = DMIC_CLK, don't touch others */
 		wlf,gpio-cfg = <0x0018>, <0xffff>, <0xffff>, <0xffff>;
+		/* DMIC is connected to IN1L */
+		wlf,in1l-as-dmicdat1;
 	};
 };
 
@@ -176,6 +178,10 @@ &mu7 {
 	status = "okay";
 };
 
+&netc_blk_ctrl {
+	status = "okay";
+};
+
 &netc_emdio {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_emdio>, <&pinctrl_phy0res>;
@@ -213,6 +219,10 @@ &netc_timer {
 	status = "okay";
 };
 
+&netcmix_blk_ctrl {
+	status = "okay";
+};
+
 &sai3 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_sai3>;
diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
index d6c549c160476..305751046d66e 100644
--- a/arch/arm64/boot/dts/freescale/imx95.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
@@ -1867,45 +1867,45 @@ pmu@49252000 {
 			};
 		};
 
-		usb3: usb@4c010010 {
-			compatible = "fsl,imx95-dwc3", "fsl,imx8mp-dwc3";
-			reg = <0x0 0x4c010010 0x0 0x04>,
-			      <0x0 0x4c1f0000 0x0 0x20>;
-			clocks = <&scmi_clk IMX95_CLK_HSIO>,
-				 <&scmi_clk IMX95_CLK_32K>;
-			clock-names = "hsio", "suspend";
-			interrupts = <GIC_SPI 173 IRQ_TYPE_LEVEL_HIGH>;
+		hsio_blk_ctl: syscon@4c0100c0 {
+			compatible = "nxp,imx95-hsio-blk-ctl", "syscon";
+			reg = <0x0 0x4c0100c0 0x0 0x1>;
+			#clock-cells = <1>;
+			clocks = <&clk_sys100m>;
+			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
+		};
+
+		bus@4c100000 {
+			compatible = "simple-bus";
 			#address-cells = <2>;
 			#size-cells = <2>;
-			ranges;
-			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
 			dma-ranges = <0x0 0x0 0x0 0x0 0x10 0x0>;
-			status = "disabled";
+			ranges;
 
-			usb3_dwc3: usb@4c100000 {
-				compatible = "snps,dwc3";
-				reg = <0x0 0x4c100000 0x0 0x10000>;
+			usb3: usb3_dwc3: usb@4c100000 {
+				compatible = "nxp,imx95-dwc3", "nxp,imx8mp-dwc3";
+				reg = <0x0 0x4c100000 0x0 0x10000>,
+				      <0x0 0x4c010010 0x0 0x04>,
+				      <0x0 0x4c1f0000 0x0 0x20>;
+				reg-names = "core", "blkctl", "glue";
 				clocks = <&scmi_clk IMX95_CLK_HSIO>,
+					 <&scmi_clk IMX95_CLK_HSIO>,
 					 <&scmi_clk IMX95_CLK_24M>,
 					 <&scmi_clk IMX95_CLK_32K>;
-				clock-names = "bus_early", "ref", "suspend";
-				interrupts = <GIC_SPI 175 IRQ_TYPE_LEVEL_HIGH>;
+				clock-names = "hsio", "bus_early", "ref", "suspend";
+				interrupts = <GIC_SPI 175 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 173 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "dwc_usb3", "wakeup";
+				power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
 				phys = <&usb3_phy>, <&usb3_phy>;
 				phy-names = "usb2-phy", "usb3-phy";
 				snps,gfladj-refclk-lpm-sel-quirk;
 				snps,parkmode-disable-ss-quirk;
 				iommus = <&smmu 0xe>;
+				status = "disabled";
 			};
 		};
 
-		hsio_blk_ctl: syscon@4c0100c0 {
-			compatible = "nxp,imx95-hsio-blk-ctl", "syscon";
-			reg = <0x0 0x4c0100c0 0x0 0x1>;
-			#clock-cells = <1>;
-			clocks = <&clk_sys100m>;
-			power-domains = <&scmi_devpd IMX95_PD_HSIO_TOP>;
-		};
-
 		usb3_phy: phy@4c1f0040 {
 			compatible = "fsl,imx95-usb-phy", "fsl,imx8mp-usb-phy";
 			reg = <0x0 0x4c1f0040 0x0 0x40>,
diff --git a/arch/arm64/boot/dts/freescale/s32g2.dtsi b/arch/arm64/boot/dts/freescale/s32g2.dtsi
index 809019ea0e299..8dc0c5d9f368f 100644
--- a/arch/arm64/boot/dts/freescale/s32g2.dtsi
+++ b/arch/arm64/boot/dts/freescale/s32g2.dtsi
@@ -135,7 +135,26 @@ pinctrl: pinctrl@4009c240 {
 				/* IMCR119-IMCR397 registers on siul2_1 */
 			      <0x44010c1c 0x45c>,
 				/* IMCR430-IMCR495 registers on siul2_1 */
-			      <0x440110f8 0x108>;
+			      <0x440110f8 0x108>,
+				/* PGPDO registers on siul2_0 */
+			      <0x4009d700 0x10>,
+				/* PGPDI registers on siul2_0 */
+			      <0x4009d740 0x10>,
+				/* PGPDO registers on siul2_1 */
+			      <0x44011700 0x18>,
+				/* PGPDI registers on siul2_1 */
+			      <0x44011740 0x18>,
+				/* EIRQ window: DISR0..IFEER0 */
+			      <0x44010010 0x34>;
+
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&pinctrl 0 0 102>,
+					<&pinctrl 112 112 79>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			interrupts = <GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH>;
 
 			jtag_pins: jtag-pins {
 				jtag-grp0 {
diff --git a/arch/arm64/boot/dts/freescale/s32g3.dtsi b/arch/arm64/boot/dts/freescale/s32g3.dtsi
index 22e80fc03f9c7..24fa3057e9299 100644
--- a/arch/arm64/boot/dts/freescale/s32g3.dtsi
+++ b/arch/arm64/boot/dts/freescale/s32g3.dtsi
@@ -193,7 +193,26 @@ pinctrl: pinctrl@4009c240 {
 				/* IMCR119-IMCR397 registers on siul2_1 */
 			      <0x44010c1c 0x45c>,
 				/* IMCR430-IMCR495 registers on siul2_1 */
-			      <0x440110f8 0x108>;
+			      <0x440110f8 0x108>,
+				/* PGPDO registers on siul2_0 */
+			      <0x4009d700 0x10>,
+				/* PGPDI registers on siul2_0 */
+			      <0x4009d740 0x10>,
+				/* PGPDO registers on siul2_1 */
+			      <0x44011700 0x18>,
+				/* PGPDI registers on siul2_1 */
+			      <0x44011740 0x18>,
+				/* EIRQ window: DISR0..IFEER0 */
+			      <0x44010010 0x34>;
+
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&pinctrl 0 0 102>,
+					<&pinctrl 112 112 79>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			interrupts = <GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH>;
 
 			jtag_pins: jtag-pins {
 				jtag-grp0 {
@@ -534,22 +553,22 @@ usbmisc: usbmisc@44064200 {
 			reg = <0x44064200 0x200>;
 		};
 
-                usbotg: usb@44064000 {
-                        compatible = "nxp,s32g3-usb", "nxp,s32g2-usb";
-                        reg = <0x44064000 0x200>;
-                        interrupt-parent = <&gic>;
-                        interrupts = <GIC_SPI 211 IRQ_TYPE_LEVEL_HIGH>, /* OTG Core */
-                                     <GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH>; /* OTG Wakeup */
-                        clocks = <&clks 94>, <&clks 95>;
-                        fsl,usbmisc = <&usbmisc 0>;
-                        ahb-burst-config = <0x3>;
-                        tx-burst-size-dword = <0x10>;
-                        rx-burst-size-dword = <0x10>;
-                        phy_type = "ulpi";
-                        dr_mode = "host";
-                        maximum-speed = "high-speed";
-                        status = "disabled";
-                };
+		usbotg: usb@44064000 {
+			compatible = "nxp,s32g3-usb", "nxp,s32g2-usb";
+			reg = <0x44064000 0x200>;
+			interrupt-parent = <&gic>;
+			interrupts = <GIC_SPI 211 IRQ_TYPE_LEVEL_HIGH>, /* OTG Core */
+				     <GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH>; /* OTG Wakeup */
+			clocks = <&clks 94>, <&clks 95>;
+			fsl,usbmisc = <&usbmisc 0>;
+			ahb-burst-config = <0x3>;
+			tx-burst-size-dword = <0x10>;
+			rx-burst-size-dword = <0x10>;
+			phy_type = "ulpi";
+			dr_mode = "host";
+			maximum-speed = "high-speed";
+			status = "disabled";
+		};
 
 		spi0: spi@401d4000 {
 			compatible = "nxp,s32g3-dspi", "nxp,s32g2-dspi";
diff --git a/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi b/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi
index 2f4ad5da5e33c..e5010c9d4c392 100644
--- a/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi
+++ b/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi
@@ -633,9 +633,7 @@ ohci: usb@9880000 {
 			clocks = <&crg HISTB_USB2_BUS_CLK>,
 				 <&crg HISTB_USB2_12M_CLK>,
 				 <&crg HISTB_USB2_48M_CLK>;
-			clock-names = "bus", "clk12", "clk48";
 			resets = <&crg 0xb8 12>;
-			reset-names = "bus";
 			phys = <&usb2_phy1_port0>;
 			phy-names = "usb";
 			status = "disabled";
@@ -648,11 +646,9 @@ ehci: usb@9890000 {
 			clocks = <&crg HISTB_USB2_BUS_CLK>,
 				 <&crg HISTB_USB2_PHY_CLK>,
 				 <&crg HISTB_USB2_UTMI_CLK>;
-			clock-names = "bus", "phy", "utmi";
 			resets = <&crg 0xb8 12>,
 				 <&crg 0xb8 16>,
 				 <&crg 0xb8 13>;
-			reset-names = "bus", "phy", "utmi";
 			phys = <&usb2_phy1_port0>;
 			phy-names = "usb";
 			status = "disabled";
diff --git a/arch/arm64/boot/dts/hisilicon/hip05.dtsi b/arch/arm64/boot/dts/hisilicon/hip05.dtsi
index d0912ca5f237f..2d376b3d30dd1 100644
--- a/arch/arm64/boot/dts/hisilicon/hip05.dtsi
+++ b/arch/arm64/boot/dts/hisilicon/hip05.dtsi
@@ -236,13 +236,13 @@ cluster3_l2: l2-cache3 {
 
 	gic: interrupt-controller@8d000000 {
 		compatible = "arm,gic-v3";
-                #interrupt-cells = <3>;
-                #address-cells = <2>;
-                #size-cells = <2>;
-                ranges;
-                interrupt-controller;
-                #redistributor-regions = <1>;
-                redistributor-stride = <0x0 0x30000>;
+		#interrupt-cells = <3>;
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		interrupt-controller;
+		#redistributor-regions = <1>;
+		redistributor-stride = <0x0 0x30000>;
 		reg = <0x0 0x8d000000 0 0x10000>,	/* GICD */
 		      <0x0 0x8d100000 0 0x300000>,	/* GICR */
 		      <0x0 0xfe000000 0 0x10000>,	/* GICC */
diff --git a/arch/arm64/boot/dts/intel/Makefile b/arch/arm64/boot/dts/intel/Makefile
index 088a03b89c995..270c70fdf084b 100644
--- a/arch/arm64/boot/dts/intel/Makefile
+++ b/arch/arm64/boot/dts/intel/Makefile
@@ -8,6 +8,7 @@ dtb-$(CONFIG_ARCH_INTEL_SOCFPGA) += socfpga_agilex_n6000.dtb \
 				socfpga_agilex5_socdk_013b.dtb \
 				socfpga_agilex5_socdk_modular.dtb \
 				socfpga_agilex5_socdk_nand.dtb \
+				socfpga_agilex72_socdk.dtb \
 				socfpga_agilex7m_socdk.dtb \
 				socfpga_n5x_socdk.dtb
 dtb-$(CONFIG_ARCH_KEEMBAY) += keembay-evm.dtb
diff --git a/arch/arm64/boot/dts/intel/keembay-soc.dtsi b/arch/arm64/boot/dts/intel/keembay-soc.dtsi
index ae00e9e54e82c..b6323ebe7a5f4 100644
--- a/arch/arm64/boot/dts/intel/keembay-soc.dtsi
+++ b/arch/arm64/boot/dts/intel/keembay-soc.dtsi
@@ -71,7 +71,7 @@ timer {
 
 	pmu {
 		compatible = "arm,cortex-a53-pmu";
-		interrupts = <GIC_PPI 0x7 IRQ_TYPE_LEVEL_HIGH>;
+		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_HIGH>;
 	};
 
 	soc {
diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi b/arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi
index b06c6d5d60eed..f54767d1526e1 100644
--- a/arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi
+++ b/arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi
@@ -385,7 +385,6 @@ smmu: iommu@16000000 {
 			interrupt-names = "eventq", "gerror", "priq";
 			dma-coherent;
 			#iommu-cells = <1>;
-			status = "disabled";
 		};
 
 		spi0: spi@10da4000 {
@@ -557,8 +556,40 @@ gmac0: ethernet@10810000 {
 			compatible = "altr,socfpga-stmmac-agilex5",
 				     "snps,dwxgmac-2.10";
 			reg = <0x10810000 0x3500>;
-			interrupts = <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "macirq";
+			interrupts = <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 194 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 195 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 198 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 199 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 204 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 206 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "macirq",
+					  "tx-queue-0",
+					  "tx-queue-1",
+					  "tx-queue-2",
+					  "tx-queue-3",
+					  "tx-queue-4",
+					  "tx-queue-5",
+					  "tx-queue-6",
+					  "tx-queue-7",
+					  "rx-queue-0",
+					  "rx-queue-1",
+					  "rx-queue-2",
+					  "rx-queue-3",
+					  "rx-queue-4",
+					  "rx-queue-5",
+					  "rx-queue-6",
+					  "rx-queue-7";
 			resets = <&rst EMAC0_RESET>, <&rst EMAC0_OCP_RESET>;
 			reset-names = "stmmaceth", "ahb";
 			clocks = <&clkmgr AGILEX5_EMAC0_CLK>,
@@ -577,6 +608,7 @@ gmac0: ethernet@10810000 {
 			altr,sysmgr-syscon = <&sysmgr 0x44 0>;
 			snps,clk-csr = <0>;
 			iommus = <&smmu 1>;
+			dma-coherent;
 			status = "disabled";
 
 			stmmac_axi_emac0_setup: stmmac-axi-config {
@@ -670,8 +702,40 @@ gmac1: ethernet@10820000 {
 			compatible = "altr,socfpga-stmmac-agilex5",
 				     "snps,dwxgmac-2.10";
 			reg = <0x10820000 0x3500>;
-			interrupts = <GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "macirq";
+			interrupts = <GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 209 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 211 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 213 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 214 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 215 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 217 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 218 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 219 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 220 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 221 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 222 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 223 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "macirq",
+					  "tx-queue-0",
+					  "tx-queue-1",
+					  "tx-queue-2",
+					  "tx-queue-3",
+					  "tx-queue-4",
+					  "tx-queue-5",
+					  "tx-queue-6",
+					  "tx-queue-7",
+					  "rx-queue-0",
+					  "rx-queue-1",
+					  "rx-queue-2",
+					  "rx-queue-3",
+					  "rx-queue-4",
+					  "rx-queue-5",
+					  "rx-queue-6",
+					  "rx-queue-7";
 			resets = <&rst EMAC1_RESET>, <&rst EMAC1_OCP_RESET>;
 			reset-names = "stmmaceth", "ahb";
 			clocks = <&clkmgr AGILEX5_EMAC1_CLK>,
@@ -690,6 +754,7 @@ gmac1: ethernet@10820000 {
 			altr,sysmgr-syscon = <&sysmgr 0x48 0>;
 			snps,clk-csr = <0>;
 			iommus = <&smmu 2>;
+			dma-coherent;
 			status = "disabled";
 
 			stmmac_axi_emac1_setup: stmmac-axi-config {
@@ -783,8 +848,40 @@ gmac2: ethernet@10830000 {
 			compatible = "altr,socfpga-stmmac-agilex5",
 				     "snps,dwxgmac-2.10";
 			reg = <0x10830000 0x3500>;
-			interrupts = <GIC_SPI 224 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "macirq";
+			interrupts = <GIC_SPI 224 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 225 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 226 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 227 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 228 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 229 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 230 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 231 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 232 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 233 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 234 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 235 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 236 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 237 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 238 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 239 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 240 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "macirq",
+					  "tx-queue-0",
+					  "tx-queue-1",
+					  "tx-queue-2",
+					  "tx-queue-3",
+					  "tx-queue-4",
+					  "tx-queue-5",
+					  "tx-queue-6",
+					  "tx-queue-7",
+					  "rx-queue-0",
+					  "rx-queue-1",
+					  "rx-queue-2",
+					  "rx-queue-3",
+					  "rx-queue-4",
+					  "rx-queue-5",
+					  "rx-queue-6",
+					  "rx-queue-7";
 			resets = <&rst EMAC2_RESET>, <&rst EMAC2_OCP_RESET>;
 			reset-names = "stmmaceth", "ahb";
 			clocks = <&clkmgr AGILEX5_EMAC2_CLK>,
@@ -803,6 +900,7 @@ gmac2: ethernet@10830000 {
 			altr,sysmgr-syscon = <&sysmgr 0x4c 0>;
 			snps,clk-csr = <0>;
 			iommus = <&smmu 3>;
+			dma-coherent;
 			status = "disabled";
 
 			stmmac_axi_emac2_setup: stmmac-axi-config {
diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex72.dtsi b/arch/arm64/boot/dts/intel/socfpga_agilex72.dtsi
new file mode 100644
index 0000000000000..c29c2afcaab70
--- /dev/null
+++ b/arch/arm64/boot/dts/intel/socfpga_agilex72.dtsi
@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026, Altera Corporation
+ */
+/dts-v1/;
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+	compatible = "intel,socfpga-agilex72";
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		atf_reserved: atf@80000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x80000000 0x0 0x100000>;
+			alignment = <0x1000>;
+			no-map;
+		};
+
+		service_reserved: svcbuffer@80100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x80100000 0x0 0xf00000>;
+			alignment = <0x1000>;
+			no-map;
+		};
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			compatible = "arm,cortex-a520";
+			device_type = "cpu";
+			enable-method = "psci";
+			reg = <0x0>;
+		};
+
+		cpu1: cpu@100 {
+			compatible = "arm,cortex-a520";
+			device_type = "cpu";
+			enable-method = "psci";
+			reg = <0x100>;
+		};
+
+		cpu2: cpu@200 {
+			compatible = "arm,cortex-a720";
+			device_type = "cpu";
+			enable-method = "psci";
+			reg = <0x200>;
+		};
+
+		cpu3: cpu@300 {
+			compatible = "arm,cortex-a720";
+			device_type = "cpu";
+			enable-method = "psci";
+			reg = <0x300>;
+		};
+	};
+
+	clocks {
+		uart_clk: uart-clk {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <125000000>;
+		};
+	};
+
+	psci {
+		compatible = "arm,psci-0.2";
+		method = "smc";
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupt-parent = <&intc>;
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	intc: interrupt-controller@7000000 {
+		compatible = "arm,gic-v3";
+		reg = <0x0 0x7000000 0x0 0x10000>,
+		      <0x0 0x7080000 0x0 0x100000>;
+		ranges;
+		#interrupt-cells = <3>;
+		#address-cells = <2>;
+		#size-cells = <2>;
+		interrupt-controller;
+		#redistributor-regions = <1>;
+		redistributor-stride = <0x0 0x40000>;
+
+		its: msi-controller@7040000 {
+			compatible = "arm,gic-v3-its";
+			reg = <0x0 0x7040000 0x0 0x20000>;
+			msi-controller;
+			#msi-cells = <1>;
+		};
+	};
+
+	soc: soc@0 {
+		compatible = "simple-bus";
+		ranges = <0 0 0 0xffffffff>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		device_type = "soc";
+		interrupt-parent = <&intc>;
+
+		smmu: iommu@c100000 {
+			compatible = "arm,smmu-v3";
+			reg = <0x0c100000 0x30000>;
+			interrupts = <GIC_SPI 134 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 129 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 132 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "eventq", "gerror", "priq";
+			dma-coherent;
+			#iommu-cells = <1>;
+		};
+
+		ocram: sram@0 {
+			compatible = "mmio-sram";
+			reg = <0x00000000 0x80000>;
+			ranges = <0 0 0x80000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+		};
+
+		uart0: serial@9038000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x9038000 0x100>;
+			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			clocks = <&uart_clk>;
+			status = "disabled";
+		};
+
+		uart1: serial@9039000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x9039000 0x100>;
+			interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			clocks = <&uart_clk>;
+			status = "disabled";
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex72_socdk.dts b/arch/arm64/boot/dts/intel/socfpga_agilex72_socdk.dts
new file mode 100644
index 0000000000000..998f19f492b36
--- /dev/null
+++ b/arch/arm64/boot/dts/intel/socfpga_agilex72_socdk.dts
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026, Altera Corporation
+ */
+#include "socfpga_agilex72.dtsi"
+
+/ {
+	model = "Altera SoCFPGA Agilex72 SoCDK";
+	compatible = "intel,socfpga-agilex72-socdk", "intel,socfpga-agilex72";
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 0x0 0x80000000>;
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
index 360fc24fdde22..8418777042b85 100644
--- a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
@@ -437,6 +437,7 @@ crypto: crypto@90000 {
 				interrupt-names = "ring0", "ring1", "ring2",
 						  "ring3", "eip", "mem";
 				clocks = <&nb_periph_clk 15>;
+				dma-coherent;
 			};
 
 			rwtm: mailbox@b0000 {
diff --git a/arch/arm64/boot/dts/mediatek/mt6795.dtsi b/arch/arm64/boot/dts/mediatek/mt6795.dtsi
index 134cfa77e3b1f..6dd706b278305 100644
--- a/arch/arm64/boot/dts/mediatek/mt6795.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt6795.dtsi
@@ -547,8 +547,7 @@ uart1: serial@11003000 {
 		};
 
 		apdma: dma-controller@11000380 {
-			compatible = "mediatek,mt6795-uart-dma",
-				     "mediatek,mt6577-uart-dma";
+			compatible = "mediatek,mt6795-uart-dma";
 			reg = <0 0x11000380 0 0x60>,
 			      <0 0x11000400 0 0x60>,
 			      <0 0x11000480 0 0x60>,
@@ -568,7 +567,6 @@ apdma: dma-controller@11000380 {
 			dma-requests = <8>;
 			clocks = <&pericfg CLK_PERI_AP_DMA>;
 			clock-names = "apdma";
-			mediatek,dma-33bits;
 			#dma-cells = <1>;
 		};
 
diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-kodama.dtsi b/arch/arm64/boot/dts/mediatek/mt8183-kukui-kodama.dtsi
index b702ff066636e..b210845277e10 100644
--- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-kodama.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-kodama.dtsi
@@ -86,7 +86,7 @@ touchscreen: touchscreen@10 {
 };
 
 &i2c2 {
-        pinctrl-names = "default";
+	pinctrl-names = "default";
 	pinctrl-0 = <&i2c2_pins>;
 	status = "okay";
 	clock-frequency = <400000>;
@@ -101,7 +101,7 @@ eeprom@58 {
 };
 
 &i2c4 {
-        pinctrl-names = "default";
+	pinctrl-names = "default";
 	pinctrl-0 = <&i2c4_pins>;
 	status = "okay";
 	clock-frequency = <400000>;
@@ -381,5 +381,5 @@ &qca_wifi {
 };
 
 &i2c_tunnel {
-        google,remote-bus = <2>;
+	google,remote-bus = <2>;
 };
diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui-krane-sku176.dts b/arch/arm64/boot/dts/mediatek/mt8183-kukui-krane-sku176.dts
index 095279e55d50c..cc8c913d27637 100644
--- a/arch/arm64/boot/dts/mediatek/mt8183-kukui-krane-sku176.dts
+++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui-krane-sku176.dts
@@ -19,6 +19,6 @@ / {
 };
 
 &panel {
-        status = "okay";
-        compatible = "boe,tv101wum-nl6";
+	status = "okay";
+	compatible = "boe,tv101wum-nl6";
 };
diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui.dtsi b/arch/arm64/boot/dts/mediatek/mt8183-kukui.dtsi
index a8e257b21a88c..5ea499f349f64 100644
--- a/arch/arm64/boot/dts/mediatek/mt8183-kukui.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui.dtsi
@@ -849,6 +849,8 @@ w25q64dw: flash@0 {
 		compatible = "winbond,w25q64dw", "jedec,spi-nor";
 		reg = <0>;
 		spi-max-frequency = <25000000>;
+		/* Power actually controlled by security chip */
+		vcc-supply = <&pp1800_alw>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/mediatek/mt8183.dtsi b/arch/arm64/boot/dts/mediatek/mt8183.dtsi
index 95cc067995336..8b0992548431c 100644
--- a/arch/arm64/boot/dts/mediatek/mt8183.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8183.dtsi
@@ -1071,6 +1071,22 @@ gce: mailbox@10238000 {
 			clock-names = "gce";
 		};
 
+		apdma: dma-controller@11000780 {
+			compatible = "mediatek,mt8183-uart-dma", "mediatek,mt6577-uart-dma";
+			reg = <0 0x11000780 0 0x80>, <0 0x11000800 0 0x80>,
+			      <0 0x11000880 0 0x80>, <0 0x11000900 0 0x80>,
+			      <0 0x11000980 0 0x80>, <0 0x11000a00 0 0x80>;
+			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_LOW>,
+				     <GIC_SPI 109 IRQ_TYPE_LEVEL_LOW>,
+				     <GIC_SPI 110 IRQ_TYPE_LEVEL_LOW>,
+				     <GIC_SPI 111 IRQ_TYPE_LEVEL_LOW>,
+				     <GIC_SPI 112 IRQ_TYPE_LEVEL_LOW>,
+				     <GIC_SPI 113 IRQ_TYPE_LEVEL_LOW>;
+			clocks = <&infracfg CLK_INFRA_AP_DMA>;
+			#dma-cells = <1>;
+			dma-requests = <6>;
+		};
+
 		auxadc: auxadc@11001000 {
 			compatible = "mediatek,mt8183-auxadc",
 				     "mediatek,mt8173-auxadc";
@@ -1088,6 +1104,8 @@ uart0: serial@11002000 {
 			interrupts = <GIC_SPI 91 IRQ_TYPE_LEVEL_LOW>;
 			clocks = <&clk26m>, <&infracfg CLK_INFRA_UART0>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 0>, <&apdma 1>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
@@ -1098,6 +1116,8 @@ uart1: serial@11003000 {
 			interrupts = <GIC_SPI 92 IRQ_TYPE_LEVEL_LOW>;
 			clocks = <&clk26m>, <&infracfg CLK_INFRA_UART1>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 2>, <&apdma 3>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
@@ -1108,6 +1128,8 @@ uart2: serial@11004000 {
 			interrupts = <GIC_SPI 93 IRQ_TYPE_LEVEL_LOW>;
 			clocks = <&clk26m>, <&infracfg CLK_INFRA_UART2>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 4>, <&apdma 5>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
diff --git a/arch/arm64/boot/dts/mediatek/mt8186.dtsi b/arch/arm64/boot/dts/mediatek/mt8186.dtsi
index b91f88ffae0e8..621408439ffa7 100644
--- a/arch/arm64/boot/dts/mediatek/mt8186.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8186.dtsi
@@ -48,79 +48,79 @@ cci_opp: opp-table-cci {
 		compatible = "operating-points-v2";
 		opp-shared;
 
-		cci_opp_0: opp-500000000 {
-			opp-hz = /bits/ 64 <500000000>;
-			opp-microvolt = <600000>;
+		cci_opp_0: opp-800000000 {
+			opp-hz = /bits/ 64 <800000000>;
+			opp-microvolt = <800000>;
 		};
 
-		cci_opp_1: opp-560000000 {
-			opp-hz = /bits/ 64 <560000000>;
-			opp-microvolt = <675000>;
+		cci_opp_1: opp-840000000 {
+			opp-hz = /bits/ 64 <840000000>;
+			opp-microvolt = <806250>;
 		};
 
-		cci_opp_2: opp-612000000 {
-			opp-hz = /bits/ 64 <612000000>;
-			opp-microvolt = <693750>;
+		cci_opp_2: opp-880000000 {
+			opp-hz = /bits/ 64 <880000000>;
+			opp-microvolt = <812500>;
 		};
 
-		cci_opp_3: opp-682000000 {
-			opp-hz = /bits/ 64 <682000000>;
-			opp-microvolt = <718750>;
+		cci_opp_3: opp-920000000 {
+			opp-hz = /bits/ 64 <920000000>;
+			opp-microvolt = <825000>;
 		};
 
-		cci_opp_4: opp-752000000 {
-			opp-hz = /bits/ 64 <752000000>;
-			opp-microvolt = <743750>;
+		cci_opp_4: opp-960000000 {
+			opp-hz = /bits/ 64 <960000000>;
+			opp-microvolt = <831250>;
 		};
 
-		cci_opp_5: opp-822000000 {
-			opp-hz = /bits/ 64 <822000000>;
-			opp-microvolt = <768750>;
+		cci_opp_5: opp-1000000000 {
+			opp-hz = /bits/ 64 <1000000000>;
+			opp-microvolt = <837500>;
 		};
 
-		cci_opp_6: opp-875000000 {
-			opp-hz = /bits/ 64 <875000000>;
-			opp-microvolt = <781250>;
+		cci_opp_6: opp-1040000000 {
+			opp-hz = /bits/ 64 <1040000000>;
+			opp-microvolt = <850000>;
 		};
 
-		cci_opp_7: opp-927000000 {
-			opp-hz = /bits/ 64 <927000000>;
-			opp-microvolt = <800000>;
+		cci_opp_7: opp-1080000000 {
+			opp-hz = /bits/ 64 <1080000000>;
+			opp-microvolt = <856250>;
 		};
 
-		cci_opp_8: opp-980000000 {
-			opp-hz = /bits/ 64 <980000000>;
-			opp-microvolt = <818750>;
+		cci_opp_8: opp-1120000000 {
+			opp-hz = /bits/ 64 <1120000000>;
+			opp-microvolt = <862500>;
 		};
 
-		cci_opp_9: opp-1050000000 {
-			opp-hz = /bits/ 64 <1050000000>;
-			opp-microvolt = <843750>;
+		cci_opp_9: opp-1160000000 {
+			opp-hz = /bits/ 64 <1160000000>;
+			opp-microvolt = <887500>;
 		};
 
-		cci_opp_10: opp-1120000000 {
-			opp-hz = /bits/ 64 <1120000000>;
-			opp-microvolt = <862500>;
+		cci_opp_10: opp-1200000000 {
+			opp-hz = /bits/ 64 <1200000000>;
+			opp-microvolt = <912500>;
 		};
 
-		cci_opp_11: opp-1155000000 {
-			opp-hz = /bits/ 64 <1155000000>;
-			opp-microvolt = <887500>;
+		cci_opp_11: opp-1240000000 {
+			opp-hz = /bits/ 64 <1240000000>;
+			opp-microvolt = <937500>;
 		};
 
-		cci_opp_12: opp-1190000000 {
-			opp-hz = /bits/ 64 <1190000000>;
-			opp-microvolt = <906250>;
+		cci_opp_12: opp-1280000000 {
+			opp-hz = /bits/ 64 <1280000000>;
+			opp-microvolt = <962500>;
 		};
 
-		cci_opp_13: opp-1260000000 {
-			opp-hz = /bits/ 64 <1260000000>;
-			opp-microvolt = <950000>;
+		cci_opp_13: opp-1320000000 {
+			opp-hz = /bits/ 64 <1320000000>;
+			opp-microvolt = <987500>;
 		};
 
-		cci_opp_14: opp-1330000000 {
-			opp-hz = /bits/ 64 <1330000000>;
-			opp-microvolt = <993750>;
+		cci_opp_14: opp-1360000000 {
+			opp-hz = /bits/ 64 <1360000000>;
+			opp-microvolt = <1012500>;
 		};
 
 		cci_opp_15: opp-1400000000 {
@@ -1142,6 +1142,22 @@ systimer: timer@10017000 {
 			clocks = <&clk13m>;
 		};
 
+		apdma: dma-controller@10200d80 {
+			compatible = "mediatek,mt8186-uart-dma", "mediatek,mt6835-uart-dma";
+			reg = <0 0x10200d80 0 0x80>, <0 0x10200e00 0 0x80>,
+			      <0 0x10200e80 0 0x80>, <0 0x10200f00 0 0x80>,
+			      <0 0x10200f80 0 0x80>, <0 0x10201000 0 0x80>;
+			interrupts = <GIC_SPI 376 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 377 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 378 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 379 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 380 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 381 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&infracfg_ao CLK_INFRA_AO_AP_DMA>;
+			#dma-cells = <1>;
+			dma-requests = <6>;
+		};
+
 		gce: mailbox@1022c000 {
 			compatible = "mediatek,mt8186-gce";
 			reg = <0 0X1022c000 0 0x4000>;
@@ -1218,6 +1234,8 @@ uart0: serial@11002000 {
 			interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH 0>;
 			clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART0>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 0>, <&apdma 1>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
@@ -1228,6 +1246,8 @@ uart1: serial@11003000 {
 			interrupts = <GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH 0>;
 			clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART1>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 2>, <&apdma 3>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
@@ -1482,6 +1502,8 @@ uart2: serial@11018000 {
 			interrupts = <GIC_SPI 246 IRQ_TYPE_LEVEL_HIGH 0>;
 			clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART2>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 4>, <&apdma 5>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
diff --git a/arch/arm64/boot/dts/mediatek/mt8188-geralt.dtsi b/arch/arm64/boot/dts/mediatek/mt8188-geralt.dtsi
index 4cb23595d17b4..f382f90c48f5e 100644
--- a/arch/arm64/boot/dts/mediatek/mt8188-geralt.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8188-geralt.dtsi
@@ -645,6 +645,7 @@ flash@0 {
 		compatible = "jedec,spi-nor";
 		reg = <0>;
 		spi-max-frequency = <52000000>;
+		vcc-supply = <&pp1800_ldo_z1>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/mediatek/mt8188.dtsi b/arch/arm64/boot/dts/mediatek/mt8188.dtsi
index 75133794cec38..14a320f0c70cb 100644
--- a/arch/arm64/boot/dts/mediatek/mt8188.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8188.dtsi
@@ -1360,6 +1360,31 @@ spmi: spmi@10027000 {
 			clock-names = "pmif_sys_ck", "pmif_tmr_ck", "spmimst_clk_mux";
 		};
 
+		apdma: dma-controller@10220880 {
+			compatible = "mediatek,mt8188-uart-dma", "mediatek,mt6835-uart-dma";
+			reg = <0 0x10220880 0 0x80>, <0 0x10220900 0 0x80>,
+			      <0 0x10220980 0 0x80>, <0 0x10220a00 0 0x80>,
+			      <0 0x10220a80 0 0x80>, <0 0x10220b00 0 0x80>,
+			      <0 0x10220b80 0 0x80>, <0 0x10220c00 0 0x80>,
+			      <0 0x10220c80 0 0x80>, <0 0x10220d00 0 0x80>,
+			      <0 0x10220d80 0 0x80>, <0 0x10220e00 0 0x80>;
+			interrupts = <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 175 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 176 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 177 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 178 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 179 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 180 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 181 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 182 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 183 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 184 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 185 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&infracfg_ao CLK_INFRA_AO_APDMA_BCLK>;
+			#dma-cells = <1>;
+			dma-requests = <12>;
+		};
+
 		infra_iommu: iommu@10315000 {
 			compatible = "mediatek,mt8188-iommu-infra";
 			reg = <0 0x10315000 0 0x1000>;
@@ -1512,6 +1537,8 @@ uart0: serial@11001100 {
 			interrupts = <GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH 0>;
 			clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART0>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 0>, <&apdma 1>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
@@ -1521,6 +1548,8 @@ uart1: serial@11001200 {
 			interrupts = <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH 0>;
 			clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART1>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 2>, <&apdma 3>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
@@ -1530,6 +1559,8 @@ uart2: serial@11001300 {
 			interrupts = <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH 0>;
 			clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART2>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 4>, <&apdma 5>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
@@ -1539,6 +1570,8 @@ uart3: serial@11001400 {
 			interrupts = <GIC_SPI 723 IRQ_TYPE_LEVEL_HIGH 0>;
 			clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART3>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 6>, <&apdma 7>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
diff --git a/arch/arm64/boot/dts/mediatek/mt8192.dtsi b/arch/arm64/boot/dts/mediatek/mt8192.dtsi
index 9f8f115edd4cc..898953acdb610 100644
--- a/arch/arm64/boot/dts/mediatek/mt8192.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8192.dtsi
@@ -741,6 +741,19 @@ spmi: spmi@10027000 {
 			assigned-clock-parents = <&topckgen CLK_TOP_OSC_D10>;
 		};
 
+		apdma: dma-controller@10217a80 {
+			compatible = "mediatek,mt8192-uart-dma", "mediatek,mt6835-uart-dma";
+			reg = <0 0x10217a80 0 0x80>, <0 0x10217b00 0 0x80>,
+			      <0 0x10217b80 0 0x80>, <0 0x10217c00 0 0x80>;
+			interrupts = <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&infracfg CLK_INFRA_AP_DMA>;
+			#dma-cells = <1>;
+			dma-requests = <4>;
+		};
+
 		gce: mailbox@10228000 {
 			compatible = "mediatek,mt8192-gce";
 			reg = <0 0x10228000 0 0x4000>;
@@ -765,6 +778,8 @@ uart0: serial@11002000 {
 			interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH 0>;
 			clocks = <&clk26m>, <&infracfg CLK_INFRA_UART0>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 0>, <&apdma 1>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
@@ -775,6 +790,8 @@ uart1: serial@11003000 {
 			interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH 0>;
 			clocks = <&clk26m>, <&infracfg CLK_INFRA_UART1>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 2>, <&apdma 3>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
diff --git a/arch/arm64/boot/dts/mediatek/mt8195.dtsi b/arch/arm64/boot/dts/mediatek/mt8195.dtsi
index c72e34c57629d..9aaf8b6edcb50 100644
--- a/arch/arm64/boot/dts/mediatek/mt8195.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8195.dtsi
@@ -904,6 +904,31 @@ spmi: spmi@10027000 {
 			assigned-clock-parents = <&topckgen CLK_TOP_ULPOSC1_D10>;
 		};
 
+		apdma: dma-controller@10220880 {
+			compatible = "mediatek,mt8195-uart-dma", "mediatek,mt6835-uart-dma";
+			reg = <0 0x10220880 0 0x80>, <0 0x10220900 0 0x80>,
+			      <0 0x10220980 0 0x80>, <0 0x10220a00 0 0x80>,
+			      <0 0x10220a80 0 0x80>, <0 0x10220b00 0 0x80>,
+			      <0 0x10220b80 0 0x80>, <0 0x10220c00 0 0x80>,
+			      <0 0x10220c80 0 0x80>, <0 0x10220d00 0 0x80>,
+			      <0 0x10220d80 0 0x80>, <0 0x10220e00 0 0x80>;
+			interrupts = <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 175 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 176 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 177 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 178 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 179 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 180 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 181 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 182 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 183 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 184 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 185 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&infracfg_ao CLK_INFRA_AO_APDMA_B>;
+			#dma-cells = <1>;
+			dma-requests = <12>;
+		};
+
 		iommu_infra: infra-iommu@10315000 {
 			compatible = "mediatek,mt8195-iommu-infra";
 			reg = <0 0x10315000 0 0x5000>;
@@ -1040,6 +1065,8 @@ uart0: serial@11001100 {
 			interrupts = <GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH 0>;
 			clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART0>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 0>, <&apdma 1>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
@@ -1050,6 +1077,8 @@ uart1: serial@11001200 {
 			interrupts = <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH 0>;
 			clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART1>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 2>, <&apdma 3>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
@@ -1060,6 +1089,8 @@ uart2: serial@11001300 {
 			interrupts = <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH 0>;
 			clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART2>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 4>, <&apdma 5>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
@@ -1070,6 +1101,8 @@ uart3: serial@11001400 {
 			interrupts = <GIC_SPI 723 IRQ_TYPE_LEVEL_HIGH 0>;
 			clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART3>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 6>, <&apdma 7>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
@@ -1080,6 +1113,8 @@ uart4: serial@11001500 {
 			interrupts = <GIC_SPI 724 IRQ_TYPE_LEVEL_HIGH 0>;
 			clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART4>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 8>, <&apdma 9>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
@@ -1090,6 +1125,8 @@ uart5: serial@11001600 {
 			interrupts = <GIC_SPI 725 IRQ_TYPE_LEVEL_HIGH 0>;
 			clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART5>;
 			clock-names = "baud", "bus";
+			dmas = <&apdma 10>, <&apdma 11>;
+			dma-names = "tx", "rx";
 			status = "disabled";
 		};
 
diff --git a/arch/arm64/boot/dts/mediatek/mt8390-genio-common.dtsi b/arch/arm64/boot/dts/mediatek/mt8390-genio-common.dtsi
index b0c97930a0e6c..ce27f314d19d6 100644
--- a/arch/arm64/boot/dts/mediatek/mt8390-genio-common.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8390-genio-common.dtsi
@@ -7,7 +7,7 @@
  *
  * Copyright (C) 2025 Collabora Ltd.
  *                    Louis-Alexis Eyraud <louisalexis.eyraud@collabora.com>
- *                    AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>   
+ *                    AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
  */
 
 #include "mt6365.dtsi"
diff --git a/arch/arm64/boot/dts/mediatek/mt8390-tungsten-smarc.dtsi b/arch/arm64/boot/dts/mediatek/mt8390-tungsten-smarc.dtsi
index 9f5a0ec563e8c..8256279c56299 100644
--- a/arch/arm64/boot/dts/mediatek/mt8390-tungsten-smarc.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8390-tungsten-smarc.dtsi
@@ -265,8 +265,6 @@ &disp_pwm0 {
 };
 
 &disp_dsi0 {
-	#address-cells = <1>;
-	#size-cells = <0>;
 	status = "okay";
 
 	ports {
@@ -1049,8 +1047,6 @@ &xhci2 {
 	ethernet@1 {
 		compatible = "usb424,7850";
 		reg = <1>;
-		#address-cells = <1>;
-		#size-cells = <0>;
 
 		mdio {
 			#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/nvidia/tegra234.dtsi b/arch/arm64/boot/dts/nvidia/tegra234.dtsi
index 8e0c51e496e20..820670dd60426 100644
--- a/arch/arm64/boot/dts/nvidia/tegra234.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra234.dtsi
@@ -5355,7 +5355,7 @@ cpus {
 		#size-cells = <0>;
 
 		cpu0_0: cpu@0 {
-			compatible = "arm,cortex-a78";
+			compatible = "arm,cortex-a78ae";
 			device_type = "cpu";
 			reg = <0x00000>;
 
@@ -5374,7 +5374,7 @@ cpu0_0: cpu@0 {
 		};
 
 		cpu0_1: cpu@100 {
-			compatible = "arm,cortex-a78";
+			compatible = "arm,cortex-a78ae";
 			device_type = "cpu";
 			reg = <0x00100>;
 
@@ -5393,7 +5393,7 @@ cpu0_1: cpu@100 {
 		};
 
 		cpu0_2: cpu@200 {
-			compatible = "arm,cortex-a78";
+			compatible = "arm,cortex-a78ae";
 			device_type = "cpu";
 			reg = <0x00200>;
 
@@ -5412,7 +5412,7 @@ cpu0_2: cpu@200 {
 		};
 
 		cpu0_3: cpu@300 {
-			compatible = "arm,cortex-a78";
+			compatible = "arm,cortex-a78ae";
 			device_type = "cpu";
 			reg = <0x00300>;
 
@@ -5431,7 +5431,7 @@ cpu0_3: cpu@300 {
 		};
 
 		cpu1_0: cpu@10000 {
-			compatible = "arm,cortex-a78";
+			compatible = "arm,cortex-a78ae";
 			device_type = "cpu";
 			reg = <0x10000>;
 
@@ -5450,7 +5450,7 @@ cpu1_0: cpu@10000 {
 		};
 
 		cpu1_1: cpu@10100 {
-			compatible = "arm,cortex-a78";
+			compatible = "arm,cortex-a78ae";
 			device_type = "cpu";
 			reg = <0x10100>;
 
@@ -5469,7 +5469,7 @@ cpu1_1: cpu@10100 {
 		};
 
 		cpu1_2: cpu@10200 {
-			compatible = "arm,cortex-a78";
+			compatible = "arm,cortex-a78ae";
 			device_type = "cpu";
 			reg = <0x10200>;
 
@@ -5488,7 +5488,7 @@ cpu1_2: cpu@10200 {
 		};
 
 		cpu1_3: cpu@10300 {
-			compatible = "arm,cortex-a78";
+			compatible = "arm,cortex-a78ae";
 			device_type = "cpu";
 			reg = <0x10300>;
 
@@ -5507,7 +5507,7 @@ cpu1_3: cpu@10300 {
 		};
 
 		cpu2_0: cpu@20000 {
-			compatible = "arm,cortex-a78";
+			compatible = "arm,cortex-a78ae";
 			device_type = "cpu";
 			reg = <0x20000>;
 
@@ -5526,7 +5526,7 @@ cpu2_0: cpu@20000 {
 		};
 
 		cpu2_1: cpu@20100 {
-			compatible = "arm,cortex-a78";
+			compatible = "arm,cortex-a78ae";
 			device_type = "cpu";
 			reg = <0x20100>;
 
@@ -5545,7 +5545,7 @@ cpu2_1: cpu@20100 {
 		};
 
 		cpu2_2: cpu@20200 {
-			compatible = "arm,cortex-a78";
+			compatible = "arm,cortex-a78ae";
 			device_type = "cpu";
 			reg = <0x20200>;
 
@@ -5564,7 +5564,7 @@ cpu2_2: cpu@20200 {
 		};
 
 		cpu2_3: cpu@20300 {
-			compatible = "arm,cortex-a78";
+			compatible = "arm,cortex-a78ae";
 			device_type = "cpu";
 			reg = <0x20300>;
 
diff --git a/arch/arm64/boot/dts/nvidia/tegra264-p3834.dtsi b/arch/arm64/boot/dts/nvidia/tegra264-p3834.dtsi
index 7e2c3e66c2abf..58cd81bc33d73 100644
--- a/arch/arm64/boot/dts/nvidia/tegra264-p3834.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra264-p3834.dtsi
@@ -9,6 +9,10 @@ aliases {
 	};
 
 	bus@0 {
+		dma-controller@8400000 {
+			status = "okay";
+		};
+
 		serial@c4e0000 {
 			status = "okay";
 		};
diff --git a/arch/arm64/boot/dts/nvidia/tegra264.dtsi b/arch/arm64/boot/dts/nvidia/tegra264.dtsi
index 2d8e7e37830ff..b141e8f884fed 100644
--- a/arch/arm64/boot/dts/nvidia/tegra264.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra264.dtsi
@@ -50,6 +50,76 @@ timer@8000000 {
 			status = "disabled";
 		};
 
+		hte_lic: hardware-timestamp@8380000 {
+			compatible = "nvidia,tegra264-gte-lic";
+			reg = <0x0 0x08380000 0x0 0x10000>;
+			interrupts = <GIC_SPI 0x00000268 IRQ_TYPE_LEVEL_HIGH>;
+			nvidia,int-threshold = <1>;
+			#timestamp-cells = <1>;
+			status = "disabled";
+		};
+
+		gpcdma: dma-controller@8400000 {
+			compatible = "nvidia,tegra264-gpcdma";
+			reg = <0x0 0x08400000 0x0 0x210000>;
+			interrupts = <GIC_SPI 584 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 585 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 586 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 587 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 588 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 589 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 590 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 591 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 592 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 593 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 594 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 595 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 596 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 597 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 598 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 599 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 600 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 601 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 602 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 603 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 604 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 605 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 606 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 607 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 608 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 609 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 610 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 611 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 612 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 613 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 614 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 615 IRQ_TYPE_LEVEL_HIGH>;
+			#dma-cells = <1>;
+			iommus = <&smmu1 0x00000800>;
+			iommu-map = <1 &smmu1 0x801 31>;
+			dma-coherent;
+			dma-channel-mask = <0xfffffffe>;
+			status = "disabled";
+		};
+
+		hsp_top: hsp@8800000 {
+			compatible = "nvidia,tegra264-hsp";
+			reg = <0x0 0x08800000 0x0 0xd0000>;
+			interrupts = <GIC_SPI 620 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 622 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 623 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 624 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 625 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 626 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 637 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 638 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 639 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "doorbell", "shared0", "shared1", "shared2",
+					  "shared3", "shared4", "shared5", "shared6",
+					  "shared7";
+			#mbox-cells = <2>;
+		};
+
 		aconnect@9000000 {
 			compatible = "nvidia,tegra264-aconnect",
 					"nvidia,tegra210-aconnect";
@@ -3207,66 +3277,16 @@ agic_page5: interrupt-controller@99b0000 {
 			};
 		};
 
-		gpcdma: dma-controller@8400000 {
-			compatible = "nvidia,tegra264-gpcdma", "nvidia,tegra186-gpcdma";
-			reg = <0x0 0x08400000 0x0 0x210000>;
-			interrupts = <GIC_SPI 584 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 585 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 586 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 587 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 588 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 589 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 590 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 591 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 592 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 593 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 594 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 595 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 596 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 597 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 598 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 599 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 600 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 601 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 602 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 603 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 604 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 605 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 606 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 607 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 608 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 609 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 610 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 611 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 612 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 613 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 614 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 615 IRQ_TYPE_LEVEL_HIGH>;
-			#dma-cells = <1>;
-			iommus = <&smmu1 0x00000800>;
-			dma-coherent;
-			dma-channel-mask = <0xfffffffe>;
+		hte_aon: hardware-timestamp@c2b0000 {
+			compatible = "nvidia,tegra264-gte-aon";
+			reg = <0x0 0x0c2b0000 0x0 0x10000>;
+			interrupts = <GIC_SPI 0x00000226 IRQ_TYPE_LEVEL_HIGH>;
+			nvidia,int-threshold = <1>;
+			#timestamp-cells = <1>;
+			nvidia,gpio-controller = <&gpio_aon>;
 			status = "disabled";
 		};
 
-		hsp_top: hsp@8800000 {
-			compatible = "nvidia,tegra264-hsp";
-			reg = <0x0 0x08800000 0x0 0xd0000>;
-			interrupts = <GIC_SPI 620 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 622 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 623 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 624 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 625 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 626 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 637 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 638 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 639 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "doorbell", "shared0", "shared1", "shared2",
-					  "shared3", "shared4", "shared5", "shared6",
-					  "shared7";
-			#mbox-cells = <2>;
-		};
-
 		rtc: rtc@c2c0000 {
 			compatible = "nvidia,tegra264-rtc", "nvidia,tegra20-rtc";
 			reg = <0x0 0x0c2c0000 0x0 0x10000>;
@@ -3336,6 +3356,11 @@ i2c3: i2c@c610000 {
 			status = "disabled";
 		};
 
+		pinmux_aon: pinmux@c7a2000 {
+			compatible = "nvidia,tegra264-pinmux-aon";
+			reg = <0x0 0x0c7a2000 0x0 0x2000>;
+		};
+
 		pmc: pmc@c800000 {
 			compatible = "nvidia,tegra264-pmc";
 			reg = <0x0 0x0c800000 0x0 0x100000>,
@@ -3393,7 +3418,7 @@ smmu1: iommu@5000000 {
 		cmdqv1: cmdqv@5200000 {
 			compatible = "nvidia,tegra264-cmdqv";
 			reg = <0x00 0x5200000 0x0 0x830000>;
-			interrupts = <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <GIC_SPI 19 IRQ_TYPE_EDGE_RISING>;
 			status = "disabled";
 		};
 
@@ -3413,7 +3438,7 @@ smmu2: iommu@6000000 {
 		cmdqv2: cmdqv@6200000 {
 			compatible = "nvidia,tegra264-cmdqv";
 			reg = <0x00 0x6200000 0x0 0x830000>;
-			interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <GIC_SPI 8 IRQ_TYPE_EDGE_RISING>;
 			status = "disabled";
 		};
 
@@ -3486,7 +3511,7 @@ smmu0: iommu@a000000 {
 		cmdqv0: cmdqv@a200000 {
 			compatible = "nvidia,tegra264-cmdqv";
 			reg = <0x00 0xa200000 0x0 0x830000>;
-			interrupts = <GIC_SPI 28 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <GIC_SPI 28 IRQ_TYPE_EDGE_RISING>;
 			status = "disabled";
 		};
 
@@ -3506,7 +3531,7 @@ smmu4: iommu@b000000 {
 		cmdqv4: cmdqv@b200000 {
 			compatible = "nvidia,tegra264-cmdqv";
 			reg = <0x00 0xb200000 0x0 0x830000>;
-			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <GIC_SPI 37 IRQ_TYPE_EDGE_RISING>;
 			status = "disabled";
 		};
 
@@ -3542,6 +3567,11 @@ pci@c000000 {
 			status = "disabled";
 		};
 
+		pinmux_main: pinmux@c281000 {
+			compatible = "nvidia,tegra264-pinmux-main";
+			reg = <0x00 0x0c281000 0x0 0xc000>;
+		};
+
 		gpio_main: gpio@c300000 {
 			compatible = "nvidia,tegra264-gpio";
 			reg = <0x00 0x0c300000 0x0 0x4000>,
@@ -3807,6 +3837,69 @@ its: msi-controller@40000 {
 		};
 	};
 
+	/* VISION MMIO */
+	bus@8180000000 {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		ranges = <0x000 0x00000000 0x81 0x80000000 0x00 0x10000000>, /* MMIO (256 MiB) */
+			 <0x100 0x00000000 0x00 0x20000000 0x00 0x40000000>, /* non-prefetchable memory (32-bit) */
+			 <0x200 0x00000000 0xa8 0x80000000 0x57 0x80000000>; /* I/O, ECAM, prefetchable memory (64-bit) */
+
+		host1x@1200000 {
+			compatible = "nvidia,tegra264-host1x";
+			reg = <0x0 0x1200000 0x0 0x10000>,
+			      <0x0 0x1210000 0x0 0x10000>,
+			      <0x0 0x1240000 0x0 0x10000>;
+			reg-names = "common", "hypervisor", "vm";
+			interrupts = <GIC_SPI 327 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 328 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 329 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 330 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 331 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 332 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 333 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 334 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 325 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "syncpt0", "syncpt1", "syncpt2", "syncpt3", "syncpt4",
+					  "syncpt5", "syncpt6", "syncpt7", "host1x";
+			clocks = <&bpmp TEGRA264_CLK_HOST1X>;
+			clock-names = "host1x";
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+
+			ranges = <0x000 0x00000000 0x00 0x08000000 0x00 0x01000000>,
+				 <0x000 0x02800000 0x00 0x0a800000 0x00 0x00800000>;
+
+			interconnects = <&mc TEGRA264_MEMORY_CLIENT_HOST1XR &emc>;
+			interconnect-names = "dma-mem";
+			iommus = <&smmu1 TEGRA264_SID_HOST1X>;
+			dma-coherent;
+
+			/* Context isolation domains */
+			iommu-map = <0 &smmu1 (TEGRA264_SID_VIC + 1) 16>;
+
+			vic@50000 {
+				compatible = "nvidia,tegra264-vic";
+				reg = <0x0 0x50000 0x0 0x40000>;
+				interrupts = <GIC_SPI 468 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&bpmp TEGRA264_CLK_VIC>;
+				clock-names = "vic";
+				resets = <&bpmp TEGRA264_RESET_VIC>;
+				reset-names = "vic";
+				power-domains = <&bpmp TEGRA264_POWER_DOMAIN_VIC>;
+				interconnects = <&mc TEGRA264_MEMORY_CLIENT_VICR &emc>,
+						<&mc TEGRA264_MEMORY_CLIENT_VICW &emc>;
+				interconnect-names = "dma-mem", "write";
+
+				iommus = <&smmu1 TEGRA264_SID_VIC>;
+				dma-coherent;
+			};
+		};
+	};
+
 	/* DISP_USB MMIO */
 	bus@8800000000 {
 		compatible = "simple-bus";
@@ -3831,7 +3924,7 @@ smmu3: iommu@6000000 {
 		cmdqv3: cmdqv@6200000 {
 			compatible = "nvidia,tegra264-cmdqv";
 			reg = <0x00 0x6200000 0x0 0x830000>;
-			interrupts = <GIC_SPI 232 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <GIC_SPI 232 IRQ_TYPE_EDGE_RISING>;
 			status = "disabled";
 		};
 
@@ -3862,6 +3955,11 @@ bus@a800000000 {
 			 <0x00 0x20000000 0x00 0x20000000 0x00 0x60000000>, /* non-prefetchable memory (32-bit, 1536 GiB) */
 			 <0xa8 0x80000000 0xa8 0x80000000 0x57 0x80000000>; /* I/O, ECAM, prefetchable memory (64-bit) */
 
+		pinmux_uphy: pinmux@82e0000 {
+			compatible = "nvidia,tegra264-pinmux-uphy";
+			reg = <0x00 0x082e0000 0x0 0x4000>;
+		};
+
 		gpio_uphy: gpio@8300000 {
 			compatible = "nvidia,tegra264-gpio-uphy";
 			reg = <0x00 0x08300000 0x0 0x2000>,
@@ -4051,6 +4149,28 @@ pci@8480000 {
 		};
 	};
 
+	bpmp: bpmp {
+		compatible = "nvidia,tegra264-bpmp", "nvidia,tegra186-bpmp";
+		mboxes = <&hsp_top TEGRA_HSP_MBOX_TYPE_DB
+				   TEGRA_HSP_DB_MASTER_BPMP>;
+		memory-region = <&shmem_bpmp>;
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+		#power-domain-cells = <1>;
+
+		i2c {
+			compatible = "nvidia,tegra186-bpmp-i2c";
+			nvidia,bpmp-bus-id = <5>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		thermal {
+			compatible = "nvidia,tegra186-bpmp-thermal";
+			#thermal-sensor-cells = <1>;
+		};
+	};
+
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -4068,9 +4188,10 @@ cpu0: cpu@0 {
 			d-cache-size = <65536>;
 			d-cache-line-size = <64>;
 			d-cache-sets = <256>;
+			next-level-cache = <&l2c0>;
 		};
 
-		cpu1: cpu@1 {
+		cpu1: cpu@10000 {
 			compatible = "arm,neoverse-v3ae";
 			device_type = "cpu";
 			reg = <0x10000>;
@@ -4083,29 +4204,327 @@ cpu1: cpu@1 {
 			d-cache-size = <65536>;
 			d-cache-line-size = <64>;
 			d-cache-sets = <256>;
+			next-level-cache = <&l2c1>;
 		};
-	};
 
-	bpmp: bpmp {
-		compatible = "nvidia,tegra264-bpmp", "nvidia,tegra186-bpmp";
-		mboxes = <&hsp_top TEGRA_HSP_MBOX_TYPE_DB
-				   TEGRA_HSP_DB_MASTER_BPMP>;
-		memory-region = <&shmem_bpmp>;
-		#clock-cells = <1>;
-		#reset-cells = <1>;
-		#power-domain-cells = <1>;
+		cpu2: cpu@20000 {
+			compatible = "arm,neoverse-v3ae";
+			device_type = "cpu";
+			reg = <0x20000>;
 
-		i2c {
-			compatible = "nvidia,tegra186-bpmp-i2c";
-			nvidia,bpmp-bus-id = <5>;
-			#address-cells = <1>;
-			#size-cells = <0>;
+			enable-method = "psci";
+
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&l2c2>;
 		};
 
-		thermal {
-			compatible = "nvidia,tegra186-bpmp-thermal";
-			#thermal-sensor-cells = <1>;
+		cpu3: cpu@30000 {
+			compatible = "arm,neoverse-v3ae";
+			device_type = "cpu";
+			reg = <0x30000>;
+
+			enable-method = "psci";
+
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&l2c3>;
+		};
+
+		cpu4: cpu@40000 {
+			compatible = "arm,neoverse-v3ae";
+			device_type = "cpu";
+			reg = <0x40000>;
+
+			enable-method = "psci";
+
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&l2c4>;
+		};
+
+		cpu5: cpu@50000 {
+			compatible = "arm,neoverse-v3ae";
+			device_type = "cpu";
+			reg = <0x50000>;
+
+			enable-method = "psci";
+
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&l2c5>;
 		};
+
+		cpu6: cpu@60000 {
+			compatible = "arm,neoverse-v3ae";
+			device_type = "cpu";
+			reg = <0x60000>;
+
+			enable-method = "psci";
+
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&l2c6>;
+		};
+
+		cpu7: cpu@70000 {
+			compatible = "arm,neoverse-v3ae";
+			device_type = "cpu";
+			reg = <0x70000>;
+
+			enable-method = "psci";
+
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&l2c7>;
+		};
+
+		cpu8: cpu@80000 {
+			compatible = "arm,neoverse-v3ae";
+			device_type = "cpu";
+			reg = <0x80000>;
+
+			enable-method = "psci";
+
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&l2c8>;
+		};
+
+		cpu9: cpu@90000 {
+			compatible = "arm,neoverse-v3ae";
+			device_type = "cpu";
+			reg = <0x90000>;
+
+			enable-method = "psci";
+
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&l2c9>;
+		};
+
+		cpu10: cpu@a0000 {
+			compatible = "arm,neoverse-v3ae";
+			device_type = "cpu";
+			reg = <0xa0000>;
+
+			enable-method = "psci";
+
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&l2c10>;
+		};
+
+		cpu11: cpu@b0000 {
+			compatible = "arm,neoverse-v3ae";
+			device_type = "cpu";
+			reg = <0xb0000>;
+
+			enable-method = "psci";
+
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&l2c11>;
+		};
+
+		cpu12: cpu@c0000 {
+			compatible = "arm,neoverse-v3ae";
+			device_type = "cpu";
+			reg = <0xc0000>;
+
+			enable-method = "psci";
+
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&l2c12>;
+		};
+
+		cpu13: cpu@d0000 {
+			compatible = "arm,neoverse-v3ae";
+			device_type = "cpu";
+			reg = <0xd0000>;
+
+			enable-method = "psci";
+
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&l2c13>;
+		};
+
+		l2c0: l2-cache-0 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <1048576>;
+			cache-line-size = <64>;
+			cache-sets = <2048>;
+			cache-unified;
+		};
+
+		l2c1: l2-cache-1 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <1048576>;
+			cache-line-size = <64>;
+			cache-sets = <2048>;
+			cache-unified;
+		};
+
+		l2c2: l2-cache-2 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <1048576>;
+			cache-line-size = <64>;
+			cache-sets = <2048>;
+			cache-unified;
+		};
+
+		l2c3: l2-cache-3 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <1048576>;
+			cache-line-size = <64>;
+			cache-sets = <2048>;
+			cache-unified;
+		};
+
+		l2c4: l2-cache-4 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <1048576>;
+			cache-line-size = <64>;
+			cache-sets = <2048>;
+			cache-unified;
+		};
+
+		l2c5: l2-cache-5 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <1048576>;
+			cache-line-size = <64>;
+			cache-sets = <2048>;
+			cache-unified;
+		};
+
+		l2c6: l2-cache-6 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <1048576>;
+			cache-line-size = <64>;
+			cache-sets = <2048>;
+			cache-unified;
+		};
+
+		l2c7: l2-cache-7 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <1048576>;
+			cache-line-size = <64>;
+			cache-sets = <2048>;
+			cache-unified;
+		};
+
+		l2c8: l2-cache-8 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <1048576>;
+			cache-line-size = <64>;
+			cache-sets = <2048>;
+			cache-unified;
+		};
+
+		l2c9: l2-cache-9 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <1048576>;
+			cache-line-size = <64>;
+			cache-sets = <2048>;
+			cache-unified;
+		};
+
+		l2c10: l2-cache-10 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <1048576>;
+			cache-line-size = <64>;
+			cache-sets = <2048>;
+			cache-unified;
+		};
+
+		l2c11: l2-cache-11 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <1048576>;
+			cache-line-size = <64>;
+			cache-sets = <2048>;
+			cache-unified;
+		};
+
+		l2c12: l2-cache-12 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <1048576>;
+			cache-line-size = <64>;
+			cache-sets = <2048>;
+			cache-unified;
+		};
+
+		l2c13: l2-cache-13 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <1048576>;
+			cache-line-size = <64>;
+			cache-sets = <2048>;
+			cache-unified;
+		};
+
 	};
 
 	pmu {
diff --git a/arch/arm64/boot/dts/qcom/Makefile b/arch/arm64/boot/dts/qcom/Makefile
index 6f33c4e2f09c3..e05414290d8e5 100644
--- a/arch/arm64/boot/dts/qcom/Makefile
+++ b/arch/arm64/boot/dts/qcom/Makefile
@@ -14,6 +14,7 @@ dtb-$(CONFIG_ARCH_QCOM)	+= apq8094-sony-xperia-kitakami-karin_windy.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= apq8096-db820c.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= apq8096sg-db820c.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= apq8096-ifc6640.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= eliza-cqs-evk.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= eliza-mtp.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= glymur-crd.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= hamoa-iot-evk.dtb
@@ -184,12 +185,14 @@ dtb-$(CONFIG_ARCH_QCOM)	+= qcs6490-rb3gen2-industrial-mezzanine.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs6490-rb3gen2-vision-mezzanine.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs6490-thundercomm-minipc-g1iot.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs6490-thundercomm-rubikpi3.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= qcs6490-vicharak-axon-mini.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs8300-ride.dtb
 
 qcs8300-ride-el2-dtbs := qcs8300-ride.dtb monaco-el2.dtbo
 
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs8300-ride-el2.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs8550-aim300-aiot.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= qcs8550-rb5gen2.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs9100-ride.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs9100-ride-r3.dtb
 
@@ -216,8 +219,6 @@ dtb-$(CONFIG_ARCH_QCOM)	+= qru1000-idp.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sa8155p-adp.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sa8295p-adp.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sa8540p-ride.dtb
-dtb-$(CONFIG_ARCH_QCOM)	+= sa8775p-ride.dtb
-dtb-$(CONFIG_ARCH_QCOM)	+= sa8775p-ride-r3.dtb
 sc7180-acer-aspire1-el2-dtbs	:= sc7180-acer-aspire1.dtb sc7180-el2.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= sc7180-acer-aspire1.dtb sc7180-acer-aspire1-el2.dtb
 sc7180-ecs-liva-qc710-el2-dtbs	:= sc7180-ecs-liva-qc710.dtb sc7180-el2.dtbo
@@ -334,6 +335,9 @@ dtb-$(CONFIG_ARCH_QCOM)	+= sdm850-huawei-matebook-e-2019.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sdm850-lenovo-yoga-c630.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sdm850-samsung-w737.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sdx75-idp.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= shikra-cqm-evk.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= shikra-cqs-evk.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= shikra-iqs-evk.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm4250-oneplus-billie2.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm4450-qrd.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm6115-fxtec-pro1x.dtb
@@ -348,7 +352,10 @@ dtb-$(CONFIG_ARCH_QCOM)	+= sm7125-xiaomi-curtana.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm7125-xiaomi-joyeuse.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm7225-fairphone-fp4.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm7325-motorola-dubai.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= sm7325-motorola-dubai-csot.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= sm7325-motorola-dubai-tianma.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm7325-nothing-spacewar.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= sm7325-xiaomi-taoyao.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm8150-hdk.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm8150-microsoft-surface-duo.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm8150-mtp.dtb
@@ -401,6 +408,10 @@ dtb-$(CONFIG_ARCH_QCOM)	+= sm8650-qrd.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm8750-mtp.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm8750-qrd.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= talos-evk.dtb
+
+talos-evk-el2-dtbs	:= talos-evk.dtb talos-el2.dtbo
+
+dtb-$(CONFIG_ARCH_QCOM)	+= talos-evk-el2.dtb
 talos-evk-usb1-peripheral-dtbs := talos-evk.dtb talos-evk-usb1-peripheral.dtbo
 dtb-$(CONFIG_ARCH_QCOM) += talos-evk-usb1-peripheral.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= talos-evk-camera-imx577.dtbo
@@ -431,6 +442,8 @@ x1e80100-hp-elitebook-ultra-g1q-el2-dtbs := x1e80100-hp-elitebook-ultra-g1q.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= x1e80100-hp-elitebook-ultra-g1q.dtb x1e80100-hp-elitebook-ultra-g1q-el2.dtb
 x1e80100-hp-omnibook-x14-el2-dtbs	:= x1e80100-hp-omnibook-x14.dtb x1-el2.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= x1e80100-hp-omnibook-x14.dtb x1e80100-hp-omnibook-x14-el2.dtb
+x1e80100-honor-magicbook-art-14-el2-dtbs := x1e80100-honor-magicbook-art-14.dtb x1-el2.dtbo
+dtb-$(CONFIG_ARCH_QCOM) += x1e80100-honor-magicbook-art-14.dtb x1e80100-honor-magicbook-art-14-el2.dtb
 x1e80100-lenovo-yoga-slim7x-el2-dtbs	:= x1e80100-lenovo-yoga-slim7x.dtb x1-el2.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= x1e80100-lenovo-yoga-slim7x.dtb x1e80100-lenovo-yoga-slim7x-el2.dtb
 x1e80100-medion-sprchrgd-14-s1-el2-dtbs	:= x1e80100-medion-sprchrgd-14-s1.dtb x1-el2.dtbo
@@ -457,3 +470,5 @@ x1p42100-lenovo-thinkbook-16-el2-dtbs	:= x1p42100-lenovo-thinkbook-16.dtb x1-el2
 dtb-$(CONFIG_ARCH_QCOM)	+= x1p42100-lenovo-thinkbook-16.dtb x1p42100-lenovo-thinkbook-16-el2.dtb
 x1p64100-microsoft-denali-el2-dtbs	:= x1p64100-microsoft-denali.dtb x1-el2.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= x1p64100-microsoft-denali.dtb x1p64100-microsoft-denali-el2.dtb
+x1p42100-microsoft-sp12in-el2-dtbs	:= x1p42100-microsoft-sp12in.dtb x1-el2.dtbo
+dtb-$(CONFIG_ARCH_QCOM)	+= x1p42100-microsoft-sp12in.dtb x1p42100-microsoft-sp12in-el2.dtb
diff --git a/arch/arm64/boot/dts/qcom/apq8016-sbc-usb-host.dtso b/arch/arm64/boot/dts/qcom/apq8016-sbc-usb-host.dtso
index a82c26b7eae86..9872d5cb83bda 100644
--- a/arch/arm64/boot/dts/qcom/apq8016-sbc-usb-host.dtso
+++ b/arch/arm64/boot/dts/qcom/apq8016-sbc-usb-host.dtso
@@ -4,5 +4,5 @@
 /plugin/;
 
 &usb {
-         dr_mode = "host";
+	dr_mode = "host";
 };
diff --git a/arch/arm64/boot/dts/qcom/eliza-cqs-evk.dts b/arch/arm64/boot/dts/qcom/eliza-cqs-evk.dts
new file mode 100644
index 0000000000000..43d428a4ed2df
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/eliza-cqs-evk.dts
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include "eliza-cqs-som.dtsi"
+#include "eliza-evk.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. Eliza CQS EVK";
+	compatible = "qcom,eliza-cqs-evk", "qcom,eliza-cqs-som", "qcom,eliza";
+};
diff --git a/arch/arm64/boot/dts/qcom/eliza-cqs-som.dtsi b/arch/arm64/boot/dts/qcom/eliza-cqs-som.dtsi
new file mode 100644
index 0000000000000..318ae3c223515
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/eliza-cqs-som.dtsi
@@ -0,0 +1,399 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+
+#include "eliza.dtsi"
+#include "pm7550ba-eliza.dtsi"
+#include "pm7550-eliza.dtsi"
+#include "pm8550vs-d-eliza.dtsi"
+#include "pm8550vs-g-eliza.dtsi"
+#include "pmr735d-eliza.dtsi"
+#include "pmk8550-eliza.dtsi"
+
+/ {
+	clocks {
+		xo_board: xo-board {
+			compatible = "fixed-clock";
+			clock-frequency = <76800000>;
+			#clock-cells = <0>;
+		};
+
+		sleep_clk: sleep-clk {
+			compatible = "fixed-clock";
+			clock-frequency = <32764>;
+			#clock-cells = <0>;
+		};
+
+		bi_tcxo_div2: bi-tcxo-div2-clk {
+			compatible = "fixed-factor-clock";
+			#clock-cells = <0>;
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+			clock-mult = <1>;
+			clock-div = <2>;
+		};
+
+		bi_tcxo_ao_div2: bi-tcxo-ao-div2-clk {
+			compatible = "fixed-factor-clock";
+			#clock-cells = <0>;
+
+			clocks = <&rpmhcc RPMH_CXO_CLK_A>;
+			clock-mult = <1>;
+			clock-div = <2>;
+		};
+	};
+
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pm7550-rpmh-regulators";
+
+		vdd-l1-supply = <&vreg_s3b>;
+		vdd-l2-l3-supply = <&vreg_s3b>;
+		vdd-l4-l5-supply = <&vreg_s2b>;
+		vdd-l6-supply = <&vreg_s2b>;
+		vdd-l7-supply = <&vreg_s1b>;
+		vdd-l8-supply = <&vreg_s1b>;
+		vdd-l9-l10-supply = <&vreg_s1b>;
+		vdd-l11-supply = <&vreg_s1b>;
+		vdd-l12-l14-supply = <&vreg_bob>;
+		vdd-l13-l16-supply = <&vreg_bob>;
+		vdd-l15-l17-l18-l19-l20-l21-l22-l23-supply = <&vreg_bob>;
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s3-supply = <&vph_pwr>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s6-supply = <&vph_pwr>;
+
+		vdd-bob-supply = <&vph_pwr>;
+
+		qcom,pmic-id = "b";
+
+		vreg_s1b: smps1 {
+			regulator-name = "vreg_s1b";
+			regulator-min-microvolt = <1850000>;
+			regulator-max-microvolt = <2040000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s2b: smps2 {
+			regulator-name = "vreg_s2b";
+			regulator-min-microvolt = <375000>;
+			regulator-max-microvolt = <2744000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s3b: smps3 {
+			regulator-name = "vreg_s3b";
+			regulator-min-microvolt = <375000>;
+			regulator-max-microvolt = <2744000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s4b: smps4 {
+			regulator-name = "vreg_s4b";
+			regulator-min-microvolt = <2156000>;
+			regulator-max-microvolt = <2400000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2b: ldo2 {
+			regulator-name = "vreg_l2b";
+			regulator-min-microvolt = <720000>;
+			regulator-max-microvolt = <950000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3b: ldo3 {
+			regulator-name = "vreg_l3b";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4b: ldo4 {
+			regulator-name = "vreg_l4b";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6b: ldo6 {
+			regulator-name = "vreg_l6b";
+			regulator-min-microvolt = <866000>;
+			regulator-max-microvolt = <958000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b: ldo7 {
+			regulator-name = "vreg_l7b";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b: ldo8 {
+			regulator-name = "vreg_l8b";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b: ldo9 {
+			regulator-name = "vreg_l9b";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10b: ldo10 {
+			regulator-name = "vreg_l10b";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11b: ldo11 {
+			regulator-name = "vreg_l11b";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b: ldo12 {
+			regulator-name = "vreg_l12b";
+			regulator-min-microvolt = <2400000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13b: ldo13 {
+			regulator-name = "vreg_l13b";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l14b: ldo14 {
+			regulator-name = "vreg_l14b";
+			regulator-min-microvolt = <3300000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b: ldo15 {
+			regulator-name = "vreg_l15b";
+			regulator-min-microvolt = <3300000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l16b: ldo16 {
+			regulator-name = "vreg_l16b";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b: ldo17 {
+			regulator-name = "vreg_l17b";
+			regulator-min-microvolt = <3104000>;
+			regulator-max-microvolt = <3104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l18b: ldo18 {
+			regulator-name = "vreg_l18b";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l19b: ldo19 {
+			regulator-name = "vreg_l19b";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l20b: ldo20 {
+			regulator-name = "vreg_l20b";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l21b: ldo21 {
+			regulator-name = "vreg_l21b";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l22b: ldo22 {
+			regulator-name = "vreg_l22b";
+			regulator-min-microvolt = <3200000>;
+			regulator-max-microvolt = <3200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l23b: ldo23 {
+			regulator-name = "vreg_l23b";
+			regulator-min-microvolt = <1650000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob: bob {
+			regulator-name = "vreg_bob";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+
+		vdd-l1-supply = <&vreg_s1b>;
+
+		qcom,pmic-id = "d";
+
+		vreg_l1d: ldo1 {
+			regulator-name = "vreg_l1d";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+
+		vdd-l1-supply = <&vreg_s2b>;
+		vdd-l3-supply = <&vreg_s2b>;
+
+		qcom,pmic-id = "g";
+
+		vreg_l1g: ldo1 {
+			regulator-name = "vreg_l1g";
+			regulator-min-microvolt = <1150000>;
+			regulator-max-microvolt = <1260000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3g: ldo3 {
+			regulator-name = "vreg_l3g";
+			regulator-min-microvolt = <1150000>;
+			regulator-max-microvolt = <1260000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+	};
+
+	regulators-3 {
+		compatible = "qcom,pmr735d-rpmh-regulators";
+
+		vdd-l1-l2-l5-supply = <&vreg_s3b>;
+		vdd-l3-l4-supply = <&vreg_s2b>;
+		vdd-l6-supply = <&vreg_s1b>;
+		vdd-l7-supply = <&vreg_s3b>;
+
+		qcom,pmic-id = "k";
+
+		vreg_l1k: ldo1 {
+			regulator-name = "vreg_l1k";
+			regulator-min-microvolt = <488000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2k: ldo2 {
+			regulator-name = "vreg_l2k";
+			regulator-min-microvolt = <920000>;
+			regulator-max-microvolt = <969000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3k: ldo3 {
+			regulator-name = "vreg_l3k";
+			regulator-min-microvolt = <1080000>;
+			regulator-max-microvolt = <1350000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4k: ldo4 {
+			regulator-name = "vreg_l4k";
+			regulator-min-microvolt = <960000>;
+			regulator-max-microvolt = <1980000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5k: ldo5 {
+			regulator-name = "vreg_l5k";
+			regulator-min-microvolt = <866000>;
+			regulator-max-microvolt = <931000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6k: ldo6 {
+			regulator-name = "vreg_l6k";
+			regulator-min-microvolt = <1100000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7k: ldo7 {
+			regulator-name = "vreg_l7k";
+			regulator-min-microvolt = <720000>;
+			regulator-max-microvolt = <958000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/eliza/adsp.mbn",
+			"qcom/eliza/adsp_dtb.mbn";
+
+	status = "okay";
+};
+
+&sdhc_1 {
+	vmmc-supply = <&vreg_l12b>;
+	vqmmc-supply = <&vreg_l1d>;
+	pinctrl-0 = <&sdc1_default>;
+	pinctrl-1 = <&sdc1_sleep>;
+	pinctrl-names = "default", "sleep";
+	mmc-hs400-1_8v;
+	mmc-hs200-1_8v;
+	non-removable;
+	supports-cqe;
+	no-sdio;
+	no-sd;
+
+	status = "okay";
+};
+
+&tlmm {
+	gpio-reserved-ranges = <20 4>,   /* NFC SPI */
+			       <111 2>,  /* WCN UART1 */
+			       <118 1>;  /* NFC Secure I/O */
+};
diff --git a/arch/arm64/boot/dts/qcom/eliza-evk.dtsi b/arch/arm64/boot/dts/qcom/eliza-evk.dtsi
new file mode 100644
index 0000000000000..e47b24f8b8277
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/eliza-evk.dtsi
@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/ {
+	aliases {
+		serial0 = &uart13;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+};
+
+&uart13 {
+	compatible = "qcom,geni-debug-uart";
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/eliza-mtp.dts b/arch/arm64/boot/dts/qcom/eliza-mtp.dts
index 1374afd9d14e3..5abc23b9b676a 100644
--- a/arch/arm64/boot/dts/qcom/eliza-mtp.dts
+++ b/arch/arm64/boot/dts/qcom/eliza-mtp.dts
@@ -11,6 +11,11 @@
 #include "eliza.dtsi"
 
 #include "pm7550ba-eliza.dtsi"
+#include "pm7550-eliza.dtsi"
+#include "pm8550vs-d-eliza.dtsi"
+#include "pm8550vs-g-eliza.dtsi"
+#include "pmr735d-eliza.dtsi"
+#include "pmk8550-eliza.dtsi"
 
 / {
 	model = "Qualcomm Technologies, Inc. Eliza MTP";
@@ -479,6 +484,30 @@ &remoteproc_adsp {
 	status = "okay";
 };
 
+&spi8 {
+	status = "okay";
+
+	touchscreen@0 {
+		compatible = "goodix,gt9916";
+		reg = <0>;
+
+		interrupt-parent = <&tlmm>;
+		interrupts = <13 IRQ_TYPE_LEVEL_LOW>;
+
+		reset-gpios = <&tlmm 16 GPIO_ACTIVE_LOW>;
+
+		avdd-supply = <&vreg_l22b>;
+
+		spi-max-frequency = <1000000>;
+
+		touchscreen-size-x = <1080>;
+		touchscreen-size-y = <2400>;
+
+		pinctrl-0 = <&ts_irq>, <&ts_reset>;
+		pinctrl-names = "default";
+	};
+};
+
 &tlmm {
 	gpio-reserved-ranges = <20 4>,   /* NFC SPI */
 			       <111 2>,  /* WCN UART1 */
@@ -504,6 +533,21 @@ mdp_vsync: mdp-vsync-state {
 		drive-strength = <2>;
 		bias-pull-down;
 	};
+
+	ts_irq: ts-irq-state {
+		pins = "gpio13";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-up;
+		output-disable;
+	};
+
+	ts_reset: ts-reset-state {
+		pins = "gpio16";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-up;
+	};
 };
 
 &uart13 {
diff --git a/arch/arm64/boot/dts/qcom/eliza.dtsi b/arch/arm64/boot/dts/qcom/eliza.dtsi
index 977de44b816e4..357e8066c1768 100644
--- a/arch/arm64/boot/dts/qcom/eliza.dtsi
+++ b/arch/arm64/boot/dts/qcom/eliza.dtsi
@@ -1232,8 +1232,6 @@ qupv3_1: geniqup@ac0000 {
 			#size-cells = <2>;
 			ranges;
 
-			status = "disabled";
-
 			i2c0: i2c@a80000 {
 				compatible = "qcom,geni-i2c";
 				reg = <0x0 0x00a80000 0x0 0x4000>;
@@ -2696,7 +2694,7 @@ lpass_ag_noc: interconnect@7e40000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,eliza-pdc", "qcom,pdc";
-			reg = <0x0 0x0b220000 0x0 0x40000>,
+			reg = <0x0 0x0b220000 0x0 0x10000>,
 			      <0x0 0x174000f0 0x0 0x64>;
 
 			qcom,pdc-ranges = <0 480 8>, <8 719 1>, <9 718 1>,
@@ -3616,6 +3614,102 @@ cpufreq_hw: cpufreq@17d91000 {
 			#clock-cells = <1>;
 		};
 
+		pmu@24091000 {
+			compatible = "qcom,eliza-llcc-bwmon", "qcom,sc7280-llcc-bwmon";
+			reg = <0x0 0x24091000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH>;
+
+			interconnects = <&mc_virt MASTER_LLCC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+
+			operating-points-v2 = <&llcc_bwmon_opp_table>;
+
+			llcc_bwmon_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-0 {
+					opp-peak-kBps = <2188800>;
+				};
+
+				opp-1 {
+					opp-peak-kBps = <5414400>;
+				};
+
+				opp-2 {
+					opp-peak-kBps = <6220800>;
+				};
+
+				opp-3 {
+					opp-peak-kBps = <6835200>;
+				};
+
+				opp-4 {
+					opp-peak-kBps = <8371200>;
+				};
+
+				opp-5 {
+					opp-peak-kBps = <10944000>;
+				};
+
+				opp-6 {
+					opp-peak-kBps = <12748800>;
+				};
+
+				opp-7 {
+					opp-peak-kBps = <14745600>;
+				};
+
+				opp-8 {
+					opp-peak-kBps = <16896000>;
+				};
+			};
+		};
+
+		pmu@240b7400 {
+			compatible = "qcom,eliza-cpu-bwmon", "qcom,sdm845-bwmon";
+			reg = <0x0 0x240b7400 0x0 0x600>;
+
+			interrupts = <GIC_SPI 581 IRQ_TYPE_LEVEL_HIGH>;
+
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &gem_noc SLAVE_LLCC QCOM_ICC_TAG_ACTIVE_ONLY>;
+
+			operating-points-v2 = <&cpu_bwmon_opp_table>;
+
+			cpu_bwmon_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-0 {
+					opp-peak-kBps = <307200>;
+				};
+
+				opp-1 {
+					opp-peak-kBps = <5600000>;
+				};
+
+				opp-2 {
+					opp-peak-kBps = <8528000>;
+				};
+
+				opp-3 {
+					opp-peak-kBps = <11120000>;
+				};
+
+				opp-4 {
+					opp-peak-kBps = <14000000>;
+				};
+
+				opp-5 {
+					opp-peak-kBps = <14928000>;
+				};
+
+				opp-6 {
+					opp-peak-kBps = <17056000>;
+				};
+			};
+		};
+
 		gem_noc: interconnect@24100000 {
 			compatible = "qcom,eliza-gem-noc";
 			reg = <0x0 0x24100000 0x0 0x163080>;
diff --git a/arch/arm64/boot/dts/qcom/glymur-crd.dts b/arch/arm64/boot/dts/qcom/glymur-crd.dts
index c98dfb3941fa3..6125617de82a6 100644
--- a/arch/arm64/boot/dts/qcom/glymur-crd.dts
+++ b/arch/arm64/boot/dts/qcom/glymur-crd.dts
@@ -8,198 +8,9 @@
 #include "glymur.dtsi"
 #include "glymur-crd.dtsi"
 
-#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
-
 / {
 	model = "Qualcomm Technologies, Inc. Glymur CRD";
 	compatible = "qcom,glymur-crd", "qcom,glymur";
-
-	pmic-glink {
-		compatible = "qcom,glymur-pmic-glink",
-			     "qcom,pmic-glink";
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		connector@0 {
-			compatible = "usb-c-connector";
-			reg = <0>;
-			power-role = "dual";
-			data-role = "dual";
-
-			ports {
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				port@0 {
-					reg = <0>;
-
-					pmic_glink_hs_in: endpoint {
-						remote-endpoint = <&usb_0_dwc3_hs>;
-					};
-				};
-
-				port@1 {
-					reg = <1>;
-
-					pmic_glink_ss_in: endpoint {
-						remote-endpoint = <&usb_0_qmpphy_out>;
-					};
-				};
-			};
-		};
-
-		connector@1 {
-			compatible = "usb-c-connector";
-			reg = <1>;
-			power-role = "dual";
-			data-role = "dual";
-
-			ports {
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				port@0 {
-					reg = <0>;
-
-					pmic_glink_hs_in1: endpoint {
-						remote-endpoint = <&usb_1_dwc3_hs>;
-					};
-				};
-
-				port@1 {
-					reg = <1>;
-
-					pmic_glink_ss_in1: endpoint {
-						remote-endpoint = <&usb_1_qmpphy_out>;
-					};
-				};
-			};
-		};
-	};
-
-	vreg_edp_3p3: regulator-edp-3p3 {
-		compatible = "regulator-fixed";
-
-		regulator-name = "VREG_EDP_3P3";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-
-		gpio = <&tlmm 70 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-
-		pinctrl-0 = <&edp_reg_en>;
-		pinctrl-names = "default";
-
-		regulator-boot-on;
-	};
-
-	vreg_misc_3p3: regulator-misc-3p3 {
-		compatible = "regulator-fixed";
-
-		regulator-name = "VREG_MISC_3P3";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-
-		gpio = <&pmh0110_f_e0_gpios 6 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-
-		pinctrl-0 = <&misc_3p3_reg_en>;
-		pinctrl-names = "default";
-
-		regulator-boot-on;
-	};
-};
-
-&i2c0 {
-	clock-frequency = <400000>;
-
-	status = "okay";
-
-	touchpad@2c {
-		compatible = "hid-over-i2c";
-		reg = <0x2c>;
-
-		hid-descr-addr = <0x20>;
-		interrupts-extended = <&tlmm 3 IRQ_TYPE_LEVEL_LOW>;
-
-		vdd-supply = <&vreg_misc_3p3>;
-		vddl-supply = <&vreg_l15b_e0_1p8>;
-
-		pinctrl-0 = <&tpad_default>;
-		pinctrl-names = "default";
-
-		wakeup-source;
-	};
-
-	keyboard@3a {
-		compatible = "hid-over-i2c";
-		reg = <0x3a>;
-
-		hid-descr-addr = <0x1>;
-		interrupts-extended = <&tlmm 67 IRQ_TYPE_LEVEL_LOW>;
-
-		vdd-supply = <&vreg_misc_3p3>;
-		vddl-supply = <&vreg_l15b_e0_1p8>;
-
-		pinctrl-0 = <&kybd_default>;
-		pinctrl-names = "default";
-
-		wakeup-source;
-	};
-};
-
-&i2c8 {
-	clock-frequency = <400000>;
-
-	status = "okay";
-
-	touchscreen@38 {
-		compatible = "hid-over-i2c";
-		reg = <0x38>;
-
-		hid-descr-addr = <0x1>;
-		interrupts-extended = <&tlmm 51 IRQ_TYPE_LEVEL_LOW>;
-
-		vdd-supply = <&vreg_misc_3p3>;
-		vddl-supply = <&vreg_l15b_e0_1p8>;
-
-		pinctrl-0 = <&ts0_default>;
-		pinctrl-names = "default";
-	};
-};
-
-&i2c5 {
-	clock-frequency = <400000>;
-
-	status = "okay";
-
-	ptn3222_0: redriver@43 {
-		compatible = "nxp,ptn3222";
-		reg = <0x43>;
-
-		reset-gpios = <&tlmm 8 GPIO_ACTIVE_LOW>;
-
-		vdd3v3-supply = <&vreg_l8b_e0_1p50>;
-		vdd1v8-supply = <&vreg_l15b_e0_1p8>;
-
-		#phy-cells = <0>;
-	};
-
-	ptn3222_1: redriver@47 {
-		compatible = "nxp,ptn3222";
-		reg = <0x47>;
-
-		reset-gpios = <&tlmm 9 GPIO_ACTIVE_LOW>;
-
-		vdd3v3-supply = <&vreg_l8b_e0_1p50>;
-		vdd1v8-supply = <&vreg_l15b_e0_1p8>;
-
-		#phy-cells = <0>;
-	};
-};
-
-&mdss {
-	status = "okay";
 };
 
 &mdss_dp0 {
@@ -217,213 +28,3 @@ &mdss_dp1 {
 &mdss_dp1_out {
 	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
 };
-
-&mdss_dp3 {
-	/delete-property/ #sound-dai-cells;
-
-	status = "okay";
-
-	aux-bus {
-		panel {
-			compatible = "samsung,atna60cl08", "samsung,atna33xc20";
-			enable-gpios = <&tlmm 18 GPIO_ACTIVE_HIGH>;
-			power-supply = <&vreg_edp_3p3>;
-
-			pinctrl-0 = <&edp_bl_en>;
-			pinctrl-names = "default";
-
-			port {
-				edp_panel_in: endpoint {
-					remote-endpoint = <&mdss_dp3_out>;
-				};
-			};
-		};
-	};
-};
-
-&mdss_dp3_out {
-	data-lanes = <0 1 2 3>;
-	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
-
-	remote-endpoint = <&edp_panel_in>;
-};
-
-&mdss_dp3_phy {
-	vdda-phy-supply = <&vreg_l2f_e1_0p83>;
-	vdda-pll-supply = <&vreg_l4f_e1_1p08>;
-
-	status = "okay";
-};
-
-&pmh0110_f_e0_gpios {
-	misc_3p3_reg_en: misc-3p3-reg-en-state {
-		pins = "gpio6";
-		function = "normal";
-		bias-disable;
-		input-disable;
-		output-enable;
-		drive-push-pull;
-		power-source = <1>; /* 1.8 V */
-		qcom,drive-strength = <PMIC_GPIO_STRENGTH_LOW>;
-	};
-};
-
-&smb2370_j_e2_eusb2_repeater {
-	vdd18-supply = <&vreg_l15b_e0_1p8>;
-	vdd3-supply = <&vreg_l7b_e0_2p79>;
-};
-
-&smb2370_k_e2_eusb2_repeater {
-	vdd18-supply = <&vreg_l15b_e0_1p8>;
-	vdd3-supply = <&vreg_l7b_e0_2p79>;
-};
-
-&tlmm {
-	edp_bl_en: edp-bl-en-state {
-		pins = "gpio18";
-		function = "gpio";
-		drive-strength = <16>;
-		bias-disable;
-	};
-
-	edp_reg_en: edp-reg-en-state {
-		pins = "gpio70";
-		function = "gpio";
-		drive-strength = <16>;
-		bias-disable;
-	};
-
-	kybd_default: kybd-default-state {
-		pins = "gpio67";
-		function = "gpio";
-		bias-disable;
-	};
-
-	tpad_default: tpad-default-state {
-		pins = "gpio3";
-		function = "gpio";
-		bias-disable;
-	};
-
-	ts0_default: ts0-default-state {
-		int-n-pins {
-			pins = "gpio51";
-			function = "gpio";
-			bias-disable;
-		};
-
-		reset-n-pins {
-			pins = "gpio48";
-			function = "gpio";
-			drive-strength = <16>;
-			bias-disable;
-		};
-	};
-};
-
-&usb_0 {
-	status = "okay";
-};
-
-&usb_0_dwc3_hs {
-	remote-endpoint = <&pmic_glink_hs_in>;
-};
-
-&usb_0_hsphy {
-	vdd-supply = <&vreg_l3f_e0_0p72>;
-	vdda12-supply = <&vreg_l4h_e0_1p2>;
-
-	phys = <&smb2370_j_e2_eusb2_repeater>;
-
-	status = "okay";
-};
-
-&usb_0_qmpphy {
-	vdda-phy-supply = <&vreg_l4h_e0_1p2>;
-	vdda-pll-supply = <&vreg_l3f_e0_0p72>;
-	refgen-supply = <&vreg_l2f_e0_0p82>;
-
-	status = "okay";
-};
-
-&usb_0_qmpphy_out {
-	remote-endpoint = <&pmic_glink_ss_in>;
-};
-
-&usb_1 {
-	status = "okay";
-};
-
-&usb_1_dwc3_hs {
-	remote-endpoint = <&pmic_glink_hs_in1>;
-};
-
-&usb_1_hsphy {
-	vdd-supply = <&vreg_l3f_e0_0p72>;
-	vdda12-supply = <&vreg_l4h_e0_1p2>;
-
-	phys = <&smb2370_k_e2_eusb2_repeater>;
-
-	status = "okay";
-};
-
-&usb_1_qmpphy {
-	vdda-phy-supply = <&vreg_l4h_e0_1p2>;
-	vdda-pll-supply = <&vreg_l1h_e0_0p89>;
-	refgen-supply = <&vreg_l2f_e0_0p82>;
-
-	status = "okay";
-};
-
-&usb_1_qmpphy_out {
-	remote-endpoint = <&pmic_glink_ss_in1>;
-};
-
-&usb_hs {
-	status = "okay";
-};
-
-&usb_hs_phy {
-	vdd-supply = <&vreg_l2h_e0_0p72>;
-	vdda12-supply = <&vreg_l4h_e0_1p2>;
-
-	phys = <&ptn3222_1>;
-
-	status = "okay";
-};
-
-&usb_mp {
-	status = "okay";
-};
-
-&usb_mp_hsphy0 {
-	vdd-supply = <&vreg_l2h_e0_0p72>;
-	vdda12-supply = <&vreg_l4h_e0_1p2>;
-
-	phys = <&ptn3222_0>;
-
-	status = "okay";
-};
-
-&usb_mp_hsphy1 {
-	vdd-supply = <&vreg_l2h_e0_0p72>;
-	vdda12-supply = <&vreg_l4h_e0_1p2>;
-
-	status = "okay";
-};
-
-&usb_mp_qmpphy0 {
-	vdda-phy-supply = <&vreg_l4h_e0_1p2>;
-	vdda-pll-supply = <&vreg_l2h_e0_0p72>;
-	refgen-supply = <&vreg_l4f_e1_1p08>;
-
-	status = "okay";
-};
-
-&usb_mp_qmpphy1 {
-	vdda-phy-supply = <&vreg_l4h_e0_1p2>;
-	vdda-pll-supply = <&vreg_l2h_e0_0p72>;
-	refgen-supply = <&vreg_l4f_e1_1p08>;
-
-	status = "okay";
-};
diff --git a/arch/arm64/boot/dts/qcom/glymur-crd.dtsi b/arch/arm64/boot/dts/qcom/glymur-crd.dtsi
index e784b538f42e1..b314c6dd423cc 100644
--- a/arch/arm64/boot/dts/qcom/glymur-crd.dtsi
+++ b/arch/arm64/boot/dts/qcom/glymur-crd.dtsi
@@ -11,6 +11,7 @@
 #include "smb2370.dtsi"         /* SPMI2: SID-9/10/11            */
 
 #include <dt-bindings/input/gpio-keys.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
 
 / {
 	model = "Qualcomm Technologies, Inc. Glymur CRD";
@@ -22,6 +23,7 @@ aliases {
 		i2c0 = &i2c0;
 		i2c1 = &i2c4;
 		i2c2 = &i2c5;
+		i2c3 = &i2c9;
 		spi0 = &spi18;
 	};
 
@@ -68,6 +70,101 @@ switch-lid {
 		};
 	};
 
+	pmic-glink {
+		compatible = "qcom,glymur-pmic-glink",
+			     "qcom,pmic-glink";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in: endpoint {
+						remote-endpoint = <&usb_0_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss_in: endpoint {
+						remote-endpoint = <&usb_0_qmpphy_out>;
+					};
+				};
+			};
+		};
+
+		connector@1 {
+			compatible = "usb-c-connector";
+			reg = <1>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in1: endpoint {
+						remote-endpoint = <&usb_1_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss_in1: endpoint {
+						remote-endpoint = <&usb_1_qmpphy_out>;
+					};
+				};
+			};
+		};
+	};
+
+	vreg_edp_3p3: regulator-edp-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_EDP_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 70 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&edp_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_misc_3p3: regulator-misc-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_MISC_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pmh0110_f_e0_gpios 6 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&misc_3p3_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
 	vreg_nvme: regulator-nvme {
 		compatible = "regulator-fixed";
 
@@ -140,6 +237,54 @@ vreg_wwan: regulator-wwan {
 		pinctrl-names = "default";
 	};
 
+	sound {
+		compatible = "qcom,glymur-sndcard";
+		model = "GLYMUR-CRD";
+		audio-routing = "WooferLeft IN", "WSA WSA_SPK1 OUT",
+				"TweeterLeft IN", "WSA WSA_SPK2 OUT",
+				"WooferRight IN", "WSA2 WSA_SPK1 OUT",
+				"TweeterRight IN", "WSA2 WSA_SPK2 OUT",
+				"VA DMIC0", "vdd-micb",
+				"VA DMIC1", "vdd-micb",
+				"VA DMIC2", "vdd-micb",
+				"VA DMIC3", "vdd-micb";
+
+		wsa-dai-link {
+			link-name = "WSA Playback";
+
+			cpu {
+				sound-dai = <&q6apmbedai WSA_CODEC_DMA_RX_0>;
+			};
+
+			codec {
+				sound-dai = <&left_woofer>, <&left_tweeter>,
+					    <&swr0 0>, <&lpass_wsamacro 0>,
+					    <&right_woofer>, <&right_tweeter>,
+					    <&swr3 0>, <&lpass_wsa2macro 0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		va-dai-link {
+			link-name = "VA Capture";
+
+			cpu {
+				sound-dai = <&q6apmbedai VA_CODEC_DMA_TX_0>;
+			};
+
+			codec {
+				sound-dai = <&lpass_vamacro 0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+	};
+
 	wcn7850-pmu {
 		compatible = "qcom,wcn7850-pmu";
 
@@ -444,6 +589,160 @@ vreg_l4h_e0_1p2: ldo4 {
 	};
 };
 
+&i2c9 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	embedded-controller@76 {
+		compatible = "qcom,glymur-crd-ec", "qcom,hamoa-crd-ec";
+		reg = <0x76>;
+
+		interrupts-extended = <&tlmm 66 IRQ_TYPE_EDGE_FALLING>;
+
+		pinctrl-0 = <&ec_int_n_default>;
+		pinctrl-names = "default";
+	};
+};
+
+&i2c0 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	touchpad@2c {
+		compatible = "hid-over-i2c";
+		reg = <0x2c>;
+
+		hid-descr-addr = <0x20>;
+		interrupts-extended = <&tlmm 3 IRQ_TYPE_LEVEL_LOW>;
+
+		vdd-supply = <&vreg_misc_3p3>;
+		vddl-supply = <&vreg_l15b_e0_1p8>;
+
+		pinctrl-0 = <&tpad_default>;
+		pinctrl-names = "default";
+
+		wakeup-source;
+	};
+
+	keyboard@3a {
+		compatible = "hid-over-i2c";
+		reg = <0x3a>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 67 IRQ_TYPE_LEVEL_LOW>;
+
+		vdd-supply = <&vreg_misc_3p3>;
+		vddl-supply = <&vreg_l15b_e0_1p8>;
+
+		pinctrl-0 = <&kybd_default>;
+		pinctrl-names = "default";
+
+		wakeup-source;
+	};
+};
+
+&i2c5 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	ptn3222_0: redriver@43 {
+		compatible = "nxp,ptn3222";
+		reg = <0x43>;
+
+		reset-gpios = <&tlmm 8 GPIO_ACTIVE_LOW>;
+
+		vdd3v3-supply = <&vreg_l8b_e0_1p50>;
+		vdd1v8-supply = <&vreg_l15b_e0_1p8>;
+
+		#phy-cells = <0>;
+	};
+
+	ptn3222_1: redriver@47 {
+		compatible = "nxp,ptn3222";
+		reg = <0x47>;
+
+		reset-gpios = <&tlmm 9 GPIO_ACTIVE_LOW>;
+
+		vdd3v3-supply = <&vreg_l8b_e0_1p50>;
+		vdd1v8-supply = <&vreg_l15b_e0_1p8>;
+
+		#phy-cells = <0>;
+	};
+};
+
+&i2c8 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	touchscreen@38 {
+		compatible = "hid-over-i2c";
+		reg = <0x38>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 51 IRQ_TYPE_LEVEL_LOW>;
+
+		vdd-supply = <&vreg_misc_3p3>;
+		vddl-supply = <&vreg_l15b_e0_1p8>;
+
+		reset-gpios = <&tlmm 48 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&ts0_default>;
+		pinctrl-names = "default";
+	};
+};
+
+&lpass_vamacro {
+	pinctrl-0 = <&dmic01_default>, <&dmic23_default>;
+	pinctrl-names = "default";
+	qcom,dmic-sample-rate = <4800000>;
+	vdd-micb-supply = <&vreg_l1b_e0_1p8>;
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dp3 {
+	/delete-property/ #sound-dai-cells;
+
+	status = "okay";
+
+	aux-bus {
+		panel {
+			compatible = "samsung,atna60cl08", "samsung,atna33xc20";
+			enable-gpios = <&tlmm 18 GPIO_ACTIVE_HIGH>;
+			power-supply = <&vreg_edp_3p3>;
+
+			pinctrl-0 = <&edp_bl_en>;
+			pinctrl-names = "default";
+
+			port {
+				edp_panel_in: endpoint {
+					remote-endpoint = <&mdss_dp3_out>;
+				};
+			};
+		};
+	};
+};
+
+&mdss_dp3_out {
+	data-lanes = <0 1 2 3>;
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+
+	remote-endpoint = <&edp_panel_in>;
+};
+
+&mdss_dp3_phy {
+	vdda-phy-supply = <&vreg_l2f_e1_0p83>;
+	vdda-pll-supply = <&vreg_l4f_e1_1p08>;
+
+	status = "okay";
+};
+
 &pcie3b {
 	vddpe-3v3-supply = <&vreg_nvmesec>;
 
@@ -537,14 +836,6 @@ &pcie6_port0 {
 	wake-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>;
 };
 
-&pmh0101_gpios {
-	nvme_reg_en: nvme-reg-en-state {
-		pins = "gpio14";
-		function = "normal";
-		bias-disable;
-	};
-};
-
 &pmh0110_f_e1_gpios {
 	nvme_sec_reg_en: nvme-reg-en-state {
 		pins = "gpio14";
@@ -560,6 +851,25 @@ key_vol_up_default: key-vol-up-default-state {
 		output-disable;
 		bias-pull-up;
 	};
+
+	nvme_reg_en: nvme-reg-en-state {
+		pins = "gpio14";
+		function = "normal";
+		bias-disable;
+	};
+};
+
+&pmh0110_f_e0_gpios {
+	misc_3p3_reg_en: misc-3p3-reg-en-state {
+		pins = "gpio6";
+		function = "normal";
+		bias-disable;
+		input-disable;
+		output-enable;
+		drive-push-pull;
+		power-source = <1>; /* 1.8 V */
+		qcom,drive-strength = <PMIC_GPIO_STRENGTH_LOW>;
+	};
 };
 
 &pmk8850_rtc {
@@ -585,17 +895,109 @@ &remoteproc_cdsp {
 	status = "okay";
 };
 
+&smb2370_j_e2_eusb2_repeater {
+	vdd18-supply = <&vreg_l15b_e0_1p8>;
+	vdd3-supply = <&vreg_l7b_e0_2p79>;
+};
+
+&smb2370_k_e2_eusb2_repeater {
+	vdd18-supply = <&vreg_l15b_e0_1p8>;
+	vdd3-supply = <&vreg_l7b_e0_2p79>;
+};
+
+&swr0 {
+	status = "okay";
+
+	/* WSA8845, Left Woofer */
+	left_woofer: speaker@0,0 {
+		compatible = "sdw20217020400";
+		reg = <0 0>;
+		reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "WooferLeft";
+		vdd-1p8-supply = <&vreg_l15b_e0_1p8>;
+		vdd-io-supply = <&vreg_l18b_e0_1p2>;
+		qcom,port-mapping = <1 2 3 7 12 14>;
+	};
+
+	/* WSA8845, Left Tweeter */
+	left_tweeter: speaker@0,1 {
+		compatible = "sdw20217020400";
+		reg = <0 1>;
+		reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "TweeterLeft";
+		vdd-1p8-supply = <&vreg_l15b_e0_1p8>;
+		vdd-io-supply = <&vreg_l18b_e0_1p2>;
+		qcom,port-mapping = <4 5 6 7 13 15>;
+	};
+};
+
+&swr3 {
+	status = "okay";
+
+	/* WSA8845, Right Woofer */
+	right_woofer: speaker@0,0 {
+		compatible = "sdw20217020400";
+		reg = <0 0>;
+		reset-gpios = <&lpass_tlmm 13 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "WooferRight";
+		vdd-1p8-supply = <&vreg_l15b_e0_1p8>;
+		vdd-io-supply = <&vreg_l18b_e0_1p2>;
+		qcom,port-mapping = <1 2 3 7 12 14>;
+	};
+
+	/* WSA8845, Right Tweeter */
+	right_tweeter: speaker@0,1 {
+		compatible = "sdw20217020400";
+		reg = <0 1>;
+		reset-gpios = <&lpass_tlmm 13 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "TweeterRight";
+		vdd-1p8-supply = <&vreg_l15b_e0_1p8>;
+		vdd-io-supply = <&vreg_l18b_e0_1p2>;
+		qcom,port-mapping = <4 5 6 7 13 15>;
+	};
+};
+
 &tlmm {
 	gpio-reserved-ranges = <4 4>, /* EC TZ Secure I3C */
 			       <10 2>, /* OOB UART */
 			       <44 4>; /* Security SPI (TPM) */
 
+	edp_bl_en: edp-bl-en-state {
+		pins = "gpio18";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	edp_reg_en: edp-reg-en-state {
+		pins = "gpio70";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
 	hall_int_n_default: hall-int-n-state {
 		pins = "gpio92";
 		function = "gpio";
 		bias-disable;
 	};
 
+	ec_int_n_default: ec-int-n-state {
+		pins = "gpio66";
+		function = "gpio";
+		bias-disable;
+	};
+
+	kybd_default: kybd-default-state {
+		pins = "gpio67";
+		function = "gpio";
+		bias-disable;
+	};
+
 	pcie4_default: pcie4-default-state {
 		clkreq-n-pins {
 			pins = "gpio147";
@@ -688,6 +1090,27 @@ wake-n-pins {
 		};
 	};
 
+	tpad_default: tpad-default-state {
+		pins = "gpio3";
+		function = "gpio";
+		bias-disable;
+	};
+
+	ts0_default: ts0-default-state {
+		int-n-pins {
+			pins = "gpio51";
+			function = "gpio";
+			bias-disable;
+		};
+
+		reset-n-pins {
+			pins = "gpio48";
+			function = "gpio";
+			drive-strength = <16>;
+			bias-disable;
+		};
+	};
+
 	wcn_wlan_bt_en: wcn-wlan-bt-en-state {
 		pins = "gpio116", "gpio117";
 		function = "gpio";
@@ -726,3 +1149,110 @@ bluetooth {
 		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
 	};
 };
+
+&usb_0 {
+	status = "okay";
+};
+
+&usb_0_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in>;
+};
+
+&usb_0_hsphy {
+	vdd-supply = <&vreg_l3f_e0_0p72>;
+	vdda12-supply = <&vreg_l4h_e0_1p2>;
+
+	phys = <&smb2370_j_e2_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_0_qmpphy {
+	vdda-phy-supply = <&vreg_l4h_e0_1p2>;
+	vdda-pll-supply = <&vreg_l3f_e0_0p72>;
+	refgen-supply = <&vreg_l2f_e0_0p82>;
+
+	status = "okay";
+};
+
+&usb_0_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss_in>;
+};
+
+&usb_1 {
+	status = "okay";
+};
+
+&usb_1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in1>;
+};
+
+&usb_1_hsphy {
+	vdd-supply = <&vreg_l3f_e0_0p72>;
+	vdda12-supply = <&vreg_l4h_e0_1p2>;
+
+	phys = <&smb2370_k_e2_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_1_qmpphy {
+	vdda-phy-supply = <&vreg_l4h_e0_1p2>;
+	vdda-pll-supply = <&vreg_l1h_e0_0p89>;
+	refgen-supply = <&vreg_l2f_e0_0p82>;
+
+	status = "okay";
+};
+
+&usb_1_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss_in1>;
+};
+
+&usb_hs {
+	status = "okay";
+};
+
+&usb_hs_phy {
+	vdd-supply = <&vreg_l2h_e0_0p72>;
+	vdda12-supply = <&vreg_l4h_e0_1p2>;
+
+	phys = <&ptn3222_1>;
+
+	status = "okay";
+};
+
+&usb_mp {
+	status = "okay";
+};
+
+&usb_mp_hsphy0 {
+	vdd-supply = <&vreg_l2h_e0_0p72>;
+	vdda12-supply = <&vreg_l4h_e0_1p2>;
+
+	phys = <&ptn3222_0>;
+
+	status = "okay";
+};
+
+&usb_mp_hsphy1 {
+	vdd-supply = <&vreg_l2h_e0_0p72>;
+	vdda12-supply = <&vreg_l4h_e0_1p2>;
+
+	status = "okay";
+};
+
+&usb_mp_qmpphy0 {
+	vdda-phy-supply = <&vreg_l4h_e0_1p2>;
+	vdda-pll-supply = <&vreg_l2h_e0_0p72>;
+	refgen-supply = <&vreg_l4f_e1_1p08>;
+
+	status = "okay";
+};
+
+&usb_mp_qmpphy1 {
+	vdda-phy-supply = <&vreg_l4h_e0_1p2>;
+	vdda-pll-supply = <&vreg_l2h_e0_0p72>;
+	refgen-supply = <&vreg_l4f_e1_1p08>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/glymur.dtsi b/arch/arm64/boot/dts/qcom/glymur.dtsi
index 20b49af7298e9..55d91c696a3a3 100644
--- a/arch/arm64/boot/dts/qcom/glymur.dtsi
+++ b/arch/arm64/boot/dts/qcom/glymur.dtsi
@@ -3,7 +3,9 @@
  * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
+#include <dt-bindings/clock/qcom,glymur-camcc.h>
 #include <dt-bindings/clock/qcom,glymur-dispcc.h>
+#include <dt-bindings/clock/qcom,glymur-evacc.h>
 #include <dt-bindings/clock/qcom,glymur-gcc.h>
 #include <dt-bindings/clock/qcom,glymur-gpucc.h>
 #include <dt-bindings/clock/qcom,glymur-tcsr.h>
@@ -11,6 +13,7 @@
 #include <dt-bindings/clock/qcom,kaanapali-gxclkctl.h>
 #include <dt-bindings/clock/qcom,rpmh.h>
 #include <dt-bindings/dma/qcom-gpi.h>
+#include <dt-bindings/firmware/qcom,scm.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/interconnect/qcom,icc.h>
 #include <dt-bindings/interconnect/qcom,glymur-rpmh.h>
@@ -20,8 +23,11 @@
 #include <dt-bindings/power/qcom,rpmhpd.h>
 #include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+#include <dt-bindings/soc/qcom,gpr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
+#include <dt-bindings/sound/qcom,q6dsp-lpass-ports.h>
 #include <dt-bindings/spmi/spmi.h>
+#include <dt-bindings/thermal/thermal.h>
 
 #include "glymur-ipcc.h"
 
@@ -39,6 +45,7 @@ cpu0: cpu@0 {
 			compatible = "qcom,oryon-2-2";
 			reg = <0x0 0x0>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1024>;
 			power-domains = <&cpu_pd0>, <&scmi_perf 0>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_0>;
@@ -56,6 +63,7 @@ cpu1: cpu@100 {
 			compatible = "qcom,oryon-2-2";
 			reg = <0x0 0x100>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1024>;
 			power-domains = <&cpu_pd1>, <&scmi_perf 0>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_0>;
@@ -67,6 +75,7 @@ cpu2: cpu@200 {
 			compatible = "qcom,oryon-2-2";
 			reg = <0x0 0x200>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1024>;
 			power-domains = <&cpu_pd2>, <&scmi_perf 0>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_0>;
@@ -78,6 +87,7 @@ cpu3: cpu@300 {
 			compatible = "qcom,oryon-2-2";
 			reg = <0x0 0x300>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1024>;
 			power-domains = <&cpu_pd3>, <&scmi_perf 0>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_0>;
@@ -89,6 +99,7 @@ cpu4: cpu@400 {
 			compatible = "qcom,oryon-2-2";
 			reg = <0x0 0x400>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1024>;
 			power-domains = <&cpu_pd4>, <&scmi_perf 0>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_0>;
@@ -100,6 +111,7 @@ cpu5: cpu@500 {
 			compatible = "qcom,oryon-2-2";
 			reg = <0x0 0x500>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1024>;
 			power-domains = <&cpu_pd5>, <&scmi_perf 0>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_0>;
@@ -111,6 +123,7 @@ cpu6: cpu@10000 {
 			compatible = "qcom,oryon-2-1";
 			reg = <0x0 0x10000>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1372>;
 			power-domains = <&cpu_pd6>, <&scmi_perf 1>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_1>;
@@ -128,6 +141,7 @@ cpu7: cpu@10100 {
 			compatible = "qcom,oryon-2-1";
 			reg = <0x0 0x10100>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1372>;
 			power-domains = <&cpu_pd7>, <&scmi_perf 1>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_1>;
@@ -139,6 +153,7 @@ cpu8: cpu@10200 {
 			compatible = "qcom,oryon-2-1";
 			reg = <0x0 0x10200>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1372>;
 			power-domains = <&cpu_pd8>, <&scmi_perf 1>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_1>;
@@ -150,6 +165,7 @@ cpu9: cpu@10300 {
 			compatible = "qcom,oryon-2-1";
 			reg = <0x0 0x10300>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1372>;
 			power-domains = <&cpu_pd9>, <&scmi_perf 1>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_1>;
@@ -161,6 +177,7 @@ cpu10: cpu@10400 {
 			compatible = "qcom,oryon-2-1";
 			reg = <0x0 0x10400>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1372>;
 			power-domains = <&cpu_pd10>, <&scmi_perf 1>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_1>;
@@ -172,6 +189,7 @@ cpu11: cpu@10500 {
 			compatible = "qcom,oryon-2-1";
 			reg = <0x0 0x10500>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1372>;
 			power-domains = <&cpu_pd11>, <&scmi_perf 1>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_1>;
@@ -183,6 +201,7 @@ cpu12: cpu@20000 {
 			compatible = "qcom,oryon-2-1";
 			reg = <0x0 0x20000>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1372>;
 			power-domains = <&cpu_pd12>, <&scmi_perf 2>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_2>;
@@ -200,6 +219,7 @@ cpu13: cpu@20100 {
 			compatible = "qcom,oryon-2-1";
 			reg = <0x0 0x20100>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1372>;
 			power-domains = <&cpu_pd13>, <&scmi_perf 2>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_2>;
@@ -211,6 +231,7 @@ cpu14: cpu@20200 {
 			compatible = "qcom,oryon-2-1";
 			reg = <0x0 0x20200>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1372>;
 			power-domains = <&cpu_pd14>, <&scmi_perf 2>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_2>;
@@ -222,6 +243,7 @@ cpu15: cpu@20300 {
 			compatible = "qcom,oryon-2-1";
 			reg = <0x0 0x20300>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1372>;
 			power-domains = <&cpu_pd15>, <&scmi_perf 2>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_2>;
@@ -233,6 +255,7 @@ cpu16: cpu@20400 {
 			compatible = "qcom,oryon-2-1";
 			reg = <0x0 0x20400>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1372>;
 			power-domains = <&cpu_pd16>, <&scmi_perf 2>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_2>;
@@ -244,6 +267,7 @@ cpu17: cpu@20500 {
 			compatible = "qcom,oryon-2-1";
 			reg = <0x0 0x20500>;
 			enable-method = "psci";
+			capacity-dmips-mhz = <1372>;
 			power-domains = <&cpu_pd17>, <&scmi_perf 2>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_2>;
@@ -663,6 +687,14 @@ smem_mem: smem@ffe00000 {
 			hwlocks = <&tcsr_mutex 3>;
 			no-map;
 		};
+
+		adsp_rpc_remote_heap_mem: adsp-rpc-remote-heap {
+			compatible = "shared-dma-pool";
+			alloc-ranges = <0x0 0x80000000 0x0 0x80000000>;
+			reusable;
+			alignment = <0x0 0x400000>;
+			size = <0x0 0x800000>;
+		};
 	};
 
 	smp2p-adsp {
@@ -786,8 +818,8 @@ gcc: clock-controller@100000 {
 				 <&usb_0_qmpphy QMP_USB43DP_USB3_PIPE_CLK>,
 				 <&usb_1_qmpphy QMP_USB43DP_USB3_PIPE_CLK>,
 				 <&usb_2_qmpphy QMP_USB43DP_USB3_PIPE_CLK>,
-				 <&usb_mp_qmpphy0 QMP_USB43DP_USB3_PIPE_CLK>,
-				 <&usb_mp_qmpphy1 QMP_USB43DP_USB3_PIPE_CLK>,
+				 <&usb_mp_qmpphy0>,		/* USB3 UNI PHY pipe 0 */
+				 <&usb_mp_qmpphy1>,		/* USB3 UNI PHY pipe 1 */
 				 <0>,				/* USB4 PHY 0 pcie pipe */
 				 <0>,				/* USB4 PHY 0 Max pipe */
 				 <0>,				/* USB4 PHY 1 pcie pipe */
@@ -1876,7 +1908,7 @@ &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
 			spi0: spi@b80000 {
 				compatible = "qcom,geni-spi";
 				reg = <0x0 0x00b80000 0x0 0x4000>;
-				interrupts = <GIC_SPI 1052 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts = <GIC_ESPI 92 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>;
 				clock-names = "se";
 				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
@@ -1903,7 +1935,7 @@ &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
 			i2c1: i2c@b84000 {
 				compatible = "qcom,geni-i2c";
 				reg = <0x0 0x00b84000 0x0 0x4000>;
-				interrupts = <GIC_SPI 1053 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts = <GIC_ESPI 93 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>;
 				clock-names = "se";
 				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
@@ -1930,7 +1962,7 @@ &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
 			spi1: spi@b84000 {
 				compatible = "qcom,geni-spi";
 				reg = <0x0 0x00b84000 0x0 0x4000>;
-				interrupts = <GIC_SPI 1053 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts = <GIC_ESPI 93 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>;
 				clock-names = "se";
 				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
@@ -1957,7 +1989,7 @@ &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
 			i2c2: i2c@b88000 {
 				compatible = "qcom,geni-i2c";
 				reg = <0x0 0x00b88000 0x0 0x4000>;
-				interrupts = <GIC_SPI 1054 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts = <GIC_ESPI 94 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>;
 				clock-names = "se";
 				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
@@ -1984,7 +2016,7 @@ &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
 			spi2: spi@b88000 {
 				compatible = "qcom,geni-spi";
 				reg = <0x0 0x00b88000 0x0 0x4000>;
-				interrupts = <GIC_SPI 1054 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts = <GIC_ESPI 94 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>;
 				clock-names = "se";
 				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
@@ -2011,7 +2043,7 @@ &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
 			uart2: serial@b88000 {
 				compatible = "qcom,geni-uart";
 				reg = <0x0 0x00b88000 0x0 0x4000>;
-				interrupts = <GIC_SPI 1054 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts = <GIC_ESPI 94 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>;
 				clock-names = "se";
 				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
@@ -2056,7 +2088,7 @@ &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
 			spi3: spi@b8c000 {
 				compatible = "qcom,geni-spi";
 				reg = <0x0 0x00b8c000 0x0 0x4000>;
-				interrupts = <GIC_SPI 1055 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts = <GIC_ESPI 95 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S3_CLK>;
 				clock-names = "se";
 				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
@@ -2110,7 +2142,7 @@ &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
 			spi4: spi@b90000 {
 				compatible = "qcom,geni-spi";
 				reg = <0x0 0x00b90000 0x0 0x4000>;
-				interrupts = <GIC_SPI 1056 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts = <GIC_ESPI 96 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>;
 				clock-names = "se";
 				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
@@ -2164,7 +2196,7 @@ &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
 			spi5: spi@b94000 {
 				compatible = "qcom,geni-spi";
 				reg = <0x0 0x00b94000 0x0 0x4000>;
-				interrupts = <GIC_SPI 1057 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts = <GIC_ESPI 97 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S5_CLK>;
 				clock-names = "se";
 				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
@@ -2191,7 +2223,7 @@ &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
 			i2c6: i2c@b98000 {
 				compatible = "qcom,geni-i2c";
 				reg = <0x0 0x00b98000 0x0 0x4000>;
-				interrupts = <GIC_SPI 1058 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts = <GIC_ESPI 98 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S6_CLK>;
 				clock-names = "se";
 				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
@@ -2218,7 +2250,7 @@ &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
 			spi6: spi@b98000 {
 				compatible = "qcom,geni-spi";
 				reg = <0x0 0x00b98000 0x0 0x4000>;
-				interrupts = <GIC_SPI 1058 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts = <GIC_ESPI 98 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S6_CLK>;
 				clock-names = "se";
 				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
@@ -2245,7 +2277,7 @@ &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
 			i2c7: i2c@b9c000 {
 				compatible = "qcom,geni-i2c";
 				reg = <0x0 0x00b9c000 0x0 0x4000>;
-				interrupts = <GIC_SPI 1059 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts = <GIC_ESPI 99 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S7_CLK>;
 				clock-names = "se";
 				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
@@ -2272,7 +2304,7 @@ &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
 			spi7: spi@b9c000 {
 				compatible = "qcom,geni-spi";
 				reg = <0x0 0x00b9c000 0x0 0x4000>;
-				interrupts = <GIC_SPI 1059 IRQ_TYPE_LEVEL_HIGH>;
+				interrupts = <GIC_ESPI 99 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S7_CLK>;
 				clock-names = "se";
 				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
@@ -3751,6 +3783,129 @@ hsc_noc: interconnect@2000000 {
 			#interconnect-cells = <2>;
 		};
 
+		gpu: gpu@3d00000 {
+			compatible = "qcom,adreno-44070001", "qcom,adreno";
+			reg = <0x0 0x03d00000 0x0 0x6c000>,
+			      <0x0 0x03d9e000 0x0 0x2000>;
+			reg-names = "kgsl_3d0_reg_memory",
+				    "cx_mem";
+
+			interrupts = <GIC_SPI 300 IRQ_TYPE_LEVEL_HIGH>;
+
+			iommus = <&adreno_smmu 0 0x0>,
+				 <&adreno_smmu 1 0x0>;
+
+			operating-points-v2 = <&gpu_opp_table>;
+
+			qcom,gmu = <&gmu>;
+			#cooling-cells = <2>;
+
+			interconnects = <&hsc_noc MASTER_GFX3D QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "gfx-mem";
+
+			gpu_opp_table: opp-table {
+				compatible = "operating-points-v2-adreno",
+					     "operating-points-v2";
+
+				opp-310000000 {
+					opp-hz = /bits/ 64 <310000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D1>;
+					opp-peak-kBps = <2136719>;
+					opp-supported-hw = <0xf>;
+					/* ACD is disabled */
+				};
+
+				opp-410000000 {
+					opp-hz = /bits/ 64 <410000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS>;
+					opp-peak-kBps = <6074219>;
+					opp-supported-hw = <0xf>;
+					/* ACD is disabled */
+				};
+
+				opp-572000000 {
+					opp-hz = /bits/ 64 <572000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS>;
+					opp-peak-kBps = <12449219>;
+					opp-supported-hw = <0xf>;
+					qcom,opp-acd-level = <0xe02d5ffd>;
+				};
+
+				opp-760000000 {
+					opp-hz = /bits/ 64 <760000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS_L1>;
+					opp-peak-kBps = <12449219>;
+					opp-supported-hw = <0xf>;
+					qcom,opp-acd-level = <0xc0285ffd>;
+				};
+
+				opp-820000000 {
+					opp-hz = /bits/ 64 <820000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS_L2>;
+					opp-peak-kBps = <16500000>;
+					opp-supported-hw = <0xf>;
+					qcom,opp-acd-level = <0xa82e5ffd>;
+				};
+
+				opp-915000000 {
+					opp-hz = /bits/ 64 <915000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_NOM>;
+					opp-peak-kBps = <16500000>;
+					opp-supported-hw = <0xf>;
+					qcom,opp-acd-level = <0x882d5ffd>;
+				};
+
+				opp-1070000000 {
+					opp-hz = /bits/ 64 <1070000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_NOM_L1>;
+					opp-peak-kBps = <16500000>;
+					opp-supported-hw = <0xf>;
+					qcom,opp-acd-level = <0x882b5ffd>;
+				};
+
+				opp-1185000000 {
+					opp-hz = /bits/ 64 <1185000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO>;
+					opp-peak-kBps = <16500000>;
+					opp-supported-hw = <0xf>;
+					qcom,opp-acd-level = <0x882a5ffd>;
+				};
+
+				opp-1350000000 {
+					opp-hz = /bits/ 64 <1350000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L1>;
+					opp-peak-kBps = <18597657>;
+					opp-supported-hw = <0xf>;
+					qcom,opp-acd-level = <0x882a5ffd>;
+				};
+
+				opp-1550000000 {
+					opp-hz = /bits/ 64 <1550000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L3>;
+					opp-peak-kBps = <18597657>;
+					opp-supported-hw = <0x7>;
+					qcom,opp-acd-level = <0xa8295ffd>;
+				};
+
+				opp-1700000000 {
+					opp-hz = /bits/ 64 <1700000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L4>;
+					opp-peak-kBps = <18597657>;
+					opp-supported-hw = <0x7>;
+					qcom,opp-acd-level = <0x88295ffd>;
+				};
+
+				opp-1850000000 {
+					opp-hz = /bits/ 64 <1850000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L5>;
+					opp-peak-kBps = <18597657>;
+					opp-supported-hw = <0x3>;
+					qcom,opp-acd-level = <0x88285ffd>;
+				};
+			};
+		};
+
 		gxclkctl: clock-controller@3d64000 {
 			compatible = "qcom,glymur-gxclkctl";
 			reg = <0x0 0x03d64000 0x0 0x6000>;
@@ -3762,6 +3917,66 @@ gxclkctl: clock-controller@3d64000 {
 			#power-domain-cells = <1>;
 		};
 
+		gmu: gmu@3d6c000 {
+			compatible = "qcom,adreno-gmu-x285.1", "qcom,adreno-gmu";
+
+			reg = <0x0 0x03d6c000 0x0 0x32000>;
+			reg-names = "gmu";
+
+			interrupts = <GIC_SPI 304 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 305 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "hfi",
+					  "gmu";
+
+			clocks = <&gpucc GPU_CC_AHB_CLK>,
+				 <&gpucc GPU_CC_CX_GMU_CLK>,
+				 <&gpucc GPU_CC_CXO_CLK>,
+				 <&gcc GCC_GPU_GEMNOC_GFX_CLK>,
+				 <&gpucc GPU_CC_HUB_CX_INT_CLK>,
+				 <&gpucc GPU_CC_RSCC_HUB_AON_CLK>;
+			clock-names = "ahb",
+				      "gmu",
+				      "cxo",
+				      "memnoc",
+				      "hub",
+				      "rscc";
+
+			power-domains = <&gpucc GPU_CC_CX_GDSC>,
+					<&gxclkctl GX_CLKCTL_GX_GDSC>;
+			power-domain-names = "cx",
+					     "gx";
+
+			iommus = <&adreno_smmu 5 0x0>;
+
+			qcom,qmp = <&aoss_qmp>;
+
+			operating-points-v2 = <&gmu_opp_table>;
+
+			gmu_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-575000000 {
+					opp-hz = /bits/ 64 <575000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS>;
+				};
+
+				opp-700000000 {
+					opp-hz = /bits/ 64 <700000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS>;
+				};
+
+				opp-725000000 {
+					opp-hz = /bits/ 64 <725000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS_L1>;
+				};
+
+				opp-750000000 {
+					opp-hz = /bits/ 64 <750000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_NOM>;
+				};
+			};
+		};
+
 		gpucc: clock-controller@3d90000 {
 			compatible = "qcom,glymur-gpucc";
 			reg = <0x0 0x03d90000 0x0 0x9800>;
@@ -3779,6 +3994,44 @@ gpucc: clock-controller@3d90000 {
 			#power-domain-cells = <1>;
 		};
 
+		adreno_smmu: iommu@3da0000 {
+			compatible = "qcom,glymur-smmu-500", "qcom,adreno-smmu",
+				     "qcom,smmu-500", "arm,mmu-500";
+			reg = <0x0 0x03da0000 0x0 0x40000>;
+			#iommu-cells = <2>;
+			#global-interrupts = <1>;
+			interrupts = <GIC_SPI 674 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 678 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 679 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 680 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 681 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 682 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 683 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 684 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 685 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 686 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 687 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 688 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 422 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 476 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 574 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 575 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 576 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 577 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 660 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 662 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 665 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 666 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 667 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 669 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 670 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 700 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gpucc GPU_CC_GPU_SMMU_VOTE_CLK>;
+			clock-names = "hlos";
+			power-domains = <&gpucc GPU_CC_CX_GDSC>;
+			dma-coherent;
+		};
+
 		ipcc: mailbox@3e04000 {
 			compatible = "qcom,glymur-ipcc", "qcom,ipcc";
 			reg = <0x0 0x03e04000 0x0 0x1000>;
@@ -3845,6 +4098,9 @@ fastrpc {
 					compatible = "qcom,glymur-fastrpc", "qcom,kaanapali-fastrpc";
 					qcom,glink-channels = "fastrpcglink-apps-dsp";
 					label = "adsp";
+					memory-region = <&adsp_rpc_remote_heap_mem>;
+					qcom,vmids = <QCOM_SCM_VMID_LPASS
+						      QCOM_SCM_VMID_ADSP_HEAP>;
 					#address-cells = <1>;
 					#size-cells = <0>;
 
@@ -3903,9 +4159,138 @@ compute-cb@8 {
 						dma-coherent;
 					};
 				};
+
+				gpr {
+					compatible = "qcom,gpr";
+					qcom,glink-channels = "adsp_apps";
+					qcom,domain = <GPR_DOMAIN_ID_ADSP>;
+					qcom,intents = <512 20>;
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					q6apm: service@1 {
+						compatible = "qcom,q6apm";
+						reg = <GPR_APM_MODULE_IID>;
+						#sound-dai-cells = <0>;
+						qcom,protection-domain = "avs/audio",
+									 "msm/adsp/audio_pd";
+
+						q6apmbedai: bedais {
+							compatible = "qcom,q6apm-lpass-dais";
+							#sound-dai-cells = <1>;
+						};
+
+						q6apmdai: dais {
+							compatible = "qcom,q6apm-dais";
+							iommus = <&apps_smmu 0x1001 0x80>,
+								 <&apps_smmu 0x1061 0x20>;
+						};
+					};
+
+					q6prm: service@2 {
+						compatible = "qcom,q6prm";
+						reg = <GPR_PRM_MODULE_IID>;
+						qcom,protection-domain = "avs/audio",
+									 "msm/adsp/audio_pd";
+
+						q6prmcc: clock-controller {
+							compatible = "qcom,q6prm-lpass-clocks";
+							#clock-cells = <2>;
+						};
+					};
+				};
 			};
 		};
 
+		swr0: soundwire@6c80000 {
+			compatible = "qcom,soundwire-v3.1.0";
+			reg = <0x0 0x06c80000 0x0 0x10000>;
+			interrupts = <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&lpass_wsamacro>;
+			clock-names = "iface";
+			label = "WSA";
+
+			pinctrl-0 = <&wsa_swr_active>;
+			pinctrl-names = "default";
+
+			qcom,ports-block-pack-mode =	/bits/ 8 <0x00 0x01 0x01 0x00 0x01 0x01 0x00 0x00 0x00 0x00 0x00 0x01 0x01 0x00 0x00 0x01 0x01>;
+			qcom,ports-word-length =	/bits/ 8 <0xff 0xff 0xff 0xff 0xff 0xff 0x08 0xff 0xff 0xff 0xff 0xff 0xff 0x18 0x18 0xff 0xff>;
+			qcom,ports-offset1 =		/bits/ 8 <0x01 0x03 0x05 0x02 0x04 0x15 0x00 0xff 0xff 0xff 0xff 0x06 0x0d 0x0 0x19 0x06 0x06>;
+			qcom,ports-offset2 =		/bits/ 8 <0xff 0x07 0x1f 0xff 0x07 0x1f 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff>;
+			qcom,ports-sinterval =		/bits/ 16 <0x07 0x1f 0x3f 0x07 0x1f 0x3f 0xc8 0xff 0xff 0xff 0xff 0x0f 0x0f 0x31f 0x31f 0x0f 0x0f >;
+			qcom,ports-hstart =		/bits/ 8 <0xff 0xff 0xff 0xff 0xff 0xff 0x08 0xff 0xff 0xff 0xff 0xff 0xff 0xf 0xf 0xff 0xff>;
+			qcom,ports-hstop =		/bits/ 8 <0xff 0xff 0xff 0xff 0xff 0xff 0x08 0xff 0xff 0xff 0xff 0xff 0xff 0x0f 0x0f 0xff 0xff>;
+			qcom,ports-lane-control =	/bits/ 8 <0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff>;
+			qcom,ports-block-group-count =	/bits/ 8 <0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff>;
+			#address-cells = <2>;
+			#size-cells = <0>;
+			#sound-dai-cells = <1>;
+			status = "disabled";
+		};
+
+		lpass_wsamacro: codec@6c90000 {
+			compatible = "qcom,glymur-lpass-wsa-macro", "qcom,sm8550-lpass-wsa-macro";
+			reg = <0x0 0x06c90000 0x0 0x1000>;
+			clocks = <&q6prmcc LPASS_CLK_ID_WSA_CORE_TX_MCLK LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+				 <&q6prmcc LPASS_HW_MACRO_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+				 <&q6prmcc LPASS_HW_DCODEC_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+				 <&lpass_vamacro>;
+			clock-names = "mclk",
+				      "macro",
+				      "dcodec",
+				      "fsgen";
+
+			#clock-cells = <0>;
+			clock-output-names = "mclk";
+			#sound-dai-cells = <1>;
+			sound-name-prefix = "WSA";
+		};
+
+		swr3: soundwire@6ca0000 {
+			compatible = "qcom,soundwire-v3.1.0";
+			reg = <0x0 0x06ca0000 0x0 0x10000>;
+			interrupts = <GIC_SPI 171 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&lpass_wsa2macro>;
+			clock-names = "iface";
+			label = "WSA2";
+
+			pinctrl-0 = <&wsa2_swr_active>;
+			pinctrl-names = "default";
+
+			qcom,ports-block-pack-mode =	/bits/ 8 <0x00 0x01 0x01 0x00 0x01 0x01 0x00 0x00 0x00 0x00 0x00 0x01 0x01 0x00 0x00 0x01 0x01>;
+			qcom,ports-word-length =	/bits/ 8 <0xff 0xff 0xff 0xff 0xff 0xff 0x08 0xff 0xff 0xff 0xff 0xff 0xff 0x18 0x18 0xff 0xff>;
+			qcom,ports-offset1 =		/bits/ 8 <0x01 0x03 0x05 0x02 0x04 0x15 0x00 0xff 0xff 0xff 0xff 0x06 0x0d 0x0 0x19 0x06 0x06>;
+			qcom,ports-offset2 =		/bits/ 8 <0xff 0x07 0x1f 0xff 0x07 0x1f 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff>;
+			qcom,ports-sinterval =		/bits/ 16 <0x07 0x1f 0x3f 0x07 0x1f 0x3f 0xc8 0xff 0xff 0xff 0xff 0x0f 0x0f 0x31f 0x31f 0x0f 0x0f >;
+			qcom,ports-hstart =		/bits/ 8 <0xff 0xff 0xff 0xff 0xff 0xff 0x08 0xff 0xff 0xff 0xff 0xff 0xff 0xf 0xf 0xff 0xff>;
+			qcom,ports-hstop =		/bits/ 8 <0xff 0xff 0xff 0xff 0xff 0xff 0x08 0xff 0xff 0xff 0xff 0xff 0xff 0x0f 0x0f 0xff 0xff>;
+			qcom,ports-lane-control =	/bits/ 8 <0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff>;
+			qcom,ports-block-group-count =	/bits/ 8 <0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff 0xff>;
+
+			#address-cells = <2>;
+			#size-cells = <0>;
+			#sound-dai-cells = <1>;
+			status = "disabled";
+		};
+
+		lpass_wsa2macro: codec@6cb0000 {
+			compatible = "qcom,glymur-lpass-wsa-macro", "qcom,sm8550-lpass-wsa-macro";
+			reg = <0x0 0x06cb0000 0x0 0x1000>;
+			clocks = <&q6prmcc LPASS_CLK_ID_WSA2_CORE_TX_MCLK LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+				 <&q6prmcc LPASS_HW_MACRO_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+				 <&q6prmcc LPASS_HW_DCODEC_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+				 <&lpass_vamacro>;
+			clock-names = "mclk",
+				      "macro",
+				      "dcodec",
+				      "fsgen";
+
+			#clock-cells = <0>;
+			clock-output-names = "wsa2-mclk";
+			#sound-dai-cells = <1>;
+			sound-name-prefix = "WSA2";
+		};
+
 		lpass_lpiaon_noc: interconnect@7400000 {
 			compatible = "qcom,glymur-lpass-lpiaon-noc";
 			reg = <0x0 0x07400000 0x0 0x19080>;
@@ -3920,6 +4305,138 @@ lpass_lpicx_noc: interconnect@7420000 {
 			#interconnect-cells = <2>;
 		};
 
+		lpass_vamacro: codec@7660000 {
+			compatible = "qcom,glymur-lpass-va-macro", "qcom,sm8550-lpass-va-macro";
+			reg = <0x0 0x07660000 0x0 0x2000>;
+			clocks = <&q6prmcc LPASS_CLK_ID_TX_CORE_MCLK LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+				 <&q6prmcc LPASS_HW_MACRO_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+				 <&q6prmcc LPASS_HW_DCODEC_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>;
+			clock-names = "mclk",
+				      "macro",
+				      "dcodec";
+
+			#clock-cells = <0>;
+			clock-output-names = "fsgen";
+			#sound-dai-cells = <1>;
+		};
+
+		lpass_tlmm: pinctrl@7760000 {
+			compatible = "qcom,glymur-lpass-lpi-pinctrl", "qcom,sm8650-lpass-lpi-pinctrl";
+			reg = <0x0 0x07760000 0x0 0x20000>;
+
+			clocks = <&q6prmcc LPASS_HW_MACRO_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+				 <&q6prmcc LPASS_HW_DCODEC_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>;
+			clock-names = "core", "audio";
+
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&lpass_tlmm 0 0 23>;
+
+			tx_swr_active: tx-swr-active-state {
+				clk-pins {
+					pins = "gpio0";
+					function = "swr_tx_clk";
+					drive-strength = <2>;
+					slew-rate = <1>;
+					bias-disable;
+				};
+
+				data-pins {
+					pins = "gpio1", "gpio2";
+					function = "swr_tx_data";
+					drive-strength = <2>;
+					slew-rate = <1>;
+					bias-bus-hold;
+				};
+			};
+
+			rx_swr_active: rx-swr-active-state {
+				clk-pins {
+					pins = "gpio3";
+					function = "swr_rx_clk";
+					drive-strength = <2>;
+					slew-rate = <1>;
+					bias-disable;
+				};
+
+				data-pins {
+					pins = "gpio4", "gpio5";
+					function = "swr_rx_data";
+					drive-strength = <2>;
+					slew-rate = <1>;
+					bias-bus-hold;
+				};
+			};
+
+			dmic01_default: dmic01-default-state {
+				clk-pins {
+					pins = "gpio6";
+					function = "dmic1_clk";
+					drive-strength = <8>;
+					output-high;
+				};
+
+				data-pins {
+					pins = "gpio7";
+					function = "dmic1_data";
+					drive-strength = <8>;
+					input-enable;
+				};
+			};
+
+			dmic23_default: dmic23-default-state {
+				clk-pins {
+					pins = "gpio8";
+					function = "dmic2_clk";
+					drive-strength = <8>;
+					output-high;
+				};
+
+				data-pins {
+					pins = "gpio9";
+					function = "dmic2_data";
+					drive-strength = <8>;
+					input-enable;
+				};
+			};
+
+			wsa_swr_active: wsa-swr-active-state {
+				clk-pins {
+					pins = "gpio10";
+					function = "wsa_swr_clk";
+					drive-strength = <2>;
+					slew-rate = <1>;
+					bias-disable;
+				};
+
+				data-pins {
+					pins = "gpio11";
+					function = "wsa_swr_data";
+					drive-strength = <2>;
+					slew-rate = <1>;
+					bias-bus-hold;
+				};
+			};
+
+			wsa2_swr_active: wsa2-swr-active-state {
+				clk-pins {
+					pins = "gpio15";
+					function = "wsa2_swr_clk";
+					drive-strength = <2>;
+					slew-rate = <1>;
+					bias-disable;
+				};
+
+				data-pins {
+					pins = "gpio16";
+					function = "wsa2_swr_data";
+					drive-strength = <2>;
+					slew-rate = <1>;
+					bias-bus-hold;
+				};
+			};
+		};
+
 		lpass_ag_noc: interconnect@7e40000 {
 			compatible = "qcom,glymur-lpass-ag-noc";
 			reg = <0x0 0x07e40000 0x0 0xe080>;
@@ -4362,6 +4879,22 @@ usb_mp: usb@a400000 {
 			status = "disabled";
 		};
 
+		camcc: clock-controller@ade0000 {
+			compatible = "qcom,glymur-camcc";
+			reg = <0x0 0x0ade0000 0x0 0x20000>;
+			clocks = <&gcc GCC_CAMERA_AHB_CLK>,
+				 <&rpmhcc RPMH_CXO_CLK>,
+				 <&rpmhcc RPMH_CXO_CLK_A>,
+				 <&sleep_clk>;
+			power-domains = <&rpmhpd RPMHPD_MXC>,
+					<&rpmhpd RPMHPD_MMCX>;
+			required-opps = <&rpmhpd_opp_low_svs>,
+					<&rpmhpd_opp_low_svs>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
 		mdss: display-subsystem@ae00000 {
 			compatible = "qcom,glymur-mdss";
 			reg = <0x0 0x0ae00000 0x0 0x1000>;
@@ -4804,6 +5337,23 @@ videocc: clock-controller@aaf0000 {
 			#power-domain-cells = <1>;
 		};
 
+		evacc: clock-controller@abf0000 {
+			compatible = "qcom,glymur-evacc";
+			reg = <0x0 0x0abf0000 0x0 0x10000>;
+			clocks = <&gcc GCC_EVA_AHB_CLK>,
+				 <&rpmhcc RPMH_CXO_CLK>,
+				 <&sleep_clk>;
+
+			power-domains = <&rpmhpd RPMHPD_MMCX>,
+					<&rpmhpd RPMHPD_MXC>;
+			required-opps = <&rpmhpd_opp_low_svs>,
+					<&rpmhpd_opp_low_svs>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
 		dispcc: clock-controller@af00000 {
 			compatible = "qcom,glymur-dispcc";
 			reg = <0x0 0x0af00000 0x0 0x20000>;
@@ -5029,7 +5579,7 @@ spmi_bus1: spmi@c437000 {
 				#size-cells = <0>;
 			};
 
-			spmi_bus2: spmi@c48000 {
+			spmi_bus2: spmi@c448000 {
 				reg = <0x0 0x0c448000 0x0 0x4000>,
 				      <0x0 0x0c8e0000 0x0 0x10000>,
 				      <0x0 0x0c44c000 0x0 0x8000>;
@@ -5770,6 +6320,7 @@ tpdm@1000f000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_spdm";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -5834,6 +6385,7 @@ tpdm@1102c000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_gcc";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -5852,6 +6404,7 @@ tpdm@11180000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_cdsp";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -5871,6 +6424,7 @@ tpdm@11185000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_cdsp_dpm_1";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -5890,6 +6444,7 @@ tpdm@11186000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_cdsp_dpm_2";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -6010,6 +6565,7 @@ cti@11193000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_cdsp";
 		};
 
 		cti_wpss: cti@111ab000 {
@@ -6018,6 +6574,7 @@ cti_wpss: cti@111ab000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_wpss";
 		};
 
 		tpdm@111d0000 {
@@ -6026,6 +6583,7 @@ tpdm@111d0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_qm";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -6201,6 +6759,7 @@ tpdm@11207000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_mm_dsb";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -6219,6 +6778,7 @@ tpdm@1120b000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_east_dsb";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -6237,6 +6797,7 @@ tpdm@11213000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_west_dsb";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -6255,6 +6816,7 @@ tpdm@11219000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_center_dsb";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -6273,6 +6835,7 @@ tpdm@1121a000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_ipcc";
 
 			qcom,cmb-msrs-num = <32>;
 
@@ -6291,6 +6854,7 @@ tpdm@1121b000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_qrng";
 
 			qcom,cmb-msrs-num = <32>;
 
@@ -6309,6 +6873,7 @@ tpdm@1121c000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_pmu";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -6327,6 +6892,7 @@ tpdm@1121d000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_rdpm_cx";
 
 			qcom,cmb-msrs-num = <32>;
 
@@ -6345,6 +6911,7 @@ tpdm@1121e000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_rdpm_mxc";
 
 			qcom,cmb-msrs-num = <32>;
 
@@ -6363,6 +6930,7 @@ tpdm@1121f000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_rdpm_mxa";
 
 			qcom,cmb-msrs-num = <32>;
 
@@ -6381,6 +6949,7 @@ tpdm@11220000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_center_dsb_1";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -6399,6 +6968,7 @@ tpdm@11224000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_south2_dsb";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -6417,6 +6987,7 @@ tpdm@11228000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_south_dsb";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -6435,6 +7006,7 @@ tpdm@11470000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_pcie_rscc";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -6478,6 +7050,7 @@ tpdm@11c03000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_4";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -6656,6 +7229,7 @@ tpdm@11c09000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_0";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -6675,6 +7249,7 @@ tpdm@11c0a000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_1";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -6694,6 +7269,7 @@ tpdm@11c0b000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_2";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -6713,6 +7289,7 @@ tpdm@11c0c000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_3";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -6732,6 +7309,7 @@ tpdm@11c0d000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -6745,6 +7323,22 @@ swao_tpdm_out: endpoint {
 			};
 		};
 
+		sram@14680000 {
+			compatible = "qcom,glymur-imem", "mmio-sram";
+			reg = <0x0 0x14680000 0x0 0x2c000>;
+			ranges = <0 0 0x14680000 0x2c000>;
+
+			no-memory-wc;
+
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			pil-sram@94c {
+				compatible = "qcom,pil-reloc-info";
+				reg = <0x94c 0xc8>;
+			};
+		};
+
 		apps_smmu: iommu@15000000 {
 			compatible = "qcom,glymur-smmu-500",
 				     "qcom,smmu-500",
@@ -6874,9 +7468,9 @@ apps_smmu: iommu@15000000 {
 		pcie_smmu: iommu@15480000 {
 			compatible = "arm,smmu-v3";
 			reg = <0x0 0x15480000 0x0 0x20000>;
-			interrupts = <GIC_SPI 964 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 962 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 960 IRQ_TYPE_LEVEL_HIGH>;
+			interrupts = <GIC_ESPI 4 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 2 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 0 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "eventq", "cmdq-sync", "gerror";
 			dma-coherent;
 			#iommu-cells = <1>;
@@ -8307,13 +8901,22 @@ aoss-7-critical {
 		};
 
 		thermal_gpu_0_0: gpu-0-0-thermal {
+			polling-delay-passive = <100>;
+
 			thermal-sensors = <&tsens7 1>;
 
+			cooling-maps {
+				map0 {
+					trip = <&gpu00_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
 			trips {
-				trip-point0 {
-					temperature = <90000>;
-					hysteresis = <5000>;
-					type = "hot";
+				gpu00_alert0: trip-point0 {
+					temperature = <95000>;
+					hysteresis = <1000>;
+					type = "passive";
 				};
 
 				gpu-0-0-critical {
@@ -8322,16 +8925,26 @@ gpu-0-0-critical {
 					type = "critical";
 				};
 			};
+
 		};
 
 		thermal_gpu_0_1: gpu-0-1-thermal {
+			polling-delay-passive = <100>;
+
 			thermal-sensors = <&tsens7 2>;
 
+			cooling-maps {
+				map0 {
+					trip = <&gpu01_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
 			trips {
-				trip-point0 {
-					temperature = <90000>;
-					hysteresis = <5000>;
-					type = "hot";
+				gpu01_alert0: trip-point0 {
+					temperature = <95000>;
+					hysteresis = <1000>;
+					type = "passive";
 				};
 
 				gpu-0-1-critical {
@@ -8343,13 +8956,22 @@ gpu-0-1-critical {
 		};
 
 		thermal_gpu_0_2: gpu-0-2-thermal {
+			polling-delay-passive = <100>;
+
 			thermal-sensors = <&tsens7 3>;
 
+			cooling-maps {
+				map0 {
+					trip = <&gpu02_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
 			trips {
-				trip-point0 {
-					temperature = <90000>;
-					hysteresis = <5000>;
-					type = "hot";
+				gpu02_alert0: trip-point0 {
+					temperature = <95000>;
+					hysteresis = <1000>;
+					type = "passive";
 				};
 
 				gpu-0-2-critical {
@@ -8361,13 +8983,22 @@ gpu-0-2-critical {
 		};
 
 		thermal_gpu_1_0: gpu-1-0-thermal {
+			polling-delay-passive = <100>;
+
 			thermal-sensors = <&tsens7 4>;
 
+			cooling-maps {
+				map0 {
+					trip = <&gpu10_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
 			trips {
-				trip-point0 {
-					temperature = <90000>;
-					hysteresis = <5000>;
-					type = "hot";
+				gpu10_alert0: trip-point0 {
+					temperature = <95000>;
+					hysteresis = <1000>;
+					type = "passive";
 				};
 
 				gpu-1-0-critical {
@@ -8379,13 +9010,22 @@ gpu-1-0-critical {
 		};
 
 		thermal_gpu_1_1: gpu-1-1-thermal {
+			polling-delay-passive = <100>;
+
 			thermal-sensors = <&tsens7 5>;
 
+			cooling-maps {
+				map0 {
+					trip = <&gpu11_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
 			trips {
-				trip-point0 {
-					temperature = <90000>;
-					hysteresis = <5000>;
-					type = "hot";
+				gpu11_alert0: trip-point0 {
+					temperature = <95000>;
+					hysteresis = <1000>;
+					type = "passive";
 				};
 
 				gpu-1-1-critical {
@@ -8397,13 +9037,22 @@ gpu-1-1-critical {
 		};
 
 		thermal_gpu_1_2: gpu-1-2-thermal {
+			polling-delay-passive = <100>;
+
 			thermal-sensors = <&tsens7 6>;
 
+			cooling-maps {
+				map0 {
+					trip = <&gpu12_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
 			trips {
-				trip-point0 {
-					temperature = <90000>;
-					hysteresis = <5000>;
-					type = "hot";
+				gpu12_alert0: trip-point0 {
+					temperature = <95000>;
+					hysteresis = <1000>;
+					type = "passive";
 				};
 
 				gpu-1-2-critical {
@@ -8415,13 +9064,22 @@ gpu-1-2-critical {
 		};
 
 		thermal_gpu_2_0: gpu-2-0-thermal {
+			polling-delay-passive = <100>;
+
 			thermal-sensors = <&tsens7 7>;
 
+			cooling-maps {
+				map0 {
+					trip = <&gpu20_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
 			trips {
-				trip-point0 {
-					temperature = <90000>;
-					hysteresis = <5000>;
-					type = "hot";
+				gpu20_alert0: trip-point0 {
+					temperature = <95000>;
+					hysteresis = <1000>;
+					type = "passive";
 				};
 
 				gpu-2-0-critical {
@@ -8433,13 +9091,22 @@ gpu-2-0-critical {
 		};
 
 		thermal_gpu_2_1: gpu-2-1-thermal {
+			polling-delay-passive = <100>;
+
 			thermal-sensors = <&tsens7 8>;
 
+			cooling-maps {
+				map0 {
+					trip = <&gpu21_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
 			trips {
-				trip-point0 {
-					temperature = <90000>;
-					hysteresis = <5000>;
-					type = "hot";
+				gpu21_alert0: trip-point0 {
+					temperature = <95000>;
+					hysteresis = <1000>;
+					type = "passive";
 				};
 
 				gpu-2-1-critical {
@@ -8451,13 +9118,22 @@ gpu-2-1-critical {
 		};
 
 		thermal_gpu_2_2: gpu-2-2-thermal {
+			polling-delay-passive = <100>;
+
 			thermal-sensors = <&tsens7 9>;
 
+			cooling-maps {
+				map0 {
+					trip = <&gpu22_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
 			trips {
-				trip-point0 {
-					temperature = <90000>;
-					hysteresis = <5000>;
-					type = "hot";
+				gpu22_alert0: trip-point0 {
+					temperature = <95000>;
+					hysteresis = <1000>;
+					type = "passive";
 				};
 
 				gpu-2-2-critical {
@@ -8469,13 +9145,22 @@ gpu-2-2-critical {
 		};
 
 		thermal_gpu_3_0: gpu-3-0-thermal {
+			polling-delay-passive = <100>;
+
 			thermal-sensors = <&tsens7 10>;
 
+			cooling-maps {
+				map0 {
+					trip = <&gpu30_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
 			trips {
-				trip-point0 {
-					temperature = <90000>;
-					hysteresis = <5000>;
-					type = "hot";
+				gpu30_alert0: trip-point0 {
+					temperature = <95000>;
+					hysteresis = <1000>;
+					type = "passive";
 				};
 
 				gpu-3-0-critical {
@@ -8487,13 +9172,22 @@ gpu-3-0-critical {
 		};
 
 		thermal_gpu_3_1: gpu-3-1-thermal {
+			polling-delay-passive = <100>;
+
 			thermal-sensors = <&tsens7 11>;
 
+			cooling-maps {
+				map0 {
+					trip = <&gpu31_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
 			trips {
-				trip-point0 {
-					temperature = <90000>;
-					hysteresis = <5000>;
-					type = "hot";
+				gpu31_alert0: trip-point0 {
+					temperature = <95000>;
+					hysteresis = <1000>;
+					type = "passive";
 				};
 
 				gpu-3-1-critical {
@@ -8505,13 +9199,22 @@ gpu-3-1-critical {
 		};
 
 		thermal_gpu_3_2: gpu-3-2-thermal {
+			polling-delay-passive = <100>;
+
 			thermal-sensors = <&tsens7 12>;
 
+			cooling-maps {
+				map0 {
+					trip = <&gpu32_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
 			trips {
-				trip-point0 {
-					temperature = <90000>;
-					hysteresis = <5000>;
-					type = "hot";
+				gpu32_alert0: trip-point0 {
+					temperature = <95000>;
+					hysteresis = <1000>;
+					type = "passive";
 				};
 
 				gpu-3-2-critical {
@@ -8523,13 +9226,22 @@ gpu-3-2-critical {
 		};
 
 		thermal_gpuss_0: gpuss-0-thermal {
+			polling-delay-passive = <100>;
+
 			thermal-sensors = <&tsens7 13>;
 
+			cooling-maps {
+				map0 {
+					trip = <&gpuss0_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
 			trips {
-				trip-point0 {
-					temperature = <90000>;
-					hysteresis = <5000>;
-					type = "hot";
+				gpuss0_alert0: trip-point0 {
+					temperature = <95000>;
+					hysteresis = <1000>;
+					type = "passive";
 				};
 
 				gpuss-0-critical {
@@ -8541,13 +9253,22 @@ gpuss-0-critical {
 		};
 
 		thermal_gpuss_1: gpuss-1-thermal {
+			polling-delay-passive = <100>;
+
 			thermal-sensors = <&tsens7 14>;
 
+			cooling-maps {
+				map0 {
+					trip = <&gpuss1_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
 			trips {
-				trip-point0 {
-					temperature = <90000>;
-					hysteresis = <5000>;
-					type = "hot";
+				gpuss1_alert0: trip-point0 {
+					temperature = <95000>;
+					hysteresis = <1000>;
+					type = "passive";
 				};
 
 				gpuss-1-critical {
diff --git a/arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts b/arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts
index c7ba4b353ddd2..9fa86bb6438ec 100644
--- a/arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts
+++ b/arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts
@@ -839,6 +839,16 @@ eusb6_repeater: redriver@4f {
 		pinctrl-0 = <&eusb6_reset_n>;
 		pinctrl-names = "default";
 	};
+
+	embedded-controller@76 {
+		compatible = "qcom,hamoa-iot-evk-ec", "qcom,hamoa-crd-ec";
+		reg = <0x76>;
+
+		interrupts-extended = <&tlmm 66 IRQ_TYPE_EDGE_FALLING>;
+
+		pinctrl-0 = <&ec_int_n_default>;
+		pinctrl-names = "default";
+	};
 };
 
 &i2c7 {
@@ -1341,6 +1351,12 @@ right_tweeter: speaker@0,1 {
 };
 
 &tlmm {
+	ec_int_n_default: ec-int-n-state {
+		pins = "gpio66";
+		function = "gpio";
+		bias-disable;
+	};
+
 	edp_reg_en: edp-reg-en-state {
 		pins = "gpio70";
 		function = "gpio";
diff --git a/arch/arm64/boot/dts/qcom/hamoa.dtsi b/arch/arm64/boot/dts/qcom/hamoa.dtsi
index 4ba751a65142b..09527dcf95767 100644
--- a/arch/arm64/boot/dts/qcom/hamoa.dtsi
+++ b/arch/arm64/boot/dts/qcom/hamoa.dtsi
@@ -6,11 +6,13 @@
 #include <dt-bindings/clock/qcom,rpmh.h>
 #include <dt-bindings/clock/qcom,sc8280xp-lpasscc.h>
 #include <dt-bindings/clock/qcom,sm8450-videocc.h>
+#include <dt-bindings/clock/qcom,x1e80100-camcc.h>
 #include <dt-bindings/clock/qcom,x1e80100-dispcc.h>
 #include <dt-bindings/clock/qcom,x1e80100-gcc.h>
 #include <dt-bindings/clock/qcom,x1e80100-gpucc.h>
 #include <dt-bindings/clock/qcom,x1e80100-tcsr.h>
 #include <dt-bindings/dma/qcom-gpi.h>
+#include <dt-bindings/firmware/qcom,scm.h>
 #include <dt-bindings/interconnect/qcom,icc.h>
 #include <dt-bindings/interconnect/qcom,x1e80100-rpmh.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
@@ -474,16 +476,6 @@ gunyah_hyp_mem: gunyah-hyp@80000000 {
 			no-map;
 		};
 
-		hyp_elf_package_mem: hyp-elf-package@80800000 {
-			reg = <0x0 0x80800000 0x0 0x200000>;
-			no-map;
-		};
-
-		ncc_mem: ncc@80a00000 {
-			reg = <0x0 0x80a00000 0x0 0x400000>;
-			no-map;
-		};
-
 		cpucp_log_mem: cpucp-log@80e00000 {
 			reg = <0x0 0x80e00000 0x0 0x40000>;
 			no-map;
@@ -504,11 +496,6 @@ tags_mem: tags-region@81400000 {
 			no-map;
 		};
 
-		xbl_dtlog_mem: xbl-dtlog@81a00000 {
-			reg = <0x0 0x81a00000 0x0 0x40000>;
-			no-map;
-		};
-
 		xbl_ramdump_mem: xbl-ramdump@81a40000 {
 			reg = <0x0 0x81a40000 0x0 0x1c0000>;
 			no-map;
@@ -550,51 +537,6 @@ secdata_apss_mem: secdata-apss@81cff000 {
 			no-map;
 		};
 
-		pdp_ns_shared_mem: pdp-ns-shared@81e00000 {
-			reg = <0x0 0x81e00000 0x0 0x100000>;
-			no-map;
-		};
-
-		gpu_prr_mem: gpu-prr@81f00000 {
-			reg = <0x0 0x81f00000 0x0 0x10000>;
-			no-map;
-		};
-
-		tpm_control_mem: tpm-control@81f10000 {
-			reg = <0x0 0x81f10000 0x0 0x10000>;
-			no-map;
-		};
-
-		usb_ucsi_shared_mem: usb-ucsi-shared@81f20000 {
-			reg = <0x0 0x81f20000 0x0 0x10000>;
-			no-map;
-		};
-
-		pld_pep_mem: pld-pep@81f30000 {
-			reg = <0x0 0x81f30000 0x0 0x6000>;
-			no-map;
-		};
-
-		pld_gmu_mem: pld-gmu@81f36000 {
-			reg = <0x0 0x81f36000 0x0 0x1000>;
-			no-map;
-		};
-
-		pld_pdp_mem: pld-pdp@81f37000 {
-			reg = <0x0 0x81f37000 0x0 0x1000>;
-			no-map;
-		};
-
-		tz_stat_mem: tz-stat@82700000 {
-			reg = <0x0 0x82700000 0x0 0x100000>;
-			no-map;
-		};
-
-		xbl_tmp_buffer_mem: xbl-tmp-buffer@82800000 {
-			reg = <0x0 0x82800000 0x0 0xc00000>;
-			no-map;
-		};
-
 		adsp_rpc_remote_heap_mem: adsp-rpc-remote-heap@84b00000 {
 			reg = <0x0 0x84b00000 0x0 0x800000>;
 			no-map;
@@ -2868,7 +2810,7 @@ usb_1_ss0_hsphy: phy@fd3000 {
 			reg = <0 0x00fd3000 0 0x154>;
 			#phy-cells = <0>;
 
-			clocks = <&tcsr TCSR_USB2_1_CLKREF_EN>;
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
 			clock-names = "ref";
 
 			resets = <&gcc GCC_QUSB2PHY_PRIM_BCR>;
@@ -3010,7 +2952,7 @@ usb_1_ss2_hsphy: phy@fde000 {
 			reg = <0 0x00fde000 0 0x154>;
 			#phy-cells = <0>;
 
-			clocks = <&tcsr TCSR_USB2_1_CLKREF_EN>;
+			clocks = <&tcsr TCSR_USB2_2_CLKREF_EN>;
 			clock-names = "ref";
 
 			resets = <&gcc GCC_QUSB2PHY_TERT_BCR>;
@@ -4371,6 +4313,9 @@ fastrpc {
 					compatible = "qcom,fastrpc";
 					qcom,glink-channels = "fastrpcglink-apps-dsp";
 					label = "adsp";
+					memory-region = <&adsp_rpc_remote_heap_mem>;
+					qcom,vmids = <QCOM_SCM_VMID_LPASS
+						      QCOM_SCM_VMID_ADSP_HEAP>;
 					qcom,non-secure-domain;
 					#address-cells = <1>;
 					#size-cells = <0>;
@@ -5318,6 +5263,13 @@ usb_1_ss0: usb@a600000 {
 
 			resets = <&gcc GCC_USB30_PRIM_BCR>;
 
+			interconnects = <&usb_south_anoc MASTER_USB3_0 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_USB3_0 QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "usb-ddr",
+					     "apps-usb";
+
 			wakeup-source;
 
 			iommus = <&apps_smmu 0x1420 0x0>;
@@ -5550,6 +5502,22 @@ videocc: clock-controller@aaf0000 {
 			#power-domain-cells = <1>;
 		};
 
+		camcc: clock-controller@ade0000 {
+			compatible = "qcom,x1e80100-camcc";
+			reg = <0x0 0x0ade0000 0x0 0x20000>;
+			clocks = <&gcc GCC_CAMERA_AHB_CLK>,
+				 <&bi_tcxo_div2>,
+				 <&bi_tcxo_ao_div2>,
+				 <&sleep_clk>;
+			power-domains = <&rpmhpd RPMHPD_MXC>,
+					<&rpmhpd RPMHPD_MMCX>;
+			required-opps = <&rpmhpd_opp_low_svs>,
+					<&rpmhpd_opp_low_svs>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
 		mdss: display-subsystem@ae00000 {
 			compatible = "qcom,x1e80100-mdss";
 			reg = <0 0x0ae00000 0 0x1000>;
@@ -6047,7 +6015,7 @@ dispcc: clock-controller@af00000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,x1e80100-pdc", "qcom,pdc";
-			reg = <0 0x0b220000 0 0x30000>, <0 0x174000f0 0 0x64>;
+			reg = <0 0x0b220000 0 0x10000>, <0 0x174000f0 0 0x64>;
 
 			qcom,pdc-ranges = <0 480 42>, <42 251 5>,
 					  <47 522 52>, <99 609 32>,
@@ -6804,6 +6772,35 @@ data-pins {
 			};
 		};
 
+		ctcu@10001000 {
+			compatible = "qcom,x1e80100-ctcu", "qcom,sa8775p-ctcu";
+			reg = <0x0 0x10001000 0x0 0x1000>;
+
+			clocks = <&aoss_qmp>;
+			clock-names = "apb";
+
+			in-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					ctcu_in0: endpoint {
+						remote-endpoint = <&etr0_out>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					ctcu_in1: endpoint {
+						remote-endpoint = <&etr1_out>;
+					};
+				};
+			};
+		};
+
 		stm@10002000 {
 			compatible = "arm,coresight-stm", "arm,primecell";
 			reg = <0x0 0x10002000 0x0 0x1000>,
@@ -6829,6 +6826,7 @@ tpdm@10003000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_prng";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -6886,6 +6884,7 @@ tpdm@1000f000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_dcc";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -7018,12 +7017,129 @@ qdss_funnel_out: endpoint {
 			};
 		};
 
+		replicator@10046000 {
+			compatible = "arm,coresight-dynamic-replicator", "arm,primecell";
+			reg = <0x0 0x10046000 0x0 0x1000>;
+
+			clocks = <&aoss_qmp>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				port {
+					qdss_rep_in: endpoint {
+						remote-endpoint = <&swao_rep_out0>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					qdss_rep_out0: endpoint {
+						remote-endpoint = <&etr_rep_in>;
+					};
+				};
+			};
+		};
+
+		tmc_etr: tmc@10048000 {
+			compatible = "arm,coresight-tmc", "arm,primecell";
+			reg = <0x0 0x10048000 0x0 0x1000>;
+
+			clocks = <&aoss_qmp>;
+			clock-names = "apb_pclk";
+			iommus = <&apps_smmu 0x04e0 0x0>;
+
+			arm,scatter-gather;
+
+			in-ports {
+				port {
+					etr0_in: endpoint {
+						remote-endpoint = <&etr_rep_out0>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					etr0_out: endpoint {
+						remote-endpoint = <&ctcu_in0>;
+					};
+				};
+			};
+		};
+
+		replicator@1004e000 {
+			compatible = "arm,coresight-dynamic-replicator", "arm,primecell";
+			reg = <0x0 0x1004e000 0x0 0x1000>;
+
+			clocks = <&aoss_qmp>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				port {
+					etr_rep_in: endpoint {
+						remote-endpoint = <&qdss_rep_out0>;
+					};
+				};
+			};
+
+			out-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					etr_rep_out0: endpoint {
+						remote-endpoint = <&etr0_in>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					etr_rep_out1: endpoint {
+						remote-endpoint = <&etr1_in>;
+					};
+				};
+			};
+		};
+
+		tmc_etr1: tmc@1004f000 {
+			compatible = "arm,coresight-tmc", "arm,primecell";
+			reg = <0x0 0x1004f000 0x0 0x1000>;
+
+			clocks = <&aoss_qmp>;
+			clock-names = "apb_pclk";
+			iommus = <&apps_smmu 0x0500 0x0>;
+
+			arm,scatter-gather;
+			arm,buffer-size = <0x400000>;
+
+			in-ports {
+				port {
+					etr1_in: endpoint {
+						remote-endpoint = <&etr_rep_out1>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					etr1_out: endpoint {
+						remote-endpoint = <&ctcu_in1>;
+					};
+				};
+			};
+		};
+
 		tpdm@10800000 {
 			compatible = "qcom,coresight-tpdm", "arm,primecell";
 			reg = <0x0 0x10800000 0x0 0x1000>;
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_tmess_0";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -7043,6 +7159,7 @@ tpdm@1082c000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_dl_south_0";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -7062,6 +7179,7 @@ tpdm@10841000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_spdm";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -7081,6 +7199,7 @@ tpdm@10844000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_qm";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -7124,6 +7243,7 @@ cti@1098b000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_cdsp_cscti";
 		};
 
 		tpdm@109d0000 {
@@ -7132,6 +7252,7 @@ tpdm@109d0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_lpass";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -7152,6 +7273,7 @@ tpdm@10ac0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_gcc";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -7172,6 +7294,7 @@ tpdm@10ac1000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_rpdm_mxa";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -7331,7 +7454,20 @@ swao_rep_in: endpoint {
 			};
 
 			out-ports {
-				port {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					swao_rep_out0: endpoint {
+						remote-endpoint = <&qdss_rep_in>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
 					swao_rep_out1: endpoint {
 						remote-endpoint = <&eud_in>;
 					};
@@ -7406,6 +7542,7 @@ tpdm@10b09000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_dl_south_1";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -7425,6 +7562,7 @@ tpdm@10b0a000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_3";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -7444,6 +7582,7 @@ tpdm@10b0b000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_1";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -7463,6 +7602,7 @@ tpdm@10b0c000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_0";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -7482,6 +7622,7 @@ tpdm@10b0d000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_dlct";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -7501,6 +7642,7 @@ tpdm@10b20000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_dlmm";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -7571,6 +7713,7 @@ tpdm@10c08000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_1";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -7619,6 +7762,7 @@ tpdm@10c28000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_ddr_lpi";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -7638,6 +7782,7 @@ tpdm@10c29000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_2";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -7771,6 +7916,7 @@ tpdm@10c38000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_ipcc";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -7790,6 +7936,7 @@ tpdm@10c39000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_rdpm_mx";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -7886,6 +8033,7 @@ tpdm@10cc1000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_rdpm";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -7990,6 +8138,7 @@ tpdm@10d08000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_llcc_7";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -8009,6 +8158,7 @@ tpdm@10d09000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_llcc_6";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -8028,6 +8178,7 @@ tpdm@10d0a000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_llcc_5";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -8047,6 +8198,7 @@ tpdm@10d0b000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_llcc_4";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -8066,6 +8218,7 @@ tpdm@10d0c000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_llcc_3";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -8085,6 +8238,7 @@ tpdm@10d0d000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_llcc_0";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -8104,6 +8258,7 @@ tpdm@10d0e000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_llcc_1";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -8123,6 +8278,7 @@ tpdm@10d0f000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_llcc_2";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
diff --git a/arch/arm64/boot/dts/qcom/ipq5018.dtsi b/arch/arm64/boot/dts/qcom/ipq5018.dtsi
index 6f8004a22a1ff..b5bfd5dcabad4 100644
--- a/arch/arm64/boot/dts/qcom/ipq5018.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq5018.dtsi
@@ -229,6 +229,9 @@ ge_phy: ethernet-phy@7 {
 				compatible = "ethernet-phy-id004d.d0c0";
 				reg = <7>;
 
+				clocks = <&gcc GCC_GEPHY_RX_CLK>,
+					 <&gcc GCC_GEPHY_TX_CLK>;
+				clock-names = "rx", "tx";
 				resets = <&gcc GCC_GEPHY_MISC_ARES>;
 			};
 		};
@@ -256,7 +259,7 @@ cmn_pll: clock-controller@9b000 {
 				      "sys";
 			#clock-cells = <1>;
 			assigned-clocks = <&cmn_pll IPQ5018_CMN_PLL_CLK>;
-			assigned-clock-rates-u64 = /bits/ 64 <9600000000>;
+			assigned-clock-rates-u64 = /bits/ 64 <4800000000>;
 		};
 
 		qfprom: qfprom@a0000 {
@@ -453,6 +456,16 @@ tcsr: syscon@1937000 {
 			reg = <0x01937000 0x21000>;
 		};
 
+		pwm: pwm@1941010 {
+			compatible = "qcom,ipq5018-pwm", "qcom,ipq6018-pwm";
+			reg = <0x01941010 0x20>;
+			clocks = <&gcc GCC_ADSS_PWM_CLK>;
+			assigned-clocks = <&gcc GCC_ADSS_PWM_CLK>;
+			assigned-clock-rates = <100000000>;
+			#pwm-cells = <3>;
+			status = "disabled";
+		};
+
 		sdhc_1: mmc@7804000 {
 			compatible = "qcom,ipq5018-sdhci", "qcom,sdhci-msm-v5";
 			reg = <0x7804000 0x1000>;
diff --git a/arch/arm64/boot/dts/qcom/ipq5332-rdp-common.dtsi b/arch/arm64/boot/dts/qcom/ipq5332-rdp-common.dtsi
index 8967861be5fda..3bbf748f2e5c5 100644
--- a/arch/arm64/boot/dts/qcom/ipq5332-rdp-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq5332-rdp-common.dtsi
@@ -2,7 +2,7 @@
 /*
  * IPQ5332 RDP board common device tree source
  *
- * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 /dts-v1/;
@@ -55,12 +55,32 @@ &blsp1_uart0 {
 	status = "okay";
 };
 
+/*
+ * The bootstrap pins for the board select the XO clock frequency that
+ * supports 48 MHZ or 96 MHZ. This setting automatically enables the
+ * right dividers, to ensure the reference clock output from WiFi to
+ * the CMN PLL is 48 MHZ.
+ */
+&ref_48mhz_clk {
+	clock-div = <1>;
+	clock-mult = <1>;
+};
+
 &sleep_clk {
 	clock-frequency = <32000>;
 };
 
+/*
+ * The frequency of xo_board is fixed to 24 MHZ, which is routed
+ * from WiFi output clock 48 MHZ divided by 2.
+ */
 &xo_board {
-	clock-frequency = <24000000>;
+	clock-div = <2>;
+	clock-mult = <1>;
+};
+
+&xo_clk {
+	clock-frequency = <48000000>;
 };
 
 /* PINCTRL */
diff --git a/arch/arm64/boot/dts/qcom/ipq5332-rdp441.dts b/arch/arm64/boot/dts/qcom/ipq5332-rdp441.dts
index 79ec77cfe5527..7fcf632e289f0 100644
--- a/arch/arm64/boot/dts/qcom/ipq5332-rdp441.dts
+++ b/arch/arm64/boot/dts/qcom/ipq5332-rdp441.dts
@@ -36,9 +36,6 @@ &pcie0 {
 	pinctrl-0 = <&pcie0_default>;
 	pinctrl-names = "default";
 
-	perst-gpios = <&tlmm 38 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 39 GPIO_ACTIVE_LOW>;
-
 	status = "okay";
 };
 
@@ -46,13 +43,15 @@ &pcie0_phy {
 	status = "okay";
 };
 
+&pcie0_port0 {
+	reset-gpios = <&tlmm 38 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 39 GPIO_ACTIVE_LOW>;
+};
+
 &pcie1 {
 	pinctrl-0 = <&pcie1_default>;
 	pinctrl-names = "default";
 
-	perst-gpios = <&tlmm 47 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 48 GPIO_ACTIVE_LOW>;
-
 	status = "okay";
 };
 
@@ -60,6 +59,11 @@ &pcie1_phy {
 	status = "okay";
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 47 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 48 GPIO_ACTIVE_LOW>;
+};
+
 &tlmm {
 	i2c_1_pins: i2c-1-state {
 		pins = "gpio29", "gpio30";
diff --git a/arch/arm64/boot/dts/qcom/ipq5332.dtsi b/arch/arm64/boot/dts/qcom/ipq5332.dtsi
index e227730d99a66..74d9de8d7641d 100644
--- a/arch/arm64/boot/dts/qcom/ipq5332.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq5332.dtsi
@@ -2,10 +2,11 @@
 /*
  * IPQ5332 device tree source
  *
- * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #include <dt-bindings/clock/qcom,apss-ipq.h>
+#include <dt-bindings/clock/qcom,ipq5332-cmn-pll.h>
 #include <dt-bindings/clock/qcom,ipq5332-gcc.h>
 #include <dt-bindings/interconnect/qcom,ipq5332.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
@@ -16,12 +17,24 @@ / {
 	#size-cells = <2>;
 
 	clocks {
+		ref_48mhz_clk: ref-48mhz-clk {
+			compatible = "fixed-factor-clock";
+			clocks = <&xo_clk>;
+			#clock-cells = <0>;
+		};
+
 		sleep_clk: sleep-clk {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
 		};
 
 		xo_board: xo-board-clk {
+			compatible = "fixed-factor-clock";
+			clocks = <&ref_48mhz_clk>;
+			#clock-cells = <0>;
+		};
+
+		xo_clk: xo-clk {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
 		};
@@ -167,6 +180,20 @@ usbphy0: phy@7b000 {
 			status = "disabled";
 		};
 
+		cmn_pll: clock-controller@9b000 {
+			compatible = "qcom,ipq5332-cmn-pll";
+			reg = <0x0009b000 0x800>;
+			clocks = <&ref_48mhz_clk>,
+				 <&gcc GCC_CMN_12GPLL_AHB_CLK>,
+				 <&gcc GCC_CMN_12GPLL_SYS_CLK>;
+			clock-names = "ref",
+				      "ahb",
+				      "sys";
+			#clock-cells = <1>;
+			assigned-clocks = <&cmn_pll IPQ5332_CMN_PLL_CLK>;
+			assigned-clock-rates-u64 = /bits/ 64 <6000000000>;
+		};
+
 		qfprom: efuse@a4000 {
 			compatible = "qcom,ipq5332-qfprom", "qcom,qfprom";
 			reg = <0x000a4000 0x721>;
@@ -334,6 +361,16 @@ tcsr: syscon@1937000 {
 			reg = <0x01937000 0x21000>;
 		};
 
+		pwm: pwm@1941010 {
+			compatible = "qcom,ipq5332-pwm", "qcom,ipq6018-pwm";
+			reg = <0x01941010 0x20>;
+			clocks = <&gcc GCC_ADSS_PWM_CLK>;
+			assigned-clocks = <&gcc GCC_ADSS_PWM_CLK>;
+			assigned-clock-rates = <100000000>;
+			#pwm-cells = <3>;
+			status = "disabled";
+		};
+
 		sdhc: mmc@7804000 {
 			compatible = "qcom,ipq5332-sdhci", "qcom,sdhci-msm-v5";
 			reg = <0x07804000 0x1000>, <0x07805000 0x1000>;
@@ -704,19 +741,18 @@ pcie1: pcie@18000000 {
 				      "aux",
 				      "ahb";
 
-			phys = <&pcie1_phy>;
-			phy-names = "pciephy";
-
 			interconnects = <&gcc MASTER_SNOC_PCIE3_2_M &gcc SLAVE_SNOC_PCIE3_2_M>,
 					<&gcc MASTER_ANOC_PCIE3_2_S &gcc SLAVE_ANOC_PCIE3_2_S>;
 			interconnect-names = "pcie-mem", "cpu-pcie";
 
 			status = "disabled";
 
-			pcie@0 {
+			pcie1_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 
+				phys = <&pcie1_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -808,19 +844,18 @@ pcie0: pcie@20000000 {
 				      "aux",
 				      "ahb";
 
-			phys = <&pcie0_phy>;
-			phy-names = "pciephy";
-
 			interconnects = <&gcc MASTER_SNOC_PCIE3_1_M &gcc SLAVE_SNOC_PCIE3_1_M>,
 					<&gcc MASTER_ANOC_PCIE3_1_S &gcc SLAVE_ANOC_PCIE3_1_S>;
 			interconnect-names = "pcie-mem", "cpu-pcie";
 
 			status = "disabled";
 
-			pcie@0 {
+			pcie0_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 
+				phys = <&pcie0_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
diff --git a/arch/arm64/boot/dts/qcom/ipq5424-rdp466.dts b/arch/arm64/boot/dts/qcom/ipq5424-rdp466.dts
index de71b72ae6dcc..be8657239c46b 100644
--- a/arch/arm64/boot/dts/qcom/ipq5424-rdp466.dts
+++ b/arch/arm64/boot/dts/qcom/ipq5424-rdp466.dts
@@ -86,8 +86,6 @@ &pcie2 {
 	pinctrl-0 = <&pcie2_default_state>;
 	pinctrl-names = "default";
 
-	perst-gpios = <&tlmm 31 GPIO_ACTIVE_LOW>;
-
 	status = "okay";
 };
 
@@ -95,12 +93,14 @@ &pcie2_phy {
 	status = "okay";
 };
 
+&pcie2_port0 {
+	reset-gpios = <&tlmm 31 GPIO_ACTIVE_LOW>;
+};
+
 &pcie3 {
 	pinctrl-0 = <&pcie3_default_state>;
 	pinctrl-names = "default";
 
-	perst-gpios = <&tlmm 34 GPIO_ACTIVE_LOW>;
-
 	status = "okay";
 };
 
@@ -108,6 +108,10 @@ &pcie3_phy {
 	status = "okay";
 };
 
+&pcie3_port0 {
+	reset-gpios = <&tlmm 34 GPIO_ACTIVE_LOW>;
+};
+
 &qusb_phy_0 {
 	vdd-supply = <&vreg_misc_0p925>;
 	vdda-pll-supply = <&vreg_misc_1p8>;
diff --git a/arch/arm64/boot/dts/qcom/ipq5424.dtsi b/arch/arm64/boot/dts/qcom/ipq5424.dtsi
index 876bf6a8b8ff0..702061e16a58d 100644
--- a/arch/arm64/boot/dts/qcom/ipq5424.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq5424.dtsi
@@ -968,19 +968,19 @@ pcie3: pcie@40000000 {
 				      "aux",
 				      "ahb";
 
-			phys = <&pcie3_phy>;
-			phy-names = "pciephy";
 			interconnects = <&gcc MASTER_ANOC_PCIE3 &gcc SLAVE_ANOC_PCIE3>,
 					<&gcc MASTER_CNOC_PCIE3 &gcc SLAVE_CNOC_PCIE3>;
 			interconnect-names = "pcie-mem", "cpu-pcie";
 
 			status = "disabled";
 
-			pcie@0 {
+			pcie3_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie3_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -1071,19 +1071,19 @@ pcie2: pcie@50000000 {
 				      "aux",
 				      "ahb";
 
-			phys = <&pcie2_phy>;
-			phy-names = "pciephy";
 			interconnects = <&gcc MASTER_ANOC_PCIE2 &gcc SLAVE_ANOC_PCIE2>,
 					<&gcc MASTER_CNOC_PCIE2 &gcc SLAVE_CNOC_PCIE2>;
 			interconnect-names = "pcie-mem", "cpu-pcie";
 
 			status = "disabled";
 
-			pcie@0 {
+			pcie2_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie2_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -1174,19 +1174,19 @@ pcie1: pcie@60000000 {
 				      "aux",
 				      "ahb";
 
-			phys = <&pcie1_phy>;
-			phy-names = "pciephy";
 			interconnects = <&gcc MASTER_ANOC_PCIE1	&gcc SLAVE_ANOC_PCIE1>,
 					<&gcc MASTER_CNOC_PCIE1	&gcc SLAVE_CNOC_PCIE1>;
 			interconnect-names = "pcie-mem", "cpu-pcie";
 
 			status = "disabled";
 
-			pcie@0 {
+			pcie1_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie1_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -1277,19 +1277,19 @@ pcie0: pcie@70000000 {
 				      "aux",
 				      "ahb";
 
-			phys = <&pcie0_phy>;
-			phy-names = "pciephy";
 			interconnects = <&gcc MASTER_ANOC_PCIE0 &gcc SLAVE_ANOC_PCIE0>,
 					<&gcc MASTER_CNOC_PCIE0	&gcc SLAVE_CNOC_PCIE0>;
 			interconnect-names = "pcie-mem", "cpu-pcie";
 
 			status = "disabled";
 
-			pcie@0 {
+			pcie0_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie0_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
diff --git a/arch/arm64/boot/dts/qcom/ipq6018.dtsi b/arch/arm64/boot/dts/qcom/ipq6018.dtsi
index 40f1c262126ef..7866844cc09fd 100644
--- a/arch/arm64/boot/dts/qcom/ipq6018.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq6018.dtsi
@@ -413,6 +413,16 @@ tcsr: syscon@1937000 {
 			reg = <0x0 0x01937000 0x0 0x21000>;
 		};
 
+		pwm: pwm@1941010 {
+			compatible = "qcom,ipq6018-pwm";
+			reg = <0x0 0x01941010 0x0 0x20>;
+			clocks = <&gcc GCC_ADSS_PWM_CLK>;
+			assigned-clocks = <&gcc GCC_ADSS_PWM_CLK>;
+			assigned-clock-rates = <100000000>;
+			#pwm-cells = <3>;
+			status = "disabled";
+		};
+
 		usb2: usb@70f8800 {
 			compatible = "qcom,ipq6018-dwc3", "qcom,dwc3";
 			reg = <0x0 0x070f8800 0x0 0x400>;
diff --git a/arch/arm64/boot/dts/qcom/ipq9574.dtsi b/arch/arm64/boot/dts/qcom/ipq9574.dtsi
index 622cfa96ed2b3..451c2076f6a7c 100644
--- a/arch/arm64/boot/dts/qcom/ipq9574.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq9574.dtsi
@@ -445,6 +445,16 @@ tcsr: syscon@1937000 {
 			reg = <0x01937000 0x21000>;
 		};
 
+		pwm: pwm@1941010 {
+			compatible = "qcom,ipq9574-pwm", "qcom,ipq6018-pwm";
+			reg = <0x01941010 0x20>;
+			clocks = <&gcc GCC_ADSS_PWM_CLK>;
+			assigned-clocks = <&gcc GCC_ADSS_PWM_CLK>;
+			assigned-clock-rates = <100000000>;
+			#pwm-cells = <3>;
+			status = "disabled";
+		};
+
 		sdhc_1: mmc@7804000 {
 			compatible = "qcom,ipq9574-sdhci", "qcom,sdhci-msm-v5";
 			reg = <0x07804000 0x1000>,
@@ -945,7 +955,8 @@ pcie1: pcie@10000000 {
 				     <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
+				     <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "msi0",
 					  "msi1",
 					  "msi2",
@@ -953,7 +964,8 @@ pcie1: pcie@10000000 {
 					  "msi4",
 					  "msi5",
 					  "msi6",
-					  "msi7";
+					  "msi7",
+					  "global";
 
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 0x7>;
@@ -1031,7 +1043,8 @@ pcie3: pcie@18000000 {
 				     <GIC_SPI 326 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 415 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 494 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 495 IRQ_TYPE_LEVEL_HIGH>;
+				     <GIC_SPI 495 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "msi0",
 					  "msi1",
 					  "msi2",
@@ -1039,7 +1052,8 @@ pcie3: pcie@18000000 {
 					  "msi4",
 					  "msi5",
 					  "msi6",
-					  "msi7";
+					  "msi7",
+					  "global";
 
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 0x7>;
@@ -1117,7 +1131,8 @@ pcie2: pcie@20000000 {
 				     <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>;
+				     <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "msi0",
 					  "msi1",
 					  "msi2",
@@ -1125,7 +1140,8 @@ pcie2: pcie@20000000 {
 					  "msi4",
 					  "msi5",
 					  "msi6",
-					  "msi7";
+					  "msi7",
+					  "global";
 
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 0x7>;
@@ -1202,7 +1218,8 @@ pcie0: pcie@28000000 {
 				     <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>;
+				     <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "msi0",
 					  "msi1",
 					  "msi2",
@@ -1210,7 +1227,8 @@ pcie0: pcie@28000000 {
 					  "msi4",
 					  "msi5",
 					  "msi6",
-					  "msi7";
+					  "msi7",
+					  "global";
 
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 0x7>;
diff --git a/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts b/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts
index f9b5b5718b904..7ccff3d03b78b 100644
--- a/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts
+++ b/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts
@@ -163,6 +163,49 @@ platform {
 		};
 	};
 
+	pmic-glink {
+		compatible = "qcom,kaanapali-pmic-glink",
+			     "qcom,pmic-glink";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in: endpoint {
+						remote-endpoint = <&usb_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss_in: endpoint {
+						remote-endpoint = <&usb_dp_qmpphy_out>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_sbu: endpoint {
+					};
+				};
+			};
+		};
+	};
+
 	wcd939x: audio-codec {
 		compatible = "qcom,wcd9395-codec", "qcom,wcd9390-codec";
 
@@ -1076,6 +1119,11 @@ &remoteproc_cdsp {
 	status = "okay";
 };
 
+&remoteproc_soccp {
+	firmware-name = "qcom/kaanapali/soccp.mbn",
+			"qcom/kaanapali/soccp_dtb.mbn";
+};
+
 &sdhc_2 {
 	cd-gpios = <&tlmm 55 GPIO_ACTIVE_LOW>;
 
@@ -1328,8 +1376,6 @@ &ufs_mem_phy {
 };
 
 &usb {
-	dr_mode = "peripheral";
-
 	status = "okay";
 };
 
@@ -1348,3 +1394,11 @@ &usb_dp_qmpphy {
 
 	status = "okay";
 };
+
+&usb_dp_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss_in>;
+};
+
+&usb_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in>;
+};
diff --git a/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts b/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts
index 55d02219ef4e9..6efbe45605a04 100644
--- a/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts
+++ b/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts
@@ -80,6 +80,49 @@ key-volume-up {
 			wakeup-source;
 		};
 	};
+
+	pmic-glink {
+		compatible = "qcom,kaanapali-pmic-glink",
+			     "qcom,pmic-glink";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in: endpoint {
+						remote-endpoint = <&usb_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss_in: endpoint {
+						remote-endpoint = <&usb_dp_qmpphy_out>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_sbu: endpoint {
+					};
+				};
+			};
+		};
+	};
 };
 
 &apps_rsc {
@@ -786,6 +829,11 @@ &remoteproc_cdsp {
 	status = "okay";
 };
 
+&remoteproc_soccp {
+	firmware-name = "qcom/kaanapali/soccp.mbn",
+			"qcom/kaanapali/soccp_dtb.mbn";
+};
+
 &tlmm {
 	gpio-reserved-ranges = <36 4>, /* NFC eSE SPI */
 			       <74 1>, /* eSE */
@@ -823,8 +871,6 @@ &ufs_mem_phy {
 };
 
 &usb {
-	dr_mode = "peripheral";
-
 	status = "okay";
 };
 
@@ -843,3 +889,11 @@ &usb_dp_qmpphy {
 
 	status = "okay";
 };
+
+&usb_dp_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss_in>;
+};
+
+&usb_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in>;
+};
diff --git a/arch/arm64/boot/dts/qcom/kaanapali.dtsi b/arch/arm64/boot/dts/qcom/kaanapali.dtsi
index 7aa9653bd456e..74699bc3eadbe 100644
--- a/arch/arm64/boot/dts/qcom/kaanapali.dtsi
+++ b/arch/arm64/boot/dts/qcom/kaanapali.dtsi
@@ -45,9 +45,9 @@ cpu0: cpu@0 {
 			reg = <0x0 0x0>;
 			enable-method = "psci";
 			next-level-cache = <&l2_0>;
-			power-domains = <&cpu_pd0>;
-			power-domain-names = "psci";
-			clocks = <&pdp_scmi_perf 0>;
+			power-domains = <&cpu_pd0>, <&scmi_perf 0>;
+			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 
 			l2_0: l2-cache {
 				compatible = "cache";
@@ -62,9 +62,9 @@ cpu1: cpu@100 {
 			reg = <0x0 0x100>;
 			enable-method = "psci";
 			next-level-cache = <&l2_0>;
-			power-domains = <&cpu_pd1>;
-			power-domain-names = "psci";
-			clocks = <&pdp_scmi_perf 0>;
+			power-domains = <&cpu_pd1>, <&scmi_perf 0>;
+			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu2: cpu@200 {
@@ -73,9 +73,9 @@ cpu2: cpu@200 {
 			reg = <0x0 0x200>;
 			enable-method = "psci";
 			next-level-cache = <&l2_0>;
-			power-domains = <&cpu_pd2>;
-			power-domain-names = "psci";
-			clocks = <&pdp_scmi_perf 0>;
+			power-domains = <&cpu_pd2>, <&scmi_perf 0>;
+			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu3: cpu@300 {
@@ -84,9 +84,9 @@ cpu3: cpu@300 {
 			reg = <0x0 0x300>;
 			enable-method = "psci";
 			next-level-cache = <&l2_0>;
-			power-domains = <&cpu_pd3>;
-			power-domain-names = "psci";
-			clocks = <&pdp_scmi_perf 0>;
+			power-domains = <&cpu_pd3>, <&scmi_perf 0>;
+			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu4: cpu@400 {
@@ -95,9 +95,9 @@ cpu4: cpu@400 {
 			reg = <0x0 0x400>;
 			enable-method = "psci";
 			next-level-cache = <&l2_0>;
-			power-domains = <&cpu_pd4>;
-			power-domain-names = "psci";
-			clocks = <&pdp_scmi_perf 0>;
+			power-domains = <&cpu_pd4>, <&scmi_perf 0>;
+			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu5: cpu@500 {
@@ -106,9 +106,9 @@ cpu5: cpu@500 {
 			reg = <0x0 0x500>;
 			enable-method = "psci";
 			next-level-cache = <&l2_0>;
-			power-domains = <&cpu_pd5>;
-			power-domain-names = "psci";
-			clocks = <&pdp_scmi_perf 0>;
+			power-domains = <&cpu_pd5>, <&scmi_perf 0>;
+			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu6: cpu@10000 {
@@ -117,9 +117,9 @@ cpu6: cpu@10000 {
 			reg = <0x0 0x10000>;
 			enable-method = "psci";
 			next-level-cache = <&l2_1>;
-			power-domains = <&cpu_pd6>;
-			power-domain-names = "psci";
-			clocks = <&pdp_scmi_perf 1>;
+			power-domains = <&cpu_pd6>, <&scmi_perf 1>;
+			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 
 			l2_1: l2-cache {
 				compatible = "cache";
@@ -134,9 +134,9 @@ cpu7: cpu@10100 {
 			reg = <0x0 0x10100>;
 			enable-method = "psci";
 			next-level-cache = <&l2_1>;
-			power-domains = <&cpu_pd7>;
-			power-domain-names = "psci";
-			clocks = <&pdp_scmi_perf 1>;
+			power-domains = <&cpu_pd7>, <&scmi_perf 1>;
+			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu-map {
@@ -235,9 +235,9 @@ scmi: scmi {
 			#address-cells = <1>;
 			#size-cells = <0>;
 
-			pdp_scmi_perf: protocol@13 {
+			scmi_perf: protocol@13 {
 				reg = <0x13>;
-				#clock-cells = <1>;
+				#power-domain-cells = <1>;
 			};
 		};
 	};
@@ -504,6 +504,32 @@ smp2p_cdsp_in: slave-kernel {
 		};
 	};
 
+	smp2p-soccp {
+		compatible = "qcom,smp2p";
+
+		interrupts-extended = <&ipcc IPCC_MPROC_SOCCP
+					     IPCC_MPROC_SIGNAL_SMP2P
+					     IRQ_TYPE_EDGE_RISING>;
+
+		mboxes = <&ipcc IPCC_MPROC_SOCCP
+				IPCC_MPROC_SIGNAL_SMP2P>;
+
+		qcom,smem = <617>, <616>;
+		qcom,local-pid = <0>;
+		qcom,remote-pid = <19>;
+
+		soccp_smp2p_out: master-kernel {
+			qcom,entry-name = "master-kernel";
+			#qcom,smem-state-cells = <1>;
+		};
+
+		soccp_smp2p_in: slave-kernel {
+			qcom,entry-name = "slave-kernel";
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+
 	soc: soc@0 {
 		compatible = "simple-bus";
 
@@ -1513,6 +1539,55 @@ &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
 			};
 		};
 
+		remoteproc_soccp: remoteproc@d00000 {
+			compatible = "qcom,kaanapali-soccp-pas";
+			reg = <0x0 0x00d00000 0x0 0x200000>;
+
+			interrupts-extended = <&intc GIC_SPI 167 IRQ_TYPE_EDGE_RISING>,
+					      <&soccp_smp2p_in 0 IRQ_TYPE_EDGE_RISING>,
+					      <&soccp_smp2p_in 1 IRQ_TYPE_EDGE_RISING>,
+					      <&soccp_smp2p_in 2 IRQ_TYPE_EDGE_RISING>,
+					      <&soccp_smp2p_in 3 IRQ_TYPE_EDGE_RISING>,
+					      <&soccp_smp2p_in 9 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "wdog",
+					  "fatal",
+					  "ready",
+					  "handover",
+					  "stop-ack",
+					  "pong";
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+			clock-names = "xo";
+
+			power-domains = <&rpmhpd RPMHPD_CX>,
+					<&rpmhpd RPMHPD_MX>;
+			power-domain-names = "cx",
+					     "mx";
+
+			memory-region = <&soccp_mem>,
+					<&soccp_dtb_mem>;
+
+			qcom,smem-states = <&soccp_smp2p_out 0>,
+					   <&soccp_smp2p_out 8>;
+			qcom,smem-state-names = "stop",
+						"ping";
+
+			glink-edge {
+				interrupts-extended = <&ipcc IPCC_MPROC_SOCCP
+							     IPCC_MPROC_SIGNAL_GLINK_QMP
+							     IRQ_TYPE_EDGE_RISING>;
+				mboxes = <&ipcc IPCC_MPROC_SOCCP
+						IPCC_MPROC_SIGNAL_GLINK_QMP>;
+				qcom,remote-pid = <19>;
+				label = "soccp";
+			};
+		};
+
+		rng: rng@10c3000 {
+			compatible = "qcom,kaanapali-trng", "qcom,trng";
+			reg = <0x0 0x010c3000 0x0 0x1000>;
+		};
+
 		ipcc: mailbox@1106000 {
 			compatible = "qcom,kaanapali-ipcc", "qcom,ipcc";
 			reg = <0x0 0x01106000 0x0 0x1000>;
@@ -2545,6 +2620,31 @@ ice: crypto@1d88000 {
 			power-domains = <&gcc GCC_UFS_PHY_GDSC>;
 		};
 
+		cryptobam: dma-controller@1dc4000 {
+			compatible = "qcom,bam-v2.0.0";
+			reg = <0x0 0x01dc4000 0x0 0x22000>;
+			interrupts = <GIC_SPI 272 IRQ_TYPE_LEVEL_HIGH>;
+			#dma-cells = <1>;
+			iommus = <&apps_smmu 0xc0 0x0>,
+				 <&apps_smmu 0xc1 0x0>;
+			qcom,ee = <0>;
+			qcom,num-ees = <4>;
+			num-channels = <20>;
+			qcom,controlled-remotely;
+		};
+
+		crypto: crypto@1dfa000 {
+			compatible = "qcom,kaanapali-qce", "qcom,sm8150-qce", "qcom,qce";
+			reg = <0x0 0x01dfa000 0x0 0x6000>;
+			interconnects = <&aggre_noc MASTER_CRYPTO QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "memory";
+			dmas = <&cryptobam 4>, <&cryptobam 5>;
+			dma-names = "rx", "tx";
+			iommus = <&apps_smmu 0xc0 0x0>,
+				 <&apps_smmu 0xc1 0x0>;
+		};
+
 		tcsr_mutex: hwlock@1f40000 {
 			compatible = "qcom,tcsr-mutex";
 			reg = <0x0 0x01f40000 0x0 0x20000>;
@@ -3557,6 +3657,9 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
 			tx-fifo-resize;
 			dma-coherent;
 
+			usb-role-switch;
+			wakeup-source;
+
 			status = "disabled";
 
 			ports {
@@ -4359,6 +4462,7 @@ tpdm@10003000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_dcc";
 
 			qcom,cmb-element-bits = <32>;
 
@@ -4414,6 +4518,7 @@ tpdm@1000f000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_spdm";
 
 			qcom,cmb-element-bits = <32>;
 
@@ -4477,6 +4582,7 @@ tpdm@11000000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_modem_0";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -4565,6 +4671,7 @@ tpdm@1102c000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_gcc";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -4583,6 +4690,7 @@ tpdm@11180000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_cdsp";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -4602,6 +4710,7 @@ tpdm@11183000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_cdsp_cmsr1";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,cmb-element-bits = <32>;
@@ -4621,6 +4730,7 @@ tpdm@11184000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_cdsp_cmsr2";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,cmb-element-bits = <32>;
@@ -4640,6 +4750,7 @@ tpdm@11185000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_cdsp_dpm1";
 
 			qcom,cmb-element-bits = <64>;
 
@@ -4658,6 +4769,7 @@ tpdm@11186000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_cdsp_dpm2";
 
 			qcom,cmb-element-bits = <64>;
 
@@ -4777,6 +4889,7 @@ tpdm@111a3000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_pmu";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -4795,6 +4908,7 @@ tpdm@111a4000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_qrng";
 
 			out-ports {
 				port {
@@ -4811,6 +4925,7 @@ tpdm@111a5000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_dlmm";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -4829,6 +4944,7 @@ tpdm@111a6000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_north_dsb";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -4847,6 +4963,7 @@ tpdm@111a7000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_south_dsb";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -4865,6 +4982,7 @@ tpdm@111a8000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_rdpm_cx";
 
 			out-ports {
 				port {
@@ -4881,6 +4999,7 @@ tpdm@111a9000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_rdpm_mxa";
 
 			out-ports {
 				port {
@@ -4897,6 +5016,7 @@ tpdm@111aa000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_rdpm_mxc";
 
 			out-ports {
 				port {
@@ -4913,6 +5033,7 @@ tpdm@111ab000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_ipcc_cmb0";
 
 			out-ports {
 				port {
@@ -4929,6 +5050,7 @@ tpdm@111ac000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_ipcc_cmb1";
 
 			out-ports {
 				port {
@@ -4945,6 +5067,7 @@ tpdm@111ad000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_ipcc_cmb2";
 
 			out-ports {
 				port {
@@ -4961,6 +5084,7 @@ tpdm@111ae000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_ipcc_cmb3";
 
 			out-ports {
 				port {
@@ -4977,6 +5101,7 @@ tpdm@111af000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_ipcc_cmb4";
 
 			out-ports {
 				port {
@@ -4993,6 +5118,7 @@ tpdm@111b3000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_pcie_rscc";
 
 			out-ports {
 				port {
@@ -5182,6 +5308,7 @@ tpdm@111d0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_qm";
 
 			out-ports {
 				port {
@@ -5198,6 +5325,7 @@ tpdm@11303000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_4";
 
 			qcom,cmb-element-bits = <64>;
 
@@ -5339,6 +5467,7 @@ tpdm@11309000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_0";
 
 			qcom,cmb-element-bits = <64>;
 
@@ -5357,6 +5486,7 @@ tpdm@1130a000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_1";
 
 			qcom,cmb-element-bits = <64>;
 
@@ -5375,6 +5505,7 @@ tpdm@1130b000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_2";
 
 			qcom,cmb-element-bits = <64>;
 
@@ -5393,6 +5524,7 @@ tpdm@1130c000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_3";
 
 			qcom,cmb-element-bits = <64>;
 
@@ -5411,6 +5543,7 @@ tpdm@1130d000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_1";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -5430,6 +5563,7 @@ tpdm@11422000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_ipa";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -7116,6 +7250,7 @@ timer {
 
 	tpdm-cdsp-llm {
 		compatible = "qcom,coresight-static-tpdm";
+		label = "tpdm_cdsp_llm";
 		qcom,cmb-element-bits = <32>;
 
 		out-ports {
@@ -7129,6 +7264,7 @@ tpdm_cdsp_llm_out: endpoint {
 
 	tpdm-cdsp-llm2 {
 		compatible = "qcom,coresight-static-tpdm";
+		label = "tpdm_cdsp_llm2";
 		qcom,cmb-element-bits = <32>;
 
 		out-ports {
@@ -7142,6 +7278,7 @@ tpdm_cdsp_llm2_out: endpoint {
 
 	tpdm-modem1 {
 		compatible = "qcom,coresight-static-tpdm";
+		label = "tpdm_modem_1";
 		qcom,cmb-element-bits = <32>;
 
 		out-ports {
@@ -7155,6 +7292,7 @@ tpdm_modem1_out: endpoint {
 
 	tpdm-modem2 {
 		compatible = "qcom,coresight-static-tpdm";
+		label = "tpdm_modem_2";
 		qcom,cmb-element-bits = <64>;
 
 		out-ports {
diff --git a/arch/arm64/boot/dts/qcom/kodiak.dtsi b/arch/arm64/boot/dts/qcom/kodiak.dtsi
index fa540d8c2615d..7e73348fe0d02 100644
--- a/arch/arm64/boot/dts/qcom/kodiak.dtsi
+++ b/arch/arm64/boot/dts/qcom/kodiak.dtsi
@@ -191,9 +191,12 @@ rmtfs_mem: rmtfs@9c900000 {
 			qcom,vmid = <QCOM_SCM_VMID_MSS_MSA>;
 		};
 
-		adsp_rpc_remote_heap_mem: adsp-rpc-remote-heap@9cb80000 {
-			reg = <0x0 0x9cb80000 0x0 0x800000>;
-			no-map;
+		adsp_rpc_remote_heap_mem: adsp-rpc-remote-heap {
+			compatible = "shared-dma-pool";
+			alloc-ranges = <0x0 0x80000000 0x0 0x80000000>;
+			reusable;
+			alignment = <0x0 0x400000>;
+			size = <0x0 0x800000>;
 		};
 	};
 
@@ -2286,9 +2289,6 @@ pcie0: pcie@1c00000 {
 
 			power-domains = <&gcc GCC_PCIE_0_GDSC>;
 
-			phys = <&pcie0_phy>;
-			phy-names = "pciephy";
-
 			pinctrl-names = "default";
 			pinctrl-0 = <&pcie0_clkreq_n>;
 			dma-coherent;
@@ -2300,6 +2300,8 @@ pcie0_port: pcie@0 {
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie0_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -2416,9 +2418,6 @@ pcie1: pcie@1c08000 {
 
 			power-domains = <&gcc GCC_PCIE_1_GDSC>;
 
-			phys = <&pcie1_phy>;
-			phy-names = "pciephy";
-
 			pinctrl-names = "default";
 			pinctrl-0 = <&pcie1_clkreq_n>;
 
@@ -2434,6 +2433,8 @@ pcie1_port0: pcie@0 {
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie1_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -2646,6 +2647,9 @@ ipa: ipa@1e40000 {
 			qcom,smem-state-names = "ipa-clock-enabled-valid",
 						"ipa-clock-enabled";
 
+			qcom,gsi-loader = "self";
+			memory-region = <&ipa_fw_mem>;
+
 			sram = <&ipa_modem_tables>;
 
 			status = "disabled";
@@ -3517,6 +3521,7 @@ tpdm@600f000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_spdm";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -3536,6 +3541,7 @@ cti@6010000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdss";
 		};
 
 		funnel@6041000 {
@@ -3685,6 +3691,7 @@ cti@6b00000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_swao";
 		};
 
 		cti@6b01000 {
@@ -3693,6 +3700,7 @@ cti@6b01000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_swao_1";
 		};
 
 		cti@6b02000 {
@@ -3701,6 +3709,7 @@ cti@6b02000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_swao_2";
 		};
 
 		cti@6b03000 {
@@ -3709,6 +3718,7 @@ cti@6b03000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_swao_3";
 		};
 
 		funnel@6b04000 {
@@ -3863,6 +3873,7 @@ tpdm@6b09000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_0";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -3882,6 +3893,7 @@ tpdm@6b0a000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_1";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -3901,6 +3913,7 @@ tpdm@6b0b000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_2";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -3920,6 +3933,7 @@ tpdm@6b0c000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_3";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -3939,6 +3953,7 @@ tpdm@6b0d000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_1";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -3958,6 +3973,7 @@ cti@6b11000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_aoss";
 		};
 
 		etm@7040000 {
@@ -5743,7 +5759,7 @@ opp-810000000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sc7280-pdc", "qcom,pdc";
-			reg = <0 0x0b220000 0 0x30000>;
+			reg = <0 0x0b220000 0 0x10000>;
 			qcom,pdc-ranges = <0 480 40>, <40 140 14>, <54 263 1>,
 					  <55 306 4>, <59 312 3>, <62 374 2>,
 					  <64 434 2>, <66 438 3>, <69 86 1>,
diff --git a/arch/arm64/boot/dts/qcom/lemans-auto.dtsi b/arch/arm64/boot/dts/qcom/lemans-auto.dtsi
deleted file mode 100644
index 8db958d60fd1d..0000000000000
--- a/arch/arm64/boot/dts/qcom/lemans-auto.dtsi
+++ /dev/null
@@ -1,104 +0,0 @@
-// SPDX-License-Identifier: BSD-3-Clause
-/*
- * Copyright (c) 2023, Linaro Limited
- */
-
-/dts-v1/;
-
-#include "lemans.dtsi"
-
-/delete-node/ &pil_camera_mem;
-/delete-node/ &pil_adsp_mem;
-/delete-node/ &q6_adsp_dtb_mem;
-/delete-node/ &q6_gdsp0_dtb_mem;
-/delete-node/ &pil_gdsp0_mem;
-/delete-node/ &pil_gdsp1_mem;
-/delete-node/ &q6_gdsp1_dtb_mem;
-/delete-node/ &q6_cdsp0_dtb_mem;
-/delete-node/ &pil_cdsp0_mem;
-/delete-node/ &pil_gpu_mem;
-/delete-node/ &pil_cdsp1_mem;
-/delete-node/ &q6_cdsp1_dtb_mem;
-/delete-node/ &pil_cvp_mem;
-/delete-node/ &pil_video_mem;
-/delete-node/ &gunyah_md_mem;
-
-/ {
-	reserved-memory {
-		#address-cells = <2>;
-		#size-cells = <2>;
-		ranges;
-
-		tz_ffi_mem: tz-ffi@91c00000 {
-			compatible = "shared-dma-pool";
-			reg = <0x0 0x91c00000 0x0 0x1400000>;
-			no-map;
-		};
-
-		pil_camera_mem: pil-camera@95200000 {
-			reg = <0x0 0x95200000 0x0 0x500000>;
-			no-map;
-		};
-
-		pil_adsp_mem: pil-adsp@95c00000 {
-			reg = <0x0 0x95c00000 0x0 0x1e00000>;
-			no-map;
-		};
-
-		pil_gdsp0_mem: pil-gdsp0@97b00000 {
-			reg = <0x0 0x97b00000 0x0 0x1e00000>;
-			no-map;
-		};
-
-		pil_gdsp1_mem: pil-gdsp1@99900000 {
-			reg = <0x0 0x99900000 0x0 0x1e00000>;
-			no-map;
-		};
-
-		pil_cdsp0_mem: pil-cdsp0@9b800000 {
-			reg = <0x0 0x9b800000 0x0 0x1e00000>;
-			no-map;
-		};
-
-		pil_gpu_mem: pil-gpu@9d600000 {
-			reg = <0x0 0x9d600000 0x0 0x2000>;
-			no-map;
-		};
-
-		pil_cdsp1_mem: pil-cdsp1@9d700000 {
-			reg = <0x0 0x9d700000 0x0 0x1e00000>;
-			no-map;
-		};
-
-		pil_cvp_mem: pil-cvp@9f500000 {
-			reg = <0x0 0x9f500000 0x0 0x700000>;
-			no-map;
-		};
-
-		pil_video_mem: pil-video@9fc00000 {
-			reg = <0x0 0x9fc00000 0x0 0x700000>;
-			no-map;
-		};
-
-		audio_mdf_mem: audio-mdf-region@ae000000 {
-			reg = <0x0 0xae000000 0x0 0x1000000>;
-			no-map;
-		};
-
-		hyptz_reserved_mem: hyptz-reserved@beb00000 {
-			reg = <0x0 0xbeb00000 0x0 0x11500000>;
-			no-map;
-		};
-
-		trusted_apps_mem: trusted-apps@d1900000 {
-			reg = <0x0 0xd1900000 0x0 0x3800000>;
-			no-map;
-		};
-	};
-
-	firmware {
-		scm {
-			memory-region = <&tz_ffi_mem>;
-		};
-	};
-};
diff --git a/arch/arm64/boot/dts/qcom/lemans-evk-camera.dtso b/arch/arm64/boot/dts/qcom/lemans-evk-camera.dtso
index 4600d5441cce4..a4c6d39cf7778 100644
--- a/arch/arm64/boot/dts/qcom/lemans-evk-camera.dtso
+++ b/arch/arm64/boot/dts/qcom/lemans-evk-camera.dtso
@@ -14,7 +14,7 @@
 #include <dt-bindings/gpio/gpio.h>
 
 &{/} {
-	vreg_cam1_1p8: vreg_cam1_1p8 {
+	vreg_cam1_1p8: cam1-1p8-regulator {
 		compatible = "regulator-fixed";
 		regulator-name = "vreg_cam1_1p8";
 		startup-delay-us = <10000>;
diff --git a/arch/arm64/boot/dts/qcom/lemans-evk.dts b/arch/arm64/boot/dts/qcom/lemans-evk.dts
index 34dfc8d22b6a5..0c53640c42a65 100644
--- a/arch/arm64/boot/dts/qcom/lemans-evk.dts
+++ b/arch/arm64/boot/dts/qcom/lemans-evk.dts
@@ -21,6 +21,7 @@ aliases {
 		ethernet0 = &ethernet0;
 		mmc1 = &sdhc;
 		serial0 = &uart10;
+		serial1 = &uart17;
 		serial2 = &uart0;
 	};
 
@@ -63,12 +64,43 @@ port@1 {
 				reg = <1>;
 
 				usb0_con_ss_ep: endpoint {
-					remote-endpoint = <&hd3ss3220_in_ep>;
+					remote-endpoint = <&hd3ss3220_0_in_ep>;
 				};
 			};
 		};
 	};
 
+	connector-1 {
+		compatible = "usb-c-connector";
+		label = "USB1-Type-C";
+		data-role = "host";
+		power-role = "source";
+
+		vbus-supply = <&usb1_vbus>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				usb1_con_hs_ep: endpoint {
+					remote-endpoint = <&usb_hub_2_1>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				usb1_con_ss_ep: endpoint {
+					remote-endpoint = <&hd3ss3220_1_in_ep>;
+				};
+
+			};
+		};
+	};
+
 	connector-2 {
 		compatible = "gpio-usb-b-connector", "usb-b-connector";
 		label = "micro-USB";
@@ -88,6 +120,38 @@ usb2_con_hs_ep: endpoint {
 		};
 	};
 
+	connector-3 {
+		compatible = "pcie-m2-e-connector";
+		vpcie3v3-supply = <&vreg_wcn_3p3>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				m2_e_pcie_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&pcieport0_ep>;
+				};
+			};
+
+			port@3 {
+				reg = <3>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				m2_e_uart_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&uart17_ep>;
+				};
+			};
+		};
+	};
+
 	edp0-connector {
 		compatible = "dp-connector";
 		label = "EDP0";
@@ -161,6 +225,15 @@ usb0_vbus: regulator-usb0-vbus {
 		enable-active-high;
 	};
 
+	usb1_vbus: regulator-usb1-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb1_vbus";
+		gpio = <&expander1 3 GPIO_ACTIVE_HIGH>;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		enable-active-high;
+	};
+
 	usb2_vbus: regulator-usb2-vbus {
 		compatible = "regulator-fixed";
 		regulator-name = "usb2_vbus";
@@ -178,6 +251,17 @@ vmmc_sdc: regulator-vmmc-sdc {
 		regulator-max-microvolt = <2950000>;
 	};
 
+	vreg_dcin_12v: regulator-dcin-12v {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_DCIN_12V";
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
 	vreg_sdc: regulator-vreg-sdc {
 		compatible = "regulator-gpio";
 
@@ -191,6 +275,19 @@ vreg_sdc: regulator-vreg-sdc {
 
 		startup-delay-us = <100>;
 	};
+
+	vreg_wcn_3p3: regulator-wcn-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		vin-supply = <&vreg_dcin_12v>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
 };
 
 &apps_rsc {
@@ -551,7 +648,7 @@ ports {
 			port@0 {
 				reg = <0>;
 
-				hd3ss3220_in_ep: endpoint {
+				hd3ss3220_0_in_ep: endpoint {
 					remote-endpoint = <&usb0_con_ss_ep>;
 				};
 			};
@@ -559,12 +656,45 @@ hd3ss3220_in_ep: endpoint {
 			port@1 {
 				reg = <1>;
 
-				hd3ss3220_out_ep: endpoint {
+				hd3ss3220_0_out_ep: endpoint {
 					remote-endpoint = <&usb_0_dwc3_ss>;
 				};
 			};
 		};
 	};
+
+	usb-typec@47 {
+		compatible = "ti,hd3ss3220";
+		reg = <0x47>;
+
+		interrupts-extended = <&pmm8654au_2_gpios 6 IRQ_TYPE_EDGE_FALLING>;
+
+		id-gpios = <&tlmm 51 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&usb1_id>, <&usb1_intr>;
+		pinctrl-names = "default";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				hd3ss3220_1_in_ep: endpoint {
+					remote-endpoint = <&usb1_con_ss_ep>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				hd3ss3220_1_out_ep: endpoint {
+					remote-endpoint = <&usb_hub_3_1>;
+				};
+			};
+		};
+	};
 };
 
 &i2c18 {
@@ -742,6 +872,14 @@ &pcie1_phy {
 	status = "okay";
 };
 
+&pcieport0 {
+	port {
+		pcieport0_ep: endpoint {
+			remote-endpoint = <&m2_e_pcie_ep>;
+		};
+	};
+};
+
 &pmm8654au_0_pon_resin {
 	linux,code = <KEY_VOLUMEDOWN>;
 	status = "okay";
@@ -756,6 +894,14 @@ usb0_intr_state: usb0-intr-state {
 		power-source = <0>;
 	};
 
+	usb1_intr: usb1-intr-state {
+		pins = "gpio6";
+		function = "normal";
+		input-enable;
+		bias-pull-up;
+		power-source = <0>;
+	};
+
 	usb2_id: usb2-id-state {
 		pins = "gpio11";
 		function = "normal";
@@ -956,6 +1102,12 @@ usb_id: usb-id-state {
 		function = "gpio";
 		bias-pull-up;
 	};
+
+	usb1_id: usb1-id-state {
+		pins = "gpio51";
+		function = "gpio";
+		bias-pull-up;
+	};
 };
 
 &uart0 {
@@ -970,6 +1122,16 @@ &uart10 {
 	status = "okay";
 };
 
+&uart17 {
+	status = "okay";
+
+	port {
+		uart17_ep: endpoint {
+			remote-endpoint = <&m2_e_uart_ep>;
+		};
+	};
+};
+
 &ufs_mem_hc {
 	reset-gpios = <&tlmm 149 GPIO_ACTIVE_LOW>;
 	vcc-supply = <&vreg_l8a>;
@@ -996,7 +1158,7 @@ &usb_0_dwc3_hs {
 };
 
 &usb_0_dwc3_ss {
-	remote-endpoint = <&hd3ss3220_out_ep>;
+	remote-endpoint = <&hd3ss3220_0_out_ep>;
 };
 
 &usb_0_hsphy {
@@ -1014,6 +1176,87 @@ &usb_0_qmpphy {
 	status = "okay";
 };
 
+&usb_1 {
+	dr_mode = "host";
+
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	status = "okay";
+
+	usb_hub_2_x: hub@1 {
+		compatible = "usb5e3,610";
+		reg = <1>;
+
+		peer-hub = <&usb_hub_3_x>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@1 {
+				reg = <1>;
+
+				usb_hub_2_1: endpoint {
+					remote-endpoint = <&usb1_con_hs_ep>;
+				};
+			};
+
+			/*
+			 * Port-4 is connected to M.2 E key connector on corekit.
+			 */
+			port@4 {
+				reg = <4>;
+
+				usb_hub_2_4: endpoint {
+				};
+			};
+		};
+	};
+
+	usb_hub_3_x: hub@2 {
+		compatible = "usb5e3,625";
+		reg = <2>;
+
+		peer-hub = <&usb_hub_2_x>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@1 {
+				reg = <1>;
+
+				usb_hub_3_1: endpoint {
+					remote-endpoint = <&hd3ss3220_1_out_ep>;
+				};
+			};
+
+			port@4 {
+				reg = <4>;
+
+				usb_hub_3_4: endpoint {
+				};
+			};
+		};
+	};
+};
+
+&usb_1_hsphy {
+	vdda-pll-supply = <&vreg_l7a>;
+	vdda18-supply = <&vreg_l6c>;
+	vdda33-supply = <&vreg_l9a>;
+
+	status = "okay";
+};
+
+&usb_1_qmpphy {
+	vdda-phy-supply = <&vreg_l1c>;
+	vdda-pll-supply = <&vreg_l7a>;
+
+	status = "okay";
+};
+
 &usb_2 {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi b/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi
index 40f88498999bd..cefb8ff008063 100644
--- a/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi
@@ -63,7 +63,7 @@ vreg_3p3: vreg-3p3-regulator {
 		regulator-max-microvolt = <3300000>;
 	};
 
-	vreg_conn_1p8: vreg_conn_1p8 {
+	vreg_conn_1p8: vreg-conn-1p8-regulator {
 		compatible = "regulator-fixed";
 		regulator-name = "vreg_conn_1p8";
 		startup-delay-us = <4000>;
@@ -71,7 +71,7 @@ vreg_conn_1p8: vreg_conn_1p8 {
 		gpio = <&pmm8654au_1_gpios 4 GPIO_ACTIVE_HIGH>;
 	};
 
-	vreg_conn_pa: vreg_conn_pa {
+	vreg_conn_pa: vreg-conn-pa-regulator {
 		compatible = "regulator-fixed";
 		regulator-name = "vreg_conn_pa";
 		startup-delay-us = <4000>;
diff --git a/arch/arm64/boot/dts/qcom/lemans.dtsi b/arch/arm64/boot/dts/qcom/lemans.dtsi
index 353a6e6fd3acb..3b0539e27b519 100644
--- a/arch/arm64/boot/dts/qcom/lemans.dtsi
+++ b/arch/arm64/boot/dts/qcom/lemans.dtsi
@@ -297,17 +297,7 @@ core3 {
 		idle-states {
 			entry-method = "psci";
 
-			gold_cpu_sleep_0: cpu-sleep-0 {
-				compatible = "arm,idle-state";
-				idle-state-name = "gold-power-collapse";
-				arm,psci-suspend-param = <0x40000003>;
-				entry-latency-us = <549>;
-				exit-latency-us = <901>;
-				min-residency-us = <1774>;
-				local-timer-stop;
-			};
-
-			gold_rail_cpu_sleep_0: cpu-sleep-1 {
+			gold_rail_cpu_sleep_0: cpu-sleep-0 {
 				compatible = "arm,idle-state";
 				idle-state-name = "gold-rail-power-collapse";
 				arm,psci-suspend-param = <0x40000004>;
@@ -566,57 +556,49 @@ psci {
 		cpu_pd0: power-domain-cpu0 {
 			#power-domain-cells = <0>;
 			power-domains = <&cluster_0_pd>;
-			domain-idle-states = <&gold_cpu_sleep_0>,
-					     <&gold_rail_cpu_sleep_0>;
+			domain-idle-states = <&gold_rail_cpu_sleep_0>;
 		};
 
 		cpu_pd1: power-domain-cpu1 {
 			#power-domain-cells = <0>;
 			power-domains = <&cluster_0_pd>;
-			domain-idle-states = <&gold_cpu_sleep_0>,
-					     <&gold_rail_cpu_sleep_0>;
+			domain-idle-states = <&gold_rail_cpu_sleep_0>;
 		};
 
 		cpu_pd2: power-domain-cpu2 {
 			#power-domain-cells = <0>;
 			power-domains = <&cluster_0_pd>;
-			domain-idle-states = <&gold_cpu_sleep_0>,
-					     <&gold_rail_cpu_sleep_0>;
+			domain-idle-states = <&gold_rail_cpu_sleep_0>;
 		};
 
 		cpu_pd3: power-domain-cpu3 {
 			#power-domain-cells = <0>;
 			power-domains = <&cluster_0_pd>;
-			domain-idle-states = <&gold_cpu_sleep_0>,
-					     <&gold_rail_cpu_sleep_0>;
+			domain-idle-states = <&gold_rail_cpu_sleep_0>;
 		};
 
 		cpu_pd4: power-domain-cpu4 {
 			#power-domain-cells = <0>;
 			power-domains = <&cluster_1_pd>;
-			domain-idle-states = <&gold_cpu_sleep_0>,
-					     <&gold_rail_cpu_sleep_0>;
+			domain-idle-states = <&gold_rail_cpu_sleep_0>;
 		};
 
 		cpu_pd5: power-domain-cpu5 {
 			#power-domain-cells = <0>;
 			power-domains = <&cluster_1_pd>;
-			domain-idle-states = <&gold_cpu_sleep_0>,
-					     <&gold_rail_cpu_sleep_0>;
+			domain-idle-states = <&gold_rail_cpu_sleep_0>;
 		};
 
 		cpu_pd6: power-domain-cpu6 {
 			#power-domain-cells = <0>;
 			power-domains = <&cluster_1_pd>;
-			domain-idle-states = <&gold_cpu_sleep_0>,
-					     <&gold_rail_cpu_sleep_0>;
+			domain-idle-states = <&gold_rail_cpu_sleep_0>;
 		};
 
 		cpu_pd7: power-domain-cpu7 {
 			#power-domain-cells = <0>;
 			power-domains = <&cluster_1_pd>;
-			domain-idle-states = <&gold_cpu_sleep_0>,
-					     <&gold_rail_cpu_sleep_0>;
+			domain-idle-states = <&gold_rail_cpu_sleep_0>;
 		};
 
 		cluster_0_pd: power-domain-cluster0 {
@@ -1041,7 +1023,7 @@ qfprom: efuse@784000 {
 			#address-cells = <1>;
 			#size-cells = <1>;
 
-			gpu_speed_bin: gpu_speed_bin@240c {
+			gpu_speed_bin: gpu-speed-bin@240c {
 				reg = <0x240c 0x1>;
 				bits = <0 8>;
 			};
@@ -2779,6 +2761,7 @@ pcie0: pcie@1c00000 {
 			status = "disabled";
 
 			pcieport0: pcie@0 {
+				compatible = "pciclass,0604";
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
@@ -3197,6 +3180,7 @@ tpdm@4003000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_dcc";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -3256,6 +3240,7 @@ tpdm@400f000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_spdm";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -3724,6 +3709,7 @@ tpdm@4b09000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_0";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -3744,6 +3730,7 @@ tpdm@4b0a000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_1";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -3764,6 +3751,7 @@ tpdm@4b0b000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_2";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -3784,6 +3772,7 @@ tpdm@4b0c000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_3";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -3804,6 +3793,7 @@ tpdm@4b0d000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_1";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -3824,6 +3814,7 @@ aoss_cti: cti@4b13000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_aoss";
 		};
 
 		funnel@4b83000 {
@@ -4145,6 +4136,7 @@ tpdm@6860000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_actpm";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -4165,6 +4157,7 @@ tpdm@6861000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_apss";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -4247,6 +4240,7 @@ tpdm@68a0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_llm_silver";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -4267,6 +4261,7 @@ tpdm@68b0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_llm_gold";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -4287,6 +4282,7 @@ tpdm@68c0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_llm_ext";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -5873,7 +5869,7 @@ dispcc0: clock-controller@af00000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sa8775p-pdc", "qcom,pdc";
-			reg = <0x0 0x0b220000 0x0 0x30000>,
+			reg = <0x0 0x0b220000 0x0 0x10000>,
 			      <0x0 0x17c000f0 0x0 0x64>;
 			qcom,pdc-ranges = <0 480 40>,
 					  <40 140 14>,
@@ -9179,6 +9175,7 @@ arch_timer: timer {
 
 	turing-llm-tpdm {
 		compatible = "qcom,coresight-static-tpdm";
+		label = "tpdm_cdsp_llm_0";
 
 		qcom,cmb-element-bits = <32>;
 
diff --git a/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts b/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts
index ebdca0e2d2f3a..bb3b67e2fd018 100644
--- a/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts
+++ b/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts
@@ -449,16 +449,16 @@ vreg_l2c: ldo2 {
 	regulators-2 {
 		compatible = "qcom,pmr735b-rpmh-regulators";
 
-		vdd-l1-l2-supply= <&vreg_s3b>;
-		vdd-l3-supply= <&vreg_s3b>;
-		vdd-l4-supply= <&vreg_s1b>;
-		vdd-l5-supply= <&vreg_s2b>;
-		vdd-l6-supply= <&vreg_s2b>;
-		vdd-l7-l8-supply= <&vreg_s2b>;
-		vdd-l9-supply= <&vreg_s3b>;
-		vdd-l10-supply= <&vreg_s1b>;
-		vdd-l11-supply= <&vreg_s3b>;
-		vdd-l12-supply= <&vreg_s3b>;
+		vdd-l1-l2-supply = <&vreg_s3b>;
+		vdd-l3-supply = <&vreg_s3b>;
+		vdd-l4-supply = <&vreg_s1b>;
+		vdd-l5-supply = <&vreg_s2b>;
+		vdd-l6-supply = <&vreg_s2b>;
+		vdd-l7-l8-supply = <&vreg_s2b>;
+		vdd-l9-supply = <&vreg_s3b>;
+		vdd-l10-supply = <&vreg_s1b>;
+		vdd-l11-supply = <&vreg_s3b>;
+		vdd-l12-supply = <&vreg_s3b>;
 
 		qcom,pmic-id = "f";
 
@@ -700,6 +700,15 @@ vibrator@5a {
 	};
 };
 
+&ipa {
+	firmware-name = "qcom/milos/fairphone/fp6/ipa_fws.mbn";
+	memory-region = <&ipa_fw_mem>;
+
+	qcom,gsi-loader = "self";
+
+	status = "okay";
+};
+
 &pm8550vs_c {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/milos-nothing-asteroids.dts b/arch/arm64/boot/dts/qcom/milos-nothing-asteroids.dts
index 7393978926e44..787acef4a4c85 100644
--- a/arch/arm64/boot/dts/qcom/milos-nothing-asteroids.dts
+++ b/arch/arm64/boot/dts/qcom/milos-nothing-asteroids.dts
@@ -25,31 +25,7 @@ aliases {
 	};
 
 	chosen {
-		#address-cells = <2>;
-		#size-cells = <2>;
-		ranges;
-
 		stdout-path = "serial0:115200n8";
-
-		framebuffer0: framebuffer@e3940000 {
-			compatible = "simple-framebuffer";
-			reg = <0x0 0xe3940000 0x0 (2392 * 1080 * 4)>;
-			width = <1080>;
-			height = <2392>;
-			stride = <(1080 * 4)>;
-			format = "a8r8g8b8";
-			panel = <&panel>;
-			interconnects = <&mmss_noc MASTER_MDP QCOM_ICC_TAG_ALWAYS
-					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
-			clocks = <&gcc GCC_DISP_HF_AXI_CLK>;
-
-			/* Dummy panel for simple-framebuffer dimension info */
-			panel: dummy-panel {
-				compatible = "boe,bf068mwm-td0";
-				height-mm = <157>;
-				width-mm = <71>;
-			};
-		};
 	};
 
 	gpio-keys {
@@ -131,15 +107,6 @@ ramoops@81f20000 {
 			mem-type = <2>;
 			pmsg-size = <0x200000>;
 		};
-
-		/*
-		 * ABL is powering down display and controller if this node is
-		 * not named exactly "splash_region".
-		 */
-		splash_region@e3940000 {
-			reg = <0x0 0xe3940000 0x0 0x2b00000>;
-			no-map;
-		};
 	};
 };
 
@@ -507,11 +474,6 @@ vreg_l12f: ldo12 {
 	};
 };
 
-&dispcc {
-	/* Disable for now so simple-framebuffer continues working */
-	status = "disabled";
-};
-
 &gcc {
 	protected-clocks = <GCC_PCIE_1_AUX_CLK>, <GCC_PCIE_1_AUX_CLK_SRC>,
 			   <GCC_PCIE_1_CFG_AHB_CLK>, <GCC_PCIE_1_MSTR_AXI_CLK>,
diff --git a/arch/arm64/boot/dts/qcom/milos.dtsi b/arch/arm64/boot/dts/qcom/milos.dtsi
index 8c92329889538..e91c0cf5e093d 100644
--- a/arch/arm64/boot/dts/qcom/milos.dtsi
+++ b/arch/arm64/boot/dts/qcom/milos.dtsi
@@ -3,6 +3,7 @@
  * Copyright (c) 2025, Luca Weiss <luca.weiss@fairphone.com>
  */
 
+#include <dt-bindings/clock/qcom,dsi-phy-28nm.h>
 #include <dt-bindings/clock/qcom,milos-camcc.h>
 #include <dt-bindings/clock/qcom,milos-dispcc.h>
 #include <dt-bindings/clock/qcom,milos-gcc.h>
@@ -1647,6 +1648,50 @@ adreno_smmu: iommu@3da0000 {
 			dma-coherent;
 		};
 
+		ipa: ipa@3f40000 {
+			compatible = "qcom,milos-ipa";
+
+			reg = <0x0 0x03f40000 0x0 0x10000>,
+			      <0x0 0x03f50000 0x0 0x5000>,
+			      <0x0 0x03e04000 0x0 0xfc000>;
+			reg-names = "ipa-reg",
+				    "ipa-shared",
+				    "gsi";
+
+			interrupts-extended = <&intc GIC_SPI 654 IRQ_TYPE_EDGE_RISING 0>,
+					      <&intc GIC_SPI 432 IRQ_TYPE_LEVEL_HIGH 0>,
+					      <&smp2p_ipa_in 0 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_ipa_in 1 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "ipa",
+					  "gsi",
+					  "ipa-clock-query",
+					  "ipa-setup-ready";
+
+			clocks = <&rpmhcc RPMH_IPA_CLK>;
+			clock-names = "core";
+
+			interconnects = <&aggre2_noc MASTER_IPA QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &cnoc_main SLAVE_IPA_CFG QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "memory",
+					     "config";
+
+			iommus = <&apps_smmu 0x4a0 0x0>,
+				 <&apps_smmu 0x4a2 0x0>;
+
+			qcom,qmp = <&aoss_qmp>;
+
+			qcom,smem-states = <&smp2p_ipa_out 0>,
+					   <&smp2p_ipa_out 1>;
+			qcom,smem-state-names = "ipa-clock-enabled-valid",
+						"ipa-clock-enabled";
+
+			sram = <&ipa_modem_tables>;
+
+			status = "disabled";
+		};
+
 		remoteproc_mpss: remoteproc@4080000 {
 			compatible = "qcom,milos-mpss-pas";
 			reg = <0x0 0x04080000 0x0 0x10000>;
@@ -1724,6 +1769,8 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
 
 			iommus = <&apps_smmu 0x540 0>;
 
+			resets = <&gcc GCC_SDCC2_BCR>;
+
 			bus-width = <4>;
 
 			qcom,dll-config = <0x0007442c>;
@@ -1971,11 +2018,220 @@ camcc: clock-controller@adb0000 {
 				 <&sleep_clk>,
 				 <&gcc GCC_CAMERA_AHB_CLK>;
 
+			interconnects = <&mmss_noc MASTER_CAMNOC_HF QCOM_ICC_TAG_ALWAYS
+					 &mmss_noc SLAVE_MNOC_HF_MEM_NOC QCOM_ICC_TAG_ALWAYS>;
+
 			#clock-cells = <1>;
 			#reset-cells = <1>;
 			#power-domain-cells = <1>;
 		};
 
+		mdss: display-subsystem@ae00000 {
+			compatible = "qcom,milos-mdss";
+			reg = <0x0 0x0ae00000 0x0 0x1000>;
+			reg-names = "mdss";
+
+			interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH 0>;
+
+			clocks = <&dispcc DISP_CC_MDSS_AHB_CLK>,
+				 <&gcc GCC_DISP_HF_AXI_CLK>,
+				 <&dispcc DISP_CC_MDSS_MDP_CLK>;
+
+			resets = <&dispcc DISP_CC_MDSS_CORE_BCR>;
+
+			interconnects = <&mmss_noc MASTER_MDP QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &cnoc_main SLAVE_DISPLAY_CFG QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "mdp0-mem",
+					     "cpu-cfg";
+
+			power-domains = <&dispcc DISP_CC_MDSS_CORE_GDSC>;
+
+			iommus = <&apps_smmu 0x1c00 0x2>;
+
+			interrupt-controller;
+			#interrupt-cells = <1>;
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			status = "disabled";
+
+			mdss_mdp: display-controller@ae01000 {
+				compatible = "qcom,milos-dpu";
+				reg = <0x0 0x0ae01000 0x0 0x8f000>,
+				      <0x0 0x0aeb0000 0x0 0x3000>;
+				reg-names = "mdp",
+					    "vbif";
+
+				interrupts-extended = <&mdss 0>;
+
+				clocks = <&gcc GCC_DISP_HF_AXI_CLK>,
+					 <&dispcc DISP_CC_MDSS_AHB_CLK>,
+					 <&dispcc DISP_CC_MDSS_MDP_LUT_CLK>,
+					 <&dispcc DISP_CC_MDSS_MDP_CLK>,
+					 <&dispcc DISP_CC_MDSS_VSYNC_CLK>;
+				clock-names = "nrt_bus",
+					      "iface",
+					      "lut",
+					      "core",
+					      "vsync";
+
+				assigned-clocks = <&dispcc DISP_CC_MDSS_VSYNC_CLK>;
+				assigned-clock-rates = <19200000>;
+
+				operating-points-v2 = <&mdp_opp_table>;
+
+				power-domains = <&rpmhpd RPMHPD_CX>;
+
+				ports {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					port@0 {
+						reg = <0>;
+
+						dpu_intf1_out: endpoint {
+							remote-endpoint = <&mdss_dsi0_in>;
+						};
+					};
+				};
+
+				mdp_opp_table: opp-table {
+					compatible = "operating-points-v2";
+
+					opp-200000000 {
+						opp-hz = /bits/ 64 <200000000>;
+						required-opps = <&rpmhpd_opp_low_svs>;
+					};
+
+					opp-342000000 {
+						opp-hz = /bits/ 64 <342000000>;
+						required-opps = <&rpmhpd_opp_svs>;
+					};
+
+					opp-402000000 {
+						opp-hz = /bits/ 64 <402000000>;
+						required-opps = <&rpmhpd_opp_svs_l1>;
+					};
+
+					opp-535000000 {
+						opp-hz = /bits/ 64 <535000000>;
+						required-opps = <&rpmhpd_opp_nom>;
+					};
+
+					opp-600000000 {
+						opp-hz = /bits/ 64 <600000000>;
+						required-opps = <&rpmhpd_opp_nom_l1>;
+					};
+
+					opp-630000000 {
+						opp-hz = /bits/ 64 <630000000>;
+						required-opps = <&rpmhpd_opp_turbo>;
+					};
+				};
+			};
+
+			mdss_dsi0: dsi@ae94000 {
+				compatible = "qcom,milos-dsi-ctrl", "qcom,mdss-dsi-ctrl";
+				reg = <0x0 0x0ae94000 0x0 0x1000>;
+				reg-names = "dsi_ctrl";
+
+				interrupts-extended = <&mdss 4>;
+
+				clocks = <&dispcc DISP_CC_MDSS_BYTE0_CLK>,
+					 <&dispcc DISP_CC_MDSS_BYTE0_INTF_CLK>,
+					 <&dispcc DISP_CC_MDSS_PCLK0_CLK>,
+					 <&dispcc DISP_CC_MDSS_ESC0_CLK>,
+					 <&dispcc DISP_CC_MDSS_AHB_CLK>,
+					 <&gcc GCC_DISP_HF_AXI_CLK>;
+				clock-names = "byte",
+					      "byte_intf",
+					      "pixel",
+					      "core",
+					      "iface",
+					      "bus";
+
+				assigned-clocks = <&dispcc DISP_CC_MDSS_BYTE0_CLK_SRC>,
+						  <&dispcc DISP_CC_MDSS_PCLK0_CLK_SRC>;
+				assigned-clock-parents = <&mdss_dsi0_phy DSI_BYTE_PLL_CLK>,
+							 <&mdss_dsi0_phy DSI_PIXEL_PLL_CLK>;
+
+				operating-points-v2 = <&mdss_dsi_opp_table>;
+
+				power-domains = <&rpmhpd RPMHPD_CX>;
+
+				phys = <&mdss_dsi0_phy>;
+				phy-names = "dsi";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+
+				ports {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					port@0 {
+						reg = <0>;
+
+						mdss_dsi0_in: endpoint {
+							remote-endpoint = <&dpu_intf1_out>;
+						};
+					};
+
+					port@1 {
+						reg = <1>;
+
+						mdss_dsi0_out: endpoint {
+						};
+					};
+				};
+
+				mdss_dsi_opp_table: opp-table {
+					compatible = "operating-points-v2";
+
+					opp-187500000 {
+						opp-hz = /bits/ 64 <187500000>;
+						required-opps = <&rpmhpd_opp_low_svs>;
+					};
+
+					opp-300000000 {
+						opp-hz = /bits/ 64 <300000000>;
+						required-opps = <&rpmhpd_opp_svs>;
+					};
+
+					opp-358000000 {
+						opp-hz = /bits/ 64 <358000000>;
+						required-opps = <&rpmhpd_opp_svs_l1>;
+					};
+				};
+			};
+
+			mdss_dsi0_phy: phy@ae95000 {
+				compatible = "qcom,milos-dsi-phy-4nm";
+				reg = <0x0 0x0ae95000 0x0 0x200>,
+				      <0x0 0x0ae95200 0x0 0x300>,
+				      <0x0 0x0ae95500 0x0 0x400>;
+				reg-names = "dsi_phy",
+					    "dsi_phy_lane",
+					    "dsi_pll";
+
+				clocks = <&dispcc DISP_CC_MDSS_AHB_CLK>,
+					 <&rpmhcc RPMH_CXO_CLK>;
+				clock-names = "iface",
+					      "ref";
+
+				#clock-cells = <1>;
+				#phy-cells = <0>;
+
+				status = "disabled";
+			};
+		};
+
 		dispcc: clock-controller@af00000 {
 			compatible = "qcom,milos-dispcc";
 			reg = <0x0 0x0af00000 0x0 0x20000>;
@@ -1984,8 +2240,8 @@ dispcc: clock-controller@af00000 {
 				 <&sleep_clk>,
 				 <&gcc GCC_DISP_AHB_CLK>,
 				 <&gcc GCC_DISP_GPLL0_DIV_CLK_SRC>,
-				 <0>, /* dsi0_phy_pll_out_byteclk */
-				 <0>, /* dsi0_phy_pll_out_dsiclk */
+				 <&mdss_dsi0_phy DSI_BYTE_PLL_CLK>,
+				 <&mdss_dsi0_phy DSI_PIXEL_PLL_CLK>,
 				 <0>, /* dp0_phy_pll_link_clk */
 				 <0>; /* dp0_phy_pll_vco_div_clk */
 
@@ -1996,7 +2252,7 @@ dispcc: clock-controller@af00000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,milos-pdc", "qcom,pdc";
-			reg = <0x0 0x0b220000 0x0 0x30000>,
+			reg = <0x0 0x0b220000 0x0 0x10000>,
 			      <0x0 0x174000f0 0x0 0x64>;
 			interrupt-parent = <&intc>;
 
@@ -2337,6 +2593,26 @@ scl-pins {
 			};
 		};
 
+		sram@14680000 {
+			compatible = "qcom,milos-imem", "mmio-sram";
+			reg = <0x0 0x14680000 0x0 0x2c000>;
+			ranges = <0x0 0x0 0x14680000 0x2c000>;
+
+			no-memory-wc;
+
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			pil-reloc-sram@94c {
+				compatible = "qcom,pil-reloc-info";
+				reg = <0x94c 0xc8>;
+			};
+
+			ipa_modem_tables: modem-tables-sram@3000 {
+				reg = <0x3000 0x2000>;
+			};
+		};
+
 		apps_smmu: iommu@15000000 {
 			compatible = "qcom,milos-smmu-500", "qcom,smmu-500", "arm,mmu-500";
 			reg = <0x0 0x15000000 0x0 0x100000>;
diff --git a/arch/arm64/boot/dts/qcom/monaco-arduino-monza.dts b/arch/arm64/boot/dts/qcom/monaco-arduino-monza.dts
index 379b796f261f4..d1346ca5dd4ef 100644
--- a/arch/arm64/boot/dts/qcom/monaco-arduino-monza.dts
+++ b/arch/arm64/boot/dts/qcom/monaco-arduino-monza.dts
@@ -7,10 +7,9 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
+#include <dt-bindings/pwm/pwm.h>
 #include <dt-bindings/sound/qcom,q6dsp-lpass-ports.h>
 
-#include "monaco.dtsi"
-#include "monaco-pmics.dtsi"
 #include "monaco-monza-som.dtsi"
 
 / {
@@ -118,6 +117,22 @@ platform {
 				sound-dai = <&q6apm>;
 			};
 		};
+
+		displayport-0-dai-link {
+			link-name = "DisplayPort0 Playback";
+
+			codec {
+				sound-dai = <&mdss_dp0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai DISPLAY_PORT_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
 	};
 
 	vdc_3v3: regulator-3v3 {
@@ -156,6 +171,26 @@ vreg_nvme: regulator-3p3-m2 {
 	};
 };
 
+&cci1 {
+	status = "okay";
+};
+
+&cci1_i2c1 {
+	fan_controller: fan-controller@2f {
+		compatible = "microchip,emc2305";
+		reg = <0x2f>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		#pwm-cells = <3>;
+
+		fan: fan@0 {
+			reg = <0x0>;
+			pwms = <&fan_controller 26000 PWM_POLARITY_INVERTED 1>;
+			#cooling-cells = <2>;
+		};
+	};
+};
+
 &ethernet0 {
 	phy-mode = "2500base-x";
 	phy-handle = <&hsgmii_phy0>;
@@ -246,6 +281,46 @@ &i2c12 {
 
 	status = "okay";
 
+	typec@8 {
+		compatible = "cypress,cypd4226";
+		reg = <0x08>;
+		pinctrl-0 = <&typec_default_state>;
+		pinctrl-names = "default";
+		interrupts-extended = <&tlmm 3 IRQ_TYPE_LEVEL_LOW>;
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+			label = "USB-C";
+			data-role = "dual";
+			power-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					ucsi_hs_ccg_p0: endpoint {
+						remote-endpoint = <&usb_1_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					ucsi_ss_ccg_p0: endpoint {
+						remote-endpoint = <&usb_1_dwc3_ss>;
+					};
+				};
+			};
+		};
+	};
+
 	max98091: audio-codec@10 {
 		compatible = "maxim,max98091";
 		reg = <0x10>;
@@ -291,6 +366,12 @@ adv7535_out: endpoint {
 			};
 		};
 	};
+
+	power-sensor@40 {
+		compatible = "ti,ina232";
+		reg = <0x40>;
+		shunt-resistor = <2000>;
+	};
 };
 
 &mdss {
@@ -369,6 +450,27 @@ &pcieport1 {
 	reset-gpios = <&tlmm 23 GPIO_ACTIVE_LOW>;
 };
 
+&thermal_zones {
+	cpuss-0-0-thermal {
+		polling-delay = <2000>;
+
+		trips {
+			cpuss0_active: cpuss-active {
+				temperature = <32000>;
+				hysteresis = <2000>;
+				type = "active";
+			};
+		};
+
+		cooling-maps {
+			map-active {
+				trip = <&cpuss0_active>;
+				cooling-device = <&fan 0 THERMAL_NO_LIMIT>;
+			};
+		};
+	};
+};
+
 &tlmm {
 	pcie0_default_state: pcie0-default-state {
 		wake-pins {
@@ -393,6 +495,12 @@ perst-pins {
 		};
 	};
 
+	typec_default_state: typec-default-state {
+		pins = "gpio3";
+		function = "gpio";
+		bias-disable;
+	};
+
 	ethernet0_default: ethernet0-default-state {
 		ethernet0_mdc: ethernet0-mdc-pins {
 			pins = "gpio5";
@@ -469,6 +577,14 @@ &usb_1 {
 	status = "okay";
 };
 
+&usb_1_dwc3_hs {
+	remote-endpoint = <&ucsi_hs_ccg_p0>;
+};
+
+&usb_1_dwc3_ss {
+	remote-endpoint = <&ucsi_ss_ccg_p0>;
+};
+
 /* Internally connected to the MCU (e.g. for DFU). */
 &usb_2 {
 	dr_mode = "host";
diff --git a/arch/arm64/boot/dts/qcom/monaco.dtsi b/arch/arm64/boot/dts/qcom/monaco.dtsi
index e4c8466f941bd..64fc0d592282c 100644
--- a/arch/arm64/boot/dts/qcom/monaco.dtsi
+++ b/arch/arm64/boot/dts/qcom/monaco.dtsi
@@ -309,16 +309,6 @@ idle-states {
 			entry-method = "psci";
 
 			little_cpu_sleep_0: cpu-sleep-0-0 {
-				compatible = "arm,idle-state";
-				idle-state-name = "silver-power-collapse";
-				arm,psci-suspend-param = <0x40000003>;
-				entry-latency-us = <449>;
-				exit-latency-us = <801>;
-				min-residency-us = <1574>;
-				local-timer-stop;
-			};
-
-			little_cpu_sleep_1: cpu-sleep-0-1 {
 				compatible = "arm,idle-state";
 				idle-state-name = "silver-rail-power-collapse";
 				arm,psci-suspend-param = <0x40000004>;
@@ -329,16 +319,6 @@ little_cpu_sleep_1: cpu-sleep-0-1 {
 			};
 
 			big_cpu_sleep_0: cpu-sleep-1-0 {
-				compatible = "arm,idle-state";
-				idle-state-name = "gold-power-collapse";
-				arm,psci-suspend-param = <0x40000003>;
-				entry-latency-us = <549>;
-				exit-latency-us = <901>;
-				min-residency-us = <1774>;
-				local-timer-stop;
-			};
-
-			big_cpu_sleep_1: cpu-sleep-1-1 {
 				compatible = "arm,idle-state";
 				idle-state-name = "gold-rail-power-collapse";
 				arm,psci-suspend-param = <0x40000004>;
@@ -681,49 +661,49 @@ psci {
 		cpu_pd0: power-domain-cpu0 {
 			#power-domain-cells = <0>;
 			power-domains = <&cluster_pd0>;
-			domain-idle-states = <&big_cpu_sleep_0 &big_cpu_sleep_1>;
+			domain-idle-states = <&big_cpu_sleep_0>;
 		};
 
 		cpu_pd1: power-domain-cpu1 {
 			#power-domain-cells = <0>;
 			power-domains = <&cluster_pd0>;
-			domain-idle-states = <&big_cpu_sleep_0 &big_cpu_sleep_1>;
+			domain-idle-states = <&big_cpu_sleep_0>;
 		};
 
 		cpu_pd2: power-domain-cpu2 {
 			#power-domain-cells = <0>;
 			power-domains = <&cluster_pd0>;
-			domain-idle-states = <&big_cpu_sleep_0 &big_cpu_sleep_1>;
+			domain-idle-states = <&big_cpu_sleep_0>;
 		};
 
 		cpu_pd3: power-domain-cpu3 {
 			#power-domain-cells = <0>;
 			power-domains = <&cluster_pd0>;
-			domain-idle-states = <&big_cpu_sleep_0 &big_cpu_sleep_1>;
+			domain-idle-states = <&big_cpu_sleep_0>;
 		};
 
 		cpu_pd4: power-domain-cpu4 {
 			#power-domain-cells = <0>;
 			power-domains = <&cluster_pd1>;
-			domain-idle-states = <&little_cpu_sleep_0 &little_cpu_sleep_1>;
+			domain-idle-states = <&little_cpu_sleep_0>;
 		};
 
 		cpu_pd5: power-domain-cpu5 {
 			#power-domain-cells = <0>;
 			power-domains = <&cluster_pd1>;
-			domain-idle-states = <&little_cpu_sleep_0 &little_cpu_sleep_1>;
+			domain-idle-states = <&little_cpu_sleep_0>;
 		};
 
 		cpu_pd6: power-domain-cpu6 {
 			#power-domain-cells = <0>;
 			power-domains = <&cluster_pd1>;
-			domain-idle-states = <&little_cpu_sleep_0 &little_cpu_sleep_1>;
+			domain-idle-states = <&little_cpu_sleep_0>;
 		};
 
 		cpu_pd7: power-domain-cpu7 {
 			#power-domain-cells = <0>;
 			power-domains = <&cluster_pd1>;
-			domain-idle-states = <&little_cpu_sleep_0 &little_cpu_sleep_1>;
+			domain-idle-states = <&little_cpu_sleep_0>;
 		};
 
 		cluster_pd0: power-domain-cluster0 {
@@ -2927,7 +2907,7 @@ ws-pins {
 				};
 			};
 
-			lpi_i2s4_active: lpi_i2s4-active-state {
+			lpi_i2s4_active: lpi-i2s4-active-state {
 				data0-pins {
 					pins = "gpio17";
 					function = "i2s4_data";
@@ -3017,14 +2997,6 @@ in-ports {
 				#address-cells = <1>;
 				#size-cells = <0>;
 
-				port@0 {
-					reg = <0>;
-
-					swao_rep_out0: endpoint {
-						remote-endpoint = <&qdss_rep_in>;
-					};
-				};
-
 				port@1 {
 					reg = <1>;
 
@@ -3049,6 +3021,7 @@ tpdm@400f000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_spdm";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -3311,6 +3284,7 @@ tpdm@4841000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_prng";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -3330,6 +3304,7 @@ tpdm@4850000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_pimem";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -3351,6 +3326,7 @@ tpdm@4860000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_dl_ch_south";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -3444,6 +3420,7 @@ tpdm@4980000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_cdsp";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -3487,6 +3464,7 @@ tpdm@4ac0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_mmnoc_0";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -3559,6 +3537,7 @@ tpdm@4ad0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_dlct";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -3734,6 +3713,14 @@ out-ports {
 				#address-cells = <1>;
 				#size-cells = <0>;
 
+				port@0 {
+					reg = <0>;
+
+					swao_rep_out0: endpoint {
+						remote-endpoint = <&qdss_rep_in>;
+					};
+				};
+
 				port@1 {
 					reg = <1>;
 
@@ -3811,6 +3798,7 @@ tpdm@4b09000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_0";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -3830,6 +3818,7 @@ tpdm@4b0a000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_1";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -3849,6 +3838,7 @@ tpdm@4b0b000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_2";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -3868,6 +3858,7 @@ tpdm@4b0c000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_3";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -3887,6 +3878,7 @@ tpdm@4b0d000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_1";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -3906,6 +3898,7 @@ cti@4b13000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_aoss";
 		};
 
 		tpdm@4b80000 {
@@ -3914,6 +3907,7 @@ tpdm@4b80000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_cdsp_0";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -3981,6 +3975,7 @@ cti@4b8b000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_cdsp_1";
 		};
 
 		tpdm@4c40000 {
@@ -3989,6 +3984,7 @@ tpdm@4c40000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_gpdsp_0";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -4082,6 +4078,7 @@ tpdm@4c50000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_dl_south";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -4154,6 +4151,7 @@ tpdm@4e00000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_ddr";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -4244,6 +4242,7 @@ tpdm@4e10000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_ddr_ch0";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -4287,6 +4286,7 @@ tpdm@4e20000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_ddr_ch1";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -4612,6 +4612,7 @@ cti@682b000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_apss";
 		};
 
 		tpdm@6860000 {
@@ -4620,6 +4621,7 @@ tpdm@6860000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_actpm";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -4639,6 +4641,7 @@ tpdm@6861000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_apss";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -4719,6 +4722,7 @@ tpdm@68a0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_llm_gold";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -4738,6 +4742,7 @@ tpdm@68b0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_llm_silver";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -4757,6 +4762,7 @@ tpdm@68c0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_ext_dsb";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -4776,6 +4782,7 @@ cti@68e0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_llm_gold";
 		};
 
 		cti@68f0000 {
@@ -4784,6 +4791,7 @@ cti@68f0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_llm_silver";
 		};
 
 		cti@6900000 {
@@ -4792,6 +4800,7 @@ cti@6900000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_ext_dsb";
 		};
 
 		sdhc_1: mmc@87c4000 {
@@ -6077,7 +6086,7 @@ dispcc: clock-controller@af00000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,qcs8300-pdc", "qcom,pdc";
-			reg = <0x0 0xb220000 0x0 0x30000>,
+			reg = <0x0 0xb220000 0x0 0x10000>,
 			      <0x0 0x17c000f0 0x0 0x64>;
 			interrupt-parent = <&intc>;
 			#interrupt-cells = <2>;
@@ -7384,6 +7393,7 @@ intc: interrupt-controller@17a00000 {
 			interrupt-controller;
 			#redistributor-regions = <1>;
 			redistributor-stride = <0x0 0x20000>;
+			#address-cells = <0>;
 		};
 
 		watchdog@17c10000 {
diff --git a/arch/arm64/boot/dts/qcom/msm8953-flipkart-rimob.dts b/arch/arm64/boot/dts/qcom/msm8953-flipkart-rimob.dts
index ef4faf7631327..978820348fa56 100644
--- a/arch/arm64/boot/dts/qcom/msm8953-flipkart-rimob.dts
+++ b/arch/arm64/boot/dts/qcom/msm8953-flipkart-rimob.dts
@@ -47,9 +47,18 @@ framebuffer@90001000 {
 	gpio-keys {
 		compatible = "gpio-keys";
 
-		pinctrl-0 = <&gpio_key_default>;
+		pinctrl-0 = <&gpio_hall_sensor_default>, <&gpio_key_default>;
 		pinctrl-names = "default";
 
+		event-hall-sensor {
+			label = "Hall Effect Sensor";
+			gpios = <&tlmm 46 GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_SW>;
+			linux,code = <SW_LID>;
+			linux,can-disable;
+			wakeup-source;
+		};
+
 		key-volume-up {
 			label = "Volume Up";
 			gpios = <&tlmm 85 GPIO_ACTIVE_LOW>;
@@ -78,6 +87,13 @@ vph_pwr: vph-pwr-regulator {
 	};
 };
 
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/msm8953/flipkart/rimob/a506_zap.mbn";
+};
 
 &hsusb_phy {
 	vdd-supply = <&pm8953_l3>;
@@ -87,11 +103,95 @@ &hsusb_phy {
 	status = "okay";
 };
 
+&i2c_3 {
+	status = "okay";
+
+	touchscreen@5d {
+		compatible = "goodix,gt5688";
+		reg = <0x5d>;
+
+		interrupts-extended = <&tlmm 65 IRQ_TYPE_LEVEL_LOW>;
+
+		pinctrl-0 = <&tsp_int_rst_default>;
+		pinctrl-names = "default";
+
+		irq-gpios = <&tlmm 65 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&tlmm 64 GPIO_ACTIVE_HIGH>;
+
+		VDDIO-supply = <&pm8953_l6>;
+		AVDD28-supply = <&pm8953_l10>;
+
+		touchscreen-size-x = <1080>;
+		touchscreen-size-y = <1920>;
+		touchscreen-inverted-x;
+		touchscreen-inverted-y;
+	};
+};
+
+&ibb {
+	qcom,discharge-resistor-kohms = <32>;
+};
+
+&lab {
+	qcom,soft-start-us = <800>;
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dsi0 {
+	vdda-supply = <&pm8953_s3>;
+	vddio-supply = <&pm8953_l6>;
+
+	status = "okay";
+
+	panel: panel@0 {
+		compatible = "flipkart,rimob-panel-nt35532-cs";
+		reg = <0>;
+
+		backlight = <&pmi8950_wled>;
+		reset-gpios = <&tlmm 61 GPIO_ACTIVE_LOW>;
+		avdd-supply = <&lab>;
+		avee-supply = <&ibb>;
+		vci-supply = <&pm8953_l17>;
+		vddi-supply = <&pm8953_l6>;
+
+		pinctrl-0 = <&panel_default>;
+		pinctrl-names = "default";
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mdss_dsi0_out>;
+			};
+		};
+	};
+};
+
+&mdss_dsi0_out {
+	data-lanes = <0 1 2 3>;
+	remote-endpoint = <&panel_in>;
+};
+
+&mdss_dsi0_phy {
+	vcca-supply = <&pm8953_l3>;
+
+	status = "okay";
+};
+
 &pm8953_resin {
 	linux,code = <KEY_VOLUMEDOWN>;
 	status = "okay";
 };
 
+&pmi8950_wled {
+	qcom,current-limit-microamp = <10000>;
+	qcom,num-strings = <3>;
+	qcom,ovp-millivolt = <29500>;
+
+	status = "okay";
+};
+
 &rpm_requests {
 	regulators {
 		compatible = "qcom,rpm-pm8953-regulators";
@@ -238,6 +338,27 @@ &sdhc_2 {
 &tlmm {
 	gpio-reserved-ranges = <0 4>, <135 4>;
 
+	gpio_hall_sensor_default: gpio-hall-sensor-default-state {
+		pins = "gpio46";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	panel_default: panel-default-state {
+		pins = "gpio61";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
+
+	tsp_int_rst_default: tsp-int-rst-default-state {
+		pins = "gpio64", "gpio65";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-up;
+	};
+
 	gpio_key_default: gpio-key-default-state {
 		pins = "gpio85";
 		function = "gpio";
@@ -253,3 +374,18 @@ &usb3 {
 &usb3_dwc3 {
 	dr_mode = "peripheral";
 };
+
+&wcnss {
+	vddpx-supply = <&pm8953_l5>;
+
+	status = "okay";
+};
+
+&wcnss_iris {
+	compatible = "qcom,wcn3660b";
+
+	vddxo-supply = <&pm8953_l7>;
+	vddrfa-supply = <&pm8953_l19>;
+	vddpa-supply = <&pm8953_l9>;
+	vdddig-supply = <&pm8953_l5>;
+};
diff --git a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts
index fd3a2121465b0..ca22e2f9d20a2 100644
--- a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts
+++ b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts
@@ -39,7 +39,7 @@ &blsp2_i2c3 {
 	haptics: drv2604@5a {
 		compatible = "ti,drv2604";
 		reg = <0x5a>;
-		enable-gpio = <&tlmm 93 0x00>;
+		enable-gpios = <&tlmm 93 GPIO_ACTIVE_HIGH>;
 		mode = <DRV260X_LRA_MODE>;
 		library-sel = <DRV260X_LIB_LRA>;
 		pinctrl-names = "default","sleep";
diff --git a/arch/arm64/boot/dts/qcom/msm8996.dtsi b/arch/arm64/boot/dts/qcom/msm8996.dtsi
index 2f67e665996f3..818e5660df91d 100644
--- a/arch/arm64/boot/dts/qcom/msm8996.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8996.dtsi
@@ -1632,6 +1632,30 @@ blsp2_i2c5_default: blsp2-i2c5-state {
 
 			/* Sleep state for BLSP2_I2C5 is missing.. */
 
+			blsp2_spi5_default: blsp2-spi5-default-state {
+				spi-pins {
+					pins = "gpio58", "gpio59", "gpio61";
+					function = "blsp_spi11";
+					drive-strength = <12>;
+					bias-disable;
+				};
+
+				cs-pins {
+					pins = "gpio60";
+					function = "blsp_spi11";
+					drive-strength = <16>;
+					bias-disable;
+					output-high;
+				};
+			};
+
+			blsp2_spi5_sleep: blsp2-spi5-sleep-state {
+				pins = "gpio58", "gpio59", "gpio60", "gpio61";
+				function = "gpio";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
 			cdc_reset_active: cdc-reset-active-state {
 				pins = "gpio64";
 				function = "gpio";
@@ -3433,6 +3457,23 @@ blsp2_i2c5: i2c@75b9000 {
 			status = "disabled";
 		};
 
+		blsp2_spi5: spi@75b9000 {
+			compatible = "qcom,spi-qup-v2.2.1";
+			reg = <0x75b9000 0x600>;
+			interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc GCC_BLSP2_QUP5_SPI_APPS_CLK>,
+				 <&gcc GCC_BLSP2_AHB_CLK>;
+			clock-names = "core", "iface";
+			pinctrl-names = "default", "sleep";
+			pinctrl-0 = <&blsp2_spi5_default>;
+			pinctrl-1 = <&blsp2_spi5_sleep>;
+			dmas = <&blsp2_dma 20>, <&blsp2_dma 21>;
+			dma-names = "tx", "rx";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		blsp2_i2c6: i2c@75ba000 {
 			compatible = "qcom,i2c-qup-v2.2.1";
 			reg = <0x75ba000 0x1000>;
diff --git a/arch/arm64/boot/dts/qcom/pm4125.dtsi b/arch/arm64/boot/dts/qcom/pm4125.dtsi
index 542e8fe030da4..3dc8d667d091c 100644
--- a/arch/arm64/boot/dts/qcom/pm4125.dtsi
+++ b/arch/arm64/boot/dts/qcom/pm4125.dtsi
@@ -37,7 +37,7 @@ pm4125_resin: resin {
 		};
 
 		pm4125_vbus: usb-vbus-regulator@1100 {
-			compatible = "qcom,pm4125-vbus-reg", "qcom,pm8150b-vbus-reg";
+			compatible = "qcom,pm4125-vbus-reg";
 			reg = <0x1100>;
 			status = "disabled";
 		};
diff --git a/arch/arm64/boot/dts/qcom/pm7550-eliza.dtsi b/arch/arm64/boot/dts/qcom/pm7550-eliza.dtsi
new file mode 100644
index 0000000000000..9b907f69264b2
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/pm7550-eliza.dtsi
@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2026 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/spmi/spmi.h>
+
+/ {
+	thermal-zones {
+		pm7550-thermal {
+			polling-delay-passive = <100>;
+
+			thermal-sensors = <&pm7550_temp_alarm>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					/*
+					 * Current Linux driver currently only supports up to
+					 * 125°C, should be updated to 145°C once available.
+					 */
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+	};
+};
+
+&spmi_bus0 {
+	pm7550: pmic@1 {
+		compatible = "qcom,pm7550", "qcom,spmi-pmic";
+		reg = <1 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pm7550_temp_alarm: temp-alarm@a00 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0xa00>;
+			interrupts = <0x1 0xa 0x0 IRQ_TYPE_EDGE_BOTH>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		pm7550_gpios: gpio@8800 {
+			compatible = "qcom,pm7550-gpio", "qcom,spmi-gpio";
+			reg = <0x8800>;
+			gpio-controller;
+			gpio-ranges = <&pm7550_gpios 0 0 12>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/pm8010-kaanapali.dtsi b/arch/arm64/boot/dts/qcom/pm8010-kaanapali.dtsi
index bfc58a6589d3b..6a8f1accd260f 100644
--- a/arch/arm64/boot/dts/qcom/pm8010-kaanapali.dtsi
+++ b/arch/arm64/boot/dts/qcom/pm8010-kaanapali.dtsi
@@ -75,6 +75,16 @@ pm8010_m_e1_temp_alarm: temp-alarm@2400 {
 			interrupts = <0xc 0x24 0x0 IRQ_TYPE_EDGE_BOTH>;
 			#thermal-sensor-cells = <0>;
 		};
+
+		pm8010_m_e1_gpios: gpio@c000 {
+			compatible = "qcom,pm8010-gpio", "qcom,spmi-gpio";
+			reg = <0xc000>;
+			gpio-controller;
+			gpio-ranges = <&pm8010_m_e1_gpios 0 0 2>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
 	};
 
 	pm8010_n_e1: pmic@d {
@@ -89,5 +99,15 @@ pm8010_n_e1_temp_alarm: temp-alarm@2400 {
 			interrupts = <0xd 0x24 0x0 IRQ_TYPE_EDGE_BOTH>;
 			#thermal-sensor-cells = <0>;
 		};
+
+		pm8010_n_e1_gpios: gpio@c000 {
+			compatible = "qcom,pm8010-gpio", "qcom,spmi-gpio";
+			reg = <0xc000>;
+			gpio-controller;
+			gpio-ranges = <&pm8010_n_e1_gpios 0 0 2>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/qcom/pm8010.dtsi b/arch/arm64/boot/dts/qcom/pm8010.dtsi
index ef330194946b6..031fcf38f3d17 100644
--- a/arch/arm64/boot/dts/qcom/pm8010.dtsi
+++ b/arch/arm64/boot/dts/qcom/pm8010.dtsi
@@ -64,6 +64,16 @@ pm8010_m_temp_alarm: temp-alarm@2400 {
 			interrupts = <0xc 0x24 0x0 IRQ_TYPE_EDGE_BOTH>;
 			#thermal-sensor-cells = <0>;
 		};
+
+		pm8010_m_gpios: gpio@c000 {
+			compatible = "qcom,pm8010-gpio", "qcom,spmi-gpio";
+			reg = <0xc000>;
+			gpio-controller;
+			gpio-ranges = <&pm8010_m_gpios 0 0 2>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
 	};
 
 	pm8010_n: pmic@d {
@@ -78,5 +88,15 @@ pm8010_n_temp_alarm: temp-alarm@2400 {
 			interrupts = <0xd 0x24 0x0 IRQ_TYPE_EDGE_BOTH>;
 			#thermal-sensor-cells = <0>;
 		};
+
+		pm8010_n_gpios: gpio@c000 {
+			compatible = "qcom,pm8010-gpio", "qcom,spmi-gpio";
+			reg = <0xc000>;
+			gpio-controller;
+			gpio-ranges = <&pm8010_n_gpios 0 0 2>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/qcom/pm8550vs-d-eliza.dtsi b/arch/arm64/boot/dts/qcom/pm8550vs-d-eliza.dtsi
new file mode 100644
index 0000000000000..548d2aa839623
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/pm8550vs-d-eliza.dtsi
@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2026 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/spmi/spmi.h>
+
+/ {
+	thermal-zones {
+		pm8550vs-d-thermal {
+			polling-delay-passive = <100>;
+
+			thermal-sensors = <&pm8550vs_d_temp_alarm>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "hot";
+				};
+
+				trip2 {
+					temperature = <145000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+	};
+};
+
+&spmi_bus0 {
+	pm8550vs_d: pmic@3 {
+		compatible = "qcom,pm8550vs", "qcom,spmi-pmic";
+		reg = <3 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pm8550vs_d_temp_alarm: temp-alarm@a00 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0xa00>;
+			interrupts = <0x3 0xa 0x0 IRQ_TYPE_EDGE_BOTH>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		pm8550vs_d_gpios: gpio@8800 {
+			compatible = "qcom,pm8550vs-gpio", "qcom,spmi-gpio";
+			reg = <0x8800>;
+			gpio-controller;
+			gpio-ranges = <&pm8550vs_d_gpios 0 0 6>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/pm8550vs-g-eliza.dtsi b/arch/arm64/boot/dts/qcom/pm8550vs-g-eliza.dtsi
new file mode 100644
index 0000000000000..4ad2cd189a6c1
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/pm8550vs-g-eliza.dtsi
@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2026 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/spmi/spmi.h>
+
+/ {
+	thermal-zones {
+		pm8550vs-g-thermal {
+			polling-delay-passive = <100>;
+
+			thermal-sensors = <&pm8550vs_g_temp_alarm>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "hot";
+				};
+
+				trip2 {
+					temperature = <145000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+	};
+};
+
+&spmi_bus0 {
+	pm8550vs_g: pmic@6 {
+		compatible = "qcom,pm8550vs", "qcom,spmi-pmic";
+		reg = <6 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pm8550vs_g_temp_alarm: temp-alarm@a00 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0xa00>;
+			interrupts = <0x6 0xa 0x0 IRQ_TYPE_EDGE_BOTH>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		pm8550vs_g_gpios: gpio@8800 {
+			compatible = "qcom,pm8550vs-gpio", "qcom,spmi-gpio";
+			reg = <0x8800>;
+			gpio-controller;
+			gpio-ranges = <&pm8550vs_g_gpios 0 0 6>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/pmk8550-eliza.dtsi b/arch/arm64/boot/dts/qcom/pmk8550-eliza.dtsi
new file mode 100644
index 0000000000000..e0ff4ff9c4693
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/pmk8550-eliza.dtsi
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2026 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/input/linux-event-codes.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/spmi/spmi.h>
+
+/ {
+	reboot-mode {
+		compatible = "nvmem-reboot-mode";
+		nvmem-cells = <&reboot_reason>;
+		nvmem-cell-names = "reboot-mode";
+		mode-recovery = <0x01>;
+		mode-bootloader = <0x02>;
+	};
+};
+
+&spmi_bus0 {
+	pmk8550: pmic@0 {
+		compatible = "qcom,pm8550", "qcom,spmi-pmic";
+		reg = <0x0 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmk8550_pon: pon@1300 {
+			compatible = "qcom,pmk8350-pon";
+			reg = <0x1300>, <0x800>;
+			reg-names = "hlos", "pbs";
+
+			pon_pwrkey: pwrkey {
+				compatible = "qcom,pmk8350-pwrkey";
+				interrupts = <0x0 0x13 0x7 IRQ_TYPE_EDGE_BOTH>;
+				linux,code = <KEY_POWER>;
+				status = "disabled";
+			};
+
+			pon_resin: resin {
+				compatible = "qcom,pmk8350-resin";
+				interrupts = <0x0 0x13 0x6 IRQ_TYPE_EDGE_BOTH>;
+				status = "disabled";
+			};
+		};
+
+		pmk8550_rtc: rtc@6100 {
+			compatible = "qcom,pmk8350-rtc";
+			reg = <0x6100>, <0x6200>;
+			reg-names = "rtc", "alarm";
+			interrupts = <0x0 0x62 0x1 IRQ_TYPE_EDGE_RISING>;
+		};
+
+		pmk8550_sdam_2: nvram@7100 {
+			compatible = "qcom,spmi-sdam";
+			reg = <0x7100>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x7100 0x100>;
+
+			reboot_reason: reboot-reason@48 {
+				reg = <0x48 0x1>;
+				bits = <1 7>;
+			};
+		};
+
+		pmk8550_gpios: gpio@b800 {
+			compatible = "qcom,pmk8550-gpio", "qcom,spmi-gpio";
+			reg = <0xb800>;
+			gpio-controller;
+			gpio-ranges = <&pmk8550_gpios 0 0 6>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		pmk8550_pwm: pwm {
+			compatible = "qcom,pmk8550-pwm";
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+			#pwm-cells = <2>;
+
+			status = "disabled";
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/pmr735d-eliza.dtsi b/arch/arm64/boot/dts/qcom/pmr735d-eliza.dtsi
new file mode 100644
index 0000000000000..1c29d8ddea65d
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/pmr735d-eliza.dtsi
@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2026 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/spmi/spmi.h>
+
+/ {
+	thermal-zones {
+		pmr735d-thermal {
+			polling-delay-passive = <100>;
+
+			thermal-sensors = <&pmr735d_temp_alarm>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "hot";
+				};
+
+				trip2 {
+					temperature = <145000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+	};
+};
+
+&spmi_bus0 {
+	pmr735d: pmic@a {
+		compatible = "qcom,pmr735d", "qcom,spmi-pmic";
+		reg = <0xa SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmr735d_temp_alarm: temp-alarm@a00 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0xa00>;
+			interrupts = <0xa 0xa 0x0 IRQ_TYPE_EDGE_BOTH>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		pmr735d_gpios: gpio@8800 {
+			compatible = "qcom,pmr735d-gpio", "qcom,spmi-gpio";
+			reg = <0x8800>;
+			gpio-controller;
+			gpio-ranges = <&pmr735d_gpios 0 0 2>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi b/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi
index 394e65518ac50..ff8621f875058 100644
--- a/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi
+++ b/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi
@@ -389,6 +389,10 @@ &gpu_zap_shader {
 	firmware-name = "qcom/x1p42100/gen71500_zap.mbn";
 };
 
+&iris {
+	status = "okay";
+};
+
 &pcie3 {
 	pinctrl-0 = <&pcie3_default>;
 	pinctrl-names = "default";
diff --git a/arch/arm64/boot/dts/qcom/purwa.dtsi b/arch/arm64/boot/dts/qcom/purwa.dtsi
index 9ab4f26b35f29..c698e6cb2543c 100644
--- a/arch/arm64/boot/dts/qcom/purwa.dtsi
+++ b/arch/arm64/boot/dts/qcom/purwa.dtsi
@@ -6,6 +6,8 @@
 /* X1P42100 is heavily based on hamoa, with some meaningful differences */
 #include "hamoa.dtsi"
 
+#include <dt-bindings/clock/qcom,x1p42100-videocc.h>
+
 /delete-node/ &bwmon_cluster0;
 /delete-node/ &cluster_pd2;
 /delete-node/ &cpu_map_cluster2;
@@ -19,6 +21,7 @@
 /delete-node/ &cpu_pd11;
 /delete-node/ &gpu_opp_table;
 /delete-node/ &gpu_speed_bin;
+/delete-node/ &iris_opp_table;
 /delete-node/ &pcie3_phy;
 /delete-node/ &thermal_aoss3;
 /delete-node/ &thermal_cpu2_0_btm;
@@ -36,6 +39,10 @@
 /delete-node/ &thermal_gpuss_6;
 /delete-node/ &thermal_gpuss_7;
 
+&camcc {
+	compatible = "qcom,x1p42100-camcc";
+};
+
 &gcc {
 	compatible = "qcom,x1p42100-gcc", "qcom,x1e80100-gcc";
 };
@@ -47,6 +54,8 @@ &gmu {
 &gpu {
 	compatible = "qcom,adreno-43030c00", "qcom,adreno";
 
+	iommus = <&adreno_smmu 0 0x0>;
+
 	nvmem-cells = <&gpu_speed_bin>;
 	nvmem-cell-names = "speed_bin";
 
@@ -157,6 +166,55 @@ &gpucc {
 	compatible = "qcom,x1p42100-gpucc";
 };
 
+&iris {
+	compatible = "qcom,x1p42100-iris";
+
+	clocks = <&gcc GCC_VIDEO_AXI0_CLK>,
+		 <&videocc VIDEO_CC_MVS0C_CLK>,
+		 <&videocc VIDEO_CC_MVS0_CLK>,
+		 <&videocc VIDEO_CC_MVS0_BSE_CLK>;
+	clock-names = "iface",
+		      "core",
+		      "vcodec0_core",
+		      "vcodec0_bse";
+
+	operating-points-v2 = <&iris_opp_table_x1p42100>;
+
+	iris_opp_table_x1p42100: opp-table {
+		compatible = "operating-points-v2";
+
+		opp-210000000 {
+			opp-hz = /bits/ 64 <210000000 105000000>;
+			required-opps = <&rpmhpd_opp_low_svs_d1>,
+					<&rpmhpd_opp_low_svs>;
+		};
+
+		opp-300000000 {
+			opp-hz = /bits/ 64 <300000000 150000000>;
+			required-opps = <&rpmhpd_opp_low_svs_d1>,
+					<&rpmhpd_opp_svs>;
+		};
+
+		opp-335000000 {
+			opp-hz = /bits/ 64 <335000000 167500000>;
+			required-opps = <&rpmhpd_opp_svs>,
+					<&rpmhpd_opp_svs_l1>;
+		};
+
+		opp-424000000 {
+			opp-hz = /bits/ 64 <424000000 212000000>;
+			required-opps = <&rpmhpd_opp_svs>,
+					<&rpmhpd_opp_nom>;
+		};
+
+		opp-500000000 {
+			opp-hz = /bits/ 64 <500000000 250000000>;
+			required-opps = <&rpmhpd_opp_svs>,
+					<&rpmhpd_opp_turbo>;
+		};
+	};
+};
+
 /* PCIe3 has half the lanes compared to X1E80100 */
 &pcie3 {
 	num-lanes = <4>;
@@ -256,3 +314,7 @@ &thermal_nsp3 {
 &tsens3 {
 	status = "disabled";
 };
+
+&videocc {
+	compatible = "qcom,x1p42100-videocc";
+};
diff --git a/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts b/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts
index 04cb9230d29fd..f072bd96c7d40 100644
--- a/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts
+++ b/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts
@@ -40,9 +40,9 @@ chosen {
 		#size-cells = <2>;
 		ranges;
 
-		framebuffer0: framebuffer@a000000 {
+		framebuffer {
 			compatible = "simple-framebuffer";
-			reg = <0x0 0xe1000000 0x0 (2700 * 1224 * 4)>;
+			memory-region = <&cont_splash_mem>;
 			width = <1224>;
 			height = <2700>;
 			stride = <(1224 * 4)>;
@@ -866,8 +866,33 @@ vibrator@5a {
 &i2c2 {
 	status = "okay";
 
-	/* AW88261FCR amplifier @ 34 */
-	/* AW88261FCR amplifier @ 35 */
+	/* Top speaker / ear speaker */
+	aw88261_l: audio-codec@34 {
+		compatible = "awinic,aw88261";
+		reg = <0x34>;
+
+		dvdd-supply = <&vreg_l18b>;
+		sound-name-prefix = "Amplifier L";
+		firmware-name = "qcom/qcm6490/fairphone5/aw88261_acf.bin";
+
+		awinic,audio-channel = <0>;
+
+		#sound-dai-cells = <0>;
+	};
+
+	/* Bottom speaker */
+	aw88261_r: audio-codec@35 {
+		compatible = "awinic,aw88261";
+		reg = <0x35>;
+
+		dvdd-supply = <&vreg_l18b>;
+		sound-name-prefix = "Amplifier R";
+		firmware-name = "qcom/qcm6490/fairphone5/aw88261_acf.bin";
+
+		awinic,audio-channel = <1>;
+
+		#sound-dai-cells = <0>;
+	};
 };
 
 &i2c4 {
@@ -913,9 +938,8 @@ &i2c9 {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	firmware-name = "qcom/qcm6490/fairphone5/ipa_fws.mbn";
+
 	status = "okay";
 };
 
@@ -1161,6 +1185,13 @@ &pon_resin {
 	status = "okay";
 };
 
+&q6afedai {
+	dai@127 {
+		reg = <QUINARY_MI2S_RX>;
+		qcom,sd-lines = <0>;
+	};
+};
+
 &qup_spi13_cs {
 	drive-strength = <6>;
 	bias-disable;
@@ -1238,6 +1269,11 @@ &sound {
 	compatible = "fairphone,fp5-sndcard";
 	model = "Fairphone 5";
 
+	pinctrl-0 = <&lpass_i2s1_active>;
+	pinctrl-1 = <&lpass_i2s1_sleep>;
+	pinctrl-names = "default",
+			"sleep";
+
 	mm1-dai-link {
 		link-name = "MultiMedia1";
 
@@ -1246,6 +1282,22 @@ cpu {
 		};
 	};
 
+	i2s-dai-link {
+		link-name = "Quinary MI2S Playback";
+
+		codec {
+			sound-dai = <&aw88261_l>, <&aw88261_r>;
+		};
+
+		cpu {
+			sound-dai = <&q6afedai QUINARY_MI2S_RX>;
+		};
+
+		platform {
+			sound-dai = <&q6routing>;
+		};
+	};
+
 	displayport-rx-dai-link {
 		link-name = "DisplayPort Playback";
 
diff --git a/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts b/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts
index bf18c48520813..94eebb7de7b90 100644
--- a/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts
+++ b/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts
@@ -501,8 +501,6 @@ &i2c1 {
 
 &ipa {
 	firmware-name = "qcom/qcm6490/particle/tachyon/ipa_fws.mbn";
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 
 	status = "okay";
 };
@@ -545,9 +543,6 @@ &mdss_dp_out {
 };
 
 &pcie0 {
-	perst-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 89 GPIO_ACTIVE_HIGH>;
-
 	pinctrl-0 = <&pcie0_reset_n>, <&pcie0_wake_n>, <&pcie0_clkreq_n>;
 	pinctrl-names = "default";
 
@@ -561,9 +556,12 @@ &pcie0_phy {
 	status = "okay";
 };
 
-&pcie1 {
-	perst-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
+&pcie0_port {
+	reset-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 89 GPIO_ACTIVE_HIGH>;
+};
 
+&pcie1 {
 	pinctrl-0 = <&pcie1_reset_n>, <&pcie1_wake_n>, <&pcie1_clkreq_n>;
 	pinctrl-names = "default";
 
@@ -579,6 +577,10 @@ &pcie1_phy {
 	status = "okay";
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
+};
+
 &pmk8350_adc_tm {
 	status = "okay";
 
diff --git a/arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts b/arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts
index 797f37596bf19..e5366cfd3a02a 100644
--- a/arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts
+++ b/arch/arm64/boot/dts/qcom/qcm6490-shift-otter.dts
@@ -39,9 +39,9 @@ chosen {
 
 		stdout-path = "serial0:115200n8";
 
-		framebuffer0: framebuffer@a000000 {
+		framebuffer {
 			compatible = "simple-framebuffer";
-			reg = <0x0 0xe1000000 0x0 (2400 * 1080 * 4)>;
+			memory-region = <&cont_splash_mem>;
 			width = <1080>;
 			height = <2400>;
 			stride = <(1080 * 4)>;
@@ -553,9 +553,8 @@ &i2c13 {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	firmware-name = "qcom/qcm6490/SHIFT/otter/ipa_fws.mbn";
+
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/qcs615-ride.dts b/arch/arm64/boot/dts/qcom/qcs615-ride.dts
index 51b13b492472f..047b4efb04b94 100644
--- a/arch/arm64/boot/dts/qcom/qcs615-ride.dts
+++ b/arch/arm64/boot/dts/qcom/qcs615-ride.dts
@@ -232,8 +232,8 @@ vreg_l1a: ldo1 {
 
 		vreg_l2a: ldo2 {
 			regulator-name = "vreg_l2a";
-			regulator-min-microvolt = <1650000>;
-			regulator-max-microvolt = <3100000>;
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2960000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_LPM>;
 			regulator-allow-set-load;
 			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
@@ -599,7 +599,7 @@ &sdhc_2 {
 	cd-gpios = <&tlmm 99 GPIO_ACTIVE_LOW>;
 
 	vmmc-supply = <&vreg_l10a>;
-	vqmmc-supply = <&vreg_s4a>;
+	vqmmc-supply = <&vreg_l2a>;
 
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts b/arch/arm64/boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts
index bb5a42b038f19..82ebd88f9aba3 100644
--- a/arch/arm64/boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts
+++ b/arch/arm64/boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts
@@ -504,6 +504,10 @@ &i2c13 {
 	status = "okay";
 };
 
+&ipa {
+	/delete-property/ memory-region;
+};
+
 &lpass_audiocc {
 	compatible = "qcom,qcm6490-lpassaudiocc";
 	/delete-property/ power-domains;
@@ -522,9 +526,6 @@ &lpass_va_macro {
 };
 
 &pcie0 {
-	perst-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 89 GPIO_ACTIVE_HIGH>;
-
 	pinctrl-0 = <&pcie0_clkreq_n>, <&pcie0_reset_n>, <&pcie0_wake_n>;
 	pinctrl-names = "default";
 
@@ -538,10 +539,12 @@ &pcie0_phy {
 	status = "okay";
 };
 
-&pcie1 {
-	perst-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 3 GPIO_ACTIVE_HIGH>;
+&pcie0_port {
+	reset-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 89 GPIO_ACTIVE_HIGH>;
+};
 
+&pcie1 {
 	pinctrl-0 = <&pcie1_clkreq_n>, <&pcie1_reset_n>, <&pcie1_wake_n>;
 	pinctrl-names = "default";
 
@@ -566,6 +569,11 @@ &pcie1_phy {
 	status = "okay";
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 3 GPIO_ACTIVE_HIGH>;
+};
+
 &pm7325_gpios {
 	pm7325_adc_default: adc-default-state {
 		pins = "gpio2";
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts
index 37a3b51323ce5..e9a932c90d143 100644
--- a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts
+++ b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts
@@ -806,7 +806,7 @@ &mdss_dp {
 };
 
 &mdss_dp_out {
-	data-lanes = <0 1>;
+	data-lanes = <0 1 2 3>;
 	remote-endpoint = <&usb_dp_qmpphy_dp_in>;
 };
 
@@ -841,8 +841,6 @@ &mdss_edp_phy {
 };
 
 &pcie1 {
-	perst-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
-
 	pinctrl-0 = <&pcie1_reset_n>, <&pcie1_wake_n>, <&pcie1_clkreq_n>;
 	pinctrl-names = "default";
 
@@ -867,6 +865,8 @@ &pcie1_phy {
 };
 
 &pcie1_port0 {
+	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
+
 	pcie@0,0 {
 		compatible = "pci1179,0623";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
@@ -1422,6 +1422,7 @@ &usb_1_qmpphy {
 	vdda-phy-supply = <&vreg_l6b_1p2>;
 	vdda-pll-supply = <&vreg_l1b_0p912>;
 
+	mode-switch;
 	orientation-switch;
 
 	status = "okay";
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts
index a5ad796cb65d0..c8eed8ba93f5b 100644
--- a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts
+++ b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts
@@ -653,6 +653,10 @@ redriver_usb_con_sbu: endpoint {
 	};
 };
 
+&ipa {
+	/delete-property/ memory-region;
+};
+
 &mdss {
 	status = "okay";
 };
@@ -684,9 +688,6 @@ &mdss_dsi_phy {
 };
 
 &pcie0 {
-	perst-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 89 GPIO_ACTIVE_HIGH>;
-
 	pinctrl-0 = <&pcie0_clkreq_n>,
 		    <&pcie0_reset_n>,
 		    <&pcie0_wake_n>;
@@ -702,10 +703,12 @@ &pcie0_phy {
 	status = "okay";
 };
 
-&pcie1 {
-	perst-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 3 GPIO_ACTIVE_HIGH>;
+&pcie0_port {
+	reset-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 89 GPIO_ACTIVE_HIGH>;
+};
 
+&pcie1 {
 	pinctrl-0 = <&pcie1_clkreq_n>,
 		    <&pcie1_reset_n>,
 		    <&pcie1_wake_n>;
@@ -732,6 +735,9 @@ &pcie1_phy {
 };
 
 &pcie1_port0 {
+	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 3 GPIO_ACTIVE_HIGH>;
+
 	pcie@0,0 {
 		compatible = "pci1179,0623";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts
index f47efca42d48d..d8a9a7e47bdc4 100644
--- a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts
+++ b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts
@@ -774,6 +774,10 @@ lt9611_out: endpoint {
 	};
 };
 
+&ipa {
+	/delete-property/ memory-region;
+};
+
 &lpass_audiocc {
 	compatible = "qcom,qcm6490-lpassaudiocc";
 	/delete-property/ power-domains;
@@ -812,9 +816,6 @@ &mdss_dsi_phy {
 };
 
 &pcie0 {
-	perst-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 89 GPIO_ACTIVE_HIGH>;
-
 	pinctrl-0 = <&pcie0_clkreq_n>,
 		    <&pcie0_reset_n>,
 		    <&pcie0_wake_n>;
@@ -830,10 +831,12 @@ &pcie0_phy {
 	status = "okay";
 };
 
-&pcie1 {
-	perst-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 3 GPIO_ACTIVE_LOW>;
+&pcie0_port {
+	reset-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 89 GPIO_ACTIVE_HIGH>;
+};
 
+&pcie1 {
 	pinctrl-0 = <&pcie1_clkreq_n>,
 		    <&pcie1_reset_n>,
 		    <&pcie1_wake_n>;
@@ -849,6 +852,11 @@ &pcie1_phy {
 	status = "okay";
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 3 GPIO_ACTIVE_LOW>;
+};
+
 &pm7325_gpios {
 	kypd_vol_up_n: kypd-vol-up-n-state {
 		pins = "gpio6";
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-vicharak-axon-mini.dts b/arch/arm64/boot/dts/qcom/qcs6490-vicharak-axon-mini.dts
new file mode 100644
index 0000000000000..4f2e0109f49bf
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/qcs6490-vicharak-axon-mini.dts
@@ -0,0 +1,1014 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2026 Vicharak Computers Pvt. Ltd.
+ */
+
+/dts-v1/;
+
+/* PM7250B is configured to use SID8/9 */
+#define PM7250B_SID 8
+#define PM7250B_SID1 9
+
+#include <dt-bindings/iio/qcom,spmi-adc7-pmk8350.h>
+#include <dt-bindings/iio/qcom,spmi-adc7-pm7325.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+
+#include "kodiak.dtsi"
+#include "pm7250b.dtsi"
+#include "pm7325.dtsi"
+#include "pm8350c.dtsi"
+#include "pmk8350.dtsi"
+
+/delete-node/ &adsp_mem;
+/delete-node/ &cdsp_mem;
+/delete-node/ &ipa_fw_mem;
+/delete-node/ &mpss_mem;
+/delete-node/ &remoteproc_mpss;
+/delete-node/ &remoteproc_wpss;
+/delete-node/ &rmtfs_mem;
+/delete-node/ &video_mem;
+/delete-node/ &wifi;
+/delete-node/ &wlan_ce_mem;
+/delete-node/ &wlan_fw_mem;
+/delete-node/ &wpss_mem;
+/delete-node/ &xbl_mem;
+
+/ {
+	model = "Vicharak Axon Mini";
+	compatible = "vicharak,axon-mini", "qcom,qcm6490";
+	chassis-type = "embedded";
+
+	aliases {
+		serial0 = &uart5;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		pinctrl-0 = <&user_leds>;
+		pinctrl-names = "default";
+
+		user-led {
+			color = <LED_COLOR_ID_BLUE>;
+			function = LED_FUNCTION_INDICATOR;
+			gpios = <&tlmm 19 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "default-on";
+			default-state = "on";
+			panic-indicator;
+			retain-state-suspended;
+		};
+
+		status-led {
+			color = <LED_COLOR_ID_BLUE>;
+			function = LED_FUNCTION_INDICATOR;
+			gpios = <&tlmm 55 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+			default-state = "on";
+			retain-state-suspended;
+		};
+	};
+
+	reserved-memory {
+		xbl_mem: xbl@80700000 {
+			reg = <0x0 0x80700000 0x0 0x100000>;
+			no-map;
+		};
+
+		cdsp_secure_heap_mem: cdsp-secure-heap@81800000 {
+			reg = <0x0 0x81800000 0x0 0x1e00000>;
+			no-map;
+		};
+
+		camera_mem: camera@84300000 {
+			reg = <0x0 0x84300000 0x0 0x500000>;
+			no-map;
+		};
+
+		wpss_mem: wpss@84800000 {
+			reg = <0x0 0x84800000 0x0 0x1900000>;
+			no-map;
+		};
+
+		adsp_mem: adsp@86100000 {
+			reg = <0x0 0x86100000 0x0 0x2800000>;
+			no-map;
+		};
+
+		cdsp_mem: cdsp@88900000 {
+			reg = <0x0 0x88900000 0x0 0x1e00000>;
+			no-map;
+		};
+
+		video_mem: video@8a700000 {
+			reg = <0x0 0x8a700000 0x0 0x700000>;
+			no-map;
+		};
+
+		cvp_mem: cvp@8ae00000 {
+			reg = <0x0 0x8ae00000 0x0 0x500000>;
+			no-map;
+		};
+
+		ipa_fw_mem: ipa-fw@8b300000 {
+			reg = <0x0 0x8b300000 0x0 0x10000>;
+			no-map;
+		};
+
+		ipa_gsi_mem: ipa-gsi@8b310000 {
+			reg = <0x0 0x8b310000 0x0 0xa000>;
+			no-map;
+		};
+
+		gpu_microcode_mem: gpu-microcode@8b31a000 {
+			reg = <0x0 0x8b31a000 0x0 0x2000>;
+			no-map;
+		};
+
+		tz_stat_mem: tz-stat@c0000000 {
+			reg = <0x0 0xc0000000 0x0 0x100000>;
+			no-map;
+		};
+
+		tags_mem: tags@c0100000 {
+			reg = <0x0 0xc0100000 0x0 0x1200000>;
+			no-map;
+		};
+
+		qtee_mem: qtee@c1300000 {
+			reg = <0x0 0xc1300000 0x0 0x500000>;
+			no-map;
+		};
+
+		trusted_apps_mem: trusted-apps@c1800000 {
+			reg = <0x0 0xc1800000 0x0 0x1c00000>;
+			no-map;
+		};
+
+		debug_vm_mem: debug-vm@d0600000 {
+			reg = <0x0 0xd0600000 0x0 0x100000>;
+			no-map;
+		};
+	};
+
+	thermal-zones {
+		chg-skin-thermal {
+			polling-delay-passive = <0>;
+
+			thermal-sensors = <&pm7250b_adc_tm 0>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		conn-thermal {
+			polling-delay-passive = <0>;
+
+			thermal-sensors = <&pm7250b_adc_tm 1>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		quiet-thermal {
+			polling-delay-passive = <0>;
+
+			thermal-sensors = <&pmk8350_adc_tm 1>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		sdm-skin-thermal {
+			polling-delay-passive = <0>;
+
+			thermal-sensors = <&pmk8350_adc_tm 3>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		xo-thermal {
+			polling-delay-passive = <0>;
+
+			thermal-sensors = <&pmk8350_adc_tm 0>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+	};
+
+	vcc_5v0: regulator-vcc-5v-peri {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_5v0";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+
+		vin-supply = <&vph_pwr>;
+
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	vcc_5v0_usb2_0: regulator-vcc-5v0-usb2-0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_5v0_usb2_0";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+
+		vin-supply = <&vcc_5v0>;
+
+		gpio = <&tlmm 117 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&vcc5v0_usb2_0_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	vcc_pcie1_3v3: regulator-vcc-pcie1-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_pcie1_3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		vin-supply = <&vcc_5v0>;
+
+		gpio = <&tlmm 115 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&vcc_pcie1_3v3_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	vcc_pcie0_dsi_3v3: regulator-vcc-pcie0-dsi-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_pcie0_dsi_3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		vin-supply = <&vcc_5v0>;
+
+		gpio = <&tlmm 114 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&vcc_pcie0_dsi_3v3_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	wlan_pwrseq: wlan-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+
+		pinctrl-0 = <&wl_enable_h>;
+		pinctrl-names = "default";
+
+		reset-gpios = <&tlmm 84 GPIO_ACTIVE_LOW>;
+		post-power-on-delay-ms = <200>;
+		power-off-delay-us = <20000>;
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pm7325-rpmh-regulators";
+		qcom,pmic-id = "b";
+
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s3-supply = <&vph_pwr>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s6-supply = <&vph_pwr>;
+		vdd-s7-supply = <&vph_pwr>;
+		vdd-s8-supply = <&vph_pwr>;
+		vdd-l1-l4-l12-l15-supply = <&vreg_s7b_0p972>;
+		vdd-l2-l7-supply = <&vreg_bob_3p296>;
+		vdd-l6-l9-l10-supply = <&vreg_s8b_1p272>;
+		vdd-l8-supply = <&vreg_s7b_0p972>;
+		vdd-l11-l17-l18-l19-supply = <&vreg_s1b_1p872>;
+		vdd-l13-supply = <&vreg_s7b_0p972>;
+		vdd-l14-l16-supply = <&vreg_s8b_1p272>;
+
+		vreg_s1b_1p872: smps1 {
+			regulator-name = "vreg_s1b_1p872";
+			regulator-min-microvolt = <1840000>;
+			regulator-max-microvolt = <2040000>;
+		};
+
+		vreg_s7b_0p972: smps7 {
+			regulator-name = "vreg_s7b_0p972";
+			regulator-min-microvolt = <535000>;
+			regulator-max-microvolt = <1120000>;
+		};
+
+		vreg_s8b_1p272: smps8 {
+			regulator-name = "vreg_s8b_1p272";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1500000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_RET>;
+		};
+
+		vreg_l1b_0p912: ldo1 {
+			regulator-name = "vreg_l1b_0p912";
+			regulator-min-microvolt = <825000>;
+			regulator-max-microvolt = <925000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2b_3p072: ldo2 {
+			regulator-name = "vreg_l2b_3p072";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3b_0p504: ldo3 {
+			regulator-name = "vreg_l3b_0p504";
+			regulator-min-microvolt = <312000>;
+			regulator-max-microvolt = <650000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6b_1p2: ldo6 {
+			regulator-name = "vreg_l6b_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1260000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b_2p96: ldo7 {
+			regulator-name = "vreg_l7b_2p96";
+			regulator-min-microvolt = <2960000>;
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b_0p904: ldo8 {
+			regulator-name = "vreg_l8b_0p904";
+			regulator-min-microvolt = <870000>;
+			regulator-max-microvolt = <970000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b_1p2: ldo9 {
+			regulator-name = "vreg_l9b_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11b_1p504: ldo11 {
+			regulator-name = "vreg_l11b_1p504";
+			regulator-min-microvolt = <1776000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b_0p751: ldo12 {
+			regulator-name = "vreg_l12b_0p751";
+			regulator-min-microvolt = <751000>;
+			regulator-max-microvolt = <824000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13b_0p53: ldo13 {
+			regulator-name = "vreg_l13b_0p53";
+			regulator-min-microvolt = <530000>;
+			regulator-max-microvolt = <824000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l14b_1p08: ldo14 {
+			regulator-name = "vreg_l14b_1p08";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_0p765: ldo15 {
+			regulator-name = "vreg_l15b_0p765";
+			regulator-min-microvolt = <765000>;
+			regulator-max-microvolt = <1020000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l16b_1p1: ldo16 {
+			regulator-name = "vreg_l16b_1p1";
+			regulator-min-microvolt = <1100000>;
+			regulator-max-microvolt = <1300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b_1p7: ldo17 {
+			regulator-name = "vreg_l17b_1p7";
+			regulator-min-microvolt = <1700000>;
+			regulator-max-microvolt = <1900000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l18b_1p8: ldo18 {
+			regulator-name = "vreg_l18b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l19b_1p8: ldo19 {
+			regulator-name = "vreg_l19b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pm8350c-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s3-supply = <&vph_pwr>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s6-supply = <&vph_pwr>;
+		vdd-s7-supply = <&vph_pwr>;
+		vdd-s8-supply = <&vph_pwr>;
+		vdd-s9-supply = <&vph_pwr>;
+		vdd-s10-supply = <&vph_pwr>;
+		vdd-l1-l12-supply = <&vreg_s1b_1p872>;
+		vdd-l2-l8-supply = <&vreg_s1b_1p872>;
+		vdd-l3-l4-l5-l7-l13-supply = <&vreg_bob_3p296>;
+		vdd-l6-l9-l11-supply = <&vreg_bob_3p296>;
+		vdd-l10-supply = <&vreg_s7b_0p972>;
+		vdd-bob-supply = <&vph_pwr>;
+
+		vreg_s1c_2p19: smps1 {
+			regulator-name = "vreg_s1c_2p19";
+			regulator-min-microvolt = <2200000>;
+			regulator-max-microvolt = <2208000>;
+		};
+
+		vreg_s9c_1p084: smps9 {
+			regulator-name = "vreg_s9c_1p084";
+			regulator-min-microvolt = <1010000>;
+			regulator-max-microvolt = <1170000>;
+		};
+
+		vreg_l1c_1p8: ldo1 {
+			regulator-name = "vreg_l1c_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1980000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2c_1p62: ldo2 {
+			regulator-name = "vreg_l2c_1p62";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <1976000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3c_2p8: ldo3 {
+			regulator-name = "vreg_l3c_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <3540000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4c_1p62: ldo4 {
+			regulator-name = "vreg_l4c_1p62";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5c_1p62: ldo5 {
+			regulator-name = "vreg_l5c_1p62";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6c_2p96: ldo6 {
+			regulator-name = "vreg_l6c_2p96";
+			regulator-min-microvolt = <1650000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7c_3p0: ldo7 {
+			regulator-name = "vreg_l7c_3p0";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8c_1p62: ldo8 {
+			regulator-name = "vreg_l8c_1p62";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9c_2p96: ldo9 {
+			regulator-name = "vreg_l9c_2p96";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10c_0p88: ldo10 {
+			regulator-name = "vreg_l10c_0p88";
+			regulator-min-microvolt = <720000>;
+			regulator-max-microvolt = <1050000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11c_2p8: ldo11 {
+			regulator-name = "vreg_l11c_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12c_1p8: ldo12 {
+			regulator-name = "vreg_l12c_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+
+			/*
+			 * VREG_L12C_1P8 supplies the Ampak WLAN/BT module
+			 * VDDIO and the external 32.768 kHz oscillator.
+			 */
+			regulator-always-on;
+			regulator-boot-on;
+		};
+
+		vreg_l13c_2p7: ldo13 {
+			regulator-name = "vreg_l13c_2p7";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob_3p296: bob {
+			regulator-name = "vreg_bob_3p296";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3300000>;
+		};
+	};
+};
+
+&gcc {
+	protected-clocks = <GCC_CFG_NOC_LPASS_CLK>,
+			   <GCC_MSS_CFG_AHB_CLK>,
+			   <GCC_MSS_GPLL0_MAIN_DIV_CLK_SRC>,
+			   <GCC_MSS_OFFLINE_AXI_CLK>,
+			   <GCC_MSS_Q6SS_BOOT_CLK_SRC>,
+			   <GCC_MSS_Q6_MEMNOC_AXI_CLK>,
+			   <GCC_MSS_SNOC_AXI_CLK>,
+			   <GCC_QSPI_CNOC_PERIPH_AHB_CLK>,
+			   <GCC_QSPI_CORE_CLK>,
+			   <GCC_QSPI_CORE_CLK_SRC>,
+			   <GCC_SEC_CTRL_CLK_SRC>,
+			   <GCC_WPSS_AHB_BDG_MST_CLK>,
+			   <GCC_WPSS_AHB_CLK>,
+			   <GCC_WPSS_RSCP_CLK>;
+};
+
+&gpi_dma0 {
+	status = "okay";
+};
+
+&gpi_dma1 {
+	status = "okay";
+};
+
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/qcs6490/a660_zap.mbn";
+};
+
+&ice {
+	status = "disabled";
+};
+
+&pcie0 {
+	perst-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 89 GPIO_ACTIVE_HIGH>;
+
+	pinctrl-0 = <&pcie0_clkreq_n>, <&pcie0_reset_n>, <&pcie0_wake_n>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie0_phy {
+	vdda-phy-supply = <&vreg_l10c_0p88>;
+	vdda-pll-supply = <&vreg_l6b_1p2>;
+
+	status = "okay";
+};
+
+&pcie1 {
+	perst-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 3 GPIO_ACTIVE_HIGH>;
+
+	pinctrl-0 = <&pcie1_reset_n>, <&pcie1_wake_n>, <&pcie1_clkreq_n>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie1_phy {
+	vdda-phy-supply = <&vreg_l10c_0p88>;
+	vdda-pll-supply = <&vreg_l6b_1p2>;
+
+	status = "okay";
+};
+
+&pm7250b_adc {
+	channel@4d {
+		reg = <ADC5_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "charger_skin_therm";
+	};
+
+	channel@4f {
+		reg = <ADC5_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "conn_therm";
+	};
+};
+
+&pm7250b_adc_tm {
+	status = "okay";
+
+	charger-skin-therm@0 {
+		reg = <0>;
+		io-channels = <&pm7250b_adc ADC5_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	conn-therm@1 {
+		reg = <1>;
+		io-channels = <&pm7250b_adc ADC5_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+};
+
+&pm7325_temp_alarm {
+	io-channels = <&pmk8350_vadc PM7325_ADC7_DIE_TEMP>;
+	io-channel-names = "thermal";
+};
+
+&pmk8350_adc_tm {
+	status = "okay";
+
+	xo-therm@0 {
+		reg = <0>;
+		io-channels = <&pmk8350_vadc PMK8350_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	quiet-therm@1 {
+		reg = <1>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	sdm-skin-therm@3 {
+		reg = <3>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+};
+
+&pmk8350_rtc {
+	status = "okay";
+};
+
+&pmk8350_vadc {
+	status = "okay";
+
+	channel@3 {
+		reg = <PMK8350_ADC7_DIE_TEMP>;
+		qcom,pre-scaling = <1 1>;
+		label = "pmk8350_die_therm";
+	};
+
+	channel@44 {
+		reg = <PMK8350_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pmk8350_xo_therm";
+	};
+
+	channel@103 {
+		reg = <PM7325_ADC7_DIE_TEMP>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_die_therm";
+	};
+
+	channel@144 {
+		reg = <PM7325_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_quiet_therm";
+	};
+
+	channel@146 {
+		reg = <PM7325_ADC7_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_sdm_skin_therm";
+	};
+};
+
+&pon_pwrkey {
+	status = "okay";
+};
+
+&pon_resin {
+	linux,code = <KEY_RESTART>;
+
+	status = "okay";
+};
+
+&qupv3_id_0 {
+	firmware-name = "qcom/qcs6490/qupv3fw.elf";
+
+	status = "okay";
+};
+
+&qupv3_id_1 {
+	firmware-name = "qcom/qcs6490/qupv3fw.elf";
+
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/qcs6490/adsp.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/qcs6490/cdsp.mbn";
+
+	status = "okay";
+};
+
+&sdhc_1 {
+	vqmmc-supply = <&vreg_l19b_1p8>;
+	vmmc-supply = <&vreg_bob_3p296>;
+
+	non-removable;
+	no-sd;
+	no-sdio;
+
+	status = "okay";
+};
+
+&sdhc_2 {
+	vqmmc-supply = <&vreg_l2c_1p62>;
+	vmmc-supply = <&vreg_l6c_2p96>;
+
+	mmc-pwrseq = <&wlan_pwrseq>;
+
+	bus-width = <4>;
+	non-removable;
+	no-sd;
+	no-mmc;
+	cap-sdio-irq;
+	keep-power-in-suspend;
+	max-frequency = <100000000>;
+
+	status = "okay";
+};
+
+&tlmm {
+	gpio-reserved-ranges = <48 4>; /* NFC */
+
+	pcie0_reset_n: pcie0-reset-n-state {
+		pins = "gpio87";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	pcie0_wake_n: pcie0-wake-n-state {
+		pins = "gpio89";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	pcie1_reset_n: pcie1-reset-n-state {
+		pins = "gpio2";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	pcie1_wake_n: pcie1-wake-n-state {
+		pins = "gpio3";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	user_leds: user-led-state {
+		pins = "gpio19", "gpio55";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	vcc5v0_usb2_0_en: vcc-5v0-usb2-0-en-state {
+		pins = "gpio117";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	vcc_pcie0_dsi_3v3_en: vcc-pcie0-dsi-3v3-en-state {
+		pins = "gpio114";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	vcc_pcie1_3v3_en: vcc-pcie1-3v3-en-state {
+		pins = "gpio115";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	wl_enable_h: wl-enable-h-state {
+		pins = "gpio84";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
+};
+
+&uart5 {
+	status = "okay";
+};
+
+&ufs_mem_hc {
+	/delete-property/ qcom,ice;
+
+	reset-gpios = <&tlmm 175 GPIO_ACTIVE_LOW>;
+	vcc-supply = <&vreg_l7b_2p96>;
+	vcc-max-microamp = <800000>;
+	vccq-supply = <&vreg_l9b_1p2>;
+	vccq-max-microamp = <900000>;
+	vccq2-supply = <&vreg_l9b_1p2>;
+	vccq2-max-microamp = <900000>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vreg_l10c_0p88>;
+	vdda-pll-supply = <&vreg_l6b_1p2>;
+
+	status = "okay";
+};
+
+&usb_2 {
+	/* Routed to an onboard USB hub for two USB-A host ports. */
+	dr_mode = "host";
+
+	status = "okay";
+};
+
+&usb_2_hsphy {
+	vdda-pll-supply = <&vreg_l10c_0p88>;
+	vdda33-supply = <&vreg_l2b_3p072>;
+	vdda18-supply = <&vreg_l1c_1p8>;
+
+	status = "okay";
+};
+
+&venus {
+	status = "okay";
+};
+
+/* pinctrl */
+&pcie0_clkreq_n {
+	bias-pull-up;
+	drive-strength = <2>;
+};
+
+&pcie1_clkreq_n {
+	bias-pull-up;
+	drive-strength = <2>;
+};
+
+&sdc1_clk {
+	bias-disable;
+	drive-strength = <16>;
+};
+
+&sdc1_cmd {
+	bias-pull-up;
+	drive-strength = <10>;
+};
+
+&sdc1_data {
+	bias-pull-up;
+	drive-strength = <10>;
+};
+
+&sdc1_rclk {
+	bias-pull-down;
+};
+
+&sdc2_clk {
+	bias-disable;
+	drive-strength = <16>;
+};
+
+&sdc2_cmd {
+	bias-pull-up;
+	drive-strength = <10>;
+};
+
+&sdc2_data {
+	bias-pull-up;
+	drive-strength = <10>;
+};
diff --git a/arch/arm64/boot/dts/qcom/qcs8550-rb5gen2.dts b/arch/arm64/boot/dts/qcom/qcs8550-rb5gen2.dts
new file mode 100644
index 0000000000000..9df8dcd6c62b6
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/qcs8550-rb5gen2.dts
@@ -0,0 +1,1573 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2026 Axon Enterprise, Inc.
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+#include "qcs8550.dtsi"
+#include "pm8010.dtsi"
+#include "pm8550.dtsi"
+#include "pm8550b.dtsi"
+#define PMK8550VE_SID 5
+#include "pm8550ve.dtsi"
+#include "pm8550vs.dtsi"
+#include "pmk8550.dtsi"
+#include "pmr735d_a.dtsi"
+#include "pmr735d_b.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. QCS8550 RB5Gen2";
+	compatible = "qcom,qcs8550-rb5gen2", "qcom,qcs8550", "qcom,sm8550";
+	chassis-type = "embedded";
+
+	aliases {
+		serial0 = &uart7;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	clocks {
+		clk40m: can-clk {
+			compatible = "fixed-clock";
+			clock-frequency = <40000000>;
+			#clock-cells = <0>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&volume_up_n>;
+		pinctrl-names = "default";
+
+		key-volume-up {
+			label = "Volume Up";
+			linux,code = <KEY_VOLUMEUP>;
+			gpios = <&pm8550_gpios 6 GPIO_ACTIVE_LOW>;
+			debounce-interval = <15>;
+			linux,can-disable;
+			wakeup-source;
+		};
+	};
+
+	hdmi-connector {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con: endpoint {
+				remote-endpoint = <&lt9611_out>;
+			};
+		};
+	};
+
+	/* Lontium LT9611UXC fails FW upgrade and has timeouts with geni-i2c */
+	/* Workaround is to use bit-banged I2C */
+	i2c_hub_3_gpio: i2c {
+		compatible = "i2c-gpio";
+
+		sda-gpios = <&tlmm 22 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+		scl-gpios = <&tlmm 23 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-0 {
+			label = "green:status-3";
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_GREEN>;
+			gpios = <&pm8550_gpios 2 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-1 {
+			label = "blue:bt-power";
+			function = LED_FUNCTION_BLUETOOTH;
+			color = <LED_COLOR_ID_BLUE>;
+			gpios = <&pm8550b_gpios 7 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "bluetooth-power";
+			default-state = "off";
+		};
+
+		led-2 {
+			label = "yellow:wlan";
+			function = LED_FUNCTION_WLAN;
+			color = <LED_COLOR_ID_YELLOW>;
+			gpios = <&pm8550b_gpios 9 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "phy0tx";
+			default-state = "off";
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,sm8550-pmic-glink", "qcom,pmic-glink";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in: endpoint {
+						remote-endpoint = <&usb_1_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss_in: endpoint {
+						remote-endpoint = <&redriver_usb_con_ss>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_sbu_in: endpoint {
+						remote-endpoint = <&redriver_usb_con_sbu>;
+					};
+				};
+			};
+		};
+	};
+
+	lt9611_1v2: regulator-lt9611-1v2 {
+		compatible = "regulator-fixed";
+		regulator-name = "LT9611_1V2";
+
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+
+		vin-supply = <&vreg_l14b_3p2>;
+	};
+
+	lt9611_3v3: regulator-lt9611-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "LT9611_3V3";
+
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		vin-supply = <&vreg_l14b_3p2>;
+	};
+
+	pcie_upd_1p05: regulator-pcie-upd-1p05 {
+		compatible = "regulator-fixed";
+		regulator-name = "PCIE_UPD_1P05";
+		gpio = <&tlmm 179 GPIO_ACTIVE_HIGH>;
+		vin-supply = <&vdd_ntn_0p9>;
+		regulator-min-microvolt = <1050000>;
+		regulator-max-microvolt = <1050000>;
+		enable-active-high;
+		regulator-enable-ramp-delay = <5000>;
+		pinctrl-0 = <&upd_1p05_en>;
+		pinctrl-names = "default";
+	};
+
+	pcie_upd_3p3: regulator-pcie-upd-3p3 {
+		compatible = "regulator-fixed";
+		regulator-name = "PCIE_UPD_3P3";
+		gpio = <&tlmm 13 GPIO_ACTIVE_HIGH>;
+		vin-supply = <&pcie_upd_1p05>;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		enable-active-high;
+		regulator-enable-ramp-delay = <10000>;
+		pinctrl-0 = <&upd_3p3_en>;
+		pinctrl-names = "default";
+	};
+
+	vdd_ntn_0p9: regulator-vdd-ntn-0p9 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD_NTN_0P9";
+		vin-supply = <&vdd_ntn_1p8>;
+		regulator-min-microvolt = <899400>;
+		regulator-max-microvolt = <899400>;
+		regulator-enable-ramp-delay = <4300>;
+	};
+
+	vdd_ntn_1p8: regulator-vdd-ntn-1p8 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD_NTN_1P8";
+		gpio = <&tlmm 67 GPIO_ACTIVE_HIGH>;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		enable-active-high;
+		pinctrl-0 = <&ntn0_en>;
+		pinctrl-names = "default";
+		regulator-enable-ramp-delay = <10000>;
+	};
+
+	vdd_ntn1_0p9: regulator-vdd-ntn1-0p9 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD_NTN1_0P9";
+		vin-supply = <&vdd_ntn1_1p8>;
+		regulator-min-microvolt = <899400>;
+		regulator-max-microvolt = <899400>;
+		regulator-enable-ramp-delay = <4300>;
+	};
+
+	vdd_ntn1_1p8: regulator-vdd-ntn1-1p8 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD_NTN1_1P8";
+		gpio = <&tlmm 42 GPIO_ACTIVE_HIGH>;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		enable-active-high;
+		pinctrl-0 = <&ntn1_en>;
+		pinctrl-names = "default";
+		regulator-enable-ramp-delay = <10000>;
+	};
+
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	sound {
+		compatible = "qcom,sm8550-sndcard", "qcom,sm8450-sndcard";
+		model = "QCS8550-RB5Gen2";
+		audio-routing = "SpkrLeft IN", "WSA_SPK1 OUT",
+				"SpkrRight IN", "WSA_SPK2 OUT",
+				"VA DMIC0", "vdd-micb",
+				"VA DMIC1", "vdd-micb";
+
+		wsa-dai-link {
+			link-name = "WSA Playback";
+
+			codec {
+				sound-dai = <&left_spkr>, <&right_spkr>,
+					    <&swr0 0>, <&lpass_wsamacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai WSA_CODEC_DMA_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		va-dai-link {
+			link-name = "VA Capture";
+
+			codec {
+				sound-dai = <&lpass_vamacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai VA_CODEC_DMA_TX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+	};
+
+	wcn7850-pmu {
+		compatible = "qcom,wcn7850-pmu";
+
+		pinctrl-0 = <&wlan_en>, <&bt_default>, <&sw_ctrl_default>,
+			    <&pmk8550_sleep_clk>;
+		pinctrl-names = "default";
+
+		wlan-enable-gpios = <&tlmm 80 GPIO_ACTIVE_HIGH>;
+		bt-enable-gpios = <&tlmm 81 GPIO_ACTIVE_HIGH>;
+		swctrl-gpios = <&tlmm 82 GPIO_ACTIVE_HIGH>;
+
+		vdd-supply = <&vreg_s5g_0p85>;
+		vddio-supply = <&vreg_l15b_1p8>;
+		vddaon-supply = <&vreg_s2g_0p852>;
+		vdddig-supply = <&vreg_s4e_0p95>;
+		vddrfa1p2-supply = <&vreg_s4g_1p25>;
+		vddrfa1p8-supply = <&vreg_s6g_1p86>;
+
+		regulators {
+			vreg_pmu_rfa_cmn: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn";
+			};
+
+			vreg_pmu_aon_0p59: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p59";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p85: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p85";
+			};
+
+			vreg_pmu_btcmx_0p85: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p85";
+			};
+
+			vreg_pmu_rfa_0p8: ldo5 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo6 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p8: ldo7 {
+				regulator-name = "vreg_pmu_rfa_1p8";
+			};
+
+			vreg_pmu_pcie_0p9: ldo8 {
+				regulator-name = "vreg_pmu_pcie_0p9";
+			};
+
+			vreg_pmu_pcie_1p8: ldo9 {
+				regulator-name = "vreg_pmu_pcie_1p8";
+			};
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pm8550-rpmh-regulators";
+		qcom,pmic-id = "b";
+
+		vdd-bob1-supply = <&vph_pwr>;
+		vdd-bob2-supply = <&vph_pwr>;
+		vdd-l1-l4-l10-supply = <&vreg_s6g_1p86>;
+		vdd-l2-l13-l14-supply = <&vreg_bob1>;
+		vdd-l3-supply = <&vreg_s4g_1p25>;
+		vdd-l5-l16-supply = <&vreg_bob1>;
+		vdd-l6-l7-supply = <&vreg_bob1>;
+		vdd-l8-l9-supply = <&vreg_bob1>;
+		vdd-l11-supply = <&vreg_s4g_1p25>;
+		vdd-l12-supply = <&vreg_s6g_1p86>;
+		vdd-l15-supply = <&vreg_s6g_1p86>;
+		vdd-l17-supply = <&vreg_bob2>;
+
+		vreg_bob1: bob1 {
+			regulator-name = "vreg_bob1";
+			regulator-min-microvolt = <3296000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob2: bob2 {
+			regulator-name = "vreg_bob2";
+			regulator-min-microvolt = <2720000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1b_1p8: ldo1 {
+			regulator-name = "vreg_l1b_1p8";
+			regulator-min-microvolt = <1710000>;
+			regulator-max-microvolt = <1950000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2b_3p0: ldo2 {
+			regulator-name = "vreg_l2b_3p0";
+			regulator-min-microvolt = <2900000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5b_3p1: ldo5 {
+			regulator-name = "vreg_l5b_3p1";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6b_1p8: ldo6 {
+			regulator-name = "vreg_l6b_1p8";
+			regulator-min-microvolt = <1700000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b_1p8: ldo7 {
+			regulator-name = "vreg_l7b_1p8";
+			regulator-min-microvolt = <1700000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b_1p8: ldo8 {
+			regulator-name = "vreg_l8b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b_2p9: ldo9 {
+			regulator-name = "vreg_l9b_2p9";
+			regulator-min-microvolt = <2960000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11b_1p2: ldo11 {
+			regulator-name = "vreg_l11b_1p2";
+			regulator-min-microvolt = <1100000>;
+			regulator-max-microvolt = <1300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b_1p8: ldo12 {
+			regulator-name = "vreg_l12b_1p8";
+			regulator-min-microvolt = <1710000>;
+			regulator-max-microvolt = <1950000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13b_3p0: ldo13 {
+			regulator-name = "vreg_l13b_3p0";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l14b_3p2: ldo14 {
+			regulator-name = "vreg_l14b_3p2";
+			regulator-min-microvolt = <3200000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+		};
+
+		vreg_l15b_1p8: ldo15 {
+			regulator-name = "vreg_l15b_1p8";
+			regulator-min-microvolt = <1760000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l16b_2p8: ldo16 {
+			regulator-name = "vreg_l16b_2p8";
+			regulator-min-microvolt = <2000000>;
+			regulator-max-microvolt = <3400000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b_2p5: ldo17 {
+			regulator-name = "vreg_l17b_2p5";
+			regulator-min-microvolt = <2400000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vdd-l1-supply = <&vreg_s4g_1p25>;
+		vdd-l2-supply = <&vreg_s4e_0p95>;
+		vdd-l3-supply = <&vreg_s4e_0p95>;
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s6-supply = <&vph_pwr>;
+
+		vreg_l3c_0p9: ldo3 {
+			regulator-name = "vreg_l3c_0p9";
+			regulator-min-microvolt = <835000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+		qcom,pmic-id = "d";
+
+		vdd-l1-supply = <&vreg_s4e_0p95>;
+		vdd-l2-supply = <&vreg_s4e_0p95>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+
+		vreg_l1d_0p88: ldo1 {
+			regulator-name = "vreg_l1d_0p88";
+			regulator-min-microvolt = <825000>;
+			regulator-max-microvolt = <958000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2d_0p752: ldo2 {
+			regulator-name = "vreg_l2d_0p752";
+			regulator-min-microvolt = <675000>;
+			regulator-max-microvolt = <808000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s4d_0p628: smps4 {
+			regulator-name = "vreg_s4d_0p628";
+			regulator-min-microvolt = <572000>;
+			regulator-max-microvolt = <988000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s5d_0p728: smps5 {
+			regulator-name = "vreg_s5d_0p728";
+			regulator-min-microvolt = <572000>;
+			regulator-max-microvolt = <988000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-3 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+		qcom,pmic-id = "e";
+
+		vdd-l1-supply = <&vreg_s4e_0p95>;
+		vdd-l2-supply = <&vreg_s4e_0p95>;
+		vdd-l3-supply = <&vreg_s4g_1p25>;
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s3-supply = <&vph_pwr>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s6-supply = <&vph_pwr>;
+
+		vreg_l1e_0p88: ldo1 {
+			regulator-name = "vreg_l1e_0p88";
+			regulator-min-microvolt = <831000>;
+			regulator-max-microvolt = <904000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2e_0p9: ldo2 {
+			regulator-name = "vreg_l2e_0p9";
+			regulator-min-microvolt = <870000>;
+			regulator-max-microvolt = <970000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3e_1p2: ldo3 {
+			regulator-name = "vreg_l3e_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s1e_0p72: smps1 {
+			regulator-name = "vreg_s1e_0p72";
+			regulator-min-microvolt = <532000>;
+			regulator-max-microvolt = <852000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s3e_0p75: smps3 {
+			regulator-name = "vreg_s3e_0p75";
+			regulator-min-microvolt = <716000>;
+			regulator-max-microvolt = <884000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s4e_0p95: smps4 {
+			regulator-name = "vreg_s4e_0p95";
+			regulator-min-microvolt = <870100>;
+			regulator-max-microvolt = <1152000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s5e_1p08: smps5 {
+			regulator-name = "vreg_s5e_1p08";
+			regulator-min-microvolt = <1010000>;
+			regulator-max-microvolt = <1120000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s6e_0p728: smps6 {
+			regulator-name = "vreg_s6e_0p728";
+			regulator-min-microvolt = <528000>;
+			regulator-max-microvolt = <904000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-4 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "f";
+
+		vdd-l1-supply = <&vreg_s4e_0p95>;
+		vdd-l2-supply = <&vreg_s4e_0p95>;
+		vdd-l3-supply = <&vreg_s4e_0p95>;
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s3-supply = <&vph_pwr>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s7-supply = <&vph_pwr>;
+
+		vreg_l1f_0p9: ldo1 {
+			regulator-name = "vreg_l1f_0p9";
+			regulator-min-microvolt = <866000>;
+			regulator-max-microvolt = <958000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2f_0p88: ldo2 {
+			regulator-name = "vreg_l2f_0p88";
+			regulator-min-microvolt = <866000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3f_0p88: ldo3 {
+			regulator-name = "vreg_l3f_0p88";
+			regulator-min-microvolt = <830000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s1f_0p728: smps1 {
+			regulator-name = "vreg_s1f_0p728";
+			regulator-min-microvolt = <516000>;
+			regulator-max-microvolt = <904000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s3f_0p852: smps3 {
+			regulator-name = "vreg_s3f_0p852";
+			regulator-min-microvolt = <688000>;
+			regulator-max-microvolt = <952000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s4f_0p5: smps4 {
+			regulator-name = "vreg_s4f_0p5";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <500000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s5f_0p752: smps5 {
+			regulator-name = "vreg_s5f_0p752";
+			regulator-min-microvolt = <716000>;
+			regulator-max-microvolt = <884000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s7f_0p628: smps7 {
+			regulator-name = "vreg_s7f_0p628";
+			regulator-min-microvolt = <516000>;
+			regulator-max-microvolt = <812000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-5 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+		qcom,pmic-id = "g";
+
+		vdd-l1-supply = <&vreg_s4g_1p25>;
+		vdd-l2-supply = <&vreg_s4g_1p25>;
+		vdd-l3-supply = <&vreg_s4g_1p25>;
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s3-supply = <&vph_pwr>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s6-supply = <&vph_pwr>;
+
+		vreg_l1g_1p2: ldo1 {
+			regulator-name = "vreg_l1g_1p2";
+			regulator-min-microvolt = <1140000>;
+			regulator-max-microvolt = <1260000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3g_1p2: ldo3 {
+			regulator-name = "vreg_l3g_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s1g_1p256: smps1 {
+			regulator-name = "vreg_s1g_1p256";
+			regulator-min-microvolt = <1172000>;
+			regulator-max-microvolt = <1388000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s2g_0p852: smps2 {
+			regulator-name = "vreg_s2g_0p852";
+			regulator-min-microvolt = <500000>;
+			regulator-max-microvolt = <1053200>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s3g_0p752: smps3 {
+			regulator-name = "vreg_s3g_0p752";
+			regulator-min-microvolt = <532000>;
+			regulator-max-microvolt = <1148000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s4g_1p25: smps4 {
+			regulator-name = "vreg_s4g_1p25";
+			regulator-min-microvolt = <1172000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s5g_0p85: smps5 {
+			regulator-name = "vreg_s5g_0p85";
+			regulator-min-microvolt = <800000>;
+			regulator-max-microvolt = <1002600>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s6g_1p86: smps6 {
+			regulator-name = "vreg_s6g_1p86";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2192000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-6 {
+		compatible = "qcom,pm8010-rpmh-regulators";
+
+		vdd-l1-l2-supply = <&vreg_s4g_1p25>;
+		vdd-l3-l4-supply = <&vreg_bob2>;
+		vdd-l5-supply = <&vreg_s6g_1p86>;
+		vdd-l6-supply = <&vreg_s6g_1p86>;
+		vdd-l7-supply = <&vreg_bob1>;
+
+		qcom,pmic-id = "m";
+
+		vreg_l1m_1p056: ldo1 {
+			regulator-name = "vreg_l1m_1p056";
+			regulator-min-microvolt = <1056000>;
+			regulator-max-microvolt = <1056000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2m_1p056: ldo2 {
+			regulator-name = "vreg_l2m_1p056";
+			regulator-min-microvolt = <1056000>;
+			regulator-max-microvolt = <1056000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3m_2p8: ldo3 {
+			regulator-name = "vreg_l3m_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4m_2p8: ldo4 {
+			regulator-name = "vreg_l4m_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5m_1p8: ldo5 {
+			regulator-name = "vreg_l5m_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6m_1p8: ldo6 {
+			regulator-name = "vreg_l6m_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7m_2p9: ldo7 {
+			regulator-name = "vreg_l7m_2p9";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2904000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-7 {
+		compatible = "qcom,pm8010-rpmh-regulators";
+
+		vdd-l1-l2-supply = <&vreg_s4g_1p25>;
+		vdd-l3-l4-supply = <&vreg_bob2>;
+		vdd-l5-supply = <&vreg_s6g_1p86>;
+		vdd-l6-supply = <&vreg_bob1>;
+		vdd-l7-supply = <&vreg_bob1>;
+
+		qcom,pmic-id = "n";
+
+		vreg_l1n_1p1: ldo1 {
+			regulator-name = "vreg_l1n_1p1";
+			regulator-min-microvolt = <1104000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2n_1p1: ldo2 {
+			regulator-name = "vreg_l2n_1p1";
+			regulator-min-microvolt = <1104000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3n_2p8: ldo3 {
+			regulator-name = "vreg_l3n_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <3000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4n_2p8: ldo4 {
+			regulator-name = "vreg_l4n_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5n_1p8: ldo5 {
+			regulator-name = "vreg_l5n_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6n_3p3: ldo6 {
+			regulator-name = "vreg_l6n_3p3";
+			regulator-min-microvolt = <3200000>;
+			regulator-max-microvolt = <3200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7n_2p96: ldo7 {
+			regulator-name = "vreg_l7n_2p96";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&gpi_dma1 {
+	status = "okay";
+};
+
+&gpi_dma2 {
+	status = "okay";
+};
+
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/sm8550/a740_zap.mbn";
+};
+
+&i2c_hub_2 {
+	clock-frequency = <100000>;
+
+	status = "okay";
+
+	typec-mux@1c {
+		compatible = "onnn,nb7vpq904m";
+		reg = <0x1c>;
+
+		vcc-supply = <&vreg_l15b_1p8>;
+
+		retimer-switch;
+		orientation-switch;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				redriver_usb_con_ss: endpoint {
+					remote-endpoint = <&pmic_glink_ss_in>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				redriver_phy_con_ss: endpoint {
+					remote-endpoint = <&usb_dp_qmpphy_out>;
+					data-lanes = <0 1 2 3>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+
+				redriver_usb_con_sbu: endpoint {
+					remote-endpoint = <&pmic_glink_sbu_in>;
+				};
+			};
+		};
+	};
+};
+
+&i2c_hub_3_gpio {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	lt9611_codec: hdmi-bridge@2b {
+		compatible = "lontium,lt9611uxc";
+		reg = <0x2b>;
+
+		interrupts-extended = <&tlmm 40 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&tlmm 7 GPIO_ACTIVE_HIGH>;
+
+		vdd-supply = <&lt9611_1v2>;
+		vcc-supply = <&lt9611_3v3>;
+
+		pinctrl-0 = <&lt9611_irq_pin>, <&lt9611_rst_pin>;
+		pinctrl-names = "default";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				lt9611_a: endpoint {
+					remote-endpoint = <&mdss_dsi0_out>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+
+				lt9611_out: endpoint {
+					remote-endpoint = <&hdmi_con>;
+				};
+			};
+		};
+	};
+};
+
+&i2c_hub_4 {
+	status = "okay";
+};
+
+&i2c_master_hub_0 {
+	status = "okay";
+};
+
+&ipa {
+	firmware-name = "qcom/sm8550/ipa_fws.mbn";
+
+	status = "okay";
+};
+
+&iris {
+	status = "okay";
+};
+
+&lpass_vamacro {
+	pinctrl-0 = <&dmic01_default>;
+	pinctrl-names = "default";
+
+	qcom,dmic-sample-rate = <4800000>;
+
+	vdd-micb-supply = <&vreg_l15b_1p8>;
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dsi0 {
+	vdda-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&mdss_dsi0_out {
+	remote-endpoint = <&lt9611_a>;
+	data-lanes = <0 1 2 3>;
+};
+
+&mdss_dsi0_phy {
+	vdds-supply = <&vreg_l1e_0p88>;
+
+	status = "okay";
+};
+
+&mdss_dp0 {
+	status = "okay";
+};
+
+&pcie0 {
+	vddpe-3v3-supply = <&pcie_upd_3p3>;
+
+	wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>;
+	perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+
+	pinctrl-0 = <&pcie0_default_state>;
+	pinctrl-names = "default";
+
+	iommu-map = <0x0 &apps_smmu 0x1400 0x1>,
+		    <0x100 &apps_smmu 0x1401 0x1>,
+		    <0x208 &apps_smmu 0x1402 0x1>,
+		    <0x210 &apps_smmu 0x1403 0x1>,
+		    <0x218 &apps_smmu 0x1404 0x1>,
+		    <0x300 &apps_smmu 0x1407 0x1>,
+		    <0x400 &apps_smmu 0x1408 0x1>,
+		    <0x500 &apps_smmu 0x140c 0x1>,
+		    <0x501 &apps_smmu 0x140e 0x1>;
+
+	/delete-property/ msi-map;
+
+	status = "okay";
+};
+
+&pcie0_phy {
+	vdda-phy-supply = <&vreg_l1e_0p88>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&pcie0_port0 {
+	pcie@0,0 {
+		compatible = "pci1179,0623";
+		reg = <0x10000 0x0 0x0 0x0 0x0>;
+		#address-cells = <3>;
+		#size-cells = <2>;
+
+		device_type = "pci";
+		ranges;
+		bus-range = <0x2 0xff>;
+
+		vddc-supply = <&vdd_ntn_0p9>;
+		vdd18-supply = <&vdd_ntn_1p8>;
+		vdd09-supply = <&vdd_ntn_0p9>;
+		vddio1-supply = <&vdd_ntn_1p8>;
+		vddio2-supply = <&vdd_ntn_1p8>;
+		vddio18-supply = <&vdd_ntn_1p8>;
+
+		i2c-parent = <&i2c_hub_4 0x77>;
+
+		resx-gpios = <&tlmm 64 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&tc9563_0_rst>, <&upd_ponrst>;
+		pinctrl-names = "default";
+
+		pcie@1,0 {
+			reg = <0x20800 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			device_type = "pci";
+			ranges;
+			bus-range = <0x3 0xff>;
+		};
+
+		pcie@2,0 {
+			reg = <0x21000 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			device_type = "pci";
+			ranges;
+			bus-range = <0x4 0xff>;
+		};
+
+		pcie@3,0 {
+			reg = <0x21800 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+			device_type = "pci";
+			ranges;
+			bus-range = <0x5 0xff>;
+
+			pci@0,0 {
+				reg = <0x50000 0x0 0x0 0x0 0x0>;
+				#address-cells = <3>;
+				#size-cells = <2>;
+				device_type = "pci";
+				ranges;
+			};
+
+			pci@0,1 {
+				reg = <0x50100 0x0 0x0 0x0 0x0>;
+				#address-cells = <3>;
+				#size-cells = <2>;
+				device_type = "pci";
+				ranges;
+			};
+		};
+	};
+};
+
+&pcie1 {
+	wake-gpios = <&tlmm 99 GPIO_ACTIVE_HIGH>;
+	perst-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
+
+	pinctrl-0 = <&pcie1_default_state>;
+	pinctrl-names = "default";
+
+	iommu-map = <0x0 &apps_smmu 0x1480 0x1>,
+		    <0x100 &apps_smmu 0x1481 0x1>,
+		    <0x208 &apps_smmu 0x1482 0x1>,
+		    <0x210 &apps_smmu 0x1483 0x1>,
+		    <0x218 &apps_smmu 0x1484 0x1>,
+		    <0x300 &apps_smmu 0x1487 0x1>,
+		    <0x400 &apps_smmu 0x1488 0x1>,
+		    <0x500 &apps_smmu 0x148c 0x1>,
+		    <0x501 &apps_smmu 0x148e 0x1>;
+
+	/delete-property/ msi-map;
+
+	status = "okay";
+};
+
+&pcie1_phy {
+	vdda-phy-supply = <&vreg_l3c_0p9>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+	vdda-qref-supply = <&vreg_l1e_0p88>;
+
+	status = "okay";
+};
+
+&pcie1_port0 {
+	pcie@0,0 {
+		compatible = "pci1179,0623";
+		reg = <0x10000 0x0 0x0 0x0 0x0>;
+		#address-cells = <3>;
+		#size-cells = <2>;
+
+		device_type = "pci";
+		ranges;
+		bus-range = <0x2 0xff>;
+
+		vddc-supply = <&vdd_ntn1_0p9>;
+		vdd18-supply = <&vdd_ntn1_1p8>;
+		vdd09-supply = <&vdd_ntn1_0p9>;
+		vddio1-supply = <&vdd_ntn1_1p8>;
+		vddio2-supply = <&vdd_ntn1_1p8>;
+		vddio18-supply = <&vdd_ntn1_1p8>;
+
+		i2c-parent = <&i2c_hub_3_gpio 0x77>;
+
+		resx-gpios = <&tlmm 65 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&tc9563_1_rst>;
+		pinctrl-names = "default";
+
+		pcie@1,0 {
+			reg = <0x20800 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			device_type = "pci";
+			ranges;
+			bus-range = <0x3 0xff>;
+		};
+
+		pcie@2,0 {
+			reg = <0x21000 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			device_type = "pci";
+			ranges;
+			bus-range = <0x4 0xff>;
+
+			wifi@0 {
+				compatible = "pci17cb,1107";
+				reg = <0x40000 0x0 0x0 0x0 0x0>;
+
+				vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+				vddaon-supply = <&vreg_pmu_aon_0p59>;
+				vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+				vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+				vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+				vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+				vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+				vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>;
+				vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>;
+			};
+		};
+
+		pcie@3,0 {
+			reg = <0x21800 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+			device_type = "pci";
+			ranges;
+			bus-range = <0x5 0xff>;
+		};
+	};
+};
+
+&pm8550_gpios {
+	volume_up_n: volume-up-n-state {
+		pins = "gpio6";
+		function = "normal";
+		power-source = <1>;
+		bias-pull-up;
+		input-enable;
+	};
+
+	sdc2_card_det_n: sdc2-card-det-state {
+		pins = "gpio12";
+		function = "normal";
+		power-source = <1>;
+		bias-pull-up;
+		input-enable;
+	};
+};
+
+&pm8550_pwm {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	status = "okay";
+
+	led@1 {
+		reg = <1>;
+		color = <LED_COLOR_ID_GREEN>;
+		function = LED_FUNCTION_STATUS;
+		function-enumerator = <0>;
+		linux,default-trigger = "none";
+		default-state = "off";
+	};
+
+	led@2 {
+		reg = <2>;
+		color = <LED_COLOR_ID_GREEN>;
+		function = LED_FUNCTION_STATUS;
+		function-enumerator = <1>;
+		linux,default-trigger = "none";
+		default-state = "off";
+	};
+
+	led@3 {
+		reg = <3>;
+		color = <LED_COLOR_ID_GREEN>;
+		function = LED_FUNCTION_STATUS;
+		function-enumerator = <2>;
+		linux,default-trigger = "none";
+		default-state = "off";
+	};
+};
+
+&pm8550b_eusb2_repeater {
+	vdd18-supply = <&vreg_l15b_1p8>;
+	vdd3-supply = <&vreg_l5b_3p1>;
+};
+
+&pmk8550_gpios {
+	pmk8550_sleep_clk: sleep-clk-state {
+		pins = "gpio3";
+		function = "func1";
+		input-disable;
+		output-enable;
+		bias-disable;
+		power-source = <0>;
+	};
+};
+
+&pon_pwrkey {
+	status = "okay";
+};
+
+&pon_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+
+	status = "okay";
+};
+
+&qupv3_id_0 {
+	status = "okay";
+};
+
+&qupv3_id_1 {
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/sm8550/adsp.mbn",
+			"qcom/sm8550/adsp_dtb.mbn";
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/sm8550/cdsp.mbn",
+			"qcom/sm8550/cdsp_dtb.mbn";
+	status = "okay";
+};
+
+&remoteproc_mpss {
+	firmware-name = "qcom/sm8550/modem.mbn",
+			"qcom/sm8550/modem_dtb.mbn";
+	status = "okay";
+};
+
+&sdhc_2 {
+	cd-gpios = <&pm8550_gpios 12 GPIO_ACTIVE_LOW>;
+
+	pinctrl-0 = <&sdc2_default>, <&sdc2_card_det_n>;
+	pinctrl-1 = <&sdc2_sleep>, <&sdc2_card_det_n>;
+	pinctrl-names = "default", "sleep";
+
+	vmmc-supply = <&vreg_l9b_2p9>;
+	vqmmc-supply = <&vreg_l8b_1p8>;
+
+	no-sdio;
+	no-mmc;
+
+	status = "okay";
+};
+
+&sleep_clk {
+	clock-frequency = <32764>;
+};
+
+&spi11 {
+	status = "okay";
+
+	can@0 {
+		compatible = "microchip,mcp2518fd";
+		reg = <0>;
+		interrupts-extended = <&tlmm 55 IRQ_TYPE_LEVEL_LOW>;
+		clocks = <&clk40m>;
+		spi-max-frequency = <10000000>;
+		vdd-supply = <&vreg_l14b_3p2>;
+		xceiver-supply = <&vreg_l14b_3p2>;
+	};
+};
+
+&swr0 {
+	status = "okay";
+
+	left_spkr: speaker@0,0 {
+		compatible = "sdw20217020400";
+		reg = <0 0>;
+
+		reset-gpios = <&tlmm 133 GPIO_ACTIVE_LOW>;
+
+		vdd-1p8-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l15b_1p8>;
+
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "SpkrLeft";
+		qcom,port-mapping = <1 2 3 7 10 13>;
+	};
+
+	right_spkr: speaker@0,1 {
+		compatible = "sdw20217020400";
+		reg = <0 1>;
+
+		reset-gpios = <&tlmm 133 GPIO_ACTIVE_LOW>;
+
+		vdd-1p8-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l15b_1p8>;
+
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "SpkrRight";
+		qcom,port-mapping = <4 5 6 7 11 13>;
+	};
+};
+
+&tlmm {
+	/* 32-35 for NFC secure element */
+	/* 36-39 for fingerprint sensor */
+	gpio-reserved-ranges = <32 8>;
+
+	lt9611_rst_pin: lt9611-rst-state {
+		pins = "gpio7";
+		function = "gpio";
+	};
+
+	upd_3p3_en: upd-3p3-en-state {
+		pins = "gpio13";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	lt9611_irq_pin: lt9611-irq-state {
+		pins = "gpio40";
+		function = "gpio";
+		bias-disable;
+	};
+
+	ntn1_en: ntn1-en-state {
+		pins = "gpio42";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	tc9563_0_rst: tc9563-0-rst-state {
+		pins = "gpio64";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	tc9563_1_rst: tc9563-1-rst-state {
+		pins = "gpio65";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	ntn0_en: ntn0-en-state {
+		pins = "gpio67";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	wlan_en: wlan-en-state {
+		pins = "gpio80";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-down;
+	};
+
+	bt_default: bt-default-state {
+		pins = "gpio81";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	sw_ctrl_default: sw-ctrl-default-state {
+		pins = "gpio82";
+		function = "gpio";
+		bias-pull-down;
+	};
+
+	upd_1p05_en: upd-1p05-en-state {
+		pins = "gpio179";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	upd_ponrst: upd-ponrst-state {
+		pins = "gpio182";
+		function = "gpio";
+		drive-strength = <2>;
+		output-high;
+	};
+};
+
+&uart7 {
+	status = "okay";
+};
+
+&uart14 {
+	status = "okay";
+
+	bluetooth {
+		compatible = "qcom,wcn7850-bt";
+
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+	};
+};
+
+&ufs_mem_hc {
+	reset-gpios = <&tlmm 210 GPIO_ACTIVE_LOW>;
+	vcc-supply = <&vreg_l17b_2p5>;
+	vcc-max-microamp = <1300000>;
+	vccq-supply = <&vreg_l1g_1p2>;
+	vccq-max-microamp = <1200000>;
+	vdd-hba-supply = <&vreg_l3g_1p2>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vreg_l1d_0p88>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&usb_1 {
+	status = "okay";
+};
+
+&usb_1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in>;
+};
+
+&usb_1_hsphy {
+	vdd-supply = <&vreg_l1e_0p88>;
+	vdda12-supply = <&vreg_l3e_1p2>;
+
+	phys = <&pm8550b_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_dp_qmpphy {
+	vdda-phy-supply = <&vreg_l3e_1p2>;
+	vdda-pll-supply = <&vreg_l3f_0p88>;
+
+	status = "okay";
+};
+
+&usb_dp_qmpphy_out {
+	remote-endpoint = <&redriver_phy_con_ss>;
+};
+
+&xo_board {
+	clock-frequency = <76800000>;
+};
diff --git a/arch/arm64/boot/dts/qcom/qdu1000.dtsi b/arch/arm64/boot/dts/qcom/qdu1000.dtsi
index 952d4270d1181..26cc86d12f725 100644
--- a/arch/arm64/boot/dts/qcom/qdu1000.dtsi
+++ b/arch/arm64/boot/dts/qcom/qdu1000.dtsi
@@ -1054,7 +1054,7 @@ usb_1_dwc3_ss: endpoint {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,qdu1000-pdc", "qcom,pdc";
-			reg = <0x0 0xb220000 0x0 0x30000>, <0x0 0x174000f0 0x0 0x64>;
+			reg = <0x0 0xb220000 0x0 0x10000>, <0x0 0x174000f0 0x0 0x64>;
 			qcom,pdc-ranges = <0 480 12>, <14 494 24>, <40 520 54>,
 					  <94 609 31>, <125 63 1>;
 			#interrupt-cells = <2>;
diff --git a/arch/arm64/boot/dts/qcom/qrb2210-arduino-imola.dts b/arch/arm64/boot/dts/qcom/qrb2210-arduino-imola.dts
index bf088fa9807f0..c472e13fceca4 100644
--- a/arch/arm64/boot/dts/qcom/qrb2210-arduino-imola.dts
+++ b/arch/arm64/boot/dts/qcom/qrb2210-arduino-imola.dts
@@ -235,8 +235,8 @@ &mdss_dsi0_phy {
 };
 
 &pm4125_vbus {
-	regulator-min-microamp = <500000>;
-	regulator-max-microamp = <500000>;
+	regulator-min-microvolt = <5000000>;
+	regulator-max-microvolt = <5000000>;
 
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/qrb2210-rb1-vision-mezzanine.dtso b/arch/arm64/boot/dts/qcom/qrb2210-rb1-vision-mezzanine.dtso
index c314cd6dd484a..f5e6021a3354c 100644
--- a/arch/arm64/boot/dts/qcom/qrb2210-rb1-vision-mezzanine.dtso
+++ b/arch/arm64/boot/dts/qcom/qrb2210-rb1-vision-mezzanine.dtso
@@ -60,7 +60,7 @@ ov9282_ep: endpoint {
 				link-frequencies = /bits/ 64 <400000000>;
 				data-lanes = <1 2>;
 				remote-endpoint = <&csiphy0_ep>;
-                        };
-                };
+			};
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/qcom/qrb2210-rb1.dts b/arch/arm64/boot/dts/qcom/qrb2210-rb1.dts
index 4ace2d6c06cee..bf6fb12ad9906 100644
--- a/arch/arm64/boot/dts/qcom/qrb2210-rb1.dts
+++ b/arch/arm64/boot/dts/qcom/qrb2210-rb1.dts
@@ -433,8 +433,8 @@ pm4125_ss_in: endpoint {
 };
 
 &pm4125_vbus {
-	regulator-min-microamp = <500000>;
-	regulator-max-microamp = <500000>;
+	regulator-min-microvolt = <5000000>;
+	regulator-max-microvolt = <5000000>;
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts b/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts
index 22baee407fbee..9b05f6d843cfb 100644
--- a/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts
+++ b/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts
@@ -741,6 +741,12 @@ &usb_qmpphy_out {
 	remote-endpoint = <&pmi632_ss_in>;
 };
 
+&venus {
+	firmware-name = "qcom/venus-6.0/venus.mbn";
+
+	status = "okay";
+};
+
 &wifi {
 	/* SoC */
 	vdd-0.8-cx-mx-supply = <&vreg_l8a_0p664>;
diff --git a/arch/arm64/boot/dts/qcom/sa8295p-adp.dts b/arch/arm64/boot/dts/qcom/sa8295p-adp.dts
index d28d691624279..512de3597581b 100644
--- a/arch/arm64/boot/dts/qcom/sa8295p-adp.dts
+++ b/arch/arm64/boot/dts/qcom/sa8295p-adp.dts
@@ -453,15 +453,17 @@ &mdss1_dp3_phy {
 };
 
 &pcie2a {
-	perst-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>;
-
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie2a_default>;
 
 	status = "okay";
 };
 
+&pcie2a_port0 {
+	reset-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>;
+};
+
 &pcie2a_phy {
 	vdda-phy-supply = <&vreg_l11a>;
 	vdda-pll-supply = <&vreg_l3a>;
@@ -472,15 +474,17 @@ &pcie2a_phy {
 &pcie3a {
 	num-lanes = <2>;
 
-	perst-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 56 GPIO_ACTIVE_LOW>;
-
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie3a_default>;
 
 	status = "okay";
 };
 
+&pcie3a_port0 {
+	reset-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 56 GPIO_ACTIVE_LOW>;
+};
+
 &pcie3a_phy {
 	vdda-phy-supply = <&vreg_l11a>;
 	vdda-pll-supply = <&vreg_l3a>;
@@ -489,15 +493,17 @@ &pcie3a_phy {
 };
 
 &pcie3b {
-	perst-gpios = <&tlmm 153 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 130 GPIO_ACTIVE_LOW>;
-
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie3b_default>;
 
 	status = "okay";
 };
 
+&pcie3b_port0 {
+	reset-gpios = <&tlmm 153 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 130 GPIO_ACTIVE_LOW>;
+};
+
 &pcie3b_phy {
 	vdda-phy-supply = <&vreg_l11a>;
 	vdda-pll-supply = <&vreg_l3a>;
@@ -506,15 +512,17 @@ &pcie3b_phy {
 };
 
 &pcie4 {
-	perst-gpios = <&tlmm 141 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 139 GPIO_ACTIVE_LOW>;
-
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie4_default>;
 
 	status = "okay";
 };
 
+&pcie4_port0 {
+	reset-gpios = <&tlmm 141 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 139 GPIO_ACTIVE_LOW>;
+};
+
 &pcie4_phy {
 	vdda-phy-supply = <&vreg_l11a>;
 	vdda-pll-supply = <&vreg_l3a>;
diff --git a/arch/arm64/boot/dts/qcom/sa8540p-ride.dts b/arch/arm64/boot/dts/qcom/sa8540p-ride.dts
index 44177e9b64b52..83d5f1d9e79ba 100644
--- a/arch/arm64/boot/dts/qcom/sa8540p-ride.dts
+++ b/arch/arm64/boot/dts/qcom/sa8540p-ride.dts
@@ -366,15 +366,17 @@ &pcie2a {
 		 <0x02000000 0x0 0x3c300000 0x0 0x3c300000 0x0 0x1d00000>,
 		 <0x03000000 0x5 0x00000000 0x5 0x00000000 0x1 0x00000000>;
 
-	perst-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 145 GPIO_ACTIVE_HIGH>;
-
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie2a_default>;
 
 	status = "disabled";
 };
 
+&pcie2a_port0 {
+	reset-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 145 GPIO_ACTIVE_HIGH>;
+};
+
 &pcie2a_phy {
 	vdda-phy-supply = <&vreg_l11a>;
 	vdda-pll-supply = <&vreg_l3a>;
@@ -387,15 +389,17 @@ &pcie3a {
 		 <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x20000000>,
 		 <0x03000000 0x6 0x00000000 0x6 0x00000000 0x2 0x00000000>;
 
-	perst-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 56 GPIO_ACTIVE_HIGH>;
-
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie3a_default>;
 
 	status = "okay";
 };
 
+&pcie3a_port0 {
+	reset-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 56 GPIO_ACTIVE_HIGH>;
+};
+
 &pcie3a_phy {
 	vdda-phy-supply = <&vreg_l11a>;
 	vdda-pll-supply = <&vreg_l3a>;
diff --git a/arch/arm64/boot/dts/qcom/sa8775p-ride-r3.dts b/arch/arm64/boot/dts/qcom/sa8775p-ride-r3.dts
deleted file mode 100644
index b25f0b2c94104..0000000000000
--- a/arch/arm64/boot/dts/qcom/sa8775p-ride-r3.dts
+++ /dev/null
@@ -1,17 +0,0 @@
-// SPDX-License-Identifier: BSD-3-Clause
-/*
- * Copyright (c) 2023, Linaro Limited
- */
-
-/dts-v1/;
-
-#include "lemans-auto.dtsi"
-
-#include "lemans-pmics.dtsi"
-#include "lemans-ride-common.dtsi"
-#include "lemans-ride-ethernet-aqr115c.dtsi"
-
-/ {
-	model = "Qualcomm SA8775P Ride Rev3";
-	compatible = "qcom,sa8775p-ride-r3", "qcom,sa8775p";
-};
diff --git a/arch/arm64/boot/dts/qcom/sa8775p-ride.dts b/arch/arm64/boot/dts/qcom/sa8775p-ride.dts
deleted file mode 100644
index 2d9028cd60be4..0000000000000
--- a/arch/arm64/boot/dts/qcom/sa8775p-ride.dts
+++ /dev/null
@@ -1,17 +0,0 @@
-// SPDX-License-Identifier: BSD-3-Clause
-/*
- * Copyright (c) 2023, Linaro Limited
- */
-
-/dts-v1/;
-
-#include "lemans-auto.dtsi"
-
-#include "lemans-pmics.dtsi"
-#include "lemans-ride-common.dtsi"
-#include "lemans-ride-ethernet-88ea1512.dtsi"
-
-/ {
-	model = "Qualcomm SA8775P Ride";
-	compatible = "qcom,sa8775p-ride", "qcom,sa8775p";
-};
diff --git a/arch/arm64/boot/dts/qcom/sar2130p.dtsi b/arch/arm64/boot/dts/qcom/sar2130p.dtsi
index 3c9529bb2f76f..767f7cc218f16 100644
--- a/arch/arm64/boot/dts/qcom/sar2130p.dtsi
+++ b/arch/arm64/boot/dts/qcom/sar2130p.dtsi
@@ -2417,7 +2417,7 @@ dispcc: clock-controller@af00000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sar2130p-pdc", "qcom,pdc";
-			reg = <0x0 0x0b220000 0x0 0x30000>, <0x0 0x174000f0 0x0 0x64>;
+			reg = <0x0 0x0b220000 0x0 0x10000>, <0x0 0x174000f0 0x0 0x64>;
 			qcom,pdc-ranges = <0 480 94>,
 					  <94 609 31>,
 					  <125 63 1>,
diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi b/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi
index b398f69917f0e..cd4a0e281cf8f 100644
--- a/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi
@@ -99,6 +99,11 @@ chosen {
 		stdout-path = "serial0:115200n8";
 	};
 
+	firmware {
+		#address-cells = <2>;
+		#size-cells = <2>;
+	};
+
 	/* FIXED REGULATORS - parents above children */
 
 	/* This is the top level supply and variable voltage */
diff --git a/arch/arm64/boot/dts/qcom/sc7180.dtsi b/arch/arm64/boot/dts/qcom/sc7180.dtsi
index fa65c485172f9..18f9cb82639a6 100644
--- a/arch/arm64/boot/dts/qcom/sc7180.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7180.dtsi
@@ -3514,7 +3514,7 @@ dispcc: clock-controller@af00000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sc7180-pdc", "qcom,pdc";
-			reg = <0 0x0b220000 0 0x30000>;
+			reg = <0 0x0b220000 0 0x10000>;
 			qcom,pdc-ranges = <0 480 94>, <94 609 31>, <125 63 1>;
 			#interrupt-cells = <2>;
 			interrupt-parent = <&intc>;
diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-lte-sku.dtsi b/arch/arm64/boot/dts/qcom/sc7280-herobrine-lte-sku.dtsi
index b721a8546800c..ac89724ee0993 100644
--- a/arch/arm64/boot/dts/qcom/sc7280-herobrine-lte-sku.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-lte-sku.dtsi
@@ -22,7 +22,9 @@ mdata_mem: mpss-metadata {
 };
 
 &ipa {
+	/delete-property/ memory-region;
 	qcom,gsi-loader = "modem";
+
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi b/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi
index 5c5e4f1dd2217..58ea0532c0fbb 100644
--- a/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi
@@ -25,6 +25,11 @@ chosen {
 		stdout-path = "serial0:115200n8";
 	};
 
+	firmware {
+		#address-cells = <2>;
+		#size-cells = <2>;
+	};
+
 	/*
 	 * FIXED REGULATORS
 	 *
diff --git a/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi b/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi
index 8cac4ce9c8515..6bb806931b792 100644
--- a/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi
@@ -418,7 +418,6 @@ &lpass_va_macro {
 
 &pcie1 {
 	status = "okay";
-	perst-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
 
 	vddpe-3v3-supply = <&nvme_3v3_regulator>;
 
@@ -433,6 +432,10 @@ &pcie1_phy {
 	vdda-pll-supply = <&vreg_l6b_1p2>;
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
+};
+
 &pmk8350_vadc {
 	channel@3 {
 		reg = <PMK8350_ADC7_DIE_TEMP>;
diff --git a/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts b/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts
index d86a31ddede29..7601afc3d7bf9 100644
--- a/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts
+++ b/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts
@@ -26,6 +26,7 @@ backlight: backlight {
 		compatible = "pwm-backlight";
 		pwms = <&pmc8180c_lpg 4 1000000>;
 		enable-gpios = <&pmc8180c_gpios 8 GPIO_ACTIVE_HIGH>;
+		power-supply = <&vled_bl_pw>;
 
 		pinctrl-0 = <&bl_pwm_default>;
 		pinctrl-names = "default";
@@ -157,14 +158,39 @@ cdsp_mem: cdsp-region@98900000 {
 		};
 	};
 
-	vph_pwr: vph-pwr-regulator {
+	vled_bl_pw: regulator-vled-bl-pw {
 		compatible = "regulator-fixed";
-		regulator-name = "vph_pwr";
-		regulator-min-microvolt = <3700000>;
-		regulator-max-microvolt = <3700000>;
+
+		regulator-name = "VLED_BL_PW";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pmc8180_2_gpios 1 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&bl_pwr_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
 	};
 
-	vreg_s4a_1p8: pm8150-s4-regulator {
+	vreg_lcm_3v3: regulator-edp-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_LCM_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 130 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&lcm_3v3_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_s4a_1p8: regulator-pm8150-s4 {
 		compatible = "regulator-fixed";
 		regulator-name = "vreg_s4a_1p8";
 
@@ -177,6 +203,13 @@ vreg_s4a_1p8: pm8150-s4-regulator {
 		vin-supply = <&vph_pwr>;
 	};
 
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+	};
+
 	usbprim-sbu-mux {
 		compatible = "pericom,pi3usb102", "gpio-sbu-mux";
 
@@ -438,6 +471,7 @@ &mdss_edp {
 	aux-bus {
 		panel {
 			compatible = "edp-panel";
+			power-supply = <&vreg_lcm_3v3>;
 			no-hpd;
 
 			backlight = <&backlight>;
@@ -472,6 +506,13 @@ &pcie3_phy {
 	status = "okay";
 };
 
+&pmc8180_2_gpios {
+	bl_pwr_en: bl-pwr-en-state {
+		pins = "gpio1";
+		function = "normal";
+	};
+};
+
 &pmc8180_pwrkey {
 	status = "okay";
 };
@@ -765,6 +806,12 @@ ts_int_default: ts-int-default-state {
 		drive-strength = <2>;
 	};
 
+	lcm_3v3_en: lcm-3v3-en-state {
+		pins = "gpio130";
+		function = "gpio";
+		bias-disable;
+	};
+
 	usbprim_sbu_default: usbprim-sbu-state {
 		oe-n-pins {
 			pins = "gpio152";
diff --git a/arch/arm64/boot/dts/qcom/sc8180x-primus.dts b/arch/arm64/boot/dts/qcom/sc8180x-primus.dts
index aff398390eba7..e34f4758ebe28 100644
--- a/arch/arm64/boot/dts/qcom/sc8180x-primus.dts
+++ b/arch/arm64/boot/dts/qcom/sc8180x-primus.dts
@@ -29,9 +29,10 @@ backlight: backlight {
 		compatible = "pwm-backlight";
 		pwms = <&pmc8180c_lpg 4 1000000>;
 		enable-gpios = <&pmc8180c_gpios 8 GPIO_ACTIVE_HIGH>;
+		power-supply = <&vled_bl_pw>;
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&bl_pwm_default>;
+		pinctrl-names = "default";
 	};
 
 	chosen {
@@ -167,7 +168,39 @@ reserved-region@9a500000 {
 		};
 	};
 
-	vreg_nvme_0p9: nvme-0p9-regulator {
+	vled_bl_pw: regulator-vled-bl-pw {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VLED_BL_PW";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pmc8180_2_gpios 1 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&bl_pwr_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_lcm_3v3: regulator-edp-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_LCM_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 130 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&lcm_3v3_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_nvme_0p9: regulator-nvme-0p9 {
 		compatible = "regulator-fixed";
 		regulator-name = "vreg_nvme_0p9";
 
@@ -177,7 +210,7 @@ vreg_nvme_0p9: nvme-0p9-regulator {
 		regulator-always-on;
 	};
 
-	vreg_nvme_3p3: nvme-3p3-regulator {
+	vreg_nvme_3p3: regulator-nvme-3p3 {
 		compatible = "regulator-fixed";
 		regulator-name = "vreg_nvme_3p3";
 
@@ -190,7 +223,7 @@ vreg_nvme_3p3: nvme-3p3-regulator {
 		regulator-always-on;
 	};
 
-	vdd_kb_tp_3v3: vdd-kb-tp-3v3-regulator {
+	vdd_kb_tp_3v3: regulator-vdd-kb-tp-3v3 {
 		compatible = "regulator-fixed";
 		regulator-name = "vdd_kb_tp_3v3";
 		regulator-min-microvolt = <3300000>;
@@ -205,7 +238,7 @@ vdd_kb_tp_3v3: vdd-kb-tp-3v3-regulator {
 		pinctrl-0 = <&kb_tp_3v3_en_active_state>;
 	};
 
-	vph_pwr: vph-pwr-regulator {
+	vph_pwr: regulator-vph-pwr {
 		compatible = "regulator-fixed";
 		regulator-name = "vph_pwr";
 		regulator-min-microvolt = <3700000>;
@@ -540,6 +573,7 @@ &mdss_edp {
 	aux-bus {
 		panel {
 			compatible = "edp-panel";
+			power-supply = <&vreg_lcm_3v3>;
 
 			backlight = <&backlight>;
 
@@ -769,6 +803,12 @@ &wifi {
 };
 
 /* PINCTRL */
+&pmc8180_2_gpios {
+	bl_pwr_en: bl-pwr-en-state {
+		pins = "gpio1";
+		function = "normal";
+	};
+};
 
 &pmc8180c_gpios {
 	bl_pwm_default: bl-pwm-default-state {
@@ -950,4 +990,10 @@ rx-pins {
 			bias-pull-up;
 		};
 	};
+
+	lcm_3v3_en: lcm-3v3-en-state {
+		pins = "gpio130";
+		function = "gpio";
+		bias-disable;
+	};
 };
diff --git a/arch/arm64/boot/dts/qcom/sc8180x.dtsi b/arch/arm64/boot/dts/qcom/sc8180x.dtsi
index 45391768e2458..b532e33a26397 100644
--- a/arch/arm64/boot/dts/qcom/sc8180x.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc8180x.dtsi
@@ -3554,7 +3554,7 @@ dispcc: clock-controller@af00000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sc8180x-pdc", "qcom,pdc";
-			reg = <0 0x0b220000 0 0x30000>;
+			reg = <0 0x0b220000 0 0x10000>;
 			qcom,pdc-ranges = <0 480 94>, <94 609 31>;
 			#interrupt-cells = <2>;
 			interrupt-parent = <&intc>;
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts
index dcdeefd287283..67f0576f2824d 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts
@@ -628,9 +628,6 @@ keyboard@68 {
 };
 
 &pcie2a {
-	perst-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>;
-
 	vddpe-3v3-supply = <&vreg_nvme>;
 
 	pinctrl-names = "default";
@@ -646,10 +643,12 @@ &pcie2a_phy {
 	status = "okay";
 };
 
-&pcie3a {
-	perst-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+&pcie2a_port0 {
+	reset-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>;
+};
 
+&pcie3a {
 	vddpe-3v3-supply = <&vreg_wwan>;
 
 	pinctrl-names = "default";
@@ -665,12 +664,14 @@ &pcie3a_phy {
 	status = "okay";
 };
 
+&pcie3a_port0 {
+	reset-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+};
+
 &pcie4 {
 	max-link-speed = <2>;
 
-	perst-gpios = <&tlmm 141 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 139 GPIO_ACTIVE_LOW>;
-
 	vddpe-3v3-supply = <&vreg_wlan>;
 
 	pinctrl-names = "default";
@@ -687,6 +688,9 @@ &pcie4_phy {
 };
 
 &pcie4_port0 {
+	reset-gpios = <&tlmm 141 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 139 GPIO_ACTIVE_LOW>;
+
 	wifi@0 {
 		compatible = "pci17cb,1103";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-huawei-gaokun3.dts b/arch/arm64/boot/dts/qcom/sc8280xp-huawei-gaokun3.dts
index f3c00be67081a..adce0302d92e3 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-huawei-gaokun3.dts
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-huawei-gaokun3.dts
@@ -637,7 +637,7 @@ embedded-controller@38 {
 		compatible = "huawei,gaokun3-ec";
 		reg = <0x38>;
 
-		interrupts-extended = <&tlmm 107 IRQ_TYPE_LEVEL_LOW>;
+		interrupts-extended = <&tlmm 103 IRQ_TYPE_LEVEL_LOW>;
 
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -739,9 +739,6 @@ &mdss0_dp1_out {
 };
 
 &pcie2a {
-	perst-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>;
-
 	vddpe-3v3-supply = <&vreg_nvme>;
 
 	pinctrl-0 = <&pcie2a_default>;
@@ -757,12 +754,14 @@ &pcie2a_phy {
 	status = "okay";
 };
 
+&pcie2a_port0 {
+	reset-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>;
+};
+
 &pcie4 {
 	max-link-speed = <2>;
 
-	perst-gpios = <&tlmm 141 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 139 GPIO_ACTIVE_LOW>;
-
 	vddpe-3v3-supply = <&vreg_wlan>;
 
 	pinctrl-0 = <&pcie4_default>;
@@ -772,6 +771,9 @@ &pcie4 {
 };
 
 &pcie4_port0 {
+	reset-gpios = <&tlmm 141 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 139 GPIO_ACTIVE_LOW>;
+
 	wifi@0 {
 		compatible = "pci17cb,1103";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts
index abd9c5a67b9ff..218573a97785b 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts
@@ -674,8 +674,9 @@ camera@10 {
 		reg = <0x10>;
 
 		reset-gpios = <&tlmm 15 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&cam_rgb_default>, <&cam_mclk3_default>;
 		pinctrl-names = "default";
-		pinctrl-0 = <&cam_rgb_default>;
 
 		leds = <&privacy_led>;
 		led-names = "privacy";
@@ -932,10 +933,13 @@ keyboard@68 {
 	};
 };
 
-&pcie2a {
-	perst-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>;
+&iris {
+	firmware-name = "qcom/sc8280xp/LENOVO/21BX/qcvss8280.mbn";
+
+	status = "okay";
+};
 
+&pcie2a {
 	vddpe-3v3-supply = <&vreg_nvme>;
 
 	pinctrl-names = "default";
@@ -951,10 +955,12 @@ &pcie2a_phy {
 	status = "okay";
 };
 
-&pcie3a {
-	perst-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+&pcie2a_port0 {
+	reset-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>;
+};
 
+&pcie3a {
 	vddpe-3v3-supply = <&vreg_wwan>;
 
 	pinctrl-names = "default";
@@ -970,12 +976,14 @@ &pcie3a_phy {
 	status = "okay";
 };
 
+&pcie3a_port0 {
+	reset-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+};
+
 &pcie4 {
 	max-link-speed = <2>;
 
-	perst-gpios = <&tlmm 141 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 139 GPIO_ACTIVE_LOW>;
-
 	vddpe-3v3-supply = <&vreg_wlan>;
 
 	pinctrl-names = "default";
@@ -985,6 +993,9 @@ &pcie4 {
 };
 
 &pcie4_port0 {
+	reset-gpios = <&tlmm 141 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 139 GPIO_ACTIVE_LOW>;
+
 	wifi@0 {
 		compatible = "pci17cb,1103";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
@@ -1552,13 +1563,6 @@ cam_indicator_en: cam-indicator-en-state {
 	};
 
 	cam_rgb_default: cam-rgb-default-state {
-		mclk-pins {
-			pins = "gpio17";
-			function = "cam_mclk";
-			drive-strength = <16>;
-			bias-disable;
-		};
-
 		sc-rgb-xshut-n-pins {
 			pins = "gpio15";
 			function = "gpio";
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts
index f2b4470d4407f..89c4ba267b078 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts
@@ -6,6 +6,8 @@
 /dts-v1/;
 
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/gpio-keys.h>
+#include <dt-bindings/input/input.h>
 #include <dt-bindings/regulator/qcom,rpmh-regulator.h>
 
 #include "sc8280xp.dtsi"
@@ -46,11 +48,56 @@ wcd938x: audio-codec {
 		#sound-dai-cells = <1>;
 	};
 
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&pmc8280c_lpg 3 1000000>;
+		enable-gpios = <&pmc8280_1_gpios 8 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&edp_bl_en>, <&edp_bl_pwm>;
+		pinctrl-names = "default";
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&hall_int_n_default>, <&vol_down_n>, <&vol_up_n>;
+		pinctrl-names = "default";
+
+		key-vol-down {
+			label = "Volume Down";
+			gpios = <&pmc8280_1_gpios 6 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEDOWN>;
+			debounce-interval = <15>;
+			linux,can-disable;
+			wakeup-source;
+		};
+
+		key-vol-up {
+			label = "Volume Up";
+			gpios = <&pmc8280_1_gpios 9 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
+			debounce-interval = <15>;
+			linux,can-disable;
+			wakeup-source;
+		};
+
+		switch-lid {
+			label = "lid";
+			gpios = <&tlmm 180 GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_SW>;
+			linux,code = <SW_LID>;
+			wakeup-source;
+			wakeup-event-action = <EV_ACT_DEASSERTED>;
+		};
+	};
+
 	pmic-glink {
 		compatible = "qcom,sc8280xp-pmic-glink", "qcom,pmic-glink";
 
 		#address-cells = <1>;
 		#size-cells = <0>;
+		orientation-gpios = <&tlmm 166 GPIO_ACTIVE_HIGH>,
+				    <&tlmm 49 GPIO_ACTIVE_HIGH>;
 
 		/* Left-side top port */
 		connector@0 {
@@ -127,6 +174,22 @@ pmic_glink_con1_sbu: endpoint {
 		};
 	};
 
+	vreg_edp_3p3: regulator-edp-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_EDP_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 36 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&edp_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
 	vreg_nvme: regulator-nvme {
 		compatible = "regulator-fixed";
 
@@ -243,7 +306,7 @@ map1 {
 	usb0-sbu-mux {
 		compatible = "pericom,pi3usb102", "gpio-sbu-mux";
 
-		enable-gpios = <&tlmm 101 GPIO_ACTIVE_LOW>;
+		enable-gpios = <&tlmm 100 GPIO_ACTIVE_LOW>;
 		select-gpios = <&tlmm 164 GPIO_ACTIVE_HIGH>;
 
 		pinctrl-0 = <&usb0_sbu_default>;
@@ -277,6 +340,70 @@ usb1_sbu_mux: endpoint {
 			};
 		};
 	};
+
+	wcn6855-pmu {
+		compatible = "qcom,wcn6855-pmu";
+
+		pinctrl-0 = <&bt_default>, <&wlan_en>;
+		pinctrl-names = "default";
+
+		wlan-enable-gpios = <&tlmm 134 GPIO_ACTIVE_HIGH>;
+		bt-enable-gpios = <&tlmm 133 GPIO_ACTIVE_HIGH>;
+		swctrl-gpios = <&tlmm 132 GPIO_ACTIVE_HIGH>;
+
+		vddio-supply = <&vreg_s10b>;
+		vddaon-supply = <&vreg_s12b>;
+		vddpmu-supply = <&vreg_s12b>;
+		vddpmumx-supply = <&vreg_s12b>;
+		vddpmucx-supply = <&vreg_s12b>;
+		vddrfa0p95-supply = <&vreg_s12b>;
+		vddrfa1p3-supply = <&vreg_s11b>;
+		vddrfa1p9-supply = <&vreg_s1c>;
+		vddpcie1p3-supply = <&vreg_s11b>;
+		vddpcie1p9-supply = <&vreg_s1c>;
+
+		regulators {
+			vreg_pmu_rfa_cmn_0p8: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn_0p8";
+			};
+
+			vreg_pmu_aon_0p8: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p8";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p8: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p8";
+			};
+
+			vreg_pmu_btcmx_0p8: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p8";
+			};
+
+			vreg_pmu_pcie_1p8: ldo5 {
+				regulator-name = "vreg_pmu_pcie_1p8";
+			};
+
+			vreg_pmu_pcie_0p9: ldo6 {
+				regulator-name = "vreg_pmu_pcie_0p9";
+			};
+
+			vreg_pmu_rfa_0p8: ldo7 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo8 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p7: ldo9 {
+				regulator-name = "vreg_pmu_rfa_1p7";
+			};
+		};
+	};
 };
 
 &apps_rsc {
@@ -303,7 +430,6 @@ vreg_s11b: smps11 {
 			regulator-min-microvolt = <1272000>;
 			regulator-max-microvolt = <1272000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-always-on;
 		};
 
 		vreg_s12b: smps12 {
@@ -311,7 +437,6 @@ vreg_s12b: smps12 {
 			regulator-min-microvolt = <984000>;
 			regulator-max-microvolt = <984000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-always-on;
 		};
 
 		vreg_l3b: ldo3 {
@@ -335,7 +460,6 @@ vreg_l6b: ldo6 {
 			regulator-max-microvolt = <880000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
 			regulator-boot-on;
-			regulator-always-on;	// FIXME: VDD_A_EDP_0_0P9
 		};
 	};
 
@@ -439,10 +563,6 @@ &dispcc0 {
 	status = "okay";
 };
 
-&dispcc1 {
-	status = "okay";
-};
-
 &gpi_dma0 {
 	status = "okay";
 };
@@ -472,7 +592,7 @@ &mdss0_dp0 {
 };
 
 &mdss0_dp0_out {
-	data-lanes = <0 1>;
+	data-lanes = <0 1 2 3>;
 	remote-endpoint = <&usb_0_qmpphy_dp_in>;
 };
 
@@ -481,14 +601,47 @@ &mdss0_dp1 {
 };
 
 &mdss0_dp1_out {
-	data-lanes = <0 1>;
+	data-lanes = <0 1 2 3>;
 	remote-endpoint = <&usb_1_qmpphy_dp_in>;
 };
 
-&pcie2a {
-	perst-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>;
+&mdss0_dp3 {
+	compatible = "qcom,sc8280xp-edp";
+	/delete-property/ #sound-dai-cells;
+
+	data-lanes = <0 1 2 3>;
+
+	status = "okay";
+
+	aux-bus {
+		panel {
+			compatible = "edp-panel";
+
+			backlight = <&backlight>;
+			power-supply = <&vreg_edp_3p3>;
 
+			port {
+				edp_panel_in: endpoint {
+					remote-endpoint = <&mdss0_dp3_out>;
+				};
+			};
+		};
+	};
+};
+
+&mdss0_dp3_out {
+	remote-endpoint = <&edp_panel_in>;
+};
+
+&mdss0_dp3_phy {
+	compatible = "qcom,sc8280xp-edp-phy";
+	vdda-phy-supply = <&vreg_l6b>;
+	vdda-pll-supply = <&vreg_l3b>;
+
+	status = "okay";
+};
+
+&pcie2a {
 	vddpe-3v3-supply = <&vreg_nvme>;
 
 	pinctrl-0 = <&pcie2a_default>;
@@ -504,10 +657,12 @@ &pcie2a_phy {
 	status = "okay";
 };
 
-&pcie3a {
-	perst-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+&pcie2a_port0 {
+	reset-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>;
+};
 
+&pcie3a {
 	vddpe-3v3-supply = <&vreg_wwan>;
 
 	pinctrl-0 = <&pcie3a_default>;
@@ -523,12 +678,14 @@ &pcie3a_phy {
 	status = "okay";
 };
 
+&pcie3a_port0 {
+	reset-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+};
+
 &pcie4 {
 	max-link-speed = <2>;
 
-	perst-gpios = <&tlmm 141 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 139 GPIO_ACTIVE_LOW>;
-
 	vddpe-3v3-supply = <&vreg_wlan>;
 
 	pinctrl-0 = <&pcie4_default>;
@@ -538,10 +695,23 @@ &pcie4 {
 };
 
 &pcie4_port0 {
+	reset-gpios = <&tlmm 141 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 139 GPIO_ACTIVE_LOW>;
+
 	wifi@0 {
 		compatible = "pci17cb,1103";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
 
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn_0p8>;
+		vddaon-supply = <&vreg_pmu_aon_0p8>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p8>;
+		vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>;
+		vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>;
+
 		qcom,calibration-variant = "MS_SP9_5G";
 	};
 };
@@ -553,6 +723,10 @@ &pcie4_phy {
 	status = "okay";
 };
 
+&pmc8280c_lpg {
+	status = "okay";
+};
+
 &pmk8280_pon_pwrkey {
 	status = "okay";
 };
@@ -562,18 +736,9 @@ &pmk8280_pon_resin {
 };
 
 &pmk8280_rtc {
-	nvmem-cells = <&rtc_offset>;
-	nvmem-cell-names = "offset";
-
-	status = "okay";
-};
+	qcom,uefi-rtc-info;
 
-&pmk8280_sdam_6 {
 	status = "okay";
-
-	rtc_offset: rtc-offset@bc {
-		reg = <0xbc 0x4>;
-	};
 };
 
 &qup0 {
@@ -615,9 +780,6 @@ &sound {
 			"VA DMIC0", "MIC BIAS1",
 			"VA DMIC1", "MIC BIAS1",
 			"VA DMIC2", "MIC BIAS3",
-			"VA DMIC0", "VA MIC BIAS1",
-			"VA DMIC1", "VA MIC BIAS1",
-			"VA DMIC2", "VA MIC BIAS3",
 			"TX SWR_ADC1", "ADC2_OUTPUT";
 
 	wcd-playback-dai-link {
@@ -722,20 +884,16 @@ &uart2 {
 	bluetooth {
 		compatible = "qcom,wcn6855-bt";
 
-		vddio-supply = <&vreg_s10b>;
-		vddbtcxmx-supply = <&vreg_s12b>;
-		vddrfacmn-supply = <&vreg_s12b>;
-		vddrfa0p8-supply = <&vreg_s12b>;
-		vddrfa1p2-supply = <&vreg_s11b>;
-		vddrfa1p7-supply = <&vreg_s1c>;
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn_0p8>;
+		vddaon-supply = <&vreg_pmu_aon_0p8>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p8>;
+		vddbtcmx-supply = <&vreg_pmu_btcmx_0p8>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>;
 
 		max-speed = <3200000>;
-
-		enable-gpios = <&tlmm 133 GPIO_ACTIVE_HIGH>;
-		swctrl-gpios = <&tlmm 132 GPIO_ACTIVE_HIGH>;
-
-		pinctrl-0 = <&bt_default>;
-		pinctrl-names = "default";
 	};
 };
 
@@ -778,6 +936,7 @@ &usb_0_qmpphy {
 	vdda-phy-supply = <&vreg_l9d>;
 	vdda-pll-supply = <&vreg_l4d>;
 
+	mode-switch;
 	orientation-switch;
 
 	status = "okay";
@@ -815,6 +974,7 @@ &usb_1_qmpphy {
 	vdda-phy-supply = <&vreg_l4b>;
 	vdda-pll-supply = <&vreg_l3b>;
 
+	mode-switch;
 	orientation-switch;
 
 	status = "okay";
@@ -853,6 +1013,29 @@ &lpass_tlmm {
 	status = "okay";
 };
 
+&pmc8280_1_gpios {
+	edp_bl_en: edp-bl-en-state {
+		pins = "gpio8";
+		function = "normal";
+	};
+
+	vol_down_n: vol-down-n-state {
+		pins = "gpio6";
+		function = "normal";
+		power-source = <1>;
+		input-enable;
+		bias-pull-up;
+	};
+
+	vol_up_n: vol-up-n-state {
+		pins = "gpio9";
+		function = "normal";
+		power-source = <1>;
+		input-enable;
+		bias-pull-up;
+	};
+};
+
 &pmc8280_2_gpios {
 	wwan_sw_en: wwan-sw-en-state {
 		pins = "gpio1";
@@ -860,6 +1043,13 @@ wwan_sw_en: wwan-sw-en-state {
 	};
 };
 
+&pmc8280c_gpios {
+	edp_bl_pwm: edp-bl-pwm-state {
+		pins = "gpio8";
+		function = "func1";
+	};
+};
+
 &pmr735a_gpios {
 	hastings_reg_en: hastings-reg-en-state {
 		pins = "gpio1";
@@ -883,6 +1073,20 @@ hstp-sw-ctrl-pins {
 		};
 	};
 
+	edp_reg_en: edp-reg-en-state {
+		pins = "gpio36";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	hall_int_n_default: hall-int-n-state {
+		pins = "gpio180";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
 	nvme_reg_en: nvme-reg-en-state {
 		pins = "gpio135";
 		function = "gpio";
@@ -996,7 +1200,7 @@ tx-pins {
 
 	usb0_sbu_default: usb0-sbu-state {
 		oe-n-pins {
-			pins = "gpio101";
+			pins = "gpio100";
 			function = "gpio";
 			bias-disable;
 			drive-strength = <16>;
@@ -1035,4 +1239,11 @@ reset-pins {
 			bias-disable;
 		};
 	};
+
+	wlan_en: wlan-en-state {
+		pins = "gpio134";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-down;
+	};
 };
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts
index 125af356e24b9..9da2ffb318d04 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts
@@ -624,9 +624,6 @@ &mdss0_dp2_phy {
 };
 
 &pcie2a {
-	perst-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>;
-
 	vddpe-3v3-supply = <&vreg_nvme>;
 
 	pinctrl-0 = <&pcie2a_default>;
@@ -642,12 +639,14 @@ &pcie2a_phy {
 	status = "okay";
 };
 
+&pcie2a_port0 {
+	reset-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>;
+};
+
 &pcie4 {
 	max-link-speed = <2>;
 
-	perst-gpios = <&tlmm 141 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 139 GPIO_ACTIVE_LOW>;
-
 	vddpe-3v3-supply = <&vreg_wlan>;
 
 	pinctrl-0 = <&pcie4_default>;
@@ -657,6 +656,9 @@ &pcie4 {
 };
 
 &pcie4_port0 {
+	reset-gpios = <&tlmm 141 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 139 GPIO_ACTIVE_LOW>;
+
 	wifi@0 {
 		compatible = "pci17cb,1103";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
@@ -763,20 +765,11 @@ &pmk8280_pon_resin {
 };
 
 &pmk8280_rtc {
-	nvmem-cells = <&rtc_offset>;
-	nvmem-cell-names = "offset";
+	qcom,uefi-rtc-info;
 
 	status = "okay";
 };
 
-&pmk8280_sdam_6 {
-	status = "okay";
-
-	rtc_offset: rtc-offset@bc {
-		reg = <0xbc 0x4>;
-	};
-};
-
 &pmk8280_vadc {
 	channel@144 {
 		reg = <PM8350_ADC7_AMUX_THM1_100K_PU(1)>;
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
index a2bd6b10e4752..fbcfd3883bbb4 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
@@ -11,13 +11,16 @@
 #include <dt-bindings/clock/qcom,rpmh.h>
 #include <dt-bindings/clock/qcom,sc8280xp-camcc.h>
 #include <dt-bindings/clock/qcom,sc8280xp-lpasscc.h>
+#include <dt-bindings/clock/qcom,sm8350-videocc.h>
 #include <dt-bindings/dma/qcom-gpi.h>
+#include <dt-bindings/interconnect/qcom,icc.h>
 #include <dt-bindings/interconnect/qcom,osm-l3.h>
 #include <dt-bindings/interconnect/qcom,sc8280xp.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/mailbox/qcom-ipcc.h>
 #include <dt-bindings/phy/phy-qcom-qmp.h>
 #include <dt-bindings/power/qcom-rpmpd.h>
+#include <dt-bindings/reset/qcom,sm8350-videocc.h>
 #include <dt-bindings/soc/qcom,gpr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
 #include <dt-bindings/sound/qcom,q6afe.h>
@@ -314,78 +317,18 @@ scm: scm {
 		};
 	};
 
-	aggre1_noc: interconnect-aggre1-noc {
-		compatible = "qcom,sc8280xp-aggre1-noc";
-		#interconnect-cells = <2>;
-		qcom,bcm-voters = <&apps_bcm_voter>;
-	};
-
-	aggre2_noc: interconnect-aggre2-noc {
-		compatible = "qcom,sc8280xp-aggre2-noc";
-		#interconnect-cells = <2>;
-		qcom,bcm-voters = <&apps_bcm_voter>;
-	};
-
 	clk_virt: interconnect-clk-virt {
 		compatible = "qcom,sc8280xp-clk-virt";
 		#interconnect-cells = <2>;
 		qcom,bcm-voters = <&apps_bcm_voter>;
 	};
 
-	config_noc: interconnect-config-noc {
-		compatible = "qcom,sc8280xp-config-noc";
-		#interconnect-cells = <2>;
-		qcom,bcm-voters = <&apps_bcm_voter>;
-	};
-
-	dc_noc: interconnect-dc-noc {
-		compatible = "qcom,sc8280xp-dc-noc";
-		#interconnect-cells = <2>;
-		qcom,bcm-voters = <&apps_bcm_voter>;
-	};
-
-	gem_noc: interconnect-gem-noc {
-		compatible = "qcom,sc8280xp-gem-noc";
-		#interconnect-cells = <2>;
-		qcom,bcm-voters = <&apps_bcm_voter>;
-	};
-
-	lpass_noc: interconnect-lpass-ag-noc {
-		compatible = "qcom,sc8280xp-lpass-ag-noc";
-		#interconnect-cells = <2>;
-		qcom,bcm-voters = <&apps_bcm_voter>;
-	};
-
 	mc_virt: interconnect-mc-virt {
 		compatible = "qcom,sc8280xp-mc-virt";
 		#interconnect-cells = <2>;
 		qcom,bcm-voters = <&apps_bcm_voter>;
 	};
 
-	mmss_noc: interconnect-mmss-noc {
-		compatible = "qcom,sc8280xp-mmss-noc";
-		#interconnect-cells = <2>;
-		qcom,bcm-voters = <&apps_bcm_voter>;
-	};
-
-	nspa_noc: interconnect-nspa-noc {
-		compatible = "qcom,sc8280xp-nspa-noc";
-		#interconnect-cells = <2>;
-		qcom,bcm-voters = <&apps_bcm_voter>;
-	};
-
-	nspb_noc: interconnect-nspb-noc {
-		compatible = "qcom,sc8280xp-nspb-noc";
-		#interconnect-cells = <2>;
-		qcom,bcm-voters = <&apps_bcm_voter>;
-	};
-
-	system_noc: interconnect-system-noc {
-		compatible = "qcom,sc8280xp-system-noc";
-		#interconnect-cells = <2>;
-		qcom,bcm-voters = <&apps_bcm_voter>;
-	};
-
 	memory@80000000 {
 		device_type = "memory";
 		/* We expect the bootloader to fill in the size */
@@ -692,8 +635,8 @@ reserved-region@85b00000 {
 			no-map;
 		};
 
-		pil_gpu_mem: gpu-mem@8bf00000 {
-			reg = <0 0x8bf00000 0 0x2000>;
+		pil_video_mem: video-region@86700000 {
+			reg = <0 0x86700000 0 0x500000>;
 			no-map;
 		};
 
@@ -712,6 +655,11 @@ pil_nsp0_mem: cdsp0-region@8a100000 {
 			no-map;
 		};
 
+		pil_gpu_mem: gpu-mem@8bf00000 {
+			reg = <0 0x8bf00000 0 0x2000>;
+			no-map;
+		};
+
 		pil_nsp1_mem: cdsp1-region@8c600000 {
 			reg = <0 0x8c600000 0 0x1e00000>;
 			no-map;
@@ -2152,6 +2100,63 @@ rng: rng@10d3000 {
 			clock-names = "core";
 		};
 
+		config_noc: interconnect@1500000 {
+			compatible = "qcom,sc8280xp-config-noc";
+			reg = <0 0x01500000 0 0x2c000>;
+
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		system_noc: interconnect@1680000 {
+			compatible = "qcom,sc8280xp-system-noc";
+			reg = <0 0x01680000 0 0x1a400>;
+
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		aggre1_noc: interconnect@16c0000 {
+			compatible = "qcom,sc8280xp-aggre1-noc";
+			reg = <0 0x016c0000 0 0x3af80>;
+
+			clocks = <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB3_PRIM_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB3_SEC_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB3_MP_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB4_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB4_1_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB_NOC_SOUTH_AXI_CLK>,
+				 <&rpmhcc RPMH_IPA_CLK>;
+
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		aggre2_noc: interconnect@1700000 {
+			compatible = "qcom,sc8280xp-aggre2-noc";
+			reg = <0 0x01700000 0 0x3af80>;
+
+			clocks = <&gcc GCC_AGGRE_NOC_PCIE0_TUNNEL_AXI_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE1_TUNNEL_AXI_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE_4_AXI_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE_SOUTH_SF_AXI_CLK>,
+				 <&gcc GCC_AGGRE_UFS_CARD_AXI_CLK>,
+				 <&gcc GCC_DDRSS_GPU_AXI_CLK>,
+				 <&gcc GCC_DDRSS_PCIE_SF_TBU_CLK>;
+
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		mmss_noc: interconnect@1740000 {
+			compatible = "qcom,sc8280xp-mmss-noc";
+			reg = <0 0x01740000 0 0x1fa80>;
+
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
 		pcie4: pcie@1c00000 {
 			device_type = "pci";
 			compatible = "qcom,pcie-sc8280xp";
@@ -2220,9 +2225,6 @@ pcie4: pcie@1c00000 {
 			power-domains = <&gcc PCIE_4_GDSC>;
 			required-opps = <&rpmhpd_opp_nom>;
 
-			phys = <&pcie4_phy>;
-			phy-names = "pciephy";
-
 			status = "disabled";
 
 			pcie4_port0: pcie@0 {
@@ -2230,6 +2232,8 @@ pcie4_port0: pcie@0 {
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie4_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -2331,9 +2335,6 @@ pcie3b: pcie@1c08000 {
 			power-domains = <&gcc PCIE_3B_GDSC>;
 			required-opps = <&rpmhpd_opp_nom>;
 
-			phys = <&pcie3b_phy>;
-			phy-names = "pciephy";
-
 			status = "disabled";
 
 			pcie3b_port0: pcie@0 {
@@ -2341,6 +2342,8 @@ pcie3b_port0: pcie@0 {
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie3b_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -2442,9 +2445,6 @@ pcie3a: pcie@1c10000 {
 			power-domains = <&gcc PCIE_3A_GDSC>;
 			required-opps = <&rpmhpd_opp_nom>;
 
-			phys = <&pcie3a_phy>;
-			phy-names = "pciephy";
-
 			status = "disabled";
 
 			pcie3a_port0: pcie@0 {
@@ -2452,6 +2452,8 @@ pcie3a_port0: pcie@0 {
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie3a_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -2556,9 +2558,6 @@ pcie2b: pcie@1c18000 {
 			power-domains = <&gcc PCIE_2B_GDSC>;
 			required-opps = <&rpmhpd_opp_nom>;
 
-			phys = <&pcie2b_phy>;
-			phy-names = "pciephy";
-
 			status = "disabled";
 
 			pcie2b_port0: pcie@0 {
@@ -2566,6 +2565,8 @@ pcie2b_port0: pcie@0 {
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie2b_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -2667,9 +2668,6 @@ pcie2a: pcie@1c20000 {
 			power-domains = <&gcc PCIE_2A_GDSC>;
 			required-opps = <&rpmhpd_opp_nom>;
 
-			phys = <&pcie2a_phy>;
-			phy-names = "pciephy";
-
 			status = "disabled";
 
 			pcie2a_port0: pcie@0 {
@@ -2677,6 +2675,8 @@ pcie2a_port0: pcie@0 {
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie2a_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -3379,6 +3379,14 @@ lpasscc: clock-controller@33e0000 {
 			#reset-cells = <1>;
 		};
 
+		lpass_noc: interconnect@3c40000 {
+			compatible = "qcom,sc8280xp-lpass-ag-noc";
+			reg = <0 0x03c40000 0 0xf080>;
+
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
 		gpu: gpu@3d00000 {
 			compatible = "qcom,adreno-690.0", "qcom,adreno";
 
@@ -3954,6 +3962,22 @@ opp-6 {
 			};
 		};
 
+		dc_noc: interconnect@90e0000 {
+			compatible = "qcom,sc8280xp-dc-noc";
+			reg = <0 0x090e0000 0 0x5080>;
+
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		gem_noc: interconnect@9100000 {
+			compatible = "qcom,sc8280xp-gem-noc";
+			reg = <0 0x09100000 0 0xb8400>;
+
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
 		system-cache-controller@9200000 {
 			compatible = "qcom,sc8280xp-llcc";
 			reg = <0 0x09200000 0 0x58000>, <0 0x09280000 0 0x58000>,
@@ -4209,6 +4233,97 @@ usb_1_dwc3_ss: endpoint {
 			};
 		};
 
+		iris: video-codec@aa00000 {
+			compatible = "qcom,sc8280xp-iris", "qcom,sm8250-venus";
+			reg = <0x0 0x0aa00000 0x0 0x100000>;
+			interrupts = <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH>;
+
+			clocks = <&gcc GCC_VIDEO_AXI0_CLK>,
+				 <&videocc VIDEO_CC_MVS0C_CLK>,
+				 <&videocc VIDEO_CC_MVS0_CLK>;
+			clock-names = "iface",
+				      "core",
+				      "vcodec0_core";
+			power-domains = <&videocc MVS0C_GDSC>,
+					<&videocc MVS0_GDSC>,
+					<&rpmhpd SC8280XP_MX>,
+					<&rpmhpd SC8280XP_MMCX>;
+			power-domain-names = "venus",
+					     "vcodec0",
+					     "mx",
+					     "mmcx";
+
+			resets = <&gcc GCC_VIDEO_AXI0_CLK_ARES>,
+				 <&videocc VIDEO_CC_MVS0C_CLK_ARES>;
+			reset-names = "bus", "core";
+
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_VENUS_CFG QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&mmss_noc MASTER_VIDEO_P0 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "cpu-cfg",
+					     "video-mem";
+
+			operating-points-v2 = <&iris_opp_table>;
+			iommus = <&apps_smmu 0x2a00 0x400>;
+			memory-region = <&pil_video_mem>;
+
+			status = "disabled";
+
+			iris_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-240000000 {
+					opp-hz = /bits/ 64 <240000000>;
+					required-opps = <&rpmhpd_opp_svs>,
+							<&rpmhpd_opp_low_svs>;
+				};
+
+				opp-338000000 {
+					opp-hz = /bits/ 64 <338000000>;
+					required-opps = <&rpmhpd_opp_svs>,
+							<&rpmhpd_opp_svs>;
+				};
+
+				opp-366000000 {
+					opp-hz = /bits/ 64 <366000000>;
+					required-opps = <&rpmhpd_opp_svs_l1>,
+							<&rpmhpd_opp_svs_l1>;
+				};
+
+				opp-444000000 {
+					opp-hz = /bits/ 64 <444000000>;
+					required-opps = <&rpmhpd_opp_svs_l1>,
+							<&rpmhpd_opp_nom>;
+				};
+
+				opp-533000000 {
+					opp-hz = /bits/ 64 <533000000>;
+					required-opps = <&rpmhpd_opp_nom>,
+							<&rpmhpd_opp_turbo>;
+				};
+
+				opp-560000000 {
+					opp-hz = /bits/ 64 <560000000>;
+					required-opps = <&rpmhpd_opp_nom>,
+							<&rpmhpd_opp_turbo_l1>;
+				};
+			};
+		};
+
+		videocc: clock-controller@abf0000 {
+			compatible = "qcom,sc8280xp-videocc";
+			reg = <0 0x0abf0000 0 0x10000>;
+			clocks = <&rpmhcc RPMH_CXO_CLK>,
+				 <&rpmhcc RPMH_CXO_CLK_A>,
+				 <&sleep_clk>;
+			power-domains = <&rpmhpd SC8280XP_MMCX>;
+			required-opps = <&rpmhpd_opp_low_svs>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
 		cci0: cci@ac4a000 {
 			compatible = "qcom,sc8280xp-cci", "qcom,msm8996-cci";
 			reg = <0 0x0ac4a000 0 0x1000>;
@@ -5326,7 +5441,7 @@ dispcc0: clock-controller@af00000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sc8280xp-pdc", "qcom,pdc";
-			reg = <0 0x0b220000 0 0x30000>, <0 0x17c000f0 0 0x60>;
+			reg = <0 0x0b220000 0 0x10000>, <0 0x17c000f0 0 0x60>;
 			qcom,pdc-ranges = <0 480 40>,
 					  <40 140 14>,
 					  <54 263 1>,
@@ -5356,7 +5471,7 @@ pdc: interrupt-controller@b220000 {
 					  <211 699 1>,
 					  <212 705 1>,
 					  <213 450 1>,
-					  <214 643 1>,
+					  <214 643 2>,
 					  <216 646 5>,
 					  <221 390 5>,
 					  <226 700 3>,
@@ -5379,7 +5494,7 @@ pdc: interrupt-controller@b220000 {
 					  <252 798 1>,
 					  <253 765 1>,
 					  <254 763 1>,
-					  <255 454 1>,
+					  <255 454 3>,
 					  <258 139 1>,
 					  <259 786 2>,
 					  <261 370 2>,
@@ -5484,6 +5599,62 @@ tlmm: pinctrl@f100000 {
 			gpio-ranges = <&tlmm 0 0 230>;
 			wakeup-parent = <&pdc>;
 
+			cam_mclk0_default: cam-mclk0-default-state {
+				pins = "gpio119";
+				function = "cam_mclk";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			cam_mclk1_default: cam-mclk1-default-state {
+				pins = "gpio120";
+				function = "cam_mclk";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			cam_mclk2_default: cam-mclk2-default-state {
+				pins = "gpio16";
+				function = "cam_mclk";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			cam_mclk3_default: cam-mclk3-default-state {
+				pins = "gpio17";
+				function = "cam_mclk";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			cam_mclk4_default: cam-mclk4-default-state {
+				pins = "gpio6";
+				function = "cam_mclk";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			cam_mclk5_default: cam-mclk5-default-state {
+				pins = "gpio7";
+				function = "cam_mclk";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			cam_mclk6_default: cam-mclk6-default-state {
+				pins = "gpio33";
+				function = "cam_mclk";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			cam_mclk7_default: cam-mclk7-default-state {
+				pins = "gpio34";
+				function = "cam_mclk";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
 			cci0_default: cci0-default-state {
 				cci0_i2c0_default: cci0-i2c0-default-pins {
 					/* cci_i2c_sda0, cci_i2c_scl0 */
@@ -6004,6 +6175,14 @@ cpufreq_hw: cpufreq@18591000 {
 			#clock-cells = <1>;
 		};
 
+		nspa_noc: interconnect@1b0c0000 {
+			compatible = "qcom,sc8280xp-nspa-noc";
+			reg = <0 0x1b0c0000 0 0x10000>;
+
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
 		remoteproc_nsp0: remoteproc@1b300000 {
 			compatible = "qcom,sc8280xp-nsp0-pas";
 			reg = <0 0x1b300000 0 0x10000>;
@@ -6139,6 +6318,14 @@ compute-cb@14 {
 			};
 		};
 
+		nspb_noc: interconnect@210c0000 {
+			compatible = "qcom,sc8280xp-nspb-noc";
+			reg = <0 0x210c0000 0 0x10000>;
+
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
 		remoteproc_nsp1: remoteproc@21300000 {
 			compatible = "qcom,sc8280xp-nsp1-pas";
 			reg = <0 0x21300000 0 0x10000>;
diff --git a/arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi b/arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi
index 0f57b915186b7..b4854801a5f5e 100644
--- a/arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi
@@ -522,6 +522,11 @@ rmi4_f12: rmi4-f12@12 {
 	};
 };
 
+&lpi_tlmm {
+	/* sensor gpios are protected */
+	gpio-reserved-ranges = <0 8>, <12 6>;
+};
+
 &mdss {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/sdm670.dtsi b/arch/arm64/boot/dts/qcom/sdm670.dtsi
index 30844b150c801..5c60ea36f04a1 100644
--- a/arch/arm64/boot/dts/qcom/sdm670.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm670.dtsi
@@ -1646,7 +1646,7 @@ usb_1_dwc3: usb@a600000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sdm670-pdc", "qcom,pdc";
-			reg = <0 0x0b220000 0 0x30000>;
+			reg = <0 0x0b220000 0 0x10000>;
 			qcom,pdc-ranges = <0 480 40>, <41 521 7>, <49 529 4>,
 					  <54 534 24>, <79 559 15>, <94 609 15>,
 					  <115 630 7>;
diff --git a/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi
index b2dc0327dff85..9368fd6b87515 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi
@@ -66,8 +66,8 @@ venus_mem: venus@97800000 {
 			no-map;
 		};
 
-		cdsp_mem: cdsp-mem@97D00000 {
-			reg = <0 0x97D00000 0 0x800000>;
+		cdsp_mem: cdsp-mem@97d00000 {
+			reg = <0 0x97d00000 0 0x800000>;
 			no-map;
 		};
 
@@ -81,8 +81,8 @@ slpi_mem: slpi@98700000 {
 			no-map;
 		};
 
-		spss_mem: spss@99B00000 {
-			reg = <0 0x99B00000 0 0x100000>;
+		spss_mem: spss@99b00000 {
+			reg = <0 0x99b00000 0 0x100000>;
 			no-map;
 		};
 
diff --git a/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi
index e0c3566761bf8..5ac2e058827cc 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi
@@ -472,9 +472,6 @@ &gpu {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
-
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sdm845-mtp.dts b/arch/arm64/boot/dts/qcom/sdm845-mtp.dts
index 091568642faa7..97c79b51ff425 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-mtp.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-mtp.dts
@@ -428,8 +428,6 @@ &i2c10 {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi
index a6c2519a418d9..439779f308e38 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi
@@ -520,7 +520,7 @@ &i2c12 {
 	clock-frequency = <400000>;
 
 	synaptics-rmi4-i2c@20 {
-		compatible = "syna,rmi4-i2c";
+		compatible = "syna,rmi4-s3706b", "syna,rmi4-i2c";
 		reg = <0x20>;
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -551,8 +551,6 @@ rmi4_f12: rmi4-f12@12 {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	firmware-name = "qcom/sdm845/OnePlus/enchilada/ipa_fws.mbn";
 
 	status = "okay";
diff --git a/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts b/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts
index 0d915ecb0444f..a404a23086d47 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts
@@ -913,8 +913,6 @@ &mss_pil {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	firmware-name = "qcom/sdm845/Samsung/starqltechn/ipa_fws.mbn";
 
 	status = "okay";
diff --git a/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts b/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts
index f21b914f22b51..e31c784eff3de 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts
@@ -530,8 +530,6 @@ audio-codec@34 {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	firmware-name = "qcom/sdm845/SHIFT/axolotl/ipa_fws.mbn";
 
 	status = "okay";
diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi
index 54e58ca046934..cc14d9fa3b051 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi
@@ -285,8 +285,6 @@ &ibb {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	firmware-name = "qcom/sdm845/Xiaomi/beryllium/ipa_fws.mbn";
 
 	status = "okay";
diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts
index a44d6e776c82e..96d4f7496f2f8 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts
@@ -409,8 +409,6 @@ &ibb {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	firmware-name = "qcom/sdm845/Xiaomi/polaris/ipa_fws.mbn";
 
 	status = "okay";
diff --git a/arch/arm64/boot/dts/qcom/sdm845.dtsi b/arch/arm64/boot/dts/qcom/sdm845.dtsi
index 4ae8627d6dbc3..745ca60fc8ed8 100644
--- a/arch/arm64/boot/dts/qcom/sdm845.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845.dtsi
@@ -2780,6 +2780,9 @@ ipa: ipa@1e40000 {
 			qcom,smem-state-names = "ipa-clock-enabled-valid",
 						"ipa-clock-enabled";
 
+			qcom,gsi-loader = "self";
+			memory-region = <&ipa_fw_mem>;
+
 			sram = <&ipa_modem_tables>;
 
 			status = "disabled";
@@ -5102,7 +5105,7 @@ dispcc: clock-controller@af00000 {
 
 		pdc_intc: interrupt-controller@b220000 {
 			compatible = "qcom,sdm845-pdc", "qcom,pdc";
-			reg = <0 0x0b220000 0 0x30000>;
+			reg = <0 0x0b220000 0 0x10000>;
 			qcom,pdc-ranges = <0 480 94>, <94 609 15>, <115 630 7>;
 			#interrupt-cells = <2>;
 			interrupt-parent = <&intc>;
diff --git a/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts b/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts
index f048653818702..e38b768d16a1d 100644
--- a/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts
+++ b/arch/arm64/boot/dts/qcom/sdm850-huawei-matebook-e-2019.dts
@@ -565,8 +565,6 @@ panel_in_edp: endpoint {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	firmware-name = "qcom/sdm850/HUAWEI/AL09/ipa_fws.elf";
 
 	status = "okay";
diff --git a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts
index e41200839dbec..9b6f0c1fa811d 100644
--- a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts
+++ b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts
@@ -347,6 +347,10 @@ &cdsp_pas {
 	status = "okay";
 };
 
+&cluster_sleep_0 {
+	arm,psci-suspend-param = <0x41008244>;
+};
+
 &gcc {
 	protected-clocks = <GCC_QSPI_CORE_CLK>,
 			   <GCC_QSPI_CORE_CLK_SRC>,
@@ -584,8 +588,6 @@ ecsh: hid@5c {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	firmware-name = "qcom/sdm850/LENOVO/81JL/ipa_fws.elf";
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts b/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts
index a676d3ea01b99..1dab0582eae6f 100644
--- a/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts
+++ b/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts
@@ -384,9 +384,8 @@ digitizer@9 {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	firmware-name = "qcom/sdm850/samsung/w737/ipa_fws.elf";
+
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sdx75.dtsi b/arch/arm64/boot/dts/qcom/sdx75.dtsi
index d1b61530b562f..85cc9c6e6f83f 100644
--- a/arch/arm64/boot/dts/qcom/sdx75.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdx75.dtsi
@@ -1100,7 +1100,7 @@ usb_1_dwc3_ss: endpoint {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sdx75-pdc", "qcom,pdc";
-			reg = <0x0 0xb220000 0x0 0x30000>,
+			reg = <0x0 0xb220000 0x0 0x10000>,
 			      <0x0 0x174000f0 0x0 0x64>;
 			qcom,pdc-ranges = <0 147 52>,
 					  <52 266 32>,
diff --git a/arch/arm64/boot/dts/qcom/shikra-cqm-evk.dts b/arch/arm64/boot/dts/qcom/shikra-cqm-evk.dts
new file mode 100644
index 0000000000000..0a52ab9b7a4c3
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/shikra-cqm-evk.dts
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include "shikra-cqm-som.dtsi"
+#include "shikra-evk.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. Shikra CQM EVK";
+	compatible = "qcom,shikra-cqm-evk", "qcom,shikra-cqm-som", "qcom,shikra";
+	chassis-type = "embedded";
+
+	aliases {
+		mmc0 = &sdhc_1;
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+};
+
+&sdhc_1 {
+	vmmc-supply = <&pm4125_l20>;
+	vqmmc-supply = <&pm4125_l14>;
+
+	pinctrl-0 = <&sdc1_state_on>;
+	pinctrl-1 = <&sdc1_state_off>;
+	pinctrl-names = "default", "sleep";
+
+	non-removable;
+	supports-cqe;
+	no-sdio;
+	no-sd;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/shikra-cqm-som.dtsi b/arch/arm64/boot/dts/qcom/shikra-cqm-som.dtsi
new file mode 100644
index 0000000000000..dc3861489f64d
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/shikra-cqm-som.dtsi
@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+
+#include "shikra.dtsi"
+#include "pm4125.dtsi"
+#include "pm8005.dtsi"
+
+/ {
+	gpio-keys {
+		compatible = "gpio-keys";
+		label = "gpio-keys";
+		pinctrl-0 = <&vol_up_n>;
+		pinctrl-names = "default";
+
+		key-volume-up {
+			label = "Volume Up";
+			gpios = <&pm4125_gpios 9 GPIO_ACTIVE_LOW>;
+			linux,input-type = <1>;
+			linux,code = <KEY_VOLUMEUP>;
+			wakeup-source;
+			debounce-interval = <15>;
+			linux,can-disable;
+		};
+	};
+};
+
+&pm4125_gpios {
+	vol_up_n: vol-up-n-state {
+		pins = "gpio9";
+		function = PMIC_GPIO_FUNC_NORMAL;
+		input-enable;
+		bias-pull-up;
+		power-source = <0>;
+	};
+};
+
+&pm4125_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+
+	status = "okay";
+};
+
+&pm8005_regulators {
+	status = "disabled";
+};
+
+&rpm_requests {
+	regulators {
+		compatible = "qcom,rpm-pm2250-regulators";
+
+		pm4125_s2: s2 {
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1200000>;
+		};
+
+		pm4125_l3: l3 {
+			regulator-min-microvolt = <624000>;
+			regulator-max-microvolt = <650000>;
+		};
+
+		pm4125_l4: l4 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2960000>;
+		};
+
+		pm4125_l5: l5 {
+			regulator-min-microvolt = <1232000>;
+			regulator-max-microvolt = <1304000>;
+		};
+
+		pm4125_l6: l6 {
+			regulator-min-microvolt = <788000>;
+			regulator-max-microvolt = <1050000>;
+		};
+
+		pm4125_l7: l7 {
+			regulator-min-microvolt = <664000>;
+			regulator-max-microvolt = <664000>;
+		};
+
+		pm4125_l8: l8 {
+			regulator-min-microvolt = <928000>;
+			regulator-max-microvolt = <1000000>;
+		};
+
+		pm4125_l9: l9 {
+			regulator-min-microvolt = <875000>;
+			regulator-max-microvolt = <1000000>;
+		};
+
+		pm4125_l10: l10 {
+			regulator-min-microvolt = <1304000>;
+			regulator-max-microvolt = <1304000>;
+		};
+
+		pm4125_l12: l12 {
+			regulator-min-microvolt = <928000>;
+			regulator-max-microvolt = <975000>;
+		};
+
+		pm4125_l13: l13 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		pm4125_l14: l14 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		pm4125_l15: l15 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		pm4125_l16: l16 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		pm4125_l17: l17 {
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3544000>;
+		};
+
+		pm4125_l18: l18 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2960000>;
+		};
+
+		pm4125_l19: l19 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2960000>;
+		};
+
+		pm4125_l20: l20 {
+			regulator-min-microvolt = <2952000>;
+			regulator-max-microvolt = <2952000>;
+		};
+
+		pm4125_l21: l21 {
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3056000>;
+		};
+
+		pm4125_l22: l22 {
+			regulator-min-microvolt = <3304000>;
+			regulator-max-microvolt = <3304000>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/shikra-cqs-evk.dts b/arch/arm64/boot/dts/qcom/shikra-cqs-evk.dts
new file mode 100644
index 0000000000000..b3f19a64d7aed
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/shikra-cqs-evk.dts
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include "shikra-cqm-som.dtsi"
+#include "shikra-evk.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. Shikra CQS EVK";
+	compatible = "qcom,shikra-cqs-evk", "qcom,shikra-cqs-som", "qcom,shikra";
+	chassis-type = "embedded";
+
+	aliases {
+		mmc0 = &sdhc_1;
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+};
+
+&sdhc_1 {
+	vmmc-supply = <&pm4125_l20>;
+	vqmmc-supply = <&pm4125_l14>;
+
+	pinctrl-0 = <&sdc1_state_on>;
+	pinctrl-1 = <&sdc1_state_off>;
+	pinctrl-names = "default", "sleep";
+
+	non-removable;
+	supports-cqe;
+	no-sdio;
+	no-sd;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/shikra-evk.dtsi b/arch/arm64/boot/dts/qcom/shikra-evk.dtsi
new file mode 100644
index 0000000000000..6eb4184f76422
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/shikra-evk.dtsi
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+&qupv3_0 {
+	firmware-name = "qcom/shikra/qupv3fw.elf";
+
+	status = "okay";
+};
+
+&uart0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/shikra-iqs-evk.dts b/arch/arm64/boot/dts/qcom/shikra-iqs-evk.dts
new file mode 100644
index 0000000000000..3003a47bd7594
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/shikra-iqs-evk.dts
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include "shikra-iqs-som.dtsi"
+#include "shikra-evk.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. Shikra IQS EVK";
+	compatible = "qcom,shikra-iqs-evk", "qcom,shikra-iqs-som", "qcom,shikra";
+	chassis-type = "embedded";
+
+	aliases {
+		mmc0 = &sdhc_1;
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+};
+
+&sdhc_1 {
+	vmmc-supply = <&pm8150_l17>;
+	vqmmc-supply = <&pm8150_s4>;
+
+	pinctrl-0 = <&sdc1_state_on>;
+	pinctrl-1 = <&sdc1_state_off>;
+	pinctrl-names = "default", "sleep";
+
+	non-removable;
+	supports-cqe;
+	no-sdio;
+	no-sd;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/shikra-iqs-som.dtsi b/arch/arm64/boot/dts/qcom/shikra-iqs-som.dtsi
new file mode 100644
index 0000000000000..73945bf42112d
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/shikra-iqs-som.dtsi
@@ -0,0 +1,170 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+
+#include "shikra.dtsi"
+#include "pm8150.dtsi"
+
+/ {
+	gpio-key {
+		compatible = "gpio-keys";
+		label = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&vol_up_n>;
+
+		key-volume-up {
+			label = "Volume Up";
+			gpios = <&pm8150_gpios 6 GPIO_ACTIVE_LOW>;
+			linux,input-type = <1>;
+			linux,code = <KEY_VOLUMEUP>;
+			wakeup-source;
+			debounce-interval = <15>;
+			linux,can-disable;
+		};
+	};
+};
+
+&pm8150_gpios {
+	vol_up_n: vol-up-n-state {
+		pins = "gpio6";
+		function = PMIC_GPIO_FUNC_NORMAL;
+		input-enable;
+		bias-pull-up;
+		power-source = <0>;
+	};
+
+};
+
+&pon_pwrkey {
+	status = "okay";
+};
+
+&pon_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+	status = "okay";
+};
+
+&rpm_requests {
+	regulators {
+		compatible = "qcom,rpm-pm8150-regulators";
+
+		pm8150_s4: s4 {
+			regulator-min-microvolt = <1080000>;
+			regulator-max-microvolt = <2040000>;
+		};
+
+		pm8150_s5: s5 {
+			regulator-min-microvolt = <1574000>;
+			regulator-max-microvolt = <2040000>;
+		};
+
+		pm8150_s6: s6 {
+			regulator-min-microvolt = <382000>;
+			regulator-max-microvolt = <1352000>;
+		};
+
+		pm8150_s7: s7 {
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1200000>;
+		};
+
+		pm8150_s8: s8 {
+			regulator-min-microvolt = <570000>;
+			regulator-max-microvolt = <650000>;
+		};
+
+		pm8150_l1: l1 {
+			regulator-min-microvolt = <312000>;
+			regulator-max-microvolt = <1304000>;
+		};
+
+		pm8150_l2: l2 {
+			regulator-min-microvolt = <1650000>;
+			regulator-max-microvolt = <3300000>;
+		};
+
+		pm8150_l3: l3 {
+			regulator-min-microvolt = <312000>;
+			regulator-max-microvolt = <1304000>;
+		};
+
+		pm8150_l4: l4 {
+			regulator-min-microvolt = <875000>;
+			regulator-max-microvolt = <975000>;
+		};
+
+		pm8150_l5: l5 {
+			regulator-min-microvolt = <788000>;
+			regulator-max-microvolt = <1050000>;
+		};
+
+		pm8150_l6: l6 {
+			regulator-min-microvolt = <875000>;
+			regulator-max-microvolt = <1000000>;
+		};
+
+		pm8150_l7: l7 {
+			regulator-min-microvolt = <1504000>;
+			regulator-max-microvolt = <2000000>;
+		};
+
+		pm8150_l8: l8 {
+			regulator-min-microvolt = <1150000>;
+			regulator-max-microvolt = <1304000>;
+		};
+
+		pm8150_l9: l9 {
+			regulator-min-microvolt = <875000>;
+			regulator-max-microvolt = <1000000>;
+		};
+
+		pm8150_l10: l10 {
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3544000>;
+		};
+
+		pm8150_l11: l11 {
+			regulator-min-microvolt = <1100000>;
+			regulator-max-microvolt = <1304000>;
+		};
+
+		pm8150_l12: l12 {
+			regulator-min-microvolt = <1650000>;
+			regulator-max-microvolt = <1950000>;
+		};
+
+		pm8150_l13: l13 {
+			regulator-min-microvolt = <2921000>;
+			regulator-max-microvolt = <3230000>;
+		};
+
+		pm8150_l14: l14 {
+			regulator-min-microvolt = <1700000>;
+			regulator-max-microvolt = <1910000>;
+		};
+
+		pm8150_l15: l15 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1900000>;
+		};
+
+		pm8150_l16: l16 {
+			regulator-min-microvolt = <1504000>;
+			regulator-max-microvolt = <3544000>;
+		};
+
+		pm8150_l17: l17 {
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3544000>;
+		};
+
+		pm8150_l18: l18 {
+			regulator-min-microvolt = <400000>;
+			regulator-max-microvolt = <728000>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/shikra.dtsi b/arch/arm64/boot/dts/qcom/shikra.dtsi
new file mode 100644
index 0000000000000..4e5bc9e17c8ed
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/shikra.dtsi
@@ -0,0 +1,842 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/clock/qcom,rpmcc.h>
+#include <dt-bindings/clock/qcom,shikra-gcc.h>
+#include <dt-bindings/interconnect/qcom,icc.h>
+#include <dt-bindings/interconnect/qcom,rpm-icc.h>
+#include <dt-bindings/interconnect/qcom,shikra.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/power/qcom-rpmpd.h>
+
+/ {
+	interrupt-parent = <&intc>;
+
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	clocks {
+		xo_board: xo-board {
+			compatible = "fixed-clock";
+			clock-frequency = <38400000>;
+			#clock-cells = <0>;
+		};
+
+		sleep_clk: sleep-clk {
+			compatible = "fixed-clock";
+			clock-frequency = <32764>;
+			#clock-cells = <0>;
+		};
+	};
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x0 0x0>;
+			enable-method = "psci";
+			next-level-cache = <&l3>;
+			capacity-dmips-mhz = <1024>;
+			dynamic-power-coefficient = <100>;
+		};
+
+		cpu1: cpu@100 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x0 0x100>;
+			enable-method = "psci";
+			next-level-cache = <&l3>;
+			capacity-dmips-mhz = <1024>;
+			dynamic-power-coefficient = <100>;
+		};
+
+		cpu2: cpu@200 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x0 0x200>;
+			enable-method = "psci";
+			next-level-cache = <&l3>;
+			capacity-dmips-mhz = <1024>;
+			dynamic-power-coefficient = <100>;
+		};
+
+		cpu3: cpu@300 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a78c";
+			reg = <0x0 0x300>;
+			enable-method = "psci";
+			next-level-cache = <&l2_3>;
+			capacity-dmips-mhz = <1946>;
+			dynamic-power-coefficient = <489>;
+
+			l2_3: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3>;
+				cache-size = <0x40000>;
+			};
+		};
+
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&cpu0>;
+				};
+
+				core1 {
+					cpu = <&cpu1>;
+				};
+
+				core2 {
+					cpu = <&cpu2>;
+				};
+			};
+
+			cluster1 {
+				core0 {
+					cpu = <&cpu3>;
+				};
+			};
+		};
+
+		l3: l3-cache {
+			compatible = "cache";
+			cache-level = <3>;
+			cache-unified;
+			cache-size = <0x80000>;
+		};
+	};
+
+	firmware {
+		scm {
+			compatible = "qcom,scm-shikra", "qcom,scm";
+			clocks = <&rpmcc RPM_SMD_CE1_CLK>;
+			clock-names = "core";
+			qcom,dload-mode = <&tcsr_regs 0x13000>;
+			#reset-cells = <1>;
+			interconnects = <&system_noc MASTER_CRYPTO_CORE0 RPM_ALWAYS_TAG
+					 &mc_virt SLAVE_EBI_CH0 RPM_ALWAYS_TAG>;
+		};
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		/* We expect the bootloader to fill in the size */
+		reg = <0x0 0x80000000 0x0 0x0>;
+	};
+
+	pmu-a55 {
+		compatible = "arm,cortex-a55-pmu";
+		interrupts = <GIC_PPI 5 IRQ_TYPE_LEVEL_HIGH &ppi_cluster0>;
+	};
+
+	pmu-a78c {
+		compatible = "arm,cortex-a78-pmu";
+		interrupts = <GIC_PPI 5 IRQ_TYPE_LEVEL_HIGH &ppi_cluster1>;
+	};
+
+	psci {
+		compatible = "arm,psci-1.0";
+		method = "smc";
+	};
+
+	rpm: remoteproc {
+		compatible = "qcom,shikra-rpm-proc", "qcom,rpm-proc";
+
+		glink-edge {
+			compatible = "qcom,glink-rpm";
+			interrupts = <GIC_SPI 194 IRQ_TYPE_EDGE_RISING 0>;
+			qcom,rpm-msg-ram = <&rpm_msg_ram>;
+			mboxes = <&apcs_glb 0>;
+
+			rpm_requests: rpm-requests {
+				compatible = "qcom,rpm-shikra", "qcom,glink-smd-rpm";
+				qcom,glink-channels = "rpm_requests";
+
+				rpmcc: clock-controller {
+					compatible = "qcom,rpmcc-shikra", "qcom,rpmcc-qcm2290", "qcom,rpmcc";
+					clocks = <&xo_board>;
+					clock-names = "xo";
+					#clock-cells = <1>;
+				};
+
+				rpmpd: power-controller {
+					compatible = "qcom,shikra-rpmpd";
+					#power-domain-cells = <1>;
+					operating-points-v2 = <&rpmpd_opp_table>;
+
+					rpmpd_opp_table: opp-table {
+						compatible = "operating-points-v2";
+
+						rpmpd_opp_min_svs: opp1 {
+							opp-level = <RPM_SMD_LEVEL_MIN_SVS>;
+						};
+
+						rpmpd_opp_low_svs: opp2 {
+							opp-level = <RPM_SMD_LEVEL_LOW_SVS>;
+						};
+
+						rpmpd_opp_svs: opp3 {
+							opp-level = <RPM_SMD_LEVEL_SVS>;
+						};
+
+						rpmpd_opp_svs_plus: opp4 {
+							opp-level = <RPM_SMD_LEVEL_SVS_PLUS>;
+						};
+
+						rpmpd_opp_nom: opp5 {
+							opp-level = <RPM_SMD_LEVEL_NOM>;
+						};
+
+						rpmpd_opp_nom_plus: opp6 {
+							opp-level = <RPM_SMD_LEVEL_NOM_PLUS>;
+						};
+
+						rpmpd_opp_turbo: opp7 {
+							opp-level = <RPM_SMD_LEVEL_TURBO>;
+						};
+
+						rpmpd_opp_turbo_plus: opp8 {
+							opp-level = <RPM_SMD_LEVEL_TURBO_NO_CPR>;
+						};
+					};
+				};
+			};
+		};
+
+		mpm: interrupt-controller {
+			compatible = "qcom,mpm";
+			qcom,rpm-msg-ram = <&apss_mpm>;
+			interrupts = <GIC_SPI 197 IRQ_TYPE_EDGE_RISING 0>;
+			mboxes = <&apcs_glb 1>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			#power-domain-cells = <0>;
+			interrupt-parent = <&intc>;
+			qcom,mpm-pin-count = <96>;
+			qcom,mpm-pin-map = <2 275>,  /* TSENS0 uplow */
+					   <12 422>, /* DWC3 ss_phy_irq */
+					   <58 272>, /* QUSB2_PHY dmse_hv_vddmx */
+					   <59 273>, /* QUSB2_PHY dpse_hv_vddmx */
+					   <86 183>, /* MPM wake, SPMI */
+					   <90 157>, /* QUSB2_PHY DM */
+					   <91 158>; /* QUSB2_PHY DP */
+		};
+	};
+
+	reserved_memory: reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		hyp_mem: hyp@80000000 {
+			reg = <0x0 0x80000000 0x0 0x1600000>;
+			no-map;
+		};
+
+		xblboot_mem: xblboot@85e00000 {
+			reg = <0x0 0x85e00000 0x0 0x100000>;
+			no-map;
+		};
+
+		secdata_apss_mem: secdata-apss@85fff000 {
+			reg = <0x0 0x85fff000 0x0 0x1000>;
+			no-map;
+		};
+
+		smem_mem: smem@86000000 {
+			compatible = "qcom,smem";
+			reg = <0x0 0x86000000 0x0 0x200000>;
+			no-map;
+
+			hwlocks = <&tcsr_mutex 3>;
+		};
+
+		audio_heap_mem: audio-heap@86200000 {
+			reg = <0x0 0x86200000 0x0 0x100000>;
+			no-map;
+		};
+
+		tz_stat_mem: tz-stat@a0000000 {
+			reg = <0x0 0xa0000000 0x0 0x100000>;
+			no-map;
+		};
+
+		qtee_mem: qtee@a1300000 {
+			reg = <0x0 0xa1300000 0x0 0x500000>;
+			no-map;
+		};
+
+		tz_apps_mem: tz-apps@a1800000 {
+			reg = <0x0 0xa1800000 0x0 0x2100000>;
+			no-map;
+		};
+
+		mpss_wlan_mem: mpss-wlan@ab000000 {
+			reg = <0x0 0xab000000 0x0 0x6e00000>;
+			no-map;
+		};
+
+		wlan_mem: wlan@b2300000 {
+			reg = <0x0 0xb2300000 0x0 0x100000>;
+			no-map;
+		};
+
+		cdsp_mem: cdsp@b2400000 {
+			reg = <0x0 0xb2400000 0x0 0x1900000>;
+			no-map;
+		};
+
+		gpu_micro_code_mem: gpu-micro-code@b3d00000 {
+			reg = <0x0 0xb3d00000 0x0 0x2000>;
+			no-map;
+		};
+
+		video_mem: video@b3d02000 {
+			reg = <0x0 0xb3d02000 0x0 0x700000>;
+			no-map;
+		};
+
+		lmcu_mem: lmcu@b4402000 {
+			reg = <0x0 0xb4402000 0x0 0x300000>;
+			no-map;
+		};
+
+		lmcu_dtb_mem: lmcu-dtb@b4702000 {
+			reg = <0x0 0xb4702000 0x0 0x40000>;
+			no-map;
+		};
+	};
+
+	soc: soc@0 {
+		compatible = "simple-bus";
+
+		#address-cells = <2>;
+		#size-cells = <2>;
+		dma-ranges = <0x0 0x0 0x0 0x0 0x10 0x0>;
+		ranges = <0x0 0x0 0x0 0x0 0x10 0x0>;
+
+		tcsr_mutex: syscon@340000 {
+			compatible = "qcom,tcsr-mutex";
+			reg = <0x0 0x00340000 0x0 0x20000>;
+			#hwlock-cells = <1>;
+		};
+
+		tcsr_regs: syscon@3c0000 {
+			compatible = "qcom,shikra-tcsr", "syscon";
+			reg = <0x0 0x003c0000 0x0 0x40000>;
+		};
+
+		tlmm: pinctrl@500000 {
+			compatible = "qcom,shikra-tlmm";
+			reg = <0x0 0x00500000 0x0 0x700000>;
+
+			interrupts = <GIC_SPI 227 IRQ_TYPE_LEVEL_HIGH 0>;
+
+			gpio-controller;
+			#gpio-cells = <2>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+
+			gpio-ranges = <&tlmm 0 0 165>;
+			wakeup-parent = <&mpm>;
+
+			qup_uart0_default: qup-uart0-default-state {
+				pins = "gpio0", "gpio1";
+				function = "qup0_se0";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
+			sdc1_state_on: sdc1-on-state {
+				clk-pins {
+					pins = "sdc1_clk";
+					drive-strength = <6>;
+					bias-disable;
+				};
+
+				cmd-pins {
+					pins = "sdc1_cmd";
+					drive-strength = <6>;
+					bias-pull-up;
+				};
+
+				data-pins {
+					pins = "sdc1_data";
+					drive-strength = <6>;
+					bias-pull-up;
+				};
+
+				rclk-pins {
+					pins = "sdc1_rclk";
+					bias-pull-down;
+				};
+			};
+
+			sdc1_state_off: sdc1-off-state {
+				clk-pins {
+					pins = "sdc1_clk";
+					drive-strength = <2>;
+					bias-bus-hold;
+				};
+
+				cmd-pins {
+					pins = "sdc1_cmd";
+					drive-strength = <2>;
+					bias-bus-hold;
+				};
+
+				data-pins {
+					pins = "sdc1_data";
+					drive-strength = <2>;
+					bias-bus-hold;
+				};
+
+				rclk-pins {
+					pins = "sdc1_rclk";
+					bias-bus-hold;
+				};
+			};
+		};
+
+		mem_noc: interconnect@d00000 {
+			compatible = "qcom,shikra-mem-noc-core";
+			reg = <0x0 0x00d00000 0x0 0x43080>;
+			clocks = <&gcc GCC_DDRSS_GPU_AXI_CLK>;
+			clock-names = "gpu_axi";
+			#interconnect-cells = <2>;
+		};
+
+		llcc: system-cache-controller@e00000 {
+			compatible = "qcom,shikra-llcc";
+			reg = <0x0 0x00e00000 0x0 0x80000>,
+			      <0x0 0x00f00000 0x0 0x80000>,
+			      <0x0 0x01000000 0x0 0x80000>;
+			reg-names = "llcc0_base",
+				    "llcc1_base",
+				    "llcc_broadcast_base";
+			interrupts = <GIC_SPI 539 IRQ_TYPE_LEVEL_HIGH 0>;
+		};
+
+		gcc: clock-controller@1400000 {
+			compatible = "qcom,shikra-gcc";
+			reg = <0x0 0x01400000 0x0 0x1f0000>;
+			clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>,
+				 <&sleep_clk>,
+				 <0>,
+				 <0>,
+				 <0>,
+				 <0>,
+				 <0>,
+				 <0>;
+			power-domains = <&rpmpd RPMPD_VDDCX>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
+		system_noc: interconnect@1880000 {
+			compatible = "qcom,shikra-sys-noc";
+			reg = <0x0 0x01880000 0x0 0x6a080>;
+			clocks = <&gcc GCC_EMAC0_AXI_SYS_NOC_CLK>,
+				 <&gcc GCC_EMAC1_AXI_SYS_NOC_CLK>,
+				 <&gcc GCC_SYS_NOC_USB2_PRIM_AXI_CLK>,
+				 <&gcc GCC_SYS_NOC_USB3_PRIM_AXI_CLK>;
+			clock-names = "emac0_axi",
+				      "emac1_axi",
+				      "usb2_axi",
+				      "usb3_axi";
+			#interconnect-cells = <2>;
+
+			clk_virt: interconnect-clk {
+				compatible = "qcom,shikra-clk-virt";
+				#interconnect-cells = <2>;
+			};
+
+			mc_virt: interconnect-mc {
+				compatible = "qcom,shikra-mc-virt";
+				#interconnect-cells = <2>;
+			};
+
+			mmrt_virt: interconnect-mmrt {
+				compatible = "qcom,shikra-mmrt-virt";
+				#interconnect-cells = <2>;
+			};
+
+			mmnrt_virt: interconnect-mmnrt {
+				compatible = "qcom,shikra-mmnrt-virt";
+				#interconnect-cells = <2>;
+			};
+		};
+
+		config_noc: interconnect@1900000 {
+			compatible = "qcom,shikra-config-noc";
+			reg = <0x0 0x01900000 0x0 0x8080>;
+			#interconnect-cells = <2>;
+		};
+
+		qfprom: efuse@1b44000 {
+			compatible = "qcom,shikra-qfprom", "qcom,qfprom";
+			reg = <0x0 0x01b44000 0x0 0x3000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			qusb2_hstx_trim_1: hstx-trim@25b {
+				reg = <0x25b 0x1>;
+				bits = <1 4>;
+			};
+
+			gpu_speed_bin: gpu-speed-bin@2006 {
+				reg = <0x2006 0x2>;
+				bits = <5 8>;
+			};
+		};
+
+		spmi_bus: spmi@1c40000 {
+			compatible = "qcom,spmi-pmic-arb";
+			reg = <0x0 0x01c40000 0x0 0x1100>,
+			      <0x0 0x01e00000 0x0 0x2000000>,
+			      <0x0 0x03e00000 0x0 0x100000>,
+			      <0x0 0x03f00000 0x0 0xa0000>,
+			      <0x0 0x01c0a000 0x0 0x26000>;
+			reg-names = "core",
+				    "chnls",
+				    "obsrvr",
+				    "intr",
+				    "cnfg";
+			interrupts-extended = <&mpm 86 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "periph_irq";
+			interrupt-controller;
+			#interrupt-cells = <4>;
+			#address-cells = <2>;
+			#size-cells = <0>;
+			qcom,channel = <0>;
+			qcom,ee = <0>;
+		};
+
+		rpm_msg_ram: sram@45f0000 {
+			compatible = "qcom,rpm-msg-ram", "mmio-sram";
+			reg = <0x0 0x045f0000 0x0 0x7000>;
+
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0x0 0x0 0x045f0000 0x7000>;
+
+			apss_mpm: sram@1b8 {
+				reg = <0x1b8 0x48>;
+			};
+		};
+
+		sram@4690000 {
+			compatible = "qcom,rpm-stats";
+			reg = <0x0 0x04690000 0x0 0x14000>;
+		};
+
+		sdhc_1: mmc@4744000 {
+			compatible = "qcom,shikra-sdhci", "qcom,sdhci-msm-v5";
+
+			reg = <0x0 0x04744000 0x0 0x1000>,
+			      <0x0 0x04745000 0x0 0x1000>;
+			reg-names = "hc",
+				    "cqhci";
+
+			iommus = <&apps_smmu 0xc0 0x0>;
+
+			interrupts = <GIC_SPI 348 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 352 IRQ_TYPE_LEVEL_HIGH 0>;
+			interrupt-names = "hc_irq",
+					  "pwr_irq";
+
+			clocks = <&gcc GCC_SDCC1_AHB_CLK>,
+				 <&gcc GCC_SDCC1_APPS_CLK>,
+				 <&rpmcc RPM_SMD_XO_CLK_SRC>;
+			clock-names = "iface",
+				      "core",
+				      "xo";
+
+			interconnects = <&system_noc MASTER_SDCC_1 RPM_ALWAYS_TAG
+					 &mc_virt SLAVE_EBI_CH0 RPM_ALWAYS_TAG>,
+					<&mem_noc MASTER_AMPSS_M0 RPM_ACTIVE_TAG
+					 &config_noc SLAVE_SDCC_1 RPM_ACTIVE_TAG>;
+			interconnect-names = "sdhc-ddr",
+					     "cpu-sdhc";
+
+			power-domains = <&rpmpd RPMPD_VDDCX>;
+			operating-points-v2 = <&sdhc1_opp_table>;
+
+			qcom,dll-config = <0x000f642c>;
+			qcom,ddr-config = <0x80040868>;
+
+			bus-width = <8>;
+
+			mmc-ddr-1_8v;
+			mmc-hs200-1_8v;
+			mmc-hs400-1_8v;
+			mmc-hs400-enhanced-strobe;
+
+			resets = <&gcc GCC_SDCC1_BCR>;
+
+			status = "disabled";
+
+			sdhc1_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-100000000 {
+					opp-hz = /bits/ 64 <100000000>;
+					required-opps = <&rpmpd_opp_low_svs>;
+					opp-peak-kBps = <250000 133320>;
+					opp-avg-kBps = <104000 0>;
+				};
+
+				opp-384000000 {
+					opp-hz = /bits/ 64 <384000000>;
+					required-opps = <&rpmpd_opp_nom>;
+					opp-peak-kBps = <800000 300000>;
+					opp-avg-kBps = <400000 0>;
+				};
+			};
+		};
+
+		qupv3_0: geniqup@4ac0000 {
+			compatible = "qcom,geni-se-qup";
+			reg = <0x0 0x04ac0000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_QUPV3_WRAP_0_M_AHB_CLK>,
+				 <&gcc GCC_QUPV3_WRAP_0_S_AHB_CLK>;
+			clock-names = "m-ahb",
+				      "s-ahb";
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			uart0: serial@4a80000 {
+				compatible = "qcom,geni-debug-uart";
+				reg = <0x0 0x04a80000 0x0 0x4000>;
+
+				interrupts = <GIC_SPI 527 IRQ_TYPE_LEVEL_HIGH 0>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 RPM_ALWAYS_TAG
+						 &clk_virt SLAVE_QUP_CORE_0 RPM_ALWAYS_TAG>,
+						<&mem_noc MASTER_AMPSS_M0 RPM_ALWAYS_TAG
+						 &config_noc SLAVE_QUP_0 RPM_ALWAYS_TAG>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart0_default>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+		};
+
+		sram@c11e000 {
+			compatible = "qcom,shikra-imem", "mmio-sram";
+			reg = <0x0 0x0c11e000 0x0 0x1000>;
+			ranges = <0x0 0x0 0x0c11e000 0x1000>;
+
+			no-memory-wc;
+
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			pil-sram@94c {
+				compatible = "qcom,pil-reloc-info";
+				reg = <0x94c 0xc8>;
+			};
+		};
+
+		apps_smmu: iommu@c600000 {
+			compatible = "qcom,shikra-smmu-500", "qcom,smmu-500", "arm,mmu-500";
+			reg = <0x0 0x0c600000 0x0 0x80000>;
+			#iommu-cells = <2>;
+			#global-interrupts = <1>;
+
+			interrupts = <GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 91 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 92 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 93 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 124 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 126 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 128 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 129 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 130 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 132 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 133 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 134 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 135 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 136 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 139 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH 0>;
+		};
+
+		intc: interrupt-controller@f200000 {
+			compatible = "arm,gic-v3";
+			reg = <0x0 0xf200000 0x0 0x10000>,
+			      <0x0 0xf240000 0x0 0x80000>;
+
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH 0>;
+
+			#interrupt-cells = <4>;
+			interrupt-controller;
+
+			#redistributor-regions = <1>;
+			redistributor-stride = <0x0 0x20000>;
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			ppi-partitions {
+				ppi_cluster0: interrupt-partition-0 {
+					affinity = <&cpu0 &cpu1 &cpu2>;
+				};
+
+				ppi_cluster1: interrupt-partition-1 {
+					affinity = <&cpu3>;
+				};
+			};
+		};
+
+		apcs_glb: mailbox@f400000 {
+			compatible = "qcom,shikra-apss-shared", "qcom,sdm845-apss-shared";
+			reg = <0x0 0x0f400000 0x0 0x1000>;
+			#mbox-cells = <1>;
+		};
+
+		watchdog@f410000 {
+			compatible = "qcom,apss-wdt-shikra", "qcom,kpss-wdt";
+			reg = <0x0 0x0f410000 0x0 0x1000>;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&sleep_clk>;
+		};
+
+		timer@f420000 {
+			compatible = "arm,armv7-timer-mem";
+			reg = <0x0 0x0f420000  0x0 0x1000>;
+
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0x0 0x0 0x0 0x10000000>;
+
+			frame@f421000 {
+				reg = <0x0f421000 0x1000>,
+				      <0x0f422000 0x1000>;
+				frame-number = <0>;
+				interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH 0>,
+					     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH 0>;
+			};
+
+			frame@f423000 {
+				reg = <0x0f423000 0x1000>;
+				frame-number = <1>;
+				interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH 0>;
+				status = "disabled";
+			};
+
+			frame@f425000 {
+				reg = <0x0f425000 0x1000>;
+				frame-number = <2>;
+				interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH 0>;
+				status = "disabled";
+			};
+
+			frame@f427000 {
+				reg = <0x0f427000 0x1000>;
+				frame-number = <3>;
+				interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH 0>;
+				status = "disabled";
+			};
+
+			frame@f429000 {
+				reg = <0x0f429000 0x1000>;
+				frame-number = <4>;
+				interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH 0>;
+				status = "disabled";
+			};
+
+			frame@f42b000 {
+				reg = <0x0f42b000 0x1000>;
+				frame-number = <5>;
+				interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH 0>;
+				status = "disabled";
+			};
+
+			frame@f42d000 {
+				reg = <0x0f42d000 0x1000>;
+				frame-number = <6>;
+				interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH 0>;
+				status = "disabled";
+			};
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+
+		interrupts = <GIC_PPI 1 IRQ_TYPE_LEVEL_LOW 0>,
+			     <GIC_PPI 2 IRQ_TYPE_LEVEL_LOW 0>,
+			     <GIC_PPI 3 IRQ_TYPE_LEVEL_LOW 0>,
+			     <GIC_PPI 0 IRQ_TYPE_LEVEL_LOW 0>;
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/sm4450.dtsi b/arch/arm64/boot/dts/qcom/sm4450.dtsi
index 696e2e0841ad9..b7dc40f3778c4 100644
--- a/arch/arm64/boot/dts/qcom/sm4450.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm4450.dtsi
@@ -464,7 +464,7 @@ dispcc: clock-controller@af00000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sm4450-pdc", "qcom,pdc";
-			reg = <0 0x0b220000 0 0x30000>, <0 0x174000f0 0 0x64>;
+			reg = <0 0x0b220000 0 0x10000>, <0 0x174000f0 0 0x64>;
 			qcom,pdc-ranges = <0 480 94>, <94 494 31>,
 					  <125 63 1>;
 			#interrupt-cells = <2>;
diff --git a/arch/arm64/boot/dts/qcom/sm6115-fxtec-pro1x.dts b/arch/arm64/boot/dts/qcom/sm6115-fxtec-pro1x.dts
index 0f23eaef01f21..b290f0f4c0e10 100644
--- a/arch/arm64/boot/dts/qcom/sm6115-fxtec-pro1x.dts
+++ b/arch/arm64/boot/dts/qcom/sm6115-fxtec-pro1x.dts
@@ -151,7 +151,7 @@ touchscreen@14 {
 
 		interrupts-extended = <&tlmm 80 IRQ_TYPE_LEVEL_LOW>;
 
-		irq-gpios = <&tlmm 80 IRQ_TYPE_LEVEL_LOW>;
+		irq-gpios = <&tlmm 80 GPIO_ACTIVE_LOW>;
 		reset-gpios = <&tlmm 71 GPIO_ACTIVE_HIGH>;
 		AVDD28-supply = <&ts_vdd_supply>;
 		VDDIO-supply = <&ts_vddio_supply>;
diff --git a/arch/arm64/boot/dts/qcom/sm6115.dtsi b/arch/arm64/boot/dts/qcom/sm6115.dtsi
index bd94eb87d6f93..3520802d1fed5 100644
--- a/arch/arm64/boot/dts/qcom/sm6115.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm6115.dtsi
@@ -1853,6 +1853,71 @@ adreno_smmu: iommu@59a0000 {
 			#iommu-cells = <2>;
 		};
 
+		venus: video-codec@5a00000 {
+			compatible = "qcom,sm6115-venus",
+				     "qcom,qcm2290-venus";
+			reg = <0x0 0x5a00000 0x0 0xf0000>;
+			interrupts = <GIC_SPI 225 IRQ_TYPE_LEVEL_HIGH>;
+
+			power-domains = <&gcc GCC_VENUS_GDSC>,
+					<&gcc GCC_VCODEC0_GDSC>,
+					<&rpmpd SM6115_VDDCX>;
+			power-domain-names = "venus",
+					     "vcodec0",
+					     "cx";
+			operating-points-v2 = <&venus_opp_table>;
+
+			clocks = <&gcc GCC_VIDEO_VENUS_CTL_CLK>,
+				 <&gcc GCC_VIDEO_AHB_CLK>,
+				 <&gcc GCC_VENUS_CTL_AXI_CLK>,
+				 <&gcc GCC_VIDEO_THROTTLE_CORE_CLK>,
+				 <&gcc GCC_VIDEO_VCODEC0_SYS_CLK>,
+				 <&gcc GCC_VCODEC0_AXI_CLK>;
+			clock-names = "core",
+				      "iface",
+				      "bus",
+				      "throttle",
+				      "vcodec0_core",
+				      "vcodec0_bus";
+
+			memory-region = <&pil_video_mem>;
+			iommus = <&apps_smmu 0x860 0x0>,
+				 <&apps_smmu 0x880 0x0>;
+
+			interconnects = <&mmnrt_virt MASTER_VIDEO_P0 RPM_ALWAYS_TAG
+					 &bimc SLAVE_EBI_CH0 RPM_ALWAYS_TAG>,
+					<&bimc MASTER_AMPSS_M0 RPM_ACTIVE_TAG
+					 &config_noc SLAVE_VENUS_CFG RPM_ACTIVE_TAG>;
+			interconnect-names = "video-mem",
+					     "cpu-cfg";
+
+			status = "disabled";
+
+			venus_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-133333333 {
+					opp-hz = /bits/ 64 <133333333>;
+					required-opps = <&rpmpd_opp_low_svs>;
+				};
+
+				opp-240000000 {
+					opp-hz = /bits/ 64 <240000000>;
+					required-opps = <&rpmpd_opp_svs>;
+				};
+
+				opp-300000000 {
+					opp-hz = /bits/ 64 <300000000>;
+					required-opps = <&rpmpd_opp_svs_plus>;
+				};
+
+				opp-384000000 {
+					opp-hz = /bits/ 64 <384000000>;
+					required-opps = <&rpmpd_opp_nom>;
+				};
+			};
+		};
+
 		mdss: display-subsystem@5e00000 {
 			compatible = "qcom,sm6115-mdss";
 			reg = <0x0 0x05e00000 0x0 0x1000>;
diff --git a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-ginkgo-common.dtsi b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-ginkgo-common.dtsi
index 7eecd9dc30280..a5458ab41629b 100644
--- a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-ginkgo-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-ginkgo-common.dtsi
@@ -12,6 +12,7 @@
 #include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
 #include "sm6125.dtsi"
 #include "pm6125.dtsi"
+#include "pmi632.dtsi"
 
 /delete-node/ &adsp_pil_mem;
 /delete-node/ &cont_splash_mem;
@@ -46,22 +47,22 @@ framebuffer {
 	};
 
 	reserved-memory {
-		adsp_pil_mem: adsp_pil_mem@55300000 {
+		adsp_pil_mem: adsp-pil-mem@55300000 {
 			reg = <0x0 0x55300000 0x0 0x2200000>;
 			no-map;
 		};
 
-		ipa_fw_mem: ipa_fw_mem@57500000 {
+		ipa_fw_mem: ipa-fw-mem@57500000 {
 			reg = <0x0 0x57500000 0x0 0x10000>;
 			no-map;
 		};
 
-		ipa_gsi_mem: ipa_gsi_mem@57510000 {
+		ipa_gsi_mem: ipa-gsi-mem@57510000 {
 			reg = <0x0 0x57510000 0x0 0x5000>;
 			no-map;
 		};
 
-		gpu_mem: gpu_mem@57515000 {
+		gpu_mem: gpu-mem@57515000 {
 			reg = <0x0 0x57515000 0x0 0x2000>;
 			no-map;
 		};
@@ -98,6 +99,10 @@ key-volume-up {
 	};
 };
 
+&gpi_dma1 {
+	status = "okay";
+};
+
 &pm6125_gpios {
 	vol_up_n: vol-up-n-state {
 		pins = "gpio5";
@@ -115,6 +120,37 @@ &hsusb_phy1 {
 	status = "okay";
 };
 
+&pmi632_typec {
+	status = "okay";
+
+	connector {
+		compatible = "usb-c-connector";
+
+		power-role = "dual";
+		data-role = "dual";
+		self-powered;
+
+		typec-power-opmode = "default";
+		pd-disable;
+
+		port {
+			pmi632_hs_in: endpoint {
+				remote-endpoint = <&usb_dwc3_hs>;
+			};
+		};
+	};
+};
+
+&pmi632_vbus {
+	regulator-min-microamp = <500000>;
+	regulator-max-microamp = <1000000>;
+	status = "okay";
+};
+
+&pmi632_vib {
+	status = "okay";
+};
+
 &pon_pwrkey {
 	status = "okay";
 };
@@ -128,6 +164,10 @@ &qupv3_id_0 {
 	status = "okay";
 };
 
+&qupv3_id_1 {
+	status = "okay";
+};
+
 &rpm_requests {
 	regulators-0 {
 		compatible = "qcom,rpm-pm6125-regulators";
@@ -300,6 +340,18 @@ &sdhc_2 {
 	status = "okay";
 };
 
+&spi8 {
+	status = "okay";
+
+	irled@1 {
+		compatible = "ir-spi-led";
+		reg = <1>;
+
+		duty-cycle = /bits/ 8 <30>;
+		spi-max-frequency = <1000000>;
+	};
+};
+
 &tlmm {
 	gpio-reserved-ranges = <0 4>, <30 4>;
 };
@@ -311,3 +363,7 @@ &uart4 {
 &usb3 {
 	status = "okay";
 };
+
+&usb_dwc3_hs {
+	remote-endpoint = <&pmi632_hs_in>;
+};
diff --git a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts
index b9d9439e230b4..139f2b401af50 100644
--- a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts
+++ b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts
@@ -198,7 +198,7 @@ &mdss_dsi0 {
 	status = "okay";
 
 	panel@0 {
-		compatible = "samsung,s6e8fco-m1906f9";
+		compatible = "samsung,s6e8fc0-m1906f9";
 		reg = <0>;
 
 		reset-gpios = <&tlmm 90 GPIO_ACTIVE_LOW>;
diff --git a/arch/arm64/boot/dts/qcom/sm6125.dtsi b/arch/arm64/boot/dts/qcom/sm6125.dtsi
index 6e84c226948c0..86de1560bdefa 100644
--- a/arch/arm64/boot/dts/qcom/sm6125.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm6125.dtsi
@@ -374,22 +374,23 @@ smem: smem {
 	};
 
 	soc@0 {
-		#address-cells = <1>;
-		#size-cells = <1>;
-		ranges = <0x00 0x00 0x00 0xffffffff>;
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges = <0 0 0 0 0x10 0>;
+		dma-ranges = <0 0 0 0 0x10 0>;
 		compatible = "simple-bus";
 
 		tcsr_mutex: hwlock@340000 {
 			compatible = "qcom,tcsr-mutex";
-			reg = <0x00340000 0x20000>;
+			reg = <0x0 0x00340000 0x0 0x20000>;
 			#hwlock-cells = <1>;
 		};
 
 		tlmm: pinctrl@500000 {
 			compatible = "qcom,sm6125-tlmm";
-			reg = <0x00500000 0x400000>,
-			      <0x00900000 0x400000>,
-			      <0x00d00000 0x400000>;
+			reg = <0x0 0x00500000 0x0 0x400000>,
+			      <0x0 0x00900000 0x0 0x400000>,
+			      <0x0 0x00d00000 0x0 0x400000>;
 			reg-names = "west", "south", "east";
 			interrupts = <GIC_SPI 227 IRQ_TYPE_LEVEL_HIGH>;
 			gpio-controller;
@@ -672,7 +673,7 @@ qup_uart4_default: qup-uart4-default-state {
 
 		gcc: clock-controller@1400000 {
 			compatible = "qcom,gcc-sm6125";
-			reg = <0x01400000 0x1f0000>;
+			reg = <0x0 0x01400000 0x0 0x1f0000>;
 			#clock-cells = <1>;
 			#reset-cells = <1>;
 			#power-domain-cells = <1>;
@@ -682,7 +683,7 @@ gcc: clock-controller@1400000 {
 
 		hsusb_phy1: phy@1613000 {
 			compatible = "qcom,msm8996-qusb2-phy";
-			reg = <0x01613000 0x180>;
+			reg = <0x0 0x01613000 0x0 0x180>;
 			#phy-cells = <0>;
 
 			clocks = <&gcc GCC_AHB2PHY_USB_CLK>,
@@ -695,18 +696,18 @@ hsusb_phy1: phy@1613000 {
 
 		rng: rng@1b53000 {
 			compatible = "qcom,prng-ee";
-			reg = <0x01b53000 0x1000>;
+			reg = <0x0 0x01b53000 0x0 0x1000>;
 			clocks = <&gcc GCC_PRNG_AHB_CLK>;
 			clock-names = "core";
 		};
 
 		spmi_bus: spmi@1c40000 {
 			compatible = "qcom,spmi-pmic-arb";
-			reg = <0x01c40000 0x1100>,
-			      <0x01e00000 0x2000000>,
-			      <0x03e00000 0x100000>,
-			      <0x03f00000 0xa0000>,
-			      <0x01c0a000 0x26000>;
+			reg = <0x0 0x01c40000 0x0 0x1100>,
+			      <0x0 0x01e00000 0x0 0x2000000>,
+			      <0x0 0x03e00000 0x0 0x100000>,
+			      <0x0 0x03f00000 0x0 0xa0000>,
+			      <0x0 0x01c0a000 0x0 0x26000>;
 			reg-names = "core", "chnls", "obsrvr", "intr", "cnfg";
 			interrupt-names = "periph_irq";
 			interrupts = <GIC_SPI 183 IRQ_TYPE_LEVEL_HIGH>;
@@ -720,12 +721,13 @@ spmi_bus: spmi@1c40000 {
 
 		rpm_msg_ram: sram@45f0000 {
 			compatible = "qcom,rpm-msg-ram";
-			reg = <0x045f0000 0x7000>;
+			reg = <0x0 0x045f0000 0x0 0x7000>;
 		};
 
 		sdhc_1: mmc@4744000 {
 			compatible = "qcom,sm6125-sdhci", "qcom,sdhci-msm-v5";
-			reg = <0x04744000 0x1000>, <0x04745000 0x1000>;
+			reg = <0x0 0x04744000 0x0 0x1000>,
+			      <0x0 0x04745000 0x0 0x1000>;
 			reg-names = "hc", "cqhci";
 
 			interrupts = <GIC_SPI 348 IRQ_TYPE_LEVEL_HIGH>,
@@ -752,7 +754,7 @@ sdhc_1: mmc@4744000 {
 
 		sdhc_2: mmc@4784000 {
 			compatible = "qcom,sm6125-sdhci", "qcom,sdhci-msm-v5";
-			reg = <0x04784000 0x1000>;
+			reg = <0x0 0x04784000 0x0 0x1000>;
 			reg-names = "hc";
 
 			interrupts = <GIC_SPI 350 IRQ_TYPE_LEVEL_HIGH>,
@@ -780,7 +782,8 @@ sdhc_2: mmc@4784000 {
 
 		ufs_mem_hc: ufshc@4804000 {
 			compatible = "qcom,sm6125-ufshc", "qcom,ufshc", "jedec,ufs-2.0";
-			reg = <0x04804000 0x3000>, <0x04810000 0x8000>;
+			reg = <0x0 0x04804000 0x0 0x3000>,
+			      <0x0 0x04810000 0x0 0x8000>;
 			reg-names = "std", "ice";
 			interrupts = <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>;
 
@@ -825,7 +828,7 @@ ufs_mem_hc: ufshc@4804000 {
 
 		ufs_mem_phy: phy@4807000 {
 			compatible = "qcom,sm6125-qmp-ufs-phy";
-			reg = <0x04807000 0xdb8>;
+			reg = <0x0 0x04807000 0x0 0xdb8>;
 
 			clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>,
 				 <&gcc GCC_UFS_PHY_PHY_AUX_CLK>,
@@ -846,7 +849,7 @@ ufs_mem_phy: phy@4807000 {
 
 		gpi_dma0: dma-controller@4a00000 {
 			compatible = "qcom,sm6125-gpi-dma", "qcom,sdm845-gpi-dma";
-			reg = <0x04a00000 0x60000>;
+			reg = <0x0 0x04a00000 0x0 0x60000>;
 			interrupts = <GIC_SPI 335 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 336 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 337 IRQ_TYPE_LEVEL_HIGH>,
@@ -864,19 +867,19 @@ gpi_dma0: dma-controller@4a00000 {
 
 		qupv3_id_0: geniqup@4ac0000 {
 			compatible = "qcom,geni-se-qup";
-			reg = <0x04ac0000 0x2000>;
+			reg = <0x0 0x04ac0000 0x0 0x2000>;
 			clocks = <&gcc GCC_QUPV3_WRAP_0_M_AHB_CLK>,
 				 <&gcc GCC_QUPV3_WRAP_0_S_AHB_CLK>;
 			clock-names = "m-ahb", "s-ahb";
 			iommus = <&apps_smmu 0x123 0x0>;
-			#address-cells = <1>;
-			#size-cells = <1>;
+			#address-cells = <2>;
+			#size-cells = <2>;
 			ranges;
 			status = "disabled";
 
 			i2c0: i2c@4a80000 {
 				compatible = "qcom,geni-i2c";
-				reg = <0x04a80000 0x4000>;
+				reg = <0x0 0x04a80000 0x0 0x4000>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>;
 				clock-names = "se";
 				interrupts = <GIC_SPI 327 IRQ_TYPE_LEVEL_HIGH>;
@@ -893,7 +896,7 @@ i2c0: i2c@4a80000 {
 
 			spi0: spi@4a80000 {
 				compatible = "qcom,geni-spi";
-				reg = <0x04a80000 0x4000>;
+				reg = <0x0 0x04a80000 0x0 0x4000>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S0_CLK>;
 				clock-names = "se";
 				interrupts = <GIC_SPI 327 IRQ_TYPE_LEVEL_HIGH>;
@@ -910,7 +913,7 @@ spi0: spi@4a80000 {
 
 			i2c1: i2c@4a84000 {
 				compatible = "qcom,geni-i2c";
-				reg = <0x04a84000 0x4000>;
+				reg = <0x0 0x04a84000 0x0 0x4000>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>;
 				clock-names = "se";
 				interrupts = <GIC_SPI 328 IRQ_TYPE_LEVEL_HIGH>;
@@ -927,7 +930,7 @@ i2c1: i2c@4a84000 {
 
 			i2c2: i2c@4a88000 {
 				compatible = "qcom,geni-i2c";
-				reg = <0x04a88000 0x4000>;
+				reg = <0x0 0x04a88000 0x0 0x4000>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>;
 				clock-names = "se";
 				interrupts = <GIC_SPI 329 IRQ_TYPE_LEVEL_HIGH>;
@@ -944,7 +947,7 @@ i2c2: i2c@4a88000 {
 
 			spi2: spi@4a88000 {
 				compatible = "qcom,geni-spi";
-				reg = <0x04a88000 0x4000>;
+				reg = <0x0 0x04a88000 0x0 0x4000>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S2_CLK>;
 				clock-names = "se";
 				interrupts = <GIC_SPI 329 IRQ_TYPE_LEVEL_HIGH>;
@@ -961,7 +964,7 @@ spi2: spi@4a88000 {
 
 			i2c3: i2c@4a8c000 {
 				compatible = "qcom,geni-i2c";
-				reg = <0x04a8c000 0x4000>;
+				reg = <0x0 0x04a8c000 0x0 0x4000>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S3_CLK>;
 				clock-names = "se";
 				interrupts = <GIC_SPI 330 IRQ_TYPE_LEVEL_HIGH>;
@@ -978,7 +981,7 @@ i2c3: i2c@4a8c000 {
 
 			i2c4: i2c@4a90000 {
 				compatible = "qcom,geni-i2c";
-				reg = <0x04a90000 0x4000>;
+				reg = <0x0 0x04a90000 0x0 0x4000>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>;
 				clock-names = "se";
 				interrupts = <GIC_SPI 331 IRQ_TYPE_LEVEL_HIGH>;
@@ -995,7 +998,7 @@ i2c4: i2c@4a90000 {
 
 			uart4: serial@4a90000 {
 				compatible = "qcom,geni-debug-uart";
-				reg = <0x04a90000 0x4000>;
+				reg = <0x0 0x04a90000 0x0 0x4000>;
 				clocks = <&gcc GCC_QUPV3_WRAP0_S4_CLK>;
 				clock-names = "se";
 				interrupts = <GIC_SPI 331 IRQ_TYPE_LEVEL_HIGH>;
@@ -1007,7 +1010,7 @@ uart4: serial@4a90000 {
 
 		gpi_dma1: dma-controller@4c00000 {
 			compatible = "qcom,sm6125-gpi-dma", "qcom,sdm845-gpi-dma";
-			reg = <0x04c00000 0x60000>;
+			reg = <0x0 0x04c00000 0x0 0x60000>;
 			interrupts = <GIC_SPI 314 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 315 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 316 IRQ_TYPE_LEVEL_HIGH>,
@@ -1025,19 +1028,19 @@ gpi_dma1: dma-controller@4c00000 {
 
 		qupv3_id_1: geniqup@4cc0000 {
 			compatible = "qcom,geni-se-qup";
-			reg = <0x04cc0000 0x2000>;
+			reg = <0x0 0x04cc0000 0x0 0x2000>;
 			clocks = <&gcc GCC_QUPV3_WRAP_1_M_AHB_CLK>,
 				 <&gcc GCC_QUPV3_WRAP_1_S_AHB_CLK>;
 			clock-names = "m-ahb", "s-ahb";
 			iommus = <&apps_smmu 0x143 0x0>;
-			#address-cells = <1>;
-			#size-cells = <1>;
+			#address-cells = <2>;
+			#size-cells = <2>;
 			ranges;
 			status = "disabled";
 
 			i2c5: i2c@4c80000 {
 				compatible = "qcom,geni-i2c";
-				reg = <0x04c80000 0x4000>;
+				reg = <0x0 0x04c80000 0x0 0x4000>;
 				clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>;
 				clock-names = "se";
 				interrupts = <GIC_SPI 308 IRQ_TYPE_LEVEL_HIGH>;
@@ -1054,7 +1057,7 @@ i2c5: i2c@4c80000 {
 
 			spi5: spi@4c80000 {
 				compatible = "qcom,geni-spi";
-				reg = <0x04c80000 0x4000>;
+				reg = <0x0 0x04c80000 0x0 0x4000>;
 				clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>;
 				clock-names = "se";
 				interrupts = <GIC_SPI 308 IRQ_TYPE_LEVEL_HIGH>;
@@ -1071,7 +1074,7 @@ spi5: spi@4c80000 {
 
 			i2c6: i2c@4c84000 {
 				compatible = "qcom,geni-i2c";
-				reg = <0x04c84000 0x4000>;
+				reg = <0x0 0x04c84000 0x0 0x4000>;
 				clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>;
 				clock-names = "se";
 				interrupts = <GIC_SPI 309 IRQ_TYPE_LEVEL_HIGH>;
@@ -1088,7 +1091,7 @@ i2c6: i2c@4c84000 {
 
 			spi6: spi@4c84000 {
 				compatible = "qcom,geni-spi";
-				reg = <0x04c84000 0x4000>;
+				reg = <0x0 0x04c84000 0x0 0x4000>;
 				clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>;
 				clock-names = "se";
 				interrupts = <GIC_SPI 309 IRQ_TYPE_LEVEL_HIGH>;
@@ -1105,7 +1108,7 @@ spi6: spi@4c84000 {
 
 			i2c7: i2c@4c88000 {
 				compatible = "qcom,geni-i2c";
-				reg = <0x04c88000 0x4000>;
+				reg = <0x0 0x04c88000 0x0 0x4000>;
 				clocks = <&gcc GCC_QUPV3_WRAP1_S2_CLK>;
 				clock-names = "se";
 				interrupts = <GIC_SPI 310 IRQ_TYPE_LEVEL_HIGH>;
@@ -1122,7 +1125,7 @@ i2c7: i2c@4c88000 {
 
 			i2c8: i2c@4c8c000 {
 				compatible = "qcom,geni-i2c";
-				reg = <0x04c8c000 0x4000>;
+				reg = <0x0 0x04c8c000 0x0 0x4000>;
 				clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>;
 				clock-names = "se";
 				interrupts = <GIC_SPI 311 IRQ_TYPE_LEVEL_HIGH>;
@@ -1139,7 +1142,7 @@ i2c8: i2c@4c8c000 {
 
 			spi8: spi@4c8c000 {
 				compatible = "qcom,geni-spi";
-				reg = <0x04c8c000 0x4000>;
+				reg = <0x0 0x04c8c000 0x0 0x4000>;
 				clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>;
 				clock-names = "se";
 				interrupts = <GIC_SPI 311 IRQ_TYPE_LEVEL_HIGH>;
@@ -1156,7 +1159,7 @@ spi8: spi@4c8c000 {
 
 			i2c9: i2c@4c90000 {
 				compatible = "qcom,geni-i2c";
-				reg = <0x04c90000 0x4000>;
+				reg = <0x0 0x04c90000 0x0 0x4000>;
 				clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>;
 				clock-names = "se";
 				interrupts = <GIC_SPI 312 IRQ_TYPE_LEVEL_HIGH>;
@@ -1173,7 +1176,7 @@ i2c9: i2c@4c90000 {
 
 			spi9: spi@4c90000 {
 				compatible = "qcom,geni-spi";
-				reg = <0x04c90000 0x4000>;
+				reg = <0x0 0x04c90000 0x0 0x4000>;
 				clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>;
 				clock-names = "se";
 				interrupts = <GIC_SPI 312 IRQ_TYPE_LEVEL_HIGH>;
@@ -1191,9 +1194,9 @@ spi9: spi@4c90000 {
 
 		usb3: usb@4ef8800 {
 			compatible = "qcom,sm6125-dwc3", "qcom,dwc3";
-			reg = <0x04ef8800 0x400>;
-			#address-cells = <1>;
-			#size-cells = <1>;
+			reg = <0x0 0x04ef8800 0x0 0x400>;
+			#address-cells = <2>;
+			#size-cells = <2>;
 			ranges;
 
 			clocks = <&gcc GCC_CFG_NOC_USB3_PRIM_AXI_CLK>,
@@ -1228,7 +1231,7 @@ usb3: usb@4ef8800 {
 
 			usb3_dwc3: usb@4e00000 {
 				compatible = "snps,dwc3";
-				reg = <0x04e00000 0xcd00>;
+				reg = <0x0 0x04e00000 0x0 0xcd00>;
 				interrupts = <GIC_SPI 255 IRQ_TYPE_LEVEL_HIGH>;
 				iommus = <&apps_smmu 0x100 0x0>;
 				phys = <&hsusb_phy1>;
@@ -1238,18 +1241,31 @@ usb3_dwc3: usb@4e00000 {
 				snps,dis-u1-entry-quirk;
 				snps,dis-u2-entry-quirk;
 				maximum-speed = "high-speed";
-				dr_mode = "peripheral";
+
+				usb-role-switch;
+
+				ports {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					port@0 {
+						reg = <0>;
+
+						usb_dwc3_hs: endpoint {
+						};
+					};
+				};
 			};
 		};
 
 		sram@4690000 {
 			compatible = "qcom,rpm-stats";
-			reg = <0x04690000 0x10000>;
+			reg = <0x0 0x04690000 0x0 0x10000>;
 		};
 
 		mdss: display-subsystem@5e00000 {
 			compatible = "qcom,sm6125-mdss";
-			reg = <0x05e00000 0x1000>;
+			reg = <0x0 0x05e00000 0x0 0x1000>;
 			reg-names = "mdss";
 
 			interrupts = <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>;
@@ -1269,16 +1285,16 @@ mdss: display-subsystem@5e00000 {
 
 			iommus = <&apps_smmu 0x400 0x0>;
 
-			#address-cells = <1>;
-			#size-cells = <1>;
+			#address-cells = <2>;
+			#size-cells = <2>;
 			ranges;
 
 			status = "disabled";
 
 			mdss_mdp: display-controller@5e01000 {
 				compatible = "qcom,sm6125-dpu";
-				reg = <0x05e01000 0x83208>,
-				      <0x05eb0000 0x3000>;
+				reg = <0x0 0x05e01000 0x0 0x83208>,
+				      <0x0 0x05eb0000 0x0 0x3000>;
 				reg-names = "mdp", "vbif";
 
 				interrupt-parent = <&mdss>;
@@ -1348,7 +1364,7 @@ opp-400000000 {
 
 			mdss_dsi0: dsi@5e94000 {
 				compatible = "qcom,sm6125-dsi-ctrl", "qcom,mdss-dsi-ctrl";
-				reg = <0x05e94000 0x400>;
+				reg = <0x0 0x05e94000 0x0 0x400>;
 				reg-names = "dsi_ctrl";
 
 				interrupt-parent = <&mdss>;
@@ -1417,9 +1433,9 @@ opp-187500000 {
 
 			mdss_dsi0_phy: phy@5e94400 {
 				compatible = "qcom,sm6125-dsi-phy-14nm";
-				reg = <0x05e94400 0x100>,
-				      <0x05e94500 0x300>,
-				      <0x05e94800 0x188>;
+				reg = <0x0 0x05e94400 0x0 0x100>,
+				      <0x0 0x05e94500 0x0 0x300>,
+				      <0x0 0x05e94800 0x0 0x188>;
 				reg-names = "dsi_phy",
 					    "dsi_phy_lane",
 					    "dsi_pll";
@@ -1441,7 +1457,7 @@ mdss_dsi0_phy: phy@5e94400 {
 
 		dispcc: clock-controller@5f00000 {
 			compatible = "qcom,sm6125-dispcc";
-			reg = <0x05f00000 0x20000>;
+			reg = <0x0 0x05f00000 0x0 0x20000>;
 
 			clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>,
 				 <&mdss_dsi0_phy DSI_BYTE_PLL_CLK>,
@@ -1470,7 +1486,7 @@ dispcc: clock-controller@5f00000 {
 
 		apps_smmu: iommu@c600000 {
 			compatible = "qcom,sm6125-smmu-500", "qcom,smmu-500", "arm,mmu-500";
-			reg = <0x0c600000 0x80000>;
+			reg = <0x0 0x0c600000 0x0 0x80000>;
 			interrupts = <GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>,
@@ -1544,74 +1560,74 @@ apps_smmu: iommu@c600000 {
 		apcs_glb: mailbox@f111000 {
 			compatible = "qcom,sm6125-apcs-hmss-global",
 				     "qcom,msm8994-apcs-kpss-global";
-			reg = <0x0f111000 0x1000>;
+			reg = <0x0 0x0f111000 0x0 0x1000>;
 
 			#mbox-cells = <1>;
 		};
 
 		timer@f120000 {
 			compatible = "arm,armv7-timer-mem";
-			#address-cells = <1>;
+			#address-cells = <2>;
 			#size-cells = <1>;
-			ranges;
-			reg = <0x0f120000 0x1000>;
+			reg = <0x0 0x0f120000 0x0 0x1000>;
+			ranges = <0x0 0x0 0x0 0x0 0x20000000>;
 			clock-frequency = <19200000>;
 
 			frame@f121000 {
 				frame-number = <0>;
 				interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
-				reg = <0x0f121000 0x1000>,
-				      <0x0f122000 0x1000>;
+				reg = <0x0 0x0f121000 0x1000>,
+				      <0x0 0x0f122000 0x1000>;
 			};
 
 			frame@f123000 {
 				frame-number = <1>;
 				interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>;
-				reg = <0x0f123000 0x1000>;
+				reg = <0x0 0x0f123000 0x1000>;
 				status = "disabled";
 			};
 
 			frame@f124000 {
 				frame-number = <2>;
 				interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>;
-				reg = <0x0f124000 0x1000>;
+				reg = <0x0 0x0f124000 0x1000>;
 				status = "disabled";
 			};
 
 			frame@f125000 {
 				frame-number = <3>;
 				interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
-				reg = <0x0f125000 0x1000>;
+				reg = <0x0 0x0f125000 0x1000>;
 				status = "disabled";
 			};
 
 			frame@f126000 {
 				frame-number = <4>;
 				interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
-				reg = <0x0f126000 0x1000>;
+				reg = <0x0 0x0f126000 0x1000>;
 				status = "disabled";
 			};
 
 			frame@f127000 {
 				frame-number = <5>;
 				interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
-				reg = <0x0f127000 0x1000>;
+				reg = <0x0 0x0f127000 0x1000>;
 				status = "disabled";
 			};
 
 			frame@f128000 {
 				frame-number = <6>;
 				interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
-				reg = <0x0f128000 0x1000>;
+				reg = <0x0 0x0f128000 0x1000>;
 				status = "disabled";
 			};
 		};
 
 		intc: interrupt-controller@f200000 {
 			compatible = "arm,gic-v3";
-			reg = <0x0f200000 0x20000>,
-			      <0x0f300000 0x100000>;
+			reg = <0x0 0x0f200000 0x0 0x20000>,
+			      <0x0 0x0f300000 0x0 0x100000>;
 			#interrupt-cells = <3>;
 			interrupt-controller;
 			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm64/boot/dts/qcom/sm6350.dtsi b/arch/arm64/boot/dts/qcom/sm6350.dtsi
index d6adf68563cb0..51e0cb5cce66f 100644
--- a/arch/arm64/boot/dts/qcom/sm6350.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm6350.dtsi
@@ -1307,6 +1307,9 @@ ipa: ipa@1e40000 {
 			qcom,smem-state-names = "ipa-clock-enabled-valid",
 						"ipa-clock-enabled";
 
+			qcom,gsi-loader = "self";
+			memory-region = <&pil_ipa_fw_mem>;
+
 			sram = <&ipa_modem_tables>;
 
 			status = "disabled";
@@ -1527,6 +1530,9 @@ gpu: gpu@3d00000 {
 			iommus = <&adreno_smmu 0>;
 			operating-points-v2 = <&gpu_opp_table>;
 			qcom,gmu = <&gmu>;
+			interconnects = <&gem_noc MASTER_GRAPHICS_3D QCOM_ICC_TAG_ALWAYS
+					 &clk_virt SLAVE_EBI_CH0 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "gfx-mem";
 			nvmem-cells = <&gpu_speed_bin>;
 			nvmem-cell-names = "speed_bin";
 			#cooling-cells = <2>;
@@ -1688,6 +1694,9 @@ mpss: remoteproc@4080000 {
 					<&rpmhpd SM6350_MSS>;
 			power-domain-names = "cx", "mss";
 
+			interconnects = <&clk_virt MASTER_LLCC QCOM_ICC_TAG_ALWAYS
+					 &clk_virt SLAVE_EBI_CH0 QCOM_ICC_TAG_ALWAYS>;
+
 			memory-region = <&pil_modem_mem>;
 
 			qcom,qmp = <&aoss_qmp>;
@@ -1727,6 +1736,9 @@ cdsp: remoteproc@8300000 {
 					<&rpmhpd SM6350_MX>;
 			power-domain-names = "cx", "mx";
 
+			interconnects = <&compute_noc MASTER_NPU QCOM_ICC_TAG_ALWAYS
+					 &compute_noc SLAVE_CDSP_GEM_NOC QCOM_ICC_TAG_ALWAYS>;
+
 			memory-region = <&pil_cdsp_mem>;
 
 			qcom,qmp = <&aoss_qmp>;
@@ -2557,7 +2569,7 @@ dispcc: clock-controller@af00000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sm6350-pdc", "qcom,pdc";
-			reg = <0x0 0x0b220000 0x0 0x30000>, <0x0 0x17c000f0 0x0 0x64>;
+			reg = <0x0 0x0b220000 0x0 0x10000>, <0x0 0x17c000f0 0x0 0x64>;
 			qcom,pdc-ranges = <0 480 94>, <94 609 31>,
 					  <125 63 1>, <126 655 12>, <138 139 15>;
 			#interrupt-cells = <2>;
@@ -2811,6 +2823,11 @@ sram@14680000 {
 			ipa_modem_tables: modem-tables@28000 {
 				reg = <0x28000 0x2000>;
 			};
+
+			pil-reloc@2a94c {
+				compatible = "qcom,pil-reloc-info";
+				reg = <0x2a94c 0xc8>;
+			};
 		};
 
 		apps_smmu: iommu@15000000 {
diff --git a/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts b/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts
index 23f950067a08b..711a120753ad7 100644
--- a/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts
+++ b/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts
@@ -48,7 +48,7 @@ chosen {
 
 		stdout-path = "serial0:115200n8";
 
-		framebuffer0: framebuffer@a000000 {
+		framebuffer0: framebuffer@a0000000 {
 			compatible = "simple-framebuffer";
 			reg = <0 0xa0000000 0 (2340 * 1080 * 4)>;
 			width = <1080>;
@@ -711,9 +711,8 @@ vreg_l7p: ldo7 {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&pil_ipa_fw_mem>;
 	firmware-name = "qcom/sm7225/fairphone4/ipa_fws.mbn";
+
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sm7325-motorola-dubai-common.dtsi b/arch/arm64/boot/dts/qcom/sm7325-motorola-dubai-common.dtsi
new file mode 100644
index 0000000000000..3860ab8508a09
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sm7325-motorola-dubai-common.dtsi
@@ -0,0 +1,1454 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (c) 2026, Val Packett <val@packett.cool>
+ */
+
+/dts-v1/;
+
+/* PM7250B is configured to use SID8/9 */
+#define PM7250B_SID 8
+#define PM7250B_SID1 9
+
+#include <dt-bindings/arm/qcom,ids.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/iio/qcom,spmi-adc7-pm7325.h>
+#include <dt-bindings/iio/qcom,spmi-adc7-pmk8350.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+
+#include "sm7325.dtsi"
+#include "pm7325.dtsi"
+#include "pm7250b.dtsi"
+#include "pm8350c.dtsi" /* PM7350C */
+#include "pmk8350.dtsi" /* PMK7325 */
+
+/ {
+	chassis-type = "handset";
+
+	aliases {
+		bluetooth0 = &bluetooth;
+		serial0 = &uart5;
+		serial1 = &uart7;
+		wifi0 = &wifi;
+	};
+
+	chosen {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		stdout-path = "serial0:115200n8";
+
+		framebuffer {
+			compatible = "simple-framebuffer";
+			memory-region = <&framebuffer_mem>;
+			width = <1080>;
+			height = <2400>;
+			stride = <(1080 * 4)>;
+			format = "a8r8g8b8";
+
+			clocks = <&gcc GCC_DISP_HF_AXI_CLK>,
+				 <&dispcc DISP_CC_MDSS_MDP_CLK>,
+				 <&dispcc DISP_CC_MDSS_BYTE0_CLK>,
+				 <&dispcc DISP_CC_MDSS_BYTE0_INTF_CLK>,
+				 <&dispcc DISP_CC_MDSS_PCLK0_CLK>,
+				 <&dispcc DISP_CC_MDSS_VSYNC_CLK>;
+			power-domains = <&dispcc DISP_CC_MDSS_CORE_GDSC>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&kypd_vol_up_n>;
+		pinctrl-names = "default";
+
+		key-volume-up {
+			label = "Volume Up";
+			gpios = <&pm7325_gpios 6 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,sm7325-pmic-glink",
+			     "qcom,qcm6490-pmic-glink",
+			     "qcom,pmic-glink";
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		orientation-gpios = <&tlmm 140 GPIO_ACTIVE_HIGH>;
+
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in: endpoint {
+						remote-endpoint = <&usb_1_dwc3_hs>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_sbu: endpoint {
+						remote-endpoint = <&fsa4480_sbu_mux>;
+					};
+				};
+			};
+		};
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ramoops@ae000000 {
+			compatible = "ramoops";
+			reg = <0x0 0xae000000 0x0 0xc0000>;
+			console-size = <0x40000>;
+			mem-type = <2>;
+			pmsg-size = <0x40000>;
+			record-size = <0x3f800>;
+		};
+
+		removed@c0000000 {
+			reg = <0x0 0xc0000000 0x0 0x5100000>;
+			no-map;
+		};
+
+		framebuffer_mem: framebuffer@e1000000 {
+			reg = <0x0 0xe1000000 0x0 (1080 * 2400 * 4)>;
+			no-map;
+		};
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			size = <0x0 0x8000000>;
+			reusable;
+			linux,cma-default;
+		};
+	};
+
+	thermal-zones {
+		cam-flash-thermal {
+			thermal-sensors = <&pmk8350_adc_tm 2>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		chg-skin-thermal {
+			thermal-sensors = <&pm7250b_adc_tm 0>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		chg-thermal {
+			thermal-sensors = <&pmk8350_adc_tm 4>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		conn-thermal {
+			thermal-sensors = <&pm7250b_adc_tm 1>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		pa-1-thermal {
+			thermal-sensors = <&pmk8350_adc_tm 5>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		pa-2-thermal {
+			thermal-sensors = <&pmk8350_adc_tm 6>;
+
+			/* Reports negative temperature. */
+			status = "disabled";
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		quiet-thermal {
+			thermal-sensors = <&pmk8350_adc_tm 1>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		sdm-skin-thermal {
+			thermal-sensors = <&pmk8350_adc_tm 3>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		xo-thermal {
+			thermal-sensors = <&pmk8350_adc_tm 0>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+	};
+
+	// S2B is really ebi.lvl but it's there for supply map completeness sake.
+	vreg_s2b_0p7: smpb2-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_s2b_0p7";
+
+		regulator-min-microvolt = <65535>;
+		regulator-max-microvolt = <65535>;
+		regulator-always-on;
+		vin-supply = <&vph_pwr>;
+	};
+
+	vph_pwr: vph-pwr-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+	};
+
+	wcn6750-pmu {
+		compatible = "qcom,wcn6750-pmu";
+		pinctrl-0 = <&bt_en>;
+		pinctrl-names = "default";
+		vddaon-supply = <&mos_0p95_aon_s10c>;
+		vddasd-supply = <&vreg_l11c_2p8>;
+		vddpmu-supply = <&mos_0p95_dig_s10c>;
+		vddio-supply = <&vdd18_io>;
+		vddrfa0p8-supply = <&vdd09_pmu_rfa_i>;
+		vddrfa1p2-supply = <&vdd13_pmu_rfa_i>;
+		vddrfa1p7-supply = <&vdd19_pmu_rfa_i>;
+		vddrfa2p2-supply = <&vdd22_wlpa_s1c>;
+
+		bt-enable-gpios = <&tlmm 85 GPIO_ACTIVE_HIGH>;
+
+		regulators {
+			vreg_pmu_rfa_cmn: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn";
+			};
+
+			vreg_pmu_aon_0p59: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p59";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p85: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p85";
+			};
+
+			vreg_pmu_btcmx_0p85: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p85";
+			};
+
+			vreg_pmu_rfa_0p8: ldo5 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo6 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p7: ldo7 {
+				regulator-name = "vreg_pmu_rfa_1p7";
+			};
+
+			vreg_pmu_pcie_0p9: ldo8 {
+				regulator-name = "vreg_pmu_pcie_0p9";
+			};
+
+			vreg_pmu_pcie_1p8: ldo9 {
+				regulator-name = "vreg_pmu_pcie_1p8";
+			};
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pm7325-rpmh-regulators";
+		qcom,pmic-id = "b";
+
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s7-supply = <&vph_pwr>;
+		vdd-s8-supply = <&vph_pwr>;
+
+		vdd-l1-l4-l12-l15-supply = <&vreg_s7b_0p952>;
+		vdd-l2-l7-supply = <&vreg_bob>;
+		vdd-l3-supply = <&vreg_s2b_0p7>;
+		vdd-l5-supply = <&vreg_s2b_0p7>;
+		vdd-l6-l9-l10-supply = <&vreg_s8b_1p256>;
+		vdd-l8-supply = <&vreg_s7b_0p952>;
+		vdd-l11-l17-l18-l19-supply = <&vreg_s1b_1p856>;
+		vdd-l13-supply = <&vreg_s7b_0p952>;
+		vdd-l14-l16-supply = <&vreg_s8b_1p256>;
+
+		/*
+		 * S2, L4-L5 are ARCs:
+		 * S2 - ebi.lvl,
+		 * L4 - lmx.lvl,
+		 * l5 - lcx.lvl.
+		 *
+		 * L10 are unused.
+		 */
+
+		vdd19_pmu_rfa_i:
+		vreg_s1b_1p856: smps1 {
+			regulator-name = "vreg_s1b_1p856";
+			regulator-min-microvolt = <1840000>;
+			regulator-max-microvolt = <2040000>;
+		};
+
+		mos_0p95_aon_s10c:
+		mos_0p95_dig_s10c:
+		vdd09_pmu_rfa_i:
+		vreg_s7b_0p952: smps7 {
+			regulator-name = "vreg_s7b_0p952";
+			regulator-min-microvolt = <535000>;
+			regulator-max-microvolt = <1120000>;
+		};
+
+		vdd13_pmu_rfa_i:
+		vreg_s8b_1p256: smps8 {
+			regulator-name = "vreg_s8b_1p256";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1500000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_RET>;
+		};
+
+		vreg_l1b_0p912: ldo1 {
+			regulator-name = "vreg_l1b_0p912";
+			regulator-min-microvolt = <825000>;
+			regulator-max-microvolt = <925000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vdd_a_usbhs_3p1:
+		vreg_l2b_3p072: ldo2 {
+			regulator-name = "vreg_l2b_3p072";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3b_0p6: ldo3 {
+			regulator-name = "vreg_l3b_0p6";
+			regulator-min-microvolt = <312000>;
+			regulator-max-microvolt = <910000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vdd_a_dsi_0_1p2:
+		vdd_a_ufs_0_1p2:
+		vreg_l6b_1p2: ldo6 {
+			regulator-name = "vreg_l6b_1p2";
+			regulator-min-microvolt = <1140000>;
+			regulator-max-microvolt = <1260000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b_2p96: ldo7 {
+			regulator-name = "vreg_l7b_2p96";
+			regulator-min-microvolt = <2400000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b_0p904: ldo8 {
+			regulator-name = "vreg_l8b_0p904";
+			regulator-min-microvolt = <870000>;
+			regulator-max-microvolt = <970000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b_1p2: ldo9 {
+			regulator-name = "vreg_l9b_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11b_1p776: ldo11 {
+			regulator-name = "vreg_l11b_1p776";
+			regulator-min-microvolt = <1504000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b_0p8: ldo12 {
+			regulator-name = "vreg_l12b_0p8";
+			regulator-min-microvolt = <751000>;
+			regulator-max-microvolt = <824000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13b_0p8: ldo13 {
+			regulator-name = "vreg_l13b_0p8";
+			regulator-min-microvolt = <530000>;
+			regulator-max-microvolt = <824000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l14b_1p2: ldo14 {
+			regulator-name = "vreg_l14b_1p2";
+			regulator-min-microvolt = <1080000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_0p88: ldo15 {
+			regulator-name = "vreg_l15b_0p88";
+			regulator-min-microvolt = <765000>;
+			regulator-max-microvolt = <1020000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l16b_1p2: ldo16 {
+			regulator-name = "vreg_l16b_1p2";
+			regulator-min-microvolt = <1100000>;
+			regulator-max-microvolt = <1300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b_1p8: ldo17 {
+			regulator-name = "vreg_l17b_1p8";
+			regulator-min-microvolt = <1700000>;
+			regulator-max-microvolt = <1900000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l18b_1p8: ldo18 {
+			regulator-name = "vreg_l18b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+			regulator-boot-on;
+		};
+
+		vdd18_io:
+		vreg_l19b_1p8: ldo19 {
+			regulator-name = "vreg_l19b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pm8350c-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s7-supply = <&vph_pwr>;
+		vdd-s9-supply = <&vph_pwr>;
+		vdd-s10-supply = <&vph_pwr>;
+
+		vdd-l1-l12-supply = <&vreg_s1b_1p856>;
+		vdd-l2-l8-supply = <&vreg_s1b_1p856>;
+		vdd-l3-l4-l5-l7-l13-supply = <&vreg_bob>;
+		vdd-l6-l9-l11-supply = <&vreg_bob>;
+		vdd-l10-supply = <&vreg_s7b_0p952>;
+
+		vdd-bob-supply = <&vph_pwr>;
+
+		/*
+		 * S2, S5, S7, S10 are ARCs:
+		 * S2 - cx.lvl,
+		 * S5 - mss.lvl,
+		 * S7 - gfx.lvl,
+		 * S10 - mx.lvl.
+		 */
+
+		vdd22_wlpa_s1c:
+		vreg_s1c_2p2: smps1 {
+			regulator-name = "vreg_s1c_2p2";
+			regulator-min-microvolt = <2190000>;
+			regulator-max-microvolt = <2210000>;
+		};
+
+		vreg_s9c_0p676: smps9 {
+			regulator-name = "vreg_s9c_0p676";
+			regulator-min-microvolt = <1010000>;
+			regulator-max-microvolt = <1170000>;
+		};
+
+		vdd_a_usbhs_1p8:
+		vdd_qfprom:
+		vreg_l1c_1p8: ldo1 {
+			regulator-name = "vreg_l1c_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1980000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		iovdd_ts:
+		vreg_l2c_1p8: ldo2 {
+			regulator-name = "vreg_l2c_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vdd_ts:
+		vreg_l3c_3p0: ldo3 {
+			regulator-name = "vreg_l3c_3p0";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4c_1p8_3p0: ldo4 {
+			regulator-name = "vreg_l4c_1p8_3p0";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5c_1p8_3p0: ldo5 {
+			regulator-name = "vreg_l5c_1p8_3p0";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6c_2p96: ldo6 {
+			regulator-name = "vreg_l6c_2p96";
+			regulator-min-microvolt = <1650000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7c_3p0: ldo7 {
+			regulator-name = "vreg_l7c_3p0";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8c_1p8: ldo8 {
+			regulator-name = "vreg_l8c_1p8";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9c_3p3: ldo9 {
+			regulator-name = "vreg_l9c_2p96";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vdd_a_dsi_0_0p9:
+		vdd_a_ufs_0_core:
+		vdd_a_usbhs_core:
+		vreg_l10c_0p88: ldo10 {
+			regulator-name = "vreg_l10c_0p88";
+			regulator-min-microvolt = <720000>;
+			regulator-max-microvolt = <1050000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11c_2p8: ldo11 {
+			regulator-name = "vreg_l11c_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		disp_iovcc_1p8:
+		vreg_l12c_1p8: ldo12 {
+			regulator-name = "vreg_l12c_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		disp_vci_3p0:
+		vreg_l13c_3p0: ldo13 {
+			regulator-name = "vreg_l13c_3p0";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		audio_sw_vcc:
+		vreg_bob: bob {
+			regulator-name = "vreg_bob";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+};
+
+&gcc {
+	protected-clocks = <GCC_CFG_NOC_LPASS_CLK>,
+			   <GCC_MSS_CFG_AHB_CLK>,
+			   <GCC_MSS_OFFLINE_AXI_CLK>,
+			   <GCC_MSS_Q6SS_BOOT_CLK_SRC>,
+			   <GCC_MSS_Q6_MEMNOC_AXI_CLK>,
+			   <GCC_MSS_SNOC_AXI_CLK>,
+			   <GCC_QSPI_CNOC_PERIPH_AHB_CLK>,
+			   <GCC_QSPI_CORE_CLK>,
+			   <GCC_QSPI_CORE_CLK_SRC>,
+			   <GCC_SEC_CTRL_CLK_SRC>,
+			   <GCC_WPSS_AHB_BDG_MST_CLK>,
+			   <GCC_WPSS_AHB_CLK>,
+			   <GCC_WPSS_RSCP_CLK>;
+};
+
+&gpi_dma0 {
+	status = "okay";
+};
+
+&gpi_dma1 {
+	status = "okay";
+};
+
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/sm7325/motorola/dubai/a660_zap.mbn";
+};
+
+&i2c4 {
+	clock-frequency = <100000>;
+
+	status = "okay";
+
+	typec-mux@42 {
+		compatible = "fcs,fsa4480";
+		reg = <0x42>;
+
+		interrupts-extended = <&tlmm 6 IRQ_TYPE_LEVEL_LOW>;
+
+		vcc-supply = <&audio_sw_vcc>;
+
+		mode-switch;
+		orientation-switch;
+
+		port {
+			fsa4480_sbu_mux: endpoint {
+				remote-endpoint = <&pmic_glink_sbu>;
+			};
+		};
+	};
+};
+
+&ipa {
+	firmware-name = "qcom/sm7325/motorola/dubai/yupik_ipa_fws.mbn";
+	memory-region = <&ipa_fw_mem>;
+
+	qcom,gsi-loader = "self";
+
+	status = "okay";
+};
+
+&pm7250b_adc {
+	channel@4e {
+		reg = <ADC5_AMUX_THM2_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "chg_skin_therm";
+	};
+
+	channel@4f {
+		reg = <ADC5_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "conn_therm";
+	};
+};
+
+&pm7250b_adc_tm {
+	status = "okay";
+
+	chg-skin-therm@0 {
+		reg = <0>;
+		io-channels = <&pm7250b_adc ADC5_AMUX_THM2_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	conn-therm@1 {
+		reg = <1>;
+		io-channels = <&pm7250b_adc ADC5_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+};
+
+&pm7250b_gpios {
+	gpio-line-names = "FG_ALERT_N", /* GPIO_1 */
+			  "SLAVECP_INT",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "DOUBLER_STAT",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "NC", /* GPIO_10 */
+			  "NC",
+			  "NC";
+};
+
+&pm7325_gpios {
+	gpio-line-names = "PA_THERM1", /* GPIO_1 */
+			  "NC",
+			  "NC",
+			  "PA_THERM2",
+			  "CBL_PWR_N",
+			  "KYPD_VOL_UP_N",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "NC"; /* GPIO_10 */
+
+	kypd_vol_up_n: kypd-volp-n-state {
+		pins = "gpio6";
+		function = PMIC_GPIO_FUNC_NORMAL;
+		bias-pull-up;
+		input-enable;
+		power-source = <1>;
+	};
+};
+
+&pm8350c_gpios {
+	gpio-line-names = "NC", /* GPIO_1 */
+			  "NC",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "NC"; /* GPIO_9 */
+};
+
+&pm8350c_flash {
+	status = "okay";
+
+	led-0 {
+		function = LED_FUNCTION_FLASH;
+		color = <LED_COLOR_ID_WHITE>;
+		led-sources = <1>, <4>;
+		led-max-microamp = <500000>;
+		flash-max-microamp = <1500000>;
+		flash-max-timeout-us = <400000>;
+	};
+
+	led-1 {
+		function = LED_FUNCTION_FLASH;
+		color = <LED_COLOR_ID_WHITE>;
+		led-sources = <2>, <3>;
+		led-max-microamp = <500000>;
+		flash-max-microamp = <1500000>;
+		flash-max-timeout-us = <400000>;
+	};
+};
+
+&pmk8350_adc_tm {
+	status = "okay";
+
+	xo-therm@0 {
+		reg = <0>;
+		io-channels = <&pmk8350_vadc PMK8350_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	quiet-therm@1 {
+		reg = <1>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	cam-flash-therm@2 {
+		reg = <2>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_AMUX_THM2_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	sdm-skin-therm@3 {
+		reg = <3>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	chg-therm@4 {
+		reg = <4>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_AMUX_THM4_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	pa-therm-1@5 {
+		reg = <5>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_GPIO1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	pa-therm-2@6 {
+		reg = <6>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_GPIO4_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+};
+
+&pmk8350_gpios {
+	gpio-line-names = "NC", /* GPIO_0 */
+			  "NC",
+			  "TP_PMK_GPIO_3",
+			  "PMK_OPTION"; /* GPIO_4 */
+};
+
+&pmk8350_rtc {
+	status = "okay";
+};
+
+&pmk8350_vadc {
+	status = "okay";
+
+	channel@44 {
+		reg = <PMK8350_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pmk8350_xo_therm";
+	};
+
+	channel@144 {
+		reg = <PM7325_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_quiet_therm";
+	};
+
+	channel@145 {
+		reg = <PM7325_ADC7_AMUX_THM2_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_cam_flash_therm";
+	};
+
+	channel@146 {
+		reg = <PM7325_ADC7_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_sdm_skin_therm";
+	};
+
+	channel@147 {
+		reg = <PM7325_ADC7_AMUX_THM4_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_chg_therm";
+	};
+
+	channel@14a {
+		reg = <PM7325_ADC7_GPIO1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_pa_therm1";
+	};
+
+	channel@14d {
+		reg = <PM7325_ADC7_GPIO4_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_pa_therm2";
+	};
+};
+
+&pon_pwrkey {
+	status = "okay";
+};
+
+&pon_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+
+	status = "okay";
+};
+
+&qfprom {
+	vcc-supply = <&vdd_qfprom>;
+};
+
+&qup_spi13_cs {
+	drive-strength = <6>;
+	bias-pull-down;
+};
+
+&qup_spi13_data_clk {
+	drive-strength = <6>;
+	bias-pull-down;
+};
+
+&qup_uart5_rx {
+	drive-strength = <2>;
+	bias-disable;
+};
+
+&qup_uart5_tx {
+	drive-strength = <2>;
+	bias-disable;
+};
+
+&qup_uart7_cts {
+	/*
+	 * Configure a bias-bus-hold on CTS to lower power
+	 * usage when Bluetooth is turned off. Bus hold will
+	 * maintain a low power state regardless of whether
+	 * the Bluetooth module drives the pin in either
+	 * direction or leaves the pin fully unpowered.
+	 */
+	bias-bus-hold;
+};
+
+&qup_uart7_rts {
+	/* We'll drive RTS, so no pull */
+	drive-strength = <2>;
+	bias-disable;
+};
+
+&qup_uart7_rx {
+	/*
+	 * Configure a pull-up on RX. This is needed to avoid
+	 * garbage data when the TX pin of the Bluetooth module is
+	 * in tri-state (module powered off or not driving the
+	 * signal yet).
+	 */
+	bias-pull-up;
+};
+
+&qup_uart7_tx {
+	/* We'll drive TX, so no pull */
+	drive-strength = <2>;
+	bias-disable;
+};
+
+&qupv3_id_0 {
+	status = "okay";
+};
+
+&qupv3_id_1 {
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/sm7325/motorola/dubai/adsp.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/sm7325/motorola/dubai/cdsp.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_mpss {
+	firmware-name = "qcom/sm7325/motorola/dubai/modem.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_wpss {
+	firmware-name = "qcom/sm7325/motorola/dubai/wpss.mbn";
+
+	status = "okay";
+};
+
+&rmtfs_mem {
+	qcom,vmid = <QCOM_SCM_VMID_MSS_MSA>,
+		    <QCOM_SCM_VMID_NAV>;
+};
+
+&spi13 {
+	status = "okay";
+
+	touchscreen@0 {
+		compatible = "goodix,gt9916";
+		reg = <0>;
+
+		interrupts-extended = <&tlmm 81 IRQ_TYPE_LEVEL_LOW>;
+
+		reset-gpios = <&tlmm 105 GPIO_ACTIVE_LOW>;
+
+		avdd-supply = <&vdd_ts>;
+		vddio-supply = <&iovdd_ts>;
+
+		spi-max-frequency = <1000000>;
+
+		touchscreen-size-x = <1080>;
+		touchscreen-size-y = <2400>;
+
+		pinctrl-0 = <&ts_int_n>, <&ts_reset_n>;
+		pinctrl-names = "default";
+	};
+};
+
+&tlmm {
+	gpio-line-names = "NC", /* GPIO_0 */
+			  "NC",
+			  "NC",
+			  "NC",
+			  "SMARTPA_I2C_SDA",
+			  "SMARTPA_I2C_SCK",
+			  "AUD_4480_DET",
+			  "AUD_4480_INT",
+			  "FASTCHARGE_I2C_SDA",
+			  "FASTCHARGE_I2C_SCL",
+			  "SM_GPSLNA_EN_GPIO10", /* GPIO_10 */
+			  "NC",
+			  "PERI_I2C_SDA",
+			  "PERI_I2C_SCL",
+			  "NC",
+			  "NC",
+			  "APPS_I2C_SDA",
+			  "APPS_I2C_SCL",
+			  "NC",
+			  "DISP_LDO_EN",
+			  "CAM_FW_RST_N", /* GPIO_20 */
+			  "CAM_RM_RST_N",
+			  "DBG_UART_TXD",
+			  "DBG_UART_RXD",
+			  "TOPSB_INT_N",
+			  "MCAM_LDO_EN",
+			  "HOST2WLAN_SOL",
+			  "WLAN2HOST_SOL",
+			  "UART_BT_RTS_AP_CTS",
+			  "UART_BT_CTS_AP_RTS",
+			  "UART_BT_RX_AP_TX", /* GPIO_30 */
+			  "UART_BT_TX_AP_RX",
+			  "FRONT_60M_INT",
+			  "FRONT_60M_RST",
+			  "FP_INT",
+			  "FP_RST",
+			  "NFC_I2C_SDA",
+			  "NFC_I2C_SCL",
+			  "NFC_EN",
+			  "NFC_CLK_REQ",
+			  "NFC_SE_SPI_NREST", /* GPIO_40 */
+			  "NFC_IRQ",
+			  "RF_LDO_GPIO42_EN",
+			  "GOOGLE_KEY_INT", /* no such key exists */
+			  "DISP_RST_N",
+			  "SB_INT_N",
+			  "HAP_RST_N",
+			  "HAP_INT",
+			  "ESE_SPI_MISO",
+			  "ESE_SPI_MOSI",
+			  "ESE_SPI_CLK", /* GPIO_50 */
+			  "ESE_SPI_CS",
+			  "TP_SPI_MISO",
+			  "TP_SPI_MOSI",
+			  "TP_SPI_CLK",
+			  "TP_SPI_CS_N",
+			  "FP_SPI_MISO",
+			  "FP_SPI_MOSI",
+			  "FP_SPI_CLK",
+			  "FP_SPI_CS_N",
+			  "HW_ID_1", /* GPIO_60 */
+			  "HW_ID_2",
+			  "CAM_PMU_EN",
+			  "WIDE_DVDD_LDO_EN",
+			  "CAM_MCLK0",
+			  "CAM_MCLK1",
+			  "CAM_MCLK2",
+			  "CAM_MCLK3",
+			  "NC",
+			  "CCI_I2C_SDA0",
+			  "CCI_I2C_SCL0", /* GPIO_70 */
+			  "CCI_I2C_SDA1",
+			  "CCI_I2C_SCL1",
+			  "CCI_I2C_SDA2",
+			  "CCI_I2C_SCL2",
+			  "CCI_I2C_SDA3",
+			  "CCI_I2C_SCL3",
+			  "CAM_RT_RST_N",
+			  "CAM_RU_RST_N",
+			  "FCAM_LDO_EN",
+			  "DISP_TE", /* GPIO_80 */
+			  "TP_INT_N",
+			  "FORCED_USB_BOOT",
+			  "SM_CDC_RST_N",
+			  "WLAN_EN",
+			  "BT_EN",
+			  "WCN_SW_CTRL",
+			  "PCIE0_RESET_N",
+			  "PCIE0_CLK_REQ_N",
+			  "PCIE0_WAKE_N",
+			  "MOS_AS_EN", /* GPIO_90 */
+			  "NC",
+			  "",
+			  "NC",
+			  "BT_FM_SLIMBUS_CLK",
+			  "BT_FM_SLIMBUS_DATA",
+			  "NC",
+			  "RFCONN_DET_1",
+			  "RF_CON_DET_2",
+			  "RF_CON_DET_3",
+			  "NC", /* GPIO_100 */
+			  "NC",
+			  "NC",
+			  "ACCEL_INT",
+			  "NC",
+			  "TP_RST_N",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "UIM2_DATA",
+			  "UIM2_CLK", /* GPIO_110 */
+			  "UIM2_RESET",
+			  "UIM2_PRESENT",
+			  "UIM1_DATA",
+			  "UIM1_CLK",
+			  "UIM1_RESET",
+			  "UIM1_PRESENT",
+			  "SM_RFFE0_CLK",
+			  "SM_RFFE0_DATA",
+			  "SM_RFFE1_CLK",
+			  "SM_RFFE1_DATA", /* GPIO_120 */
+			  "PA_MUTING",
+			  "SM_GRFC5",
+			  "LAA_RX",
+			  "SM_GRFC7",
+			  "SM_RFFE4_CLK",
+			  "SM_RFFE4_DATA",
+			  "WLAN_COEX_UART_RX",
+			  "WLAN_COEX_UART_TX",
+			  "NC",
+			  "NC", /* GPIO_130 */
+			  "SM_GRFC12",
+			  "NC",
+			  "QLINK0_REQUEST",
+			  "QLINK0_ENABLE",
+			  "QLINK0_WMSS_RESET_N",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "USB_CC_DIR", /* GPIO_140 */
+			  "SAR_INT_N",
+			  "PROX_INT_N",
+			  "",
+			  "SM_SWR_TX_CLK",
+			  "SM_SWR_TX_DATA0",
+			  "SM_SWR_TX_DATA1",
+			  "SM_SWR_RX_CLK",
+			  "SM_SWR_RX_DATA0",
+			  "SM_SWR_RX_DATA1",
+			  "NC", /* GPIO_150 */
+			  "NC",
+			  "NC",
+			  "NC",
+			  "SB_MI2S_SCK",
+			  "SB_MI2S_WS",
+			  "SB_MI2S_RXDAT_AP_TX",
+			  "SB_MI2S_TXDAT_AP_RX",
+			  "NC",
+			  "SNS_I3C0_SDA",
+			  "SNS_I3C0_SCL", /* GPIO_160 */
+			  "SSC_I2C4_SDA",
+			  "SSC_I2C4_SCL",
+			  "MAG_I2C_SDA",
+			  "MAG_I2C_SCL",
+			  "NC",
+			  "NC",
+			  "",
+			  "",
+			  "",
+			  "", /* GPIO_170 */
+			  "SSC_BT_UART4_TX",
+			  "SSC_BT_UART4_RX",
+			  "NC",
+			  "NC";
+	gpio-reserved-ranges = <48 4>, /* SPI (eSE - embedded Secure Element) */
+			       <56 4>; /* SPI (fingerprint reader) */
+
+	qup_uart7_sleep_cts: qup-uart7-sleep-cts-state {
+		pins = "gpio28";
+		function = "gpio";
+		/*
+		 * Configure a bias-bus-hold on CTS to lower power
+		 * usage when Bluetooth is turned off. Bus hold will
+		 * maintain a low power state regardless of whether
+		 * the Bluetooth module drives the pin in either
+		 * direction or leaves the pin fully unpowered.
+		 */
+		bias-bus-hold;
+	};
+
+	qup_uart7_sleep_rts: qup-uart7-sleep-rts-state {
+		pins = "gpio29";
+		function = "gpio";
+		/*
+		 * Configure pull-down on RTS. As RTS is active low
+		 * signal, pull it low to indicate the BT SoC that it
+		 * can wakeup the system anytime from suspend state by
+		 * pulling RX low (by sending wakeup bytes).
+		 */
+		bias-pull-down;
+	};
+
+	qup_uart7_sleep_tx: qup-uart7-sleep-tx-state {
+		pins = "gpio30";
+		function = "gpio";
+		/*
+		 * Configure pull-up on TX when it isn't actively driven
+		 * to prevent BT SoC from receiving garbage during sleep.
+		 */
+		bias-pull-up;
+	};
+
+	qup_uart7_sleep_rx: qup-uart7-sleep-rx-state {
+		pins = "gpio31";
+		function = "gpio";
+		/*
+		 * Configure a pull-up on RX. This is needed to avoid
+		 * garbage data when the TX pin of the Bluetooth module
+		 * is floating which may cause spurious wakeups.
+		 */
+		bias-pull-up;
+	};
+
+	oled_reset_n: oled-reset-n-state {
+		pins = "gpio44";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
+
+	aw86224_reset_default: aw86224-reset-default-state {
+		pins = "gpio46";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
+	aw86224_int_default: aw86224-int-default-state {
+		pins = "gpio47";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	mdp_vsync_p: mdp-vsync-p-state {
+		pins = "gpio80";
+		function = "mdp_vsync";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
+	ts_int_n: ts-int-n-state {
+		pins = "gpio81";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-up;
+	};
+
+	bt_en: bt-en-state {
+		pins = "gpio85";
+		function = "gpio";
+		output-low;
+		bias-disable;
+	};
+
+	ts_reset_n: ts-int-n-state {
+		pins = "gpio105";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-up;
+	};
+};
+
+&uart5 {
+	status = "okay";
+};
+
+&uart7 {
+	/delete-property/ interrupts;
+	interrupts-extended = <&intc GIC_SPI 608 IRQ_TYPE_LEVEL_HIGH>,
+			      <&tlmm 31 IRQ_TYPE_EDGE_FALLING>;
+	pinctrl-1 = <&qup_uart7_sleep_cts>,
+		    <&qup_uart7_sleep_rts>,
+		    <&qup_uart7_sleep_tx>,
+		    <&qup_uart7_sleep_rx>;
+	pinctrl-names = "default",
+			"sleep";
+
+	status = "okay";
+
+	bluetooth: bluetooth {
+		compatible = "qcom,wcn6750-bt";
+
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddbtcmx-supply = <&vreg_pmu_btcmx_0p85>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+
+		max-speed = <3200000>;
+
+		qcom,local-bd-address-broken;
+	};
+};
+
+&ufs_mem_hc {
+	reset-gpios = <&tlmm 175 GPIO_ACTIVE_LOW>;
+
+	vcc-supply = <&vreg_l7b_2p96>;
+	vcc-max-microamp = <800000>;
+
+	vccq-supply = <&vreg_l9b_1p2>;
+	vccq-max-microamp = <900000>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vdd_a_ufs_0_core>;
+	vdda-pll-supply = <&vdd_a_ufs_0_1p2>;
+
+	status = "okay";
+};
+
+&usb_1 {
+	/* USB 2.0 only */
+	qcom,select-utmi-as-pipe-clk;
+	maximum-speed = "high-speed";
+
+	/* Remove USB3 phy */
+	phys = <&usb_1_hsphy>;
+	phy-names = "usb2-phy";
+
+	status = "okay";
+};
+
+&usb_1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in>;
+};
+
+&usb_1_hsphy {
+	vdda-pll-supply = <&vdd_a_usbhs_core>;
+	vdda18-supply = <&vdd_a_usbhs_1p8>;
+	vdda33-supply = <&vdd_a_usbhs_3p1>;
+
+	status = "okay";
+};
+
+&venus {
+	firmware-name = "qcom/sm7325/motorola/dubai/vpu20_1v.mbn";
+
+	status = "okay";
+};
+
+&wifi {
+	qcom,calibration-variant = "Motorola_dubai";
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/sm7325-motorola-dubai-csot.dts b/arch/arm64/boot/dts/qcom/sm7325-motorola-dubai-csot.dts
new file mode 100644
index 0000000000000..d53a729f71dab
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sm7325-motorola-dubai-csot.dts
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (c) 2026, Val Packett <val@packett.cool>
+ */
+
+/dts-v1/;
+
+#include "sm7325-motorola-dubai-common.dtsi"
+
+/ {
+	model = "Motorola Edge 30 (CSOT)";
+	compatible = "motorola,dubai", "qcom,sm7325";
+};
diff --git a/arch/arm64/boot/dts/qcom/sm7325-motorola-dubai-tianma.dts b/arch/arm64/boot/dts/qcom/sm7325-motorola-dubai-tianma.dts
new file mode 100644
index 0000000000000..7b843c407f88e
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sm7325-motorola-dubai-tianma.dts
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (c) 2026, Val Packett <val@packett.cool>
+ */
+
+/dts-v1/;
+
+#include "sm7325-motorola-dubai-common.dtsi"
+
+/ {
+	model = "Motorola Edge 30 (Tianma)";
+	compatible = "motorola,dubai", "qcom,sm7325";
+};
diff --git a/arch/arm64/boot/dts/qcom/sm7325-motorola-dubai.dts b/arch/arm64/boot/dts/qcom/sm7325-motorola-dubai.dts
index 3c836e196b19c..d94571c05a59a 100644
--- a/arch/arm64/boot/dts/qcom/sm7325-motorola-dubai.dts
+++ b/arch/arm64/boot/dts/qcom/sm7325-motorola-dubai.dts
@@ -727,9 +727,6 @@ fsa4480_sbu_mux: endpoint {
 
 &ipa {
 	firmware-name = "qcom/sm7325/motorola/dubai/yupik_ipa_fws.mbn";
-	memory-region = <&ipa_fw_mem>;
-
-	qcom,gsi-loader = "self";
 
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/sm7325-nothing-spacewar.dts b/arch/arm64/boot/dts/qcom/sm7325-nothing-spacewar.dts
index cb59c122f6f6a..7db9ae1c3cef9 100644
--- a/arch/arm64/boot/dts/qcom/sm7325-nothing-spacewar.dts
+++ b/arch/arm64/boot/dts/qcom/sm7325-nothing-spacewar.dts
@@ -971,8 +971,6 @@ nfc@28 {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	firmware-name = "qcom/sm7325/nothing/spacewar/ipa_fws.mbn";
 
 	status = "okay";
diff --git a/arch/arm64/boot/dts/qcom/sm7325-xiaomi-taoyao.dts b/arch/arm64/boot/dts/qcom/sm7325-xiaomi-taoyao.dts
new file mode 100644
index 0000000000000..6f8bc65bb99b8
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sm7325-xiaomi-taoyao.dts
@@ -0,0 +1,913 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (c) 2025, Stanislav Zaikin <zstaseg@gmail.com>
+ */
+
+/dts-v1/;
+
+/* PM7250B is configured to use SID8/9 */
+#define PM7250B_SID 8
+#define PM7250B_SID1 9
+
+#include <dt-bindings/arm/qcom,ids.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/iio/qcom,spmi-adc7-pm7325.h>
+#include <dt-bindings/iio/qcom,spmi-adc7-pmk8350.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+#include <dt-bindings/sound/qcom,q6afe.h>
+#include <dt-bindings/sound/qcom,q6asm.h>
+
+#include "sm7325.dtsi"
+#include "pm7325.dtsi"
+#include "pm7250b.dtsi" /* PM7250B */
+#include "pm8350c.dtsi" /* PM7350C */
+#include "pmk8350.dtsi" /* PMK7325 */
+
+/* The following reserved memory regions have different addresses or sizes */
+/delete-node/ &adsp_mem;
+/delete-node/ &adsp_rpc_remote_heap_mem;
+/delete-node/ &cdsp_mem;
+/delete-node/ &rmtfs_mem;
+
+/ {
+	model = "Xiaomi 12 Lite 5G";
+	compatible = "xiaomi,taoyao", "qcom,sm7325";
+	chassis-type = "handset";
+
+	aliases {
+		serial0 = &uart5;
+		serial1 = &uart7;
+	};
+
+	chosen {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		framebuffer0: framebuffer {
+			compatible = "simple-framebuffer";
+			memory-region = <&framebuffer_reserved>;
+			width = <1080>;
+			height = <2400>;
+			stride = <(1080 * 4)>;
+			format = "a8r8g8b8";
+
+			clocks = <&gcc GCC_DISP_HF_AXI_CLK>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&key_vol_up>;
+		pinctrl-names = "default";
+
+		key-volume-up {
+			label = "Volume Up";
+			gpios = <&pm7325_gpios 6 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,sm7325-pmic-glink",
+			     "qcom,qcm6490-pmic-glink",
+			     "qcom,pmic-glink";
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		orientation-gpios = <&tlmm 140 GPIO_ACTIVE_HIGH>;
+
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in: endpoint {
+						remote-endpoint = <&usb_1_dwc3_hs>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_sbu: endpoint {
+						remote-endpoint = <&fsa4480_sbu_mux>;
+					};
+				};
+			};
+		};
+	};
+
+	vreg_oled_dvdd: regulator-oled-dvdd {
+		compatible = "regulator-fixed";
+		regulator-name = "oled_dvdd";
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		gpio = <&tlmm 46 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		vin-supply = <&vreg_s1b_1p856>;
+		regulator-boot-on;
+	};
+
+	/* S2B is really ebi.lvl but it's there for supply map completeness sake. */
+	vreg_s2b_0p7: regulator-smpb2 {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_s2b_0p7";
+
+		regulator-min-microvolt = <700000>;
+		regulator-max-microvolt = <700000>;
+		regulator-always-on;
+		vin-supply = <&vph_pwr>;
+	};
+
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+	};
+
+	reserved-memory {
+		cdsp_secure_heap_mem: cdsp-secure-heap@81800000 {
+			reg = <0x0 0x81800000 0x0 0x1e00000>;
+			no-map;
+		};
+
+		cvp_mem: cvp@86200000 {
+			reg = <0x0 0x86200000 0x0 0x500000>;
+			no-map;
+		};
+
+		adsp_mem: adsp@86700000 {
+			reg = <0x0 0x86700000 0x0 0x4000000>;
+			no-map;
+		};
+
+		camera_mem: camera@8ad00000 {
+			reg = <0x0 0x8ad00000 0x0 0x500000>;
+			no-map;
+		};
+
+		ipa_gsi_mem: ipa-gsi@8b710000 {
+			reg = <0x0 0x8b710000 0x0 0xa000>;
+			no-map;
+		};
+
+		cdsp_mem: cdsp@9c700000 {
+			reg = <0x0 0x9c700000 0x0 0x1e00000>;
+			no-map;
+		};
+
+		ramoops@a9000000 {
+			compatible = "ramoops";
+			reg = <0x0 0xa9000000 0x0 0x200000>;
+			mem-type = <2>;
+			pmsg-size = <0x100000>;
+			record-size = <0x100000>;
+		};
+
+		removed_mem: removed@c0000000 {
+			reg = <0x0 0xc0000000 0x0 0x6800000>;
+			no-map;
+		};
+
+		pil_trustedvm_mem: pil-trustedvm-region@d0800000 {
+			reg = <0x0 0xd0800000 0x0 0x76f7000>;
+			no-map;
+		};
+
+		qrtr_shmem: qrtr-shmem@d7ef7000 {
+			reg = <0x0 0xd7ef7000 0x0 0x9000>;
+			no-map;
+		};
+
+		neuron_block_0_mem: neuron-block@d7f00000 {
+			reg = <0x0 0xd7f00000 0x0 0x80000>;
+			no-map;
+		};
+
+		neuron_block_1_mem: neuron-block@d7f80000 {
+			reg = <0x0 0xd7f80000 0x0 0x80000>;
+			no-map;
+		};
+
+		adsp_rpc_remote_heap_mem: adsp-rpc-remote-heap@e0400000 {
+			reg = <0x0 0xe0400000 0x0 0x800000>;
+			no-map;
+		};
+
+		framebuffer_reserved: framebuffer@e1000000 {
+			reg = <0x0 0xe1000000 0x0 (1080 * 2400 * 4)>;
+			no-map;
+		};
+
+		rmtfs_mem: rmtfs@e7d00000 {
+			compatible = "qcom,rmtfs-mem";
+			reg = <0x0 0xe7d00000 0x0 0x280000>;
+			no-map;
+
+			qcom,client-id = <1>;
+			qcom,vmid = <QCOM_SCM_VMID_MSS_MSA>,
+				    <QCOM_SCM_VMID_NAV>;
+		};
+	};
+
+	wcn6750-pmu {
+		compatible = "qcom,wcn6750-pmu";
+		pinctrl-0 = <&hst_bt_en>,
+			    <&hst_sw_ctrl>;
+		pinctrl-names = "default";
+		vddio-supply = <&vreg_l19b_1p8>;
+		vddaon-supply = <&vreg_s7b_0p952>;
+		vddasd-supply = <&vreg_l11c_2p8>;
+		vddpmu-supply = <&vreg_s7b_0p952>;
+		vddrfa0p8-supply = <&vreg_s7b_0p952>;
+		vddrfa1p2-supply = <&vreg_s8b_1p256>;
+		vddrfa1p7-supply = <&vreg_s1b_1p856>;
+		vddrfa2p2-supply = <&vreg_s1c_2p2>;
+
+		bt-enable-gpios = <&tlmm 85 GPIO_ACTIVE_HIGH>;
+		swctrl-gpios = <&tlmm 86 GPIO_ACTIVE_HIGH>;
+
+		regulators {
+			vreg_pmu_rfa_cmn: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn";
+			};
+
+			vreg_pmu_aon_0p59: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p59";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p85: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p85";
+			};
+
+			vreg_pmu_btcmx_0p85: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p85";
+			};
+
+			vreg_pmu_rfa_0p8: ldo5 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo6 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p7: ldo7 {
+				regulator-name = "vreg_pmu_rfa_1p7";
+			};
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pm7325-rpmh-regulators";
+		qcom,pmic-id = "b";
+
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s7-supply = <&vph_pwr>;
+		vdd-s8-supply = <&vph_pwr>;
+
+		vdd-l1-l4-l12-l15-supply = <&vreg_s7b_0p952>;
+		vdd-l2-l7-supply = <&vreg_bob>;
+		vdd-l3-supply = <&vreg_s2b_0p7>;
+		vdd-l5-supply = <&vreg_s2b_0p7>;
+		vdd-l6-l9-l10-supply = <&vreg_s8b_1p256>;
+		vdd-l8-supply = <&vreg_s7b_0p952>;
+		vdd-l11-l17-l18-l19-supply = <&vreg_s1b_1p856>;
+		vdd-l13-supply = <&vreg_s7b_0p952>;
+		vdd-l14-l16-supply = <&vreg_s8b_1p256>;
+
+		/*
+		 * S2, L4-L5 are ARCs:
+		 * S2 - ebi.lvl,
+		 * L4 - lmx.lvl,
+		 * L5 - lcx.lvl.
+		 *
+		 * L10 are unused.
+		 */
+
+		vreg_s1b_1p856: smps1 {
+			regulator-name = "vreg_s1b_1p856";
+			regulator-min-microvolt = <1840000>;
+			regulator-max-microvolt = <2040000>;
+		};
+
+		vreg_s7b_0p952: smps7 {
+			regulator-name = "vreg_s7b_0p952";
+			regulator-min-microvolt = <535000>;
+			regulator-max-microvolt = <1120000>;
+		};
+
+		vreg_s8b_1p256: smps8 {
+			regulator-name = "vreg_s8b_1p256";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1500000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_RET>;
+		};
+
+		vreg_l1b_0p912: ldo1 {
+			regulator-name = "vreg_l1b_0p912";
+			regulator-min-microvolt = <825000>;
+			regulator-max-microvolt = <925000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2b_3p072: ldo2 {
+			regulator-name = "vreg_l2b_3p072";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3b_0p6: ldo3 {
+			regulator-name = "vreg_l3b_0p6";
+			regulator-min-microvolt = <312000>;
+			regulator-max-microvolt = <910000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6b_1p2: ldo6 {
+			regulator-name = "vreg_l6b_1p2";
+			regulator-min-microvolt = <1140000>;
+			regulator-max-microvolt = <1260000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b_2p96: ldo7 {
+			regulator-name = "vreg_l7b_2p96";
+			/* Constrained for UFS VCC, at least until UFS driver scales voltage */
+			regulator-min-microvolt = <2952000>;
+			regulator-max-microvolt = <2952000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b_0p904: ldo8 {
+			regulator-name = "vreg_l8b_0p904";
+			regulator-min-microvolt = <870000>;
+			regulator-max-microvolt = <970000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b_1p2: ldo9 {
+			regulator-name = "vreg_l9b_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11b_1p776: ldo11 {
+			regulator-name = "vreg_l11b_1p776";
+			regulator-min-microvolt = <1504000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b_0p8: ldo12 {
+			regulator-name = "vreg_l12b_0p8";
+			regulator-min-microvolt = <751000>;
+			regulator-max-microvolt = <824000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13b_0p8: ldo13 {
+			regulator-name = "vreg_l13b_0p8";
+			regulator-min-microvolt = <530000>;
+			regulator-max-microvolt = <824000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l14b_1p2: ldo14 {
+			regulator-name = "vreg_l14b_1p2";
+			regulator-min-microvolt = <1080000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_0p88: ldo15 {
+			regulator-name = "vreg_l15b_0p88";
+			regulator-min-microvolt = <765000>;
+			regulator-max-microvolt = <1020000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l16b_1p2: ldo16 {
+			regulator-name = "vreg_l16b_1p2";
+			regulator-min-microvolt = <1100000>;
+			regulator-max-microvolt = <1300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b_1p8: ldo17 {
+			regulator-name = "vreg_l17b_1p8";
+			regulator-min-microvolt = <1700000>;
+			regulator-max-microvolt = <1900000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l18b_1p8: ldo18 {
+			regulator-name = "vreg_l18b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l19b_1p8: ldo19 {
+			regulator-name = "vreg_l19b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pm8350c-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s7-supply = <&vph_pwr>;
+		vdd-s9-supply = <&vph_pwr>;
+		vdd-s10-supply = <&vph_pwr>;
+
+		vdd-l1-l12-supply = <&vreg_s1b_1p856>;
+		vdd-l2-l8-supply = <&vreg_s1b_1p856>;
+		vdd-l3-l4-l5-l7-l13-supply = <&vreg_bob>;
+		vdd-l6-l9-l11-supply = <&vreg_bob>;
+		vdd-l10-supply = <&vreg_s7b_0p952>;
+
+		vdd-bob-supply = <&vph_pwr>;
+
+		/*
+		 * S2, S5, S7, S10 are ARCs:
+		 * S2 - cx.lvl,
+		 * S5 - mss.lvl,
+		 * S7 - gfx.lvl,
+		 * S10 - mx.lvl.
+		 */
+
+		vreg_s1c_2p2: smps1 {
+			regulator-name = "vreg_s1c_2p2";
+			regulator-min-microvolt = <2190000>;
+			regulator-max-microvolt = <2210000>;
+		};
+
+		vreg_s9c_0p676: smps9 {
+			regulator-name = "vreg_s9c_0p676";
+			regulator-min-microvolt = <1010000>;
+			regulator-max-microvolt = <1170000>;
+		};
+
+		vreg_l1c_1p8: ldo1 {
+			regulator-name = "vreg_l1c_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1980000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2c_1p8: ldo2 {
+			regulator-name = "vreg_l2c_1p8";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <1980000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3c_3p0: ldo3 {
+			regulator-name = "vreg_l3c_3p0";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <3540000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4c_1p8_3p0: ldo4 {
+			regulator-name = "vreg_l4c_1p8_3p0";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5c_1p8_3p0: ldo5 {
+			regulator-name = "vreg_l5c_1p8_3p0";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6c_2p96: ldo6 {
+			regulator-name = "vreg_l6c_2p96";
+			regulator-min-microvolt = <1650000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7c_3p0: ldo7 {
+			regulator-name = "vreg_l7c_3p0";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8c_1p8: ldo8 {
+			regulator-name = "vreg_l8c_1p8";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9c_2p96: ldo9 {
+			regulator-name = "vreg_l9c_2p96";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10c_0p88: ldo10 {
+			regulator-name = "vreg_l10c_0p88";
+			regulator-min-microvolt = <720000>;
+			regulator-max-microvolt = <1050000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11c_2p8: ldo11 {
+			regulator-name = "vreg_l11c_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12c_1p8: ldo12 {
+			regulator-name = "vreg_l12c_1p8";
+			regulator-min-microvolt = <1650000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13c_3p0: ldo13 {
+			regulator-name = "vreg_l13c_3p0";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob: bob {
+			regulator-name = "vreg_bob";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+};
+
+&dispcc {
+	status = "disabled";
+};
+
+&gcc {
+	protected-clocks = <GCC_CFG_NOC_LPASS_CLK>,
+			   <GCC_MSS_CFG_AHB_CLK>,
+			   <GCC_MSS_OFFLINE_AXI_CLK>,
+			   <GCC_MSS_Q6SS_BOOT_CLK_SRC>,
+			   <GCC_MSS_Q6_MEMNOC_AXI_CLK>,
+			   <GCC_MSS_SNOC_AXI_CLK>,
+			   <GCC_QSPI_CNOC_PERIPH_AHB_CLK>,
+			   <GCC_QSPI_CORE_CLK>,
+			   <GCC_QSPI_CORE_CLK_SRC>,
+			   <GCC_SEC_CTRL_CLK_SRC>,
+			   <GCC_WPSS_AHB_BDG_MST_CLK>,
+			   <GCC_WPSS_AHB_CLK>,
+			   <GCC_WPSS_RSCP_CLK>;
+};
+
+&gpi_dma0 {
+	status = "okay";
+};
+
+&gpi_dma1 {
+	status = "okay";
+};
+
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/sm7325/xiaomi/taoyao/a660_zap.mbn";
+};
+
+&i2c1 {
+	clock-frequency = <100000>;
+
+	status = "okay";
+
+	typec-mux@42 {
+		compatible = "fcs,fsa4480";
+		reg = <0x42>;
+
+		vcc-supply = <&vreg_bob>;
+
+		mode-switch;
+		orientation-switch;
+
+		port {
+			fsa4480_sbu_mux: endpoint {
+				remote-endpoint = <&pmic_glink_sbu>;
+			};
+		};
+	};
+};
+
+&ipa {
+	firmware-name = "qcom/sm7325/xiaomi/taoyao/ipa_fws.mbn";
+
+	status = "okay";
+};
+
+&lpass_audiocc {
+	compatible = "qcom,qcm6490-lpassaudiocc";
+	/delete-property/ power-domains;
+};
+
+&pm7325_gpios {
+	key_vol_up: key-vol-up-n-state {
+		pins = "gpio6";
+		function = PMIC_GPIO_FUNC_NORMAL;
+		power-source = <1>;
+		bias-pull-up;
+		input-enable;
+	};
+};
+
+&pmk8350_rtc {
+	status = "okay";
+};
+
+&pon_pwrkey {
+	status = "okay";
+};
+
+&pon_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+
+	status = "okay";
+};
+
+&qup_spi13_cs {
+	drive-strength = <6>;
+	bias-disable;
+};
+
+&qup_spi13_data_clk {
+	drive-strength = <6>;
+	bias-disable;
+};
+
+&qupv3_id_0 {
+	status = "okay";
+};
+
+&qupv3_id_1 {
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/sm7325/xiaomi/taoyao/adsp.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/sm7325/xiaomi/taoyao/cdsp.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_mpss {
+	firmware-name = "qcom/sm7325/xiaomi/taoyao/modem.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_wpss {
+	firmware-name = "qcom/sm7325/xiaomi/taoyao/wpss.mbn";
+
+	status = "okay";
+};
+
+&spi13 {
+	status = "okay";
+
+	touchscreen@0 {
+		compatible = "goodix,gt9916";
+		reg = <0>;
+
+		interrupts-extended = <&tlmm 81 IRQ_TYPE_LEVEL_LOW>;
+
+		reset-gpios = <&tlmm 105 GPIO_ACTIVE_LOW>;
+
+		avdd-supply = <&vreg_l7c_3p0>;
+		vddio-supply = <&vreg_l2c_1p8>;
+
+		spi-max-frequency = <5000000>;
+
+		touchscreen-size-x = <10800>;
+		touchscreen-size-y = <24000>;
+
+		pinctrl-0 = <&ts_irq>, <&ts_reset>;
+		pinctrl-names = "default";
+	};
+};
+
+&tlmm {
+	/*
+	 * 48-51: presumably NFC SE
+	 * 56-59: presumably fingerprint reader
+	 */
+	gpio-reserved-ranges = <48 4>, <56 4>;
+
+	bt_uart_sleep_cts: bt-uart-sleep-cts-state {
+		pins = "gpio28";
+		function = "gpio";
+		bias-bus-hold;
+	};
+
+	bt_uart_sleep_rts: bt-uart-sleep-rts-state {
+		pins = "gpio29";
+		function = "gpio";
+		bias-pull-down;
+	};
+
+	bt_uart_sleep_txd: bt-uart-sleep-txd-state {
+		pins = "gpio30";
+		function = "gpio";
+		bias-pull-up;
+	};
+
+	bt_uart_sleep_rxd: bt-uart-sleep-rxd-state {
+		pins = "gpio31";
+		function = "gpio";
+		bias-pull-up;
+	};
+
+	sde_dsi_active: sde-dsi-active-state {
+		pins = "gpio44";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
+
+	sde_dsi_sleep: sde-dsi-sleep-state {
+		pins = "gpio44";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
+	sde_te_active_sleep: sde-te-active-state {
+		pins = "gpio80";
+		function = "mdp_vsync";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
+	ts_irq: ts-irq-state {
+		pins = "gpio81";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-up;
+	};
+
+	hst_bt_en: hst-bt-en-state {
+		pins = "gpio85";
+		function = "gpio";
+		bias-disable;
+	};
+
+	hst_sw_ctrl: hst-sw-ctrl-state {
+		pins = "gpio86";
+		function = "gpio";
+		bias-pull-down;
+	};
+
+	ts_reset: ts-reset-state {
+		pins = "gpio105";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-up;
+	};
+};
+
+&uart5 {
+	status = "okay";
+};
+
+&uart7 {
+	/delete-property/interrupts;
+	interrupts-extended = <&intc GIC_SPI 608 IRQ_TYPE_LEVEL_HIGH>,
+			      <&tlmm 31 IRQ_TYPE_EDGE_FALLING>;
+
+	pinctrl-1 = <&bt_uart_sleep_cts>,
+		    <&bt_uart_sleep_rts>,
+		    <&bt_uart_sleep_txd>,
+		    <&bt_uart_sleep_rxd>;
+	pinctrl-names = "default", "sleep";
+
+	status = "okay";
+
+	bluetooth: bluetooth {
+		compatible = "qcom,wcn6750-bt";
+
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddbtcmx-supply = <&vreg_pmu_btcmx_0p85>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		max-speed = <3200000>;
+	};
+};
+
+&ufs_mem_hc {
+	reset-gpios = <&tlmm 175 GPIO_ACTIVE_LOW>;
+
+	vcc-supply = <&vreg_l7b_2p96>;
+	vcc-max-microamp = <800000>;
+	/*
+	 * Technically l9b enables an eLDO (supplied by s1b) which then powers
+	 * VCCQ2 of the UFS.
+	 */
+	vccq2-supply = <&vreg_l9b_1p2>;
+	vccq2-max-microamp = <900000>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vreg_l10c_0p88>;
+	vdda-pll-supply = <&vreg_l6b_1p2>;
+
+	status = "okay";
+};
+
+&usb_1 {
+	/* USB 2.0 only */
+	qcom,select-utmi-as-pipe-clk;
+
+	maximum-speed = "high-speed";
+	/* Remove USB3 phy */
+	phys = <&usb_1_hsphy>;
+	phy-names = "usb2-phy";
+
+	status = "okay";
+};
+
+&usb_1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in>;
+};
+
+&usb_1_hsphy {
+	vdda-pll-supply = <&vreg_l10c_0p88>;
+	vdda18-supply = <&vreg_l1c_1p8>;
+	vdda33-supply = <&vreg_l2b_3p072>;
+
+	status = "okay";
+};
+
+&venus {
+	firmware-name = "qcom/sm7325/xiaomi/taoyao/vpu20_1v.mbn";
+
+	status = "okay";
+};
+
+&wifi {
+	qcom,calibration-variant = "Xiaomi_taoyao";
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/sm8150.dtsi b/arch/arm64/boot/dts/qcom/sm8150.dtsi
index 0e101096209ab..661848ef5eee2 100644
--- a/arch/arm64/boot/dts/qcom/sm8150.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8150.dtsi
@@ -4174,7 +4174,7 @@ dispcc: clock-controller@af00000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sm8150-pdc", "qcom,pdc";
-			reg = <0 0x0b220000 0 0x30000>;
+			reg = <0 0x0b220000 0 0x10000>;
 			qcom,pdc-ranges = <0 480 94>, <94 609 31>,
 					  <125 63 1>;
 			#interrupt-cells = <2>;
diff --git a/arch/arm64/boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi b/arch/arm64/boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi
index 51b57c697a753..19aff83ecf7b6 100644
--- a/arch/arm64/boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi
@@ -4,6 +4,7 @@
  */
 
 #include <dt-bindings/arm/qcom,ids.h>
+#include <dt-bindings/media/video-interfaces.h>
 #include <dt-bindings/phy/phy.h>
 #include <dt-bindings/regulator/qcom,rpmh-regulator.h>
 #include <dt-bindings/usb/pd.h>
@@ -531,6 +532,61 @@ vreg_l7f_1p8: ldo7 {
 	};
 };
 
+&camss {
+	status = "okay";
+
+	vdda-phy-supply = <&vreg_l5a_0p88>;
+	vdda-pll-supply = <&vreg_l9a_1p2>;
+
+	ports {
+		port@4 {
+			csiphy4_ep: endpoint {
+				clock-lanes = <7>;
+				data-lanes = <0 1>;
+				bus-type = <MEDIA_BUS_TYPE_CSI2_DPHY>;
+				remote-endpoint = <&ov8856_front_ep>;
+			};
+		};
+	};
+};
+
+&cci1 {
+	status = "okay";
+};
+
+&cci1_i2c1 {
+	camera_front: camera@10 {
+		compatible = "ovti,ov8856";
+		reg = <0x10>;
+
+		avdd-supply = <&vreg_l5p>;
+		dovdd-supply = <&vreg_l1c_1p8>;
+		dvdd-supply = <&vreg_l1p>;
+
+		clocks = <&camcc CAM_CC_MCLK3_CLK>;
+		clock-names = "xvclk";
+		assigned-clocks = <&camcc CAM_CC_MCLK3_CLK>;
+		assigned-clock-rates = <19200000>;
+
+		reset-gpios = <&tlmm 109 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&cam_mclk3_default &camera_front_active>;
+		pinctrl-names = "default";
+
+		orientation = <0>; /* Front facing */
+		rotation = <270>;
+
+		port {
+			ov8856_front_ep: endpoint {
+				data-lanes = <1 2>;
+				link-frequencies = /bits/ 64
+					<720000000 360000000>;
+				remote-endpoint = <&csiphy4_ep>;
+			};
+		};
+	};
+};
+
 &cdsp {
 	firmware-name = "qcom/sm8250/xiaomi/elish/cdsp.mbn";
 	status = "okay";
@@ -599,6 +655,82 @@ fuel-gauge@55 {
 	};
 };
 
+&i2c15 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	pm8008: pmic@8 {
+		compatible = "qcom,pm8008";
+		reg = <0x8>;
+
+		interrupts-extended = <&tlmm 84 IRQ_TYPE_EDGE_RISING>;
+		reset-gpios = <&tlmm 76 GPIO_ACTIVE_LOW>;
+
+		vdd-l1-l2-supply = <&vreg_s8c_1p35>;
+		vdd-l3-l4-supply = <&vreg_bob>;
+		vdd-l5-supply = <&vreg_bob>;
+		vdd-l6-supply = <&vreg_bob>;
+		vdd-l7-supply = <&vreg_bob>;
+
+		pinctrl-0 = <&pm8008_int_default>, <&pm8008_reset_default>;
+		pinctrl-names = "default";
+
+		gpio-controller;
+		#gpio-cells = <2>;
+		gpio-ranges = <&pm8008 0 0 2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+
+		#thermal-sensor-cells = <0>;
+
+		regulators {
+			vreg_l1p: ldo1 {
+				regulator-name = "vreg_l1p";
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+			};
+
+			vreg_l2p: ldo2 {
+				regulator-name = "vreg_l2p";
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+			};
+
+			vreg_l3p: ldo3 {
+				regulator-name = "vreg_l3p";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+			};
+
+			vreg_l4p: ldo4 {
+				regulator-name = "vreg_l4p";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+			};
+
+			vreg_l5p: ldo5 {
+				regulator-name = "vreg_l5p";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+			};
+
+			vreg_l6p: ldo6 {
+				regulator-name = "vreg_l6p";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			vreg_l7p: ldo7 {
+				regulator-name = "vreg_l7p";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2900000>;
+			};
+		};
+	};
+};
+
 &mdss {
 	status = "okay";
 };
@@ -801,6 +933,27 @@ bt_en_state: bt-default-state {
 		bias-pull-up;
 	};
 
+	pm8008_int_default: pm8008-int-default-state {
+		pins = "gpio84";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	pm8008_reset_default: pm8008-reset-default-state {
+		pins = "gpio76";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	camera_front_active: camera-front-active-state {
+		pins = "gpio109";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
 	wlan_en_state: wlan-default-state {
 		pins = "gpio20";
 		function = "gpio";
diff --git a/arch/arm64/boot/dts/qcom/sm8250.dtsi b/arch/arm64/boot/dts/qcom/sm8250.dtsi
index 7076720413ab2..f11da2ca41818 100644
--- a/arch/arm64/boot/dts/qcom/sm8250.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8250.dtsi
@@ -10,6 +10,7 @@
 #include <dt-bindings/clock/qcom,gpucc-sm8250.h>
 #include <dt-bindings/clock/qcom,rpmh.h>
 #include <dt-bindings/dma/qcom-gpi.h>
+#include <dt-bindings/firmware/qcom,scm.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/interconnect/qcom,osm-l3.h>
 #include <dt-bindings/interconnect/qcom,sm8250.h>
@@ -863,6 +864,14 @@ cdsp_secure_heap: memory@8bf00000 {
 			reg = <0x0 0x8bf00000 0x0 0x4600000>;
 			no-map;
 		};
+
+		adsp_rpc_remote_heap_mem: adsp-rpc-remote-heap {
+			compatible = "shared-dma-pool";
+			alloc-ranges = <0x0 0x80000000 0x0 0x80000000>;
+			reusable;
+			alignment = <0x0 0x400000>;
+			size = <0x0 0x800000>;
+		};
 	};
 
 	smem {
@@ -4326,8 +4335,12 @@ venus: video-codec@aa00000 {
 			interrupts = <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH>;
 			power-domains = <&videocc MVS0C_GDSC>,
 					<&videocc MVS0_GDSC>,
-					<&rpmhpd RPMHPD_MX>;
-			power-domain-names = "venus", "vcodec0", "mx";
+					<&rpmhpd RPMHPD_MX>,
+					<&rpmhpd RPMHPD_MMCX>;
+			power-domain-names = "venus",
+					     "vcodec0",
+					     "mx",
+					     "mmcx";
 			operating-points-v2 = <&venus_opp_table>;
 
 			clocks = <&gcc GCC_VIDEO_AXI0_CLK>,
@@ -4351,24 +4364,28 @@ venus: video-codec@aa00000 {
 			venus_opp_table: opp-table {
 				compatible = "operating-points-v2";
 
-				opp-720000000 {
-					opp-hz = /bits/ 64 <720000000>;
-					required-opps = <&rpmhpd_opp_low_svs>;
+				opp-240000000 {
+					opp-hz = /bits/ 64 <240000000>;
+					required-opps = <&rpmhpd_opp_svs>,
+							<&rpmhpd_opp_low_svs>;
 				};
 
-				opp-1014000000 {
-					opp-hz = /bits/ 64 <1014000000>;
-					required-opps = <&rpmhpd_opp_svs>;
+				opp-338000000 {
+					opp-hz = /bits/ 64 <338000000>;
+					required-opps = <&rpmhpd_opp_svs>,
+							<&rpmhpd_opp_svs>;
 				};
 
-				opp-1098000000 {
-					opp-hz = /bits/ 64 <1098000000>;
-					required-opps = <&rpmhpd_opp_svs_l1>;
+				opp-366000000 {
+					opp-hz = /bits/ 64 <366000000>;
+					required-opps = <&rpmhpd_opp_svs_l1>,
+							<&rpmhpd_opp_svs_l1>;
 				};
 
-				opp-1332000000 {
-					opp-hz = /bits/ 64 <1332000000>;
-					required-opps = <&rpmhpd_opp_nom>;
+				opp-444000000 {
+					opp-hz = /bits/ 64 <444000000>;
+					required-opps = <&rpmhpd_opp_svs_l1>,
+							<&rpmhpd_opp_nom>;
 				};
 			};
 		};
@@ -5053,7 +5070,7 @@ dispcc: clock-controller@af00000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sm8250-pdc", "qcom,pdc";
-			reg = <0 0x0b220000 0 0x30000>, <0 0x17c000f0 0 0x60>;
+			reg = <0 0x0b220000 0 0x10000>, <0 0x17c000f0 0 0x60>;
 			qcom,pdc-ranges = <0 480 94>, <94 609 31>,
 					  <125 63 1>, <126 716 12>;
 			#interrupt-cells = <2>;
@@ -5165,6 +5182,13 @@ mclk-pins {
 				};
 			};
 
+			cam_mclk3_default: cam-mclk3-default-state {
+				pins = "gpio97";
+				function = "cam_mclk";
+				drive-strength = <4>;
+				bias-disable;
+			};
+
 			cci0_default: cci0-default-state {
 				cci0_i2c0_default: cci0-i2c0-default-pins {
 					/* SDA, SCL */
@@ -6077,6 +6101,9 @@ fastrpc {
 					compatible = "qcom,fastrpc";
 					qcom,glink-channels = "fastrpcglink-apps-dsp";
 					label = "adsp";
+					memory-region = <&adsp_rpc_remote_heap_mem>;
+					qcom,vmids = <QCOM_SCM_VMID_LPASS
+						      QCOM_SCM_VMID_ADSP_HEAP>;
 					qcom,non-secure-domain;
 					#address-cells = <1>;
 					#size-cells = <0>;
diff --git a/arch/arm64/boot/dts/qcom/sm8350-hdk.dts b/arch/arm64/boot/dts/qcom/sm8350-hdk.dts
index 5f975d0094658..d98ec4bcfd87f 100644
--- a/arch/arm64/boot/dts/qcom/sm8350-hdk.dts
+++ b/arch/arm64/boot/dts/qcom/sm8350-hdk.dts
@@ -472,6 +472,12 @@ lt9611_out: endpoint {
 	};
 };
 
+&iris {
+	firmware-name = "qcom/vpu/vpu20_p4_sm8350.mbn";
+
+	status = "okay";
+};
+
 &mdss {
 	status = "okay";
 };
@@ -923,8 +929,7 @@ irq-pins {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&pil_ipa_fw_mem>;
-	status = "okay";
 	firmware-name = "qcom/sm8350/ipa_fws.mbn";
+
+	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/sm8350-microsoft-surface-duo2.dts b/arch/arm64/boot/dts/qcom/sm8350-microsoft-surface-duo2.dts
index 3bd5e57cbcdaa..6b10464e8774c 100644
--- a/arch/arm64/boot/dts/qcom/sm8350-microsoft-surface-duo2.dts
+++ b/arch/arm64/boot/dts/qcom/sm8350-microsoft-surface-duo2.dts
@@ -290,8 +290,6 @@ &i2c11 {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&pil_ipa_fw_mem>;
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sm8350-mtp.dts b/arch/arm64/boot/dts/qcom/sm8350-mtp.dts
index 8bee57f3b25a1..b25f223d6190e 100644
--- a/arch/arm64/boot/dts/qcom/sm8350-mtp.dts
+++ b/arch/arm64/boot/dts/qcom/sm8350-mtp.dts
@@ -378,7 +378,5 @@ &usb_2_qmpphy {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&pil_ipa_fw_mem>;
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi b/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi
index 7ae1eb0a7ccea..ba862b4f9890b 100644
--- a/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi
@@ -553,9 +553,8 @@ &i2c17 {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&pil_ipa_fw_mem>;
 	firmware-name = "qcom/sm8350/Sony/sagami/ipa_fws.mbn";
+
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sm8350.dtsi b/arch/arm64/boot/dts/qcom/sm8350.dtsi
index c830953156ec6..72ecfaeefdd39 100644
--- a/arch/arm64/boot/dts/qcom/sm8350.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8350.dtsi
@@ -10,6 +10,7 @@
 #include <dt-bindings/clock/qcom,gcc-sm8350.h>
 #include <dt-bindings/clock/qcom,gpucc-sm8350.h>
 #include <dt-bindings/clock/qcom,rpmh.h>
+#include <dt-bindings/clock/qcom,sm8350-videocc.h>
 #include <dt-bindings/dma/qcom-gpi.h>
 #include <dt-bindings/firmware/qcom,scm.h>
 #include <dt-bindings/gpio/gpio.h>
@@ -19,6 +20,7 @@
 #include <dt-bindings/phy/phy-qcom-qmp.h>
 #include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
+#include <dt-bindings/reset/qcom,sm8350-videocc.h>
 #include <dt-bindings/soc/qcom,apr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
 #include <dt-bindings/sound/qcom,q6afe.h>
@@ -1866,6 +1868,9 @@ ipa: ipa@1e40000 {
 			qcom,smem-state-names = "ipa-clock-enabled-valid",
 						"ipa-clock-enabled";
 
+			qcom,gsi-loader = "self";
+			memory-region = <&pil_ipa_fw_mem>;
+
 			sram = <&ipa_modem_tables>;
 
 			status = "disabled";
@@ -2747,6 +2752,85 @@ usb_2_dwc3: usb@a800000 {
 			};
 		};
 
+		iris: video-codec@aa00000 {
+			compatible = "qcom,sm8350-iris", "qcom,sm8250-venus";
+			reg = <0x0 0x0aa00000 0x0 0x100000>;
+			interrupts = <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH>;
+
+			clocks = <&gcc GCC_VIDEO_AXI0_CLK>,
+				 <&videocc VIDEO_CC_MVS0C_CLK>,
+				 <&videocc VIDEO_CC_MVS0_CLK>;
+			clock-names = "iface",
+				      "core",
+				      "vcodec0_core";
+			power-domains = <&videocc MVS0C_GDSC>,
+					<&videocc MVS0_GDSC>,
+					<&rpmhpd RPMHPD_MX>,
+					<&rpmhpd RPMHPD_MMCX>;
+			power-domain-names = "venus",
+					     "vcodec0",
+					     "mx",
+					     "mmcx";
+
+			resets = <&gcc GCC_VIDEO_AXI0_CLK_ARES>,
+				 <&videocc VIDEO_CC_MVS0C_CLK_ARES>;
+			reset-names = "bus", "core";
+
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_VENUS_CFG QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&mmss_noc MASTER_VIDEO_P0 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "cpu-cfg",
+					     "video-mem";
+
+			operating-points-v2 = <&iris_opp_table>;
+			iommus = <&apps_smmu 0x2100 0x400>;
+			memory-region = <&pil_video_mem>;
+
+			status = "disabled";
+
+			iris_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-240000000 {
+					opp-hz = /bits/ 64 <240000000>;
+					required-opps = <&rpmhpd_opp_svs>,
+							<&rpmhpd_opp_low_svs>;
+				};
+
+				opp-338000000 {
+					opp-hz = /bits/ 64 <338000000>;
+					required-opps = <&rpmhpd_opp_svs>,
+							<&rpmhpd_opp_svs>;
+				};
+
+				opp-366000000 {
+					opp-hz = /bits/ 64 <366000000>;
+					required-opps = <&rpmhpd_opp_svs_l1>,
+							<&rpmhpd_opp_svs_l1>;
+				};
+
+				opp-444000000 {
+					opp-hz = /bits/ 64 <444000000>;
+					required-opps = <&rpmhpd_opp_svs_l1>,
+							<&rpmhpd_opp_nom>;
+				};
+			};
+		};
+
+		videocc: clock-controller@abf0000 {
+			compatible = "qcom,sm8350-videocc";
+			reg = <0 0x0abf0000 0 0x10000>;
+			clocks = <&rpmhcc RPMH_CXO_CLK>,
+				 <&rpmhcc RPMH_CXO_CLK_A>,
+				 <&sleep_clk>;
+			power-domains = <&rpmhpd RPMHPD_MMCX>;
+			required-opps = <&rpmhpd_opp_low_svs>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
 		mdss: display-subsystem@ae00000 {
 			compatible = "qcom,sm8350-mdss";
 			reg = <0 0x0ae00000 0 0x1000>;
@@ -3174,7 +3258,7 @@ dispcc: clock-controller@af00000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sm8350-pdc", "qcom,pdc";
-			reg = <0 0x0b220000 0 0x30000>, <0 0x17c000f0 0 0x60>;
+			reg = <0 0x0b220000 0 0x10000>, <0 0x17c000f0 0 0x60>;
 			qcom,pdc-ranges = <0 480 40>, <40 140 14>, <54 263 1>,   <55 306 4>,
 					  <59 312 3>, <62 374 2>,  <64 434 2>,   <66 438 3>,
 					  <69 86 1>,  <70 520 54>, <124 609 31>, <155 63 1>,
diff --git a/arch/arm64/boot/dts/qcom/sm8450.dtsi b/arch/arm64/boot/dts/qcom/sm8450.dtsi
index 56cb6e959e4ee..2488928656d94 100644
--- a/arch/arm64/boot/dts/qcom/sm8450.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8450.dtsi
@@ -752,11 +752,6 @@ oem_vm_mem: memory@bb000000 {
 			no-map;
 		};
 
-		mte_mem: memory@c0000000 {
-			reg = <0x0 0xc0000000 0x0 0x20000000>;
-			no-map;
-		};
-
 		qheebsp_reserved_mem: memory@e0000000 {
 			reg = <0x0 0xe0000000 0x0 0x600000>;
 			no-map;
@@ -806,16 +801,6 @@ qtee_mem: memory@e9b00000 {
 			reg = <0x0 0xe9b00000 0x0 0x500000>;
 			no-map;
 		};
-
-		trusted_apps_mem: memory@ea000000 {
-			reg = <0x0 0xea000000 0x0 0x3900000>;
-			no-map;
-		};
-
-		trusted_apps_ext_mem: memory@ed900000 {
-			reg = <0x0 0xed900000 0x0 0x3b00000>;
-			no-map;
-		};
 	};
 
 	smp2p-adsp {
@@ -2669,6 +2654,7 @@ usb_1_qmpphy: phy@88e8000 {
 			#clock-cells = <1>;
 			#phy-cells = <1>;
 
+			mode-switch;
 			orientation-switch;
 
 			status = "disabled";
@@ -2944,20 +2930,20 @@ swr1: soundwire@3210000 {
 			clock-names = "iface";
 			label = "RX";
 			qcom,din-ports = <0>;
-			qcom,dout-ports = <5>;
+			qcom,dout-ports = <6>;
 
 			pinctrl-0 = <&rx_swr_active>;
 			pinctrl-names = "default";
 
-			qcom,ports-sinterval-low =	/bits/ 8 <0x03 0x1f 0x1f 0x07 0x00>;
-			qcom,ports-offset1 =		/bits/ 8 <0x00 0x00 0x0b 0x01 0x00>;
-			qcom,ports-offset2 =		/bits/ 8 <0x00 0x00 0x0b 0x00 0x00>;
-			qcom,ports-hstart =		/bits/ 8 <0xff 0x03 0xff 0xff 0xff>;
-			qcom,ports-hstop =		/bits/ 8 <0xff 0x06 0xff 0xff 0xff>;
-			qcom,ports-word-length =	/bits/ 8 <0x01 0x07 0x04 0xff 0xff>;
-			qcom,ports-block-pack-mode =	/bits/ 8 <0xff 0x00 0x01 0xff 0xff>;
-			qcom,ports-block-group-count =	/bits/ 8 <0xff 0xff 0xff 0xff 0x00>;
-			qcom,ports-lane-control =	/bits/ 8 <0x01 0x00 0x00 0x00 0x00>;
+			qcom,ports-sinterval =		/bits/ 16 <0x03 0x3f 0x1f 0x07 0x00 0x18f>;
+			qcom,ports-offset1 =		/bits/ 8 <0x00 0x00 0x0b 0x01 0x00 0x00>;
+			qcom,ports-offset2 =		/bits/ 8 <0x00 0x00 0x0b 0x00 0x00 0x00>;
+			qcom,ports-hstart =		/bits/ 8 <0xff 0x03 0xff 0xff 0xff 0x08>;
+			qcom,ports-hstop =		/bits/ 8 <0xff 0x06 0xff 0xff 0xff 0x08>;
+			qcom,ports-word-length =	/bits/ 8 <0x01 0x07 0x04 0xff 0xff 0x0f>;
+			qcom,ports-block-pack-mode =	/bits/ 8 <0xff 0x00 0x01 0xff 0xff 0x00>;
+			qcom,ports-block-group-count =	/bits/ 8 <0xff 0xff 0xff 0xff 0x00 0x00>;
+			qcom,ports-lane-control =	/bits/ 8 <0x01 0x00 0x00 0x00 0x00 0x00>;
 
 			#address-cells = <2>;
 			#size-cells = <0>;
@@ -3728,7 +3714,7 @@ dispcc: clock-controller@af00000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sm8450-pdc", "qcom,pdc";
-			reg = <0 0x0b220000 0 0x30000>, <0 0x174000f0 0 0x64>;
+			reg = <0 0x0b220000 0 0x10000>, <0 0x174000f0 0 0x64>;
 			qcom,pdc-ranges = <0 480 12>, <14 494 24>, <40 520 54>,
 					  <94 609 31>, <125 63 1>, <126 716 12>;
 			#interrupt-cells = <2>;
diff --git a/arch/arm64/boot/dts/qcom/sm8550-hdk.dts b/arch/arm64/boot/dts/qcom/sm8550-hdk.dts
index ee13e6136a825..b795589385520 100644
--- a/arch/arm64/boot/dts/qcom/sm8550-hdk.dts
+++ b/arch/arm64/boot/dts/qcom/sm8550-hdk.dts
@@ -939,8 +939,6 @@ &i2c_master_hub_0 {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	firmware-name = "qcom/sm8550/ipa_fws.mbn";
 	status = "okay";
 };
@@ -1012,7 +1010,7 @@ &pcie0 {
 	status = "okay";
 };
 
-&pcieport0 {
+&pcie0_port0 {
 	wifi@0 {
 		compatible = "pci17cb,1107";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
diff --git a/arch/arm64/boot/dts/qcom/sm8550-qrd.dts b/arch/arm64/boot/dts/qcom/sm8550-qrd.dts
index 2fb2e0be5e4c6..f2effa55d8197 100644
--- a/arch/arm64/boot/dts/qcom/sm8550-qrd.dts
+++ b/arch/arm64/boot/dts/qcom/sm8550-qrd.dts
@@ -818,8 +818,6 @@ fsa4480_sbu_mux: endpoint {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	firmware-name = "qcom/sm8550/ipa_fws.mbn";
 	status = "okay";
 };
@@ -912,7 +910,7 @@ &pcie0 {
 	status = "okay";
 };
 
-&pcieport0 {
+&pcie0_port0 {
 	wifi@0 {
 		compatible = "pci17cb,1107";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
diff --git a/arch/arm64/boot/dts/qcom/sm8550.dtsi b/arch/arm64/boot/dts/qcom/sm8550.dtsi
index 396201905ef25..7599b7b7e645f 100644
--- a/arch/arm64/boot/dts/qcom/sm8550.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8550.dtsi
@@ -2321,8 +2321,14 @@ pcie0: pcie@1c00000 {
 			      <0 0x60000000 0 0xf1d>,
 			      <0 0x60000f20 0 0xa8>,
 			      <0 0x60001000 0 0x1000>,
-			      <0 0x60100000 0 0x100000>;
-			reg-names = "parf", "dbi", "elbi", "atu", "config";
+			      <0 0x60100000 0 0x100000>,
+			      <0 0x01c03000 0 0x1000>;
+			reg-names = "parf",
+				    "dbi",
+				    "elbi",
+				    "atu",
+				    "config",
+				    "mhi";
 			#address-cells = <3>;
 			#size-cells = <2>;
 			ranges = <0x01000000 0x0 0x00000000 0x0 0x60200000 0x0 0x100000>,
@@ -2449,7 +2455,7 @@ opp-16000000-3 {
 				};
 			};
 
-			pcieport0: pcie@0 {
+			pcie0_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
@@ -2495,8 +2501,14 @@ pcie1: pcie@1c08000 {
 			      <0x0 0x40000000 0x0 0xf1d>,
 			      <0x0 0x40000f20 0x0 0xa8>,
 			      <0x0 0x40001000 0x0 0x1000>,
-			      <0x0 0x40100000 0x0 0x100000>;
-			reg-names = "parf", "dbi", "elbi", "atu", "config";
+			      <0x0 0x40100000 0x0 0x100000>,
+			      <0x0 0x01c0b000 0x0 0x1000>;
+			reg-names = "parf",
+				    "dbi",
+				    "elbi",
+				    "atu",
+				    "config",
+				    "mhi";
 			#address-cells = <3>;
 			#size-cells = <2>;
 			ranges = <0x01000000 0x0 0x00000000 0x0 0x40200000 0x0 0x100000>,
@@ -2645,7 +2657,7 @@ opp-32000000-4 {
 				};
 			};
 
-			pcie@0 {
+			pcie1_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
@@ -3079,6 +3091,8 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
 			qcom,smem-state-names = "ipa-clock-enabled-valid",
 						"ipa-clock-enabled";
 
+			qcom,gsi-loader = "self";
+			memory-region = <&ipa_fw_mem>;
 			sram = <&ipa_modem_tables>;
 
 			status = "disabled";
@@ -4606,7 +4620,7 @@ usb_1_dwc3_ss: endpoint {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sm8550-pdc", "qcom,pdc";
-			reg = <0 0x0b220000 0 0x30000>, <0 0x174000f0 0 0x64>;
+			reg = <0 0x0b220000 0 0x10000>, <0 0x174000f0 0 0x64>;
 			qcom,pdc-ranges = <0 480 94>, <94 609 31>,
 					  <125 63 1>, <126 716 12>,
 					  <138 251 5>;
diff --git a/arch/arm64/boot/dts/qcom/sm8650-hdk.dts b/arch/arm64/boot/dts/qcom/sm8650-hdk.dts
index eabc828c05b4c..80bd2437f154f 100644
--- a/arch/arm64/boot/dts/qcom/sm8650-hdk.dts
+++ b/arch/arm64/boot/dts/qcom/sm8650-hdk.dts
@@ -888,9 +888,8 @@ lt9611_out: endpoint {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	firmware-name = "qcom/sm8650/ipa_fws.mbn";
+
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sm8650-qrd.dts b/arch/arm64/boot/dts/qcom/sm8650-qrd.dts
index a3982ae229290..a28d70c85fa37 100644
--- a/arch/arm64/boot/dts/qcom/sm8650-qrd.dts
+++ b/arch/arm64/boot/dts/qcom/sm8650-qrd.dts
@@ -861,9 +861,8 @@ redriver_ss_in: endpoint {
 };
 
 &ipa {
-	qcom,gsi-loader = "self";
-	memory-region = <&ipa_fw_mem>;
 	firmware-name = "qcom/sm8650/ipa_fws.mbn";
+
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sm8650.dtsi b/arch/arm64/boot/dts/qcom/sm8650.dtsi
index 160ead25ecf74..9ecbc0010694f 100644
--- a/arch/arm64/boot/dts/qcom/sm8650.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8650.dtsi
@@ -75,6 +75,11 @@ cpu0: cpu@0 {
 			compatible = "arm,cortex-a520";
 			reg = <0 0>;
 
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+
 			clocks = <&cpufreq_hw 0>;
 
 			power-domains = <&cpu_pd0>;
@@ -103,11 +108,15 @@ l2_0: l2-cache {
 				cache-level = <2>;
 				cache-unified;
 				next-level-cache = <&l3_0>;
+				cache-size = <524288>;
+				cache-line-size = <64>;
 
 				l3_0: l3-cache {
 					compatible = "cache";
 					cache-level = <3>;
 					cache-unified;
+					cache-size = <12582912>;
+					cache-line-size = <64>;
 				};
 			};
 		};
@@ -117,6 +126,11 @@ cpu1: cpu@100 {
 			compatible = "arm,cortex-a520";
 			reg = <0 0x100>;
 
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+
 			clocks = <&cpufreq_hw 0>;
 
 			power-domains = <&cpu_pd1>;
@@ -146,6 +160,11 @@ cpu2: cpu@200 {
 			compatible = "arm,cortex-a720";
 			reg = <0 0x200>;
 
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+
 			clocks = <&cpufreq_hw 3>;
 
 			power-domains = <&cpu_pd2>;
@@ -153,7 +172,7 @@ cpu2: cpu@200 {
 
 			enable-method = "psci";
 			next-level-cache = <&l2_200>;
-			capacity-dmips-mhz = <1792>;
+			capacity-dmips-mhz = <2909>;
 			dynamic-power-coefficient = <238>;
 
 			qcom,freq-domain = <&cpufreq_hw 3>;
@@ -174,6 +193,8 @@ l2_200: l2-cache {
 				cache-level = <2>;
 				cache-unified;
 				next-level-cache = <&l3_0>;
+				cache-size = <524288>;
+				cache-line-size = <64>;
 			};
 		};
 
@@ -182,6 +203,11 @@ cpu3: cpu@300 {
 			compatible = "arm,cortex-a720";
 			reg = <0 0x300>;
 
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+
 			clocks = <&cpufreq_hw 3>;
 
 			power-domains = <&cpu_pd3>;
@@ -189,7 +215,7 @@ cpu3: cpu@300 {
 
 			enable-method = "psci";
 			next-level-cache = <&l2_300>;
-			capacity-dmips-mhz = <1792>;
+			capacity-dmips-mhz = <2909>;
 			dynamic-power-coefficient = <238>;
 
 			qcom,freq-domain = <&cpufreq_hw 3>;
@@ -210,6 +236,8 @@ l2_300: l2-cache {
 				cache-level = <2>;
 				cache-unified;
 				next-level-cache = <&l3_0>;
+				cache-size = <524288>;
+				cache-line-size = <64>;
 			};
 		};
 
@@ -218,6 +246,11 @@ cpu4: cpu@400 {
 			compatible = "arm,cortex-a720";
 			reg = <0 0x400>;
 
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+
 			clocks = <&cpufreq_hw 3>;
 
 			power-domains = <&cpu_pd4>;
@@ -225,7 +258,7 @@ cpu4: cpu@400 {
 
 			enable-method = "psci";
 			next-level-cache = <&l2_400>;
-			capacity-dmips-mhz = <1792>;
+			capacity-dmips-mhz = <2909>;
 			dynamic-power-coefficient = <238>;
 
 			qcom,freq-domain = <&cpufreq_hw 3>;
@@ -246,6 +279,8 @@ l2_400: l2-cache {
 				cache-level = <2>;
 				cache-unified;
 				next-level-cache = <&l3_0>;
+				cache-size = <524288>;
+				cache-line-size = <64>;
 			};
 		};
 
@@ -254,6 +289,11 @@ cpu5: cpu@500 {
 			compatible = "arm,cortex-a720";
 			reg = <0 0x500>;
 
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+
 			clocks = <&cpufreq_hw 1>;
 
 			power-domains = <&cpu_pd5>;
@@ -261,7 +301,7 @@ cpu5: cpu@500 {
 
 			enable-method = "psci";
 			next-level-cache = <&l2_500>;
-			capacity-dmips-mhz = <1792>;
+			capacity-dmips-mhz = <2909>;
 			dynamic-power-coefficient = <238>;
 
 			qcom,freq-domain = <&cpufreq_hw 1>;
@@ -282,6 +322,8 @@ l2_500: l2-cache {
 				cache-level = <2>;
 				cache-unified;
 				next-level-cache = <&l3_0>;
+				cache-size = <524288>;
+				cache-line-size = <64>;
 			};
 		};
 
@@ -290,6 +332,11 @@ cpu6: cpu@600 {
 			compatible = "arm,cortex-a720";
 			reg = <0 0x600>;
 
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+
 			clocks = <&cpufreq_hw 1>;
 
 			power-domains = <&cpu_pd6>;
@@ -297,7 +344,7 @@ cpu6: cpu@600 {
 
 			enable-method = "psci";
 			next-level-cache = <&l2_600>;
-			capacity-dmips-mhz = <1792>;
+			capacity-dmips-mhz = <2909>;
 			dynamic-power-coefficient = <238>;
 
 			qcom,freq-domain = <&cpufreq_hw 1>;
@@ -318,6 +365,8 @@ l2_600: l2-cache {
 				cache-level = <2>;
 				cache-unified;
 				next-level-cache = <&l3_0>;
+				cache-size = <524288>;
+				cache-line-size = <64>;
 			};
 		};
 
@@ -326,6 +375,11 @@ cpu7: cpu@700 {
 			compatible = "arm,cortex-x4";
 			reg = <0 0x700>;
 
+			i-cache-size = <65536>;
+			i-cache-line-size = <64>;
+			d-cache-size = <65536>;
+			d-cache-line-size = <64>;
+
 			clocks = <&cpufreq_hw 2>;
 
 			power-domains = <&cpu_pd7>;
@@ -333,7 +387,7 @@ cpu7: cpu@700 {
 
 			enable-method = "psci";
 			next-level-cache = <&l2_700>;
-			capacity-dmips-mhz = <1894>;
+			capacity-dmips-mhz = <3591>;
 			dynamic-power-coefficient = <588>;
 
 			qcom,freq-domain = <&cpufreq_hw 2>;
@@ -354,6 +408,8 @@ l2_700: l2-cache {
 				cache-level = <2>;
 				cache-unified;
 				next-level-cache = <&l3_0>;
+				cache-size = <2097152>;
+				cache-line-size = <64>;
 			};
 		};
 
@@ -4397,6 +4453,9 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
 			qcom,smem-state-names = "ipa-clock-enabled-valid",
 						"ipa-clock-enabled";
 
+			qcom,gsi-loader = "self";
+			memory-region = <&ipa_fw_mem>;
+
 			sram = <&ipa_modem_tables>;
 
 			status = "disabled";
@@ -4678,18 +4737,18 @@ swr1: soundwire@6ad0000 {
 			pinctrl-0 = <&rx_swr_active>;
 			pinctrl-names = "default";
 
-			qcom,din-ports = <0>;
+			qcom,din-ports = <1>;
 			qcom,dout-ports = <11>;
 
-			qcom,ports-sinterval =		/bits/ 16 <0x03 0x1f 0x1f 0x07 0x03 0xff 0xff 0x31 0xff 0xff 0xff>;
-			qcom,ports-offset1 =		/bits/ 8 <0x00 0x00 0x0b 0x09 0x01 0xff 0xff 0x00 0xff 0xff 0xff>;
-			qcom,ports-offset2 =		/bits/ 8 <0x00 0x00 0x0b 0x00 0x00 0xff 0xff 0x00 0xff 0xff 0xff>;
-			qcom,ports-hstart =		/bits/ 8 <0xff 0x03 0xff 0xff 0xff 0xff 0xff 0x00 0xff 0xff 0xff>;
-			qcom,ports-hstop =		/bits/ 8 <0xff 0x06 0xff 0xff 0xff 0xff 0xff 0x0f 0xff 0xff 0xff>;
-			qcom,ports-word-length =	/bits/ 8 <0x01 0x07 0x04 0xff 0xff 0xff 0xff 0x18 0xff 0xff 0xff>;
-			qcom,ports-block-pack-mode =	/bits/ 8 <0xff 0x00 0x01 0xff 0xff 0xff 0xff 0x01 0xff 0xff 0xff>;
-			qcom,ports-block-group-count =	/bits/ 8 <0xff 0xff 0xff 0x01 0x03 0xff 0xff 0x00 0xff 0xff 0xff>;
-			qcom,ports-lane-control =	/bits/ 8 <0x01 0x00 0x00 0x00 0x00 0xff 0xff 0x01 0xff 0xff 0xff>;
+			qcom,ports-sinterval =		/bits/ 16 <0x03 0x1f 0x1f 0x07 0x03 0xff 0xff 0x31 0xff 0xff 0xff 0xff>;
+			qcom,ports-offset1 =		/bits/ 8 <0x00 0x00 0x0b 0x09 0x01 0xff 0xff 0x00 0xff 0xff 0xff 0xff>;
+			qcom,ports-offset2 =		/bits/ 8 <0x00 0x00 0x0b 0x00 0x00 0xff 0xff 0x00 0xff 0xff 0xff 0xff>;
+			qcom,ports-hstart =		/bits/ 8 <0xff 0x03 0xff 0xff 0xff 0xff 0xff 0x00 0xff 0xff 0xff 0xff>;
+			qcom,ports-hstop =		/bits/ 8 <0xff 0x06 0xff 0xff 0xff 0xff 0xff 0x0f 0xff 0xff 0xff 0xff>;
+			qcom,ports-word-length =	/bits/ 8 <0x01 0x07 0x04 0xff 0xff 0xff 0xff 0x18 0xff 0xff 0xff 0xff>;
+			qcom,ports-block-pack-mode =	/bits/ 8 <0xff 0x00 0x01 0xff 0xff 0xff 0xff 0x01 0xff 0xff 0xff 0xff>;
+			qcom,ports-block-group-count =	/bits/ 8 <0xff 0xff 0xff 0x01 0x03 0xff 0xff 0x00 0xff 0xff 0xff 0xff>;
+			qcom,ports-lane-control =	/bits/ 8 <0x01 0x00 0x00 0x00 0x00 0xff 0xff 0x01 0xff 0xff 0xff 0xff>;
 
 			#address-cells = <2>;
 			#size-cells = <0>;
@@ -4775,17 +4834,17 @@ swr2: soundwire@6d30000 {
 			pinctrl-names = "default";
 
 			qcom,din-ports = <4>;
-			qcom,dout-ports = <0>;
-
-			qcom,ports-sinterval-low =	/bits/ 8 <0x01 0x01 0x03 0x03>;
-			qcom,ports-offset1 =		/bits/ 8 <0x00 0x00 0x01 0x01>;
-			qcom,ports-offset2 =		/bits/ 8 <0x00 0x00 0x00 0x00>;
-			qcom,ports-hstart =		/bits/ 8 <0xff 0xff 0xff 0xff>;
-			qcom,ports-hstop =		/bits/ 8 <0xff 0xff 0xff 0xff>;
-			qcom,ports-word-length =	/bits/ 8 <0xff 0xff 0xff 0xff>;
-			qcom,ports-block-pack-mode =	/bits/ 8 <0xff 0xff 0xff 0xff>;
-			qcom,ports-block-group-count =	/bits/ 8 <0xff 0xff 0xff 0xff>;
-			qcom,ports-lane-control =	/bits/ 8 <0x01 0x02 0x00 0x00>;
+			qcom,dout-ports = <1>;
+
+			qcom,ports-sinterval-low =	/bits/ 8 <0x00 0x01 0x01 0x03 0x03>;
+			qcom,ports-offset1 =		/bits/ 8 <0x00 0x00 0x00 0x01 0x01>;
+			qcom,ports-offset2 =		/bits/ 8 <0x00 0x00 0x00 0x00 0x00>;
+			qcom,ports-hstart =		/bits/ 8 <0xff 0xff 0xff 0xff 0xff>;
+			qcom,ports-hstop =		/bits/ 8 <0xff 0xff 0xff 0xff 0xff>;
+			qcom,ports-word-length =	/bits/ 8 <0xff 0xff 0xff 0xff 0xff>;
+			qcom,ports-block-pack-mode =	/bits/ 8 <0xff 0xff 0xff 0xff 0xff>;
+			qcom,ports-block-group-count =	/bits/ 8 <0xff 0xff 0xff 0xff 0xff>;
+			qcom,ports-lane-control =	/bits/ 8 <0xff 0x01 0x02 0x00 0x00>;
 
 			#address-cells = <2>;
 			#size-cells = <0>;
@@ -5872,11 +5931,15 @@ mdss_dsi1_phy: phy@ae97000 {
 
 			mdss_dp0: displayport-controller@af54000 {
 				compatible = "qcom,sm8650-dp";
-				reg = <0 0xaf54000 0 0x104>,
-				      <0 0xaf54200 0 0xc0>,
-				      <0 0xaf55000 0 0x770>,
-				      <0 0xaf56000 0 0x9c>,
-				      <0 0xaf57000 0 0x9c>;
+				reg = <0x0 0x0af54000 0x0 0x200>,
+				      <0x0 0x0af54200 0x0 0x200>,
+				      <0x0 0x0af55000 0x0 0xc00>,
+				      <0x0 0x0af56000 0x0 0x400>,
+				      <0x0 0x0af57000 0x0 0x400>,
+				      <0x0 0x0af58000 0x0 0x400>,
+				      <0x0 0x0af59000 0x0 0x400>,
+				      <0x0 0x0af5a000 0x0 0x600>,
+				      <0x0 0x0af5b000 0x0 0x600>;
 
 				interrupts-extended = <&mdss 12>;
 
@@ -5991,7 +6054,7 @@ dispcc: clock-controller@af00000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sm8650-pdc", "qcom,pdc";
-			reg = <0 0x0b220000 0 0x30000>, <0 0x174000f0 0 0x64>;
+			reg = <0 0x0b220000 0 0x10000>, <0 0x174000f0 0 0x64>;
 
 			interrupt-parent = <&intc>;
 
@@ -7091,8 +7154,8 @@ sram@14680000 {
 
 			ranges = <0 0 0x14680000 0x2c000>;
 
-			ipa_modem_tables: modem-tables@8000 {
-				reg = <0x8000 0x2000>;
+			ipa_modem_tables: modem-tables@3000 {
+				reg = <0x3000 0x2000>;
 			};
 		};
 
diff --git a/arch/arm64/boot/dts/qcom/sm8750.dtsi b/arch/arm64/boot/dts/qcom/sm8750.dtsi
index fafed417c66fc..3496f00ac4f55 100644
--- a/arch/arm64/boot/dts/qcom/sm8750.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8750.dtsi
@@ -1218,6 +1218,28 @@ &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
 
 				status = "disabled";
 			};
+
+			uart15: serial@89c000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x0089c000 0x0 0x4000>;
+
+				interrupts = <GIC_SPI 462 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP2_S7_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart15_default>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
 		};
 
 		i2c_master_hub_0: geniqup@9c0000 {
@@ -2085,12 +2107,12 @@ aggre2_noc: interconnect@1700000 {
 		};
 
 		cambistmclkcc: clock-controller@1760000 {
-		       compatible = "qcom,sm8750-cambistmclkcc";
-		       reg = <0x0 0x1760000 0x0 0x6000>;
-		       clocks = <&gcc GCC_CAM_BIST_MCLK_AHB_CLK>,
-				<&bi_tcxo_div2>,
-				<&bi_tcxo_ao_div2>,
-				<&sleep_clk>;
+			compatible = "qcom,sm8750-cambistmclkcc";
+			reg = <0x0 0x1760000 0x0 0x6000>;
+			clocks = <&gcc GCC_CAM_BIST_MCLK_AHB_CLK>,
+				 <&bi_tcxo_div2>,
+				 <&bi_tcxo_ao_div2>,
+				 <&sleep_clk>;
 			power-domains = <&rpmhpd RPMHPD_MMCX>,
 					<&rpmhpd RPMHPD_MX>;
 			required-opps = <&rpmhpd_opp_low_svs>,
@@ -2893,6 +2915,7 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
 
 			dma-coherent;
 			usb-role-switch;
+			wakeup-source;
 
 			status = "disabled";
 
@@ -3073,7 +3096,7 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
 			mdss_mdp: display-controller@ae01000 {
 				compatible = "qcom,sm8750-dpu";
 				reg = <0x0 0x0ae01000 0x0 0x93000>,
-				      <0x0 0x0aeb0000 0x0 0x2008>;
+				      <0x0 0x0aeb0000 0x0 0x3000>;
 				reg-names = "mdp",
 					    "vbif";
 
@@ -3357,11 +3380,15 @@ mdss_dsi1_phy: phy@ae97000 {
 
 			mdss_dp0: displayport-controller@af54000 {
 				compatible = "qcom,sm8750-dp", "qcom,sm8650-dp";
-				reg = <0x0 0xaf54000 0x0 0x104>,
-				      <0x0 0xaf54200 0x0 0xc0>,
-				      <0x0 0xaf55000 0x0 0x770>,
-				      <0x0 0xaf56000 0x0 0x9c>,
-				      <0x0 0xaf57000 0x0 0x9c>;
+				reg = <0x0 0x0af54000 0x0 0x200>,
+				      <0x0 0x0af54200 0x0 0x200>,
+				      <0x0 0x0af55000 0x0 0xc00>,
+				      <0x0 0x0af56000 0x0 0x400>,
+				      <0x0 0x0af57000 0x0 0x400>,
+				      <0x0 0x0af58000 0x0 0x400>,
+				      <0x0 0x0af59000 0x0 0x400>,
+				      <0x0 0x0af5a000 0x0 0x600>,
+				      <0x0 0x0af5b000 0x0 0x600>;
 
 				interrupts-extended = <&mdss 12>;
 
@@ -4055,6 +4082,14 @@ qup_uart14_cts_rts: qup-uart14-cts-rts-state {
 				bias-pull-down;
 			};
 
+			qup_uart15_default: qup-uart15-default-state {
+				/* TX, RX */
+				pins = "gpio30", "gpio31";
+				function = "qup2_se7";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
 			sdc2_sleep: sdc2-sleep-state {
 				clk-pins {
 					pins = "sdc2_clk";
@@ -4106,6 +4141,35 @@ tcsrcc: clock-controller@f204008 {
 			#reset-cells = <1>;
 		};
 
+		ctcu@10001000 {
+			compatible = "qcom,sm8750-ctcu", "qcom,sa8775p-ctcu";
+			reg = <0x0 0x10001000 0x0 0x1000>;
+
+			clocks = <&aoss_qmp>;
+			clock-names = "apb";
+
+			in-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					ctcu_in0: endpoint {
+						remote-endpoint = <&etr0_out>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					ctcu_in1: endpoint {
+						remote-endpoint = <&etr1_out>;
+					};
+				};
+			};
+		};
+
 		stm@10002000 {
 			compatible = "arm,coresight-stm", "arm,primecell";
 			reg = <0x0 0x10002000 0x0 0x1000>,
@@ -4161,6 +4225,7 @@ tpdm@1000f000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_spdm";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -4219,12 +4284,129 @@ funnel_in0_out: endpoint {
 			};
 		};
 
+		replicator@10046000 {
+			compatible = "arm,coresight-dynamic-replicator", "arm,primecell";
+			reg = <0x0 0x10046000 0x0 0x1000>;
+
+			clocks = <&aoss_qmp>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				port {
+					qdss_rep_in: endpoint {
+						remote-endpoint = <&swao_rep_out0>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					qdss_rep_out0: endpoint {
+						remote-endpoint = <&etr_rep_in>;
+					};
+				};
+			};
+		};
+
+		tmc@10048000 {
+			compatible = "arm,coresight-tmc", "arm,primecell";
+			reg = <0x0 0x10048000 0x0 0x1000>;
+
+			clocks = <&aoss_qmp>;
+			clock-names = "apb_pclk";
+
+			iommus = <&apps_smmu 0x04e0 0x0>;
+			arm,scatter-gather;
+
+			in-ports {
+				port {
+					etr0_in: endpoint {
+						remote-endpoint = <&etr_rep_out0>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					etr0_out: endpoint {
+						remote-endpoint = <&ctcu_in0>;
+					};
+				};
+			};
+		};
+
+		replicator@1004e000 {
+			compatible = "arm,coresight-dynamic-replicator", "arm,primecell";
+			reg = <0x0 0x1004e000 0x0 0x1000>;
+
+			clocks = <&aoss_qmp>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				port {
+					etr_rep_in: endpoint {
+						remote-endpoint = <&qdss_rep_out0>;
+					};
+				};
+			};
+
+			out-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					etr_rep_out0: endpoint {
+						remote-endpoint = <&etr0_in>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					etr_rep_out1: endpoint {
+						remote-endpoint = <&etr1_in>;
+					};
+				};
+			};
+		};
+
+		tmc_etr1: tmc@1004f000 {
+			compatible = "arm,coresight-tmc", "arm,primecell";
+			reg = <0x0 0x1004f000 0x0 0x1000>;
+
+			clocks = <&aoss_qmp>;
+			clock-names = "apb_pclk";
+
+			iommus = <&apps_smmu 0x0500 0x0>;
+			arm,scatter-gather;
+			arm,buffer-size = <0x400000>;
+
+			in-ports {
+				port {
+					etr1_in: endpoint {
+						remote-endpoint = <&etr_rep_out1>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					etr1_out: endpoint {
+						remote-endpoint = <&ctcu_in1>;
+					};
+				};
+			};
+		};
+
 		tpdm@10800000 {
 			compatible = "qcom,coresight-tpdm", "arm,primecell";
 			reg = <0x0 0x10800000 0x0 0x1000>;
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_modem_0";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -4305,6 +4487,7 @@ cti@1080b000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_mss_qdsp6";
 		};
 
 		tpdm@1082c000 {
@@ -4313,6 +4496,7 @@ tpdm@1082c000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_gcc";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -4331,6 +4515,7 @@ tpdm@10841000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_prng";
 
 			qcom,cmb-msrs-num = <32>;
 
@@ -4349,6 +4534,7 @@ tpdm@1084e000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_mm_bcv";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -4368,6 +4554,7 @@ tpdm@1084f000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_mm_lmh";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -4387,6 +4574,7 @@ tpdm@10850000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_mm_dpm";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -4451,6 +4639,7 @@ tpdm@10980000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_cdsp";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -4539,6 +4728,7 @@ cti@1098b000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_cdsp_qdsp";
 		};
 
 		tpdm@109a3000 {
@@ -4547,6 +4737,7 @@ tpdm@109a3000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_pmu";
 
 			qcom,cmb-msrs-num = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -4566,6 +4757,7 @@ tpdm@109a4000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_ipcc";
 
 			qcom,cmb-msrs-num = <32>;
 
@@ -4584,6 +4776,7 @@ tpdm@109a5000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_dlmm";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -4602,6 +4795,7 @@ tpdm@109a6000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_north_dsb";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -4620,6 +4814,7 @@ tpdm@109a7000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_south_dsb";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -4638,6 +4833,7 @@ tpdm@109a8000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_rdpm_cmb0";
 
 			qcom,cmb-msrs-num = <32>;
 
@@ -4656,6 +4852,7 @@ tpdm@109a9000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_rdpm_cmb1";
 
 			qcom,cmb-msrs-num = <32>;
 
@@ -4674,6 +4871,7 @@ tpdm@109aa000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_rdpm_cmb2";
 
 			qcom,cmb-msrs-num = <32>;
 
@@ -4825,6 +5023,7 @@ tpdm@109d0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_qm";
 
 			qcom,dsb-msrs-num = <32>;
 
@@ -4889,6 +5088,38 @@ tmc_etf_in: endpoint {
 					};
 				};
 			};
+
+			out-ports {
+				port {
+					tmc_etf_out: endpoint {
+						remote-endpoint = <&swao_rep_in>;
+					};
+				};
+			};
+		};
+
+		replicator@10b06000 {
+			compatible = "arm,coresight-dynamic-replicator", "arm,primecell";
+			reg = <0x0 0x10b06000 0x0 0x1000>;
+
+			clocks = <&aoss_qmp>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				port {
+					swao_rep_in: endpoint {
+						remote-endpoint = <&tmc_etf_out>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					swao_rep_out0: endpoint {
+						remote-endpoint = <&qdss_rep_in>;
+					};
+				};
+			};
 		};
 
 		tpda@10b08000 {
@@ -4938,7 +5169,7 @@ port@4 {
 					reg = <4>;
 
 					tpda_aoss_in4: endpoint {
-						remote-endpoint =<&tpdm_swao_out>;
+						remote-endpoint = <&tpdm_swao_out>;
 					};
 				};
 			};
@@ -4958,6 +5189,7 @@ tpdm@10b09000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_0";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -4977,6 +5209,7 @@ tpdm@10b0a000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_1";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -4996,6 +5229,7 @@ tpdm@10b0b000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_2";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -5015,6 +5249,7 @@ tpdm@10b0c000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_prio_3";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -5034,6 +5269,7 @@ tpdm@10b0d000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_1";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -5445,6 +5681,7 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
 			phy-names = "ufsphy";
 
 			#reset-cells = <1>;
+			qcom,ice = <&ice>;
 
 			status = "disabled";
 
@@ -6850,6 +7087,7 @@ timer {
 
 	tpdm-cdsp-llm {
 		compatible = "qcom,coresight-static-tpdm";
+		label = "tpdm_cdsp_llm";
 		qcom,cmb-element-bits = <32>;
 
 		out-ports {
@@ -6863,6 +7101,7 @@ tpdm_cdsp_llm_out: endpoint {
 
 	tpdm-cdsp-llm2 {
 		compatible = "qcom,coresight-static-tpdm";
+		label = "tpdm_cdsp_llm2";
 		qcom,cmb-element-bits = <32>;
 
 		out-ports {
@@ -6876,6 +7115,7 @@ tpdm_cdsp_llm2_out: endpoint {
 
 	tpdm-modem1 {
 		compatible = "qcom,coresight-static-tpdm";
+		label = "tpdm_modem_1";
 		qcom,dsb-element-bits = <32>;
 
 		out-ports {
diff --git a/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi b/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi
index 294354c034c37..6d2cff3cf23cc 100644
--- a/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi
+++ b/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi
@@ -317,6 +317,14 @@ &gpi_dma1 {
 	status = "okay";
 };
 
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/qcs615/a612_zap.mbn";
+};
+
 &i2c5 {
 	clock-frequency = <400000>;
 	status = "okay";
diff --git a/arch/arm64/boot/dts/qcom/talos.dtsi b/arch/arm64/boot/dts/qcom/talos.dtsi
index fb1bbc51bb8a4..bdfb70ca97407 100644
--- a/arch/arm64/boot/dts/qcom/talos.dtsi
+++ b/arch/arm64/boot/dts/qcom/talos.dtsi
@@ -11,6 +11,7 @@
 #include <dt-bindings/clock/qcom,qcs615-videocc.h>
 #include <dt-bindings/clock/qcom,rpmh.h>
 #include <dt-bindings/dma/qcom-gpi.h>
+#include <dt-bindings/firmware/qcom,scm.h>
 #include <dt-bindings/interconnect/qcom,icc.h>
 #include <dt-bindings/interconnect/qcom,osm-l3.h>
 #include <dt-bindings/interconnect/qcom,qcs615-rpmh.h>
@@ -50,10 +51,10 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
 					<&osm_l3 MASTER_OSM_L3_APPS &osm_l3 SLAVE_OSM_L3>;
 
 			l2_0: l2-cache {
-			      compatible = "cache";
-			      cache-level = <2>;
-			      cache-unified;
-			      next-level-cache = <&l3_0>;
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3_0>;
 			};
 		};
 
@@ -75,10 +76,10 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
 					<&osm_l3 MASTER_OSM_L3_APPS &osm_l3 SLAVE_OSM_L3>;
 
 			l2_100: l2-cache {
-			      compatible = "cache";
-			      cache-level = <2>;
-			      cache-unified;
-			      next-level-cache = <&l3_0>;
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3_0>;
 			};
 		};
 
@@ -100,10 +101,10 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
 					<&osm_l3 MASTER_OSM_L3_APPS &osm_l3 SLAVE_OSM_L3>;
 
 			l2_200: l2-cache {
-			      compatible = "cache";
-			      cache-level = <2>;
-			      cache-unified;
-			      next-level-cache = <&l3_0>;
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3_0>;
 			};
 		};
 
@@ -125,10 +126,10 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
 					<&osm_l3 MASTER_OSM_L3_APPS &osm_l3 SLAVE_OSM_L3>;
 
 			l2_300: l2-cache {
-			      compatible = "cache";
-			      cache-level = <2>;
-			      cache-unified;
-			      next-level-cache = <&l3_0>;
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3_0>;
 			};
 		};
 
@@ -150,10 +151,10 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
 					<&osm_l3 MASTER_OSM_L3_APPS &osm_l3 SLAVE_OSM_L3>;
 
 			l2_400: l2-cache {
-			      compatible = "cache";
-			      cache-level = <2>;
-			      cache-unified;
-			      next-level-cache = <&l3_0>;
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3_0>;
 			};
 		};
 
@@ -175,10 +176,10 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
 					<&osm_l3 MASTER_OSM_L3_APPS &osm_l3 SLAVE_OSM_L3>;
 
 			l2_500: l2-cache {
-			      compatible = "cache";
-			      cache-level = <2>;
-			      cache-unified;
-			      next-level-cache = <&l3_0>;
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3_0>;
 			};
 		};
 
@@ -201,10 +202,10 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
 					<&osm_l3 MASTER_OSM_L3_APPS &osm_l3 SLAVE_OSM_L3>;
 
 			l2_600: l2-cache {
-			      compatible = "cache";
-			      cache-level = <2>;
-			      cache-unified;
-			      next-level-cache = <&l3_0>;
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3_0>;
 			};
 		};
 
@@ -226,10 +227,10 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
 					<&osm_l3 MASTER_OSM_L3_APPS &osm_l3 SLAVE_OSM_L3>;
 
 			l2_700: l2-cache {
-			      compatible = "cache";
-			      cache-level = <2>;
-			      cache-unified;
-			      next-level-cache = <&l3_0>;
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3_0>;
 			};
 		};
 
@@ -677,6 +678,14 @@ pil_gpu_mem: pil-gpu@97715000 {
 			reg = <0x0 0x97715000 0x0 0x2000>;
 			no-map;
 		};
+
+		adsp_rpc_remote_heap_mem: adsp-rpc-remote-heap {
+			compatible = "shared-dma-pool";
+			alloc-ranges = <0x0 0x80000000 0x0 0x80000000>;
+			reusable;
+			alignment = <0x0 0x400000>;
+			size = <0x0 0x800000>;
+		};
 	};
 
 	soc: soc@0 {
@@ -2017,7 +2026,7 @@ opp-435000000 {
 			};
 		};
 
-		gmu: gmu@506a000 {
+		gmu: gmu@506d000 {
 			compatible = "qcom,adreno-rgmu-612.0", "qcom,adreno-rgmu";
 			reg = <0x0 0x0506d000 0x0 0x2c000>;
 
@@ -2242,6 +2251,7 @@ cti@6010000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdss";
 		};
 
 		cti@6011000 {
@@ -2250,6 +2260,7 @@ cti@6011000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdss_1";
 		};
 
 		cti@6012000 {
@@ -2258,6 +2269,7 @@ cti@6012000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdss_2";
 		};
 
 		cti@6013000 {
@@ -2266,6 +2278,7 @@ cti@6013000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdss_3";
 		};
 
 		cti@6014000 {
@@ -2274,6 +2287,7 @@ cti@6014000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdss_4";
 		};
 
 		cti@6015000 {
@@ -2282,6 +2296,7 @@ cti@6015000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdss_5";
 		};
 
 		cti@6016000 {
@@ -2290,6 +2305,7 @@ cti@6016000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdss_6";
 		};
 
 		cti@6017000 {
@@ -2298,6 +2314,7 @@ cti@6017000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdss_7";
 		};
 
 		cti@6018000 {
@@ -2306,6 +2323,7 @@ cti@6018000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdss_8";
 		};
 
 		cti@6019000 {
@@ -2314,6 +2332,7 @@ cti@6019000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdss_9";
 		};
 
 		cti@601a000 {
@@ -2322,6 +2341,7 @@ cti@601a000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdss_10";
 		};
 
 		cti@601b000 {
@@ -2330,6 +2350,7 @@ cti@601b000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdss_11";
 		};
 
 		cti@601c000 {
@@ -2338,6 +2359,7 @@ cti@601c000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdss_12";
 		};
 
 		cti@601d000 {
@@ -2346,6 +2368,7 @@ cti@601d000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdss_13";
 		};
 
 		cti@601e000 {
@@ -2354,6 +2377,7 @@ cti@601e000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdss_14";
 		};
 
 		cti@601f000 {
@@ -2362,6 +2386,7 @@ cti@601f000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdss_15";
 		};
 
 		funnel@6041000 {
@@ -2594,6 +2619,7 @@ cti@683b000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_qdsp6";
 		};
 
 		tpdm@6840000 {
@@ -2602,6 +2628,7 @@ tpdm@6840000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_vsense";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -2622,6 +2649,7 @@ tpdm@684c000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_prng";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -2641,6 +2669,7 @@ tpdm@6850000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_pimem";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -2662,6 +2691,7 @@ tpdm@6860000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_cdsp";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -2705,6 +2735,7 @@ cti@6867000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_cdsp";
 		};
 
 		tpdm@6870000 {
@@ -2713,6 +2744,7 @@ tpdm@6870000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_dcc";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -2733,6 +2765,7 @@ tpdm@699c000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_wcss";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -2755,6 +2788,7 @@ tpdm@69c0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_monaq";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -2798,6 +2832,7 @@ tpdm@69d0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_qm";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -2818,6 +2853,7 @@ tpdm@6a00000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_ddr";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -2838,6 +2874,7 @@ cti@6a02000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_ddr_dl0";
 		};
 
 		cti@6a03000 {
@@ -2846,6 +2883,7 @@ cti@6a03000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_ddr_dl0_1";
 		};
 
 		cti@6a10000 {
@@ -2854,6 +2892,7 @@ cti@6a10000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_ddr_dl1";
 		};
 
 		cti@6a11000 {
@@ -2862,6 +2901,7 @@ cti@6a11000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_ddr_dl1_1";
 		};
 
 		funnel@6a05000 {
@@ -2932,6 +2972,7 @@ tpdm@6b02000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_0";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -2952,6 +2993,7 @@ tpdm@6b03000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_swao_1";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -2972,6 +3014,7 @@ cti@6b04000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_swao";
 		};
 
 		cti@6b05000 {
@@ -2980,6 +3023,7 @@ cti@6b05000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_swao_1";
 		};
 
 		cti@6b06000 {
@@ -2988,6 +3032,7 @@ cti@6b06000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_swao_2";
 		};
 
 		cti@6b07000 {
@@ -2996,6 +3041,7 @@ cti@6b07000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_swao_3";
 		};
 
 		funnel@6b08000 {
@@ -3102,6 +3148,7 @@ cti@6b21000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_aop_m3";
 		};
 
 		tpdm@6b48000 {
@@ -3110,6 +3157,7 @@ tpdm@6b48000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_west";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -3129,6 +3177,7 @@ cti@6c13000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_titan";
 
 			/* Not all required clocks can be enabled from the OS */
 			status = "fail";
@@ -3140,6 +3189,7 @@ cti@6c20000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_venus";
 			status = "disabled";
 		};
 
@@ -3149,6 +3199,7 @@ tpdm@6c28000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_center";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -3168,6 +3219,7 @@ cti@6c29000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_dlct";
 		};
 
 		cti@6c2a000 {
@@ -3176,6 +3228,7 @@ cti@6c2a000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_dlct_1";
 		};
 
 		cti@7020000 {
@@ -3184,6 +3237,7 @@ cti@7020000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_apss_apb_2";
 		};
 
 		etm@7040000 {
@@ -3212,6 +3266,7 @@ cti@7120000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_apss_apb_3";
 		};
 
 		etm@7140000 {
@@ -3240,6 +3295,7 @@ cti@7220000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_apss_apb_4";
 		};
 
 		etm@7240000 {
@@ -3268,6 +3324,7 @@ cti@7320000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_apss_apb_5";
 		};
 
 		etm@7340000 {
@@ -3296,6 +3353,7 @@ cti@7420000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_apss_apb_6";
 		};
 
 		etm@7440000 {
@@ -3324,6 +3382,7 @@ cti@7520000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_apss_apb_7";
 		};
 
 		etm@7540000 {
@@ -3352,6 +3411,7 @@ cti@7620000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_apss_apb_8";
 		};
 
 		etm@7640000 {
@@ -3380,6 +3440,7 @@ cti@7720000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_apss_apb_9";
 		};
 
 		etm@7740000 {
@@ -3554,6 +3615,7 @@ tpdm@7830000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_olc";
 
 			qcom,cmb-element-bits = <64>;
 			qcom,cmb-msrs-num = <32>;
@@ -3597,6 +3659,7 @@ tpdm@7860000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_apss";
 
 			qcom,dsb-element-bits = <32>;
 			qcom,dsb-msrs-num = <32>;
@@ -3640,6 +3703,7 @@ tpdm@78a0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_llm_silver";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -3659,6 +3723,7 @@ tpdm@78b0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "tpdm_llm_gold";
 
 			qcom,cmb-element-bits = <32>;
 			qcom,cmb-msrs-num = <32>;
@@ -3726,6 +3791,7 @@ cti@78e0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_apss";
 		};
 
 		cti@78f0000 {
@@ -3734,6 +3800,7 @@ cti@78f0000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_apss_1";
 		};
 
 		cti@7900000 {
@@ -3742,6 +3809,7 @@ cti@7900000 {
 
 			clocks = <&aoss_qmp>;
 			clock-names = "apb_pclk";
+			label = "cti_apss_2";
 		};
 
 		remoteproc_cdsp: remoteproc@8300000 {
@@ -4624,7 +4692,7 @@ dispcc: clock-controller@af00000 {
 
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,qcs615-pdc", "qcom,pdc";
-			reg = <0x0 0x0b220000 0x0 0x30000>,
+			reg = <0x0 0x0b220000 0x0 0x10000>,
 			      <0x0 0x17c000f0 0x0 0x64>;
 			qcom,pdc-ranges = <0 480 94>, <94 609 31>, <125 63 1>;
 			interrupt-parent = <&intc>;
@@ -5195,6 +5263,9 @@ fastrpc {
 					compatible = "qcom,fastrpc";
 					qcom,glink-channels = "fastrpcglink-apps-dsp";
 					label = "adsp";
+					memory-region = <&adsp_rpc_remote_heap_mem>;
+					qcom,vmids = <QCOM_SCM_VMID_LPASS
+						      QCOM_SCM_VMID_ADSP_HEAP>;
 					#address-cells = <1>;
 					#size-cells = <0>;
 
@@ -5401,6 +5472,7 @@ cpu-critical {
 
 		gpu-thermal {
 			thermal-sensors = <&tsens0 9>;
+			polling-delay-passive = <200>;
 
 			trips {
 				gpu_alert0: trip-point0 {
diff --git a/arch/arm64/boot/dts/qcom/x1-crd.dtsi b/arch/arm64/boot/dts/qcom/x1-crd.dtsi
index a9c5c523575ee..9602d65c8b3db 100644
--- a/arch/arm64/boot/dts/qcom/x1-crd.dtsi
+++ b/arch/arm64/boot/dts/qcom/x1-crd.dtsi
@@ -1074,6 +1074,16 @@ eusb6_repeater: redriver@4f {
 
 		#phy-cells = <0>;
 	};
+
+	embedded-controller@76 {
+		compatible = "qcom,hamoa-crd-ec";
+		reg = <0x76>;
+
+		interrupts-extended = <&tlmm 66 IRQ_TYPE_EDGE_FALLING>;
+
+		pinctrl-0 = <&ec_int_n_default>;
+		pinctrl-names = "default";
+	};
 };
 
 &i2c7 {
@@ -1523,6 +1533,12 @@ &tlmm {
 			       <44 4>, /* SPI (TPM) */
 			       <238 1>; /* UFS Reset */
 
+	ec_int_n_default: ec-int-n-state {
+		pins = "gpio66";
+		function = "gpio";
+		bias-disable;
+	};
+
 	edp_reg_en: edp-reg-en-state {
 		pins = "gpio70";
 		function = "gpio";
diff --git a/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi b/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi
index db291730130c7..d6de4da02dcd3 100644
--- a/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi
+++ b/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi
@@ -589,6 +589,7 @@ vreg_l12b_1p2: ldo12 {
 			regulator-min-microvolt = <1200000>;
 			regulator-max-microvolt = <1200000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
 		};
 
 		vreg_l13b_3p0: ldo13 {
@@ -610,6 +611,7 @@ vreg_l15b_1p8: ldo15 {
 			regulator-min-microvolt = <1800000>;
 			regulator-max-microvolt = <1800000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
 		};
 	};
 
diff --git a/arch/arm64/boot/dts/qcom/x1e80100-honor-magicbook-art-14.dts b/arch/arm64/boot/dts/qcom/x1e80100-honor-magicbook-art-14.dts
new file mode 100644
index 0000000000000..b70c1e094bbf2
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/x1e80100-honor-magicbook-art-14.dts
@@ -0,0 +1,1341 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2025 Kirill A. Korinsky <kirill@korins.ky>
+ * Copyright (c) 2025 Valentin Manea <valentin.manea@mrs.ro>
+ * Copyright (c) 2026 Konstantin Shabanov <mail@etehtsea.me>
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/gpio-keys.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+
+#include "hamoa.dtsi"
+#include "hamoa-pmics.dtsi"
+
+/ {
+	model = "HONOR MagicBook Art 14 Snapdragon";
+	compatible = "honor,magicbook-art-14-snapdragon", "qcom,x1e80100";
+	chassis-type = "laptop";
+
+	aliases {
+		serial0 = &uart21;
+		serial1 = &uart14;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&hall_int_n_default>;
+		pinctrl-names = "default";
+
+		switch-lid {
+			gpios = <&tlmm 92 GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_SW>;
+			linux,code = <SW_LID>;
+			wakeup-source;
+			wakeup-event-action = <EV_ACT_DEASSERTED>;
+		};
+	};
+
+	hdmi-bridge {
+		compatible = "realtek,rtd2171";
+
+		pinctrl-0 = <&hdmi_hpd_default>;
+		pinctrl-names = "default";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				hdmi_bridge_dp_in: endpoint {
+					remote-endpoint = <&usb_1_ss2_qmpphy_out_dp>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				hdmi_bridge_tmds_out: endpoint {
+					remote-endpoint = <&hdmi_con>;
+				};
+			};
+		};
+	};
+
+	hdmi-connector {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con: endpoint {
+				remote-endpoint = <&hdmi_bridge_tmds_out>;
+			};
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,x1e80100-pmic-glink",
+			     "qcom,sm8550-pmic-glink",
+			     "qcom,pmic-glink";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		orientation-gpios = <&tlmm 121 GPIO_ACTIVE_HIGH>,
+				    <&tlmm 123 GPIO_ACTIVE_HIGH>;
+
+		/* Left-side port, closer to the screen */
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_ss0_hs_in: endpoint {
+						remote-endpoint = <&usb_1_ss0_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss0_ss_in: endpoint {
+						remote-endpoint = <&usb_1_ss0_qmpphy_out>;
+					};
+				};
+			};
+		};
+
+		/* Left-side port, farther from the screen */
+		connector@1 {
+			compatible = "usb-c-connector";
+			reg = <1>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_ss1_hs_in: endpoint {
+						remote-endpoint = <&usb_1_ss1_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss1_ss_in: endpoint {
+						remote-endpoint = <&usb_1_ss1_qmpphy_out>;
+					};
+				};
+			};
+		};
+	};
+
+	reserved-memory {
+		linux,cma {
+			compatible = "shared-dma-pool";
+			size = <0x0 0x8000000>;
+			reusable;
+			linux,cma-default;
+		};
+	};
+
+	sound {
+		compatible = "qcom,x1e80100-sndcard";
+		model = "X1E80100-HONOR-MagicBook-Art-14-Snapdragon";
+		audio-routing = "WooferLeft IN", "WSA WSA_SPK1 OUT",
+				"TweeterLeft IN", "WSA WSA_SPK2 OUT",
+				"WooferRight IN", "WSA2 WSA_SPK2 OUT",
+				"TweeterRight IN", "WSA2 WSA_SPK2 OUT",
+				"IN1_HPHL", "HPHL_OUT",
+				"IN2_HPHR", "HPHR_OUT",
+				"AMIC2", "MIC BIAS2",
+				"VA DMIC0", "MIC BIAS3",
+				"VA DMIC1", "MIC BIAS3",
+				"VA DMIC2", "MIC BIAS1",
+				"VA DMIC3", "MIC BIAS1",
+				"TX SWR_INPUT1", "ADC2_OUTPUT";
+
+		wcd-playback-dai-link {
+			link-name = "WCD Playback";
+
+			codec {
+				sound-dai = <&wcd938x 0>, <&swr1 0>, <&lpass_rxmacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai RX_CODEC_DMA_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		wcd-capture-dai-link {
+			link-name = "WCD Capture";
+
+			codec {
+				sound-dai = <&wcd938x 1>, <&swr2 1>, <&lpass_txmacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai TX_CODEC_DMA_TX_3>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		wsa-dai-link {
+			link-name = "WSA Playback";
+
+			codec {
+				sound-dai = <&left_woofer>, <&left_tweeter>,
+					    <&swr0 0>, <&lpass_wsamacro 0>,
+					    <&right_woofer>, <&right_tweeter>,
+					    <&swr3 0>, <&lpass_wsa2macro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai WSA_CODEC_DMA_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		va-dai-link {
+			link-name = "VA Capture";
+
+			codec {
+				sound-dai = <&lpass_vamacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai VA_CODEC_DMA_TX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+	};
+
+	vreg_misc_3p3: regulator-misc-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_MISC_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pm8550ve_8_gpios 6 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&misc_3p3_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vreg_wcn_0p95: regulator-wcn-0p95 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_0P95";
+		regulator-min-microvolt = <950000>;
+		regulator-max-microvolt = <950000>;
+
+		vin-supply = <&vreg_wcn_3p3>;
+	};
+
+	vreg_wcn_1p9: regulator-wcn-1p9 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_1P9";
+		regulator-min-microvolt = <1900000>;
+		regulator-max-microvolt = <1900000>;
+
+		vin-supply = <&vreg_wcn_3p3>;
+	};
+
+	vreg_wcn_3p3: regulator-wcn-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 214 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&wcn_sw_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	wcd938x: audio-codec {
+		compatible = "qcom,wcd9385-codec";
+
+		pinctrl-0 = <&wcd_default>;
+		pinctrl-names = "default";
+
+		qcom,micbias1-microvolt = <1800000>;
+		qcom,micbias2-microvolt = <1800000>;
+		qcom,micbias3-microvolt = <1800000>;
+		qcom,micbias4-microvolt = <1800000>;
+		qcom,mbhc-buttons-vthreshold-microvolt = <75000 150000 237000 500000 500000 500000 500000 500000>;
+		qcom,mbhc-headset-vthreshold-microvolt = <1700000>;
+		qcom,mbhc-headphone-vthreshold-microvolt = <50000>;
+		qcom,rx-device = <&wcd_rx>;
+		qcom,tx-device = <&wcd_tx>;
+
+		reset-gpios = <&tlmm 191 GPIO_ACTIVE_LOW>;
+
+		vdd-buck-supply = <&vreg_l15b_1p8>;
+		vdd-rxtx-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l15b_1p8>;
+		vdd-mic-bias-supply = <&vreg_bob1>;
+
+		#sound-dai-cells = <1>;
+	};
+
+	wcn7850-pmu {
+		compatible = "qcom,wcn7850-pmu";
+
+		wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>;
+		bt-enable-gpios = <&tlmm 116 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&wcn_wlan_bt_en>;
+		pinctrl-names = "default";
+
+		vdd-supply = <&vreg_wcn_0p95>;
+		vddio-supply = <&vreg_l15b_1p8>;
+		vddaon-supply = <&vreg_wcn_0p95>;
+		vdddig-supply = <&vreg_wcn_0p95>;
+		vddrfa1p2-supply = <&vreg_wcn_1p9>;
+		vddrfa1p8-supply = <&vreg_wcn_1p9>;
+
+		regulators {
+			vreg_pmu_rfa_cmn: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn";
+			};
+
+			vreg_pmu_aon_0p59: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p59";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p85: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p85";
+			};
+
+			vreg_pmu_btcmx_0p85: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p85";
+			};
+
+			vreg_pmu_rfa_0p8: ldo5 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo6 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p8: ldo7 {
+				regulator-name = "vreg_pmu_rfa_1p8";
+			};
+
+			vreg_pmu_pcie_0p9: ldo8 {
+				regulator-name = "vreg_pmu_pcie_0p9";
+			};
+
+			vreg_pmu_pcie_1p8: ldo9 {
+				regulator-name = "vreg_pmu_pcie_1p8";
+			};
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pm8550-rpmh-regulators";
+		qcom,pmic-id = "b";
+
+		vdd-bob1-supply = <&vph_pwr>;
+		vdd-bob2-supply = <&vph_pwr>;
+		vdd-l1-l4-l10-supply = <&vreg_s4c_1p8>;
+		vdd-l2-l13-l14-supply = <&vreg_bob1>;
+		vdd-l5-l16-supply = <&vreg_bob1>;
+		vdd-l6-l7-supply = <&vreg_bob2>;
+		vdd-l8-l9-supply = <&vreg_bob1>;
+		vdd-l12-supply = <&vreg_s5j_1p2>;
+		vdd-l15-supply = <&vreg_s4c_1p8>;
+		vdd-l17-supply = <&vreg_bob2>;
+
+		vreg_bob1: bob1 {
+			regulator-name = "vreg_bob1";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob2: bob2 {
+			regulator-name = "vreg_bob2";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1b_1p8: ldo1 {
+			regulator-name = "vreg_l1b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2b_3p0: ldo2 {
+			regulator-name = "vreg_l2b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4b_1p8: ldo4 {
+			regulator-name = "vreg_l4b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5b_3p0: ldo5 {
+			regulator-name = "vreg_l5b_3p0";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6b_1p8: ldo6 {
+			regulator-name = "vreg_l6b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b_2p8: ldo7 {
+			regulator-name = "vreg_l7b_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b_3p0: ldo8 {
+			regulator-name = "vreg_l8b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b_2p9: ldo9 {
+			regulator-name = "vreg_l9b_2p9";
+			regulator-min-microvolt = <2960000>;
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10b_1p8: ldo10 {
+			regulator-name = "vreg_l10b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b_1p2: ldo12 {
+			regulator-name = "vreg_l12b_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+		};
+
+		vreg_l13b_3p0: ldo13 {
+			regulator-name = "vreg_l13b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l14b_3p0: ldo14 {
+			regulator-name = "vreg_l14b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_1p8: ldo15 {
+			regulator-name = "vreg_l15b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+		};
+
+		vreg_l16b_2p9: ldo16 {
+			regulator-name = "vreg_l16b_2p9";
+			regulator-min-microvolt = <2912000>;
+			regulator-max-microvolt = <2912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b_2p5: ldo17 {
+			regulator-name = "vreg_l17b_2p5";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <2504000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vdd-l1-supply = <&vreg_s5j_1p2>;
+		vdd-l2-supply = <&vreg_s1f_0p7>;
+		vdd-l3-supply = <&vreg_s1f_0p7>;
+		vdd-s4-supply = <&vph_pwr>;
+
+		vreg_s4c_1p8: smps4 {
+			regulator-name = "vreg_s4c_1p8";
+			regulator-min-microvolt = <1856000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1c_1p2: ldo1 {
+			regulator-name = "vreg_l1c_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2c_0p8: ldo2 {
+			regulator-name = "vreg_l2c_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3c_0p8: ldo3 {
+			regulator-name = "vreg_l3c_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pmc8380-rpmh-regulators";
+		qcom,pmic-id = "d";
+
+		vdd-l1-supply = <&vreg_s1f_0p7>;
+		vdd-l2-supply = <&vreg_s1f_0p7>;
+		vdd-l3-supply = <&vreg_s4c_1p8>;
+		vdd-s1-supply = <&vph_pwr>;
+
+		vreg_l1d_0p8: ldo1 {
+			regulator-name = "vreg_l1d_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2d_0p9: ldo2 {
+			regulator-name = "vreg_l2d_0p9";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3d_1p8: ldo3 {
+			regulator-name = "vreg_l3d_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-3 {
+		compatible = "qcom,pmc8380-rpmh-regulators";
+		qcom,pmic-id = "e";
+
+		vdd-l2-supply = <&vreg_s1f_0p7>;
+		vdd-l3-supply = <&vreg_s5j_1p2>;
+
+		vreg_l2e_0p8: ldo2 {
+			regulator-name = "vreg_l2e_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3e_1p2: ldo3 {
+			regulator-name = "vreg_l3e_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-4 {
+		compatible = "qcom,pmc8380-rpmh-regulators";
+		qcom,pmic-id = "f";
+
+		vdd-l1-supply = <&vreg_s5j_1p2>;
+		vdd-l2-supply = <&vreg_s5j_1p2>;
+		vdd-l3-supply = <&vreg_s5j_1p2>;
+		vdd-s1-supply = <&vph_pwr>;
+
+		vreg_s1f_0p7: smps1 {
+			regulator-name = "vreg_s1f_0p7";
+			regulator-min-microvolt = <700000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1f_1p0: ldo1 {
+			regulator-name = "vreg_l1f_1p0";
+			regulator-min-microvolt = <1024000>;
+			regulator-max-microvolt = <1024000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2f_1p0: ldo2 {
+			regulator-name = "vreg_l2f_1p0";
+			regulator-min-microvolt = <1024000>;
+			regulator-max-microvolt = <1024000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3f_1p0: ldo3 {
+			regulator-name = "vreg_l3f_1p0";
+			regulator-min-microvolt = <1024000>;
+			regulator-max-microvolt = <1024000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-6 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "i";
+
+		vdd-l1-supply = <&vreg_s4c_1p8>;
+		vdd-l2-supply = <&vreg_s5j_1p2>;
+		vdd-l3-supply = <&vreg_s1f_0p7>;
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+
+		vreg_s1i_0p9: smps1 {
+			regulator-name = "vreg_s1i_0p9";
+			regulator-min-microvolt = <900000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s2i_1p0: smps2 {
+			regulator-name = "vreg_s2i_1p0";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1i_1p8: ldo1 {
+			regulator-name = "vreg_l1i_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2i_1p2: ldo2 {
+			regulator-name = "vreg_l2i_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3i_0p8: ldo3 {
+			regulator-name = "vreg_l3i_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-7 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "j";
+
+		vdd-l1-supply = <&vreg_s1f_0p7>;
+		vdd-l2-supply = <&vreg_s5j_1p2>;
+		vdd-l3-supply = <&vreg_s1f_0p7>;
+		vdd-s5-supply = <&vph_pwr>;
+
+		vreg_s5j_1p2: smps5 {
+			regulator-name = "vreg_s5j_1p2";
+			regulator-min-microvolt = <1256000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1j_0p8: ldo1 {
+			regulator-name = "vreg_l1j_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2j_1p2: ldo2 {
+			regulator-name = "vreg_l2j_1p2";
+			regulator-min-microvolt = <1256000>;
+			regulator-max-microvolt = <1256000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3j_0p8: ldo3 {
+			regulator-name = "vreg_l3j_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&gpu {
+	status = "okay";
+
+	zap-shader {
+		firmware-name = "qcom/x1e80100/HONOR/MRO-XXX/qcdxkmsuc8380.mbn";
+	};
+};
+
+&i2c1 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	touchpad@5d {
+		compatible = "hid-over-i2c";
+		reg = <0x5d>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 24 IRQ_TYPE_LEVEL_LOW>;
+
+		vdd-supply = <&vreg_misc_3p3>;
+		vddl-supply = <&vreg_l12b_1p2>;
+
+		pinctrl-0 = <&tpad_default>;
+		pinctrl-names = "default";
+
+		wakeup-source;
+	};
+};
+
+&i2c5 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	eusb5_repeater: redriver@43 {
+		compatible = "nxp,ptn3222";
+		reg = <0x43>;
+		#phy-cells = <0>;
+
+		vdd3v3-supply = <&vreg_l13b_3p0>;
+		vdd1v8-supply = <&vreg_l4b_1p8>;
+
+		reset-gpios = <&tlmm 7 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&eusb5_reset_n>;
+		pinctrl-names = "default";
+	};
+
+	eusb6_repeater: redriver@4f {
+		compatible = "nxp,ptn3222";
+		reg = <0x4f>;
+		#phy-cells = <0>;
+
+		vdd3v3-supply = <&vreg_l13b_3p0>;
+		vdd1v8-supply = <&vreg_l4b_1p8>;
+
+		reset-gpios = <&tlmm 184 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&eusb6_reset_n>;
+		pinctrl-names = "default";
+	};
+};
+
+&i2c8 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	touchscreen@38 {
+		compatible = "hid-over-i2c";
+		reg = <0x38>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 51 IRQ_TYPE_LEVEL_LOW>;
+
+		vdd-supply = <&vreg_misc_3p3>;
+
+		pinctrl-0 = <&ts0_default>;
+		pinctrl-names = "default";
+	};
+};
+
+&i2c9 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	keyboard@3a {
+		compatible = "hid-over-i2c";
+		reg = <0x3a>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 67 IRQ_TYPE_LEVEL_LOW>;
+
+		vdd-supply = <&vreg_misc_3p3>;
+		vddl-supply = <&vreg_l12b_1p2>;
+
+		pinctrl-0 = <&kybd_default>;
+		pinctrl-names = "default";
+
+		wakeup-source;
+	};
+};
+
+&iris {
+	firmware-name = "qcom/x1e80100/HONOR/MRO-XXX/qcvss8380.mbn";
+
+	status = "okay";
+};
+
+&lpass_tlmm {
+	spkr_01_sd_n_active: spkr-01-sd-n-active-state {
+		pins = "gpio12";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+		output-low;
+	};
+
+	spkr_23_sd_n_active: spkr-23-sd-n-active-state {
+		pins = "gpio13";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+		output-low;
+	};
+};
+
+&lpass_vamacro {
+	pinctrl-0 = <&dmic01_default>, <&dmic23_default>;
+	pinctrl-names = "default";
+
+	vdd-micb-supply = <&vreg_l1b_1p8>;
+	qcom,dmic-sample-rate = <4800000>;
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dp0 {
+	status = "okay";
+};
+
+&mdss_dp0_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp1 {
+	status = "okay";
+};
+
+&mdss_dp1_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp2 {
+	status = "okay";
+};
+
+&mdss_dp2_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&pcie4 {
+	pinctrl-0 = <&pcie4_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie4_phy {
+	vdda-phy-supply = <&vreg_l3i_0p8>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&pcie4_port0 {
+	reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+
+	wifi@0 {
+		compatible = "pci17cb,1107";
+		reg = <0x10000 0x0 0x0 0x0 0x0>;
+
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+		vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>;
+		vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>;
+	};
+};
+
+&pm8550ve_8_gpios {
+	misc_3p3_reg_en: misc-3p3-reg-en-state {
+		pins = "gpio6";
+		function = "normal";
+		bias-disable;
+		input-disable;
+		output-enable;
+		drive-push-pull;
+		power-source = <1>; /* 1.8 V */
+		qcom,drive-strength = <PMIC_GPIO_STRENGTH_LOW>;
+	};
+};
+
+&qupv3_0 {
+	status = "okay";
+};
+
+&qupv3_1 {
+	status = "okay";
+};
+
+&qupv3_2 {
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/x1e80100/HONOR/MRO-XXX/qcadsp8380.mbn",
+			"qcom/x1e80100/HONOR/MRO-XXX/adsp_dtbs.elf";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/x1e80100/HONOR/MRO-XXX/qccdsp8380.mbn",
+			"qcom/x1e80100/HONOR/MRO-XXX/cdsp_dtbs.elf";
+
+	status = "okay";
+};
+
+&swr0 {
+	status = "okay";
+
+	pinctrl-0 = <&wsa_swr_active>, <&spkr_01_sd_n_active>;
+	pinctrl-names = "default";
+
+	/* WSA8845, Left Woofer */
+	left_woofer: speaker@0,0 {
+		compatible = "sdw20217020400";
+		reg = <0 0>;
+		reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "WooferLeft";
+		vdd-1p8-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l12b_1p2>;
+		qcom,port-mapping = <1 2 3 7 10 13>;
+	};
+
+	/* WSA8845, Left Tweeter */
+	left_tweeter: speaker@0,1 {
+		compatible = "sdw20217020400";
+		reg = <0 1>;
+		reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "TweeterLeft";
+		vdd-1p8-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l12b_1p2>;
+		qcom,port-mapping = <4 5 6 7 11 13>;
+	};
+};
+
+&swr1 {
+	status = "okay";
+
+	/* WCD9385 RX */
+	wcd_rx: codec@0,4 {
+		compatible = "sdw20217010d00";
+		reg = <0 4>;
+		qcom,rx-port-mapping = <1 2 3 4 5>;
+	};
+};
+
+&swr2 {
+	status = "okay";
+
+	/* WCD9385 TX */
+	wcd_tx: codec@0,3 {
+		compatible = "sdw20217010d00";
+		reg = <0 3>;
+		qcom,tx-port-mapping = <2 2 3 4>;
+	};
+};
+
+&swr3 {
+	pinctrl-0 = <&wsa2_swr_active>, <&spkr_23_sd_n_active>;
+	pinctrl-names = "default";
+
+	status = "okay";
+
+	/* WSA8845, Right Woofer */
+	right_woofer: speaker@0,0 {
+		compatible = "sdw20217020400";
+		reg = <0 0>;
+		reset-gpios = <&lpass_tlmm 13 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "WooferRight";
+		vdd-1p8-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l12b_1p2>;
+		qcom,port-mapping = <1 2 3 7 10 13>;
+	};
+
+	/* WSA8845, Right Tweeter */
+	right_tweeter: speaker@0,1 {
+		compatible = "sdw20217020400";
+		reg = <0 1>;
+		reset-gpios = <&lpass_tlmm 13 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "TweeterRight";
+		vdd-1p8-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l12b_1p2>;
+		qcom,port-mapping = <4 5 6 7 11 13>;
+	};
+};
+
+&tlmm {
+	gpio-reserved-ranges = <34 2>, /* Unused */
+			       <44 4>; /* SPI (TPM) */
+
+	tpad_default: tpad-default-state {
+		pins = "gpio3";
+		function = "gpio";
+		bias-disable;
+	};
+
+	eusb5_reset_n: eusb5-reset-n-state {
+		pins = "gpio7";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	ts0_default: ts0-default-state {
+		reset-n-pins {
+			pins = "gpio48";
+			function = "gpio";
+			drive-strength = <16>;
+		};
+
+		int-n-pins {
+			pins = "gpio51";
+			function = "gpio";
+			bias-disable;
+		};
+	};
+
+	kybd_default: kybd-default-state {
+		pins = "gpio67";
+		function = "gpio";
+		bias-disable;
+	};
+
+	hall_int_n_default: hall-int-n-state {
+		pins = "gpio92";
+		function = "gpio";
+		bias-disable;
+	};
+
+	wcn_wlan_bt_en: wcn-wlan-bt-en-state {
+		pins = "gpio116", "gpio117";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	hdmi_hpd_default: hdmi-hpd-default-state {
+		pins = "gpio126";
+		function = "usb2_dp";
+		bias-disable;
+	};
+
+	pcie4_default: pcie4-default-state {
+		perst-n-pins {
+			pins = "gpio146";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		clkreq-n-pins {
+			pins = "gpio147";
+			function = "pcie4_clk";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		wake-n-pins {
+			pins = "gpio148";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	eusb6_reset_n: eusb6-reset-n-state {
+		pins = "gpio184";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	wcd_default: wcd-reset-n-active-state {
+		pins = "gpio191";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	wcn_sw_en: wcn-sw-en-state {
+		pins = "gpio214";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+};
+
+&uart14 {
+	status = "okay";
+
+	bluetooth {
+		compatible = "qcom,wcn7850-bt";
+		max-speed = <3200000>;
+
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+	};
+};
+
+&uart21 {
+	compatible = "qcom,geni-debug-uart";
+
+	status = "okay";
+};
+
+&ufs_mem_hc {
+	reset-gpios = <&tlmm 238 GPIO_ACTIVE_LOW>;
+
+	vcc-supply = <&vreg_l17b_2p5>;
+	vcc-max-microamp = <600000>;
+	vccq-supply = <&vreg_l2i_1p2>;
+	vccq-max-microamp = <600000>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vreg_l3i_0p8>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&usb_1_ss0 {
+	dr_mode = "host";
+
+	status = "okay";
+};
+
+&usb_1_ss0_dwc3_hs {
+	remote-endpoint = <&pmic_glink_ss0_hs_in>;
+};
+
+&usb_1_ss0_hsphy {
+	vdd-supply = <&vreg_l3j_0p8>;
+	vdda12-supply = <&vreg_l2j_1p2>;
+
+	status = "okay";
+};
+
+&usb_1_ss0_qmpphy {
+	vdda-phy-supply = <&vreg_l2j_1p2>;
+	vdda-pll-supply = <&vreg_l1j_0p8>;
+
+	status = "okay";
+};
+
+&usb_1_ss0_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss0_ss_in>;
+};
+
+&usb_1_ss1 {
+	dr_mode = "host";
+
+	status = "okay";
+};
+
+&usb_1_ss1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_ss1_hs_in>;
+};
+
+&usb_1_ss1_hsphy {
+	vdd-supply = <&vreg_l3j_0p8>;
+	vdda12-supply = <&vreg_l2j_1p2>;
+
+	status = "okay";
+};
+
+&usb_1_ss1_qmpphy {
+	vdda-phy-supply = <&vreg_l2j_1p2>;
+	vdda-pll-supply = <&vreg_l2d_0p9>;
+
+	status = "okay";
+};
+
+&usb_1_ss1_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss1_ss_in>;
+};
+
+&usb_1_ss2_qmpphy {
+	vdda-phy-supply = <&vreg_l2j_1p2>;
+	vdda-pll-supply = <&vreg_l2d_0p9>;
+
+	/delete-property/ mode-switch;
+	/delete-property/ orientation-switch;
+
+	status = "okay";
+
+	ports {
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			/delete-node/ endpoint;
+
+			usb_1_ss2_qmpphy_out_dp: endpoint@0 {
+				reg = <0>;
+
+				data-lanes = <3 2 1 0>;
+				remote-endpoint = <&hdmi_bridge_dp_in>;
+			};
+		};
+	};
+};
+
+/* usb_2 is for camera, USB port on top of the screen */
+&usb_2 {
+	dr_mode = "host";
+
+	status = "okay";
+};
+
+&usb_2_hsphy {
+	vdd-supply = <&vreg_l3j_0p8>;
+	vdda12-supply = <&vreg_l2j_1p2>;
+	phys = <&eusb5_repeater>;
+
+	status = "okay";
+};
+
+/* MP0 goes to the USB-A port (USB3) and FPC */
+&usb_mp {
+	status = "okay";
+};
+
+&usb_mp_hsphy0 {
+	vdd-supply = <&vreg_l2e_0p8>;
+	vdda12-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&usb_mp_hsphy1 {
+	vdd-supply = <&vreg_l2e_0p8>;
+	vdda12-supply = <&vreg_l3e_1p2>;
+
+	phys = <&eusb6_repeater>;
+
+	status = "okay";
+};
+
+&usb_mp_qmpphy0 {
+	vdda-phy-supply = <&vreg_l3e_1p2>;
+	vdda-pll-supply = <&vreg_l3c_0p8>;
+
+	status = "okay";
+};
+
+&usb_mp_qmpphy1 {
+	vdda-phy-supply = <&vreg_l3e_1p2>;
+	vdda-pll-supply = <&vreg_l3c_0p8>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/x1p42100-microsoft-sp12in.dts b/arch/arm64/boot/dts/qcom/x1p42100-microsoft-sp12in.dts
new file mode 100644
index 0000000000000..0314b2246cdb7
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/x1p42100-microsoft-sp12in.dts
@@ -0,0 +1,1194 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2024, Linaro Limited
+ * Copyright (c) 2025, Jens Glathe
+ * Copyright (c) 2025, Harrison Vanderbyl
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/gpio-keys.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+#include <dt-bindings/clock/qcom,x1e80100-tcsr.h>
+
+#include "purwa.dtsi"
+#include "hamoa-pmics.dtsi"
+
+/delete-node/ &pmc8380_6;
+/delete-node/ &pmc8380_6_thermal;
+
+/ {
+	model = "Surface Pro 12in 1st Edition";
+	compatible = "microsoft,surface-pro-12in", "qcom,x1p42100";
+	chassis-type = "tablet";
+
+	aliases {
+		serial0 = &uart2;
+		serial1 = &uart14;
+	};
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&pmk8550_pwm 0 5000000>;
+
+		power-supply = <&vreg_edp_3p3>;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&hall_int_n_default>, <&vol_up_n_default>, <&vol_down_n_default>;
+		pinctrl-names = "default";
+
+		switch-lid {
+			gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_SW>;
+			linux,code = <SW_LID>;
+			wakeup-source;
+			wakeup-event-action = <EV_ACT_DEASSERTED>;
+		};
+
+		key-vol-up {
+			gpios = <&pm8550_gpios 8 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
+			debounce-interval = <15>;
+			linux,can-disable;
+			wakeup-source;
+		};
+
+		key-vol-down {
+			gpios = <&pm8550_gpios 6 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEDOWN>;
+			debounce-interval = <15>;
+			linux,can-disable;
+			wakeup-source;
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,x1e80100-pmic-glink",
+			     "qcom,sm8550-pmic-glink",
+			     "qcom,pmic-glink";
+		orientation-gpios = <&tlmm 121 GPIO_ACTIVE_HIGH>,
+				    <&tlmm 123 GPIO_ACTIVE_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* Right-side upper port */
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_ss0_hs_in: endpoint {
+						remote-endpoint = <&usb_1_ss0_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss0_ss_in: endpoint {
+						remote-endpoint = <&usb_1_ss0_qmpphy_out>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_ss0_sbu: endpoint {
+						remote-endpoint = <&usb_1_ss0_sbu_mux>;
+					};
+				};
+			};
+		};
+
+		/* Right-side lower port */
+		connector@1 {
+			compatible = "usb-c-connector";
+			reg = <1>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_ss1_hs_in: endpoint {
+						remote-endpoint = <&usb_1_ss1_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss1_ss_in: endpoint {
+						remote-endpoint = <&usb_1_ss1_qmpphy_out>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_ss1_sbu: endpoint {
+						remote-endpoint = <&usb_1_ss1_sbu_mux>;
+					};
+				};
+			};
+		};
+	};
+
+	reserved-memory {
+		linux,cma {
+			compatible = "shared-dma-pool";
+			size = <0x0 0x20000000>;
+			reusable;
+			linux,cma-default;
+		};
+	};
+
+	vreg_edp_3p3: regulator-edp-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_EDP_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 70 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&edp_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+		vin-supply = <&vreg_panel_en>;
+	};
+
+	vreg_panel_en: regulator-panel-en {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_PANEL_EN";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 29 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		regulator-enable-ramp-delay = <150000>;
+
+		pinctrl-0 = <&panel_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+	};
+
+	vreg_wcn_0p95: regulator-wcn-0p95 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_0P95";
+		regulator-min-microvolt = <950000>;
+		regulator-max-microvolt = <950000>;
+
+		vin-supply = <&vreg_wcn_3p3>;
+	};
+
+	vreg_wcn_1p9: regulator-wcn-1p9 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_1P9";
+		regulator-min-microvolt = <1900000>;
+		regulator-max-microvolt = <1900000>;
+
+		vin-supply = <&vreg_wcn_3p3>;
+	};
+
+	vreg_wcn_3p3: regulator-wcn-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 214 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&wcn_sw_en>;
+		pinctrl-names = "default";
+	};
+
+	sound {
+		compatible = "qcom,x1e80100-sndcard";
+		model = "X1P42100-Microsoft-Surface-Pro-12in";
+		audio-routing = "SpkrLeft IN", "WSA WSA_SPK1 OUT",
+				"SpkrRight IN", "WSA WSA_SPK2 OUT",
+				"VA DMIC0", "vdd-micb",
+				"VA DMIC1", "vdd-micb";
+
+		va-dai-link {
+			link-name = "VA Capture";
+
+			codec {
+				sound-dai = <&lpass_vamacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai VA_CODEC_DMA_TX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		wsa-dai-link {
+			link-name = "WSA Playback";
+
+			codec {
+				sound-dai = <&left_spkr>, <&right_spkr>,
+					    <&swr0 0>, <&lpass_wsamacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai WSA_CODEC_DMA_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+	};
+
+	wcn7850-pmu {
+		compatible = "qcom,wcn7850-pmu";
+
+		vdd-supply = <&vreg_wcn_0p95>;
+		vddio-supply = <&vreg_l15b_1p8>;
+		vddaon-supply = <&vreg_wcn_0p95>;
+		vdddig-supply = <&vreg_wcn_0p95>;
+		vddrfa1p2-supply = <&vreg_wcn_1p9>;
+		vddrfa1p8-supply = <&vreg_wcn_1p9>;
+
+		wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>;
+		bt-enable-gpios = <&tlmm 116 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&wcn_wlan_bt_en>;
+		pinctrl-names = "default";
+
+		regulators {
+			vreg_pmu_rfa_cmn: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn";
+			};
+
+			vreg_pmu_aon_0p59: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p59";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p85: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p85";
+			};
+
+			vreg_pmu_btcmx_0p85: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p85";
+			};
+
+			vreg_pmu_rfa_0p8: ldo5 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo6 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p8: ldo7 {
+				regulator-name = "vreg_pmu_rfa_1p8";
+			};
+
+			vreg_pmu_pcie_0p9: ldo8 {
+				regulator-name = "vreg_pmu_pcie_0p9";
+			};
+
+			vreg_pmu_pcie_1p8: ldo9 {
+				regulator-name = "vreg_pmu_pcie_1p8";
+			};
+		};
+	};
+
+	usb-1-ss0-sbu-mux {
+		compatible = "onnn,fsusb42", "gpio-sbu-mux";
+
+		enable-gpios = <&tlmm 168 GPIO_ACTIVE_LOW>;
+		select-gpios = <&tlmm 167 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&usb_1_ss0_sbu_default>;
+		pinctrl-names = "default";
+
+		mode-switch;
+		orientation-switch;
+
+		port {
+			usb_1_ss0_sbu_mux: endpoint {
+				remote-endpoint = <&pmic_glink_ss0_sbu>;
+			};
+		};
+	};
+
+	usb-1-ss1-sbu-mux {
+		compatible = "onnn,fsusb42", "gpio-sbu-mux";
+
+		enable-gpios = <&tlmm 179 GPIO_ACTIVE_LOW>;
+		select-gpios = <&tlmm 178 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&usb_1_ss1_sbu_default>;
+		pinctrl-names = "default";
+
+		mode-switch;
+		orientation-switch;
+
+		port {
+			usb_1_ss1_sbu_mux: endpoint {
+				remote-endpoint = <&pmic_glink_ss1_sbu>;
+			};
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pm8550-rpmh-regulators";
+		qcom,pmic-id = "b";
+
+		vdd-bob1-supply = <&vph_pwr>;
+		vdd-bob2-supply = <&vph_pwr>;
+		vdd-l1-l4-l10-supply = <&vreg_s4c_1p8>;
+		vdd-l2-l13-l14-supply = <&vreg_bob1>;
+		vdd-l5-l16-supply = <&vreg_bob1>;
+		vdd-l6-l7-supply = <&vreg_bob2>;
+		vdd-l8-l9-supply = <&vreg_bob1>;
+		vdd-l12-supply = <&vreg_s5j_1p2>;
+		vdd-l15-supply = <&vreg_s4c_1p8>;
+		vdd-l17-supply = <&vreg_bob2>;
+
+		vreg_bob1: bob1 {
+			regulator-name = "vreg_bob1";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob2: bob2 {
+			regulator-name = "vreg_bob2";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1b_1p8: ldo1 {
+			regulator-name = "vreg_l1b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2b_3p0: ldo2 {
+			regulator-name = "vreg_l2b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4b_1p8: ldo4 {
+			regulator-name = "vreg_l4b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5b_3p0: ldo5 {
+			regulator-name = "vreg_l5b_3p0";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6b_1p8: ldo6 {
+			regulator-name = "vreg_l6b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b_2p8: ldo7 {
+			regulator-name = "vreg_l7b_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+		};
+
+		vreg_l8b_3p0: ldo8 {
+			regulator-name = "vreg_l8b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b_2p9: ldo9 {
+			regulator-name = "vreg_l9b_2p9";
+			regulator-min-microvolt = <2960000>;
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10b_1p8: ldo10 {
+			regulator-name = "vreg_l10b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b_1p2: ldo12 {
+			regulator-name = "vreg_l12b_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13b_3p0: ldo13 {
+			regulator-name = "vreg_l13b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l14b_3p0: ldo14 {
+			regulator-name = "vreg_l14b_3p0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_1p8: ldo15 {
+			regulator-name = "vreg_l15b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l16b_2p9: ldo16 {
+			regulator-name = "vreg_l16b_2p9";
+			regulator-min-microvolt = <2900000>;
+			regulator-max-microvolt = <2912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b_2p5: ldo17 {
+			regulator-name = "vreg_l17b_2p5";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <2504000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vdd-l1-supply = <&vreg_s5j_1p2>;
+		vdd-s4-supply = <&vph_pwr>;
+
+		vreg_s4c_1p8: smps4 {
+			regulator-name = "vreg_s4c_1p8";
+			regulator-min-microvolt = <1856000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1c_1p2: ldo1 {
+			regulator-name = "vreg_l1c_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2c_0p8: ldo2 {
+			regulator-name = "vreg_l2c_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3c_0p8: ldo3 {
+			regulator-name = "vreg_l3c_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pmc8380-rpmh-regulators";
+		qcom,pmic-id = "d";
+
+		vdd-l3-supply = <&vreg_s4c_1p8>;
+		vdd-s1-supply = <&vph_pwr>;
+
+		vreg_l1d_0p8: ldo1 {
+			regulator-name = "vreg_l1d_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2d_0p9: ldo2 {
+			regulator-name = "vreg_l2d_0p9";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3d_1p8: ldo3 {
+			regulator-name = "vreg_l3d_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-3 {
+		compatible = "qcom,pmc8380-rpmh-regulators";
+		qcom,pmic-id = "e";
+
+		vdd-l3-supply = <&vreg_s5j_1p2>;
+
+		vreg_l2e_0p8: ldo2 {
+			regulator-name = "vreg_l2e_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3e_1p2: ldo3 {
+			regulator-name = "vreg_l3e_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-6 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "i";
+
+		vdd-l1-supply = <&vreg_s4c_1p8>;
+		vdd-l2-supply = <&vreg_s5j_1p2>;
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+
+		vreg_s1i_0p9: smps1 {
+			regulator-name = "vreg_s1i_0p9";
+			regulator-min-microvolt = <900000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s2i_1p0: smps2 {
+			regulator-name = "vreg_s2i_1p0";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1i_1p8: ldo1 {
+			regulator-name = "vreg_l1i_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2i_1p2: ldo2 {
+			regulator-name = "vreg_l2i_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3i_0p8: ldo3 {
+			regulator-name = "vreg_l3i_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-7 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "j";
+
+		vdd-l2-supply = <&vreg_s5j_1p2>;
+		vdd-s5-supply = <&vph_pwr>;
+
+		vreg_s5j_1p2: smps5 {
+			regulator-name = "vreg_s5j_1p2";
+			regulator-min-microvolt = <1256000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1j_0p8: ldo1 {
+			regulator-name = "vreg_l1j_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2j_1p2: ldo2 {
+			regulator-name = "vreg_l2j_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1256000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3j_0p8: ldo3 {
+			regulator-name = "vreg_l3j_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/x1p42100/Microsoft/Surface12/qcdxkmsucpurwa.mbn";
+};
+
+&i2c8 {
+	clock-frequency = <1000000>;
+
+	status = "okay";
+
+	touchscreen@16 {
+		compatible = "hid-over-i2c";
+		reg = <0x16>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 38 IRQ_TYPE_LEVEL_LOW>;
+		wakeup-source;
+
+		vddl-supply = <&vreg_l15b_1p8>;
+
+		pinctrl-0 = <&ts0_default>;
+		pinctrl-names = "default";
+	};
+};
+
+&i2c0 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+};
+
+&i2c4 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	/* MAX34417 @12 */
+	/* MAX34417 @14 */
+	/* MAX34417 @16 */
+	/* MAX34417 @18 */
+	/* MAX34417 @1a */
+};
+
+&i2c9 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	/* NFC @28 Business 2109 sku only */
+};
+
+&lpass_tlmm {
+	spkr_01_sd_n_active: spkr-01-sd-n-active-state {
+		pins = "gpio12";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+		output-low;
+	};
+};
+
+&lpass_vamacro {
+	qcom,dmic-sample-rate = <4800000>;
+
+	vdd-micb-supply = <&vreg_l1b_1p8>;
+
+	pinctrl-0 = <&dmic01_default>;
+	pinctrl-names = "default";
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dp0 {
+	status = "okay";
+};
+
+&mdss_dp0_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp1 {
+	status = "okay";
+};
+
+&mdss_dp1_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp3 {
+	/delete-property/ #sound-dai-cells;
+
+	pinctrl-0 = <&edp0_hpd_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+
+	aux-bus {
+		panel: panel {
+			compatible = "edp-panel";
+
+			backlight = <&backlight>;
+
+			power-supply = <&vreg_edp_3p3>;
+
+			port {
+				edp_panel_in: endpoint {
+					remote-endpoint = <&mdss_dp3_out>;
+				};
+			};
+		};
+	};
+};
+
+&mdss_dp3_out {
+	data-lanes = <0 1 2 3>;
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+
+	remote-endpoint = <&edp_panel_in>;
+};
+
+&mdss_dp3_phy {
+	vdda-phy-supply = <&vreg_l3j_0p8>;
+	vdda-pll-supply = <&vreg_l2j_1p2>;
+
+	status = "okay";
+};
+
+&pcie4 {
+	pinctrl-0 = <&pcie4_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie4_phy {
+	vdda-phy-supply = <&vreg_l3i_0p8>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&pcie4_port0 {
+	reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+
+	wifi@0 {
+		compatible = "pci17cb,1107";
+		reg = <0x10000 0x0 0x0 0x0 0x0>;
+
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+		vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>;
+		vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>;
+	};
+};
+
+&pm8550_gpios {
+	vol_up_n_default: vol-up-n-state {
+		pins = "gpio8";
+		function = "normal";
+		power-source = <1>;
+		bias-pull-up;
+		input-enable;
+	};
+
+	vol_down_n_default: vol-down-n-state {
+		pins = "gpio6";
+		function = "normal";
+		power-source = <1>;
+		bias-pull-up;
+		input-enable;
+	};
+};
+
+&pmk8550_pwm {
+	status = "okay";
+};
+
+&qupv3_0 {
+	status = "okay";
+};
+
+&qupv3_1 {
+	status = "okay";
+};
+
+&qupv3_2 {
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/x1p42100/Microsoft/Surface12/qcadsp8380.mbn",
+			"qcom/x1p42100/Microsoft/Surface12/adsp_dtbs.elf";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/x1p42100/Microsoft/Surface12/qccdsp8380.mbn",
+			"qcom/x1p42100/Microsoft/Surface12/cdsp_dtbs.elf";
+
+	status = "okay";
+};
+
+&smb2360_0 {
+	status = "okay";
+};
+
+&smb2360_0_eusb2_repeater {
+	vdd18-supply = <&vreg_l3d_1p8>;
+	vdd3-supply = <&vreg_l2b_3p0>;
+};
+
+&smb2360_1 {
+	status = "okay";
+};
+
+&smb2360_1_eusb2_repeater {
+	vdd18-supply = <&vreg_l3d_1p8>;
+	vdd3-supply = <&vreg_l14b_3p0>;
+};
+
+&swr0 {
+	pinctrl-0 = <&wsa_swr_active>, <&spkr_01_sd_n_active>;
+	pinctrl-names = "default";
+
+	status = "okay";
+
+	/* WSA8845, Left speaker */
+	left_spkr: speaker@0,0 {
+		compatible = "sdw20217020400";
+		reg = <0 0>;
+		reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "SpkrLeft";
+		vdd-1p8-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l12b_1p2>;
+		qcom,port-mapping = <1 2 3 7 10 13>;
+	};
+
+	/* WSA8845, Right speaker */
+	right_spkr: speaker@0,1 {
+		compatible = "sdw20217020400";
+		reg = <0 1>;
+		reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "SpkrRight";
+		vdd-1p8-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l12b_1p2>;
+		qcom,port-mapping = <4 5 6 7 11 13>;
+	};
+};
+
+&tlmm {
+	gpio-reserved-ranges = <34 2>, /* Unused */
+			       <44 4>; /* SPI (TPM) */
+
+	edp_reg_en: edp-reg-en-state {
+		pins = "gpio70";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	hall_int_n_default: hall-int-n-state {
+		pins = "gpio2";
+		function = "gpio";
+		bias-disable;
+	};
+
+	panel_en: panel-en-state {
+		pins = "gpio29";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	pcie4_default: pcie4-default-state {
+		clkreq-n-pins {
+			pins = "gpio147";
+			function = "pcie4_clk";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio146";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio148";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	ssam_state: ssam-state-state {
+		pins = "gpio91";
+		function = "gpio";
+		bias-disable;
+	};
+
+	ts0_default: ts0-default-state {
+		int-n-pins {
+			pins = "gpio38";
+			function = "gpio";
+			bias-disable;
+		};
+
+		reset-n-pins {
+			pins = "gpio48";
+			function = "gpio";
+			output-high;
+			drive-strength = <16>;
+		};
+	};
+
+	wcn_sw_en: wcn-sw-en-state {
+		pins = "gpio214";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	wcn_wlan_bt_en: wcn-wlan-bt-en-state {
+		pins = "gpio116", "gpio117";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	usb_1_ss0_sbu_default: usb-1-ss0-sbu-state {
+		oe-n-pins {
+			pins = "gpio168";
+			function = "gpio";
+			bias-disable;
+			drive-strength = <2>;
+		};
+
+		sel-pins {
+			pins = "gpio167";
+			function = "gpio";
+			bias-disable;
+			drive-strength = <2>;
+		};
+	};
+
+	usb_1_ss1_sbu_default: usb-1-ss1-sbu-state {
+		oe-n-pins {
+			pins = "gpio179";
+			function = "gpio";
+			bias-disable;
+			drive-strength = <2>;
+		};
+
+		sel-pins {
+			pins = "gpio178";
+			function = "gpio";
+			bias-disable;
+			drive-strength = <2>;
+		};
+	};
+};
+
+&uart2 {
+	status = "okay";
+
+	embedded-controller {
+		compatible = "microsoft,surface-sam";
+
+		interrupts-extended = <&tlmm 91 IRQ_TYPE_EDGE_RISING>;
+
+		current-speed = <4000000>;
+
+		pinctrl-0 = <&ssam_state>;
+		pinctrl-names = "default";
+	};
+};
+
+&uart14 {
+	status = "okay";
+
+	bluetooth {
+		compatible = "qcom,wcn7850-bt";
+		max-speed = <3200000>;
+
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+	};
+};
+
+&ufs_mem_hc {
+	vcc-supply = <&vreg_l17b_2p5>;
+	vcc-max-microamp = <800000>;
+	vccq-supply = <&vreg_l2i_1p2>;
+	vccq-max-microamp = <900000>;
+
+	vdd-hba-supply = <&vreg_l3j_0p8>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	status = "okay";
+
+	vdda-phy-supply = <&vreg_l3i_0p8>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+};
+
+&usb_1_ss0_hsphy {
+	vdd-supply = <&vreg_l3j_0p8>;
+	vdda12-supply = <&vreg_l2j_1p2>;
+
+	phys = <&smb2360_0_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_1_ss0_qmpphy {
+	vdda-phy-supply = <&vreg_l2j_1p2>;
+	vdda-pll-supply = <&vreg_l1j_0p8>;
+
+	status = "okay";
+};
+
+&usb_1_ss0 {
+	dr_mode = "host";
+
+	status = "okay";
+};
+
+&usb_1_ss0_dwc3_hs {
+	remote-endpoint = <&pmic_glink_ss0_hs_in>;
+};
+
+&usb_1_ss0_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss0_ss_in>;
+};
+
+&usb_1_ss1_hsphy {
+	vdd-supply = <&vreg_l3j_0p8>;
+	vdda12-supply = <&vreg_l2j_1p2>;
+
+	phys = <&smb2360_1_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_1_ss1_qmpphy {
+	vdda-phy-supply = <&vreg_l2j_1p2>;
+	vdda-pll-supply = <&vreg_l2d_0p9>;
+
+	status = "okay";
+};
+
+&usb_1_ss1 {
+	dr_mode = "host";
+
+	status = "okay";
+};
+
+&usb_1_ss1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_ss1_hs_in>;
+};
+
+&usb_1_ss1_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss1_ss_in>;
+};
diff --git a/arch/arm64/boot/dts/realtek/kent.dtsi b/arch/arm64/boot/dts/realtek/kent.dtsi
index 8d4293cd4c036..df0d5ee3b1b94 100644
--- a/arch/arm64/boot/dts/realtek/kent.dtsi
+++ b/arch/arm64/boot/dts/realtek/kent.dtsi
@@ -23,7 +23,7 @@ timer {
 			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
 			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
 			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>,
-			     <GIC_PPI  9 IRQ_TYPE_LEVEL_HIGH>;
+			     <GIC_PPI 12 IRQ_TYPE_LEVEL_HIGH>;
 	};
 
 	cpus {
@@ -151,6 +151,37 @@ uart0: serial@7800 {
 				status = "disabled";
 			};
 
+			gpio: gpio@31000 {
+				compatible = "realtek,rtd1625-iso-gpio";
+				reg = <0x31000 0x398>;
+				gpio-controller;
+				gpio-ranges = <&isom_pinctrl 0 0 2>,
+					      <&ve4_pinctrl 2 0 6>,
+					      <&iso_pinctrl 8 0 4>,
+					      <&ve4_pinctrl 12 6 2>,
+					      <&main2_pinctrl 14 0 2>,
+					      <&ve4_pinctrl 16 8 4>,
+					      <&main2_pinctrl 20 2 3>,
+					      <&ve4_pinctrl 23 12 3>,
+					      <&iso_pinctrl 26 4 2>,
+					      <&isom_pinctrl 28 2 2>,
+					      <&ve4_pinctrl 30 15 6>,
+					      <&main2_pinctrl 36 5 6>,
+					      <&ve4_pinctrl 42 21 3>,
+					      <&iso_pinctrl 45 6 6>,
+					      <&ve4_pinctrl 51 24 1>,
+					      <&iso_pinctrl 52 12 1>,
+					      <&ve4_pinctrl 53 25 11>,
+					      <&main2_pinctrl 64 11 28>,
+					      <&ve4_pinctrl 92 36 2>,
+					      <&iso_pinctrl 94 13 19>,
+					      <&iso_pinctrl 128 32 4>,
+					      <&ve4_pinctrl 132 38 13>,
+					      <&iso_pinctrl 145 36 19>,
+					      <&ve4_pinctrl 164 51 2>;
+				#gpio-cells = <2>;
+			};
+
 			iso_pinctrl: pinctrl@4e000 {
 				compatible = "realtek,rtd1625-iso-pinctrl";
 				reg = <0x4e000 0x1a4>;
@@ -161,6 +192,14 @@ main2_pinctrl: pinctrl@4f200 {
 				reg = <0x4f200 0x50>;
 			};
 
+			iso_m_gpio: gpio@89100 {
+				compatible = "realtek,rtd1625-isom-gpio";
+				reg = <0x89100 0x30>;
+				gpio-controller;
+				gpio-ranges = <&isom_pinctrl 0 0 4>;
+				#gpio-cells = <2>;
+			};
+
 			isom_pinctrl: pinctrl@146200 {
 				compatible = "realtek,rtd1625-isom-pinctrl";
 				reg = <0x146200 0x34>;
diff --git a/arch/arm64/boot/dts/realtek/rtd16xx.dtsi b/arch/arm64/boot/dts/realtek/rtd16xx.dtsi
index 3a7f6e35b7f74..43b13d133c324 100644
--- a/arch/arm64/boot/dts/realtek/rtd16xx.dtsi
+++ b/arch/arm64/boot/dts/realtek/rtd16xx.dtsi
@@ -105,7 +105,8 @@ timer {
 		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
 			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
 			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
-			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>;
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 12 IRQ_TYPE_LEVEL_LOW>;
 	};
 
 	arm_pmu: pmu {
diff --git a/arch/arm64/boot/dts/renesas/r8a774a1.dtsi b/arch/arm64/boot/dts/renesas/r8a774a1.dtsi
index 36675f5bcdeaf..e66d86db6e6c4 100644
--- a/arch/arm64/boot/dts/renesas/r8a774a1.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a774a1.dtsi
@@ -262,7 +262,7 @@ scif_clk: scif {
 		clock-frequency = <0>;
 	};
 
-	soc {
+	soc: soc {
 		compatible = "simple-bus";
 		bootph-all;
 
diff --git a/arch/arm64/boot/dts/renesas/r8a774b1.dtsi b/arch/arm64/boot/dts/renesas/r8a774b1.dtsi
index ceef0104f75e8..62c6703917db4 100644
--- a/arch/arm64/boot/dts/renesas/r8a774b1.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a774b1.dtsi
@@ -146,7 +146,7 @@ scif_clk: scif {
 		clock-frequency = <0>;
 	};
 
-	soc {
+	soc: soc {
 		compatible = "simple-bus";
 		bootph-all;
 
diff --git a/arch/arm64/boot/dts/renesas/r8a774e1.dtsi b/arch/arm64/boot/dts/renesas/r8a774e1.dtsi
index 9df5f1a424004..0ae9bb72d2dda 100644
--- a/arch/arm64/boot/dts/renesas/r8a774e1.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a774e1.dtsi
@@ -326,7 +326,7 @@ scif_clk: scif {
 		clock-frequency = <0>;
 	};
 
-	soc {
+	soc: soc {
 		compatible = "simple-bus";
 		bootph-all;
 
diff --git a/arch/arm64/boot/dts/renesas/r8a77960.dtsi b/arch/arm64/boot/dts/renesas/r8a77960.dtsi
index ad36aa8e75435..4f9989b5e77a8 100644
--- a/arch/arm64/boot/dts/renesas/r8a77960.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77960.dtsi
@@ -311,7 +311,7 @@ scif_clk: scif {
 		clock-frequency = <0>;
 	};
 
-	soc {
+	soc: soc {
 		compatible = "simple-bus";
 		bootph-all;
 
diff --git a/arch/arm64/boot/dts/renesas/r8a77961.dtsi b/arch/arm64/boot/dts/renesas/r8a77961.dtsi
index 9d76e39eab72e..ad4491ba948f2 100644
--- a/arch/arm64/boot/dts/renesas/r8a77961.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77961.dtsi
@@ -311,7 +311,7 @@ scif_clk: scif {
 		clock-frequency = <0>;
 	};
 
-	soc {
+	soc: soc {
 		compatible = "simple-bus";
 		bootph-all;
 
diff --git a/arch/arm64/boot/dts/renesas/r8a77965-salvator-x.dts b/arch/arm64/boot/dts/renesas/r8a77965-salvator-x.dts
index f84c64ed4df7b..af8cfdccd2103 100644
--- a/arch/arm64/boot/dts/renesas/r8a77965-salvator-x.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77965-salvator-x.dts
@@ -30,3 +30,7 @@ &du {
 	clock-names = "du.0", "du.1", "du.3",
 		      "dclkin.0", "dclkin.1", "dclkin.3";
 };
+
+&gpu {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a77965-salvator-xs.dts b/arch/arm64/boot/dts/renesas/r8a77965-salvator-xs.dts
index a1d3c8d531cfe..f0d9c0124f69c 100644
--- a/arch/arm64/boot/dts/renesas/r8a77965-salvator-xs.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77965-salvator-xs.dts
@@ -30,3 +30,7 @@ &du {
 	clock-names = "du.0", "du.1", "du.3",
 		      "dclkin.0", "dclkin.1", "dclkin.3";
 };
+
+&gpu {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a77965-ulcb.dts b/arch/arm64/boot/dts/renesas/r8a77965-ulcb.dts
index 71704b67a20e1..127eb2ea3c920 100644
--- a/arch/arm64/boot/dts/renesas/r8a77965-ulcb.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77965-ulcb.dts
@@ -31,3 +31,7 @@ &du {
 	clock-names = "du.0", "du.1", "du.3",
 		      "dclkin.0", "dclkin.1", "dclkin.3";
 };
+
+&gpu {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a77965.dtsi b/arch/arm64/boot/dts/renesas/r8a77965.dtsi
index 611a9335c63ad..70708f5cf7467 100644
--- a/arch/arm64/boot/dts/renesas/r8a77965.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77965.dtsi
@@ -182,7 +182,7 @@ scif_clk: scif {
 		clock-frequency = <0>;
 	};
 
-	soc {
+	soc: soc {
 		compatible = "simple-bus";
 		bootph-all;
 
diff --git a/arch/arm64/boot/dts/renesas/r8a77970.dtsi b/arch/arm64/boot/dts/renesas/r8a77970.dtsi
index 1f6676e2795a4..f7f1f280fa0b6 100644
--- a/arch/arm64/boot/dts/renesas/r8a77970.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77970.dtsi
@@ -91,7 +91,7 @@ scif_clk: scif {
 		clock-frequency = <0>;
 	};
 
-	soc {
+	soc: soc {
 		compatible = "simple-bus";
 		bootph-all;
 
diff --git a/arch/arm64/boot/dts/renesas/r8a77980.dtsi b/arch/arm64/boot/dts/renesas/r8a77980.dtsi
index 86b7792d68fac..514dafe344786 100644
--- a/arch/arm64/boot/dts/renesas/r8a77980.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77980.dtsi
@@ -120,7 +120,7 @@ scif_clk: scif {
 		clock-frequency = <0>;
 	};
 
-	soc {
+	soc: soc {
 		compatible = "simple-bus";
 		bootph-all;
 
diff --git a/arch/arm64/boot/dts/renesas/r8a77995.dtsi b/arch/arm64/boot/dts/renesas/r8a77995.dtsi
index 5f3fcef7560cb..522a49db02587 100644
--- a/arch/arm64/boot/dts/renesas/r8a77995.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77995.dtsi
@@ -85,7 +85,7 @@ scif_clk: scif {
 		clock-frequency = <0>;
 	};
 
-	soc {
+	soc: soc {
 		compatible = "simple-bus";
 		bootph-all;
 
diff --git a/arch/arm64/boot/dts/renesas/r8a779g0.dtsi b/arch/arm64/boot/dts/renesas/r8a779g0.dtsi
index 82a7278836e57..8a291447b9045 100644
--- a/arch/arm64/boot/dts/renesas/r8a779g0.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a779g0.dtsi
@@ -589,6 +589,23 @@ tmu4: timer@ffc00000 {
 			status = "disabled";
 		};
 
+		mfis: system-controller@e6260000 {
+			compatible = "renesas,r8a779g0-mfis";
+			reg = <0 0xe6260000 0 0x10000>;
+			reg-names = "common";
+			interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 129 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 916 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 918 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 920 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 922 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 133 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 135 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0e", "ch1e", "ch2e", "ch3e", "ch4e", "ch5e",
+					  "ch6e", "ch7e", "ch8e", "ch9e", "ch10e", "ch11e";
+			#hwlock-cells = <1>;
+			#mbox-cells = <2>;
+			status = "disabled";
+		};
+
 		tsn0: ethernet@e6460000 {
 			compatible = "renesas,r8a779g0-ethertsn", "renesas,rcar-gen4-ethertsn";
 			reg = <0 0xe6460000 0 0x7000>,
diff --git a/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts b/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts
index d2b3fc08954a1..0ab303863155e 100644
--- a/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts
+++ b/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts
@@ -107,6 +107,47 @@ tee@8c400000 {
 			reg = <0x0 0x8c400000 0x0 0x02000000>;
 			no-map;
 		};
+
+		/* DRAM controller inline ECC areas */
+		ecc@10cccc0000 {
+			reg = <0x10 0xcccc0000 0x0 0x33340000>;
+			no-map;
+		};
+
+		ecc@12cccc0000 {
+			reg = <0x12 0xcccc0000 0x0 0x33340000>;
+			no-map;
+		};
+
+		ecc@14cccc0000 {
+			reg = <0x14 0xcccc0000 0x0 0x33340000>;
+			no-map;
+		};
+
+		ecc@16cccc0000 {
+			reg = <0x16 0xcccc0000 0x0 0x33340000>;
+			no-map;
+		};
+
+		ecc@18cccc0000 {
+			reg = <0x18 0xcccc0000 0x0 0x33340000>;
+			no-map;
+		};
+
+		ecc@1a66660000 {
+			reg = <0x1a 0x66660000 0x0 0x999a0000>;
+			no-map;
+		};
+
+		ecc@1c66660000 {
+			reg = <0x1c 0x66660000 0x0 0x999a0000>;
+			no-map;
+		};
+
+		ecc@1e66660000 {
+			reg = <0x1e 0x66660000 0x0 0x999a0000>;
+			no-map;
+		};
 	};
 };
 
diff --git a/arch/arm64/boot/dts/renesas/r8a78000.dtsi b/arch/arm64/boot/dts/renesas/r8a78000.dtsi
index 7780fb4e8351d..fb71974ef3905 100644
--- a/arch/arm64/boot/dts/renesas/r8a78000.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a78000.dtsi
@@ -143,6 +143,7 @@ a720_0: cpu@0 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x0>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_0>;
 		};
 
@@ -150,6 +151,7 @@ a720_1: cpu@100 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x100>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_1>;
 		};
 
@@ -157,6 +159,7 @@ a720_2: cpu@200 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x200>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_2>;
 		};
 
@@ -164,6 +167,7 @@ a720_3: cpu@300 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x300>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_3>;
 		};
 
@@ -171,6 +175,7 @@ a720_4: cpu@10000 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x10000>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_4>;
 		};
 
@@ -178,6 +183,7 @@ a720_5: cpu@10100 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x10100>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_5>;
 		};
 
@@ -185,6 +191,7 @@ a720_6: cpu@10200 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x10200>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_6>;
 		};
 
@@ -192,6 +199,7 @@ a720_7: cpu@10300 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x10300>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_7>;
 		};
 
@@ -199,6 +207,7 @@ a720_8: cpu@20000 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x20000>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_8>;
 		};
 
@@ -206,6 +215,7 @@ a720_9: cpu@20100 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x20100>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_9>;
 		};
 
@@ -213,6 +223,7 @@ a720_10: cpu@20200 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x20200>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_10>;
 		};
 
@@ -220,6 +231,7 @@ a720_11: cpu@20300 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x20300>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_11>;
 		};
 
@@ -227,6 +239,7 @@ a720_12: cpu@30000 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x30000>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_12>;
 		};
 
@@ -234,6 +247,7 @@ a720_13: cpu@30100 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x30100>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_13>;
 		};
 
@@ -241,6 +255,7 @@ a720_14: cpu@30200 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x30200>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_14>;
 		};
 
@@ -248,6 +263,7 @@ a720_15: cpu@30300 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x30300>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_15>;
 		};
 
@@ -255,6 +271,7 @@ a720_16: cpu@40000 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x40000>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_16>;
 		};
 
@@ -262,6 +279,7 @@ a720_17: cpu@40100 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x40100>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_17>;
 		};
 
@@ -269,6 +287,7 @@ a720_18: cpu@40200 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x40200>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_18>;
 		};
 
@@ -276,6 +295,7 @@ a720_19: cpu@40300 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x40300>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_19>;
 		};
 
@@ -283,6 +303,7 @@ a720_20: cpu@50000 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x50000>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_20>;
 		};
 
@@ -290,6 +311,7 @@ a720_21: cpu@50100 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x50100>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_21>;
 		};
 
@@ -297,6 +319,7 @@ a720_22: cpu@50200 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x50200>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_22>;
 		};
 
@@ -304,6 +327,7 @@ a720_23: cpu@50300 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x50300>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_23>;
 		};
 
@@ -311,6 +335,7 @@ a720_24: cpu@60000 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x60000>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_24>;
 		};
 
@@ -318,6 +343,7 @@ a720_25: cpu@60100 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x60100>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_25>;
 		};
 
@@ -325,6 +351,7 @@ a720_26: cpu@60200 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x60200>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_26>;
 		};
 
@@ -332,6 +359,7 @@ a720_27: cpu@60300 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x60300>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_27>;
 		};
 
@@ -339,6 +367,7 @@ a720_28: cpu@70000 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x70000>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_28>;
 		};
 
@@ -346,6 +375,7 @@ a720_29: cpu@70100 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x70100>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_29>;
 		};
 
@@ -353,6 +383,7 @@ a720_30: cpu@70200 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x70200>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_30>;
 		};
 
@@ -360,6 +391,7 @@ a720_31: cpu@70300 {
 			compatible = "arm,cortex-a720ae";
 			reg = <0x0 0x70300>;
 			device_type = "cpu";
+			enable-method = "psci";
 			next-level-cache = <&L2_CA720_31>;
 		};
 
diff --git a/arch/arm64/boot/dts/renesas/r9a08g046.dtsi b/arch/arm64/boot/dts/renesas/r9a08g046.dtsi
index 03bdee8705281..c63a857f0e5b0 100644
--- a/arch/arm64/boot/dts/renesas/r9a08g046.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a08g046.dtsi
@@ -358,6 +358,75 @@ i2c3: i2c@10090c00 {
 			status = "disabled";
 		};
 
+		rspi0: spi@100b0000 {
+			compatible = "renesas,r9a08g046-rspi";
+			reg = <0 0x100b0000 0 0x1000>;
+			interrupts = <GIC_SPI 440 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 441 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 442 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 443 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 444 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "idle", "error", "end", "rx", "tx";
+			clocks = <&cpg CPG_MOD R9A08G046_RSPI0_PCLK>,
+				 <&cpg CPG_MOD R9A08G046_RSPI0_TCLK>;
+			clock-names = "pclk", "tclk";
+			resets = <&cpg R9A08G046_RSPI0_PRESETN>,
+				 <&cpg R9A08G046_RSPI0_TRESETN>;
+			reset-names = "presetn", "tresetn";
+			dmas = <&dmac 0x26f2>, <&dmac 0x26f1>;
+			dma-names = "rx", "tx";
+			power-domains = <&cpg>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		rspi1: spi@100e9000 {
+			compatible = "renesas,r9a08g046-rspi";
+			reg = <0 0x100e9000 0 0x1000>;
+			interrupts = <GIC_SPI 445 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 446 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 447 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 448 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 449 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "idle", "error", "end", "rx", "tx";
+			clocks = <&cpg CPG_MOD R9A08G046_RSPI1_PCLK>,
+				 <&cpg CPG_MOD R9A08G046_RSPI1_TCLK>;
+			clock-names = "pclk", "tclk";
+			resets = <&cpg R9A08G046_RSPI1_PRESETN>,
+				 <&cpg R9A08G046_RSPI1_TRESETN>;
+			reset-names = "presetn", "tresetn";
+			dmas = <&dmac 0x26f6>, <&dmac 0x26f5>;
+			dma-names = "rx", "tx";
+			power-domains = <&cpg>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		rspi2: spi@100ea000 {
+			compatible = "renesas,r9a08g046-rspi";
+			reg = <0 0x100ea000 0 0x1000>;
+			interrupts = <GIC_SPI 450 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 451 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 452 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 453 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 454 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "idle", "error", "end", "rx", "tx";
+			clocks = <&cpg CPG_MOD R9A08G046_RSPI2_PCLK>,
+				 <&cpg CPG_MOD R9A08G046_RSPI2_TCLK>;
+			clock-names = "pclk", "tclk";
+			resets = <&cpg R9A08G046_RSPI2_PRESETN>,
+				 <&cpg R9A08G046_RSPI2_TRESETN>;
+			reset-names = "presetn", "tresetn";
+			dmas = <&dmac 0x26fa>, <&dmac 0x26f9>;
+			dma-names = "rx", "tx";
+			power-domains = <&cpg>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		rsci0: serial@100b8000 {
 			compatible = "renesas,r9a08g046-rsci";
 			reg = <0 0x100b8000 0 0x1000>;
diff --git a/arch/arm64/boot/dts/renesas/r9a08g046l48-smarc.dts b/arch/arm64/boot/dts/renesas/r9a08g046l48-smarc.dts
index 32d3b08a3cf32..5289efd1a430a 100644
--- a/arch/arm64/boot/dts/renesas/r9a08g046l48-smarc.dts
+++ b/arch/arm64/boot/dts/renesas/r9a08g046l48-smarc.dts
@@ -13,6 +13,7 @@
 #define SW_DPI_EN		0
 #define SW_GPIO4		1
 #define SW_I3C_EN		0
+#define SW_SER0_PMOD		1
 
 #define PMOD_GPIO4		0
 #define PMOD_GPIO6		0
@@ -37,6 +38,9 @@ / {
 	aliases {
 		i2c2 = &i2c2;
 		i2c3 = &i2c3;
+		serial0 = &rsci2;
+		serial1 = &rsci3;
+		serial2 = &rsci1;
 		serial3 = &scif0;
 	};
 
@@ -134,6 +138,38 @@ i2c3_pins: i2c3 {
 			 <RZG3L_PORT_PINMUX(2, 1, 4)>; /* RIIC3_SDA */
 	};
 
+	rsci1_pins: rsci1 {
+		pinmux = <RZG3L_PORT_PINMUX(D, 4, 5)>, /* RSCI1_RXD_MISO_SCL */
+			 <RZG3L_PORT_PINMUX(D, 5, 5)>, /* RSCI1_TXD_MOSI_SDA */
+			 <RZG3L_PORT_PINMUX(D, 6, 6)>, /* RSCI1_CTS */
+			 <RZG3L_PORT_PINMUX(D, 7, 5)>; /* RSCI1_SS_CTS#_RTS# */
+		bias-pull-up;
+	};
+
+	rsci2_pins: rsci2 {
+		pinmux = <RZG3L_PORT_PINMUX(7, 6, 5)>, /* RSCI2_RXD_MISO_SCL */
+			 <RZG3L_PORT_PINMUX(7, 7, 5)>, /* RSCI2_TXD_MOSI_SDA */
+			 <RZG3L_PORT_PINMUX(8, 0, 6)>, /* RSCI2_CTS */
+			 <RZG3L_PORT_PINMUX(8, 1, 5)>; /* RSCI2_SS_CTS#_RTS# */
+		bias-pull-up;
+		power-source = <1800>;
+	};
+
+	rsci3_pins: rsci3 {
+		pinmux = <RZG3L_PORT_PINMUX(8, 2, 5)>, /* RSCI3_RXD_MISO_SCL */
+			 <RZG3L_PORT_PINMUX(8, 3, 5)>; /* RSCI3_TXD_MOSI_SDA */
+		bias-pull-up;
+		power-source = <1800>;
+	};
+
+	rspi2_pins: rspi2 {
+		pinmux = <RZG3L_PORT_PINMUX(6, 0, 2)>, /* RSPI2_MISO */
+			 <RZG3L_PORT_PINMUX(6, 1, 2)>, /* RSPI2_MOSI */
+			 <RZG3L_PORT_PINMUX(6, 2, 2)>, /* RSPI2_SCK */
+			 <RZG3L_PORT_PINMUX(6, 3, 2)>; /* RSPI2_SSLA */
+			 power-source = <1800>;
+	};
+
 	scif0_pins: scif0 {
 		pins = "SCIF0_TXD", "SCIF0_RXD";
 		power-source = <1800>;
@@ -147,6 +183,48 @@ ssi0_pins: ssi0 {
 	};
 };
 
+#if SW_SER0_PMOD
+&rsci1 {
+	pinctrl-0 = <&rsci1_pins>;
+	pinctrl-names = "default";
+
+	uart-has-rtscts;
+
+	status = "okay";
+};
+#endif
+
+#if !SW_DPI_EN
+&rsci2 {
+	pinctrl-0 = <&rsci2_pins>;
+	pinctrl-names = "default";
+
+	uart-has-rtscts;
+
+	status = "okay";
+};
+#endif
+
+#if (!SW_DPI_EN)
+&rsci3 {
+	pinctrl-0 = <&rsci3_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+#endif
+
+#if !SW_DPI_EN
+&rspi2 {
+	pinctrl-0 = <&rspi2_pins>;
+	pinctrl-names = "default";
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	status = "okay";
+};
+#endif
+
 &scif0 {
 	pinctrl-0 = <&scif0_pins>;
 	pinctrl-names = "default";
diff --git a/arch/arm64/boot/dts/renesas/r9a09g047.dtsi b/arch/arm64/boot/dts/renesas/r9a09g047.dtsi
index b48da8534a3df..73757e8e21970 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g047.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g047.dtsi
@@ -21,6 +21,20 @@ audio_extal_clk: audio-clk {
 		clock-frequency = <0>;
 	};
 
+	audio_clkb: audio-b-clk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board */
+		clock-frequency = <0>;
+	};
+
+	audio_clkc: audio-c-clk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board */
+		clock-frequency = <0>;
+	};
+
 	/*
 	 * The default cluster table is based on the assumption that the PLLCA55 clock
 	 * frequency is set to 1.7GHz. The PLLCA55 clock frequency can be set to
@@ -271,8 +285,10 @@ pinctrl: pinctrl@10410000 {
 		cpg: clock-controller@10420000 {
 			compatible = "renesas,r9a09g047-cpg";
 			reg = <0 0x10420000 0 0x10000>;
-			clocks = <&audio_extal_clk>, <&rtxin_clk>, <&qextal_clk>;
-			clock-names = "audio_extal", "rtxin", "qextal";
+			clocks = <&audio_extal_clk>, <&rtxin_clk>, <&qextal_clk>,
+				 <&audio_clkb>, <&audio_clkc>;
+			clock-names = "audio_extal", "rtxin", "qextal",
+				      "audio_clkb", "audio_clkc";
 			#clock-cells = <2>;
 			#reset-cells = <1>;
 			#power-domain-cells = <0>;
@@ -896,6 +912,506 @@ rsci9: serial@12803000 {
 			status = "disabled";
 		};
 
+		snd_rzg3e: sound@13c00000 {
+			compatible = "renesas,r9a09g047-sound";
+			reg = <0 0x13c00000 0 0x20000>, /* SCU */
+			      <0 0x13c20000 0 0x10000>, /* ADG */
+			      <0 0x13c30000 0 0x1000>,  /* SSIU */
+			      <0 0x13c31000 0 0x1f000>, /* SSI */
+			      <0 0x13c50000 0 0x10000>; /* Audio DMAC peri peri */
+			reg-names = "scu", "adg", "ssiu", "ssi", "audmapp";
+			clocks = <&cpg CPG_MOD 0xf5>,
+				 <&cpg CPG_MOD 0x181>, <&cpg CPG_MOD 0x182>,
+				 <&cpg CPG_MOD 0x183>, <&cpg CPG_MOD 0x184>,
+				 <&cpg CPG_MOD 0x185>, <&cpg CPG_MOD 0x186>,
+				 <&cpg CPG_MOD 0x187>, <&cpg CPG_MOD 0x188>,
+				 <&cpg CPG_MOD 0x189>, <&cpg CPG_MOD 0x18a>,
+				 <&cpg CPG_MOD 0x174>, <&cpg CPG_MOD 0x175>,
+				 <&cpg CPG_MOD 0x176>, <&cpg CPG_MOD 0x177>,
+				 <&cpg CPG_MOD 0x178>, <&cpg CPG_MOD 0x179>,
+				 <&cpg CPG_MOD 0x17a>, <&cpg CPG_MOD 0x17b>,
+				 <&cpg CPG_MOD 0x17c>, <&cpg CPG_MOD 0x17d>,
+				 <&cpg CPG_MOD 0x172>, <&cpg CPG_MOD 0x173>,
+				 <&cpg CPG_MOD 0x172>, <&cpg CPG_MOD 0x173>,
+				 <&cpg CPG_MOD 0x170>, <&cpg CPG_MOD 0x171>,
+				 <&cpg CPG_MOD 0xfb>, <&cpg CPG_MOD 0xfc>,
+				 <&cpg CPG_MOD 0xfd>, <&cpg CPG_MOD 0xfa>,
+				 <&cpg CPG_MOD 0x180>,
+				 <&cpg CPG_MOD 0xf6>, <&cpg CPG_MOD 0xf7>,
+				 <&cpg CPG_MOD 0x17e>,
+				 <&cpg CPG_MOD 0x160>, <&cpg CPG_MOD 0x161>,
+				 <&cpg CPG_MOD 0x162>, <&cpg CPG_MOD 0x163>,
+				 <&cpg CPG_MOD 0x164>, <&cpg CPG_MOD 0x165>,
+				 <&cpg CPG_MOD 0x166>, <&cpg CPG_MOD 0x167>,
+				 <&cpg CPG_MOD 0x168>, <&cpg CPG_MOD 0x169>,
+				 <&cpg CPG_MOD 0xf8>, <&cpg CPG_MOD 0xf9>;
+			clock-names = "ssi-all",
+				      "ssi-0", "ssi-1",
+				      "ssi-2", "ssi-3",
+				      "ssi-4", "ssi-5",
+				      "ssi-6", "ssi-7",
+				      "ssi-8", "ssi-9",
+				      "src-0", "src-1",
+				      "src-2", "src-3",
+				      "src-4", "src-5",
+				      "src-6", "src-7",
+				      "src-8", "src-9",
+				      "mix-0", "mix-1",
+				      "ctu-0", "ctu-1",
+				      "dvc-0", "dvc-1",
+				      "audio-clka", "audio-clkb",
+				      "audio-clkc", "audio-clki",
+				      "ssif_supply",
+				      "scu", "scu_x2",
+				      "scu_supply",
+				      "adg-ssi-0", "adg-ssi-1",
+				      "adg-ssi-2", "adg-ssi-3",
+				      "adg-ssi-4", "adg-ssi-5",
+				      "adg-ssi-6", "adg-ssi-7",
+				      "adg-ssi-8", "adg-ssi-9",
+				      "audmapp", "adg";
+			power-domains = <&cpg>;
+			resets = <&cpg 0xe1>,
+				 <&cpg 0xe2>, <&cpg 0xe3>,
+				 <&cpg 0xe4>, <&cpg 0xe5>,
+				 <&cpg 0xe6>, <&cpg 0xe7>,
+				 <&cpg 0xe8>, <&cpg 0xe9>,
+				 <&cpg 0xea>, <&cpg 0xeb>,
+				 <&cpg 0xec>, <&cpg 0xee>,
+				 <&cpg 0xed>;
+			reset-names = "ssi-all",
+				      "ssi-0", "ssi-1",
+				      "ssi-2", "ssi-3",
+				      "ssi-4", "ssi-5",
+				      "ssi-6", "ssi-7",
+				      "ssi-8", "ssi-9",
+				      "scu", "adg",
+				      "audmapp";
+			status = "disabled";
+
+			ctu {
+				ctu00: ctu-0 { };
+				ctu01: ctu-1 { };
+				ctu02: ctu-2 { };
+				ctu03: ctu-3 { };
+				ctu10: ctu-4 { };
+				ctu11: ctu-5 { };
+				ctu12: ctu-6 { };
+				ctu13: ctu-7 { };
+			};
+
+			dvc {
+				dvc0: dvc-0 {
+					dmas = <&dmac0 0x1db3>, <&dmac1 0x1db3>,
+					       <&dmac2 0x1db3>, <&dmac3 0x1db3>,
+					       <&dmac4 0x1db3>;
+					dma-names = "tx", "tx", "tx", "tx", "tx";
+				};
+				dvc1: dvc-1 {
+					dmas = <&dmac0 0x1db4>, <&dmac1 0x1db4>,
+					       <&dmac2 0x1db4>, <&dmac3 0x1db4>,
+					       <&dmac4 0x1db4>;
+					dma-names = "tx", "tx", "tx", "tx", "tx";
+				};
+			};
+
+			mix {
+				mix0: mix-0 { };
+				mix1: mix-1 { };
+			};
+
+			src {
+				src0: src-0 {
+					interrupts = <GIC_SPI 902 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&dmac0 0x1d9f>, <&dmac0 0x1da9>,
+					       <&dmac1 0x1d9f>, <&dmac1 0x1da9>,
+					       <&dmac2 0x1d9f>, <&dmac2 0x1da9>,
+					       <&dmac3 0x1d9f>, <&dmac3 0x1da9>,
+					       <&dmac4 0x1d9f>, <&dmac4 0x1da9>;
+					dma-names = "rx", "tx", "rx", "tx", "rx", "tx", "rx", "tx",
+						    "rx", "tx";
+				};
+				src1: src-1 {
+					interrupts = <GIC_SPI 903 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&dmac0 0x1da0>, <&dmac0 0x1daa>,
+					       <&dmac1 0x1da0>, <&dmac1 0x1daa>,
+					       <&dmac2 0x1da0>, <&dmac2 0x1daa>,
+					       <&dmac3 0x1da0>, <&dmac3 0x1daa>,
+					       <&dmac4 0x1da0>, <&dmac4 0x1daa>;
+					dma-names = "rx", "tx", "rx", "tx", "rx", "tx", "rx", "tx",
+						    "rx", "tx";
+				};
+				src2: src-2 {
+					interrupts = <GIC_SPI 904 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&dmac0 0x1da1>, <&dmac0 0x1dab>,
+					       <&dmac1 0x1da1>, <&dmac1 0x1dab>,
+					       <&dmac2 0x1da1>, <&dmac2 0x1dab>,
+					       <&dmac3 0x1da1>, <&dmac3 0x1dab>,
+					       <&dmac4 0x1da1>, <&dmac4 0x1dab>;
+					dma-names = "rx", "tx", "rx", "tx", "rx", "tx", "rx", "tx",
+						    "rx", "tx";
+				};
+				src3: src-3 {
+					interrupts = <GIC_SPI 905 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&dmac0 0x1da2>, <&dmac0 0x1dac>,
+					       <&dmac1 0x1da2>, <&dmac1 0x1dac>,
+					       <&dmac2 0x1da2>, <&dmac2 0x1dac>,
+					       <&dmac3 0x1da2>, <&dmac3 0x1dac>,
+					       <&dmac4 0x1da2>, <&dmac4 0x1dac>;
+					dma-names = "rx", "tx", "rx", "tx", "rx", "tx", "rx", "tx",
+						    "rx", "tx";
+				};
+				src4: src-4 {
+					interrupts = <GIC_SPI 906 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&dmac0 0x1da3>, <&dmac0 0x1dad>,
+					       <&dmac1 0x1da3>, <&dmac1 0x1dad>,
+					       <&dmac2 0x1da3>, <&dmac2 0x1dad>,
+					       <&dmac3 0x1da3>, <&dmac3 0x1dad>,
+					       <&dmac4 0x1da3>, <&dmac4 0x1dad>;
+					dma-names = "rx", "tx", "rx", "tx", "rx", "tx", "rx", "tx",
+						    "rx", "tx";
+				};
+				src5: src-5 {
+					interrupts = <GIC_SPI 907 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&dmac0 0x1da4>, <&dmac0 0x1dae>,
+					       <&dmac1 0x1da4>, <&dmac1 0x1dae>,
+					       <&dmac2 0x1da4>, <&dmac2 0x1dae>,
+					       <&dmac3 0x1da4>, <&dmac3 0x1dae>,
+					       <&dmac4 0x1da4>, <&dmac4 0x1dae>;
+					dma-names = "rx", "tx", "rx", "tx", "rx", "tx", "rx", "tx",
+						    "rx", "tx";
+				};
+				src6: src-6 {
+					interrupts = <GIC_SPI 908 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&dmac0 0x1da5>, <&dmac0 0x1daf>,
+					       <&dmac1 0x1da5>, <&dmac1 0x1daf>,
+					       <&dmac2 0x1da5>, <&dmac2 0x1daf>,
+					       <&dmac3 0x1da5>, <&dmac3 0x1daf>,
+					       <&dmac4 0x1da5>, <&dmac4 0x1daf>;
+					dma-names = "rx", "tx", "rx", "tx", "rx", "tx", "rx", "tx",
+						    "rx", "tx";
+				};
+				src7: src-7 {
+					interrupts = <GIC_SPI 909 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&dmac0 0x1da6>, <&dmac0 0x1db0>,
+					       <&dmac1 0x1da6>, <&dmac1 0x1db0>,
+					       <&dmac2 0x1da6>, <&dmac2 0x1db0>,
+					       <&dmac3 0x1da6>, <&dmac3 0x1db0>,
+					       <&dmac4 0x1da6>, <&dmac4 0x1db0>;
+					dma-names = "rx", "tx", "rx", "tx", "rx", "tx", "rx", "tx",
+						    "rx", "tx";
+				};
+				src8: src-8 {
+					interrupts = <GIC_SPI 910 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&dmac0 0x1da7>, <&dmac0 0x1db1>,
+					       <&dmac1 0x1da7>, <&dmac1 0x1db1>,
+					       <&dmac2 0x1da7>, <&dmac2 0x1db1>,
+					       <&dmac3 0x1da7>, <&dmac3 0x1db1>,
+					       <&dmac4 0x1da7>, <&dmac4 0x1db1>;
+					dma-names = "rx", "tx", "rx", "tx", "rx", "tx", "rx", "tx",
+						    "rx", "tx";
+				};
+				src9: src-9 {
+					interrupts = <GIC_SPI 911 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&dmac0 0x1da8>, <&dmac0 0x1db2>,
+					       <&dmac1 0x1da8>, <&dmac1 0x1db2>,
+					       <&dmac2 0x1da8>, <&dmac2 0x1db2>,
+					       <&dmac3 0x1da8>, <&dmac3 0x1db2>,
+					       <&dmac4 0x1da8>, <&dmac4 0x1db2>;
+					dma-names = "rx", "tx", "rx", "tx", "rx", "tx", "rx", "tx",
+						    "rx", "tx";
+				};
+			};
+
+			ssi {
+				ssi0: ssi-0 {
+					interrupts = <GIC_SPI 889 IRQ_TYPE_LEVEL_HIGH>;
+				};
+				ssi1: ssi-1 {
+					interrupts = <GIC_SPI 890 IRQ_TYPE_LEVEL_HIGH>;
+				};
+				ssi2: ssi-2 {
+					interrupts = <GIC_SPI 891 IRQ_TYPE_LEVEL_HIGH>;
+				};
+				ssi3: ssi-3 {
+					interrupts = <GIC_SPI 892 IRQ_TYPE_LEVEL_HIGH>;
+				};
+				ssi4: ssi-4 {
+					interrupts = <GIC_SPI 893 IRQ_TYPE_LEVEL_HIGH>;
+				};
+				ssi5: ssi-5 {
+					interrupts = <GIC_SPI 894 IRQ_TYPE_LEVEL_HIGH>;
+				};
+				ssi6: ssi-6 {
+					interrupts = <GIC_SPI 895 IRQ_TYPE_LEVEL_HIGH>;
+				};
+				ssi7: ssi-7 {
+					interrupts = <GIC_SPI 896 IRQ_TYPE_LEVEL_HIGH>;
+				};
+				ssi8: ssi-8 {
+					interrupts = <GIC_SPI 897 IRQ_TYPE_LEVEL_HIGH>;
+				};
+				ssi9: ssi-9 {
+					interrupts = <GIC_SPI 898 IRQ_TYPE_LEVEL_HIGH>;
+				};
+			};
+
+			ssiu {
+				ssiu00: ssiu-0 {
+					dmas = <&dmac0 0x1d61>, <&dmac0 0x1d62>,
+					       <&dmac1 0x1d61>, <&dmac1 0x1d62>,
+					       <&dmac2 0x1d61>, <&dmac2 0x1d62>,
+					       <&dmac3 0x1d61>, <&dmac3 0x1d62>,
+					       <&dmac4 0x1d61>, <&dmac4 0x1d62>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu01: ssiu-1 {
+					dmas = <&dmac0 0x1d63>, <&dmac0 0x1d64>,
+					       <&dmac1 0x1d63>, <&dmac1 0x1d64>,
+					       <&dmac2 0x1d63>, <&dmac2 0x1d64>,
+					       <&dmac3 0x1d63>, <&dmac3 0x1d64>,
+					       <&dmac4 0x1d63>, <&dmac4 0x1d64>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu02: ssiu-2 {
+					dmas = <&dmac0 0x1d65>, <&dmac0 0x1d66>,
+					       <&dmac1 0x1d65>, <&dmac1 0x1d66>,
+					       <&dmac2 0x1d65>, <&dmac2 0x1d66>,
+					       <&dmac3 0x1d65>, <&dmac3 0x1d66>,
+					       <&dmac4 0x1d65>, <&dmac4 0x1d66>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu03: ssiu-3 {
+					dmas = <&dmac0 0x1d67>, <&dmac0 0x1d68>,
+					       <&dmac1 0x1d67>, <&dmac1 0x1d68>,
+					       <&dmac2 0x1d67>, <&dmac2 0x1d68>,
+					       <&dmac3 0x1d67>, <&dmac3 0x1d68>,
+					       <&dmac4 0x1d67>, <&dmac4 0x1d68>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu10: ssiu-4 {
+					dmas = <&dmac0 0x1d69>, <&dmac0 0x1d6a>,
+					       <&dmac1 0x1d69>, <&dmac1 0x1d6a>,
+					       <&dmac2 0x1d69>, <&dmac2 0x1d6a>,
+					       <&dmac3 0x1d69>, <&dmac3 0x1d6a>,
+					       <&dmac4 0x1d69>, <&dmac4 0x1d6a>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu11: ssiu-5 {
+					dmas = <&dmac0 0x1d6b>, <&dmac0 0x1d6c>,
+					       <&dmac1 0x1d6b>, <&dmac1 0x1d6c>,
+					       <&dmac2 0x1d6b>, <&dmac2 0x1d6c>,
+					       <&dmac3 0x1d6b>, <&dmac3 0x1d6c>,
+					       <&dmac4 0x1d6b>, <&dmac4 0x1d6c>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu12: ssiu-6 {
+					dmas = <&dmac0 0x1d6d>, <&dmac0 0x1d6e>,
+					       <&dmac1 0x1d6d>, <&dmac1 0x1d6e>,
+					       <&dmac2 0x1d6d>, <&dmac2 0x1d6e>,
+					       <&dmac3 0x1d6d>, <&dmac3 0x1d6e>,
+					       <&dmac4 0x1d6d>, <&dmac4 0x1d6e>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu13: ssiu-7 {
+					dmas = <&dmac0 0x1d6f>, <&dmac0 0x1d70>,
+					       <&dmac1 0x1d6f>, <&dmac1 0x1d70>,
+					       <&dmac2 0x1d6f>, <&dmac2 0x1d70>,
+					       <&dmac3 0x1d6f>, <&dmac3 0x1d70>,
+					       <&dmac4 0x1d6f>, <&dmac4 0x1d70>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu20: ssiu-8 {
+					dmas = <&dmac0 0x1d71>, <&dmac0 0x1d72>,
+					       <&dmac1 0x1d71>, <&dmac1 0x1d72>,
+					       <&dmac2 0x1d71>, <&dmac2 0x1d72>,
+					       <&dmac3 0x1d71>, <&dmac3 0x1d72>,
+					       <&dmac4 0x1d71>, <&dmac4 0x1d72>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu21: ssiu-9 {
+					dmas = <&dmac0 0x1d73>, <&dmac0 0x1d74>,
+					       <&dmac1 0x1d73>, <&dmac1 0x1d74>,
+					       <&dmac2 0x1d73>, <&dmac2 0x1d74>,
+					       <&dmac3 0x1d73>, <&dmac3 0x1d74>,
+					       <&dmac4 0x1d73>, <&dmac4 0x1d74>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu22: ssiu-10 {
+					dmas = <&dmac0 0x1d75>, <&dmac0 0x1d76>,
+					       <&dmac1 0x1d75>, <&dmac1 0x1d76>,
+					       <&dmac2 0x1d75>, <&dmac2 0x1d76>,
+					       <&dmac3 0x1d75>, <&dmac3 0x1d76>,
+					       <&dmac4 0x1d75>, <&dmac4 0x1d76>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu23: ssiu-11 {
+					dmas = <&dmac0 0x1d77>, <&dmac0 0x1d78>,
+					       <&dmac1 0x1d77>, <&dmac1 0x1d78>,
+					       <&dmac2 0x1d77>, <&dmac2 0x1d78>,
+					       <&dmac3 0x1d77>, <&dmac3 0x1d78>,
+					       <&dmac4 0x1d77>, <&dmac4 0x1d78>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu30: ssiu-12 {
+					dmas = <&dmac0 0x1d79>, <&dmac0 0x1d7a>,
+					       <&dmac1 0x1d79>, <&dmac1 0x1d7a>,
+					       <&dmac2 0x1d79>, <&dmac2 0x1d7a>,
+					       <&dmac3 0x1d79>, <&dmac3 0x1d7a>,
+					       <&dmac4 0x1d79>, <&dmac4 0x1d7a>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu31: ssiu-13 {
+					dmas = <&dmac0 0x1d7b>, <&dmac0 0x1d7c>,
+					       <&dmac1 0x1d7b>, <&dmac1 0x1d7c>,
+					       <&dmac2 0x1d7b>, <&dmac2 0x1d7c>,
+					       <&dmac3 0x1d7b>, <&dmac3 0x1d7c>,
+					       <&dmac4 0x1d7b>, <&dmac4 0x1d7c>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu32: ssiu-14 {
+					dmas = <&dmac0 0x1d7d>, <&dmac0 0x1d7e>,
+					       <&dmac1 0x1d7d>, <&dmac1 0x1d7e>,
+					       <&dmac2 0x1d7d>, <&dmac2 0x1d7e>,
+					       <&dmac3 0x1d7d>, <&dmac3 0x1d7e>,
+					       <&dmac4 0x1d7d>, <&dmac4 0x1d7e>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu33: ssiu-15 {
+					dmas = <&dmac0 0x1d7f>, <&dmac0 0x1d80>,
+					       <&dmac1 0x1d7f>, <&dmac1 0x1d80>,
+					       <&dmac2 0x1d7f>, <&dmac2 0x1d80>,
+					       <&dmac3 0x1d7f>, <&dmac3 0x1d80>,
+					       <&dmac4 0x1d7f>, <&dmac4 0x1d80>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu40: ssiu-16 {
+					dmas = <&dmac0 0x1d81>, <&dmac0 0x1d82>,
+					       <&dmac1 0x1d81>, <&dmac1 0x1d82>,
+					       <&dmac2 0x1d81>, <&dmac2 0x1d82>,
+					       <&dmac3 0x1d81>, <&dmac3 0x1d82>,
+					       <&dmac4 0x1d81>, <&dmac4 0x1d82>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu41: ssiu-17 {
+					dmas = <&dmac0 0x1d83>, <&dmac0 0x1d84>,
+					       <&dmac1 0x1d83>, <&dmac1 0x1d84>,
+					       <&dmac2 0x1d83>, <&dmac2 0x1d84>,
+					       <&dmac3 0x1d83>, <&dmac3 0x1d84>,
+					       <&dmac4 0x1d83>, <&dmac4 0x1d84>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu42: ssiu-18 {
+					dmas = <&dmac0 0x1d85>, <&dmac0 0x1d86>,
+					       <&dmac1 0x1d85>, <&dmac1 0x1d86>,
+					       <&dmac2 0x1d85>, <&dmac2 0x1d86>,
+					       <&dmac3 0x1d85>, <&dmac3 0x1d86>,
+					       <&dmac4 0x1d85>, <&dmac4 0x1d86>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu43: ssiu-19 {
+					dmas = <&dmac0 0x1d87>, <&dmac0 0x1d88>,
+					       <&dmac1 0x1d87>, <&dmac1 0x1d88>,
+					       <&dmac2 0x1d87>, <&dmac2 0x1d88>,
+					       <&dmac3 0x1d87>, <&dmac3 0x1d88>,
+					       <&dmac4 0x1d87>, <&dmac4 0x1d88>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu50: ssiu-20 {
+					dmas = <&dmac0 0x1d89>, <&dmac0 0x1d8a>,
+					       <&dmac1 0x1d89>, <&dmac1 0x1d8a>,
+					       <&dmac2 0x1d89>, <&dmac2 0x1d8a>,
+					       <&dmac3 0x1d89>, <&dmac3 0x1d8a>,
+					       <&dmac4 0x1d89>, <&dmac4 0x1d8a>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu60: ssiu-21 {
+					dmas = <&dmac0 0x1d8b>, <&dmac0 0x1d8c>,
+					       <&dmac1 0x1d8b>, <&dmac1 0x1d8c>,
+					       <&dmac2 0x1d8b>, <&dmac2 0x1d8c>,
+					       <&dmac3 0x1d8b>, <&dmac3 0x1d8c>,
+					       <&dmac4 0x1d8b>, <&dmac4 0x1d8c>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu70: ssiu-22 {
+					dmas = <&dmac0 0x1d8d>, <&dmac0 0x1d8e>,
+					       <&dmac1 0x1d8d>, <&dmac1 0x1d8e>,
+					       <&dmac2 0x1d8d>, <&dmac2 0x1d8e>,
+					       <&dmac3 0x1d8d>, <&dmac3 0x1d8e>,
+					       <&dmac4 0x1d8d>, <&dmac4 0x1d8e>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu80: ssiu-23 {
+					dmas = <&dmac0 0x1d8f>, <&dmac0 0x1d90>,
+					       <&dmac1 0x1d8f>, <&dmac1 0x1d90>,
+					       <&dmac2 0x1d8f>, <&dmac2 0x1d90>,
+					       <&dmac3 0x1d8f>, <&dmac3 0x1d90>,
+					       <&dmac4 0x1d8f>, <&dmac4 0x1d90>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu90: ssiu-24 {
+					dmas = <&dmac0 0x1d91>, <&dmac0 0x1d92>,
+					       <&dmac1 0x1d91>, <&dmac1 0x1d92>,
+					       <&dmac2 0x1d91>, <&dmac2 0x1d92>,
+					       <&dmac3 0x1d91>, <&dmac3 0x1d92>,
+					       <&dmac4 0x1d91>, <&dmac4 0x1d92>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu91: ssiu-25 {
+					dmas = <&dmac0 0x1d93>, <&dmac0 0x1d94>,
+					       <&dmac1 0x1d93>, <&dmac1 0x1d94>,
+					       <&dmac2 0x1d93>, <&dmac2 0x1d94>,
+					       <&dmac3 0x1d93>, <&dmac3 0x1d94>,
+					       <&dmac4 0x1d93>, <&dmac4 0x1d94>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu92: ssiu-26 {
+					dmas = <&dmac0 0x1d95>, <&dmac0 0x1d96>,
+					       <&dmac1 0x1d95>, <&dmac1 0x1d96>,
+					       <&dmac2 0x1d95>, <&dmac2 0x1d96>,
+					       <&dmac3 0x1d95>, <&dmac3 0x1d96>,
+					       <&dmac4 0x1d95>, <&dmac4 0x1d96>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+				ssiu93: ssiu-27 {
+					dmas = <&dmac0 0x1d97>, <&dmac0 0x1d98>,
+					       <&dmac1 0x1d97>, <&dmac1 0x1d98>,
+					       <&dmac2 0x1d97>, <&dmac2 0x1d98>,
+					       <&dmac3 0x1d97>, <&dmac3 0x1d98>,
+					       <&dmac4 0x1d97>, <&dmac4 0x1d98>;
+					dma-names = "tx", "rx", "tx", "rx", "tx", "rx", "tx", "rx",
+						    "tx", "rx";
+				};
+			};
+		};
+
 		wdt1: watchdog@14400000 {
 			compatible = "renesas,r9a09g047-wdt", "renesas,r9a09g057-wdt";
 			reg = <0 0x14400000 0 0x400>;
@@ -931,7 +1447,7 @@ pcie: pcie@13400000 {
 			reg = <0 0x13400000 0 0x10000>;
 			ranges = <0x02000000 0 0x30000000 0 0x30000000 0 0x08000000>,
 				 <0x43000000 4 0x40000000 4 0x40000000 6 0x00000000>;
-			dma-ranges = <0x42000000 0 0x40000000 0 0x40000000 2 0x00000000>;
+			dma-ranges = <0x43000000 0 0x40000000 0 0x40000000 2 0x00000000>;
 			bus-range = <0x0 0xff>;
 			interrupts = <GIC_SPI 800 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 801 IRQ_TYPE_LEVEL_HIGH>,
@@ -1265,6 +1781,7 @@ sdhi0: mmc@15c00000  {
 			clocks = <&cpg CPG_MOD 0xa3>, <&cpg CPG_MOD 0xa5>,
 				 <&cpg CPG_MOD 0xa4>, <&cpg CPG_MOD 0xa6>;
 			clock-names = "core", "clkh", "cd", "aclk";
+			max-frequency = <200000000>;
 			resets = <&cpg 0xa7>;
 			power-domains = <&cpg>;
 			status = "disabled";
@@ -1285,6 +1802,7 @@ sdhi1: mmc@15c10000 {
 			clocks = <&cpg CPG_MOD 0xa7>, <&cpg CPG_MOD 0xa9>,
 				 <&cpg CPG_MOD 0xa8>, <&cpg CPG_MOD 0xaa>;
 			clock-names = "core", "clkh", "cd", "aclk";
+			max-frequency = <200000000>;
 			resets = <&cpg 0xa8>;
 			power-domains = <&cpg>;
 			status = "disabled";
@@ -1305,6 +1823,7 @@ sdhi2: mmc@15c20000 {
 			clocks = <&cpg CPG_MOD 0xab>, <&cpg CPG_MOD 0xad>,
 				 <&cpg CPG_MOD 0xac>, <&cpg CPG_MOD 0xae>;
 			clock-names = "core", "clkh", "cd", "aclk";
+			max-frequency = <200000000>;
 			resets = <&cpg 0xa9>;
 			power-domains = <&cpg>;
 			status = "disabled";
diff --git a/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts b/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts
index 6372f582a7c41..4eed095b683b3 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts
+++ b/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts
@@ -32,6 +32,37 @@
 #include "rzg3e-smarc-som.dtsi"
 #include "renesas-smarc2.dtsi"
 
+/*
+ * SSI-DA7212
+ *
+ * These commands are required when Playback/Capture
+ *
+ *	amixer -q cset name='Aux Switch' on
+ *	amixer -q cset name='Mixin Left Aux Left Switch' on
+ *	amixer -q cset name='Mixin Right Aux Right Switch' on
+ *	amixer -q cset name='ADC Switch' on
+ *	amixer -q cset name='Mixout Right Mixin Right Switch' off
+ *	amixer -q cset name='Mixout Left Mixin Left Switch' off
+ *	amixer -q cset name='Headphone Volume' 70%
+ *	amixer -q cset name='Headphone Switch' on
+ *	amixer -q cset name='Mixout Left DAC Left Switch' on
+ *	amixer -q cset name='Mixout Right DAC Right Switch' on
+ *	amixer -q cset name='DAC Left Source MUX' 'DAI Input Left'
+ *	amixer -q cset name='DAC Right Source MUX' 'DAI Input Right'
+ *	amixer -q sset 'Mic 1 Amp Source MUX' 'MIC_P'
+ *	amixer -q sset 'Mic 2 Amp Source MUX' 'MIC_P'
+ *	amixer -q sset 'Mixin Left Mic 1' on
+ *	amixer -q sset 'Mixin Right Mic 2' on
+ *	amixer -q sset 'Mic 1' 90% on
+ *	amixer -q sset 'Mic 2' 90% on
+ *	amixer -q sset 'Lineout' 80% on
+ *	amixer -q set "Headphone" 100% on
+ *
+ * When Capture chained with DVC, use this command to amplify sound
+ *	amixer set 'DVC In',0 80%
+ * For playback, use: amixer set 'DVC Out',0 80%
+ */
+
 / {
 	model = "Renesas SMARC EVK version 2 based on r9a09g047e57";
 	compatible = "renesas,smarc2-evk", "renesas,rzg3e-smarcm",
@@ -55,6 +86,22 @@ vqmmc_sd1_pvdd: regulator-vqmmc-sd1-pvdd {
 		gpios-states = <0>;
 		states = <3300000 0>, <1800000 1>;
 	};
+
+	sound_card: sound {
+		compatible = "audio-graph-card";
+
+		label = "snd-rzg3e";
+
+		dais = <&rsnd_port0>;	/* DA7212 */
+	};
+};
+
+&audio_clkb {
+	clock-frequency = <11289600>;
+};
+
+&audio_clkc {
+	clock-frequency = <12288000>;
 };
 
 &canfd {
@@ -99,6 +146,35 @@ &i2c0 {
 	pinctrl-names = "default";
 };
 
+&i2c1 {
+	da7212: codec@1a {
+		compatible = "dlg,da7212";
+		#sound-dai-cells = <0>;
+		reg = <0x1a>;
+
+		clocks = <&versa3 1>;
+		clock-names = "mclk";
+
+		dlg,micbias1-lvl = <2500>;
+		dlg,micbias2-lvl = <2500>;
+		dlg,dmic-data-sel = "lrise_rfall";
+		dlg,dmic-samplephase = "between_clkedge";
+		dlg,dmic-clkrate = <3000000>;
+
+		VDDA-supply = <&reg_1p8v>;
+		VDDSP-supply = <&reg_3p3v>;
+		VDDMIC-supply = <&reg_3p3v>;
+		VDDIO-supply = <&reg_1p8v>;
+
+		port {
+			da7212_endpoint: endpoint {
+				remote-endpoint = <&rsnd_endpoint0>;
+				mclk-fs = <256>;
+			};
+		};
+	};
+};
+
 &keys {
 	pinctrl-0 = <&nmi_pins>;
 	pinctrl-names = "default";
@@ -237,7 +313,7 @@ &rsci2 {
 };
 #endif
 
-#if (!SW_LCD_EN) && (SW_SER0_PMOD)
+#if !SW_LCD_EN
 &rsci4 {
 	pinctrl-0 = <&rsci4_pins>;
 	pinctrl-names = "default";
@@ -280,6 +356,42 @@ &sdhi1 {
 	vqmmc-supply = <&vqmmc_sd1_pvdd>;
 };
 
+&snd_rzg3e {
+	pinctrl-0 = <&sound_clk_pins &sound_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+
+	/* audio_clkout */
+	#clock-cells = <0>;
+	clock-frequency = <11289600>;
+
+	/* Multi DAI */
+	#sound-dai-cells = <1>;
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		rsnd_port0: port@0 {
+			reg = <0>;
+			rsnd_endpoint0: endpoint {
+				remote-endpoint = <&da7212_endpoint>;
+
+				dai-format = "i2s";
+				bitclock-master = <&rsnd_endpoint0>;
+				frame-master = <&rsnd_endpoint0>;
+
+				playback = <&ssi3>, <&src1>, <&dvc1>;
+				capture = <&ssi4>, <&src0>, <&dvc0>;
+			};
+		};
+	};
+};
+
+&ssi4 {
+	shared-pin;
+};
+
 &usb3_phy {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/renesas/r9a09g056.dtsi b/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
index d6c8c39df2a4d..76fa34ff3d07e 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
@@ -39,6 +39,20 @@ audio_extal_clk: audio-clk {
 		clock-frequency = <0>;
 	};
 
+	audio_clkb: audio-b-clk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board */
+		clock-frequency = <0>;
+	};
+
+	audio_clkc: audio-c-clk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board */
+		clock-frequency = <0>;
+	};
+
 	/*
 	 * The default cluster table is based on the assumption that the PLLCA55 clock
 	 * frequency is set to 1.7GHz. The PLLCA55 clock frequency can be set to
@@ -294,8 +308,10 @@ pinctrl: pinctrl@10410000 {
 		cpg: clock-controller@10420000 {
 			compatible = "renesas,r9a09g056-cpg";
 			reg = <0 0x10420000 0 0x10000>;
-			clocks = <&audio_extal_clk>, <&rtxin_clk>, <&qextal_clk>;
-			clock-names = "audio_extal", "rtxin", "qextal";
+			clocks = <&audio_extal_clk>, <&rtxin_clk>, <&qextal_clk>,
+				 <&audio_clkb>, <&audio_clkc>;
+			clock-names = "audio_extal", "rtxin", "qextal",
+				      "audio_clkb", "audio_clkc";
 			#clock-cells = <2>;
 			#reset-cells = <1>;
 			#power-domain-cells = <0>;
@@ -1049,7 +1065,7 @@ pcie: pcie@13400000 {
 			reg = <0 0x13400000 0 0x10000>;
 			ranges = <0x02000000 0 0x30000000 0 0x30000000 0 0x8000000>,
 				 <0x43000000 4 0x40000000 4 0x40000000 6 0x00000000>;
-			dma-ranges = <0x42000000 0 0x40000000 0 0x40000000 2 0x00000000>;
+			dma-ranges = <0x43000000 0 0x40000000 0 0x40000000 2 0x00000000>;
 			bus-range = <0x0 0xff>;
 			interrupts = <GIC_SPI 800 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 801 IRQ_TYPE_LEVEL_HIGH>,
diff --git a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
index 1e94366bb7eee..639693d464a77 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
@@ -21,6 +21,20 @@ audio_extal_clk: audio-clk {
 		clock-frequency = <0>;
 	};
 
+	audio_clkb: audio-b-clk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board */
+		clock-frequency = <0>;
+	};
+
+	audio_clkc: audio-c-clk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board */
+		clock-frequency = <0>;
+	};
+
 	/*
 	 * The default cluster table is based on the assumption that the PLLCA55 clock
 	 * frequency is set to 1.7GHz. The PLLCA55 clock frequency can be set to
@@ -276,8 +290,10 @@ pinctrl: pinctrl@10410000 {
 		cpg: clock-controller@10420000 {
 			compatible = "renesas,r9a09g057-cpg";
 			reg = <0 0x10420000 0 0x10000>;
-			clocks = <&audio_extal_clk>, <&rtxin_clk>, <&qextal_clk>;
-			clock-names = "audio_extal", "rtxin", "qextal";
+			clocks = <&audio_extal_clk>, <&rtxin_clk>, <&qextal_clk>,
+				 <&audio_clkb>, <&audio_clkc>;
+			clock-names = "audio_extal", "rtxin", "qextal",
+				      "audio_clkb", "audio_clkc";
 			#clock-cells = <2>;
 			#reset-cells = <1>;
 			#power-domain-cells = <0>;
@@ -1025,6 +1041,146 @@ rsci9: serial@12803000 {
 			status = "disabled";
 		};
 
+		pcie0: pcie@13400000 {
+			compatible = "renesas,r9a09g057-pcie";
+			reg = <0 0x13400000 0 0x10000>;
+			ranges = <0x02000000 0x0 0x30000000 0x0 0x30000000 0x0 0x08000000>,
+				 <0x43000000 0x4 0x40000000 0x4 0x40000000 0x6 0x00000000>;
+			dma-ranges = <0x43000000 0x0 0x40000000 0x0 0x40000000 0x4 0x00000000>;
+			bus-range = <0x0 0xff>;
+			interrupts = <GIC_SPI 800 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 801 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 802 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 803 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 806 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 792 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 793 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 794 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 795 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 796 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 797 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 799 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 804 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 805 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 807 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 791 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 798 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 808 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 809 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 810 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 811 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 812 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 813 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "serr", "serr_cor", "serr_nonfatal",
+					  "serr_fatal", "axi_err", "inta",
+					  "intb", "intc", "intd", "msi",
+					  "link_bandwidth", "pm_pme", "dma",
+					  "pcie_evt", "msg", "all",
+					  "link_equalization_request",
+					  "turn_off_event", "pmu_poweroff",
+					  "d3_event_f0", "d3_event_f1",
+					  "cfg_pmcsr_writeclear_f0",
+					  "cfg_pmcsr_writeclear_f1";
+			#interrupt-cells = <1>;
+			interrupt-controller;
+			interrupt-map-mask = <0 0 0 7>;
+			interrupt-map = <0 0 0 1 &pcie0 0 0 0 0>, /* INTA */
+					<0 0 0 2 &pcie0 0 0 0 1>, /* INTB */
+					<0 0 0 3 &pcie0 0 0 0 2>, /* INTC */
+					<0 0 0 4 &pcie0 0 0 0 3>; /* INTD */
+			clocks = <&cpg CPG_MOD 0xc4>, <&cpg CPG_MOD 0xc5>;
+			clock-names = "aclk", "pmu";
+			resets = <&cpg 0xb2>;
+			reset-names = "aresetn";
+			power-domains = <&cpg>;
+			device_type = "pci";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			num-lanes = <2>;
+			renesas,sysc = <&sys 0>;
+			status = "disabled";
+
+			pcie_port0: pcie@0,0 {
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				ranges;
+				device_type = "pci";
+				vendor-id = <0x1912>;
+				device-id = <0x003b>;
+				#address-cells = <3>;
+				#size-cells = <2>;
+			};
+		};
+
+		pcie1: pcie@13410000 {
+			compatible = "renesas,r9a09g057-pcie";
+			reg = <0 0x13410000 0 0x10000>;
+			ranges = <0x02000000 0x0 0x38000000 0x0 0x38000000 0x0 0x08000000>,
+				 <0x43000000 0xa 0x40000000 0xa 0x40000000 0x5 0xc0000000>;
+			dma-ranges = <0x43000000 0x0 0x40000000 0x0 0x40000000 0x4 0x00000000>;
+			bus-range = <0x0 0xff>;
+			interrupts = <GIC_SPI 823 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 824 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 825 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 826 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 829 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 815 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 816 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 817 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 818 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 819 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 820 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 822 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 827 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 828 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 830 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 814 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 821 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 831 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 832 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 833 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 834 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 835 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 836 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "serr", "serr_cor", "serr_nonfatal",
+					  "serr_fatal", "axi_err", "inta",
+					  "intb", "intc", "intd", "msi",
+					  "link_bandwidth", "pm_pme", "dma",
+					  "pcie_evt", "msg", "all",
+					  "link_equalization_request",
+					  "turn_off_event", "pmu_poweroff",
+					  "d3_event_f0", "d3_event_f1",
+					  "cfg_pmcsr_writeclear_f0",
+					  "cfg_pmcsr_writeclear_f1";
+			#interrupt-cells = <1>;
+			interrupt-controller;
+			interrupt-map-mask = <0 0 0 7>;
+			interrupt-map = <0 0 0 1 &pcie1 0 0 0 0>, /* INTA */
+					<0 0 0 2 &pcie1 0 0 0 1>, /* INTB */
+					<0 0 0 3 &pcie1 0 0 0 2>, /* INTC */
+					<0 0 0 4 &pcie1 0 0 0 3>; /* INTD */
+			clocks = <&cpg CPG_MOD 0xc4>, <&cpg CPG_MOD 0xc5>;
+			clock-names = "aclk", "pmu";
+			resets = <&cpg 0xb2>;
+			reset-names = "aresetn";
+			power-domains = <&cpg>;
+			device_type = "pci";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			num-lanes = <2>;
+			renesas,sysc = <&sys 1>;
+			status = "disabled";
+
+			pcie_port1: pcie@0,0 {
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				ranges;
+				device_type = "pci";
+				vendor-id = <0x1912>;
+				device-id = <0x003b>;
+				#address-cells = <3>;
+				#size-cells = <2>;
+			};
+		};
+
 		i2c0: i2c@14400400 {
 			compatible = "renesas,riic-r9a09g057";
 			reg = <0 0x14400400 0 0x400>;
diff --git a/arch/arm64/boot/dts/renesas/r9a09g057h44-rzv2h-evk.dts b/arch/arm64/boot/dts/renesas/r9a09g057h44-rzv2h-evk.dts
index 3c1ddacc09441..637fc92dcc263 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g057h44-rzv2h-evk.dts
+++ b/arch/arm64/boot/dts/renesas/r9a09g057h44-rzv2h-evk.dts
@@ -384,6 +384,18 @@ &ostm7 {
 	status = "okay";
 };
 
+&pcie0 {
+	pinctrl-0 = <&pcie0_pins>;
+	pinctrl-names = "default";
+	num-lanes = <4>;
+	status = "okay";
+};
+
+&pcie_port0 {
+	clocks = <&versa3 5>;
+	clock-names = "ref";
+};
+
 &pinctrl {
 	eth0_pins: eth0 {
 		pins = "ET0_TXC_TXCLK";
@@ -430,6 +442,12 @@ i2c8_pins: i2c8 {
 			 <RZV2H_PORT_PINMUX(0, 7, 1)>; /* I2C8_SCL */
 	};
 
+	pcie0_pins: pcie0 {
+		pins = "PCIE0_RSTOUTB";
+		slew-rate = <0>;
+		renesas,output-impedance = <2>;
+	};
+
 	scif_pins: scif {
 		pins = "SCIF_TXD", "SCIF_RXD";
 		renesas,output-impedance = <1>;
diff --git a/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts b/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts
index e9ed2de128f6f..572be8aa6d7ef 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts
+++ b/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts
@@ -256,60 +256,112 @@ can0_pins: can0-pins {
 	/*
 	 * GMAC1 Pin Configuration:
 	 *
-	 * SW2[8] ON - use pins P33_2-P33_7, P34_0-P34_5, P34_7 and
-	 * P35_0-P35_2 for Ethernet port 3
+	 * SW2[8] ON - use pins P33_2-P33_7 and P34_0-P34_5 for Ethernet port 3
 	 */
-	gmac1_pins: gmac1-pins {
-		pinmux = <RZT2H_PORT_PINMUX(33, 2, 0xf)>, /* ETH3_TXCLK */
-			 <RZT2H_PORT_PINMUX(33, 3, 0xf)>, /* ETH3_TXD0 */
-			 <RZT2H_PORT_PINMUX(33, 4, 0xf)>, /* ETH3_TXD1 */
-			 <RZT2H_PORT_PINMUX(33, 5, 0xf)>, /* ETH3_TXD2 */
-			 <RZT2H_PORT_PINMUX(33, 6, 0xf)>, /* ETH3_TXD3 */
-			 <RZT2H_PORT_PINMUX(33, 7, 0xf)>, /* ETH3_TXEN */
-			 <RZT2H_PORT_PINMUX(34, 0, 0xf)>, /* ETH3_RXCLK */
-			 <RZT2H_PORT_PINMUX(34, 1, 0xf)>, /* ETH3_RXD0 */
-			 <RZT2H_PORT_PINMUX(34, 2, 0xf)>, /* ETH3_RXD1 */
-			 <RZT2H_PORT_PINMUX(34, 3, 0xf)>, /* ETH3_RXD2 */
-			 <RZT2H_PORT_PINMUX(34, 4, 0xf)>, /* ETH3_RXD3 */
-			 <RZT2H_PORT_PINMUX(34, 5, 0xf)>, /* ETH3_RXDV */
-			 <RZT2H_PORT_PINMUX(34, 7, 0xf)>, /* ETH3_TXER */
-			 <RZT2H_PORT_PINMUX(35, 0, 0xf)>, /* ETH3_RXER */
-			 <RZT2H_PORT_PINMUX(35, 1, 0xf)>, /* ETH3_CRS */
-			 <RZT2H_PORT_PINMUX(35, 2, 0xf)>, /* ETH3_COL */
-			 <RZT2H_PORT_PINMUX(26, 1, 0x10)>, /* GMAC1_MDC */
-			 <RZT2H_PORT_PINMUX(26, 2, 0x10)>, /* GMAC1_MDIO */
-			 <RZT2H_PORT_PINMUX(34, 6, 0x2)>, /* ETH3_REFCLK */
-			 <RZT2H_PORT_PINMUX(27, 2, 0x0)>; /* IRQ3 */
+	gmac1_pins: gmac1-group {
+		irq-pins {
+			pinmux = <RZT2H_PORT_PINMUX(27, 2, 0x0)>; /* IRQ3 */
+			drive-strength-microamp = <5000>;
+			slew-rate = <0>;
+			input-schmitt-disable;
+		};
+
+		md-pins {
+			pinmux = <RZT2H_PORT_PINMUX(26, 1, 0x10)>, /* GMAC1_MDC */
+				 <RZT2H_PORT_PINMUX(26, 2, 0x10)>; /* GMAC1_MDIO */
+			drive-strength-microamp = <5000>;
+			slew-rate = <0>;
+			input-schmitt-disable;
+		};
+
+		refclk-pins {
+			pinmux = <RZT2H_PORT_PINMUX(34, 6, 0x2)>; /* ETH3_REFCLK */
+			drive-strength-microamp = <5000>;
+			slew-rate = <1>;
+		};
+
+		rx-pins {
+			pinmux = <RZT2H_PORT_PINMUX(34, 0, 0xf)>, /* ETH3_RXCLK */
+				 <RZT2H_PORT_PINMUX(34, 1, 0xf)>, /* ETH3_RXD0 */
+				 <RZT2H_PORT_PINMUX(34, 2, 0xf)>, /* ETH3_RXD1 */
+				 <RZT2H_PORT_PINMUX(34, 3, 0xf)>, /* ETH3_RXD2 */
+				 <RZT2H_PORT_PINMUX(34, 4, 0xf)>, /* ETH3_RXD3 */
+				 <RZT2H_PORT_PINMUX(34, 5, 0xf)>; /* ETH3_RXDV */
+			input-schmitt-disable;
+		};
+
+		txclk-pins {
+			pinmux = <RZT2H_PORT_PINMUX(33, 2, 0xf)>; /* ETH3_TXCLK */
+			drive-strength-microamp = <11800>;
+			slew-rate = <1>;
+			input-schmitt-disable;
+		};
+
+		txd-en-pins {
+			pinmux = <RZT2H_PORT_PINMUX(33, 3, 0xf)>, /* ETH3_TXD0 */
+				 <RZT2H_PORT_PINMUX(33, 4, 0xf)>, /* ETH3_TXD1 */
+				 <RZT2H_PORT_PINMUX(33, 5, 0xf)>, /* ETH3_TXD2 */
+				 <RZT2H_PORT_PINMUX(33, 6, 0xf)>, /* ETH3_TXD3 */
+				 <RZT2H_PORT_PINMUX(33, 7, 0xf)>; /* ETH3_TXEN */
+			drive-strength-microamp = <11800>;
+			slew-rate = <1>;
+		};
 	};
 
 	/*
 	 * GMAC2 Pin Configuration:
 	 *
 	 * SW2[6] OFF - connect MDC/MDIO of Ethernet port 2 to GMAC2
-	 * SW2[7] ON - use pins P29_1-P29_7, P30_0-P30_4, and P31_2-P31_5
-	 * for Ethernet port 2
+	 * SW2[7] ON - use pins P29_1-P29_7 and P30_0-P30_4 for Ethernet port 2
 	 */
-	gmac2_pins: gmac2-pins {
-		pinmux = <RZT2H_PORT_PINMUX(29, 1, 0xf)>, /* ETH2_TXCLK */
-			 <RZT2H_PORT_PINMUX(29, 2, 0xf)>, /* ETH2_TXD0 */
-			 <RZT2H_PORT_PINMUX(29, 3, 0xf)>, /* ETH2_TXD1 */
-			 <RZT2H_PORT_PINMUX(29, 4, 0xf)>, /* ETH2_TXD2 */
-			 <RZT2H_PORT_PINMUX(29, 5, 0xf)>, /* ETH2_TXD3 */
-			 <RZT2H_PORT_PINMUX(29, 6, 0xf)>, /* ETH2_TXEN */
-			 <RZT2H_PORT_PINMUX(29, 7, 0xf)>, /* ETH2_RXCLK */
-			 <RZT2H_PORT_PINMUX(30, 0, 0xf)>, /* ETH2_RXD0 */
-			 <RZT2H_PORT_PINMUX(30, 1, 0xf)>, /* ETH2_RXD1 */
-			 <RZT2H_PORT_PINMUX(30, 2, 0xf)>, /* ETH2_RXD2 */
-			 <RZT2H_PORT_PINMUX(30, 3, 0xf)>, /* ETH2_RXD3 */
-			 <RZT2H_PORT_PINMUX(30, 4, 0xf)>, /* ETH2_RXDV */
-			 <RZT2H_PORT_PINMUX(31, 2, 0xf)>, /* ETH2_TXER */
-			 <RZT2H_PORT_PINMUX(31, 3, 0xf)>, /* ETH2_RXER */
-			 <RZT2H_PORT_PINMUX(31, 4, 0xf)>, /* ETH2_CRS */
-			 <RZT2H_PORT_PINMUX(31, 5, 0xf)>, /* ETH2_COL */
-			 <RZT2H_PORT_PINMUX(30, 5, 0x10)>, /* GMAC2_MDC */
-			 <RZT2H_PORT_PINMUX(30, 6, 0x10)>, /* GMAC2_MDIO */
-			 <RZT2H_PORT_PINMUX(31, 0, 0x2)>, /* ETH2_REFCLK */
-			 <RZT2H_PORT_PINMUX(31, 1, 0x0)>; /* IRQ13 */
+	gmac2_pins: gmac2-group {
+		irq-pins {
+			pinmux = <RZT2H_PORT_PINMUX(31, 1, 0x0)>; /* IRQ13 */
+			drive-strength-microamp = <5000>;
+			slew-rate = <0>;
+			input-schmitt-disable;
+		};
+
+		md-pins {
+			pinmux = <RZT2H_PORT_PINMUX(30, 5, 0x10)>, /* GMAC2_MDC */
+				 <RZT2H_PORT_PINMUX(30, 6, 0x10)>; /* GMAC2_MDIO */
+			drive-strength-microamp = <5000>;
+			slew-rate = <0>;
+			input-schmitt-disable;
+		};
+
+		refclk-pins {
+			pinmux = <RZT2H_PORT_PINMUX(31, 0, 0x2)>; /* ETH2_REFCLK */
+			drive-strength-microamp = <5000>;
+			slew-rate = <1>;
+		};
+
+		rx-pins {
+			pinmux = <RZT2H_PORT_PINMUX(29, 7, 0xf)>, /* ETH2_RXCLK */
+				 <RZT2H_PORT_PINMUX(30, 0, 0xf)>, /* ETH2_RXD0 */
+				 <RZT2H_PORT_PINMUX(30, 1, 0xf)>, /* ETH2_RXD1 */
+				 <RZT2H_PORT_PINMUX(30, 2, 0xf)>, /* ETH2_RXD2 */
+				 <RZT2H_PORT_PINMUX(30, 3, 0xf)>, /* ETH2_RXD3 */
+				 <RZT2H_PORT_PINMUX(30, 4, 0xf)>; /* ETH2_RXDV */
+			input-schmitt-disable;
+		};
+
+		txclk-pins {
+			pinmux = <RZT2H_PORT_PINMUX(29, 1, 0xf)>; /* ETH2_TXCLK */
+			drive-strength-microamp = <11800>;
+			slew-rate = <1>;
+			input-schmitt-disable;
+		};
+
+		txd-en-pins {
+			pinmux = <RZT2H_PORT_PINMUX(29, 2, 0xf)>, /* ETH2_TXD0 */
+				 <RZT2H_PORT_PINMUX(29, 3, 0xf)>, /* ETH2_TXD1 */
+				 <RZT2H_PORT_PINMUX(29, 4, 0xf)>, /* ETH2_TXD2 */
+				 <RZT2H_PORT_PINMUX(29, 5, 0xf)>, /* ETH2_TXD3 */
+				 <RZT2H_PORT_PINMUX(29, 6, 0xf)>; /* ETH2_TXEN */
+			drive-strength-microamp = <11800>;
+			slew-rate = <1>;
+		};
 	};
 
 	/*
diff --git a/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts b/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts
index ef6cc7497c2c4..4e57d4fe195ca 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts
+++ b/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts
@@ -339,63 +339,113 @@ can1_pins: can1-pins {
 	/*
 	 * GMAC1 Pin Configuration:
 	 *
-	 * DSW5[8] ON - use pins P00_0-P00_2, P33_2-P33_7, P34_0-P34_6
-	 * for Ethernet port 3
-	 * DSW12[1] OFF; DSW12[2] ON - use pin P00_3 for Ethernet port 3
+	 * DSW5[8] ON - use pins P33_2-P33_7 and P34_0-P34_6 for Ethernet port 3
 	 */
-	gmac1_pins: gmac1-pins {
-		pinmux = <RZT2H_PORT_PINMUX(33, 2, 0xf)>, /* ETH3_TXCLK */
-			 <RZT2H_PORT_PINMUX(33, 3, 0xf)>, /* ETH3_TXD0 */
-			 <RZT2H_PORT_PINMUX(33, 4, 0xf)>, /* ETH3_TXD0 */
-			 <RZT2H_PORT_PINMUX(33, 5, 0xf)>, /* ETH3_TXD2 */
-			 <RZT2H_PORT_PINMUX(33, 6, 0xf)>, /* ETH3_TXD3 */
-			 <RZT2H_PORT_PINMUX(33, 7, 0xf)>, /* ETH3_TXEN */
-			 <RZT2H_PORT_PINMUX(34, 0, 0xf)>, /* ETH3_RXCLK */
-			 <RZT2H_PORT_PINMUX(34, 1, 0xf)>, /* ETH3_RXD0 */
-			 <RZT2H_PORT_PINMUX(34, 2, 0xf)>, /* ETH3_RXD1 */
-			 <RZT2H_PORT_PINMUX(34, 3, 0xf)>, /* ETH3_RXD2 */
-			 <RZT2H_PORT_PINMUX(34, 4, 0xf)>, /* ETH3_RXD3 */
-			 <RZT2H_PORT_PINMUX(34, 5, 0xf)>, /* ETH3_RXDV */
-			 <RZT2H_PORT_PINMUX(0, 0, 0xf)>, /* ETH3_TXER */
-			 <RZT2H_PORT_PINMUX(0, 1, 0xf)>, /* ETH3_RXER */
-			 <RZT2H_PORT_PINMUX(0, 2, 0xf)>, /* ETH3_CRS */
-			 <RZT2H_PORT_PINMUX(0, 3, 0xf)>, /* ETH3_COL */
-			 <RZT2H_PORT_PINMUX(26, 1, 0x10)>, /* GMAC1_MDC */
-			 <RZT2H_PORT_PINMUX(26, 2, 0x10)>, /* GMAC1_MDIO */
-			 <RZT2H_PORT_PINMUX(34, 6, 0x2)>, /* ETH3_REFCLK */
-			 <RZT2H_PORT_PINMUX(17, 3, 0x0)>; /* IRQ15 */
+	gmac1_pins: gmac1-group {
+		irq-pins {
+			pinmux = <RZT2H_PORT_PINMUX(17, 3, 0x0)>; /* IRQ15 */
+			drive-strength-microamp = <5000>;
+			slew-rate = <0>;
+			input-schmitt-disable;
+		};
+
+		md-pins {
+			pinmux = <RZT2H_PORT_PINMUX(26, 1, 0x10)>, /* GMAC1_MDC */
+				 <RZT2H_PORT_PINMUX(26, 2, 0x10)>; /* GMAC1_MDIO */
+			drive-strength-microamp = <5000>;
+			slew-rate = <0>;
+			input-schmitt-disable;
+		};
+
+		refclk-pins {
+			pinmux = <RZT2H_PORT_PINMUX(34, 6, 0x2)>; /* ETH3_REFCLK */
+			drive-strength-microamp = <5000>;
+			slew-rate = <1>;
+		};
+
+		rx-pins {
+			pinmux = <RZT2H_PORT_PINMUX(34, 0, 0xf)>, /* ETH3_RXCLK */
+				 <RZT2H_PORT_PINMUX(34, 1, 0xf)>, /* ETH3_RXD0 */
+				 <RZT2H_PORT_PINMUX(34, 2, 0xf)>, /* ETH3_RXD1 */
+				 <RZT2H_PORT_PINMUX(34, 3, 0xf)>, /* ETH3_RXD2 */
+				 <RZT2H_PORT_PINMUX(34, 4, 0xf)>, /* ETH3_RXD3 */
+				 <RZT2H_PORT_PINMUX(34, 5, 0xf)>; /* ETH3_RXDV */
+			input-schmitt-disable;
+		};
+
+		txclk-pins {
+			pinmux = <RZT2H_PORT_PINMUX(33, 2, 0xf)>; /* ETH3_TXCLK */
+			drive-strength-microamp = <11800>;
+			slew-rate = <1>;
+			input-schmitt-disable;
+		};
+
+		txd-en-pins {
+			pinmux = <RZT2H_PORT_PINMUX(33, 3, 0xf)>, /* ETH3_TXD0 */
+				 <RZT2H_PORT_PINMUX(33, 4, 0xf)>, /* ETH3_TXD1 */
+				 <RZT2H_PORT_PINMUX(33, 5, 0xf)>, /* ETH3_TXD2 */
+				 <RZT2H_PORT_PINMUX(33, 6, 0xf)>, /* ETH3_TXD3 */
+				 <RZT2H_PORT_PINMUX(33, 7, 0xf)>; /* ETH3_TXEN */
+			drive-strength-microamp = <11800>;
+			slew-rate = <1>;
+		};
 	};
 
 	/*
 	 * GMAC2 Pin Configuration:
 	 *
 	 * DSW5[6] OFF - connect MDC/MDIO of Ethernet port 2 to GMAC2
-	 * DSW5[7] ON - use pins P29_1-P29_7, P30_0-P30_4, P30_7,
-	 * P31_2, P31_4 and P31_5 are used for Ethernet port 2
+	 * DSW5[7] ON - use pins P29_1-P29_7 and P30_0-P30_4 for Ethernet port 2
 	 * DSW13[7] OFF; DSW13[8] ON - use pin P13_7 for IRQ14
 	 */
-	gmac2_pins: gmac2-pins {
-		pinmux = <RZT2H_PORT_PINMUX(29, 1, 0xf)>, /* ETH2_TXCLK */
-			 <RZT2H_PORT_PINMUX(29, 2, 0xf)>, /* ETH2_TXD0 */
-			 <RZT2H_PORT_PINMUX(29, 3, 0xf)>, /* ETH2_TXD1 */
-			 <RZT2H_PORT_PINMUX(29, 4, 0xf)>, /* ETH2_TXD2 */
-			 <RZT2H_PORT_PINMUX(29, 5, 0xf)>, /* ETH2_TXD3 */
-			 <RZT2H_PORT_PINMUX(29, 6, 0xf)>, /* ETH2_TXEN */
-			 <RZT2H_PORT_PINMUX(29, 7, 0xf)>, /* ETH2_RXCLK */
-			 <RZT2H_PORT_PINMUX(30, 0, 0xf)>, /* ETH2_RXD0 */
-			 <RZT2H_PORT_PINMUX(30, 1, 0xf)>, /* ETH2_RXD1 */
-			 <RZT2H_PORT_PINMUX(30, 2, 0xf)>, /* ETH2_RXD2 */
-			 <RZT2H_PORT_PINMUX(30, 3, 0xf)>, /* ETH2_RXD3 */
-			 <RZT2H_PORT_PINMUX(30, 4, 0xf)>, /* ETH2_RXDV */
-			 <RZT2H_PORT_PINMUX(31, 2, 0xf)>, /* ETH2_TXER */
-			 <RZT2H_PORT_PINMUX(31, 1, 0xf)>, /* ETH2_RXER */
-			 <RZT2H_PORT_PINMUX(31, 4, 0xf)>, /* ETH2_CRS */
-			 <RZT2H_PORT_PINMUX(31, 5, 0xf)>, /* ETH2_COL */
-			 <RZT2H_PORT_PINMUX(30, 5, 0x10)>, /* GMAC2_MDC */
-			 <RZT2H_PORT_PINMUX(30, 6, 0x10)>, /* GMAC2_MDIO */
-			 <RZT2H_PORT_PINMUX(31, 0, 0x2)>, /* ETH2_REFCLK */
-			 <RZT2H_PORT_PINMUX(13, 7, 0x0)>; /* IRQ14 */
+	gmac2_pins: gmac2-group {
+		irq-pins {
+			pinmux = <RZT2H_PORT_PINMUX(13, 7, 0x0)>; /* IRQ14 */
+			drive-strength-microamp = <5000>;
+			slew-rate = <0>;
+			input-schmitt-disable;
+		};
+
+		md-pins {
+			pinmux = <RZT2H_PORT_PINMUX(30, 5, 0x10)>, /* GMAC2_MDC */
+				 <RZT2H_PORT_PINMUX(30, 6, 0x10)>; /* GMAC2_MDIO */
+			drive-strength-microamp = <5000>;
+			slew-rate = <0>;
+			input-schmitt-disable;
+		};
+
+		refclk-pins {
+			pinmux = <RZT2H_PORT_PINMUX(31, 0, 0x2)>; /* ETH2_REFCLK */
+			drive-strength-microamp = <5000>;
+			slew-rate = <1>;
+		};
 
+		rx-pins {
+			pinmux = <RZT2H_PORT_PINMUX(29, 7, 0xf)>, /* ETH2_RXCLK */
+				 <RZT2H_PORT_PINMUX(30, 0, 0xf)>, /* ETH2_RXD0 */
+				 <RZT2H_PORT_PINMUX(30, 1, 0xf)>, /* ETH2_RXD1 */
+				 <RZT2H_PORT_PINMUX(30, 2, 0xf)>, /* ETH2_RXD2 */
+				 <RZT2H_PORT_PINMUX(30, 3, 0xf)>, /* ETH2_RXD3 */
+				 <RZT2H_PORT_PINMUX(30, 4, 0xf)>; /* ETH2_RXDV */
+			input-schmitt-disable;
+		};
+
+		txclk-pins {
+			pinmux = <RZT2H_PORT_PINMUX(29, 1, 0xf)>; /* ETH2_TXCLK */
+			drive-strength-microamp = <11800>;
+			slew-rate = <1>;
+			input-schmitt-disable;
+		};
+
+		txd-en-pins {
+			pinmux = <RZT2H_PORT_PINMUX(29, 2, 0xf)>, /* ETH2_TXD0 */
+				 <RZT2H_PORT_PINMUX(29, 3, 0xf)>, /* ETH2_TXD1 */
+				 <RZT2H_PORT_PINMUX(29, 4, 0xf)>, /* ETH2_TXD2 */
+				 <RZT2H_PORT_PINMUX(29, 5, 0xf)>, /* ETH2_TXD3 */
+				 <RZT2H_PORT_PINMUX(29, 6, 0xf)>; /* ETH2_TXEN */
+			drive-strength-microamp = <11800>;
+			slew-rate = <1>;
+		};
 	};
 
 	/*
diff --git a/arch/arm64/boot/dts/renesas/rzg3e-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg3e-smarc-som.dtsi
index 2e1d9686df884..0e6dc84e01990 100644
--- a/arch/arm64/boot/dts/renesas/rzg3e-smarc-som.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg3e-smarc-som.dtsi
@@ -32,6 +32,7 @@ / {
 	aliases {
 		ethernet0 = &eth0;
 		ethernet1 = &eth1;
+		i2c1 = &i2c1;
 		i2c2 = &i2c2;
 		mmc0 = &sdhi0;
 		mmc2 = &sdhi2;
@@ -77,6 +78,12 @@ reg_vdd0p8v_others: regulator-vdd0p8v-others {
 		regulator-always-on;
 	};
 
+	x2: x2-clock {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <24000000>;
+	};
+
 	/* 32.768kHz crystal */
 	x3: x3-clock {
 		compatible = "fixed-clock";
@@ -112,6 +119,12 @@ &gpu {
 	mali-supply = <&reg_vdd0p8v_others>;
 };
 
+&i2c1 {
+	pinctrl-0 = <&i2c1_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
 &i2c2 {
 	pinctrl-0 = <&i2c2_pins>;
 	pinctrl-names = "default";
@@ -130,6 +143,20 @@ raa215300: pmic@12 {
 
 		interrupts-extended = <&pinctrl RZG3E_GPIO(S, 1) IRQ_TYPE_EDGE_FALLING>;
 	};
+
+	versa3: clock-generator@68 {
+		compatible = "renesas,5p35023";
+		reg = <0x68>;
+		#clock-cells = <1>;
+		clocks = <&x2>;
+
+		assigned-clocks = <&versa3 0>, <&versa3 1>,
+				  <&versa3 2>, <&versa3 3>,
+				  <&versa3 4>;
+		assigned-clock-rates = <24000000>, <12288000>,
+				       <11289600>, <12288000>,
+				       <25000000>;
+	};
 };
 
 &i3c {
@@ -234,6 +261,11 @@ ctrl {
 		};
 	};
 
+	i2c1_pins: i2c1 {
+		pinmux = <RZG3E_PORT_PINMUX(3, 2, 1)>, /* SCL1 */
+			 <RZG3E_PORT_PINMUX(3, 3, 1)>; /* SDA1 */
+	};
+
 	i2c2_pins: i2c {
 		pinmux = <RZG3E_PORT_PINMUX(3, 4, 1)>, /* SCL2 */
 			 <RZG3E_PORT_PINMUX(3, 5, 1)>; /* SDA2 */
@@ -320,6 +352,18 @@ sd2-pwen {
 		};
 	};
 
+	sound_clk_pins: sound-clk {
+		pinmux = <RZG3E_PORT_PINMUX(4, 2, 8)>, /* AUDIO_CLKB */
+			 <RZG3E_PORT_PINMUX(4, 3, 8)>; /* AUDIO_CLKC */
+	};
+
+	sound_pins: sound {
+		pinmux = <RZG3E_PORT_PINMUX(0, 2, 9)>, /* SSI3_SDATA */
+			 <RZG3E_PORT_PINMUX(0, 3, 9)>, /* SSI3_SCK */
+			 <RZG3E_PORT_PINMUX(0, 4, 9)>, /* SSI3_WS */
+			 <RZG3E_PORT_PINMUX(0, 5, 9)>; /* SSI4_SDATA */
+	};
+
 	xspi_pins: xspi0 {
 		pinmux = <RZG3E_PORT_PINMUX(M, 0, 0)>, /* XSPI0_IO0 */
 			 <RZG3E_PORT_PINMUX(M, 1, 0)>, /* XSPI0_IO1 */
diff --git a/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi b/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi
index e86e6d3aa8a34..ffcc0960903bf 100644
--- a/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi
@@ -29,6 +29,17 @@ chosen {
 		stdout-path = "serial0:115200n8";
 	};
 
+	memory@c8000000 {
+		device_type = "memory";
+		/* first 128MiB is reserved for secure area. */
+		reg = <0x0 0xc8000000 0x0 0x38000000>;
+	};
+
+	memory@240000000 {
+		device_type = "memory";
+		reg = <0x2 0x40000000 0x1 0xc0000000>;
+	};
+
 	reg_1p8v: regulator-1p8v {
 		compatible = "regulator-fixed";
 		regulator-name = "fixed-1.8V";
@@ -257,6 +268,9 @@ &pinctrl {
 	sci0_pins: sci0-pins {
 		pinmux = <RZT2H_PORT_PINMUX(27, 4, 0x14)>,
 			 <RZT2H_PORT_PINMUX(27, 5, 0x14)>;
+		drive-strength-microamp = <9000>;
+		slew-rate = <1>;
+		input-schmitt-disable;
 	};
 
 #if SD0_EMMC
diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile
index 761d82b4f4f2a..3fc54b1a09b69 100644
--- a/arch/arm64/boot/dts/rockchip/Makefile
+++ b/arch/arm64/boot/dts/rockchip/Makefile
@@ -92,6 +92,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-sapphire.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-sapphire-excavator.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399pro-rock-pi-n10.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3528-armsom-sige1.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3528-hinlink-h28k.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3528-nanopi-zero2.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3528-radxa-e20c.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3528-rock-2a.dtb
@@ -144,6 +145,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-easepi-r1.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-evb1-v10.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-fastrhino-r66s.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-fastrhino-r68s.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-graperain-g3568-v2.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-hinlink-h66k.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-hinlink-h68k.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3568-lubancat-2.dtb
@@ -171,6 +173,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-evb1-v10.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-evb1-v10-pcie1.dtbo
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-evb2-v10.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-khadas-edge-2l.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-lckfb-tspi-3m.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-luckfox-omni3576.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-nanopi-m5.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3576-nanopi-r76s.dtb
@@ -213,6 +216,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-tiger-haikou.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-tiger-haikou-video-demo.dtbo
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-toybrick-x0.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-turing-rk1.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-youyeetoo-yy3588.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588s-coolpi-4b.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588s-evb1-v10.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588s-gameforce-ace.dtb
@@ -223,6 +227,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588s-nanopi-r6c.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588s-odroid-m2.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588s-orangepi-5.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588s-orangepi-5b.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588s-orangepi-5-pro.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588s-orangepi-cm5-base.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588s-radxa-cm5-io.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588s-roc-pc.dtb
diff --git a/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi b/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi
index add917af5de78..f14f9eca7d342 100644
--- a/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi
+++ b/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi
@@ -35,7 +35,7 @@ emmc_pwrseq: emmc-pwrseq {
 		compatible = "mmc-pwrseq-emmc";
 		pinctrl-0 = <&emmc_reset>;
 		pinctrl-names = "default";
-		reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_LOW>;
 	};
 
 	gpio-leds {
diff --git a/arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi b/arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi
index 192791993f059..02200de695d31 100644
--- a/arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi
+++ b/arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi
@@ -33,7 +33,7 @@ emmc_pwrseq: emmc-pwrseq {
 		compatible = "mmc-pwrseq-emmc";
 		pinctrl-0 = <&emmc_reset>;
 		pinctrl-names = "default";
-		reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_LOW>;
 	};
 
 	gpio-leds {
diff --git a/arch/arm64/boot/dts/rockchip/px30-ringneck-haikou-video-demo.dtso b/arch/arm64/boot/dts/rockchip/px30-ringneck-haikou-video-demo.dtso
index 760d5139f95df..d0725595ade07 100644
--- a/arch/arm64/boot/dts/rockchip/px30-ringneck-haikou-video-demo.dtso
+++ b/arch/arm64/boot/dts/rockchip/px30-ringneck-haikou-video-demo.dtso
@@ -45,11 +45,11 @@ cam_avdd_2v8: regulator-cam-avdd-2v8 {
 
 	cam_dovdd_1v8: regulator-cam-dovdd-1v8 {
 		compatible = "regulator-fixed";
-	        gpio = <&pca9670 3 GPIO_ACTIVE_LOW>;
-	        regulator-max-microvolt = <1800000>;
-	        regulator-min-microvolt = <1800000>;
-	        regulator-name = "cam-dovdd-1v8";
-	        vin-supply = <&vcc1v8_video>;
+		gpio = <&pca9670 3 GPIO_ACTIVE_LOW>;
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+		regulator-name = "cam-dovdd-1v8";
+		vin-supply = <&vcc1v8_video>;
 	};
 
 	cam_dvdd_1v2: regulator-cam-dvdd-1v2 {
diff --git a/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi b/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi
index 973b4c5880e24..29794216592d8 100644
--- a/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi
+++ b/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi
@@ -26,7 +26,7 @@ emmc_pwrseq: emmc-pwrseq {
 		compatible = "mmc-pwrseq-emmc";
 		pinctrl-0 = <&emmc_reset>;
 		pinctrl-names = "default";
-		reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_LOW>;
 	};
 
 	leds {
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts b/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts
index 8d26bd9b75003..d46cdfe3f7849 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts
@@ -734,12 +734,6 @@ light_int_l: light-int-l {
 		};
 	};
 
-	wifi {
-		wifi_host_wake_l: wifi-host-wake-l {
-			rockchip,pins = <4 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>;
-		};
-	};
-
 	wireless-bluetooth {
 		bt_wake_pin: bt-wake-pin {
 			rockchip,pins = <2 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>;
@@ -766,19 +760,7 @@ &sdio0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>;
 	sd-uhs-sdr104;
-	#address-cells = <1>;
-	#size-cells = <0>;
 	status = "okay";
-
-	brcmf: wifi@1 {
-		compatible = "brcm,bcm4329-fmac";
-		reg = <1>;
-		interrupt-parent = <&gpio4>;
-		interrupts = <RK_PD0 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-names = "host-wake";
-		pinctrl-names = "default";
-		pinctrl-0 = <&wifi_host_wake_l>;
-	};
 };
 
 &pwm0 {
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-rock960.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-rock960.dtsi
index 7b1086682d119..acf44f8b47ed7 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-rock960.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399-rock960.dtsi
@@ -382,7 +382,7 @@ &i2c4 {
 };
 
 &i2s2 {
-        status = "okay";
+	status = "okay";
 };
 
 &io_domains {
diff --git a/arch/arm64/boot/dts/rockchip/rk3528-hinlink-h28k.dts b/arch/arm64/boot/dts/rockchip/rk3528-hinlink-h28k.dts
new file mode 100644
index 0000000000000..0ec50e9156eb1
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3528-hinlink-h28k.dts
@@ -0,0 +1,318 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+
+/dts-v1/;
+
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pwm/pwm.h>
+#include "rk3528.dtsi"
+
+/ {
+	model = "HINLINK H28K";
+	compatible = "hinlink,h28k", "rockchip,rk3528";
+
+	aliases {
+		ethernet0 = &gmac1;
+		mmc0 = &sdhci;
+		mmc1 = &sdmmc;
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:1500000n8";
+	};
+
+	keys {
+		compatible = "adc-keys";
+		io-channels = <&saradc 0>;
+		io-channel-names = "buttons";
+		keyup-threshold-microvolt = <1800000>;
+		poll-interval = <100>;
+
+		button-boot {
+			label = "BOOT";
+			linux,code = <KEY_SETUP>;
+			press-threshold-microvolt = <0>;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&lan_led>, <&wan_led>, <&work_led>;
+
+		led-0 {
+			color = <LED_COLOR_ID_AMBER>;
+			function = LED_FUNCTION_LAN;
+			gpios = <&gpio4 RK_PC1 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "netdev";
+		};
+
+		led-1 {
+			color = <LED_COLOR_ID_BLUE>;
+			function = LED_FUNCTION_WAN;
+			gpios = <&gpio4 RK_PC0 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "netdev";
+		};
+
+		led-2 {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_STATUS;
+			gpios = <&gpio4 RK_PB7 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-on";
+		};
+	};
+
+	vdd_0v9: regulator-0v9-vdd {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_0v9";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		vin-supply = <&vcc5v0_sys>;
+	};
+
+	vcc_ddr: regulator-1v1-vcc-ddr {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_ddr";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1100000>;
+		regulator-max-microvolt = <1100000>;
+		vin-supply = <&vcc5v0_sys>;
+	};
+
+	vcc_1v8: regulator-1v8-vcc {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&vcc_3v3>;
+	};
+
+	vcc_3v3: regulator-3v3-vcc {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc5v0_sys>;
+	};
+
+	vcc3v3_sd: regulator-3v3-vcc-sd {
+		compatible = "regulator-fixed";
+		gpios = <&gpio4 RK_PA1 GPIO_ACTIVE_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&sdmmc_pwren_l>;
+		regulator-name = "vcc3v3_sd";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc_3v3>;
+	};
+
+	vcc5v0_sys: regulator-5v0-vcc-sys {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc5v0_sys";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+	};
+
+	vccio_sd: regulator-vccio-sd {
+		compatible = "regulator-gpio";
+		gpios = <&gpio4 RK_PB6 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&sdmmc_vol_ctrl_h>;
+		regulator-name = "vccio_sd";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		states = <1800000 0x0>, <3300000 0x1>;
+		vin-supply = <&vcc5v0_sys>;
+	};
+
+	vdd_arm: regulator-vdd-arm {
+		compatible = "pwm-regulator";
+		pwms = <&pwm1 0 5000 PWM_POLARITY_INVERTED>;
+		pwm-supply = <&vcc5v0_sys>;
+		regulator-name = "vdd_arm";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <746000>;
+		regulator-max-microvolt = <1201000>;
+		regulator-settling-time-up-us = <250>;
+	};
+
+	vdd_logic: regulator-vdd-logic {
+		compatible = "pwm-regulator";
+		pwms = <&pwm2 0 5000 PWM_POLARITY_INVERTED>;
+		pwm-supply = <&vcc5v0_sys>;
+		regulator-name = "vdd_logic";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <705000>;
+		regulator-max-microvolt = <1006000>;
+		regulator-settling-time-up-us = <250>;
+	};
+};
+
+&combphy {
+	status = "okay";
+};
+
+&cpu0 {
+	cpu-supply = <&vdd_arm>;
+};
+
+&cpu1 {
+	cpu-supply = <&vdd_arm>;
+};
+
+&cpu2 {
+	cpu-supply = <&vdd_arm>;
+};
+
+&cpu3 {
+	cpu-supply = <&vdd_arm>;
+};
+
+&gmac1 {
+	clock_in_out = "output";
+	phy-handle = <&rgmii_phy>;
+	phy-mode = "rgmii-id";
+	phy-supply = <&vcc_3v3>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&rgmii_miim>,
+		    <&rgmii_tx_bus2>,
+		    <&rgmii_rx_bus2>,
+		    <&rgmii_rgmii_clk>,
+		    <&rgmii_rgmii_bus>;
+	status = "okay";
+};
+
+&gpu {
+	mali-supply = <&vdd_logic>;
+	status = "okay";
+};
+
+&mdio1 {
+	rgmii_phy: ethernet-phy@1 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <0x1>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&gmac1_rstn_l>;
+		reset-assert-us = <20000>;
+		reset-deassert-us = <100000>;
+		reset-gpios = <&gpio4 RK_PC2 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&pcie {
+	pinctrl-names = "default";
+	pinctrl-0 = <&rtl8111hs_isolateb_l>;
+	reset-gpios = <&gpio4 RK_PA3 GPIO_ACTIVE_HIGH>;
+	vpcie3v3-supply = <&vcc_3v3>;
+	status = "okay";
+};
+
+&pinctrl {
+	gmac {
+		gmac1_rstn_l: gmac1-rstn-l {
+			rockchip,pins = <4 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	leds {
+		lan_led: lan-led {
+			rockchip,pins = <4 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		wan_led: wan-led {
+			rockchip,pins = <4 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		work_led: work-led {
+			rockchip,pins = <4 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	pcie {
+		rtl8111hs_isolateb_l: rtl8111hs-isolateb-l {
+			rockchip,pins = <4 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	sdmmc {
+		sdmmc_pwren_l: sdmmc-pwren-l {
+			rockchip,pins = <4 RK_PA1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		sdmmc_vol_ctrl_h: sdmmc-vol-ctrl-h {
+			rockchip,pins = <4 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+};
+
+&pwm1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm1m0_pins>;
+	status = "okay";
+};
+
+&pwm2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm2m0_pins>;
+	status = "okay";
+};
+
+&saradc {
+	vref-supply = <&vcc_1v8>;
+	status = "okay";
+};
+
+&sdhci {
+	bus-width = <8>;
+	cap-mmc-highspeed;
+	mmc-hs200-1_8v;
+	non-removable;
+	vmmc-supply = <&vcc_3v3>;
+	vqmmc-supply = <&vcc_1v8>;
+	status = "okay";
+};
+
+&sdmmc {
+	bus-width = <4>;
+	cap-sd-highspeed;
+	disable-wp;
+	sd-uhs-sdr104;
+	vmmc-supply = <&vcc3v3_sd>;
+	vqmmc-supply = <&vccio_sd>;
+	status = "okay";
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0m0_xfer>;
+	status = "okay";
+};
+
+&usb2phy {
+	status = "okay";
+};
+
+&usb2phy_host {
+	phy-supply = <&vcc5v0_sys>;
+	status = "okay";
+};
+
+&usb_host0_ehci {
+	status = "okay";
+};
+
+&usb_host0_ohci {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3568-9tripod-x3568-v4.dts b/arch/arm64/boot/dts/rockchip/rk3568-9tripod-x3568-v4.dts
index 4db00489be40f..4903e32d0ead0 100644
--- a/arch/arm64/boot/dts/rockchip/rk3568-9tripod-x3568-v4.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3568-9tripod-x3568-v4.dts
@@ -205,7 +205,7 @@ vcc5v0_usb: regulator-vcc5v0-usb {
 	vcc5v0_usb_host: regulator-vcc5v0-usb-host {
 		compatible = "regulator-fixed";
 		enable-active-high;
-		gpio = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>;
+		gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&vcc5v0_usb_host_en>;
 		regulator-name = "vcc5v0_usb_host";
@@ -217,7 +217,7 @@ vcc5v0_usb_host: regulator-vcc5v0-usb-host {
 	vcc5v0_usb_otg: regulator-vcc5v0-usb-otg {
 		compatible = "regulator-fixed";
 		enable-active-high;
-		gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>;
+		gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&vcc5v0_usb_otg_en>;
 		regulator-name = "vcc5v0_usb_otg";
@@ -582,9 +582,9 @@ &i2s1_8ch {
 	status = "okay";
 };
 
-/* used for AP6275S Bluetooth Sound */
+/* required rk3568-codec-digital for AP6275S Bluetooth Sound */
 &i2s3_2ch {
-	status = "okay";
+	status = "disabled";
 };
 
 &mdio0 {
@@ -664,6 +664,20 @@ &pdm {
 };
 
 &pinctrl {
+	bluetooth {
+		bt_reg_on_h: bt-reg-on-h {
+			rockchip,pins = <3 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		bt_wake_host_h: bt-wake-host-h {
+			rockchip,pins = <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+
+		host_wake_bt_h: host-wake-bt-h {
+			rockchip,pins = <3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	leds {
 		led_work_en: led_work_en {
 			rockchip,pins = <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>;
@@ -699,6 +713,12 @@ vcc3v3_pcie_en_pin: vcc3v3-pcie-en-pin {
 			rockchip,pins = <3 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
 	};
+
+	wifi-irq {
+		wifi_host_wake_irq: wifi-host-wake-irq {
+			rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+	};
 };
 
 &pmu_io_domains {
@@ -770,7 +790,19 @@ &sdmmc2 {
 	sd-uhs-sdr104;
 	vmmc-supply = <&vcc3v3_sys>;
 	vqmmc-supply = <&vcc_1v8>;
+	#address-cells = <1>;
+	#size-cells = <0>;
 	status = "okay";
+
+	wifi@1 {
+		compatible = "brcm,bcm43752-fmac", "brcm,bcm4329-fmac";
+		reg = <1>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <RK_PD4 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "host-wake";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wifi_host_wake_irq>;
+	};
 };
 
 &spdif {
@@ -798,10 +830,26 @@ &uart4 {
 	status = "okay";
 };
 
-/* used for WiFi/BT AP6275S */
+/* used for AP6275S Bluetooth */
 &uart8 {
-	pinctrl-0 = <&uart8m0_xfer &uart8m0_ctsn>;
+	dma-names = "tx", "rx";
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart8m0_xfer &uart8m0_ctsn &uart8m0_rtsn>;
+	uart-has-rtscts;
 	status = "okay";
+
+	bluetooth {
+		compatible = "brcm,bcm43438-bt";
+		clocks = <&rk809 1>;
+		clock-names = "lpo";
+		device-wakeup-gpios = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>;
+		host-wakeup-gpios = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>;
+		max-speed = <1500000>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&bt_reg_on_h &bt_wake_host_h &host_wake_bt_h>;
+		shutdown-gpios = <&gpio3 RK_PA0 GPIO_ACTIVE_HIGH>;
+		vddio-supply = <&vcc_1v8>;
+	};
 };
 
 &uart9 {
diff --git a/arch/arm64/boot/dts/rockchip/rk3568-graperain-g3568-v2.dts b/arch/arm64/boot/dts/rockchip/rk3568-graperain-g3568-v2.dts
new file mode 100644
index 0000000000000..a4e7af6bba8a3
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3568-graperain-g3568-v2.dts
@@ -0,0 +1,943 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+
+/dts-v1/;
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/rockchip.h>
+#include <dt-bindings/soc/rockchip,vop2.h>
+#include "rk3568.dtsi"
+
+/ {
+	model = "Graperain G3568 v2";
+	compatible = "graperain,g3568-v2", "rockchip,rk3568";
+
+	aliases {
+		ethernet0 = &gmac0;
+		ethernet1 = &gmac1;
+		mmc0 = &sdhci;
+		mmc1 = &sdmmc0;
+		mmc2 = &sdmmc2;
+		rtc0 = &rtc0;
+	};
+
+	chosen {
+		stdout-path = "serial2:1500000n8";
+	};
+
+	adc-keys {
+		compatible = "adc-keys";
+		io-channels = <&saradc 0>;
+		io-channel-names = "buttons";
+		keyup-threshold-microvolt = <1800000>;
+		poll-interval = <100>;
+
+		button-esc {
+			label = "esc";
+			linux,code = <KEY_ESC>;
+			press-threshold-microvolt = <1250000>;
+		};
+
+		button-menu {
+			label = "menu";
+			linux,code = <KEY_MENU>;
+			press-threshold-microvolt = <1600000>;
+		};
+
+		button-vol-down {
+			label = "volume down";
+			linux,code = <KEY_VOLUMEDOWN>;
+			press-threshold-microvolt = <600000>;
+		};
+
+		button-vol-up {
+			label = "volume up";
+			linux,code = <KEY_VOLUMEUP>;
+			press-threshold-microvolt = <100000>;
+		};
+	};
+
+	hdmi-con {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con_in: endpoint {
+				remote-endpoint = <&hdmi_out_con>;
+			};
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led_work: led-0 {
+			color = <LED_COLOR_ID_BLUE>;
+			function = LED_FUNCTION_HEARTBEAT;
+			gpios = <&gpio0 RK_PC0 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+			pinctrl-names = "default";
+			pinctrl-0 = <&led_work_en>;
+		};
+	};
+
+	rk809-sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,name = "Analog RK809";
+		simple-audio-card,mclk-fs = <256>;
+
+		simple-audio-card,cpu {
+			sound-dai = <&i2s1_8ch>;
+		};
+		simple-audio-card,codec {
+			sound-dai = <&rk809>;
+		};
+	};
+
+	pdm_codec: pdm-codec {
+		compatible = "dmic-codec";
+		num-channels = <2>;
+		#sound-dai-cells = <0>;
+	};
+
+	pdm_sound: pdm-sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "microphone";
+
+		simple-audio-card,cpu {
+			sound-dai = <&pdm>;
+		};
+
+		simple-audio-card,codec {
+			sound-dai = <&pdm_codec>;
+		};
+	};
+
+	spdif_dit: spdif-dit {
+		compatible = "linux,spdif-dit";
+		#sound-dai-cells = <0>;
+	};
+
+	spdif_sound: spdif-sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "SPDIF";
+
+		simple-audio-card,cpu {
+			sound-dai = <&spdif>;
+		};
+		simple-audio-card,codec {
+			sound-dai = <&spdif_dit>;
+		};
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		clocks = <&rk809 1>;
+		clock-names = "ext_clock";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wifi_enable>;
+		post-power-on-delay-ms = <100>;
+		power-off-delay-us = <300>;
+		reset-gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_LOW>;
+	};
+
+	dc_12v: regulator-dc-12v {
+		compatible = "regulator-fixed";
+		regulator-name = "dc_12v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+	};
+
+	pcie30_avdd0v9: regulator-pcie30-avdd0v9 {
+		compatible = "regulator-fixed";
+		regulator-name = "pcie30_avdd0v9";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		vin-supply = <&vcc3v3_sys>;
+	};
+
+	pcie30_avdd1v8: regulator-pcie30-avdd1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "pcie30_avdd1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&vcc3v3_sys>;
+	};
+
+	vcc3v3_sys: regulator-vcc3v3-sys {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc3v3_sys";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&dc_12v>;
+	};
+
+	vcc3v3_pcie: regulator-vcc3v3-pcie {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio3 RK_PC3 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&vcc3v3_pcie_en_pin>;
+		regulator-name = "vcc3v3_pcie";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		startup-delay-us = <5000>;
+		vin-supply = <&dc_12v>;
+	};
+
+	vcc5v0_sys: regulator-vcc5v0-sys {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc5v0_sys";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&dc_12v>;
+	};
+
+	vcc5v0_usb: regulator-vcc5v0-usb {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc5v0_usb";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&dc_12v>;
+	};
+
+	vcc5v0_usb_host: regulator-vcc5v0-usb-host {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&vcc5v0_usb_host_en>;
+		regulator-name = "vcc5v0_usb_host";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&vcc5v0_usb>;
+	};
+
+	vcc5v0_usb_otg: regulator-vcc5v0-usb-otg {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&vcc5v0_usb_otg_en>;
+		regulator-name = "vcc5v0_usb_otg";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&vcc5v0_usb>;
+	};
+};
+
+&can1 {
+	assigned-clocks = <&cru CLK_CAN1>;
+	assigned-clock-rates = <150000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&can1m1_pins>;
+	status = "okay";
+};
+
+/* used for usb_host0_xhci */
+&combphy0 {
+	status = "okay";
+};
+
+/* used for usb_host1_xhci */
+&combphy1 {
+	status = "okay";
+};
+
+/* connected to sata2 */
+&combphy2 {
+	status = "okay";
+};
+
+&cpu0 {
+	cpu-supply = <&vdd_cpu>;
+};
+
+&cpu1 {
+	cpu-supply = <&vdd_cpu>;
+};
+
+&cpu2 {
+	cpu-supply = <&vdd_cpu>;
+};
+
+&cpu3 {
+	cpu-supply = <&vdd_cpu>;
+};
+
+&gmac0 {
+	assigned-clocks = <&cru SCLK_GMAC0_RX_TX>, <&cru SCLK_GMAC0>;
+	assigned-clock-parents = <&cru SCLK_GMAC0_RGMII_SPEED>;
+	assigned-clock-rates = <0>, <125000000>;
+	clock_in_out = "output";
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac0_miim
+		     &gmac0_tx_bus2
+		     &gmac0_rx_bus2
+		     &gmac0_rgmii_clk
+		     &gmac0_rgmii_bus>;
+	phy-handle = <&rgmii_phy0>;
+	phy-mode = "rgmii-id";
+	status = "okay";
+};
+
+&gmac1 {
+	assigned-clocks = <&cru SCLK_GMAC1_RX_TX>, <&cru SCLK_GMAC1>;
+	assigned-clock-parents = <&cru SCLK_GMAC1_RGMII_SPEED>;
+	assigned-clock-rates = <0>, <125000000>;
+	clock_in_out = "output";
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac1m1_miim
+		     &gmac1m1_tx_bus2
+		     &gmac1m1_rx_bus2
+		     &gmac1m1_rgmii_clk
+		     &gmac1m1_rgmii_bus>;
+	phy-handle = <&rgmii_phy1>;
+	phy-mode = "rgmii-id";
+	status = "okay";
+};
+
+&gpu {
+	mali-supply = <&vdd_gpu>;
+	status = "okay";
+};
+
+&hdmi {
+	avdd-0v9-supply = <&vdda0v9_image>;
+	avdd-1v8-supply = <&vcca1v8_image>;
+	status = "okay";
+};
+
+&hdmi_in {
+	hdmi_in_vp0: endpoint {
+		remote-endpoint = <&vp0_out_hdmi>;
+	};
+};
+
+&hdmi_out {
+	hdmi_out_con: endpoint {
+		remote-endpoint = <&hdmi_con_in>;
+	};
+};
+
+&hdmi_sound {
+	status = "okay";
+};
+
+&i2c0 {
+	status = "okay";
+
+	vdd_cpu: regulator@1c {
+		compatible = "tcs,tcs4525";
+		reg = <0x1c>;
+		regulator-name = "vdd_cpu";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <800000>;
+		regulator-max-microvolt = <1150000>;
+		regulator-ramp-delay = <2300>;
+		vin-supply = <&vcc5v0_sys>;
+		fcs,suspend-voltage-selector = <1>;
+
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+
+	rk809: pmic@20 {
+		compatible = "rockchip,rk809";
+		reg = <0x20>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PA3 IRQ_TYPE_LEVEL_LOW>;
+		assigned-clocks = <&cru I2S1_MCLKOUT_TX>;
+		assigned-clock-parents = <&cru CLK_I2S1_8CH_TX>;
+		#clock-cells = <1>;
+		clock-names = "mclk";
+		clocks = <&cru I2S1_MCLKOUT_TX>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pmic_int>, <&i2s1m0_mclk>;
+		system-power-controller;
+		#sound-dai-cells = <0>;
+		vcc1-supply = <&vcc3v3_sys>;
+		vcc2-supply = <&vcc3v3_sys>;
+		vcc3-supply = <&vcc3v3_sys>;
+		vcc4-supply = <&vcc3v3_sys>;
+		vcc5-supply = <&vcc3v3_sys>;
+		vcc6-supply = <&vcc3v3_sys>;
+		vcc7-supply = <&vcc3v3_sys>;
+		vcc8-supply = <&vcc3v3_sys>;
+		vcc9-supply = <&vcc3v3_sys>;
+		wakeup-source;
+
+		regulators {
+			vdd_logic: DCDC_REG1 {
+				regulator-name = "vdd_logic";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-initial-mode = <0x2>;
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-ramp-delay = <6001>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_gpu: DCDC_REG2 {
+				regulator-name = "vdd_gpu";
+				regulator-always-on;
+				regulator-initial-mode = <0x2>;
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-ramp-delay = <6001>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_ddr: DCDC_REG3 {
+				regulator-name = "vcc_ddr";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-initial-mode = <0x2>;
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			vdd_npu: DCDC_REG4 {
+				regulator-name = "vdd_npu";
+				regulator-initial-mode = <0x2>;
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-ramp-delay = <6001>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_1v8: DCDC_REG5 {
+				regulator-name = "vcc_1v8";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdda0v9_image: LDO_REG1 {
+				regulator-name = "vdda0v9_image";
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdda_0v9: LDO_REG2 {
+				regulator-name = "vdda_0v9";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdda0v9_pmu: LDO_REG3 {
+				regulator-name = "vdda0v9_pmu";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <900000>;
+				};
+			};
+
+			vccio_acodec: LDO_REG4 {
+				regulator-name = "vccio_acodec";
+				regulator-always-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vccio_sd: LDO_REG5 {
+				regulator-name = "vccio_sd";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc3v3_pmu: LDO_REG6 {
+				regulator-name = "vcc3v3_pmu";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <3300000>;
+				};
+			};
+
+			vcca_1v8: LDO_REG7 {
+				regulator-name = "vcca_1v8";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcca1v8_pmu: LDO_REG8 {
+				regulator-name = "vcca1v8_pmu";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vcca1v8_image: LDO_REG9 {
+				regulator-name = "vcca1v8_image";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_3v3: SWITCH_REG1 {
+				regulator-name = "vcc_3v3";
+				regulator-always-on;
+				regulator-boot-on;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc3v3_sd: SWITCH_REG2 {
+				regulator-name = "vcc3v3_sd";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+		};
+
+		codec {
+			rockchip,mic-in-differential;
+		};
+	};
+};
+
+&i2c5 {
+	status = "okay";
+
+	rtc0: rtc@51 {
+		compatible = "nxp,pcf8563";
+		reg = <0x51>;
+		#clock-cells = <0>;
+	};
+};
+
+&i2s0_8ch {
+	status = "okay";
+};
+
+&i2s1_8ch {
+	pinctrl-0 = <&i2s1m0_sclktx &i2s1m0_lrcktx &i2s1m0_sdi0 &i2s1m0_sdo0>;
+	rockchip,trcm-sync-tx-only;
+	status = "okay";
+};
+
+/* required rk3568-codec-digital for AP6356S Bluetooth Sound */
+&i2s3_2ch {
+	status = "disabled";
+};
+
+&mdio0 {
+	rgmii_phy0: ethernet-phy@1 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <1>;
+		reset-assert-us = <20000>;
+		reset-deassert-us = <100000>;
+		reset-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_LOW>;
+
+		leds {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			led@1 {
+				reg = <1>;
+				color = <LED_COLOR_ID_GREEN>;
+				function = LED_FUNCTION_LAN;
+				default-state = "keep";
+			};
+
+			led@2 {
+				reg = <2>;
+				color = <LED_COLOR_ID_AMBER>;
+				function = LED_FUNCTION_LAN;
+				default-state = "keep";
+			};
+		};
+	};
+};
+
+&mdio1 {
+	rgmii_phy1: ethernet-phy@1 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <1>;
+		reset-assert-us = <20000>;
+		reset-deassert-us = <100000>;
+		reset-gpios = <&gpio2 RK_PD1 GPIO_ACTIVE_LOW>;
+
+		leds {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			led@1 {
+				reg = <1>;
+				color = <LED_COLOR_ID_GREEN>;
+				function = LED_FUNCTION_LAN;
+				default-state = "keep";
+			};
+
+			led@2 {
+				reg = <2>;
+				color = <LED_COLOR_ID_AMBER>;
+				function = LED_FUNCTION_LAN;
+				default-state = "keep";
+			};
+		};
+	};
+};
+
+&pcie30phy {
+	status = "okay";
+};
+
+&pcie3x2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie_reset_pin>;
+	reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_HIGH>;
+	vpcie3v3-supply = <&vcc3v3_pcie>;
+	status = "okay";
+};
+
+&pdm {
+	status = "okay";
+};
+
+&pinctrl {
+	bluetooth {
+		bt_reg_on_h: bt-reg-on-h {
+			rockchip,pins = <3 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		bt_wake_host_h: bt-wake-host-h {
+			rockchip,pins = <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+
+		host_wake_bt_h: host-wake-bt-h {
+			rockchip,pins = <3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	leds {
+		led_work_en: led_work_en {
+			rockchip,pins = <0 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	pmic {
+		pmic_int: pmic_int {
+			rockchip,pins = <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	pcie {
+		pcie_reset_pin: pcie-reset-pin {
+			rockchip,pins = <2 RK_PD6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+		vcc3v3_pcie_en_pin: vcc3v3-pcie-en-pin {
+			rockchip,pins = <3 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	sdio-pwrseq {
+		wifi_enable: wifi-enable {
+			rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	usb {
+		vcc5v0_usb_host_en: vcc5v0_usb_host_en {
+			rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+		vcc5v0_usb_otg_en: vcc5v0_usb_otg_en {
+			rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	wifi-irq {
+		wifi_host_wake_irq: wifi-host-wake-irq {
+			rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+	};
+};
+
+&pmu_io_domains {
+	pmuio1-supply = <&vcc3v3_pmu>;
+	pmuio2-supply = <&vcc3v3_pmu>;
+	vccio1-supply = <&vccio_acodec>;
+	vccio2-supply = <&vcc_1v8>;
+	vccio3-supply = <&vccio_sd>;
+	vccio4-supply = <&vcc_1v8>;
+	vccio5-supply = <&vcc_3v3>;
+	vccio6-supply = <&vcc_1v8>;
+	vccio7-supply = <&vcc_3v3>;
+	status = "okay";
+};
+
+&pwm4 {
+	status = "okay";
+};
+
+&pwm5 {
+	status = "okay";
+};
+
+/* Required remotectl for IR receiver */
+&pwm7 {
+	status = "disabled";
+};
+
+&saradc {
+	vref-supply = <&vcca_1v8>;
+	status = "okay";
+};
+
+&sata2 {
+	status = "okay";
+};
+
+/* used for eMMC */
+&sdhci {
+	bus-width = <8>;
+	max-frequency = <200000000>;
+	mmc-hs200-1_8v;
+	non-removable;
+	pinctrl-names = "default";
+	pinctrl-0 = <&emmc_bus8 &emmc_clk &emmc_cmd &emmc_datastrobe>;
+	status = "okay";
+};
+
+/* used for microSD (TF) Slot */
+&sdmmc0 {
+	bus-width = <4>;
+	cap-sd-highspeed;
+	cd-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_LOW>;
+	disable-wp;
+	pinctrl-names = "default";
+	pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>;
+	sd-uhs-sdr104;
+	vmmc-supply = <&vcc3v3_sd>;
+	vqmmc-supply = <&vccio_sd>;
+	status = "okay";
+};
+
+/* used for AP6356S WiFi */
+&sdmmc2 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	cap-sdio-irq;
+	keep-power-in-suspend;
+	mmc-pwrseq = <&sdio_pwrseq>;
+	non-removable;
+	pinctrl-names = "default";
+	pinctrl-0 = <&sdmmc2m0_bus4 &sdmmc2m0_cmd &sdmmc2m0_clk>;
+	sd-uhs-sdr104;
+	vmmc-supply = <&vcc3v3_sys>;
+	vqmmc-supply = <&vcc_1v8>;
+	status = "okay";
+
+	wifi@1 {
+		compatible = "brcm,bcm4356-fmac", "brcm,bcm4329-fmac";
+		reg = <1>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <RK_PD4 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "host-wake";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wifi_host_wake_irq>;
+	};
+};
+
+&spdif {
+	status = "okay";
+};
+
+&tsadc {
+	rockchip,hw-tshut-mode = <1>;
+	rockchip,hw-tshut-polarity = <0>;
+	status = "okay";
+};
+
+/* used for Debug */
+&uart2 {
+	status = "okay";
+};
+
+&uart3 {
+	pinctrl-0 = <&uart3m1_xfer>;
+	status = "okay";
+};
+
+&uart4 {
+	pinctrl-0 = <&uart4m1_xfer>;
+	status = "okay";
+};
+
+/* used for AP6356S Bluetooth */
+&uart8 {
+	dma-names = "tx", "rx";
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart8m0_xfer &uart8m0_ctsn &uart8m0_rtsn>;
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "brcm,bcm4345c5";
+		clocks = <&rk809 1>;
+		clock-names = "lpo";
+		device-wakeup-gpios = <&gpio3 RK_PA2 GPIO_ACTIVE_HIGH>;
+		host-wakeup-gpios = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>;
+		max-speed = <1500000>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&bt_reg_on_h &bt_wake_host_h &host_wake_bt_h>;
+		shutdown-gpios = <&gpio3 RK_PA0 GPIO_ACTIVE_HIGH>;
+		vddio-supply = <&vcc_1v8>;
+	};
+};
+
+&uart9 {
+	pinctrl-0 = <&uart9m1_xfer>;
+	status = "okay";
+};
+
+&usb_host0_ehci {
+	status = "okay";
+};
+
+&usb_host0_ohci {
+	status = "okay";
+};
+
+&usb_host0_xhci {
+	extcon = <&usb2phy0>;
+	status = "okay";
+};
+
+&usb_host1_ehci {
+	status = "okay";
+};
+
+&usb_host1_ohci {
+	status = "okay";
+};
+
+&usb_host1_xhci {
+	status = "okay";
+};
+
+&usb2phy0 {
+	status = "okay";
+};
+
+&usb2phy0_host {
+	phy-supply = <&vcc5v0_usb_host>;
+	status = "okay";
+};
+
+&usb2phy0_otg {
+	phy-supply = <&vcc5v0_usb_otg>;
+	status = "okay";
+};
+
+&usb2phy1 {
+	status = "okay";
+};
+
+&usb2phy1_host {
+	phy-supply = <&vcc5v0_usb_host>;
+	status = "okay";
+};
+
+&usb2phy1_otg {
+	phy-supply = <&vcc5v0_usb_host>;
+	status = "okay";
+};
+
+&vop {
+	assigned-clocks = <&cru DCLK_VOP0>, <&cru DCLK_VOP1>;
+	assigned-clock-parents = <&pmucru PLL_HPLL>, <&cru PLL_VPLL>;
+	status = "okay";
+};
+
+&vop_mmu {
+	status = "okay";
+};
+
+&vp0 {
+	vp0_out_hdmi: endpoint@ROCKCHIP_VOP2_EP_HDMI0 {
+		reg = <ROCKCHIP_VOP2_EP_HDMI0>;
+		remote-endpoint = <&hdmi_in_vp0>;
+	};
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3576-lckfb-tspi-3m.dts b/arch/arm64/boot/dts/rockchip/rk3576-lckfb-tspi-3m.dts
new file mode 100644
index 0000000000000..4ba450c7308bb
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3576-lckfb-tspi-3m.dts
@@ -0,0 +1,1003 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2026 LCKFB
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/rockchip.h>
+#include <dt-bindings/soc/rockchip,vop2.h>
+#include <dt-bindings/usb/pd.h>
+#include "rk3576.dtsi"
+
+/ {
+	model = "LCKFB TaishanPi 3M";
+	compatible = "lckfb,tspi-3m-rk3576", "rockchip,rk3576";
+
+	aliases {
+		mmc0 = &sdhci;
+		mmc1 = &sdmmc;
+	};
+
+	chosen {
+		stdout-path = "serial0:1500000n8";
+	};
+
+	adc-keys {
+		compatible = "adc-keys";
+		io-channels = <&saradc 1>;
+		io-channel-names = "buttons";
+		keyup-threshold-microvolt = <1800000>;
+		poll-interval = <100>;
+
+		button-recovery {
+			label = "Recovery";
+			linux,code = <KEY_VENDOR>;
+			press-threshold-microvolt = <18000>;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-0 {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_HEARTBEAT;
+			gpios = <&gpio0 RK_PA0 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "heartbeat";
+		};
+	};
+
+	hdmi-con {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con_in: endpoint {
+				remote-endpoint = <&hdmi_out_con>;
+			};
+		};
+	};
+
+	vcc_5v0_usb2_host: regulator-vcc-5v0-usb2-host {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio2 RK_PB5 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&usb2_host_pwren>;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "vcc_5v0_usb2_host";
+		vin-supply = <&vcc_5v0_sys>;
+	};
+
+	vcc_5v0_usb3_host: regulator-vcc-5v0-usb3-host {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio0 RK_PC3 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&usb3_host_pwren>;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "vcc_5v0_usb3_host";
+		vin-supply = <&vcc_5v0_sys>;
+	};
+
+	vcc_5v0_typec: regulator-vcc-5v0-typec {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio0 RK_PB6 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&typec_vbus_en>;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "vcc_5v0_typec";
+		vin-supply = <&vcc_5v0_sys>;
+	};
+
+	vcc_5v0_sys: regulator-vcc-5v0-sys {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_5v0_sys";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+	};
+
+	vcc_1v1_nldo_s3: regulator-vcc-1v1-nldo-s3 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_1v1_nldo_s3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1100000>;
+		regulator-max-microvolt = <1100000>;
+		vin-supply = <&vcc_5v0_sys>;
+	};
+
+	vcc_2v0_pldo_s3: regulator-vcc-2v0-pldo-s3 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_2v0_pldo_s3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <2000000>;
+		regulator-max-microvolt = <2000000>;
+		vin-supply = <&vcc_5v0_sys>;
+	};
+
+	vcc_3v3_minipcie: regulator-vcc-3v3-minipcie {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio1 RK_PD5 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&minipcie_pwren>;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "vcc_3v3_minipcie";
+		startup-delay-us = <5000>;
+		vin-supply = <&vcc_5v0_sys>;
+	};
+
+	vcc_3v3_wl: regulator-3v3-wl {
+		compatible = "regulator-fixed";
+		gpio = <&gpio0 RK_PD1 GPIO_ACTIVE_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&wifi_pwren_l>;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "vcc_3v3_wl";
+		vin-supply = <&vcc_3v3_s0>;
+	};
+
+	vcc_3v3_s0: regulator-vcc-3v3-s0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_3v3_s0";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc_3v3_s3>;
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		clock-names = "ext_clock";
+		clocks = <&hym8563>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&wifi_reg_on_h>;
+		post-power-on-delay-ms = <200>;
+		reset-gpios = <&gpio1 RK_PC6 GPIO_ACTIVE_LOW>;
+	};
+
+	usb_a_3_0: usb-a-connector-0 {
+		compatible = "usb-a-connector";
+		label = "USB 3.0 Type-A";
+		vbus-supply = <&vcc_5v0_usb3_host>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				usb_a_3_0_hs: endpoint {
+					remote-endpoint = <&usb_hub_port1>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				usb_a_3_0_ss: endpoint {
+					remote-endpoint = <&usb_drd1_ss>;
+				};
+			};
+		};
+	};
+
+	usb_a_2_0_1: usb-a-connector-1 {
+		compatible = "usb-a-connector";
+		label = "USB 2.0 Type-A 1";
+		vbus-supply = <&vcc_5v0_usb3_host>;
+
+		port {
+			usb_a_2_0_1_hs: endpoint {
+				remote-endpoint = <&usb_hub_port2>;
+			};
+		};
+	};
+
+	usb_a_2_0_2: usb-a-connector-2 {
+		compatible = "usb-a-connector";
+		label = "USB 2.0 Type-A 2";
+		vbus-supply = <&vcc_5v0_usb2_host>;
+
+		port {
+			usb_a_2_0_2_hs: endpoint {
+				remote-endpoint = <&usb_hub_port3>;
+			};
+		};
+	};
+
+	usb_a_2_0_3: usb-a-connector-3 {
+		compatible = "usb-a-connector";
+		label = "USB 2.0 Type-A 3";
+		vbus-supply = <&vcc_5v0_usb2_host>;
+
+		port {
+			usb_a_2_0_3_hs: endpoint {
+				remote-endpoint = <&usb_hub_port4>;
+			};
+		};
+	};
+};
+
+&combphy0_ps {
+	status = "okay";
+};
+
+&combphy1_psu {
+	status = "okay";
+};
+
+&cpu_b0 {
+	cpu-supply = <&vdd_cpu_big_s0>;
+};
+
+&cpu_b1 {
+	cpu-supply = <&vdd_cpu_big_s0>;
+};
+
+&cpu_b2 {
+	cpu-supply = <&vdd_cpu_big_s0>;
+};
+
+&cpu_b3 {
+	cpu-supply = <&vdd_cpu_big_s0>;
+};
+
+&cpu_l0 {
+	cpu-supply = <&vdd_cpu_lit_s0>;
+};
+
+&cpu_l1 {
+	cpu-supply = <&vdd_cpu_lit_s0>;
+};
+
+&cpu_l2 {
+	cpu-supply = <&vdd_cpu_lit_s0>;
+};
+
+&cpu_l3 {
+	cpu-supply = <&vdd_cpu_lit_s0>;
+};
+
+&gmac0 {
+	clock_in_out = "input";
+	phy-handle = <&rgmii_phy0>;
+	/* RX delay is added by the PHY, TX delay by the GMAC. */
+	phy-mode = "rgmii-rxid";
+	pinctrl-names = "default";
+	pinctrl-0 = <&eth0m0_miim
+		     &eth0m0_tx_bus2
+		     &eth0m0_rx_bus2
+		     &eth0m0_rgmii_clk
+		     &eth0m0_rgmii_bus
+		     &eth0m0_mclk>;
+	tx_delay = <0x24>;
+	status = "okay";
+};
+
+&hdmi {
+	/* The external level shifters must be on for TMDS operation. */
+	frl-enable-gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&hdmi_in {
+	hdmi_in_vp0: endpoint {
+		remote-endpoint = <&vp0_out_hdmi>;
+	};
+};
+
+&hdmi_out {
+	hdmi_out_con: endpoint {
+		remote-endpoint = <&hdmi_con_in>;
+	};
+};
+
+&hdmi_sound {
+	simple-audio-card,mclk-fs = <128>;
+	status = "okay";
+
+	simple-audio-card,cpu {
+		dai-tdm-slot-num = <2>;
+		dai-tdm-slot-width = <32>;
+	};
+};
+
+&hdptxphy {
+	status = "okay";
+};
+
+&i2c1 {
+	status = "okay";
+
+	pmic@23 {
+		compatible = "rockchip,rk806";
+		reg = <0x23>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		interrupt-parent = <&gpio0>;
+		interrupts = <6 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pmic_pins
+			     &rk806_dvs1_null
+			     &rk806_dvs2_null
+			     &rk806_dvs3_null>;
+		system-power-controller;
+		vcc1-supply = <&vcc_5v0_sys>;
+		vcc2-supply = <&vcc_5v0_sys>;
+		vcc3-supply = <&vcc_5v0_sys>;
+		vcc4-supply = <&vcc_5v0_sys>;
+		vcc5-supply = <&vcc_5v0_sys>;
+		vcc6-supply = <&vcc_5v0_sys>;
+		vcc7-supply = <&vcc_5v0_sys>;
+		vcc8-supply = <&vcc_5v0_sys>;
+		vcc9-supply = <&vcc_5v0_sys>;
+		vcc10-supply = <&vcc_5v0_sys>;
+		vcc11-supply = <&vcc_2v0_pldo_s3>;
+		vcc12-supply = <&vcc_5v0_sys>;
+		vcc13-supply = <&vcc_1v1_nldo_s3>;
+		vcc14-supply = <&vcc_1v1_nldo_s3>;
+		vcca-supply = <&vcc_5v0_sys>;
+
+		rk806_dvs1_null: dvs1-null-pins {
+			pins = "gpio_pwrctrl1";
+			function = "pin_fun0";
+		};
+
+		rk806_dvs2_null: dvs2-null-pins {
+			pins = "gpio_pwrctrl2";
+			function = "pin_fun0";
+		};
+
+		rk806_dvs3_null: dvs3-null-pins {
+			pins = "gpio_pwrctrl3";
+			function = "pin_fun0";
+		};
+
+		regulators {
+			vdd_cpu_big_s0: dcdc-reg1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-enable-ramp-delay = <400>;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <950000>;
+				regulator-name = "vdd_cpu_big_s0";
+				regulator-ramp-delay = <12500>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_npu_s0: dcdc-reg2 {
+				regulator-boot-on;
+				regulator-enable-ramp-delay = <400>;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <950000>;
+				regulator-name = "vdd_npu_s0";
+				regulator-ramp-delay = <12500>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_cpu_lit_s0: dcdc-reg3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <950000>;
+				regulator-name = "vdd_cpu_lit_s0";
+				regulator-ramp-delay = <12500>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <750000>;
+				};
+			};
+
+			vcc_3v3_s3: dcdc-reg4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc_3v3_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <3300000>;
+				};
+			};
+
+			vdd_gpu_s0: dcdc-reg5 {
+				regulator-boot-on;
+				regulator-enable-ramp-delay = <400>;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <900000>;
+				regulator-name = "vdd_gpu_s0";
+				regulator-ramp-delay = <12500>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <850000>;
+				};
+			};
+
+			vddq_ddr_s0: dcdc-reg6 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-name = "vddq_ddr_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_logic_s0: dcdc-reg7 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <800000>;
+				regulator-name = "vdd_logic_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_1v8_s3: dcdc-reg8 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcc_1v8_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vdd2_ddr_s3: dcdc-reg9 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-name = "vdd2_ddr_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			vdd_ddr_s0: dcdc-reg10 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-name = "vdd_ddr_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcca_1v8_s0: pldo-reg1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcca_1v8_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcca1v8_pldo2_s0: pldo-reg2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcca1v8_pldo2_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdda_1v2_s0: pldo-reg3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-name = "vdda_1v2_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcca_3v3_s0: pldo-reg4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcca_3v3_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vccio_sd_s0: pldo-reg5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vccio_sd_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcca1v8_pldo6_s3: pldo-reg6 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcca1v8_pldo6_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vdd_0v75_s3: nldo-reg1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt = <750000>;
+				regulator-name = "vdd_0v75_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <750000>;
+				};
+			};
+
+			vdda_ddr_pll_s0: nldo-reg2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <850000>;
+				regulator-name = "vdda_ddr_pll_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdda_hdmi_s0: nldo-reg3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <837500>;
+				regulator-max-microvolt = <837500>;
+				regulator-name = "vdda_hdmi_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdda_0v85_s0: nldo-reg4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <850000>;
+				regulator-name = "vdda_0v85_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdda_0v75_s0: nldo-reg5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt = <750000>;
+				regulator-name = "vdda_0v75_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+		};
+	};
+};
+
+&i2c2 {
+	status = "okay";
+
+	typec@22 {
+		compatible = "fcs,fusb302";
+		reg = <0x22>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PB4 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&fusb302_int>;
+		vbus-supply = <&vcc_5v0_typec>;
+
+		connector {
+			compatible = "usb-c-connector";
+			data-role = "host";
+			label = "USB-C";
+			power-role = "source";
+			source-pdos = <PDO_FIXED(5000, 500, PDO_FIXED_USB_COMM)>;
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					usb_c_hs: endpoint {
+						remote-endpoint = <&usb_drd0_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					usb_c_ss: endpoint {
+						remote-endpoint = <&usbdp_phy_ep>;
+					};
+				};
+			};
+		};
+	};
+
+	hym8563: rtc@51 {
+		compatible = "haoyu,hym8563";
+		reg = <0x51>;
+		#clock-cells = <0>;
+		clock-output-names = "hym8563";
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PA5 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&rtc_int_l>;
+		wakeup-source;
+	};
+};
+
+&mdio0 {
+	rgmii_phy0: ethernet-phy@1 {
+		compatible = "ethernet-phy-id7b74.4412";
+		reg = <0x1>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&gmac0_phy_reset>;
+		reset-assert-us = <20000>;
+		reset-deassert-us = <100000>;
+		reset-gpios = <&gpio2 RK_PB3 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&pinctrl {
+	ethernet {
+		gmac0_phy_reset: gmac0-phy-reset {
+			rockchip,pins = <2 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	minipcie {
+		minipcie_pwren: minipcie-pwren {
+			rockchip,pins = <1 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	rtc {
+		rtc_int_l: rtc-int-l {
+			rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	wireless-bluetooth {
+		bt_host_wake_h: bt-host-wake-h {
+			rockchip,pins = <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+
+		bt_reg_on_h: bt-reg-on-h {
+			rockchip,pins = <1 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		bt_wake_h: bt-wake-h {
+			rockchip,pins = <1 RK_PD4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	wireless-wlan {
+		wifi_host_wake_h: wifi-host-wake-h {
+			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+
+		wifi_pwren_l: wifi-pwren-l {
+			rockchip,pins = <0 RK_PD1 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+
+		wifi_reg_on_h: wifi-reg-on-h {
+			rockchip,pins = <1 RK_PC6 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	sdmmc {
+		sdmmc0_det_l: sdmmc0-det-l {
+			rockchip,pins = <0 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	usb-host {
+		usb2_host_pwren: usb2-host-pwren {
+			rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		usb3_host_pwren: usb3-host-pwren {
+			rockchip,pins = <0 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	usb-typec {
+		fusb302_int: fusb302-int {
+			rockchip,pins = <0 RK_PB4 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+
+		typec_vbus_en: typec-vbus-en {
+			rockchip,pins = <0 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+};
+
+&sai6 {
+	rockchip,sai-tx-route = <0>;
+	status = "okay";
+};
+
+&saradc {
+	vref-supply = <&vcca1v8_pldo2_s0>;
+	status = "okay";
+};
+
+&sata0 {
+	target-supply = <&vcc_3v3_minipcie>;
+	status = "okay";
+};
+
+&sdio {
+	bus-width = <4>;
+	cap-sd-highspeed;
+	cap-sdio-irq;
+	keep-power-in-suspend;
+	max-frequency = <100000000>;
+	mmc-pwrseq = <&sdio_pwrseq>;
+	no-mmc;
+	no-sd;
+	non-removable;
+	sd-uhs-sdr104;
+	vmmc-supply = <&vcc_3v3_wl>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	wifi@1 {
+		compatible = "brcm,bcm43456-fmac", "brcm,bcm4329-fmac";
+		reg = <1>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PB0 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "host-wake";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wifi_host_wake_h>;
+	};
+};
+
+&sdmmc {
+	bus-width = <4>;
+	cap-sd-highspeed;
+	cd-gpios = <&gpio0 RK_PA7 GPIO_ACTIVE_LOW>;
+	disable-wp;
+	max-frequency = <50000000>;
+	no-mmc;
+	no-sdio;
+	pinctrl-names = "default";
+	/* GPIO0_B6/SDMMC0_PWREN is not connected on this board. */
+	pinctrl-0 = <&sdmmc0_clk &sdmmc0_cmd &sdmmc0_det_l &sdmmc0_bus4>;
+	vmmc-supply = <&vcc_3v3_s0>;
+	vqmmc-supply = <&vccio_sd_s0>;
+	status = "okay";
+};
+
+&sdhci {
+	bus-width = <8>;
+	full-pwr-cycle-in-suspend;
+	mmc-hs400-1_8v;
+	mmc-hs400-enhanced-strobe;
+	no-sd;
+	no-sdio;
+	non-removable;
+	vmmc-supply = <&vcc_3v3_s3>;
+	vqmmc-supply = <&vcc_1v8_s3>;
+	status = "okay";
+};
+
+&tsadc {
+	status = "okay";
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&uart6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart6m1_xfer &uart6m1_ctsn &uart6m1_rtsn>;
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "brcm,bcm4345c5";
+		clocks = <&hym8563>;
+		clock-names = "lpo";
+		device-wakeup-gpios = <&gpio1 RK_PD4 GPIO_ACTIVE_HIGH>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PB1 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "host-wakeup";
+		max-speed = <1500000>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&bt_host_wake_h &bt_reg_on_h &bt_wake_h>;
+		shutdown-gpios = <&gpio1 RK_PC7 GPIO_ACTIVE_HIGH>;
+		vbat-supply = <&vcc_3v3_wl>;
+	};
+};
+
+&u2phy0 {
+	status = "okay";
+};
+
+&u2phy0_otg {
+	status = "okay";
+};
+
+&u2phy1 {
+	status = "okay";
+};
+
+&u2phy1_otg {
+	status = "okay";
+};
+
+&usbdp_phy {
+	mode-switch;
+	orientation-switch;
+	status = "okay";
+
+	port {
+		usbdp_phy_ep: endpoint {
+			remote-endpoint = <&usb_c_ss>;
+		};
+	};
+};
+
+&usb_drd0_dwc3 {
+	usb-role-switch;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+
+			usb_drd0_hs: endpoint {
+				remote-endpoint = <&usb_c_hs>;
+			};
+		};
+	};
+};
+
+&usb_drd1_dwc3 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	dr_mode = "host";
+	status = "okay";
+
+	usb_hub: hub@1 {
+		compatible = "usb1a40,0201";
+		reg = <1>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@1 {
+				reg = <1>;
+
+				usb_hub_port1: endpoint {
+					remote-endpoint = <&usb_a_3_0_hs>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+
+				usb_hub_port2: endpoint {
+					remote-endpoint = <&usb_a_2_0_1_hs>;
+				};
+			};
+
+			port@3 {
+				reg = <3>;
+
+				usb_hub_port3: endpoint {
+					remote-endpoint = <&usb_a_2_0_2_hs>;
+				};
+			};
+
+			port@4 {
+				reg = <4>;
+
+				usb_hub_port4: endpoint {
+					remote-endpoint = <&usb_a_2_0_3_hs>;
+				};
+			};
+		};
+	};
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@1 {
+			reg = <1>;
+
+			usb_drd1_ss: endpoint {
+				remote-endpoint = <&usb_a_3_0_ss>;
+			};
+		};
+	};
+};
+
+&vop {
+	status = "okay";
+};
+
+&vop_mmu {
+	status = "okay";
+};
+
+&vp0 {
+	vp0_out_hdmi: endpoint@ROCKCHIP_VOP2_EP_HDMI0 {
+		reg = <ROCKCHIP_VOP2_EP_HDMI0>;
+		remote-endpoint = <&hdmi_in_vp0>;
+	};
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3576.dtsi b/arch/arm64/boot/dts/rockchip/rk3576.dtsi
index e12a2a0cfb891..b0c0d3c8b1b14 100644
--- a/arch/arm64/boot/dts/rockchip/rk3576.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3576.dtsi
@@ -1496,6 +1496,34 @@ hdmi_out: port@1 {
 			};
 		};
 
+		edp: edp@27dc0000 {
+			compatible = "rockchip,rk3576-edp";
+			reg = <0x0 0x27dc0000 0x0 0x1000>;
+			clocks = <&cru CLK_EDP0_24M>, <&cru PCLK_EDP0>, <&cru HCLK_VO0_ROOT>;
+			clock-names = "dp", "pclk", "hclk";
+			interrupts = <GIC_SPI 365 IRQ_TYPE_LEVEL_HIGH>;
+			phys = <&hdptxphy>;
+			phy-names = "dp";
+			power-domains = <&power RK3576_PD_VO0>;
+			resets = <&cru SRST_EDP0_24M>, <&cru SRST_P_EDP0>;
+			reset-names = "dp", "apb";
+			rockchip,grf = <&vo0_grf>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				edp_in: port@0 {
+					reg = <0>;
+				};
+
+				edp_out: port@1 {
+					reg = <1>;
+				};
+			};
+		};
+
 		dp: dp@27e40000 {
 			compatible = "rockchip,rk3576-dp";
 			reg = <0x0 0x27e40000 0x0 0x30000>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi
index fc1fdbfd31622..376ad04e07869 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi
@@ -1910,8 +1910,8 @@ hdmi0_out: port@1 {
 	edp0: edp@fdec0000 {
 		compatible = "rockchip,rk3588-edp";
 		reg = <0x0 0xfdec0000 0x0 0x1000>;
-		clocks = <&cru CLK_EDP0_24M>, <&cru PCLK_EDP0>;
-		clock-names = "dp", "pclk";
+		clocks = <&cru CLK_EDP0_24M>, <&cru PCLK_EDP0>, <&cru HCLK_VO1>;
+		clock-names = "dp", "pclk", "hclk";
 		interrupts = <GIC_SPI 163 IRQ_TYPE_LEVEL_HIGH 0>;
 		phys = <&hdptxphy0>;
 		phy-names = "dp";
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi
index a2640014ee042..b251bb129cdbf 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi
@@ -285,8 +285,8 @@ hdmi1_out: port@1 {
 	edp1: edp@fded0000 {
 		compatible = "rockchip,rk3588-edp";
 		reg = <0x0 0xfded0000 0x0 0x1000>;
-		clocks = <&cru CLK_EDP1_24M>, <&cru PCLK_EDP1>;
-		clock-names = "dp", "pclk";
+		clocks = <&cru CLK_EDP1_24M>, <&cru PCLK_EDP1>, <&cru HCLK_VO1>;
+		clock-names = "dp", "pclk", "hclk";
 		interrupts = <GIC_SPI 164 IRQ_TYPE_LEVEL_HIGH 0>;
 		phys = <&hdptxphy1>;
 		phy-names = "dp";
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-friendlyelec-cm3588-nas.dts b/arch/arm64/boot/dts/rockchip/rk3588-friendlyelec-cm3588-nas.dts
index 0dce96ca8c288..f4c34da61c4df 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-friendlyelec-cm3588-nas.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-friendlyelec-cm3588-nas.dts
@@ -12,6 +12,7 @@
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/pinctrl/rockchip.h>
 #include <dt-bindings/soc/rockchip,vop2.h>
+#include <dt-bindings/thermal/thermal.h>
 #include <dt-bindings/usb/pd.h>
 #include "rk3588-friendlyelec-cm3588.dtsi"
 
@@ -457,6 +458,36 @@ &i2c8 {
 	status = "okay";
 };
 
+&package_thermal {
+	polling-delay = <1000>;
+
+	trips {
+		package_warm: package-warm {
+			temperature = <55000>;
+			hysteresis = <2000>;
+			type = "active";
+		};
+
+		package_hot: package-hot {
+			temperature = <65000>;
+			hysteresis = <2000>;
+			type = "active";
+		};
+	};
+
+	cooling-maps {
+		map0 {
+			trip = <&package_warm>;
+			cooling-device = <&fan THERMAL_NO_LIMIT 1>;
+		};
+
+		map1 {
+			trip = <&package_hot>;
+			cooling-device = <&fan 2 THERMAL_NO_LIMIT>;
+		};
+	};
+};
+
 &pcie2x1l0 {
 	/* 2. M.2 socket, CON14: pcie30phy port0 lane1, @fe170000 */
 	max-link-speed = <3>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi
index 7495af5bd9731..cfdb5c13f8606 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi
@@ -201,25 +201,25 @@ vbus5v0_usb: regulator-vbus5v0-usb {
 		vin-supply = <&vcc5v0_sys>;
 	};
 
-	vcc3v3_pcie2x1l0: regulator-vcc3v3-pcie2x1l0 {
+	vdd_mpcie_3v3: regulator-vdd-mpcie-3v3 {
 		compatible = "regulator-fixed";
 		enable-active-high;
 		gpio = <&gpio4 RK_PC2 GPIO_ACTIVE_HIGH>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&pcie_m2_1_pwren>;
-		regulator-name = "vcc3v3_pcie2x1l0";
+		regulator-name = "vdd_mpcie_3v3";
 		regulator-min-microvolt = <3300000>;
 		regulator-max-microvolt = <3300000>;
 		vin-supply = <&vcc5v0_sys>;
 	};
 
-	vcc3v3_pcie30: regulator-vcc3v3-pcie30 {
+	vcc3v3_pcie_m2_0: regulator-vcc3v3-pcie-m2-0 {
 		compatible = "regulator-fixed";
 		enable-active-high;
 		gpios = <&gpio2 RK_PC5 GPIO_ACTIVE_HIGH>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&pcie_m2_0_pwren>;
-		regulator-name = "vcc3v3_pcie30";
+		regulator-name = "vcc3v3_pcie_m2_0";
 		regulator-min-microvolt = <3300000>;
 		regulator-max-microvolt = <3300000>;
 		vin-supply = <&vcc5v0_sys>;
@@ -648,7 +648,7 @@ &pcie2x1l0 {
 
 &pcie2x1l1 {
 	reset-gpios = <&gpio4 RK_PA2 GPIO_ACTIVE_HIGH>;
-	vpcie3v3-supply = <&vcc3v3_pcie2x1l0>;
+	vpcie3v3-supply = <&vdd_mpcie_3v3>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie2_1_rst>;
 	status = "okay";
@@ -668,7 +668,7 @@ &pcie30phy {
 
 &pcie3x4 {
 	reset-gpios = <&gpio4 RK_PB6 GPIO_ACTIVE_HIGH>;
-	vpcie3v3-supply = <&vcc3v3_pcie30>;
+	vpcie3v3-supply = <&vcc3v3_pcie_m2_0>;
 	status = "okay";
 };
 
@@ -985,7 +985,7 @@ vcc_2v0_pldo_s3: dcdc-reg7 {
 				regulator-min-microvolt = <2000000>;
 				regulator-max-microvolt = <2000000>;
 				regulator-ramp-delay = <12500>;
-				regulator-name = "vdd_2v0_pldo_s3";
+				regulator-name = "vcc_2v0_pldo_s3";
 
 				regulator-state-mem {
 					regulator-on-in-suspend;
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-youyeetoo-yy3588.dts b/arch/arm64/boot/dts/rockchip/rk3588-youyeetoo-yy3588.dts
new file mode 100644
index 0000000000000..e31d04b85edf7
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3588-youyeetoo-yy3588.dts
@@ -0,0 +1,1190 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2024 Youyeetoo
+ * Copyright (c) 2026 Daniele Briguglio <hello@superkali.me>
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/rockchip.h>
+#include <dt-bindings/soc/rockchip,vop2.h>
+#include <dt-bindings/usb/pd.h>
+#include "rk3588.dtsi"
+
+/ {
+	model = "Youyeetoo YY3588";
+	compatible = "youyeetoo,yy3588", "rockchip,rk3588";
+
+	aliases {
+		ethernet0 = &gmac1;
+		ethernet1 = &r8125;
+		mmc0 = &sdhci;
+		mmc1 = &sdmmc;
+	};
+
+	adc-keys {
+		compatible = "adc-keys";
+		io-channels = <&saradc 1>;
+		io-channel-names = "buttons";
+		keyup-threshold-microvolt = <1800000>;
+		poll-interval = <100>;
+
+		button-recovery {
+			label = "Recovery";
+			linux,code = <KEY_VENDOR>;
+			press-threshold-microvolt = <17000>;
+		};
+	};
+
+	analog-sound {
+		compatible = "simple-audio-card";
+		pinctrl-names = "default";
+		pinctrl-0 = <&hp_det>;
+		simple-audio-card,name = "rockchip-es8388";
+		simple-audio-card,aux-devs = <&speaker_amp>;
+		simple-audio-card,format = "i2s";
+		simple-audio-card,hp-det-gpios = <&gpio1 RK_PC4 GPIO_ACTIVE_LOW>;
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,pin-switches = "Headphones", "Speaker";
+		simple-audio-card,routing =
+			"Headphones", "LOUT2",
+			"Headphones", "ROUT2",
+			"Speaker Amp INL", "LOUT1",
+			"Speaker Amp INR", "ROUT1",
+			"Speaker", "Speaker Amp OUTL",
+			"Speaker", "Speaker Amp OUTR",
+			"LINPUT1", "Microphone Jack",
+			"RINPUT1", "Microphone Jack",
+			"LINPUT2", "Onboard Microphone",
+			"RINPUT2", "Onboard Microphone";
+		simple-audio-card,widgets =
+			"Microphone", "Microphone Jack",
+			"Microphone", "Onboard Microphone",
+			"Headphone", "Headphones",
+			"Speaker", "Speaker";
+
+		simple-audio-card,codec {
+			sound-dai = <&es8388>;
+			system-clock-frequency = <12288000>;
+		};
+
+		simple-audio-card,cpu {
+			sound-dai = <&i2s0_8ch>;
+		};
+	};
+
+	chosen {
+		stdout-path = "serial2:1500000n8";
+	};
+
+	hdmi0-con {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi0_con_in: endpoint {
+				remote-endpoint = <&hdmi0_out_con>;
+			};
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&led_pins>;
+
+		led-0 {
+			color = <LED_COLOR_ID_BLUE>;
+			function = LED_FUNCTION_STATUS;
+			gpios = <&gpio1 RK_PD6 GPIO_ACTIVE_HIGH>;
+		};
+
+		led-1 {
+			function = LED_FUNCTION_HEARTBEAT;
+			gpios = <&gpio1 RK_PD7 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+		};
+	};
+
+	/* PI6C557-05BLE PCIe 3.0 reference clock generator */
+	pcie30_port0_refclk: pcie-oscillator {
+		compatible = "gated-fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <100000000>;
+		clock-output-names = "pcie30_refclk";
+		vdd-supply = <&vcc3v3_pi6c_05>;
+	};
+
+	fan: pwm-fan {
+		compatible = "pwm-fan";
+		#cooling-cells = <2>;
+		cooling-levels = <0 50 100 150 200 255>;
+		fan-supply = <&vcc12v_dcin>;
+		pwms = <&pwm2 0 50000 0>;
+	};
+
+	pcie20_avdd0v85: regulator-pcie20-avdd0v85 {
+		compatible = "regulator-fixed";
+		regulator-name = "pcie20_avdd0v85";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <850000>;
+		regulator-max-microvolt = <850000>;
+		vin-supply = <&vdd_0v85_s0>;
+	};
+
+	pcie20_avdd1v8: regulator-pcie20-avdd1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "pcie20_avdd1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&avcc_1v8_s0>;
+	};
+
+	pcie30_avdd0v75: regulator-pcie30-avdd0v75 {
+		compatible = "regulator-fixed";
+		regulator-name = "pcie30_avdd0v75";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <750000>;
+		regulator-max-microvolt = <750000>;
+		vin-supply = <&avdd_0v75_s0>;
+	};
+
+	pcie30_avdd1v8: regulator-pcie30-avdd1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "pcie30_avdd1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&avcc_1v8_s0>;
+	};
+
+	vbus5v0_typec: regulator-vbus5v0-typec {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio4 RK_PB0 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&typec5v_pwren>;
+		regulator-name = "vbus5v0_typec";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&vcc5v0_sys>;
+	};
+
+	vcc_1v1_nldo_s3: regulator-vcc-1v1-nldo-s3 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_1v1_nldo_s3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1100000>;
+		regulator-max-microvolt = <1100000>;
+		vin-supply = <&vcc5v0_sys>;
+	};
+
+	vcc12v_dcin: regulator-vcc12v-dcin {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc12v_dcin";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+	};
+
+	vcc3v3_pi6c_05: regulator-vcc3v3-pi6c-05 {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio2 RK_PC5 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pi6c_05_pwren>;
+		regulator-name = "vcc3v3_pi6c_05";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		startup-delay-us = <5000>;
+		vin-supply = <&vcc5v0_sys>;
+	};
+
+	vcc3v3_sd_s0: regulator-vcc3v3-sd-s0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc3v3_sd_s0";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc3v3_sys>;
+	};
+
+	vcc3v3_sys: regulator-vcc3v3-sys {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc3v3_sys";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc12v_dcin>;
+	};
+
+	vcc5v0_sys: regulator-vcc5v0-sys {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc5v0_sys";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&vcc12v_dcin>;
+	};
+
+	speaker_amp: speaker-amplifier {
+		compatible = "simple-audio-amplifier";
+		enable-gpios = <&gpio4 RK_PB5 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&spk_en>;
+		sound-name-prefix = "Speaker Amp";
+		VCC-supply = <&vcc5v0_sys>;
+	};
+};
+
+&combphy0_ps {
+	status = "okay";
+};
+
+&combphy1_ps {
+	status = "okay";
+};
+
+&combphy2_psu {
+	status = "okay";
+};
+
+&cpu_b0 {
+	cpu-supply = <&vdd_cpu_big0_s0>;
+};
+
+&cpu_b1 {
+	cpu-supply = <&vdd_cpu_big0_s0>;
+};
+
+&cpu_b2 {
+	cpu-supply = <&vdd_cpu_big1_s0>;
+};
+
+&cpu_b3 {
+	cpu-supply = <&vdd_cpu_big1_s0>;
+};
+
+&cpu_l0 {
+	cpu-supply = <&vdd_cpu_lit_s0>;
+};
+
+&cpu_l1 {
+	cpu-supply = <&vdd_cpu_lit_s0>;
+};
+
+&cpu_l2 {
+	cpu-supply = <&vdd_cpu_lit_s0>;
+};
+
+&cpu_l3 {
+	cpu-supply = <&vdd_cpu_lit_s0>;
+};
+
+&gmac1 {
+	clock_in_out = "input";
+	phy-handle = <&rgmii_phy>;
+	/* RX delay is added by the PHY, TX delay by the GMAC */
+	phy-mode = "rgmii-rxid";
+	pinctrl-names = "default";
+	pinctrl-0 = <&gmac1_miim
+		     &gmac1_tx_bus2
+		     &gmac1_rx_bus2
+		     &gmac1_rgmii_clk
+		     &gmac1_rgmii_bus
+		     &gmac1_clkinout>;
+	tx_delay = <0x43>;
+	status = "okay";
+};
+
+&gpu {
+	mali-supply = <&vdd_gpu_s0>;
+	status = "okay";
+};
+
+&hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB1 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&hdmi0_in {
+	hdmi0_in_vp0: endpoint {
+		remote-endpoint = <&vp0_out_hdmi0>;
+	};
+};
+
+&hdmi0_out {
+	hdmi0_out_con: endpoint {
+		remote-endpoint = <&hdmi0_con_in>;
+	};
+};
+
+&hdmi0_sound {
+	status = "okay";
+};
+
+&hdmi_receiver_cma {
+	status = "okay";
+};
+
+&hdmi_receiver {
+	hpd-gpios = <&gpio1 RK_PD5 GPIO_ACTIVE_LOW>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&hdmim1_rx_cec &hdmim1_rx_hpdin &hdmim1_rx_scl &hdmim1_rx_sda &hdmirx_5v_det>;
+	status = "okay";
+};
+
+&hdptxphy0 {
+	status = "okay";
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0m2_xfer>;
+	status = "okay";
+
+	vdd_cpu_big0_s0: regulator@42 {
+		compatible = "rockchip,rk8602";
+		reg = <0x42>;
+		fcs,suspend-voltage-selector = <1>;
+		regulator-name = "vdd_cpu_big0_s0";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <550000>;
+		regulator-max-microvolt = <1050000>;
+		regulator-ramp-delay = <2300>;
+		vin-supply = <&vcc5v0_sys>;
+
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+
+	vdd_cpu_big1_s0: regulator@43 {
+		compatible = "rockchip,rk8603", "rockchip,rk8602";
+		reg = <0x43>;
+		fcs,suspend-voltage-selector = <1>;
+		regulator-name = "vdd_cpu_big1_s0";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <550000>;
+		regulator-max-microvolt = <1050000>;
+		regulator-ramp-delay = <2300>;
+		vin-supply = <&vcc5v0_sys>;
+
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1m2_xfer>;
+	status = "okay";
+
+	vdd_npu_s0: regulator@42 {
+		compatible = "rockchip,rk8602";
+		reg = <0x42>;
+		fcs,suspend-voltage-selector = <1>;
+		regulator-name = "vdd_npu_s0";
+		regulator-boot-on;
+		regulator-min-microvolt = <550000>;
+		regulator-max-microvolt = <950000>;
+		regulator-ramp-delay = <2300>;
+		vin-supply = <&vcc5v0_sys>;
+
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+};
+
+&i2c6 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c6m0_xfer>;
+	status = "okay";
+
+	usbc0: usb-typec@22 {
+		compatible = "fcs,fusb302";
+		reg = <0x22>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PD3 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&usbc0_int>;
+		vbus-supply = <&vbus5v0_typec>;
+
+		usb_con: connector {
+			compatible = "usb-c-connector";
+			data-role = "dual";
+			label = "USB-C";
+			power-role = "source";
+			source-pdos = <PDO_FIXED(5000, 1500, PDO_FIXED_USB_COMM)>;
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					usbc0_hs: endpoint {
+						remote-endpoint = <&usb_host0_xhci_drd_sw>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					usbc0_ss: endpoint {
+						remote-endpoint = <&usbdp_phy0_typec_ss>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					usbc0_sbu: endpoint {
+						remote-endpoint = <&usbdp_phy0_typec_sbu>;
+					};
+				};
+			};
+		};
+	};
+
+	hym8563: rtc@51 {
+		compatible = "haoyu,hym8563";
+		reg = <0x51>;
+		#clock-cells = <0>;
+		clock-output-names = "hym8563";
+	};
+};
+
+&i2c7 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c7m0_xfer>;
+	status = "okay";
+
+	es8388: audio-codec@11 {
+		compatible = "everest,es8388", "everest,es8328";
+		reg = <0x11>;
+		assigned-clocks = <&cru I2S0_8CH_MCLKOUT>;
+		assigned-clock-rates = <12288000>;
+		clocks = <&cru I2S0_8CH_MCLKOUT>;
+		AVDD-supply = <&vcc3v3_sys>;
+		DVDD-supply = <&vcc_1v8_s3>;
+		HPVDD-supply = <&vcc3v3_sys>;
+		PVDD-supply = <&vcc_1v8_s3>;
+		#sound-dai-cells = <0>;
+	};
+};
+
+&i2s0_8ch {
+	pinctrl-0 = <&i2s0_lrck
+		     &i2s0_mclk
+		     &i2s0_sclk
+		     &i2s0_sdi0
+		     &i2s0_sdo0>;
+	status = "okay";
+};
+
+&i2s5_8ch {
+	status = "okay";
+};
+
+&i2s7_8ch {
+	status = "okay";
+};
+
+&mdio1 {
+	rgmii_phy: ethernet-phy@1 {
+		/* RTL8211F */
+		compatible = "ethernet-phy-id001c.c916";
+		reg = <0x1>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&rtl8211f_rst>;
+		reset-assert-us = <20000>;
+		reset-deassert-us = <100000>;
+		reset-gpios = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&package_thermal {
+	polling-delay = <1000>;
+
+	trips {
+		package_fan0: package-fan0 {
+			temperature = <55000>;
+			hysteresis = <2000>;
+			type = "active";
+		};
+
+		package_fan1: package-fan1 {
+			temperature = <65000>;
+			hysteresis = <2000>;
+			type = "active";
+		};
+	};
+
+	cooling-maps {
+		map0 {
+			trip = <&package_fan0>;
+			cooling-device = <&fan THERMAL_NO_LIMIT 1>;
+		};
+
+		map1 {
+			trip = <&package_fan1>;
+			cooling-device = <&fan 2 THERMAL_NO_LIMIT>;
+		};
+	};
+};
+
+&pcie2x1l0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie2_0_rst>;
+	reset-gpios = <&gpio4 RK_PA5 GPIO_ACTIVE_HIGH>;
+	vpcie3v3-supply = <&vcc3v3_sys>;
+	status = "okay";
+};
+
+&pcie2x1l1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie2_1_rst>;
+	reset-gpios = <&gpio4 RK_PA2 GPIO_ACTIVE_HIGH>;
+	vpcie3v3-supply = <&vcc3v3_sys>;
+	status = "okay";
+
+	pcie@0,0 {
+		reg = <0x300000 0 0 0 0>;
+		#address-cells = <3>;
+		#size-cells = <2>;
+		ranges;
+		device_type = "pci";
+		bus-range = <0x30 0x3f>;
+
+		r8125: ethernet@0,0 {
+			reg = <0x310000 0 0 0 0>;
+		};
+	};
+};
+
+&pcie30phy {
+	status = "okay";
+};
+
+&pcie3x4 {
+	clocks = <&cru ACLK_PCIE_4L_MSTR>, <&cru ACLK_PCIE_4L_SLV>,
+		 <&cru ACLK_PCIE_4L_DBI>, <&cru PCLK_PCIE_4L>,
+		 <&cru CLK_PCIE_AUX0>, <&cru CLK_PCIE4L_PIPE>,
+		 <&pcie30_port0_refclk>;
+	clock-names = "aclk_mst", "aclk_slv",
+		      "aclk_dbi", "pclk",
+		      "aux", "pipe",
+		      "ref";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie3x4_perstn>;
+	reset-gpios = <&gpio1 RK_PB4 GPIO_ACTIVE_HIGH>;
+	vpcie3v3-supply = <&vcc3v3_sys>;
+	status = "okay";
+};
+
+&pinctrl {
+	hdmirx {
+		hdmirx_5v_det: hdmirx-5v-det {
+			rockchip,pins = <1 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	leds {
+		led_pins: led-pins {
+			rockchip,pins = <1 RK_PD6 RK_FUNC_GPIO &pcfg_pull_none>,
+					<1 RK_PD7 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	pcie {
+		pcie2_0_rst: pcie2-0-rst {
+			rockchip,pins = <4 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		pcie2_1_rst: pcie2-1-rst {
+			rockchip,pins = <4 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		pi6c_05_pwren: pi6c-05-pwren {
+			rockchip,pins = <2 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		pcie3x4_perstn: pcie3x4-perstn {
+			rockchip,pins = <1 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	rtl8211f {
+		rtl8211f_rst: rtl8211f-rst {
+			rockchip,pins = <3 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	sound {
+		hp_det: hp-det {
+			rockchip,pins = <1 RK_PC4 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+
+		spk_en: spk-en {
+			rockchip,pins = <4 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	usb {
+		typec5v_pwren: typec5v-pwren {
+			rockchip,pins = <4 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		usbc0_int: usbc0-int {
+			rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+};
+
+&pwm2 {
+	pinctrl-0 = <&pwm2m2_pins>;
+	status = "okay";
+};
+
+&rknn_core_0 {
+	npu-supply = <&vdd_npu_s0>;
+	sram-supply = <&vdd_npu_s0>;
+	status = "okay";
+};
+
+&rknn_core_1 {
+	npu-supply = <&vdd_npu_s0>;
+	sram-supply = <&vdd_npu_s0>;
+	status = "okay";
+};
+
+&rknn_core_2 {
+	npu-supply = <&vdd_npu_s0>;
+	sram-supply = <&vdd_npu_s0>;
+	status = "okay";
+};
+
+&rknn_mmu_0 {
+	status = "okay";
+};
+
+&rknn_mmu_1 {
+	status = "okay";
+};
+
+&rknn_mmu_2 {
+	status = "okay";
+};
+
+&saradc {
+	vref-supply = <&avcc_1v8_s0>;
+	status = "okay";
+};
+
+&sata0 {
+	status = "okay";
+};
+
+&sdhci {
+	bus-width = <8>;
+	mmc-hs400-1_8v;
+	mmc-hs400-enhanced-strobe;
+	no-sd;
+	no-sdio;
+	non-removable;
+	status = "okay";
+};
+
+&sdmmc {
+	bus-width = <4>;
+	cap-mmc-highspeed;
+	cap-sd-highspeed;
+	disable-wp;
+	max-frequency = <150000000>;
+	no-mmc;
+	no-sdio;
+	sd-uhs-sdr104;
+	vmmc-supply = <&vcc3v3_sd_s0>;
+	vqmmc-supply = <&vccio_sd_s0>;
+	status = "okay";
+};
+
+&spi2 {
+	assigned-clocks = <&cru CLK_SPI2>;
+	assigned-clock-rates = <200000000>;
+	num-cs = <1>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi2m2_cs0 &spi2m2_pins>;
+	status = "okay";
+
+	pmic@0 {
+		compatible = "rockchip,rk806";
+		reg = <0x0>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <7 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pmic_pins>, <&rk806_dvs1_null>,
+			    <&rk806_dvs2_null>, <&rk806_dvs3_null>;
+		spi-max-frequency = <1000000>;
+		system-power-controller;
+
+		vcc1-supply = <&vcc5v0_sys>;
+		vcc2-supply = <&vcc5v0_sys>;
+		vcc3-supply = <&vcc5v0_sys>;
+		vcc4-supply = <&vcc5v0_sys>;
+		vcc5-supply = <&vcc5v0_sys>;
+		vcc6-supply = <&vcc5v0_sys>;
+		vcc7-supply = <&vcc5v0_sys>;
+		vcc8-supply = <&vcc5v0_sys>;
+		vcc9-supply = <&vcc5v0_sys>;
+		vcc10-supply = <&vcc5v0_sys>;
+		vcc11-supply = <&vcc_2v0_pldo_s3>;
+		vcc12-supply = <&vcc5v0_sys>;
+		vcc13-supply = <&vcc_1v1_nldo_s3>;
+		vcc14-supply = <&vcc_1v1_nldo_s3>;
+		vcca-supply = <&vcc5v0_sys>;
+
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		rk806_dvs1_null: dvs1-null-pins {
+			pins = "gpio_pwrctrl1";
+			function = "pin_fun0";
+		};
+
+		rk806_dvs2_null: dvs2-null-pins {
+			pins = "gpio_pwrctrl2";
+			function = "pin_fun0";
+		};
+
+		rk806_dvs3_null: dvs3-null-pins {
+			pins = "gpio_pwrctrl3";
+			function = "pin_fun0";
+		};
+
+		regulators {
+			vdd_gpu_s0: vdd_gpu_mem_s0: dcdc-reg1 {
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <950000>;
+				regulator-ramp-delay = <12500>;
+				regulator-name = "vdd_gpu_s0";
+				regulator-enable-ramp-delay = <400>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_cpu_lit_s0: vdd_cpu_lit_mem_s0: dcdc-reg2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <950000>;
+				regulator-ramp-delay = <12500>;
+				regulator-name = "vdd_cpu_lit_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_log_s0: dcdc-reg3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <675000>;
+				regulator-max-microvolt = <750000>;
+				regulator-ramp-delay = <12500>;
+				regulator-name = "vdd_log_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <750000>;
+				};
+			};
+
+			vdd_vdenc_s0: vdd_vdenc_mem_s0: dcdc-reg4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <950000>;
+				regulator-ramp-delay = <12500>;
+				regulator-name = "vdd_vdenc_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_ddr_s0: dcdc-reg5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <675000>;
+				regulator-max-microvolt = <900000>;
+				regulator-ramp-delay = <12500>;
+				regulator-name = "vdd_ddr_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <850000>;
+				};
+			};
+
+			vdd2_ddr_s3: dcdc-reg6 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-name = "vdd2_ddr_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			vcc_2v0_pldo_s3: dcdc-reg7 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <2000000>;
+				regulator-max-microvolt = <2000000>;
+				regulator-ramp-delay = <12500>;
+				regulator-name = "vdd_2v0_pldo_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <2000000>;
+				};
+			};
+
+			vcc_3v3_s3: dcdc-reg8 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc_3v3_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <3300000>;
+				};
+			};
+
+			vddq_ddr_s0: dcdc-reg9 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-name = "vddq_ddr_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_1v8_s3: dcdc-reg10 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcc_1v8_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			avcc_1v8_s0: pldo-reg1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "avcc_1v8_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vcc_1v8_s0: pldo-reg2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcc_1v8_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			avdd_1v2_s0: pldo-reg3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-name = "avdd_1v2_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_3v3_s0: pldo-reg4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-ramp-delay = <12500>;
+				regulator-name = "vcc_3v3_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vccio_sd_s0: pldo-reg5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-ramp-delay = <12500>;
+				regulator-name = "vccio_sd_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			pldo6_s3: pldo-reg6 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "pldo6_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vdd_0v75_s3: nldo-reg1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt = <750000>;
+				regulator-name = "vdd_0v75_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <750000>;
+				};
+			};
+
+			vdd_ddr_pll_s0: nldo-reg2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <850000>;
+				regulator-name = "vdd_ddr_pll_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <850000>;
+				};
+			};
+
+			avdd_0v75_s0: nldo-reg3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt = <750000>;
+				regulator-name = "avdd_0v75_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_0v85_s0: nldo-reg4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <850000>;
+				regulator-name = "vdd_0v85_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_0v75_s0: nldo-reg5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt = <750000>;
+				regulator-name = "vdd_0v75_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+		};
+	};
+};
+
+&tsadc {
+	status = "okay";
+};
+
+&u2phy0 {
+	status = "okay";
+};
+
+&u2phy0_otg {
+	status = "okay";
+};
+
+&u2phy1 {
+	status = "okay";
+};
+
+&u2phy1_otg {
+	status = "okay";
+};
+
+&u2phy2 {
+	status = "okay";
+};
+
+&u2phy2_host {
+	phy-supply = <&vcc5v0_sys>;
+	status = "okay";
+};
+
+&u2phy3 {
+	status = "okay";
+};
+
+&u2phy3_host {
+	phy-supply = <&vcc5v0_sys>;
+	status = "okay";
+};
+
+&uart1 {
+	pinctrl-0 = <&uart1m0_xfer>;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-0 = <&uart2m0_xfer>;
+	status = "okay";
+};
+
+&uart6 {
+	pinctrl-0 = <&uart6m0_xfer>;
+	status = "okay";
+};
+
+&uart7 {
+	pinctrl-0 = <&uart7m0_xfer>;
+	status = "okay";
+};
+
+&uart9 {
+	pinctrl-0 = <&uart9m0_xfer>;
+	status = "okay";
+};
+
+&usb_host0_ehci {
+	status = "okay";
+};
+
+&usb_host0_ohci {
+	status = "okay";
+};
+
+&usb_host0_xhci {
+	usb-role-switch;
+	status = "okay";
+
+	port {
+		usb_host0_xhci_drd_sw: endpoint {
+			remote-endpoint = <&usbc0_hs>;
+		};
+	};
+};
+
+&usb_host1_ehci {
+	status = "okay";
+};
+
+&usb_host1_ohci {
+	status = "okay";
+};
+
+&usb_host1_xhci {
+	dr_mode = "host";
+	status = "okay";
+};
+
+&usbdp_phy0 {
+	mode-switch;
+	orientation-switch;
+	sbu1-dc-gpios = <&gpio4 RK_PA6 GPIO_ACTIVE_HIGH>;
+	sbu2-dc-gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_HIGH>;
+	status = "okay";
+
+	port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		usbdp_phy0_typec_ss: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&usbc0_ss>;
+		};
+
+		usbdp_phy0_typec_sbu: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&usbc0_sbu>;
+		};
+	};
+};
+
+&usbdp_phy1 {
+	status = "okay";
+};
+
+&vop {
+	status = "okay";
+};
+
+&vop_mmu {
+	status = "okay";
+};
+
+&vp0 {
+	vp0_out_hdmi0: endpoint@ROCKCHIP_VOP2_EP_HDMI0 {
+		reg = <ROCKCHIP_VOP2_EP_HDMI0>;
+		remote-endpoint = <&hdmi0_in_vp0>;
+	};
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5-5b.dtsi b/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5-5b.dtsi
new file mode 100644
index 0000000000000..6e081bc06cfe0
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5-5b.dtsi
@@ -0,0 +1,256 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Device tree definitions shared by the Orange Pi 5 and Orange Pi 5B
+ * but not the Orange Pi 5 Pro.
+ */
+
+#include <dt-bindings/usb/pd.h>
+#include "rk3588s-orangepi-5.dtsi"
+
+/ {
+	aliases {
+		ethernet0 = &gmac1;
+	};
+
+	analog-sound {
+		compatible = "simple-audio-card";
+		pinctrl-names = "default";
+		pinctrl-0 = <&hp_detect>;
+		simple-audio-card,name = "rockchip,es8388";
+		simple-audio-card,bitclock-master = <&masterdai>;
+		simple-audio-card,format = "i2s";
+		simple-audio-card,frame-master = <&masterdai>;
+		simple-audio-card,hp-det-gpios = <&gpio1 RK_PD5 GPIO_ACTIVE_HIGH>;
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,routing =
+			"Headphones", "LOUT1",
+			"Headphones", "ROUT1",
+			"LINPUT1", "Microphone Jack",
+			"RINPUT1", "Microphone Jack",
+			"LINPUT2", "Onboard Microphone",
+			"RINPUT2", "Onboard Microphone";
+		simple-audio-card,widgets =
+			"Microphone", "Microphone Jack",
+			"Microphone", "Onboard Microphone",
+			"Headphone", "Headphones";
+
+		simple-audio-card,cpu {
+			sound-dai = <&i2s1_8ch>;
+		};
+
+		masterdai: simple-audio-card,codec {
+			sound-dai = <&es8388>;
+			system-clock-frequency = <12288000>;
+		};
+	};
+
+	pwm-leds {
+		compatible = "pwm-leds";
+
+		led {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_STATUS;
+			linux,default-trigger = "heartbeat";
+			max-brightness = <255>;
+			pwms = <&pwm0 0 25000 0>;
+		};
+	};
+
+	vbus_typec: regulator-vbus-typec {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio3 RK_PC0 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&typec5v_pwren>;
+		regulator-name = "vbus_typec";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&vcc5v0_sys>;
+	};
+};
+
+&gmac1 {
+	clock_in_out = "output";
+	phy-handle = <&rgmii_phy1>;
+	phy-mode = "rgmii-rxid";
+	pinctrl-0 = <&gmac1_miim
+			&gmac1_tx_bus2
+			&gmac1_rx_bus2
+			&gmac1_rgmii_clk
+			&gmac1_rgmii_bus>;
+	pinctrl-names = "default";
+	tx_delay = <0x42>;
+	status = "okay";
+};
+
+&i2c6 {
+	es8388: audio-codec@10 {
+		compatible = "everest,es8388", "everest,es8328";
+		reg = <0x10>;
+		clocks = <&cru I2S1_8CH_MCLKOUT>;
+		AVDD-supply = <&vcca_3v3_s0>;
+		DVDD-supply = <&vcca_1v8_s0>;
+		HPVDD-supply = <&vcca_3v3_s0>;
+		PVDD-supply = <&vcca_3v3_s0>;
+		assigned-clocks = <&cru I2S1_8CH_MCLKOUT>;
+		assigned-clock-rates = <12288000>;
+		#sound-dai-cells = <0>;
+	};
+
+	usbc0: usb-typec@22 {
+		compatible = "fcs,fusb302";
+		reg = <0x22>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PD3 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&usbc0_int>;
+		vbus-supply = <&vbus_typec>;
+		status = "okay";
+
+		usb_con: connector {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			data-role = "dual";
+			op-sink-microwatt = <1000000>;
+			power-role = "dual";
+			sink-pdos =
+				<PDO_FIXED(5000, 1000, PDO_FIXED_USB_COMM)>;
+			source-pdos =
+				<PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
+			try-power-role = "source";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					usbc0_hs: endpoint {
+						remote-endpoint = <&usb_host0_xhci_drd_sw>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+					usbc0_ss: endpoint {
+						remote-endpoint = <&usbdp_phy0_typec_ss>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+					usbc0_sbu: endpoint {
+						remote-endpoint = <&usbdp_phy0_typec_sbu>;
+					};
+				};
+			};
+		};
+	};
+};
+
+&mdio1 {
+	rgmii_phy1: ethernet-phy@1 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <0x1>;
+		reset-assert-us = <20000>;
+		reset-deassert-us = <100000>;
+		reset-gpios = <&gpio3 RK_PB2 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&pinctrl {
+	usb-typec {
+		usbc0_int: usbc0-int {
+			rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+
+		typec5v_pwren: typec5v-pwren {
+			rockchip,pins = <3 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+};
+
+&i2s1_8ch {
+	rockchip,i2s-tx-route = <3 2 1 0>;
+	rockchip,i2s-rx-route = <1 3 2 0>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2s1m0_sclk
+			   &i2s1m0_mclk
+			   &i2s1m0_lrck
+			   &i2s1m0_sdi1
+			   &i2s1m0_sdo3>;
+	status = "okay";
+};
+
+&pwm0 {
+	pinctrl-0 = <&pwm0m2_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&rk806 {
+	regulators {
+		vcc_1v8_s0: pldo-reg1 {
+			regulator-name = "vcc_1v8_s0";
+			regulator-always-on;
+			regulator-boot-on;
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+
+			regulator-state-mem {
+				regulator-off-in-suspend;
+			};
+		};
+
+		vcca_1v8_s0: pldo-reg2 {
+			regulator-name = "vcca_1v8_s0";
+			regulator-always-on;
+			regulator-boot-on;
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+
+			regulator-state-mem {
+				regulator-off-in-suspend;
+				regulator-suspend-microvolt = <1800000>;
+			};
+		};
+	};
+};
+
+
+&usb_host0_xhci {
+	dr_mode = "otg";
+	usb-role-switch;
+
+	port {
+		usb_host0_xhci_drd_sw: endpoint {
+			remote-endpoint = <&usbc0_hs>;
+		};
+	};
+};
+
+&usb_host2_xhci {
+	status = "okay";
+};
+
+&usbdp_phy0 {
+	mode-switch;
+	orientation-switch;
+	sbu1-dc-gpios = <&gpio4 RK_PA5 GPIO_ACTIVE_HIGH>;
+	sbu2-dc-gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_HIGH>;
+
+	port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		usbdp_phy0_typec_ss: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&usbc0_ss>;
+		};
+
+		usbdp_phy0_typec_sbu: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&usbc0_sbu>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5-pro.dts b/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5-pro.dts
new file mode 100644
index 0000000000000..c9679bac1db73
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5-pro.dts
@@ -0,0 +1,358 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+
+/dts-v1/;
+
+#include "rk3588s-orangepi-5.dtsi"
+
+/ {
+	model = "Xunlong Orange Pi 5 Pro";
+	compatible = "xunlong,orangepi-5-pro", "rockchip,rk3588s";
+
+	aliases {
+		mmc0 = &sdhci;
+		mmc1 = &sdmmc;
+		mmc2 = &sdio;
+	};
+
+	analog-sound {
+		compatible = "simple-audio-card";
+		pinctrl-names = "default";
+		pinctrl-0 = <&hp_detect>;
+		simple-audio-card,format = "i2s";
+		simple-audio-card,hp-det-gpios = <&gpio1 RK_PD5 GPIO_ACTIVE_HIGH>;
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,name = "rockchip,es8388";
+		simple-audio-card,routing =
+			"Headphones", "LOUT1",
+			"Headphones", "ROUT1",
+			"LINPUT1", "Microphone Jack",
+			"RINPUT1", "Microphone Jack",
+			"LINPUT2", "Onboard Microphone",
+			"RINPUT2", "Onboard Microphone";
+		simple-audio-card,widgets =
+			"Microphone", "Microphone Jack",
+			"Microphone", "Onboard Microphone",
+			"Headphone", "Headphones";
+
+		simple-audio-card,cpu {
+			sound-dai = <&i2s2_2ch>;
+		};
+
+		simple-audio-card,codec {
+			sound-dai = <&es8388>;
+			system-clock-frequency = <12288000>;
+		};
+	};
+
+	pwm-leds {
+		compatible = "pwm-leds";
+
+		led-0 {
+			color = <LED_COLOR_ID_BLUE>;
+			function = LED_FUNCTION_STATUS;
+			linux,default-trigger = "heartbeat";
+			max-brightness = <255>;
+			pwms = <&pwm15 0 1000000 0>;
+		};
+
+		led-1 {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_ACTIVITY;
+			linux,default-trigger = "heartbeat";
+			max-brightness = <255>;
+			pwms = <&pwm3 0 1000000 0>;
+		};
+	};
+
+	fan: pwm-fan {
+		compatible = "pwm-fan";
+		#cooling-cells = <2>;
+		cooling-levels = <0 50 100 150 200 255>;
+		fan-supply = <&vcc5v0_sys>;
+		pwms = <&pwm2 0 20000000 0>;
+	};
+
+	vcc3v3_eth: regulator-vcc3v3-eth {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio3 RK_PB7 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&ethernet_en>;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "vcc3v3_eth";
+		startup-delay-us = <50000>;
+		vin-supply = <&vcc_3v3_s3>;
+	};
+
+	vcc5v0_otg: regulator-vcc5v0-otg {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio0 RK_PC4 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&vcc5v0_otg_en>;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "vcc5v0_otg";
+		vin-supply = <&vcc5v0_sys>;
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		clocks = <&hym8563>;
+		clock-names = "ext_clock";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wifi_enable_h>;
+		post-power-on-delay-ms = <200>;
+		reset-gpios = <&gpio0 RK_PD0 GPIO_ACTIVE_LOW>;
+	};
+
+	typea_con: usb-a-connector {
+		compatible = "usb-a-connector";
+		data-role = "host";
+		label = "USB3 Type-A";
+		power-role = "source";
+		vbus-supply = <&vcc5v0_otg>;
+	};
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1m4_xfer>;
+	status = "okay";
+};
+
+&i2c3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c3m0_xfer>;
+	status = "okay";
+
+	es8388: audio-codec@11 {
+		compatible = "everest,es8388", "everest,es8328";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		AVDD-supply = <&vcca_3v3_s0>;
+		DVDD-supply = <&vcca_1v8_s0>;
+		HPVDD-supply = <&vcca_3v3_s0>;
+		PVDD-supply = <&vcca_1v8_s0>;
+		assigned-clock-rates = <12288000>;
+		assigned-clocks = <&cru I2S2_2CH_MCLKOUT>;
+		clocks = <&cru I2S2_2CH_MCLKOUT>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&i2s2m1_mclk>;
+	};
+};
+
+&i2c4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c4m3_xfer>;
+	status = "okay";
+};
+
+&i2s2_2ch {
+	pinctrl-0 = <&i2s2m1_lrck &i2s2m1_sclk
+		     &i2s2m1_sdi &i2s2m1_sdo>;
+	status = "okay";
+};
+
+&package_thermal {
+	polling-delay = <1000>;
+
+	cooling-maps {
+		map0 {
+			trip = <&package_fan0>;
+			cooling-device = <&fan THERMAL_NO_LIMIT 1>;
+		};
+
+		map1 {
+			trip = <&package_fan1>;
+			cooling-device = <&fan 2 THERMAL_NO_LIMIT>;
+		};
+	};
+
+	trips {
+		package_fan0: package-fan0 {
+			hysteresis = <2000>;
+			temperature = <55000>;
+			type = "active";
+		};
+
+		package_fan1: package-fan1 {
+			hysteresis = <2000>;
+			temperature = <65000>;
+			type = "active";
+		};
+	};
+};
+
+/* NVMe */
+&pcie2x1l1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie30x1m1_1_perstn>, <&pcie30x1m1_1_clkreqn>,
+		    <&pcie30x1m1_1_waken>;
+	reset-gpios = <&gpio4 RK_PA2 GPIO_ACTIVE_HIGH>;
+	supports-clkreq;
+	vpcie3v3-supply = <&vcc_3v3_s3>;
+	status = "okay";
+};
+
+/* NIC */
+&pcie2x1l2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie20x1m0_perstn>, <&pcie20x1m0_clkreqn>,
+		    <&pcie20x1m0_waken>;
+	reset-gpios = <&gpio3 RK_PD1 GPIO_ACTIVE_HIGH>;
+	vpcie3v3-supply = <&vcc3v3_eth>;
+	status = "okay";
+};
+
+&pinctrl {
+	bluetooth {
+		bt_wake_gpio: bt-wake-pin {
+			rockchip,pins = <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+
+		bt_wake_host_irq: bt-wake-host-irq {
+			rockchip,pins = <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+	};
+
+	ethernet {
+		ethernet_en: ethernet-en {
+			rockchip,pins = <3 RK_PB7 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+	};
+
+	usb {
+		vcc5v0_otg_en: vcc5v0-otg-en {
+			rockchip,pins = <0 RK_PC4 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+	};
+
+	wlan {
+		wifi_enable_h: wifi-enable-h {
+			rockchip,pins = <0 RK_PD0 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+
+		wifi_host_wake_irq: wifi-host-wake-irq {
+			rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+	};
+};
+
+&pwm15 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm15m2_pins>;
+	status = "okay";
+};
+
+&pwm2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm2m1_pins>;
+	status = "okay";
+};
+
+&pwm3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm3m2_pins>;
+	status = "okay";
+};
+
+&rk806 {
+	regulators {
+		vcca_1v8_s0: pldo-reg1 {
+			regulator-name = "vcca_1v8_s0";
+			regulator-always-on;
+			regulator-boot-on;
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+
+			regulator-state-mem {
+				regulator-off-in-suspend;
+				regulator-suspend-microvolt = <1800000>;
+			};
+		};
+
+		vcc_1v8_s0: pldo-reg2 {
+			regulator-name = "vcc_1v8_s0";
+			regulator-always-on;
+			regulator-boot-on;
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+
+			regulator-state-mem {
+				regulator-off-in-suspend;
+			};
+		};
+	};
+};
+
+&sdhci {
+	status = "okay";
+};
+
+&sdio {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	bus-width = <4>;
+	cap-sd-highspeed;
+	cap-sdio-irq;
+	keep-power-in-suspend;
+	max-frequency = <150000000>;
+	mmc-pwrseq = <&sdio_pwrseq>;
+	no-mmc;
+	no-sd;
+	non-removable;
+	sd-uhs-sdr104;
+	status = "okay";
+
+	ap6256: wifi@1 {
+		compatible = "brcm,bcm43456-fmac", "brcm,bcm4329-fmac";
+		reg = <1>;
+		interrupt-names = "host-wake";
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PA0 IRQ_TYPE_LEVEL_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&wifi_host_wake_irq>;
+	};
+};
+
+&uart9 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart9m2_xfer &uart9m2_ctsn &uart9m2_rtsn>;
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "brcm,bcm4345c5";
+		clocks = <&hym8563>;
+		clock-names = "lpo";
+		device-wakeup-gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>;
+		interrupt-names = "host-wakeup";
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PC5 IRQ_TYPE_LEVEL_HIGH>;
+		max-speed = <1500000>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&bt_wake_host_irq &bt_wake_gpio>;
+		shutdown-gpios = <&gpio0 RK_PD5 GPIO_ACTIVE_HIGH>;
+		vbat-supply = <&vcc_3v3_s3>;
+		vddio-supply = <&vcc_1v8_s3>;
+	};
+};
+
+&u2phy0_otg {
+	phy-supply = <&vcc5v0_otg>;
+};
+
+&usb_host0_xhci {
+	dr_mode = "host";
+};
+
+&usbdp_phy0 {
+	/*
+	 * The USB3 Type-A host connector is wired to PHY lanes 2/3. Lanes 0/1
+	 * are connected to the LT8711UXD DP-to-HDMI bridge on the board.
+	 * dp-lane-mux routes USB3 SuperSpeed to the correct lanes.
+	 */
+	rockchip,dp-lane-mux = <0 1>;
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5.dts b/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5.dts
index 83b9b6645a1e4..d76bdf1b5e90e 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5.dts
@@ -2,12 +2,16 @@
 
 /dts-v1/;
 
-#include "rk3588s-orangepi-5.dtsi"
+#include "rk3588s-orangepi-5-5b.dtsi"
 
 / {
 	model = "Xunlong Orange Pi 5";
 	compatible = "xunlong,orangepi-5", "rockchip,rk3588s";
 
+	aliases {
+		mmc0 = &sdmmc;
+	};
+
 	vcc3v3_pcie20: regulator-vcc3v3-pcie20 {
 		compatible = "regulator-fixed";
 		enable-active-high;
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5.dtsi b/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5.dtsi
index 9da13f96f13a1..fd0efa8d1bfbb 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5.dtsi
@@ -3,19 +3,13 @@
 /dts-v1/;
 
 #include <dt-bindings/gpio/gpio.h>
-#include <dt-bindings/leds/common.h>
 #include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
 #include <dt-bindings/pinctrl/rockchip.h>
 #include <dt-bindings/soc/rockchip,vop2.h>
-#include <dt-bindings/usb/pd.h>
 #include "rk3588s.dtsi"
 
 / {
-	aliases {
-		ethernet0 = &gmac1;
-		mmc0 = &sdmmc;
-	};
-
 	chosen {
 		stdout-path = "serial2:1500000n8";
 	};
@@ -34,38 +28,6 @@ button-recovery {
 		};
 	};
 
-	analog-sound {
-		compatible = "simple-audio-card";
-		pinctrl-names = "default";
-		pinctrl-0 = <&hp_detect>;
-		simple-audio-card,name = "rockchip,es8388";
-		simple-audio-card,bitclock-master = <&masterdai>;
-		simple-audio-card,format = "i2s";
-		simple-audio-card,frame-master = <&masterdai>;
-		simple-audio-card,hp-det-gpios = <&gpio1 RK_PD5 GPIO_ACTIVE_HIGH>;
-		simple-audio-card,mclk-fs = <256>;
-		simple-audio-card,routing =
-			"Headphones", "LOUT1",
-			"Headphones", "ROUT1",
-			"LINPUT1", "Microphone Jack",
-			"RINPUT1", "Microphone Jack",
-			"LINPUT2", "Onboard Microphone",
-			"RINPUT2", "Onboard Microphone";
-		simple-audio-card,widgets =
-			"Microphone", "Microphone Jack",
-			"Microphone", "Onboard Microphone",
-			"Headphone", "Headphones";
-
-		simple-audio-card,cpu {
-			sound-dai = <&i2s1_8ch>;
-		};
-
-		masterdai: simple-audio-card,codec {
-			sound-dai = <&es8388>;
-			system-clock-frequency = <12288000>;
-		};
-	};
-
 	hdmi0-con {
 		compatible = "hdmi-connector";
 		type = "a";
@@ -77,28 +39,14 @@ hdmi0_con_in: endpoint {
 		};
 	};
 
-	pwm-leds {
-		compatible = "pwm-leds";
-
-		led {
-			color = <LED_COLOR_ID_GREEN>;
-			function = LED_FUNCTION_STATUS;
-			linux,default-trigger = "heartbeat";
-			max-brightness = <255>;
-			pwms = <&pwm0 0 25000 0>;
-		};
-	};
-
-	vbus_typec: regulator-vbus-typec {
+	vcc_3v3_sd_s0: regulator-vcc-3v3-sd-s0 {
 		compatible = "regulator-fixed";
-		enable-active-high;
-		gpio = <&gpio3 RK_PC0 GPIO_ACTIVE_HIGH>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&typec5v_pwren>;
-		regulator-name = "vbus_typec";
-		regulator-min-microvolt = <5000000>;
-		regulator-max-microvolt = <5000000>;
-		vin-supply = <&vcc5v0_sys>;
+		gpios = <&gpio4 RK_PB5 GPIO_ACTIVE_LOW>;
+		regulator-name = "vcc_3v3_sd_s0";
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc_3v3_s3>;
 	};
 
 	vcc5v0_sys: regulator-vcc5v0-sys {
@@ -109,16 +57,6 @@ vcc5v0_sys: regulator-vcc5v0-sys {
 		regulator-min-microvolt = <5000000>;
 		regulator-max-microvolt = <5000000>;
 	};
-
-	vcc_3v3_sd_s0: regulator-vcc-3v3-sd-s0 {
-		compatible = "regulator-fixed";
-		gpios = <&gpio4 RK_PB5 GPIO_ACTIVE_LOW>;
-		regulator-name = "vcc_3v3_sd_s0";
-		regulator-boot-on;
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		vin-supply = <&vcc_3v3_s3>;
-	};
 };
 
 &combphy0_ps {
@@ -161,20 +99,6 @@ &cpu_l3 {
 	cpu-supply = <&vdd_cpu_lit_s0>;
 };
 
-&gmac1 {
-	clock_in_out = "output";
-	phy-handle = <&rgmii_phy1>;
-	phy-mode = "rgmii-rxid";
-	pinctrl-0 = <&gmac1_miim
-		     &gmac1_tx_bus2
-		     &gmac1_rx_bus2
-		     &gmac1_rgmii_clk
-		     &gmac1_rgmii_bus>;
-	pinctrl-names = "default";
-	tx_delay = <0x42>;
-	status = "okay";
-};
-
 &gpu {
 	mali-supply = <&vdd_gpu_s0>;
 	status = "okay";
@@ -274,69 +198,6 @@ &i2c6 {
 	pinctrl-0 = <&i2c6m3_xfer>;
 	status = "okay";
 
-	es8388: audio-codec@10 {
-		compatible = "everest,es8388", "everest,es8328";
-		reg = <0x10>;
-		clocks = <&cru I2S1_8CH_MCLKOUT>;
-		AVDD-supply = <&vcc_3v3_s0>;
-		DVDD-supply = <&vcc_1v8_s0>;
-		HPVDD-supply = <&vcc_3v3_s0>;
-		PVDD-supply = <&vcc_3v3_s0>;
-		assigned-clocks = <&cru I2S1_8CH_MCLKOUT>;
-		assigned-clock-rates = <12288000>;
-		#sound-dai-cells = <0>;
-	};
-
-	usbc0: usb-typec@22 {
-		compatible = "fcs,fusb302";
-		reg = <0x22>;
-		interrupt-parent = <&gpio0>;
-		interrupts = <RK_PD3 IRQ_TYPE_LEVEL_LOW>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&usbc0_int>;
-		vbus-supply = <&vbus_typec>;
-		status = "okay";
-
-		usb_con: connector {
-			compatible = "usb-c-connector";
-			label = "USB-C";
-			data-role = "dual";
-			op-sink-microwatt = <1000000>;
-			power-role = "dual";
-			sink-pdos =
-				<PDO_FIXED(5000, 1000, PDO_FIXED_USB_COMM)>;
-			source-pdos =
-				<PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
-			try-power-role = "source";
-
-			ports {
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				port@0 {
-					reg = <0>;
-					usbc0_hs: endpoint {
-						remote-endpoint = <&usb_host0_xhci_drd_sw>;
-					};
-				};
-
-				port@1 {
-					reg = <1>;
-					usbc0_ss: endpoint {
-						remote-endpoint = <&usbdp_phy0_typec_ss>;
-					};
-				};
-
-				port@2 {
-					reg = <2>;
-					usbc0_sbu: endpoint {
-						remote-endpoint = <&usbdp_phy0_typec_sbu>;
-					};
-				};
-			};
-		};
-	};
-
 	hym8563: rtc@51 {
 		compatible = "haoyu,hym8563";
 		reg = <0x51>;
@@ -350,32 +211,10 @@ hym8563: rtc@51 {
 	};
 };
 
-&i2s1_8ch {
-	rockchip,i2s-tx-route = <3 2 1 0>;
-	rockchip,i2s-rx-route = <1 3 2 0>;
-	pinctrl-names = "default";
-	pinctrl-0 = <&i2s1m0_sclk
-	             &i2s1m0_mclk
-	             &i2s1m0_lrck
-	             &i2s1m0_sdi1
-	             &i2s1m0_sdo3>;
-	status = "okay";
-};
-
 &i2s5_8ch {
 	status = "okay";
 };
 
-&mdio1 {
-	rgmii_phy1: ethernet-phy@1 {
-		compatible = "ethernet-phy-ieee802.3-c22";
-		reg = <0x1>;
-		reset-assert-us = <20000>;
-		reset-deassert-us = <100000>;
-		reset-gpios = <&gpio3 RK_PB2 GPIO_ACTIVE_LOW>;
-	};
-};
-
 &pd_gpu {
 	domain-supply = <&vdd_gpu_s0>;
 };
@@ -402,22 +241,6 @@ hp_detect: hp-detect {
 			rockchip,pins = <1 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
 	};
-
-	usb-typec {
-		usbc0_int: usbc0-int {
-			rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_up>;
-		};
-
-		typec5v_pwren: typec5v-pwren {
-			rockchip,pins = <3 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>;
-		};
-	};
-};
-
-&pwm0 {
-	pinctrl-0 = <&pwm0m2_pins>;
-	pinctrl-names = "default";
-	status = "okay";
 };
 
 &rknn_core_0 {
@@ -451,7 +274,7 @@ &rknn_mmu_2 {
 };
 
 &saradc {
-	vref-supply = <&avcc_1v8_s0>;
+	vref-supply = <&vcc_1v8_s0>;
 	status = "okay";
 };
 
@@ -501,7 +324,7 @@ &spi2 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&spi2m2_cs0 &spi2m2_pins>;
 
-	pmic@0 {
+	rk806: pmic@0 {
 		compatible = "rockchip,rk806";
 		reg = <0x0>;
 		interrupt-parent = <&gpio0>;
@@ -676,33 +499,8 @@ regulator-state-mem {
 				};
 			};
 
-			avcc_1v8_s0: pldo-reg1 {
-				regulator-name = "avcc_1v8_s0";
-				regulator-always-on;
-				regulator-boot-on;
-				regulator-min-microvolt = <1800000>;
-				regulator-max-microvolt = <1800000>;
-
-				regulator-state-mem {
-					regulator-off-in-suspend;
-				};
-			};
-
-			vcc_1v8_s0: pldo-reg2 {
-				regulator-name = "vcc_1v8_s0";
-				regulator-always-on;
-				regulator-boot-on;
-				regulator-min-microvolt = <1800000>;
-				regulator-max-microvolt = <1800000>;
-
-				regulator-state-mem {
-					regulator-off-in-suspend;
-					regulator-suspend-microvolt = <1800000>;
-				};
-			};
-
-			avdd_1v2_s0: pldo-reg3 {
-				regulator-name = "avdd_1v2_s0";
+			vdda_1v2_s0: pldo-reg3 {
+				regulator-name = "vdda_1v2_s0";
 				regulator-always-on;
 				regulator-boot-on;
 				regulator-min-microvolt = <1200000>;
@@ -713,8 +511,8 @@ regulator-state-mem {
 				};
 			};
 
-			vcc_3v3_s0: pldo-reg4 {
-				regulator-name = "vcc_3v3_s0";
+			vcca_3v3_s0: pldo-reg4 {
+				regulator-name = "vcca_3v3_s0";
 				regulator-always-on;
 				regulator-boot-on;
 				regulator-min-microvolt = <3300000>;
@@ -852,26 +650,7 @@ &uart2 {
 };
 
 &usbdp_phy0 {
-	mode-switch;
-	orientation-switch;
-	sbu1-dc-gpios = <&gpio4 RK_PA5 GPIO_ACTIVE_HIGH>;
-	sbu2-dc-gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_HIGH>;
 	status = "okay";
-
-	port {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		usbdp_phy0_typec_ss: endpoint@0 {
-			reg = <0>;
-			remote-endpoint = <&usbc0_ss>;
-		};
-
-		usbdp_phy0_typec_sbu: endpoint@1 {
-			reg = <1>;
-			remote-endpoint = <&usbc0_sbu>;
-		};
-	};
 };
 
 &usb_host0_ehci {
@@ -883,15 +662,7 @@ &usb_host0_ohci {
 };
 
 &usb_host0_xhci {
-	dr_mode = "otg";
-	usb-role-switch;
 	status = "okay";
-
-	port {
-		usb_host0_xhci_drd_sw: endpoint {
-			remote-endpoint = <&usbc0_hs>;
-		};
-	};
 };
 
 &usb_host1_ehci {
@@ -902,7 +673,7 @@ &usb_host1_ohci {
 	status = "okay";
 };
 
-&usb_host2_xhci {
+&vop {
 	status = "okay";
 };
 
@@ -910,10 +681,6 @@ &vop_mmu {
 	status = "okay";
 };
 
-&vop {
-	status = "okay";
-};
-
 &vp0 {
 	vp0_out_hdmi0: endpoint@ROCKCHIP_VOP2_EP_HDMI0 {
 		reg = <ROCKCHIP_VOP2_EP_HDMI0>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5b.dts b/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5b.dts
index d21ec320d2950..8af174777809a 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5b.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5b.dts
@@ -2,7 +2,7 @@
 
 /dts-v1/;
 
-#include "rk3588s-orangepi-5.dtsi"
+#include "rk3588s-orangepi-5-5b.dtsi"
 
 / {
 	model = "Xunlong Orange Pi 5B";
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-radxa-cm5-io.dts b/arch/arm64/boot/dts/rockchip/rk3588s-radxa-cm5-io.dts
index af4a9bc015e16..dc4cc44182f95 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-radxa-cm5-io.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-radxa-cm5-io.dts
@@ -19,6 +19,7 @@ / {
 	aliases {
 		ethernet0 = &gmac1;
 		mmc1 = &sdmmc;
+		rtc0 = &hym8563;
 	};
 
 	chosen {
@@ -198,6 +199,18 @@ usbc0_dp_altmode_mux: endpoint {
 			};
 		};
 	};
+
+	hym8563: rtc@51 {
+		compatible = "haoyu,hym8563";
+		reg = <0x51>;
+		#clock-cells = <0>;
+		clock-output-names = "hym8563";
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PB0 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&rtc_int_l>;
+		wakeup-source;
+	};
 };
 
 &i2s5_8ch {
@@ -227,6 +240,12 @@ hdmi0_tx_on_h: hdmi0-tx-on-h {
 		};
 	};
 
+	hym8563 {
+		rtc_int_l: rtc-int-l {
+			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
 	usb {
 		vcc5v0_host_en: vcc5v0-host-en {
 			rockchip,pins = <1 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts b/arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts
index d534d662c40f1..7150d5f7684dc 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts
@@ -23,16 +23,19 @@ analog-sound {
 		compatible = "simple-audio-card";
 		pinctrl-names = "default";
 		pinctrl-0 = <&hp_detect>;
+		simple-audio-card,aux-devs = <&headphones_amp>;
 		simple-audio-card,name = "rockchip,es8388";
-		simple-audio-card,bitclock-master = <&masterdai>;
+		simple-audio-card,bitclock-master = <&cpudai>;
 		simple-audio-card,format = "i2s";
-		simple-audio-card,frame-master = <&masterdai>;
+		simple-audio-card,frame-master = <&cpudai>;
 		simple-audio-card,hp-det-gpios = <&gpio1 RK_PA6 GPIO_ACTIVE_LOW>;
 		simple-audio-card,mclk-fs = <256>;
 		simple-audio-card,pin-switches = "Headphones";
 		simple-audio-card,routing =
-			"Headphones", "LOUT1",
-			"Headphones", "ROUT1",
+			"Headphones", "Headphone Amp OUTL",
+			"Headphones", "Headphone Amp OUTR",
+			"Headphone Amp INL", "LOUT2",
+			"Headphone Amp INR", "ROUT2",
 			"LINPUT1", "Microphone Jack",
 			"RINPUT1", "Microphone Jack",
 			"LINPUT2", "Onboard Microphone",
@@ -47,11 +50,17 @@ masterdai: simple-audio-card,codec {
 			system-clock-frequency = <12288000>;
 		};
 
-		simple-audio-card,cpu {
+		cpudai: simple-audio-card,cpu {
 			sound-dai = <&i2s0_8ch>;
 		};
 	};
 
+	headphones_amp: audio-amplifier-headphones {
+		compatible = "simple-audio-amplifier";
+		enable-gpios = <&gpio1 RK_PA4 GPIO_ACTIVE_HIGH>;
+		sound-name-prefix = "Headphone Amp";
+	};
+
 	chosen {
 		stdout-path = "serial2:1500000n8";
 	};
@@ -327,12 +336,12 @@ &i2c3 {
 	es8388: audio-codec@11 {
 		compatible = "everest,es8388", "everest,es8328";
 		reg = <0x11>;
-		clocks = <&cru I2S1_8CH_MCLKOUT>;
+		clocks = <&cru I2S0_8CH_MCLKOUT>;
 		AVDD-supply = <&vcc_3v3_s0>;
 		DVDD-supply = <&vcc_1v8_s0>;
 		HPVDD-supply = <&vcc_3v3_s0>;
 		PVDD-supply = <&vcc_3v3_s0>;
-		assigned-clocks = <&cru I2S1_8CH_MCLKOUT>;
+		assigned-clocks = <&cru I2S0_8CH_MCLKOUT>;
 		assigned-clock-rates = <12288000>;
 		#sound-dai-cells = <0>;
 	};
@@ -824,6 +833,12 @@ &usb_host0_ohci {
 
 &usb_host0_xhci {
 	extcon = <&u2phy0>;
+	role-switch-default-mode = "peripheral";
+	usb-role-switch;
+	status = "okay";
+};
+
+&usbdp_phy0 {
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-sck-kv-g-revA.dtso b/arch/arm64/boot/dts/xilinx/zynqmp-sck-kv-g-revA.dtso
index 923a70d750bfc..52954761c9f3d 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-sck-kv-g-revA.dtso
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-sck-kv-g-revA.dtso
@@ -218,23 +218,23 @@ &pinctrl0 {
 	status = "okay";
 
 	pinctrl_gpio0_default: gpio0-default {
-                conf {
-                        groups = "gpio0_38_grp";
-                        bias-pull-up;
-                        power-source = <IO_STANDARD_LVCMOS18>;
-                };
-
-                mux {
-                        groups = "gpio0_38_grp";
-                        function = "gpio0";
-                };
-
-                conf-tx {
-                        pins = "MIO38";
-                        bias-disable;
-                        output-enable;
-                };
-        };
+		conf {
+			groups = "gpio0_38_grp";
+			bias-pull-up;
+			power-source = <IO_STANDARD_LVCMOS18>;
+		};
+
+		mux {
+			groups = "gpio0_38_grp";
+			function = "gpio0";
+		};
+
+		conf-tx {
+			pins = "MIO38";
+			bias-disable;
+			output-enable;
+		};
+	};
 
 	pinctrl_uart1_default: uart1-default {
 		conf {
@@ -374,7 +374,6 @@ conf {
 
 		conf-cd {
 			groups = "sdio1_cd_0_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <SLEW_RATE_SLOW>;
 			power-source = <IO_STANDARD_LVCMOS18>;
@@ -393,9 +392,9 @@ mux {
 };
 
 &gpio {
-        status = "okay";
-        pinctrl-names = "default";
-        pinctrl-0 = <&pinctrl_gpio0_default>;
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio0_default>;
 };
 
 &uart1 {
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-sck-kv-g-revB.dtso b/arch/arm64/boot/dts/xilinx/zynqmp-sck-kv-g-revB.dtso
index 563e750b0e082..49732de5fa4b9 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-sck-kv-g-revB.dtso
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-sck-kv-g-revB.dtso
@@ -365,7 +365,6 @@ conf {
 
 		conf-cd {
 			groups = "sdio1_cd_0_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <SLEW_RATE_SLOW>;
 			power-source = <IO_STANDARD_LVCMOS18>;
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm015-dc1.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm015-dc1.dts
index 6aff22d433616..f57987dad50fd 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm015-dc1.dts
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm015-dc1.dts
@@ -270,7 +270,6 @@ mux-cd {
 
 		conf-cd {
 			groups = "sdio0_cd_0_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <SLEW_RATE_SLOW>;
 			power-source = <IO_STANDARD_LVCMOS18>;
@@ -283,7 +282,6 @@ mux-wp {
 
 		conf-wp {
 			groups = "sdio0_wp_0_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <SLEW_RATE_SLOW>;
 			power-source = <IO_STANDARD_LVCMOS18>;
@@ -310,7 +308,6 @@ mux-cd {
 
 		conf-cd {
 			groups = "sdio1_cd_0_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <SLEW_RATE_SLOW>;
 			power-source = <IO_STANDARD_LVCMOS18>;
@@ -323,7 +320,6 @@ mux-wp {
 
 		conf-wp {
 			groups = "sdio1_wp_0_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <SLEW_RATE_SLOW>;
 			power-source = <IO_STANDARD_LVCMOS18>;
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm019-dc5.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm019-dc5.dts
index 53aa3dca1dca2..737d445dc16b7 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm019-dc5.dts
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm019-dc5.dts
@@ -270,7 +270,6 @@ mux-cd {
 
 		conf-cd {
 			groups = "sdio0_cd_0_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <SLEW_RATE_SLOW>;
 			power-source = <IO_STANDARD_LVCMOS18>;
@@ -283,7 +282,6 @@ mux-wp {
 
 		conf-wp {
 			groups = "sdio0_wp_0_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <SLEW_RATE_SLOW>;
 			power-source = <IO_STANDARD_LVCMOS18>;
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zcu100-revC.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zcu100-revC.dts
index 4ec8a400494e3..41f312a82bb44 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-zcu100-revC.dts
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-zcu100-revC.dts
@@ -320,7 +320,6 @@ mux-cd {
 
 		conf-cd {
 			groups = "sdio0_cd_0_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <SLEW_RATE_SLOW>;
 			power-source = <IO_STANDARD_LVCMOS18>;
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revA.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revA.dts
index e172a30e7b217..a5bc521ab6797 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revA.dts
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-zcu102-revA.dts
@@ -896,7 +896,6 @@ mux-cd {
 
 		conf-cd {
 			groups = "sdio1_cd_0_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <SLEW_RATE_SLOW>;
 			power-source = <IO_STANDARD_LVCMOS18>;
@@ -909,7 +908,6 @@ mux-wp {
 
 		conf-wp {
 			groups = "sdio1_wp_0_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <SLEW_RATE_SLOW>;
 			power-source = <IO_STANDARD_LVCMOS18>;
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revA.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revA.dts
index fe8f151ed706f..32509083e54f4 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revA.dts
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revA.dts
@@ -359,7 +359,6 @@ mux-cd {
 
 		conf-cd {
 			groups = "sdio1_cd_0_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <SLEW_RATE_SLOW>;
 			power-source = <IO_STANDARD_LVCMOS18>;
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revC.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revC.dts
index 3ee8ab2247221..96699be8430fa 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revC.dts
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-zcu104-revC.dts
@@ -371,7 +371,6 @@ mux-cd {
 
 		conf-cd {
 			groups = "sdio1_cd_0_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <SLEW_RATE_SLOW>;
 			power-source = <IO_STANDARD_LVCMOS18>;
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zcu106-revA.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zcu106-revA.dts
index 7f6c87d4d77e1..52441e5c87390 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-zcu106-revA.dts
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-zcu106-revA.dts
@@ -895,7 +895,6 @@ mux-cd {
 
 		conf-cd {
 			groups = "sdio1_cd_0_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <SLEW_RATE_SLOW>;
 			power-source = <IO_STANDARD_LVCMOS18>;
@@ -908,7 +907,6 @@ mux-wp {
 
 		conf-wp {
 			groups = "sdio1_wp_0_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <SLEW_RATE_SLOW>;
 			power-source = <IO_STANDARD_LVCMOS18>;
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zcu111-revA.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zcu111-revA.dts
index 428b5558fbbaf..b34e4c93d249d 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-zcu111-revA.dts
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-zcu111-revA.dts
@@ -750,7 +750,6 @@ mux-cd {
 
 		conf-cd {
 			groups = "sdio1_cd_0_grp";
-			bias-high-impedance;
 			bias-pull-up;
 			slew-rate = <SLEW_RATE_SLOW>;
 			power-source = <IO_STANDARD_LVCMOS18>;
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index 654a102cb5bc5..91a3b2cd10b34 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -521,12 +521,14 @@ CONFIG_WLCORE_SDIO=m
 CONFIG_WWAN=m
 CONFIG_MHI_WWAN_CTRL=m
 CONFIG_MHI_WWAN_MBIM=m
+CONFIG_QCOM_BAM_DMUX=m
 CONFIG_INPUT_EVDEV=y
 CONFIG_KEYBOARD_ADC=m
 CONFIG_KEYBOARD_GPIO=y
 CONFIG_KEYBOARD_GPIO_POLLED=m
 CONFIG_KEYBOARD_SNVS_PWRKEY=m
 CONFIG_KEYBOARD_IMX_SC_KEY=m
+CONFIG_KEYBOARD_SUN4I_LRADC=m
 CONFIG_KEYBOARD_CROS_EC=y
 CONFIG_KEYBOARD_MTK_PMIC=m
 CONFIG_MOUSE_ELAN_I2C=m
@@ -601,6 +603,7 @@ CONFIG_HW_RANDOM=y
 CONFIG_HW_RANDOM_VIRTIO=y
 CONFIG_HW_RANDOM_HISI_TRNG=m
 CONFIG_HW_RANDOM_XILINX=m
+CONFIG_HW_RANDOM_QCOM=m
 CONFIG_TCG_TPM=y
 CONFIG_TCG_TIS=m
 CONFIG_TCG_TIS_SPI=m
@@ -1132,6 +1135,7 @@ CONFIG_SND_SOC_TEGRA210_AMX=m
 CONFIG_SND_SOC_TEGRA210_ADX=m
 CONFIG_SND_SOC_TEGRA210_MIXER=m
 CONFIG_SND_SOC_TEGRA_AUDIO_GRAPH_CARD=m
+CONFIG_SND_SOC_DMIC=m
 CONFIG_SND_SOC_J721E_EVM=m
 CONFIG_SND_SOC_XILINX_I2S=m
 CONFIG_SND_SOC_XILINX_AUDIO_FORMATTER=m
@@ -1425,6 +1429,7 @@ CONFIG_EC_ACER_ASPIRE1=m
 CONFIG_EC_HUAWEI_GAOKUN=m
 CONFIG_EC_LENOVO_YOGA_C630=m
 CONFIG_EC_LENOVO_THINKPAD_T14S=m
+CONFIG_EC_QCOM_HAMOA=m
 CONFIG_COMMON_CLK_APPLE_NCO=m
 CONFIG_COMMON_CLK_RK808=y
 CONFIG_COMMON_CLK_SCMI=y
@@ -1939,8 +1944,6 @@ CONFIG_CRYPTO_AES_ARM64_CE_CCM=y
 CONFIG_CRYPTO_DEV_SUN8I_CE=m
 CONFIG_CRYPTO_DEV_FSL_CAAM=m
 CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM=m
-CONFIG_CRYPTO_DEV_QCE=m
-CONFIG_CRYPTO_DEV_QCOM_RNG=m
 CONFIG_CRYPTO_DEV_TEGRA=m
 CONFIG_CRYPTO_DEV_ZYNQMP_AES=m
 CONFIG_CRYPTO_DEV_ZYNQMP_SHA3=m
diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h
index 65555284446e7..7075a3bd2c616 100644
--- a/arch/arm64/include/asm/fixmap.h
+++ b/arch/arm64/include/asm/fixmap.h
@@ -78,8 +78,12 @@ enum fixed_addresses {
 	/*
 	 * Temporary boot-time mappings, used by early_ioremap(),
 	 * before ioremap() is functional.
+	 *
+	 * Reserve one extra page so a 256K mapping may start at any
+	 * offset within a page. early_ioremap() maps the page-aligned
+	 * physical range, so the initial offset can consume an extra page.
 	 */
-#define NR_FIX_BTMAPS		(SZ_256K / PAGE_SIZE)
+#define NR_FIX_BTMAPS		((SZ_256K / PAGE_SIZE) + 1)
 #define FIX_BTMAPS_SLOTS	7
 #define TOTAL_FIX_BTMAPS	(NR_FIX_BTMAPS * FIX_BTMAPS_SLOTS)
 
diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h
index cdf3e8422ea12..012d711034d17 100644
--- a/arch/arm64/include/asm/kvm_nested.h
+++ b/arch/arm64/include/asm/kvm_nested.h
@@ -388,6 +388,14 @@ struct s1_walk_result {
 	bool	failed;
 };
 
+static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw)
+{
+	wr->fst		= fst;
+	wr->ptw		= s1ptw;
+	wr->s2		= s1ptw;
+	wr->failed	= true;
+}
+
 int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi,
 		       struct s1_walk_result *wr, u64 va);
 int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa,
diff --git a/arch/arm64/include/asm/module.lds.h b/arch/arm64/include/asm/module.lds.h
index 0b3aacd22c590..603f92a8a73ed 100644
--- a/arch/arm64/include/asm/module.lds.h
+++ b/arch/arm64/include/asm/module.lds.h
@@ -4,19 +4,6 @@ SECTIONS {
 	.text.ftrace_trampoline 0 : { BYTE(0) }
 	.init.text.ftrace_trampoline 0 : { BYTE(0) }
 
-#ifdef CONFIG_KASAN_SW_TAGS
-	/*
-	 * Outlined checks go into comdat-deduplicated sections named .text.hot.
-	 * Because they are in comdats they are not combined by the linker and
-	 * we otherwise end up with multiple sections with the same .text.hot
-	 * name in the .ko file. The kernel module loader warns if it sees
-	 * multiple sections with the same name so we use this sections
-	 * directive to force them into a single section and silence the
-	 * warning.
-	 */
-	.text.hot 0 : { *(.text.hot) }
-#endif
-
 #ifdef CONFIG_UNWIND_TABLES
 	/*
 	 * Currently, we only use unwind info at module load time, so we can
diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h
index 212ce1b02e15e..09d7c00cf4054 100644
--- a/arch/arm64/include/asm/pgtable-prot.h
+++ b/arch/arm64/include/asm/pgtable-prot.h
@@ -28,11 +28,11 @@
 #define PTE_PRESENT_VALID_KERNEL (PTE_VALID | PTE_MAYBE_NG)
 
 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
-#define PTE_UFFD_WP		(_AT(pteval_t, 1) << 58) /* uffd-wp tracking */
-#define PTE_SWP_UFFD_WP		(_AT(pteval_t, 1) << 3)	 /* only for swp ptes */
+#define PTE_UFFD		(_AT(pteval_t, 1) << 58) /* userfaultfd tracking */
+#define PTE_SWP_UFFD		(_AT(pteval_t, 1) << 3)	 /* only for swp ptes */
 #else
-#define PTE_UFFD_WP		(_AT(pteval_t, 0))
-#define PTE_SWP_UFFD_WP		(_AT(pteval_t, 0))
+#define PTE_UFFD		(_AT(pteval_t, 0))
+#define PTE_SWP_UFFD		(_AT(pteval_t, 0))
 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
 
 #define _PROT_DEFAULT		(PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 27689c62bd25d..a2681d7553584 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -341,19 +341,19 @@ static inline pmd_t pmd_mknoncont(pmd_t pmd)
 }
 
 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
-static inline int pte_uffd_wp(pte_t pte)
+static inline int pte_uffd(pte_t pte)
 {
-	return !!(pte_val(pte) & PTE_UFFD_WP);
+	return !!(pte_val(pte) & PTE_UFFD);
 }
 
-static inline pte_t pte_mkuffd_wp(pte_t pte)
+static inline pte_t pte_mkuffd(pte_t pte)
 {
-	return pte_wrprotect(set_pte_bit(pte, __pgprot(PTE_UFFD_WP)));
+	return pte_wrprotect(set_pte_bit(pte, __pgprot(PTE_UFFD)));
 }
 
-static inline pte_t pte_clear_uffd_wp(pte_t pte)
+static inline pte_t pte_clear_uffd(pte_t pte)
 {
-	return clear_pte_bit(pte, __pgprot(PTE_UFFD_WP));
+	return clear_pte_bit(pte, __pgprot(PTE_UFFD));
 }
 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
 
@@ -537,26 +537,23 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte)
 }
 
 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
-static inline pte_t pte_swp_mkuffd_wp(pte_t pte)
+static inline pte_t pte_swp_mkuffd(pte_t pte)
 {
-	return set_pte_bit(pte, __pgprot(PTE_SWP_UFFD_WP));
+	return set_pte_bit(pte, __pgprot(PTE_SWP_UFFD));
 }
 
-static inline int pte_swp_uffd_wp(pte_t pte)
+static inline int pte_swp_uffd(pte_t pte)
 {
-	return !!(pte_val(pte) & PTE_SWP_UFFD_WP);
+	return !!(pte_val(pte) & PTE_SWP_UFFD);
 }
 
-static inline pte_t pte_swp_clear_uffd_wp(pte_t pte)
+static inline pte_t pte_swp_clear_uffd(pte_t pte)
 {
-	return clear_pte_bit(pte, __pgprot(PTE_SWP_UFFD_WP));
+	return clear_pte_bit(pte, __pgprot(PTE_SWP_UFFD));
 }
 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
 
-#ifdef CONFIG_NUMA_BALANCING
-/*
- * See the comment in include/linux/pgtable.h
- */
+#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE
 static inline int pte_protnone(pte_t pte)
 {
 	/*
@@ -575,7 +572,7 @@ static inline int pmd_protnone(pmd_t pmd)
 {
 	return pte_protnone(pmd_pte(pmd));
 }
-#endif
+#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */
 
 #define pmd_present(pmd)	pte_present(pmd_pte(pmd))
 #define pmd_dirty(pmd)		pte_dirty(pmd_pte(pmd))
@@ -593,13 +590,13 @@ static inline int pmd_protnone(pmd_t pmd)
 #define pmd_mkvalid_k(pmd)	pte_pmd(pte_mkvalid_k(pmd_pte(pmd)))
 #define pmd_mkinvalid(pmd)	pte_pmd(pte_mkinvalid(pmd_pte(pmd)))
 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
-#define pmd_uffd_wp(pmd)	pte_uffd_wp(pmd_pte(pmd))
-#define pmd_mkuffd_wp(pmd)	pte_pmd(pte_mkuffd_wp(pmd_pte(pmd)))
-#define pmd_clear_uffd_wp(pmd)	pte_pmd(pte_clear_uffd_wp(pmd_pte(pmd)))
-#define pmd_swp_uffd_wp(pmd)	pte_swp_uffd_wp(pmd_pte(pmd))
-#define pmd_swp_mkuffd_wp(pmd)	pte_pmd(pte_swp_mkuffd_wp(pmd_pte(pmd)))
-#define pmd_swp_clear_uffd_wp(pmd) \
-				pte_pmd(pte_swp_clear_uffd_wp(pmd_pte(pmd)))
+#define pmd_uffd(pmd)	pte_uffd(pmd_pte(pmd))
+#define pmd_mkuffd(pmd)	pte_pmd(pte_mkuffd(pmd_pte(pmd)))
+#define pmd_clear_uffd(pmd)	pte_pmd(pte_clear_uffd(pmd_pte(pmd)))
+#define pmd_swp_uffd(pmd)	pte_swp_uffd(pmd_pte(pmd))
+#define pmd_swp_mkuffd(pmd)	pte_pmd(pte_swp_mkuffd(pmd_pte(pmd)))
+#define pmd_swp_clear_uffd(pmd) \
+				pte_pmd(pte_swp_clear_uffd(pmd_pte(pmd)))
 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
 
 #define pmd_write(pmd)		pte_write(pmd_pte(pmd))
@@ -1515,7 +1512,7 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
  * Encode and decode a swap entry:
  *	bits 0-1:	present (must be zero)
  *	bits 2:		remember PG_anon_exclusive
- *	bit  3:		remember uffd-wp state
+ *	bit  3:		remember uffd state
  *	bits 6-10:	swap type
  *	bit  11:	PTE_PRESENT_INVALID (must be zero)
  *	bits 12-61:	swap offset
@@ -1534,10 +1531,10 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
 #define __pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
 #define __swp_entry_to_pte(swp)	((pte_t) { (swp).val })
 
-#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
 #define __pmd_to_swp_entry(pmd)		((swp_entry_t) { pmd_val(pmd) })
 #define __swp_entry_to_pmd(swp)		__pmd((swp).val)
-#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
+#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */
 
 /*
  * Ensure that there are not more swap files than can be encoded in the kernel
diff --git a/arch/arm64/include/asm/ptdump.h b/arch/arm64/include/asm/ptdump.h
index 5b374a6ab34a4..50a195eda8ed2 100644
--- a/arch/arm64/include/asm/ptdump.h
+++ b/arch/arm64/include/asm/ptdump.h
@@ -7,8 +7,6 @@
 
 #include <linux/ptdump.h>
 
-DECLARE_STATIC_KEY_FALSE(arm64_ptdump_lock_key);
-
 #ifdef CONFIG_PTDUMP
 
 #include <linux/mm_types.h>
diff --git a/arch/arm64/include/asm/vmalloc.h b/arch/arm64/include/asm/vmalloc.h
index 4ec1acd3c1b34..d665f9d687422 100644
--- a/arch/arm64/include/asm/vmalloc.h
+++ b/arch/arm64/include/asm/vmalloc.h
@@ -23,10 +23,14 @@ static inline unsigned long arch_vmap_pte_range_map_size(unsigned long addr,
 						unsigned long end, u64 pfn,
 						unsigned int max_page_shift)
 {
+	unsigned long size;
+
 	/*
 	 * If the block is at least CONT_PTE_SIZE in size, and is naturally
 	 * aligned in both virtual and physical space, then we can pte-map the
 	 * block using the PTE_CONT bit for more efficient use of the TLB.
+	 * The returned mapping size may cover multiple CONT_PTE_SIZE blocks,
+	 * capped below PMD_SIZE.
 	 */
 	if (max_page_shift < CONT_PTE_SHIFT)
 		return PAGE_SIZE;
@@ -40,7 +44,9 @@ static inline unsigned long arch_vmap_pte_range_map_size(unsigned long addr,
 	if (!IS_ALIGNED(PFN_PHYS(pfn), CONT_PTE_SIZE))
 		return PAGE_SIZE;
 
-	return CONT_PTE_SIZE;
+	size = min3(end - addr, 1UL << max_page_shift, PMD_SIZE >> 1);
+	size = rounddown_pow_of_two(size);
+	return size;
 }
 
 #define arch_vmap_pte_range_unmap_size arch_vmap_pte_range_unmap_size
diff --git a/arch/arm64/kernel/machine_kexec_file.c b/arch/arm64/kernel/machine_kexec_file.c
index e31fabed378a5..854d872dfd0f5 100644
--- a/arch/arm64/kernel/machine_kexec_file.c
+++ b/arch/arm64/kernel/machine_kexec_file.c
@@ -40,46 +40,30 @@ int arch_kimage_file_post_load_cleanup(struct kimage *image)
 }
 
 #ifdef CONFIG_CRASH_DUMP
-static int prepare_elf_headers(void **addr, unsigned long *sz)
+unsigned int arch_get_system_nr_ranges(void)
 {
-	struct crash_mem *cmem;
-	unsigned int nr_ranges;
-	int ret;
-	u64 i;
+	unsigned int nr_ranges = 2 + crashk_cma_cnt; /* for exclusion of crashkernel region */
 	phys_addr_t start, end;
+	u64 i;
 
-	nr_ranges = 2; /* for exclusion of crashkernel region */
 	for_each_mem_range(i, &start, &end)
 		nr_ranges++;
 
-	cmem = kmalloc_flex(*cmem, ranges, nr_ranges);
-	if (!cmem)
-		return -ENOMEM;
+	return nr_ranges;
+}
+
+int arch_crash_populate_cmem(struct crash_mem *cmem)
+{
+	phys_addr_t start, end;
+	u64 i;
 
-	cmem->max_nr_ranges = nr_ranges;
-	cmem->nr_ranges = 0;
 	for_each_mem_range(i, &start, &end) {
 		cmem->ranges[cmem->nr_ranges].start = start;
 		cmem->ranges[cmem->nr_ranges].end = end - 1;
 		cmem->nr_ranges++;
 	}
 
-	/* Exclude crashkernel region */
-	ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end);
-	if (ret)
-		goto out;
-
-	if (crashk_low_res.end) {
-		ret = crash_exclude_mem_range(cmem, crashk_low_res.start, crashk_low_res.end);
-		if (ret)
-			goto out;
-	}
-
-	ret = crash_prepare_elf64_headers(cmem, true, addr, sz);
-
-out:
-	kfree(cmem);
-	return ret;
+	return 0;
 }
 #endif
 
@@ -109,7 +93,7 @@ int load_other_segments(struct kimage *image,
 	void *headers;
 	unsigned long headers_sz;
 	if (image->type == KEXEC_TYPE_CRASH) {
-		ret = prepare_elf_headers(&headers, &headers_sz);
+		ret = crash_prepare_headers(true, &headers, &headers_sz, NULL);
 		if (ret) {
 			pr_err("Preparing elf core header failed\n");
 			goto out_err;
diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c
index 4d08598e2891d..f43d36c0d7f31 100644
--- a/arch/arm64/kernel/ptrace.c
+++ b/arch/arm64/kernel/ptrace.c
@@ -2379,7 +2379,7 @@ static int report_syscall_entry(struct pt_regs *regs)
 	int regno, ret;
 
 	saved_reg = ptrace_save_reg(regs, PTRACE_SYSCALL_ENTER, &regno);
-	ret = ptrace_report_syscall_entry(regs);
+	ret = !ptrace_report_syscall_permit_entry(regs);
 	if (ret)
 		forget_syscall(regs);
 	regs->regs[regno] = saved_reg;
@@ -2408,6 +2408,21 @@ static void report_syscall_exit(struct pt_regs *regs)
 	}
 }
 
+static void update_syscall_orig_x0_after_ptrace(struct pt_regs *regs)
+{
+	/*
+	 * Keep orig_x0 authoritative so that seccomp (via
+	 * syscall_get_arguments()), audit and the restart path all see the same
+	 * first argument the syscall is dispatched with, even if it has been
+	 * updated by a tracer. Skip this for NO_SYSCALL (set either by the user
+	 * or the tracer), as regs[0] holds the return value (see the comment in
+	 * el0_svc_common()) and can be unwound using syscall_rollback().
+	 * For compat tasks, orig_r0 is provided directly through GPR index 17.
+	 */
+	if (!is_compat_task() && regs->syscallno != NO_SYSCALL)
+		regs->orig_x0 = regs->regs[0];
+}
+
 int syscall_trace_enter(struct pt_regs *regs)
 {
 	unsigned long flags = read_thread_flags();
@@ -2417,12 +2432,26 @@ int syscall_trace_enter(struct pt_regs *regs)
 		ret = report_syscall_entry(regs);
 		if (ret || (flags & _TIF_SYSCALL_EMU))
 			return NO_SYSCALL;
+
+		/*
+		 * Ensure ptrace changes to x0 during a regular
+		 * syscall-enter-stop (PTRACE_SYSCALL) are visible to
+		 * subsequent seccomp checks, tracepoints and audit.
+		 */
+		update_syscall_orig_x0_after_ptrace(regs);
 	}
 
 	/* Do the secure computing after ptrace; failures should be fast. */
-	if (secure_computing() == -1)
+	if (!seccomp_permit_syscall())
 		return NO_SYSCALL;
 
+	/*
+	 * Ensure tracer changes to x0 during seccomp ptrace exit
+	 * processing (SECCOMP_RET_TRACE) are visible to tracepoints and
+	 * audit.
+	 */
+	update_syscall_orig_x0_after_ptrace(regs);
+
 	if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
 		trace_sys_enter(regs, regs->syscallno);
 
diff --git a/arch/arm64/kernel/vdso32/vdso.lds.S b/arch/arm64/kernel/vdso32/vdso.lds.S
index c374fb0146f34..12bfc39e8aab6 100644
--- a/arch/arm64/kernel/vdso32/vdso.lds.S
+++ b/arch/arm64/kernel/vdso32/vdso.lds.S
@@ -82,9 +82,11 @@ VERSION
 {
 	LINUX_2.6 {
 	global:
+#ifdef CONFIG_COMPAT_32BIT_TIME
 		__vdso_clock_gettime;
 		__vdso_gettimeofday;
 		__vdso_clock_getres;
+#endif /* CONFIG_COMPAT_32BIT_TIME */
 		__vdso_clock_gettime64;
 		__vdso_clock_getres_time64;
 	local: *;
diff --git a/arch/arm64/kernel/vdso32/vgettimeofday.c b/arch/arm64/kernel/vdso32/vgettimeofday.c
index 0c6998ebe491a..12d0255cc2cf1 100644
--- a/arch/arm64/kernel/vdso32/vgettimeofday.c
+++ b/arch/arm64/kernel/vdso32/vgettimeofday.c
@@ -8,16 +8,17 @@
 #define BUILD_VDSO32_64
 #include <vdso/gettime.h>
 
+#ifdef CONFIG_COMPAT_32BIT_TIME
 int __vdso_clock_gettime(clockid_t clock,
 			 struct old_timespec32 *ts)
 {
 	return __cvdso_clock_gettime32(clock, ts);
 }
 
-int __vdso_clock_gettime64(clockid_t clock,
-			   struct __kernel_timespec *ts)
+int __vdso_clock_getres(clockid_t clock_id,
+			struct old_timespec32 *res)
 {
-	return __cvdso_clock_gettime(clock, ts);
+	return __cvdso_clock_getres_time32(clock_id, res);
 }
 
 int __vdso_gettimeofday(struct __kernel_old_timeval *tv,
@@ -25,11 +26,12 @@ int __vdso_gettimeofday(struct __kernel_old_timeval *tv,
 {
 	return __cvdso_gettimeofday(tv, tz);
 }
+#endif /* CONFIG_COMPAT_32BIT_TIME */
 
-int __vdso_clock_getres(clockid_t clock_id,
-			struct old_timespec32 *res)
+int __vdso_clock_gettime64(clockid_t clock,
+			   struct __kernel_timespec *ts)
 {
-	return __cvdso_clock_getres_time32(clock_id, res);
+	return __cvdso_clock_gettime(clock, ts);
 }
 
 int __vdso_clock_getres_time64(clockid_t clock_id, struct __kernel_timespec *res)
diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c
index b8ded434c63f9..640f2dc00a8ba 100644
--- a/arch/arm64/kvm/at.c
+++ b/arch/arm64/kvm/at.c
@@ -11,14 +11,6 @@
 #include <asm/kvm_mmu.h>
 #include <asm/lsui.h>
 
-static void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw)
-{
-	wr->fst		= fst;
-	wr->ptw		= s1ptw;
-	wr->s2		= s1ptw;
-	wr->failed	= true;
-}
-
 #define S1_MMU_DISABLED		(-127)
 
 static int get_ia_size(struct s1_walk_info *wi)
diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c
index e688bc5139c1f..3c82f392845d1 100644
--- a/arch/arm64/kvm/emulate-nested.c
+++ b/arch/arm64/kvm/emulate-nested.c
@@ -2746,17 +2746,33 @@ static u64 kvm_check_illegal_exception_return(struct kvm_vcpu *vcpu, u64 spsr)
 	    (spsr & PSR_MODE32_BIT) ||
 	    (vcpu_el2_tge_is_set(vcpu) && (mode == PSR_MODE_EL1t ||
 					   mode == PSR_MODE_EL1h))) {
+		u64 mask;
+
 		/*
-		 * The guest is playing with our nerves. Preserve EL, SP,
-		 * masks, flags from the existing PSTATE, and set IL.
-		 * The HW will then generate an Illegal State Exception
-		 * immediately after ERET.
+		 * On an illegal exception return, the flags and masks are
+		 * taken from the SPSR while PSTATE.{EL,SP,nRW} and, if
+		 * FEAT_GCS, PSTATE.EXLOCK are unchanged (R_VWJHB). Set IL
+		 * so the HW generates an Illegal State Exception right
+		 * after ERET.
 		 */
-		spsr = *vcpu_cpsr(vcpu);
+		mask = PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT |
+		       PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT;
+
+		if (kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, PAN, IMP))
+			mask |= PSR_PAN_BIT;
+		if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, NMI, IMP))
+			mask |= ALLINT_ALLINT;
+		/* FEAT_SPE_EXC and FEAT_TRBE_EXC also gate PSTATE.PM one day... */
+		if (kvm_has_feat(vcpu->kvm, ID_AA64DFR1_EL1, EBEP, IMP))
+			mask |= BIT_ULL(32);	/* SPSR_ELx.PM */
+
+		spsr &= mask;
 
-		spsr &= (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT |
-			 PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT |
-			 PSR_MODE_MASK | PSR_MODE32_BIT);
+		mask = PSR_MODE_MASK | PSR_MODE32_BIT;
+		if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, GCS, IMP))
+			mask |= BIT_ULL(34);	/* PSTATE.EXLOCK */
+
+		spsr |= *vcpu_cpsr(vcpu) & mask;
 		spsr |= PSR_IL_BIT;
 	}
 
@@ -2784,7 +2800,7 @@ void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu)
 		 * ERET handling, and the guest will have a little surprise.
 		 */
 		if (kvm_has_pauth(vcpu->kvm, FPACCOMBINE) && !(spsr & PSR_IL_BIT)) {
-			esr &= ESR_ELx_ERET_ISS_ERETA;
+			esr &= (ESR_ELx_ERET_ISS_ERETA | ESR_ELx_IL);
 			esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_FPAC);
 			kvm_inject_nested_sync(vcpu, esr);
 			return;
@@ -2826,6 +2842,7 @@ static void kvm_inject_el2_exception(struct kvm_vcpu *vcpu, u64 esr_el2,
 		break;
 	case except_type_serror:
 		kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SERR);
+		vcpu_write_sys_reg(vcpu, esr_el2, ESR_EL2);
 		break;
 	default:
 		WARN_ONCE(1, "Unsupported EL2 exception injection %d\n", type);
@@ -2950,6 +2967,6 @@ int kvm_inject_nested_serror(struct kvm_vcpu *vcpu, u64 esr)
 	 * vSError injection. Manually populate EC for an emulated SError
 	 * exception.
 	 */
-	esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR);
+	esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR) | ESR_ELx_IL;
 	return kvm_inject_nested(vcpu, esr, except_type_serror);
 }
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index 18131e395e24c..4bf624a49591d 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -448,16 +448,19 @@ static inline bool __populate_fault_info(struct kvm_vcpu *vcpu)
 
 static inline bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code)
 {
+	u64 spsr;
+
 	*vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR);
 	arm64_mops_reset_regs(vcpu_gp_regs(vcpu), vcpu->arch.fault.esr_el2);
 	write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR);
 
 	/*
 	 * Finish potential single step before executing the prologue
-	 * instruction.
+	 * instruction. Modify the hardware SPSR_EL2 directly, as vcpu_cpsr()
+	 * may hold a synthetic (vEL2) value for a guest hypervisor.
 	 */
-	*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
-	write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR);
+	spsr = read_sysreg_el2(SYS_SPSR);
+	write_sysreg_el2(spsr & ~DBG_SPSR_SS, SYS_SPSR);
 
 	return true;
 }
@@ -602,8 +605,6 @@ static inline bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)
 			return false;
 		break;
 	case ESR_ELx_EC_SYS64:
-		if (WARN_ON_ONCE(!is_hyp_ctxt(vcpu)))
-			return false;
 		fallthrough;
 	case ESR_ELx_EC_SVE:
 		if (!sve_guest)
diff --git a/arch/arm64/kvm/hyp/include/nvhe/clock.h b/arch/arm64/kvm/hyp/include/nvhe/clock.h
index 9f429f5c06640..ae03ec6965af3 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/clock.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/clock.h
@@ -6,11 +6,11 @@
 #include <asm/kvm_hyp.h>
 
 #ifdef CONFIG_NVHE_EL2_TRACING
-void trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc);
-u64 trace_clock(void);
+void trace_hyp_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc);
+u64 trace_hyp_clock(void);
 #else
 static inline void
-trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) { }
-static inline u64 trace_clock(void) { return 0; }
+trace_hyp_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) { }
+static inline u64 trace_hyp_clock(void) { return 0; }
 #endif
 #endif
diff --git a/arch/arm64/kvm/hyp/nvhe/clock.c b/arch/arm64/kvm/hyp/nvhe/clock.c
index a7fc61976fd0d..f3e2619db4e44 100644
--- a/arch/arm64/kvm/hyp/nvhe/clock.c
+++ b/arch/arm64/kvm/hyp/nvhe/clock.c
@@ -30,7 +30,7 @@ static u64 __clock_mult_uint128(u64 cyc, u32 mult, u32 shift)
 }
 
 /* Does not guarantee no reader on the modified bank. */
-void trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc)
+void trace_hyp_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc)
 {
 	struct clock_data *clock = &trace_clock_data;
 	u64 bank = clock->cur ^ 1;
@@ -48,7 +48,7 @@ void trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc)
 }
 
 /* Use untrusted host data */
-u64 trace_clock(void)
+u64 trace_hyp_clock(void)
 {
 	struct clock_data *clock = &trace_clock_data;
 	u64 bank = smp_load_acquire(&clock->cur);
diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c
index 1af722771178a..a327c2bbb6b64 100644
--- a/arch/arm64/kvm/hyp/nvhe/ffa.c
+++ b/arch/arm64/kvm/hyp/nvhe/ffa.c
@@ -352,7 +352,7 @@ static u32 __ffa_host_share_ranges(struct ffa_mem_region_addr_range *ranges,
 		u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE;
 		u64 pfn = hyp_phys_to_pfn(range->address);
 
-		if (!PAGE_ALIGNED(sz))
+		if (!PAGE_ALIGNED(sz | range->address))
 			break;
 
 		if (__pkvm_host_share_ffa(pfn, sz / PAGE_SIZE))
@@ -372,7 +372,7 @@ static u32 __ffa_host_unshare_ranges(struct ffa_mem_region_addr_range *ranges,
 		u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE;
 		u64 pfn = hyp_phys_to_pfn(range->address);
 
-		if (!PAGE_ALIGNED(sz))
+		if (!PAGE_ALIGNED(sz | range->address))
 			break;
 
 		if (__pkvm_host_unshare_ffa(pfn, sz / PAGE_SIZE))
@@ -476,11 +476,12 @@ static void __do_ffa_mem_xfer(const u64 func_id,
 	DECLARE_REG(u32, fraglen, ctxt, 2);
 	DECLARE_REG(u64, addr_mbz, ctxt, 3);
 	DECLARE_REG(u32, npages_mbz, ctxt, 4);
+	u32 offset, nr_ranges, checked_offset, em_mem_access_off;
 	struct ffa_mem_region_attributes *ep_mem_access;
 	struct ffa_composite_mem_region *reg;
 	struct ffa_mem_region *buf;
-	u32 offset, nr_ranges, checked_offset;
 	int ret = 0;
+	size_t mem_region_len = FFA_MEM_REGION_SZ(hyp_ffa_version);
 
 	if (addr_mbz || npages_mbz || fraglen > len ||
 	    fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) {
@@ -488,8 +489,7 @@ static void __do_ffa_mem_xfer(const u64 func_id,
 		goto out;
 	}
 
-	if (fraglen < sizeof(struct ffa_mem_region) +
-		      sizeof(struct ffa_mem_region_attributes)) {
+	if (fraglen < mem_region_len + ffa_emad_size_get(hyp_ffa_version)) {
 		ret = FFA_RET_INVALID_PARAMETERS;
 		goto out;
 	}
@@ -508,8 +508,13 @@ static void __do_ffa_mem_xfer(const u64 func_id,
 	buf = hyp_buffers.tx;
 	memcpy(buf, host_buffers.tx, fraglen);
 
-	ep_mem_access = (void *)buf +
-			ffa_mem_desc_offset(buf, 0, hyp_ffa_version);
+	em_mem_access_off = ffa_mem_desc_offset(buf, 0, hyp_ffa_version);
+	if ((u64)em_mem_access_off + ffa_emad_size_get(hyp_ffa_version) > fraglen) {
+		ret = FFA_RET_INVALID_PARAMETERS;
+		goto out_unlock;
+	}
+
+	ep_mem_access = (void *)buf + em_mem_access_off;
 	offset = ep_mem_access->composite_off;
 	if (!offset || buf->ep_count != 1 || buf->sender_id != HOST_FFA_ID) {
 		ret = FFA_RET_INVALID_PARAMETERS;
@@ -574,9 +579,9 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res,
 	DECLARE_REG(u32, handle_lo, ctxt, 1);
 	DECLARE_REG(u32, handle_hi, ctxt, 2);
 	DECLARE_REG(u32, flags, ctxt, 3);
+	u32 offset, len, fraglen, fragoff, em_mem_access_off;
 	struct ffa_mem_region_attributes *ep_mem_access;
 	struct ffa_composite_mem_region *reg;
-	u32 offset, len, fraglen, fragoff;
 	struct ffa_mem_region *buf;
 	int ret = 0;
 	u64 handle;
@@ -599,16 +604,22 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res,
 	len = res->a1;
 	fraglen = res->a2;
 
-	ep_mem_access = (void *)buf +
-			ffa_mem_desc_offset(buf, 0, hyp_ffa_version);
+	em_mem_access_off = ffa_mem_desc_offset(buf, 0, hyp_ffa_version);
+	if ((u64)em_mem_access_off + ffa_emad_size_get(hyp_ffa_version) > fraglen) {
+		ret = FFA_RET_INVALID_PARAMETERS;
+		ffa_rx_release(res);
+		goto out_unlock;
+	}
+
+	ep_mem_access = (void *)buf + em_mem_access_off;
 	offset = ep_mem_access->composite_off;
 	/*
 	 * We can trust the SPMD to get this right, but let's at least
 	 * check that we end up with something that doesn't look _completely_
 	 * bogus.
 	 */
-	if (WARN_ON(offset > len ||
-		    fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE)) {
+	if (offset + CONSTITUENTS_OFFSET(0) > len ||
+	    fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) {
 		ret = FFA_RET_ABORTED;
 		ffa_rx_release(res);
 		goto out_unlock;
@@ -636,11 +647,16 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res,
 		ffa_rx_release(res);
 	}
 
+	reg = (void *)buf + offset;
+	if (offset + CONSTITUENTS_OFFSET(reg->addr_range_cnt) > len) {
+		ret = FFA_RET_ABORTED;
+		goto out_unlock;
+	}
+
 	ffa_mem_reclaim(res, handle_lo, handle_hi, flags);
 	if (res->a0 != FFA_SUCCESS)
 		goto out_unlock;
 
-	reg = (void *)buf + offset;
 	/* If the SPMD was happy, then we should be too. */
 	WARN_ON(ffa_host_unshare_ranges(reg->constituents,
 					reg->addr_range_cnt));
@@ -864,7 +880,7 @@ static void do_ffa_part_get(struct arm_smccc_1_2_regs *res,
 
 bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id)
 {
-	struct arm_smccc_1_2_regs res;
+	struct arm_smccc_1_2_regs res = {0};
 
 	/*
 	 * There's no way we can tell what a non-standard SMC call might
diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c
index 3b2c4fbc34d8e..d67833053aae3 100644
--- a/arch/arm64/kvm/hyp/nvhe/pkvm.c
+++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c
@@ -433,7 +433,6 @@ static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm,
 	hyp_vm->host_kvm = host_kvm;
 	hyp_vm->kvm.created_vcpus = nr_vcpus;
 	hyp_vm->kvm.arch.pkvm.is_protected = READ_ONCE(host_kvm->arch.pkvm.is_protected);
-	hyp_vm->kvm.arch.pkvm.is_created = true;
 	hyp_vm->kvm.arch.flags = 0;
 	pkvm_init_features_from_host(hyp_vm, host_kvm);
 
@@ -1056,7 +1055,8 @@ static u64 __pkvm_memshare_page_req(struct kvm_vcpu *vcpu, u64 ipa)
 
 	/* Fake up a data abort (level 3 translation fault on write) */
 	vcpu->arch.fault.esr_el2 = (ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT) |
-				   ESR_ELx_WNR | ESR_ELx_FSC_FAULT |
+				   ESR_ELx_IL | ESR_ELx_WNR |
+				   ESR_ELx_FSC_FAULT |
 				   FIELD_PREP(ESR_ELx_FSC_LEVEL, 3);
 
 	/* Shuffle the IPA around into the HPFAR */
diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c
index 8c3fbb413a06d..b1411fb541397 100644
--- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c
+++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c
@@ -268,6 +268,7 @@ static void inject_sync64(struct kvm_vcpu *vcpu, u64 esr)
 
 	write_sysreg_el1(esr, SYS_ESR);
 	write_sysreg_el1(read_sysreg_el2(SYS_ELR), SYS_ELR);
+	write_sysreg_el1(read_sysreg_el2(SYS_SPSR), SYS_SPSR);
 	write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR);
 	write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR);
 }
@@ -278,7 +279,7 @@ static void inject_sync64(struct kvm_vcpu *vcpu, u64 esr)
  */
 static void inject_undef64(struct kvm_vcpu *vcpu)
 {
-	inject_sync64(vcpu, (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT));
+	inject_sync64(vcpu, (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT) | ESR_ELx_IL);
 }
 
 static u64 read_id_reg(const struct kvm_vcpu *vcpu,
diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c
index e7e150ab265ff..65be6c9fb379d 100644
--- a/arch/arm64/kvm/hyp/nvhe/trace.c
+++ b/arch/arm64/kvm/hyp/nvhe/trace.c
@@ -35,7 +35,7 @@ static bool hyp_trace_buffer_loaded(struct hyp_trace_buffer *trace_buffer)
 void *tracing_reserve_entry(unsigned long length)
 {
 	return simple_ring_buffer_reserve(this_cpu_ptr(trace_buffer.simple_rbs), length,
-					  trace_clock());
+					  trace_hyp_clock());
 }
 
 void tracing_commit_entry(void)
@@ -290,7 +290,7 @@ void __tracing_update_clock(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc)
 	}
 
 	/* ...we can now override the old one and swap. */
-	trace_clock_update(mult, shift, epoch_ns, epoch_cyc);
+	trace_hyp_clock_update(mult, shift, epoch_ns, epoch_cyc);
 }
 
 int __tracing_reset(unsigned int cpu)
diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
index 91a7dfad66866..b74dd5ce1efd3 100644
--- a/arch/arm64/kvm/hyp/pgtable.c
+++ b/arch/arm64/kvm/hyp/pgtable.c
@@ -1370,16 +1370,19 @@ int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr,
 	if (prot & KVM_PGTABLE_PROT_W)
 		set |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W;
 
-	ret = stage2_set_xn_attr(prot, &xn);
-	if (ret)
-		return ret;
+	if (prot & KVM_PGTABLE_PROT_X) {
+		ret = stage2_set_xn_attr(prot, &xn);
+		if (ret)
+			return ret;
 
-	set |= xn & KVM_PTE_LEAF_ATTR_HI_S2_XN;
-	clr |= ~xn & KVM_PTE_LEAF_ATTR_HI_S2_XN;
+		set |= xn & KVM_PTE_LEAF_ATTR_HI_S2_XN;
+		clr |= ~xn & KVM_PTE_LEAF_ATTR_HI_S2_XN;
+	}
 
 	ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level, flags);
 	if (!ret || ret == -EAGAIN)
-		kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, addr, level);
+		kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, addr,
+			     (ret == -EAGAIN) ? TLBI_TTL_UNKNOWN : level);
 	return ret;
 }
 
diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c
index 89982bd3345f6..d6c4fc16f8795 100644
--- a/arch/arm64/kvm/inject_fault.c
+++ b/arch/arm64/kvm/inject_fault.c
@@ -138,11 +138,10 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr
 		pend_sync_exception(vcpu);
 
 	/*
-	 * Build an {i,d}abort, depending on the level and the
-	 * instruction set. Report an external synchronous abort.
+	 * Build an {i,d}abort, depending on the level.
+	 * Report an external synchronous abort.
 	 */
-	if (kvm_vcpu_trap_il_is32bit(vcpu))
-		esr |= ESR_ELx_IL;
+	esr |= ESR_ELx_IL;
 
 	/*
 	 * Here, the guest runs in AArch64 mode when in EL1. If we get
@@ -170,14 +169,7 @@ void kvm_inject_sync(struct kvm_vcpu *vcpu, u64 esr)
 
 static void inject_undef64(struct kvm_vcpu *vcpu)
 {
-	u64 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT);
-
-	/*
-	 * Build an unknown exception, depending on the instruction
-	 * set.
-	 */
-	if (kvm_vcpu_trap_il_is32bit(vcpu))
-		esr |= ESR_ELx_IL;
+	u64 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT) | ESR_ELx_IL;
 
 	kvm_inject_sync(vcpu, esr);
 }
@@ -389,7 +381,7 @@ int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr)
 	 */
 	if (!serror_is_masked(vcpu)) {
 		pend_serror_exception(vcpu);
-		esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR);
+		esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR) | ESR_ELx_IL;
 		vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
 		return 1;
 	}
diff --git a/arch/arm64/kvm/mmio.c b/arch/arm64/kvm/mmio.c
index e2285ed8c91de..d1c3a352d5a22 100644
--- a/arch/arm64/kvm/mmio.c
+++ b/arch/arm64/kvm/mmio.c
@@ -126,6 +126,10 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu)
 		len = kvm_vcpu_dabt_get_as(vcpu);
 		data = kvm_mmio_read_buf(run->mmio.data, len);
 
+		trace_kvm_mmio(KVM_TRACE_MMIO_READ, len, run->mmio.phys_addr,
+			       &data);
+		data = vcpu_data_host_to_guest(vcpu, data, len);
+
 		if (kvm_vcpu_dabt_issext(vcpu) &&
 		    len < sizeof(unsigned long)) {
 			mask = 1U << ((len * 8) - 1);
@@ -135,9 +139,6 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu)
 		if (!kvm_vcpu_dabt_issf(vcpu))
 			data = data & 0xffffffff;
 
-		trace_kvm_mmio(KVM_TRACE_MMIO_READ, len, run->mmio.phys_addr,
-			       &data);
-		data = vcpu_data_host_to_guest(vcpu, data, len);
 		vcpu_set_reg(vcpu, kvm_vcpu_dabt_get_rd(vcpu), data);
 	}
 
diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c
index fb54f6dad995c..dfb96edbdc43c 100644
--- a/arch/arm64/kvm/nested.c
+++ b/arch/arm64/kvm/nested.c
@@ -24,6 +24,7 @@ struct vncr_tlb {
 	struct s1_walk_result	wr;
 
 	u64			hpa;
+	bool			hpa_writable;
 
 	/* -1 when not mapped on a CPU */
 	int			cpu;
@@ -1401,15 +1402,19 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem)
 
 	gfn = vt->wr.pa >> PAGE_SHIFT;
 	memslot = gfn_to_memslot(vcpu->kvm, gfn);
-	if (!memslot)
+	if (!memslot) {
+		fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false);
 		return -EFAULT;
+	}
 
 	*is_gmem = kvm_slot_has_gmem(memslot);
 	if (!*is_gmem) {
 		pfn = __kvm_faultin_pfn(memslot, gfn, write_fault ? FOLL_WRITE : 0,
 					&writable, &page);
-		if (is_error_noslot_pfn(pfn) || (write_fault && !writable))
+		if (is_error_noslot_pfn(pfn)) {
+			fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false);
 			return -EFAULT;
+		}
 	} else {
 		ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL);
 		if (ret) {
@@ -1417,6 +1422,19 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem)
 					      write_fault, false, false);
 			return ret;
 		}
+
+		writable = !(memslot->flags & KVM_MEM_READONLY);
+	}
+
+	/*
+	 * FIXME: This check is too restrictive as KVM allows cacheable memory
+	 * attributes for PFNMAP VMAs that have cacheable attributes in host
+	 * stage-1.
+	 */
+	if (!pfn_is_map_memory(pfn)) {
+		kvm_release_faultin_page(vcpu->kvm, page, true, false);
+		fail_s1_walk(&vt->wr, ESR_ELx_FSC_EXTABT, false);
+		return -EINVAL;
 	}
 
 	scoped_guard(write_lock, &vcpu->kvm->mmu_lock) {
@@ -1427,116 +1445,100 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem)
 
 		vt->gva = va;
 		vt->hpa = pfn << PAGE_SHIFT;
+		vt->hpa_writable = writable;
 		vt->valid = true;
 		vt->cpu = -1;
 
 		kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu);
-		kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw);
+		kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw && vt->hpa_writable);
 	}
 
-	if (vt->wr.pw)
+	if (vt->wr.pw && vt->hpa_writable)
 		mark_page_dirty(vcpu->kvm, gfn);
 
 	return 0;
 }
 
-static void inject_vncr_perm(struct kvm_vcpu *vcpu)
+static void handle_vncr_perm(struct kvm_vcpu *vcpu)
 {
 	struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
 	u64 esr = kvm_vcpu_get_esr(vcpu);
+	u64 fsc;
+
+	/*
+	 * Promote to an external abort if the stage-1 permits writes but the
+	 * HPA is read-only (e.g. RO memslot).
+	 */
+	if (kvm_is_write_fault(vcpu) && vt->wr.pw && !vt->hpa_writable)
+		fsc = ESR_ELx_FSC_EXTABT;
+	/*
+	 * Otherwise, inject a permission fault using the guest's translation
+	 * level rather than the host's.
+	 */
+	else
+		fsc = ESR_ELx_FSC_PERM_L(vt->wr.level);
 
-	/* Adjust the fault level to reflect that of the guest's */
 	esr &= ~ESR_ELx_FSC;
-	esr |= FIELD_PREP(ESR_ELx_FSC,
-			  ESR_ELx_FSC_PERM_L(vt->wr.level));
+	esr |= FIELD_PREP(ESR_ELx_FSC, fsc);
 
 	kvm_inject_nested_sync(vcpu, esr);
 }
 
-static bool kvm_vncr_tlb_lookup(struct kvm_vcpu *vcpu)
-{
-	struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
-
-	lockdep_assert_held_read(&vcpu->kvm->mmu_lock);
-
-	if (!vt->valid)
-		return false;
-
-	if (read_vncr_el2(vcpu) != vt->gva)
-		return false;
-
-	if (vt->wr.nG)
-		return get_asid_by_regime(vcpu, TR_EL20) == vt->wr.asid;
-
-	return true;
-}
-
 int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu)
 {
 	struct vncr_tlb *vt = vcpu->arch.vncr_tlb;
 	u64 esr = kvm_vcpu_get_esr(vcpu);
+	bool is_gmem = false;
+	bool perm;
+	int ret;
 
 	WARN_ON_ONCE(!(esr & ESR_ELx_VNCR));
 
 	if (kvm_vcpu_abt_issea(vcpu))
 		return kvm_handle_guest_sea(vcpu);
 
-	if (esr_fsc_is_permission_fault(esr)) {
-		inject_vncr_perm(vcpu);
-	} else if (esr_fsc_is_translation_fault(esr)) {
-		bool valid, is_gmem = false;
-		int ret;
-
-		scoped_guard(read_lock, &vcpu->kvm->mmu_lock)
-			valid = kvm_vncr_tlb_lookup(vcpu);
-
-		if (!valid)
-			ret = kvm_translate_vncr(vcpu, &is_gmem);
-		else
-			ret = -EPERM;
-
-		switch (ret) {
-		case -EAGAIN:
-			/* Let's try again... */
-			break;
-		case -ENOMEM:
-			/*
-			 * For guest_memfd, this indicates that it failed to
-			 * create a folio to back the memory. Inform userspace.
-			 */
-			if (is_gmem)
-				return 0;
-			/* Otherwise, let's try again... */
-			break;
-		case -EFAULT:
-		case -EIO:
-		case -EHWPOISON:
-			if (is_gmem)
-				return 0;
-			fallthrough;
-		case -EINVAL:
-		case -ENOENT:
-		case -EACCES:
-			/*
-			 * Translation failed, inject the corresponding
-			 * exception back to EL2.
-			 */
-			BUG_ON(!vt->wr.failed);
+	if (!esr_fsc_is_translation_fault(esr) && !esr_fsc_is_permission_fault(esr)) {
+		KVM_BUG(1, vcpu->kvm, "Unhandled VNCR abort, ESR=%llx\n", esr);
+		return -EIO;
+	}
 
-			esr &= ~ESR_ELx_FSC;
-			esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst);
+	ret = kvm_translate_vncr(vcpu, &is_gmem);
+	switch (ret) {
+	case -EAGAIN:
+		/* Let's try again... */
+		return 1;
+	case -ENOMEM:
+		/*
+		 * For guest_memfd, this indicates that it failed to
+		 * create a folio to back the memory. Inform userspace.
+		 */
+		if (is_gmem)
+			return 0;
+		/* Otherwise, let's try again... */
+		break;
+	case -EFAULT:
+	case -EIO:
+	case -EHWPOISON:
+		if (is_gmem)
+			return 0;
+		fallthrough;
+	case -EINVAL:
+	case -ENOENT:
+	case -EACCES:
+		/*
+		 * Translation failed, inject the corresponding
+		 * exception back to EL2.
+		 */
+		esr &= ~ESR_ELx_FSC;
+		esr |= FIELD_PREP(ESR_ELx_FSC, vt->wr.fst);
 
-			kvm_inject_nested_sync(vcpu, esr);
-			break;
-		case -EPERM:
-			/* Hack to deal with POE until we get kernel support */
-			inject_vncr_perm(vcpu);
-			break;
-		case 0:
-			break;
-		}
-	} else {
-		WARN_ONCE(1, "Unhandled VNCR abort, ESR=%llx\n", esr);
+		kvm_inject_nested_sync(vcpu, esr);
+		break;
+	case 0:
+		perm = kvm_is_write_fault(vcpu) ? vt->wr.pw && vt->hpa_writable : vt->wr.pr;
+		if (!perm)
+			handle_vncr_perm(vcpu);
+		break;
 	}
 
 	return 1;
@@ -1574,7 +1576,7 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu)
 
 	vt->cpu = smp_processor_id();
 
-	if (vt->wr.pw && vt->wr.pr)
+	if (vt->hpa_writable && vt->wr.pw && vt->wr.pr)
 		prot = PAGE_KERNEL;
 	else if (vt->wr.pr)
 		prot = PAGE_KERNEL_RO;
diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c
index 053e4f733e4be..17b6584caec52 100644
--- a/arch/arm64/kvm/pkvm.c
+++ b/arch/arm64/kvm/pkvm.c
@@ -185,7 +185,11 @@ static int __pkvm_create_hyp_vm(struct kvm *kvm)
 
 bool pkvm_hyp_vm_is_created(struct kvm *kvm)
 {
-	return READ_ONCE(kvm->arch.pkvm.is_created);
+	/*
+	 * Serialised by config_lock/slots_lock, or by VM lifecycle at
+	 * teardown, so a plain read suffices.
+	 */
+	return kvm->arch.pkvm.is_created;
 }
 
 int pkvm_create_hyp_vm(struct kvm *kvm)
@@ -230,13 +234,6 @@ int pkvm_init_host_vm(struct kvm *kvm, unsigned long type)
 	int ret;
 	bool protected = type & KVM_VM_TYPE_ARM_PROTECTED;
 
-	if (pkvm_hyp_vm_is_created(kvm))
-		return -EINVAL;
-
-	/* VM is already reserved, no need to proceed. */
-	if (kvm->arch.pkvm.handle)
-		return 0;
-
 	/* Reserve the VM in hyp and obtain a hyp handle for the VM. */
 	ret = kvm_call_hyp_nvhe(__pkvm_reserve_vm);
 	if (ret < 0)
@@ -352,7 +349,7 @@ static int __pkvm_pgtable_stage2_reclaim(struct kvm_pgtable *pgt, u64 start, u64
 		page = pfn_to_page(mapping->pfn);
 		WARN_ON_ONCE(mapping->nr_pages != 1);
 		unpin_user_pages_dirty_lock(&page, 1, true);
-		account_locked_vm(current->mm, 1, false);
+		account_locked_vm(kvm->mm, 1, false);
 		pkvm_mapping_remove(mapping, &pgt->pkvm_mappings);
 		kfree(mapping);
 	}
diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c
index 4477f870c7b36..740b39875728d 100644
--- a/arch/arm64/kvm/vgic/vgic-its.c
+++ b/arch/arm64/kvm/vgic/vgic-its.c
@@ -508,6 +508,8 @@ static struct vgic_its *__vgic_doorbell_to_its(struct kvm *kvm, gpa_t db)
 	struct kvm_io_device *kvm_io_dev;
 	struct vgic_io_device *iodev;
 
+	guard(srcu)(&kvm->srcu);
+
 	kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, db);
 	if (!kvm_io_dev)
 		return ERR_PTR(-EINVAL);
diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c
index 5a4768d8cd4f3..ccb7e3a90cd07 100644
--- a/arch/arm64/kvm/vgic/vgic.c
+++ b/arch/arm64/kvm/vgic/vgic.c
@@ -203,6 +203,7 @@ void vgic_flush_pending_lpis(struct kvm_vcpu *vcpu)
 	list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) {
 		if (irq_is_lpi(vcpu->kvm, irq->intid)) {
 			raw_spin_lock(&irq->irq_lock);
+			irq->pending_latch = false;
 			list_del(&irq->ap_list);
 			irq->vcpu = NULL;
 			raw_spin_unlock(&irq->irq_lock);
@@ -792,7 +793,11 @@ static void vgic_prune_ap_list(struct kvm_vcpu *vcpu)
 			continue;
 		}
 
-		/* This interrupt looks like it has to be migrated. */
+		/*
+		 * This interrupt looks like it has to be migrated,
+		 * make sure it is kept alive while locks are dropped.
+		 */
+		vgic_get_irq_ref(irq);
 
 		raw_spin_unlock(&irq->irq_lock);
 		raw_spin_unlock(&vgic_cpu->ap_list_lock);
@@ -815,15 +820,16 @@ static void vgic_prune_ap_list(struct kvm_vcpu *vcpu)
 		raw_spin_lock(&irq->irq_lock);
 
 		/*
-		 * If the affinity has been preserved, move the
-		 * interrupt around. Otherwise, it means things have
-		 * changed while the interrupt was unlocked, and we
-		 * need to replay this.
+		 * If the interrupt is still ours and its affinity has
+		 * been preserved, move it around. Otherwise, it means
+		 * things have changed while the interrupt was unlocked
+		 * (it may even have been taken off the list with its
+		 * affinity left untouched), and we need to replay this.
 		 *
 		 * In all cases, we cannot trust the list not to have
 		 * changed, so we restart from the beginning.
 		 */
-		if (target_vcpu == vgic_target_oracle(irq)) {
+		if (irq->vcpu == vcpu && target_vcpu == vgic_target_oracle(irq)) {
 			struct vgic_cpu *new_cpu = &target_vcpu->arch.vgic_cpu;
 
 			list_del(&irq->ap_list);
@@ -836,6 +842,8 @@ static void vgic_prune_ap_list(struct kvm_vcpu *vcpu)
 		raw_spin_unlock(&vcpuB->arch.vgic_cpu.ap_list_lock);
 		raw_spin_unlock(&vcpuA->arch.vgic_cpu.ap_list_lock);
 
+		deleted_lpis |= vgic_put_irq_norelease(vcpu->kvm, irq);
+
 		if (target_vcpu_needs_kick) {
 			kvm_make_request(KVM_REQ_IRQ_PENDING, target_vcpu);
 			kvm_vcpu_kick(target_vcpu);
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 85e23388f9bb0..0b52557652be6 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -76,6 +76,8 @@ static void data_abort_decode(unsigned long esr)
 		pr_alert("  SF = %lu, AR = %lu\n",
 			 (esr & ESR_ELx_SF) >> ESR_ELx_SF_SHIFT,
 			 (esr & ESR_ELx_AR) >> ESR_ELx_AR_SHIFT);
+		pr_alert("  Xs = %llu\n",
+			 (iss2 & ESR_ELx_Xs_MASK) >> ESR_ELx_Xs_SHIFT);
 	} else {
 		pr_alert("  ISV = 0, ISS = 0x%08lx, ISS2 = 0x%08lx\n",
 			 esr & ESR_ELx_ISS_MASK, iss2);
@@ -87,11 +89,10 @@ static void data_abort_decode(unsigned long esr)
 		 (iss2 & ESR_ELx_TnD) >> ESR_ELx_TnD_SHIFT,
 		 (iss2 & ESR_ELx_TagAccess) >> ESR_ELx_TagAccess_SHIFT);
 
-	pr_alert("  GCS = %ld, Overlay = %lu, DirtyBit = %lu, Xs = %llu\n",
+	pr_alert("  GCS = %ld, Overlay = %lu, DirtyBit = %lu\n",
 		 (iss2 & ESR_ELx_GCS) >> ESR_ELx_GCS_SHIFT,
 		 (iss2 & ESR_ELx_Overlay) >> ESR_ELx_Overlay_SHIFT,
-		 (iss2 & ESR_ELx_DirtyBit) >> ESR_ELx_DirtyBit_SHIFT,
-		 (iss2 & ESR_ELx_Xs_MASK) >> ESR_ELx_Xs_SHIFT);
+		 (iss2 & ESR_ELx_DirtyBit) >> ESR_ELx_DirtyBit_SHIFT);
 }
 
 static void mem_abort_decode(unsigned long esr)
diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c
index 30772a909aea3..8429d220660b8 100644
--- a/arch/arm64/mm/hugetlbpage.c
+++ b/arch/arm64/mm/hugetlbpage.c
@@ -87,13 +87,18 @@ static int find_num_contig(struct mm_struct *mm, unsigned long addr,
 	p4dp = p4d_offset(pgdp, addr);
 	pudp = pud_offset(p4dp, addr);
 	pmdp = pmd_offset(pudp, addr);
-	if ((pte_t *)pmdp == ptep) {
+	if ((pte_t *)PTR_ALIGN_DOWN(pmdp, sizeof(*pmdp) * CONT_PMDS) == ptep) {
 		*pgsize = PMD_SIZE;
 		return CONT_PMDS;
 	}
 	return CONT_PTES;
 }
 
+/*
+ * num_contig_ptes(), set_huge_pte_at() and arch_make_huge_pte() can be
+ * used by non-hugetlbfs(vmalloc) mm code to set multiple huge mappings
+ * at the PTE level.
+ */
 static inline int num_contig_ptes(unsigned long size, size_t *pgsize)
 {
 	int contig_ptes = 1;
@@ -110,6 +115,12 @@ static inline int num_contig_ptes(unsigned long size, size_t *pgsize)
 		contig_ptes = CONT_PTES;
 		break;
 	default:
+		if (size > 0 && size < PMD_SIZE &&
+				IS_ALIGNED(size, CONT_PTE_SIZE)) {
+			*pgsize = PAGE_SIZE;
+			contig_ptes = size >> PAGE_SHIFT;
+			break;
+		}
 		WARN_ON(!__hugetlb_valid_size(size));
 	}
 
@@ -359,6 +370,10 @@ pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags)
 	case CONT_PTE_SIZE:
 		return pte_mkcont(entry);
 	default:
+		if (pagesize > 0 && pagesize < PMD_SIZE &&
+				IS_ALIGNED(pagesize, CONT_PTE_SIZE))
+			return pte_mkcont(entry);
+
 		break;
 	}
 	pr_warn("%s: unrecognized huge page size 0x%lx\n",
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 97987f850a33c..227f58522dad2 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -96,8 +96,8 @@ phys_addr_t __ro_after_init arm64_dma_phys_limit;
 
 static void __init arch_reserve_crashkernel(void)
 {
+	unsigned long long crash_base, crash_size, cma_size = 0;
 	unsigned long long low_size = 0;
-	unsigned long long crash_base, crash_size;
 	bool high = false;
 	int ret;
 
@@ -106,11 +106,12 @@ static void __init arch_reserve_crashkernel(void)
 
 	ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(),
 				&crash_size, &crash_base,
-				&low_size, NULL, &high);
+				&low_size, &cma_size, &high);
 	if (ret)
 		return;
 
 	reserve_crashkernel_generic(crash_size, crash_base, low_size, high);
+	reserve_crashkernel_cma(cma_size);
 }
 
 static phys_addr_t __init max_zone_phys(phys_addr_t zone_limit)
diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c
index 3fcad956fdf7c..45fbdce684c89 100644
--- a/arch/arm64/mm/kasan_init.c
+++ b/arch/arm64/mm/kasan_init.c
@@ -353,9 +353,6 @@ static void __init kasan_init_shadow(void)
 		void *start = (void *)__phys_to_virt(pa_start);
 		void *end = (void *)__phys_to_virt(pa_end);
 
-		if (start >= end)
-			break;
-
 		kasan_map_populate((unsigned long)kasan_mem_to_shadow(start),
 				   (unsigned long)kasan_mem_to_shadow(end),
 				   early_pfn_to_nid(virt_to_pfn(start)));
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index a25d8beacc831..bd52fca6e8720 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -49,8 +49,6 @@
 #define NO_CONT_MAPPINGS	BIT(1)
 #define NO_EXEC_MAPPINGS	BIT(2)	/* assumes FEAT_HPDS is not used */
 
-DEFINE_STATIC_KEY_FALSE(arm64_ptdump_lock_key);
-
 u64 kimage_voffset __ro_after_init;
 EXPORT_SYMBOL(kimage_voffset);
 
@@ -1864,8 +1862,7 @@ int pmd_clear_huge(pmd_t *pmdp)
 	return 1;
 }
 
-static int __pmd_free_pte_page(pmd_t *pmdp, unsigned long addr,
-			       bool acquire_mmap_lock)
+int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr)
 {
 	pte_t *table;
 	pmd_t pmd;
@@ -1877,25 +1874,13 @@ static int __pmd_free_pte_page(pmd_t *pmdp, unsigned long addr,
 		return 1;
 	}
 
-	/* See comment in pud_free_pmd_page for static key logic */
 	table = pte_offset_kernel(pmdp, addr);
 	pmd_clear(pmdp);
 	__flush_tlb_kernel_pgtable(addr);
-	if (static_branch_unlikely(&arm64_ptdump_lock_key) && acquire_mmap_lock) {
-		mmap_read_lock(&init_mm);
-		mmap_read_unlock(&init_mm);
-	}
-
 	pte_free_kernel(NULL, table);
 	return 1;
 }
 
-int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr)
-{
-	/* If ptdump is walking the pagetables, acquire init_mm.mmap_lock */
-	return __pmd_free_pte_page(pmdp, addr, /* acquire_mmap_lock = */ true);
-}
-
 int pud_free_pmd_page(pud_t *pudp, unsigned long addr)
 {
 	pmd_t *table;
@@ -1911,36 +1896,16 @@ int pud_free_pmd_page(pud_t *pudp, unsigned long addr)
 	}
 
 	table = pmd_offset(pudp, addr);
-
-	/*
-	 * Our objective is to prevent ptdump from reading a PMD table which has
-	 * been freed. In this race, if pud_free_pmd_page observes the key on
-	 * (which got flipped by ptdump) then the mmap lock sequence here will,
-	 * as a result of the mmap write lock/unlock sequence in ptdump, give
-	 * us the correct synchronization. If not, this means that ptdump has
-	 * yet not started walking the pagetables - the sequence of barriers
-	 * issued by __flush_tlb_kernel_pgtable() guarantees that ptdump will
-	 * observe an empty PUD.
-	 */
-	pud_clear(pudp);
-	__flush_tlb_kernel_pgtable(addr);
-	if (static_branch_unlikely(&arm64_ptdump_lock_key)) {
-		mmap_read_lock(&init_mm);
-		mmap_read_unlock(&init_mm);
-	}
-
 	pmdp = table;
 	next = addr;
 	end = addr + PUD_SIZE;
 	do {
 		if (pmd_present(pmdp_get(pmdp)))
-			/*
-			 * PMD has been isolated, so ptdump won't see it. No
-			 * need to acquire init_mm.mmap_lock.
-			 */
-			__pmd_free_pte_page(pmdp, next, /* acquire_mmap_lock = */ false);
+			pmd_free_pte_page(pmdp, next);
 	} while (pmdp++, next += PMD_SIZE, next != end);
 
+	pud_clear(pudp);
+	__flush_tlb_kernel_pgtable(addr);
 	pmd_free(NULL, table);
 	return 1;
 }
diff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c
index 1c20144700d7d..5a76c59b5ada5 100644
--- a/arch/arm64/mm/ptdump.c
+++ b/arch/arm64/mm/ptdump.c
@@ -283,13 +283,6 @@ void note_page_flush(struct ptdump_state *pt_st)
 	note_page(pt_st, 0, -1, pte_val(pte_zero));
 }
 
-static void arm64_ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm)
-{
-	static_branch_inc(&arm64_ptdump_lock_key);
-	ptdump_walk_pgd(st, mm, NULL);
-	static_branch_dec(&arm64_ptdump_lock_key);
-}
-
 void ptdump_walk(struct seq_file *s, struct ptdump_info *info)
 {
 	unsigned long end = ~0UL;
@@ -318,7 +311,7 @@ void ptdump_walk(struct seq_file *s, struct ptdump_info *info)
 		}
 	};
 
-	arm64_ptdump_walk_pgd(&st.ptdump, info->mm);
+	ptdump_walk_pgd(&st.ptdump, info->mm, NULL);
 }
 
 static void __init ptdump_initialize(void)
@@ -360,7 +353,7 @@ bool ptdump_check_wx(void)
 		}
 	};
 
-	arm64_ptdump_walk_pgd(&st.ptdump, &init_mm);
+	ptdump_walk_pgd(&st.ptdump, &init_mm, NULL);
 
 	if (st.wx_pages || st.uxn_pages) {
 		pr_warn("Checked W+X mappings: FAILED, %lu W+X pages found, %lu non-UXN pages found\n",
diff --git a/arch/csky/abiv1/inc/abi/cacheflush.h b/arch/csky/abiv1/inc/abi/cacheflush.h
index d011a81575d21..21cc7965c53d5 100644
--- a/arch/csky/abiv1/inc/abi/cacheflush.h
+++ b/arch/csky/abiv1/inc/abi/cacheflush.h
@@ -42,7 +42,12 @@ static inline void flush_anon_page(struct vm_area_struct *vma,
  * Use cache_wbinv_all() here and need to be improved in future.
  */
 extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
-#define flush_cache_vmap(start, end)		cache_wbinv_all()
+
+static inline void flush_cache_vmap(unsigned long start, unsigned long end)
+{
+	cache_wbinv_all();
+}
+
 #define flush_cache_vmap_early(start, end)	do { } while (0)
 #define flush_cache_vunmap(start, end)		cache_wbinv_all()
 
diff --git a/arch/csky/kernel/entry.S b/arch/csky/kernel/entry.S
index c68cdcc76d60e..3261f46f22442 100644
--- a/arch/csky/kernel/entry.S
+++ b/arch/csky/kernel/entry.S
@@ -93,11 +93,11 @@ csky_syscall_trace:
 	ldw	a2, (sp, LSAVE_A2)
 	ldw	a3, (sp, LSAVE_A3)
 #if defined(__CSKYABIV2__)
-	subi	sp, 8
 	ldw	r9, (sp, LSAVE_A4)
+	ldw	r10, (sp, LSAVE_A5)
+	subi	sp, 8
 	stw	r9, (sp, 0x0)
-	ldw	r9, (sp, LSAVE_A5)
-	stw	r9, (sp, 0x4)
+	stw	r10, (sp, 0x4)
 	jsr	syscallid                     /* Do system call */
 	addi	sp, 8
 #else
diff --git a/arch/csky/kernel/ptrace.c b/arch/csky/kernel/ptrace.c
index 6bb685a2646b5..ee2867a1576e9 100644
--- a/arch/csky/kernel/ptrace.c
+++ b/arch/csky/kernel/ptrace.c
@@ -320,10 +320,10 @@ long arch_ptrace(struct task_struct *child, long request,
 asmlinkage int syscall_trace_enter(struct pt_regs *regs)
 {
 	if (test_thread_flag(TIF_SYSCALL_TRACE))
-		if (ptrace_report_syscall_entry(regs))
+		if (!ptrace_report_syscall_permit_entry(regs))
 			return -1;
 
-	if (secure_computing() == -1)
+	if (!seccomp_permit_syscall())
 		return -1;
 
 	if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
diff --git a/arch/hexagon/kernel/traps.c b/arch/hexagon/kernel/traps.c
index e732aa01c2ff0..6fe0c1b62c1b3 100644
--- a/arch/hexagon/kernel/traps.c
+++ b/arch/hexagon/kernel/traps.c
@@ -345,7 +345,7 @@ void do_trap0(struct pt_regs *regs)
 
 		/* allow strace to catch syscall args  */
 		if (unlikely(test_thread_flag(TIF_SYSCALL_TRACE) &&
-			ptrace_report_syscall_entry(regs)))
+			!ptrace_report_syscall_permit_entry(regs)))
 			return;  /*  return -ENOSYS somewhere?  */
 
 		/* Interrupts should be re-enabled for syscall processing */
diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig
index d8d2523250179..a21f51e5815e9 100644
--- a/arch/loongarch/Kconfig
+++ b/arch/loongarch/Kconfig
@@ -12,7 +12,7 @@ config LOONGARCH
 	select ARCH_NEEDS_DEFER_KASAN
 	select ARCH_DISABLE_KASAN_INLINE
 	select ARCH_ENABLE_MEMORY_HOTPLUG
-	select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
+	select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE
 	select ARCH_HAS_ACPI_TABLE_UPGRADE	if ACPI
 	select ARCH_HAS_CPU_FINALIZE_INIT
 	select ARCH_HAS_CURRENT_STACK_POINTER
@@ -67,6 +67,7 @@ config LOONGARCH
 	select ARCH_SUPPORTS_LTO_CLANG
 	select ARCH_SUPPORTS_LTO_CLANG_THIN
 	select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS
+	select ARCH_HAS_PTE_PROTNONE if 64BIT
 	select ARCH_SUPPORTS_NUMA_BALANCING if NUMA
 	select ARCH_SUPPORTS_PER_VMA_LOCK
 	select ARCH_SUPPORTS_RT
@@ -153,7 +154,6 @@ config LOONGARCH
 	select HAVE_FUNCTION_GRAPH_TRACER
 	select HAVE_FUNCTION_TRACER
 	select HAVE_GCC_PLUGINS
-	select HAVE_GENERIC_VDSO
 	select HAVE_HW_BREAKPOINT if PERF_EVENTS
 	select HAVE_IOREMAP_PROT
 	select HAVE_IRQ_EXIT_ON_IRQ_STACK
@@ -213,6 +213,7 @@ config LOONGARCH
 	select TRACE_IRQFLAGS_SUPPORT
 	select USE_PERCPU_NUMA_NODE_ID
 	select USER_STACKTRACE_SUPPORT
+	select VDSO_DATASTORE
 	select VDSO_GETRANDOM
 	select ZONE_DMA32 if 64BIT
 
diff --git a/arch/loongarch/include/asm/asmmacro.h b/arch/loongarch/include/asm/asmmacro.h
index a648be5f723fc..b7423d1ac5684 100644
--- a/arch/loongarch/include/asm/asmmacro.h
+++ b/arch/loongarch/include/asm/asmmacro.h
@@ -14,7 +14,7 @@
 #ifdef CONFIG_64BIT
 #define TASK_STRUCT_OFFSET 0
 #else
-#define TASK_STRUCT_OFFSET 2000
+#define TASK_STRUCT_OFFSET 2040
 #endif
 
 	.macro	cpu_save_nonscratch thread
diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h
index 223528c04d739..1952e34bc8ee0 100644
--- a/arch/loongarch/include/asm/pgtable.h
+++ b/arch/loongarch/include/asm/pgtable.h
@@ -623,7 +623,7 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
 
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
-#ifdef CONFIG_NUMA_BALANCING
+#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE
 static inline long pte_protnone(pte_t pte)
 {
 	return (pte_val(pte) & _PAGE_PROTNONE);
@@ -633,7 +633,7 @@ static inline long pmd_protnone(pmd_t pmd)
 {
 	return (pmd_val(pmd) & _PAGE_PROTNONE);
 }
-#endif /* CONFIG_NUMA_BALANCING */
+#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */
 
 #define pmd_leaf(pmd)		((pmd_val(pmd) & _PAGE_HUGE) != 0)
 #define pud_leaf(pud)		((pud_val(pud) & _PAGE_HUGE) != 0)
diff --git a/arch/loongarch/include/asm/syscall.h b/arch/loongarch/include/asm/syscall.h
index df8ea223c77b1..d9275010f5480 100644
--- a/arch/loongarch/include/asm/syscall.h
+++ b/arch/loongarch/include/asm/syscall.h
@@ -84,10 +84,4 @@ static inline int syscall_get_arch(struct task_struct *task)
 	return AUDIT_ARCH_LOONGARCH64;
 #endif
 }
-
-static inline bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs)
-{
-	return false;
-}
-
 #endif	/* __ASM_LOONGARCH_SYSCALL_H */
diff --git a/arch/loongarch/kernel/acpi.c b/arch/loongarch/kernel/acpi.c
index 8f650c9ffecde..c5c3dc69859a6 100644
--- a/arch/loongarch/kernel/acpi.c
+++ b/arch/loongarch/kernel/acpi.c
@@ -201,10 +201,12 @@ static void __init acpi_process_madt(void)
 }
 
 int pptt_enabled;
+static int acpi_nr_packages;
+static int acpi_package_ids[MAX_PACKAGES];
 
 int __init parse_acpi_topology(void)
 {
-	int cpu, topology_id;
+	int i, cpu, topology_id;
 
 	for_each_possible_cpu(cpu) {
 		topology_id = find_acpi_cpu_topology(cpu, 0);
@@ -222,6 +224,29 @@ int __init parse_acpi_topology(void)
 
 			cpu_data[cpu].core = topology_id;
 		}
+
+		topology_id = find_acpi_cpu_topology_package(cpu);
+		if (topology_id < 0) {
+			pr_warn("Invalid BIOS PPTT\n");
+			return -ENOENT;
+		}
+
+		for (i = 0; i < acpi_nr_packages; i++)
+			if (acpi_package_ids[i] == topology_id)
+				break;
+
+		if (i == acpi_nr_packages)
+			acpi_package_ids[acpi_nr_packages++] = topology_id;
+
+		cpu_data[cpu].package = topology_id;
+	}
+
+	for_each_possible_cpu(cpu) {
+		for (int i = 0; i < acpi_nr_packages; i++)
+			if (cpu_data[cpu].package == acpi_package_ids[i]) {
+				cpu_data[cpu].package = i; /* Canonicalize */
+				break;
+			}
 	}
 
 	pptt_enabled = 1;
diff --git a/arch/loongarch/kernel/machine_kexec.c b/arch/loongarch/kernel/machine_kexec.c
index d7fafda1d5417..1883cae93bc31 100644
--- a/arch/loongarch/kernel/machine_kexec.c
+++ b/arch/loongarch/kernel/machine_kexec.c
@@ -42,6 +42,7 @@ static unsigned long first_ind_entry;
 int machine_kexec_prepare(struct kimage *kimage)
 {
 	int i;
+	char head[8];
 	char *bootloader = "kexec";
 	void *cmdline_ptr = (void *)KEXEC_CMDLINE_ADDR;
 
@@ -59,7 +60,9 @@ int machine_kexec_prepare(struct kimage *kimage)
 	} else {
 		/* Find the command line */
 		for (i = 0; i < kimage->nr_segments; i++) {
-			if (!strncmp(bootloader, (char __user *)kimage->segment[i].buf, strlen(bootloader))) {
+			if (copy_from_user(head, kimage->segment[i].buf, strlen(bootloader)))
+				continue;
+			if (!strncmp(bootloader, head, strlen(bootloader))) {
 				if (!copy_from_user(cmdline_ptr, kimage->segment[i].buf, COMMAND_LINE_SIZE))
 					kimage->arch.cmdline_ptr = (unsigned long)cmdline_ptr;
 				break;
diff --git a/arch/loongarch/kernel/machine_kexec_file.c b/arch/loongarch/kernel/machine_kexec_file.c
index 5584b798ba464..5412aa9f35687 100644
--- a/arch/loongarch/kernel/machine_kexec_file.c
+++ b/arch/loongarch/kernel/machine_kexec_file.c
@@ -56,46 +56,30 @@ static void cmdline_add_initrd(struct kimage *image, unsigned long *cmdline_tmpl
 }
 
 #ifdef CONFIG_CRASH_DUMP
-
-static int prepare_elf_headers(void **addr, unsigned long *sz)
+unsigned int arch_get_system_nr_ranges(void)
 {
-	int ret, nr_ranges;
-	uint64_t i;
+	int nr_ranges = 2; /* for exclusion of crashkernel region */
 	phys_addr_t start, end;
-	struct crash_mem *cmem;
+	uint64_t i;
 
-	nr_ranges = 2; /* for exclusion of crashkernel region */
 	for_each_mem_range(i, &start, &end)
 		nr_ranges++;
 
-	cmem = kmalloc_flex(*cmem, ranges, nr_ranges);
-	if (!cmem)
-		return -ENOMEM;
+	return nr_ranges;
+}
+
+int arch_crash_populate_cmem(struct crash_mem *cmem)
+{
+	phys_addr_t start, end;
+	uint64_t i;
 
-	cmem->max_nr_ranges = nr_ranges;
-	cmem->nr_ranges = 0;
 	for_each_mem_range(i, &start, &end) {
 		cmem->ranges[cmem->nr_ranges].start = start;
 		cmem->ranges[cmem->nr_ranges].end = end - 1;
 		cmem->nr_ranges++;
 	}
 
-	/* Exclude crashkernel region */
-	ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end);
-	if (ret < 0)
-		goto out;
-
-	if (crashk_low_res.end) {
-		ret = crash_exclude_mem_range(cmem, crashk_low_res.start, crashk_low_res.end);
-		if (ret < 0)
-			goto out;
-	}
-
-	ret = crash_prepare_elf64_headers(cmem, true, addr, sz);
-
-out:
-	kfree(cmem);
-	return ret;
+	return 0;
 }
 
 /*
@@ -163,7 +147,7 @@ int load_other_segments(struct kimage *image,
 		void *headers;
 		unsigned long headers_sz;
 
-		ret = prepare_elf_headers(&headers, &headers_sz);
+		ret = crash_prepare_headers(true, &headers, &headers_sz, NULL);
 		if (ret < 0) {
 			pr_err("Preparing elf core header failed\n");
 			goto out_err;
diff --git a/arch/loongarch/kernel/numa.c b/arch/loongarch/kernel/numa.c
index 8b89898e20df1..c96c536237151 100644
--- a/arch/loongarch/kernel/numa.c
+++ b/arch/loongarch/kernel/numa.c
@@ -216,7 +216,6 @@ static int __init fake_numa_init(void)
 	phys_addr_t start = memblock_start_of_DRAM();
 	phys_addr_t end = memblock_end_of_DRAM() - 1;
 
-	node_set(0, numa_nodes_parsed);
 	pr_info("Faking a node at [mem %pap-%pap]\n", &start, &end);
 
 	return numa_add_memblk(0, start, end + 1);
diff --git a/arch/loongarch/kernel/rethook_trampoline.S b/arch/loongarch/kernel/rethook_trampoline.S
index d4ceb2fa2a5ce..2e009fbea53f2 100644
--- a/arch/loongarch/kernel/rethook_trampoline.S
+++ b/arch/loongarch/kernel/rethook_trampoline.S
@@ -71,27 +71,27 @@
 	cfi_ld	s7, PT_R30
 	cfi_ld	s8, PT_R31
 	LONG_L  t0, sp, PT_CRMD
-	li.d	t1, 0x7 /* mask bit[1:0] PLV, bit[2] IE */
+	LONG_LI	t1, 0x7 /* mask bit[1:0] PLV, bit[2] IE */
 	csrxchg t0, t1, LOONGARCH_CSR_CRMD
 	.endm
 
 SYM_CODE_START(arch_rethook_trampoline)
 	UNWIND_HINT_UNDEFINED
-	addi.d	sp, sp, -PT_SIZE
+	PTR_ADDI sp, sp, -PT_SIZE
 	save_all_base_regs
 
-	addi.d	t0, sp, PT_SIZE
-	LONG_S	t0, sp, PT_R3
+	PTR_ADDI t0, sp, PT_SIZE
+	LONG_S	 t0, sp, PT_R3
 
-	move a0, sp /* pt_regs */
+	move	 a0, sp /* pt_regs */
 
-	bl arch_rethook_trampoline_callback
+	bl	 arch_rethook_trampoline_callback
 
 	/* use the result as the return-address */
-	move ra, a0
+	move	 ra, a0
 
 	restore_all_base_regs
-	addi.d	sp, sp, PT_SIZE
+	PTR_ADDI sp, sp, PT_SIZE
 
-	jr ra
+	jr	 ra
 SYM_CODE_END(arch_rethook_trampoline)
diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c
index 5d792256bbb99..d4b5d1b6bb015 100644
--- a/arch/loongarch/kernel/smp.c
+++ b/arch/loongarch/kernel/smp.c
@@ -426,10 +426,10 @@ void loongson_init_secondary(void)
 	numa_add_cpu(cpu);
 #endif
 	per_cpu(cpu_state, cpu) = CPU_ONLINE;
-	cpu_data[cpu].package =
-		     cpu_logical_map(cpu) / loongson_sysconf.cores_per_package;
 	cpu_data[cpu].core = pptt_enabled ? cpu_data[cpu].core :
 		     cpu_logical_map(cpu) % loongson_sysconf.cores_per_package;
+	cpu_data[cpu].package = pptt_enabled ? cpu_data[cpu].package :
+		     cpu_logical_map(cpu) / loongson_sysconf.cores_per_package;
 	cpu_data[cpu].global_id = cpu_logical_map(cpu);
 }
 
diff --git a/arch/loongarch/kernel/syscall.c b/arch/loongarch/kernel/syscall.c
index 94c1c3b5b0b5a..62264e4224673 100644
--- a/arch/loongarch/kernel/syscall.c
+++ b/arch/loongarch/kernel/syscall.c
@@ -11,7 +11,6 @@
 #include <linux/linkage.h>
 #include <linux/nospec.h>
 #include <linux/objtool.h>
-#include <linux/randomize_kstack.h>
 #include <linux/syscalls.h>
 #include <linux/unistd.h>
 
@@ -58,8 +57,8 @@ typedef long (*sys_call_fn)(unsigned long, unsigned long,
 
 void noinstr __no_stack_protector do_syscall(struct pt_regs *regs)
 {
-	unsigned long nr;
 	sys_call_fn syscall_fn;
+	unsigned long nr;
 
 	nr = regs->regs[11];
 	/* Set for syscall restarting */
@@ -70,14 +69,12 @@ void noinstr __no_stack_protector do_syscall(struct pt_regs *regs)
 	regs->orig_a0 = regs->regs[4];
 	regs->regs[4] = -ENOSYS;
 
-	nr = syscall_enter_from_user_mode(regs, nr);
-
-	add_random_kstack_offset();
-
-	if (nr < NR_syscalls) {
-		syscall_fn = sys_call_table[array_index_nospec(nr, NR_syscalls)];
-		regs->regs[4] = syscall_fn(regs->orig_a0, regs->regs[5], regs->regs[6],
-					   regs->regs[7], regs->regs[8], regs->regs[9]);
+	if (likely(syscall_enter_from_user_mode_randomize_stack(regs, &nr))) {
+		if (nr < NR_syscalls) {
+			syscall_fn = sys_call_table[array_index_nospec(nr, NR_syscalls)];
+			regs->regs[4] = syscall_fn(regs->orig_a0, regs->regs[5], regs->regs[6],
+						   regs->regs[7], regs->regs[8], regs->regs[9]);
+		}
 	}
 
 	syscall_exit_to_user_mode(regs);
diff --git a/arch/loongarch/mm/kasan_init.c b/arch/loongarch/mm/kasan_init.c
index 0fc02ca064573..92ca0ba864061 100644
--- a/arch/loongarch/mm/kasan_init.c
+++ b/arch/loongarch/mm/kasan_init.c
@@ -305,9 +305,6 @@ void __init kasan_init(void)
 		void *start = (void *)phys_to_virt(pa_start);
 		void *end   = (void *)phys_to_virt(pa_end);
 
-		if (start >= end)
-			break;
-
 		kasan_map_populate((unsigned long)mem_to_shadow(start),
 			(unsigned long)mem_to_shadow(end), NUMA_NO_NODE);
 	}
diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index 2738b4db11653..c91d474faba73 100644
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -835,7 +835,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
 			move_reg(ctx, t1, src);
 			emit_sext_32(ctx, t1, is32);
 			emit_insn(ctx, divd, dst, dst, t1);
-			emit_sext_32(ctx, dst, is32);
+			emit_zext_32(ctx, dst, is32);
 		}
 		break;
 
@@ -852,7 +852,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
 			emit_sext_32(ctx, t1, is32);
 			emit_sext_32(ctx, dst, is32);
 			emit_insn(ctx, divd, dst, dst, t1);
-			emit_sext_32(ctx, dst, is32);
+			emit_zext_32(ctx, dst, is32);
 		}
 		break;
 
@@ -870,7 +870,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
 			move_reg(ctx, t1, src);
 			emit_sext_32(ctx, t1, is32);
 			emit_insn(ctx, modd, dst, dst, t1);
-			emit_sext_32(ctx, dst, is32);
+			emit_zext_32(ctx, dst, is32);
 		}
 		break;
 
@@ -887,7 +887,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
 			emit_sext_32(ctx, t1, is32);
 			emit_sext_32(ctx, dst, is32);
 			emit_insn(ctx, modd, dst, dst, t1);
-			emit_sext_32(ctx, dst, is32);
+			emit_zext_32(ctx, dst, is32);
 		}
 		break;
 
diff --git a/arch/m68k/coldfire/device.c b/arch/m68k/coldfire/device.c
index 1420bae0964f3..9a0258acd9981 100644
--- a/arch/m68k/coldfire/device.c
+++ b/arch/m68k/coldfire/device.c
@@ -12,7 +12,7 @@
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/spi/spi.h>
-#include <linux/gpio.h>
+#include <linux/gpio/legacy.h>
 #include <linux/fec.h>
 #include <linux/dmaengine.h>
 #include <asm/traps.h>
diff --git a/arch/m68k/configs/m5235evb_defconfig b/arch/m68k/configs/m5235evb_defconfig
new file mode 100644
index 0000000000000..fcac4ccd332d2
--- /dev/null
+++ b/arch/m68k/configs/m5235evb_defconfig
@@ -0,0 +1,52 @@
+CONFIG_LOG_BUF_SHIFT=14
+CONFIG_EXPERT=y
+# CONFIG_FUTEX is not set
+# CONFIG_EPOLL is not set
+# CONFIG_SIGNALFD is not set
+# CONFIG_TIMERFD is not set
+# CONFIG_EVENTFD is not set
+# CONFIG_AIO is not set
+# CONFIG_KALLSYMS is not set
+# CONFIG_MMU is not set
+CONFIG_M523x=y
+CONFIG_M5235EVB=y
+CONFIG_RAMBASE=0x00000000
+CONFIG_RAMSIZE=0x01000000
+CONFIG_VECTORBASE=0x00000000
+CONFIG_IPSBAR=0x40000000
+CONFIG_KERNELBASE=0x00020000
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_BINFMT_FLAT=y
+# CONFIG_VM_EVENT_COUNTERS is not set
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_UNIX=y
+CONFIG_INET=y
+# CONFIG_INET_DIAG is not set
+# CONFIG_IPV6 is not set
+# CONFIG_FW_LOADER is not set
+CONFIG_MTD=y
+CONFIG_MTD_BLOCK=y
+CONFIG_MTD_RAM=y
+CONFIG_MTD_UCLINUX=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_NETDEVICES=y
+CONFIG_FEC=y
+# CONFIG_INPUT is not set
+# CONFIG_SERIO is not set
+# CONFIG_VT is not set
+# CONFIG_UNIX98_PTYS is not set
+CONFIG_SERIAL_MCF=y
+CONFIG_SERIAL_MCF_CONSOLE=y
+# CONFIG_HWMON is not set
+# CONFIG_USB_SUPPORT is not set
+CONFIG_EXT2_FS=y
+# CONFIG_FILE_LOCKING is not set
+# CONFIG_DNOTIFY is not set
+CONFIG_ROMFS_FS=y
+CONFIG_ROMFS_BACKED_BY_MTD=y
+# CONFIG_NETWORK_FILESYSTEMS is not set
+# CONFIG_SLUB_DEBUG is not set
+CONFIG_BOOTPARAM=y
+CONFIG_BOOTPARAM_STRING="root=/dev/mtdblock0"
diff --git a/arch/m68k/configs/m5282evb_defconfig b/arch/m68k/configs/m5282evb_defconfig
new file mode 100644
index 0000000000000..85040dc3e45e0
--- /dev/null
+++ b/arch/m68k/configs/m5282evb_defconfig
@@ -0,0 +1,51 @@
+CONFIG_LOG_BUF_SHIFT=14
+CONFIG_EXPERT=y
+# CONFIG_FUTEX is not set
+# CONFIG_EPOLL is not set
+# CONFIG_SIGNALFD is not set
+# CONFIG_TIMERFD is not set
+# CONFIG_EVENTFD is not set
+# CONFIG_AIO is not set
+# CONFIG_KALLSYMS is not set
+# CONFIG_MMU is not set
+CONFIG_M528x=y
+CONFIG_RAMBASE=0x00000000
+CONFIG_RAMSIZE=0x00800000
+CONFIG_VECTORBASE=0x00000000
+CONFIG_IPSBAR=0x40000000
+CONFIG_KERNELBASE=0x00020000
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_BINFMT_FLAT=y
+# CONFIG_VM_EVENT_COUNTERS is not set
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_UNIX=y
+CONFIG_INET=y
+# CONFIG_INET_DIAG is not set
+# CONFIG_IPV6 is not set
+# CONFIG_FW_LOADER is not set
+CONFIG_MTD=y
+CONFIG_MTD_BLOCK=y
+CONFIG_MTD_RAM=y
+CONFIG_MTD_UCLINUX=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_NETDEVICES=y
+CONFIG_FEC=y
+# CONFIG_INPUT is not set
+# CONFIG_SERIO is not set
+# CONFIG_VT is not set
+# CONFIG_UNIX98_PTYS is not set
+CONFIG_SERIAL_MCF=y
+CONFIG_SERIAL_MCF_CONSOLE=y
+# CONFIG_HWMON is not set
+# CONFIG_USB_SUPPORT is not set
+CONFIG_EXT2_FS=y
+# CONFIG_FILE_LOCKING is not set
+# CONFIG_DNOTIFY is not set
+CONFIG_ROMFS_FS=y
+CONFIG_ROMFS_BACKED_BY_MTD=y
+# CONFIG_NETWORK_FILESYSTEMS is not set
+# CONFIG_SLUB_DEBUG is not set
+CONFIG_BOOTPARAM=y
+CONFIG_BOOTPARAM_STRING="root=/dev/mtdblock0"
diff --git a/arch/m68k/include/asm/mcfgpio.h b/arch/m68k/include/asm/mcfgpio.h
index 7103cfa4edb6c..29726aa40eb68 100644
--- a/arch/m68k/include/asm/mcfgpio.h
+++ b/arch/m68k/include/asm/mcfgpio.h
@@ -16,7 +16,7 @@ int __mcfgpio_request(unsigned gpio);
 void __mcfgpio_free(unsigned gpio);
 
 #ifdef CONFIG_GPIOLIB
-#include <linux/gpio.h>
+#include <linux/gpio/legacy.h>
 #else
 
 /* our alternate 'gpiolib' functions */
diff --git a/arch/m68k/include/asm/page_mm.h b/arch/m68k/include/asm/page_mm.h
index ed782609ca413..0971a0651d490 100644
--- a/arch/m68k/include/asm/page_mm.h
+++ b/arch/m68k/include/asm/page_mm.h
@@ -55,10 +55,12 @@ static inline void clear_page(void *page)
 #define clear_user_page(addr, vaddr, page)	\
 	do {	clear_page(addr);		\
 		flush_dcache_page(page);	\
+		(void)(vaddr);			\
 	} while (0)
 #define copy_user_page(to, from, vaddr, page)	\
 	do {	copy_page(to, from);		\
 		flush_dcache_page(page);	\
+		(void)(vaddr);			\
 	} while (0)
 
 extern unsigned long m68k_memoffset;
diff --git a/arch/m68k/kernel/ptrace.c b/arch/m68k/kernel/ptrace.c
index cfa2df24eced1..4575f6487a9aa 100644
--- a/arch/m68k/kernel/ptrace.c
+++ b/arch/m68k/kernel/ptrace.c
@@ -279,9 +279,9 @@ asmlinkage int syscall_trace_enter(void)
 	int ret = 0;
 
 	if (test_thread_flag(TIF_SYSCALL_TRACE))
-		ret = ptrace_report_syscall_entry(task_pt_regs(current));
+		ret = !ptrace_report_syscall_permit_entry(task_pt_regs(current));
 
-	if (secure_computing() == -1)
+	if (!seccomp_permit_syscall())
 		return -1;
 
 	return ret;
diff --git a/arch/microblaze/kernel/ptrace.c b/arch/microblaze/kernel/ptrace.c
index 5234d0c1dcaad..236264e932d64 100644
--- a/arch/microblaze/kernel/ptrace.c
+++ b/arch/microblaze/kernel/ptrace.c
@@ -139,7 +139,7 @@ asmlinkage unsigned long do_syscall_trace_enter(struct pt_regs *regs)
 	secure_computing_strict(regs->r12);
 
 	if (test_thread_flag(TIF_SYSCALL_TRACE) &&
-	    ptrace_report_syscall_entry(regs))
+	    !ptrace_report_syscall_permit_entry(regs))
 		/*
 		 * Tracing decided this syscall should not happen.
 		 * We'll return a bogus call number to get an ENOSYS
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index 8555bbf47c633..a06f243173060 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -3163,7 +3163,6 @@ config MIPS_EXTERNAL_TIMER
 config MIPS_GENERIC_GETTIMEOFDAY
 	def_bool y
 	select GENERIC_GETTIMEOFDAY
-	select HAVE_GENERIC_VDSO
 	depends on CSRC_R4K || CLKSRC_MIPS_GIC
 	# GCC (at least up to version 9.2) appears to emit function calls that make use
 	# of the GOT when targeting microMIPS, which we can't use in the VDSO due to
diff --git a/arch/mips/bcm47xx/buttons.c b/arch/mips/bcm47xx/buttons.c
index 46994f9bb8219..151a4ee2803f0 100644
--- a/arch/mips/bcm47xx/buttons.c
+++ b/arch/mips/bcm47xx/buttons.c
@@ -1,9 +1,14 @@
 // SPDX-License-Identifier: GPL-2.0
 #include "bcm47xx_private.h"
 
-#include <linux/input.h>
-#include <linux/gpio_keys.h>
+#include "linux/err.h"
+#include <linux/gpio/machine.h>
+#include <linux/gpio/property.h>
+#include <linux/input-event-codes.h>
 #include <linux/interrupt.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+
 #include <bcm47xx_board.h>
 #include <bcm47xx.h>
 
@@ -11,29 +16,34 @@
  * Database
  **************************************************/
 
-#define BCM47XX_GPIO_KEY(_gpio, _code)					\
-	{								\
-		.code		= _code,				\
-		.gpio		= _gpio,				\
-		.active_low	= 1,					\
+struct bcm47xx_gpio_key {
+	u16 code;
+	u8 pin;
+	u8 flags;
+};
+
+#define BCM47XX_GPIO_KEY(_gpio, _code)			\
+	{						\
+		.code	= _code,			\
+		.pin	= _gpio,			\
+		.flags	= GPIO_ACTIVE_LOW,		\
 	}
 
-#define BCM47XX_GPIO_KEY_H(_gpio, _code)				\
-	{								\
-		.code		= _code,				\
-		.gpio		= _gpio,				\
+#define BCM47XX_GPIO_KEY_H(_gpio, _code)		\
+	{						\
+		.code	= _code,			\
+		.pin	= _gpio,			\
+		.flags	= GPIO_ACTIVE_HIGH,		\
 	}
 
 /* Asus */
 
-static const struct gpio_keys_button
-bcm47xx_buttons_asus_rtn10u[] __initconst = {
+static const struct bcm47xx_gpio_key bcm47xx_buttons_asus_rtn10u[] __initconst = {
 	BCM47XX_GPIO_KEY(20, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(21, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
-bcm47xx_buttons_asus_rtn12[] __initconst = {
+static const struct bcm47xx_gpio_key bcm47xx_buttons_asus_rtn12[] __initconst = {
 	BCM47XX_GPIO_KEY(0, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(1, KEY_RESTART),
 	BCM47XX_GPIO_KEY(4, BTN_0), /* Router mode */
@@ -41,74 +51,73 @@ bcm47xx_buttons_asus_rtn12[] __initconst = {
 	BCM47XX_GPIO_KEY(6, BTN_2), /* AP mode */
 };
 
-static const struct gpio_keys_button
-bcm47xx_buttons_asus_rtn16[] __initconst = {
+static const struct bcm47xx_gpio_key bcm47xx_buttons_asus_rtn16[] __initconst = {
 	BCM47XX_GPIO_KEY(6, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(8, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_asus_rtn66u[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(9, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_asus_wl300g[] __initconst = {
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_asus_wl320ge[] __initconst = {
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_asus_wl330ge[] __initconst = {
 	BCM47XX_GPIO_KEY(2, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_asus_wl500g[] __initconst = {
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_asus_wl500gd[] __initconst = {
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_asus_wl500gpv1[] __initconst = {
 	BCM47XX_GPIO_KEY(0, KEY_RESTART),
 	BCM47XX_GPIO_KEY(4, KEY_WPS_BUTTON),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_asus_wl500gpv2[] __initconst = {
 	BCM47XX_GPIO_KEY(2, KEY_RESTART),
 	BCM47XX_GPIO_KEY(3, KEY_WPS_BUTTON),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_asus_wl500w[] __initconst = {
 	BCM47XX_GPIO_KEY_H(6, KEY_RESTART),
 	BCM47XX_GPIO_KEY_H(7, KEY_WPS_BUTTON),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_asus_wl520gc[] __initconst = {
 	BCM47XX_GPIO_KEY(2, KEY_RESTART),
 	BCM47XX_GPIO_KEY(3, KEY_WPS_BUTTON),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_asus_wl520gu[] __initconst = {
 	BCM47XX_GPIO_KEY(2, KEY_RESTART),
 	BCM47XX_GPIO_KEY(3, KEY_WPS_BUTTON),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_asus_wl700ge[] __initconst = {
 	BCM47XX_GPIO_KEY(0, KEY_POWER), /* Hard disk power switch */
 	BCM47XX_GPIO_KEY(4, KEY_WPS_BUTTON), /* EZSetup */
@@ -116,21 +125,21 @@ bcm47xx_buttons_asus_wl700ge[] __initconst = {
 	BCM47XX_GPIO_KEY(7, KEY_RESTART), /* Hard reset */
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_asus_wlhdd[] __initconst = {
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
 /* Huawei */
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_huawei_e970[] __initconst = {
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
 /* Belkin */
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_belkin_f7d4301[] __initconst = {
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 	BCM47XX_GPIO_KEY(8, KEY_WPS_BUTTON),
@@ -138,44 +147,44 @@ bcm47xx_buttons_belkin_f7d4301[] __initconst = {
 
 /* Buffalo */
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_buffalo_whr2_a54g54[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_buffalo_whr_g125[] __initconst = {
 	BCM47XX_GPIO_KEY(0, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(4, KEY_RESTART),
 	BCM47XX_GPIO_KEY(5, BTN_0), /* Router / AP mode switch */
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_buffalo_whr_g54s[] __initconst = {
 	BCM47XX_GPIO_KEY(0, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY_H(4, KEY_RESTART),
 	BCM47XX_GPIO_KEY(5, BTN_0), /* Router / AP mode switch */
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_buffalo_whr_hp_g54[] __initconst = {
 	BCM47XX_GPIO_KEY(0, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(4, KEY_RESTART),
 	BCM47XX_GPIO_KEY(5, BTN_0), /* Router / AP mode switch */
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_buffalo_wzr_g300n[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_buffalo_wzr_rs_g54[] __initconst = {
 	BCM47XX_GPIO_KEY(0, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(4, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_buffalo_wzr_rs_g54hp[] __initconst = {
 	BCM47XX_GPIO_KEY(0, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(4, KEY_RESTART),
@@ -183,20 +192,20 @@ bcm47xx_buttons_buffalo_wzr_rs_g54hp[] __initconst = {
 
 /* Dell */
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_dell_tm2300[] __initconst = {
 	BCM47XX_GPIO_KEY(0, KEY_RESTART),
 };
 
 /* D-Link */
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_dlink_dir130[] __initconst = {
 	BCM47XX_GPIO_KEY(3, KEY_RESTART),
 	BCM47XX_GPIO_KEY(7, KEY_UNKNOWN),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_dlink_dir330[] __initconst = {
 	BCM47XX_GPIO_KEY(3, KEY_RESTART),
 	BCM47XX_GPIO_KEY(7, KEY_UNKNOWN),
@@ -204,127 +213,127 @@ bcm47xx_buttons_dlink_dir330[] __initconst = {
 
 /* Linksys */
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_e1000v1[] __initconst = {
 	BCM47XX_GPIO_KEY(5, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_e1000v21[] __initconst = {
 	BCM47XX_GPIO_KEY(9, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(10, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_e2000v1[] __initconst = {
 	BCM47XX_GPIO_KEY(5, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(8, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_e2500v3[] __initconst = {
 	BCM47XX_GPIO_KEY(9, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(10, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_e3000v1[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_e3200v1[] __initconst = {
 	BCM47XX_GPIO_KEY(5, KEY_RESTART),
 	BCM47XX_GPIO_KEY(8, KEY_WPS_BUTTON),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_e4200v1[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_wrt150nv1[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_wrt150nv11[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_wrt160nv1[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_wrt160nv3[] __initconst = {
 	BCM47XX_GPIO_KEY(5, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_wrt300n_v1[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_wrt300nv11[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_UNKNOWN),
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_wrt310nv1[] __initconst = {
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 	BCM47XX_GPIO_KEY(8, KEY_UNKNOWN),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_wrt310n_v2[] __initconst = {
 	BCM47XX_GPIO_KEY(5, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_wrt320n_v1[] __initconst = {
 	BCM47XX_GPIO_KEY(5, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(8, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_wrt54g3gv2[] __initconst = {
 	BCM47XX_GPIO_KEY(5, KEY_WIMAX),
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_wrt54g_generic[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_wrt610nv1[] __initconst = {
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 	BCM47XX_GPIO_KEY(8, KEY_WPS_BUTTON),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_wrt610nv2[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_linksys_wrtsl54gs[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
@@ -332,154 +341,154 @@ bcm47xx_buttons_linksys_wrtsl54gs[] __initconst = {
 
 /* Luxul */
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_luxul_abr_4400_v1[] = {
 	BCM47XX_GPIO_KEY(14, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_luxul_xap_310_v1[] = {
 	BCM47XX_GPIO_KEY(20, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_luxul_xap_1210_v1[] = {
 	BCM47XX_GPIO_KEY(8, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_luxul_xap_1230_v1[] = {
 	BCM47XX_GPIO_KEY(8, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_luxul_xap_1240_v1[] = {
 	BCM47XX_GPIO_KEY(8, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_luxul_xap_1500_v1[] = {
 	BCM47XX_GPIO_KEY(14, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_luxul_xbr_4400_v1[] = {
 	BCM47XX_GPIO_KEY(14, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_luxul_xvw_p30_v1[] = {
 	BCM47XX_GPIO_KEY(20, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_luxul_xwr_600_v1[] = {
 	BCM47XX_GPIO_KEY(8, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_luxul_xwr_1750_v1[] = {
 	BCM47XX_GPIO_KEY(14, KEY_RESTART),
 };
 
 /* Microsoft */
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_microsoft_nm700[] __initconst = {
 	BCM47XX_GPIO_KEY(7, KEY_RESTART),
 };
 
 /* Motorola */
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_motorola_we800g[] __initconst = {
 	BCM47XX_GPIO_KEY(0, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_motorola_wr850gp[] __initconst = {
 	BCM47XX_GPIO_KEY(5, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_motorola_wr850gv2v3[] __initconst = {
 	BCM47XX_GPIO_KEY(5, KEY_RESTART),
 };
 
 /* Netgear */
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_netgear_r6200_v1[] __initconst = {
 	BCM47XX_GPIO_KEY(2, KEY_RFKILL),
 	BCM47XX_GPIO_KEY(3, KEY_RESTART),
 	BCM47XX_GPIO_KEY(4, KEY_WPS_BUTTON),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_netgear_r6300_v1[] __initconst = {
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_netgear_wn2500rp_v1[] __initconst = {
 	BCM47XX_GPIO_KEY(12, KEY_RESTART),
 	BCM47XX_GPIO_KEY(31, KEY_WPS_BUTTON),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_netgear_wndr3400v1[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_RESTART),
 	BCM47XX_GPIO_KEY(6, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(8, KEY_RFKILL),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_netgear_wndr3400_v3[] __initconst = {
 	BCM47XX_GPIO_KEY(12, KEY_RESTART),
 	BCM47XX_GPIO_KEY(23, KEY_WPS_BUTTON),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_netgear_wndr3700v3[] __initconst = {
 	BCM47XX_GPIO_KEY(2, KEY_RFKILL),
 	BCM47XX_GPIO_KEY(3, KEY_RESTART),
 	BCM47XX_GPIO_KEY(4, KEY_WPS_BUTTON),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_netgear_wndr4500v1[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(5, KEY_RFKILL),
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_netgear_wnr1000_v3[] __initconst = {
 	BCM47XX_GPIO_KEY(2, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(3, KEY_RESTART),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_netgear_wnr3500lv1[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_RESTART),
 	BCM47XX_GPIO_KEY(6, KEY_WPS_BUTTON),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_netgear_wnr3500lv2[] __initconst = {
 	BCM47XX_GPIO_KEY(4, KEY_RESTART),
 	BCM47XX_GPIO_KEY(6, KEY_WPS_BUTTON),
 	BCM47XX_GPIO_KEY(8, KEY_RFKILL),
 };
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_netgear_wnr834bv2[] __initconst = {
 	BCM47XX_GPIO_KEY(6, KEY_RESTART),
 };
 
 /* SimpleTech */
 
-static const struct gpio_keys_button
+static const struct bcm47xx_gpio_key
 bcm47xx_buttons_simpletech_simpleshare[] __initconst = {
 	BCM47XX_GPIO_KEY(0, KEY_RESTART),
 };
@@ -488,31 +497,96 @@ bcm47xx_buttons_simpletech_simpleshare[] __initconst = {
  * Init
  **************************************************/
 
-static struct gpio_keys_platform_data bcm47xx_button_pdata;
-
-static struct platform_device bcm47xx_buttons_gpio_keys = {
-	.name = "gpio-keys",
-	.dev = {
-		.platform_data = &bcm47xx_button_pdata,
+static int __init
+bcm47xx_buttons_add(const struct bcm47xx_gpio_key *buttons, int nbuttons)
+{
+	struct platform_device *pdev;
+	const struct software_node *gpio_swnode;
+	struct property_entry *p;
+	int error;
+	int i;
+
+	switch (bcm47xx_bus_type) {
+#ifdef CONFIG_BCM47XX_BCMA
+	case BCM47XX_BUS_TYPE_BCMA:
+		gpio_swnode = &bcma_gpio_swnode;
+		break;
+#endif
+#ifdef CONFIG_BCM47XX_SSB
+	case BCM47XX_BUS_TYPE_SSB:
+		gpio_swnode = &ssb_gpio_swnode;
+		break;
+#endif
+	default:
+		return -ENODEV;
 	}
-};
 
-/* Copy data from __initconst */
-static int __init bcm47xx_buttons_copy(const struct gpio_keys_button *buttons,
-				       size_t nbuttons)
-{
-	size_t size = nbuttons * sizeof(*buttons);
+	/* 1 node for gpio-keys device, 1 node for each button, 1 terminator */
+	const struct software_node **node_group __free(kfree) =
+		kcalloc(1 + nbuttons + 1, sizeof(*node_group), GFP_KERNEL);
+	if (!node_group)
+		return -ENOMEM;
 
-	bcm47xx_button_pdata.buttons = kmemdup(buttons, size, GFP_KERNEL);
-	if (!bcm47xx_button_pdata.buttons)
+	/* 1 code property, 1 gpio property, 1 terminator */
+	struct property_entry *props __free(kfree) =
+		kcalloc(nbuttons * 3, sizeof(*props), GFP_KERNEL);
+	if (!props)
 		return -ENOMEM;
-	bcm47xx_button_pdata.nbuttons = nbuttons;
 
+	/* 1 node for gpio-keys device, 1 node for each button */
+	struct software_node *nodes __free(kfree) =
+		kcalloc(1 + nbuttons, sizeof(*nodes), GFP_KERNEL);
+	if (!nodes)
+		return -ENOMEM;
+
+	struct software_node_ref_args *ref_args __free(kfree) =
+		kcalloc(nbuttons, sizeof(*ref_args), GFP_KERNEL);
+	if (!ref_args)
+		return -ENOMEM;
+
+	/* gpio-keys node */
+	nodes[0].name = "bcm47xx-gpio-buttons";
+
+	p = props;
+	for (i = 0; i < nbuttons; i++) {
+		const struct bcm47xx_gpio_key *button = &buttons[i];
+		struct software_node *node = &nodes[1 + i];
+		struct software_node_ref_args *ref = &ref_args[i];
+
+		node->parent = &nodes[0];
+		node->properties = p;
+
+		*ref = SOFTWARE_NODE_REFERENCE(gpio_swnode, button->pin, button->flags);
+		*p++ = PROPERTY_ENTRY_REF("gpios", &ref_args[i]);
+		*p++ = PROPERTY_ENTRY_U32("linux,code", button->code);
+		p++;
+	}
+
+	for (i = 0; i < nbuttons + 1; i++)
+		node_group[i] = &nodes[i];
+
+	error = software_node_register_node_group(node_group);
+	if (error)
+		return error;
+
+	pdev = platform_device_register_full(&(struct platform_device_info){
+		.name = "gpio-keys",
+		.swnode = &nodes[0],
+	});
+	error = PTR_ERR_OR_ZERO(pdev);
+	if (error) {
+		software_node_unregister_node_group(node_group);
+		return error;
+	}
+
+	retain_and_null_ptr(props);
+	retain_and_null_ptr(nodes);
+	retain_and_null_ptr(ref_args);
 	return 0;
 }
 
-#define bcm47xx_copy_bdata(dev_buttons)					\
-	bcm47xx_buttons_copy(dev_buttons, ARRAY_SIZE(dev_buttons));
+#define bcm47xx_add_bdata(dev_buttons)					\
+	bcm47xx_buttons_add(dev_buttons, ARRAY_SIZE(dev_buttons))
 
 int __init bcm47xx_buttons_register(void)
 {
@@ -521,52 +595,52 @@ int __init bcm47xx_buttons_register(void)
 
 	switch (board) {
 	case BCM47XX_BOARD_ASUS_RTN10U:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_rtn10u);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_asus_rtn10u);
 		break;
 	case BCM47XX_BOARD_ASUS_RTN12:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_rtn12);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_asus_rtn12);
 		break;
 	case BCM47XX_BOARD_ASUS_RTN16:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_rtn16);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_asus_rtn16);
 		break;
 	case BCM47XX_BOARD_ASUS_RTN66U:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_rtn66u);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_asus_rtn66u);
 		break;
 	case BCM47XX_BOARD_ASUS_WL300G:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_wl300g);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_asus_wl300g);
 		break;
 	case BCM47XX_BOARD_ASUS_WL320GE:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_wl320ge);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_asus_wl320ge);
 		break;
 	case BCM47XX_BOARD_ASUS_WL330GE:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_wl330ge);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_asus_wl330ge);
 		break;
 	case BCM47XX_BOARD_ASUS_WL500G:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_wl500g);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_asus_wl500g);
 		break;
 	case BCM47XX_BOARD_ASUS_WL500GD:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_wl500gd);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_asus_wl500gd);
 		break;
 	case BCM47XX_BOARD_ASUS_WL500GPV1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_wl500gpv1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_asus_wl500gpv1);
 		break;
 	case BCM47XX_BOARD_ASUS_WL500GPV2:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_wl500gpv2);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_asus_wl500gpv2);
 		break;
 	case BCM47XX_BOARD_ASUS_WL500W:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_wl500w);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_asus_wl500w);
 		break;
 	case BCM47XX_BOARD_ASUS_WL520GC:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_wl520gc);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_asus_wl520gc);
 		break;
 	case BCM47XX_BOARD_ASUS_WL520GU:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_wl520gu);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_asus_wl520gu);
 		break;
 	case BCM47XX_BOARD_ASUS_WL700GE:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_wl700ge);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_asus_wl700ge);
 		break;
 	case BCM47XX_BOARD_ASUS_WLHDD:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_asus_wlhdd);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_asus_wlhdd);
 		break;
 
 	case BCM47XX_BOARD_BELKIN_F7D3301:
@@ -574,193 +648,193 @@ int __init bcm47xx_buttons_register(void)
 	case BCM47XX_BOARD_BELKIN_F7D4301:
 	case BCM47XX_BOARD_BELKIN_F7D4302:
 	case BCM47XX_BOARD_BELKIN_F7D4401:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_belkin_f7d4301);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_belkin_f7d4301);
 		break;
 
 	case BCM47XX_BOARD_BUFFALO_WHR2_A54G54:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_buffalo_whr2_a54g54);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_buffalo_whr2_a54g54);
 		break;
 	case BCM47XX_BOARD_BUFFALO_WHR_G125:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_buffalo_whr_g125);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_buffalo_whr_g125);
 		break;
 	case BCM47XX_BOARD_BUFFALO_WHR_G54S:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_buffalo_whr_g54s);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_buffalo_whr_g54s);
 		break;
 	case BCM47XX_BOARD_BUFFALO_WHR_HP_G54:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_buffalo_whr_hp_g54);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_buffalo_whr_hp_g54);
 		break;
 	case BCM47XX_BOARD_BUFFALO_WZR_G300N:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_buffalo_wzr_g300n);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_buffalo_wzr_g300n);
 		break;
 	case BCM47XX_BOARD_BUFFALO_WZR_RS_G54:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_buffalo_wzr_rs_g54);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_buffalo_wzr_rs_g54);
 		break;
 	case BCM47XX_BOARD_BUFFALO_WZR_RS_G54HP:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_buffalo_wzr_rs_g54hp);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_buffalo_wzr_rs_g54hp);
 		break;
 
 	case BCM47XX_BOARD_DELL_TM2300:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_dell_tm2300);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_dell_tm2300);
 		break;
 
 	case BCM47XX_BOARD_DLINK_DIR130:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_dlink_dir130);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_dlink_dir130);
 		break;
 	case BCM47XX_BOARD_DLINK_DIR330:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_dlink_dir330);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_dlink_dir330);
 		break;
 
 	case BCM47XX_BOARD_HUAWEI_E970:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_huawei_e970);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_huawei_e970);
 		break;
 
 	case BCM47XX_BOARD_LINKSYS_E1000V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_e1000v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_e1000v1);
 		break;
 	case BCM47XX_BOARD_LINKSYS_E1000V21:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_e1000v21);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_e1000v21);
 		break;
 	case BCM47XX_BOARD_LINKSYS_E2000V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_e2000v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_e2000v1);
 		break;
 	case BCM47XX_BOARD_LINKSYS_E2500V3:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_e2500v3);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_e2500v3);
 		break;
 	case BCM47XX_BOARD_LINKSYS_E3000V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_e3000v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_e3000v1);
 		break;
 	case BCM47XX_BOARD_LINKSYS_E3200V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_e3200v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_e3200v1);
 		break;
 	case BCM47XX_BOARD_LINKSYS_E4200V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_e4200v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_e4200v1);
 		break;
 	case BCM47XX_BOARD_LINKSYS_WRT150NV1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrt150nv1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_wrt150nv1);
 		break;
 	case BCM47XX_BOARD_LINKSYS_WRT150NV11:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrt150nv11);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_wrt150nv11);
 		break;
 	case BCM47XX_BOARD_LINKSYS_WRT160NV1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrt160nv1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_wrt160nv1);
 		break;
 	case BCM47XX_BOARD_LINKSYS_WRT160NV3:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrt160nv3);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_wrt160nv3);
 		break;
 	case BCM47XX_BOARD_LINKSYS_WRT300N_V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrt300n_v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_wrt300n_v1);
 		break;
 	case BCM47XX_BOARD_LINKSYS_WRT300NV11:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrt300nv11);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_wrt300nv11);
 		break;
 	case BCM47XX_BOARD_LINKSYS_WRT310NV1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrt310nv1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_wrt310nv1);
 		break;
 	case BCM47XX_BOARD_LINKSYS_WRT310NV2:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrt310n_v2);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_wrt310n_v2);
 		break;
 	case BCM47XX_BOARD_LINKSYS_WRT320N_V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrt320n_v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_wrt320n_v1);
 		break;
 	case BCM47XX_BOARD_LINKSYS_WRT54G3GV2:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrt54g3gv2);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_wrt54g3gv2);
 		break;
 	case BCM47XX_BOARD_LINKSYS_WRT54G_TYPE_0101:
 	case BCM47XX_BOARD_LINKSYS_WRT54G_TYPE_0467:
 	case BCM47XX_BOARD_LINKSYS_WRT54G_TYPE_0708:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrt54g_generic);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_wrt54g_generic);
 		break;
 	case BCM47XX_BOARD_LINKSYS_WRT610NV1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrt610nv1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_wrt610nv1);
 		break;
 	case BCM47XX_BOARD_LINKSYS_WRT610NV2:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrt610nv2);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_wrt610nv2);
 		break;
 	case BCM47XX_BOARD_LINKSYS_WRTSL54GS:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_linksys_wrtsl54gs);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_linksys_wrtsl54gs);
 		break;
 
 	case BCM47XX_BOARD_LUXUL_ABR_4400_V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_luxul_abr_4400_v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_luxul_abr_4400_v1);
 		break;
 	case BCM47XX_BOARD_LUXUL_XAP_310_V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_luxul_xap_310_v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_luxul_xap_310_v1);
 		break;
 	case BCM47XX_BOARD_LUXUL_XAP_1210_V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_luxul_xap_1210_v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_luxul_xap_1210_v1);
 		break;
 	case BCM47XX_BOARD_LUXUL_XAP_1230_V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_luxul_xap_1230_v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_luxul_xap_1230_v1);
 		break;
 	case BCM47XX_BOARD_LUXUL_XAP_1240_V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_luxul_xap_1240_v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_luxul_xap_1240_v1);
 		break;
 	case BCM47XX_BOARD_LUXUL_XAP_1500_V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_luxul_xap_1500_v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_luxul_xap_1500_v1);
 		break;
 	case BCM47XX_BOARD_LUXUL_XBR_4400_V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_luxul_xbr_4400_v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_luxul_xbr_4400_v1);
 		break;
 	case BCM47XX_BOARD_LUXUL_XVW_P30_V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_luxul_xvw_p30_v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_luxul_xvw_p30_v1);
 		break;
 	case BCM47XX_BOARD_LUXUL_XWR_600_V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_luxul_xwr_600_v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_luxul_xwr_600_v1);
 		break;
 	case BCM47XX_BOARD_LUXUL_XWR_1750_V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_luxul_xwr_1750_v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_luxul_xwr_1750_v1);
 		break;
 
 	case BCM47XX_BOARD_MICROSOFT_MN700:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_microsoft_nm700);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_microsoft_nm700);
 		break;
 
 	case BCM47XX_BOARD_MOTOROLA_WE800G:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_motorola_we800g);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_motorola_we800g);
 		break;
 	case BCM47XX_BOARD_MOTOROLA_WR850GP:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_motorola_wr850gp);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_motorola_wr850gp);
 		break;
 	case BCM47XX_BOARD_MOTOROLA_WR850GV2V3:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_motorola_wr850gv2v3);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_motorola_wr850gv2v3);
 		break;
 
 	case BCM47XX_BOARD_NETGEAR_R6200_V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_r6200_v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_netgear_r6200_v1);
 		break;
 	case BCM47XX_BOARD_NETGEAR_R6300_V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_r6300_v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_netgear_r6300_v1);
 		break;
 	case BCM47XX_BOARD_NETGEAR_WN2500RP_V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_wn2500rp_v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_netgear_wn2500rp_v1);
 		break;
 	case BCM47XX_BOARD_NETGEAR_WNDR3400V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_wndr3400v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_netgear_wndr3400v1);
 		break;
 	case BCM47XX_BOARD_NETGEAR_WNDR3400_V3:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_wndr3400_v3);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_netgear_wndr3400_v3);
 		break;
 	case BCM47XX_BOARD_NETGEAR_WNDR3700V3:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_wndr3700v3);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_netgear_wndr3700v3);
 		break;
 	case BCM47XX_BOARD_NETGEAR_WNDR4500V1:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_wndr4500v1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_netgear_wndr4500v1);
 		break;
 	case BCM47XX_BOARD_NETGEAR_WNR1000_V3:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_wnr1000_v3);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_netgear_wnr1000_v3);
 		break;
 	case BCM47XX_BOARD_NETGEAR_WNR3500L:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_wnr3500lv1);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_netgear_wnr3500lv1);
 		break;
 	case BCM47XX_BOARD_NETGEAR_WNR3500L_V2:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_wnr3500lv2);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_netgear_wnr3500lv2);
 		break;
 	case BCM47XX_BOARD_NETGEAR_WNR834BV2:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_netgear_wnr834bv2);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_netgear_wnr834bv2);
 		break;
 
 	case BCM47XX_BOARD_SIMPLETECH_SIMPLESHARE:
-		err = bcm47xx_copy_bdata(bcm47xx_buttons_simpletech_simpleshare);
+		err = bcm47xx_add_bdata(bcm47xx_buttons_simpletech_simpleshare);
 		break;
 
 	default:
@@ -769,13 +843,7 @@ int __init bcm47xx_buttons_register(void)
 	}
 
 	if (err)
-		return -ENOMEM;
-
-	err = platform_device_register(&bcm47xx_buttons_gpio_keys);
-	if (err) {
-		pr_err("Failed to register platform device: %d\n", err);
 		return err;
-	}
 
 	return 0;
 }
diff --git a/arch/mips/include/asm/gio_device.h b/arch/mips/include/asm/gio_device.h
index f9c102f038d6a..6e9d2897347d6 100644
--- a/arch/mips/include/asm/gio_device.h
+++ b/arch/mips/include/asm/gio_device.h
@@ -1,6 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #include <linux/device.h>
-#include <linux/mod_devicetable.h>
 
 struct gio_device_id {
 	__u8 id;
diff --git a/arch/mips/include/asm/txx9/generic.h b/arch/mips/include/asm/txx9/generic.h
index 9a2c47bf3c404..653315fb6ca55 100644
--- a/arch/mips/include/asm/txx9/generic.h
+++ b/arch/mips/include/asm/txx9/generic.h
@@ -45,7 +45,6 @@ extern int (*txx9_irq_dispatch)(int pending);
 const char *prom_getenv(const char *name);
 void txx9_wdt_init(unsigned long base);
 void txx9_wdt_now(unsigned long base);
-void txx9_spi_init(int busid, unsigned long base, int irq);
 void txx9_ethaddr_init(unsigned int id, unsigned char *ethaddr);
 void txx9_sio_init(unsigned long baseaddr, int irq,
 		   unsigned int line, unsigned int sclk, int nocts);
@@ -80,15 +79,9 @@ static inline unsigned int __fls8(unsigned char x)
 	return r;
 }
 
-void txx9_iocled_init(unsigned long baseaddr,
-		      int basenum, unsigned int num, int lowactive,
+void txx9_iocled_init(unsigned long baseaddr, unsigned int num,
 		      const char *color, char **deftriggers);
 
-/* 7SEG LED */
-void txx9_7segled_init(unsigned int num,
-		       void (*putc)(unsigned int pos, unsigned char val));
-int txx9_7segled_putc(unsigned int pos, char c);
-
 void __init txx9_aclc_init(unsigned long baseaddr, int irq,
 			   unsigned int dmac_id,
 			   unsigned int dma_chan_out,
diff --git a/arch/mips/include/asm/txx9/tx4938.h b/arch/mips/include/asm/txx9/tx4938.h
index 6ca767ee6467c..765f7d6a44d36 100644
--- a/arch/mips/include/asm/txx9/tx4938.h
+++ b/arch/mips/include/asm/txx9/tx4938.h
@@ -281,7 +281,6 @@ void tx4938_wdt_init(void);
 void tx4938_setup(void);
 void tx4938_time_init(unsigned int tmrnr);
 void tx4938_sio_init(unsigned int sclk, unsigned int cts_mask);
-void tx4938_spi_init(int busid);
 void tx4938_ethaddr_init(unsigned char *addr0, unsigned char *addr1);
 int tx4938_report_pciclk(void);
 void tx4938_report_pci1clk(void);
diff --git a/arch/mips/include/asm/txx9pio.h b/arch/mips/include/asm/txx9pio.h
index 3d6fa9f8d5135..c33a977244005 100644
--- a/arch/mips/include/asm/txx9pio.h
+++ b/arch/mips/include/asm/txx9pio.h
@@ -23,7 +23,6 @@ struct txx9_pio_reg {
 	__u32 maskext;
 };
 
-int txx9_gpio_init(unsigned long baseaddr,
-		   unsigned int base, unsigned int num);
+int txx9_gpio_init(unsigned long baseaddr, unsigned int num);
 
 #endif /* __ASM_TXX9PIO_H */
diff --git a/arch/mips/kernel/gpio_txx9.c b/arch/mips/kernel/gpio_txx9.c
index 96ac40d20c238..b2e73263d94c8 100644
--- a/arch/mips/kernel/gpio_txx9.c
+++ b/arch/mips/kernel/gpio_txx9.c
@@ -76,13 +76,12 @@ static struct gpio_chip txx9_gpio_chip = {
 	.label = "TXx9",
 };
 
-int __init txx9_gpio_init(unsigned long baseaddr,
-			  unsigned int base, unsigned int num)
+int __init txx9_gpio_init(unsigned long baseaddr, unsigned int num)
 {
 	txx9_pioptr = ioremap(baseaddr, sizeof(struct txx9_pio_reg));
 	if (!txx9_pioptr)
 		return -ENODEV;
-	txx9_gpio_chip.base = base;
+	txx9_gpio_chip.base = -1;
 	txx9_gpio_chip.ngpio = num;
 	return gpiochip_add_data(&txx9_gpio_chip, NULL);
 }
diff --git a/arch/mips/kernel/proc.c b/arch/mips/kernel/proc.c
index 5401c679813ab..5f49fd4653a76 100644
--- a/arch/mips/kernel/proc.c
+++ b/arch/mips/kernel/proc.c
@@ -109,7 +109,7 @@ static int show_cpuinfo(struct seq_file *m, void *v)
 		seq_puts(m, " mips64r5");
 	if (cpu_has_mips64r6)
 		seq_puts(m, " mips64r6");
-	seq_puts(m, "\n");
+	seq_putc(m, '\n');
 
 	seq_puts(m, "ASEs implemented\t:");
 	if (cpu_has_mips16)
@@ -273,7 +273,7 @@ static int show_cpuinfo(struct seq_file *m, void *v)
 		seq_puts(m, " mm_sysad");
 	if (cpu_has_mm_full)
 		seq_puts(m, " mm_full");
-	seq_puts(m, "\n");
+	seq_putc(m, '\n');
 
 	seq_printf(m, "shadow register sets\t: %d\n",
 		      cpu_data[n].srsets);
diff --git a/arch/mips/kernel/ptrace.c b/arch/mips/kernel/ptrace.c
index 3f4c94c881241..5a0e298857b65 100644
--- a/arch/mips/kernel/ptrace.c
+++ b/arch/mips/kernel/ptrace.c
@@ -1324,11 +1324,11 @@ asmlinkage long syscall_trace_enter(struct pt_regs *regs)
 	user_exit();
 
 	if (test_thread_flag(TIF_SYSCALL_TRACE)) {
-		if (ptrace_report_syscall_entry(regs))
+		if (!ptrace_report_syscall_permit_entry(regs))
 			return -1;
 	}
 
-	if (secure_computing())
+	if (!seccomp_permit_syscall())
 		return -1;
 
 	if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT)))
diff --git a/arch/mips/kernel/vdso.c b/arch/mips/kernel/vdso.c
index bd1fc17d3975d..8165dd3fc1f86 100644
--- a/arch/mips/kernel/vdso.c
+++ b/arch/mips/kernel/vdso.c
@@ -91,8 +91,8 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
 
 		/* Map delay slot emulation page */
 		base = do_mmap(NULL, STACK_TOP, PAGE_SIZE, PROT_READ | PROT_EXEC,
-			       MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, 0, 0, &unused,
-			       NULL);
+			       MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED,
+			       EMPTY_VMA_FLAGS, 0, &unused, NULL);
 		if (IS_ERR_VALUE(base)) {
 			ret = base;
 			goto out;
@@ -129,7 +129,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
 	 * This ensures that when the kernel updates the VDSO data userland
 	 * will observe it without requiring cache invalidations.
 	 */
-	if (cpu_has_dc_aliases && IS_ENABLED(CONFIG_HAVE_GENERIC_VDSO)) {
+	if (cpu_has_dc_aliases && IS_ENABLED(CONFIG_MIPS_GENERIC_GETTIMEOFDAY)) {
 		base = __ALIGN_MASK(base, shm_align_mask);
 		base += ((unsigned long)vdso_k_time_data - gic_size) & shm_align_mask;
 	}
@@ -137,7 +137,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
 	data_addr = base + gic_size;
 	vdso_addr = data_addr + VDSO_NR_PAGES * PAGE_SIZE;
 
-	if (IS_ENABLED(CONFIG_HAVE_GENERIC_VDSO)) {
+	if (IS_ENABLED(CONFIG_MIPS_GENERIC_GETTIMEOFDAY)) {
 		vma = vdso_install_vvar_mapping(mm, data_addr);
 		if (IS_ERR(vma)) {
 			ret = PTR_ERR(vma);
diff --git a/arch/mips/txx9/Kconfig b/arch/mips/txx9/Kconfig
index 92b759a434c0e..7335efa4d5280 100644
--- a/arch/mips/txx9/Kconfig
+++ b/arch/mips/txx9/Kconfig
@@ -37,7 +37,6 @@ config SOC_TX4927
 	select IRQ_TXX9
 	select PCI_TX4927
 	select GPIO_TXX9
-	select GPIOLIB_LEGACY
 
 config SOC_TX4938
 	bool
diff --git a/arch/mips/txx9/generic/setup.c b/arch/mips/txx9/generic/setup.c
index 6c5025806914f..5f66236038c8f 100644
--- a/arch/mips/txx9/generic/setup.c
+++ b/arch/mips/txx9/generic/setup.c
@@ -19,6 +19,7 @@
 #include <linux/clkdev.h>
 #include <linux/err.h>
 #include <linux/gpio/driver.h>
+#include <linux/gpio/machine.h>
 #include <linux/platform_device.h>
 #include <linux/platform_data/txx9/ndfmc.h>
 #include <linux/serial_core.h>
@@ -341,23 +342,6 @@ void txx9_wdt_now(unsigned long base)
 		     &tmrptr->tcr);
 }
 
-/* SPI support */
-void __init txx9_spi_init(int busid, unsigned long base, int irq)
-{
-	struct resource res[] = {
-		{
-			.start	= base,
-			.end	= base + 0x20 - 1,
-			.flags	= IORESOURCE_MEM,
-		}, {
-			.start	= irq,
-			.flags	= IORESOURCE_IRQ,
-		},
-	};
-	platform_device_register_simple("spi_txx9", busid,
-					res, ARRAY_SIZE(res));
-}
-
 void __init txx9_ethaddr_init(unsigned int id, unsigned char *ethaddr)
 {
 	struct platform_device *pdev =
@@ -586,7 +570,7 @@ void __init txx9_ndfmc_init(unsigned long baseaddr,
 #if IS_ENABLED(CONFIG_LEDS_GPIO)
 static DEFINE_SPINLOCK(txx9_iocled_lock);
 
-#define TXX9_IOCLED_MAXLEDS 8
+#define TXX9_IOCLED_MAXLEDS 3	/* rbtx4927 */
 
 struct txx9_iocled_data {
 	struct gpio_chip chip;
@@ -632,8 +616,16 @@ static int txx9_iocled_dir_out(struct gpio_chip *chip, unsigned int offset,
 	return 0;
 }
 
-void __init txx9_iocled_init(unsigned long baseaddr,
-			     int basenum, unsigned int num, int lowactive,
+static struct gpiod_lookup_table txx9_iocled_table = {
+	.table = {
+		GPIO_LOOKUP_IDX("iocled", 0, NULL, 0, GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP_IDX("iocled", 1, NULL, 1, GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP_IDX("iocled", 2, NULL, 2, GPIO_ACTIVE_LOW),
+		{ },
+	},
+};
+
+void __init txx9_iocled_init(unsigned long baseaddr, unsigned int num,
 			     const char *color, char **deftriggers)
 {
 	struct txx9_iocled_data *iocled;
@@ -659,14 +651,12 @@ void __init txx9_iocled_init(unsigned long baseaddr,
 	iocled->chip.direction_input = txx9_iocled_dir_in;
 	iocled->chip.direction_output = txx9_iocled_dir_out;
 	iocled->chip.label = "iocled";
-	iocled->chip.base = basenum;
+	iocled->chip.base = -1;
 	iocled->chip.ngpio = num;
 	if (gpiochip_add_data(&iocled->chip, iocled))
 		goto out_unmap;
-	if (basenum < 0)
-		basenum = iocled->chip.base;
 
-	pdev = platform_device_alloc("leds-gpio", basenum);
+	pdev = platform_device_alloc("leds-gpio", iocled->chip.base);
 	if (!pdev)
 		goto out_gpio;
 	iocled->pdata.num_leds = num;
@@ -676,14 +666,14 @@ void __init txx9_iocled_init(unsigned long baseaddr,
 		snprintf(iocled->names[i], sizeof(iocled->names[i]),
 			 "iocled:%s:%u", color, i);
 		led->name = iocled->names[i];
-		led->gpio = basenum + i;
-		led->active_low = lowactive;
 		if (deftriggers && *deftriggers)
 			led->default_trigger = *deftriggers++;
 	}
 	pdev->dev.platform_data = &iocled->pdata;
 	if (platform_device_add(pdev))
 		goto out_pdev;
+	txx9_iocled_table.dev_id = dev_name(&pdev->dev);
+	gpiod_add_lookup_table(&txx9_iocled_table);
 	return;
 
 out_pdev:
@@ -696,8 +686,7 @@ void __init txx9_iocled_init(unsigned long baseaddr,
 	kfree(iocled);
 }
 #else /* CONFIG_LEDS_GPIO */
-void __init txx9_iocled_init(unsigned long baseaddr,
-			     int basenum, unsigned int num, int lowactive,
+void __init txx9_iocled_init(unsigned long baseaddr, unsigned int num,
 			     const char *color, char **deftriggers)
 {
 }
diff --git a/arch/mips/txx9/generic/setup_tx4938.c b/arch/mips/txx9/generic/setup_tx4938.c
index ba646548c5f69..bfd6540f8f6a4 100644
--- a/arch/mips/txx9/generic/setup_tx4938.c
+++ b/arch/mips/txx9/generic/setup_tx4938.c
@@ -301,12 +301,6 @@ void __init tx4938_sio_init(unsigned int sclk, unsigned int cts_mask)
 	}
 }
 
-void __init tx4938_spi_init(int busid)
-{
-	txx9_spi_init(busid, TX4938_SPI_REG & 0xfffffffffULL,
-		      TXX9_IRQ_BASE + TX4938_IR_SPI);
-}
-
 void __init tx4938_ethaddr_init(unsigned char *addr0, unsigned char *addr1)
 {
 	u64 pcfg = __raw_readq(&tx4938_ccfgptr->pcfg);
diff --git a/arch/mips/txx9/rbtx4927/setup.c b/arch/mips/txx9/rbtx4927/setup.c
index 31955c1d55550..223889c031866 100644
--- a/arch/mips/txx9/rbtx4927/setup.c
+++ b/arch/mips/txx9/rbtx4927/setup.c
@@ -49,6 +49,7 @@
 #include <linux/platform_device.h>
 #include <linux/delay.h>
 #include <linux/gpio.h>
+#include <linux/gpio/machine.h>
 #include <linux/leds.h>
 #include <asm/io.h>
 #include <asm/reboot.h>
@@ -157,18 +158,27 @@ static inline void tx4927_pci_setup(void) {}
 static inline void tx4937_pci_setup(void) {}
 #endif /* CONFIG_PCI */
 
+/* TX4927-SIO DTR on (PIO[15]) */
+GPIO_LOOKUP_SINGLE(sio_gpio_table, NULL, "TXx9", 15, "sio-dtr",
+		   GPIO_ACTIVE_HIGH);
+
 static void __init rbtx4927_gpio_init(void)
 {
-	/* TX4927-SIO DTR on (PIO[15]) */
-	gpio_request(15, "sio-dtr");
-	gpio_direction_output(15, 1);
+	struct gpio_desc *d;
+
+	gpiod_add_lookup_table(&sio_gpio_table);
+	d = gpiod_get(NULL, "sio-dtr", GPIOD_OUT_HIGH);
+	if (IS_ERR(d))
+		pr_err("Unable to get sio-dtr GPIO descriptor\n");
+	else
+		gpiod_put(d);
 
 	tx4927_sio_init(0, 0);
 }
 
 static void __init rbtx4927_arch_init(void)
 {
-	txx9_gpio_init(TX4927_PIO_REG & 0xfffffffffULL, 0, TX4927_NUM_PIO);
+	txx9_gpio_init(TX4927_PIO_REG & 0xfffffffffULL, TX4927_NUM_PIO);
 
 	rbtx4927_gpio_init();
 
@@ -177,7 +187,7 @@ static void __init rbtx4927_arch_init(void)
 
 static void __init rbtx4937_arch_init(void)
 {
-	txx9_gpio_init(TX4938_PIO_REG & 0xfffffffffULL, 0, TX4938_NUM_PIO);
+	txx9_gpio_init(TX4938_PIO_REG & 0xfffffffffULL, TX4938_NUM_PIO);
 
 	rbtx4927_gpio_init();
 
@@ -317,11 +327,19 @@ static void __init rbtx4927_mtd_init(void)
 		tx4927_mtd_init(i);
 }
 
+static struct gpiod_lookup_table rbtx4927_gpioled_table = {
+	.table = {
+		GPIO_LOOKUP_IDX("TXx9", 0, NULL, 0, GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP_IDX("TXx9", 1, NULL, 1, GPIO_ACTIVE_LOW),
+		{ },
+	},
+};
+
 static void __init rbtx4927_gpioled_init(void)
 {
 	static const struct gpio_led leds[] = {
-		{ .name = "gpioled:green:0", .gpio = 0, .active_low = 1, },
-		{ .name = "gpioled:green:1", .gpio = 1, .active_low = 1, },
+		{ .name = "gpioled:green:0", },
+		{ .name = "gpioled:green:1", },
 	};
 	static struct gpio_led_platform_data pdata = {
 		.num_leds = ARRAY_SIZE(leds),
@@ -332,8 +350,13 @@ static void __init rbtx4927_gpioled_init(void)
 	if (!pdev)
 		return;
 	pdev->dev.platform_data = &pdata;
-	if (platform_device_add(pdev))
+	if (platform_device_add(pdev)) {
 		platform_device_put(pdev);
+		return;
+	}
+
+	rbtx4927_gpioled_table.dev_id = dev_name(&pdev->dev);
+	gpiod_add_lookup_table(&rbtx4927_gpioled_table);
 }
 
 static void __init rbtx4927_device_init(void)
@@ -350,7 +373,7 @@ static void __init rbtx4927_device_init(void)
 		tx4938_aclc_init();
 	}
 	platform_device_register_simple("txx9aclc-generic", -1, NULL, 0);
-	txx9_iocled_init(RBTX4927_LED_ADDR - IO_BASE, -1, 3, 1, "green", NULL);
+	txx9_iocled_init(RBTX4927_LED_ADDR - IO_BASE, 3, "green", NULL);
 	rbtx4927_gpioled_init();
 }
 
diff --git a/arch/mips/vdso/vdso.lds.S b/arch/mips/vdso/vdso.lds.S
index 278ab6444e985..b11ee493c67f3 100644
--- a/arch/mips/vdso/vdso.lds.S
+++ b/arch/mips/vdso/vdso.lds.S
@@ -97,9 +97,11 @@ VERSION
 	LINUX_2.6 {
 #ifdef CONFIG_GENERIC_GETTIMEOFDAY
 	global:
+#if _MIPS_SIM == _MIPS_SIM_ABI64 || defined(CONFIG_COMPAT_32BIT_TIME)
 		__vdso_clock_gettime;
 		__vdso_gettimeofday;
 		__vdso_clock_getres;
+#endif
 #if _MIPS_SIM != _MIPS_SIM_ABI64
 		__vdso_clock_gettime64;
 		__vdso_clock_getres_time64;
diff --git a/arch/mips/vdso/vgettimeofday.c b/arch/mips/vdso/vgettimeofday.c
index 00f9fcfc327e8..a1fb06b8973e5 100644
--- a/arch/mips/vdso/vgettimeofday.c
+++ b/arch/mips/vdso/vgettimeofday.c
@@ -12,6 +12,8 @@
 #include <vdso/gettime.h>
 
 #if _MIPS_SIM != _MIPS_SIM_ABI64
+
+#ifdef CONFIG_COMPAT_32BIT_TIME
 int __vdso_clock_gettime(clockid_t clock,
 			 struct old_timespec32 *ts)
 {
@@ -29,6 +31,7 @@ int __vdso_clock_getres(clockid_t clock_id,
 {
 	return __cvdso_clock_getres_time32(clock_id, res);
 }
+#endif /* CONFIG_COMPAT_32BIT_TIME */
 
 int __vdso_clock_gettime64(clockid_t clock,
 			   struct __kernel_timespec *ts)
diff --git a/arch/nios2/kernel/ptrace.c b/arch/nios2/kernel/ptrace.c
index c88f5cabc0c1b..2cd7e1ecbd74b 100644
--- a/arch/nios2/kernel/ptrace.c
+++ b/arch/nios2/kernel/ptrace.c
@@ -133,7 +133,7 @@ asmlinkage int do_syscall_trace_enter(void)
 	int ret = 0;
 
 	if (test_thread_flag(TIF_SYSCALL_TRACE))
-		ret = ptrace_report_syscall_entry(task_pt_regs(current));
+		ret = !ptrace_report_syscall_permit_entry(task_pt_regs(current));
 
 	return ret;
 }
diff --git a/arch/nios2/mm/cacheflush.c b/arch/nios2/mm/cacheflush.c
index 8321182eb9276..10232476031fc 100644
--- a/arch/nios2/mm/cacheflush.c
+++ b/arch/nios2/mm/cacheflush.c
@@ -82,7 +82,7 @@ static void flush_aliases(struct address_space *mapping, struct folio *folio)
 	pgoff = folio->index;
 
 	flush_dcache_mmap_lock_irqsave(mapping, flags);
-	vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff + nr - 1) {
+	mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) {
 		unsigned long start;
 
 		if (vma->vm_mm != mm)
diff --git a/arch/openrisc/kernel/ptrace.c b/arch/openrisc/kernel/ptrace.c
index 552489b24855c..287a9718f0c60 100644
--- a/arch/openrisc/kernel/ptrace.c
+++ b/arch/openrisc/kernel/ptrace.c
@@ -293,7 +293,7 @@ asmlinkage long do_syscall_trace_enter(struct pt_regs *regs)
 	long ret = 0;
 
 	if (test_thread_flag(TIF_SYSCALL_TRACE) &&
-	    ptrace_report_syscall_entry(regs))
+	    !ptrace_report_syscall_permit_entry(regs))
 		/*
 		 * Tracing decided this syscall should not happen.
 		 * We'll return a bogus call number to get an ENOSYS
diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c
index 0170b69a21d31..98200b6b5b820 100644
--- a/arch/parisc/kernel/cache.c
+++ b/arch/parisc/kernel/cache.c
@@ -503,7 +503,7 @@ void flush_dcache_folio(struct folio *folio)
 	 * on machines that support equivalent aliasing
 	 */
 	flush_dcache_mmap_lock_irqsave(mapping, flags);
-	vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff + nr - 1) {
+	mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff + nr - 1) {
 		unsigned long offset = pgoff - vma->vm_pgoff;
 		unsigned long pfn = folio_pfn(folio);
 
diff --git a/arch/parisc/kernel/ptrace.c b/arch/parisc/kernel/ptrace.c
index 8a17ab7e6e0b9..1d9e210702e1f 100644
--- a/arch/parisc/kernel/ptrace.c
+++ b/arch/parisc/kernel/ptrace.c
@@ -326,7 +326,7 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request,
 long do_syscall_trace_enter(struct pt_regs *regs)
 {
 	if (test_thread_flag(TIF_SYSCALL_TRACE)) {
-		int rc = ptrace_report_syscall_entry(regs);
+		bool permit = ptrace_report_syscall_permit_entry(regs);
 
 		/*
 		 * As tracesys_next does not set %r28 to -ENOSYS
@@ -334,12 +334,10 @@ long do_syscall_trace_enter(struct pt_regs *regs)
 		 */
 		regs->gr[28] = -ENOSYS;
 
-		if (rc) {
+		if (!permit) {
 			/*
-			 * A nonzero return code from
-			 * ptrace_report_syscall_entry() tells us
-			 * to prevent the syscall execution.  Skip
-			 * the syscall call and the syscall restart handling.
+			 * Skip the syscall call and the syscall restart
+			 * handling.
 			 *
 			 * Note that the tracer may also just change
 			 * regs->gr[20] to an invalid syscall number,
@@ -351,7 +349,7 @@ long do_syscall_trace_enter(struct pt_regs *regs)
 	}
 
 	/* Do the secure computing check after ptrace. */
-	if (secure_computing() == -1)
+	if (!seccomp_permit_syscall())
 		return -1;
 
 #ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
diff --git a/arch/parisc/kernel/smp.c b/arch/parisc/kernel/smp.c
index b2d12ab728b17..36be17e32948c 100644
--- a/arch/parisc/kernel/smp.c
+++ b/arch/parisc/kernel/smp.c
@@ -343,7 +343,7 @@ static int smp_boot_one_cpu(int cpuid, struct task_struct *idle)
 	for (i = 0; i < NR_IRQS; i++) {
 		struct irq_desc *desc = irq_to_desc(i);
 
-		if (desc && desc->kstat_irqs)
+		if (desc)
 			*per_cpu_ptr(desc->kstat_irqs, cpuid) = (struct irqstat) { };
 	}
 #endif
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index f7ce5fff81f03..c6bc2cd6cc836 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -256,7 +256,6 @@ config PPC
 	select HAVE_FUNCTION_GRAPH_TRACER
 	select HAVE_FUNCTION_TRACER		if !COMPILE_TEST && (PPC64 || (PPC32 && CC_IS_GCC))
 	select HAVE_GCC_PLUGINS
-	select HAVE_GENERIC_VDSO
 	select HAVE_HARDLOCKUP_DETECTOR_ARCH	if PPC_BOOK3S_64 && SMP
 	select HAVE_HARDLOCKUP_DETECTOR_PERF	if PERF_EVENTS && HAVE_PERF_EVENTS_NMI
 	select HAVE_HW_BREAKPOINT		if PERF_EVENTS && (PPC_BOOK3S || PPC_8xx)
@@ -328,6 +327,7 @@ config PPC
 	select SYSCTL_EXCEPTION_TRACE
 	select THREAD_INFO_IN_TASK
 	select TRACE_IRQFLAGS_SUPPORT
+	select VDSO_DATASTORE
 	select VDSO_GETRANDOM
 	#
 	# Please keep this list sorted alphabetically.
diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig
index f795b74602ec1..1eb8e3457e8bf 100644
--- a/arch/powerpc/configs/ppc64_defconfig
+++ b/arch/powerpc/configs/ppc64_defconfig
@@ -210,7 +210,6 @@ CONFIG_BNX2X=m
 CONFIG_CHELSIO_T1=m
 CONFIG_BE2NET=m
 CONFIG_IBMVETH=m
-CONFIG_EHEA=m
 CONFIG_IBMVNIC=m
 CONFIG_E100=y
 CONFIG_E1000=y
diff --git a/arch/powerpc/crypto/aes_cbc.c b/arch/powerpc/crypto/aes_cbc.c
index 4a9f285f09708..9c271b4642c84 100644
--- a/arch/powerpc/crypto/aes_cbc.c
+++ b/arch/powerpc/crypto/aes_cbc.c
@@ -72,7 +72,7 @@ static int p8_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
 	return ret ? -EINVAL : 0;
 }
 
-static int p8_aes_cbc_crypt(struct skcipher_request *req, int enc)
+static int p8_aes_cbc_crypt(struct skcipher_request *req, bool enc)
 {
 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 	const struct p8_aes_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
@@ -110,12 +110,12 @@ static int p8_aes_cbc_crypt(struct skcipher_request *req, int enc)
 
 static int p8_aes_cbc_encrypt(struct skcipher_request *req)
 {
-	return p8_aes_cbc_crypt(req, 1);
+	return p8_aes_cbc_crypt(req, true);
 }
 
 static int p8_aes_cbc_decrypt(struct skcipher_request *req)
 {
-	return p8_aes_cbc_crypt(req, 0);
+	return p8_aes_cbc_crypt(req, false);
 }
 
 struct skcipher_alg p8_aes_cbc_alg = {
diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h
index e67e64ac6e8cf..f4db7d7fbd5c6 100644
--- a/arch/powerpc/include/asm/book3s/64/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable.h
@@ -490,13 +490,13 @@ static inline pte_t pte_clear_soft_dirty(pte_t pte)
 }
 #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
 
-#ifdef CONFIG_NUMA_BALANCING
+#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE
 static inline int pte_protnone(pte_t pte)
 {
 	return (pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE | _PAGE_RWX)) ==
 		cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE);
 }
-#endif /* CONFIG_NUMA_BALANCING */
+#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */
 
 static inline bool pte_hw_valid(pte_t pte)
 {
@@ -1060,19 +1060,19 @@ static inline pte_t *pmdp_ptep(pmd_t *pmd)
 #define pmd_mksoft_dirty(pmd)  pte_pmd(pte_mksoft_dirty(pmd_pte(pmd)))
 #define pmd_clear_soft_dirty(pmd) pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd)))
 
-#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
 #define pmd_swp_mksoft_dirty(pmd)	pte_pmd(pte_swp_mksoft_dirty(pmd_pte(pmd)))
 #define pmd_swp_soft_dirty(pmd)		pte_swp_soft_dirty(pmd_pte(pmd))
 #define pmd_swp_clear_soft_dirty(pmd)	pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd)))
 #endif
 #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
 
-#ifdef CONFIG_NUMA_BALANCING
+#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE
 static inline int pmd_protnone(pmd_t pmd)
 {
 	return pte_protnone(pmd_pte(pmd));
 }
-#endif /* CONFIG_NUMA_BALANCING */
+#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */
 
 #define pmd_write(pmd)		pte_write(pmd_pte(pmd))
 
diff --git a/arch/powerpc/include/asm/kexec_ranges.h b/arch/powerpc/include/asm/kexec_ranges.h
index 14055896cbcbc..8489e844b4475 100644
--- a/arch/powerpc/include/asm/kexec_ranges.h
+++ b/arch/powerpc/include/asm/kexec_ranges.h
@@ -7,7 +7,6 @@
 void sort_memory_ranges(struct crash_mem *mrngs, bool merge);
 struct crash_mem *realloc_mem_ranges(struct crash_mem **mem_ranges);
 int add_mem_range(struct crash_mem **mem_ranges, u64 base, u64 size);
-int remove_mem_range(struct crash_mem **mem_ranges, u64 base, u64 size);
 int get_exclude_memory_ranges(struct crash_mem **mem_ranges);
 int get_reserved_memory_ranges(struct crash_mem **mem_ranges);
 int get_crash_memory_ranges(struct crash_mem **mem_ranges);
diff --git a/arch/powerpc/include/asm/preempt.h b/arch/powerpc/include/asm/preempt.h
deleted file mode 100644
index 000e2b9681f30..0000000000000
--- a/arch/powerpc/include/asm/preempt.h
+++ /dev/null
@@ -1,16 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __ASM_POWERPC_PREEMPT_H
-#define __ASM_POWERPC_PREEMPT_H
-
-#include <asm-generic/preempt.h>
-
-#if defined(CONFIG_PREEMPT_DYNAMIC)
-#include <linux/jump_label.h>
-DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
-#define need_irq_preemption() \
-	(static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched))
-#else
-#define need_irq_preemption()   (IS_ENABLED(CONFIG_PREEMPTION))
-#endif
-
-#endif /* __ASM_POWERPC_PREEMPT_H */
diff --git a/arch/powerpc/include/asm/syscall.h b/arch/powerpc/include/asm/syscall.h
index 834fcc4f7b543..4b3c52ed6e9d2 100644
--- a/arch/powerpc/include/asm/syscall.h
+++ b/arch/powerpc/include/asm/syscall.h
@@ -139,9 +139,4 @@ static inline int syscall_get_arch(struct task_struct *task)
 	else
 		return AUDIT_ARCH_PPC64;
 }
-
-static inline bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs)
-{
-	return false;
-}
 #endif	/* _ASM_SYSCALL_H */
diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h
index 7b8c56962c311..49039074b33fd 100644
--- a/arch/powerpc/include/asm/uaccess.h
+++ b/arch/powerpc/include/asm/uaccess.h
@@ -537,7 +537,7 @@ static inline void __user *mask_user_address(const void __user *ptr)
 
 	if (IS_ENABLED(CONFIG_PPC64))
 		return mask_user_address_simple(ptr);
-	if (IS_ENABLED(CONFIG_E500))
+	if (IS_ENABLED(CONFIG_PPC_E500))
 		return mask_user_address_isel(ptr);
 	if (TASK_SIZE <= UL(SZ_2G) && border >= UL(SZ_2G))
 		return mask_user_address_simple(ptr);
diff --git a/arch/powerpc/kernel/cacheinfo.c b/arch/powerpc/kernel/cacheinfo.c
index 04e5ea38bdc0a..cb09330610061 100644
--- a/arch/powerpc/kernel/cacheinfo.c
+++ b/arch/powerpc/kernel/cacheinfo.c
@@ -690,7 +690,8 @@ show_shared_cpumap(struct kobject *k, struct kobj_attribute *attr, char *buf, bo
 
 	mask = &cache->shared_cpu_map;
 
-	return cpumap_print_to_pagebuf(list, buf, mask);
+	return sysfs_emit(buf, list ? "%*pbl\n" : "%*pb\n",
+			  cpumask_pr_args(mask));
 }
 
 static ssize_t shared_cpu_map_show(struct kobject *k, struct kobj_attribute *attr, char *buf)
diff --git a/arch/powerpc/kernel/dt_cpu_ftrs.c b/arch/powerpc/kernel/dt_cpu_ftrs.c
index 3af6c06af02f3..e5853daa6a486 100644
--- a/arch/powerpc/kernel/dt_cpu_ftrs.c
+++ b/arch/powerpc/kernel/dt_cpu_ftrs.c
@@ -704,6 +704,15 @@ static void __init cpufeatures_setup_start(u32 isa)
 	if (isa >= ISA_V3_1) {
 		cur_cpu_spec->cpu_features |= CPU_FTR_ARCH_31;
 		cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_ARCH_3_1;
+
+		/*
+		 * CPU_FTR_P11_PVR is a kernel-internal flag to identify
+		 * Power11 and later processors. While ISA v3.1 is supported
+		 * by Power10+, this flag specifically indicates Power11+
+		 * for code that needs to distinguish between P10 and P11.
+		 */
+		if (PVR_VER(mfspr(SPRN_PVR)) >= PVR_POWER11)
+			cur_cpu_spec->cpu_features |= CPU_FTR_P11_PVR;
 	}
 }
 
diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c
index 3c4ca90e2ab77..4fc52c21fe5d4 100644
--- a/arch/powerpc/kernel/pci-common.c
+++ b/arch/powerpc/kernel/pci-common.c
@@ -626,19 +626,14 @@ int pci_legacy_write(struct pci_bus *bus, loff_t port, u32 val, size_t size)
 		return -ENXIO;
 	addr = hose->io_base_virt + port;
 
-	/* WARNING: The generic code is idiotic. It gets passed a pointer
-	 * to what can be a 1, 2 or 4 byte quantity and always reads that
-	 * as a u32, which means that we have to correct the location of
-	 * the data read within those 32 bits for size 1 and 2
-	 */
 	switch(size) {
 	case 1:
-		out_8(addr, val >> 24);
+		out_8(addr, val);
 		return 1;
 	case 2:
 		if (port & 1)
 			return -EINVAL;
-		out_le16(addr, val >> 16);
+		out_le16(addr, val);
 		return 2;
 	case 4:
 		if (port & 3)
diff --git a/arch/powerpc/kernel/syscall.c b/arch/powerpc/kernel/syscall.c
index a9da2af6efa87..1440dcabe0526 100644
--- a/arch/powerpc/kernel/syscall.c
+++ b/arch/powerpc/kernel/syscall.c
@@ -2,7 +2,6 @@
 
 #include <linux/compat.h>
 #include <linux/context_tracking.h>
-#include <linux/randomize_kstack.h>
 #include <linux/entry-common.h>
 
 #include <asm/interrupt.h>
@@ -19,8 +18,8 @@ notrace long system_call_exception(struct pt_regs *regs, unsigned long r0)
 	long ret;
 	syscall_fn f;
 
-	add_random_kstack_offset();
-	r0 = syscall_enter_from_user_mode(regs, r0);
+	if (unlikely(!syscall_enter_from_user_mode_randomize_stack(regs, &r0)))
+		return syscall_get_error(current, regs);
 
 	if (unlikely(r0 >= NR_syscalls)) {
 		if (unlikely(trap_is_unsupported_scv(regs))) {
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index 3460d1a5a97c2..11145c40183dd 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -377,7 +377,6 @@ void vtime_task_switch(struct task_struct *prev)
 	}
 }
 
-#ifdef CONFIG_NO_HZ_COMMON
 /**
  * vtime_reset - Fast forward vtime entry clocks
  *
@@ -394,6 +393,7 @@ void vtime_reset(void)
 #endif
 }
 
+#ifdef CONFIG_NO_HZ_COMMON
 /**
  * vtime_dyntick_start - Inform vtime about entry to idle-dynticks
  *
@@ -933,6 +933,7 @@ static void __init set_decrementer_max(void)
 static void __init init_decrementer_clockevent(void)
 {
 	register_decrementer_clockevent(smp_processor_id());
+	vtime_reset();
 }
 
 void secondary_cpu_time_init(void)
@@ -948,6 +949,7 @@ void secondary_cpu_time_init(void)
 	/* FIME: Should make unrelated change to move snapshot_timebase
 	 * call here ! */
 	register_decrementer_clockevent(smp_processor_id());
+	vtime_reset();
 }
 
 /*
diff --git a/arch/powerpc/kernel/vdso/gettimeofday.S b/arch/powerpc/kernel/vdso/gettimeofday.S
index 1c8e51691bf88..c635cd1e77bee 100644
--- a/arch/powerpc/kernel/vdso/gettimeofday.S
+++ b/arch/powerpc/kernel/vdso/gettimeofday.S
@@ -67,9 +67,11 @@
  * int __kernel_gettimeofday(struct timeval *tv, struct timezone *tz);
  *
  */
+#if defined(__powerpc64__) || defined(CONFIG_COMPAT_32BIT_TIME)
 V_FUNCTION_BEGIN(__kernel_gettimeofday)
 	cvdso_call __c_kernel_gettimeofday
 V_FUNCTION_END(__kernel_gettimeofday)
+#endif
 
 /*
  * Exact prototype of clock_gettime()
@@ -77,9 +79,11 @@ V_FUNCTION_END(__kernel_gettimeofday)
  * int __kernel_clock_gettime(clockid_t clock_id, struct timespec *tp);
  *
  */
+#if defined(__powerpc64__) || defined(CONFIG_COMPAT_32BIT_TIME)
 V_FUNCTION_BEGIN(__kernel_clock_gettime)
 	cvdso_call __c_kernel_clock_gettime
 V_FUNCTION_END(__kernel_clock_gettime)
+#endif
 
 /*
  * Exact prototype of clock_gettime64()
@@ -99,9 +103,11 @@ V_FUNCTION_END(__kernel_clock_gettime64)
  * int __kernel_clock_getres(clockid_t clock_id, struct timespec *res);
  *
  */
+#if defined(__powerpc64__) || defined(CONFIG_COMPAT_32BIT_TIME)
 V_FUNCTION_BEGIN(__kernel_clock_getres)
 	cvdso_call __c_kernel_clock_getres
 V_FUNCTION_END(__kernel_clock_getres)
+#endif
 
 /*
  * Exact prototype of clock_getres_time64()
@@ -122,6 +128,8 @@ V_FUNCTION_END(__kernel_clock_getres_time64)
  * time_t time(time *t);
  *
  */
+#if defined(__powerpc64__) || defined(CONFIG_COMPAT_32BIT_TIME)
 V_FUNCTION_BEGIN(__kernel_time)
 	cvdso_call __c_kernel_time call_time=1
 V_FUNCTION_END(__kernel_time)
+#endif
diff --git a/arch/powerpc/kernel/vdso/vdso32.lds.S b/arch/powerpc/kernel/vdso/vdso32.lds.S
index 3f384a2526aeb..5e87c18fd282d 100644
--- a/arch/powerpc/kernel/vdso/vdso32.lds.S
+++ b/arch/powerpc/kernel/vdso/vdso32.lds.S
@@ -119,13 +119,15 @@ VERSION
 {
 	VDSO_VERSION_STRING {
 	global:
-		__kernel_get_syscall_map;
+#ifdef CONFIG_COMPAT_32BIT_TIME
 		__kernel_gettimeofday;
 		__kernel_clock_gettime;
-		__kernel_clock_gettime64;
 		__kernel_clock_getres;
-		__kernel_clock_getres_time64;
 		__kernel_time;
+#endif /* CONFIG_COMPAT_32BIT_TIME */
+		__kernel_get_syscall_map;
+		__kernel_clock_gettime64;
+		__kernel_clock_getres_time64;
 		__kernel_get_tbfreq;
 		__kernel_sync_dicache;
 		__kernel_sigtramp32;
diff --git a/arch/powerpc/kernel/vdso/vgettimeofday.c b/arch/powerpc/kernel/vdso/vgettimeofday.c
index 3c194e1ab5620..4b712fb01a3fc 100644
--- a/arch/powerpc/kernel/vdso/vgettimeofday.c
+++ b/arch/powerpc/kernel/vdso/vgettimeofday.c
@@ -18,23 +18,25 @@ int __c_kernel_clock_getres(clockid_t clock_id, struct __kernel_timespec *res,
 	return __cvdso_clock_getres_data(vd, clock_id, res);
 }
 #else
+#ifdef CONFIG_COMPAT_32BIT_TIME
 int __c_kernel_clock_gettime(clockid_t clock, struct old_timespec32 *ts,
 			     const struct vdso_time_data *vd)
 {
 	return __cvdso_clock_gettime32_data(vd, clock, ts);
 }
 
-int __c_kernel_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts,
-			       const struct vdso_time_data *vd)
-{
-	return __cvdso_clock_gettime_data(vd, clock, ts);
-}
-
 int __c_kernel_clock_getres(clockid_t clock_id, struct old_timespec32 *res,
 			    const struct vdso_time_data *vd)
 {
 	return __cvdso_clock_getres_time32_data(vd, clock_id, res);
 }
+#endif /* CONFIG_COMPAT_32BIT_TIME */
+
+int __c_kernel_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts,
+			       const struct vdso_time_data *vd)
+{
+	return __cvdso_clock_gettime_data(vd, clock, ts);
+}
 
 int __c_kernel_clock_getres_time64(clockid_t clock_id, struct __kernel_timespec *res,
 				   const struct vdso_time_data *vd)
@@ -43,6 +45,7 @@ int __c_kernel_clock_getres_time64(clockid_t clock_id, struct __kernel_timespec
 }
 #endif
 
+#if defined(__powerpc64__) || defined(CONFIG_COMPAT_32BIT_TIME)
 int __c_kernel_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz,
 			    const struct vdso_time_data *vd)
 {
@@ -53,3 +56,4 @@ __kernel_old_time_t __c_kernel_time(__kernel_old_time_t *time, const struct vdso
 {
 	return __cvdso_time_data(vd, time);
 }
+#endif
diff --git a/arch/powerpc/kexec/crash.c b/arch/powerpc/kexec/crash.c
index e6539f213b3d1..60a917a6beaae 100644
--- a/arch/powerpc/kexec/crash.c
+++ b/arch/powerpc/kexec/crash.c
@@ -493,7 +493,7 @@ static void update_crash_elfcorehdr(struct kimage *image, struct memory_notify *
 	struct crash_mem *cmem = NULL;
 	struct kexec_segment *ksegment;
 	void *ptr, *mem, *elfbuf = NULL;
-	unsigned long elfsz, memsz, base_addr, size;
+	unsigned long elfsz, memsz, base_addr, size, end;
 
 	ksegment = &image->segment[image->elfcorehdr_index];
 	mem = (void *) ksegment->mem;
@@ -512,7 +512,8 @@ static void update_crash_elfcorehdr(struct kimage *image, struct memory_notify *
 	if (image->hp_action == KEXEC_CRASH_HP_REMOVE_MEMORY) {
 		base_addr = PFN_PHYS(mn->start_pfn);
 		size = mn->nr_pages * PAGE_SIZE;
-		ret = remove_mem_range(&cmem, base_addr, size);
+		end = base_addr + size - 1;
+		ret = arch_crash_exclude_mem_range(&cmem, base_addr, end);
 		if (ret) {
 			pr_err("Failed to remove hot-unplugged memory from crash memory ranges\n");
 			goto out;
diff --git a/arch/powerpc/kexec/ranges.c b/arch/powerpc/kexec/ranges.c
index 867135560e5c8..e5fea23b191b6 100644
--- a/arch/powerpc/kexec/ranges.c
+++ b/arch/powerpc/kexec/ranges.c
@@ -553,9 +553,9 @@ int get_usable_memory_ranges(struct crash_mem **mem_ranges)
 #endif /* CONFIG_KEXEC_FILE */
 
 #ifdef CONFIG_CRASH_DUMP
-static int crash_exclude_mem_range_guarded(struct crash_mem **mem_ranges,
-					   unsigned long long mstart,
-					   unsigned long long mend)
+int arch_crash_exclude_mem_range(struct crash_mem **mem_ranges,
+				 unsigned long long mstart,
+				 unsigned long long mend)
 {
 	struct crash_mem *tmem = *mem_ranges;
 
@@ -604,18 +604,10 @@ int get_crash_memory_ranges(struct crash_mem **mem_ranges)
 			sort_memory_ranges(*mem_ranges, true);
 	}
 
-	/* Exclude crashkernel region */
-	ret = crash_exclude_mem_range_guarded(mem_ranges, crashk_res.start, crashk_res.end);
+	ret = crash_exclude_core_ranges(mem_ranges);
 	if (ret)
 		goto out;
 
-	for (i = 0; i < crashk_cma_cnt; ++i) {
-		ret = crash_exclude_mem_range_guarded(mem_ranges, crashk_cma_ranges[i].start,
-					      crashk_cma_ranges[i].end);
-		if (ret)
-			goto out;
-	}
-
 	/*
 	 * FIXME: For now, stay in parity with kexec-tools but if RTAS/OPAL
 	 *        regions are exported to save their context at the time of
@@ -641,89 +633,4 @@ int get_crash_memory_ranges(struct crash_mem **mem_ranges)
 		pr_err("Failed to setup crash memory ranges\n");
 	return ret;
 }
-
-/**
- * remove_mem_range - Removes the given memory range from the range list.
- * @mem_ranges:    Range list to remove the memory range to.
- * @base:          Base address of the range to remove.
- * @size:          Size of the memory range to remove.
- *
- * (Re)allocates memory, if needed.
- *
- * Returns 0 on success, negative errno on error.
- */
-int remove_mem_range(struct crash_mem **mem_ranges, u64 base, u64 size)
-{
-	u64 end;
-	int ret = 0;
-	unsigned int i;
-	u64 mstart, mend;
-	struct crash_mem *mem_rngs = *mem_ranges;
-
-	if (!size)
-		return 0;
-
-	/*
-	 * Memory range are stored as start and end address, use
-	 * the same format to do remove operation.
-	 */
-	end = base + size - 1;
-
-	for (i = 0; i < mem_rngs->nr_ranges; i++) {
-		mstart = mem_rngs->ranges[i].start;
-		mend = mem_rngs->ranges[i].end;
-
-		/*
-		 * Memory range to remove is not part of this range entry
-		 * in the memory range list
-		 */
-		if (!(base >= mstart && end <= mend))
-			continue;
-
-		/*
-		 * Memory range to remove is equivalent to this entry in the
-		 * memory range list. Remove the range entry from the list.
-		 */
-		if (base == mstart && end == mend) {
-			for (; i < mem_rngs->nr_ranges - 1; i++) {
-				mem_rngs->ranges[i].start = mem_rngs->ranges[i+1].start;
-				mem_rngs->ranges[i].end = mem_rngs->ranges[i+1].end;
-			}
-			mem_rngs->nr_ranges--;
-			goto out;
-		}
-		/*
-		 * Start address of the memory range to remove and the
-		 * current memory range entry in the list is same. Just
-		 * move the start address of the current memory range
-		 * entry in the list to end + 1.
-		 */
-		else if (base == mstart) {
-			mem_rngs->ranges[i].start = end + 1;
-			goto out;
-		}
-		/*
-		 * End address of the memory range to remove and the
-		 * current memory range entry in the list is same.
-		 * Just move the end address of the current memory
-		 * range entry in the list to base - 1.
-		 */
-		else if (end == mend)  {
-			mem_rngs->ranges[i].end = base - 1;
-			goto out;
-		}
-		/*
-		 * Memory range to remove is not at the edge of current
-		 * memory range entry. Split the current memory entry into
-		 * two half.
-		 */
-		else {
-			size = mem_rngs->ranges[i].end - end + 1;
-			mem_rngs->ranges[i].end = base - 1;
-			ret = add_mem_range(mem_ranges, end + 1, size);
-		}
-	}
-out:
-	return ret;
-}
 #endif /* CONFIG_CRASH_DUMP */
diff --git a/arch/powerpc/lib/vmx-helper.c b/arch/powerpc/lib/vmx-helper.c
index 57e897b60db86..cc9fb72cb4ebe 100644
--- a/arch/powerpc/lib/vmx-helper.c
+++ b/arch/powerpc/lib/vmx-helper.c
@@ -46,7 +46,7 @@ int exit_vmx_usercopy(void)
 	 * set and we are preemptible. The hack here is to schedule a
 	 * decrementer to fire here and reschedule for us if necessary.
 	 */
-	if (need_irq_preemption() && need_resched())
+	if (need_resched())
 		set_dec(1);
 	return 0;
 }
diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c
index 10aced261cff4..cf692b2b5f7bc 100644
--- a/arch/powerpc/mm/book3s64/radix_pgtable.c
+++ b/arch/powerpc/mm/book3s64/radix_pgtable.c
@@ -1314,7 +1314,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
 	 * covering out both edges.
 	 */
 	unsigned long addr;
-	unsigned long addr_pfn = start_pfn;
 	unsigned long next;
 	pgd_t *pgd;
 	p4d_t *p4d;
@@ -1335,7 +1334,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
 
 		if (pmd_leaf(READ_ONCE(*pmd))) {
 			/* existing huge mapping. Skip the range */
-			addr_pfn += (PMD_SIZE >> PAGE_SHIFT);
 			next = pmd_addr_end(addr, end);
 			continue;
 		}
@@ -1348,11 +1346,11 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
 			 * page whose VMEMMAP_RESERVE_NR pages were mapped and
 			 * this request fall in those pages.
 			 */
-			addr_pfn += 1;
 			next = addr + PAGE_SIZE;
 			continue;
 		} else {
 			unsigned long nr_pages = pgmap_vmemmap_nr(pgmap);
+			unsigned long addr_pfn = page_to_pfn((struct page *)addr);
 			unsigned long pfn_offset = addr_pfn - ALIGN_DOWN(addr_pfn, nr_pages);
 			pte_t *tail_page_pte;
 
@@ -1376,7 +1374,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
 				if (!pte)
 					return -ENOMEM;
 
-				addr_pfn += 2;
 				next = addr + 2 * PAGE_SIZE;
 				continue;
 			}
@@ -1392,7 +1389,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
 					return -ENOMEM;
 				vmemmap_verify(pte, node, addr, addr + PAGE_SIZE);
 
-				addr_pfn += 1;
 				next = addr + PAGE_SIZE;
 				continue;
 			}
@@ -1402,7 +1398,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
 				return -ENOMEM;
 			vmemmap_verify(pte, node, addr, addr + PAGE_SIZE);
 
-			addr_pfn += 1;
 			next = addr + PAGE_SIZE;
 			continue;
 		}
diff --git a/arch/powerpc/mm/book3s64/slice.c b/arch/powerpc/mm/book3s64/slice.c
index 28bec5bc78794..82127e31dca6d 100644
--- a/arch/powerpc/mm/book3s64/slice.c
+++ b/arch/powerpc/mm/book3s64/slice.c
@@ -659,7 +659,8 @@ unsigned long arch_get_unmapped_area(struct file *filp,
 	unsigned int psize;
 
 	if (radix_enabled())
-		return generic_get_unmapped_area(filp, addr, len, pgoff, flags, vm_flags);
+		return generic_get_unmapped_area(filp, addr, len, pgoff, flags,
+						 legacy_to_vma_flags(vm_flags));
 
 	if (filp && is_file_hugepages(filp))
 		psize = file_to_psize(filp);
@@ -679,7 +680,8 @@ unsigned long arch_get_unmapped_area_topdown(struct file *filp,
 	unsigned int psize;
 
 	if (radix_enabled())
-		return generic_get_unmapped_area_topdown(filp, addr0, len, pgoff, flags, vm_flags);
+		return generic_get_unmapped_area_topdown(filp, addr0, len,
+				pgoff, flags, legacy_to_vma_flags(vm_flags));
 
 	if (filp && is_file_hugepages(filp))
 		psize = file_to_psize(filp);
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index 558fafb82b8ac..a298746dc1431 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -104,17 +104,14 @@ void __init pseries_add_gpage(u64 addr, u64 page_size, unsigned long number_of_p
 	}
 }
 
-static int __init pseries_alloc_bootmem_huge_page(struct hstate *hstate)
+static __init void *pseries_alloc_bootmem_huge_page(struct hstate *hstate)
 {
-	struct huge_bootmem_page *m;
+	void *m;
 	if (nr_gpages == 0)
-		return 0;
+		return NULL;
 	m = phys_to_virt(gpage_freearray[--nr_gpages]);
 	gpage_freearray[nr_gpages] = 0;
-	list_add(&m->list, &huge_boot_pages[0]);
-	m->hstate = hstate;
-	m->flags = 0;
-	return 1;
+	return m;
 }
 
 bool __init hugetlb_node_alloc_supported(void)
@@ -124,7 +121,7 @@ bool __init hugetlb_node_alloc_supported(void)
 #endif
 
 
-int __init alloc_bootmem_huge_page(struct hstate *h, int nid)
+void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid)
 {
 
 #ifdef CONFIG_PPC_BOOK3S_64
diff --git a/arch/powerpc/mm/kasan/init_book3e_64.c b/arch/powerpc/mm/kasan/init_book3e_64.c
index 0d3a73d6d4b0b..0ed372fb8b096 100644
--- a/arch/powerpc/mm/kasan/init_book3e_64.c
+++ b/arch/powerpc/mm/kasan/init_book3e_64.c
@@ -68,9 +68,6 @@ static void __init kasan_init_phys_region(void *start, void *end)
 	unsigned long k_start, k_end, k_cur;
 	void *va;
 
-	if (start >= end)
-		return;
-
 	k_start = ALIGN_DOWN((unsigned long)kasan_mem_to_shadow(start), PAGE_SIZE);
 	k_end = ALIGN((unsigned long)kasan_mem_to_shadow(end), PAGE_SIZE);
 
diff --git a/arch/powerpc/mm/kasan/init_book3s_64.c b/arch/powerpc/mm/kasan/init_book3s_64.c
index dcafa641804c0..ccfbfb8946376 100644
--- a/arch/powerpc/mm/kasan/init_book3s_64.c
+++ b/arch/powerpc/mm/kasan/init_book3s_64.c
@@ -24,9 +24,6 @@ static void __init kasan_init_phys_region(void *start, void *end)
 	unsigned long k_start, k_end, k_cur;
 	void *va;
 
-	if (start >= end)
-		return;
-
 	k_start = ALIGN_DOWN((unsigned long)kasan_mem_to_shadow(start), PAGE_SIZE);
 	k_end = ALIGN((unsigned long)kasan_mem_to_shadow(end), PAGE_SIZE);
 
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index 4c1afab919963..b617b69452cd6 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -371,12 +371,6 @@ int devmem_is_allowed(unsigned long pfn)
 }
 #endif /* CONFIG_STRICT_DEVMEM */
 
-/*
- * This is defined in kernel/resource.c but only powerpc needs to export it, for
- * the EHEA driver. Drop this when drivers/net/ethernet/ibm/ehea is removed.
- */
-EXPORT_SYMBOL_GPL(walk_system_ram_range);
-
 #ifdef CONFIG_EXECMEM
 static struct execmem_info execmem_info __ro_after_init;
 
diff --git a/arch/powerpc/perf/hv-24x7.c b/arch/powerpc/perf/hv-24x7.c
index abb4cfb11fcc9..b0eedf1859606 100644
--- a/arch/powerpc/perf/hv-24x7.c
+++ b/arch/powerpc/perf/hv-24x7.c
@@ -429,7 +429,7 @@ static char *memdup_to_str(char *maybe_str, int max_len, gfp_t gfp)
 static ssize_t cpumask_show(struct device *dev,
 			    struct device_attribute *attr, char *buf)
 {
-	return cpumap_print_to_pagebuf(true, buf, &hv_24x7_cpumask);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&hv_24x7_cpumask));
 }
 
 static ssize_t sockets_show(struct device *dev,
diff --git a/arch/powerpc/perf/hv-gpci.c b/arch/powerpc/perf/hv-gpci.c
index 7269273d3aa88..76495744f52ae 100644
--- a/arch/powerpc/perf/hv-gpci.c
+++ b/arch/powerpc/perf/hv-gpci.c
@@ -100,7 +100,7 @@ static ssize_t kernel_version_show(struct device *dev,
 static ssize_t cpumask_show(struct device *dev,
 			    struct device_attribute *attr, char *buf)
 {
-	return cpumap_print_to_pagebuf(true, buf, &hv_gpci_cpumask);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&hv_gpci_cpumask));
 }
 
 /* Interface attribute array index to store system information */
diff --git a/arch/powerpc/perf/imc-pmu.c b/arch/powerpc/perf/imc-pmu.c
index e3822f36c4199..0be4e98f7ad18 100644
--- a/arch/powerpc/perf/imc-pmu.c
+++ b/arch/powerpc/perf/imc-pmu.c
@@ -117,7 +117,7 @@ static ssize_t imc_pmu_cpumask_get_attr(struct device *dev,
 		return 0;
 	}
 
-	return cpumap_print_to_pagebuf(true, buf, active_mask);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(active_mask));
 }
 
 static DEVICE_ATTR(cpumask, S_IRUGO, imc_pmu_cpumask_get_attr, NULL);
diff --git a/arch/powerpc/platforms/85xx/common.c b/arch/powerpc/platforms/85xx/common.c
index 757811155587d..c11deb2f50ed4 100644
--- a/arch/powerpc/platforms/85xx/common.c
+++ b/arch/powerpc/platforms/85xx/common.c
@@ -42,6 +42,8 @@ static const struct of_device_id mpc85xx_common_ids[] __initconst = {
 	{ .compatible = "fsl,qoriq-pcie-v2.3", },
 	{ .compatible = "fsl,qoriq-pcie-v2.2", },
 	{ .compatible = "fsl,fman", },
+	/* IFC NAND and NOR controllers */
+	{ .compatible = "fsl,ifc", },
 	{},
 };
 
diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype
index bac02c83bb3e4..f0adbf7ecb7f5 100644
--- a/arch/powerpc/platforms/Kconfig.cputype
+++ b/arch/powerpc/platforms/Kconfig.cputype
@@ -87,6 +87,7 @@ config PPC_BOOK3S_64
 	select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION
 	select ARCH_ENABLE_SPLIT_PMD_PTLOCK
 	select ARCH_SUPPORTS_HUGETLBFS
+	select ARCH_HAS_PTE_PROTNONE
 	select ARCH_SUPPORTS_NUMA_BALANCING
 	select HAVE_MOVE_PMD
 	select HAVE_MOVE_PUD
@@ -112,7 +113,7 @@ config PPC_THP
        depends on PPC_RADIX_MMU || (PPC_64S_HASH_MMU && PAGE_SIZE_64KB)
        select HAVE_ARCH_TRANSPARENT_HUGEPAGE
        select HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
-       select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
+       select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE
 
 choice
 	prompt "CPU selection"
diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c
index f6de8c1169d5b..de7494748fecd 100644
--- a/arch/powerpc/platforms/cell/spufs/file.c
+++ b/arch/powerpc/platforms/cell/spufs/file.c
@@ -268,10 +268,12 @@ static int spufs_mem_mmap_access(struct vm_area_struct *vma,
 
 	if (write && !(vma->vm_flags & VM_WRITE))
 		return -EACCES;
+	if (offset >= LS_SIZE)
+		return -EFAULT;
 	if (spu_acquire(ctx))
 		return -EINTR;
-	if ((offset + len) > vma->vm_end)
-		len = vma->vm_end - offset;
+	if ((offset + len) > LS_SIZE)
+		len = LS_SIZE - offset;
 	local_store = ctx->ops->get_ls(ctx);
 	if (write)
 		memcpy_toio(local_store + offset, buf, len);
diff --git a/arch/powerpc/platforms/powermac/low_i2c.c b/arch/powerpc/platforms/powermac/low_i2c.c
index da72a30ab8657..a175a32a222ba 100644
--- a/arch/powerpc/platforms/powermac/low_i2c.c
+++ b/arch/powerpc/platforms/powermac/low_i2c.c
@@ -1471,7 +1471,7 @@ static int __init pmac_i2c_create_platform_devices(void)
 		if (bus->platform_dev == NULL)
 			return -ENOMEM;
 		bus->platform_dev->dev.platform_data = bus;
-		bus->platform_dev->dev.of_node = bus->busnode;
+		platform_device_set_of_node(bus->platform_dev, bus->busnode);
 		platform_device_add(bus->platform_dev);
 	}
 
diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig
index f7052b131a4c5..74910ce3a541c 100644
--- a/arch/powerpc/platforms/pseries/Kconfig
+++ b/arch/powerpc/platforms/pseries/Kconfig
@@ -154,6 +154,7 @@ config HV_PERF_CTRS
 config VPA_PMU
 	tristate "VPA PMU events"
 	depends on KVM_BOOK3S_64_HV && HV_PERF_CTRS
+	default m
 	help
 	  Enable access to the VPA PMU counters via perf. This enables
 	  code that support measurement for KVM on PowerVM(KoP) feature.
diff --git a/arch/powerpc/platforms/pseries/papr_platform_attributes.c b/arch/powerpc/platforms/pseries/papr_platform_attributes.c
index c6159870de0e1..9c3758aa54c6e 100644
--- a/arch/powerpc/platforms/pseries/papr_platform_attributes.c
+++ b/arch/powerpc/platforms/pseries/papr_platform_attributes.c
@@ -271,11 +271,9 @@ static int __init papr_init(void)
 		esi_buf_size = ESI_HDR_SIZE + (CURR_MAX_ESI_ATTRS * max_esi_attrs);
 
 		temp_esi_buf = krealloc(esi_buf, esi_buf_size, GFP_KERNEL);
-		if (temp_esi_buf)
-			esi_buf = temp_esi_buf;
-		else
-			return -ENOMEM;
-
+		if (!temp_esi_buf)
+			goto out_free_esi_buf;
+		esi_buf = temp_esi_buf;
 		goto retry;
 	}
 
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index c0a6992933e41..ef1101bec76c3 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -22,7 +22,8 @@ config RISCV
 	select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION
 	select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM_VMEMMAP
 	select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2
-	select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
+	select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE
+	select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI
 	select ARCH_HAS_BINFMT_FLAT
 	select ARCH_HAS_CC_CAN_LINK
 	select ARCH_HAS_CURRENT_STACK_POINTER
@@ -54,7 +55,7 @@ config RISCV
 	select ARCH_HAS_SYSCALL_WRAPPER
 	select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
 	select ARCH_HAS_UBSAN
-	select ARCH_HAS_VDSO_ARCH_DATA if HAVE_GENERIC_VDSO
+	select ARCH_HAS_VDSO_ARCH_DATA
 	select ARCH_HAVE_NMI_SAFE_CMPXCHG
 	select ARCH_KEEP_MEMBLOCK if ACPI || KEXEC
 	select ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE	if 64BIT && MMU
@@ -71,6 +72,7 @@ config RISCV
 	select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS if 64BIT && MMU
 	select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU
 	select ARCH_SUPPORTS_PER_VMA_LOCK if MMU
+	select ARCH_HAS_PTE_PROTNONE if MMU
 	select ARCH_SUPPORTS_RT
 	select ARCH_SUPPORTS_SHADOW_CALL_STACK if HAVE_SHADOW_CALL_STACK
 	select ARCH_SUPPORTS_SCHED_MC if SMP
@@ -110,7 +112,7 @@ config RISCV
 	select GENERIC_CPU_VULNERABILITIES
 	select GENERIC_EARLY_IOREMAP
 	select GENERIC_ENTRY
-	select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO && 64BIT
+	select GENERIC_GETTIMEOFDAY if MMU && 64BIT
 	select GENERIC_IDLE_POLL_SETUP
 	select GENERIC_IOREMAP if MMU
 	select HAVE_IOREMAP_PROT if MMU
@@ -157,7 +159,7 @@ config RISCV
 	select HAVE_DEBUG_KMEMLEAK
 	select HAVE_DMA_CONTIGUOUS if MMU
 	select HAVE_DYNAMIC_FTRACE if MMU && (CLANG_SUPPORTS_DYNAMIC_FTRACE || GCC_SUPPORTS_DYNAMIC_FTRACE)
-	select FUNCTION_ALIGNMENT_4B if HAVE_DYNAMIC_FTRACE && RISCV_ISA_C
+	select FUNCTION_ALIGNMENT_4B if DYNAMIC_FTRACE && RISCV_ISA_C
 	select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS if HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS
 	select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if (DYNAMIC_FTRACE_WITH_ARGS && !CFI)
 	select HAVE_DYNAMIC_FTRACE_WITH_ARGS if HAVE_DYNAMIC_FTRACE
@@ -171,7 +173,6 @@ config RISCV
 	select HAVE_FUNCTION_ARG_ACCESS_API
 	select HAVE_FUNCTION_ERROR_INJECTION
 	select HAVE_GCC_PLUGINS
-	select HAVE_GENERIC_VDSO if MMU
 	select HAVE_IRQ_TIME_ACCOUNTING
 	select HAVE_KERNEL_BZIP2 if !EFI_ZBOOT
 	select HAVE_KERNEL_GZIP if !EFI_ZBOOT
@@ -227,7 +228,8 @@ config RISCV
 	select THREAD_INFO_IN_TASK
 	select TRACE_IRQFLAGS_SUPPORT
 	select UACCESS_MEMCPY if !MMU
-	select VDSO_GETRANDOM if HAVE_GENERIC_VDSO && 64BIT
+	select VDSO_DATASTORE if MMU
+	select VDSO_GETRANDOM if MMU && 64BIT
 	select USER_STACKTRACE_SUPPORT
 	select ZONE_DMA32 if 64BIT
 
diff --git a/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi b/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi
index a810ad3eb2a2b..13b1e57166040 100644
--- a/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi
+++ b/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi
@@ -191,6 +191,12 @@ uart3_pb_pins: uart3-pb-pins {
 				pins = "PB6", "PB7";
 				function = "uart3";
 			};
+
+			/omit-if-no-ref/
+			uart4_pb_pins: uart4-pb-pins {
+				pins = "PB2", "PB3";
+				function = "uart4";
+			};
 		};
 
 		ccu: clock-controller@2001000 {
diff --git a/arch/riscv/boot/dts/canaan/Makefile b/arch/riscv/boot/dts/canaan/Makefile
index 987d1f0c41f0e..7d54ea5c6f3d2 100644
--- a/arch/riscv/boot/dts/canaan/Makefile
+++ b/arch/riscv/boot/dts/canaan/Makefile
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
 dtb-$(CONFIG_ARCH_CANAAN) += canaan_kd233.dtb
 dtb-$(CONFIG_ARCH_CANAAN) += k210_generic.dtb
+dtb-$(CONFIG_ARCH_CANAAN) += k230-canmv.dtb
+dtb-$(CONFIG_ARCH_CANAAN) += k230-evb.dtb
 dtb-$(CONFIG_ARCH_CANAAN) += sipeed_maix_bit.dtb
 dtb-$(CONFIG_ARCH_CANAAN) += sipeed_maix_dock.dtb
 dtb-$(CONFIG_ARCH_CANAAN) += sipeed_maix_go.dtb
diff --git a/arch/riscv/boot/dts/canaan/k230-canmv.dts b/arch/riscv/boot/dts/canaan/k230-canmv.dts
new file mode 100644
index 0000000000000..2ba555f6de131
--- /dev/null
+++ b/arch/riscv/boot/dts/canaan/k230-canmv.dts
@@ -0,0 +1,343 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2024 Yangyu Chen <cyy@cyyself.name>
+ */
+
+#include "k230.dtsi"
+
+/ {
+	model = "Canaan CanMV-K230";
+	compatible = "canaan,canmv-k230", "canaan,kendryte-k230";
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	ddr: memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x0 0x0 0x20000000>;
+	};
+
+	timerx_pulse_in: clock-50000000 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <50000000>;
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&pinctrl {
+	jtag_pins: jtag-pins {
+		jtag-tck-cfg {
+			pinmux = <K230_PINMUX(2, 1)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-pull-down;
+			input-schmitt-enable;
+		};
+
+		jtag-tdi-cfg {
+			pinmux = <K230_PINMUX(3, 1)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-disable;
+		};
+
+		jtag-tdo-cfg {
+			pinmux = <K230_PINMUX(4, 1)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+		};
+
+		jtag-tms-cfg {
+			pinmux = <K230_PINMUX(5, 1)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-pull-up;
+		};
+	};
+
+	uart2_pins: uart2-pins {
+		uart2-pins-cfg {
+			pinmux = <K230_PINMUX(5, 3)>, /* uart2 txd */
+				 <K230_PINMUX(6, 3)>; /* uart2 rxd */
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-disable;
+		};
+	};
+
+	pwm2_pins: pwm2-pins {
+		pwm2-pin-cfg {
+			pinmux = <K230_PINMUX(7, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	pwm3_pins: pwm3-pins {
+		pwm3-pin-cfg {
+			pinmux = <K230_PINMUX(8, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	pwm4_pins: pwm4-pins {
+		pwm4-pin-cfg {
+			pinmux = <K230_PINMUX(9, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	iis_pins: iis-pins {
+		iis-clk-cfg {
+			pinmux = <K230_PINMUX(32, 2)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+		};
+
+		iis-ws-cfg {
+			pinmux = <K230_PINMUX(33, 2)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+		};
+
+		iis-din0-cfg {
+			pinmux = <K230_PINMUX(34, 2)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-disable;
+		};
+
+		iis-dout0-cfg {
+			pinmux = <K230_PINMUX(35, 2)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+		};
+	};
+
+	uart4_pins: uart4-pins {
+		uart4-txd-cfg {
+			pinmux = <K230_PINMUX(36, 4)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+
+		uart4-rxd-cfg {
+			pinmux = <K230_PINMUX(37, 4)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	uart0_pins: uart0-pins {
+		uart0-txd-cfg {
+			pinmux = <K230_PINMUX(38, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+
+		uart0-rxd-cfg {
+			pinmux = <K230_PINMUX(39, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	iic1_pins: iic1-pins {
+		iic1-pins-cfg {
+			pinmux = <K230_PINMUX(40, 2)>, /* iic1 scl */
+				 <K230_PINMUX(41, 2)>; /* iic1 sda */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+	};
+
+	iic3_pins: iic3-pins {
+		iic3-pins-cfg {
+			pinmux = <K230_PINMUX(44, 2)>, /* iic3 scl */
+				 <K230_PINMUX(45, 2)>; /* iic3 sda */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+	};
+
+	iic4_pins: iic4-pins {
+		iic4-pins-cfg {
+			pinmux = <K230_PINMUX(46, 3)>, /* iic4 scl */
+				 <K230_PINMUX(47, 3)>; /* iic4 sda */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+	};
+
+	iic0_pins: iic0-pins {
+		iic0-pins-cfg {
+			pinmux = <K230_PINMUX(48, 3)>, /* iic0 scl */
+				 <K230_PINMUX(49, 3)>; /* iic0 sda */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+	};
+
+	uart3_pins: uart3-pins {
+		uart3-txd-cfg {
+			pinmux = <K230_PINMUX(50, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+
+		uart3-rxd-cfg {
+			pinmux = <K230_PINMUX(51, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			input-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	key_pins: key-pins {
+		key-pins-cfg {
+			pinmux = <K230_PINMUX(52, 0)>, /* key0 */
+				 <K230_PINMUX(53, 0)>; /* key1 */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			input-enable;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	mmc1_pins: mmc1-pins {
+		mmc1-cmd-cfg {
+			pinmux = <K230_PINMUX(54, 2)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+
+		mmc1-clk-cfg {
+			pinmux = <K230_PINMUX(55, 2)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+
+		mmc1-data-cfg {
+			pinmux = <K230_PINMUX(56, 2)>, /* mmc1 data0 */
+				 <K230_PINMUX(57, 2)>, /* mmc1 data1 */
+				 <K230_PINMUX(58, 2)>, /* mmc1 data2 */
+				 <K230_PINMUX(59, 2)>; /* mmc1 data3 */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+	};
+};
+
+&sysclk {
+	clocks = <&osc24m>, <&timerx_pulse_in>;
+	clock-names = "osc24m", "timer-pulse-in";
+};
diff --git a/arch/riscv/boot/dts/canaan/k230-evb.dts b/arch/riscv/boot/dts/canaan/k230-evb.dts
new file mode 100644
index 0000000000000..4246e4bba6ccd
--- /dev/null
+++ b/arch/riscv/boot/dts/canaan/k230-evb.dts
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2024 Yangyu Chen <cyy@cyyself.name>
+ */
+
+#include "k230.dtsi"
+
+/ {
+	model = "Kendryte K230 EVB";
+	compatible = "canaan,k230-usip-lp3-evb", "canaan,kendryte-k230";
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	ddr: memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x0 0x0 0x20000000>;
+	};
+
+	timerx_pulse_in: clock-50000000 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <50000000>;
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&sysclk {
+	clocks = <&osc24m>, <&timerx_pulse_in>;
+	clock-names = "osc24m", "timer-pulse-in";
+};
diff --git a/arch/riscv/boot/dts/canaan/k230-pinctrl.h b/arch/riscv/boot/dts/canaan/k230-pinctrl.h
new file mode 100644
index 0000000000000..63dd999ca55b6
--- /dev/null
+++ b/arch/riscv/boot/dts/canaan/k230-pinctrl.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
+/*
+ * Copyright (C) 2024 Canaan Bright Sight Co. Ltd
+ * Copyright (C) 2024 Ze Huang <18771902331@163.com>
+ */
+
+#ifndef _K230_PINCTRL_H
+#define _K230_PINCTRL_H
+
+#define K230_MSC_3V3 0
+#define K230_MSC_1V8 1
+
+#define BANK_VOLTAGE_DEFAULT       K230_MSC_1V8
+#define BANK_VOLTAGE_IO50_IO61     K230_MSC_3V3
+
+#define K230_PINMUX(pin, mode) (((pin) << 8) | (mode))
+
+#endif /* _K230_PINCTRL_H */
diff --git a/arch/riscv/boot/dts/canaan/k230.dtsi b/arch/riscv/boot/dts/canaan/k230.dtsi
new file mode 100644
index 0000000000000..6dcb628a960b0
--- /dev/null
+++ b/arch/riscv/boot/dts/canaan/k230.dtsi
@@ -0,0 +1,167 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2024 Yangyu Chen <cyy@cyyself.name>
+ */
+
+#include <dt-bindings/clock/canaan,k230-clk.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/reset/canaan,k230-rst.h>
+#include "k230-pinctrl.h"
+
+/dts-v1/;
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	compatible = "canaan,kendryte-k230";
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <27000000>;
+
+		cpu@0 {
+			compatible = "thead,c908", "riscv";
+			device_type = "cpu";
+			reg = <0>;
+			riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicsr_zifencei_zihpm_zfh_zba_zbb_zbc_zbs_zvfh_svpbmt";
+			riscv,isa-base = "rv64i";
+			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom",
+					       "zicbop", "zicboz", "zicntr", "zicsr", "zifencei",
+					       "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zvfh",
+					       "svpbmt";
+			riscv,cbom-block-size = <64>;
+			riscv,cbop-block-size = <64>;
+			riscv,cboz-block-size = <64>;
+			d-cache-block-size = <64>;
+			d-cache-sets = <128>;
+			d-cache-size = <32768>;
+			i-cache-block-size = <64>;
+			i-cache-sets = <128>;
+			i-cache-size = <32768>;
+			next-level-cache = <&l2_cache>;
+			mmu-type = "riscv,sv39";
+
+			cpu0_intc: interrupt-controller {
+				compatible = "riscv,cpu-intc";
+				interrupt-controller;
+				#interrupt-cells = <1>;
+			};
+		};
+
+		l2_cache: l2-cache {
+			compatible = "cache";
+			cache-block-size = <64>;
+			cache-level = <2>;
+			cache-size = <262144>;
+			cache-sets = <256>;
+			cache-unified;
+		};
+	};
+
+	osc24m: clock-24000000 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <24000000>;
+		clock-output-names = "osc24m";
+	};
+
+	soc {
+		compatible = "simple-bus";
+		interrupt-parent = <&plic>;
+		#address-cells = <2>;
+		#size-cells = <2>;
+		dma-noncoherent;
+		ranges;
+
+		plic: interrupt-controller@f00000000 {
+			compatible = "canaan,k230-plic", "thead,c900-plic";
+			reg = <0xf 0x00000000 0x0 0x04000000>;
+			interrupts-extended = <&cpu0_intc 11>, <&cpu0_intc 9>;
+			interrupt-controller;
+			#address-cells = <0>;
+			#interrupt-cells = <2>;
+			riscv,ndev = <208>;
+		};
+
+		clint: timer@f04000000 {
+			compatible = "canaan,k230-clint", "thead,c900-clint";
+			reg = <0xf 0x04000000 0x0 0x00010000>;
+			interrupts-extended = <&cpu0_intc 3>, <&cpu0_intc 7>;
+		};
+
+		rst: reset-controller@91101000 {
+			compatible = "canaan,k230-rst";
+			reg = <0x0 0x91101000 0x0 0x1000>;
+			#reset-cells = <1>;
+		};
+
+		pinctrl: pinctrl@91105000 {
+			compatible = "canaan,k230-pinctrl";
+			reg = <0x0 0x91105000 0x0 0x100>;
+		};
+
+		sysclk: clock-controller@91102000 {
+			compatible = "canaan,k230-clk";
+			reg = <0x0 0x91102000 0x0 0x40>,
+			      <0x0 0x91100000 0x0 0x108>;
+			clocks = <&osc24m>;
+			clock-names = "osc24m";
+			#clock-cells = <1>;
+		};
+
+		uart0: serial@91400000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x91400000 0x0 0x1000>;
+			clocks = <&sysclk K230_LS_UART0_RATE>;
+			interrupts = <16 IRQ_TYPE_LEVEL_HIGH>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			resets = <&rst RST_UART0>;
+			status = "disabled";
+		};
+
+		uart1: serial@91401000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x91401000 0x0 0x1000>;
+			clocks = <&sysclk K230_LS_UART1_RATE>;
+			interrupts = <17 IRQ_TYPE_LEVEL_HIGH>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			resets = <&rst RST_UART1>;
+			status = "disabled";
+		};
+
+		uart2: serial@91402000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x91402000 0x0 0x1000>;
+			clocks = <&sysclk K230_LS_UART2_RATE>;
+			interrupts = <18 IRQ_TYPE_LEVEL_HIGH>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			resets = <&rst RST_UART2>;
+			status = "disabled";
+		};
+
+		uart3: serial@91403000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x91403000 0x0 0x1000>;
+			clocks = <&sysclk K230_LS_UART3_RATE>;
+			interrupts = <19 IRQ_TYPE_LEVEL_HIGH>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			resets = <&rst RST_UART3>;
+			status = "disabled";
+		};
+
+		uart4: serial@91404000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x91404000 0x0 0x1000>;
+			clocks = <&sysclk K230_LS_UART4_RATE>;
+			interrupts = <20 IRQ_TYPE_LEVEL_HIGH>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			resets = <&rst RST_UART4>;
+			status = "disabled";
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts
index 72f77e9edd63d..307a7e4a89434 100644
--- a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts
+++ b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts
@@ -4,6 +4,7 @@
  */
 
 #include "k1.dtsi"
+#include "k1-opp.dtsi"
 #include "k1-pinctrl.dtsi"
 
 / {
@@ -87,6 +88,38 @@ &combo_phy {
 	status = "okay";
 };
 
+&cpu_0 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_1 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_2 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_3 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_4 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_5 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_6 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_7 {
+	cpu-supply = <&buck1_0v9>;
+};
+
 &emmc {
 	bus-width = <8>;
 	mmc-hs400-1_8v;
@@ -202,7 +235,7 @@ pmic@41 {
 		dldoin2-supply = <&buck5>;
 
 		regulators {
-			buck1 {
+			buck1_0v9: buck1 {
 				regulator-min-microvolt = <500000>;
 				regulator-max-microvolt = <3450000>;
 				regulator-ramp-delay = <5000>;
diff --git a/arch/riscv/boot/dts/spacemit/k1-opp.dtsi b/arch/riscv/boot/dts/spacemit/k1-opp.dtsi
new file mode 100644
index 0000000000000..4cebfcd87485f
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k1-opp.dtsi
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+
+/ {
+	cluster0_opp_table: opp-table-cluster0 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-614400000 {
+			opp-hz = /bits/ 64 <614400000>;
+			opp-microvolt = <950000 950000 1050000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp-819000000 {
+			opp-hz = /bits/ 64 <819000000>;
+			opp-microvolt = <950000 950000 1050000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp-1000000000 {
+			opp-hz = /bits/ 64 <1000000000>;
+			opp-microvolt = <950000 950000 1050000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp-1228800000 {
+			opp-hz = /bits/ 64 <1228800000>;
+			opp-microvolt = <950000 950000 1050000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp-1600000000 {
+			opp-hz = /bits/ 64 <1600000000>;
+			opp-microvolt = <1050000 1050000 1050000>;
+			clock-latency-ns = <200000>;
+		};
+	};
+
+	cluster1_opp_table: opp-table-cluster1 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-614400000 {
+			opp-hz = /bits/ 64 <614400000>;
+			opp-microvolt = <950000 950000 1050000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp-819000000 {
+			opp-hz = /bits/ 64 <819000000>;
+			opp-microvolt = <950000 950000 1050000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp-1000000000 {
+			opp-hz = /bits/ 64 <1000000000>;
+			opp-microvolt = <950000 950000 1050000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp-1228800000 {
+			opp-hz = /bits/ 64 <1228800000>;
+			opp-microvolt = <950000 950000 1050000>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp-1600000000 {
+			opp-hz = /bits/ 64 <1600000000>;
+			opp-microvolt = <1050000 1050000 1050000>;
+			clock-latency-ns = <200000>;
+		};
+	};
+};
+
+&cpu_0 {
+	operating-points-v2 = <&cluster0_opp_table>;
+};
+
+&cpu_1 {
+	operating-points-v2 = <&cluster0_opp_table>;
+};
+
+&cpu_2 {
+	operating-points-v2 = <&cluster0_opp_table>;
+};
+
+&cpu_3 {
+	operating-points-v2 = <&cluster0_opp_table>;
+};
+
+&cpu_4 {
+	operating-points-v2 = <&cluster1_opp_table>;
+};
+
+&cpu_5 {
+	operating-points-v2 = <&cluster1_opp_table>;
+};
+
+&cpu_6 {
+	operating-points-v2 = <&cluster1_opp_table>;
+};
+
+&cpu_7 {
+	operating-points-v2 = <&cluster1_opp_table>;
+};
diff --git a/arch/riscv/boot/dts/spacemit/k1-pdma.h b/arch/riscv/boot/dts/spacemit/k1-pdma.h
new file mode 100644
index 0000000000000..7e5ad3d7111d4
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k1-pdma.h
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * DMA request number (DRQ) definitions for non-secure peripherals of
+ * the SpacemiT K1 PDMA.
+ *
+ * Copyright (c) 2026 Guodong Xu <docular.xu@gmail.com>
+ */
+
+#ifndef _DTS_SPACEMIT_K1_PDMA_H
+#define _DTS_SPACEMIT_K1_PDMA_H
+
+#define K1_PDMA_UART0_TX	3
+#define K1_PDMA_UART0_RX	4
+#define K1_PDMA_UART2_TX	5
+#define K1_PDMA_UART2_RX	6
+#define K1_PDMA_UART3_TX	7
+#define K1_PDMA_UART3_RX	8
+#define K1_PDMA_UART4_TX	9
+#define K1_PDMA_UART4_RX	10
+#define K1_PDMA_I2C0_TX		11
+#define K1_PDMA_I2C0_RX		12
+#define K1_PDMA_I2C1_TX		13
+#define K1_PDMA_I2C1_RX		14
+#define K1_PDMA_I2C2_TX		15
+#define K1_PDMA_I2C2_RX		16
+#define K1_PDMA_I2C4_TX		17
+#define K1_PDMA_I2C4_RX		18
+#define K1_PDMA_SPI3_TX		19
+#define K1_PDMA_SPI3_RX		20
+#define K1_PDMA_I2S0_TX		21
+#define K1_PDMA_I2S0_RX		22
+#define K1_PDMA_I2S1_TX		23
+#define K1_PDMA_I2S1_RX		24
+#define K1_PDMA_UART5_TX	25
+#define K1_PDMA_UART5_RX	26
+#define K1_PDMA_UART6_TX	27
+#define K1_PDMA_UART6_RX	28
+#define K1_PDMA_UART7_TX	29
+#define K1_PDMA_UART7_RX	30
+#define K1_PDMA_UART8_TX	31
+#define K1_PDMA_UART8_RX	32
+#define K1_PDMA_UART9_TX	33
+#define K1_PDMA_UART9_RX	34
+#define K1_PDMA_I2C5_TX		35
+#define K1_PDMA_I2C5_RX		36
+#define K1_PDMA_I2C6_TX		37
+#define K1_PDMA_I2C6_RX		38
+#define K1_PDMA_I2C7_TX		39
+#define K1_PDMA_I2C7_RX		40
+#define K1_PDMA_I2C8_TX		41
+#define K1_PDMA_I2C8_RX		42
+#define K1_PDMA_CAN0_RX		43
+#define K1_PDMA_QSPI_RX		44
+#define K1_PDMA_QSPI_TX		45
+
+#endif /* _DTS_SPACEMIT_K1_PDMA_H */
diff --git a/arch/riscv/boot/dts/spacemit/k1.dtsi b/arch/riscv/boot/dts/spacemit/k1.dtsi
index 08a0f28d011fe..3fabfd34114b2 100644
--- a/arch/riscv/boot/dts/spacemit/k1.dtsi
+++ b/arch/riscv/boot/dts/spacemit/k1.dtsi
@@ -6,6 +6,8 @@
 #include <dt-bindings/clock/spacemit,k1-syscon.h>
 #include <dt-bindings/phy/phy.h>
 
+#include "k1-pdma.h"
+
 /dts-v1/;
 / {
 	#address-cells = <2>;
@@ -54,6 +56,7 @@ cpu_0: cpu@0 {
 			compatible = "spacemit,x60", "riscv";
 			device_type = "cpu";
 			reg = <0>;
+			clocks = <&syscon_apmu CLK_CPU_C0_CORE>;
 			riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt";
 			riscv,isa-base = "rv64i";
 			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom",
@@ -84,6 +87,7 @@ cpu_1: cpu@1 {
 			compatible = "spacemit,x60", "riscv";
 			device_type = "cpu";
 			reg = <1>;
+			clocks = <&syscon_apmu CLK_CPU_C0_CORE>;
 			riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt";
 			riscv,isa-base = "rv64i";
 			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom",
@@ -114,6 +118,7 @@ cpu_2: cpu@2 {
 			compatible = "spacemit,x60", "riscv";
 			device_type = "cpu";
 			reg = <2>;
+			clocks = <&syscon_apmu CLK_CPU_C0_CORE>;
 			riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt";
 			riscv,isa-base = "rv64i";
 			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom",
@@ -144,6 +149,7 @@ cpu_3: cpu@3 {
 			compatible = "spacemit,x60", "riscv";
 			device_type = "cpu";
 			reg = <3>;
+			clocks = <&syscon_apmu CLK_CPU_C0_CORE>;
 			riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt";
 			riscv,isa-base = "rv64i";
 			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom",
@@ -174,6 +180,7 @@ cpu_4: cpu@4 {
 			compatible = "spacemit,x60", "riscv";
 			device_type = "cpu";
 			reg = <4>;
+			clocks = <&syscon_apmu CLK_CPU_C1_CORE>;
 			riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt";
 			riscv,isa-base = "rv64i";
 			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom",
@@ -204,6 +211,7 @@ cpu_5: cpu@5 {
 			compatible = "spacemit,x60", "riscv";
 			device_type = "cpu";
 			reg = <5>;
+			clocks = <&syscon_apmu CLK_CPU_C1_CORE>;
 			riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt";
 			riscv,isa-base = "rv64i";
 			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom",
@@ -234,6 +242,7 @@ cpu_6: cpu@6 {
 			compatible = "spacemit,x60", "riscv";
 			device_type = "cpu";
 			reg = <6>;
+			clocks = <&syscon_apmu CLK_CPU_C1_CORE>;
 			riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt";
 			riscv,isa-base = "rv64i";
 			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom",
@@ -264,6 +273,7 @@ cpu_7: cpu@7 {
 			compatible = "spacemit,x60", "riscv";
 			device_type = "cpu";
 			reg = <7>;
+			clocks = <&syscon_apmu CLK_CPU_C1_CORE>;
 			riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zba_zbb_zbc_zbs_zkt_zvfh_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt";
 			riscv,isa-base = "rv64i";
 			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom",
@@ -1094,7 +1104,7 @@ spi3: spi@d401c000 {
 				clock-names = "core", "bus";
 				resets = <&syscon_apbc RESET_SSP3>;
 				interrupts = <55>;
-				dmas = <&pdma 20>, <&pdma 19>;
+				dmas = <&pdma K1_PDMA_SPI3_RX>, <&pdma K1_PDMA_SPI3_TX>;
 				dma-names = "rx", "tx";
 				status = "disabled";
 			};
diff --git a/arch/riscv/boot/dts/spacemit/k3-com260-ifx.dts b/arch/riscv/boot/dts/spacemit/k3-com260-ifx.dts
index 238bb03d0e9e2..e7f76cd24b903 100644
--- a/arch/riscv/boot/dts/spacemit/k3-com260-ifx.dts
+++ b/arch/riscv/boot/dts/spacemit/k3-com260-ifx.dts
@@ -19,3 +19,21 @@ chosen {
 		stdout-path = "serial0:115200n8";
 	};
 };
+
+&sdhci0 {
+	pinctrl-names = "default","uhs";
+	pinctrl-0 = <&mmc1_cfg>;
+	pinctrl-1 = <&mmc1_uhs_cfg>;
+	bus-width = <4>;
+	cd-gpios = <&gpio K3_GPIO(4) GPIO_ACTIVE_HIGH>;
+	no-mmc;
+	no-sdio;
+	disable-wp;
+	cap-sd-highspeed;
+	vmmc-supply = <&aldo4>;
+	vqmmc-supply = <&aldo1>;
+	sd-uhs-sdr25;
+	sd-uhs-sdr50;
+	sd-uhs-sdr104;
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts b/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts
index b89c1521e6649..48b71cc419f09 100644
--- a/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts
+++ b/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts
@@ -14,11 +14,13 @@ / {
 
 	aliases {
 		ethernet0 = &eth0;
+		i2c2 = &i2c2;
+		i2c8 = &i2c8;
 		serial0 = &uart0;
 	};
 
 	chosen {
-		stdout-path = "serial0";
+		stdout-path = "serial0:115200n8";
 	};
 
 	memory@100000000 {
@@ -43,6 +45,42 @@ reg_aux_vcc5v: regulator-aux-vcc5v {
 	};
 };
 
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_1_cfg>;
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		pagesize = <8>;
+		read-only;
+		size = <256>;
+		status = "okay";
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			mac-address {
+				#nvmem-cell-cells = <1>;
+			};
+
+			num-macs {
+			};
+
+			part-number {
+			};
+
+			product-name {
+			};
+
+			serial-number {
+			};
+
+		};
+	};
+};
+
 &i2c8 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&i2c8_cfg>;
@@ -186,6 +224,7 @@ &eth0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&gmac0_rgmii_0_cfg>, <&gmac0_phy_0_cfg>;
 	phy-mode = "rgmii-id";
+	tx-internal-delay-ps = <400>;
 	phy-handle = <&phy0>;
 	status = "okay";
 
@@ -196,6 +235,64 @@ phy0: phy@1 {
 			reset-gpios = <&gpio 0 15 GPIO_ACTIVE_LOW>;
 			reset-assert-us = <10000>;
 			reset-deassert-us = <10000>;
+			realtek,aldps-enable;
+			realtek,clkout-disable;
+		};
+	};
+};
+
+&qspi_cfg {
+	qspi-pins {
+		power-source = <1800>;
+	};
+
+	qspi-cs0-pins {
+		power-source = <1800>;
+	};
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_cfg>;
+	status = "okay";
+
+	flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <26500000>;
+		spi-rx-bus-width = <4>;
+		spi-tx-bus-width = <4>;
+		vcc-supply = <&aldo2>; /* PMIC_VCC1V8_QSPI */
+		m25p,fast-read;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			bootinfo@0 {
+				reg = <0x0 0x20000>;
+			};
+
+			fsbl@20000 {
+				reg = <0x20000 0x80000>;
+			};
+
+			env@a0000 {
+				reg = <0xa0000 0x10000>;
+			};
+
+			esos@b0000 {
+				reg = <0xb0000 0x100000>;
+			};
+
+			opensbi@1b0000 {
+				reg = <0x1b0000 0x60000>;
+			};
+
+			uboot@210000 {
+				reg = <0x210000 0x5f0000>;
+			};
 		};
 	};
 };
diff --git a/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi b/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi
index 3ee1471f37985..5b9045e014345 100644
--- a/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi
+++ b/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi
@@ -78,6 +78,17 @@ gmac1-phy-0-pins {
 		};
 	};
 
+	/omit-if-no-ref/
+	i2c2_1_cfg: i2c2-1-cfg {
+		i2c2-1-pins {
+			pinmux = <K3_PADCONF(46, 5)>,	/* i2c2 scl */
+				 <K3_PADCONF(47, 5)>;	/* i2c2 sda */
+
+			bias-pull-up = <0>;
+			drive-strength = <25>;
+		};
+	};
+
 	/omit-if-no-ref/
 	i2c8_cfg: i2c8-cfg {
 		i2c8-pins {
@@ -679,6 +690,27 @@ pwm19-2-pins {
 		};
 	};
 
+	/omit-if-no-ref/
+	qspi_cfg: qspi-cfg {
+		qspi-pins {
+			pinmux = <K3_PADCONF(138, 0)>,	/* qspi dat0 */
+				 <K3_PADCONF(139, 0)>,	/* qspi dat1 */
+				 <K3_PADCONF(140, 0)>,	/* qspi dat2 */
+				 <K3_PADCONF(141, 0)>,	/* qspi dat3 */
+				 <K3_PADCONF(144, 0)>;	/* qspi clk */
+
+			bias-disable;
+			drive-strength = <25>;
+		};
+
+		qspi-cs0-pins {
+			pinmux = <K3_PADCONF(142, 0)>;	/* qspi cs0 */
+
+			bias-disable;
+			drive-strength = <25>;
+		};
+	};
+
 	/omit-if-no-ref/
 	uart0_0_cfg: uart0-0-cfg {
 		uart0-0-pins {
@@ -689,4 +721,48 @@ uart0-0-pins {
 			drive-strength = <25>;
 		};
 	};
+
+	mmc1_cfg: mmc1-cfg {
+		mmc1-data-cmd-pins {
+			pinmux = <K3_PADCONF(132, 0)>,	/* mmc1 dat3 */
+				 <K3_PADCONF(133, 0)>,	/* mmc1 dat2 */
+				 <K3_PADCONF(134, 0)>,	/* mmc1 dat1 */
+				 <K3_PADCONF(135, 0)>,	/* mmc1 dat0 */
+				 <K3_PADCONF(136, 0)>;	/* mmc1 cmd */
+
+			bias-pull-up = <0>;
+			drive-strength = <25>;
+			power-source = <3300>;
+		};
+
+		mmc1-clk-pins {
+			pinmux = <K3_PADCONF(137, 0)>;	/* mmc1 clk */
+
+			bias-pull-down;
+			drive-strength = <25>;
+			power-source = <3300>;
+		};
+	};
+
+	mmc1_uhs_cfg: mmc1-uhs-cfg {
+		mmc1-uhs-data-cmd-pins {
+			pinmux = <K3_PADCONF(132, 0)>,	/* mmc1 dat3 */
+				 <K3_PADCONF(133, 0)>,	/* mmc1 dat2 */
+				 <K3_PADCONF(134, 0)>,	/* mmc1 dat1 */
+				 <K3_PADCONF(135, 0)>,	/* mmc1 dat0 */
+				 <K3_PADCONF(136, 0)>;	/* mmc1 cmd */
+
+			bias-pull-up = <0>;
+			drive-strength = <21>;
+			power-source = <1800>;
+		};
+
+		mmc1-uhs-clk-pins {
+			pinmux = <K3_PADCONF(137, 0)>;	/* mmc1 clk */
+
+			bias-pull-down;
+			drive-strength = <21>;
+			power-source = <1800>;
+		};
+	};
 };
diff --git a/arch/riscv/boot/dts/spacemit/k3.dtsi b/arch/riscv/boot/dts/spacemit/k3.dtsi
index 19fc9b49668ed..fc46e34465e47 100644
--- a/arch/riscv/boot/dts/spacemit/k3.dtsi
+++ b/arch/riscv/boot/dts/spacemit/k3.dtsi
@@ -1099,6 +1099,36 @@ pll: clock-controller@d4090000 {
 			#clock-cells = <1>;
 		};
 
+		qspi: spi@d420c000 {
+			compatible = "spacemit,k3-qspi",
+				     "spacemit,k1-qspi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0xd420c000 0x0 0x1000>,
+			      <0x0 0xb8000000 0x0 0xc00000>;
+			reg-names = "QuadSPI", "QuadSPI-memory";
+			clocks = <&syscon_apmu CLK_APMU_QSPI_BUS>,
+				 <&syscon_apmu CLK_APMU_QSPI>;
+			clock-names = "qspi_en", "qspi";
+			resets = <&syscon_apmu RESET_APMU_QSPI>,
+				 <&syscon_apmu RESET_APMU_QSPI_BUS>;
+			interrupts = <117 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		sdhci0: mmc@d4280000 {
+			compatible = "spacemit,k3-sdhci";
+			reg = <0x0 0xd4280000 0x0 0x200>;
+			clocks = <&syscon_apmu CLK_APMU_SDH_AXI>,
+				 <&syscon_apmu CLK_APMU_SDH0>;
+			clock-names = "core", "io";
+			resets = <&syscon_apmu RESET_APMU_SDH_AXI>,
+				 <&syscon_apmu RESET_APMU_SDH0>;
+			reset-names = "axi", "sdh";
+			interrupts = <99 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
 		syscon_apmu: system-controller@d4282800 {
 			compatible = "spacemit,k3-syscon-apmu";
 			reg = <0x0 0xd4282800 0x0 0x400>;
diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig
index ed605b5e3162d..04ae305d5511b 100644
--- a/arch/riscv/configs/defconfig
+++ b/arch/riscv/configs/defconfig
@@ -175,6 +175,8 @@ CONFIG_PINCTRL_SOPHGO_SG2002=y
 CONFIG_GPIO_DWAPB=y
 CONFIG_GPIO_SIFIVE=y
 CONFIG_GPIO_SPACEMIT_K1=y
+CONFIG_GPIO_PCA953X=m
+CONFIG_GPIO_PCA953X_IRQ=y
 CONFIG_POWER_RESET_GPIO_RESTART=y
 CONFIG_SENSORS_SFCTEMP=m
 CONFIG_CPU_THERMAL=y
diff --git a/arch/riscv/include/asm/acpi.h b/arch/riscv/include/asm/acpi.h
index 26ab37c171bcf..4e01ef15e2019 100644
--- a/arch/riscv/include/asm/acpi.h
+++ b/arch/riscv/include/asm/acpi.h
@@ -67,6 +67,7 @@ int acpi_get_riscv_isa(struct acpi_table_header *table,
 
 void acpi_get_cbo_block_size(struct acpi_table_header *table, u32 *cbom_size,
 			     u32 *cboz_size, u32 *cbop_size);
+acpi_handle acpi_get_riscv_gsi_handle(u32 gsi);
 #else
 static inline void acpi_init_rintc_map(void) { }
 static inline struct acpi_madt_rintc *acpi_cpu_get_madt_rintc(int cpu)
@@ -92,4 +93,6 @@ void acpi_map_cpus_to_nodes(void);
 static inline void acpi_map_cpus_to_nodes(void) { }
 #endif /* CONFIG_ACPI_NUMA */
 
+#define ACPI_TABLE_UPGRADE_MAX_PHYS MEMBLOCK_ALLOC_ACCESSIBLE
+
 #endif /*_ASM_ACPI_H*/
diff --git a/arch/riscv/include/asm/cacheflush.h b/arch/riscv/include/asm/cacheflush.h
index 8cfe59483a8f2..c6297a1d26459 100644
--- a/arch/riscv/include/asm/cacheflush.h
+++ b/arch/riscv/include/asm/cacheflush.h
@@ -6,7 +6,9 @@
 #ifndef _ASM_RISCV_CACHEFLUSH_H
 #define _ASM_RISCV_CACHEFLUSH_H
 
+#include <linux/array_size.h>
 #include <linux/mm.h>
+#include <asm/barrier.h>
 
 static inline void local_flush_icache_all(void)
 {
@@ -40,23 +42,35 @@ do {							\
 		flush_icache_mm(vma->vm_mm, 0);		\
 } while (0)
 
-#ifdef CONFIG_64BIT
+#if defined(CONFIG_64BIT) && defined(CONFIG_MMU)
 /* This is accessed in assembly code. cpumask_var_t would be too complex. */
 extern DECLARE_BITMAP(new_valid_map_cpus, NR_CPUS);
 extern char _end[];
 static inline void mark_new_valid_map(void)
 {
+	/*
+	 * Orders any previous page table writes before setting bits in
+	 * new_valid_map_cpus. Pairs with the sfence.vma in
+	 * new_valid_map_cpus_check.
+	 */
+	smp_wmb();
+
 	/*
 	 * We don't care if concurrently a cpu resets this value since
 	 * the only place this can happen is in handle_exception() where
 	 * an sfence.vma is emitted.
+	 *
+	 * Not memset() or bitmap_fill() to avoid any possible compiler
+	 * shenanigans.
 	 */
-	bitmap_fill(new_valid_map_cpus, NR_CPUS);
+	for (size_t i = 0; i < ARRAY_SIZE(new_valid_map_cpus); i++)
+		WRITE_ONCE(new_valid_map_cpus[i], -1UL);
 }
 #define flush_cache_vmap flush_cache_vmap
 static inline void flush_cache_vmap(unsigned long start, unsigned long end)
 {
-	if (is_vmalloc_or_module_addr((void *)start))
+	if (is_vmalloc_or_module_addr((void *)start) ||
+	    (start >= VMEMMAP_START && end <= VMEMMAP_END))
 		mark_new_valid_map();
 }
 #define flush_cache_vmap_early(start, end)	local_flush_tlb_kernel_range(start, end)
diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h
index 09bb5f57a9d34..92d5f831f3495 100644
--- a/arch/riscv/include/asm/io.h
+++ b/arch/riscv/include/asm/io.h
@@ -102,12 +102,14 @@ __io_reads_ins(reads, u32, l, __io_br(), __io_ar(addr))
 #define readsw(addr, buffer, count) __readsw(addr, buffer, count)
 #define readsl(addr, buffer, count) __readsl(addr, buffer, count)
 
+#ifdef CONFIG_HAS_IOPORT
 __io_reads_ins(ins,  u8, b, __io_pbr(), __io_par(addr))
 __io_reads_ins(ins, u16, w, __io_pbr(), __io_par(addr))
 __io_reads_ins(ins, u32, l, __io_pbr(), __io_par(addr))
 #define insb(addr, buffer, count) __insb(PCI_IOBASE + (addr), buffer, count)
 #define insw(addr, buffer, count) __insw(PCI_IOBASE + (addr), buffer, count)
 #define insl(addr, buffer, count) __insl(PCI_IOBASE + (addr), buffer, count)
+#endif
 
 __io_writes_outs(writes,  u8, b, __io_bw(), __io_aw())
 __io_writes_outs(writes, u16, w, __io_bw(), __io_aw())
@@ -116,26 +118,32 @@ __io_writes_outs(writes, u32, l, __io_bw(), __io_aw())
 #define writesw(addr, buffer, count) __writesw(addr, buffer, count)
 #define writesl(addr, buffer, count) __writesl(addr, buffer, count)
 
+#ifdef CONFIG_HAS_IOPORT
 __io_writes_outs(outs,  u8, b, __io_pbw(), __io_paw())
 __io_writes_outs(outs, u16, w, __io_pbw(), __io_paw())
 __io_writes_outs(outs, u32, l, __io_pbw(), __io_paw())
 #define outsb(addr, buffer, count) __outsb(PCI_IOBASE + (addr), buffer, count)
 #define outsw(addr, buffer, count) __outsw(PCI_IOBASE + (addr), buffer, count)
 #define outsl(addr, buffer, count) __outsl(PCI_IOBASE + (addr), buffer, count)
+#endif
 
 #ifdef CONFIG_64BIT
 __io_reads_ins(reads, u64, q, __io_br(), __io_ar(addr))
 #define readsq(addr, buffer, count) __readsq(addr, buffer, count)
 
+#ifdef CONFIG_HAS_IOPORT
 __io_reads_ins(ins, u64, q, __io_pbr(), __io_par(addr))
 #define insq(addr, buffer, count) __insq(PCI_IOBASE + (addr), buffer, count)
+#endif
 
 __io_writes_outs(writes, u64, q, __io_bw(), __io_aw())
 #define writesq(addr, buffer, count) __writesq(addr, buffer, count)
 
+#ifdef CONFIG_HAS_IOPORT
 __io_writes_outs(outs, u64, q, __io_pbr(), __io_paw())
 #define outsq(addr, buffer, count) __outsq(PCI_IOBASE + (addr), buffer, count)
 #endif
+#endif
 
 #include <asm-generic/io.h>
 
diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h
index 60017ceec9d2a..e2d5808169e44 100644
--- a/arch/riscv/include/asm/kvm_host.h
+++ b/arch/riscv/include/asm/kvm_host.h
@@ -209,13 +209,13 @@ struct kvm_vcpu_arch {
 	/*
 	 * VCPU interrupts
 	 *
-	 * We have a lockless approach for tracking pending VCPU interrupts
-	 * implemented using atomic bitops. The irqs_pending bitmap represent
-	 * pending interrupts whereas irqs_pending_mask represent bits changed
-	 * in irqs_pending. Our approach is modeled around multiple producer
-	 * and single consumer problem where the consumer is the VCPU itself.
+	 * The irqs_pending bitmap represents pending interrupts whereas
+	 * irqs_pending_mask represents bits changed in irqs_pending. Updates
+	 * to these bitmaps are serialized so vcpu interrupt sync/flush cannot
+	 * drop a newly injected interrupt while syncing guest-visible HVIP.
 	 */
 #define KVM_RISCV_VCPU_NR_IRQS	64
+	raw_spinlock_t irqs_pending_lock;
 	DECLARE_BITMAP(irqs_pending, KVM_RISCV_VCPU_NR_IRQS);
 	DECLARE_BITMAP(irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS);
 
diff --git a/arch/riscv/include/asm/pgtable-bits.h b/arch/riscv/include/asm/pgtable-bits.h
index b422d9691e60d..d5a86b4df3ce6 100644
--- a/arch/riscv/include/asm/pgtable-bits.h
+++ b/arch/riscv/include/asm/pgtable-bits.h
@@ -40,20 +40,20 @@
 
 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
 
-/* ext_svrsw60t59b: Bit(60) for uffd-wp tracking */
-#define _PAGE_UFFD_WP							\
+/* ext_svrsw60t59b: Bit(60) for userfaultfd tracking */
+#define _PAGE_UFFD							\
 	((riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)) ?	\
 	 (1UL << 60) : 0)
 /*
  * Bit 4 is not involved into swap entry computation, so we
- * can borrow it for swap page uffd-wp tracking.
+ * can borrow it for swap page userfaultfd tracking.
  */
-#define _PAGE_SWP_UFFD_WP						\
+#define _PAGE_SWP_UFFD						\
 	((riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)) ?	\
 	 _PAGE_USER : 0)
 #else
-#define _PAGE_UFFD_WP		0
-#define _PAGE_SWP_UFFD_WP	0
+#define _PAGE_UFFD		0
+#define _PAGE_SWP_UFFD		0
 #endif
 
 #define _PAGE_TABLE     _PAGE_PRESENT
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 5d5756bda82e3..3c17ad0786ab8 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -410,37 +410,37 @@ static inline pte_t pte_wrprotect(pte_t pte)
 }
 
 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
-#define pgtable_supports_uffd_wp()	\
+#define pgtable_supports_uffd()	\
 	riscv_has_extension_unlikely(RISCV_ISA_EXT_SVRSW60T59B)
 
-static inline bool pte_uffd_wp(pte_t pte)
+static inline bool pte_uffd(pte_t pte)
 {
-	return !!(pte_val(pte) & _PAGE_UFFD_WP);
+	return !!(pte_val(pte) & _PAGE_UFFD);
 }
 
-static inline pte_t pte_mkuffd_wp(pte_t pte)
+static inline pte_t pte_mkuffd(pte_t pte)
 {
-	return pte_wrprotect(__pte(pte_val(pte) | _PAGE_UFFD_WP));
+	return pte_wrprotect(__pte(pte_val(pte) | _PAGE_UFFD));
 }
 
-static inline pte_t pte_clear_uffd_wp(pte_t pte)
+static inline pte_t pte_clear_uffd(pte_t pte)
 {
-	return __pte(pte_val(pte) & ~(_PAGE_UFFD_WP));
+	return __pte(pte_val(pte) & ~(_PAGE_UFFD));
 }
 
-static inline bool pte_swp_uffd_wp(pte_t pte)
+static inline bool pte_swp_uffd(pte_t pte)
 {
-	return !!(pte_val(pte) & _PAGE_SWP_UFFD_WP);
+	return !!(pte_val(pte) & _PAGE_SWP_UFFD);
 }
 
-static inline pte_t pte_swp_mkuffd_wp(pte_t pte)
+static inline pte_t pte_swp_mkuffd(pte_t pte)
 {
-	return __pte(pte_val(pte) | _PAGE_SWP_UFFD_WP);
+	return __pte(pte_val(pte) | _PAGE_SWP_UFFD);
 }
 
-static inline pte_t pte_swp_clear_uffd_wp(pte_t pte)
+static inline pte_t pte_swp_clear_uffd(pte_t pte)
 {
-	return __pte(pte_val(pte) & ~(_PAGE_SWP_UFFD_WP));
+	return __pte(pte_val(pte) & ~(_PAGE_SWP_UFFD));
 }
 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
 
@@ -534,10 +534,7 @@ static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
 					PAGE_SIZE)
 #endif
 
-#ifdef CONFIG_NUMA_BALANCING
-/*
- * See the comment in include/asm-generic/pgtable.h
- */
+#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE
 static inline int pte_protnone(pte_t pte)
 {
 	return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE)) == _PAGE_PROT_NONE;
@@ -547,7 +544,7 @@ static inline int pmd_protnone(pmd_t pmd)
 {
 	return pte_protnone(pmd_pte(pmd));
 }
-#endif
+#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */
 
 /* Modify page protection bits */
 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
@@ -568,6 +565,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 		struct vm_area_struct *vma, unsigned long address,
 		pte_t *ptep, unsigned int nr)
 {
+	unsigned long asid;
+
 	/*
 	 * Svvptc guarantees that the new valid pte will be visible within
 	 * a bounded timeframe, so when the uarch does not cache invalid
@@ -576,6 +575,14 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC))
 		return;
 
+	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)) {
+		local_sfence_w_inval();
+		while (nr--)
+			local_sinval_vma(address + nr * PAGE_SIZE, asid);
+		local_sfence_inval_ir();
+		return;
+	}
+
 	/*
 	 * The kernel assumes that TLBs don't cache invalid entries, but
 	 * in RISC-V, SFENCE.VMA specifies an ordering constraint, not a
@@ -583,10 +590,11 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
 	 * Relying on flush_tlb_fix_spurious_fault would suffice, but
 	 * the extra traps reduce performance.  So, eagerly SFENCE.VMA.
 	 */
+	asid = get_mm_asid(vma->vm_mm);
 	while (nr--)
-		local_flush_tlb_page(address + nr * PAGE_SIZE);
-
+		local_flush_tlb_page_asid(address + nr * PAGE_SIZE, asid);
 }
+
 #define update_mmu_cache(vma, addr, ptep) \
 	update_mmu_cache_range(NULL, vma, addr, ptep, 1)
 
@@ -899,34 +907,34 @@ static inline pud_t pud_mkspecial(pud_t pud)
 #endif
 
 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
-static inline bool pmd_uffd_wp(pmd_t pmd)
+static inline bool pmd_uffd(pmd_t pmd)
 {
-	return pte_uffd_wp(pmd_pte(pmd));
+	return pte_uffd(pmd_pte(pmd));
 }
 
-static inline pmd_t pmd_mkuffd_wp(pmd_t pmd)
+static inline pmd_t pmd_mkuffd(pmd_t pmd)
 {
-	return pte_pmd(pte_mkuffd_wp(pmd_pte(pmd)));
+	return pte_pmd(pte_mkuffd(pmd_pte(pmd)));
 }
 
-static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd)
+static inline pmd_t pmd_clear_uffd(pmd_t pmd)
 {
-	return pte_pmd(pte_clear_uffd_wp(pmd_pte(pmd)));
+	return pte_pmd(pte_clear_uffd(pmd_pte(pmd)));
 }
 
-static inline bool pmd_swp_uffd_wp(pmd_t pmd)
+static inline bool pmd_swp_uffd(pmd_t pmd)
 {
-	return pte_swp_uffd_wp(pmd_pte(pmd));
+	return pte_swp_uffd(pmd_pte(pmd));
 }
 
-static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd)
+static inline pmd_t pmd_swp_mkuffd(pmd_t pmd)
 {
-	return pte_pmd(pte_swp_mkuffd_wp(pmd_pte(pmd)));
+	return pte_pmd(pte_swp_mkuffd(pmd_pte(pmd)));
 }
 
-static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd)
+static inline pmd_t pmd_swp_clear_uffd(pmd_t pmd)
 {
-	return pte_pmd(pte_swp_clear_uffd_wp(pmd_pte(pmd)));
+	return pte_pmd(pte_swp_clear_uffd(pmd_pte(pmd)));
 }
 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
 
@@ -946,7 +954,7 @@ static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
 	return pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd)));
 }
 
-#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
 static inline bool pmd_swp_soft_dirty(pmd_t pmd)
 {
 	return pte_swp_soft_dirty(pmd_pte(pmd));
@@ -961,7 +969,7 @@ static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
 {
 	return pte_pmd(pte_swp_clear_soft_dirty(pmd_pte(pmd)));
 }
-#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
+#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */
 #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
 
 static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
@@ -986,7 +994,14 @@ static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long
 
 static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
 {
-	return pmd_leaf(pmd) && pmd_user(pmd);
+	/*
+	 * page_table_check() must ignore THP split invalidation entries created by
+	 * pmd_mkinvalid(). These retain _PAGE_LEAF so pmd_present()/pmd_leaf() stay
+	 * true during the split, but they no longer describe a user-accessible
+	 * mapping once both _PAGE_PRESENT and _PAGE_PROT_NONE are cleared.
+	 */
+	return (pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROT_NONE)) &&
+		(pmd_val(pmd) & _PAGE_LEAF) && pmd_user(pmd);
 }
 
 static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
@@ -1170,7 +1185,7 @@ static inline pud_t pud_modify(pud_t pud, pgprot_t newprot)
  *	bit            0:	_PAGE_PRESENT (zero)
  *	bit       1 to 2:	(zero)
  *	bit            3:	_PAGE_SWP_SOFT_DIRTY
- *	bit            4:	_PAGE_SWP_UFFD_WP
+ *	bit            4:	_PAGE_SWP_UFFD
  *	bit            5:	_PAGE_PROT_NONE (zero)
  *	bit            6:	exclusive marker
  *	bits      7 to 11:	swap type
@@ -1208,10 +1223,10 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte)
 	return __pte(pte_val(pte) & ~_PAGE_SWP_EXCLUSIVE);
 }
 
-#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
 #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) })
 #define __swp_entry_to_pmd(swp) __pmd((swp).val)
-#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
+#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */
 
 /*
  * In the RV64 Linux scheme, we give the user half of the virtual-address space
diff --git a/arch/riscv/include/asm/syscall.h b/arch/riscv/include/asm/syscall.h
index 8067e666a4ca6..987c9a78806f7 100644
--- a/arch/riscv/include/asm/syscall.h
+++ b/arch/riscv/include/asm/syscall.h
@@ -112,11 +112,6 @@ static inline void syscall_handler(struct pt_regs *regs, ulong syscall)
 	regs->a0 = fn(regs);
 }
 
-static inline bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs)
-{
-	return false;
-}
-
 asmlinkage long sys_riscv_flush_icache(uintptr_t, uintptr_t, uintptr_t);
 
 asmlinkage long sys_riscv_hwprobe(struct riscv_hwprobe *, size_t, size_t,
diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h
index eed0abc405143..9636d07fe9eed 100644
--- a/arch/riscv/include/asm/tlbflush.h
+++ b/arch/riscv/include/asm/tlbflush.h
@@ -15,6 +15,29 @@
 #define FLUSH_TLB_NO_ASID       ((unsigned long)-1)
 
 #ifdef CONFIG_MMU
+static inline unsigned long get_mm_asid(struct mm_struct *mm)
+{
+	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
+}
+
+static inline void local_sfence_inval_ir(void)
+{
+	asm volatile(SFENCE_INVAL_IR() ::: "memory");
+}
+
+static inline void local_sfence_w_inval(void)
+{
+	asm volatile(SFENCE_W_INVAL() ::: "memory");
+}
+
+static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
+{
+	if (asid != FLUSH_TLB_NO_ASID)
+		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
+	else
+		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
+}
+
 static inline void local_flush_tlb_all(void)
 {
 	__asm__ __volatile__ ("sfence.vma" : : : "memory");
diff --git a/arch/riscv/kernel/acpi.c b/arch/riscv/kernel/acpi.c
index 068e0b404b6f4..efd52a5d05b5c 100644
--- a/arch/riscv/kernel/acpi.c
+++ b/arch/riscv/kernel/acpi.c
@@ -353,3 +353,8 @@ int acpi_get_cpu_uid(unsigned int cpu, u32 *uid)
 	return 0;
 }
 EXPORT_SYMBOL_GPL(acpi_get_cpu_uid);
+
+void __init arch_reserve_mem_area(acpi_physical_address addr, size_t size)
+{
+	memblock_mark_nomap(addr, size);
+}
diff --git a/arch/riscv/kernel/efi.c b/arch/riscv/kernel/efi.c
index b64bf1624a052..2d3cc57b45352 100644
--- a/arch/riscv/kernel/efi.c
+++ b/arch/riscv/kernel/efi.c
@@ -95,3 +95,8 @@ int __init efi_set_mapping_permissions(struct mm_struct *mm,
 				   md->num_pages << EFI_PAGE_SHIFT,
 				   set_permissions, md);
 }
+
+bool efi_poweroff_required(void)
+{
+	return efi_enabled(EFI_RUNTIME_SERVICES);
+}
diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
index 08df724e13b9d..4bb78dff1f388 100644
--- a/arch/riscv/kernel/entry.S
+++ b/arch/riscv/kernel/entry.S
@@ -76,6 +76,8 @@
 	amoxor.d	a0, a1, (a0)
 
 	/*
+	 * Pairs with the smp_wmb() in mark_new_valid_map()
+	 *
 	 * A sfence.vma is required here. Even if we had Svvptc, there's no
 	 * guarantee that after returning we wouldn't just fault again.
 	 */
@@ -137,17 +139,12 @@ SYM_CODE_START(handle_exception)
 .Lrestore_kernel_tpsp:
 	csrr tp, CSR_SCRATCH
 
-#ifdef CONFIG_64BIT
+#if defined(CONFIG_64BIT) && defined(CONFIG_MMU)
 	/*
 	 * The RISC-V kernel does not flush TLBs on all CPUS after each new
 	 * vmalloc mapping or kfence_unprotect(), which may result in
-	 * exceptions:
-	 *
-	 * - if the uarch caches invalid entries, the new mapping would not be
-	 *   observed by the page table walker and an invalidation is needed.
-	 * - if the uarch does not cache invalid entries, a reordered access
-	 *   could "miss" the new mapping and traps: in that case, we only need
-	 *   to retry the access, no sfence.vma is required.
+	 * exceptions. In that case, we need to sfence.vma to "receive" the new
+	 * mappings and retry, whether or not we have Svvptc.
 	 */
 	new_valid_map_cpus_check
 #endif
diff --git a/arch/riscv/kernel/machine_kexec_file.c b/arch/riscv/kernel/machine_kexec_file.c
index 59d4bbc848a89..26cd2a8bd0cd3 100644
--- a/arch/riscv/kernel/machine_kexec_file.c
+++ b/arch/riscv/kernel/machine_kexec_file.c
@@ -45,6 +45,15 @@ static int get_nr_ram_ranges_callback(struct resource *res, void *arg)
 	return 0;
 }
 
+unsigned int arch_get_system_nr_ranges(void)
+{
+	unsigned int nr_ranges = 2 + crashk_cma_cnt; /* For exclusion of crashkernel region */
+
+	walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback);
+
+	return nr_ranges;
+}
+
 static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg)
 {
 	struct crash_mem *cmem = arg;
@@ -56,33 +65,9 @@ static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg)
 	return 0;
 }
 
-static int prepare_elf_headers(void **addr, unsigned long *sz)
+int arch_crash_populate_cmem(struct crash_mem *cmem)
 {
-	struct crash_mem *cmem;
-	unsigned int nr_ranges;
-	int ret;
-
-	nr_ranges = 1; /* For exclusion of crashkernel region */
-	walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback);
-
-	cmem = kmalloc_flex(*cmem, ranges, nr_ranges);
-	if (!cmem)
-		return -ENOMEM;
-
-	cmem->max_nr_ranges = nr_ranges;
-	cmem->nr_ranges = 0;
-	ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback);
-	if (ret)
-		goto out;
-
-	/* Exclude crashkernel region */
-	ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end);
-	if (!ret)
-		ret = crash_prepare_elf64_headers(cmem, true, addr, sz);
-
-out:
-	kfree(cmem);
-	return ret;
+	return walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback);
 }
 
 static char *setup_kdump_cmdline(struct kimage *image, char *cmdline,
@@ -274,7 +259,7 @@ int load_extra_segments(struct kimage *image, unsigned long kernel_start,
 	if (image->type == KEXEC_TYPE_CRASH) {
 		void *headers;
 		unsigned long headers_sz;
-		ret = prepare_elf_headers(&headers, &headers_sz);
+		ret = crash_prepare_headers(true, &headers, &headers_sz, NULL);
 		if (ret) {
 			pr_err("Preparing elf core header failed\n");
 			goto out;
diff --git a/arch/riscv/kernel/probes/decode-insn.c b/arch/riscv/kernel/probes/decode-insn.c
index 65d9590bfb9ff..433d9035b8882 100644
--- a/arch/riscv/kernel/probes/decode-insn.c
+++ b/arch/riscv/kernel/probes/decode-insn.c
@@ -29,12 +29,12 @@ riscv_probe_decode_insn(probe_opcode_t *addr, struct arch_probe_insn *api)
 	 * TODO: the REJECTED ones below need to be implemented
 	 */
 #ifdef CONFIG_RISCV_ISA_C
-	RISCV_INSN_REJECTED(c_jal,		insn);
 	RISCV_INSN_REJECTED(c_ebreak,		insn);
 
 	RISCV_INSN_SET_SIMULATE(c_j,		insn);
 	RISCV_INSN_SET_SIMULATE(c_jr,		insn);
 	RISCV_INSN_SET_SIMULATE(c_jalr,		insn);
+	RISCV_INSN_SET_SIMULATE(c_jal,		insn);
 	RISCV_INSN_SET_SIMULATE(c_beqz,		insn);
 	RISCV_INSN_SET_SIMULATE(c_bnez,		insn);
 #endif
diff --git a/arch/riscv/kernel/probes/simulate-insn.c b/arch/riscv/kernel/probes/simulate-insn.c
index fa581590c1f8b..f8a2f68578774 100644
--- a/arch/riscv/kernel/probes/simulate-insn.c
+++ b/arch/riscv/kernel/probes/simulate-insn.c
@@ -163,6 +163,13 @@ bool __kprobes simulate_c_j(u32 opcode, unsigned long addr, struct pt_regs *regs
 	return true;
 }
 
+bool __kprobes simulate_c_jal(u32 opcode, unsigned long addr, struct pt_regs *regs)
+{
+	regs->ra = addr + 2;
+
+	return simulate_c_j(opcode, addr, regs);
+}
+
 static bool __kprobes simulate_c_jr_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs,
 					 bool is_jalr)
 {
diff --git a/arch/riscv/kernel/probes/simulate-insn.h b/arch/riscv/kernel/probes/simulate-insn.h
index 44ebbc444db9d..b89e1bb01842e 100644
--- a/arch/riscv/kernel/probes/simulate-insn.h
+++ b/arch/riscv/kernel/probes/simulate-insn.h
@@ -25,6 +25,7 @@ bool simulate_branch(u32 opcode, unsigned long addr, struct pt_regs *regs);
 bool simulate_jal(u32 opcode, unsigned long addr, struct pt_regs *regs);
 bool simulate_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs);
 bool simulate_c_j(u32 opcode, unsigned long addr, struct pt_regs *regs);
+bool simulate_c_jal(u32 opcode, unsigned long addr, struct pt_regs *regs);
 bool simulate_c_jr(u32 opcode, unsigned long addr, struct pt_regs *regs);
 bool simulate_c_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs);
 bool simulate_c_bnez(u32 opcode, unsigned long addr, struct pt_regs *regs);
diff --git a/arch/riscv/kernel/reset.c b/arch/riscv/kernel/reset.c
index 9122885722265..541e2162c85ac 100644
--- a/arch/riscv/kernel/reset.c
+++ b/arch/riscv/kernel/reset.c
@@ -3,6 +3,7 @@
  * Copyright (C) 2012 Regents of the University of California
  */
 
+#include <linux/efi.h>
 #include <linux/reboot.h>
 #include <linux/pm.h>
 
@@ -17,6 +18,12 @@ EXPORT_SYMBOL(pm_power_off);
 
 void machine_restart(char *cmd)
 {
+	/*
+	 * UpdateCapsule() depends on the system being reset via ResetSystem().
+	 */
+	if (efi_enabled(EFI_RUNTIME_SERVICES))
+		efi_reboot(reboot_mode, NULL);
+
 	do_kernel_restart(cmd);
 	while (1);
 }
diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c
index 52d1d2b8f338b..a32344bb220df 100644
--- a/arch/riscv/kernel/setup.c
+++ b/arch/riscv/kernel/setup.c
@@ -321,6 +321,8 @@ void __init setup_arch(char **cmdline_p)
 	efi_init();
 	paging_init();
 
+	acpi_table_upgrade();
+
 	/* Parse the ACPI tables for possible boot-time configuration */
 	acpi_boot_table_init();
 
diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c
index 1659d31fd288f..caf6762427c87 100644
--- a/arch/riscv/kernel/sys_hwprobe.c
+++ b/arch/riscv/kernel/sys_hwprobe.c
@@ -450,6 +450,7 @@ static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs,
 	if (cpusetsize > cpumask_size())
 		cpusetsize = cpumask_size();
 
+	cpumask_clear(&cpus);
 	ret = copy_from_user(&cpus, cpus_user, cpusetsize);
 	if (ret)
 		return -EFAULT;
diff --git a/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S b/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S
index f16deee9e0915..15e50934bbd7b 100644
--- a/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S
+++ b/arch/riscv/kernel/tests/kprobes/test-kprobes-asm.S
@@ -179,6 +179,23 @@ test_kprobes_c_bnez_addr3:
 	ret
 SYM_FUNC_END(test_kprobes_c_bnez)
 
+#ifdef CONFIG_32BIT
+SYM_FUNC_START(test_kprobes_c_jal)
+	li a0, 0
+	mv a1, ra
+test_kprobes_c_jal_addr1:
+	c.jal 2f
+	ret
+1:	li a0, KPROBE_TEST_MAGIC_UPPER
+	ret
+test_kprobes_c_jal_addr2:
+2:	c.jal 1b
+	li a2, KPROBE_TEST_MAGIC_LOWER
+	add a0, a0, a2
+	jr a1
+SYM_FUNC_END(test_kprobes_c_jal)
+#endif
+
 #endif /* CONFIG_RISCV_ISA_C */
 
 .section .rodata
@@ -209,6 +226,10 @@ SYM_DATA_START(test_kprobes_addresses)
 	RISCV_PTR test_kprobes_c_bnez_addr1
 	RISCV_PTR test_kprobes_c_bnez_addr2
 	RISCV_PTR test_kprobes_c_bnez_addr3
+#ifdef CONFIG_32BIT
+	RISCV_PTR test_kprobes_c_jal_addr1
+	RISCV_PTR test_kprobes_c_jal_addr2
+#endif
 #endif /* CONFIG_RISCV_ISA_C */
 	RISCV_PTR 0
 SYM_DATA_END(test_kprobes_addresses)
@@ -226,6 +247,9 @@ SYM_DATA_START(test_kprobes_functions)
 	RISCV_PTR test_kprobes_c_jalr
 	RISCV_PTR test_kprobes_c_beqz
 	RISCV_PTR test_kprobes_c_bnez
+#ifdef CONFIG_32BIT
+	RISCV_PTR test_kprobes_c_jal
+#endif
 #endif /* CONFIG_RISCV_ISA_C */
 	RISCV_PTR 0
 SYM_DATA_END(test_kprobes_functions)
diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c
index 8c62c771a6566..2f57fd48ec28b 100644
--- a/arch/riscv/kernel/traps.c
+++ b/arch/riscv/kernel/traps.c
@@ -7,7 +7,6 @@
 #include <linux/kernel.h>
 #include <linux/init.h>
 #include <linux/irqflags.h>
-#include <linux/randomize_kstack.h>
 #include <linux/sched.h>
 #include <linux/sched/debug.h>
 #include <linux/sched/signal.h>
@@ -333,15 +332,12 @@ void do_trap_ecall_u(struct pt_regs *regs)
 
 		riscv_v_vstate_discard(regs);
 
-		syscall = syscall_enter_from_user_mode(regs, syscall);
-
-		add_random_kstack_offset();
-
-		if (syscall >= 0 && syscall < NR_syscalls) {
-			syscall = array_index_nospec(syscall, NR_syscalls);
-			syscall_handler(regs, syscall);
+		if (likely(syscall_enter_from_user_mode_randomize_stack(regs, &syscall))) {
+			if (syscall >= 0 && syscall < NR_syscalls) {
+				syscall = array_index_nospec(syscall, NR_syscalls);
+				syscall_handler(regs, syscall);
+			}
 		}
-
 		syscall_exit_to_user_mode(regs);
 	} else {
 		irqentry_state_t state = irqentry_nmi_enter(regs);
diff --git a/arch/riscv/kernel/usercfi.c b/arch/riscv/kernel/usercfi.c
index 5a7113d69badc..f027e6e05251e 100644
--- a/arch/riscv/kernel/usercfi.c
+++ b/arch/riscv/kernel/usercfi.c
@@ -110,7 +110,7 @@ void set_indir_lp_lock(struct task_struct *task, bool lock)
 }
 /*
  * The shadow stack only stores the return address and not any variables
- * this should be more than sufficient for most applications.
+ * 512M should be more than sufficient for most applications.
  * Else PAGE_ALIGN it and return back
  */
 static unsigned long calc_shstk_size(unsigned long size)
@@ -118,7 +118,7 @@ static unsigned long calc_shstk_size(unsigned long size)
 	if (size)
 		return PAGE_ALIGN(size);
 
-	return PAGE_ALIGN(min(rlimit(RLIMIT_STACK) / 2, SZ_2G));
+	return PAGE_ALIGN(min(rlimit(RLIMIT_STACK) / 8, SZ_512M));
 }
 
 /*
diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile
index a842dc034571d..8dbf2532a5730 100644
--- a/arch/riscv/kernel/vdso/Makefile
+++ b/arch/riscv/kernel/vdso/Makefile
@@ -41,6 +41,8 @@ ccflags-y += $(CFI_FULL)
 asflags-y += $(KBUILD_BASE_ISA)$(CFI_MARCH)
 asflags-y += $(CFI_FULL)
 
+ccflags-remove-y += $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_LTO)
+
 ifneq ($(c-gettimeofday-y),)
   CFLAGS_vgettimeofday.o += -fPIC -include $(c-gettimeofday-y)
 endif
@@ -68,11 +70,6 @@ ifneq ($(filter vgettimeofday, $(vdso-syms)),)
 CPPFLAGS_$(vdso_lds) += -DHAS_VGETTIMEOFDAY
 endif
 
-# Disable -pg to prevent insert call site
-CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS)
-CFLAGS_REMOVE_getrandom.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS)
-CFLAGS_REMOVE_hwprobe.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS)
-
 # Force dependency
 $(obj)/$(vdso_o): $(obj)/$(vdso_so)
 
diff --git a/arch/riscv/kernel/vdso/rt_sigreturn.S b/arch/riscv/kernel/vdso/rt_sigreturn.S
index e82987dc37394..d6f96b1abe409 100644
--- a/arch/riscv/kernel/vdso/rt_sigreturn.S
+++ b/arch/riscv/kernel/vdso/rt_sigreturn.S
@@ -7,11 +7,19 @@
 #include <asm/unistd.h>
 #include <asm/assembler.h>
 
+/*
+ * WARNING: Do NOT add a CFI landing pad at the start of this function.
+ * Unwinders such as libgcc identify the sigreturn trampoline by matching the
+ * instruction sequence. Adding a landing pad here would break unwinding from
+ * signal handlers.
+ *
+ * This trampoline is used only for signal return and not via an indirect
+ * call/jump from userspace, so adding CFI landing pad is unnecessary.
+ */
 	.text
 SYM_FUNC_START(__vdso_rt_sigreturn)
 	.cfi_startproc
 	.cfi_signal_frame
-	vdso_lpad
 	li a7, __NR_rt_sigreturn
 	ecall
 	.cfi_endproc
diff --git a/arch/riscv/kvm/aia.c b/arch/riscv/kvm/aia.c
index bafb009c5ce55..9a653b4ad40a5 100644
--- a/arch/riscv/kvm/aia.c
+++ b/arch/riscv/kvm/aia.c
@@ -53,12 +53,15 @@ void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu)
 	struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
 	unsigned long mask, val;
 
+	lockdep_assert_held(&vcpu->arch.irqs_pending_lock);
+
 	if (!kvm_riscv_aia_available())
 		return;
 
-	if (READ_ONCE(vcpu->arch.irqs_pending_mask[1])) {
-		mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[1], 0);
-		val = READ_ONCE(vcpu->arch.irqs_pending[1]) & mask;
+	mask = vcpu->arch.irqs_pending_mask[1];
+	if (mask) {
+		vcpu->arch.irqs_pending_mask[1] = 0;
+		val = vcpu->arch.irqs_pending[1] & mask;
 
 		csr->hviph &= ~mask;
 		csr->hviph |= val;
@@ -69,6 +72,8 @@ void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu)
 {
 	struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
 
+	lockdep_assert_held(&vcpu->arch.irqs_pending_lock);
+
 	if (kvm_riscv_aia_available())
 		csr->vsieh = ncsr_read(CSR_VSIEH);
 }
@@ -77,13 +82,22 @@ void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu)
 bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)
 {
 	unsigned long seip;
+#ifdef CONFIG_32BIT
+	unsigned long flags;
+	bool pending;
+#endif
 
 	if (!kvm_riscv_aia_available())
 		return false;
 
 #ifdef CONFIG_32BIT
-	if (READ_ONCE(vcpu->arch.irqs_pending[1]) &
-	    (vcpu->arch.aia_context.guest_csr.vsieh & upper_32_bits(mask)))
+	raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
+	pending = vcpu->arch.irqs_pending[1] &
+		  (vcpu->arch.aia_context.guest_csr.vsieh &
+		   upper_32_bits(mask));
+	raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
+
+	if (pending)
 		return true;
 #endif
 
@@ -207,6 +221,9 @@ int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu,
 {
 	struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
 	unsigned long regs_max = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long);
+#ifdef CONFIG_32BIT
+	unsigned long flags;
+#endif
 
 	if (!riscv_isa_extension_available(vcpu->arch.isa, SSAIA))
 		return -ENOENT;
@@ -219,8 +236,12 @@ int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu,
 		((unsigned long *)csr)[reg_num] = val;
 
 #ifdef CONFIG_32BIT
-		if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph))
-			WRITE_ONCE(vcpu->arch.irqs_pending_mask[1], 0);
+		if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) {
+			raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
+			vcpu->arch.irqs_pending_mask[1] = 0;
+			raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock,
+						   flags);
+		}
 #endif
 	}
 
diff --git a/arch/riscv/kvm/gstage.c b/arch/riscv/kvm/gstage.c
index c4c3b79567f10..b0474fcf065aa 100644
--- a/arch/riscv/kvm/gstage.c
+++ b/arch/riscv/kvm/gstage.c
@@ -5,11 +5,13 @@
  */
 
 #include <linux/bitops.h>
+#include <linux/cpufeature.h>
 #include <linux/errno.h>
 #include <linux/kvm_host.h>
 #include <linux/module.h>
 #include <linux/pgtable.h>
 #include <asm/kvm_gstage.h>
+#include <asm/hwcap.h>
 
 #ifdef CONFIG_64BIT
 unsigned long kvm_riscv_gstage_max_pgd_levels __ro_after_init = 3;
@@ -171,8 +173,10 @@ int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage,
 	}
 
 	if (pte_val(*ptep) != pte_val(map->pte)) {
+		bool was_invalid = !pte_val(*ptep);
 		set_pte(ptep, map->pte);
-		if (gstage_pte_leaf(ptep))
+		if (gstage_pte_leaf(ptep) &&
+		    !(was_invalid && riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC)))
 			gstage_tlb_flush(gstage, current_level, map->addr);
 	}
 
diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c
index 082f9b2617338..8a0aa5e0e216e 100644
--- a/arch/riscv/kvm/mmu.c
+++ b/arch/riscv/kvm/mmu.c
@@ -41,6 +41,7 @@ int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa,
 	pgprot_t prot;
 	unsigned long pfn;
 	phys_addr_t addr, end;
+	unsigned long pgd_levels = kvm->arch.pgd_levels;
 	struct kvm_mmu_memory_cache pcache = {
 		.gfp_custom = (in_atomic) ? GFP_ATOMIC | __GFP_ACCOUNT : 0,
 		.gfp_zero = __GFP_ZERO,
@@ -63,7 +64,7 @@ int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa,
 		if (!writable)
 			map.pte = pte_wrprotect(map.pte);
 
-		ret = kvm_mmu_topup_memory_cache(&pcache, kvm->arch.pgd_levels);
+		ret = __kvm_mmu_topup_memory_cache(&pcache, pgd_levels, pgd_levels);
 		if (ret)
 			goto out;
 
diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c
index cf6e231e76e25..977e36ab83d3f 100644
--- a/arch/riscv/kvm/vcpu.c
+++ b/arch/riscv/kvm/vcpu.c
@@ -80,6 +80,7 @@ static void kvm_riscv_vcpu_context_reset(struct kvm_vcpu *vcpu,
 
 static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset)
 {
+	unsigned long flags;
 	bool loaded;
 
 	/**
@@ -104,8 +105,10 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset)
 
 	kvm_riscv_vcpu_aia_reset(vcpu);
 
+	raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
 	bitmap_zero(vcpu->arch.irqs_pending, KVM_RISCV_VCPU_NR_IRQS);
 	bitmap_zero(vcpu->arch.irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS);
+	raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
 
 	kvm_riscv_vcpu_pmu_reset(vcpu);
 
@@ -151,6 +154,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
 
 	/* Setup VCPU hfence queue */
 	spin_lock_init(&vcpu->arch.hfence_lock);
+	raw_spin_lock_init(&vcpu->arch.irqs_pending_lock);
 
 	spin_lock_init(&vcpu->arch.reset_state.lock);
 
@@ -352,10 +356,14 @@ void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu)
 {
 	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 	unsigned long mask, val;
+	unsigned long flags;
 
-	if (READ_ONCE(vcpu->arch.irqs_pending_mask[0])) {
-		mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[0], 0);
-		val = READ_ONCE(vcpu->arch.irqs_pending[0]) & mask;
+	raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
+
+	mask = vcpu->arch.irqs_pending_mask[0];
+	if (mask) {
+		vcpu->arch.irqs_pending_mask[0] = 0;
+		val = vcpu->arch.irqs_pending[0] & mask;
 
 		csr->hvip &= ~mask;
 		csr->hvip |= val;
@@ -363,11 +371,14 @@ void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu)
 
 	/* Flush AIA high interrupts */
 	kvm_riscv_vcpu_aia_flush_interrupts(vcpu);
+
+	raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
 }
 
 void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu)
 {
 	unsigned long hvip;
+	unsigned long flags;
 	struct kvm_vcpu_arch *v = &vcpu->arch;
 	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 
@@ -376,34 +387,41 @@ void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu)
 
 	/* Sync-up HVIP.VSSIP bit changes does by Guest */
 	hvip = ncsr_read(CSR_HVIP);
+
+	raw_spin_lock_irqsave(&v->irqs_pending_lock, flags);
+
 	if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) {
 		if (hvip & (1UL << IRQ_VS_SOFT)) {
-			if (!test_and_set_bit(IRQ_VS_SOFT,
-					      v->irqs_pending_mask))
-				set_bit(IRQ_VS_SOFT, v->irqs_pending);
+			if (!__test_and_set_bit(IRQ_VS_SOFT,
+						v->irqs_pending_mask))
+				__set_bit(IRQ_VS_SOFT, v->irqs_pending);
 		} else {
-			if (!test_and_set_bit(IRQ_VS_SOFT,
-					      v->irqs_pending_mask))
-				clear_bit(IRQ_VS_SOFT, v->irqs_pending);
+			if (!__test_and_set_bit(IRQ_VS_SOFT,
+						v->irqs_pending_mask))
+				__clear_bit(IRQ_VS_SOFT, v->irqs_pending);
 		}
 	}
 
 	/* Sync up the HVIP.LCOFIP bit changes (only clear) by the guest */
 	if ((csr->hvip ^ hvip) & (1UL << IRQ_PMU_OVF)) {
 		if (!(hvip & (1UL << IRQ_PMU_OVF)) &&
-		    !test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask))
-			clear_bit(IRQ_PMU_OVF, v->irqs_pending);
+		    !__test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask))
+			__clear_bit(IRQ_PMU_OVF, v->irqs_pending);
 	}
 
 	/* Sync-up AIA high interrupts */
 	kvm_riscv_vcpu_aia_sync_interrupts(vcpu);
 
+	raw_spin_unlock_irqrestore(&v->irqs_pending_lock, flags);
+
 	/* Sync-up timer CSRs */
 	kvm_riscv_vcpu_timer_sync(vcpu);
 }
 
 int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
 {
+	unsigned long flags;
+
 	/*
 	 * We only allow VS-mode software, timer, and external
 	 * interrupts when irq is one of the local interrupts
@@ -416,9 +434,10 @@ int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
 	    irq != IRQ_PMU_OVF)
 		return -EINVAL;
 
-	set_bit(irq, vcpu->arch.irqs_pending);
-	smp_mb__before_atomic();
-	set_bit(irq, vcpu->arch.irqs_pending_mask);
+	raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
+	__set_bit(irq, vcpu->arch.irqs_pending);
+	__set_bit(irq, vcpu->arch.irqs_pending_mask);
+	raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
 
 	kvm_vcpu_kick(vcpu);
 
@@ -427,6 +446,8 @@ int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
 
 int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
 {
+	unsigned long flags;
+
 	/*
 	 * We only allow VS-mode software, timer, counter overflow and external
 	 * interrupts when irq is one of the local interrupts
@@ -439,26 +460,33 @@ int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
 	    irq != IRQ_PMU_OVF)
 		return -EINVAL;
 
-	clear_bit(irq, vcpu->arch.irqs_pending);
-	smp_mb__before_atomic();
-	set_bit(irq, vcpu->arch.irqs_pending_mask);
+	raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
+	__clear_bit(irq, vcpu->arch.irqs_pending);
+	__set_bit(irq, vcpu->arch.irqs_pending_mask);
+	raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
 
 	return 0;
 }
 
 bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)
 {
+	unsigned long flags;
 	unsigned long ie;
+	bool ret;
 
+	raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
 	ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK)
 		<< VSIP_TO_HVIP_SHIFT) & (unsigned long)mask;
 	ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK &
 		(unsigned long)mask;
-	if (READ_ONCE(vcpu->arch.irqs_pending[0]) & ie)
-		return true;
+	ret = vcpu->arch.irqs_pending[0] & ie;
+	raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
 
 	/* Check AIA high interrupts */
-	return kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask);
+	if (!ret)
+		ret = kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask);
+
+	return ret;
 }
 
 void __kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu)
diff --git a/arch/riscv/kvm/vcpu_exit.c b/arch/riscv/kvm/vcpu_exit.c
index 0bb0c51e3c890..6c8530b9f29ed 100644
--- a/arch/riscv/kvm/vcpu_exit.c
+++ b/arch/riscv/kvm/vcpu_exit.c
@@ -38,6 +38,25 @@ static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run,
 			return kvm_riscv_vcpu_mmio_store(vcpu, run,
 							 fault_addr,
 							 trap->htinst);
+		case EXC_INST_GUEST_PAGE_FAULT: {
+			/*
+			 * No memslot backs this GPA and an instruction fetch
+			 * cannot be emulated as MMIO. On bare metal a fetch
+			 * from an unbacked physical address raises an
+			 * instruction access fault, so reflect that back to
+			 * the guest.
+			 */
+			struct kvm_cpu_trap inst_trap = {
+				.sepc	= trap->sepc,
+				.scause	= EXC_INST_ACCESS,
+				.stval	= trap->stval,
+				.htval	= 0,
+				.htinst	= 0,
+			};
+
+			kvm_riscv_vcpu_trap_redirect(vcpu, &inst_trap);
+			return 1;
+		}
 		default:
 			return -EOPNOTSUPP;
 		};
diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c
index bb920e8923c93..99b9107b1ac18 100644
--- a/arch/riscv/kvm/vcpu_onereg.c
+++ b/arch/riscv/kvm/vcpu_onereg.c
@@ -50,19 +50,13 @@ static int kvm_riscv_vcpu_get_reg_config(struct kvm_vcpu *vcpu,
 		reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK;
 		break;
 	case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size):
-		if (kvm_riscv_isa_check_host(ZICBOM))
-			return -ENOENT;
-		reg_val = riscv_cbom_block_size;
+		reg_val = (kvm_riscv_isa_check_host(ZICBOM)) ? 0 : riscv_cbom_block_size;
 		break;
 	case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size):
-		if (kvm_riscv_isa_check_host(ZICBOZ))
-			return -ENOENT;
-		reg_val = riscv_cboz_block_size;
+		reg_val = (kvm_riscv_isa_check_host(ZICBOZ)) ? 0 : riscv_cboz_block_size;
 		break;
 	case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size):
-		if (kvm_riscv_isa_check_host(ZICBOP))
-			return -ENOENT;
-		reg_val = riscv_cbop_block_size;
+		reg_val = (kvm_riscv_isa_check_host(ZICBOP)) ? 0 : riscv_cbop_block_size;
 		break;
 	case KVM_REG_RISCV_CONFIG_REG(mvendorid):
 		reg_val = vcpu->arch.mvendorid;
@@ -144,21 +138,15 @@ static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu,
 		}
 		break;
 	case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size):
-		if (kvm_riscv_isa_check_host(ZICBOM))
-			return -ENOENT;
-		if (reg_val != riscv_cbom_block_size)
+		if (reg_val && reg_val != riscv_cbom_block_size)
 			return -EINVAL;
 		break;
 	case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size):
-		if (kvm_riscv_isa_check_host(ZICBOZ))
-			return -ENOENT;
-		if (reg_val != riscv_cboz_block_size)
+		if (reg_val && reg_val != riscv_cboz_block_size)
 			return -EINVAL;
 		break;
 	case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size):
-		if (kvm_riscv_isa_check_host(ZICBOP))
-			return -ENOENT;
-		if (reg_val != riscv_cbop_block_size)
+		if (reg_val && reg_val != riscv_cbop_block_size)
 			return -EINVAL;
 		break;
 	case KVM_REG_RISCV_CONFIG_REG(mvendorid):
@@ -298,6 +286,7 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu,
 {
 	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 	unsigned long regs_max = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long);
+	unsigned long flags;
 
 	if (reg_num >= regs_max)
 		return -ENOENT;
@@ -311,8 +300,11 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu,
 
 	((unsigned long *)csr)[reg_num] = reg_val;
 
-	if (reg_num == KVM_REG_RISCV_CSR_REG(sip))
-		WRITE_ONCE(vcpu->arch.irqs_pending_mask[0], 0);
+	if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) {
+		raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
+		vcpu->arch.irqs_pending_mask[0] = 0;
+		raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags);
+	}
 
 	return 0;
 }
@@ -614,20 +606,6 @@ static int copy_config_reg_indices(const struct kvm_vcpu *vcpu,
 		u64 size;
 		u64 reg;
 
-		/*
-		 * Avoid reporting config reg if the corresponding extension
-		 * was not available.
-		 */
-		if (i == KVM_REG_RISCV_CONFIG_REG(zicbom_block_size) &&
-		    kvm_riscv_isa_check_host(ZICBOM))
-			continue;
-		else if (i == KVM_REG_RISCV_CONFIG_REG(zicboz_block_size) &&
-			 kvm_riscv_isa_check_host(ZICBOZ))
-			continue;
-		else if (i == KVM_REG_RISCV_CONFIG_REG(zicbop_block_size) &&
-			 kvm_riscv_isa_check_host(ZICBOP))
-			continue;
-
 		size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64;
 		reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CONFIG | i;
 
diff --git a/arch/riscv/kvm/vcpu_pmu.c b/arch/riscv/kvm/vcpu_pmu.c
index bb46dcbfb24da..2025b664961c8 100644
--- a/arch/riscv/kvm/vcpu_pmu.c
+++ b/arch/riscv/kvm/vcpu_pmu.c
@@ -586,7 +586,7 @@ int kvm_riscv_vcpu_pmu_ctr_start(struct kvm_vcpu *vcpu, unsigned long ctr_base,
 		}
 	}
 	/* Start the counters that have been configured and requested by the guest */
-	for_each_set_bit(i, &ctr_mask, RISCV_MAX_COUNTERS) {
+	for_each_set_bit(i, &ctr_mask, BITS_PER_LONG) {
 		pmc_index = array_index_nospec(i + ctr_base,
 					       RISCV_KVM_MAX_COUNTERS);
 		if (!test_bit(pmc_index, kvpmu->pmc_in_use))
@@ -658,7 +658,7 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base,
 	}
 
 	/* Stop the counters that have been configured and requested by the guest */
-	for_each_set_bit(i, &ctr_mask, RISCV_MAX_COUNTERS) {
+	for_each_set_bit(i, &ctr_mask, BITS_PER_LONG) {
 		pmc_index = array_index_nospec(i + ctr_base,
 					       RISCV_KVM_MAX_COUNTERS);
 		if (!test_bit(pmc_index, kvpmu->pmc_in_use))
diff --git a/arch/riscv/kvm/vcpu_sbi_fwft.c b/arch/riscv/kvm/vcpu_sbi_fwft.c
index ab39ac464ffd8..1342adb3180c6 100644
--- a/arch/riscv/kvm/vcpu_sbi_fwft.c
+++ b/arch/riscv/kvm/vcpu_sbi_fwft.c
@@ -327,9 +327,11 @@ static int kvm_sbi_fwft_set(struct kvm_vcpu *vcpu, u32 feature,
 	if (conf->flags & SBI_FWFT_SET_FLAG_LOCK)
 		return SBI_ERR_DENIED_LOCKED;
 
-	conf->flags = flags;
+	ret = conf->feature->set(vcpu, conf, false, value);
+	if (ret == SBI_SUCCESS)
+		conf->flags = flags;
 
-	return conf->feature->set(vcpu, conf, false, value);
+	return ret;
 }
 
 static int kvm_sbi_fwft_get(struct kvm_vcpu *vcpu, unsigned long feature,
diff --git a/arch/riscv/kvm/vcpu_vector.c b/arch/riscv/kvm/vcpu_vector.c
index 62d2fb77bb9b9..3708616e2c327 100644
--- a/arch/riscv/kvm/vcpu_vector.c
+++ b/arch/riscv/kvm/vcpu_vector.c
@@ -10,6 +10,7 @@
 #include <linux/errno.h>
 #include <linux/err.h>
 #include <linux/kvm_host.h>
+#include <linux/nospec.h>
 #include <linux/uaccess.h>
 #include <asm/cpufeature.h>
 #include <asm/kvm_isa.h>
@@ -129,11 +130,20 @@ static int kvm_riscv_vcpu_vreg_addr(struct kvm_vcpu *vcpu,
 			return -ENOENT;
 		}
 	} else if (reg_num <= KVM_REG_RISCV_VECTOR_REG(31)) {
+		unsigned long reg_offset;
+
 		if (reg_size != vlenb)
 			return -EINVAL;
 		WARN_ON(!cntx->vector.datap);
-		*reg_addr = cntx->vector.datap +
-			    (reg_num - KVM_REG_RISCV_VECTOR_REG(0)) * vlenb;
+		/*
+		 * The reg_num is derived from the userspace-provided ONE_REG
+		 * id. Sanitize it with array_index_nospec() to prevent
+		 * speculative out-of-bounds access to the vector register
+		 * buffer (32 vector registers: v0..v31).
+		 */
+		reg_offset = array_index_nospec(
+				reg_num - KVM_REG_RISCV_VECTOR_REG(0), 32);
+		*reg_addr = cntx->vector.datap + reg_offset * vlenb;
 	} else {
 		return -ENOENT;
 	}
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index 5b1b3c88b4d13..e2be64bacf160 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -37,7 +37,9 @@
 
 #include "../kernel/head.h"
 
+#if defined(CONFIG_64BIT) && defined(CONFIG_MMU)
 DECLARE_BITMAP(new_valid_map_cpus, NR_CPUS);
+#endif
 
 struct kernel_mapping kernel_map __ro_after_init;
 EXPORT_SYMBOL(kernel_map);
@@ -1229,8 +1231,6 @@ static void __init create_linear_mapping_page_table(void)
 
 	/* Map all memory banks in the linear mapping */
 	for_each_mem_range(i, &start, &end) {
-		if (start >= end)
-			break;
 		if (start <= __pa(PAGE_OFFSET) &&
 		    __pa(PAGE_OFFSET) < end)
 			start = __pa(PAGE_OFFSET);
@@ -1320,7 +1320,7 @@ static inline void setup_vm_final(void)
  */
 static void __init arch_reserve_crashkernel(void)
 {
-	unsigned long long low_size = 0;
+	unsigned long long low_size = 0, cma_size = 0;
 	unsigned long long crash_base, crash_size;
 	bool high = false;
 	int ret;
@@ -1330,11 +1330,12 @@ static void __init arch_reserve_crashkernel(void)
 
 	ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(),
 				&crash_size, &crash_base,
-				&low_size, NULL, &high);
+				&low_size, &cma_size, &high);
 	if (ret)
 		return;
 
 	reserve_crashkernel_generic(crash_size, crash_base, low_size, high);
+	reserve_crashkernel_cma(cma_size);
 }
 
 void __init paging_init(void)
diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c
index c4a2a9e5586e7..1f3aa9611187f 100644
--- a/arch/riscv/mm/kasan_init.c
+++ b/arch/riscv/mm/kasan_init.c
@@ -512,9 +512,6 @@ void __init kasan_init(void)
 		void *start = (void *)__va(p_start);
 		void *end = (void *)__va(p_end);
 
-		if (start >= end)
-			break;
-
 		kasan_populate(kasan_mem_to_shadow(start), kasan_mem_to_shadow(end));
 	}
 
diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c
index 8404530ec00f9..962db300a1665 100644
--- a/arch/riscv/mm/tlbflush.c
+++ b/arch/riscv/mm/tlbflush.c
@@ -11,24 +11,6 @@
 
 #define has_svinval()	riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)
 
-static inline void local_sfence_inval_ir(void)
-{
-	asm volatile(SFENCE_INVAL_IR() ::: "memory");
-}
-
-static inline void local_sfence_w_inval(void)
-{
-	asm volatile(SFENCE_W_INVAL() ::: "memory");
-}
-
-static inline void local_sinval_vma(unsigned long vma, unsigned long asid)
-{
-	if (asid != FLUSH_TLB_NO_ASID)
-		asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory");
-	else
-		asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory");
-}
-
 /*
  * Flush entire TLB if number of entries to be flushed is greater
  * than the threshold below.
@@ -110,11 +92,6 @@ static void __ipi_flush_tlb_range_asid(void *info)
 	local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
 }
 
-static inline unsigned long get_mm_asid(struct mm_struct *mm)
-{
-	return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID;
-}
-
 static void __flush_tlb_range(struct mm_struct *mm,
 			      const struct cpumask *cmask,
 			      unsigned long start, unsigned long size,
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 84404e6778d50..df8bae04f6845 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -20,6 +20,9 @@ config ARCH_HAS_ILOG2_U64
 config ARCH_PROC_KCORE_TEXT
 	def_bool y
 
+config ARCH_HAS_CPU_RELAX
+	def_bool y
+
 config GENERIC_HWEIGHT
 	def_bool !HAVE_MARCH_Z196_FEATURES
 
@@ -82,7 +85,7 @@ config S390
 	select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE
 	select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM
 	select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2
-	select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
+	select ARCH_HAS_PMD_SOFTLEAVES if TRANSPARENT_HUGEPAGE
 	select ARCH_HAS_CC_CAN_LINK
 	select ARCH_HAS_CPU_FINALIZE_INIT
 	select ARCH_HAS_CURRENT_STACK_POINTER
@@ -148,6 +151,7 @@ config S390
 	select ARCH_SUPPORTS_HUGETLBFS
 	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && CC_IS_CLANG
 	select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS
+	select ARCH_HAS_PTE_PROTNONE
 	select ARCH_SUPPORTS_NUMA_BALANCING
 	select ARCH_SUPPORTS_PAGE_TABLE_CHECK
 	select ARCH_SUPPORTS_PER_VMA_LOCK
@@ -215,7 +219,6 @@ config S390
 	select HAVE_FUNCTION_GRAPH_TRACER
 	select HAVE_FUNCTION_TRACER
 	select HAVE_GCC_PLUGINS
-	select HAVE_GENERIC_VDSO
 	select HAVE_IOREMAP_PROT if PCI
 	select HAVE_KERNEL_BZIP2
 	select HAVE_KERNEL_GZIP
@@ -705,6 +708,8 @@ config KERNEL_IMAGE_BASE
 
 endmenu
 
+source "drivers/cpuidle/Kconfig"
+
 menu "Memory setup"
 
 config ARCH_SPARSEMEM_ENABLE
diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig
index 54637be87fb73..007668f950289 100644
--- a/arch/s390/configs/debug_defconfig
+++ b/arch/s390/configs/debug_defconfig
@@ -29,6 +29,8 @@ CONFIG_CGROUP_PIDS=y
 CONFIG_CGROUP_RDMA=y
 CONFIG_CGROUP_FREEZER=y
 CONFIG_CGROUP_HUGETLB=y
+CONFIG_CPU_IDLE=y
+CONFIG_CPU_IDLE_GOV_TEO=y
 CONFIG_CPUSETS=y
 CONFIG_CGROUP_DEVICE=y
 CONFIG_CGROUP_CPUACCT=y
diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig
index 5f5114a253cf7..6f5722634b4d3 100644
--- a/arch/s390/configs/defconfig
+++ b/arch/s390/configs/defconfig
@@ -27,6 +27,8 @@ CONFIG_CGROUP_PIDS=y
 CONFIG_CGROUP_RDMA=y
 CONFIG_CGROUP_FREEZER=y
 CONFIG_CGROUP_HUGETLB=y
+CONFIG_CPU_IDLE=y
+CONFIG_CPU_IDLE_GOV_TEO=y
 CONFIG_CPUSETS=y
 CONFIG_CGROUP_DEVICE=y
 CONFIG_CGROUP_CPUACCT=y
diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h
index e33a5b587ee4e..aea754b67c89a 100644
--- a/arch/s390/include/asm/hugetlb.h
+++ b/arch/s390/include/asm/hugetlb.h
@@ -42,9 +42,9 @@ static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
 				  pte_t *ptep, unsigned long sz)
 {
 	if ((pte_val(ptep_get(ptep)) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
-		set_pte(ptep, __pte(_REGION3_ENTRY_EMPTY));
+		set_pud((pud_t *)ptep, __pud(_REGION3_ENTRY_EMPTY));
 	else
-		set_pte(ptep, __pte(_SEGMENT_ENTRY_EMPTY));
+		set_pmd((pmd_t *)ptep, __pmd(_SEGMENT_ENTRY_EMPTY));
 }
 
 #define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH
@@ -77,20 +77,20 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
 	__set_huge_pte_at(mm, addr, ptep, pte_wrprotect(pte));
 }
 
-#define __HAVE_ARCH_HUGE_PTE_MKUFFD_WP
-static inline pte_t huge_pte_mkuffd_wp(pte_t pte)
+#define __HAVE_ARCH_HUGE_PTE_MKUFFD
+static inline pte_t huge_pte_mkuffd(pte_t pte)
 {
 	return pte;
 }
 
-#define __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD_WP
-static inline pte_t huge_pte_clear_uffd_wp(pte_t pte)
+#define __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD
+static inline pte_t huge_pte_clear_uffd(pte_t pte)
 {
 	return pte;
 }
 
-#define __HAVE_ARCH_HUGE_PTE_UFFD_WP
-static inline int huge_pte_uffd_wp(pte_t pte)
+#define __HAVE_ARCH_HUGE_PTE_UFFD
+static inline int huge_pte_uffd(pte_t pte)
 {
 	return 0;
 }
diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 859ce7c7d4544..e882663a58e77 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -842,7 +842,7 @@ static inline int pte_same(pte_t a, pte_t b)
 	return pte_val(a) == pte_val(b);
 }
 
-#ifdef CONFIG_NUMA_BALANCING
+#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE
 static inline int pte_protnone(pte_t pte)
 {
 	return pte_present(pte) && !(pte_val(pte) & _PAGE_READ);
@@ -853,7 +853,7 @@ static inline int pmd_protnone(pmd_t pmd)
 	/* pmd_leaf(pmd) implies pmd_present(pmd) */
 	return pmd_leaf(pmd) && !(pmd_val(pmd) & _SEGMENT_ENTRY_READ);
 }
-#endif
+#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */
 
 static inline bool pte_swp_exclusive(pte_t pte)
 {
@@ -903,7 +903,7 @@ static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
 	return clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_SOFT_DIRTY));
 }
 
-#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
 #define pmd_swp_soft_dirty(pmd)		pmd_soft_dirty(pmd)
 #define pmd_swp_mksoft_dirty(pmd)	pmd_mksoft_dirty(pmd)
 #define pmd_swp_clear_soft_dirty(pmd)	pmd_clear_soft_dirty(pmd)
diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index ecd3341686eb6..be8369115f6df 100644
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -14,12 +14,10 @@
 
 #include <linux/bits.h>
 
-#define CIF_NOHZ_DELAY		2	/* delay HZ disable for a tick */
 #define CIF_ENABLED_WAIT	5	/* in enabled wait state */
 #define CIF_MCCK_GUEST		6	/* machine check happening in guest */
 #define CIF_DEDICATED_CPU	7	/* this CPU is dedicated */
 
-#define _CIF_NOHZ_DELAY		BIT(CIF_NOHZ_DELAY)
 #define _CIF_ENABLED_WAIT	BIT(CIF_ENABLED_WAIT)
 #define _CIF_MCCK_GUEST		BIT(CIF_MCCK_GUEST)
 #define _CIF_DEDICATED_CPU	BIT(CIF_DEDICATED_CPU)
@@ -97,8 +95,6 @@ static __always_inline bool test_cpu_flag_of(int flag, int cpu)
 	return test_bit(flag, &per_cpu(pcpu_devices, cpu).flags);
 }
 
-#define arch_needs_cpu() test_cpu_flag(CIF_NOHZ_DELAY)
-
 static inline void get_cpu_id(struct cpuid *ptr)
 {
 	asm volatile("stidp %0" : "=Q" (*ptr));
diff --git a/arch/s390/include/asm/syscall.h b/arch/s390/include/asm/syscall.h
index 4271e4169f45b..5f310caad1fce 100644
--- a/arch/s390/include/asm/syscall.h
+++ b/arch/s390/include/asm/syscall.h
@@ -89,11 +89,6 @@ static inline int syscall_get_arch(struct task_struct *task)
 	return AUDIT_ARCH_S390X;
 }
 
-static inline bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs)
-{
-	return false;
-}
-
 #define SYSCALL_FMT_0
 #define SYSCALL_FMT_1 , "0" (r2)
 #define SYSCALL_FMT_2 , "d" (r3) SYSCALL_FMT_1
diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h
index 6a548a8194006..2554cef3a6a54 100644
--- a/arch/s390/include/asm/thread_info.h
+++ b/arch/s390/include/asm/thread_info.h
@@ -61,6 +61,7 @@ void arch_setup_new_exec(void);
  */
 #define HAVE_TIF_NEED_RESCHED_LAZY
 #define HAVE_TIF_RESTORE_SIGMASK
+#define HAVE_TIF_POLLING_NRFLAG
 
 #include <asm-generic/thread_info_tif.h>
 
diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c
index 08f3520c67853..dd12efb582019 100644
--- a/arch/s390/kernel/idle.c
+++ b/arch/s390/kernel/idle.c
@@ -136,7 +136,6 @@ void noinstr arch_cpu_idle(void)
 	/* Wait for external, I/O or machine check interrupt. */
 	psw_mask = PSW_KERNEL_BITS | PSW_MASK_WAIT |
 		   PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK;
-	clear_cpu_flag(CIF_NOHZ_DELAY);
 	set_cpu_flag(CIF_ENABLED_WAIT);
 	if (smp_cpu_mtid)
 		stcctm(MT_DIAG, smp_cpu_mtid, (u64 *)&idle->mt_cycles_enter);
diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c
index 04d528639b946..c923496aa7b48 100644
--- a/arch/s390/kernel/irq.c
+++ b/arch/s390/kernel/irq.c
@@ -166,7 +166,6 @@ void noinstr do_io_irq(struct pt_regs *regs)
 	if (from_idle)
 		account_idle_time_irq();
 
-	set_cpu_flag(CIF_NOHZ_DELAY);
 	do {
 		regs->tpi_info = get_lowcore()->tpi_info;
 		if (get_lowcore()->tpi_info.adapter_IO)
@@ -369,9 +368,6 @@ static irqreturn_t do_ext_interrupt(int irq, void *dummy)
 	int index;
 
 	ext_code.int_code = regs->int_code;
-	if (ext_code.code != EXT_IRQ_CLK_COMP)
-		set_cpu_flag(CIF_NOHZ_DELAY);
-
 	index = ext_hash(ext_code.code);
 	rcu_read_lock();
 	hlist_for_each_entry_rcu(p, &ext_int_hash[index], entry) {
diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c
index 7aa655664eccb..2076ac22e2c49 100644
--- a/arch/s390/kernel/perf_cpum_cf.c
+++ b/arch/s390/kernel/perf_cpum_cf.c
@@ -15,6 +15,7 @@
 #include <linux/init.h>
 #include <linux/miscdevice.h>
 #include <linux/perf_event.h>
+#include <linux/nospec.h>
 
 #include <asm/cpu_mf.h>
 #include <asm/hwctrset.h>
@@ -768,6 +769,7 @@ static int __hw_perf_event_init(struct perf_event *event, unsigned int type)
 			if (!is_userspace_event(ev)) {
 				if (ev >= ARRAY_SIZE(cpumf_generic_events_user))
 					return -EOPNOTSUPP;
+				ev = array_index_nospec(ev, ARRAY_SIZE(cpumf_generic_events_user));
 				ev = cpumf_generic_events_user[ev];
 			}
 		} else if (!attr->exclude_kernel && attr->exclude_user) {
@@ -778,6 +780,7 @@ static int __hw_perf_event_init(struct perf_event *event, unsigned int type)
 			if (!is_userspace_event(ev)) {
 				if (ev >= ARRAY_SIZE(cpumf_generic_events_basic))
 					return -EOPNOTSUPP;
+				ev = array_index_nospec(ev, ARRAY_SIZE(cpumf_generic_events_basic));
 				ev = cpumf_generic_events_basic[ev];
 			}
 		}
diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
index 0ba7f89b81611..4cc2c119318db 100644
--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -909,7 +909,6 @@ int __cpu_disable(void)
 	cregs[6].val  &= ~0xff000000UL;	/* disable all I/O interrupts */
 	cregs[14].val &= ~0x1f000000UL;	/* disable most machine checks */
 	__local_ctl_load(0, 15, cregs);
-	clear_cpu_flag(CIF_NOHZ_DELAY);
 	return 0;
 }
 
diff --git a/arch/s390/kernel/syscall.c b/arch/s390/kernel/syscall.c
index 75d5a3cab14e6..4ac80e2478d7a 100644
--- a/arch/s390/kernel/syscall.c
+++ b/arch/s390/kernel/syscall.c
@@ -96,9 +96,10 @@ SYSCALL_DEFINE0(ni_syscall)
 void noinstr __do_syscall(struct pt_regs *regs, int per_trap)
 {
 	unsigned long nr;
+	bool permit;
+
+	enter_from_user_mode_randomize_stack(regs);
 
-	enter_from_user_mode(regs);
-	add_random_kstack_offset();
 	regs->psw = get_lowcore()->svc_old_psw;
 	regs->int_code = get_lowcore()->svc_int_code;
 	update_timer_sys();
@@ -121,7 +122,9 @@ void noinstr __do_syscall(struct pt_regs *regs, int per_trap)
 		regs->psw.addr = current->restart_block.arch_data;
 		current->restart_block.arch_data = 1;
 	}
-	nr = syscall_enter_from_user_mode_work(regs, nr);
+
+	permit = syscall_enter_from_user_mode_work(regs, &nr);
+
 	/*
 	 * In the s390 ptrace ABI, both the syscall number and the return value
 	 * use gpr2. However, userspace puts the syscall number either in the
@@ -129,7 +132,7 @@ void noinstr __do_syscall(struct pt_regs *regs, int per_trap)
 	 * work, the ptrace code sets PIF_SYSCALL_RET_SET, which is checked here
 	 * and if set, the syscall will be skipped.
 	 */
-	if (unlikely(test_and_clear_pt_regs_flag(regs, PIF_SYSCALL_RET_SET)))
+	if (unlikely(test_and_clear_pt_regs_flag(regs, PIF_SYSCALL_RET_SET) || !permit))
 		goto out;
 	regs->gprs[2] = -ENOSYS;
 	if (likely(nr < NR_syscalls)) {
diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c
index 564403496a7ce..fcd8c1122ef57 100644
--- a/arch/s390/kernel/traps.c
+++ b/arch/s390/kernel/traps.c
@@ -9,7 +9,9 @@
  *    Copyright (C) 1991, 1992 Linus Torvalds
  */
 
+#include <linux/capability.h>
 #include <linux/cpufeature.h>
+#include <linux/debugfs.h>
 #include <linux/kprobes.h>
 #include <linux/kdebug.h>
 #include <linux/randomize_kstack.h>
@@ -33,6 +35,12 @@
 #include <asm/fault.h>
 #include "entry.h"
 
+struct pgm_stat {
+	unsigned int count[128];
+};
+
+static DEFINE_PER_CPU_SHARED_ALIGNED(struct pgm_stat, pgm_stat);
+
 static inline void __user *get_trap_ip(struct pt_regs *regs)
 {
 	unsigned long address;
@@ -332,6 +340,7 @@ void noinstr __do_pgm_check(struct pt_regs *regs)
 	struct lowcore *lc = get_lowcore();
 	bool percpu_needs_fixup;
 	irqentry_state_t state;
+	struct pgm_stat *stat;
 	unsigned int trapnr;
 	union teid teid;
 
@@ -339,6 +348,10 @@ void noinstr __do_pgm_check(struct pt_regs *regs)
 	regs->int_code = lc->pgm_int_code;
 	regs->int_parm_long = teid.val;
 	regs->monitor_code = lc->monitor_code;
+
+	trapnr = regs->int_code & PGM_INT_CODE_MASK;
+	stat = this_cpu_ptr(&pgm_stat);
+	stat->count[trapnr]++;
 	/*
 	 * In case of a guest fault, short-circuit the fault handler and return.
 	 * This way the sie64a() function will return 0; fault address and
@@ -383,7 +396,6 @@ void noinstr __do_pgm_check(struct pt_regs *regs)
 	if (!irqs_disabled_flags(regs->psw.mask))
 		trace_hardirqs_on();
 	__arch_local_irq_ssm(regs->psw.mask & ~PSW_MASK_PER);
-	trapnr = regs->int_code & PGM_INT_CODE_MASK;
 	if (trapnr)
 		pgm_check_table[trapnr](regs);
 out:
@@ -393,6 +405,33 @@ void noinstr __do_pgm_check(struct pt_regs *regs)
 	percpu_exit(regs, percpu_needs_fixup);
 }
 
+static int pgm_check_stat_show(struct seq_file *p, void *v)
+{
+	int i, cpu;
+
+	cpus_read_lock();
+	seq_puts(p, "          ");
+	for_each_online_cpu(cpu)
+		seq_printf(p, "CPU%-8d", cpu);
+	seq_putc(p, '\n');
+	for (i = 0; i < 128; i++) {
+		seq_printf(p, "%02x: ", i);
+		for_each_online_cpu(cpu)
+			seq_printf(p, "%10u ", per_cpu(pgm_stat, cpu).count[i]);
+		seq_putc(p, '\n');
+	}
+	cpus_read_unlock();
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(pgm_check_stat);
+
+static int __init debugfs_pgm_check_init(void)
+{
+	debugfs_create_file("exceptions", 0400, arch_debugfs_dir, NULL, &pgm_check_stat_fops);
+	return 0;
+}
+late_initcall(debugfs_pgm_check_init);
+
 /*
  * The program check table contains exactly 128 (0x00-0x7f) entries. Each
  * line defines the function to be called corresponding to the program check
diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c
index 5f1960ec982d0..ed4259d176295 100644
--- a/arch/s390/kvm/dat.c
+++ b/arch/s390/kvm/dat.c
@@ -570,6 +570,8 @@ static long dat_crste_walk_range(gfn_t start, gfn_t end, struct crst_table *tabl
 			else if (walk->ops->pte_entry)
 				rc = dat_pte_walk_range(max(start, cur), min(end, next),
 							dereference_pmd(crste.pmd), walk);
+			if (rc)
+				break;
 		}
 	}
 	return rc;
diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c
index 298fbaecec28d..8abb4f55b306b 100644
--- a/arch/s390/kvm/gmap.c
+++ b/arch/s390/kvm/gmap.c
@@ -1374,8 +1374,13 @@ struct gmap *gmap_create_shadow(struct kvm_s390_mmu_cache *mc, struct gmap *pare
 			/* Only allow one real-space gmap shadow. */
 			list_for_each_entry(sg, &parent->children, list) {
 				if (sg->guest_asce.r) {
-					scoped_guard(write_lock, &parent->kvm->mmu_lock)
+					if (write_trylock(&parent->kvm->mmu_lock)) {
 						gmap_unshadow(sg);
+						write_unlock(&parent->kvm->mmu_lock);
+					} else {
+						gmap_put(new);
+						return ERR_PTR(-EAGAIN);
+					}
 					break;
 				}
 			}
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 23c817595e28d..150b5dd2170e2 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -1280,8 +1280,10 @@ static int kvm_s390_vm_stop_migration(struct kvm *kvm)
 	 * PGSTEs might have cmma_d set.
 	 */
 	WRITE_ONCE(kvm->arch.migration_mode, 0);
-	if (kvm->arch.use_cmma)
-		kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
+	if (!kvm->arch.use_cmma)
+		return 0;
+
+	kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
 	/* Clear cmma_d on all existing PGSTEs and set cmma_dirty_pages to 0. */
 	gmap_set_cmma_all_clean(kvm->arch.gmap);
 	atomic64_set(&kvm->arch.cmma_dirty_pages, 0);
diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c
index 5b075c38998e3..720bb58cabe2e 100644
--- a/arch/s390/kvm/pci.c
+++ b/arch/s390/kvm/pci.c
@@ -300,9 +300,14 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib,
 
 	gaite->gisc = fib->fmt0.isc;
 	gaite->count++;
-	gaite->aisbo = fib->fmt0.aisbo;
-	gaite->aisb = virt_to_phys(page_address(aisb_page) + (fib->fmt0.aisb &
-							      ~PAGE_MASK));
+	if (fib->fmt0.sum == 1) {
+		gaite->aisbo = fib->fmt0.aisbo;
+		gaite->aisb = virt_to_phys(page_address(aisb_page) +
+					   (fib->fmt0.aisb & ~PAGE_MASK));
+	} else {
+		gaite->aisbo = 0;
+		gaite->aisb = 0;
+	}
 	aift->kzdev[zdev->aisb] = zdev->kzdev;
 	spin_unlock_irq(&aift->gait_lock);
 
@@ -328,6 +333,7 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib,
 unpin1:
 	unpin_user_page(aibv_page);
 out:
+	kvm_s390_gisc_unregister(kvm, fib->fmt0.isc);
 	return rc;
 }
 
diff --git a/arch/s390/lib/csum-partial.c b/arch/s390/lib/csum-partial.c
index 458abd9bac702..9d74ceff136c5 100644
--- a/arch/s390/lib/csum-partial.c
+++ b/arch/s390/lib/csum-partial.c
@@ -23,7 +23,7 @@ static __always_inline __wsum csum_copy(void *dst, const void *src, int len, __w
 	if (!cpu_has_vx()) {
 		if (copy)
 			memcpy(dst, src, len);
-		return cksm(dst, len, sum);
+		return cksm(src, len, sum);
 	}
 	kernel_fpu_begin(&vxstate, KERNEL_VXR_V16V23);
 	fpu_vlvgf(16, (__force u32)sum, 1);
diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c
index db35d8fe86090..f84aa9265430f 100644
--- a/arch/s390/mm/hugetlbpage.c
+++ b/arch/s390/mm/hugetlbpage.c
@@ -147,10 +147,12 @@ void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
 		if (likely(pte_present(pte)))
 			rste |= _REGION3_ENTRY_LARGE;
 		rste |= _REGION_ENTRY_TYPE_R3;
-	} else if (likely(pte_present(pte)))
-		rste |= _SEGMENT_ENTRY_LARGE;
-
-	set_pte(ptep, __pte(rste));
+		set_pud((pud_t *)ptep, __pud(rste));
+	} else {
+		if (likely(pte_present(pte)))
+			rste |= _SEGMENT_ENTRY_LARGE;
+		set_pmd((pmd_t *)ptep, __pmd(rste));
+	}
 }
 
 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c
index 94ee06ae89134..d2879ce860a13 100644
--- a/arch/s390/mm/vmem.c
+++ b/arch/s390/mm/vmem.c
@@ -39,7 +39,6 @@ static void __ref *vmem_alloc_pages(unsigned int order)
 
 static void vmem_free_pages(unsigned long addr, int order, struct vmem_altmap *altmap)
 {
-	unsigned int nr_pages = 1 << order;
 	struct page *page;
 
 	if (altmap) {
@@ -49,8 +48,7 @@ static void vmem_free_pages(unsigned long addr, int order, struct vmem_altmap *a
 	page = virt_to_page((void *)addr);
 	if (PageReserved(page)) {
 		/* allocated from memblock */
-		while (nr_pages--)
-			free_reserved_page(page++);
+		free_reserved_pages(page, order);
 	} else {
 		free_pages(addr, order);
 	}
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index 31749c0362ca2..9ddd89f71f282 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -1783,8 +1783,8 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
 		    insn->imm == BPF_FUNC_get_smp_processor_id) {
 			const u32 *cpu_nr = &get_lowcore()->cpu_nr;
 
-			/* ly %b0, cpu_nr */
-			EMIT6_DISP_LH(0xe3000000, 0x0058, BPF_REG_0, REG_0, REG_0,
+			/* llgf %b0, cpu_nr */
+			EMIT6_DISP_LH(0xe3000000, 0x0016, BPF_REG_0, REG_0, REG_0,
 				      (unsigned long)cpu_nr);
 			break;
 		}
diff --git a/arch/sh/boards/mach-rsk/devices-rsk7203.c b/arch/sh/boards/mach-rsk/devices-rsk7203.c
index e6b05d4588b7b..e8a8fc1d2ca9d 100644
--- a/arch/sh/boards/mach-rsk/devices-rsk7203.c
+++ b/arch/sh/boards/mach-rsk/devices-rsk7203.c
@@ -4,17 +4,18 @@
  *
  * Copyright (C) 2008 - 2010  Paul Mundt
  */
+#include <linux/err.h>
 #include <linux/init.h>
 #include <linux/types.h>
 #include <linux/platform_device.h>
 #include <linux/interrupt.h>
 #include <linux/smsc911x.h>
 #include <linux/input.h>
-#include <linux/gpio.h>
-#include <linux/gpio_keys.h>
-#include <linux/leds.h>
+#include <linux/io.h>
+#include <linux/gpio/machine.h>
+#include <linux/gpio/property.h>
 #include <asm/machvec.h>
-#include <asm/io.h>
+#include <cpu/pfc.h>
 #include <cpu/sh7203.h>
 
 static struct smsc911x_platform_config smsc911x_config = {
@@ -37,101 +38,236 @@ static struct resource smsc911x_resources[] = {
 	},
 };
 
-static struct platform_device smsc911x_device = {
-	.name		= "smsc911x",
-	.id		= -1,
-	.num_resources	= ARRAY_SIZE(smsc911x_resources),
-	.resource	= smsc911x_resources,
-	.dev		= {
-		.platform_data = &smsc911x_config,
+static const struct software_node rsk7203_gpio_leds_node = {
+	.name = "rsk7203-gpio-leds",
+};
+
+static const struct software_node rsk7203_green_led_node = {
+	.name = "green",
+	.parent = &rsk7203_gpio_leds_node,
+	.properties = (const struct property_entry[]) {
+		PROPERTY_ENTRY_STRING("label", "green"),
+		PROPERTY_ENTRY_GPIO("gpios", &pfc_gpiochip_node,
+				    GPIO_PE10, GPIO_ACTIVE_LOW),
+		{ }
 	},
 };
 
-static struct gpio_led rsk7203_gpio_leds[] = {
-	{
-		.name			= "green",
-		.gpio			= GPIO_PE10,
-		.active_low		= 1,
-	}, {
-		.name			= "orange",
-		.default_trigger	= "nand-disk",
-		.gpio			= GPIO_PE12,
-		.active_low		= 1,
-	}, {
-		.name			= "red:timer",
-		.default_trigger	= "timer",
-		.gpio			= GPIO_PC14,
-		.active_low		= 1,
-	}, {
-		.name			= "red:heartbeat",
-		.default_trigger	= "heartbeat",
-		.gpio			= GPIO_PE11,
-		.active_low		= 1,
+static const struct software_node rsk7203_orange_led_node = {
+	.name = "orange",
+	.parent = &rsk7203_gpio_leds_node,
+	.properties = (const struct property_entry[]) {
+		PROPERTY_ENTRY_STRING("label", "orange"),
+		PROPERTY_ENTRY_STRING("linux,default-trigger", "nand-disk"),
+		PROPERTY_ENTRY_GPIO("gpios", &pfc_gpiochip_node,
+				    GPIO_PE12, GPIO_ACTIVE_LOW),
+		{ }
 	},
 };
 
-static struct gpio_led_platform_data rsk7203_gpio_leds_info = {
-	.leds		= rsk7203_gpio_leds,
-	.num_leds	= ARRAY_SIZE(rsk7203_gpio_leds),
+static const struct software_node rsk7203_red1_led_node = {
+	.name = "red:timer",
+	.parent = &rsk7203_gpio_leds_node,
+	.properties = (const struct property_entry[]) {
+		PROPERTY_ENTRY_STRING("label", "red:timer"),
+		PROPERTY_ENTRY_STRING("linux,default-trigger", "timer"),
+		PROPERTY_ENTRY_GPIO("gpios", &pfc_gpiochip_node,
+				    GPIO_PC14, GPIO_ACTIVE_LOW),
+		{ }
+	},
 };
 
-static struct platform_device led_device = {
-	.name		= "leds-gpio",
-	.id		= -1,
-	.dev		= {
-		.platform_data	= &rsk7203_gpio_leds_info,
+static const struct software_node rsk7203_red2_led_node = {
+	.name = "red:heartbeat",
+	.parent = &rsk7203_gpio_leds_node,
+	.properties = (const struct property_entry[]) {
+		PROPERTY_ENTRY_STRING("label", "red:heartbeat"),
+		PROPERTY_ENTRY_STRING("linux,default-trigger", "heartbeat"),
+		PROPERTY_ENTRY_GPIO("gpios", &pfc_gpiochip_node,
+				    GPIO_PE11, GPIO_ACTIVE_LOW),
+		{ }
 	},
 };
 
-static struct gpio_keys_button rsk7203_gpio_keys_table[] = {
-	{
-		.code		= BTN_0,
-		.gpio		= GPIO_PB0,
-		.active_low	= 1,
-		.desc		= "SW1",
-	}, {
-		.code		= BTN_1,
-		.gpio		= GPIO_PB1,
-		.active_low	= 1,
-		.desc		= "SW2",
-	}, {
-		.code		= BTN_2,
-		.gpio		= GPIO_PB2,
-		.active_low	= 1,
-		.desc		= "SW3",
+static const struct software_node rsk7203_gpio_keys_node = {
+	.name = "rsk7203-gpio-keys",
+	.properties = (const struct property_entry[]) {
+		PROPERTY_ENTRY_U32("poll-interval", 50),
+		{ }
+	},
+};
+
+static const struct software_node rsk7203_sw1_key_node = {
+	.parent = &rsk7203_gpio_keys_node,
+	.properties = (const struct property_entry[]) {
+		PROPERTY_ENTRY_U32("linux,code", BTN_0),
+		PROPERTY_ENTRY_GPIO("gpios", &pfc_gpiochip_node,
+				    GPIO_PB0, GPIO_ACTIVE_LOW),
+		PROPERTY_ENTRY_STRING("label", "SW1"),
+		{ }
+	},
+};
+
+static const struct software_node rsk7203_sw2_key_node = {
+	.parent = &rsk7203_gpio_keys_node,
+	.properties = (const struct property_entry[]) {
+		PROPERTY_ENTRY_U32("linux,code", BTN_1),
+		PROPERTY_ENTRY_GPIO("gpios", &pfc_gpiochip_node,
+				    GPIO_PB1, GPIO_ACTIVE_LOW),
+		PROPERTY_ENTRY_STRING("label", "SW2"),
+		{ }
 	},
 };
 
-static struct gpio_keys_platform_data rsk7203_gpio_keys_info = {
-	.buttons	= rsk7203_gpio_keys_table,
-	.nbuttons	= ARRAY_SIZE(rsk7203_gpio_keys_table),
-	.poll_interval	= 50, /* default to 50ms */
+static const struct software_node rsk7203_sw3_key_node = {
+	.parent = &rsk7203_gpio_keys_node,
+	.properties = (const struct property_entry[]) {
+		PROPERTY_ENTRY_U32("linux,code", BTN_2),
+		PROPERTY_ENTRY_GPIO("gpios", &pfc_gpiochip_node,
+				    GPIO_PB2, GPIO_ACTIVE_LOW),
+		PROPERTY_ENTRY_STRING("label", "SW3"),
+		{ }
+	},
+};
+
+/* The base of the function GPIOs in the flat enum */
+#define SH7203_FN_BASE GPIO_FN_PINT7_PB
+
+static const struct software_node rsk7203_pfc_functions_node = {
+	.name = "functions",
+	.parent = &pfc_gpiochip_node,
+};
+
+static const struct software_node rsk7203_txd0_hog_node = {
+	.name = "txd0-hog",
+	.parent = &rsk7203_pfc_functions_node,
+	.properties = (const struct property_entry[]) {
+		PROPERTY_ENTRY_BOOL("gpio-hog"),
+		PROPERTY_ENTRY_U32_ARRAY("gpios", ((u32[]){
+			GPIO_FN_TXD0 - SH7203_FN_BASE, GPIO_ACTIVE_HIGH
+		})),
+		PROPERTY_ENTRY_BOOL("input"),
+		PROPERTY_ENTRY_STRING("line-name", "TXD0"),
+		{ }
+	},
 };
 
-static struct platform_device keys_device = {
-	.name		= "gpio-keys-polled",
-	.dev		= {
-		.platform_data	= &rsk7203_gpio_keys_info,
+static const struct software_node rsk7203_rxd0_hog_node = {
+	.name = "rxd0-hog",
+	.parent = &rsk7203_pfc_functions_node,
+	.properties = (const struct property_entry[]) {
+		PROPERTY_ENTRY_BOOL("gpio-hog"),
+		PROPERTY_ENTRY_U32_ARRAY("gpios", ((u32[]){
+			GPIO_FN_RXD0 - SH7203_FN_BASE, GPIO_ACTIVE_HIGH
+		})),
+		PROPERTY_ENTRY_BOOL("input"),
+		PROPERTY_ENTRY_STRING("line-name", "RXD0"),
+		{ }
 	},
 };
 
-static struct platform_device *rsk7203_devices[] __initdata = {
-	&smsc911x_device,
-	&led_device,
-	&keys_device,
+static const struct software_node rsk7203_irq0_hog_node = {
+	.name = "irq0-hog",
+	.parent = &rsk7203_pfc_functions_node,
+	.properties = (const struct property_entry[]) {
+		PROPERTY_ENTRY_BOOL("gpio-hog"),
+		PROPERTY_ENTRY_U32_ARRAY("gpios", ((u32[]){
+			GPIO_FN_IRQ0_PB - SH7203_FN_BASE, GPIO_ACTIVE_HIGH
+		})),
+		PROPERTY_ENTRY_BOOL("input"),
+		PROPERTY_ENTRY_STRING("line-name", "IRQ0_PB"),
+		{ }
+	},
 };
 
+static const struct software_node * const rsk7203_swnodes[] __initconst = {
+	&rsk7203_gpio_leds_node,
+	&rsk7203_green_led_node,
+	&rsk7203_orange_led_node,
+	&rsk7203_red1_led_node,
+	&rsk7203_red2_led_node,
+	&rsk7203_gpio_keys_node,
+	&rsk7203_sw1_key_node,
+	&rsk7203_sw2_key_node,
+	&rsk7203_sw3_key_node,
+	&rsk7203_pfc_functions_node,
+	&rsk7203_txd0_hog_node,
+	&rsk7203_rxd0_hog_node,
+	&rsk7203_irq0_hog_node,
+	NULL
+};
+
+static const struct platform_device_info rsk7203_devices[] __initconst = {
+	{
+		.name		= "smsc911x",
+		.id		= PLATFORM_DEVID_NONE,
+		.res		= smsc911x_resources,
+		.num_res	= ARRAY_SIZE(smsc911x_resources),
+		.data		= &smsc911x_config,
+		.size_data	= sizeof(smsc911x_config),
+	},
+	{
+		.name		= "leds-gpio",
+		.id		= PLATFORM_DEVID_NONE,
+		.swnode		= &rsk7203_gpio_leds_node,
+	},
+	{
+		.name		= "gpio-keys-polled",
+		.id		= PLATFORM_DEVID_NONE,
+		.swnode		= &rsk7203_gpio_keys_node,
+	},
+};
+
+/*
+ * The pfc-sh7203 device is registered at arch_initcall level, and the
+ * sh-pfc driver (registered at postcore_initcall level) probes as soon
+ * as the device is created.
+ *
+ * We need to register our software nodes at postcore_initcall level so
+ * they are already present in the system when the driver probes and
+ * tries to apply GPIO hogs.
+ */
+static int __init rsk7203_sw_nodes_setup(void)
+{
+	int error;
+
+	error = software_node_register(&pfc_gpiochip_node);
+	if (error && error != -EEXIST) {
+		pr_err("RSK7203: failed to register PFC software node: %d\n",
+		       error);
+		return error;
+	}
+
+	error = software_node_register_node_group(rsk7203_swnodes);
+	if (error) {
+		pr_err("RSK7203: failed to register board software nodes: %d\n",
+		       error);
+		return error;
+	}
+
+	return 0;
+}
+postcore_initcall(rsk7203_sw_nodes_setup);
+
 static int __init rsk7203_devices_setup(void)
 {
-	/* Select pins for SCIF0 */
-	gpio_request(GPIO_FN_TXD0, NULL);
-	gpio_request(GPIO_FN_RXD0, NULL);
+	struct platform_device *pd;
+	int error;
+	int i;
 
 	/* Setup LAN9118: CS1 in 16-bit Big Endian Mode, IRQ0 at Port B */
 	__raw_writel(0x36db0400, 0xfffc0008); /* CS1BCR */
-	gpio_request(GPIO_FN_IRQ0_PB, NULL);
 
-	return platform_add_devices(rsk7203_devices,
-				    ARRAY_SIZE(rsk7203_devices));
+	for (i = 0; i < ARRAY_SIZE(rsk7203_devices); i++) {
+		pd = platform_device_register_full(&rsk7203_devices[i]);
+		error = PTR_ERR_OR_ZERO(pd);
+		if (error) {
+			pr_err("failed to create platform device %s: %d\n",
+			       rsk7203_devices[i].name, error);
+			return error;
+		}
+	}
+
+	return 0;
 }
 device_initcall(rsk7203_devices_setup);
diff --git a/arch/sh/include/cpu-common/cpu/pfc.h b/arch/sh/include/cpu-common/cpu/pfc.h
index 879d2c9da537f..d57e38c05bdbb 100644
--- a/arch/sh/include/cpu-common/cpu/pfc.h
+++ b/arch/sh/include/cpu-common/cpu/pfc.h
@@ -11,6 +11,9 @@
 #include <linux/types.h>
 
 struct resource;
+struct software_node;
+
+extern const struct software_node pfc_gpiochip_node;
 
 int sh_pfc_register(const char *name,
 		    struct resource *resource, u32 num_resources);
diff --git a/arch/sh/kernel/cpu/pfc.c b/arch/sh/kernel/cpu/pfc.c
index 062056ede88de..5a8d804d607b8 100644
--- a/arch/sh/kernel/cpu/pfc.c
+++ b/arch/sh/kernel/cpu/pfc.c
@@ -5,21 +5,29 @@
  * Copyright (C) 2012  Renesas Solutions Corp.
  */
 
+#include <linux/err.h>
 #include <linux/init.h>
 #include <linux/platform_device.h>
+#include <linux/property.h>
 
 #include <cpu/pfc.h>
 
-static struct platform_device sh_pfc_device = {
-	.id		= -1,
+const struct software_node pfc_gpiochip_node = {
+	.name = "sh-pfc",
 };
 
 int __init sh_pfc_register(const char *name,
 			   struct resource *resource, u32 num_resources)
 {
-	sh_pfc_device.name = name;
-	sh_pfc_device.num_resources = num_resources;
-	sh_pfc_device.resource = resource;
+	struct platform_device_info pdev_info = {
+		.name		= name,
+		.id		= PLATFORM_DEVID_NONE,
+		.res		= resource,
+		.num_res	= num_resources,
+		.swnode		= &pfc_gpiochip_node,
+	};
+	struct platform_device *pdev;
 
-	return platform_device_register(&sh_pfc_device);
+	pdev = platform_device_register_full(&pdev_info);
+	return PTR_ERR_OR_ZERO(pdev);
 }
diff --git a/arch/sh/kernel/ptrace_32.c b/arch/sh/kernel/ptrace_32.c
index 06f765d71a295..8794081483fbd 100644
--- a/arch/sh/kernel/ptrace_32.c
+++ b/arch/sh/kernel/ptrace_32.c
@@ -455,12 +455,12 @@ long arch_ptrace(struct task_struct *child, long request,
 asmlinkage long do_syscall_trace_enter(struct pt_regs *regs)
 {
 	if (test_thread_flag(TIF_SYSCALL_TRACE) &&
-	    ptrace_report_syscall_entry(regs)) {
+	    !ptrace_report_syscall_permit_entry(regs)) {
 		regs->regs[0] = -ENOSYS;
 		return -1;
 	}
 
-	if (secure_computing() == -1)
+	if (!seccomp_permit_syscall())
 		return -1;
 
 	if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT)))
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index 0e9c906c4b5da..4dd79311133ff 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -113,7 +113,6 @@ config SPARC64
 	select ARCH_SUPPORTS_SCHED_SMT if SMP
 	select ARCH_SUPPORTS_SCHED_MC  if SMP
 	select ARCH_HAS_LAZY_MMU_MODE
-	select HAVE_GENERIC_VDSO
 	select GENERIC_GETTIMEOFDAY
 
 config ARCH_PROC_KCORE_TEXT
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 74ede706fb325..0837ebbc5dce6 100644
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -945,10 +945,6 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
 #define pte_clear(mm,addr,ptep)		\
 	set_pte_at((mm), (addr), (ptep), __pte(0UL))
 
-#define __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL
-#define pte_clear_not_present_full(mm,addr,ptep,fullmm)	\
-	__set_pte_at((mm), (addr), (ptep), __pte(0UL), (fullmm))
-
 #ifdef DCACHE_ALIASING_POSSIBLE
 #define __HAVE_ARCH_MOVE_PTE
 #define move_pte(pte, old_addr, new_addr)				\
diff --git a/arch/sparc/kernel/ptrace_32.c b/arch/sparc/kernel/ptrace_32.c
index c56333975fb13..5991b1731feed 100644
--- a/arch/sparc/kernel/ptrace_32.c
+++ b/arch/sparc/kernel/ptrace_32.c
@@ -441,7 +441,7 @@ asmlinkage int syscall_trace(struct pt_regs *regs, int syscall_exit_p)
 		if (syscall_exit_p)
 			ptrace_report_syscall_exit(regs, 0);
 		else
-			ret = ptrace_report_syscall_entry(regs);
+			ret = !ptrace_report_syscall_permit_entry(regs);
 	}
 
 	return ret;
diff --git a/arch/sparc/kernel/ptrace_64.c b/arch/sparc/kernel/ptrace_64.c
index 9fc67fa9336fe..825ddf55fece3 100644
--- a/arch/sparc/kernel/ptrace_64.c
+++ b/arch/sparc/kernel/ptrace_64.c
@@ -1093,7 +1093,7 @@ asmlinkage int syscall_trace_enter(struct pt_regs *regs)
 		user_exit();
 
 	if (test_thread_flag(TIF_SYSCALL_TRACE))
-		ret = ptrace_report_syscall_entry(regs);
+		ret = !ptrace_report_syscall_permit_entry(regs);
 
 	if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT)))
 		trace_sys_enter(regs, regs->u_regs[UREG_G1]);
diff --git a/arch/sparc/vdso/vclock_gettime.c b/arch/sparc/vdso/vclock_gettime.c
index 1d9859392e4cf..221bd4ed19f5c 100644
--- a/arch/sparc/vdso/vclock_gettime.c
+++ b/arch/sparc/vdso/vclock_gettime.c
@@ -21,6 +21,7 @@
 
 #include "../../../../lib/vdso/gettimeofday.c"
 
+#if defined(CONFIG_SPARC64) || defined(CONFIG_COMPAT_32BIT_TIME)
 int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
 {
 	return __cvdso_gettimeofday(tv, tz);
@@ -28,6 +29,7 @@ int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
 
 int gettimeofday(struct __kernel_old_timeval *, struct timezone *)
 	__weak __alias(__vdso_gettimeofday);
+#endif
 
 #if defined(CONFIG_SPARC64)
 int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
@@ -40,6 +42,7 @@ int clock_gettime(clockid_t, struct __kernel_timespec *)
 
 #else
 
+#if defined(CONFIG_COMPAT_32BIT_TIME)
 int __vdso_clock_gettime(clockid_t clock, struct old_timespec32 *ts)
 {
 	return __cvdso_clock_gettime32(clock, ts);
@@ -47,6 +50,7 @@ int __vdso_clock_gettime(clockid_t clock, struct old_timespec32 *ts)
 
 int clock_gettime(clockid_t, struct old_timespec32 *)
 	__weak __alias(__vdso_clock_gettime);
+#endif
 
 int __vdso_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts)
 {
diff --git a/arch/sparc/vdso/vdso32/vdso32.lds.S b/arch/sparc/vdso/vdso32/vdso32.lds.S
index a14e4f77e6f22..28052168b875e 100644
--- a/arch/sparc/vdso/vdso32/vdso32.lds.S
+++ b/arch/sparc/vdso/vdso32/vdso32.lds.S
@@ -15,12 +15,14 @@
 VERSION {
 	LINUX_2.6 {
 	global:
+#ifdef CONFIG_COMPAT_32BIT_TIME
 		clock_gettime;
 		__vdso_clock_gettime;
-		clock_gettime64;
-		__vdso_clock_gettime64;
 		gettimeofday;
 		__vdso_gettimeofday;
+#endif
+		clock_gettime64;
+		__vdso_clock_gettime64;
 	local: *;
 	};
 }
diff --git a/arch/um/include/asm/cpufeature.h b/arch/um/include/asm/cpufeature.h
index 4354f6984271c..f7770083c0a4e 100644
--- a/arch/um/include/asm/cpufeature.h
+++ b/arch/um/include/asm/cpufeature.h
@@ -49,7 +49,7 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
  * is not relevant.
  */
 #define cpu_feature_enabled(bit)	\
-	(__builtin_constant_p(bit) && DISABLED_MASK_BIT_SET(bit) ? 0 : static_cpu_has(bit))
+	(__builtin_constant_p(bit) && DISABLED_MASK_BIT_SET(bit) ? 0 : _static_cpu_has(bit))
 
 #define boot_cpu_has(bit)	cpu_has(&boot_cpu_data, bit)
 
@@ -64,15 +64,7 @@ extern void setup_clear_cpu_cap(unsigned int bit);
 
 #define setup_force_cpu_bug(bit) setup_force_cpu_cap(bit)
 
-/*
- * Static testing of CPU features. Used the same as boot_cpu_has(). It
- * statically patches the target code for additional performance. Use
- * static_cpu_has() only in fast paths, where every cycle counts. Which
- * means that the boot_cpu_has() variant is already fast enough for the
- * majority of cases and you should stick to using it as it is generally
- * only two instructions: a RIP-relative MOV and a TEST.
- */
-static __always_inline bool _static_cpu_has(u16 bit)
+static __always_inline bool __static_cpu_has(u16 bit)
 {
 	asm goto("1: jmp 6f\n"
 		 "2:\n"
@@ -116,7 +108,7 @@ static __always_inline bool _static_cpu_has(u16 bit)
 	return false;
 }
 
-#define static_cpu_has(bit)					\
+#define _static_cpu_has(bit)					\
 (								\
 	__builtin_constant_p(boot_cpu_has(bit)) ?		\
 		boot_cpu_has(bit) :				\
@@ -126,7 +118,7 @@ static __always_inline bool _static_cpu_has(u16 bit)
 #define cpu_has_bug(c, bit)		cpu_has(c, (bit))
 #define set_cpu_bug(c, bit)		set_cpu_cap(c, (bit))
 
-#define static_cpu_has_bug(bit)		static_cpu_has((bit))
+#define static_cpu_has_bug(bit)		_static_cpu_has((bit))
 #define boot_cpu_has_bug(bit)		cpu_has_bug(&boot_cpu_data, (bit))
 #define boot_cpu_set_bug(bit)		set_cpu_cap(&boot_cpu_data, (bit))
 
diff --git a/arch/um/kernel/ptrace.c b/arch/um/kernel/ptrace.c
index fdbb37b5c3996..7da0a5223aa69 100644
--- a/arch/um/kernel/ptrace.c
+++ b/arch/um/kernel/ptrace.c
@@ -135,7 +135,7 @@ int syscall_trace_enter(struct pt_regs *regs)
 	if (!test_thread_flag(TIF_SYSCALL_TRACE))
 		return 0;
 
-	return ptrace_report_syscall_entry(regs);
+	return !ptrace_report_syscall_permit_entry(regs);
 }
 
 void syscall_trace_leave(struct pt_regs *regs)
diff --git a/arch/um/kernel/skas/syscall.c b/arch/um/kernel/skas/syscall.c
index ba7494f9bfe41..bc117733e8eff 100644
--- a/arch/um/kernel/skas/syscall.c
+++ b/arch/um/kernel/skas/syscall.c
@@ -27,7 +27,7 @@ void handle_syscall(struct uml_pt_regs *r)
 		goto out;
 
 	/* Do the seccomp check after ptrace; failures should be fast. */
-	if (secure_computing() == -1)
+	if (!seccomp_permit_syscall())
 		goto out;
 
 	syscall = UPT_SYSCALL_NR(r);
diff --git a/arch/um/kernel/um_arch.c b/arch/um/kernel/um_arch.c
index 2141f5f1f5a20..e4ee693961e4e 100644
--- a/arch/um/kernel/um_arch.c
+++ b/arch/um/kernel/um_arch.c
@@ -451,18 +451,6 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
 {
 }
 
-#if IS_ENABLED(CONFIG_SMP)
-void alternatives_smp_module_add(struct module *mod, char *name,
-				 void *locks, void *locks_end,
-				 void *text,  void *text_end)
-{
-}
-
-void alternatives_smp_module_del(struct module *mod)
-{
-}
-#endif
-
 void *text_poke(void *addr, const void *opcode, size_t len)
 {
 	/*
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index bdad90f210e4b..583b4cf4922a3 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -70,7 +70,7 @@ config X86
 	select ARCH_ENABLE_HUGEPAGE_MIGRATION if X86_64 && HUGETLB_PAGE && MIGRATION
 	select ARCH_ENABLE_MEMORY_HOTPLUG if X86_64
 	select ARCH_ENABLE_SPLIT_PMD_PTLOCK if (PGTABLE_LEVELS > 2) && (X86_64 || X86_PAE)
-	select ARCH_ENABLE_THP_MIGRATION if X86_64 && TRANSPARENT_HUGEPAGE
+	select ARCH_HAS_PMD_SOFTLEAVES if X86_64 && TRANSPARENT_HUGEPAGE
 	select ARCH_HAS_ACPI_TABLE_UPGRADE	if ACPI
 	select ARCH_HAS_CPU_ATTACK_VECTORS	if CPU_MITIGATIONS
 	select ARCH_HAS_CACHE_LINE_SIZE
@@ -124,9 +124,11 @@ config X86
 	select ARCH_SUPPORTS_DEBUG_PAGEALLOC
 	select ARCH_SUPPORTS_HUGETLBFS
 	select ARCH_SUPPORTS_PAGE_TABLE_CHECK	if X86_64
+	select ARCH_HAS_PTE_PROTNONE		if X86_64
 	select ARCH_SUPPORTS_NUMA_BALANCING	if X86_64
 	select ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP	if NR_CPUS <= 4096
 	select ARCH_SUPPORTS_CFI		if X86_64
+	select ARCH_SUPPORTS_CMDLINE_FROM_BOOTCONFIG
 	select ARCH_USES_CFI_TRAPS		if X86_64 && CFI
 	select ARCH_SUPPORTS_LTO_CLANG
 	select ARCH_SUPPORTS_LTO_CLANG_THIN
@@ -302,7 +304,6 @@ config X86
 	select HAVE_UNSTABLE_SCHED_CLOCK
 	select HAVE_UNWIND_USER_FP		if X86_64
 	select HAVE_USER_RETURN_NOTIFIER
-	select HAVE_GENERIC_VDSO
 	select VDSO_GETRANDOM			if X86_64
 	select HOTPLUG_PARALLEL			if SMP && X86_64
 	select HOTPLUG_SMT			if SMP
@@ -335,6 +336,7 @@ config X86
 	select SCHED_SMT			if SMP
 	select ARCH_SUPPORTS_SCHED_CLUSTER	if SMP
 	select ARCH_SUPPORTS_SCHED_MC		if SMP
+	select ARCH_SUPPORTS_SYSCALL_USER_DISPATCH
 	select HAVE_SINGLE_FTRACE_DIRECT_OPS	if X86_64 && DYNAMIC_FTRACE_WITH_DIRECT_CALLS
 
 config INSTRUCTION_DECODER
diff --git a/arch/x86/boot/compressed/acpi.c b/arch/x86/boot/compressed/acpi.c
index f196b1d1ddf86..aed27604c11f0 100644
--- a/arch/x86/boot/compressed/acpi.c
+++ b/arch/x86/boot/compressed/acpi.c
@@ -184,10 +184,15 @@ static unsigned long get_cmdline_acpi_rsdp(void)
 	char val[MAX_ADDR_LEN] = { };
 	int ret;
 
-	ret = cmdline_find_option("acpi_rsdp", val, MAX_ADDR_LEN);
+	ret = cmdline_find_option("acpi_rsdp", val, sizeof(val));
 	if (ret < 0)
 		return 0;
 
+	if (ret >= sizeof(val)) {
+		warn("acpi_rsdp= value too long; ignoring");
+		return 0;
+	}
+
 	if (boot_kstrtoul(val, 16, &addr))
 		return 0;
 #endif
diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S
index d9dab940ff626..1aca51390f73a 100644
--- a/arch/x86/boot/compressed/head_64.S
+++ b/arch/x86/boot/compressed/head_64.S
@@ -172,19 +172,23 @@ SYM_FUNC_START(startup_32)
  /*
   * Build early 4G boot pagetable
   */
-	/*
-	 * If SEV is active then set the encryption mask in the page tables.
-	 * This will ensure that when the kernel is copied and decompressed
-	 * it will be done so encrypted.
-	 */
 	xorl	%edx, %edx
 #ifdef	CONFIG_AMD_MEM_ENCRYPT
 	call	get_sev_encryption_bit
 	xorl	%edx, %edx
 	testl	%eax, %eax
 	jz	1f
-	subl	$32, %eax	/* Encryption bit is always above bit 31 */
-	bts	%eax, %edx	/* Set encryption mask for page tables */
+
+	/* Encryption bit is always above bit 31 */
+	subl	$32, %eax
+
+	/*
+	 * If SEV is active then set the encryption mask in the page tables.
+	 * This will ensure that when the kernel is copied and decompressed it
+	 * will be done so encrypted.
+	 */
+	bts	%eax, %edx
+
 	/*
 	 * Set MSR_AMD64_SEV_ENABLED_BIT in sev_status so that
 	 * startup32_check_sev_cbit() will do a check. sev_enable() will
diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c
index 8e4bf5365ac63..22267a83e064a 100644
--- a/arch/x86/boot/compressed/kaslr.c
+++ b/arch/x86/boot/compressed/kaslr.c
@@ -32,6 +32,8 @@
 #include <asm/setup.h>	/* For COMMAND_LINE_SIZE */
 #undef _SETUP
 
+#include <asm/kexec_handover.h>
+
 extern unsigned long get_cmd_line_ptr(void);
 
 /* Simplified build-specific string for starting entropy. */
diff --git a/arch/x86/boot/early_serial_console.c b/arch/x86/boot/early_serial_console.c
index 023bf1c3de8b7..5b83beab89e15 100644
--- a/arch/x86/boot/early_serial_console.c
+++ b/arch/x86/boot/early_serial_console.c
@@ -117,7 +117,7 @@ static unsigned int probe_baud(int port)
 static void parse_console_uart8250(void)
 {
 	char optstr[64], *options;
-	int baud = DEFAULT_BAUD;
+	int baud;
 	int port = 0;
 
 	/*
@@ -136,10 +136,13 @@ static void parse_console_uart8250(void)
 	else
 		return;
 
-	if (options && (options[0] == ','))
-		baud = simple_strtoull(options + 1, &options, 0);
-	else
+	if (options && (options[0] == ',')) {
+		baud = simple_strtoull(options + 1, NULL, 0);
+		if (!baud)
+			baud = DEFAULT_BAUD;
+	} else {
 		baud = probe_baud(port);
+	}
 
 	if (port)
 		early_serial_init(port, baud);
diff --git a/arch/x86/coco/tdx/tdx.c b/arch/x86/coco/tdx/tdx.c
index 29b6f1ed59ec5..f904a636d449b 100644
--- a/arch/x86/coco/tdx/tdx.c
+++ b/arch/x86/coco/tdx/tdx.c
@@ -694,8 +694,8 @@ static bool handle_in(struct pt_regs *regs, int size, int port)
 		.r13 = PORT_READ,
 		.r14 = port,
 	};
-	u64 mask = GENMASK(BITS_PER_BYTE * size, 0);
 	bool success;
+	u64 val;
 
 	/*
 	 * Emulate the I/O read via hypercall. More info about ABI can be found
@@ -703,18 +703,16 @@ static bool handle_in(struct pt_regs *regs, int size, int port)
 	 * "TDG.VP.VMCALL<Instruction.IO>".
 	 */
 	success = !__tdx_hypercall(&args);
+	val = success ? args.r11 : 0;
 
-	/* Update part of the register affected by the emulated instruction */
-	regs->ax &= ~mask;
-	if (success)
-		regs->ax |= args.r11 & mask;
+	insn_assign_reg(&regs->ax, val, size);
 
 	return success;
 }
 
 static bool handle_out(struct pt_regs *regs, int size, int port)
 {
-	u64 mask = GENMASK(BITS_PER_BYTE * size, 0);
+	u64 mask = GENMASK(BITS_PER_BYTE * size - 1, 0);
 
 	/*
 	 * Emulate the I/O write via hypercall. More info about ABI can be found
diff --git a/arch/x86/entry/entry_64_fred.S b/arch/x86/entry/entry_64_fred.S
index 0d2768ab836c4..b98f8945dfff4 100644
--- a/arch/x86/entry/entry_64_fred.S
+++ b/arch/x86/entry/entry_64_fred.S
@@ -7,6 +7,7 @@
 #include <linux/kvm_types.h>
 
 #include <asm/asm.h>
+#include <asm/frame.h>
 #include <asm/fred.h>
 #include <asm/segment.h>
 
@@ -19,6 +20,7 @@
 	UNWIND_HINT_END_OF_STACK
 	ANNOTATE_NOENDBR
 	PUSH_AND_CLEAR_REGS
+	ENCODE_FRAME_POINTER
 	movq	%rsp, %rdi	/* %rdi -> pt_regs */
 .endm
 
diff --git a/arch/x86/entry/syscall_32.c b/arch/x86/entry/syscall_32.c
index 31b9492fe851d..91123a90710c4 100644
--- a/arch/x86/entry/syscall_32.c
+++ b/arch/x86/entry/syscall_32.c
@@ -41,7 +41,9 @@ const sys_call_ptr_t sys_call_table[] = {
 #endif
 
 #define __SYSCALL(nr, sym) case nr: return __ia32_##sym(regs);
-long ia32_sys_call(const struct pt_regs *regs, unsigned int nr)
+
+/* The unsigned int @nr argument is intentional as it creates denser code in a 64-bit build */
+static noinline long ia32_sys_call(const struct pt_regs *regs, unsigned int nr)
 {
 	switch (nr) {
 	#include <asm/syscalls_32.h>
@@ -49,7 +51,7 @@ long ia32_sys_call(const struct pt_regs *regs, unsigned int nr)
 	}
 }
 
-static __always_inline int syscall_32_enter(struct pt_regs *regs)
+static __always_inline long syscall_32_enter(struct pt_regs *regs)
 {
 	if (IS_ENABLED(CONFIG_IA32_EMULATION))
 		current_thread_info()->status |= TS_COMPAT;
@@ -70,19 +72,11 @@ early_param("ia32_emulation", ia32_emulation_override_cmdline);
 /*
  * Invoke a 32-bit syscall.  Called with IRQs on in CT_STATE_KERNEL.
  */
-static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs, int nr)
+static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs, unsigned long nr)
 {
-	/*
-	 * Convert negative numbers to very high and thus out of range
-	 * numbers for comparisons.
-	 */
-	unsigned int unr = nr;
-
-	if (likely(unr < IA32_NR_syscalls)) {
-		unr = array_index_nospec(unr, IA32_NR_syscalls);
-		regs->ax = ia32_sys_call(regs, unr);
-	} else if (nr != -1) {
-		regs->ax = __ia32_sys_ni_syscall(regs);
+	if (likely(nr < IA32_NR_syscalls)) {
+		nr = array_index_nospec(nr, IA32_NR_syscalls);
+		regs->ax = ia32_sys_call(regs, (unsigned int)nr);
 	}
 }
 
@@ -128,7 +122,7 @@ static __always_inline bool int80_is_external(void)
  */
 __visible noinstr void do_int80_emulation(struct pt_regs *regs)
 {
-	int nr;
+	long nr;
 
 	/* Kernel does not use INT $0x80! */
 	if (unlikely(!user_mode(regs))) {
@@ -142,10 +136,9 @@ __visible noinstr void do_int80_emulation(struct pt_regs *regs)
 	 * int80_is_external() below which calls into the APIC driver.
 	 * Identical for soft and external interrupts.
 	 */
-	enter_from_user_mode(regs);
+	enter_from_user_mode_randomize_stack(regs);
 
 	instrumentation_begin();
-	add_random_kstack_offset();
 
 	/* Validate that this is a soft interrupt to the extent possible */
 	if (unlikely(int80_is_external()))
@@ -168,8 +161,9 @@ __visible noinstr void do_int80_emulation(struct pt_regs *regs)
 	nr = syscall_32_enter(regs);
 
 	local_irq_enable();
-	nr = syscall_enter_from_user_mode_work(regs, nr);
-	do_syscall_32_irqs_on(regs, nr);
+
+	if (likely(syscall_enter_from_user_mode_work(regs, &nr)))
+		do_syscall_32_irqs_on(regs, nr);
 
 	instrumentation_end();
 	syscall_exit_to_user_mode(regs);
@@ -208,13 +202,11 @@ __visible noinstr void do_int80_emulation(struct pt_regs *regs)
  */
 DEFINE_FREDENTRY_RAW(int80_emulation)
 {
-	int nr;
+	long nr;
 
-	enter_from_user_mode(regs);
+	enter_from_user_mode_randomize_stack(regs);
 
 	instrumentation_begin();
-	add_random_kstack_offset();
-
 	/*
 	 * FRED pushed 0 into regs::orig_ax and regs::ax contains the
 	 * syscall number.
@@ -232,8 +224,8 @@ DEFINE_FREDENTRY_RAW(int80_emulation)
 	nr = syscall_32_enter(regs);
 
 	local_irq_enable();
-	nr = syscall_enter_from_user_mode_work(regs, nr);
-	do_syscall_32_irqs_on(regs, nr);
+	if (likely(syscall_enter_from_user_mode_work(regs, &nr)))
+		do_syscall_32_irqs_on(regs, nr);
 
 	instrumentation_end();
 	syscall_exit_to_user_mode(regs);
@@ -245,38 +237,32 @@ DEFINE_FREDENTRY_RAW(int80_emulation)
 /* Handles int $0x80 on a 32bit kernel */
 __visible noinstr void do_int80_syscall_32(struct pt_regs *regs)
 {
-	int nr = syscall_32_enter(regs);
+	long nr = syscall_32_enter(regs);
 
 	/*
 	 * Subtlety here: if ptrace pokes something larger than 2^31-1 into
 	 * orig_ax, the int return value truncates it. This matches
 	 * the semantics of syscall_get_nr().
 	 */
-	nr = syscall_enter_from_user_mode(regs, nr);
-	instrumentation_begin();
+	if (likely(syscall_enter_from_user_mode_randomize_stack(regs, &nr))) {
+		instrumentation_begin();
 
-	add_random_kstack_offset();
-	do_syscall_32_irqs_on(regs, nr);
+		do_syscall_32_irqs_on(regs, nr);
 
-	instrumentation_end();
+		instrumentation_end();
+	}
 	syscall_exit_to_user_mode(regs);
 }
 #endif /* !CONFIG_IA32_EMULATION */
 
 static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
 {
-	int nr = syscall_32_enter(regs);
+	long nr = syscall_32_enter(regs);
 	int res;
 
-	/*
-	 * This cannot use syscall_enter_from_user_mode() as it has to
-	 * fetch EBP before invoking any of the syscall entry work
-	 * functions.
-	 */
-	enter_from_user_mode(regs);
+	enter_from_user_mode_randomize_stack(regs);
 
 	instrumentation_begin();
-	add_random_kstack_offset();
 	local_irq_enable();
 	/* Fetch EBP from where the vDSO stashed it. */
 	if (IS_ENABLED(CONFIG_X86_64)) {
@@ -301,10 +287,8 @@ static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
 		return false;
 	}
 
-	nr = syscall_enter_from_user_mode_work(regs, nr);
-
-	/* Now this is just like a normal syscall. */
-	do_syscall_32_irqs_on(regs, nr);
+	if (likely(syscall_enter_from_user_mode_work(regs, &nr)))
+		do_syscall_32_irqs_on(regs, nr);
 
 	instrumentation_end();
 	syscall_exit_to_user_mode(regs);
diff --git a/arch/x86/entry/syscall_64.c b/arch/x86/entry/syscall_64.c
index 71f032504e731..966d3d1b75868 100644
--- a/arch/x86/entry/syscall_64.c
+++ b/arch/x86/entry/syscall_64.c
@@ -32,7 +32,9 @@ const sys_call_ptr_t sys_call_table[] = {
 #undef  __SYSCALL
 
 #define __SYSCALL(nr, sym) case nr: return __x64_##sym(regs);
-long x64_sys_call(const struct pt_regs *regs, unsigned int nr)
+
+/* The unsigned int @nr argument is intentional as it creates denser code */
+static noinline long x64_sys_call(const struct pt_regs *regs, unsigned int nr)
 {
 	switch (nr) {
 	#include <asm/syscalls_64.h>
@@ -40,63 +42,50 @@ long x64_sys_call(const struct pt_regs *regs, unsigned int nr)
 	}
 }
 
-#ifdef CONFIG_X86_X32_ABI
-long x32_sys_call(const struct pt_regs *regs, unsigned int nr)
+static noinline long x32_sys_call(const struct pt_regs *regs, unsigned int nr)
 {
+#ifdef CONFIG_X86_X32_ABI
 	switch (nr) {
 	#include <asm/syscalls_x32.h>
 	default: return __x64_sys_ni_syscall(regs);
 	}
-}
+#else
+	return -ENOSYS;
 #endif
+}
 
-static __always_inline bool do_syscall_x64(struct pt_regs *regs, int nr)
+static __always_inline bool do_syscall_x64(struct pt_regs *regs, unsigned long nr)
 {
-	/*
-	 * Convert negative numbers to very high and thus out of range
-	 * numbers for comparisons.
-	 */
-	unsigned int unr = nr;
-
-	if (likely(unr < NR_syscalls)) {
-		unr = array_index_nospec(unr, NR_syscalls);
-		regs->ax = x64_sys_call(regs, unr);
+	if (likely(nr < NR_syscalls)) {
+		nr = array_index_nospec(nr, NR_syscalls);
+		regs->ax = x64_sys_call(regs, (unsigned int)nr);
 		return true;
 	}
 	return false;
 }
 
-static __always_inline bool do_syscall_x32(struct pt_regs *regs, int nr)
+static __always_inline void do_syscall_x32(struct pt_regs *regs, unsigned long nr)
 {
-	/*
-	 * Adjust the starting offset of the table, and convert numbers
-	 * < __X32_SYSCALL_BIT to very high and thus out of range
-	 * numbers for comparisons.
-	 */
-	unsigned int xnr = nr - __X32_SYSCALL_BIT;
+	/* Adjust the starting offset of the table */
+	nr -= __X32_SYSCALL_BIT;
 
-	if (IS_ENABLED(CONFIG_X86_X32_ABI) && likely(xnr < X32_NR_syscalls)) {
-		xnr = array_index_nospec(xnr, X32_NR_syscalls);
-		regs->ax = x32_sys_call(regs, xnr);
-		return true;
+	if (IS_ENABLED(CONFIG_X86_X32_ABI) && likely(nr < X32_NR_syscalls)) {
+		nr = array_index_nospec(nr, X32_NR_syscalls);
+		regs->ax = x32_sys_call(regs, (unsigned int)nr);
 	}
-	return false;
 }
 
 /* Returns true to return using SYSRET, or false to use IRET */
-__visible noinstr bool do_syscall_64(struct pt_regs *regs, int nr)
+__visible noinstr bool do_syscall_64(struct pt_regs *regs, long nr)
 {
-	nr = syscall_enter_from_user_mode(regs, nr);
+	if (likely(syscall_enter_from_user_mode_randomize_stack(regs, &nr))) {
+		instrumentation_begin();
 
-	instrumentation_begin();
-	add_random_kstack_offset();
+		if (!do_syscall_x64(regs, nr))
+			do_syscall_x32(regs, nr);
 
-	if (!do_syscall_x64(regs, nr) && !do_syscall_x32(regs, nr) && nr != -1) {
-		/* Invalid system call, but still a system call. */
-		regs->ax = __x64_sys_ni_syscall(regs);
+		instrumentation_end();
 	}
-
-	instrumentation_end();
 	syscall_exit_to_user_mode(regs);
 
 	/*
diff --git a/arch/x86/entry/vdso/common/vclock_gettime.c b/arch/x86/entry/vdso/common/vclock_gettime.c
index 57066f346b3fa..304dbd1f9db43 100644
--- a/arch/x86/entry/vdso/common/vclock_gettime.c
+++ b/arch/x86/entry/vdso/common/vclock_gettime.c
@@ -15,6 +15,7 @@
 
 #include "lib/vdso/gettimeofday.c"
 
+#if defined(__x86_64__) || defined(CONFIG_COMPAT_32BIT_TIME)
 int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
 {
 	return __cvdso_gettimeofday(tv, tz);
@@ -29,6 +30,7 @@ __kernel_old_time_t __vdso_time(__kernel_old_time_t *t)
 }
 
 __kernel_old_time_t time(__kernel_old_time_t *t)	__attribute__((weak, alias("__vdso_time")));
+#endif /* CONFIG_COMPAT_32BIT_TIME */
 
 
 #if defined(CONFIG_X86_64) && !defined(BUILD_VDSO32_64)
@@ -51,6 +53,7 @@ int clock_getres(clockid_t, struct __kernel_timespec *)
 
 #else
 /* i386 only */
+#ifdef CONFIG_COMPAT_32BIT_TIME
 int __vdso_clock_gettime(clockid_t clock, struct old_timespec32 *ts)
 {
 	return __cvdso_clock_gettime32(clock, ts);
@@ -59,14 +62,6 @@ int __vdso_clock_gettime(clockid_t clock, struct old_timespec32 *ts)
 int clock_gettime(clockid_t, struct old_timespec32 *)
 	__attribute__((weak, alias("__vdso_clock_gettime")));
 
-int __vdso_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts)
-{
-	return __cvdso_clock_gettime(clock, ts);
-}
-
-int clock_gettime64(clockid_t, struct __kernel_timespec *)
-	__attribute__((weak, alias("__vdso_clock_gettime64")));
-
 int __vdso_clock_getres(clockid_t clock, struct old_timespec32 *res)
 {
 	return __cvdso_clock_getres_time32(clock, res);
@@ -74,6 +69,15 @@ int __vdso_clock_getres(clockid_t clock, struct old_timespec32 *res)
 
 int clock_getres(clockid_t, struct old_timespec32 *)
 	__attribute__((weak, alias("__vdso_clock_getres")));
+#endif /* CONFIG_COMPAT_32BIT_TIME */
+
+int __vdso_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts)
+{
+	return __cvdso_clock_gettime(clock, ts);
+}
+
+int clock_gettime64(clockid_t, struct __kernel_timespec *)
+	__attribute__((weak, alias("__vdso_clock_gettime64")));
 
 int __vdso_clock_getres_time64(clockid_t clock, struct __kernel_timespec *ts)
 {
diff --git a/arch/x86/entry/vdso/vdso32/vdso32.lds.S b/arch/x86/entry/vdso/vdso32/vdso32.lds.S
index cee8f7f9fe80d..00629192db56c 100644
--- a/arch/x86/entry/vdso/vdso32/vdso32.lds.S
+++ b/arch/x86/entry/vdso/vdso32/vdso32.lds.S
@@ -23,10 +23,12 @@ VERSION
 {
 	LINUX_2.6 {
 	global:
+#ifdef CONFIG_COMPAT_32BIT_TIME
 		__vdso_clock_gettime;
 		__vdso_gettimeofday;
 		__vdso_time;
 		__vdso_clock_getres;
+#endif /* CONFIG_COMPAT_32BIT_TIME */
 		__vdso_clock_gettime64;
 		__vdso_clock_getres_time64;
 		__vdso_getcpu;
diff --git a/arch/x86/entry/vsyscall/vsyscall_64.c b/arch/x86/entry/vsyscall/vsyscall_64.c
index ea36de9fa8641..1b68c1cc0d4f6 100644
--- a/arch/x86/entry/vsyscall/vsyscall_64.c
+++ b/arch/x86/entry/vsyscall/vsyscall_64.c
@@ -118,10 +118,10 @@ static bool write_ok_or_segv(unsigned long ptr, size_t size)
 
 static bool __emulate_vsyscall(struct pt_regs *regs, unsigned long address)
 {
-	unsigned long caller;
-	int vsyscall_nr, syscall_nr, tmp;
+	unsigned long caller, orig_dx;
+	int vsyscall_nr, syscall_nr;
+	bool skip;
 	long ret;
-	unsigned long orig_dx;
 
 	/* Confirm that the fault happened in 64-bit user mode */
 	if (!user_64bit_mode(regs))
@@ -197,16 +197,16 @@ static bool __emulate_vsyscall(struct pt_regs *regs, unsigned long address)
 	 */
 	regs->orig_ax = syscall_nr;
 	regs->ax = -ENOSYS;
-	tmp = secure_computing();
-	if ((!tmp && regs->orig_ax != syscall_nr) || regs->ip != address) {
+	skip = !seccomp_permit_syscall();
+	if ((!skip && regs->orig_ax != syscall_nr) || regs->ip != address) {
 		warn_bad_vsyscall(KERN_DEBUG, regs,
 				  "seccomp tried to change syscall nr or ip");
 		force_exit_sig(SIGSYS);
 		return true;
 	}
 	regs->orig_ax = -1;
-	if (tmp)
-		goto do_ret;  /* skip requested */
+	if (skip)
+		goto do_ret;
 
 	/*
 	 * With a real vsyscall, page faults cause SIGSEGV.
diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c
index a787409f5a625..49b6b8fce5664 100644
--- a/arch/x86/events/amd/core.c
+++ b/arch/x86/events/amd/core.c
@@ -1392,7 +1392,7 @@ static struct attribute *amd_brs_events_attrs[] = {
 static umode_t
 amd_brs_is_visible(struct kobject *kobj, struct attribute *attr, int i)
 {
-	return static_cpu_has(X86_FEATURE_BRS) && x86_pmu.lbr_nr ?
+	return cpu_feature_enabled(X86_FEATURE_BRS) && x86_pmu.lbr_nr ?
 	       attr->mode : 0;
 }
 
diff --git a/arch/x86/events/amd/iommu.c b/arch/x86/events/amd/iommu.c
index 07b110e8418ab..f332c7089bd59 100644
--- a/arch/x86/events/amd/iommu.c
+++ b/arch/x86/events/amd/iommu.c
@@ -137,7 +137,7 @@ static ssize_t _iommu_cpumask_show(struct device *dev,
 				   struct device_attribute *attr,
 				   char *buf)
 {
-	return cpumap_print_to_pagebuf(true, buf, &iommu_cpumask);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&iommu_cpumask));
 }
 static DEVICE_ATTR(cpumask, S_IRUGO, _iommu_cpumask_show, NULL);
 
diff --git a/arch/x86/events/amd/power.c b/arch/x86/events/amd/power.c
index 744dffa42deea..66197214b0107 100644
--- a/arch/x86/events/amd/power.c
+++ b/arch/x86/events/amd/power.c
@@ -150,7 +150,7 @@ static void pmu_event_read(struct perf_event *event)
 static ssize_t
 get_attr_cpumask(struct device *dev, struct device_attribute *attr, char *buf)
 {
-	return cpumap_print_to_pagebuf(true, buf, &cpu_mask);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&cpu_mask));
 }
 
 static DEVICE_ATTR(cpumask, S_IRUGO, get_attr_cpumask, NULL);
diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c
index dbc00b6dd69ee..7181973b5b127 100644
--- a/arch/x86/events/amd/uncore.c
+++ b/arch/x86/events/amd/uncore.c
@@ -265,6 +265,29 @@ static void amd_uncore_del(struct perf_event *event, int flags)
 	hwc->idx = -1;
 }
 
+static bool amd_uncore_group_valid(struct perf_event *event)
+{
+	struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
+	struct perf_event *leader = event->group_leader;
+	struct perf_event *sibling;
+	int counters = 0;
+
+	if (leader->pmu == event->pmu)
+		counters++;
+
+	for_each_sibling_event(sibling, leader) {
+		if (sibling->pmu == event->pmu &&
+		    sibling->state > PERF_EVENT_STATE_OFF)
+			counters++;
+	}
+
+	/*
+	 * When pmu->event_init() is called, the event is yet to be linked to
+	 * its leader's sibling list, so it is counted separately
+	 */
+	return (counters + 1) <= pmu->num_counters;
+}
+
 static int amd_uncore_event_init(struct perf_event *event)
 {
 	struct amd_uncore_pmu *pmu;
@@ -282,6 +305,14 @@ static int amd_uncore_event_init(struct perf_event *event)
 	if (!ctx)
 		return -ENODEV;
 
+	/*
+	 * Ensure that all events in a group can be scheduled together so that
+	 * a failure can be reported at perf_event_open() time rather than
+	 * silently at pmu->add() time when no free counter is found
+	 */
+	if (event->group_leader != event && !amd_uncore_group_valid(event))
+		return -EINVAL;
+
 	/*
 	 * NB and Last level cache counters (MSRs) are shared across all cores
 	 * that share the same NB / Last level cache.  On family 16h and below,
@@ -322,7 +353,7 @@ static ssize_t amd_uncore_attr_show_cpumask(struct device *dev,
 	struct pmu *ptr = dev_get_drvdata(dev);
 	struct amd_uncore_pmu *pmu = container_of(ptr, struct amd_uncore_pmu, pmu);
 
-	return cpumap_print_to_pagebuf(true, buf, &pmu->active_mask);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&pmu->active_mask));
 }
 static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL);
 
diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c
index d1af33d96d0a3..af0b67ffb43d8 100644
--- a/arch/x86/events/core.c
+++ b/arch/x86/events/core.c
@@ -2539,7 +2539,8 @@ static int x86_pmu_event_init(struct perf_event *event)
 	}
 
 	if (READ_ONCE(x86_pmu.attr_rdpmc) &&
-	    !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS))
+	    !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS) &&
+	    !(event->hw.config & ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE))
 		event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT;
 
 	return err;
diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index 2b35483e2b707..4dafb9eaba702 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -3533,7 +3533,7 @@ static void intel_pmu_update_rdpmc_user_disable(struct perf_event *event)
 	 */
 	if (x86_pmu.attr_rdpmc == X86_USER_RDPMC_ALWAYS_ENABLE ||
 	    (x86_pmu.attr_rdpmc == X86_USER_RDPMC_CONDITIONAL_ENABLE &&
-	     event->ctx->task))
+	     (event->attach_state & PERF_ATTACH_TASK)))
 		event->hw.config &= ~ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE;
 	else
 		event->hw.config |= ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE;
@@ -3547,8 +3547,6 @@ static void intel_pmu_enable_event(struct perf_event *event)
 	struct hw_perf_event *hwc = &event->hw;
 	int idx = hwc->idx;
 
-	intel_pmu_update_rdpmc_user_disable(event);
-
 	if (unlikely(event->attr.precise_ip))
 		static_call(x86_pmu_pebs_enable)(event);
 
@@ -5147,6 +5145,8 @@ static int intel_pmu_hw_config(struct perf_event *event)
 		leader->hw.flags |= PERF_X86_EVENT_ACR;
 	}
 
+	intel_pmu_update_rdpmc_user_disable(event);
+
 	if ((event->attr.type == PERF_TYPE_HARDWARE) ||
 	    (event->attr.type == PERF_TYPE_HW_CACHE))
 		return 0;
@@ -6360,7 +6360,7 @@ static void intel_pmu_cpu_starting(int cpu)
 	 * Deal with CPUs that don't clear their LBRs on power-up, and that may
 	 * even boot with LBRs enabled.
 	 */
-	if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && x86_pmu.lbr_nr)
+	if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR) && x86_pmu.lbr_nr)
 		msr_clear_bit(MSR_IA32_DEBUGCTLMSR, DEBUGCTLMSR_LBR_BIT);
 	intel_pmu_lbr_reset();
 
@@ -7564,7 +7564,7 @@ static ssize_t intel_hybrid_get_attr_cpus(struct device *dev,
 	struct x86_hybrid_pmu *pmu =
 		container_of(dev_get_drvdata(dev), struct x86_hybrid_pmu, pmu);
 
-	return cpumap_print_to_pagebuf(true, buf, &pmu->supported_cpus);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&pmu->supported_cpus));
 }
 
 static DEVICE_ATTR(cpus, S_IRUGO, intel_hybrid_get_attr_cpus, NULL);
@@ -7679,8 +7679,10 @@ static __always_inline int intel_pmu_init_hybrid(enum hybrid_pmu_type pmus)
 	x86_pmu.num_hybrid_pmus = hweight_long(pmus_mask);
 	x86_pmu.hybrid_pmu = kzalloc_objs(struct x86_hybrid_pmu,
 					  x86_pmu.num_hybrid_pmus);
-	if (!x86_pmu.hybrid_pmu)
+	if (!x86_pmu.hybrid_pmu) {
+		x86_pmu.num_hybrid_pmus = 0;
 		return -ENOMEM;
+	}
 
 	static_branch_enable(&perf_is_hybrid);
 	x86_pmu.filter = intel_pmu_filter;
@@ -7863,14 +7865,14 @@ __init int intel_pmu_init(void)
 	struct attribute **td_attr    = &empty_attrs;
 	struct attribute **mem_attr   = &empty_attrs;
 	struct attribute **tsx_attr   = &empty_attrs;
+	struct x86_hybrid_pmu *pmu;
+	unsigned int fixed_mask;
 	union cpuid10_edx edx;
 	union cpuid10_eax eax;
 	union cpuid10_ebx ebx;
-	unsigned int fixed_mask;
+	int version, i, ret;
 	bool pmem = false;
-	int version, i;
 	char *name;
-	struct x86_hybrid_pmu *pmu;
 
 	/* Architectural Perfmon was introduced starting with Core "Yonah" */
 	if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
@@ -7940,19 +7942,10 @@ __init int intel_pmu_init(void)
 		x86_pmu.lbr_read = intel_pmu_lbr_read_32;
 	}
 
-	if (boot_cpu_has(X86_FEATURE_ARCH_LBR))
-		intel_pmu_arch_lbr_init();
-
 	intel_pebs_init();
 
 	x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */
 
-	if (version >= 5) {
-		x86_pmu.intel_cap.anythread_deprecated = edx.split.anythread_deprecated;
-		if (x86_pmu.intel_cap.anythread_deprecated)
-			pr_cont(" AnyThread deprecated, ");
-	}
-
 	/* The perf side of core PMU is ready to support the mediated vPMU. */
 	x86_get_pmu(smp_processor_id())->capabilities |= PERF_PMU_CAP_MEDIATED_VPMU;
 
@@ -8546,7 +8539,9 @@ __init int intel_pmu_init(void)
 		 *
 		 * Initialize the common PerfMon capabilities here.
 		 */
-		intel_pmu_init_hybrid(hybrid_big_small);
+		ret = intel_pmu_init_hybrid(hybrid_big_small);
+		if (ret)
+			return ret;
 
 		x86_pmu.pebs_latency_data = grt_latency_data;
 		x86_pmu.get_event_constraints = adl_get_event_constraints;
@@ -8604,7 +8599,9 @@ __init int intel_pmu_init(void)
 	case INTEL_METEORLAKE:
 	case INTEL_METEORLAKE_L:
 	case INTEL_ARROWLAKE_U:
-		intel_pmu_init_hybrid(hybrid_big_small);
+		ret = intel_pmu_init_hybrid(hybrid_big_small);
+		if (ret)
+			return ret;
 
 		x86_pmu.pebs_latency_data = cmt_latency_data;
 		x86_pmu.get_event_constraints = mtl_get_event_constraints;
@@ -8635,7 +8632,9 @@ __init int intel_pmu_init(void)
 		pr_cont("Pantherlake Hybrid events, ");
 		name = "pantherlake_hybrid";
 
-		intel_pmu_init_hybrid(hybrid_big_small);
+		ret = intel_pmu_init_hybrid(hybrid_big_small);
+		if (ret)
+			return ret;
 
 		/* Initialize big core specific PerfMon capabilities.*/
 		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
@@ -8650,7 +8649,9 @@ __init int intel_pmu_init(void)
 		pr_cont("Arrowlake Hybrid events, ");
 		name = "arrowlake_hybrid";
 
-		intel_pmu_init_hybrid(hybrid_big_small);
+		ret = intel_pmu_init_hybrid(hybrid_big_small);
+		if (ret)
+			return ret;
 
 		/* Initialize big core specific PerfMon capabilities.*/
 		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
@@ -8667,7 +8668,9 @@ __init int intel_pmu_init(void)
 		pr_cont("Lunarlake Hybrid events, ");
 		name = "lunarlake_hybrid";
 
-		intel_pmu_init_hybrid(hybrid_big_small);
+		ret = intel_pmu_init_hybrid(hybrid_big_small);
+		if (ret)
+			return ret;
 
 		/* Initialize big core specific PerfMon capabilities.*/
 		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
@@ -8692,7 +8695,9 @@ __init int intel_pmu_init(void)
 		break;
 
 	case INTEL_ARROWLAKE_H:
-		intel_pmu_init_hybrid(hybrid_big_small_tiny);
+		ret = intel_pmu_init_hybrid(hybrid_big_small_tiny);
+		if (ret)
+			return ret;
 
 		x86_pmu.pebs_latency_data = arl_h_latency_data;
 		x86_pmu.get_event_constraints = arl_h_get_event_constraints;
@@ -8727,7 +8732,9 @@ __init int intel_pmu_init(void)
 	case INTEL_NOVALAKE_L:
 		pr_cont("Novalake Hybrid events, ");
 		name = "novalake_hybrid";
-		intel_pmu_init_hybrid(hybrid_big_small);
+		ret = intel_pmu_init_hybrid(hybrid_big_small);
+		if (ret)
+			return ret;
 
 		x86_pmu.pebs_latency_data = nvl_latency_data;
 		x86_pmu.get_event_constraints = mtl_get_event_constraints;
@@ -8829,8 +8836,13 @@ __init int intel_pmu_init(void)
 				      &x86_pmu.intel_ctrl);
 
 	/* AnyThread may be deprecated on arch perfmon v5 or later */
-	if (x86_pmu.intel_cap.anythread_deprecated)
+	if (version >= 5 && edx.split.anythread_deprecated) {
 		x86_pmu.format_attrs = intel_arch_formats_attr;
+		pr_cont("AnyThread deprecated, ");
+	}
+
+	if (boot_cpu_has(X86_FEATURE_ARCH_LBR))
+		intel_pmu_arch_lbr_init();
 
 	intel_pmu_check_event_constraints_all(NULL);
 
diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c
index 91a093d8cf2e7..8940f0292229f 100644
--- a/arch/x86/events/intel/ds.c
+++ b/arch/x86/events/intel/ds.c
@@ -832,7 +832,7 @@ static void *dsalloc_pages(size_t size, gfp_t flags, int cpu)
 	int node = cpu_to_node(cpu);
 	struct page *page;
 
-	page = __alloc_pages_node(node, flags | __GFP_ZERO, order);
+	page = alloc_pages_node(node, flags | __GFP_ZERO, order);
 	return page ? page_address(page) : NULL;
 }
 
@@ -1088,9 +1088,9 @@ void init_arch_pebs_on_cpu(int cpu)
 
 	/*
 	 * 4KB-aligned pointer of the output buffer
-	 * (__alloc_pages_node() return page aligned address)
+	 * (alloc_pages_node() returns page aligned address)
 	 * Buffer Size = 4KB * 2^SIZE
-	 * contiguous physical buffer (__alloc_pages_node() with order)
+	 * contiguous physical buffer (alloc_pages_node() with order)
 	 */
 	arch_pebs_base = virt_to_phys(cpuc->pebs_vaddr) | PEBS_BUFFER_SHIFT;
 	wrmsrq_on_cpu(cpu, MSR_IA32_PEBS_BASE, arch_pebs_base);
@@ -1444,10 +1444,6 @@ struct event_constraint intel_skl_pebs_event_constraints[] = {
 };
 
 struct event_constraint intel_icl_pebs_event_constraints[] = {
-	INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x100000000ULL),	/* old INST_RETIRED.PREC_DIST */
-	INTEL_FLAGS_UEVENT_CONSTRAINT(0x0100, 0x100000000ULL),	/* INST_RETIRED.PREC_DIST */
-	INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL),	/* SLOTS */
-
 	INTEL_PLD_CONSTRAINT(0x1cd, 0xff),			/* MEM_TRANS_RETIRED.LOAD_LATENCY */
 	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf),	/* MEM_INST_RETIRED.STLB_MISS_LOADS */
 	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf),	/* MEM_INST_RETIRED.STLB_MISS_STORES */
@@ -1470,9 +1466,6 @@ struct event_constraint intel_icl_pebs_event_constraints[] = {
 };
 
 struct event_constraint intel_glc_pebs_event_constraints[] = {
-	INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL),	/* INST_RETIRED.PREC_DIST */
-	INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL),
-
 	INTEL_FLAGS_EVENT_CONSTRAINT(0xc0, 0xfe),
 	INTEL_PLD_CONSTRAINT(0x1cd, 0xfe),
 	INTEL_PSD_CONSTRAINT(0x2cd, 0x1),
@@ -1497,9 +1490,6 @@ struct event_constraint intel_glc_pebs_event_constraints[] = {
 };
 
 struct event_constraint intel_lnc_pebs_event_constraints[] = {
-	INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL),	/* INST_RETIRED.PREC_DIST */
-	INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL),
-
 	INTEL_FLAGS_UEVENT_CONSTRAINT(0x012a, 0x1),		/* OCR.* events */
 	INTEL_FLAGS_UEVENT_CONSTRAINT(0x012b, 0x1),		/* OCR.* events */
 
@@ -1531,9 +1521,6 @@ struct event_constraint intel_lnc_pebs_event_constraints[] = {
 };
 
 struct event_constraint intel_pnc_pebs_event_constraints[] = {
-	INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL),	/* INST_RETIRED.PREC_DIST */
-	INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL),
-
 	INTEL_HYBRID_LDLAT_CONSTRAINT(0x1cd, 0xfc),
 	INTEL_HYBRID_STLAT_CONSTRAINT(0x2cd, 0x3),
 	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf),	/* MEM_INST_RETIRED.STLB_MISS_LOADS */
diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c
index cae2e02fe6cc4..cbe5c762008d2 100644
--- a/arch/x86/events/intel/lbr.c
+++ b/arch/x86/events/intel/lbr.c
@@ -109,7 +109,7 @@ static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc);
 
 static __always_inline bool is_lbr_call_stack_bit_set(u64 config)
 {
-	if (static_cpu_has(X86_FEATURE_ARCH_LBR))
+	if (cpu_feature_enabled(X86_FEATURE_ARCH_LBR))
 		return !!(config & ARCH_LBR_CALL_STACK);
 
 	return !!(config & LBR_CALL_STACK);
@@ -138,13 +138,13 @@ static void __intel_pmu_lbr_enable(bool pmi)
 	 */
 	if (cpuc->lbr_sel)
 		lbr_select = cpuc->lbr_sel->config & x86_pmu.lbr_sel_mask;
-	if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && !pmi && cpuc->lbr_sel)
+	if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR) && !pmi && cpuc->lbr_sel)
 		wrmsrq(MSR_LBR_SELECT, lbr_select);
 
 	rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl);
 	orig_debugctl = debugctl;
 
-	if (!static_cpu_has(X86_FEATURE_ARCH_LBR))
+	if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR))
 		debugctl |= DEBUGCTLMSR_LBR;
 	/*
 	 * LBR callstack does not work well with FREEZE_LBRS_ON_PMI.
@@ -159,7 +159,7 @@ static void __intel_pmu_lbr_enable(bool pmi)
 	if (orig_debugctl != debugctl)
 		wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctl);
 
-	if (static_cpu_has(X86_FEATURE_ARCH_LBR))
+	if (cpu_feature_enabled(X86_FEATURE_ARCH_LBR))
 		wrmsrq(MSR_ARCH_LBR_CTL, lbr_select | ARCH_LBR_CTL_LBREN);
 }
 
@@ -200,7 +200,7 @@ void intel_pmu_lbr_reset(void)
 
 	cpuc->last_task_ctx = NULL;
 	cpuc->last_log_id = 0;
-	if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && cpuc->lbr_select)
+	if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR) && cpuc->lbr_select)
 		wrmsrq(MSR_LBR_SELECT, 0);
 }
 
@@ -418,7 +418,7 @@ static void intel_pmu_arch_lbr_xrstors(void *ctx)
 
 static __always_inline bool lbr_is_reset_in_cstate(void *ctx)
 {
-	if (static_cpu_has(X86_FEATURE_ARCH_LBR))
+	if (cpu_feature_enabled(X86_FEATURE_ARCH_LBR))
 		return x86_pmu.lbr_deep_c_reset && !rdlbr_from(0, NULL);
 
 	return !rdlbr_from(((struct x86_perf_task_context *)ctx)->tos, NULL);
@@ -624,7 +624,7 @@ void release_lbr_buffers(void)
 	struct cpu_hw_events *cpuc;
 	int cpu;
 
-	if (!static_cpu_has(X86_FEATURE_ARCH_LBR))
+	if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR))
 		return;
 
 	for_each_possible_cpu(cpu) {
@@ -643,7 +643,7 @@ void reserve_lbr_buffers(void)
 	struct cpu_hw_events *cpuc;
 	int cpu;
 
-	if (!static_cpu_has(X86_FEATURE_ARCH_LBR))
+	if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR))
 		return;
 
 	for_each_possible_cpu(cpu) {
@@ -730,7 +730,7 @@ void intel_pmu_lbr_disable_all(void)
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
 	if (cpuc->lbr_users && !vlbr_exclude_host()) {
-		if (static_cpu_has(X86_FEATURE_ARCH_LBR))
+		if (cpu_feature_enabled(X86_FEATURE_ARCH_LBR))
 			return __intel_pmu_arch_lbr_disable();
 
 		__intel_pmu_lbr_disable();
@@ -889,7 +889,7 @@ static __always_inline u16 get_lbr_cycles(u64 info)
 {
 	u16 cycles = info & LBR_INFO_CYCLES;
 
-	if (static_cpu_has(X86_FEATURE_ARCH_LBR) &&
+	if (cpu_feature_enabled(X86_FEATURE_ARCH_LBR) &&
 	    (!static_branch_likely(&x86_lbr_cycles) ||
 	     !(info & LBR_INFO_CYC_CNT_VALID)))
 		cycles = 0;
@@ -1124,7 +1124,7 @@ static int intel_pmu_setup_hw_lbr_filter(struct perf_event *event)
 	reg = &event->hw.branch_reg;
 	reg->idx = EXTRA_REG_LBR;
 
-	if (static_cpu_has(X86_FEATURE_ARCH_LBR)) {
+	if (cpu_feature_enabled(X86_FEATURE_ARCH_LBR)) {
 		reg->config = mask;
 
 		/*
@@ -1213,7 +1213,7 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
 {
 	u64 from, to;
 	int br_sel = cpuc->br_sel;
-	int i, j, type, to_plm;
+	int i, j, type, from_plm, to_plm;
 	bool compress = false;
 
 	/* if sampling all branches, then nothing to filter */
@@ -1232,7 +1232,7 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
 		 * Doesn't support OTHER_BRANCH decoding for now.
 		 * OTHER_BRANCH branch type still rely on software decoding.
 		 */
-		if (static_cpu_has(X86_FEATURE_ARCH_LBR) &&
+		if (static_branch_likely(&x86_lbr_type) &&
 		    type <= ARCH_LBR_BR_TYPE_KNOWN_MAX) {
 			to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER;
 			type = arch_lbr_br_type_map[type] | to_plm;
@@ -1245,8 +1245,14 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
 				type |= X86_BR_NO_TX;
 		}
 
-		/* if type does not correspond, then discard */
-		if (type == X86_BR_NONE || (br_sel & type) != type) {
+		from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER;
+		/*
+		 * If type does not correspond, then discard.
+		 * Specifically reject entries whose from address is in
+		 * kernel space when only X86_BR_USER is requested.
+		 */
+		if (type == X86_BR_NONE || (br_sel & type) != type ||
+		    (!(br_sel & X86_BR_KERNEL) && (from_plm & X86_BR_KERNEL))) {
 			cpuc->lbr_entries[i].from = 0;
 			compress = true;
 		}
@@ -1279,7 +1285,7 @@ void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr)
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
 	/* Cannot get TOS for large PEBS and Arch LBR */
-	if (static_cpu_has(X86_FEATURE_ARCH_LBR) ||
+	if (cpu_feature_enabled(X86_FEATURE_ARCH_LBR) ||
 	    (cpuc->n_pebs == cpuc->n_large_pebs))
 		cpuc->lbr_stack.hw_idx = -1ULL;
 	else
diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c
index 7857959c6e823..b2109b37ea7f4 100644
--- a/arch/x86/events/intel/uncore.c
+++ b/arch/x86/events/intel/uncore.c
@@ -757,7 +757,7 @@ static int uncore_pmu_event_init(struct perf_event *event)
 
 	pmu = uncore_event_to_pmu(event);
 	/* no device found for this pmu */
-	if (!pmu->registered)
+	if (!uncore_pmu_available(pmu))
 		return -ENOENT;
 
 	/* Sampling not supported yet */
@@ -861,7 +861,7 @@ static ssize_t uncore_get_attr_cpumask(struct device *dev,
 {
 	struct intel_uncore_pmu *pmu = container_of(dev_get_drvdata(dev), struct intel_uncore_pmu, pmu);
 
-	return cpumap_print_to_pagebuf(true, buf, &pmu->cpu_mask);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&pmu->cpu_mask));
 }
 
 static DEVICE_ATTR(cpumask, S_IRUGO, uncore_get_attr_cpumask, NULL);
@@ -953,16 +953,18 @@ static int uncore_pmu_register(struct intel_uncore_pmu *pmu)
 
 	ret = perf_pmu_register(&pmu->pmu, pmu->name, -1);
 	if (!ret)
-		pmu->registered = true;
+		uncore_pmu_set_registered(pmu);
 	return ret;
 }
 
 static void uncore_pmu_unregister(struct intel_uncore_pmu *pmu)
 {
-	if (!pmu->registered)
+	if (!uncore_pmu_registered(pmu))
 		return;
 	perf_pmu_unregister(&pmu->pmu);
-	pmu->registered = false;
+
+	/* Keep PMU_BROKEN_BIT sticky. */
+	uncore_pmu_clear_registered(pmu);
 }
 
 static void uncore_free_boxes(struct intel_uncore_pmu *pmu)
@@ -1148,6 +1150,44 @@ uncore_pci_find_dev_pmu(struct pci_dev *pdev, const struct pci_device_id *ids)
 	return pmu;
 }
 
+static int uncore_box_setup(struct intel_uncore_pmu *pmu,
+			    struct intel_uncore_box *box)
+{
+	int ret;
+
+	if (uncore_pmu_broken(pmu))
+		return -ENODEV;
+
+	ret = uncore_box_init(box);
+	if (ret)
+		goto err;
+
+	/* First active box registers the pmu. */
+	if (atomic_inc_return(&pmu->activeboxes) > 1)
+		return 0;
+
+	ret = uncore_pmu_register(pmu);
+	if (ret) {
+		atomic_dec(&pmu->activeboxes);
+		goto err;
+	}
+
+	return 0;
+err:
+	/*
+	 * If any box fails, mark the per-package PMU as broken regardless of
+	 * whether it was registered or not.
+	 *
+	 * Don't decrement refcnt to avoid other in-die CPUs from trying to set
+	 * up the PMU box again.
+	 *
+	 * Don't kfree box; MSR and MMIO boxes are freed at module exit only.
+	 */
+	uncore_pmu_set_broken(pmu);
+	uncore_box_exit(box);
+	return ret;
+}
+
 /*
  * Register the PMU for a PCI device
  * @pdev: The PCI device.
@@ -1167,26 +1207,22 @@ static int uncore_pci_pmu_register(struct pci_dev *pdev,
 		return -EINVAL;
 
 	box = uncore_alloc_box(type, NUMA_NO_NODE);
-	if (!box)
+	if (!box) {
+		uncore_pmu_set_broken(pmu);
 		return -ENOMEM;
+	}
 
 	atomic_inc(&box->refcnt);
 	box->dieid = die;
 	box->pci_dev = pdev;
 	box->pmu = pmu;
-	uncore_box_init(box);
-
-	pmu->boxes[die] = box;
-	if (atomic_inc_return(&pmu->activeboxes) > 1)
-		return 0;
 
-	/* First active box registers the pmu */
-	ret = uncore_pmu_register(pmu);
-	if (ret) {
-		pmu->boxes[die] = NULL;
-		uncore_box_exit(box);
+	ret = uncore_box_setup(pmu, box);
+	if (!ret)
+		pmu->boxes[die] = box;
+	else
 		kfree(box);
-	}
+
 	return ret;
 }
 
@@ -1248,11 +1284,16 @@ static void uncore_pci_pmu_unregister(struct intel_uncore_pmu *pmu, int die)
 {
 	struct intel_uncore_box *box = pmu->boxes[die];
 
+	if (!box)
+		return;
+
 	pmu->boxes[die] = NULL;
 	if (atomic_dec_return(&pmu->activeboxes) == 0)
 		uncore_pmu_unregister(pmu);
-	uncore_box_exit(box);
-	kfree(box);
+	if (atomic_dec_return(&box->refcnt) == 0) {
+		uncore_box_exit(box);
+		kfree(box);
+	}
 }
 
 static void uncore_pci_remove(struct pci_dev *pdev)
@@ -1272,7 +1313,6 @@ static void uncore_pci_remove(struct pci_dev *pdev)
 				break;
 			}
 		}
-		WARN_ON_ONCE(i >= UNCORE_EXTRA_PCI_DEV_MAX);
 		return;
 	}
 
@@ -1486,7 +1526,8 @@ static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu,
 
 		if (old_cpu < 0) {
 			WARN_ON_ONCE(box->cpu != -1);
-			if (uncore_die_has_box(type, die, pmu->pmu_idx)) {
+			if (uncore_die_has_box(type, die, pmu->pmu_idx) &&
+			    !uncore_pmu_broken(pmu)) {
 				box->cpu = new_cpu;
 				cpumask_set_cpu(new_cpu, &pmu->cpu_mask);
 			}
@@ -1494,12 +1535,14 @@ static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu,
 		}
 
 		WARN_ON_ONCE(box->cpu != -1 && box->cpu != old_cpu);
-		box->cpu = -1;
 		cpumask_clear_cpu(old_cpu, &pmu->cpu_mask);
-		if (new_cpu < 0)
+		if (new_cpu < 0) {
+			box->cpu = -1;
 			continue;
+		}
 
-		if (!uncore_die_has_box(type, die, pmu->pmu_idx))
+		/* An inactive box doesn't need migration. */
+		if (box->cpu == -1)
 			continue;
 		uncore_pmu_cancel_hrtimer(box);
 		perf_pmu_migrate_context(&pmu->pmu, old_cpu, new_cpu);
@@ -1515,7 +1558,7 @@ static void uncore_change_context(struct intel_uncore_type **uncores,
 		uncore_change_type_ctx(*uncores, old_cpu, new_cpu);
 }
 
-static void uncore_box_unref(struct intel_uncore_type **types, int id)
+static void uncore_box_unref(struct intel_uncore_type **types, int die)
 {
 	struct intel_uncore_type *type;
 	struct intel_uncore_pmu *pmu;
@@ -1526,9 +1569,14 @@ static void uncore_box_unref(struct intel_uncore_type **types, int id)
 		type = *types;
 		pmu = type->pmus;
 		for (i = 0; i < type->num_boxes; i++, pmu++) {
-			box = pmu->boxes[id];
-			if (box && box->cpu >= 0 && atomic_dec_return(&box->refcnt) == 0)
+			box = pmu->boxes[die];
+			if (box && box->cpu >= 0 &&
+			    atomic_dec_return(&box->refcnt) == 0) {
+				if (uncore_box_active(box) &&
+				    atomic_dec_return(&pmu->activeboxes) == 0)
+					uncore_pmu_unregister(pmu);
 				uncore_box_exit(box);
+			}
 		}
 	}
 }
@@ -1537,9 +1585,15 @@ static int uncore_event_cpu_offline(unsigned int cpu)
 {
 	int die, target;
 
+	/* Clear the references */
+	die = topology_logical_die_id(cpu);
+	uncore_box_unref(uncore_msr_uncores, die);
+	uncore_box_unref(uncore_mmio_uncores, die);
+
 	/* Check if exiting cpu is used for collecting uncore events */
 	if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
-		goto unref;
+		return 0;
+
 	/* Find a new cpu to collect uncore events */
 	target = cpumask_any_but(topology_die_cpumask(cpu), cpu);
 
@@ -1552,16 +1606,10 @@ static int uncore_event_cpu_offline(unsigned int cpu)
 	uncore_change_context(uncore_msr_uncores, cpu, target);
 	uncore_change_context(uncore_mmio_uncores, cpu, target);
 	uncore_change_context(uncore_pci_uncores, cpu, target);
-
-unref:
-	/* Clear the references */
-	die = topology_logical_die_id(cpu);
-	uncore_box_unref(uncore_msr_uncores, die);
-	uncore_box_unref(uncore_mmio_uncores, die);
 	return 0;
 }
 
-static int allocate_boxes(struct intel_uncore_type **types,
+static void allocate_boxes(struct intel_uncore_type **types,
 			 unsigned int die, unsigned int cpu)
 {
 	struct intel_uncore_box *box, *tmp;
@@ -1575,11 +1623,13 @@ static int allocate_boxes(struct intel_uncore_type **types,
 		type = *types;
 		pmu = type->pmus;
 		for (i = 0; i < type->num_boxes; i++, pmu++) {
-			if (pmu->boxes[die])
+			if (pmu->boxes[die] || uncore_pmu_broken(pmu))
 				continue;
 			box = uncore_alloc_box(type, cpu_to_node(cpu));
-			if (!box)
+			if (!box) {
+				uncore_pmu_set_broken(pmu);
 				goto cleanup;
+			}
 			box->pmu = pmu;
 			box->dieid = die;
 			list_add(&box->active_list, &allocated);
@@ -1590,35 +1640,30 @@ static int allocate_boxes(struct intel_uncore_type **types,
 		list_del_init(&box->active_list);
 		box->pmu->boxes[die] = box;
 	}
-	return 0;
+	return;
 
 cleanup:
 	list_for_each_entry_safe(box, tmp, &allocated, active_list) {
 		list_del_init(&box->active_list);
 		kfree(box);
 	}
-	return -ENOMEM;
 }
 
 static int uncore_box_ref(struct intel_uncore_type **types,
-			  int id, unsigned int cpu)
+			  int die, unsigned int cpu)
 {
 	struct intel_uncore_type *type;
 	struct intel_uncore_pmu *pmu;
 	struct intel_uncore_box *box;
-	int i, ret;
-
-	ret = allocate_boxes(types, id, cpu);
-	if (ret)
-		return ret;
+	int i;
 
 	for (; *types; types++) {
 		type = *types;
 		pmu = type->pmus;
 		for (i = 0; i < type->num_boxes; i++, pmu++) {
-			box = pmu->boxes[id];
+			box = pmu->boxes[die];
 			if (box && box->cpu >= 0 && atomic_inc_return(&box->refcnt) == 1)
-				uncore_box_init(box);
+				uncore_box_setup(pmu, box);
 		}
 	}
 	return 0;
@@ -1626,93 +1671,35 @@ static int uncore_box_ref(struct intel_uncore_type **types,
 
 static int uncore_event_cpu_online(unsigned int cpu)
 {
-	int die, target, msr_ret, mmio_ret;
+	int die, target;
 
 	die = topology_logical_die_id(cpu);
-	msr_ret = uncore_box_ref(uncore_msr_uncores, die, cpu);
-	mmio_ret = uncore_box_ref(uncore_mmio_uncores, die, cpu);
-	if (msr_ret && mmio_ret)
-		return -ENOMEM;
+	allocate_boxes(uncore_msr_uncores, die, cpu);
+	allocate_boxes(uncore_mmio_uncores, die, cpu);
 
 	/*
 	 * Check if there is an online cpu in the package
 	 * which collects uncore events already.
 	 */
 	target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu));
-	if (target < nr_cpu_ids)
-		return 0;
-
-	cpumask_set_cpu(cpu, &uncore_cpu_mask);
-
-	if (!msr_ret)
+	if (target >= nr_cpu_ids) {
+		cpumask_set_cpu(cpu, &uncore_cpu_mask);
 		uncore_change_context(uncore_msr_uncores, -1, cpu);
-	if (!mmio_ret)
 		uncore_change_context(uncore_mmio_uncores, -1, cpu);
-	uncore_change_context(uncore_pci_uncores, -1, cpu);
-	return 0;
-}
-
-static int __init type_pmu_register(struct intel_uncore_type *type)
-{
-	int i, ret;
-
-	for (i = 0; i < type->num_boxes; i++) {
-		ret = uncore_pmu_register(&type->pmus[i]);
-		if (ret)
-			return ret;
+		uncore_change_context(uncore_pci_uncores, -1, cpu);
 	}
-	return 0;
-}
-
-static int __init uncore_msr_pmus_register(void)
-{
-	struct intel_uncore_type **types = uncore_msr_uncores;
-	int ret;
 
-	for (; *types; types++) {
-		ret = type_pmu_register(*types);
-		if (ret)
-			return ret;
-	}
-	return 0;
-}
-
-static int __init uncore_cpu_init(void)
-{
-	int ret;
-
-	ret = uncore_types_init(uncore_msr_uncores);
-	if (ret)
-		goto err;
-
-	ret = uncore_msr_pmus_register();
-	if (ret)
-		goto err;
+	uncore_box_ref(uncore_msr_uncores, die, cpu);
+	uncore_box_ref(uncore_mmio_uncores, die, cpu);
 	return 0;
-err:
-	uncore_types_exit(uncore_msr_uncores);
-	uncore_msr_uncores = empty_uncore;
-	return ret;
 }
 
-static int __init uncore_mmio_init(void)
+static int __init uncore_pmu_types_init(struct intel_uncore_type **types)
 {
-	struct intel_uncore_type **types = uncore_mmio_uncores;
-	int ret;
-
-	ret = uncore_types_init(types);
+	int ret = uncore_types_init(types);
 	if (ret)
-		goto err;
+		uncore_types_exit(types);
 
-	for (; *types; types++) {
-		ret = type_pmu_register(*types);
-		if (ret)
-			goto err;
-	}
-	return 0;
-err:
-	uncore_types_exit(uncore_mmio_uncores);
-	uncore_mmio_uncores = empty_uncore;
 	return ret;
 }
 
@@ -2013,12 +2000,16 @@ static int __init intel_uncore_init(void)
 
 	if (uncore_init->cpu_init) {
 		uncore_init->cpu_init();
-		cret = uncore_cpu_init();
+		cret = uncore_pmu_types_init(uncore_msr_uncores);
+		if (cret)
+			uncore_msr_uncores = empty_uncore;
 	}
 
 	if (uncore_init->mmio_init) {
 		uncore_init->mmio_init();
-		mret = uncore_mmio_init();
+		mret = uncore_pmu_types_init(uncore_mmio_uncores);
+		if (mret)
+			uncore_mmio_uncores = empty_uncore;
 	}
 
 	if (cret && pret && mret) {
diff --git a/arch/x86/events/intel/uncore.h b/arch/x86/events/intel/uncore.h
index c2e5ccb1d72c4..c8dfa2d21bfd9 100644
--- a/arch/x86/events/intel/uncore.h
+++ b/arch/x86/events/intel/uncore.h
@@ -129,7 +129,7 @@ struct intel_uncore_type {
 #define events_group attr_groups[2]
 
 struct intel_uncore_ops {
-	void (*init_box)(struct intel_uncore_box *);
+	int (*init_box)(struct intel_uncore_box *);
 	void (*exit_box)(struct intel_uncore_box *);
 	void (*disable_box)(struct intel_uncore_box *);
 	void (*enable_box)(struct intel_uncore_box *);
@@ -146,13 +146,24 @@ struct intel_uncore_pmu {
 	struct pmu			pmu;
 	char				name[UNCORE_PMU_NAME_LEN];
 	int				pmu_idx;
-	bool				registered;
+	unsigned long			flags;
 	atomic_t			activeboxes;
 	cpumask_t			cpu_mask;
 	struct intel_uncore_type	*type;
 	struct intel_uncore_box		**boxes;
 };
 
+#define PMU_REGISTERED_BIT	0
+#define PMU_BROKEN_BIT		1
+
+#define uncore_pmu_registered(pmu)	test_bit(PMU_REGISTERED_BIT, &(pmu)->flags)
+#define uncore_pmu_broken(pmu)		test_bit(PMU_BROKEN_BIT, &(pmu)->flags)
+#define uncore_pmu_available(pmu)	(uncore_pmu_registered(pmu) &&  \
+					 !uncore_pmu_broken(pmu))
+#define uncore_pmu_set_registered(pmu)	set_bit(PMU_REGISTERED_BIT, &(pmu)->flags)
+#define uncore_pmu_set_broken(pmu)	set_bit(PMU_BROKEN_BIT, &(pmu)->flags)
+#define uncore_pmu_clear_registered(pmu) clear_bit(PMU_REGISTERED_BIT, &(pmu)->flags)
+
 struct intel_uncore_extra_reg {
 	raw_spinlock_t lock;
 	u64 config, config1, config2;
@@ -185,7 +196,7 @@ struct intel_uncore_box {
 #define CFL_UNC_CBO_7_PERFEVTSEL0		0xf70
 #define CFL_UNC_CBO_7_PER_CTR0			0xf76
 
-#define UNCORE_BOX_FLAG_INITIATED		0
+#define UNCORE_BOX_FLAG_INITIALIZED		0
 /* event config registers are 8-byte apart */
 #define UNCORE_BOX_FLAG_CTL_OFFS8		1
 /* CFL 8th CBOX has different MSR space */
@@ -557,17 +568,29 @@ static inline u64 uncore_read_counter(struct intel_uncore_box *box,
 	return box->pmu->type->ops->read_counter(box, event);
 }
 
-static inline void uncore_box_init(struct intel_uncore_box *box)
+static inline bool uncore_box_active(struct intel_uncore_box *box)
 {
-	if (!test_and_set_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) {
-		if (box->pmu->type->ops->init_box)
-			box->pmu->type->ops->init_box(box);
+	return (!box->pmu->type->ops->init_box ||
+		test_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags));
+}
+
+static inline int uncore_box_init(struct intel_uncore_box *box)
+{
+	int ret = 0;
+
+	if (!test_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags) &&
+	    box->pmu->type->ops->init_box) {
+		ret = box->pmu->type->ops->init_box(box);
+		if (!ret)
+			__set_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags);
 	}
+
+	return ret;
 }
 
 static inline void uncore_box_exit(struct intel_uncore_box *box)
 {
-	if (test_and_clear_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) {
+	if (test_and_clear_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags)) {
 		if (box->pmu->type->ops->exit_box)
 			box->pmu->type->ops->exit_box(box);
 	}
diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c
index e507762222568..0a22edf4d509a 100644
--- a/arch/x86/events/intel/uncore_discovery.c
+++ b/arch/x86/events/intel/uncore_discovery.c
@@ -489,14 +489,15 @@ static u64 intel_generic_uncore_box_ctl(struct intel_uncore_box *box)
 	return unit->addr;
 }
 
-void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box)
+int intel_generic_uncore_msr_init_box(struct intel_uncore_box *box)
 {
 	u64 box_ctl = intel_generic_uncore_box_ctl(box);
 
 	if (!box_ctl)
-		return;
+		return -ENODEV;
 
 	wrmsrq(box_ctl, GENERIC_PMON_BOX_CTL_INT);
+	return 0;
 }
 
 void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box)
@@ -578,15 +579,16 @@ static inline int intel_pci_uncore_box_ctl(struct intel_uncore_box *box)
 	return UNCORE_DISCOVERY_PCI_BOX_CTRL(intel_generic_uncore_box_ctl(box));
 }
 
-void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box)
+int intel_generic_uncore_pci_init_box(struct intel_uncore_box *box)
 {
 	int box_ctl = intel_pci_uncore_box_ctl(box);
 
 	if (!box_ctl)
-		return;
+		return -ENODEV;
 
 	__set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags);
-	pci_write_config_dword(box->pci_dev, box_ctl, GENERIC_PMON_BOX_CTL_INT);
+	return pci_write_config_dword(box->pci_dev, box_ctl,
+				      GENERIC_PMON_BOX_CTL_INT);
 }
 
 void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box)
@@ -648,7 +650,7 @@ static struct intel_uncore_ops generic_uncore_pci_ops = {
 
 #define UNCORE_GENERIC_MMIO_SIZE		0x4000
 
-void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box)
+int intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box)
 {
 	static struct intel_uncore_discovery_unit *unit;
 	struct intel_uncore_type *type = box->pmu->type;
@@ -658,13 +660,13 @@ void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box)
 	if (!unit) {
 		pr_warn("Uncore type %d id %d: Cannot find box control address.\n",
 			type->type_id, box->pmu->pmu_idx);
-		return;
+		return -ENODEV;
 	}
 
 	if (!unit->addr) {
 		pr_warn("Uncore type %d box %d: Invalid box control address.\n",
 			type->type_id, unit->id);
-		return;
+		return -ENODEV;
 	}
 
 	addr = unit->addr;
@@ -672,10 +674,11 @@ void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box)
 	if (!box->io_addr) {
 		pr_warn("Uncore type %d box %d: ioremap error for 0x%llx.\n",
 			type->type_id, unit->id, (unsigned long long)addr);
-		return;
+		return -ENOMEM;
 	}
 
 	writel(GENERIC_PMON_BOX_CTL_INT, box->io_addr);
+	return 0;
 }
 
 void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box)
diff --git a/arch/x86/events/intel/uncore_discovery.h b/arch/x86/events/intel/uncore_discovery.h
index e1330342b92ee..142e1b56cfc2e 100644
--- a/arch/x86/events/intel/uncore_discovery.h
+++ b/arch/x86/events/intel/uncore_discovery.h
@@ -148,11 +148,11 @@ void intel_uncore_generic_uncore_cpu_init(void);
 int intel_uncore_generic_uncore_pci_init(void);
 void intel_uncore_generic_uncore_mmio_init(void);
 
-void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box);
+int intel_generic_uncore_msr_init_box(struct intel_uncore_box *box);
 void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box);
 void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box);
 
-void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box);
+int intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box);
 void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box);
 void intel_generic_uncore_mmio_enable_box(struct intel_uncore_box *box);
 void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box,
@@ -160,7 +160,7 @@ void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box,
 void intel_generic_uncore_mmio_enable_event(struct intel_uncore_box *box,
 					    struct perf_event *event);
 
-void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box);
+int intel_generic_uncore_pci_init_box(struct intel_uncore_box *box);
 void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box);
 void intel_generic_uncore_pci_enable_box(struct intel_uncore_box *box);
 void intel_generic_uncore_pci_disable_event(struct intel_uncore_box *box,
diff --git a/arch/x86/events/intel/uncore_nhmex.c b/arch/x86/events/intel/uncore_nhmex.c
index 8962e7cb21e3e..7a6855281102f 100644
--- a/arch/x86/events/intel/uncore_nhmex.c
+++ b/arch/x86/events/intel/uncore_nhmex.c
@@ -199,9 +199,10 @@ DEFINE_UNCORE_FORMAT_ATTR(counter, counter, "config:6-7");
 DEFINE_UNCORE_FORMAT_ATTR(match, match, "config1:0-63");
 DEFINE_UNCORE_FORMAT_ATTR(mask, mask, "config2:0-63");
 
-static void nhmex_uncore_msr_init_box(struct intel_uncore_box *box)
+static int nhmex_uncore_msr_init_box(struct intel_uncore_box *box)
 {
 	wrmsrq(NHMEX_U_MSR_PMON_GLOBAL_CTL, NHMEX_U_PMON_GLOBAL_EN_ALL);
+	return 0;
 }
 
 static void nhmex_uncore_msr_exit_box(struct intel_uncore_box *box)
diff --git a/arch/x86/events/intel/uncore_snb.c b/arch/x86/events/intel/uncore_snb.c
index edddd4f9ab5fc..055131c508ffa 100644
--- a/arch/x86/events/intel/uncore_snb.c
+++ b/arch/x86/events/intel/uncore_snb.c
@@ -295,12 +295,14 @@ static void snb_uncore_msr_disable_event(struct intel_uncore_box *box, struct pe
 	wrmsrq(event->hw.config_base, 0);
 }
 
-static void snb_uncore_msr_init_box(struct intel_uncore_box *box)
+static int snb_uncore_msr_init_box(struct intel_uncore_box *box)
 {
 	if (box->pmu->pmu_idx == 0) {
 		wrmsrq(SNB_UNC_PERF_GLOBAL_CTL,
 			SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL);
 	}
+
+	return 0;
 }
 
 static void snb_uncore_msr_enable_box(struct intel_uncore_box *box)
@@ -394,7 +396,7 @@ void snb_uncore_cpu_init(void)
 		snb_uncore_cbox.num_boxes = topology_num_cores_per_package();
 }
 
-static void skl_uncore_msr_init_box(struct intel_uncore_box *box)
+static int skl_uncore_msr_init_box(struct intel_uncore_box *box)
 {
 	if (box->pmu->pmu_idx == 0) {
 		wrmsrq(SKL_UNC_PERF_GLOBAL_CTL,
@@ -404,6 +406,8 @@ static void skl_uncore_msr_init_box(struct intel_uncore_box *box)
 	/* The 8th CBOX has different MSR space */
 	if (box->pmu->pmu_idx == 7)
 		__set_bit(UNCORE_BOX_FLAG_CFL8_CBOX_MSR_OFFS, &box->flags);
+
+	return 0;
 }
 
 static void skl_uncore_msr_enable_box(struct intel_uncore_box *box)
@@ -547,10 +551,12 @@ static struct intel_uncore_type *tgl_msr_uncores[] = {
 	NULL,
 };
 
-static void rkl_uncore_msr_init_box(struct intel_uncore_box *box)
+static int rkl_uncore_msr_init_box(struct intel_uncore_box *box)
 {
 	if (box->pmu->pmu_idx == 0)
 		wrmsrq(SKL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN);
+
+	return 0;
 }
 
 void tgl_uncore_cpu_init(void)
@@ -707,9 +713,10 @@ static struct intel_uncore_type mtl_uncore_hac_cbox = {
 	.format_group	= &adl_uncore_format_group,
 };
 
-static void mtl_uncore_msr_init_box(struct intel_uncore_box *box)
+static int mtl_uncore_msr_init_box(struct intel_uncore_box *box)
 {
 	wrmsrq(uncore_msr_box_ctl(box), SNB_UNC_GLOBAL_CTL_EN);
+	return 0;
 }
 
 static struct intel_uncore_ops mtl_uncore_msr_ops = {
@@ -773,10 +780,12 @@ static struct intel_uncore_type *lnl_msr_uncores[] = {
 
 #define LNL_UNC_MSR_GLOBAL_CTL			0x240e
 
-static void lnl_uncore_msr_init_box(struct intel_uncore_box *box)
+static int lnl_uncore_msr_init_box(struct intel_uncore_box *box)
 {
 	if (box->pmu->pmu_idx == 0)
 		wrmsrq(LNL_UNC_MSR_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN);
+
+	return 0;
 }
 
 static struct intel_uncore_ops lnl_uncore_msr_ops = {
@@ -874,7 +883,7 @@ static const struct attribute_group snb_uncore_imc_format_group = {
 	.attrs = snb_uncore_imc_formats_attr,
 };
 
-static void snb_uncore_imc_init_box(struct intel_uncore_box *box)
+static int snb_uncore_imc_init_box(struct intel_uncore_box *box)
 {
 	struct intel_uncore_type *type = box->pmu->type;
 	struct pci_dev *pdev = box->pci_dev;
@@ -893,10 +902,13 @@ static void snb_uncore_imc_init_box(struct intel_uncore_box *box)
 	addr &= ~(PAGE_SIZE - 1);
 
 	box->io_addr = ioremap(addr, type->mmio_map_size);
-	if (!box->io_addr)
+	if (!box->io_addr) {
 		pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name);
+		return -ENOMEM;
+	}
 
 	box->hrtimer_duration = UNCORE_SNB_IMC_HRTIMER_INTERVAL;
+	return 0;
 }
 
 static void snb_uncore_imc_enable_box(struct intel_uncore_box *box)
@@ -928,7 +940,7 @@ static int snb_uncore_imc_event_init(struct perf_event *event)
 
 	pmu = uncore_event_to_pmu(event);
 	/* no device found for this pmu */
-	if (!pmu->registered)
+	if (!uncore_pmu_available(pmu))
 		return -ENOENT;
 
 	/* Sampling not supported yet */
@@ -1532,7 +1544,7 @@ static struct pci_dev *tgl_uncore_get_mc_dev(void)
 #define TGL_UNCORE_MMIO_IMC_MEM_OFFSET		0x10000
 #define TGL_UNCORE_PCI_IMC_MAP_SIZE		0xe000
 
-static void
+static int
 uncore_get_box_mmio_addr(struct intel_uncore_box *box,
 			 unsigned int base_offset,
 			 int bar_offset, int step)
@@ -1541,19 +1553,20 @@ uncore_get_box_mmio_addr(struct intel_uncore_box *box,
 	struct intel_uncore_pmu *pmu = box->pmu;
 	struct intel_uncore_type *type = pmu->type;
 	resource_size_t addr;
+	int ret = 0;
 	u32 bar;
 
 	if (!pdev) {
 		pr_warn("perf uncore: Cannot find matched IMC device.\n");
-		return;
+		return -ENODEV;
 	}
 
 	pci_read_config_dword(pdev, bar_offset, &bar);
 	if (!(bar & BIT(0))) {
 		pr_warn("perf uncore: BAR 0x%x is disabled. Failed to map %s counters.\n",
 			bar_offset, type->name);
-		pci_dev_put(pdev);
-		return;
+		ret = -ENODEV;
+		goto out;
 	}
 	bar &= ~BIT(0);
 	addr = (resource_size_t)(bar + step * pmu->pmu_idx);
@@ -1565,23 +1578,26 @@ uncore_get_box_mmio_addr(struct intel_uncore_box *box,
 
 	addr += base_offset;
 	box->io_addr = ioremap(addr, type->mmio_map_size);
-	if (!box->io_addr)
+	if (!box->io_addr) {
+		ret = -ENOMEM;
 		pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name);
-
+	}
+out:
 	pci_dev_put(pdev);
+	return ret;
 }
 
-static void __uncore_imc_init_box(struct intel_uncore_box *box,
+static int __uncore_imc_init_box(struct intel_uncore_box *box,
 				  unsigned int base_offset)
 {
-	uncore_get_box_mmio_addr(box, base_offset,
+	return uncore_get_box_mmio_addr(box, base_offset,
 				 SNB_UNCORE_PCI_IMC_BAR_OFFSET,
 				 TGL_UNCORE_MMIO_IMC_MEM_OFFSET);
 }
 
-static void tgl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box)
+static int tgl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box)
 {
-	__uncore_imc_init_box(box, 0);
+	return __uncore_imc_init_box(box, 0);
 }
 
 static struct intel_uncore_ops tgl_uncore_imc_freerunning_ops = {
@@ -1648,13 +1664,15 @@ void tgl_uncore_mmio_init(void)
 #define ADL_UNCORE_IMC_CTL_INT			(ADL_UNCORE_IMC_CTL_RST_CTRL | \
 						ADL_UNCORE_IMC_CTL_RST_CTRS)
 
-static void adl_uncore_imc_init_box(struct intel_uncore_box *box)
+static int adl_uncore_imc_init_box(struct intel_uncore_box *box)
 {
-	__uncore_imc_init_box(box, ADL_UNCORE_IMC_BASE);
+	int ret = __uncore_imc_init_box(box, ADL_UNCORE_IMC_BASE);
 
 	/* The global control in MC1 can control both MCs. */
-	if (box->io_addr && (box->pmu->pmu_idx == 1))
+	if (!ret && (box->pmu->pmu_idx == 1))
 		writel(ADL_UNCORE_IMC_CTL_INT, box->io_addr + ADL_UNCORE_IMC_GLOBAL_CTL);
+
+	return ret;
 }
 
 static void adl_uncore_mmio_disable_box(struct intel_uncore_box *box)
@@ -1731,9 +1749,9 @@ static struct freerunning_counters adl_uncore_imc_freerunning[] = {
 	[ADL_MMIO_UNCORE_IMC_DATA_WRITE]	= { 0xA0, 0x0, 0x0, 1, 64 },
 };
 
-static void adl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box)
+static int adl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box)
 {
-	__uncore_imc_init_box(box, ADL_UNCORE_IMC_FREERUNNING_BASE);
+	return __uncore_imc_init_box(box, ADL_UNCORE_IMC_FREERUNNING_BASE);
 }
 
 static struct intel_uncore_ops adl_uncore_imc_freerunning_ops = {
@@ -1803,9 +1821,9 @@ static const struct attribute_group lnl_uncore_format_group = {
 	.attrs		= lnl_uncore_formats_attr,
 };
 
-static void lnl_uncore_hbo_init_box(struct intel_uncore_box *box)
+static int lnl_uncore_hbo_init_box(struct intel_uncore_box *box)
 {
-	uncore_get_box_mmio_addr(box, LNL_UNCORE_HBO_BASE,
+	return uncore_get_box_mmio_addr(box, LNL_UNCORE_HBO_BASE,
 				 LNL_UNCORE_PCI_SAFBAR_OFFSET,
 				 LNL_UNCORE_HBO_OFFSET);
 }
@@ -1829,14 +1847,16 @@ static struct intel_uncore_type lnl_uncore_hbo = {
 	.format_group	= &lnl_uncore_format_group,
 };
 
-static void lnl_uncore_sncu_init_box(struct intel_uncore_box *box)
+static int lnl_uncore_sncu_init_box(struct intel_uncore_box *box)
 {
-	uncore_get_box_mmio_addr(box, LNL_UNCORE_SNCU_BASE,
+	int ret = uncore_get_box_mmio_addr(box, LNL_UNCORE_SNCU_BASE,
 				 LNL_UNCORE_PCI_SAFBAR_OFFSET,
 				 0);
 
-	if (box->io_addr)
+	if (!ret)
 		writel(ADL_UNCORE_IMC_CTL_INT, box->io_addr + LNL_UNCORE_GLOBAL_CTL);
+
+	return ret;
 }
 
 static struct intel_uncore_ops lnl_uncore_sncu_ops = {
@@ -1887,13 +1907,15 @@ static struct intel_uncore_type ptl_uncore_imc = {
 	.mmio_map_size		= 0xf00,
 };
 
-static void ptl_uncore_sncu_init_box(struct intel_uncore_box *box)
+static int ptl_uncore_sncu_init_box(struct intel_uncore_box *box)
 {
-	intel_generic_uncore_mmio_init_box(box);
+	int ret = intel_generic_uncore_mmio_init_box(box);
 
 	/* Clear the global freeze bit */
 	if (box->io_addr)
 		writel(0, box->io_addr + PTL_UNCORE_GLOBAL_CTL_OFFSET);
+
+	return ret;
 }
 
 static struct intel_uncore_ops ptl_uncore_sncu_ops = {
diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c
index 334dc384b5b93..a97cd029db366 100644
--- a/arch/x86/events/intel/uncore_snbep.c
+++ b/arch/x86/events/intel/uncore_snbep.c
@@ -627,12 +627,12 @@ static u64 snbep_uncore_pci_read_counter(struct intel_uncore_box *box, struct pe
 	return count;
 }
 
-static void snbep_uncore_pci_init_box(struct intel_uncore_box *box)
+static int snbep_uncore_pci_init_box(struct intel_uncore_box *box)
 {
 	struct pci_dev *pdev = box->pci_dev;
 	int box_ctl = uncore_pci_box_ctl(box);
 
-	pci_write_config_dword(pdev, box_ctl, SNBEP_PMON_BOX_CTL_INT);
+	return pci_write_config_dword(pdev, box_ctl, SNBEP_PMON_BOX_CTL_INT);
 }
 
 static void snbep_uncore_msr_disable_box(struct intel_uncore_box *box)
@@ -680,12 +680,14 @@ static void snbep_uncore_msr_disable_event(struct intel_uncore_box *box,
 	wrmsrq(hwc->config_base, hwc->config);
 }
 
-static void snbep_uncore_msr_init_box(struct intel_uncore_box *box)
+static int snbep_uncore_msr_init_box(struct intel_uncore_box *box)
 {
 	unsigned msr = uncore_msr_box_ctl(box);
 
 	if (msr)
 		wrmsrq(msr, SNBEP_PMON_BOX_CTL_INT);
+
+	return 0;
 }
 
 static struct attribute *snbep_uncore_formats_attr[] = {
@@ -1507,18 +1509,21 @@ int snbep_uncore_pci_init(void)
 /* end of Sandy Bridge-EP uncore support */
 
 /* IvyTown uncore support */
-static void ivbep_uncore_msr_init_box(struct intel_uncore_box *box)
+static int ivbep_uncore_msr_init_box(struct intel_uncore_box *box)
 {
 	unsigned msr = uncore_msr_box_ctl(box);
 	if (msr)
 		wrmsrq(msr, IVBEP_PMON_BOX_CTL_INT);
+
+	return 0;
 }
 
-static void ivbep_uncore_pci_init_box(struct intel_uncore_box *box)
+static int ivbep_uncore_pci_init_box(struct intel_uncore_box *box)
 {
 	struct pci_dev *pdev = box->pci_dev;
 
-	pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT);
+	return pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL,
+				      IVBEP_PMON_BOX_CTL_INT);
 }
 
 #define IVBEP_UNCORE_MSR_OPS_COMMON_INIT()			\
@@ -2784,7 +2789,7 @@ static struct intel_uncore_type hswep_uncore_cbox = {
 /*
  * Write SBOX Initialization register bit by bit to avoid spurious #GPs
  */
-static void hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box)
+static int hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box)
 {
 	unsigned msr = uncore_msr_box_ctl(box);
 
@@ -2798,6 +2803,8 @@ static void hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box)
 			wrmsrq(msr, flags);
 		}
 	}
+
+	return 0;
 }
 
 static struct intel_uncore_ops hswep_uncore_sbox_msr_ops = {
@@ -4162,12 +4169,13 @@ static const struct attribute_group skx_upi_uncore_format_group = {
 	.attrs = skx_upi_uncore_formats_attr,
 };
 
-static void skx_upi_uncore_pci_init_box(struct intel_uncore_box *box)
+static int skx_upi_uncore_pci_init_box(struct intel_uncore_box *box)
 {
 	struct pci_dev *pdev = box->pci_dev;
 
 	__set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags);
-	pci_write_config_dword(pdev, SKX_UPI_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT);
+	return pci_write_config_dword(pdev, SKX_UPI_PCI_PMON_BOX_CTL,
+				      IVBEP_PMON_BOX_CTL_INT);
 }
 
 static struct intel_uncore_ops skx_upi_uncore_pci_ops = {
@@ -4323,12 +4331,13 @@ static struct intel_uncore_type skx_uncore_upi = {
 	.cleanup_mapping = skx_upi_cleanup_mapping,
 };
 
-static void skx_m2m_uncore_pci_init_box(struct intel_uncore_box *box)
+static int skx_m2m_uncore_pci_init_box(struct intel_uncore_box *box)
 {
 	struct pci_dev *pdev = box->pci_dev;
 
 	__set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags);
-	pci_write_config_dword(pdev, SKX_M2M_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT);
+	return pci_write_config_dword(pdev, SKX_M2M_PCI_PMON_BOX_CTL,
+				      IVBEP_PMON_BOX_CTL_INT);
 }
 
 static struct intel_uncore_ops skx_m2m_uncore_pci_ops = {
@@ -4831,13 +4840,13 @@ void snr_uncore_cpu_init(void)
 	uncore_msr_uncores = snr_msr_uncores;
 }
 
-static void snr_m2m_uncore_pci_init_box(struct intel_uncore_box *box)
+static int snr_m2m_uncore_pci_init_box(struct intel_uncore_box *box)
 {
 	struct pci_dev *pdev = box->pci_dev;
 	int box_ctl = uncore_pci_box_ctl(box);
 
 	__set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags);
-	pci_write_config_dword(pdev, box_ctl, IVBEP_PMON_BOX_CTL_INT);
+	return pci_write_config_dword(pdev, box_ctl, IVBEP_PMON_BOX_CTL_INT);
 }
 
 static struct intel_uncore_ops snr_m2m_uncore_pci_ops = {
@@ -5010,17 +5019,22 @@ static int snr_uncore_mmio_map(struct intel_uncore_box *box,
 	return 0;
 }
 
-static void __snr_uncore_mmio_init_box(struct intel_uncore_box *box,
+static int __snr_uncore_mmio_init_box(struct intel_uncore_box *box,
 				       unsigned int box_ctl, int mem_offset,
 				       unsigned int device)
 {
-	if (!snr_uncore_mmio_map(box, box_ctl, mem_offset, device))
+	int ret;
+
+	ret = snr_uncore_mmio_map(box, box_ctl, mem_offset, device);
+	if (!ret)
 		writel(IVBEP_PMON_BOX_CTL_INT, box->io_addr);
+
+	return ret;
 }
 
-static void snr_uncore_mmio_init_box(struct intel_uncore_box *box)
+static int snr_uncore_mmio_init_box(struct intel_uncore_box *box)
 {
-	__snr_uncore_mmio_init_box(box, uncore_mmio_box_ctl(box),
+	return __snr_uncore_mmio_init_box(box, uncore_mmio_box_ctl(box),
 				   SNR_IMC_MMIO_MEM0_OFFSET,
 				   SNR_MC_DEVICE_ID);
 }
@@ -5637,14 +5651,14 @@ int icx_uncore_pci_init(void)
 	return 0;
 }
 
-static void icx_uncore_imc_init_box(struct intel_uncore_box *box)
+static int icx_uncore_imc_init_box(struct intel_uncore_box *box)
 {
 	unsigned int box_ctl = box->pmu->type->box_ctl +
 			       box->pmu->type->mmio_offset * (box->pmu->pmu_idx % ICX_NUMBER_IMC_CHN);
 	int mem_offset = (box->pmu->pmu_idx / ICX_NUMBER_IMC_CHN) * ICX_IMC_MEM_STRIDE +
 			 SNR_IMC_MMIO_MEM0_OFFSET;
 
-	__snr_uncore_mmio_init_box(box, box_ctl, mem_offset,
+	return __snr_uncore_mmio_init_box(box, box_ctl, mem_offset,
 				   SNR_MC_DEVICE_ID);
 }
 
@@ -5701,12 +5715,12 @@ static struct uncore_event_desc icx_uncore_imc_freerunning_events[] = {
 	{ /* end: all zeroes */ },
 };
 
-static void icx_uncore_imc_freerunning_init_box(struct intel_uncore_box *box)
+static int icx_uncore_imc_freerunning_init_box(struct intel_uncore_box *box)
 {
 	int mem_offset = box->pmu->pmu_idx * ICX_IMC_MEM_STRIDE +
 			 SNR_IMC_MMIO_MEM0_OFFSET;
 
-	snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box),
+	return snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box),
 			    mem_offset, SNR_MC_DEVICE_ID);
 }
 
@@ -6003,10 +6017,10 @@ static struct intel_uncore_type spr_uncore_mdf = {
 	.name			= "mdf",
 };
 
-static void spr_uncore_mmio_offs8_init_box(struct intel_uncore_box *box)
+static int spr_uncore_mmio_offs8_init_box(struct intel_uncore_box *box)
 {
 	__set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags);
-	intel_generic_uncore_mmio_init_box(box);
+	return intel_generic_uncore_mmio_init_box(box);
 }
 
 static struct intel_uncore_ops spr_uncore_mmio_offs8_ops = {
@@ -6187,12 +6201,11 @@ static struct uncore_event_desc spr_uncore_imc_freerunning_events[] = {
 
 #define SPR_MC_DEVICE_ID	0x3251
 
-static void spr_uncore_imc_freerunning_init_box(struct intel_uncore_box *box)
+static int spr_uncore_imc_freerunning_init_box(struct intel_uncore_box *box)
 {
 	int mem_offset = box->pmu->pmu_idx * ICX_IMC_MEM_STRIDE + SNR_IMC_MMIO_MEM0_OFFSET;
-
-	snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box),
-			    mem_offset, SPR_MC_DEVICE_ID);
+	return snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box),
+				   mem_offset, SPR_MC_DEVICE_ID);
 }
 
 static struct intel_uncore_ops spr_uncore_imc_freerunning_ops = {
@@ -6881,20 +6894,24 @@ static unsigned int dmr_iio_freerunning_box_offsets[] = {
 	0x0, 0x8000, 0x18000, 0x20000
 };
 
-static void dmr_uncore_freerunning_init_box(struct intel_uncore_box *box)
+static int dmr_uncore_freerunning_init_box(struct intel_uncore_box *box)
 {
 	struct intel_uncore_type *type = box->pmu->type;
 	u64 mmio_base;
 
 	if (box->pmu->pmu_idx >= type->num_boxes)
-		return;
+		return -ENODEV;
 
 	mmio_base = DMR_IMH1_HIOP_MMIO_BASE;
 	mmio_base += dmr_iio_freerunning_box_offsets[box->pmu->pmu_idx];
 
 	box->io_addr = ioremap(mmio_base, type->mmio_map_size);
-	if (!box->io_addr)
+	if (!box->io_addr) {
 		pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name);
+		return -ENOMEM;
+	}
+
+	return 0;
 }
 
 static struct intel_uncore_ops dmr_uncore_freerunning_ops = {
diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h
index eae24bb35dc14..8c0b32ffab358 100644
--- a/arch/x86/events/perf_event.h
+++ b/arch/x86/events/perf_event.h
@@ -668,7 +668,7 @@ union perf_capabilities {
 		u64	perf_metrics:1;
 		u64	pebs_output_pt_available:1;
 		u64	pebs_timing_info:1;
-		u64	anythread_deprecated:1;
+		u64	__reserved:1;
 		u64	rdpmc_metrics_clear:1;
 	};
 	u64	capabilities;
@@ -1175,7 +1175,7 @@ DECLARE_STATIC_CALL(x86_pmu_pebs_disable_all, *x86_pmu.pebs_disable_all);
 
 static __always_inline struct x86_perf_task_context_opt *task_context_opt(void *ctx)
 {
-	if (static_cpu_has(X86_FEATURE_ARCH_LBR))
+	if (cpu_feature_enabled(X86_FEATURE_ARCH_LBR))
 		return &((struct x86_perf_task_context_arch_lbr *)ctx)->opt;
 
 	return &((struct x86_perf_task_context *)ctx)->opt;
@@ -1344,7 +1344,7 @@ static inline u64 x86_pmu_get_event_config(struct perf_event *event)
 static inline bool x86_pmu_has_rdpmc_user_disable(struct pmu *pmu)
 {
 	return !!(hybrid(pmu, config_mask) &
-		 ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE);
+		  ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE);
 }
 
 extern struct event_constraint emptyconstraint;
diff --git a/arch/x86/hyperv/hv_apic.c b/arch/x86/hyperv/hv_apic.c
index a8de503def370..95f1782d1e172 100644
--- a/arch/x86/hyperv/hv_apic.c
+++ b/arch/x86/hyperv/hv_apic.c
@@ -60,17 +60,15 @@ void hv_enable_coco_interrupt(unsigned int cpu, unsigned int vector, bool set)
 
 static u32 hv_apic_read(u32 reg)
 {
-	u32 reg_val, hi;
+	struct msr reg_val;
 
 	switch (reg) {
 	case APIC_EOI:
-		rdmsr(HV_X64_MSR_EOI, reg_val, hi);
-		(void)hi;
-		return reg_val;
+		rdmsrq(HV_X64_MSR_EOI, reg_val.q);
+		return reg_val.l;
 	case APIC_TASKPRI:
-		rdmsr(HV_X64_MSR_TPR, reg_val, hi);
-		(void)hi;
-		return reg_val;
+		rdmsrq(HV_X64_MSR_TPR, reg_val.q);
+		return reg_val.l;
 
 	default:
 		return native_apic_mem_read(reg);
diff --git a/arch/x86/include/asm/Kbuild b/arch/x86/include/asm/Kbuild
index 078fd2c0d69df..47ef8cb482e3b 100644
--- a/arch/x86/include/asm/Kbuild
+++ b/arch/x86/include/asm/Kbuild
@@ -15,3 +15,4 @@ generic-y += fprobe.h
 generic-y += mcs_spinlock.h
 generic-y += mmzone.h
 generic-y += ring_buffer.h
+generic-y += kexec_handover.h
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index 03364510d5fe2..08af86ef090a8 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -20,39 +20,9 @@
 
 #include <linux/stddef.h>
 
-/*
- * Alternative inline assembly for SMP.
- *
- * The LOCK_PREFIX macro defined here replaces the LOCK and
- * LOCK_PREFIX macros used everywhere in the source tree.
- *
- * SMP alternatives use the same data structures as the other
- * alternatives and the X86_FEATURE_UP flag to indicate the case of a
- * UP system running a SMP kernel.  The existing apply_alternatives()
- * works fine for patching a SMP kernel for UP.
- *
- * The SMP alternative tables can be kept after boot and contain both
- * UP and SMP versions of the instructions to allow switching back to
- * SMP at runtime, when hotplugging in a new CPU, which is especially
- * useful in virtualized environments.
- *
- * The very common lock prefix is handled as special case in a
- * separate table which is a pure address list without replacement ptr
- * and size information.  That keeps the table sizes small.
- */
-
 #ifdef CONFIG_SMP
-#define LOCK_PREFIX_HERE \
-		".pushsection .smp_locks,\"a\"\n"	\
-		".balign 4\n"				\
-		".long 671f - .\n" /* offset */		\
-		".popsection\n"				\
-		"671:"
-
-#define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock "
-
-#else /* ! CONFIG_SMP */
-#define LOCK_PREFIX_HERE ""
+#define LOCK_PREFIX "lock "
+#else
 #define LOCK_PREFIX ""
 #endif
 
@@ -87,7 +57,6 @@ extern s32 __retpoline_sites[], __retpoline_sites_end[];
 extern s32 __return_sites[],	__return_sites_end[];
 extern s32 __cfi_sites[],	__cfi_sites_end[];
 extern s32 __ibt_endbr_seal[],	__ibt_endbr_seal_end[];
-extern s32 __smp_locks[],	__smp_locks_end[];
 
 /*
  * Debug flag that can be tested to see whether alternative
@@ -162,26 +131,6 @@ static __always_inline bool cpu_wants_rethunk_at(void *addr)
 }
 #endif
 
-#ifdef CONFIG_SMP
-extern void alternatives_smp_module_add(struct module *mod, char *name,
-					void *locks, void *locks_end,
-					void *text, void *text_end);
-extern void alternatives_smp_module_del(struct module *mod);
-extern void alternatives_enable_smp(void);
-extern int alternatives_text_reserved(void *start, void *end);
-extern bool skip_smp_alternatives;
-#else
-static inline void alternatives_smp_module_add(struct module *mod, char *name,
-					       void *locks, void *locks_end,
-					       void *text, void *text_end) {}
-static inline void alternatives_smp_module_del(struct module *mod) {}
-static inline void alternatives_enable_smp(void) {}
-static inline int alternatives_text_reserved(void *start, void *end)
-{
-	return 0;
-}
-#endif	/* CONFIG_SMP */
-
 #define ALT_CALL_INSTR		"call BUG_func"
 
 #define alt_slen		"772b-771b"
@@ -319,18 +268,11 @@ void nop_func(void);
 
 #else /* __ASSEMBLER__ */
 
+.macro LOCK_PREFIX
 #ifdef CONFIG_SMP
-	.macro LOCK_PREFIX
-672:	lock
-	.pushsection .smp_locks,"a"
-	.balign 4
-	.long 672b - .
-	.popsection
-	.endm
-#else
-	.macro LOCK_PREFIX
-	.endm
+	lock
 #endif
+	.endm
 
 /*
  * Issue one struct alt_instr descriptor entry (need to put it into
diff --git a/arch/x86/include/asm/archrandom.h b/arch/x86/include/asm/archrandom.h
index 4c305305871b3..cb1f1e15a0b9b 100644
--- a/arch/x86/include/asm/archrandom.h
+++ b/arch/x86/include/asm/archrandom.h
@@ -45,12 +45,12 @@ static inline bool __must_check rdseed_long(unsigned long *v)
 
 static inline size_t __must_check arch_get_random_longs(unsigned long *v, size_t max_longs)
 {
-	return max_longs && static_cpu_has(X86_FEATURE_RDRAND) && rdrand_long(v) ? 1 : 0;
+	return max_longs && cpu_feature_enabled(X86_FEATURE_RDRAND) && rdrand_long(v) ? 1 : 0;
 }
 
 static inline size_t __must_check arch_get_random_seed_longs(unsigned long *v, size_t max_longs)
 {
-	return max_longs && static_cpu_has(X86_FEATURE_RDSEED) && rdseed_long(v) ? 1 : 0;
+	return max_longs && cpu_feature_enabled(X86_FEATURE_RDSEED) && rdseed_long(v) ? 1 : 0;
 }
 
 #ifndef CONFIG_UML
diff --git a/arch/x86/include/asm/cmpxchg_32.h b/arch/x86/include/asm/cmpxchg_32.h
index 1f80a62be9690..e85c221ed1c98 100644
--- a/arch/x86/include/asm/cmpxchg_32.h
+++ b/arch/x86/include/asm/cmpxchg_32.h
@@ -104,7 +104,7 @@ static __always_inline bool __try_cmpxchg64_local(volatile u64 *ptr, u64 *oldp,
 
 static __always_inline u64 arch_cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
 {
-	return __arch_cmpxchg64_emu(ptr, old, new, LOCK_PREFIX_HERE, "lock ");
+	return __arch_cmpxchg64_emu(ptr, old, new, , "lock ");
 }
 #define arch_cmpxchg64 arch_cmpxchg64
 
@@ -138,7 +138,7 @@ static __always_inline u64 arch_cmpxchg64_local(volatile u64 *ptr, u64 old, u64
 
 static __always_inline bool arch_try_cmpxchg64(volatile u64 *ptr, u64 *oldp, u64 new)
 {
-	return __arch_try_cmpxchg64_emu(ptr, oldp, new, LOCK_PREFIX_HERE, "lock ");
+	return __arch_try_cmpxchg64_emu(ptr, oldp, new, , "lock ");
 }
 #define arch_try_cmpxchg64 arch_try_cmpxchg64
 
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 90680f978d43b..b4cc5fd1ba2cf 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -69,7 +69,7 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
  * is not relevant.
  */
 #define cpu_feature_enabled(bit)	\
-	(__builtin_constant_p(bit) && DISABLED_MASK_BIT_SET(bit) ? 0 : static_cpu_has(bit))
+	(__builtin_constant_p(bit) && DISABLED_MASK_BIT_SET(bit) ? 0 : _static_cpu_has(bit))
 
 #define boot_cpu_has(bit)	cpu_has(&boot_cpu_data, bit)
 
@@ -96,7 +96,7 @@ void check_cpufeature_deps(struct cpuinfo_x86 *c);
  * it to manifest the address of boot_cpu_data in a register, fouling
  * the mainline (post-initialization) code.
  */
-static __always_inline bool _static_cpu_has(u16 bit)
+static __always_inline bool __static_cpu_has(u16 bit)
 {
 	asm goto(ALTERNATIVE_TERNARY("jmp 6f", %c[feature], "", "jmp %l[t_no]")
 		".pushsection .altinstr_aux,\"ax\"\n"
@@ -116,17 +116,17 @@ static __always_inline bool _static_cpu_has(u16 bit)
 	return false;
 }
 
-#define static_cpu_has(bit)					\
+#define _static_cpu_has(bit)					\
 (								\
 	__builtin_constant_p(boot_cpu_has(bit)) ?		\
 		boot_cpu_has(bit) :				\
-		_static_cpu_has(bit)				\
+		__static_cpu_has(bit)				\
 )
 
 #define cpu_has_bug(c, bit)		cpu_has(c, (bit))
 #define set_cpu_bug(c, bit)		set_cpu_cap(c, (bit))
 
-#define static_cpu_has_bug(bit)		static_cpu_has((bit))
+#define static_cpu_has_bug(bit)		_static_cpu_has((bit))
 #define boot_cpu_has_bug(bit)		cpu_has_bug(&boot_cpu_data, (bit))
 #define boot_cpu_set_bug(bit)		set_cpu_cap(&boot_cpu_data, (bit))
 
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 1b4a48bff18f7..ea69a77ac0528 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -78,7 +78,7 @@
 #define X86_FEATURE_ZEN6		( 3*32+ 6) /* CPU based on Zen6 microarchitecture */
 /* Free                                 ( 3*32+ 7) */
 #define X86_FEATURE_CONSTANT_TSC	( 3*32+ 8) /* "constant_tsc" TSC ticks at a constant rate */
-#define X86_FEATURE_UP			( 3*32+ 9) /* "up" SMP kernel running on UP */
+/* free: was #define X86_FEATURE_UP	( 3*32+ 9) * "up" SMP kernel running on UP */
 #define X86_FEATURE_ART			( 3*32+10) /* "art" Always running timer (ART) */
 #define X86_FEATURE_ARCH_PERFMON	( 3*32+11) /* "arch_perfmon" Intel Architectural PerfMon */
 #define X86_FEATURE_PEBS		( 3*32+12) /* "pebs" Precise-Event Based Sampling */
@@ -136,8 +136,12 @@
 #define X86_FEATURE_HYPERVISOR		( 4*32+31) /* "hypervisor" Running on a hypervisor */
 
 /* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000001, word 5 */
+#define X86_FEATURE_SM2			( 5*32+ 0) /* "sm2" SM2 algorithm */
+#define X86_FEATURE_SM2_EN		( 5*32+ 1) /* "sm2_en" SM2 enabled */
 #define X86_FEATURE_XSTORE		( 5*32+ 2) /* "rng" RNG present (xstore) */
 #define X86_FEATURE_XSTORE_EN		( 5*32+ 3) /* "rng_en" RNG enabled */
+#define X86_FEATURE_CCS			( 5*32+ 4) /* "ccs" SM3 + SM4 instructions */
+#define X86_FEATURE_CCS_EN		( 5*32+ 5) /* "ccs_en" CCS enabled */
 #define X86_FEATURE_XCRYPT		( 5*32+ 6) /* "ace" on-CPU crypto (xcrypt) */
 #define X86_FEATURE_XCRYPT_EN		( 5*32+ 7) /* "ace_en" on-CPU crypto enabled */
 #define X86_FEATURE_ACE2		( 5*32+ 8) /* "ace2" Advanced Cryptography Engine v2 */
@@ -146,6 +150,12 @@
 #define X86_FEATURE_PHE_EN		( 5*32+11) /* "phe_en" PHE enabled */
 #define X86_FEATURE_PMM			( 5*32+12) /* "pmm" PadLock Montgomery Multiplier */
 #define X86_FEATURE_PMM_EN		( 5*32+13) /* "pmm_en" PMM enabled */
+#define X86_FEATURE_RNG2		( 5*32+22) /* "rng2" RNG v2 */
+#define X86_FEATURE_RNG2_EN		( 5*32+23) /* "rng2_en" RNG2 enabled */
+#define X86_FEATURE_PHE2		( 5*32+25) /* "phe2" PadLock Hash Engine v2 */
+#define X86_FEATURE_PHE2_EN		( 5*32+26) /* "phe2_en" PHE2 enabled */
+#define X86_FEATURE_RSA			( 5*32+27) /* "rsa" Big-number arithmetic */
+#define X86_FEATURE_RSA_EN		( 5*32+28) /* "rsa_en" RSA enabled */
 
 /* More extended AMD flags: CPUID level 0x80000001, ECX, word 6 */
 #define X86_FEATURE_LAHF_LM		( 6*32+ 0) /* "lahf_lm" LAHF/SAHF in long mode */
@@ -365,6 +375,8 @@
 #define X86_FEATURE_HWP_HIGHEST_PERF_CHANGE (14*32+15) /* HWP Highest perf change */
 #define X86_FEATURE_HFI			(14*32+19) /* "hfi" Hardware Feedback Interface */
 
+#define X86_FEATURE_DPTI		(14*32+24) /* Intel Directed Package Thermal Interrupt */
+
 /* AMD SVM Feature Identification, CPUID level 0x8000000a (EDX), word 15 */
 #define X86_FEATURE_NPT			(15*32+ 0) /* "npt" Nested Page Table support */
 #define X86_FEATURE_LBRV		(15*32+ 1) /* "lbrv" LBR Virtualization support */
diff --git a/arch/x86/include/asm/debugreg.h b/arch/x86/include/asm/debugreg.h
index a2c1f2d24b645..854d82b88ff4e 100644
--- a/arch/x86/include/asm/debugreg.h
+++ b/arch/x86/include/asm/debugreg.h
@@ -129,7 +129,7 @@ static __always_inline unsigned long local_db_save(void)
 {
 	unsigned long dr7;
 
-	if (static_cpu_has(X86_FEATURE_HYPERVISOR) && !hw_breakpoint_active())
+	if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR) && !hw_breakpoint_active())
 		return 0;
 
 	get_debugreg(dr7, 7);
diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/entry-common.h
index eca24b5e07f4c..3fed46bd96856 100644
--- a/arch/x86/include/asm/entry-common.h
+++ b/arch/x86/include/asm/entry-common.h
@@ -2,7 +2,6 @@
 #ifndef _ASM_X86_ENTRY_COMMON_H
 #define _ASM_X86_ENTRY_COMMON_H
 
-#include <linux/randomize_kstack.h>
 #include <linux/user-return-notifier.h>
 
 #include <asm/nospec-branch.h>
diff --git a/arch/x86/include/asm/insn-eval.h b/arch/x86/include/asm/insn-eval.h
index 4733e9064ee55..ae05647a0afbf 100644
--- a/arch/x86/include/asm/insn-eval.h
+++ b/arch/x86/include/asm/insn-eval.h
@@ -9,6 +9,7 @@
 #include <linux/compiler.h>
 #include <linux/bug.h>
 #include <linux/err.h>
+#include <asm/insn.h>
 #include <asm/ptrace.h>
 
 #define INSN_CODE_SEG_ADDR_SZ(params) ((params >> 4) & 0xf)
@@ -46,4 +47,39 @@ enum insn_mmio_type insn_decode_mmio(struct insn *insn, int *bytes);
 
 bool insn_is_nop(struct insn *insn);
 
+/*
+ * Write @val into *@reg following the x86 rules for writes to
+ * general-purpose registers (Intel SDM Vol. 1, "General-Purpose
+ * Registers in 64-Bit Mode"): an 8- or 16-bit write leaves the rest of
+ * the register untouched, a 32-bit write zero-extends the result into
+ * the upper 32 bits, and a 64-bit write replaces the whole register.
+ *
+ * @bytes is the width of the write, not a property of the instruction:
+ * an instruction that, say, sign-extends a 32-bit immediate into a
+ * 64-bit register does a 64-bit write here.
+ *
+ * @reg need not be 8-byte aligned: KVM's instruction emulator offsets
+ * the pointer by one byte to address the high-byte registers (AH, CH,
+ * DH, BH).  Use narrow stores for the sub-word cases so the access
+ * width matches @bytes and the adjacent bytes are left alone.
+ */
+static inline void insn_assign_reg(unsigned long *reg, u64 val, int bytes)
+{
+	switch (bytes) {
+	case 1:
+		*(u8 *)reg = (u8)val;
+		break;
+	case 2:
+		*(u16 *)reg = (u16)val;
+		break;
+	case 4:
+		/* A 32-bit write zero-extends into the upper 32 bits. */
+		*reg = (u32)val;
+		break;
+	case 8:
+		*reg = val;
+		break;
+	}
+}
+
 #endif /* _ASM_X86_INSN_EVAL_H */
diff --git a/arch/x86/include/asm/kvm-x86-nested-ops.h b/arch/x86/include/asm/kvm-x86-nested-ops.h
new file mode 100644
index 0000000000000..4b1be5bcecaa4
--- /dev/null
+++ b/arch/x86/include/asm/kvm-x86-nested-ops.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#if !defined(KVM_X86_NESTED_OP) || \
+    !defined(KVM_X86_NESTED_OP_OPTIONAL) || \
+    !defined(KVM_X86_NESTED_OP_OPTIONAL_RET0)
+#error Missing one or more KVM_X86_NESTED_OP #defines
+#else
+/*
+ * KVM_X86_NESTED_OP() and KVM_X86_NESTED_OP_OPTIONAL() are used to help
+ * generate both DECLARE/DEFINE_STATIC_CALL() invocations and
+ * "static_call_update()" calls.
+ *
+ * KVM_X86_NESTED_OP_OPTIONAL() can be used for those functions that can have
+ * a NULL definition.  KVM_X86_NESTED_OP_OPTIONAL_RET0() can be used likewise
+ * to make a definition optional, but in this case the default will
+ * be __static_call_return0.
+ */
+KVM_X86_NESTED_OP(leave_nested)
+KVM_X86_NESTED_OP(is_exception_vmexit)
+KVM_X86_NESTED_OP(check_events)
+KVM_X86_NESTED_OP_OPTIONAL_RET0(has_events)
+KVM_X86_NESTED_OP(triple_fault)
+KVM_X86_NESTED_OP(get_state)
+KVM_X86_NESTED_OP(set_state)
+KVM_X86_NESTED_OP(get_nested_state_pages)
+KVM_X86_NESTED_OP_OPTIONAL_RET0(write_log_dirty)
+KVM_X86_NESTED_OP(translate_nested_gpa)
+#ifdef CONFIG_KVM_HYPERV
+KVM_X86_NESTED_OP_OPTIONAL(enable_evmcs)
+KVM_X86_NESTED_OP_OPTIONAL_RET0(get_evmcs_version)
+KVM_X86_NESTED_OP(hv_inject_synthetic_vmexit_post_tlb_flush)
+#endif
+#endif
+
+#undef KVM_X86_NESTED_OP
+#undef KVM_X86_NESTED_OP_OPTIONAL
+#undef KVM_X86_NESTED_OP_OPTIONAL_RET0
diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h
index 83dc5086138b3..736129db272ab 100644
--- a/arch/x86/include/asm/kvm-x86-ops.h
+++ b/arch/x86/include/asm/kvm-x86-ops.h
@@ -134,6 +134,7 @@ KVM_X86_OP_OPTIONAL(mem_enc_unregister_region)
 KVM_X86_OP_OPTIONAL(vm_copy_enc_context_from)
 KVM_X86_OP_OPTIONAL(vm_move_enc_context_from)
 KVM_X86_OP_OPTIONAL(guest_memory_reclaimed)
+KVM_X86_OP_OPTIONAL(reload_vmsa)
 KVM_X86_OP(get_feature_msr)
 KVM_X86_OP(check_emulate_instruction)
 KVM_X86_OP(apic_init_signal_blocked)
@@ -145,9 +146,14 @@ KVM_X86_OP(vcpu_deliver_sipi_vector)
 KVM_X86_OP_OPTIONAL_RET0(vcpu_get_apicv_inhibit_reasons);
 KVM_X86_OP_OPTIONAL(get_untagged_addr)
 KVM_X86_OP_OPTIONAL(alloc_apic_backing_page)
+#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE
 KVM_X86_OP_OPTIONAL_RET0(gmem_prepare)
-KVM_X86_OP_OPTIONAL_RET0(gmem_max_mapping_level)
+#endif
 KVM_X86_OP_OPTIONAL(gmem_invalidate)
+#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
+KVM_X86_OP_OPTIONAL(gmem_invalidate_range)
+#endif
+KVM_X86_OP_OPTIONAL_RET0(gmem_max_mapping_level)
 #endif
 
 #undef KVM_X86_OP
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 5f6c1ce9673b7..58f156ae31e75 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -78,12 +78,6 @@
 #define KVM_DIRTY_LOG_MANUAL_CAPS   (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \
 					KVM_DIRTY_LOG_INITIALLY_SET)
 
-#define KVM_BUS_LOCK_DETECTION_VALID_MODE	(KVM_BUS_LOCK_DETECTION_OFF | \
-						 KVM_BUS_LOCK_DETECTION_EXIT)
-
-#define KVM_X86_NOTIFY_VMEXIT_VALID_BITS	(KVM_X86_NOTIFY_VMEXIT_ENABLED | \
-						 KVM_X86_NOTIFY_VMEXIT_USER)
-
 /* x86-specific vcpu->requests bit members */
 #define KVM_REQ_MIGRATE_TIMER		KVM_ARCH_REQ(0)
 #define KVM_REQ_REPORT_TPR_ACCESS	KVM_ARCH_REQ(1)
@@ -128,27 +122,11 @@
 	KVM_ARCH_REQ_FLAGS(31, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
 #define KVM_REQ_HV_TLB_FLUSH \
 	KVM_ARCH_REQ_FLAGS(32, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
+#define KVM_REQ_VMSA_PAGE_RELOAD \
+	KVM_ARCH_REQ_FLAGS(33, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
 #define KVM_REQ_UPDATE_PROTECTED_GUEST_STATE \
 	KVM_ARCH_REQ_FLAGS(34, KVM_REQUEST_WAIT)
 
-#define CR0_RESERVED_BITS                                               \
-	(~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
-			  | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
-			  | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
-
-#define CR4_RESERVED_BITS                                               \
-	(~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
-			  | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE     \
-			  | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR | X86_CR4_PCIDE \
-			  | X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_FSGSBASE \
-			  | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_VMXE \
-			  | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP \
-			  | X86_CR4_LAM_SUP | X86_CR4_CET))
-
-#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
-
-
-
 #define INVALID_PAGE (~(hpa_t)0)
 #define VALID_PAGE(x) ((x) != INVALID_PAGE)
 
@@ -161,12 +139,6 @@
 #define KVM_HPAGE_MASK(x)	(~(KVM_HPAGE_SIZE(x) - 1))
 #define KVM_PAGES_PER_HPAGE(x)	(KVM_HPAGE_SIZE(x) / PAGE_SIZE)
 
-#define KVM_MEMSLOT_PAGES_TO_MMU_PAGES_RATIO 50
-#define KVM_MIN_ALLOC_MMU_PAGES 64UL
-#define KVM_MMU_HASH_SHIFT 12
-#define KVM_NUM_MMU_PAGES (1 << KVM_MMU_HASH_SHIFT)
-#define KVM_MIN_FREE_MMU_PAGES 5
-#define KVM_REFILL_PAGES 25
 #define KVM_MAX_CPUID_ENTRIES 256
 #define KVM_NR_VAR_MTRR 8
 
@@ -242,37 +214,6 @@ enum x86_intercept_stage;
 
 #define KVM_NR_DB_REGS	4
 
-#define DR6_BUS_LOCK   (1 << 11)
-#define DR6_BD		(1 << 13)
-#define DR6_BS		(1 << 14)
-#define DR6_BT		(1 << 15)
-#define DR6_RTM		(1 << 16)
-/*
- * DR6_ACTIVE_LOW combines fixed-1 and active-low bits.
- * We can regard all the bits in DR6_FIXED_1 as active_low bits;
- * they will never be 0 for now, but when they are defined
- * in the future it will require no code change.
- *
- * DR6_ACTIVE_LOW is also used as the init/reset value for DR6.
- */
-#define DR6_ACTIVE_LOW	0xffff0ff0
-#define DR6_VOLATILE	0x0001e80f
-#define DR6_FIXED_1	(DR6_ACTIVE_LOW & ~DR6_VOLATILE)
-
-#define DR7_BP_EN_MASK	0x000000ff
-#define DR7_GE		(1 << 9)
-#define DR7_GD		(1 << 13)
-#define DR7_VOLATILE	0xffff2bff
-
-#define KVM_GUESTDBG_VALID_MASK \
-	(KVM_GUESTDBG_ENABLE | \
-	KVM_GUESTDBG_SINGLESTEP | \
-	KVM_GUESTDBG_USE_HW_BP | \
-	KVM_GUESTDBG_USE_SW_BP | \
-	KVM_GUESTDBG_INJECT_BP | \
-	KVM_GUESTDBG_INJECT_DB | \
-	KVM_GUESTDBG_BLOCKIRQ)
-
 #define PFERR_PRESENT_MASK	BIT(0)
 #define PFERR_WRITE_MASK	BIT(1)
 #define PFERR_USER_MASK		BIT(2)
@@ -302,19 +243,58 @@ enum x86_intercept_stage;
 #define PFERR_PRIVATE_ACCESS   BIT_ULL(49)
 #define PFERR_SYNTHETIC_MASK   (PFERR_IMPLICIT_ACCESS | PFERR_PRIVATE_ACCESS)
 
-/* apic attention bits */
-#define KVM_APIC_CHECK_VAPIC	0
-/*
- * The following bit is set with PV-EOI, unset on EOI.
- * We detect PV-EOI changes by guest by comparing
- * this bit with PV-EOI in guest memory.
- * See the implementation in apic_update_pv_eoi.
- */
-#define KVM_APIC_PV_EOI_PENDING	1
-
 struct kvm_kernel_irqfd;
 struct kvm_kernel_irq_routing_entry;
 
+struct kvm_apic_map;
+
+struct kvm_x86_msr_filter;
+struct kvm_x86_pmu_event_filter;
+
+struct kvm_caps {
+	/* control of guest tsc rate supported? */
+	bool has_tsc_control;
+	/* maximum supported tsc_khz for guests */
+	u32  max_guest_tsc_khz;
+	/* number of bits of the fractional part of the TSC scaling ratio */
+	u8   tsc_scaling_ratio_frac_bits;
+	/* maximum allowed value of TSC scaling ratio */
+	u64  max_tsc_scaling_ratio;
+	/* 1ull << kvm_caps.tsc_scaling_ratio_frac_bits */
+	u64  default_tsc_scaling_ratio;
+	/* bus lock detection supported? */
+	bool has_bus_lock_exit;
+	/* notify VM exit supported? */
+	bool has_notify_vmexit;
+	/* bit mask of VM types */
+	u32 supported_vm_types;
+
+	u64 supported_mce_cap;
+	u64 supported_xcr0;
+	u64 supported_xss;
+	u64 supported_perf_cap;
+
+	u64 supported_quirks;
+	u64 inapplicable_quirks;
+};
+extern struct kvm_caps kvm_caps;
+
+struct kvm_host_values {
+	/*
+	 * The host's raw MAXPHYADDR, i.e. the number of non-reserved physical
+	 * address bits irrespective of features that repurpose legal bits,
+	 * e.g. MKTME.
+	 */
+	u8 maxphyaddr;
+
+	u64 efer;
+	u64 xcr0;
+	u64 xss;
+	u64 s_cet;
+	u64 arch_capabilities;
+};
+extern struct kvm_host_values kvm_host;
+
 /*
  * kvm_mmu_page_role tracks the properties of a shadow page (where shadow page
  * also includes TDP pages) to determine whether or not a page can be used in
@@ -452,9 +432,24 @@ struct kvm_pio_request {
 
 #define PT64_ROOT_MAX_LEVEL 5
 
-struct rsvd_bits_validate {
+struct kvm_page_format {
 	u64 rsvd_bits_mask[2][PT64_ROOT_MAX_LEVEL];
 	u64 bad_mt_xwr;
+
+	/*
+	* The pkru_mask indicates if protection key checks are needed.  It
+	* consists of 16 domains indexed by page fault error code bits [4:1],
+	* with PFEC.RSVD replaced by ACC_USER_MASK from the page tables.
+	* Each domain has 2 bits which are ANDed with AD and WD from PKRU.
+	*/
+	u32 pkru_mask;
+
+	/*
+	 * Bitmap; bit set = permission fault
+	 * Array index: page fault error code [4:1]
+	 * Bit index: pte permissions in ACC_* format
+	 */
+	u16 permissions[16];
 };
 
 struct kvm_mmu_root_info {
@@ -478,43 +473,35 @@ struct kvm_page_fault;
 
 /*
  * x86 supports 4 paging modes (5-level 64-bit, 4-level 64-bit, 3-level 32-bit,
- * and 2-level 32-bit).  The kvm_mmu structure abstracts the details of the
+ * and 2-level 32-bit).  The kvm_pagewalk structure abstracts the details of the
  * current mmu mode.
  */
-struct kvm_mmu {
+struct kvm_pagewalk {
 	unsigned long (*get_guest_pgd)(struct kvm_vcpu *vcpu);
 	u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
-	int (*page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
 	void (*inject_page_fault)(struct kvm_vcpu *vcpu,
 				  struct x86_exception *fault,
 				  bool from_hardware);
-	gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
+	gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
 			    gpa_t gva_or_gpa, u64 access,
 			    struct x86_exception *exception);
+
+	union kvm_cpu_role cpu_role;
+	struct kvm_page_format fmt;
+};
+
+struct kvm_mmu {
+	int (*page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
 	int (*sync_spte)(struct kvm_vcpu *vcpu,
 			 struct kvm_mmu_page *sp, int i);
+	struct kvm_pagewalk *w;
+
 	struct kvm_mmu_root_info root;
 	hpa_t mirror_root_hpa;
-	union kvm_cpu_role cpu_role;
 	union kvm_mmu_page_role root_role;
 
-	/*
-	* The pkru_mask indicates if protection key checks are needed.  It
-	* consists of 16 domains indexed by page fault error code bits [4:1],
-	* with PFEC.RSVD replaced by ACC_USER_MASK from the page tables.
-	* Each domain has 2 bits which are ANDed with AD and WD from PKRU.
-	*/
-	u32 pkru_mask;
-
 	struct kvm_mmu_root_info prev_roots[KVM_MMU_NUM_PREV_ROOTS];
 
-	/*
-	 * Bitmap; bit set = permission fault
-	 * Byte index: page fault error code [4:1]
-	 * Bit index: pte permissions in ACC_* format
-	 */
-	u16 permissions[16];
-
 	u64 *pae_root;
 	u64 *pml4_root;
 	u64 *pml5_root;
@@ -524,8 +511,7 @@ struct kvm_mmu {
 	 * bits include not only hardware reserved bits but also
 	 * the bits spte never used.
 	 */
-	struct rsvd_bits_validate shadow_zero_check;
-	struct rsvd_bits_validate guest_rsvd_check;
+	struct kvm_page_format fmt;
 };
 
 enum pmc_type {
@@ -664,95 +650,6 @@ struct kvm_mtrr {
 	u64 deftype;
 };
 
-/* Hyper-V SynIC timer */
-struct kvm_vcpu_hv_stimer {
-	struct hrtimer timer;
-	int index;
-	union hv_stimer_config config;
-	u64 count;
-	u64 exp_time;
-	struct hv_message msg;
-	bool msg_pending;
-};
-
-/* Hyper-V synthetic interrupt controller (SynIC)*/
-struct kvm_vcpu_hv_synic {
-	u64 version;
-	u64 control;
-	u64 msg_page;
-	u64 evt_page;
-	atomic64_t sint[HV_SYNIC_SINT_COUNT];
-	atomic_t sint_to_gsi[HV_SYNIC_SINT_COUNT];
-	DECLARE_BITMAP(auto_eoi_bitmap, 256);
-	DECLARE_BITMAP(vec_bitmap, 256);
-	bool active;
-	bool dont_zero_synic_pages;
-};
-
-/* The maximum number of entries on the TLB flush fifo. */
-#define KVM_HV_TLB_FLUSH_FIFO_SIZE (16)
-/*
- * Note: the following 'magic' entry is made up by KVM to avoid putting
- * anything besides GVA on the TLB flush fifo. It is theoretically possible
- * to observe a request to flush 4095 PFNs starting from 0xfffffffffffff000
- * which will look identical. KVM's action to 'flush everything' instead of
- * flushing these particular addresses is, however, fully legitimate as
- * flushing more than requested is always OK.
- */
-#define KVM_HV_TLB_FLUSHALL_ENTRY  ((u64)-1)
-
-enum hv_tlb_flush_fifos {
-	HV_L1_TLB_FLUSH_FIFO,
-	HV_L2_TLB_FLUSH_FIFO,
-	HV_NR_TLB_FLUSH_FIFOS,
-};
-
-struct kvm_vcpu_hv_tlb_flush_fifo {
-	spinlock_t write_lock;
-	DECLARE_KFIFO(entries, u64, KVM_HV_TLB_FLUSH_FIFO_SIZE);
-};
-
-/* Hyper-V per vcpu emulation context */
-struct kvm_vcpu_hv {
-	struct kvm_vcpu *vcpu;
-	u32 vp_index;
-	u64 hv_vapic;
-	s64 runtime_offset;
-	struct kvm_vcpu_hv_synic synic;
-	struct kvm_hyperv_exit exit;
-	struct kvm_vcpu_hv_stimer stimer[HV_SYNIC_STIMER_COUNT];
-	DECLARE_BITMAP(stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT);
-	bool enforce_cpuid;
-	struct {
-		u32 features_eax; /* HYPERV_CPUID_FEATURES.EAX */
-		u32 features_ebx; /* HYPERV_CPUID_FEATURES.EBX */
-		u32 features_edx; /* HYPERV_CPUID_FEATURES.EDX */
-		u32 enlightenments_eax; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EAX */
-		u32 enlightenments_ebx; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EBX */
-		u32 syndbg_cap_eax; /* HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES.EAX */
-		u32 nested_eax; /* HYPERV_CPUID_NESTED_FEATURES.EAX */
-		u32 nested_ebx; /* HYPERV_CPUID_NESTED_FEATURES.EBX */
-	} cpuid_cache;
-
-	struct kvm_vcpu_hv_tlb_flush_fifo tlb_flush_fifo[HV_NR_TLB_FLUSH_FIFOS];
-
-	/*
-	 * Preallocated buffers for handling hypercalls that pass sparse vCPU
-	 * sets (for high vCPU counts, they're too large to comfortably fit on
-	 * the stack).
-	 */
-	u64 sparse_banks[HV_MAX_SPARSE_VCPU_BANKS];
-	DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS);
-
-	struct hv_vp_assist_page vp_assist_page;
-
-	struct {
-		u64 pa_page_gpa;
-		u64 vm_id;
-		u32 vp_id;
-	} nested;
-};
-
 struct kvm_hypervisor_cpuid {
 	u32 base;
 	u32 limit;
@@ -783,6 +680,8 @@ struct kvm_vcpu_xen {
 };
 #endif
 
+struct kvm_vcpu_hv;
+
 struct kvm_queued_exception {
 	bool pending;
 	bool injected;
@@ -866,24 +765,14 @@ struct kvm_vcpu_arch {
 	/* Non-nested MMU for L1 */
 	struct kvm_mmu root_mmu;
 
-	/* L1 MMU when running nested */
+	/* L1 TDP when running nested */
 	struct kvm_mmu guest_mmu;
+	struct kvm_pagewalk ngpa_walk;
 
 	/*
-	 * Paging state of an L2 guest (used for nested npt)
-	 *
-	 * This context will save all necessary information to walk page tables
-	 * of an L2 guest. This context is only initialized for page table
-	 * walking and not for faulting since we never handle l2 page faults on
-	 * the host.
-	 */
-	struct kvm_mmu nested_mmu;
-
-	/*
-	 * Pointer to the mmu context currently used for
-	 * gva_to_gpa translations.
+	 * Pagewalk context used for gva_to_gpa translations.
 	 */
-	struct kvm_mmu *walk_mmu;
+	struct kvm_pagewalk gva_walk;
 
 	u64 pdptrs[4]; /* pae */
 
@@ -1150,39 +1039,6 @@ struct kvm_arch_memory_slot {
 	unsigned short *gfn_write_track;
 };
 
-/*
- * Track the mode of the optimized logical map, as the rules for decoding the
- * destination vary per mode.  Enabling the optimized logical map requires all
- * software-enabled local APIs to be in the same mode, each addressable APIC to
- * be mapped to only one MDA, and each MDA to map to at most one APIC.
- */
-enum kvm_apic_logical_mode {
-	/* All local APICs are software disabled. */
-	KVM_APIC_MODE_SW_DISABLED,
-	/* All software enabled local APICs in xAPIC cluster addressing mode. */
-	KVM_APIC_MODE_XAPIC_CLUSTER,
-	/* All software enabled local APICs in xAPIC flat addressing mode. */
-	KVM_APIC_MODE_XAPIC_FLAT,
-	/* All software enabled local APICs in x2APIC mode. */
-	KVM_APIC_MODE_X2APIC,
-	/*
-	 * Optimized map disabled, e.g. not all local APICs in the same logical
-	 * mode, same logical ID assigned to multiple APICs, etc.
-	 */
-	KVM_APIC_MODE_MAP_DISABLED,
-};
-
-struct kvm_apic_map {
-	struct rcu_head rcu;
-	enum kvm_apic_logical_mode logical_mode;
-	u32 max_apic_id;
-	union {
-		struct kvm_lapic *xapic_flat_map[8];
-		struct kvm_lapic *xapic_cluster_map[16][4];
-	};
-	struct kvm_lapic *phys_map[];
-};
-
 /* Hyper-V synthetic debugger (SynDbg)*/
 struct kvm_hv_syndbg {
 	struct {
@@ -1246,13 +1102,6 @@ struct kvm_hv {
 };
 #endif
 
-struct msr_bitmap_range {
-	u32 flags;
-	u32 nmsrs;
-	u32 base;
-	unsigned long *bitmap;
-};
-
 #ifdef CONFIG_KVM_XEN
 /* Xen emulation context */
 struct kvm_xen {
@@ -1283,132 +1132,6 @@ enum kvm_suppress_eoi_broadcast_mode {
 	KVM_SUPPRESS_EOI_BROADCAST_DISABLED /* Disable Suppress EOI broadcast */
 };
 
-struct kvm_x86_msr_filter {
-	u8 count;
-	bool default_allow:1;
-	struct msr_bitmap_range ranges[16];
-};
-
-struct kvm_x86_pmu_event_filter {
-	__u32 action;
-	__u32 nevents;
-	__u32 fixed_counter_bitmap;
-	__u32 flags;
-	__u32 nr_includes;
-	__u32 nr_excludes;
-	__u64 *includes;
-	__u64 *excludes;
-	__u64 events[] __counted_by(nevents);
-};
-
-enum kvm_apicv_inhibit {
-
-	/********************************************************************/
-	/* INHIBITs that are relevant to both Intel's APICv and AMD's AVIC. */
-	/********************************************************************/
-
-	/*
-	 * APIC acceleration is disabled by a module parameter
-	 * and/or not supported in hardware.
-	 */
-	APICV_INHIBIT_REASON_DISABLED,
-
-	/*
-	 * APIC acceleration is inhibited because AutoEOI feature is
-	 * being used by a HyperV guest.
-	 */
-	APICV_INHIBIT_REASON_HYPERV,
-
-	/*
-	 * APIC acceleration is inhibited because the userspace didn't yet
-	 * enable the kernel/split irqchip.
-	 */
-	APICV_INHIBIT_REASON_ABSENT,
-
-	/* APIC acceleration is inhibited because KVM_GUESTDBG_BLOCKIRQ
-	 * (out of band, debug measure of blocking all interrupts on this vCPU)
-	 * was enabled, to avoid AVIC/APICv bypassing it.
-	 */
-	APICV_INHIBIT_REASON_BLOCKIRQ,
-
-	/*
-	 * APICv is disabled because not all vCPUs have a 1:1 mapping between
-	 * APIC ID and vCPU, _and_ KVM is not applying its x2APIC hotplug hack.
-	 */
-	APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED,
-
-	/*
-	 * For simplicity, the APIC acceleration is inhibited
-	 * first time either APIC ID or APIC base are changed by the guest
-	 * from their reset values.
-	 */
-	APICV_INHIBIT_REASON_APIC_ID_MODIFIED,
-	APICV_INHIBIT_REASON_APIC_BASE_MODIFIED,
-
-	/******************************************************/
-	/* INHIBITs that are relevant only to the AMD's AVIC. */
-	/******************************************************/
-
-	/*
-	 * AVIC is inhibited on a vCPU because it runs a nested guest.
-	 *
-	 * This is needed because unlike APICv, the peers of this vCPU
-	 * cannot use the doorbell mechanism to signal interrupts via AVIC when
-	 * a vCPU runs nested.
-	 */
-	APICV_INHIBIT_REASON_NESTED,
-
-	/*
-	 * On SVM, the wait for the IRQ window is implemented with pending vIRQ,
-	 * which cannot be injected when the AVIC is enabled, thus AVIC
-	 * is inhibited while KVM waits for IRQ window.
-	 */
-	APICV_INHIBIT_REASON_IRQWIN,
-
-	/*
-	 * PIT (i8254) 're-inject' mode, relies on EOI intercept,
-	 * which AVIC doesn't support for edge triggered interrupts.
-	 */
-	APICV_INHIBIT_REASON_PIT_REINJ,
-
-	/*
-	 * AVIC is disabled because SEV doesn't support it.
-	 */
-	APICV_INHIBIT_REASON_SEV,
-
-	/*
-	 * AVIC is disabled because not all vCPUs with a valid LDR have a 1:1
-	 * mapping between logical ID and vCPU.
-	 */
-	APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED,
-
-	/*
-	 * AVIC is disabled because the vCPU's APIC ID is beyond the max
-	 * supported by AVIC/x2AVIC, i.e. the vCPU is unaddressable.
-	 */
-	APICV_INHIBIT_REASON_PHYSICAL_ID_TOO_BIG,
-
-	NR_APICV_INHIBIT_REASONS,
-};
-
-#define __APICV_INHIBIT_REASON(reason)			\
-	{ BIT(APICV_INHIBIT_REASON_##reason), #reason }
-
-#define APICV_INHIBIT_REASONS				\
-	__APICV_INHIBIT_REASON(DISABLED),		\
-	__APICV_INHIBIT_REASON(HYPERV),			\
-	__APICV_INHIBIT_REASON(ABSENT),			\
-	__APICV_INHIBIT_REASON(BLOCKIRQ),		\
-	__APICV_INHIBIT_REASON(PHYSICAL_ID_ALIASED),	\
-	__APICV_INHIBIT_REASON(APIC_ID_MODIFIED),	\
-	__APICV_INHIBIT_REASON(APIC_BASE_MODIFIED),	\
-	__APICV_INHIBIT_REASON(NESTED),			\
-	__APICV_INHIBIT_REASON(IRQWIN),			\
-	__APICV_INHIBIT_REASON(PIT_REINJ),		\
-	__APICV_INHIBIT_REASON(SEV),			\
-	__APICV_INHIBIT_REASON(LOGICAL_ID_ALIASED),	\
-	__APICV_INHIBIT_REASON(PHYSICAL_ID_TOO_BIG)
-
 struct kvm_possible_nx_huge_pages {
 	/*
 	 * A list of kvm_mmu_page structs that, if zapped, could possibly be
@@ -1771,11 +1494,6 @@ struct kvm_lapic_irq {
 	bool msi_redir_hint;
 };
 
-static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical)
-{
-	return dest_mode_logical ? APIC_DEST_LOGICAL : APIC_DEST_PHYSICAL;
-}
-
 enum kvm_x86_run_flags {
 	KVM_RUN_FORCE_IMMEDIATE_EXIT	= BIT(0),
 	KVM_RUN_LOAD_GUEST_DR6		= BIT(1),
@@ -1949,8 +1667,6 @@ struct kvm_x86_ops {
 
 	void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu);
 
-	const struct kvm_x86_nested_ops *nested_ops;
-
 	void (*vcpu_blocking)(struct kvm_vcpu *vcpu);
 	void (*vcpu_unblocking)(struct kvm_vcpu *vcpu);
 
@@ -1985,6 +1701,7 @@ struct kvm_x86_ops {
 	int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd);
 	int (*vm_move_enc_context_from)(struct kvm *kvm, unsigned int source_fd);
 	void (*guest_memory_reclaimed)(struct kvm *kvm);
+	void (*reload_vmsa)(struct kvm_vcpu *vcpu);
 
 	int (*get_feature_msr)(u32 msr, u64 *data);
 
@@ -2007,12 +1724,19 @@ struct kvm_x86_ops {
 
 	gva_t (*get_untagged_addr)(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags);
 	void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu);
+#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE
 	int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order);
+#endif
 	void (*gmem_invalidate)(kvm_pfn_t start, kvm_pfn_t end);
+#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
+	void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range);
+#endif
 	int (*gmem_max_mapping_level)(struct kvm *kvm, kvm_pfn_t pfn, bool is_private);
 };
 
 struct kvm_x86_nested_ops {
+	bool enabled;
+
 	void (*leave_nested)(struct kvm_vcpu *vcpu);
 	bool (*is_exception_vmexit)(struct kvm_vcpu *vcpu, u8 vector,
 				    u32 error_code);
@@ -2044,6 +1768,7 @@ struct kvm_x86_init_ops {
 
 	struct kvm_x86_ops *runtime_ops;
 	struct kvm_pmu_ops *pmu_ops;
+	struct kvm_x86_nested_ops *nested_ops;
 };
 
 struct kvm_arch_async_pf {
@@ -2054,12 +1779,12 @@ struct kvm_arch_async_pf {
 	u64 error_code;
 };
 
-extern u32 __read_mostly kvm_nr_uret_msrs;
 extern bool __read_mostly allow_smaller_maxphyaddr;
 extern bool __read_mostly enable_apicv;
 extern bool __read_mostly enable_ipiv;
 extern bool __read_mostly enable_device_posted_irqs;
 extern struct kvm_x86_ops kvm_x86_ops;
+extern struct kvm_x86_nested_ops kvm_nested_ops __read_mostly;
 
 #define kvm_x86_call(func) static_call(kvm_x86_##func)
 
@@ -2069,8 +1794,13 @@ extern struct kvm_x86_ops kvm_x86_ops;
 #define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP
 #include <asm/kvm-x86-ops.h>
 
-int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops);
-void kvm_x86_vendor_exit(void);
+#define kvm_nested_call(func) static_call(kvm_x86_nested_##func)
+
+#define KVM_X86_NESTED_OP(func) \
+	DECLARE_STATIC_CALL(kvm_x86_nested_##func, *(((struct kvm_x86_nested_ops *)0)->func));
+#define KVM_X86_NESTED_OP_OPTIONAL KVM_X86_NESTED_OP
+#define KVM_X86_NESTED_OP_OPTIONAL_RET0 KVM_X86_NESTED_OP
+#include <asm/kvm-x86-nested-ops.h>
 
 #define __KVM_HAVE_ARCH_VM_ALLOC
 static inline struct kvm *kvm_arch_alloc_vm(void)
@@ -2114,304 +1844,12 @@ enum kvm_intr_type {
 	((vcpu) && (vcpu)->arch.handling_intr_from_guest && \
 	 (!!in_nmi() == ((vcpu)->arch.handling_intr_from_guest == KVM_HANDLING_NMI)))
 
-void __init kvm_mmu_x86_module_init(void);
-int kvm_mmu_vendor_module_init(void);
-void kvm_mmu_vendor_module_exit(void);
-
-void kvm_mmu_destroy(struct kvm_vcpu *vcpu);
-int kvm_mmu_create(struct kvm_vcpu *vcpu);
-int kvm_mmu_init_vm(struct kvm *kvm);
-void kvm_mmu_uninit_vm(struct kvm *kvm);
-
-void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm,
-					    struct kvm_memory_slot *slot);
-
-void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu);
-void kvm_mmu_reset_context(struct kvm_vcpu *vcpu);
-void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
-				      const struct kvm_memory_slot *memslot,
-				      int start_level);
-void kvm_mmu_slot_try_split_huge_pages(struct kvm *kvm,
-				       const struct kvm_memory_slot *memslot,
-				       int target_level);
-void kvm_mmu_try_split_huge_pages(struct kvm *kvm,
-				  const struct kvm_memory_slot *memslot,
-				  u64 start, u64 end,
-				  int target_level);
-void kvm_mmu_recover_huge_pages(struct kvm *kvm,
-				const struct kvm_memory_slot *memslot);
-void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
-				   const struct kvm_memory_slot *memslot);
-void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen);
-void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long kvm_nr_mmu_pages);
-void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end);
-
-int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3);
-
-extern bool tdp_enabled;
-
-/*
- * EMULTYPE_NO_DECODE - Set when re-emulating an instruction (after completing
- *			userspace I/O) to indicate that the emulation context
- *			should be reused as is, i.e. skip initialization of
- *			emulation context, instruction fetch and decode.
- *
- * EMULTYPE_TRAP_UD - Set when emulating an intercepted #UD from hardware.
- *		      Indicates that only select instructions (tagged with
- *		      EmulateOnUD) should be emulated (to minimize the emulator
- *		      attack surface).  See also EMULTYPE_TRAP_UD_FORCED.
- *
- * EMULTYPE_SKIP - Set when emulating solely to skip an instruction, i.e. to
- *		   decode the instruction length.  For use *only* by
- *		   kvm_x86_ops.skip_emulated_instruction() implementations if
- *		   EMULTYPE_COMPLETE_USER_EXIT is not set.
- *
- * EMULTYPE_ALLOW_RETRY_PF - Set when the emulator should resume the guest to
- *			     retry native execution under certain conditions,
- *			     Can only be set in conjunction with EMULTYPE_PF.
- *
- * EMULTYPE_TRAP_UD_FORCED - Set when emulating an intercepted #UD that was
- *			     triggered by KVM's magic "force emulation" prefix,
- *			     which is opt in via module param (off by default).
- *			     Bypasses EmulateOnUD restriction despite emulating
- *			     due to an intercepted #UD (see EMULTYPE_TRAP_UD).
- *			     Used to test the full emulator from userspace.
- *
- * EMULTYPE_VMWARE_GP - Set when emulating an intercepted #GP for VMware
- *			backdoor emulation, which is opt in via module param.
- *			VMware backdoor emulation handles select instructions
- *			and reinjects the #GP for all other cases.
- *
- * EMULTYPE_PF - Set when an intercepted #PF triggers the emulation, in which case
- *		 the CR2/GPA value pass on the stack is valid.
- *
- * EMULTYPE_COMPLETE_USER_EXIT - Set when the emulator should update interruptibility
- *				 state and inject single-step #DBs after skipping
- *				 an instruction (after completing userspace I/O).
- *
- * EMULTYPE_WRITE_PF_TO_SP - Set when emulating an intercepted page fault that
- *			     is attempting to write a gfn that contains one or
- *			     more of the PTEs used to translate the write itself,
- *			     and the owning page table is being shadowed by KVM.
- *			     If emulation of the faulting instruction fails and
- *			     this flag is set, KVM will exit to userspace instead
- *			     of retrying emulation as KVM cannot make forward
- *			     progress.
- *
- *			     If emulation fails for a write to guest page tables,
- *			     KVM unprotects (zaps) the shadow page for the target
- *			     gfn and resumes the guest to retry the non-emulatable
- *			     instruction (on hardware).  Unprotecting the gfn
- *			     doesn't allow forward progress for a self-changing
- *			     access because doing so also zaps the translation for
- *			     the gfn, i.e. retrying the instruction will hit a
- *			     !PRESENT fault, which results in a new shadow page
- *			     and sends KVM back to square one.
- *
- * EMULTYPE_SKIP_SOFT_INT - Set in combination with EMULTYPE_SKIP to only skip
- *                          an instruction if it could generate a given software
- *                          interrupt, which must be encoded via
- *                          EMULTYPE_SET_SOFT_INT_VECTOR().
- */
-#define EMULTYPE_NO_DECODE	    (1 << 0)
-#define EMULTYPE_TRAP_UD	    (1 << 1)
-#define EMULTYPE_SKIP		    (1 << 2)
-#define EMULTYPE_ALLOW_RETRY_PF	    (1 << 3)
-#define EMULTYPE_TRAP_UD_FORCED	    (1 << 4)
-#define EMULTYPE_VMWARE_GP	    (1 << 5)
-#define EMULTYPE_PF		    (1 << 6)
-#define EMULTYPE_COMPLETE_USER_EXIT (1 << 7)
-#define EMULTYPE_WRITE_PF_TO_SP	    (1 << 8)
-#define EMULTYPE_SKIP_SOFT_INT	    (1 << 9)
-
-#define EMULTYPE_SET_SOFT_INT_VECTOR(v)	((u32)((v) & 0xff) << 16)
-#define EMULTYPE_GET_SOFT_INT_VECTOR(e)	(((e) >> 16) & 0xff)
-
-static inline bool kvm_can_emulate_event_vectoring(int emul_type)
-{
-	return !(emul_type & EMULTYPE_PF);
-}
-
-int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type);
-int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
-					void *insn, int insn_len);
-void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu,
-					  u64 *data, u8 ndata);
-void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu);
-
-void kvm_prepare_event_vectoring_exit(struct kvm_vcpu *vcpu, gpa_t gpa);
-void kvm_prepare_unexpected_reason_exit(struct kvm_vcpu *vcpu, u64 exit_reason);
-
-void kvm_enable_efer_bits(u64);
-bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer);
-int kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data);
-int kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data);
-int __kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data);
-int __kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data);
-int kvm_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data);
-int kvm_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data);
-int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu);
-int kvm_emulate_rdmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg);
-int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu);
-int kvm_emulate_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg);
-int kvm_emulate_as_nop(struct kvm_vcpu *vcpu);
-int kvm_emulate_invd(struct kvm_vcpu *vcpu);
-int kvm_emulate_mwait(struct kvm_vcpu *vcpu);
-int kvm_handle_invalid_op(struct kvm_vcpu *vcpu);
-int kvm_emulate_monitor(struct kvm_vcpu *vcpu);
-
-int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in);
-int kvm_emulate_cpuid(struct kvm_vcpu *vcpu);
-int kvm_emulate_halt(struct kvm_vcpu *vcpu);
-int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu);
-int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu);
-int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu);
-
-void kvm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
-void kvm_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
-void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector);
-
-int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
-		    int reason, bool has_error_code, u32 error_code);
-
-void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0);
-void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4);
-int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
-int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3);
-int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
-int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8);
-int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val);
-unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr);
-unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu);
-void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw);
-int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr);
-int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu);
-
-int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);
-int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);
-
-unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu);
-void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
-int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu);
-
-void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr);
-void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
-void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr, unsigned long payload);
-void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned int nr,
-			   bool has_error_code, u32 error_code);
-void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault,
-			   bool from_hardware);
-void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
-				      struct x86_exception *fault,
-				      bool from_hardware);
-
-static inline void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
-						  struct x86_exception *fault)
-{
-	__kvm_inject_emulated_page_fault(vcpu, fault, false);
-}
-
-bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr);
-
-static inline int __kvm_irq_line_state(unsigned long *irq_state,
-				       int irq_source_id, int level)
-{
-	/* Logical OR for level trig interrupt */
-	if (level)
-		__set_bit(irq_source_id, irq_state);
-	else
-		__clear_bit(irq_source_id, irq_state);
-
-	return !!(*irq_state);
-}
-
-void kvm_inject_nmi(struct kvm_vcpu *vcpu);
-int kvm_get_nr_pending_nmis(struct kvm_vcpu *vcpu);
-
-void kvm_update_dr7(struct kvm_vcpu *vcpu);
-
-bool __kvm_mmu_unprotect_gfn_and_retry(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
-				       bool always_retry);
-
-static inline bool kvm_mmu_unprotect_gfn_and_retry(struct kvm_vcpu *vcpu,
-						   gpa_t cr2_or_gpa)
-{
-	return __kvm_mmu_unprotect_gfn_and_retry(vcpu, cr2_or_gpa, false);
-}
-
-void kvm_mmu_free_roots(struct kvm *kvm, struct kvm_mmu *mmu,
-			ulong roots_to_free);
-void kvm_mmu_free_guest_mode_roots(struct kvm *kvm, struct kvm_mmu *mmu);
-gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
-			      struct x86_exception *exception);
-gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
-			       struct x86_exception *exception);
-gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
-				struct x86_exception *exception);
-
-bool kvm_apicv_activated(struct kvm *kvm);
-bool kvm_vcpu_apicv_activated(struct kvm_vcpu *vcpu);
-void __kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu);
-void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
-				      enum kvm_apicv_inhibit reason, bool set);
-void kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
-				    enum kvm_apicv_inhibit reason, bool set);
-
-static inline void kvm_set_apicv_inhibit(struct kvm *kvm,
-					 enum kvm_apicv_inhibit reason)
-{
-	kvm_set_or_clear_apicv_inhibit(kvm, reason, true);
-}
-
-static inline void kvm_clear_apicv_inhibit(struct kvm *kvm,
-					   enum kvm_apicv_inhibit reason)
-{
-	kvm_set_or_clear_apicv_inhibit(kvm, reason, false);
-}
-
-void kvm_inc_or_dec_irq_window_inhibit(struct kvm *kvm, bool inc);
-
-static inline void kvm_inc_apicv_irq_window_req(struct kvm *kvm)
-{
-	kvm_inc_or_dec_irq_window_inhibit(kvm, true);
-}
-
-static inline void kvm_dec_apicv_irq_window_req(struct kvm *kvm)
-{
-	kvm_inc_or_dec_irq_window_inhibit(kvm, false);
-}
-
-int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
-		       void *insn, int insn_len);
-void kvm_mmu_print_sptes(struct kvm_vcpu *vcpu, gpa_t gpa, const char *msg);
-void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
-void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
-			     u64 addr, unsigned long roots);
-void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid);
-void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd);
-
-void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level,
-		       int tdp_max_root_level, int tdp_huge_page_level);
-
-
 #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
 #define kvm_arch_has_private_mem(kvm) ((kvm)->arch.has_private_mem)
 #endif
 
 #define kvm_arch_has_readonly_mem(kvm) (!(kvm)->arch.has_protected_state)
 
-static inline u16 kvm_read_ldt(void)
-{
-	u16 ldt;
-	asm("sldt %0" : "=g"(ldt));
-	return ldt;
-}
-
-static inline void kvm_load_ldt(u16 sel)
-{
-	asm("lldt %0" : : "rm"(sel));
-}
-
 #ifdef CONFIG_X86_64
 static inline unsigned long read_msr(unsigned long msr)
 {
@@ -2422,25 +1860,6 @@ static inline unsigned long read_msr(unsigned long msr)
 }
 #endif
 
-static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code)
-{
-	kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
-}
-
-#define TSS_IOPB_BASE_OFFSET 0x66
-#define TSS_BASE_SIZE 0x68
-#define TSS_IOPB_SIZE (65536 / 8)
-#define TSS_REDIRECTION_SIZE (256 / 8)
-#define RMODE_TSS_SIZE							\
-	(TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1)
-
-enum {
-	TASK_SWITCH_CALL = 0,
-	TASK_SWITCH_IRET = 1,
-	TASK_SWITCH_JMP = 2,
-	TASK_SWITCH_GATE = 3,
-};
-
 #define HF_GUEST_MASK		(1 << 0) /* VCPU is in guest-mode */
 
 #ifdef CONFIG_KVM_SMM
@@ -2456,40 +1875,6 @@ enum {
 # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, 0)
 #endif
 
-int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v);
-int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu);
-int kvm_cpu_has_extint(struct kvm_vcpu *v);
-int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu);
-int kvm_cpu_get_extint(struct kvm_vcpu *v);
-int kvm_cpu_get_interrupt(struct kvm_vcpu *v);
-void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event);
-
-int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
-		    unsigned long ipi_bitmap_high, u32 min,
-		    unsigned long icr, int op_64_bit);
-
-int kvm_add_user_return_msr(u32 msr);
-int kvm_find_user_return_msr(u32 msr);
-int kvm_set_user_return_msr(unsigned index, u64 val, u64 mask);
-u64 kvm_get_user_return_msr(unsigned int slot);
-
-static inline bool kvm_is_supported_user_return_msr(u32 msr)
-{
-	return kvm_find_user_return_msr(msr) >= 0;
-}
-
-u64 kvm_scale_tsc(u64 tsc, u64 ratio);
-u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc);
-u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier);
-u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier);
-
-unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu);
-bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip);
-
-void kvm_make_scan_ioapic_request(struct kvm *kvm);
-void kvm_make_scan_ioapic_request_mask(struct kvm *kvm,
-				       unsigned long *vcpu_bitmap);
-
 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
 				     struct kvm_async_pf *work);
 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
@@ -2498,22 +1883,6 @@ void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
 			       struct kvm_async_pf *work);
 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu);
 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu);
-extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn);
-
-int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu);
-int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err);
-
-void __user *__x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa,
-				     u32 size);
-bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu);
-bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu);
-
-static inline bool kvm_irq_is_postable(struct kvm_lapic_irq *irq)
-{
-	/* We can only post Fixed and LowPrio IRQs */
-	return (irq->delivery_mode == APIC_DM_FIXED ||
-		irq->delivery_mode == APIC_DM_LOWEST);
-}
 
 static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
 {
@@ -2525,36 +1894,6 @@ static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
 	kvm_x86_call(vcpu_unblocking)(vcpu);
 }
 
-int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages);
-
-#define KVM_CLOCK_VALID_FLAGS						\
-	(KVM_CLOCK_TSC_STABLE | KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC)
-
-#define KVM_X86_VALID_QUIRKS			\
-	(KVM_X86_QUIRK_LINT0_REENABLED |	\
-	 KVM_X86_QUIRK_CD_NW_CLEARED |		\
-	 KVM_X86_QUIRK_LAPIC_MMIO_HOLE |	\
-	 KVM_X86_QUIRK_OUT_7E_INC_RIP |		\
-	 KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT |	\
-	 KVM_X86_QUIRK_FIX_HYPERCALL_INSN |	\
-	 KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS |	\
-	 KVM_X86_QUIRK_SLOT_ZAP_ALL |		\
-	 KVM_X86_QUIRK_STUFF_FEATURE_MSRS |	\
-	 KVM_X86_QUIRK_IGNORE_GUEST_PAT |	\
-	 KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM |	\
-	 KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT)
-
-#define KVM_X86_CONDITIONAL_QUIRKS		\
-	(KVM_X86_QUIRK_CD_NW_CLEARED |		\
-	 KVM_X86_QUIRK_IGNORE_GUEST_PAT)
-
-/*
- * KVM previously used a u32 field in kvm_run to indicate the hypercall was
- * initiated from long mode. KVM now sets bit 0 to indicate long mode, but the
- * remaining 31 lower bits must be 0 to preserve ABI.
- */
-#define KVM_EXIT_HYPERCALL_MBZ		GENMASK_ULL(31, 1)
-
 static inline bool kvm_arch_has_irq_bypass(void)
 {
 	return enable_device_posted_irqs;
diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h
index a7294656ad908..c9769a7b6e66c 100644
--- a/arch/x86/include/asm/linkage.h
+++ b/arch/x86/include/asm/linkage.h
@@ -103,7 +103,7 @@
 	.byte 0xb8 ASM_NL					\
 	.long __kcfi_typeid_##name ASM_NL			\
 	CFI_POST_PADDING					\
-	SYM_FUNC_END(__cfi_##name)
+	SYM_END(__cfi_##name, SYM_T_FUNC)
 
 /* UML needs to be able to override memcpy() and friends for KASAN. */
 #ifdef CONFIG_UML
diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h
index 9cd136d4515c4..645e65ac1586d 100644
--- a/arch/x86/include/asm/microcode.h
+++ b/arch/x86/include/asm/microcode.h
@@ -62,8 +62,6 @@ static inline int intel_microcode_get_datasize(struct microcode_header_intel *hd
 	return hdr->datasize ? : DEFAULT_UCODE_DATASIZE;
 }
 
-extern u32 intel_get_platform_id(void);
-
 static inline u32 intel_get_microcode_revision(void)
 {
 	u32 rev, dummy;
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 18c4be75e9271..3a8e51a0c9e88 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -1007,6 +1007,7 @@
 #define THERM_INT_HIGH_ENABLE		(1 << 0)
 #define THERM_INT_LOW_ENABLE		(1 << 1)
 #define THERM_INT_PLN_ENABLE		(1 << 24)
+#define THERM_INT_DPTI_ENABLE		(1 << 25)
 
 #define MSR_IA32_THERM_STATUS		0x0000019c
 
@@ -1036,6 +1037,7 @@
 
 #define PACKAGE_THERM_STATUS_PROCHOT		(1 << 0)
 #define PACKAGE_THERM_STATUS_POWER_LIMIT	(1 << 10)
+#define PACKAGE_THERM_STATUS_DPTI_ACK		(1 << 25)
 #define PACKAGE_THERM_STATUS_HFI_UPDATED	(1 << 26)
 
 #define MSR_IA32_PACKAGE_THERM_INTERRUPT	0x000001b2
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index b68892e6d58c4..0b8a1c5ba9a7f 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -6,6 +6,7 @@
 #include <linux/static_key.h>
 #include <linux/objtool.h>
 #include <linux/linkage.h>
+#include <linux/types.h>
 
 #include <asm/alternative.h>
 #include <asm/cpufeatures.h>
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index ac295ca6c92f9..d5f4917c1edcb 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -50,7 +50,7 @@ void ptdump_walk_user_pgd_level_checkwx(void);
 extern spinlock_t pgd_lock;
 extern struct list_head pgd_list;
 
-extern struct mm_struct *pgd_page_get_mm(struct page *page);
+struct mm_struct *pgd_page_get_mm(struct ptdesc *pt);
 
 extern pmdval_t early_pmd_flags;
 
@@ -411,19 +411,19 @@ static inline pte_t pte_wrprotect(pte_t pte)
 }
 
 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
-static inline int pte_uffd_wp(pte_t pte)
+static inline int pte_uffd(pte_t pte)
 {
-	return pte_flags(pte) & _PAGE_UFFD_WP;
+	return pte_flags(pte) & _PAGE_UFFD;
 }
 
-static inline pte_t pte_mkuffd_wp(pte_t pte)
+static inline pte_t pte_mkuffd(pte_t pte)
 {
-	return pte_wrprotect(pte_set_flags(pte, _PAGE_UFFD_WP));
+	return pte_wrprotect(pte_set_flags(pte, _PAGE_UFFD));
 }
 
-static inline pte_t pte_clear_uffd_wp(pte_t pte)
+static inline pte_t pte_clear_uffd(pte_t pte)
 {
-	return pte_clear_flags(pte, _PAGE_UFFD_WP);
+	return pte_clear_flags(pte, _PAGE_UFFD);
 }
 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
 
@@ -526,19 +526,19 @@ static inline pmd_t pmd_wrprotect(pmd_t pmd)
 }
 
 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
-static inline int pmd_uffd_wp(pmd_t pmd)
+static inline int pmd_uffd(pmd_t pmd)
 {
-	return pmd_flags(pmd) & _PAGE_UFFD_WP;
+	return pmd_flags(pmd) & _PAGE_UFFD;
 }
 
-static inline pmd_t pmd_mkuffd_wp(pmd_t pmd)
+static inline pmd_t pmd_mkuffd(pmd_t pmd)
 {
-	return pmd_wrprotect(pmd_set_flags(pmd, _PAGE_UFFD_WP));
+	return pmd_wrprotect(pmd_set_flags(pmd, _PAGE_UFFD));
 }
 
-static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd)
+static inline pmd_t pmd_clear_uffd(pmd_t pmd)
 {
-	return pmd_clear_flags(pmd, _PAGE_UFFD_WP);
+	return pmd_clear_flags(pmd, _PAGE_UFFD);
 }
 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
 
@@ -911,7 +911,7 @@ pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd);
  */
 static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
 {
-	if (!static_cpu_has(X86_FEATURE_PTI))
+	if (!cpu_feature_enabled(X86_FEATURE_PTI))
 		return pgd;
 	return __pti_set_user_pgtbl(pgdp, pgd);
 }
@@ -985,11 +985,7 @@ static inline int pmd_present(pmd_t pmd)
 	return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE);
 }
 
-#ifdef CONFIG_NUMA_BALANCING
-/*
- * These work without NUMA balancing but the kernel does not care. See the
- * comment in include/linux/pgtable.h
- */
+#ifdef CONFIG_ARCH_HAS_PTE_PROTNONE
 static inline int pte_protnone(pte_t pte)
 {
 	return (pte_flags(pte) & (_PAGE_PROTNONE | _PAGE_PRESENT))
@@ -1001,7 +997,7 @@ static inline int pmd_protnone(pmd_t pmd)
 	return (pmd_flags(pmd) & (_PAGE_PROTNONE | _PAGE_PRESENT))
 		== _PAGE_PROTNONE;
 }
-#endif /* CONFIG_NUMA_BALANCING */
+#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */
 
 static inline int pmd_none(pmd_t pmd)
 {
@@ -1471,7 +1467,7 @@ static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count)
 {
 	memcpy(dst, src, count * sizeof(pgd_t));
 #ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION
-	if (!static_cpu_has(X86_FEATURE_PTI))
+	if (!cpu_feature_enabled(X86_FEATURE_PTI))
 		return;
 	/* Clone the user space pgd as well */
 	memcpy(kernel_to_user_pgdp(dst), kernel_to_user_pgdp(src),
@@ -1545,7 +1541,7 @@ static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
 	return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY);
 }
 
-#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
 {
 	return pmd_set_flags(pmd, _PAGE_SWP_SOFT_DIRTY);
@@ -1564,34 +1560,34 @@ static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
 #endif
 
 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
-static inline pte_t pte_swp_mkuffd_wp(pte_t pte)
+static inline pte_t pte_swp_mkuffd(pte_t pte)
 {
-	return pte_set_flags(pte, _PAGE_SWP_UFFD_WP);
+	return pte_set_flags(pte, _PAGE_SWP_UFFD);
 }
 
-static inline int pte_swp_uffd_wp(pte_t pte)
+static inline int pte_swp_uffd(pte_t pte)
 {
-	return pte_flags(pte) & _PAGE_SWP_UFFD_WP;
+	return pte_flags(pte) & _PAGE_SWP_UFFD;
 }
 
-static inline pte_t pte_swp_clear_uffd_wp(pte_t pte)
+static inline pte_t pte_swp_clear_uffd(pte_t pte)
 {
-	return pte_clear_flags(pte, _PAGE_SWP_UFFD_WP);
+	return pte_clear_flags(pte, _PAGE_SWP_UFFD);
 }
 
-static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd)
+static inline pmd_t pmd_swp_mkuffd(pmd_t pmd)
 {
-	return pmd_set_flags(pmd, _PAGE_SWP_UFFD_WP);
+	return pmd_set_flags(pmd, _PAGE_SWP_UFFD);
 }
 
-static inline int pmd_swp_uffd_wp(pmd_t pmd)
+static inline int pmd_swp_uffd(pmd_t pmd)
 {
-	return pmd_flags(pmd) & _PAGE_SWP_UFFD_WP;
+	return pmd_flags(pmd) & _PAGE_SWP_UFFD;
 }
 
-static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd)
+static inline pmd_t pmd_swp_clear_uffd(pmd_t pmd)
 {
-	return pmd_clear_flags(pmd, _PAGE_SWP_UFFD_WP);
+	return pmd_clear_flags(pmd, _PAGE_SWP_UFFD);
 }
 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
 
diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h
index 2ec250ba467e2..b1a669e775f4c 100644
--- a/arch/x86/include/asm/pgtable_types.h
+++ b/arch/x86/include/asm/pgtable_types.h
@@ -31,7 +31,7 @@
 
 #define _PAGE_BIT_SPECIAL	_PAGE_BIT_SOFTW1
 #define _PAGE_BIT_CPA_TEST	_PAGE_BIT_SOFTW1
-#define _PAGE_BIT_UFFD_WP	_PAGE_BIT_SOFTW2 /* userfaultfd wrprotected */
+#define _PAGE_BIT_UFFD		_PAGE_BIT_SOFTW2 /* userfaultfd tracking */
 #define _PAGE_BIT_SOFT_DIRTY	_PAGE_BIT_SOFTW3 /* software dirty tracking */
 #define _PAGE_BIT_KERNEL_4K	_PAGE_BIT_SOFTW3 /* page must not be converted to large */
 
@@ -39,7 +39,7 @@
 #define _PAGE_BIT_SAVED_DIRTY	_PAGE_BIT_SOFTW5 /* Saved Dirty bit (leaf) */
 #define _PAGE_BIT_NOPTISHADOW	_PAGE_BIT_SOFTW5 /* No PTI shadow (root PGD) */
 #else
-/* Shared with _PAGE_BIT_UFFD_WP which is not supported on 32 bit */
+/* Shared with _PAGE_BIT_UFFD which is not supported on 32 bit */
 #define _PAGE_BIT_SAVED_DIRTY	_PAGE_BIT_SOFTW2 /* Saved Dirty bit (leaf) */
 #define _PAGE_BIT_NOPTISHADOW	_PAGE_BIT_SOFTW2 /* No PTI shadow (root PGD) */
 #endif
@@ -111,11 +111,11 @@
 #endif
 
 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
-#define _PAGE_UFFD_WP		(_AT(pteval_t, 1) << _PAGE_BIT_UFFD_WP)
-#define _PAGE_SWP_UFFD_WP	_PAGE_USER
+#define _PAGE_UFFD		(_AT(pteval_t, 1) << _PAGE_BIT_UFFD)
+#define _PAGE_SWP_UFFD		_PAGE_USER
 #else
-#define _PAGE_UFFD_WP		(_AT(pteval_t, 0))
-#define _PAGE_SWP_UFFD_WP	(_AT(pteval_t, 0))
+#define _PAGE_UFFD		(_AT(pteval_t, 0))
+#define _PAGE_SWP_UFFD		(_AT(pteval_t, 0))
 #endif
 
 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
@@ -129,7 +129,7 @@
 /*
  * The hardware requires shadow stack to be Write=0,Dirty=1. However,
  * there are valid cases where the kernel might create read-only PTEs that
- * are dirty (e.g., fork(), mprotect(), uffd-wp(), soft-dirty tracking). In
+ * are dirty (e.g., fork(), mprotect(), userfaultfd, soft-dirty tracking). In
  * this case, the _PAGE_SAVED_DIRTY bit is used instead of the HW-dirty bit,
  * to avoid creating a wrong "shadow stack" PTEs. Such PTEs have
  * (Write=0,SavedDirty=1,Dirty=0) set.
@@ -151,7 +151,7 @@
 #define _COMMON_PAGE_CHG_MASK	(PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT |	\
 				 _PAGE_SPECIAL | _PAGE_ACCESSED |	\
 				 _PAGE_DIRTY_BITS | _PAGE_SOFT_DIRTY |	\
-				 _PAGE_CC | _PAGE_UFFD_WP)
+				 _PAGE_CC | _PAGE_UFFD)
 #define _PAGE_CHG_MASK	(_COMMON_PAGE_CHG_MASK | _PAGE_PAT)
 #define _HPAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PSE | _PAGE_PAT_LARGE)
 
@@ -512,7 +512,7 @@ static inline pgprot_t pgprot_large_2_4k(pgprot_t pgprot)
 	return __pgprot(protval_large_2_4k(pgprot_val(pgprot)));
 }
 
-
+struct ptdesc;
 typedef struct page *pgtable_t;
 
 extern pteval_t __supported_pte_mask;
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 87b1d4c0727e9..8d8f890c4bc0c 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -238,6 +238,7 @@ extern void early_cpu_init(void);
 extern void identify_secondary_cpu(unsigned int cpu);
 extern void print_cpu_info(struct cpuinfo_x86 *);
 void print_cpu_msr(struct cpuinfo_x86 *);
+extern u32 intel_get_platform_id(void);
 
 /*
  * Friendlier CR3 helpers.
diff --git a/arch/x86/include/asm/resctrl.h b/arch/x86/include/asm/resctrl.h
index 575f8408a9e7c..8f6edcdcfd87b 100644
--- a/arch/x86/include/asm/resctrl.h
+++ b/arch/x86/include/asm/resctrl.h
@@ -102,6 +102,7 @@ static inline void __resctrl_sched_in(struct task_struct *tsk)
 	struct resctrl_pqr_state *state = this_cpu_ptr(&pqr_state);
 	u32 closid = READ_ONCE(state->default_closid);
 	u32 rmid = READ_ONCE(state->default_rmid);
+	struct msr val;
 	u32 tmp;
 
 	/*
@@ -123,7 +124,9 @@ static inline void __resctrl_sched_in(struct task_struct *tsk)
 	if (closid != state->cur_closid || rmid != state->cur_rmid) {
 		state->cur_closid = closid;
 		state->cur_rmid = rmid;
-		wrmsr(MSR_IA32_PQR_ASSOC, rmid, closid);
+		val.l = rmid;
+		val.h = closid;
+		wrmsrq(MSR_IA32_PQR_ASSOC, val.q);
 	}
 }
 
diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h
index 914eb32581c73..895d09faaf832 100644
--- a/arch/x86/include/asm/setup.h
+++ b/arch/x86/include/asm/setup.h
@@ -69,8 +69,6 @@ extern void x86_ce4100_early_setup(void);
 static inline void x86_ce4100_early_setup(void) { }
 #endif
 
-#include <linux/kexec_handover.h>
-
 #ifndef _SETUP
 
 #include <asm/espfix.h>
diff --git a/arch/x86/include/asm/sync_core.h b/arch/x86/include/asm/sync_core.h
index 96bda43538ee7..df22113db9cd7 100644
--- a/arch/x86/include/asm/sync_core.h
+++ b/arch/x86/include/asm/sync_core.h
@@ -61,7 +61,7 @@ static __always_inline void sync_core(void)
 	 * The SERIALIZE instruction is the most straightforward way to
 	 * do this, but it is not universally available.
 	 */
-	if (static_cpu_has(X86_FEATURE_SERIALIZE)) {
+	if (cpu_feature_enabled(X86_FEATURE_SERIALIZE)) {
 		serialize();
 		return;
 	}
@@ -96,7 +96,7 @@ static __always_inline void sync_core(void)
 static inline void sync_core_before_usermode(void)
 {
 	/* With PTI, we unconditionally serialize before running user code. */
-	if (static_cpu_has(X86_FEATURE_PTI))
+	if (cpu_feature_enabled(X86_FEATURE_PTI))
 		return;
 
 	/*
diff --git a/arch/x86/include/asm/syscall.h b/arch/x86/include/asm/syscall.h
index c10dbb74cd001..cfe4251fc1fe3 100644
--- a/arch/x86/include/asm/syscall.h
+++ b/arch/x86/include/asm/syscall.h
@@ -20,14 +20,6 @@
 typedef long (*sys_call_ptr_t)(const struct pt_regs *);
 extern const sys_call_ptr_t sys_call_table[];
 
-/*
- * These may not exist, but still put the prototypes in so we
- * can use IS_ENABLED().
- */
-extern long ia32_sys_call(const struct pt_regs *, unsigned int nr);
-extern long x32_sys_call(const struct pt_regs *, unsigned int nr);
-extern long x64_sys_call(const struct pt_regs *, unsigned int nr);
-
 /*
  * Only the low 32 bits of orig_ax are meaningful, so we return int.
  * This importantly ignores the high bits on 64-bit, so comparisons
@@ -172,7 +164,7 @@ static inline int syscall_get_arch(struct task_struct *task)
 		? AUDIT_ARCH_I386 : AUDIT_ARCH_X86_64;
 }
 
-bool do_syscall_64(struct pt_regs *regs, int nr);
+bool do_syscall_64(struct pt_regs *regs, long nr);
 void do_int80_emulation(struct pt_regs *regs);
 
 #endif	/* CONFIG_X86_32 */
diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c
index 91fa262f0e305..8dfe98784bf92 100644
--- a/arch/x86/kernel/acpi/sleep.c
+++ b/arch/x86/kernel/acpi/sleep.c
@@ -61,6 +61,7 @@ int x86_acpi_suspend_lowlevel(void)
 {
 	struct wakeup_header *header =
 		(struct wakeup_header *) __va(real_mode_header->wakeup_header);
+	struct msr val;
 
 	if (header->signature != WAKEUP_HEADER_SIGNATURE) {
 		printk(KERN_ERR "wakeup header does not match\n");
@@ -82,13 +83,10 @@ int x86_acpi_suspend_lowlevel(void)
 	 * with 2-MB L2 Cache and Intel® Processor A100 and A110 on 90
 	 * nm process with 512-KB L2 Cache Specification Update".
 	 */
-	if (!rdmsr_safe(MSR_EFER,
-			&header->pmode_efer_low,
-			&header->pmode_efer_high) &&
-	    !wrmsr_safe(MSR_EFER,
-			header->pmode_efer_low,
-			header->pmode_efer_high))
+	if (!rdmsrq_safe(MSR_EFER, &val.q) && !wrmsrq_safe(MSR_EFER, val.q))
 		header->pmode_behavior |= (1 << WAKEUP_BEHAVIOR_RESTORE_EFER);
+	header->pmode_efer_low = val.l;
+	header->pmode_efer_high = val.h;
 #endif /* !CONFIG_64BIT */
 
 	header->pmode_cr0 = read_cr0();
@@ -96,14 +94,12 @@ int x86_acpi_suspend_lowlevel(void)
 		header->pmode_cr4 = __read_cr4();
 		header->pmode_behavior |= (1 << WAKEUP_BEHAVIOR_RESTORE_CR4);
 	}
-	if (!rdmsr_safe(MSR_IA32_MISC_ENABLE,
-			&header->pmode_misc_en_low,
-			&header->pmode_misc_en_high) &&
-	    !wrmsr_safe(MSR_IA32_MISC_ENABLE,
-			header->pmode_misc_en_low,
-			header->pmode_misc_en_high))
+	if (!rdmsrq_safe(MSR_IA32_MISC_ENABLE, &val.q) &&
+	    !wrmsrq_safe(MSR_IA32_MISC_ENABLE, val.q))
 		header->pmode_behavior |=
 			(1 << WAKEUP_BEHAVIOR_RESTORE_MISC_ENABLE);
+	header->pmode_misc_en_low = val.l;
+	header->pmode_misc_en_high = val.h;
 	header->realmode_flags = acpi_realmode_flags;
 	header->real_magic = 0x12345678;
 
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 62936a3bde19b..da97a5c744254 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -41,15 +41,6 @@ static int __init debug_alt(char *str)
 }
 __setup("debug-alternative", debug_alt);
 
-static int noreplace_smp;
-
-static int __init setup_noreplace_smp(char *str)
-{
-	noreplace_smp = 1;
-	return 1;
-}
-__setup("noreplace-smp", setup_noreplace_smp);
-
 #define DPRINTK(type, fmt, args...)					\
 do {									\
 	if (debug_alternative & DA_##type)				\
@@ -1775,8 +1766,8 @@ static int cfi_rewrite_callers(s32 *start, s32 *end)
 #define FINEIBT_WARN(_f, _v) \
 	WARN_ONCE((_f) != (_v), "FineIBT: " #_f " %ld != %d\n", _f, _v)
 
-static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
-			    s32 *start_cfi, s32 *end_cfi, bool builtin)
+static void __init_or_module __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
+					     s32 *start_cfi, s32 *end_cfi, bool builtin)
 {
 	int ret;
 
@@ -2088,8 +2079,8 @@ bool decode_fineibt_insn(struct pt_regs *regs, unsigned long *target, u32 *type)
 
 #else /* !CONFIG_FINEIBT: */
 
-static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
-			    s32 *start_cfi, s32 *end_cfi, bool builtin)
+static void __init_or_module __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
+					     s32 *start_cfi, s32 *end_cfi, bool builtin)
 {
 	if (IS_ENABLED(CONFIG_CFI) && builtin)
 		pr_info("CFI: Using standard kCFI\n");
@@ -2101,167 +2092,14 @@ static void poison_cfi(void *addr) { }
 
 #endif /* !CONFIG_FINEIBT */
 
-void apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
-		   s32 *start_cfi, s32 *end_cfi)
+void __init_or_module apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
+				    s32 *start_cfi, s32 *end_cfi)
 {
 	return __apply_fineibt(start_retpoline, end_retpoline,
 			       start_cfi, end_cfi,
 			       /* .builtin = */ false);
 }
 
-#ifdef CONFIG_SMP
-static void alternatives_smp_lock(const s32 *start, const s32 *end,
-				  u8 *text, u8 *text_end)
-{
-	const s32 *poff;
-
-	for (poff = start; poff < end; poff++) {
-		u8 *ptr = (u8 *)poff + *poff;
-
-		if (!*poff || ptr < text || ptr >= text_end)
-			continue;
-		/* turn DS segment override prefix into lock prefix */
-		if (*ptr == 0x3e)
-			text_poke(ptr, ((unsigned char []){0xf0}), 1);
-	}
-}
-
-static void alternatives_smp_unlock(const s32 *start, const s32 *end,
-				    u8 *text, u8 *text_end)
-{
-	const s32 *poff;
-
-	for (poff = start; poff < end; poff++) {
-		u8 *ptr = (u8 *)poff + *poff;
-
-		if (!*poff || ptr < text || ptr >= text_end)
-			continue;
-		/* turn lock prefix into DS segment override prefix */
-		if (*ptr == 0xf0)
-			text_poke(ptr, ((unsigned char []){0x3E}), 1);
-	}
-}
-
-struct smp_alt_module {
-	/* what is this ??? */
-	struct module	*mod;
-	char		*name;
-
-	/* ptrs to lock prefixes */
-	const s32	*locks;
-	const s32	*locks_end;
-
-	/* .text segment, needed to avoid patching init code ;) */
-	u8		*text;
-	u8		*text_end;
-
-	struct list_head next;
-};
-static LIST_HEAD(smp_alt_modules);
-static bool uniproc_patched = false;	/* protected by text_mutex */
-
-void __init_or_module alternatives_smp_module_add(struct module *mod,
-						  char *name,
-						  void *locks, void *locks_end,
-						  void *text,  void *text_end)
-{
-	struct smp_alt_module *smp;
-
-	mutex_lock(&text_mutex);
-	if (!uniproc_patched)
-		goto unlock;
-
-	if (num_possible_cpus() == 1)
-		/* Don't bother remembering, we'll never have to undo it. */
-		goto smp_unlock;
-
-	smp = kzalloc_obj(*smp);
-	if (NULL == smp)
-		/* we'll run the (safe but slow) SMP code then ... */
-		goto unlock;
-
-	smp->mod	= mod;
-	smp->name	= name;
-	smp->locks	= locks;
-	smp->locks_end	= locks_end;
-	smp->text	= text;
-	smp->text_end	= text_end;
-	DPRINTK(SMP, "locks %p -> %p, text %p -> %p, name %s\n",
-		smp->locks, smp->locks_end,
-		smp->text, smp->text_end, smp->name);
-
-	list_add_tail(&smp->next, &smp_alt_modules);
-smp_unlock:
-	alternatives_smp_unlock(locks, locks_end, text, text_end);
-unlock:
-	mutex_unlock(&text_mutex);
-}
-
-void __init_or_module alternatives_smp_module_del(struct module *mod)
-{
-	struct smp_alt_module *item;
-
-	mutex_lock(&text_mutex);
-	list_for_each_entry(item, &smp_alt_modules, next) {
-		if (mod != item->mod)
-			continue;
-		list_del(&item->next);
-		kfree(item);
-		break;
-	}
-	mutex_unlock(&text_mutex);
-}
-
-void alternatives_enable_smp(void)
-{
-	struct smp_alt_module *mod;
-
-	/* Why bother if there are no other CPUs? */
-	BUG_ON(num_possible_cpus() == 1);
-
-	mutex_lock(&text_mutex);
-
-	if (uniproc_patched) {
-		pr_info("switching to SMP code\n");
-		BUG_ON(num_online_cpus() != 1);
-		clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
-		clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
-		list_for_each_entry(mod, &smp_alt_modules, next)
-			alternatives_smp_lock(mod->locks, mod->locks_end,
-					      mod->text, mod->text_end);
-		uniproc_patched = false;
-	}
-	mutex_unlock(&text_mutex);
-}
-
-/*
- * Return 1 if the address range is reserved for SMP-alternatives.
- * Must hold text_mutex.
- */
-int alternatives_text_reserved(void *start, void *end)
-{
-	struct smp_alt_module *mod;
-	const s32 *poff;
-	u8 *text_start = start;
-	u8 *text_end = end;
-
-	lockdep_assert_held(&text_mutex);
-
-	list_for_each_entry(mod, &smp_alt_modules, next) {
-		if (mod->text > text_end || mod->text_end < text_start)
-			continue;
-		for (poff = mod->locks; poff < mod->locks_end; poff++) {
-			const u8 *ptr = (const u8 *)poff + *poff;
-
-			if (text_start <= ptr && text_end > ptr)
-				return 1;
-		}
-	}
-
-	return 0;
-}
-#endif /* CONFIG_SMP */
-
 /*
  * Self-test for the INT3 based CALL emulation code.
  *
@@ -2440,40 +2278,12 @@ void __init alternative_instructions(void)
 
 	ibt_restore(ibt);
 
-#ifdef CONFIG_SMP
-	/* Patch to UP if other cpus not imminent. */
-	if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) {
-		uniproc_patched = true;
-		alternatives_smp_module_add(NULL, "core kernel",
-					    __smp_locks, __smp_locks_end,
-					    _text, _etext);
-	}
-#endif
-
 	restart_nmi();
 	alternatives_patched = 1;
 
 	alt_reloc_selftest();
 }
 
-#ifdef CONFIG_SMP
-/*
- * With CONFIG_DEFERRED_STRUCT_PAGE_INIT enabled we can free_init_pages() only
- * after the deferred initialization of the memory map is complete.
- */
-static int __init free_smp_locks(void)
-{
-	if (!uniproc_patched || num_possible_cpus() == 1) {
-		free_init_pages("SMP alternatives",
-				(unsigned long)__smp_locks,
-				(unsigned long)__smp_locks_end);
-	}
-
-	return 0;
-}
-arch_initcall(free_smp_locks);
-#endif
-
 /**
  * text_poke_early - Update instructions on a live kernel at boot time
  * @addr: address to modify
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index aa1e19979aa8e..90025451ace2f 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -1191,11 +1191,11 @@ void disable_local_APIC(void)
 	 * restore the disabled state.
 	 */
 	if (enabled_via_apicbase) {
-		unsigned int l, h;
+		struct msr val;
 
-		rdmsr(MSR_IA32_APICBASE, l, h);
-		l &= ~MSR_IA32_APICBASE_ENABLE;
-		wrmsr(MSR_IA32_APICBASE, l, h);
+		rdmsrq(MSR_IA32_APICBASE, val.q);
+		val.l &= ~MSR_IA32_APICBASE_ENABLE;
+		wrmsrq(MSR_IA32_APICBASE, val.q);
 	}
 #endif
 }
@@ -1960,7 +1960,8 @@ static bool __init detect_init_APIC(void)
 
 static bool __init apic_verify(unsigned long addr)
 {
-	u32 features, h, l;
+	struct msr val;
+	u32 features;
 
 	/*
 	 * The APIC feature bit should now be enabled
@@ -1975,9 +1976,9 @@ static bool __init apic_verify(unsigned long addr)
 
 	/* The BIOS may have set up the APIC at some other address */
 	if (boot_cpu_data.x86 >= 6) {
-		rdmsr(MSR_IA32_APICBASE, l, h);
-		if (l & MSR_IA32_APICBASE_ENABLE)
-			addr = l & MSR_IA32_APICBASE_BASE;
+		rdmsrq(MSR_IA32_APICBASE, val.q);
+		if (val.l & MSR_IA32_APICBASE_ENABLE)
+			addr = val.l & MSR_IA32_APICBASE_BASE;
 	}
 
 	register_lapic_address(addr);
@@ -1987,7 +1988,7 @@ static bool __init apic_verify(unsigned long addr)
 
 bool __init apic_force_enable(unsigned long addr)
 {
-	u32 h, l;
+	struct msr val;
 
 	if (apic_is_disabled)
 		return false;
@@ -1998,12 +1999,12 @@ bool __init apic_force_enable(unsigned long addr)
 	 * and AMD K7 (Model > 1) or later.
 	 */
 	if (boot_cpu_data.x86 >= 6) {
-		rdmsr(MSR_IA32_APICBASE, l, h);
-		if (!(l & MSR_IA32_APICBASE_ENABLE)) {
+		rdmsrq(MSR_IA32_APICBASE, val.q);
+		if (!(val.l & MSR_IA32_APICBASE_ENABLE)) {
 			pr_info("Local APIC disabled by BIOS -- reenabling.\n");
-			l &= ~MSR_IA32_APICBASE_BASE;
-			l |= MSR_IA32_APICBASE_ENABLE | addr;
-			wrmsr(MSR_IA32_APICBASE, l, h);
+			val.l &= ~MSR_IA32_APICBASE_BASE;
+			val.l |= MSR_IA32_APICBASE_ENABLE | addr;
+			wrmsrq(MSR_IA32_APICBASE, val.q);
 			enabled_via_apicbase = 1;
 		}
 	}
@@ -2442,7 +2443,7 @@ static int lapic_suspend(void *data)
 
 static void lapic_resume(void *data)
 {
-	unsigned int l, h;
+	struct msr val;
 	unsigned long flags;
 	int maxlvt;
 
@@ -2475,10 +2476,10 @@ static void lapic_resume(void *data)
 		 * SMP! We'll need to do this as part of the CPU restore!
 		 */
 		if (boot_cpu_data.x86 >= 6) {
-			rdmsr(MSR_IA32_APICBASE, l, h);
-			l &= ~MSR_IA32_APICBASE_BASE;
-			l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
-			wrmsr(MSR_IA32_APICBASE, l, h);
+			rdmsrq(MSR_IA32_APICBASE, val.q);
+			val.l &= ~MSR_IA32_APICBASE_BASE;
+			val.l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
+			wrmsrq(MSR_IA32_APICBASE, val.q);
 		}
 	}
 
diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c
index 5c5be2d582424..a60c8960bbfd0 100644
--- a/arch/x86/kernel/apic/apic_numachip.c
+++ b/arch/x86/kernel/apic/apic_numachip.c
@@ -31,7 +31,7 @@ static u32 numachip1_get_apic_id(u32 x)
 	unsigned long value;
 	unsigned int id = (x >> 24) & 0xff;
 
-	if (static_cpu_has(X86_FEATURE_NODEID_MSR)) {
+	if (cpu_feature_enabled(X86_FEATURE_NODEID_MSR)) {
 		rdmsrq(MSR_FAM10H_NODE_ID, value);
 		id |= (value << 2) & 0xff00;
 	}
diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index d2e8a849f1805..5b1070ec85d94 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -46,7 +46,6 @@ obj-$(CONFIG_CPU_SUP_HYGON)		+= hygon.o
 obj-$(CONFIG_CPU_SUP_CYRIX_32)		+= cyrix.o
 obj-$(CONFIG_CPU_SUP_CENTAUR)		+= centaur.o
 obj-$(CONFIG_CPU_SUP_TRANSMETA_32)	+= transmeta.o
-obj-$(CONFIG_CPU_SUP_UMC_32)		+= umc.o
 obj-$(CONFIG_CPU_SUP_ZHAOXIN)		+= zhaoxin.o
 obj-$(CONFIG_CPU_SUP_VORTEX_32)		+= vortex.o
 
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 487ac147e11f5..169e373418bb9 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -113,7 +113,7 @@ static void init_amd_k5(struct cpuinfo_x86 *c)
 static void init_amd_k6(struct cpuinfo_x86 *c)
 {
 #ifdef CONFIG_X86_32
-	u32 l, h;
+	struct msr val;
 	int mbytes = get_num_physpages() >> (20-PAGE_SHIFT);
 
 	if (c->x86_model < 6) {
@@ -160,13 +160,13 @@ static void init_amd_k6(struct cpuinfo_x86 *c)
 		if (mbytes > 508)
 			mbytes = 508;
 
-		rdmsr(MSR_K6_WHCR, l, h);
-		if ((l&0x0000FFFF) == 0) {
+		rdmsrq(MSR_K6_WHCR, val.q);
+		if ((val.l & 0x0000FFFF) == 0) {
 			unsigned long flags;
-			l = (1<<0)|((mbytes/4)<<1);
+			val.l = (1 << 0) | ((mbytes / 4) << 1);
 			local_irq_save(flags);
 			wbinvd();
-			wrmsr(MSR_K6_WHCR, l, h);
+			wrmsrq(MSR_K6_WHCR, val.q);
 			local_irq_restore(flags);
 			pr_info("Enabling old style K6 write allocation for %d Mb\n",
 				mbytes);
@@ -181,13 +181,13 @@ static void init_amd_k6(struct cpuinfo_x86 *c)
 		if (mbytes > 4092)
 			mbytes = 4092;
 
-		rdmsr(MSR_K6_WHCR, l, h);
-		if ((l&0xFFFF0000) == 0) {
+		rdmsrq(MSR_K6_WHCR, val.q);
+		if ((val.l & 0xFFFF0000) == 0) {
 			unsigned long flags;
-			l = ((mbytes>>2)<<22)|(1<<16);
+			val.l = ((mbytes >> 2) << 22) | (1 << 16);
 			local_irq_save(flags);
 			wbinvd();
-			wrmsr(MSR_K6_WHCR, l, h);
+			wrmsrq(MSR_K6_WHCR, val.q);
 			local_irq_restore(flags);
 			pr_info("Enabling new style K6 write allocation for %d Mb\n",
 				mbytes);
@@ -207,7 +207,7 @@ static void init_amd_k6(struct cpuinfo_x86 *c)
 static void init_amd_k7(struct cpuinfo_x86 *c)
 {
 #ifdef CONFIG_X86_32
-	u32 l, h;
+	struct msr val;
 
 	/*
 	 * Bit 15 of Athlon specific MSR 15, needs to be 0
@@ -228,11 +228,12 @@ static void init_amd_k7(struct cpuinfo_x86 *c)
 	 * As per AMD technical note 27212 0.2
 	 */
 	if ((c->x86_model == 8 && c->x86_stepping >= 1) || (c->x86_model > 8)) {
-		rdmsr(MSR_K7_CLK_CTL, l, h);
-		if ((l & 0xfff00000) != 0x20000000) {
+		rdmsrq(MSR_K7_CLK_CTL, val.q);
+		if ((val.l & 0xfff00000) != 0x20000000) {
 			pr_info("CPU: CLK_CTL MSR was %x. Reprogramming to %x\n",
-				l, ((l & 0x000fffff)|0x20000000));
-			wrmsr(MSR_K7_CLK_CTL, (l & 0x000fffff)|0x20000000, h);
+				val.l, ((val.l & 0x000fffff) | 0x20000000));
+			val.l = (val.l & 0x000fffff) | 0x20000000;
+			wrmsrq(MSR_K7_CLK_CTL, val.q);
 		}
 	}
 
@@ -614,12 +615,13 @@ static void early_detect_mem_encrypt(struct cpuinfo_x86 *c)
 
 static void early_init_amd(struct cpuinfo_x86 *c)
 {
-	u32 dummy;
+	u64 val;
 
 	if (c->x86 >= 0xf)
 		set_cpu_cap(c, X86_FEATURE_K8);
 
-	rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy);
+	rdmsrq_safe(MSR_AMD64_PATCH_LEVEL, &val);
+	c->microcode = (u32)val;
 
 	/*
 	 * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index d9af230c05125..ddf0db5326fa1 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -192,8 +192,8 @@ x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest)
 	 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
 	 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
 	 */
-	if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
-	    !static_cpu_has(X86_FEATURE_VIRT_SSBD))
+	if (!cpu_feature_enabled(X86_FEATURE_LS_CFG_SSBD) &&
+	    !cpu_feature_enabled(X86_FEATURE_VIRT_SSBD))
 		return;
 
 	/*
@@ -201,7 +201,7 @@ x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest)
 	 * virtual MSR value. If its not permanently enabled, evaluate
 	 * current's TIF_SSBD thread flag.
 	 */
-	if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
+	if (cpu_feature_enabled(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
 		hostval = SPEC_CTRL_SSBD;
 	else
 		hostval = ssbd_tif_to_spec_ctrl(ti->flags);
@@ -2499,8 +2499,8 @@ static void __init ssb_apply_mitigation(void)
 		 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
 		 * use a completely different MSR and bit dependent on family.
 		 */
-		if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
-		    !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
+		if (!cpu_feature_enabled(X86_FEATURE_SPEC_CTRL_SSBD) &&
+		    !cpu_feature_enabled(X86_FEATURE_AMD_SSBD)) {
 			x86_amd_ssb_disable();
 		} else {
 			x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c
index 51a95b07831fa..13ed16527905f 100644
--- a/arch/x86/kernel/cpu/cacheinfo.c
+++ b/arch/x86/kernel/cpu/cacheinfo.c
@@ -654,7 +654,7 @@ static DEFINE_RAW_SPINLOCK(cache_disable_lock);
  */
 static void maybe_flush_caches(void)
 {
-	if (!static_cpu_has(X86_FEATURE_SELFSNOOP))
+	if (!cpu_feature_enabled(X86_FEATURE_SELFSNOOP))
 		wbinvd();
 }
 
diff --git a/arch/x86/kernel/cpu/centaur.c b/arch/x86/kernel/cpu/centaur.c
index 681d2da493412..513fa1f640f9a 100644
--- a/arch/x86/kernel/cpu/centaur.c
+++ b/arch/x86/kernel/cpu/centaur.c
@@ -22,7 +22,7 @@
 
 static void init_c3(struct cpuinfo_x86 *c)
 {
-	u32  lo, hi;
+	u64 msr;
 
 	/* Test for Centaur Extended Feature Flags presence */
 	if (cpuid_eax(0xC0000000) >= 0xC0000001) {
@@ -30,17 +30,17 @@ static void init_c3(struct cpuinfo_x86 *c)
 
 		/* enable ACE unit, if present and disabled */
 		if ((tmp & (ACE_PRESENT | ACE_ENABLED)) == ACE_PRESENT) {
-			rdmsr(MSR_VIA_FCR, lo, hi);
-			lo |= ACE_FCR;		/* enable ACE unit */
-			wrmsr(MSR_VIA_FCR, lo, hi);
+			rdmsrq(MSR_VIA_FCR, msr);
+			/* enable ACE unit */
+			wrmsrq(MSR_VIA_FCR, msr | ACE_FCR);
 			pr_info("CPU: Enabled ACE h/w crypto\n");
 		}
 
 		/* enable RNG unit, if present and disabled */
 		if ((tmp & (RNG_PRESENT | RNG_ENABLED)) == RNG_PRESENT) {
-			rdmsr(MSR_VIA_RNG, lo, hi);
-			lo |= RNG_ENABLE;	/* enable RNG unit */
-			wrmsr(MSR_VIA_RNG, lo, hi);
+			rdmsrq(MSR_VIA_RNG, msr);
+			/* enable RNG unit */
+			wrmsrq(MSR_VIA_RNG, msr | RNG_ENABLE);
 			pr_info("CPU: Enabled h/w RNG\n");
 		}
 
@@ -52,9 +52,8 @@ static void init_c3(struct cpuinfo_x86 *c)
 #ifdef CONFIG_X86_32
 	/* Cyrix III family needs CX8 & PGE explicitly enabled. */
 	if (c->x86_model >= 6 && c->x86_model <= 13) {
-		rdmsr(MSR_VIA_FCR, lo, hi);
-		lo |= (1<<1 | 1<<7);
-		wrmsr(MSR_VIA_FCR, lo, hi);
+		rdmsrq(MSR_VIA_FCR, msr);
+		wrmsrq(MSR_VIA_FCR, msr | (1 << 1 | 1 << 7));
 		set_cpu_cap(c, X86_FEATURE_CX8);
 	}
 
@@ -115,8 +114,9 @@ static void init_centaur(struct cpuinfo_x86 *c)
 	char *name;
 	u32  fcr_set = 0;
 	u32  fcr_clr = 0;
-	u32  lo, hi, newlo;
+	u32  newlo;
 	u32  aa, bb, cc, dd;
+	struct msr val;
 #endif
 	early_init_centaur(c);
 	init_intel_cacheinfo(c);
@@ -169,15 +169,16 @@ static void init_centaur(struct cpuinfo_x86 *c)
 			name = "??";
 		}
 
-		rdmsr(MSR_IDT_FCR1, lo, hi);
-		newlo = (lo|fcr_set) & (~fcr_clr);
+		rdmsrq(MSR_IDT_FCR1, val.q);
+		newlo = (val.l | fcr_set) & (~fcr_clr);
 
-		if (newlo != lo) {
+		if (newlo != val.l) {
 			pr_info("Centaur FCR was 0x%X now 0x%X\n",
-				lo, newlo);
-			wrmsr(MSR_IDT_FCR1, newlo, hi);
+				val.l, newlo);
+			val.l = newlo;
+			wrmsrq(MSR_IDT_FCR1, val.q);
 		} else {
-			pr_info("Centaur FCR is 0x%X\n", lo);
+			pr_info("Centaur FCR is 0x%X\n", val.l);
 		}
 		/* Emulate MTRRs using Centaur's MCR. */
 		set_cpu_cap(c, X86_FEATURE_CENTAUR_MCR);
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index a3df21d264606..cbef2c6c84785 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -339,16 +339,16 @@ bool cpuid_feature(void)
 
 static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
 {
-	unsigned long lo, hi;
+	struct msr val;
 
 	if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
 		return;
 
 	/* Disable processor serial number: */
 
-	rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
-	lo |= 0x200000;
-	wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
+	rdmsrq(MSR_IA32_BBL_CR_CTL, val.q);
+	val.l |= 0x200000;
+	wrmsrq(MSR_IA32_BBL_CR_CTL, val.q);
 
 	pr_notice("CPU serial number disabled.\n");
 	clear_cpu_cap(c, X86_FEATURE_PN);
@@ -2299,8 +2299,10 @@ static inline void idt_syscall_init(void)
 /* May not be marked __init: used by software suspend */
 void syscall_init(void)
 {
+	struct msr val = { .h = (__USER32_CS << 16) | __KERNEL_CS };
+
 	/* The default user and kernel segments */
-	wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS);
+	wrmsrq(MSR_STAR, val.q);
 
 	/*
 	 * Except the IA32_STAR MSR, there is NO need to setup SYSCALL and
diff --git a/arch/x86/kernel/cpu/feat_ctl.c b/arch/x86/kernel/cpu/feat_ctl.c
index d69757246bde7..10a92927a515c 100644
--- a/arch/x86/kernel/cpu/feat_ctl.c
+++ b/arch/x86/kernel/cpu/feat_ctl.c
@@ -25,7 +25,8 @@ enum vmx_feature_leafs {
 
 static void init_vmx_capabilities(struct cpuinfo_x86 *c)
 {
-	u32 supported, funcs, ept, vpid, ign, low, high;
+	struct msr val;
+	u32 supported, funcs, ept, vpid;
 
 	BUILD_BUG_ON(NVMXINTS != NR_VMX_FEATURE_WORDS);
 
@@ -39,25 +40,31 @@ static void init_vmx_capabilities(struct cpuinfo_x86 *c)
 	 * as they exist on any CPU that supports VMX, i.e. we want the WARN if
 	 * the RDMSR faults.
 	 */
-	rdmsr(MSR_IA32_VMX_PROCBASED_CTLS, ign, supported);
+	rdmsrq(MSR_IA32_VMX_PROCBASED_CTLS, val.q);
+	supported = val.h;
 	c->vmx_capability[PRIMARY_CTLS] = supported;
 
-	rdmsr_safe(MSR_IA32_VMX_PROCBASED_CTLS2, &ign, &supported);
+	rdmsrq_safe(MSR_IA32_VMX_PROCBASED_CTLS2, &val.q);
+	supported = val.h;
 	c->vmx_capability[SECONDARY_CTLS] = supported;
 
 	/* All 64 bits of tertiary controls MSR are allowed-1 settings. */
-	rdmsr_safe(MSR_IA32_VMX_PROCBASED_CTLS3, &low, &high);
-	c->vmx_capability[TERTIARY_CTLS_LOW] = low;
-	c->vmx_capability[TERTIARY_CTLS_HIGH] = high;
+	rdmsrq_safe(MSR_IA32_VMX_PROCBASED_CTLS3, &val.q);
+	c->vmx_capability[TERTIARY_CTLS_LOW] = val.l;
+	c->vmx_capability[TERTIARY_CTLS_HIGH] = val.h;
 
-	rdmsr(MSR_IA32_VMX_PINBASED_CTLS, ign, supported);
-	rdmsr_safe(MSR_IA32_VMX_VMFUNC, &ign, &funcs);
+	rdmsrq(MSR_IA32_VMX_PINBASED_CTLS, val.q);
+	supported = val.h;
+	rdmsrq_safe(MSR_IA32_VMX_VMFUNC, &val.q);
+	funcs = val.h;
 
 	/*
 	 * Except for EPT+VPID, which enumerates support for both in a single
 	 * MSR, low for EPT, high for VPID.
 	 */
-	rdmsr_safe(MSR_IA32_VMX_EPT_VPID_CAP, &ept, &vpid);
+	rdmsrq_safe(MSR_IA32_VMX_EPT_VPID_CAP, &val.q);
+	ept = val.l;
+	vpid = val.h;
 
 	/* Pin, EPT, VPID and VM-Func are merged into a single word. */
 	WARN_ON_ONCE(supported >> 16);
diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c
index 3e8891a9caf27..ec51c2b9a257c 100644
--- a/arch/x86/kernel/cpu/hygon.c
+++ b/arch/x86/kernel/cpu/hygon.c
@@ -125,11 +125,12 @@ static void bsp_init_hygon(struct cpuinfo_x86 *c)
 
 static void early_init_hygon(struct cpuinfo_x86 *c)
 {
-	u32 dummy;
+	u64 val;
 
 	set_cpu_cap(c, X86_FEATURE_K8);
 
-	rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy);
+	rdmsrq_safe(MSR_AMD64_PATCH_LEVEL, &val);
+	c->microcode = (u32)val;
 
 	/*
 	 * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index abb3984336ebf..4297ceb2cb244 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -199,6 +199,41 @@ void intel_unlock_cpuid_leafs(struct cpuinfo_x86 *c)
 		c->cpuid_level = cpuid_eax(0);
 }
 
+/*
+ * Use CPUID to generate a "vfm" value. Useful before cpuinfo_x86
+ * structures are populated.
+ */
+static u32 intel_cpuid_vfm(void)
+{
+	u32 eax   = cpuid_eax(1);
+	u32 fam   = x86_family(eax);
+	u32 model = x86_model(eax);
+
+	return IFM(fam, model);
+}
+
+u32 intel_get_platform_id(void)
+{
+	unsigned int val[2];
+
+	if (x86_hypervisor_present)
+		return 0;
+
+	/*
+	 * This can be called early. Use CPUID directly instead of
+	 * relying on cpuinfo_x86 which may not be fully initialized.
+	 * The PII does not have MSR_IA32_PLATFORM_ID. Everything
+	 * before _it_ has no microcode (for Linux at least).
+	 */
+	if (intel_cpuid_vfm() <= INTEL_PENTIUM_II_KLAMATH)
+		return 0;
+
+	/* get processor flags from MSR 0x17 */
+	native_rdmsr(MSR_IA32_PLATFORM_ID, val[0], val[1]);
+
+	return (val[1] >> 18) & 7;
+}
+
 static void early_init_intel(struct cpuinfo_x86 *c)
 {
 	u64 misc_enable;
@@ -542,12 +577,12 @@ static void init_intel(struct cpuinfo_x86 *c)
 		set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC);
 
 	if (boot_cpu_has(X86_FEATURE_DS)) {
-		unsigned int l1, l2;
+		u64 l;
 
-		rdmsr(MSR_IA32_MISC_ENABLE, l1, l2);
-		if (!(l1 & MSR_IA32_MISC_ENABLE_BTS_UNAVAIL))
+		rdmsrq(MSR_IA32_MISC_ENABLE, l);
+		if (!(l & MSR_IA32_MISC_ENABLE_BTS_UNAVAIL))
 			set_cpu_cap(c, X86_FEATURE_BTS);
-		if (!(l1 & MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL))
+		if (!(l & MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL))
 			set_cpu_cap(c, X86_FEATURE_PEBS);
 	}
 
diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c
index 36e0df4d1342d..f916fb4c5d138 100644
--- a/arch/x86/kernel/cpu/mce/amd.c
+++ b/arch/x86/kernel/cpu/mce/amd.c
@@ -280,11 +280,11 @@ static void smca_configure(unsigned int bank, unsigned int cpu)
 	u8 *bank_counts = this_cpu_ptr(smca_bank_counts);
 	const struct smca_hwid *s_hwid;
 	unsigned int i, hwid_mcatype;
-	u32 high, low;
+	struct msr val;
 	u32 smca_config = MSR_AMD64_SMCA_MCx_CONFIG(bank);
 
 	/* Set appropriate bits in MCA_CONFIG */
-	if (!rdmsr_safe(smca_config, &low, &high)) {
+	if (!rdmsrq_safe(smca_config, &val.q)) {
 		/*
 		 * OS is required to set the MCAX bit to acknowledge that it is
 		 * now using the new MSR ranges and new registers under each
@@ -294,7 +294,7 @@ static void smca_configure(unsigned int bank, unsigned int cpu)
 		 *
 		 * MCA_CONFIG[MCAX] is bit 32 (0 in the high portion of the MSR.)
 		 */
-		high |= BIT(0);
+		val.h |= BIT(0);
 
 		/*
 		 * SMCA sets the Deferred Error Interrupt type per bank.
@@ -307,9 +307,9 @@ static void smca_configure(unsigned int bank, unsigned int cpu)
 		 * APIC based interrupt. First, check that no interrupt has been
 		 * set.
 		 */
-		if ((low & BIT(5)) && !((high >> 5) & 0x3) && data->dfr_intr_en) {
+		if ((val.l & BIT(5)) && !((val.h >> 5) & 0x3) && data->dfr_intr_en) {
 			__set_bit(bank, data->dfr_intr_banks);
-			high |= BIT(5);
+			val.h |= BIT(5);
 		}
 
 		/*
@@ -324,33 +324,33 @@ static void smca_configure(unsigned int bank, unsigned int cpu)
 		 * The OS should set this to inform the platform that the OS is ready
 		 * to handle the MCA Thresholding interrupt.
 		 */
-		if ((low & BIT(10)) && data->thr_intr_en) {
+		if ((val.l & BIT(10)) && data->thr_intr_en) {
 			__set_bit(bank, data->thr_intr_banks);
-			high |= BIT(8);
+			val.h |= BIT(8);
 		}
 
-		this_cpu_ptr(mce_banks_array)[bank].lsb_in_status = !!(low & BIT(8));
+		this_cpu_ptr(mce_banks_array)[bank].lsb_in_status = !!(val.l & BIT(8));
 
-		if (low & MCI_CONFIG_PADDRV)
+		if (val.l & MCI_CONFIG_PADDRV)
 			this_cpu_ptr(smca_banks)[bank].paddrv = 1;
 
-		wrmsr(smca_config, low, high);
+		wrmsrq(smca_config, val.q);
 	}
 
-	if (rdmsr_safe(MSR_AMD64_SMCA_MCx_IPID(bank), &low, &high)) {
+	if (rdmsrq_safe(MSR_AMD64_SMCA_MCx_IPID(bank), &val.q)) {
 		pr_warn("Failed to read MCA_IPID for bank %d\n", bank);
 		return;
 	}
 
-	hwid_mcatype = HWID_MCATYPE(high & MCI_IPID_HWID,
-				    (high & MCI_IPID_MCATYPE) >> 16);
+	hwid_mcatype = HWID_MCATYPE(val.h & MCI_IPID_HWID,
+				    (val.h & MCI_IPID_MCATYPE) >> 16);
 
 	for (i = 0; i < ARRAY_SIZE(smca_hwid_mcatypes); i++) {
 		s_hwid = &smca_hwid_mcatypes[i];
 
 		if (hwid_mcatype == s_hwid->hwid_mcatype) {
 			this_cpu_ptr(smca_banks)[bank].hwid = s_hwid;
-			this_cpu_ptr(smca_banks)[bank].id = low;
+			this_cpu_ptr(smca_banks)[bank].id = val.l;
 			this_cpu_ptr(smca_banks)[bank].sysfs_id = bank_counts[s_hwid->bank_type]++;
 			break;
 		}
@@ -432,50 +432,50 @@ static bool lvt_off_valid(struct threshold_block *b, int apic, u32 lo, u32 hi)
 static void threshold_restart_block(void *_tr)
 {
 	struct thresh_restart *tr = _tr;
-	u32 hi, lo;
+	struct msr val;
 
 	/* sysfs write might race against an offline operation */
 	if (!this_cpu_read(threshold_banks) && !tr->set_lvt_off)
 		return;
 
-	rdmsr(tr->b->address, lo, hi);
+	rdmsrq(tr->b->address, val.q);
 
 	/*
 	 * Reset error count and overflow bit.
 	 * This is done during init or after handling an interrupt.
 	 */
-	if (hi & MASK_OVERFLOW_HI || tr->set_lvt_off) {
-		hi &= ~(MASK_ERR_COUNT_HI | MASK_OVERFLOW_HI);
-		hi |= THRESHOLD_MAX - tr->b->threshold_limit;
+	if (val.h & MASK_OVERFLOW_HI || tr->set_lvt_off) {
+		val.h &= ~(MASK_ERR_COUNT_HI | MASK_OVERFLOW_HI);
+		val.h |= THRESHOLD_MAX - tr->b->threshold_limit;
 	} else if (tr->old_limit) {	/* change limit w/o reset */
-		int new_count = (hi & THRESHOLD_MAX) +
+		int new_count = (val.h & THRESHOLD_MAX) +
 		    (tr->old_limit - tr->b->threshold_limit);
 
-		hi = (hi & ~MASK_ERR_COUNT_HI) |
+		val.h = (val.h & ~MASK_ERR_COUNT_HI) |
 		    (new_count & THRESHOLD_MAX);
 	}
 
 	/* clear IntType */
-	hi &= ~MASK_INT_TYPE_HI;
+	val.h &= ~MASK_INT_TYPE_HI;
 
 	if (!tr->b->interrupt_capable)
 		goto done;
 
 	if (tr->set_lvt_off) {
-		if (lvt_off_valid(tr->b, tr->lvt_off, lo, hi)) {
+		if (lvt_off_valid(tr->b, tr->lvt_off, val.l, val.h)) {
 			/* set new lvt offset */
-			hi &= ~MASK_LVTOFF_HI;
-			hi |= tr->lvt_off << 20;
+			val.h &= ~MASK_LVTOFF_HI;
+			val.h |= tr->lvt_off << 20;
 		}
 	}
 
 	if (tr->b->interrupt_enable)
-		hi |= INT_TYPE_APIC;
+		val.h |= INT_TYPE_APIC;
 
  done:
 
-	hi |= MASK_COUNT_EN_HI;
-	wrmsr(tr->b->address, lo, hi);
+	val.h |= MASK_COUNT_EN_HI;
+	wrmsrq(tr->b->address, val.q);
 }
 
 static void threshold_restart_bank(unsigned int bank, bool intr_en)
@@ -726,7 +726,8 @@ static void smca_enable_interrupt_vectors(void)
 void mce_amd_feature_init(struct cpuinfo_x86 *c)
 {
 	unsigned int bank, block, cpu = smp_processor_id();
-	u32 low = 0, high = 0, address = 0;
+	struct msr val = { .q = 0 };
+	u32 address = 0;
 	int offset = -1;
 
 	amd_apply_cpu_quirks(c);
@@ -746,21 +747,21 @@ void mce_amd_feature_init(struct cpuinfo_x86 *c)
 		disable_err_thresholding(c, bank);
 
 		for (block = 0; block < NR_BLOCKS; ++block) {
-			address = get_block_address(address, low, high, bank, block, cpu);
+			address = get_block_address(address, val.l, val.h, bank, block, cpu);
 			if (!address)
 				break;
 
-			if (rdmsr_safe(address, &low, &high))
+			if (rdmsrq_safe(address, &val.q))
 				break;
 
-			if (!(high & MASK_VALID_HI))
+			if (!(val.h & MASK_VALID_HI))
 				continue;
 
-			if (!(high & MASK_CNTP_HI)  ||
-			     (high & MASK_LOCKED_HI))
+			if (!(val.h & MASK_CNTP_HI)  ||
+			     (val.h & MASK_LOCKED_HI))
 				continue;
 
-			offset = prepare_threshold_block(bank, block, address, offset, high);
+			offset = prepare_threshold_block(bank, block, address, offset, val.h);
 		}
 	}
 }
@@ -1083,24 +1084,24 @@ static int allocate_threshold_blocks(unsigned int cpu, struct threshold_bank *tb
 				     u32 address)
 {
 	struct threshold_block *b = NULL;
-	u32 low, high;
+	struct msr val;
 	int err;
 
 	if ((bank >= this_cpu_read(mce_num_banks)) || (block >= NR_BLOCKS))
 		return 0;
 
-	if (rdmsr_safe(address, &low, &high))
+	if (rdmsrq_safe(address, &val.q))
 		return 0;
 
-	if (!(high & MASK_VALID_HI)) {
+	if (!(val.h & MASK_VALID_HI)) {
 		if (block)
 			goto recurse;
 		else
 			return 0;
 	}
 
-	if (!(high & MASK_CNTP_HI)  ||
-	     (high & MASK_LOCKED_HI))
+	if (!(val.h & MASK_CNTP_HI)  ||
+	     (val.h & MASK_LOCKED_HI))
 		goto recurse;
 
 	b = kzalloc_obj(struct threshold_block);
@@ -1112,7 +1113,7 @@ static int allocate_threshold_blocks(unsigned int cpu, struct threshold_bank *tb
 	b->cpu			= cpu;
 	b->address		= address;
 	b->interrupt_enable	= 0;
-	b->interrupt_capable	= lvt_interrupt_supported(bank, high);
+	b->interrupt_capable	= lvt_interrupt_supported(bank, val.h);
 	b->threshold_limit	= get_thr_limit();
 
 	if (b->interrupt_capable) {
@@ -1124,13 +1125,13 @@ static int allocate_threshold_blocks(unsigned int cpu, struct threshold_bank *tb
 
 	list_add(&b->miscj, &tb->miscj);
 
-	mce_threshold_block_init(b, (high & MASK_LVTOFF_HI) >> 20);
+	mce_threshold_block_init(b, (val.h & MASK_LVTOFF_HI) >> 20);
 
 	err = kobject_init_and_add(&b->kobj, &threshold_ktype, tb->kobj, get_name(cpu, bank, b));
 	if (err)
 		goto out_free;
 recurse:
-	address = get_block_address(address, low, high, bank, ++block, cpu);
+	address = get_block_address(address, val.l, val.h, bank, ++block, cpu);
 	if (!address)
 		return 0;
 
diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c
index 9bba1e2f03af7..017aaf57ba473 100644
--- a/arch/x86/kernel/cpu/mce/core.c
+++ b/arch/x86/kernel/cpu/mce/core.c
@@ -1866,7 +1866,7 @@ static void __mcheck_cpu_init_generic(void)
 
 	rdmsrq(MSR_IA32_MCG_CAP, cap);
 	if (cap & MCG_CTL_P)
-		wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);
+		wrmsrq(MSR_IA32_MCG_CTL, ~0ULL);
 }
 
 static void __mcheck_cpu_init_prepare_banks(void)
diff --git a/arch/x86/kernel/cpu/mce/p5.c b/arch/x86/kernel/cpu/mce/p5.c
index 2272ad53fc339..eb99f384d747d 100644
--- a/arch/x86/kernel/cpu/mce/p5.c
+++ b/arch/x86/kernel/cpu/mce/p5.c
@@ -23,16 +23,16 @@ int mce_p5_enabled __read_mostly;
 /* Machine check handler for Pentium class Intel CPUs: */
 noinstr void pentium_machine_check(struct pt_regs *regs)
 {
-	u32 loaddr, hi, lotype;
+	u64 addr, type;
 
 	instrumentation_begin();
-	rdmsr(MSR_IA32_P5_MC_ADDR, loaddr, hi);
-	rdmsr(MSR_IA32_P5_MC_TYPE, lotype, hi);
+	rdmsrq(MSR_IA32_P5_MC_ADDR, addr);
+	rdmsrq(MSR_IA32_P5_MC_TYPE, type);
 
 	pr_emerg("CPU#%d: Machine Check Exception:  0x%8X (type 0x%8X).\n",
-		 smp_processor_id(), loaddr, lotype);
+		 smp_processor_id(), (u32)addr, (u32)type);
 
-	if (lotype & (1<<5)) {
+	if (type & (1<<5)) {
 		pr_emerg("CPU#%d: Possible thermal failure (CPU on fire ?).\n",
 			 smp_processor_id());
 	}
@@ -44,7 +44,7 @@ noinstr void pentium_machine_check(struct pt_regs *regs)
 /* Set up machine check reporting for processors with Intel style MCE: */
 void intel_p5_mcheck_init(struct cpuinfo_x86 *c)
 {
-	u32 l, h;
+	u64 q;
 
 	/* Default P5 to off as its often misconnected: */
 	if (!mce_p5_enabled)
@@ -55,8 +55,8 @@ void intel_p5_mcheck_init(struct cpuinfo_x86 *c)
 		return;
 
 	/* Read registers before enabling: */
-	rdmsr(MSR_IA32_P5_MC_ADDR, l, h);
-	rdmsr(MSR_IA32_P5_MC_TYPE, l, h);
+	rdmsrq(MSR_IA32_P5_MC_ADDR, q);
+	rdmsrq(MSR_IA32_P5_MC_TYPE, q);
 	pr_info("Intel old style machine check architecture supported.\n");
 
 	/* Enable MCE: */
diff --git a/arch/x86/kernel/cpu/mce/winchip.c b/arch/x86/kernel/cpu/mce/winchip.c
index 6c99f29419090..7040243533d9f 100644
--- a/arch/x86/kernel/cpu/mce/winchip.c
+++ b/arch/x86/kernel/cpu/mce/winchip.c
@@ -28,12 +28,12 @@ noinstr void winchip_machine_check(struct pt_regs *regs)
 /* Set up machine check reporting on the Winchip C6 series */
 void winchip_mcheck_init(struct cpuinfo_x86 *c)
 {
-	u32 lo, hi;
+	struct msr val;
 
-	rdmsr(MSR_IDT_FCR1, lo, hi);
-	lo |= (1<<2);	/* Enable EIERRINT (int 18 MCE) */
-	lo &= ~(1<<4);	/* Enable MCE */
-	wrmsr(MSR_IDT_FCR1, lo, hi);
+	rdmsrq(MSR_IDT_FCR1, val.q);
+	val.l |= (1<<2);	/* Enable EIERRINT (int 18 MCE) */
+	val.l &= ~(1<<4);	/* Enable MCE */
+	wrmsrq(MSR_IDT_FCR1, val.q);
 
 	cr4_set_bits(X86_CR4_MCE);
 
diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c
index f4a444e6114df..1142183c950c0 100644
--- a/arch/x86/kernel/cpu/microcode/intel.c
+++ b/arch/x86/kernel/cpu/microcode/intel.c
@@ -121,42 +121,6 @@ static inline unsigned int exttable_size(struct extended_sigtable *et)
 	return et->count * EXT_SIGNATURE_SIZE + EXT_HEADER_SIZE;
 }
 
-
-/*
- * Use CPUID to generate a "vfm" value. Useful before cpuinfo_x86
- * structures are populated.
- */
-static u32 intel_cpuid_vfm(void)
-{
-	u32 eax   = cpuid_eax(1);
-	u32 fam   = x86_family(eax);
-	u32 model = x86_model(eax);
-
-	return IFM(fam, model);
-}
-
-u32 intel_get_platform_id(void)
-{
-	unsigned int val[2];
-
-	if (x86_hypervisor_present)
-		return 0;
-
-	/*
-	 * This can be called early. Use CPUID directly instead of
-	 * relying on cpuinfo_x86 which may not be fully initialized.
-	 * The PII does not have MSR_IA32_PLATFORM_ID. Everything
-	 * before _it_ has no microcode (for Linux at least).
-	 */
-	if (intel_cpuid_vfm() <= INTEL_PENTIUM_II_KLAMATH)
-		return 0;
-
-	/* get processor flags from MSR 0x17 */
-	native_rdmsr(MSR_IA32_PLATFORM_ID, val[0], val[1]);
-
-	return (val[1] >> 18) & 7;
-}
-
 void intel_collect_cpu_info(struct cpu_signature *sig)
 {
 	sig->sig = cpuid_eax(1);
diff --git a/arch/x86/kernel/cpu/mtrr/amd.c b/arch/x86/kernel/cpu/mtrr/amd.c
index ef3e8e42b7820..a73715d6f05c1 100644
--- a/arch/x86/kernel/cpu/mtrr/amd.c
+++ b/arch/x86/kernel/cpu/mtrr/amd.c
@@ -10,20 +10,23 @@ static void
 amd_get_mtrr(unsigned int reg, unsigned long *base,
 	     unsigned long *size, mtrr_type *type)
 {
-	unsigned long low, high;
+	unsigned long val;
+	struct msr msr;
 
-	rdmsr(MSR_K6_UWCCR, low, high);
+	rdmsrq(MSR_K6_UWCCR, msr.q);
 	/* Upper dword is region 1, lower is region 0 */
 	if (reg == 1)
-		low = high;
+		val = msr.h;
+	else
+		val = msr.l;
 	/* The base masks off on the right alignment */
-	*base = (low & 0xFFFE0000) >> PAGE_SHIFT;
+	*base = (val & 0xFFFE0000) >> PAGE_SHIFT;
 	*type = 0;
-	if (low & 1)
+	if (val & 1)
 		*type = MTRR_TYPE_UNCACHABLE;
-	if (low & 2)
+	if (val & 2)
 		*type = MTRR_TYPE_WRCOMB;
-	if (!(low & 3)) {
+	if (!(val & 3)) {
 		*size = 0;
 		return;
 	}
@@ -42,8 +45,8 @@ amd_get_mtrr(unsigned int reg, unsigned long *base,
 	 * +1              000 0000 0000 0100
 	 * *128K   ...
 	 */
-	low = (~low) & 0x1FFFC;
-	*size = (low + 4) << (15 - PAGE_SHIFT);
+	val = (~val) & 0x1FFFC;
+	*size = (val + 4) << (15 - PAGE_SHIFT);
 }
 
 /**
@@ -59,12 +62,16 @@ amd_get_mtrr(unsigned int reg, unsigned long *base,
 static void
 amd_set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type)
 {
+	struct msr msr;
 	u32 regs[2];
 
 	/*
 	 * Low is MTRR0, High MTRR 1
 	 */
-	rdmsr(MSR_K6_UWCCR, regs[0], regs[1]);
+	rdmsrq(MSR_K6_UWCCR, msr.q);
+	regs[0] = msr.l;
+	regs[1] = msr.h;
+
 	/*
 	 * Blank to disable
 	 */
@@ -89,7 +96,9 @@ amd_set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type
 	 * disable local interrupts, write back the cache, set the mtrr
 	 */
 	wbinvd();
-	wrmsr(MSR_K6_UWCCR, regs[0], regs[1]);
+	msr.l = regs[0];
+	msr.h = regs[1];
+	wrmsrq(MSR_K6_UWCCR, msr.q);
 }
 
 static int
diff --git a/arch/x86/kernel/cpu/mtrr/centaur.c b/arch/x86/kernel/cpu/mtrr/centaur.c
index 6f6c3ae929437..e32cca1caf594 100644
--- a/arch/x86/kernel/cpu/mtrr/centaur.c
+++ b/arch/x86/kernel/cpu/mtrr/centaur.c
@@ -65,26 +65,26 @@ static void
 centaur_set_mcr(unsigned int reg, unsigned long base,
 		unsigned long size, mtrr_type type)
 {
-	unsigned long low, high;
+	struct msr val;
 
 	if (size == 0) {
 		/* Disable */
-		high = low = 0;
+		val.q = 0;
 	} else {
-		high = base << PAGE_SHIFT;
+		val.h = base << PAGE_SHIFT;
 		if (centaur_mcr_type == 0) {
 			/* Only support write-combining... */
-			low = -size << PAGE_SHIFT | 0x1f;
+			val.l = -size << PAGE_SHIFT | 0x1f;
 		} else {
 			if (type == MTRR_TYPE_UNCACHABLE)
-				low = -size << PAGE_SHIFT | 0x02; /* NC */
+				val.l = -size << PAGE_SHIFT | 0x02; /* NC */
 			else
-				low = -size << PAGE_SHIFT | 0x09; /* WWO, WC */
+				val.l = -size << PAGE_SHIFT | 0x09; /* WWO, WC */
 		}
 	}
-	centaur_mcr[reg].high = high;
-	centaur_mcr[reg].low = low;
-	wrmsr(MSR_IDT_MCR0 + reg, low, high);
+	centaur_mcr[reg].high = val.h;
+	centaur_mcr[reg].low = val.l;
+	wrmsrq(MSR_IDT_MCR0 + reg, val.q);
 }
 
 static int
diff --git a/arch/x86/kernel/cpu/mtrr/cleanup.c b/arch/x86/kernel/cpu/mtrr/cleanup.c
index e3eee9ae41414..cd1a6dec40645 100644
--- a/arch/x86/kernel/cpu/mtrr/cleanup.c
+++ b/arch/x86/kernel/cpu/mtrr/cleanup.c
@@ -658,8 +658,8 @@ static int __init mtrr_search_optimal_index(void)
 int __init mtrr_cleanup(void)
 {
 	unsigned long x_remove_base, x_remove_size;
-	unsigned long base, size, def, dummy;
-	u64 chunk_size, gran_size;
+	u64 def, chunk_size, gran_size;
+	unsigned long base, size;
 	mtrr_type type;
 	int index_good;
 	int i;
@@ -670,7 +670,7 @@ int __init mtrr_cleanup(void)
 	if (!cpu_feature_enabled(X86_FEATURE_MTRR) || enable_mtrr_cleanup < 1)
 		return 0;
 
-	rdmsr(MSR_MTRRdefType, def, dummy);
+	rdmsrq(MSR_MTRRdefType, def);
 	def &= 0xff;
 	if (def != MTRR_TYPE_UNCACHABLE)
 		return 0;
@@ -806,7 +806,7 @@ early_param("disable_mtrr_trim", disable_mtrr_trim_setup);
 
 int __init amd_special_default_mtrr(void)
 {
-	u32 l, h;
+	u64 q;
 
 	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
 	    boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
@@ -814,13 +814,13 @@ int __init amd_special_default_mtrr(void)
 	if (boot_cpu_data.x86 < 0xf)
 		return 0;
 	/* In case some hypervisor doesn't pass SYSCFG through: */
-	if (rdmsr_safe(MSR_AMD64_SYSCFG, &l, &h) < 0)
+	if (rdmsrq_safe(MSR_AMD64_SYSCFG, &q) < 0)
 		return 0;
 	/*
 	 * Memory between 4GB and top of mem is forced WB by this magic bit.
 	 * Reserved before K8RevF, but should be zero there.
 	 */
-	if ((l & (Tom2Enabled | Tom2ForceMemTypeWB)) ==
+	if ((q & (Tom2Enabled | Tom2ForceMemTypeWB)) ==
 		 (Tom2Enabled | Tom2ForceMemTypeWB))
 		return 1;
 	return 0;
@@ -854,9 +854,9 @@ real_trim_memory(unsigned long start_pfn, unsigned long limit_pfn)
  */
 int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
 {
-	unsigned long i, base, size, highest_pfn = 0, def, dummy;
+	unsigned long i, base, size, highest_pfn = 0;
 	mtrr_type type;
-	u64 total_trim_size;
+	u64 def, total_trim_size;
 	/* extra one for all 0 */
 	int num[MTRR_NUM_TYPES + 1];
 
@@ -870,7 +870,7 @@ int __init mtrr_trim_uncached_memory(unsigned long end_pfn)
 	if (!cpu_feature_enabled(X86_FEATURE_MTRR) || disable_mtrr_trim)
 		return 0;
 
-	rdmsr(MSR_MTRRdefType, def, dummy);
+	rdmsrq(MSR_MTRRdefType, def);
 	def &= MTRR_DEF_TYPE_TYPE;
 	if (def != MTRR_TYPE_UNCACHABLE)
 		return 0;
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
index 3a83170607323..67cf69f24b005 100644
--- a/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
@@ -103,7 +103,7 @@ u32 phys_hi_rsvd;
  */
 static inline void k8_check_syscfg_dram_mod_en(void)
 {
-	u32 lo, hi;
+	struct msr val;
 
 	if (!((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) &&
 	      (boot_cpu_data.x86 >= 0x0f)))
@@ -112,13 +112,13 @@ static inline void k8_check_syscfg_dram_mod_en(void)
 	if (cc_platform_has(CC_ATTR_HOST_SEV_SNP))
 		return;
 
-	rdmsr(MSR_AMD64_SYSCFG, lo, hi);
-	if (lo & K8_MTRRFIXRANGE_DRAM_MODIFY) {
+	rdmsrq(MSR_AMD64_SYSCFG, val.q);
+	if (val.l & K8_MTRRFIXRANGE_DRAM_MODIFY) {
 		pr_err(FW_WARN "MTRR: CPU %u: SYSCFG[MtrrFixDramModEn]"
 		       " not cleared by BIOS, clearing this bit\n",
 		       smp_processor_id());
-		lo &= ~K8_MTRRFIXRANGE_DRAM_MODIFY;
-		mtrr_wrmsr(MSR_AMD64_SYSCFG, lo, hi);
+		val.l &= ~K8_MTRRFIXRANGE_DRAM_MODIFY;
+		mtrr_wrmsr(MSR_AMD64_SYSCFG, val.l, val.h);
 	}
 }
 
@@ -557,8 +557,14 @@ u8 mtrr_type_lookup(u64 start, u64 end, u8 *uniform)
 static void
 get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr)
 {
-	rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
-	rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
+	struct msr val;
+
+	rdmsrq(MTRRphysBase_MSR(index), val.q);
+	vr->base_lo = val.l;
+	vr->base_hi = val.h;
+	rdmsrq(MTRRphysMask_MSR(index), val.q);
+	vr->mask_lo = val.l;
+	vr->mask_hi = val.h;
 }
 
 /* Fill the MSR pair relating to a var range */
@@ -577,17 +583,17 @@ void fill_mtrr_var_range(unsigned int index,
 
 static void get_fixed_ranges(mtrr_type *frs)
 {
-	unsigned int *p = (unsigned int *)frs;
+	u64 *p = (u64 *)frs;
 	int i;
 
 	k8_check_syscfg_dram_mod_en();
 
-	rdmsr(MSR_MTRRfix64K_00000, p[0], p[1]);
+	rdmsrq(MSR_MTRRfix64K_00000, p[0]);
 
 	for (i = 0; i < 2; i++)
-		rdmsr(MSR_MTRRfix16K_80000 + i, p[2 + i * 2], p[3 + i * 2]);
+		rdmsrq(MSR_MTRRfix16K_80000 + i, p[1 + i]);
 	for (i = 0; i < 8; i++)
-		rdmsr(MSR_MTRRfix4K_C0000 + i, p[6 + i * 2], p[7 + i * 2]);
+		rdmsrq(MSR_MTRRfix4K_C0000 + i, p[3 + i]);
 }
 
 void mtrr_save_fixed_ranges(void *info)
@@ -689,31 +695,26 @@ static void __init print_mtrr_state(void)
 bool __init get_mtrr_state(void)
 {
 	struct mtrr_var_range *vrs;
-	unsigned lo, dummy;
 	unsigned int i;
+	u64 q;
 
 	vrs = mtrr_state.var_ranges;
 
-	rdmsr(MSR_MTRRcap, lo, dummy);
-	mtrr_state.have_fixed = lo & MTRR_CAP_FIX;
+	rdmsrq(MSR_MTRRcap, q);
+	mtrr_state.have_fixed = q & MTRR_CAP_FIX;
 
 	for (i = 0; i < num_var_ranges; i++)
 		get_mtrr_var_range(i, &vrs[i]);
 	if (mtrr_state.have_fixed)
 		get_fixed_ranges(mtrr_state.fixed_ranges);
 
-	rdmsr(MSR_MTRRdefType, lo, dummy);
-	mtrr_state.def_type = lo & MTRR_DEF_TYPE_TYPE;
-	mtrr_state.enabled = (lo & MTRR_DEF_TYPE_ENABLE) >> MTRR_STATE_SHIFT;
+	rdmsrq(MSR_MTRRdefType, q);
+	mtrr_state.def_type = q & MTRR_DEF_TYPE_TYPE;
+	mtrr_state.enabled = (q & MTRR_DEF_TYPE_ENABLE) >> MTRR_STATE_SHIFT;
 
 	if (amd_special_default_mtrr()) {
-		unsigned low, high;
-
 		/* TOP_MEM2 */
-		rdmsr(MSR_K8_TOP_MEM2, low, high);
-		mtrr_tom2 = high;
-		mtrr_tom2 <<= 32;
-		mtrr_tom2 |= low;
+		rdmsrq(MSR_K8_TOP_MEM2, mtrr_tom2);
 		mtrr_tom2 &= 0xffffff800000ULL;
 	}
 
@@ -750,7 +751,9 @@ void __init mtrr_state_warn(void)
  */
 void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b)
 {
-	if (wrmsr_safe(msr, a, b) < 0) {
+	struct msr val = { .l = a, .h = b };
+
+	if (wrmsrq_safe(msr, val.q) < 0) {
 		pr_err("MTRR: CPU %u: Writing MSR %x to %x:%x failed\n",
 			smp_processor_id(), msr, a, b);
 	}
@@ -765,11 +768,11 @@ void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b)
  */
 static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords)
 {
-	unsigned lo, hi;
+	struct msr val;
 
-	rdmsr(msr, lo, hi);
+	rdmsrq(msr, val.q);
 
-	if (lo != msrwords[0] || hi != msrwords[1]) {
+	if (val.l != msrwords[0] || val.h != msrwords[1]) {
 		mtrr_wrmsr(msr, msrwords[0], msrwords[1]);
 		*changed = true;
 	}
@@ -806,9 +809,8 @@ generic_get_free_region(unsigned long base, unsigned long size, int replace_reg)
 static void generic_get_mtrr(unsigned int reg, unsigned long *base,
 			     unsigned long *size, mtrr_type *type)
 {
-	u32 mask_lo, mask_hi, base_lo, base_hi;
+	u64 tmp, mask, base_msr;
 	unsigned int hi;
-	u64 tmp, mask;
 
 	/*
 	 * get_mtrr doesn't need to update mtrr_state, also it could be called
@@ -816,9 +818,9 @@ static void generic_get_mtrr(unsigned int reg, unsigned long *base,
 	 */
 	get_cpu();
 
-	rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi);
+	rdmsrq(MTRRphysMask_MSR(reg), mask);
 
-	if (!(mask_lo & MTRR_PHYSMASK_V)) {
+	if (!(mask & MTRR_PHYSMASK_V)) {
 		/*  Invalid (i.e. free) range */
 		*base = 0;
 		*size = 0;
@@ -826,10 +828,10 @@ static void generic_get_mtrr(unsigned int reg, unsigned long *base,
 		goto out_put_cpu;
 	}
 
-	rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi);
+	rdmsrq(MTRRphysBase_MSR(reg), base_msr);
 
 	/* Work out the shifted address mask: */
-	tmp = (u64)mask_hi << 32 | (mask_lo & PAGE_MASK);
+	tmp = mask & PAGE_MASK;
 	mask = (u64)phys_hi_rsvd << 32 | tmp;
 
 	/* Expand tmp with high bits to all 1s: */
@@ -849,8 +851,8 @@ static void generic_get_mtrr(unsigned int reg, unsigned long *base,
 	 * contiguous range:
 	 */
 	*size = -mask >> PAGE_SHIFT;
-	*base = (u64)base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT;
-	*type = base_lo & MTRR_PHYSBASE_TYPE;
+	*base = base_msr >> PAGE_SHIFT;
+	*type = base_msr & MTRR_PHYSBASE_TYPE;
 
 out_put_cpu:
 	put_cpu();
@@ -884,21 +886,21 @@ static int set_fixed_ranges(mtrr_type *frs)
  */
 static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)
 {
-	unsigned int lo, hi;
 	bool changed = false;
+	struct msr val;
 
-	rdmsr(MTRRphysBase_MSR(index), lo, hi);
-	if ((vr->base_lo & ~MTRR_PHYSBASE_RSVD) != (lo & ~MTRR_PHYSBASE_RSVD)
-	    || (vr->base_hi & ~phys_hi_rsvd) != (hi & ~phys_hi_rsvd)) {
+	rdmsrq(MTRRphysBase_MSR(index), val.q);
+	if ((vr->base_lo & ~MTRR_PHYSBASE_RSVD) != (val.l & ~MTRR_PHYSBASE_RSVD)
+	    || (vr->base_hi & ~phys_hi_rsvd) != (val.h & ~phys_hi_rsvd)) {
 
 		mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
 		changed = true;
 	}
 
-	rdmsr(MTRRphysMask_MSR(index), lo, hi);
+	rdmsrq(MTRRphysMask_MSR(index), val.q);
 
-	if ((vr->mask_lo & ~MTRR_PHYSMASK_RSVD) != (lo & ~MTRR_PHYSMASK_RSVD)
-	    || (vr->mask_hi & ~phys_hi_rsvd) != (hi & ~phys_hi_rsvd)) {
+	if ((vr->mask_lo & ~MTRR_PHYSMASK_RSVD) != (val.l & ~MTRR_PHYSMASK_RSVD)
+	    || (vr->mask_hi & ~phys_hi_rsvd) != (val.h & ~phys_hi_rsvd)) {
 		mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
 		changed = true;
 	}
@@ -947,8 +949,12 @@ static unsigned long set_mtrr_state(void)
 
 void mtrr_disable(void)
 {
+	struct msr val;
+
 	/* Save MTRR state */
-	rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
+	rdmsrq(MSR_MTRRdefType, val.q);
+	deftype_lo = val.l;
+	deftype_hi = val.h;
 
 	/* Disable MTRRs, and set the default type to uncached */
 	mtrr_wrmsr(MSR_MTRRdefType, deftype_lo & MTRR_DEF_TYPE_DISABLE, deftype_hi);
@@ -1057,8 +1063,9 @@ int generic_validate_add_page(unsigned long base, unsigned long size,
 
 static int generic_have_wrcomb(void)
 {
-	unsigned long config, dummy;
-	rdmsr(MSR_MTRRcap, config, dummy);
+	u64 config;
+
+	rdmsrq(MSR_MTRRcap, config);
 	return config & MTRR_CAP_WC;
 }
 
diff --git a/arch/x86/kernel/cpu/mtrr/mtrr.c b/arch/x86/kernel/cpu/mtrr/mtrr.c
index 4b3d492afe17c..468c53b20acf4 100644
--- a/arch/x86/kernel/cpu/mtrr/mtrr.c
+++ b/arch/x86/kernel/cpu/mtrr/mtrr.c
@@ -547,7 +547,7 @@ void __init mtrr_bp_init(void)
 {
 	bool generic_mtrrs = cpu_feature_enabled(X86_FEATURE_MTRR);
 	const char *why = "(not available)";
-	unsigned long config, dummy;
+	unsigned long config;
 
 	phys_hi_rsvd = GENMASK(31, boot_cpu_data.x86_phys_bits - 32);
 
@@ -571,7 +571,7 @@ void __init mtrr_bp_init(void)
 	if (mtrr_enabled()) {
 		/* Get the number of variable MTRR ranges. */
 		if (mtrr_if == &generic_mtrr_ops)
-			rdmsr(MSR_MTRRcap, config, dummy);
+			rdmsrq(MSR_MTRRcap, config);
 		else
 			config = mtrr_if->var_regs;
 		num_var_ranges = config & MTRR_CAP_VCNT;
diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c
index 9c01d2562b7a8..cf780c1969e92 100644
--- a/arch/x86/kernel/cpu/resctrl/core.c
+++ b/arch/x86/kernel/cpu/resctrl/core.c
@@ -515,14 +515,12 @@ static void domain_add_cpu_ctrl(int cpu, struct rdt_resource *r)
 		return;
 	}
 
-	list_add_tail_rcu(&d->hdr.list, add_pos);
-
 	err = resctrl_online_ctrl_domain(r, d);
 	if (err) {
-		list_del_rcu(&d->hdr.list);
-		synchronize_rcu();
 		ctrl_domain_free(hw_dom);
+		return;
 	}
+	list_add_tail_rcu(&d->hdr.list, add_pos);
 }
 
 static void l3_mon_domain_setup(int cpu, int id, struct rdt_resource *r, struct list_head *add_pos)
@@ -556,14 +554,12 @@ static void l3_mon_domain_setup(int cpu, int id, struct rdt_resource *r, struct
 		return;
 	}
 
-	list_add_tail_rcu(&d->hdr.list, add_pos);
-
 	err = resctrl_online_mon_domain(r, &d->hdr);
 	if (err) {
-		list_del_rcu(&d->hdr.list);
-		synchronize_rcu();
 		l3_mon_domain_free(hw_dom);
+		return;
 	}
+	list_add_tail_rcu(&d->hdr.list, add_pos);
 }
 
 static void domain_add_cpu_mon(int cpu, struct rdt_resource *r)
@@ -642,9 +638,9 @@ static void domain_remove_cpu_ctrl(int cpu, struct rdt_resource *r)
 	d = container_of(hdr, struct rdt_ctrl_domain, hdr);
 	hw_dom = resctrl_to_arch_ctrl_dom(d);
 
-	resctrl_offline_ctrl_domain(r, d);
 	list_del_rcu(&hdr->list);
 	synchronize_rcu();
+	resctrl_offline_ctrl_domain(r, d);
 
 	/*
 	 * rdt_ctrl_domain "d" is going to be freed below, so clear
@@ -689,9 +685,9 @@ static void domain_remove_cpu_mon(int cpu, struct rdt_resource *r)
 
 		d = container_of(hdr, struct rdt_l3_mon_domain, hdr);
 		hw_dom = resctrl_to_arch_mon_dom(d);
-		resctrl_offline_mon_domain(r, hdr);
 		list_del_rcu(&hdr->list);
 		synchronize_rcu();
+		resctrl_offline_mon_domain(r, hdr);
 		l3_mon_domain_free(hw_dom);
 		break;
 	}
@@ -702,9 +698,9 @@ static void domain_remove_cpu_mon(int cpu, struct rdt_resource *r)
 			return;
 
 		pkgd = container_of(hdr, struct rdt_perf_pkg_mon_domain, hdr);
-		resctrl_offline_mon_domain(r, hdr);
 		list_del_rcu(&hdr->list);
 		synchronize_rcu();
+		resctrl_offline_mon_domain(r, hdr);
 		kfree(pkgd);
 		break;
 	}
@@ -725,13 +721,16 @@ static void domain_remove_cpu(int cpu, struct rdt_resource *r)
 static void clear_closid_rmid(int cpu)
 {
 	struct resctrl_pqr_state *state = this_cpu_ptr(&pqr_state);
+	struct msr val = {
+		.l = RESCTRL_RESERVED_RMID,
+		.h = RESCTRL_RESERVED_CLOSID
+	};
 
 	state->default_closid = RESCTRL_RESERVED_CLOSID;
 	state->default_rmid = RESCTRL_RESERVED_RMID;
 	state->cur_closid = RESCTRL_RESERVED_CLOSID;
 	state->cur_rmid = RESCTRL_RESERVED_RMID;
-	wrmsr(MSR_IA32_PQR_ASSOC, RESCTRL_RESERVED_RMID,
-	      RESCTRL_RESERVED_CLOSID);
+	wrmsrq(MSR_IA32_PQR_ASSOC, val.q);
 }
 
 static int resctrl_arch_online_cpu(unsigned int cpu)
diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
index b20e705606b8f..e74f1ed54b86c 100644
--- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
+++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
@@ -53,7 +53,7 @@ int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid)
 	/* Walking r->domains, ensure it can't race with cpuhp */
 	lockdep_assert_cpus_held();
 
-	list_for_each_entry(d, &r->ctrl_domains, hdr.list) {
+	list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) {
 		hw_dom = resctrl_to_arch_ctrl_dom(d);
 		msr_param.res = NULL;
 		for (t = 0; t < CDP_NUM_TYPES; t++) {
@@ -115,7 +115,7 @@ static void _resctrl_sdciae_enable(struct rdt_resource *r, bool enable)
 	lockdep_assert_cpus_held();
 
 	/* Update MSR_IA32_L3_QOS_EXT_CFG MSR on all the CPUs in all domains */
-	list_for_each_entry(d, &r->ctrl_domains, hdr.list)
+	list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held())
 		on_each_cpu_mask(&d->hdr.cpu_mask, resctrl_sdciae_set_one_amd, &enable, 1);
 }
 
diff --git a/arch/x86/kernel/cpu/resctrl/intel_aet.c b/arch/x86/kernel/cpu/resctrl/intel_aet.c
index 89b8b619d5d53..c22c3cf5167d2 100644
--- a/arch/x86/kernel/cpu/resctrl/intel_aet.c
+++ b/arch/x86/kernel/cpu/resctrl/intel_aet.c
@@ -398,12 +398,11 @@ void intel_aet_mon_domain_setup(int cpu, int id, struct rdt_resource *r,
 	d->hdr.type = RESCTRL_MON_DOMAIN;
 	d->hdr.rid = RDT_RESOURCE_PERF_PKG;
 	cpumask_set_cpu(cpu, &d->hdr.cpu_mask);
-	list_add_tail_rcu(&d->hdr.list, add_pos);
 
 	err = resctrl_online_mon_domain(r, &d->hdr);
 	if (err) {
-		list_del_rcu(&d->hdr.list);
-		synchronize_rcu();
 		kfree(d);
+		return;
 	}
+	list_add_tail_rcu(&d->hdr.list, add_pos);
 }
diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c
index 569894d6e5c85..3838e0a13d36f 100644
--- a/arch/x86/kernel/cpu/resctrl/monitor.c
+++ b/arch/x86/kernel/cpu/resctrl/monitor.c
@@ -136,7 +136,7 @@ static int logical_rmid_to_physical_rmid(int cpu, int lrmid)
 
 static int __rmid_read_phys(u32 prmid, enum resctrl_event_id eventid, u64 *val)
 {
-	u64 msr_val;
+	struct msr msr_val = { .l = eventid, .h = prmid };
 
 	/*
 	 * As per the SDM, when IA32_QM_EVTSEL.EvtID (bits 7:0) is configured
@@ -146,15 +146,15 @@ static int __rmid_read_phys(u32 prmid, enum resctrl_event_id eventid, u64 *val)
 	 * IA32_QM_CTR.Error (bit 63) and IA32_QM_CTR.Unavailable (bit 62)
 	 * are error bits.
 	 */
-	wrmsr(MSR_IA32_QM_EVTSEL, eventid, prmid);
-	rdmsrq(MSR_IA32_QM_CTR, msr_val);
+	wrmsrq(MSR_IA32_QM_EVTSEL, msr_val.q);
+	rdmsrq(MSR_IA32_QM_CTR, msr_val.q);
 
-	if (msr_val & RMID_VAL_ERROR)
+	if (msr_val.q & RMID_VAL_ERROR)
 		return -EIO;
-	if (msr_val & RMID_VAL_UNAVAIL)
+	if (msr_val.q & RMID_VAL_UNAVAIL)
 		return -EINVAL;
 
-	*val = msr_val;
+	*val = msr_val.q;
 	return 0;
 }
 
@@ -283,7 +283,10 @@ int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr,
 
 static int __cntr_id_read(u32 cntr_id, u64 *val)
 {
-	u64 msr_val;
+	struct msr msr_val = {
+		.l = ABMC_EXTENDED_EVT_ID | ABMC_EVT_ID,
+		.h = cntr_id
+	};
 
 	/*
 	 * QM_EVTSEL Register definition:
@@ -306,15 +309,15 @@ static int __cntr_id_read(u32 cntr_id, u64 *val)
 	 * ID is set in the QM_EVTSEL.RMID field.  The RMID_VAL_UNAVAIL bit
 	 * is set if the counter data is unavailable.
 	 */
-	wrmsr(MSR_IA32_QM_EVTSEL, ABMC_EXTENDED_EVT_ID | ABMC_EVT_ID, cntr_id);
-	rdmsrq(MSR_IA32_QM_CTR, msr_val);
+	wrmsrq(MSR_IA32_QM_EVTSEL, msr_val.q);
+	rdmsrq(MSR_IA32_QM_CTR, msr_val.q);
 
-	if (msr_val & RMID_VAL_ERROR)
+	if (msr_val.q & RMID_VAL_ERROR)
 		return -EIO;
-	if (msr_val & RMID_VAL_UNAVAIL)
+	if (msr_val.q & RMID_VAL_UNAVAIL)
 		return -EINVAL;
 
-	*val = msr_val;
+	*val = msr_val.q;
 	return 0;
 }
 
@@ -510,7 +513,7 @@ static void _resctrl_abmc_enable(struct rdt_resource *r, bool enable)
 
 	lockdep_assert_cpus_held();
 
-	list_for_each_entry(d, &r->mon_domains, hdr.list) {
+	list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
 		on_each_cpu_mask(&d->hdr.cpu_mask, resctrl_abmc_set_one_amd,
 				 &enable, 1);
 		resctrl_arch_reset_rmid_all(r, d);
diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c
index de580eca33636..d7caab0409b61 100644
--- a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c
+++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c
@@ -241,16 +241,16 @@ int resctrl_arch_pseudo_lock_fn(void *_plr)
 int resctrl_arch_measure_cycles_lat_fn(void *_plr)
 {
 	struct pseudo_lock_region *plr = _plr;
-	u32 saved_low, saved_high;
 	unsigned long i;
 	u64 start, end;
 	void *mem_r;
+	u64 saved;
 
 	local_irq_disable();
 	/*
 	 * Disable hardware prefetchers.
 	 */
-	rdmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high);
+	rdmsrq(MSR_MISC_FEATURE_CONTROL, saved);
 	wrmsrq(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits);
 	mem_r = READ_ONCE(plr->kmem);
 	/*
@@ -267,7 +267,7 @@ int resctrl_arch_measure_cycles_lat_fn(void *_plr)
 		end = rdtsc_ordered();
 		trace_pseudo_lock_mem_latency((u32)(end - start));
 	}
-	wrmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high);
+	wrmsrq(MSR_MISC_FEATURE_CONTROL, saved);
 	local_irq_enable();
 	plr->thread_done = 1;
 	wake_up_interruptible(&plr->lock_thread_wq);
@@ -312,11 +312,11 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr,
 	u64 hits_before = 0, hits_after = 0, miss_before = 0, miss_after = 0;
 	struct perf_event *miss_event, *hit_event;
 	int hit_pmcnum, miss_pmcnum;
-	u32 saved_low, saved_high;
 	unsigned int line_size;
 	unsigned int size;
 	unsigned long i;
 	void *mem_r;
+	u64 saved;
 	u64 tmp;
 
 	miss_event = perf_event_create_kernel_counter(miss_attr, plr->cpu,
@@ -346,7 +346,7 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr,
 	/*
 	 * Disable hardware prefetchers.
 	 */
-	rdmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high);
+	rdmsrq(MSR_MISC_FEATURE_CONTROL, saved);
 	wrmsrq(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits);
 
 	/* Initialize rest of local variables */
@@ -405,7 +405,7 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr,
 	 */
 	rmb();
 	/* Re-enable hardware prefetchers */
-	wrmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high);
+	wrmsrq(MSR_MISC_FEATURE_CONTROL, saved);
 	local_irq_enable();
 out_hit:
 	perf_event_release_kernel(hit_event);
diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c
index 8850264684405..5ffa39fa86fa8 100644
--- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c
+++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c
@@ -151,7 +151,7 @@ static int set_cache_qos_cfg(int level, bool enable)
 		return -ENOMEM;
 
 	r_l = &rdt_resources_all[level].r_resctrl;
-	list_for_each_entry(d, &r_l->ctrl_domains, hdr.list) {
+	list_for_each_entry_rcu(d, &r_l->ctrl_domains, hdr.list, lockdep_is_cpus_held()) {
 		if (r_l->cache.arch_has_per_cpu_cfg)
 			/* Pick all the CPUs in the domain instance */
 			for_each_cpu(cpu, &d->hdr.cpu_mask)
@@ -249,7 +249,7 @@ void resctrl_arch_reset_all_ctrls(struct rdt_resource *r)
 	 * CBMs in all ctrl_domains to the maximum mask value. Pick one CPU
 	 * from each domain to update the MSRs below.
 	 */
-	list_for_each_entry(d, &r->ctrl_domains, hdr.list) {
+	list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) {
 		hw_dom = resctrl_to_arch_ctrl_dom(d);
 
 		for (i = 0; i < hw_res->num_closid; i++)
diff --git a/arch/x86/kernel/cpu/sgx/virt.c b/arch/x86/kernel/cpu/sgx/virt.c
index db6806c40483f..6a1933ddc6fcd 100644
--- a/arch/x86/kernel/cpu/sgx/virt.c
+++ b/arch/x86/kernel/cpu/sgx/virt.c
@@ -9,6 +9,7 @@
 #include <linux/miscdevice.h>
 #include <linux/mm.h>
 #include <linux/mman.h>
+#include <linux/pagemap.h>
 #include <linux/sched/mm.h>
 #include <linux/sched/signal.h>
 #include <linux/slab.h>
@@ -41,7 +42,7 @@ static int __sgx_vepc_fault(struct sgx_vepc *vepc,
 	WARN_ON(!mutex_is_locked(&vepc->lock));
 
 	/* Calculate index of EPC page in virtual EPC's page_array */
-	index = vma->vm_pgoff + PFN_DOWN(addr - vma->vm_start);
+	index = linear_page_index(vma, addr);
 
 	epc_page = xa_load(&vepc->page_array, index);
 	if (epc_page)
diff --git a/arch/x86/kernel/cpu/transmeta.c b/arch/x86/kernel/cpu/transmeta.c
index 1fdcd69c625c2..c670fbb6ee509 100644
--- a/arch/x86/kernel/cpu/transmeta.c
+++ b/arch/x86/kernel/cpu/transmeta.c
@@ -24,7 +24,8 @@ static void early_init_transmeta(struct cpuinfo_x86 *c)
 
 static void init_transmeta(struct cpuinfo_x86 *c)
 {
-	unsigned int cap_mask, uk, max, dummy;
+	u64 msr;
+	unsigned int max, dummy;
 	unsigned int cms_rev1, cms_rev2;
 	unsigned int cpu_rev, cpu_freq = 0, cpu_flags, new_cpu_rev;
 	char cpu_info[65];
@@ -86,10 +87,10 @@ static void init_transmeta(struct cpuinfo_x86 *c)
 	}
 
 	/* Unhide possibly hidden capability flags */
-	rdmsr(0x80860004, cap_mask, uk);
-	wrmsr(0x80860004, ~0, uk);
+	rdmsrq(0x80860004, msr);
+	wrmsrq(0x80860004, msr | ~0U);
 	c->x86_capability[CPUID_1_EDX] = cpuid_edx(0x00000001);
-	wrmsr(0x80860004, cap_mask, uk);
+	wrmsrq(0x80860004, msr);
 
 	/* All Transmeta CPUs have a constant TSC */
 	set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
diff --git a/arch/x86/kernel/cpu/zhaoxin.c b/arch/x86/kernel/cpu/zhaoxin.c
index 761aef5590ac4..fe504fd43c777 100644
--- a/arch/x86/kernel/cpu/zhaoxin.c
+++ b/arch/x86/kernel/cpu/zhaoxin.c
@@ -21,7 +21,7 @@
 
 static void init_zhaoxin_cap(struct cpuinfo_x86 *c)
 {
-	u32  lo, hi;
+	u64 msr;
 
 	/* Test for Extended Feature Flags presence */
 	if (cpuid_eax(0xC0000000) >= 0xC0000001) {
@@ -29,19 +29,17 @@ static void init_zhaoxin_cap(struct cpuinfo_x86 *c)
 
 		/* Enable ACE unit, if present and disabled */
 		if ((tmp & (ACE_PRESENT | ACE_ENABLED)) == ACE_PRESENT) {
-			rdmsr(MSR_ZHAOXIN_FCR57, lo, hi);
+			rdmsrq(MSR_ZHAOXIN_FCR57, msr);
 			/* Enable ACE unit */
-			lo |= ACE_FCR;
-			wrmsr(MSR_ZHAOXIN_FCR57, lo, hi);
+			wrmsrq(MSR_ZHAOXIN_FCR57, msr | ACE_FCR);
 			pr_info("CPU: Enabled ACE h/w crypto\n");
 		}
 
 		/* Enable RNG unit, if present and disabled */
 		if ((tmp & (RNG_PRESENT | RNG_ENABLED)) == RNG_PRESENT) {
-			rdmsr(MSR_ZHAOXIN_FCR57, lo, hi);
+			rdmsrq(MSR_ZHAOXIN_FCR57, msr);
 			/* Enable RNG unit */
-			lo |= RNG_ENABLE;
-			wrmsr(MSR_ZHAOXIN_FCR57, lo, hi);
+			wrmsrq(MSR_ZHAOXIN_FCR57, msr | RNG_ENABLE);
 			pr_info("CPU: Enabled h/w RNG\n");
 		}
 
diff --git a/arch/x86/kernel/crash.c b/arch/x86/kernel/crash.c
index 623d4474631a6..e681ec9cf1dc8 100644
--- a/arch/x86/kernel/crash.c
+++ b/arch/x86/kernel/crash.c
@@ -155,16 +155,8 @@ static int get_nr_ram_ranges_callback(struct resource *res, void *arg)
 	return 0;
 }
 
-/* Gather all the required information to prepare elf headers for ram regions */
-static struct crash_mem *fill_up_crash_elf_data(void)
+unsigned int arch_get_system_nr_ranges(void)
 {
-	unsigned int nr_ranges = 0;
-	struct crash_mem *cmem;
-
-	walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback);
-	if (!nr_ranges)
-		return NULL;
-
 	/*
 	 * Exclusion of crash region, crashk_low_res and/or crashk_cma_ranges
 	 * may cause range splits. So add extra slots here.
@@ -179,49 +171,16 @@ static struct crash_mem *fill_up_crash_elf_data(void)
 	 * But in order to lest the low 1M could be changed in the future,
 	 * (e.g. [start, 1M]), add a extra slot.
 	 */
-	nr_ranges += 3 + crashk_cma_cnt;
-	cmem = vzalloc(struct_size(cmem, ranges, nr_ranges));
-	if (!cmem)
-		return NULL;
-
-	cmem->max_nr_ranges = nr_ranges;
+	unsigned int nr_ranges = 3 + crashk_cma_cnt;
 
-	return cmem;
+	walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback);
+	return nr_ranges;
 }
 
-/*
- * Look for any unwanted ranges between mstart, mend and remove them. This
- * might lead to split and split ranges are put in cmem->ranges[] array
- */
-static int elf_header_exclude_ranges(struct crash_mem *cmem)
+int arch_crash_exclude_ranges(struct crash_mem *cmem)
 {
-	int ret = 0;
-	int i;
-
 	/* Exclude the low 1M because it is always reserved */
-	ret = crash_exclude_mem_range(cmem, 0, SZ_1M - 1);
-	if (ret)
-		return ret;
-
-	/* Exclude crashkernel region */
-	ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end);
-	if (ret)
-		return ret;
-
-	if (crashk_low_res.end)
-		ret = crash_exclude_mem_range(cmem, crashk_low_res.start,
-					      crashk_low_res.end);
-	if (ret)
-		return ret;
-
-	for (i = 0; i < crashk_cma_cnt; ++i) {
-		ret = crash_exclude_mem_range(cmem, crashk_cma_ranges[i].start,
-					      crashk_cma_ranges[i].end);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
+	return crash_exclude_mem_range(cmem, 0, SZ_1M - 1);
 }
 
 static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg)
@@ -235,35 +194,9 @@ static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg)
 	return 0;
 }
 
-/* Prepare elf headers. Return addr and size */
-static int prepare_elf_headers(void **addr, unsigned long *sz,
-			       unsigned long *nr_mem_ranges)
+int arch_crash_populate_cmem(struct crash_mem *cmem)
 {
-	struct crash_mem *cmem;
-	int ret;
-
-	cmem = fill_up_crash_elf_data();
-	if (!cmem)
-		return -ENOMEM;
-
-	ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback);
-	if (ret)
-		goto out;
-
-	/* Exclude unwanted mem ranges */
-	ret = elf_header_exclude_ranges(cmem);
-	if (ret)
-		goto out;
-
-	/* Return the computed number of memory ranges, for hotplug usage */
-	*nr_mem_ranges = cmem->nr_ranges;
-
-	/* By default prepare 64bit headers */
-	ret = crash_prepare_elf64_headers(cmem, IS_ENABLED(CONFIG_X86_64), addr, sz);
-
-out:
-	vfree(cmem);
-	return ret;
+	return walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback);
 }
 #endif
 
@@ -421,7 +354,8 @@ int crash_load_segments(struct kimage *image)
 				  .buf_max = ULONG_MAX, .top_down = false };
 
 	/* Prepare elf headers and add a segment */
-	ret = prepare_elf_headers(&kbuf.buffer, &kbuf.bufsz, &pnum);
+	ret = crash_prepare_headers(IS_ENABLED(CONFIG_X86_64), &kbuf.buffer,
+				    &kbuf.bufsz, &pnum);
 	if (ret)
 		return ret;
 
@@ -514,7 +448,6 @@ unsigned int arch_crash_get_elfcorehdr_size(void)
 void arch_crash_handle_hotplug_event(struct kimage *image, void *arg)
 {
 	void *elfbuf = NULL, *old_elfcorehdr;
-	unsigned long nr_mem_ranges;
 	unsigned long mem, memsz;
 	unsigned long elfsz = 0;
 
@@ -532,7 +465,7 @@ void arch_crash_handle_hotplug_event(struct kimage *image, void *arg)
 	 * Create the new elfcorehdr reflecting the changes to CPU and/or
 	 * memory resources.
 	 */
-	if (prepare_elf_headers(&elfbuf, &elfsz, &nr_mem_ranges)) {
+	if (crash_prepare_headers(IS_ENABLED(CONFIG_X86_64), &elfbuf, &elfsz, NULL)) {
 		pr_err("unable to create new elfcorehdr");
 		goto out;
 	}
diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c
index 584fb9913be42..d1aeecd57f5ed 100644
--- a/arch/x86/kernel/fpu/core.c
+++ b/arch/x86/kernel/fpu/core.c
@@ -294,15 +294,15 @@ void fpu_free_guest_fpstate(struct fpu_guest *gfpu)
 }
 EXPORT_SYMBOL_FOR_KVM(fpu_free_guest_fpstate);
 
-/*
-  * fpu_enable_guest_xfd_features - Check xfeatures against guest perm and enable
-  * @guest_fpu:         Pointer to the guest FPU container
-  * @xfeatures:         Features requested by guest CPUID
-  *
-  * Enable all dynamic xfeatures according to guest perm and requested CPUID.
-  *
-  * Return: 0 on success, error code otherwise
-  */
+/**
+ * fpu_enable_guest_xfd_features - Check xfeatures against guest perm and enable
+ * @guest_fpu:		Pointer to the guest FPU container
+ * @xfeatures:		Features requested by guest CPUID
+ *
+ * Enable all dynamic xfeatures according to guest perm and requested CPUID.
+ *
+ * Return: 0 on success, error code otherwise
+ */
 int fpu_enable_guest_xfd_features(struct fpu_guest *guest_fpu, u64 xfeatures)
 {
 	lockdep_assert_preemption_enabled();
@@ -874,7 +874,7 @@ void fpu_flush_thread(void)
  */
 void switch_fpu_return(void)
 {
-	if (!static_cpu_has(X86_FEATURE_FPU))
+	if (!cpu_feature_enabled(X86_FEATURE_FPU))
 		return;
 
 	fpregs_restore_userregs();
diff --git a/arch/x86/kernel/fpu/signal.c b/arch/x86/kernel/fpu/signal.c
index 20b638c507ca2..33e1284bf3e43 100644
--- a/arch/x86/kernel/fpu/signal.c
+++ b/arch/x86/kernel/fpu/signal.c
@@ -187,7 +187,7 @@ bool copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size, u
 	ia32_fxstate &= (IS_ENABLED(CONFIG_X86_32) ||
 			 IS_ENABLED(CONFIG_IA32_EMULATION));
 
-	if (!static_cpu_has(X86_FEATURE_FPU)) {
+	if (!cpu_feature_enabled(X86_FEATURE_FPU)) {
 		struct user_i387_ia32_struct fp;
 
 		fpregs_soft_get(current, NULL, (struct membuf){.p = &fp,
diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c
index c1fac3a9fecc2..4e5f8c1736ec1 100644
--- a/arch/x86/kernel/kprobes/core.c
+++ b/arch/x86/kernel/kprobes/core.c
@@ -762,9 +762,6 @@ int arch_prepare_kprobe(struct kprobe *p)
 {
 	int ret;
 
-	if (alternatives_text_reserved(p->addr, p->addr))
-		return -EINVAL;
-
 	if (!can_probe((unsigned long)p->addr))
 		return -EILSEQ;
 
diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c
index 6f826a00eca29..3f8fea52619f9 100644
--- a/arch/x86/kernel/kprobes/opt.c
+++ b/arch/x86/kernel/kprobes/opt.c
@@ -77,7 +77,7 @@ unsigned long __recover_optprobed_insn(kprobe_opcode_t *buf, unsigned long addr)
 static void synthesize_clac(kprobe_opcode_t *addr)
 {
 	/*
-	 * Can't be static_cpu_has() due to how objtool treats this feature bit.
+	 * Can't be cpu_feature_enabled() due to how objtool treats this feature bit.
 	 * This isn't a fast path anyway.
 	 */
 	if (!boot_cpu_has(X86_FEATURE_SMAP))
@@ -213,7 +213,6 @@ static int copy_optimized_instructions(u8 *dest, u8 *src, u8 *real)
 	}
 	/* Check whether the address range is reserved */
 	if (ftrace_text_reserved(src, src + len - 1) ||
-	    alternatives_text_reserved(src, src + len - 1) ||
 	    jump_label_text_reserved(src, src + len - 1) ||
 	    static_call_text_reserved(src, src + len - 1))
 		return -EBUSY;
diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c
index 40c5bf97dd5cc..4fa7a86adece1 100644
--- a/arch/x86/kernel/ldt.c
+++ b/arch/x86/kernel/ldt.c
@@ -61,7 +61,7 @@ void load_mm_ldt(struct mm_struct *mm)
 	 */
 
 	if (unlikely(ldt)) {
-		if (static_cpu_has(X86_FEATURE_PTI)) {
+		if (cpu_feature_enabled(X86_FEATURE_PTI)) {
 			if (WARN_ON_ONCE((unsigned long)ldt->slot > 1)) {
 				/*
 				 * Whoops -- either the new LDT isn't mapped
diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
index b5b4de4f08e6c..33080ad889b71 100644
--- a/arch/x86/kernel/module.c
+++ b/arch/x86/kernel/module.c
@@ -227,7 +227,7 @@ int module_finalize(const Elf_Ehdr *hdr,
 		    const Elf_Shdr *sechdrs,
 		    struct module *me)
 {
-	const Elf_Shdr *s, *alt = NULL, *locks = NULL,
+	const Elf_Shdr *s, *alt = NULL,
 		*orc = NULL, *orc_ip = NULL,
 		*retpolines = NULL, *returns = NULL, *ibt_endbr = NULL,
 		*calls = NULL, *cfi = NULL;
@@ -236,8 +236,6 @@ int module_finalize(const Elf_Ehdr *hdr,
 	for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
 		if (!strcmp(".altinstructions", secstrings + s->sh_name))
 			alt = s;
-		if (!strcmp(".smp_locks", secstrings + s->sh_name))
-			locks = s;
 		if (!strcmp(".orc_unwind", secstrings + s->sh_name))
 			orc = s;
 		if (!strcmp(".orc_unwind_ip", secstrings + s->sh_name))
@@ -300,14 +298,6 @@ int module_finalize(const Elf_Ehdr *hdr,
 		void *iseg = (void *)ibt_endbr->sh_addr;
 		apply_seal_endbr(iseg, iseg + ibt_endbr->sh_size);
 	}
-	if (locks) {
-		void *lseg = (void *)locks->sh_addr;
-		void *text = me->mem[MOD_TEXT].base;
-		void *text_end = text + me->mem[MOD_TEXT].size;
-		alternatives_smp_module_add(me, me->name,
-					    lseg, lseg + locks->sh_size,
-					    text, text_end);
-	}
 
 	if (orc && orc_ip)
 		unwind_module_init(me, (void *)orc_ip->sh_addr, orc_ip->sh_size,
@@ -318,6 +308,5 @@ int module_finalize(const Elf_Ehdr *hdr,
 
 void module_arch_cleanup(struct module *mod)
 {
-	alternatives_smp_module_del(mod);
 	its_free_mod(mod);
 }
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index 85435044e33cd..346c438ac8801 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -579,7 +579,7 @@ static __always_inline void amd_set_core_ssb_state(unsigned long tifn)
 	struct ssb_state *st = this_cpu_ptr(&ssb_state);
 	u64 msr = x86_amd_ls_cfg_base;
 
-	if (!static_cpu_has(X86_FEATURE_ZEN)) {
+	if (!cpu_feature_enabled(X86_FEATURE_ZEN)) {
 		msr |= ssbd_tif_to_amd_ls_cfg(tifn);
 		wrmsrq(MSR_AMD64_LS_CFG, msr);
 		return;
@@ -646,14 +646,14 @@ static __always_inline void __speculation_ctrl_update(unsigned long tifp,
 	lockdep_assert_irqs_disabled();
 
 	/* Handle change of TIF_SSBD depending on the mitigation method. */
-	if (static_cpu_has(X86_FEATURE_VIRT_SSBD)) {
+	if (cpu_feature_enabled(X86_FEATURE_VIRT_SSBD)) {
 		if (tif_diff & _TIF_SSBD)
 			amd_set_ssb_virt_state(tifn);
-	} else if (static_cpu_has(X86_FEATURE_LS_CFG_SSBD)) {
+	} else if (cpu_feature_enabled(X86_FEATURE_LS_CFG_SSBD)) {
 		if (tif_diff & _TIF_SSBD)
 			amd_set_core_ssb_state(tifn);
-	} else if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
-		   static_cpu_has(X86_FEATURE_AMD_SSBD)) {
+	} else if (cpu_feature_enabled(X86_FEATURE_SPEC_CTRL_SSBD) ||
+		   cpu_feature_enabled(X86_FEATURE_AMD_SSBD)) {
 		updmsr |= !!(tif_diff & _TIF_SSBD);
 		msr |= ssbd_tif_to_spec_ctrl(tifn);
 	}
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index d44afbe005bb5..2bce7b3f97ede 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -277,7 +277,7 @@ static __always_inline void save_fsgs(struct task_struct *task)
 {
 	savesegment(fs, task->thread.fsindex);
 	savesegment(gs, task->thread.gsindex);
-	if (static_cpu_has(X86_FEATURE_FSGSBASE)) {
+	if (cpu_feature_enabled(X86_FEATURE_FSGSBASE)) {
 		/*
 		 * If FSGSBASE is enabled, we can't make any useful guesses
 		 * about the base, and user code expects us to save the current
@@ -391,7 +391,7 @@ static __always_inline void x86_pkru_load(struct thread_struct *prev,
 static __always_inline void x86_fsgsbase_load(struct thread_struct *prev,
 					      struct thread_struct *next)
 {
-	if (static_cpu_has(X86_FEATURE_FSGSBASE)) {
+	if (cpu_feature_enabled(X86_FEATURE_FSGSBASE)) {
 		/* Update the FS and GS selectors if they could have changed. */
 		if (unlikely(prev->fsindex || next->fsindex))
 			loadseg(FS, next->fsindex);
@@ -533,7 +533,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,
 {
 	WARN_ON_ONCE(regs != current_pt_regs());
 
-	if (static_cpu_has(X86_BUG_NULL_SEG)) {
+	if (cpu_feature_enabled(X86_BUG_NULL_SEG)) {
 		/* Loading zero below won't clear the base. */
 		loadsegment(fs, __USER_DS);
 		load_gs_index(__USER_DS);
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 46882ce79c3a4..dcde6383c0560 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -6,6 +6,7 @@
  * parts of early kernel initialization.
  */
 #include <linux/acpi.h>
+#include <linux/bootconfig.h>
 #include <linux/console.h>
 #include <linux/cpu.h>
 #include <linux/crash_dump.h>
@@ -16,6 +17,7 @@
 #include <linux/init_ohci1394_dma.h>
 #include <linux/initrd.h>
 #include <linux/iscsi_ibft.h>
+#include <linux/kexec_handover.h>
 #include <linux/memblock.h>
 #include <linux/panic_notifier.h>
 #include <linux/pci.h>
@@ -880,7 +882,6 @@ static void __init x86_report_nx(void)
  *
  * Note: On x86_64, fixmaps are ready for use even before this is called.
  */
-
 void __init setup_arch(char **cmdline_p)
 {
 #ifdef CONFIG_X86_32
@@ -924,6 +925,18 @@ void __init setup_arch(char **cmdline_p)
 	builtin_cmdline_added = true;
 #endif
 
+#ifdef CONFIG_CMDLINE_FROM_BOOTCONFIG
+	/*
+	 * Prepend the build-time-rendered embedded "kernel" keys here so
+	 * parse_early_param() below sees them, using the same opt-in as the
+	 * runtime parser, plus the build-time CONFIG_BOOT_CONFIG_FORCE.
+	 */
+	if (bootconfig_cmdline_requested(boot_command_line, NULL) ||
+	    IS_ENABLED(CONFIG_BOOT_CONFIG_FORCE))
+		xbc_prepend_embedded_cmdline(boot_command_line,
+					     COMMAND_LINE_SIZE);
+#endif
+
 	strscpy(command_line, boot_command_line, COMMAND_LINE_SIZE);
 	*cmdline_p = command_line;
 
diff --git a/arch/x86/kernel/signal_32.c b/arch/x86/kernel/signal_32.c
index e55cf19e68fe0..8e87ab5864373 100644
--- a/arch/x86/kernel/signal_32.c
+++ b/arch/x86/kernel/signal_32.c
@@ -357,7 +357,7 @@ int ia32_setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs)
 	unsafe_put_user(ptr_to_compat(&frame->uc), &frame->puc, Efault);
 
 	/* Create the ucontext.  */
-	if (static_cpu_has(X86_FEATURE_XSAVE))
+	if (cpu_feature_enabled(X86_FEATURE_XSAVE))
 		unsafe_put_user(UC_FP_XSTATE, &frame->uc.uc_flags, Efault);
 	else
 		unsafe_put_user(0, &frame->uc.uc_flags, Efault);
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index cb999feb66b02..ba01a9e919b78 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -991,9 +991,6 @@ int common_cpu_up(unsigned int cpu, struct task_struct *idle)
 {
 	int ret;
 
-	/* Just in case we booted with a single CPU. */
-	alternatives_enable_smp();
-
 	per_cpu(current_task, cpu) = idle;
 	cpu_init_stack_canary(cpu, idle);
 
diff --git a/arch/x86/kernel/time.c b/arch/x86/kernel/time.c
index 52e1f3f0b361c..4061430ac74c5 100644
--- a/arch/x86/kernel/time.c
+++ b/arch/x86/kernel/time.c
@@ -82,7 +82,7 @@ static __init void x86_late_time_init(void)
 	x86_init.irqs.intr_mode_init();
 	tsc_init();
 
-	if (static_cpu_has(X86_FEATURE_WAITPKG))
+	if (cpu_feature_enabled(X86_FEATURE_WAITPKG))
 		use_tpause_delay();
 }
 
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index ce10ae4b298b7..723347e2cf7f8 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -1221,11 +1221,11 @@ static void __init check_system_tsc_reliable(void)
 	if (is_geode_lx()) {
 		/* RTSC counts during suspend */
 #define RTSC_SUSP 0x100
-		unsigned long res_low, res_high;
+		u64 res;
 
-		rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high);
+		rdmsrq_safe(MSR_GEODE_BUSCONT_CONF0, &res);
 		/* Geode_LX - the OLPC CPU has a very reliable TSC */
-		if (res_low & RTSC_SUSP)
+		if (res & RTSC_SUSP)
 			tsc_clocksource_reliable = 1;
 	}
 #endif
diff --git a/arch/x86/kernel/tsc_msr.c b/arch/x86/kernel/tsc_msr.c
index 48e6cc1cb017a..d74743c8d2a47 100644
--- a/arch/x86/kernel/tsc_msr.c
+++ b/arch/x86/kernel/tsc_msr.c
@@ -165,7 +165,8 @@ static const struct x86_cpu_id tsc_msr_cpu_ids[] = {
  */
 unsigned long cpu_khz_from_msr(void)
 {
-	u32 lo, hi, ratio, freq, tscref;
+	u32 ratio, freq, tscref;
+	struct msr val;
 	const struct freq_desc *freq_desc;
 	const struct x86_cpu_id *id;
 	const struct muldiv *md;
@@ -178,16 +179,16 @@ unsigned long cpu_khz_from_msr(void)
 
 	freq_desc = (struct freq_desc *)id->driver_data;
 	if (freq_desc->use_msr_plat) {
-		rdmsr(MSR_PLATFORM_INFO, lo, hi);
-		ratio = (lo >> 8) & 0xff;
+		rdmsrq(MSR_PLATFORM_INFO, val.q);
+		ratio = (val.l >> 8) & 0xff;
 	} else {
-		rdmsr(MSR_IA32_PERF_STATUS, lo, hi);
-		ratio = (hi >> 8) & 0x1f;
+		rdmsrq(MSR_IA32_PERF_STATUS, val.q);
+		ratio = (val.h >> 8) & 0x1f;
 	}
 
 	/* Get FSB FREQ ID */
-	rdmsr(MSR_FSB_FREQ, lo, hi);
-	index = lo & freq_desc->mask;
+	rdmsrq(MSR_FSB_FREQ, val.q);
+	index = val.l & freq_desc->mask;
 	md = &freq_desc->muldiv[index];
 
 	/*
diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c
index 3af979fb41d38..65a2de82ecd29 100644
--- a/arch/x86/kernel/uprobes.c
+++ b/arch/x86/kernel/uprobes.c
@@ -276,15 +276,9 @@ static bool is_prefix_bad(struct insn *insn)
 	return false;
 }
 
-static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool x86_64)
+static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn)
 {
-	enum insn_mode m = x86_64 ? INSN_MODE_64 : INSN_MODE_32;
 	u32 volatile *good_insns;
-	int ret;
-
-	ret = insn_decode(insn, auprobe->insn, sizeof(auprobe->insn), m);
-	if (ret < 0)
-		return -ENOEXEC;
 
 	if (is_prefix_bad(insn))
 		return -ENOTSUPP;
@@ -293,7 +287,7 @@ static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool
 	if (insn_masking_exception(insn))
 		return -ENOTSUPP;
 
-	if (x86_64)
+	if (insn->x86_64)
 		good_insns = good_insns_64;
 	else
 		good_insns = good_insns_32;
@@ -631,14 +625,29 @@ static struct vm_special_mapping tramp_mapping = {
 	.pages  = tramp_mapping_pages,
 };
 
-struct uprobe_trampoline {
-	struct hlist_node	node;
-	unsigned long		vaddr;
-};
+
+#define LEA_INSN_SIZE		5
+#define OPT_INSN_SIZE		(LEA_INSN_SIZE + CALL_INSN_SIZE)
+#define REDZONE_SIZE		0x80
+
+static const u8 lea_rsp[] = { 0x48, 0x8d, 0x64, 0x24, 0x80 };
+
+static bool is_opt_insns(const uprobe_opcode_t *insn)
+{
+	return !memcmp(insn, lea_rsp, LEA_INSN_SIZE) &&
+	       insn[LEA_INSN_SIZE] == CALL_INSN_OPCODE;
+}
+
+static bool is_swbp_opt_insns(uprobe_opcode_t *insn)
+{
+	return is_swbp_insn(&insn[0]) &&
+	       !memcmp(&insn[1], &lea_rsp[1], LEA_INSN_SIZE - 1) &&
+	       insn[LEA_INSN_SIZE] == CALL_INSN_OPCODE;
+}
 
 static bool is_reachable_by_call(unsigned long vtramp, unsigned long vaddr)
 {
-	long delta = (long)(vaddr + 5 - vtramp);
+	long delta = (long)(vaddr + OPT_INSN_SIZE - vtramp);
 
 	return delta >= INT_MIN && delta <= INT_MAX;
 }
@@ -651,7 +660,7 @@ static unsigned long find_nearest_trampoline(unsigned long vaddr)
 	};
 	unsigned long low_limit, high_limit;
 	unsigned long low_tramp, high_tramp;
-	unsigned long call_end = vaddr + 5;
+	unsigned long call_end = vaddr + OPT_INSN_SIZE;
 
 	if (check_add_overflow(call_end, INT_MIN, &low_limit))
 		low_limit = PAGE_SIZE;
@@ -682,83 +691,32 @@ static unsigned long find_nearest_trampoline(unsigned long vaddr)
 	return high_tramp;
 }
 
-static struct uprobe_trampoline *create_uprobe_trampoline(unsigned long vaddr)
+static struct vm_area_struct *get_uprobe_trampoline(struct mm_struct *mm, unsigned long vaddr,
+						    bool *new_mapping)
 {
-	struct pt_regs *regs = task_pt_regs(current);
-	struct mm_struct *mm = current->mm;
-	struct uprobe_trampoline *tramp;
+	VMA_ITERATOR(vmi, mm, 0);
 	struct vm_area_struct *vma;
 
-	if (!user_64bit_mode(regs))
-		return NULL;
-
-	vaddr = find_nearest_trampoline(vaddr);
-	if (IS_ERR_VALUE(vaddr))
-		return NULL;
-
-	tramp = kzalloc_obj(*tramp);
-	if (unlikely(!tramp))
-		return NULL;
-
-	tramp->vaddr = vaddr;
-	vma = _install_special_mapping(mm, tramp->vaddr, PAGE_SIZE,
-				VM_READ|VM_EXEC|VM_MAYEXEC|VM_MAYREAD|VM_DONTCOPY|VM_IO,
-				&tramp_mapping);
-	if (IS_ERR(vma)) {
-		kfree(tramp);
-		return NULL;
-	}
-	return tramp;
-}
-
-static struct uprobe_trampoline *get_uprobe_trampoline(unsigned long vaddr, bool *new)
-{
-	struct uprobes_state *state = &current->mm->uprobes_state;
-	struct uprobe_trampoline *tramp = NULL;
+	*new_mapping = false;
 
 	if (vaddr > TASK_SIZE || vaddr < PAGE_SIZE)
-		return NULL;
+		return ERR_PTR(-EINVAL);
 
-	hlist_for_each_entry(tramp, &state->head_tramps, node) {
-		if (is_reachable_by_call(tramp->vaddr, vaddr)) {
-			*new = false;
-			return tramp;
-		}
+	for_each_vma(vmi, vma) {
+		if (!vma_is_special_mapping(vma, &tramp_mapping))
+			continue;
+		if (is_reachable_by_call(vma->vm_start, vaddr))
+			return vma;
 	}
 
-	tramp = create_uprobe_trampoline(vaddr);
-	if (!tramp)
-		return NULL;
-
-	*new = true;
-	hlist_add_head(&tramp->node, &state->head_tramps);
-	return tramp;
-}
-
-static void destroy_uprobe_trampoline(struct uprobe_trampoline *tramp)
-{
-	/*
-	 * We do not unmap and release uprobe trampoline page itself,
-	 * because there's no easy way to make sure none of the threads
-	 * is still inside the trampoline.
-	 */
-	hlist_del(&tramp->node);
-	kfree(tramp);
-}
-
-void arch_uprobe_init_state(struct mm_struct *mm)
-{
-	INIT_HLIST_HEAD(&mm->uprobes_state.head_tramps);
-}
-
-void arch_uprobe_clear_state(struct mm_struct *mm)
-{
-	struct uprobes_state *state = &mm->uprobes_state;
-	struct uprobe_trampoline *tramp;
-	struct hlist_node *n;
+	vaddr = find_nearest_trampoline(vaddr);
+	if (IS_ERR_VALUE(vaddr))
+		return ERR_PTR(vaddr);
 
-	hlist_for_each_entry_safe(tramp, n, &state->head_tramps, node)
-		destroy_uprobe_trampoline(tramp);
+	*new_mapping = true;
+	return _install_special_mapping(mm, vaddr, PAGE_SIZE,
+				VM_READ|VM_EXEC|VM_MAYEXEC|VM_MAYREAD|VM_IO,
+				&tramp_mapping);
 }
 
 static bool __in_uprobe_trampoline(struct mm_struct *mm, unsigned long ip)
@@ -810,7 +768,7 @@ SYSCALL_DEFINE0(uprobe)
 
 	/* Allow execution only from uprobe trampolines. */
 	if (!in_uprobe_trampoline(regs->ip))
-		return -ENXIO;
+		return -EPROTO;
 
 	err = copy_from_user(&args, (void __user *)regs->sp, sizeof(args));
 	if (err)
@@ -826,8 +784,8 @@ SYSCALL_DEFINE0(uprobe)
 	regs->ax  = args.ax;
 	regs->r11 = args.r11;
 	regs->cx  = args.cx;
-	regs->ip  = args.retaddr - 5;
-	regs->sp += sizeof(args);
+	regs->ip  = args.retaddr - OPT_INSN_SIZE;
+	regs->sp += sizeof(args) + REDZONE_SIZE;
 	regs->orig_ax = -1;
 
 	sp = regs->sp;
@@ -844,12 +802,12 @@ SYSCALL_DEFINE0(uprobe)
 	 */
 	if (regs->sp != sp) {
 		/* skip the trampoline call */
-		if (args.retaddr - 5 == regs->ip)
-			regs->ip += 5;
+		if (args.retaddr - OPT_INSN_SIZE == regs->ip)
+			regs->ip += OPT_INSN_SIZE;
 		return regs->ax;
 	}
 
-	regs->sp -= sizeof(args);
+	regs->sp -= sizeof(args) + REDZONE_SIZE;
 
 	/* for the case uprobe_consumer has changed ax/r11/cx */
 	args.ax  = regs->ax;
@@ -857,7 +815,7 @@ SYSCALL_DEFINE0(uprobe)
 	args.cx  = regs->cx;
 
 	/* keep return address unless we are instructed otherwise */
-	if (args.retaddr - 5 != regs->ip)
+	if (args.retaddr - OPT_INSN_SIZE != regs->ip)
 		args.retaddr = regs->ip;
 
 	if (shstk_push(args.retaddr) == -EFAULT)
@@ -891,7 +849,7 @@ asm (
 	"pop %rax\n"
 	"pop %r11\n"
 	"pop %rcx\n"
-	"ret\n"
+	"ret $" __stringify(REDZONE_SIZE) "\n"
 	"int3\n"
 	".balign " __stringify(PAGE_SIZE) "\n"
 	".popsection\n"
@@ -909,7 +867,8 @@ late_initcall(arch_uprobes_init);
 
 enum {
 	EXPECT_SWBP,
-	EXPECT_CALL,
+	EXPECT_OPTIMIZED,
+	EXPECT_SWBP_OPTIMIZED,
 };
 
 struct write_opcode_ctx {
@@ -917,30 +876,29 @@ struct write_opcode_ctx {
 	int expect;
 };
 
-static int is_call_insn(uprobe_opcode_t *insn)
-{
-	return *insn == CALL_INSN_OPCODE;
-}
-
 /*
- * Verification callback used by int3_update uprobe_write calls to make sure
- * the underlying instruction is as expected - either int3 or call.
+ * Verification callback used by uprobe_write calls to make sure the underlying
+ * instruction is in the expected stage of the INT3 update sequence.
  */
 static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode,
 		       int nbytes, void *data)
 {
 	struct write_opcode_ctx *ctx = data;
-	uprobe_opcode_t old_opcode[5];
+	uprobe_opcode_t old_opcode[OPT_INSN_SIZE];
 
-	uprobe_copy_from_page(page, ctx->base, (uprobe_opcode_t *) &old_opcode, 5);
+	uprobe_copy_from_page(page, ctx->base, old_opcode, OPT_INSN_SIZE);
 
 	switch (ctx->expect) {
 	case EXPECT_SWBP:
 		if (is_swbp_insn(&old_opcode[0]))
 			return 1;
 		break;
-	case EXPECT_CALL:
-		if (is_call_insn(&old_opcode[0]))
+	case EXPECT_OPTIMIZED:
+		if (is_opt_insns(&old_opcode[0]))
+			return 1;
+		break;
+	case EXPECT_SWBP_OPTIMIZED:
+		if (is_swbp_opt_insns(&old_opcode[0]))
 			return 1;
 		break;
 	}
@@ -949,48 +907,137 @@ static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t *
 }
 
 /*
- * Modify multi-byte instructions by using INT3 breakpoints on SMP.
+ * Modify the optimized instruction by using INT3 breakpoints on SMP.
  * We completely avoid using stop_machine() here, and achieve the
  * synchronization using INT3 breakpoints and SMP cross-calls.
  * (borrowed comment from smp_text_poke_batch_finish)
  *
- * The way it is done:
- *   - Add an INT3 trap to the address that will be patched
- *   - SMP sync all CPUs
- *   - Update all but the first byte of the patched range
- *   - SMP sync all CPUs
- *   - Replace the first byte (INT3) by the first byte of the replacing opcode
- *   - SMP sync all CPUs
+ * For optimization (int3_update_optimize):
+ *   1) Start with the uprobe INT3 trap already installed
+ *   2) Update everything but the first byte
+ *   3) Replace the first INT3 by the first byte of the LEA instruction
+ *
+ * For unoptimization (int3_update_unoptimize):
+ *   1) Start with the optimized uprobe lea/call instructions
+ *   2) Add an INT3 trap to the address that will be patched
+ *   3) Restore the NOP bytes before the call opcode
+ *   4) Replace the first INT3 by the first byte of the NOP instruction
+ *
+ * Note that unoptimization deliberately keeps the call opcode and displacement
+ * in bytes 5..9. Those bytes become operands of the restored 10-byte NOP.
+ *
+ * Since there is only a single target uprobe-trampoline for the given nop10
+ * instruction address, the CALL instruction will not be changed across
+ * unoptimization/optimization cycles.
+ * Therefore, any task that is preempted at the CALL instruction is guaranteed
+ * to observe that CALL and not anything else.
  */
-static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
-		       unsigned long vaddr, char *insn, bool optimize)
+static int int3_update_optimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
+				unsigned long vaddr, uprobe_opcode_t *insn)
 {
-	uprobe_opcode_t int3 = UPROBE_SWBP_INSN;
 	struct write_opcode_ctx ctx = {
 		.base = vaddr,
 	};
 	int err;
 
 	/*
-	 * Write int3 trap.
+	 * 1) Initial state after set_swbp() installed the uprobe:
+	 *    cc 2e 0f 1f 84 00 00 00 00 00
 	 *
-	 * The swbp_optimize path comes with breakpoint already installed,
-	 * so we can skip this step for optimize == true.
+	 *    After a previous unoptimization bytes 5..9 may still contain the
+	 *    old call instruction, which remains valid for threads already there.
 	 */
-	if (!optimize) {
-		ctx.expect = EXPECT_CALL;
-		err = uprobe_write(auprobe, vma, vaddr, &int3, 1, verify_insn,
-				   true /* is_register */, false /* do_update_ref_ctr */,
-				   &ctx);
-		if (err)
-			return err;
-	}
+	smp_text_poke_sync_each_cpu();
+
+	/*
+	 * 2) Rewrite the LEA tail and call displacement:
+	 *    cc [8d 64 24 80 e8 d0 d1 d2 d3]
+	 */
+	ctx.expect = EXPECT_SWBP;
+	err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1,
+			   OPT_INSN_SIZE - 1, verify_insn,
+			   true /* is_register */, false /* do_update_ref_ctr */,
+			   &ctx);
+	if (err)
+		return err;
 
 	smp_text_poke_sync_each_cpu();
 
-	/* Write all but the first byte of the patched range. */
+	/*
+	 * 3) Publish the first LEA byte:
+	 *    [48] 8d 64 24 80 e8 d0 d1 d2 d3
+	 *
+	 *    From offset 0 this is:
+	 *      lea -0x80(%rsp), %rsp
+	 *      call <uprobe-trampoline>
+	 */
+	ctx.expect = EXPECT_SWBP_OPTIMIZED;
+	err = uprobe_write(auprobe, vma, vaddr, insn, 1, verify_insn,
+			   true /* is_register */, false /* do_update_ref_ctr */,
+			   &ctx);
+	if (err)
+		goto error;
+
+	smp_text_poke_sync_each_cpu();
+	return 0;
+
+error:
+	/*
+	 * In all intermediate states byte 0 is INT3, so EXPECT_SWBP covers every
+	 * case. Restore NOP bytes 1..4, but keep the valid CALL at bytes 5..9
+	 * for a thread that had already executed the LEA before a previous
+	 * unoptimization.
+	 */
 	ctx.expect = EXPECT_SWBP;
-	err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, 4, verify_insn,
+	uprobe_write(auprobe, vma, vaddr + 1, auprobe->insn + 1,
+		     LEA_INSN_SIZE - 1, verify_insn, true, false, &ctx);
+	smp_text_poke_sync_each_cpu();
+	return err;
+}
+
+static int int3_update_unoptimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
+				  unsigned long vaddr, uprobe_opcode_t *insn)
+{
+	uprobe_opcode_t int3 = UPROBE_SWBP_INSN;
+	struct write_opcode_ctx ctx = {
+		.base = vaddr,
+		.expect = EXPECT_OPTIMIZED,
+	};
+	int err;
+
+	/*
+	 * Note the first two uprobe_write calls use is_register=true, because they
+	 * are intermediate patching states while the probe is still active, so
+	 * we force the exclusive anonymous page for the update.
+	 * Also we use do_update_ref_ctr=false because refctr was already updated by
+	 * the initial int3 install.
+	 *
+	 * The last uprobe_write to nop10 instruction is called with is_register=false
+	 * and do_update_ref_ctr=true to trigger the refctr update and to instruct
+	 * uprobe_write to zap the anonymous page if it now matches the file page.
+	 *
+	 * 1) Initial optimized state:
+	 *    48 8d 64 24 80 e8 d0 d1 d2 d3
+	 *
+	 * 2) Trap new entries before restoring the NOP bytes:
+	 *    [cc] 8d 64 24 80 e8 d0 d1 d2 d3
+	 */
+	err = uprobe_write(auprobe, vma, vaddr, &int3, 1, verify_insn,
+			   true /* is_register */, false /* do_update_ref_ctr */,
+			   &ctx);
+	if (err)
+		return err;
+
+	smp_text_poke_sync_each_cpu();
+
+	/*
+	 * 3) Restore bytes 1..4 of the original NOP while keeping byte 0 trapped
+	 *    and byte 5 as CALL:
+	 *    cc [2e 0f 1f 84] e8 d0 d1 d2 d3
+	 */
+	ctx.expect = EXPECT_SWBP_OPTIMIZED;
+	err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1,
+			   LEA_INSN_SIZE - 1, verify_insn,
 			   true /* is_register */, false /* do_update_ref_ctr */,
 			   &ctx);
 	if (err)
@@ -999,13 +1046,16 @@ static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
 	smp_text_poke_sync_each_cpu();
 
 	/*
-	 * Write first byte.
+	 * 4) Publish the first byte of the original NOP:
+	 *    [66] 2e 0f 1f 84 e8 d0 d1 d2 d3
 	 *
-	 * The swbp_unoptimize needs to finish uprobe removal together
-	 * with ref_ctr update, using uprobe_write with proper flags.
+	 * From offset 0 this is the restored 10-byte NOP; the CALL opcode and
+	 * displacement are now only NOP operands.  Offset 5 still decodes as
+	 * CALL for a thread that was already there.
 	 */
+	ctx.expect = EXPECT_SWBP;
 	err = uprobe_write(auprobe, vma, vaddr, insn, 1, verify_insn,
-			   optimize /* is_register */, !optimize /* do_update_ref_ctr */,
+			   false /* is_register */, true /* do_update_ref_ctr */,
 			   &ctx);
 	if (err)
 		return err;
@@ -1017,17 +1067,25 @@ static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
 static int swbp_optimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
 			 unsigned long vaddr, unsigned long tramp)
 {
-	u8 call[5];
+	u8 insn[OPT_INSN_SIZE], *call = &insn[LEA_INSN_SIZE];
 
-	__text_gen_insn(call, CALL_INSN_OPCODE, (const void *) vaddr,
+	/*
+	 * We have nop10 instruction (with first byte overwritten to int3),
+	 * changing it to:
+	 *   lea -0x80(%rsp), %rsp
+	 *   call tramp
+	 */
+	memcpy(insn, lea_rsp, LEA_INSN_SIZE);
+	__text_gen_insn(call, CALL_INSN_OPCODE,
+			(const void *) (vaddr + LEA_INSN_SIZE),
 			(const void *) tramp, CALL_INSN_SIZE);
-	return int3_update(auprobe, vma, vaddr, call, true /* optimize */);
+	return int3_update_optimize(auprobe, vma, vaddr, insn);
 }
 
 static int swbp_unoptimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
 			   unsigned long vaddr)
 {
-	return int3_update(auprobe, vma, vaddr, auprobe->insn, false /* optimize */);
+	return int3_update_unoptimize(auprobe, vma, vaddr, auprobe->insn);
 }
 
 static int copy_from_vaddr(struct mm_struct *mm, unsigned long vaddr, void *dst, int len)
@@ -1049,19 +1107,19 @@ static bool __is_optimized(struct mm_struct *mm, uprobe_opcode_t *insn, unsigned
 	struct __packed __arch_relative_insn {
 		u8 op;
 		s32 raddr;
-	} *call = (struct __arch_relative_insn *) insn;
+	} *call = (struct __arch_relative_insn *)(insn + LEA_INSN_SIZE);
 
-	if (!is_call_insn(insn))
+	if (!is_opt_insns(insn))
 		return false;
-	return __in_uprobe_trampoline(mm, vaddr + 5 + call->raddr);
+	return __in_uprobe_trampoline(mm, vaddr + OPT_INSN_SIZE + call->raddr);
 }
 
 static int is_optimized(struct mm_struct *mm, unsigned long vaddr)
 {
-	uprobe_opcode_t insn[5];
+	uprobe_opcode_t insn[OPT_INSN_SIZE];
 	int err;
 
-	err = copy_from_vaddr(mm, vaddr, &insn, 5);
+	err = copy_from_vaddr(mm, vaddr, &insn, OPT_INSN_SIZE);
 	if (err)
 		return err;
 	return __is_optimized(mm, (uprobe_opcode_t *)&insn, vaddr);
@@ -1111,27 +1169,29 @@ int set_orig_insn(struct arch_uprobe *auprobe, struct vm_area_struct *vma,
 static int __arch_uprobe_optimize(struct arch_uprobe *auprobe, struct mm_struct *mm,
 				  unsigned long vaddr)
 {
-	struct uprobe_trampoline *tramp;
-	struct vm_area_struct *vma;
-	bool new = false;
-	int err = 0;
+	struct pt_regs *regs = task_pt_regs(current);
+	struct vm_area_struct *vma, *tramp;
+	bool new_mapping;
+	int ret;
 
+	if (!user_64bit_mode(regs))
+		return -EINVAL;
 	vma = find_vma(mm, vaddr);
 	if (!vma)
 		return -EINVAL;
-	tramp = get_uprobe_trampoline(vaddr, &new);
-	if (!tramp)
-		return -EINVAL;
-	err = swbp_optimize(auprobe, vma, vaddr, tramp->vaddr);
-	if (WARN_ON_ONCE(err) && new)
-		destroy_uprobe_trampoline(tramp);
-	return err;
+	tramp = get_uprobe_trampoline(mm, vaddr, &new_mapping);
+	if (IS_ERR(tramp))
+		return PTR_ERR(tramp);
+	ret = swbp_optimize(auprobe, vma, vaddr, tramp->vm_start);
+	if (WARN_ON_ONCE(ret) && new_mapping)
+		WARN_ON_ONCE(do_munmap(mm, tramp->vm_start, PAGE_SIZE, NULL));
+	return ret;
 }
 
 void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr)
 {
 	struct mm_struct *mm = current->mm;
-	uprobe_opcode_t insn[5];
+	uprobe_opcode_t insn[OPT_INSN_SIZE];
 
 	if (!should_optimize(auprobe))
 		return;
@@ -1142,7 +1202,7 @@ void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr)
 	 * Check if some other thread already optimized the uprobe for us,
 	 * if it's the case just go away silently.
 	 */
-	if (copy_from_vaddr(mm, vaddr, &insn, 5))
+	if (copy_from_vaddr(mm, vaddr, &insn, OPT_INSN_SIZE))
 		goto unlock;
 	if (!is_swbp_insn((uprobe_opcode_t*) &insn))
 		goto unlock;
@@ -1158,16 +1218,32 @@ void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr)
 	mmap_write_unlock(mm);
 }
 
+static bool is_optimizable_nop10(struct insn *insn)
+{
+	static const u8 nop10_prefix[] = {
+		0x66, 0x2e, 0x0f, 0x1f, 0x84
+	};
+
+	/*
+	 * Restrict this to the 10-byte NOP form whose last 5 bytes are
+	 * SIB/displacement operands. Unoptimization keeps the call opcode and
+	 * displacement in those bytes, so other NOP encodings are not safe.
+	 */
+	return insn->length == OPT_INSN_SIZE &&
+	       insn_is_nop(insn) &&
+	       !memcmp(insn->kaddr, nop10_prefix, ARRAY_SIZE(nop10_prefix));
+}
+
 static bool can_optimize(struct insn *insn, unsigned long vaddr)
 {
-	if (!insn->x86_64 || insn->length != 5)
+	if (!insn->x86_64)
 		return false;
 
-	if (!insn_is_nop(insn))
+	if (!is_optimizable_nop10(insn))
 		return false;
 
 	/* We can't do cross page atomic writes yet. */
-	return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= 5;
+	return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= OPT_INSN_SIZE;
 }
 #else /* 32-bit: */
 /*
@@ -1549,16 +1625,26 @@ static int push_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn)
  */
 int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long addr)
 {
+	enum insn_mode m = is_64bit_mm(mm) ? INSN_MODE_64 : INSN_MODE_32;
 	u8 fix_ip_or_call = UPROBE_FIX_IP;
 	struct insn insn;
 	int ret;
 
-	ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm));
-	if (ret)
-		return ret;
+	ret = insn_decode(&insn, auprobe->insn, sizeof(auprobe->insn), m);
+	if (ret < 0)
+		return -ENOEXEC;
 
-	if (can_optimize(&insn, addr))
+	/*
+	 * No need to check instruction in uprobe_init_insn in case we
+	 * are on top of optimizable nop10.
+	 */
+	if (can_optimize(&insn, addr)) {
 		set_bit(ARCH_UPROBE_FLAG_CAN_OPTIMIZE, &auprobe->flags);
+	} else {
+		ret = uprobe_init_insn(auprobe, &insn);
+		if (ret)
+			return ret;
+	}
 
 	ret = branch_setup_xol_ops(auprobe, &insn);
 	if (ret != -ENOSYS)
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 74e336d7f9dd2..2438b89a4620a 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -65,8 +65,14 @@ const_cpu_current_top_of_stack = cpu_current_top_of_stack;
 		__end_rodata_hpage_align = .;			\
 		__end_rodata_aligned = .;
 
+#if defined(CONFIG_MITIGATION_PAGE_TABLE_ISOLATION) || defined(CONFIG_MITIGATION_SRSO)
 #define ALIGN_ENTRY_TEXT_BEGIN	. = ALIGN(PMD_SIZE);
 #define ALIGN_ENTRY_TEXT_END	. = ALIGN(PMD_SIZE);
+#else
+#define ALIGN_ENTRY_TEXT_BEGIN
+#define ALIGN_ENTRY_TEXT_END
+#endif
+
 #else
 
 #define X86_ALIGN_RODATA_BEGIN
@@ -221,7 +227,7 @@ SECTIONS
 	 * references to such code must be patched out by alternatives, normally
 	 * by using X86_FEATURE_ALWAYS CPU feature bit.
 	 *
-	 * See static_cpu_has() for an example.
+	 * See cpu_feature_enabled() for an example.
 	 */
 	.altinstr_aux : AT(ADDR(.altinstr_aux) - LOAD_OFFSET) {
 		*(.altinstr_aux)
@@ -346,18 +352,6 @@ SECTIONS
 		__init_end = .;
 	}
 
-	/*
-	 * smp_locks might be freed after init
-	 * start/end must be page aligned
-	 */
-	. = ALIGN(PAGE_SIZE);
-	.smp_locks : AT(ADDR(.smp_locks) - LOAD_OFFSET) {
-		__smp_locks = .;
-		*(.smp_locks)
-		. = ALIGN(PAGE_SIZE);
-		__smp_locks_end = .;
-	}
-
 #ifdef CONFIG_X86_64
 	.data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) {
 		NOSAVE_DATA
diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index 77337c37324b1..0474604ab8a14 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -5,8 +5,8 @@ ccflags-$(CONFIG_KVM_WERROR) += -Werror
 
 include $(srctree)/virt/kvm/Makefile.kvm
 
-kvm-y			+= x86.o emulate.o irq.o lapic.o cpuid.o pmu.o mtrr.o \
-			   debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o
+kvm-y			+= x86.o emulate.o irq.o lapic.o cpuid.o msrs.o pmu.o regs.o \
+			   mtrr.o debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o
 
 kvm-$(CONFIG_X86_64) += mmu/tdp_iter.o mmu/tdp_mmu.o
 kvm-$(CONFIG_KVM_IOAPIC) += i8259.o i8254.o ioapic.o
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index 591d2294acd75..df92537dba716 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -28,6 +28,7 @@
 #include "trace.h"
 #include "pmu.h"
 #include "xen.h"
+#include "x86.h"
 
 /*
  * Unlike "struct cpuinfo_x86.x86_capability", kvm_cpu_caps doesn't need to be
@@ -369,7 +370,7 @@ static u32 cpuid_get_reg_unsafe(struct kvm_cpuid_entry2 *entry, u32 reg)
 	}
 }
 
-static int cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, u32 func,
+static int cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, u32 func, u32 index,
 			       bool include_partially_emulated);
 
 void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
@@ -399,7 +400,7 @@ void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
 		if (!entry)
 			continue;
 
-		cpuid_func_emulated(&emulated, cpuid.function, true);
+		cpuid_func_emulated(&emulated, cpuid.function, cpuid.index, true);
 
 		/*
 		 * A vCPU has a feature if it's supported by KVM and is enabled
@@ -1272,8 +1273,12 @@ void kvm_initialize_cpu_caps(void)
 		kvm_cpu_cap_set(X86_FEATURE_NULL_SEL_CLR_BASE);
 
 	kvm_cpu_cap_init(CPUID_C000_0001_EDX,
+		F(SM2),
+		F(SM2_EN),
 		F(XSTORE),
 		F(XSTORE_EN),
+		F(CCS),
+		F(CCS_EN),
 		F(XCRYPT),
 		F(XCRYPT_EN),
 		F(ACE2),
@@ -1282,6 +1287,12 @@ void kvm_initialize_cpu_caps(void)
 		F(PHE_EN),
 		F(PMM),
 		F(PMM_EN),
+		F(RNG2),
+		F(RNG2_EN),
+		F(PHE2),
+		F(PHE2_EN),
+		F(RSA),
+		F(RSA_EN),
 	);
 
 	/*
@@ -1368,11 +1379,15 @@ static struct kvm_cpuid_entry2 *do_host_cpuid(struct kvm_cpuid_array *array,
 	return entry;
 }
 
-static int cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, u32 func,
+static int cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, u32 func, u32 index,
 			       bool include_partially_emulated)
 {
 	memset(entry, 0, sizeof(*entry));
 
+	/* KVM doesn't currently emulate any non-zero indices. */
+	if (cpuid_function_is_indexed(func) && index)
+		return 0;
+
 	entry->function = func;
 	entry->index = 0;
 	entry->flags = 0;
@@ -1410,7 +1425,7 @@ static int __do_cpuid_func_emulated(struct kvm_cpuid_array *array, u32 func)
 	if (array->nent >= array->maxnent)
 		return -E2BIG;
 
-	array->nent += cpuid_func_emulated(&array->entries[array->nent], func, false);
+	array->nent += cpuid_func_emulated(&array->entries[array->nent], func, 0, false);
 	return 0;
 }
 
@@ -1507,7 +1522,7 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function)
 		union cpuid10_eax eax = { };
 		union cpuid10_edx edx = { };
 
-		if (!enable_pmu || !static_cpu_has(X86_FEATURE_ARCH_PERFMON)) {
+		if (!enable_pmu || !cpu_feature_enabled(X86_FEATURE_ARCH_PERFMON)) {
 			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
 			break;
 		}
@@ -1748,7 +1763,7 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function)
 		 * loop if said highest leaf has no subleaves indexed by ECX.
 		 */
 		if (entry->eax >= 0x8000001d &&
-		    (static_cpu_has(X86_FEATURE_LFENCE_RDTSC)
+		    (cpu_feature_enabled(X86_FEATURE_LFENCE_RDTSC)
 		     || !static_cpu_has_bug(X86_BUG_NULL_SEG)))
 			entry->eax = max(entry->eax, 0x80000021);
 		break;
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index b566ab5c75156..c1b21282187f3 100644
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -24,6 +24,7 @@
 #include "kvm_emulate.h"
 #include <linux/stringify.h>
 #include <asm/debugreg.h>
+#include <asm/insn-eval.h>
 #include <asm/nospec-branch.h>
 #include <asm/ibt.h>
 #include <asm/text-patching.h>
@@ -439,25 +440,6 @@ static void assign_masked(ulong *dest, ulong src, ulong mask)
 	*dest = (*dest & ~mask) | (src & mask);
 }
 
-static void assign_register(unsigned long *reg, u64 val, int bytes)
-{
-	/* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
-	switch (bytes) {
-	case 1:
-		*(u8 *)reg = (u8)val;
-		break;
-	case 2:
-		*(u16 *)reg = (u16)val;
-		break;
-	case 4:
-		*reg = (u32)val;
-		break;	/* 64b: zero-extend */
-	case 8:
-		*reg = val;
-		break;
-	}
-}
-
 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
 {
 	return (1UL << (ctxt->ad_bytes << 3)) - 1;
@@ -505,7 +487,7 @@ register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
 {
 	ulong *preg = reg_rmw(ctxt, reg);
 
-	assign_register(preg, *preg + inc, ctxt->ad_bytes);
+	insn_assign_reg(preg, *preg + inc, ctxt->ad_bytes);
 }
 
 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
@@ -1767,7 +1749,7 @@ static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
 
 static void write_register_operand(struct operand *op)
 {
-	return assign_register(op->addr.reg, op->val, op->bytes);
+	return insn_assign_reg(op->addr.reg, op->val, op->bytes);
 }
 
 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
@@ -2008,7 +1990,7 @@ static int em_popa(struct x86_emulate_ctxt *ctxt)
 		rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
 		if (rc != X86EMUL_CONTINUE)
 			break;
-		assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
+		insn_assign_reg(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
 		--reg;
 	}
 	return rc;
@@ -3298,8 +3280,12 @@ static int em_dr_write(struct x86_emulate_ctxt *ctxt)
 	else
 		val = ctxt->src.val & ~0U;
 
-	/* #UD condition is already handled. */
-	if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
+	/*
+	 * A #GP due to an illegal value should be impossible at this point, as
+	 * such #GPs have priority over MOV DR intercepts on SVM, i.e. KVM must
+	 * manually check the value *before* emulating the write.
+	 */
+	if (WARN_ON_ONCE(ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val)))
 		return emulate_gp(ctxt, 0);
 
 	/* Disable writeback. */
@@ -3834,31 +3820,53 @@ static int check_cr_access(struct x86_emulate_ctxt *ctxt)
 
 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
 {
+	bool is_intel = ctxt->ops->guest_cpuid_is_intel_compatible(ctxt);
 	int dr = ctxt->modrm_reg;
-	u64 cr4;
 
 	if (dr > 7)
 		return emulate_ud(ctxt);
 
-	cr4 = ctxt->ops->get_cr(ctxt, 4);
-	if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
+	if ((dr == 4 || dr == 5) && (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_DE))
 		return emulate_ud(ctxt);
 
+	/* Intel CPUs prioritize the DR7.GD=1 #DB over the CPL>0 #GP. */
+	if (!is_intel && ctxt->ops->cpl(ctxt))
+		return emulate_gp(ctxt, 0);
+
 	if (ctxt->ops->get_effective_dr7(ctxt) & DR7_GD)
 		return emulate_db(ctxt, DR6_BD);
 
+	if (is_intel && ctxt->ops->cpl(ctxt))
+		return emulate_gp(ctxt, 0);
+
 	return X86EMUL_CONTINUE;
 }
 
 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
 {
 	u64 new_val = ctxt->src.val64;
-	int dr = ctxt->modrm_reg;
+	int rc;
 
-	if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
-		return emulate_gp(ctxt, 0);
+	rc = check_dr_read(ctxt);
+	if (rc != X86EMUL_CONTINUE)
+		return rc;
+
+	switch (ctxt->modrm_reg) {
+	case 4:
+	case 6:
+		if (!kvm_dr6_valid(new_val))
+			return emulate_gp(ctxt, 0);
+		break;
+	case 5:
+	case 7:
+		if (!kvm_dr7_valid(new_val))
+			return emulate_gp(ctxt, 0);
+		break;
+	default:
+		break;
+	}
 
-	return check_dr_read(ctxt);
+	return X86EMUL_CONTINUE;
 }
 
 static int check_svme(struct x86_emulate_ctxt *ctxt)
@@ -4367,11 +4375,10 @@ static const struct opcode twobyte_table[256] = {
 	D(ImplicitOps | ModRM | SrcMem | NoAccess), /* NOP + 7 * reserved NOP */
 	/* 0x20 - 0x2F */
 	DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_access),
-	DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
+	DIP(ModRM | DstMem | Op3264 | NoMod, dr_read, check_dr_read),
 	IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
 						check_cr_access),
-	IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
-						check_dr_write),
+	IIP(ModRM | SrcMem | Op3264 | NoMod, em_dr_write, dr_write, check_dr_write),
 	N, N, N, N,
 	GP(ModRM | DstReg | SrcMem | Mov | Sse | Avx, &pfx_0f_28_0f_29),
 	GP(ModRM | DstMem | SrcReg | Mov | Sse | Avx, &pfx_0f_28_0f_29),
diff --git a/arch/x86/kvm/fpu.h b/arch/x86/kvm/fpu.h
index f898781b6a068..6b7b628f530da 100644
--- a/arch/x86/kvm/fpu.h
+++ b/arch/x86/kvm/fpu.h
@@ -3,8 +3,34 @@
 #ifndef __KVM_FPU_H_
 #define __KVM_FPU_H_
 
+#include <linux/kvm_host.h>
+
+#include <trace/events/kvm.h>
+
 #include <asm/fpu/api.h>
 
+/* Swap (qemu) user FPU context for the guest FPU context. */
+static inline void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
+{
+	if (KVM_BUG_ON(vcpu->arch.guest_fpu.fpstate->in_use, vcpu->kvm))
+		return;
+
+	/* Exclude PKRU, it's restored separately immediately after VM-Exit. */
+	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true);
+	trace_kvm_fpu(1);
+}
+
+/* When vcpu_run ends, restore user space FPU context. */
+static inline void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
+{
+	if (KVM_BUG_ON(!vcpu->arch.guest_fpu.fpstate->in_use, vcpu->kvm))
+		return;
+
+	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false);
+	++vcpu->stat.fpu_reload;
+	trace_kvm_fpu(0);
+}
+
 typedef u32		__attribute__((vector_size(16))) sse128_t;
 #define __sse128_u	union { sse128_t vec; u64 as_u64[2]; u32 as_u32[4]; }
 #define sse128_lo(x)	({ __sse128_u t; t.vec = x; t.as_u64[0]; })
diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c
index fd4eb1e561f7f..39b58f56308b7 100644
--- a/arch/x86/kvm/hyperv.c
+++ b/arch/x86/kvm/hyperv.c
@@ -206,14 +206,20 @@ static struct kvm_vcpu *get_vcpu_by_vpidx(struct kvm *kvm, u32 vpidx)
 
 static struct kvm_vcpu_hv_synic *synic_get(struct kvm *kvm, u32 vpidx)
 {
-	struct kvm_vcpu *vcpu;
 	struct kvm_vcpu_hv_synic *synic;
+	struct kvm_vcpu_hv *hv_vcpu;
+	struct kvm_vcpu *vcpu;
 
 	vcpu = get_vcpu_by_vpidx(kvm, vpidx);
-	if (!vcpu || !to_hv_vcpu(vcpu))
+	if (!vcpu)
 		return NULL;
-	synic = to_hv_synic(vcpu);
-	return (synic->active) ? synic : NULL;
+
+	hv_vcpu = to_hv_vcpu_safe(vcpu);
+	if (!hv_vcpu)
+		return NULL;
+
+	synic = &hv_vcpu->synic;
+	return READ_ONCE(synic->active) ? synic : NULL;
 }
 
 static void kvm_hv_notify_acked_sint(struct kvm_vcpu *vcpu, u32 sint)
@@ -593,8 +599,7 @@ static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer,
 {
 	struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer);
 
-	set_bit(stimer->index,
-		to_hv_vcpu(vcpu)->stimer_pending_bitmap);
+	set_bit(stimer->index, vcpu->arch.hyperv->stimer_pending_bitmap);
 	kvm_make_request(KVM_REQ_HV_STIMER, vcpu);
 	if (vcpu_kick)
 		kvm_vcpu_kick(vcpu);
@@ -608,8 +613,7 @@ static void stimer_cleanup(struct kvm_vcpu_hv_stimer *stimer)
 				    stimer->index);
 
 	hrtimer_cancel(&stimer->timer);
-	clear_bit(stimer->index,
-		  to_hv_vcpu(vcpu)->stimer_pending_bitmap);
+	clear_bit(stimer->index, vcpu->arch.hyperv->stimer_pending_bitmap);
 	stimer->msg_pending = false;
 	stimer->exp_time = 0;
 }
@@ -972,7 +976,6 @@ int kvm_hv_vcpu_init(struct kvm_vcpu *vcpu)
 	if (!hv_vcpu)
 		return -ENOMEM;
 
-	vcpu->arch.hyperv = hv_vcpu;
 	hv_vcpu->vcpu = vcpu;
 
 	synic_init(&hv_vcpu->synic);
@@ -988,6 +991,14 @@ int kvm_hv_vcpu_init(struct kvm_vcpu *vcpu)
 		spin_lock_init(&hv_vcpu->tlb_flush_fifo[i].write_lock);
 	}
 
+	/*
+	 * Ensure the structure is fully initialized before it's visible to
+	 * other tasks, as much of the state can be legally accessed without
+	 * holding vcpu->mutex.
+	 *
+	 * Pairs with the smp_load_acquire() in to_hv_vcpu_safe().
+	 */
+	smp_store_release(&vcpu->arch.hyperv, hv_vcpu);
 	return 0;
 }
 
@@ -1002,7 +1013,7 @@ int kvm_hv_activate_synic(struct kvm_vcpu *vcpu, bool dont_zero_synic_pages)
 
 	synic = to_hv_synic(vcpu);
 
-	synic->active = true;
+	WRITE_ONCE(synic->active, true);
 	synic->dont_zero_synic_pages = dont_zero_synic_pages;
 	synic->control = HV_SYNIC_CONTROL_ENABLE;
 	return 0;
@@ -1935,14 +1946,14 @@ static int kvm_hv_get_tlb_flush_entries(struct kvm *kvm, struct kvm_hv_hcall *hc
 	return kvm_hv_get_hc_data(kvm, hc, hc->rep_cnt, hc->rep_cnt, entries);
 }
 
-static void hv_tlb_flush_enqueue(struct kvm_vcpu *vcpu,
-				 struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo,
-				 u64 *entries, int count)
+static void hv_tlb_flush_enqueue(struct kvm_vcpu *vcpu, u64 *entries, int count,
+				 bool is_guest_mode)
 {
-	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
+	struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo;
 	u64 flush_all_entry = KVM_HV_TLB_FLUSHALL_ENTRY;
 
-	if (!hv_vcpu)
+	tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(vcpu, is_guest_mode);
+	if (!tlb_flush_fifo)
 		return;
 
 	spin_lock(&tlb_flush_fifo->write_lock);
@@ -1970,15 +1981,16 @@ static void hv_tlb_flush_enqueue(struct kvm_vcpu *vcpu,
 int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu)
 {
 	struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo;
-	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
 	u64 entries[KVM_HV_TLB_FLUSH_FIFO_SIZE];
 	int i, j, count;
 	gva_t gva;
 
-	if (!tdp_enabled || !hv_vcpu)
+	if (!tdp_enabled)
 		return -EINVAL;
 
 	tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(vcpu, is_guest_mode(vcpu));
+	if (!tlb_flush_fifo)
+		return -EINVAL;
 
 	count = kfifo_out(&tlb_flush_fifo->entries, entries, KVM_HV_TLB_FLUSH_FIFO_SIZE);
 
@@ -2017,7 +2029,6 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
 	struct kvm *kvm = vcpu->kvm;
 	struct hv_tlb_flush_ex flush_ex;
 	struct hv_tlb_flush flush;
-	struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo;
 	/*
 	 * Normally, there can be no more than 'KVM_HV_TLB_FLUSH_FIFO_SIZE'
 	 * entries on the TLB flush fifo. The last entry, however, needs to be
@@ -2045,10 +2056,9 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
 	 * flush).  Translate the address here so the memory can be uniformly
 	 * read with kvm_read_guest().
 	 */
-	if (!hc->fast && mmu_is_nested(vcpu)) {
-		hc->ingpa = kvm_x86_ops.nested_ops->translate_nested_gpa(
-					vcpu, hc->ingpa,
-					PFERR_GUEST_FINAL_MASK, NULL, 0);
+	if (!hc->fast) {
+		hc->ingpa = kvm_translate_gpa(vcpu, &vcpu->arch.gva_walk, hc->ingpa,
+					      PFERR_GUEST_FINAL_MASK, NULL, 0);
 		if (unlikely(hc->ingpa == INVALID_GPA))
 			return HV_STATUS_INVALID_HYPERCALL_INPUT;
 	}
@@ -2145,11 +2155,8 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
 	 * analyze it here, flush TLB regardless of the specified address space.
 	 */
 	if (all_cpus && !is_guest_mode(vcpu)) {
-		kvm_for_each_vcpu(i, v, kvm) {
-			tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(v, false);
-			hv_tlb_flush_enqueue(v, tlb_flush_fifo,
-					     tlb_flush_entries, hc->rep_cnt);
-		}
+		kvm_for_each_vcpu(i, v, kvm)
+			hv_tlb_flush_enqueue(v, tlb_flush_entries, hc->rep_cnt, false);
 
 		kvm_make_all_cpus_request(kvm, KVM_REQ_HV_TLB_FLUSH);
 	} else if (!is_guest_mode(vcpu)) {
@@ -2159,9 +2166,7 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
 			v = kvm_get_vcpu(kvm, i);
 			if (!v)
 				continue;
-			tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(v, false);
-			hv_tlb_flush_enqueue(v, tlb_flush_fifo,
-					     tlb_flush_entries, hc->rep_cnt);
+			hv_tlb_flush_enqueue(v, tlb_flush_entries, hc->rep_cnt, false);
 		}
 
 		kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH, vcpu_mask);
@@ -2171,7 +2176,7 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
 		bitmap_zero(vcpu_mask, KVM_MAX_VCPUS);
 
 		kvm_for_each_vcpu(i, v, kvm) {
-			hv_v = to_hv_vcpu(v);
+			hv_v = to_hv_vcpu_safe(v);
 
 			/*
 			 * The following check races with nested vCPUs entering/exiting
@@ -2192,9 +2197,7 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
 				continue;
 
 			__set_bit(i, vcpu_mask);
-			tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(v, true);
-			hv_tlb_flush_enqueue(v, tlb_flush_fifo,
-					     tlb_flush_entries, hc->rep_cnt);
+			hv_tlb_flush_enqueue(v, tlb_flush_entries, hc->rep_cnt, true);
 		}
 
 		kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH, vcpu_mask);
@@ -2407,7 +2410,7 @@ static int kvm_hv_hypercall_complete(struct kvm_vcpu *vcpu, u64 result)
 	ret = kvm_skip_emulated_instruction(vcpu);
 
 	if (tlb_lock_count)
-		kvm_x86_ops.nested_ops->hv_inject_synthetic_vmexit_post_tlb_flush(vcpu);
+		kvm_nested_call(hv_inject_synthetic_vmexit_post_tlb_flush)(vcpu);
 
 	return ret;
 }
@@ -2788,8 +2791,8 @@ int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid,
 	};
 	int i, nent = ARRAY_SIZE(cpuid_entries);
 
-	if (kvm_x86_ops.nested_ops->get_evmcs_version)
-		evmcs_ver = kvm_x86_ops.nested_ops->get_evmcs_version(vcpu);
+	if (kvm_nested_ops.enabled)
+		evmcs_ver = kvm_nested_call(get_evmcs_version)(vcpu);
 
 	if (cpuid->nent < nent)
 		return -E2BIG;
diff --git a/arch/x86/kvm/hyperv.h b/arch/x86/kvm/hyperv.h
index 65e89ed653497..622a6553e9ac2 100644
--- a/arch/x86/kvm/hyperv.h
+++ b/arch/x86/kvm/hyperv.h
@@ -22,10 +22,101 @@
 #define __ARCH_X86_KVM_HYPERV_H__
 
 #include <linux/kvm_host.h>
-#include "x86.h"
+
+#include "regs.h"
 
 #ifdef CONFIG_KVM_HYPERV
 
+
+/* Hyper-V SynIC timer */
+struct kvm_vcpu_hv_stimer {
+	struct hrtimer timer;
+	int index;
+	union hv_stimer_config config;
+	u64 count;
+	u64 exp_time;
+	struct hv_message msg;
+	bool msg_pending;
+};
+
+/* Hyper-V synthetic interrupt controller (SynIC)*/
+struct kvm_vcpu_hv_synic {
+	u64 version;
+	u64 control;
+	u64 msg_page;
+	u64 evt_page;
+	atomic64_t sint[HV_SYNIC_SINT_COUNT];
+	atomic_t sint_to_gsi[HV_SYNIC_SINT_COUNT];
+	DECLARE_BITMAP(auto_eoi_bitmap, 256);
+	DECLARE_BITMAP(vec_bitmap, 256);
+	bool active;
+	bool dont_zero_synic_pages;
+};
+
+/* The maximum number of entries on the TLB flush fifo. */
+#define KVM_HV_TLB_FLUSH_FIFO_SIZE (16)
+/*
+ * Note: the following 'magic' entry is made up by KVM to avoid putting
+ * anything besides GVA on the TLB flush fifo. It is theoretically possible
+ * to observe a request to flush 4095 PFNs starting from 0xfffffffffffff000
+ * which will look identical. KVM's action to 'flush everything' instead of
+ * flushing these particular addresses is, however, fully legitimate as
+ * flushing more than requested is always OK.
+ */
+#define KVM_HV_TLB_FLUSHALL_ENTRY  ((u64)-1)
+
+enum hv_tlb_flush_fifos {
+	HV_L1_TLB_FLUSH_FIFO,
+	HV_L2_TLB_FLUSH_FIFO,
+	HV_NR_TLB_FLUSH_FIFOS,
+};
+
+struct kvm_vcpu_hv_tlb_flush_fifo {
+	spinlock_t write_lock;
+	DECLARE_KFIFO(entries, u64, KVM_HV_TLB_FLUSH_FIFO_SIZE);
+};
+
+/* Hyper-V per vcpu emulation context */
+struct kvm_vcpu_hv {
+	struct kvm_vcpu *vcpu;
+	u32 vp_index;
+	u64 hv_vapic;
+	s64 runtime_offset;
+	struct kvm_vcpu_hv_synic synic;
+	struct kvm_hyperv_exit exit;
+	struct kvm_vcpu_hv_stimer stimer[HV_SYNIC_STIMER_COUNT];
+	DECLARE_BITMAP(stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT);
+	bool enforce_cpuid;
+	struct {
+		u32 features_eax; /* HYPERV_CPUID_FEATURES.EAX */
+		u32 features_ebx; /* HYPERV_CPUID_FEATURES.EBX */
+		u32 features_edx; /* HYPERV_CPUID_FEATURES.EDX */
+		u32 enlightenments_eax; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EAX */
+		u32 enlightenments_ebx; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EBX */
+		u32 syndbg_cap_eax; /* HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES.EAX */
+		u32 nested_eax; /* HYPERV_CPUID_NESTED_FEATURES.EAX */
+		u32 nested_ebx; /* HYPERV_CPUID_NESTED_FEATURES.EBX */
+	} cpuid_cache;
+
+	struct kvm_vcpu_hv_tlb_flush_fifo tlb_flush_fifo[HV_NR_TLB_FLUSH_FIFOS];
+
+	/*
+	 * Preallocated buffers for handling hypercalls that pass sparse vCPU
+	 * sets (for high vCPU counts, they're too large to comfortably fit on
+	 * the stack).
+	 */
+	u64 sparse_banks[HV_MAX_SPARSE_VCPU_BANKS];
+	DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS);
+
+	struct hv_vp_assist_page vp_assist_page;
+
+	struct {
+		u64 pa_page_gpa;
+		u64 vm_id;
+		u32 vp_id;
+	} nested;
+};
+
 /* "Hv#1" signature */
 #define HYPERV_CPUID_SIGNATURE_EAX 0x31237648
 
@@ -61,8 +152,22 @@ static inline struct kvm_hv *to_kvm_hv(struct kvm *kvm)
 	return &kvm->arch.hyperv;
 }
 
+static inline struct kvm_vcpu_hv *to_hv_vcpu_safe(struct kvm_vcpu *vcpu)
+{
+	/*
+	 * Ensure the HyperV structure is fully initialized when accessing it
+	 * without holding vcpu->mutex (or some other guarantee that KVM can't
+	 * concurrently instantiate the structure).
+	 *
+	 * Pairs with the smp_store_release() in kvm_hv_vcpu_init().
+	 */
+	return smp_load_acquire(&vcpu->arch.hyperv);
+}
+
 static inline struct kvm_vcpu_hv *to_hv_vcpu(struct kvm_vcpu *vcpu)
 {
+	kvm_lockdep_assert_vcpu_is_locked_or_unreachable(vcpu);
+
 	return vcpu->arch.hyperv;
 }
 
@@ -87,7 +192,7 @@ static inline struct kvm_hv_syndbg *to_hv_syndbg(struct kvm_vcpu *vcpu)
 
 static inline u32 kvm_hv_get_vpindex(struct kvm_vcpu *vcpu)
 {
-	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
+	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu_safe(vcpu);
 
 	return hv_vcpu ? hv_vcpu->vp_index : vcpu->vcpu_idx;
 }
@@ -141,7 +246,7 @@ static inline struct kvm_vcpu *hv_stimer_to_vcpu(struct kvm_vcpu_hv_stimer *stim
 
 static inline bool kvm_hv_has_stimer_pending(struct kvm_vcpu *vcpu)
 {
-	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
+	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu_safe(vcpu);
 
 	if (!hv_vcpu)
 		return false;
@@ -197,10 +302,13 @@ int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid,
 static inline struct kvm_vcpu_hv_tlb_flush_fifo *kvm_hv_get_tlb_flush_fifo(struct kvm_vcpu *vcpu,
 									   bool is_guest_mode)
 {
-	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
+	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu_safe(vcpu);
 	int i = is_guest_mode ? HV_L2_TLB_FLUSH_FIFO :
 				HV_L1_TLB_FLUSH_FIFO;
 
+	if (!hv_vcpu)
+		return NULL;
+
 	return &hv_vcpu->tlb_flush_fifo[i];
 }
 
@@ -208,10 +316,12 @@ static inline void kvm_hv_vcpu_purge_flush_tlb(struct kvm_vcpu *vcpu)
 {
 	struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo;
 
-	if (!to_hv_vcpu(vcpu) || !kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu))
+	if (!kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu))
 		return;
 
 	tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(vcpu, is_guest_mode(vcpu));
+	if (!tlb_flush_fifo)
+		return;
 
 	kfifo_reset_out(&tlb_flush_fifo->entries);
 }
diff --git a/arch/x86/kvm/ioapic.c b/arch/x86/kvm/ioapic.c
index 757667fb2bfa0..0d59b9c758c21 100644
--- a/arch/x86/kvm/ioapic.c
+++ b/arch/x86/kvm/ioapic.c
@@ -33,6 +33,7 @@
 #include "lapic.h"
 #include "irq.h"
 #include "trace.h"
+#include "x86.h"
 
 static int ioapic_service(struct kvm_ioapic *vioapic, int irq,
 		bool line_status);
diff --git a/arch/x86/kvm/ioapic.h b/arch/x86/kvm/ioapic.h
index 3dadae0936902..81b5765131167 100644
--- a/arch/x86/kvm/ioapic.h
+++ b/arch/x86/kvm/ioapic.h
@@ -113,6 +113,18 @@ void kvm_get_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state);
 void kvm_set_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state);
 void kvm_ioapic_scan_entry(struct kvm_vcpu *vcpu,
 			   ulong *ioapic_handled_vectors);
+
+static inline int __kvm_irq_line_state(unsigned long *irq_state,
+				       int irq_source_id, int level)
+{
+	/* Logical OR for level trig interrupt */
+	if (level)
+		__set_bit(irq_source_id, irq_state);
+	else
+		__clear_bit(irq_source_id, irq_state);
+
+	return !!(*irq_state);
+}
 #endif /* CONFIG_KVM_IOAPIC */
 
 static inline int ioapic_in_kernel(struct kvm *kvm)
diff --git a/arch/x86/kvm/irq.c b/arch/x86/kvm/irq.c
index 8c62c6d4d5c17..8719772882722 100644
--- a/arch/x86/kvm/irq.c
+++ b/arch/x86/kvm/irq.c
@@ -423,6 +423,13 @@ void kvm_arch_irq_routing_update(struct kvm *kvm)
 		kvm_make_scan_ioapic_request(kvm);
 }
 
+static bool kvm_irq_is_postable(struct kvm_lapic_irq *irq)
+{
+	/* We can only post Fixed and LowPrio IRQs */
+	return (irq->delivery_mode == APIC_DM_FIXED ||
+		irq->delivery_mode == APIC_DM_LOWEST);
+}
+
 static int kvm_pi_update_irte(struct kvm_kernel_irqfd *irqfd,
 			      struct kvm_kernel_irq_routing_entry *entry)
 {
@@ -488,8 +495,10 @@ int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
 
 	if (irqfd->irq_entry.type == KVM_IRQ_ROUTING_MSI) {
 		ret = kvm_pi_update_irte(irqfd, &irqfd->irq_entry);
-		if (ret)
+		if (ret) {
 			kvm->arch.nr_possible_bypass_irqs--;
+			irqfd->producer = NULL;
+		}
 	}
 	spin_unlock_irq(&kvm->irqfds.lock);
 
diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h
index 34f4a78a7a010..1a84ea31e7fd2 100644
--- a/arch/x86/kvm/irq.h
+++ b/arch/x86/kvm/irq.h
@@ -112,6 +112,12 @@ static inline int irqchip_in_kernel(struct kvm *kvm)
 	return mode != KVM_IRQCHIP_NONE;
 }
 
+int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v);
+int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu);
+int kvm_cpu_has_extint(struct kvm_vcpu *v);
+int kvm_cpu_get_extint(struct kvm_vcpu *v);
+int kvm_cpu_get_interrupt(struct kvm_vcpu *v);
+
 void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu);
 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu);
 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu);
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 6f30bbdddb5aa..62169ef5f3542 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -75,6 +75,16 @@ module_param(lapic_timer_advance, bool, 0444);
 /* step-by-step approximation to mitigate fluctuation */
 #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8
 
+/* apic attention bits */
+#define KVM_APIC_CHECK_VAPIC	0
+/*
+ * The following bit is set with PV-EOI, unset on EOI.
+ * We detect PV-EOI changes by guest by comparing
+ * this bit with PV-EOI in guest memory.
+ * See the implementation in apic_update_pv_eoi.
+ */
+#define KVM_APIC_PV_EOI_PENDING	1
+
 static bool __read_mostly vector_hashing_enabled = true;
 module_param_named(vector_hashing, vector_hashing_enabled, bool, 0444);
 
@@ -3371,6 +3381,12 @@ static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
 					struct kvm_lapic *apic)
 {
 	int vector;
+
+	if (unlikely(!pv_eoi_enabled(vcpu))) {
+		__clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
+		return;
+	}
+
 	/*
 	 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
 	 * and KVM_PV_EOI_ENABLED in guest memory as follows:
@@ -3382,8 +3398,6 @@ static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
 	 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
 	 * 	-> host enabled PV EOI, guest executed EOI.
 	 */
-	BUG_ON(!pv_eoi_enabled(vcpu));
-
 	if (pv_eoi_test_and_clr_pending(vcpu))
 		return;
 	vector = apic_set_eoi(apic);
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index 71970213dc1fc..bd1098c89d998 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -32,6 +32,39 @@ enum lapic_mode {
 	LAPIC_MODE_X2APIC = MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE,
 };
 
+/*
+ * Track the mode of the optimized logical map, as the rules for decoding the
+ * destination vary per mode.  Enabling the optimized logical map requires all
+ * software-enabled local APIs to be in the same mode, each addressable APIC to
+ * be mapped to only one MDA, and each MDA to map to at most one APIC.
+ */
+enum kvm_apic_logical_mode {
+	/* All local APICs are software disabled. */
+	KVM_APIC_MODE_SW_DISABLED,
+	/* All software enabled local APICs in xAPIC cluster addressing mode. */
+	KVM_APIC_MODE_XAPIC_CLUSTER,
+	/* All software enabled local APICs in xAPIC flat addressing mode. */
+	KVM_APIC_MODE_XAPIC_FLAT,
+	/* All software enabled local APICs in x2APIC mode. */
+	KVM_APIC_MODE_X2APIC,
+	/*
+	 * Optimized map disabled, e.g. not all local APICs in the same logical
+	 * mode, same logical ID assigned to multiple APICs, etc.
+	 */
+	KVM_APIC_MODE_MAP_DISABLED,
+};
+
+struct kvm_apic_map {
+	struct rcu_head rcu;
+	enum kvm_apic_logical_mode logical_mode;
+	u32 max_apic_id;
+	union {
+		struct kvm_lapic *xapic_flat_map[8];
+		struct kvm_lapic *xapic_cluster_map[16][4];
+	};
+	struct kvm_lapic *phys_map[];
+};
+
 enum lapic_lvt_entry {
 	LVT_TIMER,
 	LVT_THERMAL_MONITOR,
@@ -131,6 +164,9 @@ static inline int kvm_irq_delivery_to_apic(struct kvm *kvm,
 }
 
 void kvm_apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high);
+int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
+		    unsigned long ipi_bitmap_high, u32 min,
+		    unsigned long icr, int op_64_bit);
 
 int kvm_apic_set_base(struct kvm_vcpu *vcpu, u64 value, bool host_initiated);
 int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s);
@@ -237,6 +273,11 @@ static inline int kvm_lapic_latched_init(struct kvm_vcpu *vcpu)
 	return lapic_in_kernel(vcpu) && test_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
 }
 
+static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical)
+{
+	return dest_mode_logical ? APIC_DEST_LOGICAL : APIC_DEST_PHYSICAL;
+}
+
 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector);
 
 bool kvm_lapic_suppress_eoi_broadcast(struct kvm_lapic *apic);
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index e1bb663ebbd58..2ae7f9ed4cf86 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -4,10 +4,23 @@
 
 #include <linux/kvm_host.h>
 #include "regs.h"
-#include "x86.h"
 #include "cpuid.h"
 
+extern bool __read_mostly tdp_enabled;
+#ifdef CONFIG_X86_64
+extern bool __read_mostly tdp_mmu_enabled;
+#else
+#define tdp_mmu_enabled false
+#endif
 extern bool __read_mostly enable_mmio_caching;
+extern bool __read_mostly eager_page_split;
+
+#define KVM_MEMSLOT_PAGES_TO_MMU_PAGES_RATIO 50
+#define KVM_MIN_ALLOC_MMU_PAGES 64UL
+#define KVM_MMU_HASH_SHIFT 12
+#define KVM_NUM_MMU_PAGES (1 << KVM_MMU_HASH_SHIFT)
+#define KVM_MIN_FREE_MMU_PAGES 5
+#define KVM_REFILL_PAGES 25
 
 #define PT_WRITABLE_SHIFT 1
 #define PT_USER_SHIFT 2
@@ -90,6 +103,38 @@ static inline bool mmu_has_mbec(struct kvm_mmu *mmu)
 
 u8 kvm_mmu_get_max_tdp_level(void);
 
+void __init kvm_mmu_x86_module_init(void);
+int kvm_mmu_vendor_module_init(void);
+void kvm_mmu_vendor_module_exit(void);
+
+void kvm_mmu_destroy(struct kvm_vcpu *vcpu);
+int kvm_mmu_create(struct kvm_vcpu *vcpu);
+int kvm_mmu_init_vm(struct kvm *kvm);
+void kvm_mmu_uninit_vm(struct kvm *kvm);
+
+void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm,
+					    struct kvm_memory_slot *slot);
+
+void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu);
+void kvm_mmu_reset_context(struct kvm_vcpu *vcpu);
+void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
+				      const struct kvm_memory_slot *memslot,
+				      int start_level);
+void kvm_mmu_slot_try_split_huge_pages(struct kvm *kvm,
+				       const struct kvm_memory_slot *memslot,
+				       int target_level);
+void kvm_mmu_try_split_huge_pages(struct kvm *kvm,
+				  const struct kvm_memory_slot *memslot,
+				  u64 start, u64 end,
+				  int target_level);
+void kvm_mmu_recover_huge_pages(struct kvm *kvm,
+				const struct kvm_memory_slot *memslot);
+void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
+				   const struct kvm_memory_slot *memslot);
+void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen);
+void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long kvm_nr_mmu_pages);
+void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end);
+
 void kvm_mmu_set_mmio_spte_mask(u64 mmio_value, u64 mmio_mask, u64 access_mask);
 void kvm_mmu_set_mmio_spte_value(struct kvm *kvm, u64 mmio_value);
 void kvm_mmu_set_me_spte_mask(u64 me_value, u64 me_mask);
@@ -101,11 +146,24 @@ void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr4,
 void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
 			     int huge_page_level, bool accessed_dirty,
 			     bool mbec, gpa_t new_eptp);
+
+int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
+		       void *insn, int insn_len);
+void kvm_mmu_print_sptes(struct kvm_vcpu *vcpu, gpa_t gpa, const char *msg);
+void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
+void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
+			     u64 addr, unsigned long roots);
+void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid);
+void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd);
+
+void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level,
+		       int tdp_max_root_level, int tdp_huge_page_level);
+
 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu);
 int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
 				u64 fault_address, char *insn, int insn_len);
 void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
-					struct kvm_mmu *mmu);
+					struct kvm_pagewalk *pw);
 
 int kvm_mmu_load(struct kvm_vcpu *vcpu);
 void kvm_mmu_unload(struct kvm_vcpu *vcpu);
@@ -115,6 +173,25 @@ void kvm_mmu_sync_prev_roots(struct kvm_vcpu *vcpu);
 void kvm_mmu_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new,
 			 int bytes);
 
+bool __kvm_mmu_unprotect_gfn_and_retry(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
+				       bool always_retry);
+
+static inline bool kvm_mmu_unprotect_gfn_and_retry(struct kvm_vcpu *vcpu,
+						   gpa_t cr2_or_gpa)
+{
+	return __kvm_mmu_unprotect_gfn_and_retry(vcpu, cr2_or_gpa, false);
+}
+
+void kvm_mmu_free_roots(struct kvm *kvm, struct kvm_mmu *mmu,
+			ulong roots_to_free);
+void kvm_mmu_free_guest_mode_roots(struct kvm *kvm, struct kvm_mmu *mmu);
+gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
+			      struct x86_exception *exception);
+gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
+			       struct x86_exception *exception);
+gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
+				struct x86_exception *exception);
+
 static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu)
 {
 	if (kvm_check_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu))
@@ -169,21 +246,21 @@ static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu)
 }
 
 static inline void kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
-						    struct kvm_mmu *mmu)
+						    struct kvm_pagewalk *w)
 {
 	/*
 	 * When EPT is enabled, KVM may passthrough CR0.WP to the guest, i.e.
-	 * @mmu's snapshot of CR0.WP and thus all related paging metadata may
+	 * @w's snapshot of CR0.WP and thus all related paging metadata may
 	 * be stale.  Refresh CR0.WP and the metadata on-demand when checking
 	 * for permission faults.  Exempt nested MMUs, i.e. MMUs for shadowing
-	 * nEPT and nNPT, as CR0.WP is ignored in both cases.  Note, KVM does
-	 * need to refresh nested_mmu, a.k.a. the walker used to translate L2
-	 * GVAs to GPAs, as that "MMU" needs to honor L2's CR0.WP.
+	 * nEPT and nNPT, as CR0.WP is ignored in both cases.  Note, KVM will
+	 * still refresh gva_walk, so as to honor L2's CR0.WP when translating
+	 * L2 GVAs to GPAs.
 	 */
-	if (!tdp_enabled || mmu == &vcpu->arch.guest_mmu)
+	if (!tdp_enabled || w == &vcpu->arch.ngpa_walk)
 		return;
 
-	__kvm_mmu_refresh_passthrough_bits(vcpu, mmu);
+	__kvm_mmu_refresh_passthrough_bits(vcpu, w);
 }
 
 /*
@@ -194,7 +271,7 @@ static inline void kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
  * Return zero if the access does not fault; return the page fault error code
  * if the access faults.
  */
-static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
+static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
 				  unsigned pte_access, unsigned pte_pkey,
 				  u64 access)
 {
@@ -217,15 +294,16 @@ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 	u64 implicit_access = access & PFERR_IMPLICIT_ACCESS;
 	bool not_smap = ((rflags & X86_EFLAGS_AC) | implicit_access) == X86_EFLAGS_AC;
 	int index = (pfec | (not_smap ? PFERR_RSVD_MASK : 0)) >> 1;
+	struct kvm_page_format *fmt = &w->fmt;
 	u32 errcode = PFERR_PRESENT_MASK;
 	bool fault;
 
-	kvm_mmu_refresh_passthrough_bits(vcpu, mmu);
+	kvm_mmu_refresh_passthrough_bits(vcpu, w);
 
-	fault = (mmu->permissions[index] >> pte_access) & 1;
+	fault = (fmt->permissions[index] >> pte_access) & 1;
 
 	WARN_ON_ONCE(pfec & (PFERR_PK_MASK | PFERR_SS_MASK | PFERR_RSVD_MASK));
-	if (unlikely(mmu->pkru_mask)) {
+	if (unlikely(fmt->pkru_mask)) {
 		u32 pkru_bits, offset;
 
 		/*
@@ -239,7 +317,7 @@ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 		/* clear present bit, replace PFEC.RSVD with ACC_USER_MASK. */
 		offset = (pfec & ~1) | ((pte_access & PT_USER_MASK) ? PFERR_RSVD_MASK : 0);
 
-		pkru_bits &= mmu->pkru_mask >> offset;
+		pkru_bits &= fmt->pkru_mask >> offset;
 		errcode |= -pkru_bits & PFERR_PK_MASK;
 		fault |= (pkru_bits != 0);
 	}
@@ -261,12 +339,6 @@ static inline bool kvm_shadow_root_allocated(struct kvm *kvm)
 	return smp_load_acquire(&kvm->arch.shadow_root_allocated);
 }
 
-#ifdef CONFIG_X86_64
-extern bool tdp_mmu_enabled;
-#else
-#define tdp_mmu_enabled false
-#endif
-
 int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn);
 
 static inline bool kvm_memslots_have_rmaps(struct kvm *kvm)
@@ -300,17 +372,21 @@ static inline void kvm_update_page_stats(struct kvm *kvm, int level, int count)
 	atomic64_add(count, &kvm->stat.pages[level - 1]);
 }
 
+static inline bool mmu_is_nested(struct kvm_vcpu *vcpu)
+{
+	return vcpu->arch.mmu == &vcpu->arch.guest_mmu;
+}
+
 static inline gpa_t kvm_translate_gpa(struct kvm_vcpu *vcpu,
-				      struct kvm_mmu *mmu,
+				      struct kvm_pagewalk *w,
 				      gpa_t gpa, u64 access,
 				      struct x86_exception *exception,
 				      u64 pte_access)
 {
-	if (mmu != &vcpu->arch.nested_mmu)
+	if (!mmu_is_nested(vcpu) || w == &vcpu->arch.ngpa_walk)
 		return gpa;
-	return kvm_x86_ops.nested_ops->translate_nested_gpa(vcpu, gpa, access,
-							    exception,
-							    pte_access);
+	return kvm_nested_call(translate_nested_gpa)(vcpu, gpa, access,
+						     exception, pte_access);
 }
 
 static inline bool kvm_has_mirrored_tdp(const struct kvm *kvm)
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 234d0a95abf53..7757f0ad65c6f 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -105,7 +105,7 @@ module_param_named(flush_on_reuse, force_flush_and_sync_on_reuse, bool, 0644);
  * 2. while doing 1. it walks guest-physical to host-physical
  * If the hardware supports that we don't need to do shadow paging.
  */
-bool tdp_enabled = false;
+bool __read_mostly tdp_enabled = false;
 
 static bool __ro_after_init tdp_mmu_allowed;
 
@@ -115,6 +115,9 @@ module_param_named(tdp_mmu, tdp_mmu_enabled, bool, 0444);
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(tdp_mmu_enabled);
 #endif
 
+bool __read_mostly eager_page_split = true;
+module_param(eager_page_split, bool, 0644);
+
 static int max_huge_page_level __read_mostly;
 static int tdp_root_level __read_mostly;
 static int max_tdp_level __read_mostly;
@@ -225,9 +228,9 @@ BUILD_MMU_ROLE_REGS_ACCESSOR(efer, lma, EFER_LMA);
  * and the vCPU may be incorrect/irrelevant.
  */
 #define BUILD_MMU_ROLE_ACCESSOR(base_or_ext, reg, name)		\
-static inline bool __maybe_unused is_##reg##_##name(struct kvm_mmu *mmu)	\
+static inline bool __maybe_unused is_##reg##_##name(struct kvm_pagewalk *w)	\
 {								\
-	return !!(mmu->cpu_role. base_or_ext . reg##_##name);	\
+	return !!(w->cpu_role. base_or_ext . reg##_##name);	\
 }
 BUILD_MMU_ROLE_ACCESSOR(base, cr0, wp);
 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, pse);
@@ -238,19 +241,19 @@ BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, la57);
 BUILD_MMU_ROLE_ACCESSOR(base, efer, nx);
 BUILD_MMU_ROLE_ACCESSOR(ext,  efer, lma);
 
-static inline bool has_pferr_fetch(struct kvm_mmu *mmu)
+static inline bool has_pferr_fetch(struct kvm_pagewalk *w)
 {
-	return mmu->cpu_role.ext.has_pferr_fetch;
+	return w->cpu_role.ext.has_pferr_fetch;
 }
 
-static inline bool is_cr0_pg(struct kvm_mmu *mmu)
+static inline bool is_cr0_pg(struct kvm_pagewalk *w)
 {
-        return mmu->cpu_role.base.level > 0;
+        return w->cpu_role.base.level > 0;
 }
 
-static inline bool is_cr4_pae(struct kvm_mmu *mmu)
+static inline bool is_cr4_pae(struct kvm_pagewalk *w)
 {
-        return !mmu->cpu_role.base.has_4_byte_gpte;
+        return !w->cpu_role.base.has_4_byte_gpte;
 }
 
 static struct kvm_mmu_role_regs vcpu_to_role_regs(struct kvm_vcpu *vcpu)
@@ -270,12 +273,12 @@ static unsigned long get_guest_cr3(struct kvm_vcpu *vcpu)
 }
 
 static inline unsigned long kvm_mmu_get_guest_pgd(struct kvm_vcpu *vcpu,
-						  struct kvm_mmu *mmu)
+						  struct kvm_pagewalk *w)
 {
-	if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) && mmu->get_guest_pgd == get_guest_cr3)
+	if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) && w->get_guest_pgd == get_guest_cr3)
 		return kvm_read_cr3(vcpu);
 
-	return mmu->get_guest_pgd(vcpu);
+	return w->get_guest_pgd(vcpu);
 }
 
 static inline bool kvm_available_flush_remote_tlbs_range(void)
@@ -2476,12 +2479,14 @@ static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterato
 					struct kvm_vcpu *vcpu, hpa_t root,
 					u64 addr)
 {
+	struct kvm_pagewalk *w = vcpu->arch.mmu->w;
+
 	iterator->addr = addr;
 	iterator->shadow_addr = root;
 	iterator->level = vcpu->arch.mmu->root_role.level;
 
 	if (iterator->level >= PT64_ROOT_4LEVEL &&
-	    vcpu->arch.mmu->cpu_role.base.level < PT64_ROOT_4LEVEL &&
+	    w->cpu_role.base.level < PT64_ROOT_4LEVEL &&
 	    !vcpu->arch.mmu->root_role.direct)
 		iterator->level = PT32E_ROOT_LEVEL;
 
@@ -3690,6 +3695,7 @@ static u64 *fast_pf_get_last_sptep(struct kvm_vcpu *vcpu, gpa_t gpa, u64 *spte)
  */
 static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 {
+	struct kvm_mmu *mmu;
 	struct kvm_mmu_page *sp;
 	int ret = RET_PF_INVALID;
 	u64 spte;
@@ -3699,6 +3705,7 @@ static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 	if (!page_fault_can_be_fast(vcpu->kvm, fault))
 		return ret;
 
+	mmu = vcpu->arch.mmu;
 	walk_shadow_page_lockless_begin(vcpu);
 
 	do {
@@ -3734,7 +3741,7 @@ static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 		 * Need not check the access of upper level table entries since
 		 * they are always ACC_ALL.
 		 */
-		if (is_access_allowed(fault, spte)) {
+		if (!spte_permission_fault(mmu, spte, fault)) {
 			ret = RET_PF_SPURIOUS;
 			break;
 		}
@@ -3757,7 +3764,7 @@ static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 		 * that were write-protected for dirty-logging or access
 		 * tracking are handled here.  Don't bother checking if the
 		 * SPTE is writable to prioritize running with A/D bits enabled.
-		 * The is_access_allowed() check above handles the common case
+		 * The spte_permission_fault() check above handles the common case
 		 * of the fault being spurious, and the SPTE is known to be
 		 * shadow-present, i.e. except for access tracking restoration
 		 * making the new SPTE writable, the check is wasteful.
@@ -3782,7 +3789,7 @@ static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 
 		/* Verify that the fault can be handled in the fast path */
 		if (new_spte == spte ||
-		    !is_access_allowed(fault, new_spte))
+		    spte_permission_fault(mmu, new_spte, fault))
 			break;
 
 		/*
@@ -4088,12 +4095,13 @@ static int mmu_first_shadow_root_alloc(struct kvm *kvm)
 static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
 {
 	struct kvm_mmu *mmu = vcpu->arch.mmu;
+	struct kvm_pagewalk *w = mmu->w;
 	u64 pdptrs[4], pm_mask;
 	gfn_t root_gfn, root_pgd;
 	int quadrant, i, r;
 	hpa_t root;
 
-	root_pgd = kvm_mmu_get_guest_pgd(vcpu, mmu);
+	root_pgd = kvm_mmu_get_guest_pgd(vcpu, w);
 	root_gfn = (root_pgd & __PT_BASE_ADDR_MASK) >> PAGE_SHIFT;
 
 	if (!kvm_vcpu_is_visible_gfn(vcpu, root_gfn)) {
@@ -4105,9 +4113,9 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
 	 * On SVM, reading PDPTRs might access guest memory, which might fault
 	 * and thus might sleep.  Grab the PDPTRs before acquiring mmu_lock.
 	 */
-	if (mmu->cpu_role.base.level == PT32E_ROOT_LEVEL) {
+	if (w->cpu_role.base.level == PT32E_ROOT_LEVEL) {
 		for (i = 0; i < 4; ++i) {
-			pdptrs[i] = mmu->get_pdptr(vcpu, i);
+			pdptrs[i] = w->get_pdptr(vcpu, i);
 			if (!(pdptrs[i] & PT_PRESENT_MASK))
 				continue;
 
@@ -4129,7 +4137,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
 	 * Do we shadow a long mode page table? If so we need to
 	 * write-protect the guests page table root.
 	 */
-	if (mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL) {
+	if (w->cpu_role.base.level >= PT64_ROOT_4LEVEL) {
 		root = mmu_alloc_root(vcpu, root_gfn, 0,
 				      mmu->root_role.level);
 		mmu->root.hpa = root;
@@ -4168,7 +4176,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
 	for (i = 0; i < 4; ++i) {
 		WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i]));
 
-		if (mmu->cpu_role.base.level == PT32E_ROOT_LEVEL) {
+		if (w->cpu_role.base.level == PT32E_ROOT_LEVEL) {
 			if (!(pdptrs[i] & PT_PRESENT_MASK)) {
 				mmu->pae_root[i] = INVALID_PAE_ROOT;
 				continue;
@@ -4182,7 +4190,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
 		 * directory. Othwerise each PAE page direct shadows one guest
 		 * PAE page directory so that quadrant should be 0.
 		 */
-		quadrant = (mmu->cpu_role.base.level == PT32_ROOT_LEVEL) ? i : 0;
+		quadrant = (w->cpu_role.base.level == PT32_ROOT_LEVEL) ? i : 0;
 
 		root = mmu_alloc_root(vcpu, root_gfn, quadrant, PT32_ROOT_LEVEL);
 		mmu->pae_root[i] = root | pm_mask;
@@ -4206,6 +4214,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
 static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu)
 {
 	struct kvm_mmu *mmu = vcpu->arch.mmu;
+	struct kvm_pagewalk *w = mmu->w;
 	bool need_pml5 = mmu->root_role.level > PT64_ROOT_4LEVEL;
 	u64 *pml5_root = NULL;
 	u64 *pml4_root = NULL;
@@ -4218,7 +4227,7 @@ static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu)
 	 * on demand, as running a 32-bit L1 VMM on 64-bit KVM is very rare.
 	 */
 	if (mmu->root_role.direct ||
-	    mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL ||
+	    w->cpu_role.base.level >= PT64_ROOT_4LEVEL ||
 	    mmu->root_role.level < PT64_ROOT_4LEVEL)
 		return 0;
 
@@ -4323,7 +4332,7 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
 
 	vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
 
-	if (vcpu->arch.mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL) {
+	if (vcpu->arch.mmu->w->cpu_role.base.level >= PT64_ROOT_4LEVEL) {
 		hpa_t root = vcpu->arch.mmu->root.hpa;
 
 		if (!is_unsync_root(root))
@@ -4364,7 +4373,7 @@ void kvm_mmu_sync_prev_roots(struct kvm_vcpu *vcpu)
 	kvm_mmu_free_roots(vcpu->kvm, vcpu->arch.mmu, roots_to_free);
 }
 
-static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
+static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
 				  gpa_t vaddr, u64 access,
 				  struct x86_exception *exception)
 {
@@ -4376,7 +4385,7 @@ static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 	 * user-mode address if CR0.PG=0.  Therefore *include* ACC_USER_MASK in
 	 * the last argument to kvm_translate_gpa (which NPT does not use).
 	 */
-	return kvm_translate_gpa(vcpu, mmu, vaddr, access | PFERR_GUEST_FINAL_MASK,
+	return kvm_translate_gpa(vcpu, w, vaddr, access | PFERR_GUEST_FINAL_MASK,
 				 exception, ACC_ALL);
 }
 
@@ -4440,7 +4449,7 @@ static int get_sptes_lockless(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes,
 static bool get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)
 {
 	u64 sptes[PT64_ROOT_MAX_LEVEL + 1];
-	struct rsvd_bits_validate *rsvd_check;
+	struct kvm_page_format *rsvd_check;
 	int root, leaf, level;
 	bool reserved = false;
 
@@ -4461,7 +4470,7 @@ static bool get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)
 	if (!is_shadow_present_pte(sptes[leaf]))
 		leaf++;
 
-	rsvd_check = &vcpu->arch.mmu->shadow_zero_check;
+	rsvd_check = &vcpu->arch.mmu->fmt;
 
 	for (level = root; level >= leaf; level--)
 		reserved |= is_rsvd_spte(rsvd_check, sptes[level], level);
@@ -4565,43 +4574,12 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu,
 	if (arch.direct_map)
 		arch.cr3 = (unsigned long)INVALID_GPA;
 	else
-		arch.cr3 = kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu);
+		arch.cr3 = kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu->w);
 
 	return kvm_setup_async_pf(vcpu, fault->addr,
 				  kvm_vcpu_gfn_to_hva(vcpu, fault->gfn), &arch);
 }
 
-void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
-{
-	int r;
-
-	if (WARN_ON_ONCE(work->arch.error_code & PFERR_PRIVATE_ACCESS))
-		return;
-
-	if ((vcpu->arch.mmu->root_role.direct != work->arch.direct_map) ||
-	      work->wakeup_all)
-		return;
-
-	r = kvm_mmu_reload(vcpu);
-	if (unlikely(r))
-		return;
-
-	if (!vcpu->arch.mmu->root_role.direct &&
-	      work->arch.cr3 != kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu))
-		return;
-
-	r = kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, work->arch.error_code,
-				  true, NULL, NULL);
-
-	/*
-	 * Account fixed page faults, otherwise they'll never be counted, but
-	 * ignore stats for all other return times.  Page-ready "faults" aren't
-	 * truly spurious and never trigger emulation
-	 */
-	if (r == RET_PF_FIXED)
-		vcpu->stat.pf_fixed++;
-}
-
 static void kvm_mmu_finish_page_fault(struct kvm_vcpu *vcpu,
 				      struct kvm_page_fault *fault, int r)
 {
@@ -4958,7 +4936,7 @@ static int kvm_tdp_mmu_page_fault(struct kvm_vcpu *vcpu,
 }
 #endif
 
-int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
+static int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 {
 #ifdef CONFIG_X86_64
 	if (tdp_mmu_enabled)
@@ -4968,6 +4946,71 @@ int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 	return direct_page_fault(vcpu, fault);
 }
 
+static int kvm_mmu_do_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
+				 u64 err, bool prefetch, int *emulation_type,
+				 u8 *level)
+{
+	struct kvm_page_fault fault = {
+		.addr = cr2_or_gpa,
+		.error_code = err,
+		.exec = err & PFERR_FETCH_MASK,
+		.write = err & PFERR_WRITE_MASK,
+		.present = err & PFERR_PRESENT_MASK,
+		.rsvd = err & PFERR_RSVD_MASK,
+		.user = err & PFERR_USER_MASK,
+		.prefetch = prefetch,
+		.is_tdp = likely(vcpu->arch.mmu->page_fault == kvm_tdp_page_fault),
+		.nx_huge_page_workaround_enabled =
+			is_nx_huge_page_enabled(vcpu->kvm),
+
+		.max_level = KVM_MAX_HUGEPAGE_LEVEL,
+		.req_level = PG_LEVEL_4K,
+		.goal_level = PG_LEVEL_4K,
+		.is_private = err & PFERR_PRIVATE_ACCESS,
+
+		.pfn = KVM_PFN_ERR_FAULT,
+	};
+	int r;
+
+	if (vcpu->arch.mmu->root_role.direct) {
+		/*
+		 * Things like memslots don't understand the concept of a shared
+		 * bit. Strip it so that the GFN can be used like normal, and the
+		 * fault.addr can be used when the shared bit is needed.
+		 */
+		fault.gfn = gpa_to_gfn(fault.addr) & ~kvm_gfn_direct_bits(vcpu->kvm);
+		fault.slot = kvm_vcpu_gfn_to_memslot(vcpu, fault.gfn);
+	}
+
+	/*
+	 * With retpoline being active an indirect call is rather expensive,
+	 * so do a direct call in the most common case.
+	 */
+	if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) && fault.is_tdp)
+		r = kvm_tdp_page_fault(vcpu, &fault);
+	else
+		r = vcpu->arch.mmu->page_fault(vcpu, &fault);
+
+	/*
+	 * Not sure what's happening, but punt to userspace and hope that
+	 * they can fix it by changing memory to shared, or they can
+	 * provide a better error.
+	 */
+	if (r == RET_PF_EMULATE && fault.is_private) {
+		pr_warn_ratelimited("kvm: unexpected emulation request on private memory\n");
+		kvm_mmu_prepare_memory_fault_exit(vcpu, &fault);
+		return -EFAULT;
+	}
+
+	if (fault.write_fault_to_shadow_pgtable && emulation_type)
+		*emulation_type |= EMULTYPE_WRITE_PF_TO_SP;
+	if (level)
+		*level = fault.goal_level;
+
+	return r;
+}
+
+
 static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa,
 				 u64 error_code, u8 *level)
 {
@@ -5058,6 +5101,37 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
 	return min(range->size, end - range->gpa);
 }
 
+void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
+{
+	int r;
+
+	if (WARN_ON_ONCE(work->arch.error_code & PFERR_PRIVATE_ACCESS))
+		return;
+
+	if ((vcpu->arch.mmu->root_role.direct != work->arch.direct_map) ||
+	      work->wakeup_all)
+		return;
+
+	r = kvm_mmu_reload(vcpu);
+	if (unlikely(r))
+		return;
+
+	if (!vcpu->arch.mmu->root_role.direct &&
+	      work->arch.cr3 != kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu->w))
+		return;
+
+	r = kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, work->arch.error_code,
+				  true, NULL, NULL);
+
+	/*
+	 * Account fixed page faults, otherwise they'll never be counted, but
+	 * ignore stats for all other return times.  Page-ready "faults" aren't
+	 * truly spurious and never trigger emulation
+	 */
+	if (r == RET_PF_FIXED)
+		vcpu->stat.pf_fixed++;
+}
+
 #ifdef CONFIG_KVM_GUEST_MEMFD
 static void kvm_assert_gmem_invalidate_lock_held(struct kvm_memory_slot *slot)
 {
@@ -5141,7 +5215,6 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_tdp_mmu_map_private_pfn);
 static void nonpaging_init_context(struct kvm_mmu *context)
 {
 	context->page_fault = nonpaging_page_fault;
-	context->gva_to_gpa = nonpaging_gva_to_gpa;
 	context->sync_spte = NULL;
 }
 
@@ -5317,7 +5390,7 @@ static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
 #include "paging_tmpl.h"
 #undef PTTYPE
 
-static void __reset_rsvds_bits_mask(struct rsvd_bits_validate *rsvd_check,
+static void __reset_rsvds_bits_mask(struct kvm_page_format *fmt,
 				    u64 pa_bits_rsvd, int level, bool nx,
 				    bool gbpages, bool pse, bool amd)
 {
@@ -5325,7 +5398,7 @@ static void __reset_rsvds_bits_mask(struct rsvd_bits_validate *rsvd_check,
 	u64 nonleaf_bit8_rsvd = 0;
 	u64 high_bits_rsvd;
 
-	rsvd_check->bad_mt_xwr = 0;
+	fmt->bad_mt_xwr = 0;
 
 	if (!gbpages)
 		gbpages_bit_rsvd = rsvd_bits(7, 7);
@@ -5349,75 +5422,75 @@ static void __reset_rsvds_bits_mask(struct rsvd_bits_validate *rsvd_check,
 	switch (level) {
 	case PT32_ROOT_LEVEL:
 		/* no rsvd bits for 2 level 4K page table entries */
-		rsvd_check->rsvd_bits_mask[0][1] = 0;
-		rsvd_check->rsvd_bits_mask[0][0] = 0;
-		rsvd_check->rsvd_bits_mask[1][0] =
-			rsvd_check->rsvd_bits_mask[0][0];
+		fmt->rsvd_bits_mask[0][1] = 0;
+		fmt->rsvd_bits_mask[0][0] = 0;
+		fmt->rsvd_bits_mask[1][0] =
+			fmt->rsvd_bits_mask[0][0];
 
 		if (!pse) {
-			rsvd_check->rsvd_bits_mask[1][1] = 0;
+			fmt->rsvd_bits_mask[1][1] = 0;
 			break;
 		}
 
 		if (is_cpuid_PSE36())
 			/* 36bits PSE 4MB page */
-			rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
+			fmt->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
 		else
 			/* 32 bits PSE 4MB page */
-			rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
+			fmt->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
 		break;
 	case PT32E_ROOT_LEVEL:
-		rsvd_check->rsvd_bits_mask[0][2] = rsvd_bits(63, 63) |
+		fmt->rsvd_bits_mask[0][2] = rsvd_bits(63, 63) |
 						   high_bits_rsvd |
 						   rsvd_bits(5, 8) |
 						   rsvd_bits(1, 2);	/* PDPTE */
-		rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;	/* PDE */
-		rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;	/* PTE */
-		rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |
+		fmt->rsvd_bits_mask[0][1] = high_bits_rsvd;	/* PDE */
+		fmt->rsvd_bits_mask[0][0] = high_bits_rsvd;	/* PTE */
+		fmt->rsvd_bits_mask[1][1] = high_bits_rsvd |
 						   rsvd_bits(13, 20);	/* large page */
-		rsvd_check->rsvd_bits_mask[1][0] =
-			rsvd_check->rsvd_bits_mask[0][0];
+		fmt->rsvd_bits_mask[1][0] =
+			fmt->rsvd_bits_mask[0][0];
 		break;
 	case PT64_ROOT_5LEVEL:
-		rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd |
+		fmt->rsvd_bits_mask[0][4] = high_bits_rsvd |
 						   nonleaf_bit8_rsvd |
 						   rsvd_bits(7, 7);
-		rsvd_check->rsvd_bits_mask[1][4] =
-			rsvd_check->rsvd_bits_mask[0][4];
+		fmt->rsvd_bits_mask[1][4] =
+			fmt->rsvd_bits_mask[0][4];
 		fallthrough;
 	case PT64_ROOT_4LEVEL:
-		rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd |
+		fmt->rsvd_bits_mask[0][3] = high_bits_rsvd |
 						   nonleaf_bit8_rsvd |
 						   rsvd_bits(7, 7);
-		rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd |
+		fmt->rsvd_bits_mask[0][2] = high_bits_rsvd |
 						   gbpages_bit_rsvd;
-		rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;
-		rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
-		rsvd_check->rsvd_bits_mask[1][3] =
-			rsvd_check->rsvd_bits_mask[0][3];
-		rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd |
+		fmt->rsvd_bits_mask[0][1] = high_bits_rsvd;
+		fmt->rsvd_bits_mask[0][0] = high_bits_rsvd;
+		fmt->rsvd_bits_mask[1][3] =
+			fmt->rsvd_bits_mask[0][3];
+		fmt->rsvd_bits_mask[1][2] = high_bits_rsvd |
 						   gbpages_bit_rsvd |
 						   rsvd_bits(13, 29);
-		rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |
+		fmt->rsvd_bits_mask[1][1] = high_bits_rsvd |
 						   rsvd_bits(13, 20); /* large page */
-		rsvd_check->rsvd_bits_mask[1][0] =
-			rsvd_check->rsvd_bits_mask[0][0];
+		fmt->rsvd_bits_mask[1][0] =
+			fmt->rsvd_bits_mask[0][0];
 		break;
 	}
 }
 
 static void reset_guest_rsvds_bits_mask(struct kvm_vcpu *vcpu,
-					struct kvm_mmu *context)
+					struct kvm_pagewalk *w)
 {
-	__reset_rsvds_bits_mask(&context->guest_rsvd_check,
+	__reset_rsvds_bits_mask(&w->fmt,
 				vcpu->arch.reserved_gpa_bits,
-				context->cpu_role.base.level, is_efer_nx(context),
+				w->cpu_role.base.level, is_efer_nx(w),
 				guest_cpu_cap_has(vcpu, X86_FEATURE_GBPAGES),
-				is_cr4_pse(context),
+				is_cr4_pse(w),
 				guest_cpuid_is_amd_compatible(vcpu));
 }
 
-static void __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
+static void __reset_rsvds_bits_mask_ept(struct kvm_page_format *fmt,
 					u64 pa_bits_rsvd, bool execonly,
 					int huge_page_level)
 {
@@ -5430,18 +5503,18 @@ static void __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
 	if (huge_page_level < PG_LEVEL_2M)
 		large_2m_rsvd = rsvd_bits(7, 7);
 
-	rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd | rsvd_bits(3, 7);
-	rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd | rsvd_bits(3, 7);
-	rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd | rsvd_bits(3, 6) | large_1g_rsvd;
-	rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd | rsvd_bits(3, 6) | large_2m_rsvd;
-	rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
+	fmt->rsvd_bits_mask[0][4] = high_bits_rsvd | rsvd_bits(3, 7);
+	fmt->rsvd_bits_mask[0][3] = high_bits_rsvd | rsvd_bits(3, 7);
+	fmt->rsvd_bits_mask[0][2] = high_bits_rsvd | rsvd_bits(3, 6) | large_1g_rsvd;
+	fmt->rsvd_bits_mask[0][1] = high_bits_rsvd | rsvd_bits(3, 6) | large_2m_rsvd;
+	fmt->rsvd_bits_mask[0][0] = high_bits_rsvd;
 
 	/* large page */
-	rsvd_check->rsvd_bits_mask[1][4] = rsvd_check->rsvd_bits_mask[0][4];
-	rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];
-	rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd | rsvd_bits(12, 29) | large_1g_rsvd;
-	rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd | rsvd_bits(12, 20) | large_2m_rsvd;
-	rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];
+	fmt->rsvd_bits_mask[1][4] = fmt->rsvd_bits_mask[0][4];
+	fmt->rsvd_bits_mask[1][3] = fmt->rsvd_bits_mask[0][3];
+	fmt->rsvd_bits_mask[1][2] = high_bits_rsvd | rsvd_bits(12, 29) | large_1g_rsvd;
+	fmt->rsvd_bits_mask[1][1] = high_bits_rsvd | rsvd_bits(12, 20) | large_2m_rsvd;
+	fmt->rsvd_bits_mask[1][0] = fmt->rsvd_bits_mask[0][0];
 
 	bad_mt_xwr = 0xFFull << (2 * 8);	/* bits 3..5 must not be 2 */
 	bad_mt_xwr |= 0xFFull << (3 * 8);	/* bits 3..5 must not be 3 */
@@ -5452,13 +5525,13 @@ static void __reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
 		/* bits 0..2 must not be 100 unless VMX capabilities allow it */
 		bad_mt_xwr |= REPEAT_BYTE(1ull << 4);
 	}
-	rsvd_check->bad_mt_xwr = bad_mt_xwr;
+	fmt->bad_mt_xwr = bad_mt_xwr;
 }
 
 static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
-		struct kvm_mmu *context, bool execonly, int huge_page_level)
+		bool execonly, int huge_page_level)
 {
-	__reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,
+	__reset_rsvds_bits_mask_ept(&vcpu->arch.ngpa_walk.fmt,
 				    vcpu->arch.reserved_gpa_bits, execonly,
 				    huge_page_level);
 }
@@ -5480,13 +5553,13 @@ static void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
 	bool is_amd = true;
 	/* KVM doesn't use 2-level page tables for the shadow MMU. */
 	bool is_pse = false;
-	struct rsvd_bits_validate *shadow_zero_check;
+	struct kvm_page_format *fmt;
 	int i;
 
 	WARN_ON_ONCE(context->root_role.level < PT32E_ROOT_LEVEL);
 
-	shadow_zero_check = &context->shadow_zero_check;
-	__reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(),
+	fmt = &context->fmt;
+	__reset_rsvds_bits_mask(fmt, reserved_hpa_bits(),
 				context->root_role.level,
 				context->root_role.efer_nx,
 				guest_cpu_cap_has(vcpu, X86_FEATURE_GBPAGES),
@@ -5502,10 +5575,10 @@ static void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
 		 * Bits in shadow_me_mask but not in shadow_me_value are
 		 * not allowed to be set.
 		 */
-		shadow_zero_check->rsvd_bits_mask[0][i] |= shadow_me_mask;
-		shadow_zero_check->rsvd_bits_mask[1][i] |= shadow_me_mask;
-		shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_value;
-		shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_value;
+		fmt->rsvd_bits_mask[0][i] |= shadow_me_mask;
+		fmt->rsvd_bits_mask[1][i] |= shadow_me_mask;
+		fmt->rsvd_bits_mask[0][i] &= ~shadow_me_value;
+		fmt->rsvd_bits_mask[1][i] &= ~shadow_me_value;
 	}
 
 }
@@ -5522,18 +5595,18 @@ static inline bool boot_cpu_is_amd(void)
  */
 static void reset_tdp_shadow_zero_bits_mask(struct kvm_mmu *context)
 {
-	struct rsvd_bits_validate *shadow_zero_check;
+	struct kvm_page_format *fmt;
 	int i;
 
-	shadow_zero_check = &context->shadow_zero_check;
+	fmt = &context->fmt;
 
 	if (boot_cpu_is_amd())
-		__reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(),
+		__reset_rsvds_bits_mask(fmt, reserved_hpa_bits(),
 					context->root_role.level, true,
 					boot_cpu_has(X86_FEATURE_GBPAGES),
 					false, true);
 	else
-		__reset_rsvds_bits_mask_ept(shadow_zero_check,
+		__reset_rsvds_bits_mask_ept(fmt,
 					    reserved_hpa_bits(), false,
 					    max_huge_page_level);
 
@@ -5541,8 +5614,8 @@ static void reset_tdp_shadow_zero_bits_mask(struct kvm_mmu *context)
 		return;
 
 	for (i = context->root_role.level; --i >= 0;) {
-		shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask;
-		shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask;
+		fmt->rsvd_bits_mask[0][i] &= ~shadow_me_mask;
+		fmt->rsvd_bits_mask[1][i] &= ~shadow_me_mask;
 	}
 }
 
@@ -5553,7 +5626,7 @@ static void reset_tdp_shadow_zero_bits_mask(struct kvm_mmu *context)
 static void
 reset_ept_shadow_zero_bits_mask(struct kvm_mmu *context, bool execonly)
 {
-	__reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
+	__reset_rsvds_bits_mask_ept(&context->fmt,
 				    reserved_hpa_bits(), execonly,
 				    max_huge_page_level);
 }
@@ -5588,18 +5661,15 @@ reset_ept_shadow_zero_bits_mask(struct kvm_mmu *context, bool execonly)
 	 (14 & (access) ? 1 << 14 : 0) | \
 	 (15 & (access) ? 1 << 15 : 0))
 
-static void update_permission_bitmask(struct kvm_mmu *mmu, bool tdp, bool ept)
+static void __update_permission_bitmask(struct kvm_page_format *fmt, bool tdp,
+					bool ept, bool cr4_smep, bool cr4_smap,
+					bool cr0_wp, bool efer_nx)
 {
 	unsigned index;
 
 	const u16 w = ACC_BITS_MASK(ACC_WRITE_MASK);
 	const u16 r = ACC_BITS_MASK(ACC_READ_MASK);
 
-	bool cr4_smep = is_cr4_smep(mmu);
-	bool cr4_smap = is_cr4_smap(mmu);
-	bool cr0_wp = is_cr0_wp(mmu);
-	bool efer_nx = is_efer_nx(mmu);
-
 	/*
 	 * In hardware, page fault error codes are generated (as the name
 	 * suggests) on any kind of page fault.  permission_fault() and
@@ -5612,7 +5682,7 @@ static void update_permission_bitmask(struct kvm_mmu *mmu, bool tdp, bool ept)
 	 * permission_fault() to indicate accesses that are *not* subject to
 	 * SMAP restrictions.
 	 */
-	for (index = 0; index < ARRAY_SIZE(mmu->permissions); ++index) {
+	for (index = 0; index < ARRAY_SIZE(fmt->permissions); ++index) {
 		unsigned pfec = index << 1;
 
 		/*
@@ -5686,10 +5756,23 @@ static void update_permission_bitmask(struct kvm_mmu *mmu, bool tdp, bool ept)
 				smapf = (pfec & (PFERR_RSVD_MASK|PFERR_FETCH_MASK)) ? 0 : kf;
 		}
 
-		mmu->permissions[index] = ff | uf | wf | rf | smapf;
+		fmt->permissions[index] = ff | uf | wf | rf | smapf;
 	}
 }
 
+static void update_permission_bitmask(struct kvm_pagewalk *w, bool tdp, bool ept)
+{
+	__update_permission_bitmask(&w->fmt, tdp, ept,
+				    is_cr4_smep(w), is_cr4_smap(w),
+				    is_cr0_wp(w), is_efer_nx(w));
+}
+
+static void update_spte_permission_bitmask(struct kvm_mmu *mmu, bool tdp, bool ept)
+{
+	__update_permission_bitmask(&mmu->fmt, tdp, ept,
+				    mmu->root_role.cr4_smep, false, true, true);
+}
+
 /*
 * PKU is an additional mechanism by which the paging controls access to
 * user-mode addresses based on the value in the PKRU register.  Protection
@@ -5714,19 +5797,19 @@ static void update_permission_bitmask(struct kvm_mmu *mmu, bool tdp, bool ept)
 * away both AD and WD.  For all reads or if the last condition holds, WD
 * only will be masked away.
 */
-static void update_pkru_bitmask(struct kvm_mmu *mmu)
+static void update_pkru_bitmask(struct kvm_pagewalk *w)
 {
 	unsigned bit;
 	bool wp;
 
-	mmu->pkru_mask = 0;
+	w->fmt.pkru_mask = 0;
 
-	if (!is_cr4_pke(mmu))
+	if (!is_cr4_pke(w))
 		return;
 
-	wp = is_cr0_wp(mmu);
+	wp = is_cr0_wp(w);
 
-	for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) {
+	for (bit = 0; bit < ARRAY_SIZE(w->fmt.permissions); ++bit) {
 		unsigned pfec, pkey_bits;
 		bool check_pkey, check_write, ff, uf, wf, pte_user;
 
@@ -5754,32 +5837,30 @@ static void update_pkru_bitmask(struct kvm_mmu *mmu)
 		/* PKRU.WD stops write access. */
 		pkey_bits |= (!!check_write) << 1;
 
-		mmu->pkru_mask |= (pkey_bits & 3) << pfec;
+		w->fmt.pkru_mask |= (pkey_bits & 3) << pfec;
 	}
 }
 
 static void reset_guest_paging_metadata(struct kvm_vcpu *vcpu,
-					struct kvm_mmu *mmu)
+					struct kvm_pagewalk *w)
 {
-	if (!is_cr0_pg(mmu))
+	if (!is_cr0_pg(w))
 		return;
 
-	reset_guest_rsvds_bits_mask(vcpu, mmu);
-	update_permission_bitmask(mmu, mmu == &vcpu->arch.guest_mmu, false);
-	update_pkru_bitmask(mmu);
+	reset_guest_rsvds_bits_mask(vcpu, w);
+	update_permission_bitmask(w, w == &vcpu->arch.ngpa_walk, false);
+	update_pkru_bitmask(w);
 }
 
 static void paging64_init_context(struct kvm_mmu *context)
 {
 	context->page_fault = paging64_page_fault;
-	context->gva_to_gpa = paging64_gva_to_gpa;
 	context->sync_spte = paging64_sync_spte;
 }
 
 static void paging32_init_context(struct kvm_mmu *context)
 {
 	context->page_fault = paging32_page_fault;
-	context->gva_to_gpa = paging32_gva_to_gpa;
 	context->sync_spte = paging32_sync_spte;
 }
 
@@ -5825,18 +5906,18 @@ static union kvm_cpu_role kvm_calc_cpu_role(struct kvm_vcpu *vcpu,
 }
 
 void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
-					struct kvm_mmu *mmu)
+					struct kvm_pagewalk *w)
 {
 	const bool cr0_wp = kvm_is_cr0_bit_set(vcpu, X86_CR0_WP);
 
 	BUILD_BUG_ON((KVM_MMU_CR0_ROLE_BITS & KVM_POSSIBLE_CR0_GUEST_BITS) != X86_CR0_WP);
 	BUILD_BUG_ON((KVM_MMU_CR4_ROLE_BITS & KVM_POSSIBLE_CR4_GUEST_BITS));
 
-	if (is_cr0_wp(mmu) == cr0_wp)
+	if (is_cr0_wp(w) == cr0_wp)
 		return;
 
-	mmu->cpu_role.base.cr0_wp = cr0_wp;
-	reset_guest_paging_metadata(vcpu, mmu);
+	w->cpu_role.base.cr0_wp = cr0_wp;
+	reset_guest_paging_metadata(vcpu, w);
 }
 
 static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu)
@@ -5894,52 +5975,37 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu,
 	struct kvm_mmu *context = &vcpu->arch.root_mmu;
 	union kvm_mmu_page_role root_role = kvm_calc_tdp_mmu_root_page_role(vcpu, cpu_role);
 
-	if (cpu_role.as_u64 == context->cpu_role.as_u64 &&
-	    root_role.word == context->root_role.word)
+	if (root_role.word == context->root_role.word)
 		return;
 
-	context->cpu_role.as_u64 = cpu_role.as_u64;
 	context->root_role.word = root_role.word;
 	context->page_fault = kvm_tdp_page_fault;
 	context->sync_spte = NULL;
-	context->get_guest_pgd = get_guest_cr3;
-	context->get_pdptr = kvm_pdptr_read;
-	context->inject_page_fault = kvm_inject_page_fault;
-
-	if (!is_cr0_pg(context))
-		context->gva_to_gpa = nonpaging_gva_to_gpa;
-	else if (is_cr4_pae(context))
-		context->gva_to_gpa = paging64_gva_to_gpa;
-	else
-		context->gva_to_gpa = paging32_gva_to_gpa;
 
-	reset_guest_paging_metadata(vcpu, context);
+	update_spte_permission_bitmask(context, true, shadow_xs_mask);
 	reset_tdp_shadow_zero_bits_mask(context);
 }
 
 static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *context,
-				    union kvm_cpu_role cpu_role,
 				    union kvm_mmu_page_role root_role)
 {
-	if (cpu_role.as_u64 == context->cpu_role.as_u64 &&
-	    root_role.word == context->root_role.word)
+	if (root_role.word == context->root_role.word)
 		return;
 
-	context->cpu_role.as_u64 = cpu_role.as_u64;
 	context->root_role.word = root_role.word;
 
-	if (!is_cr0_pg(context))
+	if (!is_cr0_pg(context->w))
 		nonpaging_init_context(context);
-	else if (is_cr4_pae(context))
+	else if (is_cr4_pae(context->w))
 		paging64_init_context(context);
 	else
 		paging32_init_context(context);
 
-	reset_guest_paging_metadata(vcpu, context);
+	update_spte_permission_bitmask(context, context == &vcpu->arch.guest_mmu, false);
 	reset_shadow_zero_bits_mask(vcpu, context);
 }
 
-static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu,
+static void init_kvm_shadow_mmu(struct kvm_vcpu *vcpu,
 				union kvm_cpu_role cpu_role)
 {
 	struct kvm_mmu *context = &vcpu->arch.root_mmu;
@@ -5961,7 +6027,28 @@ static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu,
 	 */
 	root_role.efer_nx = true;
 
-	shadow_mmu_init_context(vcpu, context, cpu_role, root_role);
+	shadow_mmu_init_context(vcpu, context, root_role);
+}
+
+static void init_kvm_page_walk(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
+			       union kvm_cpu_role cpu_role)
+{
+	if (cpu_role.as_u64 == w->cpu_role.as_u64)
+		return;
+
+	w->cpu_role.as_u64   = cpu_role.as_u64;
+	w->inject_page_fault = kvm_inject_page_fault;
+	w->get_pdptr         = kvm_pdptr_read;
+	w->get_guest_pgd     = get_guest_cr3;
+
+	if (!is_cr0_pg(w))
+		w->gva_to_gpa = nonpaging_gva_to_gpa;
+	else if (is_cr4_pae(w))
+		w->gva_to_gpa = paging64_gva_to_gpa;
+	else
+		w->gva_to_gpa = paging32_gva_to_gpa;
+
+	reset_guest_paging_metadata(vcpu, w);
 }
 
 void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr4,
@@ -5980,13 +6067,15 @@ void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr4,
 	WARN_ON_ONCE(cpu_role.base.direct || !cpu_role.base.guest_mode);
 	cpu_role.base.cr4_smep = (misc_ctl & SVM_MISC_ENABLE_GMET) != 0;
 
+	init_kvm_page_walk(vcpu, &vcpu->arch.ngpa_walk, cpu_role);
+
 	root_role = cpu_role.base;
 	root_role.level = kvm_mmu_get_tdp_level(vcpu);
 	if (root_role.level == PT64_ROOT_5LEVEL &&
 	    cpu_role.base.level == PT64_ROOT_4LEVEL)
 		root_role.passthrough = 1;
 
-	shadow_mmu_init_context(vcpu, context, cpu_role, root_role);
+	shadow_mmu_init_context(vcpu, context, root_role);
 	kvm_mmu_new_pgd(vcpu, nested_cr3);
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_init_shadow_npt_mmu);
@@ -6027,18 +6116,22 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
 		kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty,
 						   execonly, level, mbec);
 
-	if (new_mode.as_u64 != context->cpu_role.as_u64) {
+	struct kvm_pagewalk *ngpa_walk = &vcpu->arch.ngpa_walk;
+
+	if (new_mode.as_u64 != ngpa_walk->cpu_role.as_u64) {
 		/* EPT, and thus nested EPT, does not consume CR0, CR4, nor EFER. */
-		context->cpu_role.as_u64 = new_mode.as_u64;
+		ngpa_walk->cpu_role.as_u64 = new_mode.as_u64;
 		context->root_role.word = new_mode.base.word;
 
 		context->page_fault = ept_page_fault;
-		context->gva_to_gpa = ept_gva_to_gpa;
+		ngpa_walk->gva_to_gpa = ept_gva_to_gpa;
 		context->sync_spte = ept_sync_spte;
 
-		update_permission_bitmask(context, true, true);
-		context->pkru_mask = 0;
-		reset_rsvds_bits_mask_ept(vcpu, context, execonly, huge_page_level);
+		update_permission_bitmask(ngpa_walk, true, true);
+		ngpa_walk->fmt.pkru_mask = 0;
+		reset_rsvds_bits_mask_ept(vcpu, execonly, huge_page_level);
+
+		update_spte_permission_bitmask(context, true, true);
 		reset_ept_shadow_zero_bits_mask(context, execonly);
 	}
 
@@ -6046,68 +6139,19 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_init_shadow_ept_mmu);
 
-static void init_kvm_softmmu(struct kvm_vcpu *vcpu,
-			     union kvm_cpu_role cpu_role)
-{
-	struct kvm_mmu *context = &vcpu->arch.root_mmu;
-
-	kvm_init_shadow_mmu(vcpu, cpu_role);
-
-	context->get_guest_pgd     = get_guest_cr3;
-	context->get_pdptr         = kvm_pdptr_read;
-	context->inject_page_fault = kvm_inject_page_fault;
-}
-
-static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu,
-				union kvm_cpu_role new_mode)
-{
-	struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;
-
-	if (new_mode.as_u64 == g_context->cpu_role.as_u64)
-		return;
-
-	g_context->cpu_role.as_u64   = new_mode.as_u64;
-	g_context->get_guest_pgd     = get_guest_cr3;
-	g_context->get_pdptr         = kvm_pdptr_read;
-	g_context->inject_page_fault = kvm_inject_page_fault;
-
-	/*
-	 * L2 page tables are never shadowed, so there is no need to sync
-	 * SPTEs.
-	 */
-	g_context->sync_spte         = NULL;
-
-	/*
-	 * Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using
-	 * L1's nested page tables (e.g. EPT12). The nested translation
-	 * of l2_gva to l1_gpa is done by arch.nested_mmu.gva_to_gpa using
-	 * L2's page tables as the first level of translation and L1's
-	 * nested page tables as the second level of translation. Basically
-	 * the gva_to_gpa functions between mmu and nested_mmu are swapped.
-	 */
-	if (!is_paging(vcpu))
-		g_context->gva_to_gpa = nonpaging_gva_to_gpa;
-	else if (is_long_mode(vcpu))
-		g_context->gva_to_gpa = paging64_gva_to_gpa;
-	else if (is_pae(vcpu))
-		g_context->gva_to_gpa = paging64_gva_to_gpa;
-	else
-		g_context->gva_to_gpa = paging32_gva_to_gpa;
-
-	reset_guest_paging_metadata(vcpu, g_context);
-}
-
 void kvm_init_mmu(struct kvm_vcpu *vcpu)
 {
 	struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu);
 	union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, &regs);
 
-	if (mmu_is_nested(vcpu))
-		init_kvm_nested_mmu(vcpu, cpu_role);
-	else if (tdp_enabled)
-		init_kvm_tdp_mmu(vcpu, cpu_role);
-	else
-		init_kvm_softmmu(vcpu, cpu_role);
+	init_kvm_page_walk(vcpu, &vcpu->arch.gva_walk, cpu_role);
+
+	if (!mmu_is_nested(vcpu)) {
+		if (tdp_enabled)
+			init_kvm_tdp_mmu(vcpu, cpu_role);
+		else
+			init_kvm_shadow_mmu(vcpu, cpu_role);
+	}
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_init_mmu);
 
@@ -6127,10 +6171,8 @@ void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu)
 	 */
 	vcpu->arch.root_mmu.root_role.invalid = 1;
 	vcpu->arch.guest_mmu.root_role.invalid = 1;
-	vcpu->arch.nested_mmu.root_role.invalid = 1;
-	vcpu->arch.root_mmu.cpu_role.ext.valid = 0;
-	vcpu->arch.guest_mmu.cpu_role.ext.valid = 0;
-	vcpu->arch.nested_mmu.cpu_role.ext.valid = 0;
+	vcpu->arch.ngpa_walk.cpu_role.ext.valid = 0;
+	vcpu->arch.gva_walk.cpu_role.ext.valid = 0;
 	kvm_mmu_reset_context(vcpu);
 
 	KVM_BUG_ON(!kvm_can_set_cpuid_and_feature_msrs(vcpu), vcpu->kvm);
@@ -6617,22 +6659,31 @@ static void __kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu
 	write_unlock(&vcpu->kvm->mmu_lock);
 }
 
-void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
+void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
 			     u64 addr, unsigned long roots)
 {
+	struct kvm_mmu *mmu;
 	int i;
 
 	WARN_ON_ONCE(roots & ~KVM_MMU_ROOTS_ALL);
 
 	/* It's actually a GPA for vcpu->arch.guest_mmu.  */
-	if (mmu != &vcpu->arch.guest_mmu) {
+	if (w == &vcpu->arch.gva_walk) {
 		/* INVLPG on a non-canonical address is a NOP according to the SDM.  */
 		if (is_noncanonical_invlpg_address(addr, vcpu))
 			return;
 
 		kvm_x86_call(flush_tlb_gva)(vcpu, addr);
+
+		if (tdp_enabled)
+			return;
+
+		mmu = &vcpu->arch.root_mmu;
+	} else {
+		mmu = &vcpu->arch.guest_mmu;
 	}
 
+	/* Invalidate shadow pages, whether GPA->GVA or nGPA->GPA.  */
 	if (!mmu->sync_spte)
 		return;
 
@@ -6658,7 +6709,7 @@ void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
 	 * be synced when switching to that new cr3, so nothing needs to be
 	 * done here for them.
 	 */
-	kvm_mmu_invalidate_addr(vcpu, vcpu->arch.walk_mmu, gva, KVM_MMU_ROOTS_ALL);
+	kvm_mmu_invalidate_addr(vcpu, &vcpu->arch.gva_walk, gva, KVM_MMU_ROOTS_ALL);
 	++vcpu->stat.invlpg;
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_invlpg);
@@ -6680,7 +6731,7 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
 	}
 
 	if (roots)
-		kvm_mmu_invalidate_addr(vcpu, mmu, gva, roots);
+		kvm_mmu_invalidate_addr(vcpu, &vcpu->arch.gva_walk, gva, roots);
 	++vcpu->stat.invlpg;
 
 	/*
@@ -6725,11 +6776,12 @@ static void free_mmu_pages(struct kvm_mmu *mmu)
 	free_page((unsigned long)mmu->pml5_root);
 }
 
-static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
+static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, struct kvm_pagewalk *w)
 {
 	struct page *page;
 	int i;
 
+	mmu->w = w;
 	mmu->root.hpa = INVALID_PAGE;
 	mmu->root.pgd = 0;
 	mmu->mirror_root_hpa = INVALID_PAGE;
@@ -6795,13 +6847,12 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu)
 		vcpu->arch.mmu_shadow_page_cache.gfp_zero = __GFP_ZERO;
 
 	vcpu->arch.mmu = &vcpu->arch.root_mmu;
-	vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
 
-	ret = __kvm_mmu_create(vcpu, &vcpu->arch.guest_mmu);
+	ret = __kvm_mmu_create(vcpu, &vcpu->arch.guest_mmu, &vcpu->arch.ngpa_walk);
 	if (ret)
 		return ret;
 
-	ret = __kvm_mmu_create(vcpu, &vcpu->arch.root_mmu);
+	ret = __kvm_mmu_create(vcpu, &vcpu->arch.root_mmu, &vcpu->arch.gva_walk);
 	if (ret)
 		goto fail_allocate_root;
 
@@ -6871,20 +6922,11 @@ static void kvm_zap_obsolete_pages(struct kvm *kvm)
 	kvm_mmu_commit_zap_page(kvm, &invalid_list);
 }
 
-/*
- * Fast invalidate all shadow pages and use lock-break technique
- * to zap obsolete pages.
- *
- * It's required when memslot is being deleted or VM is being
- * destroyed, in these cases, we should ensure that KVM MMU does
- * not use any resource of the being-deleted slot or all slots
- * after calling the function.
- */
-static void kvm_mmu_zap_all_fast(struct kvm *kvm)
+static void __kvm_mmu_zap_all_fast_front_half(struct kvm *kvm)
 {
 	lockdep_assert_held(&kvm->slots_lock);
+	lockdep_assert_held_write(&kvm->mmu_lock);
 
-	write_lock(&kvm->mmu_lock);
 	trace_kvm_mmu_zap_all_fast(kvm);
 
 	/*
@@ -6921,8 +6963,12 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm)
 	kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS);
 
 	kvm_zap_obsolete_pages(kvm);
+}
 
-	write_unlock(&kvm->mmu_lock);
+static void __kvm_mmu_zap_all_fast_back_half(struct kvm *kvm)
+{
+	lockdep_assert_held(&kvm->slots_lock);
+	lockdep_assert_not_held(&kvm->mmu_lock);
 
 	/*
 	 * Zap the invalidated TDP MMU roots, all SPTEs must be dropped before
@@ -6936,6 +6982,24 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm)
 		kvm_tdp_mmu_zap_invalidated_roots(kvm, true);
 }
 
+/*
+ * Fast invalidate all shadow pages and use lock-break technique
+ * to zap obsolete pages.
+ *
+ * It's required when memslot is being deleted or VM is being
+ * destroyed, in these cases, we should ensure that KVM MMU does
+ * not use any resource of the being-deleted slot or all slots
+ * after calling the function.
+ */
+static void kvm_mmu_zap_all_fast(struct kvm *kvm)
+{
+	write_lock(&kvm->mmu_lock);
+	__kvm_mmu_zap_all_fast_front_half(kvm);
+	write_unlock(&kvm->mmu_lock);
+
+	__kvm_mmu_zap_all_fast_back_half(kvm);
+}
+
 int kvm_mmu_init_vm(struct kvm *kvm)
 {
 	int r, i;
@@ -7510,8 +7574,8 @@ static void kvm_mmu_zap_memslot_pages_and_flush(struct kvm *kvm,
 	kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
 }
 
-static void kvm_mmu_zap_memslot(struct kvm *kvm,
-				struct kvm_memory_slot *slot)
+void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
+				   struct kvm_memory_slot *slot)
 {
 	struct kvm_gfn_range range = {
 		.slot = slot,
@@ -7520,27 +7584,28 @@ static void kvm_mmu_zap_memslot(struct kvm *kvm,
 		.may_block = true,
 		.attr_filter = KVM_FILTER_PRIVATE | KVM_FILTER_SHARED,
 	};
+	bool zap_all = kvm->arch.vm_type == KVM_X86_DEFAULT_VM &&
+		       kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL);
 	bool flush;
 
 	write_lock(&kvm->mmu_lock);
-	flush = kvm_unmap_gfn_range(kvm, &range);
-	kvm_mmu_zap_memslot_pages_and_flush(kvm, slot, flush);
-	write_unlock(&kvm->mmu_lock);
-}
 
-static inline bool kvm_memslot_flush_zap_all(struct kvm *kvm)
-{
-	return kvm->arch.vm_type == KVM_X86_DEFAULT_VM &&
-	       kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL);
-}
+#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
+	if (slot->gmem.file)
+		kvm_arch_gmem_invalidate_range(kvm, &range);
+#endif
 
-void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
-				   struct kvm_memory_slot *slot)
-{
-	if (kvm_memslot_flush_zap_all(kvm))
-		kvm_mmu_zap_all_fast(kvm);
-	else
-		kvm_mmu_zap_memslot(kvm, slot);
+	if (zap_all) {
+		__kvm_mmu_zap_all_fast_front_half(kvm);
+	} else {
+		flush = kvm_unmap_gfn_range(kvm, &range);
+		kvm_mmu_zap_memslot_pages_and_flush(kvm, slot, flush);
+	}
+
+	write_unlock(&kvm->mmu_lock);
+
+	if (zap_all)
+		__kvm_mmu_zap_all_fast_back_half(kvm);
 }
 
 void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen)
diff --git a/arch/x86/kvm/mmu/mmu_internal.h b/arch/x86/kvm/mmu/mmu_internal.h
index 73cdcbccc89e8..c29002c601265 100644
--- a/arch/x86/kvm/mmu/mmu_internal.h
+++ b/arch/x86/kvm/mmu/mmu_internal.h
@@ -290,8 +290,6 @@ struct kvm_page_fault {
 	bool write_fault_to_shadow_pgtable;
 };
 
-int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
-
 /*
  * Return values of handle_mmio_page_fault(), mmu.page_fault(), fast_page_fault(),
  * and of course kvm_mmu_do_page_fault().
@@ -337,70 +335,6 @@ static inline void kvm_mmu_prepare_memory_fault_exit(struct kvm_vcpu *vcpu,
 				      fault->is_private);
 }
 
-static inline int kvm_mmu_do_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
-					u64 err, bool prefetch,
-					int *emulation_type, u8 *level)
-{
-	struct kvm_page_fault fault = {
-		.addr = cr2_or_gpa,
-		.error_code = err,
-		.exec = err & PFERR_FETCH_MASK,
-		.write = err & PFERR_WRITE_MASK,
-		.present = err & PFERR_PRESENT_MASK,
-		.rsvd = err & PFERR_RSVD_MASK,
-		.user = err & PFERR_USER_MASK,
-		.prefetch = prefetch,
-		.is_tdp = likely(vcpu->arch.mmu->page_fault == kvm_tdp_page_fault),
-		.nx_huge_page_workaround_enabled =
-			is_nx_huge_page_enabled(vcpu->kvm),
-
-		.max_level = KVM_MAX_HUGEPAGE_LEVEL,
-		.req_level = PG_LEVEL_4K,
-		.goal_level = PG_LEVEL_4K,
-		.is_private = err & PFERR_PRIVATE_ACCESS,
-
-		.pfn = KVM_PFN_ERR_FAULT,
-	};
-	int r;
-
-	if (vcpu->arch.mmu->root_role.direct) {
-		/*
-		 * Things like memslots don't understand the concept of a shared
-		 * bit. Strip it so that the GFN can be used like normal, and the
-		 * fault.addr can be used when the shared bit is needed.
-		 */
-		fault.gfn = gpa_to_gfn(fault.addr) & ~kvm_gfn_direct_bits(vcpu->kvm);
-		fault.slot = kvm_vcpu_gfn_to_memslot(vcpu, fault.gfn);
-	}
-
-	/*
-	 * With retpoline being active an indirect call is rather expensive,
-	 * so do a direct call in the most common case.
-	 */
-	if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) && fault.is_tdp)
-		r = kvm_tdp_page_fault(vcpu, &fault);
-	else
-		r = vcpu->arch.mmu->page_fault(vcpu, &fault);
-
-	/*
-	 * Not sure what's happening, but punt to userspace and hope that
-	 * they can fix it by changing memory to shared, or they can
-	 * provide a better error.
-	 */
-	if (r == RET_PF_EMULATE && fault.is_private) {
-		pr_warn_ratelimited("kvm: unexpected emulation request on private memory\n");
-		kvm_mmu_prepare_memory_fault_exit(vcpu, &fault);
-		return -EFAULT;
-	}
-
-	if (fault.write_fault_to_shadow_pgtable && emulation_type)
-		*emulation_type |= EMULTYPE_WRITE_PF_TO_SP;
-	if (level)
-		*level = fault.goal_level;
-
-	return r;
-}
-
 int kvm_mmu_max_mapping_level(struct kvm *kvm, struct kvm_page_fault *fault,
 			      const struct kvm_memory_slot *slot, gfn_t gfn);
 void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index df3ae0c7ec2c3..4ee7b03e762d0 100644
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -55,7 +55,7 @@
 	#define PT_LEVEL_BITS 9
 	#define PT_GUEST_DIRTY_SHIFT 9
 	#define PT_GUEST_ACCESSED_SHIFT 8
-	#define PT_HAVE_ACCESSED_DIRTY(mmu) (!(mmu)->cpu_role.base.ad_disabled)
+	#define PT_HAVE_ACCESSED_DIRTY(w) (!(w)->cpu_role.base.ad_disabled)
 	#define PT_MAX_FULL_LEVELS PT64_ROOT_MAX_LEVEL
 #else
 	#error Invalid PTTYPE value
@@ -106,13 +106,13 @@ static gfn_t gpte_to_gfn_lvl(pt_element_t gpte, int lvl)
 	return (gpte & PT_LVL_ADDR_MASK(lvl)) >> PAGE_SHIFT;
 }
 
-static inline void FNAME(protect_clean_gpte)(struct kvm_mmu *mmu, unsigned *access,
+static inline void FNAME(protect_clean_gpte)(struct kvm_pagewalk *w, unsigned *access,
 					     unsigned gpte)
 {
 	unsigned mask;
 
 	/* dirty bit is not supported, so no need to track it */
-	if (!PT_HAVE_ACCESSED_DIRTY(mmu))
+	if (!PT_HAVE_ACCESSED_DIRTY(w))
 		return;
 
 	BUILD_BUG_ON(PT_WRITABLE_MASK != ACC_WRITE_MASK);
@@ -124,7 +124,7 @@ static inline void FNAME(protect_clean_gpte)(struct kvm_mmu *mmu, unsigned *acce
 	*access &= mask;
 }
 
-static inline int FNAME(is_present_gpte)(struct kvm_mmu *mmu,
+static inline int FNAME(is_present_gpte)(struct kvm_pagewalk *w,
 					 unsigned long pte)
 {
 #if PTTYPE != PTTYPE_EPT
@@ -134,38 +134,40 @@ static inline int FNAME(is_present_gpte)(struct kvm_mmu *mmu,
 	 * For EPT, an entry is present if any of bits 2:0 are set.
 	 * With mode-based execute control, bit 10 also indicates presence.
 	 */
-	return pte & (7 | (mmu_has_mbec(mmu) ? VMX_EPT_USER_EXECUTABLE_MASK : 0));
+	return pte & (7 | (is_cr4_smep(w) ? VMX_EPT_USER_EXECUTABLE_MASK : 0));
 #endif
 }
 
-static bool FNAME(is_bad_mt_xwr)(struct rsvd_bits_validate *rsvd_check, u64 gpte)
+static bool FNAME(is_bad_mt_xwr)(struct kvm_page_format *fmt, u64 gpte)
 {
 #if PTTYPE != PTTYPE_EPT
 	return false;
 #else
-	return __is_bad_mt_xwr(rsvd_check, gpte);
+	return __is_bad_mt_xwr(fmt, gpte);
 #endif
 }
 
-static bool FNAME(is_rsvd_bits_set)(struct kvm_mmu *mmu, u64 gpte, int level)
+static bool FNAME(is_rsvd_bits_set)(struct kvm_page_format *fmt, u64 gpte, int level)
 {
-	return __is_rsvd_bits_set(&mmu->guest_rsvd_check, gpte, level) ||
-	       FNAME(is_bad_mt_xwr)(&mmu->guest_rsvd_check, gpte);
+	return __is_rsvd_bits_set(fmt, gpte, level) ||
+	       FNAME(is_bad_mt_xwr)(fmt, gpte);
 }
 
 static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu,
 				  struct kvm_mmu_page *sp, u64 *spte,
 				  u64 gpte)
 {
-	if (!FNAME(is_present_gpte)(vcpu->arch.mmu, gpte))
+	struct kvm_pagewalk *w = vcpu->arch.mmu->w;
+
+	if (!FNAME(is_present_gpte)(w, gpte))
 		goto no_present;
 
 	/* Prefetch only accessed entries (unless A/D bits are disabled). */
-	if (PT_HAVE_ACCESSED_DIRTY(vcpu->arch.mmu) &&
+	if (PT_HAVE_ACCESSED_DIRTY(w) &&
 	    !(gpte & PT_GUEST_ACCESSED_MASK))
 		goto no_present;
 
-	if (FNAME(is_rsvd_bits_set)(vcpu->arch.mmu, gpte, PG_LEVEL_4K))
+	if (FNAME(is_rsvd_bits_set)(&w->fmt, gpte, PG_LEVEL_4K))
 		goto no_present;
 
 	return false;
@@ -206,7 +208,7 @@ static inline unsigned FNAME(gpte_access)(u64 gpte)
 }
 
 static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu,
-					     struct kvm_mmu *mmu,
+					     struct kvm_pagewalk *w,
 					     struct guest_walker *walker,
 					     gpa_t addr, int write_fault)
 {
@@ -217,7 +219,7 @@ static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu,
 	int ret;
 
 	/* dirty/accessed bits are not supported, so no need to update them */
-	if (!PT_HAVE_ACCESSED_DIRTY(mmu))
+	if (!PT_HAVE_ACCESSED_DIRTY(w))
 		return 0;
 
 	for (level = walker->max_level; level >= walker->level; --level) {
@@ -233,7 +235,7 @@ static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu,
 				!(pte & PT_GUEST_DIRTY_MASK)) {
 			trace_kvm_mmu_set_dirty_bit(table_gfn, index, sizeof(pte));
 #if PTTYPE == PTTYPE_EPT
-			if (kvm_x86_ops.nested_ops->write_log_dirty(vcpu, addr))
+			if (kvm_nested_call(write_log_dirty)(vcpu, addr))
 				return -EINVAL;
 #endif
 			pte |= PT_GUEST_DIRTY_MASK;
@@ -278,7 +280,7 @@ static inline unsigned FNAME(gpte_pkeys)(struct kvm_vcpu *vcpu, u64 gpte)
 	return pkeys;
 }
 
-static inline bool FNAME(is_last_gpte)(struct kvm_mmu *mmu,
+static inline bool FNAME(is_last_gpte)(struct kvm_pagewalk *w,
 				       unsigned int level, unsigned int gpte)
 {
 	/*
@@ -296,7 +298,7 @@ static inline bool FNAME(is_last_gpte)(struct kvm_mmu *mmu,
 	 * is not reserved and does not indicate a large page at this level,
 	 * so clear PT_PAGE_SIZE_MASK in gpte if that is the case.
 	 */
-	gpte &= level - (PT32_ROOT_LEVEL + mmu->cpu_role.ext.cr4_pse);
+	gpte &= level - (PT32_ROOT_LEVEL + w->cpu_role.ext.cr4_pse);
 #endif
 	/*
 	 * PG_LEVEL_4K always terminates.  The RHS has bit 7 set
@@ -311,7 +313,7 @@ static inline bool FNAME(is_last_gpte)(struct kvm_mmu *mmu,
  * Fetch a guest pte for a guest virtual address, or for an L2's GPA.
  */
 static int FNAME(walk_addr_generic)(struct guest_walker *walker,
-				    struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
+				    struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
 				    gpa_t addr, u64 access)
 {
 	int ret;
@@ -340,16 +342,16 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 
 	trace_kvm_mmu_pagetable_walk(addr, access);
 retry_walk:
-	walker->level = mmu->cpu_role.base.level;
-	pte           = kvm_mmu_get_guest_pgd(vcpu, mmu);
-	have_ad       = PT_HAVE_ACCESSED_DIRTY(mmu);
+	walker->level = w->cpu_role.base.level;
+	pte           = kvm_mmu_get_guest_pgd(vcpu, w);
+	have_ad       = PT_HAVE_ACCESSED_DIRTY(w);
 
 #if PTTYPE == 64
 	walk_nx_mask = 1ULL << PT64_NX_SHIFT;
 	if (walker->level == PT32E_ROOT_LEVEL) {
-		pte = mmu->get_pdptr(vcpu, (addr >> 30) & 3);
+		pte = w->get_pdptr(vcpu, (addr >> 30) & 3);
 		trace_kvm_mmu_paging_element(pte, walker->level);
-		if (!FNAME(is_present_gpte)(mmu, pte))
+		if (!FNAME(is_present_gpte)(w, pte))
 			goto error;
 		--walker->level;
 	}
@@ -393,7 +395,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 		walker->table_gfn[walker->level - 1] = table_gfn;
 		walker->pte_gpa[walker->level - 1] = pte_gpa;
 
-		real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(table_gfn),
+		real_gpa = kvm_translate_gpa(vcpu, w, gfn_to_gpa(table_gfn),
 					     nested_access | PFERR_GUEST_PAGE_MASK,
 					     &walker->fault, 0);
 
@@ -422,10 +424,10 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 		 */
 		pte_access = pt_access & (pte ^ walk_nx_mask);
 
-		if (unlikely(!FNAME(is_present_gpte)(mmu, pte)))
+		if (unlikely(!FNAME(is_present_gpte)(w, pte)))
 			goto error;
 
-		if (unlikely(FNAME(is_rsvd_bits_set)(mmu, pte, walker->level))) {
+		if (unlikely(FNAME(is_rsvd_bits_set)(&w->fmt, pte, walker->level))) {
 			errcode = PFERR_RSVD_MASK | PFERR_PRESENT_MASK;
 			goto error;
 		}
@@ -434,14 +436,14 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 
 		/* Convert to ACC_*_MASK flags for struct guest_walker.  */
 		walker->pt_access[walker->level - 1] = FNAME(gpte_access)(pt_access ^ walk_nx_mask);
-	} while (!FNAME(is_last_gpte)(mmu, walker->level, pte));
+	} while (!FNAME(is_last_gpte)(w, walker->level, pte));
 
 	pte_pkey = FNAME(gpte_pkeys)(vcpu, pte);
 	accessed_dirty = have_ad ? pte_access & PT_GUEST_ACCESSED_MASK : 0;
 
 	/* Convert to ACC_*_MASK flags for struct guest_walker.  */
 	walker->pte_access = FNAME(gpte_access)(pte_access ^ walk_nx_mask);
-	errcode = permission_fault(vcpu, mmu, walker->pte_access, pte_pkey, access);
+	errcode = permission_fault(vcpu, w, walker->pte_access, pte_pkey, access);
 	if (unlikely(errcode))
 		goto error;
 
@@ -453,7 +455,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 		gfn += pse36_gfn_delta(pte);
 #endif
 
-	real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(gfn),
+	real_gpa = kvm_translate_gpa(vcpu, w, gfn_to_gpa(gfn),
 				     access | PFERR_GUEST_FINAL_MASK,
 				     &walker->fault, walker->pte_access);
 	if (real_gpa == INVALID_GPA)
@@ -462,7 +464,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 	walker->gfn = real_gpa >> PAGE_SHIFT;
 
 	if (!write_fault)
-		FNAME(protect_clean_gpte)(mmu, &walker->pte_access, pte);
+		FNAME(protect_clean_gpte)(w, &walker->pte_access, pte);
 	else
 		/*
 		 * On a write fault, fold the dirty bit into accessed_dirty.
@@ -473,7 +475,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 			(PT_GUEST_DIRTY_SHIFT - PT_GUEST_ACCESSED_SHIFT);
 
 	if (unlikely(!accessed_dirty)) {
-		ret = FNAME(update_accessed_dirty_bits)(vcpu, mmu, walker,
+		ret = FNAME(update_accessed_dirty_bits)(vcpu, w, walker,
 							addr, write_fault);
 		if (unlikely(ret < 0))
 			goto error;
@@ -485,7 +487,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 
 error:
 	errcode |= write_fault | user_fault;
-	if (fetch_fault && has_pferr_fetch(mmu))
+	if (fetch_fault && has_pferr_fetch(w))
 		errcode |= PFERR_FETCH_MASK;
 
 	walker->fault.vector = PF_VECTOR;
@@ -540,13 +542,13 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 		 * ACC_*_MASK flags!
 		 */
 		walker->fault.exit_qualification |= EPT_VIOLATION_RWX_TO_PROT(pte_access);
-		if (mmu_has_mbec(mmu))
+		if (is_cr4_smep(w))
 			walker->fault.exit_qualification |=
 				EPT_VIOLATION_USER_EXEC_TO_PROT(pte_access);
 	}
 #endif
 	walker->fault.address = addr;
-	walker->fault.nested_page_fault = mmu != vcpu->arch.walk_mmu;
+	walker->fault.nested_page_fault = w != &vcpu->arch.gva_walk;
 	walker->fault.async_page_fault = false;
 
 #if PTTYPE != PTTYPE_EPT
@@ -561,7 +563,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 static int FNAME(walk_addr)(struct guest_walker *walker,
 			    struct kvm_vcpu *vcpu, gpa_t addr, u64 access)
 {
-	return FNAME(walk_addr_generic)(walker, vcpu, vcpu->arch.mmu, addr,
+	return FNAME(walk_addr_generic)(walker, vcpu, vcpu->arch.mmu->w, addr,
 					access);
 }
 
@@ -577,7 +579,7 @@ FNAME(prefetch_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
 
 	gfn = gpte_to_gfn(gpte);
 	pte_access = sp->role.access & FNAME(gpte_access)(gpte);
-	FNAME(protect_clean_gpte)(vcpu->arch.mmu, &pte_access, gpte);
+	FNAME(protect_clean_gpte)(vcpu->arch.mmu->w, &pte_access, gpte);
 
 	return kvm_mmu_prefetch_sptes(vcpu, gfn, spte, 1, pte_access);
 }
@@ -660,7 +662,7 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
 	WARN_ON_ONCE(gw->gfn != base_gfn);
 	direct_access = gw->pte_access;
 
-	top_level = vcpu->arch.mmu->cpu_role.base.level;
+	top_level = vcpu->arch.mmu->w->cpu_role.base.level;
 	if (top_level == PT32E_ROOT_LEVEL)
 		top_level = PT32_ROOT_LEVEL;
 	/*
@@ -849,7 +851,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
 	 * otherwise KVM will cache incorrect access information in the SPTE.
 	 */
 	if (fault->write && !(walker.pte_access & ACC_WRITE_MASK) &&
-	    !is_cr0_wp(vcpu->arch.mmu) && !fault->user && fault->slot) {
+	    !is_cr0_wp(vcpu->arch.mmu->w) && !fault->user && fault->slot) {
 		walker.pte_access |= ACC_WRITE_MASK;
 		walker.pte_access &= ~ACC_USER_MASK;
 
@@ -859,7 +861,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
 		 * then we should prevent the kernel from executing it
 		 * if SMEP is enabled.
 		 */
-		if (is_cr4_smep(vcpu->arch.mmu))
+		if (is_cr4_smep(vcpu->arch.mmu->w))
 			walker.pte_access &= ~ACC_EXEC_MASK;
 	}
 #endif
@@ -894,7 +896,7 @@ static gpa_t FNAME(get_level1_sp_gpa)(struct kvm_mmu_page *sp)
 }
 
 /* Note, @addr is a GPA when gva_to_gpa() translates an L2 GPA to an L1 GPA. */
-static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
+static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
 			       gpa_t addr, u64 access,
 			       struct x86_exception *exception)
 {
@@ -904,10 +906,10 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 
 #ifndef CONFIG_X86_64
 	/* A 64-bit GVA should be impossible on 32-bit KVM. */
-	WARN_ON_ONCE((addr >> 32) && mmu == vcpu->arch.walk_mmu);
+	WARN_ON_ONCE((addr >> 32) && w == &vcpu->arch.gva_walk);
 #endif
 
-	r = FNAME(walk_addr_generic)(&walker, vcpu, mmu, addr, access);
+	r = FNAME(walk_addr_generic)(&walker, vcpu, w, addr, access);
 
 	if (r) {
 		gpa = gfn_to_gpa(walker.gfn);
@@ -957,7 +959,7 @@ static int FNAME(sync_spte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, int
 	gfn = gpte_to_gfn(gpte);
 	pte_access = sp->role.access;
 	pte_access &= FNAME(gpte_access)(gpte);
-	FNAME(protect_clean_gpte)(vcpu->arch.mmu, &pte_access, gpte);
+	FNAME(protect_clean_gpte)(vcpu->arch.mmu->w, &pte_access, gpte);
 
 	if (sync_mmio_spte(vcpu, &sp->spt[i], gfn, pte_access))
 		return 0;
diff --git a/arch/x86/kvm/mmu/spte.c b/arch/x86/kvm/mmu/spte.c
index 72d2394e089c9..5fc27e9733b38 100644
--- a/arch/x86/kvm/mmu/spte.c
+++ b/arch/x86/kvm/mmu/spte.c
@@ -281,9 +281,9 @@ bool make_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
 	if (prefetch && !synchronizing)
 		spte = mark_spte_for_access_track(spte);
 
-	WARN_ONCE(is_rsvd_spte(&vcpu->arch.mmu->shadow_zero_check, spte, level),
+	WARN_ONCE(is_rsvd_spte(&vcpu->arch.mmu->fmt, spte, level),
 		  "spte = 0x%llx, level = %d, rsvd bits = 0x%llx", spte, level,
-		  get_rsvd_bits(&vcpu->arch.mmu->shadow_zero_check, spte, level));
+		  get_rsvd_bits(&vcpu->arch.mmu->fmt, spte, level));
 
 	/*
 	 * Mark the memslot dirty *after* modifying it for access tracking.
diff --git a/arch/x86/kvm/mmu/spte.h b/arch/x86/kvm/mmu/spte.h
index 13eea94dd2128..e730717824b38 100644
--- a/arch/x86/kvm/mmu/spte.h
+++ b/arch/x86/kvm/mmu/spte.h
@@ -357,17 +357,6 @@ static inline bool is_last_spte(u64 pte, int level)
 	return (level == PG_LEVEL_4K) || is_large_pte(pte);
 }
 
-static inline bool is_executable_pte(u64 spte)
-{
-	/*
-	 * For now, return true if either the XS or XU bit is set
-	 * This function is only used for fast_page_fault,
-	 * which never processes shadow EPT, and regular page
-	 * tables always have XS==XU.
-	 */
-	return (spte & (shadow_xs_mask | shadow_xu_mask | shadow_nx_mask)) != shadow_nx_mask;
-}
-
 static inline kvm_pfn_t spte_to_pfn(u64 pte)
 {
 	return (pte & SPTE_BASE_ADDR_MASK) >> PAGE_SHIFT;
@@ -378,33 +367,33 @@ static inline bool is_accessed_spte(u64 spte)
 	return spte & shadow_accessed_mask;
 }
 
-static inline u64 get_rsvd_bits(struct rsvd_bits_validate *rsvd_check, u64 pte,
+static inline u64 get_rsvd_bits(struct kvm_page_format *fmt, u64 pte,
 				int level)
 {
 	int bit7 = (pte >> 7) & 1;
 
-	return rsvd_check->rsvd_bits_mask[bit7][level-1];
+	return fmt->rsvd_bits_mask[bit7][level-1];
 }
 
-static inline bool __is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check,
+static inline bool __is_rsvd_bits_set(struct kvm_page_format *fmt,
 				      u64 pte, int level)
 {
-	return pte & get_rsvd_bits(rsvd_check, pte, level);
+	return pte & get_rsvd_bits(fmt, pte, level);
 }
 
-static inline bool __is_bad_mt_xwr(struct rsvd_bits_validate *rsvd_check,
+static inline bool __is_bad_mt_xwr(struct kvm_page_format *fmt,
 				   u64 pte)
 {
 	if (pte & VMX_EPT_USER_EXECUTABLE_MASK)
 		pte |= VMX_EPT_EXECUTABLE_MASK;
-	return rsvd_check->bad_mt_xwr & BIT_ULL(pte & 0x3f);
+	return fmt->bad_mt_xwr & BIT_ULL(pte & 0x3f);
 }
 
-static __always_inline bool is_rsvd_spte(struct rsvd_bits_validate *rsvd_check,
+static __always_inline bool is_rsvd_spte(struct kvm_page_format *fmt,
 					 u64 spte, int level)
 {
-	return __is_bad_mt_xwr(rsvd_check, spte) ||
-	       __is_rsvd_bits_set(rsvd_check, spte, level);
+	return __is_bad_mt_xwr(fmt, spte) ||
+	       __is_rsvd_bits_set(fmt, spte, level);
 }
 
 /*
@@ -496,20 +485,40 @@ static inline bool is_mmu_writable_spte(u64 spte)
 }
 
 /*
- * Returns true if the access indicated by @fault is allowed by the existing
- * SPTE protections.  Note, the caller is responsible for checking that the
- * SPTE is a shadow-present, leaf SPTE (either before or after).
+ * Returns true if the access indicated by @fault is forbidden by the existing
+ * SPTE protections.
  */
-static inline bool is_access_allowed(struct kvm_page_fault *fault, u64 spte)
+static inline bool spte_permission_fault(struct kvm_mmu *mmu, u64 spte,
+					 struct kvm_page_fault *fault)
 {
-	if (fault->exec)
-		return is_executable_pte(spte);
+	unsigned pfec, pte_access;
 
-	if (fault->write)
-		return is_writable_pte(spte);
+	if (!is_shadow_present_pte(spte))
+		return true;
 
-	/* Fault was on Read access */
-	return spte & PT_PRESENT_MASK;
+	BUILD_BUG_ON(PT_PRESENT_MASK != ACC_READ_MASK);
+	BUILD_BUG_ON(PT_WRITABLE_MASK != ACC_WRITE_MASK);
+	BUILD_BUG_ON(VMX_EPT_READABLE_MASK != ACC_READ_MASK);
+	BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != ACC_WRITE_MASK);
+
+	/* strip nested paging fault error codes */
+	pte_access = spte & (PT_PRESENT_MASK | PT_WRITABLE_MASK);
+	if (shadow_nx_mask) {
+		pte_access |= spte & shadow_user_mask ? ACC_USER_MASK : 0;
+		pte_access |= spte & shadow_nx_mask ? 0 : ACC_EXEC_MASK;
+	} else {
+		pte_access |= spte & shadow_xs_mask ? ACC_EXEC_MASK : 0;
+		pte_access |= spte & shadow_xu_mask ? ACC_USER_EXEC_MASK : 0;
+	}
+
+	/*
+	 * RSVD is handled elsewhere, and is used for SMAP in the context
+	 * of accessing fmt.permissions[].  SPTEs never use PK or SS, as
+	 * they are not supported for shadow paging and irrelevant for TDP.
+	 */
+	pfec = fault->error_code & (
+		PFERR_WRITE_MASK | PFERR_USER_MASK | PFERR_FETCH_MASK);
+	return (mmu->fmt.permissions[pfec >> 1] >> pte_access) & 1;
 }
 
 /*
diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
index c1cbae65d239f..ce3f2efadb05b 100644
--- a/arch/x86/kvm/mmu/tdp_mmu.c
+++ b/arch/x86/kvm/mmu/tdp_mmu.c
@@ -1122,6 +1122,7 @@ static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu,
 					  struct kvm_page_fault *fault,
 					  struct tdp_iter *iter)
 {
+	struct kvm_mmu *mmu = vcpu->arch.mmu;
 	struct kvm_mmu_page *sp = sptep_to_sp(rcu_dereference(iter->sptep));
 	u64 new_spte;
 	int ret = RET_PF_FIXED;
@@ -1131,7 +1132,7 @@ static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu,
 		return RET_PF_RETRY;
 
 	if (is_shadow_present_pte(iter->old_spte) &&
-	    (fault->prefetch || is_access_allowed(fault, iter->old_spte)) &&
+	    (fault->prefetch || !spte_permission_fault(mmu, iter->old_spte, fault)) &&
 	    is_last_spte(iter->old_spte, iter->level)) {
 		WARN_ON_ONCE(fault->pfn != spte_to_pfn(iter->old_spte));
 		return RET_PF_SPURIOUS;
diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c
new file mode 100644
index 0000000000000..1e9244d690fcc
--- /dev/null
+++ b/arch/x86/kvm/msrs.c
@@ -0,0 +1,2745 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/kvm_host.h>
+#include <asm/intel_pt.h>
+#include <asm/vmx.h>
+
+#include "hyperv.h"
+#include "lapic.h"
+#include "msrs.h"
+#include "pmu.h"
+#include "trace.h"
+#include "vmx/vmx.h"
+#include "xen.h"
+#include "x86.h"
+
+bool __read_mostly ignore_msrs = 0;
+module_param(ignore_msrs, bool, 0644);
+
+bool __read_mostly report_ignored_msrs = true;
+module_param(report_ignored_msrs, bool, 0644);
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(report_ignored_msrs);
+
+/* EFER defaults:
+ * - enable syscall per default because its emulated by KVM
+ * - enable LME and LMA per default on 64 bit KVM
+ */
+#ifdef CONFIG_X86_64
+static
+u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
+#else
+static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
+#endif
+
+#define MAX_IO_MSRS 256
+
+struct msr_bitmap_range {
+	u32 flags;
+	u32 nmsrs;
+	u32 base;
+	unsigned long *bitmap;
+};
+
+struct kvm_x86_msr_filter {
+	u8 count;
+	bool default_allow:1;
+	struct msr_bitmap_range ranges[16];
+};
+
+/*
+ * Restoring the host value for MSRs that are only consumed when running in
+ * usermode, e.g. SYSCALL MSRs and TSC_AUX, can be deferred until the CPU
+ * returns to userspace, i.e. the kernel can run with the guest's value.
+ */
+#define KVM_MAX_NR_USER_RETURN_MSRS 16
+
+struct kvm_user_return_msrs {
+	struct user_return_notifier urn;
+	bool registered;
+	struct kvm_user_return_msr_values {
+		u64 host;
+		u64 curr;
+	} values[KVM_MAX_NR_USER_RETURN_MSRS];
+};
+
+u32 __read_mostly kvm_nr_uret_msrs;
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_nr_uret_msrs);
+static u32 __read_mostly kvm_uret_msrs_list[KVM_MAX_NR_USER_RETURN_MSRS];
+static DEFINE_PER_CPU(struct kvm_user_return_msrs, user_return_msrs);
+
+void kvm_destroy_user_return_msrs(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu)
+		WARN_ON_ONCE(per_cpu(user_return_msrs, cpu).registered);
+
+	kvm_nr_uret_msrs = 0;
+}
+
+static void kvm_on_user_return(struct user_return_notifier *urn)
+{
+	unsigned slot;
+	struct kvm_user_return_msrs *msrs
+		= container_of(urn, struct kvm_user_return_msrs, urn);
+	struct kvm_user_return_msr_values *values;
+
+	msrs->registered = false;
+	user_return_notifier_unregister(urn);
+
+	for (slot = 0; slot < kvm_nr_uret_msrs; ++slot) {
+		values = &msrs->values[slot];
+		if (values->host != values->curr) {
+			wrmsrq(kvm_uret_msrs_list[slot], values->host);
+			values->curr = values->host;
+		}
+	}
+}
+
+static int kvm_probe_user_return_msr(u32 msr)
+{
+	u64 val;
+	int ret;
+
+	preempt_disable();
+	ret = rdmsrq_safe(msr, &val);
+	if (ret)
+		goto out;
+	ret = wrmsrq_safe(msr, val);
+out:
+	preempt_enable();
+	return ret;
+}
+
+int kvm_add_user_return_msr(u32 msr)
+{
+	BUG_ON(kvm_nr_uret_msrs >= KVM_MAX_NR_USER_RETURN_MSRS);
+
+	if (kvm_probe_user_return_msr(msr))
+		return -1;
+
+	kvm_uret_msrs_list[kvm_nr_uret_msrs] = msr;
+	return kvm_nr_uret_msrs++;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_add_user_return_msr);
+
+int kvm_find_user_return_msr(u32 msr)
+{
+	int i;
+
+	for (i = 0; i < kvm_nr_uret_msrs; ++i) {
+		if (kvm_uret_msrs_list[i] == msr)
+			return i;
+	}
+	return -1;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_find_user_return_msr);
+
+void kvm_user_return_msr_cpu_online(void)
+{
+	struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs);
+	u64 value;
+	int i;
+
+	for (i = 0; i < kvm_nr_uret_msrs; ++i) {
+		rdmsrq_safe(kvm_uret_msrs_list[i], &value);
+		msrs->values[i].host = value;
+		msrs->values[i].curr = value;
+	}
+}
+
+static void kvm_user_return_register_notifier(struct kvm_user_return_msrs *msrs)
+{
+	if (!msrs->registered) {
+		msrs->urn.on_user_return = kvm_on_user_return;
+		user_return_notifier_register(&msrs->urn);
+		msrs->registered = true;
+	}
+}
+
+int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask)
+{
+	struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs);
+	int err;
+
+	value = (value & mask) | (msrs->values[slot].host & ~mask);
+	if (value == msrs->values[slot].curr)
+		return 0;
+	err = wrmsrq_safe(kvm_uret_msrs_list[slot], value);
+	if (err)
+		return 1;
+
+	msrs->values[slot].curr = value;
+	kvm_user_return_register_notifier(msrs);
+	return 0;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_user_return_msr);
+
+u64 kvm_get_user_return_msr(unsigned int slot)
+{
+	return this_cpu_ptr(&user_return_msrs)->values[slot].curr;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_user_return_msr);
+
+void drop_user_return_notifiers(void)
+{
+	struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs);
+
+	if (msrs->registered)
+		kvm_on_user_return(&msrs->urn);
+}
+
+/*
+ * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features) track
+ * the set of MSRs that KVM exposes to userspace through KVM_GET_MSRS,
+ * KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.  msrs_to_save holds MSRs that
+ * require host support, i.e. should be probed via RDMSR.  emulated_msrs holds
+ * MSRs that KVM emulates without strictly requiring host support.
+ * msr_based_features holds MSRs that enumerate features, i.e. are effectively
+ * CPUID leafs.  Note, msr_based_features isn't mutually exclusive with
+ * msrs_to_save and emulated_msrs.
+ */
+
+static const u32 msrs_to_save_base[] = {
+	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
+	MSR_STAR,
+#ifdef CONFIG_X86_64
+	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
+#endif
+	MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
+	MSR_IA32_FEAT_CTL, MSR_IA32_BNDCFGS, MSR_TSC_AUX,
+	MSR_IA32_SPEC_CTRL, MSR_IA32_TSX_CTRL,
+	MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH,
+	MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK,
+	MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B,
+	MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B,
+	MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B,
+	MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B,
+	MSR_IA32_UMWAIT_CONTROL,
+
+	MSR_IA32_XFD, MSR_IA32_XFD_ERR, MSR_IA32_XSS,
+
+	MSR_IA32_U_CET, MSR_IA32_S_CET,
+	MSR_IA32_PL0_SSP, MSR_IA32_PL1_SSP, MSR_IA32_PL2_SSP,
+	MSR_IA32_PL3_SSP, MSR_IA32_INT_SSP_TAB,
+	MSR_IA32_DEBUGCTLMSR,
+	MSR_IA32_LASTBRANCHFROMIP, MSR_IA32_LASTBRANCHTOIP,
+	MSR_IA32_LASTINTFROMIP, MSR_IA32_LASTINTTOIP,
+};
+
+static const u32 msrs_to_save_pmu[] = {
+	MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1,
+	MSR_ARCH_PERFMON_FIXED_CTR0 + 2,
+	MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS,
+	MSR_CORE_PERF_GLOBAL_CTRL,
+	MSR_IA32_PEBS_ENABLE, MSR_IA32_DS_AREA, MSR_PEBS_DATA_CFG,
+
+	/* This part of MSRs should match KVM_MAX_NR_INTEL_GP_COUNTERS. */
+	MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1,
+	MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3,
+	MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5,
+	MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7,
+	MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1,
+	MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3,
+	MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5,
+	MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7,
+
+	MSR_K7_EVNTSEL0, MSR_K7_EVNTSEL1, MSR_K7_EVNTSEL2, MSR_K7_EVNTSEL3,
+	MSR_K7_PERFCTR0, MSR_K7_PERFCTR1, MSR_K7_PERFCTR2, MSR_K7_PERFCTR3,
+
+	/* This part of MSRs should match KVM_MAX_NR_AMD_GP_COUNTERS. */
+	MSR_F15H_PERF_CTL0, MSR_F15H_PERF_CTL1, MSR_F15H_PERF_CTL2,
+	MSR_F15H_PERF_CTL3, MSR_F15H_PERF_CTL4, MSR_F15H_PERF_CTL5,
+	MSR_F15H_PERF_CTR0, MSR_F15H_PERF_CTR1, MSR_F15H_PERF_CTR2,
+	MSR_F15H_PERF_CTR3, MSR_F15H_PERF_CTR4, MSR_F15H_PERF_CTR5,
+
+	MSR_AMD64_PERF_CNTR_GLOBAL_CTL,
+	MSR_AMD64_PERF_CNTR_GLOBAL_STATUS,
+	MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR,
+	MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET,
+};
+
+static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_base) +
+			ARRAY_SIZE(msrs_to_save_pmu)];
+static unsigned num_msrs_to_save;
+
+static const u32 emulated_msrs_all[] = {
+	MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
+	MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
+
+#ifdef CONFIG_KVM_HYPERV
+	HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
+	HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
+	HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY,
+	HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2,
+	HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL,
+	HV_X64_MSR_RESET,
+	HV_X64_MSR_VP_INDEX,
+	HV_X64_MSR_VP_RUNTIME,
+	HV_X64_MSR_SCONTROL,
+	HV_X64_MSR_STIMER0_CONFIG,
+	HV_X64_MSR_VP_ASSIST_PAGE,
+	HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL,
+	HV_X64_MSR_TSC_EMULATION_STATUS, HV_X64_MSR_TSC_INVARIANT_CONTROL,
+	HV_X64_MSR_SYNDBG_OPTIONS,
+	HV_X64_MSR_SYNDBG_CONTROL, HV_X64_MSR_SYNDBG_STATUS,
+	HV_X64_MSR_SYNDBG_SEND_BUFFER, HV_X64_MSR_SYNDBG_RECV_BUFFER,
+	HV_X64_MSR_SYNDBG_PENDING_BUFFER,
+#endif
+
+	MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
+	MSR_KVM_PV_EOI_EN, MSR_KVM_ASYNC_PF_INT, MSR_KVM_ASYNC_PF_ACK,
+
+	MSR_IA32_TSC_ADJUST,
+	MSR_IA32_TSC_DEADLINE,
+	MSR_IA32_ARCH_CAPABILITIES,
+	MSR_IA32_PERF_CAPABILITIES,
+	MSR_IA32_MISC_ENABLE,
+	MSR_IA32_MCG_STATUS,
+	MSR_IA32_MCG_CTL,
+	MSR_IA32_MCG_EXT_CTL,
+	MSR_IA32_SMBASE,
+	MSR_SMI_COUNT,
+	MSR_PLATFORM_INFO,
+	MSR_MISC_FEATURES_ENABLES,
+	MSR_AMD64_VIRT_SPEC_CTRL,
+	MSR_AMD64_TSC_RATIO,
+	MSR_IA32_POWER_CTL,
+	MSR_IA32_UCODE_REV,
+
+	/*
+	 * KVM always supports the "true" VMX control MSRs, even if the host
+	 * does not.  The VMX MSRs as a whole are considered "emulated" as KVM
+	 * doesn't strictly require them to exist in the host (ignoring that
+	 * KVM would refuse to load in the first place if the core set of MSRs
+	 * aren't supported).
+	 */
+	MSR_IA32_VMX_BASIC,
+	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
+	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
+	MSR_IA32_VMX_TRUE_EXIT_CTLS,
+	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
+	MSR_IA32_VMX_MISC,
+	MSR_IA32_VMX_CR0_FIXED0,
+	MSR_IA32_VMX_CR4_FIXED0,
+	MSR_IA32_VMX_VMCS_ENUM,
+	MSR_IA32_VMX_PROCBASED_CTLS2,
+	MSR_IA32_VMX_EPT_VPID_CAP,
+	MSR_IA32_VMX_VMFUNC,
+
+	MSR_K7_HWCR,
+	MSR_KVM_POLL_CONTROL,
+};
+
+static u32 emulated_msrs[ARRAY_SIZE(emulated_msrs_all)];
+static unsigned num_emulated_msrs;
+
+/*
+ * List of MSRs that control the existence of MSR-based features, i.e. MSRs
+ * that are effectively CPUID leafs.  VMX MSRs are also included in the set of
+ * feature MSRs, but are handled separately to allow expedited lookups.
+ */
+static const u32 msr_based_features_all_except_vmx[] = {
+	MSR_AMD64_DE_CFG,
+	MSR_IA32_UCODE_REV,
+	MSR_IA32_ARCH_CAPABILITIES,
+	MSR_IA32_PERF_CAPABILITIES,
+	MSR_PLATFORM_INFO,
+};
+
+static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all_except_vmx) +
+			      (KVM_LAST_EMULATED_VMX_MSR - KVM_FIRST_EMULATED_VMX_MSR + 1)];
+static unsigned int num_msr_based_features;
+
+int kvm_get_msr_index_list(struct kvm_msr_list __user *user_msr_list)
+{
+	struct kvm_msr_list msr_list;
+	unsigned int n;
+
+	if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
+		return -EFAULT;
+
+	n = msr_list.nmsrs;
+	msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs;
+	if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
+		return -EFAULT;
+
+	if (n < msr_list.nmsrs)
+		return -E2BIG;
+
+	if (copy_to_user(user_msr_list->indices, &msrs_to_save,
+			 num_msrs_to_save * sizeof(u32)))
+		return -EFAULT;
+
+	if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
+			 &emulated_msrs, num_emulated_msrs * sizeof(u32)))
+		return -EFAULT;
+
+	return 0;
+}
+
+int kvm_get_feature_msr_index_list(struct kvm_msr_list __user *user_msr_list)
+{
+	struct kvm_msr_list msr_list;
+	unsigned int n;
+
+	if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
+		return -EFAULT;
+
+	n = msr_list.nmsrs;
+	msr_list.nmsrs = num_msr_based_features;
+	if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
+		return -EFAULT;
+
+	if (n < msr_list.nmsrs)
+		return -E2BIG;
+
+	if (copy_to_user(user_msr_list->indices, &msr_based_features,
+			 num_msr_based_features * sizeof(u32)))
+		return -EFAULT;
+
+	return 0;
+}
+
+/*
+ * All feature MSRs except uCode revID, which tracks the currently loaded uCode
+ * patch, are immutable once the vCPU model is defined.
+ */
+static bool kvm_is_immutable_feature_msr(u32 msr)
+{
+	int i;
+
+	if (msr >= KVM_FIRST_EMULATED_VMX_MSR && msr <= KVM_LAST_EMULATED_VMX_MSR)
+		return true;
+
+	for (i = 0; i < ARRAY_SIZE(msr_based_features_all_except_vmx); i++) {
+		if (msr == msr_based_features_all_except_vmx[i])
+			return msr != MSR_IA32_UCODE_REV;
+	}
+
+	return false;
+}
+
+static bool kvm_is_advertised_msr(u32 msr_index)
+{
+	unsigned int i;
+
+	for (i = 0; i < num_msrs_to_save; i++) {
+		if (msrs_to_save[i] == msr_index)
+			return true;
+	}
+
+	for (i = 0; i < num_emulated_msrs; i++) {
+		if (emulated_msrs[i] == msr_index)
+			return true;
+	}
+
+	return false;
+}
+
+
+/*
+ * Some IA32_ARCH_CAPABILITIES bits have dependencies on MSRs that KVM
+ * does not yet virtualize. These include:
+ *   10 - MISC_PACKAGE_CTRLS
+ *   11 - ENERGY_FILTERING_CTL
+ *   12 - DOITM
+ *   18 - FB_CLEAR_CTRL
+ *   21 - XAPIC_DISABLE_STATUS
+ *   23 - OVERCLOCKING_STATUS
+ */
+
+#define KVM_SUPPORTED_ARCH_CAP \
+	(ARCH_CAP_RDCL_NO | ARCH_CAP_IBRS_ALL | ARCH_CAP_RSBA | \
+	 ARCH_CAP_SKIP_VMENTRY_L1DFLUSH | ARCH_CAP_SSB_NO | ARCH_CAP_MDS_NO | \
+	 ARCH_CAP_PSCHANGE_MC_NO | ARCH_CAP_TSX_CTRL_MSR | ARCH_CAP_TAA_NO | \
+	 ARCH_CAP_SBDR_SSDP_NO | ARCH_CAP_FBSDP_NO | ARCH_CAP_PSDP_NO | \
+	 ARCH_CAP_FB_CLEAR | ARCH_CAP_RRSBA | ARCH_CAP_PBRSB_NO | ARCH_CAP_GDS_NO | \
+	 ARCH_CAP_RFDS_NO | ARCH_CAP_RFDS_CLEAR | ARCH_CAP_BHI_NO | ARCH_CAP_ITS_NO)
+
+u64 kvm_get_arch_capabilities(void)
+{
+	u64 data = kvm_host.arch_capabilities & KVM_SUPPORTED_ARCH_CAP;
+
+	/*
+	 * If nx_huge_pages is enabled, KVM's shadow paging will ensure that
+	 * the nested hypervisor runs with NX huge pages.  If it is not,
+	 * L1 is anyway vulnerable to ITLB_MULTIHIT exploits from other
+	 * L1 guests, so it need not worry about its own (L2) guests.
+	 */
+	data |= ARCH_CAP_PSCHANGE_MC_NO;
+
+	/*
+	 * If we're doing cache flushes (either "always" or "cond")
+	 * we will do one whenever the guest does a vmlaunch/vmresume.
+	 * If an outer hypervisor is doing the cache flush for us
+	 * (ARCH_CAP_SKIP_VMENTRY_L1DFLUSH), we can safely pass that
+	 * capability to the guest too, and if EPT is disabled we're not
+	 * vulnerable.  Overall, only VMENTER_L1D_FLUSH_NEVER will
+	 * require a nested hypervisor to do a flush of its own.
+	 */
+	if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER)
+		data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH;
+
+	if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
+		data |= ARCH_CAP_RDCL_NO;
+	if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
+		data |= ARCH_CAP_SSB_NO;
+	if (!boot_cpu_has_bug(X86_BUG_MDS))
+		data |= ARCH_CAP_MDS_NO;
+	if (!boot_cpu_has_bug(X86_BUG_RFDS))
+		data |= ARCH_CAP_RFDS_NO;
+	if (!boot_cpu_has_bug(X86_BUG_ITS))
+		data |= ARCH_CAP_ITS_NO;
+
+	if (!boot_cpu_has(X86_FEATURE_RTM)) {
+		/*
+		 * If RTM=0 because the kernel has disabled TSX, the host might
+		 * have TAA_NO or TSX_CTRL.  Clear TAA_NO (the guest sees RTM=0
+		 * and therefore knows that there cannot be TAA) but keep
+		 * TSX_CTRL: some buggy userspaces leave it set on tsx=on hosts,
+		 * and we want to allow migrating those guests to tsx=off hosts.
+		 */
+		data &= ~ARCH_CAP_TAA_NO;
+	} else if (!boot_cpu_has_bug(X86_BUG_TAA)) {
+		data |= ARCH_CAP_TAA_NO;
+	} else {
+		/*
+		 * Nothing to do here; we emulate TSX_CTRL if present on the
+		 * host so the guest can choose between disabling TSX or
+		 * using VERW to clear CPU buffers.
+		 */
+	}
+
+	if (!boot_cpu_has_bug(X86_BUG_GDS) || gds_ucode_mitigated())
+		data |= ARCH_CAP_GDS_NO;
+
+	return data;
+}
+
+static int kvm_get_feature_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
+			       bool host_initiated)
+{
+	WARN_ON_ONCE(!host_initiated);
+
+	switch (index) {
+	case MSR_IA32_ARCH_CAPABILITIES:
+		*data = kvm_get_arch_capabilities();
+		break;
+	case MSR_IA32_PERF_CAPABILITIES:
+		*data = kvm_caps.supported_perf_cap;
+		break;
+	case MSR_PLATFORM_INFO:
+		*data = MSR_PLATFORM_INFO_CPUID_FAULT;
+		break;
+	case MSR_IA32_UCODE_REV:
+		rdmsrq_safe(index, data);
+		break;
+	default:
+		return kvm_x86_call(get_feature_msr)(index, data);
+	}
+	return 0;
+}
+
+typedef int (*msr_access_t)(struct kvm_vcpu *vcpu, u32 index, u64 *data,
+			    bool host_initiated);
+
+static __always_inline int kvm_do_msr_access(struct kvm_vcpu *vcpu, u32 msr,
+					     u64 *data, bool host_initiated,
+					     enum kvm_msr_access rw,
+					     msr_access_t msr_access_fn)
+{
+	const char *op = rw == MSR_TYPE_W ? "wrmsr" : "rdmsr";
+	int ret;
+
+	BUILD_BUG_ON(rw != MSR_TYPE_R && rw != MSR_TYPE_W);
+
+	/*
+	 * Zero the data on read failures to avoid leaking stack data to the
+	 * guest and/or userspace, e.g. if the failure is ignored below.
+	 */
+	ret = msr_access_fn(vcpu, msr, data, host_initiated);
+	if (ret && rw == MSR_TYPE_R)
+		*data = 0;
+
+	if (ret != KVM_MSR_RET_UNSUPPORTED)
+		return ret;
+
+	/*
+	 * Userspace is allowed to read MSRs, and write '0' to MSRs, that KVM
+	 * advertises to userspace, even if an MSR isn't fully supported.
+	 * Simply check that @data is '0', which covers both the write '0' case
+	 * and all reads (in which case @data is zeroed on failure; see above).
+	 */
+	if (host_initiated && !*data && kvm_is_advertised_msr(msr))
+		return 0;
+
+	if (!ignore_msrs) {
+		kvm_debug_ratelimited("unhandled %s: 0x%x data 0x%llx\n",
+				      op, msr, *data);
+		return ret;
+	}
+
+	if (report_ignored_msrs)
+		kvm_pr_unimpl("ignored %s: 0x%x data 0x%llx\n", op, msr, *data);
+
+	return 0;
+}
+
+static int do_get_feature_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
+{
+	return kvm_do_msr_access(vcpu, index, data, true, MSR_TYPE_R,
+				 kvm_get_feature_msr);
+}
+
+static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
+{
+	if (efer & EFER_AUTOIBRS && !guest_cpu_cap_has(vcpu, X86_FEATURE_AUTOIBRS))
+		return false;
+
+	if (efer & EFER_FFXSR && !guest_cpu_cap_has(vcpu, X86_FEATURE_FXSR_OPT))
+		return false;
+
+	if (efer & EFER_SVME && !guest_cpu_cap_has(vcpu, X86_FEATURE_SVM))
+		return false;
+
+	if (efer & (EFER_LME | EFER_LMA) &&
+	    !guest_cpu_cap_has(vcpu, X86_FEATURE_LM))
+		return false;
+
+	if (efer & EFER_NX && !guest_cpu_cap_has(vcpu, X86_FEATURE_NX))
+		return false;
+
+	return true;
+
+}
+bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
+{
+	if (efer & efer_reserved_bits)
+		return false;
+
+	return __kvm_valid_efer(vcpu, efer);
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_valid_efer);
+
+static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
+{
+	u64 old_efer = vcpu->arch.efer;
+	u64 efer = msr_info->data;
+	int r;
+
+	if (efer & efer_reserved_bits)
+		return 1;
+
+	if (!msr_info->host_initiated) {
+		if (!__kvm_valid_efer(vcpu, efer))
+			return 1;
+
+		if (is_paging(vcpu) &&
+		    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
+			return 1;
+	}
+
+	efer &= ~EFER_LMA;
+	efer |= vcpu->arch.efer & EFER_LMA;
+
+	r = kvm_x86_call(set_efer)(vcpu, efer);
+	if (r) {
+		WARN_ON(r > 0);
+		return r;
+	}
+
+	if ((efer ^ old_efer) & KVM_MMU_EFER_ROLE_BITS)
+		kvm_mmu_reset_context(vcpu);
+
+	if (!cpu_feature_enabled(X86_FEATURE_XSAVES) &&
+	    (efer & EFER_SVME))
+		kvm_hv_xsaves_xsavec_maybe_warn(vcpu);
+
+	return 0;
+}
+
+void kvm_enable_efer_bits(u64 mask)
+{
+       efer_reserved_bits &= ~mask;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_enable_efer_bits);
+
+bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type)
+{
+	struct kvm_x86_msr_filter *msr_filter;
+	struct msr_bitmap_range *ranges;
+	struct kvm *kvm = vcpu->kvm;
+	bool allowed;
+	int idx;
+	u32 i;
+
+	/* x2APIC MSRs do not support filtering. */
+	if (index >= 0x800 && index <= 0x8ff)
+		return true;
+
+	idx = srcu_read_lock(&kvm->srcu);
+
+	msr_filter = srcu_dereference(kvm->arch.msr_filter, &kvm->srcu);
+	if (!msr_filter) {
+		allowed = true;
+		goto out;
+	}
+
+	allowed = msr_filter->default_allow;
+	ranges = msr_filter->ranges;
+
+	for (i = 0; i < msr_filter->count; i++) {
+		u32 start = ranges[i].base;
+		u32 end = start + ranges[i].nmsrs;
+		u32 flags = ranges[i].flags;
+		unsigned long *bitmap = ranges[i].bitmap;
+
+		if ((index >= start) && (index < end) && (flags & type)) {
+			allowed = test_bit(index - start, bitmap);
+			break;
+		}
+	}
+
+out:
+	srcu_read_unlock(&kvm->srcu, idx);
+
+	return allowed;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_msr_allowed);
+
+/*
+ * Write @data into the MSR specified by @index.  Select MSR specific fault
+ * checks are bypassed if @host_initiated is %true.
+ * Returns 0 on success, non-0 otherwise.
+ * Assumes vcpu_load() was already called.
+ */
+static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
+			 bool host_initiated)
+{
+	struct msr_data msr;
+
+	switch (index) {
+	case MSR_FS_BASE:
+	case MSR_GS_BASE:
+	case MSR_KERNEL_GS_BASE:
+	case MSR_CSTAR:
+	case MSR_LSTAR:
+		if (is_noncanonical_msr_address(data, vcpu))
+			return 1;
+		break;
+	case MSR_IA32_SYSENTER_EIP:
+	case MSR_IA32_SYSENTER_ESP:
+		/*
+		 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if
+		 * non-canonical address is written on Intel but not on
+		 * AMD (which ignores the top 32-bits, because it does
+		 * not implement 64-bit SYSENTER).
+		 *
+		 * 64-bit code should hence be able to write a non-canonical
+		 * value on AMD.  Making the address canonical ensures that
+		 * vmentry does not fail on Intel after writing a non-canonical
+		 * value, and that something deterministic happens if the guest
+		 * invokes 64-bit SYSENTER.
+		 */
+		data = __canonical_address(data, max_host_virt_addr_bits());
+		break;
+	case MSR_TSC_AUX:
+		if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX))
+			return 1;
+
+		if (!host_initiated &&
+		    !guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) &&
+		    !guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID))
+			return 1;
+
+		/*
+		 * Per Intel's SDM, bits 63:32 are reserved, but AMD's APM has
+		 * incomplete and conflicting architectural behavior.  Current
+		 * AMD CPUs completely ignore bits 63:32, i.e. they aren't
+		 * reserved and always read as zeros.  Enforce Intel's reserved
+		 * bits check if the guest CPU is Intel compatible, otherwise
+		 * clear the bits.  This ensures cross-vendor migration will
+		 * provide consistent behavior for the guest.
+		 */
+		if (guest_cpuid_is_intel_compatible(vcpu) && (data >> 32) != 0)
+			return 1;
+
+		data = (u32)data;
+		break;
+	case MSR_IA32_U_CET:
+	case MSR_IA32_S_CET:
+		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) &&
+		    !guest_cpu_cap_has(vcpu, X86_FEATURE_IBT))
+			return KVM_MSR_RET_UNSUPPORTED;
+		if (!kvm_is_valid_u_s_cet(vcpu, data))
+			return 1;
+		break;
+	case MSR_KVM_INTERNAL_GUEST_SSP:
+		if (!host_initiated)
+			return 1;
+		fallthrough;
+		/*
+		 * Note that the MSR emulation here is flawed when a vCPU
+		 * doesn't support the Intel 64 architecture. The expected
+		 * architectural behavior in this case is that the upper 32
+		 * bits do not exist and should always read '0'. However,
+		 * because the actual hardware on which the virtual CPU is
+		 * running does support Intel 64, XRSTORS/XSAVES in the
+		 * guest could observe behavior that violates the
+		 * architecture. Intercepting XRSTORS/XSAVES for this
+		 * special case isn't deemed worthwhile.
+		 */
+	case MSR_IA32_PL0_SSP ... MSR_IA32_INT_SSP_TAB:
+		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK))
+			return KVM_MSR_RET_UNSUPPORTED;
+		/*
+		 * MSR_IA32_INT_SSP_TAB is not present on processors that do
+		 * not support Intel 64 architecture.
+		 */
+		if (index == MSR_IA32_INT_SSP_TAB && !guest_cpu_cap_has(vcpu, X86_FEATURE_LM))
+			return KVM_MSR_RET_UNSUPPORTED;
+		if (is_noncanonical_msr_address(data, vcpu))
+			return 1;
+		/* All SSP MSRs except MSR_IA32_INT_SSP_TAB must be 4-byte aligned */
+		if (index != MSR_IA32_INT_SSP_TAB && !IS_ALIGNED(data, 4))
+			return 1;
+		break;
+	}
+
+	msr.data = data;
+	msr.index = index;
+	msr.host_initiated = host_initiated;
+
+	return kvm_x86_call(set_msr)(vcpu, &msr);
+}
+
+static int _kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
+			bool host_initiated)
+{
+	return __kvm_set_msr(vcpu, index, *data, host_initiated);
+}
+
+static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,
+				     u32 index, u64 data, bool host_initiated)
+{
+	return kvm_do_msr_access(vcpu, index, &data, host_initiated, MSR_TYPE_W,
+				 _kvm_set_msr);
+}
+
+/*
+ * Read the MSR specified by @index into @data.  Select MSR specific fault
+ * checks are bypassed if @host_initiated is %true.
+ * Returns 0 on success, non-0 otherwise.
+ * Assumes vcpu_load() was already called.
+ */
+static int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
+			 bool host_initiated)
+{
+	struct msr_data msr;
+	int ret;
+
+	switch (index) {
+	case MSR_TSC_AUX:
+		if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX))
+			return 1;
+
+		if (!host_initiated &&
+		    !guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) &&
+		    !guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID))
+			return 1;
+		break;
+	case MSR_IA32_U_CET:
+	case MSR_IA32_S_CET:
+		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) &&
+		    !guest_cpu_cap_has(vcpu, X86_FEATURE_IBT))
+			return KVM_MSR_RET_UNSUPPORTED;
+		break;
+	case MSR_KVM_INTERNAL_GUEST_SSP:
+		if (!host_initiated)
+			return 1;
+		fallthrough;
+	case MSR_IA32_PL0_SSP ... MSR_IA32_INT_SSP_TAB:
+		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK))
+			return KVM_MSR_RET_UNSUPPORTED;
+		break;
+	}
+
+	msr.index = index;
+	msr.host_initiated = host_initiated;
+
+	ret = kvm_x86_call(get_msr)(vcpu, &msr);
+	if (!ret)
+		*data = msr.data;
+	return ret;
+}
+
+static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu,
+				     u32 index, u64 *data, bool host_initiated)
+{
+	return kvm_do_msr_access(vcpu, index, data, host_initiated, MSR_TYPE_R,
+				 __kvm_get_msr);
+}
+
+int kvm_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data)
+{
+	return __kvm_set_msr(vcpu, index, data, true);
+}
+
+int kvm_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data)
+{
+	return __kvm_get_msr(vcpu, index, data, true);
+}
+
+int __kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data)
+{
+	return kvm_get_msr_ignored_check(vcpu, index, data, false);
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_emulate_msr_read);
+
+int __kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data)
+{
+	return kvm_set_msr_ignored_check(vcpu, index, data, false);
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_emulate_msr_write);
+
+int kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data)
+{
+	if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ))
+		return KVM_MSR_RET_FILTERED;
+
+	return __kvm_emulate_msr_read(vcpu, index, data);
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_msr_read);
+
+int kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data)
+{
+	if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE))
+		return KVM_MSR_RET_FILTERED;
+
+	return __kvm_emulate_msr_write(vcpu, index, data);
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_msr_write);
+
+static fastpath_t __handle_fastpath_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
+{
+	if (!kvm_pmu_is_fastpath_emulation_allowed(vcpu))
+		return EXIT_FASTPATH_NONE;
+
+	switch (msr) {
+	case APIC_BASE_MSR + (APIC_ICR >> 4):
+		if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic) ||
+		    kvm_x2apic_icr_write_fast(vcpu->arch.apic, data))
+			return EXIT_FASTPATH_NONE;
+		break;
+	case MSR_IA32_TSC_DEADLINE:
+		kvm_set_lapic_tscdeadline_msr(vcpu, data);
+		break;
+	default:
+		return EXIT_FASTPATH_NONE;
+	}
+
+	trace_kvm_msr_write(msr, data);
+
+	if (!kvm_skip_emulated_instruction(vcpu))
+		return EXIT_FASTPATH_EXIT_USERSPACE;
+
+	return EXIT_FASTPATH_REENTER_GUEST;
+}
+
+fastpath_t handle_fastpath_wrmsr(struct kvm_vcpu *vcpu)
+{
+	return __handle_fastpath_wrmsr(vcpu, kvm_ecx_read(vcpu),
+				       kvm_read_edx_eax(vcpu));
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_wrmsr);
+
+fastpath_t handle_fastpath_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg)
+{
+	return __handle_fastpath_wrmsr(vcpu, msr, kvm_register_read(vcpu, reg));
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_wrmsr_imm);
+
+static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu)
+{
+	if (!vcpu->run->msr.error) {
+		kvm_eax_write(vcpu, vcpu->run->msr.data);
+		kvm_edx_write(vcpu, vcpu->run->msr.data >> 32);
+	}
+}
+
+static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err)
+{
+	if (err) {
+		kvm_inject_gp(vcpu, 0);
+		return 1;
+	}
+
+	return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE | EMULTYPE_SKIP |
+				       EMULTYPE_COMPLETE_USER_EXIT);
+}
+
+static int complete_emulated_msr_access(struct kvm_vcpu *vcpu)
+{
+	return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error);
+}
+
+static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)
+{
+	complete_userspace_rdmsr(vcpu);
+	return complete_emulated_msr_access(vcpu);
+}
+
+static int complete_fast_msr_access(struct kvm_vcpu *vcpu)
+{
+	return kvm_x86_call(complete_emulated_msr)(vcpu, vcpu->run->msr.error);
+}
+
+static int complete_fast_rdmsr(struct kvm_vcpu *vcpu)
+{
+	complete_userspace_rdmsr(vcpu);
+	return complete_fast_msr_access(vcpu);
+}
+
+static int complete_fast_rdmsr_imm(struct kvm_vcpu *vcpu)
+{
+	if (!vcpu->run->msr.error)
+		kvm_register_write(vcpu, vcpu->arch.cui_rdmsr_imm_reg,
+				   vcpu->run->msr.data);
+
+	return complete_fast_msr_access(vcpu);
+}
+
+static u64 kvm_msr_reason(int r)
+{
+	switch (r) {
+	case KVM_MSR_RET_UNSUPPORTED:
+		return KVM_MSR_EXIT_REASON_UNKNOWN;
+	case KVM_MSR_RET_FILTERED:
+		return KVM_MSR_EXIT_REASON_FILTER;
+	default:
+		return KVM_MSR_EXIT_REASON_INVAL;
+	}
+}
+
+static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index,
+			      u32 exit_reason, u64 data,
+			      int (*completion)(struct kvm_vcpu *vcpu),
+			      int r)
+{
+	u64 msr_reason = kvm_msr_reason(r);
+
+	/* Check if the user wanted to know about this MSR fault */
+	if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason))
+		return 0;
+
+	vcpu->run->exit_reason = exit_reason;
+	vcpu->run->msr.error = 0;
+	memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad));
+	vcpu->run->msr.reason = msr_reason;
+	vcpu->run->msr.index = index;
+	vcpu->run->msr.data = data;
+	vcpu->arch.complete_userspace_io = completion;
+
+	return 1;
+}
+
+static int __kvm_emulate_rdmsr(struct kvm_vcpu *vcpu, u32 msr, int reg,
+			       int (*complete_rdmsr)(struct kvm_vcpu *))
+{
+	u64 data;
+	int r;
+
+	r = kvm_emulate_msr_read(vcpu, msr, &data);
+
+	if (!r) {
+		trace_kvm_msr_read(msr, data);
+
+		if (reg < 0) {
+			kvm_eax_write(vcpu, data);
+			kvm_edx_write(vcpu, data >> 32);
+		} else {
+			kvm_register_write(vcpu, reg, data);
+		}
+	} else {
+		/* MSR read failed? See if we should ask user space */
+		if (kvm_msr_user_space(vcpu, msr, KVM_EXIT_X86_RDMSR, 0,
+				       complete_rdmsr, r))
+			return 0;
+		trace_kvm_msr_read_ex(msr);
+	}
+
+	return kvm_x86_call(complete_emulated_msr)(vcpu, r);
+}
+
+int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
+{
+	return __kvm_emulate_rdmsr(vcpu, kvm_ecx_read(vcpu), -1,
+				   complete_fast_rdmsr);
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdmsr);
+
+int kvm_emulate_rdmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg)
+{
+	vcpu->arch.cui_rdmsr_imm_reg = reg;
+
+	return __kvm_emulate_rdmsr(vcpu, msr, reg, complete_fast_rdmsr_imm);
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdmsr_imm);
+
+static int __kvm_emulate_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
+{
+	int r;
+
+	r = kvm_emulate_msr_write(vcpu, msr, data);
+	if (!r) {
+		trace_kvm_msr_write(msr, data);
+	} else {
+		/* MSR write failed? See if we should ask user space */
+		if (kvm_msr_user_space(vcpu, msr, KVM_EXIT_X86_WRMSR, data,
+				       complete_fast_msr_access, r))
+			return 0;
+		/* Signal all other negative errors to userspace */
+		if (r < 0)
+			return r;
+		trace_kvm_msr_write_ex(msr, data);
+	}
+
+	return kvm_x86_call(complete_emulated_msr)(vcpu, r);
+}
+
+int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
+{
+	return __kvm_emulate_wrmsr(vcpu, kvm_ecx_read(vcpu),
+				   kvm_read_edx_eax(vcpu));
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_wrmsr);
+
+int kvm_emulate_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg)
+{
+	return __kvm_emulate_wrmsr(vcpu, msr, kvm_register_read(vcpu, reg));
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_wrmsr_imm);
+
+int kvm_emulator_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 msr_index,
+				     u64 *pdata)
+{
+	int r;
+
+	r = kvm_emulate_msr_read(vcpu, msr_index, pdata);
+	if (r < 0)
+		return X86EMUL_UNHANDLEABLE;
+
+	if (r) {
+		if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0,
+				       complete_emulated_rdmsr, r))
+			return X86EMUL_IO_NEEDED;
+
+		trace_kvm_msr_read_ex(msr_index);
+		return X86EMUL_PROPAGATE_FAULT;
+	}
+
+	trace_kvm_msr_read(msr_index, *pdata);
+	return X86EMUL_CONTINUE;
+}
+
+int kvm_emulator_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 msr_index,
+				     u64 data)
+{
+	int r;
+
+	r = kvm_emulate_msr_write(vcpu, msr_index, data);
+	if (r < 0)
+		return X86EMUL_UNHANDLEABLE;
+
+	if (r) {
+		if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data,
+				       complete_emulated_msr_access, r))
+			return X86EMUL_IO_NEEDED;
+
+		trace_kvm_msr_write_ex(msr_index, data);
+		return X86EMUL_PROPAGATE_FAULT;
+	}
+
+	trace_kvm_msr_write(msr_index, data);
+	return X86EMUL_CONTINUE;
+}
+
+int kvm_emulator_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
+{
+	/*
+	 * Treat emulator accesses to the current shadow stack pointer as host-
+	 * initiated, as they aren't true MSR accesses (SSP is a "just a reg"),
+	 * and this API is used only for implicit accesses, i.e. not RDMSR, and
+	 * so the index is fully KVM-controlled.
+	 */
+	if (unlikely(msr_index == MSR_KVM_INTERNAL_GUEST_SSP))
+		return kvm_msr_read(vcpu, msr_index, pdata);
+
+	return __kvm_emulate_msr_read(vcpu, msr_index, pdata);
+}
+
+/*
+ * Returns true if the MSR in question is managed via XSTATE, i.e. is context
+ * switched with the rest of guest FPU state.
+ *
+ * Note, S_CET is _not_ saved/restored via XSAVES/XRSTORS.
+ */
+static bool is_xstate_managed_msr(struct kvm_vcpu *vcpu, u32 msr)
+{
+	if (!vcpu)
+		return false;
+
+	switch (msr) {
+	case MSR_IA32_U_CET:
+		return guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) ||
+		       guest_cpu_cap_has(vcpu, X86_FEATURE_IBT);
+	case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP:
+		return guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK);
+	default:
+		return false;
+	}
+}
+
+/*
+ * Lock (and if necessary, re-load) the guest FPU, i.e. XSTATE, and access an
+ * MSR that is managed via XSTATE.  Note, the caller is responsible for doing
+ * the initial FPU load, this helper only ensures that guest state is resident
+ * in hardware (the kernel can load its FPU state in IRQ context).
+ *
+ * Note, loading guest values for U_CET and PL[0-3]_SSP while executing in the
+ * kernel is safe, as U_CET is specific to userspace, and PL[0-3]_SSP are only
+ * consumed when transitioning to lower privilege levels, i.e. are effectively
+ * only consumed by userspace as well.
+ */
+static __always_inline void kvm_access_xstate_msr(struct kvm_vcpu *vcpu,
+						  struct msr_data *msr_info,
+						  int access)
+{
+	BUILD_BUG_ON(access != MSR_TYPE_R && access != MSR_TYPE_W);
+
+	KVM_BUG_ON(!is_xstate_managed_msr(vcpu, msr_info->index), vcpu->kvm);
+	KVM_BUG_ON(!vcpu->arch.guest_fpu.fpstate->in_use, vcpu->kvm);
+
+	kvm_fpu_get();
+	if (access == MSR_TYPE_R)
+		rdmsrq(msr_info->index, msr_info->data);
+	else
+		wrmsrq(msr_info->index, msr_info->data);
+	kvm_fpu_put();
+}
+
+static void kvm_set_xstate_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
+{
+	kvm_access_xstate_msr(vcpu, msr_info, MSR_TYPE_W);
+}
+
+static void kvm_get_xstate_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
+{
+	kvm_access_xstate_msr(vcpu, msr_info, MSR_TYPE_R);
+}
+
+static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs)
+{
+	int version;
+	int r;
+	struct pvclock_wall_clock wc;
+	u32 wc_sec_hi;
+	u64 wall_nsec;
+
+	if (!wall_clock)
+		return;
+
+	r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
+	if (r)
+		return;
+
+	if (version & 1)
+		++version;  /* first time write, random junk */
+
+	++version;
+
+	if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version)))
+		return;
+
+	wall_nsec = kvm_get_wall_clock_epoch(kvm);
+
+	wc.nsec = do_div(wall_nsec, NSEC_PER_SEC);
+	wc.sec = (u32)wall_nsec; /* overflow in 2106 guest time */
+	wc.version = version;
+
+	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
+
+	if (sec_hi_ofs) {
+		wc_sec_hi = wall_nsec >> 32;
+		kvm_write_guest(kvm, wall_clock + sec_hi_ofs,
+				&wc_sec_hi, sizeof(wc_sec_hi));
+	}
+
+	version++;
+	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
+}
+
+static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time,
+				  bool old_msr, bool host_initiated)
+{
+	struct kvm_arch *ka = &vcpu->kvm->arch;
+
+	if (vcpu->vcpu_id == 0 && !host_initiated) {
+		if (ka->boot_vcpu_runs_old_kvmclock != old_msr)
+			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
+
+		ka->boot_vcpu_runs_old_kvmclock = old_msr;
+	}
+
+	vcpu->arch.time = system_time;
+	kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
+
+	/* we verify if the enable bit is set... */
+	if (system_time & 1)
+		kvm_gpc_activate(&vcpu->arch.pv_time, system_time & ~1ULL,
+				 sizeof(struct pvclock_vcpu_time_info));
+	else
+		kvm_gpc_deactivate(&vcpu->arch.pv_time);
+
+	return;
+}
+
+/* These helpers are safe iff @msr is known to be an MCx bank MSR. */
+static bool is_mci_control_msr(u32 msr)
+{
+	return (msr & 3) == 0;
+}
+static bool is_mci_status_msr(u32 msr)
+{
+	return (msr & 3) == 1;
+}
+
+/*
+ * On AMD, HWCR[McStatusWrEn] controls whether setting MCi_STATUS results in #GP.
+ */
+static bool can_set_mci_status(struct kvm_vcpu *vcpu)
+{
+	/* McStatusWrEn enabled? */
+	if (guest_cpuid_is_amd_compatible(vcpu))
+		return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));
+
+	return false;
+}
+
+static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
+{
+	u64 mcg_cap = vcpu->arch.mcg_cap;
+	unsigned bank_num = mcg_cap & 0xff;
+	u32 msr = msr_info->index;
+	u64 data = msr_info->data;
+	u32 offset, last_msr;
+
+	switch (msr) {
+	case MSR_IA32_MCG_STATUS:
+		vcpu->arch.mcg_status = data;
+		break;
+	case MSR_IA32_MCG_CTL:
+		if (!(mcg_cap & MCG_CTL_P) &&
+		    (data || !msr_info->host_initiated))
+			return 1;
+		if (data != 0 && data != ~(u64)0)
+			return 1;
+		vcpu->arch.mcg_ctl = data;
+		break;
+	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
+		last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1;
+		if (msr > last_msr)
+			return 1;
+
+		if (!(mcg_cap & MCG_CMCI_P) && (data || !msr_info->host_initiated))
+			return 1;
+		/* An attempt to write a 1 to a reserved bit raises #GP */
+		if (data & ~(MCI_CTL2_CMCI_EN | MCI_CTL2_CMCI_THRESHOLD_MASK))
+			return 1;
+		offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2,
+					    last_msr + 1 - MSR_IA32_MC0_CTL2);
+		vcpu->arch.mci_ctl2_banks[offset] = data;
+		break;
+	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
+		last_msr = MSR_IA32_MCx_CTL(bank_num) - 1;
+		if (msr > last_msr)
+			return 1;
+
+		/*
+		 * Only 0 or all 1s can be written to IA32_MCi_CTL, all other
+		 * values are architecturally undefined.  But, some Linux
+		 * kernels clear bit 10 in bank 4 to workaround a BIOS/GART TLB
+		 * issue on AMD K8s, allow bit 10 to be clear when setting all
+		 * other bits in order to avoid an uncaught #GP in the guest.
+		 *
+		 * UNIXWARE clears bit 0 of MC1_CTL to ignore correctable,
+		 * single-bit ECC data errors.
+		 */
+		if (is_mci_control_msr(msr) &&
+		    data != 0 && (data | (1 << 10) | 1) != ~(u64)0)
+			return 1;
+
+		/*
+		 * All CPUs allow writing 0 to MCi_STATUS MSRs to clear the MSR.
+		 * AMD-based CPUs allow non-zero values, but if and only if
+		 * HWCR[McStatusWrEn] is set.
+		 */
+		if (!msr_info->host_initiated && is_mci_status_msr(msr) &&
+		    data != 0 && !can_set_mci_status(vcpu))
+			return 1;
+
+		offset = array_index_nospec(msr - MSR_IA32_MC0_CTL,
+					    last_msr + 1 - MSR_IA32_MC0_CTL);
+		vcpu->arch.mce_banks[offset] = data;
+		break;
+	default:
+		return 1;
+	}
+	return 0;
+}
+
+static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
+{
+	gpa_t gpa = data & ~0x3f;
+
+	/* Bits 4:5 are reserved, Should be zero */
+	if (data & 0x30)
+		return 1;
+
+	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) &&
+	    (data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT))
+		return 1;
+
+	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) &&
+	    (data & KVM_ASYNC_PF_DELIVERY_AS_INT))
+		return 1;
+
+	if (!lapic_in_kernel(vcpu))
+		return data ? 1 : 0;
+
+	if (__kvm_pv_async_pf_enabled(data) &&
+	    kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
+				      sizeof(u64)))
+		return 1;
+
+	vcpu->arch.apf.msr_en_val = data;
+
+	if (__kvm_pv_async_pf_enabled(data)) {
+		kvm_async_pf_wakeup_all(vcpu);
+	} else {
+		kvm_clear_async_pf_completion_queue(vcpu);
+		kvm_async_pf_hash_reset(vcpu);
+	}
+	return 0;
+}
+
+static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data)
+{
+	/* Bits 8-63 are reserved */
+	if (data >> 8)
+		return 1;
+
+	if (!lapic_in_kernel(vcpu))
+		return 1;
+
+	vcpu->arch.apf.msr_int_val = data;
+
+	vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK;
+
+	return 0;
+}
+
+#ifdef CONFIG_X86_64
+static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu)
+{
+	return vcpu->arch.guest_supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC;
+}
+#endif
+
+int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
+{
+	u32 msr = msr_info->index;
+	u64 data = msr_info->data;
+
+	/*
+	 * Do not allow host-initiated writes to trigger the Xen hypercall
+	 * page setup; it could incur locking paths which are not expected
+	 * if userspace sets the MSR in an unusual location.
+	 */
+	if (kvm_xen_is_hypercall_page_msr(vcpu->kvm, msr) &&
+	    !msr_info->host_initiated)
+		return kvm_xen_write_hypercall_page(vcpu, data);
+
+	switch (msr) {
+	case MSR_AMD64_NB_CFG:
+	case MSR_IA32_UCODE_WRITE:
+	case MSR_VM_HSAVE_PA:
+	case MSR_AMD64_PATCH_LOADER:
+	case MSR_AMD64_BU_CFG2:
+	case MSR_AMD64_DC_CFG:
+	case MSR_AMD64_TW_CFG:
+	case MSR_F15H_EX_CFG:
+		break;
+
+	case MSR_IA32_UCODE_REV:
+		if (msr_info->host_initiated)
+			vcpu->arch.microcode_version = data;
+		break;
+	case MSR_IA32_ARCH_CAPABILITIES:
+		if (!msr_info->host_initiated ||
+		    !guest_cpu_cap_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
+			return KVM_MSR_RET_UNSUPPORTED;
+		vcpu->arch.arch_capabilities = data;
+		break;
+	case MSR_IA32_PERF_CAPABILITIES:
+		if (!msr_info->host_initiated ||
+		    !guest_cpu_cap_has(vcpu, X86_FEATURE_PDCM))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		if (data & ~kvm_caps.supported_perf_cap)
+			return 1;
+
+		/*
+		 * Note, this is not just a performance optimization!  KVM
+		 * disallows changing feature MSRs after the vCPU has run; PMU
+		 * refresh will bug the VM if called after the vCPU has run.
+		 */
+		if (vcpu->arch.perf_capabilities == data)
+			break;
+
+		vcpu->arch.perf_capabilities = data;
+		kvm_pmu_refresh(vcpu);
+		kvm_make_request(KVM_REQ_RECALC_INTERCEPTS, vcpu);
+		break;
+	case MSR_IA32_PRED_CMD: {
+		u64 reserved_bits = ~(PRED_CMD_IBPB | PRED_CMD_SBPB);
+
+		if (!msr_info->host_initiated) {
+			if ((!guest_has_pred_cmd_msr(vcpu)))
+				return 1;
+
+			if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SPEC_CTRL) &&
+			    !guest_cpu_cap_has(vcpu, X86_FEATURE_AMD_IBPB))
+				reserved_bits |= PRED_CMD_IBPB;
+
+			if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SBPB))
+				reserved_bits |= PRED_CMD_SBPB;
+		}
+
+		if (!boot_cpu_has(X86_FEATURE_IBPB))
+			reserved_bits |= PRED_CMD_IBPB;
+
+		if (!boot_cpu_has(X86_FEATURE_SBPB))
+			reserved_bits |= PRED_CMD_SBPB;
+
+		if (data & reserved_bits)
+			return 1;
+
+		if (!data)
+			break;
+
+		wrmsrq(MSR_IA32_PRED_CMD, data);
+		break;
+	}
+	case MSR_IA32_FLUSH_CMD:
+		if (!msr_info->host_initiated &&
+		    !guest_cpu_cap_has(vcpu, X86_FEATURE_FLUSH_L1D))
+			return 1;
+
+		if (!boot_cpu_has(X86_FEATURE_FLUSH_L1D) || (data & ~L1D_FLUSH))
+			return 1;
+		if (!data)
+			break;
+
+		wrmsrq(MSR_IA32_FLUSH_CMD, L1D_FLUSH);
+		break;
+	case MSR_EFER:
+		return set_efer(vcpu, msr_info);
+	case MSR_K7_HWCR: {
+		/*
+		 * Allow McStatusWrEn and TscFreqSel. (Linux guests from v3.2
+		 * through at least v6.6 whine if TscFreqSel is clear,
+		 * depending on F/M/S.
+		 */
+		u64 valid = BIT_ULL(18) | BIT_ULL(24);
+
+		data &= ~(u64)0x40;	/* ignore flush filter disable */
+		data &= ~(u64)0x100;	/* ignore ignne emulation enable */
+		data &= ~(u64)0x8;	/* ignore TLB cache disable */
+
+		if (guest_cpu_cap_has(vcpu, X86_FEATURE_GP_ON_USER_CPUID))
+			valid |= MSR_K7_HWCR_CPUID_USER_DIS;
+
+		if (data & ~valid) {
+			kvm_pr_unimpl_wrmsr(vcpu, msr, data);
+			return 1;
+		}
+		vcpu->arch.msr_hwcr = data;
+		break;
+	}
+	case MSR_FAM10H_MMIO_CONF_BASE:
+		if (data != 0) {
+			kvm_pr_unimpl_wrmsr(vcpu, msr, data);
+			return 1;
+		}
+		break;
+	case MSR_IA32_CR_PAT:
+		if (!kvm_pat_valid(data))
+			return 1;
+
+		vcpu->arch.pat = data;
+		break;
+	case MTRRphysBase_MSR(0) ... MSR_MTRRfix4K_F8000:
+	case MSR_MTRRdefType:
+		return kvm_mtrr_set_msr(vcpu, msr, data);
+	case MSR_IA32_APICBASE:
+		return kvm_apic_set_base(vcpu, data, msr_info->host_initiated);
+	case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff:
+		return kvm_x2apic_msr_write(vcpu, msr, data);
+	case MSR_IA32_TSC_DEADLINE:
+		kvm_set_lapic_tscdeadline_msr(vcpu, data);
+		break;
+	case MSR_IA32_TSC_ADJUST:
+		if (guest_cpu_cap_has(vcpu, X86_FEATURE_TSC_ADJUST)) {
+			if (!msr_info->host_initiated) {
+				s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
+				adjust_tsc_offset_guest(vcpu, adj);
+				/* Before back to guest, tsc_timestamp must be adjusted
+				 * as well, otherwise guest's percpu pvclock time could jump.
+				 */
+				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
+			}
+			vcpu->arch.ia32_tsc_adjust_msr = data;
+		}
+		break;
+	case MSR_IA32_MISC_ENABLE: {
+		u64 old_val = vcpu->arch.ia32_misc_enable_msr;
+
+		if (!msr_info->host_initiated) {
+			/* RO bits */
+			if ((old_val ^ data) & MSR_IA32_MISC_ENABLE_PMU_RO_MASK)
+				return 1;
+
+			/* R bits, i.e. writes are ignored, but don't fault. */
+			data = data & ~MSR_IA32_MISC_ENABLE_EMON;
+			data |= old_val & MSR_IA32_MISC_ENABLE_EMON;
+		}
+
+		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
+		    ((old_val ^ data)  & MSR_IA32_MISC_ENABLE_MWAIT)) {
+			if (!guest_cpu_cap_has(vcpu, X86_FEATURE_XMM3))
+				return 1;
+			vcpu->arch.ia32_misc_enable_msr = data;
+			vcpu->arch.cpuid_dynamic_bits_dirty = true;
+		} else {
+			vcpu->arch.ia32_misc_enable_msr = data;
+		}
+		break;
+	}
+	case MSR_IA32_SMBASE:
+		if (!IS_ENABLED(CONFIG_KVM_SMM) || !msr_info->host_initiated)
+			return 1;
+		vcpu->arch.smbase = data;
+		break;
+	case MSR_IA32_POWER_CTL:
+		vcpu->arch.msr_ia32_power_ctl = data;
+		break;
+	case MSR_IA32_TSC:
+		if (msr_info->host_initiated) {
+			kvm_synchronize_tsc(vcpu, &data);
+		} else if (!vcpu->arch.guest_tsc_protected) {
+			u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;
+			adjust_tsc_offset_guest(vcpu, adj);
+			vcpu->arch.ia32_tsc_adjust_msr += adj;
+		}
+		break;
+	case MSR_IA32_XSS:
+		if (!guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		if (data & ~vcpu->arch.guest_supported_xss)
+			return 1;
+		if (vcpu->arch.ia32_xss == data)
+			break;
+		vcpu->arch.ia32_xss = data;
+		vcpu->arch.cpuid_dynamic_bits_dirty = true;
+		break;
+	case MSR_SMI_COUNT:
+		if (!msr_info->host_initiated)
+			return 1;
+		vcpu->arch.smi_count = data;
+		break;
+	case MSR_KVM_WALL_CLOCK_NEW:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		vcpu->kvm->arch.wall_clock = data;
+		kvm_write_wall_clock(vcpu->kvm, data, 0);
+		break;
+	case MSR_KVM_WALL_CLOCK:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		vcpu->kvm->arch.wall_clock = data;
+		kvm_write_wall_clock(vcpu->kvm, data, 0);
+		break;
+	case MSR_KVM_SYSTEM_TIME_NEW:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		kvm_write_system_time(vcpu, data, false, msr_info->host_initiated);
+		break;
+	case MSR_KVM_SYSTEM_TIME:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		kvm_write_system_time(vcpu, data, true,  msr_info->host_initiated);
+		break;
+	case MSR_KVM_ASYNC_PF_EN:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		if (kvm_pv_enable_async_pf(vcpu, data))
+			return 1;
+		break;
+	case MSR_KVM_ASYNC_PF_INT:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		if (kvm_pv_enable_async_pf_int(vcpu, data))
+			return 1;
+		break;
+	case MSR_KVM_ASYNC_PF_ACK:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
+			return KVM_MSR_RET_UNSUPPORTED;
+		if (data & 0x1) {
+			/*
+			 * Pairs with the smp_mb__after_atomic() in
+			 * kvm_arch_async_page_present_queued().
+			 */
+			smp_store_mb(vcpu->arch.apf.pageready_pending, false);
+
+			kvm_check_async_pf_completion(vcpu);
+		}
+		break;
+	case MSR_KVM_STEAL_TIME:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		if (unlikely(!sched_info_on()))
+			return 1;
+
+		if (data & KVM_STEAL_RESERVED_MASK)
+			return 1;
+
+		vcpu->arch.st.msr_val = data;
+
+		if (!(data & KVM_MSR_ENABLED))
+			break;
+
+		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
+
+		break;
+	case MSR_KVM_PV_EOI_EN:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8)))
+			return 1;
+		break;
+
+	case MSR_KVM_POLL_CONTROL:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		/* only enable bit supported */
+		if (data & (-1ULL << 1))
+			return 1;
+
+		vcpu->arch.msr_kvm_poll_control = data;
+		break;
+
+	case MSR_IA32_MCG_CTL:
+	case MSR_IA32_MCG_STATUS:
+	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
+	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
+		return set_msr_mce(vcpu, msr_info);
+
+	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
+	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
+	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
+	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
+		if (kvm_pmu_is_valid_msr(vcpu, msr))
+			return kvm_pmu_set_msr(vcpu, msr_info);
+
+		if (data)
+			kvm_pr_unimpl_wrmsr(vcpu, msr, data);
+		break;
+	case MSR_K7_CLK_CTL:
+		/*
+		 * Ignore all writes to this no longer documented MSR.
+		 * Writes are only relevant for old K7 processors,
+		 * all pre-dating SVM, but a recommended workaround from
+		 * AMD for these chips. It is possible to specify the
+		 * affected processor models on the command line, hence
+		 * the need to ignore the workaround.
+		 */
+		break;
+#ifdef CONFIG_KVM_HYPERV
+	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
+	case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER:
+	case HV_X64_MSR_SYNDBG_OPTIONS:
+	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
+	case HV_X64_MSR_CRASH_CTL:
+	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
+	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
+	case HV_X64_MSR_TSC_EMULATION_CONTROL:
+	case HV_X64_MSR_TSC_EMULATION_STATUS:
+	case HV_X64_MSR_TSC_INVARIANT_CONTROL:
+		return kvm_hv_set_msr_common(vcpu, msr, data,
+					     msr_info->host_initiated);
+#endif
+	case MSR_IA32_BBL_CR_CTL3:
+		/* Drop writes to this legacy MSR -- see rdmsr
+		 * counterpart for further detail.
+		 */
+		kvm_pr_unimpl_wrmsr(vcpu, msr, data);
+		break;
+	case MSR_AMD64_OSVW_ID_LENGTH:
+		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW))
+			return 1;
+		vcpu->arch.osvw.length = data;
+		break;
+	case MSR_AMD64_OSVW_STATUS:
+		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW))
+			return 1;
+		vcpu->arch.osvw.status = data;
+		break;
+	case MSR_PLATFORM_INFO:
+		if (!msr_info->host_initiated)
+			return 1;
+		vcpu->arch.msr_platform_info = data;
+		break;
+	case MSR_MISC_FEATURES_ENABLES:
+		if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT ||
+		    (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
+		     !(vcpu->arch.msr_platform_info & MSR_PLATFORM_INFO_CPUID_FAULT)))
+			return 1;
+		vcpu->arch.msr_misc_features_enables = data;
+		break;
+#ifdef CONFIG_X86_64
+	case MSR_IA32_XFD:
+		if (!msr_info->host_initiated &&
+		    !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD))
+			return 1;
+
+		if (data & ~kvm_guest_supported_xfd(vcpu))
+			return 1;
+
+		fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data);
+		break;
+	case MSR_IA32_XFD_ERR:
+		if (!msr_info->host_initiated &&
+		    !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD))
+			return 1;
+
+		if (data & ~kvm_guest_supported_xfd(vcpu))
+			return 1;
+
+		vcpu->arch.guest_fpu.xfd_err = data;
+		break;
+#endif
+	case MSR_IA32_U_CET:
+	case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP:
+		kvm_set_xstate_msr(vcpu, msr_info);
+		break;
+	default:
+		if (kvm_pmu_is_valid_msr(vcpu, msr))
+			return kvm_pmu_set_msr(vcpu, msr_info);
+
+		return KVM_MSR_RET_UNSUPPORTED;
+	}
+	return 0;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_msr_common);
+
+static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
+{
+	u64 data;
+	u64 mcg_cap = vcpu->arch.mcg_cap;
+	unsigned bank_num = mcg_cap & 0xff;
+	u32 offset, last_msr;
+
+	switch (msr) {
+	case MSR_IA32_P5_MC_ADDR:
+	case MSR_IA32_P5_MC_TYPE:
+		data = 0;
+		break;
+	case MSR_IA32_MCG_CAP:
+		data = vcpu->arch.mcg_cap;
+		break;
+	case MSR_IA32_MCG_CTL:
+		if (!(mcg_cap & MCG_CTL_P) && !host)
+			return 1;
+		data = vcpu->arch.mcg_ctl;
+		break;
+	case MSR_IA32_MCG_STATUS:
+		data = vcpu->arch.mcg_status;
+		break;
+	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
+		last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1;
+		if (msr > last_msr)
+			return 1;
+
+		if (!(mcg_cap & MCG_CMCI_P) && !host)
+			return 1;
+		offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2,
+					    last_msr + 1 - MSR_IA32_MC0_CTL2);
+		data = vcpu->arch.mci_ctl2_banks[offset];
+		break;
+	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
+		last_msr = MSR_IA32_MCx_CTL(bank_num) - 1;
+		if (msr > last_msr)
+			return 1;
+
+		offset = array_index_nospec(msr - MSR_IA32_MC0_CTL,
+					    last_msr + 1 - MSR_IA32_MC0_CTL);
+		data = vcpu->arch.mce_banks[offset];
+		break;
+	default:
+		return 1;
+	}
+	*pdata = data;
+	return 0;
+}
+
+int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
+{
+	switch (msr_info->index) {
+	case MSR_IA32_PLATFORM_ID:
+	case MSR_IA32_EBL_CR_POWERON:
+	case MSR_IA32_LASTBRANCHFROMIP:
+	case MSR_IA32_LASTBRANCHTOIP:
+	case MSR_IA32_LASTINTFROMIP:
+	case MSR_IA32_LASTINTTOIP:
+	case MSR_AMD64_SYSCFG:
+	case MSR_K8_TSEG_ADDR:
+	case MSR_K8_TSEG_MASK:
+	case MSR_VM_HSAVE_PA:
+	case MSR_K8_INT_PENDING_MSG:
+	case MSR_AMD64_NB_CFG:
+	case MSR_FAM10H_MMIO_CONF_BASE:
+	case MSR_AMD64_BU_CFG2:
+	case MSR_IA32_PERF_CTL:
+	case MSR_AMD64_DC_CFG:
+	case MSR_AMD64_TW_CFG:
+	case MSR_F15H_EX_CFG:
+	/*
+	 * Intel Sandy Bridge CPUs must support the RAPL (running average power
+	 * limit) MSRs. Just return 0, as we do not want to expose the host
+	 * data here. Do not conditionalize this on CPUID, as KVM does not do
+	 * so for existing CPU-specific MSRs.
+	 */
+	case MSR_RAPL_POWER_UNIT:
+	case MSR_PP0_ENERGY_STATUS:	/* Power plane 0 (core) */
+	case MSR_PP1_ENERGY_STATUS:	/* Power plane 1 (graphics uncore) */
+	case MSR_PKG_ENERGY_STATUS:	/* Total package */
+	case MSR_DRAM_ENERGY_STATUS:	/* DRAM controller */
+		msr_info->data = 0;
+		break;
+	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
+	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
+	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
+	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
+		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
+			return kvm_pmu_get_msr(vcpu, msr_info);
+		msr_info->data = 0;
+		break;
+	case MSR_IA32_UCODE_REV:
+		msr_info->data = vcpu->arch.microcode_version;
+		break;
+	case MSR_IA32_ARCH_CAPABILITIES:
+		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
+			return KVM_MSR_RET_UNSUPPORTED;
+		msr_info->data = vcpu->arch.arch_capabilities;
+		break;
+	case MSR_IA32_PERF_CAPABILITIES:
+		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_PDCM))
+			return KVM_MSR_RET_UNSUPPORTED;
+		msr_info->data = vcpu->arch.perf_capabilities;
+		break;
+	case MSR_IA32_POWER_CTL:
+		msr_info->data = vcpu->arch.msr_ia32_power_ctl;
+		break;
+	case MSR_IA32_TSC: {
+		/*
+		 * Intel SDM states that MSR_IA32_TSC read adds the TSC offset
+		 * even when not intercepted. AMD manual doesn't explicitly
+		 * state this but appears to behave the same.
+		 *
+		 * On userspace reads and writes, however, we unconditionally
+		 * return L1's TSC value to ensure backwards-compatible
+		 * behavior for migration.
+		 */
+		u64 offset, ratio;
+
+		if (msr_info->host_initiated) {
+			offset = vcpu->arch.l1_tsc_offset;
+			ratio = vcpu->arch.l1_tsc_scaling_ratio;
+		} else {
+			offset = vcpu->arch.tsc_offset;
+			ratio = vcpu->arch.tsc_scaling_ratio;
+		}
+
+		msr_info->data = kvm_scale_tsc(rdtsc(), ratio) + offset;
+		break;
+	}
+	case MSR_IA32_CR_PAT:
+		msr_info->data = vcpu->arch.pat;
+		break;
+	case MSR_MTRRcap:
+	case MTRRphysBase_MSR(0) ... MSR_MTRRfix4K_F8000:
+	case MSR_MTRRdefType:
+		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);
+	case 0xcd: /* fsb frequency */
+		msr_info->data = 3;
+		break;
+		/*
+		 * MSR_EBC_FREQUENCY_ID
+		 * Conservative value valid for even the basic CPU models.
+		 * Models 0,1: 000 in bits 23:21 indicating a bus speed of
+		 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
+		 * and 266MHz for model 3, or 4. Set Core Clock
+		 * Frequency to System Bus Frequency Ratio to 1 (bits
+		 * 31:24) even though these are only valid for CPU
+		 * models > 2, however guests may end up dividing or
+		 * multiplying by zero otherwise.
+		 */
+	case MSR_EBC_FREQUENCY_ID:
+		msr_info->data = 1 << 24;
+		break;
+	case MSR_IA32_APICBASE:
+		msr_info->data = vcpu->arch.apic_base;
+		break;
+	case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff:
+		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);
+	case MSR_IA32_TSC_DEADLINE:
+		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);
+		break;
+	case MSR_IA32_TSC_ADJUST:
+		msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
+		break;
+	case MSR_IA32_MISC_ENABLE:
+		msr_info->data = vcpu->arch.ia32_misc_enable_msr;
+		break;
+	case MSR_IA32_SMBASE:
+		if (!IS_ENABLED(CONFIG_KVM_SMM) || !msr_info->host_initiated)
+			return 1;
+		msr_info->data = vcpu->arch.smbase;
+		break;
+	case MSR_SMI_COUNT:
+		msr_info->data = vcpu->arch.smi_count;
+		break;
+	case MSR_IA32_PERF_STATUS:
+		/* TSC increment by tick */
+		msr_info->data = 1000ULL;
+		/* CPU multiplier */
+		msr_info->data |= (((uint64_t)4ULL) << 40);
+		break;
+	case MSR_EFER:
+		msr_info->data = vcpu->arch.efer;
+		break;
+	case MSR_KVM_WALL_CLOCK:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		msr_info->data = vcpu->kvm->arch.wall_clock;
+		break;
+	case MSR_KVM_WALL_CLOCK_NEW:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		msr_info->data = vcpu->kvm->arch.wall_clock;
+		break;
+	case MSR_KVM_SYSTEM_TIME:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		msr_info->data = vcpu->arch.time;
+		break;
+	case MSR_KVM_SYSTEM_TIME_NEW:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		msr_info->data = vcpu->arch.time;
+		break;
+	case MSR_KVM_ASYNC_PF_EN:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		msr_info->data = vcpu->arch.apf.msr_en_val;
+		break;
+	case MSR_KVM_ASYNC_PF_INT:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		msr_info->data = vcpu->arch.apf.msr_int_val;
+		break;
+	case MSR_KVM_ASYNC_PF_ACK:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		msr_info->data = 0;
+		break;
+	case MSR_KVM_STEAL_TIME:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		msr_info->data = vcpu->arch.st.msr_val;
+		break;
+	case MSR_KVM_PV_EOI_EN:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		msr_info->data = vcpu->arch.pv_eoi.msr_val;
+		break;
+	case MSR_KVM_POLL_CONTROL:
+		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
+			return KVM_MSR_RET_UNSUPPORTED;
+
+		msr_info->data = vcpu->arch.msr_kvm_poll_control;
+		break;
+	case MSR_IA32_P5_MC_ADDR:
+	case MSR_IA32_P5_MC_TYPE:
+	case MSR_IA32_MCG_CAP:
+	case MSR_IA32_MCG_CTL:
+	case MSR_IA32_MCG_STATUS:
+	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
+	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
+		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,
+				   msr_info->host_initiated);
+	case MSR_IA32_XSS:
+		if (!msr_info->host_initiated &&
+		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
+			return 1;
+		msr_info->data = vcpu->arch.ia32_xss;
+		break;
+	case MSR_K7_CLK_CTL:
+		/*
+		 * Provide expected ramp-up count for K7. All other
+		 * are set to zero, indicating minimum divisors for
+		 * every field.
+		 *
+		 * This prevents guest kernels on AMD host with CPU
+		 * type 6, model 8 and higher from exploding due to
+		 * the rdmsr failing.
+		 */
+		msr_info->data = 0x20000000;
+		break;
+#ifdef CONFIG_KVM_HYPERV
+	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
+	case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER:
+	case HV_X64_MSR_SYNDBG_OPTIONS:
+	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
+	case HV_X64_MSR_CRASH_CTL:
+	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
+	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
+	case HV_X64_MSR_TSC_EMULATION_CONTROL:
+	case HV_X64_MSR_TSC_EMULATION_STATUS:
+	case HV_X64_MSR_TSC_INVARIANT_CONTROL:
+		return kvm_hv_get_msr_common(vcpu,
+					     msr_info->index, &msr_info->data,
+					     msr_info->host_initiated);
+#endif
+	case MSR_IA32_BBL_CR_CTL3:
+		/* This legacy MSR exists but isn't fully documented in current
+		 * silicon.  It is however accessed by winxp in very narrow
+		 * scenarios where it sets bit #19, itself documented as
+		 * a "reserved" bit.  Best effort attempt to source coherent
+		 * read data here should the balance of the register be
+		 * interpreted by the guest:
+		 *
+		 * L2 cache control register 3: 64GB range, 256KB size,
+		 * enabled, latency 0x1, configured
+		 */
+		msr_info->data = 0xbe702111;
+		break;
+	case MSR_AMD64_OSVW_ID_LENGTH:
+		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW))
+			return 1;
+		msr_info->data = vcpu->arch.osvw.length;
+		break;
+	case MSR_AMD64_OSVW_STATUS:
+		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW))
+			return 1;
+		msr_info->data = vcpu->arch.osvw.status;
+		break;
+	case MSR_PLATFORM_INFO:
+		if (!msr_info->host_initiated &&
+		    !vcpu->kvm->arch.guest_can_read_msr_platform_info)
+			return 1;
+		msr_info->data = vcpu->arch.msr_platform_info;
+		break;
+	case MSR_MISC_FEATURES_ENABLES:
+		msr_info->data = vcpu->arch.msr_misc_features_enables;
+		break;
+	case MSR_K7_HWCR:
+		msr_info->data = vcpu->arch.msr_hwcr;
+		break;
+#ifdef CONFIG_X86_64
+	case MSR_IA32_XFD:
+		if (!msr_info->host_initiated &&
+		    !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD))
+			return 1;
+
+		msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd;
+		break;
+	case MSR_IA32_XFD_ERR:
+		if (!msr_info->host_initiated &&
+		    !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD))
+			return 1;
+
+		msr_info->data = vcpu->arch.guest_fpu.xfd_err;
+		break;
+#endif
+	case MSR_IA32_U_CET:
+	case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP:
+		kvm_get_xstate_msr(vcpu, msr_info);
+		break;
+	default:
+		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
+			return kvm_pmu_get_msr(vcpu, msr_info);
+
+		return KVM_MSR_RET_UNSUPPORTED;
+	}
+	return 0;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_msr_common);
+
+static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
+{
+	return kvm_get_msr_ignored_check(vcpu, index, data, true);
+}
+
+static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
+{
+	u64 val;
+
+	/*
+	 * Reject writes to immutable feature MSRs if the vCPU model is frozen,
+	 * as KVM doesn't support modifying the guest vCPU model on the fly,
+	 * e.g. changing the VMX capabilities MSRs while L2 is active is
+	 * nonsensical.  Allow writes of the same value, e.g. so that userspace
+	 * can blindly stuff all MSRs when emulating RESET.
+	 */
+	if (!kvm_can_set_cpuid_and_feature_msrs(vcpu) &&
+	    kvm_is_immutable_feature_msr(index) &&
+	    (do_get_msr(vcpu, index, &val) || *data != val))
+		return -EINVAL;
+
+	return kvm_set_msr_ignored_check(vcpu, index, *data, true);
+}
+
+/*
+ * Read or write a bunch of msrs. All parameters are kernel addresses.
+ *
+ * @return number of msrs set successfully.
+ */
+static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
+		    struct kvm_msr_entry *entries,
+		    int (*do_msr)(struct kvm_vcpu *vcpu,
+				  unsigned index, u64 *data))
+{
+	bool fpu_loaded = false;
+	int i;
+
+	for (i = 0; i < msrs->nmsrs; ++i) {
+		/*
+		 * If userspace is accessing one or more XSTATE-managed MSRs,
+		 * temporarily load the guest's FPU state so that the guest's
+		 * MSR value(s) is resident in hardware and thus can be accessed
+		 * via RDMSR/WRMSR.
+		 */
+		if (!fpu_loaded && is_xstate_managed_msr(vcpu, entries[i].index)) {
+			kvm_load_guest_fpu(vcpu);
+			fpu_loaded = true;
+		}
+		if (do_msr(vcpu, entries[i].index, &entries[i].data))
+			break;
+	}
+	if (fpu_loaded)
+		kvm_put_guest_fpu(vcpu);
+
+	return i;
+}
+
+/*
+ * Read or write a bunch of msrs. Parameters are user addresses.
+ *
+ * @return number of msrs set successfully.
+ */
+static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
+		  int (*do_msr)(struct kvm_vcpu *vcpu,
+				unsigned index, u64 *data),
+		  int writeback)
+{
+	struct kvm_msrs msrs;
+	struct kvm_msr_entry *entries;
+	unsigned size;
+	int r;
+
+	r = -EFAULT;
+	if (copy_from_user(&msrs, user_msrs, sizeof(msrs)))
+		goto out;
+
+	r = -E2BIG;
+	if (msrs.nmsrs >= MAX_IO_MSRS)
+		goto out;
+
+	size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
+	entries = memdup_user(user_msrs->entries, size);
+	if (IS_ERR(entries)) {
+		r = PTR_ERR(entries);
+		goto out;
+	}
+
+	r = __msr_io(vcpu, &msrs, entries, do_msr);
+
+	if (writeback && copy_to_user(user_msrs->entries, entries, size))
+		r = -EFAULT;
+
+	kfree(entries);
+out:
+	return r;
+}
+
+int kvm_get_feature_msrs(struct kvm_msrs __user *user_msrs)
+{
+	return msr_io(NULL, user_msrs, do_get_feature_msr, 1);
+}
+
+int kvm_get_msrs(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs)
+{
+	guard(srcu)(&vcpu->kvm->srcu);
+
+	return msr_io(vcpu, user_msrs, do_get_msr, 1);
+}
+
+int kvm_set_msrs(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs)
+{
+	guard(srcu)(&vcpu->kvm->srcu);
+
+	return msr_io(vcpu, user_msrs, do_set_msr, 0);
+}
+
+static int kvm_get_one_msr(struct kvm_vcpu *vcpu, u32 msr, u64 __user *user_val)
+{
+	u64 val;
+
+	if (do_get_msr(vcpu, msr, &val))
+		return -EINVAL;
+
+	if (put_user(val, user_val))
+		return -EFAULT;
+
+	return 0;
+}
+
+static int kvm_set_one_msr(struct kvm_vcpu *vcpu, u32 msr, u64 __user *user_val)
+{
+	u64 val;
+
+	if (get_user(val, user_val))
+		return -EFAULT;
+
+	if (do_set_msr(vcpu, msr, &val))
+		return -EINVAL;
+
+	return 0;
+}
+
+struct kvm_x86_reg_id {
+	__u32 index;
+	__u8  type;
+	__u8  rsvd1;
+	__u8  rsvd2:4;
+	__u8  size:4;
+	__u8  x86;
+};
+
+static int kvm_translate_kvm_reg(struct kvm_vcpu *vcpu,
+				 struct kvm_x86_reg_id *reg)
+{
+	switch (reg->index) {
+	case KVM_REG_GUEST_SSP:
+		/*
+		 * FIXME: If host-initiated accesses are ever exempted from
+		 * ignore_msrs (in kvm_do_msr_access()), drop this manual check
+		 * and rely on KVM's standard checks to reject accesses to regs
+		 * that don't exist.
+		 */
+		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK))
+			return -EINVAL;
+
+		reg->type = KVM_X86_REG_TYPE_MSR;
+		reg->index = MSR_KVM_INTERNAL_GUEST_SSP;
+		break;
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
+int kvm_get_set_one_reg(struct kvm_vcpu *vcpu, unsigned int ioctl,
+			void __user *argp)
+{
+	struct kvm_one_reg one_reg;
+	struct kvm_x86_reg_id *reg;
+	u64 __user *user_val;
+	bool load_fpu;
+	int r;
+
+	if (copy_from_user(&one_reg, argp, sizeof(one_reg)))
+		return -EFAULT;
+
+	if ((one_reg.id & KVM_REG_ARCH_MASK) != KVM_REG_X86)
+		return -EINVAL;
+
+	reg = (struct kvm_x86_reg_id *)&one_reg.id;
+	if (reg->rsvd1 || reg->rsvd2)
+		return -EINVAL;
+
+	if (reg->type == KVM_X86_REG_TYPE_KVM) {
+		r = kvm_translate_kvm_reg(vcpu, reg);
+		if (r)
+			return r;
+	}
+
+	if (reg->type != KVM_X86_REG_TYPE_MSR)
+		return -EINVAL;
+
+	if ((one_reg.id & KVM_REG_SIZE_MASK) != KVM_REG_SIZE_U64)
+		return -EINVAL;
+
+	guard(srcu)(&vcpu->kvm->srcu);
+
+	load_fpu = is_xstate_managed_msr(vcpu, reg->index);
+	if (load_fpu)
+		kvm_load_guest_fpu(vcpu);
+
+	user_val = u64_to_user_ptr(one_reg.addr);
+	if (ioctl == KVM_GET_ONE_REG)
+		r = kvm_get_one_msr(vcpu, reg->index, user_val);
+	else
+		r = kvm_set_one_msr(vcpu, reg->index, user_val);
+
+	if (load_fpu)
+		kvm_put_guest_fpu(vcpu);
+	return r;
+}
+
+int kvm_get_reg_list(struct kvm_vcpu *vcpu,
+		     struct kvm_reg_list __user *user_list)
+{
+	u64 nr_regs = guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) ? 1 : 0;
+	u64 user_nr_regs;
+
+	if (get_user(user_nr_regs, &user_list->n))
+		return -EFAULT;
+
+	if (put_user(nr_regs, &user_list->n))
+		return -EFAULT;
+
+	if (user_nr_regs < nr_regs)
+		return -E2BIG;
+
+	if (nr_regs &&
+	    put_user(KVM_X86_REG_KVM(KVM_REG_GUEST_SSP), &user_list->reg[0]))
+		return -EFAULT;
+
+	return 0;
+}
+
+static struct kvm_x86_msr_filter *kvm_alloc_msr_filter(bool default_allow)
+{
+	struct kvm_x86_msr_filter *msr_filter;
+
+	msr_filter = kzalloc_obj(*msr_filter, GFP_KERNEL_ACCOUNT);
+	if (!msr_filter)
+		return NULL;
+
+	msr_filter->default_allow = default_allow;
+	return msr_filter;
+}
+
+void kvm_free_msr_filter(struct kvm_x86_msr_filter *msr_filter)
+{
+	u32 i;
+
+	if (!msr_filter)
+		return;
+
+	for (i = 0; i < msr_filter->count; i++)
+		kfree(msr_filter->ranges[i].bitmap);
+
+	kfree(msr_filter);
+}
+
+static int kvm_add_msr_filter(struct kvm_x86_msr_filter *msr_filter,
+			      struct kvm_msr_filter_range *user_range)
+{
+	unsigned long *bitmap;
+	size_t bitmap_size;
+
+	if (!user_range->nmsrs)
+		return 0;
+
+	if (user_range->flags & ~KVM_MSR_FILTER_RANGE_VALID_MASK)
+		return -EINVAL;
+
+	if (!user_range->flags)
+		return -EINVAL;
+
+	bitmap_size = BITS_TO_LONGS(user_range->nmsrs) * sizeof(long);
+	if (!bitmap_size || bitmap_size > KVM_MSR_FILTER_MAX_BITMAP_SIZE)
+		return -EINVAL;
+
+	bitmap = memdup_user((__user u8*)user_range->bitmap, bitmap_size);
+	if (IS_ERR(bitmap))
+		return PTR_ERR(bitmap);
+
+	msr_filter->ranges[msr_filter->count] = (struct msr_bitmap_range) {
+		.flags = user_range->flags,
+		.base = user_range->base,
+		.nmsrs = user_range->nmsrs,
+		.bitmap = bitmap,
+	};
+
+	msr_filter->count++;
+	return 0;
+}
+
+int kvm_vm_ioctl_set_msr_filter(struct kvm *kvm, struct kvm_msr_filter *filter)
+{
+	struct kvm_x86_msr_filter *new_filter, *old_filter;
+	bool default_allow;
+	bool empty = true;
+	int r;
+	u32 i;
+
+	if (filter->flags & ~KVM_MSR_FILTER_VALID_MASK)
+		return -EINVAL;
+
+	for (i = 0; i < ARRAY_SIZE(filter->ranges); i++)
+		empty &= !filter->ranges[i].nmsrs;
+
+	default_allow = !(filter->flags & KVM_MSR_FILTER_DEFAULT_DENY);
+	if (empty && !default_allow)
+		return -EINVAL;
+
+	new_filter = kvm_alloc_msr_filter(default_allow);
+	if (!new_filter)
+		return -ENOMEM;
+
+	for (i = 0; i < ARRAY_SIZE(filter->ranges); i++) {
+		r = kvm_add_msr_filter(new_filter, &filter->ranges[i]);
+		if (r) {
+			kvm_free_msr_filter(new_filter);
+			return r;
+		}
+	}
+
+	mutex_lock(&kvm->lock);
+	old_filter = rcu_replace_pointer(kvm->arch.msr_filter, new_filter,
+					 mutex_is_locked(&kvm->lock));
+	mutex_unlock(&kvm->lock);
+	synchronize_srcu(&kvm->srcu);
+
+	kvm_free_msr_filter(old_filter);
+
+	/*
+	 * Recalc MSR intercepts as userspace may want to intercept accesses to
+	 * MSRs that KVM would otherwise pass through to the guest.
+	 */
+	kvm_make_all_cpus_request(kvm, KVM_REQ_RECALC_INTERCEPTS);
+
+	return 0;
+}
+
+
+static void kvm_probe_feature_msr(u32 msr_index)
+{
+	u64 data;
+
+	if (kvm_get_feature_msr(NULL, msr_index, &data, true))
+		return;
+
+	msr_based_features[num_msr_based_features++] = msr_index;
+}
+
+static void kvm_probe_msr_to_save(u32 msr_index)
+{
+	u64 dummy;
+
+	if (rdmsrq_safe(msr_index, &dummy))
+		return;
+
+	/*
+	 * Even MSRs that are valid in the host may not be exposed to guests in
+	 * some cases.
+	 */
+	switch (msr_index) {
+	case MSR_IA32_BNDCFGS:
+		if (!kvm_mpx_supported())
+			return;
+		break;
+	case MSR_TSC_AUX:
+		if (!kvm_cpu_cap_has(X86_FEATURE_RDTSCP) &&
+		    !kvm_cpu_cap_has(X86_FEATURE_RDPID))
+			return;
+		break;
+	case MSR_IA32_UMWAIT_CONTROL:
+		if (!kvm_cpu_cap_has(X86_FEATURE_WAITPKG))
+			return;
+		break;
+	case MSR_IA32_RTIT_CTL:
+	case MSR_IA32_RTIT_STATUS:
+		if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT))
+			return;
+		break;
+	case MSR_IA32_RTIT_CR3_MATCH:
+		if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
+		    !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering))
+			return;
+		break;
+	case MSR_IA32_RTIT_OUTPUT_BASE:
+	case MSR_IA32_RTIT_OUTPUT_MASK:
+		if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
+		    (!intel_pt_validate_hw_cap(PT_CAP_topa_output) &&
+		     !intel_pt_validate_hw_cap(PT_CAP_single_range_output)))
+			return;
+		break;
+	case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
+		if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
+		    (msr_index - MSR_IA32_RTIT_ADDR0_A >=
+		     intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2))
+			return;
+		break;
+	case MSR_ARCH_PERFMON_PERFCTR0 ...
+	     MSR_ARCH_PERFMON_PERFCTR0 + KVM_MAX_NR_GP_COUNTERS - 1:
+		if (msr_index - MSR_ARCH_PERFMON_PERFCTR0 >=
+		    kvm_pmu_cap.num_counters_gp)
+			return;
+		break;
+	case MSR_ARCH_PERFMON_EVENTSEL0 ...
+	     MSR_ARCH_PERFMON_EVENTSEL0 + KVM_MAX_NR_GP_COUNTERS - 1:
+		if (msr_index - MSR_ARCH_PERFMON_EVENTSEL0 >=
+		    kvm_pmu_cap.num_counters_gp)
+			return;
+		break;
+	case MSR_ARCH_PERFMON_FIXED_CTR0 ...
+	     MSR_ARCH_PERFMON_FIXED_CTR0 + KVM_MAX_NR_FIXED_COUNTERS - 1:
+		if (msr_index - MSR_ARCH_PERFMON_FIXED_CTR0 >=
+		    kvm_pmu_cap.num_counters_fixed)
+			return;
+		break;
+	case MSR_AMD64_PERF_CNTR_GLOBAL_CTL:
+	case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS:
+	case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR:
+	case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET:
+		if (!kvm_cpu_cap_has(X86_FEATURE_PERFMON_V2))
+			return;
+		break;
+	case MSR_IA32_XFD:
+	case MSR_IA32_XFD_ERR:
+		if (!kvm_cpu_cap_has(X86_FEATURE_XFD))
+			return;
+		break;
+	case MSR_IA32_TSX_CTRL:
+		if (!(kvm_get_arch_capabilities() & ARCH_CAP_TSX_CTRL_MSR))
+			return;
+		break;
+	case MSR_IA32_XSS:
+		if (!kvm_caps.supported_xss)
+			return;
+		break;
+	case MSR_IA32_U_CET:
+	case MSR_IA32_S_CET:
+		if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) &&
+		    !kvm_cpu_cap_has(X86_FEATURE_IBT))
+			return;
+		break;
+	case MSR_IA32_INT_SSP_TAB:
+		if (!kvm_cpu_cap_has(X86_FEATURE_LM))
+			return;
+		fallthrough;
+	case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP:
+		if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK))
+			return;
+		break;
+	default:
+		break;
+	}
+
+	msrs_to_save[num_msrs_to_save++] = msr_index;
+}
+
+void kvm_init_msr_lists(void)
+{
+	unsigned i;
+
+	BUILD_BUG_ON_MSG(KVM_MAX_NR_FIXED_COUNTERS != 3,
+			 "Please update the fixed PMCs in msrs_to_save_pmu[]");
+
+	num_msrs_to_save = 0;
+	num_emulated_msrs = 0;
+	num_msr_based_features = 0;
+
+	for (i = 0; i < ARRAY_SIZE(msrs_to_save_base); i++)
+		kvm_probe_msr_to_save(msrs_to_save_base[i]);
+
+	if (enable_pmu) {
+		for (i = 0; i < ARRAY_SIZE(msrs_to_save_pmu); i++)
+			kvm_probe_msr_to_save(msrs_to_save_pmu[i]);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) {
+		if (!kvm_x86_call(has_emulated_msr)(NULL,
+						    emulated_msrs_all[i]))
+			continue;
+
+		emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i];
+	}
+
+	for (i = KVM_FIRST_EMULATED_VMX_MSR; i <= KVM_LAST_EMULATED_VMX_MSR; i++)
+		kvm_probe_feature_msr(i);
+
+	for (i = 0; i < ARRAY_SIZE(msr_based_features_all_except_vmx); i++)
+		kvm_probe_feature_msr(msr_based_features_all_except_vmx[i]);
+}
+
+int kvm_spec_ctrl_test_value(u64 value)
+{
+	/*
+	 * test that setting IA32_SPEC_CTRL to given value
+	 * is allowed by the host processor
+	 */
+
+	u64 saved_value;
+	unsigned long flags;
+	int ret = 0;
+
+	local_irq_save(flags);
+
+	if (rdmsrq_safe(MSR_IA32_SPEC_CTRL, &saved_value))
+		ret = 1;
+	else if (wrmsrq_safe(MSR_IA32_SPEC_CTRL, value))
+		ret = 1;
+	else
+		wrmsrq(MSR_IA32_SPEC_CTRL, saved_value);
+
+	local_irq_restore(flags);
+
+	return ret;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_spec_ctrl_test_value);
diff --git a/arch/x86/kvm/msrs.h b/arch/x86/kvm/msrs.h
new file mode 100644
index 0000000000000..9c5c6b33e58f5
--- /dev/null
+++ b/arch/x86/kvm/msrs.h
@@ -0,0 +1,156 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef ARCH_X86_KVM_MSRS_H
+#define ARCH_X86_KVM_MSRS_H
+
+#include <linux/kvm_host.h>
+#include <linux/user-return-notifier.h>
+
+#include "cpuid.h"
+#include "regs.h"
+
+extern bool report_ignored_msrs;
+extern bool ignore_msrs;
+
+extern u32 __read_mostly kvm_nr_uret_msrs;
+
+static inline void kvm_pr_unimpl_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
+{
+	if (report_ignored_msrs)
+		vcpu_unimpl(vcpu, "Unhandled WRMSR(0x%x) = 0x%llx\n", msr, data);
+}
+
+static inline void kvm_pr_unimpl_rdmsr(struct kvm_vcpu *vcpu, u32 msr)
+{
+	if (report_ignored_msrs)
+		vcpu_unimpl(vcpu, "Unhandled RDMSR(0x%x)\n", msr);
+}
+
+/*
+ * The first...last VMX feature MSRs that are emulated by KVM.  This may or may
+ * not cover all known VMX MSRs, as KVM doesn't emulate an MSR until there's an
+ * associated feature that KVM supports for nested virtualization.
+ */
+#define KVM_FIRST_EMULATED_VMX_MSR	MSR_IA32_VMX_BASIC
+#define KVM_LAST_EMULATED_VMX_MSR	MSR_IA32_VMX_VMFUNC
+
+/*
+ * KVM's internal, non-ABI indices for synthetic MSRs. The values themselves
+ * are arbitrary and have no meaning, the only requirement is that they don't
+ * conflict with "real" MSRs that KVM supports. Use values at the upper end
+ * of KVM's reserved paravirtual MSR range to minimize churn, i.e. these values
+ * will be usable until KVM exhausts its supply of paravirtual MSR indices.
+ */
+#define MSR_KVM_INTERNAL_GUEST_SSP	0x4b564dff
+
+#define MSR_IA32_CR_PAT_DEFAULT	\
+	PAT_VALUE(WB, WT, UC_MINUS, UC, WB, WT, UC_MINUS, UC)
+
+void kvm_init_msr_lists(void);
+int kvm_get_msr_index_list(struct kvm_msr_list __user *user_msr_list);
+int kvm_get_feature_msr_index_list(struct kvm_msr_list __user *user_msr_list);
+int kvm_get_feature_msrs(struct kvm_msrs __user *user_msrs);
+
+int kvm_get_msrs(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs);
+int kvm_set_msrs(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs);
+
+int kvm_get_set_one_reg(struct kvm_vcpu *vcpu, unsigned int ioctl,
+			void __user *argp);
+int kvm_get_reg_list(struct kvm_vcpu *vcpu,
+		     struct kvm_reg_list __user *user_list);
+
+void kvm_enable_efer_bits(u64);
+bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer);
+int kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data);
+int kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data);
+int __kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data);
+int __kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data);
+int kvm_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data);
+int kvm_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data);
+int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu);
+int kvm_emulate_rdmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg);
+int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu);
+int kvm_emulate_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg);
+
+fastpath_t handle_fastpath_wrmsr(struct kvm_vcpu *vcpu);
+fastpath_t handle_fastpath_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg);
+
+int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);
+int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);
+
+int kvm_add_user_return_msr(u32 msr);
+int kvm_find_user_return_msr(u32 msr);
+int kvm_set_user_return_msr(unsigned index, u64 val, u64 mask);
+u64 kvm_get_user_return_msr(unsigned int slot);
+
+static inline bool kvm_is_supported_user_return_msr(u32 msr)
+{
+	return kvm_find_user_return_msr(msr) >= 0;
+}
+
+void kvm_user_return_msr_cpu_online(void);
+void drop_user_return_notifiers(void);
+void kvm_destroy_user_return_msrs(void);
+
+int kvm_emulator_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 msr_index,
+				     u64 *pdata);
+int kvm_emulator_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 msr_index,
+				     u64 data);
+int kvm_emulator_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata);
+
+bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type);
+
+enum kvm_msr_access {
+	MSR_TYPE_R	= BIT(0),
+	MSR_TYPE_W	= BIT(1),
+	MSR_TYPE_RW	= MSR_TYPE_R | MSR_TYPE_W,
+};
+
+/*
+ * Internal error codes that are used to indicate that MSR emulation encountered
+ * an error that should result in #GP in the guest, unless userspace handles it.
+ * Note, '1', '0', and negative numbers are off limits, as they are used by KVM
+ * as part of KVM's lightly documented internal KVM_RUN return codes.
+ *
+ * UNSUPPORTED	- The MSR isn't supported, either because it is completely
+ *		  unknown to KVM, or because the MSR should not exist according
+ *		  to the vCPU model.
+ *
+ * FILTERED	- Access to the MSR is denied by a userspace MSR filter.
+ */
+#define  KVM_MSR_RET_UNSUPPORTED	2
+#define  KVM_MSR_RET_FILTERED		3
+
+int kvm_vm_ioctl_set_msr_filter(struct kvm *kvm, struct kvm_msr_filter *filter);
+void kvm_free_msr_filter(struct kvm_x86_msr_filter *msr_filter);
+
+int kvm_mtrr_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data);
+int kvm_mtrr_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata);
+
+u64 kvm_get_arch_capabilities(void);
+int kvm_spec_ctrl_test_value(u64 value);
+
+#define CET_US_RESERVED_BITS		GENMASK(9, 6)
+#define CET_US_SHSTK_MASK_BITS		GENMASK(1, 0)
+#define CET_US_IBT_MASK_BITS		(GENMASK_ULL(5, 2) | GENMASK_ULL(63, 10))
+#define CET_US_LEGACY_BITMAP_BASE(data)	((data) >> 12)
+
+static inline bool kvm_is_valid_u_s_cet(struct kvm_vcpu *vcpu, u64 data)
+{
+	if (data & CET_US_RESERVED_BITS)
+		return false;
+	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) &&
+	    (data & CET_US_SHSTK_MASK_BITS))
+		return false;
+	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_IBT) &&
+	    (data & CET_US_IBT_MASK_BITS))
+		return false;
+	if (!IS_ALIGNED(CET_US_LEGACY_BITMAP_BASE(data), 4))
+		return false;
+	/* IBT can be suppressed iff the TRACKER isn't WAIT_ENDBR. */
+	if ((data & CET_SUPPRESS) && (data & CET_WAIT_ENDBR))
+		return false;
+
+	return true;
+}
+
+#endif
diff --git a/arch/x86/kvm/mtrr.c b/arch/x86/kvm/mtrr.c
index 6f74e2b27c1ed..c4ec024943bb8 100644
--- a/arch/x86/kvm/mtrr.c
+++ b/arch/x86/kvm/mtrr.c
@@ -19,7 +19,7 @@
 #include <asm/mtrr.h>
 
 #include "cpuid.h"
-#include "x86.h"
+#include "msrs.h"
 
 static u64 *find_mtrr(struct kvm_vcpu *vcpu, unsigned int msr)
 {
diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index dd1c57593f48b..7f777049d328a 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -43,6 +43,18 @@ module_param(enable_pmu, bool, 0444);
 bool __read_mostly enable_mediated_pmu;
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_mediated_pmu);
 
+struct kvm_x86_pmu_event_filter {
+	__u32 action;
+	__u32 nevents;
+	__u32 fixed_counter_bitmap;
+	__u32 flags;
+	__u32 nr_includes;
+	__u32 nr_excludes;
+	__u64 *includes;
+	__u64 *excludes;
+	__u64 events[] __counted_by(nevents);
+};
+
 struct kvm_pmu_emulated_event_selectors {
 	u64 INSTRUCTIONS_RETIRED;
 	u64 BRANCH_INSTRUCTIONS_RETIRED;
diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c
new file mode 100644
index 0000000000000..bd8147798cc3e
--- /dev/null
+++ b/arch/x86/kvm/regs.c
@@ -0,0 +1,874 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/kvm_host.h>
+
+#include "lapic.h"
+#include "mmu.h"
+#include "regs.h"
+#include "x86.h"
+
+unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
+{
+	/* Can't read the RIP when guest state is protected, just return 0 */
+	if (vcpu->arch.guest_state_protected)
+		return 0;
+
+	if (is_64_bit_mode(vcpu))
+		return kvm_rip_read(vcpu);
+	return (u32)(kvm_get_segment_base(vcpu, VCPU_SREG_CS) +
+		     kvm_rip_read(vcpu));
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_linear_rip);
+
+bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
+{
+	return kvm_get_linear_rip(vcpu) == linear_rip;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_is_linear_rip);
+
+unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
+{
+	unsigned long rflags;
+
+	rflags = kvm_x86_call(get_rflags)(vcpu);
+	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
+		rflags &= ~X86_EFLAGS_TF;
+	return rflags;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_rflags);
+
+void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
+{
+	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
+	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
+		rflags |= X86_EFLAGS_TF;
+	kvm_x86_call(set_rflags)(vcpu, rflags);
+}
+
+void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
+{
+	__kvm_set_rflags(vcpu, rflags);
+	kvm_make_request(KVM_REQ_EVENT, vcpu);
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_rflags);
+
+static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
+{
+	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
+		/*
+		 * We are here if userspace calls get_regs() in the middle of
+		 * instruction emulation. Registers state needs to be copied
+		 * back from emulation context to vcpu. Userspace shouldn't do
+		 * that usually, but some bad designed PV devices (vmware
+		 * backdoor interface) need this to work
+		 */
+		emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);
+		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
+	}
+	regs->rax = kvm_rax_read_raw(vcpu);
+	regs->rbx = kvm_rbx_read_raw(vcpu);
+	regs->rcx = kvm_rcx_read_raw(vcpu);
+	regs->rdx = kvm_rdx_read_raw(vcpu);
+	regs->rsi = kvm_rsi_read_raw(vcpu);
+	regs->rdi = kvm_rdi_read_raw(vcpu);
+	regs->rsp = kvm_rsp_read(vcpu);
+	regs->rbp = kvm_rbp_read_raw(vcpu);
+#ifdef CONFIG_X86_64
+	regs->r8 = kvm_r8_read_raw(vcpu);
+	regs->r9 = kvm_r9_read_raw(vcpu);
+	regs->r10 = kvm_r10_read_raw(vcpu);
+	regs->r11 = kvm_r11_read_raw(vcpu);
+	regs->r12 = kvm_r12_read_raw(vcpu);
+	regs->r13 = kvm_r13_read_raw(vcpu);
+	regs->r14 = kvm_r14_read_raw(vcpu);
+	regs->r15 = kvm_r15_read_raw(vcpu);
+#endif
+
+	regs->rip = kvm_rip_read(vcpu);
+	regs->rflags = kvm_get_rflags(vcpu);
+}
+
+int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
+{
+	if (vcpu->kvm->arch.has_protected_state &&
+	    vcpu->arch.guest_state_protected)
+		return -EINVAL;
+
+	vcpu_load(vcpu);
+	__get_regs(vcpu, regs);
+	vcpu_put(vcpu);
+	return 0;
+}
+
+static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
+{
+	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
+	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
+
+	kvm_rax_write_raw(vcpu, regs->rax);
+	kvm_rbx_write_raw(vcpu, regs->rbx);
+	kvm_rcx_write_raw(vcpu, regs->rcx);
+	kvm_rdx_write_raw(vcpu, regs->rdx);
+	kvm_rsi_write_raw(vcpu, regs->rsi);
+	kvm_rdi_write_raw(vcpu, regs->rdi);
+	kvm_rsp_write(vcpu, regs->rsp);
+	kvm_rbp_write_raw(vcpu, regs->rbp);
+#ifdef CONFIG_X86_64
+	kvm_r8_write_raw(vcpu, regs->r8);
+	kvm_r9_write_raw(vcpu, regs->r9);
+	kvm_r10_write_raw(vcpu, regs->r10);
+	kvm_r11_write_raw(vcpu, regs->r11);
+	kvm_r12_write_raw(vcpu, regs->r12);
+	kvm_r13_write_raw(vcpu, regs->r13);
+	kvm_r14_write_raw(vcpu, regs->r14);
+	kvm_r15_write_raw(vcpu, regs->r15);
+#endif
+
+	kvm_rip_write(vcpu, regs->rip);
+	kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);
+
+	vcpu->arch.exception.pending = false;
+	vcpu->arch.exception_vmexit.pending = false;
+
+	kvm_make_request(KVM_REQ_EVENT, vcpu);
+}
+
+int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
+{
+	if (vcpu->kvm->arch.has_protected_state &&
+	    vcpu->arch.guest_state_protected)
+		return -EINVAL;
+
+	vcpu_load(vcpu);
+	__set_regs(vcpu, regs);
+	vcpu_put(vcpu);
+	return 0;
+}
+
+static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
+{
+	return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2);
+}
+
+/*
+ * Load the pae pdptrs.  Return 1 if they are all valid, 0 otherwise.
+ */
+int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
+{
+	struct kvm_pagewalk *w = &vcpu->arch.gva_walk;
+	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
+	gpa_t real_gpa;
+	int i;
+	int ret;
+	u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)];
+
+	/*
+	 * If the MMU is nested, CR3 holds an L2 GPA and needs to be translated
+	 * to an L1 GPA.
+	 */
+	real_gpa = kvm_translate_gpa(vcpu, w, gfn_to_gpa(pdpt_gfn),
+				     PFERR_USER_MASK | PFERR_WRITE_MASK |
+				     PFERR_GUEST_PAGE_MASK, NULL, 0);
+	if (real_gpa == INVALID_GPA)
+		return 0;
+
+	/* Note the offset, PDPTRs are 32 byte aligned when using PAE paging. */
+	ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte,
+				       cr3 & GENMASK(11, 5), sizeof(pdpte));
+	if (ret < 0)
+		return 0;
+
+	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
+		if ((pdpte[i] & PT_PRESENT_MASK) &&
+		    (pdpte[i] & pdptr_rsvd_bits(vcpu))) {
+			return 0;
+		}
+	}
+
+	/*
+	 * Marking VCPU_REG_PDPTR dirty doesn't work for !tdp_enabled.
+	 * Shadow page roots need to be reconstructed instead.
+	 */
+	if (!tdp_enabled && memcmp(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs)))
+		kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.root_mmu,
+				   KVM_MMU_ROOT_CURRENT);
+
+	memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs));
+	kvm_register_mark_dirty(vcpu, VCPU_REG_PDPTR);
+	kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);
+	vcpu->arch.pdptrs_from_userspace = false;
+
+	return 1;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(load_pdptrs);
+
+static bool kvm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
+{
+#ifdef CONFIG_X86_64
+	if (cr0 & 0xffffffff00000000UL)
+		return false;
+#endif
+
+	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
+		return false;
+
+	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
+		return false;
+
+	return kvm_x86_call(is_valid_cr0)(vcpu, cr0);
+}
+
+void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0)
+{
+	/*
+	 * CR0.WP is incorporated into the MMU role, but only for non-nested,
+	 * indirect shadow MMUs.  If paging is disabled, no updates are needed
+	 * as there are no permission bits to emulate.  If TDP is enabled, the
+	 * MMU's metadata needs to be updated, e.g. so that emulating guest
+	 * translations does the right thing, but there's no need to unload the
+	 * root as CR0.WP doesn't affect SPTEs.
+	 */
+	if ((cr0 ^ old_cr0) == X86_CR0_WP) {
+		if (!(cr0 & X86_CR0_PG))
+			return;
+
+		if (tdp_enabled) {
+			kvm_init_mmu(vcpu);
+			return;
+		}
+	}
+
+	if ((cr0 ^ old_cr0) & X86_CR0_PG) {
+		/*
+		 * Clearing CR0.PG is defined to flush the TLB from the guest's
+		 * perspective.
+		 */
+		if (!(cr0 & X86_CR0_PG))
+			kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
+		/*
+		 * Check for async #PF completion events when enabling paging,
+		 * as the vCPU may have previously encountered async #PFs (it's
+		 * entirely legal for the guest to toggle paging on/off without
+		 * waiting for the async #PF queue to drain).
+		 */
+		else if (kvm_pv_async_pf_enabled(vcpu))
+			kvm_make_request(KVM_REQ_APF_READY, vcpu);
+	}
+
+	if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS)
+		kvm_mmu_reset_context(vcpu);
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_post_set_cr0);
+
+int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
+{
+	unsigned long old_cr0 = kvm_read_cr0(vcpu);
+
+	if (!kvm_is_valid_cr0(vcpu, cr0))
+		return 1;
+
+	cr0 |= X86_CR0_ET;
+
+	/* Write to CR0 reserved bits are ignored, even on Intel. */
+	cr0 &= ~CR0_RESERVED_BITS;
+
+#ifdef CONFIG_X86_64
+	if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) &&
+	    (cr0 & X86_CR0_PG)) {
+		int cs_db, cs_l;
+
+		if (!is_pae(vcpu))
+			return 1;
+		kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
+		if (cs_l)
+			return 1;
+	}
+#endif
+	if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) &&
+	    is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) &&
+	    !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))
+		return 1;
+
+	if (!(cr0 & X86_CR0_PG) &&
+	    (is_64_bit_mode(vcpu) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)))
+		return 1;
+
+	if (!(cr0 & X86_CR0_WP) && kvm_is_cr4_bit_set(vcpu, X86_CR4_CET))
+		return 1;
+
+	kvm_x86_call(set_cr0)(vcpu, cr0);
+
+	kvm_post_set_cr0(vcpu, old_cr0, cr0);
+
+	return 0;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr0);
+
+void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
+{
+	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_lmsw);
+
+int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
+{
+	bool skip_tlb_flush = false;
+	unsigned long pcid = 0;
+#ifdef CONFIG_X86_64
+	if (kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) {
+		skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH;
+		cr3 &= ~X86_CR3_PCID_NOFLUSH;
+		pcid = cr3 & X86_CR3_PCID_MASK;
+	}
+#endif
+
+	/* PDPTRs are always reloaded for PAE paging. */
+	if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu))
+		goto handle_tlb_flush;
+
+	/*
+	 * Do not condition the GPA check on long mode, this helper is used to
+	 * stuff CR3, e.g. for RSM emulation, and there is no guarantee that
+	 * the current vCPU mode is accurate.
+	 */
+	if (!kvm_vcpu_is_legal_cr3(vcpu, cr3))
+		return 1;
+
+	if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3))
+		return 1;
+
+	if (cr3 != kvm_read_cr3(vcpu))
+		kvm_mmu_new_pgd(vcpu, cr3);
+
+	vcpu->arch.cr3 = cr3;
+	kvm_register_mark_dirty(vcpu, VCPU_REG_CR3);
+	/* Do not call post_set_cr3, we do not get here for confidential guests.  */
+
+handle_tlb_flush:
+	/*
+	 * A load of CR3 that flushes the TLB flushes only the current PCID,
+	 * even if PCID is disabled, in which case PCID=0 is flushed.  It's a
+	 * moot point in the end because _disabling_ PCID will flush all PCIDs,
+	 * and it's impossible to use a non-zero PCID when PCID is disabled,
+	 * i.e. only PCID=0 can be relevant.
+	 */
+	if (!skip_tlb_flush)
+		kvm_invalidate_pcid(vcpu, pcid);
+
+	return 0;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr3);
+
+static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
+{
+	return __kvm_is_valid_cr4(vcpu, cr4) &&
+	       kvm_x86_call(is_valid_cr4)(vcpu, cr4);
+}
+
+void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4)
+{
+	if ((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS)
+		kvm_mmu_reset_context(vcpu);
+
+	/*
+	 * If CR4.PCIDE is changed 0 -> 1, there is no need to flush the TLB
+	 * according to the SDM; however, stale prev_roots could be reused
+	 * incorrectly in the future after a MOV to CR3 with NOFLUSH=1, so we
+	 * free them all.  This is *not* a superset of KVM_REQ_TLB_FLUSH_GUEST
+	 * or KVM_REQ_TLB_FLUSH_CURRENT, because the hardware TLB is not flushed,
+	 * so fall through.
+	 */
+	if (!tdp_enabled &&
+	    (cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE))
+		kvm_mmu_unload(vcpu);
+
+	/*
+	 * The TLB has to be flushed for all PCIDs if any of the following
+	 * (architecturally required) changes happen:
+	 * - CR4.PCIDE is changed from 1 to 0
+	 * - CR4.PGE is toggled
+	 *
+	 * This is a superset of KVM_REQ_TLB_FLUSH_CURRENT.
+	 */
+	if (((cr4 ^ old_cr4) & X86_CR4_PGE) ||
+	    (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
+		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
+
+	/*
+	 * The TLB has to be flushed for the current PCID if any of the
+	 * following (architecturally required) changes happen:
+	 * - CR4.SMEP is changed from 0 to 1
+	 * - CR4.PAE is toggled
+	 */
+	else if (((cr4 ^ old_cr4) & X86_CR4_PAE) ||
+		 ((cr4 & X86_CR4_SMEP) && !(old_cr4 & X86_CR4_SMEP)))
+		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
+
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_post_set_cr4);
+
+int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
+{
+	unsigned long old_cr4 = kvm_read_cr4(vcpu);
+
+	if (!kvm_is_valid_cr4(vcpu, cr4))
+		return 1;
+
+	if (is_long_mode(vcpu)) {
+		if (!(cr4 & X86_CR4_PAE))
+			return 1;
+		if ((cr4 ^ old_cr4) & X86_CR4_LA57)
+			return 1;
+	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
+		   && ((cr4 ^ old_cr4) & X86_CR4_PDPTR_BITS)
+		   && !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))
+		return 1;
+
+	if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
+		/* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
+		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
+			return 1;
+	}
+
+	if ((cr4 & X86_CR4_CET) && !kvm_is_cr0_bit_set(vcpu, X86_CR0_WP))
+		return 1;
+
+	kvm_x86_call(set_cr4)(vcpu, cr4);
+
+	kvm_post_set_cr4(vcpu, old_cr4, cr4);
+
+	return 0;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr4);
+
+int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
+{
+	if (cr8 & CR8_RESERVED_BITS)
+		return 1;
+	if (lapic_in_kernel(vcpu))
+		kvm_lapic_set_tpr(vcpu, cr8);
+	else
+		vcpu->arch.cr8 = cr8;
+	return 0;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr8);
+
+unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
+{
+	if (lapic_in_kernel(vcpu))
+		return kvm_lapic_get_cr8(vcpu);
+	else
+		return vcpu->arch.cr8;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_cr8);
+
+static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
+{
+	struct desc_ptr dt;
+
+	if (vcpu->arch.guest_state_protected)
+		goto skip_protected_regs;
+
+	kvm_handle_exception_payload_quirk(vcpu);
+
+	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
+	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
+	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
+	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
+	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
+	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
+
+	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
+	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
+
+	kvm_x86_call(get_idt)(vcpu, &dt);
+	sregs->idt.limit = dt.size;
+	sregs->idt.base = dt.address;
+	kvm_x86_call(get_gdt)(vcpu, &dt);
+	sregs->gdt.limit = dt.size;
+	sregs->gdt.base = dt.address;
+
+	sregs->cr2 = vcpu->arch.cr2;
+	sregs->cr3 = kvm_read_cr3(vcpu);
+
+skip_protected_regs:
+	sregs->cr0 = kvm_read_cr0(vcpu);
+	sregs->cr4 = kvm_read_cr4(vcpu);
+	sregs->cr8 = kvm_get_cr8(vcpu);
+	sregs->efer = vcpu->arch.efer;
+	sregs->apic_base = vcpu->arch.apic_base;
+}
+
+static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
+{
+	__get_sregs_common(vcpu, sregs);
+
+	if (vcpu->arch.guest_state_protected)
+		return;
+
+	if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)
+		set_bit(vcpu->arch.interrupt.nr,
+			(unsigned long *)sregs->interrupt_bitmap);
+}
+
+int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
+				  struct kvm_sregs *sregs)
+{
+	if (vcpu->kvm->arch.has_protected_state &&
+	    vcpu->arch.guest_state_protected)
+		return -EINVAL;
+
+	vcpu_load(vcpu);
+	__get_sregs(vcpu, sregs);
+	vcpu_put(vcpu);
+	return 0;
+}
+
+void kvm_vcpu_ioctl_x86_get_sregs2(struct kvm_vcpu *vcpu,
+				   struct kvm_sregs2 *sregs2)
+{
+	int i;
+
+	__get_sregs_common(vcpu, (struct kvm_sregs *)sregs2);
+
+	if (vcpu->arch.guest_state_protected)
+		return;
+
+	if (is_pae_paging(vcpu)) {
+		kvm_vcpu_srcu_read_lock(vcpu);
+		for (i = 0 ; i < 4 ; i++)
+			sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i);
+		sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID;
+		kvm_vcpu_srcu_read_unlock(vcpu);
+	}
+}
+
+static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
+{
+	if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) {
+		/*
+		 * When EFER.LME and CR0.PG are set, the processor is in
+		 * 64-bit mode (though maybe in a 32-bit code segment).
+		 * CR4.PAE and EFER.LMA must be set.
+		 */
+		if (!(sregs->cr4 & X86_CR4_PAE) || !(sregs->efer & EFER_LMA))
+			return false;
+		if (!kvm_vcpu_is_legal_cr3(vcpu, sregs->cr3))
+			return false;
+	} else {
+		/*
+		 * Not in 64-bit mode: EFER.LMA is clear and the code
+		 * segment cannot be 64-bit.
+		 */
+		if (sregs->efer & EFER_LMA || sregs->cs.l)
+			return false;
+	}
+
+	return kvm_is_valid_cr4(vcpu, sregs->cr4) &&
+	       kvm_is_valid_cr0(vcpu, sregs->cr0);
+}
+
+static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs,
+			      int *mmu_reset_needed, bool update_pdptrs)
+{
+	int idx;
+	struct desc_ptr dt;
+
+	if (!kvm_is_valid_sregs(vcpu, sregs))
+		return -EINVAL;
+
+	if (kvm_apic_set_base(vcpu, sregs->apic_base, true))
+		return -EINVAL;
+
+	if (vcpu->arch.guest_state_protected)
+		return 0;
+
+	dt.size = sregs->idt.limit;
+	dt.address = sregs->idt.base;
+	kvm_x86_call(set_idt)(vcpu, &dt);
+	dt.size = sregs->gdt.limit;
+	dt.address = sregs->gdt.base;
+	kvm_x86_call(set_gdt)(vcpu, &dt);
+
+	vcpu->arch.cr2 = sregs->cr2;
+	*mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
+	vcpu->arch.cr3 = sregs->cr3;
+	kvm_register_mark_dirty(vcpu, VCPU_REG_CR3);
+	kvm_x86_call(post_set_cr3)(vcpu, sregs->cr3);
+
+	*mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
+	kvm_x86_call(set_efer)(vcpu, sregs->efer);
+
+	*mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
+	kvm_x86_call(set_cr0)(vcpu, sregs->cr0);
+
+	*mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
+	kvm_x86_call(set_cr4)(vcpu, sregs->cr4);
+
+	if (update_pdptrs) {
+		idx = srcu_read_lock(&vcpu->kvm->srcu);
+		if (is_pae_paging(vcpu)) {
+			load_pdptrs(vcpu, kvm_read_cr3(vcpu));
+			*mmu_reset_needed = 1;
+		}
+		srcu_read_unlock(&vcpu->kvm->srcu, idx);
+	}
+
+	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
+	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
+	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
+	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
+	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
+	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
+
+	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
+	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
+
+	kvm_set_cr8(vcpu, sregs->cr8);
+
+	/* Older userspace won't unhalt the vcpu on reset. */
+	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
+	    sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
+	    !is_protmode(vcpu))
+		kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE);
+
+	return 0;
+}
+
+static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
+{
+	int pending_vec, max_bits;
+	int mmu_reset_needed = 0;
+	int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true);
+
+	if (ret)
+		return ret;
+
+	if (mmu_reset_needed) {
+		kvm_mmu_reset_context(vcpu);
+		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
+	}
+
+	max_bits = KVM_NR_INTERRUPTS;
+	pending_vec = find_first_bit(
+		(const unsigned long *)sregs->interrupt_bitmap, max_bits);
+
+	if (pending_vec < max_bits) {
+		kvm_queue_interrupt(vcpu, pending_vec, false);
+		pr_debug("Set back pending irq %d\n", pending_vec);
+		kvm_make_request(KVM_REQ_EVENT, vcpu);
+	}
+	return 0;
+}
+
+int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
+				  struct kvm_sregs *sregs)
+{
+	int ret;
+
+	if (vcpu->kvm->arch.has_protected_state &&
+	    vcpu->arch.guest_state_protected)
+		return -EINVAL;
+
+	vcpu_load(vcpu);
+	ret = __set_sregs(vcpu, sregs);
+	vcpu_put(vcpu);
+	return ret;
+}
+
+int kvm_vcpu_ioctl_x86_set_sregs2(struct kvm_vcpu *vcpu,
+				  struct kvm_sregs2 *sregs2)
+{
+	int mmu_reset_needed = 0;
+	bool valid_pdptrs = sregs2->flags & KVM_SREGS2_FLAGS_PDPTRS_VALID;
+	bool pae = (sregs2->cr0 & X86_CR0_PG) && (sregs2->cr4 & X86_CR4_PAE) &&
+		!(sregs2->efer & EFER_LMA);
+	int i, ret;
+
+	if (sregs2->flags & ~KVM_SREGS2_FLAGS_PDPTRS_VALID)
+		return -EINVAL;
+
+	if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected))
+		return -EINVAL;
+
+	ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2,
+				 &mmu_reset_needed, !valid_pdptrs);
+	if (ret)
+		return ret;
+
+	if (valid_pdptrs) {
+		for (i = 0; i < 4 ; i++)
+			kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]);
+
+		kvm_register_mark_dirty(vcpu, VCPU_REG_PDPTR);
+		mmu_reset_needed = 1;
+		vcpu->arch.pdptrs_from_userspace = true;
+	}
+	if (mmu_reset_needed) {
+		kvm_mmu_reset_context(vcpu);
+		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
+	}
+	return 0;
+}
+
+void kvm_run_sync_regs_to_user(struct kvm_vcpu *vcpu)
+{
+	BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES);
+
+	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)
+		__get_regs(vcpu, &vcpu->run->s.regs.regs);
+
+	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)
+		__get_sregs(vcpu, &vcpu->run->s.regs.sregs);
+}
+
+int kvm_run_sync_regs_from_user(struct kvm_vcpu *vcpu)
+{
+	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {
+		__set_regs(vcpu, &vcpu->run->s.regs.regs);
+		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;
+	}
+
+	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {
+		struct kvm_sregs sregs = vcpu->run->s.regs.sregs;
+
+		if (__set_sregs(vcpu, &sregs))
+			return -EINVAL;
+
+		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;
+	}
+
+	return 0;
+}
+
+void kvm_update_dr0123(struct kvm_vcpu *vcpu)
+{
+	int i;
+
+	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
+		for (i = 0; i < KVM_NR_DB_REGS; i++)
+			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
+	}
+}
+
+void kvm_update_dr7(struct kvm_vcpu *vcpu)
+{
+	unsigned long dr7;
+
+	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
+		dr7 = vcpu->arch.guest_debug_dr7;
+	else
+		dr7 = vcpu->arch.dr7;
+	kvm_x86_call(set_dr7)(vcpu, dr7);
+	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
+	if (dr7 & DR7_BP_EN_MASK)
+		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7);
+
+static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)
+{
+	u64 fixed = DR6_FIXED_1;
+
+	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_RTM))
+		fixed |= DR6_RTM;
+
+	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))
+		fixed |= DR6_BUS_LOCK;
+	return fixed;
+}
+
+int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
+{
+	size_t size = ARRAY_SIZE(vcpu->arch.db);
+
+	switch (dr) {
+	case 0 ... 3:
+		vcpu->arch.db[array_index_nospec(dr, size)] = val;
+		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
+			vcpu->arch.eff_db[dr] = val;
+		break;
+	case 4:
+	case 6:
+		if (!kvm_dr6_valid(val))
+			return 1; /* #GP */
+		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
+		break;
+	case 5:
+	default: /* 7 */
+		if (!kvm_dr7_valid(val))
+			return 1; /* #GP */
+		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
+		kvm_update_dr7(vcpu);
+		break;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_dr);
+
+unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr)
+{
+	size_t size = ARRAY_SIZE(vcpu->arch.db);
+
+	switch (dr) {
+	case 0 ... 3:
+		return vcpu->arch.db[array_index_nospec(dr, size)];
+	case 4:
+	case 6:
+		return vcpu->arch.dr6;
+	case 5:
+	default: /* 7 */
+		return vcpu->arch.dr7;
+	}
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_dr);
+
+int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
+				     struct kvm_debugregs *dbgregs)
+{
+	unsigned int i;
+
+	if (vcpu->kvm->arch.has_protected_state &&
+	    vcpu->arch.guest_state_protected)
+		return -EINVAL;
+
+	kvm_handle_exception_payload_quirk(vcpu);
+
+	memset(dbgregs, 0, sizeof(*dbgregs));
+
+	BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.db) != ARRAY_SIZE(dbgregs->db));
+	for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++)
+		dbgregs->db[i] = vcpu->arch.db[i];
+
+	dbgregs->dr6 = vcpu->arch.dr6;
+	dbgregs->dr7 = vcpu->arch.dr7;
+	return 0;
+}
+
+int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
+				     struct kvm_debugregs *dbgregs)
+{
+	unsigned int i;
+
+	if (vcpu->kvm->arch.has_protected_state &&
+	    vcpu->arch.guest_state_protected)
+		return -EINVAL;
+
+	if (dbgregs->flags)
+		return -EINVAL;
+
+	if (!kvm_dr6_valid(dbgregs->dr6))
+		return -EINVAL;
+	if (!kvm_dr7_valid(dbgregs->dr7))
+		return -EINVAL;
+
+	for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++)
+		vcpu->arch.db[i] = dbgregs->db[i];
+
+	kvm_update_dr0123(vcpu);
+	vcpu->arch.dr6 = dbgregs->dr6;
+	vcpu->arch.dr7 = dbgregs->dr7;
+	kvm_update_dr7(vcpu);
+
+	return 0;
+}
diff --git a/arch/x86/kvm/regs.h b/arch/x86/kvm/regs.h
index 5bda738afb7ca..447f0ec3e63e4 100644
--- a/arch/x86/kvm/regs.h
+++ b/arch/x86/kvm/regs.h
@@ -16,6 +16,56 @@
 
 static_assert(!(KVM_POSSIBLE_CR0_GUEST_BITS & X86_CR0_PDPTR_BITS));
 
+#define CR0_RESERVED_BITS                                               \
+	(~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
+			  | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
+			  | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
+
+#define CR4_RESERVED_BITS                                               \
+	(~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
+			  | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE     \
+			  | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR | X86_CR4_PCIDE \
+			  | X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_FSGSBASE \
+			  | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_VMXE \
+			  | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP \
+			  | X86_CR4_LAM_SUP | X86_CR4_CET))
+
+#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
+
+#define DR6_BUS_LOCK   (1 << 11)
+#define DR6_BD		(1 << 13)
+#define DR6_BS		(1 << 14)
+#define DR6_BT		(1 << 15)
+#define DR6_RTM		(1 << 16)
+/*
+ * DR6_ACTIVE_LOW combines fixed-1 and active-low bits.
+ * We can regard all the bits in DR6_FIXED_1 as active_low bits;
+ * they will never be 0 for now, but when they are defined
+ * in the future it will require no code change.
+ *
+ * DR6_ACTIVE_LOW is also used as the init/reset value for DR6.
+ */
+#define DR6_ACTIVE_LOW	0xffff0ff0
+#define DR6_VOLATILE	0x0001e80f
+#define DR6_FIXED_1	(DR6_ACTIVE_LOW & ~DR6_VOLATILE)
+
+#define DR7_BP_EN_MASK	0x000000ff
+#define DR7_GE		(1 << 9)
+#define DR7_GD		(1 << 13)
+#define DR7_VOLATILE	0xffff2bff
+
+void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0);
+void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4);
+int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
+int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3);
+int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
+int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8);
+int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val);
+unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr);
+unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu);
+void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw);
+int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3);
+
 static inline bool is_long_mode(struct kvm_vcpu *vcpu)
 {
 #ifdef CONFIG_X86_64
@@ -397,6 +447,14 @@ static inline bool kvm_dr6_valid(u64 data)
 	return !(data >> 32);
 }
 
+static inline unsigned long kvm_get_effective_dr7(struct kvm_vcpu *vcpu)
+{
+	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
+		return vcpu->arch.guest_debug_dr7;
+
+	return vcpu->arch.dr7;
+}
+
 static inline void enter_guest_mode(struct kvm_vcpu *vcpu)
 {
 	vcpu->arch.hflags |= HF_GUEST_MASK;
@@ -420,4 +478,44 @@ static inline bool is_guest_mode(struct kvm_vcpu *vcpu)
 	return vcpu->arch.hflags & HF_GUEST_MASK;
 }
 
+static inline unsigned long kvm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
+{
+	return kvm_x86_call(get_segment_base)(vcpu, seg);
+}
+
+static inline void kvm_set_segment(struct kvm_vcpu *vcpu,
+				   struct kvm_segment *var, int seg)
+{
+	kvm_x86_call(set_segment)(vcpu, var, seg);
+}
+
+static inline void kvm_get_segment(struct kvm_vcpu *vcpu,
+				   struct kvm_segment *var, int seg)
+{
+	kvm_x86_call(get_segment)(vcpu, var, seg);
+}
+
+unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu);
+bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip);
+
+unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu);
+void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
+void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
+
+void kvm_vcpu_ioctl_x86_get_sregs2(struct kvm_vcpu *vcpu,
+				   struct kvm_sregs2 *sregs2);
+int kvm_vcpu_ioctl_x86_set_sregs2(struct kvm_vcpu *vcpu,
+				  struct kvm_sregs2 *sregs2);
+
+void kvm_run_sync_regs_to_user(struct kvm_vcpu *vcpu);
+int kvm_run_sync_regs_from_user(struct kvm_vcpu *vcpu);
+
+void kvm_update_dr0123(struct kvm_vcpu *vcpu);
+void kvm_update_dr7(struct kvm_vcpu *vcpu);
+int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
+				     struct kvm_debugregs *dbgregs);
+int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
+				     struct kvm_debugregs *dbgregs);
+
+
 #endif
diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c
index 58e493a80cb0e..c5b1d294b15a2 100644
--- a/arch/x86/kvm/svm/avic.c
+++ b/arch/x86/kvm/svm/avic.c
@@ -293,13 +293,10 @@ static int avic_get_physical_id_table_order(struct kvm *kvm)
 	return get_order((__avic_get_max_physical_id(kvm, NULL) + 1) * sizeof(u64));
 }
 
-int avic_alloc_physical_id_table(struct kvm *kvm)
+static int avic_alloc_physical_id_table(struct kvm *kvm)
 {
 	struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
 
-	if (!irqchip_in_kernel(kvm) || !enable_apicv)
-		return 0;
-
 	if (kvm_svm->avic_physical_id_table)
 		return 0;
 
@@ -311,39 +308,32 @@ int avic_alloc_physical_id_table(struct kvm *kvm)
 	return 0;
 }
 
-void avic_vm_destroy(struct kvm *kvm)
+static int avic_alloc_logical_id_table(struct kvm *kvm)
 {
-	unsigned long flags;
 	struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
 
-	if (!enable_apicv)
-		return;
+	if (kvm_svm->avic_logical_id_table)
+		return 0;
 
-	free_page((unsigned long)kvm_svm->avic_logical_id_table);
-	free_pages((unsigned long)kvm_svm->avic_physical_id_table,
-		   avic_get_physical_id_table_order(kvm));
+	kvm_svm->avic_logical_id_table = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
+	if (!kvm_svm->avic_logical_id_table)
+		return -ENOMEM;
 
-	spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
-	hash_del(&kvm_svm->hnode);
-	spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
+	return 0;
 }
 
-int avic_vm_init(struct kvm *kvm)
+static void avic_add_vm_to_ga_log_list(struct kvm *kvm)
 {
-	unsigned long flags;
-	int err = -ENOMEM;
 	struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
 	struct kvm_svm *k2;
 	u32 vm_id;
 
-	if (!enable_apicv)
-		return 0;
+	lockdep_assert_held(&kvm->lock);
 
-	kvm_svm->avic_logical_id_table = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
-	if (!kvm_svm->avic_logical_id_table)
-		goto free_avic;
+	if (kvm_svm->avic_vm_id)
+		return;
 
-	spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
+	guard(spinlock_irqsave)(&svm_vm_data_hash_lock);
  again:
 	vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK;
 	if (vm_id == 0) { /* id is 1-based, zero is not okay */
@@ -359,13 +349,53 @@ int avic_vm_init(struct kvm *kvm)
 	}
 	kvm_svm->avic_vm_id = vm_id;
 	hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id);
-	spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
+}
+
+int avic_vcpu_precreate(struct kvm *kvm)
+{
+	int r;
+
+	if (!irqchip_in_kernel(kvm) || WARN_ON_ONCE(!enable_apicv))
+		return 0;
+
+	/*
+	 * Don't unwind on failure, all actions must be idempotent with respect
+	 * to creating multiple vCPUs, i.e. must persist until the VM is destroyed.
+	 */
+	r = avic_alloc_physical_id_table(kvm);
+	if (r)
+		return r;
+
+	r = avic_alloc_logical_id_table(kvm);
+	if (r)
+		return r;
 
+	avic_add_vm_to_ga_log_list(kvm);
 	return 0;
+}
+
+void avic_vm_pre_destroy(struct kvm *kvm)
+{
+	struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
+
+	if (WARN_ON_ONCE(!enable_apicv) || !kvm_svm->avic_vm_id)
+		return;
+
+	guard(spinlock_irqsave)(&svm_vm_data_hash_lock);
 
-free_avic:
-	avic_vm_destroy(kvm);
-	return err;
+	hash_del(&kvm_svm->hnode);
+}
+
+void avic_vm_destroy(struct kvm *kvm)
+{
+	struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
+
+	if (!enable_apicv)
+		return;
+
+	free_page((unsigned long)kvm_svm->avic_logical_id_table);
+	free_pages((unsigned long)kvm_svm->avic_physical_id_table,
+		   avic_get_physical_id_table_order(kvm));
 }
 
 static phys_addr_t avic_get_backing_page_address(struct vcpu_svm *svm)
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index 3e6c671a8dc26..5e3e280cd483e 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -23,6 +23,7 @@
 
 #include "kvm_emulate.h"
 #include "trace.h"
+#include "irq.h"
 #include "mmu.h"
 #include "x86.h"
 #include "smm.h"
@@ -112,16 +113,15 @@ static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
 				svm->vmcb01.ptr->save.efer,
 				svm->nested.ctl.nested_cr3,
 				svm->nested.ctl.misc_ctl);
-	vcpu->arch.mmu->get_guest_pgd     = nested_svm_get_tdp_cr3;
-	vcpu->arch.mmu->get_pdptr         = nested_svm_get_tdp_pdptr;
-	vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit;
-	vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
+
+	vcpu->arch.ngpa_walk.get_guest_pgd     = nested_svm_get_tdp_cr3;
+	vcpu->arch.ngpa_walk.get_pdptr         = nested_svm_get_tdp_pdptr;
+	vcpu->arch.ngpa_walk.inject_page_fault = nested_svm_inject_npf_exit;
 }
 
 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
 {
 	vcpu->arch.mmu = &vcpu->arch.root_mmu;
-	vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
 }
 
 static bool nested_vmcb_needs_vls_intercept(struct vcpu_svm *svm)
@@ -2101,7 +2101,6 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu,
 		kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
 
 	kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
-	ret = 0;
 out_free:
 	kfree(save);
 	kfree(ctl);
@@ -2150,7 +2149,7 @@ static gpa_t svm_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa,
 				      u64 pte_access)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
-	struct kvm_mmu *mmu = vcpu->arch.mmu;
+	struct kvm_pagewalk *w = &vcpu->arch.ngpa_walk;
 
 	if (WARN_ON_ONCE(!mmu_is_nested(vcpu)))
 		return gpa;
@@ -2159,10 +2158,10 @@ static gpa_t svm_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa,
 	if (!(svm->nested.ctl.misc_ctl & SVM_MISC_ENABLE_GMET))
 		access |= PFERR_USER_MASK;
 
-	return mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);
+	return w->gva_to_gpa(vcpu, w, gpa, access, exception);
 }
 
-struct kvm_x86_nested_ops svm_nested_ops = {
+struct kvm_x86_nested_ops svm_nested_ops __initdata = {
 	.leave_nested = svm_leave_nested,
 	.translate_nested_gpa = svm_translate_nested_gpa,
 	.is_exception_vmexit = nested_svm_is_exception_vmexit,
diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c
index 4272293478766..82b983484af20 100644
--- a/arch/x86/kvm/svm/sev.c
+++ b/arch/x86/kvm/svm/sev.c
@@ -2129,8 +2129,9 @@ int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd)
 	if (ret)
 		return ret;
 
+	/* Do not allow SNP VM migration until additional state transfer is implemented  */
 	if (kvm->arch.vm_type != source_kvm->arch.vm_type ||
-	    sev_guest(kvm) || !sev_guest(source_kvm)) {
+	    sev_guest(kvm) || !sev_guest(source_kvm) || sev_snp_guest(source_kvm)) {
 		ret = -EINVAL;
 		goto out_unlock;
 	}
@@ -2331,9 +2332,6 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
 	int level;
 	int ret;
 
-	if (WARN_ON_ONCE(sev_populate_args->type != KVM_SEV_SNP_PAGE_TYPE_ZERO && !src_page))
-		return -EINVAL;
-
 	ret = snp_lookup_rmpentry((u64)pfn, &assigned, &level);
 	if (ret || assigned) {
 		pr_debug("%s: Failed to ensure GFN 0x%llx RMP entry is initial shared state, ret: %d assigned: %d\n",
@@ -2422,10 +2420,12 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp)
 	     params.type != KVM_SEV_SNP_PAGE_TYPE_CPUID))
 		return -EINVAL;
 
-	src = params.type == KVM_SEV_SNP_PAGE_TYPE_ZERO ? NULL : u64_to_user_ptr(params.uaddr);
-
-	if (!PAGE_ALIGNED(src))
+	if (params.type == KVM_SEV_SNP_PAGE_TYPE_ZERO)
+		src = NULL;
+	else if (!params.uaddr || !PAGE_ALIGNED(params.uaddr))
 		return -EINVAL;
+	else
+		src = u64_to_user_ptr(params.uaddr);
 
 	npages = params.len / PAGE_SIZE;
 
@@ -2851,8 +2851,9 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd)
 	 * disallow out-of-band SEV/SEV-ES init if the target is already an
 	 * SEV guest, or if vCPUs have been created.  KVM relies on vCPUs being
 	 * created after SEV/SEV-ES initialization, e.g. to init intercepts.
+	 * Also do not allow SNP VM mirroring until additional state transfer is implemented.
 	 */
-	if (sev_guest(kvm) || !sev_guest(source_kvm) ||
+	if (sev_guest(kvm) || !sev_guest(source_kvm) || sev_snp_guest(source_kvm) ||
 	    is_mirroring_enc_context(source_kvm) || kvm->created_vcpus) {
 		ret = -EINVAL;
 		goto e_unlock;
@@ -3980,30 +3981,25 @@ static int snp_begin_psc(struct vcpu_svm *svm)
 	return snp_do_psc(svm);
 }
 
-/*
- * Invoked as part of svm_vcpu_reset() processing of an init event.
- */
-static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu)
+static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 	struct kvm_memory_slot *slot;
+	struct kvm *kvm = vcpu->kvm;
+	gfn_t gfn = gpa_to_gfn(gpa);
+	unsigned long mmu_seq;
 	struct page *page;
 	kvm_pfn_t pfn;
-	gfn_t gfn;
-
-	guard(mutex)(&svm->sev_es.snp_vmsa_mutex);
-
-	if (!svm->sev_es.snp_ap_waiting_for_reset)
-		return;
 
-	svm->sev_es.snp_ap_waiting_for_reset = false;
+	lockdep_assert_held(&svm->sev_es.snp_vmsa_mutex);
 
-	/* Mark the vCPU as offline and not runnable */
-	vcpu->arch.pv.pv_unhalted = false;
-	kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED);
-
-	/* Clear use of the VMSA */
+	/*
+	 * Clear use of the VMSA.  Ensure snp_guest_vmsa_gpa is written exactly
+	 * once, as it is read locklessly when responding to gfn invalidations.
+	 * Pairs with the READ_ONCE() in sev_gmem_invalidate_range().
+	 */
 	svm->vmcb->control.vmsa_pa = INVALID_PAGE;
+	WRITE_ONCE(svm->sev_es.snp_guest_vmsa_gpa, INVALID_PAGE);
 
 	/*
 	 * When replacing the VMSA during SEV-SNP AP creation,
@@ -4011,16 +4007,27 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu)
 	 */
 	vmcb_mark_all_dirty(svm->vmcb);
 
-	if (!VALID_PAGE(svm->sev_es.snp_vmsa_gpa))
-		return;
+	/*
+	 * From this point forward, the VMSA will always be a guest-mapped page
+	 * rather than the initial one allocated by KVM in svm->sev_es.vmsa. In
+	 * theory, svm->sev_es.vmsa could be free'd and cleaned up here, but
+	 * that involves cleanups like flushing caches, which would ideally be
+	 * handled during teardown rather than guest boot.  Deferring that also
+	 * allows the existing logic for SEV-ES VMSAs to be re-used with
+	 * minimal SNP-specific changes.
+	 */
+	svm->sev_es.snp_has_guest_vmsa = true;
 
-	gfn = gpa_to_gfn(svm->sev_es.snp_vmsa_gpa);
-	svm->sev_es.snp_vmsa_gpa = INVALID_PAGE;
+	if (!VALID_PAGE(gpa))
+		return;
 
 	slot = gfn_to_memslot(vcpu->kvm, gfn);
 	if (!slot)
 		return;
 
+	mmu_seq = kvm->mmu_invalidate_seq;
+	smp_rmb();
+
 	/*
 	 * The new VMSA will be private memory guest memory, so retrieve the
 	 * PFN from the gmem backend.
@@ -4028,29 +4035,64 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu)
 	if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, &page, NULL))
 		return;
 
+	read_lock(&kvm->mmu_lock);
 	/*
-	 * From this point forward, the VMSA will always be a guest-mapped page
-	 * rather than the initial one allocated by KVM in svm->sev_es.vmsa. In
-	 * theory, svm->sev_es.vmsa could be free'd and cleaned up here, but
-	 * that involves cleanups like flushing caches, which would ideally be
-	 * handled during teardown rather than guest boot.  Deferring that also
-	 * allows the existing logic for SEV-ES VMSAs to be re-used with
-	 * minimal SNP-specific changes.
+	 * Save the guest-provided GPA.  If retry is needed, then KVM will try
+	 * again with the same GPA.  If the VMSA is usable, then KVM needs to
+	 * track the GPA so that the VMSA can be reloaded if the backing page
+	 * for the GPA is invalidated.
 	 */
-	svm->sev_es.snp_has_guest_vmsa = true;
+	svm->sev_es.snp_guest_vmsa_gpa = gpa;
+	if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn))
+		kvm_make_request(KVM_REQ_VMSA_PAGE_RELOAD, vcpu);
+	else
+		svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn);
+	read_unlock(&kvm->mmu_lock);
 
-	/* Use the new VMSA */
-	svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn);
+	kvm_release_page_clean(page);
+}
+
+/*
+ * Invoked as part of svm_vcpu_reset() processing of an init event.
+ */
+static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu)
+{
+	struct vcpu_svm *svm = to_svm(vcpu);
+	gpa_t gpa;
+
+	guard(mutex)(&svm->sev_es.snp_vmsa_mutex);
 
-	/* Mark the vCPU as runnable */
-	kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE);
+	if (!svm->sev_es.snp_ap_waiting_for_reset)
+		return;
+
+	svm->sev_es.snp_ap_waiting_for_reset = false;
+
+	/* Mark the vCPU as offline and not runnable */
+	vcpu->arch.pv.pv_unhalted = false;
+	kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED);
+
+	gpa = svm->sev_es.snp_pending_vmsa_gpa;
+	svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE;
+
+	__sev_snp_reload_vmsa(vcpu, gpa);
 
 	/*
-	 * gmem pages aren't currently migratable, but if this ever changes
-	 * then care should be taken to ensure svm->sev_es.vmsa is pinned
-	 * through some other means.
+	 * Mark the vCPU as runnable for CREATE requests, indicated by a valid
+	 * VMSA GPA, even if installing the VMSA failed, so that KVM_RUN will
+	 * fail instead of blocking indefinitely and hanging the vCPU, e.g. if
+	 * the backing guest_memfd page is unavailable.
 	 */
-	kvm_release_page_clean(page);
+	if (VALID_PAGE(gpa))
+		kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE);
+}
+
+void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu)
+{
+	struct vcpu_sev_es_state *sev_es = &to_svm(vcpu)->sev_es;
+
+	guard(mutex)(&sev_es->snp_vmsa_mutex);
+
+	__sev_snp_reload_vmsa(vcpu, sev_es->snp_guest_vmsa_gpa);
 }
 
 static int sev_snp_ap_creation(struct vcpu_svm *svm)
@@ -4106,10 +4148,10 @@ static int sev_snp_ap_creation(struct vcpu_svm *svm)
 			return -EINVAL;
 		}
 
-		target_svm->sev_es.snp_vmsa_gpa = svm->vmcb->control.exit_info_2;
+		target_svm->sev_es.snp_pending_vmsa_gpa = svm->vmcb->control.exit_info_2;
 		break;
 	case SVM_VMGEXIT_AP_DESTROY:
-		target_svm->sev_es.snp_vmsa_gpa = INVALID_PAGE;
+		target_svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE;
 		break;
 	default:
 		vcpu_unimpl(vcpu, "vmgexit: invalid AP creation request [%#x] from guest\n",
@@ -4792,6 +4834,8 @@ int sev_vcpu_create(struct kvm_vcpu *vcpu)
 		return -ENOMEM;
 
 	svm->sev_es.vmsa = page_address(vmsa_page);
+	svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE;
+	svm->sev_es.snp_guest_vmsa_gpa = INVALID_PAGE;
 
 	vcpu->arch.guest_tsc_protected = snp_is_secure_tsc_enabled(vcpu->kvm);
 
@@ -5180,6 +5224,41 @@ void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end)
 	}
 }
 
+void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range)
+{
+	struct kvm_vcpu *vcpu;
+	unsigned long i;
+
+	lockdep_assert_held_write(&kvm->mmu_lock);
+
+	/*
+	 * An unstable result for "is SNP" is a-ok here, thanks to mmu_lock.
+	 * The vCPU's VMSA GPA is invalidated before the vCPU is made visible
+	 * to other tasks, and can only become valid while holding mmu_lock,
+	 * after the VM is fully committed to being an SNP VM.
+	 */
+	if (!____sev_snp_guest(kvm))
+		return;
+
+	kvm_for_each_vcpu(i, vcpu, kvm) {
+		/*
+		 * Read snp_guest_vmsa_gpa without taking the vCPU's VMSA mutex
+		 * (or its generic mutex) as mmu_lock is held, i.e. this task
+		 * can't sleep.  The VMSA is invalidated outside of mmu_lock,
+		 * but can only become valid inside of mmu_lock, i.e. the below
+		 * can get false positives, but not false negatives.  A false
+		 * positive is benign, as a spurious request simply forces the
+		 * vCPU to re-establish its VMSA.
+		 */
+		gpa_t gpa = READ_ONCE(to_svm(vcpu)->sev_es.snp_guest_vmsa_gpa);
+
+		if (VALID_PAGE(gpa) &&
+		    gpa_to_gfn(gpa) >= range->start &&
+		    gpa_to_gfn(gpa) < range->end)
+			kvm_make_request_and_kick(KVM_REQ_VMSA_PAGE_RELOAD, vcpu);
+	}
+}
+
 int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private)
 {
 	int level, rc;
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index 4d2bacd00ec4f..1c6d3bc76aab0 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -310,7 +310,7 @@ static int __svm_skip_emulated_instruction(struct kvm_vcpu *vcpu,
 		goto done;
 
 	if (nrips && svm->vmcb->control.next_rip != 0) {
-		WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
+		WARN_ON_ONCE(!cpu_feature_enabled(X86_FEATURE_NRIPS));
 		svm->next_rip = svm->vmcb->control.next_rip;
 	}
 
@@ -380,7 +380,7 @@ static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu, u8 vector)
 	if (nrips)
 		kvm_rip_write(vcpu, old_rip);
 
-	if (static_cpu_has(X86_FEATURE_NRIPS))
+	if (cpu_feature_enabled(X86_FEATURE_NRIPS))
 		svm->vmcb->control.next_rip = rip;
 
 	return 0;
@@ -571,14 +571,19 @@ static int svm_enable_virtualization_cpu(void)
 		return r;
 
 	sd = per_cpu_ptr(&svm_data, me);
-	sd->asid_generation = 1;
+	/*
+	 * Bump the current asid_generation value to ensure any vCPU that
+	 * previously ran on this CPU sees a stale generation and is forced
+	 * to acquire a new ASID, preventing a latent ASID collision.
+	 */
+	sd->asid_generation++;
 	sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
 	sd->next_asid = sd->max_asid + 1;
 	sd->min_asid = max_sev_asid + 1;
 
 	wrmsrq(MSR_VM_HSAVE_PA, sd->save_area_pa);
 
-	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
+	if (cpu_feature_enabled(X86_FEATURE_TSCRATEMSR)) {
 		/*
 		 * Set the default value, even if we don't use TSC scaling
 		 * to avoid having stale value in the msr
@@ -1306,11 +1311,6 @@ void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb)
 	svm->vmcb = target_vmcb->ptr;
 }
 
-static int svm_vcpu_precreate(struct kvm *kvm)
-{
-	return avic_alloc_physical_id_table(kvm);
-}
-
 static int svm_vcpu_create(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm;
@@ -1962,7 +1962,7 @@ static int pf_interception(struct kvm_vcpu *vcpu)
 	u64 error_code = svm->vmcb->control.exit_info_1;
 
 	return kvm_handle_page_fault(vcpu, error_code, fault_address,
-			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
+			cpu_feature_enabled(X86_FEATURE_DECODEASSISTS) ?
 			svm->vmcb->control.insn_bytes : NULL,
 			svm->vmcb->control.insn_len);
 }
@@ -2022,7 +2022,7 @@ static int npf_interception(struct kvm_vcpu *vcpu)
 
 	trace_kvm_page_fault(vcpu, gpa, error_code);
 	rc = kvm_mmu_page_fault(vcpu, gpa, error_code,
-				static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
+				cpu_feature_enabled(X86_FEATURE_DECODEASSISTS) ?
 				svm->vmcb->control.insn_bytes : NULL,
 				svm->vmcb->control.insn_len);
 
@@ -2528,7 +2528,7 @@ static int iret_interception(struct kvm_vcpu *vcpu)
 
 static int invlpg_interception(struct kvm_vcpu *vcpu)
 {
-	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
+	if (!cpu_feature_enabled(X86_FEATURE_DECODEASSISTS))
 		return kvm_emulate_instruction(vcpu, 0);
 
 	kvm_mmu_invlpg(vcpu, to_svm(vcpu)->vmcb->control.exit_info_1);
@@ -2576,7 +2576,7 @@ static int cr_interception(struct kvm_vcpu *vcpu)
 	unsigned long val;
 	int err;
 
-	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
+	if (!cpu_feature_enabled(X86_FEATURE_DECODEASSISTS))
 		return emulate_on_interception(vcpu);
 
 	if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
@@ -4055,12 +4055,12 @@ static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 
-	if (vcpu->arch.nested_run_pending)
-		return -EBUSY;
-
 	if (svm_interrupt_blocked(vcpu))
 		return 0;
 
+	if (vcpu->arch.nested_run_pending)
+		return -EBUSY;
+
 	/*
 	 * An IRQ must not be injected into L2 if it's supposed to VM-Exit,
 	 * e.g. if the IRQ arrived asynchronously after checking nested events.
@@ -4185,7 +4185,7 @@ static void svm_flush_tlb_asid(struct kvm_vcpu *vcpu)
 	 * unconditionally does a TLB flush on both nested VM-Enter and nested
 	 * VM-Exit (via kvm_mmu_reset_context()).
 	 */
-	if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
+	if (cpu_feature_enabled(X86_FEATURE_FLUSHBYASID))
 		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
 	else
 		svm->current_vmcb->asid_generation--;
@@ -4549,12 +4549,12 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 	 * is no need to worry about the conditional branch over the wrmsr
 	 * being speculatively taken.
 	 */
-	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
+	if (!cpu_feature_enabled(X86_FEATURE_V_SPEC_CTRL))
 		x86_spec_ctrl_set_guest(svm->virt_spec_ctrl);
 
 	svm_vcpu_enter_exit(vcpu, enter_flags);
 
-	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
+	if (!cpu_feature_enabled(X86_FEATURE_V_SPEC_CTRL))
 		x86_spec_ctrl_restore_host(svm->virt_spec_ctrl);
 
 	/* SEV-ES guests must use the CR write traps to track CR registers. */
@@ -4931,7 +4931,7 @@ static int svm_check_intercept(struct kvm_vcpu *vcpu,
 	}
 
 	/* TODO: Advertise NRIPS to guest hypervisor unconditionally */
-	if (static_cpu_has(X86_FEATURE_NRIPS))
+	if (cpu_feature_enabled(X86_FEATURE_NRIPS))
 		vmcb->control.next_rip  = info->next_rip;
 	vmcb->control.exit_code = icpt_info.exit_code;
 	vmexit = nested_svm_exit_handled(svm);
@@ -5302,12 +5302,6 @@ static int svm_vm_init(struct kvm *kvm)
 	if (!pause_filter_count || !pause_filter_thresh)
 		kvm_disable_exits(kvm, KVM_X86_DISABLE_EXITS_PAUSE);
 
-	if (enable_apicv) {
-		int ret = avic_vm_init(kvm);
-		if (ret)
-			return ret;
-	}
-
 	svm_srso_vm_init();
 	return 0;
 }
@@ -5333,13 +5327,14 @@ struct kvm_x86_ops svm_x86_ops __initdata = {
 	.emergency_disable_virtualization_cpu = svm_emergency_disable_virtualization_cpu,
 	.has_emulated_msr = svm_has_emulated_msr,
 
-	.vcpu_precreate = svm_vcpu_precreate,
+	.vcpu_precreate = avic_vcpu_precreate,
 	.vcpu_create = svm_vcpu_create,
 	.vcpu_free = svm_vcpu_free,
 	.vcpu_reset = svm_vcpu_reset,
 
 	.vm_size = sizeof(struct kvm_svm),
 	.vm_init = svm_vm_init,
+	.vm_pre_destroy = avic_vm_pre_destroy,
 	.vm_destroy = svm_vm_destroy,
 
 	.prepare_switch_to_guest = svm_prepare_switch_to_guest,
@@ -5426,8 +5421,6 @@ struct kvm_x86_ops svm_x86_ops __initdata = {
 	.check_intercept = svm_check_intercept,
 	.handle_exit_irqoff = svm_handle_exit_irqoff,
 
-	.nested_ops = &svm_nested_ops,
-
 	.deliver_interrupt = svm_deliver_interrupt,
 	.pi_update_irte = avic_pi_update_irte,
 	.setup_mce = svm_setup_mce,
@@ -5445,9 +5438,15 @@ struct kvm_x86_ops svm_x86_ops __initdata = {
 	.mem_enc_register_region = sev_mem_enc_register_region,
 	.mem_enc_unregister_region = sev_mem_enc_unregister_region,
 	.guest_memory_reclaimed = sev_guest_memory_reclaimed,
+	.reload_vmsa = sev_snp_reload_vmsa,
 
 	.vm_copy_enc_context_from = sev_vm_copy_enc_context_from,
 	.vm_move_enc_context_from = sev_vm_move_enc_context_from,
+
+	.gmem_prepare = sev_gmem_prepare,
+	.gmem_invalidate = sev_gmem_invalidate,
+	.gmem_invalidate_range = sev_gmem_invalidate_range,
+	.gmem_max_mapping_level = sev_gmem_max_mapping_level,
 #endif
 	.check_emulate_instruction = svm_check_emulate_instruction,
 
@@ -5459,10 +5458,6 @@ struct kvm_x86_ops svm_x86_ops __initdata = {
 	.vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector,
 	.vcpu_get_apicv_inhibit_reasons = avic_vcpu_get_apicv_inhibit_reasons,
 	.alloc_apic_backing_page = svm_alloc_apic_backing_page,
-
-	.gmem_prepare = sev_gmem_prepare,
-	.gmem_invalidate = sev_gmem_invalidate,
-	.gmem_max_mapping_level = sev_gmem_max_mapping_level,
 };
 
 /*
@@ -5646,6 +5641,7 @@ static __init int svm_hardware_setup(void)
 		if (r)
 			return r;
 	}
+	svm_nested_ops.enabled = nested;
 
 	/*
 	 * KVM's MMU doesn't support using 2-level paging for itself, and thus
@@ -5712,6 +5708,8 @@ static __init int svm_hardware_setup(void)
 	enable_apicv = avic_hardware_setup();
 	if (!enable_apicv) {
 		enable_ipiv = false;
+		svm_x86_ops.vcpu_precreate = NULL;
+		svm_x86_ops.vm_pre_destroy = NULL;
 		svm_x86_ops.vcpu_blocking = NULL;
 		svm_x86_ops.vcpu_unblocking = NULL;
 		svm_x86_ops.vcpu_get_apicv_inhibit_reasons = NULL;
@@ -5772,6 +5770,7 @@ static struct kvm_x86_init_ops svm_init_ops __initdata = {
 
 	.runtime_ops = &svm_x86_ops,
 	.pmu_ops = &amd_pmu_ops,
+	.nested_ops = &svm_nested_ops,
 };
 
 static void __svm_exit(void)
diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h
index 716be21fba335..af25e4b562127 100644
--- a/arch/x86/kvm/svm/svm.h
+++ b/arch/x86/kvm/svm/svm.h
@@ -271,7 +271,8 @@ struct vcpu_sev_es_state {
 	u64 ghcb_registered_gpa;
 
 	struct mutex snp_vmsa_mutex; /* Used to handle concurrent updates of VMSA. */
-	gpa_t snp_vmsa_gpa;
+	gpa_t snp_pending_vmsa_gpa;
+	gpa_t snp_guest_vmsa_gpa;
 	bool snp_ap_waiting_for_reset;
 	bool snp_has_guest_vmsa;
 };
@@ -945,9 +946,9 @@ extern struct kvm_x86_nested_ops svm_nested_ops;
 
 bool __init avic_hardware_setup(void);
 void avic_hardware_unsetup(void);
-int avic_alloc_physical_id_table(struct kvm *kvm);
+int avic_vcpu_precreate(struct kvm *kvm);
+void avic_vm_pre_destroy(struct kvm *kvm);
 void avic_vm_destroy(struct kvm *kvm);
-int avic_vm_init(struct kvm *kvm);
 void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb);
 int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu);
 int avic_unaccelerated_access_interception(struct kvm_vcpu *vcpu);
@@ -995,6 +996,7 @@ static inline struct page *snp_safe_alloc_page(void)
 {
 	return snp_safe_alloc_page_node(numa_node_id(), GFP_KERNEL_ACCOUNT);
 }
+void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu);
 
 int sev_vcpu_create(struct kvm_vcpu *vcpu);
 void sev_free_vcpu(struct kvm_vcpu *vcpu);
@@ -1009,6 +1011,7 @@ extern unsigned int max_sev_asid;
 void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code);
 int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order);
 void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end);
+void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range);
 int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private);
 struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu);
 void sev_free_decrypted_vmsa(struct kvm_vcpu *vcpu, struct vmcb_save_area *vmsa);
@@ -1034,16 +1037,6 @@ static inline int sev_cpu_init(struct svm_cpu_data *sd) { return 0; }
 static inline int sev_dev_get_attr(u32 group, u64 attr, u64 *val) { return -ENXIO; }
 #define max_sev_asid 0
 static inline void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) {}
-static inline int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order)
-{
-	return 0;
-}
-static inline void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) {}
-static inline int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private)
-{
-	return 0;
-}
-
 static inline struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu)
 {
 	return NULL;
diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h
index 0db25bba17f6e..93de876c318ce 100644
--- a/arch/x86/kvm/trace.h
+++ b/arch/x86/kvm/trace.h
@@ -490,7 +490,7 @@ TRACE_EVENT(kvm_inj_exception,
 	TP_printk("%s%s%s%s%s",
 		  __print_symbolic(__entry->exception, kvm_trace_sym_exc),
 		  !__entry->has_error ? "" : " (",
-		  !__entry->has_error ? "" : __print_symbolic(__entry->error_code, { }),
+		  !__entry->has_error ? "" : __print_symbolic(__entry->error_code),
 		  !__entry->has_error ? "" : ")",
 		  __entry->reinjected ? " [reinjected]" : "")
 );
diff --git a/arch/x86/kvm/tss.h b/arch/x86/kvm/tss.h
index 3f9150125e701..55ced89758406 100644
--- a/arch/x86/kvm/tss.h
+++ b/arch/x86/kvm/tss.h
@@ -2,6 +2,10 @@
 #ifndef __TSS_SEGMENT_H
 #define __TSS_SEGMENT_H
 
+#include <linux/build_bug.h>
+#include <linux/stddef.h>
+#include <linux/types.h>
+
 struct tss_segment_32 {
 	u32 prev_task_link;
 	u32 esp0;
@@ -57,4 +61,14 @@ struct tss_segment_16 {
 	u16 ldt;
 };
 
+#define TSS_IOPB_BASE_OFFSET 0x66
+#define TSS_BASE_SIZE 0x68
+#define TSS_IOPB_SIZE (65536 / 8)
+#define TSS_REDIRECTION_SIZE (256 / 8)
+#define RMODE_TSS_SIZE							\
+	(TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1)
+
+static_assert(offsetof(struct tss_segment_32, io_map) == TSS_IOPB_BASE_OFFSET);
+static_assert(sizeof(struct tss_segment_32) == TSS_BASE_SIZE);
+
 #endif
diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c
index 83d9921277eac..04f986e3d439a 100644
--- a/arch/x86/kvm/vmx/main.c
+++ b/arch/x86/kvm/vmx/main.c
@@ -995,8 +995,6 @@ struct kvm_x86_ops vt_x86_ops __initdata = {
 
 	.update_cpu_dirty_logging = vt_op(update_cpu_dirty_logging),
 
-	.nested_ops = &vmx_nested_ops,
-
 	.pi_update_irte = vmx_pi_update_irte,
 	.pi_start_bypass = vmx_pi_start_bypass,
 
@@ -1038,6 +1036,7 @@ struct kvm_x86_init_ops vt_init_ops __initdata = {
 
 	.runtime_ops = &vt_x86_ops,
 	.pmu_ops = &intel_pmu_ops,
+	.nested_ops = &vmx_nested_ops,
 };
 
 static void __exit vt_exit(void)
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 6957bb6f5cf7e..eed9ca1c2668f 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -11,6 +11,7 @@
 #include "x86.h"
 #include "cpuid.h"
 #include "hyperv.h"
+#include "irq.h"
 #include "mmu.h"
 #include "nested.h"
 #include "pmu.h"
@@ -407,7 +408,7 @@ static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp,
 			roots |= KVM_MMU_ROOT_PREVIOUS(i);
 	}
 	if (roots)
-		kvm_mmu_invalidate_addr(vcpu, vcpu->arch.mmu, addr, roots);
+		kvm_mmu_invalidate_addr(vcpu, &vcpu->arch.ngpa_walk, addr, roots);
 }
 
 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
@@ -511,17 +512,15 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
 
 	vcpu->arch.mmu = &vcpu->arch.guest_mmu;
 	nested_ept_new_eptp(vcpu);
-	vcpu->arch.mmu->get_guest_pgd     = nested_ept_get_eptp;
-	vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault;
-	vcpu->arch.mmu->get_pdptr         = kvm_pdptr_read;
+	vcpu->arch.ngpa_walk.get_guest_pgd     = nested_ept_get_eptp;
+	vcpu->arch.ngpa_walk.get_pdptr       = kvm_pdptr_read;
 
-	vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
+	vcpu->arch.ngpa_walk.inject_page_fault = nested_ept_inject_page_fault;
 }
 
 static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
 {
 	vcpu->arch.mmu = &vcpu->arch.root_mmu;
-	vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
 }
 
 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
@@ -582,6 +581,9 @@ static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu,
 static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu,
 						struct vmcs12 *vmcs12)
 {
+	gpa_t vtpr_gpa = vmcs12->virtual_apic_page_addr + APIC_TASKPRI;
+	u32 vtpr;
+
 	if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
 		return 0;
 
@@ -591,6 +593,32 @@ static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu,
 	if (CC(!nested_cpu_has_vid(vmcs12) && vmcs12->tpr_threshold >> 4))
 		return -EINVAL;
 
+	/*
+	 * Do the illegal vTPR vs. TPR Threshold consistency check if and only
+	 * if KVM is configured to WARN on missed consistency checks, otherwise
+	 * it's a waste of time.  KVM needs to rely on hardware to fully detect
+	 * an illegal combination due to the vTPR being writable by L1 at all
+	 * times (it's an in-memory value, not a VMCS field).  I.e. even if the
+	 * check passes now, it might fail at the actual VM-Enter.
+	 *
+	 * If reading guest memory fails, skip the check as KVM's de facto ABI
+	 * for VMX instruction accesses to non-existent memory is to provide
+	 * PCI Bus Error semantics (reads return 0xFFs), in which case the vTPR
+	 * is guaranteed to greater than or equal to the threshold.
+	 *
+	 * Note!  Deliberately use the VM-scoped API when reading guest memory,
+	 * to ensure the read doesn't hit SMRAM when restoring L2 state on RSM,
+	 * and only perform the check when in KVM_RUN, to avoid a false failure
+	 * if userspace hasn't yet configured memslots during state restore.
+	 */
+	if (warn_on_missed_cc && vcpu->wants_to_run &&
+	    nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW) &&
+	    !nested_cpu_has_vid(vmcs12) &&
+	    !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) &&
+	    !kvm_read_guest(vcpu->kvm, vtpr_gpa, &vtpr, sizeof(vtpr)) &&
+	    CC((vmcs12->tpr_threshold & GENMASK(3, 0)) > ((vtpr >> 4) & GENMASK(3, 0))))
+		return -EINVAL;
+
 	return 0;
 }
 
@@ -1290,6 +1318,9 @@ static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu,
 	 * is the VPID incorporated into the MMU context.  I.e. KVM must assume
 	 * that the new vpid12 has never been used and thus represents a new
 	 * guest ASID that cannot have entries in the TLB.
+	 *
+	 * Note, last_vpid is initialized as 0, so the first nested VM-Enter
+	 * after VMXON will always flush the TLB to avoid using stale entries.
 	 */
 	if (is_vmenter && vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
 		vmx->nested.last_vpid = vmcs12->virtual_processor_id;
@@ -3104,38 +3135,6 @@ static int nested_vmx_check_controls(struct kvm_vcpu *vcpu,
 	return 0;
 }
 
-static int nested_vmx_check_controls_late(struct kvm_vcpu *vcpu,
-					  struct vmcs12 *vmcs12)
-{
-	void *vapic = to_vmx(vcpu)->nested.virtual_apic_map.hva;
-	u32 vtpr = vapic ? (*(u32 *)(vapic + APIC_TASKPRI)) >> 4 : 0;
-
-	/*
-	 * Don't bother with the consistency checks if KVM isn't configured to
-	 * WARN on missed consistency checks, as KVM needs to rely on hardware
-	 * to fully detect an illegal vTPR vs. TRP Threshold combination due to
-	 * the vTPR being writable by L1 at all times (it's an in-memory value,
-	 * not a VMCS field).  I.e. even if the check passes now, it might fail
-	 * at the actual VM-Enter.
-	 *
-	 * Keying off the module param also allows treating an invalid vAPIC
-	 * mapping as a consistency check failure without increasing the risk
-	 * of breaking a "real" VM.
-	 */
-	if (!warn_on_missed_cc)
-		return 0;
-
-	if ((exec_controls_get(to_vmx(vcpu)) & CPU_BASED_TPR_SHADOW) &&
-	    nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW) &&
-	    !nested_cpu_has_vid(vmcs12) &&
-	    !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) &&
-	    (CC(!vapic) ||
-	     CC((vmcs12->tpr_threshold & GENMASK(3, 0)) > (vtpr & GENMASK(3, 0)))))
-		return -EINVAL;
-
-	return 0;
-}
-
 static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu,
 				       struct vmcs12 *vmcs12)
 {
@@ -3661,19 +3660,14 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu,
 				    &vmx->nested.pre_vmenter_ssp_tbl);
 
 	/*
-	 * Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled.  In the
-	 * event of a "late" VM-Fail, i.e. a VM-Fail detected by hardware but
-	 * not KVM, KVM must unwind its software model to the pre-VM-Entry host
-	 * state.  When EPT is disabled, GUEST_CR3 holds KVM's shadow CR3, not
-	 * L1's "real" CR3, which causes nested_vmx_restore_host_state() to
-	 * corrupt vcpu->arch.cr3.  Stuffing vmcs01.GUEST_CR3 results in the
-	 * unwind naturally setting arch.cr3 to the correct value.  Smashing
-	 * vmcs01.GUEST_CR3 is safe because nested VM-Exits, and the unwind,
-	 * reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is guaranteed to be
-	 * overwritten with a shadow CR3 prior to re-entering L1.
+	 * Stash L1's CR3, so that in the event of a "late" VM-Fail, i.e. a
+	 * VM-Fail detected by hardware but not KVM, KVM can unwind its
+	 * software model to the pre-VM-Entry host state.  When EPT is
+	 * disabled, GUEST_CR3 holds KVM's shadow CR3, not L1's "real" CR3,
+	 * and so simply restoring from vmcs01.GUEST_CR3 would corrupt
+	 * vcpu->arch.cr3.
 	 */
-	if (!enable_ept)
-		vmcs_writel(GUEST_CR3, vcpu->arch.cr3);
+	vmx->nested.pre_vmenter_cr3 = kvm_read_cr3(vcpu);
 
 	vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02);
 
@@ -3685,11 +3679,6 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu,
 			return NVMX_VMENTRY_KVM_INTERNAL_ERROR;
 		}
 
-		if (nested_vmx_check_controls_late(vcpu, vmcs12)) {
-			vmx_switch_vmcs(vcpu, &vmx->vmcs01);
-			return NVMX_VMENTRY_VMFAIL;
-		}
-
 		if (nested_vmx_check_guest_state(vcpu, vmcs12,
 						 &entry_failure_code)) {
 			exit_reason.basic = EXIT_REASON_INVALID_STATE;
@@ -3774,6 +3763,8 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu,
 	if (!from_vmentry)
 		return NVMX_VMENTRY_VMEXIT;
 
+	nested_put_vmcs12_pages(vcpu);
+
 	load_vmcs12_host_state(vcpu, vmcs12);
 	vmcs12->vm_exit_reason = exit_reason.full;
 	if (enable_shadow_vmcs || nested_vmx_is_evmptr12_valid(vmx))
@@ -4990,7 +4981,7 @@ static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu)
 	vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW));
 
 	nested_ept_uninit_mmu_context(vcpu);
-	vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
+	vcpu->arch.cr3 = vmx->nested.pre_vmenter_cr3;
 	kvm_register_mark_available(vcpu, VCPU_REG_CR3);
 
 	/*
@@ -5436,6 +5427,13 @@ static int enter_vmx_operation(struct kvm_vcpu *vcpu)
 
 	vmx->nested.vpid02 = allocate_vpid();
 
+	/*
+	 * Clear last_vpid to ensure that the VPID is flushed on the first
+	 * nested VM-Enter. Otherwise, stale TLB entries from a previous life of
+	 * the VPID (e.g. different vCPU or even different VM) could be used.
+	 */
+	vmx->nested.last_vpid = 0;
+
 	vmx->nested.vmcs02_initialized = false;
 	vmx->nested.vmxon = true;
 
@@ -6063,8 +6061,8 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
 		u64 vpid;
 		u64 gla;
 	} operand;
-	u16 vpid02;
 	int r, gpr_index;
+	int cpu;
 
 	if (!(vmx->nested.msrs.secondary_ctls_high &
 	      SECONDARY_EXEC_ENABLE_VPID) ||
@@ -6098,42 +6096,34 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
 		return kvm_handle_memory_failure(vcpu, r, &e);
 
 	if (operand.vpid >> 16)
-		return nested_vmx_fail(vcpu,
-			VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
+		return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
+
+	if (type != VMX_VPID_EXTENT_ALL_CONTEXT && !operand.vpid)
+		return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
+
+	/* LAM doesn't apply to addresses that are inputs to TLB invalidation. */
+	if (type == VMX_VPID_EXTENT_INDIVIDUAL_ADDR &&
+	    is_noncanonical_invlpg_address(operand.gla, vcpu))
+		return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
 
 	/*
 	 * Always flush the effective vpid02, i.e. never flush the current VPID
 	 * and never explicitly flush vpid01.  INVVPID targets a VPID, not a
 	 * VMCS, and so whether or not the current vmcs12 has VPID enabled is
 	 * irrelevant (and there may not be a loaded vmcs12).
+	 *
+	 * If vmcs02 was last loaded on a different pCPU, then defer the flush
+	 * by invalidating the nested VPID tracking to ensure that KVM performs
+	 * the invalidation on the correct pCPU.
 	 */
-	vpid02 = nested_get_vpid02(vcpu);
-	switch (type) {
-	case VMX_VPID_EXTENT_INDIVIDUAL_ADDR:
-		/*
-		 * LAM doesn't apply to addresses that are inputs to TLB
-		 * invalidation.
-		 */
-		if (!operand.vpid ||
-		    is_noncanonical_invlpg_address(operand.gla, vcpu))
-			return nested_vmx_fail(vcpu,
-				VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
-		vpid_sync_vcpu_addr(vpid02, operand.gla);
-		break;
-	case VMX_VPID_EXTENT_SINGLE_CONTEXT:
-	case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL:
-		if (!operand.vpid)
-			return nested_vmx_fail(vcpu,
-				VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
-		vpid_sync_context(vpid02);
-		break;
-	case VMX_VPID_EXTENT_ALL_CONTEXT:
-		vpid_sync_context(vpid02);
-		break;
-	default:
-		WARN_ON_ONCE(1);
-		return kvm_skip_emulated_instruction(vcpu);
-	}
+	cpu = get_cpu();
+	if (cpu != vmx->nested.vmcs02.cpu)
+		vmx->nested.last_vpid = 0;
+	else if (type == VMX_VPID_EXTENT_INDIVIDUAL_ADDR)
+		vpid_sync_vcpu_addr(nested_get_vpid02(vcpu), operand.gla);
+	else
+		vpid_sync_context(nested_get_vpid02(vcpu));
+	put_cpu();
 
 	/*
 	 * Sync the shadow page tables if EPT is disabled, L1 is invalidating
@@ -7463,12 +7453,13 @@ __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *))
 	return 0;
 }
 
+
 static gpa_t vmx_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa,
 				      u64 access,
 				      struct x86_exception *exception,
 				      u64 pte_access)
 {
-	struct kvm_mmu *mmu = vcpu->arch.mmu;
+	struct kvm_pagewalk *w = &vcpu->arch.ngpa_walk;
 
 	if (WARN_ON_ONCE(!mmu_is_nested(vcpu)))
 		return gpa;
@@ -7481,10 +7472,10 @@ static gpa_t vmx_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa,
 	if ((pte_access & ACC_USER_MASK) && (access & PFERR_GUEST_FINAL_MASK))
 		access |= PFERR_USER_MASK;
 
-	return mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);
+	return w->gva_to_gpa(vcpu, w, gpa, access, exception);
 }
 
-struct kvm_x86_nested_ops vmx_nested_ops = {
+struct kvm_x86_nested_ops vmx_nested_ops __initdata = {
 	.leave_nested = vmx_leave_nested,
 	.translate_nested_gpa = vmx_translate_nested_gpa,
 	.is_exception_vmexit = nested_vmx_is_exception_vmexit,
diff --git a/arch/x86/kvm/vmx/nested.h b/arch/x86/kvm/vmx/nested.h
index 6d6cd5904ddf6..c6de848bd9ced 100644
--- a/arch/x86/kvm/vmx/nested.h
+++ b/arch/x86/kvm/vmx/nested.h
@@ -57,16 +57,14 @@ bool nested_vmx_check_io_bitmaps(struct kvm_vcpu *vcpu, unsigned int port,
 
 static inline struct vmcs12 *get_vmcs12(struct kvm_vcpu *vcpu)
 {
-	lockdep_assert_once(lockdep_is_held(&vcpu->mutex) ||
-			    !refcount_read(&vcpu->kvm->users_count));
+	kvm_lockdep_assert_vcpu_is_locked_or_unreachable(vcpu);
 
 	return to_vmx(vcpu)->nested.cached_vmcs12;
 }
 
 static inline struct vmcs12 *get_shadow_vmcs12(struct kvm_vcpu *vcpu)
 {
-	lockdep_assert_once(lockdep_is_held(&vcpu->mutex) ||
-			    !refcount_read(&vcpu->kvm->users_count));
+	kvm_lockdep_assert_vcpu_is_locked_or_unreachable(vcpu);
 
 	return to_vmx(vcpu)->nested.cached_shadow_vmcs12;
 }
diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c
index 989ab29b8c6fb..d1af0a752e97e 100644
--- a/arch/x86/kvm/vmx/tdx.c
+++ b/arch/x86/kvm/vmx/tdx.c
@@ -2797,7 +2797,11 @@ static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd)
 		goto out;
 	}
 
-	if (init_vm->cpuid.padding) {
+	/*
+	 * Reject the request if userspace changes cpuid.nent between the
+	 * initial read and the subsequent copy.
+	 */
+	if (init_vm->cpuid.padding || init_vm->cpuid.nent != nr_user_entries) {
 		ret = -EINVAL;
 		goto out;
 	}
@@ -3188,9 +3192,6 @@ static int tdx_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
 	if (KVM_BUG_ON(kvm_tdx->page_add_src, kvm))
 		return -EIO;
 
-	if (!src_page)
-		return -EOPNOTSUPP;
-
 	kvm_tdx->page_add_src = src_page;
 	ret = kvm_tdp_mmu_map_private_pfn(arg->vcpu, gfn, pfn);
 	kvm_tdx->page_add_src = NULL;
@@ -3237,8 +3238,8 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c
 	if (copy_from_user(&region, u64_to_user_ptr(cmd->data), sizeof(region)))
 		return -EFAULT;
 
-	if (!PAGE_ALIGNED(region.source_addr) || !PAGE_ALIGNED(region.gpa) ||
-	    !region.nr_pages ||
+	if (!PAGE_ALIGNED(region.source_addr) || !region.source_addr ||
+	    !PAGE_ALIGNED(region.gpa) || !region.nr_pages ||
 	    region.gpa + (region.nr_pages << PAGE_SHIFT) <= region.gpa ||
 	    !vt_is_tdx_private_gpa(kvm, region.gpa) ||
 	    !vt_is_tdx_private_gpa(kvm, region.gpa + (region.nr_pages << PAGE_SHIFT) - 1))
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index cc75feec05daa..82edb1ed1964b 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -72,6 +72,7 @@
 #include "x86.h"
 #include "x86_ops.h"
 #include "smm.h"
+#include "tss.h"
 #include "vmx_onhyperv.h"
 #include "vmenter.h"
 #include "posted_intr.h"
@@ -419,7 +420,7 @@ static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu)
 
 	vcpu->stat.l1d_flush++;
 
-	if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) {
+	if (cpu_feature_enabled(X86_FEATURE_FLUSH_L1D)) {
 		native_wrmsrq(MSR_IA32_FLUSH_CMD, L1D_FLUSH);
 		return;
 	}
@@ -1186,6 +1187,18 @@ static void vmx_remove_autostore_msr(struct vcpu_vmx *vmx, u32 msr)
 	vmx_remove_auto_msr(&vmx->msr_autostore, msr, VM_EXIT_MSR_STORE_COUNT);
 }
 
+static u16 vmx_store_ldt(void)
+{
+	u16 ldt;
+	asm("sldt %0" : "=g"(ldt));
+	return ldt;
+}
+
+static void vmx_load_ldt(u16 sel)
+{
+	asm("lldt %0" : : "rm"(sel));
+}
+
 #ifdef CONFIG_X86_32
 /*
  * On 32-bit kernels, VM exits still load the FS and GS bases from the
@@ -1203,7 +1216,7 @@ static unsigned long segment_base(u16 selector)
 	table = get_current_gdt_ro();
 
 	if ((selector & SEGMENT_TI_MASK) == SEGMENT_LDT) {
-		u16 ldt_selector = kvm_read_ldt();
+		u16 ldt_selector = vmx_store_ldt();
 
 		if (!(ldt_selector & ~SEGMENT_RPL_MASK))
 			return 0;
@@ -1358,7 +1371,7 @@ void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
 	 * Set host fs and gs selectors.  Unfortunately, 22.2.3 does not
 	 * allow segment selectors with cpl > 0 or ti == 1.
 	 */
-	host_state->ldt_sel = kvm_read_ldt();
+	host_state->ldt_sel = vmx_store_ldt();
 
 #ifdef CONFIG_X86_64
 	savesegment(ds, host_state->ds_sel);
@@ -1405,7 +1418,7 @@ static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx)
 	rdmsrq(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
 #endif
 	if (host_state->ldt_sel || (host_state->gs_sel & 7)) {
-		kvm_load_ldt(host_state->ldt_sel);
+		vmx_load_ldt(host_state->ldt_sel);
 #ifdef CONFIG_X86_64
 		load_gs_index(host_state->gs_sel);
 #else
@@ -1802,7 +1815,7 @@ static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
 	 * (namely Hyper-V) don't set it due to it being undefined behavior,
 	 * i.e. we end up advancing IP with some random value.
 	 */
-	if (!static_cpu_has(X86_FEATURE_HYPERVISOR) ||
+	if (!cpu_feature_enabled(X86_FEATURE_HYPERVISOR) ||
 	    exit_reason.basic != EXIT_REASON_EPT_MISCONFIG) {
 		instr_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
 
@@ -2146,7 +2159,7 @@ int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		    !guest_has_spec_ctrl_msr(vcpu))
 			return 1;
 
-		msr_info->data = to_vmx(vcpu)->spec_ctrl;
+		msr_info->data = vmx->spec_ctrl;
 		break;
 	case MSR_IA32_SYSENTER_CS:
 		msr_info->data = vmcs_read32(GUEST_SYSENTER_CS);
@@ -2178,7 +2191,7 @@ int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		if (!msr_info->host_initiated &&
 		    !guest_cpu_cap_has(vcpu, X86_FEATURE_SGX_LC))
 			return 1;
-		msr_info->data = to_vmx(vcpu)->msr_ia32_sgxlepubkeyhash
+		msr_info->data = vmx->msr_ia32_sgxlepubkeyhash
 			[msr_info->index - MSR_IA32_SGXLEPUBKEYHASH0];
 		break;
 	case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR:
@@ -2391,7 +2404,7 @@ int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 
 		vmx_guest_debugctl_write(vcpu, data);
 
-		if (intel_pmu_lbr_is_enabled(vcpu) && !to_vmx(vcpu)->lbr_desc.event &&
+		if (intel_pmu_lbr_is_enabled(vcpu) && !vmx->lbr_desc.event &&
 		    (data & DEBUGCTLMSR_LBR))
 			intel_pmu_create_guest_lbr_event(vcpu);
 		return 0;
@@ -2470,7 +2483,7 @@ int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		break;
 	case MSR_IA32_MCG_EXT_CTL:
 		if ((!msr_info->host_initiated &&
-		     !(to_vmx(vcpu)->msr_ia32_feature_control &
+		     !(vmx->msr_ia32_feature_control &
 		       FEAT_CTL_LMCE_ENABLED)) ||
 		    (data & ~MCG_EXT_CTL_LMCE_EN))
 			return 1;
@@ -2675,13 +2688,13 @@ static bool cpu_has_sgx(void)
 
 static int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt, u32 msr, u32 *result)
 {
-	u32 vmx_msr_low, vmx_msr_high;
+	struct msr vmx_msr;
 	u32 ctl = ctl_min | ctl_opt;
 
-	rdmsr(msr, vmx_msr_low, vmx_msr_high);
+	rdmsrq(msr, vmx_msr.q);
 
-	ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
-	ctl |= vmx_msr_low;  /* bit == 1 in low word  ==> must be one  */
+	ctl &= vmx_msr.h;  /* bit == 0 in high word ==> must be zero */
+	ctl |= vmx_msr.l;  /* bit == 1 in low word  ==> must be one  */
 
 	/* Ensure minimum (required) set of control bits are supported. */
 	if (ctl_min & ~ctl)
@@ -2737,6 +2750,7 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf,
 	u64 _cpu_based_3rd_exec_control = 0;
 	u32 _vmexit_control = 0;
 	u32 _vmentry_control = 0;
+	struct msr val;
 	u64 basic_msr;
 	u64 misc_msr;
 
@@ -2786,8 +2800,9 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf,
 				SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
 				SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
 
-	rdmsr_safe(MSR_IA32_VMX_EPT_VPID_CAP,
-		&vmx_cap->ept, &vmx_cap->vpid);
+	rdmsrq_safe(MSR_IA32_VMX_EPT_VPID_CAP, &val.q);
+	vmx_cap->ept = val.l;
+	vmx_cap->vpid = val.h;
 
 	if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) &&
 	    vmx_cap->ept) {
@@ -3027,7 +3042,7 @@ struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags)
 	struct page *pages;
 	struct vmcs *vmcs;
 
-	pages = __alloc_pages_node(node, flags, 0);
+	pages = alloc_pages_node(node, flags, 0);
 	if (!pages)
 		return NULL;
 	vmcs = page_address(pages);
@@ -3665,13 +3680,14 @@ void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg)
 
 u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
 {
+	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	struct kvm_segment s;
 
-	if (to_vmx(vcpu)->rmode.vm86_active) {
+	if (vmx->rmode.vm86_active) {
 		vmx_get_segment(vcpu, &s, seg);
 		return s.base;
 	}
-	return vmx_read_guest_seg_base(to_vmx(vcpu), seg);
+	return vmx_read_guest_seg_base(vmx, seg);
 }
 
 static int __vmx_get_cpl(struct kvm_vcpu *vcpu, bool no_cache)
@@ -4434,7 +4450,7 @@ void vmx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode,
  */
 void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
 {
-	u32 low32, high32;
+	struct msr val;
 	unsigned long tmpl;
 	unsigned long cr0, cr3, cr4;
 
@@ -4475,8 +4491,8 @@ void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
 
 	vmcs_writel(HOST_RIP, (unsigned long)vmx_vmexit); /* 22.2.5 */
 
-	rdmsr(MSR_IA32_SYSENTER_CS, low32, high32);
-	vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
+	rdmsrq(MSR_IA32_SYSENTER_CS, val.q);
+	vmcs_write32(HOST_IA32_SYSENTER_CS, val.l);
 
 	/*
 	 * SYSENTER is used for 32-bit system calls on either 32-bit or
@@ -4491,8 +4507,8 @@ void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
 	vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl);   /* 22.2.3 */
 
 	if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
-		rdmsr(MSR_IA32_CR_PAT, low32, high32);
-		vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32));
+		rdmsrq(MSR_IA32_CR_PAT, val.q);
+		vmcs_write64(HOST_IA32_PAT, val.q);
 	}
 
 	if (cpu_has_load_ia32_efer())
@@ -5238,6 +5254,9 @@ bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu)
 
 int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
 {
+	if (vmx_interrupt_blocked(vcpu))
+		return 0;
+
 	if (vcpu->arch.nested_run_pending)
 		return -EBUSY;
 
@@ -5248,7 +5267,7 @@ int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
 	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(vcpu))
 		return -EBUSY;
 
-	return !vmx_interrupt_blocked(vcpu);
+	return 1;
 }
 
 int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
@@ -5746,9 +5765,6 @@ static int handle_dr(struct kvm_vcpu *vcpu)
 	if (!kvm_require_dr(vcpu, dr))
 		return 1;
 
-	if (vmx_get_cpl(vcpu) > 0)
-		goto out;
-
 	dr7 = vmcs_readl(GUEST_DR7);
 	if (dr7 & DR7_GD) {
 		/*
@@ -5769,6 +5785,9 @@ static int handle_dr(struct kvm_vcpu *vcpu)
 		}
 	}
 
+	if (vmx_get_cpl(vcpu) > 0)
+		goto out;
+
 	if (vcpu->guest_debug == 0) {
 		exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING);
 
@@ -8703,7 +8722,7 @@ __init int vmx_hardware_setup(void)
 
 	/*
 	 * Setup shadow_me_value/shadow_me_mask to include MKTME KeyID
-	 * bits to shadow_zero_check.
+	 * bits into the MMU's struct kvm_page_format.
 	 */
 	vmx_setup_me_spte_mask();
 
@@ -8770,6 +8789,7 @@ __init int vmx_hardware_setup(void)
 		if (r)
 			return r;
 	}
+	vmx_nested_ops.enabled = nested;
 
 	kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler);
 
@@ -8791,7 +8811,7 @@ __init int vmx_hardware_setup(void)
 	 * caches properly.  This also requires honoring guest PAT, and is forced
 	 * independent of the quirk in vmx_ignore_guest_pat().
 	 */
-	if (!static_cpu_has(X86_FEATURE_SELFSNOOP))
+	if (!cpu_feature_enabled(X86_FEATURE_SELFSNOOP))
 		kvm_caps.supported_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT;
 
 	kvm_caps.inapplicable_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT;
diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h
index de9de0d2016ca..dc8517f15bc46 100644
--- a/arch/x86/kvm/vmx/vmx.h
+++ b/arch/x86/kvm/vmx/vmx.h
@@ -159,6 +159,13 @@ struct nested_vmx {
 	bool has_preemption_timer_deadline;
 	bool preemption_timer_expired;
 
+	/*
+	 * Used to restore L1's CR3 if hardware detects a VM-Fail Consistency
+	 * Check that KVM does not, in which case KVM needs to unwind CR3 back
+	 * to its pre-VM-Enter state, NOT to vmcs01.HOST_CR3.
+	 */
+	unsigned long pre_vmenter_cr3;
+
 	/*
 	 * Used to snapshot MSRs that are conditionally loaded on VM-Enter in
 	 * order to propagate the guest's pre-VM-Enter value into vmcs02.  For
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index afcac1042947a..8210ed1295624 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -80,7 +80,6 @@
 #include <asm/mshyperv.h>
 #include <asm/hypervisor.h>
 #include <asm/tlbflush.h>
-#include <asm/intel_pt.h>
 #include <asm/emulate_prefix.h>
 #include <asm/sgx.h>
 #include <asm/virt.h>
@@ -90,8 +89,6 @@
 #define CREATE_TRACE_POINTS
 #include "trace.h"
 
-#define MAX_IO_MSRS 256
-
 /*
  * Note, kvm_caps fields should *never* have default values, all fields must be
  * recomputed from scratch during vendor module load, e.g. to account for a
@@ -108,39 +105,65 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_host);
 #define emul_to_vcpu(ctxt) \
 	((struct kvm_vcpu *)(ctxt)->vcpu)
 
-/* EFER defaults:
- * - enable syscall per default because its emulated by KVM
- * - enable LME and LMA per default on 64 bit KVM
+/*
+ * KVM previously used a u32 field in kvm_run to indicate the hypercall was
+ * initiated from long mode. KVM now sets bit 0 to indicate long mode, but the
+ * remaining 31 lower bits must be 0 to preserve ABI.
  */
-#ifdef CONFIG_X86_64
-static
-u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
-#else
-static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
-#endif
-
+#define KVM_EXIT_HYPERCALL_MBZ		GENMASK_ULL(31, 1)
 #define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE)
 
 #define KVM_CAP_PMU_VALID_MASK KVM_PMU_CAP_DISABLE
 
+#define KVM_GUESTDBG_VALID_MASK \
+	(KVM_GUESTDBG_ENABLE | \
+	KVM_GUESTDBG_SINGLESTEP | \
+	KVM_GUESTDBG_USE_HW_BP | \
+	KVM_GUESTDBG_USE_SW_BP | \
+	KVM_GUESTDBG_INJECT_BP | \
+	KVM_GUESTDBG_INJECT_DB | \
+	KVM_GUESTDBG_BLOCKIRQ)
+
 #define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS		| \
 				    KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK	| \
 				    KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST	| \
 				    KVM_X2APIC_DISABLE_SUPPRESS_EOI_BROADCAST)
 
+#define KVM_CLOCK_VALID_FLAGS						\
+	(KVM_CLOCK_TSC_STABLE | KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC)
+
+#define KVM_X86_VALID_QUIRKS			\
+	(KVM_X86_QUIRK_LINT0_REENABLED |	\
+	 KVM_X86_QUIRK_CD_NW_CLEARED |		\
+	 KVM_X86_QUIRK_LAPIC_MMIO_HOLE |	\
+	 KVM_X86_QUIRK_OUT_7E_INC_RIP |		\
+	 KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT |	\
+	 KVM_X86_QUIRK_FIX_HYPERCALL_INSN |	\
+	 KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS |	\
+	 KVM_X86_QUIRK_SLOT_ZAP_ALL |		\
+	 KVM_X86_QUIRK_STUFF_FEATURE_MSRS |	\
+	 KVM_X86_QUIRK_IGNORE_GUEST_PAT |	\
+	 KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM |	\
+	 KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT)
+
+#define KVM_X86_CONDITIONAL_QUIRKS		\
+	(KVM_X86_QUIRK_CD_NW_CLEARED |		\
+	 KVM_X86_QUIRK_IGNORE_GUEST_PAT)
+
+#define KVM_BUS_LOCK_DETECTION_VALID_MODE	(KVM_BUS_LOCK_DETECTION_OFF | \
+						 KVM_BUS_LOCK_DETECTION_EXIT)
+
+#define KVM_X86_NOTIFY_VMEXIT_VALID_BITS	(KVM_X86_NOTIFY_VMEXIT_ENABLED | \
+						 KVM_X86_NOTIFY_VMEXIT_USER)
+
 static void process_nmi(struct kvm_vcpu *vcpu);
-static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
 static void store_regs(struct kvm_vcpu *vcpu);
 static int sync_regs(struct kvm_vcpu *vcpu);
 
-static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
-static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
-
 static DEFINE_MUTEX(vendor_module_lock);
-static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu);
-static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu);
 
 struct kvm_x86_ops kvm_x86_ops __read_mostly;
+struct kvm_x86_nested_ops kvm_nested_ops __read_mostly;
 
 #define KVM_X86_OP(func)					     \
 	DEFINE_STATIC_CALL_NULL(kvm_x86_##func,			     \
@@ -152,12 +175,12 @@ EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits);
 EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg);
 EXPORT_STATIC_CALL_GPL(kvm_x86_get_cpl);
 
-static bool __read_mostly ignore_msrs = 0;
-module_param(ignore_msrs, bool, 0644);
-
-bool __read_mostly report_ignored_msrs = true;
-module_param(report_ignored_msrs, bool, 0644);
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(report_ignored_msrs);
+#define KVM_X86_NESTED_OP(func)							\
+	DEFINE_STATIC_CALL_NULL(kvm_x86_nested_##func,				\
+				*(((struct kvm_x86_nested_ops *)0)->func));
+#define KVM_X86_NESTED_OP_OPTIONAL KVM_X86_NESTED_OP
+#define KVM_X86_NESTED_OP_OPTIONAL_RET0 KVM_X86_NESTED_OP
+#include <asm/kvm-x86-nested-ops.h>
 
 unsigned int min_timer_period_us = 200;
 module_param(min_timer_period_us, uint, 0644);
@@ -181,34 +204,10 @@ module_param(force_emulation_prefix, int, 0644);
 int __read_mostly pi_inject_timer = -1;
 module_param(pi_inject_timer, bint, 0644);
 
-bool __read_mostly eager_page_split = true;
-module_param(eager_page_split, bool, 0644);
-
 /* Enable/disable SMT_RSB bug mitigation */
 static bool __read_mostly mitigate_smt_rsb;
 module_param(mitigate_smt_rsb, bool, 0444);
 
-/*
- * Restoring the host value for MSRs that are only consumed when running in
- * usermode, e.g. SYSCALL MSRs and TSC_AUX, can be deferred until the CPU
- * returns to userspace, i.e. the kernel can run with the guest's value.
- */
-#define KVM_MAX_NR_USER_RETURN_MSRS 16
-
-struct kvm_user_return_msrs {
-	struct user_return_notifier urn;
-	bool registered;
-	struct kvm_user_return_msr_values {
-		u64 host;
-		u64 curr;
-	} values[KVM_MAX_NR_USER_RETURN_MSRS];
-};
-
-u32 __read_mostly kvm_nr_uret_msrs;
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_nr_uret_msrs);
-static u32 __read_mostly kvm_uret_msrs_list[KVM_MAX_NR_USER_RETURN_MSRS];
-static DEFINE_PER_CPU(struct kvm_user_return_msrs, user_return_msrs);
-
 #define KVM_SUPPORTED_XCR0     (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \
 				| XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \
 				| XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \
@@ -311,249 +310,6 @@ const struct kvm_stats_header kvm_vcpu_stats_header = {
 
 static struct kmem_cache *x86_emulator_cache;
 
-/*
- * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features) track
- * the set of MSRs that KVM exposes to userspace through KVM_GET_MSRS,
- * KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.  msrs_to_save holds MSRs that
- * require host support, i.e. should be probed via RDMSR.  emulated_msrs holds
- * MSRs that KVM emulates without strictly requiring host support.
- * msr_based_features holds MSRs that enumerate features, i.e. are effectively
- * CPUID leafs.  Note, msr_based_features isn't mutually exclusive with
- * msrs_to_save and emulated_msrs.
- */
-
-static const u32 msrs_to_save_base[] = {
-	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
-	MSR_STAR,
-#ifdef CONFIG_X86_64
-	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
-#endif
-	MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
-	MSR_IA32_FEAT_CTL, MSR_IA32_BNDCFGS, MSR_TSC_AUX,
-	MSR_IA32_SPEC_CTRL, MSR_IA32_TSX_CTRL,
-	MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH,
-	MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK,
-	MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B,
-	MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B,
-	MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B,
-	MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B,
-	MSR_IA32_UMWAIT_CONTROL,
-
-	MSR_IA32_XFD, MSR_IA32_XFD_ERR, MSR_IA32_XSS,
-
-	MSR_IA32_U_CET, MSR_IA32_S_CET,
-	MSR_IA32_PL0_SSP, MSR_IA32_PL1_SSP, MSR_IA32_PL2_SSP,
-	MSR_IA32_PL3_SSP, MSR_IA32_INT_SSP_TAB,
-	MSR_IA32_DEBUGCTLMSR,
-	MSR_IA32_LASTBRANCHFROMIP, MSR_IA32_LASTBRANCHTOIP,
-	MSR_IA32_LASTINTFROMIP, MSR_IA32_LASTINTTOIP,
-};
-
-static const u32 msrs_to_save_pmu[] = {
-	MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1,
-	MSR_ARCH_PERFMON_FIXED_CTR0 + 2,
-	MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS,
-	MSR_CORE_PERF_GLOBAL_CTRL,
-	MSR_IA32_PEBS_ENABLE, MSR_IA32_DS_AREA, MSR_PEBS_DATA_CFG,
-
-	/* This part of MSRs should match KVM_MAX_NR_INTEL_GP_COUNTERS. */
-	MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1,
-	MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3,
-	MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5,
-	MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7,
-	MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1,
-	MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3,
-	MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5,
-	MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7,
-
-	MSR_K7_EVNTSEL0, MSR_K7_EVNTSEL1, MSR_K7_EVNTSEL2, MSR_K7_EVNTSEL3,
-	MSR_K7_PERFCTR0, MSR_K7_PERFCTR1, MSR_K7_PERFCTR2, MSR_K7_PERFCTR3,
-
-	/* This part of MSRs should match KVM_MAX_NR_AMD_GP_COUNTERS. */
-	MSR_F15H_PERF_CTL0, MSR_F15H_PERF_CTL1, MSR_F15H_PERF_CTL2,
-	MSR_F15H_PERF_CTL3, MSR_F15H_PERF_CTL4, MSR_F15H_PERF_CTL5,
-	MSR_F15H_PERF_CTR0, MSR_F15H_PERF_CTR1, MSR_F15H_PERF_CTR2,
-	MSR_F15H_PERF_CTR3, MSR_F15H_PERF_CTR4, MSR_F15H_PERF_CTR5,
-
-	MSR_AMD64_PERF_CNTR_GLOBAL_CTL,
-	MSR_AMD64_PERF_CNTR_GLOBAL_STATUS,
-	MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR,
-	MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET,
-};
-
-static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_base) +
-			ARRAY_SIZE(msrs_to_save_pmu)];
-static unsigned num_msrs_to_save;
-
-static const u32 emulated_msrs_all[] = {
-	MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
-	MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
-
-#ifdef CONFIG_KVM_HYPERV
-	HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
-	HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
-	HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY,
-	HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2,
-	HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL,
-	HV_X64_MSR_RESET,
-	HV_X64_MSR_VP_INDEX,
-	HV_X64_MSR_VP_RUNTIME,
-	HV_X64_MSR_SCONTROL,
-	HV_X64_MSR_STIMER0_CONFIG,
-	HV_X64_MSR_VP_ASSIST_PAGE,
-	HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL,
-	HV_X64_MSR_TSC_EMULATION_STATUS, HV_X64_MSR_TSC_INVARIANT_CONTROL,
-	HV_X64_MSR_SYNDBG_OPTIONS,
-	HV_X64_MSR_SYNDBG_CONTROL, HV_X64_MSR_SYNDBG_STATUS,
-	HV_X64_MSR_SYNDBG_SEND_BUFFER, HV_X64_MSR_SYNDBG_RECV_BUFFER,
-	HV_X64_MSR_SYNDBG_PENDING_BUFFER,
-#endif
-
-	MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
-	MSR_KVM_PV_EOI_EN, MSR_KVM_ASYNC_PF_INT, MSR_KVM_ASYNC_PF_ACK,
-
-	MSR_IA32_TSC_ADJUST,
-	MSR_IA32_TSC_DEADLINE,
-	MSR_IA32_ARCH_CAPABILITIES,
-	MSR_IA32_PERF_CAPABILITIES,
-	MSR_IA32_MISC_ENABLE,
-	MSR_IA32_MCG_STATUS,
-	MSR_IA32_MCG_CTL,
-	MSR_IA32_MCG_EXT_CTL,
-	MSR_IA32_SMBASE,
-	MSR_SMI_COUNT,
-	MSR_PLATFORM_INFO,
-	MSR_MISC_FEATURES_ENABLES,
-	MSR_AMD64_VIRT_SPEC_CTRL,
-	MSR_AMD64_TSC_RATIO,
-	MSR_IA32_POWER_CTL,
-	MSR_IA32_UCODE_REV,
-
-	/*
-	 * KVM always supports the "true" VMX control MSRs, even if the host
-	 * does not.  The VMX MSRs as a whole are considered "emulated" as KVM
-	 * doesn't strictly require them to exist in the host (ignoring that
-	 * KVM would refuse to load in the first place if the core set of MSRs
-	 * aren't supported).
-	 */
-	MSR_IA32_VMX_BASIC,
-	MSR_IA32_VMX_TRUE_PINBASED_CTLS,
-	MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
-	MSR_IA32_VMX_TRUE_EXIT_CTLS,
-	MSR_IA32_VMX_TRUE_ENTRY_CTLS,
-	MSR_IA32_VMX_MISC,
-	MSR_IA32_VMX_CR0_FIXED0,
-	MSR_IA32_VMX_CR4_FIXED0,
-	MSR_IA32_VMX_VMCS_ENUM,
-	MSR_IA32_VMX_PROCBASED_CTLS2,
-	MSR_IA32_VMX_EPT_VPID_CAP,
-	MSR_IA32_VMX_VMFUNC,
-
-	MSR_K7_HWCR,
-	MSR_KVM_POLL_CONTROL,
-};
-
-static u32 emulated_msrs[ARRAY_SIZE(emulated_msrs_all)];
-static unsigned num_emulated_msrs;
-
-/*
- * List of MSRs that control the existence of MSR-based features, i.e. MSRs
- * that are effectively CPUID leafs.  VMX MSRs are also included in the set of
- * feature MSRs, but are handled separately to allow expedited lookups.
- */
-static const u32 msr_based_features_all_except_vmx[] = {
-	MSR_AMD64_DE_CFG,
-	MSR_IA32_UCODE_REV,
-	MSR_IA32_ARCH_CAPABILITIES,
-	MSR_IA32_PERF_CAPABILITIES,
-	MSR_PLATFORM_INFO,
-};
-
-static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all_except_vmx) +
-			      (KVM_LAST_EMULATED_VMX_MSR - KVM_FIRST_EMULATED_VMX_MSR + 1)];
-static unsigned int num_msr_based_features;
-
-/*
- * All feature MSRs except uCode revID, which tracks the currently loaded uCode
- * patch, are immutable once the vCPU model is defined.
- */
-static bool kvm_is_immutable_feature_msr(u32 msr)
-{
-	int i;
-
-	if (msr >= KVM_FIRST_EMULATED_VMX_MSR && msr <= KVM_LAST_EMULATED_VMX_MSR)
-		return true;
-
-	for (i = 0; i < ARRAY_SIZE(msr_based_features_all_except_vmx); i++) {
-		if (msr == msr_based_features_all_except_vmx[i])
-			return msr != MSR_IA32_UCODE_REV;
-	}
-
-	return false;
-}
-
-static bool kvm_is_advertised_msr(u32 msr_index)
-{
-	unsigned int i;
-
-	for (i = 0; i < num_msrs_to_save; i++) {
-		if (msrs_to_save[i] == msr_index)
-			return true;
-	}
-
-	for (i = 0; i < num_emulated_msrs; i++) {
-		if (emulated_msrs[i] == msr_index)
-			return true;
-	}
-
-	return false;
-}
-
-typedef int (*msr_access_t)(struct kvm_vcpu *vcpu, u32 index, u64 *data,
-			    bool host_initiated);
-
-static __always_inline int kvm_do_msr_access(struct kvm_vcpu *vcpu, u32 msr,
-					     u64 *data, bool host_initiated,
-					     enum kvm_msr_access rw,
-					     msr_access_t msr_access_fn)
-{
-	const char *op = rw == MSR_TYPE_W ? "wrmsr" : "rdmsr";
-	int ret;
-
-	BUILD_BUG_ON(rw != MSR_TYPE_R && rw != MSR_TYPE_W);
-
-	/*
-	 * Zero the data on read failures to avoid leaking stack data to the
-	 * guest and/or userspace, e.g. if the failure is ignored below.
-	 */
-	ret = msr_access_fn(vcpu, msr, data, host_initiated);
-	if (ret && rw == MSR_TYPE_R)
-		*data = 0;
-
-	if (ret != KVM_MSR_RET_UNSUPPORTED)
-		return ret;
-
-	/*
-	 * Userspace is allowed to read MSRs, and write '0' to MSRs, that KVM
-	 * advertises to userspace, even if an MSR isn't fully supported.
-	 * Simply check that @data is '0', which covers both the write '0' case
-	 * and all reads (in which case @data is zeroed on failure; see above).
-	 */
-	if (host_initiated && !*data && kvm_is_advertised_msr(msr))
-		return 0;
-
-	if (!ignore_msrs) {
-		kvm_debug_ratelimited("unhandled %s: 0x%x data 0x%llx\n",
-				      op, msr, *data);
-		return ret;
-	}
-
-	if (report_ignored_msrs)
-		kvm_pr_unimpl("ignored %s: 0x%x data 0x%llx\n", op, msr, *data);
-
-	return 0;
-}
-
 static struct kmem_cache *kvm_alloc_emulator_cache(void)
 {
 	unsigned int useroffset = offsetof(struct x86_emulate_ctxt, src);
@@ -567,128 +323,6 @@ static struct kmem_cache *kvm_alloc_emulator_cache(void)
 
 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
 
-static void kvm_destroy_user_return_msrs(void)
-{
-	int cpu;
-
-	for_each_possible_cpu(cpu)
-		WARN_ON_ONCE(per_cpu(user_return_msrs, cpu).registered);
-
-	kvm_nr_uret_msrs = 0;
-}
-
-static void kvm_on_user_return(struct user_return_notifier *urn)
-{
-	unsigned slot;
-	struct kvm_user_return_msrs *msrs
-		= container_of(urn, struct kvm_user_return_msrs, urn);
-	struct kvm_user_return_msr_values *values;
-
-	msrs->registered = false;
-	user_return_notifier_unregister(urn);
-
-	for (slot = 0; slot < kvm_nr_uret_msrs; ++slot) {
-		values = &msrs->values[slot];
-		if (values->host != values->curr) {
-			wrmsrq(kvm_uret_msrs_list[slot], values->host);
-			values->curr = values->host;
-		}
-	}
-}
-
-static int kvm_probe_user_return_msr(u32 msr)
-{
-	u64 val;
-	int ret;
-
-	preempt_disable();
-	ret = rdmsrq_safe(msr, &val);
-	if (ret)
-		goto out;
-	ret = wrmsrq_safe(msr, val);
-out:
-	preempt_enable();
-	return ret;
-}
-
-int kvm_add_user_return_msr(u32 msr)
-{
-	BUG_ON(kvm_nr_uret_msrs >= KVM_MAX_NR_USER_RETURN_MSRS);
-
-	if (kvm_probe_user_return_msr(msr))
-		return -1;
-
-	kvm_uret_msrs_list[kvm_nr_uret_msrs] = msr;
-	return kvm_nr_uret_msrs++;
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_add_user_return_msr);
-
-int kvm_find_user_return_msr(u32 msr)
-{
-	int i;
-
-	for (i = 0; i < kvm_nr_uret_msrs; ++i) {
-		if (kvm_uret_msrs_list[i] == msr)
-			return i;
-	}
-	return -1;
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_find_user_return_msr);
-
-static void kvm_user_return_msr_cpu_online(void)
-{
-	struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs);
-	u64 value;
-	int i;
-
-	for (i = 0; i < kvm_nr_uret_msrs; ++i) {
-		rdmsrq_safe(kvm_uret_msrs_list[i], &value);
-		msrs->values[i].host = value;
-		msrs->values[i].curr = value;
-	}
-}
-
-static void kvm_user_return_register_notifier(struct kvm_user_return_msrs *msrs)
-{
-	if (!msrs->registered) {
-		msrs->urn.on_user_return = kvm_on_user_return;
-		user_return_notifier_register(&msrs->urn);
-		msrs->registered = true;
-	}
-}
-
-int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask)
-{
-	struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs);
-	int err;
-
-	value = (value & mask) | (msrs->values[slot].host & ~mask);
-	if (value == msrs->values[slot].curr)
-		return 0;
-	err = wrmsrq_safe(kvm_uret_msrs_list[slot], value);
-	if (err)
-		return 1;
-
-	msrs->values[slot].curr = value;
-	kvm_user_return_register_notifier(msrs);
-	return 0;
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_user_return_msr);
-
-u64 kvm_get_user_return_msr(unsigned int slot)
-{
-	return this_cpu_ptr(&user_return_msrs)->values[slot].curr;
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_user_return_msr);
-
-static void drop_user_return_notifiers(void)
-{
-	struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs);
-
-	if (msrs->registered)
-		kvm_on_user_return(&msrs->urn);
-}
-
 /*
  * Handle a fault on a hardware virtualization (VMX or SVM) instruction.
  *
@@ -837,7 +471,7 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu, unsigned int nr,
 	 * wants to intercept the exception.
 	 */
 	if (is_guest_mode(vcpu) &&
-	    kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) {
+	    kvm_nested_call(is_exception_vmexit)(vcpu, nr, error_code)) {
 		kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code,
 					   has_payload, payload);
 		return;
@@ -943,17 +577,6 @@ int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_complete_insn_gp);
 
-static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err)
-{
-	if (err) {
-		kvm_inject_gp(vcpu, 0);
-		return 1;
-	}
-
-	return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE | EMULTYPE_SKIP |
-				       EMULTYPE_COMPLETE_USER_EXIT);
-}
-
 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault,
 			   bool from_hardware)
 {
@@ -976,11 +599,12 @@ void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
 				      struct x86_exception *fault,
 				      bool from_hardware)
 {
-	struct kvm_mmu *fault_mmu;
+	struct kvm_pagewalk *fault_walk;
+
 	WARN_ON_ONCE(fault->vector != PF_VECTOR);
 
-	fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :
-					       vcpu->arch.walk_mmu;
+	fault_walk = fault->nested_page_fault ? &vcpu->arch.ngpa_walk :
+						&vcpu->arch.gva_walk;
 
 	/*
 	 * Invalidate the TLB entry for the faulting address, if it exists,
@@ -988,10 +612,10 @@ void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
 	 */
 	if ((fault->error_code & PFERR_PRESENT_MASK) &&
 	    !(fault->error_code & PFERR_RSVD_MASK))
-		kvm_mmu_invalidate_addr(vcpu, fault_mmu, fault->address,
+		kvm_mmu_invalidate_addr(vcpu, fault_walk, fault->address,
 					KVM_MMU_ROOT_CURRENT);
 
-	fault_mmu->inject_page_fault(vcpu, fault, from_hardware);
+	fault_walk->inject_page_fault(vcpu, fault, from_hardware);
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_inject_emulated_page_fault);
 
@@ -1017,170 +641,6 @@ bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_require_dr);
 
-static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
-{
-	return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2);
-}
-
-/*
- * Load the pae pdptrs.  Return 1 if they are all valid, 0 otherwise.
- */
-int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
-{
-	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
-	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
-	gpa_t real_gpa;
-	int i;
-	int ret;
-	u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)];
-
-	/*
-	 * If the MMU is nested, CR3 holds an L2 GPA and needs to be translated
-	 * to an L1 GPA.
-	 */
-	real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn),
-				     PFERR_USER_MASK | PFERR_WRITE_MASK |
-				     PFERR_GUEST_PAGE_MASK, NULL, 0);
-	if (real_gpa == INVALID_GPA)
-		return 0;
-
-	/* Note the offset, PDPTRs are 32 byte aligned when using PAE paging. */
-	ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte,
-				       cr3 & GENMASK(11, 5), sizeof(pdpte));
-	if (ret < 0)
-		return 0;
-
-	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
-		if ((pdpte[i] & PT_PRESENT_MASK) &&
-		    (pdpte[i] & pdptr_rsvd_bits(vcpu))) {
-			return 0;
-		}
-	}
-
-	/*
-	 * Marking VCPU_REG_PDPTR dirty doesn't work for !tdp_enabled.
-	 * Shadow page roots need to be reconstructed instead.
-	 */
-	if (!tdp_enabled && memcmp(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs)))
-		kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT);
-
-	memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs));
-	kvm_register_mark_dirty(vcpu, VCPU_REG_PDPTR);
-	kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);
-	vcpu->arch.pdptrs_from_userspace = false;
-
-	return 1;
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(load_pdptrs);
-
-static bool kvm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
-{
-#ifdef CONFIG_X86_64
-	if (cr0 & 0xffffffff00000000UL)
-		return false;
-#endif
-
-	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
-		return false;
-
-	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
-		return false;
-
-	return kvm_x86_call(is_valid_cr0)(vcpu, cr0);
-}
-
-void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0)
-{
-	/*
-	 * CR0.WP is incorporated into the MMU role, but only for non-nested,
-	 * indirect shadow MMUs.  If paging is disabled, no updates are needed
-	 * as there are no permission bits to emulate.  If TDP is enabled, the
-	 * MMU's metadata needs to be updated, e.g. so that emulating guest
-	 * translations does the right thing, but there's no need to unload the
-	 * root as CR0.WP doesn't affect SPTEs.
-	 */
-	if ((cr0 ^ old_cr0) == X86_CR0_WP) {
-		if (!(cr0 & X86_CR0_PG))
-			return;
-
-		if (tdp_enabled) {
-			kvm_init_mmu(vcpu);
-			return;
-		}
-	}
-
-	if ((cr0 ^ old_cr0) & X86_CR0_PG) {
-		/*
-		 * Clearing CR0.PG is defined to flush the TLB from the guest's
-		 * perspective.
-		 */
-		if (!(cr0 & X86_CR0_PG))
-			kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
-		/*
-		 * Check for async #PF completion events when enabling paging,
-		 * as the vCPU may have previously encountered async #PFs (it's
-		 * entirely legal for the guest to toggle paging on/off without
-		 * waiting for the async #PF queue to drain).
-		 */
-		else if (kvm_pv_async_pf_enabled(vcpu))
-			kvm_make_request(KVM_REQ_APF_READY, vcpu);
-	}
-
-	if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS)
-		kvm_mmu_reset_context(vcpu);
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_post_set_cr0);
-
-int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
-{
-	unsigned long old_cr0 = kvm_read_cr0(vcpu);
-
-	if (!kvm_is_valid_cr0(vcpu, cr0))
-		return 1;
-
-	cr0 |= X86_CR0_ET;
-
-	/* Write to CR0 reserved bits are ignored, even on Intel. */
-	cr0 &= ~CR0_RESERVED_BITS;
-
-#ifdef CONFIG_X86_64
-	if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) &&
-	    (cr0 & X86_CR0_PG)) {
-		int cs_db, cs_l;
-
-		if (!is_pae(vcpu))
-			return 1;
-		kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
-		if (cs_l)
-			return 1;
-	}
-#endif
-	if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) &&
-	    is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) &&
-	    !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))
-		return 1;
-
-	if (!(cr0 & X86_CR0_PG) &&
-	    (is_64_bit_mode(vcpu) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)))
-		return 1;
-
-	if (!(cr0 & X86_CR0_WP) && kvm_is_cr4_bit_set(vcpu, X86_CR4_CET))
-		return 1;
-
-	kvm_x86_call(set_cr0)(vcpu, cr0);
-
-	kvm_post_set_cr0(vcpu, old_cr0, cr0);
-
-	return 0;
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr0);
-
-void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
-{
-	(void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_lmsw);
-
 static void kvm_load_xfeatures(struct kvm_vcpu *vcpu, bool load_guest)
 {
 	if (vcpu->arch.guest_state_protected)
@@ -1224,13 +684,6 @@ static void kvm_load_host_pkru(struct kvm_vcpu *vcpu)
 	}
 }
 
-#ifdef CONFIG_X86_64
-static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu)
-{
-	return vcpu->arch.guest_supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC;
-}
-#endif
-
 int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
 {
 	u64 xcr0 = xcr;
@@ -1290,100 +743,18 @@ int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu)
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_xsetbv);
 
-static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
-{
-	return __kvm_is_valid_cr4(vcpu, cr4) &&
-	       kvm_x86_call(is_valid_cr4)(vcpu, cr4);
-}
-
-void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4)
+void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid)
 {
-	if ((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS)
-		kvm_mmu_reset_context(vcpu);
+	struct kvm_mmu *mmu = vcpu->arch.mmu;
+	unsigned long roots_to_free = 0;
+	int i;
 
 	/*
-	 * If CR4.PCIDE is changed 0 -> 1, there is no need to flush the TLB
-	 * according to the SDM; however, stale prev_roots could be reused
-	 * incorrectly in the future after a MOV to CR3 with NOFLUSH=1, so we
-	 * free them all.  This is *not* a superset of KVM_REQ_TLB_FLUSH_GUEST
-	 * or KVM_REQ_TLB_FLUSH_CURRENT, because the hardware TLB is not flushed,
-	 * so fall through.
-	 */
-	if (!tdp_enabled &&
-	    (cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE))
-		kvm_mmu_unload(vcpu);
-
-	/*
-	 * The TLB has to be flushed for all PCIDs if any of the following
-	 * (architecturally required) changes happen:
-	 * - CR4.PCIDE is changed from 1 to 0
-	 * - CR4.PGE is toggled
-	 *
-	 * This is a superset of KVM_REQ_TLB_FLUSH_CURRENT.
-	 */
-	if (((cr4 ^ old_cr4) & X86_CR4_PGE) ||
-	    (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
-		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
-
-	/*
-	 * The TLB has to be flushed for the current PCID if any of the
-	 * following (architecturally required) changes happen:
-	 * - CR4.SMEP is changed from 0 to 1
-	 * - CR4.PAE is toggled
-	 */
-	else if (((cr4 ^ old_cr4) & X86_CR4_PAE) ||
-		 ((cr4 & X86_CR4_SMEP) && !(old_cr4 & X86_CR4_SMEP)))
-		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
-
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_post_set_cr4);
-
-int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
-{
-	unsigned long old_cr4 = kvm_read_cr4(vcpu);
-
-	if (!kvm_is_valid_cr4(vcpu, cr4))
-		return 1;
-
-	if (is_long_mode(vcpu)) {
-		if (!(cr4 & X86_CR4_PAE))
-			return 1;
-		if ((cr4 ^ old_cr4) & X86_CR4_LA57)
-			return 1;
-	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
-		   && ((cr4 ^ old_cr4) & X86_CR4_PDPTR_BITS)
-		   && !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))
-		return 1;
-
-	if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
-		/* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
-		if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
-			return 1;
-	}
-
-	if ((cr4 & X86_CR4_CET) && !kvm_is_cr0_bit_set(vcpu, X86_CR0_WP))
-		return 1;
-
-	kvm_x86_call(set_cr4)(vcpu, cr4);
-
-	kvm_post_set_cr4(vcpu, old_cr4, cr4);
-
-	return 0;
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr4);
-
-static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid)
-{
-	struct kvm_mmu *mmu = vcpu->arch.mmu;
-	unsigned long roots_to_free = 0;
-	int i;
-
-	/*
-	 * MOV CR3 and INVPCID are usually not intercepted when using TDP, but
-	 * this is reachable when running EPT=1 and unrestricted_guest=0,  and
-	 * also via the emulator.  KVM's TDP page tables are not in the scope of
-	 * the invalidation, but the guest's TLB entries need to be flushed as
-	 * the CPU may have cached entries in its TLB for the target PCID.
+	 * MOV CR3 and INVPCID are usually not intercepted when using TDP, but
+	 * this is reachable when running EPT=1 and unrestricted_guest=0,  and
+	 * also via the emulator.  KVM's TDP page tables are not in the scope of
+	 * the invalidation, but the guest's TLB entries need to be flushed as
+	 * the CPU may have cached entries in its TLB for the target PCID.
 	 */
 	if (unlikely(tdp_enabled)) {
 		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
@@ -1415,3297 +786,1361 @@ static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid)
 	kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free);
 }
 
-int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
+int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu)
 {
-	bool skip_tlb_flush = false;
-	unsigned long pcid = 0;
-#ifdef CONFIG_X86_64
-	if (kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) {
-		skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH;
-		cr3 &= ~X86_CR3_PCID_NOFLUSH;
-		pcid = cr3 & X86_CR3_PCID_MASK;
-	}
-#endif
-
-	/* PDPTRs are always reloaded for PAE paging. */
-	if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu))
-		goto handle_tlb_flush;
-
-	/*
-	 * Do not condition the GPA check on long mode, this helper is used to
-	 * stuff CR3, e.g. for RSM emulation, and there is no guarantee that
-	 * the current vCPU mode is accurate.
-	 */
-	if (!kvm_vcpu_is_legal_cr3(vcpu, cr3))
-		return 1;
+	u32 pmc = kvm_ecx_read(vcpu);
+	u64 data;
 
-	if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3))
+	if (kvm_pmu_rdpmc(vcpu, pmc, &data)) {
+		kvm_inject_gp(vcpu, 0);
 		return 1;
+	}
 
-	if (cr3 != kvm_read_cr3(vcpu))
-		kvm_mmu_new_pgd(vcpu, cr3);
-
-	vcpu->arch.cr3 = cr3;
-	kvm_register_mark_dirty(vcpu, VCPU_REG_CR3);
-	/* Do not call post_set_cr3, we do not get here for confidential guests.  */
-
-handle_tlb_flush:
-	/*
-	 * A load of CR3 that flushes the TLB flushes only the current PCID,
-	 * even if PCID is disabled, in which case PCID=0 is flushed.  It's a
-	 * moot point in the end because _disabling_ PCID will flush all PCIDs,
-	 * and it's impossible to use a non-zero PCID when PCID is disabled,
-	 * i.e. only PCID=0 can be relevant.
-	 */
-	if (!skip_tlb_flush)
-		kvm_invalidate_pcid(vcpu, pcid);
-
-	return 0;
+	kvm_eax_write(vcpu, data);
+	kvm_edx_write(vcpu, data >> 32);
+	return kvm_skip_emulated_instruction(vcpu);
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr3);
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdpmc);
 
-int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
+int kvm_emulate_as_nop(struct kvm_vcpu *vcpu)
 {
-	if (cr8 & CR8_RESERVED_BITS)
-		return 1;
-	if (lapic_in_kernel(vcpu))
-		kvm_lapic_set_tpr(vcpu, cr8);
-	else
-		vcpu->arch.cr8 = cr8;
-	return 0;
+	return kvm_skip_emulated_instruction(vcpu);
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr8);
 
-unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
+int kvm_emulate_invd(struct kvm_vcpu *vcpu)
 {
-	if (lapic_in_kernel(vcpu))
-		return kvm_lapic_get_cr8(vcpu);
-	else
-		return vcpu->arch.cr8;
+	/* Treat an INVD instruction as a NOP and just skip it. */
+	return kvm_emulate_as_nop(vcpu);
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_cr8);
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_invd);
 
-static void kvm_update_dr0123(struct kvm_vcpu *vcpu)
+fastpath_t handle_fastpath_invd(struct kvm_vcpu *vcpu)
 {
-	int i;
+	if (!kvm_pmu_is_fastpath_emulation_allowed(vcpu))
+		return EXIT_FASTPATH_NONE;
 
-	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
-		for (i = 0; i < KVM_NR_DB_REGS; i++)
-			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
-	}
+	if (!kvm_emulate_invd(vcpu))
+		return EXIT_FASTPATH_EXIT_USERSPACE;
+
+	return EXIT_FASTPATH_REENTER_GUEST;
 }
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_invd);
 
-void kvm_update_dr7(struct kvm_vcpu *vcpu)
+int kvm_handle_invalid_op(struct kvm_vcpu *vcpu)
 {
-	unsigned long dr7;
-
-	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
-		dr7 = vcpu->arch.guest_debug_dr7;
-	else
-		dr7 = vcpu->arch.dr7;
-	kvm_x86_call(set_dr7)(vcpu, dr7);
-	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
-	if (dr7 & DR7_BP_EN_MASK)
-		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
+	kvm_queue_exception(vcpu, UD_VECTOR);
+	return 1;
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7);
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_handle_invalid_op);
 
-static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)
-{
-	u64 fixed = DR6_FIXED_1;
 
-	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_RTM))
-		fixed |= DR6_RTM;
+static int kvm_emulate_monitor_mwait(struct kvm_vcpu *vcpu, const char *insn)
+{
+	bool enabled;
 
-	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))
-		fixed |= DR6_BUS_LOCK;
-	return fixed;
-}
+	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS))
+		goto emulate_as_nop;
 
-int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
-{
-	size_t size = ARRAY_SIZE(vcpu->arch.db);
+	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT))
+		enabled = guest_cpu_cap_has(vcpu, X86_FEATURE_MWAIT);
+	else
+		enabled = vcpu->arch.ia32_misc_enable_msr & MSR_IA32_MISC_ENABLE_MWAIT;
 
-	switch (dr) {
-	case 0 ... 3:
-		vcpu->arch.db[array_index_nospec(dr, size)] = val;
-		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
-			vcpu->arch.eff_db[dr] = val;
-		break;
-	case 4:
-	case 6:
-		if (!kvm_dr6_valid(val))
-			return 1; /* #GP */
-		vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
-		break;
-	case 5:
-	default: /* 7 */
-		if (!kvm_dr7_valid(val))
-			return 1; /* #GP */
-		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
-		kvm_update_dr7(vcpu);
-		break;
-	}
+	if (!enabled)
+		return kvm_handle_invalid_op(vcpu);
 
-	return 0;
+emulate_as_nop:
+	pr_warn_once("%s instruction emulated as NOP!\n", insn);
+	return kvm_emulate_as_nop(vcpu);
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_dr);
-
-unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr)
+int kvm_emulate_mwait(struct kvm_vcpu *vcpu)
 {
-	size_t size = ARRAY_SIZE(vcpu->arch.db);
+	return kvm_emulate_monitor_mwait(vcpu, "MWAIT");
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_mwait);
 
-	switch (dr) {
-	case 0 ... 3:
-		return vcpu->arch.db[array_index_nospec(dr, size)];
-	case 4:
-	case 6:
-		return vcpu->arch.dr6;
-	case 5:
-	default: /* 7 */
-		return vcpu->arch.dr7;
-	}
+int kvm_emulate_monitor(struct kvm_vcpu *vcpu)
+{
+	return kvm_emulate_monitor_mwait(vcpu, "MONITOR");
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_dr);
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_monitor);
 
-static unsigned long kvm_get_effective_dr7(struct kvm_vcpu *vcpu)
+static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)
 {
-	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
-		return vcpu->arch.guest_debug_dr7;
+	xfer_to_guest_mode_prepare();
 
-	return vcpu->arch.dr7;
+	return READ_ONCE(vcpu->mode) == EXITING_GUEST_MODE ||
+	       kvm_request_pending(vcpu) || xfer_to_guest_mode_work_pending();
 }
 
-int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu)
-{
-	u32 pmc = kvm_ecx_read(vcpu);
-	u64 data;
+#ifdef CONFIG_X86_64
+struct pvclock_clock {
+	int vclock_mode;
+	u64 cycle_last;
+	u64 mask;
+	u32 mult;
+	u32 shift;
+	u64 base_cycles;
+	u64 offset;
+};
 
-	if (kvm_pmu_rdpmc(vcpu, pmc, &data)) {
-		kvm_inject_gp(vcpu, 0);
-		return 1;
-	}
+struct pvclock_gtod_data {
+	seqcount_t	seq;
 
-	kvm_eax_write(vcpu, data);
-	kvm_edx_write(vcpu, data >> 32);
-	return kvm_skip_emulated_instruction(vcpu);
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdpmc);
+	struct pvclock_clock clock; /* extract of a clocksource struct */
+	struct pvclock_clock raw_clock; /* extract of a clocksource struct */
 
-/*
- * Some IA32_ARCH_CAPABILITIES bits have dependencies on MSRs that KVM
- * does not yet virtualize. These include:
- *   10 - MISC_PACKAGE_CTRLS
- *   11 - ENERGY_FILTERING_CTL
- *   12 - DOITM
- *   18 - FB_CLEAR_CTRL
- *   21 - XAPIC_DISABLE_STATUS
- *   23 - OVERCLOCKING_STATUS
- */
+	ktime_t		offs_boot;
+	u64		wall_time_sec;
+};
 
-#define KVM_SUPPORTED_ARCH_CAP \
-	(ARCH_CAP_RDCL_NO | ARCH_CAP_IBRS_ALL | ARCH_CAP_RSBA | \
-	 ARCH_CAP_SKIP_VMENTRY_L1DFLUSH | ARCH_CAP_SSB_NO | ARCH_CAP_MDS_NO | \
-	 ARCH_CAP_PSCHANGE_MC_NO | ARCH_CAP_TSX_CTRL_MSR | ARCH_CAP_TAA_NO | \
-	 ARCH_CAP_SBDR_SSDP_NO | ARCH_CAP_FBSDP_NO | ARCH_CAP_PSDP_NO | \
-	 ARCH_CAP_FB_CLEAR | ARCH_CAP_RRSBA | ARCH_CAP_PBRSB_NO | ARCH_CAP_GDS_NO | \
-	 ARCH_CAP_RFDS_NO | ARCH_CAP_RFDS_CLEAR | ARCH_CAP_BHI_NO | ARCH_CAP_ITS_NO)
+static struct pvclock_gtod_data pvclock_gtod_data;
 
-static u64 kvm_get_arch_capabilities(void)
+static void update_pvclock_gtod(struct timekeeper *tk)
 {
-	u64 data = kvm_host.arch_capabilities & KVM_SUPPORTED_ARCH_CAP;
+	struct pvclock_gtod_data *vdata = &pvclock_gtod_data;
 
-	/*
-	 * If nx_huge_pages is enabled, KVM's shadow paging will ensure that
-	 * the nested hypervisor runs with NX huge pages.  If it is not,
-	 * L1 is anyway vulnerable to ITLB_MULTIHIT exploits from other
-	 * L1 guests, so it need not worry about its own (L2) guests.
-	 */
-	data |= ARCH_CAP_PSCHANGE_MC_NO;
+	write_seqcount_begin(&vdata->seq);
 
-	/*
-	 * If we're doing cache flushes (either "always" or "cond")
-	 * we will do one whenever the guest does a vmlaunch/vmresume.
-	 * If an outer hypervisor is doing the cache flush for us
-	 * (ARCH_CAP_SKIP_VMENTRY_L1DFLUSH), we can safely pass that
-	 * capability to the guest too, and if EPT is disabled we're not
-	 * vulnerable.  Overall, only VMENTER_L1D_FLUSH_NEVER will
-	 * require a nested hypervisor to do a flush of its own.
-	 */
-	if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER)
-		data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH;
+	/* copy pvclock gtod data */
+	vdata->clock.vclock_mode	= tk->tkr_mono.clock->vdso_clock_mode;
+	vdata->clock.cycle_last		= tk->tkr_mono.cycle_last;
+	vdata->clock.mask		= tk->tkr_mono.mask;
+	vdata->clock.mult		= tk->tkr_mono.mult;
+	vdata->clock.shift		= tk->tkr_mono.shift;
+	vdata->clock.base_cycles	= tk->tkr_mono.xtime_nsec;
+	vdata->clock.offset		= tk->tkr_mono.base;
 
-	if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
-		data |= ARCH_CAP_RDCL_NO;
-	if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
-		data |= ARCH_CAP_SSB_NO;
-	if (!boot_cpu_has_bug(X86_BUG_MDS))
-		data |= ARCH_CAP_MDS_NO;
-	if (!boot_cpu_has_bug(X86_BUG_RFDS))
-		data |= ARCH_CAP_RFDS_NO;
-	if (!boot_cpu_has_bug(X86_BUG_ITS))
-		data |= ARCH_CAP_ITS_NO;
+	vdata->raw_clock.vclock_mode	= tk->tkr_raw.clock->vdso_clock_mode;
+	vdata->raw_clock.cycle_last	= tk->tkr_raw.cycle_last;
+	vdata->raw_clock.mask		= tk->tkr_raw.mask;
+	vdata->raw_clock.mult		= tk->tkr_raw.mult;
+	vdata->raw_clock.shift		= tk->tkr_raw.shift;
+	vdata->raw_clock.base_cycles	= tk->tkr_raw.xtime_nsec;
+	vdata->raw_clock.offset		= tk->tkr_raw.base;
 
-	if (!boot_cpu_has(X86_FEATURE_RTM)) {
-		/*
-		 * If RTM=0 because the kernel has disabled TSX, the host might
-		 * have TAA_NO or TSX_CTRL.  Clear TAA_NO (the guest sees RTM=0
-		 * and therefore knows that there cannot be TAA) but keep
-		 * TSX_CTRL: some buggy userspaces leave it set on tsx=on hosts,
-		 * and we want to allow migrating those guests to tsx=off hosts.
-		 */
-		data &= ~ARCH_CAP_TAA_NO;
-	} else if (!boot_cpu_has_bug(X86_BUG_TAA)) {
-		data |= ARCH_CAP_TAA_NO;
-	} else {
-		/*
-		 * Nothing to do here; we emulate TSX_CTRL if present on the
-		 * host so the guest can choose between disabling TSX or
-		 * using VERW to clear CPU buffers.
-		 */
-	}
+	vdata->wall_time_sec            = tk->xtime_sec;
 
-	if (!boot_cpu_has_bug(X86_BUG_GDS) || gds_ucode_mitigated())
-		data |= ARCH_CAP_GDS_NO;
+	vdata->offs_boot		= tk->offs_boot;
 
-	return data;
+	write_seqcount_end(&vdata->seq);
 }
 
-static int kvm_get_feature_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
-			       bool host_initiated)
+static s64 get_kvmclock_base_ns(void)
 {
-	WARN_ON_ONCE(!host_initiated);
-
-	switch (index) {
-	case MSR_IA32_ARCH_CAPABILITIES:
-		*data = kvm_get_arch_capabilities();
-		break;
-	case MSR_IA32_PERF_CAPABILITIES:
-		*data = kvm_caps.supported_perf_cap;
-		break;
-	case MSR_PLATFORM_INFO:
-		*data = MSR_PLATFORM_INFO_CPUID_FAULT;
-		break;
-	case MSR_IA32_UCODE_REV:
-		rdmsrq_safe(index, data);
-		break;
-	default:
-		return kvm_x86_call(get_feature_msr)(index, data);
-	}
-	return 0;
+	/* Count up from boot time, but with the frequency of the raw clock.  */
+	return ktime_to_ns(ktime_add(ktime_get_raw(), pvclock_gtod_data.offs_boot));
+}
+#else
+static s64 get_kvmclock_base_ns(void)
+{
+	/* Master clock not used, so we can just use CLOCK_BOOTTIME.  */
+	return ktime_get_boottime_ns();
 }
+#endif
 
-static int do_get_feature_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
+static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
 {
-	return kvm_do_msr_access(vcpu, index, data, true, MSR_TYPE_R,
-				 kvm_get_feature_msr);
+	do_shl32_div32(dividend, divisor);
+	return dividend;
 }
 
-static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
+static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz,
+			       s8 *pshift, u32 *pmultiplier)
 {
-	if (efer & EFER_AUTOIBRS && !guest_cpu_cap_has(vcpu, X86_FEATURE_AUTOIBRS))
-		return false;
+	uint64_t scaled64;
+	int32_t  shift = 0;
+	uint64_t tps64;
+	uint32_t tps32;
 
-	if (efer & EFER_FFXSR && !guest_cpu_cap_has(vcpu, X86_FEATURE_FXSR_OPT))
-		return false;
+	tps64 = base_hz;
+	scaled64 = scaled_hz;
+	while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
+		tps64 >>= 1;
+		shift--;
+	}
 
-	if (efer & EFER_SVME && !guest_cpu_cap_has(vcpu, X86_FEATURE_SVM))
-		return false;
+	tps32 = (uint32_t)tps64;
+	while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
+		if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
+			scaled64 >>= 1;
+		else
+			tps32 <<= 1;
+		shift++;
+	}
 
-	if (efer & (EFER_LME | EFER_LMA) &&
-	    !guest_cpu_cap_has(vcpu, X86_FEATURE_LM))
-		return false;
+	*pshift = shift;
+	*pmultiplier = div_frac(scaled64, tps32);
+}
 
-	if (efer & EFER_NX && !guest_cpu_cap_has(vcpu, X86_FEATURE_NX))
-		return false;
+#ifdef CONFIG_X86_64
+static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
+#endif
 
-	return true;
+static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
+static unsigned long max_tsc_khz;
 
-}
-bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
+static u32 adjust_tsc_khz(u32 khz, s32 ppm)
 {
-	if (efer & efer_reserved_bits)
-		return false;
-
-	return __kvm_valid_efer(vcpu, efer);
+	u64 v = (u64)khz * (1000000 + ppm);
+	do_div(v, 1000000);
+	return v;
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_valid_efer);
-
-static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
-{
-	u64 old_efer = vcpu->arch.efer;
-	u64 efer = msr_info->data;
-	int r;
 
-	if (efer & efer_reserved_bits)
-		return 1;
+static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier);
 
-	if (!msr_info->host_initiated) {
-		if (!__kvm_valid_efer(vcpu, efer))
-			return 1;
+static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
+{
+	u64 ratio;
 
-		if (is_paging(vcpu) &&
-		    (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
-			return 1;
+	/* Guest TSC same frequency as host TSC? */
+	if (!scale) {
+		kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);
+		return 0;
 	}
 
-	efer &= ~EFER_LMA;
-	efer |= vcpu->arch.efer & EFER_LMA;
-
-	r = kvm_x86_call(set_efer)(vcpu, efer);
-	if (r) {
-		WARN_ON(r > 0);
-		return r;
+	/* TSC scaling supported? */
+	if (!kvm_caps.has_tsc_control) {
+		if (user_tsc_khz > tsc_khz) {
+			vcpu->arch.tsc_catchup = 1;
+			vcpu->arch.tsc_always_catchup = 1;
+			return 0;
+		} else {
+			pr_warn_ratelimited("user requested TSC rate below hardware speed\n");
+			return -1;
+		}
 	}
 
-	if ((efer ^ old_efer) & KVM_MMU_EFER_ROLE_BITS)
-		kvm_mmu_reset_context(vcpu);
+	/* TSC scaling required  - calculate ratio */
+	ratio = mul_u64_u32_div(1ULL << kvm_caps.tsc_scaling_ratio_frac_bits,
+				user_tsc_khz, tsc_khz);
 
-	if (!static_cpu_has(X86_FEATURE_XSAVES) &&
-	    (efer & EFER_SVME))
-		kvm_hv_xsaves_xsavec_maybe_warn(vcpu);
+	if (ratio == 0 || ratio >= kvm_caps.max_tsc_scaling_ratio) {
+		pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n",
+			            user_tsc_khz);
+		return -1;
+	}
 
+	kvm_vcpu_write_tsc_multiplier(vcpu, ratio);
 	return 0;
 }
 
-void kvm_enable_efer_bits(u64 mask)
-{
-       efer_reserved_bits &= ~mask;
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_enable_efer_bits);
-
-bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type)
+static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
 {
-	struct kvm_x86_msr_filter *msr_filter;
-	struct msr_bitmap_range *ranges;
-	struct kvm *kvm = vcpu->kvm;
-	bool allowed;
-	int idx;
-	u32 i;
-
-	/* x2APIC MSRs do not support filtering. */
-	if (index >= 0x800 && index <= 0x8ff)
-		return true;
-
-	idx = srcu_read_lock(&kvm->srcu);
-
-	msr_filter = srcu_dereference(kvm->arch.msr_filter, &kvm->srcu);
-	if (!msr_filter) {
-		allowed = true;
-		goto out;
-	}
-
-	allowed = msr_filter->default_allow;
-	ranges = msr_filter->ranges;
-
-	for (i = 0; i < msr_filter->count; i++) {
-		u32 start = ranges[i].base;
-		u32 end = start + ranges[i].nmsrs;
-		u32 flags = ranges[i].flags;
-		unsigned long *bitmap = ranges[i].bitmap;
+	u32 thresh_lo, thresh_hi;
+	int use_scaling = 0;
 
-		if ((index >= start) && (index < end) && (flags & type)) {
-			allowed = test_bit(index - start, bitmap);
-			break;
-		}
+	/* tsc_khz can be zero if TSC calibration fails */
+	if (user_tsc_khz == 0) {
+		/* set tsc_scaling_ratio to a safe value */
+		kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);
+		return -1;
 	}
 
-out:
-	srcu_read_unlock(&kvm->srcu, idx);
-
-	return allowed;
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_msr_allowed);
-
-/*
- * Write @data into the MSR specified by @index.  Select MSR specific fault
- * checks are bypassed if @host_initiated is %true.
- * Returns 0 on success, non-0 otherwise.
- * Assumes vcpu_load() was already called.
- */
-static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
-			 bool host_initiated)
-{
-	struct msr_data msr;
-
-	switch (index) {
-	case MSR_FS_BASE:
-	case MSR_GS_BASE:
-	case MSR_KERNEL_GS_BASE:
-	case MSR_CSTAR:
-	case MSR_LSTAR:
-		if (is_noncanonical_msr_address(data, vcpu))
-			return 1;
-		break;
-	case MSR_IA32_SYSENTER_EIP:
-	case MSR_IA32_SYSENTER_ESP:
-		/*
-		 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if
-		 * non-canonical address is written on Intel but not on
-		 * AMD (which ignores the top 32-bits, because it does
-		 * not implement 64-bit SYSENTER).
-		 *
-		 * 64-bit code should hence be able to write a non-canonical
-		 * value on AMD.  Making the address canonical ensures that
-		 * vmentry does not fail on Intel after writing a non-canonical
-		 * value, and that something deterministic happens if the guest
-		 * invokes 64-bit SYSENTER.
-		 */
-		data = __canonical_address(data, max_host_virt_addr_bits());
-		break;
-	case MSR_TSC_AUX:
-		if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX))
-			return 1;
-
-		if (!host_initiated &&
-		    !guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) &&
-		    !guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID))
-			return 1;
-
-		/*
-		 * Per Intel's SDM, bits 63:32 are reserved, but AMD's APM has
-		 * incomplete and conflicting architectural behavior.  Current
-		 * AMD CPUs completely ignore bits 63:32, i.e. they aren't
-		 * reserved and always read as zeros.  Enforce Intel's reserved
-		 * bits check if the guest CPU is Intel compatible, otherwise
-		 * clear the bits.  This ensures cross-vendor migration will
-		 * provide consistent behavior for the guest.
-		 */
-		if (guest_cpuid_is_intel_compatible(vcpu) && (data >> 32) != 0)
-			return 1;
+	/* Compute a scale to convert nanoseconds in TSC cycles */
+	kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC,
+			   &vcpu->arch.virtual_tsc_shift,
+			   &vcpu->arch.virtual_tsc_mult);
+	vcpu->arch.virtual_tsc_khz = user_tsc_khz;
 
-		data = (u32)data;
-		break;
-	case MSR_IA32_U_CET:
-	case MSR_IA32_S_CET:
-		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) &&
-		    !guest_cpu_cap_has(vcpu, X86_FEATURE_IBT))
-			return KVM_MSR_RET_UNSUPPORTED;
-		if (!kvm_is_valid_u_s_cet(vcpu, data))
-			return 1;
-		break;
-	case MSR_KVM_INTERNAL_GUEST_SSP:
-		if (!host_initiated)
-			return 1;
-		fallthrough;
-		/*
-		 * Note that the MSR emulation here is flawed when a vCPU
-		 * doesn't support the Intel 64 architecture. The expected
-		 * architectural behavior in this case is that the upper 32
-		 * bits do not exist and should always read '0'. However,
-		 * because the actual hardware on which the virtual CPU is
-		 * running does support Intel 64, XRSTORS/XSAVES in the
-		 * guest could observe behavior that violates the
-		 * architecture. Intercepting XRSTORS/XSAVES for this
-		 * special case isn't deemed worthwhile.
-		 */
-	case MSR_IA32_PL0_SSP ... MSR_IA32_INT_SSP_TAB:
-		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK))
-			return KVM_MSR_RET_UNSUPPORTED;
-		/*
-		 * MSR_IA32_INT_SSP_TAB is not present on processors that do
-		 * not support Intel 64 architecture.
-		 */
-		if (index == MSR_IA32_INT_SSP_TAB && !guest_cpu_cap_has(vcpu, X86_FEATURE_LM))
-			return KVM_MSR_RET_UNSUPPORTED;
-		if (is_noncanonical_msr_address(data, vcpu))
-			return 1;
-		/* All SSP MSRs except MSR_IA32_INT_SSP_TAB must be 4-byte aligned */
-		if (index != MSR_IA32_INT_SSP_TAB && !IS_ALIGNED(data, 4))
-			return 1;
-		break;
+	/*
+	 * Compute the variation in TSC rate which is acceptable
+	 * within the range of tolerance and decide if the
+	 * rate being applied is within that bounds of the hardware
+	 * rate.  If so, no scaling or compensation need be done.
+	 */
+	thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
+	thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
+	if (user_tsc_khz < thresh_lo || user_tsc_khz > thresh_hi) {
+		pr_debug("requested TSC rate %u falls outside tolerance [%u,%u]\n",
+			 user_tsc_khz, thresh_lo, thresh_hi);
+		use_scaling = 1;
 	}
-
-	msr.data = data;
-	msr.index = index;
-	msr.host_initiated = host_initiated;
-
-	return kvm_x86_call(set_msr)(vcpu, &msr);
+	return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);
 }
 
-static int _kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
-			bool host_initiated)
+static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
 {
-	return __kvm_set_msr(vcpu, index, *data, host_initiated);
+	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
+				      vcpu->arch.virtual_tsc_mult,
+				      vcpu->arch.virtual_tsc_shift);
+	tsc += vcpu->arch.this_tsc_write;
+	return tsc;
 }
 
-static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,
-				     u32 index, u64 data, bool host_initiated)
+#ifdef CONFIG_X86_64
+static inline bool gtod_is_based_on_tsc(int mode)
 {
-	return kvm_do_msr_access(vcpu, index, &data, host_initiated, MSR_TYPE_W,
-				 _kvm_set_msr);
+	return mode == VDSO_CLOCKMODE_TSC || mode == VDSO_CLOCKMODE_HVCLOCK;
 }
+#endif
 
-/*
- * Read the MSR specified by @index into @data.  Select MSR specific fault
- * checks are bypassed if @host_initiated is %true.
- * Returns 0 on success, non-0 otherwise.
- * Assumes vcpu_load() was already called.
- */
-static int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
-			 bool host_initiated)
+static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu, bool new_generation)
 {
-	struct msr_data msr;
-	int ret;
-
-	switch (index) {
-	case MSR_TSC_AUX:
-		if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX))
-			return 1;
-
-		if (!host_initiated &&
-		    !guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) &&
-		    !guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID))
-			return 1;
-		break;
-	case MSR_IA32_U_CET:
-	case MSR_IA32_S_CET:
-		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) &&
-		    !guest_cpu_cap_has(vcpu, X86_FEATURE_IBT))
-			return KVM_MSR_RET_UNSUPPORTED;
-		break;
-	case MSR_KVM_INTERNAL_GUEST_SSP:
-		if (!host_initiated)
-			return 1;
-		fallthrough;
-	case MSR_IA32_PL0_SSP ... MSR_IA32_INT_SSP_TAB:
-		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK))
-			return KVM_MSR_RET_UNSUPPORTED;
-		break;
-	}
-
-	msr.index = index;
-	msr.host_initiated = host_initiated;
-
-	ret = kvm_x86_call(get_msr)(vcpu, &msr);
-	if (!ret)
-		*data = msr.data;
-	return ret;
-}
+#ifdef CONFIG_X86_64
+	struct kvm_arch *ka = &vcpu->kvm->arch;
+	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
 
-int kvm_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data)
-{
-	return __kvm_set_msr(vcpu, index, data, true);
-}
+	/*
+	 * To use the masterclock, the host clocksource must be based on TSC
+	 * and all vCPUs must have matching TSCs.  Note, the count for matching
+	 * vCPUs doesn't include the reference vCPU, hence "+1".
+	 */
+	bool use_master_clock = (ka->nr_vcpus_matched_tsc + 1 ==
+				 atomic_read(&vcpu->kvm->online_vcpus)) &&
+				gtod_is_based_on_tsc(gtod->clock.vclock_mode);
 
-int kvm_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data)
-{
-	return __kvm_get_msr(vcpu, index, data, true);
-}
+	/*
+	 * Request a masterclock update if the masterclock needs to be toggled
+	 * on/off, or when starting a new generation and the masterclock is
+	 * enabled (compute_guest_tsc() requires the masterclock snapshot to be
+	 * taken _after_ the new generation is created).
+	 */
+	if ((ka->use_master_clock && new_generation) ||
+	    (ka->use_master_clock != use_master_clock))
+		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
 
-static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu,
-				     u32 index, u64 *data, bool host_initiated)
-{
-	return kvm_do_msr_access(vcpu, index, data, host_initiated, MSR_TYPE_R,
-				 __kvm_get_msr);
+	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
+			    atomic_read(&vcpu->kvm->online_vcpus),
+		            ka->use_master_clock, gtod->clock.vclock_mode);
+#endif
 }
 
-int __kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data)
+/*
+ * Multiply tsc by a fixed point number represented by ratio.
+ *
+ * The most significant 64-N bits (mult) of ratio represent the
+ * integral part of the fixed point number; the remaining N bits
+ * (frac) represent the fractional part, ie. ratio represents a fixed
+ * point number (mult + frac * 2^(-N)).
+ *
+ * N equals to kvm_caps.tsc_scaling_ratio_frac_bits.
+ */
+static inline u64 __scale_tsc(u64 ratio, u64 tsc)
 {
-	return kvm_get_msr_ignored_check(vcpu, index, data, false);
+	return mul_u64_u64_shr(tsc, ratio, kvm_caps.tsc_scaling_ratio_frac_bits);
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_emulate_msr_read);
 
-int __kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data)
+u64 kvm_scale_tsc(u64 tsc, u64 ratio)
 {
-	return kvm_set_msr_ignored_check(vcpu, index, data, false);
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_emulate_msr_write);
+	u64 _tsc = tsc;
 
-int kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data)
-{
-	if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ))
-		return KVM_MSR_RET_FILTERED;
+	if (ratio != kvm_caps.default_tsc_scaling_ratio)
+		_tsc = __scale_tsc(ratio, tsc);
 
-	return __kvm_emulate_msr_read(vcpu, index, data);
+	return _tsc;
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_msr_read);
 
-int kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data)
+u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
 {
-	if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE))
-		return KVM_MSR_RET_FILTERED;
-
-	return __kvm_emulate_msr_write(vcpu, index, data);
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_msr_write);
+	u64 tsc;
 
+	tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio);
 
-static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu)
-{
-	if (!vcpu->run->msr.error) {
-		kvm_eax_write(vcpu, vcpu->run->msr.data);
-		kvm_edx_write(vcpu, vcpu->run->msr.data >> 32);
-	}
+	return target_tsc - tsc;
 }
 
-static int complete_emulated_msr_access(struct kvm_vcpu *vcpu)
+u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
 {
-	return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error);
+	return vcpu->arch.l1_tsc_offset +
+		kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio);
 }
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_read_l1_tsc);
 
-static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)
+u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier)
 {
-	complete_userspace_rdmsr(vcpu);
-	return complete_emulated_msr_access(vcpu);
-}
+	u64 nested_offset;
 
-static int complete_fast_msr_access(struct kvm_vcpu *vcpu)
-{
-	return kvm_x86_call(complete_emulated_msr)(vcpu, vcpu->run->msr.error);
-}
+	if (l2_multiplier == kvm_caps.default_tsc_scaling_ratio)
+		nested_offset = l1_offset;
+	else
+		nested_offset = mul_s64_u64_shr((s64) l1_offset, l2_multiplier,
+						kvm_caps.tsc_scaling_ratio_frac_bits);
 
-static int complete_fast_rdmsr(struct kvm_vcpu *vcpu)
-{
-	complete_userspace_rdmsr(vcpu);
-	return complete_fast_msr_access(vcpu);
+	nested_offset += l2_offset;
+	return nested_offset;
 }
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_calc_nested_tsc_offset);
 
-static int complete_fast_rdmsr_imm(struct kvm_vcpu *vcpu)
+u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier)
 {
-	if (!vcpu->run->msr.error)
-		kvm_register_write(vcpu, vcpu->arch.cui_rdmsr_imm_reg,
-				   vcpu->run->msr.data);
+	if (l2_multiplier != kvm_caps.default_tsc_scaling_ratio)
+		return mul_u64_u64_shr(l1_multiplier, l2_multiplier,
+				       kvm_caps.tsc_scaling_ratio_frac_bits);
 
-	return complete_fast_msr_access(vcpu);
+	return l1_multiplier;
 }
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_calc_nested_tsc_multiplier);
 
-static u64 kvm_msr_reason(int r)
+void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset)
 {
-	switch (r) {
-	case KVM_MSR_RET_UNSUPPORTED:
-		return KVM_MSR_EXIT_REASON_UNKNOWN;
-	case KVM_MSR_RET_FILTERED:
-		return KVM_MSR_EXIT_REASON_FILTER;
-	default:
-		return KVM_MSR_EXIT_REASON_INVAL;
-	}
-}
+	if (vcpu->arch.guest_tsc_protected)
+		return;
 
-static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index,
-			      u32 exit_reason, u64 data,
-			      int (*completion)(struct kvm_vcpu *vcpu),
-			      int r)
-{
-	u64 msr_reason = kvm_msr_reason(r);
+	trace_kvm_write_tsc_offset(vcpu->vcpu_id,
+				   vcpu->arch.l1_tsc_offset,
+				   l1_offset);
 
-	/* Check if the user wanted to know about this MSR fault */
-	if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason))
-		return 0;
+	vcpu->arch.l1_tsc_offset = l1_offset;
 
-	vcpu->run->exit_reason = exit_reason;
-	vcpu->run->msr.error = 0;
-	memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad));
-	vcpu->run->msr.reason = msr_reason;
-	vcpu->run->msr.index = index;
-	vcpu->run->msr.data = data;
-	vcpu->arch.complete_userspace_io = completion;
+	/*
+	 * If we are here because L1 chose not to trap WRMSR to TSC then
+	 * according to the spec this should set L1's TSC (as opposed to
+	 * setting L1's offset for L2).
+	 */
+	if (is_guest_mode(vcpu))
+		vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(
+			l1_offset,
+			kvm_x86_call(get_l2_tsc_offset)(vcpu),
+			kvm_x86_call(get_l2_tsc_multiplier)(vcpu));
+	else
+		vcpu->arch.tsc_offset = l1_offset;
 
-	return 1;
+	kvm_x86_call(write_tsc_offset)(vcpu);
 }
 
-static int __kvm_emulate_rdmsr(struct kvm_vcpu *vcpu, u32 msr, int reg,
-			       int (*complete_rdmsr)(struct kvm_vcpu *))
+static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier)
 {
-	u64 data;
-	int r;
-
-	r = kvm_emulate_msr_read(vcpu, msr, &data);
-
-	if (!r) {
-		trace_kvm_msr_read(msr, data);
+	vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier;
 
-		if (reg < 0) {
-			kvm_eax_write(vcpu, data);
-			kvm_edx_write(vcpu, data >> 32);
-		} else {
-			kvm_register_write(vcpu, reg, data);
-		}
-	} else {
-		/* MSR read failed? See if we should ask user space */
-		if (kvm_msr_user_space(vcpu, msr, KVM_EXIT_X86_RDMSR, 0,
-				       complete_rdmsr, r))
-			return 0;
-		trace_kvm_msr_read_ex(msr);
-	}
-
-	return kvm_x86_call(complete_emulated_msr)(vcpu, r);
-}
+	/* Userspace is changing the multiplier while L2 is active */
+	if (is_guest_mode(vcpu))
+		vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier(
+			l1_multiplier,
+			kvm_x86_call(get_l2_tsc_multiplier)(vcpu));
+	else
+		vcpu->arch.tsc_scaling_ratio = l1_multiplier;
 
-int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
-{
-	return __kvm_emulate_rdmsr(vcpu, kvm_ecx_read(vcpu), -1,
-				   complete_fast_rdmsr);
+	if (kvm_caps.has_tsc_control)
+		kvm_x86_call(write_tsc_multiplier)(vcpu);
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdmsr);
 
-int kvm_emulate_rdmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg)
+static inline bool kvm_check_tsc_unstable(void)
 {
-	vcpu->arch.cui_rdmsr_imm_reg = reg;
-
-	return __kvm_emulate_rdmsr(vcpu, msr, reg, complete_fast_rdmsr_imm);
+#ifdef CONFIG_X86_64
+	/*
+	 * TSC is marked unstable when we're running on Hyper-V,
+	 * 'TSC page' clocksource is good.
+	 */
+	if (pvclock_gtod_data.clock.vclock_mode == VDSO_CLOCKMODE_HVCLOCK)
+		return false;
+#endif
+	return check_tsc_unstable();
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdmsr_imm);
 
-static int __kvm_emulate_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
+/*
+ * Infers attempts to synchronize the guest's tsc from host writes. Sets the
+ * offset for the vcpu and tracks the TSC matching generation that the vcpu
+ * participates in.
+ */
+static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,
+				  u64 ns, bool matched, bool user_set_tsc)
 {
-	int r;
+	struct kvm *kvm = vcpu->kvm;
 
-	r = kvm_emulate_msr_write(vcpu, msr, data);
-	if (!r) {
-		trace_kvm_msr_write(msr, data);
-	} else {
-		/* MSR write failed? See if we should ask user space */
-		if (kvm_msr_user_space(vcpu, msr, KVM_EXIT_X86_WRMSR, data,
-				       complete_fast_msr_access, r))
-			return 0;
-		/* Signal all other negative errors to userspace */
-		if (r < 0)
-			return r;
-		trace_kvm_msr_write_ex(msr, data);
-	}
+	lockdep_assert_held(&kvm->arch.tsc_write_lock);
 
-	return kvm_x86_call(complete_emulated_msr)(vcpu, r);
-}
+	if (vcpu->arch.guest_tsc_protected)
+		return;
 
-int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
-{
-	return __kvm_emulate_wrmsr(vcpu, kvm_ecx_read(vcpu),
-				   kvm_read_edx_eax(vcpu));
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_wrmsr);
+	if (user_set_tsc)
+		vcpu->kvm->arch.user_set_tsc = true;
 
-int kvm_emulate_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg)
-{
-	return __kvm_emulate_wrmsr(vcpu, msr, kvm_register_read(vcpu, reg));
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_wrmsr_imm);
+	/*
+	 * We also track th most recent recorded KHZ, write and time to
+	 * allow the matching interval to be extended at each write.
+	 */
+	kvm->arch.last_tsc_nsec = ns;
+	kvm->arch.last_tsc_write = tsc;
+	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
+	kvm->arch.last_tsc_offset = offset;
 
-int kvm_emulate_as_nop(struct kvm_vcpu *vcpu)
-{
-	return kvm_skip_emulated_instruction(vcpu);
-}
+	vcpu->arch.last_guest_tsc = tsc;
 
-int kvm_emulate_invd(struct kvm_vcpu *vcpu)
-{
-	/* Treat an INVD instruction as a NOP and just skip it. */
-	return kvm_emulate_as_nop(vcpu);
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_invd);
+	kvm_vcpu_write_tsc_offset(vcpu, offset);
 
-fastpath_t handle_fastpath_invd(struct kvm_vcpu *vcpu)
-{
-	if (!kvm_pmu_is_fastpath_emulation_allowed(vcpu))
-		return EXIT_FASTPATH_NONE;
+	if (!matched) {
+		/*
+		 * We split periods of matched TSC writes into generations.
+		 * For each generation, we track the original measured
+		 * nanosecond time, offset, and write, so if TSCs are in
+		 * sync, we can match exact offset, and if not, we can match
+		 * exact software computation in compute_guest_tsc()
+		 *
+		 * These values are tracked in kvm->arch.cur_xxx variables.
+		 */
+		kvm->arch.cur_tsc_generation++;
+		kvm->arch.cur_tsc_nsec = ns;
+		kvm->arch.cur_tsc_write = tsc;
+		kvm->arch.cur_tsc_offset = offset;
+		kvm->arch.nr_vcpus_matched_tsc = 0;
+	} else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) {
+		kvm->arch.nr_vcpus_matched_tsc++;
+	}
 
-	if (!kvm_emulate_invd(vcpu))
-		return EXIT_FASTPATH_EXIT_USERSPACE;
+	/* Keep track of which generation this VCPU has synchronized to */
+	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
+	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
+	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;
 
-	return EXIT_FASTPATH_REENTER_GUEST;
+	kvm_track_tsc_matching(vcpu, !matched);
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_invd);
 
-int kvm_handle_invalid_op(struct kvm_vcpu *vcpu)
+void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)
 {
-	kvm_queue_exception(vcpu, UD_VECTOR);
-	return 1;
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_handle_invalid_op);
-
+	u64 data = user_value ? *user_value : 0;
+	struct kvm *kvm = vcpu->kvm;
+	u64 offset, ns, elapsed;
+	unsigned long flags;
+	bool matched = false;
+	bool synchronizing = false;
 
-static int kvm_emulate_monitor_mwait(struct kvm_vcpu *vcpu, const char *insn)
-{
-	bool enabled;
+	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
+	offset = kvm_compute_l1_tsc_offset(vcpu, data);
+	ns = get_kvmclock_base_ns();
+	elapsed = ns - kvm->arch.last_tsc_nsec;
 
-	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS))
-		goto emulate_as_nop;
+	if (vcpu->arch.virtual_tsc_khz) {
+		if (data == 0) {
+			/*
+			 * Force synchronization when creating a vCPU, or when
+			 * userspace explicitly writes a zero value.
+			 */
+			synchronizing = true;
+		} else if (kvm->arch.user_set_tsc) {
+			u64 tsc_exp = kvm->arch.last_tsc_write +
+						nsec_to_cycles(vcpu, elapsed);
+			u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;
+			/*
+			 * Here lies UAPI baggage: when a user-initiated TSC write has
+			 * a small delta (1 second) of virtual cycle time against the
+			 * previously set vCPU, we assume that they were intended to be
+			 * in sync and the delta was only due to the racy nature of the
+			 * legacy API.
+			 *
+			 * This trick falls down when restoring a guest which genuinely
+			 * has been running for less time than the 1 second of imprecision
+			 * which we allow for in the legacy API. In this case, the first
+			 * value written by userspace (on any vCPU) should not be subject
+			 * to this 'correction' to make it sync up with values that only
+			 * come from the kernel's default vCPU creation. Make the 1-second
+			 * slop hack only trigger if the user_set_tsc flag is already set.
+			 */
+			synchronizing = data < tsc_exp + tsc_hz &&
+					data + tsc_hz > tsc_exp;
+		}
+	}
 
-	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT))
-		enabled = guest_cpu_cap_has(vcpu, X86_FEATURE_MWAIT);
-	else
-		enabled = vcpu->arch.ia32_misc_enable_msr & MSR_IA32_MISC_ENABLE_MWAIT;
 
-	if (!enabled)
-		return kvm_handle_invalid_op(vcpu);
+	/*
+	 * For a reliable TSC, we can match TSC offsets, and for an unstable
+	 * TSC, we add elapsed time in this computation.  We could let the
+	 * compensation code attempt to catch up if we fall behind, but
+	 * it's better to try to match offsets from the beginning.
+         */
+	if (synchronizing &&
+	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
+		if (!kvm_check_tsc_unstable()) {
+			offset = kvm->arch.cur_tsc_offset;
+		} else {
+			u64 delta = nsec_to_cycles(vcpu, elapsed);
+			data += delta;
+			offset = kvm_compute_l1_tsc_offset(vcpu, data);
+		}
+		matched = true;
+	}
 
-emulate_as_nop:
-	pr_warn_once("%s instruction emulated as NOP!\n", insn);
-	return kvm_emulate_as_nop(vcpu);
-}
-int kvm_emulate_mwait(struct kvm_vcpu *vcpu)
-{
-	return kvm_emulate_monitor_mwait(vcpu, "MWAIT");
+	__kvm_synchronize_tsc(vcpu, offset, data, ns, matched, !!user_value);
+	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_mwait);
 
-int kvm_emulate_monitor(struct kvm_vcpu *vcpu)
-{
-	return kvm_emulate_monitor_mwait(vcpu, "MONITOR");
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_monitor);
+#ifdef CONFIG_X86_64
 
-static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)
+static u64 read_tsc(void)
 {
-	xfer_to_guest_mode_prepare();
+	u64 ret = (u64)rdtsc_ordered();
+	u64 last = pvclock_gtod_data.clock.cycle_last;
 
-	return READ_ONCE(vcpu->mode) == EXITING_GUEST_MODE ||
-	       kvm_request_pending(vcpu) || xfer_to_guest_mode_work_pending();
+	if (likely(ret >= last))
+		return ret;
+
+	/*
+	 * GCC likes to generate cmov here, but this branch is extremely
+	 * predictable (it's just a function of time and the likely is
+	 * very likely) and there's a data dependence, so force GCC
+	 * to generate a branch instead.  I don't barrier() because
+	 * we don't actually need a barrier, and if this function
+	 * ever gets inlined it will generate worse code.
+	 */
+	asm volatile ("");
+	return last;
 }
 
-static fastpath_t __handle_fastpath_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
+static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp,
+			  int *mode)
 {
-	if (!kvm_pmu_is_fastpath_emulation_allowed(vcpu))
-		return EXIT_FASTPATH_NONE;
+	u64 tsc_pg_val;
+	long v;
 
-	switch (msr) {
-	case APIC_BASE_MSR + (APIC_ICR >> 4):
-		if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic) ||
-		    kvm_x2apic_icr_write_fast(vcpu->arch.apic, data))
-			return EXIT_FASTPATH_NONE;
+	switch (clock->vclock_mode) {
+	case VDSO_CLOCKMODE_HVCLOCK:
+		if (hv_read_tsc_page_tsc(hv_get_tsc_page(),
+					 tsc_timestamp, &tsc_pg_val)) {
+			/* TSC page valid */
+			*mode = VDSO_CLOCKMODE_HVCLOCK;
+			v = (tsc_pg_val - clock->cycle_last) &
+				clock->mask;
+		} else {
+			/* TSC page invalid */
+			*mode = VDSO_CLOCKMODE_NONE;
+		}
 		break;
-	case MSR_IA32_TSC_DEADLINE:
-		kvm_set_lapic_tscdeadline_msr(vcpu, data);
+	case VDSO_CLOCKMODE_TSC:
+		*mode = VDSO_CLOCKMODE_TSC;
+		*tsc_timestamp = read_tsc();
+		v = (*tsc_timestamp - clock->cycle_last) &
+			clock->mask;
 		break;
 	default:
-		return EXIT_FASTPATH_NONE;
+		*mode = VDSO_CLOCKMODE_NONE;
 	}
 
-	trace_kvm_msr_write(msr, data);
-
-	if (!kvm_skip_emulated_instruction(vcpu))
-		return EXIT_FASTPATH_EXIT_USERSPACE;
-
-	return EXIT_FASTPATH_REENTER_GUEST;
-}
-
-fastpath_t handle_fastpath_wrmsr(struct kvm_vcpu *vcpu)
-{
-	return __handle_fastpath_wrmsr(vcpu, kvm_ecx_read(vcpu),
-				       kvm_read_edx_eax(vcpu));
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_wrmsr);
+	if (*mode == VDSO_CLOCKMODE_NONE)
+		*tsc_timestamp = v = 0;
 
-fastpath_t handle_fastpath_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg)
-{
-	return __handle_fastpath_wrmsr(vcpu, msr, kvm_register_read(vcpu, reg));
+	return v * clock->mult;
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_wrmsr_imm);
 
 /*
- * Adapt set_msr() to msr_io()'s calling convention
+ * As with get_kvmclock_base_ns(), this counts from boot time, at the
+ * frequency of CLOCK_MONOTONIC_RAW (hence adding gtos->offs_boot).
  */
-static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
+static int do_kvmclock_base(s64 *t, u64 *tsc_timestamp)
 {
-	return kvm_get_msr_ignored_check(vcpu, index, data, true);
-}
+	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
+	unsigned long seq;
+	int mode;
+	u64 ns;
 
-static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
-{
-	u64 val;
-
-	/*
-	 * Reject writes to immutable feature MSRs if the vCPU model is frozen,
-	 * as KVM doesn't support modifying the guest vCPU model on the fly,
-	 * e.g. changing the VMX capabilities MSRs while L2 is active is
-	 * nonsensical.  Allow writes of the same value, e.g. so that userspace
-	 * can blindly stuff all MSRs when emulating RESET.
-	 */
-	if (!kvm_can_set_cpuid_and_feature_msrs(vcpu) &&
-	    kvm_is_immutable_feature_msr(index) &&
-	    (do_get_msr(vcpu, index, &val) || *data != val))
-		return -EINVAL;
+	do {
+		seq = read_seqcount_begin(&gtod->seq);
+		ns = gtod->raw_clock.base_cycles;
+		ns += vgettsc(&gtod->raw_clock, tsc_timestamp, &mode);
+		ns >>= gtod->raw_clock.shift;
+		ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot));
+	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
+	*t = ns;
 
-	return kvm_set_msr_ignored_check(vcpu, index, *data, true);
+	return mode;
 }
 
-#ifdef CONFIG_X86_64
-struct pvclock_clock {
-	int vclock_mode;
-	u64 cycle_last;
-	u64 mask;
-	u32 mult;
-	u32 shift;
-	u64 base_cycles;
-	u64 offset;
-};
-
-struct pvclock_gtod_data {
-	seqcount_t	seq;
-
-	struct pvclock_clock clock; /* extract of a clocksource struct */
-	struct pvclock_clock raw_clock; /* extract of a clocksource struct */
-
-	ktime_t		offs_boot;
-	u64		wall_time_sec;
-};
-
-static struct pvclock_gtod_data pvclock_gtod_data;
-
-static void update_pvclock_gtod(struct timekeeper *tk)
+/*
+ * This calculates CLOCK_MONOTONIC at the time of the TSC snapshot, with
+ * no boot time offset.
+ */
+static int do_monotonic(s64 *t, u64 *tsc_timestamp)
 {
-	struct pvclock_gtod_data *vdata = &pvclock_gtod_data;
+	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
+	unsigned long seq;
+	int mode;
+	u64 ns;
 
-	write_seqcount_begin(&vdata->seq);
+	do {
+		seq = read_seqcount_begin(&gtod->seq);
+		ns = gtod->clock.base_cycles;
+		ns += vgettsc(&gtod->clock, tsc_timestamp, &mode);
+		ns >>= gtod->clock.shift;
+		ns += ktime_to_ns(gtod->clock.offset);
+	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
+	*t = ns;
 
-	/* copy pvclock gtod data */
-	vdata->clock.vclock_mode	= tk->tkr_mono.clock->vdso_clock_mode;
-	vdata->clock.cycle_last		= tk->tkr_mono.cycle_last;
-	vdata->clock.mask		= tk->tkr_mono.mask;
-	vdata->clock.mult		= tk->tkr_mono.mult;
-	vdata->clock.shift		= tk->tkr_mono.shift;
-	vdata->clock.base_cycles	= tk->tkr_mono.xtime_nsec;
-	vdata->clock.offset		= tk->tkr_mono.base;
+	return mode;
+}
 
-	vdata->raw_clock.vclock_mode	= tk->tkr_raw.clock->vdso_clock_mode;
-	vdata->raw_clock.cycle_last	= tk->tkr_raw.cycle_last;
-	vdata->raw_clock.mask		= tk->tkr_raw.mask;
-	vdata->raw_clock.mult		= tk->tkr_raw.mult;
-	vdata->raw_clock.shift		= tk->tkr_raw.shift;
-	vdata->raw_clock.base_cycles	= tk->tkr_raw.xtime_nsec;
-	vdata->raw_clock.offset		= tk->tkr_raw.base;
+static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
+{
+	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
+	unsigned long seq;
+	int mode;
+	u64 ns;
 
-	vdata->wall_time_sec            = tk->xtime_sec;
+	do {
+		seq = read_seqcount_begin(&gtod->seq);
+		ts->tv_sec = gtod->wall_time_sec;
+		ns = gtod->clock.base_cycles;
+		ns += vgettsc(&gtod->clock, tsc_timestamp, &mode);
+		ns >>= gtod->clock.shift;
+	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
 
-	vdata->offs_boot		= tk->offs_boot;
+	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
+	ts->tv_nsec = ns;
 
-	write_seqcount_end(&vdata->seq);
+	return mode;
 }
 
-static s64 get_kvmclock_base_ns(void)
+/*
+ * Calculates the kvmclock_base_ns (CLOCK_MONOTONIC_RAW + boot time) and
+ * reports the TSC value from which it do so. Returns true if host is
+ * using TSC based clocksource.
+ */
+static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
 {
-	/* Count up from boot time, but with the frequency of the raw clock.  */
-	return ktime_to_ns(ktime_add(ktime_get_raw(), pvclock_gtod_data.offs_boot));
+	/* checked again under seqlock below */
+	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
+		return false;
+
+	return gtod_is_based_on_tsc(do_kvmclock_base(kernel_ns,
+						     tsc_timestamp));
 }
-#else
-static s64 get_kvmclock_base_ns(void)
+
+/*
+ * Calculates CLOCK_MONOTONIC and reports the TSC value from which it did
+ * so. Returns true if host is using TSC based clocksource.
+ */
+bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
 {
-	/* Master clock not used, so we can just use CLOCK_BOOTTIME.  */
-	return ktime_get_boottime_ns();
+	/* checked again under seqlock below */
+	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
+		return false;
+
+	return gtod_is_based_on_tsc(do_monotonic(kernel_ns,
+						 tsc_timestamp));
 }
-#endif
 
-static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs)
+/*
+ * Calculates CLOCK_REALTIME and reports the TSC value from which it did
+ * so. Returns true if host is using TSC based clocksource.
+ *
+ * DO NOT USE this for anything related to migration. You want CLOCK_TAI
+ * for that.
+ */
+static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
+					   u64 *tsc_timestamp)
 {
-	int version;
-	int r;
-	struct pvclock_wall_clock wc;
-	u32 wc_sec_hi;
-	u64 wall_nsec;
-
-	if (!wall_clock)
-		return;
+	/* checked again under seqlock below */
+	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
+		return false;
 
-	r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
-	if (r)
-		return;
+	return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp));
+}
+#endif
 
-	if (version & 1)
-		++version;  /* first time write, random junk */
+/*
+ *
+ * Assuming a stable TSC across physical CPUS, and a stable TSC
+ * across virtual CPUs, the following condition is possible.
+ * Each numbered line represents an event visible to both
+ * CPUs at the next numbered event.
+ *
+ * "timespecX" represents host monotonic time. "tscX" represents
+ * RDTSC value.
+ *
+ * 		VCPU0 on CPU0		|	VCPU1 on CPU1
+ *
+ * 1.  read timespec0,tsc0
+ * 2.					| timespec1 = timespec0 + N
+ * 					| tsc1 = tsc0 + M
+ * 3. transition to guest		| transition to guest
+ * 4. ret0 = timespec0 + (rdtsc - tsc0) |
+ * 5.				        | ret1 = timespec1 + (rdtsc - tsc1)
+ * 				        | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
+ *
+ * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
+ *
+ * 	- ret0 < ret1
+ *	- timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
+ *		...
+ *	- 0 < N - M => M < N
+ *
+ * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
+ * always the case (the difference between two distinct xtime instances
+ * might be smaller then the difference between corresponding TSC reads,
+ * when updating guest vcpus pvclock areas).
+ *
+ * To avoid that problem, do not allow visibility of distinct
+ * system_timestamp/tsc_timestamp values simultaneously: use a master
+ * copy of host monotonic time values. Update that master copy
+ * in lockstep.
+ *
+ * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
+ *
+ */
 
-	++version;
+static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
+{
+#ifdef CONFIG_X86_64
+	struct kvm_arch *ka = &kvm->arch;
+	int vclock_mode;
+	bool host_tsc_clocksource, vcpus_matched;
 
-	if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version)))
-		return;
+	lockdep_assert_held(&kvm->arch.tsc_write_lock);
+	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
+			atomic_read(&kvm->online_vcpus));
 
-	wall_nsec = kvm_get_wall_clock_epoch(kvm);
+	/*
+	 * If the host uses TSC clock, then passthrough TSC as stable
+	 * to the guest.
+	 */
+	host_tsc_clocksource = kvm_get_time_and_clockread(
+					&ka->master_kernel_ns,
+					&ka->master_cycle_now);
 
-	wc.nsec = do_div(wall_nsec, NSEC_PER_SEC);
-	wc.sec = (u32)wall_nsec; /* overflow in 2106 guest time */
-	wc.version = version;
+	ka->use_master_clock = host_tsc_clocksource && vcpus_matched
+				&& !ka->backwards_tsc_observed
+				&& !ka->boot_vcpu_runs_old_kvmclock;
 
-	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
+	if (ka->use_master_clock)
+		atomic_set(&kvm_guest_has_master_clock, 1);
 
-	if (sec_hi_ofs) {
-		wc_sec_hi = wall_nsec >> 32;
-		kvm_write_guest(kvm, wall_clock + sec_hi_ofs,
-				&wc_sec_hi, sizeof(wc_sec_hi));
-	}
+	vclock_mode = pvclock_gtod_data.clock.vclock_mode;
+	trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
+					vcpus_matched);
+#endif
+}
 
-	version++;
-	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
+static void kvm_make_mclock_inprogress_request(struct kvm *kvm)
+{
+	kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
 }
 
-static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time,
-				  bool old_msr, bool host_initiated)
+static void __kvm_start_pvclock_update(struct kvm *kvm)
 {
-	struct kvm_arch *ka = &vcpu->kvm->arch;
+	raw_spin_lock_irq(&kvm->arch.tsc_write_lock);
+	write_seqcount_begin(&kvm->arch.pvclock_sc);
+}
 
-	if (vcpu->vcpu_id == 0 && !host_initiated) {
-		if (ka->boot_vcpu_runs_old_kvmclock != old_msr)
-			kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
+static void kvm_start_pvclock_update(struct kvm *kvm)
+{
+	kvm_make_mclock_inprogress_request(kvm);
 
-		ka->boot_vcpu_runs_old_kvmclock = old_msr;
-	}
+	/* no guest entries from this point */
+	__kvm_start_pvclock_update(kvm);
+}
 
-	vcpu->arch.time = system_time;
-	kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
+static void kvm_end_pvclock_update(struct kvm *kvm)
+{
+	struct kvm_arch *ka = &kvm->arch;
+	struct kvm_vcpu *vcpu;
+	unsigned long i;
 
-	/* we verify if the enable bit is set... */
-	if (system_time & 1)
-		kvm_gpc_activate(&vcpu->arch.pv_time, system_time & ~1ULL,
-				 sizeof(struct pvclock_vcpu_time_info));
-	else
-		kvm_gpc_deactivate(&vcpu->arch.pv_time);
+	write_seqcount_end(&ka->pvclock_sc);
+	raw_spin_unlock_irq(&ka->tsc_write_lock);
+	kvm_for_each_vcpu(i, vcpu, kvm)
+		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
 
-	return;
+	/* guest entries allowed */
+	kvm_for_each_vcpu(i, vcpu, kvm)
+		kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
 }
 
-static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
+static void kvm_update_masterclock(struct kvm *kvm)
 {
-	do_shl32_div32(dividend, divisor);
-	return dividend;
+	kvm_hv_request_tsc_page_update(kvm);
+	kvm_start_pvclock_update(kvm);
+	pvclock_update_vm_gtod_copy(kvm);
+	kvm_end_pvclock_update(kvm);
 }
 
-static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz,
-			       s8 *pshift, u32 *pmultiplier)
-{
-	uint64_t scaled64;
-	int32_t  shift = 0;
-	uint64_t tps64;
-	uint32_t tps32;
-
-	tps64 = base_hz;
-	scaled64 = scaled_hz;
-	while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
-		tps64 >>= 1;
-		shift--;
-	}
-
-	tps32 = (uint32_t)tps64;
-	while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
-		if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
-			scaled64 >>= 1;
-		else
-			tps32 <<= 1;
-		shift++;
-	}
-
-	*pshift = shift;
-	*pmultiplier = div_frac(scaled64, tps32);
-}
-
-#ifdef CONFIG_X86_64
-static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
-#endif
-
-static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
-static unsigned long max_tsc_khz;
-
-static u32 adjust_tsc_khz(u32 khz, s32 ppm)
+/*
+ * Use the kernel's tsc_khz directly if the TSC is constant, otherwise use KVM's
+ * per-CPU value (which may be zero if a CPU is going offline).  Note, tsc_khz
+ * can change during boot even if the TSC is constant, as it's possible for KVM
+ * to be loaded before TSC calibration completes.  Ideally, KVM would get a
+ * notification when calibration completes, but practically speaking calibration
+ * will complete before userspace is alive enough to create VMs.
+ */
+static unsigned long get_cpu_tsc_khz(void)
 {
-	u64 v = (u64)khz * (1000000 + ppm);
-	do_div(v, 1000000);
-	return v;
+	if (cpu_feature_enabled(X86_FEATURE_CONSTANT_TSC))
+		return tsc_khz;
+	else
+		return __this_cpu_read(cpu_tsc_khz);
 }
 
-static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier);
-
-static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
+/* Called within read_seqcount_begin/retry for kvm->pvclock_sc.  */
+static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
 {
-	u64 ratio;
+	struct kvm_arch *ka = &kvm->arch;
+	struct pvclock_vcpu_time_info hv_clock;
 
-	/* Guest TSC same frequency as host TSC? */
-	if (!scale) {
-		kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);
-		return 0;
-	}
+	/* both __this_cpu_read() and rdtsc() should be on the same cpu */
+	get_cpu();
 
-	/* TSC scaling supported? */
-	if (!kvm_caps.has_tsc_control) {
-		if (user_tsc_khz > tsc_khz) {
-			vcpu->arch.tsc_catchup = 1;
-			vcpu->arch.tsc_always_catchup = 1;
-			return 0;
-		} else {
-			pr_warn_ratelimited("user requested TSC rate below hardware speed\n");
-			return -1;
-		}
-	}
+	data->flags = 0;
+	if (ka->use_master_clock &&
+	    (cpu_feature_enabled(X86_FEATURE_CONSTANT_TSC) || __this_cpu_read(cpu_tsc_khz))) {
+#ifdef CONFIG_X86_64
+		struct timespec64 ts;
 
-	/* TSC scaling required  - calculate ratio */
-	ratio = mul_u64_u32_div(1ULL << kvm_caps.tsc_scaling_ratio_frac_bits,
-				user_tsc_khz, tsc_khz);
+		if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) {
+			data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec;
+			data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC;
+		} else
+#endif
+		data->host_tsc = rdtsc();
 
-	if (ratio == 0 || ratio >= kvm_caps.max_tsc_scaling_ratio) {
-		pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n",
-			            user_tsc_khz);
-		return -1;
+		data->flags |= KVM_CLOCK_TSC_STABLE;
+		hv_clock.tsc_timestamp = ka->master_cycle_now;
+		hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
+		kvm_get_time_scale(NSEC_PER_SEC, get_cpu_tsc_khz() * 1000LL,
+				   &hv_clock.tsc_shift,
+				   &hv_clock.tsc_to_system_mul);
+		data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc);
+	} else {
+		data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset;
 	}
 
-	kvm_vcpu_write_tsc_multiplier(vcpu, ratio);
-	return 0;
+	put_cpu();
 }
 
-static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
+static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
 {
-	u32 thresh_lo, thresh_hi;
-	int use_scaling = 0;
-
-	/* tsc_khz can be zero if TSC calibration fails */
-	if (user_tsc_khz == 0) {
-		/* set tsc_scaling_ratio to a safe value */
-		kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);
-		return -1;
-	}
-
-	/* Compute a scale to convert nanoseconds in TSC cycles */
-	kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC,
-			   &vcpu->arch.virtual_tsc_shift,
-			   &vcpu->arch.virtual_tsc_mult);
-	vcpu->arch.virtual_tsc_khz = user_tsc_khz;
+	struct kvm_arch *ka = &kvm->arch;
+	unsigned seq;
 
-	/*
-	 * Compute the variation in TSC rate which is acceptable
-	 * within the range of tolerance and decide if the
-	 * rate being applied is within that bounds of the hardware
-	 * rate.  If so, no scaling or compensation need be done.
-	 */
-	thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
-	thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
-	if (user_tsc_khz < thresh_lo || user_tsc_khz > thresh_hi) {
-		pr_debug("requested TSC rate %u falls outside tolerance [%u,%u]\n",
-			 user_tsc_khz, thresh_lo, thresh_hi);
-		use_scaling = 1;
-	}
-	return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);
+	do {
+		seq = read_seqcount_begin(&ka->pvclock_sc);
+		__get_kvmclock(kvm, data);
+	} while (read_seqcount_retry(&ka->pvclock_sc, seq));
 }
 
-static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
+u64 get_kvmclock_ns(struct kvm *kvm)
 {
-	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
-				      vcpu->arch.virtual_tsc_mult,
-				      vcpu->arch.virtual_tsc_shift);
-	tsc += vcpu->arch.this_tsc_write;
-	return tsc;
-}
+	struct kvm_clock_data data;
 
-#ifdef CONFIG_X86_64
-static inline bool gtod_is_based_on_tsc(int mode)
-{
-	return mode == VDSO_CLOCKMODE_TSC || mode == VDSO_CLOCKMODE_HVCLOCK;
+	get_kvmclock(kvm, &data);
+	return data.clock;
 }
-#endif
 
-static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu, bool new_generation)
+static void kvm_setup_guest_pvclock(struct pvclock_vcpu_time_info *ref_hv_clock,
+				    struct kvm_vcpu *vcpu,
+				    struct gfn_to_pfn_cache *gpc,
+				    unsigned int offset)
 {
-#ifdef CONFIG_X86_64
-	struct kvm_arch *ka = &vcpu->kvm->arch;
-	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
+	struct pvclock_vcpu_time_info *guest_hv_clock;
+	struct pvclock_vcpu_time_info hv_clock;
+	unsigned long flags;
 
-	/*
-	 * To use the masterclock, the host clocksource must be based on TSC
-	 * and all vCPUs must have matching TSCs.  Note, the count for matching
-	 * vCPUs doesn't include the reference vCPU, hence "+1".
-	 */
-	bool use_master_clock = (ka->nr_vcpus_matched_tsc + 1 ==
-				 atomic_read(&vcpu->kvm->online_vcpus)) &&
-				gtod_is_based_on_tsc(gtod->clock.vclock_mode);
+	memcpy(&hv_clock, ref_hv_clock, sizeof(hv_clock));
+
+	read_lock_irqsave(&gpc->lock, flags);
+	while (!kvm_gpc_check(gpc, offset + sizeof(*guest_hv_clock))) {
+		read_unlock_irqrestore(&gpc->lock, flags);
+
+		if (kvm_gpc_refresh(gpc, offset + sizeof(*guest_hv_clock)))
+			return;
+
+		read_lock_irqsave(&gpc->lock, flags);
+	}
+
+	guest_hv_clock = (void *)(gpc->khva + offset);
 
 	/*
-	 * Request a masterclock update if the masterclock needs to be toggled
-	 * on/off, or when starting a new generation and the masterclock is
-	 * enabled (compute_guest_tsc() requires the masterclock snapshot to be
-	 * taken _after_ the new generation is created).
+	 * This VCPU is paused, but it's legal for a guest to read another
+	 * VCPU's kvmclock, so we really have to follow the specification where
+	 * it says that version is odd if data is being modified, and even after
+	 * it is consistent.
 	 */
-	if ((ka->use_master_clock && new_generation) ||
-	    (ka->use_master_clock != use_master_clock))
-		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
-
-	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
-			    atomic_read(&vcpu->kvm->online_vcpus),
-		            ka->use_master_clock, gtod->clock.vclock_mode);
-#endif
-}
 
-/*
- * Multiply tsc by a fixed point number represented by ratio.
- *
- * The most significant 64-N bits (mult) of ratio represent the
- * integral part of the fixed point number; the remaining N bits
- * (frac) represent the fractional part, ie. ratio represents a fixed
- * point number (mult + frac * 2^(-N)).
- *
- * N equals to kvm_caps.tsc_scaling_ratio_frac_bits.
- */
-static inline u64 __scale_tsc(u64 ratio, u64 tsc)
-{
-	return mul_u64_u64_shr(tsc, ratio, kvm_caps.tsc_scaling_ratio_frac_bits);
-}
+	guest_hv_clock->version = hv_clock.version = (guest_hv_clock->version + 1) | 1;
+	smp_wmb();
 
-u64 kvm_scale_tsc(u64 tsc, u64 ratio)
-{
-	u64 _tsc = tsc;
+	/* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
+	hv_clock.flags |= (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED);
 
-	if (ratio != kvm_caps.default_tsc_scaling_ratio)
-		_tsc = __scale_tsc(ratio, tsc);
+	memcpy(guest_hv_clock, &hv_clock, sizeof(*guest_hv_clock));
 
-	return _tsc;
-}
+	smp_wmb();
 
-static u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
-{
-	u64 tsc;
+	guest_hv_clock->version = ++hv_clock.version;
 
-	tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio);
+	kvm_gpc_mark_dirty_in_slot(gpc);
+	read_unlock_irqrestore(&gpc->lock, flags);
 
-	return target_tsc - tsc;
+	trace_kvm_pvclock_update(vcpu->vcpu_id, &hv_clock);
 }
 
-u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
+int kvm_guest_time_update(struct kvm_vcpu *v)
 {
-	return vcpu->arch.l1_tsc_offset +
-		kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio);
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_read_l1_tsc);
+	struct pvclock_vcpu_time_info hv_clock = {};
+	unsigned long flags, tgt_tsc_khz;
+	unsigned seq;
+	struct kvm_vcpu_arch *vcpu = &v->arch;
+	struct kvm_arch *ka = &v->kvm->arch;
+	s64 kernel_ns;
+	u64 tsc_timestamp, host_tsc;
+	bool use_master_clock;
 
-u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier)
-{
-	u64 nested_offset;
-
-	if (l2_multiplier == kvm_caps.default_tsc_scaling_ratio)
-		nested_offset = l1_offset;
-	else
-		nested_offset = mul_s64_u64_shr((s64) l1_offset, l2_multiplier,
-						kvm_caps.tsc_scaling_ratio_frac_bits);
-
-	nested_offset += l2_offset;
-	return nested_offset;
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_calc_nested_tsc_offset);
-
-u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier)
-{
-	if (l2_multiplier != kvm_caps.default_tsc_scaling_ratio)
-		return mul_u64_u64_shr(l1_multiplier, l2_multiplier,
-				       kvm_caps.tsc_scaling_ratio_frac_bits);
-
-	return l1_multiplier;
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_calc_nested_tsc_multiplier);
-
-static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset)
-{
-	if (vcpu->arch.guest_tsc_protected)
-		return;
-
-	trace_kvm_write_tsc_offset(vcpu->vcpu_id,
-				   vcpu->arch.l1_tsc_offset,
-				   l1_offset);
-
-	vcpu->arch.l1_tsc_offset = l1_offset;
-
-	/*
-	 * If we are here because L1 chose not to trap WRMSR to TSC then
-	 * according to the spec this should set L1's TSC (as opposed to
-	 * setting L1's offset for L2).
-	 */
-	if (is_guest_mode(vcpu))
-		vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(
-			l1_offset,
-			kvm_x86_call(get_l2_tsc_offset)(vcpu),
-			kvm_x86_call(get_l2_tsc_multiplier)(vcpu));
-	else
-		vcpu->arch.tsc_offset = l1_offset;
-
-	kvm_x86_call(write_tsc_offset)(vcpu);
-}
-
-static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier)
-{
-	vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier;
-
-	/* Userspace is changing the multiplier while L2 is active */
-	if (is_guest_mode(vcpu))
-		vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier(
-			l1_multiplier,
-			kvm_x86_call(get_l2_tsc_multiplier)(vcpu));
-	else
-		vcpu->arch.tsc_scaling_ratio = l1_multiplier;
-
-	if (kvm_caps.has_tsc_control)
-		kvm_x86_call(write_tsc_multiplier)(vcpu);
-}
-
-static inline bool kvm_check_tsc_unstable(void)
-{
-#ifdef CONFIG_X86_64
-	/*
-	 * TSC is marked unstable when we're running on Hyper-V,
-	 * 'TSC page' clocksource is good.
-	 */
-	if (pvclock_gtod_data.clock.vclock_mode == VDSO_CLOCKMODE_HVCLOCK)
-		return false;
-#endif
-	return check_tsc_unstable();
-}
-
-/*
- * Infers attempts to synchronize the guest's tsc from host writes. Sets the
- * offset for the vcpu and tracks the TSC matching generation that the vcpu
- * participates in.
- */
-static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,
-				  u64 ns, bool matched, bool user_set_tsc)
-{
-	struct kvm *kvm = vcpu->kvm;
-
-	lockdep_assert_held(&kvm->arch.tsc_write_lock);
-
-	if (vcpu->arch.guest_tsc_protected)
-		return;
-
-	if (user_set_tsc)
-		vcpu->kvm->arch.user_set_tsc = true;
+	kernel_ns = 0;
+	host_tsc = 0;
 
 	/*
-	 * We also track th most recent recorded KHZ, write and time to
-	 * allow the matching interval to be extended at each write.
+	 * If the host uses TSC clock, then passthrough TSC as stable
+	 * to the guest.
 	 */
-	kvm->arch.last_tsc_nsec = ns;
-	kvm->arch.last_tsc_write = tsc;
-	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
-	kvm->arch.last_tsc_offset = offset;
-
-	vcpu->arch.last_guest_tsc = tsc;
-
-	kvm_vcpu_write_tsc_offset(vcpu, offset);
-
-	if (!matched) {
-		/*
-		 * We split periods of matched TSC writes into generations.
-		 * For each generation, we track the original measured
-		 * nanosecond time, offset, and write, so if TSCs are in
-		 * sync, we can match exact offset, and if not, we can match
-		 * exact software computation in compute_guest_tsc()
-		 *
-		 * These values are tracked in kvm->arch.cur_xxx variables.
-		 */
-		kvm->arch.cur_tsc_generation++;
-		kvm->arch.cur_tsc_nsec = ns;
-		kvm->arch.cur_tsc_write = tsc;
-		kvm->arch.cur_tsc_offset = offset;
-		kvm->arch.nr_vcpus_matched_tsc = 0;
-	} else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) {
-		kvm->arch.nr_vcpus_matched_tsc++;
-	}
-
-	/* Keep track of which generation this VCPU has synchronized to */
-	vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
-	vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
-	vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;
-
-	kvm_track_tsc_matching(vcpu, !matched);
-}
-
-static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)
-{
-	u64 data = user_value ? *user_value : 0;
-	struct kvm *kvm = vcpu->kvm;
-	u64 offset, ns, elapsed;
-	unsigned long flags;
-	bool matched = false;
-	bool synchronizing = false;
-
-	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
-	offset = kvm_compute_l1_tsc_offset(vcpu, data);
-	ns = get_kvmclock_base_ns();
-	elapsed = ns - kvm->arch.last_tsc_nsec;
-
-	if (vcpu->arch.virtual_tsc_khz) {
-		if (data == 0) {
-			/*
-			 * Force synchronization when creating a vCPU, or when
-			 * userspace explicitly writes a zero value.
-			 */
-			synchronizing = true;
-		} else if (kvm->arch.user_set_tsc) {
-			u64 tsc_exp = kvm->arch.last_tsc_write +
-						nsec_to_cycles(vcpu, elapsed);
-			u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;
-			/*
-			 * Here lies UAPI baggage: when a user-initiated TSC write has
-			 * a small delta (1 second) of virtual cycle time against the
-			 * previously set vCPU, we assume that they were intended to be
-			 * in sync and the delta was only due to the racy nature of the
-			 * legacy API.
-			 *
-			 * This trick falls down when restoring a guest which genuinely
-			 * has been running for less time than the 1 second of imprecision
-			 * which we allow for in the legacy API. In this case, the first
-			 * value written by userspace (on any vCPU) should not be subject
-			 * to this 'correction' to make it sync up with values that only
-			 * come from the kernel's default vCPU creation. Make the 1-second
-			 * slop hack only trigger if the user_set_tsc flag is already set.
-			 */
-			synchronizing = data < tsc_exp + tsc_hz &&
-					data + tsc_hz > tsc_exp;
+	do {
+		seq = read_seqcount_begin(&ka->pvclock_sc);
+		use_master_clock = ka->use_master_clock;
+		if (use_master_clock) {
+			host_tsc = ka->master_cycle_now;
+			kernel_ns = ka->master_kernel_ns;
 		}
-	}
-
+	} while (read_seqcount_retry(&ka->pvclock_sc, seq));
 
-	/*
-	 * For a reliable TSC, we can match TSC offsets, and for an unstable
-	 * TSC, we add elapsed time in this computation.  We could let the
-	 * compensation code attempt to catch up if we fall behind, but
-	 * it's better to try to match offsets from the beginning.
-         */
-	if (synchronizing &&
-	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
-		if (!kvm_check_tsc_unstable()) {
-			offset = kvm->arch.cur_tsc_offset;
-		} else {
-			u64 delta = nsec_to_cycles(vcpu, elapsed);
-			data += delta;
-			offset = kvm_compute_l1_tsc_offset(vcpu, data);
-		}
-		matched = true;
+	/* Keep irq disabled to prevent changes to the clock */
+	local_irq_save(flags);
+	tgt_tsc_khz = get_cpu_tsc_khz();
+	if (unlikely(tgt_tsc_khz == 0)) {
+		local_irq_restore(flags);
+		kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
+		return 1;
+	}
+	if (!use_master_clock) {
+		host_tsc = rdtsc();
+		kernel_ns = get_kvmclock_base_ns();
 	}
 
-	__kvm_synchronize_tsc(vcpu, offset, data, ns, matched, !!user_value);
-	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
-}
-
-static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,
-					   s64 adjustment)
-{
-	u64 tsc_offset = vcpu->arch.l1_tsc_offset;
-	kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);
-}
-
-static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
-{
-	if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio)
-		WARN_ON(adjustment < 0);
-	adjustment = kvm_scale_tsc((u64) adjustment,
-				   vcpu->arch.l1_tsc_scaling_ratio);
-	adjust_tsc_offset_guest(vcpu, adjustment);
-}
-
-#ifdef CONFIG_X86_64
-
-static u64 read_tsc(void)
-{
-	u64 ret = (u64)rdtsc_ordered();
-	u64 last = pvclock_gtod_data.clock.cycle_last;
-
-	if (likely(ret >= last))
-		return ret;
-
-	/*
-	 * GCC likes to generate cmov here, but this branch is extremely
-	 * predictable (it's just a function of time and the likely is
-	 * very likely) and there's a data dependence, so force GCC
-	 * to generate a branch instead.  I don't barrier() because
-	 * we don't actually need a barrier, and if this function
-	 * ever gets inlined it will generate worse code.
-	 */
-	asm volatile ("");
-	return last;
-}
-
-static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp,
-			  int *mode)
-{
-	u64 tsc_pg_val;
-	long v;
-
-	switch (clock->vclock_mode) {
-	case VDSO_CLOCKMODE_HVCLOCK:
-		if (hv_read_tsc_page_tsc(hv_get_tsc_page(),
-					 tsc_timestamp, &tsc_pg_val)) {
-			/* TSC page valid */
-			*mode = VDSO_CLOCKMODE_HVCLOCK;
-			v = (tsc_pg_val - clock->cycle_last) &
-				clock->mask;
-		} else {
-			/* TSC page invalid */
-			*mode = VDSO_CLOCKMODE_NONE;
-		}
-		break;
-	case VDSO_CLOCKMODE_TSC:
-		*mode = VDSO_CLOCKMODE_TSC;
-		*tsc_timestamp = read_tsc();
-		v = (*tsc_timestamp - clock->cycle_last) &
-			clock->mask;
-		break;
-	default:
-		*mode = VDSO_CLOCKMODE_NONE;
-	}
-
-	if (*mode == VDSO_CLOCKMODE_NONE)
-		*tsc_timestamp = v = 0;
-
-	return v * clock->mult;
-}
-
-/*
- * As with get_kvmclock_base_ns(), this counts from boot time, at the
- * frequency of CLOCK_MONOTONIC_RAW (hence adding gtos->offs_boot).
- */
-static int do_kvmclock_base(s64 *t, u64 *tsc_timestamp)
-{
-	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
-	unsigned long seq;
-	int mode;
-	u64 ns;
-
-	do {
-		seq = read_seqcount_begin(&gtod->seq);
-		ns = gtod->raw_clock.base_cycles;
-		ns += vgettsc(&gtod->raw_clock, tsc_timestamp, &mode);
-		ns >>= gtod->raw_clock.shift;
-		ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot));
-	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
-	*t = ns;
-
-	return mode;
-}
-
-/*
- * This calculates CLOCK_MONOTONIC at the time of the TSC snapshot, with
- * no boot time offset.
- */
-static int do_monotonic(s64 *t, u64 *tsc_timestamp)
-{
-	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
-	unsigned long seq;
-	int mode;
-	u64 ns;
-
-	do {
-		seq = read_seqcount_begin(&gtod->seq);
-		ns = gtod->clock.base_cycles;
-		ns += vgettsc(&gtod->clock, tsc_timestamp, &mode);
-		ns >>= gtod->clock.shift;
-		ns += ktime_to_ns(gtod->clock.offset);
-	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
-	*t = ns;
-
-	return mode;
-}
-
-static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
-{
-	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
-	unsigned long seq;
-	int mode;
-	u64 ns;
-
-	do {
-		seq = read_seqcount_begin(&gtod->seq);
-		ts->tv_sec = gtod->wall_time_sec;
-		ns = gtod->clock.base_cycles;
-		ns += vgettsc(&gtod->clock, tsc_timestamp, &mode);
-		ns >>= gtod->clock.shift;
-	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
-
-	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
-	ts->tv_nsec = ns;
-
-	return mode;
-}
-
-/*
- * Calculates the kvmclock_base_ns (CLOCK_MONOTONIC_RAW + boot time) and
- * reports the TSC value from which it do so. Returns true if host is
- * using TSC based clocksource.
- */
-static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
-{
-	/* checked again under seqlock below */
-	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
-		return false;
-
-	return gtod_is_based_on_tsc(do_kvmclock_base(kernel_ns,
-						     tsc_timestamp));
-}
-
-/*
- * Calculates CLOCK_MONOTONIC and reports the TSC value from which it did
- * so. Returns true if host is using TSC based clocksource.
- */
-bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
-{
-	/* checked again under seqlock below */
-	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
-		return false;
-
-	return gtod_is_based_on_tsc(do_monotonic(kernel_ns,
-						 tsc_timestamp));
-}
-
-/*
- * Calculates CLOCK_REALTIME and reports the TSC value from which it did
- * so. Returns true if host is using TSC based clocksource.
- *
- * DO NOT USE this for anything related to migration. You want CLOCK_TAI
- * for that.
- */
-static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
-					   u64 *tsc_timestamp)
-{
-	/* checked again under seqlock below */
-	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
-		return false;
-
-	return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp));
-}
-#endif
-
-/*
- *
- * Assuming a stable TSC across physical CPUS, and a stable TSC
- * across virtual CPUs, the following condition is possible.
- * Each numbered line represents an event visible to both
- * CPUs at the next numbered event.
- *
- * "timespecX" represents host monotonic time. "tscX" represents
- * RDTSC value.
- *
- * 		VCPU0 on CPU0		|	VCPU1 on CPU1
- *
- * 1.  read timespec0,tsc0
- * 2.					| timespec1 = timespec0 + N
- * 					| tsc1 = tsc0 + M
- * 3. transition to guest		| transition to guest
- * 4. ret0 = timespec0 + (rdtsc - tsc0) |
- * 5.				        | ret1 = timespec1 + (rdtsc - tsc1)
- * 				        | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
- *
- * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
- *
- * 	- ret0 < ret1
- *	- timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
- *		...
- *	- 0 < N - M => M < N
- *
- * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
- * always the case (the difference between two distinct xtime instances
- * might be smaller then the difference between corresponding TSC reads,
- * when updating guest vcpus pvclock areas).
- *
- * To avoid that problem, do not allow visibility of distinct
- * system_timestamp/tsc_timestamp values simultaneously: use a master
- * copy of host monotonic time values. Update that master copy
- * in lockstep.
- *
- * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
- *
- */
-
-static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
-{
-#ifdef CONFIG_X86_64
-	struct kvm_arch *ka = &kvm->arch;
-	int vclock_mode;
-	bool host_tsc_clocksource, vcpus_matched;
-
-	lockdep_assert_held(&kvm->arch.tsc_write_lock);
-	vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
-			atomic_read(&kvm->online_vcpus));
-
-	/*
-	 * If the host uses TSC clock, then passthrough TSC as stable
-	 * to the guest.
-	 */
-	host_tsc_clocksource = kvm_get_time_and_clockread(
-					&ka->master_kernel_ns,
-					&ka->master_cycle_now);
-
-	ka->use_master_clock = host_tsc_clocksource && vcpus_matched
-				&& !ka->backwards_tsc_observed
-				&& !ka->boot_vcpu_runs_old_kvmclock;
-
-	if (ka->use_master_clock)
-		atomic_set(&kvm_guest_has_master_clock, 1);
-
-	vclock_mode = pvclock_gtod_data.clock.vclock_mode;
-	trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
-					vcpus_matched);
-#endif
-}
-
-static void kvm_make_mclock_inprogress_request(struct kvm *kvm)
-{
-	kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
-}
-
-static void __kvm_start_pvclock_update(struct kvm *kvm)
-{
-	raw_spin_lock_irq(&kvm->arch.tsc_write_lock);
-	write_seqcount_begin(&kvm->arch.pvclock_sc);
-}
-
-static void kvm_start_pvclock_update(struct kvm *kvm)
-{
-	kvm_make_mclock_inprogress_request(kvm);
-
-	/* no guest entries from this point */
-	__kvm_start_pvclock_update(kvm);
-}
-
-static void kvm_end_pvclock_update(struct kvm *kvm)
-{
-	struct kvm_arch *ka = &kvm->arch;
-	struct kvm_vcpu *vcpu;
-	unsigned long i;
-
-	write_seqcount_end(&ka->pvclock_sc);
-	raw_spin_unlock_irq(&ka->tsc_write_lock);
-	kvm_for_each_vcpu(i, vcpu, kvm)
-		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
-
-	/* guest entries allowed */
-	kvm_for_each_vcpu(i, vcpu, kvm)
-		kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
-}
-
-static void kvm_update_masterclock(struct kvm *kvm)
-{
-	kvm_hv_request_tsc_page_update(kvm);
-	kvm_start_pvclock_update(kvm);
-	pvclock_update_vm_gtod_copy(kvm);
-	kvm_end_pvclock_update(kvm);
-}
-
-/*
- * Use the kernel's tsc_khz directly if the TSC is constant, otherwise use KVM's
- * per-CPU value (which may be zero if a CPU is going offline).  Note, tsc_khz
- * can change during boot even if the TSC is constant, as it's possible for KVM
- * to be loaded before TSC calibration completes.  Ideally, KVM would get a
- * notification when calibration completes, but practically speaking calibration
- * will complete before userspace is alive enough to create VMs.
- */
-static unsigned long get_cpu_tsc_khz(void)
-{
-	if (static_cpu_has(X86_FEATURE_CONSTANT_TSC))
-		return tsc_khz;
-	else
-		return __this_cpu_read(cpu_tsc_khz);
-}
-
-/* Called within read_seqcount_begin/retry for kvm->pvclock_sc.  */
-static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
-{
-	struct kvm_arch *ka = &kvm->arch;
-	struct pvclock_vcpu_time_info hv_clock;
-
-	/* both __this_cpu_read() and rdtsc() should be on the same cpu */
-	get_cpu();
-
-	data->flags = 0;
-	if (ka->use_master_clock &&
-	    (static_cpu_has(X86_FEATURE_CONSTANT_TSC) || __this_cpu_read(cpu_tsc_khz))) {
-#ifdef CONFIG_X86_64
-		struct timespec64 ts;
-
-		if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) {
-			data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec;
-			data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC;
-		} else
-#endif
-		data->host_tsc = rdtsc();
-
-		data->flags |= KVM_CLOCK_TSC_STABLE;
-		hv_clock.tsc_timestamp = ka->master_cycle_now;
-		hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
-		kvm_get_time_scale(NSEC_PER_SEC, get_cpu_tsc_khz() * 1000LL,
-				   &hv_clock.tsc_shift,
-				   &hv_clock.tsc_to_system_mul);
-		data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc);
-	} else {
-		data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset;
-	}
-
-	put_cpu();
-}
-
-static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
-{
-	struct kvm_arch *ka = &kvm->arch;
-	unsigned seq;
-
-	do {
-		seq = read_seqcount_begin(&ka->pvclock_sc);
-		__get_kvmclock(kvm, data);
-	} while (read_seqcount_retry(&ka->pvclock_sc, seq));
-}
-
-u64 get_kvmclock_ns(struct kvm *kvm)
-{
-	struct kvm_clock_data data;
-
-	get_kvmclock(kvm, &data);
-	return data.clock;
-}
-
-static void kvm_setup_guest_pvclock(struct pvclock_vcpu_time_info *ref_hv_clock,
-				    struct kvm_vcpu *vcpu,
-				    struct gfn_to_pfn_cache *gpc,
-				    unsigned int offset)
-{
-	struct pvclock_vcpu_time_info *guest_hv_clock;
-	struct pvclock_vcpu_time_info hv_clock;
-	unsigned long flags;
-
-	memcpy(&hv_clock, ref_hv_clock, sizeof(hv_clock));
-
-	read_lock_irqsave(&gpc->lock, flags);
-	while (!kvm_gpc_check(gpc, offset + sizeof(*guest_hv_clock))) {
-		read_unlock_irqrestore(&gpc->lock, flags);
-
-		if (kvm_gpc_refresh(gpc, offset + sizeof(*guest_hv_clock)))
-			return;
-
-		read_lock_irqsave(&gpc->lock, flags);
-	}
-
-	guest_hv_clock = (void *)(gpc->khva + offset);
-
-	/*
-	 * This VCPU is paused, but it's legal for a guest to read another
-	 * VCPU's kvmclock, so we really have to follow the specification where
-	 * it says that version is odd if data is being modified, and even after
-	 * it is consistent.
-	 */
-
-	guest_hv_clock->version = hv_clock.version = (guest_hv_clock->version + 1) | 1;
-	smp_wmb();
-
-	/* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
-	hv_clock.flags |= (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED);
-
-	memcpy(guest_hv_clock, &hv_clock, sizeof(*guest_hv_clock));
-
-	smp_wmb();
-
-	guest_hv_clock->version = ++hv_clock.version;
-
-	kvm_gpc_mark_dirty_in_slot(gpc);
-	read_unlock_irqrestore(&gpc->lock, flags);
-
-	trace_kvm_pvclock_update(vcpu->vcpu_id, &hv_clock);
-}
-
-int kvm_guest_time_update(struct kvm_vcpu *v)
-{
-	struct pvclock_vcpu_time_info hv_clock = {};
-	unsigned long flags, tgt_tsc_khz;
-	unsigned seq;
-	struct kvm_vcpu_arch *vcpu = &v->arch;
-	struct kvm_arch *ka = &v->kvm->arch;
-	s64 kernel_ns;
-	u64 tsc_timestamp, host_tsc;
-	bool use_master_clock;
-
-	kernel_ns = 0;
-	host_tsc = 0;
-
-	/*
-	 * If the host uses TSC clock, then passthrough TSC as stable
-	 * to the guest.
-	 */
-	do {
-		seq = read_seqcount_begin(&ka->pvclock_sc);
-		use_master_clock = ka->use_master_clock;
-		if (use_master_clock) {
-			host_tsc = ka->master_cycle_now;
-			kernel_ns = ka->master_kernel_ns;
-		}
-	} while (read_seqcount_retry(&ka->pvclock_sc, seq));
-
-	/* Keep irq disabled to prevent changes to the clock */
-	local_irq_save(flags);
-	tgt_tsc_khz = get_cpu_tsc_khz();
-	if (unlikely(tgt_tsc_khz == 0)) {
-		local_irq_restore(flags);
-		kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
-		return 1;
-	}
-	if (!use_master_clock) {
-		host_tsc = rdtsc();
-		kernel_ns = get_kvmclock_base_ns();
-	}
-
-	tsc_timestamp = kvm_read_l1_tsc(v, host_tsc);
-
-	/*
-	 * We may have to catch up the TSC to match elapsed wall clock
-	 * time for two reasons, even if kvmclock is used.
-	 *   1) CPU could have been running below the maximum TSC rate
-	 *   2) Broken TSC compensation resets the base at each VCPU
-	 *      entry to avoid unknown leaps of TSC even when running
-	 *      again on the same CPU.  This may cause apparent elapsed
-	 *      time to disappear, and the guest to stand still or run
-	 *	very slowly.
-	 */
-	if (vcpu->tsc_catchup) {
-		u64 tsc = compute_guest_tsc(v, kernel_ns);
-		if (tsc > tsc_timestamp) {
-			adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
-			tsc_timestamp = tsc;
-		}
-	}
-
-	local_irq_restore(flags);
-
-	/* With all the info we got, fill in the values */
-
-	if (kvm_caps.has_tsc_control) {
-		tgt_tsc_khz = kvm_scale_tsc(tgt_tsc_khz,
-					    v->arch.l1_tsc_scaling_ratio);
-		tgt_tsc_khz = tgt_tsc_khz ? : 1;
-	}
-
-	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
-		kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
-				   &vcpu->pvclock_tsc_shift,
-				   &vcpu->pvclock_tsc_mul);
-		vcpu->hw_tsc_khz = tgt_tsc_khz;
-	}
-
-	hv_clock.tsc_shift = vcpu->pvclock_tsc_shift;
-	hv_clock.tsc_to_system_mul = vcpu->pvclock_tsc_mul;
-	hv_clock.tsc_timestamp = tsc_timestamp;
-	hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
-	vcpu->last_guest_tsc = tsc_timestamp;
-
-	/* If the host uses TSC clocksource, then it is stable */
-	hv_clock.flags = 0;
-	if (use_master_clock)
-		hv_clock.flags |= PVCLOCK_TSC_STABLE_BIT;
-
-	if (vcpu->pv_time.active) {
-		/*
-		 * GUEST_STOPPED is only supported by kvmclock, and KVM's
-		 * historic behavior is to only process the request if kvmclock
-		 * is active/enabled.
-		 */
-		if (vcpu->pvclock_set_guest_stopped_request) {
-			hv_clock.flags |= PVCLOCK_GUEST_STOPPED;
-			vcpu->pvclock_set_guest_stopped_request = false;
-		}
-		kvm_setup_guest_pvclock(&hv_clock, v, &vcpu->pv_time, 0);
-
-		hv_clock.flags &= ~PVCLOCK_GUEST_STOPPED;
-	}
-
-	kvm_hv_setup_tsc_page(v->kvm, &hv_clock);
-
-#ifdef CONFIG_KVM_XEN
-	/*
-	 * For Xen guests we may need to override PVCLOCK_TSC_STABLE_BIT as unless
-	 * explicitly told to use TSC as its clocksource Xen will not set this bit.
-	 * This default behaviour led to bugs in some guest kernels which cause
-	 * problems if they observe PVCLOCK_TSC_STABLE_BIT in the pvclock flags.
-	 *
-	 * Note!  Clear TSC_STABLE only for Xen clocks, i.e. the order matters!
-	 */
-	if (ka->xen.hvm_config.flags & KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE)
-		hv_clock.flags &= ~PVCLOCK_TSC_STABLE_BIT;
-
-	if (vcpu->xen.vcpu_info_cache.active)
-		kvm_setup_guest_pvclock(&hv_clock, v, &vcpu->xen.vcpu_info_cache,
-					offsetof(struct compat_vcpu_info, time));
-	if (vcpu->xen.vcpu_time_info_cache.active)
-		kvm_setup_guest_pvclock(&hv_clock, v, &vcpu->xen.vcpu_time_info_cache, 0);
-#endif
-	return 0;
-}
-
-/*
- * The pvclock_wall_clock ABI tells the guest the wall clock time at
- * which it started (i.e. its epoch, when its kvmclock was zero).
- *
- * In fact those clocks are subtly different; wall clock frequency is
- * adjusted by NTP and has leap seconds, while the kvmclock is a
- * simple function of the TSC without any such adjustment.
- *
- * Perhaps the ABI should have exposed CLOCK_TAI and a ratio between
- * that and kvmclock, but even that would be subject to change over
- * time.
- *
- * Attempt to calculate the epoch at a given moment using the *same*
- * TSC reading via kvm_get_walltime_and_clockread() to obtain both
- * wallclock and kvmclock times, and subtracting one from the other.
- *
- * Fall back to using their values at slightly different moments by
- * calling ktime_get_real_ns() and get_kvmclock_ns() separately.
- */
-uint64_t kvm_get_wall_clock_epoch(struct kvm *kvm)
-{
-#ifdef CONFIG_X86_64
-	struct pvclock_vcpu_time_info hv_clock;
-	struct kvm_arch *ka = &kvm->arch;
-	unsigned long seq, local_tsc_khz;
-	struct timespec64 ts;
-	uint64_t host_tsc;
-
-	do {
-		seq = read_seqcount_begin(&ka->pvclock_sc);
-
-		local_tsc_khz = 0;
-		if (!ka->use_master_clock)
-			break;
-
-		/*
-		 * The TSC read and the call to get_cpu_tsc_khz() must happen
-		 * on the same CPU.
-		 */
-		get_cpu();
-
-		local_tsc_khz = get_cpu_tsc_khz();
-
-		if (local_tsc_khz &&
-		    !kvm_get_walltime_and_clockread(&ts, &host_tsc))
-			local_tsc_khz = 0; /* Fall back to old method */
-
-		put_cpu();
-
-		/*
-		 * These values must be snapshotted within the seqcount loop.
-		 * After that, it's just mathematics which can happen on any
-		 * CPU at any time.
-		 */
-		hv_clock.tsc_timestamp = ka->master_cycle_now;
-		hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
-
-	} while (read_seqcount_retry(&ka->pvclock_sc, seq));
-
-	/*
-	 * If the conditions were right, and obtaining the wallclock+TSC was
-	 * successful, calculate the KVM clock at the corresponding time and
-	 * subtract one from the other to get the guest's epoch in nanoseconds
-	 * since 1970-01-01.
-	 */
-	if (local_tsc_khz) {
-		kvm_get_time_scale(NSEC_PER_SEC, local_tsc_khz * NSEC_PER_USEC,
-				   &hv_clock.tsc_shift,
-				   &hv_clock.tsc_to_system_mul);
-		return ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec -
-			__pvclock_read_cycles(&hv_clock, host_tsc);
-	}
-#endif
-	return ktime_get_real_ns() - get_kvmclock_ns(kvm);
-}
-
-/*
- * kvmclock updates which are isolated to a given vcpu, such as
- * vcpu->cpu migration, should not allow system_timestamp from
- * the rest of the vcpus to remain static.
- *
- * So in those cases, request a kvmclock update for all vcpus.
- * The worst case for a remote vcpu to update its kvmclock
- * is then bounded by maximum nohz sleep latency.
- */
-static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
-{
-	unsigned long i;
-	struct kvm_vcpu *vcpu;
-	struct kvm *kvm = v->kvm;
-
-	kvm_for_each_vcpu(i, vcpu, kvm) {
-		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
-		kvm_vcpu_kick(vcpu);
-	}
-}
-
-/* These helpers are safe iff @msr is known to be an MCx bank MSR. */
-static bool is_mci_control_msr(u32 msr)
-{
-	return (msr & 3) == 0;
-}
-static bool is_mci_status_msr(u32 msr)
-{
-	return (msr & 3) == 1;
-}
-
-/*
- * On AMD, HWCR[McStatusWrEn] controls whether setting MCi_STATUS results in #GP.
- */
-static bool can_set_mci_status(struct kvm_vcpu *vcpu)
-{
-	/* McStatusWrEn enabled? */
-	if (guest_cpuid_is_amd_compatible(vcpu))
-		return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));
-
-	return false;
-}
-
-static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
-{
-	u64 mcg_cap = vcpu->arch.mcg_cap;
-	unsigned bank_num = mcg_cap & 0xff;
-	u32 msr = msr_info->index;
-	u64 data = msr_info->data;
-	u32 offset, last_msr;
-
-	switch (msr) {
-	case MSR_IA32_MCG_STATUS:
-		vcpu->arch.mcg_status = data;
-		break;
-	case MSR_IA32_MCG_CTL:
-		if (!(mcg_cap & MCG_CTL_P) &&
-		    (data || !msr_info->host_initiated))
-			return 1;
-		if (data != 0 && data != ~(u64)0)
-			return 1;
-		vcpu->arch.mcg_ctl = data;
-		break;
-	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
-		last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1;
-		if (msr > last_msr)
-			return 1;
-
-		if (!(mcg_cap & MCG_CMCI_P) && (data || !msr_info->host_initiated))
-			return 1;
-		/* An attempt to write a 1 to a reserved bit raises #GP */
-		if (data & ~(MCI_CTL2_CMCI_EN | MCI_CTL2_CMCI_THRESHOLD_MASK))
-			return 1;
-		offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2,
-					    last_msr + 1 - MSR_IA32_MC0_CTL2);
-		vcpu->arch.mci_ctl2_banks[offset] = data;
-		break;
-	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
-		last_msr = MSR_IA32_MCx_CTL(bank_num) - 1;
-		if (msr > last_msr)
-			return 1;
-
-		/*
-		 * Only 0 or all 1s can be written to IA32_MCi_CTL, all other
-		 * values are architecturally undefined.  But, some Linux
-		 * kernels clear bit 10 in bank 4 to workaround a BIOS/GART TLB
-		 * issue on AMD K8s, allow bit 10 to be clear when setting all
-		 * other bits in order to avoid an uncaught #GP in the guest.
-		 *
-		 * UNIXWARE clears bit 0 of MC1_CTL to ignore correctable,
-		 * single-bit ECC data errors.
-		 */
-		if (is_mci_control_msr(msr) &&
-		    data != 0 && (data | (1 << 10) | 1) != ~(u64)0)
-			return 1;
-
-		/*
-		 * All CPUs allow writing 0 to MCi_STATUS MSRs to clear the MSR.
-		 * AMD-based CPUs allow non-zero values, but if and only if
-		 * HWCR[McStatusWrEn] is set.
-		 */
-		if (!msr_info->host_initiated && is_mci_status_msr(msr) &&
-		    data != 0 && !can_set_mci_status(vcpu))
-			return 1;
-
-		offset = array_index_nospec(msr - MSR_IA32_MC0_CTL,
-					    last_msr + 1 - MSR_IA32_MC0_CTL);
-		vcpu->arch.mce_banks[offset] = data;
-		break;
-	default:
-		return 1;
-	}
-	return 0;
-}
-
-static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
-{
-	gpa_t gpa = data & ~0x3f;
-
-	/* Bits 4:5 are reserved, Should be zero */
-	if (data & 0x30)
-		return 1;
-
-	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) &&
-	    (data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT))
-		return 1;
-
-	if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) &&
-	    (data & KVM_ASYNC_PF_DELIVERY_AS_INT))
-		return 1;
-
-	if (!lapic_in_kernel(vcpu))
-		return data ? 1 : 0;
-
-	if (__kvm_pv_async_pf_enabled(data) &&
-	    kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
-				      sizeof(u64)))
-		return 1;
-
-	vcpu->arch.apf.msr_en_val = data;
-
-	if (__kvm_pv_async_pf_enabled(data)) {
-		kvm_async_pf_wakeup_all(vcpu);
-	} else {
-		kvm_clear_async_pf_completion_queue(vcpu);
-		kvm_async_pf_hash_reset(vcpu);
-	}
-	return 0;
-}
-
-static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data)
-{
-	/* Bits 8-63 are reserved */
-	if (data >> 8)
-		return 1;
-
-	if (!lapic_in_kernel(vcpu))
-		return 1;
-
-	vcpu->arch.apf.msr_int_val = data;
-
-	vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK;
-
-	return 0;
-}
-
-static void kvmclock_reset(struct kvm_vcpu *vcpu)
-{
-	kvm_gpc_deactivate(&vcpu->arch.pv_time);
-	vcpu->arch.time = 0;
-}
-
-static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)
-{
-	++vcpu->stat.tlb_flush;
-	kvm_x86_call(flush_tlb_all)(vcpu);
-
-	/* Flushing all ASIDs flushes the current ASID... */
-	kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
-}
-
-static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)
-{
-	++vcpu->stat.tlb_flush;
-
-	if (!tdp_enabled) {
-		/*
-		 * A TLB flush on behalf of the guest is equivalent to
-		 * INVPCID(all), toggling CR4.PGE, etc., which requires
-		 * a forced sync of the shadow page tables.  Ensure all the
-		 * roots are synced and the guest TLB in hardware is clean.
-		 */
-		kvm_mmu_sync_roots(vcpu);
-		kvm_mmu_sync_prev_roots(vcpu);
-	}
-
-	kvm_x86_call(flush_tlb_guest)(vcpu);
-
-	/*
-	 * Flushing all "guest" TLB is always a superset of Hyper-V's fine
-	 * grained flushing.
-	 */
-	kvm_hv_vcpu_purge_flush_tlb(vcpu);
-}
-
-
-static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu)
-{
-	++vcpu->stat.tlb_flush;
-	kvm_x86_call(flush_tlb_current)(vcpu);
-}
-
-/*
- * Service "local" TLB flush requests, which are specific to the current MMU
- * context.  In addition to the generic event handling in vcpu_enter_guest(),
- * TLB flushes that are targeted at an MMU context also need to be serviced
- * prior before nested VM-Enter/VM-Exit.
- */
-void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu)
-{
-	if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
-		kvm_vcpu_flush_tlb_current(vcpu);
-
-	if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu))
-		kvm_vcpu_flush_tlb_guest(vcpu);
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_service_local_tlb_flush_requests);
-
-static void record_steal_time(struct kvm_vcpu *vcpu)
-{
-	struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
-	struct kvm_steal_time __user *st;
-	struct kvm_memslots *slots;
-	gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;
-	u64 steal;
-	u32 version;
-
-	if (kvm_xen_msr_enabled(vcpu->kvm)) {
-		kvm_xen_runstate_set_running(vcpu);
-		return;
-	}
-
-	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
-		return;
-
-	if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm))
-		return;
-
-	slots = kvm_memslots(vcpu->kvm);
-
-	if (unlikely(slots->generation != ghc->generation ||
-		     gpa != ghc->gpa ||
-		     kvm_is_error_hva(ghc->hva) || !ghc->memslot)) {
-		/* We rely on the fact that it fits in a single page. */
-		BUILD_BUG_ON((sizeof(*st) - 1) & KVM_STEAL_VALID_BITS);
-
-		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st)) ||
-		    kvm_is_error_hva(ghc->hva) || !ghc->memslot)
-			return;
-	}
-
-	st = (struct kvm_steal_time __user *)ghc->hva;
-	/*
-	 * Doing a TLB flush here, on the guest's behalf, can avoid
-	 * expensive IPIs.
-	 */
-	if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) {
-		u8 st_preempted = 0;
-		int err = -EFAULT;
-
-		if (!user_access_begin(st, sizeof(*st)))
-			return;
-
-		asm volatile("1: xchgb %0, %2\n"
-			     "xor %1, %1\n"
-			     "2:\n"
-			     _ASM_EXTABLE_UA(1b, 2b)
-			     : "+q" (st_preempted),
-			       "+&r" (err),
-			       "+m" (st->preempted));
-		if (err)
-			goto out;
-
-		user_access_end();
-
-		vcpu->arch.st.preempted = 0;
-
-		trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
-				       st_preempted & KVM_VCPU_FLUSH_TLB);
-		if (st_preempted & KVM_VCPU_FLUSH_TLB)
-			kvm_vcpu_flush_tlb_guest(vcpu);
-
-		if (!user_access_begin(st, sizeof(*st)))
-			goto dirty;
-	} else {
-		if (!user_access_begin(st, sizeof(*st)))
-			return;
-
-		unsafe_put_user(0, &st->preempted, out);
-		vcpu->arch.st.preempted = 0;
-	}
-
-	unsafe_get_user(version, &st->version, out);
-	if (version & 1)
-		version += 1;  /* first time write, random junk */
-
-	version += 1;
-	unsafe_put_user(version, &st->version, out);
-
-	smp_wmb();
-
-	unsafe_get_user(steal, &st->steal, out);
-	steal += current->sched_info.run_delay -
-		vcpu->arch.st.last_steal;
-	vcpu->arch.st.last_steal = current->sched_info.run_delay;
-	unsafe_put_user(steal, &st->steal, out);
-
-	version += 1;
-	unsafe_put_user(version, &st->version, out);
-
- out:
-	user_access_end();
- dirty:
-	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
-}
-
-/*
- * Returns true if the MSR in question is managed via XSTATE, i.e. is context
- * switched with the rest of guest FPU state.
- *
- * Note, S_CET is _not_ saved/restored via XSAVES/XRSTORS.
- */
-static bool is_xstate_managed_msr(struct kvm_vcpu *vcpu, u32 msr)
-{
-	if (!vcpu)
-		return false;
-
-	switch (msr) {
-	case MSR_IA32_U_CET:
-		return guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) ||
-		       guest_cpu_cap_has(vcpu, X86_FEATURE_IBT);
-	case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP:
-		return guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK);
-	default:
-		return false;
-	}
-}
-
-/*
- * Lock (and if necessary, re-load) the guest FPU, i.e. XSTATE, and access an
- * MSR that is managed via XSTATE.  Note, the caller is responsible for doing
- * the initial FPU load, this helper only ensures that guest state is resident
- * in hardware (the kernel can load its FPU state in IRQ context).
- *
- * Note, loading guest values for U_CET and PL[0-3]_SSP while executing in the
- * kernel is safe, as U_CET is specific to userspace, and PL[0-3]_SSP are only
- * consumed when transitioning to lower privilege levels, i.e. are effectively
- * only consumed by userspace as well.
- */
-static __always_inline void kvm_access_xstate_msr(struct kvm_vcpu *vcpu,
-						  struct msr_data *msr_info,
-						  int access)
-{
-	BUILD_BUG_ON(access != MSR_TYPE_R && access != MSR_TYPE_W);
-
-	KVM_BUG_ON(!is_xstate_managed_msr(vcpu, msr_info->index), vcpu->kvm);
-	KVM_BUG_ON(!vcpu->arch.guest_fpu.fpstate->in_use, vcpu->kvm);
-
-	kvm_fpu_get();
-	if (access == MSR_TYPE_R)
-		rdmsrq(msr_info->index, msr_info->data);
-	else
-		wrmsrq(msr_info->index, msr_info->data);
-	kvm_fpu_put();
-}
-
-static void kvm_set_xstate_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
-{
-	kvm_access_xstate_msr(vcpu, msr_info, MSR_TYPE_W);
-}
-
-static void kvm_get_xstate_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
-{
-	kvm_access_xstate_msr(vcpu, msr_info, MSR_TYPE_R);
-}
-
-int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
-{
-	u32 msr = msr_info->index;
-	u64 data = msr_info->data;
+	tsc_timestamp = kvm_read_l1_tsc(v, host_tsc);
 
 	/*
-	 * Do not allow host-initiated writes to trigger the Xen hypercall
-	 * page setup; it could incur locking paths which are not expected
-	 * if userspace sets the MSR in an unusual location.
+	 * We may have to catch up the TSC to match elapsed wall clock
+	 * time for two reasons, even if kvmclock is used.
+	 *   1) CPU could have been running below the maximum TSC rate
+	 *   2) Broken TSC compensation resets the base at each VCPU
+	 *      entry to avoid unknown leaps of TSC even when running
+	 *      again on the same CPU.  This may cause apparent elapsed
+	 *      time to disappear, and the guest to stand still or run
+	 *	very slowly.
 	 */
-	if (kvm_xen_is_hypercall_page_msr(vcpu->kvm, msr) &&
-	    !msr_info->host_initiated)
-		return kvm_xen_write_hypercall_page(vcpu, data);
-
-	switch (msr) {
-	case MSR_AMD64_NB_CFG:
-	case MSR_IA32_UCODE_WRITE:
-	case MSR_VM_HSAVE_PA:
-	case MSR_AMD64_PATCH_LOADER:
-	case MSR_AMD64_BU_CFG2:
-	case MSR_AMD64_DC_CFG:
-	case MSR_AMD64_TW_CFG:
-	case MSR_F15H_EX_CFG:
-		break;
-
-	case MSR_IA32_UCODE_REV:
-		if (msr_info->host_initiated)
-			vcpu->arch.microcode_version = data;
-		break;
-	case MSR_IA32_ARCH_CAPABILITIES:
-		if (!msr_info->host_initiated ||
-		    !guest_cpu_cap_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
-			return KVM_MSR_RET_UNSUPPORTED;
-		vcpu->arch.arch_capabilities = data;
-		break;
-	case MSR_IA32_PERF_CAPABILITIES:
-		if (!msr_info->host_initiated ||
-		    !guest_cpu_cap_has(vcpu, X86_FEATURE_PDCM))
-			return KVM_MSR_RET_UNSUPPORTED;
-
-		if (data & ~kvm_caps.supported_perf_cap)
-			return 1;
-
-		/*
-		 * Note, this is not just a performance optimization!  KVM
-		 * disallows changing feature MSRs after the vCPU has run; PMU
-		 * refresh will bug the VM if called after the vCPU has run.
-		 */
-		if (vcpu->arch.perf_capabilities == data)
-			break;
-
-		vcpu->arch.perf_capabilities = data;
-		kvm_pmu_refresh(vcpu);
-		kvm_make_request(KVM_REQ_RECALC_INTERCEPTS, vcpu);
-		break;
-	case MSR_IA32_PRED_CMD: {
-		u64 reserved_bits = ~(PRED_CMD_IBPB | PRED_CMD_SBPB);
-
-		if (!msr_info->host_initiated) {
-			if ((!guest_has_pred_cmd_msr(vcpu)))
-				return 1;
-
-			if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SPEC_CTRL) &&
-			    !guest_cpu_cap_has(vcpu, X86_FEATURE_AMD_IBPB))
-				reserved_bits |= PRED_CMD_IBPB;
-
-			if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SBPB))
-				reserved_bits |= PRED_CMD_SBPB;
+	if (vcpu->tsc_catchup) {
+		u64 tsc = compute_guest_tsc(v, kernel_ns);
+		if (tsc > tsc_timestamp) {
+			adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
+			tsc_timestamp = tsc;
 		}
-
-		if (!boot_cpu_has(X86_FEATURE_IBPB))
-			reserved_bits |= PRED_CMD_IBPB;
-
-		if (!boot_cpu_has(X86_FEATURE_SBPB))
-			reserved_bits |= PRED_CMD_SBPB;
-
-		if (data & reserved_bits)
-			return 1;
-
-		if (!data)
-			break;
-
-		wrmsrq(MSR_IA32_PRED_CMD, data);
-		break;
 	}
-	case MSR_IA32_FLUSH_CMD:
-		if (!msr_info->host_initiated &&
-		    !guest_cpu_cap_has(vcpu, X86_FEATURE_FLUSH_L1D))
-			return 1;
-
-		if (!boot_cpu_has(X86_FEATURE_FLUSH_L1D) || (data & ~L1D_FLUSH))
-			return 1;
-		if (!data)
-			break;
 
-		wrmsrq(MSR_IA32_FLUSH_CMD, L1D_FLUSH);
-		break;
-	case MSR_EFER:
-		return set_efer(vcpu, msr_info);
-	case MSR_K7_HWCR: {
-		/*
-		 * Allow McStatusWrEn and TscFreqSel. (Linux guests from v3.2
-		 * through at least v6.6 whine if TscFreqSel is clear,
-		 * depending on F/M/S.
-		 */
-		u64 valid = BIT_ULL(18) | BIT_ULL(24);
+	local_irq_restore(flags);
 
-		data &= ~(u64)0x40;	/* ignore flush filter disable */
-		data &= ~(u64)0x100;	/* ignore ignne emulation enable */
-		data &= ~(u64)0x8;	/* ignore TLB cache disable */
+	/* With all the info we got, fill in the values */
 
-		if (guest_cpu_cap_has(vcpu, X86_FEATURE_GP_ON_USER_CPUID))
-			valid |= MSR_K7_HWCR_CPUID_USER_DIS;
+	if (kvm_caps.has_tsc_control) {
+		tgt_tsc_khz = kvm_scale_tsc(tgt_tsc_khz,
+					    v->arch.l1_tsc_scaling_ratio);
+		tgt_tsc_khz = tgt_tsc_khz ? : 1;
+	}
 
-		if (data & ~valid) {
-			kvm_pr_unimpl_wrmsr(vcpu, msr, data);
-			return 1;
-		}
-		vcpu->arch.msr_hwcr = data;
-		break;
+	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
+		kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
+				   &vcpu->pvclock_tsc_shift,
+				   &vcpu->pvclock_tsc_mul);
+		vcpu->hw_tsc_khz = tgt_tsc_khz;
 	}
-	case MSR_FAM10H_MMIO_CONF_BASE:
-		if (data != 0) {
-			kvm_pr_unimpl_wrmsr(vcpu, msr, data);
-			return 1;
-		}
-		break;
-	case MSR_IA32_CR_PAT:
-		if (!kvm_pat_valid(data))
-			return 1;
 
-		vcpu->arch.pat = data;
-		break;
-	case MTRRphysBase_MSR(0) ... MSR_MTRRfix4K_F8000:
-	case MSR_MTRRdefType:
-		return kvm_mtrr_set_msr(vcpu, msr, data);
-	case MSR_IA32_APICBASE:
-		return kvm_apic_set_base(vcpu, data, msr_info->host_initiated);
-	case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff:
-		return kvm_x2apic_msr_write(vcpu, msr, data);
-	case MSR_IA32_TSC_DEADLINE:
-		kvm_set_lapic_tscdeadline_msr(vcpu, data);
-		break;
-	case MSR_IA32_TSC_ADJUST:
-		if (guest_cpu_cap_has(vcpu, X86_FEATURE_TSC_ADJUST)) {
-			if (!msr_info->host_initiated) {
-				s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
-				adjust_tsc_offset_guest(vcpu, adj);
-				/* Before back to guest, tsc_timestamp must be adjusted
-				 * as well, otherwise guest's percpu pvclock time could jump.
-				 */
-				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
-			}
-			vcpu->arch.ia32_tsc_adjust_msr = data;
-		}
-		break;
-	case MSR_IA32_MISC_ENABLE: {
-		u64 old_val = vcpu->arch.ia32_misc_enable_msr;
+	hv_clock.tsc_shift = vcpu->pvclock_tsc_shift;
+	hv_clock.tsc_to_system_mul = vcpu->pvclock_tsc_mul;
+	hv_clock.tsc_timestamp = tsc_timestamp;
+	hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
+	vcpu->last_guest_tsc = tsc_timestamp;
 
-		if (!msr_info->host_initiated) {
-			/* RO bits */
-			if ((old_val ^ data) & MSR_IA32_MISC_ENABLE_PMU_RO_MASK)
-				return 1;
+	/* If the host uses TSC clocksource, then it is stable */
+	hv_clock.flags = 0;
+	if (use_master_clock)
+		hv_clock.flags |= PVCLOCK_TSC_STABLE_BIT;
 
-			/* R bits, i.e. writes are ignored, but don't fault. */
-			data = data & ~MSR_IA32_MISC_ENABLE_EMON;
-			data |= old_val & MSR_IA32_MISC_ENABLE_EMON;
+	if (vcpu->pv_time.active) {
+		/*
+		 * GUEST_STOPPED is only supported by kvmclock, and KVM's
+		 * historic behavior is to only process the request if kvmclock
+		 * is active/enabled.
+		 */
+		if (vcpu->pvclock_set_guest_stopped_request) {
+			hv_clock.flags |= PVCLOCK_GUEST_STOPPED;
+			vcpu->pvclock_set_guest_stopped_request = false;
 		}
+		kvm_setup_guest_pvclock(&hv_clock, v, &vcpu->pv_time, 0);
 
-		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
-		    ((old_val ^ data)  & MSR_IA32_MISC_ENABLE_MWAIT)) {
-			if (!guest_cpu_cap_has(vcpu, X86_FEATURE_XMM3))
-				return 1;
-			vcpu->arch.ia32_misc_enable_msr = data;
-			vcpu->arch.cpuid_dynamic_bits_dirty = true;
-		} else {
-			vcpu->arch.ia32_misc_enable_msr = data;
-		}
-		break;
+		hv_clock.flags &= ~PVCLOCK_GUEST_STOPPED;
 	}
-	case MSR_IA32_SMBASE:
-		if (!IS_ENABLED(CONFIG_KVM_SMM) || !msr_info->host_initiated)
-			return 1;
-		vcpu->arch.smbase = data;
-		break;
-	case MSR_IA32_POWER_CTL:
-		vcpu->arch.msr_ia32_power_ctl = data;
-		break;
-	case MSR_IA32_TSC:
-		if (msr_info->host_initiated) {
-			kvm_synchronize_tsc(vcpu, &data);
-		} else if (!vcpu->arch.guest_tsc_protected) {
-			u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;
-			adjust_tsc_offset_guest(vcpu, adj);
-			vcpu->arch.ia32_tsc_adjust_msr += adj;
-		}
-		break;
-	case MSR_IA32_XSS:
-		if (!guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
-			return KVM_MSR_RET_UNSUPPORTED;
-
-		if (data & ~vcpu->arch.guest_supported_xss)
-			return 1;
-		if (vcpu->arch.ia32_xss == data)
-			break;
-		vcpu->arch.ia32_xss = data;
-		vcpu->arch.cpuid_dynamic_bits_dirty = true;
-		break;
-	case MSR_SMI_COUNT:
-		if (!msr_info->host_initiated)
-			return 1;
-		vcpu->arch.smi_count = data;
-		break;
-	case MSR_KVM_WALL_CLOCK_NEW:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
-			return KVM_MSR_RET_UNSUPPORTED;
-
-		vcpu->kvm->arch.wall_clock = data;
-		kvm_write_wall_clock(vcpu->kvm, data, 0);
-		break;
-	case MSR_KVM_WALL_CLOCK:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
-			return KVM_MSR_RET_UNSUPPORTED;
-
-		vcpu->kvm->arch.wall_clock = data;
-		kvm_write_wall_clock(vcpu->kvm, data, 0);
-		break;
-	case MSR_KVM_SYSTEM_TIME_NEW:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
-			return KVM_MSR_RET_UNSUPPORTED;
-
-		kvm_write_system_time(vcpu, data, false, msr_info->host_initiated);
-		break;
-	case MSR_KVM_SYSTEM_TIME:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
-			return KVM_MSR_RET_UNSUPPORTED;
-
-		kvm_write_system_time(vcpu, data, true,  msr_info->host_initiated);
-		break;
-	case MSR_KVM_ASYNC_PF_EN:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
-			return KVM_MSR_RET_UNSUPPORTED;
-
-		if (kvm_pv_enable_async_pf(vcpu, data))
-			return 1;
-		break;
-	case MSR_KVM_ASYNC_PF_INT:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
-			return KVM_MSR_RET_UNSUPPORTED;
 
-		if (kvm_pv_enable_async_pf_int(vcpu, data))
-			return 1;
-		break;
-	case MSR_KVM_ASYNC_PF_ACK:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
-			return KVM_MSR_RET_UNSUPPORTED;
-		if (data & 0x1) {
-			/*
-			 * Pairs with the smp_mb__after_atomic() in
-			 * kvm_arch_async_page_present_queued().
-			 */
-			smp_store_mb(vcpu->arch.apf.pageready_pending, false);
+	kvm_hv_setup_tsc_page(v->kvm, &hv_clock);
 
-			kvm_check_async_pf_completion(vcpu);
-		}
-		break;
-	case MSR_KVM_STEAL_TIME:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
-			return KVM_MSR_RET_UNSUPPORTED;
+#ifdef CONFIG_KVM_XEN
+	/*
+	 * For Xen guests we may need to override PVCLOCK_TSC_STABLE_BIT as unless
+	 * explicitly told to use TSC as its clocksource Xen will not set this bit.
+	 * This default behaviour led to bugs in some guest kernels which cause
+	 * problems if they observe PVCLOCK_TSC_STABLE_BIT in the pvclock flags.
+	 *
+	 * Note!  Clear TSC_STABLE only for Xen clocks, i.e. the order matters!
+	 */
+	if (ka->xen.hvm_config.flags & KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE)
+		hv_clock.flags &= ~PVCLOCK_TSC_STABLE_BIT;
 
-		if (unlikely(!sched_info_on()))
-			return 1;
+	if (vcpu->xen.vcpu_info_cache.active)
+		kvm_setup_guest_pvclock(&hv_clock, v, &vcpu->xen.vcpu_info_cache,
+					offsetof(struct compat_vcpu_info, time));
+	if (vcpu->xen.vcpu_time_info_cache.active)
+		kvm_setup_guest_pvclock(&hv_clock, v, &vcpu->xen.vcpu_time_info_cache, 0);
+#endif
+	return 0;
+}
 
-		if (data & KVM_STEAL_RESERVED_MASK)
-			return 1;
+/*
+ * The pvclock_wall_clock ABI tells the guest the wall clock time at
+ * which it started (i.e. its epoch, when its kvmclock was zero).
+ *
+ * In fact those clocks are subtly different; wall clock frequency is
+ * adjusted by NTP and has leap seconds, while the kvmclock is a
+ * simple function of the TSC without any such adjustment.
+ *
+ * Perhaps the ABI should have exposed CLOCK_TAI and a ratio between
+ * that and kvmclock, but even that would be subject to change over
+ * time.
+ *
+ * Attempt to calculate the epoch at a given moment using the *same*
+ * TSC reading via kvm_get_walltime_and_clockread() to obtain both
+ * wallclock and kvmclock times, and subtracting one from the other.
+ *
+ * Fall back to using their values at slightly different moments by
+ * calling ktime_get_real_ns() and get_kvmclock_ns() separately.
+ */
+uint64_t kvm_get_wall_clock_epoch(struct kvm *kvm)
+{
+#ifdef CONFIG_X86_64
+	struct pvclock_vcpu_time_info hv_clock;
+	struct kvm_arch *ka = &kvm->arch;
+	unsigned long seq, local_tsc_khz;
+	struct timespec64 ts;
+	uint64_t host_tsc;
 
-		vcpu->arch.st.msr_val = data;
+	do {
+		seq = read_seqcount_begin(&ka->pvclock_sc);
 
-		if (!(data & KVM_MSR_ENABLED))
+		local_tsc_khz = 0;
+		if (!ka->use_master_clock)
 			break;
 
-		kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
-
-		break;
-	case MSR_KVM_PV_EOI_EN:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
-			return KVM_MSR_RET_UNSUPPORTED;
-
-		if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8)))
-			return 1;
-		break;
-
-	case MSR_KVM_POLL_CONTROL:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
-			return KVM_MSR_RET_UNSUPPORTED;
-
-		/* only enable bit supported */
-		if (data & (-1ULL << 1))
-			return 1;
+		/*
+		 * The TSC read and the call to get_cpu_tsc_khz() must happen
+		 * on the same CPU.
+		 */
+		get_cpu();
 
-		vcpu->arch.msr_kvm_poll_control = data;
-		break;
+		local_tsc_khz = get_cpu_tsc_khz();
 
-	case MSR_IA32_MCG_CTL:
-	case MSR_IA32_MCG_STATUS:
-	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
-	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
-		return set_msr_mce(vcpu, msr_info);
+		if (local_tsc_khz &&
+		    !kvm_get_walltime_and_clockread(&ts, &host_tsc))
+			local_tsc_khz = 0; /* Fall back to old method */
 
-	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
-	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
-	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
-	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
-		if (kvm_pmu_is_valid_msr(vcpu, msr))
-			return kvm_pmu_set_msr(vcpu, msr_info);
+		put_cpu();
 
-		if (data)
-			kvm_pr_unimpl_wrmsr(vcpu, msr, data);
-		break;
-	case MSR_K7_CLK_CTL:
 		/*
-		 * Ignore all writes to this no longer documented MSR.
-		 * Writes are only relevant for old K7 processors,
-		 * all pre-dating SVM, but a recommended workaround from
-		 * AMD for these chips. It is possible to specify the
-		 * affected processor models on the command line, hence
-		 * the need to ignore the workaround.
-		 */
-		break;
-#ifdef CONFIG_KVM_HYPERV
-	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
-	case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER:
-	case HV_X64_MSR_SYNDBG_OPTIONS:
-	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
-	case HV_X64_MSR_CRASH_CTL:
-	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
-	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
-	case HV_X64_MSR_TSC_EMULATION_CONTROL:
-	case HV_X64_MSR_TSC_EMULATION_STATUS:
-	case HV_X64_MSR_TSC_INVARIANT_CONTROL:
-		return kvm_hv_set_msr_common(vcpu, msr, data,
-					     msr_info->host_initiated);
-#endif
-	case MSR_IA32_BBL_CR_CTL3:
-		/* Drop writes to this legacy MSR -- see rdmsr
-		 * counterpart for further detail.
+		 * These values must be snapshotted within the seqcount loop.
+		 * After that, it's just mathematics which can happen on any
+		 * CPU at any time.
 		 */
-		kvm_pr_unimpl_wrmsr(vcpu, msr, data);
-		break;
-	case MSR_AMD64_OSVW_ID_LENGTH:
-		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW))
-			return 1;
-		vcpu->arch.osvw.length = data;
-		break;
-	case MSR_AMD64_OSVW_STATUS:
-		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW))
-			return 1;
-		vcpu->arch.osvw.status = data;
-		break;
-	case MSR_PLATFORM_INFO:
-		if (!msr_info->host_initiated)
-			return 1;
-		vcpu->arch.msr_platform_info = data;
-		break;
-	case MSR_MISC_FEATURES_ENABLES:
-		if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT ||
-		    (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
-		     !(vcpu->arch.msr_platform_info & MSR_PLATFORM_INFO_CPUID_FAULT)))
-			return 1;
-		vcpu->arch.msr_misc_features_enables = data;
-		break;
-#ifdef CONFIG_X86_64
-	case MSR_IA32_XFD:
-		if (!msr_info->host_initiated &&
-		    !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD))
-			return 1;
-
-		if (data & ~kvm_guest_supported_xfd(vcpu))
-			return 1;
-
-		fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data);
-		break;
-	case MSR_IA32_XFD_ERR:
-		if (!msr_info->host_initiated &&
-		    !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD))
-			return 1;
+		hv_clock.tsc_timestamp = ka->master_cycle_now;
+		hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
 
-		if (data & ~kvm_guest_supported_xfd(vcpu))
-			return 1;
+	} while (read_seqcount_retry(&ka->pvclock_sc, seq));
 
-		vcpu->arch.guest_fpu.xfd_err = data;
-		break;
+	/*
+	 * If the conditions were right, and obtaining the wallclock+TSC was
+	 * successful, calculate the KVM clock at the corresponding time and
+	 * subtract one from the other to get the guest's epoch in nanoseconds
+	 * since 1970-01-01.
+	 */
+	if (local_tsc_khz) {
+		kvm_get_time_scale(NSEC_PER_SEC, local_tsc_khz * NSEC_PER_USEC,
+				   &hv_clock.tsc_shift,
+				   &hv_clock.tsc_to_system_mul);
+		return ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec -
+			__pvclock_read_cycles(&hv_clock, host_tsc);
+	}
 #endif
-	case MSR_IA32_U_CET:
-	case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP:
-		kvm_set_xstate_msr(vcpu, msr_info);
-		break;
-	default:
-		if (kvm_pmu_is_valid_msr(vcpu, msr))
-			return kvm_pmu_set_msr(vcpu, msr_info);
+	return ktime_get_real_ns() - get_kvmclock_ns(kvm);
+}
+
+/*
+ * kvmclock updates which are isolated to a given vcpu, such as
+ * vcpu->cpu migration, should not allow system_timestamp from
+ * the rest of the vcpus to remain static.
+ *
+ * So in those cases, request a kvmclock update for all vcpus.
+ * The worst case for a remote vcpu to update its kvmclock
+ * is then bounded by maximum nohz sleep latency.
+ */
+static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
+{
+	unsigned long i;
+	struct kvm_vcpu *vcpu;
+	struct kvm *kvm = v->kvm;
 
-		return KVM_MSR_RET_UNSUPPORTED;
+	kvm_for_each_vcpu(i, vcpu, kvm) {
+		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
+		kvm_vcpu_kick(vcpu);
 	}
-	return 0;
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_msr_common);
 
-static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
+static void kvmclock_reset(struct kvm_vcpu *vcpu)
 {
-	u64 data;
-	u64 mcg_cap = vcpu->arch.mcg_cap;
-	unsigned bank_num = mcg_cap & 0xff;
-	u32 offset, last_msr;
-
-	switch (msr) {
-	case MSR_IA32_P5_MC_ADDR:
-	case MSR_IA32_P5_MC_TYPE:
-		data = 0;
-		break;
-	case MSR_IA32_MCG_CAP:
-		data = vcpu->arch.mcg_cap;
-		break;
-	case MSR_IA32_MCG_CTL:
-		if (!(mcg_cap & MCG_CTL_P) && !host)
-			return 1;
-		data = vcpu->arch.mcg_ctl;
-		break;
-	case MSR_IA32_MCG_STATUS:
-		data = vcpu->arch.mcg_status;
-		break;
-	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
-		last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1;
-		if (msr > last_msr)
-			return 1;
+	kvm_gpc_deactivate(&vcpu->arch.pv_time);
+	vcpu->arch.time = 0;
+}
 
-		if (!(mcg_cap & MCG_CMCI_P) && !host)
-			return 1;
-		offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2,
-					    last_msr + 1 - MSR_IA32_MC0_CTL2);
-		data = vcpu->arch.mci_ctl2_banks[offset];
-		break;
-	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
-		last_msr = MSR_IA32_MCx_CTL(bank_num) - 1;
-		if (msr > last_msr)
-			return 1;
+static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)
+{
+	++vcpu->stat.tlb_flush;
+	kvm_x86_call(flush_tlb_all)(vcpu);
 
-		offset = array_index_nospec(msr - MSR_IA32_MC0_CTL,
-					    last_msr + 1 - MSR_IA32_MC0_CTL);
-		data = vcpu->arch.mce_banks[offset];
-		break;
-	default:
-		return 1;
-	}
-	*pdata = data;
-	return 0;
+	/* Flushing all ASIDs flushes the current ASID... */
+	kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
 }
 
-int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
-{
-	switch (msr_info->index) {
-	case MSR_IA32_PLATFORM_ID:
-	case MSR_IA32_EBL_CR_POWERON:
-	case MSR_IA32_LASTBRANCHFROMIP:
-	case MSR_IA32_LASTBRANCHTOIP:
-	case MSR_IA32_LASTINTFROMIP:
-	case MSR_IA32_LASTINTTOIP:
-	case MSR_AMD64_SYSCFG:
-	case MSR_K8_TSEG_ADDR:
-	case MSR_K8_TSEG_MASK:
-	case MSR_VM_HSAVE_PA:
-	case MSR_K8_INT_PENDING_MSG:
-	case MSR_AMD64_NB_CFG:
-	case MSR_FAM10H_MMIO_CONF_BASE:
-	case MSR_AMD64_BU_CFG2:
-	case MSR_IA32_PERF_CTL:
-	case MSR_AMD64_DC_CFG:
-	case MSR_AMD64_TW_CFG:
-	case MSR_F15H_EX_CFG:
-	/*
-	 * Intel Sandy Bridge CPUs must support the RAPL (running average power
-	 * limit) MSRs. Just return 0, as we do not want to expose the host
-	 * data here. Do not conditionalize this on CPUID, as KVM does not do
-	 * so for existing CPU-specific MSRs.
-	 */
-	case MSR_RAPL_POWER_UNIT:
-	case MSR_PP0_ENERGY_STATUS:	/* Power plane 0 (core) */
-	case MSR_PP1_ENERGY_STATUS:	/* Power plane 1 (graphics uncore) */
-	case MSR_PKG_ENERGY_STATUS:	/* Total package */
-	case MSR_DRAM_ENERGY_STATUS:	/* DRAM controller */
-		msr_info->data = 0;
-		break;
-	case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
-	case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
-	case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
-	case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
-		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
-			return kvm_pmu_get_msr(vcpu, msr_info);
-		msr_info->data = 0;
-		break;
-	case MSR_IA32_UCODE_REV:
-		msr_info->data = vcpu->arch.microcode_version;
-		break;
-	case MSR_IA32_ARCH_CAPABILITIES:
-		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
-			return KVM_MSR_RET_UNSUPPORTED;
-		msr_info->data = vcpu->arch.arch_capabilities;
-		break;
-	case MSR_IA32_PERF_CAPABILITIES:
-		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_PDCM))
-			return KVM_MSR_RET_UNSUPPORTED;
-		msr_info->data = vcpu->arch.perf_capabilities;
-		break;
-	case MSR_IA32_POWER_CTL:
-		msr_info->data = vcpu->arch.msr_ia32_power_ctl;
-		break;
-	case MSR_IA32_TSC: {
-		/*
-		 * Intel SDM states that MSR_IA32_TSC read adds the TSC offset
-		 * even when not intercepted. AMD manual doesn't explicitly
-		 * state this but appears to behave the same.
-		 *
-		 * On userspace reads and writes, however, we unconditionally
-		 * return L1's TSC value to ensure backwards-compatible
-		 * behavior for migration.
-		 */
-		u64 offset, ratio;
-
-		if (msr_info->host_initiated) {
-			offset = vcpu->arch.l1_tsc_offset;
-			ratio = vcpu->arch.l1_tsc_scaling_ratio;
-		} else {
-			offset = vcpu->arch.tsc_offset;
-			ratio = vcpu->arch.tsc_scaling_ratio;
-		}
+static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)
+{
+	++vcpu->stat.tlb_flush;
 
-		msr_info->data = kvm_scale_tsc(rdtsc(), ratio) + offset;
-		break;
-	}
-	case MSR_IA32_CR_PAT:
-		msr_info->data = vcpu->arch.pat;
-		break;
-	case MSR_MTRRcap:
-	case MTRRphysBase_MSR(0) ... MSR_MTRRfix4K_F8000:
-	case MSR_MTRRdefType:
-		return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);
-	case 0xcd: /* fsb frequency */
-		msr_info->data = 3;
-		break;
+	if (!tdp_enabled) {
 		/*
-		 * MSR_EBC_FREQUENCY_ID
-		 * Conservative value valid for even the basic CPU models.
-		 * Models 0,1: 000 in bits 23:21 indicating a bus speed of
-		 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
-		 * and 266MHz for model 3, or 4. Set Core Clock
-		 * Frequency to System Bus Frequency Ratio to 1 (bits
-		 * 31:24) even though these are only valid for CPU
-		 * models > 2, however guests may end up dividing or
-		 * multiplying by zero otherwise.
+		 * A TLB flush on behalf of the guest is equivalent to
+		 * INVPCID(all), toggling CR4.PGE, etc., which requires
+		 * a forced sync of the shadow page tables.  Ensure all the
+		 * roots are synced and the guest TLB in hardware is clean.
 		 */
-	case MSR_EBC_FREQUENCY_ID:
-		msr_info->data = 1 << 24;
-		break;
-	case MSR_IA32_APICBASE:
-		msr_info->data = vcpu->arch.apic_base;
-		break;
-	case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff:
-		return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);
-	case MSR_IA32_TSC_DEADLINE:
-		msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);
-		break;
-	case MSR_IA32_TSC_ADJUST:
-		msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
-		break;
-	case MSR_IA32_MISC_ENABLE:
-		msr_info->data = vcpu->arch.ia32_misc_enable_msr;
-		break;
-	case MSR_IA32_SMBASE:
-		if (!IS_ENABLED(CONFIG_KVM_SMM) || !msr_info->host_initiated)
-			return 1;
-		msr_info->data = vcpu->arch.smbase;
-		break;
-	case MSR_SMI_COUNT:
-		msr_info->data = vcpu->arch.smi_count;
-		break;
-	case MSR_IA32_PERF_STATUS:
-		/* TSC increment by tick */
-		msr_info->data = 1000ULL;
-		/* CPU multiplier */
-		msr_info->data |= (((uint64_t)4ULL) << 40);
-		break;
-	case MSR_EFER:
-		msr_info->data = vcpu->arch.efer;
-		break;
-	case MSR_KVM_WALL_CLOCK:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
-			return KVM_MSR_RET_UNSUPPORTED;
+		kvm_mmu_sync_roots(vcpu);
+		kvm_mmu_sync_prev_roots(vcpu);
+	}
 
-		msr_info->data = vcpu->kvm->arch.wall_clock;
-		break;
-	case MSR_KVM_WALL_CLOCK_NEW:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
-			return KVM_MSR_RET_UNSUPPORTED;
+	kvm_x86_call(flush_tlb_guest)(vcpu);
 
-		msr_info->data = vcpu->kvm->arch.wall_clock;
-		break;
-	case MSR_KVM_SYSTEM_TIME:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
-			return KVM_MSR_RET_UNSUPPORTED;
+	/*
+	 * Flushing all "guest" TLB is always a superset of Hyper-V's fine
+	 * grained flushing.
+	 */
+	kvm_hv_vcpu_purge_flush_tlb(vcpu);
+}
 
-		msr_info->data = vcpu->arch.time;
-		break;
-	case MSR_KVM_SYSTEM_TIME_NEW:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
-			return KVM_MSR_RET_UNSUPPORTED;
 
-		msr_info->data = vcpu->arch.time;
-		break;
-	case MSR_KVM_ASYNC_PF_EN:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
-			return KVM_MSR_RET_UNSUPPORTED;
+static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu)
+{
+	++vcpu->stat.tlb_flush;
+	kvm_x86_call(flush_tlb_current)(vcpu);
+}
 
-		msr_info->data = vcpu->arch.apf.msr_en_val;
-		break;
-	case MSR_KVM_ASYNC_PF_INT:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
-			return KVM_MSR_RET_UNSUPPORTED;
+/*
+ * Service "local" TLB flush requests, which are specific to the current MMU
+ * context.  In addition to the generic event handling in vcpu_enter_guest(),
+ * TLB flushes that are targeted at an MMU context also need to be serviced
+ * prior before nested VM-Enter/VM-Exit.
+ */
+void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu)
+{
+	if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
+		kvm_vcpu_flush_tlb_current(vcpu);
 
-		msr_info->data = vcpu->arch.apf.msr_int_val;
-		break;
-	case MSR_KVM_ASYNC_PF_ACK:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
-			return KVM_MSR_RET_UNSUPPORTED;
+	if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu))
+		kvm_vcpu_flush_tlb_guest(vcpu);
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_service_local_tlb_flush_requests);
 
-		msr_info->data = 0;
-		break;
-	case MSR_KVM_STEAL_TIME:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
-			return KVM_MSR_RET_UNSUPPORTED;
+static void record_steal_time(struct kvm_vcpu *vcpu)
+{
+	struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
+	struct kvm_steal_time __user *st;
+	struct kvm_memslots *slots;
+	gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;
+	u64 steal;
+	u32 version;
 
-		msr_info->data = vcpu->arch.st.msr_val;
-		break;
-	case MSR_KVM_PV_EOI_EN:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
-			return KVM_MSR_RET_UNSUPPORTED;
+	if (kvm_xen_msr_enabled(vcpu->kvm)) {
+		kvm_xen_runstate_set_running(vcpu);
+		return;
+	}
 
-		msr_info->data = vcpu->arch.pv_eoi.msr_val;
-		break;
-	case MSR_KVM_POLL_CONTROL:
-		if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
-			return KVM_MSR_RET_UNSUPPORTED;
+	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
+		return;
 
-		msr_info->data = vcpu->arch.msr_kvm_poll_control;
-		break;
-	case MSR_IA32_P5_MC_ADDR:
-	case MSR_IA32_P5_MC_TYPE:
-	case MSR_IA32_MCG_CAP:
-	case MSR_IA32_MCG_CTL:
-	case MSR_IA32_MCG_STATUS:
-	case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
-	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
-		return get_msr_mce(vcpu, msr_info->index, &msr_info->data,
-				   msr_info->host_initiated);
-	case MSR_IA32_XSS:
-		if (!msr_info->host_initiated &&
-		    !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
-			return 1;
-		msr_info->data = vcpu->arch.ia32_xss;
-		break;
-	case MSR_K7_CLK_CTL:
-		/*
-		 * Provide expected ramp-up count for K7. All other
-		 * are set to zero, indicating minimum divisors for
-		 * every field.
-		 *
-		 * This prevents guest kernels on AMD host with CPU
-		 * type 6, model 8 and higher from exploding due to
-		 * the rdmsr failing.
-		 */
-		msr_info->data = 0x20000000;
-		break;
-#ifdef CONFIG_KVM_HYPERV
-	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
-	case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER:
-	case HV_X64_MSR_SYNDBG_OPTIONS:
-	case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
-	case HV_X64_MSR_CRASH_CTL:
-	case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
-	case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
-	case HV_X64_MSR_TSC_EMULATION_CONTROL:
-	case HV_X64_MSR_TSC_EMULATION_STATUS:
-	case HV_X64_MSR_TSC_INVARIANT_CONTROL:
-		return kvm_hv_get_msr_common(vcpu,
-					     msr_info->index, &msr_info->data,
-					     msr_info->host_initiated);
-#endif
-	case MSR_IA32_BBL_CR_CTL3:
-		/* This legacy MSR exists but isn't fully documented in current
-		 * silicon.  It is however accessed by winxp in very narrow
-		 * scenarios where it sets bit #19, itself documented as
-		 * a "reserved" bit.  Best effort attempt to source coherent
-		 * read data here should the balance of the register be
-		 * interpreted by the guest:
-		 *
-		 * L2 cache control register 3: 64GB range, 256KB size,
-		 * enabled, latency 0x1, configured
-		 */
-		msr_info->data = 0xbe702111;
-		break;
-	case MSR_AMD64_OSVW_ID_LENGTH:
-		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW))
-			return 1;
-		msr_info->data = vcpu->arch.osvw.length;
-		break;
-	case MSR_AMD64_OSVW_STATUS:
-		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW))
-			return 1;
-		msr_info->data = vcpu->arch.osvw.status;
-		break;
-	case MSR_PLATFORM_INFO:
-		if (!msr_info->host_initiated &&
-		    !vcpu->kvm->arch.guest_can_read_msr_platform_info)
-			return 1;
-		msr_info->data = vcpu->arch.msr_platform_info;
-		break;
-	case MSR_MISC_FEATURES_ENABLES:
-		msr_info->data = vcpu->arch.msr_misc_features_enables;
-		break;
-	case MSR_K7_HWCR:
-		msr_info->data = vcpu->arch.msr_hwcr;
-		break;
-#ifdef CONFIG_X86_64
-	case MSR_IA32_XFD:
-		if (!msr_info->host_initiated &&
-		    !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD))
-			return 1;
+	if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm))
+		return;
 
-		msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd;
-		break;
-	case MSR_IA32_XFD_ERR:
-		if (!msr_info->host_initiated &&
-		    !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD))
-			return 1;
+	slots = kvm_memslots(vcpu->kvm);
 
-		msr_info->data = vcpu->arch.guest_fpu.xfd_err;
-		break;
-#endif
-	case MSR_IA32_U_CET:
-	case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP:
-		kvm_get_xstate_msr(vcpu, msr_info);
-		break;
-	default:
-		if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
-			return kvm_pmu_get_msr(vcpu, msr_info);
+	if (unlikely(slots->generation != ghc->generation ||
+		     gpa != ghc->gpa ||
+		     kvm_is_error_hva(ghc->hva) || !ghc->memslot)) {
+		/* We rely on the fact that it fits in a single page. */
+		BUILD_BUG_ON((sizeof(*st) - 1) & KVM_STEAL_VALID_BITS);
 
-		return KVM_MSR_RET_UNSUPPORTED;
+		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st)) ||
+		    kvm_is_error_hva(ghc->hva) || !ghc->memslot)
+			return;
 	}
-	return 0;
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_msr_common);
 
-/*
- * Read or write a bunch of msrs. All parameters are kernel addresses.
- *
- * @return number of msrs set successfully.
- */
-static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
-		    struct kvm_msr_entry *entries,
-		    int (*do_msr)(struct kvm_vcpu *vcpu,
-				  unsigned index, u64 *data))
-{
-	bool fpu_loaded = false;
-	int i;
+	st = (struct kvm_steal_time __user *)ghc->hva;
+	/*
+	 * Doing a TLB flush here, on the guest's behalf, can avoid
+	 * expensive IPIs.
+	 */
+	if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) {
+		u8 st_preempted = 0;
+		int err = -EFAULT;
 
-	for (i = 0; i < msrs->nmsrs; ++i) {
-		/*
-		 * If userspace is accessing one or more XSTATE-managed MSRs,
-		 * temporarily load the guest's FPU state so that the guest's
-		 * MSR value(s) is resident in hardware and thus can be accessed
-		 * via RDMSR/WRMSR.
-		 */
-		if (!fpu_loaded && is_xstate_managed_msr(vcpu, entries[i].index)) {
-			kvm_load_guest_fpu(vcpu);
-			fpu_loaded = true;
-		}
-		if (do_msr(vcpu, entries[i].index, &entries[i].data))
-			break;
-	}
-	if (fpu_loaded)
-		kvm_put_guest_fpu(vcpu);
+		if (!user_access_begin(st, sizeof(*st)))
+			return;
 
-	return i;
-}
+		asm volatile("1: xchgb %0, %2\n"
+			     "xor %1, %1\n"
+			     "2:\n"
+			     _ASM_EXTABLE_UA(1b, 2b)
+			     : "+q" (st_preempted),
+			       "+&r" (err),
+			       "+m" (st->preempted));
+		if (err)
+			goto out;
 
-/*
- * Read or write a bunch of msrs. Parameters are user addresses.
- *
- * @return number of msrs set successfully.
- */
-static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
-		  int (*do_msr)(struct kvm_vcpu *vcpu,
-				unsigned index, u64 *data),
-		  int writeback)
-{
-	struct kvm_msrs msrs;
-	struct kvm_msr_entry *entries;
-	unsigned size;
-	int r;
+		user_access_end();
 
-	r = -EFAULT;
-	if (copy_from_user(&msrs, user_msrs, sizeof(msrs)))
-		goto out;
+		vcpu->arch.st.preempted = 0;
 
-	r = -E2BIG;
-	if (msrs.nmsrs >= MAX_IO_MSRS)
-		goto out;
+		trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
+				       st_preempted & KVM_VCPU_FLUSH_TLB);
+		if (st_preempted & KVM_VCPU_FLUSH_TLB)
+			kvm_vcpu_flush_tlb_guest(vcpu);
 
-	size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
-	entries = memdup_user(user_msrs->entries, size);
-	if (IS_ERR(entries)) {
-		r = PTR_ERR(entries);
-		goto out;
+		if (!user_access_begin(st, sizeof(*st)))
+			goto dirty;
+	} else {
+		if (!user_access_begin(st, sizeof(*st)))
+			return;
+
+		unsafe_put_user(0, &st->preempted, out);
+		vcpu->arch.st.preempted = 0;
 	}
 
-	r = __msr_io(vcpu, &msrs, entries, do_msr);
+	unsafe_get_user(version, &st->version, out);
+	if (version & 1)
+		version += 1;  /* first time write, random junk */
 
-	if (writeback && copy_to_user(user_msrs->entries, entries, size))
-		r = -EFAULT;
+	version += 1;
+	unsafe_put_user(version, &st->version, out);
 
-	kfree(entries);
-out:
-	return r;
+	smp_wmb();
+
+	unsafe_get_user(steal, &st->steal, out);
+	steal += current->sched_info.run_delay -
+		vcpu->arch.st.last_steal;
+	vcpu->arch.st.last_steal = current->sched_info.run_delay;
+	unsafe_put_user(steal, &st->steal, out);
+
+	version += 1;
+	unsafe_put_user(version, &st->version, out);
+
+ out:
+	user_access_end();
+ dirty:
+	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
 }
 
 static inline bool kvm_can_mwait_in_guest(void)
@@ -4928,15 +2363,14 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 			r &= ~KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST;
 		break;
 	case KVM_CAP_NESTED_STATE:
-		r = kvm_x86_ops.nested_ops->get_state ?
-			kvm_x86_ops.nested_ops->get_state(NULL, NULL, 0) : 0;
+		r = kvm_nested_ops.enabled ? kvm_nested_call(get_state)(NULL, NULL, 0) : 0;
 		break;
 #ifdef CONFIG_KVM_HYPERV
 	case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
 		r = kvm_x86_ops.enable_l2_tlb_flush != NULL;
 		break;
 	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
-		r = kvm_x86_ops.nested_ops->enable_evmcs != NULL;
+		r = kvm_nested_ops.enabled && kvm_nested_ops.enable_evmcs != NULL;
 		break;
 #endif
 	case KVM_CAP_SMALLER_MAXPHYADDR:
@@ -5026,32 +2460,9 @@ long kvm_arch_dev_ioctl(struct file *filp,
 	long r;
 
 	switch (ioctl) {
-	case KVM_GET_MSR_INDEX_LIST: {
-		struct kvm_msr_list __user *user_msr_list = argp;
-		struct kvm_msr_list msr_list;
-		unsigned n;
-
-		r = -EFAULT;
-		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
-			goto out;
-		n = msr_list.nmsrs;
-		msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs;
-		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
-			goto out;
-		r = -E2BIG;
-		if (n < msr_list.nmsrs)
-			goto out;
-		r = -EFAULT;
-		if (copy_to_user(user_msr_list->indices, &msrs_to_save,
-				 num_msrs_to_save * sizeof(u32)))
-			goto out;
-		if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
-				 &emulated_msrs,
-				 num_emulated_msrs * sizeof(u32)))
-			goto out;
-		r = 0;
+	case KVM_GET_MSR_INDEX_LIST:
+		r = kvm_get_msr_index_list(argp);
 		break;
-	}
 	case KVM_GET_SUPPORTED_CPUID:
 	case KVM_GET_EMULATED_CPUID: {
 		struct kvm_cpuid2 __user *cpuid_arg = argp;
@@ -5069,40 +2480,21 @@ long kvm_arch_dev_ioctl(struct file *filp,
 		r = -EFAULT;
 		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
 			goto out;
-		r = 0;
-		break;
-	}
-	case KVM_X86_GET_MCE_CAP_SUPPORTED:
-		r = -EFAULT;
-		if (copy_to_user(argp, &kvm_caps.supported_mce_cap,
-				 sizeof(kvm_caps.supported_mce_cap)))
-			goto out;
-		r = 0;
-		break;
-	case KVM_GET_MSR_FEATURE_INDEX_LIST: {
-		struct kvm_msr_list __user *user_msr_list = argp;
-		struct kvm_msr_list msr_list;
-		unsigned int n;
-
-		r = -EFAULT;
-		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
-			goto out;
-		n = msr_list.nmsrs;
-		msr_list.nmsrs = num_msr_based_features;
-		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
-			goto out;
-		r = -E2BIG;
-		if (n < msr_list.nmsrs)
-			goto out;
+		r = 0;
+		break;
+	}
+	case KVM_X86_GET_MCE_CAP_SUPPORTED:
 		r = -EFAULT;
-		if (copy_to_user(user_msr_list->indices, &msr_based_features,
-				 num_msr_based_features * sizeof(u32)))
+		if (copy_to_user(argp, &kvm_caps.supported_mce_cap,
+				 sizeof(kvm_caps.supported_mce_cap)))
 			goto out;
 		r = 0;
 		break;
-	}
+	case KVM_GET_MSR_FEATURE_INDEX_LIST:
+		r = kvm_get_feature_msr_index_list(argp);
+		break;
 	case KVM_GET_MSRS:
-		r = msr_io(NULL, argp, do_get_feature_msr, 1);
+		r = kvm_get_feature_msrs(argp);
 		break;
 #ifdef CONFIG_KVM_HYPERV
 	case KVM_GET_SUPPORTED_HV_CPUID:
@@ -5322,6 +2714,18 @@ static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
 	return 0;
 }
 
+static bool kvm_is_interrupt_allowed(struct kvm_vcpu *vcpu)
+{
+	/*
+	 * Note, .interrupt_allowed() returns -EBUSY if interrupts are allowed
+	 * based on CPU state, but can't be immediately delivered due to a
+	 * pending nested VM-Enter.  Treat that case as "allowed", because
+	 * the goal here is just to check if interrupts are architecturally
+	 * allowed, not to check if they can be injected.
+	 */
+	return kvm_x86_call(interrupt_allowed)(vcpu, false);
+}
+
 static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)
 {
 	/*
@@ -5347,7 +2751,7 @@ static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)
 	 * or KVM_SET_SREGS.  For that to work, we must be at an
 	 * instruction boundary and with no events half-injected.
 	 */
-	return (kvm_arch_interrupt_allowed(vcpu) &&
+	return (kvm_is_interrupt_allowed(vcpu) &&
 		kvm_cpu_accept_dm_intr(vcpu) &&
 		!kvm_event_needs_reinjection(vcpu) &&
 		!kvm_is_exception_pending(vcpu));
@@ -5472,7 +2876,8 @@ static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
 	if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
 		return -EINVAL;
 
-	banks += array_index_nospec(4 * mce->bank, 4 * bank_num);
+	mce->bank = array_index_nospec(mce->bank, bank_num);
+	banks += 4 * mce->bank;
 
 	if (is_ucna(mce))
 		return kvm_vcpu_x86_set_ucna(vcpu, mce, banks);
@@ -5532,7 +2937,7 @@ static struct kvm_queued_exception *kvm_get_exception_to_save(struct kvm_vcpu *v
 	return &vcpu->arch.exception;
 }
 
-static void kvm_handle_exception_payload_quirk(struct kvm_vcpu *vcpu)
+void kvm_handle_exception_payload_quirk(struct kvm_vcpu *vcpu)
 {
 	struct kvm_queued_exception *ex = kvm_get_exception_to_save(vcpu);
 
@@ -5736,57 +3141,6 @@ static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
 	return 0;
 }
 
-static int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
-					    struct kvm_debugregs *dbgregs)
-{
-	unsigned int i;
-
-	if (vcpu->kvm->arch.has_protected_state &&
-	    vcpu->arch.guest_state_protected)
-		return -EINVAL;
-
-	kvm_handle_exception_payload_quirk(vcpu);
-
-	memset(dbgregs, 0, sizeof(*dbgregs));
-
-	BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.db) != ARRAY_SIZE(dbgregs->db));
-	for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++)
-		dbgregs->db[i] = vcpu->arch.db[i];
-
-	dbgregs->dr6 = vcpu->arch.dr6;
-	dbgregs->dr7 = vcpu->arch.dr7;
-	return 0;
-}
-
-static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
-					    struct kvm_debugregs *dbgregs)
-{
-	unsigned int i;
-
-	if (vcpu->kvm->arch.has_protected_state &&
-	    vcpu->arch.guest_state_protected)
-		return -EINVAL;
-
-	if (dbgregs->flags)
-		return -EINVAL;
-
-	if (!kvm_dr6_valid(dbgregs->dr6))
-		return -EINVAL;
-	if (!kvm_dr7_valid(dbgregs->dr7))
-		return -EINVAL;
-
-	for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++)
-		vcpu->arch.db[i] = dbgregs->db[i];
-
-	kvm_update_dr0123(vcpu);
-	vcpu->arch.dr6 = dbgregs->dr6;
-	vcpu->arch.dr7 = dbgregs->dr7;
-	kvm_update_dr7(vcpu);
-
-	return 0;
-}
-
-
 static int kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu,
 					 u8 *state, unsigned int size)
 {
@@ -6029,9 +3383,10 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
 			uint16_t vmcs_version;
 			void __user *user_ptr;
 
-			if (!kvm_x86_ops.nested_ops->enable_evmcs)
+			if (!kvm_nested_ops.enabled ||
+			    !kvm_nested_ops.enable_evmcs)
 				return -ENOTTY;
-			r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version);
+			r = kvm_nested_call(enable_evmcs)(vcpu, &vmcs_version);
 			if (!r) {
 				user_ptr = (void __user *)(uintptr_t)cap->args[0];
 				if (copy_to_user(user_ptr, &vmcs_version,
@@ -6058,134 +3413,6 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
 	}
 }
 
-struct kvm_x86_reg_id {
-	__u32 index;
-	__u8  type;
-	__u8  rsvd1;
-	__u8  rsvd2:4;
-	__u8  size:4;
-	__u8  x86;
-};
-
-static int kvm_translate_kvm_reg(struct kvm_vcpu *vcpu,
-				 struct kvm_x86_reg_id *reg)
-{
-	switch (reg->index) {
-	case KVM_REG_GUEST_SSP:
-		/*
-		 * FIXME: If host-initiated accesses are ever exempted from
-		 * ignore_msrs (in kvm_do_msr_access()), drop this manual check
-		 * and rely on KVM's standard checks to reject accesses to regs
-		 * that don't exist.
-		 */
-		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK))
-			return -EINVAL;
-
-		reg->type = KVM_X86_REG_TYPE_MSR;
-		reg->index = MSR_KVM_INTERNAL_GUEST_SSP;
-		break;
-	default:
-		return -EINVAL;
-	}
-	return 0;
-}
-
-static int kvm_get_one_msr(struct kvm_vcpu *vcpu, u32 msr, u64 __user *user_val)
-{
-	u64 val;
-
-	if (do_get_msr(vcpu, msr, &val))
-		return -EINVAL;
-
-	if (put_user(val, user_val))
-		return -EFAULT;
-
-	return 0;
-}
-
-static int kvm_set_one_msr(struct kvm_vcpu *vcpu, u32 msr, u64 __user *user_val)
-{
-	u64 val;
-
-	if (get_user(val, user_val))
-		return -EFAULT;
-
-	if (do_set_msr(vcpu, msr, &val))
-		return -EINVAL;
-
-	return 0;
-}
-
-static int kvm_get_set_one_reg(struct kvm_vcpu *vcpu, unsigned int ioctl,
-			       void __user *argp)
-{
-	struct kvm_one_reg one_reg;
-	struct kvm_x86_reg_id *reg;
-	u64 __user *user_val;
-	bool load_fpu;
-	int r;
-
-	if (copy_from_user(&one_reg, argp, sizeof(one_reg)))
-		return -EFAULT;
-
-	if ((one_reg.id & KVM_REG_ARCH_MASK) != KVM_REG_X86)
-		return -EINVAL;
-
-	reg = (struct kvm_x86_reg_id *)&one_reg.id;
-	if (reg->rsvd1 || reg->rsvd2)
-		return -EINVAL;
-
-	if (reg->type == KVM_X86_REG_TYPE_KVM) {
-		r = kvm_translate_kvm_reg(vcpu, reg);
-		if (r)
-			return r;
-	}
-
-	if (reg->type != KVM_X86_REG_TYPE_MSR)
-		return -EINVAL;
-
-	if ((one_reg.id & KVM_REG_SIZE_MASK) != KVM_REG_SIZE_U64)
-		return -EINVAL;
-
-	guard(srcu)(&vcpu->kvm->srcu);
-
-	load_fpu = is_xstate_managed_msr(vcpu, reg->index);
-	if (load_fpu)
-		kvm_load_guest_fpu(vcpu);
-
-	user_val = u64_to_user_ptr(one_reg.addr);
-	if (ioctl == KVM_GET_ONE_REG)
-		r = kvm_get_one_msr(vcpu, reg->index, user_val);
-	else
-		r = kvm_set_one_msr(vcpu, reg->index, user_val);
-
-	if (load_fpu)
-		kvm_put_guest_fpu(vcpu);
-	return r;
-}
-
-static int kvm_get_reg_list(struct kvm_vcpu *vcpu,
-			    struct kvm_reg_list __user *user_list)
-{
-	u64 nr_regs = guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) ? 1 : 0;
-	u64 user_nr_regs;
-
-	if (get_user(user_nr_regs, &user_list->n))
-		return -EFAULT;
-
-	if (put_user(nr_regs, &user_list->n))
-		return -EFAULT;
-
-	if (user_nr_regs < nr_regs)
-		return -E2BIG;
-
-	if (nr_regs &&
-	    put_user(KVM_X86_REG_KVM(KVM_REG_GUEST_SSP), &user_list->reg[0]))
-		return -EFAULT;
-
-	return 0;
-}
-
 long kvm_arch_vcpu_ioctl(struct file *filp,
 			 unsigned int ioctl, unsigned long arg)
 {
@@ -6290,18 +3517,12 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 		r = 0;
 		break;
 	}
-	case KVM_GET_MSRS: {
-		int idx = srcu_read_lock(&vcpu->kvm->srcu);
-		r = msr_io(vcpu, argp, do_get_msr, 1);
-		srcu_read_unlock(&vcpu->kvm->srcu, idx);
+	case KVM_GET_MSRS:
+		r = kvm_get_msrs(vcpu, argp);
 		break;
-	}
-	case KVM_SET_MSRS: {
-		int idx = srcu_read_lock(&vcpu->kvm->srcu);
-		r = msr_io(vcpu, argp, do_set_msr, 0);
-		srcu_read_unlock(&vcpu->kvm->srcu, idx);
+	case KVM_SET_MSRS:
+		r = kvm_set_msrs(vcpu, argp);
 		break;
-	}
 	case KVM_GET_ONE_REG:
 	case KVM_SET_ONE_REG:
 		r = kvm_get_set_one_reg(vcpu, ioctl, argp);
@@ -6529,7 +3750,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 		u32 user_data_size;
 
 		r = -EINVAL;
-		if (!kvm_x86_ops.nested_ops->get_state)
+		if (!kvm_nested_ops.enabled)
 			break;
 
 		BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size));
@@ -6537,8 +3758,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 		if (get_user(user_data_size, &user_kvm_nested_state->size))
 			break;
 
-		r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state,
-						     user_data_size);
+		r = kvm_nested_call(get_state)(vcpu, user_kvm_nested_state, user_data_size);
 		if (r < 0)
 			break;
 
@@ -6559,7 +3779,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 		int idx;
 
 		r = -EINVAL;
-		if (!kvm_x86_ops.nested_ops->set_state)
+		if (!kvm_nested_ops.enabled)
 			break;
 
 		r = -EFAULT;
@@ -6582,7 +3802,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 			break;
 
 		idx = srcu_read_lock(&vcpu->kvm->srcu);
-		r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state);
+		r = kvm_nested_call(set_state)(vcpu, user_kvm_nested_state, &kvm_state);
 		srcu_read_unlock(&vcpu->kvm->srcu, idx);
 		break;
 	}
@@ -6623,7 +3843,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 		r = -ENOMEM;
 		if (!u.sregs2)
 			goto out;
-		__get_sregs2(vcpu, u.sregs2);
+		kvm_vcpu_ioctl_x86_get_sregs2(vcpu, u.sregs2);
 		r = -EFAULT;
 		if (copy_to_user(argp, u.sregs2, sizeof(struct kvm_sregs2)))
 			goto out;
@@ -6642,7 +3862,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 			u.sregs2 = NULL;
 			goto out;
 		}
-		r = __set_sregs2(vcpu, u.sregs2);
+		r = kvm_vcpu_ioctl_x86_set_sregs2(vcpu, u.sregs2);
 		break;
 	}
 	case KVM_HAS_DEVICE_ATTR:
@@ -6736,7 +3956,10 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
 			break;
 		fallthrough;
 	case KVM_CAP_DISABLE_QUIRKS:
-		kvm->arch.disabled_quirks |= cap->args[0] & kvm_caps.supported_quirks;
+		mutex_lock(&kvm->lock);
+		WRITE_ONCE(kvm->arch.disabled_quirks,
+			   kvm->arch.disabled_quirks | (cap->args[0] & kvm_caps.supported_quirks));
+		mutex_unlock(&kvm->lock);
 		r = 0;
 		break;
 	case KVM_CAP_SPLIT_IRQCHIP: {
@@ -6994,113 +4217,6 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
 	return r;
 }
 
-static struct kvm_x86_msr_filter *kvm_alloc_msr_filter(bool default_allow)
-{
-	struct kvm_x86_msr_filter *msr_filter;
-
-	msr_filter = kzalloc_obj(*msr_filter, GFP_KERNEL_ACCOUNT);
-	if (!msr_filter)
-		return NULL;
-
-	msr_filter->default_allow = default_allow;
-	return msr_filter;
-}
-
-static void kvm_free_msr_filter(struct kvm_x86_msr_filter *msr_filter)
-{
-	u32 i;
-
-	if (!msr_filter)
-		return;
-
-	for (i = 0; i < msr_filter->count; i++)
-		kfree(msr_filter->ranges[i].bitmap);
-
-	kfree(msr_filter);
-}
-
-static int kvm_add_msr_filter(struct kvm_x86_msr_filter *msr_filter,
-			      struct kvm_msr_filter_range *user_range)
-{
-	unsigned long *bitmap;
-	size_t bitmap_size;
-
-	if (!user_range->nmsrs)
-		return 0;
-
-	if (user_range->flags & ~KVM_MSR_FILTER_RANGE_VALID_MASK)
-		return -EINVAL;
-
-	if (!user_range->flags)
-		return -EINVAL;
-
-	bitmap_size = BITS_TO_LONGS(user_range->nmsrs) * sizeof(long);
-	if (!bitmap_size || bitmap_size > KVM_MSR_FILTER_MAX_BITMAP_SIZE)
-		return -EINVAL;
-
-	bitmap = memdup_user((__user u8*)user_range->bitmap, bitmap_size);
-	if (IS_ERR(bitmap))
-		return PTR_ERR(bitmap);
-
-	msr_filter->ranges[msr_filter->count] = (struct msr_bitmap_range) {
-		.flags = user_range->flags,
-		.base = user_range->base,
-		.nmsrs = user_range->nmsrs,
-		.bitmap = bitmap,
-	};
-
-	msr_filter->count++;
-	return 0;
-}
-
-static int kvm_vm_ioctl_set_msr_filter(struct kvm *kvm,
-				       struct kvm_msr_filter *filter)
-{
-	struct kvm_x86_msr_filter *new_filter, *old_filter;
-	bool default_allow;
-	bool empty = true;
-	int r;
-	u32 i;
-
-	if (filter->flags & ~KVM_MSR_FILTER_VALID_MASK)
-		return -EINVAL;
-
-	for (i = 0; i < ARRAY_SIZE(filter->ranges); i++)
-		empty &= !filter->ranges[i].nmsrs;
-
-	default_allow = !(filter->flags & KVM_MSR_FILTER_DEFAULT_DENY);
-	if (empty && !default_allow)
-		return -EINVAL;
-
-	new_filter = kvm_alloc_msr_filter(default_allow);
-	if (!new_filter)
-		return -ENOMEM;
-
-	for (i = 0; i < ARRAY_SIZE(filter->ranges); i++) {
-		r = kvm_add_msr_filter(new_filter, &filter->ranges[i]);
-		if (r) {
-			kvm_free_msr_filter(new_filter);
-			return r;
-		}
-	}
-
-	mutex_lock(&kvm->lock);
-	old_filter = rcu_replace_pointer(kvm->arch.msr_filter, new_filter,
-					 mutex_is_locked(&kvm->lock));
-	mutex_unlock(&kvm->lock);
-	synchronize_srcu(&kvm->srcu);
-
-	kvm_free_msr_filter(old_filter);
-
-	/*
-	 * Recalc MSR intercepts as userspace may want to intercept accesses to
-	 * MSRs that KVM would otherwise pass through to the guest.
-	 */
-	kvm_make_all_cpus_request(kvm, KVM_REQ_RECALC_INTERCEPTS);
-
-	return 0;
-}
-
 #ifdef CONFIG_KVM_COMPAT
 /* for KVM_X86_SET_MSR_FILTER */
 struct kvm_msr_filter_range_compat {
@@ -7621,157 +4737,6 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
 	return r;
 }
 
-static void kvm_probe_feature_msr(u32 msr_index)
-{
-	u64 data;
-
-	if (kvm_get_feature_msr(NULL, msr_index, &data, true))
-		return;
-
-	msr_based_features[num_msr_based_features++] = msr_index;
-}
-
-static void kvm_probe_msr_to_save(u32 msr_index)
-{
-	u32 dummy[2];
-
-	if (rdmsr_safe(msr_index, &dummy[0], &dummy[1]))
-		return;
-
-	/*
-	 * Even MSRs that are valid in the host may not be exposed to guests in
-	 * some cases.
-	 */
-	switch (msr_index) {
-	case MSR_IA32_BNDCFGS:
-		if (!kvm_mpx_supported())
-			return;
-		break;
-	case MSR_TSC_AUX:
-		if (!kvm_cpu_cap_has(X86_FEATURE_RDTSCP) &&
-		    !kvm_cpu_cap_has(X86_FEATURE_RDPID))
-			return;
-		break;
-	case MSR_IA32_UMWAIT_CONTROL:
-		if (!kvm_cpu_cap_has(X86_FEATURE_WAITPKG))
-			return;
-		break;
-	case MSR_IA32_RTIT_CTL:
-	case MSR_IA32_RTIT_STATUS:
-		if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT))
-			return;
-		break;
-	case MSR_IA32_RTIT_CR3_MATCH:
-		if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
-		    !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering))
-			return;
-		break;
-	case MSR_IA32_RTIT_OUTPUT_BASE:
-	case MSR_IA32_RTIT_OUTPUT_MASK:
-		if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
-		    (!intel_pt_validate_hw_cap(PT_CAP_topa_output) &&
-		     !intel_pt_validate_hw_cap(PT_CAP_single_range_output)))
-			return;
-		break;
-	case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
-		if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
-		    (msr_index - MSR_IA32_RTIT_ADDR0_A >=
-		     intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2))
-			return;
-		break;
-	case MSR_ARCH_PERFMON_PERFCTR0 ...
-	     MSR_ARCH_PERFMON_PERFCTR0 + KVM_MAX_NR_GP_COUNTERS - 1:
-		if (msr_index - MSR_ARCH_PERFMON_PERFCTR0 >=
-		    kvm_pmu_cap.num_counters_gp)
-			return;
-		break;
-	case MSR_ARCH_PERFMON_EVENTSEL0 ...
-	     MSR_ARCH_PERFMON_EVENTSEL0 + KVM_MAX_NR_GP_COUNTERS - 1:
-		if (msr_index - MSR_ARCH_PERFMON_EVENTSEL0 >=
-		    kvm_pmu_cap.num_counters_gp)
-			return;
-		break;
-	case MSR_ARCH_PERFMON_FIXED_CTR0 ...
-	     MSR_ARCH_PERFMON_FIXED_CTR0 + KVM_MAX_NR_FIXED_COUNTERS - 1:
-		if (msr_index - MSR_ARCH_PERFMON_FIXED_CTR0 >=
-		    kvm_pmu_cap.num_counters_fixed)
-			return;
-		break;
-	case MSR_AMD64_PERF_CNTR_GLOBAL_CTL:
-	case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS:
-	case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR:
-	case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET:
-		if (!kvm_cpu_cap_has(X86_FEATURE_PERFMON_V2))
-			return;
-		break;
-	case MSR_IA32_XFD:
-	case MSR_IA32_XFD_ERR:
-		if (!kvm_cpu_cap_has(X86_FEATURE_XFD))
-			return;
-		break;
-	case MSR_IA32_TSX_CTRL:
-		if (!(kvm_get_arch_capabilities() & ARCH_CAP_TSX_CTRL_MSR))
-			return;
-		break;
-	case MSR_IA32_XSS:
-		if (!kvm_caps.supported_xss)
-			return;
-		break;
-	case MSR_IA32_U_CET:
-	case MSR_IA32_S_CET:
-		if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) &&
-		    !kvm_cpu_cap_has(X86_FEATURE_IBT))
-			return;
-		break;
-	case MSR_IA32_INT_SSP_TAB:
-		if (!kvm_cpu_cap_has(X86_FEATURE_LM))
-			return;
-		fallthrough;
-	case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP:
-		if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK))
-			return;
-		break;
-	default:
-		break;
-	}
-
-	msrs_to_save[num_msrs_to_save++] = msr_index;
-}
-
-static void kvm_init_msr_lists(void)
-{
-	unsigned i;
-
-	BUILD_BUG_ON_MSG(KVM_MAX_NR_FIXED_COUNTERS != 3,
-			 "Please update the fixed PMCs in msrs_to_save_pmu[]");
-
-	num_msrs_to_save = 0;
-	num_emulated_msrs = 0;
-	num_msr_based_features = 0;
-
-	for (i = 0; i < ARRAY_SIZE(msrs_to_save_base); i++)
-		kvm_probe_msr_to_save(msrs_to_save_base[i]);
-
-	if (enable_pmu) {
-		for (i = 0; i < ARRAY_SIZE(msrs_to_save_pmu); i++)
-			kvm_probe_msr_to_save(msrs_to_save_pmu[i]);
-	}
-
-	for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) {
-		if (!kvm_x86_call(has_emulated_msr)(NULL,
-						    emulated_msrs_all[i]))
-			continue;
-
-		emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i];
-	}
-
-	for (i = KVM_FIRST_EMULATED_VMX_MSR; i <= KVM_LAST_EMULATED_VMX_MSR; i++)
-		kvm_probe_feature_msr(i);
-
-	for (i = 0; i < ARRAY_SIZE(msr_based_features_all_except_vmx); i++)
-		kvm_probe_feature_msr(msr_based_features_all_except_vmx[i]);
-}
-
 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
 			   void *__v)
 {
@@ -7821,36 +4786,24 @@ static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
 	return handled;
 }
 
-void kvm_set_segment(struct kvm_vcpu *vcpu,
-		     struct kvm_segment *var, int seg)
-{
-	kvm_x86_call(set_segment)(vcpu, var, seg);
-}
-
-void kvm_get_segment(struct kvm_vcpu *vcpu,
-		     struct kvm_segment *var, int seg)
-{
-	kvm_x86_call(get_segment)(vcpu, var, seg);
-}
-
 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
 			      struct x86_exception *exception)
 {
-	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
+	struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
 
 	u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
-	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
+	return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, exception);
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_gva_to_gpa_read);
 
 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
 			       struct x86_exception *exception)
 {
-	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
+	struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
 
 	u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
 	access |= PFERR_WRITE_MASK;
-	return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
+	return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, exception);
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_gva_to_gpa_write);
 
@@ -7858,21 +4811,21 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_gva_to_gpa_write);
 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
 				struct x86_exception *exception)
 {
-	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
+	struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
 
-	return mmu->gva_to_gpa(vcpu, mmu, gva, 0, exception);
+	return gva_walk->gva_to_gpa(vcpu, gva_walk, gva, 0, exception);
 }
 
 static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
 				      struct kvm_vcpu *vcpu, u64 access,
 				      struct x86_exception *exception)
 {
-	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
+	struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
 	void *data = val;
 	int r = X86EMUL_CONTINUE;
 
 	while (bytes) {
-		gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);
+		gpa_t gpa = gva_walk->gva_to_gpa(vcpu, gva_walk, addr, access, exception);
 		unsigned offset = addr & (PAGE_SIZE-1);
 		unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
 		int ret;
@@ -7900,14 +4853,14 @@ static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
 				struct x86_exception *exception)
 {
 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
-	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
+	struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
 	u64 access = (kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
 	unsigned offset;
 	int ret;
 
 	/* Inline kvm_read_guest_virt_helper for speed.  */
-	gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access|PFERR_FETCH_MASK,
-				    exception);
+	gpa_t gpa = gva_walk->gva_to_gpa(vcpu, gva_walk, addr, access|PFERR_FETCH_MASK,
+					  exception);
 	if (unlikely(gpa == INVALID_GPA))
 		return X86EMUL_PROPAGATE_FAULT;
 
@@ -7959,12 +4912,12 @@ static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes
 				      struct kvm_vcpu *vcpu, u64 access,
 				      struct x86_exception *exception)
 {
-	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
+	struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
 	void *data = val;
 	int r = X86EMUL_CONTINUE;
 
 	while (bytes) {
-		gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);
+		gpa_t gpa = gva_walk->gva_to_gpa(vcpu, gva_walk, addr, access, exception);
 		unsigned offset = addr & (PAGE_SIZE-1);
 		unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
 		int ret;
@@ -8065,7 +5018,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
 				gpa_t *gpa, struct x86_exception *exception,
 				bool write)
 {
-	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
+	struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
 	u64 access = ((kvm_x86_call(get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0)
 		     | (write ? PFERR_WRITE_MASK : 0);
 
@@ -8075,7 +5028,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
 	 * shadow page table for L2 guest.
 	 */
 	if (vcpu_match_mmio_gva(vcpu, gva) && (!is_paging(vcpu) ||
-	    !permission_fault(vcpu, vcpu->arch.walk_mmu,
+	    !permission_fault(vcpu, gva_walk,
 			      vcpu->arch.mmio_access, 0, access))) {
 		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
 					(gva & (PAGE_SIZE - 1));
@@ -8083,7 +5036,7 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
 		return 1;
 	}
 
-	*gpa = mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
+	*gpa = gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, exception);
 
 	if (*gpa == INVALID_GPA)
 		return -1;
@@ -8489,12 +5442,7 @@ static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
 				     int size, unsigned short port,
 				     const void *val, unsigned int count)
 {
-	return emulator_pio_out(emul_to_vcpu(ctxt), size, port, val, count);
-}
-
-static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
-{
-	return kvm_x86_call(get_segment_base)(vcpu, seg);
+	return emulator_pio_out(emul_to_vcpu(ctxt), size, port, val, count);
 }
 
 static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
@@ -8641,7 +5589,7 @@ static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
 static unsigned long emulator_get_cached_segment_base(
 	struct x86_emulate_ctxt *ctxt, int seg)
 {
-	return get_segment_base(emul_to_vcpu(ctxt), seg);
+	return kvm_get_segment_base(emul_to_vcpu(ctxt), seg);
 }
 
 static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
@@ -8714,61 +5662,22 @@ static int emulator_get_msr_with_filter(struct x86_emulate_ctxt *ctxt,
 					u32 msr_index, u64 *pdata)
 {
 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
-	int r;
-
-	r = kvm_emulate_msr_read(vcpu, msr_index, pdata);
-	if (r < 0)
-		return X86EMUL_UNHANDLEABLE;
-
-	if (r) {
-		if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0,
-				       complete_emulated_rdmsr, r))
-			return X86EMUL_IO_NEEDED;
-
-		trace_kvm_msr_read_ex(msr_index);
-		return X86EMUL_PROPAGATE_FAULT;
-	}
 
-	trace_kvm_msr_read(msr_index, *pdata);
-	return X86EMUL_CONTINUE;
+	return kvm_emulator_get_msr_with_filter(vcpu, msr_index, pdata);
 }
 
 static int emulator_set_msr_with_filter(struct x86_emulate_ctxt *ctxt,
 					u32 msr_index, u64 data)
 {
 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
-	int r;
-
-	r = kvm_emulate_msr_write(vcpu, msr_index, data);
-	if (r < 0)
-		return X86EMUL_UNHANDLEABLE;
-
-	if (r) {
-		if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data,
-				       complete_emulated_msr_access, r))
-			return X86EMUL_IO_NEEDED;
-
-		trace_kvm_msr_write_ex(msr_index, data);
-		return X86EMUL_PROPAGATE_FAULT;
-	}
 
-	trace_kvm_msr_write(msr_index, data);
-	return X86EMUL_CONTINUE;
+	return kvm_emulator_set_msr_with_filter(vcpu, msr_index, data);
 }
 
 static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
 			    u32 msr_index, u64 *pdata)
 {
-	/*
-	 * Treat emulator accesses to the current shadow stack pointer as host-
-	 * initiated, as they aren't true MSR accesses (SSP is a "just a reg"),
-	 * and this API is used only for implicit accesses, i.e. not RDMSR, and
-	 * so the index is fully KVM-controlled.
-	 */
-	if (unlikely(msr_index == MSR_KVM_INTERNAL_GUEST_SSP))
-		return kvm_msr_read(emul_to_vcpu(ctxt), msr_index, pdata);
-
-	return __kvm_emulate_msr_read(emul_to_vcpu(ctxt), msr_index, pdata);
+	return kvm_emulator_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
 }
 
 static int emulator_check_rdpmc_early(struct x86_emulate_ctxt *ctxt, u32 pmc)
@@ -10014,6 +6923,22 @@ static void kvm_setup_efer_caps(void)
 		kvm_enable_efer_bits(EFER_AUTOIBRS);
 }
 
+static void kvm_nested_ops_update(const struct kvm_x86_nested_ops *nested_ops)
+{
+	memcpy(&kvm_nested_ops, nested_ops, sizeof(kvm_nested_ops));
+
+#define __KVM_X86_NESTED_OP(func) \
+	static_call_update(kvm_x86_nested_##func, kvm_nested_ops.func);
+#define KVM_X86_NESTED_OP(func) \
+	WARN_ON(!kvm_nested_ops.func); __KVM_X86_NESTED_OP(func)
+#define KVM_X86_NESTED_OP_OPTIONAL __KVM_X86_NESTED_OP
+#define KVM_X86_NESTED_OP_OPTIONAL_RET0(func) \
+	static_call_update(kvm_x86_nested_##func, (void *)kvm_nested_ops.func ? : \
+						  (void *)__static_call_return0);
+#include <asm/kvm-x86-nested-ops.h>
+#undef __KVM_X86_NESTED_OP
+}
+
 static inline void kvm_ops_update(struct kvm_x86_init_ops *ops)
 {
 	memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
@@ -10029,6 +6954,8 @@ static inline void kvm_ops_update(struct kvm_x86_init_ops *ops)
 #include <asm/kvm-x86-ops.h>
 #undef __KVM_X86_OP
 
+	kvm_nested_ops_update(ops->nested_ops);
+
 	kvm_pmu_ops_update(ops->pmu_ops);
 }
 
@@ -10565,11 +7492,11 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu)
 int kvm_check_nested_events(struct kvm_vcpu *vcpu)
 {
 	if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
-		kvm_x86_ops.nested_ops->triple_fault(vcpu);
+		kvm_nested_call(triple_fault)(vcpu);
 		return 1;
 	}
 
-	return kvm_x86_ops.nested_ops->check_events(vcpu);
+	return kvm_nested_call(check_events)(vcpu);
 }
 
 static void kvm_inject_exception(struct kvm_vcpu *vcpu)
@@ -10807,9 +7734,7 @@ static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu,
 			kvm_x86_call(enable_irq_window)(vcpu);
 	}
 
-	if (is_guest_mode(vcpu) &&
-	    kvm_x86_ops.nested_ops->has_events &&
-	    kvm_x86_ops.nested_ops->has_events(vcpu, true))
+	if (is_guest_mode(vcpu) && kvm_nested_call(has_events)(vcpu, true))
 		*req_immediate_exit = true;
 
 	/*
@@ -11132,7 +8057,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 		}
 
 		if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {
-			if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) {
+			if (unlikely(!kvm_nested_call(get_nested_state_pages)(vcpu))) {
 				r = 0;
 				goto out;
 			}
@@ -11184,7 +8109,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 		}
 		if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
 			if (is_guest_mode(vcpu))
-				kvm_x86_ops.nested_ops->triple_fault(vcpu);
+				kvm_nested_call(triple_fault)(vcpu);
 
 			if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
 				vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
@@ -11278,6 +8203,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 				goto out;
 			}
 		}
+		if (kvm_check_request(KVM_REQ_VMSA_PAGE_RELOAD, vcpu))
+			kvm_x86_call(reload_vmsa)(vcpu);
 	}
 
 	if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win ||
@@ -11596,15 +8523,13 @@ bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
 	if (kvm_test_request(KVM_REQ_UPDATE_PROTECTED_GUEST_STATE, vcpu))
 		return true;
 
-	if (kvm_arch_interrupt_allowed(vcpu) && kvm_cpu_has_interrupt(vcpu))
+	if (kvm_is_interrupt_allowed(vcpu) && kvm_cpu_has_interrupt(vcpu))
 		return true;
 
 	if (kvm_hv_has_stimer_pending(vcpu))
 		return true;
 
-	if (is_guest_mode(vcpu) &&
-	    kvm_x86_ops.nested_ops->has_events &&
-	    kvm_x86_ops.nested_ops->has_events(vcpu, false))
+	if (is_guest_mode(vcpu) && kvm_nested_call(has_events)(vcpu, false))
 		return true;
 
 	if (kvm_xen_has_pending_events(vcpu))
@@ -11902,28 +8827,6 @@ static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
 	return 0;
 }
 
-/* Swap (qemu) user FPU context for the guest FPU context. */
-static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
-{
-	if (KVM_BUG_ON(vcpu->arch.guest_fpu.fpstate->in_use, vcpu->kvm))
-		return;
-
-	/* Exclude PKRU, it's restored separately immediately after VM-Exit. */
-	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true);
-	trace_kvm_fpu(1);
-}
-
-/* When vcpu_run ends, restore user space FPU context. */
-static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
-{
-	if (KVM_BUG_ON(!vcpu->arch.guest_fpu.fpstate->in_use, vcpu->kvm))
-		return;
-
-	fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false);
-	++vcpu->stat.fpu_reload;
-	trace_kvm_fpu(0);
-}
-
 static int kvm_x86_vcpu_pre_run(struct kvm_vcpu *vcpu)
 {
 	/*
@@ -12029,8 +8932,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 	 * a pending VM-Exit if L1 wants to intercept the exception.
 	 */
 	if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) &&
-	    kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, ex->vector,
-							ex->error_code)) {
+	    kvm_nested_call(is_exception_vmexit)(vcpu, ex->vector, ex->error_code)) {
 		kvm_queue_exception_vmexit(vcpu, ex->vector,
 					   ex->has_error_code, ex->error_code,
 					   ex->has_payload, ex->payload);
@@ -12073,179 +8975,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 	return r;
 }
 
-static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
-{
-	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
-		/*
-		 * We are here if userspace calls get_regs() in the middle of
-		 * instruction emulation. Registers state needs to be copied
-		 * back from emulation context to vcpu. Userspace shouldn't do
-		 * that usually, but some bad designed PV devices (vmware
-		 * backdoor interface) need this to work
-		 */
-		emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);
-		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
-	}
-	regs->rax = kvm_rax_read_raw(vcpu);
-	regs->rbx = kvm_rbx_read_raw(vcpu);
-	regs->rcx = kvm_rcx_read_raw(vcpu);
-	regs->rdx = kvm_rdx_read_raw(vcpu);
-	regs->rsi = kvm_rsi_read_raw(vcpu);
-	regs->rdi = kvm_rdi_read_raw(vcpu);
-	regs->rsp = kvm_rsp_read(vcpu);
-	regs->rbp = kvm_rbp_read_raw(vcpu);
-#ifdef CONFIG_X86_64
-	regs->r8 = kvm_r8_read_raw(vcpu);
-	regs->r9 = kvm_r9_read_raw(vcpu);
-	regs->r10 = kvm_r10_read_raw(vcpu);
-	regs->r11 = kvm_r11_read_raw(vcpu);
-	regs->r12 = kvm_r12_read_raw(vcpu);
-	regs->r13 = kvm_r13_read_raw(vcpu);
-	regs->r14 = kvm_r14_read_raw(vcpu);
-	regs->r15 = kvm_r15_read_raw(vcpu);
-#endif
-
-	regs->rip = kvm_rip_read(vcpu);
-	regs->rflags = kvm_get_rflags(vcpu);
-}
-
-int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
-{
-	if (vcpu->kvm->arch.has_protected_state &&
-	    vcpu->arch.guest_state_protected)
-		return -EINVAL;
-
-	vcpu_load(vcpu);
-	__get_regs(vcpu, regs);
-	vcpu_put(vcpu);
-	return 0;
-}
-
-static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
-{
-	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
-	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
-
-	kvm_rax_write_raw(vcpu, regs->rax);
-	kvm_rbx_write_raw(vcpu, regs->rbx);
-	kvm_rcx_write_raw(vcpu, regs->rcx);
-	kvm_rdx_write_raw(vcpu, regs->rdx);
-	kvm_rsi_write_raw(vcpu, regs->rsi);
-	kvm_rdi_write_raw(vcpu, regs->rdi);
-	kvm_rsp_write(vcpu, regs->rsp);
-	kvm_rbp_write_raw(vcpu, regs->rbp);
-#ifdef CONFIG_X86_64
-	kvm_r8_write_raw(vcpu, regs->r8);
-	kvm_r9_write_raw(vcpu, regs->r9);
-	kvm_r10_write_raw(vcpu, regs->r10);
-	kvm_r11_write_raw(vcpu, regs->r11);
-	kvm_r12_write_raw(vcpu, regs->r12);
-	kvm_r13_write_raw(vcpu, regs->r13);
-	kvm_r14_write_raw(vcpu, regs->r14);
-	kvm_r15_write_raw(vcpu, regs->r15);
-#endif
-
-	kvm_rip_write(vcpu, regs->rip);
-	kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);
-
-	vcpu->arch.exception.pending = false;
-	vcpu->arch.exception_vmexit.pending = false;
-
-	kvm_make_request(KVM_REQ_EVENT, vcpu);
-}
-
-int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
-{
-	if (vcpu->kvm->arch.has_protected_state &&
-	    vcpu->arch.guest_state_protected)
-		return -EINVAL;
-
-	vcpu_load(vcpu);
-	__set_regs(vcpu, regs);
-	vcpu_put(vcpu);
-	return 0;
-}
-
-static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
-{
-	struct desc_ptr dt;
-
-	if (vcpu->arch.guest_state_protected)
-		goto skip_protected_regs;
-
-	kvm_handle_exception_payload_quirk(vcpu);
-
-	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
-	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
-	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
-	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
-	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
-	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
-
-	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
-	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
-
-	kvm_x86_call(get_idt)(vcpu, &dt);
-	sregs->idt.limit = dt.size;
-	sregs->idt.base = dt.address;
-	kvm_x86_call(get_gdt)(vcpu, &dt);
-	sregs->gdt.limit = dt.size;
-	sregs->gdt.base = dt.address;
-
-	sregs->cr2 = vcpu->arch.cr2;
-	sregs->cr3 = kvm_read_cr3(vcpu);
-
-skip_protected_regs:
-	sregs->cr0 = kvm_read_cr0(vcpu);
-	sregs->cr4 = kvm_read_cr4(vcpu);
-	sregs->cr8 = kvm_get_cr8(vcpu);
-	sregs->efer = vcpu->arch.efer;
-	sregs->apic_base = vcpu->arch.apic_base;
-}
-
-static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
-{
-	__get_sregs_common(vcpu, sregs);
-
-	if (vcpu->arch.guest_state_protected)
-		return;
-
-	if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)
-		set_bit(vcpu->arch.interrupt.nr,
-			(unsigned long *)sregs->interrupt_bitmap);
-}
-
-static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)
-{
-	int i;
-
-	__get_sregs_common(vcpu, (struct kvm_sregs *)sregs2);
-
-	if (vcpu->arch.guest_state_protected)
-		return;
-
-	if (is_pae_paging(vcpu)) {
-		kvm_vcpu_srcu_read_lock(vcpu);
-		for (i = 0 ; i < 4 ; i++)
-			sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i);
-		sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID;
-		kvm_vcpu_srcu_read_unlock(vcpu);
-	}
-}
-
-int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
-				  struct kvm_sregs *sregs)
-{
-	if (vcpu->kvm->arch.has_protected_state &&
-	    vcpu->arch.guest_state_protected)
-		return -EINVAL;
-
-	vcpu_load(vcpu);
-	__get_sregs(vcpu, sregs);
-	vcpu_put(vcpu);
-	return 0;
-}
-
 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 				    struct kvm_mp_state *mp_state)
 {
@@ -12365,173 +9094,6 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_task_switch);
 
-static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
-{
-	if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) {
-		/*
-		 * When EFER.LME and CR0.PG are set, the processor is in
-		 * 64-bit mode (though maybe in a 32-bit code segment).
-		 * CR4.PAE and EFER.LMA must be set.
-		 */
-		if (!(sregs->cr4 & X86_CR4_PAE) || !(sregs->efer & EFER_LMA))
-			return false;
-		if (!kvm_vcpu_is_legal_cr3(vcpu, sregs->cr3))
-			return false;
-	} else {
-		/*
-		 * Not in 64-bit mode: EFER.LMA is clear and the code
-		 * segment cannot be 64-bit.
-		 */
-		if (sregs->efer & EFER_LMA || sregs->cs.l)
-			return false;
-	}
-
-	return kvm_is_valid_cr4(vcpu, sregs->cr4) &&
-	       kvm_is_valid_cr0(vcpu, sregs->cr0);
-}
-
-static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs,
-		int *mmu_reset_needed, bool update_pdptrs)
-{
-	int idx;
-	struct desc_ptr dt;
-
-	if (!kvm_is_valid_sregs(vcpu, sregs))
-		return -EINVAL;
-
-	if (kvm_apic_set_base(vcpu, sregs->apic_base, true))
-		return -EINVAL;
-
-	if (vcpu->arch.guest_state_protected)
-		return 0;
-
-	dt.size = sregs->idt.limit;
-	dt.address = sregs->idt.base;
-	kvm_x86_call(set_idt)(vcpu, &dt);
-	dt.size = sregs->gdt.limit;
-	dt.address = sregs->gdt.base;
-	kvm_x86_call(set_gdt)(vcpu, &dt);
-
-	vcpu->arch.cr2 = sregs->cr2;
-	*mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
-	vcpu->arch.cr3 = sregs->cr3;
-	kvm_register_mark_dirty(vcpu, VCPU_REG_CR3);
-	kvm_x86_call(post_set_cr3)(vcpu, sregs->cr3);
-
-	*mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
-	kvm_x86_call(set_efer)(vcpu, sregs->efer);
-
-	*mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
-	kvm_x86_call(set_cr0)(vcpu, sregs->cr0);
-
-	*mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
-	kvm_x86_call(set_cr4)(vcpu, sregs->cr4);
-
-	if (update_pdptrs) {
-		idx = srcu_read_lock(&vcpu->kvm->srcu);
-		if (is_pae_paging(vcpu)) {
-			load_pdptrs(vcpu, kvm_read_cr3(vcpu));
-			*mmu_reset_needed = 1;
-		}
-		srcu_read_unlock(&vcpu->kvm->srcu, idx);
-	}
-
-	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
-	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
-	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
-	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
-	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
-	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
-
-	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
-	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
-
-	kvm_set_cr8(vcpu, sregs->cr8);
-
-	/* Older userspace won't unhalt the vcpu on reset. */
-	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
-	    sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
-	    !is_protmode(vcpu))
-		kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE);
-
-	return 0;
-}
-
-static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
-{
-	int pending_vec, max_bits;
-	int mmu_reset_needed = 0;
-	int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true);
-
-	if (ret)
-		return ret;
-
-	if (mmu_reset_needed) {
-		kvm_mmu_reset_context(vcpu);
-		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
-	}
-
-	max_bits = KVM_NR_INTERRUPTS;
-	pending_vec = find_first_bit(
-		(const unsigned long *)sregs->interrupt_bitmap, max_bits);
-
-	if (pending_vec < max_bits) {
-		kvm_queue_interrupt(vcpu, pending_vec, false);
-		pr_debug("Set back pending irq %d\n", pending_vec);
-		kvm_make_request(KVM_REQ_EVENT, vcpu);
-	}
-	return 0;
-}
-
-static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)
-{
-	int mmu_reset_needed = 0;
-	bool valid_pdptrs = sregs2->flags & KVM_SREGS2_FLAGS_PDPTRS_VALID;
-	bool pae = (sregs2->cr0 & X86_CR0_PG) && (sregs2->cr4 & X86_CR4_PAE) &&
-		!(sregs2->efer & EFER_LMA);
-	int i, ret;
-
-	if (sregs2->flags & ~KVM_SREGS2_FLAGS_PDPTRS_VALID)
-		return -EINVAL;
-
-	if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected))
-		return -EINVAL;
-
-	ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2,
-				 &mmu_reset_needed, !valid_pdptrs);
-	if (ret)
-		return ret;
-
-	if (valid_pdptrs) {
-		for (i = 0; i < 4 ; i++)
-			kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]);
-
-		kvm_register_mark_dirty(vcpu, VCPU_REG_PDPTR);
-		mmu_reset_needed = 1;
-		vcpu->arch.pdptrs_from_userspace = true;
-	}
-	if (mmu_reset_needed) {
-		kvm_mmu_reset_context(vcpu);
-		kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
-	}
-	return 0;
-}
-
-int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
-				  struct kvm_sregs *sregs)
-{
-	int ret;
-
-	if (vcpu->kvm->arch.has_protected_state &&
-	    vcpu->arch.guest_state_protected)
-		return -EINVAL;
-
-	vcpu_load(vcpu);
-	ret = __set_sregs(vcpu, sregs);
-	vcpu_put(vcpu);
-	return ret;
-}
-
 static void kvm_arch_vcpu_guestdbg_update_apicv_inhibit(struct kvm *kvm)
 {
 	bool set = false;
@@ -12687,13 +9249,7 @@ int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 
 static void store_regs(struct kvm_vcpu *vcpu)
 {
-	BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES);
-
-	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)
-		__get_regs(vcpu, &vcpu->run->s.regs.regs);
-
-	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)
-		__get_sregs(vcpu, &vcpu->run->s.regs.sregs);
+	kvm_run_sync_regs_to_user(vcpu);
 
 	if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)
 		kvm_vcpu_ioctl_x86_get_vcpu_events(
@@ -12702,19 +9258,8 @@ static void store_regs(struct kvm_vcpu *vcpu)
 
 static int sync_regs(struct kvm_vcpu *vcpu)
 {
-	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {
-		__set_regs(vcpu, &vcpu->run->s.regs.regs);
-		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;
-	}
-
-	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {
-		struct kvm_sregs sregs = vcpu->run->s.regs.sregs;
-
-		if (__set_sregs(vcpu, &sregs))
-			return -EINVAL;
-
-		vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;
-	}
+	if (kvm_run_sync_regs_from_user(vcpu))
+		return -EINVAL;
 
 	if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {
 		struct kvm_vcpu_events events = vcpu->run->s.regs.events;
@@ -13457,13 +10002,13 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
 	if (kvm->arch.created_mediated_pmu)
 		perf_release_mediated_pmu();
 	kvm_destroy_vcpus(kvm);
-	kvm_free_msr_filter(srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1));
+	kvm_free_msr_filter((void * __force)kvm->arch.msr_filter);
 #ifdef CONFIG_KVM_IOAPIC
 	kvm_pic_destroy(kvm);
 	kvm_ioapic_destroy(kvm);
 #endif
 	kvfree(rcu_dereference_check(kvm->arch.apic_map, 1));
-	kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));
+	kfree((void * __force)kvm->arch.pmu_event_filter);
 	kvm_mmu_uninit_vm(kvm);
 	kvm_page_track_cleanup(kvm);
 	kvm_xen_destroy_vm(kvm);
@@ -13811,56 +10356,6 @@ int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
 	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
 }
 
-int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
-{
-	return kvm_x86_call(interrupt_allowed)(vcpu, false);
-}
-
-unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
-{
-	/* Can't read the RIP when guest state is protected, just return 0 */
-	if (vcpu->arch.guest_state_protected)
-		return 0;
-
-	if (is_64_bit_mode(vcpu))
-		return kvm_rip_read(vcpu);
-	return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +
-		     kvm_rip_read(vcpu));
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_linear_rip);
-
-bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
-{
-	return kvm_get_linear_rip(vcpu) == linear_rip;
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_is_linear_rip);
-
-unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
-{
-	unsigned long rflags;
-
-	rflags = kvm_x86_call(get_rflags)(vcpu);
-	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
-		rflags &= ~X86_EFLAGS_TF;
-	return rflags;
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_rflags);
-
-static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
-{
-	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
-	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
-		rflags |= X86_EFLAGS_TF;
-	kvm_x86_call(set_rflags)(vcpu, rflags);
-}
-
-void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
-{
-	__kvm_set_rflags(vcpu, rflags);
-	kvm_make_request(KVM_REQ_EVENT, vcpu);
-}
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_rflags);
-
 static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
 {
 	BUILD_BUG_ON(!is_power_of_2(ASYNC_PF_PER_VCPU));
@@ -13996,7 +10491,7 @@ bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)
 	 * If interrupts are off we cannot even use an artificial
 	 * halt state.
 	 */
-	return kvm_arch_interrupt_allowed(vcpu);
+	return kvm_is_interrupt_allowed(vcpu);
 }
 
 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
@@ -14136,46 +10631,24 @@ void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end)
 {
 	kvm_x86_call(gmem_invalidate)(start, end);
 }
-#endif
-#endif
-
-int kvm_spec_ctrl_test_value(u64 value)
+void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range)
 {
-	/*
-	 * test that setting IA32_SPEC_CTRL to given value
-	 * is allowed by the host processor
-	 */
-
-	u64 saved_value;
-	unsigned long flags;
-	int ret = 0;
-
-	local_irq_save(flags);
-
-	if (rdmsrq_safe(MSR_IA32_SPEC_CTRL, &saved_value))
-		ret = 1;
-	else if (wrmsrq_safe(MSR_IA32_SPEC_CTRL, value))
-		ret = 1;
-	else
-		wrmsrq(MSR_IA32_SPEC_CTRL, saved_value);
-
-	local_irq_restore(flags);
-
-	return ret;
+	kvm_x86_call(gmem_invalidate_range)(kvm, range);
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_spec_ctrl_test_value);
+#endif
+#endif
 
 void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code)
 {
-	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
+	struct kvm_pagewalk *gva_walk = &vcpu->arch.gva_walk;
 	struct x86_exception fault;
 	u64 access = error_code &
 		(PFERR_WRITE_MASK | PFERR_FETCH_MASK | PFERR_USER_MASK);
 
 	if (!(error_code & PFERR_PRESENT_MASK) ||
-	    mmu->gva_to_gpa(vcpu, mmu, gva, access, &fault) != INVALID_GPA) {
+	    gva_walk->gva_to_gpa(vcpu, gva_walk, gva, access, &fault) != INVALID_GPA) {
 		/*
-		 * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page
+		 * If gva_walk->gva_to_gpa succeeded, the page
 		 * tables probably do not match the TLB.  Just proceed
 		 * with the error code that the processor gave.
 		 */
@@ -14186,7 +10659,7 @@ void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_c
 		fault.address = gva;
 		fault.async_page_fault = false;
 	}
-	vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault, true);
+	gva_walk->inject_page_fault(vcpu, &fault, true);
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_fixup_and_inject_pf_error);
 
diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
index 9de577ef9c974..0f5919b092e47 100644
--- a/arch/x86/kvm/x86.h
+++ b/arch/x86/kvm/x86.h
@@ -6,53 +6,16 @@
 #include <asm/fpu/xstate.h>
 #include <asm/mce.h>
 #include <asm/pvclock.h>
+#include "msrs.h"
+#include "mmu.h"
 #include "regs.h"
 #include "kvm_emulate.h"
 #include "cpuid.h"
 
 #define KVM_MAX_MCE_BANKS 32
 
-struct kvm_caps {
-	/* control of guest tsc rate supported? */
-	bool has_tsc_control;
-	/* maximum supported tsc_khz for guests */
-	u32  max_guest_tsc_khz;
-	/* number of bits of the fractional part of the TSC scaling ratio */
-	u8   tsc_scaling_ratio_frac_bits;
-	/* maximum allowed value of TSC scaling ratio */
-	u64  max_tsc_scaling_ratio;
-	/* 1ull << kvm_caps.tsc_scaling_ratio_frac_bits */
-	u64  default_tsc_scaling_ratio;
-	/* bus lock detection supported? */
-	bool has_bus_lock_exit;
-	/* notify VM exit supported? */
-	bool has_notify_vmexit;
-	/* bit mask of VM types */
-	u32 supported_vm_types;
-
-	u64 supported_mce_cap;
-	u64 supported_xcr0;
-	u64 supported_xss;
-	u64 supported_perf_cap;
-
-	u64 supported_quirks;
-	u64 inapplicable_quirks;
-};
-
-struct kvm_host_values {
-	/*
-	 * The host's raw MAXPHYADDR, i.e. the number of non-reserved physical
-	 * address bits irrespective of features that repurpose legal bits,
-	 * e.g. MKTME.
-	 */
-	u8 maxphyaddr;
-
-	u64 efer;
-	u64 xcr0;
-	u64 xss;
-	u64 s_cet;
-	u64 arch_capabilities;
-};
+int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops);
+void kvm_x86_vendor_exit(void);
 
 void kvm_spurious_fault(void);
 
@@ -86,14 +49,6 @@ do {											\
 	failed;								\
 })
 
-/*
- * The first...last VMX feature MSRs that are emulated by KVM.  This may or may
- * not cover all known VMX MSRs, as KVM doesn't emulate an MSR until there's an
- * associated feature that KVM supports for nested virtualization.
- */
-#define KVM_FIRST_EMULATED_VMX_MSR	MSR_IA32_VMX_BASIC
-#define KVM_LAST_EMULATED_VMX_MSR	MSR_IA32_VMX_VMFUNC
-
 #define KVM_DEFAULT_PLE_GAP		128
 #define KVM_VMX_DEFAULT_PLE_WINDOW	4096
 #define KVM_DEFAULT_PLE_WINDOW_GROW	2
@@ -102,16 +57,6 @@ do {											\
 #define KVM_SVM_DEFAULT_PLE_WINDOW_MAX	USHRT_MAX
 #define KVM_SVM_DEFAULT_PLE_WINDOW	3000
 
-/*
- * KVM's internal, non-ABI indices for synthetic MSRs. The values themselves
- * are arbitrary and have no meaning, the only requirement is that they don't
- * conflict with "real" MSRs that KVM supports. Use values at the upper end
- * of KVM's reserved paravirtual MSR range to minimize churn, i.e. these values
- * will be usable until KVM exhausts its supply of paravirtual MSR indices.
- */
-
-#define MSR_KVM_INTERNAL_GUEST_SSP	0x4b564dff
-
 static inline unsigned int __grow_ple_window(unsigned int val,
 		unsigned int base, unsigned int modifier, unsigned int max)
 {
@@ -142,16 +87,13 @@ static inline unsigned int __shrink_ple_window(unsigned int val,
 	return max(val, min);
 }
 
-#define MSR_IA32_CR_PAT_DEFAULT	\
-	PAT_VALUE(WB, WT, UC_MINUS, UC, WB, WT, UC_MINUS, UC)
-
 void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu);
 int kvm_check_nested_events(struct kvm_vcpu *vcpu);
 
 /* Forcibly leave the nested mode in cases like a vCPU reset */
 static inline void kvm_leave_nested(struct kvm_vcpu *vcpu)
 {
-	kvm_x86_ops.nested_ops->leave_nested(vcpu);
+	kvm_nested_call(leave_nested)(vcpu);
 }
 
 /*
@@ -252,11 +194,6 @@ static inline bool x86_exception_has_error_code(unsigned int vector)
 	return (1U << vector) & exception_has_error_code;
 }
 
-static inline bool mmu_is_nested(struct kvm_vcpu *vcpu)
-{
-	return vcpu->arch.mmu == &vcpu->arch.guest_mmu;
-}
-
 static inline u8 vcpu_virt_addr_bits(struct kvm_vcpu *vcpu)
 {
 	return kvm_is_cr4_bit_set(vcpu, X86_CR4_LA57) ? 57 : 48;
@@ -367,7 +304,7 @@ static inline bool vcpu_match_mmio_gpa(struct kvm_vcpu *vcpu, gpa_t gpa)
 
 static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk)
 {
-	return !(kvm->arch.disabled_quirks & quirk);
+	return !(READ_ONCE(kvm->arch.disabled_quirks) & quirk);
 }
 
 static __always_inline void kvm_request_l1tf_flush_l1d(void)
@@ -384,6 +321,8 @@ static __always_inline void kvm_request_l1tf_flush_l1d(void)
 #endif
 }
 
+void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event);
+
 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip);
 
 u64 get_kvmclock_ns(struct kvm *kvm);
@@ -391,6 +330,29 @@ uint64_t kvm_get_wall_clock_epoch(struct kvm *kvm);
 bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp);
 int kvm_guest_time_update(struct kvm_vcpu *v);
 
+void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value);
+u64 kvm_scale_tsc(u64 tsc, u64 ratio);
+u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc);
+u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier);
+u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier);
+u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc);
+void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset);
+
+static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,
+					   s64 adjustment)
+{
+	kvm_vcpu_write_tsc_offset(vcpu, vcpu->arch.l1_tsc_offset + adjustment);
+}
+
+static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
+{
+	if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio)
+		WARN_ON(adjustment < 0);
+	adjustment = kvm_scale_tsc((u64) adjustment,
+				   vcpu->arch.l1_tsc_scaling_ratio);
+	adjust_tsc_offset_guest(vcpu, adjustment);
+}
+
 int kvm_read_guest_virt(struct kvm_vcpu *vcpu,
 	gva_t addr, void *val, unsigned int bytes,
 	struct x86_exception *exception);
@@ -403,21 +365,316 @@ int handle_ud(struct kvm_vcpu *vcpu);
 
 void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu,
 				   struct kvm_queued_exception *ex);
+void kvm_handle_exception_payload_quirk(struct kvm_vcpu *vcpu);
 
-int kvm_mtrr_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data);
-int kvm_mtrr_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata);
 void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code);
 int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type,
 				    void *insn, int insn_len);
 int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
 			    int emulation_type, void *insn, int insn_len);
-fastpath_t handle_fastpath_wrmsr(struct kvm_vcpu *vcpu);
-fastpath_t handle_fastpath_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg);
+/*
+ * EMULTYPE_NO_DECODE - Set when re-emulating an instruction (after completing
+ *			userspace I/O) to indicate that the emulation context
+ *			should be reused as is, i.e. skip initialization of
+ *			emulation context, instruction fetch and decode.
+ *
+ * EMULTYPE_TRAP_UD - Set when emulating an intercepted #UD from hardware.
+ *		      Indicates that only select instructions (tagged with
+ *		      EmulateOnUD) should be emulated (to minimize the emulator
+ *		      attack surface).  See also EMULTYPE_TRAP_UD_FORCED.
+ *
+ * EMULTYPE_SKIP - Set when emulating solely to skip an instruction, i.e. to
+ *		   decode the instruction length.  For use *only* by
+ *		   kvm_x86_ops.skip_emulated_instruction() implementations if
+ *		   EMULTYPE_COMPLETE_USER_EXIT is not set.
+ *
+ * EMULTYPE_ALLOW_RETRY_PF - Set when the emulator should resume the guest to
+ *			     retry native execution under certain conditions,
+ *			     Can only be set in conjunction with EMULTYPE_PF.
+ *
+ * EMULTYPE_TRAP_UD_FORCED - Set when emulating an intercepted #UD that was
+ *			     triggered by KVM's magic "force emulation" prefix,
+ *			     which is opt in via module param (off by default).
+ *			     Bypasses EmulateOnUD restriction despite emulating
+ *			     due to an intercepted #UD (see EMULTYPE_TRAP_UD).
+ *			     Used to test the full emulator from userspace.
+ *
+ * EMULTYPE_VMWARE_GP - Set when emulating an intercepted #GP for VMware
+ *			backdoor emulation, which is opt in via module param.
+ *			VMware backdoor emulation handles select instructions
+ *			and reinjects the #GP for all other cases.
+ *
+ * EMULTYPE_PF - Set when an intercepted #PF triggers the emulation, in which case
+ *		 the CR2/GPA value pass on the stack is valid.
+ *
+ * EMULTYPE_COMPLETE_USER_EXIT - Set when the emulator should update interruptibility
+ *				 state and inject single-step #DBs after skipping
+ *				 an instruction (after completing userspace I/O).
+ *
+ * EMULTYPE_WRITE_PF_TO_SP - Set when emulating an intercepted page fault that
+ *			     is attempting to write a gfn that contains one or
+ *			     more of the PTEs used to translate the write itself,
+ *			     and the owning page table is being shadowed by KVM.
+ *			     If emulation of the faulting instruction fails and
+ *			     this flag is set, KVM will exit to userspace instead
+ *			     of retrying emulation as KVM cannot make forward
+ *			     progress.
+ *
+ *			     If emulation fails for a write to guest page tables,
+ *			     KVM unprotects (zaps) the shadow page for the target
+ *			     gfn and resumes the guest to retry the non-emulatable
+ *			     instruction (on hardware).  Unprotecting the gfn
+ *			     doesn't allow forward progress for a self-changing
+ *			     access because doing so also zaps the translation for
+ *			     the gfn, i.e. retrying the instruction will hit a
+ *			     !PRESENT fault, which results in a new shadow page
+ *			     and sends KVM back to square one.
+ *
+ * EMULTYPE_SKIP_SOFT_INT - Set in combination with EMULTYPE_SKIP to only skip
+ *                          an instruction if it could generate a given software
+ *                          interrupt, which must be encoded via
+ *                          EMULTYPE_SET_SOFT_INT_VECTOR().
+ */
+#define EMULTYPE_NO_DECODE	    (1 << 0)
+#define EMULTYPE_TRAP_UD	    (1 << 1)
+#define EMULTYPE_SKIP		    (1 << 2)
+#define EMULTYPE_ALLOW_RETRY_PF	    (1 << 3)
+#define EMULTYPE_TRAP_UD_FORCED	    (1 << 4)
+#define EMULTYPE_VMWARE_GP	    (1 << 5)
+#define EMULTYPE_PF		    (1 << 6)
+#define EMULTYPE_COMPLETE_USER_EXIT (1 << 7)
+#define EMULTYPE_WRITE_PF_TO_SP	    (1 << 8)
+#define EMULTYPE_SKIP_SOFT_INT	    (1 << 9)
+
+#define EMULTYPE_SET_SOFT_INT_VECTOR(v)	((u32)((v) & 0xff) << 16)
+#define EMULTYPE_GET_SOFT_INT_VECTOR(e)	(((e) >> 16) & 0xff)
+
+static inline bool kvm_can_emulate_event_vectoring(int emul_type)
+{
+	return !(emul_type & EMULTYPE_PF);
+}
+
+int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type);
+int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
+					void *insn, int insn_len);
+void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu,
+					  u64 *data, u8 ndata);
+void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu);
+
+void kvm_prepare_event_vectoring_exit(struct kvm_vcpu *vcpu, gpa_t gpa);
+void kvm_prepare_unexpected_reason_exit(struct kvm_vcpu *vcpu, u64 exit_reason);
+
 fastpath_t handle_fastpath_hlt(struct kvm_vcpu *vcpu);
 fastpath_t handle_fastpath_invd(struct kvm_vcpu *vcpu);
 
-extern struct kvm_caps kvm_caps;
-extern struct kvm_host_values kvm_host;
+int kvm_emulate_as_nop(struct kvm_vcpu *vcpu);
+int kvm_emulate_invd(struct kvm_vcpu *vcpu);
+int kvm_emulate_mwait(struct kvm_vcpu *vcpu);
+int kvm_handle_invalid_op(struct kvm_vcpu *vcpu);
+int kvm_emulate_monitor(struct kvm_vcpu *vcpu);
+
+int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in);
+int kvm_emulate_cpuid(struct kvm_vcpu *vcpu);
+int kvm_emulate_halt(struct kvm_vcpu *vcpu);
+int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu);
+int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu);
+int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu);
+
+void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector);
+
+enum kvm_task_switch_reason {
+	TASK_SWITCH_CALL = 0,
+	TASK_SWITCH_IRET = 1,
+	TASK_SWITCH_JMP = 2,
+	TASK_SWITCH_GATE = 3,
+};
+int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
+		    int reason, bool has_error_code, u32 error_code);
+
+int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr);
+int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu);
+int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu);
+
+int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu);
+int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err);
+
+void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr);
+void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
+void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr, unsigned long payload);
+void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned int nr,
+			   bool has_error_code, u32 error_code);
+void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault,
+			   bool from_hardware);
+void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
+				      struct x86_exception *fault,
+				      bool from_hardware);
+
+static inline void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
+						  struct x86_exception *fault)
+{
+	__kvm_inject_emulated_page_fault(vcpu, fault, false);
+}
+
+bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr);
+
+static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code)
+{
+	kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
+}
+
+void kvm_inject_nmi(struct kvm_vcpu *vcpu);
+int kvm_get_nr_pending_nmis(struct kvm_vcpu *vcpu);
+
+void __user *__x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa,
+				     u32 size);
+int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages);
+
+bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu);
+bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu);
+
+enum kvm_apicv_inhibit {
+
+	/********************************************************************/
+	/* INHIBITs that are relevant to both Intel's APICv and AMD's AVIC. */
+	/********************************************************************/
+
+	/*
+	 * APIC acceleration is disabled by a module parameter
+	 * and/or not supported in hardware.
+	 */
+	APICV_INHIBIT_REASON_DISABLED,
+
+	/*
+	 * APIC acceleration is inhibited because AutoEOI feature is
+	 * being used by a HyperV guest.
+	 */
+	APICV_INHIBIT_REASON_HYPERV,
+
+	/*
+	 * APIC acceleration is inhibited because the userspace didn't yet
+	 * enable the kernel/split irqchip.
+	 */
+	APICV_INHIBIT_REASON_ABSENT,
+
+	/* APIC acceleration is inhibited because KVM_GUESTDBG_BLOCKIRQ
+	 * (out of band, debug measure of blocking all interrupts on this vCPU)
+	 * was enabled, to avoid AVIC/APICv bypassing it.
+	 */
+	APICV_INHIBIT_REASON_BLOCKIRQ,
+
+	/*
+	 * APICv is disabled because not all vCPUs have a 1:1 mapping between
+	 * APIC ID and vCPU, _and_ KVM is not applying its x2APIC hotplug hack.
+	 */
+	APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED,
+
+	/*
+	 * For simplicity, the APIC acceleration is inhibited
+	 * first time either APIC ID or APIC base are changed by the guest
+	 * from their reset values.
+	 */
+	APICV_INHIBIT_REASON_APIC_ID_MODIFIED,
+	APICV_INHIBIT_REASON_APIC_BASE_MODIFIED,
+
+	/******************************************************/
+	/* INHIBITs that are relevant only to the AMD's AVIC. */
+	/******************************************************/
+
+	/*
+	 * AVIC is inhibited on a vCPU because it runs a nested guest.
+	 *
+	 * This is needed because unlike APICv, the peers of this vCPU
+	 * cannot use the doorbell mechanism to signal interrupts via AVIC when
+	 * a vCPU runs nested.
+	 */
+	APICV_INHIBIT_REASON_NESTED,
+
+	/*
+	 * On SVM, the wait for the IRQ window is implemented with pending vIRQ,
+	 * which cannot be injected when the AVIC is enabled, thus AVIC
+	 * is inhibited while KVM waits for IRQ window.
+	 */
+	APICV_INHIBIT_REASON_IRQWIN,
+
+	/*
+	 * PIT (i8254) 're-inject' mode, relies on EOI intercept,
+	 * which AVIC doesn't support for edge triggered interrupts.
+	 */
+	APICV_INHIBIT_REASON_PIT_REINJ,
+
+	/*
+	 * AVIC is disabled because SEV doesn't support it.
+	 */
+	APICV_INHIBIT_REASON_SEV,
+
+	/*
+	 * AVIC is disabled because not all vCPUs with a valid LDR have a 1:1
+	 * mapping between logical ID and vCPU.
+	 */
+	APICV_INHIBIT_REASON_LOGICAL_ID_ALIASED,
+
+	/*
+	 * AVIC is disabled because the vCPU's APIC ID is beyond the max
+	 * supported by AVIC/x2AVIC, i.e. the vCPU is unaddressable.
+	 */
+	APICV_INHIBIT_REASON_PHYSICAL_ID_TOO_BIG,
+
+	NR_APICV_INHIBIT_REASONS,
+};
+
+#define __APICV_INHIBIT_REASON(reason)			\
+	{ BIT(APICV_INHIBIT_REASON_##reason), #reason }
+
+#define APICV_INHIBIT_REASONS				\
+	__APICV_INHIBIT_REASON(DISABLED),		\
+	__APICV_INHIBIT_REASON(HYPERV),			\
+	__APICV_INHIBIT_REASON(ABSENT),			\
+	__APICV_INHIBIT_REASON(BLOCKIRQ),		\
+	__APICV_INHIBIT_REASON(PHYSICAL_ID_ALIASED),	\
+	__APICV_INHIBIT_REASON(APIC_ID_MODIFIED),	\
+	__APICV_INHIBIT_REASON(APIC_BASE_MODIFIED),	\
+	__APICV_INHIBIT_REASON(NESTED),			\
+	__APICV_INHIBIT_REASON(IRQWIN),			\
+	__APICV_INHIBIT_REASON(PIT_REINJ),		\
+	__APICV_INHIBIT_REASON(SEV),			\
+	__APICV_INHIBIT_REASON(LOGICAL_ID_ALIASED),	\
+	__APICV_INHIBIT_REASON(PHYSICAL_ID_TOO_BIG)
+
+bool kvm_apicv_activated(struct kvm *kvm);
+bool kvm_vcpu_apicv_activated(struct kvm_vcpu *vcpu);
+void __kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu);
+void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
+				      enum kvm_apicv_inhibit reason, bool set);
+void kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
+				    enum kvm_apicv_inhibit reason, bool set);
+
+static inline void kvm_set_apicv_inhibit(struct kvm *kvm,
+					 enum kvm_apicv_inhibit reason)
+{
+	kvm_set_or_clear_apicv_inhibit(kvm, reason, true);
+}
+
+static inline void kvm_clear_apicv_inhibit(struct kvm *kvm,
+					   enum kvm_apicv_inhibit reason)
+{
+	kvm_set_or_clear_apicv_inhibit(kvm, reason, false);
+}
+
+void kvm_inc_or_dec_irq_window_inhibit(struct kvm *kvm, bool inc);
+
+static inline void kvm_inc_apicv_irq_window_req(struct kvm *kvm)
+{
+	kvm_inc_or_dec_irq_window_inhibit(kvm, true);
+}
+
+static inline void kvm_dec_apicv_irq_window_req(struct kvm *kvm)
+{
+	kvm_inc_or_dec_irq_window_inhibit(kvm, false);
+}
+
+void kvm_make_scan_ioapic_request(struct kvm *kvm);
+void kvm_make_scan_ioapic_request_mask(struct kvm *kvm,
+				       unsigned long *vcpu_bitmap);
 
 void kvm_setup_xss_caps(void);
 
@@ -461,22 +718,6 @@ extern bool enable_vmware_backdoor;
 
 extern int pi_inject_timer;
 
-extern bool report_ignored_msrs;
-
-extern bool eager_page_split;
-
-static inline void kvm_pr_unimpl_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
-{
-	if (report_ignored_msrs)
-		vcpu_unimpl(vcpu, "Unhandled WRMSR(0x%x) = 0x%llx\n", msr, data);
-}
-
-static inline void kvm_pr_unimpl_rdmsr(struct kvm_vcpu *vcpu, u32 msr)
-{
-	if (report_ignored_msrs)
-		vcpu_unimpl(vcpu, "Unhandled RDMSR(0x%x)\n", msr);
-}
-
 static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec)
 {
 	return pvclock_scale_delta(nsec, vcpu->arch.virtual_tsc_mult,
@@ -575,6 +816,8 @@ static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
 		vcpu->arch.apf.gfns[i] = ~0;
 }
 
+bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn);
+
 /*
  * Trigger machine check on the host. We assume all the MSRs are already set up
  * by the CPU and that we still run on the same CPU as the MCE occurred on.
@@ -594,32 +837,10 @@ static inline void kvm_machine_check(void)
 #endif
 }
 
-int kvm_spec_ctrl_test_value(u64 value);
 int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r,
 			      struct x86_exception *e);
+void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid);
 int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva);
-bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type);
-
-enum kvm_msr_access {
-	MSR_TYPE_R	= BIT(0),
-	MSR_TYPE_W	= BIT(1),
-	MSR_TYPE_RW	= MSR_TYPE_R | MSR_TYPE_W,
-};
-
-/*
- * Internal error codes that are used to indicate that MSR emulation encountered
- * an error that should result in #GP in the guest, unless userspace handles it.
- * Note, '1', '0', and negative numbers are off limits, as they are used by KVM
- * as part of KVM's lightly documented internal KVM_RUN return codes.
- *
- * UNSUPPORTED	- The MSR isn't supported, either because it is completely
- *		  unknown to KVM, or because the MSR should not exist according
- *		  to the vCPU model.
- *
- * FILTERED	- Access to the MSR is denied by a userspace MSR filter.
- */
-#define  KVM_MSR_RET_UNSUPPORTED	2
-#define  KVM_MSR_RET_FILTERED		3
 
 int kvm_sev_es_mmio(struct kvm_vcpu *vcpu, bool is_write, gpa_t gpa,
 		    unsigned int bytes, void *data);
@@ -679,27 +900,4 @@ int ____kvm_emulate_hypercall(struct kvm_vcpu *vcpu, int cpl,
 
 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu);
 
-#define CET_US_RESERVED_BITS		GENMASK(9, 6)
-#define CET_US_SHSTK_MASK_BITS		GENMASK(1, 0)
-#define CET_US_IBT_MASK_BITS		(GENMASK_ULL(5, 2) | GENMASK_ULL(63, 10))
-#define CET_US_LEGACY_BITMAP_BASE(data)	((data) >> 12)
-
-static inline bool kvm_is_valid_u_s_cet(struct kvm_vcpu *vcpu, u64 data)
-{
-	if (data & CET_US_RESERVED_BITS)
-		return false;
-	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) &&
-	    (data & CET_US_SHSTK_MASK_BITS))
-		return false;
-	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_IBT) &&
-	    (data & CET_US_IBT_MASK_BITS))
-		return false;
-	if (!IS_ALIGNED(CET_US_LEGACY_BITMAP_BASE(data), 4))
-		return false;
-	/* IBT can be suppressed iff the TRACKER isn't WAIT_ENDBR. */
-	if ((data & CET_SUPPRESS) && (data & CET_WAIT_ENDBR))
-		return false;
-
-	return true;
-}
 #endif
diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c
index 694b31c1fcc9a..f93c9c42e03b6 100644
--- a/arch/x86/kvm/xen.c
+++ b/arch/x86/kvm/xen.c
@@ -22,6 +22,7 @@
 #include <xen/interface/version.h>
 #include <xen/interface/event_channel.h>
 #include <xen/interface/sched.h>
+#include <xen/xen-ops.h>
 
 #include <asm/xen/cpuid.h>
 #include <asm/pvclock.h>
@@ -212,7 +213,7 @@ static void kvm_xen_start_timer(struct kvm_vcpu *vcpu, u64 guest_abs,
 		struct pvclock_vcpu_time_info hv_clock;
 		uint64_t host_tsc, guest_tsc;
 
-		if (!static_cpu_has(X86_FEATURE_CONSTANT_TSC) ||
+		if (!cpu_feature_enabled(X86_FEATURE_CONSTANT_TSC) ||
 		    !vcpu->kvm->arch.use_master_clock)
 			break;
 
@@ -1103,6 +1104,8 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data)
 		break;
 
 	case KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID:
+		BUILD_BUG_ON(XEN_VCPU_ID_INVALID < KVM_MAX_VCPUS);
+
 		if (data->u.vcpu_id >= KVM_MAX_VCPUS)
 			r = -EINVAL;
 		else {
@@ -1607,16 +1610,28 @@ static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd,
 	struct vcpu_set_singleshot_timer oneshot;
 	struct x86_exception e;
 
+	if (cmd != VCPUOP_set_singleshot_timer &&
+	    cmd != VCPUOP_stop_singleshot_timer)
+		return false;
+
 	if (!kvm_xen_timer_enabled(vcpu))
 		return false;
 
-	switch (cmd) {
-	case VCPUOP_set_singleshot_timer:
-		if (vcpu->arch.xen.vcpu_id != vcpu_id) {
-			*r = -EINVAL;
-			return true;
-		}
+	if (vcpu->arch.xen.vcpu_id == XEN_VCPU_ID_INVALID)
+		return false;
+
+	/*
+	 * Reject the hypercall if the guest is trying to start/stop the timer
+	 * for a different vCPU.  Xen per-vCPU hypercalls take a target vCPU as
+	 * a common parameter, as all per-vCPU hypercalls *except* single-shot
+	 * timer updates can be cross-vCPU.
+	 */
+	if (vcpu->arch.xen.vcpu_id != vcpu_id) {
+		*r = -EINVAL;
+		return true;
+	}
 
+	if (cmd == VCPUOP_set_singleshot_timer) {
 		/*
 		 * The only difference for 32-bit compat is the 4 bytes of
 		 * padding after the interesting part of the structure. So
@@ -1640,20 +1655,12 @@ static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd,
 		}
 
 		kvm_xen_start_timer(vcpu, oneshot.timeout_abs_ns, false);
-		*r = 0;
-		return true;
-
-	case VCPUOP_stop_singleshot_timer:
-		if (vcpu->arch.xen.vcpu_id != vcpu_id) {
-			*r = -EINVAL;
-			return true;
-		}
+	} else {
 		kvm_xen_stop_timer(vcpu);
-		*r = 0;
-		return true;
 	}
 
-	return false;
+	*r = 0;
+	return true;
 }
 
 static bool kvm_xen_hcall_set_timer_op(struct kvm_vcpu *vcpu, uint64_t timeout,
@@ -2299,7 +2306,7 @@ static bool kvm_xen_hcall_evtchn_send(struct kvm_vcpu *vcpu, u64 param, u64 *r)
 
 void kvm_xen_init_vcpu(struct kvm_vcpu *vcpu)
 {
-	vcpu->arch.xen.vcpu_id = vcpu->vcpu_idx;
+	vcpu->arch.xen.vcpu_id = XEN_VCPU_ID_INVALID;
 	vcpu->arch.xen.poll_evtchn = 0;
 
 	timer_setup(&vcpu->arch.xen.poll_timer, cancel_evtchn_poll, 0);
diff --git a/arch/x86/lib/copy_mc.c b/arch/x86/lib/copy_mc.c
index 97e88e58567bf..d509256a228e8 100644
--- a/arch/x86/lib/copy_mc.c
+++ b/arch/x86/lib/copy_mc.c
@@ -70,7 +70,7 @@ unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigne
 		instrument_memcpy_after(dst, src, len, ret);
 		return ret;
 	}
-	if (static_cpu_has(X86_FEATURE_ERMS)) {
+	if (cpu_feature_enabled(X86_FEATURE_ERMS)) {
 		instrument_memcpy_before(dst, src, len);
 		ret = copy_mc_enhanced_fast_string(dst, src, len);
 		instrument_memcpy_after(dst, src, len, ret);
@@ -93,7 +93,7 @@ unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, un
 		return ret;
 	}
 
-	if (static_cpu_has(X86_FEATURE_ERMS)) {
+	if (cpu_feature_enabled(X86_FEATURE_ERMS)) {
 		instrument_copy_to_user(dst, src, len);
 		__uaccess_begin();
 		ret = copy_mc_enhanced_fast_string((__force void *)dst, src, len);
diff --git a/arch/x86/lib/msr-smp.c b/arch/x86/lib/msr-smp.c
index 0c6627773c803..7b6cfc2c09701 100644
--- a/arch/x86/lib/msr-smp.c
+++ b/arch/x86/lib/msr-smp.c
@@ -15,7 +15,7 @@ static void __rdmsr_on_cpu(void *info)
 	else
 		reg = &rv->reg;
 
-	rdmsr(rv->msr_no, reg->l, reg->h);
+	rdmsrq(rv->msr_no, reg->q);
 }
 
 static void __wrmsr_on_cpu(void *info)
@@ -28,7 +28,7 @@ static void __wrmsr_on_cpu(void *info)
 	else
 		reg = &rv->reg;
 
-	wrmsr(rv->msr_no, reg->l, reg->h);
+	wrmsrq(rv->msr_no, reg->q);
 }
 
 int rdmsrq_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
@@ -121,7 +121,7 @@ static void __rdmsr_safe_on_cpu(void *info)
 {
 	struct msr_info_completion *rv = info;
 
-	rv->msr.err = rdmsr_safe(rv->msr.msr_no, &rv->msr.reg.l, &rv->msr.reg.h);
+	rv->msr.err = rdmsrq_safe(rv->msr.msr_no, &rv->msr.reg.q);
 	complete(&rv->done);
 }
 
@@ -129,7 +129,7 @@ static void __wrmsr_safe_on_cpu(void *info)
 {
 	struct msr_info *rv = info;
 
-	rv->err = wrmsr_safe(rv->msr_no, rv->reg.l, rv->reg.h);
+	rv->err = wrmsrq_safe(rv->msr_no, rv->reg.q);
 }
 
 int wrmsrq_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c
index ac27e39fc993d..b4c13493c6455 100644
--- a/arch/x86/lib/usercopy_32.c
+++ b/arch/x86/lib/usercopy_32.c
@@ -328,7 +328,7 @@ unsigned long copy_from_user_inatomic_nontemporal(void *to, const void __user *f
 	if (!user_access_begin(from, n))
 		return n;
 #ifdef CONFIG_X86_INTEL_USERCOPY
-	if (n > 64 && static_cpu_has(X86_FEATURE_XMM2))
+	if (n > 64 && cpu_feature_enabled(X86_FEATURE_XMM2))
 		n = __copy_user_intel_nocache(to, from, n);
 	else
 		__copy_user(to, from, n);
diff --git a/arch/x86/mm/amdtopology.c b/arch/x86/mm/amdtopology.c
index f980b0eb01054..1cb581bbf2b6f 100644
--- a/arch/x86/mm/amdtopology.c
+++ b/arch/x86/mm/amdtopology.c
@@ -150,7 +150,6 @@ int __init amd_numa_init(void)
 
 		prevbase = base;
 		numa_add_memblk(nodeid, base, limit);
-		node_set(nodeid, numa_nodes_parsed);
 	}
 
 	if (nodes_empty(numa_nodes_parsed))
diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c
index ceb8d03191ab2..fa6eebe6a69e0 100644
--- a/arch/x86/mm/extable.c
+++ b/arch/x86/mm/extable.c
@@ -194,7 +194,7 @@ static bool ex_handler_msr(const struct exception_table_entry *fixup,
 static bool ex_handler_clear_fs(const struct exception_table_entry *fixup,
 				struct pt_regs *regs)
 {
-	if (static_cpu_has(X86_BUG_NULL_SEG))
+	if (cpu_feature_enabled(X86_BUG_NULL_SEG))
 		asm volatile ("mov %0, %%fs" : : "rm" (__USER_DS));
 	asm volatile ("mov %0, %%fs" : : "rm" (0));
 	return ex_handler_default(fixup, regs);
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 45b99c3b1442d..aa88370ce7394 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -275,17 +275,17 @@ void arch_sync_kernel_mappings(unsigned long start, unsigned long end)
 	for (addr = start & PMD_MASK;
 	     addr >= TASK_SIZE_MAX && addr < VMALLOC_END;
 	     addr += PMD_SIZE) {
-		struct page *page;
+		struct ptdesc *ptdesc;
 
 		spin_lock(&pgd_lock);
-		list_for_each_entry(page, &pgd_list, lru) {
+		list_for_each_entry(ptdesc, &pgd_list, pt_list) {
 			spinlock_t *pgt_lock;
 
 			/* the pgt_lock only for Xen */
-			pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
+			pgt_lock = &pgd_page_get_mm(ptdesc)->page_table_lock;
 
 			spin_lock(pgt_lock);
-			vmalloc_sync_one(page_address(page), addr);
+			vmalloc_sync_one(ptdesc_address(ptdesc), addr);
 			spin_unlock(pgt_lock);
 		}
 		spin_unlock(&pgd_lock);
diff --git a/arch/x86/mm/ident_map.c b/arch/x86/mm/ident_map.c
index bd5d101c5c379..5a15bffe65749 100644
--- a/arch/x86/mm/ident_map.c
+++ b/arch/x86/mm/ident_map.c
@@ -111,7 +111,7 @@ static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page,
 		use_gbpage = info->direct_gbpages;
 
 		/* Don't use gbpage if it maps more than the requested region. */
-		/* at the begining: */
+		/* at the beginning: */
 		use_gbpage &= ((addr & ~PUD_MASK) == 0);
 		/* ... or at the end: */
 		use_gbpage &= ((next & ~PUD_MASK) == 0);
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index d57f29ca23a5a..70e682180291a 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -33,7 +33,6 @@
 #include <linux/nmi.h>
 #include <linux/gfp.h>
 #include <linux/kcore.h>
-#include <linux/bootmem_info.h>
 
 #include <asm/processor.h>
 #include <asm/bios_ebda.h>
@@ -136,7 +135,7 @@ static void sync_global_pgds_l5(unsigned long start, unsigned long end)
 
 	for (addr = start; addr <= end; addr = ALIGN(addr + 1, PGDIR_SIZE)) {
 		const pgd_t *pgd_ref = pgd_offset_k(addr);
-		struct page *page;
+		struct ptdesc *ptdesc;
 
 		/* Check for overflow */
 		if (addr < start)
@@ -146,13 +145,13 @@ static void sync_global_pgds_l5(unsigned long start, unsigned long end)
 			continue;
 
 		spin_lock(&pgd_lock);
-		list_for_each_entry(page, &pgd_list, lru) {
+		list_for_each_entry(ptdesc, &pgd_list, pt_list) {
 			pgd_t *pgd;
 			spinlock_t *pgt_lock;
 
-			pgd = (pgd_t *)page_address(page) + pgd_index(addr);
+			pgd = (pgd_t *)ptdesc_address(ptdesc) + pgd_index(addr);
 			/* the pgt_lock only for Xen */
-			pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
+			pgt_lock = &pgd_page_get_mm(ptdesc)->page_table_lock;
 			spin_lock(pgt_lock);
 
 			if (!pgd_none(*pgd_ref) && !pgd_none(*pgd))
@@ -174,7 +173,7 @@ static void sync_global_pgds_l4(unsigned long start, unsigned long end)
 	for (addr = start; addr <= end; addr = ALIGN(addr + 1, PGDIR_SIZE)) {
 		pgd_t *pgd_ref = pgd_offset_k(addr);
 		const p4d_t *p4d_ref;
-		struct page *page;
+		struct ptdesc *ptdesc;
 
 		/*
 		 * With folded p4d, pgd_none() is always false, we need to
@@ -187,15 +186,15 @@ static void sync_global_pgds_l4(unsigned long start, unsigned long end)
 			continue;
 
 		spin_lock(&pgd_lock);
-		list_for_each_entry(page, &pgd_list, lru) {
+		list_for_each_entry(ptdesc, &pgd_list, pt_list) {
 			pgd_t *pgd;
 			p4d_t *p4d;
 			spinlock_t *pgt_lock;
 
-			pgd = (pgd_t *)page_address(page) + pgd_index(addr);
+			pgd = (pgd_t *)ptdesc_address(ptdesc) + pgd_index(addr);
 			p4d = p4d_offset(pgd, addr);
 			/* the pgt_lock only for Xen */
-			pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
+			pgt_lock = &pgd_page_get_mm(ptdesc)->page_table_lock;
 			spin_lock(pgt_lock);
 
 			if (!p4d_none(*p4d_ref) && !p4d_none(*p4d))
@@ -1000,32 +999,12 @@ int arch_add_memory(int nid, u64 start, u64 size,
 	return add_pages(nid, start_pfn, nr_pages, params);
 }
 
-static void free_reserved_pages(struct page *page, unsigned long nr_pages)
+static void __meminit free_pagetable(struct page *page)
 {
-	while (nr_pages--)
-		free_reserved_page(page++);
-}
-
-static void __meminit free_pagetable(struct page *page, int order)
-{
-	/* bootmem page has reserved flag */
-	if (PageReserved(page)) {
-		unsigned long nr_pages = 1 << order;
-#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
-		enum bootmem_type type = bootmem_type(page);
-
-		if (type == MIX_SECTION_INFO) {
-			while (nr_pages--)
-				put_page_bootmem(page++);
-		} else {
-			free_reserved_pages(page, nr_pages);
-		}
-#else
-		free_reserved_pages(page, nr_pages);
-#endif
-	} else {
+	if (PageReserved(page))
+		free_reserved_page(page);
+	else
 		pagetable_free(page_ptdesc(page));
-	}
 }
 
 static void __meminit free_vmemmap_pages(struct page *page, unsigned int order,
@@ -1033,19 +1012,12 @@ static void __meminit free_vmemmap_pages(struct page *page, unsigned int order,
 {
 	unsigned long nr_pages = 1u << order;
 
-	if (altmap) {
+	if (altmap)
 		vmem_altmap_free(altmap, nr_pages);
-	} else if (PageReserved(page)) {
-		if (IS_ENABLED(CONFIG_HAVE_BOOTMEM_INFO_NODE) &&
-		    bootmem_type(page) == SECTION_INFO) {
-			while (nr_pages--)
-				put_page_bootmem(page++);
-		} else {
-			free_reserved_pages(page, nr_pages);
-		}
-	} else {
+	else if (PageReserved(page))
+		free_reserved_pages(page, order);
+	else
 		__free_pages(page, order);
-	}
 }
 
 static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd)
@@ -1060,7 +1032,7 @@ static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd)
 	}
 
 	/* free a pte table */
-	free_pagetable(pmd_page(*pmd), 0);
+	free_pagetable(pmd_page(*pmd));
 	spin_lock(&init_mm.page_table_lock);
 	pmd_clear(pmd);
 	spin_unlock(&init_mm.page_table_lock);
@@ -1078,7 +1050,7 @@ static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud)
 	}
 
 	/* free a pmd table */
-	free_pagetable(pud_page(*pud), 0);
+	free_pagetable(pud_page(*pud));
 	spin_lock(&init_mm.page_table_lock);
 	pud_clear(pud);
 	spin_unlock(&init_mm.page_table_lock);
@@ -1096,7 +1068,7 @@ static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d)
 	}
 
 	/* free a pud table */
-	free_pagetable(p4d_page(*p4d), 0);
+	free_pagetable(p4d_page(*p4d));
 	spin_lock(&init_mm.page_table_lock);
 	p4d_clear(p4d);
 	spin_unlock(&init_mm.page_table_lock);
@@ -1313,16 +1285,6 @@ void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
 
 static struct kcore_list kcore_vsyscall;
 
-static void __init register_page_bootmem_info(void)
-{
-#if defined(CONFIG_NUMA) || defined(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP)
-	int i;
-
-	for_each_online_node(i)
-		register_page_bootmem_info_node(NODE_DATA(i));
-#endif
-}
-
 /*
  * Pre-allocates page-table pages for the vmalloc area in the kernel page-table.
  * Only the level which needs to be synchronized between all page-tables is
@@ -1385,14 +1347,6 @@ void __init mem_init(void)
 	after_bootmem = 1;
 	x86_init.hyper.init_after_bootmem();
 
-	/*
-	 * Must be done after boot memory is put on freelist, because here we
-	 * might set fields in deferred struct pages that have not yet been
-	 * initialized, and memblock_free_all() initializes all the reserved
-	 * deferred pages for us.
-	 */
-	register_page_bootmem_info();
-
 	/* Register memory areas for /proc/kcore */
 	if (get_gate_vma(&init_mm))
 		kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR, PAGE_SIZE, KCORE_USER);
@@ -1591,72 +1545,6 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
 	return err;
 }
 
-#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
-void register_page_bootmem_memmap(unsigned long section_nr,
-				  struct page *start_page, unsigned long nr_pages)
-{
-	unsigned long addr = (unsigned long)start_page;
-	unsigned long end = (unsigned long)(start_page + nr_pages);
-	unsigned long next;
-	pgd_t *pgd;
-	p4d_t *p4d;
-	pud_t *pud;
-	pmd_t *pmd;
-	unsigned int nr_pmd_pages;
-	struct page *page;
-
-	for (; addr < end; addr = next) {
-		pte_t *pte = NULL;
-
-		pgd = pgd_offset_k(addr);
-		if (pgd_none(*pgd)) {
-			next = (addr + PAGE_SIZE) & PAGE_MASK;
-			continue;
-		}
-		get_page_bootmem(section_nr, pgd_page(*pgd), MIX_SECTION_INFO);
-
-		p4d = p4d_offset(pgd, addr);
-		if (p4d_none(*p4d)) {
-			next = (addr + PAGE_SIZE) & PAGE_MASK;
-			continue;
-		}
-		get_page_bootmem(section_nr, p4d_page(*p4d), MIX_SECTION_INFO);
-
-		pud = pud_offset(p4d, addr);
-		if (pud_none(*pud)) {
-			next = (addr + PAGE_SIZE) & PAGE_MASK;
-			continue;
-		}
-		get_page_bootmem(section_nr, pud_page(*pud), MIX_SECTION_INFO);
-
-		pmd = pmd_offset(pud, addr);
-		if (pmd_none(*pmd)) {
-			next = (addr + PAGE_SIZE) & PAGE_MASK;
-			continue;
-		}
-
-		if (!boot_cpu_has(X86_FEATURE_PSE) || !pmd_leaf(*pmd)) {
-			next = (addr + PAGE_SIZE) & PAGE_MASK;
-			get_page_bootmem(section_nr, pmd_page(*pmd),
-					 MIX_SECTION_INFO);
-
-			pte = pte_offset_kernel(pmd, addr);
-			if (pte_none(*pte))
-				continue;
-			get_page_bootmem(section_nr, pte_page(*pte),
-					 SECTION_INFO);
-		} else {
-			next = pmd_addr_end(addr, end);
-			nr_pmd_pages = (next - addr) >> PAGE_SHIFT;
-			page = pmd_page(*pmd);
-			while (nr_pmd_pages--)
-				get_page_bootmem(section_nr, page++,
-						 SECTION_INFO);
-		}
-	}
-}
-#endif
-
 void __meminit vmemmap_populate_print_last(void)
 {
 	if (p_start) {
diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c
index 99d0a9332c145..ced66e68a68e6 100644
--- a/arch/x86/mm/numa.c
+++ b/arch/x86/mm/numa.c
@@ -216,7 +216,6 @@ static int __init dummy_numa_init(void)
 	printk(KERN_INFO "Faking a node at [mem %#018Lx-%#018Lx]\n",
 	       0LLU, PFN_PHYS(max_pfn) - 1);
 
-	node_set(0, numa_nodes_parsed);
 	node_set(0, numa_phys_nodes_parsed);
 	numa_add_memblk(0, 0, PFN_PHYS(max_pfn));
 
diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c
index d023a40a1e034..629be8c7998ba 100644
--- a/arch/x86/mm/pat/set_memory.c
+++ b/arch/x86/mm/pat/set_memory.c
@@ -22,6 +22,7 @@
 #include <linux/cc_platform.h>
 #include <linux/set_memory.h>
 #include <linux/memregion.h>
+#include <linux/cleanup.h>
 
 #include <asm/e820/api.h>
 #include <asm/processor.h>
@@ -62,10 +63,9 @@ enum cpa_warn {
 static const int cpa_warn_level = CPA_PROTECT;
 
 /*
- * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings)
- * using cpa_lock. So that we don't allow any other cpu, with stale large tlb
- * entries change the page attribute in parallel to some other cpu
- * splitting a large page entry along with changing the attribute.
+ * Serialize cpa() using cpa_lock so that we don't allow any other cpu, with
+ * stale large tlb entries, to change the page attribute in parallel to some
+ * other cpu splitting a large page entry along with changing the attribute.
  */
 static DEFINE_SPINLOCK(cpa_lock);
 
@@ -410,7 +410,7 @@ static void __cpa_flush_tlb(void *data)
 
 static int collapse_large_pages(unsigned long addr, struct list_head *pgtables);
 
-static void cpa_collapse_large_pages(struct cpa_data *cpa)
+static void __cpa_collapse_large_pages(struct cpa_data *cpa)
 {
 	unsigned long start, addr, end;
 	struct ptdesc *ptdesc, *tmp;
@@ -418,6 +418,8 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa)
 	int collapsed = 0;
 	int i;
 
+	spin_lock(&cpa_lock);
+
 	if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) {
 		for (i = 0; i < cpa->numpages; i++)
 			collapsed += collapse_large_pages(__cpa_addr(cpa, i),
@@ -431,8 +433,10 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa)
 			collapsed += collapse_large_pages(addr, &pgtables);
 	}
 
-	if (!collapsed)
+	if (!collapsed) {
+		spin_unlock(&cpa_lock);
 		return;
+	}
 
 	flush_tlb_all();
 
@@ -440,6 +444,20 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa)
 		list_del(&ptdesc->pt_list);
 		pagetable_free(ptdesc);
 	}
+
+	spin_unlock(&cpa_lock);
+}
+
+static void cpa_collapse_large_pages(struct cpa_data *cpa)
+{
+	/*
+	 * Take the mmap write lock on init_mm to:
+	 * - Avoid a use-after-free if raced by ptdump (which takes its own
+	 *   write lock on init_mm).
+	 * - Serialise concurrent CPA walkers.
+	 */
+	scoped_guard(mmap_write_lock, &init_mm)
+		__cpa_collapse_large_pages(cpa);
 }
 
 static void cpa_flush(struct cpa_data *cpa, int cache)
@@ -448,7 +466,7 @@ static void cpa_flush(struct cpa_data *cpa, int cache)
 
 	BUG_ON(irqs_disabled() && !early_boot_irqs_disabled);
 
-	if (cache && !static_cpu_has(X86_FEATURE_CLFLUSH)) {
+	if (cache && !cpu_feature_enabled(X86_FEATURE_CLFLUSH)) {
 		cpa_flush_all(cache);
 		goto collapse_large_pages;
 	}
@@ -888,24 +906,23 @@ static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
 {
 	/* change init_mm */
 	set_pte_atomic(kpte, pte);
-#ifdef CONFIG_X86_32
-	{
-		struct page *page;
 
-		list_for_each_entry(page, &pgd_list, lru) {
+	if (IS_ENABLED(CONFIG_X86_32)) {
+		struct ptdesc *ptdesc;
+
+		list_for_each_entry(ptdesc, &pgd_list, pt_list) {
 			pgd_t *pgd;
 			p4d_t *p4d;
 			pud_t *pud;
 			pmd_t *pmd;
 
-			pgd = (pgd_t *)page_address(page) + pgd_index(address);
+			pgd = (pgd_t *)ptdesc_address(ptdesc) + pgd_index(address);
 			p4d = p4d_offset(pgd, address);
 			pud = pud_offset(p4d, address);
 			pmd = pmd_offset(pud, address);
 			set_pte_atomic((pte_t *)pmd, pte);
 		}
 	}
-#endif
 }
 
 static pgprot_t pgprot_clear_protnone_bits(pgprot_t prot)
@@ -1235,11 +1252,9 @@ static int split_large_page(struct cpa_data *cpa, pte_t *kpte,
 {
 	struct ptdesc *ptdesc;
 
-	if (!debug_pagealloc_enabled())
-		spin_unlock(&cpa_lock);
+	spin_unlock(&cpa_lock);
 	ptdesc = pagetable_alloc(GFP_KERNEL, 0);
-	if (!debug_pagealloc_enabled())
-		spin_lock(&cpa_lock);
+	spin_lock(&cpa_lock);
 	if (!ptdesc)
 		return -ENOMEM;
 
@@ -1298,11 +1313,11 @@ static int collapse_pmd_page(pmd_t *pmd, unsigned long addr,
 	list_add(&page_ptdesc(pmd_page(old_pmd))->pt_list, pgtables);
 
 	if (IS_ENABLED(CONFIG_X86_32)) {
-		struct page *page;
+		struct ptdesc *ptdesc;
 
 		/* Update all PGD tables to use the same large page */
-		list_for_each_entry(page, &pgd_list, lru) {
-			pgd_t *pgd = (pgd_t *)page_address(page) + pgd_index(addr);
+		list_for_each_entry(ptdesc, &pgd_list, pt_list) {
+			pgd_t *pgd = (pgd_t *)ptdesc_address(ptdesc) + pgd_index(addr);
 			p4d_t *p4d = p4d_offset(pgd, addr);
 			pud_t *pud = pud_offset(p4d, addr);
 			pmd_t *pmd = pmd_offset(pud, addr);
@@ -2023,11 +2038,9 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary)
 		if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))
 			cpa->numpages = 1;
 
-		if (!debug_pagealloc_enabled())
-			spin_lock(&cpa_lock);
+		spin_lock(&cpa_lock);
 		ret = __change_page_attr(cpa, primary);
-		if (!debug_pagealloc_enabled())
-			spin_unlock(&cpa_lock);
+		spin_unlock(&cpa_lock);
 		if (ret)
 			goto out;
 
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index f32facdb30354..cb03f5a2b2433 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -74,9 +74,9 @@ static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm)
 	virt_to_ptdesc(pgd)->pt_mm = mm;
 }
 
-struct mm_struct *pgd_page_get_mm(struct page *page)
+struct mm_struct *pgd_page_get_mm(struct ptdesc *pt)
 {
-	return page_ptdesc(page)->pt_mm;
+	return pt->pt_mm;
 }
 
 static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd)
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 1023acadd8f86..cacd2c3f78eb7 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -162,7 +162,7 @@ static inline unsigned long build_cr3(pgd_t *pgd, u16 asid, unsigned long lam)
 {
 	unsigned long cr3 = __sme_pa(pgd) | lam;
 
-	if (static_cpu_has(X86_FEATURE_PCID)) {
+	if (cpu_feature_enabled(X86_FEATURE_PCID)) {
 		cr3 |= kern_pcid(asid);
 	} else {
 		VM_WARN_ON_ONCE(asid != 0);
@@ -197,7 +197,7 @@ static void clear_asid_other(void)
 	 * This is only expected to be set if we have disabled
 	 * kernel _PAGE_GLOBAL pages.
 	 */
-	if (!static_cpu_has(X86_FEATURE_PTI)) {
+	if (!cpu_feature_enabled(X86_FEATURE_PTI)) {
 		WARN_ON_ONCE(1);
 		return;
 	}
@@ -227,7 +227,7 @@ static struct new_asid choose_new_asid(struct mm_struct *next, u64 next_tlb_gen)
 	struct new_asid ns;
 	u16 asid;
 
-	if (!static_cpu_has(X86_FEATURE_PCID)) {
+	if (!cpu_feature_enabled(X86_FEATURE_PCID)) {
 		ns.asid = 0;
 		ns.need_flush = 1;
 		return ns;
@@ -555,7 +555,7 @@ static inline void invalidate_user_asid(u16 asid)
 	if (!cpu_feature_enabled(X86_FEATURE_PCID))
 		return;
 
-	if (!static_cpu_has(X86_FEATURE_PTI))
+	if (!cpu_feature_enabled(X86_FEATURE_PTI))
 		return;
 
 	__set_bit(kern_pcid(asid),
@@ -1584,7 +1584,7 @@ void flush_tlb_one_kernel(unsigned long addr)
 	 */
 	flush_tlb_one_user(addr);
 
-	if (!static_cpu_has(X86_FEATURE_PTI))
+	if (!cpu_feature_enabled(X86_FEATURE_PTI))
 		return;
 
 	/*
@@ -1608,7 +1608,7 @@ STATIC_NOPV void native_flush_tlb_one_user(unsigned long addr)
 	invlpg(addr);
 
 	/* If PTI is off there is no user PCID and nothing to flush. */
-	if (!static_cpu_has(X86_FEATURE_PTI))
+	if (!cpu_feature_enabled(X86_FEATURE_PTI))
 		return;
 
 	loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
@@ -1637,7 +1637,7 @@ STATIC_NOPV void native_flush_tlb_global(void)
 {
 	unsigned long flags;
 
-	if (static_cpu_has(X86_FEATURE_INVPCID)) {
+	if (cpu_feature_enabled(X86_FEATURE_INVPCID)) {
 		/*
 		 * Using INVPCID is considerably faster than a pair of writes
 		 * to CR4 sandwiched inside an IRQ flag save/restore.
diff --git a/arch/x86/pci/mmconfig-shared.c b/arch/x86/pci/mmconfig-shared.c
index acdb8dcaeb52d..9749820f68f7e 100644
--- a/arch/x86/pci/mmconfig-shared.c
+++ b/arch/x86/pci/mmconfig-shared.c
@@ -189,7 +189,7 @@ static const char *__init pci_mmcfg_intel_945(void)
 
 static const char *__init pci_mmcfg_amd_fam10h(void)
 {
-	u32 low, high, address;
+	u32 address;
 	u64 base, msr;
 	int i;
 	unsigned segnbits = 0, busnbits, end_bus;
@@ -198,13 +198,9 @@ static const char *__init pci_mmcfg_amd_fam10h(void)
 		return NULL;
 
 	address = MSR_FAM10H_MMIO_CONF_BASE;
-	if (rdmsr_safe(address, &low, &high))
+	if (rdmsrq_safe(address, &msr))
 		return NULL;
 
-	msr = high;
-	msr <<= 32;
-	msr |= low;
-
 	/* ECAM is not enabled */
 	if (!(msr & FAM10H_MMIO_CONF_ENABLE))
 		return NULL;
diff --git a/arch/x86/platform/efi/quirks.c b/arch/x86/platform/efi/quirks.c
index 90a065fcb1fab..02c56a02eb9bc 100644
--- a/arch/x86/platform/efi/quirks.c
+++ b/arch/x86/platform/efi/quirks.c
@@ -832,12 +832,5 @@ void efi_crash_gracefully_on_page_fault(unsigned long phys_addr,
 	clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 	pr_info("Froze efi_rts_wq and disabled EFI Runtime Services\n");
 
-	/*
-	 * Call schedule() in an infinite loop, so that any spurious wake ups
-	 * will never run efi_rts_wq again.
-	 */
-	for (;;) {
-		set_current_state(TASK_IDLE);
-		schedule();
-	}
+	efi_rts_park_worker();
 }
diff --git a/arch/x86/platform/intel-quark/imr.c b/arch/x86/platform/intel-quark/imr.c
index ee25b032c0b3a..9238473842c76 100644
--- a/arch/x86/platform/intel-quark/imr.c
+++ b/arch/x86/platform/intel-quark/imr.c
@@ -291,8 +291,8 @@ static inline int imr_address_overlap(phys_addr_t addr, struct imr_regs *imr)
  *
  * @base:	physical base address of region aligned to 1KiB.
  * @size:	physical size of region in bytes must be aligned to 1KiB.
- * @read_mask:	read access mask.
- * @write_mask:	write access mask.
+ * @rmask:	read access mask.
+ * @wmask:	write access mask.
  * @return:	zero on success or negative value indicating error.
  */
 int imr_add_range(phys_addr_t base, size_t size,
diff --git a/arch/x86/platform/olpc/olpc-xo1-sci.c b/arch/x86/platform/olpc/olpc-xo1-sci.c
index 30751b42d54ef..97eb4738d6021 100644
--- a/arch/x86/platform/olpc/olpc-xo1-sci.c
+++ b/arch/x86/platform/olpc/olpc-xo1-sci.c
@@ -311,12 +311,13 @@ static int xo1_sci_resume(struct platform_device *pdev)
 
 static int setup_sci_interrupt(struct platform_device *pdev)
 {
-	u32 lo, hi;
+	u64 msr;
+	u32 lo;
 	u32 sts;
 	int r;
 
-	rdmsr(0x51400020, lo, hi);
-	sci_irq = (lo >> 20) & 15;
+	rdmsrq(0x51400020, msr);
+	sci_irq = (msr >> 20) & 15;
 
 	if (sci_irq) {
 		dev_info(&pdev->dev, "SCI is mapped to IRQ %d\n", sci_irq);
@@ -324,8 +325,8 @@ static int setup_sci_interrupt(struct platform_device *pdev)
 		/* Zero means masked */
 		dev_info(&pdev->dev, "SCI unmapped. Mapping to IRQ 3\n");
 		sci_irq = 3;
-		lo |= 0x00300000;
-		wrmsrq(0x51400020, lo);
+		msr |= 0x00300000;
+		wrmsrq(0x51400020, msr);
 	}
 
 	/* Select level triggered in PIC */
diff --git a/arch/x86/platform/olpc/olpc-xo15-sci.c b/arch/x86/platform/olpc/olpc-xo15-sci.c
index e486109e88c87..75ed6ceb9df33 100644
--- a/arch/x86/platform/olpc/olpc-xo15-sci.c
+++ b/arch/x86/platform/olpc/olpc-xo15-sci.c
@@ -18,8 +18,6 @@
 
 #define DRV_NAME			"olpc-xo15-sci"
 #define PFX				DRV_NAME ": "
-#define XO15_SCI_CLASS			DRV_NAME
-#define XO15_SCI_DEVICE_NAME		"OLPC XO-1.5 SCI"
 
 static unsigned long			xo15_sci_gpe;
 static bool				lid_wake_on_close;
@@ -148,9 +146,6 @@ static int xo15_sci_probe(struct platform_device *pdev)
 	if (!device)
 		return -ENODEV;
 
-	strscpy(acpi_device_name(device), XO15_SCI_DEVICE_NAME);
-	strscpy(acpi_device_class(device), XO15_SCI_CLASS);
-
 	/* Get GPE bit assignment (EC events). */
 	status = acpi_evaluate_integer(device->handle, "_GPE", NULL, &tmp);
 	if (ACPI_FAILURE(status))
diff --git a/arch/x86/ras/Kconfig b/arch/x86/ras/Kconfig
deleted file mode 100644
index 7488c715427e5..0000000000000
--- a/arch/x86/ras/Kconfig
+++ /dev/null
@@ -1,23 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-config RAS_CEC
-	bool "Correctable Errors Collector"
-	depends on X86_MCE && MEMORY_FAILURE && DEBUG_FS
-	help
-	  This is a small cache which collects correctable memory errors per 4K
-	  page PFN and counts their repeated occurrence. Once the counter for a
-	  PFN overflows, we try to soft-offline that page as we take it to mean
-	  that it has reached a relatively high error count and would probably
-	  be best if we don't use it anymore.
-
-	  Bear in mind that this is absolutely useless if your platform doesn't
-	  have ECC DIMMs and doesn't have DRAM ECC checking enabled in the BIOS.
-
-config RAS_CEC_DEBUG
-	bool "CEC debugging machinery"
-	default n
-	depends on RAS_CEC
-	help
-	  Add extra files to (debugfs)/ras/cec to test the correctable error
-	  collector feature. "pfn" is a writable file that allows user to
-	  simulate an error in a particular page frame. "array" is a read-only
-	  file that dumps out the current state of all pages logged so far.
diff --git a/arch/x86/tools/relocs.c b/arch/x86/tools/relocs.c
index e5a2b9a912d19..5ea9bc7eeb14e 100644
--- a/arch/x86/tools/relocs.c
+++ b/arch/x86/tools/relocs.c
@@ -73,7 +73,7 @@ static const char * const	sym_regex_kernel[S_NSYMTYPES] = {
 	"^(__init_(begin|end)|"
 	"__x86_cpu_dev_(start|end)|"
 	"__alt_instructions(_end)?|"
-	"(__iommu_table|__apicdrivers|__smp_locks)(_end)?|"
+	"(__iommu_table|__apicdrivers)(_end)?|"
 	"__(start|end)_pci_.*|"
 #if CONFIG_FW_LOADER
 	"__(start|end)_builtin_fw|"
diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c
index 7e9091c640be0..a236447ac7a26 100644
--- a/arch/x86/virt/hw.c
+++ b/arch/x86/virt/hw.c
@@ -196,7 +196,7 @@ static __init int __x86_vmx_init(void)
 		struct page *page;
 		struct vmcs *vmcs;
 
-		page = __alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
+		page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
 		if (WARN_ON_ONCE(!page)) {
 			x86_vmx_exit();
 			return -ENOMEM;
diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c
index 42df8ea464c47..1b9ff749dd8e0 100644
--- a/arch/x86/virt/vmx/tdx/tdx.c
+++ b/arch/x86/virt/vmx/tdx/tdx.c
@@ -1430,6 +1430,7 @@ static __init int record_keyid_partitioning(u32 *tdx_keyid_start,
 					    u32 *nr_tdx_keyids)
 {
 	u32 _nr_mktme_keyids, _tdx_keyid_start, _nr_tdx_keyids;
+	struct msr val;
 	int ret;
 
 	/*
@@ -1437,8 +1438,9 @@ static __init int record_keyid_partitioning(u32 *tdx_keyid_start,
 	 *   Bit [31:0]:	Number of MKTME KeyIDs.
 	 *   Bit [63:32]:	Number of TDX private KeyIDs.
 	 */
-	ret = rdmsr_safe(MSR_IA32_MKTME_KEYID_PARTITIONING, &_nr_mktme_keyids,
-			&_nr_tdx_keyids);
+	ret = rdmsrq_safe(MSR_IA32_MKTME_KEYID_PARTITIONING, &val.q);
+	_nr_mktme_keyids = val.l;
+	_nr_tdx_keyids = val.h;
 	if (ret || !_nr_tdx_keyids)
 		return -EINVAL;
 
diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c
index 820af6f0aa578..377c3c0d9f20f 100644
--- a/arch/x86/xen/mmu_pv.c
+++ b/arch/x86/xen/mmu_pv.c
@@ -836,15 +836,15 @@ static void xen_pgd_pin(struct mm_struct *mm)
  */
 void xen_mm_pin_all(void)
 {
-	struct page *page;
+	struct ptdesc *ptdesc;
 
 	spin_lock(&init_mm.page_table_lock);
 	spin_lock(&pgd_lock);
 
-	list_for_each_entry(page, &pgd_list, lru) {
-		if (!PagePinned(page)) {
-			__xen_pgd_pin(&init_mm, (pgd_t *)page_address(page));
-			SetPageSavePinned(page);
+	list_for_each_entry(ptdesc, &pgd_list, pt_list) {
+		if (!PagePinned(ptdesc_page(ptdesc))) {
+			__xen_pgd_pin(&init_mm, (pgd_t *)ptdesc_address(ptdesc));
+			SetPageSavePinned(ptdesc_page(ptdesc));
 		}
 	}
 
@@ -947,16 +947,16 @@ static void xen_pgd_unpin(struct mm_struct *mm)
  */
 void xen_mm_unpin_all(void)
 {
-	struct page *page;
+	struct ptdesc *ptdesc;
 
 	spin_lock(&init_mm.page_table_lock);
 	spin_lock(&pgd_lock);
 
-	list_for_each_entry(page, &pgd_list, lru) {
-		if (PageSavePinned(page)) {
-			BUG_ON(!PagePinned(page));
-			__xen_pgd_unpin(&init_mm, (pgd_t *)page_address(page));
-			ClearPageSavePinned(page);
+	list_for_each_entry(ptdesc, &pgd_list, pt_list) {
+		if (PageSavePinned(ptdesc_page(ptdesc))) {
+			BUG_ON(!PagePinned(ptdesc_page(ptdesc)));
+			__xen_pgd_unpin(&init_mm, (pgd_t *)ptdesc_address(ptdesc));
+			ClearPageSavePinned(ptdesc_page(ptdesc));
 		}
 	}
 
diff --git a/arch/xtensa/include/asm/platform.h b/arch/xtensa/include/asm/platform.h
index 94f13fabf7cd7..f2e50fc12b4d1 100644
--- a/arch/xtensa/include/asm/platform.h
+++ b/arch/xtensa/include/asm/platform.h
@@ -33,7 +33,7 @@ extern void platform_setup (char **);
 extern void platform_idle (void);
 
 /*
- * platform_calibrate_ccount calibrates cpu clock freq (CONFIG_XTENSA_CALIBRATE)
+ * platform_calibrate_ccount calibrates cpu clock freq (CONFIG_XTENSA_CALIBRATE_CCOUNT)
  */
 extern void platform_calibrate_ccount (void);
 
diff --git a/arch/xtensa/kernel/ptrace.c b/arch/xtensa/kernel/ptrace.c
index b80d54b2ea34e..364e4fdabb00d 100644
--- a/arch/xtensa/kernel/ptrace.c
+++ b/arch/xtensa/kernel/ptrace.c
@@ -547,14 +547,13 @@ int do_syscall_trace_enter(struct pt_regs *regs)
 		regs->areg[2] = -ENOSYS;
 
 	if (test_thread_flag(TIF_SYSCALL_TRACE) &&
-	    ptrace_report_syscall_entry(regs)) {
+	    !ptrace_report_syscall_permit_entry(regs)) {
 		regs->areg[2] = -ENOSYS;
 		regs->syscall = NO_SYSCALL;
 		return 0;
 	}
 
-	if (regs->syscall == NO_SYSCALL ||
-	    secure_computing() == -1) {
+	if (regs->syscall == NO_SYSCALL || !seccomp_permit_syscall()) {
 		do_syscall_trace_leave(regs);
 		return 0;
 	}
diff --git a/arch/xtensa/platforms/iss/console.c b/arch/xtensa/platforms/iss/console.c
index 8b95221375a89..8e54625cb2ba1 100644
--- a/arch/xtensa/platforms/iss/console.c
+++ b/arch/xtensa/platforms/iss/console.c
@@ -166,8 +166,9 @@ late_initcall(rs_init);
 
 static void iss_console_write(struct console *co, const char *s, unsigned count)
 {
-	if (s && *s != 0)
-		simc_write(1, s, min(count, strlen(s)));
+	count = s ? strnlen(s, count) : 0;
+	if (count)
+		simc_write(1, s, count);
 }
 
 static struct tty_driver* iss_console_device(struct console *c, int *index)
diff --git a/arch/xtensa/platforms/iss/simdisk.c b/arch/xtensa/platforms/iss/simdisk.c
index 7c7a2aa749f82..a2ddb49c269dc 100644
--- a/arch/xtensa/platforms/iss/simdisk.c
+++ b/arch/xtensa/platforms/iss/simdisk.c
@@ -41,7 +41,7 @@ module_param(simdisk_count, int, S_IRUGO);
 MODULE_PARM_DESC(simdisk_count, "Number of simdisk units.");
 
 static int n_files;
-static const char *filename[MAX_SIMDISK_COUNT] = {
+static char *filename[MAX_SIMDISK_COUNT] = {
 #ifdef CONFIG_SIMDISK0_FILENAME
 	CONFIG_SIMDISK0_FILENAME,
 #ifdef CONFIG_SIMDISK1_FILENAME
@@ -50,20 +50,48 @@ static const char *filename[MAX_SIMDISK_COUNT] = {
 #endif
 };
 
+/*
+ * The simdisk code can be built either into the kernel or as a loadable module.
+ * When built-in, CONFIG_SIMDISK{0,1}_FILENAME can be used to specify the
+ * initial simdisk filenames and additional filenames can be provided on the
+ * kernel command line. These arguments are parsed during early boot when slab
+ * is not yet available, but the command line itself is preserved for the
+ * lifetime of the kernel, so the incoming pointer is stored directly.
+ * When built as a loadable module, each value is copied with kstrdup() and all
+ * allocated memory is freed in simdisk_param_free_filename() when the module is
+ * unloaded.
+ */
 static int simdisk_param_set_filename(const char *val,
 		const struct kernel_param *kp)
 {
-	if (n_files < ARRAY_SIZE(filename))
-		filename[n_files++] = val;
-	else
+	char *str;
+
+	if (n_files >= ARRAY_SIZE(filename))
 		return -EINVAL;
+
+#ifdef MODULE
+	str = kstrdup(val, GFP_KERNEL);
+	if (!str)
+		return -ENOMEM;
+#else
+	str = (char *)val;
+#endif
+
+	filename[n_files++] = str;
 	return 0;
 }
 
+static void simdisk_param_free_filename(void *arg)
+{
+	for (int i = 0; i < n_files; i++)
+		kfree(filename[i]);
+}
+
 static const struct kernel_param_ops simdisk_param_ops_filename = {
 	.set = simdisk_param_set_filename,
+	.free = simdisk_param_free_filename,
 };
-module_param_cb(filename, &simdisk_param_ops_filename, &n_files, 0);
+module_param_cb(filename, &simdisk_param_ops_filename, NULL, 0);
 MODULE_PARM_DESC(filename, "Backing storage filename.");
 
 static int simdisk_major = SIMDISK_MAJOR;
diff --git a/block/bdev.c b/block/bdev.c
index 85ce57bd2ae4f..797d7f0ef6094 100644
--- a/block/bdev.c
+++ b/block/bdev.c
@@ -304,7 +304,12 @@ int bdev_freeze(struct block_device *bdev)
 
 	mutex_lock(&bdev->bd_fsfreeze_mutex);
 
-	if (atomic_inc_return(&bdev->bd_fsfreeze_count) > 1) {
+	/* A device being removed from its filesystem refuses freezes. */
+	if (!atomic_inc_unless_negative(&bdev->bd_fsfreeze_count)) {
+		mutex_unlock(&bdev->bd_fsfreeze_mutex);
+		return -EBUSY;
+	}
+	if (atomic_read(&bdev->bd_fsfreeze_count) > 1) {
 		mutex_unlock(&bdev->bd_fsfreeze_mutex);
 		return 0;
 	}
@@ -340,18 +345,18 @@ int bdev_thaw(struct block_device *bdev)
 
 	mutex_lock(&bdev->bd_fsfreeze_mutex);
 
-	/*
-	 * If this returns < 0 it means that @bd_fsfreeze_count was
-	 * already 0 and no decrement was performed.
-	 */
-	nr_freeze = atomic_dec_if_positive(&bdev->bd_fsfreeze_count);
-	if (nr_freeze < 0)
+	/* <= 0: not frozen (0) or a freeze deny is held (< 0); leave it. */
+	nr_freeze = atomic_read(&bdev->bd_fsfreeze_count);
+	if (nr_freeze <= 0)
 		goto out;
 
 	error = 0;
-	if (nr_freeze > 0)
+	if (nr_freeze > 1) {
+		atomic_dec(&bdev->bd_fsfreeze_count);
 		goto out;
+	}
 
+	/* Keep the count positive across the thaw so a deny is refused. */
 	mutex_lock(&bdev->bd_holder_lock);
 	if (bdev->bd_holder_ops && bdev->bd_holder_ops->thaw) {
 		error = bdev->bd_holder_ops->thaw(bdev);
@@ -360,14 +365,52 @@ int bdev_thaw(struct block_device *bdev)
 		mutex_unlock(&bdev->bd_holder_lock);
 	}
 
-	if (error)
-		atomic_inc(&bdev->bd_fsfreeze_count);
+	if (!error)
+		atomic_dec(&bdev->bd_fsfreeze_count);
 out:
 	mutex_unlock(&bdev->bd_fsfreeze_mutex);
 	return error;
 }
 EXPORT_SYMBOL(bdev_thaw);
 
+/**
+ * bdev_deny_freeze - make a block device unfreezable
+ * @bdev: block device
+ *
+ * Reserve @bdev against bdev_freeze() the way deny_write_access() reserves a
+ * file against writers.  bd_fsfreeze_count is sign-encoded: > 0 counts active
+ * freezes, < 0 counts deniers, so a deny succeeds only while no freeze is in
+ * progress.  While held, bdev_freeze() returns -EBUSY.  Pair with
+ * bdev_allow_freeze().
+ *
+ * A filesystem removing, adding or replacing a member device denies freezes on
+ * it for the duration, so a claim a freeze walk might act on is never torn down
+ * behind the freezer's back.  The deny is device-scoped, not (device,
+ * superblock)-scoped: a device shared by several superblocks is refused for all
+ * of them.  No in-tree filesystem removes a shared claim from a live superblock.
+ *
+ * Return: 0, or -EBUSY if the device is currently frozen.
+ */
+int bdev_deny_freeze(struct block_device *bdev)
+{
+	return atomic_dec_unless_positive(&bdev->bd_fsfreeze_count) ? 0 : -EBUSY;
+}
+EXPORT_SYMBOL_GPL(bdev_deny_freeze);
+
+/**
+ * bdev_allow_freeze - allow freezing a block device again
+ * @bdev: block device
+ *
+ * Undo one bdev_deny_freeze().
+ */
+void bdev_allow_freeze(struct block_device *bdev)
+{
+	/* A deny must be held, i.e. the count must be negative. */
+	WARN_ON_ONCE(atomic_read(&bdev->bd_fsfreeze_count) >= 0);
+	atomic_inc(&bdev->bd_fsfreeze_count);
+}
+EXPORT_SYMBOL_GPL(bdev_allow_freeze);
+
 /*
  * pseudo-fs
  */
@@ -1152,6 +1195,39 @@ void bdev_release(struct file *bdev_file)
 	blkdev_put_no_open(bdev);
 }
 
+/**
+ * bdev_yield_claim - give up the holder claim on an open block device
+ * @bdev_file: open block device
+ *
+ * Yield the holder and any write access for @bdev_file without closing it, so
+ * the caller can still act on the device - e.g. bdev_allow_freeze() it - before
+ * the final bdev_fput().  bdev_fput() yields too, so calling it afterwards is
+ * safe.
+ */
+void bdev_yield_claim(struct file *bdev_file)
+{
+	struct block_device *bdev;
+	struct gendisk *disk;
+
+	if (!bdev_file->private_data)
+		return;
+
+	bdev = file_bdev(bdev_file);
+	disk = bdev->bd_disk;
+
+	mutex_lock(&disk->open_mutex);
+	bdev_yield_write_access(bdev_file);
+	bd_yield_claim(bdev_file);
+	/*
+	 * Tell release we already gave up our hold on the
+	 * device and if write restrictions are available that
+	 * we already gave up write access to the device.
+	 */
+	bdev_file->private_data = BDEV_I(bdev_file->f_mapping->host);
+	mutex_unlock(&disk->open_mutex);
+}
+EXPORT_SYMBOL_GPL(bdev_yield_claim);
+
 /**
  * bdev_fput - yield claim to the block device and put the file
  * @bdev_file: open block device
@@ -1165,22 +1241,7 @@ void bdev_fput(struct file *bdev_file)
 	if (WARN_ON_ONCE(bdev_file->f_op != &def_blk_fops))
 		return;
 
-	if (bdev_file->private_data) {
-		struct block_device *bdev = file_bdev(bdev_file);
-		struct gendisk *disk = bdev->bd_disk;
-
-		mutex_lock(&disk->open_mutex);
-		bdev_yield_write_access(bdev_file);
-		bd_yield_claim(bdev_file);
-		/*
-		 * Tell release we already gave up our hold on the
-		 * device and if write restrictions are available that
-		 * we already gave up write access to the device.
-		 */
-		bdev_file->private_data = BDEV_I(bdev_file->f_mapping->host);
-		mutex_unlock(&disk->open_mutex);
-	}
-
+	bdev_yield_claim(bdev_file);
 	fput(bdev_file);
 }
 EXPORT_SYMBOL(bdev_fput);
@@ -1217,6 +1278,18 @@ int lookup_bdev(const char *pathname, dev_t *dev)
 	if (!may_open_dev(&path))
 		goto out_path_put;
 
+	/*
+	 * Reject a block device inode with i_rdev == 0.  A dev_t of 0 is
+	 * never valid for a block device: no real block device driver
+	 * registers major 0.  Fake block device inodes (e.g. fuse with
+	 * rootmode=S_IFBLK) can expose i_rdev == 0, and letting that
+	 * propagate would confuse superblock lookup and trigger warnings
+	 * in the device-to-superblock table (super_dev_register).
+	 */
+	error = -ENODEV;
+	if (!inode->i_rdev)
+		goto out_path_put;
+
 	*dev = inode->i_rdev;
 	error = 0;
 out_path_put:
diff --git a/block/bio.c b/block/bio.c
index f2a5f4d0a9672..6a2f6fc3413e8 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -555,6 +555,14 @@ struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
 		bio = bio_alloc_percpu_cache(bs);
 	} else {
 		opf &= ~REQ_ALLOC_CACHE;
+	}
+
+	/*
+	 * For a bioset without a percpu cache, or when the percpu cache was
+	 * empty, try a slab allocation with optimistic GFP_ flags before
+	 * falling back to the mempool.
+	 */
+	if (!bio) {
 		p = kmem_cache_alloc(bs->bio_slab, gfp);
 		if (p)
 			bio = p + bs->front_pad;
@@ -1191,7 +1199,7 @@ void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter)
  * for the next iteration.
  */
 static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter,
-			    unsigned len_align_mask)
+				   struct bio_vec *bv, unsigned len_align_mask)
 {
 	size_t nbytes = bio->bi_iter.bi_size & len_align_mask;
 
@@ -1200,23 +1208,16 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter,
 
 	iov_iter_revert(iter, nbytes);
 	bio->bi_iter.bi_size -= nbytes;
-	do {
-		struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
-
-		if (nbytes < bv->bv_len) {
-			bv->bv_len -= nbytes;
-			break;
-		}
-
+	while (nbytes >= bv->bv_len) {
 		if (bio_flagged(bio, BIO_PAGE_PINNED))
 			unpin_user_page(bv->bv_page);
 
-		bio->bi_vcnt--;
+		if (!--bio->bi_vcnt)
+			return -EFAULT;
 		nbytes -= bv->bv_len;
-	} while (nbytes);
-
-	if (!bio->bi_vcnt)
-		return -EFAULT;
+		bv--;
+	}
+	bv->bv_len -= nbytes;
 	return 0;
 }
 
@@ -1276,7 +1277,8 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
 
 	if (is_pci_p2pdma_page(bio->bi_io_vec->bv_page))
 		bio->bi_opf |= REQ_NOMERGE;
-	return bio_iov_iter_align_down(bio, iter, len_align_mask);
+	return bio_iov_iter_align_down(bio, iter,
+			&bio->bi_io_vec[bio->bi_vcnt - 1], len_align_mask);
 }
 
 static struct folio *folio_alloc_greedy(gfp_t gfp, size_t *size,
@@ -1285,7 +1287,8 @@ static struct folio *folio_alloc_greedy(gfp_t gfp, size_t *size,
 	struct folio *folio;
 
 	while (*size > minsize) {
-		folio = folio_alloc(gfp | __GFP_NORETRY, get_order(*size));
+		folio = folio_alloc(gfp | __GFP_NORETRY | __GFP_NOWARN,
+				    get_order(*size));
 		if (folio)
 			return folio;
 		*size = rounddown_pow_of_two(*size - 1);
@@ -1302,7 +1305,7 @@ static void bio_free_folios(struct bio *bio)
 	bio_for_each_bvec_all(bv, bio, i) {
 		struct folio *folio = bvec_folio(bv);
 
-		if (!is_zero_folio(folio))
+		if (!is_zero_folio(folio) && !is_huge_zero_folio(folio))
 			folio_put(folio);
 	}
 }
@@ -1360,7 +1363,8 @@ static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter,
 
 	if (!bio->bi_iter.bi_size)
 		return -ENOMEM;
-	return bio_iov_iter_align_down(bio, iter, minsize - 1);
+	return bio_iov_iter_align_down(bio, iter,
+			&bio->bi_io_vec[bio->bi_vcnt - 1], minsize - 1);
 }
 
 static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter,
@@ -1368,21 +1372,18 @@ static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter,
 {
 	size_t len = min3(iov_iter_count(iter), maxlen, SZ_1M);
 	struct folio *folio;
+	ssize_t ret;
 
 	folio = folio_alloc_greedy(GFP_KERNEL, &len, minsize);
 	if (!folio)
 		return -ENOMEM;
 
 	do {
-		ssize_t ret;
-
 		ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec + 1, len,
 				&bio->bi_vcnt, bio->bi_max_vecs - 1, 0);
 		if (ret <= 0) {
-			if (!bio->bi_vcnt) {
-				folio_put(folio);
-				return ret;
-			}
+			if (!bio->bi_vcnt)
+				goto out_folio_put;
 			break;
 		}
 		len -= ret;
@@ -1398,7 +1399,20 @@ static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter,
 	bvec_set_folio(&bio->bi_io_vec[0], folio, bio->bi_iter.bi_size, 0);
 	if (iov_iter_extract_will_pin(iter))
 		bio_set_flag(bio, BIO_PAGE_PINNED);
-	return bio_iov_iter_align_down(bio, iter, minsize - 1);
+
+	/* The first vec stores the bounce buffer, so do not subtract 1 here. */
+	ret = bio_iov_iter_align_down(bio, iter,
+			&bio->bi_io_vec[bio->bi_vcnt], minsize - 1);
+	if (ret)
+		goto out_folio_put;
+
+	/* Update the bounc buffer bv_len to the aligned down size. */
+	bio->bi_io_vec[0].bv_len = bio->bi_iter.bi_size;
+	return 0;
+
+out_folio_put:
+	folio_put(folio);
+	return ret;
 }
 
 /**
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index d2a1f5903f248..d9676126c5b5d 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -434,15 +434,15 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk,
 				blkg->pd[i]->online = true;
 			}
 		}
+		blkg->online = true;
 	}
-	blkg->online = true;
 	spin_unlock(&blkcg->lock);
 
 	if (!ret)
 		return blkg;
 
 	/* @blkg failed fully initialized, use the usual release path */
-	blkg_put(blkg);
+	percpu_ref_kill(&blkg->refcnt);
 	return ERR_PTR(ret);
 
 err_free_blkg:
diff --git a/block/blk-map.c b/block/blk-map.c
index 768549f19f97e..d1d6bbe0ecf1f 100644
--- a/block/blk-map.c
+++ b/block/blk-map.c
@@ -653,6 +653,7 @@ int blk_rq_map_kern(struct request *rq, void *kbuf, unsigned int len,
 		gfp_t gfp_mask)
 {
 	unsigned long addr = (unsigned long) kbuf;
+	bool do_copy;
 	struct bio *bio;
 	int ret;
 
@@ -661,7 +662,8 @@ int blk_rq_map_kern(struct request *rq, void *kbuf, unsigned int len,
 	if (!len || !kbuf)
 		return -EINVAL;
 
-	if (!blk_rq_aligned(rq->q, addr, len) || object_is_on_stack(kbuf))
+	do_copy = !blk_rq_aligned(rq->q, addr, len) || object_is_on_stack(kbuf);
+	if (do_copy)
 		bio = bio_copy_kern(rq, kbuf, len, gfp_mask);
 	else
 		bio = bio_map_kern(rq, kbuf, len, gfp_mask);
@@ -670,8 +672,11 @@ int blk_rq_map_kern(struct request *rq, void *kbuf, unsigned int len,
 		return PTR_ERR(bio);
 
 	ret = blk_rq_append_bio(rq, bio);
-	if (unlikely(ret))
+	if (unlikely(ret)) {
+		if (do_copy)
+			bio_free_pages(bio);
 		blk_mq_map_bio_put(bio);
+	}
 	return ret;
 }
 EXPORT_SYMBOL(blk_rq_map_kern);
diff --git a/block/blk.h b/block/blk.h
index fb95d3c58950c..eaac05815cb03 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -717,6 +717,7 @@ static inline int req_ref_read(struct request *req)
 static inline u64 blk_time_get_ns(void)
 {
 	struct blk_plug *plug = current->plug;
+	u64 now;
 
 	if (!plug || !in_task())
 		return ktime_get_ns();
@@ -725,12 +726,18 @@ static inline u64 blk_time_get_ns(void)
 	 * 0 could very well be a valid time, but rather than flag "this is
 	 * a valid timestamp" separately, just accept that we'll do an extra
 	 * ktime_get_ns() if we just happen to get 0 as the current time.
+	 *
+	 * cur_ktime can be zeroed by pre-emption the moment PF_BLOCK_TS is set.
 	 */
-	if (!plug->cur_ktime) {
-		plug->cur_ktime = ktime_get_ns();
+	now = READ_ONCE(plug->cur_ktime);
+	if (!now) {
+		now = ktime_get_ns();
+		WRITE_ONCE(plug->cur_ktime, now);
+		/* Ensure PF_BLOCK_TS is set after cur_ktime. */
+		barrier();
 		current->flags |= PF_BLOCK_TS;
 	}
-	return plug->cur_ktime;
+	return now;
 }
 
 static inline ktime_t blk_time_get(void)
diff --git a/block/elevator.c b/block/elevator.c
index 3bcd37c2aa340..2161b6eea680c 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -812,8 +812,13 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf,
 	 * reference during concurrent disk deletion:
 	 *   update_nr_hwq_lock -> kn->active (via del_gendisk -> kobject_del)
 	 *   kn->active -> update_nr_hwq_lock (via this sysfs write path)
+	 *
+	 * Use the writer lock instead of the reader lock of update_nr_hwq_lock
+	 * to serialize the two-stage elevator switch steps in
+	 * elevator_change(): the core switch step under the elevator lock and
+	 * the elevator_change_done() step outside the elevator lock.
 	 */
-	if (!down_read_trylock(&set->update_nr_hwq_lock)) {
+	if (!down_write_trylock(&set->update_nr_hwq_lock)) {
 		ret = -EBUSY;
 		goto out;
 	}
@@ -824,7 +829,7 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf,
 	} else {
 		ret = -ENOENT;
 	}
-	up_read(&set->update_nr_hwq_lock);
+	up_write(&set->update_nr_hwq_lock);
 
 out:
 	if (ctx.type)
diff --git a/block/error-injection.c b/block/error-injection.c
index cfb83138960c8..e14bc4b723efb 100644
--- a/block/error-injection.c
+++ b/block/error-injection.c
@@ -276,9 +276,10 @@ static int blk_error_injection_show(struct seq_file *s, void *private)
 
 	rcu_read_lock();
 	list_for_each_entry_rcu(inj, &disk->error_injection_list, entry) {
-		seq_printf(s, "%llu:%llu status=%s,chance=%u",
-			inj->start, inj->end,
-			blk_status_to_tag(inj->status), inj->chance);
+		seq_printf(s, "%llu:%llu op=%s,status=%s,chance=%u",
+			   inj->start, inj->end,
+			   blk_op_str(inj->op),
+			   blk_status_to_tag(inj->status), inj->chance);
 		seq_putc(s, '\n');
 	}
 	rcu_read_unlock();
diff --git a/block/partitions/aix.c b/block/partitions/aix.c
index f3c4174e003e9..689837deba279 100644
--- a/block/partitions/aix.c
+++ b/block/partitions/aix.c
@@ -208,7 +208,14 @@ int aix_partition(struct parsed_partitions *state)
 		if (n) {
 			int foundlvs = 0;
 
-			for (i = 0; foundlvs < numlvs && i < state->limit; i += 1) {
+			/*
+			 * The lvd array was read as a single sector; only the
+			 * struct lvd entries that fit in it are valid.  Bound the
+			 * scan so an on-disk numlvs larger than that cannot walk
+			 * the read buffer out of bounds.
+			 */
+			for (i = 0; foundlvs < numlvs && i < state->limit &&
+				    i < SECTOR_SIZE / (int)sizeof(struct lvd); i++) {
 				lvip[i].pps_per_lv = be16_to_cpu(p[i].num_lps);
 				if (lvip[i].pps_per_lv)
 					foundlvs += 1;
diff --git a/crypto/Kconfig b/crypto/Kconfig
index f1e3721952731..b61401bd3ef6e 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -358,8 +358,8 @@ config CRYPTO_AES
 	tristate "AES (Advanced Encryption Standard)"
 	select CRYPTO_ALGAPI
 	select CRYPTO_LIB_AES
-	select CRYPTO_LIB_AES_CBC_MACS if CRYPTO_CMAC || CRYPTO_XCBC || CRYPTO_CCM
-	select CRYPTO_HASH if CRYPTO_CMAC || CRYPTO_XCBC || CRYPTO_CCM
+	select CRYPTO_LIB_AES_CBC_MACS if CRYPTO_CMAC != n || CRYPTO_XCBC != n || CRYPTO_CCM != n
+	select CRYPTO_HASH if CRYPTO_CMAC != n || CRYPTO_XCBC != n || CRYPTO_CCM != n
 	help
 	  AES cipher algorithms (Rijndael)(FIPS-197, ISO/IEC 18033-3)
 
diff --git a/crypto/af_alg.c b/crypto/af_alg.c
index cce000e8590e4..34b801568fba6 100644
--- a/crypto/af_alg.c
+++ b/crypto/af_alg.c
@@ -8,6 +8,7 @@
  */
 
 #include <linux/atomic.h>
+#include <linux/capability.h>
 #include <crypto/if_alg.h>
 #include <linux/crypto.h>
 #include <linux/init.h>
@@ -22,10 +23,28 @@
 #include <linux/sched/signal.h>
 #include <linux/security.h>
 #include <linux/string.h>
+#include <linux/sysctl.h>
+#include <linux/user_namespace.h>
 #include <keys/user-type.h>
 #include <keys/trusted-type.h>
 #include <keys/encrypted-type.h>
 
+static int af_alg_restrict = 1;
+
+static const struct ctl_table af_alg_table[] = {
+	{
+		.procname       = "af_alg_restrict",
+		.data           = &af_alg_restrict,
+		.maxlen         = sizeof(int),
+		.mode           = 0644,
+		.proc_handler   = proc_dointvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_TWO,
+	},
+};
+
+static struct ctl_table_header *af_alg_header;
+
 struct alg_type_list {
 	const struct af_alg_type *type;
 	struct list_head list;
@@ -110,6 +129,39 @@ int af_alg_unregister_type(const struct af_alg_type *type)
 }
 EXPORT_SYMBOL_GPL(af_alg_unregister_type);
 
+static bool af_alg_capable(void)
+{
+	return ns_capable_noaudit(&init_user_ns, CAP_NET_ADMIN) ||
+	       capable(CAP_SYS_ADMIN);
+}
+
+int af_alg_check_restriction(const char *name,
+			     const struct af_alg_allowlist_entry allowlist[])
+{
+	int level = READ_ONCE(af_alg_restrict);
+
+	if (level == 0)
+		return 0;
+	if (level == 1) {
+		for (const struct af_alg_allowlist_entry *ent = allowlist;
+		     ent->name; ent++) {
+			if (strcmp(name, ent->name) == 0 &&
+			    (!ent->privileged || af_alg_capable()))
+				return 0;
+		}
+	}
+	/*
+	 * Use -ENOENT (the error code for "algorithm not found") instead of
+	 * -EACCES or -EPERM, for the highest chance of correctly triggering
+	 * fallback code paths in userspace programs.
+	 *
+	 * Don't log a warning, since it would be noisy.  iwd tries to bind a
+	 * bunch of algorithms that it never uses.
+	 */
+	return -ENOENT;
+}
+EXPORT_SYMBOL_GPL(af_alg_check_restriction);
+
 static void alg_do_release(const struct af_alg_type *type, void *private)
 {
 	if (!type)
@@ -506,6 +558,9 @@ static int alg_create(struct net *net, struct socket *sock, int protocol,
 	struct sock *sk;
 	int err;
 
+	if (READ_ONCE(af_alg_restrict) == 2)
+		return -EAFNOSUPPORT;
+
 	if (sock->type != SOCK_SEQPACKET)
 		return -ESOCKTNOSUPPORT;
 	if (protocol != 0)
@@ -1222,27 +1277,32 @@ EXPORT_SYMBOL_GPL(af_alg_get_rsgl);
 
 static int __init af_alg_init(void)
 {
-	int err = proto_register(&alg_proto, 0);
+	int err;
+
+	af_alg_header = register_sysctl("crypto", af_alg_table);
 
+	err = proto_register(&alg_proto, 0);
 	if (err)
-		goto out;
+		goto out_unregister_sysctl;
 
 	err = sock_register(&alg_family);
-	if (err != 0)
+	if (err)
 		goto out_unregister_proto;
 
-out:
-	return err;
+	return 0;
 
 out_unregister_proto:
 	proto_unregister(&alg_proto);
-	goto out;
+out_unregister_sysctl:
+	unregister_sysctl_table(af_alg_header);
+	return err;
 }
 
 static void __exit af_alg_exit(void)
 {
 	sock_unregister(PF_ALG);
 	proto_unregister(&alg_proto);
+	unregister_sysctl_table(af_alg_header);
 }
 
 module_init(af_alg_init);
diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c
index 787aac8aeb24e..b9217f9086aad 100644
--- a/crypto/algif_aead.c
+++ b/crypto/algif_aead.c
@@ -34,6 +34,11 @@
 #include <linux/net.h>
 #include <net/sock.h>
 
+static const struct af_alg_allowlist_entry aead_allowlist[] = {
+	{ "ccm(aes)", true }, /* bluez */
+	{},
+};
+
 static inline bool aead_sufficient_data(struct sock *sk)
 {
 	struct alg_sock *ask = alg_sk(sk);
@@ -344,6 +349,12 @@ static struct proto_ops algif_aead_ops_nokey = {
 
 static void *aead_bind(const char *name)
 {
+	int err;
+
+	err = af_alg_check_restriction(name, aead_allowlist);
+	if (err)
+		return ERR_PTR(err);
+
 	return crypto_alloc_aead(name, 0, AF_ALG_CRYPTOAPI_MASK);
 }
 
diff --git a/crypto/algif_hash.c b/crypto/algif_hash.c
index 5452ad6c15069..a8d958d51ece4 100644
--- a/crypto/algif_hash.c
+++ b/crypto/algif_hash.c
@@ -16,6 +16,24 @@
 #include <linux/net.h>
 #include <net/sock.h>
 
+static const struct af_alg_allowlist_entry hash_allowlist[] = {
+	{ "cmac(aes)", true }, /* iwd, bluez */
+	{ "hmac(md5)", true }, /* iwd */
+	{ "hmac(sha1)", true }, /* iwd */
+	{ "hmac(sha224)", true }, /* iwd */
+	{ "hmac(sha256)", true }, /* iwd */
+	{ "hmac(sha384)", true }, /* iwd */
+	{ "hmac(sha512)", true }, /* iwd, sha512hmac */
+	{ "md4", true }, /* iwd */
+	{ "md5", true }, /* iwd */
+	{ "sha1", false }, /* iwd, iproute2 < 7.0 */
+	{ "sha224", true }, /* iwd */
+	{ "sha256", true }, /* iwd */
+	{ "sha384", true }, /* iwd */
+	{ "sha512", true }, /* iwd */
+	{},
+};
+
 struct hash_ctx {
 	struct af_alg_sgl sgl;
 
@@ -382,6 +400,12 @@ static struct proto_ops algif_hash_ops_nokey = {
 
 static void *hash_bind(const char *name)
 {
+	int err;
+
+	err = af_alg_check_restriction(name, hash_allowlist);
+	if (err)
+		return ERR_PTR(err);
+
 	return crypto_alloc_ahash(name, 0, AF_ALG_CRYPTOAPI_MASK);
 }
 
diff --git a/crypto/algif_rng.c b/crypto/algif_rng.c
index 4dfe7899f8fa4..bd522915d56d7 100644
--- a/crypto/algif_rng.c
+++ b/crypto/algif_rng.c
@@ -50,6 +50,10 @@ MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>");
 MODULE_DESCRIPTION("User-space interface for random number generators");
 
+static const struct af_alg_allowlist_entry rng_allowlist[] = {
+	{},
+};
+
 struct rng_ctx {
 #define MAXSIZE 128
 	unsigned int len;
@@ -201,6 +205,11 @@ static void *rng_bind(const char *name)
 {
 	struct rng_parent_ctx *pctx;
 	struct crypto_rng *rng;
+	int err;
+
+	err = af_alg_check_restriction(name, rng_allowlist);
+	if (err)
+		return ERR_PTR(err);
 
 	pctx = kzalloc_obj(*pctx);
 	if (!pctx)
diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c
index df20bdfe1f1f4..49ae779b3b6b8 100644
--- a/crypto/algif_skcipher.c
+++ b/crypto/algif_skcipher.c
@@ -34,6 +34,23 @@
 #include <linux/net.h>
 #include <net/sock.h>
 
+static const struct af_alg_allowlist_entry skcipher_allowlist[] = {
+	{ "adiantum(xchacha12,aes)", false }, /* cryptsetup */
+	{ "adiantum(xchacha20,aes)", false }, /* cryptsetup */
+	{ "cbc(aes)", true }, /* iwd */
+	{ "cbc(des)", true }, /* iwd */
+	{ "cbc(des3_ede)", true }, /* iwd */
+	{ "ctr(aes)", true }, /* iwd */
+	{ "ecb(aes)", true }, /* iwd, bluez */
+	{ "ecb(des)", true }, /* iwd */
+	{ "hctr2(aes)", false }, /* cryptsetup */
+	{ "xts(aes)", false }, /* cryptsetup benchmark */
+	{ "xts(camellia)", false }, /* cryptsetup */
+	{ "xts(serpent)", false }, /* cryptsetup */
+	{ "xts(twofish)", false }, /* cryptsetup */
+	{},
+};
+
 static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg,
 			    size_t size)
 {
@@ -309,6 +326,12 @@ static struct proto_ops algif_skcipher_ops_nokey = {
 
 static void *skcipher_bind(const char *name)
 {
+	int err;
+
+	err = af_alg_check_restriction(name, skcipher_allowlist);
+	if (err)
+		return ERR_PTR(err);
+
 	return crypto_alloc_skcipher(name, 0, AF_ALG_CRYPTOAPI_MASK);
 }
 
diff --git a/crypto/api.c b/crypto/api.c
index 4349c2caa23aa..24227582cfcf3 100644
--- a/crypto/api.c
+++ b/crypto/api.c
@@ -116,7 +116,7 @@ struct crypto_larval *crypto_larval_alloc(const char *name, u32 type, u32 mask)
 	larval->alg.cra_priority = -1;
 	larval->alg.cra_destroy = crypto_larval_destroy;
 
-	strscpy(larval->alg.cra_name, name, CRYPTO_MAX_ALG_NAME);
+	strscpy(larval->alg.cra_name, name);
 	init_completion(&larval->completion);
 
 	return larval;
diff --git a/crypto/crypto_user.c b/crypto/crypto_user.c
index e8b6ae75f31f7..d3ccb507153b2 100644
--- a/crypto/crypto_user.c
+++ b/crypto/crypto_user.c
@@ -11,6 +11,7 @@
 #include <linux/cryptouser.h>
 #include <linux/sched.h>
 #include <linux/security.h>
+#include <linux/string.h>
 #include <net/netlink.h>
 #include <net/net_namespace.h>
 #include <net/sock.h>
@@ -87,11 +88,9 @@ static int crypto_report_one(struct crypto_alg *alg,
 {
 	memset(ualg, 0, sizeof(*ualg));
 
-	strscpy(ualg->cru_name, alg->cra_name, sizeof(ualg->cru_name));
-	strscpy(ualg->cru_driver_name, alg->cra_driver_name,
-		sizeof(ualg->cru_driver_name));
-	strscpy(ualg->cru_module_name, module_name(alg->cra_module),
-		sizeof(ualg->cru_module_name));
+	strscpy(ualg->cru_name, alg->cra_name);
+	strscpy(ualg->cru_driver_name, alg->cra_driver_name);
+	strscpy(ualg->cru_module_name, module_name(alg->cra_module));
 
 	ualg->cru_type = 0;
 	ualg->cru_mask = 0;
diff --git a/crypto/ecc.c b/crypto/ecc.c
index 6eb4d97a5f0d3..3250a464b8523 100644
--- a/crypto/ecc.c
+++ b/crypto/ecc.c
@@ -279,6 +279,48 @@ static void vli_rshift1(u64 *vli, unsigned int ndigits)
 	}
 }
 
+#ifdef __has_builtin
+#if __has_builtin(__builtin_addcll)
+#define USE_BUILTIN_ADDC
+#endif
+#endif
+
+/* Computes result = left + right + carry_in and updates carry_out */
+static inline void add_carry(u64 left, u64 right, u64 *result, u64 carry_in,
+			     u64 *carry_out)
+{
+#ifdef USE_BUILTIN_ADDC
+	*result = __builtin_addcll(left, right, carry_in, carry_out);
+#else
+	u64 sum1, sum2;
+	u64 c1 = __builtin_uaddll_overflow(left, right, &sum1);
+	u64 c2 = __builtin_uaddll_overflow(sum1, carry_in, &sum2);
+	*result = sum2;
+	*carry_out = c1 | c2;
+#endif
+}
+
+#ifdef __has_builtin
+#if __has_builtin(__builtin_subcll)
+#define USE_BUILTIN_SUBC
+#endif
+#endif
+
+/* Computes result = left - right - borrow_in and updates borrow_out */
+static inline void sub_borrow(u64 left, u64 right, u64 *result, u64 borrow_in,
+			      u64 *borrow_out)
+{
+#ifdef USE_BUILTIN_SUBC
+	*result = __builtin_subcll(left, right, borrow_in, borrow_out);
+#else
+	u64 diff1, diff2;
+	u64 b1 = __builtin_usubll_overflow(left, right, &diff1);
+	u64 b2 = __builtin_usubll_overflow(diff1, borrow_in, &diff2);
+	*result = diff2;
+	*borrow_out = b1 | b2;
+#endif
+}
+
 /* Computes result = left + right, returning carry. Can modify in place. */
 static u64 vli_add(u64 *result, const u64 *left, const u64 *right,
 		   unsigned int ndigits)
@@ -286,15 +328,8 @@ static u64 vli_add(u64 *result, const u64 *left, const u64 *right,
 	u64 carry = 0;
 	int i;
 
-	for (i = 0; i < ndigits; i++) {
-		u64 sum;
-
-		sum = left[i] + right[i] + carry;
-		if (sum != left[i])
-			carry = (sum < left[i]);
-
-		result[i] = sum;
-	}
+	for (i = 0; i < ndigits; i++)
+		add_carry(left[i], right[i], &result[i], carry, &carry);
 
 	return carry;
 }
@@ -303,40 +338,29 @@ static u64 vli_add(u64 *result, const u64 *left, const u64 *right,
 static u64 vli_uadd(u64 *result, const u64 *left, u64 right,
 		    unsigned int ndigits)
 {
-	u64 carry = right;
+	u64 carry;
 	int i;
 
-	for (i = 0; i < ndigits; i++) {
-		u64 sum;
+	if (ndigits == 0)
+		return right;
 
-		sum = left[i] + carry;
-		if (sum != left[i])
-			carry = (sum < left[i]);
-		else
-			carry = !!carry;
+	carry = __builtin_uaddll_overflow(left[0], right, &result[0]);
 
-		result[i] = sum;
-	}
+	for (i = 1; i < ndigits; i++)
+		carry = __builtin_uaddll_overflow(left[i], carry, &result[i]);
 
 	return carry;
 }
 
 /* Computes result = left - right, returning borrow. Can modify in place. */
 u64 vli_sub(u64 *result, const u64 *left, const u64 *right,
-		   unsigned int ndigits)
+	    unsigned int ndigits)
 {
 	u64 borrow = 0;
 	int i;
 
-	for (i = 0; i < ndigits; i++) {
-		u64 diff;
-
-		diff = left[i] - right[i] - borrow;
-		if (diff != left[i])
-			borrow = (diff > left[i]);
-
-		result[i] = diff;
-	}
+	for (i = 0; i < ndigits; i++)
+		sub_borrow(left[i], right[i], &result[i], borrow, &borrow);
 
 	return borrow;
 }
@@ -344,20 +368,18 @@ EXPORT_SYMBOL(vli_sub);
 
 /* Computes result = left - right, returning borrow. Can modify in place. */
 static u64 vli_usub(u64 *result, const u64 *left, u64 right,
-	     unsigned int ndigits)
+		    unsigned int ndigits)
 {
-	u64 borrow = right;
+	u64 borrow;
 	int i;
 
-	for (i = 0; i < ndigits; i++) {
-		u64 diff;
+	if (ndigits == 0)
+		return right;
 
-		diff = left[i] - borrow;
-		if (diff != left[i])
-			borrow = (diff > left[i]);
+	borrow = __builtin_usubll_overflow(left[0], right, &result[0]);
 
-		result[i] = diff;
-	}
+	for (i = 1; i < ndigits; i++)
+		borrow = __builtin_usubll_overflow(left[i], borrow, &result[i]);
 
 	return borrow;
 }
diff --git a/crypto/hctr2.c b/crypto/hctr2.c
index ad5edf9366ac8..cfc2343bcc1cc 100644
--- a/crypto/hctr2.c
+++ b/crypto/hctr2.c
@@ -354,8 +354,7 @@ static int hctr2_create_common(struct crypto_template *tmpl, struct rtattr **tb,
 	err = -EINVAL;
 	if (strncmp(xctr_alg->base.cra_name, "xctr(", 5))
 		goto err_free_inst;
-	len = strscpy(blockcipher_name, xctr_alg->base.cra_name + 5,
-		      sizeof(blockcipher_name));
+	len = strscpy(blockcipher_name, xctr_alg->base.cra_name + 5);
 	if (len < 1)
 		goto err_free_inst;
 	if (blockcipher_name[len - 1] != ')')
diff --git a/crypto/lrw.c b/crypto/lrw.c
index aa31ab03a5975..e306e85d7cedc 100644
--- a/crypto/lrw.c
+++ b/crypto/lrw.c
@@ -359,7 +359,7 @@ static int lrw_create(struct crypto_template *tmpl, struct rtattr **tb)
 	if (!memcmp(cipher_name, "ecb(", 4)) {
 		int len;
 
-		len = strscpy(ecb_name, cipher_name + 4, sizeof(ecb_name));
+		len = strscpy(ecb_name, cipher_name + 4);
 		if (len < 2)
 			goto err_free_inst;
 
diff --git a/crypto/lskcipher.c b/crypto/lskcipher.c
index e4328df6e26c0..d7ec215e2b3ae 100644
--- a/crypto/lskcipher.c
+++ b/crypto/lskcipher.c
@@ -528,8 +528,7 @@ struct lskcipher_instance *lskcipher_alloc_instance_simple(
 		int len;
 
 		err = -EINVAL;
-		len = strscpy(ecb_name, &cipher_alg->co.base.cra_name[4],
-			      sizeof(ecb_name));
+		len = strscpy(ecb_name, &cipher_alg->co.base.cra_name[4]);
 		if (len < 2)
 			goto err_free_inst;
 
diff --git a/crypto/xts.c b/crypto/xts.c
index ad97c8091582e..1dc9487454442 100644
--- a/crypto/xts.c
+++ b/crypto/xts.c
@@ -16,6 +16,7 @@
 #include <linux/module.h>
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
+#include <linux/string.h>
 
 #include <crypto/xts.h>
 #include <crypto/b128ops.h>
@@ -400,7 +401,7 @@ static int xts_create(struct crypto_template *tmpl, struct rtattr **tb)
 	if (!memcmp(cipher_name, "ecb(", 4)) {
 		int len;
 
-		len = strscpy(name, cipher_name + 4, sizeof(name));
+		len = strscpy(name, cipher_name + 4);
 		if (len < 2)
 			goto err_free_inst;
 
diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c
index 101f324ee1787..21f2817751f9e 100644
--- a/drivers/accel/amdxdna/aie2_ctx.c
+++ b/drivers/accel/amdxdna/aie2_ctx.c
@@ -1037,7 +1037,7 @@ static int aie2_populate_range(struct amdxdna_gem_obj *abo)
 	bool found;
 	int ret;
 
-	timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
+	timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
 again:
 	found = false;
 	down_write(&xdna->notifier_lock);
@@ -1062,24 +1062,9 @@ static int aie2_populate_range(struct amdxdna_gem_obj *abo)
 		return -EFAULT;
 	}
 
-	mapp->range.notifier_seq = mmu_interval_read_begin(&mapp->notifier);
-	mmap_read_lock(mm);
-	ret = hmm_range_fault(&mapp->range);
-	mmap_read_unlock(mm);
-	if (ret) {
-		if (time_after(jiffies, timeout)) {
-			ret = -ETIME;
-			goto put_mm;
-		}
-
-		if (ret == -EBUSY) {
-			amdxdna_umap_put(mapp);
-			mmput(mm);
-			goto again;
-		}
-
+	ret = hmm_range_fault_unlocked_timeout(&mapp->range, timeout);
+	if (ret)
 		goto put_mm;
-	}
 
 	down_write(&xdna->notifier_lock);
 	if (mmu_interval_read_retry(&mapp->notifier, mapp->range.notifier_seq)) {
@@ -1097,7 +1082,7 @@ static int aie2_populate_range(struct amdxdna_gem_obj *abo)
 put_mm:
 	amdxdna_umap_put(mapp);
 	mmput(mm);
-	return ret;
+	return ret == -EBUSY ? -ETIME : ret;
 }
 
 int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq)
diff --git a/drivers/accel/amdxdna/amdxdna_ctx.c b/drivers/accel/amdxdna/amdxdna_ctx.c
index 8f8df9d04ec56..31a414c3f0d96 100644
--- a/drivers/accel/amdxdna/amdxdna_ctx.c
+++ b/drivers/accel/amdxdna/amdxdna_ctx.c
@@ -577,6 +577,7 @@ void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job)
 	amdxdna_arg_bos_put(job);
 	amdxdna_gem_put_obj(job->cmd_bo);
 	dma_fence_put(job->fence);
+	mmdrop(job->mm);
 }
 
 int amdxdna_cmd_submit(struct amdxdna_client *client,
@@ -590,6 +591,10 @@ int amdxdna_cmd_submit(struct amdxdna_client *client,
 	int ret, idx;
 
 	XDNA_DBG(xdna, "Command BO hdl %d, Arg BO count %d", cmd_bo_hdl, arg_bo_cnt);
+
+	if (!xdna->dev_info->ops->cmd_submit)
+		return -EOPNOTSUPP;
+
 	job = kzalloc_flex(*job, bos, arg_bo_cnt);
 	if (!job)
 		return -ENOMEM;
@@ -603,6 +608,16 @@ int amdxdna_cmd_submit(struct amdxdna_client *client,
 			ret = -EINVAL;
 			goto free_job;
 		}
+	} else if (!drv_cmd) {
+		/*
+		 * Only internal driver commands (drv_cmd != NULL) may omit a
+		 * command BO. A user command submission with the invalid handle
+		 * would leave job->cmd_bo NULL and later fault when the scheduler
+		 * dereferences it in amdxdna_cmd_set_state().
+		 */
+		XDNA_DBG(xdna, "Command BO handle required for user submission");
+		ret = -EINVAL;
+		goto free_job;
 	}
 
 	ret = amdxdna_arg_bos_lookup(client, job, arg_bo_hdls, arg_bo_cnt);
@@ -628,6 +643,7 @@ int amdxdna_cmd_submit(struct amdxdna_client *client,
 
 	job->hwctx = hwctx;
 	job->mm = current->mm;
+	mmgrab(job->mm);
 
 	job->fence = amdxdna_fence_create(hwctx);
 	if (!job->fence) {
@@ -662,6 +678,8 @@ int amdxdna_cmd_submit(struct amdxdna_client *client,
 cmd_put:
 	amdxdna_gem_put_obj(job->cmd_bo);
 free_job:
+	if (job->mm)
+		mmdrop(job->mm);
 	kfree(job);
 	return ret;
 }
diff --git a/drivers/accel/amdxdna/amdxdna_gem.c b/drivers/accel/amdxdna/amdxdna_gem.c
index 4628a27872656..df236a9d8a0a4 100644
--- a/drivers/accel/amdxdna/amdxdna_gem.c
+++ b/drivers/accel/amdxdna/amdxdna_gem.c
@@ -753,9 +753,15 @@ static int amdxdna_gem_dev_obj_vmap(struct drm_gem_object *obj, struct iosys_map
 	return 0;
 }
 
+static struct dma_buf *amdxdna_gem_dev_obj_export(struct drm_gem_object *gobj, int flags)
+{
+	return ERR_PTR(-EOPNOTSUPP);
+}
+
 static const struct drm_gem_object_funcs amdxdna_gem_dev_obj_funcs = {
 	.free = amdxdna_gem_dev_obj_free,
 	.vmap = amdxdna_gem_dev_obj_vmap,
+	.export = amdxdna_gem_dev_obj_export,
 };
 
 static const struct drm_gem_object_funcs amdxdna_gem_shmem_funcs = {
diff --git a/drivers/accel/ethosu/Makefile b/drivers/accel/ethosu/Makefile
index 17db5a6004167..598a388b71793 100644
--- a/drivers/accel/ethosu/Makefile
+++ b/drivers/accel/ethosu/Makefile
@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0-only
 
 obj-$(CONFIG_DRM_ACCEL_ARM_ETHOSU) := ethosu.o
-ethosu-y += ethosu_drv.o ethosu_gem.o ethosu_job.o
+ethosu-y += ethosu_drv.o ethosu_gem.o ethosu_job.o ethosu_perfmon.o
diff --git a/drivers/accel/ethosu/ethosu_device.h b/drivers/accel/ethosu/ethosu_device.h
index b189fa783d6a3..d4458eac8447e 100644
--- a/drivers/accel/ethosu/ethosu_device.h
+++ b/drivers/accel/ethosu/ethosu_device.h
@@ -6,6 +6,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/bits.h>
+#include <linux/mutex.h>
 #include <linux/types.h>
 
 #include <drm/drm_device.h>
@@ -43,6 +44,15 @@ struct gen_pool;
 #define NPU_REG_BASEP_HI(x)	(0x0084 + (x) * 8)
 #define NPU_BASEP_REGION_MAX	8
 
+#define NPU_REG_PMCR		0x0180
+#define NPU_REG_PMCNTENSET	0x0184
+#define NPU_REG_PMCNTENCLR	0x0188
+#define NPU_REG_PMCCNTR_LO	0x01A0
+#define NPU_REG_PMCCNTR_HI	0x01A4
+#define NPU_REG_PMCCNTR_CFG	0x01A8
+#define NPU_REG_PMU_EVCNTR(x)	(0x0300 + (x) * 4)
+#define NPU_REG_PMU_EVTYPER(x)	(0x0380 + (x) * 4)
+
 #define ID_ARCH_MAJOR_MASK	GENMASK(31, 28)
 #define ID_ARCH_MINOR_MASK	GENMASK(27, 20)
 #define ID_ARCH_PATCH_MASK	GENMASK(19, 16)
@@ -67,6 +77,15 @@ struct gen_pool;
 
 #define PROT_ACTIVE_CSL		BIT(1)
 
+#define PMCR_NUM_EVENT_CNT_MASK	GENMASK(15, 11)
+#define PMCR_CYCLE_CNT_RST	BIT(2)
+#define PMCR_EVENT_CNT_RST	BIT(1)
+#define PMCR_CNT_EN		BIT(0)
+
+#define PMU_EV_TYPE_NONE	0
+#define PMU_EV_TYPE_CYCLES	0x11
+#define PMU_EV_TYPE_IDLE	0x20
+
 enum ethosu_cmds {
 	NPU_OP_CONV = 0x2,
 	NPU_OP_DEPTHWISE = 0x3,
@@ -152,6 +171,8 @@ enum ethosu_cmds {
 
 #define ETHOSU_SRAM_REGION	2	/* Matching Vela compiler */
 
+struct ethosu_perfmon;
+
 /**
  * struct ethosu_device - Ethosu device
  */
@@ -161,6 +182,7 @@ struct ethosu_device {
 
 	/** @iomem: CPU mapping of the registers. */
 	void __iomem *regs;
+	void __iomem *pmu_regs;
 
 	void __iomem *sram;
 	struct gen_pool *srampool;
@@ -173,8 +195,6 @@ struct ethosu_device {
 	struct drm_ethosu_npu_info npu_info;
 
 	struct ethosu_job *in_flight_job;
-	/* For in_flight_job and ethosu_job_hw_submit() */
-	struct mutex job_lock;
 
 	/* For dma_fence */
 	spinlock_t fence_lock;
@@ -184,6 +204,17 @@ struct ethosu_device {
 	struct mutex sched_lock;
 	u64 fence_context;
 	u64 emit_seqno;
+
+	/* Tracks the performance monitor state. */
+	struct {
+		/* Protects @active. */
+		struct mutex lock;
+
+		/* Perfmon currently programmed in HW (or NULL if none). */
+		struct ethosu_perfmon *active;
+	} perfmon_state;
+
+	struct ethosu_perfmon *global_perfmon;
 };
 
 #define to_ethosu_device(drm_dev) \
diff --git a/drivers/accel/ethosu/ethosu_drv.c b/drivers/accel/ethosu/ethosu_drv.c
index ed9c748a54ad5..d121fb0d77328 100644
--- a/drivers/accel/ethosu/ethosu_drv.c
+++ b/drivers/accel/ethosu/ethosu_drv.c
@@ -15,6 +15,7 @@
 #include <drm/drm_utils.h>
 #include <drm/drm_gem.h>
 #include <drm/drm_accel.h>
+#include <drm/drm_managed.h>
 #include <drm/ethosu_accel.h>
 
 #include "ethosu_drv.h"
@@ -154,6 +155,7 @@ static int ethosu_open(struct drm_device *ddev, struct drm_file *file)
 	if (ret)
 		goto err_put_mod;
 
+	ethosu_perfmon_open_file(priv);
 	file->driver_priv = no_free_ptr(priv);
 	return 0;
 
@@ -165,6 +167,7 @@ static int ethosu_open(struct drm_device *ddev, struct drm_file *file)
 static void ethosu_postclose(struct drm_device *ddev, struct drm_file *file)
 {
 	ethosu_job_close(file->driver_priv);
+	ethosu_perfmon_close_file(file->driver_priv);
 	kfree(file->driver_priv);
 	module_put(THIS_MODULE);
 }
@@ -179,6 +182,10 @@ static const struct drm_ioctl_desc ethosu_drm_driver_ioctls[] = {
 	ETHOSU_IOCTL(BO_MMAP_OFFSET, bo_mmap_offset, 0),
 	ETHOSU_IOCTL(CMDSTREAM_BO_CREATE, cmdstream_bo_create, 0),
 	ETHOSU_IOCTL(SUBMIT, submit, 0),
+	ETHOSU_IOCTL(PERFMON_CREATE, perfmon_create, 0),
+	ETHOSU_IOCTL(PERFMON_DESTROY, perfmon_destroy, 0),
+	ETHOSU_IOCTL(PERFMON_GET_VALUES, perfmon_get_values, 0),
+	ETHOSU_IOCTL(PERFMON_SET_GLOBAL, perfmon_set_global, 0),
 };
 
 DEFINE_DRM_ACCEL_FOPS(ethosu_drm_driver_fops);
@@ -314,8 +321,14 @@ static int ethosu_init(struct ethosu_device *ethosudev)
 
 	ethosu_sram_init(ethosudev);
 
+	if (!ethosu_is_u65(ethosudev))
+		ethosudev->pmu_regs += 0x1000;
+
+	ethosudev->npu_info.pmu_counters = FIELD_GET(PMCR_NUM_EVENT_CNT_MASK,
+		readl_relaxed(ethosudev->pmu_regs + NPU_REG_PMCR));
+
 	dev_info(ethosudev->base.dev,
-		 "Ethos-U NPU, arch v%ld.%ld.%ld, rev r%ldp%ld, cmd stream ver%ld, %d MACs, %dKB SRAM\n",
+		 "Ethos-U NPU, arch v%ld.%ld.%ld, rev r%ldp%ld, cmd stream ver%ld, %d MACs, %dKB SRAM, %d PMU cntrs\n",
 		 FIELD_GET(ID_ARCH_MAJOR_MASK, id),
 		 FIELD_GET(ID_ARCH_MINOR_MASK, id),
 		 FIELD_GET(ID_ARCH_PATCH_MASK, id),
@@ -323,7 +336,8 @@ static int ethosu_init(struct ethosu_device *ethosudev)
 		 FIELD_GET(ID_VER_MINOR_MASK, id),
 		 FIELD_GET(CONFIG_CMD_STREAM_VER_MASK, config),
 		 1 << FIELD_GET(CONFIG_MACS_PER_CC_MASK, config),
-		 ethosudev->npu_info.sram_size / 1024);
+		 ethosudev->npu_info.sram_size / 1024,
+		 ethosudev->npu_info.pmu_counters);
 
 	return 0;
 }
@@ -342,11 +356,16 @@ static int ethosu_probe(struct platform_device *pdev)
 	dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
 
 	ethosudev->regs = devm_platform_ioremap_resource(pdev, 0);
+	ethosudev->pmu_regs = ethosudev->regs;
 
 	ethosudev->num_clks = devm_clk_bulk_get_all(&pdev->dev, &ethosudev->clks);
 	if (ethosudev->num_clks < 0)
 		return ethosudev->num_clks;
 
+	ret = drmm_mutex_init(&ethosudev->base, &ethosudev->perfmon_state.lock);
+	if (ret)
+		return ret;
+
 	ret = ethosu_job_init(ethosudev);
 	if (ret)
 		return ret;
diff --git a/drivers/accel/ethosu/ethosu_drv.h b/drivers/accel/ethosu/ethosu_drv.h
index 9e21dfe941843..2193bc51d4258 100644
--- a/drivers/accel/ethosu/ethosu_drv.h
+++ b/drivers/accel/ethosu/ethosu_drv.h
@@ -1,15 +1,74 @@
 /* SPDX-License-Identifier: GPL-2.0-only OR MIT */
-/* Copyright 2025 Arm, Ltd. */
+/* Copyright 2025-2026 Arm, Ltd. */
 #ifndef __ETHOSU_DRV_H__
 #define __ETHOSU_DRV_H__
 
+#include <linux/mutex.h>
+#include <linux/xarray.h>
 #include <drm/gpu_scheduler.h>
 
 struct ethosu_device;
+struct drm_device;
+struct drm_file;
 
 struct ethosu_file_priv {
 	struct ethosu_device *edev;
 	struct drm_sched_entity sched_entity;
+	struct xarray perfmons;
 };
 
+/* Performance monitor object. The perfmon lifetime is controlled by userspace
+ * using perfmon related ioctls. A perfmon can be attached to a DRM_ETHOSU_SUBMIT
+ * request, and when this is the case, HW perf counters will be activated just
+ * before the job is submitted to the NPU and disabled when the job is
+ * done. This way, only events related to a specific job will be counted.
+ */
+struct ethosu_perfmon {
+	/* Tracks the number of users of the perfmon, when this counter reaches
+	 * zero the perfmon is destroyed.
+	 */
+	refcount_t refcnt;
+
+	/* Number of counters activated in this perfmon instance
+	 * (should be less than or equal to DRM_ETHOSU_MAX_PERF_COUNTERS).
+	 */
+	u8 ncounters;
+
+	/* Events counted by the HW perf counters. */
+	u16 counters[DRM_ETHOSU_MAX_PERF_EVENT_COUNTERS];
+
+	/*
+	 * Storage for counter values. Counters are incremented by the HW
+	 * perf counter values every time the perfmon is attached to an
+	 * NPU job. This way, perfmon users don't have to retrieve the
+	 * results after each job if they want to track events covering
+	 * several submissions. Note that counter values can't be reset,
+	 * but you can fake a reset by destroying the perfmon and
+	 * creating a new one.
+	 */
+	u64 values[] __counted_by(ncounters);
+};
+
+/* ethosu_perfmon.c */
+void ethosu_perfmon_get(struct ethosu_perfmon *perfmon);
+void ethosu_perfmon_put(struct ethosu_perfmon *perfmon);
+void ethosu_perfmon_start(struct ethosu_device *ethosu,
+			  struct ethosu_perfmon *perfmon);
+void ethosu_perfmon_stop(struct ethosu_device *ethosu,
+			 struct ethosu_perfmon *perfmon, bool capture);
+void ethosu_perfmon_stop_locked(struct ethosu_device *ethosu, struct ethosu_perfmon *perfmon,
+				bool capture);
+struct ethosu_perfmon *ethosu_perfmon_find(struct ethosu_file_priv *ethosu_priv,
+					   int id);
+void ethosu_perfmon_open_file(struct ethosu_file_priv *ethosu_priv);
+void ethosu_perfmon_close_file(struct ethosu_file_priv *ethosu_priv);
+int ethosu_ioctl_perfmon_create(struct drm_device *dev, void *data,
+				struct drm_file *file_priv);
+int ethosu_ioctl_perfmon_destroy(struct drm_device *dev, void *data,
+				 struct drm_file *file_priv);
+int ethosu_ioctl_perfmon_get_values(struct drm_device *dev, void *data,
+				    struct drm_file *file_priv);
+int ethosu_ioctl_perfmon_set_global(struct drm_device *dev, void *data,
+				    struct drm_file *file_priv);
+
 #endif
diff --git a/drivers/accel/ethosu/ethosu_job.c b/drivers/accel/ethosu/ethosu_job.c
index b76924645aaaf..74f3493fb11a5 100644
--- a/drivers/accel/ethosu/ethosu_job.c
+++ b/drivers/accel/ethosu/ethosu_job.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 /* Copyright 2024-2025 Tomeu Vizoso <tomeu@tomeuvizoso.net> */
-/* Copyright 2025 Arm, Ltd. */
+/* Copyright 2025-2026 Arm, Ltd. */
 
 #include <linux/bitfield.h>
 #include <linux/genalloc.h>
@@ -147,6 +147,8 @@ static void ethosu_job_err_cleanup(struct ethosu_job *job)
 {
 	unsigned int i;
 
+	ethosu_perfmon_put(job->perfmon);
+
 	for (i = 0; i < job->region_cnt; i++)
 		drm_gem_object_put(job->region_bo[i]);
 
@@ -181,6 +183,26 @@ static void ethosu_job_free(struct drm_sched_job *sched_job)
 	ethosu_job_put(job);
 }
 
+static void
+ethosu_switch_perfmon(struct ethosu_device *ethosu, struct ethosu_job *job)
+{
+	struct ethosu_perfmon *perfmon;
+
+	guard(mutex)(&ethosu->perfmon_state.lock);
+
+	perfmon = ethosu->global_perfmon;
+	if (!perfmon)
+		perfmon = job->perfmon;
+
+	if (perfmon == ethosu->perfmon_state.active)
+		return;
+
+	ethosu_perfmon_stop_locked(ethosu, ethosu->perfmon_state.active, true);
+
+	if (perfmon)
+		ethosu_perfmon_start(ethosu, perfmon);
+}
+
 static struct dma_fence *ethosu_job_run(struct drm_sched_job *sched_job)
 {
 	struct ethosu_job *job = to_ethosu_job(sched_job);
@@ -194,10 +216,10 @@ static struct dma_fence *ethosu_job_run(struct drm_sched_job *sched_job)
 		       dev->fence_context, ++dev->emit_seqno);
 	dma_fence_get(fence);
 
-	scoped_guard(mutex, &dev->job_lock) {
-		dev->in_flight_job = job;
-		ethosu_job_hw_submit(dev, job);
-	}
+	ethosu_switch_perfmon(dev, job);
+
+	WRITE_ONCE(dev->in_flight_job, job);
+	ethosu_job_hw_submit(dev, job);
 
 	return fence;
 }
@@ -205,6 +227,7 @@ static struct dma_fence *ethosu_job_run(struct drm_sched_job *sched_job)
 static void ethosu_job_handle_irq(struct ethosu_device *dev)
 {
 	u32 status = readl_relaxed(dev->regs + NPU_REG_STATUS);
+	struct ethosu_job *job;
 
 	if (status & (STATUS_BUS_STATUS | STATUS_CMD_PARSE_ERR)) {
 		dev_err(dev->base.dev, "Error IRQ - %x\n", status);
@@ -212,11 +235,10 @@ static void ethosu_job_handle_irq(struct ethosu_device *dev)
 		return;
 	}
 
-	scoped_guard(mutex, &dev->job_lock) {
-		if (dev->in_flight_job) {
-			dma_fence_signal(dev->in_flight_job->done_fence);
-			dev->in_flight_job = NULL;
-		}
+	job = READ_ONCE(dev->in_flight_job);
+	if (job) {
+		WRITE_ONCE(dev->in_flight_job, NULL);
+		dma_fence_signal(job->done_fence);
 	}
 }
 
@@ -272,8 +294,7 @@ static enum drm_gpu_sched_stat ethosu_job_timedout(struct drm_sched_job *bad)
 
 	drm_sched_stop(&dev->sched, bad);
 
-	scoped_guard(mutex, &dev->job_lock)
-		dev->in_flight_job = NULL;
+	WRITE_ONCE(dev->in_flight_job, NULL);
 
 	/* Proceed with reset now. */
 	pm_runtime_force_suspend(dev->base.dev);
@@ -304,9 +325,6 @@ int ethosu_job_init(struct ethosu_device *edev)
 	int ret;
 
 	spin_lock_init(&edev->fence_lock);
-	ret = devm_mutex_init(dev, &edev->job_lock);
-	if (ret)
-		return ret;
 	ret = devm_mutex_init(dev, &edev->sched_lock);
 	if (ret)
 		return ret;
@@ -365,7 +383,8 @@ void ethosu_job_close(struct ethosu_file_priv *ethosu_priv)
 }
 
 static int ethosu_ioctl_submit_job(struct drm_device *dev, struct drm_file *file,
-				   struct drm_ethosu_job *job)
+				   struct drm_ethosu_job *job,
+				   int perfmon_id)
 {
 	struct ethosu_device *edev = to_ethosu_device(dev);
 	struct ethosu_file_priv *file_priv = file->driver_priv;
@@ -389,6 +408,9 @@ static int ethosu_ioctl_submit_job(struct drm_device *dev, struct drm_file *file
 	ejob->dev = edev;
 	ejob->sram_size = job->sram_size;
 
+	if (perfmon_id)
+		ejob->perfmon = ethosu_perfmon_find(file_priv, perfmon_id);
+
 	ejob->done_fence = kzalloc_obj(*ejob->done_fence);
 	if (!ejob->done_fence) {
 		ret = -ENOMEM;
@@ -491,11 +513,6 @@ int ethosu_ioctl_submit(struct drm_device *dev, void *data, struct drm_file *fil
 	int ret = 0;
 	unsigned int i = 0;
 
-	if (args->pad) {
-		drm_dbg(dev, "Reserved field in drm_ethosu_submit struct should be 0.\n");
-		return -EINVAL;
-	}
-
 	struct drm_ethosu_job __free(kvfree) *jobs =
 		kvmalloc_objs(*jobs, args->job_count);
 	if (!jobs)
@@ -509,7 +526,7 @@ int ethosu_ioctl_submit(struct drm_device *dev, void *data, struct drm_file *fil
 	}
 
 	for (i = 0; i < args->job_count; i++) {
-		ret = ethosu_ioctl_submit_job(dev, file, &jobs[i]);
+		ret = ethosu_ioctl_submit_job(dev, file, &jobs[i], args->perfmon_id);
 		if (ret)
 			return ret;
 	}
diff --git a/drivers/accel/ethosu/ethosu_job.h b/drivers/accel/ethosu/ethosu_job.h
index ff1cf448d094d..8988edd00eed4 100644
--- a/drivers/accel/ethosu/ethosu_job.h
+++ b/drivers/accel/ethosu/ethosu_job.h
@@ -21,6 +21,8 @@ struct ethosu_job {
 	u8 region_cnt;
 	u32 sram_size;
 
+	struct ethosu_perfmon *perfmon;
+
 	/* Fence to be signaled by drm-sched once its done with the job */
 	struct dma_fence *inference_done_fence;
 
diff --git a/drivers/accel/ethosu/ethosu_perfmon.c b/drivers/accel/ethosu/ethosu_perfmon.c
new file mode 100644
index 0000000000000..26f625374f9d4
--- /dev/null
+++ b/drivers/accel/ethosu/ethosu_perfmon.c
@@ -0,0 +1,301 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/* Copyright 2026 Arm, Ltd. */
+/* Based on v3d_perfmon.c, Copyright (C) 2021 Raspberry Pi */
+
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/pm_runtime.h>
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+
+#include <drm/drm_file.h>
+#include <drm/drm_ioctl.h>
+
+#include <uapi/drm/ethosu_accel.h>
+
+#include "ethosu_drv.h"
+#include "ethosu_device.h"
+
+void ethosu_perfmon_get(struct ethosu_perfmon *perfmon)
+{
+	if (perfmon)
+		refcount_inc(&perfmon->refcnt);
+}
+
+void ethosu_perfmon_put(struct ethosu_perfmon *perfmon)
+{
+	if (perfmon && refcount_dec_and_test(&perfmon->refcnt))
+		kfree(perfmon);
+}
+
+void ethosu_perfmon_start(struct ethosu_device *ethosu, struct ethosu_perfmon *perfmon)
+{
+	unsigned int i;
+	u8 ncounters;
+	u32 mask;
+
+	lockdep_assert_held(&ethosu->perfmon_state.lock);
+
+	if (WARN_ON_ONCE(!perfmon || ethosu->perfmon_state.active))
+		return;
+
+	writel_relaxed(PMCR_CNT_EN, ethosu->pmu_regs + NPU_REG_PMCR);
+	writel_relaxed(PMU_EV_TYPE_CYCLES, ethosu->pmu_regs + NPU_REG_PMCCNTR_CFG);
+
+	mask = 0x80000000;
+	ncounters = perfmon->ncounters - 1;
+	if (ncounters)
+		mask |= GENMASK(ncounters - 1, 0);
+
+	for (i = 0; i < ncounters; i++)
+		writel_relaxed(perfmon->counters[i], ethosu->pmu_regs + NPU_REG_PMU_EVTYPER(i));
+
+	writel_relaxed(mask, ethosu->pmu_regs + NPU_REG_PMCNTENSET);
+	writel_relaxed(PMCR_CNT_EN | PMCR_EVENT_CNT_RST | PMCR_CYCLE_CNT_RST,
+		ethosu->pmu_regs + NPU_REG_PMCR);
+	ethosu->perfmon_state.active = perfmon;
+}
+
+void ethosu_perfmon_stop_locked(struct ethosu_device *ethosu, struct ethosu_perfmon *perfmon,
+				bool capture)
+{
+	unsigned int i;
+	u8 ncounters;
+	u32 mask;
+
+	lockdep_assert_held(&ethosu->perfmon_state.lock);
+
+	if (!perfmon || perfmon != ethosu->perfmon_state.active)
+		return;
+
+	ncounters = perfmon->ncounters - 1;
+
+	if (!pm_runtime_get_if_active(ethosu->base.dev)) {
+		ethosu->perfmon_state.active = NULL;
+		return;
+	}
+
+	if (capture) {
+		for (i = 0; i < ncounters; i++)
+			perfmon->values[i] += readl_relaxed(ethosu->pmu_regs + NPU_REG_PMU_EVCNTR(i));
+
+		perfmon->values[ncounters] +=
+			readl_relaxed(ethosu->pmu_regs + NPU_REG_PMCCNTR_LO) |
+			(u64)readl_relaxed(ethosu->pmu_regs + NPU_REG_PMCCNTR_HI) << 32;
+	}
+
+	mask = 0x80000000;
+	if (ncounters)
+		mask |= GENMASK(ncounters - 1, 0);
+	writel_relaxed(mask, ethosu->pmu_regs + NPU_REG_PMCNTENCLR);
+
+	writel_relaxed(0, ethosu->pmu_regs + NPU_REG_PMCR);
+	ethosu->perfmon_state.active = NULL;
+
+	pm_runtime_put(ethosu->base.dev);
+}
+
+void ethosu_perfmon_stop(struct ethosu_device *ethosu, struct ethosu_perfmon *perfmon,
+			 bool capture)
+{
+	if (!perfmon)
+		return;
+
+	guard(mutex)(&ethosu->perfmon_state.lock);
+	ethosu_perfmon_stop_locked(ethosu, perfmon, capture);
+}
+
+struct ethosu_perfmon *ethosu_perfmon_find(struct ethosu_file_priv *ethosu_priv, int id)
+{
+	struct ethosu_perfmon *perfmon;
+
+	xa_lock(&ethosu_priv->perfmons);
+	perfmon = xa_load(&ethosu_priv->perfmons, id);
+	ethosu_perfmon_get(perfmon);
+	xa_unlock(&ethosu_priv->perfmons);
+
+	return perfmon;
+}
+
+void ethosu_perfmon_open_file(struct ethosu_file_priv *ethosu_priv)
+{
+	xa_init_flags(&ethosu_priv->perfmons, XA_FLAGS_ALLOC1);
+}
+
+static void ethosu_perfmon_delete(struct ethosu_file_priv *ethosu_priv,
+				 struct ethosu_perfmon *perfmon)
+{
+	struct ethosu_device *ethosu = ethosu_priv->edev;
+
+	/* If the active perfmon is being destroyed, stop it first */
+	scoped_guard(mutex, &ethosu->perfmon_state.lock) {
+		/* If the global perfmon is being destroyed, set it to NULL */
+		if (ethosu->global_perfmon == perfmon) {
+			ethosu->global_perfmon = NULL;
+			ethosu_perfmon_put(perfmon);
+		}
+
+		ethosu_perfmon_stop_locked(ethosu, perfmon, false);
+	}
+
+	ethosu_perfmon_put(perfmon);
+}
+
+void ethosu_perfmon_close_file(struct ethosu_file_priv *ethosu_priv)
+{
+	struct ethosu_perfmon *perfmon;
+	unsigned long id;
+
+	xa_for_each(&ethosu_priv->perfmons, id, perfmon)
+		ethosu_perfmon_delete(ethosu_priv, perfmon);
+
+	xa_destroy(&ethosu_priv->perfmons);
+}
+
+int ethosu_ioctl_perfmon_create(struct drm_device *dev, void *data,
+			     struct drm_file *file_priv)
+{
+	struct ethosu_file_priv *ethosu_priv = file_priv->driver_priv;
+	struct drm_ethosu_perfmon_create *req = data;
+	struct ethosu_device *ethosu = to_ethosu_device(dev);
+	struct ethosu_perfmon *perfmon;
+	unsigned int i, event_max;
+	int ret;
+	u32 id;
+
+	/* Number of monitored counters cannot exceed HW limits. */
+	if (req->ncounters > ethosu->npu_info.pmu_counters)
+		return -EINVAL;
+
+	/* Make sure all counters are valid. */
+	event_max = ethosu_is_u65(ethosu) ? 433 : 671;
+	for (i = 0; i < req->ncounters; i++) {
+		if (req->counters[i] > event_max)
+			return -EINVAL;
+	}
+
+	/* Add 1 more counter for cycle counter */
+	req->ncounters++;
+
+	perfmon = kzalloc_flex(*perfmon, values, req->ncounters);
+	if (!perfmon)
+		return -ENOMEM;
+
+	for (i = 0; i < req->ncounters - 1; i++)
+		perfmon->counters[i] = req->counters[i];
+
+	perfmon->ncounters = req->ncounters;
+
+	refcount_set(&perfmon->refcnt, 1);
+
+	ret = xa_alloc(&ethosu_priv->perfmons, &id, perfmon, xa_limit_32b,
+		       GFP_KERNEL);
+
+	if (ret < 0) {
+		kfree(perfmon);
+		return ret;
+	}
+
+	req->id = id;
+
+	return 0;
+}
+
+int ethosu_ioctl_perfmon_destroy(struct drm_device *dev, void *data,
+				 struct drm_file *file_priv)
+{
+	struct ethosu_file_priv *ethosu_priv = file_priv->driver_priv;
+	struct drm_ethosu_perfmon_destroy *req = data;
+	struct ethosu_perfmon *perfmon;
+
+	perfmon = xa_erase(&ethosu_priv->perfmons, req->id);
+	if (!perfmon)
+		return -EINVAL;
+
+	ethosu_perfmon_delete(ethosu_priv, perfmon);
+
+	return 0;
+}
+
+int ethosu_ioctl_perfmon_get_values(struct drm_device *dev, void *data,
+				 struct drm_file *file_priv)
+{
+	struct ethosu_device *ethosu = to_ethosu_device(dev);
+	struct ethosu_file_priv *ethosu_priv = file_priv->driver_priv;
+	struct drm_ethosu_perfmon_get_values *req = data;
+	struct ethosu_perfmon *perfmon;
+	int ret = 0;
+
+	if (req->pad != 0)
+		return -EINVAL;
+
+	perfmon = ethosu_perfmon_find(ethosu_priv, req->id);
+	if (!perfmon)
+		return -EINVAL;
+
+	ret = pm_runtime_resume_and_get(dev->dev);
+	if (ret) {
+		ethosu_perfmon_put(perfmon);
+		return ret;
+	}
+	ethosu_perfmon_stop(ethosu, perfmon, true);
+
+	pm_runtime_put_autosuspend(dev->dev);
+
+	if (copy_to_user(u64_to_user_ptr(req->values_ptr), perfmon->values,
+			 perfmon->ncounters * sizeof(u64)))
+		ret = -EFAULT;
+
+	ethosu_perfmon_put(perfmon);
+
+	return ret;
+}
+
+int ethosu_ioctl_perfmon_set_global(struct drm_device *dev, void *data,
+				 struct drm_file *file_priv)
+{
+	struct ethosu_file_priv *ethosu_priv = file_priv->driver_priv;
+	struct drm_ethosu_perfmon_set_global *req = data;
+	struct ethosu_device *ethosu = to_ethosu_device(dev);
+	struct ethosu_perfmon *perfmon;
+
+	if (req->flags & ~DRM_ETHOSU_PERFMON_CLEAR_GLOBAL)
+		return -EINVAL;
+
+	perfmon = ethosu_perfmon_find(ethosu_priv, req->id);
+	if (!perfmon)
+		return -EINVAL;
+
+	/* If the request is to clear the global performance monitor */
+	if (req->flags & DRM_ETHOSU_PERFMON_CLEAR_GLOBAL) {
+		struct ethosu_perfmon *old;
+
+		scoped_guard(mutex, &ethosu->perfmon_state.lock) {
+			old = ethosu->global_perfmon;
+			if (!old) {
+				ethosu_perfmon_put(perfmon);
+				return -EINVAL;
+			}
+
+			ethosu->global_perfmon = NULL;
+			ethosu_perfmon_stop_locked(ethosu, old, true);
+		}
+
+		ethosu_perfmon_put(old);
+		ethosu_perfmon_put(perfmon);
+
+		return 0;
+	}
+
+	scoped_guard(mutex, &ethosu->perfmon_state.lock) {
+		if (ethosu->perfmon_state.active || ethosu->global_perfmon) {
+			ethosu_perfmon_put(perfmon);
+			return -EBUSY;
+		}
+
+		ethosu->global_perfmon = perfmon;
+	}
+
+	return 0;
+}
diff --git a/drivers/accel/ivpu/ivpu_drv.c b/drivers/accel/ivpu/ivpu_drv.c
index 35e506074d5f1..95120957f42a5 100644
--- a/drivers/accel/ivpu/ivpu_drv.c
+++ b/drivers/accel/ivpu/ivpu_drv.c
@@ -307,6 +307,11 @@ static int ivpu_open(struct drm_device *dev, struct drm_file *file)
 		return -ENODEV;
 
 	limits = ivpu_user_limits_get(vdev);
+	if (IS_ERR(limits) && PTR_ERR(limits) == -EMFILE) {
+		/* Context limit may be held by jobs pending deferred cleanup */
+		flush_work(&vdev->job_destroy_work);
+		limits = ivpu_user_limits_get(vdev);
+	}
 	if (IS_ERR(limits)) {
 		ret = PTR_ERR(limits);
 		goto err_dev_exit;
@@ -510,9 +515,9 @@ void ivpu_prepare_for_reset(struct ivpu_device *vdev)
 {
 	ivpu_hw_irq_disable(vdev);
 	disable_irq(vdev->irq);
-	flush_work(&vdev->irq_ipc_work);
 	flush_work(&vdev->irq_dct_work);
 	flush_work(&vdev->context_abort_work);
+	flush_work(&vdev->job_destroy_work);
 	ivpu_ipc_disable(vdev);
 	ivpu_mmu_disable(vdev);
 }
@@ -584,6 +589,11 @@ static const struct drm_driver driver = {
 	.major = 1,
 };
 
+static void ivpu_destroy_workqueue(void *wq)
+{
+	destroy_workqueue(wq);
+}
+
 static int ivpu_irq_init(struct ivpu_device *vdev)
 {
 	struct pci_dev *pdev = to_pci_dev(vdev->drm.dev);
@@ -595,16 +605,26 @@ static int ivpu_irq_init(struct ivpu_device *vdev)
 		return ret;
 	}
 
-	INIT_WORK(&vdev->irq_ipc_work, ivpu_ipc_irq_work_fn);
 	INIT_WORK(&vdev->irq_dct_work, ivpu_pm_irq_dct_work_fn);
 	INIT_WORK(&vdev->context_abort_work, ivpu_context_abort_work_fn);
+	init_llist_head(&vdev->job_destroy_list);
+	INIT_WORK(&vdev->job_destroy_work, ivpu_job_destroy_work_fn);
+
+	vdev->job_destroy_wq = alloc_workqueue("ivpu_job_destroy", WQ_UNBOUND | WQ_MEM_RECLAIM, 0);
+	if (!vdev->job_destroy_wq)
+		return -ENOMEM;
+
+	ret = devm_add_action_or_reset(vdev->drm.dev, ivpu_destroy_workqueue, vdev->job_destroy_wq);
+	if (ret)
+		return ret;
 
 	ivpu_irq_handlers_init(vdev);
 
 	vdev->irq = pci_irq_vector(pdev, 0);
 
-	ret = devm_request_irq(vdev->drm.dev, vdev->irq, ivpu_hw_irq_handler,
-			       IRQF_NO_AUTOEN, DRIVER_NAME, vdev);
+	ret = devm_request_threaded_irq(vdev->drm.dev, vdev->irq, ivpu_hw_irq_handler,
+					ivpu_ipc_irq_thread_handler, IRQF_NO_AUTOEN,
+					DRIVER_NAME, vdev);
 	if (ret)
 		ivpu_err(vdev, "Failed to request an IRQ %d\n", ret);
 
diff --git a/drivers/accel/ivpu/ivpu_drv.h b/drivers/accel/ivpu/ivpu_drv.h
index 9eefbbb7ba118..86d7c9966cacb 100644
--- a/drivers/accel/ivpu/ivpu_drv.h
+++ b/drivers/accel/ivpu/ivpu_drv.h
@@ -13,6 +13,7 @@
 #include <drm/drm_print.h>
 
 #include <linux/hashtable.h>
+#include <linux/llist.h>
 #include <linux/pci.h>
 #include <linux/xarray.h>
 #include <uapi/drm/ivpu_accel.h>
@@ -157,9 +158,11 @@ struct ivpu_device {
 	struct xa_limit db_limit;
 	u32 db_next;
 
-	struct work_struct irq_ipc_work;
 	struct work_struct irq_dct_work;
 	struct work_struct context_abort_work;
+	struct llist_head job_destroy_list;
+	struct work_struct job_destroy_work;
+	struct workqueue_struct *job_destroy_wq;
 
 	struct mutex bo_list_lock; /* Protects bo_list */
 	struct list_head bo_list;
diff --git a/drivers/accel/ivpu/ivpu_fw_log.c b/drivers/accel/ivpu/ivpu_fw_log.c
index 275baf844b562..716467aa31567 100644
--- a/drivers/accel/ivpu/ivpu_fw_log.c
+++ b/drivers/accel/ivpu/ivpu_fw_log.c
@@ -43,6 +43,10 @@ static int fw_log_from_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, u32 *off
 		ivpu_dbg(vdev, FW_BOOT, "Invalid header size 0x%x\n", log->header_size);
 		return -EINVAL;
 	}
+	if (log->size < log->header_size) {
+		ivpu_dbg(vdev, FW_BOOT, "Invalid log size 0x%x\n", log->size);
+		return -EINVAL;
+	}
 	if ((char *)log + log->size > (char *)ivpu_bo_vaddr(bo) + ivpu_bo_size(bo)) {
 		ivpu_dbg(vdev, FW_BOOT, "Invalid log size 0x%x\n", log->size);
 		return -EINVAL;
diff --git a/drivers/accel/ivpu/ivpu_hw.c b/drivers/accel/ivpu/ivpu_hw.c
index d4a9bcda4100f..647dc045c231a 100644
--- a/drivers/accel/ivpu/ivpu_hw.c
+++ b/drivers/accel/ivpu/ivpu_hw.c
@@ -399,6 +399,10 @@ irqreturn_t ivpu_hw_irq_handler(int irq, void *ptr)
 		return IRQ_NONE;
 
 	pm_runtime_mark_last_busy(vdev->drm.dev);
+
+	if (ip_handled)
+		return IRQ_WAKE_THREAD;
+
 	return IRQ_HANDLED;
 }
 
diff --git a/drivers/accel/ivpu/ivpu_hw_btrs.c b/drivers/accel/ivpu/ivpu_hw_btrs.c
index dac935164e11b..a17c829adb89a 100644
--- a/drivers/accel/ivpu/ivpu_hw_btrs.c
+++ b/drivers/accel/ivpu/ivpu_hw_btrs.c
@@ -927,7 +927,7 @@ static void diagnose_failure_mtl(struct ivpu_device *vdev)
 
 static void diagnose_failure_lnl(struct ivpu_device *vdev)
 {
-	u32 reg = REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) & BTRS_LNL_IRQ_MASK;
+	u32 reg = REGB_RD32(VPU_HW_BTRS_LNL_INTERRUPT_STAT) & BTRS_LNL_IRQ_MASK;
 
 	if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, ATS_ERR, reg)) {
 		ivpu_err(vdev, "ATS_ERR_LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n",
diff --git a/drivers/accel/ivpu/ivpu_ipc.c b/drivers/accel/ivpu/ivpu_ipc.c
index 978bc3d8704fb..62607ec8ca8fe 100644
--- a/drivers/accel/ivpu/ivpu_ipc.c
+++ b/drivers/accel/ivpu/ivpu_ipc.c
@@ -146,7 +146,7 @@ ivpu_ipc_rx_msg_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
 
 	lockdep_assert_held(&ipc->cons_lock);
 
-	rx_msg = kzalloc_obj(*rx_msg, GFP_ATOMIC);
+	rx_msg = kmem_cache_zalloc(ipc->rx_msg_cache, GFP_ATOMIC);
 	if (!rx_msg) {
 		ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg);
 		return;
@@ -174,7 +174,7 @@ ivpu_ipc_rx_msg_del(struct ivpu_device *vdev, struct ivpu_ipc_rx_msg *rx_msg)
 	list_del(&rx_msg->link);
 	ivpu_ipc_rx_mark_free(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg);
 	atomic_dec(&vdev->ipc->rx_msg_count);
-	kfree(rx_msg);
+	kmem_cache_free(vdev->ipc->rx_msg_cache, rx_msg);
 }
 
 void ivpu_ipc_consumer_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
@@ -462,13 +462,11 @@ void ivpu_ipc_irq_handler(struct ivpu_device *vdev)
 			ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg);
 		}
 	}
-
-	queue_work(system_percpu_wq, &vdev->irq_ipc_work);
 }
 
-void ivpu_ipc_irq_work_fn(struct work_struct *work)
+irqreturn_t ivpu_ipc_irq_thread_handler(int irq, void *ptr)
 {
-	struct ivpu_device *vdev = container_of(work, struct ivpu_device, irq_ipc_work);
+	struct ivpu_device *vdev = ptr;
 	struct ivpu_ipc_info *ipc = vdev->ipc;
 	struct ivpu_ipc_rx_msg *rx_msg, *r;
 	struct list_head cb_msg_list;
@@ -483,6 +481,8 @@ void ivpu_ipc_irq_work_fn(struct work_struct *work)
 		rx_msg->callback(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg);
 		ivpu_ipc_rx_msg_del(vdev, rx_msg);
 	}
+
+	return IRQ_HANDLED;
 }
 
 int ivpu_ipc_init(struct ivpu_device *vdev)
@@ -490,10 +490,18 @@ int ivpu_ipc_init(struct ivpu_device *vdev)
 	struct ivpu_ipc_info *ipc = vdev->ipc;
 	int ret;
 
+	ipc->rx_msg_cache = kmem_cache_create("ivpu_ipc_rx_msg", sizeof(struct ivpu_ipc_rx_msg), 0,
+					      SLAB_HWCACHE_ALIGN, NULL);
+	if (!ipc->rx_msg_cache) {
+		ivpu_err(vdev, "Failed to create rx_msg_cache\n");
+		return -ENOMEM;
+	}
+
 	ipc->mem_tx = ivpu_bo_create_global(vdev, SZ_16K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE);
 	if (!ipc->mem_tx) {
 		ivpu_err(vdev, "Failed to allocate mem_tx\n");
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto err_destroy_cache;
 	}
 
 	ipc->mem_rx = ivpu_bo_create_global(vdev, SZ_16K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE);
@@ -532,6 +540,8 @@ int ivpu_ipc_init(struct ivpu_device *vdev)
 	ivpu_bo_free(ipc->mem_rx);
 err_free_tx:
 	ivpu_bo_free(ipc->mem_tx);
+err_destroy_cache:
+	kmem_cache_destroy(ipc->rx_msg_cache);
 	return ret;
 }
 
@@ -544,6 +554,7 @@ void ivpu_ipc_fini(struct ivpu_device *vdev)
 	drm_WARN_ON(&vdev->drm, atomic_read(&ipc->rx_msg_count) > 0);
 
 	ivpu_ipc_mem_fini(vdev);
+	kmem_cache_destroy(ipc->rx_msg_cache);
 }
 
 void ivpu_ipc_enable(struct ivpu_device *vdev)
diff --git a/drivers/accel/ivpu/ivpu_ipc.h b/drivers/accel/ivpu/ivpu_ipc.h
index b524a1985b9de..fb3e5816ca505 100644
--- a/drivers/accel/ivpu/ivpu_ipc.h
+++ b/drivers/accel/ivpu/ivpu_ipc.h
@@ -70,6 +70,7 @@ struct ivpu_ipc_info {
 	struct gen_pool *mm_tx;
 	struct ivpu_bo *mem_tx;
 	struct ivpu_bo *mem_rx;
+	struct kmem_cache *rx_msg_cache;
 
 	atomic_t rx_msg_count;
 
@@ -90,7 +91,7 @@ void ivpu_ipc_disable(struct ivpu_device *vdev);
 void ivpu_ipc_reset(struct ivpu_device *vdev);
 
 void ivpu_ipc_irq_handler(struct ivpu_device *vdev);
-void ivpu_ipc_irq_work_fn(struct work_struct *work);
+irqreturn_t ivpu_ipc_irq_thread_handler(int irq, void *ptr);
 
 void ivpu_ipc_consumer_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
 			   u32 channel, ivpu_ipc_rx_callback_t callback);
diff --git a/drivers/accel/ivpu/ivpu_job.c b/drivers/accel/ivpu/ivpu_job.c
index b24f31a8b5672..ebb2c865b09a0 100644
--- a/drivers/accel/ivpu/ivpu_job.c
+++ b/drivers/accel/ivpu/ivpu_job.c
@@ -535,6 +535,20 @@ static void ivpu_job_destroy(struct ivpu_job *job)
 	kfree(job);
 }
 
+void ivpu_job_destroy_work_fn(struct work_struct *work)
+{
+	struct ivpu_device *vdev = container_of(work, struct ivpu_device, job_destroy_work);
+	struct ivpu_job *job, *tmp;
+	struct llist_node *list;
+
+	list = llist_del_all(&vdev->job_destroy_list);
+
+	llist_for_each_entry_safe(job, tmp, list, destroy_node) {
+		ivpu_job_destroy(job);
+		ivpu_rpm_put(vdev);
+	}
+}
+
 static struct ivpu_job *
 ivpu_job_create(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count)
 {
@@ -619,7 +633,7 @@ bool ivpu_job_handle_engine_error(struct ivpu_device *vdev, u32 job_id, u32 job_
 	return false;
 }
 
-static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32 job_status)
+static struct ivpu_job *ivpu_job_signal(struct ivpu_device *vdev, u32 job_id, u32 job_status)
 {
 	struct ivpu_job *job;
 
@@ -627,7 +641,7 @@ static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32
 
 	job = xa_load(&vdev->submitted_jobs_xa, job_id);
 	if (!job)
-		return -ENOENT;
+		return NULL;
 
 	ivpu_job_remove_from_submitted_jobs(vdev, job_id);
 
@@ -646,14 +660,37 @@ static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32
 		 job->job_id, job->file_priv->ctx.id, job->cmdq_id, job->engine_idx,
 		 job->job_status);
 
-	ivpu_job_destroy(job);
 	ivpu_stop_job_timeout_detection(vdev);
 
-	ivpu_rpm_put(vdev);
-
 	if (!xa_empty(&vdev->submitted_jobs_xa))
 		ivpu_start_job_timeout_detection(vdev);
 
+	return job;
+}
+
+static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32 job_status)
+{
+	struct ivpu_job *job = ivpu_job_signal(vdev, job_id, job_status);
+
+	if (!job)
+		return -ENOENT;
+
+	ivpu_job_destroy(job);
+	ivpu_rpm_put(vdev);
+
+	return 0;
+}
+
+static int ivpu_job_signal_and_defer_destroy(struct ivpu_device *vdev, u32 job_id, u32 job_status)
+{
+	struct ivpu_job *job = ivpu_job_signal(vdev, job_id, job_status);
+
+	if (!job)
+		return -ENOENT;
+
+	llist_add(&job->destroy_node, &vdev->job_destroy_list);
+	queue_work(vdev->job_destroy_wq, &vdev->job_destroy_work);
+
 	return 0;
 }
 
@@ -689,6 +726,7 @@ static int ivpu_job_submit(struct ivpu_job *job, u8 priority, u32 cmdq_id)
 	struct ivpu_file_priv *file_priv = job->file_priv;
 	struct ivpu_device *vdev = job->vdev;
 	struct ivpu_cmdq *cmdq;
+	bool flushed = false;
 	bool is_first_job;
 	int ret;
 
@@ -696,6 +734,7 @@ static int ivpu_job_submit(struct ivpu_job *job, u8 priority, u32 cmdq_id)
 	if (ret < 0)
 		return ret;
 
+retry:
 	mutex_lock(&vdev->submitted_jobs_lock);
 	mutex_lock(&file_priv->lock);
 
@@ -709,6 +748,14 @@ static int ivpu_job_submit(struct ivpu_job *job, u8 priority, u32 cmdq_id)
 	}
 
 	ret = ivpu_cmdq_register(file_priv, cmdq);
+	if (ret == -EBUSY && !flushed) {
+		/* Doorbell may be held by jobs pending deferred cleanup */
+		mutex_unlock(&file_priv->lock);
+		mutex_unlock(&vdev->submitted_jobs_lock);
+		flush_work(&vdev->job_destroy_work);
+		flushed = true;
+		goto retry;
+	}
 	if (ret) {
 		ivpu_err(vdev, "Failed to register command queue: %d\n", ret);
 		goto err_unlock;
@@ -1101,7 +1148,7 @@ ivpu_job_done_callback(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr,
 	mutex_lock(&vdev->submitted_jobs_lock);
 	if (!ivpu_job_handle_engine_error(vdev, payload->job_id, payload->job_status))
 		/* No engine error, complete the job normally */
-		ivpu_job_signal_and_destroy(vdev, payload->job_id, payload->job_status);
+		ivpu_job_signal_and_defer_destroy(vdev, payload->job_id, payload->job_status);
 	mutex_unlock(&vdev->submitted_jobs_lock);
 }
 
diff --git a/drivers/accel/ivpu/ivpu_job.h b/drivers/accel/ivpu/ivpu_job.h
index 3ab61e6a5616c..d8dbce82447a1 100644
--- a/drivers/accel/ivpu/ivpu_job.h
+++ b/drivers/accel/ivpu/ivpu_job.h
@@ -6,8 +6,10 @@
 #ifndef __IVPU_JOB_H__
 #define __IVPU_JOB_H__
 
-#include <linux/kref.h>
 #include <linux/idr.h>
+#include <linux/kref.h>
+#include <linux/llist.h>
+#include <linux/workqueue.h>
 
 #include "ivpu_gem.h"
 
@@ -47,6 +49,7 @@ struct ivpu_cmdq {
  * @vdev:                Pointer to the VPU device
  * @file_priv:           The client context that submitted this job
  * @done_fence:          Fence signaled when job completes
+ * @destroy_node:        List node for deferred resource cleanup after job completion
  * @cmd_buf_vpu_addr:    VPU address of the command buffer for this job
  * @cmdq_id:             Command queue ID used for submission
  * @job_id:              Unique job ID for tracking and status reporting
@@ -61,6 +64,7 @@ struct ivpu_job {
 	struct ivpu_device *vdev;
 	struct ivpu_file_priv *file_priv;
 	struct dma_fence *done_fence;
+	struct llist_node destroy_node;
 	u64 cmd_buf_vpu_addr;
 	u32 cmdq_id;
 	u32 job_id;
@@ -87,6 +91,7 @@ void ivpu_job_done_consumer_init(struct ivpu_device *vdev);
 void ivpu_job_done_consumer_fini(struct ivpu_device *vdev);
 bool ivpu_job_handle_engine_error(struct ivpu_device *vdev, u32 job_id, u32 job_status);
 void ivpu_context_abort_work_fn(struct work_struct *work);
+void ivpu_job_destroy_work_fn(struct work_struct *work);
 
 void ivpu_jobs_abort_all(struct ivpu_device *vdev);
 
diff --git a/drivers/accel/rocket/rocket_job.c b/drivers/accel/rocket/rocket_job.c
index 2f1861f960ccb..bb77b6bf0f231 100644
--- a/drivers/accel/rocket/rocket_job.c
+++ b/drivers/accel/rocket/rocket_job.c
@@ -8,6 +8,7 @@
 #include <drm/drm_gem.h>
 #include <drm/rocket_accel.h>
 #include <linux/interrupt.h>
+#include <linux/overflow.h>
 #include <linux/iommu.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
@@ -102,6 +103,7 @@ rocket_copy_tasks(struct drm_device *dev,
 
 fail:
 	kvfree(rjob->tasks);
+	rjob->tasks = NULL;
 	return ret;
 }
 
@@ -188,14 +190,19 @@ static int rocket_job_push(struct rocket_job *job)
 	struct rocket_device *rdev = job->rdev;
 	struct drm_gem_object **bos;
 	struct ww_acquire_ctx acquire_ctx;
+	u32 bo_count;
 	int ret = 0;
 
-	bos = kvmalloc_array(job->in_bo_count + job->out_bo_count, sizeof(void *),
-			     GFP_KERNEL);
+	if (check_add_overflow(job->in_bo_count, job->out_bo_count, &bo_count))
+		return -EINVAL;
+
+	bos = kvmalloc_array(bo_count, sizeof(*bos), GFP_KERNEL);
+	if (!bos)
+		return -ENOMEM;
 	memcpy(bos, job->in_bos, job->in_bo_count * sizeof(void *));
 	memcpy(&bos[job->in_bo_count], job->out_bos, job->out_bo_count * sizeof(void *));
 
-	ret = drm_gem_lock_reservations(bos, job->in_bo_count + job->out_bo_count, &acquire_ctx);
+	ret = drm_gem_lock_reservations(bos, bo_count, &acquire_ctx);
 	if (ret)
 		goto err;
 
@@ -220,7 +227,7 @@ static int rocket_job_push(struct rocket_job *job)
 	rocket_attach_object_fences(job->out_bos, job->out_bo_count, job->inference_done_fence);
 
 err_unlock:
-	drm_gem_unlock_reservations(bos, job->in_bo_count + job->out_bo_count, &acquire_ctx);
+	drm_gem_unlock_reservations(bos, bo_count, &acquire_ctx);
 err:
 	kvfree(bos);
 
@@ -310,13 +317,13 @@ static struct dma_fence *rocket_job_run(struct drm_sched_job *sched_job)
 		dma_fence_put(job->done_fence);
 	job->done_fence = dma_fence_get(fence);
 
-	ret = pm_runtime_get_sync(core->dev);
+	ret = pm_runtime_resume_and_get(core->dev);
 	if (ret < 0)
-		return fence;
+		goto err_put_fences;
 
 	ret = iommu_attach_group(job->domain->domain, core->iommu_group);
 	if (ret < 0)
-		return fence;
+		goto err_put_pm;
 
 	scoped_guard(mutex, &core->job_lock) {
 		core->in_flight_job = job;
@@ -324,6 +331,14 @@ static struct dma_fence *rocket_job_run(struct drm_sched_job *sched_job)
 	}
 
 	return fence;
+
+err_put_pm:
+	pm_runtime_put(core->dev);
+err_put_fences:
+	dma_fence_put(job->done_fence);
+	job->done_fence = NULL;
+	dma_fence_put(fence);
+	return ERR_PTR(ret);
 }
 
 static void rocket_job_handle_irq(struct rocket_core *core)
@@ -548,6 +563,7 @@ static int rocket_ioctl_submit_job(struct drm_device *dev, struct drm_file *file
 	kref_init(&rjob->refcount);
 
 	rjob->rdev = rdev;
+	rjob->domain = rocket_iommu_domain_get(file_priv);
 
 	ret = drm_sched_job_init(&rjob->base,
 				 &file_priv->sched_entity,
@@ -573,8 +589,6 @@ static int rocket_ioctl_submit_job(struct drm_device *dev, struct drm_file *file
 
 	rjob->out_bo_count = job->out_bo_handle_count;
 
-	rjob->domain = rocket_iommu_domain_get(file_priv);
-
 	ret = rocket_job_push(rjob);
 	if (ret)
 		goto out_cleanup_job;
diff --git a/drivers/acpi/acpi_processor.c b/drivers/acpi/acpi_processor.c
index 00775b91bd418..5fbab54171b7a 100644
--- a/drivers/acpi/acpi_processor.c
+++ b/drivers/acpi/acpi_processor.c
@@ -994,3 +994,357 @@ int acpi_processor_evaluate_cst(acpi_handle handle, u32 cpu,
 }
 EXPORT_SYMBOL_NS_GPL(acpi_processor_evaluate_cst, "ACPI_PROCESSOR_IDLE");
 #endif /* CONFIG_ACPI_PROCESSOR_CSTATE */
+
+#ifdef CONFIG_ACPI_PROCESSOR_IDLE
+struct acpi_lpi_states_array {
+	unsigned int size;
+	unsigned int composite_states_size;
+	struct acpi_lpi_state *entries;
+	struct acpi_lpi_state *composite_states[ACPI_PROCESSOR_MAX_POWER];
+};
+
+static int obj_get_integer(union acpi_object *obj, u32 *value)
+{
+	if (obj->type != ACPI_TYPE_INTEGER)
+		return -EINVAL;
+
+	*value = obj->integer.value;
+	return 0;
+}
+
+#define lpi_state_debug(handle, message, state_idx)	\
+	acpi_handle_debug(handle, message " for _LPI state %u\n", state_idx)
+
+static void process_lpi_state_package(union acpi_object *lpi_pkg,
+				      struct acpi_lpi_state *lpi_state,
+				      acpi_handle handle,
+				      unsigned int state_idx, bool strict)
+{
+	union acpi_object *lpi_pkg_elem, *obj;
+
+	if (lpi_pkg->type != ACPI_TYPE_PACKAGE || lpi_pkg->package.count < 7)
+		return;
+
+	lpi_pkg_elem = lpi_pkg->package.elements;
+
+	/* Get the entry method first and skip the state if that fails. */
+	obj = &lpi_pkg_elem[6];
+	if (obj->type == ACPI_TYPE_BUFFER) {
+		struct acpi_power_register *reg;
+
+		if (obj->buffer.length < sizeof(*reg)) {
+			lpi_state_debug(handle, "Invalid register data", state_idx);
+			return;
+		}
+
+		reg = (struct acpi_power_register *)obj->buffer.pointer;
+		if (reg->space_id != ACPI_ADR_SPACE_FIXED_HARDWARE) {
+			lpi_state_debug(handle, "Unsupported entry method", state_idx);
+			return;
+		}
+
+		lpi_state->entry_method = ACPI_CSTATE_FFH;
+		lpi_state->address = reg->address;
+	} else if (obj->type == ACPI_TYPE_INTEGER) {
+		lpi_state->entry_method = ACPI_CSTATE_INTEGER;
+		lpi_state->address = obj->integer.value;
+	} else {
+		lpi_state_debug(handle, "Invalid entry method", state_idx);
+		return;
+	}
+
+	if (obj_get_integer(&lpi_pkg_elem[0], &lpi_state->min_residency)) {
+		if (strict) {
+			lpi_state_debug(handle, "No min. residency", state_idx);
+			return;
+		}
+
+		lpi_state_debug(handle, "Assuming 10 us min. residency", state_idx);
+		lpi_state->min_residency = 10;
+	}
+
+	if (obj_get_integer(&lpi_pkg_elem[1], &lpi_state->wake_latency)) {
+		if (strict) {
+			lpi_state_debug(handle, "No wake latency", state_idx);
+			return;
+		}
+
+		lpi_state_debug(handle, "Assuming 10 us wake latency", state_idx);
+		lpi_state->wake_latency = 10;
+	}
+
+	if (obj_get_integer(&lpi_pkg_elem[2], &lpi_state->flags))
+		lpi_state->flags = 0;
+
+	if (obj_get_integer(&lpi_pkg_elem[3], &lpi_state->arch_flags))
+		lpi_state->arch_flags = 0;
+
+	if (obj_get_integer(&lpi_pkg_elem[4], &lpi_state->res_cnt_freq))
+		lpi_state->res_cnt_freq = 1;
+
+	if (obj_get_integer(&lpi_pkg_elem[5], &lpi_state->enable_parent_state))
+		lpi_state->enable_parent_state = 0;
+
+	/* Skip elements [7-8] i.e. Residency/Usage counters. */
+
+	/*
+	 * Avoid out-of-bounds access if the size of the package is less than
+	 * expected.
+	 */
+	if (lpi_pkg->package.count < 10)
+		return;
+
+	obj = &lpi_pkg_elem[9];
+	if (obj->type == ACPI_TYPE_STRING)
+		strscpy(lpi_state->desc, obj->string.pointer, ACPI_CX_DESC_LEN);
+}
+
+static int acpi_processor_evaluate_lpi(acpi_handle handle,
+				       struct acpi_lpi_states_array *info,
+				       bool strict)
+{
+	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object *lpi_data, *lpi_pkg;
+	unsigned int lpi_pkg_count, state_idx;
+	struct acpi_lpi_state *lpi_state;
+	acpi_status status;
+	int ret = 0;
+
+	status = acpi_evaluate_object(handle, "_LPI", NULL, &buffer);
+	if (ACPI_FAILURE(status)) {
+		acpi_handle_debug(handle, "No _LPI, giving up\n");
+		return -ENODEV;
+	}
+
+	lpi_data = buffer.pointer;
+
+	/* There must be at least 4 elements = 3 elements + 1 package */
+	if (!lpi_data || lpi_data->type != ACPI_TYPE_PACKAGE ||
+	    lpi_data->package.count < 4) {
+		acpi_handle_debug(handle, "Not enough elements in _LPI\n");
+		ret = -ENODATA;
+		goto end;
+	}
+
+	lpi_pkg_count = lpi_data->package.elements[2].integer.value;
+
+	/* Validate number of power states. */
+	if (!lpi_pkg_count || lpi_pkg_count != lpi_data->package.count - 3) {
+		acpi_handle_debug(handle, "Invalid _LPI state count\n");
+		ret = -ENODATA;
+		goto end;
+	}
+
+	lpi_state = kzalloc_objs(*lpi_state, lpi_pkg_count);
+	if (!lpi_state) {
+		ret = -ENOMEM;
+		goto end;
+	}
+
+	info->size = lpi_pkg_count;
+	info->entries = lpi_state;
+
+	/* _LPI State packages start at index 3. */
+	lpi_pkg = &lpi_data->package.elements[3];
+
+	for (state_idx = 1; state_idx <= lpi_pkg_count; state_idx++) {
+		lpi_state->index = state_idx;
+		process_lpi_state_package(lpi_pkg++, lpi_state++, handle,
+					  state_idx, strict);
+	}
+
+	acpi_handle_debug(handle, "Found %u power states\n", lpi_pkg_count);
+end:
+	kfree(buffer.pointer);
+	return ret;
+}
+
+/**
+ * combine_lpi_states - combine local and parent LPI states to form a composite LPI state
+ *
+ * @local: local LPI state
+ * @parent: parent LPI state
+ * @result: composite LPI state
+ */
+static bool combine_lpi_states(struct acpi_lpi_state *local,
+			       struct acpi_lpi_state *parent,
+			       struct acpi_lpi_state *result)
+{
+	if (parent->entry_method == ACPI_CSTATE_INTEGER) {
+		if (!parent->address) /* 0 means autopromotable */
+			return false;
+		result->address = local->address + parent->address;
+	} else {
+		result->address = parent->address;
+	}
+
+	result->min_residency = max(local->min_residency, parent->min_residency);
+	result->wake_latency = local->wake_latency + parent->wake_latency;
+	result->enable_parent_state = parent->enable_parent_state;
+	result->entry_method = local->entry_method;
+
+	result->flags = parent->flags;
+	result->arch_flags = parent->arch_flags;
+	result->index = parent->index;
+
+	scnprintf(result->desc, ACPI_CX_DESC_LEN, "%s+%s", local->desc, parent->desc);
+	return true;
+}
+
+#define ACPI_LPI_STATE_FLAGS_ENABLED			BIT(0)
+
+static void stash_composite_state(struct acpi_lpi_states_array *curr_level,
+				  struct acpi_lpi_state *t)
+{
+	curr_level->composite_states[curr_level->composite_states_size++] = t;
+}
+
+static bool too_many_states(acpi_handle handle, unsigned int state_count)
+{
+	if (state_count < ACPI_PROCESSOR_MAX_POWER)
+		return false;
+
+	acpi_handle_info(handle, "No space for more _LPI states than %d\n",
+			 ACPI_PROCESSOR_MAX_POWER);
+	return true;
+}
+
+static unsigned int flatten_lpi_states(acpi_handle handle,
+				       struct acpi_lpi_state *lpi_states,
+				       unsigned int state_count,
+				       struct acpi_lpi_states_array *curr,
+				       struct acpi_lpi_states_array *prev)
+{
+	struct acpi_lpi_state *parent_lpi = curr->entries;
+	unsigned int j;
+
+	/*
+	 * Combine each of the "raw" _LPI states from the current (processor
+	 * container) level with all of the composite _LPI states from the
+	 * previous (processor or processor container) level.
+	 */
+	for (j = 0; j < curr->size; j++, parent_lpi++) {
+		struct acpi_lpi_state *flpi;
+		int i;
+
+		if (!(parent_lpi->flags & ACPI_LPI_STATE_FLAGS_ENABLED))
+			continue;
+
+		if (too_many_states(handle, state_count))
+			break;
+
+		flpi = &lpi_states[state_count];
+
+		for (i = 0; i < prev->composite_states_size; i++) {
+			struct acpi_lpi_state *local_lpi = prev->composite_states[i];
+
+			if (parent_lpi->index > local_lpi->enable_parent_state)
+				continue;
+
+			if (!combine_lpi_states(local_lpi, parent_lpi, flpi))
+				continue;
+
+			stash_composite_state(curr, flpi);
+			state_count++;
+			flpi++;
+
+			if (state_count >= ACPI_PROCESSOR_MAX_POWER)
+				break;
+		}
+	}
+
+	return state_count;
+}
+
+int acpi_processor_extract_lpi_info(acpi_handle pr_handle,
+				    struct acpi_processor_power *pr_power,
+				    bool strict)
+{
+	struct acpi_lpi_states_array info[2], *prev, *curr;
+	acpi_handle handle = pr_handle;
+	unsigned int state_count = 0;
+	unsigned int i;
+	int ret;
+
+	if (!osc_pc_lpi_support_confirmed)
+		return -EOPNOTSUPP;
+
+	curr = &info[0];
+	curr->composite_states_size = 0;
+
+	ret = acpi_processor_evaluate_lpi(handle, curr, strict);
+	if (ret)
+		return ret;
+
+	/* Copy all of the usable first-level states to power.lpi_states[]. */
+	for (i = 0; i < curr->size; i++) {
+		struct acpi_lpi_state *lpi = &curr->entries[i];
+		struct acpi_lpi_state *flpi;
+
+		/*
+		 * Skip states that are not enabled or have an inadequate entry
+		 * method for this level.
+		 */
+		if (!(lpi->flags & ACPI_LPI_STATE_FLAGS_ENABLED) ||
+		    lpi->entry_method == ACPI_CSTATE_INTEGER)
+			continue;
+
+		if (too_many_states(pr_handle, state_count))
+			break;
+
+		flpi = &pr_power->lpi_states[state_count++];
+		memcpy(flpi, lpi, sizeof(*lpi));
+		stash_composite_state(curr, flpi);
+	}
+
+	kfree(curr->entries);
+
+	/*
+	 * If there are no _LPI states at the first level, there are no _LPI
+	 * states at all.
+	 */
+	if (!state_count)
+		return -ENODATA;
+
+	prev = curr;
+	curr = &info[1];
+
+	for (;;) {
+		struct acpi_lpi_states_array *tmp;
+		struct acpi_device *d;
+
+		if (ACPI_FAILURE(acpi_get_parent(handle, &handle)))
+			break;
+
+		d = acpi_fetch_acpi_dev(handle);
+		if (!d)
+			break;
+
+		if (strcmp(acpi_device_hid(d), ACPI_PROCESSOR_CONTAINER_HID))
+			break;
+
+		curr->composite_states_size = 0;
+
+		ret = acpi_processor_evaluate_lpi(handle, curr, strict);
+		if (ret)
+			break;
+
+		/* flatten all the LPI states in this level of hierarchy */
+		state_count = flatten_lpi_states(pr_handle, pr_power->lpi_states,
+						 state_count, curr, prev);
+
+		kfree(curr->entries);
+
+		tmp = prev, prev = curr, curr = tmp;
+	}
+
+	/* reset the index after flattening */
+	for (i = 0; i < state_count; i++)
+		pr_power->lpi_states[i].index = i;
+
+	pr_power->count = state_count;
+
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(acpi_processor_extract_lpi_info, "ACPI_PROCESSOR_IDLE");
+#endif /* CONFIG_ACPI_PROCESSOR_IDLE */
diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c
index 3236a3ce79d6b..48f915ee5a0de 100644
--- a/drivers/acpi/apei/ghes.c
+++ b/drivers/acpi/apei/ghes.c
@@ -1397,8 +1397,8 @@ static int ghes_in_nmi_queue_one_entry(struct ghes *ghes,
 	return rc;
 }
 
-static int ghes_in_nmi_spool_from_list(struct list_head *rcu_list,
-				       enum fixed_addresses fixmap_idx)
+static int __maybe_unused ghes_in_nmi_spool_from_list(struct list_head *rcu_list,
+			       enum fixed_addresses fixmap_idx)
 {
 	int ret = -ENOENT;
 	struct ghes *ghes;
diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c
index af7a9b2fd5bc2..34412cd697d8b 100644
--- a/drivers/acpi/arm64/iort.c
+++ b/drivers/acpi/arm64/iort.c
@@ -789,11 +789,9 @@ struct irq_domain *iort_get_device_domain(struct device *dev, u32 id,
 	return irq_find_matching_fwnode(handle, bus_token);
 }
 
-struct fwnode_handle *iort_iwb_handle(u32 iwb_id)
+acpi_handle iort_iwb_handle(u32 iwb_id)
 {
-	struct fwnode_handle *fwnode;
 	struct acpi_iort_node *node;
-	struct acpi_device *device;
 	struct acpi_iort_iwb *iwb;
 	acpi_status status;
 	acpi_handle handle;
@@ -808,6 +806,19 @@ struct fwnode_handle *iort_iwb_handle(u32 iwb_id)
 	if (ACPI_FAILURE(status))
 		return NULL;
 
+	return handle;
+}
+
+struct fwnode_handle *iort_iwb_handle_fwnode(u32 iwb_id)
+{
+	struct fwnode_handle *fwnode;
+	struct acpi_device *device;
+	acpi_handle handle;
+
+	handle = iort_iwb_handle(iwb_id);
+	if (!handle)
+		return NULL;
+
 	device = acpi_get_acpi_dev(handle);
 	if (!device)
 		return NULL;
@@ -2090,6 +2101,11 @@ static void __init iort_init_platform_devices(void)
 	}
 }
 
+u32 arch_acpi_add_auto_dep(acpi_handle handle)
+{
+	return acpi_irq_add_auto_dep(handle);
+}
+
 void __init acpi_iort_init(void)
 {
 	acpi_status status;
diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c
index a30a904f6535f..fae79cdd36102 100644
--- a/drivers/acpi/bus.c
+++ b/drivers/acpi/bus.c
@@ -620,41 +620,6 @@ static void acpi_bus_notify(acpi_handle handle, u32 type, void *data)
 	acpi_evaluate_ost(handle, type, ACPI_OST_SC_NON_SPECIFIC_FAILURE, NULL);
 }
 
-static void acpi_notify_device(acpi_handle handle, u32 event, void *data)
-{
-	struct acpi_device *device = data;
-	struct acpi_driver *acpi_drv = to_acpi_driver(device->dev.driver);
-
-	acpi_drv->ops.notify(device, event);
-}
-
-static int acpi_device_install_notify_handler(struct acpi_device *device,
-					      struct acpi_driver *acpi_drv)
-{
-	u32 type = acpi_drv->flags & ACPI_DRIVER_ALL_NOTIFY_EVENTS ?
-				ACPI_ALL_NOTIFY : ACPI_DEVICE_NOTIFY;
-	acpi_status status;
-
-	status = acpi_install_notify_handler(device->handle, type,
-					     acpi_notify_device, device);
-	if (ACPI_FAILURE(status))
-		return -EINVAL;
-
-	return 0;
-}
-
-static void acpi_device_remove_notify_handler(struct acpi_device *device,
-					      struct acpi_driver *acpi_drv)
-{
-	u32 type = acpi_drv->flags & ACPI_DRIVER_ALL_NOTIFY_EVENTS ?
-				ACPI_ALL_NOTIFY : ACPI_DEVICE_NOTIFY;
-
-	acpi_remove_notify_handler(device->handle, type,
-				   acpi_notify_device);
-
-	acpi_os_wait_events_complete();
-}
-
 int acpi_dev_install_notify_handler(struct acpi_device *adev,
 				    u32 handler_type,
 				    acpi_notify_handler handler, void *context)
@@ -1121,57 +1086,13 @@ bool acpi_driver_match_device(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(acpi_driver_match_device);
 
-/* --------------------------------------------------------------------------
-                              ACPI Driver Management
-   -------------------------------------------------------------------------- */
-
-/**
- * __acpi_bus_register_driver - register a driver with the ACPI bus
- * @driver: driver being registered
- * @owner: owning module/driver
- *
- * Registers a driver with the ACPI bus.  Searches the namespace for all
- * devices that match the driver's criteria and binds.  Returns zero for
- * success or a negative error status for failure.
- */
-int __acpi_bus_register_driver(struct acpi_driver *driver, struct module *owner)
-{
-	if (acpi_disabled)
-		return -ENODEV;
-	driver->drv.name = driver->name;
-	driver->drv.bus = &acpi_bus_type;
-	driver->drv.owner = owner;
-
-	return driver_register(&driver->drv);
-}
-
-EXPORT_SYMBOL(__acpi_bus_register_driver);
-
-/**
- * acpi_bus_unregister_driver - unregisters a driver with the ACPI bus
- * @driver: driver to unregister
- *
- * Unregisters a driver with the ACPI bus.  Searches the namespace for all
- * devices that match the driver's criteria and unbinds.
- */
-void acpi_bus_unregister_driver(struct acpi_driver *driver)
-{
-	driver_unregister(&driver->drv);
-}
-
-EXPORT_SYMBOL(acpi_bus_unregister_driver);
-
 /* --------------------------------------------------------------------------
                               ACPI Bus operations
    -------------------------------------------------------------------------- */
 
 static int acpi_bus_match(struct device *dev, const struct device_driver *drv)
 {
-	struct acpi_device *acpi_dev = to_acpi_device(dev);
-	const struct acpi_driver *acpi_drv = to_acpi_driver(drv);
-
-	return acpi_dev->flags.match_driver
-		&& !acpi_match_device_ids(acpi_dev, acpi_drv->ids);
+	return 0;
 }
 
 static int acpi_device_uevent(const struct device *dev, struct kobj_uevent_env *env)
@@ -1179,66 +1100,9 @@ static int acpi_device_uevent(const struct device *dev, struct kobj_uevent_env *
 	return __acpi_device_uevent_modalias(to_acpi_device(dev), env);
 }
 
-static int acpi_device_probe(struct device *dev)
-{
-	struct acpi_device *acpi_dev = to_acpi_device(dev);
-	struct acpi_driver *acpi_drv = to_acpi_driver(dev->driver);
-	int ret;
-
-	if (acpi_dev->handler && !acpi_is_pnp_device(acpi_dev))
-		return -EINVAL;
-
-	if (!acpi_drv->ops.add)
-		return -ENOSYS;
-
-	ret = acpi_drv->ops.add(acpi_dev);
-	if (ret) {
-		acpi_dev->driver_data = NULL;
-		return ret;
-	}
-
-	pr_debug("Driver [%s] successfully bound to device [%s]\n",
-		 acpi_drv->name, acpi_dev->pnp.bus_id);
-
-	if (acpi_drv->ops.notify) {
-		ret = acpi_device_install_notify_handler(acpi_dev, acpi_drv);
-		if (ret) {
-			if (acpi_drv->ops.remove)
-				acpi_drv->ops.remove(acpi_dev);
-
-			acpi_dev->driver_data = NULL;
-			return ret;
-		}
-	}
-
-	pr_debug("Found driver [%s] for device [%s]\n", acpi_drv->name,
-		 acpi_dev->pnp.bus_id);
-
-	get_device(dev);
-	return 0;
-}
-
-static void acpi_device_remove(struct device *dev)
-{
-	struct acpi_device *acpi_dev = to_acpi_device(dev);
-	struct acpi_driver *acpi_drv = to_acpi_driver(dev->driver);
-
-	if (acpi_drv->ops.notify)
-		acpi_device_remove_notify_handler(acpi_dev, acpi_drv);
-
-	if (acpi_drv->ops.remove)
-		acpi_drv->ops.remove(acpi_dev);
-
-	acpi_dev->driver_data = NULL;
-
-	put_device(dev);
-}
-
 const struct bus_type acpi_bus_type = {
 	.name		= "acpi",
 	.match		= acpi_bus_match,
-	.probe		= acpi_device_probe,
-	.remove		= acpi_device_remove,
 	.uevent		= acpi_device_uevent,
 };
 
diff --git a/drivers/acpi/fan_core.c b/drivers/acpi/fan_core.c
index fb08b8549ed74..624d0736b581c 100644
--- a/drivers/acpi/fan_core.c
+++ b/drivers/acpi/fan_core.c
@@ -489,26 +489,6 @@ static void acpi_fan_notify_handler(acpi_handle handle, u32 event, void *context
 	}
 }
 
-static void acpi_fan_notify_remove(void *data)
-{
-	struct acpi_fan *fan = data;
-
-	acpi_remove_notify_handler(fan->handle, ACPI_DEVICE_NOTIFY, acpi_fan_notify_handler);
-}
-
-static int devm_acpi_fan_notify_init(struct device *dev)
-{
-	struct acpi_fan *fan = dev_get_drvdata(dev);
-	acpi_status status;
-
-	status = acpi_install_notify_handler(fan->handle, ACPI_DEVICE_NOTIFY,
-					     acpi_fan_notify_handler, dev);
-	if (ACPI_FAILURE(status))
-		return -EIO;
-
-	return devm_add_action_or_reset(dev, acpi_fan_notify_remove, fan);
-}
-
 static int acpi_fan_probe(struct platform_device *pdev)
 {
 	int result = 0;
@@ -556,7 +536,10 @@ static int acpi_fan_probe(struct platform_device *pdev)
 		if (result)
 			return result;
 
-		result = devm_acpi_fan_notify_init(&pdev->dev);
+		result = devm_acpi_install_notify_handler(&pdev->dev,
+							  ACPI_DEVICE_NOTIFY,
+							  acpi_fan_notify_handler,
+							  &pdev->dev);
 		if (result)
 			return result;
 
diff --git a/drivers/acpi/irq.c b/drivers/acpi/irq.c
index d1595156c86a4..e4293458bf619 100644
--- a/drivers/acpi/irq.c
+++ b/drivers/acpi/irq.c
@@ -13,6 +13,7 @@
 enum acpi_irq_model_id acpi_irq_model;
 
 static acpi_gsi_domain_disp_fn acpi_get_gsi_domain_id;
+static acpi_gsi_handle_disp_fn acpi_get_gsi_handle;
 static u32 (*acpi_gsi_to_irq_fallback)(u32 gsi);
 
 /**
@@ -321,15 +322,19 @@ const struct cpumask *acpi_irq_get_affinity(acpi_handle handle,
 
 /**
  * acpi_set_irq_model - Setup the GSI irqdomain information
- * @model: the value assigned to acpi_irq_model
- * @fn: a dispatcher function that will return the domain fwnode
- *	for a given GSI
+ * @model:	the value assigned to acpi_irq_model
+ * @fn:		a dispatcher function that will return the domain fwnode
+ *		for a given GSI
+ * @gsi_dep_fn: a function to retrieve the acpi_handle a GSI interrupt is
+ *		dependent on
+ *
  */
 void __init acpi_set_irq_model(enum acpi_irq_model_id model,
-			       acpi_gsi_domain_disp_fn fn)
+			       acpi_gsi_domain_disp_fn fn, acpi_gsi_handle_disp_fn gsi_dep_fn)
 {
 	acpi_irq_model = model;
 	acpi_get_gsi_domain_id = fn;
+	acpi_get_gsi_handle = gsi_dep_fn;
 }
 
 /*
@@ -385,3 +390,162 @@ struct irq_domain *acpi_irq_create_hierarchy(unsigned int flags,
 					   host_data);
 }
 EXPORT_SYMBOL_GPL(acpi_irq_create_hierarchy);
+
+struct acpi_irq_dep_ctx {
+	int		rc;
+	unsigned int	index;
+	acpi_handle	handle;
+};
+
+static acpi_status acpi_irq_get_parent(struct acpi_resource *ares, void *context)
+{
+	struct acpi_irq_dep_ctx *ctx = context;
+	struct acpi_resource_irq *irq;
+	struct acpi_resource_extended_irq *eirq;
+
+	switch (ares->type) {
+	case ACPI_RESOURCE_TYPE_IRQ:
+		irq = &ares->data.irq;
+		if (ctx->index >= irq->interrupt_count) {
+			ctx->index -= irq->interrupt_count;
+			return AE_OK;
+		}
+		ctx->handle = acpi_get_gsi_handle(irq->interrupts[ctx->index]);
+		ctx->rc = 0;
+		return AE_CTRL_TERMINATE;
+	case ACPI_RESOURCE_TYPE_EXTENDED_IRQ:
+		eirq = &ares->data.extended_irq;
+		if (eirq->producer_consumer == ACPI_PRODUCER)
+			return AE_OK;
+
+		if (ctx->index >= eirq->interrupt_count) {
+			ctx->index -= eirq->interrupt_count;
+			return AE_OK;
+		}
+
+		/* Support GSIs only */
+		if (eirq->resource_source.string_length)
+			return AE_OK;
+
+		ctx->handle = acpi_get_gsi_handle(eirq->interrupts[ctx->index]);
+		ctx->rc = 0;
+		return AE_CTRL_TERMINATE;
+	}
+
+	return AE_OK;
+}
+
+static int acpi_irq_get_dep(acpi_handle handle, unsigned int index, acpi_handle *gsi_handle)
+{
+	struct acpi_irq_dep_ctx ctx = {-EINVAL, index, NULL};
+
+	if (!gsi_handle)
+		return -EINVAL;
+
+	acpi_walk_resources(handle, METHOD_NAME__CRS, acpi_irq_get_parent, &ctx);
+	*gsi_handle = ctx.handle;
+
+	return ctx.rc;
+}
+
+static bool acpi_prt_entry_valid(void *prt_entry)
+{
+	struct acpi_pci_routing_table *entry = prt_entry;
+
+	return entry && entry->length > 0;
+}
+
+static void *acpi_prt_next_entry(void *prt_entry)
+{
+	struct acpi_pci_routing_table *entry = prt_entry;
+
+	return prt_entry + entry->length;
+}
+
+static u32 acpi_add_prt_dep(acpi_handle handle)
+{
+	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
+	struct acpi_pci_routing_table *entry;
+	struct acpi_handle_list dep_devices;
+	acpi_handle gsi_handle;
+	acpi_handle link_handle;
+	acpi_status status;
+	u32 count = 0;
+
+	status = acpi_get_irq_routing_table(handle, &buffer);
+	if (ACPI_FAILURE(status)) {
+		acpi_handle_err(handle, "failed to get IRQ routing table\n");
+		kfree(buffer.pointer);
+		return 0;
+	}
+
+	entry = buffer.pointer;
+	for (; acpi_prt_entry_valid(entry); entry = acpi_prt_next_entry(entry)) {
+		if (entry->source[0]) {
+			status = acpi_get_handle(handle, entry->source, &link_handle);
+			if (ACPI_FAILURE(status))
+				continue;
+			dep_devices.count = 1;
+			dep_devices.handles = kcalloc(1, sizeof(*dep_devices.handles), GFP_KERNEL);
+			if (!dep_devices.handles) {
+				acpi_handle_err(handle, "failed to allocate memory\n");
+				continue;
+			}
+
+			dep_devices.handles[0] = link_handle;
+			count += acpi_scan_add_dep(handle, &dep_devices);
+		} else {
+			gsi_handle = acpi_get_gsi_handle(entry->source_index);
+			if (!gsi_handle)
+				continue;
+			dep_devices.count = 1;
+			dep_devices.handles = kcalloc(1, sizeof(*dep_devices.handles), GFP_KERNEL);
+			if (!dep_devices.handles) {
+				acpi_handle_err(handle, "failed to allocate memory\n");
+				continue;
+			}
+
+			dep_devices.handles[0] = gsi_handle;
+			count += acpi_scan_add_dep(handle, &dep_devices);
+		}
+	}
+
+	kfree(buffer.pointer);
+	return count;
+}
+
+static u32 acpi_add_irq_dep(acpi_handle handle)
+{
+	struct acpi_handle_list dep_devices;
+	acpi_handle gsi_handle;
+	u32 count = 0;
+	int i;
+
+	for (i = 0; !acpi_irq_get_dep(handle, i, &gsi_handle); i++) {
+		if (!gsi_handle)
+			continue;
+
+		dep_devices.count = 1;
+		dep_devices.handles = kcalloc(1, sizeof(*dep_devices.handles), GFP_KERNEL);
+		if (!dep_devices.handles) {
+			acpi_handle_err(handle, "failed to allocate memory\n");
+			continue;
+		}
+
+		dep_devices.handles[0] = gsi_handle;
+		count += acpi_scan_add_dep(handle, &dep_devices);
+	}
+
+	return count;
+}
+
+u32 acpi_irq_add_auto_dep(acpi_handle handle)
+{
+	if (!acpi_get_gsi_handle)
+		return 0;
+
+	if (acpi_has_method(handle, "_PRT"))
+		return acpi_add_prt_dep(handle);
+
+	return acpi_add_irq_dep(handle);
+}
diff --git a/drivers/acpi/numa/srat.c b/drivers/acpi/numa/srat.c
index 62d4a8df0b8c1..5c407dc6401e8 100644
--- a/drivers/acpi/numa/srat.c
+++ b/drivers/acpi/numa/srat.c
@@ -399,8 +399,6 @@ acpi_parse_memory_affinity(union acpi_subtable_headers *header,
 		goto out_err_bad_srat;
 	}
 
-	node_set(node, numa_nodes_parsed);
-
 	pr_info("SRAT: Node %u PXM %u [mem %#010Lx-%#010Lx]%s%s\n",
 		node, pxm,
 		(unsigned long long) start, (unsigned long long) end - 1,
diff --git a/drivers/acpi/power.c b/drivers/acpi/power.c
index d4131c184be8e..23a4e207a01eb 100644
--- a/drivers/acpi/power.c
+++ b/drivers/acpi/power.c
@@ -954,7 +954,6 @@ struct acpi_device *acpi_add_power_resource(acpi_handle handle)
 	INIT_LIST_HEAD(&resource->list_node);
 	INIT_LIST_HEAD(&resource->dependents);
 	device->power.state = ACPI_STATE_UNKNOWN;
-	device->flags.match_driver = true;
 
 	/* Evaluate the object to get the system level and resource order. */
 	status = acpi_evaluate_object(handle, NULL, NULL, &buffer);
diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c
index 4482cf28f56ae..e113bcbbb882e 100644
--- a/drivers/acpi/processor_idle.c
+++ b/drivers/acpi/processor_idle.c
@@ -853,223 +853,6 @@ static int acpi_processor_setup_cstates(struct acpi_processor *pr)
 
 #endif /* CONFIG_ACPI_PROCESSOR_CSTATE */
 
-struct acpi_lpi_states_array {
-	unsigned int size;
-	unsigned int composite_states_size;
-	struct acpi_lpi_state *entries;
-	struct acpi_lpi_state *composite_states[ACPI_PROCESSOR_MAX_POWER];
-};
-
-static int obj_get_integer(union acpi_object *obj, u32 *value)
-{
-	if (obj->type != ACPI_TYPE_INTEGER)
-		return -EINVAL;
-
-	*value = obj->integer.value;
-	return 0;
-}
-
-static int acpi_processor_evaluate_lpi(acpi_handle handle,
-				       struct acpi_lpi_states_array *info)
-{
-	acpi_status status;
-	int ret = 0;
-	int pkg_count, state_idx = 1, loop;
-	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
-	union acpi_object *lpi_data;
-	struct acpi_lpi_state *lpi_state;
-
-	status = acpi_evaluate_object(handle, "_LPI", NULL, &buffer);
-	if (ACPI_FAILURE(status)) {
-		acpi_handle_debug(handle, "No _LPI, giving up\n");
-		return -ENODEV;
-	}
-
-	lpi_data = buffer.pointer;
-
-	/* There must be at least 4 elements = 3 elements + 1 package */
-	if (!lpi_data || lpi_data->type != ACPI_TYPE_PACKAGE ||
-	    lpi_data->package.count < 4) {
-		pr_debug("not enough elements in _LPI\n");
-		ret = -ENODATA;
-		goto end;
-	}
-
-	pkg_count = lpi_data->package.elements[2].integer.value;
-
-	/* Validate number of power states. */
-	if (pkg_count < 1 || pkg_count != lpi_data->package.count - 3) {
-		pr_debug("count given by _LPI is not valid\n");
-		ret = -ENODATA;
-		goto end;
-	}
-
-	lpi_state = kzalloc_objs(*lpi_state, pkg_count);
-	if (!lpi_state) {
-		ret = -ENOMEM;
-		goto end;
-	}
-
-	info->size = pkg_count;
-	info->entries = lpi_state;
-
-	/* LPI States start at index 3 */
-	for (loop = 3; state_idx <= pkg_count; loop++, state_idx++, lpi_state++) {
-		union acpi_object *element, *pkg_elem, *obj;
-
-		element = &lpi_data->package.elements[loop];
-		if (element->type != ACPI_TYPE_PACKAGE || element->package.count < 7)
-			continue;
-
-		pkg_elem = element->package.elements;
-
-		obj = pkg_elem + 6;
-		if (obj->type == ACPI_TYPE_BUFFER) {
-			struct acpi_power_register *reg;
-
-			reg = (struct acpi_power_register *)obj->buffer.pointer;
-			if (reg->space_id != ACPI_ADR_SPACE_SYSTEM_IO &&
-			    reg->space_id != ACPI_ADR_SPACE_FIXED_HARDWARE)
-				continue;
-
-			lpi_state->address = reg->address;
-			lpi_state->entry_method =
-				reg->space_id == ACPI_ADR_SPACE_FIXED_HARDWARE ?
-				ACPI_CSTATE_FFH : ACPI_CSTATE_SYSTEMIO;
-		} else if (obj->type == ACPI_TYPE_INTEGER) {
-			lpi_state->entry_method = ACPI_CSTATE_INTEGER;
-			lpi_state->address = obj->integer.value;
-		} else {
-			pr_debug("Entry method of state-%d is invalid, disable it.\n",
-				 state_idx);
-			continue;
-		}
-
-		/* elements[7,8] skipped for now i.e. Residency/Usage counter*/
-
-		obj = pkg_elem + 9;
-		if (obj->type == ACPI_TYPE_STRING)
-			strscpy(lpi_state->desc, obj->string.pointer,
-				ACPI_CX_DESC_LEN);
-
-		lpi_state->index = state_idx;
-		if (obj_get_integer(pkg_elem + 0, &lpi_state->min_residency)) {
-			pr_debug("No min. residency found, assuming 10 us\n");
-			lpi_state->min_residency = 10;
-		}
-
-		if (obj_get_integer(pkg_elem + 1, &lpi_state->wake_latency)) {
-			pr_debug("No wakeup residency found, assuming 10 us\n");
-			lpi_state->wake_latency = 10;
-		}
-
-		if (obj_get_integer(pkg_elem + 2, &lpi_state->flags))
-			lpi_state->flags = 0;
-
-		if (obj_get_integer(pkg_elem + 3, &lpi_state->arch_flags))
-			lpi_state->arch_flags = 0;
-
-		if (obj_get_integer(pkg_elem + 4, &lpi_state->res_cnt_freq))
-			lpi_state->res_cnt_freq = 1;
-
-		if (obj_get_integer(pkg_elem + 5, &lpi_state->enable_parent_state))
-			lpi_state->enable_parent_state = 0;
-	}
-
-	acpi_handle_debug(handle, "Found %d power states\n", state_idx);
-end:
-	kfree(buffer.pointer);
-	return ret;
-}
-
-/**
- * combine_lpi_states - combine local and parent LPI states to form a composite LPI state
- *
- * @local: local LPI state
- * @parent: parent LPI state
- * @result: composite LPI state
- */
-static bool combine_lpi_states(struct acpi_lpi_state *local,
-			       struct acpi_lpi_state *parent,
-			       struct acpi_lpi_state *result)
-{
-	if (parent->entry_method == ACPI_CSTATE_INTEGER) {
-		if (!parent->address) /* 0 means autopromotable */
-			return false;
-		result->address = local->address + parent->address;
-	} else {
-		result->address = parent->address;
-	}
-
-	result->min_residency = max(local->min_residency, parent->min_residency);
-	result->wake_latency = local->wake_latency + parent->wake_latency;
-	result->enable_parent_state = parent->enable_parent_state;
-	result->entry_method = local->entry_method;
-
-	result->flags = parent->flags;
-	result->arch_flags = parent->arch_flags;
-	result->index = parent->index;
-
-	scnprintf(result->desc, ACPI_CX_DESC_LEN, "%s+%s", local->desc, parent->desc);
-	return true;
-}
-
-#define ACPI_LPI_STATE_FLAGS_ENABLED			BIT(0)
-
-static void stash_composite_state(struct acpi_lpi_states_array *curr_level,
-				  struct acpi_lpi_state *t)
-{
-	curr_level->composite_states[curr_level->composite_states_size++] = t;
-}
-
-static unsigned int flatten_lpi_states(struct acpi_processor *pr,
-				       unsigned int flat_state_cnt,
-				       struct acpi_lpi_states_array *curr_level,
-				       struct acpi_lpi_states_array *prev_level)
-{
-	int i, j, state_count = curr_level->size;
-	struct acpi_lpi_state *p, *t = curr_level->entries;
-
-	curr_level->composite_states_size = 0;
-	for (j = 0; j < state_count; j++, t++) {
-		struct acpi_lpi_state *flpi;
-
-		if (!(t->flags & ACPI_LPI_STATE_FLAGS_ENABLED))
-			continue;
-
-		if (flat_state_cnt >= ACPI_PROCESSOR_MAX_POWER) {
-			pr_warn("Limiting number of LPI states to max (%d)\n",
-				ACPI_PROCESSOR_MAX_POWER);
-			pr_warn("Please increase ACPI_PROCESSOR_MAX_POWER if needed.\n");
-			break;
-		}
-
-		flpi = &pr->power.lpi_states[flat_state_cnt];
-
-		if (!prev_level) { /* leaf/processor node */
-			memcpy(flpi, t, sizeof(*t));
-			stash_composite_state(curr_level, flpi);
-			flat_state_cnt++;
-			continue;
-		}
-
-		for (i = 0; i < prev_level->composite_states_size; i++) {
-			p = prev_level->composite_states[i];
-			if (t->index <= p->enable_parent_state &&
-			    combine_lpi_states(p, t, flpi)) {
-				stash_composite_state(curr_level, flpi);
-				flat_state_cnt++;
-				flpi++;
-				if (flat_state_cnt >= ACPI_PROCESSOR_MAX_POWER)
-					break;
-			}
-		}
-	}
-
-	kfree(curr_level->entries);
-	return flat_state_cnt;
-}
-
 int __weak acpi_processor_ffh_lpi_probe(unsigned int cpu)
 {
 	return -EOPNOTSUPP;
@@ -1077,64 +860,16 @@ int __weak acpi_processor_ffh_lpi_probe(unsigned int cpu)
 
 static int acpi_processor_get_lpi_info(struct acpi_processor *pr)
 {
-	int ret, i;
-	acpi_status status;
-	acpi_handle handle = pr->handle, pr_ahandle;
-	struct acpi_device *d = NULL;
-	struct acpi_lpi_states_array info[2], *tmp, *prev, *curr;
-	unsigned int state_count;
+	int ret;
 
 	/* make sure our architecture has support */
 	ret = acpi_processor_ffh_lpi_probe(pr->id);
 	if (ret == -EOPNOTSUPP)
 		return ret;
 
-	if (!osc_pc_lpi_support_confirmed)
-		return -EOPNOTSUPP;
-
-	if (!acpi_has_method(handle, "_LPI"))
-		return -EINVAL;
-
-	prev = &info[0];
-	curr = &info[1];
-	handle = pr->handle;
-	ret = acpi_processor_evaluate_lpi(handle, prev);
+	ret = acpi_processor_extract_lpi_info(pr->handle, &pr->power, false);
 	if (ret)
 		return ret;
-	state_count = flatten_lpi_states(pr, 0, prev, NULL);
-
-	status = acpi_get_parent(handle, &pr_ahandle);
-	while (ACPI_SUCCESS(status)) {
-		d = acpi_fetch_acpi_dev(pr_ahandle);
-		if (!d)
-			break;
-
-		handle = pr_ahandle;
-
-		if (strcmp(acpi_device_hid(d), ACPI_PROCESSOR_CONTAINER_HID))
-			break;
-
-		/* can be optional ? */
-		if (!acpi_has_method(handle, "_LPI"))
-			break;
-
-		ret = acpi_processor_evaluate_lpi(handle, curr);
-		if (ret)
-			break;
-
-		/* flatten all the LPI states in this level of hierarchy */
-		state_count = flatten_lpi_states(pr, state_count, curr, prev);
-
-		tmp = prev, prev = curr, curr = tmp;
-
-		status = acpi_get_parent(handle, &pr_ahandle);
-	}
-
-	/* reset the index after flattening */
-	for (i = 0; i < state_count; i++)
-		pr->power.lpi_states[i].index = i;
-
-	pr->power.count = state_count;
 
 	/* Tell driver that _LPI is supported. */
 	pr->flags.has_lpi = 1;
diff --git a/drivers/acpi/processor_perflib.c b/drivers/acpi/processor_perflib.c
index fdf55c285c9a7..9e25d6124efda 100644
--- a/drivers/acpi/processor_perflib.c
+++ b/drivers/acpi/processor_perflib.c
@@ -287,7 +287,8 @@ static int acpi_processor_get_performance_control(struct acpi_processor *pr)
  */
 static void amd_fixup_frequency(struct acpi_processor_px *px, int i)
 {
-	u32 hi, lo, fid, did;
+	struct msr val;
+	u32 fid, did;
 	int index = px->control & 0x00000007;
 
 	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
@@ -295,16 +296,16 @@ static void amd_fixup_frequency(struct acpi_processor_px *px, int i)
 
 	if ((boot_cpu_data.x86 == 0x10 && boot_cpu_data.x86_model < 10) ||
 	    boot_cpu_data.x86 == 0x11) {
-		rdmsr(MSR_AMD_PSTATE_DEF_BASE + index, lo, hi);
+		rdmsrq(MSR_AMD_PSTATE_DEF_BASE + index, val.q);
 		/*
 		 * MSR C001_0064+:
 		 * Bit 63: PstateEn. Read-write. If set, the P-state is valid.
 		 */
-		if (!(hi & BIT(31)))
+		if (!(val.h & BIT(31)))
 			return;
 
-		fid = lo & 0x3f;
-		did = (lo >> 6) & 7;
+		fid = val.l & 0x3f;
+		did = (val.l >> 6) & 7;
 		if (boot_cpu_data.x86 == 0x10)
 			px->core_frequency = (100 * (fid + 0x10)) >> did;
 		else
diff --git a/drivers/acpi/processor_throttling.c b/drivers/acpi/processor_throttling.c
index c0f92b93747da..d1605e0ab61fe 100644
--- a/drivers/acpi/processor_throttling.c
+++ b/drivers/acpi/processor_throttling.c
@@ -698,20 +698,13 @@ static int acpi_processor_get_throttling_fadt(struct acpi_processor *pr)
 #ifdef CONFIG_X86
 static int acpi_throttling_rdmsr(u64 *value)
 {
-	u64 msr_high, msr_low;
-	u64 msr = 0;
 	int ret = -1;
 
 	if ((this_cpu_read(cpu_info.x86_vendor) != X86_VENDOR_INTEL) ||
 		!this_cpu_has(X86_FEATURE_ACPI)) {
 		pr_err("HARDWARE addr space,NOT supported yet\n");
 	} else {
-		msr_low = 0;
-		msr_high = 0;
-		rdmsr_safe(MSR_IA32_THERM_CONTROL,
-			(u32 *)&msr_low, (u32 *) &msr_high);
-		msr = (msr_high << 32) | msr_low;
-		*value = (u64) msr;
+		rdmsrq_safe(MSR_IA32_THERM_CONTROL, value);
 		ret = 0;
 	}
 	return ret;
@@ -720,15 +713,12 @@ static int acpi_throttling_rdmsr(u64 *value)
 static int acpi_throttling_wrmsr(u64 value)
 {
 	int ret = -1;
-	u64 msr;
 
 	if ((this_cpu_read(cpu_info.x86_vendor) != X86_VENDOR_INTEL) ||
 		!this_cpu_has(X86_FEATURE_ACPI)) {
 		pr_err("HARDWARE addr space,NOT supported yet\n");
 	} else {
-		msr = value;
-		wrmsr_safe(MSR_IA32_THERM_CONTROL,
-			msr & 0xffffffff, msr >> 32);
+		wrmsrq_safe(MSR_IA32_THERM_CONTROL, value);
 		ret = 0;
 	}
 	return ret;
diff --git a/drivers/acpi/riscv/irq.c b/drivers/acpi/riscv/irq.c
index 9b88d0993e880..da2c42e0ebfd5 100644
--- a/drivers/acpi/riscv/irq.c
+++ b/drivers/acpi/riscv/irq.c
@@ -23,12 +23,6 @@ struct riscv_ext_intc_list {
 	struct list_head	list;
 };
 
-struct acpi_irq_dep_ctx {
-	int		rc;
-	unsigned int	index;
-	acpi_handle	handle;
-};
-
 LIST_HEAD(ext_intc_list);
 
 static int irqchip_cmp_func(const void *in0, const void *in1)
@@ -254,7 +248,7 @@ void __init riscv_acpi_init_gsi_mapping(void)
 	acpi_get_devices("RSCV0006", riscv_acpi_create_gsi_map_smsi, NULL, NULL);
 }
 
-static acpi_handle riscv_acpi_get_gsi_handle(u32 gsi)
+acpi_handle acpi_get_riscv_gsi_handle(u32 gsi)
 {
 	struct riscv_ext_intc_list *ext_intc_element;
 	struct list_head *i;
@@ -269,138 +263,7 @@ static acpi_handle riscv_acpi_get_gsi_handle(u32 gsi)
 	return NULL;
 }
 
-static acpi_status riscv_acpi_irq_get_parent(struct acpi_resource *ares, void *context)
-{
-	struct acpi_irq_dep_ctx *ctx = context;
-	struct acpi_resource_irq *irq;
-	struct acpi_resource_extended_irq *eirq;
-
-	switch (ares->type) {
-	case ACPI_RESOURCE_TYPE_IRQ:
-		irq = &ares->data.irq;
-		if (ctx->index >= irq->interrupt_count) {
-			ctx->index -= irq->interrupt_count;
-			return AE_OK;
-		}
-		ctx->handle = riscv_acpi_get_gsi_handle(irq->interrupts[ctx->index]);
-		return AE_CTRL_TERMINATE;
-	case ACPI_RESOURCE_TYPE_EXTENDED_IRQ:
-		eirq = &ares->data.extended_irq;
-		if (eirq->producer_consumer == ACPI_PRODUCER)
-			return AE_OK;
-
-		if (ctx->index >= eirq->interrupt_count) {
-			ctx->index -= eirq->interrupt_count;
-			return AE_OK;
-		}
-
-		/* Support GSIs only */
-		if (eirq->resource_source.string_length)
-			return AE_OK;
-
-		ctx->handle = riscv_acpi_get_gsi_handle(eirq->interrupts[ctx->index]);
-		return AE_CTRL_TERMINATE;
-	}
-
-	return AE_OK;
-}
-
-static int riscv_acpi_irq_get_dep(acpi_handle handle, unsigned int index, acpi_handle *gsi_handle)
-{
-	struct acpi_irq_dep_ctx ctx = {-EINVAL, index, NULL};
-
-	if (!gsi_handle)
-		return 0;
-
-	acpi_walk_resources(handle, METHOD_NAME__CRS, riscv_acpi_irq_get_parent, &ctx);
-	*gsi_handle = ctx.handle;
-	if (*gsi_handle)
-		return 1;
-
-	return 0;
-}
-
-static u32 riscv_acpi_add_prt_dep(acpi_handle handle)
-{
-	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
-	struct acpi_pci_routing_table *entry;
-	struct acpi_handle_list dep_devices;
-	acpi_handle gsi_handle;
-	acpi_handle link_handle;
-	acpi_status status;
-	u32 count = 0;
-
-	status = acpi_get_irq_routing_table(handle, &buffer);
-	if (ACPI_FAILURE(status)) {
-		acpi_handle_err(handle, "failed to get IRQ routing table\n");
-		kfree(buffer.pointer);
-		return 0;
-	}
-
-	entry = buffer.pointer;
-	while (entry && (entry->length > 0)) {
-		if (entry->source[0]) {
-			acpi_get_handle(handle, entry->source, &link_handle);
-			dep_devices.count = 1;
-			dep_devices.handles = kzalloc_objs(*dep_devices.handles,
-							   1);
-			if (!dep_devices.handles) {
-				acpi_handle_err(handle, "failed to allocate memory\n");
-				continue;
-			}
-
-			dep_devices.handles[0] = link_handle;
-			count += acpi_scan_add_dep(handle, &dep_devices);
-		} else {
-			gsi_handle = riscv_acpi_get_gsi_handle(entry->source_index);
-			dep_devices.count = 1;
-			dep_devices.handles = kzalloc_objs(*dep_devices.handles,
-							   1);
-			if (!dep_devices.handles) {
-				acpi_handle_err(handle, "failed to allocate memory\n");
-				continue;
-			}
-
-			dep_devices.handles[0] = gsi_handle;
-			count += acpi_scan_add_dep(handle, &dep_devices);
-		}
-
-		entry = (struct acpi_pci_routing_table *)
-			((unsigned long)entry + entry->length);
-	}
-
-	kfree(buffer.pointer);
-	return count;
-}
-
-static u32 riscv_acpi_add_irq_dep(acpi_handle handle)
-{
-	struct acpi_handle_list dep_devices;
-	acpi_handle gsi_handle;
-	u32 count = 0;
-	int i;
-
-	for (i = 0;
-	     riscv_acpi_irq_get_dep(handle, i, &gsi_handle);
-	     i++) {
-		dep_devices.count = 1;
-		dep_devices.handles = kzalloc_objs(*dep_devices.handles, 1);
-		if (!dep_devices.handles) {
-			acpi_handle_err(handle, "failed to allocate memory\n");
-			continue;
-		}
-
-		dep_devices.handles[0] = gsi_handle;
-		count += acpi_scan_add_dep(handle, &dep_devices);
-	}
-
-	return count;
-}
-
 u32 arch_acpi_add_auto_dep(acpi_handle handle)
 {
-	if (acpi_has_method(handle, "_PRT"))
-		return riscv_acpi_add_prt_dep(handle);
-
-	return riscv_acpi_add_irq_dep(handle);
+	return acpi_irq_add_auto_dep(handle);
 }
diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c
index 9a7ac2eb9ce0b..36c368324138c 100644
--- a/drivers/acpi/scan.c
+++ b/drivers/acpi/scan.c
@@ -273,13 +273,9 @@ static int acpi_scan_check_and_detach(struct acpi_device *adev, void *p)
 		}
 	}
 
-	adev->flags.match_driver = false;
-	if (handler) {
-		if (handler->detach)
-			handler->detach(adev);
-	} else {
-		device_release_driver(&adev->dev);
-	}
+	if (handler && handler->detach)
+		handler->detach(adev);
+
 	/*
 	 * Most likely, the device is going away, so put it into D3cold before
 	 * that.
@@ -854,6 +850,7 @@ static const char * const acpi_ignore_dep_ids[] = {
 
 /* List of HIDs for which we honor deps of matching ACPI devs, when checking _DEP lists. */
 static const char * const acpi_honor_dep_ids[] = {
+	"ARMH0003", /* ARM GICv5 IWB */
 	"INT3472", /* Camera sensor PMIC / clk and regulator info */
 	"INTC1059", /* IVSC (TGL) driver must be loaded to allow i2c access to camera sensors */
 	"INTC1095", /* IVSC (ADL) driver must be loaded to allow i2c access to camera sensors */
@@ -862,6 +859,7 @@ static const char * const acpi_honor_dep_ids[] = {
 	"INTC10DE", /* CVS (LNL) driver must be loaded to allow camera streaming */
 	"INTC10E0", /* CVS (ARL) driver must be loaded to allow camera streaming */
 	"INTC10E1", /* CVS (PTL) driver must be loaded to allow camera streaming */
+	"INTC10FA", /* CVS (NVL) driver must be loaded to allow camera streaming */
 	"RSCV0001", /* RISC-V PLIC */
 	"RSCV0002", /* RISC-V APLIC */
 	"RSCV0005", /* RISC-V SBI MPXY MBOX */
@@ -1815,13 +1813,13 @@ void acpi_init_device_object(struct acpi_device *device, acpi_handle handle,
 	device->dev.release = release;
 	device->dev.bus = &acpi_bus_type;
 	device->dev.groups = acpi_groups;
+	device_set_pm_not_required(&device->dev);
 	fwnode_init(&device->fwnode, &acpi_device_fwnode_ops);
 	acpi_set_device_status(device, ACPI_STA_DEFAULT);
 	acpi_device_get_busid(device);
 	acpi_set_pnp_ids(handle, &device->pnp, type);
 	acpi_init_properties(device);
 	acpi_bus_get_flags(device);
-	device->flags.match_driver = false;
 	device->flags.initialized = true;
 	device->flags.enumeration_by_parent =
 		acpi_device_enumeration_by_parent(device);
@@ -2375,16 +2373,11 @@ static int acpi_bus_attach(struct acpi_device *device, void *first_pass)
 	if (ret < 0)
 		return 0;
 
-	device->flags.match_driver = true;
 	if (ret > 0 && !device->flags.enumeration_by_parent) {
 		acpi_device_set_enumerated(device);
 		goto ok;
 	}
 
-	ret = device_attach(&device->dev);
-	if (ret < 0)
-		return 0;
-
 	if (device->pnp.type.platform_id || device->pnp.type.backlight ||
 	    device->flags.enumeration_by_parent)
 		acpi_default_enumeration(device);
diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c
index 4286e4af10924..eb93c060426ff 100644
--- a/drivers/acpi/tables.c
+++ b/drivers/acpi/tables.c
@@ -22,6 +22,7 @@
 #include <linux/initrd.h>
 #include <linux/security.h>
 #include <linux/kmemleak.h>
+#include <asm/fixmap.h>
 #include "internal.h"
 
 #ifdef CONFIG_ACPI_CUSTOM_DSDT
diff --git a/drivers/android/binder/allocation.rs b/drivers/android/binder/allocation.rs
index ea5846e4da16a..165cb797eb1e9 100644
--- a/drivers/android/binder/allocation.rs
+++ b/drivers/android/binder/allocation.rs
@@ -384,8 +384,8 @@ pub(crate) fn transfer_binder_object(
                 BINDER_TYPE_WEAK_BINDER
             };
             newobj.flags = obj.flags;
-            newobj.__bindgen_anon_1.binder = ptr as _;
-            newobj.cookie = cookie as _;
+            newobj.__bindgen_anon_1.binder = ptr as uapi::binder_uintptr_t;
+            newobj.cookie = cookie as uapi::binder_uintptr_t;
             self.write(offset, &newobj)?;
             // Increment the user ref count on the node. It will be decremented as part of the
             // destruction of the buffer, when we see a binder or weak-binder object.
diff --git a/drivers/android/binder/defs.rs b/drivers/android/binder/defs.rs
index 33f51b4139c7e..8ac9bdd7a4996 100644
--- a/drivers/android/binder/defs.rs
+++ b/drivers/android/binder/defs.rs
@@ -4,6 +4,8 @@
 
 use core::mem::MaybeUninit;
 use core::ops::{Deref, DerefMut};
+use core::ptr;
+
 use kernel::{
     transmute::{AsBytes, FromBytes},
     uapi::{self, *},
@@ -146,7 +148,7 @@ fn default() -> Self {
 impl BinderVersion {
     pub(crate) fn current() -> Self {
         Self(MaybeUninit::new(uapi::binder_version {
-            protocol_version: BINDER_CURRENT_PROTOCOL_VERSION as _,
+            protocol_version: BINDER_CURRENT_PROTOCOL_VERSION as i32,
         }))
     }
 }
@@ -165,8 +167,8 @@ impl BinderTransactionDataSecctx {
     pub(crate) fn tr_data(&mut self) -> &mut BinderTransactionData {
         // SAFETY: Transparent wrapper is safe to transmute.
         unsafe {
-            &mut *(&mut self.transaction_data as *mut uapi::binder_transaction_data
-                as *mut BinderTransactionData)
+            &mut *(ptr::from_mut::<uapi::binder_transaction_data>(&mut self.transaction_data)
+                .cast::<BinderTransactionData>())
         }
     }
 }
diff --git a/drivers/android/binder/freeze.rs b/drivers/android/binder/freeze.rs
index f4df14568b252..f43388ed6ae23 100644
--- a/drivers/android/binder/freeze.rs
+++ b/drivers/android/binder/freeze.rs
@@ -127,7 +127,7 @@ fn do_work(
             }
 
             let mut state_info = BinderFrozenStateInfo::default();
-            state_info.is_frozen = is_frozen as u32;
+            state_info.is_frozen = u32::from(is_frozen);
             state_info.cookie = freeze.cookie.0;
             freeze.is_pending = true;
             freeze.last_is_frozen = Some(is_frozen);
@@ -180,36 +180,58 @@ pub(crate) fn request_freeze_notif(
         let msg = FreezeMessage::new(GFP_KERNEL)?;
         let alloc = RBTreeNodeReservation::new(GFP_KERNEL)?;
 
+        let mut afl_vec_alloc = KVVec::new();
+        let mut info;
+        let mut freeze_entry;
         let mut node_refs_guard = self.node_refs.lock();
-        let node_refs = &mut *node_refs_guard;
-        let Some(info) = node_refs.by_handle.get_mut(&handle) else {
-            pr_warn!("BC_REQUEST_FREEZE_NOTIFICATION invalid ref {}\n", handle);
-            return Err(EINVAL);
-        };
-        if info.freeze().is_some() {
-            pr_warn!("BC_REQUEST_FREEZE_NOTIFICATION already set\n");
-            return Err(EINVAL);
-        }
-        let node_ref = info.node_ref();
-        let freeze_entry = node_refs.freeze_listeners.entry(cookie);
-
-        if let rbtree::Entry::Occupied(ref dupe) = freeze_entry {
-            if !dupe.get().allow_duplicate(&node_ref.node) {
-                pr_warn!("BC_REQUEST_FREEZE_NOTIFICATION duplicate cookie\n");
+        loop {
+            let node_refs = &mut *node_refs_guard;
+            info = match node_refs.by_handle.get_mut(&handle) {
+                Some(info) => info,
+                None => {
+                    pr_warn!("BC_REQUEST_FREEZE_NOTIFICATION invalid ref {}\n", handle);
+                    return Err(EINVAL);
+                }
+            };
+            if info.freeze().is_some() {
+                pr_warn!("BC_REQUEST_FREEZE_NOTIFICATION already set\n");
                 return Err(EINVAL);
             }
-        }
+            let node_ref = info.node_ref();
+            freeze_entry = node_refs.freeze_listeners.entry(cookie);
+
+            if let rbtree::Entry::Occupied(ref dupe) = freeze_entry {
+                if !dupe.get().allow_duplicate(&node_ref.node) {
+                    pr_warn!("BC_REQUEST_FREEZE_NOTIFICATION duplicate cookie\n");
+                    return Err(EINVAL);
+                }
+            }
 
-        // All failure paths must come before this call, and all modifications must come after this
-        // call.
-        node_ref.node.add_freeze_listener(self, GFP_KERNEL)?;
+            // Now we add to the node's freeze listener list, with retry and re-allocate if the
+            // vector is full.
+            //
+            // To ensure that the node is added atomically, this is the first time we modify any
+            // state. When this call succeeds, all other modifications must occur without the
+            // possibility for any failure paths.
+            match node_ref
+                .node
+                .add_freeze_listener(self, &mut afl_vec_alloc)?
+            {
+                Ok(()) => break,
+                Err(resize_target) => {
+                    drop(node_refs_guard);
+                    afl_vec_alloc = KVVec::with_capacity(resize_target, GFP_KERNEL)?;
+                    node_refs_guard = self.node_refs.lock();
+                }
+            }
+        }
 
         match freeze_entry {
             rbtree::Entry::Vacant(entry) => {
                 entry.insert(
                     FreezeListener {
                         cookie,
-                        node: node_ref.node.clone(),
+                        node: info.node_ref().node.clone(),
                         last_is_frozen: None,
                         is_pending: false,
                         is_clearing: false,
@@ -280,6 +302,7 @@ pub(crate) fn clear_freeze_notif(self: &Arc<Self>, reader: &mut UserSliceReader)
         let handle = hc.handle;
         let cookie = FreezeCookie(hc.cookie);
 
+        let _to_free_fl;
         let alloc = FreezeMessage::new(GFP_KERNEL)?;
         let mut node_refs_guard = self.node_refs.lock();
         let node_refs = &mut *node_refs_guard;
@@ -300,7 +323,7 @@ pub(crate) fn clear_freeze_notif(self: &Arc<Self>, reader: &mut UserSliceReader)
             return Err(EINVAL);
         };
         listener.is_clearing = true;
-        listener.node.remove_freeze_listener(self);
+        _to_free_fl = listener.node.remove_freeze_listener(self);
         *info.freeze() = None;
         let mut msg = None;
         if !listener.is_pending {
diff --git a/drivers/android/binder/node.rs b/drivers/android/binder/node.rs
index c10148e9069f3..59c5ab747bf4d 100644
--- a/drivers/android/binder/node.rs
+++ b/drivers/android/binder/node.rs
@@ -9,6 +9,7 @@
     seq_print,
     sync::lock::{spinlock::SpinLockBackend, Guard},
     sync::{Arc, LockedBy, SpinLock},
+    uapi,
 };
 
 use crate::{
@@ -21,6 +22,7 @@
 };
 
 use core::mem;
+use core::ptr;
 
 mod wrapper;
 pub(crate) use self::wrapper::CritIncrWrapper;
@@ -321,7 +323,7 @@ pub(crate) unsafe fn remove_node_info(
     /// An id that is unique across all binder nodes on the system. Used as the key in the
     /// `by_node` map.
     pub(crate) fn global_id(&self) -> usize {
-        self as *const Node as usize
+        ptr::from_ref(self).addr()
     }
 
     pub(crate) fn get_id(&self) -> (u64, u64) {
@@ -464,7 +466,7 @@ pub(crate) fn incr_refcount_allow_zero2one_with_wrapper(
         owner_inner: &mut ProcessInner,
     ) -> Option<DLArc<dyn DeliverToRead>> {
         match self.incr_refcount_allow_zero2one(strong, owner_inner) {
-            Ok(Some(node)) => Some(node as _),
+            Ok(Some(node)) => Some(node as DLArc<dyn DeliverToRead>),
             Ok(None) => None,
             Err(CouldNotDeliverCriticalIncrement) => {
                 assert!(strong);
@@ -489,8 +491,8 @@ pub(crate) fn populate_counts(
         guard: &Guard<'_, ProcessInner, SpinLockBackend>,
     ) {
         let inner = self.inner.access(guard);
-        out.strong_count = inner.strong.count as _;
-        out.weak_count = inner.weak.count as _;
+        out.strong_count = inner.strong.count as u32;
+        out.weak_count = inner.weak.count as u32;
     }
 
     pub(crate) fn populate_debug_info(
@@ -498,8 +500,8 @@ pub(crate) fn populate_debug_info(
         out: &mut BinderNodeDebugInfo,
         guard: &Guard<'_, ProcessInner, SpinLockBackend>,
     ) {
-        out.ptr = self.ptr as _;
-        out.cookie = self.cookie as _;
+        out.ptr = self.ptr as uapi::binder_uintptr_t;
+        out.cookie = self.cookie as uapi::binder_uintptr_t;
         let inner = self.inner.access(guard);
         if inner.strong.has_count {
             out.has_strong_ref = 1;
@@ -657,29 +659,26 @@ fn do_work_locked(
     pub(crate) fn add_freeze_listener(
         &self,
         process: &Arc<Process>,
-        flags: kernel::alloc::Flags,
-    ) -> Result {
-        let mut vec_alloc = KVVec::<Arc<Process>>::new();
-        loop {
-            let mut guard = self.owner.inner.lock();
-            // Do not check for `guard.dead`. The `dead` flag that matters here is the owner of the
-            // listener, no the target.
-            let inner = self.inner.access_mut(&mut guard);
-            let len = inner.freeze_list.len();
-            if len >= inner.freeze_list.capacity() {
-                if len >= vec_alloc.capacity() {
-                    drop(guard);
-                    vec_alloc = KVVec::with_capacity((1 + len).next_power_of_two(), flags)?;
-                    continue;
-                }
-                mem::swap(&mut inner.freeze_list, &mut vec_alloc);
-                for elem in vec_alloc.drain_all() {
-                    inner.freeze_list.push_within_capacity(elem)?;
-                }
+        // If the vector needs to be resized, it's done via this argument.
+        vec_alloc: &mut KVVec<Arc<Process>>,
+    ) -> Result<Result<(), usize>> {
+        let mut guard = self.owner.inner.lock();
+        // Do not check for `guard.dead`. The `dead` flag that matters here is the owner of the
+        // listener, not the target.
+        let inner = self.inner.access_mut(&mut guard);
+        let len = inner.freeze_list.len();
+        if len == inner.freeze_list.capacity() {
+            if len >= vec_alloc.capacity() {
+                // Request the caller to reallocate.
+                return Ok(Err((1 + len).next_power_of_two()));
+            }
+            mem::swap(&mut inner.freeze_list, vec_alloc);
+            for elem in vec_alloc.drain_all() {
+                inner.freeze_list.push_within_capacity(elem)?;
             }
-            inner.freeze_list.push_within_capacity(process.clone())?;
-            return Ok(());
         }
+        inner.freeze_list.push_within_capacity(process.clone())?;
+        Ok(Ok(()))
     }
 
     pub(crate) fn remove_freeze_listener(&self, p: &Process) -> KVVec<Arc<Process>> {
@@ -695,6 +694,8 @@ pub(crate) fn remove_freeze_listener(&self, p: &Process) -> KVVec<Arc<Process>>
                 p.pid_in_current_ns()
             );
         }
+        // If the vector is empty it needs to be freed. However, we can't free it here because that
+        // might sleep, so return it to the caller.
         if inner.freeze_list.is_empty() {
             return mem::take(&mut inner.freeze_list);
         }
diff --git a/drivers/android/binder/node/wrapper.rs b/drivers/android/binder/node/wrapper.rs
index 43294c0505029..6e4ca01c941ab 100644
--- a/drivers/android/binder/node/wrapper.rs
+++ b/drivers/android/binder/node/wrapper.rs
@@ -21,7 +21,7 @@ pub(crate) fn new() -> Result<Self> {
 
     pub(super) fn init(self, node: DArc<Node>) -> DLArc<dyn DeliverToRead> {
         match self.inner.pin_init_with(DTRWrap::new(NodeWrapper { node })) {
-            Ok(initialized) => ListArc::from(initialized) as _,
+            Ok(initialized) => ListArc::from(initialized) as DLArc<dyn DeliverToRead>,
             Err(err) => match err {},
         }
     }
diff --git a/drivers/android/binder/page_range.rs b/drivers/android/binder/page_range.rs
index e82a5523804f4..52ffbf3504e7f 100644
--- a/drivers/android/binder/page_range.rs
+++ b/drivers/android/binder/page_range.rs
@@ -312,7 +312,7 @@ pub(crate) fn register_with_vma(&self, vma: &virt::VmaNew) -> Result<usize> {
 
         // SAFETY: This just initializes the pages array.
         unsafe {
-            let self_ptr = self as *const ShrinkablePageRange;
+            let self_ptr = ptr::from_ref(self);
             for i in 0..num_pages {
                 let info = pages.as_mut_ptr().add(i);
                 (&raw mut (*info).range).write(self_ptr);
@@ -571,7 +571,7 @@ pub(crate) unsafe fn read<T: FromBytes>(&self, offset: usize) -> Result<T> {
         unsafe {
             self.iterate(offset, size_of::<T>(), |page, offset, to_copy| {
                 // SAFETY: The sum of `offset` and `to_copy` is bounded by the size of T.
-                let obj_ptr = (out.as_mut_ptr() as *mut u8).add(out_offset);
+                let obj_ptr = out.as_mut_ptr().cast::<u8>().add(out_offset);
                 // SAFETY: The pointer points is in-bounds of the `out` variable, so it is valid.
                 page.read_raw(obj_ptr, offset, to_copy)?;
                 out_offset += to_copy;
@@ -593,7 +593,7 @@ pub(crate) unsafe fn write<T: ?Sized>(&self, offset: usize, obj: &T) -> Result {
         unsafe {
             self.iterate(offset, size_of_val(obj), |page, offset, to_copy| {
                 // SAFETY: The sum of `offset` and `to_copy` is bounded by the size of T.
-                let obj_ptr = (obj as *const T as *const u8).add(obj_offset);
+                let obj_ptr = ptr::from_ref(obj).cast::<u8>().add(obj_offset);
                 // SAFETY: We have a reference to the object, so the pointer is valid.
                 page.write_raw(obj_ptr, offset, to_copy)?;
                 obj_offset += to_copy;
@@ -712,7 +712,7 @@ fn drop(self: Pin<&mut Self>) {
 
     {
         // CAST: The `list_head` field is first in `PageInfo`.
-        let info = item as *mut PageInfo;
+        let info = item.cast::<PageInfo>();
         // SAFETY: The `range` field of `PageInfo` is immutable.
         range_ptr = unsafe { (*info).range };
         // SAFETY: The `range` outlives its `PageInfo` values.
diff --git a/drivers/android/binder/process.rs b/drivers/android/binder/process.rs
index cdd1a90797266..0555c4bd503ea 100644
--- a/drivers/android/binder/process.rs
+++ b/drivers/android/binder/process.rs
@@ -30,9 +30,9 @@
     sync::{
         aref::ARef,
         lock::{spinlock::SpinLockBackend, Guard},
-        Arc, ArcBorrow, CondVar, CondVarTimeoutResult, Mutex, SpinLock, UniqueArc,
+        Arc, ArcBorrow, CondVar, CondVarTimeoutResult, SpinLock, UniqueArc,
     },
-    task::Task,
+    task::{Pid, Task},
     uaccess::{UserSlice, UserSliceReader},
     uapi,
     workqueue::{self, Work},
@@ -259,7 +259,7 @@ pub(crate) fn new_node_ref_with_thread(
         let push = match wrapper {
             None => node
                 .incr_refcount_allow_zero2one(strong, self)?
-                .map(|node| node as _),
+                .map(|node| node as DLArc<dyn DeliverToRead>),
             Some(wrapper) => node.incr_refcount_allow_zero2one_with_wrapper(strong, wrapper, self),
         };
         if let Some(node) = push {
@@ -455,7 +455,7 @@ pub(crate) struct Process {
     // Node references are in a different lock to avoid recursive acquisition when
     // incrementing/decrementing a node in another process.
     #[pin]
-    node_refs: Mutex<ProcessNodeRefs>,
+    node_refs: SpinLock<ProcessNodeRefs>,
 
     // Work node for deferred work item.
     #[pin]
@@ -510,7 +510,7 @@ fn new(ctx: Arc<Context>, cred: ARef<Credential>) -> Result<Arc<Self>> {
                 cred,
                 inner <- kernel::new_spinlock!(ProcessInner::new(), "Process::inner"),
                 pages <- ShrinkablePageRange::new(&super::BINDER_SHRINKER),
-                node_refs <- kernel::new_mutex!(ProcessNodeRefs::new(), "Process::node_refs"),
+                node_refs <- kernel::new_spinlock!(ProcessNodeRefs::new(), "Process::node_refs"),
                 freeze_wait <- kernel::new_condvar!("Process::freeze_wait"),
                 task: current.group_leader().into(),
                 defer_work <- kernel::new_work!("Process::defer_work"),
@@ -741,7 +741,7 @@ fn set_as_manager(
         } else {
             (0, 0, 0)
         };
-        let node_ref = self.get_node(ptr, cookie, flags as _, true, thread)?;
+        let node_ref = self.get_node(ptr, cookie, flags, true, thread)?;
         let node = node_ref.node.clone();
         self.ctx.set_manager_node(node_ref)?;
         self.inner.lock().is_manager = true;
@@ -861,14 +861,17 @@ pub(crate) fn insert_or_update_handle(
         let handle = unused_id.as_u32();
 
         // Do a lookup again as node may have been inserted before the lock was reacquired.
-        if let Some(handle_ref) = refs.by_node.get(&node_ref.node.global_id()) {
-            let handle = *handle_ref;
-            let info = refs.by_handle.get_mut(&handle).unwrap();
-            info.node_ref().absorb(node_ref);
-            return Ok(handle);
-        }
+        let by_node_slot = match refs.by_node.entry(node_ref.node.global_id()) {
+            rbtree::Entry::Vacant(by_node_slot) => by_node_slot,
+            rbtree::Entry::Occupied(handle_ref) => {
+                // The node was inserted by another thread while we didn't hold the lock.
+                let handle = handle_ref.get();
+                let info = refs.by_handle.get_mut(handle).unwrap();
+                info.node_ref().absorb(node_ref);
+                return Ok(*handle);
+            }
+        };
 
-        let gid = node_ref.node.global_id();
         let (info_proc, info_node) = {
             let info_init = NodeRefInfo::new(node_ref, handle, self.into());
             match info.pin_init_with(info_init) {
@@ -884,6 +887,9 @@ pub(crate) fn insert_or_update_handle(
         // first thing in `deferred_release`, process cleanup will not miss the items inserted into
         // `refs` below.
         if self.inner.lock().is_dead {
+            // Explicitly drop the lock so that `info_proc` and `info_node` are dropped outside of
+            // the lock.
+            drop(refs_lock);
             return Err(ESRCH);
         }
 
@@ -891,7 +897,7 @@ pub(crate) fn insert_or_update_handle(
         // `info_node` into the right node's `refs` list.
         unsafe { info_proc.node_ref2().node.insert_node_info(info_node) };
 
-        refs.by_node.insert(reserve1.into_node(gid, handle));
+        by_node_slot.insert(handle, reserve1);
         by_handle_slot.insert(info_proc, reserve2);
         unused_id.acquire();
         Ok(handle)
@@ -946,15 +952,19 @@ pub(crate) fn update_ref(
 
         // To preserve original binder behaviour, we only fail requests where the manager tries to
         // increment references on itself.
+        let _to_free_by_handle;
+        let _to_free_by_node;
         let _to_free_freeze_listener;
         let _to_free_freeze_listener_cleanup;
         let mut refs = self.node_refs.lock();
         if let Some(info) = refs.by_handle.get_mut(&handle) {
             if info.node_ref().update(inc, strong) {
                 // Clean up death if there is one attached to this node reference.
-                if let Some(death) = info.death().take() {
+                //
+                // We remove the entire `info` below, so no need to remove `death` from `info`.
+                if let Some(death) = info.death().as_ref() {
                     death.set_cleared(true);
-                    self.remove_from_delivered_deaths(&death);
+                    self.remove_from_delivered_deaths(death);
                 }
 
                 // Remove reference from process tables, and from the node's `refs` list.
@@ -971,8 +981,8 @@ pub(crate) fn update_ref(
                     }
                 }
 
-                refs.by_handle.remove(&handle);
-                refs.by_node.remove(&id);
+                _to_free_by_handle = refs.by_handle.remove_node(&handle);
+                _to_free_by_node = refs.by_node.remove_node(&id);
                 refs.handle_is_present.release_id(handle as usize);
 
                 if let Some(shrink) = refs.handle_is_present.shrink_request() {
@@ -1299,7 +1309,10 @@ pub(crate) fn clear_death(&self, reader: &mut UserSliceReader, thread: &Thread)
 
         // Update state and determine if we need to queue a work item. We only need to do it when
         // the node is not dead or if the user already completed the death notification.
-        if death.set_cleared(false) {
+        let should_schedule = death.set_cleared(false);
+        drop(refs);
+
+        if should_schedule {
             if let Some(death) = ListArc::try_from_arc_or_drop(death) {
                 let _ = thread.push_work_if_looper(death);
             }
@@ -1382,13 +1395,11 @@ fn deferred_release(self: Arc<Self>) {
             // SAFETY: We are removing the `NodeRefInfo` from the right node.
             unsafe { info.node_ref2().node.remove_node_info(info) };
 
-            // Remove all death notifications from the nodes (that belong to a different process).
-            let death = if let Some(existing) = info.death().take() {
-                existing
-            } else {
-                continue;
-            };
-            death.set_cleared(false);
+            // Clear death notifications from the nodes (that belong to a different process).
+            // No need to remove them from `info` as we clear info below.
+            if let Some(death) = info.death().as_ref() {
+                death.set_cleared(false);
+            }
         }
 
         // Clean up freeze listeners.
@@ -1536,13 +1547,13 @@ fn get_frozen_status(data: UserSlice) -> Result {
 
     for ctx in crate::context::get_all_contexts()? {
         ctx.for_each_proc(|proc| {
-            if proc.task.pid() == info.pid as _ {
+            if proc.task.pid() == info.pid as Pid {
                 found = true;
                 let inner = proc.inner.lock();
                 let txns_pending = inner.txns_pending_locked();
-                info.async_recv |= inner.async_recv as u32;
-                info.sync_recv |= inner.sync_recv as u32;
-                info.sync_recv |= (txns_pending as u32) << 1;
+                info.async_recv |= u32::from(inner.async_recv);
+                info.sync_recv |= u32::from(inner.sync_recv);
+                info.sync_recv |= u32::from(txns_pending) << 1;
             }
         });
     }
diff --git a/drivers/android/binder/rust_binder_main.rs b/drivers/android/binder/rust_binder_main.rs
index dc1941cd2407b..432390aab25bb 100644
--- a/drivers/android/binder/rust_binder_main.rs
+++ b/drivers/android/binder/rust_binder_main.rs
@@ -6,12 +6,6 @@
 
 #![crate_name = "rust_binder"]
 #![recursion_limit = "256"]
-#![allow(
-    clippy::as_underscore,
-    clippy::ref_as_ptr,
-    clippy::ptr_as_ptr,
-    clippy::cast_lossless
-)]
 
 use kernel::{
     bindings::{self, seq_file},
@@ -417,7 +411,7 @@ unsafe impl<T> Sync for AssertSync<T> {}
     // SAFETY: We previously set `private_data` in `rust_binder_open`.
     let f = unsafe { Arc::<Process>::borrow((*file).private_data) };
     // SAFETY: The caller ensures that the file is valid.
-    match Process::ioctl(f, unsafe { File::from_raw_file(file) }, cmd as _, arg as _) {
+    match Process::ioctl(f, unsafe { File::from_raw_file(file) }, cmd, arg) {
         Ok(()) => 0,
         Err(err) => err.to_errno() as isize,
     }
@@ -511,7 +505,7 @@ unsafe impl<T> Sync for AssertSync<T> {}
     _: *mut kernel::ffi::c_void,
 ) -> kernel::ffi::c_int {
     // SAFETY: Accessing the private field of `seq_file` is okay.
-    let pid = (unsafe { (*ptr).private }) as usize as Pid;
+    let pid = unsafe { (*ptr).private }.addr() as Pid;
     // SAFETY: The caller ensures that the pointer is valid and exclusive for the duration in which
     // this method is called.
     let m = unsafe { SeqFile::from_raw(ptr) };
diff --git a/drivers/android/binder/thread.rs b/drivers/android/binder/thread.rs
index 3b85208139410..19f881948a84b 100644
--- a/drivers/android/binder/thread.rs
+++ b/drivers/android/binder/thread.rs
@@ -673,9 +673,9 @@ fn translate_object(
                 let strong = obj.hdr.type_ == BINDER_TYPE_BINDER;
                 // SAFETY: `binder` is a `binder_uintptr_t`; any bit pattern is a valid
                 // representation.
-                let ptr = unsafe { obj.__bindgen_anon_1.binder } as _;
-                let cookie = obj.cookie as _;
-                let flags = obj.flags as _;
+                let ptr = unsafe { obj.__bindgen_anon_1.binder };
+                let cookie = obj.cookie;
+                let flags = obj.flags;
                 let node = self
                     .process
                     .as_arc_borrow()
@@ -686,7 +686,7 @@ fn translate_object(
             BinderObjectRef::Handle(obj) => {
                 let strong = obj.hdr.type_ == BINDER_TYPE_HANDLE;
                 // SAFETY: `handle` is a `u32`; any bit pattern is a valid representation.
-                let handle = unsafe { obj.__bindgen_anon_1.handle } as _;
+                let handle = unsafe { obj.__bindgen_anon_1.handle };
                 let node = self.process.get_node_from_handle(handle, strong)?;
                 security::binder_transfer_binder(&self.process.cred, &view.alloc.process.cred)?;
                 view.transfer_binder_object(offset, obj, strong, node)?;
@@ -743,7 +743,7 @@ fn translate_object(
                     ScatterGatherEntry {
                         obj_index,
                         offset: alloc_offset,
-                        sender_uaddr: obj.buffer as _,
+                        sender_uaddr: obj.buffer as usize,
                         length: obj_length,
                         pointer_fixups: KVec::new(),
                         fixup_min_offset: 0,
@@ -850,7 +850,7 @@ fn translate_object(
                     .ok_or(EINVAL)?;
 
                 let mut fda_bytes = KVec::new();
-                UserSlice::new(UserPtr::from_addr(fda_uaddr as _), fds_len)
+                UserSlice::new(UserPtr::from_addr(fda_uaddr as usize), fds_len)
                     .read_all(&mut fda_bytes, GFP_KERNEL)?;
 
                 if fds_len != fda_bytes.len() {
@@ -1392,7 +1392,7 @@ fn write(self: &Arc<Self>, req: &mut BinderWriteRead) -> Result {
         let write_start = req.write_buffer.wrapping_add(req.write_consumed);
         let write_len = req.write_size.saturating_sub(req.write_consumed);
         let mut reader =
-            UserSlice::new(UserPtr::from_addr(write_start as _), write_len as _).reader();
+            UserSlice::new(UserPtr::from_addr(write_start as usize), write_len as usize).reader();
 
         while reader.len() >= size_of::<u32>() && self.inner.lock().return_work.is_unused() {
             let before = reader.len();
@@ -1463,7 +1463,7 @@ fn read(self: &Arc<Self>, req: &mut BinderWriteRead, wait: bool) -> Result {
         let read_start = req.read_buffer.wrapping_add(req.read_consumed);
         let read_len = req.read_size.saturating_sub(req.read_consumed);
         let mut writer = BinderReturnWriter::new(
-            UserSlice::new(UserPtr::from_addr(read_start as _), read_len as _).writer(),
+            UserSlice::new(UserPtr::from_addr(read_start as usize), read_len as usize).writer(),
             self,
         );
         let (in_pool, has_transaction, thread_todo, use_proc_queue) = {
@@ -1527,9 +1527,11 @@ fn read(self: &Arc<Self>, req: &mut BinderWriteRead, wait: bool) -> Result {
 
         // Write BR_SPAWN_LOOPER if the process needs more threads for its pool.
         if has_noop_placeholder && in_pool && self.process.needs_thread() {
-            let mut writer =
-                UserSlice::new(UserPtr::from_addr(req.read_buffer as _), req.read_size as _)
-                    .writer();
+            let mut writer = UserSlice::new(
+                UserPtr::from_addr(req.read_buffer as usize),
+                req.read_size as usize,
+            )
+            .writer();
             writer.write(&BR_SPAWN_LOOPER)?;
         }
         Ok(())
diff --git a/drivers/android/binder/trace.rs b/drivers/android/binder/trace.rs
index 5539672d7285a..06aabb3cc2f14 100644
--- a/drivers/android/binder/trace.rs
+++ b/drivers/android/binder/trace.rs
@@ -4,6 +4,8 @@
 
 use crate::transaction::Transaction;
 
+use core::ptr;
+
 use kernel::bindings::{rust_binder_transaction, task_struct};
 use kernel::error::Result;
 use kernel::ffi::{c_int, c_uint, c_ulong};
@@ -26,7 +28,7 @@
 
 #[inline]
 fn raw_transaction(t: &Transaction) -> rust_binder_transaction {
-    t as *const Transaction as rust_binder_transaction
+    ptr::from_ref(t).cast_mut().cast()
 }
 
 #[inline]
diff --git a/drivers/android/binder/transaction.rs b/drivers/android/binder/transaction.rs
index 0e5d07b7e6f0a..afef5b46eac26 100644
--- a/drivers/android/binder/transaction.rs
+++ b/drivers/android/binder/transaction.rs
@@ -11,6 +11,7 @@
     task::{Kuid, Pid},
     time::{Instant, Monotonic},
     types::ScopeGuard,
+    uapi,
 };
 
 use crate::{
@@ -410,16 +411,17 @@ fn do_work(
         let tr = tr_sec.tr_data();
         if let Some(target_node) = &self.target_node {
             let (ptr, cookie) = target_node.get_id();
-            tr.target.ptr = ptr as _;
-            tr.cookie = cookie as _;
+            tr.target.ptr = ptr as uapi::binder_uintptr_t;
+            tr.cookie = cookie as uapi::binder_uintptr_t;
         };
         tr.code = self.code;
         tr.flags = self.flags;
-        tr.data_size = self.data_size as _;
-        tr.data.ptr.buffer = self.data_address as _;
-        tr.offsets_size = self.offsets_size as _;
+        tr.data_size = self.data_size as uapi::binder_size_t;
+        tr.data.ptr.buffer = self.data_address as uapi::binder_uintptr_t;
+        tr.offsets_size = self.offsets_size as uapi::binder_size_t;
         if tr.offsets_size > 0 {
-            tr.data.ptr.offsets = (self.data_address + ptr_align(self.data_size).unwrap()) as _;
+            tr.data.ptr.offsets =
+                (self.data_address + ptr_align(self.data_size).unwrap()) as uapi::binder_uintptr_t;
         }
         tr.sender_euid = self.sender_euid.into_uid_in_current_ns();
         tr.sender_pid = 0;
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
index c43bd28b20b1a..0763eb8a457b5 100644
--- a/drivers/ata/libata-core.c
+++ b/drivers/ata/libata-core.c
@@ -2488,7 +2488,7 @@ static void ata_dev_config_sense_reporting(struct ata_device *dev)
 	}
 }
 
-static void ata_dev_config_zac(struct ata_device *dev)
+static void ata_dev_config_zoned(struct ata_device *dev)
 {
 	unsigned int err_mask;
 	u8 *identify_buf = dev->sector_buf;
@@ -2497,7 +2497,7 @@ static void ata_dev_config_zac(struct ata_device *dev)
 	dev->zac_zones_optimal_nonseq = U32_MAX;
 	dev->zac_zones_max_open = U32_MAX;
 
-	if (!ata_dev_is_zac(dev))
+	if (!ata_dev_is_zoned(dev))
 		return;
 
 	if (!ata_identify_page_supported(dev, ATA_LOG_ZONED_INFORMATION)) {
@@ -2705,6 +2705,71 @@ static void ata_dev_config_cdl(struct ata_device *dev)
 	ata_dev_cleanup_cdl_resources(dev);
 }
 
+static void ata_dev_config_depop(struct ata_device *dev)
+{
+	unsigned int err_mask;
+	u64 val;
+
+	/* Ignore old drives. */
+	if (ata_id_major_version(dev->id) < 11)
+		goto not_supported;
+
+	/* NCQ Autosense is required. */
+	if (!ata_identify_page_supported(dev, ATA_LOG_SUPPORTED_CAPABILITIES) ||
+	    !ata_id_has_ncq_autosense(dev->id))
+		goto not_supported;
+
+	err_mask = ata_read_log_page(dev, ATA_LOG_IDENTIFY_DEVICE,
+				     ATA_LOG_SUPPORTED_CAPABILITIES,
+				     dev->sector_buf, 1);
+	if (err_mask)
+		goto not_supported;
+
+	/* Check depopulation capabilities bits. */
+	val = get_unaligned_le64(&dev->sector_buf[152]);
+	if (!(val & BIT_ULL(63)))
+		goto not_supported;
+
+	/*
+	 * Support for at least the GET PHYSICAL ELEMENT STATUS and
+	 * REMOVE ELEMENT AND TRUNCATE commands is mandated.
+	 */
+	if (!(val & BIT_ULL(0)) || !(val & BIT_ULL(1)))
+		goto not_supported;
+
+	dev->flags |= ATA_DFLAG_DEPOP;
+
+	/* Check if RESTORE ELEMENTS AND REBUILD is supported. */
+	if (val & BIT_ULL(2))
+		dev->flags |= ATA_DFLAG_DEPOP_RESTORE;
+
+	/*
+	 * For ZAC devices, check if REMOVE ELEMENT AND MODIFY ZONES is
+	 * supported.
+	 */
+	if (dev->class != ATA_DEV_ZAC)
+		return;
+
+	err_mask = ata_read_log_page(dev, ATA_LOG_IDENTIFY_DEVICE,
+				     ATA_LOG_ZONED_INFORMATION,
+				     dev->sector_buf, 1);
+	if (err_mask)
+		return;
+
+	val = get_unaligned_le64(&dev->sector_buf[8]);
+	if (!(val & BIT_ULL(63)))
+		return;
+
+	if (val & BIT_ULL(1))
+		dev->flags |= ATA_DFLAG_DEPOP_MODIFY;
+
+	return;
+
+not_supported:
+	dev->flags &= ~(ATA_DFLAG_DEPOP | ATA_DFLAG_DEPOP_RESTORE |
+			ATA_DFLAG_DEPOP_MODIFY);
+}
+
 static int ata_dev_config_lba(struct ata_device *dev)
 {
 	const u16 *id = dev->id;
@@ -2942,7 +3007,7 @@ static void ata_dev_print_features(struct ata_device *dev)
 		return;
 
 	ata_dev_info(dev,
-		     "Features:%s%s%s%s%s%s%s%s%s%s\n",
+		     "Features:%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
 		     dev->flags & ATA_DFLAG_FUA ? " FUA" : "",
 		     dev->flags & ATA_DFLAG_TRUSTED ? " Trust" : "",
 		     dev->flags & ATA_DFLAG_DA ? " Dev-Attention" : "",
@@ -2952,7 +3017,10 @@ static void ata_dev_print_features(struct ata_device *dev)
 		     dev->flags & ATA_DFLAG_NCQ_SEND_RECV ? " NCQ-sndrcv" : "",
 		     dev->flags & ATA_DFLAG_NCQ_PRIO ? " NCQ-prio" : "",
 		     dev->flags & ATA_DFLAG_CDL ? " CDL" : "",
-		     dev->cpr_log ? " CPR" : "");
+		     dev->cpr_log ? " CPR" : "",
+		     dev->flags & ATA_DFLAG_DEPOP ? " Depop" : "",
+		     dev->flags & ATA_DFLAG_DEPOP_RESTORE ? " Depop-Restore" : "",
+		     dev->flags & ATA_DFLAG_DEPOP_MODIFY ? " Depop-Modify" : "");
 }
 
 /**
@@ -3111,10 +3179,11 @@ int ata_dev_configure(struct ata_device *dev)
 		ata_dev_config_fua(dev);
 		ata_dev_config_devslp(dev);
 		ata_dev_config_sense_reporting(dev);
-		ata_dev_config_zac(dev);
+		ata_dev_config_zoned(dev);
 		ata_dev_config_trusted(dev);
 		ata_dev_config_cpr(dev);
 		ata_dev_config_cdl(dev);
+		ata_dev_config_depop(dev);
 		dev->cdb_len = 32;
 
 		if (print_info)
diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c
index 05df7ea6954a9..29ec0f7fef4a3 100644
--- a/drivers/ata/libata-eh.c
+++ b/drivers/ata/libata-eh.c
@@ -1230,9 +1230,10 @@ static void __ata_eh_qc_complete(struct ata_queued_cmd *qc)
  *	Indicate to the mid and upper layers that an ATA command has
  *	completed.  To be used from EH.
  */
-void ata_eh_qc_complete(struct ata_queued_cmd *qc)
+static void ata_eh_qc_complete(struct ata_queued_cmd *qc)
 {
 	struct scsi_cmnd *scmd = qc->scsicmd;
+
 	scmd->retries = scmd->allowed;
 	__ata_eh_qc_complete(qc);
 }
@@ -1248,9 +1249,10 @@ void ata_eh_qc_complete(struct ata_queued_cmd *qc)
  *	scmd->allowed is incremented for commands which get retried
  *	due to unrelated failures (qc->err_mask is zero).
  */
-void ata_eh_qc_retry(struct ata_queued_cmd *qc)
+static void ata_eh_qc_retry(struct ata_queued_cmd *qc)
 {
 	struct scsi_cmnd *scmd = qc->scsicmd;
+
 	if (!qc->err_mask)
 		scmd->allowed++;
 	__ata_eh_qc_complete(qc);
diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c
index 5868526301a2b..1d225ee9eb868 100644
--- a/drivers/ata/libata-scsi.c
+++ b/drivers/ata/libata-scsi.c
@@ -2023,7 +2023,7 @@ static unsigned int ata_scsiop_inq_std(struct ata_device *dev,
 	if (rbuf[32] == 0 || rbuf[32] == ' ')
 		memcpy(&rbuf[32], "n/a ", 4);
 
-	if (ata_id_zoned_cap(dev->id) || dev->class == ATA_DEV_ZAC)
+	if (ata_dev_is_zoned(dev))
 		memcpy(rbuf + 58, versions_zbc, sizeof(versions_zbc));
 	else
 		memcpy(rbuf + 58, versions, sizeof(versions));
@@ -2063,7 +2063,7 @@ static unsigned int ata_scsiop_inq_00(struct ata_device *dev,
 	};
 
 	for (i = 0; i < sizeof(pages); i++) {
-		if (pages[i] == 0xb6 && !ata_dev_is_zac(dev))
+		if (pages[i] == 0xb6 && !ata_dev_is_zoned(dev))
 			continue;
 		rbuf[num_pages + 4] = pages[i];
 		num_pages++;
@@ -2201,6 +2201,39 @@ static unsigned int ata_scsiop_inq_89(struct ata_device *dev,
 	return get_unaligned_be16(&rbuf[2]) + 4;
 }
 
+/**
+ * ata_dsm_trim_pages - maximum DSM TRIM payload for a device, in 512-byte pages
+ * @dev: ATA device the DATA SET MANAGEMENT TRIM command will be sent to
+ *
+ * A DATA SET MANAGEMENT TRIM payload is a list of 512-byte pages, each holding
+ * up to ATA_MAX_TRIM_RNUM (64) LBA Range Entries; the format is page-based and
+ * unrelated to the logical sector size.
+ *
+ * The logical sector size still bounds it, though: the descriptor is written
+ * directly into the WRITE SAME data-out buffer, which sd sizes to a single
+ * logical block, so it can hold at most sector_size / 512 pages.
+ *
+ * Return: the maximum number of 512-byte pages a single translated WRITE SAME
+ * command may send to @dev (never less than one), that is the smaller of:
+ *   - MAX PAGES PER DSM COMMAND (IDENTIFY DEVICE word 105), when the device
+ *     reports a non-zero limit; and
+ *   - the logical sector size expressed in 512-byte pages (see above).
+ */
+static unsigned int ata_dsm_trim_pages(struct ata_device *dev)
+{
+	unsigned int sector_size = ata_id_logical_sector_size(dev->id);
+	unsigned int max_pages = ata_id_dsm_max_pages(dev->id);
+	unsigned int pages = sector_size / ATA_SECT_SIZE;
+
+	/* If the device does not specify a limit, assume only a single page. */
+	if (!max_pages)
+		max_pages = 1;
+
+	pages = min_not_zero(pages, max_pages);
+
+	return pages;
+}
+
 /**
  *	ata_scsiop_inq_b0 - Simulate INQUIRY VPD page B0, Block Limits
  *	@dev: Target device.
@@ -2240,7 +2273,8 @@ static unsigned int ata_scsiop_inq_b0(struct ata_device *dev,
 	 * with the unmap bit set.
 	 */
 	if (ata_id_has_trim(dev->id)) {
-		u64 max_blocks = 65535 * ATA_MAX_TRIM_RNUM;
+		unsigned int max_pages = ata_dsm_trim_pages(dev);
+		u64 max_blocks = max_pages * ATA_MAX_TRIM_RNUM * (u64)U16_MAX;
 
 		if (dev->quirks & ATA_QUIRK_MAX_TRIM_128M)
 			max_blocks = 128 << (20 - SECTOR_SHIFT);
@@ -2320,7 +2354,7 @@ static unsigned int ata_scsiop_inq_b2(struct ata_device *dev,
 static unsigned int ata_scsiop_inq_b6(struct ata_device *dev,
 				      struct scsi_cmnd *cmd, u8 *rbuf)
 {
-	if (!ata_dev_is_zac(dev)) {
+	if (!ata_dev_is_zoned(dev)) {
 		ata_scsi_set_invalid_field(dev, cmd, 2, 0xff);
 		return 0;
 	}
@@ -2851,7 +2885,7 @@ static unsigned int ata_scsiop_read_cap(struct ata_device *dev,
 	rbuf[10] = sector_size >> (8 * 1);
 	rbuf[11] = sector_size;
 
-	if (ata_id_zoned_cap(dev->id) || dev->class == ATA_DEV_ZAC)
+	if (ata_dev_is_zoned(dev))
 		rbuf[12] = (1 << 4); /* RC_BASIS */
 	rbuf[13] = log2_per_phys;
 	rbuf[14] = (lowest_aligned >> 8) & 0x3f;
@@ -3429,14 +3463,14 @@ static unsigned int ata_scsi_pass_thru(struct ata_queued_cmd *qc)
 /**
  * ata_format_dsm_trim_descr() - SATL Write Same to DSM Trim
  * @cmd: SCSI command being translated
- * @trmax: Maximum number of entries that will fit in sector_size bytes.
+ * @size: DSM TRIM payload size in bytes (a multiple of 512)
  * @sector: Starting sector
  * @count: Total Range of request in logical sectors
  *
  * Rewrite the WRITE SAME descriptor to be a DSM TRIM little-endian formatted
  * descriptor.
  *
- * Upto 64 entries of the format:
+ * The payload is a list of @size / 8 entries of the format:
  *   63:48 Range Length
  *   47:0  LBA
  *
@@ -3445,39 +3479,45 @@ static unsigned int ata_scsi_pass_thru(struct ata_queued_cmd *qc)
  *
  * NOTE: this is the same format as ADD LBA(S) TO NV CACHE PINNED SET
  *
- * Return: Number of bytes copied into sglist.
+ * The descriptor is written straight into the WRITE SAME data-out buffer;
+ * ata_dsm_trim_pages() guarantees @size does not exceed that buffer (one
+ * logical block). An atomic sg_miter mapping is used so this works from the
+ * command submission path and, unlike page_address(), copes with a high
+ * memory payload.
+ *
+ * Return: Number of bytes written into the data-out buffer.
  */
-static size_t ata_format_dsm_trim_descr(struct scsi_cmnd *cmd, u32 trmax,
+static size_t ata_format_dsm_trim_descr(struct scsi_cmnd *cmd, size_t size,
 					u64 sector, u32 count)
 {
-	struct scsi_device *sdp = cmd->device;
-	size_t len = sdp->sector_size;
-	size_t r;
-	__le64 *buf;
-	u32 i = 0;
-	unsigned long flags;
+	struct sg_mapping_iter miter;
+	size_t offset = 0;
+
+	sg_miter_start(&miter, scsi_sglist(cmd), scsi_sg_count(cmd),
+		       SG_MITER_TO_SG | SG_MITER_ATOMIC);
+	while (offset < size && sg_miter_next(&miter)) {
+		__le64 *buf = miter.addr;
+		size_t chunk = min_t(size_t, miter.length, size - offset);
+		unsigned int n = chunk / sizeof(__le64);
+		unsigned int i;
 
-	WARN_ON(len > ATA_SCSI_RBUF_SIZE);
+		for (i = 0; i < n; i++) {
+			u64 entry = 0;
 
-	if (len > ATA_SCSI_RBUF_SIZE)
-		len = ATA_SCSI_RBUF_SIZE;
+			if (count) {
+				u32 rlen = min_t(u32, count, 0xffff);
 
-	spin_lock_irqsave(&ata_scsi_rbuf_lock, flags);
-	buf = ((void *)ata_scsi_rbuf);
-	memset(buf, 0, len);
-	while (i < trmax) {
-		u64 entry = sector |
-			((u64)(count > 0xffff ? 0xffff : count) << 48);
-		buf[i++] = __cpu_to_le64(entry);
-		if (count <= 0xffff)
-			break;
-		count -= 0xffff;
-		sector += 0xffff;
+				entry = sector | ((u64)rlen << 48);
+				sector += rlen;
+				count -= rlen;
+			}
+			buf[i] = cpu_to_le64(entry);
+		}
+		offset += n * sizeof(__le64);
 	}
-	r = sg_copy_from_buffer(scsi_sglist(cmd), scsi_sg_count(cmd), buf, len);
-	spin_unlock_irqrestore(&ata_scsi_rbuf_lock, flags);
+	sg_miter_stop(&miter);
 
-	return r;
+	return offset;
 }
 
 /**
@@ -3495,14 +3535,13 @@ static unsigned int ata_scsi_write_same_xlat(struct ata_queued_cmd *qc)
 {
 	struct ata_taskfile *tf = &qc->tf;
 	struct scsi_cmnd *scmd = qc->scsicmd;
-	struct scsi_device *sdp = scmd->device;
-	size_t len = sdp->sector_size;
 	struct ata_device *dev = qc->dev;
 	const u8 *cdb = scmd->cmnd;
+	unsigned int max_pages = ata_dsm_trim_pages(dev);
+	unsigned int n_pages;
+	size_t size;
 	u64 block;
 	u32 n_block;
-	const u32 trmax = len >> 3;
-	u32 size;
 	u16 fp;
 	u8 bp = 0xff;
 	u8 unmap = cdb[1] & 0x8;
@@ -3532,7 +3571,7 @@ static unsigned int ata_scsi_write_same_xlat(struct ata_queued_cmd *qc)
 		goto invalid_fld;
 	}
 	/* If the request is too large the cmd is invalid */
-	if (n_block > 0xffff * trmax) {
+	if (n_block > max_pages * ATA_MAX_TRIM_RNUM * (u64)U16_MAX) {
 		fp = 2;
 		goto invalid_fld;
 	}
@@ -3545,31 +3584,39 @@ static unsigned int ata_scsi_write_same_xlat(struct ata_queued_cmd *qc)
 		goto invalid_param_len;
 
 	/*
-	 * size must match sector size in bytes
-	 * For DATA SET MANAGEMENT TRIM in ACS-2 nsect (aka count)
-	 * is defined as number of 512 byte blocks to be transferred.
+	 * The DATA SET MANAGEMENT TRIM payload is a whole number of 512-byte
+	 * pages (each holding up to ATA_MAX_TRIM_RNUM LBA Range Entries),
+	 * independent of the logical sector size. Only use as many pages as
+	 * are needed to describe the request, capped at max_pages.
 	 */
+	n_pages = DIV_ROUND_UP(DIV_ROUND_UP(n_block, U16_MAX),
+			       ATA_MAX_TRIM_RNUM);
+	n_pages = clamp(n_pages, 1U, max_pages);
+	size = (size_t)n_pages * ATA_SECT_SIZE;
 
-	size = ata_format_dsm_trim_descr(scmd, trmax, block, n_block);
-	if (size != len)
+	if (ata_format_dsm_trim_descr(scmd, size, block, n_block) != size)
 		goto invalid_param_len;
 
+	/*
+	 * For DATA SET MANAGEMENT TRIM the COUNT field (aka nsect) is the
+	 * number of 512-byte pages to be transferred.
+	 */
 	if (ata_ncq_enabled(dev) && ata_fpdma_dsm_supported(dev)) {
 		/* Newer devices support queued TRIM commands */
 		tf->protocol = ATA_PROT_NCQ;
 		tf->command = ATA_CMD_FPDMA_SEND;
 		tf->hob_nsect = ATA_SUBCMD_FPDMA_SEND_DSM & 0x1f;
 		tf->nsect = qc->hw_tag << 3;
-		tf->hob_feature = (size / 512) >> 8;
-		tf->feature = size / 512;
+		tf->hob_feature = n_pages >> 8;
+		tf->feature = n_pages;
 
 		tf->auxiliary = 1;
 	} else {
 		tf->protocol = ATA_PROT_DMA;
 		tf->hob_feature = 0;
 		tf->feature = ATA_DSM_TRIM;
-		tf->hob_nsect = (size / 512) >> 8;
-		tf->nsect = size / 512;
+		tf->hob_nsect = n_pages >> 8;
+		tf->nsect = n_pages;
 		tf->command = ATA_CMD_DSM;
 	}
 
@@ -3577,6 +3624,12 @@ static unsigned int ata_scsi_write_same_xlat(struct ata_queued_cmd *qc)
 		     ATA_TFLAG_WRITE;
 
 	ata_qc_set_pc_nbytes(qc);
+	/*
+	 * The DSM TRIM payload (size) may be smaller than the WRITE SAME
+	 * data-out buffer (one logical block); only transfer the pages that
+	 * were actually built so the transfer length matches the COUNT field.
+	 */
+	qc->nbytes = size;
 
 	return 0;
 
@@ -3593,88 +3646,286 @@ static unsigned int ata_scsi_write_same_xlat(struct ata_queued_cmd *qc)
 	return 1;
 }
 
-static unsigned int ata_scsi_report_supported_opcodes(struct ata_device *dev,
-						      struct scsi_cmnd *cmd,
-						      u8 *rbuf)
+struct ata_scsi_cmd {
+	u8 op;
+	u8 cdb_len;
+	bool sa_valid;
+	u16 sa;
+};
+
+/*
+ * Array of commands supported with translation or emulation, sorted in
+ * ascending opcode and service action order. All of these commands are
+ * processed either in ata_xlat_func() or in ata_scsi_simulate();
+ *
+ * Note: commands that are not fully supported may be left out of this array
+ * so that they are not reported as supported for passthrough but still
+ * available through the block layer. For now, this includes the following
+ * commands:
+ *  - WRITE_SAME_16: ata_scsi_write_same_xlat() forbids passthrough commands
+ */
+static const struct ata_scsi_cmd ata_supported_cmds[] = {
+	{	.op = TEST_UNIT_READY,		.cdb_len = 6	},
+	{	.op = REZERO_UNIT,		.cdb_len = 6	},
+	{	.op = REQUEST_SENSE,		.cdb_len = 6	},
+	{	.op = READ_6,			.cdb_len = 6	},
+	{	.op = WRITE_6,			.cdb_len = 6	},
+	{	.op = SEEK_6,			.cdb_len = 6	},
+	{	.op = INQUIRY,			.cdb_len = 6	},
+	{	.op = MODE_SELECT,		.cdb_len = 6	},
+	{	.op = MODE_SENSE,		.cdb_len = 6	},
+	{	.op = START_STOP,		.cdb_len = 6	},
+	{	.op = SEND_DIAGNOSTIC,		.cdb_len = 6	},
+	{	.op = READ_CAPACITY,		.cdb_len = 10	},
+	{	.op = READ_10,			.cdb_len = 10	},
+	{	.op = WRITE_10,			.cdb_len = 10	},
+	{	.op = SEEK_10,			.cdb_len = 10	},
+	{	.op = VERIFY,			.cdb_len = 10	},
+	{	.op = SYNCHRONIZE_CACHE,	.cdb_len = 10	},
+	{	.op = MODE_SELECT_10,		.cdb_len = 10	},
+	{	.op = MODE_SENSE_10,		.cdb_len = 10	},
+	{
+		.op = VARIABLE_LENGTH_CMD,	.cdb_len = 32,
+		.sa_valid = true,
+		.sa = ATA_32
+	},
+	{	.op = ATA_16,			.cdb_len = 16	},
+	{	.op = READ_16,			.cdb_len = 16	},
+	{	.op = WRITE_16,			.cdb_len = 16	},
+	{	.op = VERIFY_16,		.cdb_len = 16	},
+	{	.op = SYNCHRONIZE_CACHE_16,	.cdb_len = 16	},
+	{
+		.op = ZBC_OUT,			.cdb_len = 16,
+		.sa_valid = true,
+		.sa = ZO_CLOSE_ZONE
+	},
+	{
+		.op = ZBC_OUT,			.cdb_len = 16,
+		.sa_valid = true,
+		.sa = ZO_FINISH_ZONE
+	},
+	{
+		.op = ZBC_OUT,			.cdb_len = 16,
+		.sa_valid = true,
+		.sa = ZO_OPEN_ZONE
+	},
+	{
+		.op = ZBC_OUT,			.cdb_len = 16,
+		.sa_valid = true,
+		.sa = ZO_RESET_WRITE_POINTER
+	},
+	{
+		.op = ZBC_IN,			.cdb_len = 16,
+		.sa_valid = true,
+		.sa = ZI_REPORT_ZONES
+	},
+	{
+		.op = SERVICE_ACTION_IN_16,	.cdb_len = 16,
+		.sa_valid = true,
+		.sa = SAI_READ_CAPACITY_16
+	},
+	{
+		.op = SERVICE_ACTION_IN_16,	.cdb_len = 16,
+		.sa_valid = true,
+		.sa = SAI_GET_PHYSICAL_ELEMENT_STATUS
+	},
+	{
+		.op = SERVICE_ACTION_IN_16,	.cdb_len = 16,
+		.sa_valid = true,
+		.sa = SAI_REMOVE_ELEMENT_AND_TRUNCATE
+	},
+	{
+		.op = SERVICE_ACTION_IN_16,	.cdb_len = 16,
+		.sa_valid = true,
+		.sa = SAI_RESTORE_ELEMENTS_AND_REBUILD
+	},
+	{
+		.op = SERVICE_ACTION_IN_16,	.cdb_len = 16,
+		.sa_valid = true,
+		.sa = SAI_REMOVE_ELEMENT_AND_MODIFY_ZONES
+	},
+	{	.op = REPORT_LUNS,		.cdb_len = 12	},
+	{	.op = ATA_12,			.cdb_len = 12	},
+	{	.op = SECURITY_PROTOCOL_IN,	.cdb_len = 12	},
+	{
+		.op = MAINTENANCE_IN,		.cdb_len = 12,
+		.sa_valid = true,
+		.sa = MI_REPORT_SUPPORTED_OPERATION_CODES
+	},
+	{	.op = SECURITY_PROTOCOL_OUT,	.cdb_len = 12	},
+};
+
+static const struct ata_scsi_cmd *ata_scsi_get_supported_cmd(u8 op, u16 sa)
 {
-	u8 *cdb = cmd->cmnd;
-	u8 supported = 0, cdlp = 0, rwcdlp = 0;
+	const struct ata_scsi_cmd *cmd;
+	int i;
 
-	if (cdb[2] != 1 && cdb[2] != 3) {
-		ata_dev_warn(dev, "invalid command format %d\n", cdb[2]);
-		ata_scsi_set_invalid_field(dev, cmd, 2, 0xff);
-		return 0;
+	for (i = 0; i < ARRAY_SIZE(ata_supported_cmds); i++) {
+		cmd = &ata_supported_cmds[i];
+		if (cmd->op == op && cmd->sa == sa)
+			return cmd;
 	}
 
-	switch (cdb[3]) {
-	case INQUIRY:
-	case MODE_SENSE:
-	case MODE_SENSE_10:
-	case READ_CAPACITY:
-	case SERVICE_ACTION_IN_16:
-	case REPORT_LUNS:
-	case REQUEST_SENSE:
-	case SYNCHRONIZE_CACHE:
-	case SYNCHRONIZE_CACHE_16:
-	case REZERO_UNIT:
-	case SEEK_6:
-	case SEEK_10:
-	case TEST_UNIT_READY:
-	case SEND_DIAGNOSTIC:
-	case MAINTENANCE_IN:
-	case READ_6:
-	case READ_10:
-	case WRITE_6:
-	case WRITE_10:
-	case ATA_12:
-	case ATA_16:
-	case VERIFY:
-	case VERIFY_16:
-	case MODE_SELECT:
-	case MODE_SELECT_10:
-	case START_STOP:
-		supported = 3;
-		break;
+	return NULL;
+}
+
+static bool ata_scsi_supported_cmd_use_sa(u8 op)
+{
+	const struct ata_scsi_cmd *cmd;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ata_supported_cmds); i++) {
+		cmd = &ata_supported_cmds[i];
+		if (cmd->op == op)
+			return cmd->sa_valid;
+	}
+
+	return false;
+}
+
+struct ata_scsi_cmd_support {
+	u8 cdlp;
+	u8 rwcdlp;
+};
+
+static bool ata_scsi_cmd_is_supported(struct ata_device *dev, u8 op, u16 sa,
+				      struct ata_scsi_cmd_support *sup)
+{
+	const struct ata_scsi_cmd *cmd;
+
+	/* First, see if we support the command. */
+	cmd = ata_scsi_get_supported_cmd(op, sa);
+	if (!cmd)
+		return false;
+
+	/* Now refine the support report depending on the device features. */
+	memset(sup, 0, sizeof(*sup));
+	switch (op) {
 	case READ_16:
-		supported = 3;
 		if (dev->flags & ATA_DFLAG_CDL) {
 			/*
 			 * CDL read descriptors map to the T2A page, that is,
 			 * rwcdlp = 0x01 and cdlp = 0x01
 			 */
-			rwcdlp = 0x01;
-			cdlp = 0x01 << 3;
+			sup->rwcdlp = 0x01;
+			sup->cdlp = 0x01;
 		}
 		break;
 	case WRITE_16:
-		supported = 3;
 		if (dev->flags & ATA_DFLAG_CDL) {
 			/*
 			 * CDL write descriptors map to the T2B page, that is,
 			 * rwcdlp = 0x01 and cdlp = 0x02
 			 */
-			rwcdlp = 0x01;
-			cdlp = 0x02 << 3;
+			sup->rwcdlp = 0x01;
+			sup->cdlp = 0x02;
 		}
 		break;
 	case ZBC_IN:
 	case ZBC_OUT:
-		if (ata_id_zoned_cap(dev->id) ||
-		    dev->class == ATA_DEV_ZAC)
-			supported = 3;
+		return ata_dev_is_zoned(dev);
+	case SERVICE_ACTION_IN_16:
+		switch (sa) {
+		case SAI_GET_PHYSICAL_ELEMENT_STATUS:
+		case SAI_REMOVE_ELEMENT_AND_TRUNCATE:
+			return dev->flags & ATA_DFLAG_DEPOP;
+		case SAI_RESTORE_ELEMENTS_AND_REBUILD:
+			return dev->flags & ATA_DFLAG_DEPOP_RESTORE;
+		case SAI_REMOVE_ELEMENT_AND_MODIFY_ZONES:
+			return dev->flags & ATA_DFLAG_DEPOP_MODIFY;
+		default:
+			return true;
+		}
 		break;
 	case SECURITY_PROTOCOL_IN:
 	case SECURITY_PROTOCOL_OUT:
-		if (dev->flags & ATA_DFLAG_TRUSTED)
-			supported = 3;
-		break;
+		return dev->flags & ATA_DFLAG_TRUSTED;
 	default:
 		break;
 	}
 
+	return true;
+}
+
+static unsigned int
+ata_scsi_report_all_supported_opcodes(struct ata_device *dev, u8 *rbuf)
+{
+	struct ata_scsi_cmd_support sup;
+	const struct ata_scsi_cmd *cmd;
+	unsigned int len = 4;
+	u8 *buf = &rbuf[len];
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ata_supported_cmds); i++) {
+		if (len > ATA_SCSI_RBUF_SIZE - 8)
+			break;
+
+		cmd = &ata_supported_cmds[i];
+
+		/* All command format */
+		if (!ata_scsi_cmd_is_supported(dev, cmd->op, cmd->sa, &sup))
+			continue;
+
+		buf[0] = cmd->op;
+		put_unaligned_be16(cmd->sa, &buf[2]);
+		buf[5] = (sup.rwcdlp << 6) | (sup.cdlp << 2);
+		if (cmd->sa_valid)
+			buf[5] |= 0x01;
+		put_unaligned_be16(cmd->cdb_len, &buf[6]);
+
+		/* CTDP == 0 */
+		len += 8;
+		buf += 8;
+	}
+
+	put_unaligned_be32(len - 4, &rbuf[0]);
+
+	return len;
+}
+
+static unsigned int ata_scsi_report_supported_opcodes(struct ata_device *dev,
+						      struct scsi_cmnd *cmd,
+						      u8 *rbuf)
+{
+	struct ata_scsi_cmd_support sup;
+	u8 *cdb = cmd->cmnd;
+	u16 sa = 0;
+
+	switch (cdb[2]) {
+	case 0:
+		/* All command format */
+		return ata_scsi_report_all_supported_opcodes(dev, rbuf);
+	case 1:
+		/* One command format with command support data, ignore sa. */
+		if (ata_scsi_supported_cmd_use_sa(cdb[3])) {
+			ata_scsi_set_invalid_field(dev, cmd, 3, 0xff);
+			return 0;
+		}
+		break;
+	case 2:
+		/* One command format, must have sa. */
+		if (!ata_scsi_supported_cmd_use_sa(cdb[3])) {
+			ata_scsi_set_invalid_field(dev, cmd, 3, 0xff);
+			return 0;
+		}
+		fallthrough;
+	case 3:
+		/* One command format */
+		sa = get_unaligned_be16(&cdb[4]);
+		break;
+	default:
+		ata_dev_warn(dev, "invalid command format %d\n", cdb[2]);
+		ata_scsi_set_invalid_field(dev, cmd, 2, 0xff);
+		return 0;
+	}
+
 	/* One command format */
-	rbuf[0] = rwcdlp;
-	rbuf[1] = cdlp | supported;
+	if (ata_scsi_cmd_is_supported(dev, cdb[3], sa, &sup)) {
+		rbuf[0] = sup.rwcdlp;
+		rbuf[1] = (sup.cdlp << 3) | 0x03;
+	} else {
+		rbuf[1] = 0x01;
+	}
 
 	return 4;
 }
@@ -4357,6 +4608,245 @@ static unsigned int ata_scsi_security_inout_xlat(struct ata_queued_cmd *qc)
 	return 0;
 }
 
+/*
+ * Convert T-13 little-endian field representation of GET PHYSICAL ELEMENT
+ * STATUS DMA command reply into T-10 big-endian field representation.
+ */
+static void ata_scsi_get_phys_element_status_complete(struct ata_queued_cmd *qc)
+{
+	struct scsi_cmnd *scmd = qc->scsicmd;
+	struct sg_mapping_iter miter;
+	unsigned int bytes = 0;
+
+	lockdep_assert_held(qc->ap->lock);
+
+	sg_miter_start(&miter, scsi_sglist(scmd), scsi_sg_count(scmd),
+		       SG_MITER_TO_SG | SG_MITER_ATOMIC);
+
+	while (sg_miter_next(&miter)) {
+		unsigned int offset = 0;
+
+		if (bytes == 0) {
+			u32 num_desc, num_desc_returned, id;
+			u16 max_depop, cur_depop;
+			char *hdr;
+
+			/* Swizzle the header */
+			hdr = miter.addr;
+			num_desc = get_unaligned_le32(&hdr[0]);
+			num_desc_returned = get_unaligned_le32(&hdr[4]);
+			id = get_unaligned_le32(&hdr[8]);
+			max_depop = get_unaligned_le16(&hdr[12]);
+			cur_depop = get_unaligned_le16(&hdr[14]);
+
+			put_unaligned_be32(num_desc, &hdr[0]);
+			put_unaligned_be32(num_desc_returned, &hdr[4]);
+			put_unaligned_be32(id, &hdr[8]);
+			put_unaligned_be16(max_depop, &hdr[12]);
+			put_unaligned_be16(cur_depop, &hdr[14]);
+
+			offset += 32;
+			bytes += 32;
+		}
+
+		/* Swizzle the descriptors. */
+		while (offset < miter.length) {
+			char *desc;
+			u32 id;
+			u8 type;
+
+			desc = miter.addr + offset;
+			id = get_unaligned_le32(&desc[4]);
+			put_unaligned_be32(id, &desc[4]);
+
+			type = desc[14];
+			if (type == SCSI_PHYS_ELEM_TYPE_ALL_ACCESS_STORAGE) {
+				u64 capacity = get_unaligned_le64(&desc[16]);
+
+				put_unaligned_be64(capacity, &desc[16]);
+			} else {
+				u64 num_zones;
+
+				id = get_unaligned_le32(&desc[16]);
+				num_zones = get_unaligned_le64(&desc[24]);
+
+				put_unaligned_be32(id, &desc[16]);
+				put_unaligned_be64(num_zones, &desc[24]);
+			}
+
+			offset += 32;
+			bytes += 32;
+		}
+	}
+	sg_miter_stop(&miter);
+
+	ata_scsi_qc_complete(qc);
+}
+
+static unsigned int
+ata_scsi_get_phys_element_status_xlat(struct ata_queued_cmd *qc)
+{
+	struct scsi_cmnd *scmd = qc->scsicmd;
+	const u8 *cdb = scmd->cmnd;
+	struct ata_device *dev = qc->dev;
+	struct ata_taskfile *tf = &qc->tf;
+	u32 starting_element, len;
+
+	/* ATA_CMD_GET_PHYS_ELEMENT_STATUS is a DMA command. */
+	if (!(dev->flags & ATA_DFLAG_DEPOP) || !ata_dma_enabled(dev)) {
+		ata_scsi_set_sense(dev, scmd, ILLEGAL_REQUEST, 0x20, 0x0);
+		return 1;
+	}
+
+	len = get_unaligned_be32(&cdb[10]) / ATA_SECT_SIZE;
+	if (!len || len > U16_MAX) {
+		ata_scsi_set_invalid_field(dev, scmd, 10, 0);
+		return 1;
+	}
+
+	tf->protocol = ATA_PROT_DMA;
+	tf->command = ATA_CMD_GET_PHYS_ELEMENT_STATUS;
+	tf->hob_feature = cdb[14];
+	tf->hob_nsect = (len >> 8) & 0xff;
+	tf->nsect = len & 0xff;
+
+	starting_element = get_unaligned_be32(&cdb[6]);
+	if (starting_element) {
+		tf->hob_lbal = (starting_element >> 24) & 0xff;
+		tf->lbah = (starting_element >> 16) & 0xff;
+		tf->lbam = (starting_element >> 8) & 0xff;
+		tf->lbal = starting_element & 0xff;
+	}
+	tf->device = ATA_LBA;
+	tf->flags |= ATA_TFLAG_ISADDR | ATA_TFLAG_DEVICE | ATA_TFLAG_LBA48;
+
+	ata_qc_set_pc_nbytes(qc);
+
+	qc->flags |= ATA_QCFLAG_RESULT_TF;
+	qc->complete_fn = ata_scsi_get_phys_element_status_complete;
+
+	return 0;
+}
+
+static void ata_scsi_depop_ua_cap_changed_complete(struct ata_queued_cmd *qc)
+{
+	struct scsi_cmnd *scmd = qc->scsicmd;
+	u8 *cdb = scmd->cmnd;
+	bool is_ata_passthru = cdb[0] == ATA_16 || cdb[0] == ATA_12;
+	bool is_success = qc->err_mask == 0;
+
+	/*
+	 * For successful non-passthrough commands, raise a UNIT ATTENTION with
+	 * the additional sense code set to CAPACITY DATA HAS CHANGED to be
+	 * raised. Note that this should be done only if the capacity has
+	 * actually changed, which may not be the case if the element that was
+	 * specified for depopulation was already depopulated, or we did not
+	 * restore any removed element. But a capacity change unit attention is
+	 * harmless, so always raise the unit attention.
+	 */
+	if (is_success && !is_ata_passthru)
+		ata_scsi_set_sense(qc->dev, scmd, UNIT_ATTENTION,
+				   UA_CHANGED_ASC, CAPACITY_CHANGED_ASCQ);
+	ata_scsi_qc_complete(qc);
+}
+
+static unsigned int
+ata_scsi_remove_element_and_truncate_xlat(struct ata_queued_cmd *qc)
+{
+	struct scsi_cmnd *scmd = qc->scsicmd;
+	const u8 *cdb = scmd->cmnd;
+	struct ata_device *dev = qc->dev;
+	struct ata_taskfile *tf = &qc->tf;
+	u64 req_capacity;
+	u32 id;
+
+	if (!(dev->flags & ATA_DFLAG_DEPOP)) {
+		ata_scsi_set_sense(dev, scmd, ILLEGAL_REQUEST, 0x20, 0x0);
+		return 1;
+	}
+
+	req_capacity = get_unaligned_be64(&cdb[2]);
+	if (req_capacity == 1) {
+		ata_scsi_set_invalid_field(dev, scmd, 2, 0);
+		return 1;
+	}
+
+	id = get_unaligned_be32(&cdb[10]);
+
+	tf->protocol = ATA_PROT_NODATA;
+	tf->command = ATA_CMD_REMOVE_ELEMENT_AND_TRUNCATE;
+	tf->hob_feature = (id >> 24) & 0xff;
+	tf->feature = (id >> 16) & 0xff;
+	tf->hob_nsect = (id >> 8) & 0xff;
+	tf->nsect = id & 0xff;
+	tf->hob_lbah = (req_capacity >> 40) & 0xff;
+	tf->hob_lbam = (req_capacity >> 32) & 0xff;
+	tf->hob_lbal = (req_capacity >> 24) & 0xff;
+	tf->lbah = (req_capacity >> 16) & 0xff;
+	tf->lbam = (req_capacity >> 8) & 0xff;
+	tf->lbal = req_capacity & 0xff;
+	tf->device = ATA_LBA;
+	tf->flags |= ATA_TFLAG_ISADDR | ATA_TFLAG_DEVICE | ATA_TFLAG_LBA48;
+
+	qc->flags |= ATA_QCFLAG_RESULT_TF;
+	qc->complete_fn = ata_scsi_depop_ua_cap_changed_complete;
+
+	return 0;
+}
+
+static unsigned int
+ata_scsi_remove_element_and_modify_zones_xlat(struct ata_queued_cmd *qc)
+{
+	struct scsi_cmnd *scmd = qc->scsicmd;
+	const u8 *cdb = scmd->cmnd;
+	struct ata_device *dev = qc->dev;
+	struct ata_taskfile *tf = &qc->tf;
+	u32 id;
+
+	if (!(dev->flags & ATA_DFLAG_DEPOP_MODIFY)) {
+		ata_scsi_set_sense(dev, scmd, ILLEGAL_REQUEST, 0x20, 0x0);
+		return 1;
+	}
+
+	id = get_unaligned_be32(&cdb[10]);
+
+	tf->protocol = ATA_PROT_NODATA;
+	tf->command = ATA_CMD_REMOVE_ELEMENT_AND_MODIFY_ZONES;
+	tf->hob_feature = (id >> 24) & 0xff;
+	tf->feature = (id >> 16) & 0xff;
+	tf->hob_nsect = (id >> 8) & 0xff;
+	tf->nsect = id & 0xff;
+	tf->device = ATA_LBA;
+	tf->flags |= ATA_TFLAG_ISADDR | ATA_TFLAG_DEVICE | ATA_TFLAG_LBA48;
+
+	qc->flags |= ATA_QCFLAG_RESULT_TF;
+
+	return 0;
+}
+
+static unsigned int
+ata_scsi_restore_elements_and_rebuild_xlat(struct ata_queued_cmd *qc)
+{
+	struct scsi_cmnd *scmd = qc->scsicmd;
+	struct ata_device *dev = qc->dev;
+	struct ata_taskfile *tf = &qc->tf;
+
+	if (!(dev->flags & ATA_DFLAG_DEPOP_RESTORE)) {
+		ata_scsi_set_sense(dev, scmd, ILLEGAL_REQUEST, 0x20, 0x0);
+		return 1;
+	}
+
+	tf->protocol = ATA_PROT_NODATA;
+	tf->command = ATA_CMD_RESTORE_ELEMENTS_AND_REBUILD;
+	tf->device = ATA_LBA;
+	tf->flags |= ATA_TFLAG_ISADDR | ATA_TFLAG_DEVICE | ATA_TFLAG_LBA48;
+
+	qc->flags |= ATA_QCFLAG_RESULT_TF;
+	qc->complete_fn = ata_scsi_depop_ua_cap_changed_complete;
+
+	return 0;
+}
+
 /**
  *	ata_scsi_var_len_cdb_xlat - SATL variable length CDB to Handler
  *	@qc: Command to be translated
@@ -4388,7 +4878,7 @@ static unsigned int ata_scsi_var_len_cdb_xlat(struct ata_queued_cmd *qc)
 /**
  *	ata_get_xlat_func - check if SCSI to ATA translation is possible
  *	@dev: ATA device
- *	@cmd: SCSI command opcode to consider
+ *	@cdb: CDB of the SCSI command to consider
  *
  *	Look up the SCSI command given, and determine whether the
  *	SCSI command is to be translated or simulated.
@@ -4397,9 +4887,12 @@ static unsigned int ata_scsi_var_len_cdb_xlat(struct ata_queued_cmd *qc)
  *	Pointer to translation function if possible, %NULL if not.
  */
 
-static inline ata_xlat_func_t ata_get_xlat_func(struct ata_device *dev, u8 cmd)
+static inline ata_xlat_func_t ata_get_xlat_func(struct ata_device *dev,
+						u8 *cdb)
 {
-	switch (cmd) {
+	u8 sa;
+
+	switch (cdb[0]) {
 	case READ_6:
 	case READ_10:
 	case READ_16:
@@ -4433,6 +4926,18 @@ static inline ata_xlat_func_t ata_get_xlat_func(struct ata_device *dev, u8 cmd)
 	case MODE_SELECT_10:
 		return ata_scsi_mode_select_xlat;
 
+	case SERVICE_ACTION_IN_16:
+		sa = cdb[1] & 0x1f;
+		if (sa == SAI_GET_PHYSICAL_ELEMENT_STATUS)
+			return ata_scsi_get_phys_element_status_xlat;
+		if (sa == SAI_REMOVE_ELEMENT_AND_TRUNCATE)
+			return ata_scsi_remove_element_and_truncate_xlat;
+		if (sa == SAI_REMOVE_ELEMENT_AND_MODIFY_ZONES)
+			return ata_scsi_remove_element_and_modify_zones_xlat;
+		if (sa == SAI_RESTORE_ELEMENTS_AND_REBUILD)
+			return ata_scsi_restore_elements_and_rebuild_xlat;
+		break;
+
 	case ZBC_IN:
 		return ata_scsi_zbc_in_xlat;
 
@@ -4530,7 +5035,8 @@ enum scsi_qc_status __ata_scsi_queuecmd(struct scsi_cmnd *scmd,
 					struct ata_port *ap)
 	__must_hold(ap->lock)
 {
-	u8 scsi_op = scmd->cmnd[0];
+	u8 *cdb = scmd->cmnd;
+	u8 scsi_op = cdb[0];
 	ata_xlat_func_t xlat_func;
 
 	/*
@@ -4550,7 +5056,7 @@ enum scsi_qc_status __ata_scsi_queuecmd(struct scsi_cmnd *scmd,
 		if (unlikely(scmd->cmd_len > dev->cdb_len))
 			goto bad_cdb_len;
 
-		xlat_func = ata_get_xlat_func(dev, scsi_op);
+		xlat_func = ata_get_xlat_func(dev, cdb);
 	} else if (likely((scsi_op != ATA_16) || !atapi_passthru16)) {
 		/* relay SCSI command to ATAPI device */
 		int len = COMMAND_SIZE(scsi_op);
@@ -4566,7 +5072,7 @@ enum scsi_qc_status __ata_scsi_queuecmd(struct scsi_cmnd *scmd,
 		if (unlikely(scmd->cmd_len > 16))
 			goto bad_cdb_len;
 
-		xlat_func = ata_get_xlat_func(dev, scsi_op);
+		xlat_func = ata_get_xlat_func(dev, cdb);
 	}
 
 	if (xlat_func)
diff --git a/drivers/ata/libata.h b/drivers/ata/libata.h
index 700627596ce11..39494ab206a29 100644
--- a/drivers/ata/libata.h
+++ b/drivers/ata/libata.h
@@ -44,7 +44,7 @@ static inline bool ata_sstatus_online(u32 sstatus)
 	return (sstatus & 0xf) == 0x3;
 }
 
-static inline bool ata_dev_is_zac(struct ata_device *dev)
+static inline bool ata_dev_is_zoned(struct ata_device *dev)
 {
 	/* Host managed device or host aware device */
 	return dev->class == ATA_DEV_ZAC ||
diff --git a/drivers/ata/pata_mpc52xx.c b/drivers/ata/pata_mpc52xx.c
index 210a63283f62b..1739743427cfc 100644
--- a/drivers/ata/pata_mpc52xx.c
+++ b/drivers/ata/pata_mpc52xx.c
@@ -769,10 +769,8 @@ static int mpc52xx_ata_probe(struct platform_device *op)
 	task_irq = bcom_get_task_irq(dmatsk);
 	rv = devm_request_irq(&op->dev, task_irq, &mpc52xx_ata_task_irq, 0,
 				"ATA task", priv);
-	if (rv) {
-		dev_err(&op->dev, "error requesting DMA IRQ\n");
+	if (rv)
 		goto err2;
-	}
 	priv->dmatsk = dmatsk;
 
 	/* Init the hw */
diff --git a/drivers/auxdisplay/ht16k33.c b/drivers/auxdisplay/ht16k33.c
index 0b8ba754b3430..19b5c14e58ffc 100644
--- a/drivers/auxdisplay/ht16k33.c
+++ b/drivers/auxdisplay/ht16k33.c
@@ -547,11 +547,8 @@ static int ht16k33_keypad_probe(struct i2c_client *client,
 					ht16k33_keypad_irq_thread,
 					IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
 					DRIVER_NAME, keypad);
-	if (err) {
-		dev_err(dev, "irq request failed %d, error %d\n", client->irq,
-			err);
+	if (err)
 		return err;
-	}
 
 	ht16k33_keypad_stop(keypad->dev);
 
diff --git a/drivers/auxdisplay/panel.c b/drivers/auxdisplay/panel.c
index d8854cf15268b..a7dfc53157bc2 100644
--- a/drivers/auxdisplay/panel.c
+++ b/drivers/auxdisplay/panel.c
@@ -1450,43 +1450,6 @@ static struct logical_input *panel_bind_key(const char *name, const char *press,
 	return key;
 }
 
-#if 0
-/* tries to bind a callback function to the signal name <name>. The function
- * <press_fct> will be called with the <press_data> arg when the signal is
- * activated, and so on for <release_fct>/<release_data>
- * Returns the pointer to the new signal if ok, NULL if the signal could not
- * be bound.
- */
-static struct logical_input *panel_bind_callback(char *name,
-						 void (*press_fct)(int),
-						 int press_data,
-						 void (*release_fct)(int),
-						 int release_data)
-{
-	struct logical_input *callback;
-
-	callback = kmalloc(sizeof(*callback), GFP_KERNEL);
-	if (!callback)
-		return NULL;
-
-	memset(callback, 0, sizeof(struct logical_input));
-	if (!input_name2mask(name, &callback->mask, &callback->value,
-			     &scan_mask_i, &scan_mask_o))
-		return NULL;
-
-	callback->type = INPUT_TYPE_STD;
-	callback->state = INPUT_ST_LOW;
-	callback->rise_time = 1;
-	callback->fall_time = 1;
-	callback->u.std.press_fct = press_fct;
-	callback->u.std.press_data = press_data;
-	callback->u.std.release_fct = release_fct;
-	callback->u.std.release_data = release_data;
-	list_add(&callback->list, &logical_inputs);
-	return callback;
-}
-#endif
-
 static void keypad_init(void)
 {
 	int keynum;
diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c
index c99f2ab105e5b..6476227b772c6 100644
--- a/drivers/base/arch_numa.c
+++ b/drivers/base/arch_numa.c
@@ -221,9 +221,6 @@ static int __init numa_register_nodes(void)
 		node_set_online(nid);
 	}
 
-	/* Setup online nodes to actual nodes*/
-	node_possible_map = numa_nodes_parsed;
-
 	return 0;
 }
 
@@ -231,10 +228,6 @@ static int __init numa_init(int (*init_func)(void))
 {
 	int ret;
 
-	nodes_clear(numa_nodes_parsed);
-	nodes_clear(node_possible_map);
-	nodes_clear(node_online_map);
-
 	ret = numa_memblks_init(init_func, /* memblock_force_top_down */ false);
 	if (ret < 0)
 		goto out_free_distance;
@@ -283,7 +276,6 @@ static int __init dummy_numa_init(void)
 		pr_err("NUMA init failed\n");
 		return ret;
 	}
-	node_set(0, numa_nodes_parsed);
 
 	numa_off = true;
 	return 0;
diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c
index 19d288a3c80cd..69e52fed42415 100644
--- a/drivers/base/cpu.c
+++ b/drivers/base/cpu.c
@@ -218,7 +218,7 @@ static ssize_t show_cpus_attr(struct device *dev,
 {
 	struct cpu_attr *ca = container_of(attr, struct cpu_attr, attr);
 
-	return cpumap_print_to_pagebuf(true, buf, ca->map);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(ca->map));
 }
 
 #define _CPU_ATTR(name, map) \
diff --git a/drivers/base/dd.c b/drivers/base/dd.c
index 60c005223844d..f6525a7ee8c54 100644
--- a/drivers/base/dd.c
+++ b/drivers/base/dd.c
@@ -790,8 +790,8 @@ static int really_probe_debug(struct device *dev, const struct device_driver *dr
 	 * CONFIG_DYNAMIC_DEBUG and we want a simple 'initcall_debug' on the
 	 * kernel commandline to print this all the time at the debug level.
 	 */
-	printk(KERN_DEBUG "probe of %s returned %d after %lld usecs\n",
-		 dev_name(dev), ret, ktime_us_delta(rettime, calltime));
+	printk(KERN_DEBUG "probe of %s with driver %s returned %d after %lld usecs\n",
+	       dev_name(dev), drv->name, ret, ktime_us_delta(rettime, calltime));
 	return ret;
 }
 
diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c
index b1c4ceb65026e..aef0fcc6aba1e 100644
--- a/drivers/base/devtmpfs.c
+++ b/drivers/base/devtmpfs.c
@@ -413,7 +413,7 @@ static noinline int __init devtmpfs_setup(void *p)
 {
 	int err;
 
-	err = ksys_unshare(CLONE_NEWNS);
+	err = ksys_unshare(UNSHARE_EMPTY_MNTNS);
 	if (err)
 		goto out;
 	err = init_mount("devtmpfs", "/", "devtmpfs", DEVTMPFS_MFLAGS, NULL);
diff --git a/drivers/base/firmware_loader/builtin/Makefile b/drivers/base/firmware_loader/builtin/Makefile
index 6c067dedc01ec..c0264b15d48e4 100644
--- a/drivers/base/firmware_loader/builtin/Makefile
+++ b/drivers/base/firmware_loader/builtin/Makefile
@@ -16,20 +16,24 @@ ASM_ALIGN = $(if $(CONFIG_64BIT),3,2)
 PROGBITS  = $(if $(CONFIG_ARM),%,@)progbits
 
 filechk_fwbin = \
-	echo "/* Generated by $(src)/Makefile */"		;\
-	echo "    .section .rodata"				;\
-	echo "    .p2align 4"					;\
-	echo "_fw_$(FWSTR)_bin:"				;\
-	echo "    .incbin \"$(fwdir)/$(FWNAME)\""		;\
-	echo "_fw_end:"						;\
-	echo "    .section .rodata.str,\"aMS\",$(PROGBITS),1"	;\
-	echo "    .p2align $(ASM_ALIGN)"			;\
-	echo "_fw_$(FWSTR)_name:"				;\
-	echo "    .string \"$(FWNAME)\""			;\
-	echo "    .section .builtin_fw,\"a\",$(PROGBITS)"	;\
-	echo "    .p2align $(ASM_ALIGN)"			;\
-	echo "    $(ASM_WORD) _fw_$(FWSTR)_name"		;\
-	echo "    $(ASM_WORD) _fw_$(FWSTR)_bin"			;\
+	if [ ! -s "$(fwdir)/$(FWNAME)" ]; then				 \
+		echo "error: empty firmware: $(fwdir)/$(FWNAME)" >&2	;\
+		exit 1							;\
+	fi								;\
+	echo "/* Generated by $(src)/Makefile */"			;\
+	echo "    .section .rodata"					;\
+	echo "    .p2align 4"						;\
+	echo "_fw_$(FWSTR)_bin:"					;\
+	echo "    .incbin \"$(fwdir)/$(FWNAME)\""			;\
+	echo "_fw_end:"							;\
+	echo "    .section .rodata.str,\"aMS\",$(PROGBITS),1"		;\
+	echo "    .p2align $(ASM_ALIGN)"				;\
+	echo "_fw_$(FWSTR)_name:"					;\
+	echo "    .string \"$(FWNAME)\""				;\
+	echo "    .section .builtin_fw,\"a\",$(PROGBITS)"		;\
+	echo "    .p2align $(ASM_ALIGN)"				;\
+	echo "    $(ASM_WORD) _fw_$(FWSTR)_name"			;\
+	echo "    $(ASM_WORD) _fw_$(FWSTR)_bin"				;\
 	echo "    $(ASM_WORD) _fw_end - _fw_$(FWSTR)_bin"
 
 $(obj)/%.gen.S: FORCE
diff --git a/drivers/base/firmware_loader/builtin/main.c b/drivers/base/firmware_loader/builtin/main.c
index d36befebb1b93..1dcebe8e7f8eb 100644
--- a/drivers/base/firmware_loader/builtin/main.c
+++ b/drivers/base/firmware_loader/builtin/main.c
@@ -53,6 +53,8 @@ bool firmware_request_builtin(struct firmware *fw, const char *name)
 
 	for (b_fw = __start_builtin_fw; b_fw != __end_builtin_fw; b_fw++) {
 		if (strcmp(name, b_fw->name) == 0) {
+			if (b_fw->size == 0)
+				return false;
 			fw->size = b_fw->size;
 			fw->data = b_fw->data;
 			return true;
diff --git a/drivers/base/isa.c b/drivers/base/isa.c
index 5887e4211f80b..4e9f68080f399 100644
--- a/drivers/base/isa.c
+++ b/drivers/base/isa.c
@@ -166,14 +166,16 @@ static int __init isa_bus_init(void)
 	int error;
 
 	error = bus_register(&isa_bus_type);
-	if (!error) {
-		isa_bus = root_device_register("isa");
-		if (IS_ERR(isa_bus)) {
-			error = PTR_ERR(isa_bus);
-			bus_unregister(&isa_bus_type);
-		}
+	if (error)
+		return error;
+
+	isa_bus = root_device_register("isa");
+	if (IS_ERR(isa_bus)) {
+		bus_unregister(&isa_bus_type);
+		return PTR_ERR(isa_bus);
 	}
-	return error;
+
+	return 0;
 }
 
 postcore_initcall(isa_bus_init);
diff --git a/drivers/base/memory.c b/drivers/base/memory.c
index bcfe2d9f4adbd..5eead3346f1e3 100644
--- a/drivers/base/memory.c
+++ b/drivers/base/memory.c
@@ -46,6 +46,15 @@ int mhp_online_type_from_str(const char *str)
 	}
 	return -EINVAL;
 }
+EXPORT_SYMBOL_GPL(mhp_online_type_from_str);
+
+const char *mhp_online_type_to_str(int online_type)
+{
+	if (online_type < 0 || online_type >= (int)ARRAY_SIZE(online_type_to_str))
+		return NULL;
+	return online_type_to_str[online_type];
+}
+EXPORT_SYMBOL_GPL(mhp_online_type_to_str);
 
 #define to_memory_block(dev) container_of(dev, struct memory_block, dev)
 
diff --git a/drivers/base/module.c b/drivers/base/module.c
index 218aaa0964552..6789dca485ab5 100644
--- a/drivers/base/module.c
+++ b/drivers/base/module.c
@@ -42,7 +42,7 @@ int module_add_driver(struct module *mod, const struct device_driver *drv)
 	if (mod)
 		mk = &mod->mkobj;
 	else if (drv->mod_name) {
-		/* Lookup or create built-in module entry in /sys/modules */
+		/* Lookup or create built-in module entry in /sys/module */
 		mk = lookup_or_create_module_kobject(drv->mod_name);
 		if (mk) {
 			/* remember our module structure */
diff --git a/drivers/base/platform.c b/drivers/base/platform.c
index fb9120b0bcfe0..a71015f1d9153 100644
--- a/drivers/base/platform.c
+++ b/drivers/base/platform.c
@@ -599,7 +599,7 @@ static void platform_device_release(struct device *dev)
 	struct platform_object *pa = container_of(dev, struct platform_object,
 						  pdev.dev);
 
-	of_node_put(pa->pdev.dev.of_node);
+	fwnode_handle_put(pa->pdev.dev.fwnode);
 	kfree(pa->pdev.dev.platform_data);
 	kfree(pa->pdev.mfd_cell);
 	kfree(pa->pdev.resource);
@@ -619,6 +619,13 @@ static void platform_device_release_full(struct device *dev)
  *
  * Create a platform device object which can have other objects attached
  * to it, and which will have attached objects freed when it is released.
+ *
+ * The following fields of the dynamically allocated platform device must not
+ * be modified manually: resource, num_resources, dev.platform_data,
+ * dev.of_node and dev.fwnode. Users wishing to do the split platform device
+ * registration with platform_device_alloc() + platform_device_add() are
+ * required to use dedicated helpers for adding resources, platform data or
+ * assigning firmware nodes.
  */
 struct platform_device *platform_device_alloc(const char *name, int id)
 {
@@ -693,6 +700,55 @@ int platform_device_add_data(struct platform_device *pdev, const void *data,
 }
 EXPORT_SYMBOL_GPL(platform_device_add_data);
 
+/**
+ * platform_device_set_of_node - assign an OF node to device
+ * @pdev: platform device to add the node for
+ * @np: new device node
+ *
+ * Assign an OF node to this platform device. Internally keep track of the
+ * reference count. Devices created with platform_device_alloc() must use this
+ * function instead of assigning the node manually.
+ */
+void platform_device_set_of_node(struct platform_device *pdev,
+				 struct device_node *np)
+{
+	platform_device_set_fwnode(pdev, of_fwnode_handle(np));
+}
+EXPORT_SYMBOL_GPL(platform_device_set_of_node);
+
+/**
+ * platform_device_set_fwnode - assign a firmware node to device
+ * @pdev: platform device to set the node for
+ * @fwnode: new firmware node
+ *
+ * Assign a firmware node to this platform device. Internally keep track of the
+ * reference count. Devices created with platform_device_alloc() must use this
+ * function instead of assigning the node manually.
+ */
+void platform_device_set_fwnode(struct platform_device *pdev,
+				struct fwnode_handle *fwnode)
+{
+	fwnode_handle_put(pdev->dev.fwnode);
+	device_set_node(&pdev->dev, fwnode_handle_get(fwnode));
+}
+EXPORT_SYMBOL_GPL(platform_device_set_fwnode);
+
+/**
+ * platform_device_set_of_node_from_dev - reuse OF node of another device
+ * @pdev: platform device to set the node for
+ * @dev2: device whose OF node to reuse
+ *
+ * Reuses the OF node of another device in this platform device while
+ * internally keeping track of reference counting.
+ */
+void platform_device_set_of_node_from_dev(struct platform_device *pdev,
+					  const struct device *dev2)
+{
+	device_set_of_node_from_dev(&pdev->dev, dev2);
+	pdev->dev.fwnode = of_fwnode_handle(pdev->dev.of_node);
+}
+EXPORT_SYMBOL_GPL(platform_device_set_of_node_from_dev);
+
 /**
  * platform_device_add - add a platform device to device hierarchy
  * @pdev: platform device we're adding
@@ -868,8 +924,7 @@ struct platform_device *platform_device_register_full(const struct platform_devi
 		return ERR_PTR(-ENOMEM);
 
 	pdev->dev.parent = pdevinfo->parent;
-	pdev->dev.fwnode = pdevinfo->fwnode;
-	pdev->dev.of_node = of_node_get(to_of_node(pdev->dev.fwnode));
+	device_set_node(&pdev->dev, fwnode_handle_get(pdevinfo->fwnode));
 	dev_assign_of_node_reused(&pdev->dev, pdevinfo->of_node_reused);
 
 	if (pdevinfo->dma_mask) {
diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c
index 335288e8b5b31..fab38bc981139 100644
--- a/drivers/base/power/runtime.c
+++ b/drivers/base/power/runtime.c
@@ -469,9 +469,6 @@ static int rpm_callback(int (*cb)(struct device *), struct device *dev)
 	if (retval == -EACCES)
 		retval = -EAGAIN;
 
-	if (retval != -EAGAIN && retval != -EBUSY)
-		dev->power.runtime_error = retval;
-
 	return retval;
 }
 
@@ -751,6 +748,9 @@ static int rpm_suspend(struct device *dev, int rpmflags)
 	dev->power.deferred_resume = false;
 	wake_up_all(&dev->power.wait_queue);
 
+	if (retval != -EAGAIN && retval != -EBUSY)
+		dev->power.runtime_error = retval;
+
 	/*
 	 * On transient errors, if the callback routine failed an autosuspend,
 	 * and if the last_busy time has been updated so that there is a new
diff --git a/drivers/base/regmap/internal.h b/drivers/base/regmap/internal.h
index 55273a6178f8d..a6e4689000af8 100644
--- a/drivers/base/regmap/internal.h
+++ b/drivers/base/regmap/internal.h
@@ -189,7 +189,7 @@ struct regcache_ops {
 	const char *name;
 	enum regcache_type type;
 	int (*init)(struct regmap *map);
-	int (*exit)(struct regmap *map);
+	void (*exit)(struct regmap *map);
 	int (*populate)(struct regmap *map);
 #ifdef CONFIG_DEBUG_FS
 	void (*debugfs_init)(struct regmap *map);
diff --git a/drivers/base/regmap/regcache-flat.c b/drivers/base/regmap/regcache-flat.c
index 025e6749bb244..be8497fd240c3 100644
--- a/drivers/base/regmap/regcache-flat.c
+++ b/drivers/base/regmap/regcache-flat.c
@@ -53,7 +53,7 @@ static int regcache_flat_init(struct regmap *map)
 	return -ENOMEM;
 }
 
-static int regcache_flat_exit(struct regmap *map)
+static void regcache_flat_exit(struct regmap *map)
 {
 	struct regcache_flat_data *cache = map->cache;
 
@@ -62,8 +62,6 @@ static int regcache_flat_exit(struct regmap *map)
 
 	kfree(cache);
 	map->cache = NULL;
-
-	return 0;
 }
 
 static int regcache_flat_populate(struct regmap *map)
diff --git a/drivers/base/regmap/regcache-maple.c b/drivers/base/regmap/regcache-maple.c
index 49ba7282e4b8f..e46fe5c32b7fc 100644
--- a/drivers/base/regmap/regcache-maple.c
+++ b/drivers/base/regmap/regcache-maple.c
@@ -112,7 +112,7 @@ static int regcache_maple_drop(struct regmap *map, unsigned int min,
 	unsigned long *entry, *lower, *upper;
 	/* initialized to work around false-positive -Wuninitialized warning */
 	unsigned long lower_index = 0, lower_last = 0;
-	unsigned long upper_index, upper_last;
+	unsigned long upper_index = 0, upper_last = 0;
 	int ret = 0;
 
 	lower = NULL;
@@ -307,7 +307,7 @@ static int regcache_maple_init(struct regmap *map)
 	return 0;
 }
 
-static int regcache_maple_exit(struct regmap *map)
+static void regcache_maple_exit(struct regmap *map)
 {
 	struct maple_tree *mt = map->cache;
 	MA_STATE(mas, mt, 0, UINT_MAX);
@@ -315,7 +315,7 @@ static int regcache_maple_exit(struct regmap *map)
 
 	/* if we've already been called then just return */
 	if (!mt)
-		return 0;
+		return;
 
 	mas_lock(&mas);
 	mas_for_each(&mas, entry, UINT_MAX)
@@ -325,8 +325,6 @@ static int regcache_maple_exit(struct regmap *map)
 
 	kfree(mt);
 	map->cache = NULL;
-
-	return 0;
 }
 
 static int regcache_maple_insert_block(struct regmap *map, int first,
diff --git a/drivers/base/regmap/regcache-rbtree.c b/drivers/base/regmap/regcache-rbtree.c
index a69e8b4c359b7..520d5f8ba3cd9 100644
--- a/drivers/base/regmap/regcache-rbtree.c
+++ b/drivers/base/regmap/regcache-rbtree.c
@@ -16,7 +16,7 @@
 
 static int regcache_rbtree_write(struct regmap *map, unsigned int reg,
 				 unsigned int value);
-static int regcache_rbtree_exit(struct regmap *map);
+static void regcache_rbtree_exit(struct regmap *map);
 
 struct regcache_rbtree_node {
 	/* block of adjacent registers */
@@ -196,7 +196,7 @@ static int regcache_rbtree_init(struct regmap *map)
 	return 0;
 }
 
-static int regcache_rbtree_exit(struct regmap *map)
+static void regcache_rbtree_exit(struct regmap *map)
 {
 	struct rb_node *next;
 	struct regcache_rbtree_ctx *rbtree_ctx;
@@ -205,7 +205,7 @@ static int regcache_rbtree_exit(struct regmap *map)
 	/* if we've already been called then just return */
 	rbtree_ctx = map->cache;
 	if (!rbtree_ctx)
-		return 0;
+		return;
 
 	/* free up the rbtree */
 	next = rb_first(&rbtree_ctx->root);
@@ -221,8 +221,6 @@ static int regcache_rbtree_exit(struct regmap *map)
 	/* release the resources */
 	kfree(map->cache);
 	map->cache = NULL;
-
-	return 0;
 }
 
 static int regcache_rbtree_populate(struct regmap *map)
diff --git a/drivers/base/regmap/regcache.c b/drivers/base/regmap/regcache.c
index aa8f2efed7798..96cdae25b9c4b 100644
--- a/drivers/base/regmap/regcache.c
+++ b/drivers/base/regmap/regcache.c
@@ -401,7 +401,8 @@ static int rbtree_all(const void *key, const struct rb_node *node)
  */
 int regcache_sync(struct regmap *map)
 {
-	int ret = 0;
+	int sync_ret = 0;
+	int selector_ret = 0;
 	unsigned int i;
 	const char *name;
 	bool bypass;
@@ -426,21 +427,21 @@ int regcache_sync(struct regmap *map)
 	/* Apply any patch first */
 	map->cache_bypass = true;
 	for (i = 0; i < map->patch_regs; i++) {
-		ret = _regmap_write(map, map->patch[i].reg, map->patch[i].def);
-		if (ret != 0) {
+		sync_ret = _regmap_write(map, map->patch[i].reg, map->patch[i].def);
+		if (sync_ret != 0) {
 			dev_err(map->dev, "Failed to write %x = %x: %d\n",
-				map->patch[i].reg, map->patch[i].def, ret);
+				map->patch[i].reg, map->patch[i].def, sync_ret);
 			goto out;
 		}
 	}
 	map->cache_bypass = false;
 
 	if (map->cache_ops->sync)
-		ret = map->cache_ops->sync(map, 0, map->max_register);
+		sync_ret = map->cache_ops->sync(map, 0, map->max_register);
 	else
-		ret = regcache_default_sync(map, 0, map->max_register);
+		sync_ret = regcache_default_sync(map, 0, map->max_register);
 
-	if (ret == 0)
+	if (sync_ret == 0)
 		map->cache_dirty = false;
 
 out:
@@ -462,10 +463,11 @@ int regcache_sync(struct regmap *map)
 		if (regcache_read(map, this->selector_reg, &i) != 0)
 			continue;
 
-		ret = _regmap_write(map, this->selector_reg, i);
-		if (ret != 0) {
+		selector_ret = _regmap_write(map, this->selector_reg, i);
+		if (selector_ret != 0) {
+			map->cache_dirty = true;
 			dev_err(map->dev, "Failed to write %x = %x: %d\n",
-				this->selector_reg, i, ret);
+				this->selector_reg, i, selector_ret);
 			break;
 		}
 	}
@@ -476,7 +478,7 @@ int regcache_sync(struct regmap *map)
 
 	trace_regcache_sync(map, name, "stop");
 
-	return ret;
+	return sync_ret ? sync_ret : selector_ret;
 }
 EXPORT_SYMBOL_GPL(regcache_sync);
 
diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c
index 07234d415b512..99b55b1053eed 100644
--- a/drivers/base/regmap/regmap-irq.c
+++ b/drivers/base/regmap/regmap-irq.c
@@ -296,6 +296,26 @@ static int regmap_irq_set_wake(struct irq_data *data, unsigned int on)
 	return 0;
 }
 
+static int regmap_irq_reqres(struct irq_data *data)
+{
+	struct regmap_irq_chip_data *d = irq_data_get_irq_chip_data(data);
+	irq_hw_number_t hwirq = irqd_to_hwirq(data);
+
+	if (d->chip->irq_reqres)
+		return d->chip->irq_reqres(d->chip->irq_drv_data, hwirq);
+
+	return 0;
+}
+
+static void regmap_irq_relres(struct irq_data *data)
+{
+	struct regmap_irq_chip_data *d = irq_data_get_irq_chip_data(data);
+	irq_hw_number_t hwirq = irqd_to_hwirq(data);
+
+	if (d->chip->irq_relres)
+		d->chip->irq_relres(d->chip->irq_drv_data, hwirq);
+}
+
 static const struct irq_chip regmap_irq_chip = {
 	.irq_bus_lock		= regmap_irq_lock,
 	.irq_bus_sync_unlock	= regmap_irq_sync_unlock,
@@ -303,6 +323,8 @@ static const struct irq_chip regmap_irq_chip = {
 	.irq_enable		= regmap_irq_enable,
 	.irq_set_type		= regmap_irq_set_type,
 	.irq_set_wake		= regmap_irq_set_wake,
+	.irq_request_resources  = regmap_irq_reqres,
+	.irq_release_resources  = regmap_irq_relres,
 };
 
 static inline int read_sub_irq_data(struct regmap_irq_chip_data *data,
diff --git a/drivers/base/swnode.c b/drivers/base/swnode.c
index 869228a65cb36..1f2315858cc3e 100644
--- a/drivers/base/swnode.c
+++ b/drivers/base/swnode.c
@@ -699,6 +699,62 @@ software_node_graph_parse_endpoint(const struct fwnode_handle *fwnode,
 	return 0;
 }
 
+static int software_node_add_links(struct fwnode_handle *fwnode)
+{
+	const struct software_node_ref_args *ref, *ref_array;
+	struct swnode *swnode = to_swnode(fwnode);
+	const struct property_entry *prop;
+	struct fwnode_handle *refnode;
+	unsigned int count;
+
+	if (!swnode || !swnode->node->properties)
+		return 0;
+
+	/*
+	 * Unlike Device Tree, where phandles appear in many non-supplier
+	 * contexts and a curated allowlist is required, a software node only
+	 * carries a DEV_PROP_REF property when the author explicitly describes
+	 * a reference to another node. Every such reference is therefore an
+	 * intentional supplier dependency, so we create fwnode links for all
+	 * of them.
+	 */
+	for (prop = swnode->node->properties; prop->name; prop++) {
+		if (prop->type != DEV_PROP_REF || prop->is_inline)
+			continue;
+
+		/*
+		 * TODO: Graph "remote-endpoint" references go both ways
+		 * between endpoint child nodes and would create endpoint
+		 * cycles. Let's leave it out for now until we have potential
+		 * users.
+		 */
+		if (!strcmp(prop->name, "remote-endpoint"))
+			continue;
+
+		ref_array = prop->pointer;
+		count = prop->length / sizeof(*ref_array);
+
+		for (unsigned int i = 0; i < count; i++) {
+			ref = &ref_array[i];
+
+			if (ref->swnode)
+				refnode = software_node_fwnode(ref->swnode);
+			else if (ref->fwnode)
+				refnode = ref->fwnode;
+			else
+				continue;
+
+			/* Supplier not registered yet, or self-reference. */
+			if (!refnode || refnode == &swnode->fwnode)
+				continue;
+
+			fwnode_link_add(&swnode->fwnode, refnode, 0);
+		}
+	}
+
+	return 0;
+}
+
 static const struct fwnode_operations software_node_ops = {
 	.get = software_node_get,
 	.put = software_node_put,
@@ -716,6 +772,7 @@ static const struct fwnode_operations software_node_ops = {
 	.graph_get_remote_endpoint = software_node_graph_get_remote_endpoint,
 	.graph_get_port_parent = software_node_graph_get_port_parent,
 	.graph_parse_endpoint = software_node_graph_parse_endpoint,
+	.add_links = software_node_add_links,
 };
 
 /* -------------------------------------------------------------------------- */
@@ -787,6 +844,8 @@ static void software_node_release(struct kobject *kobj)
 {
 	struct swnode *swnode = kobj_to_swnode(kobj);
 
+	fwnode_links_purge(&swnode->fwnode);
+
 	if (swnode->parent) {
 		ida_free(&swnode->parent->child_ids, swnode->id);
 		list_del(&swnode->entry);
@@ -1105,6 +1164,17 @@ void software_node_notify(struct device *dev)
 	if (!swnode)
 		return;
 
+	/*
+	 * When the software node is the device's secondary firmware node,
+	 * the core only records the owning device on the primary fwnode
+	 * (see device_add()). fw_devlink resolves a supplier device through
+	 * fwnode->dev, so without this a consumer referencing the software
+	 * node could never find the supplier device and would defer forever.
+	 * Make fwnode.dev point to its owner in that case.
+	 */
+	if (!device_match_fwnode(dev, &swnode->fwnode) && !swnode->fwnode.dev)
+		swnode->fwnode.dev = dev;
+
 	swnode_get(swnode);
 	ret = sysfs_create_link(&dev->kobj, &swnode->kobj, "software_node");
 	if (ret)
@@ -1127,6 +1197,15 @@ void software_node_notify_remove(struct device *dev)
 
 	sysfs_remove_link(&swnode->kobj, dev_name(dev));
 	sysfs_remove_link(&dev->kobj, "software_node");
+
+	/*
+	 * Drop the device pointer mirrored onto a secondary software node in
+	 * software_node_notify(). For a primary software node the core owns
+	 * fwnode->dev and clears it in device_del().
+	 */
+	if (!device_match_fwnode(dev, &swnode->fwnode) && swnode->fwnode.dev == dev)
+		swnode->fwnode.dev = NULL;
+
 	swnode_put(swnode);
 
 	if (swnode->managed) {
diff --git a/drivers/base/test/Kconfig b/drivers/base/test/Kconfig
index 2756870615cca..1ecf0791241a1 100644
--- a/drivers/base/test/Kconfig
+++ b/drivers/base/test/Kconfig
@@ -18,3 +18,8 @@ config DRIVER_PE_KUNIT_TEST
 	tristate "KUnit Tests for property entry API" if !KUNIT_ALL_TESTS
 	depends on KUNIT
 	default KUNIT_ALL_TESTS
+
+config DRIVER_SWNODE_KUNIT_TEST
+	tristate "KUnit Tests for software node fw_devlink links" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	default KUNIT_ALL_TESTS
diff --git a/drivers/base/test/Makefile b/drivers/base/test/Makefile
index e321dfc7e9226..9ced7bbd569fc 100644
--- a/drivers/base/test/Makefile
+++ b/drivers/base/test/Makefile
@@ -6,3 +6,5 @@ obj-$(CONFIG_DM_KUNIT_TEST)	+= platform-device-test.o
 
 obj-$(CONFIG_DRIVER_PE_KUNIT_TEST) += property-entry-test.o
 CFLAGS_property-entry-test.o += $(DISABLE_STRUCTLEAK_PLUGIN)
+
+obj-$(CONFIG_DRIVER_SWNODE_KUNIT_TEST) += swnode-devlink-test.o
diff --git a/drivers/base/test/swnode-devlink-test.c b/drivers/base/test/swnode-devlink-test.c
new file mode 100644
index 0000000000000..f53549e926118
--- /dev/null
+++ b/drivers/base/test/swnode-devlink-test.c
@@ -0,0 +1,337 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) Qualcomm Technologies, Inc. and/or its subsidiaries
+ */
+
+#include <linux/array_size.h>
+#include <linux/device.h>
+#include <linux/fwnode.h>
+#include <linux/list.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/time.h>
+#include <linux/types.h>
+#include <linux/wait.h>
+
+#include <kunit/fwnode.h>
+#include <kunit/platform_device.h>
+#include <kunit/test.h>
+
+static int swnode_count_suppliers(struct fwnode_handle *fwnode)
+{
+	struct fwnode_link *link;
+	unsigned int count = 0;
+
+	/*
+	 * The suppliers and consumers lists should typically only be accessed
+	 * with the fwnode_link_lock taken but it's private to the driver core.
+	 *
+	 * These are tests and at this point nobody should be modifying them so
+	 * let's just access the list.
+	 */
+	list_for_each_entry(link, &fwnode->suppliers, c_hook)
+		count++;
+
+	return count;
+}
+
+/* True if a supplier link con->sup exists, checked from both list ends. */
+static bool swnode_has_link(struct fwnode_handle *consumer,
+			    struct fwnode_handle *supplier)
+{
+	bool from_con = false, from_sup = false;
+	struct fwnode_link *link;
+
+	list_for_each_entry(link, &consumer->suppliers, c_hook) {
+		if (link->supplier == supplier && link->consumer == consumer)
+			from_con = true;
+	}
+
+	list_for_each_entry(link, &supplier->consumers, s_hook) {
+		if (link->supplier == supplier && link->consumer == consumer)
+			from_sup = true;
+	}
+
+	return from_con && from_sup;
+}
+
+/* A single reference creates exactly one supplier link, on both list ends. */
+static void swnode_devlink_test_single_ref(struct kunit *test)
+{
+	static const struct software_node supp_swnode = {
+		.name = "swnode-devlink-test-supplier",
+	};
+
+	struct fwnode_handle *cons_fwnode, *supp_fwnode;
+	int ret;
+
+	const struct property_entry props[] = {
+		PROPERTY_ENTRY_REF("supplier", &supp_swnode),
+		{ }
+	};
+
+	supp_fwnode = kunit_software_node_register(test, &supp_swnode);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, supp_fwnode);
+
+	cons_fwnode = kunit_fwnode_create_software_node(test, props, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cons_fwnode);
+
+	ret = fwnode_call_int_op(cons_fwnode, add_links);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+
+	KUNIT_EXPECT_EQ(test, swnode_count_suppliers(cons_fwnode), 1);
+	KUNIT_EXPECT_TRUE(test, swnode_has_link(cons_fwnode, supp_fwnode));
+}
+
+/* Multiple distinct references create multiple supplier links. */
+static void swnode_devlink_test_multiple_refs(struct kunit *test)
+{
+	static const struct software_node supp1_swnode = {
+		.name = "swnode-devlink-test-supplier-1",
+	};
+	static const struct software_node supp2_swnode = {
+		.name = "swnode-devlink-test-supplier-2",
+	};
+	static const struct software_node *supp_nodes[] = {
+		&supp1_swnode, &supp2_swnode, NULL
+	};
+
+	const struct property_entry props[] = {
+		PROPERTY_ENTRY_REF("foo", &supp1_swnode),
+		PROPERTY_ENTRY_REF("bar", &supp2_swnode),
+		{ }
+	};
+
+	struct fwnode_handle *fwnode;
+	int ret;
+
+	ret = kunit_software_node_register_node_group(test, supp_nodes);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	fwnode = kunit_fwnode_create_software_node(test, props, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fwnode);
+
+	ret = fwnode_call_int_op(fwnode, add_links);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+
+	KUNIT_EXPECT_EQ(test, swnode_count_suppliers(fwnode), 2);
+	KUNIT_EXPECT_TRUE(test, swnode_has_link(fwnode, software_node_fwnode(&supp1_swnode)));
+	KUNIT_EXPECT_TRUE(test, swnode_has_link(fwnode, software_node_fwnode(&supp2_swnode)));
+}
+
+/* A reference to an unregistered node creates no link (graceful skip). */
+static void swnode_devlink_test_unregistered_ref(struct kunit *test)
+{
+	static const struct software_node supp_swnode = {
+		.name = "swnode-devlink-test-supplier",
+	};
+
+	const struct property_entry props[] = {
+		PROPERTY_ENTRY_REF("supplier", &supp_swnode),
+		{ }
+	};
+
+	struct fwnode_handle *fwnode;
+	int ret;
+
+	fwnode = kunit_fwnode_create_software_node(test, props, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fwnode);
+
+	ret = fwnode_call_int_op(fwnode, add_links);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, swnode_count_suppliers(fwnode), 0);
+}
+
+/* Graph "remote-endpoint" references are excluded. */
+static void swnode_devlink_test_remote_endpoint_excluded(struct kunit *test)
+{
+	static const struct software_node ep_swnode = {
+		.name = "swnode-devlink-test-end-point"
+	};
+
+	const struct property_entry props[] = {
+		PROPERTY_ENTRY_REF("remote-endpoint", &ep_swnode),
+		{ }
+	};
+
+	struct fwnode_handle *cons_fwnode, *supp_fwnode;
+	int ret;
+
+	supp_fwnode = kunit_software_node_register(test, &ep_swnode);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, supp_fwnode);
+
+	cons_fwnode = kunit_fwnode_create_software_node(test, props, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cons_fwnode);
+
+	ret = fwnode_call_int_op(cons_fwnode, add_links);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, swnode_count_suppliers(cons_fwnode), 0);
+}
+
+/* A reference array creates one link per registered element. */
+static void swnode_devlink_test_ref_array(struct kunit *test)
+{
+	static const struct software_node supp1_swnode = {
+		.name = "swnode-devlink-test-supplier-1",
+	};
+	static const struct software_node supp2_swnode = {
+		.name = "swnode-devlink-test-supplier-2",
+	};
+	static const struct software_node *supp_nodes[] = {
+		&supp1_swnode, &supp2_swnode, NULL
+	};
+	static const struct software_node_ref_args refs[] = {
+		SOFTWARE_NODE_REFERENCE(&supp1_swnode),
+		SOFTWARE_NODE_REFERENCE(&supp2_swnode, 4, 2),
+	};
+
+	const struct property_entry props[] = {
+		PROPERTY_ENTRY_REF_ARRAY("suppliers", refs),
+		{ }
+	};
+
+	struct fwnode_handle *fwnode;
+	int ret;
+
+	ret = kunit_software_node_register_node_group(test, supp_nodes);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	fwnode = kunit_fwnode_create_software_node(test, props, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fwnode);
+
+	ret = fwnode_call_int_op(fwnode, add_links);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+
+	KUNIT_EXPECT_EQ(test, swnode_count_suppliers(fwnode), 2);
+	KUNIT_EXPECT_TRUE(test, swnode_has_link(fwnode, software_node_fwnode(&supp1_swnode)));
+	KUNIT_EXPECT_TRUE(test, swnode_has_link(fwnode, software_node_fwnode(&supp2_swnode)));
+}
+
+/*
+ * End-to-end test: fw_devlink must defer a consumer's probe until its
+ * supplier has probed.
+ *
+ * The reference created by software_node_add_links() is only useful if the
+ * driver core promotes it to a real device_link and uses it to order probing.
+ * This test drives actual probing through the platform bus and asserts the
+ * supplier binds before the consumer.
+ */
+
+#define SWNODE_DEVLINK_TEST_SUPPLIER	"swnode-link-supplier"
+#define SWNODE_DEVLINK_TEST_CONSUMER	"swnode-link-consumer"
+#define SWNODE_DEVLINK_TEST_TIMEOUT_MS	(2 * MSEC_PER_SEC)
+
+struct swnode_test_probe_order {
+	/* Names in the order their drivers' .probe ran. */
+	const char *probed[2];
+	unsigned int count;
+	wait_queue_head_t wq;
+};
+
+static int swnode_test_record_probe(struct platform_device *pdev)
+{
+	struct swnode_test_probe_order *order = platform_get_drvdata(pdev);
+
+	if (order && order->count < ARRAY_SIZE(order->probed)) {
+		order->probed[order->count++] = dev_name(&pdev->dev);
+		wake_up_interruptible(&order->wq);
+	}
+
+	return 0;
+}
+
+static struct platform_driver swnode_test_supplier_driver = {
+	.probe = swnode_test_record_probe,
+	.driver = {
+		.name = SWNODE_DEVLINK_TEST_SUPPLIER,
+	},
+};
+
+static struct platform_driver swnode_test_consumer_driver = {
+	.probe = swnode_test_record_probe,
+	.driver = {
+		.name = SWNODE_DEVLINK_TEST_CONSUMER,
+	},
+};
+
+static void swnode_devlink_test_probe_order(struct kunit *test)
+{
+	static const struct software_node supplier_swnode = {
+		.name = "swnode-devlink-test-supplier",
+	};
+
+	const struct property_entry consumer_props[] = {
+		PROPERTY_ENTRY_REF("supplier-ref", &supplier_swnode),
+		{ }
+	};
+
+	struct platform_device *supplier, *consumer;
+	struct swnode_test_probe_order *order;
+	struct fwnode_handle *fwnode;
+	int ret;
+
+	order = kunit_kzalloc(test, sizeof(*order), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, order);
+	init_waitqueue_head(&order->wq);
+
+	fwnode = kunit_software_node_register(test, &supplier_swnode);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fwnode);
+
+	ret = kunit_platform_driver_register(test, &swnode_test_supplier_driver);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	ret = kunit_platform_driver_register(test, &swnode_test_consumer_driver);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	supplier = kunit_platform_device_alloc(test, SWNODE_DEVLINK_TEST_SUPPLIER,
+					       PLATFORM_DEVID_NONE);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, supplier);
+	consumer = kunit_platform_device_alloc(test, SWNODE_DEVLINK_TEST_CONSUMER,
+					       PLATFORM_DEVID_NONE);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, consumer);
+
+	platform_set_drvdata(supplier, order);
+	platform_set_drvdata(consumer, order);
+
+	ret = kunit_device_add_software_node(test, &supplier->dev, &supplier_swnode);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	ret = device_create_managed_software_node(&consumer->dev,
+						  consumer_props, NULL);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	ret = kunit_platform_device_add(test, consumer);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	ret = kunit_platform_device_add(test, supplier);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	ret = wait_event_interruptible_timeout(order->wq,
+					       order->count == 2,
+					       msecs_to_jiffies(SWNODE_DEVLINK_TEST_TIMEOUT_MS));
+	KUNIT_ASSERT_GT(test, ret, 0);
+
+	KUNIT_EXPECT_STREQ(test, order->probed[0], SWNODE_DEVLINK_TEST_SUPPLIER);
+	KUNIT_EXPECT_STREQ(test, order->probed[1], SWNODE_DEVLINK_TEST_CONSUMER);
+
+	/* Tear down the consumer (and its device link) before the supplier. */
+	kunit_platform_device_unregister(test, consumer);
+}
+
+static struct kunit_case swnode_test_cases[] = {
+	KUNIT_CASE(swnode_devlink_test_single_ref),
+	KUNIT_CASE(swnode_devlink_test_multiple_refs),
+	KUNIT_CASE(swnode_devlink_test_unregistered_ref),
+	KUNIT_CASE(swnode_devlink_test_remote_endpoint_excluded),
+	KUNIT_CASE(swnode_devlink_test_ref_array),
+	KUNIT_CASE(swnode_devlink_test_probe_order),
+	{ }
+};
+
+static struct kunit_suite swnode_test_suite = {
+	.name = "software-node-links",
+	.test_cases = swnode_test_cases,
+};
+
+kunit_test_suite(swnode_test_suite);
+
+MODULE_DESCRIPTION("Test module for software node fw_devlink support");
+MODULE_AUTHOR("Bartosz Golaszewski <bartosz.golaszewski@oss.qualcomm.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/bcma/driver_gpio.c b/drivers/bcma/driver_gpio.c
index 658c7e2ac8bf6..ea45222f2fa0f 100644
--- a/drivers/bcma/driver_gpio.c
+++ b/drivers/bcma/driver_gpio.c
@@ -19,6 +19,11 @@
 
 #define BCMA_GPIO_MAX_PINS	32
 
+const struct software_node bcma_gpio_swnode = {
+	.name = "bcma-gpio",
+};
+EXPORT_SYMBOL_GPL(bcma_gpio_swnode);
+
 static int bcma_gpio_get_value(struct gpio_chip *chip, unsigned gpio)
 {
 	struct bcma_drv_cc *cc = gpiochip_get_data(chip);
@@ -190,7 +195,20 @@ int bcma_gpio_init(struct bcma_drv_cc *cc)
 	chip->direction_input	= bcma_gpio_direction_input;
 	chip->direction_output	= bcma_gpio_direction_output;
 	chip->parent		= bus->dev;
-	chip->fwnode		= dev_fwnode(&cc->core->dev);
+
+	/*
+	 * Register software node only for the host SoC bus, unless there is
+	 * already a firmware node assigned. There is only one SoC instance
+	 * in the system, so there are no concerns with registration conflicts.
+	 */
+	if (bus->hosttype == BCMA_HOSTTYPE_SOC && !dev_fwnode(&cc->core->dev)) {
+		err = software_node_register(&bcma_gpio_swnode);
+		if (err)
+			return err;
+		chip->fwnode = software_node_fwnode(&bcma_gpio_swnode);
+	} else {
+		chip->fwnode = dev_fwnode(&cc->core->dev);
+	}
 
 	switch (bus->chipinfo.id) {
 	case BCMA_CHIP_ID_BCM4707:
@@ -219,20 +237,33 @@ int bcma_gpio_init(struct bcma_drv_cc *cc)
 
 	err = bcma_gpio_irq_init(cc);
 	if (err)
-		return err;
+		goto err_unregister_swnode;
 
 	err = gpiochip_add_data(chip, cc);
-	if (err) {
-		bcma_gpio_irq_exit(cc);
-		return err;
-	}
+	if (err)
+		goto err_irq_exit;
 
 	return 0;
+
+err_irq_exit:
+	bcma_gpio_irq_exit(cc);
+err_unregister_swnode:
+	if (bus->hosttype == BCMA_HOSTTYPE_SOC &&
+	    chip->fwnode && is_software_node(chip->fwnode)) {
+		software_node_unregister(&bcma_gpio_swnode);
+		chip->fwnode = NULL;
+	}
+	return err;
 }
 
 int bcma_gpio_unregister(struct bcma_drv_cc *cc)
 {
 	bcma_gpio_irq_exit(cc);
 	gpiochip_remove(&cc->gpio);
+	if (cc->core->bus->hosttype == BCMA_HOSTTYPE_SOC &&
+	    cc->gpio.fwnode && is_software_node(cc->gpio.fwnode)) {
+		software_node_unregister(&bcma_gpio_swnode);
+		cc->gpio.fwnode = NULL;
+	}
 	return 0;
 }
diff --git a/drivers/block/loop.c b/drivers/block/loop.c
index 310de0463beb1..1faecef330092 100644
--- a/drivers/block/loop.c
+++ b/drivers/block/loop.c
@@ -1113,6 +1113,7 @@ static void __loop_clr_fd(struct loop_device *lo)
 	struct queue_limits lim;
 	struct file *filp;
 	gfp_t gfp = lo->old_gfp_mask;
+	int err;
 
 	spin_lock_irq(&lo->lo_lock);
 	filp = lo->lo_backing_file;
@@ -1146,26 +1147,21 @@ static void __loop_clr_fd(struct loop_device *lo)
 
 	disk_force_media_change(lo->lo_disk);
 
-	if (lo->lo_flags & LO_FLAGS_PARTSCAN) {
-		int err;
-
-		/*
-		 * open_mutex has been held already in release path, so don't
-		 * acquire it if this function is called in such case.
-		 *
-		 * If the reread partition isn't from release path, lo_refcnt
-		 * must be at least one and it can only become zero when the
-		 * current holder is released.
-		 */
-		err = bdev_disk_changed(lo->lo_disk, false);
-		if (err)
-			pr_warn("%s: partition scan of loop%d failed (rc=%d)\n",
-				__func__, lo->lo_number, err);
-		/* Device is gone, no point in returning error */
-	}
+	/*
+	 * Remove all partitions, including partitions added manually with
+	 * BLKPG, which may exist even if LO_FLAGS_PARTSCAN is not set.
+	 *
+	 * open_mutex has been held already in release path, so don't acquire
+	 * it here.
+	 */
+	err = bdev_disk_changed(lo->lo_disk, false);
+	if (err)
+		pr_warn("%s: partition scan of loop%d failed (rc=%d)\n",
+			__func__, lo->lo_number, err);
+	/* Device is gone, no point in returning error */
 
 	/*
-	 * lo->lo_state is set to Lo_unbound here after above partscan has
+	 * lo->lo_state is set to Lo_unbound here after removing partitions has
 	 * finished. There cannot be anybody else entering __loop_clr_fd() as
 	 * Lo_rundown state protects us from all the other places trying to
 	 * change the 'lo' device.
diff --git a/drivers/block/rnbd/rnbd-srv.c b/drivers/block/rnbd/rnbd-srv.c
index 10e8c438bb439..79c9a5fb418f2 100644
--- a/drivers/block/rnbd/rnbd-srv.c
+++ b/drivers/block/rnbd/rnbd-srv.c
@@ -11,6 +11,7 @@
 
 #include <linux/module.h>
 #include <linux/blkdev.h>
+#include <linux/fs_struct.h>
 
 #include "rnbd-srv.h"
 #include "rnbd-srv-trace.h"
@@ -734,7 +735,8 @@ static int process_msg_open(struct rnbd_srv_session *srv_sess,
 		goto reject;
 	}
 
-	bdev_file = bdev_file_open_by_path(full_path, open_flags, NULL, NULL);
+	scoped_with_init_fs()
+		bdev_file = bdev_file_open_by_path(full_path, open_flags, NULL, NULL);
 	if (IS_ERR(bdev_file)) {
 		ret = PTR_ERR(bdev_file);
 		pr_err("Opening device '%s' on session %s failed, failed to open the block device, err: %pe\n",
diff --git a/drivers/bluetooth/Makefile b/drivers/bluetooth/Makefile
index bafc26250b636..e6b1c1180d1dc 100644
--- a/drivers/bluetooth/Makefile
+++ b/drivers/bluetooth/Makefile
@@ -50,3 +50,5 @@ hci_uart-$(CONFIG_BT_HCIUART_AG6XX)	+= hci_ag6xx.o
 hci_uart-$(CONFIG_BT_HCIUART_MRVL)	+= hci_mrvl.o
 hci_uart-$(CONFIG_BT_HCIUART_AML)	+= hci_aml.o
 hci_uart-objs				:= $(hci_uart-y)
+
+CONTEXT_ANALYSIS := y
diff --git a/drivers/bluetooth/btintel_pcie.c b/drivers/bluetooth/btintel_pcie.c
index 2b7231be5973d..2e28847263ab0 100644
--- a/drivers/bluetooth/btintel_pcie.c
+++ b/drivers/bluetooth/btintel_pcie.c
@@ -1446,72 +1446,134 @@ static int btintel_pcie_dump_fwtrigger_event(struct btintel_pcie_data *data)
 	return err;
 }
 
+/* Queue a coredump dump_traces() pass.
+ *
+ * Returns true if a new coredump was queued, false if one was already
+ * in-flight (the BTINTEL_PCIE_COREDUMP_INPROGRESS bit serves as the
+ * single-writer guard for the @coredump_work item) or the workqueue is
+ * disabled (reset / remove in progress).
+ *
+ * Always queue this AFTER any companion event-reader work (hwexp /
+ * fwtrigger) so that, on the ordered @dump_workqueue, the event reader
+ * runs first and populates dmp_hdr.event_type / event_id before
+ * dump_traces consumes them.
+ */
+static bool btintel_pcie_queue_coredump(struct btintel_pcie_data *data,
+					u16 trigger_reason)
+{
+	if (test_and_set_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags))
+		return false;
+
+	data->dmp_hdr.trigger_reason = trigger_reason;
+
+	if (queue_work(data->dump_workqueue, &data->coredump_work))
+		return true;
+
+	/* Workqueue is disabled (reset/remove drained it). Release the
+	 * guard so a later trigger, after re-probe, can succeed.
+	 */
+	clear_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags);
+	return false;
+}
+
 static void btintel_pcie_msix_fw_trigger_handler(struct btintel_pcie_data *data)
 {
 	bt_dev_dbg(data->hdev, "Received firmware smart trigger cause");
 
-	if (test_and_set_bit(BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS, &data->flags))
+	/* Per-work guard: deduplicate concurrent FW-trigger interrupts.
+	 * Cleared at the tail of btintel_pcie_fwtrigger_worker().
+	 */
+	if (test_and_set_bit(BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS,
+			     &data->flags))
 		return;
 
-	/* Trigger device core dump when there is FW assert */
-	if (!test_and_set_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags))
-		data->dmp_hdr.trigger_reason = BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT;
+	if (!queue_work(data->dump_workqueue, &data->fwtrigger_work)) {
+		clear_bit(BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS, &data->flags);
+		return;
+	}
 
-	queue_work(data->coredump_workqueue, &data->coredump_work);
+	/* Queue coredump after the fwtrigger event reader so dmp_hdr.event_*
+	 * is populated before dump_traces consumes it.
+	 */
+	btintel_pcie_queue_coredump(data, BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT);
 }
 
 static void btintel_pcie_msix_hw_exp_handler(struct btintel_pcie_data *data)
 {
 	bt_dev_err(data->hdev, "Received hw exception interrupt");
 
+	/* CORE_HALTED is the single-writer guard for this handler. It is
+	 * set once on first HW exception and cleared only by re-probe
+	 * (data is reallocated), so it also serializes hwexp_work
+	 * scheduling without needing a separate bit.
+	 */
 	if (test_and_set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags))
 		return;
 
-	if (test_and_set_bit(BTINTEL_PCIE_HWEXP_INPROGRESS, &data->flags))
-		return;
-
-	/* Trigger device core dump when there is HW  exception */
-	if (!test_and_set_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags))
-		data->dmp_hdr.trigger_reason = BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT;
+	/* Queue companion coredump first so it is appended after hwexp_work
+	 * on the ordered @dump_workqueue (preserves the original
+	 * coredump-then-hwexp ordering).
+	 */
+	btintel_pcie_queue_coredump(data, BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT);
 
-	queue_work(data->coredump_workqueue, &data->coredump_work);
+	queue_work(data->dump_workqueue, &data->hwexp_work);
 }
 
 static void btintel_pcie_coredump_worker(struct work_struct *work)
 {
 	struct btintel_pcie_data *data = container_of(work,
 					struct btintel_pcie_data, coredump_work);
-	int err;
 
 	/* hdev is NULL until setup_hdev() succeeds, and is cleared on
 	 * teardown after disable_work_sync() drains us; bail in that case.
 	 */
+	if (!data->hdev)
+		goto out;
+
+	btintel_pcie_dump_traces(data->hdev);
+out:
+	/* Release guard last so a new trigger can run only after this
+	 * pass has fully completed (including dev_coredumpv()).
+	 */
+	clear_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags);
+}
+
+static void btintel_pcie_hwexp_worker(struct work_struct *work)
+{
+	struct btintel_pcie_data *data = container_of(work,
+					struct btintel_pcie_data, hwexp_work);
+
 	if (!data->hdev)
 		return;
 
-	if (test_bit(BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS, &data->flags)) {
-		err = btintel_pcie_dump_fwtrigger_event(data);
-		if (err)
-			bt_dev_warn(data->hdev, "failed to log fwtrigger event");
-		clear_bit(BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS, &data->flags);
-	}
+	/* Unlike usb products, controller will not send hardware exception
+	 * event on exception. Instead controller writes the hardware event
+	 * to device memory along with optional debug events, raises MSIX
+	 * and halts. Driver shall read the exception event from device
+	 * memory and passes it to the stack for further processing.
+	 *
+	 * Re-entry is gated by BTINTEL_PCIE_CORE_HALTED in the IRQ
+	 * handler, which is only cleared by re-probe; no per-work bit
+	 * is needed here.
+	 */
+	btintel_pcie_read_hwexp(data);
+}
 
-	if (test_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags)) {
-		btintel_pcie_dump_traces(data->hdev);
-		clear_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags);
-	}
+static void btintel_pcie_fwtrigger_worker(struct work_struct *work)
+{
+	struct btintel_pcie_data *data = container_of(work,
+					struct btintel_pcie_data, fwtrigger_work);
+	int err;
 
-	if (test_bit(BTINTEL_PCIE_HWEXP_INPROGRESS, &data->flags)) {
-		/* Unlike usb products, controller will not send hardware
-		 * exception event on exception. Instead controller writes the
-		 * hardware event to device memory along with optional debug
-		 * events, raises MSIX and halts. Driver shall read the
-		 * exception event from device memory and passes it stack for
-		 * further processing.
-		 */
-		btintel_pcie_read_hwexp(data);
-		clear_bit(BTINTEL_PCIE_HWEXP_INPROGRESS, &data->flags);
-	}
+	if (!data->hdev)
+		goto out;
+
+	err = btintel_pcie_dump_fwtrigger_event(data);
+	if (err)
+		bt_dev_warn(data->hdev, "failed to log fwtrigger event");
+out:
+	/* Release guard last; matches set in fw_trigger handler. */
+	clear_bit(BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS, &data->flags);
 }
 
 static void btintel_pcie_rx_work(struct work_struct *work)
@@ -2488,8 +2550,6 @@ static void btintel_pcie_inc_recovery_count(struct pci_dev *pdev,
 	}
 }
 
-static void btintel_pcie_reset(struct hci_dev *hdev);
-
 static int btintel_pcie_acpi_reset_method(struct btintel_pcie_data *data)
 {
 	union acpi_object *obj, argv4;
@@ -2650,20 +2710,22 @@ static void btintel_pcie_reset_work(struct work_struct *wk)
 	btintel_pcie_synchronize_irqs(data);
 
 	flush_work(&data->rx_work);
-	/* Drain any in-flight coredump and block new ones across reset.
-	 * Safe from self-deadlock: coredump_work runs on a separate wq.
+	/* Drain any in-flight dump workers and block new ones across reset.
+	 * Safe from self-deadlock: they all run on a separate wq.
 	 */
 	disable_work_sync(&data->coredump_work);
+	disable_work_sync(&data->hwexp_work);
+	disable_work_sync(&data->fwtrigger_work);
 
 	bt_dev_dbg(data->hdev, "Release bluetooth interface");
 
 	/* Both reset paths follow the same contract: on success they
 	 * destroy 'data' via device_reprobe() (a fresh probe re-INIT_WORKs
-	 * the coredump_work with disable count 0), so enable_work() must
+	 * the dump workers with disable count 0), so enable_work() must
 	 * NOT be called on the success path. Only the FLR path can fail
 	 * with 'data' still alive, in which case we balance the
-	 * disable_work_sync() above so a later successful reset is not
-	 * permanently blocked.
+	 * disable_work_sync() calls above so a later successful reset is
+	 * not permanently blocked.
 	 *
 	 * pci_lock_rescan_remove() (held above) serializes against PCI
 	 * device addition/removal (hotplug), so no device can be added to
@@ -2674,64 +2736,97 @@ static void btintel_pcie_reset_work(struct work_struct *wk)
 		goto out;
 	}
 
-	if (btintel_pcie_perform_flr(data))
+	if (btintel_pcie_perform_flr(data)) {
 		enable_work(&data->coredump_work);
+		enable_work(&data->hwexp_work);
+		enable_work(&data->fwtrigger_work);
+	}
 
 out:
 	pci_dev_put(pdev);
 	pci_unlock_rescan_remove();
 }
 
-static void btintel_pcie_reset(struct hci_dev *hdev)
+/* Schedule a device reset of the requested type.
+ *
+ * BTINTEL_PCIE_RECOVERY_IN_PROGRESS serializes all reset requesters
+ * (sysfs reset attribute, hci_cmd_timeout(), hw_error, resume error
+ * path, etc.) so that:
+ *
+ *   - dev_data->reset_type is written by exactly one caller (the
+ *     thread that wins test_and_set_bit), eliminating the race where
+ *     a second hw_error could clobber an already-scheduled reset's
+ *     type;
+ *   - the write happens AFTER the bit is set, so reset_work observes
+ *     it through schedule_work()'s memory ordering;
+ *   - losers return without touching reset_type or scheduling the
+ *     work, so concurrent triggers are silently coalesced into the
+ *     in-flight one (whose recovery will reinitialize the device
+ *     regardless of the dropped trigger's variant).
+ *
+ * The bit is cleared only by .remove() / re-probe via fresh devm
+ * allocation, which is the intended one-shot semantics: a reset
+ * tears down and re-probes 'data', so there is no "in-flight"
+ * reset to follow up after device_reprobe() succeeds.
+ */
+static void btintel_pcie_request_reset(struct btintel_pcie_data *data,
+				       enum btintel_pcie_reset_type type)
 {
-	struct btintel_pcie_data *data;
-
-	data = hci_get_drvdata(hdev);
-
 	if (!test_bit(BTINTEL_PCIE_SETUP_DONE, &data->flags))
 		return;
 
 	if (test_and_set_bit(BTINTEL_PCIE_RECOVERY_IN_PROGRESS, &data->flags))
 		return;
 
+	data->reset_type = type;
+
 	pci_dev_get(data->pdev);
 	schedule_work(&data->reset_work);
 }
 
+static void btintel_pcie_hci_reset(struct hci_dev *hdev)
+{
+	struct btintel_pcie_data *data = hci_get_drvdata(hdev);
+
+	btintel_pcie_request_reset(data, BTINTEL_PCIE_IOSF_PRR_FLR);
+}
+
 static void btintel_pcie_hw_error(struct hci_dev *hdev, u8 code)
 {
-	struct btintel_pcie_dev_recovery *data;
+	struct btintel_pcie_dev_recovery *rec;
 	struct btintel_pcie_data *dev_data = hci_get_drvdata(hdev);
 	struct pci_dev *pdev = dev_data->pdev;
+	enum btintel_pcie_reset_type type;
 	time64_t retry_window;
 
+	if (test_bit(BTINTEL_PCIE_RECOVERY_IN_PROGRESS, &dev_data->flags))
+		return;
+
 	btintel_pcie_dump_debug_registers(hdev);
 
-	data = btintel_pcie_get_recovery(pdev, &hdev->dev);
-	if (!data)
+	rec = btintel_pcie_get_recovery(pdev, &hdev->dev);
+	if (!rec)
 		return;
 
-	if (code == 0x13)
-		dev_data->reset_type = BTINTEL_PCIE_IOSF_PRR_PLDR;
-	else
-		dev_data->reset_type = BTINTEL_PCIE_IOSF_PRR_FLR;
+	type = (code == 0x13) ? BTINTEL_PCIE_IOSF_PRR_PLDR
+			      : BTINTEL_PCIE_IOSF_PRR_FLR;
 
 	bt_dev_err(hdev, "Encountered exception err:0x%x triggering: %s", code,
-		   dev_data->reset_type == BTINTEL_PCIE_IOSF_PRR_PLDR ? "PLDR" : "FLR");
-	retry_window = ktime_get_boottime_seconds() - data->last_error;
+		   type == BTINTEL_PCIE_IOSF_PRR_PLDR ? "PLDR" : "FLR");
+	retry_window = ktime_get_boottime_seconds() - rec->last_error;
 
 	if (retry_window < BTINTEL_PCIE_RESET_WINDOW_SECS &&
-	    data->count >= BTINTEL_PCIE_FLR_MAX_RETRY) {
+	    rec->count >= BTINTEL_PCIE_FLR_MAX_RETRY) {
 		bt_dev_err(hdev, "Exhausted maximum: %d recovery attempts: %d",
-			   BTINTEL_PCIE_FLR_MAX_RETRY, data->count);
+			   BTINTEL_PCIE_FLR_MAX_RETRY, rec->count);
 		bt_dev_dbg(hdev, "Boot time: %lld seconds",
 			   ktime_get_boottime_seconds());
 		bt_dev_dbg(hdev, "last error at: %lld seconds",
-			   data->last_error);
+			   rec->last_error);
 		return;
 	}
 	btintel_pcie_inc_recovery_count(pdev, &hdev->dev);
-	btintel_pcie_reset(hdev);
+	btintel_pcie_request_reset(dev_data, type);
 }
 
 static bool btintel_pcie_wakeup(struct hci_dev *hdev)
@@ -2821,7 +2916,7 @@ static int btintel_pcie_setup_hdev(struct btintel_pcie_data *data)
 	hdev->hw_error = btintel_pcie_hw_error;
 	hdev->set_diag = btintel_set_diag;
 	hdev->set_bdaddr = btintel_set_bdaddr;
-	hdev->reset = btintel_pcie_reset;
+	hdev->reset = btintel_pcie_hci_reset;
 	hdev->wakeup = btintel_pcie_wakeup;
 	hdev->hci_drv = &btintel_pcie_hci_drv;
 
@@ -2869,8 +2964,8 @@ static int btintel_pcie_probe(struct pci_dev *pdev,
 	if (!data->workqueue)
 		return -ENOMEM;
 
-	data->coredump_workqueue = alloc_ordered_workqueue(KBUILD_MODNAME "_cd", 0);
-	if (!data->coredump_workqueue) {
+	data->dump_workqueue = alloc_ordered_workqueue(KBUILD_MODNAME "_cd", 0);
+	if (!data->dump_workqueue) {
 		destroy_workqueue(data->workqueue);
 		return -ENOMEM;
 	}
@@ -2879,6 +2974,8 @@ static int btintel_pcie_probe(struct pci_dev *pdev,
 	INIT_WORK(&data->rx_work, btintel_pcie_rx_work);
 	INIT_WORK(&data->reset_work, btintel_pcie_reset_work);
 	INIT_WORK(&data->coredump_work, btintel_pcie_coredump_worker);
+	INIT_WORK(&data->hwexp_work, btintel_pcie_hwexp_worker);
+	INIT_WORK(&data->fwtrigger_work, btintel_pcie_fwtrigger_worker);
 
 	data->boot_stage_cache = 0x00;
 	data->img_resp_cache = 0x00;
@@ -2921,7 +3018,7 @@ static int btintel_pcie_probe(struct pci_dev *pdev,
 	/* reset device before exit */
 	btintel_pcie_reset_bt(data);
 
-	destroy_workqueue(data->coredump_workqueue);
+	destroy_workqueue(data->dump_workqueue);
 
 	pci_clear_master(pdev);
 
@@ -2940,12 +3037,14 @@ static void btintel_pcie_remove(struct pci_dev *pdev)
 		return;
 	}
 
-	/* Permanently block coredump triggers and drain the worker before
-	 * tearing down. Must run before cancel_work_sync(&reset_work) so
-	 * the disable counter stays >= 1 even after reset_work()'s
+	/* Permanently block all dump triggers and drain the workers before
+	 * tearing down. Must run before disable_work_sync(&reset_work) so
+	 * the disable counters stay >= 1 even after reset_work()'s
 	 * balanced enable_work() (counter 2 -> 1, never reaching 0).
 	 */
 	disable_work_sync(&data->coredump_work);
+	disable_work_sync(&data->hwexp_work);
+	disable_work_sync(&data->fwtrigger_work);
 
 	/* Cancel pending reset work. Skip only when remove() is called from
 	 * within the reset work itself (PLDR device_reprobe path) to avoid
@@ -2973,7 +3072,7 @@ static void btintel_pcie_remove(struct pci_dev *pdev)
 
 	btintel_pcie_release_hdev(data);
 
-	destroy_workqueue(data->coredump_workqueue);
+	destroy_workqueue(data->dump_workqueue);
 	destroy_workqueue(data->workqueue);
 
 	btintel_pcie_free(data);
@@ -2992,16 +3091,8 @@ static void btintel_pcie_coredump(struct device *dev)
 	if (!data)
 		return;
 
-	if (test_and_set_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags))
-		return;
-
-	data->dmp_hdr.trigger_reason  = BTINTEL_PCIE_TRIGGER_REASON_USER_TRIGGER;
-	/* queue_work() returns false if the work is disabled (reset or
-	 * remove in progress); clear the in-progress bit so a later
-	 * trigger can succeed once the work is re-enabled.
-	 */
-	if (!queue_work(data->coredump_workqueue, &data->coredump_work))
-		clear_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags);
+	btintel_pcie_queue_coredump(data,
+				    BTINTEL_PCIE_TRIGGER_REASON_USER_TRIGGER);
 }
 #endif
 
@@ -3113,8 +3204,7 @@ static int btintel_pcie_resume(struct device *dev)
 	if (data->pm_sx_event == PM_EVENT_FREEZE ||
 	    data->pm_sx_event == PM_EVENT_HIBERNATE) {
 		set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags);
-		data->reset_type = BTINTEL_PCIE_IOSF_PRR_FLR;
-		btintel_pcie_reset(data->hdev);
+		btintel_pcie_request_reset(data, BTINTEL_PCIE_IOSF_PRR_FLR);
 		return 0;
 	}
 
@@ -3138,14 +3228,10 @@ static int btintel_pcie_resume(struct device *dev)
 	if (btintel_pcie_in_error(data) ||
 			btintel_pcie_in_device_halt(data)) {
 		bt_dev_err(data->hdev, "Controller in error state for D0 entry");
-		if (!test_and_set_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS,
-				      &data->flags)) {
-			data->dmp_hdr.trigger_reason =
-				BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT;
-			queue_work(data->coredump_workqueue, &data->coredump_work);
-		}
+		btintel_pcie_queue_coredump(data,
+					    BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT);
 		set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags);
-		btintel_pcie_reset(data->hdev);
+		btintel_pcie_request_reset(data, BTINTEL_PCIE_IOSF_PRR_FLR);
 	}
 	return err;
 }
diff --git a/drivers/bluetooth/btintel_pcie.h b/drivers/bluetooth/btintel_pcie.h
index 7caee093e3164..749369b24031c 100644
--- a/drivers/bluetooth/btintel_pcie.h
+++ b/drivers/bluetooth/btintel_pcie.h
@@ -118,7 +118,6 @@ enum {
 
 enum {
 	BTINTEL_PCIE_CORE_HALTED,
-	BTINTEL_PCIE_HWEXP_INPROGRESS,
 	BTINTEL_PCIE_COREDUMP_INPROGRESS,
 	BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS,
 	BTINTEL_PCIE_RECOVERY_IN_PROGRESS,
@@ -466,8 +465,11 @@ struct btintel_pcie_dump_header {
  * @workqueue: workqueue for RX work
  * @rx_skb_q: SKB queue for RX packet
  * @rx_work: RX work struct to process the RX packet in @rx_skb_q
- * @coredump_workqueue: dedicated workqueue for coredump collection
- * @coredump_work: work struct for coredump trace collection
+ * @dump_workqueue: dedicated ordered workqueue serializing the coredump,
+ *	hardware exception, and firmware-trigger dump workers
+ * @coredump_work: work struct for DRAM trace coredump collection
+ * @hwexp_work: work struct for hardware exception event read
+ * @fwtrigger_work: work struct for firmware-triggered diagnostic event read
  * @dma_pool: DMA pool for descriptors, index array and ci
  * @dma_p_addr: DMA address for pool
  * @dma_v_addr: address of pool
@@ -516,8 +518,10 @@ struct btintel_pcie_data {
 	struct work_struct	rx_work;
 	struct work_struct      reset_work;
 
-	struct workqueue_struct	*coredump_workqueue;
+	struct workqueue_struct	*dump_workqueue;
 	struct work_struct	coredump_work;
+	struct work_struct	hwexp_work;
+	struct work_struct	fwtrigger_work;
 
 	struct dma_pool	*dma_pool;
 	dma_addr_t	dma_p_addr;
diff --git a/drivers/bluetooth/btnxpuart.c b/drivers/bluetooth/btnxpuart.c
index 6a1cffe08d5f7..0bb300eef157c 100644
--- a/drivers/bluetooth/btnxpuart.c
+++ b/drivers/bluetooth/btnxpuart.c
@@ -1913,13 +1913,15 @@ static int nxp_serdev_probe(struct serdev_device *serdev)
 	}
 
 	if (ps_setup(hdev))
-		goto probe_fail;
+		goto probe_fail_unregister;
 
 	hci_devcd_register(hdev, nxp_coredump, nxp_coredump_hdr,
 			   nxp_coredump_notify);
 
 	return 0;
 
+probe_fail_unregister:
+	hci_unregister_dev(hdev);
 probe_fail:
 	reset_control_assert(nxpdev->pdn);
 	hci_free_dev(hdev);
diff --git a/drivers/bluetooth/btqca.c b/drivers/bluetooth/btqca.c
index 04ebe290bc784..22b08ab05b829 100644
--- a/drivers/bluetooth/btqca.c
+++ b/drivers/bluetooth/btqca.c
@@ -190,25 +190,6 @@ static int qca_send_patch_config_cmd(struct hci_dev *hdev)
 	return err;
 }
 
-static int qca_send_reset(struct hci_dev *hdev)
-{
-	struct sk_buff *skb;
-	int err;
-
-	bt_dev_dbg(hdev, "QCA HCI_RESET");
-
-	skb = __hci_cmd_sync(hdev, HCI_OP_RESET, 0, NULL, HCI_INIT_TIMEOUT);
-	if (IS_ERR(skb)) {
-		err = PTR_ERR(skb);
-		bt_dev_err(hdev, "QCA Reset failed (%d)", err);
-		return err;
-	}
-
-	kfree_skb(skb);
-
-	return 0;
-}
-
 static int qca_read_fw_board_id(struct hci_dev *hdev, u16 *bid)
 {
 	u8 cmd;
@@ -415,7 +396,7 @@ static int qca_tlv_check_data(struct hci_dev *hdev,
 
 		idx = 0;
 		data = tlv->data;
-		while (idx < length - sizeof(struct tlv_type_nvm)) {
+		while (idx + sizeof(struct tlv_type_nvm) <= length) {
 			tlv_nvm = (struct tlv_type_nvm *)(data + idx);
 
 			tag_id = le16_to_cpu(tlv_nvm->tag_id);
@@ -990,11 +971,12 @@ int qca_uart_setup(struct hci_dev *hdev, uint8_t baudrate,
 	}
 
 	/* Perform HCI reset */
-	err = qca_send_reset(hdev);
+	err = __hci_reset_sync(hdev);
 	if (err < 0) {
 		bt_dev_err(hdev, "QCA Failed to run HCI_RESET (%d)", err);
 		return err;
 	}
+	bt_dev_dbg(hdev, "QCA HCI_RESET succeed");
 
 	switch (soc_type) {
 	case QCA_WCN3991:
@@ -1029,8 +1011,7 @@ int qca_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr)
 	baswap(&bdaddr_swapped, bdaddr);
 
 	skb = __hci_cmd_sync_ev(hdev, EDL_WRITE_BD_ADDR_OPCODE, 6,
-				&bdaddr_swapped, HCI_EV_VENDOR,
-				HCI_INIT_TIMEOUT);
+				&bdaddr_swapped, 0, HCI_INIT_TIMEOUT);
 	if (IS_ERR(skb)) {
 		err = PTR_ERR(skb);
 		bt_dev_err(hdev, "QCA Change address cmd failed (%d)", err);
diff --git a/drivers/bluetooth/btrtl.c b/drivers/bluetooth/btrtl.c
index 49ecb18fea45f..7f54d2d2d13a0 100644
--- a/drivers/bluetooth/btrtl.c
+++ b/drivers/bluetooth/btrtl.c
@@ -797,8 +797,9 @@ static int rtlbt_parse_firmware(struct hci_dev *hdev,
 	}
 
 	BT_DBG("length=%x offset=%x index %d", patch_length, patch_offset, i);
-	min_size = patch_offset + patch_length;
-	if (btrtl_dev->fw_len < min_size)
+	if (patch_length < sizeof(epatch_info->fw_version) ||
+	    patch_offset > btrtl_dev->fw_len ||
+	    patch_length > btrtl_dev->fw_len - patch_offset)
 		return -EINVAL;
 
 	/* Copy the firmware into a new buffer and write the version at
diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c
index 08c0a99a62c5c..4c3193ec2d521 100644
--- a/drivers/bluetooth/btusb.c
+++ b/drivers/bluetooth/btusb.c
@@ -297,6 +297,8 @@ static const struct usb_device_id quirks_table[] = {
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3501), .driver_info = BTUSB_QCA_ROME |
 						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3503), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
 
 	/* QCA WCN6855 chipset */
 	{ USB_DEVICE(0x0489, 0xe0c7), .driver_info = BTUSB_QCA_WCN6855 |
@@ -679,6 +681,8 @@ static const struct usb_device_id quirks_table[] = {
 	{ USB_DEVICE(0x13d3, 0x3606), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH },
 	/* MediaTek MT7902 Bluetooth devices */
+	{ USB_DEVICE(0x0489, 0xe156), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x0e8d, 0x1ede), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3579), .driver_info = BTUSB_MEDIATEK |
@@ -796,6 +800,8 @@ static const struct usb_device_id quirks_table[] = {
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3613), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3625), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3627), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3628), .driver_info = BTUSB_MEDIATEK |
@@ -850,6 +856,12 @@ static const struct usb_device_id quirks_table[] = {
 	{ USB_DEVICE(0x37ad, 0x0600), .driver_info = BTUSB_REALTEK |
 						     BTUSB_WIDEBAND_SPEECH },
 
+	/* Additional Realtek 8761CU Bluetooth devices */
+	{ USB_DEVICE(0x0b05, 0x1bef), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0b05, 0x1d70), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
 	/* Additional Realtek 8821AE Bluetooth devices */
 	{ USB_DEVICE(0x0b05, 0x17dc), .driver_info = BTUSB_REALTEK },
 	{ USB_DEVICE(0x13d3, 0x3414), .driver_info = BTUSB_REALTEK },
@@ -882,6 +894,8 @@ static const struct usb_device_id quirks_table[] = {
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x0bda, 0xc123), .driver_info = BTUSB_REALTEK |
 						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x1357, 0xc123), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x0cb5, 0xc547), .driver_info = BTUSB_REALTEK |
 						     BTUSB_WIDEBAND_SPEECH },
 
@@ -937,6 +951,10 @@ struct qca_dump_info {
 	u16 ram_dump_seqno;
 };
 
+struct btqca_data {
+	struct qca_dump_info qca_dump;
+};
+
 #define BTUSB_MAX_ISOC_FRAMES	10
 
 #define BTUSB_INTR_RUNNING	0
@@ -1010,6 +1028,7 @@ struct btusb_data {
 	bool usb_alt6_packet_flow;
 	int isoc_altsetting;
 	int suspend_count;
+	const struct usb_device_id *match_id;
 
 	int (*recv_event)(struct hci_dev *hdev, struct sk_buff *skb);
 	int (*recv_acl)(struct hci_dev *hdev, struct sk_buff *skb);
@@ -1022,8 +1041,6 @@ struct btusb_data {
 	int (*disconnect)(struct hci_dev *hdev);
 
 	int oob_wake_irq;   /* irq for out-of-band wake-on-bt */
-
-	struct qca_dump_info qca_dump;
 };
 
 static void btusb_reset(struct hci_dev *hdev)
@@ -3070,14 +3087,15 @@ static int btusb_set_bdaddr_ath3012(struct hci_dev *hdev,
 static int btusb_set_bdaddr_wcn6855(struct hci_dev *hdev,
 				const bdaddr_t *bdaddr)
 {
+	bdaddr_t bdaddr_swapped;
 	struct sk_buff *skb;
-	u8 buf[6];
 	long ret;
 
-	memcpy(buf, bdaddr, sizeof(bdaddr_t));
+	baswap(&bdaddr_swapped, bdaddr);
 
-	skb = __hci_cmd_sync_ev(hdev, 0xfc14, sizeof(buf), buf,
-				HCI_EV_CMD_COMPLETE, HCI_INIT_TIMEOUT);
+	skb = __hci_cmd_sync_ev(hdev, 0xfc14, sizeof(bdaddr_swapped),
+				&bdaddr_swapped, HCI_EV_CMD_COMPLETE,
+				HCI_INIT_TIMEOUT);
 	if (IS_ERR(skb)) {
 		ret = PTR_ERR(skb);
 		bt_dev_err(hdev, "Change address command failed (%ld)", ret);
@@ -3113,14 +3131,15 @@ struct qca_dump_hdr {
 static void btusb_dump_hdr_qca(struct hci_dev *hdev, struct sk_buff *skb)
 {
 	char buf[128];
-	struct btusb_data *btdata = hci_get_drvdata(hdev);
+	struct btqca_data *btqca_data = hci_get_priv(hdev);
+	struct qca_dump_info *qca_dump_ptr = &btqca_data->qca_dump;
 
 	snprintf(buf, sizeof(buf), "Controller Name: 0x%x\n",
-			btdata->qca_dump.controller_id);
+			qca_dump_ptr->controller_id);
 	skb_put_data(skb, buf, strlen(buf));
 
 	snprintf(buf, sizeof(buf), "Firmware Version: 0x%x\n",
-			btdata->qca_dump.fw_version);
+			qca_dump_ptr->fw_version);
 	skb_put_data(skb, buf, strlen(buf));
 
 	snprintf(buf, sizeof(buf), "Driver: %s\nVendor: qca\n",
@@ -3128,7 +3147,7 @@ static void btusb_dump_hdr_qca(struct hci_dev *hdev, struct sk_buff *skb)
 	skb_put_data(skb, buf, strlen(buf));
 
 	snprintf(buf, sizeof(buf), "VID: 0x%x\nPID:0x%x\n",
-			btdata->qca_dump.id_vendor, btdata->qca_dump.id_product);
+			qca_dump_ptr->id_vendor, qca_dump_ptr->id_product);
 	skb_put_data(skb, buf, strlen(buf));
 
 	snprintf(buf, sizeof(buf), "Lmp Subversion: 0x%x\n",
@@ -3157,6 +3176,8 @@ static int handle_dump_pkt_qca(struct hci_dev *hdev, struct sk_buff *skb)
 
 	struct qca_dump_hdr *dump_hdr;
 	struct btusb_data *btdata = hci_get_drvdata(hdev);
+	struct btqca_data *btqca_data = hci_get_priv(hdev);
+	struct qca_dump_info *qca_dump_ptr = &btqca_data->qca_dump;
 	struct usb_device *udev = btdata->udev;
 
 	pkt_type = hci_skb_pkt_type(skb);
@@ -3184,8 +3205,8 @@ static int handle_dump_pkt_qca(struct hci_dev *hdev, struct sk_buff *skb)
 			goto out;
 		}
 
-		btdata->qca_dump.ram_dump_size = dump_size;
-		btdata->qca_dump.ram_dump_seqno = 0;
+		qca_dump_ptr->ram_dump_size = dump_size;
+		qca_dump_ptr->ram_dump_seqno = 0;
 
 		skb_pull(skb, offsetof(struct qca_dump_hdr, data0));
 
@@ -3197,29 +3218,29 @@ static int handle_dump_pkt_qca(struct hci_dev *hdev, struct sk_buff *skb)
 		skb_pull(skb, offsetof(struct qca_dump_hdr, data));
 	}
 
-	if (!btdata->qca_dump.ram_dump_size) {
+	if (!qca_dump_ptr->ram_dump_size) {
 		ret = -EINVAL;
 		bt_dev_err(hdev, "memdump is not active");
 		goto out;
 	}
 
-	if ((seqno > btdata->qca_dump.ram_dump_seqno + 1) && (seqno != QCA_LAST_SEQUENCE_NUM)) {
-		dump_size = QCA_MEMDUMP_PKT_SIZE * (seqno - btdata->qca_dump.ram_dump_seqno - 1);
+	if ((seqno > qca_dump_ptr->ram_dump_seqno + 1) && seqno != QCA_LAST_SEQUENCE_NUM) {
+		dump_size = QCA_MEMDUMP_PKT_SIZE * (seqno - qca_dump_ptr->ram_dump_seqno - 1);
 		hci_devcd_append_pattern(hdev, 0x0, dump_size);
 		bt_dev_err(hdev,
 			   "expected memdump seqno(%u) is not received(%u)\n",
-			   btdata->qca_dump.ram_dump_seqno, seqno);
-		btdata->qca_dump.ram_dump_seqno = seqno;
+			   qca_dump_ptr->ram_dump_seqno, seqno);
+		qca_dump_ptr->ram_dump_seqno = seqno;
 		kfree_skb(skb);
 		return ret;
 	}
 
 	hci_devcd_append(hdev, skb);
-	btdata->qca_dump.ram_dump_seqno++;
+	qca_dump_ptr->ram_dump_seqno++;
 	if (seqno == QCA_LAST_SEQUENCE_NUM) {
 		bt_dev_info(hdev,
 				"memdump done: pkts(%u), total(%u)\n",
-				btdata->qca_dump.ram_dump_seqno, btdata->qca_dump.ram_dump_size);
+				qca_dump_ptr->ram_dump_seqno, qca_dump_ptr->ram_dump_size);
 
 		hci_devcd_complete(hdev);
 		goto out;
@@ -3227,10 +3248,10 @@ static int handle_dump_pkt_qca(struct hci_dev *hdev, struct sk_buff *skb)
 	return ret;
 
 out:
-	if (btdata->qca_dump.ram_dump_size)
+	if (qca_dump_ptr->ram_dump_size)
 		usb_enable_autosuspend(udev);
-	btdata->qca_dump.ram_dump_size = 0;
-	btdata->qca_dump.ram_dump_seqno = 0;
+	qca_dump_ptr->ram_dump_size = 0;
+	qca_dump_ptr->ram_dump_seqno = 0;
 	clear_bit(BTUSB_HW_SSR_ACTIVE, &btdata->flags);
 
 	if (ret < 0)
@@ -3705,8 +3726,12 @@ static int btusb_setup_qca(struct hci_dev *hdev)
 	if (err < 0)
 		return err;
 
-	btdata->qca_dump.fw_version = le32_to_cpu(ver.patch_version);
-	btdata->qca_dump.controller_id = le32_to_cpu(ver.rom_version);
+	if (btdata->match_id->driver_info & BTUSB_QCA_WCN6855) {
+		struct btqca_data *btqca_data = hci_get_priv(hdev);
+
+		btqca_data->qca_dump.fw_version = le32_to_cpu(ver.patch_version);
+		btqca_data->qca_dump.controller_id = le32_to_cpu(ver.rom_version);
+	}
 
 	if (!(status & QCA_SYSCFG_UPDATED)) {
 		err = btusb_setup_qca_load_nvm(hdev, &ver, info);
@@ -3887,16 +3912,13 @@ static bool btusb_wakeup(struct hci_dev *hdev)
 
 static int btusb_shutdown_qca(struct hci_dev *hdev)
 {
-	struct sk_buff *skb;
+	int err;
 
-	skb = __hci_cmd_sync(hdev, HCI_OP_RESET, 0, NULL, HCI_INIT_TIMEOUT);
-	if (IS_ERR(skb)) {
+	err = __hci_reset_sync(hdev);
+	if (err)
 		bt_dev_err(hdev, "HCI reset during shutdown failed");
-		return PTR_ERR(skb);
-	}
-	kfree_skb(skb);
 
-	return 0;
+	return err;
 }
 
 static ssize_t force_poll_sync_read(struct file *file, char __user *user_buf,
@@ -4080,7 +4102,7 @@ static int btusb_probe(struct usb_interface *intf,
 	struct btusb_data *data;
 	struct hci_dev *hdev;
 	unsigned ifnum_base;
-	int err, priv_size;
+	int err, priv_size = 0;
 
 	BT_DBG("intf %p id %p", intf, id);
 
@@ -4099,7 +4121,7 @@ static int btusb_probe(struct usb_interface *intf,
 			id = match;
 	}
 
-	if (id->driver_info == BTUSB_IGNORE)
+	if (id->driver_info & BTUSB_IGNORE)
 		return -ENODEV;
 
 	if (id->driver_info & BTUSB_ATH3012) {
@@ -4117,6 +4139,7 @@ static int btusb_probe(struct usb_interface *intf,
 	if (!data)
 		return -ENOMEM;
 
+	data->match_id = id;
 	err = usb_find_common_endpoints(intf->cur_altsetting, &data->bulk_rx_ep,
 					&data->bulk_tx_ep, &data->intr_ep, NULL);
 	if (err)
@@ -4150,8 +4173,6 @@ static int btusb_probe(struct usb_interface *intf,
 	init_usb_anchor(&data->ctrl_anchor);
 	spin_lock_init(&data->rxlock);
 
-	priv_size = 0;
-
 	data->recv_event = hci_recv_frame;
 	data->recv_bulk = btusb_recv_bulk;
 
@@ -4170,6 +4191,9 @@ static int btusb_probe(struct usb_interface *intf,
 	} else if (id->driver_info & BTUSB_MEDIATEK) {
 		/* Allocate extra space for Mediatek device */
 		priv_size += sizeof(struct btmtk_data);
+	} else if (id->driver_info & BTUSB_QCA_WCN6855) {
+		/* Allocate extra space for QCA WCN6855 device */
+		priv_size += sizeof(struct btqca_data);
 	}
 
 	data->recv_acl = hci_recv_frame;
@@ -4312,8 +4336,10 @@ static int btusb_probe(struct usb_interface *intf,
 	}
 
 	if (id->driver_info & BTUSB_QCA_WCN6855) {
-		data->qca_dump.id_vendor = id->idVendor;
-		data->qca_dump.id_product = id->idProduct;
+		struct btqca_data *btqca_data = hci_get_priv(hdev);
+
+		btqca_data->qca_dump.id_vendor = id->idVendor;
+		btqca_data->qca_dump.id_product = id->idProduct;
 		data->recv_event = btusb_recv_evt_qca;
 		data->recv_acl = btusb_recv_acl_qca;
 		hci_devcd_register(hdev, btusb_coredump_qca, btusb_dump_hdr_qca, NULL);
diff --git a/drivers/bluetooth/hci_nokia.c b/drivers/bluetooth/hci_nokia.c
index 1e65b541f8ade..be2923231e712 100644
--- a/drivers/bluetooth/hci_nokia.c
+++ b/drivers/bluetooth/hci_nokia.c
@@ -354,9 +354,29 @@ static int nokia_setup_fw(struct hci_uart *hu)
 		u16 opcode;
 		struct sk_buff *skb;
 
+		if (pkt_size > fw_size - 2) {
+			err = -EINVAL;
+			dev_err(dev, "%s: Malformed firmware packet\n",
+				hu->hdev->name);
+			goto done;
+		}
+
 		switch (pkt_type) {
 		case HCI_COMMAND_PKT:
+			if (pkt_size < 1 + HCI_COMMAND_HDR_SIZE) {
+				err = -EINVAL;
+				dev_err(dev, "%s: Malformed firmware command\n",
+					hu->hdev->name);
+				goto done;
+			}
+
 			cmd = (struct hci_command_hdr *)(fw_ptr + 3);
+			if (cmd->plen > pkt_size - 1 - HCI_COMMAND_HDR_SIZE) {
+				err = -EINVAL;
+				dev_err(dev, "%s: Truncated firmware command\n",
+					hu->hdev->name);
+				goto done;
+			}
 			opcode = le16_to_cpu(cmd->opcode);
 
 			skb = __hci_cmd_sync(hu->hdev, opcode, cmd->plen,
diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c
index b2d1ee3a3d115..1222f97800f4a 100644
--- a/drivers/bluetooth/hci_qca.c
+++ b/drivers/bluetooth/hci_qca.c
@@ -1087,6 +1087,10 @@ static void qca_controller_memdump(struct work_struct *work)
 			if (!(qca_memdump->ram_dump_size)) {
 				bt_dev_err(hu->hdev, "Rx invalid memdump size");
 				kfree(qca_memdump);
+				qca->qca_memdump = NULL;
+				qca->memdump_state = QCA_MEMDUMP_COLLECTED;
+				clear_and_wake_up_bit(QCA_MEMDUMP_COLLECTION, &qca->flags);
+				clear_bit(QCA_IBS_DISABLED, &qca->flags);
 				kfree_skb(skb);
 				mutex_unlock(&qca->hci_memdump_lock);
 				return;
diff --git a/drivers/bus/mhi/ep/main.c b/drivers/bus/mhi/ep/main.c
index b1213786f72c6..21bc2c50170ff 100644
--- a/drivers/bus/mhi/ep/main.c
+++ b/drivers/bus/mhi/ep/main.c
@@ -1340,14 +1340,19 @@ static int mhi_ep_create_device(struct mhi_ep_cntrl *mhi_cntrl, u32 ch_id)
 	ret = dev_set_name(&mhi_dev->dev, "%s_%s",
 		     dev_name(&mhi_cntrl->mhi_dev->dev),
 		     mhi_dev->name);
-	if (ret) {
-		put_device(&mhi_dev->dev);
-		return ret;
-	}
+	if (ret)
+		goto err_put_channels;
 
 	ret = device_add(&mhi_dev->dev);
 	if (ret)
-		put_device(&mhi_dev->dev);
+		goto err_put_channels;
+
+	return 0;
+
+err_put_channels:
+	put_device(&mhi_dev->dev); /* DL channel reference */
+	put_device(&mhi_dev->dev); /* UL channel reference */
+	put_device(&mhi_dev->dev); /* device_initialize() reference */
 
 	return ret;
 }
diff --git a/drivers/bus/mhi/host/main.c b/drivers/bus/mhi/host/main.c
index 53c0ffe300702..4d458396233ab 100644
--- a/drivers/bus/mhi/host/main.c
+++ b/drivers/bus/mhi/host/main.c
@@ -170,6 +170,9 @@ EXPORT_SYMBOL_GPL(mhi_get_mhi_state);
 
 void mhi_soc_reset(struct mhi_controller *mhi_cntrl)
 {
+	int __maybe_unused ret;
+	u32 tmp;
+
 	if (mhi_cntrl->reset) {
 		mhi_cntrl->reset(mhi_cntrl);
 		return;
@@ -178,6 +181,9 @@ void mhi_soc_reset(struct mhi_controller *mhi_cntrl)
 	/* Generic MHI SoC reset */
 	mhi_write_reg(mhi_cntrl, mhi_cntrl->regs, MHI_SOC_RESET_REQ_OFFSET,
 		      MHI_SOC_RESET_REQ);
+	/* Flush the posted write to the device (ignore return value) */
+	ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->regs, MHI_SOC_RESET_REQ_OFFSET,
+			   &tmp);
 }
 EXPORT_SYMBOL_GPL(mhi_soc_reset);
 
diff --git a/drivers/bus/mhi/host/pm.c b/drivers/bus/mhi/host/pm.c
index f799503c8f365..9d29f1591a459 100644
--- a/drivers/bus/mhi/host/pm.c
+++ b/drivers/bus/mhi/host/pm.c
@@ -651,21 +651,13 @@ static void mhi_pm_sys_error_transition(struct mhi_controller *mhi_cntrl)
 
 	/* Trigger MHI RESET so that the device will not access host memory */
 	if (reset_device) {
-		u32 in_reset = -1;
-		unsigned long timeout = msecs_to_jiffies(mhi_cntrl->timeout_ms);
-
 		dev_dbg(dev, "Triggering MHI Reset in device\n");
 		mhi_set_mhi_state(mhi_cntrl, MHI_STATE_RESET);
 
 		/* Wait for the reset bit to be cleared by the device */
-		ret = wait_event_timeout(mhi_cntrl->state_event,
-					 mhi_read_reg_field(mhi_cntrl,
-							    mhi_cntrl->regs,
-							    MHICTRL,
-							    MHICTRL_RESET_MASK,
-							    &in_reset) ||
-					!in_reset, timeout);
-		if (!ret || in_reset) {
+		ret = mhi_poll_reg_field(mhi_cntrl, mhi_cntrl->regs, MHICTRL,
+				 MHICTRL_RESET_MASK, 0, 25000, mhi_cntrl->timeout_ms);
+		if (ret) {
 			dev_err(dev, "Device failed to exit MHI Reset state\n");
 			write_lock_irq(&mhi_cntrl->pm_lock);
 			cur_state = mhi_tryset_pm_state(mhi_cntrl,
diff --git a/drivers/bus/qcom-ebi2.c b/drivers/bus/qcom-ebi2.c
index be8166565e7cc..8d2eb955dc921 100644
--- a/drivers/bus/qcom-ebi2.c
+++ b/drivers/bus/qcom-ebi2.c
@@ -302,41 +302,23 @@ static int qcom_ebi2_probe(struct platform_device *pdev)
 	u32 val;
 	int ret;
 
-	ebi2xclk = devm_clk_get(dev, "ebi2x");
+	ebi2xclk = devm_clk_get_enabled(dev, "ebi2x");
 	if (IS_ERR(ebi2xclk))
 		return PTR_ERR(ebi2xclk);
 
-	ret = clk_prepare_enable(ebi2xclk);
-	if (ret) {
-		dev_err(dev, "could not enable EBI2X clk (%d)\n", ret);
-		return ret;
-	}
-
-	ebi2clk = devm_clk_get(dev, "ebi2");
-	if (IS_ERR(ebi2clk)) {
-		ret = PTR_ERR(ebi2clk);
-		goto err_disable_2x_clk;
-	}
-
-	ret = clk_prepare_enable(ebi2clk);
-	if (ret) {
-		dev_err(dev, "could not enable EBI2 clk\n");
-		goto err_disable_2x_clk;
-	}
+	ebi2clk = devm_clk_get_enabled(dev, "ebi2");
+	if (IS_ERR(ebi2clk))
+		return PTR_ERR(ebi2clk);
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	ebi2_base = devm_ioremap_resource(dev, res);
-	if (IS_ERR(ebi2_base)) {
-		ret = PTR_ERR(ebi2_base);
-		goto err_disable_clk;
-	}
+	if (IS_ERR(ebi2_base))
+		return PTR_ERR(ebi2_base);
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
 	ebi2_xmem = devm_ioremap_resource(dev, res);
-	if (IS_ERR(ebi2_xmem)) {
-		ret = PTR_ERR(ebi2_xmem);
-		goto err_disable_clk;
-	}
+	if (IS_ERR(ebi2_xmem))
+		return PTR_ERR(ebi2_xmem);
 
 	/* Allegedly this turns the power save mode off */
 	writel(0UL, ebi2_xmem + EBI2_XMEM_CFG);
@@ -373,15 +355,9 @@ static int qcom_ebi2_probe(struct platform_device *pdev)
 	}
 
 	if (have_children)
-		return of_platform_default_populate(np, NULL, dev);
-	return 0;
+		return devm_of_platform_populate(dev);
 
-err_disable_clk:
-	clk_disable_unprepare(ebi2clk);
-err_disable_2x_clk:
-	clk_disable_unprepare(ebi2xclk);
-
-	return ret;
+	return 0;
 }
 
 static const struct of_device_id qcom_ebi2_of_match[] = {
diff --git a/drivers/bus/ti-sysc.c b/drivers/bus/ti-sysc.c
index a5b9507de37c3..e118b900c9ac8 100644
--- a/drivers/bus/ti-sysc.c
+++ b/drivers/bus/ti-sysc.c
@@ -682,6 +682,7 @@ static struct device_node *stdout_path;
 
 static void sysc_init_stdout_path(struct sysc *ddata)
 {
+	struct device_node *chosen;
 	struct device_node *np = NULL;
 	const char *uart;
 
@@ -691,15 +692,18 @@ static void sysc_init_stdout_path(struct sysc *ddata)
 	if (stdout_path)
 		return;
 
-	np = of_find_node_by_path("/chosen");
-	if (!np)
+	chosen = of_find_node_by_path("/chosen");
+	if (!chosen)
 		goto err;
 
-	uart = of_get_property(np, "stdout-path", NULL);
-	if (!uart)
+	uart = of_get_property(chosen, "stdout-path", NULL);
+	if (!uart) {
+		of_node_put(chosen);
 		goto err;
+	}
 
 	np = of_find_node_by_path(uart);
+	of_node_put(chosen);
 	if (!np)
 		goto err;
 
diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
index a5bcef4a54eea..1bf5e3f8a557b 100644
--- a/drivers/char/hw_random/Kconfig
+++ b/drivers/char/hw_random/Kconfig
@@ -193,7 +193,7 @@ config HW_RANDOM_IXP4XX
 
 config HW_RANDOM_OMAP
 	tristate "OMAP Random Number Generator support"
-	depends on ARCH_OMAP16XX || ARCH_OMAP2PLUS || ARCH_MVEBU || ARCH_K3 || COMPILE_TEST
+	depends on ARCH_OMAP16XX || ARCH_OMAP2PLUS || ARCH_MVEBU || ARCH_K3 || ARCH_RZN1 || COMPILE_TEST
 	default HW_RANDOM
 	help
 	  This driver provides kernel-side support for the Random Number
@@ -636,6 +636,17 @@ config HW_RANDOM_XILINX
 	  To compile this driver as a module, choose M here: the module
 	  will be called xilinx-trng.
 
+config HW_RANDOM_QCOM
+	tristate "Qualcomm True Random Number Generator Driver"
+	depends on ARCH_QCOM || COMPILE_TEST
+	depends on HW_RANDOM
+	help
+	  This driver provides support for the True Random Number
+	  Generator hardware found on some Qualcomm SoCs.
+
+	  To compile this driver as a module, choose M here. The
+	  module will be called qcom-rng. If unsure, say N.
+
 endif # HW_RANDOM
 
 config UML_RANDOM
diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile
index 95b5adb49560d..8fce2aa6cf7d0 100644
--- a/drivers/char/hw_random/Makefile
+++ b/drivers/char/hw_random/Makefile
@@ -54,3 +54,4 @@ obj-$(CONFIG_HW_RANDOM_POLARFIRE_SOC) += mpfs-rng.o
 obj-$(CONFIG_HW_RANDOM_ROCKCHIP) += rockchip-rng.o
 obj-$(CONFIG_HW_RANDOM_JH7110) += jh7110-trng.o
 obj-$(CONFIG_HW_RANDOM_XILINX) += xilinx-trng.o
+obj-$(CONFIG_HW_RANDOM_QCOM) += qcom-rng.o
diff --git a/drivers/char/hw_random/airoha-trng.c b/drivers/char/hw_random/airoha-trng.c
index 076519a2f1001..98c131ee98914 100644
--- a/drivers/char/hw_random/airoha-trng.c
+++ b/drivers/char/hw_random/airoha-trng.c
@@ -186,10 +186,8 @@ static int airoha_trng_probe(struct platform_device *pdev)
 	airoha_trng_irq_mask(trng);
 	ret = devm_request_irq(&pdev->dev, irq, airoha_trng_irq, 0,
 			       pdev->name, (void *)trng);
-	if (ret) {
-		dev_err(dev, "Can't get interrupt working.\n");
+	if (ret)
 		return ret;
-	}
 
 	init_completion(&trng->rng_op_done);
 
diff --git a/drivers/char/hw_random/atmel-rng.c b/drivers/char/hw_random/atmel-rng.c
index 4ebbc44fecf01..06e79d5167438 100644
--- a/drivers/char/hw_random/atmel-rng.c
+++ b/drivers/char/hw_random/atmel-rng.c
@@ -185,7 +185,7 @@ static int __maybe_unused atmel_trng_runtime_resume(struct device *dev)
 	return atmel_trng_init(trng);
 }
 
-static const struct dev_pm_ops __maybe_unused atmel_trng_pm_ops = {
+static const struct dev_pm_ops atmel_trng_pm_ops = {
 	SET_RUNTIME_PM_OPS(atmel_trng_runtime_suspend,
 			   atmel_trng_runtime_resume, NULL)
 	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
diff --git a/drivers/char/hw_random/cctrng.c b/drivers/char/hw_random/cctrng.c
index a5be9258037f8..a6925211c3b50 100644
--- a/drivers/char/hw_random/cctrng.c
+++ b/drivers/char/hw_random/cctrng.c
@@ -509,7 +509,7 @@ static int cctrng_probe(struct platform_device *pdev)
 	/* register the driver isr function */
 	rc = devm_request_irq(dev, irq, cc_isr, IRQF_SHARED, "cctrng", drvdata);
 	if (rc)
-		return dev_err_probe(dev, rc, "Could not register to interrupt %d\n", irq);
+		return rc;
 	dev_dbg(dev, "Registered to IRQ: %d\n", irq);
 
 	/* Clear all pending interrupts */
diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c
index 6931657ad2caa..e77af6578ab50 100644
--- a/drivers/char/hw_random/core.c
+++ b/drivers/char/hw_random/core.c
@@ -596,11 +596,13 @@ int hwrng_register(struct hwrng *rng)
 			 */
 			err = set_current_rng(rng);
 			if (err)
-				goto out_unlock;
+				goto out_list_del;
 		}
 	}
 	mutex_unlock(&rng_mutex);
 	return 0;
+out_list_del:
+	list_del_init(&rng->list);
 out_unlock:
 	mutex_unlock(&rng_mutex);
 out:
diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c
index 28c56c2d1bf6d..bae8cdca13fe9 100644
--- a/drivers/char/hw_random/imx-rngc.c
+++ b/drivers/char/hw_random/imx-rngc.c
@@ -296,7 +296,7 @@ static int __init imx_rngc_probe(struct platform_device *pdev)
 			irq, imx_rngc_irq, 0, pdev->name, (void *)rngc);
 	if (ret) {
 		clk_disable_unprepare(rngc->clk);
-		return dev_err_probe(&pdev->dev, ret, "Can't get interrupt working.\n");
+		return ret;
 	}
 
 	if (self_test) {
diff --git a/drivers/char/hw_random/jh7110-trng.c b/drivers/char/hw_random/jh7110-trng.c
index 4712c3c530e42..aee12caab5780 100644
--- a/drivers/char/hw_random/jh7110-trng.c
+++ b/drivers/char/hw_random/jh7110-trng.c
@@ -303,8 +303,7 @@ static int starfive_trng_probe(struct platform_device *pdev)
 	ret = devm_request_irq(&pdev->dev, irq, starfive_trng_irq, 0, pdev->name,
 			       (void *)trng);
 	if (ret)
-		return dev_err_probe(&pdev->dev, ret,
-				     "Failed to register interrupt handler\n");
+		return ret;
 
 	trng->hclk = devm_clk_get(&pdev->dev, "hclk");
 	if (IS_ERR(trng->hclk))
diff --git a/drivers/char/hw_random/ks-sa-rng.c b/drivers/char/hw_random/ks-sa-rng.c
index 9e408144a10c1..4494f1e4ab4db 100644
--- a/drivers/char/hw_random/ks-sa-rng.c
+++ b/drivers/char/hw_random/ks-sa-rng.c
@@ -242,7 +242,14 @@ static int ks_sa_rng_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, ret, "Failed to enable SA power-domain\n");
 	}
 
-	return devm_hwrng_register(&pdev->dev, &ks_sa_rng->rng);
+	ret = devm_hwrng_register(dev, &ks_sa_rng->rng);
+	if (ret) {
+		pm_runtime_put_sync(dev);
+		pm_runtime_disable(dev);
+		return ret;
+	}
+
+	return 0;
 }
 
 static void ks_sa_rng_remove(struct platform_device *pdev)
diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c
index 5e8b50f15db75..9f0545c677741 100644
--- a/drivers/char/hw_random/omap-rng.c
+++ b/drivers/char/hw_random/omap-rng.c
@@ -391,11 +391,8 @@ static int of_get_omap_rng_device_details(struct omap_rng_dev *priv,
 
 		err = devm_request_irq(dev, irq, omap4_rng_irq,
 				       IRQF_TRIGGER_NONE, dev_name(dev), priv);
-		if (err) {
-			dev_err(dev, "unable to request irq %d, err = %d\n",
-				irq, err);
+		if (err)
 			return err;
-		}
 
 		/*
 		 * On OMAP4, enabling the shutdown_oflo interrupt is
@@ -455,32 +452,40 @@ static int omap_rng_probe(struct platform_device *pdev)
 	ret = pm_runtime_resume_and_get(&pdev->dev);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to runtime_get device: %d\n", ret);
-		goto err_ioremap;
+		goto err_pm_disable;
 	}
 
 	priv->clk = devm_clk_get(&pdev->dev, NULL);
-	if (PTR_ERR(priv->clk) == -EPROBE_DEFER)
-		return -EPROBE_DEFER;
+	if (PTR_ERR(priv->clk) == -EPROBE_DEFER) {
+		ret = -EPROBE_DEFER;
+		goto err_pm_put;
+	}
 	if (!IS_ERR(priv->clk)) {
 		ret = clk_prepare_enable(priv->clk);
 		if (ret) {
 			dev_err(&pdev->dev,
 				"Unable to enable the clk: %d\n", ret);
-			goto err_register;
+			goto err_pm_put;
 		}
+	} else {
+		priv->clk = NULL;
 	}
 
 	priv->clk_reg = devm_clk_get(&pdev->dev, "reg");
-	if (PTR_ERR(priv->clk_reg) == -EPROBE_DEFER)
-		return -EPROBE_DEFER;
+	if (PTR_ERR(priv->clk_reg) == -EPROBE_DEFER) {
+		ret = -EPROBE_DEFER;
+		goto err_clk;
+	}
 	if (!IS_ERR(priv->clk_reg)) {
 		ret = clk_prepare_enable(priv->clk_reg);
 		if (ret) {
 			dev_err(&pdev->dev,
 				"Unable to enable the register clk: %d\n",
 				ret);
-			goto err_register;
+			goto err_clk;
 		}
+	} else {
+		priv->clk_reg = NULL;
 	}
 
 	ret = (dev->of_node) ? of_get_omap_rng_device_details(priv, pdev) :
@@ -498,12 +503,14 @@ static int omap_rng_probe(struct platform_device *pdev)
 	return 0;
 
 err_register:
+	clk_disable_unprepare(priv->clk_reg);
+err_clk:
+	clk_disable_unprepare(priv->clk);
+err_pm_put:
 	priv->base = NULL;
 	pm_runtime_put_sync(&pdev->dev);
+err_pm_disable:
 	pm_runtime_disable(&pdev->dev);
-
-	clk_disable_unprepare(priv->clk_reg);
-	clk_disable_unprepare(priv->clk);
 err_ioremap:
 	dev_err(dev, "initialization failed.\n");
 	return ret;
diff --git a/drivers/char/hw_random/qcom-rng.c b/drivers/char/hw_random/qcom-rng.c
new file mode 100644
index 0000000000000..4d046caafe5bb
--- /dev/null
+++ b/drivers/char/hw_random/qcom-rng.c
@@ -0,0 +1,170 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2017-18 Linaro Limited
+//
+// Based on msm-rng.c and downstream driver
+
+#include <linux/acpi.h>
+#include <linux/clk.h>
+#include <linux/hw_random.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+
+/* Device specific register offsets */
+#define PRNG_DATA_OUT		0x0000
+#define PRNG_STATUS		0x0004
+#define PRNG_LFSR_CFG		0x0100
+#define PRNG_CONFIG		0x0104
+
+/* Device specific register masks and config values */
+#define PRNG_LFSR_CFG_MASK	0x0000ffff
+#define PRNG_LFSR_CFG_CLOCKS	0x0000dddd
+#define PRNG_CONFIG_HW_ENABLE	BIT(1)
+#define PRNG_STATUS_DATA_AVAIL	BIT(0)
+
+#define WORD_SZ			4
+
+#define QCOM_TRNG_QUALITY	1024
+
+struct qcom_rng {
+	void __iomem *base;
+	struct clk *clk;
+	struct hwrng hwrng;
+};
+
+struct qcom_rng_match_data {
+	bool hwrng_support;
+};
+
+static int qcom_rng_read(struct qcom_rng *rng, u8 *data, unsigned int max)
+{
+	unsigned int currsize = 0;
+	u32 val;
+	int ret;
+
+	/* read random data from hardware */
+	do {
+		ret = readl_poll_timeout(rng->base + PRNG_STATUS, val,
+					 val & PRNG_STATUS_DATA_AVAIL,
+					 200, 10000);
+		if (ret)
+			return ret;
+
+		val = readl_relaxed(rng->base + PRNG_DATA_OUT);
+
+		if ((max - currsize) >= WORD_SZ) {
+			memcpy(data, &val, WORD_SZ);
+			data += WORD_SZ;
+			currsize += WORD_SZ;
+		} else {
+			/* copy only remaining bytes */
+			memcpy(data, &val, max - currsize);
+			currsize = max;
+		}
+	} while (currsize < max);
+
+	return currsize;
+}
+
+static int qcom_hwrng_init(struct hwrng *hwrng)
+{
+	struct qcom_rng *qrng = container_of(hwrng, struct qcom_rng, hwrng);
+
+	return clk_prepare_enable(qrng->clk);
+}
+
+static int qcom_hwrng_read(struct hwrng *hwrng, void *data, size_t max, bool wait)
+{
+	struct qcom_rng *qrng = container_of(hwrng, struct qcom_rng, hwrng);
+
+	return qcom_rng_read(qrng, data, max);
+}
+
+static void qcom_hwrng_cleanup(struct hwrng *hwrng)
+{
+	struct qcom_rng *qrng = container_of(hwrng, struct qcom_rng, hwrng);
+
+	clk_disable_unprepare(qrng->clk);
+}
+
+static int qcom_rng_probe(struct platform_device *pdev)
+{
+	const struct qcom_rng_match_data *match_data;
+	struct qcom_rng *rng;
+	int ret;
+
+	match_data = device_get_match_data(&pdev->dev);
+	if (match_data == NULL || !match_data->hwrng_support) {
+		dev_info(&pdev->dev, "TRNG support not detected\n");
+		/*
+		 * In this case the driver does nothing except the dev_info(),
+		 * but bind the device anyway to avoid effects on GCC state.
+		 */
+		return 0;
+	}
+
+	rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL);
+	if (!rng)
+		return -ENOMEM;
+
+	rng->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(rng->base))
+		return PTR_ERR(rng->base);
+
+	rng->clk = devm_clk_get_optional(&pdev->dev, "core");
+	if (IS_ERR(rng->clk))
+		return PTR_ERR(rng->clk);
+
+	rng->hwrng.name = "qcom_hwrng";
+	rng->hwrng.init = qcom_hwrng_init;
+	rng->hwrng.read = qcom_hwrng_read;
+	rng->hwrng.cleanup = qcom_hwrng_cleanup;
+	rng->hwrng.quality = QCOM_TRNG_QUALITY;
+	ret = devm_hwrng_register(&pdev->dev, &rng->hwrng);
+	if (ret)
+		dev_err(&pdev->dev, "Register hwrng failed: %d\n", ret);
+	return ret;
+}
+
+static struct qcom_rng_match_data qcom_prng_match_data = {
+	.hwrng_support = false,
+};
+
+static struct qcom_rng_match_data qcom_prng_ee_match_data = {
+	.hwrng_support = false,
+};
+
+static struct qcom_rng_match_data qcom_trng_match_data = {
+	.hwrng_support = true,
+};
+
+static const struct acpi_device_id __maybe_unused qcom_rng_acpi_match[] = {
+	{ .id = "QCOM8160", .driver_data = (kernel_ulong_t)&qcom_prng_ee_match_data },
+	{}
+};
+MODULE_DEVICE_TABLE(acpi, qcom_rng_acpi_match);
+
+static const struct of_device_id __maybe_unused qcom_rng_of_match[] = {
+	{ .compatible = "qcom,prng", .data = &qcom_prng_match_data },
+	{ .compatible = "qcom,prng-ee", .data = &qcom_prng_ee_match_data },
+	{ .compatible = "qcom,trng", .data = &qcom_trng_match_data },
+	{}
+};
+MODULE_DEVICE_TABLE(of, qcom_rng_of_match);
+
+static struct platform_driver qcom_rng_driver = {
+	.probe = qcom_rng_probe,
+	.driver = {
+		.name = KBUILD_MODNAME,
+		.of_match_table = qcom_rng_of_match,
+		.acpi_match_table = ACPI_PTR(qcom_rng_acpi_match),
+	}
+};
+module_platform_driver(qcom_rng_driver);
+
+MODULE_ALIAS("platform:" KBUILD_MODNAME);
+MODULE_DESCRIPTION("Qualcomm random number generator driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/char/hw_random/xgene-rng.c b/drivers/char/hw_random/xgene-rng.c
index 1f4b95341c2ed..629dc85c37419 100644
--- a/drivers/char/hw_random/xgene-rng.c
+++ b/drivers/char/hw_random/xgene-rng.c
@@ -336,7 +336,7 @@ static int xgene_rng_probe(struct platform_device *pdev)
 	rc = devm_request_irq(&pdev->dev, ctx->irq, xgene_rng_irq_handler, 0,
 				dev_name(&pdev->dev), ctx);
 	if (rc)
-		return dev_err_probe(&pdev->dev, rc, "Could not request RNG alarm IRQ\n");
+		return rc;
 
 	/* Enable IP clock */
 	clk = devm_clk_get_optional_enabled(&pdev->dev, NULL);
diff --git a/drivers/char/hw_random/xilinx-trng.c b/drivers/char/hw_random/xilinx-trng.c
index 0fbc22c38fbc7..290bf5bc82db8 100644
--- a/drivers/char/hw_random/xilinx-trng.c
+++ b/drivers/char/hw_random/xilinx-trng.c
@@ -86,8 +86,8 @@ static void xtrng_softreset(struct xilinx_rng *rng)
 	xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSRST_MASK, 0);
 }
 
-/* Return no. of bytes read */
-static size_t xtrng_readblock32(void __iomem *rng_base, __be32 *buf, int blocks32, bool wait)
+/* Return no. of bytes read or a negative error before any data is read. */
+static int xtrng_readblock32(void __iomem *rng_base, __be32 *buf, int blocks32, bool wait)
 {
 	int read = 0, ret;
 	int timeout = 1;
@@ -102,8 +102,11 @@ static size_t xtrng_readblock32(void __iomem *rng_base, __be32 *buf, int blocks3
 		ret = readl_poll_timeout(rng_base + TRNG_STATUS_OFFSET, val,
 					 (val & TRNG_STATUS_QCNT_MASK) ==
 					 TRNG_STATUS_QCNT_16_BYTES, !!wait, timeout);
-		if (ret)
+		if (ret) {
+			if (!read)
+				return ret;
 			break;
+		}
 
 		for (idx = 0; idx < TRNG_READ_4_WORD; idx++) {
 			*(buf + read) = cpu_to_be32(ioread32(rng_base + TRNG_CORE_OUTPUT_OFFSET));
@@ -118,27 +121,40 @@ static int xtrng_collect_random_data(struct xilinx_rng *rng, u8 *rand_gen_buf,
 {
 	u8 randbuf[TRNG_SEC_STRENGTH_BYTES];
 	int byteleft, blocks, count = 0;
+	int full_blocks_bytes;
 	int ret;
 
 	byteleft = no_of_random_bytes & (TRNG_SEC_STRENGTH_BYTES - 1);
 	blocks = no_of_random_bytes >> TRNG_SEC_STRENGTH_SHIFT;
+	full_blocks_bytes = blocks * TRNG_SEC_STRENGTH_BYTES;
 	xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSTART_MASK,
 			  TRNG_CTRL_PRNGSTART_MASK);
 	if (blocks) {
 		ret = xtrng_readblock32(rng->rng_base, (__be32 *)rand_gen_buf, blocks, wait);
-		if (!ret)
-			return 0;
+		if (ret <= 0) {
+			count = ret;
+			goto out_stop;
+		}
 		count += ret;
+		if (ret < full_blocks_bytes)
+			goto out_stop;
 	}
 
 	if (byteleft) {
 		ret = xtrng_readblock32(rng->rng_base, (__be32 *)randbuf, 1, wait);
+		if (ret < 0) {
+			if (!count)
+				count = ret;
+			goto out_stop;
+		}
 		if (!ret)
-			return count;
-		memcpy(rand_gen_buf + (blocks * TRNG_SEC_STRENGTH_BYTES), randbuf, byteleft);
-		count += byteleft;
+			goto out_stop;
+		ret = min(ret, no_of_random_bytes - count);
+		memcpy(rand_gen_buf + count, randbuf, ret);
+		count += ret;
 	}
 
+out_stop:
 	xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET,
 			  TRNG_CTRL_PRNGMODE_MASK | TRNG_CTRL_PRNGSTART_MASK, 0U);
 
diff --git a/drivers/char/ipmi/ipmb_dev_int.c b/drivers/char/ipmi/ipmb_dev_int.c
index 680ff15c30ab9..e4c50d9ae3e13 100644
--- a/drivers/char/ipmi/ipmb_dev_int.c
+++ b/drivers/char/ipmi/ipmb_dev_int.c
@@ -141,13 +141,14 @@ static ssize_t ipmb_write(struct file *file, const char __user *buf,
 	u8 msg[MAX_MSG_LEN];
 	ssize_t ret;
 
-	if (count > sizeof(msg))
+	if (!count || count > sizeof(msg))
 		return -EINVAL;
 
 	if (copy_from_user(&msg, buf, count))
 		return -EFAULT;
 
-	if (count < msg[0])
+	if (msg[IPMB_MSG_LEN_IDX] < IPMB_REQUEST_LEN_MIN ||
+	    count < (size_t)msg[IPMB_MSG_LEN_IDX] + 1)
 		return -EINVAL;
 
 	rq_sa = GET_7BIT_ADDR(msg[RQ_SA_8BIT_IDX]);
diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c
index ab4c85f3d6fe8..8d9f2e647d9b6 100644
--- a/drivers/char/ipmi/ipmi_msghandler.c
+++ b/drivers/char/ipmi/ipmi_msghandler.c
@@ -3757,6 +3757,7 @@ int ipmi_add_smi(struct module         *owner,
  out_err_bmc_reg:
 	ipmi_bmc_unregister(intf);
  out_err_started:
+	intf->in_shutdown = true;
 	if (intf->handlers->shutdown)
 		intf->handlers->shutdown(intf->send_info);
  out_err:
diff --git a/drivers/char/misc_minor_kunit.c b/drivers/char/misc_minor_kunit.c
index e930c78e1ef97..e85210cbb6401 100644
--- a/drivers/char/misc_minor_kunit.c
+++ b/drivers/char/misc_minor_kunit.c
@@ -5,6 +5,7 @@
 #include <linux/miscdevice.h>
 #include <linux/fs.h>
 #include <linux/file.h>
+#include <linux/fs_struct.h>
 #include <linux/init_syscalls.h>
 
 /* static minor (LCD_MINOR) */
@@ -160,18 +161,22 @@ static void __init miscdev_test_can_open(struct kunit *test, struct miscdevice *
 	char *devname;
 
 	devname = kasprintf(GFP_KERNEL, "/dev/%s", misc->name);
-	ret = init_mknod(devname, S_IFCHR | 0600,
-			 new_encode_dev(MKDEV(MISC_MAJOR, misc->minor)));
-	if (ret != 0)
-		KUNIT_FAIL(test, "failed to create node\n");
 
-	filp = filp_open(devname, O_RDONLY, 0);
-	if (IS_ERR(filp))
-		KUNIT_FAIL(test, "failed to open misc device: %ld\n", PTR_ERR(filp));
-	else
-		fput(filp);
+	/* Tests run in a nullfs kthread; borrow the init fs to resolve /dev. */
+	scoped_with_init_fs() {
+		ret = init_mknod(devname, S_IFCHR | 0600,
+				 new_encode_dev(MKDEV(MISC_MAJOR, misc->minor)));
+		if (ret != 0)
+			KUNIT_FAIL(test, "failed to create node\n");
+
+		filp = filp_open(devname, O_RDONLY, 0);
+		if (IS_ERR(filp))
+			KUNIT_FAIL(test, "failed to open misc device: %ld\n", PTR_ERR(filp));
+		else
+			fput(filp);
 
-	init_unlink(devname);
+		init_unlink(devname);
+	}
 	kfree(devname);
 }
 
diff --git a/drivers/char/sonypi.c b/drivers/char/sonypi.c
index 9309cfb935be2..9a88d287fa096 100644
--- a/drivers/char/sonypi.c
+++ b/drivers/char/sonypi.c
@@ -1125,8 +1125,6 @@ static int sonypi_acpi_probe(struct platform_device *pdev)
 		return -ENODEV;
 
 	sonypi_acpi_device = device;
-	strscpy(acpi_device_name(device), "Sony laptop hotkeys");
-	strscpy(acpi_device_class(device), "sony/hotkey");
 	return 0;
 }
 
diff --git a/drivers/char/tpm/tpm_i2c_nuvoton.c b/drivers/char/tpm/tpm_i2c_nuvoton.c
index d44903b299294..aa3673002f3aa 100644
--- a/drivers/char/tpm/tpm_i2c_nuvoton.c
+++ b/drivers/char/tpm/tpm_i2c_nuvoton.c
@@ -182,8 +182,10 @@ static int i2c_nuvoton_wait_for_stat(struct tpm_chip *chip, u8 mask, u8 value,
 						      timeout);
 		if (rc > 0)
 			return 0;
-		/* At this point we know that the SINT pin is asserted, so we
-		 * do not need to do i2c_nuvoton_check_status */
+
+		disable_irq(priv->irq);
+		if (rc < 0)
+			return rc;
 	} else {
 		unsigned long ten_msec, stop;
 		bool status_valid;
diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile
index cc108a75a9008..6899e32e14aa6 100644
--- a/drivers/clk/Makefile
+++ b/drivers/clk/Makefile
@@ -157,7 +157,7 @@ obj-$(CONFIG_ARCH_SUNXI)		+= sunxi/
 obj-y					+= sunxi-ng/
 obj-$(CONFIG_ARCH_TEGRA)		+= tegra/
 obj-y					+= tenstorrent/
-obj-$(CONFIG_ARCH_THEAD)		+= thead/
+obj-y					+= thead/
 obj-y					+= ti/
 obj-$(CONFIG_CLK_UNIPHIER)		+= uniphier/
 obj-$(CONFIG_ARCH_U8500)		+= ux500/
diff --git a/drivers/clk/imx/clk-imx8mm.c b/drivers/clk/imx/clk-imx8mm.c
index 319af4deec01c..89d442415a010 100644
--- a/drivers/clk/imx/clk-imx8mm.c
+++ b/drivers/clk/imx/clk-imx8mm.c
@@ -300,6 +300,7 @@ static int imx8mm_clocks_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
+	struct clk_hw *audio_pll_hws[2];
 	void __iomem *base;
 	int ret;
 
@@ -610,6 +611,11 @@ static int imx8mm_clocks_probe(struct platform_device *pdev)
 
 	imx_register_uart_clocks();
 
+	/* Add debug interface for audio PLLs */
+	audio_pll_hws[0] = hws[IMX8MM_AUDIO_PLL1];
+	audio_pll_hws[1] = hws[IMX8MM_AUDIO_PLL2];
+	imx_audio_pll_debug_init(audio_pll_hws, ARRAY_SIZE(audio_pll_hws));
+
 	return 0;
 
 unregister_hws:
diff --git a/drivers/clk/imx/clk-pll14xx.c b/drivers/clk/imx/clk-pll14xx.c
index 39600ee22be30..b6f1cc9f57005 100644
--- a/drivers/clk/imx/clk-pll14xx.c
+++ b/drivers/clk/imx/clk-pll14xx.c
@@ -8,11 +8,13 @@
 #include <linux/bitfield.h>
 #include <linux/bits.h>
 #include <linux/clk-provider.h>
+#include <linux/debugfs.h>
 #include <linux/err.h>
 #include <linux/export.h>
 #include <linux/io.h>
 #include <linux/iopoll.h>
 #include <linux/slab.h>
+#include <linux/spinlock.h>
 #include <linux/jiffies.h>
 
 #include "clk.h"
@@ -40,6 +42,8 @@ struct clk_pll14xx {
 	enum imx_pll14xx_type		type;
 	const struct imx_pll14xx_rate_table *rate_table;
 	int rate_count;
+	s16 delta_k;
+	spinlock_t lock;
 };
 
 #define to_clk_pll14xx(_hw) container_of(_hw, struct clk_pll14xx, hw)
@@ -134,6 +138,7 @@ static void imx_pll14xx_calc_settings(struct clk_pll14xx *pll, unsigned long rat
 	u32 pll_div_ctl0, pll_div_ctl1;
 	int mdiv, pdiv, sdiv, kdiv;
 	long fout, rate_min, rate_max, dist, best = LONG_MAX;
+	unsigned long flags;
 	const struct imx_pll14xx_rate_table *tt;
 
 	/*
@@ -161,11 +166,16 @@ static void imx_pll14xx_calc_settings(struct clk_pll14xx *pll, unsigned long rat
 		return;
 	}
 
+	spin_lock_irqsave(&pll->lock, flags);
+
 	pll_div_ctl0 = readl_relaxed(pll->base + DIV_CTL0);
+	pll_div_ctl1 = readl_relaxed(pll->base + DIV_CTL1);
+
+	spin_unlock_irqrestore(&pll->lock, flags);
+
 	mdiv = FIELD_GET(MDIV_MASK, pll_div_ctl0);
 	pdiv = FIELD_GET(PDIV_MASK, pll_div_ctl0);
 	sdiv = FIELD_GET(SDIV_MASK, pll_div_ctl0);
-	pll_div_ctl1 = readl_relaxed(pll->base + DIV_CTL1);
 
 	/* Then see if we can get the desired rate by only adjusting kdiv (glitch free) */
 	rate_min = pll14xx_calc_rate(pll, mdiv, pdiv, sdiv, KDIV_MIN, prate);
@@ -361,11 +371,14 @@ static int clk_pll1443x_set_rate(struct clk_hw *hw, unsigned long drate,
 {
 	struct clk_pll14xx *pll = to_clk_pll14xx(hw);
 	struct imx_pll14xx_rate_table rate;
+	unsigned long flags;
 	u32 gnrl_ctl, div_ctl0;
 	int ret;
 
 	imx_pll14xx_calc_settings(pll, drate, prate, &rate);
 
+	spin_lock_irqsave(&pll->lock, flags);
+
 	div_ctl0 = readl_relaxed(pll->base + DIV_CTL0);
 
 	if (!clk_pll14xx_mp_change(&rate, div_ctl0)) {
@@ -377,6 +390,8 @@ static int clk_pll1443x_set_rate(struct clk_hw *hw, unsigned long drate,
 		writel_relaxed(FIELD_PREP(KDIV_MASK, rate.kdiv),
 			       pll->base + DIV_CTL1);
 
+		spin_unlock_irqrestore(&pll->lock, flags);
+
 		return 0;
 	}
 
@@ -396,6 +411,8 @@ static int clk_pll1443x_set_rate(struct clk_hw *hw, unsigned long drate,
 
 	writel_relaxed(FIELD_PREP(KDIV_MASK, rate.kdiv), pll->base + DIV_CTL1);
 
+	spin_unlock_irqrestore(&pll->lock, flags);
+
 	/*
 	 * According to SPEC, t3 - t2 need to be greater than
 	 * 1us and 1/FREF, respectively.
@@ -508,6 +525,8 @@ struct clk_hw *imx_dev_clk_hw_pll14xx(struct device *dev, const char *name,
 	if (!pll)
 		return ERR_PTR(-ENOMEM);
 
+	spin_lock_init(&pll->lock);
+
 	init.name = name;
 	init.flags = pll_clk->flags;
 	init.parent_names = &parent_name;
@@ -551,3 +570,101 @@ struct clk_hw *imx_dev_clk_hw_pll14xx(struct device *dev, const char *name,
 	return hw;
 }
 EXPORT_SYMBOL_GPL(imx_dev_clk_hw_pll14xx);
+
+/*
+ * Debugfs interface for Audio PLL runtime monitoring and control
+ *
+ * This interface allows dynamic adjustment of the Audio PLL
+ * K-divider for precise frequency tuning, particularly useful
+ * for audio applications.
+ *
+ * examples for the usage of the two interfaces:
+ *   1): Get the current PLL setting of dividers
+ *     cat /sys/kernel/debug/audio_pll_monitor/audio_pll1/pll_parameter
+ *
+ *   2): Adjust the K-divider by a small delta_k
+ *     echo 1 > /sys/kernel/debug/audio_pll_monitor/audio_pll1/delta_k;
+ */
+#ifdef CONFIG_DEBUG_FS
+static int pll_delta_k_get(void *data, u64 *val)
+{
+	struct clk_pll14xx *pll = to_clk_pll14xx(data);
+	*val = pll->delta_k;
+	return 0;
+}
+
+static int pll_delta_k_set(void *data, u64 val)
+{
+	struct clk_pll14xx *pll = to_clk_pll14xx(data);
+	unsigned long flags;
+	u32 div_ctl1;
+	s16 kdiv, delta_k;
+
+	delta_k = (s16)clamp_t(s64, val, KDIV_MIN, KDIV_MAX);
+
+	spin_lock_irqsave(&pll->lock, flags);
+
+	pll->delta_k = delta_k;
+
+	div_ctl1 = readl_relaxed(pll->base + DIV_CTL1);
+	kdiv = (s16)FIELD_GET(KDIV_MASK, div_ctl1);
+	kdiv = (s16)clamp_t(s32, (s32)kdiv + delta_k, KDIV_MIN, KDIV_MAX);
+	writel_relaxed(FIELD_PREP(KDIV_MASK, kdiv), pll->base + DIV_CTL1);
+
+	spin_unlock_irqrestore(&pll->lock, flags);
+
+	return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE_SIGNED(delta_k_fops, pll_delta_k_get, pll_delta_k_set, "%lld\n");
+
+static int pll_setting_show(struct seq_file *s, void *data)
+{
+	struct clk_pll14xx *pll = to_clk_pll14xx(s->private);
+	unsigned long flags;
+	u32 div_ctl0, div_ctl1;
+	u32 mdiv, pdiv, sdiv, kdiv;
+
+	spin_lock_irqsave(&pll->lock, flags);
+
+	div_ctl0 = readl_relaxed(pll->base + DIV_CTL0);
+	div_ctl1 = readl_relaxed(pll->base + DIV_CTL1);
+
+	spin_unlock_irqrestore(&pll->lock, flags);
+
+	mdiv = FIELD_GET(MDIV_MASK, div_ctl0);
+	pdiv = FIELD_GET(PDIV_MASK, div_ctl0);
+	sdiv = FIELD_GET(SDIV_MASK, div_ctl0);
+	kdiv = FIELD_GET(KDIV_MASK, div_ctl1);
+
+	seq_printf(s, "Mdiv: 0x%x; Pdiv: 0x%x; Sdiv: 0x%x; Kdiv: 0x%x\n",
+		   mdiv, pdiv, sdiv, kdiv);
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(pll_setting);
+
+void imx_audio_pll_debug_init(struct clk_hw *hws[], unsigned int num_plls)
+{
+	struct dentry *rootdir, *audio_pll_dir;
+	const char *pll_name;
+	int i;
+
+	rootdir = debugfs_create_dir("audio_pll_monitor", NULL);
+
+	for (i = 0; i < num_plls; i++) {
+		if (!IS_ERR_OR_NULL(hws[i])) {
+			pll_name = clk_hw_get_name(hws[i]);
+			audio_pll_dir = debugfs_create_dir(pll_name, rootdir);
+			debugfs_create_file_unsafe("delta_k", 0600, audio_pll_dir,
+						   hws[i], &delta_k_fops);
+			debugfs_create_file("pll_parameter", 0444, audio_pll_dir,
+					    hws[i], &pll_setting_fops);
+		}
+	}
+}
+#else /* !CONFIG_DEBUG_FS */
+void imx_audio_pll_debug_init(struct clk_hw *hws[], unsigned int num_plls)
+{
+}
+#endif /* CONFIG_DEBUG_FS */
+EXPORT_SYMBOL_GPL(imx_audio_pll_debug_init);
diff --git a/drivers/clk/imx/clk.h b/drivers/clk/imx/clk.h
index aa5202f284f3d..40bb41e353f93 100644
--- a/drivers/clk/imx/clk.h
+++ b/drivers/clk/imx/clk.h
@@ -487,4 +487,5 @@ struct clk_hw *imx_clk_gpr_mux(const char *name, const char *compatible,
 			       u32 reg, const char **parent_names,
 			       u8 num_parents, const u32 *mux_table, u32 mask);
 
+void imx_audio_pll_debug_init(struct clk_hw *hws[], unsigned int num_plls);
 #endif
diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig
index 7d84c2f1d911a..63f271aff1771 100644
--- a/drivers/clk/qcom/Kconfig
+++ b/drivers/clk/qcom/Kconfig
@@ -23,6 +23,7 @@ config CLK_ELIZA_DISPCC
 	tristate "Eliza Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select CLK_ELIZA_GCC
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controllers on Eliza SoCs.
 	  Say Y if you want to support display devices and functionality such as
@@ -32,6 +33,7 @@ config CLK_ELIZA_GCC
 	tristate "Eliza Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on Eliza devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -41,35 +43,61 @@ config CLK_ELIZA_TCSRCC
 	tristate "Eliza TCSR Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the TCSR clock controller on Eliza devices.
 	  Say Y if you want to use peripheral devices such as USB/PCIe/UFS.
 
+config CLK_GLYMUR_CAMCC
+	tristate "Glymur Camera Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select CLK_GLYMUR_GCC
+	default m if ARCH_QCOM
+	help
+	  Support for the camera clock controller on Qualcomm Technologies, Inc
+	  Glymur devices.
+	  Say Y if you want to support camera devices and functionality such as
+	  capturing pictures.
+
 config CLK_GLYMUR_DISPCC
 	tristate "Glymur Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select CLK_GLYMUR_GCC
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controllers on Qualcomm
 	  Technologies, Inc. Glymur devices.
 	  Say Y if you want to support display devices and functionality such as
 	  splash screen.
 
+config CLK_GLYMUR_EVACC
+	tristate "Glymur EVA Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	default m if ARCH_QCOM
+	select CLK_GLYMUR_GCC
+	default m if ARCH_QCOM
+	help
+	  Support for the Enhanced Video Analytics (EVA) clock controller on
+	  Qualcomm Technologies, Inc. Glymur devices.
+	  Say Y if you want to support EVA devices and functionality such as
+	  vision processing.
+
 config CLK_GLYMUR_GCC
 	tristate "Glymur Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on Glymur devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
 	  I2C, USB, UFS, SDCC, etc.
 
 config CLK_GLYMUR_GPUCC
-	tristate "GLYMUR Graphics Clock Controller"
+	tristate "Glymur Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select CLK_GLYMUR_GCC
 	help
-	  Support for the graphics clock controller on GLYMUR devices.
+	  Support for the graphics clock controller on Glymur devices.
 	  Say Y if you want to support graphics controller devices and
 	  functionality such as 3D graphics.
 
@@ -77,6 +105,7 @@ config CLK_GLYMUR_TCSRCC
 	tristate "Glymur TCSR Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the TCSR clock controller on Glymur devices.
 	  Say Y if you want to use peripheral devices such as USB/PCIe/EDP.
@@ -85,6 +114,7 @@ config CLK_GLYMUR_VIDEOCC
 	tristate "Glymur Video Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select CLK_GLYMUR_GCC
+	default m if ARCH_QCOM
 	help
 	  Support for the video clock controller on Glymur devices.
 	  Say Y if you want to support video devices and functionality such as
@@ -94,6 +124,7 @@ config CLK_KAANAPALI_CAMCC
 	tristate "Kaanapali Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select CLK_KAANAPALI_GCC
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on Qualcomm Technologies, Inc
 	  Kaanapali devices.
@@ -104,6 +135,7 @@ config CLK_KAANAPALI_DISPCC
 	tristate "Kaanapali Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select CLK_KAANAPALI_GCC
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controller on Qualcomm Technologies, Inc
 	  Kaanapali devices.
@@ -114,6 +146,7 @@ config CLK_KAANAPALI_GCC
 	tristate "Kaanapali Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on Kaanapali devices.
 	  Say Y if you want to use peripheral devices such as UART,
@@ -123,6 +156,7 @@ config CLK_KAANAPALI_GPUCC
 	tristate "Kaanapali Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select CLK_KAANAPALI_GCC
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on Kaanapali devices.
 	  Say Y if you want to support graphics controller devices and
@@ -132,6 +166,7 @@ config CLK_KAANAPALI_TCSRCC
 	tristate "Kaanapali TCSR Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the TCSR clock controller on Kaanapali devices.
 	  Say Y if you want to use peripheral devices such as PCIe, USB, UFS.
@@ -140,6 +175,7 @@ config CLK_KAANAPALI_VIDEOCC
 	tristate "Kaanapali Video Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select CLK_KAANAPALI_GCC
+	default m if ARCH_QCOM
 	help
 	  Support for the video clock controller on Kaanapali devices.
 	  Say Y if you want to support video devices and functionality such as
@@ -149,16 +185,28 @@ config CLK_NORD_GCC
 	tristate "Nord Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on Nord devices.
 	  Say Y if you want to use peripheral devices such as UART,
 	  SPI, I2C, USB, SD/UFS, PCIe etc. The clock controller is a combination
 	  of GCC, SE_GCC, NE_GCC and NW_GCC.
 
+config CLK_SHIKRA_GCC
+	tristate "Shikra Global Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select QCOM_GDSC
+	default ARCH_QCOM
+	help
+	  Support for the global clock controller on Shikra devices.
+	  Say Y if you want to use multimedia devices or peripheral
+	  devices such as Camera, Video, UART, SPI, I2C, USB, SD/eMMC etc.
+
 config CLK_X1E80100_CAMCC
 	tristate "X1E80100 Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select CLK_X1E80100_GCC
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on X1E80100 devices.
 	  Say Y if you want to support camera devices and camera functionality.
@@ -167,6 +215,7 @@ config CLK_X1E80100_DISPCC
 	tristate "X1E80100 Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select CLK_X1E80100_GCC
+	default m if ARCH_QCOM
 	help
 	  Support for the two display clock controllers on Qualcomm
 	  Technologies, Inc. X1E80100 devices.
@@ -177,6 +226,7 @@ config CLK_X1E80100_GCC
 	tristate "X1E80100 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on Qualcomm Technologies, Inc
 	  X1E80100 devices.
@@ -187,6 +237,7 @@ config CLK_X1E80100_GPUCC
 	tristate "X1E80100 Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select CLK_X1E80100_GCC
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on X1E80100 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -196,6 +247,7 @@ config CLK_X1E80100_TCSRCC
 	tristate "X1E80100 TCSR Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the TCSR clock controller on X1E80100 devices.
 	  Say Y if you want to use peripheral devices such as SD/UFS.
@@ -215,6 +267,7 @@ config CLK_X1P42100_GPUCC
 	tristate "X1P42100 Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select CLK_X1E80100_GCC
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on X1P42100 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -235,6 +288,7 @@ config CLK_QCM2290_GPUCC
 	tristate "QCM2290 Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCM_GCC_2290
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on QCM2290 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -242,6 +296,7 @@ config CLK_QCM2290_GPUCC
 
 config QCOM_A53PLL
 	tristate "MSM8916 A53 PLL"
+	default ARCH_QCOM
 	help
 	  Support for the A53 PLL on MSM8916 devices. It provides
 	  the CPU with frequencies above 1GHz.
@@ -250,6 +305,8 @@ config QCOM_A53PLL
 
 config QCOM_A7PLL
 	tristate "A7 PLL driver for SDX55 and SDX65"
+	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for the A7 PLL on SDX55 and SDX65 devices. It provides the CPU with
 	  frequencies above 1GHz.
@@ -259,6 +316,7 @@ config QCOM_A7PLL
 config QCOM_CLK_APCS_MSM8916
 	tristate "MSM8916 APCS Clock Controller"
 	depends on QCOM_APCS_IPC || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for the APCS Clock Controller on msm8916 devices. The
 	  APCS is managing the mux and divider which feeds the CPUs.
@@ -270,6 +328,7 @@ config QCOM_CLK_APCC_MSM8996
 	select QCOM_KRYO_L2_ACCESSORS
 	select INTERCONNECT_CLK if INTERCONNECT
 	depends on ARM64
+	default ARCH_QCOM
 	help
 	  Support for the CPU clock controller on msm8996 devices.
 	  Say Y if you want to support CPU clock scaling using CPUfreq
@@ -279,6 +338,7 @@ config QCOM_CLK_APCS_SDX55
 	tristate "SDX55 and SDX65 APCS Clock Controller"
 	depends on QCOM_APCS_IPC || COMPILE_TEST
 	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for the APCS Clock Controller on SDX55, SDX65 platforms. The
 	  APCS is managing the mux and divider which feeds the CPUs.
@@ -288,6 +348,7 @@ config QCOM_CLK_APCS_SDX55
 config QCOM_CLK_RPM
 	tristate "RPM based Clock Controller"
 	depends on MFD_QCOM_RPM
+	default ARCH_QCOM
 	help
 	  The RPM (Resource Power Manager) is a dedicated hardware engine for
 	  managing the shared SoC resources in order to keep the lowest power
@@ -300,6 +361,7 @@ config QCOM_CLK_RPM
 config QCOM_CLK_SMD_RPM
 	tristate "RPM over SMD based Clock Controller"
 	depends on QCOM_SMD_RPM
+	default ARCH_QCOM
 	help
 	  The RPM (Resource Power Manager) is a dedicated hardware engine for
 	  managing the shared SoC resources in order to keep the lowest power
@@ -312,6 +374,7 @@ config QCOM_CLK_SMD_RPM
 config QCOM_CLK_RPMH
 	tristate "RPMh Clock Driver"
 	depends on QCOM_RPMH
+	default ARCH_QCOM
 	help
 	 RPMh manages shared resources on some Qualcomm Technologies, Inc.
 	 SoCs. It accepts requests from other hardware subsystems via RSC.
@@ -322,6 +385,7 @@ config CLK_HAWI_GCC
 	tristate "Hawi Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on Hawi devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -330,14 +394,26 @@ config CLK_HAWI_GCC
 config CLK_HAWI_TCSRCC
 	tristate "Hawi TCSR Clock Controller"
 	depends on ARM64 || COMPILE_TEST
+	default m if ARCH_QCOM
 	help
 	  Support for the TCSR clock controller on Hawi devices.
 	  Say Y if you want to use peripheral devices such as PCIe, USB, UFS.
 
+config CLK_HAWI_VIDEOCC
+	tristate "Hawi Video Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select CLK_HAWI_GCC
+	default m if ARCH_QCOM
+	help
+	  Support for the video clock controller on Hawi devices.
+	  Say Y if you want to support video devices and functionality such as
+	  video encode/decode.
+
 config APQ_GCC_8084
 	tristate "APQ8084 Global Clock Controller"
 	depends on ARM || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on apq8084 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -348,6 +424,7 @@ config APQ_MMCC_8084
 	depends on ARM || COMPILE_TEST
 	select APQ_GCC_8084
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the multimedia clock controller on apq8084 devices.
 	  Say Y if you want to support multimedia devices such as display,
@@ -363,6 +440,7 @@ config IPQ_APSS_PLL
 
 config IPQ_APSS_5424
 	tristate "IPQ5424 APSS Clock Controller"
+	depends on ARM64 || COMPILE_TEST
 	select IPQ_APSS_PLL
 	default y if IPQ_GCC_5424
 	help
@@ -373,8 +451,10 @@ config IPQ_APSS_5424
 config IPQ_APSS_6018
 	tristate "IPQ6018 APSS Clock Controller"
 	select IPQ_APSS_PLL
+	depends on ARM64 || COMPILE_TEST
 	depends on QCOM_APCS_IPC || COMPILE_TEST
 	depends on QCOM_SMEM
+	default ARCH_QCOM
 	help
 	  Support for APSS clock controller on IPQ platforms. The
 	  APSS clock controller manages the Mux and enable block that feeds the
@@ -384,6 +464,8 @@ config IPQ_APSS_6018
 
 config IPQ_CMN_PLL
 	tristate "IPQ CMN PLL Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	default m if ARCH_QCOM
 	help
 	  Support for CMN PLL clock controller on IPQ platform. The
 	  CMN PLL consumes the AHB/SYS clocks from GCC and supplies
@@ -393,6 +475,8 @@ config IPQ_CMN_PLL
 
 config IPQ_GCC_4019
 	tristate "IPQ4019 Global Clock Controller"
+	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on ipq4019 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -401,6 +485,7 @@ config IPQ_GCC_4019
 config IPQ_GCC_5018
 	tristate "IPQ5018 Global Clock Controller"
 	depends on ARM || ARM64 || COMPILE_TEST
+	default ARCH_QCOM && ARM64
 	help
 	  Support for global clock controller on ipq5018 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -409,6 +494,7 @@ config IPQ_GCC_5018
 config IPQ_GCC_5210
 	tristate "IPQ5210 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on ipq5210 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -417,6 +503,7 @@ config IPQ_GCC_5210
 config IPQ_GCC_5332
 	tristate "IPQ5332 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on ipq5332 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -425,6 +512,7 @@ config IPQ_GCC_5332
 config IPQ_GCC_5424
 	tristate "IPQ5424 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on ipq5424 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -432,6 +520,8 @@ config IPQ_GCC_5424
 
 config IPQ_GCC_6018
 	tristate "IPQ6018 Global Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for global clock controller on ipq6018 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -441,6 +531,7 @@ config IPQ_GCC_6018
 config IPQ_GCC_806X
 	tristate "IPQ806x Global Clock Controller"
 	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on ipq806x devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -450,6 +541,7 @@ config IPQ_LCC_806X
 	tristate "IPQ806x LPASS Clock Controller"
 	depends on ARM || COMPILE_TEST
 	select IPQ_GCC_806X
+	default ARCH_QCOM
 	help
 	  Support for the LPASS clock controller on ipq806x devices.
 	  Say Y if you want to use audio devices such as i2s, pcm,
@@ -457,7 +549,9 @@ config IPQ_LCC_806X
 
 config IPQ_GCC_8074
 	tristate "IPQ8074 Global Clock Controller"
+	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for global clock controller on ipq8074 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -466,6 +560,8 @@ config IPQ_GCC_8074
 
 config IPQ_GCC_9574
 	tristate "IPQ9574 Global Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for global clock controller on ipq9574 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -486,6 +582,7 @@ config IPQ_NSSCC_5424
 	tristate "IPQ5424 NSS Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	depends on IPQ_GCC_5424
+	default m if ARCH_QCOM
 	help
 	  Support for NSS clock controller on ipq5424 devices.
 	  NSSCC receives the clock sources from GCC, CMN PLL and UNIPHY (PCS).
@@ -497,6 +594,7 @@ config IPQ_NSSCC_9574
         tristate "IPQ9574 NSS Clock Controller"
         depends on ARM64 || COMPILE_TEST
         depends on IPQ_GCC_9574
+	default m if ARCH_QCOM
         help
           Support for NSS clock controller on ipq9574 devices.
 
@@ -512,6 +610,7 @@ config IPQ_NSSCC_QCA8K
 config MSM_GCC_8660
 	tristate "MSM8660 Global Clock Controller"
 	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on msm8660 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -529,6 +628,7 @@ config MSM_GCC_8909
 config MSM_GCC_8916
 	tristate "MSM8916 Global Clock Controller"
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on msm8916 devices.
 	  Say Y if you want to use devices such as UART, SPI i2c, USB,
@@ -555,6 +655,7 @@ config MSM_GCC_8939
 config MSM_GCC_8960
 	tristate "APQ8064/MSM8960 Global Clock Controller"
 	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on apq8064/msm8960 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -563,6 +664,7 @@ config MSM_GCC_8960
 config MSM_LCC_8960
 	tristate "APQ8064/MSM8960/MDM9650 LPASS Clock Controller"
 	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for the LPASS clock controller on apq8064/msm8960/mdm9650
 	  devices.
@@ -589,6 +691,7 @@ config MSM_MMCC_8960
 	tristate "MSM8960 Multimedia Clock Controller"
 	depends on ARM || COMPILE_TEST
 	select MSM_GCC_8960
+	default ARCH_QCOM
 	help
 	  Support for the multimedia clock controller on msm8960 devices.
 	  Say Y if you want to support multimedia devices such as display,
@@ -597,6 +700,7 @@ config MSM_MMCC_8960
 config MSM_GCC_8953
 	tristate "MSM8953 Global Clock Controller"
 	select QCOM_GDSC
+	default ARCH_QCOM if ARM64
 	help
 	  Support for the global clock controller on msm8953 devices.
 	  Say Y if you want to use devices such as UART, SPI i2c, USB,
@@ -606,6 +710,7 @@ config MSM_GCC_8974
 	tristate "MSM8974 Global Clock Controller"
 	depends on ARM || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on msm8974 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -616,6 +721,7 @@ config MSM_MMCC_8974
 	depends on ARM || COMPILE_TEST
 	select MSM_GCC_8974
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the multimedia clock controller on msm8974 devices.
 	  Say Y if you want to support multimedia devices such as display,
@@ -624,6 +730,7 @@ config MSM_MMCC_8974
 config MSM_GCC_8976
 	tristate "MSM8956/76 Global Clock Controller"
 	select QCOM_GDSC
+	default ARCH_QCOM if ARM64
 	help
 	  Support for the global clock controller on msm8956/76 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -633,6 +740,7 @@ config MSM_MMCC_8994
 	tristate "MSM8994 Multimedia Clock Controller"
 	select MSM_GCC_8994
 	select QCOM_GDSC
+	default m if ARCH_QCOM && ARM64
 	help
 	  Support for the multimedia clock controller on msm8994 devices.
 	  Say Y if you want to support multimedia devices such as display,
@@ -640,6 +748,7 @@ config MSM_MMCC_8994
 
 config MSM_GCC_8994
 	tristate "MSM8994 Global Clock Controller"
+	default ARCH_QCOM if ARM64
 	help
 	  Support for the global clock controller on msm8994 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -648,6 +757,7 @@ config MSM_GCC_8994
 config MSM_GCC_8996
 	tristate "MSM8996 Global Clock Controller"
 	select QCOM_GDSC
+	default ARCH_QCOM if ARM64
 	help
 	  Support for the global clock controller on msm8996 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -657,6 +767,7 @@ config MSM_MMCC_8996
 	tristate "MSM8996 Multimedia Clock Controller"
 	select MSM_GCC_8996
 	select QCOM_GDSC
+	default m if ARCH_QCOM && ARM64
 	help
 	  Support for the multimedia clock controller on msm8996 devices.
 	  Say Y if you want to support multimedia devices such as display,
@@ -665,6 +776,7 @@ config MSM_MMCC_8996
 config MSM_GCC_8998
 	tristate "MSM8998 Global Clock Controller"
 	select QCOM_GDSC
+	default ARCH_QCOM if ARM64
 	help
 	  Support for the global clock controller on msm8998 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -674,6 +786,7 @@ config MSM_GPUCC_8998
 	tristate "MSM8998 Graphics Clock Controller"
 	select MSM_GCC_8998
 	select QCOM_GDSC
+	default m if ARCH_QCOM && ARM64
 	help
 	  Support for the graphics clock controller on MSM8998 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -683,6 +796,7 @@ config MSM_MMCC_8998
 	tristate "MSM8998 Multimedia Clock Controller"
 	select MSM_GCC_8998
 	select QCOM_GDSC
+	default m if ARCH_QCOM && ARM64
 	help
 	  Support for the multimedia clock controller on msm8998 devices.
 	  Say Y if you want to support multimedia devices such as display,
@@ -690,7 +804,9 @@ config MSM_MMCC_8998
 
 config QCM_GCC_2290
 	tristate "QCM2290 Global Clock Controller"
+	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on QCM2290 devices.
 	  Say Y if you want to use multimedia devices or peripheral
@@ -698,7 +814,9 @@ config QCM_GCC_2290
 
 config QCM_DISPCC_2290
 	tristate "QCM2290 Display Clock Controller"
+	depends on ARM64 || COMPILE_TEST
 	select QCM_GCC_2290
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controller on Qualcomm Technologies, Inc
 	  QCM2290 devices.
@@ -709,6 +827,7 @@ config QCS_DISPCC_615
 	tristate "QCS615 Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCS_GCC_615
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controller on Qualcomm Technologies, Inc
 	  QCS615 devices.
@@ -719,6 +838,7 @@ config QCS_CAMCC_615
 	tristate "QCS615 Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCS_GCC_615
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on Qualcomm Technologies, Inc
 	  QCS615 devices.
@@ -727,6 +847,7 @@ config QCS_CAMCC_615
 
 config QCS_GCC_404
 	tristate "QCS404 Global Clock Controller"
+	default y if ARCH_QCOM && ARM64
 	help
 	  Support for the global clock controller on QCS404 devices.
 	  Say Y if you want to use multimedia devices or peripheral
@@ -735,6 +856,7 @@ config QCS_GCC_404
 config CLK_NORD_TCSRCC
 	tristate "Nord TCSR Clock Controller"
 	depends on ARM64 || COMPILE_TEST
+	default m if ARCH_QCOM
 	help
 	  Support for the TCSR clock controller on Nord devices.
 	  Say Y if you want to use peripheral devices such as PCIe, USB, UFS etc.
@@ -743,6 +865,7 @@ config SA_CAMCC_8775P
 	tristate "SA8775P Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SA_GCC_8775P
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on Qualcomm Technologies, Inc
 	  SA8775P devices.
@@ -753,6 +876,7 @@ config QCS_GCC_8300
 	tristate "QCS8300 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on Qualcomm Technologies, Inc
 	  QCS8300 devices.
@@ -763,6 +887,7 @@ config QCS_GCC_615
 	tristate "QCS615 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on QCS615 devices.
 	  Say Y if you want to use multimedia devices or peripheral
@@ -772,6 +897,7 @@ config QCS_GPUCC_615
 	tristate "QCS615 Graphics clock controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCS_GCC_615
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on QCS615 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -781,6 +907,7 @@ config QCS_VIDEOCC_615
 	tristate "QCS615 Video Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCS_GCC_615
+	default m if ARCH_QCOM
 	help
 	  Support for the video clock controller on QCS615 devices.
 	  Say Y if you want to support video devices and functionality such as
@@ -790,6 +917,7 @@ config SC_CAMCC_7180
 	tristate "SC7180 Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SC_GCC_7180
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on Qualcomm Technologies, Inc
 	  SC7180 devices.
@@ -800,6 +928,7 @@ config SC_CAMCC_7280
 	tristate "SC7280 Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SC_GCC_7280
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on Qualcomm Technologies, Inc
 	  SC7280 devices.
@@ -810,6 +939,7 @@ config SC_CAMCC_8180X
 	tristate "SC8180X Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SC_GCC_8180X
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on Qualcomm Technologies, Inc
 	  SC8180X devices.
@@ -820,6 +950,7 @@ config SC_CAMCC_8280XP
 	tristate "SC8280XP Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SC_GCC_8280XP
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on Qualcomm Technologies, Inc
 	  SC8280XP devices.
@@ -830,6 +961,7 @@ config SA_DISPCC_8775P
 	tristate "SA8775P Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SA_GCC_8775P
+	default m if ARCH_QCOM
 	help
 	  Support for the two display clock controllers on Qualcomm
 	  Technologies, Inc. SA8775P devices.
@@ -840,6 +972,7 @@ config SC_DISPCC_7180
 	tristate "SC7180 Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SC_GCC_7180
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controller on Qualcomm Technologies, Inc
 	  SC7180 devices.
@@ -850,6 +983,7 @@ config SC_DISPCC_7280
 	tristate "SC7280 Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SC_GCC_7280
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controller on Qualcomm Technologies, Inc.
 	  SC7280 devices.
@@ -860,6 +994,7 @@ config SC_DISPCC_8280XP
 	tristate "SC8280XP Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SC_GCC_8280XP
+	default m if ARCH_QCOM
 	help
 	  Support for the two display clock controllers on Qualcomm
 	  Technologies, Inc. SC8280XP devices.
@@ -869,7 +1004,9 @@ config SC_DISPCC_8280XP
 config SA_GCC_8775P
 	tristate "SA8775 Global Clock Controller"
 	select QCOM_GDSC
+	depends on ARM64 || COMPILE_TEST
 	depends on COMMON_CLK_QCOM
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SA8775 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -877,8 +1014,10 @@ config SA_GCC_8775P
 
 config SA_GPUCC_8775P
 	tristate "SA8775P Graphics clock controller"
+	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
 	select SA_GCC_8775P
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SA8775P devices.
 	  Say Y if you want to support graphics controller devices and
@@ -907,6 +1046,7 @@ config SC_GCC_7180
 	select QCOM_GDSC
 	depends on COMMON_CLK_QCOM
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SC7180 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -917,6 +1057,7 @@ config SC_GCC_7280
 	select QCOM_GDSC
 	depends on COMMON_CLK_QCOM
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SC7280 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -927,6 +1068,7 @@ config SC_GCC_8180X
 	select QCOM_GDSC
 	depends on COMMON_CLK_QCOM
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SC8180X devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -937,6 +1079,7 @@ config SC_GCC_8280XP
 	select QCOM_GDSC
 	depends on COMMON_CLK_QCOM
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SC8280XP devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -946,6 +1089,7 @@ config SC_GPUCC_7180
 	tristate "SC7180 Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SC_GCC_7180
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SC7180 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -955,6 +1099,7 @@ config SC_GPUCC_7280
 	tristate "SC7280 Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SC_GCC_7280
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SC7280 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -964,6 +1109,7 @@ config SC_GPUCC_8280XP
 	tristate "SC8280XP Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SC_GCC_8280XP
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SC8280XP devices.
 	  Say Y if you want to support graphics controller devices and
@@ -973,6 +1119,7 @@ config SC_LPASSCC_7280
 	tristate "SC7280 Low Power Audio Subsystem (LPASS) Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SC_GCC_7280
+	default m if ARCH_QCOM
 	help
 	  Support for the LPASS clock controller on SC7280 devices.
 	  Say Y if you want to use the LPASS branch clocks of the LPASS clock
@@ -982,6 +1129,7 @@ config SC_LPASSCC_8280XP
 	tristate "SC8280 Low Power Audio Subsystem (LPASS) Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SC_GCC_8280XP
+	default m if ARCH_QCOM
 	help
 	  Support for the LPASS clock controller on SC8280XP devices.
 	  Say Y if you want to use the LPASS branch clocks of the LPASS clock
@@ -1002,6 +1150,7 @@ config SC_LPASS_CORECC_7280
 	depends on ARM64 || COMPILE_TEST
 	select SC_GCC_7280
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the LPASS(Low Power Audio Subsystem) core and audio clock
 	  controller on SC7280 devices.
@@ -1012,6 +1161,7 @@ config SC_VIDEOCC_7180
 	tristate "SC7180 Video Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SC_GCC_7180
+	default m if ARCH_QCOM
 	help
 	  Support for the video clock controller on SC7180 devices.
 	  Say Y if you want to support video devices and functionality such as
@@ -1021,6 +1171,7 @@ config SC_VIDEOCC_7280
 	tristate "SC7280 Video Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SC_GCC_7280
+	default m if ARCH_QCOM
 	help
 	  Support for the video clock controller on SC7280 devices.
 	  Say Y if you want to support video devices and functionality such as
@@ -1030,6 +1181,7 @@ config SDM_CAMCC_845
 	tristate "SDM845 Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SDM_GCC_845
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on SDM845 devices.
 	  Say Y if you want to support camera devices and camera functionality.
@@ -1038,6 +1190,7 @@ config SDM_GCC_660
 	tristate "SDM660 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SDM660 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -1048,6 +1201,7 @@ config SDM_MMCC_660
 	depends on ARM64 || COMPILE_TEST
 	select SDM_GCC_660
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the multimedia clock controller on SDM660 devices.
 	  Say Y if you want to support multimedia devices such as display,
@@ -1058,6 +1212,7 @@ config SDM_GPUCC_660
 	depends on ARM64 || COMPILE_TEST
 	select SDM_GCC_660
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SDM630/636/660 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -1079,7 +1234,9 @@ config QCS_Q6SSTOP_404
 
 config QDU_GCC_1000
 	tristate "QDU1000/QRU1000 Global Clock Controller"
+	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on QDU1000 and
 	  QRU1000 devices. Say Y if you want to use peripheral
@@ -1089,6 +1246,7 @@ config QDU_ECPRICC_1000
 	tristate "QDU1000/QRU1000 ECPRI Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QDU_GCC_1000
+	default m if ARCH_QCOM
 	help
 	  Support for the ECPRI clock controller on QDU1000 and
 	  QRU1000 devices. Say Y if you want to support the ECPRI
@@ -1098,6 +1256,7 @@ config SDM_GCC_845
 	tristate "SDM845/SDM670 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SDM845 and SDM670 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -1107,6 +1266,7 @@ config SDM_GPUCC_845
 	tristate "SDM845 Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SDM_GCC_845
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SDM845 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -1117,6 +1277,7 @@ config SDM_VIDEOCC_845
 	depends on ARM64 || COMPILE_TEST
 	select SDM_GCC_845
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the video clock controller on SDM845 devices.
 	  Say Y if you want to support video devices and functionality such as
@@ -1126,6 +1287,7 @@ config SDM_DISPCC_845
 	tristate "SDM845 Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SDM_GCC_845
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controller on Qualcomm Technologies, Inc
 	  SDM845 devices.
@@ -1136,6 +1298,7 @@ config SDM_LPASSCC_845
 	tristate "SDM845 Low Power Audio Subsystem (LPAAS) Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SDM_GCC_845
+	default m if ARCH_QCOM
 	help
 	  Support for the LPASS clock controller on SDM845 devices.
 	  Say Y if you want to use the LPASS branch clocks of the LPASS clock
@@ -1145,6 +1308,7 @@ config SDX_GCC_55
 	tristate "SDX55 Global Clock Controller"
 	depends on ARM || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SDX55 devices.
 	  Say Y if you want to use peripheral devices such as UART,
@@ -1162,6 +1326,7 @@ config SDX_GCC_65
 config SDX_GCC_75
 	tristate "SDX75 Global Clock Controller"
 	select QCOM_GDSC
+	default ARCH_QCOM if ARM64
 	help
 	  Support for the global clock controller on SDX75 devices.
 	  Say Y if you want to use peripheral devices such as UART,
@@ -1171,6 +1336,7 @@ config SM_CAMCC_4450
 	tristate "SM4450 Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_4450
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on SM4450 devices.
 	  Say Y if you want to support camera devices and camera functionality.
@@ -1179,6 +1345,7 @@ config SM_CAMCC_6350
 	tristate "SM6350 Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_6350
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on SM6350 devices.
 	  Say Y if you want to support camera devices and camera functionality.
@@ -1187,6 +1354,7 @@ config SM_CAMCC_7150
 	tristate "SM7150 Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_7150
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on SM7150 devices.
 	  Say Y if you want to support camera devices and camera functionality.
@@ -1195,6 +1363,7 @@ config SM_CAMCC_MILOS
 	tristate "Milos Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_MILOS
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on Milos devices.
 	  Say Y if you want to support camera devices and camera functionality.
@@ -1203,6 +1372,7 @@ config SM_CAMCC_8150
 	tristate "SM8150 Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_8150
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on Qualcomm Technologies, Inc
 	  SM8150 devices.
@@ -1213,6 +1383,7 @@ config SM_CAMCC_8250
 	tristate "SM8250 Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_8250
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on SM8250 devices.
 	  Say Y if you want to support camera devices and camera functionality.
@@ -1221,6 +1392,7 @@ config SM_CAMCC_8450
 	tristate "SM8450 Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_8450
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on SM8450 or SM8475 devices.
 	  Say Y if you want to support camera devices and camera functionality.
@@ -1229,6 +1401,7 @@ config SM_CAMCC_8550
 	tristate "SM8550 Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_8550
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on SM8550 devices.
 	  Say Y if you want to support camera devices and camera functionality.
@@ -1237,6 +1410,7 @@ config SM_CAMCC_8650
 	tristate "SM8650 Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_8650
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on SM8650 devices.
 	  Say Y if you want to support camera devices and camera functionality.
@@ -1245,6 +1419,7 @@ config SM_CAMCC_8750
 	tristate "SM8750 Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_8750
+	default m if ARCH_QCOM
 	help
 	  Support for the camera clock controller on SM8750 devices.
 	  The camera clock controller has a separate cambist controller which
@@ -1255,6 +1430,7 @@ config SM_DISPCC_4450
 	tristate "SM4450 Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	depends on SM_GCC_4450
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controller on Qualcomm Technologies, Inc
 	  SM4450 devices.
@@ -1265,6 +1441,7 @@ config SM_DISPCC_6115
 	tristate "SM6115 Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	depends on SM_GCC_6115
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controller on Qualcomm Technologies, Inc
 	  SM6115/SM4250 devices.
@@ -1275,6 +1452,7 @@ config SM_DISPCC_6125
 	tristate "SM6125 Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	depends on SM_GCC_6125
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controller on Qualcomm Technologies, Inc
 	  SM6125 devices.
@@ -1285,6 +1463,7 @@ config SM_DISPCC_7150
 	tristate "SM7150 Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	depends on SM_GCC_7150
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controller on Qualcomm Technologies, Inc
 	  SM7150 devices.
@@ -1295,6 +1474,7 @@ config SM_DISPCC_8250
 	tristate "SM8150/SM8250/SM8350 Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	depends on SM_GCC_8150 || SM_GCC_8250 || SM_GCC_8350
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controller on Qualcomm Technologies, Inc
 	  SM8150/SM8250/SM8350 devices.
@@ -1305,6 +1485,7 @@ config SM_DISPCC_6350
 	tristate "SM6350 Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	depends on SM_GCC_6350
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controller on Qualcomm Technologies, Inc
 	  SM6350 devices.
@@ -1315,6 +1496,7 @@ config SM_DISPCC_6375
 	tristate "SM6375 Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	depends on SM_GCC_6375
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controller on Qualcomm Technologies, Inc
 	  SM6375 devices.
@@ -1325,6 +1507,7 @@ config SM_DISPCC_MILOS
 	tristate "Milos Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	depends on SM_GCC_MILOS
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controller on Qualcomm Technologies, Inc
 	  Milos devices.
@@ -1335,6 +1518,7 @@ config SM_DISPCC_8450
 	tristate "SM8450 Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	depends on SM_GCC_8450
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controller on Qualcomm Technologies, Inc
 	  SM8450 or SM8475 devices.
@@ -1345,6 +1529,7 @@ config SM_DISPCC_8550
 	tristate "SM8550 Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	depends on SM_GCC_8550 || SM_GCC_8650 || SAR_GCC_2130P
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controller on Qualcomm Technologies, Inc
 	  SAR2130P, SM8550 or SM8650 devices.
@@ -1356,6 +1541,7 @@ config SM_DISPCC_8750
 	depends on ARM64 || COMPILE_TEST
 	depends on SM_GCC_8750
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the display clock controller on Qualcomm Technologies, Inc
 	  SM8750 devices.
@@ -1366,6 +1552,7 @@ config SM_GCC_4450
 	tristate "SM4450 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SM4450 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -1375,6 +1562,7 @@ config SM_GCC_6115
 	tristate "SM6115 and SM4250 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SM6115 and SM4250 devices.
 	  Say Y if you want to use peripheral devices such as UART, SPI,
@@ -1384,6 +1572,7 @@ config SM_GCC_6125
 	tristate "SM6125 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SM6125 devices.
 	  Say Y if you want to use peripheral devices such as UART,
@@ -1393,6 +1582,7 @@ config SM_GCC_6350
 	tristate "SM6350 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SM6350 devices.
 	  Say Y if you want to use peripheral devices such as UART,
@@ -1402,6 +1592,7 @@ config SM_GCC_6375
 	tristate "SM6375 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SM6375 devices.
 	  Say Y if you want to use peripheral devices such as UART,
@@ -1411,6 +1602,7 @@ config SM_GCC_7150
 	tristate "SM7150 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SM7150 devices.
 	  Say Y if you want to use peripheral devices such as UART,
@@ -1420,6 +1612,7 @@ config SM_GCC_MILOS
 	tristate "Milos Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on Milos devices.
 	  Say Y if you want to use peripheral devices such as UART,
@@ -1429,6 +1622,7 @@ config SM_GCC_8150
 	tristate "SM8150 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SM8150 devices.
 	  Say Y if you want to use peripheral devices such as UART,
@@ -1438,6 +1632,7 @@ config SM_GCC_8250
 	tristate "SM8250 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SM8250 devices.
 	  Say Y if you want to use peripheral devices such as UART,
@@ -1447,6 +1642,7 @@ config SM_GCC_8350
 	tristate "SM8350 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SM8350 devices.
 	  Say Y if you want to use peripheral devices such as UART,
@@ -1456,6 +1652,7 @@ config SM_GCC_8450
 	tristate "SM8450 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SM8450 or SM8475
 	  devices.
@@ -1466,6 +1663,7 @@ config SM_GCC_8550
 	tristate "SM8550 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SM8550 devices.
 	  Say Y if you want to use peripheral devices such as UART,
@@ -1475,6 +1673,7 @@ config SM_GCC_8650
 	tristate "SM8650 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SM8650 devices.
 	  Say Y if you want to use peripheral devices such as UART,
@@ -1484,6 +1683,7 @@ config SM_GCC_8750
 	tristate "SM8750 Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the global clock controller on SM8750 devices.
 	  Say Y if you want to use peripheral devices such as UART,
@@ -1493,6 +1693,7 @@ config SM_GPUCC_4450
 	tristate "SM4450 Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_4450
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SM4450 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -1502,6 +1703,7 @@ config SM_GPUCC_6115
 	tristate "SM6115 Graphics Clock Controller"
 	select SM_GCC_6115
 	depends on ARM64 || COMPILE_TEST
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SM6115 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -1511,6 +1713,7 @@ config SM_GPUCC_6125
 	tristate "SM6125 Graphics Clock Controller"
 	select SM_GCC_6125
 	depends on ARM64 || COMPILE_TEST
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SM6125 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -1520,6 +1723,7 @@ config SM_GPUCC_6375
 	tristate "SM6375 Graphics Clock Controller"
 	select SM_GCC_6375
 	depends on ARM64 || COMPILE_TEST
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SM6375 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -1529,6 +1733,7 @@ config SM_GPUCC_6350
 	tristate "SM6350 Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_6350
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SM6350 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -1538,6 +1743,7 @@ config SM_GPUCC_MILOS
 	tristate "Milos Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_MILOS
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on Milos devices.
 	  Say Y if you want to support graphics controller devices and
@@ -1547,6 +1753,7 @@ config SM_GPUCC_8150
 	tristate "SM8150 Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_8150
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SM8150 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -1556,6 +1763,7 @@ config SM_GPUCC_8250
 	tristate "SM8250 Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_8250
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SM8250 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -1565,6 +1773,7 @@ config SM_GPUCC_8350
 	tristate "SM8350 Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_8350
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SM8350 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -1574,6 +1783,7 @@ config SM_GPUCC_8450
 	tristate "SM8450 Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_8450
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SM8450 or SM8475
 	  devices.
@@ -1584,6 +1794,7 @@ config SM_GPUCC_8550
 	tristate "SM8550 Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_8550
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SM8550 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -1593,6 +1804,7 @@ config SM_GPUCC_8650
 	tristate "SM8650 Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_8650
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SM8650 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -1602,6 +1814,7 @@ config SM_GPUCC_8750
 	tristate "SM8750 Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_8750
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on SM8750 devices.
 	  Say Y if you want to support graphics controller devices and
@@ -1611,6 +1824,7 @@ config SM_LPASSCC_6115
 	tristate "SM6115 Low Power Audio Subsystem (LPASS) Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_6115
+	default m if ARCH_QCOM
 	help
 	  Support for the LPASS clock controller on SM6115 devices.
 	  Say Y if you want to toggle LPASS-adjacent resets within
@@ -1620,6 +1834,7 @@ config SM_TCSRCC_8550
 	tristate "SM8550 TCSR Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the TCSR clock controller on SM8550 devices.
 	  Say Y if you want to use peripheral devices such as SD/UFS.
@@ -1628,6 +1843,7 @@ config SM_TCSRCC_8650
 	tristate "SM8650 TCSR Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default ARCH_QCOM
 	help
 	  Support for the TCSR clock controller on SM8650 devices.
 	  Say Y if you want to use peripheral devices such as SD/UFS.
@@ -1636,6 +1852,7 @@ config SM_TCSRCC_8750
 	tristate "SM8750 TCSR Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the TCSR clock controller on SM8750 devices.
 	  Say Y if you want to use peripheral devices such as UFS/USB/PCIe.
@@ -1645,6 +1862,7 @@ config SA_VIDEOCC_8775P
 	depends on ARM64 || COMPILE_TEST
 	select SA_GCC_8775P
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the video clock controller on Qualcomm Technologies, Inc.
 	  SA8775P devices.
@@ -1656,6 +1874,7 @@ config SM_VIDEOCC_6350
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_6350
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the video clock controller on SM6350 devices.
 	  Say Y if you want to support video devices and functionality such as
@@ -1666,6 +1885,7 @@ config SM_VIDEOCC_7150
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_7150
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the video clock controller on SM7150 devices.
 	  Say Y if you want to support video devices and functionality such as
@@ -1676,6 +1896,7 @@ config SM_VIDEOCC_MILOS
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_MILOS
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the video clock controller on Qualcomm Technologies, Inc.
 	  Milos devices.
@@ -1687,6 +1908,7 @@ config SM_VIDEOCC_8150
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_8150
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the video clock controller on SM8150 devices.
 	  Say Y if you want to support video devices and functionality such as
@@ -1697,6 +1919,7 @@ config SM_VIDEOCC_8250
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_8250
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the video clock controller on SM8250 devices.
 	  Say Y if you want to support video devices and functionality such as
@@ -1707,6 +1930,7 @@ config SM_VIDEOCC_8350
 	depends on ARM64 || COMPILE_TEST
 	depends on SM_GCC_8350 || SC_GCC_8280XP
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the video clock controller on SM8350 or SC8280XP devices.
 	  Say Y if you want to support video devices and functionality such as
@@ -1716,6 +1940,7 @@ config SM_VIDEOCC_8550
 	tristate "SM8550 Video Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the video clock controller on Qualcomm Technologies, Inc.
 	  SM8550 or SM8650 or X1E80100 devices.
@@ -1727,6 +1952,7 @@ config SM_VIDEOCC_8750
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_8750
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the video clock controller on Qualcomm Technologies, Inc.
 	  SM8750 devices.
@@ -1767,6 +1993,7 @@ config KRAITCC
 config CLK_GFM_LPASS_SM8250
 	tristate "SM8250 GFM LPASS Clocks"
 	depends on ARM64 || COMPILE_TEST
+	default m if ARCH_QCOM
 	help
 	  Support for the Glitch Free Mux (GFM) Low power audio
           subsystem (LPASS) clocks found on SM8250 SoCs.
@@ -1776,6 +2003,7 @@ config SM_VIDEOCC_8450
 	depends on ARM64 || COMPILE_TEST
 	select SM_GCC_8450
 	select QCOM_GDSC
+	default m if ARCH_QCOM
 	help
 	  Support for the video clock controller on Qualcomm Technologies, Inc.
 	  SM8450 or SM8475 devices.
diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile
index 58f9a5eb6fd7f..db7689ee5dc63 100644
--- a/drivers/clk/qcom/Makefile
+++ b/drivers/clk/qcom/Makefile
@@ -24,13 +24,16 @@ obj-$(CONFIG_CLK_ELIZA_DISPCC) += dispcc-eliza.o
 obj-$(CONFIG_CLK_ELIZA_GCC) += gcc-eliza.o
 obj-$(CONFIG_CLK_ELIZA_TCSRCC) += tcsrcc-eliza.o
 obj-$(CONFIG_CLK_GFM_LPASS_SM8250) += lpass-gfm-sm8250.o
+obj-$(CONFIG_CLK_GLYMUR_CAMCC) += camcc-glymur.o
 obj-$(CONFIG_CLK_GLYMUR_DISPCC) += dispcc-glymur.o
+obj-$(CONFIG_CLK_GLYMUR_EVACC) += evacc-glymur.o
 obj-$(CONFIG_CLK_GLYMUR_GCC) += gcc-glymur.o
 obj-$(CONFIG_CLK_GLYMUR_GPUCC) += gpucc-glymur.o gxclkctl-kaanapali.o
 obj-$(CONFIG_CLK_GLYMUR_TCSRCC) += tcsrcc-glymur.o
 obj-$(CONFIG_CLK_GLYMUR_VIDEOCC) += videocc-glymur.o
 obj-$(CONFIG_CLK_HAWI_GCC) += gcc-hawi.o
 obj-$(CONFIG_CLK_HAWI_TCSRCC) += tcsrcc-hawi.o
+obj-$(CONFIG_CLK_HAWI_VIDEOCC) += videocc-hawi.o
 obj-$(CONFIG_CLK_KAANAPALI_CAMCC) += cambistmclkcc-kaanapali.o camcc-kaanapali.o
 obj-$(CONFIG_CLK_KAANAPALI_DISPCC) += dispcc-kaanapali.o
 obj-$(CONFIG_CLK_KAANAPALI_GCC) += gcc-kaanapali.o
@@ -39,6 +42,7 @@ obj-$(CONFIG_CLK_KAANAPALI_TCSRCC) += tcsrcc-kaanapali.o
 obj-$(CONFIG_CLK_KAANAPALI_VIDEOCC) += videocc-kaanapali.o
 obj-$(CONFIG_CLK_NORD_GCC) += gcc-nord.o negcc-nord.o nwgcc-nord.o segcc-nord.o
 obj-$(CONFIG_CLK_NORD_TCSRCC) += tcsrcc-nord.o
+obj-$(CONFIG_CLK_SHIKRA_GCC) += gcc-shikra.o
 obj-$(CONFIG_CLK_X1E80100_CAMCC) += camcc-x1e80100.o
 obj-$(CONFIG_CLK_X1E80100_DISPCC) += dispcc-x1e80100.o
 obj-$(CONFIG_CLK_X1E80100_GCC) += gcc-x1e80100.o
diff --git a/drivers/clk/qcom/camcc-glymur.c b/drivers/clk/qcom/camcc-glymur.c
new file mode 100644
index 0000000000000..a80f05b7bc484
--- /dev/null
+++ b/drivers/clk/qcom/camcc-glymur.c
@@ -0,0 +1,2280 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,glymur-camcc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "common.h"
+#include "gdsc.h"
+#include "reset.h"
+
+enum {
+	DT_IFACE,
+	DT_BI_TCXO,
+	DT_BI_TCXO_AO,
+	DT_SLEEP_CLK,
+};
+
+enum {
+	P_BI_TCXO,
+	P_BI_TCXO_AO,
+	P_CAM_CC_PLL0_OUT_EVEN,
+	P_CAM_CC_PLL0_OUT_MAIN,
+	P_CAM_CC_PLL0_OUT_ODD,
+	P_CAM_CC_PLL1_OUT_EVEN,
+	P_CAM_CC_PLL2_OUT_EVEN,
+	P_CAM_CC_PLL2_OUT_MAIN,
+	P_CAM_CC_PLL3_OUT_EVEN,
+	P_CAM_CC_PLL4_OUT_EVEN,
+	P_CAM_CC_PLL5_OUT_EVEN,
+	P_SLEEP_CLK,
+};
+
+static const struct pll_vco rivian_eko_t_vco[] = {
+	{ 883200000, 1171200000, 0 },
+};
+
+static const struct pll_vco taycan_eko_t_vco[] = {
+	{ 249600000, 2500000000, 0 },
+};
+
+/* 1200.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll0_config = {
+	.l = 0x3e,
+	.alpha = 0x8000,
+	.config_ctl_val = 0x25c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00008408,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll cam_cc_pll0 = {
+	.offset = 0x0,
+	.config = &cam_cc_pll0_config,
+	.vco_table = taycan_eko_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eko_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll0",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eko_t_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll0_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll0_out_even = {
+	.offset = 0x0,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll0_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll0_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll0_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eko_t_ops,
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll0_out_odd[] = {
+	{ 0x2, 3 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll0_out_odd = {
+	.offset = 0x0,
+	.post_div_shift = 14,
+	.post_div_table = post_div_table_cam_cc_pll0_out_odd,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll0_out_odd),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll0_out_odd",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eko_t_ops,
+	},
+};
+
+/* 608.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll1_config = {
+	.l = 0x1f,
+	.alpha = 0xaaaa,
+	.config_ctl_val = 0x25c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000408,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll cam_cc_pll1 = {
+	.offset = 0x1000,
+	.config = &cam_cc_pll1_config,
+	.vco_table = taycan_eko_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eko_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll1",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eko_t_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll1_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll1_out_even = {
+	.offset = 0x1000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll1_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll1_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll1_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll1.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eko_t_ops,
+	},
+};
+
+/* 960.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll2_config = {
+	.l = 0x32,
+	.alpha = 0x0,
+	.config_ctl_val = 0x12000000,
+	.config_ctl_hi_val = 0x00890263,
+	.config_ctl_hi1_val = 0x1af04237,
+	.config_ctl_hi2_val = 0x00000000,
+};
+
+static struct clk_alpha_pll cam_cc_pll2 = {
+	.offset = 0x2000,
+	.config = &cam_cc_pll2_config,
+	.vco_table = rivian_eko_t_vco,
+	.num_vco = ARRAY_SIZE(rivian_eko_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_RIVIAN_EKO_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll2",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_rivian_eko_t_ops,
+		},
+	},
+};
+
+/* 691.2 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll3_config = {
+	.l = 0x24,
+	.alpha = 0x0,
+	.config_ctl_val = 0x25c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000408,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll cam_cc_pll3 = {
+	.offset = 0x3000,
+	.config = &cam_cc_pll3_config,
+	.vco_table = taycan_eko_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eko_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll3",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eko_t_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll3_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll3_out_even = {
+	.offset = 0x3000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll3_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll3_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll3_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll3.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eko_t_ops,
+	},
+};
+
+/* 691.2 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll4_config = {
+	.l = 0x24,
+	.alpha = 0x0,
+	.config_ctl_val = 0x25c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000408,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll cam_cc_pll4 = {
+	.offset = 0x4000,
+	.config = &cam_cc_pll4_config,
+	.vco_table = taycan_eko_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eko_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll4",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eko_t_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll4_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll4_out_even = {
+	.offset = 0x4000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll4_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll4_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll4_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll4.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eko_t_ops,
+	},
+};
+
+/* 960.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll5_config = {
+	.l = 0x32,
+	.alpha = 0x0,
+	.config_ctl_val = 0x25c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000408,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll cam_cc_pll5 = {
+	.offset = 0x5000,
+	.config = &cam_cc_pll5_config,
+	.vco_table = taycan_eko_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eko_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll5",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eko_t_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll5_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll5_out_even = {
+	.offset = 0x5000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll5_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll5_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll5_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll5.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eko_t_ops,
+	},
+};
+
+static const struct parent_map cam_cc_parent_map_0[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL0_OUT_MAIN, 1 },
+	{ P_CAM_CC_PLL0_OUT_EVEN, 2 },
+	{ P_CAM_CC_PLL0_OUT_ODD, 3 },
+	{ P_CAM_CC_PLL5_OUT_EVEN, 5 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_0[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll0.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_even.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_odd.clkr.hw },
+	{ .hw = &cam_cc_pll5_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_1[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL2_OUT_EVEN, 3 },
+	{ P_CAM_CC_PLL2_OUT_MAIN, 5 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_1[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll2.clkr.hw },
+	{ .hw = &cam_cc_pll2.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_2[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL3_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_2[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll3_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_3[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL4_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_3[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll4_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_4[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL1_OUT_EVEN, 4 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_4[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll1_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_5[] = {
+	{ P_SLEEP_CLK, 0 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_5[] = {
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct parent_map cam_cc_parent_map_6_ao[] = {
+	{ P_BI_TCXO_AO, 0 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_6_ao[] = {
+	{ .index = DT_BI_TCXO_AO },
+};
+
+static const struct freq_tbl ftbl_cam_cc_bps_clk_src[] = {
+	F(160000000, P_CAM_CC_PLL0_OUT_ODD, 2.5, 0, 0),
+	F(200000000, P_CAM_CC_PLL0_OUT_ODD, 2, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0),
+	F(600000000, P_CAM_CC_PLL0_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_bps_clk_src = {
+	.cmd_rcgr = 0x10278,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_bps_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_bps_clk_src",
+			.parent_data = cam_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_camnoc_axi_rt_clk_src[] = {
+	F(240000000, P_CAM_CC_PLL0_OUT_EVEN, 2.5, 0, 0),
+	F(300000000, P_CAM_CC_PLL0_OUT_EVEN, 2, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_camnoc_axi_rt_clk_src = {
+	.cmd_rcgr = 0x137b4,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_camnoc_axi_rt_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_axi_rt_clk_src",
+			.parent_data = cam_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_cci_0_clk_src[] = {
+	F(30000000, P_CAM_CC_PLL5_OUT_EVEN, 16, 0, 0),
+	F(37500000, P_CAM_CC_PLL0_OUT_EVEN, 16, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_cci_0_clk_src = {
+	.cmd_rcgr = 0x1350c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_cci_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cci_0_clk_src",
+			.parent_data = cam_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static struct clk_rcg2 cam_cc_cci_1_clk_src = {
+	.cmd_rcgr = 0x1363c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_cci_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cci_1_clk_src",
+			.parent_data = cam_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_cphy_rx_clk_src[] = {
+	F(300000000, P_CAM_CC_PLL0_OUT_MAIN, 4, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_MAIN, 3, 0, 0),
+	F(480000000, P_CAM_CC_PLL0_OUT_MAIN, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_cphy_rx_clk_src = {
+	.cmd_rcgr = 0x11168,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_cphy_rx_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cphy_rx_clk_src",
+			.parent_data = cam_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_csi0phytimer_clk_src[] = {
+	F(266666667, P_CAM_CC_PLL0_OUT_ODD, 1.5, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_csi0phytimer_clk_src = {
+	.cmd_rcgr = 0x150e0,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi0phytimer_clk_src",
+			.parent_data = cam_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static struct clk_rcg2 cam_cc_csi1phytimer_clk_src = {
+	.cmd_rcgr = 0x15104,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi1phytimer_clk_src",
+			.parent_data = cam_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static struct clk_rcg2 cam_cc_csi4phytimer_clk_src = {
+	.cmd_rcgr = 0x15124,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi4phytimer_clk_src",
+			.parent_data = cam_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static struct clk_rcg2 cam_cc_csid_clk_src = {
+	.cmd_rcgr = 0x1378c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_cphy_rx_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csid_clk_src",
+			.parent_data = cam_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_fast_ahb_clk_src[] = {
+	F(80000000, P_CAM_CC_PLL0_OUT_EVEN, 7.5, 0, 0),
+	F(100000000, P_CAM_CC_PLL0_OUT_EVEN, 6, 0, 0),
+	F(200000000, P_CAM_CC_PLL0_OUT_EVEN, 3, 0, 0),
+	F(300000000, P_CAM_CC_PLL0_OUT_MAIN, 4, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_fast_ahb_clk_src = {
+	.cmd_rcgr = 0x10018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_fast_ahb_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_fast_ahb_clk_src",
+			.parent_data = cam_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_icp_clk_src[] = {
+	F(300000000, P_CAM_CC_PLL0_OUT_EVEN, 2, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0),
+	F(480000000, P_CAM_CC_PLL5_OUT_EVEN, 1, 0, 0),
+	F(600000000, P_CAM_CC_PLL0_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_icp_clk_src = {
+	.cmd_rcgr = 0x133cc,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_icp_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_icp_clk_src",
+			.parent_data = cam_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_ife_0_clk_src[] = {
+	F(345600000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	F(432000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	F(594000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	F(675000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	F(727000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_ife_0_clk_src = {
+	.cmd_rcgr = 0x11018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_2,
+	.freq_tbl = ftbl_cam_cc_ife_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_0_clk_src",
+			.parent_data = cam_cc_parent_data_2,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_2),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_ife_1_clk_src[] = {
+	F(345600000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	F(432000000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	F(594000000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	F(675000000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	F(727000000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_ife_1_clk_src = {
+	.cmd_rcgr = 0x12018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_3,
+	.freq_tbl = ftbl_cam_cc_ife_1_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_1_clk_src",
+			.parent_data = cam_cc_parent_data_3,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_3),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_ife_lite_clk_src[] = {
+	F(266666667, P_CAM_CC_PLL0_OUT_ODD, 1.5, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0),
+	F(480000000, P_CAM_CC_PLL5_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_ife_lite_clk_src = {
+	.cmd_rcgr = 0x13000,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_ife_lite_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_clk_src",
+			.parent_data = cam_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static struct clk_rcg2 cam_cc_ife_lite_csid_clk_src = {
+	.cmd_rcgr = 0x13140,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_ife_lite_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_csid_clk_src",
+			.parent_data = cam_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_ipe_nps_clk_src[] = {
+	F(304000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0),
+	F(364000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0),
+	F(500000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0),
+	F(600000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0),
+	F(700000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_ipe_nps_clk_src = {
+	.cmd_rcgr = 0x103d0,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_4,
+	.freq_tbl = ftbl_cam_cc_ipe_nps_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_nps_clk_src",
+			.parent_data = cam_cc_parent_data_4,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_4),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_jpeg_clk_src[] = {
+	F(160000000, P_CAM_CC_PLL0_OUT_ODD, 2.5, 0, 0),
+	F(200000000, P_CAM_CC_PLL0_OUT_ODD, 2, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0),
+	F(480000000, P_CAM_CC_PLL5_OUT_EVEN, 1, 0, 0),
+	F(600000000, P_CAM_CC_PLL0_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_jpeg_clk_src = {
+	.cmd_rcgr = 0x13284,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_jpeg_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_jpeg_clk_src",
+			.parent_data = cam_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_mclk0_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(24000000, P_CAM_CC_PLL2_OUT_MAIN, 10, 1, 4),
+	F(68571429, P_CAM_CC_PLL2_OUT_MAIN, 14, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_mclk0_clk_src = {
+	.cmd_rcgr = 0x15000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_mclk0_clk_src",
+			.parent_data = cam_cc_parent_data_1,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static struct clk_rcg2 cam_cc_mclk1_clk_src = {
+	.cmd_rcgr = 0x1501c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_mclk1_clk_src",
+			.parent_data = cam_cc_parent_data_1,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static struct clk_rcg2 cam_cc_mclk2_clk_src = {
+	.cmd_rcgr = 0x15038,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_mclk2_clk_src",
+			.parent_data = cam_cc_parent_data_1,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static struct clk_rcg2 cam_cc_mclk3_clk_src = {
+	.cmd_rcgr = 0x15054,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_mclk3_clk_src",
+			.parent_data = cam_cc_parent_data_1,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static struct clk_rcg2 cam_cc_mclk4_clk_src = {
+	.cmd_rcgr = 0x15070,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_mclk4_clk_src",
+			.parent_data = cam_cc_parent_data_1,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static struct clk_rcg2 cam_cc_mclk5_clk_src = {
+	.cmd_rcgr = 0x1508c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_mclk5_clk_src",
+			.parent_data = cam_cc_parent_data_1,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static struct clk_rcg2 cam_cc_mclk6_clk_src = {
+	.cmd_rcgr = 0x150a8,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_mclk6_clk_src",
+			.parent_data = cam_cc_parent_data_1,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static struct clk_rcg2 cam_cc_mclk7_clk_src = {
+	.cmd_rcgr = 0x150c4,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_mclk7_clk_src",
+			.parent_data = cam_cc_parent_data_1,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_qdss_debug_clk_src[] = {
+	F(60000000, P_CAM_CC_PLL5_OUT_EVEN, 8, 0, 0),
+	F(75000000, P_CAM_CC_PLL0_OUT_EVEN, 8, 0, 0),
+	F(150000000, P_CAM_CC_PLL0_OUT_EVEN, 4, 0, 0),
+	F(300000000, P_CAM_CC_PLL0_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_qdss_debug_clk_src = {
+	.cmd_rcgr = 0x137fc,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_qdss_debug_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_qdss_debug_clk_src",
+			.parent_data = cam_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_sleep_clk_src[] = {
+	F(32000, P_SLEEP_CLK, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_sleep_clk_src = {
+	.cmd_rcgr = 0x13964,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_5,
+	.freq_tbl = ftbl_cam_cc_sleep_clk_src,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_sleep_clk_src",
+			.parent_data = cam_cc_parent_data_5,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_5),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_slow_ahb_clk_src[] = {
+	F(64000000, P_CAM_CC_PLL5_OUT_EVEN, 7.5, 0, 0),
+	F(80000000, P_CAM_CC_PLL0_OUT_EVEN, 7.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_slow_ahb_clk_src = {
+	.cmd_rcgr = 0x10148,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_slow_ahb_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_slow_ahb_clk_src",
+			.parent_data = cam_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_xo_clk_src[] = {
+	F(19200000, P_BI_TCXO_AO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_xo_clk_src = {
+	.cmd_rcgr = 0x13948,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_6_ao,
+	.freq_tbl = ftbl_cam_cc_xo_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_xo_clk_src",
+			.parent_data = cam_cc_parent_data_6_ao,
+			.num_parents = ARRAY_SIZE(cam_cc_parent_data_6_ao),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_bps_ahb_clk = {
+	.halt_reg = 0x10274,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10274,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_bps_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_bps_clk = {
+	.halt_reg = 0x103a4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x103a4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_bps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_bps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_bps_fast_ahb_clk = {
+	.halt_reg = 0x10144,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10144,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_bps_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_axi_nrt_clk = {
+	.halt_reg = 0x137e0,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x137e0,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x137e0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_axi_nrt_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_camnoc_axi_rt_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_axi_rt_clk = {
+	.halt_reg = 0x137cc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x137cc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_axi_rt_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_camnoc_axi_rt_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_dcd_xo_clk = {
+	.halt_reg = 0x137f0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x137f0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_dcd_xo_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_xo_clk = {
+	.halt_reg = 0x137f4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x137f4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_xo_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cci_0_clk = {
+	.halt_reg = 0x13638,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13638,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cci_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cci_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cci_1_clk = {
+	.halt_reg = 0x13768,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13768,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cci_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cci_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_core_ahb_clk = {
+	.halt_reg = 0x13944,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x13944,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_core_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cpas_ahb_clk = {
+	.halt_reg = 0x1376c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1376c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cpas_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cpas_bps_clk = {
+	.halt_reg = 0x103b4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x103b4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cpas_bps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_bps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cpas_fast_ahb_clk = {
+	.halt_reg = 0x1377c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1377c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cpas_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cpas_ife_0_clk = {
+	.halt_reg = 0x11154,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11154,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cpas_ife_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cpas_ife_1_clk = {
+	.halt_reg = 0x12040,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12040,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cpas_ife_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cpas_ife_lite_clk = {
+	.halt_reg = 0x1313c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1313c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cpas_ife_lite_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_lite_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cpas_ipe_nps_clk = {
+	.halt_reg = 0x1050c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1050c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cpas_ipe_nps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ipe_nps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi0phytimer_clk = {
+	.halt_reg = 0x150f8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x150f8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi0phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi0phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi1phytimer_clk = {
+	.halt_reg = 0x1511c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1511c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi1phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi1phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi4phytimer_clk = {
+	.halt_reg = 0x15250,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x15250,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi4phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi4phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csid_clk = {
+	.halt_reg = 0x137a4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x137a4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csid_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csid_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csid_csiphy_rx_clk = {
+	.halt_reg = 0x15100,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x15100,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csid_csiphy_rx_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy0_clk = {
+	.halt_reg = 0x150fc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x150fc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy1_clk = {
+	.halt_reg = 0x15120,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x15120,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy4_clk = {
+	.halt_reg = 0x15254,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x15254,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_icp_ahb_clk = {
+	.halt_reg = 0x13508,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13508,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_icp_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_icp_clk = {
+	.halt_reg = 0x134f8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x134f8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_icp_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_icp_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_0_clk = {
+	.halt_reg = 0x11144,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11144,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_0_dsp_clk = {
+	.halt_reg = 0x11158,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11158,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_0_dsp_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_0_fast_ahb_clk = {
+	.halt_reg = 0x11164,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11164,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_0_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_1_clk = {
+	.halt_reg = 0x12030,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12030,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_1_dsp_clk = {
+	.halt_reg = 0x12044,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12044,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_1_dsp_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_1_fast_ahb_clk = {
+	.halt_reg = 0x12050,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12050,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_1_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_lite_ahb_clk = {
+	.halt_reg = 0x13280,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13280,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_lite_clk = {
+	.halt_reg = 0x1312c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1312c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_lite_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_lite_cphy_rx_clk = {
+	.halt_reg = 0x1327c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1327c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_cphy_rx_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_lite_csid_clk = {
+	.halt_reg = 0x1326c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1326c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_csid_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_lite_csid_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_nps_ahb_clk = {
+	.halt_reg = 0x10528,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10528,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_nps_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_nps_clk = {
+	.halt_reg = 0x104fc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x104fc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_nps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ipe_nps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_nps_fast_ahb_clk = {
+	.halt_reg = 0x1052c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1052c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_nps_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_pps_clk = {
+	.halt_reg = 0x10510,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10510,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_pps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ipe_nps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_pps_fast_ahb_clk = {
+	.halt_reg = 0x10530,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10530,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_pps_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_jpeg_clk = {
+	.halt_reg = 0x133b0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x133b0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_jpeg_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_jpeg_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_mclk0_clk = {
+	.halt_reg = 0x15018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x15018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_mclk0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_mclk0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_mclk1_clk = {
+	.halt_reg = 0x15034,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x15034,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_mclk1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_mclk1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_mclk2_clk = {
+	.halt_reg = 0x15050,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x15050,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_mclk2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_mclk2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_mclk3_clk = {
+	.halt_reg = 0x1506c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1506c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_mclk3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_mclk3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_mclk4_clk = {
+	.halt_reg = 0x15088,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x15088,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_mclk4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_mclk4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_mclk5_clk = {
+	.halt_reg = 0x150a4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x150a4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_mclk5_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_mclk5_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_mclk6_clk = {
+	.halt_reg = 0x150c0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x150c0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_mclk6_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_mclk6_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_mclk7_clk = {
+	.halt_reg = 0x150dc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x150dc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_mclk7_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_mclk7_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_qdss_debug_clk = {
+	.halt_reg = 0x13928,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13928,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_qdss_debug_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_qdss_debug_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_qdss_debug_xo_clk = {
+	.halt_reg = 0x1392c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1392c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_qdss_debug_xo_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc cam_cc_titan_top_gdsc = {
+	.gdscr = 0x13930,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_titan_top_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc cam_cc_bps_gdsc = {
+	.gdscr = 0x10004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_bps_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.parent = &cam_cc_titan_top_gdsc.pd,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc cam_cc_ife_0_gdsc = {
+	.gdscr = 0x11004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_ife_0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.parent = &cam_cc_titan_top_gdsc.pd,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc cam_cc_ife_1_gdsc = {
+	.gdscr = 0x12004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_ife_1_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.parent = &cam_cc_titan_top_gdsc.pd,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc cam_cc_ipe_0_gdsc = {
+	.gdscr = 0x103bc,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_ipe_0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.parent = &cam_cc_titan_top_gdsc.pd,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct clk_regmap *cam_cc_glymur_clocks[] = {
+	[CAM_CC_BPS_AHB_CLK] = &cam_cc_bps_ahb_clk.clkr,
+	[CAM_CC_BPS_CLK] = &cam_cc_bps_clk.clkr,
+	[CAM_CC_BPS_CLK_SRC] = &cam_cc_bps_clk_src.clkr,
+	[CAM_CC_BPS_FAST_AHB_CLK] = &cam_cc_bps_fast_ahb_clk.clkr,
+	[CAM_CC_CAMNOC_AXI_NRT_CLK] = &cam_cc_camnoc_axi_nrt_clk.clkr,
+	[CAM_CC_CAMNOC_AXI_RT_CLK] = &cam_cc_camnoc_axi_rt_clk.clkr,
+	[CAM_CC_CAMNOC_AXI_RT_CLK_SRC] = &cam_cc_camnoc_axi_rt_clk_src.clkr,
+	[CAM_CC_CAMNOC_DCD_XO_CLK] = &cam_cc_camnoc_dcd_xo_clk.clkr,
+	[CAM_CC_CAMNOC_XO_CLK] = &cam_cc_camnoc_xo_clk.clkr,
+	[CAM_CC_CCI_0_CLK] = &cam_cc_cci_0_clk.clkr,
+	[CAM_CC_CCI_0_CLK_SRC] = &cam_cc_cci_0_clk_src.clkr,
+	[CAM_CC_CCI_1_CLK] = &cam_cc_cci_1_clk.clkr,
+	[CAM_CC_CCI_1_CLK_SRC] = &cam_cc_cci_1_clk_src.clkr,
+	[CAM_CC_CORE_AHB_CLK] = &cam_cc_core_ahb_clk.clkr,
+	[CAM_CC_CPAS_AHB_CLK] = &cam_cc_cpas_ahb_clk.clkr,
+	[CAM_CC_CPAS_BPS_CLK] = &cam_cc_cpas_bps_clk.clkr,
+	[CAM_CC_CPAS_FAST_AHB_CLK] = &cam_cc_cpas_fast_ahb_clk.clkr,
+	[CAM_CC_CPAS_IFE_0_CLK] = &cam_cc_cpas_ife_0_clk.clkr,
+	[CAM_CC_CPAS_IFE_1_CLK] = &cam_cc_cpas_ife_1_clk.clkr,
+	[CAM_CC_CPAS_IFE_LITE_CLK] = &cam_cc_cpas_ife_lite_clk.clkr,
+	[CAM_CC_CPAS_IPE_NPS_CLK] = &cam_cc_cpas_ipe_nps_clk.clkr,
+	[CAM_CC_CPHY_RX_CLK_SRC] = &cam_cc_cphy_rx_clk_src.clkr,
+	[CAM_CC_CSI0PHYTIMER_CLK] = &cam_cc_csi0phytimer_clk.clkr,
+	[CAM_CC_CSI0PHYTIMER_CLK_SRC] = &cam_cc_csi0phytimer_clk_src.clkr,
+	[CAM_CC_CSI1PHYTIMER_CLK] = &cam_cc_csi1phytimer_clk.clkr,
+	[CAM_CC_CSI1PHYTIMER_CLK_SRC] = &cam_cc_csi1phytimer_clk_src.clkr,
+	[CAM_CC_CSI4PHYTIMER_CLK] = &cam_cc_csi4phytimer_clk.clkr,
+	[CAM_CC_CSI4PHYTIMER_CLK_SRC] = &cam_cc_csi4phytimer_clk_src.clkr,
+	[CAM_CC_CSID_CLK] = &cam_cc_csid_clk.clkr,
+	[CAM_CC_CSID_CLK_SRC] = &cam_cc_csid_clk_src.clkr,
+	[CAM_CC_CSID_CSIPHY_RX_CLK] = &cam_cc_csid_csiphy_rx_clk.clkr,
+	[CAM_CC_CSIPHY0_CLK] = &cam_cc_csiphy0_clk.clkr,
+	[CAM_CC_CSIPHY1_CLK] = &cam_cc_csiphy1_clk.clkr,
+	[CAM_CC_CSIPHY4_CLK] = &cam_cc_csiphy4_clk.clkr,
+	[CAM_CC_FAST_AHB_CLK_SRC] = &cam_cc_fast_ahb_clk_src.clkr,
+	[CAM_CC_ICP_AHB_CLK] = &cam_cc_icp_ahb_clk.clkr,
+	[CAM_CC_ICP_CLK] = &cam_cc_icp_clk.clkr,
+	[CAM_CC_ICP_CLK_SRC] = &cam_cc_icp_clk_src.clkr,
+	[CAM_CC_IFE_0_CLK] = &cam_cc_ife_0_clk.clkr,
+	[CAM_CC_IFE_0_CLK_SRC] = &cam_cc_ife_0_clk_src.clkr,
+	[CAM_CC_IFE_0_DSP_CLK] = &cam_cc_ife_0_dsp_clk.clkr,
+	[CAM_CC_IFE_0_FAST_AHB_CLK] = &cam_cc_ife_0_fast_ahb_clk.clkr,
+	[CAM_CC_IFE_1_CLK] = &cam_cc_ife_1_clk.clkr,
+	[CAM_CC_IFE_1_CLK_SRC] = &cam_cc_ife_1_clk_src.clkr,
+	[CAM_CC_IFE_1_DSP_CLK] = &cam_cc_ife_1_dsp_clk.clkr,
+	[CAM_CC_IFE_1_FAST_AHB_CLK] = &cam_cc_ife_1_fast_ahb_clk.clkr,
+	[CAM_CC_IFE_LITE_AHB_CLK] = &cam_cc_ife_lite_ahb_clk.clkr,
+	[CAM_CC_IFE_LITE_CLK] = &cam_cc_ife_lite_clk.clkr,
+	[CAM_CC_IFE_LITE_CLK_SRC] = &cam_cc_ife_lite_clk_src.clkr,
+	[CAM_CC_IFE_LITE_CPHY_RX_CLK] = &cam_cc_ife_lite_cphy_rx_clk.clkr,
+	[CAM_CC_IFE_LITE_CSID_CLK] = &cam_cc_ife_lite_csid_clk.clkr,
+	[CAM_CC_IFE_LITE_CSID_CLK_SRC] = &cam_cc_ife_lite_csid_clk_src.clkr,
+	[CAM_CC_IPE_NPS_AHB_CLK] = &cam_cc_ipe_nps_ahb_clk.clkr,
+	[CAM_CC_IPE_NPS_CLK] = &cam_cc_ipe_nps_clk.clkr,
+	[CAM_CC_IPE_NPS_CLK_SRC] = &cam_cc_ipe_nps_clk_src.clkr,
+	[CAM_CC_IPE_NPS_FAST_AHB_CLK] = &cam_cc_ipe_nps_fast_ahb_clk.clkr,
+	[CAM_CC_IPE_PPS_CLK] = &cam_cc_ipe_pps_clk.clkr,
+	[CAM_CC_IPE_PPS_FAST_AHB_CLK] = &cam_cc_ipe_pps_fast_ahb_clk.clkr,
+	[CAM_CC_JPEG_CLK] = &cam_cc_jpeg_clk.clkr,
+	[CAM_CC_JPEG_CLK_SRC] = &cam_cc_jpeg_clk_src.clkr,
+	[CAM_CC_MCLK0_CLK] = &cam_cc_mclk0_clk.clkr,
+	[CAM_CC_MCLK0_CLK_SRC] = &cam_cc_mclk0_clk_src.clkr,
+	[CAM_CC_MCLK1_CLK] = &cam_cc_mclk1_clk.clkr,
+	[CAM_CC_MCLK1_CLK_SRC] = &cam_cc_mclk1_clk_src.clkr,
+	[CAM_CC_MCLK2_CLK] = &cam_cc_mclk2_clk.clkr,
+	[CAM_CC_MCLK2_CLK_SRC] = &cam_cc_mclk2_clk_src.clkr,
+	[CAM_CC_MCLK3_CLK] = &cam_cc_mclk3_clk.clkr,
+	[CAM_CC_MCLK3_CLK_SRC] = &cam_cc_mclk3_clk_src.clkr,
+	[CAM_CC_MCLK4_CLK] = &cam_cc_mclk4_clk.clkr,
+	[CAM_CC_MCLK4_CLK_SRC] = &cam_cc_mclk4_clk_src.clkr,
+	[CAM_CC_MCLK5_CLK] = &cam_cc_mclk5_clk.clkr,
+	[CAM_CC_MCLK5_CLK_SRC] = &cam_cc_mclk5_clk_src.clkr,
+	[CAM_CC_MCLK6_CLK] = &cam_cc_mclk6_clk.clkr,
+	[CAM_CC_MCLK6_CLK_SRC] = &cam_cc_mclk6_clk_src.clkr,
+	[CAM_CC_MCLK7_CLK] = &cam_cc_mclk7_clk.clkr,
+	[CAM_CC_MCLK7_CLK_SRC] = &cam_cc_mclk7_clk_src.clkr,
+	[CAM_CC_PLL0] = &cam_cc_pll0.clkr,
+	[CAM_CC_PLL0_OUT_EVEN] = &cam_cc_pll0_out_even.clkr,
+	[CAM_CC_PLL0_OUT_ODD] = &cam_cc_pll0_out_odd.clkr,
+	[CAM_CC_PLL1] = &cam_cc_pll1.clkr,
+	[CAM_CC_PLL1_OUT_EVEN] = &cam_cc_pll1_out_even.clkr,
+	[CAM_CC_PLL2] = &cam_cc_pll2.clkr,
+	[CAM_CC_PLL3] = &cam_cc_pll3.clkr,
+	[CAM_CC_PLL3_OUT_EVEN] = &cam_cc_pll3_out_even.clkr,
+	[CAM_CC_PLL4] = &cam_cc_pll4.clkr,
+	[CAM_CC_PLL4_OUT_EVEN] = &cam_cc_pll4_out_even.clkr,
+	[CAM_CC_PLL5] = &cam_cc_pll5.clkr,
+	[CAM_CC_PLL5_OUT_EVEN] = &cam_cc_pll5_out_even.clkr,
+	[CAM_CC_QDSS_DEBUG_CLK] = &cam_cc_qdss_debug_clk.clkr,
+	[CAM_CC_QDSS_DEBUG_CLK_SRC] = &cam_cc_qdss_debug_clk_src.clkr,
+	[CAM_CC_QDSS_DEBUG_XO_CLK] = &cam_cc_qdss_debug_xo_clk.clkr,
+	[CAM_CC_SLEEP_CLK_SRC] = &cam_cc_sleep_clk_src.clkr,
+	[CAM_CC_SLOW_AHB_CLK_SRC] = &cam_cc_slow_ahb_clk_src.clkr,
+	[CAM_CC_XO_CLK_SRC] = &cam_cc_xo_clk_src.clkr,
+};
+
+static struct gdsc *cam_cc_glymur_gdscs[] = {
+	[CAM_CC_BPS_GDSC] = &cam_cc_bps_gdsc,
+	[CAM_CC_IFE_0_GDSC] = &cam_cc_ife_0_gdsc,
+	[CAM_CC_IFE_1_GDSC] = &cam_cc_ife_1_gdsc,
+	[CAM_CC_IPE_0_GDSC] = &cam_cc_ipe_0_gdsc,
+	[CAM_CC_TITAN_TOP_GDSC] = &cam_cc_titan_top_gdsc,
+};
+
+static const struct qcom_reset_map cam_cc_glymur_resets[] = {
+	[CAM_CC_BPS_BCR] = { 0x10000 },
+	[CAM_CC_ICP_BCR] = { 0x133c8 },
+	[CAM_CC_IFE_0_BCR] = { 0x11000 },
+	[CAM_CC_IFE_1_BCR] = { 0x12000 },
+	[CAM_CC_IPE_0_BCR] = { 0x103b8 },
+	[CAM_CC_QDSS_DEBUG_BCR] = { 0x137f8 },
+};
+
+static struct clk_alpha_pll *cam_cc_glymur_plls[] = {
+	&cam_cc_pll0,
+	&cam_cc_pll1,
+	&cam_cc_pll2,
+	&cam_cc_pll3,
+	&cam_cc_pll4,
+	&cam_cc_pll5,
+};
+
+static u32 cam_cc_glymur_critical_cbcrs[] = {
+	0x13960, /* CAM_CC_GDSC_CLK */
+	0x1397c, /* CAM_CC_SLEEP_CLK */
+};
+
+static const struct regmap_config cam_cc_glymur_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1603c,
+	.fast_io = true,
+};
+
+static struct qcom_cc_driver_data cam_cc_glymur_driver_data = {
+	.alpha_plls = cam_cc_glymur_plls,
+	.num_alpha_plls = ARRAY_SIZE(cam_cc_glymur_plls),
+	.clk_cbcrs = cam_cc_glymur_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(cam_cc_glymur_critical_cbcrs),
+};
+
+static const struct qcom_cc_desc cam_cc_glymur_desc = {
+	.config = &cam_cc_glymur_regmap_config,
+	.clks = cam_cc_glymur_clocks,
+	.num_clks = ARRAY_SIZE(cam_cc_glymur_clocks),
+	.resets = cam_cc_glymur_resets,
+	.num_resets = ARRAY_SIZE(cam_cc_glymur_resets),
+	.gdscs = cam_cc_glymur_gdscs,
+	.num_gdscs = ARRAY_SIZE(cam_cc_glymur_gdscs),
+	.use_rpm = true,
+	.driver_data = &cam_cc_glymur_driver_data,
+};
+
+static const struct of_device_id cam_cc_glymur_match_table[] = {
+	{ .compatible = "qcom,glymur-camcc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, cam_cc_glymur_match_table);
+
+static int cam_cc_glymur_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &cam_cc_glymur_desc);
+}
+
+static struct platform_driver cam_cc_glymur_driver = {
+	.probe = cam_cc_glymur_probe,
+	.driver = {
+		.name = "camcc-glymur",
+		.of_match_table = cam_cc_glymur_match_table,
+	},
+};
+
+module_platform_driver(cam_cc_glymur_driver);
+
+MODULE_DESCRIPTION("QTI CAMCC GLYMUR Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/camcc-sc8280xp.c b/drivers/clk/qcom/camcc-sc8280xp.c
index e97b8d4f3c844..660d8655d3919 100644
--- a/drivers/clk/qcom/camcc-sc8280xp.c
+++ b/drivers/clk/qcom/camcc-sc8280xp.c
@@ -1753,24 +1753,6 @@ static struct clk_branch camcc_csiphy3_clk = {
 	},
 };
 
-static struct clk_branch camcc_gdsc_clk = {
-	.halt_reg = 0xc1e4,
-	.halt_check = BRANCH_HALT,
-	.clkr = {
-		.enable_reg = 0xc1e4,
-		.enable_mask = BIT(0),
-		.hw.init = &(struct clk_init_data){
-			.name = "camcc_gdsc_clk",
-			.parent_hws = (const struct clk_hw*[]){
-				&camcc_xo_clk_src.clkr.hw,
-			},
-			.num_parents = 1,
-			.flags = CLK_SET_RATE_PARENT,
-			.ops = &clk_branch2_ops,
-		},
-	},
-};
-
 static struct clk_branch camcc_icp_ahb_clk = {
 	.halt_reg = 0xc0d8,
 	.halt_check = BRANCH_HALT,
@@ -2839,7 +2821,6 @@ static struct clk_regmap *camcc_sc8280xp_clocks[] = {
 	[CAMCC_CSIPHY2_CLK] = &camcc_csiphy2_clk.clkr,
 	[CAMCC_CSIPHY3_CLK] = &camcc_csiphy3_clk.clkr,
 	[CAMCC_FAST_AHB_CLK_SRC] = &camcc_fast_ahb_clk_src.clkr,
-	[CAMCC_GDSC_CLK] = &camcc_gdsc_clk.clkr,
 	[CAMCC_ICP_AHB_CLK] = &camcc_icp_ahb_clk.clkr,
 	[CAMCC_ICP_CLK] = &camcc_icp_clk.clkr,
 	[CAMCC_ICP_CLK_SRC] = &camcc_icp_clk_src.clkr,
diff --git a/drivers/clk/qcom/camcc-x1p42100.c b/drivers/clk/qcom/camcc-x1p42100.c
index cfe24bde46527..e2b8f7bd8974a 100644
--- a/drivers/clk/qcom/camcc-x1p42100.c
+++ b/drivers/clk/qcom/camcc-x1p42100.c
@@ -2165,7 +2165,7 @@ static struct clk_alpha_pll *cam_cc_x1p42100_plls[] = {
 	&cam_cc_pll6,
 };
 
-static u32 cam_cc_x1p42100_critical_cbcrs[] = {
+static const u32 cam_cc_x1p42100_critical_cbcrs[] = {
 	0x13a9c, /* CAM_CC_GDSC_CLK */
 	0x13ab8, /* CAM_CC_SLEEP_CLK */
 };
@@ -2178,14 +2178,14 @@ static const struct regmap_config cam_cc_x1p42100_regmap_config = {
 	.fast_io = true,
 };
 
-static struct qcom_cc_driver_data cam_cc_x1p42100_driver_data = {
+static const struct qcom_cc_driver_data cam_cc_x1p42100_driver_data = {
 	.alpha_plls = cam_cc_x1p42100_plls,
 	.num_alpha_plls = ARRAY_SIZE(cam_cc_x1p42100_plls),
 	.clk_cbcrs = cam_cc_x1p42100_critical_cbcrs,
 	.num_clk_cbcrs = ARRAY_SIZE(cam_cc_x1p42100_critical_cbcrs),
 };
 
-static struct qcom_cc_desc cam_cc_x1p42100_desc = {
+static const struct qcom_cc_desc cam_cc_x1p42100_desc = {
 	.config = &cam_cc_x1p42100_regmap_config,
 	.clks = cam_cc_x1p42100_clocks,
 	.num_clks = ARRAY_SIZE(cam_cc_x1p42100_clocks),
diff --git a/drivers/clk/qcom/clk-regmap-phy-mux.c b/drivers/clk/qcom/clk-regmap-phy-mux.c
index b7d1c69d62f7f..7b7243b7107dc 100644
--- a/drivers/clk/qcom/clk-regmap-phy-mux.c
+++ b/drivers/clk/qcom/clk-regmap-phy-mux.c
@@ -15,66 +15,48 @@
 #define PHY_MUX_PHY_SRC		0
 #define PHY_MUX_REF_SRC		2
 
-#define XO_RATE			19200000UL
-
 static inline struct clk_regmap_phy_mux *to_clk_regmap_phy_mux(struct clk_regmap *clkr)
 {
 	return container_of(clkr, struct clk_regmap_phy_mux, clkr);
 }
 
-static unsigned long phy_mux_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
+static int phy_mux_is_enabled(struct clk_hw *hw)
 {
 	struct clk_regmap *clkr = to_clk_regmap(hw);
 	struct clk_regmap_phy_mux *phy_mux = to_clk_regmap_phy_mux(clkr);
-	u32 val;
+	unsigned int val;
 
 	regmap_read(clkr->regmap, phy_mux->reg, &val);
+	val = FIELD_GET(PHY_MUX_MASK, val);
+
+	WARN_ON(val != PHY_MUX_PHY_SRC && val != PHY_MUX_REF_SRC);
 
-	switch (FIELD_GET(PHY_MUX_MASK, val)) {
-	case PHY_MUX_PHY_SRC:
-		return ULONG_MAX;
-	case PHY_MUX_REF_SRC:
-		return XO_RATE;
-	default:
-		return 0;
-	}
+	return val == PHY_MUX_PHY_SRC;
 }
 
-static int phy_mux_determine_rate(struct clk_hw *hw, struct clk_rate_request *req)
+static int phy_mux_enable(struct clk_hw *hw)
 {
-	if (req->rate == XO_RATE || req->rate == ULONG_MAX)
-		return 0;
+	struct clk_regmap *clkr = to_clk_regmap(hw);
+	struct clk_regmap_phy_mux *phy_mux = to_clk_regmap_phy_mux(clkr);
 
-	return -EINVAL;
+	return regmap_update_bits(clkr->regmap, phy_mux->reg,
+				  PHY_MUX_MASK,
+				  FIELD_PREP(PHY_MUX_MASK, PHY_MUX_PHY_SRC));
 }
 
-static int phy_mux_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate)
+static void phy_mux_disable(struct clk_hw *hw)
 {
 	struct clk_regmap *clkr = to_clk_regmap(hw);
 	struct clk_regmap_phy_mux *phy_mux = to_clk_regmap_phy_mux(clkr);
-	u32 val;
-
-	switch (rate) {
-	case XO_RATE:
-		val = PHY_MUX_REF_SRC;
-		break;
-	case ULONG_MAX:
-		val = PHY_MUX_PHY_SRC;
-		break;
-	default:
-		return -EINVAL;
-	}
 
 	regmap_update_bits(clkr->regmap, phy_mux->reg,
 			   PHY_MUX_MASK,
-			   FIELD_PREP(PHY_MUX_MASK, val));
-
-	return 0;
+			   FIELD_PREP(PHY_MUX_MASK, PHY_MUX_REF_SRC));
 }
 
 const struct clk_ops clk_regmap_phy_mux_ops = {
-	.recalc_rate = phy_mux_recalc_rate,
-	.determine_rate = phy_mux_determine_rate,
-	.set_rate = phy_mux_set_rate,
+	.enable = phy_mux_enable,
+	.disable = phy_mux_disable,
+	.is_enabled = phy_mux_is_enabled,
 };
 EXPORT_SYMBOL_GPL(clk_regmap_phy_mux_ops);
diff --git a/drivers/clk/qcom/clk-smd-rpm.c b/drivers/clk/qcom/clk-smd-rpm.c
index 103db984a40b9..0b624ed4715c7 100644
--- a/drivers/clk/qcom/clk-smd-rpm.c
+++ b/drivers/clk/qcom/clk-smd-rpm.c
@@ -495,6 +495,7 @@ DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(rf_clk2, 5, 19200000);
 DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(rf_clk3, 6, 19200000);
 DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(ln_bb_clk, 8, 19200000);
 
+DEFINE_CLK_SMD_RPM_XO_BUFFER_PREFIX(38m4_, rf_clk2, 5, 38400000);
 DEFINE_CLK_SMD_RPM_XO_BUFFER_PREFIX(38m4_, rf_clk3, 6, 38400000);
 
 DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(cxo_d0, 1, 19200000);
@@ -1262,6 +1263,10 @@ static struct clk_smd_rpm *qcm2290_clks[] = {
 	[RPM_SMD_QDSS_A_CLK] = &clk_smd_rpm_branch_qdss_a_clk,
 	[RPM_SMD_LN_BB_CLK2] = &clk_smd_rpm_ln_bb_clk2,
 	[RPM_SMD_LN_BB_CLK2_A] = &clk_smd_rpm_ln_bb_clk2_a,
+	[RPM_SMD_RF_CLK1] = &clk_smd_rpm_rf_clk1,
+	[RPM_SMD_RF_CLK1_A] = &clk_smd_rpm_rf_clk1_a,
+	[RPM_SMD_RF_CLK2] = &clk_smd_rpm_38m4_rf_clk2,
+	[RPM_SMD_RF_CLK2_A] = &clk_smd_rpm_38m4_rf_clk2_a,
 	[RPM_SMD_RF_CLK3] = &clk_smd_rpm_38m4_rf_clk3,
 	[RPM_SMD_RF_CLK3_A] = &clk_smd_rpm_38m4_rf_clk3_a,
 	[RPM_SMD_IPA_CLK] = &clk_smd_rpm_ipa_clk,
diff --git a/drivers/clk/qcom/common.c b/drivers/clk/qcom/common.c
index eec369d2173b5..2c09abaf1d2a1 100644
--- a/drivers/clk/qcom/common.c
+++ b/drivers/clk/qcom/common.c
@@ -428,7 +428,7 @@ int qcom_cc_really_probe(struct device *dev,
 
 put_rpm:
 	if (desc->use_rpm)
-		pm_runtime_put(dev);
+		pm_runtime_put_sync(dev);
 
 	return ret;
 }
diff --git a/drivers/clk/qcom/dispcc-eliza.c b/drivers/clk/qcom/dispcc-eliza.c
index 760881cb10770..290018d1abe1c 100644
--- a/drivers/clk/qcom/dispcc-eliza.c
+++ b/drivers/clk/qcom/dispcc-eliza.c
@@ -756,7 +756,7 @@ static struct clk_rcg2 disp_cc_mdss_mdp_clk_src = {
 		.parent_data = disp_cc_parent_data_11,
 		.num_parents = ARRAY_SIZE(disp_cc_parent_data_11),
 		.flags = CLK_SET_RATE_PARENT,
-		.ops = &clk_rcg2_shared_ops,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
 	},
 };
 
diff --git a/drivers/clk/qcom/evacc-glymur.c b/drivers/clk/qcom/evacc-glymur.c
new file mode 100644
index 0000000000000..ba124cfc1a4de
--- /dev/null
+++ b/drivers/clk/qcom/evacc-glymur.c
@@ -0,0 +1,455 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,glymur-evacc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "common.h"
+#include "gdsc.h"
+#include "reset.h"
+
+enum {
+	DT_AHB_CLK,
+	DT_BI_TCXO,
+	DT_SLEEP_CLK,
+};
+
+enum {
+	P_BI_TCXO,
+	P_EVA_CC_PLL0_OUT_MAIN,
+	P_SLEEP_CLK,
+};
+
+static const struct pll_vco taycan_eko_t_vco[] = {
+	{ 249600000, 2500000000, 0 },
+};
+
+/* 840.0 MHz Configuration */
+static const struct alpha_pll_config eva_cc_pll0_config = {
+	.l = 0x2b,
+	.alpha = 0xc000,
+	.config_ctl_val = 0x25c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000008,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll eva_cc_pll0 = {
+	.offset = 0x0,
+	.config = &eva_cc_pll0_config,
+	.vco_table = taycan_eko_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eko_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "eva_cc_pll0",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eko_t_ops,
+		},
+	},
+};
+
+static const struct parent_map eva_cc_parent_map_0[] = {
+	{ P_BI_TCXO, 0 },
+};
+
+static const struct clk_parent_data eva_cc_parent_data_0[] = {
+	{ .index = DT_BI_TCXO },
+};
+
+static const struct parent_map eva_cc_parent_map_1[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_EVA_CC_PLL0_OUT_MAIN, 1 },
+};
+
+static const struct clk_parent_data eva_cc_parent_data_1[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &eva_cc_pll0.clkr.hw },
+};
+
+static const struct parent_map eva_cc_parent_map_2[] = {
+	{ P_SLEEP_CLK, 0 },
+};
+
+static const struct clk_parent_data eva_cc_parent_data_2[] = {
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct freq_tbl ftbl_eva_cc_ahb_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 eva_cc_ahb_clk_src = {
+	.cmd_rcgr = 0x8018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = eva_cc_parent_map_0,
+	.freq_tbl = ftbl_eva_cc_ahb_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "eva_cc_ahb_clk_src",
+			.parent_data = eva_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(eva_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_eva_cc_mvs0_clk_src[] = {
+	F(840000000, P_EVA_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1050000000, P_EVA_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1350000000, P_EVA_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1500000000, P_EVA_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1650000000, P_EVA_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 eva_cc_mvs0_clk_src = {
+	.cmd_rcgr = 0x8000,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = eva_cc_parent_map_1,
+	.freq_tbl = ftbl_eva_cc_mvs0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "eva_cc_mvs0_clk_src",
+			.parent_data = eva_cc_parent_data_1,
+			.num_parents = ARRAY_SIZE(eva_cc_parent_data_1),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_eva_cc_sleep_clk_src[] = {
+	F(32000, P_SLEEP_CLK, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 eva_cc_sleep_clk_src = {
+	.cmd_rcgr = 0x80e0,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = eva_cc_parent_map_2,
+	.freq_tbl = ftbl_eva_cc_sleep_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "eva_cc_sleep_clk_src",
+			.parent_data = eva_cc_parent_data_2,
+			.num_parents = ARRAY_SIZE(eva_cc_parent_data_2),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static struct clk_rcg2 eva_cc_xo_clk_src = {
+	.cmd_rcgr = 0x80bc,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = eva_cc_parent_map_0,
+	.freq_tbl = ftbl_eva_cc_ahb_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "eva_cc_xo_clk_src",
+			.parent_data = eva_cc_parent_data_0,
+			.num_parents = ARRAY_SIZE(eva_cc_parent_data_0),
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_rcg2_shared_ops,
+		},
+	},
+};
+
+static struct clk_regmap_div eva_cc_mvs0_div_clk_src = {
+	.reg = 0x809c,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "eva_cc_mvs0_div_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&eva_cc_mvs0_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_regmap_div eva_cc_mvs0c_div2_div_clk_src = {
+	.reg = 0x8060,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "eva_cc_mvs0c_div2_div_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&eva_cc_mvs0_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_branch eva_cc_mvs0_clk = {
+	.halt_reg = 0x807c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x807c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x807c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "eva_cc_mvs0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&eva_cc_mvs0_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch eva_cc_mvs0_freerun_clk = {
+	.halt_reg = 0x808c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x808c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "eva_cc_mvs0_freerun_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&eva_cc_mvs0_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch eva_cc_mvs0_shift_clk = {
+	.halt_reg = 0x80d8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x80d8,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x80d8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "eva_cc_mvs0_shift_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&eva_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch eva_cc_mvs0c_clk = {
+	.halt_reg = 0x804c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x804c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "eva_cc_mvs0c_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&eva_cc_mvs0c_div2_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch eva_cc_mvs0c_freerun_clk = {
+	.halt_reg = 0x805c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x805c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "eva_cc_mvs0c_freerun_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&eva_cc_mvs0c_div2_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch eva_cc_mvs0c_shift_clk = {
+	.halt_reg = 0x80dc,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x80dc,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x80dc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "eva_cc_mvs0c_shift_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&eva_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc eva_cc_mvs0c_gdsc = {
+	.gdscr = 0x8034,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x6,
+	.pd = {
+		.name = "eva_cc_mvs0c_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc eva_cc_mvs0_gdsc = {
+	.gdscr = 0x8068,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x6,
+	.pd = {
+		.name = "eva_cc_mvs0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+	.parent = &eva_cc_mvs0c_gdsc.pd,
+};
+
+static struct clk_regmap *eva_cc_glymur_clocks[] = {
+	[EVA_CC_AHB_CLK_SRC] = &eva_cc_ahb_clk_src.clkr,
+	[EVA_CC_MVS0_CLK] = &eva_cc_mvs0_clk.clkr,
+	[EVA_CC_MVS0_CLK_SRC] = &eva_cc_mvs0_clk_src.clkr,
+	[EVA_CC_MVS0_DIV_CLK_SRC] = &eva_cc_mvs0_div_clk_src.clkr,
+	[EVA_CC_MVS0_FREERUN_CLK] = &eva_cc_mvs0_freerun_clk.clkr,
+	[EVA_CC_MVS0_SHIFT_CLK] = &eva_cc_mvs0_shift_clk.clkr,
+	[EVA_CC_MVS0C_CLK] = &eva_cc_mvs0c_clk.clkr,
+	[EVA_CC_MVS0C_DIV2_DIV_CLK_SRC] = &eva_cc_mvs0c_div2_div_clk_src.clkr,
+	[EVA_CC_MVS0C_FREERUN_CLK] = &eva_cc_mvs0c_freerun_clk.clkr,
+	[EVA_CC_MVS0C_SHIFT_CLK] = &eva_cc_mvs0c_shift_clk.clkr,
+	[EVA_CC_PLL0] = &eva_cc_pll0.clkr,
+	[EVA_CC_SLEEP_CLK_SRC] = &eva_cc_sleep_clk_src.clkr,
+	[EVA_CC_XO_CLK_SRC] = &eva_cc_xo_clk_src.clkr,
+};
+
+static struct gdsc *eva_cc_glymur_gdscs[] = {
+	[EVA_CC_MVS0_GDSC] = &eva_cc_mvs0_gdsc,
+	[EVA_CC_MVS0C_GDSC] = &eva_cc_mvs0c_gdsc,
+};
+
+static const struct qcom_reset_map eva_cc_glymur_resets[] = {
+	[EVA_CC_INTERFACE_BCR] = { 0x80a0 },
+	[EVA_CC_MVS0_BCR] = { 0x8064 },
+	[EVA_CC_MVS0C_CLK_ARES] = { 0x804c, 2 },
+	[EVA_CC_MVS0C_BCR] = { 0x8030 },
+	[EVA_CC_MVS0C_FREERUN_CLK_ARES] = { 0x805c, 2 },
+};
+
+static struct clk_alpha_pll *eva_cc_glymur_plls[] = {
+	&eva_cc_pll0,
+};
+
+static const u32 eva_cc_glymur_critical_cbcrs[] = {
+	0x80a4, /* EVA_CC_AHB_CLK */
+	0x80f8, /* EVA_CC_SLEEP_CLK */
+	0x80d4, /* EVA_CC_XO_CLK */
+};
+
+static const struct regmap_config eva_cc_glymur_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x9f50,
+	.fast_io = true,
+};
+
+static void clk_glymur_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/*
+	 * Update CTRL_IN register as per HW recommendation to ensure clocks
+	 * stay cycle‑aligned when the EVA core is ON.
+	 */
+	regmap_set_bits(regmap, 0x9f24, BIT(0));
+}
+
+static const struct qcom_cc_driver_data eva_cc_glymur_driver_data = {
+	.alpha_plls = eva_cc_glymur_plls,
+	.num_alpha_plls = ARRAY_SIZE(eva_cc_glymur_plls),
+	.clk_cbcrs = eva_cc_glymur_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(eva_cc_glymur_critical_cbcrs),
+	.clk_regs_configure = clk_glymur_regs_configure,
+};
+
+static const struct qcom_cc_desc eva_cc_glymur_desc = {
+	.config = &eva_cc_glymur_regmap_config,
+	.clks = eva_cc_glymur_clocks,
+	.num_clks = ARRAY_SIZE(eva_cc_glymur_clocks),
+	.resets = eva_cc_glymur_resets,
+	.num_resets = ARRAY_SIZE(eva_cc_glymur_resets),
+	.gdscs = eva_cc_glymur_gdscs,
+	.num_gdscs = ARRAY_SIZE(eva_cc_glymur_gdscs),
+	.use_rpm = true,
+	.driver_data = &eva_cc_glymur_driver_data,
+};
+
+static const struct of_device_id eva_cc_glymur_match_table[] = {
+	{ .compatible = "qcom,glymur-evacc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, eva_cc_glymur_match_table);
+
+static int eva_cc_glymur_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &eva_cc_glymur_desc);
+}
+
+static struct platform_driver eva_cc_glymur_driver = {
+	.probe = eva_cc_glymur_probe,
+	.driver = {
+		.name = "evacc-glymur",
+		.of_match_table = eva_cc_glymur_match_table,
+	},
+};
+
+module_platform_driver(eva_cc_glymur_driver);
+
+MODULE_DESCRIPTION("QTI EVACC Glymur Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-glymur.c b/drivers/clk/qcom/gcc-glymur.c
index f4ede4a3a1c07..6925c6865089c 100644
--- a/drivers/clk/qcom/gcc-glymur.c
+++ b/drivers/clk/qcom/gcc-glymur.c
@@ -3668,21 +3668,6 @@ static struct clk_branch gcc_disp_hf_axi_clk = {
 	},
 };
 
-static struct clk_branch gcc_eva_ahb_clk = {
-	.halt_reg = 0x9b004,
-	.halt_check = BRANCH_HALT_VOTED,
-	.hwcg_reg = 0x9b004,
-	.hwcg_bit = 1,
-	.clkr = {
-		.enable_reg = 0x9b004,
-		.enable_mask = BIT(0),
-		.hw.init = &(const struct clk_init_data) {
-			.name = "gcc_eva_ahb_clk",
-			.ops = &clk_branch2_ops,
-		},
-	},
-};
-
 static struct clk_branch gcc_eva_axi0_clk = {
 	.halt_reg = 0x9b008,
 	.halt_check = BRANCH_HALT_SKIP,
@@ -3713,19 +3698,6 @@ static struct clk_branch gcc_eva_axi0c_clk = {
 	},
 };
 
-static struct clk_branch gcc_eva_xo_clk = {
-	.halt_reg = 0x9b024,
-	.halt_check = BRANCH_HALT,
-	.clkr = {
-		.enable_reg = 0x9b024,
-		.enable_mask = BIT(0),
-		.hw.init = &(const struct clk_init_data) {
-			.name = "gcc_eva_xo_clk",
-			.ops = &clk_branch2_ops,
-		},
-	},
-};
-
 static struct clk_branch gcc_gp1_clk = {
 	.halt_reg = 0x64000,
 	.halt_check = BRANCH_HALT,
@@ -7992,10 +7964,8 @@ static struct clk_regmap *gcc_glymur_clocks[] = {
 	[GCC_CFG_NOC_USB_ANOC_AHB_CLK] = &gcc_cfg_noc_usb_anoc_ahb_clk.clkr,
 	[GCC_CFG_NOC_USB_ANOC_SOUTH_AHB_CLK] = &gcc_cfg_noc_usb_anoc_south_ahb_clk.clkr,
 	[GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr,
-	[GCC_EVA_AHB_CLK] = &gcc_eva_ahb_clk.clkr,
 	[GCC_EVA_AXI0_CLK] = &gcc_eva_axi0_clk.clkr,
 	[GCC_EVA_AXI0C_CLK] = &gcc_eva_axi0c_clk.clkr,
-	[GCC_EVA_XO_CLK] = &gcc_eva_xo_clk.clkr,
 	[GCC_GP1_CLK] = &gcc_gp1_clk.clkr,
 	[GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr,
 	[GCC_GP2_CLK] = &gcc_gp2_clk.clkr,
@@ -8544,6 +8514,8 @@ static const u32 gcc_glymur_critical_cbcrs[] = {
 	0x71004, /* GCC_GPU_CFG_AHB_CLK */
 	0x32004, /* GCC_VIDEO_AHB_CLK */
 	0x32058, /* GCC_VIDEO_XO_CLK */
+	0x9b004, /* GCC_EVA_AHB_CLK */
+	0x9b024, /* GCC_EVA_XO_CLK */
 };
 
 static const struct regmap_config gcc_glymur_regmap_config = {
diff --git a/drivers/clk/qcom/gcc-ipq5424.c b/drivers/clk/qcom/gcc-ipq5424.c
index 35af6ffeeb855..3ddd4b3aedea9 100644
--- a/drivers/clk/qcom/gcc-ipq5424.c
+++ b/drivers/clk/qcom/gcc-ipq5424.c
@@ -139,17 +139,6 @@ static struct clk_alpha_pll gpll4 = {
 			.parent_data = &gcc_parent_data_xo,
 			.num_parents = 1,
 			.ops = &clk_alpha_pll_ops,
-			/*
-			 * There are no consumers for this GPLL in kernel yet,
-			 * (will be added soon), so the clock framework
-			 * disables this source. But some of the clocks
-			 * initialized by boot loaders uses this source. So we
-			 * need to keep this clock ON. Add the
-			 * CLK_IGNORE_UNUSED flag so the clock will not be
-			 * disabled. Once the consumer in kernel is added, we
-			 * can get rid of this flag.
-			 */
-			.flags = CLK_IGNORE_UNUSED,
 		},
 	},
 };
diff --git a/drivers/clk/qcom/gcc-ipq9650.c b/drivers/clk/qcom/gcc-ipq9650.c
index c556c2bbfd96b..e65a52fdfa91f 100644
--- a/drivers/clk/qcom/gcc-ipq9650.c
+++ b/drivers/clk/qcom/gcc-ipq9650.c
@@ -3022,6 +3022,90 @@ static struct clk_branch gcc_usb1_sleep_clk = {
 	},
 };
 
+static struct clk_rcg2 gcc_refgen_core_clk_src = {
+	.cmd_rcgr = 0x23004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_nss_ts_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_refgen_core_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_branch gcc_refgen_cmn_uphy_core_clk = {
+	.halt_reg = 0x2300c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2300c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_refgen_cmn_uphy_core_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_refgen_core_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_refgen_pcie_core_clk = {
+	.halt_reg = 0x23020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x23020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_refgen_pcie_core_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_refgen_core_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_refgen_pcie_hclk = {
+	.halt_reg = 0x23024,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x23024,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_refgen_pcie_hclk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_refgen_cmn_uphy_hclk = {
+	.halt_reg = 0x23010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x23010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_refgen_cmn_uphy_hclk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
 static struct clk_regmap *gcc_ipq9650_clocks[] = {
 	[GCC_ADSS_PWM_CLK] = &gcc_adss_pwm_clk.clkr,
 	[GCC_ADSS_PWM_CLK_SRC] = &gcc_adss_pwm_clk_src.clkr,
@@ -3179,6 +3263,11 @@ static struct clk_regmap *gcc_ipq9650_clocks[] = {
 	[GPLL2] = &gpll2.clkr,
 	[GPLL2_OUT_MAIN] = &gpll2_out_main.clkr,
 	[GPLL4] = &gpll4.clkr,
+	[GCC_REFGEN_CORE_CLK_SRC] = &gcc_refgen_core_clk_src.clkr,
+	[GCC_REFGEN_PCIE_CORE_CLK] = &gcc_refgen_pcie_core_clk.clkr,
+	[GCC_REFGEN_PCIE_HCLK] = &gcc_refgen_pcie_hclk.clkr,
+	[GCC_REFGEN_CMN_UPHY_HCLK] = &gcc_refgen_cmn_uphy_hclk.clkr,
+	[GCC_REFGEN_CMN_UPHY_CORE_CLK] = &gcc_refgen_cmn_uphy_core_clk.clkr,
 };
 
 static const struct qcom_reset_map gcc_ipq9650_resets[] = {
diff --git a/drivers/clk/qcom/gcc-qcs8300.c b/drivers/clk/qcom/gcc-qcs8300.c
index 07218d9c96a7d..31fd870b10f7a 100644
--- a/drivers/clk/qcom/gcc-qcs8300.c
+++ b/drivers/clk/qcom/gcc-qcs8300.c
@@ -3267,7 +3267,7 @@ static struct gdsc gcc_pcie_0_gdsc = {
 	.pd = {
 		.name = "gcc_pcie_0_gdsc",
 	},
-	.pwrsts = PWRSTS_OFF_ON,
+	.pwrsts = PWRSTS_RET_ON,
 	.flags = VOTABLE | RETAIN_FF_ENABLE | POLL_CFG_GDSCR,
 };
 
@@ -3281,7 +3281,7 @@ static struct gdsc gcc_pcie_1_gdsc = {
 	.pd = {
 		.name = "gcc_pcie_1_gdsc",
 	},
-	.pwrsts = PWRSTS_OFF_ON,
+	.pwrsts = PWRSTS_RET_ON,
 	.flags = VOTABLE | RETAIN_FF_ENABLE | POLL_CFG_GDSCR,
 };
 
@@ -3305,7 +3305,7 @@ static struct gdsc gcc_usb20_prim_gdsc = {
 	.pd = {
 		.name = "gcc_usb20_prim_gdsc",
 	},
-	.pwrsts = PWRSTS_OFF_ON,
+	.pwrsts = PWRSTS_RET_ON,
 	.flags = RETAIN_FF_ENABLE | POLL_CFG_GDSCR,
 };
 
@@ -3317,7 +3317,7 @@ static struct gdsc gcc_usb30_prim_gdsc = {
 	.pd = {
 		.name = "gcc_usb30_prim_gdsc",
 	},
-	.pwrsts = PWRSTS_OFF_ON,
+	.pwrsts = PWRSTS_RET_ON,
 	.flags = RETAIN_FF_ENABLE | POLL_CFG_GDSCR,
 };
 
diff --git a/drivers/clk/qcom/gcc-shikra.c b/drivers/clk/qcom/gcc-shikra.c
new file mode 100644
index 0000000000000..d5222756f214c
--- /dev/null
+++ b/drivers/clk/qcom/gcc-shikra.c
@@ -0,0 +1,4431 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,shikra-gcc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "clk-regmap-phy-mux.h"
+#include "common.h"
+#include "gdsc.h"
+#include "reset.h"
+
+enum {
+	DT_BI_TCXO,
+	DT_SLEEP_CLK,
+	DT_EMAC0_SGMIIPHY_RCLK,
+	DT_EMAC0_SGMIIPHY_TCLK,
+	DT_EMAC1_SGMIIPHY_RCLK,
+	DT_EMAC1_SGMIIPHY_TCLK,
+	DT_PCIE_PIPE_CLK,
+	DT_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK,
+};
+
+enum {
+	P_BI_TCXO,
+	P_EMAC0_SGMIIPHY_RCLK,
+	P_EMAC0_SGMIIPHY_TCLK,
+	P_EMAC1_SGMIIPHY_RCLK,
+	P_EMAC1_SGMIIPHY_TCLK,
+	P_GPLL0_OUT_AUX2,
+	P_GPLL0_OUT_EARLY,
+	P_GPLL10_OUT_MAIN,
+	P_GPLL11_OUT_AUX,
+	P_GPLL11_OUT_AUX2,
+	P_GPLL11_OUT_MAIN,
+	P_GPLL12_OUT_AUX2,
+	P_GPLL12_OUT_EARLY,
+	P_GPLL3_OUT_EARLY,
+	P_GPLL3_OUT_MAIN,
+	P_GPLL4_OUT_MAIN,
+	P_GPLL5_OUT_MAIN,
+	P_GPLL6_OUT_EARLY,
+	P_GPLL6_OUT_MAIN,
+	P_GPLL7_OUT_MAIN,
+	P_GPLL8_OUT_EARLY,
+	P_GPLL8_OUT_MAIN,
+	P_GPLL9_OUT_EARLY,
+	P_GPLL9_OUT_MAIN,
+	P_PCIE_PIPE_CLK,
+	P_SLEEP_CLK,
+	P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK,
+};
+
+static const struct pll_vco brammo_vco[] = {
+	{ 500000000, 1250000000, 0 },
+};
+
+static const struct pll_vco default_vco[] = {
+	{ 500000000, 1000000000, 2 },
+};
+
+static const struct pll_vco spark_vco[] = {
+	{ 750000000, 1500000000, 1 },
+};
+
+static struct clk_alpha_pll gpll0 = {
+	.offset = 0x0,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.clkr = {
+		.enable_reg = 0x79000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll0",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_gpll0_out_aux2[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv gpll0_out_aux2 = {
+	.offset = 0x0,
+	.post_div_shift = 8,
+	.post_div_table = post_div_table_gpll0_out_aux2,
+	.num_post_div = ARRAY_SIZE(post_div_table_gpll0_out_aux2),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpll0_out_aux2",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gpll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ro_ops,
+	},
+};
+
+/* 1152.0 MHz Configuration */
+static const struct alpha_pll_config gpll10_config = {
+	.l = 0x3c,
+	.alpha = 0x0,
+	.vco_val = BIT(20),
+	.vco_mask = GENMASK(21, 20),
+	.main_output_mask = BIT(0),
+	.config_ctl_val = 0x4001055b,
+	.test_ctl_hi1_val = 0x1,
+};
+
+static struct clk_alpha_pll gpll10 = {
+	.offset = 0xa000,
+	.config = &gpll10_config,
+	.vco_table = spark_vco,
+	.num_vco = ARRAY_SIZE(spark_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.clkr = {
+		.enable_reg = 0x79000,
+		.enable_mask = BIT(10),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll10",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		},
+	},
+};
+
+/* 600.0 MHz Configuration */
+static const struct alpha_pll_config gpll11_config = {
+	.l = 0x1f,
+	.alpha = 0x0,
+	.alpha_hi = 0x40,
+	.alpha_en_mask = BIT(24),
+	.vco_val = BIT(21),
+	.vco_mask = GENMASK(21, 20),
+	.main_output_mask = BIT(0),
+	.config_ctl_val = 0x4001055b,
+	.test_ctl_hi1_val = 0x1,
+};
+
+static struct clk_alpha_pll gpll11 = {
+	.offset = 0xb000,
+	.config = &gpll11_config,
+	.vco_table = default_vco,
+	.num_vco = ARRAY_SIZE(default_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.flags = SUPPORTS_DYNAMIC_UPDATE,
+	.clkr = {
+		.enable_reg = 0x79000,
+		.enable_mask = BIT(11),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll11",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll gpll12 = {
+	.offset = 0xc000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.clkr = {
+		.enable_reg = 0x79000,
+		.enable_mask = BIT(12),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll12",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_gpll12_out_aux2[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv gpll12_out_aux2 = {
+	.offset = 0xc000,
+	.post_div_shift = 8,
+	.post_div_table = post_div_table_gpll12_out_aux2,
+	.num_post_div = ARRAY_SIZE(post_div_table_gpll12_out_aux2),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpll12_out_aux2",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gpll12.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ro_ops,
+	},
+};
+
+static struct clk_alpha_pll gpll3 = {
+	.offset = 0x3000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.clkr = {
+		.enable_reg = 0x79000,
+		.enable_mask = BIT(3),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll3",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_gpll3_out_main[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv gpll3_out_main = {
+	.offset = 0x3000,
+	.post_div_shift = 8,
+	.post_div_table = post_div_table_gpll3_out_main,
+	.num_post_div = ARRAY_SIZE(post_div_table_gpll3_out_main),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpll3_out_main",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gpll3.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ro_ops,
+	},
+};
+
+static struct clk_alpha_pll gpll4 = {
+	.offset = 0x4000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.clkr = {
+		.enable_reg = 0x79000,
+		.enable_mask = BIT(4),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll4",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll gpll5 = {
+	.offset = 0x5000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.clkr = {
+		.enable_reg = 0x79000,
+		.enable_mask = BIT(5),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll5",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll gpll6 = {
+	.offset = 0x6000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.clkr = {
+		.enable_reg = 0x79000,
+		.enable_mask = BIT(6),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll6",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_gpll6_out_main[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv gpll6_out_main = {
+	.offset = 0x6000,
+	.post_div_shift = 8,
+	.post_div_table = post_div_table_gpll6_out_main,
+	.num_post_div = ARRAY_SIZE(post_div_table_gpll6_out_main),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpll6_out_main",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gpll6.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ro_ops,
+	},
+};
+
+static struct clk_alpha_pll gpll7 = {
+	.offset = 0x7000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.clkr = {
+		.enable_reg = 0x79000,
+		.enable_mask = BIT(7),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll7",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_ops,
+		},
+	},
+};
+
+/* 533.2 MHz Configuration */
+static const struct alpha_pll_config gpll8_config = {
+	.l = 0x1b,
+	.alpha = 0x55555555,
+	.alpha_hi = 0xc5,
+	.alpha_en_mask = BIT(24),
+	.vco_val = BIT(21),
+	.vco_mask = GENMASK(21, 20),
+	.main_output_mask = BIT(0),
+	.early_output_mask = BIT(3),
+	.post_div_val = BIT(8),
+	.post_div_mask = GENMASK(11, 8),
+	.config_ctl_val = 0x4001055b,
+	.test_ctl_hi1_val = 0x1,
+};
+
+static struct clk_alpha_pll gpll8 = {
+	.offset = 0x8000,
+	.config = &gpll8_config,
+	.vco_table = default_vco,
+	.num_vco = ARRAY_SIZE(default_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.flags = SUPPORTS_DYNAMIC_UPDATE,
+	.clkr = {
+		.enable_reg = 0x79000,
+		.enable_mask = BIT(8),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll8",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_gpll8_out_main[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv gpll8_out_main = {
+	.offset = 0x8000,
+	.post_div_shift = 8,
+	.post_div_table = post_div_table_gpll8_out_main,
+	.num_post_div = ARRAY_SIZE(post_div_table_gpll8_out_main),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpll8_out_main",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gpll8.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_ro_ops,
+	},
+};
+
+/* 1152.0 MHz Configuration */
+static const struct alpha_pll_config gpll9_config = {
+	.l = 0x3c,
+	.alpha = 0x0,
+	.post_div_val = BIT(8),
+	.post_div_mask = GENMASK(9, 8),
+	.main_output_mask = BIT(0),
+	.early_output_mask = BIT(3),
+	.config_ctl_val = 0x00004289,
+	.test_ctl_val = 0x08000000,
+};
+
+static struct clk_alpha_pll gpll9 = {
+	.offset = 0x9000,
+	.config = &gpll9_config,
+	.vco_table = brammo_vco,
+	.num_vco = ARRAY_SIZE(brammo_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_BRAMMO_EVO],
+	.clkr = {
+		.enable_reg = 0x79000,
+		.enable_mask = BIT(9),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll9",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_gpll9_out_main[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv gpll9_out_main = {
+	.offset = 0x9000,
+	.post_div_shift = 8,
+	.post_div_table = post_div_table_gpll9_out_main,
+	.num_post_div = ARRAY_SIZE(post_div_table_gpll9_out_main),
+	.width = 2,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_BRAMMO_EVO],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpll9_out_main",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gpll9.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_ro_ops,
+	},
+};
+
+static const struct parent_map gcc_parent_map_0[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_EARLY, 1 },
+	{ P_GPLL0_OUT_AUX2, 2 },
+};
+
+static const struct clk_parent_data gcc_parent_data_0[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll0_out_aux2.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_1[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_EARLY, 1 },
+	{ P_GPLL0_OUT_AUX2, 2 },
+	{ P_GPLL6_OUT_MAIN, 4 },
+};
+
+static const struct clk_parent_data gcc_parent_data_1[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll0_out_aux2.clkr.hw },
+	{ .hw = &gpll6_out_main.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_2[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_EARLY, 1 },
+	{ P_GPLL0_OUT_AUX2, 2 },
+	{ P_SLEEP_CLK, 5 },
+};
+
+static const struct clk_parent_data gcc_parent_data_2[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll0_out_aux2.clkr.hw },
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct parent_map gcc_parent_map_3[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_EARLY, 1 },
+	{ P_GPLL9_OUT_EARLY, 2 },
+	{ P_GPLL10_OUT_MAIN, 3 },
+	{ P_GPLL9_OUT_MAIN, 5 },
+	{ P_GPLL3_OUT_MAIN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_3[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll9.clkr.hw },
+	{ .hw = &gpll10.clkr.hw },
+	{ .hw = &gpll9_out_main.clkr.hw },
+	{ .hw = &gpll3_out_main.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_4[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_SLEEP_CLK, 5 },
+};
+
+static const struct clk_parent_data gcc_parent_data_4[] = {
+	{ .index = DT_BI_TCXO },
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct parent_map gcc_parent_map_5[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_EARLY, 1 },
+	{ P_GPLL0_OUT_AUX2, 2 },
+	{ P_GPLL10_OUT_MAIN, 3 },
+	{ P_GPLL4_OUT_MAIN, 5 },
+	{ P_GPLL3_OUT_EARLY, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_5[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll0_out_aux2.clkr.hw },
+	{ .hw = &gpll10.clkr.hw },
+	{ .hw = &gpll4.clkr.hw },
+	{ .hw = &gpll3.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_6[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_EARLY, 1 },
+	{ P_GPLL0_OUT_AUX2, 2 },
+	{ P_GPLL4_OUT_MAIN, 5 },
+	{ P_GPLL3_OUT_MAIN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_6[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll0_out_aux2.clkr.hw },
+	{ .hw = &gpll4.clkr.hw },
+	{ .hw = &gpll3_out_main.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_7[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_EARLY, 1 },
+	{ P_GPLL8_OUT_EARLY, 2 },
+	{ P_GPLL10_OUT_MAIN, 3 },
+	{ P_GPLL8_OUT_MAIN, 4 },
+	{ P_GPLL9_OUT_MAIN, 5 },
+	{ P_GPLL3_OUT_EARLY, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_7[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll8.clkr.hw },
+	{ .hw = &gpll10.clkr.hw },
+	{ .hw = &gpll8_out_main.clkr.hw },
+	{ .hw = &gpll9.clkr.hw },
+	{ .hw = &gpll3.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_8[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_EARLY, 1 },
+	{ P_GPLL8_OUT_EARLY, 2 },
+	{ P_GPLL10_OUT_MAIN, 3 },
+	{ P_GPLL8_OUT_MAIN, 4 },
+	{ P_GPLL9_OUT_MAIN, 5 },
+	{ P_GPLL3_OUT_MAIN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_8[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll8.clkr.hw },
+	{ .hw = &gpll10.clkr.hw },
+	{ .hw = &gpll8_out_main.clkr.hw },
+	{ .hw = &gpll9.clkr.hw },
+	{ .hw = &gpll3_out_main.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_9[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_EARLY, 1 },
+	{ P_GPLL8_OUT_EARLY, 2 },
+	{ P_GPLL10_OUT_MAIN, 3 },
+	{ P_GPLL6_OUT_MAIN, 4 },
+	{ P_GPLL9_OUT_MAIN, 5 },
+	{ P_GPLL3_OUT_EARLY, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_9[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll8.clkr.hw },
+	{ .hw = &gpll10.clkr.hw },
+	{ .hw = &gpll6_out_main.clkr.hw },
+	{ .hw = &gpll9.clkr.hw },
+	{ .hw = &gpll3.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_10[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_EARLY, 1 },
+	{ P_SLEEP_CLK, 5 },
+};
+
+static const struct clk_parent_data gcc_parent_data_10[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct parent_map gcc_parent_map_11[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL12_OUT_EARLY, 1 },
+	{ P_GPLL12_OUT_AUX2, 4 },
+	{ P_GPLL3_OUT_EARLY, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_11[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll12.clkr.hw },
+	{ .hw = &gpll12_out_aux2.clkr.hw },
+	{ .hw = &gpll3.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_12[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL12_OUT_EARLY, 1 },
+	{ P_GPLL0_OUT_AUX2, 2 },
+	{ P_GPLL12_OUT_AUX2, 4 },
+};
+
+static const struct clk_parent_data gcc_parent_data_12[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll12.clkr.hw },
+	{ .hw = &gpll0_out_aux2.clkr.hw },
+	{ .hw = &gpll12_out_aux2.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_13[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_EARLY, 1 },
+};
+
+static const struct clk_parent_data gcc_parent_data_13[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_14[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_EARLY, 1 },
+	{ P_GPLL0_OUT_AUX2, 2 },
+	{ P_GPLL10_OUT_MAIN, 3 },
+	{ P_GPLL8_OUT_MAIN, 4 },
+	{ P_GPLL9_OUT_MAIN, 5 },
+	{ P_GPLL3_OUT_EARLY, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_14[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll0_out_aux2.clkr.hw },
+	{ .hw = &gpll10.clkr.hw },
+	{ .hw = &gpll8_out_main.clkr.hw },
+	{ .hw = &gpll9.clkr.hw },
+	{ .hw = &gpll3.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_15[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_EARLY, 1 },
+	{ P_GPLL8_OUT_EARLY, 2 },
+	{ P_GPLL10_OUT_MAIN, 3 },
+	{ P_GPLL6_OUT_EARLY, 5 },
+	{ P_GPLL3_OUT_MAIN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_15[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll8.clkr.hw },
+	{ .hw = &gpll10.clkr.hw },
+	{ .hw = &gpll6.clkr.hw },
+	{ .hw = &gpll3_out_main.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_21[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_EARLY, 1 },
+	{ P_GPLL0_OUT_AUX2, 2 },
+	{ P_GPLL7_OUT_MAIN, 3 },
+	{ P_GPLL4_OUT_MAIN, 5 },
+};
+
+static const struct clk_parent_data gcc_parent_data_21[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll0_out_aux2.clkr.hw },
+	{ .hw = &gpll7.clkr.hw },
+	{ .hw = &gpll4.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_22[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL12_OUT_EARLY, 1 },
+	{ P_GPLL5_OUT_MAIN, 3 },
+	{ P_GPLL12_OUT_AUX2, 4 },
+	{ P_GPLL3_OUT_EARLY, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_22[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll12.clkr.hw },
+	{ .hw = &gpll5.clkr.hw },
+	{ .hw = &gpll12_out_aux2.clkr.hw },
+	{ .hw = &gpll3.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_24[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL11_OUT_MAIN, 1 },
+	{ P_GPLL11_OUT_AUX, 2 },
+	{ P_GPLL11_OUT_AUX2, 3 },
+};
+
+static const struct clk_parent_data gcc_parent_data_24[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll11.clkr.hw },
+	{ .hw = &gpll11.clkr.hw },
+	{ .hw = &gpll11.clkr.hw },
+};
+
+static struct clk_regmap_phy_mux gcc_emac0_cc_sgmiiphy_rx_clk_src = {
+	.reg = 0xad048,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac0_cc_sgmiiphy_rx_clk_src",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_EMAC0_SGMIIPHY_RCLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_emac0_cc_sgmiiphy_tx_clk_src = {
+	.reg = 0xad040,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac0_cc_sgmiiphy_tx_clk_src",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_EMAC0_SGMIIPHY_TCLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_emac1_cc_sgmiiphy_rx_clk_src = {
+	.reg = 0xae048,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac1_cc_sgmiiphy_rx_clk_src",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_EMAC1_SGMIIPHY_RCLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_emac1_cc_sgmiiphy_tx_clk_src = {
+	.reg = 0xae040,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac1_cc_sgmiiphy_tx_clk_src",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_EMAC1_SGMIIPHY_TCLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_pcie_pipe_clk_src = {
+	.reg = 0xaf058,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_PCIE_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_usb3_prim_phy_pipe_clk_src = {
+	.reg = 0x1a05c,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb3_prim_phy_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_axi_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(150000000, P_GPLL0_OUT_AUX2, 2, 0, 0),
+	F(200000000, P_GPLL0_OUT_AUX2, 1.5, 0, 0),
+	F(300000000, P_GPLL0_OUT_AUX2, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_camss_axi_clk_src = {
+	.cmd_rcgr = 0x5802c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_5,
+	.freq_tbl = ftbl_gcc_camss_axi_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_camss_axi_clk_src",
+		.parent_data = gcc_parent_data_5,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_5),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_cci_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(37500000, P_GPLL0_OUT_AUX2, 8, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_camss_cci_clk_src = {
+	.cmd_rcgr = 0x56000,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_14,
+	.freq_tbl = ftbl_gcc_camss_cci_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_camss_cci_clk_src",
+		.parent_data = gcc_parent_data_14,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_14),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_csi0phytimer_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(100000000, P_GPLL0_OUT_AUX2, 3, 0, 0),
+	F(200000000, P_GPLL0_OUT_AUX2, 1.5, 0, 0),
+	F(268800000, P_GPLL4_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_camss_csi0phytimer_clk_src = {
+	.cmd_rcgr = 0x45000,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_6,
+	.freq_tbl = ftbl_gcc_camss_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_camss_csi0phytimer_clk_src",
+		.parent_data = gcc_parent_data_6,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_6),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_camss_csi1phytimer_clk_src = {
+	.cmd_rcgr = 0x4501c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_6,
+	.freq_tbl = ftbl_gcc_camss_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_camss_csi1phytimer_clk_src",
+		.parent_data = gcc_parent_data_6,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_6),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_mclk0_clk_src[] = {
+	F(19200000, P_GPLL9_OUT_EARLY, 1, 1, 60),
+	F(24000000, P_GPLL9_OUT_MAIN, 1, 1, 24),
+	F(64000000, P_GPLL9_OUT_EARLY, 9, 1, 2),
+	{ }
+};
+
+static struct clk_rcg2 gcc_camss_mclk0_clk_src = {
+	.cmd_rcgr = 0x51000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_camss_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_camss_mclk0_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_camss_mclk1_clk_src = {
+	.cmd_rcgr = 0x5101c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_camss_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_camss_mclk1_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_camss_mclk2_clk_src = {
+	.cmd_rcgr = 0x51038,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_camss_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_camss_mclk2_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_camss_mclk3_clk_src = {
+	.cmd_rcgr = 0x51054,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_camss_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_camss_mclk3_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_ope_ahb_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(171428571, P_GPLL0_OUT_EARLY, 3.5, 0, 0),
+	F(240000000, P_GPLL0_OUT_EARLY, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_camss_ope_ahb_clk_src = {
+	.cmd_rcgr = 0x55024,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_7,
+	.freq_tbl = ftbl_gcc_camss_ope_ahb_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_camss_ope_ahb_clk_src",
+		.parent_data = gcc_parent_data_7,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_7),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_ope_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(200000000, P_GPLL8_OUT_MAIN, 2, 0, 0),
+	F(266600000, P_GPLL8_OUT_MAIN, 1, 0, 0),
+	F(465000000, P_GPLL8_OUT_MAIN, 1, 0, 0),
+	F(580000000, P_GPLL8_OUT_EARLY, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_camss_ope_clk_src = {
+	.cmd_rcgr = 0x55004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_7,
+	.freq_tbl = ftbl_gcc_camss_ope_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_camss_ope_clk_src",
+		.parent_data = gcc_parent_data_7,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_7),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_tfe_0_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(128000000, P_GPLL10_OUT_MAIN, 9, 0, 0),
+	F(135529412, P_GPLL10_OUT_MAIN, 8.5, 0, 0),
+	F(144000000, P_GPLL10_OUT_MAIN, 8, 0, 0),
+	F(153600000, P_GPLL10_OUT_MAIN, 7.5, 0, 0),
+	F(164571429, P_GPLL10_OUT_MAIN, 7, 0, 0),
+	F(177230769, P_GPLL10_OUT_MAIN, 6.5, 0, 0),
+	F(192000000, P_GPLL10_OUT_MAIN, 6, 0, 0),
+	F(209454545, P_GPLL10_OUT_MAIN, 5.5, 0, 0),
+	F(230400000, P_GPLL10_OUT_MAIN, 5, 0, 0),
+	F(256000000, P_GPLL10_OUT_MAIN, 4.5, 0, 0),
+	F(288000000, P_GPLL10_OUT_MAIN, 4, 0, 0),
+	F(329142857, P_GPLL10_OUT_MAIN, 3.5, 0, 0),
+	F(384000000, P_GPLL10_OUT_MAIN, 3, 0, 0),
+	F(460800000, P_GPLL10_OUT_MAIN, 2.5, 0, 0),
+	F(576000000, P_GPLL10_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_camss_tfe_0_clk_src = {
+	.cmd_rcgr = 0x52004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_8,
+	.freq_tbl = ftbl_gcc_camss_tfe_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_camss_tfe_0_clk_src",
+		.parent_data = gcc_parent_data_8,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_8),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_tfe_0_csid_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(120000000, P_GPLL0_OUT_EARLY, 5, 0, 0),
+	F(192000000, P_GPLL6_OUT_MAIN, 2, 0, 0),
+	F(240000000, P_GPLL0_OUT_EARLY, 2.5, 0, 0),
+	F(384000000, P_GPLL6_OUT_MAIN, 1, 0, 0),
+	F(426400000, P_GPLL3_OUT_EARLY, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_camss_tfe_0_csid_clk_src = {
+	.cmd_rcgr = 0x52094,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_9,
+	.freq_tbl = ftbl_gcc_camss_tfe_0_csid_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_camss_tfe_0_csid_clk_src",
+		.parent_data = gcc_parent_data_9,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_9),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_camss_tfe_1_clk_src = {
+	.cmd_rcgr = 0x52024,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_8,
+	.freq_tbl = ftbl_gcc_camss_tfe_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_camss_tfe_1_clk_src",
+		.parent_data = gcc_parent_data_8,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_8),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_camss_tfe_1_csid_clk_src = {
+	.cmd_rcgr = 0x520b4,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_9,
+	.freq_tbl = ftbl_gcc_camss_tfe_0_csid_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_camss_tfe_1_csid_clk_src",
+		.parent_data = gcc_parent_data_9,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_9),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_tfe_cphy_rx_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(240000000, P_GPLL0_OUT_EARLY, 2.5, 0, 0),
+	F(341333333, P_GPLL6_OUT_EARLY, 1, 4, 9),
+	F(384000000, P_GPLL6_OUT_EARLY, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_camss_tfe_cphy_rx_clk_src = {
+	.cmd_rcgr = 0x52064,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_15,
+	.freq_tbl = ftbl_gcc_camss_tfe_cphy_rx_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_camss_tfe_cphy_rx_clk_src",
+		.parent_data = gcc_parent_data_15,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_15),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_top_ahb_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(40000000, P_GPLL0_OUT_AUX2, 7.5, 0, 0),
+	F(80000000, P_GPLL0_OUT_EARLY, 7.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_camss_top_ahb_clk_src = {
+	.cmd_rcgr = 0x58010,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_5,
+	.freq_tbl = ftbl_gcc_camss_top_ahb_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_camss_top_ahb_clk_src",
+		.parent_data = gcc_parent_data_5,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_5),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_emac0_axi_clk_src[] = {
+	F(75000000, P_GPLL0_OUT_AUX2, 4, 0, 0),
+	F(120000000, P_GPLL0_OUT_AUX2, 2.5, 0, 0),
+	F(150000000, P_GPLL0_OUT_AUX2, 2, 0, 0),
+	F(200000000, P_GPLL0_OUT_AUX2, 1.5, 0, 0),
+	F(240000000, P_GPLL0_OUT_EARLY, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_emac0_axi_clk_src = {
+	.cmd_rcgr = 0x109dc,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_emac0_axi_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_emac0_axi_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_emac0_phy_aux_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_emac0_phy_aux_clk_src = {
+	.cmd_rcgr = 0xad01c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_10,
+	.freq_tbl = ftbl_gcc_emac0_phy_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_emac0_phy_aux_clk_src",
+		.parent_data = gcc_parent_data_10,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_10),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_emac0_ptp_clk_src[] = {
+	F(250000000, P_GPLL12_OUT_AUX2, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_emac0_ptp_clk_src = {
+	.cmd_rcgr = 0xad064,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_11,
+	.freq_tbl = ftbl_gcc_emac0_ptp_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_emac0_ptp_clk_src",
+		.parent_data = gcc_parent_data_11,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_11),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_emac0_rgmii_clk_src[] = {
+	F(50000000, P_GPLL0_OUT_AUX2, 6, 0, 0),
+	F(125000000, P_GPLL12_OUT_AUX2, 4, 0, 0),
+	F(250000000, P_GPLL12_OUT_EARLY, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_emac0_rgmii_clk_src = {
+	.cmd_rcgr = 0xad04c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_12,
+	.freq_tbl = ftbl_gcc_emac0_rgmii_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_emac0_rgmii_clk_src",
+		.parent_data = gcc_parent_data_12,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_12),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_emac1_axi_clk_src = {
+	.cmd_rcgr = 0x109fc,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_emac0_axi_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_emac1_axi_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_emac1_phy_aux_clk_src = {
+	.cmd_rcgr = 0xae01c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_10,
+	.freq_tbl = ftbl_gcc_emac0_phy_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_emac1_phy_aux_clk_src",
+		.parent_data = gcc_parent_data_10,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_10),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_emac1_ptp_clk_src = {
+	.cmd_rcgr = 0xae064,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_11,
+	.freq_tbl = ftbl_gcc_emac0_ptp_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_emac1_ptp_clk_src",
+		.parent_data = gcc_parent_data_11,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_11),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_emac1_rgmii_clk_src = {
+	.cmd_rcgr = 0xae04c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_12,
+	.freq_tbl = ftbl_gcc_emac0_rgmii_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_emac1_rgmii_clk_src",
+		.parent_data = gcc_parent_data_12,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_12),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = {
+	F(25000000, P_GPLL0_OUT_AUX2, 12, 0, 0),
+	F(50000000, P_GPLL0_OUT_AUX2, 6, 0, 0),
+	F(100000000, P_GPLL0_OUT_AUX2, 3, 0, 0),
+	F(200000000, P_GPLL0_OUT_AUX2, 1.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_gp1_clk_src = {
+	.cmd_rcgr = 0x4d004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gp1_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_gp2_clk_src = {
+	.cmd_rcgr = 0x4e004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gp2_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_gp3_clk_src = {
+	.cmd_rcgr = 0x4f004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gp3_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie_aux_clk_src = {
+	.cmd_rcgr = 0xaf074,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_4,
+	.freq_tbl = ftbl_gcc_emac0_phy_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_aux_clk_src",
+		.parent_data = gcc_parent_data_4,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_4),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie_aux_phy_clk_src = {
+	.cmd_rcgr = 0xaf05c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_4,
+	.freq_tbl = ftbl_gcc_emac0_phy_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_aux_phy_clk_src",
+		.parent_data = gcc_parent_data_4,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_4),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pcie_rchng_phy_clk_src[] = {
+	F(100000000, P_GPLL0_OUT_AUX2, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcie_rchng_phy_clk_src = {
+	.cmd_rcgr = 0xaf028,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_rchng_phy_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_rchng_phy_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pdm2_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(60000000, P_GPLL0_OUT_AUX2, 5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pdm2_clk_src = {
+	.cmd_rcgr = 0x20010,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pdm2_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pdm2_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap0_s0_clk_src[] = {
+	F(7372800, P_GPLL0_OUT_AUX2, 1, 384, 15625),
+	F(14745600, P_GPLL0_OUT_AUX2, 1, 768, 15625),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(29491200, P_GPLL0_OUT_AUX2, 1, 1536, 15625),
+	F(32000000, P_GPLL0_OUT_AUX2, 1, 8, 75),
+	F(48000000, P_GPLL0_OUT_AUX2, 1, 4, 25),
+	F(64000000, P_GPLL0_OUT_AUX2, 1, 16, 75),
+	F(75000000, P_GPLL0_OUT_AUX2, 4, 0, 0),
+	F(80000000, P_GPLL0_OUT_AUX2, 1, 4, 15),
+	F(96000000, P_GPLL0_OUT_AUX2, 1, 8, 25),
+	F(100000000, P_GPLL0_OUT_AUX2, 3, 0, 0),
+	F(102400000, P_GPLL0_OUT_AUX2, 1, 128, 375),
+	F(112000000, P_GPLL0_OUT_AUX2, 1, 28, 75),
+	F(117964800, P_GPLL0_OUT_AUX2, 1, 6144, 15625),
+	F(120000000, P_GPLL0_OUT_AUX2, 2.5, 0, 0),
+	F(128000000, P_GPLL6_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_init_data gcc_qupv3_wrap0_s0_clk_src_init = {
+	.name = "gcc_qupv3_wrap0_s0_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.ops = &clk_rcg2_shared_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s0_clk_src = {
+	.cmd_rcgr = 0x1f148,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap0_s0_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap0_s1_clk_src_init = {
+	.name = "gcc_qupv3_wrap0_s1_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.ops = &clk_rcg2_shared_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s1_clk_src = {
+	.cmd_rcgr = 0x1f278,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap0_s1_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap0_s2_clk_src_init = {
+	.name = "gcc_qupv3_wrap0_s2_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.ops = &clk_rcg2_shared_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s2_clk_src = {
+	.cmd_rcgr = 0x1f3a8,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap0_s2_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap0_s3_clk_src_init = {
+	.name = "gcc_qupv3_wrap0_s3_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.ops = &clk_rcg2_shared_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s3_clk_src = {
+	.cmd_rcgr = 0x1f4d8,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap0_s3_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap0_s4_clk_src_init = {
+	.name = "gcc_qupv3_wrap0_s4_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.ops = &clk_rcg2_shared_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s4_clk_src = {
+	.cmd_rcgr = 0x1f608,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap0_s4_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap0_s5_clk_src_init = {
+	.name = "gcc_qupv3_wrap0_s5_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.ops = &clk_rcg2_shared_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s5_clk_src = {
+	.cmd_rcgr = 0x1f738,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap0_s5_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap0_s6_clk_src_init = {
+	.name = "gcc_qupv3_wrap0_s6_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.ops = &clk_rcg2_shared_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s6_clk_src = {
+	.cmd_rcgr = 0x1f868,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap0_s6_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap0_s7_clk_src_init = {
+	.name = "gcc_qupv3_wrap0_s7_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.ops = &clk_rcg2_shared_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s7_clk_src = {
+	.cmd_rcgr = 0x1f998,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap0_s7_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap0_s8_clk_src_init = {
+	.name = "gcc_qupv3_wrap0_s8_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.ops = &clk_rcg2_shared_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s8_clk_src = {
+	.cmd_rcgr = 0x1fac8,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap0_s8_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap0_s9_clk_src_init = {
+	.name = "gcc_qupv3_wrap0_s9_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.ops = &clk_rcg2_shared_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s9_clk_src = {
+	.cmd_rcgr = 0x1fbf8,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap0_s9_clk_src_init,
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc1_apps_clk_src[] = {
+	F(144000, P_BI_TCXO, 16, 3, 25),
+	F(400000, P_BI_TCXO, 12, 1, 4),
+	F(20000000, P_GPLL0_OUT_AUX2, 5, 1, 3),
+	F(25000000, P_GPLL0_OUT_AUX2, 6, 1, 2),
+	F(50000000, P_GPLL0_OUT_AUX2, 6, 0, 0),
+	F(100000000, P_GPLL0_OUT_AUX2, 3, 0, 0),
+	F(192000000, P_GPLL6_OUT_MAIN, 2, 0, 0),
+	F(384000000, P_GPLL6_OUT_MAIN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sdcc1_apps_clk_src = {
+	.cmd_rcgr = 0x38028,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_sdcc1_apps_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_sdcc1_apps_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_shared_floor_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc1_ice_core_clk_src[] = {
+	F(75000000, P_GPLL0_OUT_AUX2, 4, 0, 0),
+	F(100000000, P_GPLL0_OUT_AUX2, 3, 0, 0),
+	F(150000000, P_GPLL0_OUT_AUX2, 2, 0, 0),
+	F(200000000, P_GPLL0_OUT_EARLY, 3, 0, 0),
+	F(300000000, P_GPLL0_OUT_AUX2, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = {
+	.cmd_rcgr = 0x38010,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_sdcc1_ice_core_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_sdcc1_ice_core_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_shared_floor_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc2_apps_clk_src[] = {
+	F(400000, P_BI_TCXO, 12, 1, 4),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(25000000, P_GPLL0_OUT_AUX2, 12, 0, 0),
+	F(50000000, P_GPLL0_OUT_AUX2, 6, 0, 0),
+	F(100000000, P_GPLL0_OUT_AUX2, 3, 0, 0),
+	F(202000000, P_GPLL7_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sdcc2_apps_clk_src = {
+	.cmd_rcgr = 0x1e00c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_21,
+	.freq_tbl = ftbl_gcc_sdcc2_apps_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_sdcc2_apps_clk_src",
+		.parent_data = gcc_parent_data_21,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_21),
+		.ops = &clk_rcg2_shared_floor_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_tscss_clk_src = {
+	.cmd_rcgr = 0xac004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_22,
+	.freq_tbl = ftbl_gcc_emac0_ptp_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_tscss_clk_src",
+		.parent_data = gcc_parent_data_22,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_22),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_usb20_master_clk_src[] = {
+	F(60000000, P_GPLL0_OUT_AUX2, 5, 0, 0),
+	F(120000000, P_GPLL0_OUT_EARLY, 5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_usb20_master_clk_src = {
+	.cmd_rcgr = 0xb003c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_usb20_master_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb20_master_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb20_mock_utmi_clk_src = {
+	.cmd_rcgr = 0xb0020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_13,
+	.freq_tbl = ftbl_gcc_emac0_phy_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb20_mock_utmi_clk_src",
+		.parent_data = gcc_parent_data_13,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_13),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_usb30_prim_master_clk_src[] = {
+	F(66666667, P_GPLL0_OUT_AUX2, 4.5, 0, 0),
+	F(133333333, P_GPLL0_OUT_EARLY, 4.5, 0, 0),
+	F(200000000, P_GPLL0_OUT_EARLY, 3, 0, 0),
+	F(240000000, P_GPLL0_OUT_EARLY, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_usb30_prim_master_clk_src = {
+	.cmd_rcgr = 0x1a01c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_usb30_prim_master_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb30_prim_master_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb30_prim_mock_utmi_clk_src = {
+	.cmd_rcgr = 0x1a034,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_emac0_phy_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb30_prim_mock_utmi_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb3_prim_phy_aux_clk_src = {
+	.cmd_rcgr = 0x1a060,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_4,
+	.freq_tbl = ftbl_gcc_emac0_phy_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb3_prim_phy_aux_clk_src",
+		.parent_data = gcc_parent_data_4,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_4),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_video_venus_clk_src[] = {
+	F(133333333, P_GPLL11_OUT_MAIN, 4.5, 0, 0),
+	F(240000000, P_GPLL11_OUT_MAIN, 2.5, 0, 0),
+	F(300000000, P_GPLL11_OUT_MAIN, 2, 0, 0),
+	F(384000000, P_GPLL11_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_video_venus_clk_src = {
+	.cmd_rcgr = 0x6d000,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_24,
+	.freq_tbl = ftbl_gcc_video_venus_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_video_venus_clk_src",
+		.parent_data = gcc_parent_data_24,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_24),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_disp_gpll0_clk_src = {
+	.reg = 0x17058,
+	.shift = 0,
+	.width = 2,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_disp_gpll0_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gpll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_regmap_div_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_usb20_mock_utmi_postdiv_clk_src = {
+	.reg = 0xb0038,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb20_mock_utmi_postdiv_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_usb20_mock_utmi_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_usb30_prim_mock_utmi_postdiv_clk_src = {
+	.reg = 0x1a04c,
+	.shift = 0,
+	.width = 2,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb30_prim_mock_utmi_postdiv_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_usb30_prim_mock_utmi_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_branch gcc_ahb2phy_csi_clk = {
+	.halt_reg = 0x1d004,
+	.halt_check = BRANCH_HALT_DELAY,
+	.hwcg_reg = 0x1d004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x1d004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ahb2phy_csi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ahb2phy_usb_clk = {
+	.halt_reg = 0x1d008,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x1d008,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x1d008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ahb2phy_usb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_boot_rom_ahb_clk = {
+	.halt_reg = 0x23004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x23004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7900c,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_boot_rom_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cam_throttle_nrt_clk = {
+	.halt_reg = 0x17070,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x17070,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(16),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cam_throttle_nrt_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cam_throttle_rt_clk = {
+	.halt_reg = 0x1706c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x1706c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(15),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cam_throttle_rt_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_axi_clk = {
+	.halt_reg = 0x58044,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x58044,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_camnoc_atb_clk = {
+	.halt_reg = 0x5804c,
+	.halt_check = BRANCH_HALT_DELAY,
+	.hwcg_reg = 0x5804c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x5804c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_camnoc_atb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_camnoc_dragonlink_atb_clk = {
+	.halt_reg = 0x58060,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x58060,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x58060,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_camnoc_dragonlink_atb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_camnoc_nts_xo_clk = {
+	.halt_reg = 0x58050,
+	.halt_check = BRANCH_HALT_DELAY,
+	.hwcg_reg = 0x58050,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x58050,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_camnoc_nts_xo_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_cci_0_clk = {
+	.halt_reg = 0x56018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x56018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_cci_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_cci_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_cphy_0_clk = {
+	.halt_reg = 0x52088,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x52088,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_cphy_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_tfe_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_cphy_1_clk = {
+	.halt_reg = 0x5208c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x5208c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_cphy_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_tfe_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi0phytimer_clk = {
+	.halt_reg = 0x45018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x45018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_csi0phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_csi0phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi1phytimer_clk = {
+	.halt_reg = 0x45034,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x45034,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_csi1phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_csi1phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_mclk0_clk = {
+	.halt_reg = 0x51018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x51018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_mclk0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_mclk0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_mclk1_clk = {
+	.halt_reg = 0x51034,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x51034,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_mclk1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_mclk1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_mclk2_clk = {
+	.halt_reg = 0x51050,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x51050,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_mclk2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_mclk2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_mclk3_clk = {
+	.halt_reg = 0x5106c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x5106c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_mclk3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_mclk3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_nrt_axi_clk = {
+	.halt_reg = 0x58054,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x58054,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_nrt_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_ope_ahb_clk = {
+	.halt_reg = 0x5503c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x5503c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_ope_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_ope_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_ope_clk = {
+	.halt_reg = 0x5501c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x5501c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_ope_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_ope_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_rt_axi_clk = {
+	.halt_reg = 0x5805c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x5805c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_rt_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_tfe_0_clk = {
+	.halt_reg = 0x5201c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x5201c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_tfe_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_tfe_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_tfe_0_cphy_rx_clk = {
+	.halt_reg = 0x5207c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x5207c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_tfe_0_cphy_rx_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_tfe_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_tfe_0_csid_clk = {
+	.halt_reg = 0x520ac,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x520ac,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_tfe_0_csid_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_tfe_0_csid_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_tfe_1_clk = {
+	.halt_reg = 0x5203c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x5203c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_tfe_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_tfe_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_tfe_1_cphy_rx_clk = {
+	.halt_reg = 0x52080,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x52080,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_tfe_1_cphy_rx_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_tfe_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_tfe_1_csid_clk = {
+	.halt_reg = 0x520cc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x520cc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_tfe_1_csid_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_tfe_1_csid_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_top_ahb_clk = {
+	.halt_reg = 0x58028,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x58028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camss_top_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_camss_top_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cfg_noc_usb2_prim_axi_clk = {
+	.halt_reg = 0x111c4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x111c4,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x111c4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cfg_noc_usb2_prim_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb20_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cfg_noc_usb3_prim_axi_clk = {
+	.halt_reg = 0x1a07c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x1a07c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x1a07c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cfg_noc_usb3_prim_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb30_prim_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ddrss_gpu_axi_clk = {
+	.halt_reg = 0x71000,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x71000,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x71000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ddrss_gpu_axi_clk",
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ddrss_memnoc_pcie_sf_clk = {
+	.halt_reg = 0x29044,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x29044,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x29044,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ddrss_memnoc_pcie_sf_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_disp_gpll0_div_clk_src = {
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(11),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_disp_gpll0_div_clk_src",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_disp_gpll0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_disp_hf_axi_clk = {
+	.halt_reg = 0x17020,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x17020,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x17020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_disp_hf_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_disp_throttle_core_clk = {
+	.halt_reg = 0x17064,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x17064,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(13),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_disp_throttle_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac0_ahb_clk = {
+	.halt_reg = 0xad010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xad010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0xad010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac0_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac0_axi_clk = {
+	.halt_reg = 0xad014,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xad014,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0xad014,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac0_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac0_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac0_axi_sys_noc_clk = {
+	.halt_reg = 0x109d4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x109d4,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x109d4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac0_axi_sys_noc_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac0_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac0_cc_sgmiiphy_rx_clk = {
+	.halt_reg = 0xad044,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0xad044,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac0_cc_sgmiiphy_rx_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac0_cc_sgmiiphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac0_cc_sgmiiphy_tx_clk = {
+	.halt_reg = 0xad03c,
+	.halt_check = BRANCH_HALT_DELAY,
+	.hwcg_reg = 0xad03c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0xad03c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac0_cc_sgmiiphy_tx_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac0_cc_sgmiiphy_tx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac0_phy_aux_clk = {
+	.halt_reg = 0xad018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xad018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac0_phy_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac0_phy_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac0_ptp_clk = {
+	.halt_reg = 0xad034,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xad034,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac0_ptp_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac0_ptp_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac0_rgmii_clk = {
+	.halt_reg = 0xad038,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xad038,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac0_rgmii_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac0_rgmii_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac1_ahb_clk = {
+	.halt_reg = 0xae010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xae010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0xae010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac1_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac1_axi_clk = {
+	.halt_reg = 0xae014,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xae014,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0xae014,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac1_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac1_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac1_axi_sys_noc_clk = {
+	.halt_reg = 0x109f4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x109f4,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x109f4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac1_axi_sys_noc_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac1_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac1_cc_sgmiiphy_rx_clk = {
+	.halt_reg = 0xae044,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0xae044,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac1_cc_sgmiiphy_rx_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac1_cc_sgmiiphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac1_cc_sgmiiphy_tx_clk = {
+	.halt_reg = 0xae03c,
+	.halt_check = BRANCH_HALT_DELAY,
+	.hwcg_reg = 0xae03c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0xae03c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac1_cc_sgmiiphy_tx_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac1_cc_sgmiiphy_tx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac1_phy_aux_clk = {
+	.halt_reg = 0xae018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xae018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac1_phy_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac1_phy_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac1_ptp_clk = {
+	.halt_reg = 0xae034,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xae034,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac1_ptp_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac1_ptp_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac1_rgmii_clk = {
+	.halt_reg = 0xae038,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xae038,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac1_rgmii_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac1_rgmii_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp1_clk = {
+	.halt_reg = 0x4d000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x4d000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gp1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gp1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp2_clk = {
+	.halt_reg = 0x4e000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x4e000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gp2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gp2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp3_clk = {
+	.halt_reg = 0x4f000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x4f000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gp3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gp3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_gpll0_clk_src = {
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x7900c,
+		.enable_mask = BIT(18),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpu_gpll0_clk_src",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpll0.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_gpll0_div_clk_src = {
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x7900c,
+		.enable_mask = BIT(19),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpu_gpll0_div_clk_src",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpll0_out_aux2.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_memnoc_gfx_clk = {
+	.halt_reg = 0x3600c,
+	.halt_check = BRANCH_VOTED,
+	.hwcg_reg = 0x3600c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x3600c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpu_memnoc_gfx_clk",
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_smmu_vote_clk = {
+	.halt_reg = 0x7d000,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7d000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpu_smmu_vote_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_snoc_dvm_gfx_clk = {
+	.halt_reg = 0x36018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x36018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpu_snoc_dvm_gfx_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_throttle_core_clk = {
+	.halt_reg = 0x36048,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x36048,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7900c,
+		.enable_mask = BIT(21),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpu_throttle_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mmu_tcu_vote_clk = {
+	.halt_reg = 0x7d06c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7d06c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_mmu_tcu_vote_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_aux_clk = {
+	.halt_reg = 0xaf044,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xaf044,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x79018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_cfg_ahb_clk = {
+	.halt_reg = 0xaf010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xaf010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7900c,
+		.enable_mask = BIT(27),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_clkref_en = {
+	.halt_reg = 0xb8000,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0xb8000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_clkref_en",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_mstr_axi_clk = {
+	.halt_reg = 0xaf020,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xaf020,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7900c,
+		.enable_mask = BIT(30),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_mstr_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_pipe_clk = {
+	.halt_reg = 0xaf050,
+	.halt_check = BRANCH_HALT_DELAY,
+	.hwcg_reg = 0xaf050,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x79018,
+		.enable_mask = BIT(2),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_pipe_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_pipe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_rchng_phy_clk = {
+	.halt_reg = 0xaf040,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xaf040,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7900c,
+		.enable_mask = BIT(31),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_rchng_phy_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_rchng_phy_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_sleep_clk = {
+	.halt_reg = 0xaf04c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xaf04c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x79018,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_sleep_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_aux_phy_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_slv_axi_clk = {
+	.halt_reg = 0xaf018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7900c,
+		.enable_mask = BIT(29),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_slv_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_slv_q2a_axi_clk = {
+	.halt_reg = 0xaf014,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xaf014,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7900c,
+		.enable_mask = BIT(28),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_slv_q2a_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_tbu_clk = {
+	.halt_reg = 0xaf098,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xaf098,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x79018,
+		.enable_mask = BIT(6),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_tbu_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_throttle_core_clk = {
+	.halt_reg = 0xaf094,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xaf094,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x79018,
+		.enable_mask = BIT(5),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_throttle_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_throttle_xo_clk = {
+	.halt_reg = 0xaf090,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x79018,
+		.enable_mask = BIT(4),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_throttle_xo_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_tile_axi_sys_noc_clk = {
+	.halt_reg = 0x10f2c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x10f2c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x10f2c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_tile_axi_sys_noc_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac0_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm2_clk = {
+	.halt_reg = 0x2000c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2000c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pdm2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pdm2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm_ahb_clk = {
+	.halt_reg = 0x20004,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x20004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x20004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pdm_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm_xo4_clk = {
+	.halt_reg = 0x20008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x20008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pdm_xo4_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pwm0_xo512_clk = {
+	.halt_reg = 0x2002c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2002c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pwm0_xo512_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_camera_nrt_ahb_clk = {
+	.halt_reg = 0x17014,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x17014,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(9),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_camera_nrt_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_camera_rt_ahb_clk = {
+	.halt_reg = 0x17060,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x17060,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(12),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_camera_rt_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_disp_ahb_clk = {
+	.halt_reg = 0x17018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x17018,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(10),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_disp_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_gpu_cfg_ahb_clk = {
+	.halt_reg = 0x36040,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x36040,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7900c,
+		.enable_mask = BIT(20),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_gpu_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_pcie_cfg_ahb_clk = {
+	.halt_reg = 0xaf08c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xaf08c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x79018,
+		.enable_mask = BIT(3),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_pcie_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qmip_video_vcodec_ahb_clk = {
+	.halt_reg = 0x17010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x17010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(8),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qmip_video_vcodec_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_core_2x_clk = {
+	.halt_reg = 0x1f014,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(21),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_core_2x_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_core_clk = {
+	.halt_reg = 0x1f00c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(20),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s0_clk = {
+	.halt_reg = 0x1f144,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(22),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_s0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap0_s0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s1_clk = {
+	.halt_reg = 0x1f274,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(23),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_s1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap0_s1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s2_clk = {
+	.halt_reg = 0x1f3a4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(24),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_s2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap0_s2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s3_clk = {
+	.halt_reg = 0x1f4d4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(25),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_s3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap0_s3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s4_clk = {
+	.halt_reg = 0x1f604,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(26),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_s4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap0_s4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s5_clk = {
+	.halt_reg = 0x1f734,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(27),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_s5_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap0_s5_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s6_clk = {
+	.halt_reg = 0x1f864,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(28),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_s6_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap0_s6_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s7_clk = {
+	.halt_reg = 0x1f994,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(29),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_s7_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap0_s7_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s8_clk = {
+	.halt_reg = 0x1fac4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(30),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_s8_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap0_s8_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s9_clk = {
+	.halt_reg = 0x1fbf4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(31),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_s9_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap0_s9_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_0_m_ahb_clk = {
+	.halt_reg = 0x1f004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x1f004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(18),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_0_m_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_0_s_ahb_clk = {
+	.halt_reg = 0x1f008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x1f008,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(19),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_0_s_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_ahb_clk = {
+	.halt_reg = 0x38008,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x38008,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x38008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc1_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_apps_clk = {
+	.halt_reg = 0x38004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x38004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc1_apps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sdcc1_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_ice_core_clk = {
+	.halt_reg = 0x3800c,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x3800c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x3800c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc1_ice_core_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sdcc1_ice_core_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_ahb_clk = {
+	.halt_reg = 0x1e008,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x1e008,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x1e008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc2_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_apps_clk = {
+	.halt_reg = 0x1e004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1e004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc2_apps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sdcc2_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sys_noc_usb2_prim_axi_clk = {
+	.halt_reg = 0x10a14,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x10a14,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x10a14,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sys_noc_usb2_prim_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb20_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sys_noc_usb3_prim_axi_clk = {
+	.halt_reg = 0x1a078,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x1a078,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x1a078,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sys_noc_usb3_prim_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb30_prim_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_tscss_ahb_clk = {
+	.halt_reg = 0xac024,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xac024,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0xac024,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_tscss_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_tscss_cntr_clk = {
+	.halt_reg = 0xac020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xac020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_tscss_cntr_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_tscss_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_tscss_etu_clk = {
+	.halt_reg = 0xac01c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xac01c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_tscss_etu_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_tscss_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_clkref_en = {
+	.halt_reg = 0x8c000,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x8c000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_clkref_en",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb20_master_clk = {
+	.halt_reg = 0xb0010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0xb0010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0xb0010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb20_master_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb20_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb20_mock_utmi_clk = {
+	.halt_reg = 0xb001c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0xb001c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb20_mock_utmi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb20_mock_utmi_postdiv_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb20_sleep_clk = {
+	.halt_reg = 0xb0018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0xb0018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb20_sleep_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_prim_master_clk = {
+	.halt_reg = 0x1a010,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x1a010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x1a010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb30_prim_master_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb30_prim_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_prim_mock_utmi_clk = {
+	.halt_reg = 0x1a018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1a018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb30_prim_mock_utmi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_prim_sleep_clk = {
+	.halt_reg = 0x1a014,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1a014,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb30_prim_sleep_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_prim_clkref_en = {
+	.halt_reg = 0x9f000,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x9f000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb3_prim_clkref_en",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_prim_phy_com_aux_clk = {
+	.halt_reg = 0x1a054,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1a054,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb3_prim_phy_com_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb3_prim_phy_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_prim_phy_pipe_clk = {
+	.halt_reg = 0x1a058,
+	.halt_check = BRANCH_HALT_DELAY,
+	.hwcg_reg = 0x1a058,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x1a058,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb3_prim_phy_pipe_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb3_prim_phy_pipe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_vcodec0_axi_clk = {
+	.halt_reg = 0x6e008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x6e008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_vcodec0_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_venus_ahb_clk = {
+	.halt_reg = 0x6e010,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x6e010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_venus_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_venus_ctl_axi_clk = {
+	.halt_reg = 0x6e004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x6e004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_venus_ctl_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_video_axi0_clk = {
+	.halt_reg = 0x1701c,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x1701c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x1701c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_video_axi0_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_video_throttle_core_clk = {
+	.halt_reg = 0x17068,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x17068,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x79004,
+		.enable_mask = BIT(14),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_video_throttle_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_video_vcodec0_sys_clk = {
+	.halt_reg = 0x6d044,
+	.halt_check = BRANCH_HALT_DELAY,
+	.hwcg_reg = 0x6d044,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x6d044,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_video_vcodec0_sys_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_video_venus_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_video_venus_ctl_clk = {
+	.halt_reg = 0x6d02c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x6d02c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_video_venus_ctl_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_video_venus_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc gcc_camss_top_gdsc = {
+	.gdscr = 0x58004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gcc_camss_top_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_emac0_gdsc = {
+	.gdscr = 0xad004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x2,
+	.pd = {
+		.name = "gcc_emac0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_emac1_gdsc = {
+	.gdscr = 0xae004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x2,
+	.pd = {
+		.name = "gcc_emac1_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_pcie_gdsc = {
+	.gdscr = 0xaf004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gcc_pcie_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_usb20_gdsc = {
+	.gdscr = 0xb0004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gcc_usb20_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_usb30_prim_gdsc = {
+	.gdscr = 0x1a004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x2,
+	.pd = {
+		.name = "gcc_usb30_prim_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_venus_gdsc = {
+	.gdscr = 0x6d01c,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gcc_venus_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_vcodec0_gdsc = {
+	.gdscr = 0x6d038,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gcc_vcodec0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.parent = &gcc_venus_gdsc.pd,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct clk_regmap *gcc_shikra_clocks[] = {
+	[GCC_AHB2PHY_CSI_CLK] = &gcc_ahb2phy_csi_clk.clkr,
+	[GCC_AHB2PHY_USB_CLK] = &gcc_ahb2phy_usb_clk.clkr,
+	[GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr,
+	[GCC_CAM_THROTTLE_NRT_CLK] = &gcc_cam_throttle_nrt_clk.clkr,
+	[GCC_CAM_THROTTLE_RT_CLK] = &gcc_cam_throttle_rt_clk.clkr,
+	[GCC_CAMSS_AXI_CLK] = &gcc_camss_axi_clk.clkr,
+	[GCC_CAMSS_AXI_CLK_SRC] = &gcc_camss_axi_clk_src.clkr,
+	[GCC_CAMSS_CAMNOC_ATB_CLK] = &gcc_camss_camnoc_atb_clk.clkr,
+	[GCC_CAMSS_CAMNOC_DRAGONLINK_ATB_CLK] = &gcc_camss_camnoc_dragonlink_atb_clk.clkr,
+	[GCC_CAMSS_CAMNOC_NTS_XO_CLK] = &gcc_camss_camnoc_nts_xo_clk.clkr,
+	[GCC_CAMSS_CCI_0_CLK] = &gcc_camss_cci_0_clk.clkr,
+	[GCC_CAMSS_CCI_CLK_SRC] = &gcc_camss_cci_clk_src.clkr,
+	[GCC_CAMSS_CPHY_0_CLK] = &gcc_camss_cphy_0_clk.clkr,
+	[GCC_CAMSS_CPHY_1_CLK] = &gcc_camss_cphy_1_clk.clkr,
+	[GCC_CAMSS_CSI0PHYTIMER_CLK] = &gcc_camss_csi0phytimer_clk.clkr,
+	[GCC_CAMSS_CSI0PHYTIMER_CLK_SRC] = &gcc_camss_csi0phytimer_clk_src.clkr,
+	[GCC_CAMSS_CSI1PHYTIMER_CLK] = &gcc_camss_csi1phytimer_clk.clkr,
+	[GCC_CAMSS_CSI1PHYTIMER_CLK_SRC] = &gcc_camss_csi1phytimer_clk_src.clkr,
+	[GCC_CAMSS_MCLK0_CLK] = &gcc_camss_mclk0_clk.clkr,
+	[GCC_CAMSS_MCLK0_CLK_SRC] = &gcc_camss_mclk0_clk_src.clkr,
+	[GCC_CAMSS_MCLK1_CLK] = &gcc_camss_mclk1_clk.clkr,
+	[GCC_CAMSS_MCLK1_CLK_SRC] = &gcc_camss_mclk1_clk_src.clkr,
+	[GCC_CAMSS_MCLK2_CLK] = &gcc_camss_mclk2_clk.clkr,
+	[GCC_CAMSS_MCLK2_CLK_SRC] = &gcc_camss_mclk2_clk_src.clkr,
+	[GCC_CAMSS_MCLK3_CLK] = &gcc_camss_mclk3_clk.clkr,
+	[GCC_CAMSS_MCLK3_CLK_SRC] = &gcc_camss_mclk3_clk_src.clkr,
+	[GCC_CAMSS_NRT_AXI_CLK] = &gcc_camss_nrt_axi_clk.clkr,
+	[GCC_CAMSS_OPE_AHB_CLK] = &gcc_camss_ope_ahb_clk.clkr,
+	[GCC_CAMSS_OPE_AHB_CLK_SRC] = &gcc_camss_ope_ahb_clk_src.clkr,
+	[GCC_CAMSS_OPE_CLK] = &gcc_camss_ope_clk.clkr,
+	[GCC_CAMSS_OPE_CLK_SRC] = &gcc_camss_ope_clk_src.clkr,
+	[GCC_CAMSS_RT_AXI_CLK] = &gcc_camss_rt_axi_clk.clkr,
+	[GCC_CAMSS_TFE_0_CLK] = &gcc_camss_tfe_0_clk.clkr,
+	[GCC_CAMSS_TFE_0_CLK_SRC] = &gcc_camss_tfe_0_clk_src.clkr,
+	[GCC_CAMSS_TFE_0_CPHY_RX_CLK] = &gcc_camss_tfe_0_cphy_rx_clk.clkr,
+	[GCC_CAMSS_TFE_0_CSID_CLK] = &gcc_camss_tfe_0_csid_clk.clkr,
+	[GCC_CAMSS_TFE_0_CSID_CLK_SRC] = &gcc_camss_tfe_0_csid_clk_src.clkr,
+	[GCC_CAMSS_TFE_1_CLK] = &gcc_camss_tfe_1_clk.clkr,
+	[GCC_CAMSS_TFE_1_CLK_SRC] = &gcc_camss_tfe_1_clk_src.clkr,
+	[GCC_CAMSS_TFE_1_CPHY_RX_CLK] = &gcc_camss_tfe_1_cphy_rx_clk.clkr,
+	[GCC_CAMSS_TFE_1_CSID_CLK] = &gcc_camss_tfe_1_csid_clk.clkr,
+	[GCC_CAMSS_TFE_1_CSID_CLK_SRC] = &gcc_camss_tfe_1_csid_clk_src.clkr,
+	[GCC_CAMSS_TFE_CPHY_RX_CLK_SRC] = &gcc_camss_tfe_cphy_rx_clk_src.clkr,
+	[GCC_CAMSS_TOP_AHB_CLK] = &gcc_camss_top_ahb_clk.clkr,
+	[GCC_CAMSS_TOP_AHB_CLK_SRC] = &gcc_camss_top_ahb_clk_src.clkr,
+	[GCC_CFG_NOC_USB2_PRIM_AXI_CLK] = &gcc_cfg_noc_usb2_prim_axi_clk.clkr,
+	[GCC_CFG_NOC_USB3_PRIM_AXI_CLK] = &gcc_cfg_noc_usb3_prim_axi_clk.clkr,
+	[GCC_DDRSS_GPU_AXI_CLK] = &gcc_ddrss_gpu_axi_clk.clkr,
+	[GCC_DDRSS_MEMNOC_PCIE_SF_CLK] = &gcc_ddrss_memnoc_pcie_sf_clk.clkr,
+	[GCC_DISP_GPLL0_CLK_SRC] = &gcc_disp_gpll0_clk_src.clkr,
+	[GCC_DISP_GPLL0_DIV_CLK_SRC] = &gcc_disp_gpll0_div_clk_src.clkr,
+	[GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr,
+	[GCC_DISP_THROTTLE_CORE_CLK] = &gcc_disp_throttle_core_clk.clkr,
+	[GCC_EMAC0_AHB_CLK] = &gcc_emac0_ahb_clk.clkr,
+	[GCC_EMAC0_AXI_CLK] = &gcc_emac0_axi_clk.clkr,
+	[GCC_EMAC0_AXI_CLK_SRC] = &gcc_emac0_axi_clk_src.clkr,
+	[GCC_EMAC0_AXI_SYS_NOC_CLK] = &gcc_emac0_axi_sys_noc_clk.clkr,
+	[GCC_EMAC0_CC_SGMIIPHY_RX_CLK] = &gcc_emac0_cc_sgmiiphy_rx_clk.clkr,
+	[GCC_EMAC0_CC_SGMIIPHY_RX_CLK_SRC] = &gcc_emac0_cc_sgmiiphy_rx_clk_src.clkr,
+	[GCC_EMAC0_CC_SGMIIPHY_TX_CLK] = &gcc_emac0_cc_sgmiiphy_tx_clk.clkr,
+	[GCC_EMAC0_CC_SGMIIPHY_TX_CLK_SRC] = &gcc_emac0_cc_sgmiiphy_tx_clk_src.clkr,
+	[GCC_EMAC0_PHY_AUX_CLK] = &gcc_emac0_phy_aux_clk.clkr,
+	[GCC_EMAC0_PHY_AUX_CLK_SRC] = &gcc_emac0_phy_aux_clk_src.clkr,
+	[GCC_EMAC0_PTP_CLK] = &gcc_emac0_ptp_clk.clkr,
+	[GCC_EMAC0_PTP_CLK_SRC] = &gcc_emac0_ptp_clk_src.clkr,
+	[GCC_EMAC0_RGMII_CLK] = &gcc_emac0_rgmii_clk.clkr,
+	[GCC_EMAC0_RGMII_CLK_SRC] = &gcc_emac0_rgmii_clk_src.clkr,
+	[GCC_EMAC1_AHB_CLK] = &gcc_emac1_ahb_clk.clkr,
+	[GCC_EMAC1_AXI_CLK] = &gcc_emac1_axi_clk.clkr,
+	[GCC_EMAC1_AXI_CLK_SRC] = &gcc_emac1_axi_clk_src.clkr,
+	[GCC_EMAC1_AXI_SYS_NOC_CLK] = &gcc_emac1_axi_sys_noc_clk.clkr,
+	[GCC_EMAC1_CC_SGMIIPHY_RX_CLK] = &gcc_emac1_cc_sgmiiphy_rx_clk.clkr,
+	[GCC_EMAC1_CC_SGMIIPHY_RX_CLK_SRC] = &gcc_emac1_cc_sgmiiphy_rx_clk_src.clkr,
+	[GCC_EMAC1_CC_SGMIIPHY_TX_CLK] = &gcc_emac1_cc_sgmiiphy_tx_clk.clkr,
+	[GCC_EMAC1_CC_SGMIIPHY_TX_CLK_SRC] = &gcc_emac1_cc_sgmiiphy_tx_clk_src.clkr,
+	[GCC_EMAC1_PHY_AUX_CLK] = &gcc_emac1_phy_aux_clk.clkr,
+	[GCC_EMAC1_PHY_AUX_CLK_SRC] = &gcc_emac1_phy_aux_clk_src.clkr,
+	[GCC_EMAC1_PTP_CLK] = &gcc_emac1_ptp_clk.clkr,
+	[GCC_EMAC1_PTP_CLK_SRC] = &gcc_emac1_ptp_clk_src.clkr,
+	[GCC_EMAC1_RGMII_CLK] = &gcc_emac1_rgmii_clk.clkr,
+	[GCC_EMAC1_RGMII_CLK_SRC] = &gcc_emac1_rgmii_clk_src.clkr,
+	[GCC_GP1_CLK] = &gcc_gp1_clk.clkr,
+	[GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr,
+	[GCC_GP2_CLK] = &gcc_gp2_clk.clkr,
+	[GCC_GP2_CLK_SRC] = &gcc_gp2_clk_src.clkr,
+	[GCC_GP3_CLK] = &gcc_gp3_clk.clkr,
+	[GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr,
+	[GCC_GPU_GPLL0_CLK_SRC] = &gcc_gpu_gpll0_clk_src.clkr,
+	[GCC_GPU_GPLL0_DIV_CLK_SRC] = &gcc_gpu_gpll0_div_clk_src.clkr,
+	[GCC_GPU_MEMNOC_GFX_CLK] = &gcc_gpu_memnoc_gfx_clk.clkr,
+	[GCC_GPU_SMMU_VOTE_CLK] = &gcc_gpu_smmu_vote_clk.clkr,
+	[GCC_GPU_SNOC_DVM_GFX_CLK] = &gcc_gpu_snoc_dvm_gfx_clk.clkr,
+	[GCC_GPU_THROTTLE_CORE_CLK] = &gcc_gpu_throttle_core_clk.clkr,
+	[GCC_MMU_TCU_VOTE_CLK] = &gcc_mmu_tcu_vote_clk.clkr,
+	[GCC_PCIE_AUX_CLK] = &gcc_pcie_aux_clk.clkr,
+	[GCC_PCIE_AUX_CLK_SRC] = &gcc_pcie_aux_clk_src.clkr,
+	[GCC_PCIE_AUX_PHY_CLK_SRC] = &gcc_pcie_aux_phy_clk_src.clkr,
+	[GCC_PCIE_CFG_AHB_CLK] = &gcc_pcie_cfg_ahb_clk.clkr,
+	[GCC_PCIE_CLKREF_EN] = &gcc_pcie_clkref_en.clkr,
+	[GCC_PCIE_MSTR_AXI_CLK] = &gcc_pcie_mstr_axi_clk.clkr,
+	[GCC_PCIE_PIPE_CLK] = &gcc_pcie_pipe_clk.clkr,
+	[GCC_PCIE_PIPE_CLK_SRC] = &gcc_pcie_pipe_clk_src.clkr,
+	[GCC_PCIE_RCHNG_PHY_CLK] = &gcc_pcie_rchng_phy_clk.clkr,
+	[GCC_PCIE_RCHNG_PHY_CLK_SRC] = &gcc_pcie_rchng_phy_clk_src.clkr,
+	[GCC_PCIE_SLEEP_CLK] = &gcc_pcie_sleep_clk.clkr,
+	[GCC_PCIE_SLV_AXI_CLK] = &gcc_pcie_slv_axi_clk.clkr,
+	[GCC_PCIE_SLV_Q2A_AXI_CLK] = &gcc_pcie_slv_q2a_axi_clk.clkr,
+	[GCC_PCIE_TBU_CLK] = &gcc_pcie_tbu_clk.clkr,
+	[GCC_PCIE_THROTTLE_CORE_CLK] = &gcc_pcie_throttle_core_clk.clkr,
+	[GCC_PCIE_THROTTLE_XO_CLK] = &gcc_pcie_throttle_xo_clk.clkr,
+	[GCC_PCIE_TILE_AXI_SYS_NOC_CLK] = &gcc_pcie_tile_axi_sys_noc_clk.clkr,
+	[GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr,
+	[GCC_PDM2_CLK_SRC] = &gcc_pdm2_clk_src.clkr,
+	[GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr,
+	[GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr,
+	[GCC_PWM0_XO512_CLK] = &gcc_pwm0_xo512_clk.clkr,
+	[GCC_QMIP_CAMERA_NRT_AHB_CLK] = &gcc_qmip_camera_nrt_ahb_clk.clkr,
+	[GCC_QMIP_CAMERA_RT_AHB_CLK] = &gcc_qmip_camera_rt_ahb_clk.clkr,
+	[GCC_QMIP_DISP_AHB_CLK] = &gcc_qmip_disp_ahb_clk.clkr,
+	[GCC_QMIP_GPU_CFG_AHB_CLK] = &gcc_qmip_gpu_cfg_ahb_clk.clkr,
+	[GCC_QMIP_PCIE_CFG_AHB_CLK] = &gcc_qmip_pcie_cfg_ahb_clk.clkr,
+	[GCC_QMIP_VIDEO_VCODEC_AHB_CLK] = &gcc_qmip_video_vcodec_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP0_CORE_2X_CLK] = &gcc_qupv3_wrap0_core_2x_clk.clkr,
+	[GCC_QUPV3_WRAP0_CORE_CLK] = &gcc_qupv3_wrap0_core_clk.clkr,
+	[GCC_QUPV3_WRAP0_S0_CLK] = &gcc_qupv3_wrap0_s0_clk.clkr,
+	[GCC_QUPV3_WRAP0_S0_CLK_SRC] = &gcc_qupv3_wrap0_s0_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S1_CLK] = &gcc_qupv3_wrap0_s1_clk.clkr,
+	[GCC_QUPV3_WRAP0_S1_CLK_SRC] = &gcc_qupv3_wrap0_s1_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S2_CLK] = &gcc_qupv3_wrap0_s2_clk.clkr,
+	[GCC_QUPV3_WRAP0_S2_CLK_SRC] = &gcc_qupv3_wrap0_s2_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S3_CLK] = &gcc_qupv3_wrap0_s3_clk.clkr,
+	[GCC_QUPV3_WRAP0_S3_CLK_SRC] = &gcc_qupv3_wrap0_s3_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S4_CLK] = &gcc_qupv3_wrap0_s4_clk.clkr,
+	[GCC_QUPV3_WRAP0_S4_CLK_SRC] = &gcc_qupv3_wrap0_s4_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S5_CLK] = &gcc_qupv3_wrap0_s5_clk.clkr,
+	[GCC_QUPV3_WRAP0_S5_CLK_SRC] = &gcc_qupv3_wrap0_s5_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S6_CLK] = &gcc_qupv3_wrap0_s6_clk.clkr,
+	[GCC_QUPV3_WRAP0_S6_CLK_SRC] = &gcc_qupv3_wrap0_s6_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S7_CLK] = &gcc_qupv3_wrap0_s7_clk.clkr,
+	[GCC_QUPV3_WRAP0_S7_CLK_SRC] = &gcc_qupv3_wrap0_s7_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S8_CLK] = &gcc_qupv3_wrap0_s8_clk.clkr,
+	[GCC_QUPV3_WRAP0_S8_CLK_SRC] = &gcc_qupv3_wrap0_s8_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S9_CLK] = &gcc_qupv3_wrap0_s9_clk.clkr,
+	[GCC_QUPV3_WRAP0_S9_CLK_SRC] = &gcc_qupv3_wrap0_s9_clk_src.clkr,
+	[GCC_QUPV3_WRAP_0_M_AHB_CLK] = &gcc_qupv3_wrap_0_m_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_0_S_AHB_CLK] = &gcc_qupv3_wrap_0_s_ahb_clk.clkr,
+	[GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr,
+	[GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr,
+	[GCC_SDCC1_APPS_CLK_SRC] = &gcc_sdcc1_apps_clk_src.clkr,
+	[GCC_SDCC1_ICE_CORE_CLK] = &gcc_sdcc1_ice_core_clk.clkr,
+	[GCC_SDCC1_ICE_CORE_CLK_SRC] = &gcc_sdcc1_ice_core_clk_src.clkr,
+	[GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr,
+	[GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr,
+	[GCC_SDCC2_APPS_CLK_SRC] = &gcc_sdcc2_apps_clk_src.clkr,
+	[GCC_SYS_NOC_USB2_PRIM_AXI_CLK] = &gcc_sys_noc_usb2_prim_axi_clk.clkr,
+	[GCC_SYS_NOC_USB3_PRIM_AXI_CLK] = &gcc_sys_noc_usb3_prim_axi_clk.clkr,
+	[GCC_TSCSS_AHB_CLK] = &gcc_tscss_ahb_clk.clkr,
+	[GCC_TSCSS_CLK_SRC] = &gcc_tscss_clk_src.clkr,
+	[GCC_TSCSS_CNTR_CLK] = &gcc_tscss_cntr_clk.clkr,
+	[GCC_TSCSS_ETU_CLK] = &gcc_tscss_etu_clk.clkr,
+	[GCC_UFS_CLKREF_EN] = &gcc_ufs_clkref_en.clkr,
+	[GCC_USB20_MASTER_CLK] = &gcc_usb20_master_clk.clkr,
+	[GCC_USB20_MASTER_CLK_SRC] = &gcc_usb20_master_clk_src.clkr,
+	[GCC_USB20_MOCK_UTMI_CLK] = &gcc_usb20_mock_utmi_clk.clkr,
+	[GCC_USB20_MOCK_UTMI_CLK_SRC] = &gcc_usb20_mock_utmi_clk_src.clkr,
+	[GCC_USB20_MOCK_UTMI_POSTDIV_CLK_SRC] = &gcc_usb20_mock_utmi_postdiv_clk_src.clkr,
+	[GCC_USB20_SLEEP_CLK] = &gcc_usb20_sleep_clk.clkr,
+	[GCC_USB30_PRIM_MASTER_CLK] = &gcc_usb30_prim_master_clk.clkr,
+	[GCC_USB30_PRIM_MASTER_CLK_SRC] = &gcc_usb30_prim_master_clk_src.clkr,
+	[GCC_USB30_PRIM_MOCK_UTMI_CLK] = &gcc_usb30_prim_mock_utmi_clk.clkr,
+	[GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC] = &gcc_usb30_prim_mock_utmi_clk_src.clkr,
+	[GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC] = &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr,
+	[GCC_USB30_PRIM_SLEEP_CLK] = &gcc_usb30_prim_sleep_clk.clkr,
+	[GCC_USB3_PRIM_CLKREF_EN] = &gcc_usb3_prim_clkref_en.clkr,
+	[GCC_USB3_PRIM_PHY_AUX_CLK_SRC] = &gcc_usb3_prim_phy_aux_clk_src.clkr,
+	[GCC_USB3_PRIM_PHY_COM_AUX_CLK] = &gcc_usb3_prim_phy_com_aux_clk.clkr,
+	[GCC_USB3_PRIM_PHY_PIPE_CLK] = &gcc_usb3_prim_phy_pipe_clk.clkr,
+	[GCC_USB3_PRIM_PHY_PIPE_CLK_SRC] = &gcc_usb3_prim_phy_pipe_clk_src.clkr,
+	[GCC_VCODEC0_AXI_CLK] = &gcc_vcodec0_axi_clk.clkr,
+	[GCC_VENUS_AHB_CLK] = &gcc_venus_ahb_clk.clkr,
+	[GCC_VENUS_CTL_AXI_CLK] = &gcc_venus_ctl_axi_clk.clkr,
+	[GCC_VIDEO_AXI0_CLK] = &gcc_video_axi0_clk.clkr,
+	[GCC_VIDEO_THROTTLE_CORE_CLK] = &gcc_video_throttle_core_clk.clkr,
+	[GCC_VIDEO_VCODEC0_SYS_CLK] = &gcc_video_vcodec0_sys_clk.clkr,
+	[GCC_VIDEO_VENUS_CLK_SRC] = &gcc_video_venus_clk_src.clkr,
+	[GCC_VIDEO_VENUS_CTL_CLK] = &gcc_video_venus_ctl_clk.clkr,
+	[GPLL0] = &gpll0.clkr,
+	[GPLL0_OUT_AUX2] = &gpll0_out_aux2.clkr,
+	[GPLL10] = &gpll10.clkr,
+	[GPLL11] = &gpll11.clkr,
+	[GPLL12] = &gpll12.clkr,
+	[GPLL12_OUT_AUX2] = &gpll12_out_aux2.clkr,
+	[GPLL3] = &gpll3.clkr,
+	[GPLL3_OUT_MAIN] = &gpll3_out_main.clkr,
+	[GPLL4] = &gpll4.clkr,
+	[GPLL5] = &gpll5.clkr,
+	[GPLL6] = &gpll6.clkr,
+	[GPLL6_OUT_MAIN] = &gpll6_out_main.clkr,
+	[GPLL7] = &gpll7.clkr,
+	[GPLL8] = &gpll8.clkr,
+	[GPLL8_OUT_MAIN] = &gpll8_out_main.clkr,
+	[GPLL9] = &gpll9.clkr,
+	[GPLL9_OUT_MAIN] = &gpll9_out_main.clkr,
+};
+
+static struct gdsc *gcc_shikra_gdscs[] = {
+	[GCC_CAMSS_TOP_GDSC] = &gcc_camss_top_gdsc,
+	[GCC_EMAC0_GDSC] = &gcc_emac0_gdsc,
+	[GCC_EMAC1_GDSC] = &gcc_emac1_gdsc,
+	[GCC_PCIE_GDSC] = &gcc_pcie_gdsc,
+	[GCC_USB20_GDSC] = &gcc_usb20_gdsc,
+	[GCC_USB30_PRIM_GDSC] = &gcc_usb30_prim_gdsc,
+	[GCC_VCODEC0_GDSC] = &gcc_vcodec0_gdsc,
+	[GCC_VENUS_GDSC] = &gcc_venus_gdsc,
+};
+
+static const struct qcom_reset_map gcc_shikra_resets[] = {
+	[GCC_CAMSS_OPE_BCR] = { 0x55000 },
+	[GCC_CAMSS_TFE_BCR] = { 0x52000 },
+	[GCC_CAMSS_TOP_BCR] = { 0x58000 },
+	[GCC_EMAC0_BCR] = { 0xad000 },
+	[GCC_EMAC1_BCR] = { 0xae000 },
+	[GCC_GPU_BCR] = { 0x36000 },
+	[GCC_MMSS_BCR] = { 0x17000 },
+	[GCC_PCIE_BCR] = { 0xaf000 },
+	[GCC_PCIE_PHY_BCR] = { 0xb1000 },
+	[GCC_PDM_BCR] = { 0x20000 },
+	[GCC_QUPV3_WRAPPER_0_BCR] = { 0x1f000 },
+	[GCC_QUSB2PHY_PRIM_BCR] = { 0x1c000 },
+	[GCC_QUSB2PHY_SEC_BCR] = { 0x1c004 },
+	[GCC_SDCC1_BCR] = { 0x38000 },
+	[GCC_SDCC2_BCR] = { 0x1e000 },
+	[GCC_TSCSS_BCR] = { 0xac000 },
+	[GCC_USB20_BCR] = { 0xb0000 },
+	[GCC_USB30_PRIM_BCR] = { 0x1a000 },
+	[GCC_USB3PHY_PHY_PRIM_SP0_BCR] = { 0x1b008 },
+	[GCC_USB3_DP_PHY_PRIM_BCR] = { 0x1b020 },
+	[GCC_USB3_PHY_PRIM_SP0_BCR] = { 0x1b000 },
+	[GCC_USB_PHY_CFG_AHB2PHY_BCR] = { 0x1d000 },
+	[GCC_VCODEC0_BCR] = { 0x6d034 },
+	[GCC_VENUS_BCR] = { 0x6d018 },
+	[GCC_VIDEO_INTERFACE_BCR] = { 0x6e000 },
+};
+
+static struct clk_alpha_pll *gcc_shikra_plls[] = {
+	&gpll10,
+	&gpll11,
+	&gpll8,
+	&gpll9,
+};
+
+static const u32 gcc_shikra_critical_cbcrs[] = {
+	0x17008, /* GCC_CAMERA_AHB_CLK */
+	0x17028, /* GCC_CAMERA_XO_CLK */
+	0x1700c, /* GCC_DISP_AHB_CLK */
+	0x1702c, /* GCC_DISP_XO_CLK */
+	0x36004, /* GCC_GPU_CFG_AHB_CLK */
+	0x36100, /* GCC_GPU_IREF_CLK */
+	0x3a00c, /* GCC_LPASS_CONFIG_CLK */
+	0x3a008, /* GCC_LPASS_CORE_AXIM_CLK */
+	0x79004, /* GCC_SYS_NOC_CPUSS_AHB_CLK */
+	0x17004, /* GCC_VIDEO_AHB_CLK */
+	0x17024, /* GCC_VIDEO_XO_CLK */
+};
+
+static const struct clk_rcg_dfs_data gcc_shikra_dfs_clocks[] = {
+	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s0_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s1_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s2_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s3_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s4_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s5_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s6_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s7_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s8_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s9_clk_src),
+};
+
+static const struct regmap_config gcc_shikra_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xc7000,
+	.fast_io = true,
+};
+
+static const struct qcom_cc_driver_data gcc_shikra_driver_data = {
+	.alpha_plls = gcc_shikra_plls,
+	.num_alpha_plls = ARRAY_SIZE(gcc_shikra_plls),
+	.clk_cbcrs = gcc_shikra_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_shikra_critical_cbcrs),
+	.dfs_rcgs = gcc_shikra_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_shikra_dfs_clocks),
+};
+
+static const struct qcom_cc_desc gcc_shikra_desc = {
+	.config = &gcc_shikra_regmap_config,
+	.clks = gcc_shikra_clocks,
+	.num_clks = ARRAY_SIZE(gcc_shikra_clocks),
+	.resets = gcc_shikra_resets,
+	.num_resets = ARRAY_SIZE(gcc_shikra_resets),
+	.gdscs = gcc_shikra_gdscs,
+	.num_gdscs = ARRAY_SIZE(gcc_shikra_gdscs),
+	.driver_data = &gcc_shikra_driver_data,
+};
+
+static const struct of_device_id gcc_shikra_match_table[] = {
+	{ .compatible = "qcom,shikra-gcc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, gcc_shikra_match_table);
+
+static int gcc_shikra_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &gcc_shikra_desc);
+}
+
+static struct platform_driver gcc_shikra_driver = {
+	.probe = gcc_shikra_probe,
+	.driver = {
+		.name = "gcc-shikra",
+		.of_match_table = gcc_shikra_match_table,
+	},
+};
+
+static int __init gcc_shikra_init(void)
+{
+	return platform_driver_register(&gcc_shikra_driver);
+}
+subsys_initcall(gcc_shikra_init);
+
+static void __exit gcc_shikra_exit(void)
+{
+	platform_driver_unregister(&gcc_shikra_driver);
+}
+module_exit(gcc_shikra_exit);
+
+MODULE_DESCRIPTION("QTI GCC Shikra Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-sm6115.c b/drivers/clk/qcom/gcc-sm6115.c
index 4c3804701e243..c5251aff98864 100644
--- a/drivers/clk/qcom/gcc-sm6115.c
+++ b/drivers/clk/qcom/gcc-sm6115.c
@@ -3218,6 +3218,7 @@ static struct gdsc gcc_vcodec0_gdsc = {
 	.pd = {
 		.name = "gcc_vcodec0",
 	},
+	.flags = HW_CTRL_TRIGGER,
 	.pwrsts = PWRSTS_OFF_ON,
 };
 
diff --git a/drivers/clk/qcom/gdsc.c b/drivers/clk/qcom/gdsc.c
index ee5f86ca50cb7..b99d70149af34 100644
--- a/drivers/clk/qcom/gdsc.c
+++ b/drivers/clk/qcom/gdsc.c
@@ -104,14 +104,21 @@ static int gdsc_hwctrl(struct gdsc *sc, bool en)
 static int gdsc_poll_status(struct gdsc *sc, enum gdsc_status status)
 {
 	ktime_t start;
+	int ret;
 
 	start = ktime_get();
 	do {
-		if (gdsc_check_status(sc, status))
+		ret = gdsc_check_status(sc, status);
+		if (ret < 0)
+			return ret;
+		if (ret)
 			return 0;
 	} while (ktime_us_delta(ktime_get(), start) < STATUS_POLL_TIMEOUT_US);
 
-	if (gdsc_check_status(sc, status))
+	ret = gdsc_check_status(sc, status);
+	if (ret < 0)
+		return ret;
+	if (ret)
 		return 0;
 
 	return -ETIMEDOUT;
@@ -493,7 +500,9 @@ static int gdsc_init(struct gdsc *sc)
 
 	} else if (sc->flags & ALWAYS_ON) {
 		/* If ALWAYS_ON GDSCs are not ON, turn them ON */
-		gdsc_enable(&sc->pd);
+		ret = gdsc_enable(&sc->pd);
+		if (ret)
+			return ret;
 		on = true;
 	}
 
@@ -669,10 +678,18 @@ int gdsc_register(struct gdsc_desc *desc,
 void gdsc_unregister(struct gdsc_desc *desc)
 {
 	struct device *dev = desc->dev;
+	struct gdsc **scs = desc->scs;
 	size_t num = desc->num;
+	int i;
 
-	gdsc_pm_subdomain_remove(desc, num);
 	of_genpd_del_provider(dev->of_node);
+	gdsc_pm_subdomain_remove(desc, num);
+
+	for (i = 0; i < num; i++) {
+		if (!scs[i])
+			continue;
+		pm_genpd_remove(&scs[i]->pd);
+	}
 }
 
 /*
@@ -708,3 +725,25 @@ int gdsc_gx_do_nothing_enable(struct generic_pm_domain *domain)
 	return ret;
 }
 EXPORT_SYMBOL_GPL(gdsc_gx_do_nothing_enable);
+
+/*
+ * GX GDSC is a special power domain. Normally, its disable sequence
+ * is managed by the GMU firmware, and high level OS must not attempt
+ * to disable it. The only exception is during GMU recovery, where the
+ * GMU driver can set GenPD’s synced_poweroff flag to allow explicitly
+ * disable GX GDSC in hardware.
+ */
+int gdsc_gx_disable(struct generic_pm_domain *domain)
+{
+	struct gdsc *sc = domain_to_gdsc(domain);
+
+	if (domain->synced_poweroff)
+		return gdsc_disable(domain);
+
+	/* Remove parent-supply placed in enable */
+	if (sc->rsupply)
+		return regulator_disable(sc->rsupply);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(gdsc_gx_disable);
diff --git a/drivers/clk/qcom/gdsc.h b/drivers/clk/qcom/gdsc.h
index 92ff6bcce7b1c..2f9665b664e60 100644
--- a/drivers/clk/qcom/gdsc.h
+++ b/drivers/clk/qcom/gdsc.h
@@ -93,6 +93,7 @@ int gdsc_register(struct gdsc_desc *desc, struct reset_controller_dev *,
 		  struct regmap *);
 void gdsc_unregister(struct gdsc_desc *desc);
 int gdsc_gx_do_nothing_enable(struct generic_pm_domain *domain);
+int gdsc_gx_disable(struct generic_pm_domain *domain);
 #else
 static inline int gdsc_register(struct gdsc_desc *desc,
 				struct reset_controller_dev *rcdev,
diff --git a/drivers/clk/qcom/gxclkctl-kaanapali.c b/drivers/clk/qcom/gxclkctl-kaanapali.c
index 10c1a8976c56c..8b71dc66ebbd6 100644
--- a/drivers/clk/qcom/gxclkctl-kaanapali.c
+++ b/drivers/clk/qcom/gxclkctl-kaanapali.c
@@ -6,6 +6,7 @@
 #include <linux/clk-provider.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 
 #include <dt-bindings/clock/qcom,kaanapali-gxclkctl.h>
@@ -25,6 +26,7 @@ static struct gdsc gx_clkctl_gx_gdsc = {
 	.pd = {
 		.name = "gx_clkctl_gx_gdsc",
 		.power_on = gdsc_gx_do_nothing_enable,
+		.power_off = gdsc_gx_disable,
 	},
 	.pwrsts = PWRSTS_OFF_ON,
 	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
@@ -60,7 +62,15 @@ MODULE_DEVICE_TABLE(of, gx_clkctl_kaanapali_match_table);
 
 static int gx_clkctl_kaanapali_probe(struct platform_device *pdev)
 {
-	return qcom_cc_probe(pdev, &gx_clkctl_kaanapali_desc);
+	int ret;
+
+	ret = qcom_cc_probe(pdev, &gx_clkctl_kaanapali_desc);
+	if (ret)
+		return ret;
+
+	pm_runtime_disable(&pdev->dev);
+
+	return ret;
 }
 
 static struct platform_driver gx_clkctl_kaanapali_driver = {
diff --git a/drivers/clk/qcom/videocc-hawi.c b/drivers/clk/qcom/videocc-hawi.c
new file mode 100644
index 0000000000000..ac3ff7202fa0b
--- /dev/null
+++ b/drivers/clk/qcom/videocc-hawi.c
@@ -0,0 +1,840 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,hawi-videocc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "common.h"
+#include "gdsc.h"
+#include "reset.h"
+
+enum {
+	DT_BI_TCXO,
+	DT_AHB_CLK,
+};
+
+enum {
+	P_BI_TCXO,
+	P_VIDEO_CC_PLL0_OUT_EVEN,
+	P_VIDEO_CC_PLL0_OUT_MAIN,
+	P_VIDEO_CC_PLL1_OUT_MAIN,
+	P_VIDEO_CC_PLL2_OUT_MAIN,
+	P_VIDEO_CC_PLL3_OUT_MAIN,
+};
+
+static const struct pll_vco taycan_eha_t_vco[] = {
+	{ 249600000, 2500000000, 0 },
+};
+
+/* 360.0 MHz Configuration */
+static const struct alpha_pll_config video_cc_pll0_config = {
+	.l = 0x12,
+	.cal_l = 0x48,
+	.alpha = 0xc000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll video_cc_pll0 = {
+	.offset = 0x0,
+	.config = &video_cc_pll0_config,
+	.vco_table = taycan_eha_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eha_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_pll0",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eha_t_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_video_cc_pll0_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv video_cc_pll0_out_even = {
+	.offset = 0x0,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_video_cc_pll0_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_video_cc_pll0_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_pll0_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&video_cc_pll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eha_t_ops,
+	},
+};
+
+/* 300.0 MHz Configuration */
+static const struct alpha_pll_config video_cc_pll1_config = {
+	.l = 0xf,
+	.cal_l = 0x48,
+	.alpha = 0xa000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000000,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll video_cc_pll1 = {
+	.offset = 0x1000,
+	.config = &video_cc_pll1_config,
+	.vco_table = taycan_eha_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eha_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_pll1",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eha_t_ops,
+		},
+	},
+};
+
+/* 300.0 MHz Configuration */
+static const struct alpha_pll_config video_cc_pll2_config = {
+	.l = 0xf,
+	.cal_l = 0x48,
+	.alpha = 0xa000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000000,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll video_cc_pll2 = {
+	.offset = 0x2000,
+	.config = &video_cc_pll2_config,
+	.vco_table = taycan_eha_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eha_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_pll2",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eha_t_ops,
+		},
+	},
+};
+
+/* 300.0 MHz Configuration */
+static const struct alpha_pll_config video_cc_pll3_config = {
+	.l = 0xf,
+	.cal_l = 0x48,
+	.alpha = 0xa000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000000,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll video_cc_pll3 = {
+	.offset = 0x3000,
+	.config = &video_cc_pll3_config,
+	.vco_table = taycan_eha_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eha_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_pll3",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eha_t_ops,
+		},
+	},
+};
+
+static const struct parent_map video_cc_parent_map_0[] = {
+	{ P_BI_TCXO, 0 },
+};
+
+static const struct clk_parent_data video_cc_parent_data_0[] = {
+	{ .index = DT_BI_TCXO },
+};
+
+static const struct parent_map video_cc_parent_map_1[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_VIDEO_CC_PLL1_OUT_MAIN, 1 },
+};
+
+static const struct clk_parent_data video_cc_parent_data_1[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &video_cc_pll1.clkr.hw },
+};
+
+static const struct parent_map video_cc_parent_map_2[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_VIDEO_CC_PLL3_OUT_MAIN, 1 },
+};
+
+static const struct clk_parent_data video_cc_parent_data_2[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &video_cc_pll3.clkr.hw },
+};
+
+static const struct parent_map video_cc_parent_map_3[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_VIDEO_CC_PLL2_OUT_MAIN, 1 },
+};
+
+static const struct clk_parent_data video_cc_parent_data_3[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &video_cc_pll2.clkr.hw },
+};
+
+static const struct parent_map video_cc_parent_map_4[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_VIDEO_CC_PLL0_OUT_MAIN, 1 },
+	{ P_VIDEO_CC_PLL0_OUT_EVEN, 2 },
+};
+
+static const struct clk_parent_data video_cc_parent_data_4[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &video_cc_pll0.clkr.hw },
+	{ .hw = &video_cc_pll0_out_even.clkr.hw },
+};
+
+static const struct freq_tbl ftbl_video_cc_ahb_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 video_cc_ahb_clk_src = {
+	.cmd_rcgr = 0x8060,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = video_cc_parent_map_0,
+	.freq_tbl = ftbl_video_cc_ahb_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_ahb_clk_src",
+		.parent_data = video_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(video_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_video_cc_mvs0_clk_src[] = {
+	F(150000000, P_VIDEO_CC_PLL1_OUT_MAIN, 2, 0, 0),
+	F(240000000, P_VIDEO_CC_PLL1_OUT_MAIN, 2, 0, 0),
+	F(285000000, P_VIDEO_CC_PLL1_OUT_MAIN, 2, 0, 0),
+	F(311000000, P_VIDEO_CC_PLL1_OUT_MAIN, 2, 0, 0),
+	F(420000000, P_VIDEO_CC_PLL1_OUT_MAIN, 2, 0, 0),
+	F(444000000, P_VIDEO_CC_PLL1_OUT_MAIN, 2, 0, 0),
+	F(533000000, P_VIDEO_CC_PLL1_OUT_MAIN, 2, 0, 0),
+	F(630000000, P_VIDEO_CC_PLL1_OUT_MAIN, 2, 0, 0),
+	F(714000000, P_VIDEO_CC_PLL1_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 video_cc_mvs0_clk_src = {
+	.cmd_rcgr = 0x8030,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = video_cc_parent_map_1,
+	.freq_tbl = ftbl_video_cc_mvs0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_mvs0_clk_src",
+		.parent_data = video_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(video_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_video_cc_mvs0a_clk_src[] = {
+	F(150000000, P_VIDEO_CC_PLL3_OUT_MAIN, 2, 0, 0),
+	F(240000000, P_VIDEO_CC_PLL3_OUT_MAIN, 2, 0, 0),
+	F(338000000, P_VIDEO_CC_PLL3_OUT_MAIN, 2, 0, 0),
+	F(420000000, P_VIDEO_CC_PLL3_OUT_MAIN, 2, 0, 0),
+	F(444000000, P_VIDEO_CC_PLL3_OUT_MAIN, 2, 0, 0),
+	F(533000000, P_VIDEO_CC_PLL3_OUT_MAIN, 2, 0, 0),
+	F(630000000, P_VIDEO_CC_PLL3_OUT_MAIN, 2, 0, 0),
+	F(710000000, P_VIDEO_CC_PLL3_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 video_cc_mvs0a_clk_src = {
+	.cmd_rcgr = 0x8000,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = video_cc_parent_map_2,
+	.freq_tbl = ftbl_video_cc_mvs0a_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_mvs0a_clk_src",
+		.parent_data = video_cc_parent_data_2,
+		.num_parents = ARRAY_SIZE(video_cc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_video_cc_mvs0b_clk_src[] = {
+	F(150000000, P_VIDEO_CC_PLL2_OUT_MAIN, 2, 0, 0),
+	F(240000000, P_VIDEO_CC_PLL2_OUT_MAIN, 2, 0, 0),
+	F(311000000, P_VIDEO_CC_PLL2_OUT_MAIN, 2, 0, 0),
+	F(420000000, P_VIDEO_CC_PLL2_OUT_MAIN, 2, 0, 0),
+	F(444000000, P_VIDEO_CC_PLL2_OUT_MAIN, 2, 0, 0),
+	F(533000000, P_VIDEO_CC_PLL2_OUT_MAIN, 2, 0, 0),
+	F(630000000, P_VIDEO_CC_PLL2_OUT_MAIN, 2, 0, 0),
+	F(667000000, P_VIDEO_CC_PLL2_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 video_cc_mvs0b_clk_src = {
+	.cmd_rcgr = 0x8018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = video_cc_parent_map_3,
+	.freq_tbl = ftbl_video_cc_mvs0b_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_mvs0b_clk_src",
+		.parent_data = video_cc_parent_data_3,
+		.num_parents = ARRAY_SIZE(video_cc_parent_data_3),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_video_cc_mvs0c_clk_src[] = {
+	F(225000000, P_VIDEO_CC_PLL0_OUT_EVEN, 1, 0, 0),
+	F(360000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(430000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(557000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(634000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(782000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(928000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1060000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 video_cc_mvs0c_clk_src = {
+	.cmd_rcgr = 0x8048,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = video_cc_parent_map_4,
+	.freq_tbl = ftbl_video_cc_mvs0c_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_mvs0c_clk_src",
+		.parent_data = video_cc_parent_data_4,
+		.num_parents = ARRAY_SIZE(video_cc_parent_data_4),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 video_cc_xo_clk_src = {
+	.cmd_rcgr = 0x8180,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = video_cc_parent_map_0,
+	.freq_tbl = ftbl_video_cc_ahb_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_xo_clk_src",
+		.parent_data = video_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(video_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_branch video_cc_cx_axi0_clk = {
+	.halt_reg = 0x81e8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x81e8,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x81e8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_cx_axi0_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0_clk = {
+	.halt_reg = 0x80cc,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x80cc,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x80cc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0_shift_clk = {
+	.halt_reg = 0x81a4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x81a4,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x81a4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0_shift_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0_vpp0_clk = {
+	.halt_reg = 0x811c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x811c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x811c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0_vpp0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0_vpp0_vpp1_gating_clk = {
+	.halt_reg = 0x80c8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x80c8,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x80c8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0_vpp0_vpp1_gating_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0_vpp1_clk = {
+	.halt_reg = 0x80f4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x80f4,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x80f4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0_vpp1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0a_clk = {
+	.halt_reg = 0x8090,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x8090,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x8090,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0a_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs0a_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0b_clk = {
+	.halt_reg = 0x80b8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x80b8,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x80b8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0b_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs0b_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0c_clk = {
+	.halt_reg = 0x814c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x814c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x814c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0c_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs0c_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0c_ctl_freerun_clk = {
+	.halt_reg = 0x8160,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x8160,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0c_ctl_freerun_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs0c_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0c_debug_clk = {
+	.halt_reg = 0x8148,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x8148,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0c_debug_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs0c_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0c_freerun_clk = {
+	.halt_reg = 0x815c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x815c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0c_freerun_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs0c_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0c_shift_clk = {
+	.halt_reg = 0x81a8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x81a8,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x81a8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0c_shift_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc video_cc_mvs0c_gdsc = {
+	.gdscr = 0x8130,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x6,
+	.pd = {
+		.name = "video_cc_mvs0c_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc video_cc_axi0_cx_int_gdsc = {
+	.gdscr = 0x81cc,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "video_cc_axi0_cx_int_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.parent = &video_cc_mvs0c_gdsc.pd,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc video_cc_mm_int_gdsc = {
+	.gdscr = 0x81b4,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x2,
+	.pd = {
+		.name = "video_cc_mm_int_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.parent = &video_cc_axi0_cx_int_gdsc.pd,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc video_cc_mvs0_gdsc = {
+	.gdscr = 0x80a4,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x6,
+	.pd = {
+		.name = "video_cc_mvs0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.parent = &video_cc_mm_int_gdsc.pd,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc video_cc_mvs0_vpp0_gdsc = {
+	.gdscr = 0x8108,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "video_cc_mvs0_vpp0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.parent = &video_cc_mvs0_gdsc.pd,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc video_cc_mvs0_vpp1_gdsc = {
+	.gdscr = 0x80e0,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "video_cc_mvs0_vpp1_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.parent = &video_cc_mvs0_gdsc.pd,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc video_cc_mvs0a_gdsc = {
+	.gdscr = 0x807c,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "video_cc_mvs0a_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.parent = &video_cc_mm_int_gdsc.pd,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct clk_regmap *video_cc_hawi_clocks[] = {
+	[VIDEO_CC_AHB_CLK_SRC] = &video_cc_ahb_clk_src.clkr,
+	[VIDEO_CC_CX_AXI0_CLK] = &video_cc_cx_axi0_clk.clkr,
+	[VIDEO_CC_MVS0_CLK] = &video_cc_mvs0_clk.clkr,
+	[VIDEO_CC_MVS0_CLK_SRC] = &video_cc_mvs0_clk_src.clkr,
+	[VIDEO_CC_MVS0_SHIFT_CLK] = &video_cc_mvs0_shift_clk.clkr,
+	[VIDEO_CC_MVS0_VPP0_CLK] = &video_cc_mvs0_vpp0_clk.clkr,
+	[VIDEO_CC_MVS0_VPP0_VPP1_GATING_CLK] = &video_cc_mvs0_vpp0_vpp1_gating_clk.clkr,
+	[VIDEO_CC_MVS0_VPP1_CLK] = &video_cc_mvs0_vpp1_clk.clkr,
+	[VIDEO_CC_MVS0A_CLK] = &video_cc_mvs0a_clk.clkr,
+	[VIDEO_CC_MVS0A_CLK_SRC] = &video_cc_mvs0a_clk_src.clkr,
+	[VIDEO_CC_MVS0B_CLK] = &video_cc_mvs0b_clk.clkr,
+	[VIDEO_CC_MVS0B_CLK_SRC] = &video_cc_mvs0b_clk_src.clkr,
+	[VIDEO_CC_MVS0C_CLK] = &video_cc_mvs0c_clk.clkr,
+	[VIDEO_CC_MVS0C_CLK_SRC] = &video_cc_mvs0c_clk_src.clkr,
+	[VIDEO_CC_MVS0C_CTL_FREERUN_CLK] = &video_cc_mvs0c_ctl_freerun_clk.clkr,
+	[VIDEO_CC_MVS0C_DEBUG_CLK] = &video_cc_mvs0c_debug_clk.clkr,
+	[VIDEO_CC_MVS0C_FREERUN_CLK] = &video_cc_mvs0c_freerun_clk.clkr,
+	[VIDEO_CC_MVS0C_SHIFT_CLK] = &video_cc_mvs0c_shift_clk.clkr,
+	[VIDEO_CC_PLL0] = &video_cc_pll0.clkr,
+	[VIDEO_CC_PLL0_OUT_EVEN] = &video_cc_pll0_out_even.clkr,
+	[VIDEO_CC_PLL1] = &video_cc_pll1.clkr,
+	[VIDEO_CC_PLL2] = &video_cc_pll2.clkr,
+	[VIDEO_CC_PLL3] = &video_cc_pll3.clkr,
+	[VIDEO_CC_XO_CLK_SRC] = &video_cc_xo_clk_src.clkr,
+};
+
+static struct gdsc *video_cc_hawi_gdscs[] = {
+	[VIDEO_CC_AXI0_CX_INT_GDSC] = &video_cc_axi0_cx_int_gdsc,
+	[VIDEO_CC_MM_INT_GDSC] = &video_cc_mm_int_gdsc,
+	[VIDEO_CC_MVS0_GDSC] = &video_cc_mvs0_gdsc,
+	[VIDEO_CC_MVS0_VPP0_GDSC] = &video_cc_mvs0_vpp0_gdsc,
+	[VIDEO_CC_MVS0_VPP1_GDSC] = &video_cc_mvs0_vpp1_gdsc,
+	[VIDEO_CC_MVS0A_GDSC] = &video_cc_mvs0a_gdsc,
+	[VIDEO_CC_MVS0C_GDSC] = &video_cc_mvs0c_gdsc,
+};
+
+static const struct qcom_reset_map video_cc_hawi_resets[] = {
+	[VIDEO_CC_AXI0_CX_INT_BCR] = { 0x81c8 },
+	[VIDEO_CC_INTERFACE_BCR] = { 0x8164 },
+	[VIDEO_CC_MM_INT_BCR] = { 0x81b0 },
+	[VIDEO_CC_MVS0_BCR] = { 0x80a0 },
+	[VIDEO_CC_MVS0_VPP0_BCR] = { 0x8104 },
+	[VIDEO_CC_MVS0_VPP1_BCR] = { 0x80dc },
+	[VIDEO_CC_MVS0A_BCR] = { 0x8078 },
+	[VIDEO_CC_MVS0C_CLK_ARES] = { 0x814c, 2 },
+	[VIDEO_CC_MVS0C_BCR] = { 0x812c },
+	[VIDEO_CC_MVS0C_CTL_FREERUN_CLK_ARES] = { 0x8160, 2 },
+	[VIDEO_CC_MVS0C_FREERUN_CLK_ARES] = { 0x815c, 2 },
+	[VIDEO_CC_XO_CLK_ARES] = { 0x8198, 2 },
+};
+
+static struct clk_alpha_pll *video_cc_hawi_plls[] = {
+	&video_cc_pll0,
+	&video_cc_pll1,
+	&video_cc_pll2,
+	&video_cc_pll3,
+};
+
+static const u32 video_cc_hawi_critical_cbcrs[] = {
+	0x8168, /* VIDEO_CC_AHB_CLK */
+	0x81e4, /* VIDEO_CC_CX_DBGCH_XO_CLK */
+	0x81e0, /* VIDEO_CC_CX_XO_CLK */
+	0x81a0, /* VIDEO_CC_DBGCH_XO_CLK */
+	0x81ac, /* VIDEO_CC_SLEEP_CLK */
+	0x819c, /* VIDEO_CC_TS_XO_CLK */
+	0x8198, /* VIDEO_CC_XO_CLK */
+};
+
+static const struct regmap_config video_cc_hawi_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xa018,
+	.fast_io = true,
+};
+
+static struct qcom_cc_driver_data video_cc_hawi_driver_data = {
+	.alpha_plls = video_cc_hawi_plls,
+	.num_alpha_plls = ARRAY_SIZE(video_cc_hawi_plls),
+	.clk_cbcrs = video_cc_hawi_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(video_cc_hawi_critical_cbcrs),
+};
+
+static const struct qcom_cc_desc video_cc_hawi_desc = {
+	.config = &video_cc_hawi_regmap_config,
+	.clks = video_cc_hawi_clocks,
+	.num_clks = ARRAY_SIZE(video_cc_hawi_clocks),
+	.resets = video_cc_hawi_resets,
+	.num_resets = ARRAY_SIZE(video_cc_hawi_resets),
+	.gdscs = video_cc_hawi_gdscs,
+	.num_gdscs = ARRAY_SIZE(video_cc_hawi_gdscs),
+	.use_rpm = true,
+	.driver_data = &video_cc_hawi_driver_data,
+};
+
+static const struct of_device_id video_cc_hawi_match_table[] = {
+	{ .compatible = "qcom,hawi-videocc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, video_cc_hawi_match_table);
+
+static int video_cc_hawi_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &video_cc_hawi_desc);
+}
+
+static struct platform_driver video_cc_hawi_driver = {
+	.probe = video_cc_hawi_probe,
+	.driver = {
+		.name = "videocc-hawi",
+		.of_match_table = video_cc_hawi_match_table,
+	},
+};
+
+module_platform_driver(video_cc_hawi_driver);
+
+MODULE_DESCRIPTION("QTI VIDEOCC Hawi Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/videocc-x1p42100.c b/drivers/clk/qcom/videocc-x1p42100.c
index 503c03210ec8f..6de596908cb73 100644
--- a/drivers/clk/qcom/videocc-x1p42100.c
+++ b/drivers/clk/qcom/videocc-x1p42100.c
@@ -526,7 +526,7 @@ static struct clk_alpha_pll *video_cc_x1p42100_plls[] = {
 	&video_cc_pll1,
 };
 
-static u32 video_cc_x1p42100_critical_cbcrs[] = {
+static const u32 video_cc_x1p42100_critical_cbcrs[] = {
 	0x80f4, /* VIDEO_CC_AHB_CLK */
 	0x8150, /* VIDEO_CC_SLEEP_CLK */
 	0x8124, /* VIDEO_CC_XO_CLK */
@@ -540,7 +540,7 @@ static const struct regmap_config video_cc_x1p42100_regmap_config = {
 	.fast_io = true,
 };
 
-static struct qcom_cc_driver_data video_cc_x1p42100_driver_data = {
+static const struct qcom_cc_driver_data video_cc_x1p42100_driver_data = {
 	.alpha_plls = video_cc_x1p42100_plls,
 	.num_alpha_plls = ARRAY_SIZE(video_cc_x1p42100_plls),
 	.clk_cbcrs = video_cc_x1p42100_critical_cbcrs,
diff --git a/drivers/clk/renesas/Kconfig b/drivers/clk/renesas/Kconfig
index 0203ecbb38825..5c0238e878b70 100644
--- a/drivers/clk/renesas/Kconfig
+++ b/drivers/clk/renesas/Kconfig
@@ -218,10 +218,12 @@ config CLK_R9A09G057
 config CLK_R9A09G077
 	bool "RZ/T2H clock support" if COMPILE_TEST
 	select CLK_RENESAS_CPG_MSSR
+	select CLK_RZV2H_CPG_LIB
 
 config CLK_R9A09G087
 	bool "RZ/N2H clock support" if COMPILE_TEST
 	select CLK_RENESAS_CPG_MSSR
+	select CLK_RZV2H_CPG_LIB
 
 config CLK_SH73A0
 	bool "SH-Mobile AG5 clock support" if COMPILE_TEST
@@ -260,8 +262,12 @@ config CLK_RZG2L
 
 config CLK_RZV2H
 	bool "RZ/{G3E,V2H(P)} family clock support" if COMPILE_TEST
+	select CLK_RZV2H_CPG_LIB
 	select RESET_CONTROLLER
 
+config CLK_RZV2H_CPG_LIB
+	bool "RZV2H CPG library functions" if COMPILE_TEST
+
 config CLK_RENESAS_VBATTB
 	tristate "Renesas VBATTB clock controller"
 	depends on ARCH_RZG2L || COMPILE_TEST
diff --git a/drivers/clk/renesas/Makefile b/drivers/clk/renesas/Makefile
index bd2bed91ab291..ac790e56034b9 100644
--- a/drivers/clk/renesas/Makefile
+++ b/drivers/clk/renesas/Makefile
@@ -52,6 +52,7 @@ obj-$(CONFIG_CLK_RCAR_GEN3_CPG)		+= rcar-gen3-cpg.o
 obj-$(CONFIG_CLK_RCAR_GEN4_CPG)		+= rcar-gen4-cpg.o
 obj-$(CONFIG_CLK_RCAR_USB2_CLOCK_SEL)	+= rcar-usb2-clock-sel.o
 obj-$(CONFIG_CLK_RZG2L)			+= rzg2l-cpg.o
+obj-$(CONFIG_CLK_RZV2H_CPG_LIB)		+= rzv2h-cpg-lib.o
 obj-$(CONFIG_CLK_RZV2H)			+= rzv2h-cpg.o
 
 # Generic
diff --git a/drivers/clk/renesas/r9a08g045-cpg.c b/drivers/clk/renesas/r9a08g045-cpg.c
index 624fc5e6fb247..ea2a6a71aebd7 100644
--- a/drivers/clk/renesas/r9a08g045-cpg.c
+++ b/drivers/clk/renesas/r9a08g045-cpg.c
@@ -77,6 +77,7 @@ enum clk_ids {
 	CLK_SEL_PLL4,
 	CLK_P1_DIV2,
 	CLK_P3_DIV2,
+	CLK_P4_DIV2,
 	CLK_SD0_DIV4,
 	CLK_SD1_DIV4,
 	CLK_SD2_DIV4,
@@ -172,6 +173,8 @@ static const struct cpg_core_clk r9a08g045_core_clks[] __initconst = {
 	DEF_G3S_DIV("P3", R9A08G045_CLK_P3, CLK_PLL3_DIV2_4, DIVPL3C, G3S_DIVPL3C_STS,
 		    dtable_1_32, 0, 0, 0, NULL),
 	DEF_FIXED("P3_DIV2", CLK_P3_DIV2, R9A08G045_CLK_P3, 1, 2),
+	DEF_FIXED("P4", R9A08G045_CLK_P4, CLK_PLL2_DIV2, 1, 5),
+	DEF_FIXED("P4_DIV2", CLK_P4_DIV2, R9A08G045_CLK_P4, 1, 2),
 	DEF_FIXED("P5", R9A08G045_CLK_P5, CLK_PLL2_DIV2, 1, 4),
 	DEF_FIXED("ZT", R9A08G045_CLK_ZT, CLK_PLL3_DIV2_8, 1, 1),
 	DEF_FIXED("S0", R9A08G045_CLK_S0, CLK_SEL_PLL4, 1, 2),
@@ -274,6 +277,10 @@ static const struct rzg2l_mod_clk r9a08g045_mod_clks[] = {
 					MSTOP(BUS_MCPU2, BIT(5))),
 	DEF_MOD("scif5_clk_pck",	R9A08G045_SCIF5_CLK_PCK, R9A08G045_CLK_P0, 0x584, 5,
 					MSTOP(BUS_MCPU3, BIT(4))),
+	DEF_MOD("canfd_pclk",		R9A08G045_CANFD_PCLK, CLK_P4_DIV2, 0x594, 0,
+					MSTOP(BUS_MCPU2, BIT(9))),
+	DEF_MOD("canfd_clk_ram",	R9A08G045_CANFD_CLK_RAM, R9A08G045_CLK_P4, 0x594, 1,
+					MSTOP(BUS_MCPU2, BIT(9))),
 	DEF_MOD("gpio_hclk",		R9A08G045_GPIO_HCLK, R9A08G045_OSCCLK, 0x598, 0,
 					MSTOP(BUS_PERI_CPU, BIT(6))),
 	DEF_MOD("adc_adclk",		R9A08G045_ADC_ADCLK, R9A08G045_CLK_TSU, 0x5a8, 0,
@@ -324,6 +331,8 @@ static const struct rzg2l_reset r9a08g045_resets[] = {
 	DEF_RST(R9A08G045_SCIF3_RST_SYSTEM_N, 0x884, 3),
 	DEF_RST(R9A08G045_SCIF4_RST_SYSTEM_N, 0x884, 4),
 	DEF_RST(R9A08G045_SCIF5_RST_SYSTEM_N, 0x884, 5),
+	DEF_RST(R9A08G045_CANFD_RSTP_N, 0x894, 0),
+	DEF_RST(R9A08G045_CANFD_RSTC_N, 0x894, 1),
 	DEF_RST(R9A08G045_GPIO_RSTN, 0x898, 0),
 	DEF_RST(R9A08G045_GPIO_PORT_RESETN, 0x898, 1),
 	DEF_RST(R9A08G045_GPIO_SPARE_RESETN, 0x898, 2),
diff --git a/drivers/clk/renesas/r9a08g046-cpg.c b/drivers/clk/renesas/r9a08g046-cpg.c
index a57638734ce7c..4488bd1988e82 100644
--- a/drivers/clk/renesas/r9a08g046-cpg.c
+++ b/drivers/clk/renesas/r9a08g046-cpg.c
@@ -17,10 +17,15 @@
 /* RZ/G3L Specific registers. */
 #define G3L_CPG_PL2_DDIV		(0x204)
 #define G3L_CPG_PL3_DDIV		(0x208)
+#define G3L_CPG_SDHI_DDIV		(0x218)
+#define G3L_CPG_GE3D_DDIV		(0x224)
 #define G3L_CPG_CA55CORE_DDIV		(0x234)
 #define G3L_CPG_RSCI_DDIV		(0x238)
 #define G3L_CPG_RSPI_DDIV		(0x23c)
+#define G3L_CPG_SDHI_DSEL		(0x244)
 #define G3L_CLKDIVSTATUS		(0x280)
+#define G3L_CLKSELSTATUS		(0x284)
+#define G3L_CPG_GE3D_SSEL		(0x40c)
 #define G3L_CPG_ETH_SSEL		(0x410)
 #define G3L_CPG_RSCI_SSEL		(0x414)
 #define G3L_CPG_RSPI_SSEL		(0x418)
@@ -30,6 +35,10 @@
 #define G3L_DIVPL2A		DDIV_PACK(G3L_CPG_PL2_DDIV, 0, 2)
 #define G3L_DIVPL2B		DDIV_PACK(G3L_CPG_PL2_DDIV, 4, 2)
 #define G3L_DIVPL3A		DDIV_PACK(G3L_CPG_PL3_DDIV, 0, 2)
+#define G3L_DIV_SDHI0		DDIV_PACK(G3L_CPG_SDHI_DDIV, 0, 2)
+#define G3L_DIV_SDHI1		DDIV_PACK(G3L_CPG_SDHI_DDIV, 4, 2)
+#define G3L_DIV_SDHI2		DDIV_PACK(G3L_CPG_SDHI_DDIV, 8, 2)
+#define G3L_DIV_GE3D		DDIV_PACK(G3L_CPG_GE3D_DDIV, 0, 3)
 #define G3L_DIV_CA55_CORE0	DDIV_PACK(G3L_CPG_CA55CORE_DDIV, 0, 3)
 #define G3L_DIV_CA55_CORE1	DDIV_PACK(G3L_CPG_CA55CORE_DDIV, 4, 3)
 #define G3L_DIV_CA55_CORE2	DDIV_PACK(G3L_CPG_CA55CORE_DDIV, 8, 3)
@@ -61,8 +70,19 @@
 #define G3L_DIV_RSPI0_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 20, 1)
 #define G3L_DIV_RSPI1_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 21, 1)
 #define G3L_DIV_RSPI2_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 22, 1)
+#define G3L_DIV_SDHI0_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 24, 1)
+#define G3L_DIV_SDHI1_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 25, 1)
+#define G3L_DIV_SDHI2_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 26, 1)
+
+#define G3L_SEL_SDHI0_STS	SEL_PLL_PACK(G3L_CLKSELSTATUS, 16, 1)
+#define G3L_SEL_SDHI1_STS	SEL_PLL_PACK(G3L_CLKSELSTATUS, 17, 1)
+#define G3L_SEL_SDHI2_STS	SEL_PLL_PACK(G3L_CLKSELSTATUS, 18, 1)
+#define G3L_DIV_GE3D_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 27, 1)
 
 /* RZ/G3L Specific clocks select. */
+#define G3L_SEL_SDHI0		SEL_PLL_PACK(G3L_CPG_SDHI_DSEL, 0, 2)
+#define G3L_SEL_SDHI1		SEL_PLL_PACK(G3L_CPG_SDHI_DSEL, 4, 2)
+#define G3L_SEL_SDHI2		SEL_PLL_PACK(G3L_CPG_SDHI_DSEL, 8, 2)
 #define G3L_SEL_ETH0_TX		SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 0, 1)
 #define G3L_SEL_ETH0_RX		SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 1, 1)
 #define G3L_SEL_ETH0_RM		SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 2, 1)
@@ -73,6 +93,7 @@
 #define G3L_SEL_ETH1_RM		SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 10, 1)
 #define G3L_SEL_ETH1_CLK_TX_I	SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 11, 1)
 #define G3L_SEL_ETH1_CLK_RX_I	SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 12, 1)
+#define G3L_SEL_GE3D		SEL_PLL_PACK(G3L_CPG_GE3D_SSEL, 0, 2)
 #define G3L_SEL_RSCI0		SEL_PLL_PACK(G3L_CPG_RSCI_SSEL, 0, 2)
 #define G3L_SEL_RSCI1		SEL_PLL_PACK(G3L_CPG_RSCI_SSEL, 2, 2)
 #define G3L_SEL_RSCI2		SEL_PLL_PACK(G3L_CPG_RSCI_SSEL, 4, 2)
@@ -94,6 +115,7 @@ enum clk_ids {
 
 	/* Internal Core Clocks */
 	CLK_PLL1,
+	CLK_PLL1_DIV2,
 	CLK_PLL2,
 	CLK_PLL2_DIV2,
 	CLK_PLL2_DIV2_4,
@@ -102,6 +124,8 @@ enum clk_ids {
 	CLK_PLL2_DIV7,
 	CLK_PLL3,
 	CLK_PLL3_DIV2,
+	CLK_PLL3_DIV2_2,
+	CLK_PLL3_DIV3,
 	CLK_PLL6,
 	CLK_PLL6_DIV10,
 	CLK_SEL_ETH0_TX,
@@ -110,6 +134,7 @@ enum clk_ids {
 	CLK_SEL_ETH1_TX,
 	CLK_SEL_ETH1_RX,
 	CLK_SEL_ETH1_RM,
+	CLK_SEL_GE3D,
 	CLK_SEL_RSCI0,
 	CLK_SEL_RSCI1,
 	CLK_SEL_RSCI2,
@@ -117,16 +142,29 @@ enum clk_ids {
 	CLK_SEL_RSPI0,
 	CLK_SEL_RSPI1,
 	CLK_SEL_RSPI2,
+	CLK_SEL_SDHI0,
+	CLK_SEL_SDHI1,
+	CLK_SEL_SDHI2,
 	CLK_ETH0_TR,
 	CLK_ETH0_RM,
 	CLK_ETH1_TR,
 	CLK_ETH1_RM,
+	CLK_SD0_DIV2,
+	CLK_SD1_DIV2,
+	CLK_SD2_DIV2,
 
 	/* Module Clocks */
 	MOD_CLK_BASE,
 };
 
 /* Divider tables */
+static const struct clk_div_table dtable_1_4[] = {
+	{ 0, 1 },
+	{ 1, 2 },
+	{ 2, 4 },
+	{ 0, 0 },
+};
+
 static const struct clk_div_table dtable_1_8[] = {
 	{ 0, 1 },
 	{ 1, 2 },
@@ -189,12 +227,17 @@ static const char * const sel_eth0_rm[] = { ".pll6_div10", "eth0_rxc_rx_clk" };
 static const char * const sel_eth1_tx[] = { ".div_eth1_tr", "eth1_txc_tx_clk" };
 static const char * const sel_eth1_rx[] = { ".div_eth1_tr", "eth1_rxc_rx_clk" };
 static const char * const sel_eth1_rm[] = { ".pll6_div10", "eth1_rxc_rx_clk" };
+static const char * const sel_ge3d[] = { ".pll1_div2", ".pll3_div3", ".pll6", ".pll3_div2_2" };
 static const char * const sel_rsci_rspi[] = { ".pll2_div5", ".pll2_div6", ".pll2_div7", ".pll2_div2_4" };
+static const char * const sel_sdhi[] = { ".pll2_div2", ".pll1_div2",  ".pll6", ".pll2_div6" };
 static const char * const sel_eth0_clk_tx_i[] = { ".sel_eth0_tx", ".div_eth0_rm" };
 static const char * const sel_eth0_clk_rx_i[] = { ".sel_eth0_rx", ".div_eth0_rm" };
 static const char * const sel_eth1_clk_tx_i[] = { ".sel_eth1_tx", ".div_eth1_rm" };
 static const char * const sel_eth1_clk_rx_i[] = { ".sel_eth1_rx", ".div_eth1_rm" };
 
+/* Mux clock indices tables. */
+static const u32 mtable_sd[] = { 0, 1, 2, 3 };
+
 static const struct cpg_core_clk r9a08g046_core_clks[] __initconst = {
 	/* External Clock Inputs */
 	DEF_INPUT("extal", CLK_EXTAL),
@@ -210,13 +253,23 @@ static const struct cpg_core_clk r9a08g046_core_clks[] __initconst = {
 	DEF_FIXED(".pll3", CLK_PLL3, CLK_EXTAL, 200, 3),
 	DEF_G3L_PLL(".pll6", CLK_PLL6, CLK_EXTAL, CPG_PLL_CONF(0x50, 0),
 		    500000000UL),
+	DEF_FIXED(".pll1_div2", CLK_PLL1_DIV2, CLK_PLL1, 1, 2),
 	DEF_FIXED(".pll2_div2", CLK_PLL2_DIV2, CLK_PLL2, 1, 2),
 	DEF_FIXED(".pll2_div2_4", CLK_PLL2_DIV2_4, CLK_PLL2_DIV2, 1, 4),
 	DEF_FIXED(".pll2_div5", CLK_PLL2_DIV5, CLK_PLL2, 1, 5),
 	DEF_FIXED(".pll2_div6", CLK_PLL2_DIV6, CLK_PLL2, 1, 6),
 	DEF_FIXED(".pll2_div7", CLK_PLL2_DIV7, CLK_PLL2, 1, 7),
 	DEF_FIXED(".pll3_div2", CLK_PLL3_DIV2, CLK_PLL3, 1, 2),
+	DEF_FIXED(".pll3_div2_2", CLK_PLL3_DIV2_2, CLK_PLL3_DIV2, 1, 2),
+	DEF_FIXED(".pll3_div3", CLK_PLL3_DIV3, CLK_PLL3, 1, 3),
 	DEF_FIXED(".pll6_div10", CLK_PLL6_DIV10, CLK_PLL6, 1, 10),
+	DEF_SD_MUX(".sel_sdhi0", CLK_SEL_SDHI0, G3L_SEL_SDHI0, G3L_SEL_SDHI0_STS, sel_sdhi,
+		   mtable_sd, 0, NULL),
+	DEF_SD_MUX(".sel_sdhi1", CLK_SEL_SDHI1, G3L_SEL_SDHI1, G3L_SEL_SDHI1_STS, sel_sdhi,
+		   mtable_sd, 0, NULL),
+	DEF_SD_MUX(".sel_sdhi2", CLK_SEL_SDHI2, G3L_SEL_SDHI2, G3L_SEL_SDHI2_STS, sel_sdhi,
+		   mtable_sd, 0, NULL),
+	DEF_MUX(".sel_ge3d", CLK_SEL_GE3D, G3L_SEL_GE3D, sel_ge3d),
 	DEF_MUX(".sel_rsci0", CLK_SEL_RSCI0, G3L_SEL_RSCI0, sel_rsci_rspi),
 	DEF_MUX(".sel_rsci1", CLK_SEL_RSCI1, G3L_SEL_RSCI1, sel_rsci_rspi),
 	DEF_MUX(".sel_rsci2", CLK_SEL_RSCI2, G3L_SEL_RSCI2, sel_rsci_rspi),
@@ -264,6 +317,18 @@ static const struct cpg_core_clk r9a08g046_core_clks[] __initconst = {
 		    dtable_1_8, 0, 200000000UL, 0, NULL),
 	DEF_G3S_DIV("P19", R9A08G046_CLK_P19, CLK_SEL_RSPI2, G3L_DIV_RSPI2, G3L_DIV_RSPI2_STS,
 		    dtable_1_8, 0, 200000000UL, 0, NULL),
+	DEF_G3S_DIV("SD0", R9A08G046_CLK_SD0, CLK_SEL_SDHI0, G3L_DIV_SDHI0, G3L_DIV_SDHI0_STS,
+		    dtable_1_4, 800000000UL, 600000000UL, CLK_SET_RATE_PARENT,
+		    rzg3s_cpg_div_clk_notifier),
+	DEF_G3S_DIV("SD1", R9A08G046_CLK_SD1, CLK_SEL_SDHI1, G3L_DIV_SDHI1, G3L_DIV_SDHI1_STS,
+		    dtable_1_4, 800000000UL, 600000000UL, CLK_SET_RATE_PARENT,
+		    rzg3s_cpg_div_clk_notifier),
+	DEF_G3S_DIV("SD2", R9A08G046_CLK_SD2, CLK_SEL_SDHI2, G3L_DIV_SDHI2, G3L_DIV_SDHI2_STS,
+		    dtable_1_4, 800000000UL, 600000000UL, CLK_SET_RATE_PARENT,
+		    rzg3s_cpg_div_clk_notifier),
+	DEF_FIXED(".sd0_div2", CLK_SD0_DIV2, R9A08G046_CLK_SD0, 1, 2),
+	DEF_FIXED(".sd1_div2", CLK_SD1_DIV2, R9A08G046_CLK_SD1, 1, 2),
+	DEF_FIXED(".sd2_div2", CLK_SD2_DIV2, R9A08G046_CLK_SD2, 1, 2),
 	DEF_FIXED("HP", R9A08G046_CLK_HP, CLK_PLL6_DIV10, 1, 1),
 	DEF_MUX_FLAGS("ETHTX01", R9A08G046_CLK_ETHTX01, G3L_SEL_ETH0_CLK_TX_I, sel_eth0_clk_tx_i,
 		      CLK_SET_RATE_PARENT),
@@ -279,6 +344,8 @@ static const struct cpg_core_clk r9a08g046_core_clks[] __initconst = {
 	DEF_FIXED("ETHRM1", R9A08G046_CLK_ETHRM1, CLK_SEL_ETH1_RM, 1, 1),
 	DEF_FIXED("ETHTX12", R9A08G046_CLK_ETHTX12, CLK_SEL_ETH1_TX, 1, 1),
 	DEF_FIXED("ETHRX12", R9A08G046_CLK_ETHRX12, CLK_SEL_ETH1_RX, 1, 1),
+	DEF_G3S_DIV("G", R9A08G046_CLK_G, CLK_SEL_GE3D, G3L_DIV_GE3D, G3L_DIV_GE3D_STS,
+		    dtable_1_32, 0, 0, 0, NULL),
 	DEF_FIXED("OSCCLK", R9A08G046_OSCCLK, CLK_EXTAL, 1, 1),
 };
 
@@ -297,6 +364,42 @@ static const struct rzg2l_mod_clk r9a08g046_mod_clks[] = {
 					MSTOP(BUS_REG0, BIT(0))),
 	DEF_MOD("wdt0_clk",		R9A08G046_WDT0_CLK, R9A08G046_OSCCLK, 0x548, 1,
 					MSTOP(BUS_REG0, BIT(0))),
+	DEF_MOD("sdhi0_imclk",		R9A08G046_SDHI0_IMCLK, CLK_SD0_DIV2, 0x554, 0,
+					MSTOP(BUS_PERI_COM, BIT(0))),
+	DEF_MOD("sdhi0_imclk2",		R9A08G046_SDHI0_IMCLK2, CLK_SD0_DIV2, 0x554, 1,
+					MSTOP(BUS_PERI_COM, BIT(0))),
+	DEF_MOD("sdhi0_clk_hs",		R9A08G046_SDHI0_CLK_HS, R9A08G046_CLK_SD0, 0x554, 2,
+					MSTOP(BUS_PERI_COM, BIT(0))),
+	DEF_MOD("sdhi0_iaclks",		R9A08G046_SDHI0_IACLKS, R9A08G046_CLK_P1, 0x554, 3,
+					MSTOP(BUS_PERI_COM, BIT(0))),
+	DEF_MOD("sdhi0_iaclkm",		R9A08G046_SDHI0_IACLKM, R9A08G046_CLK_P1, 0x554, 12,
+					MSTOP(BUS_PERI_COM, BIT(0))),
+	DEF_MOD("sdhi1_imclk",		R9A08G046_SDHI1_IMCLK, CLK_SD1_DIV2, 0x554, 4,
+					MSTOP(BUS_PERI_COM, BIT(1))),
+	DEF_MOD("sdhi1_imclk2",		R9A08G046_SDHI1_IMCLK2, CLK_SD1_DIV2, 0x554, 5,
+					MSTOP(BUS_PERI_COM, BIT(1))),
+	DEF_MOD("sdhi1_clk_hs",		R9A08G046_SDHI1_CLK_HS, R9A08G046_CLK_SD1, 0x554, 6,
+					MSTOP(BUS_PERI_COM, BIT(1))),
+	DEF_MOD("sdhi1_iaclks",		R9A08G046_SDHI1_IACLKS, R9A08G046_CLK_P1, 0x554, 7,
+					MSTOP(BUS_PERI_COM, BIT(1))),
+	DEF_MOD("sdhi1_iaclkm",		R9A08G046_SDHI1_IACLKM, R9A08G046_CLK_P1, 0x554, 13,
+					MSTOP(BUS_PERI_COM, BIT(1))),
+	DEF_MOD("sdhi2_imclk",		R9A08G046_SDHI2_IMCLK, CLK_SD2_DIV2, 0x554, 8,
+					MSTOP(BUS_PERI_COM, BIT(11))),
+	DEF_MOD("sdhi2_imclk2",		R9A08G046_SDHI2_IMCLK2, CLK_SD2_DIV2, 0x554, 9,
+					MSTOP(BUS_PERI_COM, BIT(11))),
+	DEF_MOD("sdhi2_clk_hs",		R9A08G046_SDHI2_CLK_HS, R9A08G046_CLK_SD2, 0x554, 10,
+					MSTOP(BUS_PERI_COM, BIT(11))),
+	DEF_MOD("sdhi2_iaclks",		R9A08G046_SDHI2_IACLKS, R9A08G046_CLK_P1, 0x554, 11,
+					MSTOP(BUS_PERI_COM, BIT(11))),
+	DEF_MOD("sdhi2_iaclkm",		R9A08G046_SDHI2_IACLKM, R9A08G046_CLK_P1, 0x554, 14,
+					MSTOP(BUS_PERI_COM, BIT(11))),
+	DEF_MOD("ge3d_clk",		R9A08G046_GE3D_CLK, R9A08G046_CLK_G, 0x558, 0,
+					MSTOP(BUS_PERI_VIDEO, BIT(12))),
+	DEF_MOD("ge3d_axi_clk",		R9A08G046_GE3D_AXI_CLK, R9A08G046_CLK_P1, 0x558, 1,
+					MSTOP(BUS_PERI_VIDEO, BIT(12))),
+	DEF_MOD("ge3d_ace_clk",		R9A08G046_GE3D_ACE_CLK, R9A08G046_CLK_P1, 0x558, 2,
+					MSTOP(BUS_PERI_VIDEO, BIT(12))),
 	DEF_MOD("ssi0_pclk2",		R9A08G046_SSI0_PCLK2, R9A08G046_CLK_P0,	0x570, 0,
 					MSTOP(BUS_MCPU1, BIT(10))),
 	DEF_MOD("ssi0_pclk_sfr",	R9A08G046_SSI0_PCLK_SFR, R9A08G046_CLK_P0, 0x570, 1,
@@ -313,6 +416,16 @@ static const struct rzg2l_mod_clk r9a08g046_mod_clks[] = {
 					MSTOP(BUS_MCPU1, BIT(13))),
 	DEF_MOD("ssi3_pclk_sfr",	R9A08G046_SSI3_PCLK_SFR, R9A08G046_CLK_P0, 0x570, 7,
 					MSTOP(BUS_MCPU1, BIT(13))),
+	DEF_MOD("usb_u2h0_hclk",	R9A08G046_USB_U2H0_HCLK, R9A08G046_CLK_P1, 0x578, 0,
+					MSTOP(BUS_PERI_COM, BIT(5))),
+	DEF_MOD("usb_u2h1_hclk",	R9A08G046_USB_U2H1_HCLK, R9A08G046_CLK_P1, 0x578, 1,
+					MSTOP(BUS_PERI_COM, BIT(7))),
+	DEF_MOD("usb_u2p0_exr_cpuclk",	R9A08G046_USB_U2P0_EXR_CPUCLK, R9A08G046_CLK_P1, 0x578, 2,
+					MSTOP(BUS_PERI_COM, BIT(6))),
+	DEF_MOD("usb_pclk",		R9A08G046_USB_PCLK, R9A08G046_CLK_P1, 0x578, 3,
+					MSTOP(BUS_PERI_COM, BIT(4))),
+	DEF_MOD("usb_u2p1_exr_cpuclk",	R9A08G046_USB_U2P1_EXR_CPUCLK, R9A08G046_CLK_P1, 0x578, 4,
+					MSTOP(BUS_PERI_COM, BIT(13))),
 	DEF_MOD("eth0_clk_axi",		R9A08G046_ETH0_CLK_AXI, R9A08G046_CLK_P1, 0x57c, 0,
 					MSTOP(BUS_PERI_COM, BIT(2))),
 	DEF_MOD("eth1_clk_axi",		R9A08G046_ETH1_CLK_AXI, R9A08G046_CLK_P1, 0x57c, 1,
@@ -412,10 +525,27 @@ static const struct rzg2l_reset r9a08g046_resets[] = {
 	DEF_RST(R9A08G046_DMAC_ARESETN, 0x82c, 0),
 	DEF_RST(R9A08G046_DMAC_RST_ASYNC, 0x82c, 1),
 	DEF_RST(R9A08G046_WDT0_PRESETN, 0x848, 0),
+	DEF_RST(R9A08G046_SDHI0_IXRST, 0x854, 0),
+	DEF_RST(R9A08G046_SDHI1_IXRST, 0x854, 1),
+	DEF_RST(R9A08G046_SDHI2_IXRST, 0x854, 2),
+	DEF_RST(R9A08G046_SDHI0_IXRSTAXIM, 0x854, 3),
+	DEF_RST(R9A08G046_SDHI0_IXRSTAXIS, 0x854, 4),
+	DEF_RST(R9A08G046_SDHI1_IXRSTAXIM, 0x854, 5),
+	DEF_RST(R9A08G046_SDHI1_IXRSTAXIS, 0x854, 6),
+	DEF_RST(R9A08G046_SDHI2_IXRSTAXIM, 0x854, 7),
+	DEF_RST(R9A08G046_SDHI2_IXRSTAXIS, 0x854, 8),
+	DEF_RST(R9A08G046_GE3D_RESETN, 0x858, 0),
+	DEF_RST(R9A08G046_GE3D_AXI_RESETN, 0x858, 1),
+	DEF_RST(R9A08G046_GE3D_ACE_RESETN, 0x858, 2),
 	DEF_RST(R9A08G046_SSI0_RST_M2_REG, 0x870, 0),
 	DEF_RST(R9A08G046_SSI1_RST_M2_REG, 0x870, 1),
 	DEF_RST(R9A08G046_SSI2_RST_M2_REG, 0x870, 2),
 	DEF_RST(R9A08G046_SSI3_RST_M2_REG, 0x870, 3),
+	DEF_RST(R9A08G046_USB_U2H0_HRESETN, 0x878, 0),
+	DEF_RST(R9A08G046_USB_U2H1_HRESETN, 0x878, 1),
+	DEF_RST(R9A08G046_USB_U2P0_EXL_SYSRST, 0x878, 2),
+	DEF_RST(R9A08G046_USB_PRESETN, 0x878, 3),
+	DEF_RST(R9A08G046_USB_U2P1_EXL_SYSRST, 0x878, 4),
 	DEF_RST(R9A08G046_ETH0_ARESET_N, 0x87c, 0),
 	DEF_RST(R9A08G046_ETH1_ARESET_N, 0x87c, 1),
 	DEF_RST(R9A08G046_I2C0_MRST, 0x880, 0),
diff --git a/drivers/clk/renesas/r9a09g047-cpg.c b/drivers/clk/renesas/r9a09g047-cpg.c
index 94158b6834e6e..f576dd441a114 100644
--- a/drivers/clk/renesas/r9a09g047-cpg.c
+++ b/drivers/clk/renesas/r9a09g047-cpg.c
@@ -23,6 +23,8 @@ enum clk_ids {
 	CLK_AUDIO_EXTAL,
 	CLK_RTXIN,
 	CLK_QEXTAL,
+	CLK_AUDIO_CLKB,
+	CLK_AUDIO_CLKC,
 
 	/* PLL Clocks */
 	CLK_PLLCM33,
@@ -44,6 +46,7 @@ enum clk_ids {
 	CLK_SMUX2_XSPI_CLK1,
 	CLK_PLLCM33_XSPI,
 	CLK_PLLCLN_DIV2,
+	CLK_PLLCLN_DIV4,
 	CLK_PLLCLN_DIV8,
 	CLK_PLLCLN_DIV16,
 	CLK_PLLCLN_DIV20,
@@ -159,6 +162,8 @@ static const struct cpg_core_clk r9a09g047_core_clks[] __initconst = {
 	DEF_INPUT("audio_extal", CLK_AUDIO_EXTAL),
 	DEF_INPUT("rtxin", CLK_RTXIN),
 	DEF_INPUT("qextal", CLK_QEXTAL),
+	DEF_INPUT("audio_clkb", CLK_AUDIO_CLKB),
+	DEF_INPUT("audio_clkc", CLK_AUDIO_CLKC),
 
 	/* PLL Clocks */
 	DEF_FIXED(".pllcm33", CLK_PLLCM33, CLK_QEXTAL, 200, 3),
@@ -183,6 +188,7 @@ static const struct cpg_core_clk r9a09g047_core_clks[] __initconst = {
 	DEF_CSDIV(".pllcm33_xspi", CLK_PLLCM33_XSPI, CLK_SMUX2_XSPI_CLK1, CSDIV0_DIVCTL3,
 		  dtable_2_16),
 	DEF_FIXED(".pllcln_div2", CLK_PLLCLN_DIV2, CLK_PLLCLN, 1, 2),
+	DEF_FIXED(".pllcln_div4", CLK_PLLCLN_DIV4, CLK_PLLCLN, 1, 4),
 	DEF_FIXED(".pllcln_div8", CLK_PLLCLN_DIV8, CLK_PLLCLN, 1, 8),
 	DEF_FIXED(".pllcln_div16", CLK_PLLCLN_DIV16, CLK_PLLCLN, 1, 16),
 	DEF_FIXED(".pllcln_div20", CLK_PLLCLN_DIV20, CLK_PLLCLN, 1, 20),
@@ -194,6 +200,7 @@ static const struct cpg_core_clk r9a09g047_core_clks[] __initconst = {
 	DEF_FIXED(".plldty_acpu_div2", CLK_PLLDTY_ACPU_DIV2, CLK_PLLDTY_ACPU, 1, 2),
 	DEF_FIXED(".plldty_acpu_div4", CLK_PLLDTY_ACPU_DIV4, CLK_PLLDTY_ACPU, 1, 4),
 	DEF_FIXED(".plldty_div8", CLK_PLLDTY_DIV8, CLK_PLLDTY, 1, 8),
+	DEF_FIXED(".plldty_div16", CLK_PLLDTY_DIV16, CLK_PLLDTY, 1, 16),
 
 	DEF_FIXED(".plleth_250_fix", CLK_PLLETH_DIV_250_FIX, CLK_PLLETH, 1, 4),
 	DEF_FIXED(".plleth_125_fix", CLK_PLLETH_DIV_125_FIX, CLK_PLLETH_DIV_250_FIX, 1, 2),
@@ -205,7 +212,6 @@ static const struct cpg_core_clk r9a09g047_core_clks[] __initconst = {
 	DEF_SMUX(".smux2_gbe0_rxclk", CLK_SMUX2_GBE0_RXCLK, SSEL0_SELCTL3, smux2_gbe0_rxclk),
 	DEF_SMUX(".smux2_gbe1_txclk", CLK_SMUX2_GBE1_TXCLK, SSEL1_SELCTL0, smux2_gbe1_txclk),
 	DEF_SMUX(".smux2_gbe1_rxclk", CLK_SMUX2_GBE1_RXCLK, SSEL1_SELCTL1, smux2_gbe1_rxclk),
-	DEF_FIXED(".plldty_div16", CLK_PLLDTY_DIV16, CLK_PLLDTY, 1, 16),
 	DEF_DDIV(".plldty_rcpu", CLK_PLLDTY_RCPU, CLK_PLLDTY, CDDIV3_DIVCTL2, dtable_2_64),
 	DEF_FIXED(".plldty_rcpu_div4", CLK_PLLDTY_RCPU_DIV4, CLK_PLLDTY_RCPU, 1, 4),
 
@@ -530,10 +536,110 @@ static const struct rzv2h_mod_clk r9a09g047_mod_clks[] __initconst = {
 						BUS_MSTOP(3, BIT(4))),
 	DEF_MOD("ge3d_ace_clk",			CLK_PLLDTY_ACPU_DIV2, 15, 2, 7, 18,
 						BUS_MSTOP(3, BIT(4))),
+	DEF_MOD("ssif_0_clk",			CLK_PLLCLN_DIV8, 15, 5, 7, 21,
+						BUS_MSTOP(2, BIT(3) | BIT(4))),
+	DEF_MOD("scu_0_clk",			CLK_PLLCLN_DIV8, 15, 6, 7, 22,
+						BUS_MSTOP(2, BIT(0) | BIT(1))),
+	DEF_MOD("scu_0_clkx2",			CLK_PLLCLN_DIV4, 15, 7, 7, 23,
+						BUS_MSTOP(2, BIT(0) | BIT(1))),
+	DEF_MOD("dmacpp_0_clk",			CLK_PLLCLN_DIV8, 15, 8, 7, 24,
+						BUS_MSTOP(2, BIT(5))),
+	DEF_MOD("adg_0_clks1",			CLK_PLLCLN_DIV8, 15, 9, 7, 25,
+						BUS_MSTOP(2, BIT(2))),
+	DEF_MOD("adg_0_clk_195m",		CLK_PLLCLN_DIV8, 15, 10, 7, 26,
+						BUS_MSTOP(2, BIT(2))),
+	DEF_MOD("adg_0_audio_clka",		CLK_AUDIO_EXTAL, 15, 11, 7, 27,
+						BUS_MSTOP(2, BIT(2))),
+	DEF_MOD("adg_0_audio_clkb",		CLK_AUDIO_CLKB, 15, 12, 7, 28,
+						BUS_MSTOP(2, BIT(2))),
+	DEF_MOD("adg_0_audio_clkc",		CLK_AUDIO_CLKC, 15, 13, 7, 29,
+						BUS_MSTOP(2, BIT(2))),
 	DEF_MOD("tsu_1_pclk",			CLK_QEXTAL, 16, 10, 8, 10,
 						BUS_MSTOP(2, BIT(15))),
+	DEF_MOD("adg_ssi0_clk",			CLK_PLLCLN_DIV8, 22, 0, -1, -1,
+						BUS_MSTOP(2, BIT(2))),
+	DEF_MOD("adg_ssi1_clk",			CLK_PLLCLN_DIV8, 22, 1, -1, -1,
+						BUS_MSTOP(2, BIT(2))),
+	DEF_MOD("adg_ssi2_clk",			CLK_PLLCLN_DIV8, 22, 2, -1, -1,
+						BUS_MSTOP(2, BIT(2))),
+	DEF_MOD("adg_ssi3_clk",			CLK_PLLCLN_DIV8, 22, 3, -1, -1,
+						BUS_MSTOP(2, BIT(2))),
+	DEF_MOD("adg_ssi4_clk",			CLK_PLLCLN_DIV8, 22, 4, -1, -1,
+						BUS_MSTOP(2, BIT(2))),
+	DEF_MOD("adg_ssi5_clk",			CLK_PLLCLN_DIV8, 22, 5, -1, -1,
+						BUS_MSTOP(2, BIT(2))),
+	DEF_MOD("adg_ssi6_clk",			CLK_PLLCLN_DIV8, 22, 6, -1, -1,
+						BUS_MSTOP(2, BIT(2))),
+	DEF_MOD("adg_ssi7_clk",			CLK_PLLCLN_DIV8, 22, 7, -1, -1,
+						BUS_MSTOP(2, BIT(2))),
+	DEF_MOD("adg_ssi8_clk",			CLK_PLLCLN_DIV8, 22, 8, -1, -1,
+						BUS_MSTOP(2, BIT(2))),
+	DEF_MOD("adg_ssi9_clk",			CLK_PLLCLN_DIV8, 22, 9, -1, -1,
+						BUS_MSTOP(2, BIT(2))),
+	DEF_MOD("dvc0_clk",			CLK_PLLCLN_DIV8, 23, 0, -1, -1,
+						BUS_MSTOP(2, BIT(0) | BIT(1))),
+	DEF_MOD("dvc1_clk",			CLK_PLLCLN_DIV8, 23, 1, -1, -1,
+						BUS_MSTOP(2, BIT(0) | BIT(1))),
+	DEF_MOD("ctu0_mix0_clk",		CLK_PLLCLN_DIV8, 23, 2, -1, -1,
+						BUS_MSTOP(2, BIT(0) | BIT(1))),
+	DEF_MOD("ctu1_mix1_clk",		CLK_PLLCLN_DIV8, 23, 3, -1, -1,
+						BUS_MSTOP(2, BIT(0) | BIT(1))),
+	DEF_MOD("src0_clk",			CLK_PLLCLN_DIV8, 23, 4, -1, -1,
+						BUS_MSTOP(2, BIT(0) | BIT(1))),
+	DEF_MOD("src1_clk",			CLK_PLLCLN_DIV8, 23, 5, -1, -1,
+						BUS_MSTOP(2, BIT(0) | BIT(1))),
+	DEF_MOD("src2_clk",			CLK_PLLCLN_DIV8, 23, 6, -1, -1,
+						BUS_MSTOP(2, BIT(0) | BIT(1))),
+	DEF_MOD("src3_clk",			CLK_PLLCLN_DIV8, 23, 7, -1, -1,
+						BUS_MSTOP(2, BIT(0) | BIT(1))),
+	DEF_MOD("src4_clk",			CLK_PLLCLN_DIV8, 23, 8, -1, -1,
+						BUS_MSTOP(2, BIT(0) | BIT(1))),
+	DEF_MOD("src5_clk",			CLK_PLLCLN_DIV8, 23, 9, -1, -1,
+						BUS_MSTOP(2, BIT(0) | BIT(1))),
+	DEF_MOD("src6_clk",			CLK_PLLCLN_DIV8, 23, 10, -1, -1,
+						BUS_MSTOP(2, BIT(0) | BIT(1))),
+	DEF_MOD("src7_clk",			CLK_PLLCLN_DIV8, 23, 11, -1, -1,
+						BUS_MSTOP(2, BIT(0) | BIT(1))),
+	DEF_MOD("src8_clk",			CLK_PLLCLN_DIV8, 23, 12, -1, -1,
+						BUS_MSTOP(2, BIT(0) | BIT(1))),
+	DEF_MOD("src9_clk",			CLK_PLLCLN_DIV8, 23, 13, -1, -1,
+						BUS_MSTOP(2, BIT(0) | BIT(1))),
+	DEF_MOD("scu_supply_clk",		CLK_PLLCLN_DIV8, 23, 14, -1, -1,
+						BUS_MSTOP(2, BIT(0) | BIT(1))),
+	DEF_MOD("ssiu_supply_clk",		CLK_PLLCLN_DIV8, 24, 0, -1, -1,
+						BUS_MSTOP(2, BIT(3) | BIT(4))),
+	DEF_MOD("ssi0_clk",			CLK_PLLCLN_DIV8, 24, 1, -1, -1,
+						BUS_MSTOP(2, BIT(3) | BIT(4))),
+	DEF_MOD("ssi1_clk",			CLK_PLLCLN_DIV8, 24, 2, -1, -1,
+						BUS_MSTOP(2, BIT(3) | BIT(4))),
+	DEF_MOD("ssi2_clk",			CLK_PLLCLN_DIV8, 24, 3, -1, -1,
+						BUS_MSTOP(2, BIT(3) | BIT(4))),
+	DEF_MOD("ssi3_clk",			CLK_PLLCLN_DIV8, 24, 4, -1, -1,
+						BUS_MSTOP(2, BIT(3) | BIT(4))),
+	DEF_MOD("ssi4_clk",			CLK_PLLCLN_DIV8, 24, 5, -1, -1,
+						BUS_MSTOP(2, BIT(3) | BIT(4))),
+	DEF_MOD("ssi5_clk",			CLK_PLLCLN_DIV8, 24, 6, -1, -1,
+						BUS_MSTOP(2, BIT(3) | BIT(4))),
+	DEF_MOD("ssi6_clk",			CLK_PLLCLN_DIV8, 24, 7, -1, -1,
+						BUS_MSTOP(2, BIT(3) | BIT(4))),
+	DEF_MOD("ssi7_clk",			CLK_PLLCLN_DIV8, 24, 8, -1, -1,
+						BUS_MSTOP(2, BIT(3) | BIT(4))),
+	DEF_MOD("ssi8_clk",			CLK_PLLCLN_DIV8, 24, 9, -1, -1,
+						BUS_MSTOP(2, BIT(3) | BIT(4))),
+	DEF_MOD("ssi9_clk",			CLK_PLLCLN_DIV8, 24, 10, -1, -1,
+						BUS_MSTOP(2, BIT(3) | BIT(4))),
 	DEF_MOD("dsi_0_vclk2",			CLK_SMUX2_DSI1_CLK, 25, 0, 10, 21,
 						BUS_MSTOP(9, BIT(15) | BIT(14))),
+	DEF_MOD("lvds_top_clk_ch0",		CLK_PLLDSI0, 26, 0, 10, 22,
+						BUS_MSTOP(13, BIT(0))),
+	DEF_MOD("lvds_top_clk_ch1",		CLK_PLLDSI1, 26, 1, 10, 23,
+						BUS_MSTOP(13, BIT(0))),
+	DEF_MOD("lvds_top_clk_dot_ch0",		CLK_SMUX2_DSI0_CLK, 26, 2, 10, 24,
+						BUS_MSTOP(13, BIT(0))),
+	DEF_MOD("lvds_top_clk_dot_ch1",		CLK_SMUX2_DSI1_CLK, 26, 3, 10, 25,
+						BUS_MSTOP(13, BIT(0))),
+	DEF_MOD("lvds_top_pclk",		CLK_PLLDTY_DIV16, 26, 4, 10, 26,
+						BUS_MSTOP(13, BIT(0))),
 	DEF_MOD("lcdc_1_clk_a",			CLK_PLLDTY_ACPU_DIV2, 26, 8, 10, 30,
 						BUS_MSTOP(13, BIT(5) | BIT(4) | BIT(3))),
 	DEF_MOD("lcdc_1_clk_p",			CLK_PLLDTY_DIV16, 26, 9, 10, 31,
@@ -621,7 +727,22 @@ static const struct rzv2h_reset r9a09g047_resets[] __initconst = {
 	DEF_RST(13, 13, 6, 14),		/* GE3D_RESETN */
 	DEF_RST(13, 14, 6, 15),		/* GE3D_AXI_RESETN */
 	DEF_RST(13, 15, 6, 16),		/* GE3D_ACE_RESETN */
+	DEF_RST(14, 1, 6, 18),		/* SSIF_0_ASYNC_RESET_SSI */
+	DEF_RST(14, 2, 6, 19),		/* SSIF_0_SYNC_RESET_SSI0 */
+	DEF_RST(14, 3, 6, 20),		/* SSIF_0_SYNC_RESET_SSI1 */
+	DEF_RST(14, 4, 6, 21),		/* SSIF_0_SYNC_RESET_SSI2 */
+	DEF_RST(14, 5, 6, 22),		/* SSIF_0_SYNC_RESET_SSI3 */
+	DEF_RST(14, 6, 6, 23),		/* SSIF_0_SYNC_RESET_SSI4 */
+	DEF_RST(14, 7, 6, 24),		/* SSIF_0_SYNC_RESET_SSI5 */
+	DEF_RST(14, 8, 6, 25),		/* SSIF_0_SYNC_RESET_SSI6 */
+	DEF_RST(14, 9, 6, 26),		/* SSIF_0_SYNC_RESET_SSI7 */
+	DEF_RST(14, 10, 6, 27),		/* SSIF_0_SYNC_RESET_SSI8 */
+	DEF_RST(14, 11, 6, 28),		/* SSIF_0_SYNC_RESET_SSI9 */
+	DEF_RST(14, 12, 6, 29),		/* SCU_0_RESET_SRU */
+	DEF_RST(14, 13, 6, 30),		/* DMACpp_0_ARST */
+	DEF_RST(14, 14, 6, 31),		/* ADG_0_RST_RESET_ADG */
 	DEF_RST(15, 8, 7, 9),		/* TSU_1_PRESETN */
+	DEF_RST(17, 10, 8, 11),		/* LVDS_TOP_RESET_N */
 	DEF_RST(17, 14, 8, 15),		/* LCDC_1_RESET_N */
 };
 
diff --git a/drivers/clk/renesas/r9a09g077-cpg.c b/drivers/clk/renesas/r9a09g077-cpg.c
index f777601a23b93..5640c2035e5a4 100644
--- a/drivers/clk/renesas/r9a09g077-cpg.c
+++ b/drivers/clk/renesas/r9a09g077-cpg.c
@@ -8,16 +8,23 @@
 
 #include <linux/bitfield.h>
 #include <linux/clk-provider.h>
+#include <linux/clk/renesas.h>
 #include <linux/device.h>
 #include <linux/init.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
 #include <linux/kernel.h>
 #include <linux/math.h>
+#include <linux/module.h>
 #include <linux/types.h>
+#include <linux/units.h>
 
 #include <dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h>
 #include <dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h>
 #include "renesas-cpg-mssr.h"
 
+MODULE_IMPORT_NS("RZV2H_CPG");
+
 #define RZT2H_REG_BLOCK_SHIFT	11
 #define RZT2H_REG_OFFSET_MASK	GENMASK(10, 0)
 #define RZT2H_REG_CONF(block, offset)	(((block) << RZT2H_REG_BLOCK_SHIFT) | \
@@ -66,11 +73,26 @@
 #define DIVSCI2ASYNC	CONF_PACK(SCKCR3, 10, 2)
 #define DIVSCI3ASYNC	CONF_PACK(SCKCR3, 12, 2)
 #define DIVSCI4ASYNC	CONF_PACK(SCKCR3, 14, 2)
+#define LCDCDIVSEL	CONF_PACK(SCKCR3, 20, 4)
+
+#define PLL3EN		FIELD_PREP_CONST(OFFSET_MASK, (0xc0))
+
+#define CPG_PLL_EN_EN		BIT(0)
+#define CPG_PLL3_VCO_CTR0(x)	((x) + 0x4)
+#define CPG_PLL3_VCO_CTR0_PDIV	GENMASK(21, 16)
+#define CPG_PLL3_VCO_CTR0_MDIV	GENMASK(9, 0)
+#define CPG_PLL3_VCO_CTR1(x)	((x) + 0x8)
+#define CPG_PLL3_VCO_CTR1_KDIV	GENMASK(31, 16)
+#define CPG_PLL3_VCO_CTR1_SDIV	GENMASK(2, 0)
+#define CPG_PLL_MON(x)		((x) - 0x10)
+#define CPG_PLL_MON_LOCK	BIT(0)
 
 enum rzt2h_clk_types {
 	CLK_TYPE_RZT2H_DIV = CLK_TYPE_CUSTOM,	/* Clock with divider */
 	CLK_TYPE_RZT2H_MUX,			/* Clock with clock source selector */
 	CLK_TYPE_RZT2H_FSELXSPI,		/* Clock with FSELXSPIn source selector */
+	CLK_TYPE_RZT2H_PLL3,			/* PLL3 Clock */
+	CLK_TYPE_RZT2H_LCDCDIV,			/* LCDC divider clock */
 };
 
 #define DEF_DIV(_name, _id, _parent, _conf, _dtable) \
@@ -83,10 +105,51 @@ enum rzt2h_clk_types {
 #define DEF_DIV_FSELXSPI(_name, _id, _parent, _conf, _dtable) \
 	DEF_TYPE(_name, _id, CLK_TYPE_RZT2H_FSELXSPI, .conf = _conf, \
 		 .parent = _parent, .dtable = _dtable, .flag = 0)
+#define DEF_PLL3(_name, _id, _parent, _conf) \
+	DEF_TYPE(_name, _id, CLK_TYPE_RZT2H_PLL3, .conf = _conf, \
+		 .parent = _parent)
+#define DEF_DIV_LCDC(_name, _id, _parent, _conf, _dtable) \
+	DEF_TYPE(_name, _id, CLK_TYPE_RZT2H_LCDCDIV, .conf = _conf, \
+		 .parent = _parent, .dtable = _dtable, .flag = CLK_SET_RATE_PARENT)
+
+struct pll_clk {
+	void __iomem *reg;
+	const struct rzv2h_pll_limits *limits;
+	struct device *dev;
+	struct rzv2h_pll_pars pll_parameters;
+	struct clk_hw hw;
+	unsigned long cur_rate;
+};
+
+#define to_pll(_hw)	container_of(_hw, struct pll_clk, hw)
+
+struct r9a09g077_lcdc_div_clk {
+	const struct clk_div_table *dtable;
+	void __iomem *reg;
+	struct device *dev;
+	struct clk_hw hw;
+	u32 conf;
+	u8 divider;
+};
+
+#define to_lcdc_div_clk(_hw) \
+	container_of(_hw, struct r9a09g077_lcdc_div_clk, hw)
+
+#define RZT2H_MAX_LCDC_DIV_TABLES	16
+
+static const struct rzv2h_pll_limits r9a09g077_cpg_pll3_limits = {
+	.input_fref = 48 * MEGA,
+	.fout = { .min = 25 * MEGA, .max = 430 * MEGA },
+	.fvco = { .min = 1600 * MEGA, .max = 3200 * MEGA },
+	.m = { .min = 0x40, .max = 0x3ff },
+	.p = { .min = 0x2, .max = 0x8 },
+	.s = { .min = 0x0, .max = 0x6 },
+	.k = { .min = -32768, .max = 32767 },
+};
 
 enum clk_ids {
 	/* Core Clock Outputs exported to DT */
-	LAST_DT_CORE_CLK = R9A09G077_PCLKCAN,
+	LAST_DT_CORE_CLK = R9A09G077_PCLKRTC,
 
 	/* External Input Clocks */
 	CLK_EXTAL,
@@ -96,10 +159,12 @@ enum clk_ids {
 	CLK_PLL0,
 	CLK_PLL1,
 	CLK_PLL2,
+	CLK_PLL3,
 	CLK_PLL4,
 	CLK_SEL_CLK_PLL0,
 	CLK_SEL_CLK_PLL1,
 	CLK_SEL_CLK_PLL2,
+	CLK_SEL_CLK_PLL3,
 	CLK_SEL_CLK_PLL4,
 	CLK_PLL4D1,
 	CLK_PLL4D1_DIV3,
@@ -107,6 +172,7 @@ enum clk_ids {
 	CLK_PLL4D3,
 	CLK_PLL4D3_DIV10,
 	CLK_PLL4D3_DIV20,
+	CLK_PLL4D50,
 	CLK_SCI0ASYNC,
 	CLK_SCI1ASYNC,
 	CLK_SCI2ASYNC,
@@ -119,6 +185,7 @@ enum clk_ids {
 	CLK_SPI3ASYNC,
 	CLK_DIVSELXSPI0_SCKCR,
 	CLK_DIVSELXSPI1_SCKCR,
+	CLK_LCDDIVSEL,
 
 	/* Module Clocks */
 	MOD_CLK_BASE,
@@ -130,6 +197,26 @@ static const struct clk_div_table dtable_1_2[] = {
 	{0, 0},
 };
 
+static const struct clk_div_table dtable_2_32[] = {
+	{0, 2},
+	{1, 4},
+	{2, 6},
+	{3, 8},
+	{4, 10},
+	{5, 12},
+	{6, 14},
+	{7, 16},
+	{8, 18},
+	{9, 20},
+	{10, 22},
+	{11, 24},
+	{12, 26},
+	{13, 28},
+	{14, 30},
+	{15, 32},
+	{0, 0},
+};
+
 static const struct clk_div_table dtable_6_8_16_32_64[] = {
 	{6, 64},
 	{5, 32},
@@ -152,6 +239,7 @@ static const struct clk_div_table dtable_24_25_30_32[] = {
 static const char * const sel_clk_pll0[] = { ".loco", ".pll0" };
 static const char * const sel_clk_pll1[] = { ".loco", ".pll1" };
 static const char * const sel_clk_pll2[] = { ".loco", ".pll2" };
+static const char * const sel_clk_pll3[] = { ".loco", ".pll3" };
 static const char * const sel_clk_pll4[] = { ".loco", ".pll4" };
 static const char * const sel_clk_pll4d1_div3_div4[] = { ".pll4d1_div3", ".pll4d1_div4" };
 static const char * const sel_clk_pll4d3_div10_div20[] = { ".pll4d3_div10", ".pll4d3_div20" };
@@ -173,10 +261,14 @@ static const struct cpg_core_clk r9a09g077_core_clks[] __initconst = {
 		sel_clk_pll1, ARRAY_SIZE(sel_clk_pll1), CLK_MUX_READ_ONLY),
 	DEF_MUX(".sel_clk_pll2", CLK_SEL_CLK_PLL2, SEL_PLL,
 		sel_clk_pll2, ARRAY_SIZE(sel_clk_pll2), CLK_MUX_READ_ONLY),
+	DEF_MUX(".sel_clk_pll3", CLK_SEL_CLK_PLL3, SEL_PLL,
+		sel_clk_pll3, ARRAY_SIZE(sel_clk_pll3), CLK_MUX_READ_ONLY),
 	DEF_MUX(".sel_clk_pll4", CLK_SEL_CLK_PLL4, SEL_PLL,
 		sel_clk_pll4, ARRAY_SIZE(sel_clk_pll4), CLK_MUX_READ_ONLY),
 
 	DEF_FIXED(".pll4d1", CLK_PLL4D1, CLK_SEL_CLK_PLL4, 1, 1),
+	DEF_FIXED(".pll4d50", CLK_PLL4D50, CLK_SEL_CLK_PLL4, 50, 1),
+	DEF_PLL3(".pll3", CLK_PLL3, CLK_PLL4D50, PLL3EN),
 	DEF_FIXED(".pll4d1_div3", CLK_PLL4D1_DIV3, CLK_PLL4D1, 3, 1),
 	DEF_FIXED(".pll4d1_div4", CLK_PLL4D1_DIV4, CLK_PLL4D1, 4, 1),
 	DEF_FIXED(".pll4d3", CLK_PLL4D3, CLK_SEL_CLK_PLL4, 3, 1),
@@ -229,6 +321,7 @@ static const struct cpg_core_clk r9a09g077_core_clks[] __initconst = {
 	DEF_FIXED("PCLKL", R9A09G077_CLK_PCLKL, CLK_SEL_CLK_PLL1, 16, 1),
 	DEF_FIXED("PCLKAH", R9A09G077_CLK_PCLKAH, CLK_PLL4D1, 6, 1),
 	DEF_FIXED("PCLKAM", R9A09G077_CLK_PCLKAM, CLK_PLL4D1, 12, 1),
+	DEF_FIXED("PCLKAL", R9A09G077_CLK_PCLKAL, CLK_PLL4D1, 24, 1),
 	DEF_FIXED("SDHI_CLKHS", R9A09G077_SDHI_CLKHS, CLK_SEL_CLK_PLL2, 1, 1),
 	DEF_FIXED("USB_CLK", R9A09G077_USB_CLK, CLK_PLL4D1, 48, 1),
 	DEF_FIXED("ETCLKA", R9A09G077_ETCLKA, CLK_SEL_CLK_PLL1, 5, 1),
@@ -242,6 +335,9 @@ static const struct cpg_core_clk r9a09g077_core_clks[] __initconst = {
 			 FSELXSPI1, dtable_6_8_16_32_64),
 	DEF_MUX("PCLKCAN", R9A09G077_PCLKCAN, FSELCANFD,
 		sel_clk_pll4d3_div10_div20, ARRAY_SIZE(sel_clk_pll4d3_div10_div20), 0),
+	DEF_DIV_LCDC("LCDC_CLKD", R9A09G077_LCDC_CLKD, CLK_SEL_CLK_PLL3, LCDCDIVSEL,
+		     dtable_2_32),
+	DEF_FIXED("PCLKRTC", R9A09G077_PCLKRTC, CLK_EXTAL, 128, 1),
 };
 
 static const struct mssr_mod_clk r9a09g077_mod_clks[] __initconst = {
@@ -272,6 +368,8 @@ static const struct mssr_mod_clk r9a09g077_mod_clks[] __initconst = {
 	DEF_MOD("sci5fck", 600, CLK_SCI5ASYNC),
 	DEF_MOD("iic2", 601, R9A09G077_CLK_PCLKL),
 	DEF_MOD("spi3", 602, CLK_SPI3ASYNC),
+	DEF_MOD("rtc", 605, R9A09G077_CLK_PCLKL),
+	DEF_MOD("lcdc", 1204, R9A09G077_CLK_PCLKAL),
 	DEF_MOD("sdhi0", 1212, R9A09G077_CLK_PCLKAM),
 	DEF_MOD("sdhi1", 1213, R9A09G077_CLK_PCLKAM),
 };
@@ -481,6 +579,276 @@ r9a09g077_cpg_fselxspi_div_clk_register(struct device *dev,
 	return hw->clk;
 }
 
+static unsigned long r9a09g077_cpg_pll3_clk_recalc_rate(struct clk_hw *hw,
+							unsigned long parent_rate)
+{
+	struct pll_clk *pll_clk = to_pll(hw);
+	u32 ctr0, ctr1;
+	u8 pdiv, sdiv;
+	u64 rate;
+	u16 mdiv;
+	s16 kdiv;
+
+	ctr0 = readl(CPG_PLL3_VCO_CTR0(pll_clk->reg));
+	ctr1 = readl(CPG_PLL3_VCO_CTR1(pll_clk->reg));
+
+	pdiv = FIELD_GET(CPG_PLL3_VCO_CTR0_PDIV, ctr0);
+	mdiv = FIELD_GET(CPG_PLL3_VCO_CTR0_MDIV, ctr0);
+	kdiv = (s16)FIELD_GET(CPG_PLL3_VCO_CTR1_KDIV, ctr1);
+	sdiv = FIELD_GET(CPG_PLL3_VCO_CTR1_SDIV, ctr1);
+
+	rate = mul_u64_u32_shr(parent_rate, (mdiv << 16) + kdiv, 16 + sdiv);
+
+	return DIV_ROUND_CLOSEST_ULL(rate, pdiv);
+}
+
+static int r9a09g077_cpg_pll3_determine_rate(struct clk_hw *hw,
+					     struct clk_rate_request *req)
+{
+	struct pll_clk *pll_clk = to_pll(hw);
+	u64 rate_millihz;
+
+	if (req->rate == pll_clk->cur_rate)
+		return 0;
+
+	rate_millihz = mul_u32_u32(req->rate, MILLI);
+	if (!rzv2h_cpg_get_pll_pars(pll_clk->limits, &pll_clk->pll_parameters,
+				    rate_millihz)) {
+		dev_dbg(pll_clk->dev,
+			"failed to determine rate for req->rate: %lu\n",
+			req->rate);
+		return -EINVAL;
+	}
+	req->rate = DIV_ROUND_CLOSEST_ULL(pll_clk->pll_parameters.freq_millihz, MILLI);
+	pll_clk->cur_rate = req->rate;
+
+	return 0;
+}
+
+static int r9a09g077_cpg_pll3_set_rate(struct clk_hw *hw, unsigned long rate,
+				       unsigned long parent_rate)
+{
+	struct pll_clk *pll_clk = to_pll(hw);
+	struct rzv2h_pll_pars *params = &pll_clk->pll_parameters;
+	void __iomem *offset = pll_clk->reg;
+	u32 val;
+	int ret;
+
+	/* Put PLL into standby mode */
+	writel(0, offset);
+	ret = readl_poll_timeout_atomic(CPG_PLL_MON(offset),
+					val, !(val & CPG_PLL_MON_LOCK),
+					100, 2000);
+	if (ret) {
+		dev_err(pll_clk->dev, "Failed to put PLL into standby mode");
+		return ret;
+	}
+
+	/* Output clock setting 1 */
+	val = readl(CPG_PLL3_VCO_CTR0(offset));
+	FIELD_MODIFY(CPG_PLL3_VCO_CTR0_MDIV, &val, params->m);
+	FIELD_MODIFY(CPG_PLL3_VCO_CTR0_PDIV, &val, params->p);
+	writel(val, CPG_PLL3_VCO_CTR0(offset));
+
+	/* Output clock setting 2 */
+	val = readl(CPG_PLL3_VCO_CTR1(offset));
+	FIELD_MODIFY(CPG_PLL3_VCO_CTR1_KDIV, &val, params->k);
+	FIELD_MODIFY(CPG_PLL3_VCO_CTR1_SDIV, &val, params->s);
+	writel(val, CPG_PLL3_VCO_CTR1(offset));
+
+	writel(CPG_PLL_EN_EN, offset);
+
+	/* PLL normal mode transition, output clock stability check */
+	ret = readl_poll_timeout_atomic(CPG_PLL_MON(offset),
+					val, (val & CPG_PLL_MON_LOCK),
+					100, 2000);
+	if (ret) {
+		writel(0, offset);
+		dev_err(pll_clk->dev, "Failed to put PLL into normal mode");
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct clk_ops r9a09g077_cpg_pll3_ops = {
+	.recalc_rate = r9a09g077_cpg_pll3_clk_recalc_rate,
+	.determine_rate = r9a09g077_cpg_pll3_determine_rate,
+	.set_rate = r9a09g077_cpg_pll3_set_rate,
+};
+
+static struct clk * __init
+r9a09g077_cpg_pll3_clk_register(struct device *dev,
+				const struct cpg_core_clk *core,
+				void __iomem *addr,
+				struct cpg_mssr_pub *pub,
+				const struct rzv2h_pll_limits *limits)
+{
+	struct clk_init_data init = {};
+	const struct clk *parent;
+	const char *parent_name;
+	struct pll_clk *pll_clk;
+	int ret;
+
+	parent = pub->clks[core->parent];
+	if (IS_ERR(parent))
+		return ERR_CAST(parent);
+
+	pll_clk = devm_kzalloc(dev, sizeof(*pll_clk), GFP_KERNEL);
+	if (!pll_clk)
+		return ERR_PTR(-ENOMEM);
+
+	parent_name = __clk_get_name(parent);
+	init.name = core->name;
+	init.ops = &r9a09g077_cpg_pll3_ops;
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+
+	pll_clk->dev = dev;
+	pll_clk->hw.init = &init;
+	pll_clk->reg = addr;
+	pll_clk->limits = limits;
+
+	ret = devm_clk_hw_register(dev, &pll_clk->hw);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return pll_clk->hw.clk;
+}
+
+static int r9a09g077_cpg_lcdc_div_determine_rate(struct clk_hw *hw,
+						 struct clk_rate_request *req)
+{
+	struct r9a09g077_lcdc_div_clk *dsi_div = to_lcdc_div_clk(hw);
+	struct clk_hw *mux_hw = clk_hw_get_parent(hw);
+	u8 table[RZT2H_MAX_LCDC_DIV_TABLES] = { 0 };
+	struct rzv2h_pll_div_pars dsi_params;
+	const struct clk_div_table *div;
+	struct pll_clk *pll_clk;
+	unsigned int i = 0;
+	u64 freq_millihz;
+
+	/* index 1 is always .pll3 in sel_clk_pll3[] */
+	pll_clk = to_pll(clk_hw_get_parent_by_index(mux_hw, 1));
+
+	for (div = dsi_div->dtable; div->div; div++) {
+		if (i >= RZT2H_MAX_LCDC_DIV_TABLES)
+			return -EINVAL;
+		table[i++] = div->div;
+	}
+
+	freq_millihz = mul_u32_u32(req->rate, MILLI);
+
+	if (!rzv2h_cpg_get_pll_divs_pars(pll_clk->limits, &dsi_params, table,
+					 i, freq_millihz)) {
+		dev_err(dsi_div->dev,
+			"LCDC divider failed to determine rate for req->rate: %lu\n",
+			req->rate);
+		return -EINVAL;
+	}
+
+	req->rate = DIV_ROUND_CLOSEST_ULL(dsi_params.div.freq_millihz, MILLI);
+	req->best_parent_rate = req->rate * dsi_params.div.divider_value;
+	dsi_div->divider = dsi_params.div.divider_value;
+	pll_clk->cur_rate = req->best_parent_rate;
+	pll_clk->pll_parameters = dsi_params.pll;
+
+	return 0;
+}
+
+static int r9a09g077_cpg_lcdc_div_set_rate(struct clk_hw *hw,
+					   unsigned long rate,
+					   unsigned long parent_rate)
+{
+	struct r9a09g077_lcdc_div_clk *dsi_div = to_lcdc_div_clk(hw);
+	const struct clk_div_table *clkt;
+	bool divider_found = false;
+	u32 val, shift;
+
+	for (clkt = dsi_div->dtable; clkt->div; clkt++) {
+		if (clkt->div == dsi_div->divider) {
+			divider_found = true;
+			break;
+		}
+	}
+
+	if (!divider_found)
+		return -EINVAL;
+
+	shift = GET_SHIFT(dsi_div->conf);
+	val = readl(dsi_div->reg);
+	val &= ~(clk_div_mask(GET_WIDTH(dsi_div->conf)) << shift);
+	val |= clkt->val << shift;
+	writel(val, dsi_div->reg);
+
+	return 0;
+}
+
+static unsigned long
+r9a09g077_cpg_lcdc_div_recalc_rate(struct clk_hw *hw,
+				   unsigned long parent_rate)
+{
+	struct r9a09g077_lcdc_div_clk *dsi_div = to_lcdc_div_clk(hw);
+	u32 div;
+
+	div = readl(dsi_div->reg);
+	div >>= GET_SHIFT(dsi_div->conf);
+	div &= clk_div_mask(GET_WIDTH(dsi_div->conf));
+	div = dsi_div->dtable[div].div;
+
+	return DIV_ROUND_CLOSEST_ULL(parent_rate, div);
+}
+
+static const struct clk_ops r9a09g077_cpg_lcdc_div_ops = {
+	.recalc_rate = r9a09g077_cpg_lcdc_div_recalc_rate,
+	.determine_rate = r9a09g077_cpg_lcdc_div_determine_rate,
+	.set_rate = r9a09g077_cpg_lcdc_div_set_rate,
+};
+
+static struct clk * __init
+r9a09g077_cpg_lcdc_div_clk_register(struct device *dev,
+				    const struct cpg_core_clk *core,
+				    void __iomem *addr,
+				    struct cpg_mssr_pub *pub)
+{
+	struct r9a09g077_lcdc_div_clk *clk_hw_data;
+	struct clk_init_data init = {};
+	struct clk **clks = pub->clks;
+	const struct clk *parent;
+	const char *parent_name;
+	struct clk_hw *hw;
+	int ret;
+
+	parent = clks[core->parent];
+	if (IS_ERR(parent))
+		return ERR_CAST(parent);
+
+	clk_hw_data = devm_kzalloc(dev, sizeof(*clk_hw_data), GFP_KERNEL);
+	if (!clk_hw_data)
+		return ERR_PTR(-ENOMEM);
+
+	clk_hw_data->dtable = core->dtable;
+	clk_hw_data->reg = addr;
+	clk_hw_data->conf = core->conf;
+	clk_hw_data->dev = dev;
+	clk_hw_data->divider = 32; /* Initialize divider for LCDC */
+
+	parent_name = __clk_get_name(parent);
+	init.name = core->name;
+	init.ops = &r9a09g077_cpg_lcdc_div_ops;
+	init.flags = core->flag;
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+
+	hw = &clk_hw_data->hw;
+	hw->init = &init;
+	ret = devm_clk_hw_register(dev, hw);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return hw->clk;
+}
+
 static struct clk * __init
 r9a09g077_cpg_clk_register(struct device *dev, const struct cpg_core_clk *core,
 			   const struct cpg_mssr_info *info,
@@ -497,6 +865,11 @@ r9a09g077_cpg_clk_register(struct device *dev, const struct cpg_core_clk *core,
 		return r9a09g077_cpg_mux_clk_register(dev, core, addr, pub);
 	case CLK_TYPE_RZT2H_FSELXSPI:
 		return r9a09g077_cpg_fselxspi_div_clk_register(dev, core, addr, pub);
+	case CLK_TYPE_RZT2H_PLL3:
+		return r9a09g077_cpg_pll3_clk_register(dev, core, pub->base1 + offset,
+						       pub, &r9a09g077_cpg_pll3_limits);
+	case CLK_TYPE_RZT2H_LCDCDIV:
+		return r9a09g077_cpg_lcdc_div_clk_register(dev, core, addr, pub);
 	default:
 		return ERR_PTR(-EINVAL);
 	}
diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c
index b6dab92cb2205..80f4403ea2baf 100644
--- a/drivers/clk/renesas/renesas-cpg-mssr.c
+++ b/drivers/clk/renesas/renesas-cpg-mssr.c
@@ -252,6 +252,22 @@ static void cpg_rzt2h_mstp_write(struct cpg_mssr_priv *priv, u16 offset, u32 val
 	writel(value, base + RZT2H_MSTPCR_OFFSET(offset));
 }
 
+static void cpg_rzt2h_mstp_delay(u32 idx, bool bit_valid)
+{
+	unsigned int mask = bit_valid ? GENMASK(31, 0) : GENMASK(31, 5);
+
+	if (idx == (MOD_CLK_PACK(1204) & mask)) {
+		/* LCDC needs 100 dummy reads, or 142us */
+		udelay(142);
+	} else if (idx == (MOD_CLK_PACK(605) & mask)) {
+		/* RTC needs 300 dummy reads, or 428us */
+		udelay(428);
+	} else {
+		/* default 7 dummy reads, or 10us */
+		udelay(10);
+	}
+}
+
 static int cpg_mstp_clock_endisable(struct clk_hw *hw, bool enable)
 {
 	struct mstp_clock *clock = to_mstp_clock(hw);
@@ -311,7 +327,7 @@ static int cpg_mstp_clock_endisable(struct clk_hw *hw, bool enable)
 		 * register, we simply add a delay after the read operation.
 		 */
 		cpg_rzt2h_mstp_read(priv, priv->control_regs[reg]);
-		udelay(10);
+		cpg_rzt2h_mstp_delay(clock->index, true);
 		return 0;
 	}
 
@@ -1141,7 +1157,7 @@ static int cpg_mssr_resume_noirq(struct device *dev)
 			cpg_rzt2h_mstp_write(priv, priv->control_regs[reg], newval);
 			/* See cpg_mstp_clock_endisable() on why this is necessary. */
 			cpg_rzt2h_mstp_read(priv, priv->control_regs[reg]);
-			udelay(10);
+			cpg_rzt2h_mstp_delay(reg * 32, false);
 			continue;
 		} else
 			writel(newval, priv->pub.base0 + priv->control_regs[reg]);
diff --git a/drivers/clk/renesas/rzv2h-cpg-lib.c b/drivers/clk/renesas/rzv2h-cpg-lib.c
new file mode 100644
index 0000000000000..124239c7327e4
--- /dev/null
+++ b/drivers/clk/renesas/rzv2h-cpg-lib.c
@@ -0,0 +1,217 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * RZV2H CPG Library. This library provides common functions to calculate
+ * PLL parameters for the RZV2H SoC.
+ *
+ * Copyright (C) 2026 Renesas Electronics Corp.
+ *
+ */
+
+#include <linux/clk/renesas.h>
+#include <linux/export.h>
+#include <linux/math.h>
+#include <linux/types.h>
+#include <linux/units.h>
+
+/**
+ * rzv2h_cpg_get_pll_pars - Finds the best combination of PLL parameters
+ * for a given frequency.
+ *
+ * @limits: Pointer to the structure containing the limits for the PLL parameters
+ * @pars: Pointer to the structure where the best calculated PLL parameters values
+ * will be stored
+ * @freq_millihz: Target output frequency in millihertz
+ *
+ * This function calculates the best set of PLL parameters (M, K, P, S) to achieve
+ * the desired frequency.
+ * There is no direct formula to calculate the PLL parameters, as it's an open
+ * system of equations, therefore this function uses an iterative approach to
+ * determine the best solution. The best solution is one that minimizes the error
+ * (desired frequency - actual frequency).
+ *
+ * Return: true if a valid set of parameters values is found, false otherwise.
+ */
+bool rzv2h_cpg_get_pll_pars(const struct rzv2h_pll_limits *limits,
+			    struct rzv2h_pll_pars *pars, u64 freq_millihz)
+{
+	unsigned long input_fref = limits->input_fref ?: (24 * MEGA);
+	u64 fout_min_millihz = mul_u32_u32(limits->fout.min, MILLI);
+	u64 fout_max_millihz = mul_u32_u32(limits->fout.max, MILLI);
+	struct rzv2h_pll_pars p, best;
+
+	if (freq_millihz > fout_max_millihz ||
+	    freq_millihz < fout_min_millihz)
+		return false;
+
+	/* Initialize best error to maximum possible value */
+	best.error_millihz = S64_MAX;
+
+	for (p.p = limits->p.min; p.p <= limits->p.max; p.p++) {
+		u32 fref = input_fref / p.p;
+		u16 divider;
+
+		for (divider = 1 << limits->s.min, p.s = limits->s.min;
+			p.s <= limits->s.max; p.s++, divider <<= 1) {
+			for (p.m = limits->m.min; p.m <= limits->m.max; p.m++) {
+				u64 output_m, output_k_range;
+				s64 pll_k, output_k;
+				u64 fvco, output;
+
+				/*
+				 * The frequency generated by the PLL + divider
+				 * is calculated as follows:
+				 *
+				 * With:
+				 * Freq = Ffout = Ffvco / 2^(pll_s)
+				 * Ffvco = (pll_m + (pll_k / 65536)) * Ffref
+				 * Ffref = 24MHz / pll_p
+				 *
+				 * Freq can also be rewritten as:
+				 * Freq = Ffvco / 2^(pll_s)
+				 *      = ((pll_m + (pll_k / 65536)) * Ffref) / 2^(pll_s)
+				 *      = (pll_m * Ffref) / 2^(pll_s) + ((pll_k / 65536) * Ffref) / 2^(pll_s)
+				 *      = output_m + output_k
+				 *
+				 * Every parameter has been determined at this
+				 * point, but pll_k.
+				 *
+				 * Considering that:
+				 * limits->k.min <= pll_k <= limits->k.max
+				 * Then:
+				 * -0.5 <= (pll_k / 65536) < 0.5
+				 * Therefore:
+				 * -Ffref / (2 * 2^(pll_s)) <= output_k < Ffref / (2 * 2^(pll_s))
+				 */
+
+				/* Compute output M component (in mHz) */
+				output_m = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(p.m, fref) * MILLI,
+								 divider);
+				/* Compute range for output K (in mHz) */
+				output_k_range = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(fref, MILLI),
+								       2 * divider);
+				/*
+				 * No point in continuing if we can't achieve
+				 * the desired frequency
+				 */
+				if (freq_millihz <  (output_m - output_k_range) ||
+				    freq_millihz >= (output_m + output_k_range)) {
+					continue;
+				}
+
+				/*
+				 * Compute the K component
+				 *
+				 * Since:
+				 * Freq = output_m + output_k
+				 * Then:
+				 * output_k = Freq - output_m
+				 *          = ((pll_k / 65536) * Ffref) / 2^(pll_s)
+				 * Therefore:
+				 * pll_k = (output_k * 65536 * 2^(pll_s)) / Ffref
+				 */
+				output_k = freq_millihz - output_m;
+				pll_k = div_s64(output_k * 65536ULL * divider,
+						fref);
+				pll_k = DIV_S64_ROUND_CLOSEST(pll_k, MILLI);
+
+				/* Validate K value within allowed limits */
+				if (pll_k < limits->k.min ||
+				    pll_k > limits->k.max)
+					continue;
+
+				p.k = pll_k;
+
+				/* Compute (Ffvco * 65536) */
+				fvco = mul_u32_u32(p.m * 65536 + p.k, fref);
+				if (fvco < mul_u32_u32(limits->fvco.min, 65536) ||
+				    fvco > mul_u32_u32(limits->fvco.max, 65536))
+					continue;
+
+				/* PLL_M component of (output * 65536 * PLL_P) */
+				output = mul_u32_u32(p.m * 65536, input_fref);
+				/* PLL_K component of (output * 65536 * PLL_P) */
+				output += p.k * input_fref;
+				/* Make it in mHz */
+				output *= MILLI;
+				output = DIV_U64_ROUND_CLOSEST(output, 65536 * p.p * divider);
+
+				/* Check output frequency against limits */
+				if (output < fout_min_millihz ||
+				    output > fout_max_millihz)
+					continue;
+
+				p.error_millihz = freq_millihz - output;
+				p.freq_millihz = output;
+
+				/* If an exact match is found, return immediately */
+				if (p.error_millihz == 0) {
+					*pars = p;
+					return true;
+				}
+
+				/* Update best match if error is smaller */
+				if (abs(best.error_millihz) > abs(p.error_millihz))
+					best = p;
+			}
+		}
+	}
+
+	/* If no valid parameters were found, return false */
+	if (best.error_millihz == S64_MAX)
+		return false;
+
+	*pars = best;
+	return true;
+}
+EXPORT_SYMBOL_NS_GPL(rzv2h_cpg_get_pll_pars, "RZV2H_CPG");
+
+/*
+ * rzv2h_cpg_get_pll_divs_pars - Finds the best combination of PLL parameters
+ * and divider value for a given frequency.
+ *
+ * @limits: Pointer to the structure containing the limits for the PLL parameters
+ * @pars: Pointer to the structure where the best calculated PLL parameters and
+ * divider values will be stored
+ * @table: Pointer to the array of valid divider values
+ * @table_size: Size of the divider values array
+ * @freq_millihz: Target output frequency in millihertz
+ *
+ * This function calculates the best set of PLL parameters (M, K, P, S) and divider
+ * value to achieve the desired frequency. See rzv2h_cpg_get_pll_pars() for more
+ * details on how the PLL parameters are calculated.
+ *
+ * freq_millihz is the desired frequency generated by the PLL followed by a
+ * a gear.
+ */
+bool rzv2h_cpg_get_pll_divs_pars(const struct rzv2h_pll_limits *limits,
+				 struct rzv2h_pll_div_pars *pars,
+				 const u8 *table, u8 table_size, u64 freq_millihz)
+{
+	struct rzv2h_pll_div_pars p, best;
+
+	best.div.error_millihz = S64_MAX;
+	p.div.error_millihz = S64_MAX;
+	for (unsigned int i = 0; i < table_size; i++) {
+		if (!rzv2h_cpg_get_pll_pars(limits, &p.pll, freq_millihz * table[i]))
+			continue;
+
+		p.div.divider_value = table[i];
+		p.div.freq_millihz = DIV_U64_ROUND_CLOSEST(p.pll.freq_millihz, table[i]);
+		p.div.error_millihz = freq_millihz - p.div.freq_millihz;
+
+		if (p.div.error_millihz == 0) {
+			*pars = p;
+			return true;
+		}
+
+		if (abs(best.div.error_millihz) > abs(p.div.error_millihz))
+			best = p;
+	}
+
+	if (best.div.error_millihz == S64_MAX)
+		return false;
+
+	*pars = best;
+	return true;
+}
+EXPORT_SYMBOL_NS_GPL(rzv2h_cpg_get_pll_divs_pars, "RZV2H_CPG");
diff --git a/drivers/clk/renesas/rzv2h-cpg.c b/drivers/clk/renesas/rzv2h-cpg.c
index 5bdfdc415bb6a..917fc737f254b 100644
--- a/drivers/clk/renesas/rzv2h-cpg.c
+++ b/drivers/clk/renesas/rzv2h-cpg.c
@@ -217,211 +217,8 @@ struct rzv2h_plldsi_div_clk {
 #define to_plldsi_div_clk(_hw) \
 	container_of(_hw, struct rzv2h_plldsi_div_clk, hw)
 
-#define RZ_V2H_OSC_CLK_IN_MEGA		(24 * MEGA)
 #define RZV2H_MAX_DIV_TABLES		(16)
 
-/**
- * rzv2h_get_pll_pars - Finds the best combination of PLL parameters
- * for a given frequency.
- *
- * @limits: Pointer to the structure containing the limits for the PLL parameters
- * @pars: Pointer to the structure where the best calculated PLL parameters values
- * will be stored
- * @freq_millihz: Target output frequency in millihertz
- *
- * This function calculates the best set of PLL parameters (M, K, P, S) to achieve
- * the desired frequency.
- * There is no direct formula to calculate the PLL parameters, as it's an open
- * system of equations, therefore this function uses an iterative approach to
- * determine the best solution. The best solution is one that minimizes the error
- * (desired frequency - actual frequency).
- *
- * Return: true if a valid set of parameters values is found, false otherwise.
- */
-bool rzv2h_get_pll_pars(const struct rzv2h_pll_limits *limits,
-			struct rzv2h_pll_pars *pars, u64 freq_millihz)
-{
-	u64 fout_min_millihz = mul_u32_u32(limits->fout.min, MILLI);
-	u64 fout_max_millihz = mul_u32_u32(limits->fout.max, MILLI);
-	struct rzv2h_pll_pars p, best;
-
-	if (freq_millihz > fout_max_millihz ||
-	    freq_millihz < fout_min_millihz)
-		return false;
-
-	/* Initialize best error to maximum possible value */
-	best.error_millihz = S64_MAX;
-
-	for (p.p = limits->p.min; p.p <= limits->p.max; p.p++) {
-		u32 fref = RZ_V2H_OSC_CLK_IN_MEGA / p.p;
-		u16 divider;
-
-		for (divider = 1 << limits->s.min, p.s = limits->s.min;
-			p.s <= limits->s.max; p.s++, divider <<= 1) {
-			for (p.m = limits->m.min; p.m <= limits->m.max; p.m++) {
-				u64 output_m, output_k_range;
-				s64 pll_k, output_k;
-				u64 fvco, output;
-
-				/*
-				 * The frequency generated by the PLL + divider
-				 * is calculated as follows:
-				 *
-				 * With:
-				 * Freq = Ffout = Ffvco / 2^(pll_s)
-				 * Ffvco = (pll_m + (pll_k / 65536)) * Ffref
-				 * Ffref = 24MHz / pll_p
-				 *
-				 * Freq can also be rewritten as:
-				 * Freq = Ffvco / 2^(pll_s)
-				 *      = ((pll_m + (pll_k / 65536)) * Ffref) / 2^(pll_s)
-				 *      = (pll_m * Ffref) / 2^(pll_s) + ((pll_k / 65536) * Ffref) / 2^(pll_s)
-				 *      = output_m + output_k
-				 *
-				 * Every parameter has been determined at this
-				 * point, but pll_k.
-				 *
-				 * Considering that:
-				 * limits->k.min <= pll_k <= limits->k.max
-				 * Then:
-				 * -0.5 <= (pll_k / 65536) < 0.5
-				 * Therefore:
-				 * -Ffref / (2 * 2^(pll_s)) <= output_k < Ffref / (2 * 2^(pll_s))
-				 */
-
-				/* Compute output M component (in mHz) */
-				output_m = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(p.m, fref) * MILLI,
-								 divider);
-				/* Compute range for output K (in mHz) */
-				output_k_range = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(fref, MILLI),
-								       2 * divider);
-				/*
-				 * No point in continuing if we can't achieve
-				 * the desired frequency
-				 */
-				if (freq_millihz <  (output_m - output_k_range) ||
-				    freq_millihz >= (output_m + output_k_range)) {
-					continue;
-				}
-
-				/*
-				 * Compute the K component
-				 *
-				 * Since:
-				 * Freq = output_m + output_k
-				 * Then:
-				 * output_k = Freq - output_m
-				 *          = ((pll_k / 65536) * Ffref) / 2^(pll_s)
-				 * Therefore:
-				 * pll_k = (output_k * 65536 * 2^(pll_s)) / Ffref
-				 */
-				output_k = freq_millihz - output_m;
-				pll_k = div_s64(output_k * 65536ULL * divider,
-						fref);
-				pll_k = DIV_S64_ROUND_CLOSEST(pll_k, MILLI);
-
-				/* Validate K value within allowed limits */
-				if (pll_k < limits->k.min ||
-				    pll_k > limits->k.max)
-					continue;
-
-				p.k = pll_k;
-
-				/* Compute (Ffvco * 65536) */
-				fvco = mul_u32_u32(p.m * 65536 + p.k, fref);
-				if (fvco < mul_u32_u32(limits->fvco.min, 65536) ||
-				    fvco > mul_u32_u32(limits->fvco.max, 65536))
-					continue;
-
-				/* PLL_M component of (output * 65536 * PLL_P) */
-				output = mul_u32_u32(p.m * 65536, RZ_V2H_OSC_CLK_IN_MEGA);
-				/* PLL_K component of (output * 65536 * PLL_P) */
-				output += p.k * RZ_V2H_OSC_CLK_IN_MEGA;
-				/* Make it in mHz */
-				output *= MILLI;
-				output = DIV_U64_ROUND_CLOSEST(output, 65536 * p.p * divider);
-
-				/* Check output frequency against limits */
-				if (output < fout_min_millihz ||
-				    output > fout_max_millihz)
-					continue;
-
-				p.error_millihz = freq_millihz - output;
-				p.freq_millihz = output;
-
-				/* If an exact match is found, return immediately */
-				if (p.error_millihz == 0) {
-					*pars = p;
-					return true;
-				}
-
-				/* Update best match if error is smaller */
-				if (abs(best.error_millihz) > abs(p.error_millihz))
-					best = p;
-			}
-		}
-	}
-
-	/* If no valid parameters were found, return false */
-	if (best.error_millihz == S64_MAX)
-		return false;
-
-	*pars = best;
-	return true;
-}
-EXPORT_SYMBOL_NS_GPL(rzv2h_get_pll_pars, "RZV2H_CPG");
-
-/*
- * rzv2h_get_pll_divs_pars - Finds the best combination of PLL parameters
- * and divider value for a given frequency.
- *
- * @limits: Pointer to the structure containing the limits for the PLL parameters
- * @pars: Pointer to the structure where the best calculated PLL parameters and
- * divider values will be stored
- * @table: Pointer to the array of valid divider values
- * @table_size: Size of the divider values array
- * @freq_millihz: Target output frequency in millihertz
- *
- * This function calculates the best set of PLL parameters (M, K, P, S) and divider
- * value to achieve the desired frequency. See rzv2h_get_pll_pars() for more details
- * on how the PLL parameters are calculated.
- *
- * freq_millihz is the desired frequency generated by the PLL followed by a
- * a gear.
- */
-bool rzv2h_get_pll_divs_pars(const struct rzv2h_pll_limits *limits,
-			     struct rzv2h_pll_div_pars *pars,
-			     const u8 *table, u8 table_size, u64 freq_millihz)
-{
-	struct rzv2h_pll_div_pars p, best;
-
-	best.div.error_millihz = S64_MAX;
-	p.div.error_millihz = S64_MAX;
-	for (unsigned int i = 0; i < table_size; i++) {
-		if (!rzv2h_get_pll_pars(limits, &p.pll, freq_millihz * table[i]))
-			continue;
-
-		p.div.divider_value = table[i];
-		p.div.freq_millihz = DIV_U64_ROUND_CLOSEST(p.pll.freq_millihz, table[i]);
-		p.div.error_millihz = freq_millihz - p.div.freq_millihz;
-
-		if (p.div.error_millihz == 0) {
-			*pars = p;
-			return true;
-		}
-
-		if (abs(best.div.error_millihz) > abs(p.div.error_millihz))
-			best = p;
-	}
-
-	if (best.div.error_millihz == S64_MAX)
-		return false;
-
-	*pars = best;
-	return true;
-}
-EXPORT_SYMBOL_NS_GPL(rzv2h_get_pll_divs_pars, "RZV2H_CPG");
-
 /**
  * struct rzv2h_plldsi_mux_clk - PLL DSI MUX clock
  *
diff --git a/drivers/clk/rockchip/clk-rk3588.c b/drivers/clk/rockchip/clk-rk3588.c
index 2ba9976654cf1..75d42fea2a11d 100644
--- a/drivers/clk/rockchip/clk-rk3588.c
+++ b/drivers/clk/rockchip/clk-rk3588.c
@@ -895,7 +895,7 @@ static struct rockchip_clk_branch rk3588_early_clk_branches[] __initdata = {
 	MUX(I2S2_2CH_MCLKOUT, "i2s2_2ch_mclkout", i2s2_2ch_mclkout_p, CLK_SET_RATE_PARENT,
 			RK3588_CLKSEL_CON(30), 2, 1, MFLAGS),
 	GATE_GRF(I2S2_2CH_MCLKOUT_TO_IO, "i2s2_2ch_mclkout_to_io", "i2s2_2ch_mclkout",
-			0, RK3588_SYSGRF_SOC_CON6, 2, GFLAGS, grf_type_sys),
+			CLK_IGNORE_UNUSED, RK3588_SYSGRF_SOC_CON6, 2, GFLAGS, grf_type_sys),
 
 	COMPOSITE(CLK_I2S3_2CH_SRC, "clk_i2s3_2ch_src", gpll_aupll_p, 0,
 			RK3588_CLKSEL_CON(30), 8, 1, MFLAGS, 3, 5, DFLAGS,
@@ -912,7 +912,7 @@ static struct rockchip_clk_branch rk3588_early_clk_branches[] __initdata = {
 	MUX(I2S3_2CH_MCLKOUT, "i2s3_2ch_mclkout", i2s3_2ch_mclkout_p, CLK_SET_RATE_PARENT,
 			RK3588_CLKSEL_CON(32), 2, 1, MFLAGS),
 	GATE_GRF(I2S3_2CH_MCLKOUT_TO_IO, "i2s3_2ch_mclkout_to_io", "i2s3_2ch_mclkout",
-			0, RK3588_SYSGRF_SOC_CON6, 7, GFLAGS, grf_type_sys),
+			CLK_IGNORE_UNUSED, RK3588_SYSGRF_SOC_CON6, 7, GFLAGS, grf_type_sys),
 	GATE(PCLK_ACDCDIG, "pclk_acdcdig", "pclk_audio_root", 0,
 			RK3588_CLKGATE_CON(7), 11, GFLAGS),
 	GATE(HCLK_I2S0_8CH, "hclk_i2s0_8ch", "hclk_audio_root", 0,
@@ -942,7 +942,7 @@ static struct rockchip_clk_branch rk3588_early_clk_branches[] __initdata = {
 	MUX(I2S0_8CH_MCLKOUT, "i2s0_8ch_mclkout", i2s0_8ch_mclkout_p, CLK_SET_RATE_PARENT,
 			RK3588_CLKSEL_CON(28), 2, 2, MFLAGS),
 	GATE_GRF(I2S0_8CH_MCLKOUT_TO_IO, "i2s0_8ch_mclkout_to_io", "i2s0_8ch_mclkout",
-			0, RK3588_SYSGRF_SOC_CON6, 0, GFLAGS, grf_type_sys),
+			CLK_IGNORE_UNUSED, RK3588_SYSGRF_SOC_CON6, 0, GFLAGS, grf_type_sys),
 
 	GATE(HCLK_PDM1, "hclk_pdm1", "hclk_audio_root", 0,
 			RK3588_CLKGATE_CON(9), 6, GFLAGS),
@@ -2102,7 +2102,7 @@ static struct rockchip_clk_branch rk3588_early_clk_branches[] __initdata = {
 	COMPOSITE(DCLK_VOP1_SRC, "dclk_vop1_src", gpll_cpll_v0pll_aupll_p, 0,
 			RK3588_CLKSEL_CON(111), 14, 2, MFLAGS, 9, 5, DFLAGS,
 			RK3588_CLKGATE_CON(52), 11, GFLAGS),
-	COMPOSITE(DCLK_VOP2_SRC, "dclk_vop2_src", gpll_cpll_v0pll_aupll_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT,
+	COMPOSITE(DCLK_VOP2_SRC, "dclk_vop2_src", gpll_cpll_v0pll_aupll_p, 0,
 			RK3588_CLKSEL_CON(112), 5, 2, MFLAGS, 0, 5, DFLAGS,
 			RK3588_CLKGATE_CON(52), 12, GFLAGS),
 	COMPOSITE_NODIV(DCLK_VOP0, "dclk_vop0", dclk_vop0_p,
@@ -2229,7 +2229,7 @@ static struct rockchip_clk_branch rk3588_early_clk_branches[] __initdata = {
 	MUX(I2S1_8CH_MCLKOUT, "i2s1_8ch_mclkout", i2s1_8ch_mclkout_p, CLK_SET_RATE_PARENT,
 			RK3588_PMU_CLKSEL_CON(9), 2, 2, MFLAGS),
 	GATE_GRF(I2S1_8CH_MCLKOUT_TO_IO, "i2s1_8ch_mclkout_to_io", "i2s1_8ch_mclkout",
-			0, RK3588_SYSGRF_SOC_CON6, 1, GFLAGS, grf_type_sys),
+			CLK_IGNORE_UNUSED, RK3588_SYSGRF_SOC_CON6, 1, GFLAGS, grf_type_sys),
 	GATE(PCLK_PMU1, "pclk_pmu1", "pclk_pmu0_root", CLK_IS_CRITICAL,
 			RK3588_PMU_CLKGATE_CON(1), 0, GFLAGS),
 	GATE(CLK_DDR_FAIL_SAFE, "clk_ddr_fail_safe", "clk_pmu0", CLK_IGNORE_UNUSED,
diff --git a/drivers/clk/samsung/clk-exynos990.c b/drivers/clk/samsung/clk-exynos990.c
index 7a27557e6713b..44c068bc1dc8d 100644
--- a/drivers/clk/samsung/clk-exynos990.c
+++ b/drivers/clk/samsung/clk-exynos990.c
@@ -20,7 +20,7 @@
 #define CLKS_NR_HSI0 (CLK_GOUT_HSI0_LHS_ACEL_D_HSI0_CLK + 1)
 #define CLKS_NR_PERIC0 (CLK_GOUT_PERIC0_SYSREG_PCLK + 1)
 #define CLKS_NR_PERIC1 (CLK_GOUT_PERIC1_XIU_P_ACLK + 1)
-#define CLKS_NR_PERIS (CLK_GOUT_PERIS_OTP_CON_TOP_OSCCLK + 1)
+#define CLKS_NR_PERIS (CLK_GOUT_PERIS_TMU_SUB_PCLK + 1)
 
 /* ---- CMU_TOP ------------------------------------------------------------- */
 
@@ -2550,7 +2550,7 @@ static const unsigned long peris_clk_regs[] __initconst = {
 
 /* Parent clock list for CMU_PERIS muxes */
 PNAME(mout_peris_bus_user_p)		= { "oscclk", "mout_cmu_peris_bus" };
-PNAME(mout_peris_clk_peris_gic_p)	= { "oscclk", "mout_peris_bus_user" };
+PNAME(mout_peris_clk_peris_gic_p)	= { "mout_peris_bus_user", "oscclk" };
 
 static const struct samsung_mux_clock peris_mux_clks[] __initconst = {
 	MUX(CLK_MOUT_PERIS_BUS_USER, "mout_peris_bus_user",
@@ -2583,15 +2583,15 @@ static const struct samsung_gate_clock peris_gate_clks[] __initconst = {
 	     CLK_CON_GAT_GOUT_BLK_PERIS_UID_RSTNSYNC_CLK_PERIS_BUSP_IPCLKPORT_CLK,
 	     21, 0, 0),
 	GATE(CLK_GOUT_PERIS_CLK_PERIS_OSCCLK_CLK,
-	     "gout_peris_clk_peris_oscclk_clk", "mout_peris_bus_user",
+	     "gout_peris_clk_peris_oscclk_clk", "oscclk",
 	     CLK_CON_GAT_CLK_BLK_PERIS_UID_RSTNSYNC_CLK_PERIS_OSCCLK_IPCLKPORT_CLK,
 	     21, 0, 0),
 	GATE(CLK_GOUT_PERIS_CLK_PERIS_GIC_CLK,
-	     "gout_peris_clk_peris_gic_clk", "mout_peris_bus_user",
+	     "gout_peris_clk_peris_gic_clk", "mout_peris_clk_peris_gic",
 	     CLK_CON_GAT_GOUT_BLK_PERIS_UID_RSTNSYNC_CLK_PERIS_GIC_IPCLKPORT_CLK,
 	     21, 0, 0),
 	GATE(CLK_GOUT_PERIS_AD_AXI_P_PERIS_ACLKM,
-	     "gout_peris_ad_axi_p_peris_aclkm", "mout_peris_bus_user",
+	     "gout_peris_ad_axi_p_peris_aclkm", "mout_peris_clk_peris_gic",
 	     CLK_CON_GAT_GOUT_BLK_PERIS_UID_AD_AXI_P_PERIS_IPCLKPORT_ACLKM,
 	     21, CLK_IGNORE_UNUSED, 0),
 	GATE(CLK_GOUT_PERIS_OTP_CON_BIRA_PCLK,
@@ -2599,27 +2599,31 @@ static const struct samsung_gate_clock peris_gate_clks[] __initconst = {
 	     CLK_CON_GAT_GOUT_BLK_PERIS_UID_OTP_CON_BIRA_IPCLKPORT_PCLK,
 	     21, 0, 0),
 	GATE(CLK_GOUT_PERIS_GIC_CLK,
-	     "gout_peris_gic_clk", "mout_peris_bus_user",
+	     "gout_peris_gic_clk", "mout_peris_clk_peris_gic",
 	     CLK_CON_GAT_GOUT_BLK_PERIS_UID_GIC_IPCLKPORT_CLK,
 	     21, CLK_IS_CRITICAL, 0),
 	GATE(CLK_GOUT_PERIS_LHM_AXI_P_PERIS_CLK,
-	     "gout_peris_lhm_axi_p_peris_clk", "oscclk",
+	     "gout_peris_lhm_axi_p_peris_clk", "mout_peris_bus_user",
 	     CLK_CON_GAT_GOUT_BLK_PERIS_UID_LHM_AXI_P_PERIS_IPCLKPORT_I_CLK,
 	     21, CLK_IGNORE_UNUSED, 0),
 	GATE(CLK_GOUT_PERIS_MCT_PCLK,
-	     "gout_peris_mct_pclk", "mout_peris_clk_peris_gic",
+	     "gout_peris_mct_pclk", "mout_peris_bus_user",
 	     CLK_CON_GAT_GOUT_BLK_PERIS_UID_MCT_IPCLKPORT_PCLK,
 	     21, 0, 0),
 	GATE(CLK_GOUT_PERIS_OTP_CON_TOP_PCLK,
-	     "gout_peris_otp_con_top_pclk", "mout_peris_clk_peris_gic",
+	     "gout_peris_otp_con_top_pclk", "mout_peris_bus_user",
 	     CLK_CON_GAT_GOUT_BLK_PERIS_UID_OTP_CON_TOP_IPCLKPORT_PCLK,
 	     21, 0, 0),
 	GATE(CLK_GOUT_PERIS_D_TZPC_PERIS_PCLK,
 	     "gout_peris_d_tzpc_peris_pclk", "mout_peris_bus_user",
 	     CLK_CON_GAT_GOUT_BLK_PERIS_UID_D_TZPC_PERIS_IPCLKPORT_PCLK,
 	     21, 0, 0),
+	GATE(CLK_GOUT_PERIS_TMU_SUB_PCLK,
+	     "gout_peris_tmu_sub_pclk", "mout_peris_bus_user",
+	     CLK_CON_GAT_GOUT_BLK_PERIS_UID_TMU_SUB_IPCLKPORT_PCLK,
+	     21, 0, 0),
 	GATE(CLK_GOUT_PERIS_TMU_TOP_PCLK,
-	     "gout_peris_tmu_top_pclk", "mout_peris_clk_peris_gic",
+	     "gout_peris_tmu_top_pclk", "mout_peris_bus_user",
 	     CLK_CON_GAT_GOUT_BLK_PERIS_UID_TMU_TOP_IPCLKPORT_PCLK,
 	     21, 0, 0),
 	GATE(CLK_GOUT_PERIS_OTP_CON_BIRA_OSCCLK,
diff --git a/drivers/clk/spacemit/ccu-k3.c b/drivers/clk/spacemit/ccu-k3.c
index cb0c4277f72a8..aca61ea760194 100644
--- a/drivers/clk/spacemit/ccu-k3.c
+++ b/drivers/clk/spacemit/ccu-k3.c
@@ -777,7 +777,7 @@ static const struct clk_parent_data sdh2_parents[] = {
 CCU_MUX_DIV_GATE_FC_DEFINE(sdh2_clk, sdh2_parents, APMU_SDH2_CLK_RES_CTRL, 8, 3,
 			   BIT(11), 5, 3, BIT(4), 0);
 
-CCU_GATE_DEFINE(usb2_bus_clk, CCU_PARENT_HW(axi_clk), APMU_USB_CLK_RES_CTRL, BIT(0), 0);
+CCU_GATE_DEFINE(usb2_bus_clk, CCU_PARENT_HW(axi_clk), APMU_USB_CLK_RES_CTRL, BIT(1), 0);
 CCU_GATE_DEFINE(usb3_porta_bus_clk, CCU_PARENT_HW(axi_clk), APMU_USB_CLK_RES_CTRL, BIT(4), 0);
 CCU_GATE_DEFINE(usb3_portb_bus_clk, CCU_PARENT_HW(axi_clk), APMU_USB_CLK_RES_CTRL, BIT(8), 0);
 CCU_GATE_DEFINE(usb3_portc_bus_clk, CCU_PARENT_HW(axi_clk), APMU_USB_CLK_RES_CTRL, BIT(12), 0);
@@ -926,15 +926,16 @@ CCU_MUX_DIV_GATE_FC_DEFINE(dpu_aclk, dpu_aclk_parents, APMU_LCD_CLK_RES_CTRL5, 1
 			   20, 3, BIT(16), 0);
 
 static const struct clk_parent_data ufs_aclk_parents[] = {
-	CCU_PARENT_HW(pll1_d6_409p6),
 	CCU_PARENT_HW(pll1_d5_491p52),
-	CCU_PARENT_HW(pll1_d4_614p4),
-	CCU_PARENT_HW(pll1_d8_307p2),
-	CCU_PARENT_HW(pll2_d4),
+	CCU_PARENT_HW(pll1_d6_409p6),
+	CCU_PARENT_HW(pll2_d6),
+	CCU_PARENT_HW(pll2_d5),
 };
 CCU_MUX_DIV_GATE_FC_DEFINE(ufs_aclk, ufs_aclk_parents, APMU_UFS_CLK_RES_CTRL, 5, 3, BIT(8),
 			   2, 3, BIT(1), 0);
 
+CCU_FACTOR_DEFINE(ufs_refclk, CCU_PARENT_HW(pll1_d64_38p4), 2, 1);
+
 static const struct clk_parent_data edp0_pclk_parents[] = {
 	CCU_PARENT_HW(lcd_pxclk),
 	CCU_PARENT_NAME(external_clk),
@@ -1026,7 +1027,7 @@ CCU_MUX_DIV_GATE_DEFINE(isim_vclk_out3, isim_vclk_parents, APMU_SNR_ISIM_VCLK_CT
 /* APMU clocks end */
 
 /* DCIU clocks start */
-CCU_GATE_DEFINE(hdma_clk, CCU_PARENT_HW(axi_clk), DCIU_DMASYS_CLK_EN, BIT(0), 0);
+CCU_GATE_DEFINE(hdma_clk, CCU_PARENT_HW(axi_clk), DCIU_DMASYS_CLK_EN, BIT(0), CLK_IS_CRITICAL);
 CCU_GATE_DEFINE(dma350_clk, CCU_PARENT_HW(axi_clk), DCIU_DMASYS_SDMA_CLK_EN, BIT(0), 0);
 CCU_GATE_DEFINE(c2_tcm_pipe_clk, CCU_PARENT_HW(axi_clk), DCIU_C2_TCM_PIPE_CLK, BIT(0), 0);
 CCU_GATE_DEFINE(c3_tcm_pipe_clk, CCU_PARENT_HW(axi_clk), DCIU_C3_TCM_PIPE_CLK, BIT(0), 0);
@@ -1392,6 +1393,7 @@ static struct clk_hw *k3_ccu_apmu_hws[] = {
 	[CLK_APMU_DSI4LN2_DPU_ACLK]	= &dsi4ln2_dpu_aclk.common.hw,
 	[CLK_APMU_DPU_ACLK]		= &dpu_aclk.common.hw,
 	[CLK_APMU_UFS_ACLK]		= &ufs_aclk.common.hw,
+	[CLK_APMU_UFS_REFCLK]		= &ufs_refclk.common.hw,
 	[CLK_APMU_EDP0_PXCLK]		= &edp0_pxclk.common.hw,
 	[CLK_APMU_EDP1_PXCLK]		= &edp1_pxclk.common.hw,
 	[CLK_APMU_PCIE_PORTA_MSTE]	= &pciea_mstr_clk.common.hw,
diff --git a/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c b/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c
index f6bfeba009e8e..b24c8b196e664 100644
--- a/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c
+++ b/drivers/clk/sunxi-ng/ccu-sun6i-rtc.c
@@ -52,8 +52,6 @@ struct sun6i_rtc_match_data {
 	u8				osc32k_fanout_nparents;
 };
 
-static bool have_iosc_calibration;
-
 static int ccu_iosc_enable(struct clk_hw *hw)
 {
 	struct ccu_common *cm = hw_to_ccu_common(hw);
@@ -80,7 +78,7 @@ static unsigned long ccu_iosc_recalc_rate(struct clk_hw *hw,
 {
 	struct ccu_common *cm = hw_to_ccu_common(hw);
 
-	if (have_iosc_calibration) {
+	if (cm->features & CCU_FEATURE_IOSC_CALIBRATION) {
 		u32 reg = readl(cm->base + IOSC_CLK_CALI_REG);
 
 		/*
@@ -119,7 +117,7 @@ static int ccu_iosc_32k_prepare(struct clk_hw *hw)
 	struct ccu_common *cm = hw_to_ccu_common(hw);
 	u32 val;
 
-	if (!have_iosc_calibration)
+	if (!(cm->features & CCU_FEATURE_IOSC_CALIBRATION))
 		return 0;
 
 	val = readl(cm->base + IOSC_CLK_CALI_REG);
@@ -134,7 +132,7 @@ static void ccu_iosc_32k_unprepare(struct clk_hw *hw)
 	struct ccu_common *cm = hw_to_ccu_common(hw);
 	u32 val;
 
-	if (!have_iosc_calibration)
+	if (!(cm->features & CCU_FEATURE_IOSC_CALIBRATION))
 		return;
 
 	val = readl(cm->base + IOSC_CLK_CALI_REG);
@@ -148,7 +146,7 @@ static unsigned long ccu_iosc_32k_recalc_rate(struct clk_hw *hw,
 	struct ccu_common *cm = hw_to_ccu_common(hw);
 	u32 val;
 
-	if (have_iosc_calibration) {
+	if (cm->features & CCU_FEATURE_IOSC_CALIBRATION) {
 		val = readl(cm->base + IOSC_CLK_CALI_REG);
 
 		/* Assume the calibrated 32k clock is accurate. */
@@ -167,7 +165,7 @@ static unsigned long ccu_iosc_32k_recalc_accuracy(struct clk_hw *hw,
 	struct ccu_common *cm = hw_to_ccu_common(hw);
 	u32 val;
 
-	if (have_iosc_calibration) {
+	if (cm->features & CCU_FEATURE_IOSC_CALIBRATION) {
 		val = readl(cm->base + IOSC_CLK_CALI_REG);
 
 		/* Assume the calibrated 32k clock is accurate. */
@@ -191,10 +189,8 @@ static struct ccu_common iosc_32k_clk = {
 					 CLK_GET_RATE_NOCACHE),
 };
 
-static const struct clk_hw *ext_osc32k[] = { NULL }; /* updated during probe */
-
-static SUNXI_CCU_GATE_HWS(ext_osc32k_gate_clk, "ext-osc32k-gate",
-			  ext_osc32k, 0x0, BIT(4), 0);
+static SUNXI_CCU_GATE_FW(ext_osc32k_gate_clk, "ext-osc32k-gate",
+			  "ext-osc32k", 0x0, BIT(4), 0);
 
 static const struct clk_hw *osc32k_parents[] = {
 	&iosc_32k_clk.hw,
@@ -352,7 +348,6 @@ MODULE_DEVICE_TABLE(of, sun6i_rtc_ccu_match);
 int sun6i_rtc_ccu_probe(struct device *dev, void __iomem *reg)
 {
 	const struct sun6i_rtc_match_data *data;
-	struct clk *ext_osc32k_clk = NULL;
 	const struct of_device_id *match;
 
 	/* This driver is only used for newer variants of the hardware. */
@@ -361,23 +356,12 @@ int sun6i_rtc_ccu_probe(struct device *dev, void __iomem *reg)
 		return 0;
 
 	data = match->data;
-	have_iosc_calibration = data->have_iosc_calibration;
-
-	if (data->have_ext_osc32k) {
-		const char *fw_name;
-
-		/* ext-osc32k was the only input clock in the old binding. */
-		fw_name = of_property_present(dev->of_node, "clock-names")
-			? "ext-osc32k" : NULL;
-		ext_osc32k_clk = devm_clk_get_optional(dev, fw_name);
-		if (IS_ERR(ext_osc32k_clk))
-			return PTR_ERR(ext_osc32k_clk);
+	if (data->have_iosc_calibration) {
+		iosc_clk.features |= CCU_FEATURE_IOSC_CALIBRATION;
+		iosc_32k_clk.features |= CCU_FEATURE_IOSC_CALIBRATION;
 	}
 
-	if (ext_osc32k_clk) {
-		/* Link ext-osc32k-gate to its parent. */
-		*ext_osc32k = __clk_get_hw(ext_osc32k_clk);
-	} else {
+	if (!data->have_ext_osc32k) {
 		/* ext-osc32k-gate is an orphan, so do not register it. */
 		sun6i_rtc_ccu_hw_clks.hws[CLK_EXT_OSC32K_GATE] = NULL;
 		osc32k_init_data.num_parents = 1;
diff --git a/drivers/clk/sunxi-ng/ccu_common.c b/drivers/clk/sunxi-ng/ccu_common.c
index 1c083b4d0b7e0..43d8eca6abeed 100644
--- a/drivers/clk/sunxi-ng/ccu_common.c
+++ b/drivers/clk/sunxi-ng/ccu_common.c
@@ -114,7 +114,7 @@ static int sunxi_ccu_probe(struct sunxi_ccu *ccu, struct device *dev,
 			   const struct sunxi_ccu_desc *desc)
 {
 	struct ccu_reset *reset;
-	int i, ret;
+	int i, j, ret;
 
 	ccu->desc = desc;
 
@@ -130,8 +130,8 @@ static int sunxi_ccu_probe(struct sunxi_ccu *ccu, struct device *dev,
 		cclk->lock = &ccu->lock;
 	}
 
-	for (i = 0; i < desc->hw_clks->num ; i++) {
-		struct clk_hw *hw = desc->hw_clks->hws[i];
+	for (j = 0; j < desc->hw_clks->num ; j++) {
+		struct clk_hw *hw = desc->hw_clks->hws[j];
 		const char *name;
 
 		if (!hw)
@@ -143,7 +143,7 @@ static int sunxi_ccu_probe(struct sunxi_ccu *ccu, struct device *dev,
 		else
 			ret = of_clk_hw_register(node, hw);
 		if (ret) {
-			pr_err("Couldn't register clock %d - %s\n", i, name);
+			pr_err("Couldn't register clock %d - %s\n", j, name);
 			goto err_clk_unreg;
 		}
 	}
@@ -186,8 +186,8 @@ static int sunxi_ccu_probe(struct sunxi_ccu *ccu, struct device *dev,
 err_del_provider:
 	of_clk_del_provider(node);
 err_clk_unreg:
-	while (--i >= 0) {
-		struct clk_hw *hw = desc->hw_clks->hws[i];
+	while (--j >= 0) {
+		struct clk_hw *hw = desc->hw_clks->hws[j];
 
 		if (!hw)
 			continue;
diff --git a/drivers/clk/sunxi-ng/ccu_common.h b/drivers/clk/sunxi-ng/ccu_common.h
index bbec283b9d993..d9dc24ad55039 100644
--- a/drivers/clk/sunxi-ng/ccu_common.h
+++ b/drivers/clk/sunxi-ng/ccu_common.h
@@ -21,6 +21,7 @@
 #define CCU_FEATURE_CLOSEST_RATE	BIT(9)
 #define CCU_FEATURE_DUAL_DIV		BIT(10)
 #define CCU_FEATURE_UPDATE_BIT		BIT(11)
+#define CCU_FEATURE_IOSC_CALIBRATION	BIT(12)
 
 /* MMC timing mode switch bit */
 #define CCU_MMC_NEW_TIMING_MODE		BIT(30)
diff --git a/drivers/clk/sunxi-ng/ccu_mux.c b/drivers/clk/sunxi-ng/ccu_mux.c
index 766f27cff748e..09230728c4009 100644
--- a/drivers/clk/sunxi-ng/ccu_mux.c
+++ b/drivers/clk/sunxi-ng/ccu_mux.c
@@ -66,7 +66,6 @@ unsigned long ccu_mux_helper_apply_prediv(struct ccu_common *common,
 {
 	return parent_rate / ccu_mux_get_prediv(common, cm, parent_index);
 }
-EXPORT_SYMBOL_NS_GPL(ccu_mux_helper_apply_prediv, "SUNXI_CCU");
 
 static unsigned long ccu_mux_helper_unapply_prediv(struct ccu_common *common,
 					    struct ccu_mux_internal *cm,
diff --git a/drivers/clk/ux500/clk-prcmu.c b/drivers/clk/ux500/clk-prcmu.c
index ddc86551bf574..ac96c46bd1bbf 100644
--- a/drivers/clk/ux500/clk-prcmu.c
+++ b/drivers/clk/ux500/clk-prcmu.c
@@ -7,7 +7,7 @@
  */
 
 #include <linux/clk-provider.h>
-#include <linux/mfd/dbx500-prcmu.h>
+#include <linux/mfd/db8500-prcmu.h>
 #include <linux/slab.h>
 #include <linux/io.h>
 #include <linux/err.h>
@@ -35,13 +35,13 @@ static int clk_prcmu_prepare(struct clk_hw *hw)
 {
 	struct clk_prcmu *clk = to_clk_prcmu(hw);
 
-	return prcmu_request_clock(clk->cg_sel, true);
+	return db8500_prcmu_request_clock(clk->cg_sel, true);
 }
 
 static void clk_prcmu_unprepare(struct clk_hw *hw)
 {
 	struct clk_prcmu *clk = to_clk_prcmu(hw);
-	if (prcmu_request_clock(clk->cg_sel, false))
+	if (db8500_prcmu_request_clock(clk->cg_sel, false))
 		pr_err("clk_prcmu: %s failed to disable %s.\n", __func__,
 		       clk_hw_get_name(hw));
 }
@@ -86,7 +86,7 @@ static int clk_prcmu_opp_prepare(struct clk_hw *hw)
 		clk->opp_requested = 1;
 	}
 
-	err = prcmu_request_clock(clk->cg_sel, true);
+	err = db8500_prcmu_request_clock(clk->cg_sel, true);
 	if (err) {
 		prcmu_qos_remove_requirement(PRCMU_QOS_APE_OPP,
 					(char *)clk_hw_get_name(hw));
@@ -101,7 +101,7 @@ static void clk_prcmu_opp_unprepare(struct clk_hw *hw)
 {
 	struct clk_prcmu *clk = to_clk_prcmu(hw);
 
-	if (prcmu_request_clock(clk->cg_sel, false)) {
+	if (db8500_prcmu_request_clock(clk->cg_sel, false)) {
 		pr_err("clk_prcmu: %s failed to disable %s.\n", __func__,
 			clk_hw_get_name(hw));
 		return;
@@ -120,7 +120,7 @@ static int clk_prcmu_opp_volt_prepare(struct clk_hw *hw)
 	struct clk_prcmu *clk = to_clk_prcmu(hw);
 
 	if (!clk->opp_requested) {
-		err = prcmu_request_ape_opp_100_voltage(true);
+		err = db8500_prcmu_request_ape_opp_100_voltage(true);
 		if (err) {
 			pr_err("clk_prcmu: %s fail req APE OPP VOLT for %s.\n",
 				__func__, clk_hw_get_name(hw));
@@ -129,9 +129,9 @@ static int clk_prcmu_opp_volt_prepare(struct clk_hw *hw)
 		clk->opp_requested = 1;
 	}
 
-	err = prcmu_request_clock(clk->cg_sel, true);
+	err = db8500_prcmu_request_clock(clk->cg_sel, true);
 	if (err) {
-		prcmu_request_ape_opp_100_voltage(false);
+		db8500_prcmu_request_ape_opp_100_voltage(false);
 		clk->opp_requested = 0;
 		return err;
 	}
@@ -143,14 +143,14 @@ static void clk_prcmu_opp_volt_unprepare(struct clk_hw *hw)
 {
 	struct clk_prcmu *clk = to_clk_prcmu(hw);
 
-	if (prcmu_request_clock(clk->cg_sel, false)) {
+	if (db8500_prcmu_request_clock(clk->cg_sel, false)) {
 		pr_err("clk_prcmu: %s failed to disable %s.\n", __func__,
 			clk_hw_get_name(hw));
 		return;
 	}
 
 	if (clk->opp_requested) {
-		prcmu_request_ape_opp_100_voltage(false);
+		db8500_prcmu_request_ape_opp_100_voltage(false);
 		clk->opp_requested = 0;
 	}
 }
diff --git a/drivers/clk/ux500/u8500_of_clk.c b/drivers/clk/ux500/u8500_of_clk.c
index 6f78808387b10..d2499815226f0 100644
--- a/drivers/clk/ux500/u8500_of_clk.c
+++ b/drivers/clk/ux500/u8500_of_clk.c
@@ -9,7 +9,7 @@
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/clk-provider.h>
-#include <linux/mfd/dbx500-prcmu.h>
+#include <linux/mfd/db8500-prcmu.h>
 
 #include "clk.h"
 #include "prcc.h"
diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c
index 21639d9ac753b..10ea6035f4ad6 100644
--- a/drivers/cpufreq/acpi-cpufreq.c
+++ b/drivers/cpufreq/acpi-cpufreq.c
@@ -246,32 +246,32 @@ static unsigned extract_freq(struct cpufreq_policy *policy, u32 val)
 
 static u32 cpu_freq_read_intel(struct acpi_pct_register *not_used)
 {
-	u32 val, dummy __always_unused;
+	u64 val;
 
-	rdmsr(MSR_IA32_PERF_CTL, val, dummy);
-	return val;
+	rdmsrq(MSR_IA32_PERF_CTL, val);
+	return (u32)val;
 }
 
 static void cpu_freq_write_intel(struct acpi_pct_register *not_used, u32 val)
 {
-	u32 lo, hi;
+	u64 msrval;
 
-	rdmsr(MSR_IA32_PERF_CTL, lo, hi);
-	lo = (lo & ~INTEL_MSR_RANGE) | (val & INTEL_MSR_RANGE);
-	wrmsr(MSR_IA32_PERF_CTL, lo, hi);
+	rdmsrq(MSR_IA32_PERF_CTL, msrval);
+	msrval = (msrval & ~(u64)INTEL_MSR_RANGE) | (val & INTEL_MSR_RANGE);
+	wrmsrq(MSR_IA32_PERF_CTL, msrval);
 }
 
 static u32 cpu_freq_read_amd(struct acpi_pct_register *not_used)
 {
-	u32 val, dummy __always_unused;
+	u64 val;
 
-	rdmsr(MSR_AMD_PERF_CTL, val, dummy);
-	return val;
+	rdmsrq(MSR_AMD_PERF_CTL, val);
+	return (u32)val;
 }
 
 static void cpu_freq_write_amd(struct acpi_pct_register *not_used, u32 val)
 {
-	wrmsr(MSR_AMD_PERF_CTL, val, 0);
+	wrmsrq(MSR_AMD_PERF_CTL, val);
 }
 
 static u32 cpu_freq_read_io(struct acpi_pct_register *reg)
diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c
index a74a4cf99d22e..f8d8288d644f8 100644
--- a/drivers/cpufreq/amd-pstate.c
+++ b/drivers/cpufreq/amd-pstate.c
@@ -782,6 +782,7 @@ static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy,
 static void amd_pstate_adjust_perf(struct cpufreq_policy *policy,
 				   unsigned long _min_perf,
 				   unsigned long target_perf,
+				   unsigned long _max_perf,
 				   unsigned long capacity)
 {
 	u8 max_perf, min_perf, des_perf, cap_perf;
diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c
index 638e5bf72185a..3f64f266e695b 100644
--- a/drivers/cpufreq/apple-soc-cpufreq.c
+++ b/drivers/cpufreq/apple-soc-cpufreq.c
@@ -249,21 +249,19 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy)
 		return -ENODEV;
 	}
 
-	ret = dev_pm_opp_of_add_table(cpu_dev);
-	if (ret < 0) {
-		dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret);
-		return ret;
-	}
+	priv = kzalloc_obj(*priv);
+	if (!priv)
+		return -ENOMEM;
 
 	ret = apple_soc_cpufreq_find_cluster(policy, &reg_base, &info);
 	if (ret) {
 		dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret);
-		return ret;
+		goto out_free_priv;
 	}
 
-	ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);
-	if (ret) {
-		dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret);
+	ret = dev_pm_opp_of_cpumask_add_table(policy->cpus);
+	if (ret < 0) {
+		dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret);
 		goto out_iounmap;
 	}
 
@@ -271,19 +269,13 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy)
 	if (ret <= 0) {
 		dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n");
 		ret = -EPROBE_DEFER;
-		goto out_free_opp;
-	}
-
-	priv = kzalloc_obj(*priv);
-	if (!priv) {
-		ret = -ENOMEM;
-		goto out_free_opp;
+		goto out_free_table;
 	}
 
 	ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);
 	if (ret) {
 		dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
-		goto out_free_priv;
+		goto out_free_table;
 	}
 
 	/* Get OPP levels (p-state indexes) and stash them in driver_data */
@@ -318,12 +310,12 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy)
 
 out_free_cpufreq_table:
 	dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table);
-out_free_priv:
-	kfree(priv);
-out_free_opp:
-	dev_pm_opp_remove_all_dynamic(cpu_dev);
+out_free_table:
+	dev_pm_opp_of_cpumask_remove_table(policy->cpus);
 out_iounmap:
 	iounmap(reg_base);
+out_free_priv:
+	kfree(priv);
 	return ret;
 }
 
@@ -332,7 +324,7 @@ static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy)
 	struct apple_cpu_priv *priv = policy->driver_data;
 
 	dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table);
-	dev_pm_opp_remove_all_dynamic(priv->cpu_dev);
+	dev_pm_opp_of_cpumask_remove_table(policy->cpus);
 	iounmap(priv->reg_base);
 	kfree(priv);
 }
diff --git a/drivers/cpufreq/brcmstb-avs-cpufreq.c b/drivers/cpufreq/brcmstb-avs-cpufreq.c
index 71450cca8e9f4..7d902856ac01a 100644
--- a/drivers/cpufreq/brcmstb-avs-cpufreq.c
+++ b/drivers/cpufreq/brcmstb-avs-cpufreq.c
@@ -584,11 +584,8 @@ static int brcm_avs_prepare_init(struct platform_device *pdev)
 	ret = devm_request_irq(dev, priv->host_irq, irq_handler,
 			       IRQF_TRIGGER_RISING,
 			       BRCM_AVS_HOST_INTR, priv);
-	if (ret && priv->host_irq >= 0) {
-		dev_err(dev, "IRQ request failed: %s (%d) -- %d\n",
-			BRCM_AVS_HOST_INTR, priv->host_irq, ret);
+	if (ret && priv->host_irq >= 0)
 		goto unmap_intr_base;
-	}
 
 	if (brcm_avs_is_firmware_loaded(priv))
 		return 0;
diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c
index ff1204c666b19..371ca1fe90209 100644
--- a/drivers/cpufreq/cpufreq-dt-platdev.c
+++ b/drivers/cpufreq/cpufreq-dt-platdev.c
@@ -81,6 +81,8 @@ static const struct of_device_id allowlist[] __initconst = {
 		{ .have_governor_per_policy = true, },
 	},
 
+	{ .compatible = "spacemit,k1", },
+
 	{ .compatible = "st-ericsson,u8500", },
 	{ .compatible = "st-ericsson,u8540", },
 	{ .compatible = "st-ericsson,u9500", },
@@ -200,6 +202,7 @@ static const struct of_device_id blocklist[] __initconst = {
 	{ .compatible = "ti,am62l3", },
 	{ .compatible = "ti,am62p5", },
 
+	{ .compatible = "qcom,ipq5210", },
 	{ .compatible = "qcom,ipq5332", },
 	{ .compatible = "qcom,ipq5424", },
 	{ .compatible = "qcom,ipq6018", },
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 507224c9ecd38..0d0df986fa3df 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -2225,14 +2225,17 @@ EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch);
  * @policy: cpufreq policy object of the target CPU.
  * @min_perf: Minimum (required) performance level (units of @capacity).
  * @target_perf: Target (desired) performance level (units of @capacity).
+ * @max_perf: Maximum (allowed) performance level (units of @capacity).
  * @capacity: Capacity of the target CPU.
  *
- * Carry out a fast performance level switch of @cpu without sleeping.
+ * Carry out a fast performance level adjustment for the CPU represented by
+ * @policy without sleeping.
  *
  * The driver's ->adjust_perf() callback invoked by this function must be
- * suitable for being called from within RCU-sched read-side critical sections
- * and it is expected to select a suitable performance level equal to or above
- * @min_perf and preferably equal to or below @target_perf.
+ * suitable for calling from within RCU-sched read-side critical sections and
+ * it is expected to program the processor to select suitable performance
+ * levels between @min_perf and @max_perf inclusive and preferably close to
+ * @target_perf going forward for the CPU represented by @policy.
  *
  * This function must not be called if policy->fast_switch_enabled is unset.
  *
@@ -2244,9 +2247,10 @@ EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch);
 void cpufreq_driver_adjust_perf(struct cpufreq_policy *policy,
 				 unsigned long min_perf,
 				 unsigned long target_perf,
+				 unsigned long max_perf,
 				 unsigned long capacity)
 {
-	cpufreq_driver->adjust_perf(policy, min_perf, target_perf, capacity);
+	cpufreq_driver->adjust_perf(policy, min_perf, target_perf, max_perf, capacity);
 }
 
 /**
@@ -2586,6 +2590,9 @@ static void cpufreq_update_pressure(struct cpufreq_policy *policy)
 
 	cpu = cpumask_first(policy->related_cpus);
 	max_freq = arch_scale_freq_ref(cpu);
+	if (!max_freq)
+		max_freq = policy->cpuinfo.max_freq;
+
 	capped_freq = policy->max;
 
 	/*
diff --git a/drivers/cpufreq/e_powersaver.c b/drivers/cpufreq/e_powersaver.c
index eb5a9209d8288..54689ebadeb24 100644
--- a/drivers/cpufreq/e_powersaver.c
+++ b/drivers/cpufreq/e_powersaver.c
@@ -90,7 +90,7 @@ static int eps_acpi_exit(struct cpufreq_policy *policy)
 static unsigned int eps_get(unsigned int cpu)
 {
 	struct eps_cpu_data *centaur;
-	u32 lo, hi;
+	u64 val;
 
 	if (cpu)
 		return 0;
@@ -99,50 +99,50 @@ static unsigned int eps_get(unsigned int cpu)
 		return 0;
 
 	/* Return current frequency */
-	rdmsr(MSR_IA32_PERF_STATUS, lo, hi);
-	return centaur->fsb * ((lo >> 8) & 0xff);
+	rdmsrq(MSR_IA32_PERF_STATUS, val);
+	return centaur->fsb * ((val >> 8) & 0xff);
 }
 
 static int eps_set_state(struct eps_cpu_data *centaur,
 			 struct cpufreq_policy *policy,
 			 u32 dest_state)
 {
-	u32 lo, hi;
+	u64 val;
 	int i;
 
 	/* Wait while CPU is busy */
-	rdmsr(MSR_IA32_PERF_STATUS, lo, hi);
+	rdmsrq(MSR_IA32_PERF_STATUS, val);
 	i = 0;
-	while (lo & ((1 << 16) | (1 << 17))) {
+	while (val & ((1 << 16) | (1 << 17))) {
 		udelay(16);
-		rdmsr(MSR_IA32_PERF_STATUS, lo, hi);
+		rdmsrq(MSR_IA32_PERF_STATUS, val);
 		i++;
 		if (unlikely(i > 64)) {
 			return -ENODEV;
 		}
 	}
 	/* Set new multiplier and voltage */
-	wrmsr(MSR_IA32_PERF_CTL, dest_state & 0xffff, 0);
+	wrmsrq(MSR_IA32_PERF_CTL, dest_state & 0xffff);
 	/* Wait until transition end */
 	i = 0;
 	do {
 		udelay(16);
-		rdmsr(MSR_IA32_PERF_STATUS, lo, hi);
+		rdmsrq(MSR_IA32_PERF_STATUS, val);
 		i++;
 		if (unlikely(i > 64)) {
 			return -ENODEV;
 		}
-	} while (lo & ((1 << 16) | (1 << 17)));
+	} while (val & ((1 << 16) | (1 << 17)));
 
 #ifdef DEBUG
 	{
 	u8 current_multiplier, current_voltage;
 
 	/* Print voltage and multiplier */
-	rdmsr(MSR_IA32_PERF_STATUS, lo, hi);
-	current_voltage = lo & 0xff;
+	rdmsrq(MSR_IA32_PERF_STATUS, val);
+	current_voltage = val & 0xff;
 	pr_info("Current voltage = %dmV\n", current_voltage * 16 + 700);
-	current_multiplier = (lo >> 8) & 0xff;
+	current_multiplier = (val >> 8) & 0xff;
 	pr_info("Current multiplier = %d\n", current_multiplier);
 	}
 #endif
@@ -171,7 +171,6 @@ static int eps_target(struct cpufreq_policy *policy, unsigned int index)
 static int eps_cpu_init(struct cpufreq_policy *policy)
 {
 	unsigned int i;
-	u32 lo, hi;
 	u64 val;
 	u8 current_multiplier, current_voltage;
 	u8 max_multiplier, max_voltage;
@@ -195,13 +194,13 @@ static int eps_cpu_init(struct cpufreq_policy *policy)
 
 	switch (c->x86_model) {
 	case 10:
-		rdmsr(0x1153, lo, hi);
-		brand = (((lo >> 2) ^ lo) >> 18) & 3;
+		rdmsrq(0x1153, val);
+		brand = (((val >> 2) ^ val) >> 18) & 3;
 		pr_cont("Model A ");
 		break;
 	case 13:
-		rdmsr(0x1154, lo, hi);
-		brand = (((lo >> 4) ^ (lo >> 2))) & 0x000000ff;
+		rdmsrq(0x1154, val);
+		brand = (((val >> 4) ^ (val >> 2))) & 0x000000ff;
 		pr_cont("Model D ");
 		break;
 	}
@@ -237,20 +236,20 @@ static int eps_cpu_init(struct cpufreq_policy *policy)
 	}
 
 	/* Print voltage and multiplier */
-	rdmsr(MSR_IA32_PERF_STATUS, lo, hi);
-	current_voltage = lo & 0xff;
+	rdmsrq(MSR_IA32_PERF_STATUS, val);
+	current_voltage = val & 0xff;
 	pr_info("Current voltage = %dmV\n", current_voltage * 16 + 700);
-	current_multiplier = (lo >> 8) & 0xff;
+	current_multiplier = (val >> 8) & 0xff;
 	pr_info("Current multiplier = %d\n", current_multiplier);
 
 	/* Print limits */
-	max_voltage = hi & 0xff;
+	max_voltage = (val >> 32) & 0xff;
 	pr_info("Highest voltage = %dmV\n", max_voltage * 16 + 700);
-	max_multiplier = (hi >> 8) & 0xff;
+	max_multiplier = (val >> 40) & 0xff;
 	pr_info("Highest multiplier = %d\n", max_multiplier);
-	min_voltage = (hi >> 16) & 0xff;
+	min_voltage = (val >> 48) & 0xff;
 	pr_info("Lowest voltage = %dmV\n", min_voltage * 16 + 700);
-	min_multiplier = (hi >> 24) & 0xff;
+	min_multiplier = (val >> 56) & 0xff;
 	pr_info("Lowest multiplier = %d\n", min_multiplier);
 
 	/* Sanity checks */
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index 5a0eeb84d3821..2cbf810db66e3 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -299,7 +299,6 @@ struct pstate_funcs {
 static struct pstate_funcs pstate_funcs __read_mostly;
 
 static bool hwp_active __ro_after_init;
-static int hwp_mode_bdw __ro_after_init;
 static bool per_cpu_limits __ro_after_init;
 static bool hwp_forced __ro_after_init;
 static bool hwp_boost __read_mostly;
@@ -587,21 +586,14 @@ static void intel_pstate_hybrid_hwp_adjust(struct cpudata *cpu)
 
 	hwp_is_hybrid = true;
 
-	cpu->pstate.turbo_freq = rounddown(cpu->pstate.turbo_pstate * scaling,
-					   perf_ctl_scaling);
-	cpu->pstate.max_freq = rounddown(cpu->pstate.max_pstate * scaling,
-					 perf_ctl_scaling);
-
 	freq = perf_ctl_max_phys * perf_ctl_scaling;
 	cpu->pstate.max_pstate_physical = intel_pstate_freq_to_hwp(cpu, freq);
 
-	freq = cpu->pstate.min_pstate * perf_ctl_scaling;
-	cpu->pstate.min_freq = freq;
 	/*
 	 * Cast the min P-state value retrieved via pstate_funcs.get_min() to
 	 * the effective range of HWP performance levels.
 	 */
-	cpu->pstate.min_pstate = intel_pstate_freq_to_hwp(cpu, freq);
+	cpu->pstate.min_pstate = intel_pstate_freq_to_hwp(cpu, cpu->pstate.min_freq);
 }
 
 static bool turbo_is_disabled(void)
@@ -979,12 +971,10 @@ static int hybrid_get_cost(struct device *dev, unsigned long freq,
 	 * capacity.  Similarly, P-cores start to be populated when E-cores are
 	 * utilized above 60% of the capacity.
 	 */
-	if (hybrid_get_cpu_type(dev->id) == INTEL_CPU_TYPE_ATOM) {
-		if (hybrid_has_l3(dev->id)) /* E-core */
-			*cost += 1;
-	} else { /* P-core */
+	if (hybrid_get_cpu_type(dev->id) == INTEL_CPU_TYPE_CORE) /* P-core */
 		*cost += 2;
-	}
+	else if (hybrid_has_l3(dev->id)) /* E-core */
+		*cost += 1;
 
 	return 0;
 }
@@ -1058,12 +1048,14 @@ static void hybrid_clear_cpu_capacity(unsigned int cpunum)
 
 static void hybrid_get_capacity_perf(struct cpudata *cpu)
 {
+	u64 hwp_cap = READ_ONCE(cpu->hwp_cap_cached);
+
 	if (READ_ONCE(global.no_turbo)) {
-		cpu->capacity_perf = cpu->pstate.max_pstate_physical;
+		cpu->capacity_perf = HWP_GUARANTEED_PERF(hwp_cap);
 		return;
 	}
 
-	cpu->capacity_perf = HWP_HIGHEST_PERF(READ_ONCE(cpu->hwp_cap_cached));
+	cpu->capacity_perf = HWP_HIGHEST_PERF(hwp_cap);
 }
 
 static void hybrid_set_capacity_of_cpus(void)
@@ -1183,6 +1175,22 @@ static bool hybrid_clear_max_perf_cpu(void)
 	return ret;
 }
 
+static void intel_pstate_update_freq_limits(struct cpudata *cpu)
+{
+	int scaling = cpu->pstate.scaling;
+	unsigned int turbo_freq = cpu->pstate.turbo_pstate * scaling;
+	unsigned int max_freq = cpu->pstate.max_pstate * scaling;
+	int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling;
+
+	if (scaling != perf_ctl_scaling) {
+		turbo_freq = rounddown(turbo_freq, perf_ctl_scaling);
+		max_freq = rounddown(max_freq, perf_ctl_scaling);
+	}
+
+	cpu->pstate.turbo_freq = turbo_freq;
+	cpu->pstate.max_freq = max_freq;
+}
+
 static void __intel_pstate_get_hwp_cap(struct cpudata *cpu)
 {
 	u64 cap;
@@ -1195,20 +1203,8 @@ static void __intel_pstate_get_hwp_cap(struct cpudata *cpu)
 
 static void intel_pstate_get_hwp_cap(struct cpudata *cpu)
 {
-	int scaling = cpu->pstate.scaling;
-
 	__intel_pstate_get_hwp_cap(cpu);
-
-	cpu->pstate.max_freq = cpu->pstate.max_pstate * scaling;
-	cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * scaling;
-	if (scaling != cpu->pstate.perf_ctl_scaling) {
-		int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling;
-
-		cpu->pstate.max_freq = rounddown(cpu->pstate.max_freq,
-						 perf_ctl_scaling);
-		cpu->pstate.turbo_freq = rounddown(cpu->pstate.turbo_freq,
-						   perf_ctl_scaling);
-	}
+	intel_pstate_update_freq_limits(cpu);
 }
 
 static void hybrid_update_capacity(struct cpudata *cpu)
@@ -2297,33 +2293,16 @@ static int hwp_get_cpu_scaling(int cpu)
 	return intel_pstate_cppc_get_scaling(cpu);
 }
 
-static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
-{
-	trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
-	cpu->pstate.current_pstate = pstate;
-	/*
-	 * Generally, there is no guarantee that this code will always run on
-	 * the CPU being updated, so force the register update to run on the
-	 * right CPU.
-	 */
-	wrmsrq_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL,
-		      pstate_funcs.get_val(cpu, pstate));
-}
-
-static void intel_pstate_set_min_pstate(struct cpudata *cpu)
-{
-	intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
-}
-
 static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
 {
 	int perf_ctl_scaling = pstate_funcs.get_scaling();
 
 	cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical(cpu->cpu);
 	cpu->pstate.min_pstate = pstate_funcs.get_min(cpu->cpu);
+	cpu->pstate.min_freq = cpu->pstate.min_pstate * perf_ctl_scaling;
 	cpu->pstate.perf_ctl_scaling = perf_ctl_scaling;
 
-	if (hwp_active && !hwp_mode_bdw) {
+	if (hwp_active) {
 		__intel_pstate_get_hwp_cap(cpu);
 
 		if (pstate_funcs.get_cpu_scaling) {
@@ -2343,19 +2322,13 @@ static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
 		cpu->pstate.turbo_pstate = pstate_funcs.get_turbo(cpu->cpu);
 	}
 
-	if (cpu->pstate.scaling == perf_ctl_scaling) {
-		cpu->pstate.min_freq = cpu->pstate.min_pstate * perf_ctl_scaling;
-		cpu->pstate.max_freq = cpu->pstate.max_pstate * perf_ctl_scaling;
-		cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * perf_ctl_scaling;
-	}
+	intel_pstate_update_freq_limits(cpu);
 
 	if (pstate_funcs.get_aperf_mperf_shift)
 		cpu->aperf_mperf_shift = pstate_funcs.get_aperf_mperf_shift();
 
 	if (pstate_funcs.get_vid)
 		pstate_funcs.get_vid(cpu);
-
-	intel_pstate_set_min_pstate(cpu);
 }
 
 /*
@@ -2882,6 +2855,19 @@ static void intel_pstate_update_perf_limits(struct cpudata *cpu,
 		 cpu->min_perf_ratio);
 }
 
+static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
+{
+	trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
+	cpu->pstate.current_pstate = pstate;
+	/*
+	 * Generally, there is no guarantee that this code will always run on
+	 * the CPU being updated, so force the register update to run on the
+	 * right CPU.
+	 */
+	wrmsrq_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL,
+		      pstate_funcs.get_val(cpu, pstate));
+}
+
 static int intel_pstate_set_policy(struct cpufreq_policy *policy)
 {
 	struct cpudata *cpu;
@@ -2969,6 +2955,11 @@ static int intel_pstate_verify_policy(struct cpufreq_policy_data *policy)
 	return 0;
 }
 
+static void intel_pstate_set_min_pstate(struct cpudata *cpu)
+{
+	intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
+}
+
 static int intel_cpufreq_cpu_offline(struct cpufreq_policy *policy)
 {
 	struct cpudata *cpu = all_cpu_data[policy->cpu];
@@ -3061,6 +3052,7 @@ static int __intel_pstate_cpu_init(struct cpufreq_policy *policy)
 static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
 {
 	int ret = __intel_pstate_cpu_init(policy);
+	struct cpudata *cpu;
 
 	if (ret)
 		return ret;
@@ -3071,11 +3063,11 @@ static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
 	 */
 	policy->policy = CPUFREQ_POLICY_POWERSAVE;
 
-	if (hwp_active) {
-		struct cpudata *cpu = all_cpu_data[policy->cpu];
-
+	cpu = all_cpu_data[policy->cpu];
+	if (hwp_active)
 		cpu->epp_cached = intel_pstate_get_epp(cpu, 0);
-	}
+	else
+		intel_pstate_set_min_pstate(cpu);
 
 	return 0;
 }
@@ -3241,6 +3233,7 @@ static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy,
 static void intel_cpufreq_adjust_perf(struct cpufreq_policy *policy,
 				      unsigned long min_perf,
 				      unsigned long target_perf,
+				      unsigned long max_perf,
 				      unsigned long capacity)
 {
 	struct cpudata *cpu = all_cpu_data[policy->cpu];
@@ -3271,7 +3264,13 @@ static void intel_cpufreq_adjust_perf(struct cpufreq_policy *policy,
 	if (min_pstate > cpu->max_perf_ratio)
 		min_pstate = cpu->max_perf_ratio;
 
-	max_pstate = min(cap_pstate, cpu->max_perf_ratio);
+	max_pstate = cap_pstate;
+	if (max_perf < capacity)
+		max_pstate = DIV_ROUND_UP(cap_pstate * max_perf, capacity);
+
+	if (max_pstate > cpu->max_perf_ratio)
+		max_pstate = cpu->max_perf_ratio;
+
 	if (max_pstate < min_pstate)
 		max_pstate = min_pstate;
 
@@ -3299,8 +3298,6 @@ static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy)
 		return ret;
 
 	policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY;
-	/* This reflects the intel_pstate_get_cpu_pstates() setting. */
-	policy->cur = policy->cpuinfo.min_freq;
 
 	req = kzalloc_objs(*req, 2);
 	if (!req) {
@@ -3321,9 +3318,15 @@ static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy)
 		WRITE_ONCE(cpu->hwp_req_cached, value);
 
 		cpu->epp_cached = intel_pstate_get_epp(cpu, value);
+
+		intel_cpufreq_hwp_update(cpu, cpu->pstate.min_pstate,
+					 cpu->pstate.max_pstate,
+					 cpu->pstate.min_pstate, false);
 	} else {
 		policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY;
+		intel_pstate_set_min_pstate(cpu);
 	}
+	policy->cur = policy->cpuinfo.min_freq;
 
 	freq = DIV_ROUND_UP(cpu->pstate.turbo_freq * global.min_perf_pct, 100);
 
@@ -3674,14 +3677,14 @@ static inline bool intel_pstate_has_acpi_ppc(void) { return false; }
 static inline void intel_pstate_request_control_from_smm(void) {}
 #endif /* CONFIG_ACPI */
 
-#define INTEL_PSTATE_HWP_BROADWELL	0x01
+#define INTEL_PSTATE_HWP_NOT_HYBRID	0x01
 
 #define X86_MATCH_HWP(vfm, hwp_mode)				\
 	X86_MATCH_VFM_FEATURE(vfm, X86_FEATURE_HWP, hwp_mode)
 
 static const struct x86_cpu_id hwp_support_ids[] __initconst = {
-	X86_MATCH_HWP(INTEL_BROADWELL_X,	INTEL_PSTATE_HWP_BROADWELL),
-	X86_MATCH_HWP(INTEL_BROADWELL_D,	INTEL_PSTATE_HWP_BROADWELL),
+	X86_MATCH_HWP(INTEL_BROADWELL_X,	INTEL_PSTATE_HWP_NOT_HYBRID),
+	X86_MATCH_HWP(INTEL_BROADWELL_D,	INTEL_PSTATE_HWP_NOT_HYBRID),
 	X86_MATCH_HWP(INTEL_ANY,		0),
 	{}
 };
@@ -3806,7 +3809,6 @@ static int __init intel_pstate_init(void)
 
 		if (!no_hwp) {
 			hwp_active = true;
-			hwp_mode_bdw = id->driver_data;
 			intel_pstate.attr = hwp_cpufreq_attrs;
 			intel_cpufreq.attr = hwp_cpufreq_attrs;
 			intel_cpufreq.flags |= CPUFREQ_NEED_UPDATE_LIMITS;
@@ -3814,7 +3816,8 @@ static int __init intel_pstate_init(void)
 			if (!default_driver)
 				default_driver = &intel_pstate;
 
-			pstate_funcs.get_cpu_scaling = hwp_get_cpu_scaling;
+			if (!id->driver_data)
+				pstate_funcs.get_cpu_scaling = hwp_get_cpu_scaling;
 
 			goto hwp_cpu_matched;
 		}
diff --git a/drivers/cpufreq/longhaul.c b/drivers/cpufreq/longhaul.c
index a18d1d11725f1..4c2599264333a 100644
--- a/drivers/cpufreq/longhaul.c
+++ b/drivers/cpufreq/longhaul.c
@@ -118,13 +118,14 @@ static unsigned int calc_speed(int mult)
 
 static int longhaul_get_cpu_mult(void)
 {
-	unsigned long invalue = 0, lo, hi;
+	unsigned long invalue = 0;
+	u64 val;
 
-	rdmsr(MSR_IA32_EBL_CR_POWERON, lo, hi);
-	invalue = (lo & (1<<22|1<<23|1<<24|1<<25))>>22;
+	rdmsrq(MSR_IA32_EBL_CR_POWERON, val);
+	invalue = (val & (1<<22|1<<23|1<<24|1<<25))>>22;
 	if (longhaul_version == TYPE_LONGHAUL_V2 ||
 	    longhaul_version == TYPE_POWERSAVER) {
-		if (lo & (1<<27))
+		if (val & (1<<27))
 			invalue += 16;
 	}
 	return eblcr[invalue];
@@ -761,7 +762,7 @@ static int longhaul_cpu_init(struct cpufreq_policy *policy)
 	struct cpuinfo_x86 *c = &cpu_data(0);
 	char *cpuname = NULL;
 	int ret;
-	u32 lo, hi;
+	u64 val;
 
 	/* Check what we have on this motherboard */
 	switch (c->x86_model) {
@@ -835,8 +836,8 @@ static int longhaul_cpu_init(struct cpufreq_policy *policy)
 	}
 	/* Check Longhaul ver. 2 */
 	if (longhaul_version == TYPE_LONGHAUL_V2) {
-		rdmsr(MSR_VIA_LONGHAUL, lo, hi);
-		if (lo == 0 && hi == 0)
+		rdmsrq(MSR_VIA_LONGHAUL, val);
+		if (val == 0)
 			/* Looks like MSR isn't present */
 			longhaul_version = TYPE_LONGHAUL_V1;
 	}
diff --git a/drivers/cpufreq/longrun.c b/drivers/cpufreq/longrun.c
index f3aaca0496a4c..82a7bb69c4015 100644
--- a/drivers/cpufreq/longrun.c
+++ b/drivers/cpufreq/longrun.c
@@ -35,27 +35,27 @@ static unsigned int longrun_low_freq, longrun_high_freq;
  */
 static void longrun_get_policy(struct cpufreq_policy *policy)
 {
-	u32 msr_lo, msr_hi;
+	struct msr msr;
 
-	rdmsr(MSR_TMTA_LONGRUN_FLAGS, msr_lo, msr_hi);
-	pr_debug("longrun flags are %x - %x\n", msr_lo, msr_hi);
-	if (msr_lo & 0x01)
+	rdmsrq(MSR_TMTA_LONGRUN_FLAGS, msr.q);
+	pr_debug("longrun flags are %x - %x\n", msr.l, msr.h);
+	if (msr.l & 0x01)
 		policy->policy = CPUFREQ_POLICY_PERFORMANCE;
 	else
 		policy->policy = CPUFREQ_POLICY_POWERSAVE;
 
-	rdmsr(MSR_TMTA_LONGRUN_CTRL, msr_lo, msr_hi);
-	pr_debug("longrun ctrl is %x - %x\n", msr_lo, msr_hi);
-	msr_lo &= 0x0000007F;
-	msr_hi &= 0x0000007F;
+	rdmsrq(MSR_TMTA_LONGRUN_CTRL, msr.q);
+	pr_debug("longrun ctrl is %x - %x\n", msr.l, msr.h);
+	msr.l &= 0x0000007F;
+	msr.h &= 0x0000007F;
 
 	if (longrun_high_freq <= longrun_low_freq) {
 		/* Assume degenerate Longrun table */
 		policy->min = policy->max = longrun_high_freq;
 	} else {
-		policy->min = longrun_low_freq + msr_lo *
+		policy->min = longrun_low_freq + msr.l *
 			((longrun_high_freq - longrun_low_freq) / 100);
-		policy->max = longrun_low_freq + msr_hi *
+		policy->max = longrun_low_freq + msr.h *
 			((longrun_high_freq - longrun_low_freq) / 100);
 	}
 	policy->cpu = 0;
@@ -71,7 +71,7 @@ static void longrun_get_policy(struct cpufreq_policy *policy)
  */
 static int longrun_set_policy(struct cpufreq_policy *policy)
 {
-	u32 msr_lo, msr_hi;
+	struct msr msr;
 	u32 pctg_lo, pctg_hi;
 
 	if (!policy)
@@ -93,24 +93,24 @@ static int longrun_set_policy(struct cpufreq_policy *policy)
 		pctg_lo = pctg_hi;
 
 	/* performance or economy mode */
-	rdmsr(MSR_TMTA_LONGRUN_FLAGS, msr_lo, msr_hi);
-	msr_lo &= 0xFFFFFFFE;
+	rdmsrq(MSR_TMTA_LONGRUN_FLAGS, msr.q);
+	msr.l &= 0xFFFFFFFE;
 	switch (policy->policy) {
 	case CPUFREQ_POLICY_PERFORMANCE:
-		msr_lo |= 0x00000001;
+		msr.l |= 0x00000001;
 		break;
 	case CPUFREQ_POLICY_POWERSAVE:
 		break;
 	}
-	wrmsr(MSR_TMTA_LONGRUN_FLAGS, msr_lo, msr_hi);
+	wrmsrq(MSR_TMTA_LONGRUN_FLAGS, msr.q);
 
 	/* lower and upper boundary */
-	rdmsr(MSR_TMTA_LONGRUN_CTRL, msr_lo, msr_hi);
-	msr_lo &= 0xFFFFFF80;
-	msr_hi &= 0xFFFFFF80;
-	msr_lo |= pctg_lo;
-	msr_hi |= pctg_hi;
-	wrmsr(MSR_TMTA_LONGRUN_CTRL, msr_lo, msr_hi);
+	rdmsrq(MSR_TMTA_LONGRUN_CTRL, msr.q);
+	msr.l &= 0xFFFFFF80;
+	msr.h &= 0xFFFFFF80;
+	msr.l |= pctg_lo;
+	msr.h |= pctg_hi;
+	wrmsrq(MSR_TMTA_LONGRUN_CTRL, msr.q);
 
 	return 0;
 }
@@ -160,8 +160,7 @@ static unsigned int longrun_get(unsigned int cpu)
 static int longrun_determine_freqs(unsigned int *low_freq,
 						      unsigned int *high_freq)
 {
-	u32 msr_lo, msr_hi;
-	u32 save_lo, save_hi;
+	struct msr msr, save;
 	u32 eax, ebx, ecx, edx;
 	u32 try_hi;
 	struct cpuinfo_x86 *c = &cpu_data(0);
@@ -178,15 +177,17 @@ static int longrun_determine_freqs(unsigned int *low_freq,
 		 * For maximum frequency, read out level zero.
 		 */
 		/* minimum */
-		rdmsr(MSR_TMTA_LRTI_READOUT, msr_lo, msr_hi);
-		wrmsr(MSR_TMTA_LRTI_READOUT, msr_hi, msr_hi);
-		rdmsr(MSR_TMTA_LRTI_VOLT_MHZ, msr_lo, msr_hi);
-		*low_freq = msr_lo * 1000; /* to kHz */
+		rdmsrq(MSR_TMTA_LRTI_READOUT, msr.q);
+		msr.l = msr.h;
+		wrmsrq(MSR_TMTA_LRTI_READOUT, msr.q);
+		rdmsrq(MSR_TMTA_LRTI_VOLT_MHZ, msr.q);
+		*low_freq = msr.l * 1000; /* to kHz */
 
 		/* maximum */
-		wrmsr(MSR_TMTA_LRTI_READOUT, 0, msr_hi);
-		rdmsr(MSR_TMTA_LRTI_VOLT_MHZ, msr_lo, msr_hi);
-		*high_freq = msr_lo * 1000; /* to kHz */
+		msr.l = 0;
+		wrmsrq(MSR_TMTA_LRTI_READOUT, msr.q);
+		rdmsrq(MSR_TMTA_LRTI_VOLT_MHZ, msr.q);
+		*high_freq = msr.l * 1000; /* to kHz */
 
 		pr_debug("longrun table interface told %u - %u kHz\n",
 				*low_freq, *high_freq);
@@ -202,9 +203,9 @@ static int longrun_determine_freqs(unsigned int *low_freq,
 	pr_debug("high frequency is %u kHz\n", *high_freq);
 
 	/* get current borders */
-	rdmsr(MSR_TMTA_LONGRUN_CTRL, msr_lo, msr_hi);
-	save_lo = msr_lo & 0x0000007F;
-	save_hi = msr_hi & 0x0000007F;
+	rdmsrq(MSR_TMTA_LONGRUN_CTRL, msr.q);
+	save.l = msr.l & 0x0000007F;
+	save.h = msr.h & 0x0000007F;
 
 	/* if current perf_pctg is larger than 90%, we need to decrease the
 	 * upper limit to make the calculation more accurate.
@@ -214,16 +215,16 @@ static int longrun_determine_freqs(unsigned int *low_freq,
 	 * on some barrier values */
 	for (try_hi = 80; try_hi > 0 && ecx > 90; try_hi -= 10) {
 		/* set to 0 to try_hi perf_pctg */
-		msr_lo &= 0xFFFFFF80;
-		msr_hi &= 0xFFFFFF80;
-		msr_hi |= try_hi;
-		wrmsr(MSR_TMTA_LONGRUN_CTRL, msr_lo, msr_hi);
+		msr.l &= 0xFFFFFF80;
+		msr.h &= 0xFFFFFF80;
+		msr.h |= try_hi;
+		wrmsrq(MSR_TMTA_LONGRUN_CTRL, msr.q);
 
 		/* read out current core MHz and current perf_pctg */
 		cpuid(0x80860007, &eax, &ebx, &ecx, &edx);
 
 		/* restore values */
-		wrmsr(MSR_TMTA_LONGRUN_CTRL, save_lo, save_hi);
+		wrmsrq(MSR_TMTA_LONGRUN_CTRL, save.q);
 	}
 	pr_debug("percentage is %u %%, freq is %u MHz\n", ecx, eax);
 
diff --git a/drivers/cpufreq/powernow-k6.c b/drivers/cpufreq/powernow-k6.c
index 99d2244e03b0e..2044e8a336ec1 100644
--- a/drivers/cpufreq/powernow-k6.c
+++ b/drivers/cpufreq/powernow-k6.c
@@ -83,15 +83,15 @@ static const struct {
 static int powernow_k6_get_cpu_multiplier(void)
 {
 	unsigned long invalue = 0;
-	u32 msrval;
+	u64 msrval;
 
 	local_irq_disable();
 
 	msrval = POWERNOW_IOPORT + 0x1;
-	wrmsr(MSR_K6_EPMR, msrval, 0); /* enable the PowerNow port */
+	wrmsrq(MSR_K6_EPMR, msrval); /* enable the PowerNow port */
 	invalue = inl(POWERNOW_IOPORT + 0x8);
 	msrval = POWERNOW_IOPORT + 0x0;
-	wrmsr(MSR_K6_EPMR, msrval, 0); /* disable it again */
+	wrmsrq(MSR_K6_EPMR, msrval); /* disable it again */
 
 	local_irq_enable();
 
@@ -101,8 +101,8 @@ static int powernow_k6_get_cpu_multiplier(void)
 static void powernow_k6_set_cpu_multiplier(unsigned int best_i)
 {
 	unsigned long outvalue, invalue;
-	unsigned long msrval;
 	unsigned long cr0;
+	u64 msrval;
 
 	/* we now need to transform best_i to the BVC format, see AMD#23446 */
 
@@ -118,13 +118,13 @@ static void powernow_k6_set_cpu_multiplier(unsigned int best_i)
 	outvalue = (1<<12) | (1<<10) | (1<<9) | (index_to_register[best_i]<<5);
 
 	msrval = POWERNOW_IOPORT + 0x1;
-	wrmsr(MSR_K6_EPMR, msrval, 0); /* enable the PowerNow port */
+	wrmsrq(MSR_K6_EPMR, msrval); /* enable the PowerNow port */
 	invalue = inl(POWERNOW_IOPORT + 0x8);
 	invalue = invalue & 0x1f;
 	outvalue = outvalue | invalue;
 	outl(outvalue, (POWERNOW_IOPORT + 0x8));
 	msrval = POWERNOW_IOPORT + 0x0;
-	wrmsr(MSR_K6_EPMR, msrval, 0); /* disable it again */
+	wrmsrq(MSR_K6_EPMR, msrval); /* disable it again */
 
 	write_cr0(cr0);
 	local_irq_enable();
diff --git a/drivers/cpufreq/powernow-k8.c b/drivers/cpufreq/powernow-k8.c
index 2b791f1ec51bb..fe1f499b4fc0e 100644
--- a/drivers/cpufreq/powernow-k8.c
+++ b/drivers/cpufreq/powernow-k8.c
@@ -87,10 +87,10 @@ static u32 convert_fid_to_vco_fid(u32 fid)
  */
 static int pending_bit_stuck(void)
 {
-	u32 lo, hi __always_unused;
+	u64 msr;
 
-	rdmsr(MSR_FIDVID_STATUS, lo, hi);
-	return lo & MSR_S_LO_CHANGE_PENDING ? 1 : 0;
+	rdmsrq(MSR_FIDVID_STATUS, msr);
+	return msr & MSR_S_LO_CHANGE_PENDING ? 1 : 0;
 }
 
 /*
@@ -99,7 +99,7 @@ static int pending_bit_stuck(void)
  */
 static int query_current_values_with_pending_wait(struct powernow_k8_data *data)
 {
-	u32 lo, hi;
+	struct msr msr;
 	u32 i = 0;
 
 	do {
@@ -107,11 +107,11 @@ static int query_current_values_with_pending_wait(struct powernow_k8_data *data)
 			pr_debug("detected change pending stuck\n");
 			return 1;
 		}
-		rdmsr(MSR_FIDVID_STATUS, lo, hi);
-	} while (lo & MSR_S_LO_CHANGE_PENDING);
+		rdmsrq(MSR_FIDVID_STATUS, msr.q);
+	} while (msr.l & MSR_S_LO_CHANGE_PENDING);
 
-	data->currvid = hi & MSR_S_HI_CURRENT_VID;
-	data->currfid = lo & MSR_S_LO_CURRENT_FID;
+	data->currvid = msr.h & MSR_S_HI_CURRENT_VID;
+	data->currfid = msr.l & MSR_S_LO_CURRENT_FID;
 
 	return 0;
 }
@@ -131,22 +131,22 @@ static void count_off_vst(struct powernow_k8_data *data)
 /* need to init the control msr to a safe value (for each cpu) */
 static void fidvid_msr_init(void)
 {
-	u32 lo, hi;
+	struct msr msr;
 	u8 fid, vid;
 
-	rdmsr(MSR_FIDVID_STATUS, lo, hi);
-	vid = hi & MSR_S_HI_CURRENT_VID;
-	fid = lo & MSR_S_LO_CURRENT_FID;
-	lo = fid | (vid << MSR_C_LO_VID_SHIFT);
-	hi = MSR_C_HI_STP_GNT_BENIGN;
-	pr_debug("cpu%d, init lo 0x%x, hi 0x%x\n", smp_processor_id(), lo, hi);
-	wrmsr(MSR_FIDVID_CTL, lo, hi);
+	rdmsrq(MSR_FIDVID_STATUS, msr.q);
+	vid = msr.h & MSR_S_HI_CURRENT_VID;
+	fid = msr.l & MSR_S_LO_CURRENT_FID;
+	msr.l = fid | (vid << MSR_C_LO_VID_SHIFT);
+	msr.h = MSR_C_HI_STP_GNT_BENIGN;
+	pr_debug("cpu%d, init lo 0x%x, hi 0x%x\n", smp_processor_id(), msr.l, msr.h);
+	wrmsrq(MSR_FIDVID_CTL, msr.q);
 }
 
 /* write the new fid value along with the other control fields to the msr */
 static int write_new_fid(struct powernow_k8_data *data, u32 fid)
 {
-	u32 lo;
+	struct msr msr;
 	u32 savevid = data->currvid;
 	u32 i = 0;
 
@@ -155,15 +155,15 @@ static int write_new_fid(struct powernow_k8_data *data, u32 fid)
 		return 1;
 	}
 
-	lo = fid;
-	lo |= (data->currvid << MSR_C_LO_VID_SHIFT);
-	lo |= MSR_C_LO_INIT_FID_VID;
+	msr.l = fid;
+	msr.l |= (data->currvid << MSR_C_LO_VID_SHIFT);
+	msr.l |= MSR_C_LO_INIT_FID_VID;
+	msr.h = data->plllock * PLL_LOCK_CONVERSION;
 
-	pr_debug("writing fid 0x%x, lo 0x%x, hi 0x%x\n",
-		fid, lo, data->plllock * PLL_LOCK_CONVERSION);
+	pr_debug("writing fid 0x%x, lo 0x%x, hi 0x%x\n", fid, msr.l, msr.h);
 
 	do {
-		wrmsr(MSR_FIDVID_CTL, lo, data->plllock * PLL_LOCK_CONVERSION);
+		wrmsrq(MSR_FIDVID_CTL, msr.q);
 		if (i++ > 100) {
 			pr_err("Hardware error - pending bit very stuck - no further pstate changes possible\n");
 			return 1;
@@ -190,7 +190,7 @@ static int write_new_fid(struct powernow_k8_data *data, u32 fid)
 /* Write a new vid to the hardware */
 static int write_new_vid(struct powernow_k8_data *data, u32 vid)
 {
-	u32 lo;
+	struct msr msr;
 	u32 savefid = data->currfid;
 	int i = 0;
 
@@ -199,15 +199,15 @@ static int write_new_vid(struct powernow_k8_data *data, u32 vid)
 		return 1;
 	}
 
-	lo = data->currfid;
-	lo |= (vid << MSR_C_LO_VID_SHIFT);
-	lo |= MSR_C_LO_INIT_FID_VID;
+	msr.l = data->currfid;
+	msr.l |= (vid << MSR_C_LO_VID_SHIFT);
+	msr.l |= MSR_C_LO_INIT_FID_VID;
+	msr.h = STOP_GRANT_5NS;
 
-	pr_debug("writing vid 0x%x, lo 0x%x, hi 0x%x\n",
-		vid, lo, STOP_GRANT_5NS);
+	pr_debug("writing vid 0x%x, lo 0x%x, hi 0x%x\n", vid, msr.l, msr.h);
 
 	do {
-		wrmsr(MSR_FIDVID_CTL, lo, STOP_GRANT_5NS);
+		wrmsrq(MSR_FIDVID_CTL, msr.q);
 		if (i++ > 100) {
 			pr_err("internal error - pending bit very stuck - no further pstate changes possible\n");
 			return 1;
@@ -281,9 +281,10 @@ static int transition_fid_vid(struct powernow_k8_data *data,
 static int core_voltage_pre_transition(struct powernow_k8_data *data,
 		u32 reqvid, u32 reqfid)
 {
+	struct msr msr;
 	u32 rvosteps = data->rvo;
 	u32 savefid = data->currfid;
-	u32 maxvid, lo __always_unused, rvomult = 1;
+	u32 maxvid, rvomult = 1;
 
 	pr_debug("ph1 (cpu%d): start, currfid 0x%x, currvid 0x%x, reqvid 0x%x, rvo 0x%x\n",
 		smp_processor_id(),
@@ -292,8 +293,8 @@ static int core_voltage_pre_transition(struct powernow_k8_data *data,
 	if ((savefid < LO_FID_TABLE_TOP) && (reqfid < LO_FID_TABLE_TOP))
 		rvomult = 2;
 	rvosteps *= rvomult;
-	rdmsr(MSR_FIDVID_STATUS, lo, maxvid);
-	maxvid = 0x1f & (maxvid >> 16);
+	rdmsrq(MSR_FIDVID_STATUS, msr.q);
+	maxvid = 0x1f & (msr.h >> 16);
 	pr_debug("ph1 maxvid=0x%x\n", maxvid);
 	if (reqvid < maxvid) /* lower numbers are higher voltages */
 		reqvid = maxvid;
diff --git a/drivers/cpufreq/qcom-cpufreq-nvmem.c b/drivers/cpufreq/qcom-cpufreq-nvmem.c
index e6d28d162442a..efa766e98d865 100644
--- a/drivers/cpufreq/qcom-cpufreq-nvmem.c
+++ b/drivers/cpufreq/qcom-cpufreq-nvmem.c
@@ -200,6 +200,13 @@ static int qcom_cpufreq_kryo_name_version(struct device *cpu_dev,
 	case QCOM_ID_IPQ9574:
 		drv->versions = 1 << (unsigned int)(*speedbin);
 		break;
+	case QCOM_ID_IPQ5200:
+	case QCOM_ID_IPQ5210:
+	case QCOM_ID_QCF2200:
+	case QCOM_ID_QCF3200:
+	case QCOM_ID_QCF3210:
+		drv->versions = (*speedbin == 0xcd) ? BIT(1) : BIT(0);
+		break;
 	case QCOM_ID_IPQ5424:
 	case QCOM_ID_IPQ5404:
 		drv->versions = (*speedbin == 0x3b) ? BIT(1) : BIT(0);
@@ -618,6 +625,7 @@ static const struct of_device_id qcom_cpufreq_match_list[] __initconst __maybe_u
 	{ .compatible = "qcom,msm8909", .data = &match_data_msm8909 },
 	{ .compatible = "qcom,msm8996", .data = &match_data_kryo },
 	{ .compatible = "qcom,qcs404", .data = &match_data_qcs404 },
+	{ .compatible = "qcom,ipq5210", .data = &match_data_kryo },
 	{ .compatible = "qcom,ipq5332", .data = &match_data_kryo },
 	{ .compatible = "qcom,ipq5424", .data = &match_data_kryo },
 	{ .compatible = "qcom,ipq6018", .data = &match_data_ipq6018 },
diff --git a/drivers/cpufreq/rcpufreq_dt.rs b/drivers/cpufreq/rcpufreq_dt.rs
index 10106fa13095e..5d27857261da9 100644
--- a/drivers/cpufreq/rcpufreq_dt.rs
+++ b/drivers/cpufreq/rcpufreq_dt.rs
@@ -4,12 +4,19 @@
 
 use kernel::{
     clk::Clk,
-    cpu, cpufreq,
+    cpu,
+    cpufreq, //
     cpumask::CpumaskVar,
-    device::{Core, Device},
+    device::{
+        Core,
+        Device, //
+    },
     error::code::*,
     macros::vtable,
-    module_platform_driver, of, opp, platform,
+    module_platform_driver,
+    of,
+    opp,
+    platform, //
     prelude::*,
     str::CString,
     sync::Arc,
diff --git a/drivers/cpufreq/spear-cpufreq.c b/drivers/cpufreq/spear-cpufreq.c
index 81a0780b2ebf0..ffe5eda82f0b1 100644
--- a/drivers/cpufreq/spear-cpufreq.c
+++ b/drivers/cpufreq/spear-cpufreq.c
@@ -79,9 +79,9 @@ static int spear1340_set_cpu_rate(struct clk *sys_pclk, unsigned long newfreq)
 	int ret = 0;
 
 	sys_clk = clk_get_parent(spear_cpufreq.clk);
-	if (IS_ERR(sys_clk)) {
+	if (!sys_clk) {
 		pr_err("failed to get cpu's parent (sys) clock\n");
-		return PTR_ERR(sys_clk);
+		return -EINVAL;
 	}
 
 	/* Set the rate of the source clock before changing the parent */
diff --git a/drivers/cpufreq/speedstep-centrino.c b/drivers/cpufreq/speedstep-centrino.c
index 9237ed8f2b1fa..de50fb367c6bf 100644
--- a/drivers/cpufreq/speedstep-centrino.c
+++ b/drivers/cpufreq/speedstep-centrino.c
@@ -345,7 +345,7 @@ static unsigned int get_cur_freq(unsigned int cpu)
 static int centrino_cpu_init(struct cpufreq_policy *policy)
 {
 	struct cpuinfo_x86 *cpu = &cpu_data(policy->cpu);
-	unsigned l, h;
+	u64 q;
 	int i;
 
 	/* Only Intel makes Enhanced Speedstep-capable CPUs */
@@ -378,16 +378,16 @@ static int centrino_cpu_init(struct cpufreq_policy *policy)
 
 	/* Check to see if Enhanced SpeedStep is enabled, and try to
 	   enable it if not. */
-	rdmsr(MSR_IA32_MISC_ENABLE, l, h);
+	rdmsrq(MSR_IA32_MISC_ENABLE, q);
 
-	if (!(l & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP)) {
-		l |= MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP;
-		pr_debug("trying to enable Enhanced SpeedStep (%x)\n", l);
-		wrmsr(MSR_IA32_MISC_ENABLE, l, h);
+	if (!(q & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP)) {
+		q |= MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP;
+		pr_debug("trying to enable Enhanced SpeedStep (%x)\n", (u32)q);
+		wrmsrq(MSR_IA32_MISC_ENABLE, q);
 
 		/* check to see if it stuck */
-		rdmsr(MSR_IA32_MISC_ENABLE, l, h);
-		if (!(l & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP)) {
+		rdmsrq(MSR_IA32_MISC_ENABLE, q);
+		if (!(q & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP)) {
 			pr_info("couldn't enable Enhanced SpeedStep\n");
 			return -ENODEV;
 		}
diff --git a/drivers/cpufreq/speedstep-lib.c b/drivers/cpufreq/speedstep-lib.c
index 973716c1c29ce..2afc3f177a298 100644
--- a/drivers/cpufreq/speedstep-lib.c
+++ b/drivers/cpufreq/speedstep-lib.c
@@ -69,13 +69,14 @@ static unsigned int pentium3_get_frequency(enum speedstep_processor processor)
 		{   0, 0xff}
 	};
 
+	struct msr msr;
 	u32 msr_lo, msr_tmp;
 	int i = 0, j = 0;
 
 	/* read MSR 0x2a - we only need the low 32 bits */
-	rdmsr(MSR_IA32_EBL_CR_POWERON, msr_lo, msr_tmp);
-	pr_debug("P3 - MSR_IA32_EBL_CR_POWERON: 0x%x 0x%x\n", msr_lo, msr_tmp);
-	msr_tmp = msr_lo;
+	rdmsrq(MSR_IA32_EBL_CR_POWERON, msr.q);
+	pr_debug("P3 - MSR_IA32_EBL_CR_POWERON: 0x%x 0x%x\n", msr.l, msr.h);
+	msr_tmp = msr_lo = msr.l;
 
 	/* decode the FSB */
 	msr_tmp &= 0x00c0000;
@@ -108,19 +109,20 @@ static unsigned int pentium3_get_frequency(enum speedstep_processor processor)
 
 static unsigned int pentiumM_get_frequency(void)
 {
-	u32 msr_lo, msr_tmp;
+	struct msr msr;
+	u32 msr_tmp;
 
-	rdmsr(MSR_IA32_EBL_CR_POWERON, msr_lo, msr_tmp);
-	pr_debug("PM - MSR_IA32_EBL_CR_POWERON: 0x%x 0x%x\n", msr_lo, msr_tmp);
+	rdmsrq(MSR_IA32_EBL_CR_POWERON, msr.q);
+	pr_debug("PM - MSR_IA32_EBL_CR_POWERON: 0x%x 0x%x\n", msr.l, msr.h);
 
 	/* see table B-2 of 24547212.pdf */
-	if (msr_lo & 0x00040000) {
+	if (msr.l & 0x00040000) {
 		printk(KERN_DEBUG PFX "PM - invalid FSB: 0x%x 0x%x\n",
-				msr_lo, msr_tmp);
+				msr.l, msr.h);
 		return 0;
 	}
 
-	msr_tmp = (msr_lo >> 22) & 0x1f;
+	msr_tmp = (msr.l >> 22) & 0x1f;
 	pr_debug("bits 22-26 are 0x%x, speed is %u\n",
 			msr_tmp, (msr_tmp * 100 * 1000));
 
@@ -129,13 +131,14 @@ static unsigned int pentiumM_get_frequency(void)
 
 static unsigned int pentium_core_get_frequency(void)
 {
+	struct msr msr;
 	u32 fsb = 0;
-	u32 msr_lo, msr_tmp;
+	u32 msr_tmp;
 	int ret;
 
-	rdmsr(MSR_FSB_FREQ, msr_lo, msr_tmp);
+	rdmsrq(MSR_FSB_FREQ, msr.q);
 	/* see table B-2 of 25366920.pdf */
-	switch (msr_lo & 0x07) {
+	switch (msr.l & 0x07) {
 	case 5:
 		fsb = 100000;
 		break;
@@ -158,11 +161,11 @@ static unsigned int pentium_core_get_frequency(void)
 		pr_err("PCORE - MSR_FSB_FREQ undefined value\n");
 	}
 
-	rdmsr(MSR_IA32_EBL_CR_POWERON, msr_lo, msr_tmp);
+	rdmsrq(MSR_IA32_EBL_CR_POWERON, msr.q);
 	pr_debug("PCORE - MSR_IA32_EBL_CR_POWERON: 0x%x 0x%x\n",
-			msr_lo, msr_tmp);
+			msr.l, msr.h);
 
-	msr_tmp = (msr_lo >> 22) & 0x1f;
+	msr_tmp = (msr.l >> 22) & 0x1f;
 	pr_debug("bits 22-26 are 0x%x, speed is %u\n",
 			msr_tmp, (msr_tmp * fsb));
 
@@ -174,7 +177,8 @@ static unsigned int pentium_core_get_frequency(void)
 static unsigned int pentium4_get_frequency(void)
 {
 	struct cpuinfo_x86 *c = &boot_cpu_data;
-	u32 msr_lo, msr_hi, mult;
+	struct msr msr;
+	u32 mult;
 	unsigned int fsb = 0;
 	unsigned int ret;
 	u8 fsb_code;
@@ -187,16 +191,16 @@ static unsigned int pentium4_get_frequency(void)
 	if (c->x86_model < 2)
 		return cpu_khz;
 
-	rdmsr(0x2c, msr_lo, msr_hi);
+	rdmsrq(0x2c, msr.q);
 
-	pr_debug("P4 - MSR_EBC_FREQUENCY_ID: 0x%x 0x%x\n", msr_lo, msr_hi);
+	pr_debug("P4 - MSR_EBC_FREQUENCY_ID: 0x%x 0x%x\n", msr.l, msr.h);
 
 	/* decode the FSB: see IA-32 Intel (C) Architecture Software
 	 * Developer's Manual, Volume 3: System Prgramming Guide,
 	 * revision #12 in Table B-1: MSRs in the Pentium 4 and
 	 * Intel Xeon Processors, on page B-4 and B-5.
 	 */
-	fsb_code = (msr_lo >> 16) & 0x7;
+	fsb_code = (msr.l >> 16) & 0x7;
 	switch (fsb_code) {
 	case 0:
 		fsb = 100 * 1000;
@@ -214,7 +218,7 @@ static unsigned int pentium4_get_frequency(void)
 				"Please send an e-mail to <linux@brodo.de>\n");
 
 	/* Multiplier. */
-	mult = msr_lo >> 24;
+	mult = msr.l >> 24;
 
 	pr_debug("P4 - FSB %u kHz; Multiplier %u; Speed %u kHz\n",
 			fsb, mult, (fsb * mult));
@@ -255,7 +259,8 @@ EXPORT_SYMBOL_GPL(speedstep_get_frequency);
 enum speedstep_processor speedstep_detect_processor(void)
 {
 	struct cpuinfo_x86 *c = &cpu_data(0);
-	u32 ebx, msr_lo, msr_hi;
+	struct msr msr;
+	u32 ebx;
 
 	pr_debug("x86: %x, model: %x\n", c->x86, c->x86_model);
 
@@ -343,11 +348,11 @@ enum speedstep_processor speedstep_detect_processor(void)
 
 		/* all mobile PIII Coppermines have FSB 100 MHz
 		 * ==> sort out a few desktop PIIIs. */
-		rdmsr(MSR_IA32_EBL_CR_POWERON, msr_lo, msr_hi);
+		rdmsrq(MSR_IA32_EBL_CR_POWERON, msr.q);
 		pr_debug("Coppermine: MSR_IA32_EBL_CR_POWERON is 0x%x, 0x%x\n",
-				msr_lo, msr_hi);
-		msr_lo &= 0x00c0000;
-		if (msr_lo != 0x0080000)
+				msr.l, msr.h);
+		msr.l &= 0x00c0000;
+		if (msr.l != 0x0080000)
 			return 0;
 
 		/*
@@ -356,11 +361,11 @@ enum speedstep_processor speedstep_detect_processor(void)
 		 * it has SpeedStep technology if either
 		 * bit 56 or 57 is set
 		 */
-		rdmsr(MSR_IA32_PLATFORM_ID, msr_lo, msr_hi);
+		rdmsrq(MSR_IA32_PLATFORM_ID, msr.q);
 		pr_debug("Coppermine: MSR_IA32_PLATFORM ID is 0x%x, 0x%x\n",
-				msr_lo, msr_hi);
-		if ((msr_hi & (1<<18)) &&
-		    (relaxed_check ? 1 : (msr_hi & (3<<24)))) {
+				msr.l, msr.h);
+		if ((msr.h & (1<<18)) &&
+		    (relaxed_check ? 1 : (msr.h & (3<<24)))) {
 			if (c->x86_stepping == 0x01) {
 				pr_debug("early PIII version\n");
 				return SPEEDSTEP_CPU_PIII_C_EARLY;
diff --git a/drivers/cpuidle/Kconfig b/drivers/cpuidle/Kconfig
index d6d8386d3f02b..00e2562041fdb 100644
--- a/drivers/cpuidle/Kconfig
+++ b/drivers/cpuidle/Kconfig
@@ -71,6 +71,11 @@ depends on RISCV
 source "drivers/cpuidle/Kconfig.riscv"
 endmenu
 
+menu "S390 CPU Idle Drivers"
+depends on S390
+source "drivers/cpuidle/Kconfig.s390"
+endmenu
+
 config HALTPOLL_CPUIDLE
 	tristate "Halt poll cpuidle driver"
 	depends on X86 && KVM_GUEST
diff --git a/drivers/cpuidle/Kconfig.arm b/drivers/cpuidle/Kconfig.arm
index a1ee475d180da..b88b01aa5829d 100644
--- a/drivers/cpuidle/Kconfig.arm
+++ b/drivers/cpuidle/Kconfig.arm
@@ -120,11 +120,11 @@ config ARM_QCOM_SPM_CPUIDLE
 	bool "CPU Idle Driver for Qualcomm Subsystem Power Manager (SPM)"
 	depends on (ARCH_QCOM || COMPILE_TEST) && !ARM64 && MMU
 	depends on ARCH_SUSPEND_POSSIBLE
+	depends on QCOM_SCM=y
+	depends on QCOM_SPM=y
 	select ARM_CPU_SUSPEND
 	select CPU_IDLE_MULTIPLE_DRIVERS
 	select DT_IDLE_STATES
-	select QCOM_SCM
-	select QCOM_SPM
 	help
 	  Select this to enable cpuidle for Qualcomm processors.
 	  The Subsystem Power Manager (SPM) controls low power modes for the
diff --git a/drivers/cpuidle/Kconfig.s390 b/drivers/cpuidle/Kconfig.s390
new file mode 100644
index 0000000000000..c9acf7ba8eeef
--- /dev/null
+++ b/drivers/cpuidle/Kconfig.s390
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# S390 CPU Idle drivers
+#
+
+config S390_CPUIDLE
+	bool "S390 CPU idle driver"
+	default y
+	help
+	  Select this option to enable processor idle state management
+	  through cpuidle subsystem.
diff --git a/drivers/cpuidle/Makefile b/drivers/cpuidle/Makefile
index 1de9e92c5b0fc..88cbc2a7aea86 100644
--- a/drivers/cpuidle/Makefile
+++ b/drivers/cpuidle/Makefile
@@ -42,3 +42,7 @@ obj-$(CONFIG_POWERNV_CPUIDLE)		+= cpuidle-powernv.o
 ###############################################################################
 # RISC-V drivers
 obj-$(CONFIG_RISCV_SBI_CPUIDLE)		+= cpuidle-riscv-sbi.o
+
+###############################################################################
+# S390 drivers
+obj-$(CONFIG_S390_CPUIDLE)		+= cpuidle-s390.o
diff --git a/drivers/cpuidle/cpuidle-s390.c b/drivers/cpuidle/cpuidle-s390.c
new file mode 100644
index 0000000000000..1d02a77ebce02
--- /dev/null
+++ b/drivers/cpuidle/cpuidle-s390.c
@@ -0,0 +1,115 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * s390 generic CPU idle driver.
+ *
+ * Copyright IBM Corp. 2026
+ */
+
+#define pr_fmt(fmt) "CPUidle s390: " fmt
+
+#include <linux/init.h>
+#include <linux/cpuidle.h>
+#include <linux/cpu.h>
+#include <linux/sched/clock.h>
+
+static __cpuidle int s390_enter_idle(struct cpuidle_device *dev,
+				     struct cpuidle_driver *drv,
+				     int index)
+{
+	arch_cpu_idle();
+	return index;
+}
+
+static struct cpuidle_driver s390_cpuidle_driver = {
+	.cpumask = (struct cpumask *)cpu_present_mask,
+	.name = "s390-idle",
+	.states = {
+		{ /* entry 0 is for polling */},
+		{
+			.enter			= s390_enter_idle,
+			.name			= "IDLE",
+			.desc			= "ENABLED WAIT",
+		},
+	},
+	.safe_state_index = 0,
+	.state_count = 2,
+};
+
+static int s390_cpuidle_cpu_online(unsigned int cpu)
+{
+	struct cpuidle_device *dev = &per_cpu(cpuidle_dev, cpu);
+	int rc;
+
+	if (dev->registered) {
+		cpuidle_pause_and_lock();
+		rc = cpuidle_enable_device(dev);
+		cpuidle_resume_and_unlock();
+		if (rc)
+			pr_err("Failed to enable cpuidle device on cpu %u\n", cpu);
+	} else {
+		dev->cpu = cpu;
+		rc = cpuidle_register_device(dev);
+		if (rc)
+			pr_err("Failed to register cpuidle driver on cpu %u\n", cpu);
+	}
+	return rc;
+}
+
+static int s390_cpuidle_cpu_dead(unsigned int cpu)
+{
+	struct cpuidle_device *dev = &per_cpu(cpuidle_dev, cpu);
+
+	if (!dev->registered)
+		return 0;
+	cpuidle_pause_and_lock();
+	cpuidle_disable_device(dev);
+	cpuidle_resume_and_unlock();
+	return 0;
+}
+
+/*
+ * The target_residency and exit_latency values are benchmark-derived estimates
+ * that remain non-deterministic due to s390's virtualized architecture.
+ *
+ * Configuration strategy:
+ * - Poll idle state: Values derived from the next enabled idle state (EW)
+ * - Enabled Wait state: Values selected based on idle behavior and empirical
+ *   measurement data
+ *
+ * Goal is to improve responsiveness for workloads with frequent sleep/wakeup
+ * cycles while minimizing any side effects.
+ */
+static void __init s390_cpuidle_ew_tune(void)
+{
+	struct cpuidle_state *state = &s390_cpuidle_driver.states[1];
+
+	if (machine_is_lpar()) {
+		state->target_residency = 5;
+		state->exit_latency = 5;
+	} else {
+		state->target_residency = 1;
+		state->exit_latency = 1;
+	}
+}
+
+static int __init s390_cpuidle_init(void)
+{
+	int rc;
+
+	s390_cpuidle_ew_tune();
+	cpuidle_poll_state_init(&s390_cpuidle_driver);
+	rc = cpuidle_register(&s390_cpuidle_driver, NULL);
+	if (rc)
+		return rc;
+	rc = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+				       "cpuidle/s390:online",
+				       s390_cpuidle_cpu_online,
+				       s390_cpuidle_cpu_dead);
+	if (rc < 0) {
+		cpuidle_unregister(&s390_cpuidle_driver);
+		pr_err("Failed to allocate hotplug state: cpuidle/s390:online\n");
+		return rc;
+	}
+	return 0;
+}
+device_initcall(s390_cpuidle_init);
diff --git a/drivers/cpuidle/cpuidle-ux500.c b/drivers/cpuidle/cpuidle-ux500.c
index f7d778580e9be..6d6c52c0bcc2d 100644
--- a/drivers/cpuidle/cpuidle-ux500.c
+++ b/drivers/cpuidle/cpuidle-ux500.c
@@ -11,7 +11,7 @@
 #include <linux/spinlock.h>
 #include <linux/atomic.h>
 #include <linux/smp.h>
-#include <linux/mfd/dbx500-prcmu.h>
+#include <linux/mfd/db8500-prcmu.h>
 #include <linux/platform_data/arm-ux500-pm.h>
 #include <linux/platform_device.h>
 
@@ -66,7 +66,7 @@ static inline int ux500_enter_idle(struct cpuidle_device *dev,
 		/* Go to the retention state, the prcmu will wait for the
 		 * cpu to go WFI and this is what happens after exiting this
 		 * 'master' critical section */
-		if (prcmu_set_power_state(PRCMU_AP_IDLE, true, true))
+		if (db8500_prcmu_set_power_state(PRCMU_AP_IDLE, true, true))
 			goto out;
 
 		/* When we switch to retention, the prcmu is in charge
@@ -109,7 +109,7 @@ static struct cpuidle_driver ux500_idle_driver = {
 static int dbx500_cpuidle_probe(struct platform_device *pdev)
 {
 	/* Configure wake up reasons */
-	prcmu_enable_wakeups(PRCMU_WAKEUP(ARM) | PRCMU_WAKEUP(RTC) |
+	db8500_prcmu_enable_wakeups(PRCMU_WAKEUP(ARM) | PRCMU_WAKEUP(RTC) |
 			     PRCMU_WAKEUP(ABB));
 
 	return cpuidle_register(&ux500_idle_driver, NULL);
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 216a00bad5d70..0189dfdcbbe11 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -528,13 +528,17 @@ source "drivers/crypto/intel/Kconfig"
 
 config CRYPTO_DEV_QCE
 	tristate "Qualcomm crypto engine accelerator"
-	depends on ARCH_QCOM || COMPILE_TEST
+	depends on (BROKEN && ARCH_QCOM) || COMPILE_TEST
 	depends on HAS_IOMEM
 	help
 	  This driver supports Qualcomm crypto engine accelerator
 	  hardware. To compile this driver as a module, choose M here. The
 	  module will be called qcrypto.
 
+	  This driver does not have exclusive access to the
+	  hardware, causing races with the secure world.  It
+	  is also slower than the CPU.
+
 config CRYPTO_DEV_QCE_SKCIPHER
 	bool
 	depends on CRYPTO_DEV_QCE
@@ -637,18 +641,6 @@ config CRYPTO_DEV_QCE_SW_MAX_LEN
 	  always processed by the software fallback, and all DES requests
 	  are done by the hardware.
 
-config CRYPTO_DEV_QCOM_RNG
-	tristate "Qualcomm Random Number Generator Driver"
-	depends on ARCH_QCOM || COMPILE_TEST
-	depends on HW_RANDOM
-	select CRYPTO_RNG
-	help
-	  This driver provides support for the Random Number
-	  Generator hardware found on Qualcomm SoCs.
-
-	  To compile this driver as a module, choose M here. The
-	  module will be called qcom-rng. If unsure, say N.
-
 config CRYPTO_DEV_IMGTEC_HASH
 	tristate "Imagination Technologies hardware hash accelerator"
 	depends on MIPS || COMPILE_TEST
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 5a950c7abc393..2c33b83f3cfa6 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -27,7 +27,6 @@ obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o
 obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o
 obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/
 obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/
-obj-$(CONFIG_CRYPTO_DEV_QCOM_RNG) += qcom-rng.o
 obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rockchip/
 obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o
 obj-$(CONFIG_CRYPTO_DEV_SA2UL) += sa2ul.o
diff --git a/drivers/crypto/allwinner/Kconfig b/drivers/crypto/allwinner/Kconfig
index 06ea0e9fe6f22..d86ae005fbe2e 100644
--- a/drivers/crypto/allwinner/Kconfig
+++ b/drivers/crypto/allwinner/Kconfig
@@ -70,14 +70,6 @@ config CRYPTO_DEV_SUN8I_CE_HASH
 	help
 	  Say y to enable support for hash algorithms.
 
-config CRYPTO_DEV_SUN8I_CE_PRNG
-	bool "Support for Allwinner Crypto Engine PRNG"
-	depends on CRYPTO_DEV_SUN8I_CE
-	select CRYPTO_RNG
-	help
-	  Select this option if you want to provide kernel-side support for
-	  the Pseudo-Random Number Generator found in the Crypto Engine.
-
 config CRYPTO_DEV_SUN8I_CE_TRNG
 	bool "Support for Allwinner Crypto Engine TRNG"
 	depends on CRYPTO_DEV_SUN8I_CE
@@ -113,14 +105,6 @@ config CRYPTO_DEV_SUN8I_SS_DEBUG
 	  This will create /sys/kernel/debug/sun8i-ss/stats for displaying
 	  the number of requests per flow and per algorithm.
 
-config CRYPTO_DEV_SUN8I_SS_PRNG
-	bool "Support for Allwinner Security System PRNG"
-	depends on CRYPTO_DEV_SUN8I_SS
-	select CRYPTO_RNG
-	help
-	  Select this option if you want to provide kernel-side support for
-	  the Pseudo-Random Number Generator found in the Security System.
-
 config CRYPTO_DEV_SUN8I_SS_HASH
 	bool "Enable support for hash on sun8i-ss"
 	depends on CRYPTO_DEV_SUN8I_SS
diff --git a/drivers/crypto/allwinner/sun8i-ce/Makefile b/drivers/crypto/allwinner/sun8i-ce/Makefile
index 0842eb2d9408d..ea708b427e2e4 100644
--- a/drivers/crypto/allwinner/sun8i-ce/Makefile
+++ b/drivers/crypto/allwinner/sun8i-ce/Makefile
@@ -1,5 +1,4 @@
 obj-$(CONFIG_CRYPTO_DEV_SUN8I_CE) += sun8i-ce.o
 sun8i-ce-y += sun8i-ce-core.o sun8i-ce-cipher.o
 sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_HASH) += sun8i-ce-hash.o
-sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG) += sun8i-ce-prng.o
 sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG) += sun8i-ce-trng.o
diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c
index f3b58ed6aed0d..c6402e87f8a07 100644
--- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c
+++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c
@@ -12,7 +12,6 @@
 
 #include <crypto/engine.h>
 #include <crypto/internal/hash.h>
-#include <crypto/internal/rng.h>
 #include <crypto/internal/skcipher.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
@@ -49,7 +48,6 @@ static const struct ce_variant ce_h3_variant = {
 		{ "mod", 50000000, 0 },
 		},
 	.esr = ESR_H3,
-	.prng = CE_ALG_PRNG,
 	.trng = CE_ID_NOTSUPP,
 };
 
@@ -66,7 +64,6 @@ static const struct ce_variant ce_h5_variant = {
 		{ "mod", 300000000, 0 },
 		},
 	.esr = ESR_H5,
-	.prng = CE_ALG_PRNG,
 	.trng = CE_ID_NOTSUPP,
 };
 
@@ -80,7 +77,6 @@ static const struct ce_variant ce_h6_variant = {
 	},
 	.cipher_t_dlen_in_bytes = true,
 	.hash_t_dlen_in_bits = true,
-	.prng_t_dlen_in_bytes = true,
 	.trng_t_dlen_in_bytes = true,
 	.ce_clks = {
 		{ "bus", 0, 200000000 },
@@ -88,7 +84,6 @@ static const struct ce_variant ce_h6_variant = {
 		{ "ram", 0, 400000000 },
 		},
 	.esr = ESR_H6,
-	.prng = CE_ALG_PRNG_V2,
 	.trng = CE_ALG_TRNG_V2,
 };
 
@@ -102,7 +97,6 @@ static const struct ce_variant ce_h616_variant = {
 	},
 	.cipher_t_dlen_in_bytes = true,
 	.hash_t_dlen_in_bits = true,
-	.prng_t_dlen_in_bytes = true,
 	.trng_t_dlen_in_bytes = true,
 	.needs_word_addresses = true,
 	.ce_clks = {
@@ -112,7 +106,6 @@ static const struct ce_variant ce_h616_variant = {
 		{ "trng", 0, 0 },
 		},
 	.esr = ESR_H6,
-	.prng = CE_ALG_PRNG_V2,
 	.trng = CE_ALG_TRNG_V2,
 };
 
@@ -129,7 +122,6 @@ static const struct ce_variant ce_a64_variant = {
 		{ "mod", 300000000, 0 },
 		},
 	.esr = ESR_A64,
-	.prng = CE_ALG_PRNG,
 	.trng = CE_ID_NOTSUPP,
 };
 
@@ -148,7 +140,6 @@ static const struct ce_variant ce_d1_variant = {
 		{ "trng", 0, 0 },
 		},
 	.esr = ESR_D1,
-	.prng = CE_ALG_PRNG,
 	.trng = CE_ALG_TRNG,
 };
 
@@ -165,7 +156,6 @@ static const struct ce_variant ce_r40_variant = {
 		{ "mod", 300000000, 0 },
 		},
 	.esr = ESR_R40,
-	.prng = CE_ALG_PRNG,
 	.trng = CE_ID_NOTSUPP,
 };
 
@@ -614,25 +604,6 @@ static struct sun8i_ce_alg_template ce_algs[] = {
 	},
 },
 #endif
-#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG
-{
-	.type = CRYPTO_ALG_TYPE_RNG,
-	.alg.rng = {
-		.base = {
-			.cra_name		= "stdrng",
-			.cra_driver_name	= "sun8i-ce-prng",
-			.cra_priority		= 300,
-			.cra_ctxsize		= sizeof(struct sun8i_ce_rng_tfm_ctx),
-			.cra_module		= THIS_MODULE,
-			.cra_init		= sun8i_ce_prng_init,
-			.cra_exit		= sun8i_ce_prng_exit,
-		},
-		.generate               = sun8i_ce_prng_generate,
-		.seed                   = sun8i_ce_prng_seed,
-		.seedsize               = PRNG_SEED_SIZE,
-	}
-},
-#endif
 };
 
 static int sun8i_ce_debugfs_show(struct seq_file *seq, void *v)
@@ -693,14 +664,6 @@ static int sun8i_ce_debugfs_show(struct seq_file *seq, void *v)
 			seq_printf(seq, "\tFallback due to SG numbers: %lu\n",
 				   ce_algs[i].stat_fb_maxsg);
 			break;
-#endif
-#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG
-		case CRYPTO_ALG_TYPE_RNG:
-			seq_printf(seq, "%s %s reqs=%lu bytes=%lu\n",
-				   ce_algs[i].alg.rng.base.cra_driver_name,
-				   ce_algs[i].alg.rng.base.cra_name,
-				   ce_algs[i].stat_req, ce_algs[i].stat_bytes);
-			break;
 #endif
 		}
 	}
@@ -930,25 +893,6 @@ static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce)
 				return err;
 			}
 			break;
-#endif
-#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG
-		case CRYPTO_ALG_TYPE_RNG:
-			if (ce->variant->prng == CE_ID_NOTSUPP) {
-				dev_info(ce->dev,
-					 "DEBUG: Algo of %s not supported\n",
-					 ce_algs[i].alg.rng.base.cra_name);
-				ce_algs[i].ce = NULL;
-				break;
-			}
-			dev_info(ce->dev, "Register %s\n",
-				 ce_algs[i].alg.rng.base.cra_name);
-			err = crypto_register_rng(&ce_algs[i].alg.rng);
-			if (err) {
-				dev_err(ce->dev, "Fail to register %s\n",
-					ce_algs[i].alg.rng.base.cra_name);
-				ce_algs[i].ce = NULL;
-			}
-			break;
 #endif
 		default:
 			ce_algs[i].ce = NULL;
@@ -977,13 +921,6 @@ static void sun8i_ce_unregister_algs(struct sun8i_ce_dev *ce)
 				 ce_algs[i].alg.hash.base.halg.base.cra_name);
 			crypto_engine_unregister_ahash(&ce_algs[i].alg.hash);
 			break;
-#endif
-#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG
-		case CRYPTO_ALG_TYPE_RNG:
-			dev_info(ce->dev, "Unregister %d %s\n", i,
-				 ce_algs[i].alg.rng.base.cra_name);
-			crypto_unregister_rng(&ce_algs[i].alg.rng);
-			break;
 #endif
 		}
 	}
diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c
deleted file mode 100644
index d0a1ac66738bf..0000000000000
--- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c
+++ /dev/null
@@ -1,159 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * sun8i-ce-prng.c - hardware cryptographic offloader for
- * Allwinner H3/A64/H5/H2+/H6/R40 SoC
- *
- * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com>
- *
- * This file handle the PRNG
- *
- * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst
- */
-#include "sun8i-ce.h"
-#include <linux/dma-mapping.h>
-#include <linux/pm_runtime.h>
-#include <crypto/internal/rng.h>
-
-int sun8i_ce_prng_init(struct crypto_tfm *tfm)
-{
-	struct sun8i_ce_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	memset(ctx, 0, sizeof(struct sun8i_ce_rng_tfm_ctx));
-	return 0;
-}
-
-void sun8i_ce_prng_exit(struct crypto_tfm *tfm)
-{
-	struct sun8i_ce_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	kfree_sensitive(ctx->seed);
-	ctx->seed = NULL;
-	ctx->slen = 0;
-}
-
-int sun8i_ce_prng_seed(struct crypto_rng *tfm, const u8 *seed,
-		       unsigned int slen)
-{
-	struct sun8i_ce_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm);
-
-	if (ctx->seed && ctx->slen != slen) {
-		kfree_sensitive(ctx->seed);
-		ctx->slen = 0;
-		ctx->seed = NULL;
-	}
-	if (!ctx->seed)
-		ctx->seed = kmalloc(slen, GFP_KERNEL | GFP_DMA);
-	if (!ctx->seed)
-		return -ENOMEM;
-
-	memcpy(ctx->seed, seed, slen);
-	ctx->slen = slen;
-
-	return 0;
-}
-
-int sun8i_ce_prng_generate(struct crypto_rng *tfm, const u8 *src,
-			   unsigned int slen, u8 *dst, unsigned int dlen)
-{
-	struct sun8i_ce_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm);
-	struct rng_alg *alg = crypto_rng_alg(tfm);
-	struct sun8i_ce_alg_template *algt;
-	struct sun8i_ce_dev *ce;
-	dma_addr_t dma_iv, dma_dst;
-	int err = 0;
-	int flow = 3;
-	unsigned int todo;
-	struct sun8i_ce_flow *chan;
-	struct ce_task *cet;
-	u32 common, sym;
-	void *d;
-
-	algt = container_of(alg, struct sun8i_ce_alg_template, alg.rng);
-	ce = algt->ce;
-
-	if (ctx->slen == 0) {
-		dev_err(ce->dev, "not seeded\n");
-		return -EINVAL;
-	}
-
-	/* we want dlen + seedsize rounded up to a multiple of PRNG_DATA_SIZE */
-	todo = dlen + ctx->slen + PRNG_DATA_SIZE * 2;
-	todo -= todo % PRNG_DATA_SIZE;
-
-	d = kzalloc(todo, GFP_KERNEL | GFP_DMA);
-	if (!d) {
-		err = -ENOMEM;
-		goto err_mem;
-	}
-
-	dev_dbg(ce->dev, "%s PRNG slen=%u dlen=%u todo=%u multi=%u\n", __func__,
-		slen, dlen, todo, todo / PRNG_DATA_SIZE);
-
-#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
-	algt->stat_req++;
-	algt->stat_bytes += todo;
-#endif
-
-	dma_iv = dma_map_single(ce->dev, ctx->seed, ctx->slen, DMA_TO_DEVICE);
-	if (dma_mapping_error(ce->dev, dma_iv)) {
-		dev_err(ce->dev, "Cannot DMA MAP IV\n");
-		err = -EFAULT;
-		goto err_iv;
-	}
-
-	dma_dst = dma_map_single(ce->dev, d, todo, DMA_FROM_DEVICE);
-	if (dma_mapping_error(ce->dev, dma_dst)) {
-		dev_err(ce->dev, "Cannot DMA MAP DST\n");
-		err = -EFAULT;
-		goto err_dst;
-	}
-
-	err = pm_runtime_resume_and_get(ce->dev);
-	if (err < 0)
-		goto err_pm;
-
-	mutex_lock(&ce->rnglock);
-	chan = &ce->chanlist[flow];
-
-	cet = &chan->tl[0];
-	memset(cet, 0, sizeof(struct ce_task));
-
-	cet->t_id = cpu_to_le32(flow);
-	common = ce->variant->prng | CE_COMM_INT;
-	cet->t_common_ctl = cpu_to_le32(common);
-
-	/* recent CE (H6) need length in bytes, in word otherwise */
-	if (ce->variant->prng_t_dlen_in_bytes)
-		cet->t_dlen = cpu_to_le32(todo);
-	else
-		cet->t_dlen = cpu_to_le32(todo / 4);
-
-	sym = PRNG_LD;
-	cet->t_sym_ctl = cpu_to_le32(sym);
-	cet->t_asym_ctl = 0;
-
-	cet->t_key = desc_addr_val_le32(ce, dma_iv);
-	cet->t_iv = desc_addr_val_le32(ce, dma_iv);
-
-	cet->t_dst[0].addr = desc_addr_val_le32(ce, dma_dst);
-	cet->t_dst[0].len = cpu_to_le32(todo / 4);
-
-	err = sun8i_ce_run_task(ce, 3, "PRNG");
-	mutex_unlock(&ce->rnglock);
-
-	pm_runtime_put(ce->dev);
-
-err_pm:
-	dma_unmap_single(ce->dev, dma_dst, todo, DMA_FROM_DEVICE);
-err_dst:
-	dma_unmap_single(ce->dev, dma_iv, ctx->slen, DMA_TO_DEVICE);
-
-	if (!err) {
-		memcpy(dst, d, dlen);
-		memcpy(ctx->seed, d + dlen, ctx->slen);
-	}
-err_iv:
-	kfree_sensitive(d);
-err_mem:
-	return err;
-}
diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h
index 71f5a0cd3d458..468d99bf5bf66 100644
--- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h
+++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h
@@ -15,7 +15,6 @@
 #include <linux/hw_random.h>
 #include <crypto/internal/hash.h>
 #include <crypto/md5.h>
-#include <crypto/rng.h>
 #include <crypto/sha1.h>
 #include <crypto/sha2.h>
 
@@ -58,9 +57,7 @@
 #define CE_ALG_SHA384           20
 #define CE_ALG_SHA512           21
 #define CE_ALG_TRNG		48
-#define CE_ALG_PRNG		49
 #define CE_ALG_TRNG_V2		0x1c
-#define CE_ALG_PRNG_V2		0x1d
 
 /* Used in ce_variant */
 #define CE_ID_NOTSUPP		0xFF
@@ -96,10 +93,6 @@
 #define ESR_H6	4
 #define ESR_D1	5
 
-#define PRNG_DATA_SIZE (160 / 8)
-#define PRNG_SEED_SIZE DIV_ROUND_UP(175, 8)
-#define PRNG_LD BIT(17)
-
 #define CE_DIE_ID_SHIFT	16
 #define CE_DIE_ID_MASK	0x07
 
@@ -136,13 +129,10 @@ struct ce_clock {
  *				bytes or words
  * @hash_t_dlen_in_bytes:	Does the request size for hash is in
  *				bits or words
- * @prng_t_dlen_in_bytes:	Does the request size for PRNG is in
- *				bytes or words
  * @trng_t_dlen_in_bytes:	Does the request size for TRNG is in
  *				bytes or words
  * @ce_clks:	list of clocks needed by this variant
  * @esr:	The type of error register
- * @prng:	The CE_ALG_XXX value for the PRNG
  * @trng:	The CE_ALG_XXX value for the TRNG
  */
 struct ce_variant {
@@ -151,12 +141,10 @@ struct ce_variant {
 	u32 op_mode[CE_ID_OP_MAX];
 	bool cipher_t_dlen_in_bytes;
 	bool hash_t_dlen_in_bits;
-	bool prng_t_dlen_in_bytes;
 	bool trng_t_dlen_in_bytes;
 	bool needs_word_addresses;
 	struct ce_clock ce_clks[CE_MAX_CLOCKS];
 	int esr;
-	unsigned char prng;
 	unsigned char trng;
 };
 
@@ -327,16 +315,6 @@ struct sun8i_ce_hash_reqctx {
 	struct ahash_request fallback_req; // keep at the end
 };
 
-/*
- * struct sun8i_ce_prng_ctx - context for PRNG TFM
- * @seed:	The seed to use
- * @slen:	The size of the seed
- */
-struct sun8i_ce_rng_tfm_ctx {
-	void *seed;
-	unsigned int slen;
-};
-
 /*
  * struct sun8i_ce_alg_template - crypto_alg template
  * @type:		the CRYPTO_ALG_TYPE for this template
@@ -357,7 +335,6 @@ struct sun8i_ce_alg_template {
 	union {
 		struct skcipher_engine_alg skcipher;
 		struct ahash_engine_alg hash;
-		struct rng_alg rng;
 	} alg;
 	unsigned long stat_req;
 	unsigned long stat_fb;
@@ -398,11 +375,5 @@ int sun8i_ce_hash_finup(struct ahash_request *areq);
 int sun8i_ce_hash_digest(struct ahash_request *areq);
 int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq);
 
-int sun8i_ce_prng_generate(struct crypto_rng *tfm, const u8 *src,
-			   unsigned int slen, u8 *dst, unsigned int dlen);
-int sun8i_ce_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen);
-void sun8i_ce_prng_exit(struct crypto_tfm *tfm);
-int sun8i_ce_prng_init(struct crypto_tfm *tfm);
-
 int sun8i_ce_hwrng_register(struct sun8i_ce_dev *ce);
 void sun8i_ce_hwrng_unregister(struct sun8i_ce_dev *ce);
diff --git a/drivers/crypto/allwinner/sun8i-ss/Makefile b/drivers/crypto/allwinner/sun8i-ss/Makefile
index aabfd893c8174..2d6458a42e583 100644
--- a/drivers/crypto/allwinner/sun8i-ss/Makefile
+++ b/drivers/crypto/allwinner/sun8i-ss/Makefile
@@ -1,4 +1,3 @@
 obj-$(CONFIG_CRYPTO_DEV_SUN8I_SS) += sun8i-ss.o
 sun8i-ss-y += sun8i-ss-core.o sun8i-ss-cipher.o
-sun8i-ss-$(CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG) += sun8i-ss-prng.o
 sun8i-ss-$(CONFIG_CRYPTO_DEV_SUN8I_SS_HASH) += sun8i-ss-hash.o
diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c
index 59c9bc45ec0f2..0b22fcddb882c 100644
--- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c
+++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c
@@ -11,7 +11,6 @@
  */
 
 #include <crypto/engine.h>
-#include <crypto/internal/rng.h>
 #include <crypto/internal/skcipher.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
@@ -283,25 +282,6 @@ static struct sun8i_ss_alg_template ss_algs[] = {
 		.do_one_request = sun8i_ss_handle_cipher_request,
 	},
 },
-#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG
-{
-	.type = CRYPTO_ALG_TYPE_RNG,
-	.alg.rng = {
-		.base = {
-			.cra_name		= "stdrng",
-			.cra_driver_name	= "sun8i-ss-prng",
-			.cra_priority		= 300,
-			.cra_ctxsize = sizeof(struct sun8i_ss_rng_tfm_ctx),
-			.cra_module		= THIS_MODULE,
-			.cra_init		= sun8i_ss_prng_init,
-			.cra_exit		= sun8i_ss_prng_exit,
-		},
-		.generate               = sun8i_ss_prng_generate,
-		.seed                   = sun8i_ss_prng_seed,
-		.seedsize               = PRNG_SEED_SIZE,
-	}
-},
-#endif
 #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH
 {	.type = CRYPTO_ALG_TYPE_AHASH,
 	.ss_algo_id = SS_ID_HASH_MD5,
@@ -501,14 +481,6 @@ static int sun8i_ss_debugfs_show(struct seq_file *seq, void *v)
 			seq_printf(seq, "\tFallback due to SG numbers: %lu\n",
 				   ss_algs[i].stat_fb_sgnum);
 			break;
-#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG
-		case CRYPTO_ALG_TYPE_RNG:
-			seq_printf(seq, "%s %s reqs=%lu tsize=%lu\n",
-				   ss_algs[i].alg.rng.base.cra_driver_name,
-				   ss_algs[i].alg.rng.base.cra_name,
-				   ss_algs[i].stat_req, ss_algs[i].stat_bytes);
-			break;
-#endif
 #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH
 		case CRYPTO_ALG_TYPE_AHASH:
 			seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n",
@@ -711,16 +683,6 @@ static int sun8i_ss_register_algs(struct sun8i_ss_dev *ss)
 				return err;
 			}
 			break;
-#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG
-		case CRYPTO_ALG_TYPE_RNG:
-			err = crypto_register_rng(&ss_algs[i].alg.rng);
-			if (err) {
-				dev_err(ss->dev, "Fail to register %s\n",
-					ss_algs[i].alg.rng.base.cra_name);
-				ss_algs[i].ss = NULL;
-			}
-			break;
-#endif
 #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH
 		case CRYPTO_ALG_TYPE_AHASH:
 			id = ss_algs[i].ss_algo_id;
@@ -764,13 +726,6 @@ static void sun8i_ss_unregister_algs(struct sun8i_ss_dev *ss)
 				 ss_algs[i].alg.skcipher.base.base.cra_name);
 			crypto_engine_unregister_skcipher(&ss_algs[i].alg.skcipher);
 			break;
-#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG
-		case CRYPTO_ALG_TYPE_RNG:
-			dev_info(ss->dev, "Unregister %d %s\n", i,
-				 ss_algs[i].alg.rng.base.cra_name);
-			crypto_unregister_rng(&ss_algs[i].alg.rng);
-			break;
-#endif
 #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH
 		case CRYPTO_ALG_TYPE_AHASH:
 			dev_info(ss->dev, "Unregister %d %s\n", i,
diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c
deleted file mode 100644
index a923cfc6553f4..0000000000000
--- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c
+++ /dev/null
@@ -1,177 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * sun8i-ss-prng.c - hardware cryptographic offloader for
- * Allwinner A80/A83T SoC
- *
- * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com>
- *
- * This file handle the PRNG found in the SS
- *
- * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst
- */
-#include "sun8i-ss.h"
-#include <linux/dma-mapping.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/pm_runtime.h>
-#include <crypto/internal/rng.h>
-
-int sun8i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed,
-		       unsigned int slen)
-{
-	struct sun8i_ss_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm);
-
-	if (ctx->seed && ctx->slen != slen) {
-		kfree_sensitive(ctx->seed);
-		ctx->slen = 0;
-		ctx->seed = NULL;
-	}
-	if (!ctx->seed)
-		ctx->seed = kmalloc(slen, GFP_KERNEL);
-	if (!ctx->seed)
-		return -ENOMEM;
-
-	memcpy(ctx->seed, seed, slen);
-	ctx->slen = slen;
-
-	return 0;
-}
-
-int sun8i_ss_prng_init(struct crypto_tfm *tfm)
-{
-	struct sun8i_ss_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	memset(ctx, 0, sizeof(struct sun8i_ss_rng_tfm_ctx));
-	return 0;
-}
-
-void sun8i_ss_prng_exit(struct crypto_tfm *tfm)
-{
-	struct sun8i_ss_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	kfree_sensitive(ctx->seed);
-	ctx->seed = NULL;
-	ctx->slen = 0;
-}
-
-int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src,
-			   unsigned int slen, u8 *dst, unsigned int dlen)
-{
-	struct sun8i_ss_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm);
-	struct rng_alg *alg = crypto_rng_alg(tfm);
-	struct sun8i_ss_alg_template *algt;
-	unsigned int todo_with_padding;
-	struct sun8i_ss_dev *ss;
-	dma_addr_t dma_iv, dma_dst;
-	unsigned int todo;
-	int err = 0;
-	int flow;
-	void *d;
-	u32 v;
-
-	algt = container_of(alg, struct sun8i_ss_alg_template, alg.rng);
-	ss = algt->ss;
-
-	if (ctx->slen == 0) {
-		dev_err(ss->dev, "The PRNG is not seeded\n");
-		return -EINVAL;
-	}
-
-	/* The SS does not give an updated seed, so we need to get a new one.
-	 * So we will ask for an extra PRNG_SEED_SIZE data.
-	 * We want dlen + seedsize rounded up to a multiple of PRNG_DATA_SIZE
-	 */
-	todo = dlen + PRNG_SEED_SIZE + PRNG_DATA_SIZE;
-	todo -= todo % PRNG_DATA_SIZE;
-
-	todo_with_padding = ALIGN(todo, dma_get_cache_alignment());
-	if (todo_with_padding < todo || todo < dlen)
-		return -EOVERFLOW;
-
-	d = kzalloc(todo_with_padding, GFP_KERNEL);
-	if (!d)
-		return -ENOMEM;
-
-	flow = sun8i_ss_get_engine_number(ss);
-
-#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
-	algt->stat_req++;
-	algt->stat_bytes += todo;
-#endif
-
-	v = SS_ALG_PRNG | SS_PRNG_CONTINUE | SS_START;
-	if (flow)
-		v |= SS_FLOW1;
-	else
-		v |= SS_FLOW0;
-
-	dma_iv = dma_map_single(ss->dev, ctx->seed, ctx->slen, DMA_TO_DEVICE);
-	if (dma_mapping_error(ss->dev, dma_iv)) {
-		dev_err(ss->dev, "Cannot DMA MAP IV\n");
-		err = -EFAULT;
-		goto err_free;
-	}
-
-	dma_dst = dma_map_single(ss->dev, d, todo, DMA_FROM_DEVICE);
-	if (dma_mapping_error(ss->dev, dma_dst)) {
-		dev_err(ss->dev, "Cannot DMA MAP DST\n");
-		err = -EFAULT;
-		goto err_iv;
-	}
-
-	err = pm_runtime_resume_and_get(ss->dev);
-	if (err < 0)
-		goto err_pm;
-	err = 0;
-
-	mutex_lock(&ss->mlock);
-	writel(dma_iv, ss->base + SS_IV_ADR_REG);
-	/* the PRNG act badly (failing rngtest) without SS_KEY_ADR_REG set */
-	writel(dma_iv, ss->base + SS_KEY_ADR_REG);
-	writel(dma_dst, ss->base + SS_DST_ADR_REG);
-	writel(todo / 4, ss->base + SS_LEN_ADR_REG);
-
-	reinit_completion(&ss->flows[flow].complete);
-	ss->flows[flow].status = 0;
-	/* Be sure all data is written before enabling the task */
-	wmb();
-
-	writel(v, ss->base + SS_CTL_REG);
-
-	wait_for_completion_interruptible_timeout(&ss->flows[flow].complete,
-						  msecs_to_jiffies(todo));
-	if (ss->flows[flow].status == 0) {
-		dev_err(ss->dev, "DMA timeout for PRNG (size=%u)\n", todo);
-		err = -EFAULT;
-	}
-	/* Since cipher and hash use the linux/cryptoengine and that we have
-	 * a cryptoengine per flow, we are sure that they will issue only one
-	 * request per flow.
-	 * Since the cryptoengine wait for completion before submitting a new
-	 * one, the mlock could be left just after the final writel.
-	 * But cryptoengine cannot handle crypto_rng, so we need to be sure
-	 * nothing will use our flow.
-	 * The easiest way is to grab mlock until the hardware end our requests.
-	 * We could have used a per flow lock, but this would increase
-	 * complexity.
-	 * The drawback is that no request could be handled for the other flow.
-	 */
-	mutex_unlock(&ss->mlock);
-
-	pm_runtime_put(ss->dev);
-
-err_pm:
-	dma_unmap_single(ss->dev, dma_dst, todo, DMA_FROM_DEVICE);
-err_iv:
-	dma_unmap_single(ss->dev, dma_iv, ctx->slen, DMA_TO_DEVICE);
-
-	if (!err) {
-		memcpy(dst, d, dlen);
-		/* Update seed */
-		memcpy(ctx->seed, d + dlen, ctx->slen);
-	}
-err_free:
-	kfree_sensitive(d);
-
-	return err;
-}
diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h
index 3fc86225edaf3..289fb22abfa27 100644
--- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h
+++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h
@@ -8,7 +8,6 @@
 #include <crypto/aes.h>
 #include <crypto/des.h>
 #include <crypto/engine.h>
-#include <crypto/rng.h>
 #include <crypto/skcipher.h>
 #include <linux/atomic.h>
 #include <linux/debugfs.h>
@@ -27,7 +26,6 @@
 #define SS_ALG_DES		(1 << 2)
 #define SS_ALG_3DES		(2 << 2)
 #define SS_ALG_MD5		(3 << 2)
-#define SS_ALG_PRNG		(4 << 2)
 #define SS_ALG_SHA1		(6 << 2)
 #define SS_ALG_SHA224		(7 << 2)
 #define SS_ALG_SHA256		(8 << 2)
@@ -68,8 +66,6 @@
 #define SS_FLOW0	BIT(30)
 #define SS_FLOW1	BIT(31)
 
-#define SS_PRNG_CONTINUE	BIT(18)
-
 #define MAX_SG 8
 
 #define MAXFLOW 2
@@ -79,9 +75,6 @@
 #define SS_DIE_ID_SHIFT	20
 #define SS_DIE_ID_MASK	0x07
 
-#define PRNG_DATA_SIZE (160 / 8)
-#define PRNG_SEED_SIZE DIV_ROUND_UP(175, 8)
-
 #define MAX_PAD_SIZE 4096
 
 /*
@@ -213,16 +206,6 @@ struct sun8i_cipher_tfm_ctx {
 	struct crypto_skcipher *fallback_tfm;
 };
 
-/*
- * struct sun8i_ss_prng_ctx - context for PRNG TFM
- * @seed:	The seed to use
- * @slen:	The size of the seed
- */
-struct sun8i_ss_rng_tfm_ctx {
-	void *seed;
-	unsigned int slen;
-};
-
 /*
  * struct sun8i_ss_hash_tfm_ctx - context for an ahash TFM
  * @fallback_tfm:	pointer to the fallback TFM
@@ -274,7 +257,6 @@ struct sun8i_ss_alg_template {
 	struct sun8i_ss_dev *ss;
 	union {
 		struct skcipher_engine_alg skcipher;
-		struct rng_alg rng;
 		struct ahash_engine_alg hash;
 	} alg;
 	unsigned long stat_req;
@@ -300,11 +282,6 @@ int sun8i_ss_skencrypt(struct skcipher_request *areq);
 int sun8i_ss_get_engine_number(struct sun8i_ss_dev *ss);
 
 int sun8i_ss_run_task(struct sun8i_ss_dev *ss, struct sun8i_cipher_req_ctx *rctx, const char *name);
-int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src,
-			   unsigned int slen, u8 *dst, unsigned int dlen);
-int sun8i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen);
-int sun8i_ss_prng_init(struct crypto_tfm *tfm);
-void sun8i_ss_prng_exit(struct crypto_tfm *tfm);
 
 int sun8i_ss_hash_init_tfm(struct crypto_ahash *tfm);
 void sun8i_ss_hash_exit_tfm(struct crypto_ahash *tfm);
diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c
index 001da785af079..0271b5e4d9230 100644
--- a/drivers/crypto/amcc/crypto4xx_core.c
+++ b/drivers/crypto/amcc/crypto4xx_core.c
@@ -1251,6 +1251,10 @@ static int crypto4xx_probe(struct platform_device *ofdev)
 	if (!core_dev->dev)
 		return -ENOMEM;
 
+	core_dev->dev->ce_base = devm_platform_ioremap_resource(ofdev, 0);
+	if (IS_ERR(core_dev->dev->ce_base))
+		return PTR_ERR(core_dev->dev->ce_base);
+
 	/*
 	 * Older version of 460EX/GT have a hardware bug.
 	 * Hence they do not support H/W based security intr coalescing
@@ -1286,13 +1290,6 @@ static int crypto4xx_probe(struct platform_device *ofdev)
 	tasklet_init(&core_dev->tasklet, crypto4xx_bh_tasklet_cb,
 		     (unsigned long) dev);
 
-	core_dev->dev->ce_base = devm_platform_ioremap_resource(ofdev, 0);
-	if (IS_ERR(core_dev->dev->ce_base)) {
-		dev_err(&ofdev->dev, "failed to ioremap resource");
-		rc = PTR_ERR(core_dev->dev->ce_base);
-		goto err_build_sdr;
-	}
-
 	/* Register for Crypto isr, Crypto Engine IRQ */
 	core_dev->irq = platform_get_irq(ofdev, 0);
 	if (core_dev->irq < 0) {
diff --git a/drivers/crypto/aspeed/aspeed-acry.c b/drivers/crypto/aspeed/aspeed-acry.c
index 5993bcba97163..301612556a769 100644
--- a/drivers/crypto/aspeed/aspeed-acry.c
+++ b/drivers/crypto/aspeed/aspeed-acry.c
@@ -728,7 +728,7 @@ static int aspeed_acry_probe(struct platform_device *pdev)
 	/* Get irq number and register it */
 	acry_dev->irq = platform_get_irq(pdev, 0);
 	if (acry_dev->irq < 0)
-		return -ENXIO;
+		return acry_dev->irq;
 
 	rc = devm_request_irq(dev, acry_dev->irq, aspeed_acry_irq, 0,
 			      dev_name(dev), acry_dev);
diff --git a/drivers/crypto/aspeed/aspeed-hace.c b/drivers/crypto/aspeed/aspeed-hace.c
index 3fe644bfe0373..1f9afa002ae8c 100644
--- a/drivers/crypto/aspeed/aspeed-hace.c
+++ b/drivers/crypto/aspeed/aspeed-hace.c
@@ -127,7 +127,7 @@ static int aspeed_hace_probe(struct platform_device *pdev)
 	/* Get irq number and register it */
 	hace_dev->irq = platform_get_irq(pdev, 0);
 	if (hace_dev->irq < 0)
-		return -ENXIO;
+		return hace_dev->irq;
 
 	rc = devm_request_irq(&pdev->dev, hace_dev->irq, aspeed_hace_irq, 0,
 			      dev_name(&pdev->dev), hace_dev);
diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c
index f79d7c4565462..30acfd937d54b 100644
--- a/drivers/crypto/atmel-aes.c
+++ b/drivers/crypto/atmel-aes.c
@@ -794,7 +794,6 @@ static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd,
 	struct dma_slave_config config;
 	dma_async_tx_callback callback;
 	struct atmel_aes_dma *dma;
-	int err;
 
 	memset(&config, 0, sizeof(config));
 	config.src_addr_width = addr_width;
@@ -819,12 +818,9 @@ static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd,
 		return -EINVAL;
 	}
 
-	err = dmaengine_slave_config(dma->chan, &config);
-	if (err)
-		return err;
-
-	desc = dmaengine_prep_slave_sg(dma->chan, dma->sg, dma->sg_len, dir,
-				       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+	desc = dmaengine_prep_config_sg(dma->chan, dma->sg, dma->sg_len, dir,
+					DMA_PREP_INTERRUPT | DMA_CTRL_ACK,
+					&config);
 	if (!desc)
 		return -ENOMEM;
 
diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c
index 9da9dd6585dfc..4add3b2ddd0b6 100644
--- a/drivers/crypto/atmel-ecc.c
+++ b/drivers/crypto/atmel-ecc.c
@@ -27,21 +27,18 @@ static struct atmel_ecc_driver_data driver_data;
 
 /**
  * struct atmel_ecdh_ctx - transformation context
- * @client     : pointer to i2c client device
- * @fallback   : used for unsupported curves or when user wants to use its own
- *               private key.
- * @public_key : generated when calling set_secret(). It's the responsibility
- *               of the user to not call set_secret() while
- *               generate_public_key() or compute_shared_secret() are in flight.
- * @curve_id   : elliptic curve id
- * @do_fallback: true when the device doesn't support the curve or when the user
- *               wants to use its own private key.
+ * @client: I2C client device
+ * @fallback: ECDH fallback used for caller-provided private keys
+ * @public_key: cached public key for the device-generated private key
+ * @do_fallback: true when ECDH operations should use @fallback
+ *
+ * The caller must not invoke set_secret() while generate_public_key()
+ * or compute_shared_secret() are in flight.
  */
 struct atmel_ecdh_ctx {
 	struct i2c_client *client;
 	struct crypto_kpp *fallback;
 	const u8 *public_key;
-	unsigned int curve_id;
 	bool do_fallback;
 };
 
@@ -55,7 +52,7 @@ static void atmel_ecdh_done(struct atmel_i2c_work_data *work_data, void *areq,
 	if (status)
 		goto free_work_data;
 
-	/* might want less than we've got */
+	/* copy only as much as requested, capped at 32 bytes */
 	n_sz = min(ATMEL_ECC_NIST_P256_N_SIZE, req->dst_len);
 
 	/* copy the shared secret */
@@ -64,15 +61,15 @@ static void atmel_ecdh_done(struct atmel_i2c_work_data *work_data, void *areq,
 	if (copied != n_sz)
 		status = -EINVAL;
 
-	/* fall through */
 free_work_data:
 	kfree_sensitive(work_data);
 	kpp_request_complete(req, status);
 }
 
 /*
- * A random private key is generated and stored in the device. The device
- * returns the pair public key.
+ * If no private key is provided, generate one in the device and cache
+ * the corresponding public key. The generated private key never leaves
+ * the device.
  */
 static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
 				 unsigned int len)
@@ -83,10 +80,9 @@ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
 	struct ecdh params;
 	int ret = -ENOMEM;
 
-	/* free the old public key, if any */
 	kfree(ctx->public_key);
-	/* make sure you don't free the old public key twice */
 	ctx->public_key = NULL;
+	ctx->do_fallback = false;
 
 	if (crypto_ecdh_decode_key(buf, len, &params) < 0) {
 		dev_err(&ctx->client->dev, "crypto_ecdh_decode_key failed\n");
@@ -94,33 +90,25 @@ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
 	}
 
 	if (params.key_size) {
-		/* fallback to ecdh software implementation */
-		ctx->do_fallback = true;
-		return crypto_kpp_set_secret(ctx->fallback, buf, len);
+		ret = crypto_kpp_set_secret(ctx->fallback, buf, len);
+		ctx->do_fallback = !ret;
+		return ret;
 	}
 
 	cmd = kmalloc_obj(*cmd);
 	if (!cmd)
 		return -ENOMEM;
 
-	/*
-	 * The device only supports NIST P256 ECC keys. The public key size will
-	 * always be the same. Use a macro for the key size to avoid unnecessary
-	 * computations.
-	 */
 	public_key = kmalloc(ATMEL_ECC_PUBKEY_SIZE, GFP_KERNEL);
 	if (!public_key)
 		goto free_cmd;
 
-	ctx->do_fallback = false;
-
 	atmel_i2c_init_genkey_cmd(cmd, DATA_SLOT_2);
 
 	ret = atmel_i2c_send_receive(ctx->client, cmd);
 	if (ret)
 		goto free_public_key;
 
-	/* save the public key */
 	memcpy(public_key, &cmd->data[RSP_DATA_IDX], ATMEL_ECC_PUBKEY_SIZE);
 	ctx->public_key = public_key;
 
@@ -149,7 +137,7 @@ static int atmel_ecdh_generate_public_key(struct kpp_request *req)
 	if (!ctx->public_key)
 		return -EINVAL;
 
-	/* might want less than we've got */
+	/* copy only as much as requested, capped at 64 bytes */
 	nbytes = min(ATMEL_ECC_PUBKEY_SIZE, req->dst_len);
 
 	/* public key was saved at private key generation */
@@ -175,7 +163,10 @@ static int atmel_ecdh_compute_shared_secret(struct kpp_request *req)
 		return crypto_kpp_compute_shared_secret(req);
 	}
 
-	/* must have exactly two points to be on the curve */
+	if (!ctx->public_key)
+		return -EINVAL;
+
+	/* A P-256 public key must contain two 32-byte coordinates */
 	if (req->src_len != ATMEL_ECC_PUBKEY_SIZE)
 		return -EINVAL;
 
@@ -250,7 +241,6 @@ static int atmel_ecdh_init_tfm(struct crypto_kpp *tfm)
 	struct crypto_kpp *fallback;
 	struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
 
-	ctx->curve_id = ECC_CURVE_NIST_P256;
 	ctx->client = atmel_ecc_i2c_client_alloc();
 	if (IS_ERR(ctx->client)) {
 		pr_err("tfm - i2c_client binding failed\n");
diff --git a/drivers/crypto/atmel-i2c.c b/drivers/crypto/atmel-i2c.c
index ff19857894d05..24bded47a32be 100644
--- a/drivers/crypto/atmel-i2c.c
+++ b/drivers/crypto/atmel-i2c.c
@@ -138,9 +138,8 @@ int atmel_i2c_init_ecdh_cmd(struct atmel_i2c_cmd *cmd,
 	cmd->param2 = cpu_to_le16(DATA_SLOT_2);
 
 	/*
-	 * The device only supports NIST P256 ECC keys. The public key size will
-	 * always be the same. Use a macro for the key size to avoid unnecessary
-	 * computations.
+	 * The device only supports P-256. Its public key is encoded as
+	 * two 32-byte coordinates.
 	 */
 	copied = sg_copy_to_buffer(pubkey,
 				   sg_nents_for_len(pubkey,
diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c
index 4c9af737b33aa..21a84c29c9b79 100644
--- a/drivers/crypto/atmel-sha204a.c
+++ b/drivers/crypto/atmel-sha204a.c
@@ -15,6 +15,7 @@
 #include <linux/module.h>
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
+#include <linux/string.h>
 #include <linux/sysfs.h>
 #include <linux/workqueue.h>
 #include "atmel-i2c.h"
@@ -31,10 +32,14 @@ static void atmel_sha204a_rng_done(struct atmel_i2c_work_data *work_data,
 	struct atmel_i2c_client_priv *i2c_priv = work_data->ctx;
 	struct hwrng *rng = areq;
 
-	if (status)
+	if (status) {
 		dev_warn_ratelimited(&i2c_priv->client->dev,
 				     "i2c transaction failed (%d)\n",
 				     status);
+		kfree_sensitive(work_data);
+		atomic_dec(&i2c_priv->tfm_count);
+		return;
+	}
 
 	rng->priv = (unsigned long)work_data;
 	atomic_dec(&i2c_priv->tfm_count);
@@ -91,12 +96,15 @@ static int atmel_sha204a_rng_read(struct hwrng *rng, void *data, size_t max,
 
 	ret = atmel_i2c_send_receive(i2c_priv->client, &cmd);
 	if (ret)
-		return ret;
+		goto out;
 
 	max = min(RANDOM_RSP_SIZE - CMD_OVERHEAD_SIZE, max);
 	memcpy(data, &cmd.data[RSP_DATA_IDX], max);
+	ret = max;
 
-	return max;
+out:
+	memzero_explicit(&cmd, sizeof(cmd));
+	return ret;
 }
 
 static int atmel_sha204a_otp_read(struct i2c_client *client, u16 addr, u8 *otp)
@@ -205,7 +213,7 @@ static void atmel_sha204a_remove(struct i2c_client *client)
 	devm_hwrng_unregister(&client->dev, &i2c_priv->hwrng);
 	atmel_i2c_flush_queue();
 
-	kfree((void *)i2c_priv->hwrng.priv);
+	kfree_sensitive((void *)i2c_priv->hwrng.priv);
 }
 
 static const struct of_device_id atmel_sha204a_dt_ids[] = {
diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c
index 3c0eacacfc87b..78443ae388f39 100644
--- a/drivers/crypto/atmel-tdes.c
+++ b/drivers/crypto/atmel-tdes.c
@@ -284,7 +284,6 @@ static int atmel_tdes_write_ctrl(struct atmel_tdes_dev *dd)
 
 static int atmel_tdes_crypt_pdc_stop(struct atmel_tdes_dev *dd)
 {
-	int err = 0;
 	size_t count;
 
 	atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS);
@@ -296,24 +295,23 @@ static int atmel_tdes_crypt_pdc_stop(struct atmel_tdes_dev *dd)
 		dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out,
 					dd->dma_size, DMA_FROM_DEVICE);
 
-		/* copy data */
 		count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset,
 				dd->buf_out, dd->buflen, dd->dma_size, 1);
 		if (count != dd->dma_size) {
-			err = -EINVAL;
 			dev_dbg(dd->dev, "not all data converted: %zu\n", count);
+			return -EINVAL;
 		}
 	}
 
-	return err;
+	return 0;
 }
 
 static int atmel_tdes_buff_init(struct atmel_tdes_dev *dd)
 {
 	int err = -ENOMEM;
 
-	dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, 0);
-	dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, 0);
+	dd->buf_in = (void *)__get_free_page(GFP_KERNEL);
+	dd->buf_out = (void *)__get_free_page(GFP_KERNEL);
 	dd->buflen = PAGE_SIZE;
 	dd->buflen &= ~(DES_BLOCK_SIZE - 1);
 
@@ -450,26 +448,19 @@ static int atmel_tdes_crypt_dma(struct atmel_tdes_dev *dd,
 
 static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd)
 {
-	int err, fast = 0, in, out;
+	bool fast;
+	int err;
 	size_t count;
 	dma_addr_t addr_in, addr_out;
 
-	if ((!dd->in_offset) && (!dd->out_offset)) {
-		/* check for alignment */
-		in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32)) &&
-			IS_ALIGNED(dd->in_sg->length, dd->ctx->block_size);
-		out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32)) &&
-			IS_ALIGNED(dd->out_sg->length, dd->ctx->block_size);
-		fast = in && out;
-
-		if (sg_dma_len(dd->in_sg) != sg_dma_len(dd->out_sg))
-			fast = 0;
-	}
-
+	fast = !dd->in_offset && !dd->out_offset &&
+		dd->in_sg->length == dd->out_sg->length &&
+		IS_ALIGNED(dd->in_sg->offset, sizeof(u32)) &&
+		IS_ALIGNED(dd->out_sg->offset, sizeof(u32)) &&
+		IS_ALIGNED(dd->in_sg->length, dd->ctx->block_size);
 
-	if (fast)  {
-		count = min_t(size_t, dd->total, sg_dma_len(dd->in_sg));
-		count = min_t(size_t, count, sg_dma_len(dd->out_sg));
+	if (fast) {
+		count = min(dd->total, dd->in_sg->length);
 
 		err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 		if (!err) {
@@ -609,28 +600,25 @@ static int atmel_tdes_handle_queue(struct atmel_tdes_dev *dd,
 
 static int atmel_tdes_crypt_dma_stop(struct atmel_tdes_dev *dd)
 {
-	int err = -EINVAL;
 	size_t count;
 
-	if (dd->flags & TDES_FLAGS_DMA) {
-		err = 0;
-		if  (dd->flags & TDES_FLAGS_FAST) {
-			dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
-			dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
-		} else {
-			dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out,
-						dd->dma_size, DMA_FROM_DEVICE);
-
-			/* copy data */
-			count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset,
-				dd->buf_out, dd->buflen, dd->dma_size, 1);
-			if (count != dd->dma_size) {
-				err = -EINVAL;
-				dev_dbg(dd->dev, "not all data converted: %zu\n", count);
-			}
+	if  (dd->flags & TDES_FLAGS_FAST) {
+		dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
+		dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
+	} else {
+		dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out, dd->dma_size,
+					DMA_FROM_DEVICE);
+
+		count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset,
+					   dd->buf_out, dd->buflen,
+					   dd->dma_size, 1);
+		if (count != dd->dma_size) {
+			dev_dbg(dd->dev, "not all data converted: %zu\n", count);
+			return -EINVAL;
 		}
 	}
-	return err;
+
+	return 0;
 }
 
 static int atmel_tdes_crypt(struct skcipher_request *req, unsigned long mode)
diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig
index 05210a0edb8af..ec57bf6aaf6c6 100644
--- a/drivers/crypto/caam/Kconfig
+++ b/drivers/crypto/caam/Kconfig
@@ -145,20 +145,11 @@ config CRYPTO_DEV_FSL_CAAM_PKC_API
 config CRYPTO_DEV_FSL_CAAM_RNG_API
 	bool "Register caam device for hwrng API"
 	default y
-	select CRYPTO_RNG
 	select HW_RANDOM
 	help
 	  Selecting this will register the SEC4 hardware rng to
 	  the hw_random API for supplying the kernel entropy pool.
 
-config CRYPTO_DEV_FSL_CAAM_PRNG_API
-	bool "Register Pseudo random number generation implementation with Crypto API"
-	default y
-	select CRYPTO_RNG
-	help
-	  Selecting this will register the SEC hardware prng to
-	  the Crypto API.
-
 config CRYPTO_DEV_FSL_CAAM_BLOB_GEN
 	bool
 
diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile
index d2eaf5205b1ca..a17e3cf14c61a 100644
--- a/drivers/crypto/caam/Makefile
+++ b/drivers/crypto/caam/Makefile
@@ -20,7 +20,6 @@ caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o
 caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o
 caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o
 caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o
-caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PRNG_API) += caamprng.o
 caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caampkc.o pkc_desc.o
 caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_BLOB_GEN) += blob_gen.o
 
diff --git a/drivers/crypto/caam/caamprng.c b/drivers/crypto/caam/caamprng.c
deleted file mode 100644
index 6e4c1191cb280..0000000000000
--- a/drivers/crypto/caam/caamprng.c
+++ /dev/null
@@ -1,241 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * Driver to expose SEC4 PRNG via crypto RNG API
- *
- * Copyright 2022 NXP
- *
- */
-
-#include <linux/completion.h>
-#include <crypto/internal/rng.h>
-#include <linux/dma-mapping.h>
-#include <linux/kernel.h>
-#include "compat.h"
-#include "regs.h"
-#include "intern.h"
-#include "desc_constr.h"
-#include "jr.h"
-#include "error.h"
-
-/*
- * Length of used descriptors, see caam_init_desc()
- */
-#define CAAM_PRNG_MAX_DESC_LEN (CAAM_CMD_SZ +				\
-			    CAAM_CMD_SZ +				\
-			    CAAM_CMD_SZ + CAAM_PTR_SZ_MAX)
-
-/* prng per-device context */
-struct caam_prng_ctx {
-	int err;
-	struct completion done;
-};
-
-struct caam_prng_alg {
-	struct rng_alg rng;
-	bool registered;
-};
-
-static void caam_prng_done(struct device *jrdev, u32 *desc, u32 err,
-			  void *context)
-{
-	struct caam_prng_ctx *jctx = context;
-
-	jctx->err = err ? caam_jr_strstatus(jrdev, err) : 0;
-
-	complete(&jctx->done);
-}
-
-static u32 *caam_init_reseed_desc(u32 *desc)
-{
-	init_job_desc(desc, 0);	/* + 1 cmd_sz */
-	/* Generate random bytes: + 1 cmd_sz */
-	append_operation(desc, OP_TYPE_CLASS1_ALG | OP_ALG_ALGSEL_RNG |
-			OP_ALG_AS_FINALIZE);
-
-	print_hex_dump_debug("prng reseed desc@: ", DUMP_PREFIX_ADDRESS,
-			     16, 4, desc, desc_bytes(desc), 1);
-
-	return desc;
-}
-
-static u32 *caam_init_prng_desc(u32 *desc, dma_addr_t dst_dma, u32 len)
-{
-	init_job_desc(desc, 0);	/* + 1 cmd_sz */
-	/* Generate random bytes: + 1 cmd_sz */
-	append_operation(desc, OP_ALG_ALGSEL_RNG | OP_TYPE_CLASS1_ALG);
-	/* Store bytes: + 1 cmd_sz + caam_ptr_sz  */
-	append_fifo_store(desc, dst_dma,
-			  len, FIFOST_TYPE_RNGSTORE);
-
-	print_hex_dump_debug("prng job desc@: ", DUMP_PREFIX_ADDRESS,
-			     16, 4, desc, desc_bytes(desc), 1);
-
-	return desc;
-}
-
-static int caam_prng_generate(struct crypto_rng *tfm,
-			     const u8 *src, unsigned int slen,
-			     u8 *dst, unsigned int dlen)
-{
-	unsigned int aligned_dlen = ALIGN(dlen, dma_get_cache_alignment());
-	struct caam_prng_ctx ctx;
-	struct device *jrdev;
-	dma_addr_t dst_dma;
-	u32 *desc;
-	u8 *buf;
-	int ret;
-
-	if (aligned_dlen < dlen)
-		return -EOVERFLOW;
-
-	buf = kzalloc(aligned_dlen, GFP_KERNEL);
-	if (!buf)
-		return -ENOMEM;
-
-	jrdev = caam_jr_alloc();
-	ret = PTR_ERR_OR_ZERO(jrdev);
-	if (ret) {
-		pr_err("Job Ring Device allocation failed\n");
-		kfree(buf);
-		return ret;
-	}
-
-	desc = kzalloc(CAAM_PRNG_MAX_DESC_LEN, GFP_KERNEL);
-	if (!desc) {
-		ret = -ENOMEM;
-		goto out1;
-	}
-
-	dst_dma = dma_map_single(jrdev, buf, dlen, DMA_FROM_DEVICE);
-	if (dma_mapping_error(jrdev, dst_dma)) {
-		dev_err(jrdev, "Failed to map destination buffer memory\n");
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	init_completion(&ctx.done);
-	ret = caam_jr_enqueue(jrdev,
-			      caam_init_prng_desc(desc, dst_dma, dlen),
-			      caam_prng_done, &ctx);
-
-	if (ret == -EINPROGRESS) {
-		wait_for_completion(&ctx.done);
-		ret = ctx.err;
-	}
-
-	dma_unmap_single(jrdev, dst_dma, dlen, DMA_FROM_DEVICE);
-
-	if (!ret)
-		memcpy(dst, buf, dlen);
-out:
-	kfree(desc);
-out1:
-	caam_jr_free(jrdev);
-	kfree(buf);
-	return ret;
-}
-
-static void caam_prng_exit(struct crypto_tfm *tfm) {}
-
-static int caam_prng_init(struct crypto_tfm *tfm)
-{
-	return 0;
-}
-
-static int caam_prng_seed(struct crypto_rng *tfm,
-			 const u8 *seed, unsigned int slen)
-{
-	struct caam_prng_ctx ctx;
-	struct device *jrdev;
-	u32 *desc;
-	int ret;
-
-	if (slen) {
-		pr_err("Seed length should be zero\n");
-		return -EINVAL;
-	}
-
-	jrdev = caam_jr_alloc();
-	ret = PTR_ERR_OR_ZERO(jrdev);
-	if (ret) {
-		pr_err("Job Ring Device allocation failed\n");
-		return ret;
-	}
-
-	desc = kzalloc(CAAM_PRNG_MAX_DESC_LEN, GFP_KERNEL);
-	if (!desc) {
-		caam_jr_free(jrdev);
-		return -ENOMEM;
-	}
-
-	init_completion(&ctx.done);
-	ret = caam_jr_enqueue(jrdev,
-			      caam_init_reseed_desc(desc),
-			      caam_prng_done, &ctx);
-
-	if (ret == -EINPROGRESS) {
-		wait_for_completion(&ctx.done);
-		ret = ctx.err;
-	}
-
-	kfree(desc);
-	caam_jr_free(jrdev);
-	return ret;
-}
-
-static struct caam_prng_alg caam_prng_alg = {
-	.rng = {
-		.generate = caam_prng_generate,
-		.seed = caam_prng_seed,
-		.seedsize = 0,
-		.base = {
-			.cra_name = "stdrng",
-			.cra_driver_name = "prng-caam",
-			.cra_priority = 500,
-			.cra_ctxsize = sizeof(struct caam_prng_ctx),
-			.cra_module = THIS_MODULE,
-			.cra_init = caam_prng_init,
-			.cra_exit = caam_prng_exit,
-		},
-	}
-};
-
-void caam_prng_unregister(void *data)
-{
-	if (caam_prng_alg.registered)
-		crypto_unregister_rng(&caam_prng_alg.rng);
-}
-
-int caam_prng_register(struct device *ctrldev)
-{
-	struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
-	u32 rng_inst;
-	int ret = 0;
-
-	/* Check for available RNG blocks before registration */
-	if (priv->era < 10)
-		rng_inst = (rd_reg32(&priv->jr[0]->perfmon.cha_num_ls) &
-			    CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT;
-	else
-		rng_inst = rd_reg32(&priv->jr[0]->vreg.rng) & CHA_VER_NUM_MASK;
-
-	if (!rng_inst) {
-		dev_dbg(ctrldev, "RNG block is not available... skipping registering algorithm\n");
-		return ret;
-	}
-
-	ret = crypto_register_rng(&caam_prng_alg.rng);
-	if (ret) {
-		dev_err(ctrldev,
-			"couldn't register rng crypto alg: %d\n",
-			ret);
-		return ret;
-	}
-
-	caam_prng_alg.registered = true;
-
-	dev_info(ctrldev,
-		 "rng crypto API alg registered %s\n", caam_prng_alg.rng.base.cra_driver_name);
-
-	return 0;
-}
diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h
index a88da0d31b23b..6e48bf7d60543 100644
--- a/drivers/crypto/caam/intern.h
+++ b/drivers/crypto/caam/intern.h
@@ -212,21 +212,6 @@ static inline void caam_rng_exit(struct device *dev) {}
 
 #endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API */
 
-#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_PRNG_API
-
-int caam_prng_register(struct device *dev);
-void caam_prng_unregister(void *data);
-
-#else
-
-static inline int caam_prng_register(struct device *dev)
-{
-	return 0;
-}
-
-static inline void caam_prng_unregister(void *data) {}
-#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_PRNG_API */
-
 #ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI
 
 int caam_qi_algapi_init(struct device *dev);
diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c
index 0ef00df9730e5..bddeaaaca487c 100644
--- a/drivers/crypto/caam/jr.c
+++ b/drivers/crypto/caam/jr.c
@@ -40,7 +40,6 @@ static void register_algs(struct caam_drv_private_jr *jrpriv,
 	caam_algapi_hash_init(dev);
 	caam_pkc_init(dev);
 	jrpriv->hwrng = !caam_rng_init(dev);
-	caam_prng_register(dev);
 	caam_qi_algapi_init(dev);
 
 algs_unlock:
@@ -55,7 +54,6 @@ static void unregister_algs(void)
 		goto algs_unlock;
 
 	caam_qi_algapi_exit();
-	caam_prng_unregister(NULL);
 	caam_pkc_exit();
 	caam_algapi_hash_exit();
 	caam_algapi_exit();
diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.c b/drivers/crypto/cavium/nitrox/nitrox_hal.c
index 1b5abdb6cc5e1..e36c1741bb786 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_hal.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_hal.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <linux/delay.h>
+#include <linux/string.h>
 
 #include "nitrox_dev.h"
 #include "nitrox_csr.h"
@@ -647,7 +648,7 @@ void nitrox_get_hwinfo(struct nitrox_device *ndev)
 		 ndev->hw.revision_id);
 
 	/* copy partname */
-	strscpy(ndev->hw.partname, name, sizeof(ndev->hw.partname));
+	strscpy(ndev->hw.partname, name);
 }
 
 void enable_pf2vf_mbox_interrupts(struct nitrox_device *ndev)
diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c
index 85058a89f35b2..ff9bb253dbb20 100644
--- a/drivers/crypto/ccp/ccp-crypto-sha.c
+++ b/drivers/crypto/ccp/ccp-crypto-sha.c
@@ -426,7 +426,7 @@ static int ccp_register_hmac_alg(struct list_head *head,
 	*ccp_alg = *base_alg;
 	INIT_LIST_HEAD(&ccp_alg->entry);
 
-	strscpy(ccp_alg->child_alg, def->name, CRYPTO_MAX_ALG_NAME);
+	strscpy(ccp_alg->child_alg, def->name);
 
 	alg = &ccp_alg->alg;
 	alg->setkey = ccp_sha_setkey;
diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c
index ca473ca198b81..f833cb7e4da36 100644
--- a/drivers/crypto/ccp/sev-dev.c
+++ b/drivers/crypto/ccp/sev-dev.c
@@ -245,6 +245,7 @@ static int sev_cmd_buffer_len(int cmd)
 	case SEV_CMD_SNP_LAUNCH_FINISH:		return sizeof(struct sev_data_snp_launch_finish);
 	case SEV_CMD_SNP_DBG_DECRYPT:		return sizeof(struct sev_data_snp_dbg);
 	case SEV_CMD_SNP_DBG_ENCRYPT:		return sizeof(struct sev_data_snp_dbg);
+	case SEV_CMD_SNP_VERIFY_MITIGATION:	return sizeof(struct sev_data_snp_verify_mitigation);
 	case SEV_CMD_SNP_PAGE_UNSMASH:		return sizeof(struct sev_data_snp_page_unsmash);
 	case SEV_CMD_SNP_PLATFORM_STATUS:	return sizeof(struct sev_data_snp_addr);
 	case SEV_CMD_SNP_GUEST_REQUEST:		return sizeof(struct sev_data_snp_guest_request);
@@ -260,20 +261,16 @@ static int sev_cmd_buffer_len(int cmd)
 
 static struct file *open_file_as_root(const char *filename, int flags, umode_t mode)
 {
-	struct path root __free(path_put) = {};
-
-	task_lock(&init_task);
-	get_fs_root(init_task.fs, &root);
-	task_unlock(&init_task);
-
 	CLASS(prepare_creds, cred)();
 	if (!cred)
 		return ERR_PTR(-ENOMEM);
 
 	cred->fsuid = GLOBAL_ROOT_UID;
 
-	scoped_with_creds(cred)
-		return file_open_root(&root, filename, flags, mode);
+	scoped_with_init_fs() {
+		scoped_with_creds(cred)
+			return filp_open(filename, flags, mode);
+	}
 }
 
 static int sev_read_init_ex_file(void)
@@ -1352,6 +1349,162 @@ static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg)
 	return 0;
 }
 
+#ifdef CONFIG_SYSFS
+static int snp_verify_mitigation(u16 command, u64 vector,
+				 struct sev_data_snp_verify_mitigation_dst *dst)
+{
+	struct sev_data_snp_verify_mitigation_dst *mit_dst = NULL;
+	struct sev_data_snp_verify_mitigation data = {0};
+	struct sev_device *sev = psp_master->sev_data;
+	int ret, error = 0;
+
+	mit_dst = snp_alloc_firmware_page(GFP_KERNEL | __GFP_ZERO);
+	if (!mit_dst)
+		return -ENOMEM;
+
+	data.length = sizeof(data);
+	data.subcommand = command;
+	data.vector = vector;
+	data.dst_paddr = __psp_pa(mit_dst);
+	data.dst_paddr_en = true;
+
+	ret = sev_do_cmd(SEV_CMD_SNP_VERIFY_MITIGATION, &data, &error);
+	if (!ret)
+		memcpy(dst, mit_dst, sizeof(*mit_dst));
+	else
+		dev_err(sev->dev, "SNP_VERIFY_MITIGATION command failed, ret = %d, error = %#x\n",
+			ret, error);
+
+	snp_free_firmware_page(mit_dst);
+
+	return ret;
+}
+
+static ssize_t supported_mitigations_show(struct kobject *kobj,
+					  struct kobj_attribute *attr, char *buf)
+{
+	struct sev_data_snp_verify_mitigation_dst dst;
+	int ret;
+
+	ret = snp_verify_mitigation(SNP_MIT_SUBCMD_REQ_STATUS, 0, &dst);
+	if (ret)
+		return ret;
+
+	return sysfs_emit(buf, "0x%llx\n", dst.mit_supported_vector);
+}
+
+static struct kobj_attribute supported_attr =
+		__ATTR_RO_MODE(supported_mitigations, 0400);
+
+static ssize_t verified_mitigations_show(struct kobject *kobj,
+					 struct kobj_attribute *attr, char *buf)
+{
+	struct sev_data_snp_verify_mitigation_dst dst;
+	int ret;
+
+	ret = snp_verify_mitigation(SNP_MIT_SUBCMD_REQ_STATUS, 0, &dst);
+	if (ret)
+		return ret;
+
+	return sysfs_emit(buf, "0x%llx\n", dst.mit_verified_vector);
+}
+
+static ssize_t verified_mitigations_store(struct kobject *kobj,
+					  struct kobj_attribute *attr,
+					  const char *buf, size_t count)
+{
+	struct sev_data_snp_verify_mitigation_dst dst;
+	struct sev_device *sev = psp_master->sev_data;
+	u64 vector;
+	int ret;
+
+	ret = kstrtoull(buf, 0, &vector);
+	if (ret)
+		return ret;
+
+	/*
+	 * The firmware verifies a single mitigation per call. Reject vectors
+	 * with more than one bit set early to avoid a guaranteed-to-fail call
+	 */
+	if (hweight64(vector) != 1)
+		return -EINVAL;
+
+	ret = snp_verify_mitigation(SNP_MIT_SUBCMD_REQ_VERIFY, vector, &dst);
+	if (ret)
+		return ret;
+
+	if (dst.mit_failure_status) {
+		dev_err(sev->dev, "Verify Mitigation - failure status: 0x%x\n",
+			dst.mit_failure_status);
+		return -EINVAL;
+	}
+
+	return count;
+}
+
+static struct kobj_attribute verified_attr =
+		__ATTR_RW_MODE(verified_mitigations, 0600);
+
+static struct attribute *mitigation_attrs[] = {
+	&supported_attr.attr,
+	&verified_attr.attr,
+	NULL
+};
+
+static const struct attribute_group mit_attr_group = {
+	.attrs = mitigation_attrs,
+};
+
+static void sev_snp_register_verify_mitigation(struct sev_device *sev)
+{
+	int rc;
+
+	if (!(sev->snp_feat_info_0.ecx & SNP_VERIFY_MITIGATION_SUPPORTED) ||
+	    sev->verify_mit)
+		return;
+
+	if (!sev->sev_kobj) {
+		sev->sev_kobj = kobject_create_and_add("sev", firmware_kobj);
+		if (!sev->sev_kobj)
+			return;
+	}
+
+	sev->verify_mit = kobject_create_and_add("vulnerabilities", sev->sev_kobj);
+	if (!sev->verify_mit)
+		goto err_sev_kobj;
+
+	rc = sysfs_create_group(sev->verify_mit, &mit_attr_group);
+	if (rc)
+		goto err_verify_mit;
+
+	return;
+
+err_verify_mit:
+	kobject_put(sev->verify_mit);
+	sev->verify_mit = NULL;
+err_sev_kobj:
+	kobject_put(sev->sev_kobj);
+	sev->sev_kobj = NULL;
+}
+
+static void sev_snp_unregister_verify_mitigation(struct sev_device *sev)
+{
+	if (sev->verify_mit) {
+		sysfs_remove_group(sev->verify_mit, &mit_attr_group);
+		kobject_put(sev->verify_mit);
+		sev->verify_mit = NULL;
+	}
+
+	if (sev->sev_kobj) {
+		kobject_put(sev->sev_kobj);
+		sev->sev_kobj = NULL;
+	}
+}
+#else	// CONFIG_SYSFS
+static void sev_snp_register_verify_mitigation(struct sev_device *sev) { }
+static void sev_snp_unregister_verify_mitigation(struct sev_device *sev) { }
+#endif	// CONFIG_SYSFS
+
 static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid)
 {
 	struct sev_data_range_list *snp_range_list __free(kfree) = NULL;
@@ -1545,7 +1698,7 @@ static int __sev_platform_init_handle_init_ex_path(struct sev_device *sev)
 	if (sev_init_ex_buffer)
 		return 0;
 
-	page = alloc_pages(GFP_KERNEL, get_order(NV_LENGTH));
+	page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(NV_LENGTH));
 	if (!page) {
 		dev_err(sev->dev, "SEV: INIT_EX NV memory allocation failed\n");
 		return -ENOMEM;
@@ -1555,20 +1708,31 @@ static int __sev_platform_init_handle_init_ex_path(struct sev_device *sev)
 
 	rc = sev_read_init_ex_file();
 	if (rc)
-		return rc;
+		goto err_free;
 
 	/* If SEV-SNP is initialized, transition to firmware page. */
 	if (sev->snp_initialized) {
 		unsigned long npages;
 
 		npages = 1UL << get_order(NV_LENGTH);
-		if (rmp_mark_pages_firmware(__pa(sev_init_ex_buffer), npages, false)) {
+		if (rmp_mark_pages_firmware(__pa(sev_init_ex_buffer), npages, true)) {
 			dev_err(sev->dev, "SEV: INIT_EX NV memory page state change failed.\n");
-			return -ENOMEM;
+			rc = -ENOMEM;
+			/*
+			 * Pages can be in an inconsistent state, don't release them back to the
+			 * system.
+			 */
+			goto err_reset;
 		}
 	}
 
 	return 0;
+
+err_free:
+	__free_pages(page, get_order(NV_LENGTH));
+err_reset:
+	sev_init_ex_buffer = NULL;
+	return rc;
 }
 
 static int __sev_platform_init_locked(int *error)
@@ -1675,6 +1839,17 @@ int sev_platform_init(struct sev_platform_init_args *args)
 	rc = _sev_platform_init_locked(args);
 	mutex_unlock(&sev_cmd_mutex);
 
+	/*
+	 * Register the sysfs interface outside the sev_cmd_mutex. The
+	 * _show()/_store() handlers issue SEV commands that acquire the
+	 * sev_cmd_mutex, so creating (and on the shutdown path, removing) the
+	 * sysfs group must stay outside that lock. sysfs provides its own
+	 * synchronization between group creation/removal and concurrent
+	 * attribute access.
+	 */
+	if (!rc)
+		sev_snp_register_verify_mitigation(psp_master->sev_data);
+
 	return rc;
 }
 EXPORT_SYMBOL_GPL(sev_platform_init);
@@ -2769,6 +2944,15 @@ static void sev_firmware_shutdown(struct sev_device *sev)
 	if (sev->tio_status)
 		sev_tsm_uninit(sev);
 
+	/*
+	 * Remove the sysfs interface before taking the sev_cmd_mutex.
+	 * sysfs_remove_group() waits for in-flight _show()/_store() handlers
+	 * to drain, and those handlers issue SNP_VERIFY_MITIGATION via
+	 * sev_do_cmd() which acquires the sev_cmd_mutex. Removing the group
+	 * while holding the mutex could therefore deadlock.
+	 */
+	sev_snp_unregister_verify_mitigation(sev);
+
 	mutex_lock(&sev_cmd_mutex);
 
 	__sev_firmware_shutdown(sev, false);
diff --git a/drivers/crypto/ccp/sev-dev.h b/drivers/crypto/ccp/sev-dev.h
index b1cd556bbbf65..d5e596606defc 100644
--- a/drivers/crypto/ccp/sev-dev.h
+++ b/drivers/crypto/ccp/sev-dev.h
@@ -59,6 +59,9 @@ struct sev_device {
 
 	bool snp_initialized;
 
+	struct kobject *sev_kobj;
+	struct kobject *verify_mit;
+
 	struct sev_user_data_status sev_plat_status;
 
 	struct sev_user_data_snp_status snp_plat_status;
diff --git a/drivers/crypto/gemini/sl3516-ce-cipher.c b/drivers/crypto/gemini/sl3516-ce-cipher.c
index 583010b2d0071..02ec4282333b6 100644
--- a/drivers/crypto/gemini/sl3516-ce-cipher.c
+++ b/drivers/crypto/gemini/sl3516-ce-cipher.c
@@ -56,10 +56,6 @@ static bool sl3516_ce_need_fallback(struct skcipher_request *areq)
 			ce->fallback_mod16++;
 			return true;
 		}
-		if ((sg_dma_len(sg) % 16) != 0) {
-			ce->fallback_mod16++;
-			return true;
-		}
 		if (!IS_ALIGNED(sg->offset, 16)) {
 			ce->fallback_align16++;
 			return true;
@@ -72,10 +68,6 @@ static bool sl3516_ce_need_fallback(struct skcipher_request *areq)
 			ce->fallback_mod16++;
 			return true;
 		}
-		if ((sg_dma_len(sg) % 16) != 0) {
-			ce->fallback_mod16++;
-			return true;
-		}
 		if (!IS_ALIGNED(sg->offset, 16)) {
 			ce->fallback_align16++;
 			return true;
diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c
index a951d2ef78330..c01966a4a33fa 100644
--- a/drivers/crypto/hisilicon/qm.c
+++ b/drivers/crypto/hisilicon/qm.c
@@ -2944,11 +2944,8 @@ static int qm_alloc_uacce(struct hisi_qm *qm)
 		.flags = UACCE_DEV_SVA,
 		.ops = &uacce_qm_ops,
 	};
-	int ret;
 
-	ret = strscpy(interface.name, dev_driver_string(&pdev->dev),
-		      sizeof(interface.name));
-	if (ret < 0)
+	if (strscpy(interface.name, dev_driver_string(&pdev->dev)) < 0)
 		return -ENAMETOOLONG;
 
 	uacce = uacce_alloc(&pdev->dev, &interface);
diff --git a/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c b/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c
index 5b90cf0fb0e41..fdda046724548 100644
--- a/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c
+++ b/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c
@@ -1591,6 +1591,7 @@ static const struct of_device_id ixp4xx_crypto_of_match[] = {
 	},
 	{},
 };
+MODULE_DEVICE_TABLE(of, ixp4xx_crypto_of_match);
 
 static struct platform_driver ixp_crypto_driver = {
 	.probe = ixp_crypto_probe,
diff --git a/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c
index 8a8f6c81e010c..0e424024224e5 100644
--- a/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c
+++ b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c
@@ -1541,7 +1541,7 @@ static int register_aes_algs(struct ocs_aes_dev *aes_dev)
 
 	/*
 	 * If any algorithm fails to register, all preceding algorithms that
-	 * were successfully registered will be automatically unregistered.
+	 * were registered in the same call are automatically unregistered.
 	 */
 	ret = crypto_engine_register_aeads(algs_aead, ARRAY_SIZE(algs_aead));
 	if (ret)
@@ -1549,7 +1549,7 @@ static int register_aes_algs(struct ocs_aes_dev *aes_dev)
 
 	ret = crypto_engine_register_skciphers(algs, ARRAY_SIZE(algs));
 	if (ret)
-		crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs));
+		crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead));
 
 	return ret;
 }
diff --git a/drivers/crypto/intel/keembay/keembay-ocs-ecc.c b/drivers/crypto/intel/keembay/keembay-ocs-ecc.c
index e61a95f66a0c7..9e555b02086c1 100644
--- a/drivers/crypto/intel/keembay/keembay-ocs-ecc.c
+++ b/drivers/crypto/intel/keembay/keembay-ocs-ecc.c
@@ -978,6 +978,7 @@ static const struct of_device_id kmb_ocs_ecc_of_match[] = {
 	},
 	{}
 };
+MODULE_DEVICE_TABLE(of, kmb_ocs_ecc_of_match);
 
 /* The OCS driver is a platform device. */
 static struct platform_driver kmb_ocs_ecc_driver = {
diff --git a/drivers/crypto/intel/qat/qat_common/adf_aer.c b/drivers/crypto/intel/qat/qat_common/adf_aer.c
index d58cd7fbf7077..afded3030e9a3 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_aer.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_aer.c
@@ -189,6 +189,8 @@ static void adf_device_reset_worker(struct work_struct *work)
 	queue_work(device_sriov_wq, &sriov_data.sriov_work);
 	if (wait_for_completion_timeout(&sriov_data.compl, wait_jiffies))
 		adf_pf2vf_notify_restarted(accel_dev);
+	else
+		cancel_work_sync(&sriov_data.sriov_work);
 
 	adf_dev_restarted_notify(accel_dev);
 	clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status);
diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg.c b/drivers/crypto/intel/qat/qat_common/adf_cfg.c
index ea5d72d5090c0..d97ee10000452 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_cfg.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_cfg.c
@@ -2,6 +2,7 @@
 /* Copyright(c) 2014 - 2020 Intel Corporation */
 #include <linux/mutex.h>
 #include <linux/slab.h>
+#include <linux/string.h>
 #include <linux/list.h>
 #include <linux/seq_file.h>
 #include "adf_accel_devices.h"
@@ -284,13 +285,13 @@ int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev,
 		return -ENOMEM;
 
 	INIT_LIST_HEAD(&key_val->list);
-	strscpy(key_val->key, key, sizeof(key_val->key));
+	strscpy(key_val->key, key);
 
 	if (type == ADF_DEC) {
 		snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES,
 			 "%ld", (*((long *)val)));
 	} else if (type == ADF_STR) {
-		strscpy(key_val->val, (char *)val, sizeof(key_val->val));
+		strscpy(key_val->val, (char *)val);
 	} else if (type == ADF_HEX) {
 		snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES,
 			 "0x%lx", (unsigned long)val);
@@ -350,7 +351,7 @@ int adf_cfg_section_add(struct adf_accel_dev *accel_dev, const char *name)
 	if (!sec)
 		return -ENOMEM;
 
-	strscpy(sec->name, name, sizeof(sec->name));
+	strscpy(sec->name, name);
 	INIT_LIST_HEAD(&sec->param_head);
 	down_write(&cfg->lock);
 	list_add_tail(&sec->list, &cfg->sec_list);
diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c
index 1af6da8b263fb..f7237c7ef2b15 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c
@@ -49,18 +49,17 @@ static_assert(sizeof(ADF_CFG_SYM ADF_SERVICES_DELIMITER
 		     ADF_CFG_DCC) < ADF_CFG_MAX_VAL_LEN_IN_BYTES);
 
 static int adf_service_string_to_mask(struct adf_accel_dev *accel_dev, const char *buf,
-				      size_t len, unsigned long *out_mask)
+				      unsigned long *out_mask)
 {
 	struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
-	char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { };
+	char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
 	unsigned long mask = 0;
 	char *substr, *token;
 	int id, num_svc = 0;
 
-	if (len > ADF_CFG_MAX_VAL_LEN_IN_BYTES - 1)
+	if (strscpy_pad(services, buf) < 0)
 		return -EINVAL;
 
-	strscpy(services, buf, ADF_CFG_MAX_VAL_LEN_IN_BYTES);
 	substr = services;
 
 	while ((token = strsep(&substr, ADF_SERVICES_DELIMITER))) {
@@ -103,12 +102,12 @@ static int adf_service_mask_to_string(unsigned long mask, char *buf, size_t len)
 }
 
 int adf_parse_service_string(struct adf_accel_dev *accel_dev, const char *in,
-			     size_t in_len, char *out, size_t out_len)
+			     char *out, size_t out_len)
 {
 	unsigned long mask;
 	int ret;
 
-	ret = adf_service_string_to_mask(accel_dev, in, in_len, &mask);
+	ret = adf_service_string_to_mask(accel_dev, in, &mask);
 	if (ret)
 		return ret;
 
@@ -121,7 +120,6 @@ int adf_parse_service_string(struct adf_accel_dev *accel_dev, const char *in,
 int adf_get_service_mask(struct adf_accel_dev *accel_dev, unsigned long *mask)
 {
 	char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { };
-	size_t len;
 	int ret;
 
 	ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC,
@@ -132,8 +130,7 @@ int adf_get_service_mask(struct adf_accel_dev *accel_dev, unsigned long *mask)
 		return ret;
 	}
 
-	len = strnlen(services, ADF_CFG_MAX_VAL_LEN_IN_BYTES);
-	ret = adf_service_string_to_mask(accel_dev, services, len, mask);
+	ret = adf_service_string_to_mask(accel_dev, services, mask);
 	if (ret)
 		dev_err(&GET_DEV(accel_dev), "Invalid value of %s param: %s\n",
 			ADF_SERVICES_ENABLED, services);
diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h
index 913d717280afa..89be2f2c72336 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h
@@ -35,7 +35,7 @@ enum {
 #define MAX_NUM_CONCURR_SVC	ADF_THREE_SERVICES
 
 int adf_parse_service_string(struct adf_accel_dev *accel_dev, const char *in,
-			     size_t in_len, char *out, size_t out_len);
+			     char *out, size_t out_len);
 int adf_get_service_enabled(struct adf_accel_dev *accel_dev);
 int adf_get_service_mask(struct adf_accel_dev *accel_dev, unsigned long *mask);
 enum adf_cfg_service_type adf_srv_to_cfg_svc_type(enum adf_base_services svc);
diff --git a/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c b/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c
index f9017e03ec0f2..32aeb795cc036 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c
@@ -2,6 +2,7 @@
 /* Copyright(c) 2024 Intel Corporation */
 
 #include <linux/slab.h>
+#include <linux/string.h>
 #include <linux/types.h>
 #include "adf_mstate_mgr.h"
 
@@ -158,7 +159,7 @@ static struct adf_mstate_sect_h *adf_mstate_sect_add_header(struct adf_mstate_mg
 		return NULL;
 	}
 
-	strscpy(sect->id, id, sizeof(sect->id));
+	strscpy(sect->id, id);
 	sect->size = 0;
 	sect->sub_sects = 0;
 	mgr->state += sizeof(*sect);
diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs.c b/drivers/crypto/intel/qat/qat_common/adf_sysfs.c
index 79c63dfa8ff30..8daa69a76b01a 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_sysfs.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs.c
@@ -125,7 +125,7 @@ static ssize_t cfg_services_store(struct device *dev, struct device_attribute *a
 	if (!accel_dev)
 		return -EINVAL;
 
-	ret = adf_parse_service_string(accel_dev, buf, count, services,
+	ret = adf_parse_service_string(accel_dev, buf, services,
 				       ADF_CFG_MAX_VAL_LEN_IN_BYTES);
 	if (ret)
 		return ret;
diff --git a/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c b/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c
index a8f853516a3f9..fc5d88a2bb172 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c
@@ -2,6 +2,7 @@
 /* Copyright(c) 2014 - 2020 Intel Corporation */
 #include <linux/mutex.h>
 #include <linux/slab.h>
+#include <linux/string.h>
 #include <linux/seq_file.h>
 #include "adf_accel_devices.h"
 #include "adf_transport_internal.h"
@@ -103,7 +104,7 @@ int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, const char *name)
 	if (!ring_debug)
 		return -ENOMEM;
 
-	strscpy(ring_debug->ring_name, name, sizeof(ring_debug->ring_name));
+	strscpy(ring_debug->ring_name, name);
 	snprintf(entry_name, sizeof(entry_name), "ring_%02d",
 		 ring->ring_number);
 
diff --git a/drivers/crypto/intel/qat/qat_common/qat_algs.c b/drivers/crypto/intel/qat/qat_common/qat_algs.c
index 7f638a62e3ade..91663805d9e60 100644
--- a/drivers/crypto/intel/qat/qat_common/qat_algs.c
+++ b/drivers/crypto/intel/qat/qat_common/qat_algs.c
@@ -405,6 +405,7 @@ static void qat_alg_xts_reverse_key(const u8 *key_forward, unsigned int keylen,
 		memcpy(key_reverse + AES_BLOCK_SIZE, key - AES_BLOCK_SIZE,
 		       AES_BLOCK_SIZE);
 	}
+	memzero_explicit(&aes_expanded, sizeof(aes_expanded));
 }
 
 static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx,
diff --git a/drivers/crypto/intel/qat/qat_common/qat_compression.c b/drivers/crypto/intel/qat/qat_common/qat_compression.c
index 1424d7a9bcd36..8129ad0c32d84 100644
--- a/drivers/crypto/intel/qat/qat_common/qat_compression.c
+++ b/drivers/crypto/intel/qat/qat_common/qat_compression.c
@@ -2,6 +2,7 @@
 /* Copyright(c) 2022 Intel Corporation */
 #include <linux/module.h>
 #include <linux/slab.h>
+#include <linux/string.h>
 #include "adf_accel_devices.h"
 #include "adf_common_drv.h"
 #include "adf_transport.h"
@@ -144,7 +145,7 @@ static int qat_compression_create_instances(struct adf_accel_dev *accel_dev)
 	int i;
 
 	INIT_LIST_HEAD(&accel_dev->compression_list);
-	strscpy(key, ADF_NUM_DC, sizeof(key));
+	strscpy(key, ADF_NUM_DC);
 	ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);
 	if (ret)
 		return ret;
diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c
index 205579a6ba2bf..58dd996c7f3ac 100644
--- a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c
+++ b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c
@@ -99,7 +99,7 @@ static int dev_supports_eng_type(struct otx_cpt_eng_grps *eng_grps,
 static void set_ucode_filename(struct otx_cpt_ucode *ucode,
 			       const char *filename)
 {
-	strscpy(ucode->filename, filename, OTX_CPT_UCODE_NAME_LENGTH);
+	strscpy(ucode->filename, filename);
 }
 
 static char *get_eng_type_str(int eng_type)
@@ -140,7 +140,7 @@ static int get_ucode_type(struct otx_cpt_ucode_hdr *ucode_hdr, int *ucode_type)
 	u32 i, val = 0;
 	u8 nn;
 
-	strscpy(tmp_ver_str, ucode_hdr->ver_str, OTX_CPT_UCODE_VER_STR_SZ);
+	strscpy(tmp_ver_str, ucode_hdr->ver_str);
 	for (i = 0; i < strlen(tmp_ver_str); i++)
 		tmp_ver_str[i] = tolower(tmp_ver_str[i]);
 
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c
index 9b0887d7e62ca..465f00e746232 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c
@@ -74,7 +74,7 @@ static int is_2nd_ucode_used(struct otx2_cpt_eng_grp_info *eng_grp)
 static void set_ucode_filename(struct otx2_cpt_ucode *ucode,
 			       const char *filename)
 {
-	strscpy(ucode->filename, filename, OTX2_CPT_NAME_LENGTH);
+	strscpy(ucode->filename, filename);
 }
 
 static char *get_eng_type_str(int eng_type)
@@ -130,7 +130,7 @@ static int get_ucode_type(struct device *dev,
 	int i, val = 0;
 	u8 nn;
 
-	strscpy(tmp_ver_str, ucode_hdr->ver_str, OTX2_CPT_UCODE_VER_STR_SZ);
+	strscpy(tmp_ver_str, ucode_hdr->ver_str);
 	for (i = 0; i < strlen(tmp_ver_str); i++)
 		tmp_ver_str[i] = tolower(tmp_ver_str[i]);
 
diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c
index 133ebc9982362..595b2fd84667e 100644
--- a/drivers/crypto/mxs-dcp.c
+++ b/drivers/crypto/mxs-dcp.c
@@ -353,7 +353,7 @@ static int mxs_dcp_aes_block_crypt(struct crypto_async_request *arq)
 
 	for_each_sg(req->src, src, sg_nents(req->src), i) {
 		src_buf = sg_virt(src);
-		len = sg_dma_len(src);
+		len = src->length;
 		tlen += len;
 		limit_hit = tlen > req->cryptlen;
 
diff --git a/drivers/crypto/qce/aead.c b/drivers/crypto/qce/aead.c
index 9cb11fada2c4d..92d84941d3dbf 100644
--- a/drivers/crypto/qce/aead.c
+++ b/drivers/crypto/qce/aead.c
@@ -9,8 +9,6 @@
 #include <crypto/gcm.h>
 #include <crypto/authenc.h>
 #include <crypto/internal/aead.h>
-#include <crypto/internal/des.h>
-#include <crypto/sha1.h>
 #include <crypto/sha2.h>
 #include <crypto/scatterwalk.h>
 #include "aead.h"
@@ -592,7 +590,6 @@ static int qce_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int
 	struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm);
 	struct crypto_authenc_keys authenc_keys;
 	unsigned long flags = to_aead_tmpl(tfm)->alg_flags;
-	u32 _key[6];
 	int err;
 
 	err = crypto_authenc_extractkeys(&authenc_keys, key, keylen);
@@ -603,26 +600,7 @@ static int qce_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int
 	    authenc_keys.authkeylen > QCE_MAX_KEY_SIZE)
 		return -EINVAL;
 
-	if (IS_DES(flags)) {
-		err = verify_aead_des_key(tfm, authenc_keys.enckey, authenc_keys.enckeylen);
-		if (err)
-			return err;
-	} else if (IS_3DES(flags)) {
-		err = verify_aead_des3_key(tfm, authenc_keys.enckey, authenc_keys.enckeylen);
-		if (err)
-			return err;
-		/*
-		 * The crypto engine does not support any two keys
-		 * being the same for triple des algorithms. The
-		 * verify_skcipher_des3_key does not check for all the
-		 * below conditions. Schedule fallback in this case.
-		 */
-		memcpy(_key, authenc_keys.enckey, DES3_EDE_KEY_SIZE);
-		if (!((_key[0] ^ _key[2]) | (_key[1] ^ _key[3])) ||
-		    !((_key[2] ^ _key[4]) | (_key[3] ^ _key[5])) ||
-		    !((_key[0] ^ _key[4]) | (_key[1] ^ _key[5])))
-			ctx->need_fallback = true;
-	} else if (IS_AES(flags)) {
+	if (IS_AES(flags)) {
 		/* No random key sizes */
 		if (authenc_keys.enckeylen != AES_KEYSIZE_128 &&
 		    authenc_keys.enckeylen != AES_KEYSIZE_192 &&
@@ -693,38 +671,6 @@ struct qce_aead_def {
 };
 
 static const struct qce_aead_def aead_def[] = {
-	{
-		.flags          = QCE_ALG_DES | QCE_MODE_CBC | QCE_HASH_SHA1_HMAC,
-		.name           = "authenc(hmac(sha1),cbc(des))",
-		.drv_name       = "authenc-hmac-sha1-cbc-des-qce",
-		.blocksize      = DES_BLOCK_SIZE,
-		.ivsize         = DES_BLOCK_SIZE,
-		.maxauthsize	= SHA1_DIGEST_SIZE,
-	},
-	{
-		.flags          = QCE_ALG_3DES | QCE_MODE_CBC | QCE_HASH_SHA1_HMAC,
-		.name           = "authenc(hmac(sha1),cbc(des3_ede))",
-		.drv_name       = "authenc-hmac-sha1-cbc-3des-qce",
-		.blocksize      = DES3_EDE_BLOCK_SIZE,
-		.ivsize         = DES3_EDE_BLOCK_SIZE,
-		.maxauthsize	= SHA1_DIGEST_SIZE,
-	},
-	{
-		.flags          = QCE_ALG_DES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC,
-		.name           = "authenc(hmac(sha256),cbc(des))",
-		.drv_name       = "authenc-hmac-sha256-cbc-des-qce",
-		.blocksize      = DES_BLOCK_SIZE,
-		.ivsize         = DES_BLOCK_SIZE,
-		.maxauthsize	= SHA256_DIGEST_SIZE,
-	},
-	{
-		.flags          = QCE_ALG_3DES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC,
-		.name           = "authenc(hmac(sha256),cbc(des3_ede))",
-		.drv_name       = "authenc-hmac-sha256-cbc-3des-qce",
-		.blocksize      = DES3_EDE_BLOCK_SIZE,
-		.ivsize         = DES3_EDE_BLOCK_SIZE,
-		.maxauthsize	= SHA256_DIGEST_SIZE,
-	},
 	{
 		.flags          =  QCE_ALG_AES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC,
 		.name           = "authenc(hmac(sha256),cbc(aes))",
diff --git a/drivers/crypto/qce/common.c b/drivers/crypto/qce/common.c
index 54a78a57f6302..3081d765a0f2c 100644
--- a/drivers/crypto/qce/common.c
+++ b/drivers/crypto/qce/common.c
@@ -8,7 +8,6 @@
 #include <linux/interrupt.h>
 #include <linux/types.h>
 #include <crypto/scatterwalk.h>
-#include <crypto/sha1.h>
 #include <crypto/sha2.h>
 
 #include "cipher.h"
@@ -115,18 +114,16 @@ static u32 qce_auth_cfg(unsigned long flags, u32 key_size, u32 auth_size)
 			cfg |= AUTH_KEY_SZ_AES256 << AUTH_KEY_SIZE_SHIFT;
 	}
 
-	if (IS_SHA1(flags) || IS_SHA1_HMAC(flags))
-		cfg |= AUTH_SIZE_SHA1 << AUTH_SIZE_SHIFT;
-	else if (IS_SHA256(flags) || IS_SHA256_HMAC(flags))
+	if (IS_SHA256(flags) || IS_SHA256_HMAC(flags))
 		cfg |= AUTH_SIZE_SHA256 << AUTH_SIZE_SHIFT;
 	else if (IS_CMAC(flags))
 		cfg |= AUTH_SIZE_ENUM_16_BYTES << AUTH_SIZE_SHIFT;
 	else if (IS_CCM(flags))
 		cfg |= (auth_size - 1) << AUTH_SIZE_SHIFT;
 
-	if (IS_SHA1(flags) || IS_SHA256(flags))
+	if (IS_SHA256(flags))
 		cfg |= AUTH_MODE_HASH << AUTH_MODE_SHIFT;
-	else if (IS_SHA1_HMAC(flags) || IS_SHA256_HMAC(flags))
+	else if (IS_SHA256_HMAC(flags))
 		cfg |= AUTH_MODE_HMAC << AUTH_MODE_SHIFT;
 	else if (IS_CCM(flags))
 		cfg |= AUTH_MODE_CCM << AUTH_MODE_SHIFT;
@@ -191,7 +188,7 @@ static int qce_setup_regs_ahash(struct crypto_async_request *async_req)
 	else
 		qce_cpu_to_be32p_array(auth, rctx->digest, digestsize);
 
-	iv_words = (IS_SHA1(rctx->flags) || IS_SHA1_HMAC(rctx->flags)) ? 5 : 8;
+	iv_words = 8;
 	qce_write_array(qce, REG_AUTH_IV0, (u32 *)auth, iv_words);
 
 	if (rctx->first_blk)
@@ -243,19 +240,8 @@ static u32 qce_encr_cfg(unsigned long flags, u32 aes_key_size)
 
 	if (IS_AES(flags))
 		cfg |= ENCR_ALG_AES << ENCR_ALG_SHIFT;
-	else if (IS_DES(flags) || IS_3DES(flags))
-		cfg |= ENCR_ALG_DES << ENCR_ALG_SHIFT;
-
-	if (IS_DES(flags))
-		cfg |= ENCR_KEY_SZ_DES << ENCR_KEY_SZ_SHIFT;
-
-	if (IS_3DES(flags))
-		cfg |= ENCR_KEY_SZ_3DES << ENCR_KEY_SZ_SHIFT;
 
 	switch (flags & QCE_MODE_MASK) {
-	case QCE_MODE_ECB:
-		cfg |= ENCR_MODE_ECB << ENCR_MODE_SHIFT;
-		break;
 	case QCE_MODE_CBC:
 		cfg |= ENCR_MODE_CBC << ENCR_MODE_SHIFT;
 		break;
@@ -340,13 +326,7 @@ static int qce_setup_regs_skcipher(struct crypto_async_request *async_req)
 
 	encr_cfg = qce_encr_cfg(flags, keylen);
 
-	if (IS_DES(flags)) {
-		enciv_words = 2;
-		enckey_words = 2;
-	} else if (IS_3DES(flags)) {
-		enciv_words = 2;
-		enckey_words = 6;
-	} else if (IS_AES(flags)) {
+	if (IS_AES(flags)) {
 		if (IS_XTS(flags))
 			qce_xtskey(qce, ctx->enc_key, ctx->enc_keylen,
 				   rctx->cryptlen);
@@ -357,14 +337,12 @@ static int qce_setup_regs_skcipher(struct crypto_async_request *async_req)
 
 	qce_write_array(qce, REG_ENCR_KEY0, (u32 *)enckey, enckey_words);
 
-	if (!IS_ECB(flags)) {
-		if (IS_XTS(flags))
-			qce_xts_swapiv(enciv, rctx->iv, ivsize);
-		else
-			qce_cpu_to_be32p_array(enciv, rctx->iv, ivsize);
+	if (IS_XTS(flags))
+		qce_xts_swapiv(enciv, rctx->iv, ivsize);
+	else
+		qce_cpu_to_be32p_array(enciv, rctx->iv, ivsize);
 
-		qce_write_array(qce, REG_CNTR0_IV0, (u32 *)enciv, enciv_words);
-	}
+	qce_write_array(qce, REG_CNTR0_IV0, (u32 *)enciv, enciv_words);
 
 	if (IS_ENCRYPT(flags))
 		encr_cfg |= BIT(ENCODE_SHIFT);
@@ -393,10 +371,6 @@ static int qce_setup_regs_skcipher(struct crypto_async_request *async_req)
 #endif
 
 #ifdef CONFIG_CRYPTO_DEV_QCE_AEAD
-static const u32 std_iv_sha1[SHA256_DIGEST_SIZE / sizeof(u32)] = {
-	SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, 0, 0, 0
-};
-
 static const u32 std_iv_sha256[SHA256_DIGEST_SIZE / sizeof(u32)] = {
 	SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
 	SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7
@@ -473,13 +447,8 @@ static int qce_setup_regs_aead(struct crypto_async_request *async_req)
 	/* Write initial authentication IV only for HMAC algorithms */
 	if (IS_SHA_HMAC(rctx->flags)) {
 		/* Write default authentication iv */
-		if (IS_SHA1_HMAC(rctx->flags)) {
-			auth_ivsize = SHA1_DIGEST_SIZE;
-			memcpy(authiv, std_iv_sha1, auth_ivsize);
-		} else if (IS_SHA256_HMAC(rctx->flags)) {
-			auth_ivsize = SHA256_DIGEST_SIZE;
-			memcpy(authiv, std_iv_sha256, auth_ivsize);
-		}
+		auth_ivsize = SHA256_DIGEST_SIZE;
+		memcpy(authiv, std_iv_sha256, auth_ivsize);
 		authiv_words = auth_ivsize / sizeof(u32);
 		qce_write_array(qce, REG_AUTH_IV0, (u32 *)authiv, authiv_words);
 	} else if (IS_CCM(rctx->flags)) {
diff --git a/drivers/crypto/qce/common.h b/drivers/crypto/qce/common.h
index 02e63ad9f2455..9cd2e6ed8bbb0 100644
--- a/drivers/crypto/qce/common.h
+++ b/drivers/crypto/qce/common.h
@@ -22,7 +22,7 @@
 
 /* IV length in bytes */
 #define QCE_AES_IV_LENGTH		AES_BLOCK_SIZE
-/* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
+/* max of AES_BLOCK_SIZE */
 #define QCE_MAX_IV_SIZE			AES_BLOCK_SIZE
 
 /* maximum nonce bytes  */
@@ -33,14 +33,10 @@
 #define QCE_MAX_ALIGN_SIZE		64
 
 /* cipher algorithms */
-#define QCE_ALG_DES			BIT(0)
-#define QCE_ALG_3DES			BIT(1)
 #define QCE_ALG_AES			BIT(2)
 
 /* hash and hmac algorithms */
-#define QCE_HASH_SHA1			BIT(3)
 #define QCE_HASH_SHA256			BIT(4)
-#define QCE_HASH_SHA1_HMAC		BIT(5)
 #define QCE_HASH_SHA256_HMAC		BIT(6)
 #define QCE_HASH_AES_CMAC		BIT(7)
 
@@ -58,21 +54,15 @@
 #define QCE_ENCRYPT			BIT(30)
 #define QCE_DECRYPT			BIT(31)
 
-#define IS_DES(flags)			(flags & QCE_ALG_DES)
-#define IS_3DES(flags)			(flags & QCE_ALG_3DES)
 #define IS_AES(flags)			(flags & QCE_ALG_AES)
 
-#define IS_SHA1(flags)			(flags & QCE_HASH_SHA1)
 #define IS_SHA256(flags)		(flags & QCE_HASH_SHA256)
-#define IS_SHA1_HMAC(flags)		(flags & QCE_HASH_SHA1_HMAC)
 #define IS_SHA256_HMAC(flags)		(flags & QCE_HASH_SHA256_HMAC)
 #define IS_CMAC(flags)			(flags & QCE_HASH_AES_CMAC)
-#define IS_SHA(flags)			(IS_SHA1(flags) || IS_SHA256(flags))
-#define IS_SHA_HMAC(flags)		\
-		(IS_SHA1_HMAC(flags) || IS_SHA256_HMAC(flags))
+#define IS_SHA(flags)			IS_SHA256(flags)
+#define IS_SHA_HMAC(flags)		IS_SHA256_HMAC(flags)
 
 #define IS_CBC(mode)			(mode & QCE_MODE_CBC)
-#define IS_ECB(mode)			(mode & QCE_MODE_ECB)
 #define IS_CTR(mode)			(mode & QCE_MODE_CTR)
 #define IS_XTS(mode)			(mode & QCE_MODE_XTS)
 #define IS_CCM(mode)			(mode & QCE_MODE_CCM)
diff --git a/drivers/crypto/qce/regs-v5.h b/drivers/crypto/qce/regs-v5.h
index d59ed27989062..431a7db1a4e72 100644
--- a/drivers/crypto/qce/regs-v5.h
+++ b/drivers/crypto/qce/regs-v5.h
@@ -203,7 +203,6 @@
 
 #define AUTH_SIZE_SHIFT			9
 #define AUTH_SIZE_MASK			GENMASK(13, 9)
-#define AUTH_SIZE_SHA1			0
 #define AUTH_SIZE_SHA256		1
 #define AUTH_SIZE_ENUM_1_BYTES		0
 #define AUTH_SIZE_ENUM_2_BYTES		1
@@ -284,15 +283,12 @@
 
 #define ENCR_KEY_SZ_SHIFT		3
 #define ENCR_KEY_SZ_MASK		GENMASK(5, 3)
-#define ENCR_KEY_SZ_DES			0
-#define ENCR_KEY_SZ_3DES		1
 #define ENCR_KEY_SZ_AES128		0
 #define ENCR_KEY_SZ_AES256		2
 
 #define ENCR_ALG_SHIFT			0
 #define ENCR_ALG_MASK			GENMASK(2, 0)
 #define ENCR_ALG_NONE			0
-#define ENCR_ALG_DES			1
 #define ENCR_ALG_AES			2
 #define ENCR_ALG_KASUMI			4
 #define ENCR_ALG_SNOW_3G		5
diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c
index 1b37121cbcdcb..eb8a5b0c4173f 100644
--- a/drivers/crypto/qce/sha.c
+++ b/drivers/crypto/qce/sha.c
@@ -25,10 +25,6 @@ struct qce_sha_saved_state {
 
 static LIST_HEAD(ahash_algs);
 
-static const u32 std_iv_sha1[SHA256_DIGEST_SIZE / sizeof(u32)] = {
-	SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, 0, 0, 0
-};
-
 static const u32 std_iv_sha256[SHA256_DIGEST_SIZE / sizeof(u32)] = {
 	SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
 	SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7
@@ -187,10 +183,8 @@ static int qce_ahash_update(struct ahash_request *req)
 	struct qce_sha_reqctx *rctx = ahash_request_ctx_dma(req);
 	struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm);
 	struct qce_device *qce = tmpl->qce;
-	struct scatterlist *sg_last, *sg;
-	unsigned int total, len;
+	unsigned int total;
 	unsigned int hash_later;
-	unsigned int nbytes;
 	unsigned int blocksize;
 
 	blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
@@ -238,28 +232,8 @@ static int qce_ahash_update(struct ahash_request *req)
 	if (!hash_later)
 		hash_later = blocksize;
 
-	if (hash_later) {
-		unsigned int src_offset = req->nbytes - hash_later;
-		scatterwalk_map_and_copy(rctx->buf, req->src, src_offset,
-					 hash_later, 0);
-	}
-
-	/* here nbytes is multiple of blocksize */
-	nbytes = total - hash_later;
-
-	len = rctx->buflen;
-	sg = sg_last = req->src;
-
-	while (len < nbytes && sg) {
-		if (len + sg_dma_len(sg) > nbytes)
-			break;
-		len += sg_dma_len(sg);
-		sg_last = sg;
-		sg = sg_next(sg);
-	}
-
-	if (!sg_last)
-		return -EINVAL;
+	scatterwalk_map_and_copy(rctx->buf, req->src, req->nbytes - hash_later,
+				 hash_later, 0);
 
 	if (rctx->buflen) {
 		sg_init_table(rctx->sg, 2);
@@ -268,7 +242,8 @@ static int qce_ahash_update(struct ahash_request *req)
 		req->src = rctx->sg;
 	}
 
-	req->nbytes = nbytes;
+	/* hash only complete blocks */
+	req->nbytes = total - hash_later;
 	rctx->buflen = hash_later;
 
 	return qce->async_req_enqueue(tmpl->qce, &req->base);
@@ -349,9 +324,7 @@ static int qce_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key,
 		return 0;
 	}
 
-	if (digestsize == SHA1_DIGEST_SIZE)
-		alg_name = "sha1-qce";
-	else if (digestsize == SHA256_DIGEST_SIZE)
+	if (digestsize == SHA256_DIGEST_SIZE)
 		alg_name = "sha256-qce";
 	else
 		return -EINVAL;
@@ -412,15 +385,6 @@ struct qce_ahash_def {
 };
 
 static const struct qce_ahash_def ahash_def[] = {
-	{
-		.flags		= QCE_HASH_SHA1,
-		.name		= "sha1",
-		.drv_name	= "sha1-qce",
-		.digestsize	= SHA1_DIGEST_SIZE,
-		.blocksize	= SHA1_BLOCK_SIZE,
-		.statesize	= sizeof(struct qce_sha_saved_state),
-		.std_iv		= std_iv_sha1,
-	},
 	{
 		.flags		= QCE_HASH_SHA256,
 		.name		= "sha256",
@@ -430,15 +394,6 @@ static const struct qce_ahash_def ahash_def[] = {
 		.statesize	= sizeof(struct qce_sha_saved_state),
 		.std_iv		= std_iv_sha256,
 	},
-	{
-		.flags		= QCE_HASH_SHA1_HMAC,
-		.name		= "hmac(sha1)",
-		.drv_name	= "hmac-sha1-qce",
-		.digestsize	= SHA1_DIGEST_SIZE,
-		.blocksize	= SHA1_BLOCK_SIZE,
-		.statesize	= sizeof(struct qce_sha_saved_state),
-		.std_iv		= std_iv_sha1,
-	},
 	{
 		.flags		= QCE_HASH_SHA256_HMAC,
 		.name		= "hmac(sha256)",
@@ -476,9 +431,7 @@ static int qce_ahash_register_one(const struct qce_ahash_def *def,
 	alg->halg.digestsize = def->digestsize;
 	alg->halg.statesize = def->statesize;
 
-	if (IS_SHA1(def->flags))
-		tmpl->hash_zero = sha1_zero_message_hash;
-	else if (IS_SHA256(def->flags))
+	if (IS_SHA256(def->flags))
 		tmpl->hash_zero = sha256_zero_message_hash;
 
 	base = &alg->halg.base;
diff --git a/drivers/crypto/qce/sha.h b/drivers/crypto/qce/sha.h
index a22695361f165..cb822fc334dc1 100644
--- a/drivers/crypto/qce/sha.h
+++ b/drivers/crypto/qce/sha.h
@@ -7,7 +7,6 @@
 #define _SHA_H_
 
 #include <crypto/scatterwalk.h>
-#include <crypto/sha1.h>
 #include <crypto/sha2.h>
 
 #include "common.h"
diff --git a/drivers/crypto/qce/skcipher.c b/drivers/crypto/qce/skcipher.c
index db0b648a56eb1..ff4ee9541f7cb 100644
--- a/drivers/crypto/qce/skcipher.c
+++ b/drivers/crypto/qce/skcipher.c
@@ -11,7 +11,6 @@
 #include <linux/types.h>
 #include <linux/errno.h>
 #include <crypto/aes.h>
-#include <crypto/internal/des.h>
 #include <crypto/internal/skcipher.h>
 
 #include "cipher.h"
@@ -209,51 +208,6 @@ static int qce_skcipher_setkey(struct crypto_skcipher *ablk, const u8 *key,
 	return ret;
 }
 
-static int qce_des_setkey(struct crypto_skcipher *ablk, const u8 *key,
-			  unsigned int keylen)
-{
-	struct qce_cipher_ctx *ctx = crypto_skcipher_ctx(ablk);
-	int err;
-
-	err = verify_skcipher_des_key(ablk, key);
-	if (err)
-		return err;
-
-	ctx->enc_keylen = keylen;
-	memcpy(ctx->enc_key, key, keylen);
-	return 0;
-}
-
-static int qce_des3_setkey(struct crypto_skcipher *ablk, const u8 *key,
-			   unsigned int keylen)
-{
-	struct qce_cipher_ctx *ctx = crypto_skcipher_ctx(ablk);
-	u32 _key[6];
-	int err;
-
-	err = verify_skcipher_des3_key(ablk, key);
-	if (err)
-		return err;
-
-	/*
-	 * The crypto engine does not support any two keys
-	 * being the same for triple des algorithms. The
-	 * verify_skcipher_des3_key does not check for all the
-	 * below conditions. Return -ENOKEY in case any two keys
-	 * are the same. Revisit to see if a fallback cipher
-	 * is needed to handle this condition.
-	 */
-	memcpy(_key, key, DES3_EDE_KEY_SIZE);
-	if (!((_key[0] ^ _key[2]) | (_key[1] ^ _key[3])) ||
-	    !((_key[2] ^ _key[4]) | (_key[3] ^ _key[5])) ||
-	    !((_key[0] ^ _key[4]) | (_key[1] ^ _key[5])))
-		return -ENOKEY;
-
-	ctx->enc_keylen = keylen;
-	memcpy(ctx->enc_key, key, keylen);
-	return 0;
-}
-
 static int qce_skcipher_crypt(struct skcipher_request *req, int encrypt)
 {
 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -276,7 +230,7 @@ static int qce_skcipher_crypt(struct skcipher_request *req, int encrypt)
 	 * ECB and CBC algorithms require message lengths to be
 	 * multiples of block size.
 	 */
-	if (IS_ECB(rctx->flags) || IS_CBC(rctx->flags))
+	if (IS_CBC(rctx->flags))
 		if (!IS_ALIGNED(req->cryptlen, blocksize))
 			return -EINVAL;
 
@@ -359,15 +313,6 @@ struct qce_skcipher_def {
 };
 
 static const struct qce_skcipher_def skcipher_def[] = {
-	{
-		.flags		= QCE_ALG_AES | QCE_MODE_ECB,
-		.name		= "ecb(aes)",
-		.drv_name	= "ecb-aes-qce",
-		.blocksize	= AES_BLOCK_SIZE,
-		.ivsize		= 0,
-		.min_keysize	= AES_MIN_KEY_SIZE,
-		.max_keysize	= AES_MAX_KEY_SIZE,
-	},
 	{
 		.flags		= QCE_ALG_AES | QCE_MODE_CBC,
 		.name		= "cbc(aes)",
@@ -396,42 +341,6 @@ static const struct qce_skcipher_def skcipher_def[] = {
 		.min_keysize	= AES_MIN_KEY_SIZE * 2,
 		.max_keysize	= AES_MAX_KEY_SIZE * 2,
 	},
-	{
-		.flags		= QCE_ALG_DES | QCE_MODE_ECB,
-		.name		= "ecb(des)",
-		.drv_name	= "ecb-des-qce",
-		.blocksize	= DES_BLOCK_SIZE,
-		.ivsize		= 0,
-		.min_keysize	= DES_KEY_SIZE,
-		.max_keysize	= DES_KEY_SIZE,
-	},
-	{
-		.flags		= QCE_ALG_DES | QCE_MODE_CBC,
-		.name		= "cbc(des)",
-		.drv_name	= "cbc-des-qce",
-		.blocksize	= DES_BLOCK_SIZE,
-		.ivsize		= DES_BLOCK_SIZE,
-		.min_keysize	= DES_KEY_SIZE,
-		.max_keysize	= DES_KEY_SIZE,
-	},
-	{
-		.flags		= QCE_ALG_3DES | QCE_MODE_ECB,
-		.name		= "ecb(des3_ede)",
-		.drv_name	= "ecb-3des-qce",
-		.blocksize	= DES3_EDE_BLOCK_SIZE,
-		.ivsize		= 0,
-		.min_keysize	= DES3_EDE_KEY_SIZE,
-		.max_keysize	= DES3_EDE_KEY_SIZE,
-	},
-	{
-		.flags		= QCE_ALG_3DES | QCE_MODE_CBC,
-		.name		= "cbc(des3_ede)",
-		.drv_name	= "cbc-3des-qce",
-		.blocksize	= DES3_EDE_BLOCK_SIZE,
-		.ivsize		= DES3_EDE_BLOCK_SIZE,
-		.min_keysize	= DES3_EDE_KEY_SIZE,
-		.max_keysize	= DES3_EDE_KEY_SIZE,
-	},
 };
 
 static int qce_skcipher_register_one(const struct qce_skcipher_def *def,
@@ -455,9 +364,7 @@ static int qce_skcipher_register_one(const struct qce_skcipher_def *def,
 	alg->ivsize			= def->ivsize;
 	alg->min_keysize		= def->min_keysize;
 	alg->max_keysize		= def->max_keysize;
-	alg->setkey			= IS_3DES(def->flags) ? qce_des3_setkey :
-					  IS_DES(def->flags) ? qce_des_setkey :
-					  qce_skcipher_setkey;
+	alg->setkey			= qce_skcipher_setkey;
 	alg->encrypt			= qce_skcipher_encrypt;
 	alg->decrypt			= qce_skcipher_decrypt;
 
diff --git a/drivers/crypto/qcom-rng.c b/drivers/crypto/qcom-rng.c
deleted file mode 100644
index 150e5802e3515..0000000000000
--- a/drivers/crypto/qcom-rng.c
+++ /dev/null
@@ -1,276 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-// Copyright (c) 2017-18 Linaro Limited
-//
-// Based on msm-rng.c and downstream driver
-
-#include <crypto/internal/rng.h>
-#include <linux/acpi.h>
-#include <linux/clk.h>
-#include <linux/crypto.h>
-#include <linux/hw_random.h>
-#include <linux/io.h>
-#include <linux/iopoll.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
-
-/* Device specific register offsets */
-#define PRNG_DATA_OUT		0x0000
-#define PRNG_STATUS		0x0004
-#define PRNG_LFSR_CFG		0x0100
-#define PRNG_CONFIG		0x0104
-
-/* Device specific register masks and config values */
-#define PRNG_LFSR_CFG_MASK	0x0000ffff
-#define PRNG_LFSR_CFG_CLOCKS	0x0000dddd
-#define PRNG_CONFIG_HW_ENABLE	BIT(1)
-#define PRNG_STATUS_DATA_AVAIL	BIT(0)
-
-#define WORD_SZ			4
-
-#define QCOM_TRNG_QUALITY	1024
-
-struct qcom_rng {
-	struct mutex lock;
-	void __iomem *base;
-	struct clk *clk;
-	struct hwrng hwrng;
-	struct qcom_rng_match_data *match_data;
-};
-
-struct qcom_rng_ctx {
-	struct qcom_rng *rng;
-};
-
-struct qcom_rng_match_data {
-	bool skip_init;
-	bool hwrng_support;
-};
-
-static struct qcom_rng *qcom_rng_dev;
-
-static int qcom_rng_read(struct qcom_rng *rng, u8 *data, unsigned int max)
-{
-	unsigned int currsize = 0;
-	u32 val;
-	int ret;
-
-	/* read random data from hardware */
-	do {
-		ret = readl_poll_timeout(rng->base + PRNG_STATUS, val,
-					 val & PRNG_STATUS_DATA_AVAIL,
-					 200, 10000);
-		if (ret)
-			return ret;
-
-		val = readl_relaxed(rng->base + PRNG_DATA_OUT);
-		if (!val)
-			return -EINVAL;
-
-		if ((max - currsize) >= WORD_SZ) {
-			memcpy(data, &val, WORD_SZ);
-			data += WORD_SZ;
-			currsize += WORD_SZ;
-		} else {
-			/* copy only remaining bytes */
-			memcpy(data, &val, max - currsize);
-			currsize = max;
-		}
-	} while (currsize < max);
-
-	return currsize;
-}
-
-static int qcom_rng_generate(struct crypto_rng *tfm,
-			     const u8 *src, unsigned int slen,
-			     u8 *dstn, unsigned int dlen)
-{
-	struct qcom_rng_ctx *ctx = crypto_rng_ctx(tfm);
-	struct qcom_rng *rng = ctx->rng;
-	int ret;
-
-	ret = clk_prepare_enable(rng->clk);
-	if (ret)
-		return ret;
-
-	mutex_lock(&rng->lock);
-
-	ret = qcom_rng_read(rng, dstn, dlen);
-
-	mutex_unlock(&rng->lock);
-	clk_disable_unprepare(rng->clk);
-
-	if (ret >= 0)
-		ret = 0;
-
-	return ret;
-}
-
-static int qcom_rng_seed(struct crypto_rng *tfm, const u8 *seed,
-			 unsigned int slen)
-{
-	return 0;
-}
-
-static int qcom_hwrng_read(struct hwrng *hwrng, void *data, size_t max, bool wait)
-{
-	struct qcom_rng *qrng = container_of(hwrng, struct qcom_rng, hwrng);
-
-	return qcom_rng_read(qrng, data, max);
-}
-
-static int qcom_rng_enable(struct qcom_rng *rng)
-{
-	u32 val;
-	int ret;
-
-	ret = clk_prepare_enable(rng->clk);
-	if (ret)
-		return ret;
-
-	/* Enable PRNG only if it is not already enabled */
-	val = readl_relaxed(rng->base + PRNG_CONFIG);
-	if (val & PRNG_CONFIG_HW_ENABLE)
-		goto already_enabled;
-
-	val = readl_relaxed(rng->base + PRNG_LFSR_CFG);
-	val &= ~PRNG_LFSR_CFG_MASK;
-	val |= PRNG_LFSR_CFG_CLOCKS;
-	writel(val, rng->base + PRNG_LFSR_CFG);
-
-	val = readl_relaxed(rng->base + PRNG_CONFIG);
-	val |= PRNG_CONFIG_HW_ENABLE;
-	writel(val, rng->base + PRNG_CONFIG);
-
-already_enabled:
-	clk_disable_unprepare(rng->clk);
-
-	return 0;
-}
-
-static int qcom_rng_init(struct crypto_tfm *tfm)
-{
-	struct qcom_rng_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	ctx->rng = qcom_rng_dev;
-
-	if (!ctx->rng->match_data->skip_init)
-		return qcom_rng_enable(ctx->rng);
-
-	return 0;
-}
-
-static struct rng_alg qcom_rng_alg = {
-	.generate	= qcom_rng_generate,
-	.seed		= qcom_rng_seed,
-	.seedsize	= 0,
-	.base		= {
-		.cra_name		= "stdrng",
-		.cra_driver_name	= "qcom-rng",
-		.cra_flags		= CRYPTO_ALG_TYPE_RNG,
-		.cra_priority		= 300,
-		.cra_ctxsize		= sizeof(struct qcom_rng_ctx),
-		.cra_module		= THIS_MODULE,
-		.cra_init		= qcom_rng_init,
-	}
-};
-
-static int qcom_rng_probe(struct platform_device *pdev)
-{
-	struct qcom_rng *rng;
-	int ret;
-
-	rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL);
-	if (!rng)
-		return -ENOMEM;
-
-	platform_set_drvdata(pdev, rng);
-	mutex_init(&rng->lock);
-
-	rng->base = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(rng->base))
-		return PTR_ERR(rng->base);
-
-	rng->clk = devm_clk_get_optional(&pdev->dev, "core");
-	if (IS_ERR(rng->clk))
-		return PTR_ERR(rng->clk);
-
-	rng->match_data = (struct qcom_rng_match_data *)device_get_match_data(&pdev->dev);
-
-	qcom_rng_dev = rng;
-	ret = crypto_register_rng(&qcom_rng_alg);
-	if (ret) {
-		dev_err(&pdev->dev, "Register crypto rng failed: %d\n", ret);
-		qcom_rng_dev = NULL;
-		return ret;
-	}
-
-	if (rng->match_data->hwrng_support) {
-		rng->hwrng.name = "qcom_hwrng";
-		rng->hwrng.read = qcom_hwrng_read;
-		rng->hwrng.quality = QCOM_TRNG_QUALITY;
-		ret = devm_hwrng_register(&pdev->dev, &rng->hwrng);
-		if (ret) {
-			dev_err(&pdev->dev, "Register hwrng failed: %d\n", ret);
-			qcom_rng_dev = NULL;
-			goto fail;
-		}
-	}
-
-	return ret;
-fail:
-	crypto_unregister_rng(&qcom_rng_alg);
-	return ret;
-}
-
-static void qcom_rng_remove(struct platform_device *pdev)
-{
-	crypto_unregister_rng(&qcom_rng_alg);
-
-	qcom_rng_dev = NULL;
-}
-
-static struct qcom_rng_match_data qcom_prng_match_data = {
-	.skip_init = false,
-	.hwrng_support = false,
-};
-
-static struct qcom_rng_match_data qcom_prng_ee_match_data = {
-	.skip_init = true,
-	.hwrng_support = false,
-};
-
-static struct qcom_rng_match_data qcom_trng_match_data = {
-	.skip_init = true,
-	.hwrng_support = true,
-};
-
-static const struct acpi_device_id __maybe_unused qcom_rng_acpi_match[] = {
-	{ .id = "QCOM8160", .driver_data = (kernel_ulong_t)&qcom_prng_ee_match_data },
-	{}
-};
-MODULE_DEVICE_TABLE(acpi, qcom_rng_acpi_match);
-
-static const struct of_device_id __maybe_unused qcom_rng_of_match[] = {
-	{ .compatible = "qcom,prng", .data = &qcom_prng_match_data },
-	{ .compatible = "qcom,prng-ee", .data = &qcom_prng_ee_match_data },
-	{ .compatible = "qcom,trng", .data = &qcom_trng_match_data },
-	{}
-};
-MODULE_DEVICE_TABLE(of, qcom_rng_of_match);
-
-static struct platform_driver qcom_rng_driver = {
-	.probe = qcom_rng_probe,
-	.remove =  qcom_rng_remove,
-	.driver = {
-		.name = KBUILD_MODNAME,
-		.of_match_table = qcom_rng_of_match,
-		.acpi_match_table = ACPI_PTR(qcom_rng_acpi_match),
-	}
-};
-module_platform_driver(qcom_rng_driver);
-
-MODULE_ALIAS("platform:" KBUILD_MODNAME);
-MODULE_DESCRIPTION("Qualcomm random number generator driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/crypto/rockchip/rk3288_crypto_ahash.c b/drivers/crypto/rockchip/rk3288_crypto_ahash.c
index b9f5a8b42e661..d3482619aa2f1 100644
--- a/drivers/crypto/rockchip/rk3288_crypto_ahash.c
+++ b/drivers/crypto/rockchip/rk3288_crypto_ahash.c
@@ -324,7 +324,12 @@ static int rk_hash_run(struct crypto_engine *engine, void *breq)
 	 * efficiency, and make it response quickly when dma
 	 * complete.
 	 */
-	readl_poll_timeout(rkc->reg + RK_CRYPTO_HASH_STS, v, v == 0, 10, 1000);
+	err = readl_poll_timeout(rkc->reg + RK_CRYPTO_HASH_STS, v,
+				 v == 0, 10, 1000);
+	if (err) {
+		dev_err(rkc->dev, "HASH idle timeout\n");
+		goto theend;
+	}
 
 	for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++) {
 		v = readl(rkc->reg + RK_CRYPTO_HASH_DOUT_0 + i * 4);
diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c
index 965a03d5b27ae..d865fd4a098cb 100644
--- a/drivers/crypto/sa2ul.c
+++ b/drivers/crypto/sa2ul.c
@@ -2395,7 +2395,10 @@ static int sa_ul_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	sa_init_mem(dev_data);
+	ret = sa_init_mem(dev_data);
+	if (ret)
+		goto disable_pm;
+
 	ret = sa_dma_init(dev_data);
 	if (ret)
 		goto destroy_dma_pool;
@@ -2430,6 +2433,7 @@ static int sa_ul_probe(struct platform_device *pdev)
 destroy_dma_pool:
 	dma_pool_destroy(dev_data->sc_pool);
 
+disable_pm:
 	pm_runtime_put_sync(dev);
 	pm_runtime_disable(dev);
 
diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h
index 07555ae638594..f7cebb0265523 100644
--- a/drivers/cxl/core/core.h
+++ b/drivers/cxl/core/core.h
@@ -101,6 +101,8 @@ void __iomem *devm_cxl_iomap_block(struct device *dev, resource_size_t addr,
 struct dentry *cxl_debugfs_create_dir(const char *dir);
 int cxl_dpa_set_part(struct cxl_endpoint_decoder *cxled,
 		     enum cxl_partition_mode mode);
+struct cxl_memdev_state;
+int cxl_mem_get_partition_info(struct cxl_memdev_state *mds);
 int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, u64 size);
 int cxl_dpa_free(struct cxl_endpoint_decoder *cxled);
 resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled);
diff --git a/drivers/cxl/core/features.c b/drivers/cxl/core/features.c
index 85185af46b72d..ba6d2a5acb74a 100644
--- a/drivers/cxl/core/features.c
+++ b/drivers/cxl/core/features.c
@@ -225,7 +225,7 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
 		       void *feat_out, size_t feat_out_size, u16 offset,
 		       u16 *return_code)
 {
-	size_t data_to_rd_size, size_out;
+	size_t data_to_rd_size;
 	struct cxl_mbox_get_feat_in pi;
 	struct cxl_mbox_cmd mbox_cmd;
 	size_t data_rcvd_size = 0;
@@ -237,9 +237,10 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
 	if (!feat_out || !feat_out_size)
 		return 0;
 
-	size_out = min(feat_out_size, cxl_mbox->payload_size);
 	uuid_copy(&pi.uuid, feat_uuid);
 	pi.selection = selection;
+
+	guard(mutex)(&cxl_mbox->feat_mutex);
 	do {
 		data_to_rd_size = min(feat_out_size - data_rcvd_size,
 				      cxl_mbox->payload_size);
@@ -250,7 +251,7 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
 			.opcode = CXL_MBOX_OP_GET_FEATURE,
 			.size_in = sizeof(pi),
 			.payload_in = &pi,
-			.size_out = size_out,
+			.size_out = data_to_rd_size,
 			.payload_out = feat_out + data_rcvd_size,
 			.min_out = data_to_rd_size,
 		};
@@ -314,6 +315,7 @@ int cxl_set_feature(struct cxl_mailbox *cxl_mbox,
 		data_in_size = cxl_mbox->payload_size - hdr_size;
 	}
 
+	guard(mutex)(&cxl_mbox->feat_mutex);
 	do {
 		int rc;
 
@@ -471,6 +473,10 @@ static void *cxlctl_get_feature(struct cxl_features_state *cxlfs,
 	if (!count)
 		return ERR_PTR(-EINVAL);
 
+	if (out_size < offsetof(struct fwctl_rpc_cxl_out, payload) ||
+	    count > out_size - offsetof(struct fwctl_rpc_cxl_out, payload))
+		return ERR_PTR(-EINVAL);
+
 	struct fwctl_rpc_cxl_out *rpc_out __free(kvfree) =
 		kvzalloc(out_size, GFP_KERNEL);
 	if (!rpc_out)
@@ -516,6 +522,9 @@ static void *cxlctl_set_feature(struct cxl_features_state *cxlfs,
 	flags = le32_to_cpu(feat_in->flags);
 	out_size = *out_len;
 
+	if (out_size < offsetof(struct fwctl_rpc_cxl_out, payload))
+		return ERR_PTR(-EINVAL);
+
 	struct fwctl_rpc_cxl_out *rpc_out __free(kvfree) =
 		kvzalloc(out_size, GFP_KERNEL);
 	if (!rpc_out)
@@ -649,7 +658,13 @@ static void *cxlctl_fw_rpc(struct fwctl_uctx *uctx, enum fwctl_rpc_scope scope,
 	struct cxl_memdev *cxlmd = fwctl_to_memdev(fwctl_dev);
 	struct cxl_features_state *cxlfs = to_cxlfs(cxlmd->cxlds);
 	const struct fwctl_rpc_cxl *rpc_in = in;
-	u16 opcode = rpc_in->opcode;
+	u16 opcode;
+
+	if (in_len < sizeof(rpc_in->hdr) ||
+	    rpc_in->op_size > in_len - sizeof(rpc_in->hdr))
+		return ERR_PTR(-EINVAL);
+
+	opcode = rpc_in->opcode;
 
 	if (!cxlctl_validate_hw_command(cxlfs, rpc_in, scope, opcode))
 		return ERR_PTR(-EINVAL);
diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
index 7c6c5b7450a54..3cd64cc50ff63 100644
--- a/drivers/cxl/core/mbox.c
+++ b/drivers/cxl/core/mbox.c
@@ -11,7 +11,6 @@
 
 #include "core.h"
 #include "trace.h"
-#include "mce.h"
 
 static bool cxl_raw_allow_all;
 
@@ -380,11 +379,7 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox_cmd,
 		}
 	}
 
-	/* Prepare to handle a full payload for variable sized output */
-	if (out_size == CXL_VARIABLE_PAYLOAD)
-		mbox_cmd->size_out = cxl_mbox->payload_size;
-	else
-		mbox_cmd->size_out = out_size;
+	mbox_cmd->size_out = min_t(size_t, out_size, cxl_mbox->payload_size);
 
 	if (mbox_cmd->size_out) {
 		mbox_cmd->payload_out = kvzalloc(mbox_cmd->size_out, GFP_KERNEL);
@@ -1152,7 +1147,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_mem_get_event_records, "CXL");
  *
  * See CXL @8.2.9.5.2.1 Get Partition Info
  */
-static int cxl_mem_get_partition_info(struct cxl_memdev_state *mds)
+int cxl_mem_get_partition_info(struct cxl_memdev_state *mds)
 {
 	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
 	struct cxl_mbox_get_partition_info pi;
@@ -1308,55 +1303,6 @@ int cxl_mem_sanitize(struct cxl_memdev *cxlmd, u16 cmd)
 	return -EBUSY;
 }
 
-static void add_part(struct cxl_dpa_info *info, u64 start, u64 size, enum cxl_partition_mode mode)
-{
-	int i = info->nr_partitions;
-
-	if (size == 0)
-		return;
-
-	info->part[i].range = (struct range) {
-		.start = start,
-		.end = start + size - 1,
-	};
-	info->part[i].mode = mode;
-	info->nr_partitions++;
-}
-
-int cxl_mem_dpa_fetch(struct cxl_memdev_state *mds, struct cxl_dpa_info *info)
-{
-	struct cxl_dev_state *cxlds = &mds->cxlds;
-	struct device *dev = cxlds->dev;
-	int rc;
-
-	if (!cxlds->media_ready) {
-		info->size = 0;
-		return 0;
-	}
-
-	info->size = mds->total_bytes;
-
-	if (mds->partition_align_bytes == 0) {
-		add_part(info, 0, mds->volatile_only_bytes, CXL_PARTMODE_RAM);
-		add_part(info, mds->volatile_only_bytes,
-			 mds->persistent_only_bytes, CXL_PARTMODE_PMEM);
-		return 0;
-	}
-
-	rc = cxl_mem_get_partition_info(mds);
-	if (rc) {
-		dev_err(dev, "Failed to query partition information\n");
-		return rc;
-	}
-
-	add_part(info, 0, mds->active_volatile_bytes, CXL_PARTMODE_RAM);
-	add_part(info, mds->active_volatile_bytes, mds->active_persistent_bytes,
-		 CXL_PARTMODE_PMEM);
-
-	return 0;
-}
-EXPORT_SYMBOL_NS_GPL(cxl_mem_dpa_fetch, "CXL");
-
 int cxl_get_dirty_count(struct cxl_memdev_state *mds, u32 *count)
 {
 	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
@@ -1455,6 +1401,11 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
 		if (rc)
 			break;
 
+		if (!le16_to_cpu(po->count)) {
+			dev_dbg(&cxlmd->dev, "Poison empty payload!\n");
+			break;
+		}
+
 		for (int i = 0; i < le16_to_cpu(po->count); i++)
 			trace_cxl_poison(cxlmd, cxlr, &po->record[i],
 					 po->flags, po->overflow_ts,
@@ -1516,6 +1467,7 @@ int cxl_mailbox_init(struct cxl_mailbox *cxl_mbox, struct device *host)
 
 	cxl_mbox->host = host;
 	mutex_init(&cxl_mbox->mbox_mutex);
+	mutex_init(&cxl_mbox->feat_mutex);
 	rcuwait_init(&cxl_mbox->mbox_wait);
 
 	return 0;
@@ -1526,7 +1478,6 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev, u64 serial,
 						 u16 dvsec)
 {
 	struct cxl_memdev_state *mds;
-	int rc;
 
 	mds = devm_cxl_dev_state_create(dev, CXL_DEVTYPE_CLASSMEM, serial,
 					dvsec, struct cxl_memdev_state, cxlds,
@@ -1538,12 +1489,6 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev, u64 serial,
 
 	mutex_init(&mds->event.log_lock);
 
-	rc = devm_cxl_register_mce_notifier(dev, &mds->mce_notifier);
-	if (rc == -EOPNOTSUPP)
-		dev_warn(dev, "CXL MCE unsupported\n");
-	else if (rc)
-		return ERR_PTR(rc);
-
 	return mds;
 }
 EXPORT_SYMBOL_NS_GPL(cxl_memdev_state_create, "CXL");
diff --git a/drivers/cxl/core/mce.c b/drivers/cxl/core/mce.c
index ff8d078c6ca1e..65fed913b2217 100644
--- a/drivers/cxl/core/mce.c
+++ b/drivers/cxl/core/mce.c
@@ -4,16 +4,16 @@
 #include <linux/notifier.h>
 #include <linux/set_memory.h>
 #include <asm/mce.h>
-#include <cxlmem.h>
+#include <cxl.h>
+#include "core.h"
 #include "mce.h"
 
 static int cxl_handle_mce(struct notifier_block *nb, unsigned long val,
 			  void *data)
 {
-	struct cxl_memdev_state *mds = container_of(nb, struct cxl_memdev_state,
-						    mce_notifier);
-	struct cxl_memdev *cxlmd = mds->cxlds.cxlmd;
-	struct cxl_port *endpoint = cxlmd->endpoint;
+	struct cxl_region *cxlr = container_of(nb, struct cxl_region,
+					       mce_notifier);
+	struct cxl_region_params *p = &cxlr->params;
 	struct mce *mce = data;
 	u64 spa, spa_alias;
 	unsigned long pfn;
@@ -21,26 +21,25 @@ static int cxl_handle_mce(struct notifier_block *nb, unsigned long val,
 	if (!mce || !mce_usable_address(mce))
 		return NOTIFY_DONE;
 
-	if (!endpoint)
-		return NOTIFY_DONE;
-
 	spa = mce->addr & MCI_ADDR_PHYSADDR;
 
-	pfn = spa >> PAGE_SHIFT;
-	if (!pfn_valid(pfn))
+	if (!cxl_resource_contains_addr(p->res, spa))
 		return NOTIFY_DONE;
 
-	spa_alias = cxl_port_get_spa_cache_alias(endpoint, spa);
-	if (spa_alias == ~0ULL)
-		return NOTIFY_DONE;
+	if (spa >= p->res->start + p->cache_size)
+		spa_alias = spa - p->cache_size;
+	else
+		spa_alias = spa + p->cache_size;
 
 	pfn = spa_alias >> PAGE_SHIFT;
+	if (!pfn_valid(pfn))
+		return NOTIFY_DONE;
 
 	/*
 	 * Take down the aliased memory page. The original memory page flagged
 	 * by the MCE will be taken cared of by the standard MCE handler.
 	 */
-	dev_emerg(mds->cxlds.dev, "Offlining aliased SPA address0: %#llx\n",
+	dev_emerg(&cxlr->dev, "Offlining aliased SPA address0: %#llx\n",
 		  spa_alias);
 	if (!memory_failure(pfn, 0))
 		set_mce_nospec(pfn);
diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
index 33a3d2e7b13af..b3419df586b9c 100644
--- a/drivers/cxl/core/memdev.c
+++ b/drivers/cxl/core/memdev.c
@@ -594,6 +594,73 @@ bool is_cxl_memdev(const struct device *dev)
 }
 EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, "CXL");
 
+static void add_part(struct cxl_dpa_info *info, u64 start, u64 size, enum cxl_partition_mode mode)
+{
+	int i = info->nr_partitions;
+
+	if (size == 0)
+		return;
+
+	info->part[i].range = (struct range) {
+		.start = start,
+		.end = start + size - 1,
+	};
+	info->part[i].mode = mode;
+	info->nr_partitions++;
+}
+
+int cxl_mem_dpa_fetch(struct cxl_memdev_state *mds, struct cxl_dpa_info *info)
+{
+	struct cxl_dev_state *cxlds = &mds->cxlds;
+	struct device *dev = cxlds->dev;
+	int rc;
+
+	if (!cxlds->media_ready) {
+		info->size = 0;
+		return 0;
+	}
+
+	info->size = mds->total_bytes;
+
+	if (mds->partition_align_bytes == 0) {
+		add_part(info, 0, mds->volatile_only_bytes, CXL_PARTMODE_RAM);
+		add_part(info, mds->volatile_only_bytes,
+			 mds->persistent_only_bytes, CXL_PARTMODE_PMEM);
+		return 0;
+	}
+
+	rc = cxl_mem_get_partition_info(mds);
+	if (rc) {
+		dev_err(dev, "Failed to query partition information\n");
+		return rc;
+	}
+
+	add_part(info, 0, mds->active_volatile_bytes, CXL_PARTMODE_RAM);
+	add_part(info, mds->active_volatile_bytes, mds->active_persistent_bytes,
+		 CXL_PARTMODE_PMEM);
+
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_mem_dpa_fetch, "CXL");
+
+
+/**
+ * cxl_set_capacity: initialize dpa by a driver without a mailbox.
+ *
+ * @cxlds: pointer to cxl_dev_state
+ * @capacity: device volatile memory size
+ */
+int cxl_set_capacity(struct cxl_dev_state *cxlds, u64 capacity)
+{
+	struct cxl_dpa_info range_info = {
+		.size = capacity,
+	};
+
+	add_part(&range_info, 0, capacity, CXL_PARTMODE_RAM);
+	return cxl_dpa_setup(cxlds, &range_info);
+}
+EXPORT_SYMBOL_NS_GPL(cxl_set_capacity, "CXL");
+
 /**
  * set_exclusive_cxl_commands() - atomically disable user cxl commands
  * @mds: The device state to operate on
@@ -921,7 +988,7 @@ static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data,
 	if (!size)
 		return FW_UPLOAD_ERR_INVALID_SIZE;
 
-	mds->fw.oneshot = struct_size(transfer, data, size) <
+	mds->fw.oneshot = struct_size(transfer, data, size) <=
 			    cxl_mbox->payload_size;
 
 	if (cxl_mem_get_fw_info(mds))
diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
index e4338fd7e01b4..9d807c1a002ca 100644
--- a/drivers/cxl/core/pci.c
+++ b/drivers/cxl/core/pci.c
@@ -6,6 +6,7 @@
 #include <linux/delay.h>
 #include <linux/pci.h>
 #include <linux/pci-doe.h>
+#include <cxl/pci.h>
 #include <linux/aer.h>
 #include <cxlpci.h>
 #include <cxlmem.h>
diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
index 1215ee4f40351..3122101984354 100644
--- a/drivers/cxl/core/port.c
+++ b/drivers/cxl/core/port.c
@@ -11,6 +11,7 @@
 #include <linux/idr.h>
 #include <linux/node.h>
 #include <cxl/einj.h>
+#include <cxl/pci.h>
 #include <cxlmem.h>
 #include <cxlpci.h>
 #include <cxl.h>
@@ -1749,8 +1750,8 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd,
 					     parent_dport, uport_dev,
 					     dport_dev);
 		if (IS_ERR(dport)) {
-			/* Port already exists, restart iteration */
-			if (PTR_ERR(dport) == -EAGAIN)
+			/* Port or dport already exists, restart iteration */
+			if (PTR_ERR(dport) == -EAGAIN || PTR_ERR(dport) == -EBUSY)
 				return 0;
 			return PTR_ERR(dport);
 		}
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index 1e211542b6b64..578622240401d 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -15,6 +15,7 @@
 #include <cxlmem.h>
 #include <cxl.h>
 #include "core.h"
+#include "mce.h"
 
 /**
  * DOC: cxl core region
@@ -3859,34 +3860,6 @@ int cxl_add_to_region(struct cxl_endpoint_decoder *cxled)
 }
 EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, "CXL");
 
-u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, u64 spa)
-{
-	struct cxl_region_ref *iter;
-	unsigned long index;
-
-	if (!endpoint)
-		return ~0ULL;
-
-	guard(rwsem_write)(&cxl_rwsem.region);
-
-	xa_for_each(&endpoint->regions, index, iter) {
-		struct cxl_region_params *p = &iter->region->params;
-
-		if (cxl_resource_contains_addr(p->res, spa)) {
-			if (!p->cache_size)
-				return ~0ULL;
-
-			if (spa >= p->res->start + p->cache_size)
-				return spa - p->cache_size;
-
-			return spa + p->cache_size;
-		}
-	}
-
-	return ~0ULL;
-}
-EXPORT_SYMBOL_NS_GPL(cxl_port_get_spa_cache_alias, "CXL");
-
 static int is_system_ram(struct resource *res, void *arg)
 {
 	struct cxl_region *cxlr = arg;
@@ -4217,6 +4190,20 @@ static int cxl_region_probe(struct device *dev)
 	if (rc)
 		return rc;
 
+	/*
+	 * Regions fronted by an extended linear cache need the MCE notifier to
+	 * offline the aliased page on a memory error.
+	 */
+	if (p->cache_size) {
+		rc = devm_cxl_register_mce_notifier(&cxlr->dev,
+						    &cxlr->mce_notifier);
+		if (rc == -EOPNOTSUPP)
+			dev_warn(&cxlr->dev,
+				 "CONFIG_CXL_MCE disabled, MCE notifier not registered\n");
+		else if (rc)
+			return rc;
+	}
+
 	rc = cxl_region_setup_poison(cxlr);
 	if (rc)
 		return rc;
diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
index 93710cf4f0a69..20c2d9fbcfe7d 100644
--- a/drivers/cxl/core/regs.c
+++ b/drivers/cxl/core/regs.c
@@ -4,6 +4,7 @@
 #include <linux/device.h>
 #include <linux/slab.h>
 #include <linux/pci.h>
+#include <cxl/pci.h>
 #include <cxlmem.h>
 #include <cxlpci.h>
 #include <pmu.h>
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index c0e5308e4d1b8..bdd0132c999dc 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -478,6 +478,7 @@ struct cxl_region_params {
  * @coord: QoS access coordinates for the region
  * @node_notifier: notifier for setting the access coordinates to node
  * @adist_notifier: notifier for calculating the abstract distance of node
+ * @mce_notifier: notifier for MCE
  */
 struct cxl_region {
 	struct device dev;
@@ -493,6 +494,7 @@ struct cxl_region {
 	struct access_coordinate coord[ACCESS_COORDINATE_MAX];
 	struct notifier_block node_notifier;
 	struct notifier_block adist_notifier;
+	struct notifier_block mce_notifier;
 };
 
 struct cxl_nvdimm_bridge {
@@ -870,7 +872,6 @@ bool is_cxl_pmem_region(struct device *dev);
 struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev);
 int cxl_add_to_region(struct cxl_endpoint_decoder *cxled);
 struct cxl_dax_region *to_cxl_dax_region(struct device *dev);
-u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, u64 spa);
 bool cxl_region_contains_resource(const struct resource *res);
 #else
 static inline bool is_cxl_pmem_region(struct device *dev)
@@ -889,11 +890,6 @@ static inline struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
 {
 	return NULL;
 }
-static inline u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint,
-					       u64 spa)
-{
-	return 0;
-}
 static inline bool cxl_region_contains_resource(const struct resource *res)
 {
 	return false;
diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
index ed419d0c59f2f..c401e3a1af06f 100644
--- a/drivers/cxl/cxlmem.h
+++ b/drivers/cxl/cxlmem.h
@@ -183,7 +183,7 @@ static inline struct cxl_ep *cxl_ep_load(struct cxl_port *port,
 	C(MBUNSUPPORTED, -ENXIO, "unsupported on the mailbox it was issued on"),\
 	C(PAYLOADLEN, -ENXIO, "invalid payload length"),			\
 	C(LOG, -ENXIO, "invalid or unsupported log page"),			\
-	C(INTERRUPTED, -ENXIO, "asynchronous event occured"),			\
+	C(INTERRUPTED, -ENXIO, "asynchronous event occurred"),			\
 	C(FEATUREVERSION, -ENXIO, "unsupported feature version"),		\
 	C(FEATURESELVALUE, -ENXIO, "unsupported feature selection value"),	\
 	C(FEATURETRANSFERIP, -ENXIO, "feature transfer in progress"),		\
@@ -431,7 +431,6 @@ static inline struct cxl_dev_state *mbox_to_cxlds(struct cxl_mailbox *cxl_mbox)
  * @poison: poison driver state info
  * @security: security driver state info
  * @fw: firmware upload / activation state
- * @mce_notifier: MCE notifier
  *
  * See CXL 3.0 8.2.9.8.2 Capacity Configuration and Label Storage for
  * details on capacity parameters.
@@ -451,7 +450,6 @@ struct cxl_memdev_state {
 	struct cxl_poison_state poison;
 	struct cxl_security_state security;
 	struct cxl_fw_state fw;
-	struct notifier_block mce_notifier;
 };
 
 static inline struct cxl_memdev_state *
diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
index b826eb53cf7ba..110ec9c44f09f 100644
--- a/drivers/cxl/cxlpci.h
+++ b/drivers/cxl/cxlpci.h
@@ -13,16 +13,6 @@
  */
 #define CXL_PCI_DEFAULT_MAX_VECTORS 16
 
-/* Register Block Identifier (RBI) */
-enum cxl_regloc_type {
-	CXL_REGLOC_RBI_EMPTY = 0,
-	CXL_REGLOC_RBI_COMPONENT,
-	CXL_REGLOC_RBI_VIRT,
-	CXL_REGLOC_RBI_MEMDEV,
-	CXL_REGLOC_RBI_PMU,
-	CXL_REGLOC_RBI_TYPES
-};
-
 /*
  * Table Access DOE, CDAT Read Entry Response
  *
@@ -112,6 +102,4 @@ static inline void devm_cxl_port_ras_setup(struct cxl_port *port)
 }
 #endif
 
-int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
-		       struct cxl_register_map *map);
 #endif /* __CXL_PCI_H__ */
diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
index 267c679b0b3c2..d7f499cf1d8fb 100644
--- a/drivers/cxl/pci.c
+++ b/drivers/cxl/pci.c
@@ -11,6 +11,7 @@
 #include <linux/pci.h>
 #include <linux/aer.h>
 #include <linux/io.h>
+#include <cxl/pci.h>
 #include <cxl/mailbox.h>
 #include "cxlmem.h"
 #include "cxlpci.h"
@@ -691,12 +692,6 @@ static int cxl_pci_type3_init_mailbox(struct cxl_dev_state *cxlds)
 {
 	int rc;
 
-	/*
-	 * Fail the init if there's no mailbox. For a type3 this is out of spec.
-	 */
-	if (!cxlds->reg_map.device_map.mbox.valid)
-		return -ENODEV;
-
 	rc = cxl_mailbox_init(&cxlds->cxl_mbox, cxlds->dev);
 	if (rc)
 		return rc;
@@ -829,10 +824,13 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	 */
 	rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT,
 				&cxlds->reg_map);
-	if (rc)
+	if (rc) {
+		if (rc == -EPROBE_DEFER)
+			return rc;
 		dev_warn(&pdev->dev, "No component registers (%d)\n", rc);
-	else if (!cxlds->reg_map.component_map.ras.valid)
+	} else if (!cxlds->reg_map.component_map.ras.valid) {
 		dev_dbg(&pdev->dev, "RAS registers not found\n");
+	}
 
 	rc = cxl_pci_type3_init_mailbox(cxlds);
 	if (rc)
diff --git a/drivers/dax/bus.h b/drivers/dax/bus.h
index 5909171a4428b..bfc31923f3ff4 100644
--- a/drivers/dax/bus.h
+++ b/drivers/dax/bus.h
@@ -16,6 +16,8 @@ struct dax_region;
 #define IORESOURCE_DAX_STATIC BIT(0)
 #define IORESOURCE_DAX_KMEM BIT(1)
 
+#define DAX_KMEM_UNPLUGGED	(-1) /* Do not create memory blocks */
+
 struct dax_region *alloc_dax_region(struct device *parent, int region_id,
 		struct range *range, int target_node, unsigned int align,
 		unsigned long flags);
diff --git a/drivers/dax/dax-private.h b/drivers/dax/dax-private.h
index 81e4af49e39c1..607a53a91f58b 100644
--- a/drivers/dax/dax-private.h
+++ b/drivers/dax/dax-private.h
@@ -69,7 +69,6 @@ struct dev_dax_range {
  * data while the device is activated in the driver.
  * @region: parent region
  * @dax_dev: core dax functionality
- * @virt_addr: kva from memremap; used by fsdev_dax
  * @cached_size: size of daxdev cached by fsdev_dax
  * @align: alignment of this instance
  * @target_node: effective numa node if dev_dax memory range is onlined
@@ -85,7 +84,6 @@ struct dev_dax_range {
 struct dev_dax {
 	struct dax_region *region;
 	struct dax_device *dax_dev;
-	void *virt_addr;
 	u64 cached_size;
 	unsigned int align;
 	int target_node;
diff --git a/drivers/dax/fsdev.c b/drivers/dax/fsdev.c
index 188b2526bee45..598604bf5ac5c 100644
--- a/drivers/dax/fsdev.c
+++ b/drivers/dax/fsdev.c
@@ -45,15 +45,13 @@ static void fsdev_write_dax(void *addr, struct page *page,
 }
 
 static long __fsdev_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
-			long nr_pages, enum dax_access_mode mode, void **kaddr,
-			unsigned long *pfn)
+		long nr_pages, enum dax_access_mode mode, void **kaddr,
+		unsigned long *pfn)
 {
 	struct dev_dax *dev_dax = dax_get_private(dax_dev);
 	size_t size = nr_pages << PAGE_SHIFT;
 	size_t offset = pgoff << PAGE_SHIFT;
-	void *virt_addr = dev_dax->virt_addr + offset;
 	phys_addr_t phys;
-	unsigned long local_pfn;
 
 	phys = dax_pgoff_to_phys(dev_dax, pgoff, size);
 	if (phys == -1) {
@@ -63,11 +61,10 @@ static long __fsdev_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
 	}
 
 	if (kaddr)
-		*kaddr = virt_addr;
+		*kaddr = __va(phys);
 
-	local_pfn = PHYS_PFN(phys);
 	if (pfn)
-		*pfn = local_pfn;
+		*pfn = PHYS_PFN(phys);
 
 	/*
 	 * Use cached_size which was computed at probe time. The size cannot
@@ -83,7 +80,8 @@ static int fsdev_dax_zero_page_range(struct dax_device *dax_dev,
 	long rc;
 
 	WARN_ONCE(nr_pages > 1, "%s: nr_pages > 1\n", __func__);
-	rc = __fsdev_dax_direct_access(dax_dev, pgoff, 1, DAX_ACCESS, &kaddr, NULL);
+	rc = __fsdev_dax_direct_access(dax_dev, pgoff, 1, DAX_ACCESS,
+				       &kaddr, NULL);
 	if (rc < 0)
 		return rc;
 	fsdev_write_dax(kaddr, ZERO_PAGE(0), 0, PAGE_SIZE);
@@ -91,15 +89,15 @@ static int fsdev_dax_zero_page_range(struct dax_device *dax_dev,
 }
 
 static long fsdev_dax_direct_access(struct dax_device *dax_dev,
-		  pgoff_t pgoff, long nr_pages, enum dax_access_mode mode,
-		  void **kaddr, unsigned long *pfn)
+		pgoff_t pgoff, long nr_pages, enum dax_access_mode mode,
+		void **kaddr, unsigned long *pfn)
 {
 	return __fsdev_dax_direct_access(dax_dev, pgoff, nr_pages, mode,
 					 kaddr, pfn);
 }
 
-static size_t fsdev_dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
-		void *addr, size_t bytes, struct iov_iter *i)
+static size_t fsdev_dax_recovery_write(struct dax_device *dax_dev,
+		pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i)
 {
 	return _copy_from_iter_flushcache(addr, bytes, i);
 }
@@ -127,6 +125,23 @@ static void fsdev_clear_ops(void *data)
 	dax_set_ops(dev_dax->dax_dev, NULL);
 }
 
+static void fsdev_clear_pgmap_ops(void *data)
+{
+	struct dev_pagemap *pgmap = data;
+
+	/*
+	 * fsdev installs pgmap->ops and ->owner at probe. For a static device
+	 * the pgmap is shared and long-lived (owned by the dax bus), so
+	 * leaving fsdev's ops behind on unbind would let a later
+	 * memory_failure -- after rebind to another driver, or after this
+	 * module is unloaded -- dispatch through a stale or freed
+	 * ->memory_failure handler. Clear them so the pgmap carries no fsdev
+	 * state once we are unbound.
+	 */
+	pgmap->ops = NULL;
+	pgmap->owner = NULL;
+}
+
 /*
  * Page map operations for FS-DAX mode
  * Similar to fsdax_pagemap_ops in drivers/nvdimm/pmem.c
@@ -135,11 +150,26 @@ static void fsdev_clear_ops(void *data)
  * The core mm code in free_zone_device_folio() handles the wake_up_var()
  * directly for this memory type.
  */
+static u64 fsdev_pfn_to_offset(struct dev_dax *dev_dax, unsigned long pfn)
+{
+	phys_addr_t phys = PFN_PHYS(pfn);
+	u64 offset = 0;
+
+	for (int i = 0; i < dev_dax->nr_range; i++) {
+		struct range *range = &dev_dax->ranges[i].range;
+
+		if (phys >= range->start && phys <= range->end)
+			return offset + (phys - range->start);
+		offset += range_len(range);
+	}
+	return -1ULL;
+}
+
 static int fsdev_pagemap_memory_failure(struct dev_pagemap *pgmap,
 		unsigned long pfn, unsigned long nr_pages, int mf_flags)
 {
 	struct dev_dax *dev_dax = pgmap->owner;
-	u64 offset = PFN_PHYS(pfn) - dev_dax->ranges[0].range.start;
+	u64 offset = fsdev_pfn_to_offset(dev_dax, pfn);
 	u64 len = nr_pages << PAGE_SHIFT;
 
 	return dax_holder_notify_failure(dev_dax->dax_dev, offset,
@@ -204,46 +234,62 @@ static const struct file_operations fsdev_fops = {
 	.release = fsdev_release,
 };
 
-static int fsdev_dax_probe(struct dev_dax *dev_dax)
+/*
+ * Acquire the dev_pagemap for probe: the static (pre-populated) one if
+ * present, or a devm-allocated one for the dynamic case. Note that
+ * dev_dax->pgmap is not set here; fsdev_dax_probe() sets it only once
+ * probe succeeds, so a failed probe never leaves a dangling pointer
+ * to a devres-freed pgmap.
+ */
+static struct dev_pagemap *fsdev_acquire_pgmap(struct dev_dax *dev_dax)
 {
-	struct dax_device *dax_dev = dev_dax->dax_dev;
 	struct device *dev = &dev_dax->dev;
 	struct dev_pagemap *pgmap;
-	struct inode *inode;
-	u64 data_offset = 0;
-	struct cdev *cdev;
-	void *addr;
-	int rc, i;
+	size_t pgmap_size;
 
 	if (static_dev_dax(dev_dax)) {
 		if (dev_dax->nr_range > 1) {
-			dev_warn(dev, "static pgmap / multi-range device conflict\n");
-			return -EINVAL;
+			dev_warn(dev,
+				 "static pgmap / multi-range device conflict\n");
+			return ERR_PTR(-EINVAL);
 		}
 
 		pgmap = dev_dax->pgmap;
-	} else {
-		size_t pgmap_size;
+		pgmap->vmemmap_shift = 0;
+		return pgmap;
+	}
 
-		if (dev_dax->pgmap) {
-			dev_warn(dev, "dynamic-dax with pre-populated page map\n");
-			return -EINVAL;
-		}
+	if (dev_dax->pgmap) {
+		dev_warn(dev, "dynamic-dax with pre-populated page map\n");
+		return ERR_PTR(-EINVAL);
+	}
 
-		pgmap_size = struct_size(pgmap, ranges, dev_dax->nr_range - 1);
-		pgmap = devm_kzalloc(dev, pgmap_size, GFP_KERNEL);
-		if (!pgmap)
-			return -ENOMEM;
+	pgmap_size = struct_size(pgmap, ranges, dev_dax->nr_range - 1);
+	pgmap = devm_kzalloc(dev, pgmap_size, GFP_KERNEL);
+	if (!pgmap)
+		return ERR_PTR(-ENOMEM);
 
-		pgmap->nr_range = dev_dax->nr_range;
-		dev_dax->pgmap = pgmap;
+	pgmap->nr_range = dev_dax->nr_range;
+	for (int i = 0; i < dev_dax->nr_range; i++)
+		pgmap->ranges[i] = dev_dax->ranges[i].range;
 
-		for (i = 0; i < dev_dax->nr_range; i++) {
-			struct range *range = &dev_dax->ranges[i].range;
+	return pgmap;
+}
 
-			pgmap->ranges[i] = *range;
-		}
-	}
+static int fsdev_dax_probe(struct dev_dax *dev_dax)
+{
+	struct dax_device *dax_dev = dev_dax->dax_dev;
+	struct device *dev = &dev_dax->dev;
+	struct dev_pagemap *pgmap;
+	struct inode *inode;
+	u64 data_offset = 0;
+	struct cdev *cdev;
+	void *addr;
+	int rc, i;
+
+	pgmap = fsdev_acquire_pgmap(dev_dax);
+	if (IS_ERR(pgmap))
+		return PTR_ERR(pgmap);
 
 	for (i = 0; i < dev_dax->nr_range; i++) {
 		struct range *range = &dev_dax->ranges[i].range;
@@ -275,6 +321,11 @@ static int fsdev_dax_probe(struct dev_dax *dev_dax)
 	if (IS_ERR(addr))
 		return PTR_ERR(addr);
 
+	/* Drop fsdev's pgmap->ops/owner on unbind so no stale ops survive. */
+	rc = devm_add_action_or_reset(dev, fsdev_clear_pgmap_ops, pgmap);
+	if (rc)
+		return rc;
+
 	/*
 	 * Clear any stale compound folio state left over from a previous
 	 * driver (e.g., device_dax with vmemmap_shift). Also register this
@@ -290,15 +341,18 @@ static int fsdev_dax_probe(struct dev_dax *dev_dax)
 	/* Detect whether the data is at a non-zero offset into the memory */
 	if (pgmap->range.start != dev_dax->ranges[0].range.start) {
 		u64 phys = dev_dax->ranges[0].range.start;
-		u64 pgmap_phys = dev_dax->pgmap[0].range.start;
+		u64 pgmap_phys = pgmap[0].range.start;
 
-		if (!WARN_ON(pgmap_phys > phys))
-			data_offset = phys - pgmap_phys;
+		if (pgmap_phys > phys) {
+			dev_err(dev, "pgmap start %#llx exceeds data start %#llx\n",
+				pgmap_phys, phys);
+			return -EINVAL;
+		}
+		data_offset = phys - pgmap_phys;
 
 		pr_debug("%s: offset detected phys=%llx pgmap_phys=%llx offset=%llx\n",
 		       __func__, phys, pgmap_phys, data_offset);
 	}
-	dev_dax->virt_addr = addr + data_offset;
 
 	inode = dax_inode(dax_dev);
 	cdev = inode->i_cdev;
@@ -323,7 +377,13 @@ static int fsdev_dax_probe(struct dev_dax *dev_dax)
 		return rc;
 
 	run_dax(dax_dev);
-	return devm_add_action_or_reset(dev, fsdev_kill, dev_dax);
+	rc = devm_add_action_or_reset(dev, fsdev_kill, dev_dax);
+	if (rc)
+		return rc;
+
+	/* Probe can no longer fail; expose the pgmap via dev_dax */
+	dev_dax->pgmap = pgmap;
+	return 0;
 }
 
 static struct dax_device_driver fsdev_dax_driver = {
diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c
index a18e2b968e4da..a48d699cf344c 100644
--- a/drivers/dax/kmem.c
+++ b/drivers/dax/kmem.c
@@ -33,9 +33,7 @@ static int dax_kmem_range(struct dev_dax *dev_dax, int i, struct range *r)
 	struct dev_dax_range *dax_range = &dev_dax->ranges[i];
 	struct range *range = &dax_range->range;
 
-	/* memory-block align the hotplug range */
-	r->start = ALIGN(range->start, memory_block_size_bytes());
-	r->end = ALIGN_DOWN(range->end + 1, memory_block_size_bytes()) - 1;
+	*r = memory_block_aligned_range(range);
 	if (r->start >= r->end) {
 		r->start = range->start;
 		r->end = range->end;
@@ -47,6 +45,8 @@ static int dax_kmem_range(struct dev_dax *dev_dax, int i, struct range *r)
 struct dax_kmem_data {
 	const char *res_name;
 	int mgid;
+	int state;
+	struct mutex lock; /* protects hotplug state transitions */
 	struct resource *res[];
 };
 
@@ -65,16 +65,331 @@ static void kmem_put_memory_types(void)
 	mt_put_memory_types(&kmem_memory_types);
 }
 
+/* True for the online states a kmem dax device can hold. */
+static bool dax_kmem_state_is_online(int state)
+{
+	return state == MMOP_ONLINE ||
+	       state == MMOP_ONLINE_KERNEL ||
+	       state == MMOP_ONLINE_MOVABLE;
+}
+
+/**
+ * dax_kmem_do_hotplug - hotplug memory for dax kmem device
+ * @dev_dax: the dev_dax instance
+ * @data: the dax_kmem_data structure with resource tracking
+ * @online_type: the online policy to use for the memory blocks
+ *
+ * Hotplugs all ranges in the dev_dax region as system memory with the
+ * provided online policy (offline, online, online_movable, online_kernel).
+ *
+ * Returns the number of successfully mapped ranges, or negative error.
+ */
+static int dax_kmem_do_hotplug(struct dev_dax *dev_dax,
+			       struct dax_kmem_data *data,
+			       int online_type)
+{
+	struct device *dev = &dev_dax->dev;
+	int i, rc, added = 0;
+	mhp_t mhp_flags;
+
+	if (dax_kmem_state_is_online(data->state))
+		return -EINVAL;
+
+	if (online_type < MMOP_OFFLINE || online_type > MMOP_ONLINE_MOVABLE)
+		return -EINVAL;
+
+	for (i = 0; i < dev_dax->nr_range; i++) {
+		struct range range;
+
+		rc = dax_kmem_range(dev_dax, i, &range);
+		if (rc)
+			continue;
+
+		/*
+		 * init_resources() is best-effort: if a reservation conflict
+		 * occurs it keeps the range but leaves res[i]=NULL. For hotplug
+		 * on probe systems, this means kmem will partially online.
+		 *
+		 * We have to keep this behavior not to break those systems.
+		 * For those systems - atomicity only applies to valid ranges.
+		 */
+		if (!data->res[i])
+			continue;
+
+		mhp_flags = MHP_NID_IS_MGID;
+		if (dev_dax->memmap_on_memory)
+			mhp_flags |= MHP_MEMMAP_ON_MEMORY;
+
+		/*
+		 * Ensure that future kexec'd kernels will not treat
+		 * this as RAM automatically.
+		 */
+		rc = __add_memory_driver_managed(data->mgid, range.start,
+				range_len(&range), kmem_name, mhp_flags,
+				online_type);
+
+		if (rc) {
+			dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed\n",
+				 i, range.start, range.end);
+			/*
+			 * Release the reservation for the range that failed to
+			 * add so a later hotremove does not try to remove memory
+			 * that was never added.
+			 */
+			if (data->res[i]) {
+				remove_resource(data->res[i]);
+				kfree(data->res[i]);
+				data->res[i] = NULL;
+			}
+			if (added)
+				continue;
+			return rc;
+		}
+		added++;
+	}
+
+	return added;
+}
+
+/**
+ * dax_kmem_init_resources - create memory regions for dax kmem
+ * @dev_dax: the dev_dax instance
+ * @data: the dax_kmem_data structure with resource tracking
+ *
+ * Initializes all the resources for the DAX
+ *
+ * Returns the number of successfully mapped ranges, or negative error.
+ */
+static int dax_kmem_init_resources(struct dev_dax *dev_dax,
+				   struct dax_kmem_data *data)
+{
+	struct device *dev = &dev_dax->dev;
+	int i, rc, mapped = 0;
+
+	for (i = 0; i < dev_dax->nr_range; i++) {
+		struct resource *res;
+		struct range range;
+
+		rc = dax_kmem_range(dev_dax, i, &range);
+		if (rc)
+			continue;
+
+		/* Skip ranges already added */
+		if (data->res[i])
+			continue;
+
+		/* Region is permanently reserved if hotremove fails. */
+		res = request_mem_region(range.start, range_len(&range),
+					 data->res_name);
+		if (!res) {
+			dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve region\n",
+				 i, range.start, range.end);
+			/*
+			 * Once some memory has been onlined we can't
+			 * assume that it can be un-onlined safely.
+			 */
+			if (mapped)
+				continue;
+			return -EBUSY;
+		}
+		data->res[i] = res;
+		/*
+		 * Set flags appropriate for System RAM.  Leave ..._BUSY clear
+		 * so that add_memory() can add a child resource.  Do not
+		 * inherit flags from the parent since it may set new flags
+		 * unknown to us that will break add_memory() later.
+		 */
+		res->flags = IORESOURCE_SYSTEM_RAM;
+		mapped++;
+	}
+	return mapped;
+}
+
+#ifdef CONFIG_MEMORY_HOTREMOVE
+/**
+ * dax_kmem_do_hotremove - hot-remove memory for dax kmem device
+ * @dev_dax: the dev_dax instance
+ * @data: the dax_kmem_data structure with resource tracking
+ *
+ * Offlines and removes every currently-added range in the dev_dax region
+ * atomically: either all ranges are offlined and removed, or none are and
+ * the device is returned to its prior state.
+ *
+ * Returns 0 on success, or a negative errno on failure.
+ */
+static int dax_kmem_do_hotremove(struct dev_dax *dev_dax,
+				 struct dax_kmem_data *data)
+{
+	struct device *dev = &dev_dax->dev;
+	struct range *ranges;
+	int i, nr_ranges = 0, rc;
+
+	ranges = kmalloc_objs(*ranges, dev_dax->nr_range);
+	if (!ranges)
+		return -ENOMEM;
+
+	/* Collect the ranges that were actually added during probe. */
+	for (i = 0; i < dev_dax->nr_range; i++) {
+		struct range range;
+
+		if (!data->res[i])
+			continue;
+		if (dax_kmem_range(dev_dax, i, &range))
+			continue;
+		ranges[nr_ranges++] = range;
+	}
+
+	/* Nothing added means nothing to remove. */
+	if (!nr_ranges) {
+		kfree(ranges);
+		return 0;
+	}
+
+	rc = offline_and_remove_memory_ranges(ranges, nr_ranges);
+	kfree(ranges);
+	if (rc) {
+		/* Recoverable: the ranges rolled back, nothing is leaked yet. */
+		dev_err(dev, "hotremove failed, device left online: %d\n", rc);
+		return rc;
+	}
+
+	/* All ranges removed; release the reserved resources. */
+	for (i = 0; i < dev_dax->nr_range; i++) {
+		if (!data->res[i])
+			continue;
+		remove_resource(data->res[i]);
+		kfree(data->res[i]);
+		data->res[i] = NULL;
+	}
+
+	return 0;
+}
+#else
+static int dax_kmem_do_hotremove(struct dev_dax *dev_dax,
+				 struct dax_kmem_data *data)
+{
+	return -EBUSY;
+}
+#endif /* CONFIG_MEMORY_HOTREMOVE */
+
+/**
+ * dax_kmem_cleanup_resources - remove the dax memory resources
+ * @dev_dax: the dev_dax instance
+ * @data: the dax_kmem_data structure with resource tracking
+ *
+ * Removes all resources in the dev_dax region.
+ */
+static void dax_kmem_cleanup_resources(struct dev_dax *dev_dax,
+				       struct dax_kmem_data *data)
+{
+	int i;
+
+	/*
+	 * If the device unbind occurs before memory is hotremoved, we can never
+	 * remove the memory (requires reboot).  Attempting an offline operation
+	 * here may cause deadlock and a failure to finish the unbind.
+	 *
+	 * Note: This leaks the resources.
+	 */
+	if (WARN(((data->state != DAX_KMEM_UNPLUGGED) &&
+		  (data->state != MMOP_OFFLINE)),
+		 "Hotplug memory regions stuck online until reboot"))
+		return;
+
+	for (i = 0; i < dev_dax->nr_range; i++) {
+		if (!data->res[i])
+			continue;
+		remove_resource(data->res[i]);
+		kfree(data->res[i]);
+		data->res[i] = NULL;
+	}
+}
+
+static int dax_kmem_parse_state(const char *buf)
+{
+	int online_type;
+
+	/* "unplugged" is kmem-specific - the rest map to MMOP_ */
+	if (sysfs_streq(buf, "unplugged"))
+		return DAX_KMEM_UNPLUGGED;
+
+	online_type = mhp_online_type_from_str(buf);
+	/* Disallow "offline": it's not useful and creates race conditions */
+	if (online_type == MMOP_OFFLINE)
+		return -EINVAL;
+	return online_type;
+}
+
+static ssize_t state_show(struct device *dev,
+			    struct device_attribute *attr, char *buf)
+{
+	struct dax_kmem_data *data = dev_get_drvdata(dev);
+	const char *state_str;
+
+	if (data->state == DAX_KMEM_UNPLUGGED)
+		state_str = "unplugged";
+	else
+		state_str = mhp_online_type_to_str(data->state);
+
+	return sysfs_emit(buf, "%s\n", state_str ?: "unknown");
+}
+
+static ssize_t state_store(struct device *dev, struct device_attribute *attr,
+			     const char *buf, size_t len)
+{
+	struct dev_dax *dev_dax = to_dev_dax(dev);
+	struct dax_kmem_data *data = dev_get_drvdata(dev);
+	int online_type;
+	int rc;
+
+	online_type = dax_kmem_parse_state(buf);
+	if (online_type < DAX_KMEM_UNPLUGGED)
+		return online_type;
+
+	guard(mutex)(&data->lock);
+
+	/* Already in requested state */
+	if (data->state == online_type)
+		return len;
+
+	if (online_type == DAX_KMEM_UNPLUGGED) {
+		rc = dax_kmem_do_hotremove(dev_dax, data);
+		if (rc)
+			return rc;
+		data->state = DAX_KMEM_UNPLUGGED;
+		return len;
+	}
+
+	/* Onlining is only allowed from the unplugged state. */
+	if (data->state != DAX_KMEM_UNPLUGGED)
+		return -EBUSY;
+
+	/* Re-acquire resources if previously unplugged, otherwise no-op */
+	rc = dax_kmem_init_resources(dev_dax, data);
+	if (rc < 0)
+		return rc;
+
+	rc = dax_kmem_do_hotplug(dev_dax, data, online_type);
+	if (rc < 0) {
+		/* Total failure, drop the reservations we took. */
+		dax_kmem_cleanup_resources(dev_dax, data);
+		return rc;
+	}
+
+	data->state = online_type;
+	return len;
+}
+
 static int dev_dax_kmem_probe(struct dev_dax *dev_dax)
 {
 	struct device *dev = &dev_dax->dev;
 	unsigned long total_len = 0, orig_len = 0;
 	struct dax_kmem_data *data;
 	struct memory_dev_type *mtype;
-	int i, rc, mapped = 0;
-	mhp_t mhp_flags;
+	int i, rc;
 	int numa_node;
 	int adist = MEMTIER_DEFAULT_DAX_ADISTANCE;
+	int online_type = mhp_get_default_online_type();
 
 	/*
 	 * Ensure good NUMA information for the persistent memory.
@@ -133,68 +448,26 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax)
 	if (rc < 0)
 		goto err_reg_mgid;
 	data->mgid = rc;
+	data->state = DAX_KMEM_UNPLUGGED;
+	mutex_init(&data->lock);
 
-	for (i = 0; i < dev_dax->nr_range; i++) {
-		struct resource *res;
-		struct range range;
-
-		rc = dax_kmem_range(dev_dax, i, &range);
-		if (rc)
-			continue;
-
-		/* Region is permanently reserved if hotremove fails. */
-		res = request_mem_region(range.start, range_len(&range), data->res_name);
-		if (!res) {
-			dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve region\n",
-					i, range.start, range.end);
-			/*
-			 * Once some memory has been onlined we can't
-			 * assume that it can be un-onlined safely.
-			 */
-			if (mapped)
-				continue;
-			rc = -EBUSY;
-			goto err_request_mem;
-		}
-		data->res[i] = res;
-
-		/*
-		 * Set flags appropriate for System RAM.  Leave ..._BUSY clear
-		 * so that add_memory() can add a child resource.  Do not
-		 * inherit flags from the parent since it may set new flags
-		 * unknown to us that will break add_memory() below.
-		 */
-		res->flags = IORESOURCE_SYSTEM_RAM;
-
-		mhp_flags = MHP_NID_IS_MGID;
-		if (dev_dax->memmap_on_memory)
-			mhp_flags |= MHP_MEMMAP_ON_MEMORY;
-
-		/*
-		 * Ensure that future kexec'd kernels will not treat
-		 * this as RAM automatically.
-		 */
-		rc = add_memory_driver_managed(data->mgid, range.start,
-				range_len(&range), kmem_name, mhp_flags);
+	dev_set_drvdata(dev, data);
 
-		if (rc) {
-			dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed\n",
-					i, range.start, range.end);
-			remove_resource(res);
-			kfree(res);
-			data->res[i] = NULL;
-			if (mapped)
-				continue;
-			goto err_request_mem;
-		}
-		mapped++;
-	}
+	rc = dax_kmem_init_resources(dev_dax, data);
+	if (rc < 0)
+		goto err_resources;
 
-	dev_set_drvdata(dev, data);
+	rc = dax_kmem_do_hotplug(dev_dax, data, online_type);
+	if (rc < 0)
+		goto err_hotplug;
+	data->state = online_type;
 
 	return 0;
 
-err_request_mem:
+err_hotplug:
+	dax_kmem_cleanup_resources(dev_dax, data);
+err_resources:
+	dev_set_drvdata(dev, NULL);
 	memory_group_unregister(data->mgid);
 err_reg_mgid:
 	kfree(data->res_name);
@@ -206,61 +479,71 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax)
 }
 
 #ifdef CONFIG_MEMORY_HOTREMOVE
-static void dev_dax_kmem_remove(struct dev_dax *dev_dax)
+/*
+ * Remove the device's added ranges with remove_memory().
+ * Unlike the sysfs unplug path it never offlines and fails if the blocks are
+ * online (-EBUSY), so it is safe from unbind. Failures leak until reboot.
+ *
+ * Returns 0 only if every added range was removed.
+ */
+static int dax_kmem_remove_ranges(struct dev_dax *dev_dax,
+				  struct dax_kmem_data *data)
 {
-	int i, success = 0;
-	int node = dev_dax->target_node;
 	struct device *dev = &dev_dax->dev;
-	struct dax_kmem_data *data = dev_get_drvdata(dev);
+	int i, rc = 0;
 
-	/*
-	 * We have one shot for removing memory, if some memory blocks were not
-	 * offline prior to calling this function remove_memory() will fail, and
-	 * there is no way to hotremove this memory until reboot because device
-	 * unbind will succeed even if we return failure.
-	 */
 	for (i = 0; i < dev_dax->nr_range; i++) {
 		struct range range;
-		int rc;
 
-		rc = dax_kmem_range(dev_dax, i, &range);
-		if (rc)
+		if (!data->res[i] || dax_kmem_range(dev_dax, i, &range))
 			continue;
-
-		/* range was never added during probe */
-		if (!data->res[i]) {
-			success++;
+		if (remove_memory(range.start, range_len(&range))) {
+			dev_warn(dev, "mapping%d: %#llx-%#llx stuck online until reboot\n",
+				 i, range.start, range.end);
+			rc = -EBUSY;
 			continue;
 		}
+		remove_resource(data->res[i]);
+		kfree(data->res[i]);
+		data->res[i] = NULL;
+	}
+	return rc;
+}
 
-		rc = remove_memory(range.start, range_len(&range));
-		if (rc == 0) {
-			remove_resource(data->res[i]);
-			kfree(data->res[i]);
-			data->res[i] = NULL;
-			success++;
-			continue;
-		}
+static void dev_dax_kmem_remove(struct dev_dax *dev_dax)
+{
+	int node = dev_dax->target_node;
+	struct device *dev = &dev_dax->dev;
+	struct dax_kmem_data *data = dev_get_drvdata(dev);
+
+	/*
+	 * Remove every range that is still added.  dax_kmem_remove_ranges()
+	 * uses remove_memory(), which never offlines: an online block fails
+	 * with -EBUSY rather than deadlocking an uninterruptible unbind.
+	 *
+	 * data->state only tracks daxX.Y/state writes, so it can be stale if
+	 * blocks were toggled via memoryX/state. Do not trust it here and
+	 * attempt simply remove_memory() - which reports the true state of
+	 * each range anyway. Anything left online is leaked until reboot.
+	 */
+	if (dax_kmem_remove_ranges(dev_dax, data)) {
+		dev_err(dev, "Hotplug regions stuck online until reboot\n");
 		any_hotremove_failed = true;
-		dev_err(dev,
-			"mapping%d: %#llx-%#llx cannot be hotremoved until the next reboot\n",
-				i, range.start, range.end);
+		return;
 	}
 
-	if (success >= dev_dax->nr_range) {
-		memory_group_unregister(data->mgid);
-		kfree(data->res_name);
-		kfree(data);
-		dev_set_drvdata(dev, NULL);
-		/*
-		 * Clear the memtype association on successful unplug.
-		 * If not, we have memory blocks left which can be
-		 * offlined/onlined later. We need to keep memory_dev_type
-		 * for that. This implies this reference will be around
-		 * till next reboot.
-		 */
-		clear_node_memory_type(node, NULL);
-	}
+	memory_group_unregister(data->mgid);
+	kfree(data->res_name);
+	kfree(data);
+	dev_set_drvdata(dev, NULL);
+	/*
+	 * Clear the memtype association on successful unplug.
+	 * If not, we have memory blocks left which can be
+	 * offlined/onlined later. We need to keep memory_dev_type
+	 * for that. This implies this reference will be around
+	 * till next reboot.
+	 */
+	clear_node_memory_type(node, NULL);
 }
 #else
 static void dev_dax_kmem_remove(struct dev_dax *dev_dax)
@@ -276,10 +559,21 @@ static void dev_dax_kmem_remove(struct dev_dax *dev_dax)
 }
 #endif /* CONFIG_MEMORY_HOTREMOVE */
 
+static DEVICE_ATTR_RW(state);
+
+static struct attribute *dev_dax_kmem_attrs[] = {
+	&dev_attr_state.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(dev_dax_kmem);
+
 static struct dax_device_driver device_dax_kmem_driver = {
 	.probe = dev_dax_kmem_probe,
 	.remove = dev_dax_kmem_remove,
 	.type = DAXDRV_KMEM_TYPE,
+	.drv = {
+		.dev_groups = dev_dax_kmem_groups,
+	},
 };
 
 static int __init dax_kmem_init(void)
diff --git a/drivers/dax/super.c b/drivers/dax/super.c
index 25cf99dd9360b..45f84b0eb909a 100644
--- a/drivers/dax/super.c
+++ b/drivers/dax/super.c
@@ -116,11 +116,47 @@ EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev);
 
 #if IS_ENABLED(CONFIG_FS_DAX)
 
+/**
+ * fs_put_dax() - release holder ownership of a dax_device
+ * @dax_dev: dax device to release (may be NULL)
+ * @holder: the holder pointer previously passed to fs_dax_get() or
+ *          fs_dax_get_by_bdev(); must match exactly, as it is used
+ *          in a cmpxchg to atomically release ownership
+ *
+ * Must only be called by the current holder. Clears holder_ops before
+ * holder_data to avoid a race where a concurrent fs_dax_get() could have
+ * its newly installed holder_ops overwritten.
+ */
 void fs_put_dax(struct dax_device *dax_dev, void *holder)
 {
-	if (dax_dev && holder &&
-	    cmpxchg(&dax_dev->holder_data, holder, NULL) == holder)
-		dax_dev->holder_ops = NULL;
+	if (dax_dev && holder) {
+		void *prev;
+
+		/*
+		 * Clear holder_ops before releasing holder_data. A concurrent
+		 * dax_holder_notify_failure() that sees NULL ops returns
+		 * -EOPNOTSUPP cleanly. A concurrent fs_dax_get() that acquires
+		 * holder_data after the cmpxchg below is guaranteed to observe
+		 * holder_ops=NULL first (cmpxchg provides release ordering), so
+		 * its subsequent store of new ops will not be overwritten.
+		 */
+		WRITE_ONCE(dax_dev->holder_ops, NULL);
+		prev = cmpxchg(&dax_dev->holder_data, holder, NULL);
+
+		/*
+		 * prev == holder: normal release.
+		 * prev == NULL:   already released by kill_dax() when the
+		 *                 device was removed under a live holder;
+		 *                 not a bug.
+		 * prev != holder (non-NULL): fs_put_dax() called by something
+		 *                 that is not the current holder; an API
+		 *                 contract violation. A lock would be needed
+		 *                 to guard against this, but we WARN_ON()
+		 *                 instead since violating the contract is
+		 *                 a bug.
+		 */
+		WARN_ON(prev && prev != holder);
+	}
 	put_dax(dax_dev);
 }
 EXPORT_SYMBOL_GPL(fs_put_dax);
@@ -303,6 +339,7 @@ EXPORT_SYMBOL_GPL(dax_recovery_write);
 int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off,
 			      u64 len, int mf_flags)
 {
+	const struct dax_holder_operations *ops;
 	int rc, id;
 
 	id = dax_read_lock();
@@ -311,12 +348,19 @@ int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off,
 		goto out;
 	}
 
-	if (!dax_dev->holder_ops) {
+	/*
+	 * Read holder_ops once: a concurrent fs_put_dax() can clear it without
+	 * synchronizing against readers. Without the single fetch the compiler
+	 * could reload between the NULL check and the call and dereference a
+	 * NULL ops.
+	 */
+	ops = READ_ONCE(dax_dev->holder_ops);
+	if (!ops) {
 		rc = -EOPNOTSUPP;
 		goto out;
 	}
 
-	rc = dax_dev->holder_ops->notify_failure(dax_dev, off, len, mf_flags);
+	rc = ops->notify_failure(dax_dev, off, len, mf_flags);
 out:
 	dax_read_unlock(id);
 	return rc;
diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c
index 82dd9a43dc621..f08fc6966eaee 100644
--- a/drivers/devfreq/devfreq.c
+++ b/drivers/devfreq/devfreq.c
@@ -1261,7 +1261,6 @@ void devfreq_resume(void)
 int devfreq_add_governor(struct devfreq_governor *governor)
 {
 	struct devfreq_governor *g;
-	struct devfreq *devfreq;
 	int err = 0;
 
 	if (!governor) {
@@ -1280,38 +1279,6 @@ int devfreq_add_governor(struct devfreq_governor *governor)
 
 	list_add(&governor->node, &devfreq_governor_list);
 
-	list_for_each_entry(devfreq, &devfreq_list, node) {
-		int ret = 0;
-		struct device *dev = devfreq->dev.parent;
-
-		if (!strncmp(devfreq->governor->name, governor->name,
-			     DEVFREQ_NAME_LEN)) {
-			/* The following should never occur */
-			if (devfreq->governor) {
-				dev_warn(dev,
-					 "%s: Governor %s already present\n",
-					 __func__, devfreq->governor->name);
-				ret = devfreq->governor->event_handler(devfreq,
-							DEVFREQ_GOV_STOP, NULL);
-				if (ret) {
-					dev_warn(dev,
-						 "%s: Governor %s stop = %d\n",
-						 __func__,
-						 devfreq->governor->name, ret);
-				}
-				/* Fall through */
-			}
-			devfreq->governor = governor;
-			ret = devfreq->governor->event_handler(devfreq,
-						DEVFREQ_GOV_START, NULL);
-			if (ret) {
-				dev_warn(dev, "%s: Governor %s start=%d\n",
-					 __func__, devfreq->governor->name,
-					 ret);
-			}
-		}
-	}
-
 err_out:
 	mutex_unlock(&devfreq_list_lock);
 
@@ -1410,7 +1377,7 @@ static ssize_t governor_show(struct device *dev,
 	struct devfreq *df = to_devfreq(dev);
 
 	if (!df->governor)
-		return -EINVAL;
+		return -ENOENT;
 
 	return sprintf(buf, "%s\n", df->governor->name);
 }
@@ -1423,9 +1390,6 @@ static ssize_t governor_store(struct device *dev, struct device_attribute *attr,
 	char str_governor[DEVFREQ_NAME_LEN + 1];
 	const struct devfreq_governor *governor, *prev_governor;
 
-	if (!df->governor)
-		return -EINVAL;
-
 	ret = sscanf(buf, "%" __stringify(DEVFREQ_NAME_LEN) "s", str_governor);
 	if (ret != 1)
 		return -EINVAL;
@@ -1436,6 +1400,9 @@ static ssize_t governor_store(struct device *dev, struct device_attribute *attr,
 		ret = PTR_ERR(governor);
 		goto out;
 	}
+	if (!df->governor)
+		goto start_new_governor;
+
 	if (df->governor == governor) {
 		ret = 0;
 		goto out;
@@ -1456,6 +1423,7 @@ static ssize_t governor_store(struct device *dev, struct device_attribute *attr,
 		goto out;
 	}
 
+start_new_governor:
 	/*
 	 * Start the new governor and create the specific sysfs files
 	 * which depend on the new governor.
@@ -1469,6 +1437,9 @@ static ssize_t governor_store(struct device *dev, struct device_attribute *attr,
 
 		/* Restore previous governor */
 		df->governor = prev_governor;
+		if (!df->governor)
+			goto out;
+
 		ret = df->governor->event_handler(df, DEVFREQ_GOV_START, NULL);
 		if (ret) {
 			dev_err(dev,
@@ -1501,16 +1472,13 @@ static ssize_t available_governors_show(struct device *d,
 	struct devfreq *df = to_devfreq(d);
 	ssize_t count = 0;
 
-	if (!df->governor)
-		return -EINVAL;
-
 	mutex_lock(&devfreq_list_lock);
 
 	/*
 	 * The devfreq with immutable governor (e.g., passive) shows
 	 * only own governor.
 	 */
-	if (IS_SUPPORTED_FLAG(df->governor->flags, IMMUTABLE)) {
+	if (df->governor && IS_SUPPORTED_FLAG(df->governor->flags, IMMUTABLE)) {
 		count = scnprintf(&buf[count], DEVFREQ_NAME_LEN,
 				  "%s ", df->governor->name);
 	/*
diff --git a/drivers/devfreq/event/rockchip-dfi.c b/drivers/devfreq/event/rockchip-dfi.c
index 5e6e7e900bda0..64014f95fb66d 100644
--- a/drivers/devfreq/event/rockchip-dfi.c
+++ b/drivers/devfreq/event/rockchip-dfi.c
@@ -185,8 +185,10 @@ static int rockchip_dfi_enable(struct rockchip_dfi *dfi)
 	}
 
 	ret = rockchip_dfi_ddrtype_to_ctrl(dfi, &ctrl);
-	if (ret)
+	if (ret) {
+		clk_disable_unprepare(dfi->clk);
 		goto out;
+	}
 
 	for (i = 0; i < dfi->max_channels; i++) {
 
@@ -354,7 +356,7 @@ static ssize_t ddr_perf_cpumask_show(struct device *dev,
 	struct pmu *pmu = dev_get_drvdata(dev);
 	struct rockchip_dfi *dfi = container_of(pmu, struct rockchip_dfi, pmu);
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(dfi->cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(dfi->cpu)));
 }
 
 static struct device_attribute ddr_perf_cpumask_attr =
diff --git a/drivers/devfreq/hisi_uncore_freq.c b/drivers/devfreq/hisi_uncore_freq.c
index bef718d6ae35d..e1f64b7230823 100644
--- a/drivers/devfreq/hisi_uncore_freq.c
+++ b/drivers/devfreq/hisi_uncore_freq.c
@@ -540,7 +540,7 @@ static ssize_t related_cpus_show(struct device *dev,
 {
 	struct hisi_uncore_freq *uncore = dev_get_drvdata(dev->parent);
 
-	return cpumap_print_to_pagebuf(true, buf, &uncore->related_cpus);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&uncore->related_cpus));
 }
 
 static DEVICE_ATTR_RO(related_cpus);
diff --git a/drivers/dma-buf/dma-fence-unwrap.c b/drivers/dma-buf/dma-fence-unwrap.c
index 364cbf79ad734..120fe0c2a75af 100644
--- a/drivers/dma-buf/dma-fence-unwrap.c
+++ b/drivers/dma-buf/dma-fence-unwrap.c
@@ -93,9 +93,9 @@ static int fence_cmp(const void *_a, const void *_b)
  *
  * Return: Number of unique fences remaining in the array.
  */
-int dma_fence_dedup_array(struct dma_fence **fences, int num_fences)
+size_t dma_fence_dedup_array(struct dma_fence **fences, size_t num_fences)
 {
-	int i, j;
+	size_t i, j;
 
 	if (!num_fences)
 		return 0;
@@ -118,14 +118,14 @@ int dma_fence_dedup_array(struct dma_fence **fences, int num_fences)
 EXPORT_SYMBOL_GPL(dma_fence_dedup_array);
 
 /* Implementation for the dma_fence_merge() marco, don't use directly */
-struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences,
+struct dma_fence *__dma_fence_unwrap_merge(size_t num_fences,
 					   struct dma_fence **fences,
 					   struct dma_fence_unwrap *iter)
 {
 	struct dma_fence *tmp, *unsignaled = NULL, **array;
 	struct dma_fence_array *result;
 	ktime_t timestamp;
-	int i, count;
+	size_t i, count;
 
 	count = 0;
 	timestamp = ns_to_ktime(0);
diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c
index 87797bea91cbd..05090fb0fd5ae 100644
--- a/drivers/dma-buf/dma-fence.c
+++ b/drivers/dma-buf/dma-fence.c
@@ -534,7 +534,7 @@ dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
 
 	__dma_fence_might_wait();
 
-	dma_fence_enable_sw_signaling(fence);
+	dma_fence_enable_signaling(fence);
 
 	rcu_read_lock();
 	ops = rcu_dereference(fence->ops);
@@ -656,14 +656,14 @@ static bool __dma_fence_enable_signaling(struct dma_fence *fence)
 }
 
 /**
- * dma_fence_enable_sw_signaling - enable signaling on fence
+ * dma_fence_enable_signaling - enable signaling on fence
  * @fence: the fence to enable
  *
  * This will request for sw signaling to be enabled, to make the fence
  * complete as soon as possible. This calls &dma_fence_ops.enable_signaling
  * internally.
  */
-void dma_fence_enable_sw_signaling(struct dma_fence *fence)
+void dma_fence_enable_signaling(struct dma_fence *fence)
 {
 	unsigned long flags;
 
@@ -671,7 +671,7 @@ void dma_fence_enable_sw_signaling(struct dma_fence *fence)
 	__dma_fence_enable_signaling(fence);
 	dma_fence_unlock_irqrestore(fence, flags);
 }
-EXPORT_SYMBOL(dma_fence_enable_sw_signaling);
+EXPORT_SYMBOL(dma_fence_enable_signaling);
 
 /**
  * dma_fence_add_callback - add a callback to be called when the fence
@@ -1102,9 +1102,12 @@ __dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
  * context and seqno are used for easy comparison between fences, allowing
  * to check which fence is later by simply using dma_fence_later().
  *
- * It is strongly discouraged to provide an external lock because this couples
- * lock and fence life time. This is only allowed for legacy use cases when
- * multiple fences need to be prevented from signaling out of order.
+ * External locks are a relic of legacy use cases that needed a shared lock
+ * to serialize signaling when no out-of-order signaling was possible through
+ * &dma_fence_ops.signaled. Drivers have abandoned this concept since the
+ * introduction of the callback, but the external lock is still around. New
+ * users MUST NOT use external locks, as they force the issuer to outlive all
+ * fences that reference the lock.
  */
 void
 dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
@@ -1129,9 +1132,8 @@ EXPORT_SYMBOL(dma_fence_init);
  * Context and seqno are used for easy comparison between fences, allowing
  * to check which fence is later by simply using dma_fence_later().
  *
- * It is strongly discouraged to provide an external lock because this couples
- * lock and fence life time. This is only allowed for legacy use cases when
- * multiple fences need to be prevented from signaling out of order.
+ * New users MUST NOT use external locks. Check the documentation in
+ * dma_fence_init() to understand the motives behind the legacy use cases.
  */
 void
 dma_fence_init64(struct dma_fence *fence, const struct dma_fence_ops *ops,
diff --git a/drivers/dma-buf/dma-heap.c b/drivers/dma-buf/dma-heap.c
index a76bf3f8b071a..3937dd41bb0f6 100644
--- a/drivers/dma-buf/dma-heap.c
+++ b/drivers/dma-buf/dma-heap.c
@@ -13,6 +13,7 @@
 #include <linux/err.h>
 #include <linux/export.h>
 #include <linux/list.h>
+#include <linux/minmax.h>
 #include <linux/nospec.h>
 #include <linux/syscalls.h>
 #include <linux/uaccess.h>
@@ -154,7 +155,7 @@ static long dma_heap_ioctl(struct file *file, unsigned int ucmd,
 		in_size = 0;
 	if ((ucmd & kcmd & IOC_OUT) == 0)
 		out_size = 0;
-	ksize = max(max(in_size, out_size), drv_size);
+	ksize = max3(in_size, out_size, drv_size);
 
 	/* If necessary, allocate buffer for ioctl argument */
 	if (ksize > sizeof(stack_kdata)) {
diff --git a/drivers/dma-buf/st-dma-fence-chain.c b/drivers/dma-buf/st-dma-fence-chain.c
index a3023d3fedc9d..e0d9b69bfa76a 100644
--- a/drivers/dma-buf/st-dma-fence-chain.c
+++ b/drivers/dma-buf/st-dma-fence-chain.c
@@ -82,7 +82,7 @@ static void test_sanitycheck(struct kunit *test)
 
 	chain = mock_chain(NULL, f, 1);
 	if (chain)
-		dma_fence_enable_sw_signaling(chain);
+		dma_fence_enable_signaling(chain);
 	else
 		KUNIT_FAIL(test, "Failed to create chain");
 
@@ -139,7 +139,7 @@ static int fence_chains_init(struct fence_chains *fc, unsigned int count,
 
 		fc->tail = fc->chains[i];
 
-		dma_fence_enable_sw_signaling(fc->chains[i]);
+		dma_fence_enable_signaling(fc->chains[i]);
 	}
 
 	fc->chain_length = i;
diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c b/drivers/dma-buf/st-dma-fence-unwrap.c
index 4e7ee25372ba6..4d9d313b460cb 100644
--- a/drivers/dma-buf/st-dma-fence-unwrap.c
+++ b/drivers/dma-buf/st-dma-fence-unwrap.c
@@ -103,7 +103,7 @@ static void test_sanitycheck(struct kunit *test)
 	f = mock_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f);
 
-	dma_fence_enable_sw_signaling(f);
+	dma_fence_enable_signaling(f);
 
 	array = mock_array(1, f);
 	KUNIT_ASSERT_NOT_NULL(test, array);
@@ -122,7 +122,7 @@ static void test_unwrap_array(struct kunit *test)
 	f1 = mock_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f1);
 
-	dma_fence_enable_sw_signaling(f1);
+	dma_fence_enable_signaling(f1);
 
 	f2 = mock_fence();
 	if (!f2) {
@@ -131,7 +131,7 @@ static void test_unwrap_array(struct kunit *test)
 		return;
 	}
 
-	dma_fence_enable_sw_signaling(f2);
+	dma_fence_enable_signaling(f2);
 
 	array = mock_array(2, f1, f2);
 	KUNIT_ASSERT_NOT_NULL(test, array);
@@ -160,7 +160,7 @@ static void test_unwrap_chain(struct kunit *test)
 	f1 = mock_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f1);
 
-	dma_fence_enable_sw_signaling(f1);
+	dma_fence_enable_signaling(f1);
 
 	f2 = mock_fence();
 	if (!f2) {
@@ -169,7 +169,7 @@ static void test_unwrap_chain(struct kunit *test)
 		return;
 	}
 
-	dma_fence_enable_sw_signaling(f2);
+	dma_fence_enable_signaling(f2);
 
 	chain = mock_chain(f1, f2);
 	KUNIT_ASSERT_NOT_NULL(test, chain);
@@ -198,7 +198,7 @@ static void test_unwrap_chain_array(struct kunit *test)
 	f1 = mock_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f1);
 
-	dma_fence_enable_sw_signaling(f1);
+	dma_fence_enable_signaling(f1);
 
 	f2 = mock_fence();
 	if (!f2) {
@@ -207,7 +207,7 @@ static void test_unwrap_chain_array(struct kunit *test)
 		return;
 	}
 
-	dma_fence_enable_sw_signaling(f2);
+	dma_fence_enable_signaling(f2);
 
 	array = mock_array(2, f1, f2);
 	KUNIT_ASSERT_NOT_NULL(test, array);
@@ -239,7 +239,7 @@ static void test_unwrap_merge(struct kunit *test)
 	f1 = mock_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f1);
 
-	dma_fence_enable_sw_signaling(f1);
+	dma_fence_enable_signaling(f1);
 
 	f2 = mock_fence();
 	if (!f2) {
@@ -247,7 +247,7 @@ static void test_unwrap_merge(struct kunit *test)
 		goto error_put_f1;
 	}
 
-	dma_fence_enable_sw_signaling(f2);
+	dma_fence_enable_signaling(f2);
 
 	f3 = dma_fence_unwrap_merge(f1, f2);
 	if (!f3) {
@@ -285,7 +285,7 @@ static void test_unwrap_merge_duplicate(struct kunit *test)
 	f1 = mock_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f1);
 
-	dma_fence_enable_sw_signaling(f1);
+	dma_fence_enable_signaling(f1);
 
 	f2 = dma_fence_unwrap_merge(f1, f1);
 	if (!f2) {
@@ -322,7 +322,7 @@ static void test_unwrap_merge_seqno(struct kunit *test)
 	f1 = __mock_fence(ctx[1], 1);
 	KUNIT_ASSERT_NOT_NULL(test, f1);
 
-	dma_fence_enable_sw_signaling(f1);
+	dma_fence_enable_signaling(f1);
 
 	f2 = __mock_fence(ctx[1], 2);
 	if (!f2) {
@@ -330,7 +330,7 @@ static void test_unwrap_merge_seqno(struct kunit *test)
 		goto error_put_f1;
 	}
 
-	dma_fence_enable_sw_signaling(f2);
+	dma_fence_enable_signaling(f2);
 
 	f3 = __mock_fence(ctx[0], 1);
 	if (!f3) {
@@ -338,7 +338,7 @@ static void test_unwrap_merge_seqno(struct kunit *test)
 		goto error_put_f2;
 	}
 
-	dma_fence_enable_sw_signaling(f3);
+	dma_fence_enable_signaling(f3);
 
 	f4 = dma_fence_unwrap_merge(f1, f2, f3);
 	if (!f4) {
@@ -378,7 +378,7 @@ static void test_unwrap_merge_order(struct kunit *test)
 	f1 = mock_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f1);
 
-	dma_fence_enable_sw_signaling(f1);
+	dma_fence_enable_signaling(f1);
 
 	f2 = mock_fence();
 	if (!f2) {
@@ -387,7 +387,7 @@ static void test_unwrap_merge_order(struct kunit *test)
 		return;
 	}
 
-	dma_fence_enable_sw_signaling(f2);
+	dma_fence_enable_signaling(f2);
 
 	a1 = mock_array(2, f1, f2);
 	KUNIT_ASSERT_NOT_NULL(test, a1);
@@ -442,7 +442,7 @@ static void test_unwrap_merge_complex(struct kunit *test)
 	f1 = mock_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f1);
 
-	dma_fence_enable_sw_signaling(f1);
+	dma_fence_enable_signaling(f1);
 
 	f2 = mock_fence();
 	if (!f2) {
@@ -450,7 +450,7 @@ static void test_unwrap_merge_complex(struct kunit *test)
 		goto error_put_f1;
 	}
 
-	dma_fence_enable_sw_signaling(f2);
+	dma_fence_enable_signaling(f2);
 
 	f3 = dma_fence_unwrap_merge(f1, f2);
 	if (!f3) {
@@ -510,7 +510,7 @@ static void test_unwrap_merge_complex_seqno(struct kunit *test)
 	f1 = __mock_fence(ctx[0], 2);
 	KUNIT_ASSERT_NOT_NULL(test, f1);
 
-	dma_fence_enable_sw_signaling(f1);
+	dma_fence_enable_signaling(f1);
 
 	f2 = __mock_fence(ctx[1], 1);
 	if (!f2) {
@@ -518,7 +518,7 @@ static void test_unwrap_merge_complex_seqno(struct kunit *test)
 		goto error_put_f1;
 	}
 
-	dma_fence_enable_sw_signaling(f2);
+	dma_fence_enable_signaling(f2);
 
 	f3 = __mock_fence(ctx[0], 1);
 	if (!f3) {
@@ -526,7 +526,7 @@ static void test_unwrap_merge_complex_seqno(struct kunit *test)
 		goto error_put_f2;
 	}
 
-	dma_fence_enable_sw_signaling(f3);
+	dma_fence_enable_signaling(f3);
 
 	f4 = __mock_fence(ctx[1], 2);
 	if (!f4) {
@@ -534,7 +534,7 @@ static void test_unwrap_merge_complex_seqno(struct kunit *test)
 		goto error_put_f3;
 	}
 
-	dma_fence_enable_sw_signaling(f4);
+	dma_fence_enable_signaling(f4);
 
 	f5 = mock_array(2, dma_fence_get(f1), dma_fence_get(f2));
 	if (!f5) {
diff --git a/drivers/dma-buf/st-dma-fence.c b/drivers/dma-buf/st-dma-fence.c
index 4992722296968..856d0d302a5df 100644
--- a/drivers/dma-buf/st-dma-fence.c
+++ b/drivers/dma-buf/st-dma-fence.c
@@ -42,7 +42,7 @@ static void test_sanitycheck(struct kunit *test)
 	f = mock_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f);
 
-	dma_fence_enable_sw_signaling(f);
+	dma_fence_enable_signaling(f);
 
 	dma_fence_signal(f);
 	dma_fence_put(f);
@@ -55,7 +55,7 @@ static void test_signaling(struct kunit *test)
 	f = mock_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f);
 
-	dma_fence_enable_sw_signaling(f);
+	dma_fence_enable_signaling(f);
 
 	if (dma_fence_is_signaled(f)) {
 		KUNIT_FAIL(test, "Fence unexpectedly signaled on creation");
@@ -127,7 +127,7 @@ static void test_late_add_callback(struct kunit *test)
 	f = mock_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f);
 
-	dma_fence_enable_sw_signaling(f);
+	dma_fence_enable_signaling(f);
 
 	dma_fence_signal(f);
 
@@ -209,7 +209,7 @@ static void test_status(struct kunit *test)
 	f = mock_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f);
 
-	dma_fence_enable_sw_signaling(f);
+	dma_fence_enable_signaling(f);
 
 	if (dma_fence_get_status(f)) {
 		KUNIT_FAIL(test, "Fence unexpectedly has signaled status on creation");
@@ -233,7 +233,7 @@ static void test_error(struct kunit *test)
 	f = mock_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f);
 
-	dma_fence_enable_sw_signaling(f);
+	dma_fence_enable_signaling(f);
 
 	dma_fence_set_error(f, -EIO);
 
@@ -260,7 +260,7 @@ static void test_wait(struct kunit *test)
 	f = mock_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f);
 
-	dma_fence_enable_sw_signaling(f);
+	dma_fence_enable_signaling(f);
 
 	if (dma_fence_wait_timeout(f, false, 0) != 0) {
 		KUNIT_FAIL(test, "Wait reported complete before being signaled");
@@ -300,7 +300,7 @@ static void test_wait_timeout(struct kunit *test)
 	wt.f = mock_fence();
 	KUNIT_ASSERT_NOT_NULL(test, wt.f);
 
-	dma_fence_enable_sw_signaling(wt.f);
+	dma_fence_enable_signaling(wt.f);
 
 	if (dma_fence_wait_timeout(wt.f, false, 1) != 0) {
 		KUNIT_FAIL(test, "Wait reported complete before being signaled");
@@ -379,7 +379,7 @@ static int thread_signal_callback(void *arg)
 			break;
 		}
 
-		dma_fence_enable_sw_signaling(f1);
+		dma_fence_enable_signaling(f1);
 
 		rcu_assign_pointer(t->fences[t->id], f1);
 		smp_wmb();
diff --git a/drivers/dma-buf/st-dma-resv.c b/drivers/dma-buf/st-dma-resv.c
index 95a4becdb8926..0b96136bbd546 100644
--- a/drivers/dma-buf/st-dma-resv.c
+++ b/drivers/dma-buf/st-dma-resv.c
@@ -48,7 +48,7 @@ static void test_sanitycheck(struct kunit *test)
 	f = alloc_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f);
 
-	dma_fence_enable_sw_signaling(f);
+	dma_fence_enable_signaling(f);
 
 	dma_fence_signal(f);
 	dma_fence_put(f);
@@ -73,7 +73,7 @@ static void test_signaling(struct kunit *test)
 	f = alloc_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f);
 
-	dma_fence_enable_sw_signaling(f);
+	dma_fence_enable_signaling(f);
 
 	dma_resv_init(&resv);
 	r = dma_resv_lock(&resv, NULL);
@@ -117,7 +117,7 @@ static void test_for_each(struct kunit *test)
 	f = alloc_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f);
 
-	dma_fence_enable_sw_signaling(f);
+	dma_fence_enable_signaling(f);
 
 	dma_resv_init(&resv);
 	r = dma_resv_lock(&resv, NULL);
@@ -176,7 +176,7 @@ static void test_for_each_unlocked(struct kunit *test)
 	f = alloc_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f);
 
-	dma_fence_enable_sw_signaling(f);
+	dma_fence_enable_signaling(f);
 
 	dma_resv_init(&resv);
 	r = dma_resv_lock(&resv, NULL);
@@ -246,7 +246,7 @@ static void test_get_fences(struct kunit *test)
 	f = alloc_fence();
 	KUNIT_ASSERT_NOT_NULL(test, f);
 
-	dma_fence_enable_sw_signaling(f);
+	dma_fence_enable_signaling(f);
 
 	dma_resv_init(&resv);
 	r = dma_resv_lock(&resv, NULL);
diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c
index bced421c0d658..08f57bc1294df 100644
--- a/drivers/dma-buf/udmabuf.c
+++ b/drivers/dma-buf/udmabuf.c
@@ -224,21 +224,22 @@ static int begin_cpu_udmabuf(struct dma_buf *buf,
 {
 	struct udmabuf *ubuf = buf->priv;
 	struct device *dev = ubuf->device->this_device;
-	int ret = 0;
 
 	if (!ubuf->sg) {
 		ubuf->sg = get_sg_table(dev, buf, direction);
 		if (IS_ERR(ubuf->sg)) {
+			int ret;
+
 			ret = PTR_ERR(ubuf->sg);
 			ubuf->sg = NULL;
+			return ret;
 		} else {
 			ubuf->sg_dir = direction;
 		}
-	} else {
-		dma_sync_sgtable_for_cpu(dev, ubuf->sg, direction);
 	}
 
-	return ret;
+	dma_sync_sgtable_for_cpu(dev, ubuf->sg, direction);
+	return 0;
 }
 
 static int end_cpu_udmabuf(struct dma_buf *buf,
diff --git a/drivers/dma/arm-dma350.c b/drivers/dma/arm-dma350.c
index 84220fa83029d..09403aca8bb06 100644
--- a/drivers/dma/arm-dma350.c
+++ b/drivers/dma/arm-dma350.c
@@ -13,6 +13,11 @@
 #include "dmaengine.h"
 #include "virt-dma.h"
 
+#define DMANSECCTRL		0x200
+
+#define NSEC_CTRL		0x0c
+#define INTREN_ANYCHINTR_EN	BIT(0)
+
 #define DMAINFO			0x0f00
 
 #define DMA_BUILDCFG0		0xb0
@@ -582,6 +587,10 @@ static int d350_probe(struct platform_device *pdev)
 	dmac->dma.device_issue_pending = d350_issue_pending;
 	INIT_LIST_HEAD(&dmac->dma.channels);
 
+	reg = readl_relaxed(base + DMANSECCTRL + NSEC_CTRL);
+	writel_relaxed(reg | INTREN_ANYCHINTR_EN,
+		       base + DMANSECCTRL + NSEC_CTRL);
+
 	/* Would be nice to have per-channel caps for this... */
 	memset = true;
 	for (int i = 0; i < nchan; i++) {
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 9049171df8578..6ffd8bd82154a 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -814,10 +814,15 @@ static const struct dma_slave_map *dma_filter_match(struct dma_device *device,
  */
 struct dma_chan *dma_request_chan(struct device *dev, const char *name)
 {
-	struct fwnode_handle *fwnode = dev_fwnode(dev);
+	struct fwnode_handle *fwnode;
 	struct dma_device *d, *_d;
 	struct dma_chan *chan = NULL;
 
+	if (WARN_ON(!dev || !name))
+		return ERR_PTR(-EINVAL);
+
+	fwnode = dev_fwnode(dev);
+
 	if (is_of_node(fwnode))
 		chan = of_dma_request_slave_channel(to_of_node(fwnode), name);
 	else if (is_acpi_device_node(fwnode))
@@ -1100,6 +1105,8 @@ static int __dma_async_device_channel_register(struct dma_device *device,
 	chan->dev->device.parent = device->dev;
 	chan->dev->chan = chan;
 	chan->dev->dev_id = device->dev_id;
+	spin_lock_init(&chan->lock);
+
 	if (!name)
 		dev_set_name(&chan->dev->device, "dma%dchan%d", device->dev_id, chan->chan_id);
 	else
diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
index bcefaff03b5c6..eebed2474210d 100644
--- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
+++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
@@ -1126,7 +1126,7 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan)
 				hw_desc = &desc->hw_desc[i];
 				if (hw_desc->llp == llp) {
 					axi_chan_irq_clear(chan, hw_desc->lli->status_lo);
-					hw_desc->lli->ctl_hi |= CH_CTL_H_LLI_VALID;
+					hw_desc->lli->ctl_hi |= cpu_to_le32(CH_CTL_H_LLI_VALID);
 					desc->completed_blocks = i;
 
 					if (((hw_desc->len * (i + 1)) % desc->period_len) == 0)
diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index 89a4c498a17b8..384c482058d1b 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -40,141 +40,63 @@ u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr)
 	return cpu_addr;
 }
 
-static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk)
-{
-	struct dw_edma_burst *burst;
-
-	burst = kzalloc_obj(*burst, GFP_NOWAIT);
-	if (unlikely(!burst))
-		return NULL;
-
-	INIT_LIST_HEAD(&burst->list);
-	if (chunk->burst) {
-		/* Create and add new element into the linked list */
-		chunk->bursts_alloc++;
-		list_add_tail(&burst->list, &chunk->burst->list);
-	} else {
-		/* List head */
-		chunk->bursts_alloc = 0;
-		chunk->burst = burst;
-	}
-
-	return burst;
-}
-
-static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc)
-{
-	struct dw_edma_chip *chip = desc->chan->dw->chip;
-	struct dw_edma_chan *chan = desc->chan;
-	struct dw_edma_chunk *chunk;
-
-	chunk = kzalloc_obj(*chunk, GFP_NOWAIT);
-	if (unlikely(!chunk))
-		return NULL;
-
-	INIT_LIST_HEAD(&chunk->list);
-	chunk->chan = chan;
-	/* Toggling change bit (CB) in each chunk, this is a mechanism to
-	 * inform the eDMA HW block that this is a new linked list ready
-	 * to be consumed.
-	 *  - Odd chunks originate CB equal to 0
-	 *  - Even chunks originate CB equal to 1
-	 */
-	chunk->cb = !(desc->chunks_alloc % 2);
-	if (chan->dir == EDMA_DIR_WRITE) {
-		chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr;
-		chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr;
-	} else {
-		chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr;
-		chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr;
-	}
-
-	if (desc->chunk) {
-		/* Create and add new element into the linked list */
-		if (!dw_edma_alloc_burst(chunk)) {
-			kfree(chunk);
-			return NULL;
-		}
-		desc->chunks_alloc++;
-		list_add_tail(&chunk->list, &desc->chunk->list);
-	} else {
-		/* List head */
-		chunk->burst = NULL;
-		desc->chunks_alloc = 0;
-		desc->chunk = chunk;
-	}
-
-	return chunk;
-}
-
-static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan)
+static struct dw_edma_desc *
+dw_edma_alloc_desc(struct dw_edma_chan *chan, size_t nburst)
 {
 	struct dw_edma_desc *desc;
 
-	desc = kzalloc_obj(*desc, GFP_NOWAIT);
+	desc = kzalloc_flex(*desc, burst, nburst, GFP_NOWAIT);
 	if (unlikely(!desc))
 		return NULL;
 
 	desc->chan = chan;
-	if (!dw_edma_alloc_chunk(desc)) {
-		kfree(desc);
-		return NULL;
-	}
+	desc->nburst = nburst;
+	desc->cb = true;
 
 	return desc;
 }
 
-static void dw_edma_free_burst(struct dw_edma_chunk *chunk)
+static void vchan_free_desc(struct virt_dma_desc *vdesc)
 {
-	struct dw_edma_burst *child, *_next;
-
-	/* Remove all the list elements */
-	list_for_each_entry_safe(child, _next, &chunk->burst->list, list) {
-		list_del(&child->list);
-		kfree(child);
-		chunk->bursts_alloc--;
-	}
-
-	/* Remove the list head */
-	kfree(child);
-	chunk->burst = NULL;
+	kfree(vd2dw_edma_desc(vdesc));
 }
 
-static void dw_edma_free_chunk(struct dw_edma_desc *desc)
+static void dw_edma_core_start(struct dw_edma_desc *desc, bool first)
 {
-	struct dw_edma_chunk *child, *_next;
+	struct dw_edma_chan *chan = desc->chan;
+	size_t i = 0;
 
-	if (!desc->chunk)
+	if (chan->non_ll) {
+		chan->dw->core->non_ll_start(chan, &desc->burst[desc->start_burst]);
+		desc->done_burst = desc->start_burst;
+		desc->start_burst += 1;
 		return;
+	}
+
+	for (i = 0; i + desc->start_burst < desc->nburst; i++) {
+		u32 idx = i + desc->start_burst;
+
+		if (i == chan->ll_max)
+			break;
 
-	/* Remove all the list elements */
-	list_for_each_entry_safe(child, _next, &desc->chunk->list, list) {
-		dw_edma_free_burst(child);
-		list_del(&child->list);
-		kfree(child);
-		desc->chunks_alloc--;
+		dw_edma_core_ll_data(chan, &desc->burst[idx],
+				     i, desc->cb,
+				     idx == desc->nburst - 1 || i == chan->ll_max - 1);
 	}
 
-	/* Remove the list head */
-	kfree(child);
-	desc->chunk = NULL;
-}
+	desc->done_burst = desc->start_burst;
+	desc->start_burst += i;
 
-static void dw_edma_free_desc(struct dw_edma_desc *desc)
-{
-	dw_edma_free_chunk(desc);
-	kfree(desc);
-}
+	dw_edma_core_ll_link(chan, i, desc->cb, chan->ll_region.paddr);
 
-static void vchan_free_desc(struct virt_dma_desc *vdesc)
-{
-	dw_edma_free_desc(vd2dw_edma_desc(vdesc));
+	if (first)
+		dw_edma_core_ch_enable(chan);
+
+	dw_edma_core_ch_doorbell(chan);
 }
 
 static int dw_edma_start_transfer(struct dw_edma_chan *chan)
 {
-	struct dw_edma *dw = chan->dw;
-	struct dw_edma_chunk *child;
 	struct dw_edma_desc *desc;
 	struct virt_dma_desc *vd;
 
@@ -186,17 +108,9 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan)
 	if (!desc)
 		return 0;
 
-	child = list_first_entry_or_null(&desc->chunk->list,
-					 struct dw_edma_chunk, list);
-	if (!child)
-		return 0;
+	dw_edma_core_start(desc, !desc->start_burst);
 
-	dw_edma_core_start(dw, child, !desc->xfer_sz);
-	desc->xfer_sz += child->ll_region.sz;
-	dw_edma_free_burst(child);
-	list_del(&child->list);
-	kfree(child);
-	desc->chunks_alloc--;
+	desc->cb = !desc->cb;
 
 	return 1;
 }
@@ -267,6 +181,20 @@ static int dw_edma_device_config(struct dma_chan *dchan,
 	return 0;
 }
 
+static struct dma_slave_config *
+dw_edma_device_get_config(struct dma_chan *dchan,
+			  struct dma_slave_config *config)
+{
+	struct dw_edma_chan *chan;
+
+	if (config)
+		return config;
+
+	chan = dchan2dw_edma_chan(dchan);
+
+	return &chan->config;
+}
+
 static int dw_edma_device_pause(struct dma_chan *dchan)
 {
 	struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
@@ -373,8 +301,10 @@ dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
 	vd = vchan_find_desc(&chan->vc, cookie);
 	if (vd) {
 		desc = vd2dw_edma_desc(vd);
-		if (desc)
-			residue = desc->alloc_sz - desc->xfer_sz;
+
+		residue = desc->alloc_sz;
+		if (desc && desc->done_burst)
+			residue -= desc->burst[desc->done_burst - 1].xfer_sz;
 	}
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
 
@@ -385,19 +315,18 @@ dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
 }
 
 static struct dma_async_tx_descriptor *
-dw_edma_device_transfer(struct dw_edma_transfer *xfer)
+dw_edma_device_transfer(struct dw_edma_transfer *xfer,
+			struct dma_slave_config *config)
 {
 	struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan);
 	enum dma_transfer_direction dir = xfer->direction;
 	struct scatterlist *sg = NULL;
-	struct dw_edma_chunk *chunk;
 	struct dw_edma_burst *burst;
 	struct dw_edma_desc *desc;
 	u64 src_addr, dst_addr;
 	size_t fsz = 0;
-	u32 bursts_max;
-	u32 cnt = 0;
-	int i;
+	size_t cnt = 0;
+	u32 i;
 
 	if (!chan->configured)
 		return NULL;
@@ -453,27 +382,12 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
 		return NULL;
 	}
 
-	/*
-	 * For non-LL mode, only a single burst can be handled
-	 * in a single chunk unlike LL mode where multiple bursts
-	 * can be configured in a single chunk.
-	 */
-	bursts_max = chan->non_ll ? 1 : chan->ll_max;
-
-	desc = dw_edma_alloc_desc(chan);
-	if (unlikely(!desc))
-		goto err_alloc;
-
-	chunk = dw_edma_alloc_chunk(desc);
-	if (unlikely(!chunk))
-		goto err_alloc;
-
 	if (xfer->type == EDMA_XFER_INTERLEAVED) {
 		src_addr = xfer->xfer.il->src_start;
 		dst_addr = xfer->xfer.il->dst_start;
 	} else {
-		src_addr = chan->config.src_addr;
-		dst_addr = chan->config.dst_addr;
+		src_addr = config->src_addr;
+		dst_addr = config->dst_addr;
 	}
 
 	if (dir == DMA_DEV_TO_MEM)
@@ -491,19 +405,15 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
 		fsz = xfer->xfer.il->frame_size;
 	}
 
+	desc = dw_edma_alloc_desc(chan, cnt);
+	if (unlikely(!desc))
+		return NULL;
+
 	for (i = 0; i < cnt; i++) {
 		if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg)
 			break;
 
-		if (chunk->bursts_alloc == bursts_max) {
-			chunk = dw_edma_alloc_chunk(desc);
-			if (unlikely(!chunk))
-				goto err_alloc;
-		}
-
-		burst = dw_edma_alloc_burst(chunk);
-		if (unlikely(!burst))
-			goto err_alloc;
+		burst = desc->burst + i;
 
 		if (xfer->type == EDMA_XFER_CYCLIC)
 			burst->sz = xfer->xfer.cyclic.len;
@@ -512,8 +422,8 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
 		else if (xfer->type == EDMA_XFER_INTERLEAVED)
 			burst->sz = xfer->xfer.il->sgl[i % fsz].size;
 
-		chunk->ll_region.sz += burst->sz;
 		desc->alloc_sz += burst->sz;
+		burst->xfer_sz = desc->alloc_sz;
 
 		if (dir == DMA_DEV_TO_MEM) {
 			burst->sar = src_addr;
@@ -568,19 +478,14 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
 	}
 
 	return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags);
-
-err_alloc:
-	if (desc)
-		dw_edma_free_desc(desc);
-
-	return NULL;
 }
 
 static struct dma_async_tx_descriptor *
-dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
-			     unsigned int len,
-			     enum dma_transfer_direction direction,
-			     unsigned long flags, void *context)
+dw_edma_device_prep_config_sg(struct dma_chan *dchan, struct scatterlist *sgl,
+			      unsigned int len,
+			      enum dma_transfer_direction direction,
+			      unsigned long flags,
+			      struct dma_slave_config *config)
 {
 	struct dw_edma_transfer xfer;
 
@@ -591,7 +496,10 @@ dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
 	xfer.flags = flags;
 	xfer.type = EDMA_XFER_SCATTER_GATHER;
 
-	return dw_edma_device_transfer(&xfer);
+	if (config && dw_edma_device_config(dchan, config))
+		return NULL;
+
+	return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, config));
 }
 
 static struct dma_async_tx_descriptor *
@@ -610,7 +518,7 @@ dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr,
 	xfer.flags = flags;
 	xfer.type = EDMA_XFER_CYCLIC;
 
-	return dw_edma_device_transfer(&xfer);
+	return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, NULL));
 }
 
 static struct dma_async_tx_descriptor *
@@ -626,7 +534,7 @@ dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,
 	xfer.flags = flags;
 	xfer.type = EDMA_XFER_INTERLEAVED;
 
-	return dw_edma_device_transfer(&xfer);
+	return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, NULL));
 }
 
 static void dw_hdma_set_callback_result(struct virt_dma_desc *vd,
@@ -640,8 +548,14 @@ static void dw_hdma_set_callback_result(struct virt_dma_desc *vd,
 		return;
 
 	desc = vd2dw_edma_desc(vd);
-	if (desc)
-		residue = desc->alloc_sz - desc->xfer_sz;
+	if (desc) {
+		residue = desc->alloc_sz;
+
+		if (result == DMA_TRANS_NOERROR)
+			residue -= desc->burst[desc->start_burst - 1].xfer_sz;
+		else if (desc->done_burst)
+			residue -= desc->burst[desc->done_burst - 1].xfer_sz;
+	}
 
 	res = &vd->tx_result;
 	res->result = result;
@@ -660,7 +574,7 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
 		switch (chan->request) {
 		case EDMA_REQ_NONE:
 			desc = vd2dw_edma_desc(vd);
-			if (!desc->chunks_alloc) {
+			if (desc->start_burst >= desc->nburst) {
 				dw_hdma_set_callback_result(vd,
 							    DMA_TRANS_NOERROR);
 				list_del(&vd->node);
@@ -906,10 +820,11 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
 		chan->status = EDMA_ST_IDLE;
 
 		if (chan->dir == EDMA_DIR_WRITE)
-			chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ);
+			chan->ll_region = chip->ll_region_wr[chan->id];
 		else
-			chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ);
-		chan->ll_max -= 1;
+			chan->ll_region = chip->ll_region_rd[chan->id];
+
+		chan->ll_max = chan->ll_region.sz / EDMA_LL_SZ - 1;
 
 		dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n",
 			 str_write_read(chan->dir == EDMA_DIR_WRITE),
@@ -925,9 +840,9 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
 		irq = &dw->irq[pos];
 
 		if (chan->dir == EDMA_DIR_WRITE)
-			irq->wr_mask |= BIT(chan->id);
+			bitmap_set(irq->wr_mask, chan->id, 1);
 		else
-			irq->rd_mask |= BIT(chan->id);
+			bitmap_set(irq->rd_mask, chan->id, 1);
 
 		irq->dw = dw;
 		memcpy(&chan->msi, &irq->msi, sizeof(chan->msi));
@@ -970,7 +885,7 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
 	dma->device_terminate_all = dw_edma_device_terminate_all;
 	dma->device_issue_pending = dw_edma_device_issue_pending;
 	dma->device_tx_status = dw_edma_device_tx_status;
-	dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg;
+	dma->device_prep_config_sg = dw_edma_device_prep_config_sg;
 	dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic;
 	dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma;
 
@@ -1068,6 +983,8 @@ int dw_edma_probe(struct dw_edma_chip *chip)
 	struct dw_edma *dw;
 	u32 wr_alloc = 0;
 	u32 rd_alloc = 0;
+	u16 max_wr_cnt;
+	u16 max_rd_cnt;
 	int i, err;
 
 	if (!chip)
@@ -1083,20 +1000,25 @@ int dw_edma_probe(struct dw_edma_chip *chip)
 
 	dw->chip = chip;
 
-	if (dw->chip->mf == EDMA_MF_HDMA_NATIVE)
+	if (dw->chip->mf == EDMA_MF_HDMA_NATIVE) {
 		dw_hdma_v0_core_register(dw);
-	else
+		max_wr_cnt = HDMA_MAX_WR_CH;
+		max_rd_cnt = HDMA_MAX_RD_CH;
+	} else {
 		dw_edma_v0_core_register(dw);
+		max_wr_cnt = EDMA_MAX_WR_CH;
+		max_rd_cnt = EDMA_MAX_RD_CH;
+	}
 
 	raw_spin_lock_init(&dw->lock);
 
 	dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt,
 			      dw_edma_core_ch_count(dw, EDMA_DIR_WRITE));
-	dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH);
+	dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, max_wr_cnt);
 
 	dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt,
 			      dw_edma_core_ch_count(dw, EDMA_DIR_READ));
-	dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH);
+	dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, max_rd_cnt);
 
 	if (!dw->wr_ch_cnt && !dw->rd_ch_cnt)
 		return -EINVAL;
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index 6474cacf71953..e39c11bc91fbf 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -43,32 +43,24 @@ struct dw_edma_chan;
 struct dw_edma_chunk;
 
 struct dw_edma_burst {
-	struct list_head		list;
 	u64				sar;
 	u64				dar;
 	u32				sz;
-};
-
-struct dw_edma_chunk {
-	struct list_head		list;
-	struct dw_edma_chan		*chan;
-	struct dw_edma_burst		*burst;
-
-	u32				bursts_alloc;
-
-	u8				cb;
-	struct dw_edma_region		ll_region;	/* Linked list */
+	/* precalulate summary of previous burst total size */
+	u32				xfer_sz;
 };
 
 struct dw_edma_desc {
 	struct virt_dma_desc		vd;
 	struct dw_edma_chan		*chan;
-	struct dw_edma_chunk		*chunk;
-
-	u32				chunks_alloc;
 
 	u32				alloc_sz;
-	u32				xfer_sz;
+
+	size_t				done_burst;
+	size_t				start_burst;
+	u8				cb;
+	size_t				nburst;
+	struct dw_edma_burst            burst[] __counted_by(nburst);
 };
 
 struct dw_edma_chan {
@@ -78,6 +70,7 @@ struct dw_edma_chan {
 	enum dw_edma_dir		dir;
 
 	u32				ll_max;
+	struct dw_edma_region		ll_region;	/* Linked list */
 
 	struct msi_msg			msi;
 
@@ -91,9 +84,10 @@ struct dw_edma_chan {
 
 struct dw_edma_irq {
 	struct msi_msg                  msi;
-	u32				wr_mask;
-	u32				rd_mask;
 	struct dw_edma			*dw;
+
+	DECLARE_BITMAP(wr_mask, HDMA_MAX_WR_CH);
+	DECLARE_BITMAP(rd_mask, HDMA_MAX_RD_CH);
 };
 
 struct dw_edma {
@@ -124,7 +118,12 @@ struct dw_edma_core_ops {
 	enum dma_status (*ch_status)(struct dw_edma_chan *chan);
 	irqreturn_t (*handle_int)(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 				  dw_edma_handler_t done, dw_edma_handler_t abort);
-	void (*start)(struct dw_edma_chunk *chunk, bool first);
+	void (*non_ll_start)(struct dw_edma_chan *chan, struct dw_edma_burst *child);
+	void (*ll_data)(struct dw_edma_chan *chan, struct dw_edma_burst *burst,
+			u32 idx, bool cb, bool irq);
+	void (*ll_link)(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr);
+	void (*ch_doorbell)(struct dw_edma_chan *chan);
+	void (*ch_enable)(struct dw_edma_chan *chan);
 	void (*ch_config)(struct dw_edma_chan *chan);
 	void (*debugfs_on)(struct dw_edma *dw);
 	void (*ack_emulated_irq)(struct dw_edma *dw);
@@ -166,6 +165,14 @@ struct dw_edma_chan *dchan2dw_edma_chan(struct dma_chan *dchan)
 	return vc2dw_edma_chan(to_virt_chan(dchan));
 }
 
+static inline u64 dw_edma_core_get_ll_paddr(struct dw_edma_chan *chan)
+{
+	if (chan->dir == EDMA_DIR_WRITE)
+		return chan->dw->chip->ll_region_wr[chan->id].paddr;
+
+	return chan->dw->chip->ll_region_rd[chan->id].paddr;
+}
+
 static inline
 void dw_edma_core_off(struct dw_edma *dw)
 {
@@ -192,15 +199,32 @@ dw_edma_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 }
 
 static inline
-void dw_edma_core_start(struct dw_edma *dw, struct dw_edma_chunk *chunk, bool first)
+void dw_edma_core_ch_config(struct dw_edma_chan *chan)
 {
-	dw->core->start(chunk, first);
+	chan->dw->core->ch_config(chan);
 }
 
-static inline
-void dw_edma_core_ch_config(struct dw_edma_chan *chan)
+static inline void
+dw_edma_core_ll_data(struct dw_edma_chan *chan, struct dw_edma_burst *burst,
+		     u32 idx, bool cb, bool irq)
 {
-	chan->dw->core->ch_config(chan);
+	chan->dw->core->ll_data(chan, burst, idx, cb, irq);
+}
+
+static inline void
+dw_edma_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr)
+{
+	chan->dw->core->ll_link(chan, idx, cb, addr);
+}
+
+static inline void dw_edma_core_ch_doorbell(struct dw_edma_chan *chan)
+{
+	chan->dw->core->ch_doorbell(chan);
+}
+
+static inline void dw_edma_core_ch_enable(struct dw_edma_chan *chan)
+{
+	chan->dw->core->ch_enable(chan);
 }
 
 static inline
diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c
index 791c46e8ae4c4..b34e1e45aeb30 100644
--- a/drivers/dma/dw-edma/dw-edma-pcie.c
+++ b/drivers/dma/dw-edma/dw-edma-pcie.c
@@ -62,11 +62,11 @@ struct dw_edma_pcie_data {
 	/* eDMA registers location */
 	struct dw_edma_block		rg;
 	/* eDMA memory linked list location */
-	struct dw_edma_block		ll_wr[EDMA_MAX_WR_CH];
-	struct dw_edma_block		ll_rd[EDMA_MAX_RD_CH];
+	struct dw_edma_block		ll_wr[HDMA_MAX_WR_CH];
+	struct dw_edma_block		ll_rd[HDMA_MAX_RD_CH];
 	/* eDMA memory data location */
-	struct dw_edma_block		dt_wr[EDMA_MAX_WR_CH];
-	struct dw_edma_block		dt_rd[EDMA_MAX_RD_CH];
+	struct dw_edma_block		dt_wr[HDMA_MAX_WR_CH];
+	struct dw_edma_block		dt_rd[HDMA_MAX_RD_CH];
 	/* Other */
 	enum dw_edma_map_format		mf;
 	u8				irqs;
diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c
index cfdd6463252e6..7bb3ec461cce4 100644
--- a/drivers/dma/dw-edma/dw-edma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-edma-v0-core.c
@@ -239,7 +239,7 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 	irqreturn_t ret = IRQ_NONE;
 	struct dw_edma_chan *chan;
 	unsigned long off;
-	u32 mask;
+	unsigned long *mask;
 
 	if (dir == EDMA_DIR_WRITE) {
 		total = dw->wr_ch_cnt;
@@ -252,7 +252,7 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 	}
 
 	val = dw_edma_v0_core_status_done_int(dw, dir);
-	val &= mask;
+	val &= *mask;
 	for_each_set_bit(pos, &val, total) {
 		chan = &dw->chan[pos + off];
 
@@ -263,7 +263,7 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 	}
 
 	val = dw_edma_v0_core_status_abort_int(dw, dir);
-	val &= mask;
+	val &= *mask;
 	for_each_set_bit(pos, &val, total) {
 		chan = &dw->chan[pos + off];
 
@@ -276,77 +276,110 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 	return ret;
 }
 
-static void dw_edma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i,
+static void dw_edma_v0_write_ll_data(struct dw_edma_chan *chan, int i,
 				     u32 control, u32 size, u64 sar, u64 dar)
 {
 	ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli);
 
-	if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
-		struct dw_edma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs;
+	if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
+		struct dw_edma_v0_lli *lli = chan->ll_region.vaddr.mem + ofs;
 
-		lli->control = control;
 		lli->transfer_size = size;
 		lli->sar.reg = sar;
 		lli->dar.reg = dar;
+		dma_wmb();
+		lli->control = control;
 	} else {
-		struct dw_edma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs;
+		struct dw_edma_v0_lli __iomem *lli = chan->ll_region.vaddr.io + ofs;
 
-		writel(control, &lli->control);
 		writel(size, &lli->transfer_size);
 		writeq(sar, &lli->sar.reg);
 		writeq(dar, &lli->dar.reg);
+		writel(control, &lli->control);
 	}
 }
 
-static void dw_edma_v0_write_ll_link(struct dw_edma_chunk *chunk,
+static void dw_edma_v0_write_ll_link(struct dw_edma_chan *chan,
 				     int i, u32 control, u64 pointer)
 {
 	ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli);
 
-	if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
-		struct dw_edma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs;
+	if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
+		struct dw_edma_v0_llp *llp = chan->ll_region.vaddr.mem + ofs;
 
-		llp->control = control;
 		llp->llp.reg = pointer;
+		dma_wmb();
+		llp->control = control;
 	} else {
-		struct dw_edma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs;
+		struct dw_edma_v0_llp __iomem *llp = chan->ll_region.vaddr.io + ofs;
 
-		writel(control, &llp->control);
 		writeq(pointer, &llp->llp.reg);
+		writel(control, &llp->control);
 	}
 }
 
-static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
+static void dw_edma_v0_core_ch_enable(struct dw_edma_chan *chan)
 {
-	struct dw_edma_burst *child;
-	struct dw_edma_chan *chan = chunk->chan;
-	u32 control = 0, i = 0;
-	int j;
-
-	if (chunk->cb)
-		control = DW_EDMA_V0_CB;
-
-	j = chunk->bursts_alloc;
-	list_for_each_entry(child, &chunk->burst->list, list) {
-		j--;
-		if (!j) {
-			control |= DW_EDMA_V0_LIE;
-			if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
-				control |= DW_EDMA_V0_RIE;
-		}
+	struct dw_edma *dw = chan->dw;
+	unsigned long flags;
+	u32 tmp;
 
-		dw_edma_v0_write_ll_data(chunk, i++, control, child->sz,
-					 child->sar, child->dar);
+	 /* Enable engine */
+	SET_RW_32(dw, chan->dir, engine_en, BIT(0));
+	if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) {
+		switch (chan->id) {
+		case 0:
+		SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en, BIT(0));
+			break;
+		case 1:
+			SET_RW_COMPAT(dw, chan->dir, ch1_pwr_en, BIT(0));
+			break;
+		case 2:
+			SET_RW_COMPAT(dw, chan->dir, ch2_pwr_en, BIT(0));
+			break;
+		case 3:
+			SET_RW_COMPAT(dw, chan->dir, ch3_pwr_en, BIT(0));
+			break;
+		case 4:
+			SET_RW_COMPAT(dw, chan->dir, ch4_pwr_en, BIT(0));
+			break;
+		case 5:
+			SET_RW_COMPAT(dw, chan->dir, ch5_pwr_en, BIT(0));
+			break;
+		case 6:
+			SET_RW_COMPAT(dw, chan->dir, ch6_pwr_en, BIT(0));
+			break;
+		case 7:
+			SET_RW_COMPAT(dw, chan->dir, ch7_pwr_en, BIT(0));
+			break;
+		}
 	}
-
-	control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB;
-	if (!chunk->cb)
-		control |= DW_EDMA_V0_CB;
-
-	dw_edma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr);
+	/* Interrupt unmask - done, abort */
+	raw_spin_lock_irqsave(&dw->lock, flags);
+
+	tmp = GET_RW_32(dw, chan->dir, int_mask);
+	tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id));
+	tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id));
+	SET_RW_32(dw, chan->dir, int_mask, tmp);
+	/* Linked list error */
+	tmp = GET_RW_32(dw, chan->dir, linked_list_err_en);
+	tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id));
+	SET_RW_32(dw, chan->dir, linked_list_err_en, tmp);
+
+	raw_spin_unlock_irqrestore(&dw->lock, flags);
+
+	/* Channel control */
+	SET_CH_32(dw, chan->dir, chan->id, ch_control1,
+		  (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE));
+	/* Linked list */
+	/* llp is not aligned on 64bit -> keep 32bit accesses */
+	SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
+		  lower_32_bits(chan->ll_region.paddr));
+	SET_CH_32(dw, chan->dir, chan->id, llp.msb,
+		  upper_32_bits(chan->ll_region.paddr));
 }
 
-static void dw_edma_v0_sync_ll_data(struct dw_edma_chunk *chunk)
+static void dw_edma_v0_sync_ll_data(struct dw_edma_chan *chan)
 {
 	/*
 	 * In case of remote eDMA engine setup, the DW PCIe RP/EP internal
@@ -356,88 +389,8 @@ static void dw_edma_v0_sync_ll_data(struct dw_edma_chunk *chunk)
 	 * LL memory in a hope that the MRd TLP will return only after the
 	 * last MWr TLP is completed
 	 */
-	if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
-		readl(chunk->ll_region.vaddr.io);
-}
-
-static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
-{
-	struct dw_edma_chan *chan = chunk->chan;
-	struct dw_edma *dw = chan->dw;
-	unsigned long flags;
-	u32 tmp;
-
-	dw_edma_v0_core_write_chunk(chunk);
-
-	if (first) {
-		/* Enable engine */
-		SET_RW_32(dw, chan->dir, engine_en, BIT(0));
-		if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) {
-			switch (chan->id) {
-			case 0:
-				SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en,
-					      BIT(0));
-				break;
-			case 1:
-				SET_RW_COMPAT(dw, chan->dir, ch1_pwr_en,
-					      BIT(0));
-				break;
-			case 2:
-				SET_RW_COMPAT(dw, chan->dir, ch2_pwr_en,
-					      BIT(0));
-				break;
-			case 3:
-				SET_RW_COMPAT(dw, chan->dir, ch3_pwr_en,
-					      BIT(0));
-				break;
-			case 4:
-				SET_RW_COMPAT(dw, chan->dir, ch4_pwr_en,
-					      BIT(0));
-				break;
-			case 5:
-				SET_RW_COMPAT(dw, chan->dir, ch5_pwr_en,
-					      BIT(0));
-				break;
-			case 6:
-				SET_RW_COMPAT(dw, chan->dir, ch6_pwr_en,
-					      BIT(0));
-				break;
-			case 7:
-				SET_RW_COMPAT(dw, chan->dir, ch7_pwr_en,
-					      BIT(0));
-				break;
-			}
-		}
-		/* Interrupt unmask - done, abort */
-		raw_spin_lock_irqsave(&dw->lock, flags);
-
-		tmp = GET_RW_32(dw, chan->dir, int_mask);
-		tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id));
-		tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id));
-		SET_RW_32(dw, chan->dir, int_mask, tmp);
-		/* Linked list error */
-		tmp = GET_RW_32(dw, chan->dir, linked_list_err_en);
-		tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id));
-		SET_RW_32(dw, chan->dir, linked_list_err_en, tmp);
-
-		raw_spin_unlock_irqrestore(&dw->lock, flags);
-
-		/* Channel control */
-		SET_CH_32(dw, chan->dir, chan->id, ch_control1,
-			  (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE));
-		/* Linked list */
-		/* llp is not aligned on 64bit -> keep 32bit accesses */
-		SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
-			  lower_32_bits(chunk->ll_region.paddr));
-		SET_CH_32(dw, chan->dir, chan->id, llp.msb,
-			  upper_32_bits(chunk->ll_region.paddr));
-	}
-
-	dw_edma_v0_sync_ll_data(chunk);
-
-	/* Doorbell */
-	SET_RW_32(dw, chan->dir, doorbell,
-		  FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id));
+	if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
+		readl(chan->ll_region.vaddr.io);
 }
 
 static void dw_edma_v0_core_ch_config(struct dw_edma_chan *chan)
@@ -509,6 +462,48 @@ static void dw_edma_v0_core_ch_config(struct dw_edma_chan *chan)
 	}
 }
 
+static void
+dw_edma_v0_core_ll_data(struct dw_edma_chan *chan, struct dw_edma_burst *burst,
+			u32 idx, bool cb, bool irq)
+{
+	u32 control = 0;
+
+	if (cb)
+		control |= DW_EDMA_V0_CB;
+
+	if (irq) {
+		control |= DW_EDMA_V0_LIE;
+
+		if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
+			control |= DW_EDMA_V0_RIE;
+	}
+
+	dw_edma_v0_write_ll_data(chan, idx, control, burst->sz, burst->sar,
+				 burst->dar);
+}
+
+static void
+dw_edma_v0_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr)
+{
+	u32 control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB;
+
+	if (!cb)
+		control |= DW_EDMA_V0_CB;
+
+	dw_edma_v0_write_ll_link(chan, idx, control, addr);
+}
+
+static void dw_edma_v0_core_ch_doorbell(struct dw_edma_chan *chan)
+{
+	struct dw_edma *dw = chan->dw;
+
+	dw_edma_v0_sync_ll_data(chan);
+
+	/* Doorbell */
+	SET_RW_32(dw, chan->dir, doorbell,
+		  FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id));
+}
+
 /* eDMA debugfs callbacks */
 static void dw_edma_v0_core_debugfs_on(struct dw_edma *dw)
 {
@@ -539,7 +534,10 @@ static const struct dw_edma_core_ops dw_edma_v0_core = {
 	.ch_count = dw_edma_v0_core_ch_count,
 	.ch_status = dw_edma_v0_core_ch_status,
 	.handle_int = dw_edma_v0_core_handle_int,
-	.start = dw_edma_v0_core_start,
+	.ll_data = dw_edma_v0_core_ll_data,
+	.ll_link = dw_edma_v0_core_ll_link,
+	.ch_doorbell = dw_edma_v0_core_ch_doorbell,
+	.ch_enable = dw_edma_v0_core_ch_enable,
 	.ch_config = dw_edma_v0_core_ch_config,
 	.debugfs_on = dw_edma_v0_core_debugfs_on,
 	.ack_emulated_irq = dw_edma_v0_core_ack_emulated_irq,
diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c
index 632abb8b481cf..87ba3e8c9fa19 100644
--- a/drivers/dma/dw-edma/dw-hdma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c
@@ -53,13 +53,24 @@ __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)
 static void dw_hdma_v0_core_off(struct dw_edma *dw)
 {
 	int id;
+	enum dw_edma_dir dir;
+
+	dir = EDMA_DIR_WRITE;
+	for (id = 0; id < dw->wr_ch_cnt; id++) {
+		SET_CH_32(dw, dir, id, int_setup,
+			  HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+		SET_CH_32(dw, dir, id, int_clear,
+			  HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+		SET_CH_32(dw, dir, id, ch_en, 0);
+	}
 
-	for (id = 0; id < HDMA_V0_MAX_NR_CH; id++) {
-		SET_BOTH_CH_32(dw, id, int_setup,
-			       HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
-		SET_BOTH_CH_32(dw, id, int_clear,
-			       HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
-		SET_BOTH_CH_32(dw, id, ch_en, 0);
+	dir = EDMA_DIR_READ;
+	for (id = 0; id < dw->rd_ch_cnt; id++) {
+		SET_CH_32(dw, dir, id, int_setup,
+			  HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+		SET_CH_32(dw, dir, id, int_clear,
+			  HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+		SET_CH_32(dw, dir, id, ch_en, 0);
 	}
 }
 
@@ -118,7 +129,7 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 	unsigned long total, pos, val;
 	irqreturn_t ret = IRQ_NONE;
 	struct dw_edma_chan *chan;
-	unsigned long off, mask;
+	unsigned long off, *mask;
 
 	if (dir == EDMA_DIR_WRITE) {
 		total = dw->wr_ch_cnt;
@@ -130,7 +141,7 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 		mask = dw_irq->rd_mask;
 	}
 
-	for_each_set_bit(pos, &mask, total) {
+	for_each_set_bit(pos, mask, total) {
 		chan = &dw->chan[pos + off];
 
 		val = dw_hdma_v0_core_status_int(chan);
@@ -152,66 +163,77 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 	return ret;
 }
 
-static void dw_hdma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i,
+static void dw_hdma_v0_write_ll_data(struct dw_edma_chan *chan, int i,
 				     u32 control, u32 size, u64 sar, u64 dar)
 {
 	ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli);
 
-	if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
-		struct dw_hdma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs;
+	if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
+		struct dw_hdma_v0_lli *lli = chan->ll_region.vaddr.mem + ofs;
 
-		lli->control = control;
 		lli->transfer_size = size;
 		lli->sar.reg = sar;
 		lli->dar.reg = dar;
+		dma_wmb();
+		lli->control = control;
 	} else {
-		struct dw_hdma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs;
+		struct dw_hdma_v0_lli __iomem *lli = chan->ll_region.vaddr.io + ofs;
 
-		writel(control, &lli->control);
 		writel(size, &lli->transfer_size);
 		writeq(sar, &lli->sar.reg);
 		writeq(dar, &lli->dar.reg);
+		writel(control, &lli->control);
 	}
 }
 
-static void dw_hdma_v0_write_ll_link(struct dw_edma_chunk *chunk,
+static void dw_hdma_v0_write_ll_link(struct dw_edma_chan *chan,
 				     int i, u32 control, u64 pointer)
 {
 	ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli);
 
-	if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
-		struct dw_hdma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs;
+	if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
+		struct dw_hdma_v0_llp *llp = chan->ll_region.vaddr.mem + ofs;
 
-		llp->control = control;
 		llp->llp.reg = pointer;
+		dma_wmb();
+		llp->control = control;
 	} else {
-		struct dw_hdma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs;
+		struct dw_hdma_v0_llp __iomem *llp = chan->ll_region.vaddr.io + ofs;
 
-		writel(control, &llp->control);
 		writeq(pointer, &llp->llp.reg);
+		writel(control, &llp->control);
 	}
 }
 
-static void dw_hdma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
+static void dw_hdma_v0_core_ch_enable(struct dw_edma_chan *chan)
 {
-	struct dw_edma_burst *child;
-	u32 control = 0, i = 0;
-
-	if (chunk->cb)
-		control = DW_HDMA_V0_CB;
-
-	list_for_each_entry(child, &chunk->burst->list, list)
-		dw_hdma_v0_write_ll_data(chunk, i++, control, child->sz,
-					 child->sar, child->dar);
-
-	control = DW_HDMA_V0_LLP | DW_HDMA_V0_TCB;
-	if (!chunk->cb)
-		control |= DW_HDMA_V0_CB;
+	struct dw_edma *dw = chan->dw;
+	u32 tmp;
 
-	dw_hdma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr);
+	/* Enable engine */
+	SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0));
+	/* Interrupt unmask - stop, abort */
+	tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup);
+	tmp &= ~(HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
+	/* Interrupt enable - stop, abort */
+	tmp |= HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_ABORT_INT_EN;
+	if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL))
+		tmp |= HDMA_V0_REMOTE_STOP_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN;
+	SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp);
+	/* Channel control */
+	SET_CH_32(dw, chan->dir, chan->id, control1, HDMA_V0_LINKLIST_EN);
+	/* Linked list */
+	/* llp is not aligned on 64bit -> keep 32bit accesses */
+	SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
+		  lower_32_bits(chan->ll_region.paddr));
+	SET_CH_32(dw, chan->dir, chan->id, llp.msb,
+		  upper_32_bits(chan->ll_region.paddr));
+	/* Set consumer cycle */
+	SET_CH_32(dw, chan->dir, chan->id, cycle_sync,
+		  HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT);
 }
 
-static void dw_hdma_v0_sync_ll_data(struct dw_edma_chunk *chunk)
+static void dw_hdma_v0_sync_ll_data(struct dw_edma_chan *chan)
 {
 	/*
 	 * In case of remote HDMA engine setup, the DW PCIe RP/EP internal
@@ -221,60 +243,16 @@ static void dw_hdma_v0_sync_ll_data(struct dw_edma_chunk *chunk)
 	 * LL memory in a hope that the MRd TLP will return only after the
 	 * last MWr TLP is completed
 	 */
-	if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
-		readl(chunk->ll_region.vaddr.io);
-}
-
-static void dw_hdma_v0_core_ll_start(struct dw_edma_chunk *chunk, bool first)
-{
-	struct dw_edma_chan *chan = chunk->chan;
-	struct dw_edma *dw = chan->dw;
-	u32 tmp;
-
-	dw_hdma_v0_core_write_chunk(chunk);
-
-	if (first) {
-		/* Enable engine */
-		SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0));
-		/* Interrupt unmask - stop, abort */
-		tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup);
-		tmp &= ~(HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
-		/* Interrupt enable - stop, abort */
-		tmp |= HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_ABORT_INT_EN;
-		if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL))
-			tmp |= HDMA_V0_REMOTE_STOP_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN;
-		SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp);
-		/* Channel control */
-		SET_CH_32(dw, chan->dir, chan->id, control1, HDMA_V0_LINKLIST_EN);
-		/* Linked list */
-		/* llp is not aligned on 64bit -> keep 32bit accesses */
-		SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
-			  lower_32_bits(chunk->ll_region.paddr));
-		SET_CH_32(dw, chan->dir, chan->id, llp.msb,
-			  upper_32_bits(chunk->ll_region.paddr));
-		/* Set consumer cycle */
-		SET_CH_32(dw, chan->dir, chan->id, cycle_sync,
-			HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT);
-	}
-
-	dw_hdma_v0_sync_ll_data(chunk);
-
-	/* Doorbell */
-	SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START);
+	if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
+		readl(chan->ll_region.vaddr.io);
 }
 
-static void dw_hdma_v0_core_non_ll_start(struct dw_edma_chunk *chunk)
+static void dw_hdma_v0_core_non_ll_start(struct dw_edma_chan *chan,
+					 struct dw_edma_burst *child)
 {
-	struct dw_edma_chan *chan = chunk->chan;
 	struct dw_edma *dw = chan->dw;
-	struct dw_edma_burst *child;
 	u32 val;
 
-	child = list_first_entry_or_null(&chunk->burst->list,
-					 struct dw_edma_burst, list);
-	if (!child)
-		return;
-
 	SET_CH_32(dw, chan->dir, chan->id, ch_en, HDMA_V0_CH_EN);
 
 	/* Source address */
@@ -315,16 +293,6 @@ static void dw_hdma_v0_core_non_ll_start(struct dw_edma_chunk *chunk)
 		  HDMA_V0_DOORBELL_START);
 }
 
-static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
-{
-	struct dw_edma_chan *chan = chunk->chan;
-
-	if (chan->non_ll)
-		dw_hdma_v0_core_non_ll_start(chunk);
-	else
-		dw_hdma_v0_core_ll_start(chunk, first);
-}
-
 static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan)
 {
 	struct dw_edma *dw = chan->dw;
@@ -339,6 +307,40 @@ static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan)
 	SET_CH_32(dw, chan->dir, chan->id, msi_msgdata, chan->msi.data);
 }
 
+static void
+dw_hdma_v0_core_ll_data(struct dw_edma_chan *chan, struct dw_edma_burst *burst,
+			u32 idx, bool cb, bool irq)
+{
+	u32 control = 0;
+
+	if (cb)
+		control |= DW_HDMA_V0_CB;
+
+	dw_hdma_v0_write_ll_data(chan, idx, control, burst->sz, burst->sar,
+				 burst->dar);
+}
+
+static void
+dw_hdma_v0_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr)
+{
+	u32 control = DW_HDMA_V0_LLP | DW_HDMA_V0_TCB;
+
+	if (!cb)
+		control |= DW_HDMA_V0_CB;
+
+	dw_hdma_v0_write_ll_link(chan, idx, control, addr);
+}
+
+static void dw_hdma_v0_core_ch_doorbell(struct dw_edma_chan *chan)
+{
+	struct dw_edma *dw = chan->dw;
+
+	dw_hdma_v0_sync_ll_data(chan);
+
+	/* Doorbell */
+	SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START);
+}
+
 /* HDMA debugfs callbacks */
 static void dw_hdma_v0_core_debugfs_on(struct dw_edma *dw)
 {
@@ -356,7 +358,11 @@ static const struct dw_edma_core_ops dw_hdma_v0_core = {
 	.ch_count = dw_hdma_v0_core_ch_count,
 	.ch_status = dw_hdma_v0_core_ch_status,
 	.handle_int = dw_hdma_v0_core_handle_int,
-	.start = dw_hdma_v0_core_start,
+	.non_ll_start = dw_hdma_v0_core_non_ll_start,
+	.ll_data = dw_hdma_v0_core_ll_data,
+	.ll_link = dw_hdma_v0_core_ll_link,
+	.ch_doorbell = dw_hdma_v0_core_ch_doorbell,
+	.ch_enable = dw_hdma_v0_core_ch_enable,
 	.ch_config = dw_hdma_v0_core_ch_config,
 	.debugfs_on = dw_hdma_v0_core_debugfs_on,
 	.db_offset = dw_hdma_v0_core_db_offset,
diff --git a/drivers/dma/dw-edma/dw-hdma-v0-regs.h b/drivers/dma/dw-edma/dw-hdma-v0-regs.h
index 7759ba9b4850f..48e40efceb2ea 100644
--- a/drivers/dma/dw-edma/dw-hdma-v0-regs.h
+++ b/drivers/dma/dw-edma/dw-hdma-v0-regs.h
@@ -11,7 +11,7 @@
 
 #include <linux/dmaengine.h>
 
-#define HDMA_V0_MAX_NR_CH			8
+#define HDMA_V0_MAX_NR_CH			64
 #define HDMA_V0_CH_EN				BIT(0)
 #define HDMA_V0_LOCAL_ABORT_INT_EN		BIT(6)
 #define HDMA_V0_REMOTE_ABORT_INT_EN		BIT(5)
diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c
index 36155ab1602a9..d9fb717b5b53c 100644
--- a/drivers/dma/fsl-edma-main.c
+++ b/drivers/dma/fsl-edma-main.c
@@ -414,6 +414,8 @@ static int fsl_edma3_irq_init(struct platform_device *pdev, struct fsl_edma_engi
 
 		errirq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s-err",
 					     dev_name(&pdev->dev));
+		if (!errirq_name)
+			return -ENOMEM;
 
 		ret = devm_request_irq(&pdev->dev, fsl_edma->errirq, fsl_edma3_err_handler_shared,
 				       0, errirq_name, fsl_edma);
diff --git a/drivers/dma/fsl_raid.c b/drivers/dma/fsl_raid.c
index 99945845d8b5e..2d86f61105e52 100644
--- a/drivers/dma/fsl_raid.c
+++ b/drivers/dma/fsl_raid.c
@@ -242,9 +242,9 @@ static void fill_cfd_frame(struct fsl_re_cmpnd_frame *cf, u8 index,
 	u32 efrl = length & FSL_RE_CF_LENGTH_MASK;
 
 	efrl |= final << FSL_RE_CF_FINAL_SHIFT;
-	cf[index].efrl32 = efrl;
-	cf[index].addr_high = upper_32_bits(addr);
-	cf[index].addr_low = lower_32_bits(addr);
+	cf[index].efrl32 = cpu_to_be32(efrl);
+	cf[index].addr_high = cpu_to_be32(upper_32_bits(addr));
+	cf[index].addr_low = cpu_to_be32(lower_32_bits(addr));
 }
 
 static struct fsl_re_desc *fsl_re_init_desc(struct fsl_re_chan *re_chan,
@@ -256,9 +256,10 @@ static struct fsl_re_desc *fsl_re_init_desc(struct fsl_re_chan *re_chan,
 	dma_async_tx_descriptor_init(&desc->async_tx, &re_chan->chan);
 	INIT_LIST_HEAD(&desc->node);
 
-	desc->hwdesc.fmt32 = FSL_RE_FRAME_FORMAT << FSL_RE_HWDESC_FMT_SHIFT;
-	desc->hwdesc.lbea32 = upper_32_bits(paddr);
-	desc->hwdesc.addr_low = lower_32_bits(paddr);
+	desc->hwdesc.fmt32 = cpu_to_be32(FSL_RE_FRAME_FORMAT <<
+					  FSL_RE_HWDESC_FMT_SHIFT);
+	desc->hwdesc.lbea32 = cpu_to_be32(upper_32_bits(paddr));
+	desc->hwdesc.addr_low = cpu_to_be32(lower_32_bits(paddr));
 	desc->cf_addr = cf;
 	desc->cf_paddr = paddr;
 
@@ -374,11 +375,11 @@ static struct dma_async_tx_descriptor *fsl_re_prep_dma_genq(
 	for (i = 2, j = 0; j < save_src_cnt; i++, j++)
 		fill_cfd_frame(cf, i, len, src[j], 0);
 
+	/* Fill the last frame and mark it final */
 	if (cont_q)
-		fill_cfd_frame(cf, i++, len, dest, 0);
-
-	/* Setting the final bit in the last source buffer frame in CFD */
-	cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT;
+		fill_cfd_frame(cf, i, len, dest, 1);
+	else
+		fill_cfd_frame(cf, i - 1, len, src[j - 1], 1);
 
 	return &desc->async_tx;
 }
@@ -504,16 +505,16 @@ static struct dma_async_tx_descriptor *fsl_re_prep_dma_pq(
 			p[save_src_cnt + 2] = 1;
 			fill_cfd_frame(cf, i++, len, dest[0], 0);
 			fill_cfd_frame(cf, i++, len, dest[1], 0);
-			fill_cfd_frame(cf, i++, len, dest[1], 0);
+			fill_cfd_frame(cf, i++, len, dest[1], 1);
 		} else {
 			dev_err(re_chan->dev, "PQ tx continuation error!\n");
 			return NULL;
 		}
+	} else {
+		/* Mark the last source buffer frame final */
+		fill_cfd_frame(cf, i - 1, len, src[j - 1], 1);
 	}
 
-	/* Setting the final bit in the last source buffer frame in CFD */
-	cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT;
-
 	return &desc->async_tx;
 }
 
@@ -656,8 +657,7 @@ static int fsl_re_chan_probe(struct platform_device *ofdev,
 		goto err_free;
 	}
 
-	chan->jrregs = (struct fsl_re_chan_cfg *)((u8 *)re_priv->re_regs +
-			off + ptr);
+	chan->jrregs = re_priv->base + off + ptr;
 
 	/* read irq property from dts */
 	chan->irq = irq_of_parse_and_map(np, 0);
@@ -745,38 +745,36 @@ static int fsl_re_chan_probe(struct platform_device *ofdev,
 /* Probe function for RAID Engine */
 static int fsl_re_probe(struct platform_device *ofdev)
 {
+	struct fsl_re_ctrl __iomem *re_regs;
 	struct fsl_re_drv_private *re_priv;
 	struct device_node *child;
 	u32 off;
 	u8 ridx = 0;
 	struct dma_device *dma_dev;
-	struct resource *res;
 	int rc;
 	struct device *dev = &ofdev->dev;
 
+	/* IOMAP the entire RAID Engine region */
+	re_regs = devm_platform_ioremap_resource(ofdev, 0);
+	if (IS_ERR(re_regs))
+		return PTR_ERR(re_regs);
+
 	re_priv = devm_kzalloc(dev, sizeof(*re_priv), GFP_KERNEL);
 	if (!re_priv)
 		return -ENOMEM;
 
-	res = platform_get_resource(ofdev, IORESOURCE_MEM, 0);
-	if (!res)
-		return -ENODEV;
-
-	/* IOMAP the entire RAID Engine region */
-	re_priv->re_regs = devm_ioremap(dev, res->start, resource_size(res));
-	if (!re_priv->re_regs)
-		return -EBUSY;
+	re_priv->base = re_regs;
 
 	/* Program the RE mode */
-	out_be32(&re_priv->re_regs->global_config, FSL_RE_NON_DPAA_MODE);
+	out_be32(&re_regs->global_config, FSL_RE_NON_DPAA_MODE);
 
 	/* Program Galois Field polynomial */
-	out_be32(&re_priv->re_regs->galois_field_config, FSL_RE_GFM_POLY);
+	out_be32(&re_regs->galois_field_config, FSL_RE_GFM_POLY);
 
 	dev_info(dev, "version %x, mode %x, gfp %x\n",
-		 in_be32(&re_priv->re_regs->re_version_id),
-		 in_be32(&re_priv->re_regs->global_config),
-		 in_be32(&re_priv->re_regs->galois_field_config));
+		 in_be32(&re_regs->re_version_id),
+		 in_be32(&re_regs->global_config),
+		 in_be32(&re_regs->galois_field_config));
 
 	dma_dev = &re_priv->dma_dev;
 	dma_dev->dev = dev;
diff --git a/drivers/dma/fsl_raid.h b/drivers/dma/fsl_raid.h
index 69d743c04973a..adbfede330a77 100644
--- a/drivers/dma/fsl_raid.h
+++ b/drivers/dma/fsl_raid.h
@@ -256,7 +256,7 @@ struct fsl_re_hw_desc {
 struct fsl_re_drv_private {
 	u8 total_chans;
 	struct dma_device dma_dev;
-	struct fsl_re_ctrl *re_regs;
+	void __iomem *base;
 	struct fsl_re_chan *re_jrs[FSL_RE_MAX_CHANS];
 	struct dma_pool *cf_desc_pool;
 	struct dma_pool *hw_desc_pool;
@@ -273,7 +273,7 @@ struct fsl_re_chan {
 	struct device *dev;
 	struct fsl_re_drv_private *re_dev;
 	struct dma_chan chan;
-	struct fsl_re_chan_cfg *jrregs;
+	struct fsl_re_chan_cfg __iomem *jrregs;
 	int irq;
 	struct tasklet_struct irqtask;
 	u32 alloc_count;
diff --git a/drivers/dma/hisi_dma.c b/drivers/dma/hisi_dma.c
index 28bf818f9aa63..c751a2e49e6dc 100644
--- a/drivers/dma/hisi_dma.c
+++ b/drivers/dma/hisi_dma.c
@@ -983,7 +983,7 @@ static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, chan_num),
 				GFP_KERNEL);
 	if (!hdma_dev)
-		return -EINVAL;
+		return -ENOMEM;
 
 	hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2];
 	hdma_dev->pdev = pdev;
diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c
index 0366c7cf35020..82b07cf942ef8 100644
--- a/drivers/dma/idxd/cdev.c
+++ b/drivers/dma/idxd/cdev.c
@@ -288,6 +288,7 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp)
 	fdev->parent = cdev_dev(idxd_cdev);
 	fdev->bus = &dsa_bus_type;
 	fdev->type = &idxd_cdev_file_type;
+	idxd_wq_get(wq);
 
 	rc = dev_set_name(fdev, "file%d", ctx->id);
 	if (rc < 0) {
@@ -301,13 +302,14 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp)
 		goto failed_dev_add;
 	}
 
-	idxd_wq_get(wq);
 	mutex_unlock(&wq->wq_lock);
 	return 0;
 
 failed_dev_add:
 failed_dev_name:
+	mutex_unlock(&wq->wq_lock);
 	put_device(fdev);
+	return rc;
 failed_ida:
 failed_set_pasid:
 	if (device_user_pasid_enabled(idxd))
diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c
index f1cfc7790d950..4b827a3297564 100644
--- a/drivers/dma/idxd/init.c
+++ b/drivers/dma/idxd/init.c
@@ -159,18 +159,12 @@ static void idxd_cleanup_interrupts(struct idxd_device *idxd)
 
 static void idxd_clean_wqs(struct idxd_device *idxd)
 {
-	struct idxd_wq *wq;
 	struct device *conf_dev;
 	int i;
 
 	for (i = 0; i < idxd->max_wqs; i++) {
-		wq = idxd->wqs[i];
-		if (idxd->hw.wq_cap.op_config)
-			bitmap_free(wq->opcap_bmap);
-		kfree(wq->wqcfg);
-		conf_dev = wq_confdev(wq);
+		conf_dev = wq_confdev(idxd->wqs[i]);
 		put_device(conf_dev);
-		kfree(wq);
 	}
 	bitmap_free(idxd->wq_enable_map);
 	kfree(idxd->wqs);
@@ -212,7 +206,6 @@ static int idxd_setup_wqs(struct idxd_device *idxd)
 		rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id);
 		if (rc < 0) {
 			put_device(conf_dev);
-			kfree(wq);
 			goto err_unwind;
 		}
 
@@ -227,7 +220,6 @@ static int idxd_setup_wqs(struct idxd_device *idxd)
 		wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev));
 		if (!wq->wqcfg) {
 			put_device(conf_dev);
-			kfree(wq);
 			rc = -ENOMEM;
 			goto err_unwind;
 		}
@@ -235,9 +227,7 @@ static int idxd_setup_wqs(struct idxd_device *idxd)
 		if (idxd->hw.wq_cap.op_config) {
 			wq->opcap_bmap = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL);
 			if (!wq->opcap_bmap) {
-				kfree(wq->wqcfg);
 				put_device(conf_dev);
-				kfree(wq);
 				rc = -ENOMEM;
 				goto err_unwind;
 			}
@@ -252,13 +242,8 @@ static int idxd_setup_wqs(struct idxd_device *idxd)
 
 err_unwind:
 	while (--i >= 0) {
-		wq = idxd->wqs[i];
-		if (idxd->hw.wq_cap.op_config)
-			bitmap_free(wq->opcap_bmap);
-		kfree(wq->wqcfg);
-		conf_dev = wq_confdev(wq);
+		conf_dev = wq_confdev(idxd->wqs[i]);
 		put_device(conf_dev);
-		kfree(wq);
 	}
 	bitmap_free(idxd->wq_enable_map);
 
@@ -270,15 +255,12 @@ static int idxd_setup_wqs(struct idxd_device *idxd)
 
 static void idxd_clean_engines(struct idxd_device *idxd)
 {
-	struct idxd_engine *engine;
 	struct device *conf_dev;
 	int i;
 
 	for (i = 0; i < idxd->max_engines; i++) {
-		engine = idxd->engines[i];
-		conf_dev = engine_confdev(engine);
+		conf_dev = engine_confdev(idxd->engines[i]);
 		put_device(conf_dev);
-		kfree(engine);
 	}
 	kfree(idxd->engines);
 }
@@ -313,7 +295,6 @@ static int idxd_setup_engines(struct idxd_device *idxd)
 		rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id);
 		if (rc < 0) {
 			put_device(conf_dev);
-			kfree(engine);
 			goto err;
 		}
 
@@ -324,10 +305,8 @@ static int idxd_setup_engines(struct idxd_device *idxd)
 
  err:
 	while (--i >= 0) {
-		engine = idxd->engines[i];
-		conf_dev = engine_confdev(engine);
+		conf_dev = engine_confdev(idxd->engines[i]);
 		put_device(conf_dev);
-		kfree(engine);
 	}
 	kfree(idxd->engines);
 
@@ -336,13 +315,10 @@ static int idxd_setup_engines(struct idxd_device *idxd)
 
 static void idxd_clean_groups(struct idxd_device *idxd)
 {
-	struct idxd_group *group;
 	int i;
 
 	for (i = 0; i < idxd->max_groups; i++) {
-		group = idxd->groups[i];
-		put_device(group_confdev(group));
-		kfree(group);
+		put_device(group_confdev(idxd->groups[i]));
 	}
 	kfree(idxd->groups);
 }
@@ -377,7 +353,6 @@ static int idxd_setup_groups(struct idxd_device *idxd)
 		rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id);
 		if (rc < 0) {
 			put_device(conf_dev);
-			kfree(group);
 			goto err;
 		}
 
@@ -402,7 +377,6 @@ static int idxd_setup_groups(struct idxd_device *idxd)
 	while (--i >= 0) {
 		group = idxd->groups[i];
 		put_device(group_confdev(group));
-		kfree(group);
 	}
 	kfree(idxd->groups);
 
diff --git a/drivers/dma/mediatek/mtk-uart-apdma.c b/drivers/dma/mediatek/mtk-uart-apdma.c
index c269d84d7bd2b..f74e9a3285887 100644
--- a/drivers/dma/mediatek/mtk-uart-apdma.c
+++ b/drivers/dma/mediatek/mtk-uart-apdma.c
@@ -531,7 +531,7 @@ static int mtk_uart_apdma_probe(struct platform_device *pdev)
 	for (i = 0; i < mtkd->dma_requests; i++) {
 		c = devm_kzalloc(mtkd->ddev.dev, sizeof(*c), GFP_KERNEL);
 		if (!c) {
-			rc = -ENODEV;
+			rc = -ENOMEM;
 			goto err_no_dma;
 		}
 
diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c
index 25ba84b187041..6214d9000db8e 100644
--- a/drivers/dma/pl330.c
+++ b/drivers/dma/pl330.c
@@ -502,6 +502,7 @@ struct pl330_dmac {
 	struct dma_pl330_chan *peripherals; /* keep at end */
 	int quirks;
 
+	struct dentry		*dbgfs;
 	struct reset_control	*rstc;
 	struct reset_control	*rstc_ocp;
 };
@@ -2952,14 +2953,24 @@ DEFINE_SHOW_ATTRIBUTE(pl330_debugfs);
 
 static inline void init_pl330_debugfs(struct pl330_dmac *pl330)
 {
-	debugfs_create_file(dev_name(pl330->ddma.dev),
-			    S_IFREG | 0444, NULL, pl330,
-			    &pl330_debugfs_fops);
+	pl330->dbgfs = debugfs_create_file(dev_name(pl330->ddma.dev),
+					   S_IFREG | 0444, NULL, pl330,
+					   &pl330_debugfs_fops);
+}
+
+static inline void deinit_pl330_debugfs(struct pl330_dmac *pl330)
+{
+	debugfs_remove(pl330->dbgfs);
+	pl330->dbgfs = NULL;
 }
 #else
 static inline void init_pl330_debugfs(struct pl330_dmac *pl330)
 {
 }
+
+static inline void deinit_pl330_debugfs(struct pl330_dmac *pl330)
+{
+}
 #endif
 
 /*
@@ -3204,6 +3215,8 @@ static void pl330_remove(struct amba_device *adev)
 	struct dma_pl330_chan *pch, *_p;
 	int i, irq;
 
+	deinit_pl330_debugfs(pl330);
+
 	pm_runtime_get_noresume(pl330->ddma.dev);
 
 	if (adev->dev.of_node)
diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c
index a9a254dbf8cb9..f47a326dd7ffa 100644
--- a/drivers/dma/sun6i-dma.c
+++ b/drivers/dma/sun6i-dma.c
@@ -945,16 +945,13 @@ static int sun6i_dma_terminate_all(struct dma_chan *chan)
 
 	spin_lock_irqsave(&vchan->vc.lock, flags);
 
-	if (vchan->cyclic) {
-		vchan->cyclic = false;
-		if (pchan && pchan->desc) {
-			struct virt_dma_desc *vd = &pchan->desc->vd;
-			struct virt_dma_chan *vc = &vchan->vc;
+	if (pchan && pchan->desc && pchan->desc != pchan->done) {
+		struct virt_dma_desc *vd = &pchan->desc->vd;
 
-			list_add_tail(&vd->node, &vc->desc_completed);
-		}
+		vchan_terminate_vdesc(vd);
 	}
 
+	vchan->cyclic = false;
 	vchan_get_all_descriptors(&vchan->vc, &head);
 
 	if (pchan) {
diff --git a/drivers/dma/switchtec_dma.c b/drivers/dma/switchtec_dma.c
index 3ef9286406159..71d9868ce613e 100644
--- a/drivers/dma/switchtec_dma.c
+++ b/drivers/dma/switchtec_dma.c
@@ -1099,7 +1099,7 @@ static int switchtec_dma_chan_init(struct switchtec_dma_dev *swdma_dev,
 	dev_dbg(&pdev->dev, "Channel %d: SE buffer count %d\n", i, se_buf_len);
 
 	thresh = se_buf_len / 2;
-	valid_en_se |= FIELD_GET(SE_THRESH_MASK, thresh);
+	valid_en_se |= FIELD_PREP(SE_THRESH_MASK, thresh);
 	writel(valid_en_se, &swdma_chan->mmio_chan_fw->valid_en_se);
 
 	/* request irqs */
diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c
index ceaee1e33e68a..21a381d022cfb 100644
--- a/drivers/dma/tegra210-adma.c
+++ b/drivers/dma/tegra210-adma.c
@@ -1087,15 +1087,9 @@ static int tegra_adma_probe(struct platform_device *pdev)
 		}
 	} else {
 		/* If no 'page' property found, then reg DT binding would be legacy */
-		res_base = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-		if (res_base) {
-			tdma->base_addr = devm_ioremap_resource(&pdev->dev, res_base);
-			if (IS_ERR(tdma->base_addr))
-				return PTR_ERR(tdma->base_addr);
-		} else {
-			return dev_err_probe(&pdev->dev, -ENODEV,
-					     "failed to get memory resource\n");
-		}
+		tdma->base_addr = devm_platform_ioremap_resource(pdev, 0);
+		if (IS_ERR(tdma->base_addr))
+			return PTR_ERR(tdma->base_addr);
 
 		tdma->ch_base_addr = tdma->base_addr + cdata->ch_base_offset;
 	}
diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c
index 1cf158eb7bdb5..fb21e0df5ab7b 100644
--- a/drivers/dma/ti/k3-udma.c
+++ b/drivers/dma/ti/k3-udma.c
@@ -3408,7 +3408,7 @@ static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc,
 	return 0;
 }
 
-static struct dma_descriptor_metadata_ops metadata_ops = {
+static const struct dma_descriptor_metadata_ops metadata_ops = {
 	.attach = udma_attach_metadata,
 	.get_ptr = udma_get_metadata_ptr,
 	.set_len = udma_set_metadata_len,
diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c
index 404235c173538..bef2b031dba19 100644
--- a/drivers/dma/xilinx/xilinx_dma.c
+++ b/drivers/dma/xilinx/xilinx_dma.c
@@ -165,8 +165,10 @@
 #define XILINX_DMA_FLUSH_MM2S		2
 #define XILINX_DMA_FLUSH_BOTH		1
 
-/* Delay loop counter to prevent hardware failure */
-#define XILINX_DMA_LOOP_COUNT		1000000
+/* Timeout for polling various registers */
+#define XILINX_DMA_POLL_TIMEOUT_US	1000000
+/* Delay between polls (avoid a delay of 0 to prevent CPU stalls) */
+#define XILINX_DMA_POLL_DELAY_US	10
 
 /* AXI DMA Specific Registers/Offsets */
 #define XILINX_DMA_REG_SRCDSTADDR	0x18
@@ -653,7 +655,7 @@ static void *xilinx_dma_get_metadata_ptr(struct dma_async_tx_descriptor *tx,
 	return seg->hw.app;
 }
 
-static struct dma_descriptor_metadata_ops xilinx_dma_metadata_ops = {
+static const struct dma_descriptor_metadata_ops xilinx_dma_metadata_ops = {
 	.get_ptr = xilinx_dma_get_metadata_ptr,
 };
 
@@ -1324,8 +1326,9 @@ static int xilinx_dma_stop_transfer(struct xilinx_dma_chan *chan)
 
 	/* Wait for the hardware to halt */
 	return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val,
-				       val & XILINX_DMA_DMASR_HALTED, 0,
-				       XILINX_DMA_LOOP_COUNT);
+				       val & XILINX_DMA_DMASR_HALTED,
+				       XILINX_DMA_POLL_DELAY_US,
+				       XILINX_DMA_POLL_TIMEOUT_US);
 }
 
 /**
@@ -1339,8 +1342,9 @@ static int xilinx_cdma_stop_transfer(struct xilinx_dma_chan *chan)
 	u32 val;
 
 	return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val,
-				       val & XILINX_DMA_DMASR_IDLE, 0,
-				       XILINX_DMA_LOOP_COUNT);
+				       val & XILINX_DMA_DMASR_IDLE,
+				       XILINX_DMA_POLL_DELAY_US,
+				       XILINX_DMA_POLL_TIMEOUT_US);
 }
 
 /**
@@ -1356,8 +1360,9 @@ static void xilinx_dma_start(struct xilinx_dma_chan *chan)
 
 	/* Wait for the hardware to start */
 	err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val,
-				      !(val & XILINX_DMA_DMASR_HALTED), 0,
-				      XILINX_DMA_LOOP_COUNT);
+				      !(val & XILINX_DMA_DMASR_HALTED),
+				      XILINX_DMA_POLL_DELAY_US,
+				      XILINX_DMA_POLL_TIMEOUT_US);
 
 	if (err) {
 		dev_err(chan->dev, "Cannot start channel %p: %x\n",
@@ -1580,7 +1585,14 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan)
 		return;
 	}
 
-	if (!chan->idle)
+	/*
+	 * Direct (non-SG) mode has no descriptor queue: writing the BTT
+	 * register launches a transfer immediately, so a new transfer must
+	 * not be programmed while one is in flight. Keep such transfers
+	 * serialized. SG mode supports chaining onto a running transfer via
+	 * tail-pointer extension, so it is allowed to proceed when busy.
+	 */
+	if (!chan->has_sg && !chan->idle)
 		return;
 
 	head_desc = list_first_entry(&chan->pending_list,
@@ -1596,10 +1608,9 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan)
 		reg &= ~XILINX_DMA_CR_COALESCE_MAX;
 		reg |= chan->desc_pendingcount <<
 				  XILINX_DMA_CR_COALESCE_SHIFT;
-		dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);
 	}
 
-	if (chan->has_sg)
+	if (chan->has_sg && list_empty(&chan->active_list))
 		xilinx_write(chan, XILINX_DMA_REG_CURDESC,
 			     head_desc->async_tx.phys);
 	reg  &= ~XILINX_DMA_CR_DELAY_MAX;
@@ -1607,7 +1618,8 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan)
 	reg |= XILINX_DMA_DMAXR_ALL_IRQ_MASK;
 	dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);
 
-	xilinx_dma_start(chan);
+	if (chan->idle)
+		xilinx_dma_start(chan);
 
 	if (chan->err)
 		return;
@@ -1660,9 +1672,6 @@ static void xilinx_mcdma_start_transfer(struct xilinx_dma_chan *chan)
 	if (chan->err)
 		return;
 
-	if (!chan->idle)
-		return;
-
 	if (list_empty(&chan->pending_list))
 		return;
 
@@ -1685,8 +1694,9 @@ static void xilinx_mcdma_start_transfer(struct xilinx_dma_chan *chan)
 	dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg);
 
 	/* Program current descriptor */
-	xilinx_write(chan, XILINX_MCDMA_CHAN_CDESC_OFFSET(chan->tdest),
-		     head_desc->async_tx.phys);
+	if (chan->has_sg && list_empty(&chan->active_list))
+		xilinx_write(chan, XILINX_MCDMA_CHAN_CDESC_OFFSET(chan->tdest),
+			     head_desc->async_tx.phys);
 
 	/* Program channel enable register */
 	reg = dma_ctrl_read(chan, XILINX_MCDMA_CHEN_OFFSET);
@@ -1698,7 +1708,8 @@ static void xilinx_mcdma_start_transfer(struct xilinx_dma_chan *chan)
 	reg |= XILINX_MCDMA_CR_RUNSTOP_MASK;
 	dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg);
 
-	xilinx_dma_start(chan);
+	if (chan->idle)
+		xilinx_dma_start(chan);
 
 	if (chan->err)
 		return;
@@ -1794,8 +1805,9 @@ static int xilinx_dma_reset(struct xilinx_dma_chan *chan)
 
 	/* Wait for the hardware to finish reset */
 	err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMACR, tmp,
-				      !(tmp & XILINX_DMA_DMACR_RESET), 0,
-				      XILINX_DMA_LOOP_COUNT);
+				      !(tmp & XILINX_DMA_DMACR_RESET),
+				      XILINX_DMA_POLL_DELAY_US,
+				      XILINX_DMA_POLL_TIMEOUT_US);
 
 	if (err) {
 		dev_err(chan->dev, "reset timeout, cr %x, sr %x\n",
@@ -1893,8 +1905,10 @@ static irqreturn_t xilinx_mcdma_irq_handler(int irq, void *data)
 	if (status & XILINX_MCDMA_IRQ_IOC_MASK) {
 		spin_lock(&chan->lock);
 		xilinx_dma_complete_descriptor(chan);
-		chan->idle = true;
-		chan->start_transfer(chan);
+		if (list_empty(&chan->active_list)) {
+			chan->idle = true;
+			chan->start_transfer(chan);
+		}
 		spin_unlock(&chan->lock);
 	}
 
@@ -1950,8 +1964,10 @@ static irqreturn_t xilinx_dma_irq_handler(int irq, void *data)
 		      XILINX_DMA_DMASR_DLY_CNT_IRQ)) {
 		spin_lock(&chan->lock);
 		xilinx_dma_complete_descriptor(chan);
-		chan->idle = true;
-		chan->start_transfer(chan);
+		if (list_empty(&chan->active_list)) {
+			chan->idle = true;
+			chan->start_transfer(chan);
+		}
 		spin_unlock(&chan->lock);
 	}
 
diff --git a/drivers/dma/xilinx/zynqmp_dma.c b/drivers/dma/xilinx/zynqmp_dma.c
index f6a812e49ddca..2b03584102acf 100644
--- a/drivers/dma/xilinx/zynqmp_dma.c
+++ b/drivers/dma/xilinx/zynqmp_dma.c
@@ -1159,8 +1159,6 @@ static int zynqmp_dma_probe(struct platform_device *pdev)
 /**
  * zynqmp_dma_remove - Driver remove function
  * @pdev: Pointer to the platform_device structure
- *
- * Return: Always '0'
  */
 static void zynqmp_dma_remove(struct platform_device *pdev)
 {
@@ -1170,9 +1168,9 @@ static void zynqmp_dma_remove(struct platform_device *pdev)
 	dma_async_device_unregister(&zdev->common);
 
 	zynqmp_dma_chan_remove(zdev->chan);
-	if (pm_runtime_active(zdev->dev))
-		zynqmp_dma_runtime_suspend(zdev->dev);
 	pm_runtime_disable(zdev->dev);
+	if (!pm_runtime_status_suspended(zdev->dev))
+		zynqmp_dma_runtime_suspend(zdev->dev);
 }
 
 static const struct of_device_id zynqmp_dma_of_match[] = {
diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
index 2e8690cb3c16e..bb1e8650c9d59 100644
--- a/drivers/dpll/dpll_core.c
+++ b/drivers/dpll/dpll_core.c
@@ -884,7 +884,11 @@ dpll_pin_register(struct dpll_device *dpll, struct dpll_pin *pin,
 	    WARN_ON(ops->measured_freq_get &&
 		    (!dpll_device_ops(dpll)->freq_monitor_get ||
 		     !dpll_device_ops(dpll)->freq_monitor_set)) ||
-	    WARN_ON(ops->supported_ffo && !ops->ffo_get))
+	    WARN_ON(ops->supported_ffo && !ops->ffo_get) ||
+	    WARN_ON((pin->prop.capabilities &
+		     DPLL_PIN_CAPABILITIES_STATE_CONNECTED_OVERRIDE) &&
+		    !(pin->prop.capabilities &
+		      DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE)))
 		return -EINVAL;
 
 	mutex_lock(&dpll_lock);
diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
index bf729cde796a7..5703667593a7c 100644
--- a/drivers/dpll/dpll_netlink.c
+++ b/drivers/dpll/dpll_netlink.c
@@ -567,6 +567,9 @@ dpll_msg_add_pin_ref_sync(struct sk_buff *msg, struct dpll_pin *pin,
 		if (!dpll_pin_available(ref_sync_pin))
 			continue;
 		ref_sync_pin_priv = dpll_pin_on_dpll_priv(dpll, ref_sync_pin);
+		/* Pin may have been unregistered from this dpll already */
+		if (!ref_sync_pin_priv)
+			continue;
 		if (WARN_ON(!ops->ref_sync_get))
 			return -EOPNOTSUPP;
 		ret = ops->ref_sync_get(pin, pin_priv, ref_sync_pin,
diff --git a/drivers/dpll/dpll_nl.c b/drivers/dpll/dpll_nl.c
index ed3bbe9841ea7..b1ba490e72b05 100644
--- a/drivers/dpll/dpll_nl.c
+++ b/drivers/dpll/dpll_nl.c
@@ -61,7 +61,7 @@ static const struct nla_policy dpll_pin_id_get_nl_policy[DPLL_A_PIN_TYPE + 1] =
 	[DPLL_A_PIN_BOARD_LABEL] = { .type = NLA_NUL_STRING, },
 	[DPLL_A_PIN_PANEL_LABEL] = { .type = NLA_NUL_STRING, },
 	[DPLL_A_PIN_PACKAGE_LABEL] = { .type = NLA_NUL_STRING, },
-	[DPLL_A_PIN_TYPE] = NLA_POLICY_RANGE(NLA_U32, 1, 5),
+	[DPLL_A_PIN_TYPE] = NLA_POLICY_RANGE(NLA_U32, 1, 6),
 };
 
 /* DPLL_CMD_PIN_GET - do */
diff --git a/drivers/dpll/zl3073x/chan.c b/drivers/dpll/zl3073x/chan.c
index 2fe3c3da84bb5..4ec2cf53dad46 100644
--- a/drivers/dpll/zl3073x/chan.c
+++ b/drivers/dpll/zl3073x/chan.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
 #include <linux/cleanup.h>
+#include <linux/delay.h>
 #include <linux/dev_printk.h>
 #include <linux/string.h>
 #include <linux/types.h>
@@ -31,9 +32,18 @@ int zl3073x_chan_state_update(struct zl3073x_dev *zldev, u8 index)
 	if (rc)
 		return rc;
 
-	/* Read df_offset vs tracked reference */
+	/* Read df_offset only when locked to a reference. In NCO mode
+	 * df_offset was captured at entry by nco_mode_set() - preserve it.
+	 */
+	if (!zl3073x_chan_is_locked(chan)) {
+		if (!zl3073x_chan_mode_is_nco(chan))
+			chan->df_offset = ZL_DPLL_DF_OFFSET_UNKNOWN;
+		return 0;
+	}
+
 	rc = zl3073x_poll_zero_u8(zldev, ZL_REG_DPLL_DF_READ(index),
-				  ZL_DPLL_DF_READ_SEM);
+				  ZL_DPLL_DF_READ_SEM,
+				  ZL_POLL_DF_READ_TIMEOUT_US);
 	if (rc)
 		return rc;
 
@@ -43,7 +53,8 @@ int zl3073x_chan_state_update(struct zl3073x_dev *zldev, u8 index)
 		return rc;
 
 	rc = zl3073x_poll_zero_u8(zldev, ZL_REG_DPLL_DF_READ(index),
-				  ZL_DPLL_DF_READ_SEM);
+				  ZL_DPLL_DF_READ_SEM,
+				  ZL_POLL_DF_READ_TIMEOUT_US);
 	if (rc)
 		return rc;
 
@@ -56,6 +67,96 @@ int zl3073x_chan_state_update(struct zl3073x_dev *zldev, u8 index)
 	return 0;
 }
 
+/**
+ * zl3073x_chan_nco_mode_set - switch DPLL channel to NCO mode
+ * @zldev: pointer to zl3073x_dev structure
+ * @index: DPLL channel index
+ *
+ * Switches the channel to NCO mode and reads the df_offset
+ * auto-captured by nco_auto_read directly from the register.
+ * No DF_READ handshake is needed as nco_auto_read populates
+ * the register before the mode switch completes.
+ *
+ * Return: 0 on success, <0 on error
+ */
+int zl3073x_chan_nco_mode_set(struct zl3073x_dev *zldev, u8 index)
+{
+	struct zl3073x_chan *chan = &zldev->chan[index];
+	u8 prev_mode, df_read;
+	u64 val;
+	int rc;
+
+	prev_mode = zl3073x_chan_mode_get(chan);
+
+	/* nco_auto_read captures the tracking offset at NCO entry only
+	 * from reflock, auto or holdover mode. From freerun the captured
+	 * value is not meaningful.
+	 */
+	if (prev_mode == ZL_DPLL_MODE_REFSEL_MODE_FREERUN) {
+		zl3073x_chan_mode_set(chan, ZL_DPLL_MODE_REFSEL_MODE_NCO);
+
+		rc = zl3073x_write_u8(zldev, ZL_REG_DPLL_MODE_REFSEL(index),
+				      chan->mode_refsel);
+		if (rc) {
+			zl3073x_chan_mode_set(chan, prev_mode);
+			return rc;
+		}
+
+		chan->df_offset = ZL_DPLL_DF_OFFSET_UNKNOWN;
+		return 0;
+	}
+
+	/* Configure df_read for nco_auto_read:
+	 * ref_ofst=0 - reads offset relative to master clock (not input ref)
+	 * cmd=CMD_ACC_I - accumulated I-part covering both locked and
+	 *                 holdover entry.
+	 *
+	 * No semaphore is set - this only configures what the df_offset
+	 * value represents after the mode switch; nco_auto_read performs
+	 * the actual read automatically.
+	 */
+	df_read = FIELD_PREP(ZL_DPLL_DF_READ_REF_OFST, 0) |
+		  FIELD_PREP(ZL_DPLL_DF_READ_CMD, ZL_DPLL_DF_READ_CMD_ACC_I);
+	rc = zl3073x_write_u8(zldev, ZL_REG_DPLL_DF_READ(index), df_read);
+	if (rc)
+		return rc;
+
+	/* Wait for df_read configuration to take effect before
+	 * triggering nco_auto_read via mode switch. The worst-case
+	 * internal register update time is 25 ms.
+	 */
+	fsleep(25000);
+
+	zl3073x_chan_mode_set(chan, ZL_DPLL_MODE_REFSEL_MODE_NCO);
+	rc = zl3073x_write_u8(zldev, ZL_REG_DPLL_MODE_REFSEL(index),
+			      chan->mode_refsel);
+	if (rc) {
+		zl3073x_chan_mode_set(chan, prev_mode);
+		return rc;
+	}
+
+	/* Wait for nco_auto_read to populate df_offset. The worst-case
+	 * internal register update time is 25 ms.
+	 */
+	fsleep(25000);
+
+	/* Read df_offset captured by nco_auto_read during mode switch.
+	 * No DF_READ semaphore handshake needed. Mode switch already
+	 * succeeded, so don't propagate a read failure back to userspace.
+	 */
+	rc = zl3073x_read_u48(zldev, ZL_REG_DPLL_DF_OFFSET(index), &val);
+	if (rc) {
+		dev_warn(zldev->dev,
+			 "Failed to read DPLL%u df_offset: %pe\n",
+			 index, ERR_PTR(rc));
+		chan->df_offset = ZL_DPLL_DF_OFFSET_UNKNOWN;
+	} else {
+		chan->df_offset = sign_extend64(val, 47);
+	}
+
+	return 0;
+}
+
 /**
  * zl3073x_chan_state_fetch - fetch DPLL channel state from hardware
  * @zldev: pointer to zl3073x_dev structure
@@ -71,6 +172,10 @@ int zl3073x_chan_state_fetch(struct zl3073x_dev *zldev, u8 index)
 	struct zl3073x_chan *chan = &zldev->chan[index];
 	int rc, i;
 
+	rc = zl3073x_read_u8(zldev, ZL_REG_DPLL_CTRL(index), &chan->ctrl);
+	if (rc)
+		return rc;
+
 	rc = zl3073x_read_u8(zldev, ZL_REG_DPLL_MODE_REFSEL(index),
 			     &chan->mode_refsel);
 	if (rc)
@@ -83,6 +188,13 @@ int zl3073x_chan_state_fetch(struct zl3073x_dev *zldev, u8 index)
 	if (rc)
 		return rc;
 
+	/* If firmware left the channel in NCO mode, mark df_offset as
+	 * unknown - we cannot know whether the preconditions for a valid
+	 * nco_auto_read capture were met.
+	 */
+	if (zl3073x_chan_mode_is_nco(chan))
+		chan->df_offset = ZL_DPLL_DF_OFFSET_UNKNOWN;
+
 	dev_dbg(zldev->dev,
 		"DPLL%u lock_state: %u, ho: %u, sel_state: %u, sel_ref: %u\n",
 		index, zl3073x_chan_lock_state_get(chan),
@@ -145,7 +257,15 @@ int zl3073x_chan_state_set(struct zl3073x_dev *zldev, u8 index,
 	if (!memcmp(&dchan->cfg, &chan->cfg, sizeof(chan->cfg)))
 		return 0;
 
-	/* Direct register write for mode_refsel */
+	/* Direct register writes for ctrl and mode_refsel */
+	if (dchan->ctrl != chan->ctrl) {
+		rc = zl3073x_write_u8(zldev, ZL_REG_DPLL_CTRL(index),
+				      chan->ctrl);
+		if (rc)
+			return rc;
+		dchan->ctrl = chan->ctrl;
+	}
+
 	if (dchan->mode_refsel != chan->mode_refsel) {
 		rc = zl3073x_write_u8(zldev, ZL_REG_DPLL_MODE_REFSEL(index),
 				      chan->mode_refsel);
diff --git a/drivers/dpll/zl3073x/chan.h b/drivers/dpll/zl3073x/chan.h
index 4353809c69122..dc9c6d95bdee7 100644
--- a/drivers/dpll/zl3073x/chan.h
+++ b/drivers/dpll/zl3073x/chan.h
@@ -13,6 +13,7 @@ struct zl3073x_dev;
 
 /**
  * struct zl3073x_chan - DPLL channel state
+ * @ctrl: DPLL control register value
  * @mode_refsel: mode and reference selection register value
  * @ref_prio: reference priority registers (4 bits per ref, P/N packed)
  * @mon_status: monitor status register value
@@ -21,6 +22,7 @@ struct zl3073x_dev;
  */
 struct zl3073x_chan {
 	struct_group(cfg,
+		u8	ctrl;
 		u8	mode_refsel;
 		u8	ref_prio[ZL3073X_NUM_REFS / 2];
 	);
@@ -38,6 +40,7 @@ int zl3073x_chan_state_set(struct zl3073x_dev *zldev, u8 index,
 			   const struct zl3073x_chan *chan);
 
 int zl3073x_chan_state_update(struct zl3073x_dev *zldev, u8 index);
+int zl3073x_chan_nco_mode_set(struct zl3073x_dev *zldev, u8 index);
 
 /**
  * zl3073x_chan_df_offset_get - get cached df_offset vs tracked reference
@@ -152,6 +155,51 @@ static inline u8 zl3073x_chan_lock_state_get(const struct zl3073x_chan *chan)
 	return FIELD_GET(ZL_DPLL_MON_STATUS_STATE, chan->mon_status);
 }
 
+/**
+ * zl3073x_chan_is_locked - check if channel is locked to a reference
+ * @chan: pointer to channel state
+ *
+ * Return: true if channel is locked, false otherwise
+ */
+static inline bool zl3073x_chan_is_locked(const struct zl3073x_chan *chan)
+{
+	u8 lock_state = zl3073x_chan_lock_state_get(chan);
+	return lock_state == ZL_DPLL_MON_STATUS_STATE_LOCK;
+}
+
+/**
+ * zl3073x_chan_mode_is_auto - check if channel is in automatic mode
+ * @chan: pointer to channel state
+ *
+ * Return: true if channel is in automatic mode, false otherwise
+ */
+static inline bool zl3073x_chan_mode_is_auto(const struct zl3073x_chan *chan)
+{
+	return zl3073x_chan_mode_get(chan) == ZL_DPLL_MODE_REFSEL_MODE_AUTO;
+}
+
+/**
+ * zl3073x_chan_mode_is_nco - check if channel is in NCO mode
+ * @chan: pointer to channel state
+ *
+ * Return: true if channel is in NCO mode, false otherwise
+ */
+static inline bool zl3073x_chan_mode_is_nco(const struct zl3073x_chan *chan)
+{
+	return zl3073x_chan_mode_get(chan) == ZL_DPLL_MODE_REFSEL_MODE_NCO;
+}
+
+/**
+ * zl3073x_chan_mode_is_reflock - check if channel is in reflock mode
+ * @chan: pointer to channel state
+ *
+ * Return: true if channel is in reflock mode, false otherwise
+ */
+static inline bool zl3073x_chan_mode_is_reflock(const struct zl3073x_chan *chan)
+{
+	return zl3073x_chan_mode_get(chan) == ZL_DPLL_MODE_REFSEL_MODE_REFLOCK;
+}
+
 /**
  * zl3073x_chan_is_ho_ready - check if holdover is ready
  * @chan: pointer to channel state
diff --git a/drivers/dpll/zl3073x/core.c b/drivers/dpll/zl3073x/core.c
index 8e6416a4741de..7f5afaaae6342 100644
--- a/drivers/dpll/zl3073x/core.c
+++ b/drivers/dpll/zl3073x/core.c
@@ -311,17 +311,17 @@ int zl3073x_write_u48(struct zl3073x_dev *zldev, unsigned int reg, u64 val)
  * @zldev: zl3073x device pointer
  * @reg: register to poll (has to be 8bit register)
  * @mask: bit mask for polling
+ * @timeout_us: timeout in microseconds
  *
  * Waits for bits specified by @mask in register @reg value to be cleared
  * by the device.
  *
  * Returns: 0 on success, <0 on error
  */
-int zl3073x_poll_zero_u8(struct zl3073x_dev *zldev, unsigned int reg, u8 mask)
+int zl3073x_poll_zero_u8(struct zl3073x_dev *zldev, unsigned int reg,
+			 u8 mask, unsigned int timeout_us)
 {
-	/* Register polling sleep & timeout */
-#define ZL_POLL_SLEEP_US   10
-#define ZL_POLL_TIMEOUT_US 2000000
+#define ZL_POLL_SLEEP_US 10
 	unsigned int val;
 
 	/* Check the register is 8bit */
@@ -335,7 +335,7 @@ int zl3073x_poll_zero_u8(struct zl3073x_dev *zldev, unsigned int reg, u8 mask)
 	reg = ZL_REG_ADDR(reg) + ZL_RANGE_OFFSET;
 
 	return regmap_read_poll_timeout(zldev->regmap, reg, val, !(val & mask),
-					ZL_POLL_SLEEP_US, ZL_POLL_TIMEOUT_US);
+					ZL_POLL_SLEEP_US, timeout_us);
 }
 
 int zl3073x_mb_op(struct zl3073x_dev *zldev, unsigned int op_reg, u8 op_val,
@@ -354,7 +354,8 @@ int zl3073x_mb_op(struct zl3073x_dev *zldev, unsigned int op_reg, u8 op_val,
 		return rc;
 
 	/* Wait for the operation to actually finish */
-	return zl3073x_poll_zero_u8(zldev, op_reg, op_val);
+	return zl3073x_poll_zero_u8(zldev, op_reg, op_val,
+				    ZL_POLL_MB_TIMEOUT_US);
 }
 
 /**
@@ -377,8 +378,8 @@ zl3073x_do_hwreg_op(struct zl3073x_dev *zldev, u8 op)
 		return rc;
 
 	/* Poll for completion - pending bit cleared */
-	return zl3073x_poll_zero_u8(zldev, ZL_REG_HWREG_OP,
-				    ZL_HWREG_OP_PENDING);
+	return zl3073x_poll_zero_u8(zldev, ZL_REG_HWREG_OP, ZL_HWREG_OP_PENDING,
+				    ZL_POLL_HWREG_TIMEOUT_US);
 }
 
 /**
@@ -566,19 +567,7 @@ zl3073x_dev_ref_states_update(struct zl3073x_dev *zldev)
 	}
 }
 
-static void
-zl3073x_dev_chan_states_update(struct zl3073x_dev *zldev)
-{
-	int i, rc;
 
-	for (i = 0; i < zldev->info->num_channels; i++) {
-		rc = zl3073x_chan_state_update(zldev, i);
-		if (rc)
-			dev_warn(zldev->dev,
-				 "Failed to get DPLL%u state: %pe\n", i,
-				 ERR_PTR(rc));
-	}
-}
 
 /**
  * zl3073x_ref_phase_offsets_update - update reference phase offsets
@@ -609,7 +598,8 @@ int zl3073x_ref_phase_offsets_update(struct zl3073x_dev *zldev, int channel)
 	 * to be zero to ensure that the measured data are coherent.
 	 */
 	rc = zl3073x_poll_zero_u8(zldev, ZL_REG_REF_PHASE_ERR_READ_RQST,
-				  ZL_REF_PHASE_ERR_READ_RQST_RD);
+				  ZL_REF_PHASE_ERR_READ_RQST_RD,
+				  ZL_POLL_PHASE_ERR_TIMEOUT_US);
 	if (rc)
 		return rc;
 
@@ -628,7 +618,8 @@ int zl3073x_ref_phase_offsets_update(struct zl3073x_dev *zldev, int channel)
 
 	/* Wait for finish */
 	return zl3073x_poll_zero_u8(zldev, ZL_REG_REF_PHASE_ERR_READ_RQST,
-				    ZL_REF_PHASE_ERR_READ_RQST_RD);
+				    ZL_REF_PHASE_ERR_READ_RQST_RD,
+				    ZL_POLL_PHASE_ERR_TIMEOUT_US);
 }
 
 /**
@@ -648,7 +639,8 @@ zl3073x_ref_freq_meas_latch(struct zl3073x_dev *zldev, u8 type)
 
 	/* Wait for previous measurement to finish */
 	rc = zl3073x_poll_zero_u8(zldev, ZL_REG_REF_FREQ_MEAS_CTRL,
-				  ZL_REF_FREQ_MEAS_CTRL);
+				  ZL_REF_FREQ_MEAS_CTRL,
+				  ZL_POLL_FREQ_MEAS_TIMEOUT_US);
 	if (rc)
 		return rc;
 
@@ -669,7 +661,8 @@ zl3073x_ref_freq_meas_latch(struct zl3073x_dev *zldev, u8 type)
 
 	/* Wait for finish */
 	return zl3073x_poll_zero_u8(zldev, ZL_REG_REF_FREQ_MEAS_CTRL,
-				    ZL_REF_FREQ_MEAS_CTRL);
+				    ZL_REF_FREQ_MEAS_CTRL,
+				    ZL_POLL_FREQ_MEAS_TIMEOUT_US);
 }
 
 /**
@@ -715,9 +708,6 @@ zl3073x_dev_periodic_work(struct kthread_work *work)
 	/* Update input references' states */
 	zl3073x_dev_ref_states_update(zldev);
 
-	/* Update DPLL channels' states */
-	zl3073x_dev_chan_states_update(zldev);
-
 	/* Update DPLL-to-connected-ref phase offsets registers */
 	rc = zl3073x_ref_phase_offsets_update(zldev, -1);
 	if (rc)
@@ -727,7 +717,7 @@ zl3073x_dev_periodic_work(struct kthread_work *work)
 	/* Update measured input reference frequencies if frequency
 	 * monitoring is enabled.
 	 */
-	if (zldev->freq_monitor) {
+	if (READ_ONCE(zldev->freq_monitor)) {
 		rc = zl3073x_ref_freq_meas_update(zldev);
 		if (rc)
 			dev_warn(zldev->dev,
@@ -763,7 +753,7 @@ int zl3073x_dev_phase_avg_factor_set(struct zl3073x_dev *zldev, u8 factor)
 		return rc;
 
 	/* Save the new factor */
-	zldev->phase_avg_factor = factor;
+	WRITE_ONCE(zldev->phase_avg_factor, factor);
 
 	return 0;
 }
diff --git a/drivers/dpll/zl3073x/core.h b/drivers/dpll/zl3073x/core.h
index addba378b0df4..78dc208f3eea2 100644
--- a/drivers/dpll/zl3073x/core.h
+++ b/drivers/dpll/zl3073x/core.h
@@ -7,6 +7,7 @@
 #include <linux/kthread.h>
 #include <linux/list.h>
 #include <linux/mutex.h>
+#include <linux/time64.h>
 #include <linux/types.h>
 
 #include "chan.h"
@@ -19,6 +20,12 @@ struct device;
 struct regmap;
 struct zl3073x_dpll;
 
+/* Per-operation poll timeouts */
+#define ZL_POLL_DF_READ_TIMEOUT_US	(25 * USEC_PER_MSEC)
+#define ZL_POLL_FREQ_MEAS_TIMEOUT_US	(50 * USEC_PER_MSEC)
+#define ZL_POLL_HWREG_TIMEOUT_US	(50 * USEC_PER_MSEC)
+#define ZL_POLL_MB_TIMEOUT_US		(30 * USEC_PER_MSEC)
+#define ZL_POLL_PHASE_ERR_TIMEOUT_US	(50 * USEC_PER_MSEC)
 
 enum zl3073x_flags {
 	ZL3073X_FLAG_REF_PHASE_COMP_32_BIT,
@@ -94,7 +101,7 @@ void zl3073x_dev_stop(struct zl3073x_dev *zldev);
 
 static inline u8 zl3073x_dev_phase_avg_factor_get(struct zl3073x_dev *zldev)
 {
-	return zldev->phase_avg_factor;
+	return READ_ONCE(zldev->phase_avg_factor);
 }
 
 int zl3073x_dev_phase_avg_factor_set(struct zl3073x_dev *zldev, u8 factor);
@@ -127,7 +134,8 @@ struct zl3073x_hwreg_seq_item {
 
 int zl3073x_mb_op(struct zl3073x_dev *zldev, unsigned int op_reg, u8 op_val,
 		  unsigned int mask_reg, u16 mask_val);
-int zl3073x_poll_zero_u8(struct zl3073x_dev *zldev, unsigned int reg, u8 mask);
+int zl3073x_poll_zero_u8(struct zl3073x_dev *zldev, unsigned int reg,
+			 u8 mask, unsigned int timeout_us);
 int zl3073x_read_u8(struct zl3073x_dev *zldev, unsigned int reg, u8 *val);
 int zl3073x_read_u16(struct zl3073x_dev *zldev, unsigned int reg, u16 *val);
 int zl3073x_read_u32(struct zl3073x_dev *zldev, unsigned int reg, u32 *val);
diff --git a/drivers/dpll/zl3073x/dpll.c b/drivers/dpll/zl3073x/dpll.c
index 4ab045e85a890..a2a641b8358f2 100644
--- a/drivers/dpll/zl3073x/dpll.c
+++ b/drivers/dpll/zl3073x/dpll.c
@@ -1,6 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
-#include <linux/atomic.h>
 #include <linux/bits.h>
 #include <linux/bitfield.h>
 #include <linux/bug.h>
@@ -57,7 +56,7 @@ struct zl3073x_dpll_pin {
 	s32			phase_gran;
 	enum dpll_pin_operstate	operstate;
 	s64			phase_offset;
-	atomic64_t		freq_offset;
+	s64			freq_offset;
 	u32			measured_freq;
 };
 
@@ -80,6 +79,18 @@ zl3073x_dpll_is_input_pin(struct zl3073x_dpll_pin *pin)
 	return pin->dir == DPLL_PIN_DIRECTION_INPUT;
 }
 
+/**
+ * zl3073x_dpll_is_nco_pin - check if the pin is a virtual NCO pin
+ * @pin: pin to check
+ *
+ * Return: true if pin is a virtual NCO pin, false otherwise.
+ */
+static bool
+zl3073x_dpll_is_nco_pin(struct zl3073x_dpll_pin *pin)
+{
+	return pin->id == ZL3073X_NCO_PIN_ID;
+}
+
 /**
  * zl3073x_dpll_is_p_pin - check if the pin is P-pin
  * @pin: pin to check
@@ -119,6 +130,19 @@ zl3073x_dpll_pin_get_by_ref(struct zl3073x_dpll *zldpll, u8 ref_id)
 	return NULL;
 }
 
+static struct zl3073x_dpll_pin *
+zl3073x_dpll_nco_pin_get(struct zl3073x_dpll *zldpll)
+{
+	struct zl3073x_dpll_pin *pin;
+
+	list_for_each_entry(pin, &zldpll->pins, list) {
+		if (zl3073x_dpll_is_nco_pin(pin))
+			return pin;
+	}
+
+	return NULL;
+}
+
 static int
 zl3073x_dpll_input_pin_esync_get(const struct dpll_pin *dpll_pin,
 				 void *pin_priv,
@@ -133,6 +157,8 @@ zl3073x_dpll_input_pin_esync_get(const struct dpll_pin *dpll_pin,
 	const struct zl3073x_ref *ref;
 	u8 ref_id;
 
+	guard(mutex)(&zldpll->lock);
+
 	ref_id = zl3073x_input_pin_ref_get(pin->id);
 	ref = zl3073x_ref_state_get(zldev, ref_id);
 
@@ -169,6 +195,8 @@ zl3073x_dpll_input_pin_esync_set(const struct dpll_pin *dpll_pin,
 	struct zl3073x_ref ref;
 	u8 ref_id, sync_mode;
 
+	guard(mutex)(&zldpll->lock);
+
 	ref_id = zl3073x_input_pin_ref_get(pin->id);
 	ref = *zl3073x_ref_state_get(zldev, ref_id);
 
@@ -204,6 +232,8 @@ zl3073x_dpll_input_pin_ref_sync_get(const struct dpll_pin *dpll_pin,
 	const struct zl3073x_ref *ref;
 	u8 ref_id, mode, pair;
 
+	guard(mutex)(&zldpll->lock);
+
 	ref_id = zl3073x_input_pin_ref_get(pin->id);
 	ref = zl3073x_ref_state_get(zldev, ref_id);
 	mode = zl3073x_ref_sync_mode_get(ref);
@@ -235,6 +265,8 @@ zl3073x_dpll_input_pin_ref_sync_set(const struct dpll_pin *dpll_pin,
 	struct zl3073x_ref ref;
 	int rc;
 
+	guard(mutex)(&zldpll->lock);
+
 	ref_id = zl3073x_input_pin_ref_get(pin->id);
 	sync_ref_id = zl3073x_input_pin_ref_get(sync_pin->id);
 	ref = *zl3073x_ref_state_get(zldev, ref_id);
@@ -298,12 +330,15 @@ zl3073x_dpll_input_pin_ffo_get(const struct dpll_pin *dpll_pin, void *pin_priv,
 			       struct dpll_ffo_param *ffo,
 			       struct netlink_ext_ack *extack)
 {
+	struct zl3073x_dpll *zldpll = dpll_priv;
 	struct zl3073x_dpll_pin *pin = pin_priv;
 
+	guard(mutex)(&zldpll->lock);
+
 	if (pin->operstate != DPLL_PIN_OPERSTATE_ACTIVE)
 		return -ENODATA;
 
-	ffo->ffo = atomic64_read(&pin->freq_offset);
+	ffo->ffo = pin->freq_offset;
 
 	return 0;
 }
@@ -315,8 +350,11 @@ zl3073x_dpll_input_pin_measured_freq_get(const struct dpll_pin *dpll_pin,
 					 void *dpll_priv, u64 *measured_freq,
 					 struct netlink_ext_ack *extack)
 {
+	struct zl3073x_dpll *zldpll = dpll_priv;
 	struct zl3073x_dpll_pin *pin = pin_priv;
 
+	guard(mutex)(&zldpll->lock);
+
 	*measured_freq = pin->measured_freq;
 	*measured_freq *= DPLL_PIN_MEASURED_FREQUENCY_DIVIDER;
 
@@ -334,6 +372,8 @@ zl3073x_dpll_input_pin_frequency_get(const struct dpll_pin *dpll_pin,
 	struct zl3073x_dpll_pin *pin = pin_priv;
 	u8 ref_id;
 
+	guard(mutex)(&zldpll->lock);
+
 	ref_id = zl3073x_input_pin_ref_get(pin->id);
 	*frequency = zl3073x_dev_ref_freq_get(zldpll->dev, ref_id);
 
@@ -353,6 +393,8 @@ zl3073x_dpll_input_pin_frequency_set(const struct dpll_pin *dpll_pin,
 	struct zl3073x_ref ref;
 	u8 ref_id;
 
+	guard(mutex)(&zldpll->lock);
+
 	/* Get reference state */
 	ref_id = zl3073x_input_pin_ref_get(pin->id);
 	ref = *zl3073x_ref_state_get(zldev, ref_id);
@@ -401,6 +443,8 @@ zl3073x_dpll_input_pin_phase_offset_get(const struct dpll_pin *dpll_pin,
 	u8 conn_id, ref_id;
 	s64 ref_phase;
 
+	guard(mutex)(&zldpll->lock);
+
 	/* Get currently connected reference */
 	conn_id = zl3073x_dpll_connected_ref_get(zldpll);
 
@@ -458,6 +502,8 @@ zl3073x_dpll_input_pin_phase_adjust_get(const struct dpll_pin *dpll_pin,
 	s64 phase_comp;
 	u8 ref_id;
 
+	guard(mutex)(&zldpll->lock);
+
 	/* Read reference configuration */
 	ref_id = zl3073x_input_pin_ref_get(pin->id);
 	ref = zl3073x_ref_state_get(zldev, ref_id);
@@ -490,6 +536,8 @@ zl3073x_dpll_input_pin_phase_adjust_set(const struct dpll_pin *dpll_pin,
 	struct zl3073x_ref ref;
 	u8 ref_id;
 
+	guard(mutex)(&zldpll->lock);
+
 	/* Read reference configuration */
 	ref_id = zl3073x_input_pin_ref_get(pin->id);
 	ref = *zl3073x_ref_state_get(zldev, ref_id);
@@ -523,6 +571,8 @@ zl3073x_dpll_ref_operstate_get(struct zl3073x_dpll_pin *pin,
 	const struct zl3073x_ref *ref;
 	u8 ref_id;
 
+	lockdep_assert_held(&zldpll->lock);
+
 	ref_id = zl3073x_input_pin_ref_get(pin->id);
 
 	/* Check if this pin is the currently locked reference */
@@ -556,6 +606,8 @@ zl3073x_dpll_input_pin_state_on_dpll_get(const struct dpll_pin *dpll_pin,
 	const struct zl3073x_chan *chan;
 	u8 mode, ref;
 
+	guard(mutex)(&zldpll->lock);
+
 	chan = zl3073x_chan_state_get(zldpll->dev, zldpll->id);
 	ref = zl3073x_input_pin_ref_get(pin->id);
 	mode = zl3073x_chan_mode_get(chan);
@@ -589,8 +641,11 @@ zl3073x_dpll_input_pin_operstate_on_dpll_get(const struct dpll_pin *dpll_pin,
 					     enum dpll_pin_operstate *operstate,
 					     struct netlink_ext_ack *extack)
 {
+	struct zl3073x_dpll *zldpll = dpll_priv;
 	struct zl3073x_dpll_pin *pin = pin_priv;
 
+	guard(mutex)(&zldpll->lock);
+
 	return zl3073x_dpll_ref_operstate_get(pin, operstate);
 }
 
@@ -604,9 +659,12 @@ zl3073x_dpll_input_pin_state_on_dpll_set(const struct dpll_pin *dpll_pin,
 {
 	struct zl3073x_dpll *zldpll = dpll_priv;
 	struct zl3073x_dpll_pin *pin = pin_priv;
+	struct zl3073x_dpll_pin *nco_pin = NULL;
 	struct zl3073x_chan chan;
 	u8 mode, ref;
-	int rc;
+	int rc = 0;
+
+	mutex_lock(&zldpll->lock);
 
 	chan = *zl3073x_chan_state_get(zldpll->dev, zldpll->id);
 	ref = zl3073x_input_pin_ref_get(pin->id);
@@ -633,6 +691,10 @@ zl3073x_dpll_input_pin_state_on_dpll_set(const struct dpll_pin *dpll_pin,
 			goto invalid_state;
 		}
 		break;
+	case ZL_DPLL_MODE_REFSEL_MODE_NCO:
+		if (state == DPLL_PIN_STATE_CONNECTED)
+			nco_pin = zl3073x_dpll_nco_pin_get(zldpll);
+		fallthrough;
 	case ZL_DPLL_MODE_REFSEL_MODE_FREERUN:
 	case ZL_DPLL_MODE_REFSEL_MODE_HOLDOVER:
 		if (state == DPLL_PIN_STATE_CONNECTED) {
@@ -648,13 +710,13 @@ zl3073x_dpll_input_pin_state_on_dpll_set(const struct dpll_pin *dpll_pin,
 	case ZL_DPLL_MODE_REFSEL_MODE_AUTO:
 		if (state == DPLL_PIN_STATE_SELECTABLE) {
 			if (zl3073x_chan_ref_is_selectable(&chan, ref))
-				return 0; /* Pin is already selectable */
+				goto unlock; /* Pin is already selectable */
 
 			/* Restore pin priority in HW */
 			zl3073x_chan_ref_prio_set(&chan, ref, pin->prio);
 		} else if (state == DPLL_PIN_STATE_DISCONNECTED) {
 			if (!zl3073x_chan_ref_is_selectable(&chan, ref))
-				return 0; /* Pin is already disconnected */
+				goto unlock; /* Pin is already disconnected */
 
 			/* Set pin priority to none in HW */
 			zl3073x_chan_ref_prio_set(&chan, ref,
@@ -667,18 +729,27 @@ zl3073x_dpll_input_pin_state_on_dpll_set(const struct dpll_pin *dpll_pin,
 		/* In other modes we cannot change input reference */
 		NL_SET_ERR_MSG(extack,
 			       "Pin state cannot be changed in current mode");
-		return -EOPNOTSUPP;
+		rc = -EOPNOTSUPP;
+		goto unlock;
 	}
 
 	/* Commit DPLL channel changes */
 	rc = zl3073x_chan_state_set(zldpll->dev, zldpll->id, &chan);
-	if (rc)
-		return rc;
+	goto unlock;
 
-	return 0;
 invalid_state:
 	NL_SET_ERR_MSG_MOD(extack, "Invalid pin state for this device mode");
-	return -EINVAL;
+	rc = -EINVAL;
+unlock:
+	mutex_unlock(&zldpll->lock);
+
+	/* If leaving NCO mode, notify userspace about the NCO pin
+	 * state change - the periodic worker skips the NCO pin.
+	 */
+	if (!rc && nco_pin)
+		__dpll_pin_change_ntf(nco_pin->dpll_pin);
+
+	return rc;
 }
 
 static int
@@ -686,8 +757,11 @@ zl3073x_dpll_input_pin_prio_get(const struct dpll_pin *dpll_pin, void *pin_priv,
 				const struct dpll_device *dpll, void *dpll_priv,
 				u32 *prio, struct netlink_ext_ack *extack)
 {
+	struct zl3073x_dpll *zldpll = dpll_priv;
 	struct zl3073x_dpll_pin *pin = pin_priv;
 
+	guard(mutex)(&zldpll->lock);
+
 	*prio = pin->prio;
 
 	return 0;
@@ -704,6 +778,8 @@ zl3073x_dpll_input_pin_prio_set(const struct dpll_pin *dpll_pin, void *pin_priv,
 	u8 ref;
 	int rc;
 
+	guard(mutex)(&zldpll->lock);
+
 	if (prio > ZL_DPLL_REF_PRIO_MAX)
 		return -EINVAL;
 
@@ -739,6 +815,8 @@ zl3073x_dpll_output_pin_esync_get(const struct dpll_pin *dpll_pin,
 	u32 synth_freq, out_freq;
 	u8 out_id;
 
+	guard(mutex)(&zldpll->lock);
+
 	out_id = zl3073x_output_pin_out_get(pin->id);
 	out = zl3073x_out_state_get(zldev, out_id);
 
@@ -796,6 +874,8 @@ zl3073x_dpll_output_pin_esync_set(const struct dpll_pin *dpll_pin,
 	u32 synth_freq;
 	u8 out_id;
 
+	guard(mutex)(&zldpll->lock);
+
 	out_id = zl3073x_output_pin_out_get(pin->id);
 	out = *zl3073x_out_state_get(zldev, out_id);
 
@@ -816,7 +896,7 @@ zl3073x_dpll_output_pin_esync_set(const struct dpll_pin *dpll_pin,
 
 	/* If esync is being disabled just write mailbox and finish */
 	if (!freq)
-		goto write_mailbox;
+		return zl3073x_out_state_set(zldev, out_id, &out);
 
 	/* Get attached synth frequency */
 	synth = zl3073x_synth_state_get(zldev, zl3073x_out_synth_get(&out));
@@ -833,7 +913,6 @@ zl3073x_dpll_output_pin_esync_set(const struct dpll_pin *dpll_pin,
 	 */
 	out.esync_n_width = out.div / 2;
 
-write_mailbox:
 	/* Commit output configuration */
 	return zl3073x_out_state_set(zldev, out_id, &out);
 }
@@ -848,6 +927,8 @@ zl3073x_dpll_output_pin_frequency_get(const struct dpll_pin *dpll_pin,
 	struct zl3073x_dpll *zldpll = dpll_priv;
 	struct zl3073x_dpll_pin *pin = pin_priv;
 
+	guard(mutex)(&zldpll->lock);
+
 	*frequency = zl3073x_dev_output_pin_freq_get(zldpll->dev, pin->id);
 
 	return 0;
@@ -868,6 +949,8 @@ zl3073x_dpll_output_pin_frequency_set(const struct dpll_pin *dpll_pin,
 	struct zl3073x_out out;
 	u8 out_id;
 
+	guard(mutex)(&zldpll->lock);
+
 	out_id = zl3073x_output_pin_out_get(pin->id);
 	out = *zl3073x_out_state_get(zldev, out_id);
 
@@ -941,6 +1024,8 @@ zl3073x_dpll_output_pin_phase_adjust_get(const struct dpll_pin *dpll_pin,
 	const struct zl3073x_out *out;
 	u8 out_id;
 
+	guard(mutex)(&zldpll->lock);
+
 	out_id = zl3073x_output_pin_out_get(pin->id);
 	out = zl3073x_out_state_get(zldev, out_id);
 
@@ -964,6 +1049,8 @@ zl3073x_dpll_output_pin_phase_adjust_set(const struct dpll_pin *dpll_pin,
 	struct zl3073x_out out;
 	u8 out_id;
 
+	guard(mutex)(&zldpll->lock);
+
 	out_id = zl3073x_output_pin_out_get(pin->id);
 	out = *zl3073x_out_state_get(zldev, out_id);
 
@@ -988,6 +1075,144 @@ zl3073x_dpll_output_pin_state_on_dpll_get(const struct dpll_pin *dpll_pin,
 	return 0;
 }
 
+static int
+zl3073x_dpll_nco_pin_operstate_on_dpll_get(const struct dpll_pin *dpll_pin,
+					   void *pin_priv,
+					   const struct dpll_device *dpll,
+					   void *dpll_priv,
+					   enum dpll_pin_operstate *operstate,
+					   struct netlink_ext_ack *extack)
+{
+	struct zl3073x_dpll *zldpll = dpll_priv;
+	const struct zl3073x_chan *chan;
+
+	guard(mutex)(&zldpll->lock);
+
+	chan = zl3073x_chan_state_get(zldpll->dev, zldpll->id);
+	if (zl3073x_chan_mode_is_nco(chan))
+		*operstate = DPLL_PIN_OPERSTATE_ACTIVE;
+	else
+		*operstate = DPLL_PIN_OPERSTATE_STANDBY;
+
+	return 0;
+}
+
+static int
+zl3073x_dpll_nco_pin_state_on_dpll_get(const struct dpll_pin *dpll_pin,
+				       void *pin_priv,
+				       const struct dpll_device *dpll,
+				       void *dpll_priv,
+				       enum dpll_pin_state *state,
+				       struct netlink_ext_ack *extack)
+{
+	struct zl3073x_dpll *zldpll = dpll_priv;
+	const struct zl3073x_chan *chan;
+
+	guard(mutex)(&zldpll->lock);
+
+	chan = zl3073x_chan_state_get(zldpll->dev, zldpll->id);
+	if (zl3073x_chan_mode_is_nco(chan))
+		*state = DPLL_PIN_STATE_CONNECTED;
+	else
+		*state = DPLL_PIN_STATE_DISCONNECTED;
+
+	return 0;
+}
+
+static int
+zl3073x_dpll_nco_pin_state_on_dpll_set(const struct dpll_pin *dpll_pin,
+				       void *pin_priv,
+				       const struct dpll_device *dpll,
+				       void *dpll_priv,
+				       enum dpll_pin_state state,
+				       struct netlink_ext_ack *extack)
+{
+	struct zl3073x_dpll_pin *ref_pin = NULL;
+	struct zl3073x_dpll *zldpll = dpll_priv;
+	struct zl3073x_chan chan;
+	u8 ref;
+	int rc;
+
+	mutex_lock(&zldpll->lock);
+
+	chan = *zl3073x_chan_state_get(zldpll->dev, zldpll->id);
+
+	switch (state) {
+	case DPLL_PIN_STATE_CONNECTED:
+		if (zl3073x_chan_mode_is_nco(&chan)) {
+			mutex_unlock(&zldpll->lock);
+			return 0;
+		}
+		if (zl3073x_chan_mode_is_auto(&chan)) {
+			NL_SET_ERR_MSG(extack,
+				       "NCO pin cannot be connected in automatic mode");
+			mutex_unlock(&zldpll->lock);
+			return -EINVAL;
+		}
+		if (zl3073x_chan_mode_is_reflock(&chan)) {
+			/* Get currently connected pin */
+			ref = zl3073x_chan_ref_get(&chan);
+			ref_pin = zl3073x_dpll_pin_get_by_ref(zldpll, ref);
+		}
+		rc = zl3073x_chan_nco_mode_set(zldpll->dev, zldpll->id);
+		break;
+	case DPLL_PIN_STATE_DISCONNECTED:
+		if (!zl3073x_chan_mode_is_nco(&chan)) {
+			mutex_unlock(&zldpll->lock);
+			return 0;
+		}
+		/* Switch to freerun - holdover averaging was not
+		 * updated during NCO mode.
+		 */
+		zl3073x_chan_mode_set(&chan,
+				      ZL_DPLL_MODE_REFSEL_MODE_FREERUN);
+		rc = zl3073x_chan_state_set(zldpll->dev, zldpll->id, &chan);
+		break;
+	default:
+		NL_SET_ERR_MSG(extack, "invalid pin state for NCO pin");
+		mutex_unlock(&zldpll->lock);
+		return -EINVAL;
+	}
+
+	mutex_unlock(&zldpll->lock);
+
+	if (!rc && ref_pin)
+		__dpll_pin_change_ntf(ref_pin->dpll_pin);
+
+	return rc;
+}
+
+static int
+zl3073x_dpll_nco_pin_ffo_get(const struct dpll_pin *dpll_pin, void *pin_priv,
+			     const struct dpll_device *dpll, void *dpll_priv,
+			     struct dpll_ffo_param *ffo,
+			     struct netlink_ext_ack *extack)
+{
+	struct zl3073x_dpll *zldpll = dpll_priv;
+	const struct zl3073x_chan *chan;
+	s64 df_offset;
+
+	guard(mutex)(&zldpll->lock);
+
+	chan = zl3073x_chan_state_get(zldpll->dev, zldpll->id);
+	if (!zl3073x_chan_mode_is_nco(chan))
+		return -ENODATA;
+
+	/* Do not report FFO if a failure occurred during switching to NCO. */
+	df_offset = zl3073x_chan_df_offset_get(chan);
+	if (df_offset == ZL_DPLL_DF_OFFSET_UNKNOWN)
+		return -ENODATA;
+
+	/* dpll_df_offset register has inverted sign per datasheet:
+	 * f_offset = f_nom * (-df_offset) / 2^48
+	 * NCO pin reports DPLL output offset from nominal, so negate.
+	 * Convert to PPT: ppt = -df * 5^12 / 2^36
+	 */
+	ffo->ffo = -mul_s64_u64_shr(df_offset, 244140625, 36);
+
+	return 0;
+}
+
 static int
 zl3073x_dpll_temp_get(const struct dpll_device *dpll, void *dpll_priv,
 		      s32 *temp, struct netlink_ext_ack *extack)
@@ -997,6 +1222,8 @@ zl3073x_dpll_temp_get(const struct dpll_device *dpll, void *dpll_priv,
 	u16 val;
 	int rc;
 
+	guard(mutex)(&zldpll->lock);
+
 	rc = zl3073x_read_u16(zldev, ZL_REG_DIE_TEMP_STATUS, &val);
 	if (rc)
 		return rc;
@@ -1008,14 +1235,13 @@ zl3073x_dpll_temp_get(const struct dpll_device *dpll, void *dpll_priv,
 }
 
 static int
-zl3073x_dpll_lock_status_get(const struct dpll_device *dpll, void *dpll_priv,
-			     enum dpll_lock_status *status,
-			     enum dpll_lock_status_error *status_error,
-			     struct netlink_ext_ack *extack)
+__zl3073x_dpll_lock_status_get(struct zl3073x_dpll *zldpll,
+			       enum dpll_lock_status *status)
 {
-	struct zl3073x_dpll *zldpll = dpll_priv;
 	const struct zl3073x_chan *chan;
 
+	lockdep_assert_held(&zldpll->lock);
+
 	chan = zl3073x_chan_state_get(zldpll->dev, zldpll->id);
 
 	switch (zl3073x_chan_mode_get(chan)) {
@@ -1052,23 +1278,24 @@ zl3073x_dpll_lock_status_get(const struct dpll_device *dpll, void *dpll_priv,
 }
 
 static int
-zl3073x_dpll_supported_modes_get(const struct dpll_device *dpll,
-				 void *dpll_priv, unsigned long *modes,
-				 struct netlink_ext_ack *extack)
+zl3073x_dpll_lock_status_get(const struct dpll_device *dpll, void *dpll_priv,
+			     enum dpll_lock_status *status,
+			     enum dpll_lock_status_error *status_error,
+			     struct netlink_ext_ack *extack)
 {
 	struct zl3073x_dpll *zldpll = dpll_priv;
-	const struct zl3073x_chan *chan;
 
-	chan = zl3073x_chan_state_get(zldpll->dev, zldpll->id);
+	guard(mutex)(&zldpll->lock);
 
-	/* We support switching between automatic and manual mode, except in
-	 * a case where the DPLL channel is configured to run in NCO mode.
-	 * In this case, report only the manual mode to which the NCO is mapped
-	 * as the only supported one.
-	 */
-	if (zl3073x_chan_mode_get(chan) != ZL_DPLL_MODE_REFSEL_MODE_NCO)
-		__set_bit(DPLL_MODE_AUTOMATIC, modes);
+	return __zl3073x_dpll_lock_status_get(zldpll, status);
+}
 
+static int
+zl3073x_dpll_supported_modes_get(const struct dpll_device *dpll,
+				 void *dpll_priv, unsigned long *modes,
+				 struct netlink_ext_ack *extack)
+{
+	__set_bit(DPLL_MODE_AUTOMATIC, modes);
 	__set_bit(DPLL_MODE_MANUAL, modes);
 
 	return 0;
@@ -1081,6 +1308,8 @@ zl3073x_dpll_mode_get(const struct dpll_device *dpll, void *dpll_priv,
 	struct zl3073x_dpll *zldpll = dpll_priv;
 	const struct zl3073x_chan *chan;
 
+	guard(mutex)(&zldpll->lock);
+
 	chan = zl3073x_chan_state_get(zldpll->dev, zldpll->id);
 
 	switch (zl3073x_chan_mode_get(chan)) {
@@ -1137,8 +1366,8 @@ zl3073x_dpll_phase_offset_avg_factor_set(const struct dpll_device *dpll,
 		return rc;
 	}
 
-	/* The averaging factor is common for all DPLL channels so after change
-	 * we have to send a notification for other DPLL devices.
+	/* The averaging factor is common for all DPLL channels so after
+	 * change we have to send a notification for other DPLL devices.
 	 */
 	list_for_each_entry(item, &zldpll->dev->dplls, list) {
 		struct dpll_device *dpll_dev = READ_ONCE(item->dpll_dev);
@@ -1155,10 +1384,13 @@ zl3073x_dpll_mode_set(const struct dpll_device *dpll, void *dpll_priv,
 		      enum dpll_mode mode, struct netlink_ext_ack *extack)
 {
 	struct zl3073x_dpll *zldpll = dpll_priv;
+	struct zl3073x_dpll_pin *nco_pin = NULL;
 	struct zl3073x_chan chan;
 	u8 hw_mode, ref;
 	int rc;
 
+	mutex_lock(&zldpll->lock);
+
 	chan = *zl3073x_chan_state_get(zldpll->dev, zldpll->id);
 	ref = zl3073x_chan_refsel_ref_get(&chan);
 
@@ -1178,6 +1410,9 @@ zl3073x_dpll_mode_set(const struct dpll_device *dpll, void *dpll_priv,
 		else
 			hw_mode = ZL_DPLL_MODE_REFSEL_MODE_HOLDOVER;
 	} else {
+		if (zl3073x_chan_mode_is_nco(&chan))
+			nco_pin = zl3073x_dpll_nco_pin_get(zldpll);
+
 		/* We are switching from manual to automatic mode:
 		 * - if there is a valid reference selected then ensure that
 		 *   it is selectable after switch to automatic mode
@@ -1206,9 +1441,18 @@ zl3073x_dpll_mode_set(const struct dpll_device *dpll, void *dpll_priv,
 	if (rc) {
 		NL_SET_ERR_MSG_MOD(extack,
 				   "failed to set reference selection mode");
+		mutex_unlock(&zldpll->lock);
 		return rc;
 	}
 
+	mutex_unlock(&zldpll->lock);
+
+	/* If leaving NCO mode, notify userspace about the NCO pin
+	 * state change - the periodic worker skips the NCO pin.
+	 */
+	if (nco_pin)
+		__dpll_pin_change_ntf(nco_pin->dpll_pin);
+
 	return 0;
 }
 
@@ -1220,6 +1464,8 @@ zl3073x_dpll_phase_offset_monitor_get(const struct dpll_device *dpll,
 {
 	struct zl3073x_dpll *zldpll = dpll_priv;
 
+	guard(mutex)(&zldpll->lock);
+
 	if (zldpll->phase_monitor)
 		*state = DPLL_FEATURE_STATE_ENABLE;
 	else
@@ -1236,6 +1482,8 @@ zl3073x_dpll_phase_offset_monitor_set(const struct dpll_device *dpll,
 {
 	struct zl3073x_dpll *zldpll = dpll_priv;
 
+	guard(mutex)(&zldpll->lock);
+
 	zldpll->phase_monitor = (state == DPLL_FEATURE_STATE_ENABLE);
 
 	return 0;
@@ -1249,7 +1497,7 @@ zl3073x_dpll_freq_monitor_get(const struct dpll_device *dpll,
 {
 	struct zl3073x_dpll *zldpll = dpll_priv;
 
-	if (zldpll->dev->freq_monitor)
+	if (READ_ONCE(zldpll->dev->freq_monitor))
 		*state = DPLL_FEATURE_STATE_ENABLE;
 	else
 		*state = DPLL_FEATURE_STATE_DISABLE;
@@ -1264,13 +1512,14 @@ zl3073x_dpll_freq_monitor_set(const struct dpll_device *dpll,
 			      struct netlink_ext_ack *extack)
 {
 	struct zl3073x_dpll *item, *zldpll = dpll_priv;
+	struct zl3073x_dev *zldev = zldpll->dev;
 
-	zldpll->dev->freq_monitor = (state == DPLL_FEATURE_STATE_ENABLE);
+	WRITE_ONCE(zldev->freq_monitor, state == DPLL_FEATURE_STATE_ENABLE);
 
 	/* The frequency monitoring is common for all DPLL channels so after
 	 * change we have to send a notification for other DPLL devices.
 	 */
-	list_for_each_entry(item, &zldpll->dev->dplls, list) {
+	list_for_each_entry(item, &zldev->dplls, list) {
 		struct dpll_device *dpll_dev = READ_ONCE(item->dpll_dev);
 
 		if (item != zldpll && dpll_dev)
@@ -1312,6 +1561,15 @@ static const struct dpll_pin_ops zl3073x_dpll_output_pin_ops = {
 	.state_on_dpll_get = zl3073x_dpll_output_pin_state_on_dpll_get,
 };
 
+static const struct dpll_pin_ops zl3073x_dpll_nco_pin_ops = {
+	.supported_ffo = BIT(DPLL_FFO_PIN_DEVICE),
+	.direction_get = zl3073x_dpll_pin_direction_get,
+	.ffo_get = zl3073x_dpll_nco_pin_ffo_get,
+	.operstate_on_dpll_get = zl3073x_dpll_nco_pin_operstate_on_dpll_get,
+	.state_on_dpll_get = zl3073x_dpll_nco_pin_state_on_dpll_get,
+	.state_on_dpll_set = zl3073x_dpll_nco_pin_state_on_dpll_set,
+};
+
 static const struct dpll_device_ops zl3073x_dpll_device_ops = {
 	.lock_status_get = zl3073x_dpll_lock_status_get,
 	.mode_get = zl3073x_dpll_mode_get,
@@ -1459,7 +1717,9 @@ zl3073x_dpll_pin_unregister(struct zl3073x_dpll_pin *pin)
 
 	WARN(!pin->dpll_pin, "DPLL pin is not registered\n");
 
-	if (zl3073x_dpll_is_input_pin(pin))
+	if (zl3073x_dpll_is_nco_pin(pin))
+		ops = &zl3073x_dpll_nco_pin_ops;
+	else if (zl3073x_dpll_is_input_pin(pin))
 		ops = &zl3073x_dpll_input_pin_ops;
 	else
 		ops = &zl3073x_dpll_output_pin_ops;
@@ -1512,20 +1772,13 @@ zl3073x_dpll_pin_is_registrable(struct zl3073x_dpll *zldpll,
 				enum dpll_pin_direction dir, u8 index)
 {
 	struct zl3073x_dev *zldev = zldpll->dev;
-	const struct zl3073x_chan *chan;
 	bool is_diff, is_enabled;
 	const char *name;
 
-	chan = zl3073x_chan_state_get(zldev, zldpll->id);
-
 	if (dir == DPLL_PIN_DIRECTION_INPUT) {
 		u8 ref_id = zl3073x_input_pin_ref_get(index);
 		const struct zl3073x_ref *ref;
 
-		/* Skip the pin if the DPLL is running in NCO mode */
-		if (zl3073x_chan_mode_get(chan) == ZL_DPLL_MODE_REFSEL_MODE_NCO)
-			return false;
-
 		name = "REF";
 		ref = zl3073x_ref_state_get(zldev, ref_id);
 		is_diff = zl3073x_ref_is_diff(ref);
@@ -1566,6 +1819,66 @@ zl3073x_dpll_pin_is_registrable(struct zl3073x_dpll *zldpll,
 	return true;
 }
 
+static const struct dpll_pin_properties zl3073x_dpll_nco_pin_props = {
+	.type = DPLL_PIN_TYPE_INT_NCO,
+	.package_label = "NCO",
+	.capabilities = DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE,
+};
+
+static int
+zl3073x_dpll_nco_pin_register(struct zl3073x_dpll *zldpll)
+{
+	struct zl3073x_dpll_pin *pin;
+	struct zl3073x_chan chan;
+	int rc;
+
+	/* Ensure that ctrl bits are configured for NCO operation:
+	 * - nco_auto_read: auto-capture tracking offset on NCO entry
+	 * - tod_step_reset: apply negated ToD step on NCO exit
+	 * - tie_clear: PPS DPLLs re-align outputs on NCO exit
+	 */
+	mutex_lock(&zldpll->lock);
+	chan = *zl3073x_chan_state_get(zldpll->dev, zldpll->id);
+	FIELD_MODIFY(ZL_DPLL_CTRL_NCO_AUTO_READ, &chan.ctrl, 1);
+	FIELD_MODIFY(ZL_DPLL_CTRL_TOD_STEP_RST, &chan.ctrl, 1);
+	FIELD_MODIFY(ZL_DPLL_CTRL_TIE_CLEAR, &chan.ctrl,
+		     zldpll->type == DPLL_TYPE_PPS ? 1 : 0);
+	rc = zl3073x_chan_state_set(zldpll->dev, zldpll->id, &chan);
+	mutex_unlock(&zldpll->lock);
+	if (rc)
+		return rc;
+
+	pin = zl3073x_dpll_pin_alloc(zldpll, DPLL_PIN_DIRECTION_INPUT,
+				     ZL3073X_NCO_PIN_ID);
+	if (IS_ERR(pin))
+		return PTR_ERR(pin);
+
+	pin->dpll_pin = dpll_pin_get(zldpll->dev->clock_id, ZL3073X_NCO_PIN_ID,
+				     THIS_MODULE, &zl3073x_dpll_nco_pin_props,
+				     &pin->tracker);
+	if (IS_ERR(pin->dpll_pin)) {
+		rc = PTR_ERR(pin->dpll_pin);
+		goto err_pin_get;
+	}
+
+	rc = dpll_pin_register(zldpll->dpll_dev, pin->dpll_pin,
+			       &zl3073x_dpll_nco_pin_ops, pin);
+	if (rc)
+		goto err_register;
+
+	list_add(&pin->list, &zldpll->pins);
+
+	return 0;
+
+err_register:
+	dpll_pin_put(pin->dpll_pin, &pin->tracker);
+err_pin_get:
+	pin->dpll_pin = NULL;
+	kfree(pin);
+
+	return rc;
+}
+
 /**
  * zl3073x_dpll_pins_register - register all registerable DPLL pins
  * @zldpll: pointer to zl3073x_dpll structure
@@ -1613,6 +1926,11 @@ zl3073x_dpll_pins_register(struct zl3073x_dpll *zldpll)
 		list_add(&pin->list, &zldpll->pins);
 	}
 
+	/* Register NCO virtual input pin */
+	rc = zl3073x_dpll_nco_pin_register(zldpll);
+	if (rc)
+		goto error;
+
 	return 0;
 
 error:
@@ -1648,8 +1966,8 @@ zl3073x_dpll_device_register(struct zl3073x_dpll *zldpll)
 		return rc;
 	}
 
-	rc = dpll_device_register(zldpll->dpll_dev,
-				  zl3073x_prop_dpll_type_get(zldev, zldpll->id),
+	zldpll->type = zl3073x_prop_dpll_type_get(zldev, zldpll->id);
+	rc = dpll_device_register(zldpll->dpll_dev, zldpll->type,
 				  &zldpll->ops, zldpll);
 	if (rc) {
 		dpll_device_put(zldpll->dpll_dev, &zldpll->tracker);
@@ -1696,6 +2014,8 @@ zl3073x_dpll_pin_phase_offset_check(struct zl3073x_dpll_pin *pin)
 	u8 ref_id;
 	int rc;
 
+	lockdep_assert_held(&zldpll->lock);
+
 	/* No phase offset if the ref monitor reports signal errors */
 	ref_id = zl3073x_input_pin_ref_get(pin->id);
 	if (!zl3073x_dev_ref_is_status_ok(zldev, ref_id))
@@ -1752,16 +2072,27 @@ zl3073x_dpll_pin_ffo_check(struct zl3073x_dpll_pin *pin)
 	const struct zl3073x_chan *chan;
 	s64 ffo;
 
+	lockdep_assert_held(&zldpll->lock);
+
 	if (pin->operstate != DPLL_PIN_OPERSTATE_ACTIVE)
 		return false;
 
 	chan = zl3073x_chan_state_get(zldpll->dev, zldpll->id);
+	if (zl3073x_chan_df_offset_get(chan) == ZL_DPLL_DF_OFFSET_UNKNOWN)
+		return false;
+
+	/* dpll_df_offset register has inverted sign per datasheet:
+	 * f_offset = f_nom * (-df_offset) / 2^48
+	 * Input pin FFO is pin-vs-DPLL (opposite of DPLL-vs-reference),
+	 * so the two inversions cancel out: ppt = df * 5^12 / 2^36
+	 */
 	ffo = mul_s64_u64_shr(zl3073x_chan_df_offset_get(chan),
 			      244140625, 36);
 
-	if (atomic64_xchg(&pin->freq_offset, ffo) != ffo) {
+	if (pin->freq_offset != ffo) {
 		dev_dbg(zldev->dev, "%s freq offset changed to: %lld\n",
 			pin->label, ffo);
+		pin->freq_offset = ffo;
 		return true;
 	}
 
@@ -1786,7 +2117,9 @@ zl3073x_dpll_pin_measured_freq_check(struct zl3073x_dpll_pin *pin)
 	u8 ref_id;
 	u32 freq;
 
-	if (!zldpll->dev->freq_monitor)
+	lockdep_assert_held(&zldpll->lock);
+
+	if (!READ_ONCE(zldpll->dev->freq_monitor))
 		return false;
 
 	ref_id = zl3073x_input_pin_ref_get(pin->id);
@@ -1816,27 +2149,37 @@ zl3073x_dpll_pin_measured_freq_check(struct zl3073x_dpll_pin *pin)
 void
 zl3073x_dpll_changes_check(struct zl3073x_dpll *zldpll)
 {
+	DECLARE_BITMAP(changed_pins, ZL3073X_NUM_INPUT_PINS);
 	struct zl3073x_dev *zldev = zldpll->dev;
 	enum dpll_lock_status lock_status;
 	struct device *dev = zldev->dev;
 	struct zl3073x_dpll_pin *pin;
+	bool dev_changed = false;
 	int rc;
 
+	bitmap_zero(changed_pins, ZL3073X_NUM_INPUT_PINS);
+
+	mutex_lock(&zldpll->lock);
+
 	zldpll->check_count++;
 
-	/* Get current lock status for the DPLL */
-	rc = zl3073x_dpll_lock_status_get(zldpll->dpll_dev, zldpll,
-					  &lock_status, NULL, NULL);
+	rc = zl3073x_chan_state_update(zldev, zldpll->id);
+	if (rc) {
+		dev_err(dev, "Failed to get DPLL%u state: %pe\n",
+			zldpll->id, ERR_PTR(rc));
+		goto unlock;
+	}
+
+	rc = __zl3073x_dpll_lock_status_get(zldpll, &lock_status);
 	if (rc) {
 		dev_err(dev, "Failed to get DPLL%u lock status: %pe\n",
 			zldpll->id, ERR_PTR(rc));
-		return;
+		goto unlock;
 	}
 
-	/* If lock status was changed then notify DPLL core */
 	if (zldpll->lock_status != lock_status) {
 		zldpll->lock_status = lock_status;
-		dpll_device_change_ntf(zldpll->dpll_dev);
+		dev_changed = true;
 	}
 
 	/* Update phase offset latch registers for this DPLL if the phase
@@ -1848,18 +2191,16 @@ zl3073x_dpll_changes_check(struct zl3073x_dpll *zldpll)
 			dev_err(zldev->dev,
 				"Failed to update phase offsets: %pe\n",
 				ERR_PTR(rc));
-			return;
+			goto unlock;
 		}
 	}
 
 	list_for_each_entry(pin, &zldpll->pins, list) {
 		enum dpll_pin_operstate operstate;
-		bool pin_changed = false;
 
-		/* Output pins change checks are not necessary because output
-		 * states are constant.
-		 */
-		if (!zl3073x_dpll_is_input_pin(pin))
+		/* Only physical input pins need monitoring */
+		if (!zl3073x_dpll_is_input_pin(pin) ||
+		    zl3073x_dpll_is_nco_pin(pin))
 			continue;
 
 		rc = zl3073x_dpll_ref_operstate_get(pin, &operstate);
@@ -1867,31 +2208,41 @@ zl3073x_dpll_changes_check(struct zl3073x_dpll *zldpll)
 			dev_err(dev,
 				"Failed to get %s on DPLL%u oper state: %pe\n",
 				pin->label, zldpll->id, ERR_PTR(rc));
-			return;
+			goto unlock;
 		}
 
 		if (operstate != pin->operstate) {
 			dev_dbg(dev, "%s oper state changed: %u->%u\n",
 				pin->label, pin->operstate, operstate);
 			pin->operstate = operstate;
-			pin_changed = true;
+			set_bit(pin->id, changed_pins);
 		}
 
-		/* Check for phase offset, ffo, and measured freq change
-		 * once per second.
-		 */
 		if (zldpll->check_count % 2 == 0) {
 			if (zl3073x_dpll_pin_phase_offset_check(pin))
-				pin_changed = true;
+				set_bit(pin->id, changed_pins);
 
 			if (zl3073x_dpll_pin_ffo_check(pin))
-				pin_changed = true;
+				set_bit(pin->id, changed_pins);
 
 			if (zl3073x_dpll_pin_measured_freq_check(pin))
-				pin_changed = true;
+				set_bit(pin->id, changed_pins);
 		}
+	}
+
+unlock:
+	mutex_unlock(&zldpll->lock);
 
-		if (pin_changed)
+	/* Send notifications outside the lock to avoid ABBA deadlock
+	 * with dpll_lock taken by notification functions.
+	 */
+	if (dev_changed)
+		dpll_device_change_ntf(zldpll->dpll_dev);
+
+	list_for_each_entry(pin, &zldpll->pins, list) {
+		if (zl3073x_dpll_is_input_pin(pin) &&
+		    !zl3073x_dpll_is_nco_pin(pin) &&
+		    test_bit(pin->id, changed_pins))
 			dpll_pin_change_ntf(pin->dpll_pin);
 	}
 }
@@ -1948,6 +2299,7 @@ zl3073x_dpll_alloc(struct zl3073x_dev *zldev, u8 ch)
 
 	zldpll->dev = zldev;
 	zldpll->id = ch;
+	mutex_init(&zldpll->lock);
 	INIT_LIST_HEAD(&zldpll->pins);
 
 	return zldpll;
@@ -1964,6 +2316,7 @@ zl3073x_dpll_free(struct zl3073x_dpll *zldpll)
 {
 	WARN(zldpll->dpll_dev, "DPLL device is still registered\n");
 
+	mutex_destroy(&zldpll->lock);
 	kfree(zldpll);
 }
 
diff --git a/drivers/dpll/zl3073x/dpll.h b/drivers/dpll/zl3073x/dpll.h
index 21adcc18e45e1..faebc402ba1b7 100644
--- a/drivers/dpll/zl3073x/dpll.h
+++ b/drivers/dpll/zl3073x/dpll.h
@@ -18,6 +18,8 @@
  * @ops: DPLL device operations for this instance
  * @dpll_dev: pointer to registered DPLL device
  * @tracker: tracking object for the acquired reference
+ * @lock: per-DPLL mutex serializing all operations
+ * @type: DPLL type (PPS or EEC)
  * @lock_status: last saved DPLL lock status
  * @pins: list of pins
  */
@@ -30,6 +32,8 @@ struct zl3073x_dpll {
 	struct dpll_device_ops		ops;
 	struct dpll_device		*dpll_dev;
 	dpll_tracker			tracker;
+	struct mutex			lock;
+	enum dpll_type			type;
 	enum dpll_lock_status		lock_status;
 	struct list_head		pins;
 };
diff --git a/drivers/dpll/zl3073x/regs.h b/drivers/dpll/zl3073x/regs.h
index 9578f00095282..b70ead7d4495b 100644
--- a/drivers/dpll/zl3073x/regs.h
+++ b/drivers/dpll/zl3073x/regs.h
@@ -5,6 +5,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/bits.h>
+#include <linux/limits.h>
 
 /*
  * Hardware limits for ZL3073x chip family
@@ -17,6 +18,7 @@
 #define ZL3073X_NUM_OUTPUT_PINS	(ZL3073X_NUM_OUTS * 2)
 #define ZL3073X_NUM_PINS	(ZL3073X_NUM_INPUT_PINS + \
 				 ZL3073X_NUM_OUTPUT_PINS)
+#define ZL3073X_NCO_PIN_ID	ZL3073X_NUM_PINS
 
 /*
  * Register address structure:
@@ -164,10 +166,18 @@
 #define ZL_DPLL_MODE_REFSEL_MODE_NCO		4
 #define ZL_DPLL_MODE_REFSEL_REF			GENMASK(7, 4)
 
+#define ZL_REG_DPLL_CTRL(_idx)						\
+	ZL_REG_IDX(_idx, 5, 0x05, 1, ZL3073X_MAX_CHANNELS, 4)
+#define ZL_DPLL_CTRL_TIE_CLEAR			BIT(0)
+#define ZL_DPLL_CTRL_TOD_STEP_RST		BIT(2)
+#define ZL_DPLL_CTRL_NCO_AUTO_READ		BIT(7)
+
 #define ZL_REG_DPLL_DF_READ(_idx)					\
 	ZL_REG_IDX(_idx, 5, 0x28, 1, ZL3073X_MAX_CHANNELS, 1)
 #define ZL_DPLL_DF_READ_SEM			BIT(4)
 #define ZL_DPLL_DF_READ_REF_OFST		BIT(3)
+#define ZL_DPLL_DF_READ_CMD			GENMASK(2, 0)
+#define ZL_DPLL_DF_READ_CMD_ACC_I		4
 
 #define ZL_REG_DPLL_MEAS_CTRL			ZL_REG(5, 0x50, 1)
 #define ZL_DPLL_MEAS_CTRL_EN			BIT(0)
@@ -190,6 +200,7 @@
 #define ZL_REG_DPLL_DF_OFFSET_4		ZL_REG(7, 0x00, 6)
 #define ZL_REG_DPLL_DF_OFFSET(_idx)					\
 	((_idx) < 4 ? ZL_REG_DPLL_DF_OFFSET_03(_idx) : ZL_REG_DPLL_DF_OFFSET_4)
+#define ZL_DPLL_DF_OFFSET_UNKNOWN	S64_MIN
 
 /***********************************
  * Register Page 9, Synth and Output
diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c
index 4edd2088c2db6..68846f583eeef 100644
--- a/drivers/edac/altera_edac.c
+++ b/drivers/edac/altera_edac.c
@@ -1507,6 +1507,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device)
 	int edac_idx, rc;
 	struct device_node *np;
 	const struct edac_device_prv_data *prv = &a10_sdmmceccb_data;
+	bool is_s10 = device->edac->is_s10;
 
 	rc = altr_check_ecc_deps(device);
 	if (rc)
@@ -1533,7 +1534,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device)
 	altdev = dci->pvt_info;
 	*altdev = *device;
 
-	if (!devres_open_group(&altdev->ddev, altr_portb_setup, GFP_KERNEL))
+	if (!devres_open_group(device->edac->dev, altr_portb_setup, GFP_KERNEL))
 		return -ENOMEM;
 
 	/* Update PortB specific values */
@@ -1548,21 +1549,20 @@ static int altr_portb_setup(struct altr_edac_device_dev *device)
 
 	/*
 	 * Update the PortB IRQs - A10 has 4, S10 has 2, Index accordingly
-	 *
-	 * FIXME: Instead of ifdefs with different architectures the driver
-	 *        should properly use compatibles.
 	 */
-#ifdef CONFIG_64BIT
-	altdev->sb_irq = irq_of_parse_and_map(np, 1);
-#else
-	altdev->sb_irq = irq_of_parse_and_map(np, 2);
-#endif
+
+	/* Using compatibles to determine the IRQ Index */
+	if (is_s10)
+		altdev->sb_irq = irq_of_parse_and_map(np, 1);
+	else
+		altdev->sb_irq = irq_of_parse_and_map(np, 2);
+
 	if (!altdev->sb_irq) {
 		edac_printk(KERN_ERR, EDAC_DEVICE, "Error PortB SBIRQ alloc\n");
 		rc = -ENODEV;
 		goto err_release_group_1;
 	}
-	rc = devm_request_irq(&altdev->ddev, altdev->sb_irq,
+	rc = devm_request_irq(device->edac->dev, altdev->sb_irq,
 			      prv->ecc_irq_handler, IRQF_TRIGGER_HIGH,
 			      ecc_name, altdev);
 	if (rc) {
@@ -1570,29 +1570,28 @@ static int altr_portb_setup(struct altr_edac_device_dev *device)
 		goto err_release_group_1;
 	}
 
-#ifdef CONFIG_64BIT
-	/* Use IRQ to determine SError origin instead of assigning IRQ */
-	rc = of_property_read_u32_index(np, "interrupts", 1, &altdev->db_irq);
-	if (rc) {
-		edac_printk(KERN_ERR, EDAC_DEVICE,
-			    "Error PortB DBIRQ alloc\n");
-		goto err_release_group_1;
-	}
-#else
-	altdev->db_irq = irq_of_parse_and_map(np, 3);
-	if (!altdev->db_irq) {
-		edac_printk(KERN_ERR, EDAC_DEVICE, "Error PortB DBIRQ alloc\n");
-		rc = -ENODEV;
-		goto err_release_group_1;
-	}
-	rc = devm_request_irq(&altdev->ddev, altdev->db_irq,
-			      prv->ecc_irq_handler, IRQF_TRIGGER_HIGH,
-			      ecc_name, altdev);
-	if (rc) {
-		edac_printk(KERN_ERR, EDAC_DEVICE, "PortB DBERR IRQ error\n");
-		goto err_release_group_1;
+	if (is_s10) {
+		/* Use IRQ to determine SError origin instead of assigning IRQ */
+		rc = of_property_read_u32_index(np, "interrupts", 1, &altdev->db_irq);
+		if (rc) {
+			edac_printk(KERN_ERR, EDAC_DEVICE, "Error PortB DBIRQ alloc\n");
+			goto err_release_group_1;
+		}
+	} else {
+		altdev->db_irq = irq_of_parse_and_map(np, 3);
+		if (!altdev->db_irq) {
+			edac_printk(KERN_ERR, EDAC_DEVICE, "Error PortB DBIRQ alloc\n");
+			rc = -ENODEV;
+			goto err_release_group_1;
+		}
+		rc = devm_request_irq(device->edac->dev, altdev->db_irq,
+				      prv->ecc_irq_handler, IRQF_TRIGGER_HIGH,
+				      ecc_name, altdev);
+		if (rc) {
+			edac_printk(KERN_ERR, EDAC_DEVICE, "PortB DBERR IRQ error\n");
+			goto err_release_group_1;
+		}
 	}
-#endif
 
 	rc = edac_device_add_device(dci);
 	if (rc) {
@@ -1605,13 +1604,13 @@ static int altr_portb_setup(struct altr_edac_device_dev *device)
 
 	list_add(&altdev->next, &altdev->edac->a10_ecc_devices);
 
-	devres_remove_group(&altdev->ddev, altr_portb_setup);
+	devres_remove_group(device->edac->dev, altr_portb_setup);
 
 	return 0;
 
 err_release_group_1:
 	edac_device_free_ctl_info(dci);
-	devres_release_group(&altdev->ddev, altr_portb_setup);
+	devres_release_group(device->edac->dev, altr_portb_setup);
 	edac_printk(KERN_ERR, EDAC_DEVICE,
 		    "%s:Error setting up EDAC device: %d\n", ecc_name, rc);
 	return rc;
@@ -1974,29 +1973,29 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac,
 		goto err_release_group1;
 	}
 
-#ifdef CONFIG_64BIT
-	/* Use IRQ to determine SError origin instead of assigning IRQ */
-	rc = of_property_read_u32_index(np, "interrupts", 0, &altdev->db_irq);
-	if (rc) {
-		edac_printk(KERN_ERR, EDAC_DEVICE,
-			    "Unable to parse DB IRQ index\n");
-		goto err_release_group1;
-	}
-#else
-	altdev->db_irq = irq_of_parse_and_map(np, 1);
-	if (!altdev->db_irq) {
-		edac_printk(KERN_ERR, EDAC_DEVICE, "Error allocating DBIRQ\n");
-		rc = -ENODEV;
-		goto err_release_group1;
-	}
-	rc = devm_request_irq(edac->dev, altdev->db_irq, prv->ecc_irq_handler,
-			      IRQF_TRIGGER_HIGH,
-			      ecc_name, altdev);
-	if (rc) {
-		edac_printk(KERN_ERR, EDAC_DEVICE, "No DBERR IRQ resource\n");
-		goto err_release_group1;
+	if (edac->is_s10) {
+		/* Use IRQ to determine SError origin instead of assigning IRQ */
+		rc = of_property_read_u32_index(np, "interrupts", 0, &altdev->db_irq);
+		if (rc) {
+			edac_printk(KERN_ERR, EDAC_DEVICE,
+				    "Unable to parse DB IRQ index\n");
+			goto err_release_group1;
+		}
+	} else {
+		altdev->db_irq = irq_of_parse_and_map(np, 1);
+		if (!altdev->db_irq) {
+			edac_printk(KERN_ERR, EDAC_DEVICE, "Error allocating DBIRQ\n");
+			rc = -ENODEV;
+			goto err_release_group1;
+		}
+		rc = devm_request_irq(edac->dev, altdev->db_irq, prv->ecc_irq_handler,
+				      IRQF_TRIGGER_HIGH,
+				      ecc_name, altdev);
+		if (rc) {
+			edac_printk(KERN_ERR, EDAC_DEVICE, "No DBERR IRQ resource\n");
+			goto err_release_group1;
+		}
 	}
-#endif
 
 	rc = edac_device_add_device(dci);
 	if (rc) {
@@ -2059,7 +2058,6 @@ static const struct irq_domain_ops a10_eccmgr_ic_ops = {
 /************** Stratix 10 EDAC Double Bit Error Handler ************/
 #define to_a10edac(p, m) container_of(p, struct altr_arria10_edac, m)
 
-#ifdef CONFIG_64BIT
 /* panic routine issues reboot on non-zero panic_timeout */
 extern int panic_timeout;
 
@@ -2106,7 +2104,6 @@ static int s10_edac_dberr_handler(struct notifier_block *this,
 
 	return NOTIFY_DONE;
 }
-#endif
 
 /****************** Arria 10 EDAC Probe Function *********************/
 static int altr_edac_a10_probe(struct platform_device *pdev)
@@ -2122,6 +2119,8 @@ static int altr_edac_a10_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, edac);
 	INIT_LIST_HEAD(&edac->a10_ecc_devices);
 
+	edac->is_s10 = !!device_get_match_data(&pdev->dev);
+
 	edac->ecc_mgr_map =
 		altr_sysmgr_regmap_lookup_by_phandle(pdev->dev.of_node,
 						     "altr,sysmgr-syscon");
@@ -2153,8 +2152,7 @@ static int altr_edac_a10_probe(struct platform_device *pdev)
 	irq_set_chained_handler_and_data(edac->sb_irq,
 					 altr_edac_a10_irq_handler,
 					 edac);
-
-#ifdef CONFIG_64BIT
+	if (edac->is_s10)
 	{
 		int dberror, err_addr;
 
@@ -2177,15 +2175,14 @@ static int altr_edac_a10_probe(struct platform_device *pdev)
 			regmap_write(edac->ecc_mgr_map,
 				     S10_SYSMGR_UE_ADDR_OFST, 0);
 		}
-	}
-#else
-	edac->db_irq = platform_get_irq(pdev, 1);
-	if (edac->db_irq < 0)
-		return edac->db_irq;
+	} else {
+		edac->db_irq = platform_get_irq(pdev, 1);
+		if (edac->db_irq < 0)
+			return edac->db_irq;
 
-	irq_set_chained_handler_and_data(edac->db_irq,
-					 altr_edac_a10_irq_handler, edac);
-#endif
+		irq_set_chained_handler_and_data(edac->db_irq,
+						 altr_edac_a10_irq_handler, edac);
+	}
 
 	for_each_child_of_node(pdev->dev.of_node, child) {
 		if (!of_device_is_available(child))
@@ -2207,7 +2204,7 @@ static int altr_edac_a10_probe(struct platform_device *pdev)
 
 static const struct of_device_id altr_edac_a10_of_match[] = {
 	{ .compatible = "altr,socfpga-a10-ecc-manager" },
-	{ .compatible = "altr,socfpga-s10-ecc-manager" },
+	{ .compatible = "altr,socfpga-s10-ecc-manager", .data = (void *)1 },
 	{},
 };
 MODULE_DEVICE_TABLE(of, altr_edac_a10_of_match);
diff --git a/drivers/edac/altera_edac.h b/drivers/edac/altera_edac.h
index f3e84172caa9c..9387056fd65e3 100644
--- a/drivers/edac/altera_edac.h
+++ b/drivers/edac/altera_edac.h
@@ -394,6 +394,7 @@ struct altr_arria10_edac {
 	struct irq_chip		irq_chip;
 	struct list_head	a10_ecc_devices;
 	struct notifier_block	panic_notifier;
+	bool is_s10;
 };
 
 #endif	/* #ifndef _ALTERA_EDAC_H */
diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c
index c6aa69dbd9fb1..475235c402e88 100644
--- a/drivers/edac/amd64_edac.c
+++ b/drivers/edac/amd64_edac.c
@@ -4173,6 +4173,8 @@ static int __init amd64_edac_init(void)
 		goto err_pci;
 	}
 
+	request_module_nowait("amd_atl");
+
 	/* register stuff with EDAC MCE */
 	if (boot_cpu_data.x86 >= 0x17) {
 		amd_register_ecc_decoder(decode_umc_error);
diff --git a/drivers/edac/debugfs.c b/drivers/edac/debugfs.c
index 8195fc9c9354a..447d0c6200820 100644
--- a/drivers/edac/debugfs.c
+++ b/drivers/edac/debugfs.c
@@ -1,50 +1,9 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
-#include <linux/string_choices.h>
-
 #include "edac_module.h"
 
 static struct dentry *edac_debugfs;
 
-static ssize_t edac_fake_inject_write(struct file *file,
-				      const char __user *data,
-				      size_t count, loff_t *ppos)
-{
-	struct device *dev = file->private_data;
-	struct mem_ctl_info *mci = to_mci(dev);
-	static enum hw_event_mc_err_type type;
-	u16 errcount = mci->fake_inject_count;
-
-	if (!errcount)
-		errcount = 1;
-
-	type = mci->fake_inject_ue ? HW_EVENT_ERR_UNCORRECTED
-				   : HW_EVENT_ERR_CORRECTED;
-
-	printk(KERN_DEBUG
-	       "Generating %d %s fake error%s to %d.%d.%d to test core handling. NOTE: this won't test the driver-specific decoding logic.\n",
-		errcount,
-		(type == HW_EVENT_ERR_UNCORRECTED) ? "UE" : "CE",
-		str_plural(errcount),
-		mci->fake_inject_layer[0],
-		mci->fake_inject_layer[1],
-		mci->fake_inject_layer[2]
-	       );
-	edac_mc_handle_error(type, mci, errcount, 0, 0, 0,
-			     mci->fake_inject_layer[0],
-			     mci->fake_inject_layer[1],
-			     mci->fake_inject_layer[2],
-			     "FAKE ERROR", "for EDAC testing only");
-
-	return count;
-}
-
-static const struct file_operations debug_fake_inject_fops = {
-	.open = simple_open,
-	.write = edac_fake_inject_write,
-	.llseek = generic_file_llseek,
-};
-
 void __init edac_debugfs_init(void)
 {
 	edac_debugfs = debugfs_create_dir("edac", NULL);
@@ -57,29 +16,7 @@ void edac_debugfs_exit(void)
 
 void edac_create_debugfs_nodes(struct mem_ctl_info *mci)
 {
-	struct dentry *parent;
-	char name[80];
-	int i;
-
-	parent = debugfs_create_dir(mci->dev.kobj.name, edac_debugfs);
-
-	for (i = 0; i < mci->n_layers; i++) {
-		sprintf(name, "fake_inject_%s",
-			     edac_layer_name[mci->layers[i].type]);
-		debugfs_create_u8(name, S_IRUGO | S_IWUSR, parent,
-				  &mci->fake_inject_layer[i]);
-	}
-
-	debugfs_create_bool("fake_inject_ue", S_IRUGO | S_IWUSR, parent,
-			    &mci->fake_inject_ue);
-
-	debugfs_create_u16("fake_inject_count", S_IRUGO | S_IWUSR, parent,
-			   &mci->fake_inject_count);
-
-	debugfs_create_file("fake_inject", S_IWUSR, parent, &mci->dev,
-			    &debug_fake_inject_fops);
-
-	mci->debugfs = parent;
+	mci->debugfs = debugfs_create_dir(mci->dev.kobj.name, edac_debugfs);
 }
 
 /* Create a toplevel dir under EDAC's debugfs hierarchy */
diff --git a/drivers/edac/edac_mc_sysfs.c b/drivers/edac/edac_mc_sysfs.c
index c2ed6c696e540..9b4b5582fa9fd 100644
--- a/drivers/edac/edac_mc_sysfs.c
+++ b/drivers/edac/edac_mc_sysfs.c
@@ -129,7 +129,7 @@ static ssize_t dimmdev_location_show(struct device *dev,
 	ssize_t count;
 
 	count = edac_dimm_info_location(dimm, data, PAGE_SIZE);
-	count += scnprintf(data + count, PAGE_SIZE - count, "\n");
+	count += sysfs_emit_at(data, count, "\n");
 
 	return count;
 }
diff --git a/drivers/edac/ie31200_edac.c b/drivers/edac/ie31200_edac.c
index e3bd6436669ba..94ab80197c2e8 100644
--- a/drivers/edac/ie31200_edac.c
+++ b/drivers/edac/ie31200_edac.c
@@ -261,11 +261,11 @@ static void ie31200_clear_error_info(struct mem_ctl_info *mci)
 	 * the ECC error log registers in all memory controllers.
 	 */
 	if (cfg->msr_clear_eccerrlog_offset) {
-		if (wrmsr_safe(cfg->msr_clear_eccerrlog_offset,
-			       cfg->reg_eccerrlog_ce_mask |
-			       cfg->reg_eccerrlog_ce_ovfl_mask |
-			       cfg->reg_eccerrlog_ue_mask |
-			       cfg->reg_eccerrlog_ue_ovfl_mask, 0) < 0)
+		if (wrmsrq_safe(cfg->msr_clear_eccerrlog_offset,
+			        cfg->reg_eccerrlog_ce_mask |
+			        cfg->reg_eccerrlog_ce_ovfl_mask |
+			        cfg->reg_eccerrlog_ue_mask |
+			        cfg->reg_eccerrlog_ue_ovfl_mask) < 0)
 			ie31200_printk(KERN_ERR, "Failed to wrmsr.\n");
 
 		return;
diff --git a/drivers/edac/mce_amd.c b/drivers/edac/mce_amd.c
index bd252cb3c38e7..7aee57a25b689 100644
--- a/drivers/edac/mce_amd.c
+++ b/drivers/edac/mce_amd.c
@@ -806,7 +806,7 @@ amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data)
 	struct mce *m = (struct mce *)data;
 	struct mce_hw_err *err = to_mce_hw_err(m);
 	unsigned int fam = x86_family(m->cpuid);
-	u32 mca_config_lo = 0, dummy;
+	u64 mca_config = 0;
 	int ecc;
 
 	if (m->kflags & MCE_HANDLED_CEC)
@@ -826,9 +826,9 @@ amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data)
 		((m->status & MCI_STATUS_PCC)	? "PCC"	  : "-"));
 
 	if (boot_cpu_has(X86_FEATURE_SMCA)) {
-		rdmsr_safe(MSR_AMD64_SMCA_MCx_CONFIG(m->bank), &mca_config_lo, &dummy);
+		rdmsrq_safe(MSR_AMD64_SMCA_MCx_CONFIG(m->bank), &mca_config);
 
-		if (mca_config_lo & MCI_CONFIG_MCAX)
+		if (mca_config & MCI_CONFIG_MCAX)
 			pr_cont("|%s", ((m->status & MCI_STATUS_TCC) ? "TCC" : "-"));
 
 		pr_cont("|%s", ((m->status & MCI_STATUS_SYNDV) ? "SyndV" : "-"));
@@ -863,7 +863,7 @@ amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data)
 
 		if (m->status & MCI_STATUS_SYNDV) {
 			pr_cont(", Syndrome: 0x%016llx\n", m->synd);
-			if (mca_config_lo & MCI_CONFIG_FRUTEXT) {
+			if (mca_config & MCI_CONFIG_FRUTEXT) {
 				char frutext[17];
 
 				frutext[16] = '\0';
diff --git a/drivers/extcon/extcon-adc-jack.c b/drivers/extcon/extcon-adc-jack.c
index 7e3c9f38297bd..e735f43dcdeb0 100644
--- a/drivers/extcon/extcon-adc-jack.c
+++ b/drivers/extcon/extcon-adc-jack.c
@@ -210,3 +210,4 @@ module_platform_driver(adc_jack_driver);
 MODULE_AUTHOR("MyungJoo Ham <myungjoo.ham@samsung.com>");
 MODULE_DESCRIPTION("ADC Jack extcon driver");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/firewire/ohci.c b/drivers/firewire/ohci.c
index 8153d62c58f06..4ab697014bf4a 100644
--- a/drivers/firewire/ohci.c
+++ b/drivers/firewire/ohci.c
@@ -307,7 +307,7 @@ static bool detect_vt630x_with_asm1083_on_amd_ryzen_machine(const struct pci_dev
 	const struct pci_dev *pcie_to_pci_bridge;
 
 	// Detect any type of AMD Ryzen machine.
-	if (!static_cpu_has(X86_FEATURE_ZEN))
+	if (!cpu_feature_enabled(X86_FEATURE_ZEN))
 		return false;
 
 	// Detect VIA VT6306/6307/6308.
diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig
index 12dc70254842c..544438a03f7be 100644
--- a/drivers/firmware/Kconfig
+++ b/drivers/firmware/Kconfig
@@ -187,32 +187,20 @@ config SYSFB
 	select SCREEN_INFO
 
 config SYSFB_SIMPLEFB
-	bool "Mark VGA/VBE/EFI FB as generic system framebuffer"
+	bool "Mark VGA/VBE/EFI FB as generic system framebuffer (deprecated)"
 	depends on X86 || EFI
 	select SYSFB
 	help
-	  Firmwares often provide initial graphics framebuffers so the BIOS,
+	  Firmware often provides initial graphics framebuffers so the BIOS,
 	  bootloader or kernel can show basic video-output during boot for
-	  user-guidance and debugging. Historically, x86 used the VESA BIOS
-	  Extensions and EFI-framebuffers for this, which are mostly limited
-	  to x86 BIOS or EFI systems.
-	  This option, if enabled, marks VGA/VBE/EFI framebuffers as generic
-	  framebuffers so the new generic system-framebuffer drivers can be
-	  used instead. If the framebuffer is not compatible with the generic
-	  modes, it is advertised as fallback platform framebuffer so legacy
-	  drivers like efifb, vesafb and uvesafb can pick it up.
-	  If this option is not selected, all system framebuffers are always
-	  marked as fallback platform framebuffers as usual.
-
-	  Note: Legacy fbdev drivers, including vesafb, efifb, uvesafb, will
-	  not be able to pick up generic system framebuffers if this option
-	  is selected. You are highly encouraged to enable simplefb as
-	  replacement if you select this option. simplefb can correctly deal
-	  with generic system framebuffers. But you should still keep vesafb
-	  and others enabled as fallback if a system framebuffer is
-	  incompatible with simplefb.
-
-	  If unsure, say Y.
+	  user-guidance and debugging.
+
+	  This option, if enabled, marks VBE/EFI framebuffers as system
+	  framebuffers so the generic simpledrm driver can be used.
+
+	  This option is deprecated and will be removed in the near future. If
+	  unsure, say N and select efidrm, vesadrm instead. The dedicated DRM
+	  drivers provide the same functionality plus additional features.
 
 config TH1520_AON_PROTOCOL
 	tristate "Always-On firmware protocol"
diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c
index 0f468362c2885..8654b3365c9b6 100644
--- a/drivers/firmware/arm_ffa/driver.c
+++ b/drivers/firmware/arm_ffa/driver.c
@@ -32,6 +32,7 @@
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
+#include <linux/minmax.h>
 #include <linux/module.h>
 #include <linux/mm.h>
 #include <linux/mutex.h>
@@ -59,7 +60,9 @@
 	(FIELD_PREP(SENDER_ID_MASK, (s)) | FIELD_PREP(RECEIVER_ID_MASK, (r)))
 
 #define RXTX_MAP_MIN_BUFSZ_MASK	GENMASK(1, 0)
-#define RXTX_MAP_MIN_BUFSZ(x)	((x) & RXTX_MAP_MIN_BUFSZ_MASK)
+#define RXTX_MAP_MAX_BUFSZ_MASK	GENMASK(31, 16)
+#define RXTX_MAP_MIN_BUFSZ(x)	(FIELD_GET(RXTX_MAP_MIN_BUFSZ_MASK, (x)))
+#define RXTX_MAP_MAX_BUFSZ(x)	(FIELD_GET(RXTX_MAP_MAX_BUFSZ_MASK, (x)))
 
 #define FFA_MAX_NOTIFICATIONS		64
 
@@ -713,30 +716,39 @@ ffa_setup_and_transmit(u32 func_id, void *buffer, u32 max_fragsize,
 	struct ffa_composite_mem_region *composite;
 	struct ffa_mem_region_addr_range *constituents;
 	struct ffa_mem_region_attributes *ep_mem_access;
-	u32 idx, frag_len, length, buf_sz = 0, num_entries = sg_nents(args->sg);
+	u32 idx, frag_len, length, buf_sz = 0, num_entries = sg_nents(args->sg), ep_offset;
+	u32 emad_end, emad_size = ffa_emad_size_get(drv_info->version);
 
 	mem_region->tag = args->tag;
 	mem_region->flags = args->flags;
 	mem_region->sender_id = drv_info->vm_id;
 	mem_region->attributes = ffa_memory_attributes_get(func_id);
+
+	ffa_mem_region_additional_setup(drv_info->version, mem_region);
 	composite_offset = ffa_mem_desc_offset(buffer, args->nattrs,
 					       drv_info->version);
+	if (composite_offset + sizeof(*composite) > max_fragsize)
+		return -ENXIO;
 
 	for (idx = 0; idx < args->nattrs; idx++) {
-		ep_mem_access = buffer +
-			ffa_mem_desc_offset(buffer, idx, drv_info->version);
+		ep_offset = ffa_mem_desc_offset(buffer, idx, drv_info->version);
+		if (check_add_overflow(ep_offset, emad_size, &emad_end))
+			return -ENXIO;
+
+		if (emad_end > max_fragsize)
+			return -ENXIO;
+
+		ep_mem_access = buffer + ep_offset;
+		memset(ep_mem_access, 0, emad_size);
 		ep_mem_access->receiver = args->attrs[idx].receiver;
 		ep_mem_access->attrs = args->attrs[idx].attrs;
 		ep_mem_access->composite_off = composite_offset;
-		ep_mem_access->flag = 0;
-		ep_mem_access->reserved = 0;
 		ffa_emad_impdef_value_init(drv_info->version,
 					   ep_mem_access->impdef_val,
 					   args->attrs[idx].impdef_val);
 	}
 	mem_region->handle = 0;
 	mem_region->ep_count = args->nattrs;
-	ffa_mem_region_additional_setup(drv_info->version, mem_region);
 
 	composite = buffer + composite_offset;
 	composite->total_pg_cnt = ffa_get_num_pages_sg(args->sg);
@@ -769,7 +781,7 @@ ffa_setup_and_transmit(u32 func_id, void *buffer, u32 max_fragsize,
 			constituents = buffer;
 		}
 
-		if ((void *)constituents - buffer > max_fragsize) {
+		if ((void *)constituents + sizeof(*constituents) - buffer > max_fragsize) {
 			pr_err("Memory Region Fragment > Tx Buffer size\n");
 			return -EFAULT;
 		}
@@ -778,7 +790,7 @@ ffa_setup_and_transmit(u32 func_id, void *buffer, u32 max_fragsize,
 		constituents->pg_cnt = args->sg->length / FFA_PAGE_SIZE;
 		constituents->reserved = 0;
 		constituents++;
-		frag_len += sizeof(struct ffa_mem_region_addr_range);
+		frag_len += sizeof(*constituents);
 	} while ((args->sg = sg_next(args->sg)));
 
 	return ffa_transmit_fragment(func_id, addr, buf_sz, frag_len,
@@ -1139,7 +1151,7 @@ static int ffa_partition_info_get(const char *uuid_str,
 	uuid_t uuid;
 	struct ffa_partition_info *pbuf;
 
-	if (uuid_parse(uuid_str, &uuid)) {
+	if (!uuid_str || uuid_parse(uuid_str, &uuid)) {
 		pr_err("invalid uuid (%s)\n", uuid_str);
 		return -ENODEV;
 	}
@@ -2101,7 +2113,7 @@ static int ffa_probe(struct platform_device *pdev)
 {
 	int ret;
 	u32 buf_sz;
-	size_t rxtx_bufsz = SZ_4K;
+	size_t rxtx_min_bufsz = SZ_4K, rxtx_max_bufsz = 0, rxtx_bufsz;
 
 	if (IS_BUILTIN(CONFIG_ARM_FFA_TRANSPORT) &&
 	    is_protected_kvm_enabled() && !is_pkvm_initialized())
@@ -2132,15 +2144,18 @@ static int ffa_probe(struct platform_device *pdev)
 	ret = ffa_features(FFA_FN_NATIVE(RXTX_MAP), 0, &buf_sz, NULL);
 	if (!ret) {
 		if (RXTX_MAP_MIN_BUFSZ(buf_sz) == 1)
-			rxtx_bufsz = SZ_64K;
+			rxtx_min_bufsz = SZ_64K;
 		else if (RXTX_MAP_MIN_BUFSZ(buf_sz) == 2)
-			rxtx_bufsz = SZ_16K;
+			rxtx_min_bufsz = SZ_16K;
 		else
-			rxtx_bufsz = SZ_4K;
+			rxtx_min_bufsz = SZ_4K;
+
+		rxtx_max_bufsz = RXTX_MAP_MAX_BUFSZ(buf_sz) * SZ_4K;
+		if (rxtx_max_bufsz != 0 && rxtx_max_bufsz < rxtx_min_bufsz)
+			rxtx_max_bufsz = rxtx_min_bufsz;
 	}
 
-	rxtx_bufsz = PAGE_ALIGN(rxtx_bufsz);
-	drv_info->rxtx_bufsz = rxtx_bufsz;
+	rxtx_bufsz = min_not_zero(PAGE_ALIGN(rxtx_min_bufsz), rxtx_max_bufsz);
 	drv_info->rx_buffer = alloc_pages_exact(rxtx_bufsz, GFP_KERNEL);
 	if (!drv_info->rx_buffer) {
 		ret = -ENOMEM;
@@ -2156,10 +2171,17 @@ static int ffa_probe(struct platform_device *pdev)
 	ret = ffa_rxtx_map(virt_to_phys(drv_info->tx_buffer),
 			   virt_to_phys(drv_info->rx_buffer),
 			   rxtx_bufsz / FFA_PAGE_SIZE);
+	if (ret == -EINVAL && !rxtx_max_bufsz && rxtx_min_bufsz < rxtx_bufsz) {
+		rxtx_bufsz = rxtx_min_bufsz;
+		ret = ffa_rxtx_map(virt_to_phys(drv_info->tx_buffer),
+				   virt_to_phys(drv_info->rx_buffer),
+				   rxtx_bufsz / FFA_PAGE_SIZE);
+	}
 	if (ret) {
 		pr_err("failed to register FFA RxTx buffers\n");
 		goto free_pages;
 	}
+	drv_info->rxtx_bufsz = rxtx_bufsz;
 
 	mutex_init(&drv_info->rx_lock);
 	mutex_init(&drv_info->tx_lock);
diff --git a/drivers/firmware/arm_scmi/Kconfig b/drivers/firmware/arm_scmi/Kconfig
index e3fb36825978e..783c24a20e291 100644
--- a/drivers/firmware/arm_scmi/Kconfig
+++ b/drivers/firmware/arm_scmi/Kconfig
@@ -96,7 +96,7 @@ config ARM_SCMI_POWER_CONTROL
 	  firmware.
 
 	  This driver can also be built as a module.  If so, the module will be
-	  called scmi_power_control. Note this may needed early in boot to catch
-	  early shutdown/reboot SCMI requests.
+	  called scmi_power_control. Note this may be needed early in boot to
+	  catch early shutdown/reboot SCMI requests.
 
 endmenu
diff --git a/drivers/firmware/arm_scmi/bus.c b/drivers/firmware/arm_scmi/bus.c
index 793be9eabaedd..7f06d56e49053 100644
--- a/drivers/firmware/arm_scmi/bus.c
+++ b/drivers/firmware/arm_scmi/bus.c
@@ -7,7 +7,6 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
-#include <linux/atomic.h>
 #include <linux/types.h>
 #include <linux/module.h>
 #include <linux/of.h>
@@ -33,8 +32,8 @@ struct scmi_requested_dev {
 	struct list_head node;
 };
 
-/* Track globally the creation of SCMI SystemPower related devices */
-static atomic_t scmi_syspower_registered = ATOMIC_INIT(0);
+/* Track globally the SCMI SystemPower protocol device. */
+static struct scmi_device *scmi_syspower_registered;
 
 /**
  * scmi_protocol_device_request  - Helper to request a device
@@ -201,21 +200,33 @@ scmi_protocol_table_unregister(const struct scmi_device_id *id_table)
 		scmi_protocol_device_unrequest(entry);
 }
 
-static int scmi_dev_match_by_id_table(struct scmi_device *scmi_dev,
-				      const struct scmi_device_id *id_table)
+static bool scmi_device_is_transport(const struct scmi_device *scmi_dev)
+{
+	return !strncmp(scmi_dev->name, SCMI_TRANSPORT_DEVNAME_PREFIX,
+			strlen(SCMI_TRANSPORT_DEVNAME_PREFIX));
+}
+
+static int __scmi_dev_match_by_id_table(struct scmi_device *scmi_dev,
+					const struct scmi_device_id *id_table,
+					bool skip_transport)
 {
 	if (!id_table || !id_table->name)
 		return 0;
 
-	/* Always skip transport devices from matching */
 	for (; id_table->protocol_id && id_table->name; id_table++)
 		if (id_table->protocol_id == scmi_dev->protocol_id &&
-		    strncmp(scmi_dev->name, "__scmi_transport_device", 23) &&
+		    !(skip_transport && scmi_device_is_transport(scmi_dev)) &&
 		    !strcmp(id_table->name, scmi_dev->name))
 			return 1;
 	return 0;
 }
 
+static int scmi_dev_match_by_id_table(struct scmi_device *scmi_dev,
+				      const struct scmi_device_id *id_table)
+{
+	return __scmi_dev_match_by_id_table(scmi_dev, id_table, true);
+}
+
 static int scmi_dev_match_id(struct scmi_device *scmi_dev,
 			     const struct scmi_driver *scmi_drv)
 {
@@ -235,11 +246,12 @@ static int scmi_match_by_id_table(struct device *dev, const void *data)
 	struct scmi_device *scmi_dev = to_scmi_dev(dev);
 	const struct scmi_device_id *id_table = data;
 
-	return scmi_dev_match_by_id_table(scmi_dev, id_table);
+	return __scmi_dev_match_by_id_table(scmi_dev, id_table, false);
 }
 
-static struct scmi_device *scmi_child_dev_find(struct device *parent,
-					       int prot_id, const char *name)
+/* Returns a device_find_child() reference which must be dropped by caller. */
+static struct scmi_device *
+scmi_child_dev_find_get(struct device *parent, int prot_id, const char *name)
 {
 	struct scmi_device_id id_table[2] = { 0 };
 	struct device *dev;
@@ -251,9 +263,6 @@ static struct scmi_device *scmi_child_dev_find(struct device *parent,
 	if (!dev)
 		return NULL;
 
-	/* Drop the refcnt bumped implicitly by device_find_child */
-	put_device(dev);
-
 	return to_scmi_dev(dev);
 }
 
@@ -391,10 +400,23 @@ void scmi_driver_unregister(struct scmi_driver *driver)
 }
 EXPORT_SYMBOL_GPL(scmi_driver_unregister);
 
+static void scmi_device_release_resources(struct scmi_device *scmi_dev)
+{
+	if (scmi_dev->protocol_id == SCMI_PROTOCOL_SYSTEM)
+		cmpxchg(&scmi_syspower_registered, scmi_dev, NULL);
+
+	if (scmi_dev->id) {
+		ida_free(&scmi_bus_id, scmi_dev->id);
+		scmi_dev->id = 0;
+	}
+}
+
 static void scmi_device_release(struct device *dev)
 {
 	struct scmi_device *scmi_dev = to_scmi_dev(dev);
 
+	scmi_device_release_resources(scmi_dev);
+	of_node_put(dev->of_node);
 	kfree_const(scmi_dev->name);
 	kfree(scmi_dev);
 }
@@ -406,11 +428,9 @@ static void __scmi_device_destroy(struct scmi_device *scmi_dev)
 		 dev_name(&scmi_dev->dev), scmi_dev->protocol_id,
 		 scmi_dev->name);
 
-	if (scmi_dev->protocol_id == SCMI_PROTOCOL_SYSTEM)
-		atomic_set(&scmi_syspower_registered, 0);
-
-	ida_free(&scmi_bus_id, scmi_dev->id);
-	device_unregister(&scmi_dev->dev);
+	device_del(&scmi_dev->dev);
+	scmi_device_release_resources(scmi_dev);
+	put_device(&scmi_dev->dev);
 }
 
 static struct scmi_device *
@@ -419,6 +439,7 @@ __scmi_device_create(struct device_node *np, struct device *parent,
 {
 	int id, retval;
 	struct scmi_device *scmi_dev;
+	bool syspower = (protocol == SCMI_PROTOCOL_SYSTEM);
 
 	/*
 	 * If the same protocol/name device already exist under the same parent
@@ -427,45 +448,41 @@ __scmi_device_create(struct device_node *np, struct device *parent,
 	 * each DT defined protocol at probe time, and the concurrent
 	 * registration of SCMI drivers.
 	 */
-	scmi_dev = scmi_child_dev_find(parent, protocol, name);
-	if (scmi_dev)
+	scmi_dev = scmi_child_dev_find_get(parent, protocol, name);
+	if (scmi_dev) {
+		put_device(&scmi_dev->dev);
 		return scmi_dev;
-
-	/*
-	 * Ignore any possible subsequent failures while creating the device
-	 * since we are doomed anyway at that point; not using a mutex which
-	 * spans across this whole function to keep things simple and to avoid
-	 * to serialize all the __scmi_device_create calls across possibly
-	 * different SCMI server instances (parent)
-	 */
-	if (protocol == SCMI_PROTOCOL_SYSTEM &&
-	    atomic_cmpxchg(&scmi_syspower_registered, 0, 1)) {
-		dev_warn(parent,
-			 "SCMI SystemPower protocol device must be unique !\n");
-		return NULL;
 	}
 
 	scmi_dev = kzalloc_obj(*scmi_dev);
 	if (!scmi_dev)
 		return NULL;
 
-	scmi_dev->name = kstrdup_const(name ?: "unknown", GFP_KERNEL);
-	if (!scmi_dev->name) {
+	scmi_dev->protocol_id = protocol;
+
+	/*
+	 * Reserve the singleton SystemPower protocol device using the device
+	 * pointer itself, so delayed release of an older device cannot clear
+	 * a reservation owned by a newer device.
+	 */
+	if (syspower && cmpxchg(&scmi_syspower_registered, NULL, scmi_dev)) {
+		dev_warn(parent,
+			 "SCMI SystemPower protocol device must be unique !\n");
 		kfree(scmi_dev);
 		return NULL;
 	}
 
+	scmi_dev->name = kstrdup_const(name ?: "unknown", GFP_KERNEL);
+	if (!scmi_dev->name)
+		goto free_dev;
+
 	id = ida_alloc_min(&scmi_bus_id, 1, GFP_KERNEL);
-	if (id < 0) {
-		kfree_const(scmi_dev->name);
-		kfree(scmi_dev);
-		return NULL;
-	}
+	if (id < 0)
+		goto free_name;
 
 	scmi_dev->id = id;
-	scmi_dev->protocol_id = protocol;
 	scmi_dev->dev.parent = parent;
-	device_set_node(&scmi_dev->dev, of_fwnode_handle(np));
+	device_set_node(&scmi_dev->dev, of_fwnode_handle(of_node_get(np)));
 	scmi_dev->dev.bus = &scmi_bus_type;
 	scmi_dev->dev.release = scmi_device_release;
 	dev_set_name(&scmi_dev->dev, "scmi_dev.%d", id);
@@ -479,8 +496,14 @@ __scmi_device_create(struct device_node *np, struct device *parent,
 
 	return scmi_dev;
 put_dev:
+	scmi_device_release_resources(scmi_dev);
 	put_device(&scmi_dev->dev);
-	ida_free(&scmi_bus_id, id);
+	return NULL;
+free_name:
+	kfree_const(scmi_dev->name);
+free_dev:
+	scmi_device_release_resources(scmi_dev);
+	kfree(scmi_dev);
 	return NULL;
 }
 
@@ -561,9 +584,11 @@ void scmi_device_destroy(struct device *parent, int protocol, const char *name)
 {
 	struct scmi_device *scmi_dev;
 
-	scmi_dev = scmi_child_dev_find(parent, protocol, name);
-	if (scmi_dev)
+	scmi_dev = scmi_child_dev_find_get(parent, protocol, name);
+	if (scmi_dev) {
 		__scmi_device_destroy(scmi_dev);
+		put_device(&scmi_dev->dev);
+	}
 }
 EXPORT_SYMBOL_GPL(scmi_device_destroy);
 
diff --git a/drivers/firmware/arm_scmi/clock.c b/drivers/firmware/arm_scmi/clock.c
index 42e666a628c73..0278705d809e8 100644
--- a/drivers/firmware/arm_scmi/clock.c
+++ b/drivers/firmware/arm_scmi/clock.c
@@ -718,7 +718,7 @@ static int scmi_clock_rate_set(const struct scmi_protocol_handle *ph,
 static int scmi_clock_determine_rate(const struct scmi_protocol_handle *ph,
 				     u32 clk_id, unsigned long *rate)
 {
-	u64 fmin, fmax, ftmp;
+	u64 fmin, fmax, ftmp, step;
 	struct scmi_clock_info *clk;
 	struct scmi_clock_desc *clkd;
 	struct clock_info *ci = ph->get_priv(ph);
@@ -749,11 +749,14 @@ static int scmi_clock_determine_rate(const struct scmi_protocol_handle *ph,
 		return 0;
 	}
 
+	step = clkd->r.rates[RATE_STEP];
+	if (!step)
+		return -EINVAL;
+
 	ftmp = *rate - fmin;
-	ftmp += clkd->r.rates[RATE_STEP] - 1; /* to round up */
-	ftmp = div64_ul(ftmp, clkd->r.rates[RATE_STEP]);
+	ftmp = DIV64_U64_ROUND_UP(ftmp, step);
 
-	*rate = ftmp * clkd->r.rates[RATE_STEP] + fmin;
+	*rate = ftmp * step + fmin;
 
 	return 0;
 }
diff --git a/drivers/firmware/arm_scmi/common.h b/drivers/firmware/arm_scmi/common.h
index b9723c105fc1b..fe8c22cfb9f7d 100644
--- a/drivers/firmware/arm_scmi/common.h
+++ b/drivers/firmware/arm_scmi/common.h
@@ -34,6 +34,8 @@
 
 #define SCMI_SHMEM_MAX_PAYLOAD_SIZE	104
 
+#define SCMI_TRANSPORT_DEVNAME_PREFIX	"__scmi_transport_device"
+
 enum scmi_error_codes {
 	SCMI_SUCCESS = 0,	/* Success */
 	SCMI_ERR_SUPPORT = -1,	/* Not supported */
diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c
index 3e0d975ec94c4..ef29fd223287d 100644
--- a/drivers/firmware/arm_scmi/driver.c
+++ b/drivers/firmware/arm_scmi/driver.c
@@ -33,6 +33,7 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/processor.h>
+#include <linux/rcupdate.h>
 #include <linux/refcount.h>
 #include <linux/slab.h>
 #include <linux/xarray.h>
@@ -2628,21 +2629,31 @@ static int scmi_handle_put(const struct scmi_handle *handle)
 	return 0;
 }
 
-static void scmi_device_link_add(struct device *consumer,
+static bool scmi_device_link_add(struct device *consumer,
 				 struct device *supplier)
 {
 	struct device_link *link;
 
 	link = device_link_add(consumer, supplier, DL_FLAG_AUTOREMOVE_CONSUMER);
 
-	WARN_ON(!link);
+	return !WARN_ON(!link);
+}
+
+static void scmi_clear_handle(struct scmi_device *scmi_dev)
+{
+	if (!scmi_dev->handle)
+		return;
+
+	scmi_handle_put(scmi_dev->handle);
+	scmi_dev->handle = NULL;
 }
 
 static void scmi_set_handle(struct scmi_device *scmi_dev)
 {
 	scmi_dev->handle = scmi_handle_get(&scmi_dev->dev);
-	if (scmi_dev->handle)
-		scmi_device_link_add(&scmi_dev->dev, scmi_dev->handle->dev);
+	if (scmi_dev->handle &&
+	    !scmi_device_link_add(&scmi_dev->dev, scmi_dev->handle->dev))
+		scmi_clear_handle(scmi_dev);
 }
 
 static int __scmi_xfer_info_init(struct scmi_info *sinfo,
@@ -2751,6 +2762,9 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node,
 	idx = tx ? 0 : 1;
 	idr = tx ? &info->tx_idr : &info->rx_idr;
 
+	if (idr_find(idr, prot_id))
+		return -EEXIST;
+
 	if (!info->desc->ops->chan_available(of_node, idx)) {
 		cinfo = idr_find(idr, SCMI_PROTOCOL_BASE);
 		if (unlikely(!cinfo)) /* Possible only if platform has no Rx */
@@ -2768,7 +2782,7 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node,
 	cinfo->no_completion_irq = info->desc->no_completion_irq;
 
 	/* Create a unique name for this transport device */
-	snprintf(name, 32, "__scmi_transport_device_%s_%02X",
+	snprintf(name, sizeof(name), SCMI_TRANSPORT_DEVNAME_PREFIX "_%s_%02X",
 		 idx ? "rx" : "tx", prot_id);
 	/* Create a uniquely named, dedicated transport device for this chan */
 	tdev = scmi_device_create(of_node, info->dev, prot_id, name);
@@ -2778,13 +2792,12 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node,
 		devm_kfree(info->dev, cinfo);
 		return -EINVAL;
 	}
-	of_node_get(of_node);
 
 	cinfo->id = prot_id;
 	cinfo->dev = &tdev->dev;
+	cinfo->handle = &info->handle;
 	ret = info->desc->ops->chan_setup(cinfo, info->dev, tx);
 	if (ret) {
-		of_node_put(of_node);
 		scmi_device_destroy(info->dev, prot_id, name);
 		devm_kfree(info->dev, cinfo);
 		return ret;
@@ -2807,14 +2820,13 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node,
 			"unable to allocate SCMI idr slot err %d\n", ret);
 		/* Destroy channel and device only if created by this call. */
 		if (tdev) {
-			of_node_put(of_node);
+			info->desc->ops->chan_free(prot_id, cinfo, idr);
 			scmi_device_destroy(info->dev, prot_id, name);
 			devm_kfree(info->dev, cinfo);
 		}
 		return ret;
 	}
 
-	cinfo->handle = &info->handle;
 	return 0;
 }
 
@@ -2872,9 +2884,11 @@ static int scmi_channels_setup(struct scmi_info *info)
 		if (of_property_read_u32(child, "reg", &prot_id))
 			continue;
 
-		if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id))
+		if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) {
 			dev_err(info->dev,
 				"Out of range protocol %d\n", prot_id);
+			continue;
+		}
 
 		ret = scmi_txrx_setup(info, child, prot_id);
 		if (ret)
@@ -2884,7 +2898,7 @@ static int scmi_channels_setup(struct scmi_info *info)
 	return 0;
 }
 
-static int scmi_chan_destroy(int id, void *p, void *idr)
+static int scmi_chan_destroy(int id, void *p, void *data)
 {
 	struct scmi_chan_info *cinfo = p;
 
@@ -2892,13 +2906,10 @@ static int scmi_chan_destroy(int id, void *p, void *idr)
 		struct scmi_info *info = handle_to_scmi_info(cinfo->handle);
 		struct scmi_device *sdev = to_scmi_dev(cinfo->dev);
 
-		of_node_put(cinfo->dev->of_node);
-		scmi_device_destroy(info->dev, id, sdev->name);
+		scmi_device_destroy(info->dev, cinfo->id, sdev->name);
 		cinfo->dev = NULL;
 	}
 
-	idr_remove(idr, id);
-
 	return 0;
 }
 
@@ -2925,6 +2936,7 @@ static int scmi_bus_notifier(struct notifier_block *nb,
 {
 	struct scmi_info *info = bus_nb_to_scmi_info(nb);
 	struct scmi_device *sdev = to_scmi_dev(data);
+	const char *status;
 
 	/* Skip devices of different SCMI instances */
 	if (sdev->dev.parent != info->dev)
@@ -2934,18 +2946,22 @@ static int scmi_bus_notifier(struct notifier_block *nb,
 	case BUS_NOTIFY_BIND_DRIVER:
 		/* setup handle now as the transport is ready */
 		scmi_set_handle(sdev);
+		status = "about to be BOUND.";
+		break;
+	case BUS_NOTIFY_DRIVER_NOT_BOUND:
+		scmi_clear_handle(sdev);
+		status = "NOT BOUND.";
 		break;
 	case BUS_NOTIFY_UNBOUND_DRIVER:
-		scmi_handle_put(sdev->handle);
-		sdev->handle = NULL;
+		scmi_clear_handle(sdev);
+		status = "UNBOUND.";
 		break;
 	default:
 		return NOTIFY_DONE;
 	}
 
 	dev_dbg(info->dev, "Device %s (%s) is now %s\n", dev_name(&sdev->dev),
-		sdev->name, action == BUS_NOTIFY_BIND_DRIVER ?
-		"about to be BOUND." : "UNBOUND.");
+		sdev->name, status);
 
 	return NOTIFY_OK;
 }
@@ -2957,7 +2973,9 @@ static int scmi_device_request_notifier(struct notifier_block *nb,
 	struct scmi_device_id *id_table = data;
 	struct scmi_info *info = req_nb_to_scmi_info(nb);
 
+	rcu_read_lock();
 	np = idr_find(&info->active_protocols, id_table->protocol_id);
+	rcu_read_unlock();
 	if (!np)
 		return NOTIFY_DONE;
 
@@ -3263,7 +3281,7 @@ static int scmi_probe(struct platform_device *pdev)
 	ret = scmi_channels_setup(info);
 	if (ret) {
 		err_str = "failed to setup channels\n";
-		goto clear_ida;
+		goto clear_txrx_setup;
 	}
 
 	ret = bus_register_notifier(&scmi_bus_type, &info->bus_nb);
@@ -3325,7 +3343,7 @@ static int scmi_probe(struct platform_device *pdev)
 			dev_err(dev, "%s", err_str);
 			return 0;
 		}
-		goto notification_exit;
+		goto raw_mode_cleanup;
 	}
 
 	mutex_lock(&scmi_list_mutex);
@@ -3340,8 +3358,10 @@ static int scmi_probe(struct platform_device *pdev)
 		if (of_property_read_u32(child, "reg", &prot_id))
 			continue;
 
-		if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id))
+		if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) {
 			dev_err(dev, "Out of range protocol %d\n", prot_id);
+			continue;
+		}
 
 		if (!scmi_is_protocol_implemented(handle, prot_id)) {
 			dev_err(dev, "SCMI protocol %d not implemented\n",
@@ -3367,18 +3387,18 @@ static int scmi_probe(struct platform_device *pdev)
 
 	return 0;
 
-notification_exit:
+raw_mode_cleanup:
 	if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT))
 		scmi_raw_mode_cleanup(info->raw);
-	scmi_notification_exit(&info->handle);
 clear_dev_req_notifier:
 	blocking_notifier_chain_unregister(&scmi_requested_devices_nh,
 					   &info->dev_req_nb);
 clear_bus_notifier:
 	bus_unregister_notifier(&scmi_bus_type, &info->bus_nb);
 clear_txrx_setup:
+	scmi_notification_quiesce(&info->handle);
 	scmi_cleanup_txrx_channels(info);
-clear_ida:
+	scmi_notification_exit(&info->handle);
 	ida_free(&scmi_id, info->id);
 
 out_err:
@@ -3401,6 +3421,12 @@ static void scmi_remove(struct platform_device *pdev)
 	list_del(&info->node);
 	mutex_unlock(&scmi_list_mutex);
 
+	blocking_notifier_chain_unregister(&scmi_requested_devices_nh,
+					   &info->dev_req_nb);
+
+	/* Stop transport callbacks before tearing down notifications. */
+	scmi_notification_quiesce(&info->handle);
+	scmi_cleanup_txrx_channels(info);
 	scmi_notification_exit(&info->handle);
 
 	mutex_lock(&info->protocols_mtx);
@@ -3411,13 +3437,8 @@ static void scmi_remove(struct platform_device *pdev)
 		of_node_put(child);
 	idr_destroy(&info->active_protocols);
 
-	blocking_notifier_chain_unregister(&scmi_requested_devices_nh,
-					   &info->dev_req_nb);
 	bus_unregister_notifier(&scmi_bus_type, &info->bus_nb);
 
-	/* Safe to free channels since no more users */
-	scmi_cleanup_txrx_channels(info);
-
 	ida_free(&scmi_id, info->id);
 }
 
diff --git a/drivers/firmware/arm_scmi/notify.c b/drivers/firmware/arm_scmi/notify.c
index 40ec184eedaec..fb60fb07c0ed1 100644
--- a/drivers/firmware/arm_scmi/notify.c
+++ b/drivers/firmware/arm_scmi/notify.c
@@ -209,11 +209,11 @@ struct scmi_registered_events_desc;
  * @init_work: A work item to perform final initializations of pending handlers
  * @notify_wq: A reference to the allocated Kernel cmwq
  * @pending_mtx: A mutex to protect @pending_events_handlers
+ * @pending_events_handlers: An hashtable containing all pending events'
+ *			     handlers descriptors
  * @registered_protocols: A statically allocated array containing pointers to
  *			  all the registered protocol-level specific information
  *			  related to events' handling
- * @pending_events_handlers: An hashtable containing all pending events'
- *			     handlers descriptors
  *
  * Each platform instance, represented by a handle, has its own instance of
  * the notification subsystem represented by this structure.
@@ -225,8 +225,8 @@ struct scmi_notify_instance {
 	struct workqueue_struct	*notify_wq;
 	/* lock to protect pending_events_handlers */
 	struct mutex		pending_mtx;
-	struct scmi_registered_events_desc	**registered_protocols;
 	DECLARE_HASHTABLE(pending_events_handlers, SCMI_PENDING_HASH_SZ);
+	struct scmi_registered_events_desc	*registered_protocols[SCMI_MAX_PROTO];
 };
 
 /**
@@ -276,13 +276,13 @@ struct scmi_registered_event;
  * @eh_sz: Size of the pre-allocated buffer @eh
  * @in_flight: A reference to an in flight &struct scmi_registered_event
  * @num_events: Number of events in @registered_events
- * @registered_events: A dynamically allocated array holding all the registered
- *		       events' descriptors, whose fixed-size is determined at
- *		       compile time.
  * @registered_mtx: A mutex to protect @registered_events_handlers
  * @ph: SCMI protocol handle reference
  * @registered_events_handlers: An hashtable containing all events' handlers
  *				descriptors registered for this protocol
+ * @registered_events: A dynamically allocated array holding all the registered
+ *		       events' descriptors, whose fixed-size is determined at
+ *		       compile time.
  *
  * All protocols that register at least one event have their protocol-specific
  * information stored here, together with the embedded allocated events_queue.
@@ -302,11 +302,11 @@ struct scmi_registered_events_desc {
 	size_t				eh_sz;
 	void				*in_flight;
 	int				num_events;
-	struct scmi_registered_event	**registered_events;
 	/* mutex to protect registered_events_handlers */
 	struct mutex			registered_mtx;
 	const struct scmi_protocol_handle	*ph;
 	DECLARE_HASHTABLE(registered_events_handlers, SCMI_REGISTERED_HASH_SZ);
+	struct scmi_registered_event	*registered_events[] __counted_by(num_events);
 };
 
 /**
@@ -338,9 +338,9 @@ struct scmi_registered_event {
 	void		*report;
 	u32		num_sources;
 	bool		not_supported_by_platform;
-	refcount_t	*sources;
 	/* locking to serialize the access to sources */
 	struct mutex	sources_mtx;
+	refcount_t	sources[] __counted_by(num_sources);
 };
 
 /**
@@ -600,9 +600,9 @@ int scmi_notify(const struct scmi_handle *handle, u8 proto_id, u8 evt_id,
 		return -EINVAL;
 	}
 	if (kfifo_avail(&r_evt->proto->equeue.kfifo) < sizeof(eh) + len) {
-		dev_warn(handle->dev,
-			 "queue full, dropping proto_id:%d  evt_id:%d  ts:%lld\n",
-			 proto_id, evt_id, ktime_to_ns(ts));
+		dev_warn_ratelimited(handle->dev,
+				     "queue full, dropping proto_id:%d  evt_id:%d  ts:%lld\n",
+				     proto_id, evt_id, ktime_to_ns(ts));
 		return -ENOMEM;
 	}
 
@@ -706,9 +706,13 @@ scmi_allocate_registered_events_desc(struct scmi_notify_instance *ni,
 	if (WARN_ON(ni->registered_protocols[proto_id]))
 		return ERR_PTR(-EINVAL);
 
-	pd = devm_kzalloc(ni->handle->dev, sizeof(*pd), GFP_KERNEL);
+	pd = devm_kzalloc(ni->handle->dev,
+			  struct_size(pd, registered_events, num_events),
+			  GFP_KERNEL);
 	if (!pd)
 		return ERR_PTR(-ENOMEM);
+
+	pd->num_events = num_events;
 	pd->id = proto_id;
 	pd->ops = ops;
 	pd->ni = ni;
@@ -722,12 +726,6 @@ scmi_allocate_registered_events_desc(struct scmi_notify_instance *ni,
 		return ERR_PTR(-ENOMEM);
 	pd->eh_sz = eh_sz;
 
-	pd->registered_events = devm_kcalloc(ni->handle->dev, num_events,
-					     sizeof(char *), GFP_KERNEL);
-	if (!pd->registered_events)
-		return ERR_PTR(-ENOMEM);
-	pd->num_events = num_events;
-
 	/* Initialize per protocol handlers table */
 	mutex_init(&pd->registered_mtx);
 	hash_init(pd->registered_events_handlers);
@@ -796,18 +794,16 @@ int scmi_register_protocol_events(const struct scmi_handle *handle, u8 proto_id,
 		int id;
 		struct scmi_registered_event *r_evt;
 
-		r_evt = devm_kzalloc(ni->handle->dev, sizeof(*r_evt),
+		r_evt = devm_kzalloc(ni->handle->dev,
+				     struct_size(r_evt, sources, num_sources),
 				     GFP_KERNEL);
 		if (!r_evt)
 			return -ENOMEM;
+
+		r_evt->num_sources = num_sources;
 		r_evt->proto = pd;
 		r_evt->evt = evt;
 
-		r_evt->sources = devm_kcalloc(ni->handle->dev, num_sources,
-					      sizeof(refcount_t), GFP_KERNEL);
-		if (!r_evt->sources)
-			return -ENOMEM;
-		r_evt->num_sources = num_sources;
 		mutex_init(&r_evt->sources_mtx);
 
 		r_evt->report = devm_kzalloc(ni->handle->dev,
@@ -1673,11 +1669,6 @@ int scmi_notification_init(struct scmi_handle *handle)
 	ni->gid = gid;
 	ni->handle = handle;
 
-	ni->registered_protocols = devm_kcalloc(handle->dev, SCMI_MAX_PROTO,
-						sizeof(char *), GFP_KERNEL);
-	if (!ni->registered_protocols)
-		goto err;
-
 	ni->notify_wq = alloc_workqueue(dev_name(handle->dev),
 					WQ_UNBOUND | WQ_FREEZABLE | WQ_SYSFS,
 					0);
@@ -1706,6 +1697,25 @@ int scmi_notification_init(struct scmi_handle *handle)
 	return -ENOMEM;
 }
 
+/**
+ * scmi_notification_quiesce()  - Stop notification late initialization
+ * @handle: The handle identifying the platform instance to quiesce
+ *
+ * Prevent new late-init work from being queued and wait for any already queued
+ * or running late-init work to complete before transport channels are torn
+ * down.
+ */
+void scmi_notification_quiesce(struct scmi_handle *handle)
+{
+	struct scmi_notify_instance *ni;
+
+	ni = scmi_notification_instance_data_get(handle);
+	if (!ni)
+		return;
+
+	disable_work_sync(&ni->init_work);
+}
+
 /**
  * scmi_notification_exit()  - Shutdown and clean Notification core
  * @handle: The handle identifying the platform instance to shutdown
@@ -1717,6 +1727,8 @@ void scmi_notification_exit(struct scmi_handle *handle)
 	ni = scmi_notification_instance_data_get(handle);
 	if (!ni)
 		return;
+
+	scmi_notification_quiesce(handle);
 	scmi_notification_instance_data_set(handle, NULL);
 
 	/* Destroy while letting pending work complete */
diff --git a/drivers/firmware/arm_scmi/notify.h b/drivers/firmware/arm_scmi/notify.h
index 76758a736cf47..f18f98c5ab3ba 100644
--- a/drivers/firmware/arm_scmi/notify.h
+++ b/drivers/firmware/arm_scmi/notify.h
@@ -82,6 +82,7 @@ struct scmi_protocol_events {
 };
 
 int scmi_notification_init(struct scmi_handle *handle);
+void scmi_notification_quiesce(struct scmi_handle *handle);
 void scmi_notification_exit(struct scmi_handle *handle);
 int scmi_register_protocol_events(const struct scmi_handle *handle, u8 proto_id,
 				  const struct scmi_protocol_handle *ph,
diff --git a/drivers/firmware/arm_scmi/transports/mailbox.c b/drivers/firmware/arm_scmi/transports/mailbox.c
index ae0f67e6cc45f..308736c3ead9c 100644
--- a/drivers/firmware/arm_scmi/transports/mailbox.c
+++ b/drivers/firmware/arm_scmi/transports/mailbox.c
@@ -211,13 +211,18 @@ static int mailbox_chan_setup(struct scmi_chan_info *cinfo, struct device *dev,
 	cl->tx_block = false;
 	cl->knows_txdone = tx;
 
+	cinfo->transport_info = smbox;
+	smbox->cinfo = cinfo;
+	mutex_init(&smbox->chan_lock);
+
 	smbox->chan = mbox_request_channel(cl, tx ? 0 : p2a_chan);
 	if (IS_ERR(smbox->chan)) {
 		ret = PTR_ERR(smbox->chan);
+		smbox->chan = NULL;
 		if (ret != -EPROBE_DEFER)
 			dev_err(cdev,
 				"failed to request SCMI %s mailbox\n", desc);
-		return ret;
+		goto err_clear_cinfo;
 	}
 
 	/* Additional unidirectional channel for TX if needed */
@@ -225,9 +230,10 @@ static int mailbox_chan_setup(struct scmi_chan_info *cinfo, struct device *dev,
 		smbox->chan_receiver = mbox_request_channel(cl, a2p_rx_chan);
 		if (IS_ERR(smbox->chan_receiver)) {
 			ret = PTR_ERR(smbox->chan_receiver);
+			smbox->chan_receiver = NULL;
 			if (ret != -EPROBE_DEFER)
 				dev_err(cdev, "failed to request SCMI Tx Receiver mailbox\n");
-			return ret;
+			goto err_free_chan;
 		}
 	}
 
@@ -235,17 +241,23 @@ static int mailbox_chan_setup(struct scmi_chan_info *cinfo, struct device *dev,
 		smbox->chan_platform_receiver = mbox_request_channel(cl, p2a_rx_chan);
 		if (IS_ERR(smbox->chan_platform_receiver)) {
 			ret = PTR_ERR(smbox->chan_platform_receiver);
+			smbox->chan_platform_receiver = NULL;
 			if (ret != -EPROBE_DEFER)
 				dev_err(cdev, "failed to request SCMI P2A Receiver mailbox\n");
-			return ret;
+			goto err_free_chan;
 		}
 	}
 
-	cinfo->transport_info = smbox;
-	smbox->cinfo = cinfo;
-	mutex_init(&smbox->chan_lock);
-
 	return 0;
+
+err_free_chan:
+	mbox_free_channel(smbox->chan);
+err_clear_cinfo:
+	cinfo->transport_info = NULL;
+	smbox->cinfo = NULL;
+	devm_iounmap(dev, smbox->shmem);
+	devm_kfree(dev, smbox);
+	return ret;
 }
 
 static int mailbox_chan_free(int id, void *p, void *data)
diff --git a/drivers/firmware/arm_scmi/transports/smc.c b/drivers/firmware/arm_scmi/transports/smc.c
index 21abb571e4f2f..1fce3ccdeb7fc 100644
--- a/drivers/firmware/arm_scmi/transports/smc.c
+++ b/drivers/firmware/arm_scmi/transports/smc.c
@@ -172,6 +172,13 @@ static int smc_chan_setup(struct scmi_chan_info *cinfo, struct device *dev,
 		scmi_info->param_page = SHMEM_PAGE(res.start);
 		scmi_info->param_offset = SHMEM_OFFSET(res.start);
 	}
+
+	scmi_info->func_id = func_id;
+	scmi_info->cap_id = cap_id;
+	scmi_info->cinfo = cinfo;
+	smc_channel_lock_init(scmi_info);
+	cinfo->transport_info = scmi_info;
+
 	/*
 	 * If there is an interrupt named "a2p", then the service and
 	 * completion of a message is signaled by an interrupt rather than by
@@ -183,18 +190,14 @@ static int smc_chan_setup(struct scmi_chan_info *cinfo, struct device *dev,
 				  IRQF_NO_SUSPEND, dev_name(dev), scmi_info);
 		if (ret) {
 			dev_err(dev, "failed to setup SCMI smc irq\n");
+			cinfo->transport_info = NULL;
+			scmi_info->cinfo = NULL;
 			return ret;
 		}
 	} else {
 		cinfo->no_completion_irq = true;
 	}
 
-	scmi_info->func_id = func_id;
-	scmi_info->cap_id = cap_id;
-	scmi_info->cinfo = cinfo;
-	smc_channel_lock_init(scmi_info);
-	cinfo->transport_info = scmi_info;
-
 	return 0;
 }
 
diff --git a/drivers/firmware/efi/runtime-wrappers.c b/drivers/firmware/efi/runtime-wrappers.c
index da8d296216441..2344b9d1e81f8 100644
--- a/drivers/firmware/efi/runtime-wrappers.c
+++ b/drivers/firmware/efi/runtime-wrappers.c
@@ -118,6 +118,14 @@ union efi_rts_args {
 
 struct efi_runtime_work efi_rts_work;
 
+/*
+ * Upper bound on how long we wait for a single EFI runtime service
+ * call to finish before declaring firmware wedged. Chosen to be longer
+ * than any plausible legitimate call (including UpdateCapsule on slow
+ * SPI-NOR) while still bounding userspace wait time.
+ */
+#define EFI_RTS_TIMEOUT		(120 * HZ)
+
 /*
  * efi_queue_work:	Queue EFI runtime service call and wait for completion
  * @_rts:		EFI runtime service function identifier
@@ -202,7 +210,7 @@ void efi_call_virt_check_flags(unsigned long flags, const void *caller)
  */
 static DEFINE_SEMAPHORE(efi_runtime_lock, 1);
 
-static struct task_struct *efi_runtime_lock_owner;
+static struct task_struct *efi_runtime_lock_owner __used;
 
 /*
  * Expose the EFI runtime lock to the UV platform
@@ -211,6 +219,19 @@ static struct task_struct *efi_runtime_lock_owner;
 extern struct semaphore __efi_uv_runtime_lock __alias(efi_runtime_lock);
 #endif
 
+/*
+ * Park a worker that must never run efi_rts_wq again: EFI runtime services
+ * have been disabled and its efi_rts_work is abandoned. Loop in schedule()
+ * so a spurious wakeup cannot resume it.
+ */
+void __noreturn efi_rts_park_worker(void)
+{
+	for (;;) {
+		set_current_state(TASK_IDLE);
+		schedule();
+	}
+}
+
 /*
  * Calls the appropriate efi_runtime_service() with the appropriate
  * arguments.
@@ -221,6 +242,9 @@ static void __nocfi efi_call_rts(struct work_struct *work)
 	efi_status_t status = EFI_NOT_FOUND;
 	unsigned long flags;
 
+	if (!efi_enabled(EFI_RUNTIME_SERVICES))
+		efi_rts_park_worker();
+
 	efi_runtime_lock_owner = current;
 
 	arch_efi_call_virt_setup();
@@ -312,6 +336,9 @@ static void __nocfi efi_call_rts(struct work_struct *work)
 	efi_call_virt_check_flags(flags, efi_rts_work.caller);
 	arch_efi_call_virt_teardown();
 
+	if (!efi_enabled(EFI_RUNTIME_SERVICES))
+		efi_rts_park_worker();
+
 	efi_rts_work.status = status;
 	complete(&efi_rts_work.efi_rts_comp);
 	efi_runtime_lock_owner = NULL;
@@ -320,17 +347,16 @@ static void __nocfi efi_call_rts(struct work_struct *work)
 static efi_status_t __efi_queue_work(enum efi_rts_ids id,
 				     union efi_rts_args *args)
 {
+	if (!efi_enabled(EFI_RUNTIME_SERVICES)) {
+		pr_warn_once("EFI Runtime Services are disabled!\n");
+		return EFI_DEVICE_ERROR;
+	}
+
 	efi_rts_work.efi_rts_id = id;
 	efi_rts_work.args = args;
 	efi_rts_work.caller = __builtin_return_address(0);
 	efi_rts_work.status = EFI_ABORTED;
 
-	if (!efi_enabled(EFI_RUNTIME_SERVICES)) {
-		pr_warn_once("EFI Runtime Services are disabled!\n");
-		efi_rts_work.status = EFI_DEVICE_ERROR;
-		goto exit;
-	}
-
 	init_completion(&efi_rts_work.efi_rts_comp);
 	INIT_WORK(&efi_rts_work.work, efi_call_rts);
 
@@ -338,10 +364,18 @@ static efi_status_t __efi_queue_work(enum efi_rts_ids id,
 	 * queue_work() returns 0 if work was already on queue,
 	 * _ideally_ this should never happen.
 	 */
-	if (queue_work(efi_rts_wq, &efi_rts_work.work))
-		wait_for_completion(&efi_rts_work.efi_rts_comp);
-	else
+	if (!queue_work(efi_rts_wq, &efi_rts_work.work)) {
 		pr_err("Failed to queue work to efi_rts_wq.\n");
+		goto exit;
+	}
+
+	if (!wait_for_completion_timeout(&efi_rts_work.efi_rts_comp,
+					 EFI_RTS_TIMEOUT)) {
+		pr_err("EFI runtime service %d wedged in firmware; disabling EFI runtime services\n",
+		       id);
+		clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
+		return EFI_ABORTED;
+	}
 
 	WARN_ON_ONCE(efi_rts_work.status == EFI_ABORTED);
 exit:
@@ -449,6 +483,11 @@ virt_efi_set_variable_nb(efi_char16_t *name, efi_guid_t *vendor, u32 attr,
 	if (down_trylock(&efi_runtime_lock))
 		return EFI_NOT_READY;
 
+	if (!efi_enabled(EFI_RUNTIME_SERVICES)) {
+		up(&efi_runtime_lock);
+		return EFI_DEVICE_ERROR;
+	}
+
 	efi_runtime_lock_owner = current;
 	status = efi_call_virt_pointer(efi.runtime, set_variable, name, vendor,
 				       attr, data_size, data);
@@ -488,6 +527,11 @@ virt_efi_query_variable_info_nb(u32 attr, u64 *storage_space,
 	if (down_trylock(&efi_runtime_lock))
 		return EFI_NOT_READY;
 
+	if (!efi_enabled(EFI_RUNTIME_SERVICES)) {
+		up(&efi_runtime_lock);
+		return EFI_DEVICE_ERROR;
+	}
+
 	efi_runtime_lock_owner = current;
 	status = efi_call_virt_pointer(efi.runtime, query_variable_info, attr,
 				       storage_space, remaining_space,
@@ -518,6 +562,12 @@ virt_efi_reset_system(int reset_type, efi_status_t status,
 		return;
 	}
 
+	if (!efi_enabled(EFI_RUNTIME_SERVICES)) {
+		pr_warn("EFI Runtime Services are disabled, not invoking reset_system()\n");
+		up(&efi_runtime_lock);
+		return;
+	}
+
 	efi_runtime_lock_owner = current;
 	arch_efi_call_virt_setup();
 	efi_rts_work.efi_rts_id = EFI_RESET_SYSTEM;
diff --git a/drivers/firmware/imx/sm-misc.c b/drivers/firmware/imx/sm-misc.c
index ac9af824c2d40..fb8d7bdb5b084 100644
--- a/drivers/firmware/imx/sm-misc.c
+++ b/drivers/firmware/imx/sm-misc.c
@@ -79,6 +79,9 @@ static int syslog_show(struct seq_file *file, void *priv)
 	u16 size = SZ_4K / 4;
 	int ret;
 
+	if (!syslog)
+		return -ENOMEM;
+
 	if (!ph)
 		return -ENODEV;
 
diff --git a/drivers/firmware/psci/psci_checker.c b/drivers/firmware/psci/psci_checker.c
index e67ba98910826..e045e92cd898a 100644
--- a/drivers/firmware/psci/psci_checker.c
+++ b/drivers/firmware/psci/psci_checker.c
@@ -186,7 +186,6 @@ static int hotplug_tests(void)
 {
 	int i, nb_cpu_group, err = -ENOMEM;
 	cpumask_var_t offlined_cpus, *cpu_groups;
-	char *page_buf;
 
 	if (!alloc_cpumask_var(&offlined_cpus, GFP_KERNEL))
 		return err;
@@ -194,10 +193,6 @@ static int hotplug_tests(void)
 	nb_cpu_group = alloc_init_cpu_groups(&cpu_groups);
 	if (nb_cpu_group < 0)
 		goto out_free_cpus;
-	page_buf = (char *)__get_free_page(GFP_KERNEL);
-	if (!page_buf)
-		goto out_free_cpu_groups;
-
 	/*
 	 * Of course the last CPU cannot be powered down and cpu_down() should
 	 * refuse doing that.
@@ -210,17 +205,11 @@ static int hotplug_tests(void)
 	 * off, the cpu group itself should shut down.
 	 */
 	for (i = 0; i < nb_cpu_group; ++i) {
-		ssize_t len = cpumap_print_to_pagebuf(true, page_buf,
-						      cpu_groups[i]);
-		/* Remove trailing newline. */
-		page_buf[len - 1] = '\0';
-		pr_info("Trying to turn off and on again group %d (CPUs %s)\n",
-			i, page_buf);
+		pr_info("Trying to turn off and on again group %d (CPUs %*pbl)\n",
+			i, cpumask_pr_args(cpu_groups[i]));
 		err += down_and_up_cpus(cpu_groups[i], offlined_cpus);
 	}
 
-	free_page((unsigned long)page_buf);
-out_free_cpu_groups:
 	free_cpu_groups(nb_cpu_group, &cpu_groups);
 out_free_cpus:
 	free_cpumask_var(offlined_cpus);
diff --git a/drivers/firmware/qcom/Kconfig b/drivers/firmware/qcom/Kconfig
index b477d54b495a6..c7f8413ab996c 100644
--- a/drivers/firmware/qcom/Kconfig
+++ b/drivers/firmware/qcom/Kconfig
@@ -6,9 +6,29 @@
 
 menu "Qualcomm firmware drivers"
 
+config QCOM_PAS
+	tristate "Qualcomm generic PAS interface driver"
+	help
+	  Enable the generic Peripheral Authentication Service (PAS) provided
+	  by the firmware. It acts as the common layer with different TZ
+	  backends plugged in whether it's an SCM implementation or a proper
+	  TEE bus based PAS service implementation.
+
+config QCOM_PAS_TEE
+	tristate "Qualcomm PAS TEE interface driver"
+	select QCOM_PAS
+	depends on TEE
+	depends on !CPU_BIG_ENDIAN
+	default m if ARCH_QCOM
+	help
+	  Enable the generic Peripheral Authentication Service (PAS) provided
+	  by the firmware TEE implementation as the backend.
+
 config QCOM_SCM
+	tristate "Qualcomm PAS SCM interface driver"
+	select QCOM_PAS
 	select QCOM_TZMEM
-	tristate
+	default y if ARCH_QCOM
 
 config QCOM_TZMEM
 	tristate
diff --git a/drivers/firmware/qcom/Makefile b/drivers/firmware/qcom/Makefile
index 0be40a1abc13c..88ce74d74c3e2 100644
--- a/drivers/firmware/qcom/Makefile
+++ b/drivers/firmware/qcom/Makefile
@@ -5,6 +5,9 @@
 
 obj-$(CONFIG_QCOM_SCM)		+= qcom-scm.o
 qcom-scm-objs += qcom_scm.o qcom_scm-smc.o qcom_scm-legacy.o
+CFLAGS_qcom_scm-smc.o := -I$(src)
 obj-$(CONFIG_QCOM_TZMEM)	+= qcom_tzmem.o
 obj-$(CONFIG_QCOM_QSEECOM)	+= qcom_qseecom.o
 obj-$(CONFIG_QCOM_QSEECOM_UEFISECAPP) += qcom_qseecom_uefisecapp.o
+obj-$(CONFIG_QCOM_PAS)		+= qcom_pas.o
+obj-$(CONFIG_QCOM_PAS_TEE)	+= qcom_pas_tee.o
diff --git a/drivers/firmware/qcom/qcom_pas.c b/drivers/firmware/qcom/qcom_pas.c
new file mode 100644
index 0000000000000..24485dd0fa10a
--- /dev/null
+++ b/drivers/firmware/qcom/qcom_pas.c
@@ -0,0 +1,298 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2010,2015,2019 The Linux Foundation. All rights reserved.
+ * Copyright (C) 2015 Linaro Ltd.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/device/devres.h>
+#include <linux/firmware/qcom/qcom_pas.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+#include "qcom_pas.h"
+
+static struct qcom_pas_ops *ops_ptr;
+
+/**
+ * devm_qcom_pas_context_alloc() - Allocate peripheral authentication service
+ *				   context for a given peripheral
+ *
+ * PAS context is device-resource managed, so the caller does not need
+ * to worry about freeing the context memory.
+ *
+ * @dev:	  PAS firmware device
+ * @pas_id:	  peripheral authentication service id
+ * @mem_phys:	  Subsystem reserve memory start address
+ * @mem_size:	  Subsystem reserve memory size
+ *
+ * Return: The new PAS context, or ERR_PTR() on failure.
+ */
+struct qcom_pas_context *devm_qcom_pas_context_alloc(struct device *dev,
+						     u32 pas_id,
+						     phys_addr_t mem_phys,
+						     size_t mem_size)
+{
+	struct qcom_pas_context *ctx;
+
+	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return ERR_PTR(-ENOMEM);
+
+	ctx->dev = dev;
+	ctx->pas_id = pas_id;
+	ctx->mem_phys = mem_phys;
+	ctx->mem_size = mem_size;
+
+	return ctx;
+}
+EXPORT_SYMBOL_GPL(devm_qcom_pas_context_alloc);
+
+/**
+ * qcom_pas_init_image() - Initialize peripheral authentication service state
+ *			   machine for a given peripheral, using the metadata
+ * @pas_id:	peripheral authentication service id
+ * @metadata:	pointer to memory containing ELF header, program header table
+ *		and optional blob of data used for authenticating the metadata
+ *		and the rest of the firmware
+ * @size:	size of the metadata
+ * @ctx:	optional pas context
+ *
+ * Return: 0 on success.
+ *
+ * Upon successful return, the PAS metadata context (@ctx) will be used to
+ * track the metadata allocation, this needs to be released by invoking
+ * qcom_pas_metadata_release() by the caller.
+ */
+int qcom_pas_init_image(u32 pas_id, const void *metadata, size_t size,
+			struct qcom_pas_context *ctx)
+{
+	if (!ops_ptr)
+		return -ENODEV;
+
+	return ops_ptr->init_image(ops_ptr->dev, pas_id, metadata, size, ctx);
+}
+EXPORT_SYMBOL_GPL(qcom_pas_init_image);
+
+/**
+ * qcom_pas_metadata_release() - release metadata context
+ * @ctx:	pas context
+ */
+void qcom_pas_metadata_release(struct qcom_pas_context *ctx)
+{
+	if (!ops_ptr || !ctx || !ctx->ptr)
+		return;
+
+	ops_ptr->metadata_release(ops_ptr->dev, ctx);
+}
+EXPORT_SYMBOL_GPL(qcom_pas_metadata_release);
+
+/**
+ * qcom_pas_mem_setup() - Prepare the memory related to a given peripheral
+ *			  for firmware loading
+ * @pas_id:	peripheral authentication service id
+ * @addr:	start address of memory area to prepare
+ * @size:	size of the memory area to prepare
+ *
+ * Return: 0 on success.
+ */
+int qcom_pas_mem_setup(u32 pas_id, phys_addr_t addr, phys_addr_t size)
+{
+	if (!ops_ptr)
+		return -ENODEV;
+
+	return ops_ptr->mem_setup(ops_ptr->dev, pas_id, addr, size);
+}
+EXPORT_SYMBOL_GPL(qcom_pas_mem_setup);
+
+/**
+ * qcom_pas_get_rsc_table() - Retrieve the resource table in passed output buffer
+ *			      for a given peripheral.
+ *
+ * Qualcomm remote processor may rely on both static and dynamic resources for
+ * its functionality. Static resources typically refer to memory-mapped
+ * addresses required by the subsystem and are often embedded within the
+ * firmware binary and dynamic resources, such as shared memory in DDR etc.,
+ * are determined at runtime during the boot process.
+ *
+ * On Qualcomm Technologies devices, it's possible that static resources are
+ * not embedded in the firmware binary and instead are provided by TrustZone.
+ * However, dynamic resources are always expected to come from TrustZone. This
+ * indicates that for Qualcomm devices, all resources (static and dynamic) will
+ * be provided by TrustZone PAS service.
+ *
+ * If the remote processor firmware binary does contain static resources, they
+ * should be passed in input_rt. These will be forwarded to TrustZone for
+ * authentication. TrustZone will then append the dynamic resources and return
+ * the complete resource table in output_rt_tzm.
+ *
+ * If the remote processor firmware binary does not include a resource table,
+ * the caller of this function should set input_rt as NULL and input_rt_size
+ * as zero respectively.
+ *
+ * More about documentation on resource table data structures can be found in
+ * include/linux/remoteproc.h
+ *
+ * @ctx:	    PAS context
+ * @input_rt:       resource table buffer which is present in firmware binary
+ * @input_rt_size:  size of the resource table present in firmware binary
+ * @output_rt_size: TrustZone expects caller should pass worst case size for
+ *		    the output_rt_tzm.
+ *
+ * Return:
+ *  On success, returns a pointer to the allocated buffer containing the final
+ *  resource table and output_rt_size will have actual resource table size from
+ *  TrustZone. The caller is responsible for freeing the buffer. On failure,
+ *  returns ERR_PTR(-errno).
+ */
+struct resource_table *qcom_pas_get_rsc_table(struct qcom_pas_context *ctx,
+					      void *input_rt,
+					      size_t input_rt_size,
+					      size_t *output_rt_size)
+{
+	if (!ops_ptr)
+		return ERR_PTR(-ENODEV);
+	if (!ctx)
+		return ERR_PTR(-EINVAL);
+
+	return ops_ptr->get_rsc_table(ops_ptr->dev, ctx, input_rt,
+				      input_rt_size, output_rt_size);
+}
+EXPORT_SYMBOL_GPL(qcom_pas_get_rsc_table);
+
+/**
+ * qcom_pas_auth_and_reset() - Authenticate the given peripheral firmware
+ *			       and reset the remote processor
+ * @pas_id:	peripheral authentication service id
+ *
+ * Return: 0 on success.
+ */
+int qcom_pas_auth_and_reset(u32 pas_id)
+{
+	if (!ops_ptr)
+		return -ENODEV;
+
+	return ops_ptr->auth_and_reset(ops_ptr->dev, pas_id);
+}
+EXPORT_SYMBOL_GPL(qcom_pas_auth_and_reset);
+
+/**
+ * qcom_pas_prepare_and_auth_reset() - Prepare, authenticate, and reset the
+ *				       remote processor
+ *
+ * @ctx:	Context saved during call to devm_qcom_pas_context_alloc()
+ *
+ * This function performs the necessary steps to prepare a PAS subsystem,
+ * authenticate it using the provided metadata, and initiate a reset sequence.
+ *
+ * It should be used when Linux is in control setting up the IOMMU hardware
+ * for remote subsystem during secure firmware loading processes. The
+ * preparation step sets up a shmbridge over the firmware memory before
+ * TrustZone accesses the firmware memory region for authentication. The
+ * authentication step verifies the integrity and authenticity of the firmware
+ * or configuration using secure metadata. Finally, the reset step ensures the
+ * subsystem starts in a clean and sane state.
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int qcom_pas_prepare_and_auth_reset(struct qcom_pas_context *ctx)
+{
+	if (!ops_ptr)
+		return -ENODEV;
+	if (!ctx)
+		return -EINVAL;
+
+	return ops_ptr->prepare_and_auth_reset(ops_ptr->dev, ctx);
+}
+EXPORT_SYMBOL_GPL(qcom_pas_prepare_and_auth_reset);
+
+/**
+ * qcom_pas_set_remote_state() - Set the remote processor state
+ * @state:	peripheral state
+ * @pas_id:	peripheral authentication service id
+ *
+ * Return: 0 on success.
+ */
+int qcom_pas_set_remote_state(u32 state, u32 pas_id)
+{
+	if (!ops_ptr)
+		return -ENODEV;
+
+	return ops_ptr->set_remote_state(ops_ptr->dev, state, pas_id);
+}
+EXPORT_SYMBOL_GPL(qcom_pas_set_remote_state);
+
+/**
+ * qcom_pas_shutdown() - Shut down the remote processor
+ * @pas_id:	peripheral authentication service id
+ *
+ * Return: 0 on success.
+ */
+int qcom_pas_shutdown(u32 pas_id)
+{
+	if (!ops_ptr)
+		return -ENODEV;
+
+	return ops_ptr->shutdown(ops_ptr->dev, pas_id);
+}
+EXPORT_SYMBOL_GPL(qcom_pas_shutdown);
+
+/**
+ * qcom_pas_supported() - Check if the peripheral authentication service is
+ *			  supported for the given peripheral
+ * @pas_id:	peripheral authentication service id
+ *
+ * Return: true if PAS is supported for this peripheral, otherwise false.
+ */
+bool qcom_pas_supported(u32 pas_id)
+{
+	if (!ops_ptr)
+		return false;
+
+	return ops_ptr->supported(ops_ptr->dev, pas_id);
+}
+EXPORT_SYMBOL_GPL(qcom_pas_supported);
+
+/**
+ * qcom_pas_is_available() - Check if the peripheral authentication service is
+ *			     available. Note that it is mandatory for any PAS
+ *			     client to invoke this API. If it returns true then
+ *			     only any other PAS API can be invoked.
+ *
+ * Return: true if PAS is available, otherwise false.
+ */
+bool qcom_pas_is_available(void)
+{
+	/*
+	 * The barrier for ops_ptr is intended to synchronize the data stores
+	 * for the ops data structure when client drivers are in parallel
+	 * checking for PAS service availability.
+	 *
+	 * Once the PAS backend becomes available, it is allowed for multiple
+	 * threads to enter TZ for parallel bringup of co-processors during
+	 * boot.
+	 */
+	return !!smp_load_acquire(&ops_ptr);
+}
+EXPORT_SYMBOL_GPL(qcom_pas_is_available);
+
+void qcom_pas_ops_register(struct qcom_pas_ops *ops)
+{
+	if (!qcom_pas_is_available())
+		/* Paired with smp_load_acquire() in qcom_pas_is_available() */
+		smp_store_release(&ops_ptr, ops);
+	else
+		pr_err("qcom_pas: ops already registered by %s\n",
+		       ops_ptr->drv_name);
+}
+EXPORT_SYMBOL_GPL(qcom_pas_ops_register);
+
+void qcom_pas_ops_unregister(void)
+{
+	/* Paired with smp_load_acquire() in qcom_pas_is_available() */
+	smp_store_release(&ops_ptr, NULL);
+}
+EXPORT_SYMBOL_GPL(qcom_pas_ops_unregister);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Qualcomm generic TZ PAS driver");
diff --git a/drivers/firmware/qcom/qcom_pas.h b/drivers/firmware/qcom/qcom_pas.h
new file mode 100644
index 0000000000000..8643e2760602e
--- /dev/null
+++ b/drivers/firmware/qcom/qcom_pas.h
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __QCOM_PAS_INT_H
+#define __QCOM_PAS_INT_H
+
+struct device;
+
+/**
+ * struct qcom_pas_ops - Qcom Peripheral Authentication Service (PAS) ops
+ * @drv_name:			PAS driver name.
+ * @dev:			PAS device pointer.
+ * @supported:			Peripheral supported callback.
+ * @init_image:			Peripheral image initialization callback.
+ * @mem_setup:			Peripheral memory setup callback.
+ * @get_rsc_table:		Peripheral get resource table callback.
+ * @prepare_and_auth_reset:	Peripheral prepare firmware authentication and
+ *				reset callback.
+ * @auth_and_reset:		Peripheral firmware authentication and reset
+ *				callback.
+ * @set_remote_state:		Peripheral set remote state callback.
+ * @shutdown:			Peripheral shutdown callback.
+ * @metadata_release:		Image metadata release callback.
+ */
+struct qcom_pas_ops {
+	const char *drv_name;
+	struct device *dev;
+	bool (*supported)(struct device *dev, u32 pas_id);
+	int (*init_image)(struct device *dev, u32 pas_id, const void *metadata,
+			  size_t size, struct qcom_pas_context *ctx);
+	int (*mem_setup)(struct device *dev, u32 pas_id, phys_addr_t addr,
+			 phys_addr_t size);
+	void *(*get_rsc_table)(struct device *dev, struct qcom_pas_context *ctx,
+			       void *input_rt, size_t input_rt_size,
+			       size_t *output_rt_size);
+	int (*prepare_and_auth_reset)(struct device *dev,
+				      struct qcom_pas_context *ctx);
+	int (*auth_and_reset)(struct device *dev, u32 pas_id);
+	int (*set_remote_state)(struct device *dev, u32 state, u32 pas_id);
+	int (*shutdown)(struct device *dev, u32 pas_id);
+	void (*metadata_release)(struct device *dev,
+				 struct qcom_pas_context *ctx);
+};
+
+void qcom_pas_ops_register(struct qcom_pas_ops *ops);
+void qcom_pas_ops_unregister(void);
+
+#endif /* __QCOM_PAS_INT_H */
diff --git a/drivers/firmware/qcom/qcom_pas_tee.c b/drivers/firmware/qcom/qcom_pas_tee.c
new file mode 100644
index 0000000000000..ac33a00687aa1
--- /dev/null
+++ b/drivers/firmware/qcom/qcom_pas_tee.c
@@ -0,0 +1,479 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/delay.h>
+#include <linux/of.h>
+#include <linux/firmware/qcom/qcom_pas.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/tee_drv.h>
+#include <linux/uuid.h>
+
+#include "qcom_pas.h"
+
+/*
+ * Peripheral Authentication Service (PAS) supported.
+ *
+ * [in]  params[0].value.a:	Unique 32bit remote processor identifier
+ */
+#define TA_QCOM_PAS_IS_SUPPORTED		1
+
+/*
+ * PAS capabilities.
+ *
+ * [in]  params[0].value.a:	Unique 32bit remote processor identifier
+ * [out] params[1].value.a:	PAS capability flags
+ */
+#define TA_QCOM_PAS_CAPABILITIES		2
+
+/*
+ * PAS image initialization.
+ *
+ * [in]  params[0].value.a:	Unique 32bit remote processor identifier
+ * [in]  params[1].memref:	Loadable firmware metadata
+ */
+#define TA_QCOM_PAS_INIT_IMAGE			3
+
+/*
+ * PAS memory setup.
+ *
+ * [in]  params[0].value.a:	Unique 32bit remote processor identifier
+ * [in]  params[0].value.b:	Relocatable firmware size
+ * [in]  params[1].value.a:	32bit LSB relocatable firmware memory address
+ * [in]  params[1].value.b:	32bit MSB relocatable firmware memory address
+ */
+#define TA_QCOM_PAS_MEM_SETUP			4
+
+/*
+ * PAS get resource table.
+ *
+ * [in]     params[0].value.a:	Unique 32bit remote processor identifier
+ * [inout]  params[1].memref:	Resource table config
+ */
+#define TA_QCOM_PAS_GET_RESOURCE_TABLE		5
+
+/*
+ * PAS image authentication and co-processor reset.
+ *
+ * [in]  params[0].value.a:	Unique 32bit remote processor identifier
+ * [in]  params[0].value.b:	Firmware size
+ * [in]  params[1].value.a:	32bit LSB firmware memory address
+ * [in]  params[1].value.b:	32bit MSB firmware memory address
+ * [in]  params[2].memref:	Optional fw memory space shared/lent
+ */
+#define TA_QCOM_PAS_AUTH_AND_RESET		6
+
+/*
+ * PAS co-processor set suspend/resume state.
+ *
+ * [in]  params[0].value.a:	Unique 32bit remote processor identifier
+ * [in]  params[0].value.b:	Co-processor state identifier
+ */
+#define TA_QCOM_PAS_SET_REMOTE_STATE		7
+
+/*
+ * PAS co-processor shutdown.
+ *
+ * [in]  params[0].value.a:	Unique 32bit remote processor identifier
+ */
+#define TA_QCOM_PAS_SHUTDOWN			8
+
+#define TEE_NUM_PARAMS				4
+
+/**
+ * struct qcom_pas_tee_private - PAS service private data
+ * @dev:		PAS service device.
+ * @ctx:		TEE context handler.
+ * @session_id:		PAS TA session identifier.
+ */
+struct qcom_pas_tee_private {
+	struct device *dev;
+	struct tee_context *ctx;
+	u32 session_id;
+};
+
+static bool qcom_pas_tee_supported(struct device *dev, u32 pas_id)
+{
+	struct qcom_pas_tee_private *data = dev_get_drvdata(dev);
+	struct tee_ioctl_invoke_arg inv_arg = {
+		.func = TA_QCOM_PAS_IS_SUPPORTED,
+		.session = data->session_id,
+		.num_params = TEE_NUM_PARAMS
+	};
+	struct tee_param param[4] = {
+		[0] = {
+			.attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT,
+			.u.value.a = pas_id
+		}
+	};
+	int ret;
+
+	ret = tee_client_invoke_func(data->ctx, &inv_arg, param);
+	if (ret < 0 || inv_arg.ret != 0) {
+		dev_err(dev, "PAS not supported, pas_id: %d, ret: %d, err: 0x%x\n",
+			pas_id, ret, inv_arg.ret);
+		return false;
+	}
+
+	return true;
+}
+
+static int qcom_pas_tee_init_image(struct device *dev, u32 pas_id,
+				   const void *metadata, size_t size,
+				   struct qcom_pas_context *ctx)
+{
+	struct qcom_pas_tee_private *data = dev_get_drvdata(dev);
+	struct tee_ioctl_invoke_arg inv_arg = {
+		.func = TA_QCOM_PAS_INIT_IMAGE,
+		.session = data->session_id,
+		.num_params = TEE_NUM_PARAMS
+	};
+	struct tee_param param[4] = {
+		[0] = {
+			.attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT,
+			.u.value.a = pas_id
+		},
+		[1] = {
+			.attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT,
+		}
+	};
+	struct tee_shm *mdata_shm;
+	u8 *mdata_buf = NULL;
+	int ret;
+
+	mdata_shm = tee_shm_alloc_kernel_buf(data->ctx, size);
+	if (IS_ERR(mdata_shm)) {
+		dev_err(dev, "mdata_shm allocation failed\n");
+		return PTR_ERR(mdata_shm);
+	}
+
+	mdata_buf = tee_shm_get_va(mdata_shm, 0);
+	if (IS_ERR(mdata_buf)) {
+		dev_err(dev, "mdata_buf get VA failed\n");
+		tee_shm_free(mdata_shm);
+		return PTR_ERR(mdata_buf);
+	}
+	memcpy(mdata_buf, metadata, size);
+
+	param[1].u.memref.shm = mdata_shm;
+	param[1].u.memref.size = size;
+
+	ret = tee_client_invoke_func(data->ctx, &inv_arg, param);
+	if (ret < 0 || inv_arg.ret != 0) {
+		dev_err(dev, "PAS init image failed, pas_id: %d, ret: %d, err: 0x%x\n",
+			pas_id, ret, inv_arg.ret);
+		tee_shm_free(mdata_shm);
+		return ret ?: -EINVAL;
+	}
+
+	if (ctx)
+		ctx->ptr = (void *)mdata_shm;
+	else
+		tee_shm_free(mdata_shm);
+
+	return ret;
+}
+
+static int qcom_pas_tee_mem_setup(struct device *dev, u32 pas_id,
+				  phys_addr_t addr, phys_addr_t size)
+{
+	struct qcom_pas_tee_private *data = dev_get_drvdata(dev);
+	struct tee_ioctl_invoke_arg inv_arg = {
+		.func = TA_QCOM_PAS_MEM_SETUP,
+		.session = data->session_id,
+		.num_params = TEE_NUM_PARAMS
+	};
+	struct tee_param param[4] = {
+		[0] = {
+			.attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT,
+			.u.value.a = pas_id,
+			.u.value.b = size,
+		},
+		[1] = {
+			.attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT,
+			.u.value.a = lower_32_bits(addr),
+			.u.value.b = upper_32_bits(addr),
+		}
+	};
+	int ret;
+
+	ret = tee_client_invoke_func(data->ctx, &inv_arg, param);
+	if (ret < 0 || inv_arg.ret != 0) {
+		dev_err(dev, "PAS mem setup failed, pas_id: %d, ret: %d, err: 0x%x\n",
+			pas_id, ret, inv_arg.ret);
+		return ret ?: -EINVAL;
+	}
+
+	return ret;
+}
+
+DEFINE_FREE(shm_free, struct tee_shm *, tee_shm_free(_T))
+
+static void *qcom_pas_tee_get_rsc_table(struct device *dev,
+					struct qcom_pas_context *ctx,
+					void *input_rt, size_t input_rt_size,
+					size_t *output_rt_size)
+{
+	struct qcom_pas_tee_private *data = dev_get_drvdata(dev);
+	struct tee_ioctl_invoke_arg inv_arg = {
+		.func = TA_QCOM_PAS_GET_RESOURCE_TABLE,
+		.session = data->session_id,
+		.num_params = TEE_NUM_PARAMS
+	};
+	struct tee_param param[4] = {
+		[0] = {
+			.attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT,
+			.u.value.a = ctx->pas_id,
+		},
+		[1] = {
+			.attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT,
+			.u.memref.size = input_rt_size,
+		}
+	};
+	void *rt_buf = NULL;
+	int ret;
+
+	ret = tee_client_invoke_func(data->ctx, &inv_arg, param);
+	if (ret < 0 || inv_arg.ret != 0) {
+		dev_err(dev, "PAS get RT failed, pas_id: %d, ret: %d, err: 0x%x\n",
+			ctx->pas_id, ret, inv_arg.ret);
+		return ret ? ERR_PTR(ret) : ERR_PTR(-EINVAL);
+	}
+
+	if (param[1].u.memref.size >= input_rt_size) {
+		struct tee_shm *rt_shm __free(shm_free) =
+			tee_shm_alloc_kernel_buf(data->ctx,
+						 param[1].u.memref.size);
+		void *rt_shm_va;
+
+		if (IS_ERR_OR_NULL(rt_shm)) {
+			dev_err(dev, "rt_shm allocation failed\n");
+			rt_shm = NULL;
+			return ERR_PTR(-ENOMEM);
+		}
+
+		rt_shm_va = tee_shm_get_va(rt_shm, 0);
+		if (IS_ERR(rt_shm_va)) {
+			dev_err(dev, "rt_shm get VA failed\n");
+			return ERR_CAST(rt_shm_va);
+		}
+		memcpy(rt_shm_va, input_rt, input_rt_size);
+
+		param[1].u.memref.shm = rt_shm;
+		ret = tee_client_invoke_func(data->ctx, &inv_arg, param);
+		if (ret < 0 || inv_arg.ret != 0) {
+			dev_err(dev, "PAS get RT failed, pas_id: %d, ret: %d, err: 0x%x\n",
+				ctx->pas_id, ret, inv_arg.ret);
+			return ret ? ERR_PTR(ret) : ERR_PTR(-EINVAL);
+		}
+
+		if (param[1].u.memref.size) {
+			*output_rt_size = param[1].u.memref.size;
+			rt_buf = kmemdup(rt_shm_va, *output_rt_size, GFP_KERNEL);
+			if (!rt_buf)
+				return ERR_PTR(-ENOMEM);
+		}
+	} else {
+		*output_rt_size = 0;
+	}
+
+	return rt_buf;
+}
+
+static int __qcom_pas_tee_auth_and_reset(struct device *dev, u32 pas_id,
+					 phys_addr_t mem_phys, size_t mem_size)
+{
+	struct qcom_pas_tee_private *data = dev_get_drvdata(dev);
+	struct tee_ioctl_invoke_arg inv_arg = {
+		.func = TA_QCOM_PAS_AUTH_AND_RESET,
+		.session = data->session_id,
+		.num_params = TEE_NUM_PARAMS
+	};
+	struct tee_param param[4] = {
+		[0] = {
+			.attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT,
+			.u.value.a = pas_id,
+			.u.value.b = mem_size,
+		},
+		[1] = {
+			.attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT,
+			.u.value.a = lower_32_bits(mem_phys),
+			.u.value.b = upper_32_bits(mem_phys),
+		},
+		/* Reserved for fw memory space to be shared or lent */
+		[2] = {
+			.attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT,
+		}
+	};
+	int ret;
+
+	ret = tee_client_invoke_func(data->ctx, &inv_arg, param);
+	if (ret < 0 || inv_arg.ret != 0) {
+		dev_err(dev, "PAS auth reset failed, pas_id: %d, ret: %d, err: 0x%x\n",
+			pas_id, ret, inv_arg.ret);
+		return ret ?: -EINVAL;
+	}
+
+	return ret;
+}
+
+static int qcom_pas_tee_auth_and_reset(struct device *dev, u32 pas_id)
+{
+	return __qcom_pas_tee_auth_and_reset(dev, pas_id, 0, 0);
+}
+
+static int qcom_pas_tee_prepare_and_auth_reset(struct device *dev,
+					       struct qcom_pas_context *ctx)
+{
+	return __qcom_pas_tee_auth_and_reset(dev, ctx->pas_id, ctx->mem_phys,
+					     ctx->mem_size);
+}
+
+static int qcom_pas_tee_set_remote_state(struct device *dev, u32 state,
+					 u32 pas_id)
+{
+	struct qcom_pas_tee_private *data = dev_get_drvdata(dev);
+	struct tee_ioctl_invoke_arg inv_arg = {
+		.func = TA_QCOM_PAS_SET_REMOTE_STATE,
+		.session = data->session_id,
+		.num_params = TEE_NUM_PARAMS
+	};
+	struct tee_param param[4] = {
+		[0] = {
+			.attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT,
+			.u.value.a = pas_id,
+			.u.value.b = state,
+		}
+	};
+	int ret;
+
+	ret = tee_client_invoke_func(data->ctx, &inv_arg, param);
+	if (ret < 0 || inv_arg.ret != 0) {
+		dev_err(dev, "PAS set remote state failed, pas_id: %d, ret: %d, err: 0x%x\n",
+			pas_id, ret, inv_arg.ret);
+		return ret ?: -EINVAL;
+	}
+
+	return ret;
+}
+
+static int qcom_pas_tee_shutdown(struct device *dev, u32 pas_id)
+{
+	struct qcom_pas_tee_private *data = dev_get_drvdata(dev);
+	struct tee_ioctl_invoke_arg inv_arg = {
+		.func = TA_QCOM_PAS_SHUTDOWN,
+		.session = data->session_id,
+		.num_params = TEE_NUM_PARAMS
+	};
+	struct tee_param param[4] = {
+		[0] = {
+			.attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT,
+			.u.value.a = pas_id
+		}
+	};
+	int ret;
+
+	ret = tee_client_invoke_func(data->ctx, &inv_arg, param);
+	if (ret < 0 || inv_arg.ret != 0) {
+		dev_err(dev, "PAS shutdown failed, pas_id: %d, ret: %d, err: 0x%x\n",
+			pas_id, ret, inv_arg.ret);
+		return ret ?: -EINVAL;
+	}
+
+	return ret;
+}
+
+static void qcom_pas_tee_metadata_release(struct device *dev,
+					  struct qcom_pas_context *ctx)
+{
+	struct tee_shm *mdata_shm = ctx->ptr;
+
+	tee_shm_free(mdata_shm);
+	ctx->ptr = NULL;
+}
+
+static struct qcom_pas_ops qcom_pas_ops_tee = {
+	.drv_name		= "qcom-pas-tee",
+	.supported		= qcom_pas_tee_supported,
+	.init_image		= qcom_pas_tee_init_image,
+	.mem_setup		= qcom_pas_tee_mem_setup,
+	.get_rsc_table		= qcom_pas_tee_get_rsc_table,
+	.auth_and_reset		= qcom_pas_tee_auth_and_reset,
+	.prepare_and_auth_reset	= qcom_pas_tee_prepare_and_auth_reset,
+	.set_remote_state	= qcom_pas_tee_set_remote_state,
+	.shutdown		= qcom_pas_tee_shutdown,
+	.metadata_release	= qcom_pas_tee_metadata_release,
+};
+
+static int optee_ctx_match(struct tee_ioctl_version_data *ver, const void *data)
+{
+	return ver->impl_id == TEE_IMPL_ID_OPTEE;
+}
+
+static int qcom_pas_tee_probe(struct tee_client_device *pas_dev)
+{
+	struct device *dev = &pas_dev->dev;
+	struct qcom_pas_tee_private *data;
+	struct tee_ioctl_open_session_arg sess_arg = {
+		.clnt_login = TEE_IOCTL_LOGIN_REE_KERNEL
+	};
+	int ret;
+
+	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->ctx = tee_client_open_context(NULL, optee_ctx_match, NULL, NULL);
+	if (IS_ERR(data->ctx))
+		return -ENODEV;
+
+	export_uuid(sess_arg.uuid, &pas_dev->id.uuid);
+	ret = tee_client_open_session(data->ctx, &sess_arg, NULL);
+	if (ret < 0 || sess_arg.ret != 0) {
+		dev_err(dev, "tee_client_open_session failed, ret: %d, err: 0x%x\n",
+			ret, sess_arg.ret);
+		tee_client_close_context(data->ctx);
+		return ret ?: -EINVAL;
+	}
+
+	data->session_id = sess_arg.session;
+	dev_set_drvdata(dev, data);
+	qcom_pas_ops_tee.dev = dev;
+	qcom_pas_ops_register(&qcom_pas_ops_tee);
+
+	return ret;
+}
+
+static void qcom_pas_tee_remove(struct tee_client_device *pas_dev)
+{
+	struct device *dev = &pas_dev->dev;
+	struct qcom_pas_tee_private *data = dev_get_drvdata(dev);
+
+	qcom_pas_ops_unregister();
+	tee_client_close_session(data->ctx, data->session_id);
+	tee_client_close_context(data->ctx);
+}
+
+static const struct tee_client_device_id qcom_pas_tee_id_table[] = {
+	{UUID_INIT(0xcff7d191, 0x7ca0, 0x4784,
+		   0xaf, 0x13, 0x48, 0x22, 0x3b, 0x9a, 0x4f, 0xbe)},
+	{}
+};
+MODULE_DEVICE_TABLE(tee, qcom_pas_tee_id_table);
+
+static struct tee_client_driver optee_pas_tee_driver = {
+	.probe		= qcom_pas_tee_probe,
+	.remove		= qcom_pas_tee_remove,
+	.id_table	= qcom_pas_tee_id_table,
+	.driver		= {
+		.name		= "qcom-pas-tee",
+	},
+};
+
+module_tee_client_driver(optee_pas_tee_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Qualcomm PAS TEE driver");
diff --git a/drivers/firmware/qcom/qcom_scm-smc.c b/drivers/firmware/qcom/qcom_scm-smc.c
index 574930729ddd7..01999c22659cb 100644
--- a/drivers/firmware/qcom/qcom_scm-smc.c
+++ b/drivers/firmware/qcom/qcom_scm-smc.c
@@ -24,6 +24,9 @@ struct arm_smccc_args {
 	unsigned long args[8];
 };
 
+#define CREATE_TRACE_POINTS
+#include "qcom_scm_trace.h"
+
 static DEFINE_MUTEX(qcom_scm_lock);
 
 #define QCOM_SCM_EBUSY_WAIT_MS 30
@@ -44,6 +47,7 @@ static void __scm_smc_do_quirk(const struct arm_smccc_args *smc,
 	quirk.state.a6 = 0;
 
 	do {
+		trace_scm_smc_request(a0, smc);
 		arm_smccc_smc_quirk(a0, smc->args[1], smc->args[2],
 				    smc->args[3], smc->args[4], smc->args[5],
 				    quirk.state.a6, smc->args[7], res, &quirk);
@@ -83,6 +87,7 @@ int scm_get_wq_ctx(u32 *wq_ctx, u32 *flags, u32 *more_pending)
 	if (ret)
 		return ret;
 
+	trace_scm_waitq_get_wq_ctx(get_wq_res.a1, get_wq_res.a2, get_wq_res.a3);
 	*wq_ctx = get_wq_res.a1;
 	*flags  = get_wq_res.a2;
 	*more_pending = get_wq_res.a3;
@@ -105,10 +110,12 @@ static int __scm_smc_do_quirk_handle_waitq(struct device *dev, struct arm_smccc_
 			wq_ctx = res->a1;
 			smc_call_ctx = res->a2;
 
+			trace_scm_waitq_sleep(wq_ctx, smc_call_ctx);
 			ret = qcom_scm_wait_for_wq_completion(wq_ctx);
 			if (ret)
 				return ret;
 
+			trace_scm_waitq_resume(smc_call_ctx);
 			fill_wq_resume_args(&resume, smc_call_ctx);
 			smc = &resume;
 		}
@@ -201,6 +208,9 @@ int __scm_smc_call(struct device *dev, const struct qcom_scm_desc *desc,
 	}
 
 	ret = __scm_smc_do(dev, &smc, &smc_res, atomic);
+
+	trace_scm_smc_done(ret, smc.args[0], &smc_res);
+
 	if (ret)
 		return ret;
 
diff --git a/drivers/firmware/qcom/qcom_scm.c b/drivers/firmware/qcom/qcom_scm.c
index 6b601a4b89dbf..f35f2ee391304 100644
--- a/drivers/firmware/qcom/qcom_scm.c
+++ b/drivers/firmware/qcom/qcom_scm.c
@@ -13,6 +13,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/err.h>
 #include <linux/export.h>
+#include <linux/firmware/qcom/qcom_pas.h>
 #include <linux/firmware/qcom/qcom_scm.h>
 #include <linux/firmware/qcom/qcom_tzmem.h>
 #include <linux/init.h>
@@ -33,6 +34,7 @@
 
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 
+#include "qcom_pas.h"
 #include "qcom_scm.h"
 #include "qcom_tzmem.h"
 
@@ -57,6 +59,7 @@ struct qcom_scm {
 	int scm_vote_count;
 
 	u64 dload_mode_addr;
+	void __iomem *minidump_sram;
 
 	struct qcom_tzmem_pool *mempool;
 	unsigned int wq_cnt;
@@ -141,6 +144,20 @@ static const u8 qcom_scm_cpu_warm_bits[QCOM_SCM_BOOT_MAX_CPUS] = {
 #define QCOM_DLOAD_MINIDUMP	2
 #define QCOM_DLOAD_BOTHDUMP	3
 
+/* Minidump destination values written to always-on SRAM for boot firmware */
+#define QCOM_MINIDUMP_DEST_USB		0x0
+#define QCOM_MINIDUMP_DEST_STORAGE	0x2
+
+static u32 minidump_dest = QCOM_MINIDUMP_DEST_USB;
+
+static const struct {
+	const char *name;
+	u32 val;
+} minidump_dest_map[] = {
+	{ "usb",     QCOM_MINIDUMP_DEST_USB     },
+	{ "storage", QCOM_MINIDUMP_DEST_STORAGE },
+};
+
 #define QCOM_SCM_DEFAULT_WAITQ_COUNT 1
 
 static const char * const qcom_scm_convention_names[] = {
@@ -479,25 +496,6 @@ void qcom_scm_cpu_power_down(u32 flags)
 }
 EXPORT_SYMBOL_GPL(qcom_scm_cpu_power_down);
 
-int qcom_scm_set_remote_state(u32 state, u32 id)
-{
-	struct qcom_scm_desc desc = {
-		.svc = QCOM_SCM_SVC_BOOT,
-		.cmd = QCOM_SCM_BOOT_SET_REMOTE_STATE,
-		.arginfo = QCOM_SCM_ARGS(2),
-		.args[0] = state,
-		.args[1] = id,
-		.owner = ARM_SMCCC_OWNER_SIP,
-	};
-	struct qcom_scm_res res;
-	int ret;
-
-	ret = qcom_scm_call(__scm->dev, &desc, &res);
-
-	return ret ? : res.result[0];
-}
-EXPORT_SYMBOL_GPL(qcom_scm_set_remote_state);
-
 static int qcom_scm_disable_sdi(void)
 {
 	int ret;
@@ -551,45 +549,39 @@ static int qcom_scm_io_rmw(phys_addr_t addr, unsigned int mask, unsigned int val
 	return qcom_scm_io_writel(addr, new);
 }
 
-static void qcom_scm_set_download_mode(u32 dload_mode)
+static void qcom_scm_set_download_mode(struct qcom_scm *scm, u32 dload_mode)
 {
 	int ret = 0;
 
-	if (__scm->dload_mode_addr) {
-		ret = qcom_scm_io_rmw(__scm->dload_mode_addr, QCOM_DLOAD_MASK,
+	if (scm->dload_mode_addr) {
+		ret = qcom_scm_io_rmw(scm->dload_mode_addr, QCOM_DLOAD_MASK,
 				      FIELD_PREP(QCOM_DLOAD_MASK, dload_mode));
-	} else if (__qcom_scm_is_call_available(__scm->dev, QCOM_SCM_SVC_BOOT,
+	} else if (__qcom_scm_is_call_available(scm->dev, QCOM_SCM_SVC_BOOT,
 						QCOM_SCM_BOOT_SET_DLOAD_MODE)) {
-		ret = __qcom_scm_set_dload_mode(__scm->dev, !!dload_mode);
+		ret = __qcom_scm_set_dload_mode(scm->dev, !!dload_mode);
 	} else if (dload_mode) {
-		dev_err(__scm->dev,
+		dev_err(scm->dev,
 			"No available mechanism for setting download mode\n");
 	}
 
 	if (ret)
-		dev_err(__scm->dev, "failed to set download mode: %d\n", ret);
+		dev_err(scm->dev, "failed to set download mode: %d\n", ret);
+
+	/*
+	 * Write the destination into the always-on SRAM so boot firmware
+	 * can read it before DDR is initialised on the next warm reset.
+	 * Only written when minidump is active;
+	 */
+	if (scm->minidump_sram && (dload_mode & QCOM_DLOAD_MINIDUMP))
+		writel_relaxed(minidump_dest, scm->minidump_sram);
 }
 
-/**
- * devm_qcom_scm_pas_context_alloc() - Allocate peripheral authentication service
- *				       context for a given peripheral
- *
- * PAS context is device-resource managed, so the caller does not need
- * to worry about freeing the context memory.
- *
- * @dev:	  PAS firmware device
- * @pas_id:	  peripheral authentication service id
- * @mem_phys:	  Subsystem reserve memory start address
- * @mem_size:	  Subsystem reserve memory size
- *
- * Returns: The new PAS context, or ERR_PTR() on failure.
- */
 struct qcom_scm_pas_context *devm_qcom_scm_pas_context_alloc(struct device *dev,
 							     u32 pas_id,
 							     phys_addr_t mem_phys,
 							     size_t mem_size)
 {
-	struct qcom_scm_pas_context *ctx;
+	struct qcom_pas_context *ctx;
 
 	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
 	if (!ctx)
@@ -600,11 +592,12 @@ struct qcom_scm_pas_context *devm_qcom_scm_pas_context_alloc(struct device *dev,
 	ctx->mem_phys = mem_phys;
 	ctx->mem_size = mem_size;
 
-	return ctx;
+	return (struct qcom_scm_pas_context *)ctx;
 }
 EXPORT_SYMBOL_GPL(devm_qcom_scm_pas_context_alloc);
 
-static int __qcom_scm_pas_init_image(u32 pas_id, dma_addr_t mdata_phys,
+static int __qcom_scm_pas_init_image(struct device *dev, u32 pas_id,
+				     dma_addr_t mdata_phys,
 				     struct qcom_scm_res *res)
 {
 	struct qcom_scm_desc desc = {
@@ -626,7 +619,7 @@ static int __qcom_scm_pas_init_image(u32 pas_id, dma_addr_t mdata_phys,
 
 	desc.args[1] = mdata_phys;
 
-	ret = qcom_scm_call(__scm->dev, &desc, res);
+	ret = qcom_scm_call(dev, &desc, res);
 	qcom_scm_bw_disable();
 
 disable_clk:
@@ -635,7 +628,8 @@ static int __qcom_scm_pas_init_image(u32 pas_id, dma_addr_t mdata_phys,
 	return ret;
 }
 
-static int qcom_scm_pas_prep_and_init_image(struct qcom_scm_pas_context *ctx,
+static int qcom_scm_pas_prep_and_init_image(struct device *dev,
+					    struct qcom_pas_context *ctx,
 					    const void *metadata, size_t size)
 {
 	struct qcom_scm_res res;
@@ -650,7 +644,7 @@ static int qcom_scm_pas_prep_and_init_image(struct qcom_scm_pas_context *ctx,
 	memcpy(mdata_buf, metadata, size);
 	mdata_phys = qcom_tzmem_to_phys(mdata_buf);
 
-	ret = __qcom_scm_pas_init_image(ctx->pas_id, mdata_phys, &res);
+	ret = __qcom_scm_pas_init_image(dev, ctx->pas_id, mdata_phys, &res);
 	if (ret < 0)
 		qcom_tzmem_free(mdata_buf);
 	else
@@ -659,25 +653,9 @@ static int qcom_scm_pas_prep_and_init_image(struct qcom_scm_pas_context *ctx,
 	return ret ? : res.result[0];
 }
 
-/**
- * qcom_scm_pas_init_image() - Initialize peripheral authentication service
- *			       state machine for a given peripheral, using the
- *			       metadata
- * @pas_id:	peripheral authentication service id
- * @metadata:	pointer to memory containing ELF header, program header table
- *		and optional blob of data used for authenticating the metadata
- *		and the rest of the firmware
- * @size:	size of the metadata
- * @ctx:	optional pas context
- *
- * Return: 0 on success.
- *
- * Upon successful return, the PAS metadata context (@ctx) will be used to
- * track the metadata allocation, this needs to be released by invoking
- * qcom_scm_pas_metadata_release() by the caller.
- */
-int qcom_scm_pas_init_image(u32 pas_id, const void *metadata, size_t size,
-			    struct qcom_scm_pas_context *ctx)
+static int __qcom_scm_pas_init_image2(struct device *dev, u32 pas_id,
+				      const void *metadata, size_t size,
+				      struct qcom_pas_context *ctx)
 {
 	struct qcom_scm_res res;
 	dma_addr_t mdata_phys;
@@ -685,7 +663,7 @@ int qcom_scm_pas_init_image(u32 pas_id, const void *metadata, size_t size,
 	int ret;
 
 	if (ctx && ctx->use_tzmem)
-		return qcom_scm_pas_prep_and_init_image(ctx, metadata, size);
+		return qcom_scm_pas_prep_and_init_image(dev, ctx, metadata, size);
 
 	/*
 	 * During the scm call memory protection will be enabled for the meta
@@ -699,16 +677,15 @@ int qcom_scm_pas_init_image(u32 pas_id, const void *metadata, size_t size,
 	 * If we pass a buffer that is already part of an SHM Bridge to this
 	 * call, it will fail.
 	 */
-	mdata_buf = dma_alloc_coherent(__scm->dev, size, &mdata_phys,
-				       GFP_KERNEL);
+	mdata_buf = dma_alloc_coherent(dev, size, &mdata_phys, GFP_KERNEL);
 	if (!mdata_buf)
 		return -ENOMEM;
 
 	memcpy(mdata_buf, metadata, size);
 
-	ret = __qcom_scm_pas_init_image(pas_id, mdata_phys, &res);
+	ret = __qcom_scm_pas_init_image(dev, pas_id, mdata_phys, &res);
 	if (ret < 0 || !ctx) {
-		dma_free_coherent(__scm->dev, size, mdata_buf, mdata_phys);
+		dma_free_coherent(dev, size, mdata_buf, mdata_phys);
 	} else if (ctx) {
 		ctx->ptr = mdata_buf;
 		ctx->phys = mdata_phys;
@@ -717,36 +694,35 @@ int qcom_scm_pas_init_image(u32 pas_id, const void *metadata, size_t size,
 
 	return ret ? : res.result[0];
 }
-EXPORT_SYMBOL_GPL(qcom_scm_pas_init_image);
 
-/**
- * qcom_scm_pas_metadata_release() - release metadata context
- * @ctx:	pas context
- */
-void qcom_scm_pas_metadata_release(struct qcom_scm_pas_context *ctx)
+int qcom_scm_pas_init_image(u32 pas_id, const void *metadata, size_t size,
+			    struct qcom_scm_pas_context *ctx)
 {
-	if (!ctx->ptr)
-		return;
+	return __qcom_scm_pas_init_image2(__scm->dev, pas_id, metadata, size,
+					  (struct qcom_pas_context *)ctx);
+}
+EXPORT_SYMBOL_GPL(qcom_scm_pas_init_image);
 
+static void __qcom_scm_pas_metadata_release(struct device *dev,
+					    struct qcom_pas_context *ctx)
+{
 	if (ctx->use_tzmem)
 		qcom_tzmem_free(ctx->ptr);
 	else
-		dma_free_coherent(__scm->dev, ctx->size, ctx->ptr, ctx->phys);
+		dma_free_coherent(dev, ctx->size, ctx->ptr, ctx->phys);
 
 	ctx->ptr = NULL;
 }
+
+void qcom_scm_pas_metadata_release(struct qcom_scm_pas_context *ctx)
+{
+	__qcom_scm_pas_metadata_release(__scm->dev,
+					(struct qcom_pas_context *)ctx);
+}
 EXPORT_SYMBOL_GPL(qcom_scm_pas_metadata_release);
 
-/**
- * qcom_scm_pas_mem_setup() - Prepare the memory related to a given peripheral
- *			      for firmware loading
- * @pas_id:	peripheral authentication service id
- * @addr:	start address of memory area to prepare
- * @size:	size of the memory area to prepare
- *
- * Returns 0 on success.
- */
-int qcom_scm_pas_mem_setup(u32 pas_id, phys_addr_t addr, phys_addr_t size)
+static int __qcom_scm_pas_mem_setup(struct device *dev, u32 pas_id,
+				    phys_addr_t addr, phys_addr_t size)
 {
 	int ret;
 	struct qcom_scm_desc desc = {
@@ -768,7 +744,7 @@ int qcom_scm_pas_mem_setup(u32 pas_id, phys_addr_t addr, phys_addr_t size)
 	if (ret)
 		goto disable_clk;
 
-	ret = qcom_scm_call(__scm->dev, &desc, &res);
+	ret = qcom_scm_call(dev, &desc, &res);
 	qcom_scm_bw_disable();
 
 disable_clk:
@@ -776,9 +752,15 @@ int qcom_scm_pas_mem_setup(u32 pas_id, phys_addr_t addr, phys_addr_t size)
 
 	return ret ? : res.result[0];
 }
+
+int qcom_scm_pas_mem_setup(u32 pas_id, phys_addr_t addr, phys_addr_t size)
+{
+	return __qcom_scm_pas_mem_setup(__scm->dev, pas_id, addr, size);
+}
 EXPORT_SYMBOL_GPL(qcom_scm_pas_mem_setup);
 
-static void *__qcom_scm_pas_get_rsc_table(u32 pas_id, void *input_rt_tzm,
+static void *__qcom_scm_pas_get_rsc_table(struct device *dev, u32 pas_id,
+					  void *input_rt_tzm,
 					  size_t input_rt_size,
 					  size_t *output_rt_size)
 {
@@ -813,7 +795,7 @@ static void *__qcom_scm_pas_get_rsc_table(u32 pas_id, void *input_rt_tzm,
 	 * with output_rt_tzm buffer with res.result[2] size however, It should not
 	 * be of unresonable size.
 	 */
-	ret = qcom_scm_call(__scm->dev, &desc, &res);
+	ret = qcom_scm_call(dev, &desc, &res);
 	if (!ret && res.result[2] > SZ_1G) {
 		ret = -E2BIG;
 		goto free_output_rt;
@@ -830,51 +812,11 @@ static void *__qcom_scm_pas_get_rsc_table(u32 pas_id, void *input_rt_tzm,
 	return ret ? ERR_PTR(ret) : output_rt_tzm;
 }
 
-/**
- * qcom_scm_pas_get_rsc_table() - Retrieve the resource table in passed output buffer
- *				  for a given peripheral.
- *
- * Qualcomm remote processor may rely on both static and dynamic resources for
- * its functionality. Static resources typically refer to memory-mapped addresses
- * required by the subsystem and are often embedded within the firmware binary
- * and dynamic resources, such as shared memory in DDR etc., are determined at
- * runtime during the boot process.
- *
- * On Qualcomm Technologies devices, it's possible that static resources are not
- * embedded in the firmware binary and instead are provided by TrustZone However,
- * dynamic resources are always expected to come from TrustZone. This indicates
- * that for Qualcomm devices, all resources (static and dynamic) will be provided
- * by TrustZone via the SMC call.
- *
- * If the remote processor firmware binary does contain static resources, they
- * should be passed in input_rt. These will be forwarded to TrustZone for
- * authentication. TrustZone will then append the dynamic resources and return
- * the complete resource table in output_rt_tzm.
- *
- * If the remote processor firmware binary does not include a resource table,
- * the caller of this function should set input_rt as NULL and input_rt_size
- * as zero respectively.
- *
- * More about documentation on resource table data structures can be found in
- * include/linux/remoteproc.h
- *
- * @ctx:	    PAS context
- * @pas_id:	    peripheral authentication service id
- * @input_rt:       resource table buffer which is present in firmware binary
- * @input_rt_size:  size of the resource table present in firmware binary
- * @output_rt_size: TrustZone expects caller should pass worst case size for
- *		    the output_rt_tzm.
- *
- * Return:
- *  On success, returns a pointer to the allocated buffer containing the final
- *  resource table and output_rt_size will have actual resource table size from
- *  TrustZone. The caller is responsible for freeing the buffer. On failure,
- *  returns ERR_PTR(-errno).
- */
-struct resource_table *qcom_scm_pas_get_rsc_table(struct qcom_scm_pas_context *ctx,
-						  void *input_rt,
-						  size_t input_rt_size,
-						  size_t *output_rt_size)
+static void *__qcom_scm_pas_get_rsc_table2(struct device *dev,
+					   struct qcom_pas_context *ctx,
+					   void *input_rt,
+					   size_t input_rt_size,
+					   size_t *output_rt_size)
 {
 	struct resource_table empty_rsc = {};
 	size_t size = SZ_16K;
@@ -909,11 +851,12 @@ struct resource_table *qcom_scm_pas_get_rsc_table(struct qcom_scm_pas_context *c
 
 	memcpy(input_rt_tzm, input_rt, input_rt_size);
 
-	output_rt_tzm = __qcom_scm_pas_get_rsc_table(ctx->pas_id, input_rt_tzm,
+	output_rt_tzm = __qcom_scm_pas_get_rsc_table(dev, ctx->pas_id,
+						     input_rt_tzm,
 						     input_rt_size, &size);
 	if (PTR_ERR(output_rt_tzm) == -EOVERFLOW)
 		/* Try again with the size requested by the TZ */
-		output_rt_tzm = __qcom_scm_pas_get_rsc_table(ctx->pas_id,
+		output_rt_tzm = __qcom_scm_pas_get_rsc_table(dev, ctx->pas_id,
 							     input_rt_tzm,
 							     input_rt_size,
 							     &size);
@@ -943,16 +886,20 @@ struct resource_table *qcom_scm_pas_get_rsc_table(struct qcom_scm_pas_context *c
 
 	return ret ? ERR_PTR(ret) : tbl_ptr;
 }
+
+struct resource_table *qcom_scm_pas_get_rsc_table(struct qcom_scm_pas_context *ctx,
+						  void *input_rt,
+						  size_t input_rt_size,
+						  size_t *output_rt_size)
+{
+	return __qcom_scm_pas_get_rsc_table2(__scm->dev,
+					     (struct qcom_pas_context *)ctx,
+					     input_rt, input_rt_size,
+					     output_rt_size);
+}
 EXPORT_SYMBOL_GPL(qcom_scm_pas_get_rsc_table);
 
-/**
- * qcom_scm_pas_auth_and_reset() - Authenticate the given peripheral firmware
- *				   and reset the remote processor
- * @pas_id:	peripheral authentication service id
- *
- * Return 0 on success.
- */
-int qcom_scm_pas_auth_and_reset(u32 pas_id)
+static int __qcom_scm_pas_auth_and_reset(struct device *dev, u32 pas_id)
 {
 	int ret;
 	struct qcom_scm_desc desc = {
@@ -972,7 +919,7 @@ int qcom_scm_pas_auth_and_reset(u32 pas_id)
 	if (ret)
 		goto disable_clk;
 
-	ret = qcom_scm_call(__scm->dev, &desc, &res);
+	ret = qcom_scm_call(dev, &desc, &res);
 	qcom_scm_bw_disable();
 
 disable_clk:
@@ -980,28 +927,15 @@ int qcom_scm_pas_auth_and_reset(u32 pas_id)
 
 	return ret ? : res.result[0];
 }
+
+int qcom_scm_pas_auth_and_reset(u32 pas_id)
+{
+	return __qcom_scm_pas_auth_and_reset(__scm->dev, pas_id);
+}
 EXPORT_SYMBOL_GPL(qcom_scm_pas_auth_and_reset);
 
-/**
- * qcom_scm_pas_prepare_and_auth_reset() - Prepare, authenticate, and reset the
- *					   remote processor
- *
- * @ctx:	Context saved during call to qcom_scm_pas_context_init()
- *
- * This function performs the necessary steps to prepare a PAS subsystem,
- * authenticate it using the provided metadata, and initiate a reset sequence.
- *
- * It should be used when Linux is in control setting up the IOMMU hardware
- * for remote subsystem during secure firmware loading processes. The preparation
- * step sets up a shmbridge over the firmware memory before TrustZone accesses the
- * firmware memory region for authentication. The authentication step verifies
- * the integrity and authenticity of the firmware or configuration using secure
- * metadata. Finally, the reset step ensures the subsystem starts in a clean and
- * sane state.
- *
- * Return: 0 on success, negative errno on failure.
- */
-int qcom_scm_pas_prepare_and_auth_reset(struct qcom_scm_pas_context *ctx)
+static int __qcom_scm_pas_prepare_and_auth_reset(struct device *dev,
+						 struct qcom_pas_context *ctx)
 {
 	u64 handle;
 	int ret;
@@ -1012,7 +946,7 @@ int qcom_scm_pas_prepare_and_auth_reset(struct qcom_scm_pas_context *ctx)
 	 * memory region and then invokes a call to TrustZone to authenticate.
 	 */
 	if (!ctx->use_tzmem)
-		return qcom_scm_pas_auth_and_reset(ctx->pas_id);
+		return __qcom_scm_pas_auth_and_reset(dev, ctx->pas_id);
 
 	/*
 	 * When Linux runs @ EL2 Linux must create the shmbridge itself and then
@@ -1022,20 +956,45 @@ int qcom_scm_pas_prepare_and_auth_reset(struct qcom_scm_pas_context *ctx)
 	if (ret)
 		return ret;
 
-	ret = qcom_scm_pas_auth_and_reset(ctx->pas_id);
+	ret = __qcom_scm_pas_auth_and_reset(dev, ctx->pas_id);
 	qcom_tzmem_shm_bridge_delete(handle);
 
 	return ret;
 }
+
+int qcom_scm_pas_prepare_and_auth_reset(struct qcom_scm_pas_context *ctx)
+{
+	return __qcom_scm_pas_prepare_and_auth_reset(__scm->dev,
+						     (struct qcom_pas_context *)ctx);
+}
 EXPORT_SYMBOL_GPL(qcom_scm_pas_prepare_and_auth_reset);
 
-/**
- * qcom_scm_pas_shutdown() - Shut down the remote processor
- * @pas_id:	peripheral authentication service id
- *
- * Returns 0 on success.
- */
-int qcom_scm_pas_shutdown(u32 pas_id)
+static int __qcom_scm_pas_set_remote_state(struct device *dev, u32 state,
+					   u32 pas_id)
+{
+	struct qcom_scm_desc desc = {
+		.svc = QCOM_SCM_SVC_BOOT,
+		.cmd = QCOM_SCM_BOOT_SET_REMOTE_STATE,
+		.arginfo = QCOM_SCM_ARGS(2),
+		.args[0] = state,
+		.args[1] = pas_id,
+		.owner = ARM_SMCCC_OWNER_SIP,
+	};
+	struct qcom_scm_res res;
+	int ret;
+
+	ret = qcom_scm_call(dev, &desc, &res);
+
+	return ret ? : res.result[0];
+}
+
+int qcom_scm_set_remote_state(u32 state, u32 id)
+{
+	return __qcom_scm_pas_set_remote_state(__scm->dev, state, id);
+}
+EXPORT_SYMBOL_GPL(qcom_scm_set_remote_state);
+
+static int __qcom_scm_pas_shutdown(struct device *dev, u32 pas_id)
 {
 	int ret;
 	struct qcom_scm_desc desc = {
@@ -1055,7 +1014,7 @@ int qcom_scm_pas_shutdown(u32 pas_id)
 	if (ret)
 		goto disable_clk;
 
-	ret = qcom_scm_call(__scm->dev, &desc, &res);
+	ret = qcom_scm_call(dev, &desc, &res);
 	qcom_scm_bw_disable();
 
 disable_clk:
@@ -1063,16 +1022,14 @@ int qcom_scm_pas_shutdown(u32 pas_id)
 
 	return ret ? : res.result[0];
 }
+
+int qcom_scm_pas_shutdown(u32 pas_id)
+{
+	return __qcom_scm_pas_shutdown(__scm->dev, pas_id);
+}
 EXPORT_SYMBOL_GPL(qcom_scm_pas_shutdown);
 
-/**
- * qcom_scm_pas_supported() - Check if the peripheral authentication service is
- *			      available for the given peripherial
- * @pas_id:	peripheral authentication service id
- *
- * Returns true if PAS is supported for this peripheral, otherwise false.
- */
-bool qcom_scm_pas_supported(u32 pas_id)
+static bool __qcom_scm_pas_supported(struct device *dev, u32 pas_id)
 {
 	int ret;
 	struct qcom_scm_desc desc = {
@@ -1084,16 +1041,49 @@ bool qcom_scm_pas_supported(u32 pas_id)
 	};
 	struct qcom_scm_res res;
 
-	if (!__qcom_scm_is_call_available(__scm->dev, QCOM_SCM_SVC_PIL,
+	if (!__qcom_scm_is_call_available(dev, QCOM_SCM_SVC_PIL,
 					  QCOM_SCM_PIL_PAS_IS_SUPPORTED))
 		return false;
 
-	ret = qcom_scm_call(__scm->dev, &desc, &res);
+	ret = qcom_scm_call(dev, &desc, &res);
 
 	return ret ? false : !!res.result[0];
 }
+
+bool qcom_scm_pas_supported(u32 pas_id)
+{
+	return __qcom_scm_pas_supported(__scm->dev, pas_id);
+}
 EXPORT_SYMBOL_GPL(qcom_scm_pas_supported);
 
+static struct qcom_pas_ops qcom_pas_ops_scm = {
+	.drv_name		= "qcom_scm",
+	.supported		= __qcom_scm_pas_supported,
+	.init_image		= __qcom_scm_pas_init_image2,
+	.mem_setup		= __qcom_scm_pas_mem_setup,
+	.get_rsc_table		= __qcom_scm_pas_get_rsc_table2,
+	.auth_and_reset		= __qcom_scm_pas_auth_and_reset,
+	.prepare_and_auth_reset	= __qcom_scm_pas_prepare_and_auth_reset,
+	.set_remote_state	= __qcom_scm_pas_set_remote_state,
+	.shutdown		= __qcom_scm_pas_shutdown,
+	.metadata_release	= __qcom_scm_pas_metadata_release,
+};
+
+/**
+ * qcom_scm_is_pas_available() - Check if the peripheral authentication service
+ *				 is available via SCM or not
+ *
+ * Returns true if PAS is available, otherwise false.
+ */
+static bool qcom_scm_is_pas_available(void)
+{
+	if (!__qcom_scm_is_call_available(__scm->dev, QCOM_SCM_SVC_PIL,
+					  QCOM_SCM_PIL_PAS_AUTH_AND_RESET))
+		return false;
+
+	return true;
+}
+
 static int __qcom_scm_pas_mss_reset(struct device *dev, bool reset)
 {
 	struct qcom_scm_desc desc = {
@@ -2040,6 +2030,29 @@ int qcom_scm_gpu_init_regs(u32 gpu_req)
 }
 EXPORT_SYMBOL_GPL(qcom_scm_gpu_init_regs);
 
+static int qcom_scm_map_minidump_sram(struct device *dev, void __iomem **out)
+{
+	struct device_node *np = dev->of_node;
+	struct device_node *sram_np;
+	struct resource res;
+	int ret;
+
+	sram_np = of_parse_phandle(np, "sram", 0);
+	if (!sram_np)
+		return 0;
+
+	ret = of_address_to_resource(sram_np, 0, &res);
+	of_node_put(sram_np);
+	if (ret)
+		return ret;
+
+	*out = devm_ioremap(dev, res.start, resource_size(&res));
+	if (!*out)
+		return -ENOMEM;
+
+	return 0;
+}
+
 static int qcom_scm_find_dload_address(struct device *dev, u64 *addr)
 {
 	struct device_node *tcsr;
@@ -2295,6 +2308,7 @@ static const struct of_device_id qcom_scm_qseecom_allowlist[] __maybe_unused = {
 	{ .compatible = "dell,latitude-7455" },
 	{ .compatible = "dell,xps13-9345" },
 	{ .compatible = "ecs,liva-qc710" },
+	{ .compatible = "honor,magicbook-art-14-snapdragon" },
 	{ .compatible = "hp,elitebook-ultra-g1q" },
 	{ .compatible = "hp,omnibook-x14" },
 	{ .compatible = "huawei,gaokun3" },
@@ -2705,6 +2719,7 @@ static int get_download_mode(char *buffer, const struct kernel_param *kp)
 
 static int set_download_mode(const char *val, const struct kernel_param *kp)
 {
+	struct qcom_scm *scm;
 	bool tmp;
 	int ret;
 
@@ -2720,8 +2735,10 @@ static int set_download_mode(const char *val, const struct kernel_param *kp)
 	}
 
 	download_mode = ret;
-	if (__scm)
-		qcom_scm_set_download_mode(download_mode);
+	/* Pairs with smp_store_release() in qcom_scm_probe(). */
+	scm = smp_load_acquire(&__scm);
+	if (scm)
+		qcom_scm_set_download_mode(scm, download_mode);
 
 	return 0;
 }
@@ -2734,6 +2751,47 @@ static const struct kernel_param_ops download_mode_param_ops = {
 module_param_cb(download_mode, &download_mode_param_ops, NULL, 0644);
 MODULE_PARM_DESC(download_mode, "download mode: off/0/N for no dump mode, full/on/1/Y for full dump mode, mini for minidump mode and full,mini for both full and minidump mode together are acceptable values");
 
+static int get_minidump_dest(char *buffer, const struct kernel_param *kp)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(minidump_dest_map); i++)
+		if (minidump_dest == minidump_dest_map[i].val)
+			return sysfs_emit(buffer, "%s\n", minidump_dest_map[i].name);
+
+	return sysfs_emit(buffer, "unknown\n");
+}
+
+static int set_minidump_dest(const char *val, const struct kernel_param *kp)
+{
+	struct qcom_scm *scm;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(minidump_dest_map); i++)
+		if (sysfs_streq(val, minidump_dest_map[i].name))
+			break;
+
+	if (i >= ARRAY_SIZE(minidump_dest_map))
+		return -EINVAL;
+
+	minidump_dest = minidump_dest_map[i].val;
+
+	/* Pairs with smp_store_release() in qcom_scm_probe(). */
+	scm = smp_load_acquire(&__scm);
+	if (scm && scm->minidump_sram && (download_mode & QCOM_DLOAD_MINIDUMP))
+		writel_relaxed(minidump_dest, scm->minidump_sram);
+
+	return 0;
+}
+
+static const struct kernel_param_ops minidump_dest_param_ops = {
+	.get = get_minidump_dest,
+	.set = set_minidump_dest,
+};
+
+module_param_cb(minidump_dest, &minidump_dest_param_ops, NULL, 0644);
+MODULE_PARM_DESC(minidump_dest, "Minidump SRAM destination: usb (default) or storage");
+
 static int qcom_scm_probe(struct platform_device *pdev)
 {
 	struct qcom_tzmem_pool_config pool_config;
@@ -2748,7 +2806,13 @@ static int qcom_scm_probe(struct platform_device *pdev)
 	scm->dev = &pdev->dev;
 	ret = qcom_scm_find_dload_address(&pdev->dev, &scm->dload_mode_addr);
 	if (ret < 0)
-		return ret;
+		return dev_err_probe(&pdev->dev, ret,
+				     "Failed to get download mode address\n");
+
+	ret = qcom_scm_map_minidump_sram(&pdev->dev, &scm->minidump_sram);
+	if (ret < 0)
+		return dev_err_probe(&pdev->dev, ret,
+				     "Failed to map minidump SRAM\n");
 
 	mutex_init(&scm->scm_bw_lock);
 
@@ -2837,12 +2901,17 @@ static int qcom_scm_probe(struct platform_device *pdev)
 
 	__get_convention();
 
+	if (qcom_scm_is_pas_available()) {
+		qcom_pas_ops_scm.dev = scm->dev;
+		qcom_pas_ops_register(&qcom_pas_ops_scm);
+	}
+
 	/*
 	 * If "download mode" is requested, from this point on warmboot
 	 * will cause the boot stages to enter download mode, unless
 	 * disabled below by a clean shutdown/reboot.
 	 */
-	qcom_scm_set_download_mode(download_mode);
+	qcom_scm_set_download_mode(scm, download_mode);
 
 	/*
 	 * Disable SDI if indicated by DT that it is enabled by default.
@@ -2875,7 +2944,8 @@ static int qcom_scm_probe(struct platform_device *pdev)
 static void qcom_scm_shutdown(struct platform_device *pdev)
 {
 	/* Clean shutdown, disable download mode to allow normal restart */
-	qcom_scm_set_download_mode(QCOM_DLOAD_NODUMP);
+	qcom_scm_set_download_mode(__scm, QCOM_DLOAD_NODUMP);
+	qcom_pas_ops_unregister();
 }
 
 static const struct of_device_id qcom_scm_dt_match[] = {
diff --git a/drivers/firmware/qcom/qcom_scm_trace.h b/drivers/firmware/qcom/qcom_scm_trace.h
new file mode 100644
index 0000000000000..6c911124fc56b
--- /dev/null
+++ b/drivers/firmware/qcom/qcom_scm_trace.h
@@ -0,0 +1,143 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM qcom_scm
+
+#if !defined(_TRACE_SCM_SMC_INTERFACE_H) || defined(TRACE_HEADER_MULTI_READ)
+
+#define _TRACE_SCM_SMC_INTERFACE_H
+
+#include <linux/tracepoint.h>
+
+TRACE_EVENT(scm_smc_request,
+
+	TP_PROTO(unsigned long a0, const struct arm_smccc_args *smc),
+
+	TP_ARGS(a0, smc),
+
+	TP_STRUCT__entry(
+		__field(u64, smc_id)
+		__field(u8, svc_id)
+		__field(u8, cmd_id)
+		__field(u8, args_cnt)
+		__dynamic_array(unsigned long, args,
+				min_t(u8, (smc->args[1] & 0xF), (u8)6))
+	),
+
+	TP_fast_assign(
+		__entry->smc_id = a0;
+		__entry->svc_id = (smc->args[0] >> 8) & 0xFF;
+		__entry->cmd_id = smc->args[0] & 0xFF;
+		u8 n = min_t(u8, (smc->args[1] & 0xF), (u8)6);
+
+		__entry->args_cnt = n;
+
+		unsigned long *dst = __get_dynamic_array(args);
+
+		for (int i = 0; i < n; i++)
+			dst[i] = smc->args[2 + i];
+	),
+
+	TP_printk("smc_id:0x%08llx svc_id:0x%02x cmd_id:0x%02x args_cnt:%u args:%s",
+		  __entry->smc_id, __entry->svc_id, __entry->cmd_id, __entry->args_cnt,
+		  __print_dynamic_array(args, sizeof(unsigned long)))
+);
+
+TRACE_EVENT(scm_waitq_sleep,
+
+	TP_PROTO(u32 wq_ctx, u32 smc_ctx),
+
+	TP_ARGS(wq_ctx, smc_ctx),
+
+	TP_STRUCT__entry(
+		__field(u32, wq_ctx)
+		__field(u32, smc_call_ctx)
+	),
+
+	TP_fast_assign(
+		__entry->wq_ctx = wq_ctx;
+		__entry->smc_call_ctx = smc_ctx;
+	),
+
+	TP_printk("wq_ctx:%u, smc_call_ctx:%u", __entry->wq_ctx, __entry->smc_call_ctx)
+);
+
+TRACE_EVENT(scm_waitq_resume,
+
+	TP_PROTO(u32 smc_ctx),
+
+	TP_ARGS(smc_ctx),
+
+	TP_STRUCT__entry(
+		__field(u32, smc_call_ctx)
+	),
+
+	TP_fast_assign(
+		__entry->smc_call_ctx = smc_ctx;
+	),
+
+	TP_printk("smc_call_ctx:%u", __entry->smc_call_ctx)
+);
+
+TRACE_EVENT(scm_waitq_get_wq_ctx,
+
+	TP_PROTO(u32 wq_ctx, u32 flags, u32 pending),
+
+	TP_ARGS(wq_ctx, flags, pending),
+
+	TP_STRUCT__entry(
+		__field(u32, wq_ctx)
+		__field(u32, flags)
+		__field(u32, more_pending)
+	),
+
+	TP_fast_assign(
+		__entry->wq_ctx = wq_ctx;
+		__entry->flags = flags;
+		__entry->more_pending = pending;
+	),
+
+	TP_printk("wq_ctx:%u, flags:%u, more_pending:%u",
+		  __entry->wq_ctx, __entry->flags, __entry->more_pending)
+);
+
+TRACE_EVENT(scm_smc_done,
+
+	TP_PROTO(int ret, u64 smc_id, struct arm_smccc_res *smc_res),
+
+	TP_ARGS(ret, smc_id, smc_res),
+
+	TP_STRUCT__entry(
+		__field(int, ret)
+		__field(u64, smc_id)
+		__field(unsigned long, res)
+		__field(unsigned long, res0)
+		__field(unsigned long, res1)
+		__field(unsigned long, res2)
+	),
+
+	TP_fast_assign(
+		__entry->ret = ret;
+		__entry->smc_id = smc_id;
+		__entry->res = smc_res->a0;
+		__entry->res0 = smc_res->a1;
+		__entry->res1 = smc_res->a2;
+		__entry->res2 = smc_res->a3;
+	),
+
+	TP_printk("smc_id:0x%08llx, ret:%d res_to_callee:0x%lx res0:0x%lx res1:0x%lx res2:0x%lx",
+		  __entry->smc_id, __entry->ret, __entry->res,
+		  __entry->res0, __entry->res1, __entry->res2)
+);
+
+#endif /* _TRACE_SCM_SMC_INTERFACE_H */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE qcom_scm_trace
+
+#include <trace/define_trace.h>
+
diff --git a/drivers/firmware/ti_sci.c b/drivers/firmware/ti_sci.c
index 590a464403c50..cc747ab0237f2 100644
--- a/drivers/firmware/ti_sci.c
+++ b/drivers/firmware/ti_sci.c
@@ -4225,6 +4225,10 @@ static int ti_sci_probe(struct platform_device *pdev)
 	ret = of_platform_populate(dev->of_node, NULL, NULL, dev);
 	if (ret) {
 		dev_err(dev, "platform_populate failed %pe\n", ERR_PTR(ret));
+		of_platform_depopulate(dev);
+		mutex_lock(&ti_sci_list_mutex);
+		list_del(&info->node);
+		mutex_unlock(&ti_sci_list_mutex);
 		goto out;
 	}
 	return 0;
diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c
index af838b2dc3279..f9a3a95b06380 100644
--- a/drivers/firmware/xilinx/zynqmp.c
+++ b/drivers/firmware/xilinx/zynqmp.c
@@ -1513,6 +1513,99 @@ int zynqmp_pm_request_wake(const u32 node,
 }
 EXPORT_SYMBOL_GPL(zynqmp_pm_request_wake);
 
+/**
+ * zynqmp_pm_start_rpu - Boot Real-time Processing Unit (Cortex-R) on SoC
+ *
+ * @node: power-domains id of the core
+ * @bootaddr: Boot address of elf
+ *
+ * Return: status, either success or error+reason
+ */
+int zynqmp_pm_start_rpu(const u32 node, const u64 bootaddr)
+{
+	enum rpu_boot_mem bootmem;
+	int ret;
+
+	/*
+	 * The exception vector pointers (EVP) refer to the base-address of
+	 * exception vectors (for reset, IRQ, FIQ, etc). The reset-vector
+	 * starts at the base-address and subsequent vectors are on 4-byte
+	 * boundaries.
+	 *
+	 * Exception vectors can start either from 0x0000_0000 (LOVEC) or
+	 * from 0xFFFF_0000 (HIVEC) which is mapped in the OCM (On-Chip Memory)
+	 *
+	 * Usually firmware will put Exception vectors at LOVEC.
+	 *
+	 * It is not recommend that you change the exception vector.
+	 * Changing the EVP to HIVEC will result in increased interrupt latency
+	 * and jitter. Also, if the OCM is secured and the Cortex-R5F processor
+	 * is non-secured, then the Cortex-R5F processor cannot access the
+	 * HIVEC exception vectors in the OCM.
+	 */
+	bootmem = (bootaddr >= 0xFFFC0000) ?
+		   PM_RPU_BOOTMEM_HIVEC : PM_RPU_BOOTMEM_LOVEC;
+
+	pr_debug("RPU boot addr 0x%llx from %s.", bootaddr,
+		 bootmem == PM_RPU_BOOTMEM_HIVEC ? "OCM" : "TCM");
+
+	/* Request node before starting RPU core if new version of API is supported */
+	if (zynqmp_pm_feature(PM_REQUEST_NODE) > PM_API_VERSION_1) {
+		ret = zynqmp_pm_request_node(node,
+					     ZYNQMP_PM_CAPABILITY_ACCESS, 0,
+					     ZYNQMP_PM_REQUEST_ACK_BLOCKING);
+		if (ret < 0) {
+			pr_err("failed to request 0x%x", node);
+			return ret;
+		}
+	}
+
+	ret = zynqmp_pm_request_wake(node, true,
+				     bootmem, ZYNQMP_PM_REQUEST_ACK_NO);
+	if (ret)
+		pr_err("failed to start RPU = 0x%x\n", node);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_start_rpu);
+
+/**
+ * zynqmp_pm_stop_rpu - Stop Real-time Processing Unit (Cortex-R) on SoC
+ *
+ * @node: power-domains id of the core
+ *
+ * Return: status, either success or error+reason
+ */
+int zynqmp_pm_stop_rpu(const u32 node)
+{
+	int ret;
+
+	/* Use release node API to stop core if new version of API is supported */
+	if (zynqmp_pm_feature(PM_RELEASE_NODE) > PM_API_VERSION_1) {
+		ret = zynqmp_pm_release_node(node);
+		if (ret)
+			pr_err("failed to stop remoteproc RPU %d\n", ret);
+		return ret;
+	}
+
+	/*
+	 * Check expected version of EEMI call before calling it. This avoids
+	 * any error or warning prints from firmware as it is expected that fw
+	 * doesn't support it.
+	 */
+	if (zynqmp_pm_feature(PM_FORCE_POWERDOWN) != PM_API_VERSION_1) {
+		pr_debug("EEMI interface %d ver 1 not supported\n",
+			 PM_FORCE_POWERDOWN);
+		return -EOPNOTSUPP;
+	}
+
+	/* maintain force pwr down for backward compatibility */
+	ret = zynqmp_pm_force_pwrdwn(node, ZYNQMP_PM_REQUEST_ACK_BLOCKING);
+	if (ret)
+		pr_err("core force power down failed\n");
+	return ret;
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_stop_rpu);
+
 /**
  * zynqmp_pm_set_requirement() - PM call to set requirement for PM slaves
  * @node:		Node ID of the slave
diff --git a/drivers/fpga/dfl-fme-perf.c b/drivers/fpga/dfl-fme-perf.c
index 7422d2bc6f374..495edc448b68f 100644
--- a/drivers/fpga/dfl-fme-perf.c
+++ b/drivers/fpga/dfl-fme-perf.c
@@ -183,7 +183,7 @@ static ssize_t cpumask_show(struct device *dev,
 
 	priv = to_fme_perf_priv(pmu);
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(priv->cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(priv->cpu)));
 }
 static DEVICE_ATTR_RO(cpumask);
 
@@ -925,6 +925,8 @@ static int fme_perf_pmu_register(struct platform_device *pdev,
 				PERF_PMU_CAP_NO_EXCLUDE;
 
 	name = devm_kasprintf(priv->dev, GFP_KERNEL, "dfl_fme%d", pdev->id);
+	if (!name)
+		return -ENOMEM;
 
 	ret = perf_pmu_register(pmu, name, -1);
 	if (ret)
diff --git a/drivers/fpga/stratix10-soc.c b/drivers/fpga/stratix10-soc.c
index 0a295ccf1644a..b8ec2e6f615fa 100644
--- a/drivers/fpga/stratix10-soc.c
+++ b/drivers/fpga/stratix10-soc.c
@@ -195,20 +195,18 @@ static int s10_ops_write_init(struct fpga_manager *mgr,
 	ret = s10_svc_send_msg(priv, COMMAND_RECONFIG,
 			       &ctype, sizeof(ctype));
 	if (ret < 0)
-		goto init_done;
+		goto init_error;
 
-	ret = wait_for_completion_timeout(
-		&priv->status_return_completion, S10_RECONFIG_TIMEOUT);
-	if (!ret) {
+	if (!wait_for_completion_timeout(&priv->status_return_completion,
+					 S10_RECONFIG_TIMEOUT)) {
 		dev_err(dev, "timeout waiting for RECONFIG_REQUEST\n");
 		ret = -ETIMEDOUT;
-		goto init_done;
+		goto init_error;
 	}
 
-	ret = 0;
 	if (!test_and_clear_bit(SVC_STATUS_OK, &priv->status)) {
 		ret = -ETIMEDOUT;
-		goto init_done;
+		goto init_error;
 	}
 
 	/* Allocate buffers from the service layer's pool. */
@@ -217,14 +215,16 @@ static int s10_ops_write_init(struct fpga_manager *mgr,
 		if (IS_ERR(kbuf)) {
 			s10_free_buffers(mgr);
 			ret = PTR_ERR(kbuf);
-			goto init_done;
+			goto init_error;
 		}
 
 		priv->svc_bufs[i].buf = kbuf;
 		priv->svc_bufs[i].lock = 0;
 	}
 
-init_done:
+	return 0;
+
+init_error:
 	stratix10_svc_done(priv->chan);
 	return ret;
 }
@@ -342,6 +342,9 @@ static int s10_ops_write(struct fpga_manager *mgr, const char *buf,
 	if (!s10_free_buffers(mgr))
 		dev_err(dev, "%s not all buffers were freed\n", __func__);
 
+	if (ret < 0)
+		stratix10_svc_done(priv->chan);
+
 	return ret;
 }
 
diff --git a/drivers/fpga/xilinx-pr-decoupler.c b/drivers/fpga/xilinx-pr-decoupler.c
index 6994d68e90366..45b65a3264af1 100644
--- a/drivers/fpga/xilinx-pr-decoupler.c
+++ b/drivers/fpga/xilinx-pr-decoupler.c
@@ -118,46 +118,30 @@ static int xlnx_pr_decoupler_probe(struct platform_device *pdev)
 	if (IS_ERR(priv->io_base))
 		return PTR_ERR(priv->io_base);
 
-	priv->clk = devm_clk_get(&pdev->dev, "aclk");
+	priv->clk = devm_clk_get_prepared(&pdev->dev, "aclk");
 	if (IS_ERR(priv->clk))
 		return dev_err_probe(&pdev->dev, PTR_ERR(priv->clk),
 				     "input clock not found\n");
 
-	err = clk_prepare_enable(priv->clk);
-	if (err) {
-		dev_err(&pdev->dev, "unable to enable clock\n");
-		return err;
-	}
-
-	clk_disable(priv->clk);
-
 	br = fpga_bridge_register(&pdev->dev, priv->ipconfig->name,
 				  &xlnx_pr_decoupler_br_ops, priv);
 	if (IS_ERR(br)) {
 		err = PTR_ERR(br);
 		dev_err(&pdev->dev, "unable to register %s",
 			priv->ipconfig->name);
-		goto err_clk;
+		return err;
 	}
 
 	platform_set_drvdata(pdev, br);
 
 	return 0;
-
-err_clk:
-	clk_unprepare(priv->clk);
-
-	return err;
 }
 
 static void xlnx_pr_decoupler_remove(struct platform_device *pdev)
 {
 	struct fpga_bridge *bridge = platform_get_drvdata(pdev);
-	struct xlnx_pr_decoupler_data *p = bridge->priv;
 
 	fpga_bridge_unregister(bridge);
-
-	clk_unprepare(p->clk);
 }
 
 static struct platform_driver xlnx_pr_decoupler_driver = {
diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
index 28cf6d2e83c27..f03c052883761 100644
--- a/drivers/gpio/Kconfig
+++ b/drivers/gpio/Kconfig
@@ -646,7 +646,7 @@ config GPIO_ROCKCHIP
 config GPIO_RTD
 	tristate "Realtek DHC GPIO support"
 	depends on ARCH_REALTEK || COMPILE_TEST
-	default y
+	default ARCH_REALTEK
 	select GPIOLIB_IRQCHIP
 	help
 	  This option enables support for GPIOs found on Realtek DHC(Digital
@@ -656,6 +656,18 @@ config GPIO_RTD
 	  Say yes here to support GPIO functionality and GPIO interrupt on
 	  Realtek DHC SoCs.
 
+config GPIO_RTD1625
+	tristate "Realtek DHC RTD1625 GPIO support"
+	depends on ARCH_REALTEK || COMPILE_TEST
+	default ARCH_REALTEK
+	select GPIOLIB_IRQCHIP
+	help
+	  This option enables support for the GPIO controller on Realtek
+	  DHC (Digital Home Center) RTD1625 SoC.
+
+	  Say yes here to support both basic GPIO line functionality
+	  and GPIO interrupt handling capabilities for this platform.
+
 config GPIO_SAMA5D2_PIOBU
 	tristate "SAMA5D2 PIOBU GPIO support"
 	depends on OF
diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile
index 4d0e900402fcf..fa14581e39951 100644
--- a/drivers/gpio/Makefile
+++ b/drivers/gpio/Makefile
@@ -160,6 +160,7 @@ obj-$(CONFIG_GPIO_REALTEK_OTTO)		+= gpio-realtek-otto.o
 obj-$(CONFIG_GPIO_REG)			+= gpio-reg.o
 obj-$(CONFIG_GPIO_ROCKCHIP)	+= gpio-rockchip.o
 obj-$(CONFIG_GPIO_RTD)			+= gpio-rtd.o
+obj-$(CONFIG_GPIO_RTD1625)		+= gpio-rtd1625.o
 obj-$(CONFIG_ARCH_SA1100)		+= gpio-sa1100.o
 obj-$(CONFIG_GPIO_SAMA5D2_PIOBU)	+= gpio-sama5d2-piobu.o
 obj-$(CONFIG_GPIO_SCH311X)		+= gpio-sch311x.o
diff --git a/drivers/gpio/gpio-amd-fch.c b/drivers/gpio/gpio-amd-fch.c
index 9f329938202bf..9b9d75acf35a0 100644
--- a/drivers/gpio/gpio-amd-fch.c
+++ b/drivers/gpio/gpio-amd-fch.c
@@ -48,13 +48,11 @@ static void __iomem *amd_fch_gpio_addr(struct amd_fch_gpio_priv *priv,
 static int amd_fch_gpio_direction_input(struct gpio_chip *gc,
 					unsigned int offset)
 {
-	unsigned long flags;
 	struct amd_fch_gpio_priv *priv = gpiochip_get_data(gc);
 	void __iomem *ptr = amd_fch_gpio_addr(priv, offset);
 
-	spin_lock_irqsave(&priv->lock, flags);
+	guard(spinlock_irqsave)(&priv->lock);
 	writel_relaxed(readl_relaxed(ptr) & ~AMD_FCH_GPIO_FLAG_DIRECTION, ptr);
-	spin_unlock_irqrestore(&priv->lock, flags);
 
 	return 0;
 }
@@ -62,12 +60,11 @@ static int amd_fch_gpio_direction_input(struct gpio_chip *gc,
 static int amd_fch_gpio_direction_output(struct gpio_chip *gc,
 					 unsigned int gpio, int value)
 {
-	unsigned long flags;
 	struct amd_fch_gpio_priv *priv = gpiochip_get_data(gc);
 	void __iomem *ptr = amd_fch_gpio_addr(priv, gpio);
 	u32 val;
 
-	spin_lock_irqsave(&priv->lock, flags);
+	guard(spinlock_irqsave)(&priv->lock);
 
 	val = readl_relaxed(ptr);
 	if (value)
@@ -77,33 +74,28 @@ static int amd_fch_gpio_direction_output(struct gpio_chip *gc,
 
 	writel_relaxed(val | AMD_FCH_GPIO_FLAG_DIRECTION, ptr);
 
-	spin_unlock_irqrestore(&priv->lock, flags);
-
 	return 0;
 }
 
 static int amd_fch_gpio_get_direction(struct gpio_chip *gc, unsigned int gpio)
 {
 	int ret;
-	unsigned long flags;
 	struct amd_fch_gpio_priv *priv = gpiochip_get_data(gc);
 	void __iomem *ptr = amd_fch_gpio_addr(priv, gpio);
 
-	spin_lock_irqsave(&priv->lock, flags);
+	guard(spinlock_irqsave)(&priv->lock);
 	ret = (readl_relaxed(ptr) & AMD_FCH_GPIO_FLAG_DIRECTION);
-	spin_unlock_irqrestore(&priv->lock, flags);
 
 	return ret ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN;
 }
 
 static int amd_fch_gpio_set(struct gpio_chip *gc, unsigned int gpio, int value)
 {
-	unsigned long flags;
 	struct amd_fch_gpio_priv *priv = gpiochip_get_data(gc);
 	void __iomem *ptr = amd_fch_gpio_addr(priv, gpio);
 	u32 mask;
 
-	spin_lock_irqsave(&priv->lock, flags);
+	guard(spinlock_irqsave)(&priv->lock);
 
 	mask = readl_relaxed(ptr);
 	if (value)
@@ -112,22 +104,18 @@ static int amd_fch_gpio_set(struct gpio_chip *gc, unsigned int gpio, int value)
 		mask &= ~AMD_FCH_GPIO_FLAG_WRITE;
 	writel_relaxed(mask, ptr);
 
-	spin_unlock_irqrestore(&priv->lock, flags);
-
 	return 0;
 }
 
 static int amd_fch_gpio_get(struct gpio_chip *gc,
 			    unsigned int offset)
 {
-	unsigned long flags;
 	u32 val;
 	struct amd_fch_gpio_priv *priv = gpiochip_get_data(gc);
 	void __iomem *ptr = amd_fch_gpio_addr(priv, offset);
 
-	spin_lock_irqsave(&priv->lock, flags);
+	guard(spinlock_irqsave)(&priv->lock);
 	val = readl_relaxed(ptr);
-	spin_unlock_irqrestore(&priv->lock, flags);
 
 	return FIELD_GET(AMD_FCH_GPIO_FLAG_READ, val);
 }
diff --git a/drivers/gpio/gpio-bd72720.c b/drivers/gpio/gpio-bd72720.c
index 306e234112090..c124d964e4cfc 100644
--- a/drivers/gpio/gpio-bd72720.c
+++ b/drivers/gpio/gpio-bd72720.c
@@ -85,9 +85,11 @@ static int bd72720gpi_get(struct bd72720_gpio *bdgpio, unsigned int reg_offset)
 static int bd72720gpo_get(struct bd72720_gpio *bdgpio,
 			  unsigned int offset)
 {
-	const int regs[] = { BD72720_REG_GPIO1_CTRL, BD72720_REG_GPIO2_CTRL,
-			     BD72720_REG_GPIO3_CTRL, BD72720_REG_GPIO4_CTRL,
-			     BD72720_REG_GPIO5_CTRL, BD72720_REG_EPDEN_CTRL };
+	static const int regs[] = {
+		BD72720_REG_GPIO1_CTRL, BD72720_REG_GPIO2_CTRL,
+		BD72720_REG_GPIO3_CTRL, BD72720_REG_GPIO4_CTRL,
+		BD72720_REG_GPIO5_CTRL, BD72720_REG_EPDEN_CTRL
+	};
 	int ret, val;
 
 	ret = regmap_read(bdgpio->regmap, regs[offset], &val);
@@ -111,9 +113,11 @@ static int bd72720gpo_set(struct gpio_chip *chip, unsigned int offset,
 			  int value)
 {
 	struct bd72720_gpio *bdgpio = gpiochip_get_data(chip);
-	const int regs[] = { BD72720_REG_GPIO1_CTRL, BD72720_REG_GPIO2_CTRL,
-			     BD72720_REG_GPIO3_CTRL, BD72720_REG_GPIO4_CTRL,
-			     BD72720_REG_GPIO5_CTRL, BD72720_REG_EPDEN_CTRL };
+	static const int regs[] = {
+		BD72720_REG_GPIO1_CTRL, BD72720_REG_GPIO2_CTRL,
+		BD72720_REG_GPIO3_CTRL, BD72720_REG_GPIO4_CTRL,
+		BD72720_REG_GPIO5_CTRL, BD72720_REG_EPDEN_CTRL
+	};
 
 	if (BIT(offset) & bdgpio->gpio_is_input) {
 		dev_dbg(bdgpio->dev, "pin %d not output.\n", offset);
@@ -132,9 +136,11 @@ static int bd72720_gpio_set_config(struct gpio_chip *chip, unsigned int offset,
 				   unsigned long config)
 {
 	struct bd72720_gpio *bdgpio = gpiochip_get_data(chip);
-	const int regs[] = { BD72720_REG_GPIO1_CTRL, BD72720_REG_GPIO2_CTRL,
-			     BD72720_REG_GPIO3_CTRL, BD72720_REG_GPIO4_CTRL,
-			     BD72720_REG_GPIO5_CTRL, BD72720_REG_EPDEN_CTRL };
+	static const int regs[] = {
+		BD72720_REG_GPIO1_CTRL, BD72720_REG_GPIO2_CTRL,
+		BD72720_REG_GPIO3_CTRL, BD72720_REG_GPIO4_CTRL,
+		BD72720_REG_GPIO5_CTRL, BD72720_REG_EPDEN_CTRL
+	};
 
 	/*
 	 * We can only set the output mode, which makes sense only when output
diff --git a/drivers/gpio/gpio-bt8xx.c b/drivers/gpio/gpio-bt8xx.c
index 324eeb77dbd52..f3a2321bc388a 100644
--- a/drivers/gpio/gpio-bt8xx.c
+++ b/drivers/gpio/gpio-bt8xx.c
@@ -154,12 +154,18 @@ static int bt8xxgpio_probe(struct pci_dev *dev,
 			const struct pci_device_id *pci_id)
 {
 	struct bt8xxgpio *bg;
+	void __iomem *mmio;
 	int err;
 
+	mmio = devm_ioremap_resource(&dev->dev, pci_resource_n(dev, 0));
+	if (IS_ERR(mmio))
+		return PTR_ERR(mmio);
+
 	bg = devm_kzalloc(&dev->dev, sizeof(struct bt8xxgpio), GFP_KERNEL);
 	if (!bg)
 		return -ENOMEM;
 
+	bg->mmio = mmio;
 	bg->pdev = dev;
 	spin_lock_init(&bg->lock);
 
@@ -168,24 +174,9 @@ static int bt8xxgpio_probe(struct pci_dev *dev,
 		dev_err(&dev->dev, "can't enable device.\n");
 		return err;
 	}
-	if (!devm_request_mem_region(&dev->dev, pci_resource_start(dev, 0),
-				pci_resource_len(dev, 0),
-				"bt8xxgpio")) {
-		dev_warn(&dev->dev, "can't request iomem (0x%llx).\n",
-		       (unsigned long long)pci_resource_start(dev, 0));
-		err = -EBUSY;
-		goto err_disable;
-	}
 	pci_set_master(dev);
 	pci_set_drvdata(dev, bg);
 
-	bg->mmio = devm_ioremap(&dev->dev, pci_resource_start(dev, 0), 0x1000);
-	if (!bg->mmio) {
-		dev_err(&dev->dev, "ioremap() failed\n");
-		err = -EIO;
-		goto err_disable;
-	}
-
 	/* Disable interrupts */
 	bgwrite(0, BT848_INT_MASK);
 
diff --git a/drivers/gpio/gpio-mt7621.c b/drivers/gpio/gpio-mt7621.c
index 1b0b5247d3c9a..87086c322f085 100644
--- a/drivers/gpio/gpio-mt7621.c
+++ b/drivers/gpio/gpio-mt7621.c
@@ -68,7 +68,7 @@ mt7621_gpio_gc_to_priv(struct gpio_chip *gc)
 }
 
 static inline struct mtk_gc *
-to_mediatek_gpio(struct gpio_chip *chip)
+to_mt7621_gpio(struct gpio_chip *chip)
 {
 	struct gpio_generic_chip *gen_gc = to_gpio_generic_chip(chip);
 
@@ -137,7 +137,7 @@ mt7621_gpio_hwirq_to_offset(irq_hw_number_t hwirq, struct mtk_gc *bank)
 }
 
 static void
-mediatek_gpio_irq_unmask(struct irq_data *d)
+mt7621_gpio_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
 	struct mtk_gc *rg = gpiochip_get_data(gc);
@@ -159,7 +159,7 @@ mediatek_gpio_irq_unmask(struct irq_data *d)
 }
 
 static void
-mediatek_gpio_irq_mask(struct irq_data *d)
+mt7621_gpio_irq_mask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
 	struct mtk_gc *rg = gpiochip_get_data(gc);
@@ -181,7 +181,7 @@ mediatek_gpio_irq_mask(struct irq_data *d)
 }
 
 static int
-mediatek_gpio_irq_type(struct irq_data *d, unsigned int type)
+mt7621_gpio_irq_type(struct irq_data *d, unsigned int type)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
 	struct mtk_gc *rg = gpiochip_get_data(gc);
@@ -245,11 +245,11 @@ mt7621_gpio_irq_relres(struct irq_data *d)
 }
 
 static int
-mediatek_gpio_xlate(struct gpio_chip *chip,
+mt7621_gpio_xlate(struct gpio_chip *chip,
 		    const struct of_phandle_args *spec, u32 *flags)
 {
 	int gpio = spec->args[0];
-	struct mtk_gc *rg = to_mediatek_gpio(chip);
+	struct mtk_gc *rg = to_mt7621_gpio(chip);
 
 	if (rg->bank != gpio / MTK_BANK_WIDTH)
 		return -EINVAL;
@@ -264,10 +264,10 @@ static const struct irq_chip mt7621_irq_chip = {
 	.name		= "mt7621-gpio",
 	.irq_request_resources = mt7621_gpio_irq_reqres,
 	.irq_release_resources = mt7621_gpio_irq_relres,
-	.irq_mask_ack	= mediatek_gpio_irq_mask,
-	.irq_mask	= mediatek_gpio_irq_mask,
-	.irq_unmask	= mediatek_gpio_irq_unmask,
-	.irq_set_type	= mediatek_gpio_irq_type,
+	.irq_mask_ack	= mt7621_gpio_irq_mask,
+	.irq_mask	= mt7621_gpio_irq_mask,
+	.irq_unmask	= mt7621_gpio_irq_unmask,
+	.irq_set_type	= mt7621_gpio_irq_type,
 	.flags		= IRQCHIP_IMMUTABLE,
 };
 
@@ -380,7 +380,7 @@ mt7621_gpio_to_irq(struct gpio_chip *gc, unsigned int offset)
 }
 
 static int
-mediatek_gpio_bank_probe(struct device *dev, int bank)
+mt7621_gpio_bank_probe(struct device *dev, int bank)
 {
 	struct gpio_generic_chip_config config;
 	struct mtk *mtk = dev_get_drvdata(dev);
@@ -416,7 +416,7 @@ mediatek_gpio_bank_probe(struct device *dev, int bank)
 	}
 
 	rg->chip.gc.of_gpio_n_cells = 2;
-	rg->chip.gc.of_xlate = mediatek_gpio_xlate;
+	rg->chip.gc.of_xlate = mt7621_gpio_xlate;
 	rg->chip.gc.ngpio = MTK_BANK_WIDTH;
 	rg->chip.gc.label = devm_kasprintf(dev, GFP_KERNEL, "%s-bank%d",
 					dev_name(dev), bank);
@@ -443,7 +443,7 @@ mediatek_gpio_bank_probe(struct device *dev, int bank)
 }
 
 static int
-mediatek_gpio_probe(struct platform_device *pdev)
+mt7621_gpio_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct mtk *mtk;
@@ -477,7 +477,7 @@ mediatek_gpio_probe(struct platform_device *pdev)
 		return ret;
 
 	for (i = 0; i < MTK_BANK_CNT; i++) {
-		ret = mediatek_gpio_bank_probe(dev, i);
+		ret = mt7621_gpio_bank_probe(dev, i);
 		if (ret)
 			return ret;
 	}
@@ -485,18 +485,18 @@ mediatek_gpio_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static const struct of_device_id mediatek_gpio_match[] = {
+static const struct of_device_id mt7621_gpio_match[] = {
 	{ .compatible = "mediatek,mt7621-gpio" },
 	{},
 };
-MODULE_DEVICE_TABLE(of, mediatek_gpio_match);
+MODULE_DEVICE_TABLE(of, mt7621_gpio_match);
 
-static struct platform_driver mediatek_gpio_driver = {
-	.probe = mediatek_gpio_probe,
+static struct platform_driver mt7621_gpio_driver = {
+	.probe = mt7621_gpio_probe,
 	.driver = {
 		.name = "mt7621_gpio",
-		.of_match_table = mediatek_gpio_match,
+		.of_match_table = mt7621_gpio_match,
 	},
 };
 
-builtin_platform_driver(mediatek_gpio_driver);
+builtin_platform_driver(mt7621_gpio_driver);
diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c
index a010604e5ff79..8d3acadb0d689 100644
--- a/drivers/gpio/gpio-mvebu.c
+++ b/drivers/gpio/gpio-mvebu.c
@@ -37,6 +37,7 @@
 #include <linux/gpio/consumer.h>
 #include <linux/gpio/machine.h>
 #include <linux/init.h>
+#include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/irq.h>
 #include <linux/irqchip/chained_irq.h>
@@ -114,7 +115,7 @@ struct mvebu_gpio_chip {
 	struct regmap     *regs;
 	u32		   offset;
 	struct regmap     *percpu_regs;
-	int		   irqbase;
+	int		   bank_irq[4];
 	struct irq_domain *domain;
 	int		   soc_variant;
 
@@ -603,6 +604,34 @@ static const struct regmap_config mvebu_gpio_regmap_config = {
 	.val_bits = 32,
 };
 
+/*
+ * Forward wake-up configuration to the parent bank IRQ.
+ * @d:		interrupt data
+ * @enable:	enable as wake-up if non-zero
+ *
+ * Return: 0 on success, or a negative error code.
+ */
+static int mvebu_gpio_set_wake_irq(struct irq_data *d, unsigned int enable)
+{
+	struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d);
+	struct mvebu_gpio_chip *mvchip = gc->private;
+	int bank;
+	int irq;
+
+	bank = d->hwirq / 8;
+	if (bank >= ARRAY_SIZE(mvchip->bank_irq))
+		return -EINVAL;
+
+	irq = mvchip->bank_irq[bank];
+	if (irq <= 0)
+		return -EINVAL;
+
+	if (enable)
+		return enable_irq_wake(irq);
+
+	return disable_irq_wake(irq);
+}
+
 /*
  * Functions implementing the pwm_chip methods
  */
@@ -1253,7 +1282,7 @@ static int mvebu_gpio_probe(struct platform_device *pdev)
 
 	err = irq_alloc_domain_generic_chips(
 	    mvchip->domain, ngpios, 2, np->name, handle_level_irq,
-	    IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_LEVEL, 0, 0);
+	    IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_LEVEL, 0, IRQ_GC_INIT_NESTED_LOCK);
 	if (err) {
 		dev_err(&pdev->dev, "couldn't allocate irq chips %s (DT).\n",
 			mvchip->chip.label);
@@ -1271,6 +1300,8 @@ static int mvebu_gpio_probe(struct platform_device *pdev)
 	ct->chip.irq_mask = mvebu_gpio_level_irq_mask;
 	ct->chip.irq_unmask = mvebu_gpio_level_irq_unmask;
 	ct->chip.irq_set_type = mvebu_gpio_irq_set_type;
+	ct->chip.irq_set_wake = mvebu_gpio_set_wake_irq;
+	ct->chip.flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_MASK_ON_SUSPEND;
 	ct->chip.name = mvchip->chip.label;
 
 	ct = &gc->chip_types[1];
@@ -1279,6 +1310,8 @@ static int mvebu_gpio_probe(struct platform_device *pdev)
 	ct->chip.irq_mask = mvebu_gpio_edge_irq_mask;
 	ct->chip.irq_unmask = mvebu_gpio_edge_irq_unmask;
 	ct->chip.irq_set_type = mvebu_gpio_irq_set_type;
+	ct->chip.irq_set_wake = mvebu_gpio_set_wake_irq;
+	ct->chip.flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_MASK_ON_SUSPEND;
 	ct->handler = handle_edge_irq;
 	ct->chip.name = mvchip->chip.label;
 
@@ -1287,13 +1320,14 @@ static int mvebu_gpio_probe(struct platform_device *pdev)
 	 * interrupt handlers, with each handler dealing with 8 GPIO
 	 * pins.
 	 */
-	for (i = 0; i < 4; i++) {
+	for (i = 0; i < ARRAY_SIZE(mvchip->bank_irq); i++) {
 		int irq = platform_get_irq_optional(pdev, i);
 
 		if (irq < 0)
 			continue;
 		irq_set_chained_handler_and_data(irq, mvebu_gpio_irq_handler,
 						 mvchip);
+		mvchip->bank_irq[i] = irq;
 	}
 
 	return 0;
diff --git a/drivers/gpio/gpio-nomadik.c b/drivers/gpio/gpio-nomadik.c
index 5dc9f9d5912ab..208fed631975c 100644
--- a/drivers/gpio/gpio-nomadik.c
+++ b/drivers/gpio/gpio-nomadik.c
@@ -519,22 +519,23 @@ struct nmk_gpio_chip *nmk_gpio_populate_chip(struct fwnode_handle *fwnode,
 
 	gpio_dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode);
 	if (!gpio_dev) {
-		dev_err(dev, "populate \"%pfwP\": device not found\n", fwnode);
-		return ERR_PTR(-ENODEV);
+		ret = -ENODEV;
+		dev_err_probe(dev, ret, "populate \"%pfwP\": device not found\n", fwnode);
+		return ERR_PTR(ret);
 	}
 	gpio_pdev = to_platform_device(gpio_dev);
 
 	if (device_property_read_u32(gpio_dev, "gpio-bank", &id)) {
-		dev_err(dev, "populate: gpio-bank property not found\n");
-		platform_device_put(gpio_pdev);
-		return ERR_PTR(-EINVAL);
+		ret = -EINVAL;
+		dev_err_probe(dev, ret, "populate: gpio-bank property not found\n");
+		goto err_put_pdev;
 	}
 
 #ifdef CONFIG_PINCTRL_NOMADIK
 	if (id >= ARRAY_SIZE(nmk_gpio_chips)) {
-		dev_err(dev, "populate: invalid id: %u\n", id);
-		platform_device_put(gpio_pdev);
-		return ERR_PTR(-EINVAL);
+		ret = -EINVAL;
+		dev_err_probe(dev, ret, "populate: invalid id: %u\n", id);
+		goto err_put_pdev;
 	}
 	/* Already populated? */
 	nmk_chip = nmk_gpio_chips[id];
@@ -546,8 +547,8 @@ struct nmk_gpio_chip *nmk_gpio_populate_chip(struct fwnode_handle *fwnode,
 
 	nmk_chip = devm_kzalloc(dev, sizeof(*nmk_chip), GFP_KERNEL);
 	if (!nmk_chip) {
-		platform_device_put(gpio_pdev);
-		return ERR_PTR(-ENOMEM);
+		ret = -ENOMEM;
+		goto err_put_pdev;
 	}
 
 	if (device_property_read_u32(gpio_dev, "ngpios", &ngpio)) {
@@ -568,16 +569,17 @@ struct nmk_gpio_chip *nmk_gpio_populate_chip(struct fwnode_handle *fwnode,
 	res = platform_get_resource(gpio_pdev, IORESOURCE_MEM, 0);
 	base = devm_ioremap_resource(dev, res);
 	if (IS_ERR(base)) {
-		platform_device_put(gpio_pdev);
-		return ERR_CAST(base);
+		ret = PTR_ERR(base);
+		goto err_put_pdev;
 	}
 	nmk_chip->addr = base;
 
 	/* NOTE: do not use devm_ here! */
 	clk = clk_get_optional(gpio_dev, NULL);
 	if (IS_ERR(clk)) {
-		platform_device_put(gpio_pdev);
-		return ERR_CAST(clk);
+		ret = PTR_ERR(clk);
+		dev_err_probe(dev, ret, "failed getting clock\n");
+		goto err_put_pdev;
 	}
 	clk_prepare(clk);
 	nmk_chip->clk = clk;
@@ -585,12 +587,9 @@ struct nmk_gpio_chip *nmk_gpio_populate_chip(struct fwnode_handle *fwnode,
 	/* NOTE: do not use devm_ here! */
 	reset = reset_control_get_optional_shared(gpio_dev, NULL);
 	if (IS_ERR(reset)) {
-		clk_unprepare(clk);
-		clk_put(clk);
-		platform_device_put(gpio_pdev);
-		dev_err(dev, "failed getting reset control: %pe\n",
-			reset);
-		return ERR_CAST(reset);
+		ret = PTR_ERR(reset);
+		dev_err_probe(dev, ret, "failed getting reset control\n");
+		goto err_unprepare_clk;
 	}
 
 	/*
@@ -600,18 +599,23 @@ struct nmk_gpio_chip *nmk_gpio_populate_chip(struct fwnode_handle *fwnode,
 	 */
 	ret = reset_control_deassert(reset);
 	if (ret) {
-		reset_control_put(reset);
-		clk_unprepare(clk);
-		clk_put(clk);
-		platform_device_put(gpio_pdev);
-		dev_err(dev, "failed reset deassert: %d\n", ret);
-		return ERR_PTR(ret);
+		dev_err_probe(dev, ret, "failed reset deassert\n");
+		goto err_put_reset;
 	}
 
 #ifdef CONFIG_PINCTRL_NOMADIK
 	nmk_gpio_chips[id] = nmk_chip;
 #endif
 	return nmk_chip;
+
+err_put_reset:
+	reset_control_put(reset);
+err_unprepare_clk:
+	clk_unprepare(clk);
+	clk_put(clk);
+err_put_pdev:
+	platform_device_put(gpio_pdev);
+	return ERR_PTR(ret);
 }
 
 static void nmk_gpio_irq_print_chip(struct irq_data *d, struct seq_file *p)
@@ -647,10 +651,8 @@ static int nmk_gpio_probe(struct platform_device *pdev)
 	int ret;
 
 	nmk_chip = nmk_gpio_populate_chip(dev_fwnode(dev), pdev);
-	if (IS_ERR(nmk_chip)) {
-		dev_err(dev, "could not populate nmk chip struct\n");
+	if (IS_ERR(nmk_chip))
 		return PTR_ERR(nmk_chip);
-	}
 
 	supports_sleepmode =
 		device_property_read_bool(dev, "st,supports-sleepmode");
@@ -693,7 +695,7 @@ static int nmk_gpio_probe(struct platform_device *pdev)
 	ret = devm_request_irq(dev, irq, nmk_gpio_irq_handler, IRQF_SHARED,
 			       dev_name(dev), nmk_chip);
 	if (ret) {
-		dev_err(dev, "failed requesting IRQ\n");
+		dev_err_probe(dev, ret, "failed requesting IRQ\n");
 		return ret;
 	}
 
@@ -709,8 +711,6 @@ static int nmk_gpio_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, nmk_chip);
 
-	dev_info(dev, "chip registered\n");
-
 	return 0;
 }
 
diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c
index f6b870b7b3521..e03bf1b120910 100644
--- a/drivers/gpio/gpio-pca953x.c
+++ b/drivers/gpio/gpio-pca953x.c
@@ -604,20 +604,28 @@ static int pca953x_read_regs(struct pca953x_chip *chip, int reg, unsigned long *
 	return 0;
 }
 
-static int pca953x_gpio_direction_input(struct gpio_chip *gc, unsigned off)
+static int pca953x_gpio_direction_input_unlocked(struct gpio_chip *gc,
+						 unsigned int off)
 {
 	struct pca953x_chip *chip = gpiochip_get_data(gc);
 	u8 dirreg = chip->recalc_addr(chip, chip->regs->direction, off);
 	u8 bit = pca953x_get_bit_mask(chip, off);
 
-	guard(mutex)(&chip->i2c_lock);
-
 	if (PCA_CHIP_TYPE(chip->driver_data) == TCA6418_TYPE)
 		return regmap_update_bits(chip->regmap, dirreg, bit, 0);
 
 	return regmap_update_bits(chip->regmap, dirreg, bit, bit);
 }
 
+static int pca953x_gpio_direction_input(struct gpio_chip *gc, unsigned int off)
+{
+	struct pca953x_chip *chip = gpiochip_get_data(gc);
+
+	guard(mutex)(&chip->i2c_lock);
+
+	return pca953x_gpio_direction_input_unlocked(gc, off);
+}
+
 static int pca953x_gpio_direction_output(struct gpio_chip *gc,
 		unsigned off, int val)
 {
@@ -855,9 +863,10 @@ static void pca953x_irq_bus_sync_unlock(struct irq_data *d)
 	DECLARE_BITMAP(reg_direction, MAX_LINE);
 	int level;
 
+	guard(mutex)(&chip->i2c_lock);
+
 	if (chip->driver_data & PCA_PCAL) {
 		DECLARE_BITMAP(latched_inputs, MAX_LINE);
-		guard(mutex)(&chip->i2c_lock);
 
 		/* Enable latch on edge-triggered interrupt-enabled inputs */
 		bitmap_or(latched_inputs, chip->irq_trig_fall, chip->irq_trig_raise, gc->ngpio);
@@ -879,7 +888,7 @@ static void pca953x_irq_bus_sync_unlock(struct irq_data *d)
 
 	/* Look for any newly setup interrupt */
 	for_each_andnot_bit(level, irq_mask, reg_direction, gc->ngpio)
-		pca953x_gpio_direction_input(&chip->gpio_chip, level);
+		pca953x_gpio_direction_input_unlocked(&chip->gpio_chip, level);
 
 	mutex_unlock(&chip->irq_lock);
 }
diff --git a/drivers/gpio/gpio-pca9570.c b/drivers/gpio/gpio-pca9570.c
index 7a47a9aa0414e..db027c10494f7 100644
--- a/drivers/gpio/gpio-pca9570.c
+++ b/drivers/gpio/gpio-pca9570.c
@@ -133,7 +133,7 @@ static int pca9570_probe(struct i2c_client *client)
 	gpio->chip.get = pca9570_get;
 	gpio->chip.set = pca9570_set;
 	gpio->chip.base = -1;
-	gpio->chip_data = device_get_match_data(&client->dev);
+	gpio->chip_data = i2c_get_match_data(client);
 	gpio->chip.ngpio = gpio->chip_data->ngpio;
 	gpio->chip.can_sleep = true;
 
diff --git a/drivers/gpio/gpio-pcf857x.c b/drivers/gpio/gpio-pcf857x.c
index 4196916c4f94f..c1f5e10a3c202 100644
--- a/drivers/gpio/gpio-pcf857x.c
+++ b/drivers/gpio/gpio-pcf857x.c
@@ -18,6 +18,8 @@
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 
+#include <asm/byteorder.h>
+
 static const struct i2c_device_id pcf857x_id[] = {
 	{ .name = "pcf8574", .driver_data = 8 },
 	{ .name = "pcf8574a", .driver_data = 8 },
@@ -96,22 +98,23 @@ static int i2c_read_le8(struct i2c_client *client)
 
 static int i2c_write_le16(struct i2c_client *client, unsigned int word)
 {
-	u8 buf[2] = { word & 0xff, word >> 8, };
+	__le16 buf = cpu_to_le16(word);
 	int status;
 
-	status = i2c_master_send(client, buf, 2);
+	status = i2c_master_send(client, (char *)&buf, sizeof(buf));
 	return (status < 0) ? status : 0;
 }
 
 static int i2c_read_le16(struct i2c_client *client)
 {
-	u8 buf[2];
+	__le16 buf;
 	int status;
 
-	status = i2c_master_recv(client, buf, 2);
+	status = i2c_master_recv(client, (char *)&buf, sizeof(buf));
 	if (status < 0)
 		return status;
-	return (buf[1] << 8) | buf[0];
+
+	return le16_to_cpu(buf);
 }
 
 /*-------------------------------------------------------------------------*/
diff --git a/drivers/gpio/gpio-rcar.c b/drivers/gpio/gpio-rcar.c
index 86777e097fd87..09bebde5c4260 100644
--- a/drivers/gpio/gpio-rcar.c
+++ b/drivers/gpio/gpio-rcar.c
@@ -59,7 +59,6 @@ struct gpio_rcar_priv {
 #define MSKCLR		0x1c	/* Interrupt Mask Clear Register */
 #define POSNEG		0x20	/* Positive/Negative Logic Select Register */
 #define EDGLEVEL	0x24	/* Edge/level Select Register */
-#define FILONOFF	0x28	/* Chattering Prevention On/Off Register */
 #define OUTDTSEL	0x40	/* Output Data Select Register */
 #define BOTHEDGE	0x4c	/* One Edge/Both Edge Select Register */
 #define INEN		0x50	/* General Input Enable Register */
diff --git a/drivers/gpio/gpio-rockchip.c b/drivers/gpio/gpio-rockchip.c
index 9478a58f1caa1..77b239a9a601e 100644
--- a/drivers/gpio/gpio-rockchip.c
+++ b/drivers/gpio/gpio-rockchip.c
@@ -647,21 +647,17 @@ static void rockchip_clk_put(void *data)
 
 static int rockchip_get_bank_data(struct rockchip_pin_bank *bank)
 {
-	struct resource res;
+	struct platform_device *pdev = to_platform_device(bank->dev);
 	int id = 0, ret;
 
-	if (of_address_to_resource(bank->of_node, 0, &res)) {
-		dev_err(bank->dev, "cannot find IO resource for bank\n");
-		return -ENOENT;
-	}
-
-	bank->reg_base = devm_ioremap_resource(bank->dev, &res);
+	bank->reg_base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(bank->reg_base))
 		return PTR_ERR(bank->reg_base);
 
-	bank->irq = irq_of_parse_and_map(bank->of_node, 0);
-	if (!bank->irq)
-		return -EINVAL;
+	ret = platform_get_irq(pdev, 0);
+	if (ret < 0)
+		return ret;
+	bank->irq = ret;
 
 	bank->clk = devm_clk_get_enabled(bank->dev, NULL);
 	if (IS_ERR(bank->clk))
diff --git a/drivers/gpio/gpio-rtd1625.c b/drivers/gpio/gpio-rtd1625.c
new file mode 100644
index 0000000000000..409e540bf40b3
--- /dev/null
+++ b/drivers/gpio/gpio-rtd1625.c
@@ -0,0 +1,611 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Realtek DHC RTD1625 gpio driver
+ *
+ * Copyright (c) 2023-2026 Realtek Semiconductor Corp.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/cleanup.h>
+#include <linux/gpio/driver.h>
+#include <linux/interrupt.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+
+#define RTD1625_GPIO_DIR BIT(0)
+#define RTD1625_GPIO_OUT BIT(2)
+#define RTD1625_GPIO_IN BIT(4)
+#define RTD1625_GPIO_EDGE_INT_DP BIT(6)
+#define RTD1625_GPIO_EDGE_INT_EN BIT(8)
+#define RTD1625_GPIO_LEVEL_INT_EN BIT(16)
+#define RTD1625_GPIO_LEVEL_INT_DP BIT(18)
+#define RTD1625_GPIO_DEBOUNCE GENMASK(30, 28)
+#define RTD1625_GPIO_DEBOUNCE_WREN BIT(31)
+
+#define RTD1625_GPIO_WREN(x) ((x) << 1)
+
+/* Write-enable masks for all GPIO configs and reserved hardware bits */
+#define RTD1625_ISO_GPIO_WREN_ALL 0x8000aa8a
+#define RTD1625_ISOM_GPIO_WREN_ALL 0x800aaa8a
+
+#define RTD1625_GPIO_DEBOUNCE_1US 0
+#define RTD1625_GPIO_DEBOUNCE_10US 1
+#define RTD1625_GPIO_DEBOUNCE_100US 2
+#define RTD1625_GPIO_DEBOUNCE_1MS 3
+#define RTD1625_GPIO_DEBOUNCE_10MS 4
+#define RTD1625_GPIO_DEBOUNCE_20MS 5
+#define RTD1625_GPIO_DEBOUNCE_30MS 6
+#define RTD1625_GPIO_DEBOUNCE_50MS 7
+
+#define GPIO_CONTROL(gpio) ((gpio) * 4)
+
+enum rtd1625_irq_index {
+	RTD1625_IRQ_ASSERT,
+	RTD1625_IRQ_DEASSERT,
+	RTD1625_IRQ_LEVEL,
+	RTD1625_MAX_IRQS
+};
+
+/**
+ * struct rtd1625_gpio_info - Specific GPIO register information
+ * @num_gpios: The number of GPIOs
+ * @irq_type_support: Supported IRQ types
+ * @gpa_offset: Offset for GPIO assert interrupt status registers
+ * @gpda_offset: Offset for GPIO deassert interrupt status registers
+ * @level_offset: Offset of level interrupt status register
+ * @write_en_all: Write-enable mask for all configurable bits
+ */
+struct rtd1625_gpio_info {
+	unsigned int	num_gpios;
+	unsigned int	irq_type_support;
+	unsigned int	base_offset;
+	unsigned int	gpa_offset;
+	unsigned int	gpda_offset;
+	unsigned int	level_offset;
+	unsigned int	write_en_all;
+};
+
+struct rtd1625_gpio {
+	struct gpio_chip		gpio_chip;
+	const struct rtd1625_gpio_info	*info;
+	void __iomem			*base;
+	void __iomem			*irq_base;
+	unsigned int			irqs[RTD1625_MAX_IRQS];
+	raw_spinlock_t			lock;
+	unsigned int			*save_regs;
+};
+
+static unsigned int rtd1625_gpio_gpa_offset(struct rtd1625_gpio *data, unsigned int offset)
+{
+	return data->info->gpa_offset + ((offset / 32) * 4);
+}
+
+static unsigned int rtd1625_gpio_gpda_offset(struct rtd1625_gpio *data, unsigned int offset)
+{
+	return data->info->gpda_offset + ((offset / 32) * 4);
+}
+
+static unsigned int rtd1625_gpio_level_offset(struct rtd1625_gpio *data, unsigned int offset)
+{
+	return data->info->level_offset + ((offset / 32) * 4);
+}
+
+static int rtd1625_gpio_set_debounce(struct gpio_chip *chip, unsigned int offset,
+				     unsigned int debounce)
+{
+	struct rtd1625_gpio *data = gpiochip_get_data(chip);
+	u8 deb_val;
+	u32 val;
+
+	switch (debounce) {
+	case 1:
+		deb_val = RTD1625_GPIO_DEBOUNCE_1US;
+		break;
+	case 10:
+		deb_val = RTD1625_GPIO_DEBOUNCE_10US;
+		break;
+	case 100:
+		deb_val = RTD1625_GPIO_DEBOUNCE_100US;
+		break;
+	case 1000:
+		deb_val = RTD1625_GPIO_DEBOUNCE_1MS;
+		break;
+	case 10000:
+		deb_val = RTD1625_GPIO_DEBOUNCE_10MS;
+		break;
+	case 20000:
+		deb_val = RTD1625_GPIO_DEBOUNCE_20MS;
+		break;
+	case 30000:
+		deb_val = RTD1625_GPIO_DEBOUNCE_30MS;
+		break;
+	case 50000:
+		deb_val = RTD1625_GPIO_DEBOUNCE_50MS;
+		break;
+	default:
+		return -ENOTSUPP;
+	}
+
+	val = FIELD_PREP(RTD1625_GPIO_DEBOUNCE, deb_val) | RTD1625_GPIO_DEBOUNCE_WREN;
+
+	guard(raw_spinlock_irqsave)(&data->lock);
+
+	writel_relaxed(val, data->base + GPIO_CONTROL(offset));
+
+	return 0;
+}
+
+static int rtd1625_gpio_set_config(struct gpio_chip *chip, unsigned int offset,
+				   unsigned long config)
+{
+	u32 debounce;
+
+	if (pinconf_to_config_param(config) == PIN_CONFIG_INPUT_DEBOUNCE) {
+		debounce = pinconf_to_config_argument(config);
+		return rtd1625_gpio_set_debounce(chip, offset, debounce);
+	}
+
+	return gpiochip_generic_config(chip, offset, config);
+}
+
+static int rtd1625_gpio_set(struct gpio_chip *chip, unsigned int offset, int value)
+{
+	struct rtd1625_gpio *data = gpiochip_get_data(chip);
+	u32 val = RTD1625_GPIO_WREN(RTD1625_GPIO_OUT);
+
+	if (value)
+		val |= RTD1625_GPIO_OUT;
+
+	guard(raw_spinlock_irqsave)(&data->lock);
+
+	writel_relaxed(val, data->base + GPIO_CONTROL(offset));
+
+	return 0;
+}
+
+static int rtd1625_gpio_get(struct gpio_chip *chip, unsigned int offset)
+{
+	struct rtd1625_gpio *data = gpiochip_get_data(chip);
+	u32 val;
+
+	guard(raw_spinlock_irqsave)(&data->lock);
+
+	val = readl_relaxed(data->base + GPIO_CONTROL(offset));
+
+	if (val & RTD1625_GPIO_DIR)
+		return !!(val & RTD1625_GPIO_OUT);
+	else
+		return !!(val & RTD1625_GPIO_IN);
+}
+
+static int rtd1625_gpio_get_direction(struct gpio_chip *chip, unsigned int offset)
+{
+	struct rtd1625_gpio *data = gpiochip_get_data(chip);
+	u32 val;
+
+	guard(raw_spinlock_irqsave)(&data->lock);
+
+	val = readl_relaxed(data->base + GPIO_CONTROL(offset));
+
+	if (val & RTD1625_GPIO_DIR)
+		return GPIO_LINE_DIRECTION_OUT;
+
+	return GPIO_LINE_DIRECTION_IN;
+}
+
+static int rtd1625_gpio_set_direction(struct gpio_chip *chip, unsigned int offset, bool out)
+{
+	struct rtd1625_gpio *data = gpiochip_get_data(chip);
+	u32 val = RTD1625_GPIO_WREN(RTD1625_GPIO_DIR);
+
+	if (out)
+		val |= RTD1625_GPIO_DIR;
+
+	guard(raw_spinlock_irqsave)(&data->lock);
+
+	writel_relaxed(val, data->base + GPIO_CONTROL(offset));
+
+	return 0;
+}
+
+static int rtd1625_gpio_direction_input(struct gpio_chip *chip, unsigned int offset)
+{
+	return rtd1625_gpio_set_direction(chip, offset, false);
+}
+
+static int rtd1625_gpio_direction_output(struct gpio_chip *chip, unsigned int offset, int value)
+{
+	rtd1625_gpio_set(chip, offset, value);
+
+	return rtd1625_gpio_set_direction(chip, offset, true);
+}
+
+static void rtd1625_gpio_irq_handle(struct irq_desc *desc)
+{
+	unsigned int (*get_reg_offset)(struct rtd1625_gpio *gpio, unsigned int offset);
+	struct rtd1625_gpio *data = irq_desc_get_handler_data(desc);
+	struct irq_domain *domain = data->gpio_chip.irq.domain;
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+	unsigned int irq = irq_desc_get_irq(desc);
+	unsigned long status;
+	unsigned int reg_offset, i, j;
+	unsigned int girq;
+	irq_hw_number_t hwirq;
+	u32 irq_type;
+
+	if (irq == data->irqs[RTD1625_IRQ_ASSERT])
+		get_reg_offset = &rtd1625_gpio_gpa_offset;
+	else if (irq == data->irqs[RTD1625_IRQ_DEASSERT])
+		get_reg_offset = &rtd1625_gpio_gpda_offset;
+	else if (irq == data->irqs[2])
+		get_reg_offset = &rtd1625_gpio_level_offset;
+	else
+		return;
+
+	chained_irq_enter(chip, desc);
+
+	for (i = 0; i < data->info->num_gpios; i += 32) {
+		reg_offset = get_reg_offset(data, i);
+		status = readl_relaxed(data->irq_base + reg_offset);
+
+		/*
+		 * Hardware quirk: The controller fires both "assert" and "de-assert"
+		 * interrupts simultaneously on any edge toggle.
+		 * We must pre-clear edge interrupts here. If we drop an unwanted
+		 * de-assert interrupt below, it will never reach the IRQ core
+		 * (generic_handle_domain_irq), meaning ->irq_ack() won't be called.
+		 * Failing to clear it here leads to an interrupt storm.
+		 */
+		if (irq != data->irqs[RTD1625_IRQ_LEVEL])
+			writel_relaxed(status, data->irq_base + reg_offset);
+
+		for_each_set_bit(j, &status, 32) {
+			hwirq = i + j;
+			girq = irq_find_mapping(domain, hwirq);
+			irq_type = irq_get_trigger_type(girq);
+
+			/*
+			 * Filter out the hardware-forced de-assert interrupt unless
+			 * the user explicitly requested IRQ_TYPE_EDGE_BOTH.
+			 */
+			if (irq == data->irqs[RTD1625_IRQ_DEASSERT] &&
+			    irq_type != IRQ_TYPE_EDGE_BOTH)
+				continue;
+
+			generic_handle_domain_irq(domain, hwirq);
+		}
+	}
+
+	chained_irq_exit(chip, desc);
+}
+
+static void rtd1625_gpio_ack_irq(struct irq_data *d)
+{
+	struct rtd1625_gpio *data = irq_data_get_irq_chip_data(d);
+	irq_hw_number_t hwirq = irqd_to_hwirq(d);
+	u32 irq_type = irqd_get_trigger_type(d);
+	u32 bit_mask = BIT(hwirq % 32);
+	int reg_offset;
+
+	if (irq_type & IRQ_TYPE_LEVEL_MASK) {
+		reg_offset = rtd1625_gpio_level_offset(data, hwirq);
+		writel_relaxed(bit_mask, data->irq_base + reg_offset);
+	}
+}
+
+static void rtd1625_gpio_enable_edge_irq(struct rtd1625_gpio *data, irq_hw_number_t hwirq)
+{
+	int gpda_reg_offset = rtd1625_gpio_gpda_offset(data, hwirq);
+	int gpa_reg_offset = rtd1625_gpio_gpa_offset(data, hwirq);
+	u32 clr_mask = BIT(hwirq % 32);
+	u32 val;
+
+	guard(raw_spinlock_irqsave)(&data->lock);
+
+	writel_relaxed(clr_mask, data->irq_base + gpa_reg_offset);
+	writel_relaxed(clr_mask, data->irq_base + gpda_reg_offset);
+	val = RTD1625_GPIO_EDGE_INT_EN | RTD1625_GPIO_WREN(RTD1625_GPIO_EDGE_INT_EN);
+	writel_relaxed(val, data->base + GPIO_CONTROL(hwirq));
+}
+
+static void rtd1625_gpio_disable_edge_irq(struct rtd1625_gpio *data, irq_hw_number_t hwirq)
+{
+	u32 val;
+
+	guard(raw_spinlock_irqsave)(&data->lock);
+
+	val = RTD1625_GPIO_WREN(RTD1625_GPIO_EDGE_INT_EN);
+	writel_relaxed(val, data->base + GPIO_CONTROL(hwirq));
+}
+
+static void rtd1625_gpio_enable_level_irq(struct rtd1625_gpio *data, irq_hw_number_t hwirq)
+{
+	int level_reg_offset = rtd1625_gpio_level_offset(data, hwirq);
+	u32 clr_mask = BIT(hwirq % 32);
+	u32 val;
+
+	guard(raw_spinlock_irqsave)(&data->lock);
+
+	writel_relaxed(clr_mask, data->irq_base + level_reg_offset);
+	val = RTD1625_GPIO_LEVEL_INT_EN | RTD1625_GPIO_WREN(RTD1625_GPIO_LEVEL_INT_EN);
+	writel_relaxed(val, data->base + GPIO_CONTROL(hwirq));
+}
+
+static void rtd1625_gpio_disable_level_irq(struct rtd1625_gpio *data, irq_hw_number_t hwirq)
+{
+	u32 val;
+
+	guard(raw_spinlock_irqsave)(&data->lock);
+
+	val = RTD1625_GPIO_WREN(RTD1625_GPIO_LEVEL_INT_EN);
+	writel_relaxed(val, data->base + GPIO_CONTROL(hwirq));
+}
+
+static void rtd1625_gpio_enable_irq(struct irq_data *d)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct rtd1625_gpio *data = gpiochip_get_data(gc);
+	irq_hw_number_t hwirq = irqd_to_hwirq(d);
+	u32 irq_type = irqd_get_trigger_type(d);
+
+	gpiochip_enable_irq(gc, hwirq);
+
+	if (irq_type & IRQ_TYPE_EDGE_BOTH)
+		rtd1625_gpio_enable_edge_irq(data, hwirq);
+	else if (irq_type & IRQ_TYPE_LEVEL_MASK)
+		rtd1625_gpio_enable_level_irq(data, hwirq);
+}
+
+static void rtd1625_gpio_disable_irq(struct irq_data *d)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct rtd1625_gpio *data = gpiochip_get_data(gc);
+	irq_hw_number_t hwirq = irqd_to_hwirq(d);
+	u32 irq_type = irqd_get_trigger_type(d);
+
+	if (irq_type & IRQ_TYPE_EDGE_BOTH)
+		rtd1625_gpio_disable_edge_irq(data, hwirq);
+	else if (irq_type & IRQ_TYPE_LEVEL_MASK)
+		rtd1625_gpio_disable_level_irq(data, hwirq);
+
+	gpiochip_disable_irq(gc, hwirq);
+}
+
+static int rtd1625_gpio_irq_set_level_type(struct irq_data *d, bool level)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct rtd1625_gpio *data = gpiochip_get_data(gc);
+	irq_hw_number_t hwirq = irqd_to_hwirq(d);
+	u32 val = RTD1625_GPIO_WREN(RTD1625_GPIO_LEVEL_INT_DP);
+
+	if (!(data->info->irq_type_support & IRQ_TYPE_LEVEL_MASK))
+		return -EINVAL;
+
+	if (level)
+		val |= RTD1625_GPIO_LEVEL_INT_DP;
+
+	scoped_guard(raw_spinlock_irqsave, &data->lock)
+		writel_relaxed(val, data->base + GPIO_CONTROL(hwirq));
+
+	irq_set_handler_locked(d, handle_level_irq);
+
+	return 0;
+}
+
+static int rtd1625_gpio_irq_set_edge_type(struct irq_data *d, bool polarity)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct rtd1625_gpio *data = gpiochip_get_data(gc);
+	irq_hw_number_t hwirq = irqd_to_hwirq(d);
+	u32 val = RTD1625_GPIO_WREN(RTD1625_GPIO_EDGE_INT_DP);
+
+	if (!(data->info->irq_type_support & IRQ_TYPE_EDGE_BOTH))
+		return -EINVAL;
+
+	if (polarity)
+		val |= RTD1625_GPIO_EDGE_INT_DP;
+
+	scoped_guard(raw_spinlock_irqsave, &data->lock)
+		writel_relaxed(val, data->base + GPIO_CONTROL(hwirq));
+
+	irq_set_handler_locked(d, handle_edge_irq);
+
+	return 0;
+}
+
+static int rtd1625_gpio_irq_set_type(struct irq_data *d, unsigned int type)
+{
+	switch (type & IRQ_TYPE_SENSE_MASK) {
+	case IRQ_TYPE_EDGE_RISING:
+		return rtd1625_gpio_irq_set_edge_type(d, 1);
+
+	case IRQ_TYPE_EDGE_FALLING:
+		return rtd1625_gpio_irq_set_edge_type(d, 0);
+
+	case IRQ_TYPE_EDGE_BOTH:
+		return rtd1625_gpio_irq_set_edge_type(d, 1);
+
+	case IRQ_TYPE_LEVEL_HIGH:
+		return rtd1625_gpio_irq_set_level_type(d, 0);
+
+	case IRQ_TYPE_LEVEL_LOW:
+		return rtd1625_gpio_irq_set_level_type(d, 1);
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static struct irq_chip rtd1625_iso_gpio_irq_chip = {
+	.name = "rtd1625-gpio",
+	.irq_ack = rtd1625_gpio_ack_irq,
+	.irq_mask = rtd1625_gpio_disable_irq,
+	.irq_unmask = rtd1625_gpio_enable_irq,
+	.irq_set_type = rtd1625_gpio_irq_set_type,
+	.flags = IRQCHIP_IMMUTABLE | IRQCHIP_SKIP_SET_WAKE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
+static int rtd1625_gpio_setup_irq(struct platform_device *pdev, struct rtd1625_gpio *data)
+{
+	struct gpio_irq_chip *irq_chip;
+	unsigned int num_irqs;
+	int irq;
+
+	/*
+	 * Interrupt support is optional. All IRQs must be provided together.
+	 * If index 0 is missing, we assume no interrupts are configured in DT
+	 * and fall back to basic GPIO operation.
+	 */
+	irq = platform_get_irq_optional(pdev, 0);
+	if (irq == -ENXIO)
+		return 0;
+	if (irq < 0)
+		return irq;
+
+	num_irqs = (data->info->irq_type_support & IRQ_TYPE_LEVEL_MASK) ? 3 : 2;
+	data->irqs[RTD1625_IRQ_ASSERT] = irq;
+
+	for (unsigned int i = 1; i < num_irqs; i++) {
+		irq = platform_get_irq(pdev, i);
+		if (irq < 0)
+			return irq;
+		data->irqs[i] = irq;
+	}
+
+	irq_chip = &data->gpio_chip.irq;
+	irq_chip->handler = handle_bad_irq;
+	irq_chip->default_type = IRQ_TYPE_NONE;
+	irq_chip->parent_handler = rtd1625_gpio_irq_handle;
+	irq_chip->parent_handler_data = data;
+	irq_chip->num_parents = num_irqs;
+	irq_chip->parents = data->irqs;
+
+	gpio_irq_chip_set_chip(irq_chip, &rtd1625_iso_gpio_irq_chip);
+
+	return 0;
+}
+
+static int rtd1625_gpio_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct rtd1625_gpio *data;
+	void __iomem *irq_base;
+	int ret;
+
+	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->info = device_get_match_data(dev);
+	if (!data->info)
+		return -EINVAL;
+
+	raw_spin_lock_init(&data->lock);
+
+	irq_base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(irq_base))
+		return PTR_ERR(irq_base);
+
+	data->irq_base = irq_base;
+	data->base = irq_base + data->info->base_offset;
+
+	data->save_regs = devm_kcalloc(dev, data->info->num_gpios, sizeof(*data->save_regs),
+				       GFP_KERNEL);
+	if (!data->save_regs)
+		return -ENOMEM;
+
+	data->gpio_chip.label = dev_name(dev);
+	data->gpio_chip.base = -1;
+	data->gpio_chip.ngpio = data->info->num_gpios;
+	data->gpio_chip.request = gpiochip_generic_request;
+	data->gpio_chip.free = gpiochip_generic_free;
+	data->gpio_chip.get_direction = rtd1625_gpio_get_direction;
+	data->gpio_chip.direction_input = rtd1625_gpio_direction_input;
+	data->gpio_chip.direction_output = rtd1625_gpio_direction_output;
+	data->gpio_chip.set = rtd1625_gpio_set;
+	data->gpio_chip.get = rtd1625_gpio_get;
+	data->gpio_chip.set_config = rtd1625_gpio_set_config;
+	data->gpio_chip.parent = dev;
+
+	ret = rtd1625_gpio_setup_irq(pdev, data);
+	if (ret)
+		return ret;
+
+	platform_set_drvdata(pdev, data);
+
+	return devm_gpiochip_add_data(dev, &data->gpio_chip, data);
+}
+
+static const struct rtd1625_gpio_info rtd1625_iso_gpio_info = {
+	.num_gpios        = 166,
+	.irq_type_support = IRQ_TYPE_EDGE_BOTH,
+	.base_offset      = 0x100,
+	.gpa_offset       = 0x000,
+	.gpda_offset      = 0x020,
+	.write_en_all     = RTD1625_ISO_GPIO_WREN_ALL,
+};
+
+static const struct rtd1625_gpio_info rtd1625_isom_gpio_info = {
+	.num_gpios        = 4,
+	.irq_type_support = IRQ_TYPE_EDGE_BOTH | IRQ_TYPE_LEVEL_LOW |
+			    IRQ_TYPE_LEVEL_HIGH,
+	.base_offset      = 0x20,
+	.gpa_offset       = 0x00,
+	.gpda_offset      = 0x04,
+	.level_offset     = 0x18,
+	.write_en_all     = RTD1625_ISOM_GPIO_WREN_ALL,
+};
+
+static int rtd1625_gpio_suspend(struct device *dev)
+{
+	struct rtd1625_gpio *data = dev_get_drvdata(dev);
+	const struct rtd1625_gpio_info *info = data->info;
+
+	for (unsigned int i = 0; i < info->num_gpios; i++)
+		data->save_regs[i] = readl_relaxed(data->base + GPIO_CONTROL(i));
+
+	return 0;
+}
+
+static int rtd1625_gpio_resume(struct device *dev)
+{
+	struct rtd1625_gpio *data = dev_get_drvdata(dev);
+	const struct rtd1625_gpio_info *info = data->info;
+
+	for (unsigned int i = 0; i < info->num_gpios; i++)
+		writel_relaxed(data->save_regs[i] | info->write_en_all,
+			       data->base + GPIO_CONTROL(i));
+
+	return 0;
+}
+
+static DEFINE_NOIRQ_DEV_PM_OPS(rtd1625_gpio_pm_ops, rtd1625_gpio_suspend, rtd1625_gpio_resume);
+
+static const struct of_device_id rtd1625_gpio_of_matches[] = {
+	{ .compatible = "realtek,rtd1625-iso-gpio", .data = &rtd1625_iso_gpio_info },
+	{ .compatible = "realtek,rtd1625-isom-gpio", .data = &rtd1625_isom_gpio_info },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, rtd1625_gpio_of_matches);
+
+static struct platform_driver rtd1625_gpio_platform_driver = {
+	.driver = {
+		.name = "gpio-rtd1625",
+		.of_match_table = rtd1625_gpio_of_matches,
+		.pm = pm_sleep_ptr(&rtd1625_gpio_pm_ops),
+	},
+	.probe = rtd1625_gpio_probe,
+};
+module_platform_driver(rtd1625_gpio_platform_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Realtek Semiconductor Corporation");
+MODULE_DESCRIPTION("Realtek DHC SoC RTD1625 gpio driver");
diff --git a/drivers/gpio/gpio-sifive.c b/drivers/gpio/gpio-sifive.c
index 94ef2efbd14f5..e9a992cd4b9cd 100644
--- a/drivers/gpio/gpio-sifive.c
+++ b/drivers/gpio/gpio-sifive.c
@@ -260,6 +260,7 @@ static const struct of_device_id sifive_gpio_match[] = {
 	{ .compatible = "sifive,fu540-c000-gpio" },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, sifive_gpio_match);
 
 static struct platform_driver sifive_gpio_driver = {
 	.probe		= sifive_gpio_probe,
diff --git a/drivers/gpio/gpio-sloppy-logic-analyzer.c b/drivers/gpio/gpio-sloppy-logic-analyzer.c
index 2bbd308ca08e6..9ce9848cebebb 100644
--- a/drivers/gpio/gpio-sloppy-logic-analyzer.c
+++ b/drivers/gpio/gpio-sloppy-logic-analyzer.c
@@ -160,6 +160,13 @@ static int fops_buf_size_get(void *data, u64 *val)
 	return 0;
 }
 
+static void fops_buf_release(void *data)
+{
+	struct gpio_la_poll_priv *priv = data;
+
+	vfree(priv->blob.data);
+}
+
 static int fops_buf_size_set(void *data, u64 val)
 {
 	struct gpio_la_poll_priv *priv = data;
@@ -237,7 +244,11 @@ static int gpio_la_poll_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	/* Initially allocate a buffer. It currently is NULL */
 	fops_buf_size_set(priv, GPIO_LA_DEFAULT_BUF_SIZE);
+	ret = devm_add_action_or_reset(dev, fops_buf_release, priv);
+	if (ret)
+		return ret;
 
 	priv->descs = devm_gpiod_get_array(dev, "probe", GPIOD_IN);
 	if (IS_ERR(priv->descs))
diff --git a/drivers/gpio/gpio-tb10x.c b/drivers/gpio/gpio-tb10x.c
index 705bfd80a8d00..7fb8e6223bd19 100644
--- a/drivers/gpio/gpio-tb10x.c
+++ b/drivers/gpio/gpio-tb10x.c
@@ -51,7 +51,7 @@ static inline u32 tb10x_reg_read(struct tb10x_gpio *gpio, unsigned int offs)
 	return ioread32(gpio->base + offs);
 }
 
-static int tb10x_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
+static int tb10x_gpio_to_irq(struct gpio_chip *chip, unsigned int offset)
 {
 	struct tb10x_gpio *tb10x_gpio = gpiochip_get_data(chip);
 
@@ -167,9 +167,8 @@ static int tb10x_gpio_probe(struct platform_device *pdev)
 		tb10x_gpio->domain = irq_domain_create_linear(dev_fwnode(dev),
 							      tb10x_gpio->chip.gc.ngpio,
 							      &irq_generic_chip_ops, NULL);
-		if (!tb10x_gpio->domain) {
+		if (!tb10x_gpio->domain)
 			return -ENOMEM;
-		}
 
 		ret = irq_alloc_domain_generic_chips(tb10x_gpio->domain,
 				tb10x_gpio->chip.gc.ngpio, 1, tb10x_gpio->chip.gc.label,
diff --git a/drivers/gpio/gpio-vf610.c b/drivers/gpio/gpio-vf610.c
index aa8586d8a787f..adfa2c2d5873d 100644
--- a/drivers/gpio/gpio-vf610.c
+++ b/drivers/gpio/gpio-vf610.c
@@ -83,6 +83,7 @@ static const struct of_device_id vf610_gpio_dt_ids[] = {
 	{ .compatible = "fsl,imx8ulp-gpio",	.data = &imx8ulp_data, },
 	{ /* sentinel */ }
 };
+MODULE_DEVICE_TABLE(of, vf610_gpio_dt_ids);
 
 static inline void vf610_gpio_writel(u32 val, void __iomem *reg)
 {
diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c
index 82f27db0b2304..9f3b628d57934 100644
--- a/drivers/gpio/gpiolib-cdev.c
+++ b/drivers/gpio/gpiolib-cdev.c
@@ -2682,15 +2682,8 @@ static int gpio_chrdev_open(struct inode *inode, struct file *file)
 	file->private_data = cdev;
 	cdev->fp = file;
 
-	ret = nonseekable_open(inode, file);
-	if (ret)
-		goto out_unregister_device_notifier;
-
-	return ret;
+	return nonseekable_open(inode, file);
 
-out_unregister_device_notifier:
-	blocking_notifier_chain_unregister(&gdev->device_notifier,
-					   &cdev->device_unregistered_nb);
 out_unregister_line_notifier:
 	scoped_guard(write_lock_irqsave, &gdev->line_state_lock)
 		raw_notifier_chain_unregister(&gdev->line_state_notifier,
diff --git a/drivers/gpio/gpiolib-kunit.c b/drivers/gpio/gpiolib-kunit.c
index 380b68f879e55..c9c5b43748201 100644
--- a/drivers/gpio/gpiolib-kunit.c
+++ b/drivers/gpio/gpiolib-kunit.c
@@ -18,19 +18,67 @@
 #define GPIO_TEST_PROVIDER		"gpio-test-provider"
 #define GPIO_SWNODE_TEST_CONSUMER	"gpio-swnode-test-consumer"
 #define GPIO_UNBIND_TEST_CONSUMER	"gpio-unbind-test-consumer"
+#define GPIO_CONSUMER_NAME		"gpio-swnode-consumer-test-device"
+
+#define GPIO_TEST_PROVIDER_NGPIO	4
+
+/*
+ * The test provider tracks per-line direction and value so that lines can be
+ * driven as both inputs and outputs - this is needed to exercise input as well
+ * as output GPIO hogs.
+ */
+struct gpio_test_provider_data {
+	DECLARE_BITMAP(is_output, GPIO_TEST_PROVIDER_NGPIO);
+	DECLARE_BITMAP(values, GPIO_TEST_PROVIDER_NGPIO);
+};
 
 static int gpio_test_provider_get_direction(struct gpio_chip *gc, unsigned int offset)
 {
-	return GPIO_LINE_DIRECTION_OUT;
+	struct gpio_test_provider_data *data = gpiochip_get_data(gc);
+
+	return test_bit(offset, data->is_output) ?
+		GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN;
+}
+
+static int gpio_test_provider_direction_input(struct gpio_chip *gc, unsigned int offset)
+{
+	struct gpio_test_provider_data *data = gpiochip_get_data(gc);
+
+	clear_bit(offset, data->is_output);
+
+	return 0;
+}
+
+static int gpio_test_provider_direction_output(struct gpio_chip *gc, unsigned int offset,
+					       int value)
+{
+	struct gpio_test_provider_data *data = gpiochip_get_data(gc);
+
+	set_bit(offset, data->is_output);
+	__assign_bit(offset, data->values, value);
+
+	return 0;
+}
+
+static int gpio_test_provider_get(struct gpio_chip *gc, unsigned int offset)
+{
+	struct gpio_test_provider_data *data = gpiochip_get_data(gc);
+
+	return test_bit(offset, data->values);
 }
 
 static int gpio_test_provider_set(struct gpio_chip *gc, unsigned int offset, int value)
 {
+	struct gpio_test_provider_data *data = gpiochip_get_data(gc);
+
+	__assign_bit(offset, data->values, value);
+
 	return 0;
 }
 
 static int gpio_test_provider_probe(struct platform_device *pdev)
 {
+	struct gpio_test_provider_data *data;
 	struct device *dev = &pdev->dev;
 	struct gpio_chip *gc;
 
@@ -38,16 +86,26 @@ static int gpio_test_provider_probe(struct platform_device *pdev)
 	if (!gc)
 		return -ENOMEM;
 
+	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	/* Lines start as outputs to preserve the default for lookup tests. */
+	bitmap_fill(data->is_output, GPIO_TEST_PROVIDER_NGPIO);
+
 	gc->base = -1;
-	gc->ngpio = 4;
-	gc->label = "gpio-swnode-consumer-test-device";
+	gc->ngpio = GPIO_TEST_PROVIDER_NGPIO;
+	gc->label = GPIO_CONSUMER_NAME;
 	gc->parent = dev;
 	gc->owner = THIS_MODULE;
 
 	gc->get_direction = gpio_test_provider_get_direction;
+	gc->direction_input = gpio_test_provider_direction_input;
+	gc->direction_output = gpio_test_provider_direction_output;
+	gc->get = gpio_test_provider_get;
 	gc->set = gpio_test_provider_set;
 
-	return devm_gpiochip_add_data(dev, gc, NULL);
+	return devm_gpiochip_add_data(dev, gc, data);
 }
 
 static struct platform_driver gpio_test_provider_driver = {
@@ -63,10 +121,12 @@ static const struct software_node gpio_test_provider_swnode = {
 
 struct gpio_swnode_consumer_pdata {
 	bool gpio_ok;
+	int errno;
 };
 
 static const struct gpio_swnode_consumer_pdata gpio_swnode_pdata_template = {
 	.gpio_ok = false,
+	.errno = 0,
 };
 
 static int gpio_swnode_consumer_probe(struct platform_device *pdev)
@@ -76,8 +136,10 @@ static int gpio_swnode_consumer_probe(struct platform_device *pdev)
 	struct gpio_desc *desc;
 
 	desc = devm_gpiod_get(dev, "foo", GPIOD_OUT_HIGH);
-	if (IS_ERR(desc))
+	if (IS_ERR(desc)) {
+		pdata->errno = PTR_ERR(desc);
 		return PTR_ERR(desc);
+	}
 
 	pdata->gpio_ok = true;
 
@@ -91,13 +153,8 @@ static struct platform_driver gpio_swnode_consumer_driver = {
 	},
 };
 
-static void gpio_swnode_lookup_by_primary(struct kunit *test)
+static int gpio_swnode_register_drivers(struct kunit *test)
 {
-	struct gpio_swnode_consumer_pdata *pdata;
-	struct platform_device_info pdevinfo;
-	struct property_entry properties[2];
-	struct platform_device *pdev;
-	bool bound = false;
 	int ret;
 
 	ret = kunit_platform_driver_register(test, &gpio_test_provider_driver);
@@ -106,6 +163,17 @@ static void gpio_swnode_lookup_by_primary(struct kunit *test)
 	ret = kunit_platform_driver_register(test, &gpio_swnode_consumer_driver);
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
+	return 0;
+}
+
+static void gpio_swnode_lookup_by_primary(struct kunit *test)
+{
+	struct gpio_swnode_consumer_pdata *pdata;
+	struct platform_device_info pdevinfo;
+	struct property_entry properties[2];
+	struct platform_device *pdev;
+	bool bound = false;
+
 	pdevinfo = (struct platform_device_info){
 		.name = GPIO_TEST_PROVIDER,
 		.id = PLATFORM_DEVID_NONE,
@@ -149,7 +217,6 @@ static void gpio_swnode_lookup_by_secondary(struct kunit *test)
 	struct fwnode_handle *primary;
 	struct platform_device *pdev;
 	bool bound = false;
-	int ret;
 
 	/*
 	 * Can't live on the stack as it will still get referenced in cleanup
@@ -158,12 +225,6 @@ static void gpio_swnode_lookup_by_secondary(struct kunit *test)
 	primary = kunit_kzalloc(test, sizeof(*primary), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, primary);
 
-	ret = kunit_platform_driver_register(test, &gpio_test_provider_driver);
-	KUNIT_ASSERT_EQ(test, ret, 0);
-
-	ret = kunit_platform_driver_register(test, &gpio_swnode_consumer_driver);
-	KUNIT_ASSERT_EQ(test, ret, 0);
-
 	fwnode_init(primary, NULL);
 
 	pdevinfo = (struct platform_device_info){
@@ -211,6 +272,7 @@ static struct kunit_case gpio_swnode_lookup_tests[] = {
 static struct kunit_suite gpio_swnode_lookup_test_suite = {
 	.name = "gpio-swnode-lookup",
 	.test_cases = gpio_swnode_lookup_tests,
+	.init = gpio_swnode_register_drivers,
 };
 
 static BLOCKING_NOTIFIER_HEAD(gpio_unbind_notifier);
@@ -348,9 +410,191 @@ static struct kunit_suite gpio_unbind_with_consumers_test_suite = {
 	.test_cases = gpio_unbind_with_consumers_tests,
 };
 
+/*
+ * GPIO line hogs are described by child software nodes of the provider
+ * carrying the "gpio-hog" property. They are picked up automatically when the
+ * gpiochip is registered. Each hog below sits on a distinct line of the
+ * provider.
+ */
+#define GPIO_HOG_OUTPUT_HIGH_OFFSET	0
+#define GPIO_HOG_OUTPUT_LOW_OFFSET	1
+#define GPIO_HOG_INPUT_OFFSET		2
+
+static const u32 gpio_hog_output_high_gpios[] = {
+	GPIO_HOG_OUTPUT_HIGH_OFFSET, GPIO_ACTIVE_HIGH,
+};
+
+static const struct property_entry gpio_hog_output_high_properties[] = {
+	PROPERTY_ENTRY_U32_ARRAY("gpios", gpio_hog_output_high_gpios),
+	PROPERTY_ENTRY_STRING("line-name", "hog-output-high"),
+	PROPERTY_ENTRY_BOOL("output-high"),
+	PROPERTY_ENTRY_BOOL("gpio-hog"),
+	{ }
+};
+
+static const struct software_node gpio_hog_output_high_swnode =
+	SOFTWARE_NODE("hog-output-high", gpio_hog_output_high_properties,
+		      &gpio_test_provider_swnode);
+
+static const u32 gpio_hog_output_low_gpios[] = {
+	GPIO_HOG_OUTPUT_LOW_OFFSET, GPIO_ACTIVE_HIGH,
+};
+
+static const struct property_entry gpio_hog_output_low_properties[] = {
+	PROPERTY_ENTRY_U32_ARRAY("gpios", gpio_hog_output_low_gpios),
+	PROPERTY_ENTRY_STRING("line-name", "hog-output-low"),
+	PROPERTY_ENTRY_BOOL("output-low"),
+	PROPERTY_ENTRY_BOOL("gpio-hog"),
+	{ }
+};
+
+static const struct software_node gpio_hog_output_low_swnode =
+	SOFTWARE_NODE("hog-output-low", gpio_hog_output_low_properties,
+		      &gpio_test_provider_swnode);
+
+static const u32 gpio_hog_input_gpios[] = {
+	GPIO_HOG_INPUT_OFFSET, GPIO_ACTIVE_HIGH,
+};
+
+static const struct property_entry gpio_hog_input_properties[] = {
+	PROPERTY_ENTRY_U32_ARRAY("gpios", gpio_hog_input_gpios),
+	PROPERTY_ENTRY_STRING("line-name", "hog-input"),
+	PROPERTY_ENTRY_BOOL("input"),
+	PROPERTY_ENTRY_BOOL("gpio-hog"),
+	{ }
+};
+
+static const struct software_node gpio_hog_input_swnode =
+	SOFTWARE_NODE("hog-input", gpio_hog_input_properties,
+		      &gpio_test_provider_swnode);
+
+static const struct software_node *const gpio_hog_swnodes[] = {
+	&gpio_test_provider_swnode,
+	&gpio_hog_output_high_swnode,
+	&gpio_hog_output_low_swnode,
+	&gpio_hog_input_swnode,
+	NULL
+};
+
+/*
+ * Bring up the provider with a single hog child registered and verify both
+ * that the line was configured with the expected direction and that it is now
+ * exclusively owned (a consumer asking for the same line fails to bind).
+ *
+ * The provider node is referenced by the device through its fwnode rather than
+ * being handed to .swnode, so the device takes no software node reference of
+ * its own. Both the provider and the hog child are therefore test-managed and
+ * torn down (child first) once the test case completes.
+ */
+static void gpio_hog_assert(struct kunit *test, unsigned int offset,
+			    int expected_direction)
+{
+	struct gpio_swnode_consumer_pdata *pdata;
+	struct platform_device_info pdevinfo;
+	struct property_entry properties[2];
+	struct platform_device *pdev;
+	struct fwnode_handle *fwnode;
+	struct gpio_desc *desc;
+	bool bound = true;
+	int ret;
+
+	fwnode = software_node_fwnode(&gpio_test_provider_swnode);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fwnode);
+
+	pdevinfo = (struct platform_device_info){
+		.name = GPIO_TEST_PROVIDER,
+		.id = PLATFORM_DEVID_NONE,
+		.fwnode = fwnode,
+	};
+
+	pdev = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev);
+
+	wait_for_device_probe();
+
+	/* The hog must have configured the line with the expected direction. */
+	struct gpio_device *gdev __free(gpio_device_put) =
+		gpio_device_find_by_label(GPIO_CONSUMER_NAME);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, gdev);
+
+	desc = gpio_device_get_desc(gdev, offset);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, desc);
+
+	ret = gpiod_get_direction(desc);
+	KUNIT_ASSERT_EQ(test, ret, expected_direction);
+
+	/* A hogged line is owned exclusively, so a consumer must fail to bind. */
+	properties[0] = PROPERTY_ENTRY_GPIO("foo-gpios",
+					    &gpio_test_provider_swnode,
+					    offset, GPIO_ACTIVE_HIGH);
+	properties[1] = (struct property_entry){ };
+
+	pdevinfo = (struct platform_device_info){
+		.name = GPIO_SWNODE_TEST_CONSUMER,
+		.id = PLATFORM_DEVID_NONE,
+		.data = &gpio_swnode_pdata_template,
+		.size_data = sizeof(gpio_swnode_pdata_template),
+		.properties = properties,
+	};
+
+	pdev = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev);
+
+	wait_for_device_probe();
+	scoped_guard(device, &pdev->dev)
+		bound = device_is_bound(&pdev->dev);
+
+	KUNIT_ASSERT_FALSE(test, bound);
+
+	pdata = dev_get_platdata(&pdev->dev);
+	KUNIT_ASSERT_FALSE(test, pdata->gpio_ok);
+	KUNIT_ASSERT_EQ(test, pdata->errno, -EBUSY);
+}
+
+static void gpio_hog_output_high(struct kunit *test)
+{
+	gpio_hog_assert(test, GPIO_HOG_OUTPUT_HIGH_OFFSET, GPIO_LINE_DIRECTION_OUT);
+}
+
+static void gpio_hog_output_low(struct kunit *test)
+{
+	gpio_hog_assert(test, GPIO_HOG_OUTPUT_LOW_OFFSET, GPIO_LINE_DIRECTION_OUT);
+}
+
+static void gpio_hog_input(struct kunit *test)
+{
+	gpio_hog_assert(test, GPIO_HOG_INPUT_OFFSET, GPIO_LINE_DIRECTION_IN);
+}
+
+static int gpio_hog_suite_init(struct kunit_suite *suite)
+{
+	return software_node_register_node_group(gpio_hog_swnodes);
+}
+
+static void gpio_hog_suite_exit(struct kunit_suite *suite)
+{
+	software_node_unregister_node_group(gpio_hog_swnodes);
+}
+
+static struct kunit_case gpio_swnode_hog_tests[] = {
+	KUNIT_CASE(gpio_hog_output_high),
+	KUNIT_CASE(gpio_hog_output_low),
+	KUNIT_CASE(gpio_hog_input),
+	{ }
+};
+
+static struct kunit_suite gpio_swnode_hog_test_suite = {
+	.name = "gpio-swnode-hog",
+	.test_cases = gpio_swnode_hog_tests,
+	.suite_init = gpio_hog_suite_init,
+	.suite_exit = gpio_hog_suite_exit,
+	.init = gpio_swnode_register_drivers,
+};
+
 kunit_test_suites(
 	&gpio_swnode_lookup_test_suite,
 	&gpio_unbind_with_consumers_test_suite,
+	&gpio_swnode_hog_test_suite,
 );
 
 MODULE_DESCRIPTION("Test module for the GPIO subsystem");
diff --git a/drivers/gpio/gpiolib-swnode.c b/drivers/gpio/gpiolib-swnode.c
index 8d9591aa9304d..304994c5c7d02 100644
--- a/drivers/gpio/gpiolib-swnode.c
+++ b/drivers/gpio/gpiolib-swnode.c
@@ -26,7 +26,6 @@
 static struct gpio_device *swnode_get_gpio_device(struct fwnode_handle *fwnode)
 {
 	const struct software_node *gdev_node;
-	struct gpio_device *gdev;
 
 	gdev_node = to_software_node(fwnode);
 	if (!gdev_node)
@@ -41,27 +40,7 @@ static struct gpio_device *swnode_get_gpio_device(struct fwnode_handle *fwnode)
 		return ERR_PTR(-ENOENT);
 
 fwnode_lookup:
-	gdev = gpio_device_find_by_fwnode(fwnode);
-	if (!gdev && gdev_node && gdev_node->name)
-		/*
-		 * FIXME: We shouldn't need to compare the GPIO controller's
-		 * label against the software node that is supposedly attached
-		 * to it. However there are currently GPIO users that - knowing
-		 * the expected label of the GPIO chip whose pins they want to
-		 * control - set up dummy software nodes named after those GPIO
-		 * controllers, which aren't actually attached to them. In this
-		 * case gpio_device_find_by_fwnode() will fail as no device on
-		 * the GPIO bus is actually associated with the fwnode we're
-		 * looking for.
-		 *
-		 * As a fallback: continue checking the label if we have no
-		 * match. However, the situation described above is an abuse
-		 * of the software node API and should be phased out and the
-		 * following line - eventually removed.
-		 */
-		gdev = gpio_device_find_by_label(gdev_node->name);
-
-	return gdev ?: ERR_PTR(-EPROBE_DEFER);
+	return gpio_device_find_by_fwnode(fwnode) ?: ERR_PTR(-EPROBE_DEFER);
 }
 
 static int swnode_gpio_get_reference(const struct fwnode_handle *fwnode,
diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c
index e5fb60111151f..c433a095907fe 100644
--- a/drivers/gpio/gpiolib.c
+++ b/drivers/gpio/gpiolib.c
@@ -999,14 +999,17 @@ int gpiochip_add_hog(struct gpio_chip *gc, struct fwnode_handle *fwnode)
 	if (ret < 0)
 		return ret;
 
-	if (fwnode_property_present(fwnode, "input"))
+	if (fwnode_property_present(fwnode, "input")) {
 		dflags |= GPIOD_IN;
-	else if (fwnode_property_present(fwnode, "output-low"))
+	} else if (fwnode_property_present(fwnode, "output-low")) {
 		dflags |= GPIOD_OUT_LOW;
-	else if (fwnode_property_present(fwnode, "output-high"))
+	} else if (fwnode_property_present(fwnode, "output-high")) {
 		dflags |= GPIOD_OUT_HIGH;
-	else
-		return -EINVAL;
+	} else {
+		gpiochip_warn(gc, "%pfwP: no hogging state specified, bailing out\n",
+			      fwnode);
+		return 0;
+	}
 
 	fwnode_property_read_string(fwnode, "line-name", &name);
 
diff --git a/drivers/gpu/Makefile b/drivers/gpu/Makefile
index b4e5e338efa23..e372fc02139fe 100644
--- a/drivers/gpu/Makefile
+++ b/drivers/gpu/Makefile
@@ -7,4 +7,60 @@ obj-$(CONFIG_GPU_BUDDY)	+= buddy.o
 obj-y			+= host1x/ drm/ vga/ tests/
 obj-$(CONFIG_IMX_IPUV3_CORE)	+= ipu-v3/
 obj-$(CONFIG_TRACE_GPU_MEM)		+= trace/
-obj-$(CONFIG_NOVA_CORE)		+= nova-core/
+
+# nova-core and nova-drm are built from this Makefile so nova-drm's dependency
+# on nova-core can be expressed as a plain Make prerequisite rather than a
+# recursive sub-make. This is a temporary workaround until the Rust build
+# system supports cross-crate dependencies natively.
+
+obj-$(CONFIG_NOVA_CORE) += nova-core.o
+nova-core-y := nova-core/nova_core.o nova-core/nova_core_exports.o
+
+obj-$(CONFIG_DRM_NOVA) += nova-drm.o
+nova-drm-y := drm/nova/nova.o
+
+# Export Rust symbols from nova-core only if nova-drm actually references them.
+nova-core-export-deps := $(if $(CONFIG_DRM_NOVA),$(obj)/drm/nova/nova.o)
+
+rust_needed_exports = \
+	{ $(if $(strip $(2)),$(NM) -u $(2);,) echo "__DEFINED_RUST_SYMBOLS__"; \
+	  $(NM) -p --defined-only $(1); } | \
+	awk -v fmt='$(3)' ' \
+		/^__DEFINED_RUST_SYMBOLS__$$/ { defs = 1; next } \
+		!defs { if ($$NF ~ /^_R/) needed[$$NF] = 1; next } \
+		defs && $$2 ~ /(T|R|D|B)/ && $$3 ~ /^_R/ && \
+			$$3 !~ /_(init|cleanup)_module$$/ && \
+			$$3 !~ /__(pfx|cfi|odr_asan)/ && \
+			$$3 in needed { printf fmt, $$3 } \
+	'
+
+quiet_cmd_exports = EXPORTS $@
+      cmd_exports = \
+	$(call rust_needed_exports,$<,$(nova-core-export-deps),EXPORT_SYMBOL_RUST_GPL(%s);\n) > $@
+
+$(obj)/nova-core/exports_nova_core_generated.h: $(obj)/nova-core/nova_core.o $(nova-core-export-deps) FORCE
+	$(call if_changed,exports)
+
+targets += nova-core/exports_nova_core_generated.h
+
+$(obj)/nova-core/nova_core_exports.o: $(obj)/nova-core/exports_nova_core_generated.h
+CFLAGS_nova-core/nova_core_exports.o := -I $(objtree)/$(obj)/nova-core
+
+ifdef CONFIG_MODVERSIONS
+# The C export shim declares Rust symbols as `extern int`, so reuse its export
+# list but generate symbol CRCs from the Rust object instead of the shim's DWARF.
+$(obj)/nova-core/nova_core_exports.o: private cmd_gensymtypes_c = \
+	$(call getexportsymbols,\1) | \
+	$(objtree)/scripts/gendwarfksyms/gendwarfksyms \
+		$(if $(KBUILD_GENDWARFKSYMS_STABLE), --stable) \
+		$(if $(KBUILD_SYMTYPES), --symtypes $(@:.o=.symtypes),) \
+		$(obj)/nova-core/nova_core.o
+endif
+
+# Output nova-core's crate metadata for use by nova-drm at compile time.
+RUSTFLAGS_nova-core/nova_core.o += \
+	--emit=metadata=$(objtree)/$(obj)/nova-core/libnova_core.rmeta
+
+# Allow nova-drm to import nova-core's types.
+$(obj)/drm/nova/nova.o: $(obj)/nova-core/nova_core.o
+RUSTFLAGS_drm/nova/nova.o := -L $(objtree)/$(obj)/nova-core --extern nova_core
diff --git a/drivers/gpu/buddy.c b/drivers/gpu/buddy.c
index b12d3a2ac630d..3c73ae87f3c5b 100644
--- a/drivers/gpu/buddy.c
+++ b/drivers/gpu/buddy.c
@@ -193,6 +193,9 @@ static void mark_allocated(struct gpu_buddy *mm,
 	block->header &= ~GPU_BUDDY_HEADER_STATE;
 	block->header |= GPU_BUDDY_ALLOCATED;
 
+	mm->free_scoreboard[gpu_buddy_block_order(block)]--;
+	mm->used_scoreboard[gpu_buddy_block_order(block)]++;
+
 	rbtree_remove(mm, block);
 }
 
@@ -201,9 +204,14 @@ static void mark_free(struct gpu_buddy *mm,
 {
 	enum gpu_buddy_free_tree tree;
 
+	if (gpu_buddy_block_is_allocated(block))
+		mm->used_scoreboard[gpu_buddy_block_order(block)]--;
+
 	block->header &= ~GPU_BUDDY_HEADER_STATE;
 	block->header |= GPU_BUDDY_FREE;
 
+	mm->free_scoreboard[gpu_buddy_block_order(block)]++;
+
 	tree = get_block_tree(block);
 	rbtree_insert(mm, block, tree);
 }
@@ -214,6 +222,8 @@ static void mark_split(struct gpu_buddy *mm,
 	block->header &= ~GPU_BUDDY_HEADER_STATE;
 	block->header |= GPU_BUDDY_SPLIT;
 
+	mm->free_scoreboard[gpu_buddy_block_order(block)]--;
+
 	rbtree_remove(mm, block);
 }
 
@@ -271,9 +281,13 @@ static unsigned int __gpu_buddy_free(struct gpu_buddy *mm,
 		}
 
 		rbtree_remove(mm, buddy);
+		mm->free_scoreboard[gpu_buddy_block_order(buddy)]--;
 		if (force_merge && gpu_buddy_block_is_clear(buddy))
 			mm->clear_avail -= gpu_buddy_block_size(mm, buddy);
 
+		if (gpu_buddy_block_is_allocated(block))
+			mm->used_scoreboard[gpu_buddy_block_order(block)]--;
+
 		gpu_block_free(mm, block);
 		gpu_block_free(mm, buddy);
 
@@ -335,6 +349,7 @@ static int __force_merge(struct gpu_buddy *mm,
 					iter = rb_prev(iter);
 
 				rbtree_remove(mm, block);
+				mm->free_scoreboard[gpu_buddy_block_order(block)]--;
 				if (gpu_buddy_block_is_clear(block))
 					mm->clear_avail -= gpu_buddy_block_size(mm, block);
 
@@ -384,11 +399,23 @@ int gpu_buddy_init(struct gpu_buddy *mm, u64 size, u64 chunk_size)
 
 	BUG_ON(mm->max_order > GPU_BUDDY_MAX_ORDER);
 
+	mm->free_scoreboard = kcalloc(mm->max_order + 1,
+				      sizeof(*mm->free_scoreboard),
+				      GFP_KERNEL);
+	if (!mm->free_scoreboard)
+		return -ENOMEM;
+
+	mm->used_scoreboard = kcalloc(mm->max_order + 1,
+				      sizeof(*mm->used_scoreboard),
+				      GFP_KERNEL);
+	if (!mm->used_scoreboard)
+		goto out_free_free_scoreboard;
+
 	mm->free_trees = kmalloc_array(GPU_BUDDY_MAX_FREE_TREES,
 				       sizeof(*mm->free_trees),
 				       GFP_KERNEL);
 	if (!mm->free_trees)
-		return -ENOMEM;
+		goto out_free_used_scoreboard;
 
 	for_each_free_tree(i) {
 		mm->free_trees[i] = kmalloc_array(mm->max_order + 1,
@@ -450,6 +477,10 @@ int gpu_buddy_init(struct gpu_buddy *mm, u64 size, u64 chunk_size)
 	while (i--)
 		kfree(mm->free_trees[i]);
 	kfree(mm->free_trees);
+out_free_used_scoreboard:
+	kfree(mm->used_scoreboard);
+out_free_free_scoreboard:
+	kfree(mm->free_scoreboard);
 	return -ENOMEM;
 }
 EXPORT_SYMBOL(gpu_buddy_init);
@@ -484,10 +515,15 @@ void gpu_buddy_fini(struct gpu_buddy *mm)
 
 	gpu_buddy_assert(mm->avail == mm->size);
 
+	for (i = 0; i <= mm->max_order; ++i)
+		gpu_buddy_assert(!mm->used_scoreboard[i]);
+
 	for_each_free_tree(i)
 		kfree(mm->free_trees[i]);
 	kfree(mm->free_trees);
 	kfree(mm->roots);
+	kfree(mm->free_scoreboard);
+	kfree(mm->used_scoreboard);
 }
 EXPORT_SYMBOL(gpu_buddy_fini);
 
@@ -650,6 +686,20 @@ static bool block_incompatible(struct gpu_buddy_block *block, unsigned int flags
 	return needs_clear != gpu_buddy_block_is_clear(block);
 }
 
+static void __gpu_buddy_undo_splits(struct gpu_buddy *mm,
+				    struct gpu_buddy_block *block)
+{
+	struct gpu_buddy_block *buddy = __get_buddy(block);
+
+	if (buddy &&
+	    (gpu_buddy_block_is_free(block) &&
+	     gpu_buddy_block_is_free(buddy))) {
+		rbtree_remove(mm, block);
+		mm->free_scoreboard[gpu_buddy_block_order(block)]--;
+		__gpu_buddy_free(mm, block, false);
+	}
+}
+
 static struct gpu_buddy_block *
 __alloc_range_bias(struct gpu_buddy *mm,
 		   u64 start, u64 end,
@@ -659,7 +709,6 @@ __alloc_range_bias(struct gpu_buddy *mm,
 {
 	u64 req_size = mm->chunk_size << order;
 	struct gpu_buddy_block *block;
-	struct gpu_buddy_block *buddy;
 	LIST_HEAD(dfs);
 	int err;
 	int i;
@@ -734,11 +783,7 @@ __alloc_range_bias(struct gpu_buddy *mm,
 	 * bigger is better, so make sure we merge everything back before we
 	 * free the allocated blocks.
 	 */
-	buddy = __get_buddy(block);
-	if (buddy &&
-	    (gpu_buddy_block_is_free(block) &&
-	     gpu_buddy_block_is_free(buddy)))
-		__gpu_buddy_free(mm, block, false);
+	__gpu_buddy_undo_splits(mm, block);
 	return ERR_PTR(err);
 }
 
@@ -847,8 +892,7 @@ alloc_from_freetree(struct gpu_buddy *mm,
 	return block;
 
 err_undo:
-	if (tmp != order)
-		__gpu_buddy_free(mm, block, false);
+	__gpu_buddy_undo_splits(mm, block);
 	return ERR_PTR(err);
 }
 
@@ -912,7 +956,6 @@ gpu_buddy_offset_aligned_allocation(struct gpu_buddy *mm,
 {
 	struct gpu_buddy_block *block = NULL;
 	unsigned int order, tmp, alignment;
-	struct gpu_buddy_block *buddy;
 	enum gpu_buddy_free_tree tree;
 	unsigned long pages;
 	int err;
@@ -965,11 +1008,7 @@ gpu_buddy_offset_aligned_allocation(struct gpu_buddy *mm,
 	 * bigger is better, so make sure we merge everything back before we
 	 * free the allocated blocks.
 	 */
-	buddy = __get_buddy(block);
-	if (buddy &&
-	    (gpu_buddy_block_is_free(block) &&
-	     gpu_buddy_block_is_free(buddy)))
-		__gpu_buddy_free(mm, block, false);
+	__gpu_buddy_undo_splits(mm, block);
 	return ERR_PTR(err);
 }
 
@@ -980,7 +1019,6 @@ static int __alloc_range(struct gpu_buddy *mm,
 			 u64 *total_allocated_on_err)
 {
 	struct gpu_buddy_block *block;
-	struct gpu_buddy_block *buddy;
 	u64 total_allocated = 0;
 	LIST_HEAD(allocated);
 	u64 end;
@@ -1051,11 +1089,7 @@ static int __alloc_range(struct gpu_buddy *mm,
 	 * bigger is better, so make sure we merge everything back before we
 	 * free the allocated blocks.
 	 */
-	buddy = __get_buddy(block);
-	if (buddy &&
-	    (gpu_buddy_block_is_free(block) &&
-	     gpu_buddy_block_is_free(buddy)))
-		__gpu_buddy_free(mm, block, false);
+	__gpu_buddy_undo_splits(mm, block);
 
 err_free:
 	if (err == -ENOSPC && total_allocated_on_err) {
@@ -1515,27 +1549,18 @@ void gpu_buddy_print(struct gpu_buddy *mm)
 		mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20, mm->clear_avail >> 20);
 
 	for (order = mm->max_order; order >= 0; order--) {
-		struct gpu_buddy_block *block, *tmp;
-		struct rb_root *root;
-		u64 count = 0, free;
-		unsigned int tree;
-
-		for_each_free_tree(tree) {
-			root = &mm->free_trees[tree][order];
-
-			rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) {
-				BUG_ON(!gpu_buddy_block_is_free(block));
-				count++;
-			}
-		}
-
-		free = count * (mm->chunk_size << order);
-		if (free < SZ_1M)
-			pr_info("order-%2d free: %8llu KiB, blocks: %llu\n",
-				order, free >> 10, count);
+		u64 free_count = mm->free_scoreboard[order];
+		u64 used_count = mm->used_scoreboard[order];
+		u64 block_size = mm->chunk_size << order;
+		u64 free = free_count * block_size;
+		u64 used = used_count * block_size;
+
+		if (block_size < SZ_1M)
+			pr_info("order-%2d free: %8llu KiB, used: %8llu KiB, free_blocks: %llu, used_blocks: %llu\n",
+				order, free >> 10, used >> 10, free_count, used_count);
 		else
-			pr_info("order-%2d free: %8llu MiB, blocks: %llu\n",
-				order, free >> 20, count);
+			pr_info("order-%2d free: %8llu MiB, used: %8llu MiB, free_blocks: %llu, used_blocks: %llu\n",
+				order, free >> 20, used >> 20, free_count, used_count);
 	}
 }
 EXPORT_SYMBOL(gpu_buddy_print);
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index e97faabcd7830..e635fcffd3790 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -186,7 +186,7 @@ obj-$(CONFIG_DRM_VMWGFX)+= vmwgfx/
 obj-$(CONFIG_DRM_VGEM)	+= vgem/
 obj-$(CONFIG_DRM_VKMS)	+= vkms/
 obj-$(CONFIG_DRM_NOUVEAU) +=nouveau/
-obj-$(CONFIG_DRM_NOVA) += nova/
+# nova-drm is built from drivers/gpu/Makefile together with nova-core.
 obj-$(CONFIG_DRM_EXYNOS) +=exynos/
 obj-$(CONFIG_DRM_ROCKCHIP) +=rockchip/
 obj-$(CONFIG_DRM_GMA500) += gma500/
diff --git a/drivers/gpu/drm/adp/adp-mipi.c b/drivers/gpu/drm/adp/adp-mipi.c
index cba7d32150a98..232bb95394237 100644
--- a/drivers/gpu/drm/adp/adp-mipi.c
+++ b/drivers/gpu/drm/adp/adp-mipi.c
@@ -5,6 +5,7 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 
+#include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_mipi_dsi.h>
 
@@ -222,6 +223,9 @@ static int adp_dsi_bridge_attach(struct drm_bridge *bridge,
 }
 
 static const struct drm_bridge_funcs adp_dsi_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach	= adp_dsi_bridge_attach,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index ba80542ead9d9..105b9dcc19a2b 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -70,8 +70,9 @@ amdgpu-y += amdgpu_device.o amdgpu_reg_access.o amdgpu_doorbell_mgr.o amdgpu_kms
 	amdgpu_umc.o smu_v11_0_i2c.o amdgpu_fru_eeprom.o amdgpu_rap.o \
 	amdgpu_fw_attestation.o amdgpu_securedisplay.o \
 	amdgpu_eeprom.o amdgpu_mca.o amdgpu_psp_ta.o amdgpu_lsdma.o amdgpu_lockdep.o \
-	amdgpu_ring_mux.o amdgpu_xcp.o amdgpu_seq64.o amdgpu_aca.o amdgpu_dev_coredump.o \
-	amdgpu_cper.o amdgpu_userq_fence.o amdgpu_eviction_fence.o amdgpu_ip.o
+	amdgpu_ring_mux.o amdgpu_xcp.o amdgpu_seq64.o amdgpu_dev_coredump.o \
+	amdgpu_cper.o amdgpu_userq_fence.o amdgpu_eviction_fence.o amdgpu_ip.o \
+	amdgpu_wb.o
 
 amdgpu-$(CONFIG_PROC_FS) += amdgpu_fdinfo.o
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 7b09410d6d8f0..f5d65bd0ac253 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -44,6 +44,7 @@
 #include <linux/hashtable.h>
 #include <linux/dma-fence.h>
 #include <linux/pci.h>
+#include <linux/xarray.h>
 
 #include <drm/ttm/ttm_bo.h>
 #include <drm/ttm/ttm_placement.h>
@@ -95,7 +96,6 @@
 #include "amdgpu_doorbell.h"
 #include "amdgpu_amdkfd.h"
 #include "amdgpu_discovery.h"
-#include "amdgpu_mes.h"
 #include "amdgpu_umc.h"
 #include "amdgpu_mmhub.h"
 #include "amdgpu_gfxhub.h"
@@ -103,7 +103,6 @@
 #include "amdgpu_smuio.h"
 #include "amdgpu_fdinfo.h"
 #include "amdgpu_mca.h"
-#include "amdgpu_aca.h"
 #include "amdgpu_ras.h"
 #include "amdgpu_lockdep.h"
 #include "amdgpu_cper.h"
@@ -112,7 +111,11 @@
 #include "amdgpu_reg_state.h"
 #include "amdgpu_userq.h"
 #include "amdgpu_eviction_fence.h"
+#include "amdgpu_wb.h"
 #include "amdgpu_ip.h"
+#include "amdgpu_mes.h"
+#include "amdgpu_sa.h"
+#include "amdgpu_acpi.h"
 #if defined(CONFIG_DRM_AMD_ISP)
 #include "amdgpu_isp.h"
 #endif
@@ -134,13 +137,6 @@ struct amdgpu_mgpu_info {
 	uint32_t			num_apu;
 };
 
-enum amdgpu_ss {
-	AMDGPU_SS_DRV_LOAD,
-	AMDGPU_SS_DEV_D0,
-	AMDGPU_SS_DEV_D3,
-	AMDGPU_SS_DRV_UNLOAD
-};
-
 struct amdgpu_hwip_reg_entry {
 	u32		hwip;
 	u32		inst;
@@ -272,7 +268,6 @@ extern int amdgpu_ptl;
 
 extern uint amdgpu_hdmi_hpd_debounce_delay_ms;
 
-#define AMDGPU_VM_MAX_NUM_CTX			4096
 #define AMDGPU_SG_THRESHOLD			(256*1024*1024)
 #define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS	        3000
 #define AMDGPU_MAX_USEC_TIMEOUT			100000	/* 100 ms */
@@ -305,9 +300,10 @@ extern uint amdgpu_hdmi_hpd_debounce_delay_ms;
 
 /* reset mask */
 #define AMDGPU_RESET_TYPE_FULL (1 << 0) /* full adapter reset, mode1/mode2/BACO/etc. */
-#define AMDGPU_RESET_TYPE_SOFT_RESET (1 << 1) /* IP level soft reset */
+#define AMDGPU_RESET_TYPE_SOFT_RECOVERY (1 << 1) /* soft recovery, eg. kill shaders */
 #define AMDGPU_RESET_TYPE_PER_QUEUE (1 << 2) /* per queue */
 #define AMDGPU_RESET_TYPE_PER_PIPE (1 << 3) /* per pipe */
+#define AMDGPU_RESET_TYPE_IP_BLOCK_SOFT_RESET (1 << 4) /* soft-resets an IP block */
 
 /* max cursor sizes (in pixels) */
 #define CIK_CURSOR_WIDTH 128
@@ -330,6 +326,7 @@ struct amdgpu_hive_info;
 struct amdgpu_reset_context;
 struct amdgpu_reset_control;
 struct amdgpu_coredump_info;
+struct amdgpu_video_codecs;
 
 enum amdgpu_cp_irq {
 	AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP = 0,
@@ -387,37 +384,6 @@ struct amdgpu_clock {
 	uint32_t max_pixel_clock;
 };
 
-/* sub-allocation manager, it has to be protected by another lock.
- * By conception this is an helper for other part of the driver
- * like the indirect buffer or semaphore, which both have their
- * locking.
- *
- * Principe is simple, we keep a list of sub allocation in offset
- * order (first entry has offset == 0, last entry has the highest
- * offset).
- *
- * When allocating new object we first check if there is room at
- * the end total_size - (last_object_offset + last_object_size) >=
- * alloc_size. If so we allocate new object there.
- *
- * When there is not enough room at the end, we start waiting for
- * each sub object until we reach object_offset+object_size >=
- * alloc_size, this object then become the sub object we return.
- *
- * Alignment can't be bigger than page size.
- *
- * Hole are not considered for allocation to keep things simple.
- * Assumption is that there won't be hole (all object on same
- * alignment).
- */
-
-struct amdgpu_sa_manager {
-	struct drm_suballoc_manager	base;
-	struct amdgpu_bo		*bo;
-	uint64_t			gpu_addr;
-	void				*cpu_ptr;
-};
-
 /*
  * IRQS.
  */
@@ -446,8 +412,7 @@ struct amdgpu_fpriv {
 	struct amdgpu_bo_va	*prt_va;
 	struct amdgpu_bo_va	*csa_va;
 	struct amdgpu_bo_va	*seq64_va;
-	struct mutex		bo_list_lock;
-	struct idr		bo_list_handles;
+	struct xarray		bo_list_handles;
 	struct amdgpu_ctx_mgr	ctx_mgr;
 	struct amdgpu_userq_mgr	userq_mgr;
 
@@ -460,106 +425,11 @@ struct amdgpu_fpriv {
 
 int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv);
 
-/*
- * Writeback
- */
-#define AMDGPU_MAX_WB 1024	/* Reserve at most 1024 WB slots for amdgpu-owned rings. */
-
-/**
- * struct amdgpu_wb - This struct is used for small GPU memory allocation.
- *
- * This struct is used to allocate a small amount of GPU memory that can be
- * used to shadow certain states into the memory. This is especially useful for
- * providing easy CPU access to some states without requiring register access
- * (e.g., if some block is power gated, reading register may be problematic).
- *
- * Note: the term writeback was initially used because many of the amdgpu
- * components had some level of writeback memory, and this struct initially
- * described those components.
- */
-struct amdgpu_wb {
-
-	/**
-	 * @wb_obj:
-	 *
-	 * Buffer Object used for the writeback memory.
-	 */
-	struct amdgpu_bo	*wb_obj;
-
-	/**
-	 * @wb:
-	 *
-	 * Pointer to the first writeback slot. In terms of CPU address
-	 * this value can be accessed directly by using the offset as an index.
-	 * For the GPU address, it is necessary to use gpu_addr and the offset.
-	 */
-	uint32_t		*wb;
-
-	/**
-	 * @gpu_addr:
-	 *
-	 * Writeback base address in the GPU.
-	 */
-	uint64_t		gpu_addr;
-
-	/**
-	 * @num_wb:
-	 *
-	 * Number of writeback slots reserved for amdgpu.
-	 */
-	u32			num_wb;
-
-	/**
-	 * @used:
-	 *
-	 * Track the writeback slot already used.
-	 */
-	unsigned long		used[DIV_ROUND_UP(AMDGPU_MAX_WB, BITS_PER_LONG)];
-
-	/**
-	 * @lock:
-	 *
-	 * Protects read and write of the used field array.
-	 */
-	spinlock_t		lock;
-};
-
-int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb);
-void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb);
-
 /*
  * Benchmarking
  */
 int amdgpu_benchmark(struct amdgpu_device *adev, int test_number);
 
-/*
- * ASIC specific register table accessible by UMD
- */
-struct amdgpu_allowed_register_entry {
-	uint32_t reg_offset;
-	bool grbm_indexed;
-};
-
-struct amdgpu_video_codec_info {
-	u32 codec_type;
-	u32 max_width;
-	u32 max_height;
-	u32 max_pixels_per_frame;
-	u32 max_level;
-};
-
-#define codec_info_build(type, width, height, level) \
-			 .codec_type = type,\
-			 .max_width = width,\
-			 .max_height = height,\
-			 .max_pixels_per_frame = height * width,\
-			 .max_level = level,
-
-struct amdgpu_video_codecs {
-	const u32 codec_count;
-	const struct amdgpu_video_codec_info *codec_array;
-};
-
 /*
  * ASIC specific functions.
  */
@@ -587,8 +457,6 @@ struct amdgpu_asic_funcs {
 	/* invalidate hdp read cache */
 	void (*invalidate_hdp)(struct amdgpu_device *adev,
 			       struct amdgpu_ring *ring);
-	/* check if the asic needs a full reset of if soft reset will work */
-	bool (*need_full_reset)(struct amdgpu_device *adev);
 	/* initialize doorbell layout for specific asic*/
 	void (*init_doorbell_index)(struct amdgpu_device *adev);
 	/* PCIe bandwidth usage */
@@ -665,38 +533,6 @@ struct amdgpu_uid {
 	struct amdgpu_device *adev;
 };
 
-#define MAX_UMA_OPTION_NAME	28
-#define MAX_UMA_OPTION_ENTRIES	19
-
-#define AMDGPU_UMA_FLAG_AUTO	BIT(1)
-#define AMDGPU_UMA_FLAG_CUSTOM	BIT(0)
-
-/**
- * struct amdgpu_uma_carveout_option - single UMA carveout option
- * @name: Name of the carveout option
- * @memory_carved_mb: Amount of memory carved in MB
- * @flags: ATCS flags supported by this option
- */
-struct amdgpu_uma_carveout_option {
-	char name[MAX_UMA_OPTION_NAME];
-	uint32_t memory_carved_mb;
-	uint8_t flags;
-};
-
-/**
- * struct amdgpu_uma_carveout_info - table of available UMA carveout options
- * @num_entries: Number of available options
- * @uma_option_index: The index of the option currently applied
- * @update_lock: Lock to serialize changes to the option
- * @entries: The array of carveout options
- */
-struct amdgpu_uma_carveout_info {
-	uint8_t num_entries;
-	uint8_t uma_option_index;
-	struct mutex update_lock;
-	struct amdgpu_uma_carveout_option entries[MAX_UMA_OPTION_ENTRIES];
-};
-
 struct amd_powerplay {
 	void *pp_handle;
 	const struct amd_pm_funcs *pp_funcs;
@@ -741,44 +577,6 @@ struct amd_powerplay {
 					  (rid == 0x01) || \
 					  (rid == 0x10))))
 
-enum amdgpu_mqd_update_flag {
-       AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE = 1,
-       AMDGPU_UPDATE_FLAG_DBG_WA_DISABLE = 2,
-       AMDGPU_UPDATE_FLAG_IS_GWS = 4, /* quirk for gfx9 IP */
-};
-
-struct amdgpu_mqd_prop {
-	uint64_t mqd_gpu_addr;
-	uint64_t hqd_base_gpu_addr;
-	uint64_t rptr_gpu_addr;
-	uint64_t wptr_gpu_addr;
-	uint32_t queue_size;
-	bool use_doorbell;
-	uint32_t doorbell_index;
-	uint64_t eop_gpu_addr;
-	uint32_t hqd_pipe_priority;
-	uint32_t hqd_queue_priority;
-	uint32_t mqd_stride_size;
-	bool allow_tunneling;
-	bool hqd_active;
-	uint64_t shadow_addr;
-	uint64_t gds_bkup_addr;
-	uint64_t csa_addr;
-	uint64_t fence_address;
-	bool tmz_queue;
-	bool kernel_queue;
-	uint32_t *cu_mask;
-	uint32_t cu_mask_count;
-	uint32_t cu_flags;
-	bool is_user_cu_masked;
-};
-
-struct amdgpu_mqd {
-	unsigned mqd_size;
-	int (*init_mqd)(struct amdgpu_device *adev, void *mqd,
-			struct amdgpu_mqd_prop *p);
-};
-
 struct amdgpu_pcie_reset_ctx {
 	bool in_link_reset;
 	bool occurs_dpc;
@@ -851,6 +649,7 @@ struct amdgpu_device {
 	struct dev_pm_domain		vga_pm_domain;
 	bool				have_disp_power_ref;
 	bool                            have_atomics_support;
+	bool				is_sw_smu;
 
 	/* BIOS */
 	bool				is_atom_fw;
@@ -1022,9 +821,6 @@ struct amdgpu_device {
 	/* MCA */
 	struct amdgpu_mca               mca;
 
-	/* ACA */
-	struct amdgpu_aca		aca;
-
 	/* CPER */
 	struct amdgpu_cper		cper;
 
@@ -1130,12 +926,13 @@ struct amdgpu_device {
 	bool                            debug_largebar;
 	bool                            debug_disable_soft_recovery;
 	bool                            debug_use_vram_fw_buf;
-	bool                            debug_enable_ras_aca;
 	bool                            debug_exp_resets;
 	bool                            debug_disable_gpu_ring_reset;
 	bool                            debug_vm_userptr;
 	bool                            debug_disable_ce_logs;
 	bool                            debug_enable_ce_cs;
+	bool                            debug_hibernation_thaw_resume_gpu;
+	bool                            debug_disable_ip_block_soft_reset;
 
 	/* Protection for the following isolation structure */
 	struct mutex                    enforce_isolation_mutex;
@@ -1165,14 +962,6 @@ struct amdgpu_device {
 	struct amdgpu_kfd_dev		kfd;
 };
 
-/*
- * MES FW uses address(mqd_addr + sizeof(struct mqd) + 3*sizeof(uint32_t))
- * as fence address and writes a 32 bit fence value to this address.
- * Driver needs to allocate at least 4 DWs extra memory in addition to
- * sizeof(struct mqd). Add 8 DWs and align to AMDGPU_GPU_PAGE_SIZE for safety.
- */
-#define AMDGPU_MQD_SIZE_ALIGN(mqd_size) AMDGPU_GPU_PAGE_ALIGN(((mqd_size) + 32))
-
 static inline uint32_t amdgpu_ip_version(const struct amdgpu_device *adev,
 					 uint8_t ip, uint8_t inst)
 {
@@ -1356,7 +1145,6 @@ int emu_soc_asic_init(struct amdgpu_device *adev);
 #define amdgpu_asic_read_bios_from_rom(adev, b, l) (adev)->asic_funcs->read_bios_from_rom((adev), (b), (l))
 #define amdgpu_asic_read_register(adev, se, sh, offset, v)((adev)->asic_funcs->read_register((adev), (se), (sh), (offset), (v)))
 #define amdgpu_asic_get_config_memsize(adev) (adev)->asic_funcs->get_config_memsize((adev))
-#define amdgpu_asic_need_full_reset(adev) (adev)->asic_funcs->need_full_reset((adev))
 #define amdgpu_asic_init_doorbell_index(adev) (adev)->asic_funcs->init_doorbell_index((adev))
 #define amdgpu_asic_get_pcie_usage(adev, cnt0, cnt1) ((adev)->asic_funcs->get_pcie_usage((adev), (cnt0), (cnt1)))
 #define amdgpu_asic_need_reset_on_init(adev) (adev)->asic_funcs->need_reset_on_init((adev))
@@ -1468,6 +1256,8 @@ int amdgpu_enable_vblank_kms(struct drm_crtc *crtc);
 void amdgpu_disable_vblank_kms(struct drm_crtc *crtc);
 int amdgpu_info_ioctl(struct drm_device *dev, void *data,
 		      struct drm_file *filp);
+int amdgpu_proc_options_ioctl(struct drm_device *dev, void *data,
+			      struct drm_file *filp);
 
 /*
  * functions used by amdgpu_encoder.c
@@ -1488,88 +1278,6 @@ struct amdgpu_afmt_acr {
 
 struct amdgpu_afmt_acr amdgpu_afmt_acr(uint32_t clock);
 
-/* amdgpu_acpi.c */
-
-struct amdgpu_numa_info {
-	uint64_t size;
-	int pxm;
-	int nid;
-};
-
-/* ATCS Device/Driver State */
-#define AMDGPU_ATCS_PSC_DEV_STATE_D0		0
-#define AMDGPU_ATCS_PSC_DEV_STATE_D3_HOT	3
-#define AMDGPU_ATCS_PSC_DRV_STATE_OPR		0
-#define AMDGPU_ATCS_PSC_DRV_STATE_NOT_OPR	1
-
-#if defined(CONFIG_ACPI)
-int amdgpu_acpi_init(struct amdgpu_device *adev);
-void amdgpu_acpi_fini(struct amdgpu_device *adev);
-bool amdgpu_acpi_is_pcie_performance_request_supported(struct amdgpu_device *adev);
-bool amdgpu_acpi_is_power_shift_control_supported(void);
-bool amdgpu_acpi_is_set_uma_allocation_size_supported(void);
-int amdgpu_acpi_pcie_performance_request(struct amdgpu_device *adev,
-						u8 perf_req, bool advertise);
-int amdgpu_acpi_power_shift_control(struct amdgpu_device *adev,
-				    u8 dev_state, bool drv_state);
-int amdgpu_acpi_smart_shift_update(struct amdgpu_device *adev,
-				   enum amdgpu_ss ss_state);
-int amdgpu_acpi_set_uma_allocation_size(struct amdgpu_device *adev, u8 index, u8 type);
-int amdgpu_acpi_pcie_notify_device_ready(struct amdgpu_device *adev);
-int amdgpu_acpi_get_tmr_info(struct amdgpu_device *adev, u64 *tmr_offset,
-			     u64 *tmr_size);
-int amdgpu_acpi_get_mem_info(struct amdgpu_device *adev, int xcc_id,
-			     struct amdgpu_numa_info *numa_info);
-
-void amdgpu_acpi_get_backlight_caps(struct amdgpu_dm_backlight_caps *caps);
-bool amdgpu_acpi_should_gpu_reset(struct amdgpu_device *adev);
-void amdgpu_acpi_detect(void);
-void amdgpu_acpi_release(void);
-#else
-static inline int amdgpu_acpi_init(struct amdgpu_device *adev) { return 0; }
-static inline int amdgpu_acpi_get_tmr_info(struct amdgpu_device *adev,
-					   u64 *tmr_offset, u64 *tmr_size)
-{
-	return -EINVAL;
-}
-static inline int amdgpu_acpi_get_mem_info(struct amdgpu_device *adev,
-					   int xcc_id,
-					   struct amdgpu_numa_info *numa_info)
-{
-	return -EINVAL;
-}
-static inline void amdgpu_acpi_fini(struct amdgpu_device *adev) { }
-static inline bool amdgpu_acpi_should_gpu_reset(struct amdgpu_device *adev) { return false; }
-static inline void amdgpu_acpi_detect(void) { }
-static inline void amdgpu_acpi_release(void) { }
-static inline bool amdgpu_acpi_is_power_shift_control_supported(void) { return false; }
-static inline bool amdgpu_acpi_is_set_uma_allocation_size_supported(void) { return false; }
-static inline int amdgpu_acpi_power_shift_control(struct amdgpu_device *adev,
-						  u8 dev_state, bool drv_state) { return 0; }
-static inline int amdgpu_acpi_smart_shift_update(struct amdgpu_device *adev,
-						 enum amdgpu_ss ss_state)
-{
-	return 0;
-}
-static inline int amdgpu_acpi_set_uma_allocation_size(struct amdgpu_device *adev, u8 index, u8 type)
-{
-	return -EINVAL;
-}
-static inline void amdgpu_acpi_get_backlight_caps(struct amdgpu_dm_backlight_caps *caps) { }
-#endif
-
-#if defined(CONFIG_ACPI) && defined(CONFIG_SUSPEND)
-bool amdgpu_acpi_is_s3_active(struct amdgpu_device *adev);
-bool amdgpu_acpi_is_s0ix_active(struct amdgpu_device *adev);
-#else
-static inline bool amdgpu_acpi_is_s0ix_active(struct amdgpu_device *adev) { return false; }
-static inline bool amdgpu_acpi_is_s3_active(struct amdgpu_device *adev) { return false; }
-#endif
-
-#if defined(CONFIG_DRM_AMD_ISP)
-int amdgpu_acpi_get_isp4_dev(struct acpi_device **dev);
-#endif
-
 void amdgpu_register_gpu_instance(struct amdgpu_device *adev);
 void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c
deleted file mode 100644
index db7858fe0c3d4..0000000000000
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.c
+++ /dev/null
@@ -1,985 +0,0 @@
-/*
- * Copyright 2023 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include <linux/list.h>
-#include "amdgpu.h"
-#include "amdgpu_aca.h"
-#include "amdgpu_ras.h"
-
-#define ACA_BANK_HWID(type, hwid, mcatype) [ACA_HWIP_TYPE_##type] = {hwid, mcatype}
-
-typedef int bank_handler_t(struct aca_handle *handle, struct aca_bank *bank, enum aca_smu_type type, void *data);
-
-static struct aca_hwip aca_hwid_mcatypes[ACA_HWIP_TYPE_COUNT] = {
-	ACA_BANK_HWID(SMU,	0x01,	0x01),
-	ACA_BANK_HWID(PCS_XGMI, 0x50,	0x00),
-	ACA_BANK_HWID(UMC,	0x96,	0x00),
-};
-
-static void aca_banks_init(struct aca_banks *banks)
-{
-	if (!banks)
-		return;
-
-	memset(banks, 0, sizeof(*banks));
-	INIT_LIST_HEAD(&banks->list);
-}
-
-static int aca_banks_add_bank(struct aca_banks *banks, struct aca_bank *bank)
-{
-	struct aca_bank_node *node;
-
-	if (!bank)
-		return -EINVAL;
-
-	node = kvzalloc_obj(*node);
-	if (!node)
-		return -ENOMEM;
-
-	memcpy(&node->bank, bank, sizeof(*bank));
-
-	INIT_LIST_HEAD(&node->node);
-	list_add_tail(&node->node, &banks->list);
-
-	banks->nr_banks++;
-
-	return 0;
-}
-
-static void aca_banks_release(struct aca_banks *banks)
-{
-	struct aca_bank_node *node, *tmp;
-
-	if (list_empty(&banks->list))
-		return;
-
-	list_for_each_entry_safe(node, tmp, &banks->list, node) {
-		list_del(&node->node);
-		kvfree(node);
-		banks->nr_banks--;
-	}
-}
-
-static int aca_smu_get_valid_aca_count(struct amdgpu_device *adev, enum aca_smu_type type, u32 *count)
-{
-	struct amdgpu_aca *aca = &adev->aca;
-	const struct aca_smu_funcs *smu_funcs = aca->smu_funcs;
-
-	if (!count)
-		return -EINVAL;
-
-	if (!smu_funcs || !smu_funcs->get_valid_aca_count)
-		return -EOPNOTSUPP;
-
-	return smu_funcs->get_valid_aca_count(adev, type, count);
-}
-
-static struct aca_regs_dump {
-	const char *name;
-	int reg_idx;
-} aca_regs[] = {
-	{"CONTROL",		ACA_REG_IDX_CTL},
-	{"STATUS",		ACA_REG_IDX_STATUS},
-	{"ADDR",		ACA_REG_IDX_ADDR},
-	{"MISC",		ACA_REG_IDX_MISC0},
-	{"CONFIG",		ACA_REG_IDX_CONFIG},
-	{"IPID",		ACA_REG_IDX_IPID},
-	{"SYND",		ACA_REG_IDX_SYND},
-	{"DESTAT",		ACA_REG_IDX_DESTAT},
-	{"DEADDR",		ACA_REG_IDX_DEADDR},
-	{"CONTROL_MASK",	ACA_REG_IDX_CTL_MASK},
-};
-
-static void aca_smu_bank_dump(struct amdgpu_device *adev, int idx, int total, struct aca_bank *bank,
-			      struct ras_query_context *qctx)
-{
-	u64 event_id = qctx ? qctx->evid.event_id : RAS_EVENT_INVALID_ID;
-	int i;
-
-	if (adev->debug_disable_ce_logs &&
-	    bank->smu_err_type == ACA_SMU_TYPE_CE &&
-	    !ACA_BANK_ERR_IS_DEFFERED(bank))
-		return;
-
-	RAS_EVENT_LOG(adev, event_id, HW_ERR "Accelerator Check Architecture events logged\n");
-	/* plus 1 for output format, e.g: ACA[08/08]: xxxx */
-	for (i = 0; i < ARRAY_SIZE(aca_regs); i++)
-		RAS_EVENT_LOG(adev, event_id, HW_ERR "ACA[%02d/%02d].%s=0x%016llx\n",
-			      idx + 1, total, aca_regs[i].name, bank->regs[aca_regs[i].reg_idx]);
-
-	if (ACA_REG__STATUS__SCRUB(bank->regs[ACA_REG_IDX_STATUS]))
-		RAS_EVENT_LOG(adev, event_id, HW_ERR "hardware error logged by the scrubber\n");
-}
-
-static bool aca_bank_hwip_is_matched(struct aca_bank *bank, enum aca_hwip_type type)
-{
-
-	struct aca_hwip *hwip;
-	int hwid, mcatype;
-	u64 ipid;
-
-	if (!bank || type == ACA_HWIP_TYPE_UNKNOW)
-		return false;
-
-	hwip = &aca_hwid_mcatypes[type];
-	if (!hwip->hwid)
-		return false;
-
-	ipid = bank->regs[ACA_REG_IDX_IPID];
-	hwid = ACA_REG__IPID__HARDWAREID(ipid);
-	mcatype = ACA_REG__IPID__MCATYPE(ipid);
-
-	return hwip->hwid == hwid && hwip->mcatype == mcatype;
-}
-
-static int aca_smu_get_valid_aca_banks(struct amdgpu_device *adev, enum aca_smu_type type,
-				       int start, int count,
-				       struct aca_banks *banks, struct ras_query_context *qctx)
-{
-	struct amdgpu_aca *aca = &adev->aca;
-	const struct aca_smu_funcs *smu_funcs = aca->smu_funcs;
-	struct aca_bank bank;
-	int i, max_count, ret;
-
-	if (!count)
-		return 0;
-
-	if (!smu_funcs || !smu_funcs->get_valid_aca_bank)
-		return -EOPNOTSUPP;
-
-	switch (type) {
-	case ACA_SMU_TYPE_UE:
-		max_count = smu_funcs->max_ue_bank_count;
-		break;
-	case ACA_SMU_TYPE_CE:
-		max_count = smu_funcs->max_ce_bank_count;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	if (start + count > max_count)
-		return -EINVAL;
-
-	count = min_t(int, count, max_count);
-	for (i = 0; i < count; i++) {
-		memset(&bank, 0, sizeof(bank));
-		ret = smu_funcs->get_valid_aca_bank(adev, type, start + i, &bank);
-		if (ret)
-			return ret;
-
-		bank.smu_err_type = type;
-
-		/*
-		 * Poison being consumed when injecting a UE while running background workloads,
-		 * which are unexpected.
-		 */
-		if (type == ACA_SMU_TYPE_UE &&
-		    ACA_REG__STATUS__POISON(bank.regs[ACA_REG_IDX_STATUS]) &&
-		    !aca_bank_hwip_is_matched(&bank, ACA_HWIP_TYPE_UMC))
-			continue;
-
-		aca_smu_bank_dump(adev, i, count, &bank, qctx);
-
-		ret = aca_banks_add_bank(banks, &bank);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static bool aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, enum aca_smu_type type)
-{
-	const struct aca_bank_ops *bank_ops = handle->bank_ops;
-
-	/* Parse all deferred errors with UMC aca handle */
-	if (ACA_BANK_ERR_IS_DEFFERED(bank))
-		return handle->hwip == ACA_HWIP_TYPE_UMC;
-
-	if (!aca_bank_hwip_is_matched(bank, handle->hwip))
-		return false;
-
-	if (!bank_ops->aca_bank_is_valid)
-		return true;
-
-	return bank_ops->aca_bank_is_valid(handle, bank, type, handle->data);
-}
-
-static struct aca_bank_error *new_bank_error(struct aca_error *aerr, struct aca_bank_info *info)
-{
-	struct aca_bank_error *bank_error;
-
-	bank_error = kvzalloc_obj(*bank_error);
-	if (!bank_error)
-		return NULL;
-
-	INIT_LIST_HEAD(&bank_error->node);
-	memcpy(&bank_error->info, info, sizeof(*info));
-
-	mutex_lock(&aerr->lock);
-	list_add_tail(&bank_error->node, &aerr->list);
-	aerr->nr_errors++;
-	mutex_unlock(&aerr->lock);
-
-	return bank_error;
-}
-
-static struct aca_bank_error *find_bank_error(struct aca_error *aerr, struct aca_bank_info *info)
-{
-	struct aca_bank_error *bank_error = NULL;
-	struct aca_bank_info *tmp_info;
-	bool found = false;
-
-	mutex_lock(&aerr->lock);
-	list_for_each_entry(bank_error, &aerr->list, node) {
-		tmp_info = &bank_error->info;
-		if (tmp_info->socket_id == info->socket_id &&
-		    tmp_info->die_id == info->die_id) {
-			found = true;
-			goto out_unlock;
-		}
-	}
-
-out_unlock:
-	mutex_unlock(&aerr->lock);
-
-	return found ? bank_error : NULL;
-}
-
-static void aca_bank_error_remove(struct aca_error *aerr, struct aca_bank_error *bank_error)
-{
-	if (!aerr || !bank_error)
-		return;
-
-	list_del(&bank_error->node);
-	aerr->nr_errors--;
-
-	kvfree(bank_error);
-}
-
-static struct aca_bank_error *get_bank_error(struct aca_error *aerr, struct aca_bank_info *info)
-{
-	struct aca_bank_error *bank_error;
-
-	if (!aerr || !info)
-		return NULL;
-
-	bank_error = find_bank_error(aerr, info);
-	if (bank_error)
-		return bank_error;
-
-	return new_bank_error(aerr, info);
-}
-
-int aca_error_cache_log_bank_error(struct aca_handle *handle, struct aca_bank_info *info,
-				   enum aca_error_type type, u64 count)
-{
-	struct aca_error_cache *error_cache = &handle->error_cache;
-	struct aca_bank_error *bank_error;
-	struct aca_error *aerr;
-
-	if (!handle || !info || type >= ACA_ERROR_TYPE_COUNT)
-		return -EINVAL;
-
-	if (!count)
-		return 0;
-
-	aerr = &error_cache->errors[type];
-	bank_error = get_bank_error(aerr, info);
-	if (!bank_error)
-		return -ENOMEM;
-
-	bank_error->count += count;
-
-	return 0;
-}
-
-static int aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank, enum aca_smu_type type)
-{
-	const struct aca_bank_ops *bank_ops = handle->bank_ops;
-
-	if (!bank)
-		return -EINVAL;
-
-	if (!bank_ops->aca_bank_parser)
-		return -EOPNOTSUPP;
-
-	return bank_ops->aca_bank_parser(handle, bank, type,
-					 handle->data);
-}
-
-static int handler_aca_log_bank_error(struct aca_handle *handle, struct aca_bank *bank,
-				      enum aca_smu_type type, void *data)
-{
-	int ret;
-
-	ret = aca_bank_parser(handle, bank, type);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static int aca_dispatch_bank(struct aca_handle_manager *mgr, struct aca_bank *bank,
-			     enum aca_smu_type type, bank_handler_t handler, void *data)
-{
-	struct aca_handle *handle;
-	int ret;
-
-	if (list_empty(&mgr->list))
-		return 0;
-
-	list_for_each_entry(handle, &mgr->list, node) {
-		if (!aca_bank_is_valid(handle, bank, type))
-			continue;
-
-		ret = handler(handle, bank, type, data);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static int aca_dispatch_banks(struct aca_handle_manager *mgr, struct aca_banks *banks,
-			      enum aca_smu_type type, bank_handler_t handler, void *data)
-{
-	struct aca_bank_node *node;
-	struct aca_bank *bank;
-	int ret;
-
-	if (!mgr || !banks)
-		return -EINVAL;
-
-	/* pre check to avoid unnecessary operations */
-	if (list_empty(&mgr->list) || list_empty(&banks->list))
-		return 0;
-
-	list_for_each_entry(node, &banks->list, node) {
-		bank = &node->bank;
-
-		ret = aca_dispatch_bank(mgr, bank, type, handler, data);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static bool aca_bank_should_update(struct amdgpu_device *adev, enum aca_smu_type type)
-{
-	struct amdgpu_aca *aca = &adev->aca;
-	bool ret = true;
-
-	/*
-	 * Because the UE Valid MCA count will only be cleared after reset,
-	 * in order to avoid repeated counting of the error count,
-	 * the aca bank is only updated once during the gpu recovery stage.
-	 */
-	if (type == ACA_SMU_TYPE_UE) {
-		if (amdgpu_ras_intr_triggered())
-			ret = atomic_cmpxchg(&aca->ue_update_flag, 0, 1) == 0;
-		else
-			atomic_set(&aca->ue_update_flag, 0);
-	}
-
-	return ret;
-}
-
-static void aca_banks_generate_cper(struct amdgpu_device *adev,
-				    enum aca_smu_type type,
-				    struct aca_banks *banks,
-				    int count)
-{
-	struct aca_bank_node *node;
-	struct aca_bank *bank;
-	int r;
-
-	if (!adev->cper.enabled)
-		return;
-
-	if (!banks || !count) {
-		dev_warn(adev->dev, "fail to generate cper records\n");
-		return;
-	}
-
-	/* UEs must be encoded into separate CPER entries */
-	if (type == ACA_SMU_TYPE_UE) {
-		struct aca_banks de_banks;
-
-		aca_banks_init(&de_banks);
-		list_for_each_entry(node, &banks->list, node) {
-			bank = &node->bank;
-			if (bank->aca_err_type == ACA_ERROR_TYPE_DEFERRED) {
-				r = aca_banks_add_bank(&de_banks, bank);
-				if (r)
-					dev_warn(adev->dev, "fail to add de banks, ret = %d\n", r);
-			} else {
-				if (amdgpu_cper_generate_ue_record(adev, bank))
-					dev_warn(adev->dev, "fail to generate ue cper records\n");
-			}
-		}
-
-		if (!list_empty(&de_banks.list)) {
-			if (amdgpu_cper_generate_ce_records(adev, &de_banks, de_banks.nr_banks))
-				dev_warn(adev->dev, "fail to generate de cper records\n");
-		}
-
-		aca_banks_release(&de_banks);
-	} else {
-		/*
-		 * SMU_TYPE_CE banks are combined into 1 CPER entries,
-		 * they could be CEs or DEs or both
-		 */
-		if (amdgpu_cper_generate_ce_records(adev, banks, count))
-			dev_warn(adev->dev, "fail to generate ce cper records\n");
-	}
-}
-
-static int aca_banks_update(struct amdgpu_device *adev, enum aca_smu_type type,
-			    bank_handler_t handler, struct ras_query_context *qctx, void *data)
-{
-	struct amdgpu_aca *aca = &adev->aca;
-	struct aca_banks banks;
-	u32 count = 0;
-	int ret;
-
-	if (list_empty(&aca->mgr.list))
-		return 0;
-
-	if (!aca_bank_should_update(adev, type))
-		return 0;
-
-	ret = aca_smu_get_valid_aca_count(adev, type, &count);
-	if (ret)
-		return ret;
-
-	if (!count)
-		return 0;
-
-	aca_banks_init(&banks);
-
-	ret = aca_smu_get_valid_aca_banks(adev, type, 0, count, &banks, qctx);
-	if (ret)
-		goto err_release_banks;
-
-	if (list_empty(&banks.list)) {
-		ret = 0;
-		goto err_release_banks;
-	}
-
-	ret = aca_dispatch_banks(&aca->mgr, &banks, type,
-				 handler, data);
-	if (ret)
-		goto err_release_banks;
-
-	aca_banks_generate_cper(adev, type, &banks, count);
-
-err_release_banks:
-	aca_banks_release(&banks);
-
-	return ret;
-}
-
-static int aca_log_aca_error_data(struct aca_bank_error *bank_error, enum aca_error_type type, struct ras_err_data *err_data)
-{
-	struct aca_bank_info *info;
-	struct amdgpu_smuio_mcm_config_info mcm_info;
-	u64 count;
-
-	if (type >= ACA_ERROR_TYPE_COUNT)
-		return -EINVAL;
-
-	count = bank_error->count;
-	if (!count)
-		return 0;
-
-	info = &bank_error->info;
-	mcm_info.die_id = info->die_id;
-	mcm_info.socket_id = info->socket_id;
-
-	switch (type) {
-	case ACA_ERROR_TYPE_UE:
-		amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, count);
-		break;
-	case ACA_ERROR_TYPE_CE:
-		amdgpu_ras_error_statistic_ce_count(err_data, &mcm_info, count);
-		break;
-	case ACA_ERROR_TYPE_DEFERRED:
-		amdgpu_ras_error_statistic_de_count(err_data, &mcm_info, count);
-		break;
-	default:
-		break;
-	}
-
-	return 0;
-}
-
-static int aca_log_aca_error(struct aca_handle *handle, enum aca_error_type type, struct ras_err_data *err_data)
-{
-	struct aca_error_cache *error_cache = &handle->error_cache;
-	struct aca_error *aerr = &error_cache->errors[type];
-	struct aca_bank_error *bank_error, *tmp;
-
-	mutex_lock(&aerr->lock);
-
-	if (list_empty(&aerr->list))
-		goto out_unlock;
-
-	list_for_each_entry_safe(bank_error, tmp, &aerr->list, node) {
-		aca_log_aca_error_data(bank_error, type, err_data);
-		aca_bank_error_remove(aerr, bank_error);
-	}
-
-out_unlock:
-	mutex_unlock(&aerr->lock);
-
-	return 0;
-}
-
-static int __aca_get_error_data(struct amdgpu_device *adev, struct aca_handle *handle, enum aca_error_type type,
-				struct ras_err_data *err_data, struct ras_query_context *qctx)
-{
-	enum aca_smu_type smu_type;
-	int ret;
-
-	switch (type) {
-	case ACA_ERROR_TYPE_UE:
-		smu_type = ACA_SMU_TYPE_UE;
-		break;
-	case ACA_ERROR_TYPE_CE:
-	case ACA_ERROR_TYPE_DEFERRED:
-		smu_type = ACA_SMU_TYPE_CE;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	/* update aca bank to aca source error_cache first */
-	ret = aca_banks_update(adev, smu_type, handler_aca_log_bank_error, qctx, NULL);
-	if (ret)
-		return ret;
-
-	/* DEs may contain in CEs or UEs */
-	if (type != ACA_ERROR_TYPE_DEFERRED)
-		aca_log_aca_error(handle, ACA_ERROR_TYPE_DEFERRED, err_data);
-
-	return aca_log_aca_error(handle, type, err_data);
-}
-
-static bool aca_handle_is_valid(struct aca_handle *handle)
-{
-	if (!handle->mask || !list_empty(&handle->node))
-		return false;
-
-	return true;
-}
-
-int amdgpu_aca_get_error_data(struct amdgpu_device *adev, struct aca_handle *handle,
-			      enum aca_error_type type, struct ras_err_data *err_data,
-			      struct ras_query_context *qctx)
-{
-	if (!handle || !err_data)
-		return -EINVAL;
-
-	if (aca_handle_is_valid(handle))
-		return -EOPNOTSUPP;
-
-	if ((type < 0) || (!(BIT(type) & handle->mask)))
-		return  0;
-
-	return __aca_get_error_data(adev, handle, type, err_data, qctx);
-}
-
-static void aca_error_init(struct aca_error *aerr, enum aca_error_type type)
-{
-	mutex_init(&aerr->lock);
-	INIT_LIST_HEAD(&aerr->list);
-	aerr->type = type;
-	aerr->nr_errors = 0;
-}
-
-static void aca_init_error_cache(struct aca_handle *handle)
-{
-	struct aca_error_cache *error_cache = &handle->error_cache;
-	int type;
-
-	for (type = ACA_ERROR_TYPE_UE; type < ACA_ERROR_TYPE_COUNT; type++)
-		aca_error_init(&error_cache->errors[type], type);
-}
-
-static void aca_error_fini(struct aca_error *aerr)
-{
-	struct aca_bank_error *bank_error, *tmp;
-
-	mutex_lock(&aerr->lock);
-	if (list_empty(&aerr->list))
-		goto out_unlock;
-
-	list_for_each_entry_safe(bank_error, tmp, &aerr->list, node)
-		aca_bank_error_remove(aerr, bank_error);
-
-out_unlock:
-	mutex_unlock(&aerr->lock);
-	mutex_destroy(&aerr->lock);
-}
-
-static void aca_fini_error_cache(struct aca_handle *handle)
-{
-	struct aca_error_cache *error_cache = &handle->error_cache;
-	int type;
-
-	for (type = ACA_ERROR_TYPE_UE; type < ACA_ERROR_TYPE_COUNT; type++)
-		aca_error_fini(&error_cache->errors[type]);
-}
-
-static int add_aca_handle(struct amdgpu_device *adev, struct aca_handle_manager *mgr, struct aca_handle *handle,
-			  const char *name, const struct aca_info *ras_info, void *data)
-{
-	memset(handle, 0, sizeof(*handle));
-
-	handle->adev = adev;
-	handle->mgr = mgr;
-	handle->name = name;
-	handle->hwip = ras_info->hwip;
-	handle->mask = ras_info->mask;
-	handle->bank_ops = ras_info->bank_ops;
-	handle->data = data;
-	aca_init_error_cache(handle);
-
-	INIT_LIST_HEAD(&handle->node);
-	list_add_tail(&handle->node, &mgr->list);
-	mgr->nr_handles++;
-
-	return 0;
-}
-
-static ssize_t aca_sysfs_read(struct device *dev,
-			      struct device_attribute *attr, char *buf)
-{
-	struct aca_handle *handle = container_of(attr, struct aca_handle, aca_attr);
-
-	/* NOTE: the aca cache will be auto cleared once read,
-	 * So the driver should unify the query entry point, forward request to ras query interface directly */
-	return amdgpu_ras_aca_sysfs_read(dev, attr, handle, buf, handle->data);
-}
-
-static int add_aca_sysfs(struct amdgpu_device *adev, struct aca_handle *handle)
-{
-	struct device_attribute *aca_attr = &handle->aca_attr;
-
-	snprintf(handle->attr_name, sizeof(handle->attr_name) - 1, "aca_%s", handle->name);
-	aca_attr->show = aca_sysfs_read;
-	aca_attr->attr.name = handle->attr_name;
-	aca_attr->attr.mode = S_IRUGO;
-	sysfs_attr_init(&aca_attr->attr);
-
-	return sysfs_add_file_to_group(&adev->dev->kobj,
-				       &aca_attr->attr,
-				       "ras");
-}
-
-int amdgpu_aca_add_handle(struct amdgpu_device *adev, struct aca_handle *handle,
-			  const char *name, const struct aca_info *ras_info, void *data)
-{
-	struct amdgpu_aca *aca = &adev->aca;
-	int ret;
-
-	if (!amdgpu_aca_is_enabled(adev))
-		return 0;
-
-	ret = add_aca_handle(adev, &aca->mgr, handle, name, ras_info, data);
-	if (ret)
-		return ret;
-
-	return add_aca_sysfs(adev, handle);
-}
-
-static void remove_aca_handle(struct aca_handle *handle)
-{
-	struct aca_handle_manager *mgr = handle->mgr;
-
-	aca_fini_error_cache(handle);
-	list_del(&handle->node);
-	mgr->nr_handles--;
-}
-
-static void remove_aca_sysfs(struct aca_handle *handle)
-{
-	struct amdgpu_device *adev = handle->adev;
-	struct device_attribute *aca_attr = &handle->aca_attr;
-
-	if (adev->dev->kobj.sd)
-		sysfs_remove_file_from_group(&adev->dev->kobj,
-					     &aca_attr->attr,
-					     "ras");
-}
-
-void amdgpu_aca_remove_handle(struct aca_handle *handle)
-{
-	if (!handle || list_empty(&handle->node))
-		return;
-
-	remove_aca_sysfs(handle);
-	remove_aca_handle(handle);
-}
-
-static int aca_manager_init(struct aca_handle_manager *mgr)
-{
-	INIT_LIST_HEAD(&mgr->list);
-	mgr->nr_handles = 0;
-
-	return 0;
-}
-
-static void aca_manager_fini(struct aca_handle_manager *mgr)
-{
-	struct aca_handle *handle, *tmp;
-
-	if (list_empty(&mgr->list))
-		return;
-
-	list_for_each_entry_safe(handle, tmp, &mgr->list, node)
-		amdgpu_aca_remove_handle(handle);
-}
-
-bool amdgpu_aca_is_enabled(struct amdgpu_device *adev)
-{
-	return (adev->aca.is_enabled ||
-		adev->debug_enable_ras_aca);
-}
-
-int amdgpu_aca_init(struct amdgpu_device *adev)
-{
-	struct amdgpu_aca *aca = &adev->aca;
-	int ret;
-
-	atomic_set(&aca->ue_update_flag, 0);
-
-	ret = aca_manager_init(&aca->mgr);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-void amdgpu_aca_fini(struct amdgpu_device *adev)
-{
-	struct amdgpu_aca *aca = &adev->aca;
-
-	aca_manager_fini(&aca->mgr);
-
-	atomic_set(&aca->ue_update_flag, 0);
-}
-
-int amdgpu_aca_reset(struct amdgpu_device *adev)
-{
-	struct amdgpu_aca *aca = &adev->aca;
-
-	atomic_set(&aca->ue_update_flag, 0);
-
-	return 0;
-}
-
-void amdgpu_aca_set_smu_funcs(struct amdgpu_device *adev, const struct aca_smu_funcs *smu_funcs)
-{
-	struct amdgpu_aca *aca = &adev->aca;
-
-	WARN_ON(aca->smu_funcs);
-	aca->smu_funcs = smu_funcs;
-}
-
-int aca_bank_info_decode(struct aca_bank *bank, struct aca_bank_info *info)
-{
-	u64 ipid;
-	u32 instidhi, instidlo;
-
-	if (!bank || !info)
-		return -EINVAL;
-
-	ipid = bank->regs[ACA_REG_IDX_IPID];
-	info->hwid = ACA_REG__IPID__HARDWAREID(ipid);
-	info->mcatype = ACA_REG__IPID__MCATYPE(ipid);
-	/*
-	 * Unfied DieID Format: SAASS. A:AID, S:Socket.
-	 * Unfied DieID[4:4] = InstanceId[0:0]
-	 * Unfied DieID[0:3] = InstanceIdHi[0:3]
-	 */
-	instidhi = ACA_REG__IPID__INSTANCEIDHI(ipid);
-	instidlo = ACA_REG__IPID__INSTANCEIDLO(ipid);
-	info->die_id = ((instidhi >> 2) & 0x03);
-	info->socket_id = ((instidlo & 0x1) << 2) | (instidhi & 0x03);
-
-	return 0;
-}
-
-static int aca_bank_get_error_code(struct amdgpu_device *adev, struct aca_bank *bank)
-{
-	struct amdgpu_aca *aca = &adev->aca;
-	const struct aca_smu_funcs *smu_funcs = aca->smu_funcs;
-
-	if (!smu_funcs || !smu_funcs->parse_error_code)
-		return -EOPNOTSUPP;
-
-	return smu_funcs->parse_error_code(adev, bank);
-}
-
-int aca_bank_check_error_codes(struct amdgpu_device *adev, struct aca_bank *bank, int *err_codes, int size)
-{
-	int i, error_code;
-
-	if (!bank || !err_codes)
-		return -EINVAL;
-
-	error_code = aca_bank_get_error_code(adev, bank);
-	if (error_code < 0)
-		return error_code;
-
-	for (i = 0; i < size; i++) {
-		if (err_codes[i] == error_code)
-			return 0;
-	}
-
-	return -EINVAL;
-}
-
-int amdgpu_aca_smu_set_debug_mode(struct amdgpu_device *adev, bool en)
-{
-	struct amdgpu_aca *aca = &adev->aca;
-	const struct aca_smu_funcs *smu_funcs = aca->smu_funcs;
-
-	if (!smu_funcs || !smu_funcs->set_debug_mode)
-		return -EOPNOTSUPP;
-
-	return smu_funcs->set_debug_mode(adev, en);
-}
-
-#if defined(CONFIG_DEBUG_FS)
-static int amdgpu_aca_smu_debug_mode_set(void *data, u64 val)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)data;
-	int ret;
-
-	ret = amdgpu_ras_set_aca_debug_mode(adev, val ? true : false);
-	if (ret)
-		return ret;
-
-	dev_info(adev->dev, "amdgpu set smu aca debug mode %s success\n", val ? "on" : "off");
-
-	return 0;
-}
-
-static void aca_dump_entry(struct seq_file *m, struct aca_bank *bank, enum aca_smu_type type, int idx)
-{
-	struct aca_bank_info info;
-	int i, ret;
-
-	ret = aca_bank_info_decode(bank, &info);
-	if (ret)
-		return;
-
-	seq_printf(m, "aca entry[%d].type: %s\n", idx, type ==  ACA_SMU_TYPE_UE ? "UE" : "CE");
-	seq_printf(m, "aca entry[%d].info: socketid:%d aid:%d hwid:0x%03x mcatype:0x%04x\n",
-		   idx, info.socket_id, info.die_id, info.hwid, info.mcatype);
-
-	for (i = 0; i < ARRAY_SIZE(aca_regs); i++)
-		seq_printf(m, "aca entry[%d].regs[%d]: 0x%016llx\n", idx, aca_regs[i].reg_idx, bank->regs[aca_regs[i].reg_idx]);
-}
-
-struct aca_dump_context {
-	struct seq_file *m;
-	int idx;
-};
-
-static int handler_aca_bank_dump(struct aca_handle *handle, struct aca_bank *bank,
-				 enum aca_smu_type type, void *data)
-{
-	struct aca_dump_context *ctx = (struct aca_dump_context *)data;
-
-	aca_dump_entry(ctx->m, bank, type, ctx->idx++);
-
-	return handler_aca_log_bank_error(handle, bank, type, NULL);
-}
-
-static int aca_dump_show(struct seq_file *m, enum aca_smu_type type)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
-	struct aca_dump_context context = {
-		.m = m,
-		.idx = 0,
-	};
-
-	return aca_banks_update(adev, type, handler_aca_bank_dump, NULL, (void *)&context);
-}
-
-static int aca_dump_ce_show(struct seq_file *m, void *unused)
-{
-	return aca_dump_show(m, ACA_SMU_TYPE_CE);
-}
-
-static int aca_dump_ce_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, aca_dump_ce_show, inode->i_private);
-}
-
-static const struct file_operations aca_ce_dump_debug_fops = {
-	.owner = THIS_MODULE,
-	.open = aca_dump_ce_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
-
-static int aca_dump_ue_show(struct seq_file *m, void *unused)
-{
-	return aca_dump_show(m, ACA_SMU_TYPE_UE);
-}
-
-static int aca_dump_ue_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, aca_dump_ue_show, inode->i_private);
-}
-
-static const struct file_operations aca_ue_dump_debug_fops = {
-	.owner = THIS_MODULE,
-	.open = aca_dump_ue_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
-
-DEFINE_DEBUGFS_ATTRIBUTE(aca_debug_mode_fops, NULL, amdgpu_aca_smu_debug_mode_set, "%llu\n");
-#endif
-
-void amdgpu_aca_smu_debugfs_init(struct amdgpu_device *adev, struct dentry *root)
-{
-#if defined(CONFIG_DEBUG_FS)
-	if (!root)
-		return;
-
-	debugfs_create_file("aca_debug_mode", 0200, root, adev, &aca_debug_mode_fops);
-	debugfs_create_file("aca_ue_dump", 0400, root, adev, &aca_ue_dump_debug_fops);
-	debugfs_create_file("aca_ce_dump", 0400, root, adev, &aca_ce_dump_debug_fops);
-#endif
-}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.h
deleted file mode 100644
index 38c88897e1ecb..0000000000000
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_aca.h
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Copyright 2023 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#ifndef __AMDGPU_ACA_H__
-#define __AMDGPU_ACA_H__
-
-#include <linux/list.h>
-
-struct ras_err_data;
-struct ras_query_context;
-
-#define ACA_MAX_REGS_COUNT	(16)
-
-#define ACA_REG_FIELD(x, h, l)			(((x) & GENMASK_ULL(h, l)) >> l)
-#define ACA_REG__STATUS__VAL(x)			ACA_REG_FIELD(x, 63, 63)
-#define ACA_REG__STATUS__OVERFLOW(x)		ACA_REG_FIELD(x, 62, 62)
-#define ACA_REG__STATUS__UC(x)			ACA_REG_FIELD(x, 61, 61)
-#define ACA_REG__STATUS__EN(x)			ACA_REG_FIELD(x, 60, 60)
-#define ACA_REG__STATUS__MISCV(x)		ACA_REG_FIELD(x, 59, 59)
-#define ACA_REG__STATUS__ADDRV(x)		ACA_REG_FIELD(x, 58, 58)
-#define ACA_REG__STATUS__PCC(x)			ACA_REG_FIELD(x, 57, 57)
-#define ACA_REG__STATUS__ERRCOREIDVAL(x)	ACA_REG_FIELD(x, 56, 56)
-#define ACA_REG__STATUS__TCC(x)			ACA_REG_FIELD(x, 55, 55)
-#define ACA_REG__STATUS__SYNDV(x)		ACA_REG_FIELD(x, 53, 53)
-#define ACA_REG__STATUS__CECC(x)		ACA_REG_FIELD(x, 46, 46)
-#define ACA_REG__STATUS__UECC(x)		ACA_REG_FIELD(x, 45, 45)
-#define ACA_REG__STATUS__DEFERRED(x)		ACA_REG_FIELD(x, 44, 44)
-#define ACA_REG__STATUS__POISON(x)		ACA_REG_FIELD(x, 43, 43)
-#define ACA_REG__STATUS__SCRUB(x)		ACA_REG_FIELD(x, 40, 40)
-#define ACA_REG__STATUS__ERRCOREID(x)		ACA_REG_FIELD(x, 37, 32)
-#define ACA_REG__STATUS__ADDRLSB(x)		ACA_REG_FIELD(x, 29, 24)
-#define ACA_REG__STATUS__ERRORCODEEXT(x)	ACA_REG_FIELD(x, 21, 16)
-#define ACA_REG__STATUS__ERRORCODE(x)		ACA_REG_FIELD(x, 15, 0)
-
-#define ACA_REG__IPID__MCATYPE(x)		ACA_REG_FIELD(x, 63, 48)
-#define ACA_REG__IPID__INSTANCEIDHI(x)		ACA_REG_FIELD(x, 47, 44)
-#define ACA_REG__IPID__HARDWAREID(x)		ACA_REG_FIELD(x, 43, 32)
-#define ACA_REG__IPID__INSTANCEIDLO(x)		ACA_REG_FIELD(x, 31, 0)
-
-#define ACA_REG__MISC0__VALID(x)		ACA_REG_FIELD(x, 63, 63)
-#define ACA_REG__MISC0__OVRFLW(x)		ACA_REG_FIELD(x, 48, 48)
-#define ACA_REG__MISC0__ERRCNT(x)		ACA_REG_FIELD(x, 43, 32)
-
-#define ACA_REG__SYND__ERRORINFORMATION(x)	ACA_REG_FIELD(x, 17, 0)
-
-/* NOTE: The following codes refers to the smu header file */
-#define ACA_EXTERROR_CODE_CE			0x3a
-#define ACA_EXTERROR_CODE_FAULT			0x3b
-
-#define ACA_ERROR_UE_MASK		BIT_MASK(ACA_ERROR_TYPE_UE)
-#define ACA_ERROR_CE_MASK		BIT_MASK(ACA_ERROR_TYPE_CE)
-#define ACA_ERROR_DEFERRED_MASK		BIT_MASK(ACA_ERROR_TYPE_DEFERRED)
-
-#define mmSMNAID_AID0_MCA_SMU		0x03b30400	/* SMN AID AID0 */
-#define mmSMNAID_XCD0_MCA_SMU		0x36430400	/* SMN AID XCD0 */
-#define mmSMNAID_XCD1_MCA_SMU		0x38430400	/* SMN AID XCD1 */
-#define mmSMNXCD_XCD0_MCA_SMU		0x40430400	/* SMN XCD XCD0 */
-
-#define ACA_BANK_ERR_IS_DEFFERED(bank)                                \
-	(ACA_REG__STATUS__POISON((bank)->regs[ACA_REG_IDX_STATUS]) || \
-	 ACA_REG__STATUS__DEFERRED((bank)->regs[ACA_REG_IDX_STATUS]))
-
-enum aca_reg_idx {
-	ACA_REG_IDX_CTL			= 0,
-	ACA_REG_IDX_STATUS		= 1,
-	ACA_REG_IDX_ADDR		= 2,
-	ACA_REG_IDX_MISC0		= 3,
-	ACA_REG_IDX_CONFIG		= 4,
-	ACA_REG_IDX_IPID		= 5,
-	ACA_REG_IDX_SYND		= 6,
-	ACA_REG_IDX_DESTAT		= 8,
-	ACA_REG_IDX_DEADDR		= 9,
-	ACA_REG_IDX_CTL_MASK		= 10,
-	ACA_REG_IDX_COUNT		= 16,
-};
-
-enum aca_hwip_type {
-	ACA_HWIP_TYPE_UNKNOW = -1,
-	ACA_HWIP_TYPE_PSP = 0,
-	ACA_HWIP_TYPE_UMC,
-	ACA_HWIP_TYPE_SMU,
-	ACA_HWIP_TYPE_PCS_XGMI,
-	ACA_HWIP_TYPE_COUNT,
-};
-
-enum aca_error_type {
-	ACA_ERROR_TYPE_INVALID = -1,
-	ACA_ERROR_TYPE_UE = 0,
-	ACA_ERROR_TYPE_CE,
-	ACA_ERROR_TYPE_DEFERRED,
-	ACA_ERROR_TYPE_COUNT
-};
-
-enum aca_smu_type {
-	ACA_SMU_TYPE_INVALID = -1,
-	ACA_SMU_TYPE_UE = 0,
-	ACA_SMU_TYPE_CE,
-	ACA_SMU_TYPE_COUNT,
-};
-
-struct aca_hwip {
-	int hwid;
-	int mcatype;
-};
-
-struct aca_bank {
-	enum aca_error_type aca_err_type;
-	enum aca_smu_type smu_err_type;
-	u64 regs[ACA_MAX_REGS_COUNT];
-};
-
-struct aca_bank_node {
-	struct aca_bank bank;
-	struct list_head node;
-};
-
-struct aca_banks {
-	int nr_banks;
-	struct list_head list;
-};
-
-struct aca_bank_info {
-	int die_id;
-	int socket_id;
-	int hwid;
-	int mcatype;
-};
-
-struct aca_bank_error {
-	struct list_head node;
-	struct aca_bank_info info;
-	u64 count;
-};
-
-struct aca_error {
-	struct list_head list;
-	struct mutex lock;
-	enum aca_error_type type;
-	int nr_errors;
-};
-
-struct aca_handle_manager {
-	struct list_head list;
-	int nr_handles;
-};
-
-struct aca_error_cache {
-	struct aca_error errors[ACA_ERROR_TYPE_COUNT];
-};
-
-struct aca_handle {
-	struct list_head node;
-	enum aca_hwip_type hwip;
-	struct amdgpu_device *adev;
-	struct aca_handle_manager *mgr;
-	struct aca_error_cache error_cache;
-	const struct aca_bank_ops *bank_ops;
-	struct device_attribute aca_attr;
-	char attr_name[64];
-	const char *name;
-	u32 mask;
-	void *data;
-};
-
-struct aca_bank_ops {
-	int (*aca_bank_parser)(struct aca_handle *handle, struct aca_bank *bank, enum aca_smu_type type, void *data);
-	bool (*aca_bank_is_valid)(struct aca_handle *handle, struct aca_bank *bank, enum aca_smu_type type,
-				  void *data);
-};
-
-struct aca_smu_funcs {
-	int max_ue_bank_count;
-	int max_ce_bank_count;
-	int (*set_debug_mode)(struct amdgpu_device *adev, bool enable);
-	int (*get_valid_aca_count)(struct amdgpu_device *adev, enum aca_smu_type type, u32 *count);
-	int (*get_valid_aca_bank)(struct amdgpu_device *adev, enum aca_smu_type type, int idx, struct aca_bank *bank);
-	int (*parse_error_code)(struct amdgpu_device *adev, struct aca_bank *bank);
-};
-
-struct amdgpu_aca {
-	struct aca_handle_manager mgr;
-	const struct aca_smu_funcs *smu_funcs;
-	atomic_t ue_update_flag;
-	bool is_enabled;
-};
-
-struct aca_info {
-	enum aca_hwip_type hwip;
-	const struct aca_bank_ops *bank_ops;
-	u32 mask;
-};
-
-int amdgpu_aca_init(struct amdgpu_device *adev);
-void amdgpu_aca_fini(struct amdgpu_device *adev);
-int amdgpu_aca_reset(struct amdgpu_device *adev);
-void amdgpu_aca_set_smu_funcs(struct amdgpu_device *adev, const struct aca_smu_funcs *smu_funcs);
-bool amdgpu_aca_is_enabled(struct amdgpu_device *adev);
-
-int aca_bank_info_decode(struct aca_bank *bank, struct aca_bank_info *info);
-int aca_bank_check_error_codes(struct amdgpu_device *adev, struct aca_bank *bank, int *err_codes, int size);
-
-int amdgpu_aca_add_handle(struct amdgpu_device *adev, struct aca_handle *handle,
-			  const char *name, const struct aca_info *aca_info, void *data);
-void amdgpu_aca_remove_handle(struct aca_handle *handle);
-int amdgpu_aca_get_error_data(struct amdgpu_device *adev, struct aca_handle *handle,
-			      enum aca_error_type type, struct ras_err_data *err_data,
-			      struct ras_query_context *qctx);
-int amdgpu_aca_smu_set_debug_mode(struct amdgpu_device *adev, bool en);
-void amdgpu_aca_smu_debugfs_init(struct amdgpu_device *adev, struct dentry *root);
-int aca_error_cache_log_bank_error(struct aca_handle *handle, struct aca_bank_info *info,
-				   enum aca_error_type type, u64 count);
-#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
index 516ab9cf88fc0..7f5abb03be1b3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
@@ -140,13 +140,15 @@ static struct amdgpu_acpi_priv {
  * @atif: atif structure
  * @function: the ATIF function to execute
  * @params: ATIF function params
+ * @min_size: minimum size of the expected output buffer in bytes
  *
  * Executes the requested ATIF function (all asics).
  * Returns a pointer to the acpi output buffer.
  */
 static union acpi_object *amdgpu_atif_call(struct amdgpu_atif *atif,
 					   int function,
-					   struct acpi_buffer *params)
+					   struct acpi_buffer *params,
+					   size_t min_size)
 {
 	acpi_status status;
 	union acpi_object *obj;
@@ -189,6 +191,28 @@ static union acpi_object *amdgpu_atif_call(struct amdgpu_atif *atif,
 		return NULL;
 	}
 
+	if (obj->buffer.length < sizeof(u16)) {
+		DRM_DEBUG_DRIVER("ATIF buffer too small to hold size field: %u\n",
+				 obj->buffer.length);
+		kfree(obj);
+		return NULL;
+	}
+
+	if (obj->buffer.length < *(u16 *)obj->buffer.pointer) {
+		DRM_DEBUG_DRIVER("ATIF buffer length mismatch: reported %u, actual %u\n",
+				 *(u16 *)obj->buffer.pointer,
+				 obj->buffer.length);
+		kfree(obj);
+		return NULL;
+	}
+
+	if (*(u16 *)obj->buffer.pointer < min_size) {
+		DRM_DEBUG_DRIVER("ATIF buffer too small: expected %zu, got %u\n",
+				 min_size, *(u16 *)obj->buffer.pointer);
+		kfree(obj);
+		return NULL;
+	}
+
 	return obj;
 }
 
@@ -251,19 +275,14 @@ int amdgpu_atif_verify_interface(struct amdgpu_atif *atif)
 	size_t size;
 	int err = 0;
 
-	info = amdgpu_atif_call(atif, ATIF_FUNCTION_VERIFY_INTERFACE, NULL);
+	info = amdgpu_atif_call(atif, ATIF_FUNCTION_VERIFY_INTERFACE, NULL,
+				sizeof(output));
 	if (!info)
 		return -EIO;
 
 	memset(&output, 0, sizeof(output));
 
-	size = *(u16 *) info->buffer.pointer;
-	if (size < 12) {
-		DRM_INFO("ATIF buffer is too small: %zu\n", size);
-		err = -EINVAL;
-		goto out;
-	}
-	size = min(sizeof(output), size);
+	size = min(sizeof(output), (size_t)*(u16 *)info->buffer.pointer);
 
 	memcpy(&output, info->buffer.pointer, size);
 
@@ -273,7 +292,6 @@ int amdgpu_atif_verify_interface(struct amdgpu_atif *atif)
 	amdgpu_atif_parse_notification(&atif->notifications, output.notification_mask);
 	amdgpu_atif_parse_functions(&atif->functions, output.function_bits);
 
-out:
 	kfree(info);
 	return err;
 }
@@ -299,20 +317,14 @@ int amdgpu_atif_get_notification_params(struct amdgpu_atif *atif)
 	int err = 0;
 
 	info = amdgpu_atif_call(atif, ATIF_FUNCTION_GET_SYSTEM_PARAMETERS,
-				NULL);
+				NULL, offsetof(struct atif_system_params, command_code));
 	if (!info) {
 		err = -EIO;
 		goto out;
 	}
 
-	size = *(u16 *) info->buffer.pointer;
-	if (size < 10) {
-		err = -EINVAL;
-		goto out;
-	}
-
 	memset(&params, 0, sizeof(params));
-	size = min(sizeof(params), size);
+	size = min(sizeof(params), (size_t)*(u16 *)info->buffer.pointer);
 	memcpy(&params, info->buffer.pointer, size);
 
 	DRM_DEBUG_DRIVER("SYSTEM_PARAMS: mask = %#x, flags = %#x\n",
@@ -376,20 +388,14 @@ int amdgpu_atif_query_backlight_caps(struct amdgpu_atif *atif)
 
 	info = amdgpu_atif_call(atif,
 		ATIF_FUNCTION_QUERY_BRIGHTNESS_TRANSFER_CHARACTERISTICS,
-		&params);
+		&params, offsetof(struct atif_qbtc_output, data_points));
 	if (!info) {
 		err = -EIO;
 		goto out;
 	}
 
-	size = *(u16 *) info->buffer.pointer;
-	if (size < 10) {
-		err = -EINVAL;
-		goto out;
-	}
-
 	memset(&characteristics, 0, sizeof(characteristics));
-	size = min(sizeof(characteristics), size);
+	size = min(sizeof(characteristics), (size_t)*(u16 *)info->buffer.pointer);
 	memcpy(&characteristics, info->buffer.pointer, size);
 
 	atif->backlight_caps.caps_valid = true;
@@ -427,24 +433,18 @@ static int amdgpu_atif_get_sbios_requests(struct amdgpu_atif *atif,
 	int count = 0;
 
 	info = amdgpu_atif_call(atif, ATIF_FUNCTION_GET_SYSTEM_BIOS_REQUESTS,
-				NULL);
+				NULL, sizeof(*req));
 	if (!info)
 		return -EIO;
 
-	size = *(u16 *)info->buffer.pointer;
-	if (size < 0xd) {
-		count = -EINVAL;
-		goto out;
-	}
 	memset(req, 0, sizeof(*req));
 
-	size = min(sizeof(*req), size);
+	size = min(sizeof(*req), (size_t)*(u16 *)info->buffer.pointer);
 	memcpy(req, info->buffer.pointer, size);
 	DRM_DEBUG_DRIVER("SBIOS pending requests: %#x\n", req->pending);
 
 	count = hweight32(req->pending);
 
-out:
 	kfree(info);
 	return count;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.h
new file mode 100644
index 0000000000000..6569a4db5dae8
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.h
@@ -0,0 +1,151 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef __AMDGPU_ACPI_H__
+#define __AMDGPU_ACPI_H__
+
+#include <linux/types.h>
+#include <linux/mutex_types.h>
+
+struct amdgpu_device;
+struct acpi_device;
+struct amdgpu_dm_backlight_caps;
+
+#define MAX_UMA_OPTION_NAME	28
+#define MAX_UMA_OPTION_ENTRIES	19
+
+#define AMDGPU_UMA_FLAG_AUTO	BIT(1)
+#define AMDGPU_UMA_FLAG_CUSTOM	BIT(0)
+
+/* ATCS Device/Driver State */
+#define AMDGPU_ATCS_PSC_DEV_STATE_D0		0
+#define AMDGPU_ATCS_PSC_DEV_STATE_D3_HOT	3
+#define AMDGPU_ATCS_PSC_DRV_STATE_OPR		0
+#define AMDGPU_ATCS_PSC_DRV_STATE_NOT_OPR	1
+
+enum amdgpu_ss {
+	AMDGPU_SS_DRV_LOAD,
+	AMDGPU_SS_DEV_D0,
+	AMDGPU_SS_DEV_D3,
+	AMDGPU_SS_DRV_UNLOAD
+};
+
+/**
+ * struct amdgpu_uma_carveout_option - single UMA carveout option
+ * @name: Name of the carveout option
+ * @memory_carved_mb: Amount of memory carved in MB
+ * @flags: ATCS flags supported by this option
+ */
+struct amdgpu_uma_carveout_option {
+	char name[MAX_UMA_OPTION_NAME];
+	uint32_t memory_carved_mb;
+	uint8_t flags;
+};
+
+/**
+ * struct amdgpu_uma_carveout_info - table of available UMA carveout options
+ * @num_entries: Number of available options
+ * @uma_option_index: The index of the option currently applied
+ * @update_lock: Lock to serialize changes to the option
+ * @entries: The array of carveout options
+ */
+struct amdgpu_uma_carveout_info {
+	uint8_t num_entries;
+	uint8_t uma_option_index;
+	struct mutex update_lock;
+	struct amdgpu_uma_carveout_option entries[MAX_UMA_OPTION_ENTRIES];
+};
+
+struct amdgpu_numa_info {
+	uint64_t size;
+	int pxm;
+	int nid;
+};
+
+#if defined(CONFIG_ACPI)
+int amdgpu_acpi_init(struct amdgpu_device *adev);
+void amdgpu_acpi_fini(struct amdgpu_device *adev);
+bool amdgpu_acpi_is_pcie_performance_request_supported(struct amdgpu_device *adev);
+bool amdgpu_acpi_is_power_shift_control_supported(void);
+bool amdgpu_acpi_is_set_uma_allocation_size_supported(void);
+int amdgpu_acpi_pcie_performance_request(struct amdgpu_device *adev,
+						u8 perf_req, bool advertise);
+int amdgpu_acpi_power_shift_control(struct amdgpu_device *adev,
+				    u8 dev_state, bool drv_state);
+int amdgpu_acpi_smart_shift_update(struct amdgpu_device *adev,
+				   enum amdgpu_ss ss_state);
+int amdgpu_acpi_set_uma_allocation_size(struct amdgpu_device *adev, u8 index, u8 type);
+int amdgpu_acpi_pcie_notify_device_ready(struct amdgpu_device *adev);
+int amdgpu_acpi_get_tmr_info(struct amdgpu_device *adev, u64 *tmr_offset,
+			     u64 *tmr_size);
+int amdgpu_acpi_get_mem_info(struct amdgpu_device *adev, int xcc_id,
+			     struct amdgpu_numa_info *numa_info);
+
+void amdgpu_acpi_get_backlight_caps(struct amdgpu_dm_backlight_caps *caps);
+bool amdgpu_acpi_should_gpu_reset(struct amdgpu_device *adev);
+void amdgpu_acpi_detect(void);
+void amdgpu_acpi_release(void);
+#else
+static inline int amdgpu_acpi_init(struct amdgpu_device *adev) { return 0; }
+static inline int amdgpu_acpi_get_tmr_info(struct amdgpu_device *adev,
+					   u64 *tmr_offset, u64 *tmr_size)
+{
+	return -EINVAL;
+}
+static inline int amdgpu_acpi_get_mem_info(struct amdgpu_device *adev,
+					   int xcc_id,
+					   struct amdgpu_numa_info *numa_info)
+{
+	return -EINVAL;
+}
+static inline void amdgpu_acpi_fini(struct amdgpu_device *adev) { }
+static inline bool amdgpu_acpi_should_gpu_reset(struct amdgpu_device *adev) { return false; }
+static inline void amdgpu_acpi_detect(void) { }
+static inline void amdgpu_acpi_release(void) { }
+static inline bool amdgpu_acpi_is_power_shift_control_supported(void) { return false; }
+static inline bool amdgpu_acpi_is_set_uma_allocation_size_supported(void) { return false; }
+static inline int amdgpu_acpi_power_shift_control(struct amdgpu_device *adev,
+						  u8 dev_state, bool drv_state) { return 0; }
+static inline int amdgpu_acpi_smart_shift_update(struct amdgpu_device *adev,
+						 enum amdgpu_ss ss_state)
+{
+	return 0;
+}
+static inline int amdgpu_acpi_set_uma_allocation_size(struct amdgpu_device *adev, u8 index, u8 type)
+{
+	return -EINVAL;
+}
+static inline void amdgpu_acpi_get_backlight_caps(struct amdgpu_dm_backlight_caps *caps) { }
+#endif
+
+#if defined(CONFIG_ACPI) && defined(CONFIG_SUSPEND)
+bool amdgpu_acpi_is_s3_active(struct amdgpu_device *adev);
+bool amdgpu_acpi_is_s0ix_active(struct amdgpu_device *adev);
+#else
+static inline bool amdgpu_acpi_is_s0ix_active(struct amdgpu_device *adev) { return false; }
+static inline bool amdgpu_acpi_is_s3_active(struct amdgpu_device *adev) { return false; }
+#endif
+
+#if defined(CONFIG_DRM_AMD_ISP)
+int amdgpu_acpi_get_isp4_dev(struct acpi_device **dev);
+#endif
+#endif /* __AMDGPU_ACPI_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index da325863ad764..121282dd30c16 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -711,7 +711,8 @@ int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev,
 		goto err;
 	}
 
-	ret = amdgpu_job_alloc(adev, NULL, NULL, NULL, 1, &job, 0);
+	ret = amdgpu_job_alloc(adev, NULL, NULL, NULL, 1, 0, GFP_KERNEL,
+			       &job);
 	if (ret)
 		goto err;
 
@@ -957,3 +958,17 @@ int amdgpu_amdkfd_config_sq_perfmon(struct amdgpu_device *adev, uint32_t xcp_id,
 
 	return r;
 }
+
+/* Reset an MES queue */
+int amdgpu_amdkfd_reset_mes_queue(struct amdgpu_device *adev,
+				  uint32_t node_id,
+				  int queue_type,
+				  int pipe, int queue,
+				  unsigned int db)
+{
+	if (!adev->kfd.init_complete)
+		return 0;
+
+	return kgd2kfd_reset_mes_queue(adev->kfd.dev, node_id, queue_type,
+				       pipe, queue, db);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index e443a72772994..338412a750edf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -210,6 +210,7 @@ int amdgpu_amdkfd_evict_userptr(struct mmu_interval_notifier *mni,
 int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo,
 					uint32_t domain,
 					struct dma_fence *fence);
+int amdgpu_amdkfd_set_sigbus_delay(struct task_struct *task, u32 ms);
 #else
 static inline
 bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
@@ -241,6 +242,11 @@ int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo,
 {
 	return 0;
 }
+static inline
+int amdgpu_amdkfd_set_sigbus_delay(struct task_struct *task, u32 ms)
+{
+	return -EOPNOTSUPP;
+}
 #endif
 /* Shared API */
 int amdgpu_amdkfd_alloc_kernel_mem(struct amdgpu_device *adev, size_t size,
@@ -275,7 +281,11 @@ int amdgpu_amdkfd_stop_sched(struct amdgpu_device *adev, uint32_t node_id);
 int amdgpu_amdkfd_config_sq_perfmon(struct amdgpu_device *adev, uint32_t xcp_id,
 	bool core_override_enable, bool reg_override_enable, bool perfmon_override_enable);
 bool amdgpu_amdkfd_compute_active(struct amdgpu_device *adev, uint32_t node_id);
-
+int amdgpu_amdkfd_reset_mes_queue(struct amdgpu_device *adev,
+				  uint32_t node_id,
+				  int queue_type,
+				  int pipe, int queue,
+				  unsigned int db);
 
 /* Read user wptr from a specified user address space with page fault
  * disabled. The memory must be pinned and mapped to the hardware when
@@ -326,9 +336,9 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
 int amdgpu_amdkfd_gpuvm_dmaunmap_mem(struct kgd_mem *mem, void *drm_priv);
 int amdgpu_amdkfd_gpuvm_sync_memory(
 		struct amdgpu_device *adev, struct kgd_mem *mem, bool intr);
-int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,
-					     void **kptr, uint64_t *size);
-void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem);
+int amdgpu_amdkfd_gpuvm_map_bo_to_kernel(struct kgd_mem *mem, void **kptr,
+					 u64 *size, u32 domain);
+void amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(struct kgd_mem *mem);
 
 int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo, struct amdgpu_bo **bo_gart);
 
@@ -446,6 +456,9 @@ bool kgd2kfd_vmfault_fast_path(struct amdgpu_device *adev, struct amdgpu_iv_entr
 			       bool retry_fault);
 void kgd2kfd_lock_kfd(void);
 void kgd2kfd_teardown_processes(struct amdgpu_device *adev);
+int kgd2kfd_reset_mes_queue(struct kfd_dev *kfd, uint32_t node_id,
+			    int queue_type, int pipe, int queue,
+			    unsigned int db);
 
 #else
 static inline int kgd2kfd_init(void)
@@ -576,5 +589,12 @@ static inline void kgd2kfd_teardown_processes(struct amdgpu_device *adev)
 {
 }
 
+static inline int kgd2kfd_reset_mes_queue(struct kfd_dev *kfd, uint32_t node_id,
+					  int queue_type, int pipe, int queue,
+					  unsigned int db)
+{
+	return 0;
+}
+
 #endif
 #endif /* AMDGPU_AMDKFD_H_INCLUDED */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c
index 6ed399163547a..bc079b95fc529 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c
@@ -530,6 +530,66 @@ static uint32_t kgd_v9_4_3_ptl_ctrl(struct amdgpu_device *adev,
 			ptl_state, fmt1, fmt2);
 }
 
+static int kgd_gfx_v9_4_3_hqd_sdma_get_counter(struct amdgpu_device *adev,
+					void *mqd, uint32_t num_sdma_queues_per_eng,
+					uint64_t *val)
+{
+	struct v9_sdma_mqd *m = get_sdma_mqd(mqd);
+	uint32_t sdma_rlc_reg_offset = 0;
+	uint32_t sdma_rlc_rb_cntl;
+	uint32_t engine_id, queue_id;
+	uint32_t engines = adev->sdma.num_instances;
+	uint32_t sdma_rlcx_rb_base, sdma_rlcx_rb_base_hi;
+	bool found = false;
+
+	if (!m)
+		return -EINVAL;
+
+	if (((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
+	     amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4)) &&
+	    adev->gfx.mec_fw_version < 194) ||
+	    (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0) &&
+	    adev->gfx.mec_fw_version < 44)) {
+		pr_warn_once("MEC FW doesn't support SDMA counter!\n");
+		return -EOPNOTSUPP;
+	}
+
+	/* SDMA doesn't support over-subscription, there must be
+	 * a HQD associated with a MQD, so found must be true in
+	 * the finding loop.
+	 */
+	for (engine_id = 0; engine_id < engines && !found; engine_id++) {
+		for (queue_id = 0; queue_id < num_sdma_queues_per_eng; queue_id++) {
+			sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev,
+						engine_id, queue_id);
+			sdma_rlcx_rb_base = RREG32(sdma_rlc_reg_offset +
+						regSDMA_RLC0_RB_BASE);
+			sdma_rlcx_rb_base_hi = RREG32(sdma_rlc_reg_offset +
+						regSDMA_RLC0_RB_BASE_HI);
+
+			if (m->sdmax_rlcx_rb_base == sdma_rlcx_rb_base &&
+			    m->sdmax_rlcx_rb_base_hi == sdma_rlcx_rb_base_hi) {
+				found = true;
+				break;
+			}
+		}
+	}
+
+	sdma_rlc_rb_cntl = RREG32(sdma_rlc_reg_offset + regSDMA_RLC0_RB_CNTL);
+
+	/* Read sdma activity counter from utilization register
+	 * if hw queue is enabled, otherwise read from MQD.
+	 */
+	if (sdma_rlc_rb_cntl & SDMA_RLC0_RB_CNTL__RB_ENABLE_MASK)
+		*val = (uint64_t)RREG32(sdma_rlc_reg_offset + regSDMA_RLC0_UTILIZATION_HI) << 32 |
+			RREG32(sdma_rlc_reg_offset + regSDMA_RLC0_UTILIZATION_LO);
+	else
+		*val = (uint64_t)m->sdmax_rlcx_utilization_hi << 32 |
+			m->sdmax_rlcx_utilization_lo;
+
+	return 0;
+}
+
 const struct kfd2kgd_calls gc_9_4_3_kfd2kgd = {
 	.program_sh_mem_settings = kgd_gfx_v9_program_sh_mem_settings,
 	.set_pasid_vmid_mapping = kgd_gfx_v9_4_3_set_pasid_vmid_mapping,
@@ -566,5 +626,6 @@ const struct kfd2kgd_calls gc_9_4_3_kfd2kgd = {
 	.hqd_get_pq_addr = kgd_gfx_v9_hqd_get_pq_addr,
 	.hqd_reset = kgd_gfx_v9_hqd_reset,
 	.hqd_sdma_get_doorbell = kgd_gfx_v9_4_3_hqd_sdma_get_doorbell,
-	.ptl_ctrl = kgd_v9_4_3_ptl_ctrl
+	.ptl_ctrl = kgd_v9_4_3_ptl_ctrl,
+	.hqd_sdma_get_counter = kgd_gfx_v9_4_3_hqd_sdma_get_counter
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c
index 8aa068a4d3e3c..724beb96ed1aa 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c
@@ -139,7 +139,8 @@ static uint32_t get_sdma_rlc_reg_offset(struct amdgpu_device *adev,
 				regSDMA1_QUEUE0_RB_CNTL) - regSDMA0_QUEUE0_RB_CNTL;
 		break;
 	default:
-		BUG();
+		WARN(1, "Invalid SDMA engine id %d\n", engine_id);
+		break;
 	}
 
 	sdma_rlc_reg_offset = sdma_engine_reg_base
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12.c
index bf0bd7688ee4b..e11ba3e918411 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12.c
@@ -90,7 +90,8 @@ static uint32_t get_sdma_rlc_reg_offset(struct amdgpu_device *adev,
 				regSDMA1_QUEUE0_RB_CNTL) - regSDMA0_QUEUE0_RB_CNTL;
 		break;
 	default:
-		BUG();
+		WARN(1, "Invalid SDMA engine id %d\n", engine_id);
+		break;
 	}
 
 	sdma_rlc_reg_offset = sdma_engine_reg_base
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c
index bcb180f9d3ff8..38ca1aea33b2f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c
@@ -93,7 +93,8 @@ static uint32_t get_sdma_rlc_reg_offset(struct amdgpu_device *adev,
 				regSDMA1_SDMA_QUEUE0_RB_CNTL) - regSDMA0_SDMA_QUEUE0_RB_CNTL;
 		break;
 	default:
-		BUG();
+		WARN(1, "Invalid SDMA engine id %d\n", engine_id);
+		break;
 	}
 
 	sdma_rlc_reg_offset = sdma_engine_reg_base
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 35fe2c9746991..20831dbebc31d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -2271,11 +2271,14 @@ int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo, struct amdgpu_bo **bo
 	return ret;
 }
 
-/** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Map a GTT BO for kernel CPU access
+/** amdgpu_amdkfd_gpuvm_map_bo_to_kernel() - Map GTT or VRAM BO for kernel CPU access
  *
  * @mem: Buffer object to be mapped for CPU access
  * @kptr[out]: pointer in kernel CPU address space
  * @size[out]: size of the buffer
+ * @domain[IN]: domain for pinning (AMDGPU_GEM_DOMAIN_GTT, AMDGPU_GEM_DOMAIN_VRAM,
+ *              or their combination to let the driver choose). CPU visibility is
+ *              automatically enforced by amdgpu_bo_pin()
  *
  * Pins the BO and maps it for kernel CPU access. The eviction fence is removed
  * from the BO, since pinned BOs cannot be evicted. The bo must remain on the
@@ -2284,8 +2287,8 @@ int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo, struct amdgpu_bo **bo
  *
  * Return: 0 on success, error code on failure
  */
-int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,
-					     void **kptr, uint64_t *size)
+int amdgpu_amdkfd_gpuvm_map_bo_to_kernel(struct kgd_mem *mem, void **kptr,
+					 u64 *size, u32 domain)
 {
 	int ret;
 	struct amdgpu_bo *bo = mem->bo;
@@ -2295,6 +2298,11 @@ int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,
 		return -EINVAL;
 	}
 
+	if (!(domain & (AMDGPU_GEM_DOMAIN_GTT | AMDGPU_GEM_DOMAIN_VRAM))) {
+		pr_debug("Invalid domain 0x%x for kernel mapping\n", domain);
+		return -EINVAL;
+	}
+
 	mutex_lock(&mem->process_info->lock);
 
 	ret = amdgpu_bo_reserve(bo, true);
@@ -2303,7 +2311,7 @@ int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,
 		goto bo_reserve_failed;
 	}
 
-	ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
+	ret = amdgpu_bo_pin(bo, domain);
 	if (ret) {
 		pr_err("Failed to pin bo. ret %d\n", ret);
 		goto pin_failed;
@@ -2336,7 +2344,7 @@ int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,
 	return ret;
 }
 
-/** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Unmap a GTT BO for kernel CPU access
+/** amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel() - Unmap GTT or VRAM BO for kernel CPU access
  *
  * @mem: Buffer object to be unmapped for CPU access
  *
@@ -2344,7 +2352,7 @@ int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,
  * eviction fence, so this function should only be used for cleanup before the
  * BO is destroyed.
  */
-void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem)
+void amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(struct kgd_mem *mem)
 {
 	struct amdgpu_bo *bo = mem->bo;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
index acd22bff1882d..27c0dc8f6137a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
@@ -1923,7 +1923,7 @@ int amdgpu_atombios_init(struct amdgpu_device *adev)
 	atom_card_info->pll_read = cail_pll_read;
 	atom_card_info->pll_write = cail_pll_write;
 
-	adev->mode_info.atom_context = amdgpu_atom_parse(atom_card_info, adev->bios);
+	adev->mode_info.atom_context = amdgpu_atom_parse(atom_card_info, adev->bios, adev->bios_size);
 	if (!adev->mode_info.atom_context) {
 		amdgpu_atombios_fini(adev);
 		return -ENOMEM;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index 6860a3a4d4667..1e21e3444802b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -917,8 +917,6 @@ int amdgpu_atomfirmware_get_clock_info(struct amdgpu_device *adev)
 				   (crev != 6)) {
 				spll->reference_freq = le32_to_cpu(gfx_info->v22.rlc_gpu_timer_refclk);
 				ret = 0;
-			} else {
-				BUG();
 			}
 		}
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
index 3893e6fc2f037..e2a4644896ca8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
@@ -89,6 +89,15 @@ bool amdgpu_is_atpx_hybrid(void)
 	return amdgpu_atpx_priv.atpx.is_hybrid;
 }
 
+static bool amdgpu_atpx_buffer_validate(const union acpi_object *obj,
+					size_t min_size)
+{
+	return obj && obj->type == ACPI_TYPE_BUFFER &&
+	       obj->buffer.length >= sizeof(u16) &&
+	       obj->buffer.length >= *(u16 *)obj->buffer.pointer &&
+	       *(u16 *)obj->buffer.pointer >= min_size;
+}
+
 /**
  * amdgpu_atpx_call - call an ATPX method
  *
@@ -179,15 +188,15 @@ static int amdgpu_atpx_validate(struct amdgpu_atpx *atpx)
 		if (!info)
 			return -EIO;
 
-		memset(&output, 0, sizeof(output));
-
-		size = *(u16 *) info->buffer.pointer;
-		if (size < 10) {
-			pr_err("ATPX buffer is too small: %zu\n", size);
+		if (!amdgpu_atpx_buffer_validate(info, sizeof(output))) {
+			pr_err("Invalid ATPX GET_PX_PARAMETERS response\n");
 			kfree(info);
 			return -EINVAL;
 		}
-		size = min(sizeof(output), size);
+
+		memset(&output, 0, sizeof(output));
+
+		size = min(sizeof(output), (size_t)*(u16 *)info->buffer.pointer);
 
 		memcpy(&output, info->buffer.pointer, size);
 
@@ -258,15 +267,15 @@ static int amdgpu_atpx_verify_interface(struct amdgpu_atpx *atpx)
 	if (!info)
 		return -EIO;
 
-	memset(&output, 0, sizeof(output));
-
-	size = *(u16 *) info->buffer.pointer;
-	if (size < 8) {
-		pr_err("ATPX buffer is too small: %zu\n", size);
+	if (!amdgpu_atpx_buffer_validate(info, sizeof(output))) {
+		pr_err("Invalid ATPX VERIFY_INTERFACE response\n");
 		err = -EINVAL;
 		goto out;
 	}
-	size = min(sizeof(output), size);
+
+	memset(&output, 0, sizeof(output));
+
+	size = min(sizeof(output), (size_t)*(u16 *)info->buffer.pointer);
 
 	memcpy(&output, info->buffer.pointer, size);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
index aa039e148a5ee..08ed1eb695586 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
@@ -296,8 +296,14 @@ static int amdgpu_atrm_call(acpi_handle atrm_handle, uint8_t *bios,
 	}
 
 	obj = (union acpi_object *)buffer.pointer;
-	memcpy(bios+offset, obj->buffer.pointer, obj->buffer.length);
-	len = obj->buffer.length;
+	if (!obj || obj->type != ACPI_TYPE_BUFFER) {
+		DRM_ERROR("ATRM returned an invalid object\n");
+		kfree(buffer.pointer);
+		return -EINVAL;
+	}
+
+	len = min_t(size_t, obj->buffer.length, len);
+	memcpy(bios+offset, obj->buffer.pointer, len);
 	kfree(buffer.pointer);
 	return len;
 }
@@ -372,19 +378,59 @@ static bool amdgpu_read_disabled_bios(struct amdgpu_device *adev)
 }
 
 #ifdef CONFIG_ACPI
+/**
+ * amdgpu_acpi_vfct_match() - Check if a VFCT entry matches the device
+ * @adev: AMDGPU device
+ * @vhdr: VFCT image header to check
+ *
+ * VFCT entries contain the PCI bus number as recorded during BIOS POST.
+ * On systems where the kernel renumbers PCI buses (e.g. pci=realloc or
+ * resource conflicts), the runtime bus number may differ from the POST
+ * value.  Match by device identity (vendor + device + function) and use
+ * the bus number as a preference: exact bus match is preferred, but when
+ * the bus numbers disagree we accept the entry if the device identity
+ * matches.
+ *
+ * Returns: 0 on match, -ENODEV on no match
+ */
+static int amdgpu_acpi_vfct_match(struct amdgpu_device *adev,
+				  VFCT_IMAGE_HEADER *vhdr)
+{
+	/* Vendor and device IDs must always match */
+	if (vhdr->VendorID != adev->pdev->vendor ||
+	    vhdr->DeviceID != adev->pdev->device)
+		return -ENODEV;
+
+	if (vhdr->PCIDevice != PCI_SLOT(adev->pdev->devfn) ||
+	    vhdr->PCIFunction != PCI_FUNC(adev->pdev->devfn))
+		return -ENODEV;
+
+	/* Exact bus number match - preferred */
+	if (vhdr->PCIBus == adev->pdev->bus->number)
+		return 0;
+
+	/* Bus mismatch but device identity matches (PCI renumbering case) */
+	dev_notice(adev->dev,
+		   "VFCT bus number mismatch: table %u != runtime %u, matching by device identity (vendor 0x%04x device 0x%04x)\n",
+		   vhdr->PCIBus, adev->pdev->bus->number,
+		   adev->pdev->vendor, adev->pdev->device);
+	return 0;
+}
+
 static bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev)
 {
 	struct acpi_table_header *hdr;
 	acpi_size tbl_size;
 	UEFI_ACPI_VFCT *vfct;
 	unsigned int offset;
+	bool r = false;
 
 	if (!ACPI_SUCCESS(acpi_get_table("VFCT", 1, &hdr)))
 		return false;
 	tbl_size = hdr->length;
 	if (tbl_size < sizeof(UEFI_ACPI_VFCT)) {
 		dev_info(adev->dev, "ACPI VFCT table present but broken (too short #1),skipping\n");
-		return false;
+		goto out;
 	}
 
 	vfct = (UEFI_ACPI_VFCT *)hdr;
@@ -397,36 +443,36 @@ static bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev)
 		offset += sizeof(VFCT_IMAGE_HEADER);
 		if (offset > tbl_size) {
 			dev_info(adev->dev, "ACPI VFCT image header truncated,skipping\n");
-			return false;
+			goto out;
 		}
 
 		offset += vhdr->ImageLength;
 		if (offset > tbl_size) {
 			dev_info(adev->dev, "ACPI VFCT image truncated,skipping\n");
-			return false;
+			goto out;
 		}
 
 		if (vhdr->ImageLength &&
-		    vhdr->PCIBus == adev->pdev->bus->number &&
-		    vhdr->PCIDevice == PCI_SLOT(adev->pdev->devfn) &&
-		    vhdr->PCIFunction == PCI_FUNC(adev->pdev->devfn) &&
-		    vhdr->VendorID == adev->pdev->vendor &&
-		    vhdr->DeviceID == adev->pdev->device) {
+		    !amdgpu_acpi_vfct_match(adev, vhdr)) {
 			adev->bios = kmemdup(&vbios->VbiosContent,
 					     vhdr->ImageLength,
 					     GFP_KERNEL);
 
 			if (!check_atom_bios(adev, vhdr->ImageLength)) {
 				amdgpu_bios_release(adev);
-				return false;
+				goto out;
 			}
 			adev->bios_size = vhdr->ImageLength;
-			return true;
+			r = true;
+			goto out;
 		}
 	}
 
 	dev_info(adev->dev, "ACPI VFCT table present but broken (too short #2),skipping\n");
-	return false;
+
+out:
+	acpi_put_table(hdr);
+	return r;
 }
 #else
 static inline bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
index 43864df8af046..ce1d08f112a85 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
@@ -38,14 +38,6 @@
 #define AMDGPU_BO_LIST_NUM_BUCKETS	(AMDGPU_BO_LIST_MAX_PRIORITY + 1)
 #define AMDGPU_BO_LIST_MAX_ENTRIES	(128 * 1024)
 
-static void amdgpu_bo_list_free_rcu(struct rcu_head *rcu)
-{
-	struct amdgpu_bo_list *list = container_of(rcu, struct amdgpu_bo_list,
-						   rhead);
-	mutex_destroy(&list->bo_list_mutex);
-	kvfree(list);
-}
-
 static void amdgpu_bo_list_free(struct kref *ref)
 {
 	struct amdgpu_bo_list *list = container_of(ref, struct amdgpu_bo_list,
@@ -54,7 +46,8 @@ static void amdgpu_bo_list_free(struct kref *ref)
 
 	amdgpu_bo_list_for_each_entry(e, list)
 		amdgpu_bo_unref(&e->bo);
-	call_rcu(&list->rhead, amdgpu_bo_list_free_rcu);
+
+	kvfree(list);
 }
 
 static int amdgpu_bo_list_entry_cmp(const void *_a, const void *_b)
@@ -66,9 +59,9 @@ static int amdgpu_bo_list_entry_cmp(const void *_a, const void *_b)
 	return (int)a->priority - (int)b->priority;
 }
 
-int amdgpu_bo_list_create(struct amdgpu_device *adev, struct drm_file *filp,
-			  struct drm_amdgpu_bo_list_entry *info,
-			  size_t num_entries, struct amdgpu_bo_list **result)
+struct amdgpu_bo_list *
+amdgpu_bo_list_create(struct amdgpu_device *adev, struct drm_file *filp,
+		      struct drm_amdgpu_bo_list_entry *info, size_t num_entries)
 {
 	unsigned last_entry = 0, first_userptr = num_entries;
 	struct amdgpu_bo_list_entry *array;
@@ -79,7 +72,7 @@ int amdgpu_bo_list_create(struct amdgpu_device *adev, struct drm_file *filp,
 
 	list = kvzalloc_flex(*list, entries, num_entries);
 	if (!list)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
 	kref_init(&list->refcount);
 
@@ -134,9 +127,7 @@ int amdgpu_bo_list_create(struct amdgpu_device *adev, struct drm_file *filp,
 
 	trace_amdgpu_cs_bo_status(list->num_entries, total_size);
 
-	mutex_init(&list->bo_list_mutex);
-	*result = list;
-	return 0;
+	return list;
 
 error_free:
 	for (i = 0; i < last_entry; ++i)
@@ -144,150 +135,125 @@ int amdgpu_bo_list_create(struct amdgpu_device *adev, struct drm_file *filp,
 	for (i = first_userptr; i < num_entries; ++i)
 		amdgpu_bo_unref(&array[i].bo);
 	kvfree(list);
-	return r;
+	return ERR_PTR(r);
 
 }
 
-static void amdgpu_bo_list_destroy(struct amdgpu_fpriv *fpriv, int id)
+struct amdgpu_bo_list *amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, u32 id)
 {
 	struct amdgpu_bo_list *list;
 
-	mutex_lock(&fpriv->bo_list_lock);
-	list = idr_remove(&fpriv->bo_list_handles, id);
-	mutex_unlock(&fpriv->bo_list_lock);
+	xa_lock(&fpriv->bo_list_handles);
+	list = xa_load(&fpriv->bo_list_handles, id);
 	if (list)
-		kref_put(&list->refcount, amdgpu_bo_list_free);
-}
-
-int amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, int id,
-		       struct amdgpu_bo_list **result)
-{
-	rcu_read_lock();
-	*result = idr_find(&fpriv->bo_list_handles, id);
-
-	if (*result && kref_get_unless_zero(&(*result)->refcount)) {
-		rcu_read_unlock();
-		return 0;
-	}
+		kref_get(&list->refcount);
+	else
+		list = ERR_PTR(-ENOENT);
+	xa_unlock(&fpriv->bo_list_handles);
 
-	rcu_read_unlock();
-	*result = NULL;
-	return -ENOENT;
+	return list;
 }
 
 void amdgpu_bo_list_put(struct amdgpu_bo_list *list)
 {
-	kref_put(&list->refcount, amdgpu_bo_list_free);
+	if (list)
+		kref_put(&list->refcount, amdgpu_bo_list_free);
 }
 
-int amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in,
-				      struct drm_amdgpu_bo_list_entry **info_param)
+struct drm_amdgpu_bo_list_entry *
+amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in)
 {
-	const uint32_t info_size = sizeof(struct drm_amdgpu_bo_list_entry);
 	const void __user *uptr = u64_to_user_ptr(in->bo_info_ptr);
-	const uint32_t bo_info_size = in->bo_info_size;
 	const uint32_t bo_number = in->bo_number;
-	struct drm_amdgpu_bo_list_entry *info;
 
 	if (bo_number > AMDGPU_BO_LIST_MAX_ENTRIES)
-		return -EINVAL;
+		return ERR_PTR(-EINVAL);
 
-	/* copy the handle array from userspace to a kernel buffer */
-	if (likely(info_size == bo_info_size)) {
-		info = vmemdup_array_user(uptr, bo_number, info_size);
-		if (IS_ERR(info))
-			return PTR_ERR(info);
-	} else {
-		const uint32_t bytes = min(bo_info_size, info_size);
-		unsigned i;
-
-		info = kvmalloc_array(bo_number, info_size, GFP_KERNEL);
-		if (!info)
-			return -ENOMEM;
-
-		memset(info, 0, bo_number * info_size);
-		for (i = 0; i < bo_number; ++i, uptr += bo_info_size) {
-			if (copy_from_user(&info[i], uptr, bytes)) {
-				kvfree(info);
-				return -EFAULT;
-			}
-		}
-	}
+	if (in->bo_info_size != sizeof(struct drm_amdgpu_bo_list_entry))
+		return ERR_PTR(-EINVAL);
 
-	*info_param = info;
-	return 0;
+	return vmemdup_array_user(uptr, bo_number,
+				  sizeof(struct drm_amdgpu_bo_list_entry));
 }
 
 int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *filp)
 {
-	struct amdgpu_device *adev = drm_to_adev(dev);
 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
+	struct amdgpu_device *adev = drm_to_adev(dev);
+	struct amdgpu_bo_list *list, *prev, *curr;
 	union drm_amdgpu_bo_list *args = data;
 	uint32_t handle = args->in.list_handle;
-	struct drm_amdgpu_bo_list_entry *info = NULL;
-	struct amdgpu_bo_list *list, *old;
+	struct drm_amdgpu_bo_list_entry *info;
 	int r;
 
-	r = amdgpu_bo_create_list_entry_array(&args->in, &info);
-	if (r)
-		return r;
-
 	switch (args->in.operation) {
 	case AMDGPU_BO_LIST_OP_CREATE:
-		r = amdgpu_bo_list_create(adev, filp, info, args->in.bo_number,
-					  &list);
-		if (r)
-			goto error_free;
+	case AMDGPU_BO_LIST_OP_UPDATE:
+		info = amdgpu_bo_create_list_entry_array(&args->in);
+		if (IS_ERR(info))
+			return PTR_ERR(info);
 
-		mutex_lock(&fpriv->bo_list_lock);
-		r = idr_alloc(&fpriv->bo_list_handles, list, 1, 0, GFP_KERNEL);
-		mutex_unlock(&fpriv->bo_list_lock);
-		if (r < 0) {
-			goto error_put_list;
-		}
+		list = amdgpu_bo_list_create(adev, filp, info,
+					     args->in.bo_number);
+		kvfree(info);
+		if (IS_ERR(list))
+			return PTR_ERR(list);
 
-		handle = r;
 		break;
 
 	case AMDGPU_BO_LIST_OP_DESTROY:
-		amdgpu_bo_list_destroy(fpriv, handle);
+		list = xa_erase(&fpriv->bo_list_handles, handle);
+		amdgpu_bo_list_put(list);
 		handle = 0;
+
 		break;
 
-	case AMDGPU_BO_LIST_OP_UPDATE:
-		r = amdgpu_bo_list_create(adev, filp, info, args->in.bo_number,
-					  &list);
+	default:
+		return -EINVAL;
+	};
+
+	switch (args->in.operation) {
+	case AMDGPU_BO_LIST_OP_CREATE:
+		r = xa_alloc(&fpriv->bo_list_handles, &handle, list,
+			     xa_limit_32b, GFP_KERNEL);
 		if (r)
-			goto error_free;
+			goto error_put_list;
+
+		break;
 
-		mutex_lock(&fpriv->bo_list_lock);
-		old = idr_replace(&fpriv->bo_list_handles, list, handle);
-		mutex_unlock(&fpriv->bo_list_lock);
+	case AMDGPU_BO_LIST_OP_UPDATE:
+		curr = xa_load(&fpriv->bo_list_handles, handle);
+		if (!curr) {
+			r = -ENOENT;
+			goto error_put_list;
+		}
 
-		if (IS_ERR(old)) {
-			r = PTR_ERR(old);
+		prev = xa_cmpxchg(&fpriv->bo_list_handles, handle, curr, list,
+				  GFP_KERNEL);
+		if (xa_is_err(prev)) {
+			r = xa_err(prev);
+			goto error_put_list;
+		} else if (prev != curr) {
+			r = -ENOENT;
 			goto error_put_list;
 		}
 
-		amdgpu_bo_list_put(old);
+		amdgpu_bo_list_put(curr);
 		break;
 
+	case AMDGPU_BO_LIST_OP_DESTROY:
 	default:
-		r = -EINVAL;
-		goto error_free;
+		/* Handled above. */
+		break;
 	}
 
 	memset(args, 0, sizeof(*args));
 	args->out.list_handle = handle;
-	kvfree(info);
 
 	return 0;
 
 error_put_list:
 	amdgpu_bo_list_put(list);
-
-error_free:
-	kvfree(info);
 	return r;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h
index 2b5e7c46a39d4..bde9121508249 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h
@@ -43,7 +43,6 @@ struct amdgpu_bo_list_entry {
 };
 
 struct amdgpu_bo_list {
-	struct rcu_head rhead;
 	struct kref refcount;
 	struct amdgpu_bo *gds_obj;
 	struct amdgpu_bo *gws_obj;
@@ -51,24 +50,19 @@ struct amdgpu_bo_list {
 	unsigned first_userptr;
 	unsigned num_entries;
 
-	/* Protect access during command submission.
-	 */
-	struct mutex bo_list_mutex;
-
 	struct amdgpu_bo_list_entry entries[] __counted_by(num_entries);
 };
 
-int amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, int id,
-		       struct amdgpu_bo_list **result);
+struct amdgpu_bo_list *amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, u32 id);
 void amdgpu_bo_list_put(struct amdgpu_bo_list *list);
-int amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in,
-				      struct drm_amdgpu_bo_list_entry **info_param);
+struct drm_amdgpu_bo_list_entry *
+amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in);
 
-int amdgpu_bo_list_create(struct amdgpu_device *adev,
-				 struct drm_file *filp,
-				 struct drm_amdgpu_bo_list_entry *info,
-				 size_t num_entries,
-				 struct amdgpu_bo_list **list);
+struct amdgpu_bo_list *
+amdgpu_bo_list_create(struct amdgpu_device *adev,
+		      struct drm_file *filp,
+		      struct drm_amdgpu_bo_list_entry *info,
+		      size_t num_entries);
 
 #define amdgpu_bo_list_for_each_entry(e, list) \
 	for (e = list->entries; \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index babd23e5a27ee..d3531f0c90b45 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -76,10 +76,9 @@ static uint32_t amdgpu_cgs_read_ind_register(struct cgs_device *cgs_device,
 		DRM_ERROR("audio endpt register access not implemented.\n");
 		return 0;
 	default:
-		BUG();
+		WARN(1, "Invalid indirect register space");
+		return 0;
 	}
-	WARN(1, "Invalid indirect register space");
-	return 0;
 }
 
 static void amdgpu_cgs_write_ind_register(struct cgs_device *cgs_device,
@@ -104,9 +103,8 @@ static void amdgpu_cgs_write_ind_register(struct cgs_device *cgs_device,
 		DRM_ERROR("audio endpt register access not implemented.\n");
 		return;
 	default:
-		BUG();
+		WARN(1, "Invalid indirect register space");
 	}
-	WARN(1, "Invalid indirect register space");
 }
 
 static uint32_t fw_type_convert(struct cgs_device *cgs_device, uint32_t fw_type)
@@ -241,7 +239,7 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 		info->fw_version = amdgpu_get_firmware_version(cgs_device, type);
 		info->feature_version = (uint16_t)le32_to_cpu(header->ucode_feature_version);
 	} else {
-		char fw_name[30] = {0};
+		const char *fw_name = NULL;
 		int err = 0;
 		uint32_t ucode_size;
 		uint32_t ucode_start_address;
@@ -257,17 +255,17 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 					(adev->pdev->revision == 0x81) ||
 					(adev->pdev->device == 0x665f)) {
 					info->is_kicker = true;
-					strscpy(fw_name, "amdgpu/bonaire_k_smc.bin");
+					fw_name = "bonaire_k_smc.bin";
 				} else {
-					strscpy(fw_name, "amdgpu/bonaire_smc.bin");
+					fw_name = "bonaire_smc.bin";
 				}
 				break;
 			case CHIP_HAWAII:
 				if (adev->pdev->revision == 0x80) {
 					info->is_kicker = true;
-					strscpy(fw_name, "amdgpu/hawaii_k_smc.bin");
+					fw_name = "hawaii_k_smc.bin";
 				} else {
-					strscpy(fw_name, "amdgpu/hawaii_smc.bin");
+					fw_name = "hawaii_smc.bin";
 				}
 				break;
 			case CHIP_TOPAZ:
@@ -277,76 +275,76 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 				    ((adev->pdev->device == 0x6900) && (adev->pdev->revision == 0xD1)) ||
 				    ((adev->pdev->device == 0x6900) && (adev->pdev->revision == 0xD3))) {
 					info->is_kicker = true;
-					strscpy(fw_name, "amdgpu/topaz_k_smc.bin");
+					fw_name = "topaz_k_smc.bin";
 				} else
-					strscpy(fw_name, "amdgpu/topaz_smc.bin");
+					fw_name = "topaz_smc.bin";
 				break;
 			case CHIP_TONGA:
 				if (((adev->pdev->device == 0x6939) && (adev->pdev->revision == 0xf1)) ||
 				    ((adev->pdev->device == 0x6938) && (adev->pdev->revision == 0xf1))) {
 					info->is_kicker = true;
-					strscpy(fw_name, "amdgpu/tonga_k_smc.bin");
+					fw_name = "tonga_k_smc.bin";
 				} else
-					strscpy(fw_name, "amdgpu/tonga_smc.bin");
+					fw_name = "tonga_smc.bin";
 				break;
 			case CHIP_FIJI:
-				strscpy(fw_name, "amdgpu/fiji_smc.bin");
+				fw_name = "fiji_smc.bin";
 				break;
 			case CHIP_POLARIS11:
 				if (type == CGS_UCODE_ID_SMU) {
 					if (ASICID_IS_P21(adev->pdev->device, adev->pdev->revision)) {
 						info->is_kicker = true;
-						strscpy(fw_name, "amdgpu/polaris11_k_smc.bin");
+						fw_name = "polaris11_k_smc.bin";
 					} else if (ASICID_IS_P31(adev->pdev->device, adev->pdev->revision)) {
 						info->is_kicker = true;
-						strscpy(fw_name, "amdgpu/polaris11_k2_smc.bin");
+						fw_name = "polaris11_k2_smc.bin";
 					} else {
-						strscpy(fw_name, "amdgpu/polaris11_smc.bin");
+						fw_name = "polaris11_smc.bin";
 					}
 				} else if (type == CGS_UCODE_ID_SMU_SK) {
-					strscpy(fw_name, "amdgpu/polaris11_smc_sk.bin");
+					fw_name = "polaris11_smc_sk.bin";
 				}
 				break;
 			case CHIP_POLARIS10:
 				if (type == CGS_UCODE_ID_SMU) {
 					if (ASICID_IS_P20(adev->pdev->device, adev->pdev->revision)) {
 						info->is_kicker = true;
-						strscpy(fw_name, "amdgpu/polaris10_k_smc.bin");
+						fw_name = "polaris10_k_smc.bin";
 					} else if (ASICID_IS_P30(adev->pdev->device, adev->pdev->revision)) {
 						info->is_kicker = true;
-						strscpy(fw_name, "amdgpu/polaris10_k2_smc.bin");
+						fw_name = "polaris10_k2_smc.bin";
 					} else {
-						strscpy(fw_name, "amdgpu/polaris10_smc.bin");
+						fw_name = "polaris10_smc.bin";
 					}
 				} else if (type == CGS_UCODE_ID_SMU_SK) {
-					strscpy(fw_name, "amdgpu/polaris10_smc_sk.bin");
+					fw_name = "polaris10_smc_sk.bin";
 				}
 				break;
 			case CHIP_POLARIS12:
 				if (ASICID_IS_P23(adev->pdev->device, adev->pdev->revision)) {
 					info->is_kicker = true;
-					strscpy(fw_name, "amdgpu/polaris12_k_smc.bin");
+					fw_name = "polaris12_k_smc.bin";
 				} else {
-					strscpy(fw_name, "amdgpu/polaris12_smc.bin");
+					fw_name = "polaris12_smc.bin";
 				}
 				break;
 			case CHIP_VEGAM:
-				strscpy(fw_name, "amdgpu/vegam_smc.bin");
+				fw_name = "vegam_smc.bin";
 				break;
 			case CHIP_VEGA10:
 				if ((adev->pdev->device == 0x687f) &&
 					((adev->pdev->revision == 0xc0) ||
 					(adev->pdev->revision == 0xc1) ||
 					(adev->pdev->revision == 0xc3)))
-					strscpy(fw_name, "amdgpu/vega10_acg_smc.bin");
+					fw_name = "vega10_acg_smc.bin";
 				else
-					strscpy(fw_name, "amdgpu/vega10_smc.bin");
+					fw_name = "vega10_smc.bin";
 				break;
 			case CHIP_VEGA12:
-				strscpy(fw_name, "amdgpu/vega12_smc.bin");
+				fw_name = "vega12_smc.bin";
 				break;
 			case CHIP_VEGA20:
-				strscpy(fw_name, "amdgpu/vega20_smc.bin");
+				fw_name = "vega20_smc.bin";
 				break;
 			default:
 				drm_err(adev_to_drm(adev), "SMC firmware not supported\n");
@@ -355,10 +353,8 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 
 			err = amdgpu_ucode_request(adev, &adev->pm.fw,
 						   AMDGPU_UCODE_REQUIRED,
-						   "%s", fw_name);
+						   "amdgpu/%s", fw_name);
 			if (err) {
-				drm_err(adev_to_drm(adev),
-					"Failed to load firmware \"%s\"\n", fw_name);
 				amdgpu_ucode_release(&adev->pm.fw);
 				return err;
 			}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c
index d5e59c24d907d..0af8b7be326ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c
@@ -289,40 +289,6 @@ struct cper_hdr *amdgpu_cper_alloc_entry(struct amdgpu_device *adev,
 	return hdr;
 }
 
-int amdgpu_cper_generate_ue_record(struct amdgpu_device *adev,
-				   struct aca_bank *bank)
-{
-	struct cper_hdr *fatal = NULL;
-	struct cper_sec_crashdump_reg_data reg_data = { 0 };
-	struct amdgpu_ring *ring = &adev->cper.ring_buf;
-	int ret;
-
-	fatal = amdgpu_cper_alloc_entry(adev, AMDGPU_CPER_TYPE_FATAL, 1);
-	if (!fatal) {
-		dev_err(adev->dev, "fail to alloc cper entry for ue record\n");
-		return -ENOMEM;
-	}
-
-	reg_data.status_lo = lower_32_bits(bank->regs[ACA_REG_IDX_STATUS]);
-	reg_data.status_hi = upper_32_bits(bank->regs[ACA_REG_IDX_STATUS]);
-	reg_data.addr_lo   = lower_32_bits(bank->regs[ACA_REG_IDX_ADDR]);
-	reg_data.addr_hi   = upper_32_bits(bank->regs[ACA_REG_IDX_ADDR]);
-	reg_data.ipid_lo   = lower_32_bits(bank->regs[ACA_REG_IDX_IPID]);
-	reg_data.ipid_hi   = upper_32_bits(bank->regs[ACA_REG_IDX_IPID]);
-	reg_data.synd_lo   = lower_32_bits(bank->regs[ACA_REG_IDX_SYND]);
-	reg_data.synd_hi   = upper_32_bits(bank->regs[ACA_REG_IDX_SYND]);
-
-	amdgpu_cper_entry_fill_hdr(adev, fatal, AMDGPU_CPER_TYPE_FATAL, CPER_SEV_FATAL_UNCORRECTED);
-	ret = amdgpu_cper_entry_fill_fatal_section(adev, fatal, 0, reg_data);
-	if (ret)
-		return ret;
-
-	amdgpu_cper_ring_write(ring, fatal, fatal->record_length);
-	kfree(fatal);
-
-	return 0;
-}
-
 int amdgpu_cper_generate_bp_threshold_record(struct amdgpu_device *adev)
 {
 	struct cper_hdr *bp_threshold = NULL;
@@ -348,83 +314,6 @@ int amdgpu_cper_generate_bp_threshold_record(struct amdgpu_device *adev)
 	return 0;
 }
 
-static enum cper_error_severity amdgpu_aca_err_type_to_cper_sev(struct amdgpu_device *adev,
-								enum aca_error_type aca_err_type)
-{
-	switch (aca_err_type) {
-	case ACA_ERROR_TYPE_UE:
-		return CPER_SEV_FATAL_UNCORRECTED;
-	case ACA_ERROR_TYPE_CE:
-		return CPER_SEV_NON_FATAL_CORRECTED;
-	case ACA_ERROR_TYPE_DEFERRED:
-		return CPER_SEV_NON_FATAL_UNCORRECTED;
-	default:
-		dev_err(adev->dev, "Unknown ACA error type!\n");
-		return CPER_SEV_FATAL_UNCORRECTED;
-	}
-}
-
-int amdgpu_cper_generate_ce_records(struct amdgpu_device *adev,
-				    struct aca_banks *banks,
-				    uint16_t bank_count)
-{
-	struct cper_hdr *corrected = NULL;
-	enum cper_error_severity sev = CPER_SEV_NON_FATAL_CORRECTED;
-	struct amdgpu_ring *ring = &adev->cper.ring_buf;
-	uint32_t reg_data[CPER_ACA_REG_COUNT] = { 0 };
-	struct aca_bank_node *node;
-	struct aca_bank *bank;
-	uint32_t i = 0;
-	int ret;
-
-	corrected = amdgpu_cper_alloc_entry(adev, AMDGPU_CPER_TYPE_RUNTIME, bank_count);
-	if (!corrected) {
-		dev_err(adev->dev, "fail to allocate cper entry for ce records\n");
-		return -ENOMEM;
-	}
-
-	/* Raise severity if any DE is detected in the ACA bank list */
-	list_for_each_entry(node, &banks->list, node) {
-		bank = &node->bank;
-		if (bank->aca_err_type == ACA_ERROR_TYPE_DEFERRED) {
-			sev = CPER_SEV_NON_FATAL_UNCORRECTED;
-			break;
-		}
-	}
-
-	amdgpu_cper_entry_fill_hdr(adev, corrected, AMDGPU_CPER_TYPE_RUNTIME, sev);
-
-	/* Combine CE and DE in cper record */
-	list_for_each_entry(node, &banks->list, node) {
-		bank = &node->bank;
-		reg_data[CPER_ACA_REG_CTL_LO]    = lower_32_bits(bank->regs[ACA_REG_IDX_CTL]);
-		reg_data[CPER_ACA_REG_CTL_HI]    = upper_32_bits(bank->regs[ACA_REG_IDX_CTL]);
-		reg_data[CPER_ACA_REG_STATUS_LO] = lower_32_bits(bank->regs[ACA_REG_IDX_STATUS]);
-		reg_data[CPER_ACA_REG_STATUS_HI] = upper_32_bits(bank->regs[ACA_REG_IDX_STATUS]);
-		reg_data[CPER_ACA_REG_ADDR_LO]   = lower_32_bits(bank->regs[ACA_REG_IDX_ADDR]);
-		reg_data[CPER_ACA_REG_ADDR_HI]   = upper_32_bits(bank->regs[ACA_REG_IDX_ADDR]);
-		reg_data[CPER_ACA_REG_MISC0_LO]  = lower_32_bits(bank->regs[ACA_REG_IDX_MISC0]);
-		reg_data[CPER_ACA_REG_MISC0_HI]  = upper_32_bits(bank->regs[ACA_REG_IDX_MISC0]);
-		reg_data[CPER_ACA_REG_CONFIG_LO] = lower_32_bits(bank->regs[ACA_REG_IDX_CONFIG]);
-		reg_data[CPER_ACA_REG_CONFIG_HI] = upper_32_bits(bank->regs[ACA_REG_IDX_CONFIG]);
-		reg_data[CPER_ACA_REG_IPID_LO]   = lower_32_bits(bank->regs[ACA_REG_IDX_IPID]);
-		reg_data[CPER_ACA_REG_IPID_HI]   = upper_32_bits(bank->regs[ACA_REG_IDX_IPID]);
-		reg_data[CPER_ACA_REG_SYND_LO]   = lower_32_bits(bank->regs[ACA_REG_IDX_SYND]);
-		reg_data[CPER_ACA_REG_SYND_HI]   = upper_32_bits(bank->regs[ACA_REG_IDX_SYND]);
-
-		ret = amdgpu_cper_entry_fill_runtime_section(adev, corrected, i++,
-				amdgpu_aca_err_type_to_cper_sev(adev, bank->aca_err_type),
-				reg_data, CPER_ACA_REG_COUNT);
-		if (ret)
-			return ret;
-	}
-
-	amdgpu_cper_ring_write(ring, corrected, corrected->record_length);
-	kfree(corrected);
-
-	return 0;
-}
-
 static bool amdgpu_cper_is_hdr(struct amdgpu_ring *ring, u64 pos)
 {
 	char signature[CPER_SIGNATURE_SZ];
@@ -592,8 +481,7 @@ int amdgpu_cper_init(struct amdgpu_device *adev)
 
 	if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_cper_en(adev))
 		return 0;
-	else if (!amdgpu_sriov_vf(adev) && !amdgpu_uniras_enabled(adev) &&
-		!amdgpu_aca_is_enabled(adev))
+	else if (!amdgpu_sriov_vf(adev) && !amdgpu_uniras_enabled(adev))
 		return 0;
 
 	r = amdgpu_cper_ring_init(adev);
@@ -610,9 +498,28 @@ int amdgpu_cper_init(struct amdgpu_device *adev)
 	return 0;
 }
 
+int amdgpu_cper_deferred_init(struct amdgpu_device *adev)
+{
+	int r;
+
+	if (adev->cper.enabled)
+		return 0;
+
+	r = amdgpu_cper_init(adev);
+	if (r || !adev->cper.enabled)
+		return r;
+
+#if defined(CONFIG_DEBUG_FS)
+	if (adev_to_drm(adev)->primary->debugfs_root)
+		amdgpu_debugfs_ring_init(adev, &adev->cper.ring_buf);
+#endif
+
+	return 0;
+}
+
 int amdgpu_cper_fini(struct amdgpu_device *adev)
 {
-	if (!amdgpu_aca_is_enabled(adev) && !amdgpu_sriov_ras_cper_en(adev))
+	if (amdgpu_sriov_vf(adev))
 		return 0;
 
 	adev->cper.enabled = false;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.h
index 353421807387e..eea30be91b47b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.h
@@ -26,7 +26,6 @@
 #define __AMDGPU_CPER_H__
 
 #include "amd_cper.h"
-#include "amdgpu_aca.h"
 
 #define CPER_MAX_ALLOWED_COUNT		0x1000
 #define CPER_MAX_RING_SIZE		0X100000
@@ -88,18 +87,12 @@ int amdgpu_cper_entry_fill_bad_page_threshold_section(struct amdgpu_device *adev
 struct cper_hdr *amdgpu_cper_alloc_entry(struct amdgpu_device *adev,
 					 enum amdgpu_cper_type type,
 					 uint16_t section_count);
-/* UE must be encoded into separated cper entries, 1 UE 1 cper */
-int amdgpu_cper_generate_ue_record(struct amdgpu_device *adev,
-				   struct aca_bank *bank);
-/* CEs and DEs are combined into 1 cper entry */
-int amdgpu_cper_generate_ce_records(struct amdgpu_device *adev,
-				    struct aca_banks *banks,
-				    uint16_t bank_count);
 /* Bad page threshold is encoded into separated cper entry */
 int amdgpu_cper_generate_bp_threshold_record(struct amdgpu_device *adev);
 void amdgpu_cper_ring_write(struct amdgpu_ring *ring,
 			void *src, int count);
 int amdgpu_cper_init(struct amdgpu_device *adev);
+int amdgpu_cper_deferred_init(struct amdgpu_device *adev);
 int amdgpu_cper_fini(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index e714cee2997aa..e8eaa3908ca0c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -140,24 +140,19 @@ static int amdgpu_cs_p1_bo_handles(struct amdgpu_cs_parser *p,
 				   struct drm_amdgpu_bo_list_in *data)
 {
 	struct drm_amdgpu_bo_list_entry *info;
-	int r;
+	struct amdgpu_bo_list *list;
 
-	r = amdgpu_bo_create_list_entry_array(data, &info);
-	if (r)
-		return r;
-
-	r = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number,
-				  &p->bo_list);
-	if (r)
-		goto error_free;
+	info = amdgpu_bo_create_list_entry_array(data);
+	if (IS_ERR(info))
+		return PTR_ERR(info);
 
+	list = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number);
 	kvfree(info);
-	return 0;
+	if (IS_ERR(list))
+		return PTR_ERR(list);
 
-error_free:
-	kvfree(info);
-
-	return r;
+	p->bo_list = list;
+	return 0;
 }
 
 /* Copy the data from userspace and go over it the first time */
@@ -272,8 +267,8 @@ static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p,
 
 	for (i = 0; i < p->gang_size; ++i) {
 		ret = amdgpu_job_alloc(p->adev, vm, p->entities[i], vm,
-				       num_ibs[i], &p->jobs[i],
-				       p->filp->client_id);
+				       num_ibs[i], p->filp->client_id,
+				       GFP_KERNEL, &p->jobs[i]);
 		if (ret)
 			goto free_all_kdata;
 		switch (p->adev->enforce_isolation[fpriv->xcp_id]) {
@@ -846,6 +841,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 {
 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
 	struct ttm_operation_ctx ctx = { true, false };
+	struct amdgpu_bo_list *list = NULL;
 	struct amdgpu_vm *vm = &fpriv->vm;
 	struct amdgpu_bo_list_entry *e;
 	struct drm_gem_object *obj;
@@ -857,25 +853,24 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 		if (p->bo_list)
 			return -EINVAL;
 
-		r = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle,
-				       &p->bo_list);
-		if (r)
-			return r;
+		list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);
 	} else if (!p->bo_list) {
 		/* Create a empty bo_list when no handle is provided */
-		r = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0,
-					  &p->bo_list);
-		if (r)
-			return r;
+		list = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0);
 	}
 
-	mutex_lock(&p->bo_list->bo_list_mutex);
+	if (IS_ERR(list))
+		return PTR_ERR(list);
+	else if (list)
+		p->bo_list = list;
+	else
+		list = p->bo_list;
 
 	/* Get userptr backing pages. If pages are updated after registered
 	 * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do
 	 * amdgpu_ttm_backend_bind() to flush and invalidate new pages
 	 */
-	amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
+	amdgpu_bo_list_for_each_userptr_entry(e, list) {
 		bool userpage_invalidated = false;
 		struct amdgpu_bo *bo = e->bo;
 
@@ -905,7 +900,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 		if (unlikely(r))
 			goto out_free_user_pages;
 
-		amdgpu_bo_list_for_each_entry(e, p->bo_list) {
+		amdgpu_bo_list_for_each_entry(e, list) {
 			r = drm_exec_prepare_obj(&p->exec, &e->bo->tbo.base,
 						 TTM_NUM_MOVE_FENCES + p->gang_size);
 			drm_exec_retry_on_contention(&p->exec);
@@ -924,7 +919,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 		}
 	}
 
-	amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
+	amdgpu_bo_list_for_each_userptr_entry(e, list) {
 		struct mm_struct *usermm;
 
 		usermm = amdgpu_ttm_tt_get_usermm(e->bo->tbo.ttm);
@@ -977,17 +972,15 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 				     p->bytes_moved_vis);
 
 	for (i = 0; i < p->gang_size; ++i)
-		amdgpu_job_set_resources(p->jobs[i], p->bo_list->gds_obj,
-					 p->bo_list->gws_obj,
-					 p->bo_list->oa_obj);
+		amdgpu_job_set_resources(p->jobs[i], list->gds_obj,
+					 list->gws_obj, list->oa_obj);
 	return 0;
 
 out_free_user_pages:
-	amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
+	amdgpu_bo_list_for_each_userptr_entry(e, list) {
 		amdgpu_hmm_range_free(e->range);
 		e->range = NULL;
 	}
-	mutex_unlock(&p->bo_list->bo_list_mutex);
 	return r;
 }
 
@@ -1125,7 +1118,8 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
 
 	if (fpriv->csa_va) {
 		bo_va = fpriv->csa_va;
-		BUG_ON(!bo_va);
+		if (!bo_va)
+			return -ENOMEM;
 		r = amdgpu_vm_bo_update(adev, bo_va, false);
 		if (r)
 			return r;
@@ -1371,7 +1365,6 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
 	amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm);
 
 	mutex_unlock(&p->adev->notifier_lock);
-	mutex_unlock(&p->bo_list->bo_list_mutex);
 	return 0;
 }
 
@@ -1443,28 +1436,25 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 
 	r = amdgpu_cs_patch_jobs(&parser);
 	if (r)
-		goto error_backoff;
+		goto error_fini;
 
 	r = amdgpu_cs_vm_handling(&parser);
 	if (r)
-		goto error_backoff;
+		goto error_fini;
 
 	r = amdgpu_cs_sync_rings(&parser);
 	if (r)
-		goto error_backoff;
+		goto error_fini;
 
 	trace_amdgpu_cs_ibs(&parser);
 
 	r = amdgpu_cs_submit(&parser, data);
 	if (r)
-		goto error_backoff;
+		goto error_fini;
 
 	amdgpu_cs_parser_fini(&parser);
 	return 0;
 
-error_backoff:
-	mutex_unlock(&parser.bo_list->bo_list_mutex);
-
 error_fini:
 	amdgpu_cs_parser_fini(&parser);
 	return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index ce35b415093d6..d53259a5b82f6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -283,6 +283,8 @@ static ktime_t amdgpu_ctx_fini_entity(struct amdgpu_device *adev,
 	if (!entity)
 		return res;
 
+	drm_sched_entity_destroy(&entity->entity);
+
 	for (i = 0; i < amdgpu_sched_jobs; ++i) {
 		res = ktime_add(res, amdgpu_ctx_fence_time(entity->fences[i]));
 		dma_fence_put(entity->fences[i]);
@@ -294,32 +296,20 @@ static ktime_t amdgpu_ctx_fini_entity(struct amdgpu_device *adev,
 	return res;
 }
 
-static int amdgpu_ctx_get_stable_pstate(struct amdgpu_ctx *ctx,
-					u32 *stable_pstate)
+static u32 amdgpu_get_stable_pstate(struct amdgpu_device *adev)
 {
-	struct amdgpu_device *adev = ctx->mgr->adev;
-	enum amd_dpm_forced_level current_level;
-
-	current_level = amdgpu_dpm_get_performance_level(adev);
-
-	switch (current_level) {
+	switch (amdgpu_dpm_get_performance_level(adev)) {
 	case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
-		*stable_pstate = AMDGPU_CTX_STABLE_PSTATE_STANDARD;
-		break;
+		return AMDGPU_CTX_STABLE_PSTATE_STANDARD;
 	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK:
-		*stable_pstate = AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK;
-		break;
+		return AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK;
 	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK:
-		*stable_pstate = AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK;
-		break;
+		return AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK;
 	case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK:
-		*stable_pstate = AMDGPU_CTX_STABLE_PSTATE_PEAK;
-		break;
+		return AMDGPU_CTX_STABLE_PSTATE_PEAK;
 	default:
-		*stable_pstate = AMDGPU_CTX_STABLE_PSTATE_NONE;
-		break;
+		return AMDGPU_CTX_STABLE_PSTATE_NONE;
 	}
-	return 0;
 }
 
 static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority,
@@ -383,9 +373,9 @@ static int __amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx,
 	if (current_ctx && current_ctx != ctx)
 		return -EBUSY;
 
-	r = amdgpu_ctx_get_stable_pstate(ctx, &current_stable_pstate);
-	if (r || current_stable_pstate == stable_pstate)
-		return r;
+	current_stable_pstate = amdgpu_get_stable_pstate(adev);
+	if (current_stable_pstate == stable_pstate)
+		return 0;
 
 	r = amdgpu_dpm_force_performance_level(adev, level);
 	if (r)
@@ -416,7 +406,7 @@ static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx,
 	return r;
 }
 
-static void amdgpu_ctx_fini(struct kref *ref)
+void amdgpu_ctx_fini(struct kref *ref)
 {
 	struct amdgpu_ctx *ctx = container_of(ref, struct amdgpu_ctx, refcount);
 	struct amdgpu_ctx_mgr *mgr = ctx->mgr;
@@ -504,53 +494,26 @@ static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
 	if (!ctx)
 		return -ENOMEM;
 
-	mutex_lock(&mgr->lock);
-	r = idr_alloc(&mgr->ctx_handles, ctx, 1, AMDGPU_VM_MAX_NUM_CTX, GFP_KERNEL);
-	if (r < 0) {
-		mutex_unlock(&mgr->lock);
-		kfree(ctx);
-		return r;
-	}
-
-	*id = (uint32_t)r;
 	r = amdgpu_ctx_init(mgr, priority, filp, ctx);
 	if (r) {
-		idr_remove(&mgr->ctx_handles, *id);
-		*id = 0;
 		kfree(ctx);
+		return r;
 	}
-	mutex_unlock(&mgr->lock);
-	return r;
-}
-
-static void amdgpu_ctx_do_release(struct kref *ref)
-{
-	struct amdgpu_ctx *ctx;
-	u32 i, j;
 
-	ctx = container_of(ref, struct amdgpu_ctx, refcount);
-	for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
-		for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
-			if (!ctx->entities[i][j])
-				continue;
-
-			drm_sched_entity_destroy(&ctx->entities[i][j]->entity);
-		}
-	}
+	r = xa_alloc(&mgr->ctx_handles, id, ctx, xa_limit_32b, GFP_KERNEL);
+	if (r)
+		amdgpu_ctx_put(ctx);
 
-	amdgpu_ctx_fini(ref);
+	return r;
 }
 
 static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id)
 {
-	struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
 	struct amdgpu_ctx *ctx;
 
-	mutex_lock(&mgr->lock);
-	ctx = idr_remove(&mgr->ctx_handles, id);
-	if (ctx)
-		kref_put(&ctx->refcount, amdgpu_ctx_do_release);
-	mutex_unlock(&mgr->lock);
+	ctx = xa_erase(&fpriv->ctx_mgr.ctx_handles, id);
+	amdgpu_ctx_put(ctx);
+
 	return ctx ? 0 : -EINVAL;
 }
 
@@ -559,19 +522,11 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev,
 			    union drm_amdgpu_ctx_out *out)
 {
 	struct amdgpu_ctx *ctx;
-	struct amdgpu_ctx_mgr *mgr;
 	unsigned reset_counter;
 
-	if (!fpriv)
-		return -EINVAL;
-
-	mgr = &fpriv->ctx_mgr;
-	mutex_lock(&mgr->lock);
-	ctx = idr_find(&mgr->ctx_handles, id);
-	if (!ctx) {
-		mutex_unlock(&mgr->lock);
+	ctx = amdgpu_ctx_get(fpriv, id);
+	if (!ctx)
 		return -EINVAL;
-	}
 
 	/* TODO: these two are always zero */
 	out->state.flags = 0x0;
@@ -586,7 +541,8 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev,
 		out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET;
 	ctx->reset_counter_query = reset_counter;
 
-	mutex_unlock(&mgr->lock);
+	amdgpu_ctx_put(ctx);
+
 	return 0;
 }
 
@@ -619,18 +575,10 @@ static int amdgpu_ctx_query2(struct amdgpu_device *adev,
 {
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 	struct amdgpu_ctx *ctx;
-	struct amdgpu_ctx_mgr *mgr;
 
-	if (!fpriv)
-		return -EINVAL;
-
-	mgr = &fpriv->ctx_mgr;
-	mutex_lock(&mgr->lock);
-	ctx = idr_find(&mgr->ctx_handles, id);
-	if (!ctx) {
-		mutex_unlock(&mgr->lock);
+	ctx = amdgpu_ctx_get(fpriv, id);
+	if (!ctx)
 		return -EINVAL;
-	}
 
 	out->state.flags = 0x0;
 	out->state.hangs = 0x0;
@@ -671,7 +619,8 @@ static int amdgpu_ctx_query2(struct amdgpu_device *adev,
 				      msecs_to_jiffies(AMDGPU_RAS_COUNTE_DELAY_MS));
 	}
 
-	mutex_unlock(&mgr->lock);
+	amdgpu_ctx_put(ctx);
+
 	return 0;
 }
 
@@ -680,26 +629,26 @@ static int amdgpu_ctx_stable_pstate(struct amdgpu_device *adev,
 				    bool set, u32 *stable_pstate)
 {
 	struct amdgpu_ctx *ctx;
-	struct amdgpu_ctx_mgr *mgr;
-	int r;
+	int r = 0;
 
-	if (!fpriv)
+	ctx = amdgpu_ctx_get(fpriv, id);
+	if (!ctx)
 		return -EINVAL;
 
-	mgr = &fpriv->ctx_mgr;
-	mutex_lock(&mgr->lock);
-	ctx = idr_find(&mgr->ctx_handles, id);
-	if (!ctx) {
-		mutex_unlock(&mgr->lock);
-		return -EINVAL;
-	}
+	/*
+	 * The get path is odd in this uapi - it will check whether the context
+	 * id exist, but otherwise does nothing with it. In other words, the
+	 * uapi has historically been implemented as being able to query the
+	 * global device state, as long as the caller supplies a random valid
+	 * context id.
+	 */
 
 	if (set)
 		r = amdgpu_ctx_set_stable_pstate(ctx, *stable_pstate);
 	else
-		r = amdgpu_ctx_get_stable_pstate(ctx, stable_pstate);
+		*stable_pstate = amdgpu_get_stable_pstate(adev);
 
-	mutex_unlock(&mgr->lock);
+	amdgpu_ctx_put(ctx);
 	return r;
 }
 
@@ -778,23 +727,14 @@ struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id)
 
 	mgr = &fpriv->ctx_mgr;
 
-	mutex_lock(&mgr->lock);
-	ctx = idr_find(&mgr->ctx_handles, id);
+	xa_lock(&mgr->ctx_handles);
+	ctx = xa_load(&mgr->ctx_handles, id);
 	if (ctx)
 		kref_get(&ctx->refcount);
-	mutex_unlock(&mgr->lock);
+	xa_unlock(&mgr->ctx_handles);
 	return ctx;
 }
 
-int amdgpu_ctx_put(struct amdgpu_ctx *ctx)
-{
-	if (ctx == NULL)
-		return -EINVAL;
-
-	kref_put(&ctx->refcount, amdgpu_ctx_do_release);
-	return 0;
-}
-
 uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx,
 			      struct drm_sched_entity *entity,
 			      struct dma_fence *fence)
@@ -928,8 +868,7 @@ void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr,
 	unsigned int i;
 
 	mgr->adev = adev;
-	mutex_init(&mgr->lock);
-	idr_init_base(&mgr->ctx_handles, 1);
+	xa_init_flags(&mgr->ctx_handles, XA_FLAGS_ALLOC1);
 
 	for (i = 0; i < AMDGPU_HW_IP_NUM; ++i)
 		atomic64_set(&mgr->time_spend[i], 0);
@@ -938,13 +877,13 @@ void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr,
 long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout)
 {
 	struct amdgpu_ctx *ctx;
-	struct idr *idp;
-	uint32_t id, i, j;
+	unsigned long id;
+	int i, j;
 
-	idp = &mgr->ctx_handles;
-
-	mutex_lock(&mgr->lock);
-	idr_for_each_entry(idp, ctx, id) {
+	xa_lock(&mgr->ctx_handles);
+	xa_for_each(&mgr->ctx_handles, id, ctx) {
+		kref_get(&ctx->refcount);
+		xa_unlock(&mgr->ctx_handles);
 		for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
 			for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
 				struct drm_sched_entity *entity;
@@ -956,45 +895,21 @@ long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout)
 				timeout = drm_sched_entity_flush(entity, timeout);
 			}
 		}
+		amdgpu_ctx_put(ctx);
+		xa_lock(&mgr->ctx_handles);
 	}
-	mutex_unlock(&mgr->lock);
+	xa_unlock(&mgr->ctx_handles);
 	return timeout;
 }
 
-static void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr)
+void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr)
 {
 	struct amdgpu_ctx *ctx;
-	struct idr *idp;
-	uint32_t id, i, j;
-
-	idp = &mgr->ctx_handles;
-
-	idr_for_each_entry(idp, ctx, id) {
-		if (kref_read(&ctx->refcount) != 1) {
-			drm_err(adev_to_drm(mgr->adev), "ctx %p is still alive\n", ctx);
-			continue;
-		}
+	unsigned long id;
 
-		for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
-			for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
-				struct drm_sched_entity *entity;
-
-				if (!ctx->entities[i][j])
-					continue;
-
-				entity = &ctx->entities[i][j]->entity;
-				drm_sched_entity_fini(entity);
-			}
-		}
-		kref_put(&ctx->refcount, amdgpu_ctx_fini);
-	}
-}
-
-void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr)
-{
-	amdgpu_ctx_mgr_entity_fini(mgr);
-	idr_destroy(&mgr->ctx_handles);
-	mutex_destroy(&mgr->lock);
+	xa_for_each(&mgr->ctx_handles, id, ctx)
+		amdgpu_ctx_put(ctx);
+	xa_destroy(&mgr->ctx_handles);
 }
 
 void amdgpu_ctx_mgr_usage(struct amdgpu_ctx_mgr *mgr,
@@ -1002,21 +917,21 @@ void amdgpu_ctx_mgr_usage(struct amdgpu_ctx_mgr *mgr,
 {
 	struct amdgpu_ctx *ctx;
 	unsigned int hw_ip, i;
-	uint32_t id;
+	unsigned long id;
 
 	/*
 	 * This is a little bit racy because it can be that a ctx or a fence are
 	 * destroyed just in the moment we try to account them. But that is ok
 	 * since exactly that case is explicitely allowed by the interface.
 	 */
-	mutex_lock(&mgr->lock);
 	for (hw_ip = 0; hw_ip < AMDGPU_HW_IP_NUM; ++hw_ip) {
 		uint64_t ns = atomic64_read(&mgr->time_spend[hw_ip]);
 
 		usage[hw_ip] = ns_to_ktime(ns);
 	}
 
-	idr_for_each_entry(&mgr->ctx_handles, ctx, id) {
+	xa_lock(&mgr->ctx_handles);
+	xa_for_each(&mgr->ctx_handles, id, ctx) {
 		for (hw_ip = 0; hw_ip < AMDGPU_HW_IP_NUM; ++hw_ip) {
 			for (i = 0; i < amdgpu_ctx_num_entities[hw_ip]; ++i) {
 				struct amdgpu_ctx_entity *centity;
@@ -1030,5 +945,5 @@ void amdgpu_ctx_mgr_usage(struct amdgpu_ctx_mgr *mgr,
 			}
 		}
 	}
-	mutex_unlock(&mgr->lock);
+	xa_unlock(&mgr->ctx_handles);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
index e444b2088d400..a4b89eca41693 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
@@ -25,6 +25,7 @@
 
 #include <linux/ktime.h>
 #include <linux/types.h>
+#include <linux/xarray.h>
 
 #include "amdgpu_ring.h"
 
@@ -60,16 +61,21 @@ struct amdgpu_ctx {
 
 struct amdgpu_ctx_mgr {
 	struct amdgpu_device	*adev;
-	struct mutex		lock;
-	/* protected by lock */
-	struct idr		ctx_handles;
+	struct xarray		ctx_handles;
 	atomic64_t		time_spend[AMDGPU_HW_IP_NUM];
 };
 
 extern const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM];
 
 struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id);
-int amdgpu_ctx_put(struct amdgpu_ctx *ctx);
+
+void amdgpu_ctx_fini(struct kref *kref);
+
+static inline void amdgpu_ctx_put(struct amdgpu_ctx *ctx)
+{
+	if (ctx)
+		kref_put(&ctx->refcount, amdgpu_ctx_fini);
+}
 
 int amdgpu_ctx_get_entity(struct amdgpu_ctx *ctx, u32 hw_ip, u32 instance,
 			  u32 ring, struct drm_sched_entity **entity);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
index 389bad7242731..deab64765388c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
@@ -26,6 +26,7 @@
 #include <linux/kthread.h>
 #include <linux/pci.h>
 #include <linux/uaccess.h>
+#include <linux/security.h>
 #include <linux/pm_runtime.h>
 
 #include "amdgpu.h"
@@ -1739,6 +1740,12 @@ int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
 	struct dentry *ent, *root = minor->debugfs_root;
 	unsigned int i;
 
+	if (security_locked_down(LOCKDOWN_PCI_ACCESS)) {
+		drm_info(adev_to_drm(adev),
+			 "amdgpu: HW debugfs nodes disabled (kernel lockdown)\n");
+		return 0;
+	}
+
 	for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
 		ent = debugfs_create_file(debugfs_regs_names[i],
 					  S_IFREG | 0400, root,
@@ -2171,6 +2178,8 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev)
 
 		if (!ring)
 			continue;
+		if (ring == &adev->cper.ring_buf && !adev->cper.enabled)
+			continue;
 
 		amdgpu_debugfs_ring_init(adev, ring);
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c
index e77db76b48b8d..39b2a4c0e011a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c
@@ -64,6 +64,7 @@ const char *hw_ip_names[MAX_HWIP] = {
 	[VCN1_HWIP]		= "VCN1",
 	[VCE_HWIP]		= "VCE",
 	[VPE_HWIP]		= "VPE",
+	[UMSCH_HWIP]		= "UMSCH",
 	[DF_HWIP]		= "DF",
 	[DCE_HWIP]		= "DCE",
 	[OSSSYS_HWIP]		= "OSSSYS",
@@ -234,6 +235,9 @@ amdgpu_devcoredump_print_ibs(struct drm_printer *p,
 			drm_printf(p, "\nIB #%d 0x%llx %d dw\n", i,
 				   coredump->ibs[i].gpu_addr,
 				   coredump->ibs[i].ib_size_dw);
+
+			for (int j = 0; j < coredump->ibs[i].ib_size_dw; j++)
+				drm_printf(p, "0xffffffff\n");
 		}
 		return;
 	}
@@ -355,10 +359,14 @@ amdgpu_devcoredump_format(char *buffer, size_t count, struct amdgpu_coredump_inf
 	drm_printf(&p, "kernel: %s\n", init_utsname()->release);
 	drm_printf(&p, "module: " KBUILD_MODNAME "\n");
 	drm_printf(&p, "time: %ptSp\n", &coredump->reset_time);
+	drm_printf(&p, "pasid: %u\n", coredump->pasid);
+	drm_printf(&p, "vmid: %u\n", coredump->vmid);
 
 	if (coredump->reset_task_info.task.pid)
-		drm_printf(&p, "process_name: %s PID: %d\n",
+		drm_printf(&p, "process_name: %s TGID: %d thread: %s PID: %d\n",
 			   coredump->reset_task_info.process_name,
+			   coredump->reset_task_info.tgid,
+			   coredump->reset_task_info.task.comm,
 			   coredump->reset_task_info.task.pid);
 
 	/* SOC Information */
@@ -562,6 +570,7 @@ void amdgpu_coredump(struct amdgpu_device *adev, bool skip_vram_check,
 			amdgpu_vm_put_task_info(ti);
 		}
 		coredump->pasid = job->pasid;
+		coredump->vmid = job->vmid;
 		coredump->num_ibs = job->num_ibs;
 		for (i = 0; i < job->num_ibs; ++i) {
 			coredump->ibs[i].gpu_addr = job->ibs[i].gpu_addr;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.h
index 2371e20fc68bb..63f27337c09ad 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.h
@@ -63,6 +63,7 @@ struct amdgpu_coredump_info {
 	char				*formatted;
 
 	unsigned int			pasid;
+	unsigned int			vmid;
 	int				num_ibs;
 	struct amdgpu_coredump_ib_info	ibs[] __counted_by(num_ibs);
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 53335ca96b1d6..472e96ae884e5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -72,8 +72,10 @@
 
 #include "amdgpu_xgmi.h"
 #include "amdgpu_ras.h"
+#include "amdgpu_wb.h"
 #include "amdgpu_ras_mgr.h"
 #include "amdgpu_pmu.h"
+#include "amdgpu_smu.h"
 #include "amdgpu_fru_eeprom.h"
 #include "amdgpu_reset.h"
 #include "amdgpu_virt.h"
@@ -1005,109 +1007,6 @@ int amdgpu_device_pci_reset(struct amdgpu_device *adev)
 	return pci_reset_function(adev->pdev);
 }
 
-/*
- * amdgpu_device_wb_*()
- * Writeback is the method by which the GPU updates special pages in memory
- * with the status of certain GPU events (fences, ring pointers,etc.).
- */
-
-/**
- * amdgpu_device_wb_fini - Disable Writeback and free memory
- *
- * @adev: amdgpu_device pointer
- *
- * Disables Writeback and frees the Writeback memory (all asics).
- * Used at driver shutdown.
- */
-static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
-{
-	if (adev->wb.wb_obj) {
-		amdgpu_bo_free_kernel(&adev->wb.wb_obj,
-				      &adev->wb.gpu_addr,
-				      (void **)&adev->wb.wb);
-		adev->wb.wb_obj = NULL;
-	}
-}
-
-/**
- * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
- *
- * @adev: amdgpu_device pointer
- *
- * Initializes writeback and allocates writeback memory (all asics).
- * Used at driver startup.
- * Returns 0 on success or an -error on failure.
- */
-static int amdgpu_device_wb_init(struct amdgpu_device *adev)
-{
-	int r;
-
-	if (adev->wb.wb_obj == NULL) {
-		/* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
-		r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
-					    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
-					    &adev->wb.wb_obj, &adev->wb.gpu_addr,
-					    (void **)&adev->wb.wb);
-		if (r) {
-			dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
-			return r;
-		}
-
-		adev->wb.num_wb = AMDGPU_MAX_WB;
-		memset(&adev->wb.used, 0, sizeof(adev->wb.used));
-
-		/* clear wb memory */
-		memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
-	}
-
-	return 0;
-}
-
-/**
- * amdgpu_device_wb_get - Allocate a wb entry
- *
- * @adev: amdgpu_device pointer
- * @wb: wb index
- *
- * Allocate a wb slot for use by the driver (all asics).
- * Returns 0 on success or -EINVAL on failure.
- */
-int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
-{
-	unsigned long flags, offset;
-
-	spin_lock_irqsave(&adev->wb.lock, flags);
-	offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
-	if (offset < adev->wb.num_wb) {
-		__set_bit(offset, adev->wb.used);
-		spin_unlock_irqrestore(&adev->wb.lock, flags);
-		*wb = offset << 3; /* convert to dw offset */
-		return 0;
-	} else {
-		spin_unlock_irqrestore(&adev->wb.lock, flags);
-		return -EINVAL;
-	}
-}
-
-/**
- * amdgpu_device_wb_free - Free a wb entry
- *
- * @adev: amdgpu_device pointer
- * @wb: wb index
- *
- * Free a wb slot allocated for use by the driver (all asics)
- */
-void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
-{
-	unsigned long flags;
-
-	wb >>= 3;
-	spin_lock_irqsave(&adev->wb.lock, flags);
-	if (wb < adev->wb.num_wb)
-		__clear_bit(wb, adev->wb.used);
-	spin_unlock_irqrestore(&adev->wb.lock, flags);
-}
-
 /**
  * amdgpu_device_resize_fb_bar - try to resize FB BAR
  *
@@ -1304,28 +1203,44 @@ bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev)
 	return amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0);
 }
 
-/*
- * Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids
- * don't support dynamic speed switching. Until we have confirmation from Intel
- * that a specific host supports it, it's safer that we keep it disabled for all.
- *
- * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/
- * https://gitlab.freedesktop.org/drm/amd/-/issues/2663
- */
-static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev)
-{
 #if IS_ENABLED(CONFIG_X86)
-	struct cpuinfo_x86 *c = &cpu_data(0);
+static const struct x86_cpu_id amdgpu_pcie_dynamic_switching_quirks[] = {
+	/*
+	 * Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids
+	 * don't support dynamic speed switching. Until we have confirmation from Intel
+	 * that a specific host supports it, it's safer that we keep it disabled for all.
+	 *
+	 * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/
+	 * https://gitlab.freedesktop.org/drm/amd/-/issues/2663
+	 */
+	X86_MATCH_VENDOR_FAM(INTEL, X86_FAMILY_ANY, NULL),
+	/*
+	 * AMD Ryzen Pinnacle Ridge (Zen+, family 0x17 model 0x08) CPUs don't
+	 * support PCIe dynamic speed switching.
+	 * https://gitlab.freedesktop.org/drm/amd/-/work_items/5436
+	 */
+	X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x17, 0x08, NULL),
+	{}
+};
 
+static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev)
+{
 	/* eGPU change speeds based on USB4 fabric conditions */
 	if (dev_is_removable(adev->dev))
 		return true;
 
-	if (c->x86_vendor == X86_VENDOR_INTEL)
+	/* Hosts have problems with dynamic speed switching */
+	if (x86_match_cpu(amdgpu_pcie_dynamic_switching_quirks))
 		return false;
-#endif
+
 	return true;
 }
+#else
+static inline bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev)
+{
+	return true;
+}
+#endif
 
 static bool amdgpu_device_aspm_support_quirk(struct amdgpu_device *adev)
 {
@@ -2131,6 +2046,8 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
 	adev->cg_flags &= amdgpu_cg_mask;
 	adev->pg_flags &= amdgpu_pg_mask;
 
+	amdgpu_smu_early_init(adev);
+
 	return 0;
 }
 
@@ -2371,10 +2288,10 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
 					r);
 				goto init_failed;
 			}
-			r = amdgpu_device_wb_init(adev);
+			r = amdgpu_wb_init(adev);
 			if (r) {
 				dev_err(adev->dev,
-					"amdgpu_device_wb_init failed %d\n", r);
+					"amdgpu_wb_init failed %d\n", r);
 				goto init_failed;
 			}
 			adev->ip_blocks[i].status.hw = true;
@@ -2906,7 +2823,7 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
 			amdgpu_ucode_free_bo(adev);
 			amdgpu_free_static_csa(&adev->virt.csa_obj);
-			amdgpu_device_wb_fini(adev);
+			amdgpu_wb_fini(adev);
 			amdgpu_device_mem_scratch_fini(adev);
 			amdgpu_ib_pool_fini(adev);
 			amdgpu_seq64_fini(adev);
@@ -3678,6 +3595,10 @@ static void amdgpu_device_sys_interface_fini(struct amdgpu_device *adev)
 		amdgpu_pm_sysfs_fini(adev);
 	if (adev->ucode_sysfs_en)
 		amdgpu_ucode_sysfs_fini(adev);
+
+	amdgpu_discovery_sysfs_fini(adev);
+	amdgpu_preempt_mgr_sysfs_fini(adev);
+
 	amdgpu_device_attr_sysfs_fini(adev);
 	amdgpu_fru_sysfs_fini(adev);
 
@@ -3774,6 +3695,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 
 	spin_lock_init(&adev->irq.lock);
 
+	amdgpu_early_init_rlc_reg_funcs(adev);
 	amdgpu_device_init_apu_flags(adev);
 
 	r = amdgpu_device_check_arguments(adev);
@@ -4209,6 +4131,7 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev)
 
 	if (adev->mman.initialized)
 		drain_workqueue(adev->mman.bdev.wq);
+
 	adev->shutdown = true;
 
 	unregister_pm_notifier(&adev->pm_nb);
@@ -4707,161 +4630,6 @@ int amdgpu_device_resume(struct drm_device *dev, bool notify_clients)
 	return 0;
 }
 
-/**
- * amdgpu_device_ip_check_soft_reset - did soft reset succeed
- *
- * @adev: amdgpu_device pointer
- *
- * The list of all the hardware IPs that make up the asic is walked and
- * the check_soft_reset callbacks are run.  check_soft_reset determines
- * if the asic is still hung or not.
- * Returns true if any of the IPs are still in a hung state, false if not.
- */
-static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
-{
-	int i;
-	bool asic_hang = false;
-
-	if (amdgpu_sriov_vf(adev))
-		return true;
-
-	if (amdgpu_asic_need_full_reset(adev))
-		return true;
-
-	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_blocks[i].status.valid)
-			continue;
-		if (adev->ip_blocks[i].version->funcs->check_soft_reset)
-			adev->ip_blocks[i].status.hang =
-				adev->ip_blocks[i].version->funcs->check_soft_reset(
-					&adev->ip_blocks[i]);
-		if (adev->ip_blocks[i].status.hang) {
-			dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
-			asic_hang = true;
-		}
-	}
-	return asic_hang;
-}
-
-/**
- * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
- *
- * @adev: amdgpu_device pointer
- *
- * The list of all the hardware IPs that make up the asic is walked and the
- * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
- * handles any IP specific hardware or software state changes that are
- * necessary for a soft reset to succeed.
- * Returns 0 on success, negative error code on failure.
- */
-static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
-{
-	int i, r = 0;
-
-	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_blocks[i].status.valid)
-			continue;
-		if (adev->ip_blocks[i].status.hang &&
-		    adev->ip_blocks[i].version->funcs->pre_soft_reset) {
-			r = adev->ip_blocks[i].version->funcs->pre_soft_reset(&adev->ip_blocks[i]);
-			if (r)
-				return r;
-		}
-	}
-
-	return 0;
-}
-
-/**
- * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
- *
- * @adev: amdgpu_device pointer
- *
- * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
- * reset is necessary to recover.
- * Returns true if a full asic reset is required, false if not.
- */
-static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
-{
-	int i;
-
-	if (amdgpu_asic_need_full_reset(adev))
-		return true;
-
-	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_blocks[i].status.valid)
-			continue;
-		if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
-		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
-		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
-		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
-		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
-			if (adev->ip_blocks[i].status.hang) {
-				dev_info(adev->dev, "Some block need full reset!\n");
-				return true;
-			}
-		}
-	}
-	return false;
-}
-
-/**
- * amdgpu_device_ip_soft_reset - do a soft reset
- *
- * @adev: amdgpu_device pointer
- *
- * The list of all the hardware IPs that make up the asic is walked and the
- * soft_reset callbacks are run if the block is hung.  soft_reset handles any
- * IP specific hardware or software state changes that are necessary to soft
- * reset the IP.
- * Returns 0 on success, negative error code on failure.
- */
-static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
-{
-	int i, r = 0;
-
-	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_blocks[i].status.valid)
-			continue;
-		if (adev->ip_blocks[i].status.hang &&
-		    adev->ip_blocks[i].version->funcs->soft_reset) {
-			r = adev->ip_blocks[i].version->funcs->soft_reset(&adev->ip_blocks[i]);
-			if (r)
-				return r;
-		}
-	}
-
-	return 0;
-}
-
-/**
- * amdgpu_device_ip_post_soft_reset - clean up from soft reset
- *
- * @adev: amdgpu_device pointer
- *
- * The list of all the hardware IPs that make up the asic is walked and the
- * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
- * handles any IP specific hardware or software state changes that are
- * necessary after the IP has been soft reset.
- * Returns 0 on success, negative error code on failure.
- */
-static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
-{
-	int i, r = 0;
-
-	for (i = 0; i < adev->num_ip_blocks; i++) {
-		if (!adev->ip_blocks[i].status.valid)
-			continue;
-		if (adev->ip_blocks[i].status.hang &&
-		    adev->ip_blocks[i].version->funcs->post_soft_reset)
-			r = adev->ip_blocks[i].version->funcs->post_soft_reset(&adev->ip_blocks[i]);
-		if (r)
-			return r;
-	}
-
-	return 0;
-}
-
 /**
  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
  *
@@ -5153,20 +4921,7 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
 
 	/* Don't suspend on bare metal if we are not going to HW reset the ASIC */
 	if (!amdgpu_sriov_vf(adev)) {
-
-		if (!need_full_reset)
-			need_full_reset = amdgpu_device_ip_need_full_reset(adev);
-
-		if (!need_full_reset && amdgpu_gpu_recovery &&
-		    amdgpu_device_ip_check_soft_reset(adev)) {
-			amdgpu_device_ip_pre_soft_reset(adev);
-			r = amdgpu_device_ip_soft_reset(adev);
-			amdgpu_device_ip_post_soft_reset(adev);
-			if (r || amdgpu_device_ip_check_soft_reset(adev)) {
-				dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
-				need_full_reset = true;
-			}
-		}
+		need_full_reset = true;
 
 		if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) {
 			dev_info(tmp_adev->dev, "Dumping IP State\n");
@@ -5619,8 +5374,7 @@ static void amdgpu_device_halt_activities(struct amdgpu_device *adev,
 		drm_client_dev_suspend(adev_to_drm(tmp_adev));
 
 		/* disable ras on ALL IPs */
-		if (!need_emergency_restart && !amdgpu_reset_in_dpc(adev) &&
-		    amdgpu_device_ip_need_full_reset(tmp_adev))
+		if (!need_emergency_restart && !amdgpu_reset_in_dpc(adev))
 			amdgpu_ras_suspend(tmp_adev);
 
 		amdgpu_userq_pre_reset(tmp_adev);
@@ -6892,7 +6646,7 @@ ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring)
 
 	if (unlikely(!ring->adev->debug_disable_soft_recovery) &&
 	    !amdgpu_sriov_vf(ring->adev) && ring->funcs->soft_recovery)
-		size |= AMDGPU_RESET_TYPE_SOFT_RESET;
+		size |= AMDGPU_RESET_TYPE_SOFT_RECOVERY;
 
 	return size;
 }
@@ -6908,8 +6662,8 @@ ssize_t amdgpu_show_reset_mask(char *buf, uint32_t supported_reset)
 
 	}
 
-	if (supported_reset & AMDGPU_RESET_TYPE_SOFT_RESET)
-		size += sysfs_emit_at(buf, size, "soft ");
+	if (supported_reset & AMDGPU_RESET_TYPE_SOFT_RECOVERY)
+		size += sysfs_emit_at(buf, size, "soft_recovery ");
 
 	if (supported_reset & AMDGPU_RESET_TYPE_PER_QUEUE)
 		size += sysfs_emit_at(buf, size, "queue ");
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
index 7b9bb998906d3..44e7d2e3e6df4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
@@ -22,6 +22,7 @@
  */
 
 #include <linux/firmware.h>
+#include <linux/kernfs.h>
 
 #include "amdgpu.h"
 #include "amdgpu_discovery.h"
@@ -148,6 +149,26 @@ MODULE_FIRMWARE("amdgpu/aldebaran_ip_discovery.bin");
 #define mmDRIVER_SCRATCH_1	0x95
 #define mmDRIVER_SCRATCH_2	0x96
 
+struct ip_discovery_top {
+	struct kobject kobj;
+	struct kset die_kset;
+	struct pci_dev *pdev;
+	struct amdgpu_device *adev;
+	uint8_t *discovery_bin;
+	uint32_t bin_size;
+	bool standalone_mode;
+};
+
+/* List to track early-initialized ip_discovery_top entries */
+struct early_ip_discovery {
+	struct list_head list;
+	struct pci_dev *pdev;
+	struct ip_discovery_top *ip_top;
+};
+
+static LIST_HEAD(early_ip_discovery_list);
+static DEFINE_MUTEX(early_ip_discovery_mutex);
+
 static const char *hw_id_names[HW_ID_MAX] = {
 	[MP1_HWID]		= "MP1",
 	[MP2_HWID]		= "MP2",
@@ -226,6 +247,7 @@ static const char *hw_id_names[HW_ID_MAX] = {
 	[XGBE_HWID]		= "XGBE",
 	[MP0_HWID]		= "MP0",
 	[VPE_HWID]		= "VPE",
+	[UMSCH_HWID]		= "UMSCH",
 	[ATU_HWID]		= "ATU",
 	[AIGC_HWID]		= "AIGC",
 };
@@ -258,6 +280,7 @@ static int hw_id_map[MAX_HWIP] = {
 	[DCI_HWIP]	= DCI_HWID,
 	[PCIE_HWIP]	= PCIE_HWID,
 	[VPE_HWIP]	= VPE_HWID,
+	[UMSCH_HWIP]	= UMSCH_HWID,
 	[ISP_HWIP]	= ISP_HWID,
 	[ATU_HWIP]	= ATU_HWID,
 };
@@ -542,25 +565,37 @@ static const char *amdgpu_discovery_get_fw_name(struct amdgpu_device *adev)
 	}
 }
 
-static int amdgpu_discovery_get_table_info(struct amdgpu_device *adev,
-					   struct table_info **info,
-					   uint16_t table_id)
+static struct table_info *
+amdgpu_discovery_get_table_info_from_bin(uint8_t *discovery_bin,
+					 uint16_t table_id)
 {
-	struct binary_header *bhdr =
-		(struct binary_header *)adev->discovery.bin;
+	struct binary_header *bhdr = (struct binary_header *)discovery_bin;
 	struct binary_header_v2 *bhdrv2;
 
 	switch (bhdr->version_major) {
 	case 2:
-		bhdrv2 = (struct binary_header_v2 *)adev->discovery.bin;
-		*info = &bhdrv2->table_list[table_id];
-		break;
+		bhdrv2 = (struct binary_header_v2 *)discovery_bin;
+		return &bhdrv2->table_list[table_id];
 	case 1:
 	case 0:
-		*info = &bhdr->table_list[table_id];
-		break;
+		return &bhdr->table_list[table_id];
 	default:
-		dev_err(adev->dev, "Invalid ip discovery table version %d\n",bhdr->version_major);
+		return NULL;
+	}
+}
+
+static int amdgpu_discovery_get_table_info(struct amdgpu_device *adev,
+					   struct table_info **info,
+					   uint16_t table_id)
+{
+	struct binary_header *bhdr =
+		(struct binary_header *)adev->discovery.bin;
+
+	*info = amdgpu_discovery_get_table_info_from_bin(adev->discovery.bin,
+							 table_id);
+	if (!*info) {
+		dev_err(adev->dev, "Invalid ip discovery table version %d\n",
+			bhdr->version_major);
 		return -EINVAL;
 	}
 
@@ -724,11 +759,11 @@ static int amdgpu_discovery_init(struct amdgpu_device *adev)
 	return r;
 }
 
-static void amdgpu_discovery_sysfs_fini(struct amdgpu_device *adev);
-
 void amdgpu_discovery_fini(struct amdgpu_device *adev)
 {
-	amdgpu_discovery_sysfs_fini(adev);
+	if (adev->discovery.ip_top && !adev->discovery.ip_top->standalone_mode)
+		amdgpu_discovery_sysfs_fini(adev);
+
 	kfree(adev->discovery.bin);
 	adev->discovery.bin = NULL;
 }
@@ -737,15 +772,17 @@ static int amdgpu_discovery_validate_ip(struct amdgpu_device *adev,
 					uint8_t instance, uint16_t hw_id)
 {
 	if (instance >= HWIP_MAX_INSTANCE) {
-		dev_err(adev->dev,
-			"Unexpected instance_number (%d) from ip discovery blob\n",
-			instance);
+		if (adev)
+			dev_err(adev->dev,
+				"Unexpected instance_number (%d) from ip discovery blob\n",
+				instance);
 		return -EINVAL;
 	}
 	if (hw_id >= HW_ID_MAX) {
-		dev_err(adev->dev,
-			"Unexpected hw_id (%d) from ip discovery blob\n",
-			hw_id);
+		if (adev)
+			dev_err(adev->dev,
+				"Unexpected hw_id (%d) from ip discovery blob\n",
+				hw_id);
 		return -EINVAL;
 	}
 
@@ -1111,12 +1148,6 @@ static const struct kobj_type ip_discovery_ktype = {
 	.sysfs_ops = &kobj_sysfs_ops,
 };
 
-struct ip_discovery_top {
-	struct kobject kobj;    /* ip_discovery/ */
-	struct kset die_kset;   /* ip_discovery/die/, contains ip_die_entry */
-	struct amdgpu_device *adev;
-};
-
 static void die_kobj_release(struct kobject *kobj)
 {
 	struct ip_discovery_top *ip_top = container_of(to_kset(kobj),
@@ -1132,8 +1163,14 @@ static void ip_disc_release(struct kobject *kobj)
 						       kobj);
 	struct amdgpu_device *adev = ip_top->adev;
 
+	/* In standalone mode, discovery_bin is managed by devm and will be
+	 * freed automatically when the PCI device is removed. Do not manually
+	 * free it here to avoid double-free.
+	 */
+
 	kfree(ip_top);
-	adev->discovery.ip_top = NULL;
+	if (adev)
+		adev->discovery.ip_top = NULL;
 }
 
 static uint8_t amdgpu_discovery_get_harvest_info(struct amdgpu_device *adev,
@@ -1141,6 +1178,10 @@ static uint8_t amdgpu_discovery_get_harvest_info(struct amdgpu_device *adev,
 {
 	uint8_t harvest = 0;
 
+	/* In early init mode (adev == NULL), harvest info is not available */
+	if (!adev)
+		return 0;
+
 	/* Until a uniform way is figured, get mask based on hwid */
 	switch (hw_id) {
 	case VCN_HWID:
@@ -1169,11 +1210,14 @@ static uint8_t amdgpu_discovery_get_harvest_info(struct amdgpu_device *adev,
 }
 
 static int amdgpu_discovery_sysfs_ips(struct amdgpu_device *adev,
+				      struct ip_discovery_top *ip_top,
 				      struct ip_die_entry *ip_die_entry,
 				      const size_t _ip_offset, const int num_ips,
 				      bool reg_base_64)
 {
-	uint8_t *discovery_bin = adev->discovery.bin;
+	uint8_t *discovery_bin = ip_top->standalone_mode ?
+				 ip_top->discovery_bin :
+				 adev->discovery.bin;
 	int ii, jj, kk, res;
 	uint16_t hw_id;
 	uint8_t inst;
@@ -1270,10 +1314,12 @@ static int amdgpu_discovery_sysfs_ips(struct amdgpu_device *adev,
 	return 0;
 }
 
-static int amdgpu_discovery_sysfs_recurse(struct amdgpu_device *adev)
+static int amdgpu_discovery_sysfs_recurse(struct amdgpu_device *adev,
+					  struct ip_discovery_top *ip_top)
 {
-	struct ip_discovery_top *ip_top = adev->discovery.ip_top;
-	uint8_t *discovery_bin = adev->discovery.bin;
+	uint8_t *discovery_bin = ip_top->standalone_mode ?
+				 ip_top->discovery_bin :
+				 adev->discovery.bin;
 	struct table_info *info;
 	struct ip_discovery_header *ihdr;
 	struct die_header *dhdr;
@@ -1282,9 +1328,10 @@ static int amdgpu_discovery_sysfs_recurse(struct amdgpu_device *adev)
 	size_t ip_offset;
 	int ii, res;
 
-	res = amdgpu_discovery_get_table_info(adev, &info, IP_DISCOVERY);
-	if (res)
-		return res;
+	info = amdgpu_discovery_get_table_info_from_bin(discovery_bin,
+							IP_DISCOVERY);
+	if (!info)
+		return -EINVAL;
 	ihdr = (struct ip_discovery_header
 			*)(discovery_bin +
 			   le16_to_cpu(info->offset));
@@ -1322,7 +1369,8 @@ static int amdgpu_discovery_sysfs_recurse(struct amdgpu_device *adev)
 			return res;
 		}
 
-		amdgpu_discovery_sysfs_ips(adev, ip_die_entry, ip_offset, num_ips, !!ihdr->base_addr_64_bit);
+		amdgpu_discovery_sysfs_ips(adev, ip_top, ip_die_entry, ip_offset,
+					   num_ips, !!ihdr->base_addr_64_bit);
 	}
 
 	return 0;
@@ -1338,12 +1386,30 @@ static int amdgpu_discovery_sysfs_init(struct amdgpu_device *adev)
 	if (!discovery_bin)
 		return -EINVAL;
 
+	/* If early init already created sysfs in standalone mode, skip normal init */
+	if (adev->discovery.ip_top && adev->discovery.ip_top->standalone_mode)
+		return 0;
+
 	ip_top = kzalloc_obj(*ip_top);
 	if (!ip_top)
 		return -ENOMEM;
 
 	ip_top->adev = adev;
-	adev->discovery.ip_top = ip_top;
+
+	/* Check if ip_discovery already exists before creating.
+	 * This shouldn't normally happen but handle it gracefully.
+	 */
+	if (adev->dev->kobj.sd) {
+		struct kernfs_node *existing;
+
+		existing = kernfs_find_and_get(adev->dev->kobj.sd, "ip_discovery");
+		if (existing) {
+			kernfs_put(existing);
+			kfree(ip_top);
+			return 0;
+		}
+	}
+
 	res = kobject_init_and_add(&ip_top->kobj, &ip_discovery_ktype,
 				   &adev->dev->kobj, "ip_discovery");
 	if (res) {
@@ -1351,6 +1417,8 @@ static int amdgpu_discovery_sysfs_init(struct amdgpu_device *adev)
 		goto Err;
 	}
 
+	adev->discovery.ip_top = ip_top;
+
 	die_kset = &ip_top->die_kset;
 	kobject_set_name(&die_kset->kobj, "%s", "die");
 	die_kset->kobj.parent = &ip_top->kobj;
@@ -1365,7 +1433,7 @@ static int amdgpu_discovery_sysfs_init(struct amdgpu_device *adev)
 		ip_hw_instance_attrs[ii] = &ip_hw_attr[ii].attr;
 	ip_hw_instance_attrs[ii] = NULL;
 
-	res = amdgpu_discovery_sysfs_recurse(adev);
+	res = amdgpu_discovery_sysfs_recurse(adev, ip_top);
 
 	return res;
 Err:
@@ -1412,7 +1480,7 @@ static void amdgpu_discovery_sysfs_die_free(struct ip_die_entry *ip_die_entry)
 	kobject_put(&ip_die_entry->ip_kset.kobj);
 }
 
-static void amdgpu_discovery_sysfs_fini(struct amdgpu_device *adev)
+void amdgpu_discovery_sysfs_fini(struct amdgpu_device *adev)
 {
 	struct ip_discovery_top *ip_top = adev->discovery.ip_top;
 	struct list_head *el, *tmp;
@@ -1421,6 +1489,16 @@ static void amdgpu_discovery_sysfs_fini(struct amdgpu_device *adev)
 	if (!ip_top)
 		return;
 
+	/*
+	 * In standalone mode the sysfs hierarchy is tied to the PCI device
+	 * lifetime and is torn down by amdgpu_discovery_sysfs_early_fini().
+	 * Freeing it here would leave a dangling pointer in the early
+	 * discovery list, causing a use-after-free on driver unbind.
+	 */
+	if (ip_top->standalone_mode)
+		return;
+
+	adev->discovery.ip_top = NULL;
 	die_kset = &ip_top->die_kset;
 	spin_lock(&die_kset->list_lock);
 	list_for_each_prev_safe(el, tmp, &die_kset->list) {
@@ -1479,6 +1557,150 @@ void amdgpu_discovery_dump(struct amdgpu_device *adev, struct drm_printer *p)
 	spin_unlock(&die_kset->list_lock);
 }
 
+int amdgpu_discovery_sysfs_early_init(struct amdgpu_device *adev, struct pci_dev *pdev)
+{
+	struct ip_discovery_top *ip_top;
+	struct early_ip_discovery *early_entry, *tmp;
+	struct kset *die_kset;
+	uint8_t *discovery_bin;
+	int res, ii;
+
+	if (!adev || !adev->discovery.bin)
+		return -EINVAL;
+
+	if (adev->discovery.ip_top)
+		return 0;
+
+	mutex_lock(&early_ip_discovery_mutex);
+	list_for_each_entry_safe(early_entry, tmp, &early_ip_discovery_list, list) {
+		if (early_entry->pdev == pdev) {
+			adev->discovery.ip_top = early_entry->ip_top;
+			early_entry->ip_top->adev = adev;
+			mutex_unlock(&early_ip_discovery_mutex);
+			return 0;
+		}
+	}
+	mutex_unlock(&early_ip_discovery_mutex);
+
+	discovery_bin = adev->discovery.bin;
+
+	early_entry = kzalloc(sizeof(*early_entry), GFP_KERNEL);
+	if (!early_entry)
+		return -ENOMEM;
+
+	ip_top = kzalloc(sizeof(*ip_top), GFP_KERNEL);
+	if (!ip_top) {
+		kfree(early_entry);
+		return -ENOMEM;
+	}
+
+	ip_top->discovery_bin = devm_kmemdup(&pdev->dev, discovery_bin,
+					     DISCOVERY_TMR_SIZE, GFP_KERNEL);
+	if (!ip_top->discovery_bin) {
+		kfree(ip_top);
+		kfree(early_entry);
+		return -ENOMEM;
+	}
+
+	ip_top->bin_size = DISCOVERY_TMR_SIZE;
+	ip_top->pdev = pdev;
+	ip_top->adev = adev;
+	ip_top->standalone_mode = true;
+
+	/* Check if ip_discovery already exists (from previous probe attempt).
+	 * This can happen if the module was unloaded and reloaded but the
+	 * sysfs persisted (tied to PCI device lifetime).
+	 */
+	if (pdev->dev.kobj.sd) {
+		struct kernfs_node *existing;
+
+		existing = kernfs_find_and_get(pdev->dev.kobj.sd, "ip_discovery");
+		if (existing) {
+			kernfs_put(existing);
+			kfree(ip_top);
+			kfree(early_entry);
+			return 0;
+		}
+	}
+
+	res = kobject_init_and_add(&ip_top->kobj, &ip_discovery_ktype,
+				   &pdev->dev.kobj, "ip_discovery");
+	if (res)
+		goto err_put_kobj;
+
+	adev->discovery.ip_top = ip_top;
+
+	die_kset = &ip_top->die_kset;
+	kobject_set_name(&die_kset->kobj, "%s", "die");
+	die_kset->kobj.parent = &ip_top->kobj;
+	die_kset->kobj.ktype = &die_kobj_ktype;
+	res = kset_register(&ip_top->die_kset);
+	if (res)
+		goto err_put_die_kset;
+
+	for (ii = 0; ii < ARRAY_SIZE(ip_hw_attr); ii++)
+		ip_hw_instance_attrs[ii] = &ip_hw_attr[ii].attr;
+	ip_hw_instance_attrs[ii] = NULL;
+
+	res = amdgpu_discovery_sysfs_recurse(NULL, ip_top);
+	if (res)
+		goto err_put_die_kset;
+
+	early_entry->pdev = pdev;
+	early_entry->ip_top = ip_top;
+	mutex_lock(&early_ip_discovery_mutex);
+	list_add(&early_entry->list, &early_ip_discovery_list);
+	mutex_unlock(&early_ip_discovery_mutex);
+
+	return 0;
+
+err_put_die_kset:
+	kobject_put(&ip_top->die_kset.kobj);
+err_put_kobj:
+	kobject_put(&ip_top->kobj);
+	kfree(early_entry);
+	adev->discovery.ip_top = NULL;
+	return res;
+}
+
+void amdgpu_discovery_sysfs_early_fini(struct pci_dev *pdev)
+{
+	struct early_ip_discovery *entry, *tmp_entry;
+	struct ip_discovery_top *ip_top = NULL;
+	struct list_head *el, *tmp;
+	struct kset *die_kset;
+
+	/* Find the entry in our tracking list */
+	mutex_lock(&early_ip_discovery_mutex);
+	list_for_each_entry_safe(entry, tmp_entry, &early_ip_discovery_list, list) {
+		if (entry->pdev == pdev) {
+			ip_top = entry->ip_top;
+			list_del(&entry->list);
+			kfree(entry);
+			break;
+		}
+	}
+	mutex_unlock(&early_ip_discovery_mutex);
+
+	if (!ip_top)
+		return;
+
+	/* Clean up sysfs hierarchy */
+	die_kset = &ip_top->die_kset;
+
+	spin_lock(&die_kset->list_lock);
+	list_for_each_prev_safe(el, tmp, &die_kset->list) {
+		list_del_init(el);
+		spin_unlock(&die_kset->list_lock);
+		amdgpu_discovery_sysfs_die_free(to_ip_die_entry(list_to_kobj(el)));
+		spin_lock(&die_kset->list_lock);
+	}
+	spin_unlock(&die_kset->list_lock);
+
+	kobject_put(&ip_top->die_kset.kobj);
+	kobject_put(&ip_top->kobj);
+	/* ip_top itself will be freed by kobject_put via ip_disc_release */
+}
 
 /* ================================================== */
 
@@ -1504,6 +1726,9 @@ static int amdgpu_discovery_reg_base_init(struct amdgpu_device *adev)
 	r = amdgpu_discovery_init(adev);
 	if (r)
 		return r;
+
+	amdgpu_discovery_sysfs_early_init(adev, adev->pdev);
+
 	discovery_bin = adev->discovery.bin;
 	wafl_ver = 0;
 	adev->gfx.xcc_mask = 0;
@@ -2638,7 +2863,12 @@ static int amdgpu_discovery_set_mm_ip_blocks(struct amdgpu_device *adev)
 			return -EINVAL;
 		}
 	} else {
-		switch (amdgpu_ip_version(adev, UVD_HWIP, 0)) {
+		uint32_t vcn_version = amdgpu_ip_version(adev, UVD_HWIP, 0);
+
+		/* no VCN discovered; nothing to add */
+		if (!vcn_version)
+			return 0;
+		switch (vcn_version) {
 		case IP_VERSION(1, 0, 0):
 		case IP_VERSION(1, 0, 1):
 			amdgpu_device_ip_block_add(adev, &vcn_v1_0_ip_block);
@@ -2706,7 +2936,7 @@ static int amdgpu_discovery_set_mm_ip_blocks(struct amdgpu_device *adev)
 		default:
 			dev_err(adev->dev,
 				"Failed to add vcn/jpeg ip block(UVD_HWIP:0x%x)\n",
-				amdgpu_ip_version(adev, UVD_HWIP, 0));
+				vcn_version);
 			return -EINVAL;
 		}
 	}
@@ -3137,9 +3367,11 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(11, 5, 3):
 	case IP_VERSION(11, 5, 4):
 	case IP_VERSION(11, 5, 6):
+		adev->family = AMDGPU_FAMILY_GC_11_5_0;
+		break;
 	case IP_VERSION(11, 7, 0):
 	case IP_VERSION(11, 7, 1):
-		adev->family = AMDGPU_FAMILY_GC_11_5_0;
+		adev->family = AMDGPU_FAMILY_GC_11_5_4;
 		break;
 	case IP_VERSION(12, 0, 0):
 	case IP_VERSION(12, 0, 1):
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
index e0010f6a3eda5..5b2b16f68576c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
@@ -41,6 +41,7 @@ struct amdgpu_discovery_info {
 	bool reserve_tmr;
 };
 
+void amdgpu_discovery_sysfs_fini(struct amdgpu_device *adev);
 void amdgpu_discovery_fini(struct amdgpu_device *adev);
 int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev);
 
@@ -53,4 +54,9 @@ int amdgpu_discovery_get_gc_major_minor_version(struct amdgpu_device *adev,
 
 void amdgpu_discovery_dump(struct amdgpu_device *adev, struct drm_printer *p);
 
+/* Early sysfs functions for persistent ip_discovery export */
+int amdgpu_discovery_sysfs_early_init(struct amdgpu_device *adev,
+				       struct pci_dev *pdev);
+void amdgpu_discovery_sysfs_early_fini(struct pci_dev *pdev);
+
 #endif /* __AMDGPU_DISCOVERY__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 4c0c77eafbd1c..5362705143bc7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -33,7 +33,6 @@
 #include <drm/drm_vblank.h>
 
 #include <linux/cc_platform.h>
-#include <linux/console.h>
 #include <linux/dynamic_debug.h>
 #include <linux/module.h>
 #include <linux/mmu_notifier.h>
@@ -140,13 +139,14 @@ enum AMDGPU_DEBUG_MASK {
 	AMDGPU_DEBUG_LARGEBAR = BIT(1),
 	AMDGPU_DEBUG_DISABLE_GPU_SOFT_RECOVERY = BIT(2),
 	AMDGPU_DEBUG_USE_VRAM_FW_BUF = BIT(3),
-	AMDGPU_DEBUG_ENABLE_RAS_ACA = BIT(4),
 	AMDGPU_DEBUG_ENABLE_EXP_RESETS = BIT(5),
 	AMDGPU_DEBUG_DISABLE_GPU_RING_RESET = BIT(6),
 	AMDGPU_DEBUG_SMU_POOL = BIT(7),
 	AMDGPU_DEBUG_VM_USERPTR = BIT(8),
 	AMDGPU_DEBUG_DISABLE_RAS_CE_LOG = BIT(9),
-	AMDGPU_DEBUG_ENABLE_CE_CS = BIT(10)
+	AMDGPU_DEBUG_ENABLE_CE_CS = BIT(10),
+	AMDGPU_DEBUG_HIBERNATION_THAW_RESUME_GPU = BIT(11),
+	AMDGPU_DEBUG_DISABLE_IP_BLOCK_SOFT_RESET = BIT(12),
 };
 
 unsigned int amdgpu_vram_limit = UINT_MAX;
@@ -1939,6 +1939,7 @@ static const struct pci_device_id pciidlist[] = {
 	{0x1002, 0x6646, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMD_IS_MOBILITY},
 	{0x1002, 0x6647, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMD_IS_MOBILITY},
 	{0x1002, 0x6649, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE},
+	{0x1002, 0x664D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE},
 	{0x1002, 0x6650, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE},
 	{0x1002, 0x6651, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE},
 	{0x1002, 0x6658, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE},
@@ -2008,6 +2009,7 @@ static const struct pci_device_id pciidlist[] = {
 	{0x1002, 0x6930, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TONGA},
 	{0x1002, 0x6938, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TONGA},
 	{0x1002, 0x6939, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TONGA},
+	{0x1002, 0x693B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TONGA},
 	/* fiji */
 	{0x1002, 0x7300, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_FIJI},
 	{0x1002, 0x730F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_FIJI},
@@ -2036,6 +2038,7 @@ static const struct pci_device_id pciidlist[] = {
 	{0x1002, 0x67C4, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS10},
 	{0x1002, 0x67C7, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS10},
 	{0x1002, 0x67D0, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS10},
+	{0x1002, 0x67D4, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS10},
 	{0x1002, 0x67DF, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS10},
 	{0x1002, 0x67C8, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS10},
 	{0x1002, 0x67C9, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS10},
@@ -2049,6 +2052,7 @@ static const struct pci_device_id pciidlist[] = {
 	{0x1002, 0x6985, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12},
 	{0x1002, 0x6986, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12},
 	{0x1002, 0x6987, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12},
+	{0x1002, 0x698F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12},
 	{0x1002, 0x6995, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12},
 	{0x1002, 0x6997, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12},
 	{0x1002, 0x699F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12},
@@ -2250,7 +2254,7 @@ static void amdgpu_init_debug_options(struct amdgpu_device *adev)
 	}
 
 	if (amdgpu_debug_mask & AMDGPU_DEBUG_DISABLE_GPU_SOFT_RECOVERY) {
-		pr_info("debug: soft reset for GPU recovery disabled\n");
+		pr_info("debug: soft recovery disabled\n");
 		adev->debug_disable_soft_recovery = true;
 	}
 
@@ -2259,11 +2263,6 @@ static void amdgpu_init_debug_options(struct amdgpu_device *adev)
 		adev->debug_use_vram_fw_buf = true;
 	}
 
-	if (amdgpu_debug_mask & AMDGPU_DEBUG_ENABLE_RAS_ACA) {
-		pr_info("debug: enable RAS ACA\n");
-		adev->debug_enable_ras_aca = true;
-	}
-
 	if (amdgpu_debug_mask & AMDGPU_DEBUG_ENABLE_EXP_RESETS) {
 		pr_info("debug: enable experimental reset features\n");
 		adev->debug_exp_resets = true;
@@ -2291,6 +2290,16 @@ static void amdgpu_init_debug_options(struct amdgpu_device *adev)
 		pr_info("debug: allowing command submission to CE engine\n");
 		adev->debug_enable_ce_cs = true;
 	}
+
+	if (amdgpu_debug_mask & AMDGPU_DEBUG_HIBERNATION_THAW_RESUME_GPU) {
+		pr_info("debug: resume gpu in thaw() of hibernation\n");
+		adev->debug_hibernation_thaw_resume_gpu = true;
+	}
+
+	if (amdgpu_debug_mask & AMDGPU_DEBUG_DISABLE_IP_BLOCK_SOFT_RESET) {
+		pr_info("debug: IP block soft reset disabled\n");
+		adev->debug_disable_ip_block_soft_reset = true;
+	}
 }
 
 static unsigned long amdgpu_fix_asic_type(struct pci_dev *pdev, unsigned long flags)
@@ -2552,6 +2561,8 @@ amdgpu_pci_remove(struct pci_dev *pdev)
 
 	amdgpu_driver_unload_kms(dev);
 
+	amdgpu_discovery_sysfs_early_fini(pdev);
+
 	/*
 	 * Flush any in flight DMA operations from device.
 	 * Clear the Bus Master Enable bit and then wait on the PCIe Device
@@ -2705,9 +2716,10 @@ static int amdgpu_pmops_freeze(struct device *dev)
 static int amdgpu_pmops_thaw(struct device *dev)
 {
 	struct drm_device *drm_dev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(drm_dev);
 
 	/* do not resume device if it's normal hibernation */
-	if (console_suspend_enabled &&
+	if (!adev->debug_hibernation_thaw_resume_gpu &&
 	    !pm_hibernate_is_recovering() &&
 	    !pm_hibernation_mode_is_suspend())
 		return 0;
@@ -3076,6 +3088,7 @@ const struct drm_ioctl_desc amdgpu_ioctls_kms[] = {
 	DRM_IOCTL_DEF_DRV(AMDGPU_USERQ_SIGNAL, amdgpu_userq_signal_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(AMDGPU_USERQ_WAIT, amdgpu_userq_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(AMDGPU_GEM_LIST_HANDLES, amdgpu_gem_list_handles_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(AMDGPU_PROC_OPTIONS, amdgpu_proc_options_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
 };
 
 static const struct drm_driver amdgpu_kms_driver = {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index ea69b1bac7c6e..3043ad041bb42 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -727,6 +727,15 @@ void amdgpu_ring_set_fence_errors_and_reemit(struct amdgpu_ring *ring,
 	last_seq = amdgpu_fence_read(ring) & ring->fence_drv.num_fences_mask;
 	seq = ring->fence_drv.sync_seq & ring->fence_drv.num_fences_mask;
 
+	/* If there is nothing to reemit, return early and set an error on the fence
+	 * if applicable. If all of the fences are siganlled, this will be a nop.
+	 * if there are still fences and ring_backup_entries_to_copy is 0, then
+	 * we are skipping it on purpose.
+	 */
+	if (!ring->ring_backup_entries_to_copy) {
+		amdgpu_fence_driver_force_completion(ring, &guilty_fence->base);
+		return;
+	}
 	ring->reemit = true;
 	amdgpu_ring_alloc(ring, ring->ring_backup_entries_to_copy);
 	spin_lock_irqsave(&ring->fence_drv.lock, flags);
@@ -741,7 +750,8 @@ void amdgpu_ring_set_fence_errors_and_reemit(struct amdgpu_ring *ring,
 		if (unprocessed && !dma_fence_is_signaled_locked(unprocessed)) {
 			fence = container_of(unprocessed, struct amdgpu_fence, base);
 			is_guilty_fence = fence == guilty_fence;
-			is_guilty_context = fence->context == guilty_fence->context;
+			is_guilty_context = guilty_fence ?
+				(fence->context == guilty_fence->context) : false;
 
 			/* mark all fences from the guilty context with an error */
 			if (is_guilty_fence)
@@ -794,6 +804,17 @@ void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring,
 	seq = ring->fence_drv.sync_seq & ring->fence_drv.num_fences_mask;
 	ring->ring_backup_entries_to_copy = 0;
 
+	/* if we've already seen this fence, return early.
+	 * ring->ring_backup_entries_to_copy is set to 0 so
+	 * the reemit helper will return early as well to
+	 * avoid getting stuck in a reemit loop.
+	 */
+	if (ring->guilty_fence == guilty_fence) {
+		ring->guilty_fence = NULL;
+		return;
+	}
+	ring->guilty_fence = guilty_fence;
+
 	do {
 		last_seq++;
 		last_seq &= ring->fence_drv.num_fences_mask;
@@ -811,6 +832,36 @@ void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring,
 	} while (last_seq != seq);
 }
 
+struct amdgpu_fence *
+amdgpu_ring_find_guilty_fence(struct amdgpu_ring *ring)
+{
+	struct dma_fence *unprocessed;
+	struct dma_fence __rcu **ptr;
+	struct amdgpu_fence *fence;
+	u32 seq, last_seq;
+
+	last_seq = amdgpu_fence_read(ring) & ring->fence_drv.num_fences_mask;
+	seq = ring->fence_drv.sync_seq & ring->fence_drv.num_fences_mask;
+
+	do {
+		last_seq++;
+		last_seq &= ring->fence_drv.num_fences_mask;
+
+		ptr = &ring->fence_drv.fences[last_seq];
+		rcu_read_lock();
+		unprocessed = rcu_dereference(*ptr);
+
+		if (unprocessed && !dma_fence_is_signaled(unprocessed)) {
+			fence = container_of(unprocessed, struct amdgpu_fence, base);
+			rcu_read_unlock();
+			return fence;
+		}
+		rcu_read_unlock();
+	} while (last_seq != seq);
+
+	return NULL;
+}
+
 /*
  * Common fence implementation
  */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 85372af1216d0..9d3b40c385c92 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -34,6 +34,7 @@
 #include "amdgpu_xcp.h"
 #include "amdgpu_xgmi.h"
 #include "amdgpu_mes.h"
+#include "mes_userqueue.h"
 #include "nvd.h"
 
 /* delay 0.1 second to enable gfx off feature */
@@ -377,6 +378,30 @@ int amdgpu_gfx_kiq_init(struct amdgpu_device *adev,
 	return 0;
 }
 
+static void amdgpu_gfx_mqd_reset_restore(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+	int mqd_idx, mqd_size;
+
+	/* restore mqd with the backup copy */
+	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
+		mqd_idx = ring - &adev->gfx.compute_ring[0];
+		mqd_size = adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size;
+		if (adev->gfx.mec.mqd_backup[mqd_idx])
+			memcpy_toio(ring->mqd_ptr, adev->gfx.mec.mqd_backup[mqd_idx], mqd_size);
+	} else if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
+		mqd_size = adev->mqds[AMDGPU_HW_IP_GFX].mqd_size;
+		mqd_idx = ring - &adev->gfx.gfx_ring[0];
+
+		if (adev->gfx.me.mqd_backup[mqd_idx])
+			memcpy_toio(ring->mqd_ptr, adev->gfx.me.mqd_backup[mqd_idx], mqd_size);
+	}
+	/* reset the ring */
+	ring->wptr = 0;
+	atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
+	amdgpu_ring_clear_ring(ring);
+}
+
 /* create MQD for each compute/gfx queue */
 int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev,
 			   unsigned int mqd_size, int xcc_id)
@@ -1145,7 +1170,7 @@ uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg, uint32_t xcc_
 	BUG_ON(!ring->funcs->emit_rreg);
 
 	spin_lock_irqsave(&kiq->ring_lock, flags);
-	if (amdgpu_device_wb_get(adev, &reg_val_offs)) {
+	if (amdgpu_wb_get(adev, &reg_val_offs)) {
 		pr_err("critical bug! too many kiq readers\n");
 		goto failed_unlock;
 	}
@@ -1188,7 +1213,7 @@ uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg, uint32_t xcc_
 
 	mb();
 	value = adev->wb.wb[reg_val_offs];
-	amdgpu_device_wb_free(adev, reg_val_offs);
+	amdgpu_wb_free(adev, reg_val_offs);
 	return value;
 
 failed_undo:
@@ -1197,7 +1222,7 @@ uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg, uint32_t xcc_
 	spin_unlock_irqrestore(&kiq->ring_lock, flags);
 failed_kiq_read:
 	if (reg_val_offs)
-		amdgpu_device_wb_free(adev, reg_val_offs);
+		amdgpu_wb_free(adev, reg_val_offs);
 	dev_err(adev->dev, "failed to read reg:%x\n", reg);
 	return ~0;
 }
@@ -1688,8 +1713,9 @@ static int amdgpu_gfx_run_cleaner_shader_job(struct amdgpu_ring *ring)
 	owner = (void *)(unsigned long)atomic_inc_return(&counter);
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, &entity, owner,
-				     ib_size_dw * sizeof(uint32_t), 0, &job,
-				     AMDGPU_KERNEL_JOB_ID_CLEANER_SHADER);
+				     ib_size_dw * sizeof(uint32_t), 0,
+				     AMDGPU_KERNEL_JOB_ID_CLEANER_SHADER,
+				     &job);
 	if (r)
 		goto err;
 
@@ -1964,6 +1990,60 @@ static ssize_t amdgpu_gfx_get_compute_reset_mask(struct device *dev,
 	return amdgpu_show_reset_mask(buf, adev->gfx.compute_supported_reset);
 }
 
+static int amdgpu_gfx_mes_reset_queue_start(struct amdgpu_ring *ring,
+					     unsigned int vmid,
+					     struct amdgpu_fence *timedout_fence,
+					     bool use_mmio)
+{
+	struct amdgpu_device *adev = ring->adev;
+	bool reinit_queue;
+	int r;
+
+	if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) &&
+	    adev->mes.compute_pipe_reset_enabled)
+		reinit_queue = true;
+	else if ((ring->funcs->type == AMDGPU_RING_TYPE_GFX) &&
+		 adev->mes.gfx_pipe_reset_enabled)
+		reinit_queue = true;
+	else
+		reinit_queue = use_mmio;
+
+	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
+
+	r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, use_mmio, 0);
+	if (r)
+		return r;
+
+	if (reinit_queue) {
+		r = amdgpu_mes_unmap_legacy_queue(adev, ring,
+						  RESET_QUEUES, 0, 0, 0);
+		if (r)
+			return r;
+		amdgpu_gfx_mqd_reset_restore(ring);
+
+		r = amdgpu_mes_map_legacy_queue(adev, ring, 0);
+		if (r) {
+			dev_err(adev->dev, "failed to remap kgq\n");
+			return r;
+		}
+	}
+	return 0;
+}
+
+int amdgpu_gfx_mes_reset_queue(struct amdgpu_ring *ring,
+			       unsigned int vmid,
+			       struct amdgpu_fence *timedout_fence,
+			       bool use_mmio)
+{
+	int r;
+
+	r = amdgpu_gfx_mes_reset_queue_start(ring, vmid, timedout_fence,
+					      use_mmio);
+	if (r)
+		return r;
+	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
+}
+
 static DEVICE_ATTR(run_cleaner_shader, 0200,
 		   NULL, amdgpu_gfx_set_run_cleaner_shader);
 
@@ -2122,6 +2202,200 @@ void amdgpu_gfx_sysfs_fini(struct amdgpu_device *adev)
 	}
 }
 
+static void amdgpu_gfx_reset_start_compute_scheds(struct amdgpu_device *adev,
+						  struct amdgpu_ring *guilty_ring)
+{
+	struct amdgpu_ring *ring;
+	int i;
+
+	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+		ring = &adev->gfx.compute_ring[i];
+		if (ring == guilty_ring)
+			continue;
+		drm_sched_wqueue_start(&ring->sched);
+	}
+}
+
+static void amdgpu_gfx_reset_stop_compute_scheds(struct amdgpu_device *adev,
+						 struct amdgpu_ring *guilty_ring)
+{
+	struct amdgpu_ring *ring;
+	int i;
+
+	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+		ring = &adev->gfx.compute_ring[i];
+		if (ring == guilty_ring)
+			continue;
+		drm_sched_wqueue_stop(&ring->sched);
+	}
+}
+
+/*
+ * Match the MES-reported hung doorbell against a compute ring and run
+ * the reset. On hit, the matched ring and its guilty fence are returned
+ * via *out_ring / *out_fence so the caller can defer reset end until
+ * after MES has resumed all gangs.
+ */
+static int amdgpu_gfx_reset_mes_kcq(struct amdgpu_device *adev,
+				    struct amdgpu_ring *guilty_ring,
+				    unsigned int db,
+				    struct amdgpu_ring **out_ring,
+				    struct amdgpu_fence **out_fence)
+{
+	bool use_mmio = adev->gfx.mec.use_mmio_for_reset;
+	struct amdgpu_fence *fence;
+	struct amdgpu_ring *ring;
+	int i, r;
+
+	*out_ring = NULL;
+	*out_fence = NULL;
+	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+		ring = &adev->gfx.compute_ring[i];
+		if (ring == guilty_ring)
+			continue;
+		if (ring->doorbell_index == db) {
+			fence = amdgpu_ring_find_guilty_fence(ring);
+			r = amdgpu_gfx_mes_reset_queue_start(ring, 0, fence,
+							      use_mmio);
+			if (r)
+				return r;
+			*out_ring = ring;
+			*out_fence = fence;
+			break;
+		}
+	}
+	return 0;
+}
+
+int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
+				 struct amdgpu_ring *ring,
+				 struct amdgpu_fence *guilty_fence,
+				 struct amdgpu_usermode_queue *uq,
+				 unsigned int *hung_queue_count,
+				 void *faulty_queue_input)
+{
+	struct amdgpu_mes_hung_queue_hqd_info *hqd_info =
+		(struct amdgpu_mes_hung_queue_hqd_info *)
+		&adev->gfx.mec.mes_hung_db_array[adev->mes.hung_queue_hqd_info_offset];
+	int i, r, pipe, queue, queue_type;
+	unsigned int num_hung = 0;
+	bool use_mmio = adev->gfx.mec.use_mmio_for_reset;
+	struct mes_remove_queue_input *queue_input = (struct mes_remove_queue_input *)faulty_queue_input;
+	struct amdgpu_gfx_deferred_entry deferred_end[AMDGPU_MAX_COMPUTE_RINGS + 1];
+	int n_deferred = 0;
+	int ring_err;
+
+	guard(mutex)(&adev->gfx.mec.reset_mutex);
+	/* stop the drm schedulers for all compute queues */
+	amdgpu_gfx_reset_stop_compute_scheds(adev, ring);
+	/* suspend all will determine which queues are hung.
+	 * reset detect will return the array of bad queue doorbells
+	 */
+	r = amdgpu_mes_suspend(adev, 0);
+	/* if suspend all success, it should no hang queue */
+	if (!r)
+		/* always reset the KCQ/userq since we need to signal the fence
+		 * and we could be stuck in a loop which is preemptable.
+		 */
+		goto fence_reset;
+	r = amdgpu_mes_detect_and_reset_hung_queues(adev, AMDGPU_RING_TYPE_COMPUTE,
+						    true, &num_hung, adev->gfx.mec.mes_hung_db_array, 0);
+	if (r)
+		goto out;
+	if (hung_queue_count)
+		*hung_queue_count = num_hung;
+
+fence_reset:
+	/* reset the queue this came from if specified */
+	if (ring) {
+		r = amdgpu_gfx_mes_reset_queue_start(ring, 0, guilty_fence,
+						      use_mmio);
+		if (r)
+			goto out;
+		deferred_end[n_deferred].ring = ring;
+		deferred_end[n_deferred].fence = guilty_fence;
+		n_deferred++;
+	}
+	if (uq) {
+		r = mes_userq_reset(uq);
+		if (r)
+			goto out;
+	}
+	for (i = 0; i < num_hung; i++) {
+		struct amdgpu_ring *hr = NULL;
+		struct amdgpu_fence *hf = NULL;
+
+		pipe = hqd_info[i].pipe_index;
+		queue = hqd_info[i].queue_index;
+		queue_type = hqd_info[i].queue_type;
+
+		/* reset any KCQs */
+		r = amdgpu_gfx_reset_mes_kcq(adev, ring,
+					     adev->gfx.mec.mes_hung_db_array[i],
+					     &hr, &hf);
+		if (r)
+			goto out;
+		if (hr) {
+			deferred_end[n_deferred].ring = hr;
+			deferred_end[n_deferred].fence = hf;
+			n_deferred++;
+		}
+		/* reset any KFD queues */
+		r = amdgpu_amdkfd_reset_mes_queue(adev, 0, queue_type, pipe, queue,
+						  adev->gfx.mec.mes_hung_db_array[i]);
+		if (r)
+			goto out;
+		/* reset KGD user queues */
+		r = mes_userq_reset_queue(adev, uq, queue_type, pipe, queue,
+					  adev->gfx.mec.mes_hung_db_array[i]);
+		if (r)
+			goto out;
+	}
+
+	/* MES doesn't detect any hung queue but we have a known bad queue
+	 * and it is not KCQ
+	 */
+	if (!num_hung && queue_input && !ring) {
+		/* MES suspend_all is successful means this bad queue is
+		 * preempted successfuly. Remove it before resume all so it
+		 * doesn't get mapped back
+		 */
+		if (!down_read_trylock(&adev->reset_domain->sem)) {
+			r = -EIO;
+			goto out;
+		}
+		amdgpu_mes_lock(&adev->mes);
+		r = adev->mes.funcs->remove_hw_queue(&adev->mes, queue_input);
+		amdgpu_mes_unlock(&adev->mes);
+		up_read(&adev->reset_domain->sem);
+	}
+
+out:
+	/* resume all will enable the non-hung queues */
+	amdgpu_mes_resume(adev, 0);
+
+	/* Now CP is running again — replay backed-up commands and ring
+	 * doorbells on each reset queue.
+	 */
+	ring_err = r;
+	for (i = 0; i < n_deferred; i++) {
+		int er = amdgpu_ring_reset_helper_end(deferred_end[i].ring,
+						      deferred_end[i].fence);
+
+		if (er && !ring_err)
+			ring_err = er;
+	}
+
+	if (!ring_err)
+		amdgpu_gfx_reset_start_compute_scheds(adev, ring);
+
+	/* If this reset is triggered by non-KCQ, the KCQ result after resume must
+	 * not override the reset result; otherwise a false reset failure is returned
+	 * to the non-KCQ caller
+	 */
+	return ring ? ring_err : r;
+}
+
 int amdgpu_gfx_cleaner_shader_sw_init(struct amdgpu_device *adev,
 				      unsigned int cleaner_shader_size)
 {
@@ -2460,9 +2734,8 @@ void amdgpu_gfx_profile_ring_begin_use(struct amdgpu_ring *ring)
 	else
 		profile = PP_SMC_POWER_PROFILE_COMPUTE;
 
-	atomic_inc(&adev->gfx.total_submission_cnt);
-
-	cancel_delayed_work_sync(&adev->gfx.idle_work);
+	if (!atomic_fetch_inc(&adev->gfx.total_submission_cnt))
+		cancel_delayed_work_sync(&adev->gfx.idle_work);
 
 	/* We can safely return early here because we've cancelled the
 	 * the delayed work so there is no one else to set it to false
@@ -2490,9 +2763,9 @@ void amdgpu_gfx_profile_ring_end_use(struct amdgpu_ring *ring)
 	if (amdgpu_dpm_is_overdrive_enabled(adev))
 		return;
 
-	atomic_dec(&ring->adev->gfx.total_submission_cnt);
-
-	schedule_delayed_work(&ring->adev->gfx.idle_work, GFX_PROFILE_IDLE_TIMEOUT);
+	if (atomic_dec_and_test(&ring->adev->gfx.total_submission_cnt))
+		schedule_delayed_work(&ring->adev->gfx.idle_work,
+				      GFX_PROFILE_IDLE_TIMEOUT);
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index 54c1eb9c499ba..aefd4f03b4433 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -36,6 +36,8 @@
 #include "amdgpu_ring_mux.h"
 #include "amdgpu_xcp.h"
 
+struct amdgpu_usermode_queue;
+
 /* GFX current status */
 #define AMDGPU_GFX_NORMAL_MODE			0x00000000L
 #define AMDGPU_GFX_SAFE_MODE			0x00000001L
@@ -116,6 +118,9 @@ struct amdgpu_mec {
 	u32 num_pipe_per_mec;
 	u32 num_queue_per_pipe;
 	void			*mqd_backup[AMDGPU_MAX_COMPUTE_RINGS * AMDGPU_MAX_GC_INSTANCES];
+	bool use_mmio_for_reset;
+	u32 *mes_hung_db_array;
+	struct mutex		reset_mutex;
 };
 
 struct amdgpu_mec_bitmap {
@@ -401,6 +406,7 @@ struct amdgpu_me {
 	uint32_t			num_pipe_per_me;
 	uint32_t			num_queue_per_pipe;
 	void				*mqd_backup[AMDGPU_MAX_GFX_RINGS];
+	bool				use_mmio_for_reset;
 
 	/* These are the resources for which amdgpu takes ownership */
 	DECLARE_BITMAP(queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
@@ -479,8 +485,6 @@ struct amdgpu_gfx {
 	const struct amdgpu_gfx_funcs	*funcs;
 
 	/* reset mask */
-	uint32_t                        grbm_soft_reset;
-	uint32_t                        srbm_soft_reset;
 	uint32_t 			gfx_supported_reset;
 	uint32_t 			compute_supported_reset;
 
@@ -543,6 +547,11 @@ struct amdgpu_gfx {
 	bool				disable_uq;
 };
 
+struct amdgpu_gfx_deferred_entry {
+	struct amdgpu_ring	*ring;
+	struct amdgpu_fence	*fence;
+};
+
 struct amdgpu_gfx_ras_reg_entry {
 	struct amdgpu_ras_err_status_reg_entry reg_entry;
 	enum amdgpu_gfx_ras_mem_id_type mem_id_type;
@@ -641,6 +650,12 @@ int amdgpu_gfx_poison_consumption_handler(struct amdgpu_device *adev,
 bool amdgpu_gfx_is_master_xcc(struct amdgpu_device *adev, int xcc_id);
 int amdgpu_gfx_sysfs_init(struct amdgpu_device *adev);
 void amdgpu_gfx_sysfs_fini(struct amdgpu_device *adev);
+int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev,
+				 struct amdgpu_ring *ring,
+				 struct amdgpu_fence *guilty_fence,
+				 struct amdgpu_usermode_queue *uq,
+				 unsigned int *hung_queue_count,
+				 void *faulty_queue_input);
 void amdgpu_gfx_ras_error_func(struct amdgpu_device *adev,
 		void *ras_error_status,
 		void (*func)(struct amdgpu_device *adev, void *ras_error_status,
@@ -667,6 +682,11 @@ void amdgpu_debugfs_compute_sched_mask_init(struct amdgpu_device *adev);
 
 int amdgpu_gfx_ring_preempt_ib(struct amdgpu_ring *ring);
 
+int amdgpu_gfx_mes_reset_queue(struct amdgpu_ring *ring,
+			       unsigned int vmid,
+			       struct amdgpu_fence *timedout_fence,
+			       bool use_mmio);
+
 static inline const char *amdgpu_gfx_compute_mode_desc(int mode)
 {
 	switch (mode) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 5d6149ba7ab78..3da50132c5d04 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -761,7 +761,8 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 	r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.default_entity.base,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
-				     &job, AMDGPU_KERNEL_JOB_ID_FLUSH_GPU_TLB);
+				     AMDGPU_KERNEL_JOB_ID_FLUSH_GPU_TLB,
+				     &job);
 	if (r)
 		goto error_alloc;
 
@@ -1763,10 +1764,15 @@ int amdgpu_gmc_init_mem_ranges(struct amdgpu_device *adev)
 		valid = true;
 	else
 		valid = amdgpu_gmc_validate_partition_info(adev);
-	if (!valid) {
-		/* TODO: handle invalid case */
+	if (!valid)
 		dev_warn(adev->dev,
 			 "Mem ranges not matching with hardware config\n");
+
+	if (!adev->gmc.num_mem_partitions) {
+		dev_err(adev->dev, "num_mem_partitions is zero\n");
+		kfree(adev->gmc.mem_partitions);
+		adev->gmc.mem_partitions = NULL;
+		return -EINVAL;
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index ddb0d500e0faa..3ca187f5ade85 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -286,7 +286,6 @@ struct amdgpu_gmc {
 	struct amdgpu_irq_src	vm_fault;
 	uint32_t		vram_type;
 	uint8_t			vram_vendor;
-	uint32_t                srbm_soft_reset;
 	bool			prt_warning;
 	uint32_t		sdpif_register;
 	/* apertures */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index 63f62c670df56..634b3f7a5fff9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -351,6 +351,30 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
  */
 int amdgpu_ib_pool_init(struct amdgpu_device *adev)
 {
+	const int sizes[AMDGPU_IB_POOL_MAX] = {
+		[AMDGPU_IB_POOL_DELAYED] = SZ_1M,
+		[AMDGPU_IB_POOL_IMMEDIATE] = SZ_128K,
+		[AMDGPU_IB_POOL_DIRECT] = SZ_512K
+	};
+	const gfp_t gfp_flags[AMDGPU_IB_POOL_MAX] = {
+		/*
+		 * For normal page table updates and recoverable retry faults
+		 * (for SVM), further restricted by the VM eviction lock to not
+		 * wait for memory reclaim.
+		 */
+		[AMDGPU_IB_POOL_DELAYED] = GFP_KERNEL,
+		/*
+		 * For redirecting unrecoverable retry faults to the dummy page
+		 * or set the PRT bits. dma_fence submissions might depend on
+		 * that so we need the emmergency reserves.
+		 */
+		[AMDGPU_IB_POOL_IMMEDIATE] = GFP_ATOMIC,
+		/*
+		 * For IB tests during GPU resets. Only very small and temporary
+		 * allocation to allow dma_fences to signal.
+		 */
+		[AMDGPU_IB_POOL_DIRECT] = GFP_ATOMIC
+	};
 	int r, i;
 
 	if (adev->ib_pool_ready)
@@ -358,8 +382,7 @@ int amdgpu_ib_pool_init(struct amdgpu_device *adev)
 
 	for (i = 0; i < AMDGPU_IB_POOL_MAX; i++) {
 		r = amdgpu_sa_bo_manager_init(adev, &adev->ib_pools[i],
-					      AMDGPU_IB_POOL_SIZE, 256,
-					      AMDGPU_GEM_DOMAIN_GTT);
+					      sizes[i], gfp_flags[i]);
 		if (r)
 			goto error;
 	}
@@ -393,6 +416,17 @@ void amdgpu_ib_pool_fini(struct amdgpu_device *adev)
 	adev->ib_pool_ready = false;
 }
 
+/**
+ * amdgpu_ib_pool_gfp_flags - Returns the gfp flags to use for each pool
+ * @adev: amdgpu device pointer
+ * @type: the IB pool type
+ */
+gfp_t amdgpu_ib_pool_gfp_flags(struct amdgpu_device *adev,
+			       enum amdgpu_ib_pool_type type)
+{
+	return adev->ib_pools[type].gfp_flags;
+}
+
 /**
  * amdgpu_ib_ring_tests - test IBs on the rings
  *
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
index a6419246e9c24..c2a10b41804af 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
@@ -75,13 +75,13 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih,
 	} else {
 		unsigned wptr_offs, rptr_offs;
 
-		r = amdgpu_device_wb_get(adev, &wptr_offs);
+		r = amdgpu_wb_get(adev, &wptr_offs);
 		if (r)
 			return r;
 
-		r = amdgpu_device_wb_get(adev, &rptr_offs);
+		r = amdgpu_wb_get(adev, &rptr_offs);
 		if (r) {
-			amdgpu_device_wb_free(adev, wptr_offs);
+			amdgpu_wb_free(adev, wptr_offs);
 			return r;
 		}
 
@@ -90,8 +90,8 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih,
 					    &ih->ring_obj, &ih->gpu_addr,
 					    (void **)&ih->ring);
 		if (r) {
-			amdgpu_device_wb_free(adev, rptr_offs);
-			amdgpu_device_wb_free(adev, wptr_offs);
+			amdgpu_wb_free(adev, rptr_offs);
+			amdgpu_wb_free(adev, wptr_offs);
 			return r;
 		}
 
@@ -131,8 +131,8 @@ void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih)
 	} else {
 		amdgpu_bo_free_kernel(&ih->ring_obj, &ih->gpu_addr,
 				      (void **)&ih->ring);
-		amdgpu_device_wb_free(adev, (ih->wptr_addr - ih->gpu_addr) / 4);
-		amdgpu_device_wb_free(adev, (ih->rptr_addr - ih->gpu_addr) / 4);
+		amdgpu_wb_free(adev, (ih->wptr_addr - ih->gpu_addr) / 4);
+		amdgpu_wb_free(adev, (ih->rptr_addr - ih->gpu_addr) / 4);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.c
index 6aa54156bbc97..922f4b15619d5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.c
@@ -369,43 +369,158 @@ int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
 }
 
 /**
- * amdgpu_device_ip_is_hw - is the hardware IP enabled
+ * amdgpu_device_ip_is_valid - is the hardware IP valid
  *
  * @adev: amdgpu_device pointer
  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
  *
- * Check if the hardware IP is enable or not.
- * Returns true if it the IP is enable, false if not.
+ * Check if the hardware IP is valid or not.
+ * Returns true if it the IP is valid, false if not.
  */
-bool amdgpu_device_ip_is_hw(struct amdgpu_device *adev,
-			    enum amd_ip_block_type block_type)
+bool amdgpu_device_ip_is_valid(struct amdgpu_device *adev,
+			       enum amd_ip_block_type block_type)
 {
 	struct amdgpu_ip_block *ip_block;
 
 	ip_block = amdgpu_device_ip_get_ip_block(adev, block_type);
 	if (ip_block)
-		return ip_block->status.hw;
+		return ip_block->status.valid;
 
 	return false;
 }
 
 /**
- * amdgpu_device_ip_is_valid - is the hardware IP valid
+ * amdgpu_ip_from_ring() - Find IP block type corresponding to ring type.
  *
- * @adev: amdgpu_device pointer
- * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
+ * @ring_type: The ring type whose IP block you are looking for.
+ */
+static enum amd_ip_block_type amdgpu_ip_from_ring(const enum amdgpu_ring_type ring_type)
+{
+	switch (ring_type) {
+	case AMDGPU_RING_TYPE_GFX:
+	case AMDGPU_RING_TYPE_COMPUTE:
+		return AMD_IP_BLOCK_TYPE_GFX;
+
+	case AMDGPU_RING_TYPE_SDMA:
+		return AMD_IP_BLOCK_TYPE_SDMA;
+
+	case AMDGPU_RING_TYPE_UVD:
+	case AMDGPU_RING_TYPE_UVD_ENC:
+		return AMD_IP_BLOCK_TYPE_UVD;
+
+	case AMDGPU_RING_TYPE_VCE:
+		return AMD_IP_BLOCK_TYPE_VCE;
+
+	case AMDGPU_RING_TYPE_VCN_DEC:
+	case AMDGPU_RING_TYPE_VCN_ENC:
+		return AMD_IP_BLOCK_TYPE_VCN;
+
+	case AMDGPU_RING_TYPE_VCN_JPEG:
+		return AMD_IP_BLOCK_TYPE_JPEG;
+
+	case AMDGPU_RING_TYPE_VPE:
+		return AMD_IP_BLOCK_TYPE_VPE;
+
+	default:
+		return AMD_IP_BLOCK_TYPE_NUM;
+	}
+}
+
+/**
+ * amdgpu_ring_mask_from_ip() - Find mask of ring types corresponding to an IP block type.
  *
- * Check if the hardware IP is valid or not.
- * Returns true if it the IP is valid, false if not.
+ * @ip_type: The IP block type whose rings you are looking for.
  */
-bool amdgpu_device_ip_is_valid(struct amdgpu_device *adev,
-			       enum amd_ip_block_type block_type)
+static u32 amdgpu_ring_mask_from_ip(const enum amd_ip_block_type ip_type)
+{
+	switch (ip_type) {
+	case AMD_IP_BLOCK_TYPE_GFX:
+		return BIT(AMDGPU_RING_TYPE_GFX) | BIT(AMDGPU_RING_TYPE_COMPUTE);
+
+	case AMD_IP_BLOCK_TYPE_SDMA:
+		return BIT(AMDGPU_RING_TYPE_SDMA);
+
+	case AMD_IP_BLOCK_TYPE_UVD:
+		return BIT(AMDGPU_RING_TYPE_UVD) | BIT(AMDGPU_RING_TYPE_UVD_ENC);
+
+	case AMD_IP_BLOCK_TYPE_VCE:
+		return BIT(AMD_IP_BLOCK_TYPE_VCE);
+
+	case AMD_IP_BLOCK_TYPE_VCN:
+		return BIT(AMDGPU_RING_TYPE_VCN_DEC) | BIT(AMDGPU_RING_TYPE_VCN_ENC);
+
+	case AMD_IP_BLOCK_TYPE_JPEG:
+		return BIT(AMDGPU_RING_TYPE_VCN_JPEG);
+
+	case AMD_IP_BLOCK_TYPE_VPE:
+		return BIT(AMDGPU_RING_TYPE_VPE);
+
+	default:
+		return 0;
+	}
+}
+
+/**
+ * amdgpu_device_ip_soft_reset() - Perform a graceful soft reset on an IP block.
+ *
+ * @guilty_ring: The ring which is guilty of causing a reset.
+ * @guilty_fence: The fence which didn't signal.
+ *
+ * IP block soft reset is used when attempting to recover
+ * from a GPU hang in a situation where a more fine grained
+ * reset type isn't available or didn't work. This effectively
+ * resets all rings that belong to the same device IP block
+ * and re-initializes the device IP block.
+ *
+ * The reset is handled gracefully, meaning that we try to
+ * minimize collateral damage (ie. avoid rejecting non-guilty jobs)
+ * as well as back up and restore the contents of all rings
+ * so that the system can move on from the hang.
+ */
+int amdgpu_device_ip_soft_reset(struct amdgpu_ring *guilty_ring,
+				struct amdgpu_fence *guilty_fence)
 {
+	struct amdgpu_device *adev = guilty_ring->adev;
 	struct amdgpu_ip_block *ip_block;
+	enum amd_ip_block_type ip_type;
+	u32 ring_type_mask;
+	int r;
 
-	ip_block = amdgpu_device_ip_get_ip_block(adev, block_type);
-	if (ip_block)
-		return ip_block->status.valid;
+	ip_type = amdgpu_ip_from_ring(guilty_ring->funcs->type);
+	ip_block = amdgpu_device_ip_get_ip_block(adev, ip_type);
 
-	return false;
+	if (unlikely(!ip_block)) {
+		dev_warn(adev->dev, "IP block not found for ring %s\n",
+			 guilty_ring->name);
+		return -EOPNOTSUPP;
+	}
+
+	if (!ip_block->version->funcs->soft_reset) {
+		dev_warn(adev->dev, "IP block soft reset not supported on %s\n",
+			 ip_block->version->funcs->name);
+		return -EOPNOTSUPP;
+	}
+
+	dev_err(adev->dev, "Starting %s IP block soft reset\n",
+		ip_block->version->funcs->name);
+
+	ring_type_mask = amdgpu_ring_mask_from_ip(ip_type);
+
+	amdgpu_device_lock_reset_domain(adev->reset_domain);
+	amdgpu_multi_ring_reset_helper_begin(ring_type_mask, guilty_ring, guilty_fence);
+
+	r = ip_block->version->funcs->soft_reset(ip_block);
+
+	r = amdgpu_multi_ring_reset_helper_end(ring_type_mask, guilty_ring, r);
+	amdgpu_device_unlock_reset_domain(adev->reset_domain);
+
+	if (r) {
+		dev_err(adev->dev, "Failed %s IP block soft reset: %d\n",
+			ip_block->version->funcs->name, r);
+		return r;
+	}
+
+	dev_err(adev->dev, "Successful %s IP block soft reset\n",
+		ip_block->version->funcs->name);
+	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.h
index 1d0df6d93957e..70fc4e5db51f3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.h
@@ -68,6 +68,7 @@ enum amd_hw_ip_block_type {
 	ISP_HWIP,
 	ATU_HWIP,
 	AIGC_HWIP,
+	UMSCH_HWIP,
 	MAX_HWIP
 };
 
@@ -84,6 +85,9 @@ enum amd_hw_ip_block_type {
 #define IP_VERSION_SUBREV(ver) ((ver) & 0xF)
 #define IP_VERSION_MAJ_MIN_REV(ver) ((ver) >> 8)
 
+struct amdgpu_ring;
+struct amdgpu_fence;
+
 struct amdgpu_ip_map_info {
 	/* Map of logical to actual dev instances/mask */
 	uint32_t dev_inst[MAX_HWIP][HWIP_MAX_INSTANCE];
@@ -146,9 +150,9 @@ void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
 					    u64 *flags);
 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
 				   enum amd_ip_block_type block_type);
-bool amdgpu_device_ip_is_hw(struct amdgpu_device *adev,
-			    enum amd_ip_block_type block_type);
 bool amdgpu_device_ip_is_valid(struct amdgpu_device *adev,
 			       enum amd_ip_block_type block_type);
+int amdgpu_device_ip_soft_reset(struct amdgpu_ring *guilty_ring,
+				struct amdgpu_fence *guilty_fence);
 
 #endif /* __AMDGPU_IP_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
index 53be764968e45..95cceed4e971a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
@@ -545,7 +545,7 @@ void amdgpu_irq_delegate(struct amdgpu_device *adev,
 			 unsigned int num_dw)
 {
 	amdgpu_ih_ring_write(adev, &adev->irq.ih_soft, entry->iv_entry, num_dw);
-	queue_work(system_unbound_wq, &adev->irq.ih_soft_work);
+	queue_work(system_dfl_wq, &adev->irq.ih_soft_work);
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 9ecc6387c1eb4..d97cf1212e0f9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -112,7 +112,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 		amdgpu_job_core_dump(adev, job);
 
 	if (amdgpu_gpu_recovery &&
-	    amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_SOFT_RESET) &&
+	    amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_SOFT_RECOVERY) &&
 	    amdgpu_ring_soft_recovery(ring, job->vmid, s_job->s_fence->parent)) {
 		dev_err(adev->dev, "ring %s timeout, but soft recovered\n",
 			s_job->sched->name);
@@ -151,6 +151,17 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 		dev_err(adev->dev, "Ring %s reset failed\n", ring->sched.name);
 	}
 
+	/* Attempt an IP block soft reset, if supported. */
+	if (amdgpu_gpu_recovery &&
+	    amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_IP_BLOCK_SOFT_RESET)) {
+		r = amdgpu_device_ip_soft_reset(ring, job->hw_fence);
+		if (!r) {
+			atomic_inc(&ring->adev->gpu_reset_counter);
+			drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, info);
+			goto exit;
+		}
+	}
+
 	if (dma_fence_get_status(&s_job->s_fence->finished) == 0)
 		dma_fence_set_error(&s_job->s_fence->finished, -ETIME);
 
@@ -187,8 +198,8 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 
 int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		     struct drm_sched_entity *entity, void *owner,
-		     unsigned int num_ibs, struct amdgpu_job **job,
-		     u64 drm_client_id)
+		     unsigned int num_ibs, u64 drm_client_id,
+		     gfp_t gfp_flags, struct amdgpu_job **job)
 {
 	struct amdgpu_fence *af;
 	int r;
@@ -196,18 +207,18 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	if (num_ibs == 0)
 		return -EINVAL;
 
-	*job = kzalloc_flex(**job, ibs, num_ibs);
+	*job = kzalloc_flex(**job, ibs, num_ibs, gfp_flags);
 	if (!*job)
 		return -ENOMEM;
 
-	af = kzalloc_obj(struct amdgpu_fence);
+	af = kzalloc_obj(struct amdgpu_fence, gfp_flags);
 	if (!af) {
 		r = -ENOMEM;
 		goto err_job;
 	}
 	(*job)->hw_fence = af;
 
-	af = kzalloc_obj(struct amdgpu_fence);
+	af = kzalloc_obj(struct amdgpu_fence, gfp_flags);
 	if (!af) {
 		r = -ENOMEM;
 		goto err_fence;
@@ -241,12 +252,13 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev,
 			     struct drm_sched_entity *entity, void *owner,
 			     size_t size, enum amdgpu_ib_pool_type pool_type,
-			     struct amdgpu_job **job, u64 k_job_id)
+			     u64 k_job_id, struct amdgpu_job **job)
 {
 	int r;
 
-	r = amdgpu_job_alloc(adev, NULL, entity, owner, 1, job,
-			     k_job_id);
+	r = amdgpu_job_alloc(adev, NULL, entity, owner, 1, k_job_id,
+			     amdgpu_ib_pool_gfp_flags(adev, pool_type),
+			     job);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
index e70a1117b8129..7e03bd2ff639d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
@@ -115,13 +115,13 @@ static inline struct amdgpu_ring *amdgpu_job_ring(struct amdgpu_job *job)
 
 int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		     struct drm_sched_entity *entity, void *owner,
-		     unsigned int num_ibs, struct amdgpu_job **job,
-		     u64 drm_client_id);
+		     unsigned int num_ibs, u64 drm_client_id,
+		     gfp_t gfp_flags, struct amdgpu_job **job);
 int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev,
 			     struct drm_sched_entity *entity, void *owner,
 			     size_t size, enum amdgpu_ib_pool_type pool_type,
-			     struct amdgpu_job **job,
-			     u64 k_job_id);
+			     u64 k_job_id,
+			     struct amdgpu_job **job);
 void amdgpu_job_set_resources(struct amdgpu_job *job, struct amdgpu_bo *gds,
 			      struct amdgpu_bo *gws, struct amdgpu_bo *oa);
 void amdgpu_job_free_resources(struct amdgpu_job *job);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c
index 63ee6ba6a9316..363b991684788 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c
@@ -134,8 +134,8 @@ void amdgpu_jpeg_ring_begin_use(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 
-	atomic_inc(&adev->jpeg.total_submission_cnt);
-	cancel_delayed_work_sync(&adev->jpeg.idle_work);
+	if (!atomic_fetch_inc(&adev->jpeg.total_submission_cnt))
+		cancel_delayed_work_sync(&adev->jpeg.idle_work);
 
 	mutex_lock(&adev->jpeg.jpeg_pg_lock);
 	amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_JPEG,
@@ -145,8 +145,9 @@ void amdgpu_jpeg_ring_begin_use(struct amdgpu_ring *ring)
 
 void amdgpu_jpeg_ring_end_use(struct amdgpu_ring *ring)
 {
-	atomic_dec(&ring->adev->jpeg.total_submission_cnt);
-	schedule_delayed_work(&ring->adev->jpeg.idle_work, JPEG_IDLE_TIMEOUT);
+	if (atomic_dec_and_test(&ring->adev->jpeg.total_submission_cnt))
+		schedule_delayed_work(&ring->adev->jpeg.idle_work,
+				      JPEG_IDLE_TIMEOUT);
 }
 
 int amdgpu_jpeg_dec_ring_test_ring(struct amdgpu_ring *ring)
@@ -196,8 +197,9 @@ static int amdgpu_jpeg_dec_set_reg(struct amdgpu_ring *ring, uint32_t handle,
 	int i, r;
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, ib_size_dw * 4,
-				     AMDGPU_IB_POOL_DIRECT, &job,
-				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.h
index 346ae0ab09d33..fe95d9188713b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.h
@@ -149,6 +149,9 @@ struct amdgpu_jpeg {
 	u32 *ip_dump;
 	u32 reg_count;
 	const struct amdgpu_hwip_reg_entry *reg_list;
+
+	bool disable_uq;
+	bool disable_kq;
 };
 
 int amdgpu_jpeg_sw_init(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 71272f40feef8..287ede40cd91f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -46,6 +46,7 @@
 #include "amdgpu_reset.h"
 #include "amd_pcie.h"
 #include "amdgpu_userq.h"
+#include "amdgpu_video_codecs.h"
 
 void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev)
 {
@@ -1423,6 +1424,33 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 	return 0;
 }
 
+/**
+ * amdgpu_proc_options_ioctl - set per-fd user options
+ *
+ * @dev: drm dev pointer
+ * @data: pointer to struct drm_amdgpu_proc_options
+ * @filp: drm file
+ *
+ * Sets options stored on the per-file amdgpu_fpriv. Currently the only
+ * supported option is %AMDGPU_PROC_OPTIONS_OP_KFD_SIGBUS_DELAY which
+ * controls how KFD delivers SIGBUS for poison/RAS events to the calling
+ * process (immediate, suppressed, or delayed by N milliseconds).
+ */
+int amdgpu_proc_options_ioctl(struct drm_device *dev, void *data,
+			      struct drm_file *filp)
+{
+	struct drm_amdgpu_proc_options *args = data;
+
+	switch (args->op) {
+	case AMDGPU_PROC_OPTIONS_OP_KFD_SIGBUS_DELAY:
+		return amdgpu_amdkfd_set_sigbus_delay(current,
+						      args->kfd_sigbus_delay.value);
+	default:
+		DRM_DEBUG_KMS("Invalid user option op %u\n", args->op);
+		return -EINVAL;
+	}
+}
+
 /**
  * amdgpu_driver_open_kms - drm callback for open
  *
@@ -1504,8 +1532,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 	if (r)
 		goto error_vm;
 
-	mutex_init(&fpriv->bo_list_lock);
-	idr_init_base(&fpriv->bo_list_handles, 1);
+	xa_init_flags(&fpriv->bo_list_handles, XA_FLAGS_ALLOC1);
 
 	r = amdgpu_userq_mgr_init(&fpriv->userq_mgr, file_priv, adev);
 	if (r)
@@ -1550,8 +1577,8 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
 	struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
 	struct amdgpu_bo_list *list;
 	struct amdgpu_bo *pd;
+	unsigned long handle;
 	u32 pasid;
-	int handle;
 
 	if (!fpriv)
 		return;
@@ -1587,11 +1614,9 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
 		amdgpu_pasid_free_delayed(pd->tbo.base.resv, pasid);
 	amdgpu_bo_unref(&pd);
 
-	idr_for_each_entry(&fpriv->bo_list_handles, list, handle)
+	xa_for_each(&fpriv->bo_list_handles, handle, list)
 		amdgpu_bo_list_put(list);
-
-	idr_destroy(&fpriv->bo_list_handles);
-	mutex_destroy(&fpriv->bo_list_lock);
+	xa_destroy(&fpriv->bo_list_handles);
 
 	kfree(fpriv);
 	file_priv->driver_priv = NULL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c
index cc6d1a4e4c3af..9a7f7d2b2767a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c
@@ -27,16 +27,6 @@
 #include "umc/umc_6_7_0_offset.h"
 #include "umc/umc_6_7_0_sh_mask.h"
 
-static bool amdgpu_mca_is_deferred_error(struct amdgpu_device *adev,
-					uint64_t mc_status)
-{
-	if (adev->umc.ras->check_ecc_err_status)
-		return adev->umc.ras->check_ecc_err_status(adev,
-				AMDGPU_MCA_ERROR_TYPE_DE, &mc_status);
-
-	return false;
-}
-
 void amdgpu_mca_query_correctable_error_count(struct amdgpu_device *adev,
 					      uint64_t mc_status_addr,
 					      unsigned long *error_count)
@@ -155,479 +145,3 @@ int amdgpu_mca_mpio_ras_sw_init(struct amdgpu_device *adev)
 
 	return 0;
 }
-
-static void amdgpu_mca_bank_set_init(struct mca_bank_set *mca_set)
-{
-	if (!mca_set)
-		return;
-
-	memset(mca_set, 0, sizeof(*mca_set));
-	INIT_LIST_HEAD(&mca_set->list);
-}
-
-static int amdgpu_mca_bank_set_add_entry(struct mca_bank_set *mca_set, struct mca_bank_entry *entry)
-{
-	struct mca_bank_node *node;
-
-	if (!entry)
-		return -EINVAL;
-
-	node = kvzalloc_obj(*node);
-	if (!node)
-		return -ENOMEM;
-
-	memcpy(&node->entry, entry, sizeof(*entry));
-
-	INIT_LIST_HEAD(&node->node);
-	list_add_tail(&node->node, &mca_set->list);
-
-	mca_set->nr_entries++;
-
-	return 0;
-}
-
-static int amdgpu_mca_bank_set_merge(struct mca_bank_set *mca_set, struct mca_bank_set *new)
-{
-	struct mca_bank_node *node;
-
-	list_for_each_entry(node, &new->list, node)
-		amdgpu_mca_bank_set_add_entry(mca_set, &node->entry);
-
-	return 0;
-}
-
-static void amdgpu_mca_bank_set_remove_node(struct mca_bank_set *mca_set, struct mca_bank_node *node)
-{
-	if (!node)
-		return;
-
-	list_del(&node->node);
-	kvfree(node);
-
-	mca_set->nr_entries--;
-}
-
-static void amdgpu_mca_bank_set_release(struct mca_bank_set *mca_set)
-{
-	struct mca_bank_node *node, *tmp;
-
-	if (list_empty(&mca_set->list))
-		return;
-
-	list_for_each_entry_safe(node, tmp, &mca_set->list, node)
-		amdgpu_mca_bank_set_remove_node(mca_set, node);
-}
-
-void amdgpu_mca_smu_init_funcs(struct amdgpu_device *adev, const struct amdgpu_mca_smu_funcs *mca_funcs)
-{
-	struct amdgpu_mca *mca = &adev->mca;
-
-	mca->mca_funcs = mca_funcs;
-}
-
-int amdgpu_mca_init(struct amdgpu_device *adev)
-{
-	struct amdgpu_mca *mca = &adev->mca;
-	struct mca_bank_cache *mca_cache;
-	int i;
-
-	atomic_set(&mca->ue_update_flag, 0);
-
-	for (i = 0; i < ARRAY_SIZE(mca->mca_caches); i++) {
-		mca_cache = &mca->mca_caches[i];
-		mutex_init(&mca_cache->lock);
-		amdgpu_mca_bank_set_init(&mca_cache->mca_set);
-	}
-
-	return 0;
-}
-
-void amdgpu_mca_fini(struct amdgpu_device *adev)
-{
-	struct amdgpu_mca *mca = &adev->mca;
-	struct mca_bank_cache *mca_cache;
-	int i;
-
-	atomic_set(&mca->ue_update_flag, 0);
-
-	for (i = 0; i < ARRAY_SIZE(mca->mca_caches); i++) {
-		mca_cache = &mca->mca_caches[i];
-		amdgpu_mca_bank_set_release(&mca_cache->mca_set);
-		mutex_destroy(&mca_cache->lock);
-	}
-}
-
-int amdgpu_mca_reset(struct amdgpu_device *adev)
-{
-	amdgpu_mca_fini(adev);
-
-	return amdgpu_mca_init(adev);
-}
-
-int amdgpu_mca_smu_set_debug_mode(struct amdgpu_device *adev, bool enable)
-{
-	const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
-
-	if (mca_funcs && mca_funcs->mca_set_debug_mode)
-		return mca_funcs->mca_set_debug_mode(adev, enable);
-
-	return -EOPNOTSUPP;
-}
-
-static void amdgpu_mca_smu_mca_bank_dump(struct amdgpu_device *adev, int idx, struct mca_bank_entry *entry,
-					 struct ras_query_context *qctx)
-{
-	u64 event_id = qctx ? qctx->evid.event_id : RAS_EVENT_INVALID_ID;
-
-	RAS_EVENT_LOG(adev, event_id, HW_ERR "Accelerator Check Architecture events logged\n");
-	RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].STATUS=0x%016llx\n",
-		      idx, entry->regs[MCA_REG_IDX_STATUS]);
-	RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].ADDR=0x%016llx\n",
-		      idx, entry->regs[MCA_REG_IDX_ADDR]);
-	RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].MISC0=0x%016llx\n",
-		      idx, entry->regs[MCA_REG_IDX_MISC0]);
-	RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].IPID=0x%016llx\n",
-		      idx, entry->regs[MCA_REG_IDX_IPID]);
-	RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].SYND=0x%016llx\n",
-		      idx, entry->regs[MCA_REG_IDX_SYND]);
-}
-
-static int amdgpu_mca_smu_get_valid_mca_count(struct amdgpu_device *adev, enum amdgpu_mca_error_type type, uint32_t *count)
-{
-	const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
-
-	if (!count)
-		return -EINVAL;
-
-	if (mca_funcs && mca_funcs->mca_get_valid_mca_count)
-		return mca_funcs->mca_get_valid_mca_count(adev, type, count);
-
-	return -EOPNOTSUPP;
-}
-
-static int amdgpu_mca_smu_get_mca_entry(struct amdgpu_device *adev, enum amdgpu_mca_error_type type,
-					int idx, struct mca_bank_entry *entry)
-{
-	const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
-	int count;
-
-	if (!mca_funcs || !mca_funcs->mca_get_mca_entry)
-		return -EOPNOTSUPP;
-
-	switch (type) {
-	case AMDGPU_MCA_ERROR_TYPE_UE:
-		count = mca_funcs->max_ue_count;
-		break;
-	case AMDGPU_MCA_ERROR_TYPE_CE:
-		count = mca_funcs->max_ce_count;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	if (idx >= count)
-		return -EINVAL;
-
-	return mca_funcs->mca_get_mca_entry(adev, type, idx, entry);
-}
-
-static bool amdgpu_mca_bank_should_update(struct amdgpu_device *adev, enum amdgpu_mca_error_type type)
-{
-	struct amdgpu_mca *mca = &adev->mca;
-	bool ret = true;
-
-	/*
-	 * Because the UE Valid MCA count will only be cleared after reset,
-	 * in order to avoid repeated counting of the error count,
-	 * the aca bank is only updated once during the gpu recovery stage.
-	 */
-	if (type == AMDGPU_MCA_ERROR_TYPE_UE) {
-		if (amdgpu_ras_intr_triggered())
-			ret = atomic_cmpxchg(&mca->ue_update_flag, 0, 1) == 0;
-		else
-			atomic_set(&mca->ue_update_flag, 0);
-	}
-
-	return ret;
-}
-
-static bool amdgpu_mca_bank_should_dump(struct amdgpu_device *adev, enum amdgpu_mca_error_type type,
-					struct mca_bank_entry *entry)
-{
-	bool ret;
-
-	switch (type) {
-	case AMDGPU_MCA_ERROR_TYPE_CE:
-		ret = amdgpu_mca_is_deferred_error(adev, entry->regs[MCA_REG_IDX_STATUS]);
-		break;
-	case AMDGPU_MCA_ERROR_TYPE_UE:
-	default:
-		ret = true;
-		break;
-	}
-
-	return ret;
-}
-
-static int amdgpu_mca_smu_get_mca_set(struct amdgpu_device *adev, enum amdgpu_mca_error_type type, struct mca_bank_set *mca_set,
-				      struct ras_query_context *qctx)
-{
-	struct mca_bank_entry entry;
-	uint32_t count = 0, i;
-	int ret;
-
-	if (!mca_set)
-		return -EINVAL;
-
-	if (!amdgpu_mca_bank_should_update(adev, type))
-		return 0;
-
-	ret = amdgpu_mca_smu_get_valid_mca_count(adev, type, &count);
-	if (ret)
-		return ret;
-
-	for (i = 0; i < count; i++) {
-		memset(&entry, 0, sizeof(entry));
-		ret = amdgpu_mca_smu_get_mca_entry(adev, type, i, &entry);
-		if (ret)
-			return ret;
-
-		amdgpu_mca_bank_set_add_entry(mca_set, &entry);
-
-		if (amdgpu_mca_bank_should_dump(adev, type, &entry))
-			amdgpu_mca_smu_mca_bank_dump(adev, i, &entry, qctx);
-	}
-
-	return 0;
-}
-
-static int amdgpu_mca_smu_parse_mca_error_count(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
-						enum amdgpu_mca_error_type type, struct mca_bank_entry *entry, uint32_t *count)
-{
-	const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
-
-	if (!count || !entry)
-		return -EINVAL;
-
-	if (!mca_funcs || !mca_funcs->mca_parse_mca_error_count)
-		return -EOPNOTSUPP;
-
-	return mca_funcs->mca_parse_mca_error_count(adev, blk, type, entry, count);
-}
-
-static int amdgpu_mca_dispatch_mca_set(struct amdgpu_device *adev, enum amdgpu_ras_block blk, enum amdgpu_mca_error_type type,
-				       struct mca_bank_set *mca_set, struct ras_err_data *err_data)
-{
-	struct amdgpu_smuio_mcm_config_info mcm_info;
-	struct mca_bank_node *node, *tmp;
-	struct mca_bank_entry *entry;
-	uint32_t count;
-	int ret;
-
-	if (!mca_set)
-		return -EINVAL;
-
-	if (!mca_set->nr_entries)
-		return 0;
-
-	list_for_each_entry_safe(node, tmp, &mca_set->list, node) {
-		entry = &node->entry;
-
-		count = 0;
-		ret = amdgpu_mca_smu_parse_mca_error_count(adev, blk, type, entry, &count);
-		if (ret && ret != -EOPNOTSUPP)
-			return ret;
-
-		if (!count)
-			continue;
-
-		memset(&mcm_info, 0, sizeof(mcm_info));
-
-		mcm_info.socket_id = entry->info.socket_id;
-		mcm_info.die_id = entry->info.aid;
-
-		if (type == AMDGPU_MCA_ERROR_TYPE_UE) {
-			amdgpu_ras_error_statistic_ue_count(err_data,
-							    &mcm_info, (uint64_t)count);
-		} else {
-			if (amdgpu_mca_is_deferred_error(adev, entry->regs[MCA_REG_IDX_STATUS]))
-				amdgpu_ras_error_statistic_de_count(err_data,
-								    &mcm_info, (uint64_t)count);
-			else
-				amdgpu_ras_error_statistic_ce_count(err_data,
-								    &mcm_info, (uint64_t)count);
-		}
-
-		amdgpu_mca_bank_set_remove_node(mca_set, node);
-	}
-
-	return 0;
-}
-
-static int amdgpu_mca_add_mca_set_to_cache(struct amdgpu_device *adev, enum amdgpu_mca_error_type type, struct mca_bank_set *new)
-{
-	struct mca_bank_cache *mca_cache = &adev->mca.mca_caches[type];
-	int ret;
-
-	mutex_lock(&mca_cache->lock);
-	ret = amdgpu_mca_bank_set_merge(&mca_cache->mca_set, new);
-	mutex_unlock(&mca_cache->lock);
-
-	return ret;
-}
-
-int amdgpu_mca_smu_log_ras_error(struct amdgpu_device *adev, enum amdgpu_ras_block blk, enum amdgpu_mca_error_type type,
-				 struct ras_err_data *err_data, struct ras_query_context *qctx)
-{
-	struct mca_bank_set mca_set;
-	struct mca_bank_cache *mca_cache = &adev->mca.mca_caches[type];
-	int ret;
-
-	amdgpu_mca_bank_set_init(&mca_set);
-
-	ret = amdgpu_mca_smu_get_mca_set(adev, type, &mca_set, qctx);
-	if (ret)
-		goto out_mca_release;
-
-	ret = amdgpu_mca_dispatch_mca_set(adev, blk, type, &mca_set, err_data);
-	if (ret)
-		goto out_mca_release;
-
-	/* add remain mca bank to mca cache */
-	if (mca_set.nr_entries) {
-		ret = amdgpu_mca_add_mca_set_to_cache(adev, type, &mca_set);
-		if (ret)
-			goto out_mca_release;
-	}
-
-	/* dispatch mca set again if mca cache has valid data */
-	mutex_lock(&mca_cache->lock);
-	if (mca_cache->mca_set.nr_entries)
-		ret = amdgpu_mca_dispatch_mca_set(adev, blk, type, &mca_cache->mca_set, err_data);
-	mutex_unlock(&mca_cache->lock);
-
-out_mca_release:
-	amdgpu_mca_bank_set_release(&mca_set);
-
-	return ret;
-}
-
-#if defined(CONFIG_DEBUG_FS)
-static int amdgpu_mca_smu_debug_mode_set(void *data, u64 val)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)data;
-	int ret;
-
-	ret = amdgpu_ras_set_mca_debug_mode(adev, val ? true : false);
-	if (ret)
-		return ret;
-
-	dev_info(adev->dev, "amdgpu set smu mca debug mode %s success\n", val ? "on" : "off");
-
-	return 0;
-}
-
-static void mca_dump_entry(struct seq_file *m, struct mca_bank_entry *entry)
-{
-	int i, idx = entry->idx;
-	int reg_idx_array[] = {
-		MCA_REG_IDX_STATUS,
-		MCA_REG_IDX_ADDR,
-		MCA_REG_IDX_MISC0,
-		MCA_REG_IDX_IPID,
-		MCA_REG_IDX_SYND,
-	};
-
-	seq_printf(m, "mca entry[%d].type: %s\n", idx, entry->type == AMDGPU_MCA_ERROR_TYPE_UE ? "UE" : "CE");
-	seq_printf(m, "mca entry[%d].ip: %d\n", idx, entry->ip);
-	seq_printf(m, "mca entry[%d].info: socketid:%d aid:%d hwid:0x%03x mcatype:0x%04x\n",
-		   idx, entry->info.socket_id, entry->info.aid, entry->info.hwid, entry->info.mcatype);
-
-	for (i = 0; i < ARRAY_SIZE(reg_idx_array); i++)
-		seq_printf(m, "mca entry[%d].regs[%d]: 0x%016llx\n", idx, reg_idx_array[i], entry->regs[reg_idx_array[i]]);
-}
-
-static int mca_dump_show(struct seq_file *m, enum amdgpu_mca_error_type type)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
-	struct mca_bank_node *node;
-	struct mca_bank_set mca_set;
-	struct ras_query_context qctx;
-	int ret;
-
-	amdgpu_mca_bank_set_init(&mca_set);
-
-	qctx.evid.event_id = RAS_EVENT_INVALID_ID;
-	ret = amdgpu_mca_smu_get_mca_set(adev, type, &mca_set, &qctx);
-	if (ret)
-		goto err_free_mca_set;
-
-	seq_printf(m, "amdgpu smu %s valid mca count: %d\n",
-		   type == AMDGPU_MCA_ERROR_TYPE_UE ? "UE" : "CE", mca_set.nr_entries);
-
-	if (!mca_set.nr_entries)
-		goto err_free_mca_set;
-
-	list_for_each_entry(node, &mca_set.list, node)
-		mca_dump_entry(m, &node->entry);
-
-	/* add mca bank to mca bank cache */
-	ret = amdgpu_mca_add_mca_set_to_cache(adev, type, &mca_set);
-
-err_free_mca_set:
-	amdgpu_mca_bank_set_release(&mca_set);
-
-	return ret;
-}
-
-static int mca_dump_ce_show(struct seq_file *m, void *unused)
-{
-	return mca_dump_show(m, AMDGPU_MCA_ERROR_TYPE_CE);
-}
-
-static int mca_dump_ce_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, mca_dump_ce_show, inode->i_private);
-}
-
-static const struct file_operations mca_ce_dump_debug_fops = {
-	.owner = THIS_MODULE,
-	.open = mca_dump_ce_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
-
-static int mca_dump_ue_show(struct seq_file *m, void *unused)
-{
-	return mca_dump_show(m, AMDGPU_MCA_ERROR_TYPE_UE);
-}
-
-static int mca_dump_ue_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, mca_dump_ue_show, inode->i_private);
-}
-
-static const struct file_operations mca_ue_dump_debug_fops = {
-	.owner = THIS_MODULE,
-	.open = mca_dump_ue_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
-
-DEFINE_DEBUGFS_ATTRIBUTE(mca_debug_mode_fops, NULL, amdgpu_mca_smu_debug_mode_set, "%llu\n");
-#endif
-
-void amdgpu_mca_smu_debugfs_init(struct amdgpu_device *adev, struct dentry *root)
-{
-#if defined(CONFIG_DEBUG_FS)
-	if (!root)
-		return;
-
-	debugfs_create_file("mca_debug_mode", 0200, root, adev, &mca_debug_mode_fops);
-	debugfs_create_file("mca_ue_dump", 0400, root, adev, &mca_ue_dump_debug_fops);
-	debugfs_create_file("mca_ce_dump", 0400, root, adev, &mca_ce_dump_debug_fops);
-#endif
-}
-
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.h
index e80323ff90c14..8ad47c5ab0221 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.h
@@ -23,49 +23,9 @@
 
 #include "amdgpu_ras.h"
 
-#define MCA_MAX_REGS_COUNT	(16)
-
-#define MCA_REG_FIELD(x, h, l)			(((x) & GENMASK_ULL(h, l)) >> l)
-#define MCA_REG__STATUS__VAL(x)			MCA_REG_FIELD(x, 63, 63)
-#define MCA_REG__STATUS__OVERFLOW(x)		MCA_REG_FIELD(x, 62, 62)
-#define MCA_REG__STATUS__UC(x)			MCA_REG_FIELD(x, 61, 61)
-#define MCA_REG__STATUS__EN(x)			MCA_REG_FIELD(x, 60, 60)
-#define MCA_REG__STATUS__MISCV(x)		MCA_REG_FIELD(x, 59, 59)
-#define MCA_REG__STATUS__ADDRV(x)		MCA_REG_FIELD(x, 58, 58)
-#define MCA_REG__STATUS__PCC(x)			MCA_REG_FIELD(x, 57, 57)
-#define MCA_REG__STATUS__ERRCOREIDVAL(x)	MCA_REG_FIELD(x, 56, 56)
-#define MCA_REG__STATUS__TCC(x)			MCA_REG_FIELD(x, 55, 55)
-#define MCA_REG__STATUS__SYNDV(x)		MCA_REG_FIELD(x, 53, 53)
-#define MCA_REG__STATUS__CECC(x)		MCA_REG_FIELD(x, 46, 46)
-#define MCA_REG__STATUS__UECC(x)		MCA_REG_FIELD(x, 45, 45)
-#define MCA_REG__STATUS__DEFERRED(x)		MCA_REG_FIELD(x, 44, 44)
-#define MCA_REG__STATUS__POISON(x)		MCA_REG_FIELD(x, 43, 43)
-#define MCA_REG__STATUS__SCRUB(x)		MCA_REG_FIELD(x, 40, 40)
-#define MCA_REG__STATUS__ERRCOREID(x)		MCA_REG_FIELD(x, 37, 32)
-#define MCA_REG__STATUS__ADDRLSB(x)		MCA_REG_FIELD(x, 29, 24)
-#define MCA_REG__STATUS__ERRORCODEEXT(x)	MCA_REG_FIELD(x, 21, 16)
-#define MCA_REG__STATUS__ERRORCODE(x)		MCA_REG_FIELD(x, 15, 0)
-
-#define MCA_REG__MISC0__ERRCNT(x)		MCA_REG_FIELD(x, 43, 32)
-
-#define MCA_REG__SYND__ERRORINFORMATION(x)	MCA_REG_FIELD(x, 17, 0)
-
-enum amdgpu_mca_ip {
-	AMDGPU_MCA_IP_UNKNOW = -1,
-	AMDGPU_MCA_IP_PSP = 0,
-	AMDGPU_MCA_IP_SDMA,
-	AMDGPU_MCA_IP_GC,
-	AMDGPU_MCA_IP_SMU,
-	AMDGPU_MCA_IP_MP5,
-	AMDGPU_MCA_IP_UMC,
-	AMDGPU_MCA_IP_PCS_XGMI,
-	AMDGPU_MCA_IP_COUNT,
-};
-
 enum amdgpu_mca_error_type {
 	AMDGPU_MCA_ERROR_TYPE_UE = 0,
 	AMDGPU_MCA_ERROR_TYPE_CE,
-	AMDGPU_MCA_ERROR_TYPE_DE,
 };
 
 struct amdgpu_mca_ras_block {
@@ -77,77 +37,20 @@ struct amdgpu_mca_ras {
 	struct amdgpu_mca_ras_block *ras;
 };
 
-struct mca_bank_set {
-	int nr_entries;
-	struct list_head list;
-};
-
-struct mca_bank_cache {
-	struct mca_bank_set mca_set;
-	struct mutex lock;
-};
-
 struct amdgpu_mca {
 	struct amdgpu_mca_ras mp0;
 	struct amdgpu_mca_ras mp1;
 	struct amdgpu_mca_ras mpio;
-	const struct amdgpu_mca_smu_funcs *mca_funcs;
-	struct mca_bank_cache mca_caches[AMDGPU_MCA_ERROR_TYPE_DE];
-	atomic_t ue_update_flag;
-};
-
-enum mca_reg_idx {
-	MCA_REG_IDX_STATUS		= 1,
-	MCA_REG_IDX_ADDR		= 2,
-	MCA_REG_IDX_MISC0		= 3,
-	MCA_REG_IDX_IPID		= 5,
-	MCA_REG_IDX_SYND		= 6,
-	MCA_REG_IDX_COUNT		= 16,
-};
-
-struct mca_bank_info {
-	int socket_id;
-	int aid;
-	int hwid;
-	int mcatype;
-};
-
-struct mca_bank_entry {
-	int idx;
-	enum amdgpu_mca_error_type type;
-	enum amdgpu_mca_ip ip;
-	struct mca_bank_info info;
-	uint64_t regs[MCA_MAX_REGS_COUNT];
-};
-
-struct mca_bank_node {
-	struct mca_bank_entry entry;
-	struct list_head node;
-};
-
-struct amdgpu_mca_smu_funcs {
-	int max_ue_count;
-	int max_ce_count;
-	int (*mca_set_debug_mode)(struct amdgpu_device *adev, bool enable);
-	int (*mca_parse_mca_error_count)(struct amdgpu_device *adev, enum amdgpu_ras_block blk, enum amdgpu_mca_error_type type,
-					 struct mca_bank_entry *entry, uint32_t *count);
-	int (*mca_get_valid_mca_count)(struct amdgpu_device *adev, enum amdgpu_mca_error_type type,
-				       uint32_t *count);
-	int (*mca_get_mca_entry)(struct amdgpu_device *adev, enum amdgpu_mca_error_type type,
-				 int idx, struct mca_bank_entry *entry);
 };
 
 void amdgpu_mca_query_correctable_error_count(struct amdgpu_device *adev,
 					      uint64_t mc_status_addr,
 					      unsigned long *error_count);
-
 void amdgpu_mca_query_uncorrectable_error_count(struct amdgpu_device *adev,
 						uint64_t mc_status_addr,
 						unsigned long *error_count);
-
 void amdgpu_mca_reset_error_count(struct amdgpu_device *adev,
 				  uint64_t mc_status_addr);
-
 void amdgpu_mca_query_ras_error_count(struct amdgpu_device *adev,
 				      uint64_t mc_status_addr,
 				      void *ras_error_status);
@@ -155,15 +58,4 @@ int amdgpu_mca_mp0_ras_sw_init(struct amdgpu_device *adev);
 int amdgpu_mca_mp1_ras_sw_init(struct amdgpu_device *adev);
 int amdgpu_mca_mpio_ras_sw_init(struct amdgpu_device *adev);
 
-void amdgpu_mca_smu_init_funcs(struct amdgpu_device *adev, const struct amdgpu_mca_smu_funcs *mca_funcs);
-int amdgpu_mca_init(struct amdgpu_device *adev);
-void amdgpu_mca_fini(struct amdgpu_device *adev);
-int amdgpu_mca_reset(struct amdgpu_device *adev);
-int amdgpu_mca_smu_set_debug_mode(struct amdgpu_device *adev, bool enable);
-int amdgpu_mca_smu_get_mca_set_error_count(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
-					   enum amdgpu_mca_error_type type, uint32_t *total);
-void amdgpu_mca_smu_debugfs_init(struct amdgpu_device *adev, struct dentry *root);
-int amdgpu_mca_smu_log_ras_error(struct amdgpu_device *adev, enum amdgpu_ras_block blk, enum amdgpu_mca_error_type type,
-				 struct ras_err_data *err_data, struct ras_query_context *qctx);
-
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index e3972673fd641..1e3a23c815523 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -183,7 +183,7 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 			 adev->mes.sdma_hqd_mask[0]);
 
 	for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) {
-		r = amdgpu_device_wb_get(adev, &adev->mes.sch_ctx_offs[i]);
+		r = amdgpu_wb_get(adev, &adev->mes.sch_ctx_offs[i]);
 		if (r) {
 			dev_err(adev->dev,
 				"(%d) ring trail_fence_offs wb alloc failed\n",
@@ -195,7 +195,7 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 		adev->mes.sch_ctx_ptr[i] =
 			(uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs[i]];
 
-		r = amdgpu_device_wb_get(adev,
+		r = amdgpu_wb_get(adev,
 				 &adev->mes.query_status_fence_offs[i]);
 		if (r) {
 			dev_err(adev->dev,
@@ -217,7 +217,7 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 	if (r)
 		goto error_doorbell;
 
-	if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 1, 0)) {
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) {
 		/* When queue/pipe reset is done in MES instead of in the
 		 * driver, MES passes hung queues information to the driver in
 		 * hung_queue_hqd_info. Calculate required space to store this
@@ -237,7 +237,7 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 	}
 
 	if (adev->mes.hung_queue_db_array_size) {
-		for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) {
+		for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) {
 			r = amdgpu_bo_create_kernel(adev,
 						    adev->mes.hung_queue_db_array_size * sizeof(u32),
 						    PAGE_SIZE,
@@ -250,6 +250,15 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 				goto error_doorbell;
 			}
 		}
+
+		adev->gfx.mec.mes_hung_db_array =
+			kcalloc(amdgpu_mes_get_hung_queue_db_array_size(adev),
+				sizeof(u32), GFP_KERNEL);
+
+		if (!adev->gfx.mec.mes_hung_db_array) {
+			r = -ENOMEM;
+			goto error_doorbell;
+		}
 	}
 
 	return 0;
@@ -259,9 +268,9 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 error:
 	for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) {
 		if (adev->mes.sch_ctx_ptr[i])
-			amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]);
+			amdgpu_wb_free(adev, adev->mes.sch_ctx_offs[i]);
 		if (adev->mes.query_status_fence_ptr[i])
-			amdgpu_device_wb_free(adev,
+			amdgpu_wb_free(adev,
 				      adev->mes.query_status_fence_offs[i]);
 		if (adev->mes.hung_queue_db_array_gpu_obj[i])
 			amdgpu_bo_free_kernel(&adev->mes.hung_queue_db_array_gpu_obj[i],
@@ -279,6 +288,8 @@ void amdgpu_mes_fini(struct amdgpu_device *adev)
 	int i;
 	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
 
+	kfree(adev->gfx.mec.mes_hung_db_array);
+
 	amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj,
 			      &adev->mes.event_log_gpu_addr,
 			      &adev->mes.event_log_cpu_addr);
@@ -289,9 +300,9 @@ void amdgpu_mes_fini(struct amdgpu_device *adev)
 					 &adev->mes.hung_queue_db_array_gpu_addr[i],
 					 &adev->mes.hung_queue_db_array_cpu_addr[i]);
 		if (adev->mes.sch_ctx_ptr[i])
-			amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]);
+			amdgpu_wb_free(adev, adev->mes.sch_ctx_offs[i]);
 		if (adev->mes.query_status_fence_ptr[i])
-			amdgpu_device_wb_free(adev,
+			amdgpu_wb_free(adev,
 				      adev->mes.query_status_fence_offs[i]);
 	}
 
@@ -439,6 +450,59 @@ int amdgpu_mes_reset_legacy_queue(struct amdgpu_device *adev,
 	return r;
 }
 
+int amdgpu_mes_reset_queue_mmio(struct amdgpu_device *adev,
+				int queue_type,
+				unsigned int vmid,
+				unsigned int me,
+				unsigned int pipe,
+				unsigned int queue,
+				uint32_t xcc_id)
+{
+	struct mes_reset_queue_input queue_input;
+	int r;
+
+	memset(&queue_input, 0, sizeof(queue_input));
+
+	queue_input.xcc_id = xcc_id;
+	queue_input.me_id = me;
+	queue_input.pipe_id = pipe;
+	queue_input.queue_id = queue;
+	queue_input.vmid = vmid;
+	queue_input.queue_type = queue_type;
+	queue_input.use_mmio = true;
+
+	amdgpu_mes_lock(&adev->mes);
+	r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input);
+	amdgpu_mes_unlock(&adev->mes);
+	if (r)
+		dev_err(adev->dev, "failed to reset legacy queue\n");
+
+	return r;
+}
+
+int amdgpu_mes_reset_user_queue(struct amdgpu_device *adev,
+				int queue_type,
+				unsigned int doorbell_index,
+				unsigned int xcc_id)
+{
+	struct mes_reset_queue_input queue_input;
+	int r;
+
+	memset(&queue_input, 0, sizeof(queue_input));
+
+	queue_input.xcc_id = xcc_id;
+	queue_input.queue_type = queue_type;
+	queue_input.doorbell_offset = doorbell_index;
+
+	amdgpu_mes_lock(&adev->mes);
+	r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input);
+	amdgpu_mes_unlock(&adev->mes);
+	if (r)
+		dev_err(adev->dev, "failed to reset user queue\n");
+
+	return r;
+}
+
 int amdgpu_mes_get_hung_queue_db_array_size(struct amdgpu_device *adev)
 {
 	return adev->mes.hung_queue_db_array_size;
@@ -508,7 +572,7 @@ uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg,
 	uint64_t read_val_gpu_addr;
 	uint32_t *read_val_ptr;
 
-	if (amdgpu_device_wb_get(adev, &addr_offset)) {
+	if (amdgpu_wb_get(adev, &addr_offset)) {
 		dev_err(adev->dev, "critical bug! too many mes readers\n");
 		goto error;
 	}
@@ -534,7 +598,7 @@ uint32_t amdgpu_mes_rreg(struct amdgpu_device *adev, uint32_t reg,
 
 error:
 	if (addr_offset)
-		amdgpu_device_wb_free(adev, addr_offset);
+		amdgpu_wb_free(adev, addr_offset);
 	return val;
 }
 
@@ -805,8 +869,13 @@ bool amdgpu_mes_suspend_resume_all_supported(struct amdgpu_device *adev)
 
 bool amdgpu_mes_queue_reset_by_mes_supported(struct amdgpu_device *adev)
 {
-	return (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) &&
-		(adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x73);
+	u32 ip_maj = IP_VERSION_MAJ(amdgpu_ip_version(adev, GC_HWIP, 0));
+	u32 ip_min = IP_VERSION_MIN(amdgpu_ip_version(adev, GC_HWIP, 0));
+	u32 mes_sched = adev->mes.sched_version & AMDGPU_MES_VERSION_MASK;
+
+	return (ip_maj == 11 && mes_sched >= 0x8c) ||
+		((ip_maj == 12 && ip_min == 0) && mes_sched >= 0x8d) ||
+		((ip_maj == 12 && ip_min == 1) && mes_sched >= 0x73);
 }
 
 /* Fix me -- node_id is used to identify the correct MES instances in the future */
@@ -852,6 +921,206 @@ int amdgpu_mes_update_enforce_isolation(struct amdgpu_device *adev)
 	return r;
 }
 
+/**
+ * amdgpu_mes_rs64mem_init - initialize RS64 local memory context arrays
+ *
+ * @mes: MES instance
+ *
+ * Returns 0 on success, negative errno on failure.
+ */
+int amdgpu_mes_rs64mem_init(struct amdgpu_mes *mes)
+{
+	struct amdgpu_device *adev = container_of(mes, struct amdgpu_device, mes);
+	int r;
+
+	if (!mes->use_rs64mem)
+		return 0;
+
+	r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE,
+				    AMDGPU_GEM_DOMAIN_GTT,
+				    &mes->ctx_array_size_bo,
+				    &mes->ctx_array_size_gpu_addr,
+				    (void **)&mes->ctx_array_size_cpu_ptr);
+	if (r) {
+		dev_err(adev->dev,
+			"Failed to allocate ctx array size BO, r=%d\n", r);
+		return r;
+	}
+
+	memset(mes->ctx_array_size_cpu_ptr, 0, PAGE_SIZE);
+
+	return 0;
+}
+
+/**
+ * amdgpu_mes_rs64mem_fini - tear down RS64 local memory management
+ */
+void amdgpu_mes_rs64mem_fini(struct amdgpu_mes *mes)
+{
+	if (mes->ctx_array_size_bo) {
+		amdgpu_bo_free_kernel(&mes->ctx_array_size_bo,
+				      &mes->ctx_array_size_gpu_addr,
+				      (void **)&mes->ctx_array_size_cpu_ptr);
+	}
+	bitmap_free(mes->proc_ctx_bitmap);
+	bitmap_free(mes->gang_ctx_bitmap);
+	mes->use_rs64mem = false;
+}
+
+/**
+ * amdgpu_mes_rs64mem_setup_bitmaps - allocate bitmaps after querying MES
+ *
+ * Called after QUERY_SCHEDULER_STATUS returns and MES has written
+ * the array sizes to the GPU buffer. Reads the sizes and allocates
+ * the tracking bitmaps.
+ *
+ * @mes: MES instance
+ *
+ * Returns 0 on success, negative errno on failure.
+ */
+int amdgpu_mes_rs64mem_setup_bitmaps(struct amdgpu_mes *mes)
+{
+	struct amdgpu_device *adev = container_of(mes, struct amdgpu_device, mes);
+
+	if (!mes->use_rs64mem || !mes->ctx_array_size_cpu_ptr)
+		return 0;
+
+	/*
+	 * MES FW wrote the sizes to the GPU buffer:
+	 *   ctx_array_size_cpu_ptr[0] = proc_ctx_array_size (N)
+	 *   ctx_array_size_cpu_ptr[1] = gang_ctx_array_size (M)
+	 */
+	mes->proc_ctx_array_size = mes->ctx_array_size_cpu_ptr[0];
+	mes->gang_ctx_array_size = mes->ctx_array_size_cpu_ptr[1];
+
+	/* Sanity check - MES FW typically returns N=50, M=300 */
+	if (mes->proc_ctx_array_size == 0 || mes->gang_ctx_array_size == 0) {
+		dev_warn(adev->dev,
+			 "MES returned zero ctx array sizes (proc=%u, gang=%u), "
+			 "disabling RS64 local memory optimization\n",
+			 mes->proc_ctx_array_size, mes->gang_ctx_array_size);
+		mes->use_rs64mem = false;
+		return 0;
+	}
+
+	/* Cap to safety limits */
+	if (mes->proc_ctx_array_size > AMDGPU_MES_PROC_CTX_ARRAY_MAX)
+		mes->proc_ctx_array_size = AMDGPU_MES_PROC_CTX_ARRAY_MAX;
+	if (mes->gang_ctx_array_size > AMDGPU_MES_GANG_CTX_ARRAY_MAX)
+		mes->gang_ctx_array_size = AMDGPU_MES_GANG_CTX_ARRAY_MAX;
+
+	dev_info(adev->dev,
+		 "MES RS64 local memory: proc_ctx_array_size:%u, "
+		 "gang_ctx_array_size:%u\n",
+		 mes->proc_ctx_array_size, mes->gang_ctx_array_size);
+
+	/* Allocate bitmaps */
+	mes->proc_ctx_bitmap = bitmap_zalloc(mes->proc_ctx_array_size,
+					     GFP_KERNEL);
+	if (!mes->proc_ctx_bitmap) {
+		mes->use_rs64mem = false;
+		return -ENOMEM;
+	}
+
+	mes->gang_ctx_bitmap = bitmap_zalloc(mes->gang_ctx_array_size,
+					     GFP_KERNEL);
+	if (!mes->gang_ctx_bitmap) {
+		bitmap_free(mes->proc_ctx_bitmap);
+		mes->proc_ctx_bitmap = NULL;
+		mes->use_rs64mem = false;
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/**
+ * amdgpu_mes_alloc_proc_ctx_index - allocate a process context slot
+ *
+ * @mes: MES instance
+ *
+ * Returns 0 on success, -ENOSPC if all slots are used (caller should
+ * fall back to system memory path).
+ */
+int amdgpu_mes_alloc_proc_ctx_index(struct amdgpu_mes *mes,
+				    struct amdgpu_usermode_queue *queue)
+{
+	unsigned long bit;
+
+	if (!mes->use_rs64mem || !mes->proc_ctx_bitmap)
+		return -EOPNOTSUPP;
+
+	amdgpu_mes_lock(mes);
+	bit = find_first_zero_bit(mes->proc_ctx_bitmap,
+				  mes->proc_ctx_array_size);
+	if (bit >= mes->proc_ctx_array_size) {
+		amdgpu_mes_unlock(mes);
+		return -ENOSPC;
+	}
+	set_bit(bit, mes->proc_ctx_bitmap);
+	queue->proc_ctx_array_index = (uint32_t)bit;
+	amdgpu_mes_unlock(mes);
+
+	return 0;
+}
+
+/**
+ * amdgpu_mes_free_proc_ctx_index - free a process context slot
+ */
+void amdgpu_mes_free_proc_ctx_index(struct amdgpu_mes *mes,
+				    struct amdgpu_usermode_queue *queue)
+{
+	if (!mes->use_rs64mem || !mes->proc_ctx_bitmap)
+		return;
+	if (queue->proc_ctx_array_index >= mes->proc_ctx_array_size)
+		return;
+
+	amdgpu_mes_lock(mes);
+	clear_bit(queue->proc_ctx_array_index, mes->proc_ctx_bitmap);
+	amdgpu_mes_unlock(mes);
+}
+
+/**
+ * amdgpu_mes_alloc_gang_ctx_index - allocate a gang context slot
+ */
+int amdgpu_mes_alloc_gang_ctx_index(struct amdgpu_mes *mes,
+				    struct amdgpu_usermode_queue *queue)
+{
+	unsigned long bit;
+
+	if (!mes->use_rs64mem || !mes->gang_ctx_bitmap)
+		return -EOPNOTSUPP;
+
+	amdgpu_mes_lock(mes);
+	bit = find_first_zero_bit(mes->gang_ctx_bitmap,
+				  mes->gang_ctx_array_size);
+	if (bit >= mes->gang_ctx_array_size) {
+		amdgpu_mes_unlock(mes);
+		return -ENOSPC;
+	}
+	set_bit(bit, mes->gang_ctx_bitmap);
+	queue->gang_ctx_array_index = bit;
+	amdgpu_mes_unlock(mes);
+
+	return 0;
+}
+
+/**
+ * amdgpu_mes_free_gang_ctx_index - free a gang context slot
+ */
+void amdgpu_mes_free_gang_ctx_index(struct amdgpu_mes *mes,
+				    struct amdgpu_usermode_queue *queue)
+{
+	if (!mes->use_rs64mem || !mes->gang_ctx_bitmap)
+		return;
+	if (queue->gang_ctx_array_index >= mes->gang_ctx_array_size)
+		return;
+
+	amdgpu_mes_lock(mes);
+	clear_bit(queue->gang_ctx_array_index, mes->gang_ctx_bitmap);
+	amdgpu_mes_unlock(mes);
+}
+
 #if defined(CONFIG_DEBUG_FS)
 
 static int amdgpu_debugfs_mes_event_log_show(struct seq_file *m, void *unused)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
index 1aae49f4df49b..c67db2d6e122f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
@@ -54,7 +54,8 @@ enum amdgpu_mes_priority_level {
 
 #define AMDGPU_MES_PROC_CTX_SIZE 0x1000 /* one page area */
 #define AMDGPU_MES_GANG_CTX_SIZE 0x1000 /* one page area */
-
+#define AMDGPU_MES_PROC_CTX_ARRAY_MAX  128
+#define AMDGPU_MES_GANG_CTX_ARRAY_MAX  512
 struct amdgpu_mes_funcs;
 
 enum amdgpu_mes_pipe {
@@ -168,6 +169,20 @@ struct amdgpu_mes {
 	int                 master_xcc_ids[AMDGPU_MAX_MES_INST_PIPES];
 	struct amdgpu_bo    *shared_cmd_buf_obj[AMDGPU_MAX_MES_INST_PIPES];
 	uint64_t            shared_cmd_buf_gpu_addr[AMDGPU_MAX_MES_INST_PIPES];
+
+	bool			compute_pipe_reset_enabled;
+	bool			gfx_pipe_reset_enabled;
+
+	bool				use_rs64mem;
+	struct amdgpu_bo		*ctx_array_size_bo;
+	uint64_t			ctx_array_size_gpu_addr;
+	uint32_t			*ctx_array_size_cpu_ptr;
+
+	uint32_t			proc_ctx_array_size;
+	unsigned long			*proc_ctx_bitmap;
+	uint32_t			gang_ctx_array_size;
+	uint32_t			gang_ctx_array_index;
+	unsigned long			*gang_ctx_bitmap;
 };
 
 struct amdgpu_mes_hung_queue_hqd_info {
@@ -265,13 +280,17 @@ struct mes_add_queue_input {
 	uint32_t	exclusively_scheduled;
 	uint32_t	sh_mem_config_data;
 	uint32_t	vm_cntx_cntl;
+	uint32_t	process_context_array_index;
+	uint32_t	gang_context_array_index;
 };
 
 struct mes_remove_queue_input {
 	uint32_t        xcc_id;
 	uint32_t	doorbell_offset;
 	uint64_t	gang_context_addr;
+	uint32_t	queue_type;
 	bool		remove_queue_after_reset;
+	uint32_t	gang_context_array_index;
 };
 
 struct mes_map_legacy_queue_input {
@@ -437,6 +456,52 @@ struct amdgpu_mes_funcs {
 			      struct mes_inv_tlbs_pasid_input *input);
 };
 
+enum amdgpu_mqd_update_flag {
+	AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE = 1,
+	AMDGPU_UPDATE_FLAG_DBG_WA_DISABLE = 2,
+	AMDGPU_UPDATE_FLAG_IS_GWS = 4, /* quirk for gfx9 IP */
+};
+
+struct amdgpu_mqd_prop {
+	uint64_t mqd_gpu_addr;
+	uint64_t hqd_base_gpu_addr;
+	uint64_t rptr_gpu_addr;
+	uint64_t wptr_gpu_addr;
+	uint32_t queue_size;
+	bool use_doorbell;
+	uint32_t doorbell_index;
+	uint64_t eop_gpu_addr;
+	uint32_t hqd_pipe_priority;
+	uint32_t hqd_queue_priority;
+	uint32_t mqd_stride_size;
+	bool allow_tunneling;
+	bool hqd_active;
+	uint64_t shadow_addr;
+	uint64_t gds_bkup_addr;
+	uint64_t csa_addr;
+	uint64_t fence_address;
+	bool tmz_queue;
+	bool kernel_queue;
+	uint32_t *cu_mask;
+	uint32_t cu_mask_count;
+	uint32_t cu_flags;
+	bool is_user_cu_masked;
+};
+
+struct amdgpu_mqd {
+	unsigned mqd_size;
+	int (*init_mqd)(struct amdgpu_device *adev, void *mqd,
+			struct amdgpu_mqd_prop *p);
+};
+
+/*
+ * MES FW uses address(mqd_addr + sizeof(struct mqd) + 3*sizeof(uint32_t))
+ * as fence address and writes a 32 bit fence value to this address.
+ * Driver needs to allocate at least 4 DWs extra memory in addition to
+ * sizeof(struct mqd). Add 8 DWs and align to AMDGPU_GPU_PAGE_SIZE for safety.
+ */
+#define AMDGPU_MQD_SIZE_ALIGN(mqd_size) AMDGPU_GPU_PAGE_ALIGN(((mqd_size) + 32))
+
 #define amdgpu_mes_kiq_hw_init(adev, xcc_id) \
 	(adev)->mes.kiq_hw_init((adev), (xcc_id))
 #define amdgpu_mes_kiq_hw_fini(adev, xcc_id) \
@@ -461,6 +526,17 @@ int amdgpu_mes_reset_legacy_queue(struct amdgpu_device *adev,
 				  unsigned int vmid,
 				  bool use_mmio,
 				  uint32_t xcc_id);
+int amdgpu_mes_reset_queue_mmio(struct amdgpu_device *adev,
+				int queue_type,
+				unsigned int vmid,
+				unsigned int me,
+				unsigned int pipe,
+				unsigned int queue,
+				uint32_t xcc_id);
+int amdgpu_mes_reset_user_queue(struct amdgpu_device *adev,
+				int queue_type,
+				unsigned int doorbell_index,
+				unsigned int xcc_id);
 
 int amdgpu_mes_get_hung_queue_db_array_size(struct amdgpu_device *adev);
 int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
@@ -556,4 +632,15 @@ bool amdgpu_mes_queue_reset_by_mes_supported(struct amdgpu_device *adev);
 
 int amdgpu_mes_update_enforce_isolation(struct amdgpu_device *adev);
 
+int amdgpu_mes_rs64mem_init(struct amdgpu_mes *mes);
+void amdgpu_mes_rs64mem_fini(struct amdgpu_mes *mes);
+int amdgpu_mes_rs64mem_setup_bitmaps(struct amdgpu_mes *mes);
+int amdgpu_mes_alloc_proc_ctx_index(struct amdgpu_mes *mes,
+				    struct amdgpu_usermode_queue *queue);
+void amdgpu_mes_free_proc_ctx_index(struct amdgpu_mes *mes,
+				    struct amdgpu_usermode_queue *queue);
+int amdgpu_mes_alloc_gang_ctx_index(struct amdgpu_mes *mes,
+				    struct amdgpu_usermode_queue *queue);
+void amdgpu_mes_free_gang_ctx_index(struct amdgpu_mes *mes,
+				    struct amdgpu_usermode_queue *queue);
 #endif /* __AMDGPU_MES_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h
index 6b8214650e5d0..c5120ba51e248 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h
@@ -21,29 +21,6 @@
 #ifndef __AMDGPU_MMHUB_H__
 #define __AMDGPU_MMHUB_H__
 
-enum amdgpu_mmhub_ras_memory_id {
-	AMDGPU_MMHUB_WGMI_PAGEMEM = 0,
-	AMDGPU_MMHUB_RGMI_PAGEMEM = 1,
-	AMDGPU_MMHUB_WDRAM_PAGEMEM = 2,
-	AMDGPU_MMHUB_RDRAM_PAGEMEM = 3,
-	AMDGPU_MMHUB_WIO_CMDMEM = 4,
-	AMDGPU_MMHUB_RIO_CMDMEM = 5,
-	AMDGPU_MMHUB_WGMI_CMDMEM = 6,
-	AMDGPU_MMHUB_RGMI_CMDMEM = 7,
-	AMDGPU_MMHUB_WDRAM_CMDMEM = 8,
-	AMDGPU_MMHUB_RDRAM_CMDMEM = 9,
-	AMDGPU_MMHUB_MAM_DMEM0 = 10,
-	AMDGPU_MMHUB_MAM_DMEM1 = 11,
-	AMDGPU_MMHUB_MAM_DMEM2 = 12,
-	AMDGPU_MMHUB_MAM_DMEM3 = 13,
-	AMDGPU_MMHUB_WRET_TAGMEM = 19,
-	AMDGPU_MMHUB_RRET_TAGMEM = 20,
-	AMDGPU_MMHUB_WIO_DATAMEM = 21,
-	AMDGPU_MMHUB_WGMI_DATAMEM = 22,
-	AMDGPU_MMHUB_WDRAM_DATAMEM = 23,
-	AMDGPU_MMHUB_MEMORY_BLOCK_LAST,
-};
-
 struct amdgpu_mmhub_ras {
 	struct amdgpu_ras_block_object ras_block;
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index f98bfba59a2ce..d4a9d5e8fb429 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -276,10 +276,12 @@ int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
 		goto error_free;
 	}
 
-	r = amdgpu_bo_pin(*bo_ptr, domain);
-	if (r) {
-		dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
-		goto error_unreserve;
+	if (free) {
+		r = amdgpu_bo_pin(*bo_ptr, domain);
+		if (r) {
+			dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
+			goto error_unreserve;
+		}
 	}
 
 	r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo);
@@ -302,7 +304,8 @@ int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
 	return 0;
 
 error_unpin:
-	amdgpu_bo_unpin(*bo_ptr);
+	if (free)
+		amdgpu_bo_unpin(*bo_ptr);
 error_unreserve:
 	amdgpu_bo_unreserve(*bo_ptr);
 
@@ -1696,8 +1699,9 @@ u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m)
 	if (dma_resv_trylock(bo->tbo.base.resv)) {
 		dma_resv_describe(bo->tbo.base.resv, m);
 		dma_resv_unlock(bo->tbo.base.resv);
+	} else {
+		seq_puts(m, "\n");
 	}
-	seq_puts(m, "\n");
 
 	return size;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 4d68732d62235..ff11a09034997 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -312,46 +312,6 @@ uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo);
 uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,
 					    uint32_t domain);
 
-/*
- * sub allocation
- */
-static inline struct amdgpu_sa_manager *
-to_amdgpu_sa_manager(struct drm_suballoc_manager *manager)
-{
-	return container_of(manager, struct amdgpu_sa_manager, base);
-}
-
-static inline uint64_t amdgpu_sa_bo_gpu_addr(struct drm_suballoc *sa_bo)
-{
-	return to_amdgpu_sa_manager(sa_bo->manager)->gpu_addr +
-		drm_suballoc_soffset(sa_bo);
-}
-
-static inline void *amdgpu_sa_bo_cpu_addr(struct drm_suballoc *sa_bo)
-{
-	return to_amdgpu_sa_manager(sa_bo->manager)->cpu_ptr +
-		drm_suballoc_soffset(sa_bo);
-}
-
-int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
-				     struct amdgpu_sa_manager *sa_manager,
-				     unsigned size, u32 align, u32 domain);
-void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
-				      struct amdgpu_sa_manager *sa_manager);
-int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev,
-				      struct amdgpu_sa_manager *sa_manager);
-int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
-		     struct drm_suballoc **sa_bo,
-		     unsigned int size);
-void amdgpu_sa_bo_free(struct drm_suballoc **sa_bo,
-		       struct dma_fence *fence);
-#if defined(CONFIG_DEBUG_FS)
-void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
-					 struct seq_file *m);
-u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m);
-#endif
-void amdgpu_debugfs_sa_init(struct amdgpu_device *adev);
-
 bool amdgpu_bo_support_uswc(u64 bo_flags);
 
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_preempt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_preempt_mgr.c
index b1dc33301d83b..e8592970aaabf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_preempt_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_preempt_mgr.c
@@ -46,6 +46,17 @@ static ssize_t mem_info_preempt_used_show(struct device *dev,
 
 static DEVICE_ATTR_RO(mem_info_preempt_used);
 
+/**
+ * amdgpu_preempt_mgr_sysfs_fini - remove PREEMPT manager sysfs attributes
+ *
+ * @adev: amdgpu_device pointer
+ */
+void amdgpu_preempt_mgr_sysfs_fini(struct amdgpu_device *adev)
+{
+	if (adev->dev->kobj.sd)
+		device_remove_file(adev->dev, &dev_attr_mem_info_preempt_used);
+}
+
 /**
  * amdgpu_preempt_mgr_new - allocate a new node
  *
@@ -137,9 +148,6 @@ void amdgpu_preempt_mgr_fini(struct amdgpu_device *adev)
 	if (ret)
 		return;
 
-	if (adev->dev->kobj.sd)
-		device_remove_file(adev->dev, &dev_attr_mem_info_preempt_used);
-
 	ttm_resource_manager_cleanup(man);
 	ttm_set_driver_manager(&adev->mman.bdev, AMDGPU_PL_PREEMPT, NULL);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index e0c0d7872e452..fdcba94ec0d15 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -1222,6 +1222,33 @@ int psp_memory_partition(struct psp_context *psp, int mode)
 	return ret;
 }
 
+int psp_set_mmhub_eco_sec_level(struct amdgpu_device *adev)
+{
+	int ret;
+	struct psp_context *psp = &adev->psp;
+	struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
+
+	cmd->cmd_id = GFX_CMD_ID_SET_MMHUB_ECO_SEC_LEVEL;
+
+	ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
+	if (ret) {
+		dev_err(psp->adev->dev,
+			"PSP request failed to set mmuhub eco sec level with ret=%d\n", ret);
+		release_psp_cmd_buf(psp);
+		return ret;
+	}
+
+	if (cmd->resp.status) {
+		dev_err(psp->adev->dev,
+			"MMHUB ECO SEC LEVEL command 0x%x failed, PSP response status: 0x%X\n",
+				cmd->cmd_id, cmd->resp.status);
+		ret = -EIO;
+	}
+	release_psp_cmd_buf(psp);
+
+	return ret;
+}
+
 static int psp_ptl_fmt_verify(struct psp_context *psp, enum amdgpu_ptl_fmt fmt,
 						 uint32_t *ptl_fmt)
 {
@@ -1861,6 +1888,12 @@ int psp_xgmi_initialize(struct psp_context *psp, bool set_extended_data, bool lo
 	/* note down the capbility flag for XGMI TA */
 	psp->xgmi_context.xgmi_ta_caps = xgmi_cmd->caps_flag;
 
+	if (!amdgpu_sriov_vf(psp->adev))
+		psp->xgmi_context.supports_ext_link_info = psp->xgmi_context.xgmi_ta_caps &
+			EXTEND_PEER_LINK_INFO_CMD_FLAG;
+	else
+		psp->xgmi_context.supports_ext_link_info = amdgpu_sriov_xgmi_ta_ext_peer_link_en(psp->adev);
+
 	return ret;
 }
 
@@ -2038,15 +2071,13 @@ int psp_xgmi_get_topology_info(struct psp_context *psp,
 			amdgpu_ip_version(psp->adev, MP0_HWIP, 0) ==
 				IP_VERSION(13, 0, 14) ||
 			amdgpu_sriov_vf(psp->adev);
-		bool ta_port_num_support = psp->xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG ||
-			amdgpu_sriov_xgmi_ta_ext_peer_link_en(psp->adev);
 
 		/* popluate the shared output buffer rather than the cmd input buffer
 		 * with node_ids as the input for GET_PEER_LINKS command execution.
 		 * This is required for GET_PEER_LINKS per xgmi ta implementation.
 		 * The same requirement for GET_EXTEND_PEER_LINKS command.
 		 */
-		if (ta_port_num_support) {
+		if (psp->xgmi_context.supports_ext_link_info) {
 			link_extend_info_output = &xgmi_cmd->xgmi_out_message.get_extend_link_info;
 
 			for (i = 0; i < topology->num_nodes; i++)
@@ -2069,7 +2100,7 @@ int psp_xgmi_get_topology_info(struct psp_context *psp,
 			return ret;
 
 		for (i = 0; i < topology->num_nodes; i++) {
-			uint8_t node_num_links = ta_port_num_support ?
+			uint8_t node_num_links = psp->xgmi_context.supports_ext_link_info ?
 				link_extend_info_output->nodes[i].num_links : link_info_output->nodes[i].num_links;
 			/* accumulate num_links on extended data */
 			if (get_extended_data) {
@@ -2079,7 +2110,7 @@ int psp_xgmi_get_topology_info(struct psp_context *psp,
 								topology->nodes[i].num_links : node_num_links;
 			}
 			/* popluate the connected port num info if supported and available */
-			if (ta_port_num_support && topology->nodes[i].num_links) {
+			if (psp->xgmi_context.supports_ext_link_info && topology->nodes[i].num_links) {
 				memcpy(topology->nodes[i].port_num, link_extend_info_output->nodes[i].port_num,
 				       sizeof(struct xgmi_connected_port_num) * TA_XGMI__MAX_PORT_NUM);
 			}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
index 2d838b1b2b116..d80c85793e3b9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
@@ -232,6 +232,7 @@ struct psp_xgmi_context {
 	struct ta_context		context;
 	struct psp_xgmi_topology_info	top_info;
 	bool				supports_extended_data;
+	bool				supports_ext_link_info;
 	uint8_t				xgmi_ta_caps;
 };
 
@@ -656,5 +657,6 @@ int amdgpu_psp_reg_program_no_ring(struct psp_context *psp, uint32_t val,
 void amdgpu_psp_debugfs_init(struct amdgpu_device *adev);
 int amdgpu_psp_get_fw_type(struct amdgpu_firmware_info *ucode,
 			   enum psp_gfx_fw_type *type);
+int psp_set_mmhub_eco_sec_level(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c
index 0d3c18f04ac36..8ae72c862d112 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c
@@ -166,7 +166,8 @@ static ssize_t ta_if_load_debugfs_write(struct file *fp, const char *buf, size_t
 	if (ret)
 		return -EFAULT;
 
-	if (ta_bin_len > PSP_1_MEG)
+	if (ta_bin_len < sizeof(struct common_firmware_header) ||
+	    ta_bin_len > PSP_1_MEG)
 		return -EINVAL;
 
 	copy_pos += sizeof(uint32_t);
@@ -321,6 +322,8 @@ static ssize_t ta_if_invoke_debugfs_write(struct file *fp, const char *buf, size
 	ret = copy_from_user((void *)&shared_buf_len, &buf[copy_pos], sizeof(uint32_t));
 	if (ret)
 		return -EFAULT;
+	if (!shared_buf_len || shared_buf_len > PSP_1_MEG)
+		return -EINVAL;
 	copy_pos += sizeof(uint32_t);
 
 	shared_buf = memdup_user(&buf[copy_pos], shared_buf_len);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
index 764cd49504083..24fd24e8d874f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
@@ -128,12 +128,6 @@ const char *get_ras_block_str(struct ras_common_if *ras_block)
 /* typical ECC bad page rate is 1 bad page per 100MB VRAM */
 #define RAS_BAD_PAGE_COVER              (100 * 1024 * 1024ULL)
 
-#define MAX_UMC_POISON_POLLING_TIME_ASYNC  10
-
-#define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100  //ms
-
-#define MAX_FLUSH_RETIRE_DWORK_TIMES  100
-
 #define BYPASS_ALLOCATED_ADDRESS        0x0
 #define BYPASS_INITIALIZATION_ADDRESS   0x1
 
@@ -249,16 +243,12 @@ static int amdgpu_check_address_validity(struct amdgpu_device *adev,
 	    (address >= RAS_UMC_INJECT_ADDR_LIMIT))
 		return -EFAULT;
 
-	if (amdgpu_uniras_enabled(adev)) {
-		if (amdgpu_sriov_vf(adev))
-			count = amdgpu_virt_ras_convert_retired_address(adev, address,
-				page_pfns, ARRAY_SIZE(page_pfns));
-		else
-			count = amdgpu_ras_mgr_lookup_bad_pages_in_a_row(adev, address,
-				page_pfns, ARRAY_SIZE(page_pfns));
-	} else
-		count = amdgpu_umc_lookup_bad_pages_in_a_row(adev,
-				address, page_pfns, ARRAY_SIZE(page_pfns));
+	if (amdgpu_sriov_vf(adev))
+		count = amdgpu_virt_ras_convert_retired_address(adev, address,
+			page_pfns, ARRAY_SIZE(page_pfns));
+	else
+		count = amdgpu_ras_mgr_lookup_bad_pages_in_a_row(adev, address,
+			page_pfns, ARRAY_SIZE(page_pfns));
 
 	if (count <= 0)
 		return -EPERM;
@@ -606,8 +596,14 @@ static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f,
 		ret = amdgpu_ras_feature_enable(adev, &data.head, 1);
 		break;
 	case 2:
-		/* umc ce/ue error injection for a bad page is not allowed */
-		if (data.head.block == AMDGPU_RAS_BLOCK__UMC)
+		/*
+		 * UMC ce/ue error injection for a bad page is not allowed. For
+		 * uniras (SMU v13+) devices the injection address is validated by
+		 * the ras_mgr inject handler, so only run the legacy bad page
+		 * check for the legacy RAS path.
+		 */
+		if (data.head.block == AMDGPU_RAS_BLOCK__UMC &&
+		    !amdgpu_uniras_enabled(adev))
 			ret = amdgpu_ras_check_bad_page(adev, data.inject.address);
 		if (ret == -EINVAL) {
 			dev_warn(adev->dev, "RAS WARN: input address 0x%llx is invalid.",
@@ -1147,8 +1143,7 @@ static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev,
 					      struct ras_err_data *err_data,
 					      struct ras_query_context *qctx,
 					      const char *blk_name,
-					      bool is_ue,
-					      bool is_de)
+					      bool is_ue)
 {
 	struct amdgpu_smuio_mcm_config_info *mcm_info;
 	struct ras_err_node *err_node;
@@ -1178,53 +1173,29 @@ static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev,
 		}
 
 	} else {
-		if (is_de) {
-			for_each_ras_error(err_node, err_data) {
-				err_info = &err_node->err_info;
-				mcm_info = &err_info->mcm_info;
-				if (err_info->de_count) {
-					RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
-						      "%lld new deferred hardware errors detected in %s block\n",
-						      mcm_info->socket_id,
-						      mcm_info->die_id,
-						      err_info->de_count,
-						      blk_name);
-				}
-			}
+		if (adev->debug_disable_ce_logs)
+			return;
 
-			for_each_ras_error(err_node, &ras_mgr->err_data) {
-				err_info = &err_node->err_info;
-				mcm_info = &err_info->mcm_info;
+		for_each_ras_error(err_node, err_data) {
+			err_info = &err_node->err_info;
+			mcm_info = &err_info->mcm_info;
+			if (err_info->ce_count) {
 				RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
-					      "%lld deferred hardware errors detected in total in %s block\n",
-					      mcm_info->socket_id, mcm_info->die_id,
-					      err_info->de_count, blk_name);
-			}
-		} else {
-			if (adev->debug_disable_ce_logs)
-				return;
-
-			for_each_ras_error(err_node, err_data) {
-				err_info = &err_node->err_info;
-				mcm_info = &err_info->mcm_info;
-				if (err_info->ce_count) {
-					RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
-						      "%lld new correctable hardware errors detected in %s block\n",
-						      mcm_info->socket_id,
-						      mcm_info->die_id,
-						      err_info->ce_count,
-						      blk_name);
-				}
+					      "%lld new correctable hardware errors detected in %s block\n",
+					      mcm_info->socket_id,
+					      mcm_info->die_id,
+					      err_info->ce_count,
+					      blk_name);
 			}
+		}
 
-			for_each_ras_error(err_node, &ras_mgr->err_data) {
-				err_info = &err_node->err_info;
-				mcm_info = &err_info->mcm_info;
-				RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
-					      "%lld correctable hardware errors detected in total in %s block\n",
-					      mcm_info->socket_id, mcm_info->die_id,
-					      err_info->ce_count, blk_name);
-			}
+		for_each_ras_error(err_node, &ras_mgr->err_data) {
+			err_info = &err_node->err_info;
+			mcm_info = &err_info->mcm_info;
+			RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
+				      "%lld correctable hardware errors detected in total in %s block\n",
+				      mcm_info->socket_id, mcm_info->die_id,
+				      err_info->ce_count, blk_name);
 		}
 	}
 }
@@ -1245,8 +1216,7 @@ static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev,
 
 	if (err_data->ce_count) {
 		if (err_data_has_source_info(err_data)) {
-			amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
-							  blk_name, false, false);
+			amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, blk_name, false);
 		} else if (!adev->aid_mask &&
 			   adev->smuio.funcs &&
 			   adev->smuio.funcs->get_socket_id &&
@@ -1268,8 +1238,7 @@ static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev,
 
 	if (err_data->ue_count) {
 		if (err_data_has_source_info(err_data)) {
-			amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
-							  blk_name, true, false);
+			amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx, blk_name, true);
 		} else if (!adev->aid_mask &&
 			   adev->smuio.funcs &&
 			   adev->smuio.funcs->get_socket_id &&
@@ -1288,29 +1257,6 @@ static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev,
 				      blk_name);
 		}
 	}
-
-	if (err_data->de_count) {
-		if (err_data_has_source_info(err_data)) {
-			amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
-							  blk_name, false, true);
-		} else if (!adev->aid_mask &&
-			   adev->smuio.funcs &&
-			   adev->smuio.funcs->get_socket_id &&
-			   adev->smuio.funcs->get_die_id) {
-			RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
-				      "%ld deferred hardware errors "
-				      "detected in %s block\n",
-				      adev->smuio.funcs->get_socket_id(adev),
-				      adev->smuio.funcs->get_die_id(adev),
-				      ras_mgr->err_data.de_count,
-				      blk_name);
-		} else {
-			RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors "
-				      "detected in %s block\n",
-				      ras_mgr->err_data.de_count,
-				      blk_name);
-		}
-	}
 }
 
 static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev,
@@ -1357,8 +1303,7 @@ static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, s
 	if (err_data_has_source_info(err_data)) {
 		for_each_ras_error(err_node, err_data) {
 			err_info = &err_node->err_info;
-			amdgpu_ras_error_statistic_de_count(&obj->err_data,
-					&err_info->mcm_info, err_info->de_count);
+
 			amdgpu_ras_error_statistic_ce_count(&obj->err_data,
 					&err_info->mcm_info, err_info->ce_count);
 			amdgpu_ras_error_statistic_ue_count(&obj->err_data,
@@ -1368,7 +1313,6 @@ static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, s
 		/* for legacy asic path which doesn't has error source info */
 		obj->err_data.ue_count += err_data->ue_count;
 		obj->err_data.ce_count += err_data->ce_count;
-		obj->err_data.de_count += err_data->de_count;
 	}
 }
 
@@ -1381,76 +1325,6 @@ static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager
 	obj->err_data.de_count = err_data->de_count;
 }
 
-static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk)
-{
-	struct ras_common_if head;
-
-	memset(&head, 0, sizeof(head));
-	head.block = blk;
-
-	return amdgpu_ras_find_obj(adev, &head);
-}
-
-int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
-			const struct aca_info *aca_info, void *data)
-{
-	struct ras_manager *obj;
-
-	/* in resume phase, no need to create aca fs node */
-	if (adev->in_suspend || amdgpu_reset_in_recovery(adev))
-		return 0;
-
-	obj = get_ras_manager(adev, blk);
-	if (!obj)
-		return -EINVAL;
-
-	return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data);
-}
-
-int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk)
-{
-	struct ras_manager *obj;
-
-	obj = get_ras_manager(adev, blk);
-	if (!obj)
-		return -EINVAL;
-
-	amdgpu_aca_remove_handle(&obj->aca_handle);
-
-	return 0;
-}
-
-static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
-					 enum aca_error_type type, struct ras_err_data *err_data,
-					 struct ras_query_context *qctx)
-{
-	struct ras_manager *obj;
-
-	obj = get_ras_manager(adev, blk);
-	if (!obj)
-		return -EINVAL;
-
-	return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx);
-}
-
-ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr,
-				  struct aca_handle *handle, char *buf, void *data)
-{
-	struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle);
-	struct ras_query_if info = {
-		.head = obj->head,
-	};
-
-	if (!amdgpu_ras_get_error_query_ready(obj->adev))
-		return sysfs_emit(buf, "Query currently inaccessible\n");
-
-	if (amdgpu_ras_query_error_status(obj->adev, &info))
-		return -EINVAL;
-
-	return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count,
-			  "ce", info.ce_count, "de", info.de_count);
-}
-
 static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev,
 						struct ras_query_if *info,
 						struct ras_err_data *err_data,
@@ -1459,7 +1333,6 @@ static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev,
 {
 	enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT;
 	struct amdgpu_ras_block_object *block_obj = NULL;
-	int ret;
 
 	if (blk == AMDGPU_RAS_BLOCK_COUNT)
 		return -EINVAL;
@@ -1469,7 +1342,7 @@ static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev,
 
 	if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) {
 		return amdgpu_virt_req_ras_err_count(adev, blk, err_data);
-	} else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) {
+	} else {
 		if (info->head.block == AMDGPU_RAS_BLOCK__UMC) {
 			amdgpu_ras_get_ecc_info(adev, err_data);
 		} else {
@@ -1490,24 +1363,6 @@ static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev,
 					block_obj->hw_ops->query_ras_error_status(adev);
 			}
 		}
-	} else {
-		if (amdgpu_aca_is_enabled(adev)) {
-			ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx);
-			if (ret)
-				return ret;
-
-			ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx);
-			if (ret)
-				return ret;
-
-			ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx);
-			if (ret)
-				return ret;
-		} else {
-			/* FIXME: add code to check return value later */
-			amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx);
-			amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx);
-		}
 	}
 
 	return 0;
@@ -1565,7 +1420,6 @@ static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev,
 
 	info->ue_count = obj->err_data.ue_count;
 	info->ce_count = obj->err_data.ce_count;
-	info->de_count = obj->err_data.de_count;
 
 out_fini_err_data:
 	amdgpu_ras_error_data_fini(&err_data);
@@ -1624,8 +1478,6 @@ int amdgpu_ras_reset_error_count(struct amdgpu_device *adev,
 		enum amdgpu_ras_block block)
 {
 	struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
-	const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
-	const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
 
 	if (!block_obj || !block_obj->hw_ops) {
 		dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
@@ -1633,17 +1485,14 @@ int amdgpu_ras_reset_error_count(struct amdgpu_device *adev,
 		return -EOPNOTSUPP;
 	}
 
-	if (!amdgpu_ras_is_supported(adev, block) ||
-	    !amdgpu_ras_get_aca_debug_mode(adev))
+	if (!amdgpu_ras_is_supported(adev, block))
 		return -EOPNOTSUPP;
 
 	if (amdgpu_sriov_vf(adev))
 		return -EOPNOTSUPP;
 
 	/* skip ras error reset in gpu reset */
-	if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) &&
-	    ((smu_funcs && smu_funcs->set_debug_mode) ||
-	     (mca_funcs && mca_funcs->mca_set_debug_mode)))
+	if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev))
 		return -EOPNOTSUPP;
 
 	if (block_obj->hw_ops->reset_ras_error_count)
@@ -2090,9 +1939,6 @@ int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
 {
 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
 
-	if (amdgpu_aca_is_enabled(adev))
-		return 0;
-
 	if (!obj || obj->attr_inuse)
 		return -EINVAL;
 
@@ -2130,9 +1976,6 @@ int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
 {
 	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
 
-	if (amdgpu_aca_is_enabled(adev))
-		return 0;
-
 	if (!obj || !obj->attr_inuse)
 		return -EINVAL;
 
@@ -2245,25 +2088,6 @@ static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
 			    obj, &amdgpu_ras_debugfs_ops);
 }
 
-static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev)
-{
-	bool ret;
-
-	switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
-	case IP_VERSION(13, 0, 6):
-	case IP_VERSION(13, 0, 12):
-	case IP_VERSION(13, 0, 14):
-	case IP_VERSION(13, 0, 15):
-		ret = true;
-		break;
-	default:
-		ret = false;
-		break;
-	}
-
-	return ret;
-}
-
 void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev)
 {
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
@@ -2290,13 +2114,6 @@ void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev)
 			amdgpu_ras_debugfs_create(adev, &fs_info, dir);
 		}
 	}
-
-	if (amdgpu_ras_aca_is_supported(adev)) {
-		if (amdgpu_aca_is_enabled(adev))
-			amdgpu_aca_smu_debugfs_init(adev, dir);
-		else
-			amdgpu_mca_smu_debugfs_init(adev, dir);
-	}
 }
 
 /* debugfs end */
@@ -2489,14 +2306,6 @@ static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj
 	event_id = amdgpu_ras_acquire_event_id(adev, type);
 	RAS_EVENT_LOG(adev, event_id, "Poison is created\n");
 
-	if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) {
-		struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev);
-
-		atomic_inc(&con->page_retirement_req_cnt);
-		atomic_inc(&con->poison_creation_count);
-
-		wake_up(&con->page_retirement_wq);
-	}
 }
 
 static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj,
@@ -2529,7 +2338,6 @@ static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj,
 		 */
 		obj->err_data.ue_count += err_data.ue_count;
 		obj->err_data.ce_count += err_data.ce_count;
-		obj->err_data.de_count += err_data.de_count;
 	}
 
 	amdgpu_ras_error_data_fini(&err_data);
@@ -3026,77 +2834,6 @@ static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev,
 	return 0;
 }
 
-static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev,
-			struct eeprom_table_record *bps,
-			struct ras_err_data *err_data)
-{
-	struct ta_ras_query_address_input addr_in;
-	uint32_t socket = 0;
-	int ret = 0;
-
-	if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id)
-		socket = adev->smuio.funcs->get_socket_id(adev);
-
-	/* reinit err_data */
-	err_data->err_addr_cnt = 0;
-	err_data->err_addr_len = adev->umc.retire_unit;
-
-	memset(&addr_in, 0, sizeof(addr_in));
-	addr_in.ma.err_addr = bps->address;
-	addr_in.ma.socket_id = socket;
-	addr_in.ma.ch_inst = bps->mem_channel;
-	if (!amdgpu_ras_smu_eeprom_supported(adev)) {
-		/* tell RAS TA the node instance is not used */
-		addr_in.ma.node_inst = TA_RAS_INV_NODE;
-	} else {
-		addr_in.ma.umc_inst = bps->mcumc_id;
-		addr_in.ma.node_inst = bps->cu;
-	}
-
-	if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
-		ret = adev->umc.ras->convert_ras_err_addr(adev, err_data,
-				&addr_in, NULL, false);
-
-	return ret;
-}
-
-static int amdgpu_ras_mca2pa(struct amdgpu_device *adev,
-			struct eeprom_table_record *bps,
-			struct ras_err_data *err_data)
-{
-	struct ta_ras_query_address_input addr_in;
-	uint32_t die_id, socket = 0;
-
-	if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id)
-		socket = adev->smuio.funcs->get_socket_id(adev);
-
-	/* although die id is gotten from PA in nps1 mode, the id is
-	 * fitable for any nps mode
-	 */
-	if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa)
-		die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address,
-					bps->retired_page << AMDGPU_GPU_PAGE_SHIFT);
-	else
-		return -EINVAL;
-
-	/* reinit err_data */
-	err_data->err_addr_cnt = 0;
-	err_data->err_addr_len = adev->umc.retire_unit;
-
-	memset(&addr_in, 0, sizeof(addr_in));
-	addr_in.ma.err_addr = bps->address;
-	addr_in.ma.ch_inst = bps->mem_channel;
-	addr_in.ma.umc_inst = bps->mcumc_id;
-	addr_in.ma.node_inst = die_id;
-	addr_in.ma.socket_id = socket;
-
-	if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
-		return adev->umc.ras->convert_ras_err_addr(adev, err_data,
-					&addr_in, NULL, false);
-	else
-		return  -EINVAL;
-}
-
 static bool __check_record_in_range(struct amdgpu_device *adev,
 			struct eeprom_table_record *bps, int count)
 {
@@ -3157,117 +2894,13 @@ static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev,
 				struct eeprom_table_record *bps, struct ras_err_data *err_data,
 				enum amdgpu_memory_partition nps)
 {
-	int i = 0;
-	uint64_t chan_idx_v2;
-	enum amdgpu_memory_partition save_nps;
-
-	save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK;
-	chan_idx_v2 = bps[0].retired_page & UMC_CHANNEL_IDX_V2;
-
 	/*old asics just have pa in eeprom*/
-	if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) {
-		memcpy(err_data->err_addr, bps,
-			sizeof(struct eeprom_table_record) * adev->umc.retire_unit);
-		goto out;
-	}
-
-	for (i = 0; i < adev->umc.retire_unit; i++)
-		bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT);
+	memcpy(err_data->err_addr, bps,
+		sizeof(struct eeprom_table_record) * adev->umc.retire_unit);
 
-	if (save_nps || chan_idx_v2) {
-		if (save_nps == nps) {
-			if (amdgpu_umc_pages_in_a_row(adev, err_data,
-					bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT))
-				return -EINVAL;
-			for (i = 0; i < adev->umc.retire_unit; i++) {
-				err_data->err_addr[i].address = bps[0].address;
-				err_data->err_addr[i].mem_channel = bps[0].mem_channel;
-				err_data->err_addr[i].bank = bps[0].bank;
-				err_data->err_addr[i].err_type = bps[0].err_type;
-				err_data->err_addr[i].mcumc_id = bps[0].mcumc_id;
-			}
-		} else {
-			if (amdgpu_ras_mca2pa_by_idx(adev, &bps[0], err_data))
-				return -EINVAL;
-		}
-	} else {
-		if (bps[0].address == 0) {
-			/* for specific old eeprom data, mca address is not stored,
-			 * calc it from pa
-			 */
-			if (amdgpu_umc_pa2mca(adev, bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT,
-				&(bps[0].address), AMDGPU_NPS1_PARTITION_MODE))
-				return -EINVAL;
-		}
-
-		if (amdgpu_ras_mca2pa(adev, &bps[0], err_data)) {
-			if (nps == AMDGPU_NPS1_PARTITION_MODE)
-				memcpy(err_data->err_addr, bps,
-					sizeof(struct eeprom_table_record) * adev->umc.retire_unit);
-			else
-				return -EOPNOTSUPP;
-		}
-	}
-
-out:
 	return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, adev->umc.retire_unit);
 }
 
-static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev,
-				struct eeprom_table_record *bps, struct ras_err_data *err_data,
-				enum amdgpu_memory_partition nps)
-{
-	int i = 0;
-	uint64_t chan_idx_v2;
-	enum amdgpu_memory_partition save_nps;
-
-	if (!amdgpu_ras_smu_eeprom_supported(adev)) {
-		save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK;
-		chan_idx_v2 = bps->retired_page & UMC_CHANNEL_IDX_V2;
-		bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT);
-	} else {
-		/* if pmfw manages eeprom, save_nps is not stored on eeprom,
-		 * we should always convert mca address into physical address,
-		 * make save_nps different from nps
-		 */
-		save_nps = nps + 1;
-	}
-
-	if (save_nps == nps) {
-		if (amdgpu_umc_pages_in_a_row(adev, err_data,
-				bps->retired_page << AMDGPU_GPU_PAGE_SHIFT))
-			return -EINVAL;
-		for (i = 0; i < adev->umc.retire_unit; i++) {
-			err_data->err_addr[i].address = bps->address;
-			err_data->err_addr[i].mem_channel = bps->mem_channel;
-			err_data->err_addr[i].bank = bps->bank;
-			err_data->err_addr[i].err_type = bps->err_type;
-			err_data->err_addr[i].mcumc_id = bps->mcumc_id;
-		}
-	} else {
-		if (save_nps || chan_idx_v2) {
-			if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data))
-				return -EINVAL;
-		} else {
-			/* for specific old eeprom data, mca address is not stored,
-			 * calc it from pa
-			 */
-			if (bps->address == 0)
-				if (amdgpu_umc_pa2mca(adev,
-					bps->retired_page << AMDGPU_GPU_PAGE_SHIFT,
-					&(bps->address),
-					AMDGPU_NPS1_PARTITION_MODE))
-					return -EINVAL;
-
-			if (amdgpu_ras_mca2pa(adev, bps, err_data))
-				return -EOPNOTSUPP;
-		}
-	}
-
-	return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr,
-									adev->umc.retire_unit);
-}
-
 /* it deal with vram only. */
 int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
 		struct eeprom_table_record *bps, int pages, bool from_rom)
@@ -3300,8 +2933,7 @@ int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
 
 	if (from_rom) {
 		/* there is no pa recs in V3, so skip pa recs processing */
-		if ((control->tbl_hdr.version < RAS_TABLE_VER_V3) &&
-		    !amdgpu_ras_smu_eeprom_supported(adev)) {
+		if (control->tbl_hdr.version < RAS_TABLE_VER_V3) {
 			for (i = 0; i < pages; i++) {
 				if (control->ras_num_recs - i >= adev->umc.retire_unit) {
 					if ((bps[i].address == bps[i + 1].address) &&
@@ -3318,10 +2950,8 @@ int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
 				}
 			}
 		}
-		for (; i < pages; i++) {
-			ret = __amdgpu_ras_convert_rec_from_rom(adev,
-				&bps[i], &err_data, nps);
-		}
+		for (; i < pages; i++)
+			bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT);
 
 		con->eh_data->count_saved = con->eh_data->count;
 	} else {
@@ -3346,7 +2976,7 @@ int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev,
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 	struct ras_err_handler_data *data;
 	struct amdgpu_ras_eeprom_control *control;
-	int save_count, unit_num, i;
+	int save_count, unit_num;
 
 	if (!con || !con->eh_data) {
 		if (new_cnt)
@@ -3367,12 +2997,7 @@ int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev,
 	mutex_lock(&con->recovery_lock);
 	control = &con->eeprom_control;
 	data = con->eh_data;
-	if (amdgpu_ras_smu_eeprom_supported(adev))
-		unit_num = control->ras_num_recs -
-			control->ras_num_recs_old;
-	else
-		unit_num = data->count / adev->umc.retire_unit -
-			control->ras_num_recs;
+	unit_num = data->count / adev->umc.retire_unit - control->ras_num_recs;
 
 	save_count = con->bad_page_num - control->ras_num_bad_pages;
 	mutex_unlock(&con->recovery_lock);
@@ -3383,21 +3008,10 @@ int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev,
 	/* only new entries are saved */
 	if (unit_num && save_count) {
 		/*old asics only save pa to eeprom like before*/
-		if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) {
-			if (amdgpu_ras_eeprom_append(control,
-					&data->bps[data->count_saved], unit_num)) {
-				dev_err(adev->dev, "Failed to save EEPROM table data!");
-				return -EIO;
-			}
-		} else {
-			for (i = 0; i < unit_num; i++) {
-				if (amdgpu_ras_eeprom_append(control,
-						&data->bps[data->count_saved +
-						i * adev->umc.retire_unit], 1)) {
-					dev_err(adev->dev, "Failed to save EEPROM table data!");
-					return -EIO;
-				}
-			}
+		if (amdgpu_ras_eeprom_append(control,
+				&data->bps[data->count_saved], unit_num)) {
+			dev_err(adev->dev, "Failed to save EEPROM table data!");
+			return -EIO;
 		}
 
 		dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count);
@@ -3416,7 +3030,7 @@ static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev)
 	struct amdgpu_ras_eeprom_control *control =
 		&adev->psp.ras_context.ras->eeprom_control;
 	struct eeprom_table_record *bps;
-	int ret, i = 0;
+	int ret;
 
 	/* no bad page record, skip eeprom access */
 	if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0)
@@ -3430,33 +3044,6 @@ static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev)
 	if (ret) {
 		dev_err(adev->dev, "Failed to load EEPROM table records!");
 	} else {
-		if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) {
-			/*In V3, there is no pa recs, and some cases(when address==0) may be parsed
-			as pa recs, so add verion check to avoid it.
-			*/
-			if ((control->tbl_hdr.version < RAS_TABLE_VER_V3) &&
-			    !amdgpu_ras_smu_eeprom_supported(adev)) {
-				for (i = 0; i < control->ras_num_recs; i++) {
-					if ((control->ras_num_recs - i) >= adev->umc.retire_unit) {
-						if ((bps[i].address == bps[i + 1].address) &&
-							(bps[i].mem_channel == bps[i + 1].mem_channel)) {
-							control->ras_num_pa_recs += adev->umc.retire_unit;
-							i += (adev->umc.retire_unit - 1);
-						} else {
-							control->ras_num_mca_recs +=
-										(control->ras_num_recs - i);
-							break;
-						}
-					} else {
-						control->ras_num_mca_recs += (control->ras_num_recs - i);
-						break;
-					}
-				}
-			} else {
-				control->ras_num_mca_recs = control->ras_num_recs;
-			}
-		}
-
 		ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true);
 		if (ret)
 			goto out;
@@ -3550,293 +3137,6 @@ static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev,
 	}
 }
 
-int amdgpu_ras_put_poison_req(struct amdgpu_device *adev,
-		enum amdgpu_ras_block block, uint16_t pasid,
-		pasid_notify pasid_fn, void *data, uint32_t reset)
-{
-	int ret = 0;
-	struct ras_poison_msg poison_msg;
-	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-
-	memset(&poison_msg, 0, sizeof(poison_msg));
-	poison_msg.block = block;
-	poison_msg.pasid = pasid;
-	poison_msg.reset = reset;
-	poison_msg.pasid_fn = pasid_fn;
-	poison_msg.data = data;
-
-	ret = kfifo_put(&con->poison_fifo, poison_msg);
-	if (!ret) {
-		dev_err(adev->dev, "Poison message fifo is full!\n");
-		return -ENOSPC;
-	}
-
-	return 0;
-}
-
-static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev,
-		struct ras_poison_msg *poison_msg)
-{
-	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-
-	return kfifo_get(&con->poison_fifo, poison_msg);
-}
-
-static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log)
-{
-	mutex_init(&ecc_log->lock);
-
-	INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL);
-	ecc_log->de_queried_count = 0;
-	ecc_log->consumption_q_count = 0;
-}
-
-static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log)
-{
-	struct radix_tree_iter iter;
-	void __rcu **slot;
-	struct ras_ecc_err *ecc_err;
-
-	mutex_lock(&ecc_log->lock);
-	radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) {
-		ecc_err = radix_tree_deref_slot(slot);
-		kfree(ecc_err->err_pages.pfn);
-		kfree(ecc_err);
-		radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot);
-	}
-	mutex_unlock(&ecc_log->lock);
-
-	mutex_destroy(&ecc_log->lock);
-	ecc_log->de_queried_count = 0;
-	ecc_log->consumption_q_count = 0;
-}
-
-static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con,
-				uint32_t delayed_ms)
-{
-	int ret;
-
-	mutex_lock(&con->umc_ecc_log.lock);
-	ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree,
-			UMC_ECC_NEW_DETECTED_TAG);
-	mutex_unlock(&con->umc_ecc_log.lock);
-
-	if (ret)
-		schedule_delayed_work(&con->page_retirement_dwork,
-			msecs_to_jiffies(delayed_ms));
-
-	return ret ? true : false;
-}
-
-static void amdgpu_ras_do_page_retirement(struct work_struct *work)
-{
-	struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
-					      page_retirement_dwork.work);
-	struct amdgpu_device *adev = con->adev;
-	struct ras_err_data err_data;
-
-	/* If gpu reset is ongoing, delay retiring the bad pages */
-	if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) {
-		amdgpu_ras_schedule_retirement_dwork(con,
-				AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3);
-		return;
-	}
-
-	amdgpu_ras_error_data_init(&err_data);
-
-	amdgpu_umc_handle_bad_pages(adev, &err_data);
-
-	amdgpu_ras_error_data_fini(&err_data);
-
-	amdgpu_ras_schedule_retirement_dwork(con,
-			AMDGPU_RAS_RETIRE_PAGE_INTERVAL);
-}
-
-static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev,
-				uint32_t poison_creation_count)
-{
-	int ret = 0;
-	struct ras_ecc_log_info *ecc_log;
-	struct ras_query_if info;
-	u32 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC;
-	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
-	u64 de_queried_count;
-	u64 consumption_q_count;
-	enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION;
-
-	memset(&info, 0, sizeof(info));
-	info.head.block = AMDGPU_RAS_BLOCK__UMC;
-
-	ecc_log = &ras->umc_ecc_log;
-	ecc_log->de_queried_count = 0;
-	ecc_log->consumption_q_count = 0;
-
-	do {
-		ret = amdgpu_ras_query_error_status_with_event(adev, &info, type);
-		if (ret)
-			return ret;
-
-		de_queried_count = ecc_log->de_queried_count;
-		consumption_q_count = ecc_log->consumption_q_count;
-
-		if (de_queried_count && consumption_q_count)
-			break;
-
-		msleep(100);
-	} while (--timeout);
-
-	if (de_queried_count)
-		schedule_delayed_work(&ras->page_retirement_dwork, 0);
-
-	if (amdgpu_ras_is_rma(adev) && atomic_cmpxchg(&ras->rma_in_recovery, 0, 1) == 0)
-		amdgpu_ras_reset_gpu(adev);
-
-	return 0;
-}
-
-static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev)
-{
-	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-	struct ras_poison_msg msg;
-	int ret;
-
-	do {
-		ret = kfifo_get(&con->poison_fifo, &msg);
-	} while (ret);
-}
-
-static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev,
-			uint32_t msg_count, uint32_t *gpu_reset)
-{
-	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-	uint32_t reset_flags = 0, reset = 0;
-	struct ras_poison_msg msg;
-	int ret, i;
-
-	kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
-
-	for (i = 0; i < msg_count; i++) {
-		ret = amdgpu_ras_get_poison_req(adev, &msg);
-		if (!ret)
-			continue;
-
-		if (msg.pasid_fn)
-			msg.pasid_fn(adev, msg.pasid, msg.data);
-
-		reset_flags |= msg.reset;
-	}
-
-	/*
-	 * Try to ensure poison creation handler is completed first
-	 * to set rma if bad page exceed threshold.
-	 */
-	flush_delayed_work(&con->page_retirement_dwork);
-
-	/* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */
-	if (reset_flags && !amdgpu_ras_is_rma(adev)) {
-		if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET)
-			reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET;
-		else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET)
-			reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
-		else
-			reset = reset_flags;
-
-		con->gpu_reset_flags |= reset;
-		amdgpu_ras_reset_gpu(adev);
-
-		*gpu_reset = reset;
-
-		/* Wait for gpu recovery to complete */
-		flush_work(&con->recovery_work);
-	}
-
-	return 0;
-}
-
-static int amdgpu_ras_page_retirement_thread(void *param)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)param;
-	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-	uint32_t poison_creation_count, msg_count;
-	uint32_t gpu_reset;
-	int ret;
-
-	while (!kthread_should_stop()) {
-
-		wait_event_interruptible(con->page_retirement_wq,
-				kthread_should_stop() ||
-				atomic_read(&con->page_retirement_req_cnt));
-
-		if (kthread_should_stop())
-			break;
-
-		mutex_lock(&con->poison_lock);
-		gpu_reset = 0;
-
-		do {
-			poison_creation_count = atomic_read(&con->poison_creation_count);
-			ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count);
-			if (ret == -EIO)
-				break;
-
-			if (poison_creation_count) {
-				atomic_sub(poison_creation_count, &con->poison_creation_count);
-				atomic_sub(poison_creation_count, &con->page_retirement_req_cnt);
-			}
-		} while (atomic_read(&con->poison_creation_count) &&
-			!atomic_read(&con->poison_consumption_count));
-
-		if (ret != -EIO) {
-			msg_count = kfifo_len(&con->poison_fifo);
-			if (msg_count) {
-				ret = amdgpu_ras_poison_consumption_handler(adev,
-						msg_count, &gpu_reset);
-				if ((ret != -EIO) &&
-				    (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET))
-					atomic_sub(msg_count, &con->page_retirement_req_cnt);
-			}
-		}
-
-		if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) {
-			/* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */
-			/* Clear poison creation request */
-			atomic_set(&con->poison_creation_count, 0);
-			atomic_set(&con->poison_consumption_count, 0);
-
-			/* Clear poison fifo */
-			amdgpu_ras_clear_poison_fifo(adev);
-
-			/* Clear all poison requests */
-			atomic_set(&con->page_retirement_req_cnt, 0);
-
-			if (ret == -EIO) {
-				/* Wait for mode-1 reset to complete */
-				down_read(&adev->reset_domain->sem);
-				up_read(&adev->reset_domain->sem);
-			}
-
-			/* Wake up work to save bad pages to eeprom */
-			schedule_delayed_work(&con->page_retirement_dwork, 0);
-		} else if (gpu_reset) {
-			/* gpu just completed mode-2 reset or other reset */
-			/* Clear poison consumption messages cached in fifo */
-			msg_count = kfifo_len(&con->poison_fifo);
-			if (msg_count) {
-				amdgpu_ras_clear_poison_fifo(adev);
-				atomic_sub(msg_count, &con->page_retirement_req_cnt);
-			}
-
-			atomic_set(&con->poison_consumption_count, 0);
-
-			/* Wake up work to save bad pages to eeprom */
-			schedule_delayed_work(&con->page_retirement_dwork, 0);
-		}
-		mutex_unlock(&con->poison_lock);
-	}
-
-	return 0;
-}
-
 int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev)
 {
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
@@ -3846,22 +3146,21 @@ int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev)
 	if (!con || amdgpu_sriov_vf(adev))
 		return 0;
 
-	if (amdgpu_uniras_enabled(adev))
+	/*
+	 * For the reset-on-init path (e.g. an NPS memory partition,
+	 * switch) the RAS IP block hw_init has not been enabled and
+	 * the amdgpu_uniras_enabled return false, check amdgpu ras
+	 * context uniras_enabled flag, eeprom init will be called
+	 * during RAS IP block hw_init.
+	 */
+	if (amdgpu_uniras_enabled(adev) || con->uniras_enabled)
 		return 0;
 
 	control = &con->eeprom_control;
-	con->ras_smu_drv = amdgpu_dpm_get_ras_smu_driver(adev);
 
 	ret = amdgpu_ras_eeprom_init(control);
 	control->is_eeprom_valid = !ret;
 
-	if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr)
-		control->ras_num_pa_recs = control->ras_num_recs;
-
-	if (adev->umc.ras &&
-	    adev->umc.ras->get_retire_flip_bits)
-		adev->umc.ras->get_retire_flip_bits(adev);
-
 	if (control->ras_num_recs && control->is_eeprom_valid) {
 		ret = amdgpu_ras_load_bad_pages(adev);
 		if (ret) {
@@ -3877,13 +3176,6 @@ int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev)
 				adev, control->bad_channel_bitmap);
 			con->update_channel_flag = false;
 		}
-
-		/* The format action is only applied to new ASICs */
-		if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) >= 12 &&
-		    control->tbl_hdr.version < RAS_TABLE_VER_V3)
-			if (!amdgpu_ras_eeprom_reset_table(control))
-				if (amdgpu_ras_save_bad_pages(adev, NULL))
-					dev_warn(adev->dev, "Failed to format RAS EEPROM data in V3 version!\n");
 	}
 
 	return 0;
@@ -3917,10 +3209,8 @@ int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info)
 	}
 
 	mutex_init(&con->recovery_lock);
-	mutex_init(&con->poison_lock);
 	INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery);
 	atomic_set(&con->in_recovery, 0);
-	atomic_set(&con->rma_in_recovery, 0);
 	con->eeprom_control.bad_channel_bitmap = 0;
 
 	max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control);
@@ -3933,21 +3223,8 @@ int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info)
 	}
 
 	mutex_init(&con->page_rsv_lock);
-	INIT_KFIFO(con->poison_fifo);
 	mutex_init(&con->page_retirement_lock);
-	init_waitqueue_head(&con->page_retirement_wq);
-	atomic_set(&con->page_retirement_req_cnt, 0);
-	atomic_set(&con->poison_creation_count, 0);
-	atomic_set(&con->poison_consumption_count, 0);
-	con->page_retirement_thread =
-		kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement");
-	if (IS_ERR(con->page_retirement_thread)) {
-		con->page_retirement_thread = NULL;
-		dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n");
-	}
-
-	INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement);
-	amdgpu_ras_ecc_log_init(&con->umc_ecc_log);
+
 #ifdef CONFIG_X86_MCE_AMD
 	if ((adev->asic_type == CHIP_ALDEBARAN) &&
 	    (adev->gmc.xgmi.connected_to_cpu))
@@ -3978,33 +3255,15 @@ static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
 {
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 	struct ras_err_handler_data *data = con->eh_data;
-	int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES;
-	bool ret;
 
 	/* recovery_init failed to init it, fini is useless */
 	if (!data)
 		return 0;
 
-	/* Save all cached bad pages to eeprom */
-	do {
-		flush_delayed_work(&con->page_retirement_dwork);
-		ret = amdgpu_ras_schedule_retirement_dwork(con, 0);
-	} while (ret && max_flush_timeout--);
-
-	if (con->page_retirement_thread)
-		kthread_stop(con->page_retirement_thread);
-
-	atomic_set(&con->page_retirement_req_cnt, 0);
-	atomic_set(&con->poison_creation_count, 0);
-
 	mutex_destroy(&con->page_rsv_lock);
 
 	cancel_work_sync(&con->recovery_work);
 
-	cancel_delayed_work_sync(&con->page_retirement_dwork);
-
-	amdgpu_ras_ecc_log_fini(&con->umc_ecc_log);
-
 	mutex_lock(&con->recovery_lock);
 	con->eh_data = NULL;
 	kfree(data->bps);
@@ -4206,15 +3465,6 @@ static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
 	adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 :
 		adev->ras_hw_enabled & amdgpu_ras_mask;
 
-	/* aca is disabled by default except for psp v13_0_6/v13_0_12/v13_0_14 */
-	if (!amdgpu_sriov_vf(adev)) {
-		adev->aca.is_enabled =
-			(amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) ||
-			amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) ||
-			amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14) ||
-			amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 15));
-	}
-
 	/* bad page feature is not applicable to specific app platform */
 	if (adev->gmc.is_app_apu &&
 	    amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0))
@@ -4435,15 +3685,6 @@ int amdgpu_ras_init(struct amdgpu_device *adev)
 		goto release_con;
 	}
 
-	if (amdgpu_ras_aca_is_supported(adev)) {
-		if (amdgpu_aca_is_enabled(adev))
-			r = amdgpu_aca_init(adev);
-		else
-			r = amdgpu_mca_init(adev);
-		if (r)
-			goto release_con;
-	}
-
 	con->init_task_pid = task_pid_nr(current);
 	get_task_comm(con->init_task_comm, current);
 
@@ -4541,9 +3782,12 @@ int amdgpu_ras_block_late_init(struct amdgpu_device *adev,
 			goto cleanup;
 	}
 
-	if (ras_obj->hw_ops &&
-	    (ras_obj->hw_ops->query_ras_error_count ||
-	     ras_obj->hw_ops->query_ras_error_status)) {
+	if (amdgpu_uniras_enabled(adev) || (ras_obj->hw_ops &&
+	   (ras_obj->hw_ops->query_ras_error_count ||
+	    ras_obj->hw_ops->query_ras_error_status)) ||
+	    amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14) ||
+	    amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) ||
+	    amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6)) {
 		r = amdgpu_ras_sysfs_create(adev, ras_block);
 		if (r)
 			goto interrupt;
@@ -4671,28 +3915,13 @@ int amdgpu_ras_late_init(struct amdgpu_device *adev)
 
 	amdgpu_ras_event_mgr_init(adev);
 
-	if (amdgpu_ras_aca_is_supported(adev)) {
-		if (amdgpu_reset_in_recovery(adev)) {
-			if (amdgpu_aca_is_enabled(adev))
-				r = amdgpu_aca_reset(adev);
-			else
-				r = amdgpu_mca_reset(adev);
-			if (r)
-				return r;
-		}
-
-		if (!amdgpu_sriov_vf(adev)) {
-			if (amdgpu_aca_is_enabled(adev))
-				amdgpu_ras_set_aca_debug_mode(adev, false);
-			else
-				amdgpu_ras_set_mca_debug_mode(adev, false);
-		}
-	}
-
 	/* Guest side doesn't need init ras feature */
 	if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev))
 		return 0;
 
+	if (amdgpu_uniras_enabled(adev))
+		amdgpu_ras_mgr_set_debug_mode(adev, false);
+
 	list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
 		obj = node->ras_obj;
 		if (!obj) {
@@ -4773,13 +4002,6 @@ int amdgpu_ras_fini(struct amdgpu_device *adev)
 	amdgpu_ras_fs_fini(adev);
 	amdgpu_ras_interrupt_remove_all(adev);
 
-	if (amdgpu_ras_aca_is_supported(adev)) {
-		if (amdgpu_aca_is_enabled(adev))
-			amdgpu_aca_fini(adev);
-		else
-			amdgpu_mca_fini(adev);
-	}
-
 	WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared");
 
 	if (AMDGPU_RAS_GET_FEATURES(con->features))
@@ -5064,6 +4286,13 @@ static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev)
 	 * Use this list instead of mgpu_info to find the amdgpu
 	 * device on which the UMC error was reported.
 	 */
+	if (mce_adev_list.num_gpu >= MAX_GPU_INSTANCE) {
+		dev_warn_ratelimited(adev->dev,
+			"mce_adev_list full, skip notifier registration (max=%d)\n",
+			MAX_GPU_INSTANCE);
+		return;
+	}
+
 	mce_adev_list.devs[mce_adev_list.num_gpu++] = adev;
 
 	/*
@@ -5181,59 +4410,10 @@ int amdgpu_ras_reset_gpu(struct amdgpu_device *adev)
 	return 0;
 }
 
-int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable)
-{
-	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-	int ret = 0;
-
-	if (con) {
-		ret = amdgpu_mca_smu_set_debug_mode(adev, enable);
-		if (!ret)
-			con->is_aca_debug_mode = enable;
-	}
-
-	return ret;
-}
-
-int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable)
-{
-	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-	int ret = 0;
-
-	if (con) {
-		if (amdgpu_aca_is_enabled(adev))
-			ret = amdgpu_aca_smu_set_debug_mode(adev, enable);
-		else
-			ret = amdgpu_mca_smu_set_debug_mode(adev, enable);
-		if (!ret)
-			con->is_aca_debug_mode = enable;
-	}
-
-	return ret;
-}
-
-bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev)
-{
-	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-	const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
-	const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
-
-	if (!con)
-		return false;
-
-	if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) ||
-	    (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode))
-		return con->is_aca_debug_mode;
-	else
-		return true;
-}
-
 bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev,
 				     unsigned int *error_query_mode)
 {
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-	const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
-	const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
 
 	if (!con) {
 		*error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY;
@@ -5242,9 +4422,6 @@ bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev,
 
 	if (amdgpu_sriov_vf(adev)) {
 		*error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY;
-	} else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) {
-		*error_query_mode =
-			(con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY;
 	} else {
 		*error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY;
 	}
@@ -5551,28 +4728,6 @@ int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data,
 	return 0;
 }
 
-int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data,
-					struct amdgpu_smuio_mcm_config_info *mcm_info,
-					u64 count)
-{
-	struct ras_err_info *err_info;
-
-	if (!err_data || !mcm_info)
-		return -EINVAL;
-
-	if (!count)
-		return 0;
-
-	err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
-	if (!err_info)
-		return -EINVAL;
-
-	err_info->de_count += count;
-	err_data->de_count += count;
-
-	return 0;
-}
-
 #define mmMP0_SMN_C2PMSG_92	0x1609C
 #define mmMP0_SMN_C2PMSG_126	0x160BE
 static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev,
@@ -5834,3 +4989,14 @@ void amdgpu_ras_post_reset(struct amdgpu_device *adev,
 			amdgpu_ras_mgr_post_reset(tmp_adev);
 	}
 }
+
+int amdgpu_ras_resume_after_reset(struct amdgpu_device *adev)
+{
+	int r;
+
+	r = amdgpu_ras_mgr_resume_after_reset(adev);
+	if (r)
+		return r;
+
+	return amdgpu_cper_deferred_init(adev);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
index a86ab65aa2f07..c53f911d5729f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
@@ -31,7 +31,6 @@
 #include "ta_ras_if.h"
 #include "amdgpu_ras_eeprom.h"
 #include "amdgpu_smuio.h"
-#include "amdgpu_aca.h"
 
 struct amdgpu_iv_entry;
 
@@ -466,14 +465,6 @@ struct ras_query_context {
 typedef int (*pasid_notify)(struct amdgpu_device *adev,
 		uint16_t pasid, void *data);
 
-struct ras_poison_msg {
-	enum amdgpu_ras_block block;
-	uint16_t pasid;
-	uint32_t reset;
-	pasid_notify pasid_fn;
-	void *data;
-};
-
 struct ras_err_pages {
 	uint32_t count;
 	uint64_t *pfn;
@@ -492,8 +483,6 @@ struct ras_ecc_err {
 struct ras_ecc_log_info {
 	struct mutex lock;
 	struct radix_tree_root de_page_tree;
-	uint64_t de_queried_count;
-	uint64_t consumption_q_count;
 };
 
 struct ras_critical_region {
@@ -503,32 +492,6 @@ struct ras_critical_region {
 	uint64_t size;
 };
 
-struct ras_eeprom_table_version {
-	uint32_t minor    : 16;
-	uint32_t major    : 16;
-};
-
-struct ras_eeprom_smu_funcs {
-	int (*get_ras_table_version)(struct amdgpu_device *adev,
-							uint32_t *table_version);
-	int (*get_badpage_count)(struct amdgpu_device *adev, uint32_t *count, uint32_t timeout);
-	int (*get_badpage_mca_addr)(struct amdgpu_device *adev, uint16_t index, uint64_t *mca_addr);
-	int (*set_timestamp)(struct amdgpu_device *adev, uint64_t timestamp);
-	int (*get_timestamp)(struct amdgpu_device *adev,
-							uint16_t index, uint64_t *timestamp);
-	int (*get_badpage_ipid)(struct amdgpu_device *adev, uint16_t index, uint64_t *ipid);
-	int (*erase_ras_table)(struct amdgpu_device *adev, uint32_t *result);
-};
-
-enum ras_smu_feature_flags {
-	RAS_SMU_FEATURE_BIT__RAS_EEPROM = BIT_ULL(0),
-};
-
-struct ras_smu_drv {
-	const struct ras_eeprom_smu_funcs *smu_eeprom_funcs;
-	void (*ras_smu_feature_flags)(struct amdgpu_device *adev, uint64_t *flags);
-};
-
 struct amdgpu_ras {
 	void *ras_mgr;
 	/* ras infrastructure */
@@ -549,7 +512,6 @@ struct amdgpu_ras {
 	/* gpu recovery */
 	struct work_struct recovery_work;
 	atomic_t in_recovery;
-	atomic_t rma_in_recovery;
 	struct amdgpu_device *adev;
 	/* error handler data */
 	struct ras_err_handler_data *eh_data;
@@ -581,22 +543,15 @@ struct amdgpu_ras {
 	/* Indicates smu whether need update bad channel info */
 	bool update_channel_flag;
 	/* Record status of smu mca debug mode */
-	bool is_aca_debug_mode;
+	bool is_mca_debug_mode;
 	bool is_rma;
 
 	/* Record special requirements of gpu reset caller */
 	uint32_t  gpu_reset_flags;
 
-	struct task_struct *page_retirement_thread;
-	wait_queue_head_t page_retirement_wq;
 	struct mutex page_retirement_lock;
-	atomic_t page_retirement_req_cnt;
-	atomic_t poison_creation_count;
-	atomic_t poison_consumption_count;
 	struct mutex page_rsv_lock;
-	DECLARE_KFIFO(poison_fifo, struct ras_poison_msg, 128);
 	struct ras_ecc_log_info  umc_ecc_log;
-	struct delayed_work page_retirement_dwork;
 
 	/* ras errors detected */
 	unsigned long ras_err_state;
@@ -615,12 +570,8 @@ struct amdgpu_ras {
 	struct list_head critical_region_head;
 	struct mutex critical_region_lock;
 
-	/* Protect poison injection */
-	struct mutex poison_lock;
-
 	/* Disable/Enable uniras switch */
 	bool uniras_enabled;
-	const struct ras_smu_drv *ras_smu_drv;
 };
 
 struct ras_fs_data {
@@ -702,8 +653,6 @@ struct ras_manager {
 	struct ras_ih_data ih_data;
 
 	struct ras_err_data err_data;
-
-	struct aca_handle aca_handle;
 };
 
 struct ras_badpage {
@@ -964,8 +913,7 @@ struct amdgpu_ras* amdgpu_ras_get_context(struct amdgpu_device *adev);
 int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con);
 
 int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable);
-int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable);
-bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev);
+bool amdgpu_ras_get_mca_debug_mode(struct amdgpu_device *adev);
 bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev,
 				     unsigned int *mode);
 
@@ -1002,16 +950,7 @@ int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data,
 int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data,
 					struct amdgpu_smuio_mcm_config_info *mcm_info,
 					u64 count);
-int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data,
-					struct amdgpu_smuio_mcm_config_info *mcm_info,
-					u64 count);
 void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances);
-int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
-			       const struct aca_info *aca_info, void *data);
-int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk);
-
-ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr,
-				  struct aca_handle *handle, char *buf, void *data);
 
 void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status);
 bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev);
@@ -1029,10 +968,6 @@ int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn);
 int amdgpu_ras_add_critical_region(struct amdgpu_device *adev, struct amdgpu_bo *bo);
 bool amdgpu_ras_check_critical_address(struct amdgpu_device *adev, uint64_t addr);
 
-int amdgpu_ras_put_poison_req(struct amdgpu_device *adev,
-		enum amdgpu_ras_block block, uint16_t pasid,
-		pasid_notify pasid_fn, void *data, uint32_t reset);
-
 bool amdgpu_ras_in_recovery(struct amdgpu_device *adev);
 
 __printf(3, 4)
@@ -1045,4 +980,5 @@ void amdgpu_ras_pre_reset(struct amdgpu_device *adev,
 					  struct list_head *device_list);
 void amdgpu_ras_post_reset(struct amdgpu_device *adev,
 					  struct list_head *device_list);
+int amdgpu_ras_resume_after_reset(struct amdgpu_device *adev);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
index b265b4d9053f2..95468b9463fb8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
@@ -124,8 +124,6 @@
 					RAS_TABLE_V2_1_INFO_SIZE) \
 					/ RAS_TABLE_RECORD_SIZE)
 
-#define RAS_SMU_MESSAGE_TIMEOUT_MS 1000 /* 1s */
-
 /* Given a zero-based index of an EEPROM RAS record, yields the EEPROM
  * offset off of RAS_TABLE_START.  That is, this is something you can
  * add to control->i2c_address, and then tell I2C layer to read
@@ -159,6 +157,9 @@
 
 static bool __is_ras_eeprom_supported(struct amdgpu_device *adev)
 {
+	if (amdgpu_sriov_vf(adev))
+		return false;
+
 	switch (amdgpu_ip_version(adev, MP1_HWIP, 0)) {
 	case IP_VERSION(11, 0, 2): /* VEGA20 and ARCTURUS */
 	case IP_VERSION(11, 0, 7): /* Sienna cichlid */
@@ -449,57 +450,46 @@ int amdgpu_ras_eeprom_reset_table(struct amdgpu_ras_eeprom_control *control)
 	struct amdgpu_ras_eeprom_table_header *hdr = &control->tbl_hdr;
 	struct amdgpu_ras_eeprom_table_ras_info *rai = &control->tbl_rai;
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-	u32 erase_res = 0;
 	u8 csum;
 	int res;
 
 	mutex_lock(&control->ras_tbl_mutex);
 
-	if (!amdgpu_ras_smu_eeprom_supported(adev)) {
-		hdr->header = RAS_TABLE_HDR_VAL;
-		amdgpu_ras_set_eeprom_table_version(control);
-
-		if (hdr->version >= RAS_TABLE_VER_V2_1) {
-			hdr->first_rec_offset = RAS_RECORD_START_V2_1;
-			hdr->tbl_size = RAS_TABLE_HEADER_SIZE +
-					RAS_TABLE_V2_1_INFO_SIZE;
-			rai->rma_status = GPU_HEALTH_USABLE;
-
-			control->ras_record_offset = RAS_RECORD_START_V2_1;
-			control->ras_max_record_count = RAS_MAX_RECORD_COUNT_V2_1;
-			/**
-			 * GPU health represented as a percentage.
-			 * 0 means worst health, 100 means fully health.
-			 */
-			rai->health_percent = 100;
-			/* ecc_page_threshold = 0 means disable bad page retirement */
-			rai->ecc_page_threshold = con->bad_page_cnt_threshold;
-		} else {
-			hdr->first_rec_offset = RAS_RECORD_START;
-			hdr->tbl_size = RAS_TABLE_HEADER_SIZE;
+	hdr->header = RAS_TABLE_HDR_VAL;
+	amdgpu_ras_set_eeprom_table_version(control);
 
-			control->ras_record_offset = RAS_RECORD_START;
-			control->ras_max_record_count = RAS_MAX_RECORD_COUNT;
-		}
+	if (hdr->version >= RAS_TABLE_VER_V2_1) {
+		hdr->first_rec_offset = RAS_RECORD_START_V2_1;
+		hdr->tbl_size = RAS_TABLE_HEADER_SIZE +
+				RAS_TABLE_V2_1_INFO_SIZE;
+		rai->rma_status = GPU_HEALTH_USABLE;
 
-		csum = __calc_hdr_byte_sum(control);
-		if (hdr->version >= RAS_TABLE_VER_V2_1)
-			csum += __calc_ras_info_byte_sum(control);
-		csum = -csum;
-		hdr->checksum = csum;
-		res = __write_table_header(control);
-		if (!res && hdr->version > RAS_TABLE_VER_V1)
-			res = __write_table_ras_info(control);
+		control->ras_record_offset = RAS_RECORD_START_V2_1;
+		control->ras_max_record_count = RAS_MAX_RECORD_COUNT_V2_1;
+		/**
+		 * GPU health represented as a percentage.
+		 * 0 means worst health, 100 means fully health.
+		 */
+		rai->health_percent = 100;
+		/* ecc_page_threshold = 0 means disable bad page retirement */
+		rai->ecc_page_threshold = con->bad_page_cnt_threshold;
 	} else {
-		res = amdgpu_ras_smu_erase_ras_table(adev, &erase_res);
-		if (res || erase_res) {
-			dev_warn(adev->dev, "RAS EEPROM reset failed, res:%d result:%d",
-										res, erase_res);
-			if (!res)
-				res = -EIO;
-		}
+		hdr->first_rec_offset = RAS_RECORD_START;
+		hdr->tbl_size = RAS_TABLE_HEADER_SIZE;
+
+		control->ras_record_offset = RAS_RECORD_START;
+		control->ras_max_record_count = RAS_MAX_RECORD_COUNT;
 	}
 
+	csum = __calc_hdr_byte_sum(control);
+	if (hdr->version >= RAS_TABLE_VER_V2_1)
+		csum += __calc_ras_info_byte_sum(control);
+	csum = -csum;
+	hdr->checksum = csum;
+	res = __write_table_header(control);
+	if (!res && hdr->version > RAS_TABLE_VER_V1)
+		res = __write_table_ras_info(control);
+
 	control->ras_num_recs = 0;
 	control->ras_num_bad_pages = 0;
 	control->ras_num_mca_recs = 0;
@@ -662,7 +652,6 @@ amdgpu_ras_eeprom_append_table(struct amdgpu_ras_eeprom_control *control,
 			       const u32 num)
 {
 	struct amdgpu_ras *con = amdgpu_ras_get_context(to_amdgpu_device(control));
-	struct amdgpu_device *adev = to_amdgpu_device(control);
 	u32 a, b, i;
 	u8 *buf, *pp;
 	int res;
@@ -767,10 +756,7 @@ amdgpu_ras_eeprom_append_table(struct amdgpu_ras_eeprom_control *control,
 		% control->ras_max_record_count;
 
 	/*old asics only save pa to eeprom like before*/
-	if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12)
-		control->ras_num_pa_recs += num;
-	else
-		control->ras_num_mca_recs += num;
+	control->ras_num_pa_recs += num;
 
 	control->ras_num_bad_pages = con->bad_page_num;
 Out:
@@ -879,71 +865,6 @@ amdgpu_ras_eeprom_update_header(struct amdgpu_ras_eeprom_control *control)
 	return res;
 }
 
-int amdgpu_ras_eeprom_update_record_num(struct amdgpu_ras_eeprom_control *control)
-{
-	struct amdgpu_device *adev = to_amdgpu_device(control);
-	int ret, retry = 20;
-
-	if (!amdgpu_ras_smu_eeprom_supported(adev))
-		return 0;
-
-	control->ras_num_recs_old = control->ras_num_recs;
-
-	do {
-		/* 1000ms timeout is long enough, smu_get_badpage_count won't
-		 * return -EBUSY before timeout.
-		 */
-		ret = amdgpu_ras_smu_get_badpage_count(adev,
-			&(control->ras_num_recs), RAS_SMU_MESSAGE_TIMEOUT_MS);
-		if (!ret &&
-		    (control->ras_num_recs_old == control->ras_num_recs)) {
-			/* record number update in PMFW needs some time,
-			 * smu_get_badpage_count may return immediately without
-			 * count update, sleep for a while and retry again.
-			 */
-			msleep(50);
-			retry--;
-		} else {
-			break;
-		}
-	} while (retry);
-
-	/* no update of record number is not a real failure,
-	 * don't print warning here
-	 */
-	if (!ret && (control->ras_num_recs_old == control->ras_num_recs))
-		ret = -EINVAL;
-
-	return ret;
-}
-
-static int amdgpu_ras_smu_eeprom_append(struct amdgpu_ras_eeprom_control *control)
-{
-	struct amdgpu_device *adev = to_amdgpu_device(control);
-	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-
-	if (!amdgpu_ras_smu_eeprom_supported(adev) || !con)
-		return 0;
-
-	control->ras_num_bad_pages = con->bad_page_num;
-
-	if (amdgpu_bad_page_threshold != 0 &&
-	    control->ras_num_bad_pages > con->bad_page_cnt_threshold) {
-		dev_warn(adev->dev,
-			"Saved bad pages %d reaches threshold value %d\n",
-			control->ras_num_bad_pages, con->bad_page_cnt_threshold);
-
-		if (adev->cper.enabled && amdgpu_cper_generate_bp_threshold_record(adev))
-			dev_warn(adev->dev, "fail to generate bad page threshold cper records\n");
-
-		if ((amdgpu_bad_page_threshold != -1) &&
-		    (amdgpu_bad_page_threshold != -2))
-			con->is_rma = true;
-	}
-
-	return 0;
-}
-
 /**
  * amdgpu_ras_eeprom_append -- append records to the EEPROM RAS table
  * @control: pointer to control structure
@@ -968,9 +889,6 @@ int amdgpu_ras_eeprom_append(struct amdgpu_ras_eeprom_control *control,
 	if (!__is_ras_eeprom_supported(adev))
 		return 0;
 
-	if (amdgpu_ras_smu_eeprom_supported(adev))
-		return amdgpu_ras_smu_eeprom_append(control);
-
 	if (num == 0) {
 		dev_err(adev->dev, "will not append 0 records\n");
 		return -EINVAL;
@@ -1046,52 +964,6 @@ static int __amdgpu_ras_eeprom_read(struct amdgpu_ras_eeprom_control *control,
 	return res;
 }
 
-int amdgpu_ras_eeprom_read_idx(struct amdgpu_ras_eeprom_control *control,
-			struct eeprom_table_record *record, u32 rec_idx,
-			const u32 num)
-{
-	struct amdgpu_device *adev = to_amdgpu_device(control);
-	uint64_t ts, end_idx;
-	int i, ret;
-	u64 mca, ipid;
-	u32 cu, mem_channel, mcumc_id;
-
-	if (!amdgpu_ras_smu_eeprom_supported(adev))
-		return 0;
-
-	if (!adev->umc.ras || !adev->umc.ras->mca_ipid_parse)
-		return -EOPNOTSUPP;
-
-	end_idx = rec_idx + num;
-	for (i = rec_idx; i < end_idx; i++) {
-		ret = amdgpu_ras_smu_get_badpage_mca_addr(adev, i, &mca);
-		if (ret)
-			return ret;
-
-		ret = amdgpu_ras_smu_get_badpage_ipid(adev, i, &ipid);
-		if (ret)
-			return ret;
-
-		ret = amdgpu_ras_smu_get_timestamp(adev, i, &ts);
-		if (ret)
-			return ret;
-
-		record[i - rec_idx].address = mca;
-		/* retired_page (pa) is unused now */
-		record[i - rec_idx].retired_page = 0x1ULL;
-		record[i - rec_idx].ts = ts;
-		record[i - rec_idx].err_type = AMDGPU_RAS_EEPROM_ERR_NON_RECOVERABLE;
-
-		adev->umc.ras->mca_ipid_parse(adev, ipid,
-			&cu, &mem_channel, &mcumc_id, NULL);
-		record[i - rec_idx].cu = (u8)cu;
-		record[i - rec_idx].mem_channel = (u8)mem_channel;
-		record[i - rec_idx].mcumc_id = (u8)mcumc_id;
-	}
-
-	return 0;
-}
-
 /**
  * amdgpu_ras_eeprom_read -- read EEPROM
  * @control: pointer to control structure
@@ -1113,9 +985,6 @@ int amdgpu_ras_eeprom_read(struct amdgpu_ras_eeprom_control *control,
 	u8 *buf, *pp;
 	u32 g0, g1;
 
-	if (amdgpu_ras_smu_eeprom_supported(adev))
-		return amdgpu_ras_eeprom_read_idx(control, record, 0, num);
-
 	if (!__is_ras_eeprom_supported(adev))
 		return 0;
 
@@ -1287,10 +1156,6 @@ static ssize_t amdgpu_ras_debugfs_table_read(struct file *f, char __user *buf,
 	int res = -EFAULT;
 	size_t data_len;
 
-	/* pmfw manages eeprom data by itself */
-	if (amdgpu_ras_smu_eeprom_supported(adev))
-		return 0;
-
 	mutex_lock(&control->ras_tbl_mutex);
 
 	/* We want *pos - data_len > 0, which means there's
@@ -1395,6 +1260,86 @@ static ssize_t amdgpu_ras_debugfs_table_read(struct file *f, char __user *buf,
 	return res < 0 ? res : orig_size - size;
 }
 
+static ssize_t
+amdgpu_ras_debugfs_table_read_uniras(struct amdgpu_device *adev,
+				     char __user *buf,
+				     size_t size, loff_t *pos)
+{
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	struct ras_core_context *ras_core = ras_mgr ? ras_mgr->ras_core : NULL;
+	struct eeprom_umc_record *records = NULL;
+	struct ras_eeprom_control *control;
+	size_t bufsz, len = 0;
+	u32 num_recs;
+	char *kbuf;
+	ssize_t res;
+	int i;
+
+	if (!ras_core)
+		return 0;
+
+	/* pmfw manages eeprom data by itself */
+	if (ras_fw_eeprom_supported(ras_core))
+		return 0;
+
+	control = &ras_core->ras_eeprom;
+	num_recs = ras_eeprom_get_record_count(ras_core);
+
+	bufsz = strlen(tbl_hdr_str) + tbl_hdr_fmt_size +
+		strlen(rec_hdr_str) + (size_t)rec_hdr_fmt_size * num_recs + 1;
+
+	kbuf = kvmalloc(bufsz, GFP_KERNEL);
+	if (!kbuf)
+		return -ENOMEM;
+
+	if (num_recs) {
+		records = kvcalloc(num_recs, sizeof(*records), GFP_KERNEL);
+		if (!records) {
+			res = -ENOMEM;
+			goto out;
+		}
+
+		res = ras_eeprom_read(ras_core, records, num_recs);
+		if (res)
+			goto out;
+	}
+
+	len += scnprintf(kbuf + len, bufsz - len, "%s", tbl_hdr_str);
+	len += scnprintf(kbuf + len, bufsz - len, tbl_hdr_fmt,
+				 control->tbl_hdr.header,
+				 control->tbl_hdr.version,
+				 control->tbl_hdr.first_rec_offset,
+				 control->tbl_hdr.tbl_size,
+				 control->tbl_hdr.checksum);
+	len += scnprintf(kbuf + len, bufsz - len, "%s", rec_hdr_str);
+
+	for (i = 0; i < num_recs; i++) {
+		u32 ai = RAS_RI_TO_AI(control, i);
+		int et = records[i].err_type;
+		const char *ets = (et >= 0 && et < AMDGPU_RAS_EEPROM_ERR_COUNT) ?
+				  record_err_type_str[et] : "na";
+
+		len += scnprintf(kbuf + len, bufsz - len, rec_hdr_fmt,
+				 i,
+				 RAS_INDEX_TO_OFFSET(control, ai),
+				 ets,
+				 records[i].bank,
+				 records[i].ts,
+				 records[i].offset,
+				 records[i].mem_channel,
+				 records[i].mcumc_id,
+				 records[i].retired_row_pfn);
+	}
+
+	res = simple_read_from_buffer(buf, size, pos, kbuf, len);
+
+out:
+	kvfree(records);
+	kvfree(kbuf);
+
+	return res;
+}
+
 static ssize_t
 amdgpu_ras_debugfs_eeprom_table_read(struct file *f, char __user *buf,
 				     size_t size, loff_t *pos)
@@ -1408,6 +1353,10 @@ amdgpu_ras_debugfs_eeprom_table_read(struct file *f, char __user *buf,
 	if (!size)
 		return size;
 
+	if (amdgpu_uniras_enabled(adev))
+		return amdgpu_ras_debugfs_table_read_uniras(adev, buf,
+						    size, pos);
+
 	if (!ras || !control) {
 		res = snprintf(data, sizeof(data), "Not supported\n");
 		if (*pos >= res)
@@ -1521,42 +1470,6 @@ static int __read_table_ras_info(struct amdgpu_ras_eeprom_control *control)
 	return res == RAS_TABLE_V2_1_INFO_SIZE ? 0 : res;
 }
 
-static int amdgpu_ras_smu_eeprom_init(struct amdgpu_ras_eeprom_control *control)
-{
-	struct amdgpu_device *adev = to_amdgpu_device(control);
-	struct amdgpu_ras_eeprom_table_header *hdr = &control->tbl_hdr;
-	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
-	uint64_t local_time;
-	int res;
-
-	ras->is_rma = false;
-
-	if (!__is_ras_eeprom_supported(adev))
-		return 0;
-	mutex_init(&control->ras_tbl_mutex);
-
-	res = amdgpu_ras_smu_get_table_version(adev, &(hdr->version));
-	if (res)
-		return res;
-
-	res = amdgpu_ras_smu_get_badpage_count(adev,
-								&(control->ras_num_recs), 100);
-	if (res)
-		return res;
-
-	local_time = (uint64_t)ktime_get_real_seconds();
-	res = amdgpu_ras_smu_set_timestamp(adev, local_time);
-	if (res)
-		return res;
-
-	control->ras_max_record_count = 4000;
-
-	control->ras_num_mca_recs = 0;
-	control->ras_num_pa_recs = 0;
-
-	return 0;
-}
-
 int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control)
 {
 	struct amdgpu_device *adev = to_amdgpu_device(control);
@@ -1567,9 +1480,6 @@ int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control)
 	uint32_t vram_type = adev->gmc.vram_type;
 	int res;
 
-	if (amdgpu_ras_smu_eeprom_supported(adev))
-		return amdgpu_ras_smu_eeprom_init(control);
-
 	ras->is_rma = false;
 
 	if (!__is_ras_eeprom_supported(adev))
@@ -1663,47 +1573,6 @@ int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control)
 	return 0;
 }
 
-static int amdgpu_ras_smu_eeprom_check(struct amdgpu_ras_eeprom_control *control)
-{
-	struct amdgpu_device *adev = to_amdgpu_device(control);
-	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
-
-	if (!__is_ras_eeprom_supported(adev))
-		return 0;
-
-	control->ras_num_bad_pages = ras->bad_page_num;
-
-	if ((ras->bad_page_cnt_threshold < control->ras_num_bad_pages) &&
-	    amdgpu_bad_page_threshold != 0) {
-		dev_warn(adev->dev,
-			"RAS records:%d exceed threshold:%d\n",
-			control->ras_num_bad_pages, ras->bad_page_cnt_threshold);
-		if ((amdgpu_bad_page_threshold == -1) ||
-			(amdgpu_bad_page_threshold == -2)) {
-			dev_warn(adev->dev,
-				 "Please consult AMD Service Action Guide (SAG) for appropriate service procedures\n");
-		} else {
-			ras->is_rma = true;
-			dev_warn(adev->dev,
-				 "User defined threshold is set, runtime service will be halt when threshold is reached\n");
-		}
-
-		return 0;
-	}
-
-	dev_dbg(adev->dev,
-		"Found existing EEPROM table with %d records",
-		control->ras_num_bad_pages);
-
-	/* Warn if we are at 90% of the threshold or above
-	 */
-	if (10 * control->ras_num_bad_pages >= 9 * ras->bad_page_cnt_threshold)
-		dev_warn(adev->dev, "RAS records:%u exceeds 90%% of threshold:%d",
-				control->ras_num_bad_pages,
-				ras->bad_page_cnt_threshold);
-	return 0;
-}
-
 int amdgpu_ras_eeprom_check(struct amdgpu_ras_eeprom_control *control)
 {
 	struct amdgpu_device *adev = to_amdgpu_device(control);
@@ -1711,9 +1580,6 @@ int amdgpu_ras_eeprom_check(struct amdgpu_ras_eeprom_control *control)
 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
 	int res = 0;
 
-	if (amdgpu_ras_smu_eeprom_supported(adev))
-		return amdgpu_ras_smu_eeprom_check(control);
-
 	if (!__is_ras_eeprom_supported(adev))
 		return 0;
 
@@ -1806,8 +1672,7 @@ void amdgpu_ras_eeprom_check_and_recover(struct amdgpu_device *adev)
 	struct amdgpu_ras_eeprom_control *control;
 	int res;
 
-	if (!__is_ras_eeprom_supported(adev) || !ras ||
-	    amdgpu_ras_smu_eeprom_supported(adev))
+	if (!__is_ras_eeprom_supported(adev) || !ras)
 		return;
 	control = &ras->eeprom_control;
 	if (!control->is_eeprom_valid)
@@ -1829,151 +1694,12 @@ void amdgpu_ras_eeprom_check_and_recover(struct amdgpu_device *adev)
 	return;
 }
 
-static const struct ras_smu_drv *amdgpu_ras_get_smu_ras_drv(struct amdgpu_device *adev)
-{
-	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
-
-	if (!ras)
-		return NULL;
-
-	return ras->ras_smu_drv;
-}
-
-static uint64_t amdgpu_ras_smu_get_feature_flags(struct amdgpu_device *adev)
-{
-	const struct ras_smu_drv *ras_smu_drv = amdgpu_ras_get_smu_ras_drv(adev);
-	uint64_t flags = 0ULL;
-
-	if (!ras_smu_drv)
-		goto out;
-
-	if (ras_smu_drv->ras_smu_feature_flags)
-		ras_smu_drv->ras_smu_feature_flags(adev, &flags);
-
-out:
-	return flags;
-}
-
-bool amdgpu_ras_smu_eeprom_supported(struct amdgpu_device *adev)
-{
-	const struct ras_smu_drv *smu_ras_drv = amdgpu_ras_get_smu_ras_drv(adev);
-	uint64_t flags = 0ULL;
-
-	if (!__is_ras_eeprom_supported(adev) || !smu_ras_drv)
-		return false;
-
-	if (!smu_ras_drv->smu_eeprom_funcs)
-		return false;
-
-	flags = amdgpu_ras_smu_get_feature_flags(adev);
-
-	return !!(flags & RAS_SMU_FEATURE_BIT__RAS_EEPROM);
-}
-
-int amdgpu_ras_smu_get_table_version(struct amdgpu_device *adev,
-				     uint32_t *table_version)
-{
-	const struct ras_smu_drv *smu_ras_drv = amdgpu_ras_get_smu_ras_drv(adev);
-
-	if (!amdgpu_ras_smu_eeprom_supported(adev))
-		return -EOPNOTSUPP;
-
-	if (smu_ras_drv->smu_eeprom_funcs->get_ras_table_version)
-		return smu_ras_drv->smu_eeprom_funcs->get_ras_table_version(adev,
-										 table_version);
-	return -EOPNOTSUPP;
-}
-
-int amdgpu_ras_smu_get_badpage_count(struct amdgpu_device *adev,
-				     uint32_t *count, uint32_t timeout)
-{
-	const struct ras_smu_drv *smu_ras_drv = amdgpu_ras_get_smu_ras_drv(adev);
-
-	if (!amdgpu_ras_smu_eeprom_supported(adev))
-		return -EOPNOTSUPP;
-
-	if (smu_ras_drv->smu_eeprom_funcs->get_badpage_count)
-		return smu_ras_drv->smu_eeprom_funcs->get_badpage_count(adev,
-									     count, timeout);
-	return -EOPNOTSUPP;
-}
-
-int amdgpu_ras_smu_get_badpage_mca_addr(struct amdgpu_device *adev,
-					uint16_t index, uint64_t *mca_addr)
-{
-	const struct ras_smu_drv *smu_ras_drv = amdgpu_ras_get_smu_ras_drv(adev);
-
-	if (!amdgpu_ras_smu_eeprom_supported(adev))
-		return -EOPNOTSUPP;
-
-	if (smu_ras_drv->smu_eeprom_funcs->get_badpage_mca_addr)
-		return smu_ras_drv->smu_eeprom_funcs->get_badpage_mca_addr(adev,
-										index, mca_addr);
-	return -EOPNOTSUPP;
-}
-
-int amdgpu_ras_smu_set_timestamp(struct amdgpu_device *adev,
-				 uint64_t timestamp)
-{
-	const struct ras_smu_drv *smu_ras_drv = amdgpu_ras_get_smu_ras_drv(adev);
-
-	if (!amdgpu_ras_smu_eeprom_supported(adev))
-		return -EOPNOTSUPP;
-
-	if (smu_ras_drv->smu_eeprom_funcs->set_timestamp)
-		return smu_ras_drv->smu_eeprom_funcs->set_timestamp(adev,
-									 timestamp);
-	return -EOPNOTSUPP;
-}
-
-int amdgpu_ras_smu_get_timestamp(struct amdgpu_device *adev,
-				 uint16_t index, uint64_t *timestamp)
-{
-	const struct ras_smu_drv *smu_ras_drv = amdgpu_ras_get_smu_ras_drv(adev);
-
-	if (!amdgpu_ras_smu_eeprom_supported(adev))
-		return -EOPNOTSUPP;
-
-	if (smu_ras_drv->smu_eeprom_funcs->get_timestamp)
-		return smu_ras_drv->smu_eeprom_funcs->get_timestamp(adev,
-									 index, timestamp);
-	return -EOPNOTSUPP;
-}
-
-int amdgpu_ras_smu_get_badpage_ipid(struct amdgpu_device *adev,
-				    uint16_t index, uint64_t *ipid)
-{
-	const struct ras_smu_drv *smu_ras_drv = amdgpu_ras_get_smu_ras_drv(adev);
-
-	if (!amdgpu_ras_smu_eeprom_supported(adev))
-		return -EOPNOTSUPP;
-
-	if (smu_ras_drv->smu_eeprom_funcs->get_badpage_ipid)
-		return smu_ras_drv->smu_eeprom_funcs->get_badpage_ipid(adev,
-									    index, ipid);
-	return -EOPNOTSUPP;
-}
-
-int amdgpu_ras_smu_erase_ras_table(struct amdgpu_device *adev,
-				   uint32_t *result)
-{
-	const struct ras_smu_drv *smu_ras_drv = amdgpu_ras_get_smu_ras_drv(adev);
-
-	if (!amdgpu_ras_smu_eeprom_supported(adev))
-		return -EOPNOTSUPP;
-
-	if (smu_ras_drv->smu_eeprom_funcs->erase_ras_table)
-		return smu_ras_drv->smu_eeprom_funcs->erase_ras_table(adev,
-									   result);
-	return -EOPNOTSUPP;
-}
-
 void amdgpu_ras_check_bad_page_status(struct amdgpu_device *adev)
 {
 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
 	struct amdgpu_ras_eeprom_control *control = ras ? &ras->eeprom_control : NULL;
 
-	if (!control || amdgpu_bad_page_threshold == 0)
+	if (!__is_ras_eeprom_supported(adev) || !control || amdgpu_bad_page_threshold == 0)
 		return;
 
 	if (control->ras_num_bad_pages > ras->bad_page_cnt_threshold) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.h
index a62114800a92a..6e50b9f0569d9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.h
@@ -82,7 +82,6 @@ struct amdgpu_ras_eeprom_control {
 	/* Number of records in the table.
 	 */
 	u32 ras_num_recs;
-	u32 ras_num_recs_old;
 
 	/* the bad page number is ras_num_recs or
 	 * ras_num_recs * umc.retire_unit
@@ -164,35 +163,10 @@ int amdgpu_ras_eeprom_check(struct amdgpu_ras_eeprom_control *control);
 
 void amdgpu_ras_eeprom_check_and_recover(struct amdgpu_device *adev);
 
-bool amdgpu_ras_smu_eeprom_supported(struct amdgpu_device *adev);
-
-int amdgpu_ras_smu_get_table_version(struct amdgpu_device *adev,
-							uint32_t *table_version);
-
-int amdgpu_ras_smu_get_badpage_count(struct amdgpu_device *adev,
-								uint32_t *count, uint32_t timeout);
-
-int amdgpu_ras_smu_get_badpage_mca_addr(struct amdgpu_device *adev,
-								uint16_t index, uint64_t *mca_addr);
-
-int amdgpu_ras_smu_set_timestamp(struct amdgpu_device *adev,
-										uint64_t timestamp);
-
-int amdgpu_ras_smu_get_timestamp(struct amdgpu_device *adev,
-							uint16_t index, uint64_t *timestamp);
-
-int amdgpu_ras_smu_get_badpage_ipid(struct amdgpu_device *adev,
-								uint16_t index, uint64_t *ipid);
-
-int amdgpu_ras_smu_erase_ras_table(struct amdgpu_device *adev,
-									uint32_t *result);
-
 int amdgpu_ras_eeprom_read_idx(struct amdgpu_ras_eeprom_control *control,
 			struct eeprom_table_record *record, u32 rec_idx,
 			const u32 num);
 
-int amdgpu_ras_eeprom_update_record_num(struct amdgpu_ras_eeprom_control *control);
-
 void amdgpu_ras_check_bad_page_status(struct amdgpu_device *adev);
 
 extern const struct file_operations amdgpu_ras_debugfs_eeprom_size_ops;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h
index a1011af6b52b4..320c30ce4a62f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h
@@ -89,6 +89,14 @@ struct amdgpu_reg_access {
 	struct amdgpu_reg_smn_ext smn;
 };
 
+/*
+ * ASIC specific register table accessible by UMD
+ */
+struct amdgpu_allowed_register_entry {
+	uint32_t reg_offset;
+	bool grbm_indexed;
+};
+
 void amdgpu_reg_access_init(struct amdgpu_device *adev);
 uint32_t amdgpu_reg_smc_rd32(struct amdgpu_device *adev, uint32_t reg);
 void amdgpu_reg_smc_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index b97fa35bac232..0d34f0eca9917 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -264,31 +264,31 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
 			return r;
 	}
 
-	r = amdgpu_device_wb_get(adev, &ring->rptr_offs);
+	r = amdgpu_wb_get(adev, &ring->rptr_offs);
 	if (r) {
 		dev_err(adev->dev, "(%d) ring rptr_offs wb alloc failed\n", r);
 		return r;
 	}
 
-	r = amdgpu_device_wb_get(adev, &ring->wptr_offs);
+	r = amdgpu_wb_get(adev, &ring->wptr_offs);
 	if (r) {
 		dev_err(adev->dev, "(%d) ring wptr_offs wb alloc failed\n", r);
 		return r;
 	}
 
-	r = amdgpu_device_wb_get(adev, &ring->fence_offs);
+	r = amdgpu_wb_get(adev, &ring->fence_offs);
 	if (r) {
 		dev_err(adev->dev, "(%d) ring fence_offs wb alloc failed\n", r);
 		return r;
 	}
 
-	r = amdgpu_device_wb_get(adev, &ring->trail_fence_offs);
+	r = amdgpu_wb_get(adev, &ring->trail_fence_offs);
 	if (r) {
 		dev_err(adev->dev, "(%d) ring trail_fence_offs wb alloc failed\n", r);
 		return r;
 	}
 
-	r = amdgpu_device_wb_get(adev, &ring->cond_exe_offs);
+	r = amdgpu_wb_get(adev, &ring->cond_exe_offs);
 	if (r) {
 		dev_err(adev->dev, "(%d) ring cond_exec_polling wb alloc failed\n", r);
 		return r;
@@ -401,11 +401,11 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring)
 
 	ring->sched.ready = false;
 
-	amdgpu_device_wb_free(ring->adev, ring->rptr_offs);
-	amdgpu_device_wb_free(ring->adev, ring->wptr_offs);
+	amdgpu_wb_free(ring->adev, ring->rptr_offs);
+	amdgpu_wb_free(ring->adev, ring->wptr_offs);
 
-	amdgpu_device_wb_free(ring->adev, ring->cond_exe_offs);
-	amdgpu_device_wb_free(ring->adev, ring->fence_offs);
+	amdgpu_wb_free(ring->adev, ring->cond_exe_offs);
+	amdgpu_wb_free(ring->adev, ring->fence_offs);
 
 	amdgpu_bo_free_kernel(&ring->ring_obj,
 			      &ring->gpu_addr,
@@ -935,6 +935,194 @@ int amdgpu_ring_reset_helper_end(struct amdgpu_ring *ring,
 	return 0;
 }
 
+/**
+ * amdgpu_multi_ring_reset_helper_begin() - Prepare multiple rings for a reset.
+ *
+ * @ring_type_mask: Bitmask of affected ring types
+ * @guilty_ring: The ring which is guilty of causing a reset.
+ * @guilty_fence: The fence which didn't signal on the guilty ring.
+ *
+ * Useful when performing a GPU reset method that affects
+ * multiple rings at the same time, such as an IP block soft
+ * reset. For example, a GFX IP block soft reset will affect
+ * every graphics and compute queue.
+ *
+ * This function should be called before such a reset.
+ *
+ * Prepare the affected rings before the reset, make sure to
+ * minimize collateral damage, and backup the contents of
+ * the rings. Then the caller can call the actual HW specific
+ * reset function.
+ *
+ * After the reset is complete, the caller should then call
+ * amdgpu_multi_ring_reset_helper_end() to restore the rings.
+ */
+void amdgpu_multi_ring_reset_helper_begin(const u32 ring_type_mask,
+					  struct amdgpu_ring *guilty_ring,
+					  struct amdgpu_fence *guilty_fence)
+{
+	struct amdgpu_device *adev = guilty_ring->adev;
+	struct amdgpu_fence *ring_guilty_fence;
+	struct amdgpu_ring *ring;
+	bool rings_busy;
+	int i;
+	u32 t;
+
+	for (i = 0; i < adev->num_rings; ++i) {
+		ring = adev->rings[i];
+
+		if (!(BIT(ring->funcs->type) & ring_type_mask))
+			continue;
+
+		/* Don't accept new submissions on the ring. */
+		if (amdgpu_ring_sched_ready(ring) && !drm_sched_is_stopped(&ring->sched))
+			drm_sched_wqueue_stop(&ring->sched);
+
+		/*
+		 * Clear the preempt condition to stop the ring
+		 * from starting its next submission. This ensures
+		 * that only the currently executing submission
+		 * can be rejected because of the reset and helps
+		 * minimize collateral damage.
+		 */
+		if (ring->funcs->init_cond_exec)
+			amdgpu_ring_set_preempt_cond_exec(ring, false);
+	}
+
+	/* Flush HDP cache so the GPU can see the updated COND_EXEC values */
+	amdgpu_device_flush_hdp(adev, NULL);
+
+	/*
+	 * Give some time for non-guilty rings to finish their
+	 * current submission, to try to minimize collateral damage.
+	 *
+	 * Note that this is just a best effort, but really there
+	 * is no way to really know which ring is actually responsible
+	 * because different rings may share resources, eg. a compute
+	 * ring may hog shader engines, causing a graphics ring to hang.
+	 */
+	for (t = 0; t < adev->usec_timeout; t += 10000) {
+		rings_busy = false;
+
+		/* Check if any of the non-guilty rings are busy */
+		for (i = 0; i < adev->num_rings; ++i) {
+			ring = adev->rings[i];
+
+			if (!(BIT(ring->funcs->type) & ring_type_mask))
+				continue;
+
+			if (ring == guilty_ring)
+				continue;
+
+			rings_busy |=
+				atomic_read(&ring->fence_drv.last_seq) !=
+				READ_ONCE(ring->fence_drv.sync_seq);
+		}
+
+		if (!rings_busy)
+			break;
+
+		mdelay(10);
+	}
+
+	for (i = 0; i < adev->num_rings; ++i) {
+		ring = adev->rings[i];
+
+		if (!(BIT(ring->funcs->type) & ring_type_mask))
+			continue;
+
+		/*
+		 * Find guilty fences, ie. the fences that didn't signal
+		 * on each ring. At this point there is no way to know
+		 * which one is really responsible for the hang, and no
+		 * way to save any of them, so we treat all of them as guilty.
+		 */
+		ring_guilty_fence =
+			ring == guilty_ring ? guilty_fence :
+			amdgpu_ring_find_guilty_fence(ring);
+
+		/*
+		 * Backup current contents of the ring.
+		 * The helper takes care to only reemit unsignalled fences
+		 * so we don't have to worry about that here.
+		 */
+		amdgpu_ring_reset_helper_begin(ring, ring_guilty_fence);
+	}
+}
+
+/**
+ * amdgpu_multi_ring_reset_helper_end() - Prepare multiple rings for a reset.
+ *
+ * @ring_type_mask: Bitmask of affected ring types
+ * @guilty_ring: The ring which is guilty of causing a reset.
+ * @ret: Return code from the reset function.
+ *
+ * After calling amdgpu_multi_ring_reset_helper_begin()
+ * and executing the actual reset method, call this
+ * function to restore normal operation.
+ *
+ * In case the reset failed, this function should still
+ * be called to restore preemption state, but it won't attempt to
+ * fully restore the ring contents.
+ */
+int amdgpu_multi_ring_reset_helper_end(const u32 ring_type_mask,
+				       struct amdgpu_ring *guilty_ring, int ret)
+{
+	struct amdgpu_device *adev = guilty_ring->adev;
+	struct amdgpu_ring *ring;
+	int i, r;
+
+	/* Set preempt condition, rings are now allowed to execute submissions */
+	for (i = 0; i < adev->num_rings; ++i) {
+		ring = adev->rings[i];
+
+		if (!(BIT(ring->funcs->type) & ring_type_mask))
+			continue;
+
+		if (ring->funcs->init_cond_exec)
+			amdgpu_ring_set_preempt_cond_exec(ring, true);
+	}
+
+	/* Flush HDP cache so the GPU can see the updated COND_EXEC values */
+	amdgpu_device_flush_hdp(adev, NULL);
+
+	/* If the reset was unsuccessful, return without restoring anything else. */
+	if (ret)
+		return ret;
+
+	/* Restore contents of all rings */
+	for (i = 0; i < adev->num_rings; ++i) {
+		ring = adev->rings[i];
+
+		if (!(BIT(ring->funcs->type) & ring_type_mask))
+			continue;
+
+		/* Restore contents of the ring */
+		r = amdgpu_ring_reset_helper_end(ring, ring->guilty_fence);
+		if (r) {
+			dev_err(adev->dev,
+				"Failed to recover ring %s after soft reset\n",
+				ring->name);
+			return r;
+		}
+	}
+
+	/* Accept submissions on all rings again */
+	for (i = 0; i < adev->num_rings; ++i) {
+		ring = adev->rings[i];
+
+		if (!(BIT(ring->funcs->type) & ring_type_mask))
+			continue;
+
+		if (!amdgpu_ring_sched_ready(ring))
+			continue;
+
+		drm_sched_wqueue_start(&ring->sched);
+	}
+
+	return 0;
+}
+
 bool amdgpu_ring_is_reset_type_supported(struct amdgpu_ring *ring,
 					 u32 reset_type)
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 8f28b3bd70106..015623b8fd051 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -68,8 +68,6 @@ enum amdgpu_ring_priority_level {
 
 #define to_amdgpu_ring(s) container_of((s), struct amdgpu_ring, sched)
 
-#define AMDGPU_IB_POOL_SIZE	(1024 * 1024)
-
 enum amdgpu_ring_type {
 	AMDGPU_RING_TYPE_GFX		= AMDGPU_HW_IP_GFX,
 	AMDGPU_RING_TYPE_COMPUTE	= AMDGPU_HW_IP_COMPUTE,
@@ -314,6 +312,7 @@ struct amdgpu_ring {
 	uint32_t		*ring_backup;
 	unsigned int		ring_backup_entries_to_copy;
 	bool			reemit;
+	struct amdgpu_fence	*guilty_fence;
 	unsigned		rptr_offs;
 	u64			rptr_gpu_addr;
 	u32			*rptr_cpu_addr;
@@ -584,14 +583,23 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 		       struct dma_fence **f);
 int amdgpu_ib_pool_init(struct amdgpu_device *adev);
 void amdgpu_ib_pool_fini(struct amdgpu_device *adev);
+gfp_t amdgpu_ib_pool_gfp_flags(struct amdgpu_device *adev,
+			       enum amdgpu_ib_pool_type type);
 int amdgpu_ib_ring_tests(struct amdgpu_device *adev);
 bool amdgpu_ring_sched_ready(struct amdgpu_ring *ring);
 void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring,
 					     struct amdgpu_fence *guilty_fence);
+struct amdgpu_fence *
+amdgpu_ring_find_guilty_fence(struct amdgpu_ring *ring);
 void amdgpu_ring_reset_helper_begin(struct amdgpu_ring *ring,
 				    struct amdgpu_fence *guilty_fence);
 int amdgpu_ring_reset_helper_end(struct amdgpu_ring *ring,
 				 struct amdgpu_fence *guilty_fence);
+void amdgpu_multi_ring_reset_helper_begin(const u32 ring_type_mask,
+					  struct amdgpu_ring *guilty_ring,
+					  struct amdgpu_fence *guilty_fence);
+int amdgpu_multi_ring_reset_helper_end(const u32 ring_type_mask,
+				       struct amdgpu_ring *guilty_ring, int ret);
 bool amdgpu_ring_is_reset_type_supported(struct amdgpu_ring *ring,
 					 u32 reset_type);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c
index 572a60e1b3cbb..002fae3c380e1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c
@@ -583,3 +583,42 @@ int amdgpu_gfx_rlc_init_microcode(struct amdgpu_device *adev,
 		amdgpu_gfx_rlc_init_microcode_v2_5(adev);
 	return 0;
 }
+
+static const struct amdgpu_rlc_reg_funcs amdgpu_sriov_rlc_reg_funcs = {
+	.rreg32 = amdgpu_sriov_rreg,
+	.wreg32 = amdgpu_sriov_wreg,
+};
+
+static u32
+amdgpu_rlc_rreg(struct amdgpu_device *adev, u32 reg, u32 acc_flags, u32 hwip,
+		u32 xcc_id)
+{
+	return amdgpu_device_rreg(adev, reg, 0);
+}
+
+static void
+amdgpu_rlc_wreg(struct amdgpu_device *adev, u32 reg, u32 value, u32 acc_flags,
+		u32 hwip, u32 xcc_id)
+{
+	amdgpu_device_wreg(adev, reg, value, 0);
+}
+
+static const struct amdgpu_rlc_reg_funcs amdgpu_rlc_reg_funcs = {
+	.rreg32 = amdgpu_rlc_rreg,
+	.wreg32 = amdgpu_rlc_wreg,
+};
+
+void amdgpu_early_init_rlc_reg_funcs(struct amdgpu_device *adev)
+{
+	adev->gfx.rlc.reg_funcs = &amdgpu_rlc_reg_funcs;
+}
+
+void amdgpu_init_rlc_reg_funcs(struct amdgpu_device *adev)
+{
+	if (amdgpu_sriov_vf(adev) &&
+	    adev->gfx.rlc.funcs &&
+	    adev->gfx.rlc.rlcg_reg_access_supported)
+		adev->gfx.rlc.reg_funcs = &amdgpu_sriov_rlc_reg_funcs;
+	else
+		adev->gfx.rlc.reg_funcs = &amdgpu_rlc_reg_funcs;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
index e535534237a1f..959d60c90dcd2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
@@ -262,6 +262,11 @@ struct amdgpu_rlc_funcs {
 	bool (*is_rlcg_access_range)(struct amdgpu_device *adev, uint32_t reg);
 };
 
+struct amdgpu_rlc_reg_funcs {
+	u32  (*rreg32)(struct amdgpu_device *adev, u32 reg, u32 acc_flags, u32 hwip, u32 xcc_id);
+	void (*wreg32)(struct amdgpu_device *adev, u32 reg, u32 val, u32 acc_flags, u32 hwip, u32 xcc_id);
+};
+
 struct amdgpu_rlcg_reg_access_ctrl {
 	uint32_t scratch_reg0;
 	uint32_t scratch_reg1;
@@ -303,6 +308,7 @@ struct amdgpu_rlc {
 	/* safe mode for updating CG/PG state */
 	bool in_safe_mode[AMDGPU_MAX_RLC_INSTANCES];
 	const struct amdgpu_rlc_funcs *funcs;
+	const struct amdgpu_rlc_reg_funcs *reg_funcs;
 
 	/* for firmware data */
 	u32 save_and_restore_offset;
@@ -374,4 +380,8 @@ void amdgpu_gfx_rlc_fini(struct amdgpu_device *adev);
 int amdgpu_gfx_rlc_init_microcode(struct amdgpu_device *adev,
 				  uint16_t version_major,
 				  uint16_t version_minor);
+
+void amdgpu_early_init_rlc_reg_funcs(struct amdgpu_device *adev);
+void amdgpu_init_rlc_reg_funcs(struct amdgpu_device *adev);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
index 39070b2a4c04f..74124f80601ee 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
@@ -46,11 +46,13 @@
 
 int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
 			      struct amdgpu_sa_manager *sa_manager,
-			      unsigned int size, u32 suballoc_align, u32 domain)
+			      unsigned int size, gfp_t gfp_flags)
 {
 	int r;
 
-	r = amdgpu_bo_create_kernel(adev, size, AMDGPU_GPU_PAGE_SIZE, domain,
+	sa_manager->gfp_flags = gfp_flags;
+	r = amdgpu_bo_create_kernel(adev, size, AMDGPU_GPU_PAGE_SIZE,
+				    AMDGPU_GEM_DOMAIN_GTT,
 				    &sa_manager->bo, &sa_manager->gpu_addr,
 				    &sa_manager->cpu_ptr);
 	if (r) {
@@ -59,7 +61,8 @@ int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
 	}
 
 	memset(sa_manager->cpu_ptr, 0, size);
-	drm_suballoc_manager_init(&sa_manager->base, size, suballoc_align);
+	drm_suballoc_manager_init(&sa_manager->base, size, 256);
+
 	return r;
 }
 
@@ -73,7 +76,8 @@ void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
 
 	drm_suballoc_manager_fini(&sa_manager->base);
 
-	amdgpu_bo_free_kernel(&sa_manager->bo, &sa_manager->gpu_addr, &sa_manager->cpu_ptr);
+	amdgpu_bo_free_kernel(&sa_manager->bo, &sa_manager->gpu_addr,
+			      &sa_manager->cpu_ptr);
 }
 
 int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
@@ -81,7 +85,8 @@ int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
 		     unsigned int size)
 {
 	struct drm_suballoc *sa = drm_suballoc_new(&sa_manager->base, size,
-						   GFP_KERNEL, false, 0);
+						   sa_manager->gfp_flags,
+						   false, 0);
 
 	if (IS_ERR(sa)) {
 		*sa_bo = NULL;
@@ -110,6 +115,7 @@ void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
 {
 	struct drm_printer p = drm_seq_file_printer(m);
 
-	drm_suballoc_dump_debug_info(&sa_manager->base, &p, sa_manager->gpu_addr);
+	drm_suballoc_dump_debug_info(&sa_manager->base, &p,
+				     sa_manager->gpu_addr);
 }
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
new file mode 100644
index 0000000000000..1d1c89348709b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.h
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef AMDGPU_SA_H_
+#define AMDGPU_SA_H_
+
+#include <drm/drm_suballoc.h>
+
+struct amdgpu_device;
+struct amdgpu_bo;
+
+struct amdgpu_sa_manager {
+	struct drm_suballoc_manager	base;
+	struct amdgpu_bo		*bo;
+	uint64_t			gpu_addr;
+	void				*cpu_ptr;
+	gfp_t				gfp_flags;
+};
+
+static inline struct amdgpu_sa_manager *
+to_amdgpu_sa_manager(struct drm_suballoc_manager *manager)
+{
+	return container_of(manager, struct amdgpu_sa_manager, base);
+}
+
+static inline uint64_t amdgpu_sa_bo_gpu_addr(struct drm_suballoc *sa_bo)
+{
+	return to_amdgpu_sa_manager(sa_bo->manager)->gpu_addr +
+		drm_suballoc_soffset(sa_bo);
+}
+
+static inline void *amdgpu_sa_bo_cpu_addr(struct drm_suballoc *sa_bo)
+{
+	return to_amdgpu_sa_manager(sa_bo->manager)->cpu_ptr +
+		drm_suballoc_soffset(sa_bo);
+}
+
+int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
+			      struct amdgpu_sa_manager *sa_manager,
+			      unsigned size, gfp_t gfp_flags);
+void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
+			       struct amdgpu_sa_manager *sa_manager);
+int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev,
+			       struct amdgpu_sa_manager *sa_manager);
+int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
+		     struct drm_suballoc **sa_bo,
+		     unsigned int size);
+void amdgpu_sa_bo_free(struct drm_suballoc **sa_bo,
+		       struct dma_fence *fence);
+#if defined(CONFIG_DEBUG_FS)
+void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
+				  struct seq_file *m);
+u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m);
+#endif
+void amdgpu_debugfs_sa_init(struct amdgpu_device *adev);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
index 0eecfaa3a94c1..8effb1158430d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
@@ -39,7 +39,7 @@ static int amdgpu_sched_process_priority_override(struct amdgpu_device *adev,
 	struct amdgpu_fpriv *fpriv;
 	struct amdgpu_ctx_mgr *mgr;
 	struct amdgpu_ctx *ctx;
-	uint32_t id;
+	unsigned long id;
 	int r;
 
 	if (fd_empty(f))
@@ -50,10 +50,10 @@ static int amdgpu_sched_process_priority_override(struct amdgpu_device *adev,
 		return r;
 
 	mgr = &fpriv->ctx_mgr;
-	mutex_lock(&mgr->lock);
-	idr_for_each_entry(&mgr->ctx_handles, ctx, id)
+	xa_lock(&mgr->ctx_handles);
+	xa_for_each(&mgr->ctx_handles, id, ctx)
 		amdgpu_ctx_priority_override(ctx, priority);
-	mutex_unlock(&mgr->lock);
+	xa_unlock(&mgr->ctx_handles);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c
index fcd81242059e6..fbac732f3e01a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c
@@ -553,10 +553,11 @@ static int amdgpu_sdma_soft_reset(struct amdgpu_device *adev, u32 instance_id)
 int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,
 			     bool caller_handles_kernel_queues)
 {
-	int ret = 0;
 	struct amdgpu_sdma_instance *sdma_instance = &adev->sdma.instance[instance_id];
 	struct amdgpu_ring *gfx_ring = &sdma_instance->ring;
 	struct amdgpu_ring *page_ring = &sdma_instance->page;
+	struct amdgpu_fence *gfx_fence, *page_fence;
+	int ret = 0;
 
 	if (amdgpu_sriov_vf(adev))
 		return -EOPNOTSUPP;
@@ -569,9 +570,14 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,
 		 * the reset is in progress.
 		 */
 		drm_sched_wqueue_stop(&gfx_ring->sched);
+		gfx_fence = amdgpu_ring_find_guilty_fence(gfx_ring);
+		amdgpu_ring_reset_helper_begin(gfx_ring, gfx_fence);
 
-		if (adev->sdma.has_page_queue)
+		if (adev->sdma.has_page_queue) {
 			drm_sched_wqueue_stop(&page_ring->sched);
+			page_fence = amdgpu_ring_find_guilty_fence(page_ring);
+			amdgpu_ring_reset_helper_begin(page_ring, page_fence);
+		}
 	}
 
 	if (sdma_instance->funcs->stop_kernel_queue) {
@@ -600,14 +606,19 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,
 		 * to be submitted to the queues after the reset is complete.
 		 */
 		if (!ret) {
-			amdgpu_fence_driver_force_completion(gfx_ring, NULL);
+			ret = amdgpu_ring_reset_helper_end(gfx_ring, gfx_fence);
+			if (ret)
+				goto unlock;
 			drm_sched_wqueue_start(&gfx_ring->sched);
 			if (adev->sdma.has_page_queue) {
-				amdgpu_fence_driver_force_completion(page_ring, NULL);
+				ret = amdgpu_ring_reset_helper_end(page_ring, page_fence);
+				if (ret)
+					goto unlock;
 				drm_sched_wqueue_start(&page_ring->sched);
 			}
 		}
 	}
+unlock:
 	mutex_unlock(&sdma_instance->engine_reset_mutex);
 
 	return ret;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
index 2bf365609775a..4f4e56022c970 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
@@ -85,34 +85,6 @@ struct amdgpu_sdma_instance {
 	const struct amdgpu_sdma_funcs   *funcs;
 };
 
-enum amdgpu_sdma_ras_memory_id {
-	AMDGPU_SDMA_MBANK_DATA_BUF0 = 1,
-	AMDGPU_SDMA_MBANK_DATA_BUF1 = 2,
-	AMDGPU_SDMA_MBANK_DATA_BUF2 = 3,
-	AMDGPU_SDMA_MBANK_DATA_BUF3 = 4,
-	AMDGPU_SDMA_MBANK_DATA_BUF4 = 5,
-	AMDGPU_SDMA_MBANK_DATA_BUF5 = 6,
-	AMDGPU_SDMA_MBANK_DATA_BUF6 = 7,
-	AMDGPU_SDMA_MBANK_DATA_BUF7 = 8,
-	AMDGPU_SDMA_MBANK_DATA_BUF8 = 9,
-	AMDGPU_SDMA_MBANK_DATA_BUF9 = 10,
-	AMDGPU_SDMA_MBANK_DATA_BUF10 = 11,
-	AMDGPU_SDMA_MBANK_DATA_BUF11 = 12,
-	AMDGPU_SDMA_MBANK_DATA_BUF12 = 13,
-	AMDGPU_SDMA_MBANK_DATA_BUF13 = 14,
-	AMDGPU_SDMA_MBANK_DATA_BUF14 = 15,
-	AMDGPU_SDMA_MBANK_DATA_BUF15 = 16,
-	AMDGPU_SDMA_UCODE_BUF = 17,
-	AMDGPU_SDMA_RB_CMD_BUF = 18,
-	AMDGPU_SDMA_IB_CMD_BUF = 19,
-	AMDGPU_SDMA_UTCL1_RD_FIFO = 20,
-	AMDGPU_SDMA_UTCL1_RDBST_FIFO = 21,
-	AMDGPU_SDMA_UTCL1_WR_FIFO = 22,
-	AMDGPU_SDMA_DATA_LUT_FIFO = 23,
-	AMDGPU_SDMA_SPLIT_DAT_BUF = 24,
-	AMDGPU_SDMA_MEMORY_BLOCK_LAST,
-};
-
 struct amdgpu_sdma_ras {
 	struct amdgpu_ras_block_object ras_block;
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index 85724ec6aaf84..5324030a13f59 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -28,6 +28,8 @@
 #include <linux/types.h>
 #include <linux/tracepoint.h>
 
+#include "amdgpu_userq_fence.h"
+
 #undef TRACE_SYSTEM
 #define TRACE_SYSTEM amdgpu
 #define TRACE_INCLUDE_FILE amdgpu_trace
@@ -582,6 +584,154 @@ TRACE_EVENT(amdgpu_reset_reg_dumps,
 		      __entry->value)
 );
 
+DECLARE_EVENT_CLASS(amdgpu_userq_queue,
+	    TP_PROTO(struct amdgpu_usermode_queue *queue),
+	    TP_ARGS(queue),
+	    TP_STRUCT__entry(
+			     __field(void *, queue)
+			     __field(u64, doorbell_index)
+			     __field(int, queue_type)
+			     __field(int, state)
+			     __field(u32, xcp_id)
+			     ),
+	    TP_fast_assign(
+			   __entry->queue = queue;
+			   __entry->doorbell_index = queue->doorbell_index;
+			   __entry->queue_type = queue->queue_type;
+			   __entry->state = queue->state;
+			   __entry->xcp_id = queue->xcp_id;
+			   ),
+	    TP_printk("queue=%p, doorbell=%llu, type=%d, state=%d, xcp_id=%u",
+		      __entry->queue, __entry->doorbell_index,
+		      __entry->queue_type, __entry->state, __entry->xcp_id)
+);
+DEFINE_EVENT(amdgpu_userq_queue, amdgpu_userq_create_start,
+	     TP_PROTO(struct amdgpu_usermode_queue *queue),
+	     TP_ARGS(queue));
+DEFINE_EVENT(amdgpu_userq_queue, amdgpu_userq_destroy_start,
+	     TP_PROTO(struct amdgpu_usermode_queue *queue),
+	     TP_ARGS(queue));
+DECLARE_EVENT_CLASS(amdgpu_userq_queue_result,
+	    TP_PROTO(struct amdgpu_usermode_queue *queue, int result),
+	    TP_ARGS(queue, result),
+	    TP_STRUCT__entry(
+			     __field(void *, queue)
+			     __field(u64, doorbell_index)
+			     __field(int, queue_type)
+			     __field(int, state)
+			     __field(u32, xcp_id)
+			     __field(int, result)
+			     ),
+	    TP_fast_assign(
+			   __entry->queue = queue;
+			   __entry->doorbell_index = queue->doorbell_index;
+			   __entry->queue_type = queue->queue_type;
+			   __entry->state = queue->state;
+			   __entry->xcp_id = queue->xcp_id;
+			   __entry->result = result;
+			   ),
+	    TP_printk("queue=%p, doorbell=%llu, type=%d, state=%d, xcp_id=%u, result=%d",
+		      __entry->queue, __entry->doorbell_index,
+		      __entry->queue_type, __entry->state,
+		      __entry->xcp_id, __entry->result)
+);
+DEFINE_EVENT(amdgpu_userq_queue_result, amdgpu_userq_create_end,
+	     TP_PROTO(struct amdgpu_usermode_queue *queue, int result),
+	     TP_ARGS(queue, result));
+DEFINE_EVENT(amdgpu_userq_queue_result, amdgpu_userq_destroy_end,
+	     TP_PROTO(struct amdgpu_usermode_queue *queue, int result),
+	     TP_ARGS(queue, result));
+
+TRACE_EVENT(amdgpu_userq_emit_fence,
+	    TP_PROTO(struct device *device, struct amdgpu_usermode_queue *queue, struct amdgpu_userq_fence *fence),
+	    TP_ARGS(device, queue, fence),
+	    TP_STRUCT__entry(
+			     __field(u64, fence_context)
+			     __field(u64, fence_seqno)
+			     __string(dev, dev_name(device))
+			     __field(u64, doorbell_index)
+			     __field(u64, client_id)
+			     __field(u32, queue_type)
+			     ),
+	    TP_fast_assign(
+			   __entry->fence_context = fence->base.context;
+			   __entry->fence_seqno = fence->base.seqno;
+			   __assign_str(dev);
+			   __entry->doorbell_index = queue->doorbell_index;
+			   __entry->client_id = queue->userq_mgr->file->client_id;
+			   __entry->queue_type = queue->queue_type;
+			   ),
+	    TP_printk("dev=%s, client_id=%llu, type=%u, doorbell=%llu, fence=%llu:%llu",
+		      __get_str(dev), __entry->client_id, __entry->queue_type, __entry->doorbell_index,
+		      __entry->fence_context,
+		      __entry->fence_seqno)
+);
+
+TRACE_EVENT(amdgpu_userq_wait_deps,
+	    TP_PROTO(struct device *device, struct amdgpu_usermode_queue *queue, struct amdgpu_userq_fence *dep),
+	    TP_ARGS(device, queue, dep),
+	    TP_STRUCT__entry(
+			     __field(u64, context)
+			     __field(u64, dep_context)
+			     __field(u64, dep_seqno)
+			     __string(dev, dev_name(device))
+			     __field(u64, doorbell_index)
+			     __field(u64, client_id)
+			     __field(u32, queue_type)
+			     ),
+	    TP_fast_assign(
+			   __assign_str(dev);
+			   __entry->doorbell_index = queue->doorbell_index;
+			   __entry->queue_type = queue->queue_type;
+			   __entry->client_id = queue->userq_mgr->file->client_id;
+			   __entry->context = queue->fence_drv->context;
+			   __entry->dep_context = dep->base.context;
+			   __entry->dep_seqno = dep->base.seqno;
+			   ),
+	    TP_printk("dev=%s, client_id=%llu, type=%u, doorbell=%llu, context=%llu depends on fence=%llu:%llu",
+		      __get_str(dev), __entry->client_id, __entry->queue_type, __entry->doorbell_index, __entry->context,
+		      __entry->dep_context,
+		      __entry->dep_seqno)
+);
+
+TRACE_EVENT(amdgpu_userq_state_start,
+	    TP_PROTO(struct amdgpu_usermode_queue *queue),
+	    TP_ARGS(queue),
+	    TP_STRUCT__entry(
+			     __field(u64, doorbell_index)
+			     __field(u64, client_id)
+			     __field(u32, queue_type)
+			     __field(u32, from)
+			     ),
+	    TP_fast_assign(
+			   __entry->doorbell_index = queue->doorbell_index;
+			   __entry->queue_type = queue->queue_type;
+			   __entry->client_id = queue->userq_mgr->file->client_id;
+			   __entry->from = queue->state;
+			   ),
+	    TP_printk("client_id=%llu, type=%u, doorbell=%llu, from=%d",
+		      __entry->client_id, __entry->queue_type, __entry->doorbell_index, __entry->from)
+);
+
+TRACE_EVENT(amdgpu_userq_state_changed,
+	    TP_PROTO(struct amdgpu_usermode_queue *queue, enum amdgpu_userq_state new_state),
+	    TP_ARGS(queue, new_state),
+	    TP_STRUCT__entry(
+			     __field(u64, doorbell_index)
+			     __field(u64, client_id)
+			     __field(u32, queue_type)
+			     __field(u32, to)
+			     ),
+	    TP_fast_assign(
+			   __entry->doorbell_index = queue->doorbell_index;
+			   __entry->queue_type = queue->queue_type;
+			   __entry->client_id = queue->userq_mgr->file->client_id;
+			   __entry->to = new_state;
+			   ),
+	    TP_printk("client_id=%llu, type=%u, doorbell=%llu, to=%d",
+		      __entry->client_id, __entry->queue_type, __entry->doorbell_index, __entry->to)
+);
+
 #undef AMDGPU_JOB_GET_TIMELINE_NAME
 #endif
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 025625e7e800f..03c1e5e3580c6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -244,8 +244,9 @@ static int amdgpu_ttm_map_buffer(struct amdgpu_ttm_buffer_entity *entity,
 	r = amdgpu_job_alloc_with_ib(adev, &entity->base,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     num_dw * 4 + num_bytes,
-				     AMDGPU_IB_POOL_DELAYED, &job,
-				     AMDGPU_KERNEL_JOB_ID_TTM_MAP_BUFFER);
+				     AMDGPU_IB_POOL_DELAYED,
+				     AMDGPU_KERNEL_JOB_ID_TTM_MAP_BUFFER,
+				     &job);
 	if (r)
 		return r;
 
@@ -1592,8 +1593,8 @@ static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo,
 	r = amdgpu_job_alloc_with_ib(adev, &adev->mman.default_entity.base,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     num_dw * 4, AMDGPU_IB_POOL_DELAYED,
-				     &job,
-				     AMDGPU_KERNEL_JOB_ID_TTM_ACCESS_MEMORY_SDMA);
+				     AMDGPU_KERNEL_JOB_ID_TTM_ACCESS_MEMORY_SDMA,
+				     &job);
 	if (r)
 		goto out;
 
@@ -2194,7 +2195,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 		return r;
 	}
 
-	/* Create a boorbell page for kernel usages */
+	/* Create a doorbell page for kernel usages */
 	r = amdgpu_doorbell_create_kernel_doorbells(adev);
 	if (r) {
 		dev_err(adev->dev, "Failed to initialize kernel doorbells.\n");
@@ -2443,7 +2444,7 @@ static int amdgpu_ttm_prepare_job(struct amdgpu_device *adev,
 	int r;
 	r = amdgpu_job_alloc_with_ib(adev, &entity->base,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
-				     num_dw * 4, pool, job, k_job_id);
+				     num_dw * 4, pool, k_job_id, job);
 	if (r)
 		return r;
 
@@ -2684,12 +2685,22 @@ void amdgpu_sdma_set_buffer_funcs_scheds(struct amdgpu_device *adev,
 		return;
 	}
 
-	/* Navi1x's workaround requires us to limit to a single SDMA sched
-	 * for ttm.
-	 */
 	hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
-	adev->mman.num_buffer_funcs_scheds = hub->sdma_invalidation_workaround ?
-		1 : n;
+
+	/*
+	 * Allow using multiple SDMA schedulers only on GPUs where
+	 * we are allowed to do concurrent VM flushes.
+	 * This consideration is necessary because all GART windows
+	 * are mapped in VMID 0 (the kernel VMID) so each buffer
+	 * entity would flush VMID 0 concurrently.
+	 *
+	 * Also consider the SDMA invalidation workaround on
+	 * Navi 1x GPUs, which also prevents us from using
+	 * multiple SDMA engines on VMID 0 at the same time.
+	 */
+	adev->mman.num_buffer_funcs_scheds =
+		(adev->vm_manager.concurrent_flush &&
+		 !hub->sdma_invalidation_workaround) ? n : 1;
 }
 
 #if defined(CONFIG_DEBUG_FS)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index b5d938b313838..ff9e2e3466099 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -140,6 +140,7 @@ int amdgpu_gtt_mgr_init(struct amdgpu_device *adev, uint64_t gtt_size);
 void amdgpu_gtt_mgr_fini(struct amdgpu_device *adev);
 int amdgpu_preempt_mgr_init(struct amdgpu_device *adev);
 void amdgpu_preempt_mgr_fini(struct amdgpu_device *adev);
+void amdgpu_preempt_mgr_sysfs_fini(struct amdgpu_device *adev);
 int amdgpu_vram_mgr_init(struct amdgpu_device *adev);
 void amdgpu_vram_mgr_fini(struct amdgpu_device *adev);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
index 6d9e96fabd581..4f1c711df5bd7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
@@ -1478,7 +1478,8 @@ void amdgpu_ucode_ip_version_decode(struct amdgpu_device *adev, int block_type,
 		ip_name = "isp";
 		break;
 	default:
-		BUG();
+		WARN(1, "invalid HWIP %d\n", block_type);
+		return;
 	}
 
 	maj = IP_VERSION_MAJ(version);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
index b8ed931f8a407..a98a6cfd4fba8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
@@ -97,7 +97,6 @@ void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev,
 {
 	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-	struct amdgpu_ras_eeprom_control *control = &con->eeprom_control;
 	unsigned int error_query_mode;
 	int ret = 0;
 	unsigned long err_count;
@@ -118,77 +117,66 @@ void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev,
 		err_data->err_addr_len = adev->umc.max_ras_err_cnt_per_query;
 
 	mutex_lock(&con->page_retirement_lock);
-	if (!amdgpu_ras_smu_eeprom_supported(adev)) {
-		ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(con->umc_ecc));
-		if (ret == -EOPNOTSUPP &&
-		    error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) {
-			if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
-			    adev->umc.ras->ras_block.hw_ops->query_ras_error_count)
-				adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev,
-								ras_error_status);
-
-			if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
-			    adev->umc.ras->ras_block.hw_ops->query_ras_error_address &&
-			    adev->umc.max_ras_err_cnt_per_query) {
-				kfree(err_data->err_addr);
-				err_data->err_addr =
-					kzalloc_objs(struct eeprom_table_record,
-						     adev->umc.max_ras_err_cnt_per_query);
-
-				/* still call query_ras_error_address to clear error status
-				 * even NOMEM error is encountered
-				 */
-				if (!err_data->err_addr)
-					dev_warn(adev->dev,
-						"Failed to alloc memory for umc error address record!\n");
-				else
-					err_data->err_addr_len =
-						adev->umc.max_ras_err_cnt_per_query;
-
-				/* umc query_ras_error_address is also responsible for clearing
-				 * error status
-				 */
-				adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev,
-								ras_error_status);
-			}
-		} else if (error_query_mode == AMDGPU_RAS_FIRMWARE_ERROR_QUERY ||
-		    (!ret && error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY)) {
-			if (adev->umc.ras &&
-			    adev->umc.ras->ecc_info_query_ras_error_count)
-				adev->umc.ras->ecc_info_query_ras_error_count(adev,
-								ras_error_status);
-
-			if (adev->umc.ras &&
-			    adev->umc.ras->ecc_info_query_ras_error_address &&
-			    adev->umc.max_ras_err_cnt_per_query) {
-				kfree(err_data->err_addr);
-				err_data->err_addr =
-					kzalloc_objs(struct eeprom_table_record,
-						     adev->umc.max_ras_err_cnt_per_query);
-
-				/* still call query_ras_error_address to clear error status
-				 * even NOMEM error is encountered
-				 */
-				if (!err_data->err_addr)
-					dev_warn(adev->dev,
-						"Failed to alloc memory for umc error address record!\n");
-				else
-					err_data->err_addr_len =
-						adev->umc.max_ras_err_cnt_per_query;
-
-				/* umc query_ras_error_address is also responsible for clearing
-				 * error status
-				 */
-				adev->umc.ras->ecc_info_query_ras_error_address(adev,
-								ras_error_status);
-			}
+	ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(con->umc_ecc));
+	if (ret == -EOPNOTSUPP &&
+	    error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) {
+		if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
+		    adev->umc.ras->ras_block.hw_ops->query_ras_error_count)
+			adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev,
+							ras_error_status);
+
+		if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
+		    adev->umc.ras->ras_block.hw_ops->query_ras_error_address &&
+		    adev->umc.max_ras_err_cnt_per_query) {
+			err_data->err_addr =
+				kzalloc_objs(struct eeprom_table_record,
+					     adev->umc.max_ras_err_cnt_per_query);
+
+			/* still call query_ras_error_address to clear error status
+			 * even NOMEM error is encountered
+			 */
+			if (!err_data->err_addr)
+				dev_warn(adev->dev,
+					"Failed to alloc memory for umc error address record!\n");
+			else
+				err_data->err_addr_len =
+					adev->umc.max_ras_err_cnt_per_query;
+
+			/* umc query_ras_error_address is also responsible for clearing
+			 * error status
+			 */
+			adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev,
+							ras_error_status);
 		}
-	} else {
-		if (!amdgpu_ras_eeprom_update_record_num(control)) {
-			err_data->err_addr_cnt = err_data->de_count =
-				control->ras_num_recs -	control->ras_num_recs_old;
-			amdgpu_ras_eeprom_read_idx(control, err_data->err_addr,
-				control->ras_num_recs_old, err_data->de_count);
+	} else if (error_query_mode == AMDGPU_RAS_FIRMWARE_ERROR_QUERY ||
+	    (!ret && error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY)) {
+		if (adev->umc.ras &&
+		    adev->umc.ras->ecc_info_query_ras_error_count)
+			adev->umc.ras->ecc_info_query_ras_error_count(adev,
+							ras_error_status);
+
+		if (adev->umc.ras &&
+		    adev->umc.ras->ecc_info_query_ras_error_address &&
+		    adev->umc.max_ras_err_cnt_per_query) {
+			err_data->err_addr =
+				kcalloc(adev->umc.max_ras_err_cnt_per_query,
+					sizeof(struct eeprom_table_record), GFP_KERNEL);
+
+			/* still call query_ras_error_address to clear error status
+			 * even NOMEM error is encountered
+			 */
+			if (!err_data->err_addr)
+				dev_warn(adev->dev,
+					"Failed to alloc memory for umc error address record!\n");
+			else
+				err_data->err_addr_len =
+					adev->umc.max_ras_err_cnt_per_query;
+
+			/* umc query_ras_error_address is also responsible for clearing
+			 * error status
+			 */
+			adev->umc.ras->ecc_info_query_ras_error_address(adev,
+							ras_error_status);
 		}
 	}
 
@@ -198,7 +186,7 @@ void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev,
 		if ((amdgpu_bad_page_threshold != 0) &&
 			err_data->err_addr_cnt) {
 			amdgpu_ras_add_bad_pages(adev, err_data->err_addr,
-				err_data->err_addr_cnt, amdgpu_ras_smu_eeprom_supported(adev));
+				err_data->err_addr_cnt, false);
 			amdgpu_ras_save_bad_pages(adev, &err_count);
 
 			amdgpu_dpm_send_hbm_bad_pages_num(adev,
@@ -272,11 +260,10 @@ int amdgpu_umc_pasid_poison_handler(struct amdgpu_device *adev,
 			if (ret == AMDGPU_RAS_SUCCESS && obj) {
 				obj->err_data.ue_count += err_data.ue_count;
 				obj->err_data.ce_count += err_data.ce_count;
-				obj->err_data.de_count += err_data.de_count;
 			}
 
 			amdgpu_ras_error_data_fini(&err_data);
-		} else if (amdgpu_uniras_enabled(adev)) {
+		} else {
 			struct ras_ih_info ih_info = {0};
 
 			ih_info.block = block;
@@ -285,17 +272,6 @@ int amdgpu_umc_pasid_poison_handler(struct amdgpu_device *adev,
 			ih_info.pasid_fn = pasid_fn;
 			ih_info.data = data;
 			amdgpu_ras_mgr_handle_consumer_interrupt(adev, &ih_info);
-		} else {
-			struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-			int ret;
-
-			ret = amdgpu_ras_put_poison_req(adev,
-				block, pasid, pasid_fn, data, reset);
-			if (!ret) {
-				atomic_inc(&con->page_retirement_req_cnt);
-				atomic_inc(&con->poison_consumption_count);
-				wake_up(&con->page_retirement_wq);
-			}
 		}
 	} else {
 		if (adev->virt.ops && adev->virt.ops->ras_poison_handler)
@@ -512,129 +488,3 @@ int amdgpu_umc_loop_channels(struct amdgpu_device *adev,
 
 	return 0;
 }
-
-int amdgpu_umc_update_ecc_status(struct amdgpu_device *adev,
-				uint64_t status, uint64_t ipid, uint64_t addr)
-{
-	if (adev->umc.ras->update_ecc_status)
-		return adev->umc.ras->update_ecc_status(adev,
-					status, ipid, addr);
-	return 0;
-}
-
-int amdgpu_umc_logs_ecc_err(struct amdgpu_device *adev,
-		struct radix_tree_root *ecc_tree, struct ras_ecc_err *ecc_err)
-{
-	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-	struct ras_ecc_log_info *ecc_log;
-	int ret;
-
-	ecc_log = &con->umc_ecc_log;
-
-	mutex_lock(&ecc_log->lock);
-	ret = radix_tree_insert(ecc_tree, ecc_err->pa_pfn, ecc_err);
-	if (!ret)
-		radix_tree_tag_set(ecc_tree,
-			ecc_err->pa_pfn, UMC_ECC_NEW_DETECTED_TAG);
-	mutex_unlock(&ecc_log->lock);
-
-	return ret;
-}
-
-int amdgpu_umc_pages_in_a_row(struct amdgpu_device *adev,
-			struct ras_err_data *err_data, uint64_t pa_addr)
-{
-	struct ta_ras_query_address_output addr_out;
-
-	/* reinit err_data */
-	err_data->err_addr_cnt = 0;
-	err_data->err_addr_len = adev->umc.retire_unit;
-
-	addr_out.pa.pa = pa_addr;
-	if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
-		return adev->umc.ras->convert_ras_err_addr(adev, err_data, NULL,
-				&addr_out, false);
-	else
-		return -EINVAL;
-}
-
-int amdgpu_umc_lookup_bad_pages_in_a_row(struct amdgpu_device *adev,
-			uint64_t pa_addr, uint64_t *pfns, int len)
-{
-	int i, ret;
-	struct ras_err_data err_data;
-
-	err_data.err_addr = kzalloc_objs(struct eeprom_table_record,
-					 adev->umc.retire_unit);
-	if (!err_data.err_addr) {
-		dev_warn(adev->dev, "Failed to alloc memory in bad page lookup!\n");
-		return 0;
-	}
-
-	ret = amdgpu_umc_pages_in_a_row(adev, &err_data, pa_addr);
-	if (ret)
-		goto out;
-
-	for (i = 0; i < adev->umc.retire_unit; i++) {
-		if (i >= len)
-			goto out;
-
-		pfns[i] = err_data.err_addr[i].retired_page;
-	}
-	ret = i;
-	adev->umc.err_addr_cnt = err_data.err_addr_cnt;
-
-out:
-	kfree(err_data.err_addr);
-	return ret;
-}
-
-int amdgpu_umc_mca_to_addr(struct amdgpu_device *adev,
-			uint64_t err_addr, uint32_t ch, uint32_t umc,
-			uint32_t node, uint32_t socket,
-			struct ta_ras_query_address_output *addr_out, bool dump_addr)
-{
-	struct ta_ras_query_address_input addr_in;
-	int ret;
-
-	memset(&addr_in, 0, sizeof(addr_in));
-	addr_in.ma.err_addr = err_addr;
-	addr_in.ma.ch_inst = ch;
-	addr_in.ma.umc_inst = umc;
-	addr_in.ma.node_inst = node;
-	addr_in.ma.socket_id = socket;
-
-	if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) {
-		ret = adev->umc.ras->convert_ras_err_addr(adev, NULL, &addr_in,
-				addr_out, dump_addr);
-		if (ret)
-			return ret;
-	} else {
-		return 0;
-	}
-
-	return 0;
-}
-
-int amdgpu_umc_pa2mca(struct amdgpu_device *adev,
-		uint64_t pa, uint64_t *mca, enum amdgpu_memory_partition nps)
-{
-	struct ta_ras_query_address_input addr_in;
-	struct ta_ras_query_address_output addr_out;
-	int ret;
-
-	/* nps: the pa belongs to */
-	addr_in.pa.pa = pa | ((uint64_t)nps << 58);
-	addr_in.addr_type = TA_RAS_PA_TO_MCA;
-	ret = psp_ras_query_address(&adev->psp, &addr_in, &addr_out);
-	if (ret) {
-		dev_warn(adev->dev, "Failed to query RAS MCA address for 0x%llx",
-			pa);
-
-		return ret;
-	}
-
-	*mca = addr_out.ma.err_addr;
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
index 8494a55ebf76c..b2a3db60d231e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
@@ -21,7 +21,6 @@
 #ifndef __AMDGPU_UMC_H__
 #define __AMDGPU_UMC_H__
 #include "amdgpu_ras.h"
-#include "amdgpu_mca.h"
 /*
  * (addr / 256) * 4096, the higher 26 bits in ErrorAddr
  * is the index of 4KB block
@@ -78,18 +77,6 @@
 #define UMC_NPS_SHIFT 40
 #define UMC_NPS_MASK 0xffULL
 
-/* three column bits and one row bit in MCA address flip
- * in bad page retirement
- */
-#define RETIRE_FLIP_BITS_NUM 4
-
-struct amdgpu_umc_flip_bits {
-	uint32_t flip_bits_in_pa[RETIRE_FLIP_BITS_NUM];
-	uint32_t flip_row_bit;
-	uint32_t r13_in_pa;
-	uint32_t bit_num;
-};
-
 typedef int (*umc_func)(struct amdgpu_device *adev, uint32_t node_inst,
 			uint32_t umc_inst, uint32_t ch_inst, void *data);
 
@@ -101,20 +88,6 @@ struct amdgpu_umc_ras {
 				      void *ras_error_status);
 	void (*ecc_info_query_ras_error_address)(struct amdgpu_device *adev,
 					void *ras_error_status);
-	bool (*check_ecc_err_status)(struct amdgpu_device *adev,
-			enum amdgpu_mca_error_type type, void *ras_error_status);
-	int (*update_ecc_status)(struct amdgpu_device *adev,
-			uint64_t status, uint64_t ipid, uint64_t addr);
-	int (*convert_ras_err_addr)(struct amdgpu_device *adev,
-			struct ras_err_data *err_data,
-			struct ta_ras_query_address_input *addr_in,
-			struct ta_ras_query_address_output *addr_out,
-			bool dump_addr);
-	uint32_t (*get_die_id_from_pa)(struct amdgpu_device *adev,
-			uint64_t mca_addr, uint64_t retired_page);
-	void (*get_retire_flip_bits)(struct amdgpu_device *adev);
-	void (*mca_ipid_parse)(struct amdgpu_device *adev, uint64_t ipid,
-			uint32_t *did, uint32_t *ch, uint32_t *umc_inst, uint32_t *sid);
 };
 
 struct amdgpu_umc_funcs {
@@ -146,8 +119,6 @@ struct amdgpu_umc {
 	/* active mask for umc node instance */
 	unsigned long active_mask;
 
-	struct amdgpu_umc_flip_bits flip_bits;
-
 	unsigned long err_addr_cnt;
 };
 
@@ -179,21 +150,6 @@ int amdgpu_umc_page_retirement_mca(struct amdgpu_device *adev,
 int amdgpu_umc_loop_channels(struct amdgpu_device *adev,
 			umc_func func, void *data);
 
-int amdgpu_umc_update_ecc_status(struct amdgpu_device *adev,
-				uint64_t status, uint64_t ipid, uint64_t addr);
-int amdgpu_umc_logs_ecc_err(struct amdgpu_device *adev,
-		struct radix_tree_root *ecc_tree, struct ras_ecc_err *ecc_err);
-
 void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev,
 			void *ras_error_status);
-int amdgpu_umc_pages_in_a_row(struct amdgpu_device *adev,
-			struct ras_err_data *err_data, uint64_t pa_addr);
-int amdgpu_umc_lookup_bad_pages_in_a_row(struct amdgpu_device *adev,
-			uint64_t pa_addr, uint64_t *pfns, int len);
-int amdgpu_umc_mca_to_addr(struct amdgpu_device *adev,
-			uint64_t err_addr, uint32_t ch, uint32_t umc,
-			uint32_t node, uint32_t socket,
-			struct ta_ras_query_address_output *addr_out, bool dump_addr);
-int amdgpu_umc_pa2mca(struct amdgpu_device *adev,
-		uint64_t pa, uint64_t *mca, enum amdgpu_memory_partition nps);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umsch_mm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_umsch_mm.c
index cd707d70a0bf8..3985d56008cde 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umsch_mm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umsch_mm.c
@@ -272,7 +272,7 @@ static int umsch_mm_init(struct amdgpu_device *adev)
 	adev->umsch_mm.engine_mask = (1 << UMSCH_SWIP_ENGINE_TYPE_VPE);
 	adev->umsch_mm.vpe_hqd_mask = 0xfe;
 
-	r = amdgpu_device_wb_get(adev, &adev->umsch_mm.wb_index);
+	r = amdgpu_wb_get(adev, &adev->umsch_mm.wb_index);
 	if (r) {
 		dev_err(adev->dev, "failed to alloc wb for umsch: %d\n", r);
 		return r;
@@ -288,7 +288,7 @@ static int umsch_mm_init(struct amdgpu_device *adev)
 				    (void **)&adev->umsch_mm.cmd_buf_ptr);
 	if (r) {
 		dev_err(adev->dev, "failed to allocate cmdbuf bo %d\n", r);
-		amdgpu_device_wb_free(adev, adev->umsch_mm.wb_index);
+		amdgpu_wb_free(adev, adev->umsch_mm.wb_index);
 		return r;
 	}
 
@@ -380,7 +380,7 @@ static int umsch_mm_sw_fini(struct amdgpu_ip_block *ip_block)
 				    &adev->umsch_mm.log_gpu_addr,
 				    (void **)&adev->umsch_mm.log_cpu_addr);
 
-	amdgpu_device_wb_free(adev, adev->umsch_mm.wb_index);
+	amdgpu_wb_free(adev, adev->umsch_mm.wb_index);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index d854343b3734d..3fe10d6af7575 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -33,6 +33,7 @@
 #include "amdgpu_userq.h"
 #include "amdgpu_hmm.h"
 #include "amdgpu_userq_fence.h"
+#include "amdgpu_trace.h"
 
 u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev)
 {
@@ -88,14 +89,7 @@ static void amdgpu_userq_mgr_reset_work(struct work_struct *work)
 		container_of(work, struct amdgpu_userq_mgr,
 			     reset_work);
 	struct amdgpu_device *adev = uq_mgr->adev;
-	const int queue_types[] = {
-		AMDGPU_RING_TYPE_COMPUTE,
-		AMDGPU_RING_TYPE_GFX,
-		AMDGPU_RING_TYPE_SDMA
-	};
-	const int num_queue_types = ARRAY_SIZE(queue_types);
-	bool gpu_reset = false;
-	int i, r;
+	struct amdgpu_reset_context reset_context;
 
 	if (unlikely(adev->debug_disable_gpu_ring_reset)) {
 		dev_err(adev->dev, "userq reset disabled by debug mask\n");
@@ -109,42 +103,15 @@ static void amdgpu_userq_mgr_reset_work(struct work_struct *work)
 	if (!amdgpu_gpu_recovery)
 		return;
 
-	/*
-	 * Iterate through all queue types to detect and reset problematic queues
-	 * Process each queue type in the defined order
-	 */
-	for (i = 0; i < num_queue_types; i++) {
-		int ring_type = queue_types[i];
-		const struct amdgpu_userq_funcs *funcs =
-			adev->userq_funcs[ring_type];
-
-		if (!amdgpu_userq_is_reset_type_supported(adev, ring_type,
-							  AMDGPU_RESET_TYPE_PER_QUEUE))
-				continue;
-
-		if (atomic_read(&uq_mgr->userq_count[ring_type]) > 0 &&
-		    funcs && funcs->detect_and_reset) {
-			r = funcs->detect_and_reset(adev, ring_type);
-			if (r) {
-				gpu_reset = true;
-				break;
-			}
-		}
-	}
-
-	if (gpu_reset) {
-		struct amdgpu_reset_context reset_context;
+	memset(&reset_context, 0, sizeof(reset_context));
 
-		memset(&reset_context, 0, sizeof(reset_context));
+	reset_context.method = AMD_RESET_METHOD_NONE;
+	reset_context.reset_req_dev = adev;
+	reset_context.src = AMDGPU_RESET_SRC_USERQ;
+	set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
+	/*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/
 
-		reset_context.method = AMD_RESET_METHOD_NONE;
-		reset_context.reset_req_dev = adev;
-		reset_context.src = AMDGPU_RESET_SRC_USERQ;
-		set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
-		/*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/
-
-		amdgpu_device_gpu_recover(adev, NULL, &reset_context);
-	}
+	amdgpu_device_gpu_recover(adev, NULL, &reset_context);
 }
 
 static void amdgpu_userq_hang_detect_work(struct work_struct *work)
@@ -152,12 +119,61 @@ static void amdgpu_userq_hang_detect_work(struct work_struct *work)
 	struct amdgpu_usermode_queue *queue =
 		container_of(work, struct amdgpu_usermode_queue,
 			     hang_detect_work.work);
+	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
+	struct amdgpu_device *adev = uq_mgr->adev;
+	const struct amdgpu_userq_funcs *userq_funcs =
+		adev->userq_funcs[queue->queue_type];
+	struct drm_wedge_task_info *info = NULL;
+	struct amdgpu_task_info *ti = NULL;
+	bool gpu_reset = false;
+
+	if (unlikely(adev->debug_disable_gpu_ring_reset)) {
+		dev_err(adev->dev, "userq reset disabled by debug mask\n");
+		return;
+	}
+
+	/*
+	 * If GPU recovery feature is disabled system-wide,
+	 * skip all reset detection logic
+	 */
+	if (!amdgpu_gpu_recovery)
+		return;
+
+	if (queue->vm && queue->vm->pasid) {
+		ti = amdgpu_vm_get_task_info_pasid(adev, queue->vm->pasid);
+		if (ti) {
+			amdgpu_vm_print_task_info(adev, ti);
+			info = &ti->task;
+		}
+	}
+
+	if (amdgpu_userq_is_reset_type_supported(adev, queue->queue_type,
+						 AMDGPU_RESET_TYPE_PER_QUEUE)) {
+		int r;
+
+		if (queue->queue_type == AMDGPU_HW_IP_COMPUTE)
+			r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL,
+							 queue, NULL, NULL);
+		else
+			r = userq_funcs->reset(queue);
+		if (r) {
+			gpu_reset = true;
+		} else {
+			atomic_inc(&adev->gpu_reset_counter);
+			amdgpu_userq_fence_driver_force_completion(queue);
+			drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, info);
+		}
+	} else {
+		gpu_reset = true;
+	}
+	amdgpu_vm_put_task_info(ti);
 
 	/*
 	 * Don't schedule the work here! Scheduling or queue work from one reset
 	 * handler to another is illegal if you don't take extra precautions!
 	 */
-	amdgpu_userq_mgr_reset_work(&queue->userq_mgr->reset_work);
+	if (gpu_reset)
+		amdgpu_userq_mgr_reset_work(&queue->userq_mgr->reset_work);
 }
 
 /*
@@ -293,11 +309,15 @@ static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue)
 	int r;
 
 	if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
+		trace_amdgpu_userq_state_start(queue);
+
 		r = userq_funcs->preempt(queue);
 		if (r) {
+			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
 			queue->state = AMDGPU_USERQ_STATE_HUNG;
 			return r;
 		} else {
+			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_PREEMPTED);
 			queue->state = AMDGPU_USERQ_STATE_PREEMPTED;
 		}
 	}
@@ -313,10 +333,14 @@ static int amdgpu_userq_restore_helper(struct amdgpu_usermode_queue *queue)
 	int r = 0;
 
 	if (queue->state == AMDGPU_USERQ_STATE_PREEMPTED) {
+		trace_amdgpu_userq_state_start(queue);
+
 		r = userq_funcs->restore(queue);
 		if (r) {
+			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
 			queue->state = AMDGPU_USERQ_STATE_HUNG;
 		} else {
+			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED);
 			queue->state = AMDGPU_USERQ_STATE_MAPPED;
 		}
 	}
@@ -334,12 +358,15 @@ static int amdgpu_userq_unmap_helper(struct amdgpu_usermode_queue *queue)
 
 	if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) ||
 	    (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {
+		trace_amdgpu_userq_state_start(queue);
 
 		r = userq_funcs->unmap(queue);
 		if (r) {
+			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
 			queue->state = AMDGPU_USERQ_STATE_HUNG;
 			return r;
 		} else {
+			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_UNMAPPED);
 			queue->state = AMDGPU_USERQ_STATE_UNMAPPED;
 		}
 	}
@@ -356,11 +383,15 @@ static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue)
 	int r;
 
 	if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) {
+		trace_amdgpu_userq_state_start(queue);
+
 		r = userq_funcs->map(queue);
 		if (r) {
+			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
 			queue->state = AMDGPU_USERQ_STATE_HUNG;
 			return r;
 		} else {
+			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED);
 			queue->state = AMDGPU_USERQ_STATE_MAPPED;
 		}
 	}
@@ -507,6 +538,8 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_que
 	const struct amdgpu_userq_funcs *uq_funcs = adev->userq_funcs[queue->queue_type];
 	int r = 0;
 
+	trace_amdgpu_userq_destroy_start(queue);
+
 	cancel_delayed_work_sync(&uq_mgr->resume_work);
 
 	/* Cancel any pending hang detection work and cleanup */
@@ -541,6 +574,7 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_que
 	amdgpu_bo_unreserve(queue->db_obj.obj);
 	amdgpu_bo_unref(&queue->db_obj.obj);
 
+	trace_amdgpu_userq_destroy_end(queue, r);
 	kfree(queue);
 
 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
@@ -638,6 +672,8 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 	queue->queue_type = args->in.ip_type;
 	queue->vm = &fpriv->vm;
 	queue->priority = priority;
+	queue->xcp_id = (fpriv->xcp_id != AMDGPU_XCP_NO_PARTITION) ?
+				fpriv->xcp_id : 0;
 	queue->userq_mgr = uq_mgr;
 	INIT_DELAYED_WORK(&queue->hang_detect_work,
 			  amdgpu_userq_hang_detect_work);
@@ -680,6 +716,8 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 	}
 
 	queue->doorbell_index = index;
+	queue->doorbell_offset = (u32)args->in.doorbell_offset;
+	trace_amdgpu_userq_create_start(queue);
 	r = uq_funcs->mqd_create(queue, &args->in);
 	if (r) {
 		drm_file_err(uq_mgr->file, "Failed to create Queue\n");
@@ -703,6 +741,7 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 		r = amdgpu_userq_map_helper(queue);
 		if (r) {
 			drm_file_err(uq_mgr->file, "Failed to map Queue\n");
+			trace_amdgpu_userq_create_end(queue, r);
 			mutex_unlock(&uq_mgr->userq_mutex);
 			goto erase_doorbell;
 		}
@@ -719,11 +758,13 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 		 * This drops the last reference which should take care of
 		 * all cleanup.
 		 */
+		trace_amdgpu_userq_create_end(queue, r);
 		amdgpu_userq_put(queue);
 		return r;
 	}
 
 	amdgpu_debugfs_userq_init(filp, queue, qid);
+	trace_amdgpu_userq_create_end(queue, 0);
 	args->out.queue_id = qid;
 	return 0;
 
@@ -739,6 +780,7 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 free_fence_drv:
 	amdgpu_userq_fence_driver_free(queue);
 free_queue:
+	trace_amdgpu_userq_create_end(queue, r);
 	kfree(queue);
 err_pm_runtime:
 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
@@ -871,16 +913,10 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data,
 static int
 amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr)
 {
-	struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
-	struct amdgpu_vm *vm = &fpriv->vm;
 	struct amdgpu_usermode_queue *queue;
 	unsigned long queue_id;
 	int ret = 0, r;
 
-
-	if (amdgpu_bo_reserve(vm->root.bo, false))
-		return false;
-
 	mutex_lock(&uq_mgr->userq_mutex);
 	/* Resume all the queues for this process */
 	xa_for_each(&uq_mgr->userq_xa, queue_id, queue) {
@@ -888,6 +924,7 @@ amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr)
 		if (!amdgpu_userq_buffer_vas_mapped(queue)) {
 			drm_file_err(uq_mgr->file,
 				     "trying restore queue without va mapping\n");
+			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_INVALID_VA);
 			queue->state = AMDGPU_USERQ_STATE_INVALID_VA;
 			continue;
 		}
@@ -895,10 +932,8 @@ amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr)
 		r = amdgpu_userq_map_helper(queue);
 		if (r)
 			ret = r;
-
 	}
 	mutex_unlock(&uq_mgr->userq_mutex);
-	amdgpu_bo_unreserve(vm->root.bo);
 
 	if (ret)
 		drm_file_err(uq_mgr->file,
@@ -932,7 +967,8 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
 		spin_unlock(&vm->individual_lock);
 
 		bo = bo_va->base.bo;
-		ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2);
+		ret = drm_exec_prepare_obj(exec, &bo->tbo.base,
+					   TTM_NUM_MOVE_FENCES + 1);
 		if (unlikely(ret))
 			return ret;
 
@@ -955,7 +991,7 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
 
 /* Make sure the whole VM is ready to be used */
 static int
-amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
+amdgpu_userq_vm_validate_and_restore_queue(struct amdgpu_userq_mgr *uq_mgr)
 {
 	struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
 	bool invalidated = false, new_addition = false;
@@ -965,6 +1001,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
 	struct amdgpu_vm *vm = &fpriv->vm;
 	unsigned long key, tmp_key;
 	struct amdgpu_bo_va *bo_va;
+	struct amdgpu_usermode_queue *queue;
 	struct amdgpu_bo *bo;
 	struct drm_exec exec;
 	struct xarray xa;
@@ -1080,9 +1117,31 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
 		dma_fence_wait(bo_va->last_pt_update, false);
 	dma_fence_wait(vm->last_update, false);
 
+	xa_for_each(&uq_mgr->userq_xa, tmp_key, queue) {
+		bo = queue->wptr_obj.obj;
+		if (!bo) {
+			ret = -EINVAL;
+			goto unlock_all;
+		}
+
+		ret = amdgpu_ttm_alloc_gart(&bo->tbo);
+		if (unlikely(ret)) {
+			drm_file_err(uq_mgr->file,
+				     "failed to bind wptr bo to gart on resume, qid=%lu ret=%d\n",
+				     tmp_key, ret);
+			goto unlock_all;
+		}
+
+		queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(bo);
+	}
+
 	ret = amdgpu_evf_mgr_rearm(&fpriv->evf_mgr, &exec);
-	if (ret)
+	if (ret) {
 		drm_file_err(uq_mgr->file, "Failed to replace eviction fence\n");
+		goto unlock_all;
+	}
+
+	ret = amdgpu_userq_restore_all(uq_mgr);
 
 unlock_all:
 	drm_exec_fini(&exec);
@@ -1108,18 +1167,34 @@ static void amdgpu_userq_restore_worker(struct work_struct *work)
 	if (!dma_fence_is_signaled(ev_fence))
 		goto put_fence;
 
-	ret = amdgpu_userq_vm_validate(uq_mgr);
+	ret = amdgpu_userq_vm_validate_and_restore_queue(uq_mgr);
 	if (ret) {
 		drm_file_err(uq_mgr->file, "Failed to validate BOs to restore ret=%d\n", ret);
 		goto put_fence;
 	}
 
-	amdgpu_userq_restore_all(uq_mgr);
-
 put_fence:
 	dma_fence_put(ev_fence);
 }
 
+void amdgpu_userq_process_reset_irq(struct amdgpu_device *adev,
+				    u32 pasid, u32 doorbell_offset)
+{
+	struct xarray *xa = &adev->userq_doorbell_xa;
+	struct amdgpu_usermode_queue *queue;
+	unsigned long flags, idx;
+
+	xa_lock_irqsave(xa, flags);
+	xa_for_each(xa, idx, queue) {
+		if (queue->vm && queue->vm->pasid == pasid &&
+		    queue->doorbell_offset == doorbell_offset) {
+			amdgpu_userq_start_hang_detect_work(queue);
+			break;
+		}
+	}
+	xa_unlock_irqrestore(xa, flags);
+}
+
 static int
 amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
 {
@@ -1175,6 +1250,7 @@ int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *f
 	xa_init_flags(&userq_mgr->userq_xa, XA_FLAGS_ALLOC);
 	userq_mgr->adev = adev;
 	userq_mgr->file = file_priv;
+	mutex_init(&userq_mgr->proc_ctx_lock);
 
 	INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userq_restore_worker);
 	INIT_WORK(&userq_mgr->reset_work, amdgpu_userq_mgr_reset_work);
@@ -1228,6 +1304,11 @@ void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr)
 	 */
 	cancel_work_sync(&userq_mgr->reset_work);
 
+	amdgpu_bo_free_kernel(&userq_mgr->proc_ctx_obj.obj,
+			      &userq_mgr->proc_ctx_obj.gpu_addr,
+			      &userq_mgr->proc_ctx_obj.cpu_ptr);
+
+	mutex_destroy(&userq_mgr->proc_ctx_lock);
 	mutex_destroy(&userq_mgr->userq_mutex);
 }
 
@@ -1376,16 +1457,21 @@ void amdgpu_userq_pre_reset(struct amdgpu_device *adev)
 
 	/* TODO: We probably need a new lock for the queue state */
 	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
-		if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
-			continue;
-
-		userq_funcs = adev->userq_funcs[queue->queue_type];
-		userq_funcs->unmap(queue);
-		/* just mark all queues as hung at this point.
-		 * if unmap succeeds, we could map again
-		 * in amdgpu_userq_post_reset() if vram is not lost
+		if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
+			trace_amdgpu_userq_state_start(queue);
+			userq_funcs = adev->userq_funcs[queue->queue_type];
+			userq_funcs->unmap(queue);
+			/* just mark all queues as hung at this point.
+			 * if unmap succeeds, we could map again
+			 * in amdgpu_userq_post_reset() if vram is not lost
+			 */
+			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
+			queue->state = AMDGPU_USERQ_STATE_HUNG;
+		}
+		/* Force-complete any pending fence regardless of queue state so
+		 * that eviction/suspend and queue teardown waiters don't block
+		 * forever on a fence that will never signal after the reset.
 		 */
-		queue->state = AMDGPU_USERQ_STATE_HUNG;
 		amdgpu_userq_fence_driver_force_completion(queue);
 	}
 }
@@ -1403,6 +1489,8 @@ int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost)
 
 	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
 		if (queue->state == AMDGPU_USERQ_STATE_HUNG && !vram_lost) {
+			trace_amdgpu_userq_state_start(queue);
+
 			userq_funcs = adev->userq_funcs[queue->queue_type];
 			/* Re-map queue */
 			r = userq_funcs->map(queue);
@@ -1410,6 +1498,7 @@ int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost)
 				dev_err(adev->dev, "Failed to remap queue %ld\n", queue_id);
 				continue;
 			}
+			trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED);
 			queue->state = AMDGPU_USERQ_STATE_MAPPED;
 		}
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index d1751febaefe4..8333bab537d4a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -53,6 +53,7 @@ struct amdgpu_usermode_queue {
 	enum amdgpu_userq_state state;
 	uint64_t		doorbell_handle;
 	uint64_t		doorbell_index;
+	u32			doorbell_offset;
 	uint64_t		flags;
 	struct amdgpu_mqd_prop	*userq_prop;
 	struct amdgpu_userq_mgr *userq_mgr;
@@ -99,6 +100,9 @@ struct amdgpu_usermode_queue {
 		} va;
 		u64 va_array[6];
 	} userq_vas;
+
+	uint32_t	proc_ctx_array_index;
+	uint32_t	gang_ctx_array_index;
 };
 
 struct amdgpu_userq_funcs {
@@ -111,8 +115,7 @@ struct amdgpu_userq_funcs {
 	int (*map)(struct amdgpu_usermode_queue *queue);
 	int (*preempt)(struct amdgpu_usermode_queue *queue);
 	int (*restore)(struct amdgpu_usermode_queue *queue);
-	int (*detect_and_reset)(struct amdgpu_device *adev,
-		  int queue_type);
+	int (*reset)(struct amdgpu_usermode_queue *queue);
 };
 
 /* Usermode queues for gfx */
@@ -127,6 +130,8 @@ struct amdgpu_userq_mgr {
 	struct amdgpu_device		*adev;
 	struct delayed_work		resume_work;
 	struct drm_file			*file;
+	struct mutex			proc_ctx_lock;
+	struct amdgpu_userq_obj		proc_ctx_obj;
 
 	/**
 	 * @reset_work:
@@ -177,6 +182,16 @@ int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost);
 void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue);
 void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell);
 
+/*
+ * CP packs the per-process doorbell_id of the queue in
+ * CTXID0[9:0] on priv-fault (same encoding KFD uses via
+ * KFD_CTXID0_DOORBELL_ID_MASK)
+ */
+#define AMDGPU_CTXID0_DOORBELL_ID_MASK	0x3ff
+
+void amdgpu_userq_process_reset_irq(struct amdgpu_device *adev,
+				    u32 pasid, u32 doorbell_offset);
+
 int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
 				   struct amdgpu_usermode_queue *queue,
 				   u64 addr, u64 expected_size, u64 *va_out);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
index f74ad378e4074..7e80442ec3e5f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
@@ -30,7 +30,7 @@
 #include <drm/drm_syncobj.h>
 
 #include "amdgpu.h"
-#include "amdgpu_userq_fence.h"
+#include "amdgpu_trace.h"
 
 #define AMDGPU_USERQ_MAX_HANDLES	(1U << 16)
 
@@ -528,6 +528,8 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 	/* Create the new fence */
 	amdgpu_userq_fence_init(queue, fence, wptr);
 
+	trace_amdgpu_userq_emit_fence(dev->dev, queue, fence);
+
 	mutex_unlock(&userq_mgr->userq_mutex);
 
 	/*
@@ -701,7 +703,7 @@ amdgpu_userq_wait_add_fence(struct drm_amdgpu_userq_wait *wait_info,
 }
 
 static int
-amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
+amdgpu_userq_wait_return_fence_info(struct drm_device *dev, struct drm_file *filp,
 				    struct drm_amdgpu_userq_wait *wait_info,
 				    u32 *syncobj_handles, u64 *timeline_points,
 				    u32 *timeline_handles,
@@ -869,6 +871,8 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp,
 
 		amdgpu_userq_fence_driver_get(fence_drv);
 
+		trace_amdgpu_userq_wait_deps(dev->dev, waitq, userq_fence);
+
 		/* Store drm syncobj's gpu va address and value */
 		fence_info[cnt].va = fence_drv->va;
 		fence_info[cnt].value = fences[i]->seqno;
@@ -969,7 +973,7 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
 						   gobj_write,
 						   gobj_read);
 	} else {
-		r = amdgpu_userq_wait_return_fence_info(filp, wait_info,
+		r = amdgpu_userq_wait_return_fence_info(dev, filp, wait_info,
 							syncobj_handles,
 							timeline_points,
 							timeline_handles,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index 23383ac5323f0..e8b0c62f72be2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -1165,8 +1165,9 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 	r = amdgpu_job_alloc_with_ib(ring->adev, &adev->uvd.entity,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     64, direct ? AMDGPU_IB_POOL_DIRECT :
-				     AMDGPU_IB_POOL_DELAYED, &job,
-				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_IB_POOL_DELAYED,
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index eef3c9853a5c5..1f7a98cdce5a2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -473,7 +473,8 @@ static int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
 	r = amdgpu_job_alloc_with_ib(ring->adev, &ring->adev->vce.entity,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     ib_size_dw * 4, AMDGPU_IB_POOL_DIRECT,
-				     &job, AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
@@ -564,8 +565,9 @@ static int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     ib_size_dw * 4,
 				     direct ? AMDGPU_IB_POOL_DIRECT :
-				     AMDGPU_IB_POOL_DELAYED, &job,
-				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_IB_POOL_DELAYED,
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
index 616967519869e..17db7264269ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
@@ -506,9 +506,8 @@ void amdgpu_vcn_ring_begin_use(struct amdgpu_ring *ring)
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_vcn_inst *vcn_inst = &adev->vcn.inst[ring->me];
 
-	atomic_inc(&vcn_inst->total_submission_cnt);
-
-	cancel_delayed_work_sync(&vcn_inst->idle_work);
+	if (!atomic_fetch_inc(&vcn_inst->total_submission_cnt))
+		cancel_delayed_work_sync(&vcn_inst->idle_work);
 
 	mutex_lock(&vcn_inst->vcn_pg_lock);
 	vcn_inst->set_pg_state(vcn_inst, AMD_PG_STATE_UNGATE);
@@ -550,10 +549,9 @@ void amdgpu_vcn_ring_end_use(struct amdgpu_ring *ring)
 	    !adev->vcn.inst[ring->me].using_unified_queue)
 		atomic_dec(&ring->adev->vcn.inst[ring->me].dpg_enc_submission_cnt);
 
-	atomic_dec(&ring->adev->vcn.inst[ring->me].total_submission_cnt);
-
-	schedule_delayed_work(&ring->adev->vcn.inst[ring->me].idle_work,
-			      VCN_IDLE_TIMEOUT);
+	if (atomic_dec_and_test(&ring->adev->vcn.inst[ring->me].total_submission_cnt))
+		schedule_delayed_work(&ring->adev->vcn.inst[ring->me].idle_work,
+				      VCN_IDLE_TIMEOUT);
 }
 
 int amdgpu_vcn_dec_ring_test_ring(struct amdgpu_ring *ring)
@@ -631,7 +629,8 @@ static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring,
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL,
 				     64, AMDGPU_IB_POOL_DIRECT,
-				     &job, AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		goto err;
 
@@ -811,7 +810,8 @@ static int amdgpu_vcn_dec_sw_send_msg(struct amdgpu_ring *ring,
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL,
 				     ib_size_dw * 4, AMDGPU_IB_POOL_DIRECT,
-				     &job, AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		goto err;
 
@@ -941,7 +941,8 @@ static int amdgpu_vcn_enc_get_create_msg(struct amdgpu_ring *ring, uint32_t hand
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL,
 				     ib_size_dw * 4, AMDGPU_IB_POOL_DIRECT,
-				     &job, AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
@@ -1008,7 +1009,8 @@ static int amdgpu_vcn_enc_get_destroy_msg(struct amdgpu_ring *ring, uint32_t han
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL,
 				     ib_size_dw * 4, AMDGPU_IB_POOL_DIRECT,
-				     &job, AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
@@ -1485,6 +1487,37 @@ int vcn_set_powergating_state(struct amdgpu_ip_block *ip_block,
 	return ret;
 }
 
+static struct amdgpu_fence *
+amdgpu_vcn_ring_reset_begin_helper(struct amdgpu_ring *ring,
+				   struct amdgpu_ring *guilty_ring,
+				   struct amdgpu_fence *timedout_fence)
+{
+	struct amdgpu_fence *fence;
+
+	drm_sched_wqueue_stop(&ring->sched);
+	if (ring == guilty_ring)
+		fence = timedout_fence;
+	else
+		fence = amdgpu_ring_find_guilty_fence(ring);
+	amdgpu_ring_reset_helper_begin(ring, fence);
+
+	return fence;
+}
+
+static int
+amdgpu_vcn_ring_reset_end_helper(struct amdgpu_ring *ring,
+				 struct amdgpu_fence *fence)
+{
+	int r;
+
+	r = amdgpu_ring_reset_helper_end(ring, fence);
+	if (r)
+		return r;
+
+	drm_sched_wqueue_start(&ring->sched);
+	return 0;
+}
+
 /**
  * amdgpu_vcn_ring_reset - Reset a VCN ring
  * @ring: ring to reset
@@ -1502,48 +1535,33 @@ int amdgpu_vcn_ring_reset(struct amdgpu_ring *ring,
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[ring->me];
+	struct amdgpu_fence *dec_fence;
+	struct amdgpu_fence *enc_fence[AMDGPU_VCN_MAX_ENC_RINGS];
 	int r, i;
 
 	if (adev->vcn.inst[ring->me].using_unified_queue)
 		return -EINVAL;
 
 	mutex_lock(&vinst->engine_reset_mutex);
-	/* Stop the scheduler's work queue for the dec and enc rings if they are running.
-	 * This ensures that no new tasks are submitted to the queues while
-	 * the reset is in progress.
-	 */
-	drm_sched_wqueue_stop(&vinst->ring_dec.sched);
+	dec_fence = amdgpu_vcn_ring_reset_begin_helper(&vinst->ring_dec, ring,
+						       timedout_fence);
 	for (i = 0; i < vinst->num_enc_rings; i++)
-		drm_sched_wqueue_stop(&vinst->ring_enc[i].sched);
+		enc_fence[i] = amdgpu_vcn_ring_reset_begin_helper(&vinst->ring_enc[i], ring,
+								  timedout_fence);
 
 	/* Perform the VCN reset for the specified instance */
 	r = vinst->reset(vinst);
 	if (r)
 		goto unlock;
-	r = amdgpu_ring_test_ring(&vinst->ring_dec);
+
+	r = amdgpu_vcn_ring_reset_end_helper(&vinst->ring_dec, dec_fence);
 	if (r)
 		goto unlock;
 	for (i = 0; i < vinst->num_enc_rings; i++) {
-		r = amdgpu_ring_test_ring(&vinst->ring_enc[i]);
+		r = amdgpu_vcn_ring_reset_end_helper(&vinst->ring_enc[i], enc_fence[i]);
 		if (r)
 			goto unlock;
 	}
-	amdgpu_fence_driver_force_completion(&vinst->ring_dec,
-					     (&vinst->ring_dec == ring) ?
-					     &timedout_fence->base : NULL);
-	for (i = 0; i < vinst->num_enc_rings; i++)
-		amdgpu_fence_driver_force_completion(&vinst->ring_enc[i],
-						     (&vinst->ring_enc[i] == ring) ?
-						     &timedout_fence->base : NULL);
-
-	/* Restart the scheduler's work queue for the dec and enc rings
-	 * if they were stopped by this function. This allows new tasks
-	 * to be submitted to the queues after the reset is complete.
-	 */
-	drm_sched_wqueue_start(&vinst->ring_dec.sched);
-	for (i = 0; i < vinst->num_enc_rings; i++)
-		drm_sched_wqueue_start(&vinst->ring_enc[i].sched);
-
 unlock:
 	mutex_unlock(&vinst->engine_reset_mutex);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
index 82624b44e661a..bea95307fd42b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
@@ -368,6 +368,9 @@ struct amdgpu_vcn {
 	struct mutex            workload_profile_mutex;
 	u32 reg_count;
 	const struct amdgpu_hwip_reg_entry *reg_list;
+
+	bool			disable_uq;
+	bool			disable_kq;
 };
 
 struct amdgpu_fw_shared_rb_ptrs_struct {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_video_codecs.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_video_codecs.h
new file mode 100644
index 0000000000000..3b2a6cb8632da
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_video_codecs.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef __AMDGPU_VIDEO_CODECS_H__
+#define __AMDGPU_VIDEO_CODECS_H__
+
+#include <linux/types.h>
+
+#define codec_info_build(type, width, height, level) \
+			 .codec_type = type,\
+			 .max_width = width,\
+			 .max_height = height,\
+			 .max_pixels_per_frame = height * width,\
+			 .max_level = level,
+
+struct amdgpu_video_codec_info {
+	u32 codec_type;
+	u32 max_width;
+	u32 max_height;
+	u32 max_pixels_per_frame;
+	u32 max_level;
+};
+
+struct amdgpu_video_codecs {
+	const u32 codec_count;
+	const struct amdgpu_video_codec_info *codec_array;
+};
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 9e8f7d2b898cf..8f4b860635078 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -34,6 +34,7 @@
 #include "amdgpu_ras.h"
 #include "amdgpu_reset.h"
 #include "amdgpu_dpm.h"
+#include "amdgpu_video_codecs.h"
 #include "vi.h"
 #include "soc15.h"
 #include "nv.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
index 170adaf7e76ab..c835504fdf2b0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
@@ -2,7 +2,7 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_edid.h>
-#include <drm/drm_simple_kms_helper.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_vblank.h>
 #include <drm/drm_vblank_helper.h>
@@ -316,6 +316,10 @@ static struct drm_plane *amdgpu_vkms_plane_init(struct drm_device *dev,
 	return plane;
 }
 
+static const struct drm_encoder_funcs drm_encoder_funcs_cleanup = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static int amdgpu_vkms_output_init(struct drm_device *dev, struct
 				   amdgpu_vkms_output *output, int index)
 {
@@ -342,7 +346,9 @@ static int amdgpu_vkms_output_init(struct drm_device *dev, struct
 
 	drm_connector_helper_add(connector, &amdgpu_vkms_conn_helper_funcs);
 
-	ret = drm_simple_encoder_init(dev, encoder, DRM_MODE_ENCODER_VIRTUAL);
+	ret = drm_encoder_init(dev, encoder,
+			       &drm_encoder_funcs_cleanup,
+			       DRM_MODE_ENCODER_VIRTUAL, NULL);
 	if (ret) {
 		DRM_ERROR("Failed to init encoder\n");
 		goto err_encoder;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index bb99b7c3a010e..f224d33c4bc49 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -855,12 +855,10 @@ void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job,
 					    job->oa_size);
 	}
 
-	if (vm_flush_needed || pasid_mapping_needed || cleaner_shader_needed) {
-		amdgpu_fence_emit(ring, job->hw_vm_fence, 0);
-		fence = &job->hw_vm_fence->base;
-		/* get a ref for the job */
-		dma_fence_get(fence);
-	}
+	amdgpu_fence_emit(ring, job->hw_vm_fence, 0);
+	fence = &job->hw_vm_fence->base;
+	/* get a ref for the job */
+	dma_fence_get(fence);
 
 	if (vm_flush_needed) {
 		mutex_lock(&id_mgr->lock);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
index fd09a2b5a147f..4302742376897 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
@@ -56,7 +56,7 @@ static int amdgpu_vm_sdma_alloc_job(struct amdgpu_vm_update_params *p,
 	ndw = min(ndw, AMDGPU_VM_SDMA_MAX_NUM_DW);
 
 	r = amdgpu_job_alloc_with_ib(p->adev, entity, AMDGPU_FENCE_OWNER_VM,
-				     ndw * 4, pool, &p->job, k_job_id);
+				     ndw * 4, pool, k_job_id, &p->job);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c
index 7bf74ff93fbdb..2b45010b7ebed 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c
@@ -789,7 +789,7 @@ static int vpe_ring_test_ring(struct amdgpu_ring *ring)
 	uint64_t wb_addr;
 	int ret;
 
-	ret = amdgpu_device_wb_get(adev, &index);
+	ret = amdgpu_wb_get(adev, &index);
 	if (ret) {
 		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", ret);
 		return ret;
@@ -818,7 +818,7 @@ static int vpe_ring_test_ring(struct amdgpu_ring *ring)
 
 	ret = -ETIMEDOUT;
 out:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 
 	return ret;
 }
@@ -833,7 +833,7 @@ static int vpe_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	uint64_t wb_addr;
 	int ret;
 
-	ret = amdgpu_device_wb_get(adev, &index);
+	ret = amdgpu_wb_get(adev, &index);
 	if (ret) {
 		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", ret);
 		return ret;
@@ -872,7 +872,7 @@ static int vpe_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_wb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_wb.c
new file mode 100644
index 0000000000000..8b7061c94ccd6
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_wb.c
@@ -0,0 +1,129 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include <linux/spinlock.h>
+
+#include "amdgpu.h"
+#include "amdgpu_wb.h"
+
+/*
+ * amdgpu_wb_*()
+ * Writeback is the method by which the GPU updates special pages in memory
+ * with the status of certain GPU events (fences, ring pointers,etc.).
+ */
+
+/**
+ * amdgpu_wb_fini - Disable Writeback and free memory
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Disables Writeback and frees the Writeback memory (all asics).
+ * Used at driver shutdown.
+ */
+void amdgpu_wb_fini(struct amdgpu_device *adev)
+{
+	if (adev->wb.wb_obj) {
+		amdgpu_bo_free_kernel(&adev->wb.wb_obj,
+				      &adev->wb.gpu_addr,
+				      (void **)&adev->wb.wb);
+		adev->wb.wb_obj = NULL;
+	}
+}
+
+/**
+ * amdgpu_wb_init - Init Writeback driver info and allocate memory
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Initializes writeback and allocates writeback memory (all asics).
+ * Used at driver startup.
+ * Returns 0 on success or an -error on failure.
+ */
+int amdgpu_wb_init(struct amdgpu_device *adev)
+{
+	int r;
+
+	if (adev->wb.wb_obj == NULL) {
+		/* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
+		r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
+					    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
+					    &adev->wb.wb_obj, &adev->wb.gpu_addr,
+					    (void **)&adev->wb.wb);
+		if (r) {
+			dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
+			return r;
+		}
+
+		adev->wb.num_wb = AMDGPU_MAX_WB;
+		memset(&adev->wb.used, 0, sizeof(adev->wb.used));
+
+		/* clear wb memory */
+		memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
+	}
+
+	return 0;
+}
+
+/**
+ * amdgpu_wb_get - Allocate a wb entry
+ *
+ * @adev: amdgpu_device pointer
+ * @wb: wb index
+ *
+ * Allocate a wb slot for use by the driver (all asics).
+ * Returns 0 on success or -EINVAL on failure.
+ */
+int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
+{
+	unsigned long flags, offset;
+
+	spin_lock_irqsave(&adev->wb.lock, flags);
+	offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
+	if (offset < adev->wb.num_wb) {
+		__set_bit(offset, adev->wb.used);
+		spin_unlock_irqrestore(&adev->wb.lock, flags);
+		*wb = offset << 3; /* convert to dw offset */
+		return 0;
+	} else {
+		spin_unlock_irqrestore(&adev->wb.lock, flags);
+		return -EINVAL;
+	}
+}
+
+/**
+ * amdgpu_wb_free - Free a wb entry
+ *
+ * @adev: amdgpu_device pointer
+ * @wb: wb index
+ *
+ * Free a wb slot allocated for use by the driver (all asics)
+ */
+void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb)
+{
+	unsigned long flags;
+
+	wb >>= 3;
+	spin_lock_irqsave(&adev->wb.lock, flags);
+	if (wb < adev->wb.num_wb)
+		__clear_bit(wb, adev->wb.used);
+	spin_unlock_irqrestore(&adev->wb.lock, flags);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_wb.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_wb.h
new file mode 100644
index 0000000000000..7b8efa6a639cc
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_wb.h
@@ -0,0 +1,102 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef __AMDGPU_WB_H__
+#define __AMDGPU_WB_H__
+
+#include <linux/types.h>
+#include <linux/spinlock_types.h>
+#include <linux/math.h>
+
+/*
+ * Writeback
+ */
+#define AMDGPU_MAX_WB 1024	/* Reserve at most 1024 WB slots for amdgpu-owned rings. */
+
+/**
+ * struct amdgpu_wb - This struct is used for small GPU memory allocation.
+ *
+ * This struct is used to allocate a small amount of GPU memory that can be
+ * used to shadow certain states into the memory. This is especially useful for
+ * providing easy CPU access to some states without requiring register access
+ * (e.g., if some block is power gated, reading register may be problematic).
+ *
+ * Note: the term writeback was initially used because many of the amdgpu
+ * components had some level of writeback memory, and this struct initially
+ * described those components.
+ */
+
+struct amdgpu_bo;
+struct amdgpu_device;
+
+struct amdgpu_wb {
+
+	/**
+	 * @wb_obj:
+	 *
+	 * Buffer Object used for the writeback memory.
+	 */
+	struct amdgpu_bo	*wb_obj;
+
+	/**
+	 * @wb:
+	 *
+	 * Pointer to the first writeback slot. In terms of CPU address
+	 * this value can be accessed directly by using the offset as an index.
+	 * For the GPU address, it is necessary to use gpu_addr and the offset.
+	 */
+	uint32_t		*wb;
+
+	/**
+	 * @gpu_addr:
+	 *
+	 * Writeback base address in the GPU.
+	 */
+	uint64_t		gpu_addr;
+
+	/**
+	 * @num_wb:
+	 *
+	 * Number of writeback slots reserved for amdgpu.
+	 */
+	u32			num_wb;
+
+	/**
+	 * @used:
+	 *
+	 * Track the writeback slot already used.
+	 */
+	unsigned long		used[DIV_ROUND_UP(AMDGPU_MAX_WB, BITS_PER_LONG)];
+
+	/**
+	 * @lock:
+	 *
+	 * Protects read and write of the used field array.
+	 */
+	spinlock_t		lock;
+};
+
+void amdgpu_wb_fini(struct amdgpu_device *adev);
+int amdgpu_wb_init(struct amdgpu_device *adev);
+int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb);
+void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb);
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
index 409e103ffe8c7..35faea0ff17ff 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
@@ -381,7 +381,8 @@ int amdgpu_xcp_get_inst_details(struct amdgpu_xcp *xcp,
 				enum AMDGPU_XCP_IP_BLOCK ip,
 				uint32_t *inst_mask)
 {
-	if (!xcp->valid || !inst_mask || !(xcp->ip[ip].valid))
+	if (!xcp->valid || !inst_mask || ip >= AMDGPU_XCP_MAX_BLOCKS ||
+	    !(xcp->ip[ip].valid))
 		return -EINVAL;
 
 	*inst_mask = xcp->ip[ip].inst_mask;
@@ -468,14 +469,18 @@ void amdgpu_xcp_release_sched(struct amdgpu_device *adev,
 {
 	struct drm_gpu_scheduler *sched =
 		container_of(entity->entity.rq, typeof(*sched), rq);
+	struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
 
-	if (!adev->xcp_mgr)
+	if (!xcp_mgr)
 		return;
 
 	if (drm_sched_wqueue_ready(sched)) {
 		struct amdgpu_ring *ring = to_amdgpu_ring(sched);
 
-		atomic_dec(&adev->xcp_mgr->xcp[ring->xcp_id].ref_cnt);
+		mutex_lock(&xcp_mgr->xcp_lock);
+		if (ring->xcp_id < xcp_mgr->num_xcps && xcp_mgr->xcp[ring->xcp_id].valid)
+			atomic_dec(&xcp_mgr->xcp[ring->xcp_id].ref_cnt);
+		mutex_unlock(&xcp_mgr->xcp_lock);
 	}
 }
 
@@ -488,7 +493,9 @@ int amdgpu_xcp_select_scheds(struct amdgpu_device *adev,
 	u32 sel_xcp_id;
 	int i;
 	struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
+	int r = 0;
 
+	mutex_lock(&xcp_mgr->xcp_lock);
 	if (fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION) {
 		u32 least_ref_cnt = ~0;
 
@@ -505,19 +512,27 @@ int amdgpu_xcp_select_scheds(struct amdgpu_device *adev,
 	}
 	sel_xcp_id = fpriv->xcp_id;
 
+	if (sel_xcp_id >= xcp_mgr->num_xcps || !xcp_mgr->xcp[sel_xcp_id].valid) {
+		dev_err(adev->dev, "Selected partition #%d is not valid.", sel_xcp_id);
+		r = -ENODEV;
+		goto out;
+	}
+
 	if (xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds) {
 		*num_scheds =
-			xcp_mgr->xcp[fpriv->xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds;
+			xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds;
 		*scheds =
-			xcp_mgr->xcp[fpriv->xcp_id].gpu_sched[hw_ip][hw_prio].sched;
-		atomic_inc(&adev->xcp_mgr->xcp[sel_xcp_id].ref_cnt);
+			xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].sched;
+		atomic_inc(&xcp_mgr->xcp[sel_xcp_id].ref_cnt);
 		dev_dbg(adev->dev, "Selected partition #%d", sel_xcp_id);
 	} else {
 		dev_err(adev->dev, "Failed to schedule partition #%d.", sel_xcp_id);
-		return -ENOENT;
+		r = -ENOENT;
 	}
 
-	return 0;
+out:
+	mutex_unlock(&xcp_mgr->xcp_lock);
+	return r;
 }
 
 static void amdgpu_set_xcp_id(struct amdgpu_device *adev,
@@ -574,6 +589,9 @@ static void amdgpu_xcp_gpu_sched_update(struct amdgpu_device *adev,
 {
 	unsigned int *num_gpu_sched;
 
+	if (sel_xcp_id >= MAX_XCP || sel_xcp_id == AMDGPU_XCP_NO_PARTITION)
+		return;
+
 	num_gpu_sched = &adev->xcp_mgr->xcp[sel_xcp_id]
 			.gpu_sched[ring->funcs->type][ring->hw_prio].num_scheds;
 	adev->xcp_mgr->xcp[sel_xcp_id].gpu_sched[ring->funcs->type][ring->hw_prio]
@@ -903,7 +921,7 @@ static void amdgpu_xcp_cfg_sysfs_init(struct amdgpu_device *adev)
 {
 	struct amdgpu_xcp_res_details *xcp_res;
 	struct amdgpu_xcp_cfg *xcp_cfg;
-	int i, r, j, rid, mode;
+	int i, r, rid, mode;
 
 	if (!adev->xcp_mgr)
 		return;
@@ -949,14 +967,16 @@ static void amdgpu_xcp_cfg_sysfs_init(struct amdgpu_device *adev)
 					 &xcp_cfg_res_sysfs_ktype,
 					 &xcp_cfg->kobj, "%s",
 					 xcp_res_names[rid]);
-		if (r)
+		if (r) {
+			kobject_put(&xcp_res->kobj);
 			goto err;
+		}
 	}
 
 	adev->xcp_mgr->xcp_cfg = xcp_cfg;
 	return;
 err:
-	for (j = 0; j < i; j++) {
+	while (i--) {
 		xcp_res = &xcp_cfg->xcp_res[i];
 		kobject_put(&xcp_res->kobj);
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index e63d05c477a08..2725230aa5e39 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
@@ -106,53 +106,6 @@ static const int walf_pcs_err_noncorrectable_mask_reg_aldebaran[] = {
 	smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000
 };
 
-static const int xgmi3x16_pcs_err_status_reg_v6_4[] = {
-	smnPCS_XGMI3X16_PCS_ERROR_STATUS,
-	smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x100000
-};
-
-static const int xgmi3x16_pcs_err_noncorrectable_mask_reg_v6_4[] = {
-	smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK,
-	smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000
-};
-
-static const u64 xgmi_v6_4_0_mca_base_array[] = {
-	0x11a09200,
-	0x11b09200,
-};
-
-static const char *xgmi_v6_4_0_ras_error_code_ext[32] = {
-	[0x00] = "XGMI PCS DataLossErr",
-	[0x01] = "XGMI PCS TrainingErr",
-	[0x02] = "XGMI PCS FlowCtrlAckErr",
-	[0x03] = "XGMI PCS RxFifoUnderflowErr",
-	[0x04] = "XGMI PCS RxFifoOverflowErr",
-	[0x05] = "XGMI PCS CRCErr",
-	[0x06] = "XGMI PCS BERExceededErr",
-	[0x07] = "XGMI PCS TxMetaDataErr",
-	[0x08] = "XGMI PCS ReplayBufParityErr",
-	[0x09] = "XGMI PCS DataParityErr",
-	[0x0a] = "XGMI PCS ReplayFifoOverflowErr",
-	[0x0b] = "XGMI PCS ReplayFifoUnderflowErr",
-	[0x0c] = "XGMI PCS ElasticFifoOverflowErr",
-	[0x0d] = "XGMI PCS DeskewErr",
-	[0x0e] = "XGMI PCS FlowCtrlCRCErr",
-	[0x0f] = "XGMI PCS DataStartupLimitErr",
-	[0x10] = "XGMI PCS FCInitTimeoutErr",
-	[0x11] = "XGMI PCS RecoveryTimeoutErr",
-	[0x12] = "XGMI PCS ReadySerialTimeoutErr",
-	[0x13] = "XGMI PCS ReadySerialAttemptErr",
-	[0x14] = "XGMI PCS RecoveryAttemptErr",
-	[0x15] = "XGMI PCS RecoveryRelockAttemptErr",
-	[0x16] = "XGMI PCS ReplayAttemptErr",
-	[0x17] = "XGMI PCS SyncHdrErr",
-	[0x18] = "XGMI PCS TxReplayTimeoutErr",
-	[0x19] = "XGMI PCS RxReplayTimeoutErr",
-	[0x1a] = "XGMI PCS LinkSubTxTimeoutErr",
-	[0x1b] = "XGMI PCS LinkSubRxTimeoutErr",
-	[0x1c] = "XGMI PCS RxCMDPktErr",
-};
-
 static const struct amdgpu_pcs_ras_field xgmi_pcs_ras_fields[] = {
 	{"XGMI PCS DataLossErr",
 	 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataLossErr)},
@@ -606,7 +559,7 @@ static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev,
 		pr_err("failed to create xgmi_num_links\n");
 
 	/* Create xgmi port num file if supported */
-	if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG) {
+	if (adev->psp.xgmi_context.supports_ext_link_info) {
 		ret = device_create_file(adev->dev, &dev_attr_xgmi_port_num);
 		if (ret)
 			dev_err(adev->dev, "failed to create xgmi_port_num\n");
@@ -642,7 +595,7 @@ static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev,
 	device_remove_file(adev->dev, &dev_attr_xgmi_error);
 	device_remove_file(adev->dev, &dev_attr_xgmi_num_hops);
 	device_remove_file(adev->dev, &dev_attr_xgmi_num_links);
-	if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG)
+	if (adev->psp.xgmi_context.supports_ext_link_info)
 		device_remove_file(adev->dev, &dev_attr_xgmi_port_num);
 
 success:
@@ -660,7 +613,7 @@ static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev,
 	device_remove_file(adev->dev, &dev_attr_xgmi_error);
 	device_remove_file(adev->dev, &dev_attr_xgmi_num_hops);
 	device_remove_file(adev->dev, &dev_attr_xgmi_num_links);
-	if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG)
+	if (adev->psp.xgmi_context.supports_ext_link_info)
 		device_remove_file(adev->dev, &dev_attr_xgmi_port_num);
 
 	if (hive->kobj.parent != (&adev->dev->kobj))
@@ -1152,91 +1105,15 @@ int amdgpu_xgmi_remove_device(struct amdgpu_device *adev)
 	return 0;
 }
 
-static int xgmi_v6_4_0_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
-				       enum aca_smu_type type, void *data)
-{
-	struct amdgpu_device *adev = handle->adev;
-	struct aca_bank_info info;
-	const char *error_str;
-	u64 status, count;
-	int ret, ext_error_code;
-
-	ret = aca_bank_info_decode(bank, &info);
-	if (ret)
-		return ret;
-
-	status = bank->regs[ACA_REG_IDX_STATUS];
-	ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status);
-
-	error_str = ext_error_code < ARRAY_SIZE(xgmi_v6_4_0_ras_error_code_ext) ?
-		xgmi_v6_4_0_ras_error_code_ext[ext_error_code] : NULL;
-	if (error_str)
-		dev_info(adev->dev, "%s detected\n", error_str);
-
-	count = ACA_REG__MISC0__ERRCNT(bank->regs[ACA_REG_IDX_MISC0]);
-
-	switch (type) {
-	case ACA_SMU_TYPE_UE:
-		if (ext_error_code != 0 && ext_error_code != 1 && ext_error_code != 9)
-			count = 0ULL;
-
-		bank->aca_err_type = ACA_ERROR_TYPE_UE;
-		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, count);
-		break;
-	case ACA_SMU_TYPE_CE:
-		count = ext_error_code == 6 ? count : 0ULL;
-		bank->aca_err_type = ACA_ERROR_TYPE_CE;
-		ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, count);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-static const struct aca_bank_ops xgmi_v6_4_0_aca_bank_ops = {
-	.aca_bank_parser = xgmi_v6_4_0_aca_bank_parser,
-};
-
-static const struct aca_info xgmi_v6_4_0_aca_info = {
-	.hwip = ACA_HWIP_TYPE_PCS_XGMI,
-	.mask = ACA_ERROR_UE_MASK | ACA_ERROR_CE_MASK,
-	.bank_ops = &xgmi_v6_4_0_aca_bank_ops,
-};
-
 static int amdgpu_xgmi_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
 {
-	int r;
-
 	if (!adev->gmc.xgmi.supported ||
 	    adev->gmc.xgmi.num_physical_nodes == 0)
 		return 0;
 
 	amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL);
 
-	r = amdgpu_ras_block_late_init(adev, ras_block);
-	if (r)
-		return r;
-
-	switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
-	case IP_VERSION(6, 4, 0):
-	case IP_VERSION(6, 4, 1):
-		r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL,
-					&xgmi_v6_4_0_aca_info, NULL);
-		if (r)
-			goto late_fini;
-		break;
-	default:
-		break;
-	}
-
-	return 0;
-
-late_fini:
-	amdgpu_ras_block_late_fini(adev, ras_block);
-
-	return r;
+	return amdgpu_ras_block_late_init(adev, ras_block);
 }
 
 uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device *adev,
@@ -1252,7 +1129,7 @@ static void pcs_clear_status(struct amdgpu_device *adev, uint32_t pcs_status_reg
 	WREG32_PCIE(pcs_status_reg, 0);
 }
 
-static void amdgpu_xgmi_legacy_reset_ras_error_count(struct amdgpu_device *adev)
+static void amdgpu_xgmi_reset_ras_error_count(struct amdgpu_device *adev)
 {
 	uint32_t i;
 
@@ -1278,54 +1155,6 @@ static void amdgpu_xgmi_legacy_reset_ras_error_count(struct amdgpu_device *adev)
 	default:
 		break;
 	}
-
-	switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
-	case IP_VERSION(6, 4, 0):
-	case IP_VERSION(6, 4, 1):
-		for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_v6_4); i++)
-			pcs_clear_status(adev,
-					xgmi3x16_pcs_err_status_reg_v6_4[i]);
-		break;
-	default:
-		break;
-	}
-}
-
-static void __xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst, u64 mca_base)
-{
-	uint64_t smn_base =
-		amdgpu_reg_get_smn_base64(adev, XGMI_HWIP, xgmi_inst);
-
-	WREG64_MCA(smn_base, mca_base, ACA_REG_IDX_STATUS, 0ULL);
-}
-
-static void xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(xgmi_v6_4_0_mca_base_array); i++)
-		__xgmi_v6_4_0_reset_error_count(adev, xgmi_inst, xgmi_v6_4_0_mca_base_array[i]);
-}
-
-static void xgmi_v6_4_0_reset_ras_error_count(struct amdgpu_device *adev)
-{
-	int i;
-
-	for_each_inst(i, adev->aid_mask)
-		xgmi_v6_4_0_reset_error_count(adev, i);
-}
-
-static void amdgpu_xgmi_reset_ras_error_count(struct amdgpu_device *adev)
-{
-	switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
-	case IP_VERSION(6, 4, 0):
-	case IP_VERSION(6, 4, 1):
-		xgmi_v6_4_0_reset_ras_error_count(adev);
-		break;
-	default:
-		amdgpu_xgmi_legacy_reset_ras_error_count(adev);
-		break;
-	}
 }
 
 static int amdgpu_xgmi_query_pcs_error_status(struct amdgpu_device *adev,
@@ -1343,11 +1172,7 @@ static int amdgpu_xgmi_query_pcs_error_status(struct amdgpu_device *adev,
 
 	if (is_xgmi_pcs) {
 		if (amdgpu_ip_version(adev, XGMI_HWIP, 0) ==
-		    IP_VERSION(6, 1, 0) ||
-		    amdgpu_ip_version(adev, XGMI_HWIP, 0) ==
-		    IP_VERSION(6, 4, 0) ||
-		    amdgpu_ip_version(adev, XGMI_HWIP, 0) ==
-		    IP_VERSION(6, 4, 1)) {
+		    IP_VERSION(6, 1, 0)) {
 			pcs_ras_fields = &xgmi3x16_pcs_ras_fields[0];
 			field_array_size = ARRAY_SIZE(xgmi3x16_pcs_ras_fields);
 		} else {
@@ -1381,11 +1206,11 @@ static int amdgpu_xgmi_query_pcs_error_status(struct amdgpu_device *adev,
 	return 0;
 }
 
-static void amdgpu_xgmi_legacy_query_ras_error_count(struct amdgpu_device *adev,
+static void amdgpu_xgmi_query_ras_error_count(struct amdgpu_device *adev,
 						     void *ras_error_status)
 {
 	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
-	int i, supported = 1;
+	int i;
 	uint32_t data, mask_data = 0;
 	uint32_t ue_cnt = 0, ce_cnt = 0;
 
@@ -1449,26 +1274,6 @@ static void amdgpu_xgmi_legacy_query_ras_error_count(struct amdgpu_device *adev,
 		}
 		break;
 	default:
-		supported = 0;
-		break;
-	}
-
-	switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
-	case IP_VERSION(6, 4, 0):
-	case IP_VERSION(6, 4, 1):
-		/* check xgmi3x16 pcs error */
-		for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_v6_4); i++) {
-			data = RREG32_PCIE(xgmi3x16_pcs_err_status_reg_v6_4[i]);
-			mask_data =
-				RREG32_PCIE(xgmi3x16_pcs_err_noncorrectable_mask_reg_v6_4[i]);
-			if (data)
-				amdgpu_xgmi_query_pcs_error_status(adev, data,
-						mask_data, &ue_cnt, &ce_cnt, true, true);
-		}
-		break;
-	default:
-		if (!supported)
-			dev_warn(adev->dev, "XGMI RAS error query not supported");
 		break;
 	}
 
@@ -1478,90 +1283,6 @@ static void amdgpu_xgmi_legacy_query_ras_error_count(struct amdgpu_device *adev,
 	err_data->ce_count += ce_cnt;
 }
 
-static enum aca_error_type xgmi_v6_4_0_pcs_mca_get_error_type(struct amdgpu_device *adev, u64 status)
-{
-	const char *error_str;
-	int ext_error_code;
-
-	ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status);
-
-	error_str = ext_error_code < ARRAY_SIZE(xgmi_v6_4_0_ras_error_code_ext) ?
-		xgmi_v6_4_0_ras_error_code_ext[ext_error_code] : NULL;
-	if (error_str)
-		dev_info(adev->dev, "%s detected\n", error_str);
-
-	switch (ext_error_code) {
-	case 0:
-		return ACA_ERROR_TYPE_UE;
-	case 6:
-		return ACA_ERROR_TYPE_CE;
-	default:
-		return -EINVAL;
-	}
-
-	return -EINVAL;
-}
-
-static void __xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, struct amdgpu_smuio_mcm_config_info *mcm_info,
-					    u64 mca_base, struct ras_err_data *err_data)
-{
-	int xgmi_inst = mcm_info->die_id;
-	uint64_t smn_base;
-	u64 status = 0;
-
-	status = RREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS);
-	if (!ACA_REG__STATUS__VAL(status))
-		return;
-
-	switch (xgmi_v6_4_0_pcs_mca_get_error_type(adev, status)) {
-	case ACA_ERROR_TYPE_UE:
-		amdgpu_ras_error_statistic_ue_count(err_data, mcm_info, 1ULL);
-		break;
-	case ACA_ERROR_TYPE_CE:
-		amdgpu_ras_error_statistic_ce_count(err_data, mcm_info, 1ULL);
-		break;
-	default:
-		break;
-	}
-	smn_base = amdgpu_reg_get_smn_base64(adev, XGMI_HWIP, xgmi_inst);
-	WREG64_MCA(smn_base, mca_base, ACA_REG_IDX_STATUS, 0ULL);
-}
-
-static void xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, int xgmi_inst, struct ras_err_data *err_data)
-{
-	struct amdgpu_smuio_mcm_config_info mcm_info = {
-		.socket_id = adev->smuio.funcs->get_socket_id(adev),
-		.die_id = xgmi_inst,
-	};
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(xgmi_v6_4_0_mca_base_array); i++)
-		__xgmi_v6_4_0_query_error_count(adev, &mcm_info, xgmi_v6_4_0_mca_base_array[i], err_data);
-}
-
-static void xgmi_v6_4_0_query_ras_error_count(struct amdgpu_device *adev, void *ras_error_status)
-{
-	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
-	int i;
-
-	for_each_inst(i, adev->aid_mask)
-		xgmi_v6_4_0_query_error_count(adev, i, err_data);
-}
-
-static void amdgpu_xgmi_query_ras_error_count(struct amdgpu_device *adev,
-					      void *ras_error_status)
-{
-	switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
-	case IP_VERSION(6, 4, 0):
-	case IP_VERSION(6, 4, 1):
-		xgmi_v6_4_0_query_ras_error_count(adev, ras_error_status);
-		break;
-	default:
-		amdgpu_xgmi_legacy_query_ras_error_count(adev, ras_error_status);
-		break;
-	}
-}
-
 /* Trigger XGMI/WAFL error */
 static int amdgpu_ras_error_inject_xgmi(struct amdgpu_device *adev,
 			void *inject_if, uint32_t instance_mask)
@@ -1663,6 +1384,19 @@ static void amdgpu_xgmi_reset_on_init_work(struct work_struct *work)
 		if (r && r != -EHWPOISON)
 			dev_err(tmp_adev->dev,
 				"error during bad page data initialization");
+
+		/*
+		 * For the reset-on-init path (e.g. an NPS memory partition
+		 * switch) the RAS IP block hw_init was skipped under the
+		 * minimal init level, so uniras was never enabled. Bring it
+		 * up now that the reset domain has been unlocked. This is a
+		 * no-op for any other reset path where RAS is already
+		 * initialized, and for non-uniras devices.
+		 */
+		r = amdgpu_ras_resume_after_reset(tmp_adev);
+		if (r)
+			dev_err(tmp_adev->dev,
+				"failed to resume RAS after XGMI reset-on-init\n");
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c
index 72ea37dbfea8d..cddfe4015f531 100644
--- a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c
+++ b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c
@@ -273,8 +273,10 @@ static int aqua_vanjaram_get_xcp_res_info(struct amdgpu_xcp_mgr *xcp_mgr,
 	xcp_cfg->num_res = ARRAY_SIZE(max_res);
 
 	for (i = 0; i < xcp_cfg->num_res; i++) {
-		res_lt_xcp = max_res[i] < num_xcp;
 		xcp_cfg->xcp_res[i].id = i;
+		if (!max_res[i])
+			continue;
+		res_lt_xcp = max_res[i] < num_xcp;
 		xcp_cfg->xcp_res[i].num_inst =
 			res_lt_xcp ? 1 : max_res[i] / num_xcp;
 		xcp_cfg->xcp_res[i].num_inst =
@@ -589,6 +591,29 @@ static struct aqua_reg_list pcie_reg_addrs[] = {
 	{ smreg_0x1A380088, 6, DW_ADDR_INCR },
 };
 
+/*
+ * Return the GPU's internal US switch port, or NULL if it is not visible
+ * (e.g. passthrough) or the EP is parented under an unrelated bridge.
+ */
+static struct pci_dev *aqua_vanjaram_get_us_pdev(struct amdgpu_device *adev)
+{
+	struct pci_dev *ds_pdev, *us_pdev;
+
+	ds_pdev = pci_upstream_bridge(adev->pdev);
+	if (!ds_pdev || ds_pdev->vendor != PCI_VENDOR_ID_ATI ||
+	    pci_pcie_type(ds_pdev) != PCI_EXP_TYPE_DOWNSTREAM)
+		return NULL;
+
+	us_pdev = pci_upstream_bridge(ds_pdev);
+	if (!us_pdev ||
+	    (us_pdev->vendor != PCI_VENDOR_ID_ATI &&
+	     us_pdev->vendor != PCI_VENDOR_ID_AMD) ||
+	    pci_pcie_type(us_pdev) != PCI_EXP_TYPE_UPSTREAM)
+		return NULL;
+
+	return us_pdev;
+}
+
 static ssize_t aqua_vanjaram_read_pcie_state(struct amdgpu_device *adev,
 					     void *buf, size_t max_size)
 {
@@ -596,7 +621,7 @@ static ssize_t aqua_vanjaram_read_pcie_state(struct amdgpu_device *adev,
 	uint32_t start_addr, incrx, num_regs, szbuf;
 	struct amdgpu_regs_pcie_v1_0 *pcie_regs;
 	struct amdgpu_smn_reg_data *reg_data;
-	struct pci_dev *us_pdev, *ds_pdev;
+	struct pci_dev *us_pdev;
 	int aer_cap, r, n;
 
 	if (!buf || !max_size)
@@ -628,25 +653,27 @@ static ssize_t aqua_vanjaram_read_pcie_state(struct amdgpu_device *adev,
 		}
 	}
 
-	ds_pdev = pci_upstream_bridge(adev->pdev);
-	us_pdev = pci_upstream_bridge(ds_pdev);
+	us_pdev = aqua_vanjaram_get_us_pdev(adev);
+	if (us_pdev) {
+		pcie_capability_read_word(us_pdev, PCI_EXP_DEVSTA,
+					  &pcie_regs->device_status);
+		pcie_capability_read_word(us_pdev, PCI_EXP_LNKSTA,
+					  &pcie_regs->link_status);
+
+		aer_cap = pci_find_ext_capability(us_pdev, PCI_EXT_CAP_ID_ERR);
+		if (aer_cap) {
+			pci_read_config_dword(us_pdev,
+					      aer_cap + PCI_ERR_COR_STATUS,
+					      &pcie_regs->pcie_corr_err_status);
+			pci_read_config_dword(us_pdev,
+					      aer_cap + PCI_ERR_UNCOR_STATUS,
+					      &pcie_regs->pcie_uncorr_err_status);
+		}
 
-	pcie_capability_read_word(us_pdev, PCI_EXP_DEVSTA,
-				  &pcie_regs->device_status);
-	pcie_capability_read_word(us_pdev, PCI_EXP_LNKSTA,
-				  &pcie_regs->link_status);
-
-	aer_cap = pci_find_ext_capability(us_pdev, PCI_EXT_CAP_ID_ERR);
-	if (aer_cap) {
-		pci_read_config_dword(us_pdev, aer_cap + PCI_ERR_COR_STATUS,
-				      &pcie_regs->pcie_corr_err_status);
-		pci_read_config_dword(us_pdev, aer_cap + PCI_ERR_UNCOR_STATUS,
-				      &pcie_regs->pcie_uncorr_err_status);
+		pci_read_config_dword(us_pdev, PCI_PRIMARY_BUS,
+				      &pcie_regs->sub_bus_number_latency);
 	}
 
-	pci_read_config_dword(us_pdev, PCI_PRIMARY_BUS,
-			      &pcie_regs->sub_bus_number_latency);
-
 	pcie_reg_state->common_header.structure_size = szbuf;
 	pcie_reg_state->common_header.format_revision = 1;
 	pcie_reg_state->common_header.content_revision = 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/atom.c b/drivers/gpu/drm/amd/amdgpu/atom.c
index ca5d091549e1f..e0e585f280e21 100644
--- a/drivers/gpu/drm/amd/amdgpu/atom.c
+++ b/drivers/gpu/drm/amd/amdgpu/atom.c
@@ -114,8 +114,10 @@ static uint32_t atom_iio_execute(struct atom_context *ctx, int base,
 				 uint32_t index, uint32_t data)
 {
 	uint32_t temp = 0xCDCDCDCD;
+	int start = base;
 
-	while (1)
+	/* IIO opcodes read up to base+3; keep within the BIOS image */
+	while (base + 3 < ctx->bios_size)
 		switch (CU8(base)) {
 		case ATOM_IIO_NOP:
 			base++;
@@ -180,6 +182,9 @@ static uint32_t atom_iio_execute(struct atom_context *ctx, int base,
 			pr_info("Unknown IIO opcode\n");
 			return 0;
 		}
+
+	pr_info("IIO method starting at offset %d runs past BIOS image\n", start);
+	return 0;
 }
 
 static uint32_t atom_get_src_int(atom_exec_context *ctx, uint8_t attr,
@@ -1327,11 +1332,25 @@ static void atom_index_iio(struct atom_context *ctx, int base)
 	ctx->iio = kzalloc(2 * 256, GFP_KERNEL);
 	if (!ctx->iio)
 		return;
-	while (CU8(base) == ATOM_IIO_START) {
-		ctx->iio[CU8(base + 1)] = base + 2;
+	while (base + 1 < ctx->bios_size && CU8(base) == ATOM_IIO_START) {
+		uint8_t index = CU8(base + 1);
+		int start = base + 2;
 		base += 2;
-		while (CU8(base) != ATOM_IIO_END)
-			base += atom_iio_len[CU8(base)];
+		while (base < ctx->bios_size && CU8(base) != ATOM_IIO_END) {
+			uint8_t op = CU8(base);
+
+			/*
+			 * Unknown opcode: its length is unknown so the byte
+			 * stream cannot be resynced reliably.
+			 */
+			if (op >= ARRAY_SIZE(atom_iio_len))
+				return;
+			base += atom_iio_len[op];
+		}
+		if (base >= ctx->bios_size)
+			return;
+		/* Only index well-formed methods, others stay 0 */
+		ctx->iio[index] = start;
 		base += 3;
 	}
 }
@@ -1339,6 +1358,7 @@ static void atom_index_iio(struct atom_context *ctx, int base)
 static void atom_get_vbios_name(struct atom_context *ctx)
 {
 	unsigned char *p_rom;
+	unsigned char *p_end;
 	unsigned char str_num;
 	unsigned short off_to_vbios_str;
 	unsigned char *c_ptr;
@@ -1349,39 +1369,48 @@ static void atom_get_vbios_name(struct atom_context *ctx)
 	char *back;
 
 	p_rom = ctx->bios;
+	p_end = p_rom + ctx->bios_size;
+
+	if (p_rom + OFFSET_TO_GET_ATOMBIOS_STRING_START + 1 >= p_end)
+		goto no_name;
 
 	str_num = *(p_rom + OFFSET_TO_GET_ATOMBIOS_NUMBER_OF_STRINGS);
-	if (str_num != 0) {
-		off_to_vbios_str =
-			*(unsigned short *)(p_rom + OFFSET_TO_GET_ATOMBIOS_STRING_START);
+	if (!str_num)
+		goto no_name;
 
-		c_ptr = (unsigned char *)(p_rom + off_to_vbios_str);
-	} else {
-		/* do not know where to find name */
-		memcpy(ctx->name, na, 7);
-		ctx->name[7] = 0;
-		return;
-	}
+	off_to_vbios_str =
+		*(unsigned short *)(p_rom + OFFSET_TO_GET_ATOMBIOS_STRING_START);
+
+	c_ptr = (unsigned char *)(p_rom + off_to_vbios_str);
+	if (c_ptr >= p_end)
+		goto no_name;
 
 	/*
 	 * skip the atombios strings, usually 4
 	 * 1st is P/N, 2nd is ASIC, 3rd is PCI type, 4th is Memory type
 	 */
 	for (i = 0; i < str_num; i++) {
-		while (*c_ptr != 0)
+		while (c_ptr < p_end && *c_ptr != 0)
 			c_ptr++;
 		c_ptr++;
 	}
 
 	/* skip the following 2 chars: 0x0D 0x0A */
 	c_ptr += 2;
+	if (c_ptr >= p_end)
+		goto no_name;
 
-	name_size = strnlen(c_ptr, STRLEN_LONG - 1);
+	name_size = strnlen(c_ptr, min(STRLEN_LONG - 1, (int)(p_end - c_ptr)));
 	memcpy(ctx->name, c_ptr, name_size);
 	back = ctx->name + name_size;
 	while ((*--back) == ' ')
 		;
 	*(back + 1) = '\0';
+	return;
+
+no_name:
+	/* do not know where to find name */
+	strscpy(ctx->name, na, sizeof(ctx->name));
 }
 
 static void atom_get_vbios_date(struct atom_context *ctx)
@@ -1553,7 +1582,7 @@ static inline void atom_print_vbios_info(struct atom_context *ctx)
 		drm_info(ctx->card->dev, "ATOM BIOS: %s\n", vbios_info);
 }
 
-struct atom_context *amdgpu_atom_parse(struct card_info *card, void *bios)
+struct atom_context *amdgpu_atom_parse(struct card_info *card, void *bios, uint32_t bios_size)
 {
 	int base;
 	struct atom_context *ctx =
@@ -1567,6 +1596,7 @@ struct atom_context *amdgpu_atom_parse(struct card_info *card, void *bios)
 
 	ctx->card = card;
 	ctx->bios = bios;
+	ctx->bios_size = bios_size;
 
 	if (CU16(0) != ATOM_BIOS_MAGIC) {
 		pr_info("Invalid BIOS magic\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/atom.h b/drivers/gpu/drm/amd/amdgpu/atom.h
index bb3d9eb7eb6b5..4687c019cbe30 100644
--- a/drivers/gpu/drm/amd/amdgpu/atom.h
+++ b/drivers/gpu/drm/amd/amdgpu/atom.h
@@ -133,6 +133,7 @@ struct atom_context {
 	struct card_info *card;
 	struct mutex mutex;
 	void *bios;
+	uint32_t bios_size;
 	uint32_t cmd_table, data_table;
 	uint16_t *iio;
 
@@ -160,7 +161,7 @@ struct atom_context {
 
 extern int amdgpu_atom_debug;
 
-struct atom_context *amdgpu_atom_parse(struct card_info *card, void *bios);
+struct atom_context *amdgpu_atom_parse(struct card_info *card, void *bios, uint32_t bios_size);
 int amdgpu_atom_execute_table(struct atom_context *ctx, int index, uint32_t *params, int params_size);
 int amdgpu_atom_asic_init(struct atom_context *ctx);
 void amdgpu_atom_destroy(struct atom_context *ctx);
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index 29954c7d61b06..e037c08c85479 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -36,6 +36,7 @@
 #include "cikd.h"
 #include "atom.h"
 #include "amd_pcie.h"
+#include "amdgpu_video_codecs.h"
 
 #include "cik.h"
 #include "gmc_v7_0.h"
@@ -1876,12 +1877,6 @@ static void cik_invalidate_hdp(struct amdgpu_device *adev,
 	}
 }
 
-static bool cik_need_full_reset(struct amdgpu_device *adev)
-{
-	/* change this when we support soft reset */
-	return true;
-}
-
 static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
 			       uint64_t *count1)
 {
@@ -1971,7 +1966,6 @@ static const struct amdgpu_asic_funcs cik_asic_funcs =
 	.get_config_memsize = &cik_get_config_memsize,
 	.flush_hdp = &cik_flush_hdp,
 	.invalidate_hdp = &cik_invalidate_hdp,
-	.need_full_reset = &cik_need_full_reset,
 	.init_doorbell_index = &legacy_doorbell_index_init,
 	.get_pcie_usage = &cik_get_pcie_usage,
 	.need_reset_on_init = &cik_need_reset_on_init,
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index 120da838ac282..b951328d94cfb 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -125,7 +125,8 @@ static int cik_sdma_init_microcode(struct amdgpu_device *adev)
 	case CHIP_MULLINS:
 		chip_name = "mullins";
 		break;
-	default: BUG();
+	default:
+		return -EINVAL;
 	}
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
@@ -605,7 +606,7 @@ static int cik_sdma_ring_test_ring(struct amdgpu_ring *ring)
 	u32 tmp;
 	u64 gpu_addr;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -635,7 +636,7 @@ static int cik_sdma_ring_test_ring(struct amdgpu_ring *ring)
 		r = -ETIMEDOUT;
 
 error_free_wb:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -658,7 +659,7 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	u64 gpu_addr;
 	long r;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -699,7 +700,7 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index c8f465158e711..f2977fe6d8242 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -410,36 +410,6 @@ static u32 dce_v10_0_hpd_get_gpio_reg(struct amdgpu_device *adev)
 	return mmDC_GPIO_HPD_A;
 }
 
-static bool dce_v10_0_is_display_hung(struct amdgpu_device *adev)
-{
-	u32 crtc_hung = 0;
-	u32 crtc_status[6];
-	u32 i, j, tmp;
-
-	for (i = 0; i < adev->mode_info.num_crtc; i++) {
-		tmp = RREG32(mmCRTC_CONTROL + crtc_offsets[i]);
-		if (REG_GET_FIELD(tmp, CRTC_CONTROL, CRTC_MASTER_EN)) {
-			crtc_status[i] = RREG32(mmCRTC_STATUS_HV_COUNT + crtc_offsets[i]);
-			crtc_hung |= (1 << i);
-		}
-	}
-
-	for (j = 0; j < 10; j++) {
-		for (i = 0; i < adev->mode_info.num_crtc; i++) {
-			if (crtc_hung & (1 << i)) {
-				tmp = RREG32(mmCRTC_STATUS_HV_COUNT + crtc_offsets[i]);
-				if (tmp != crtc_status[i])
-					crtc_hung &= ~(1 << i);
-			}
-		}
-		if (crtc_hung == 0)
-			return false;
-		udelay(100);
-	}
-
-	return true;
-}
-
 static void dce_v10_0_set_vga_render_state(struct amdgpu_device *adev,
 					   bool render)
 {
@@ -2956,40 +2926,6 @@ static bool dce_v10_0_is_idle(struct amdgpu_ip_block *ip_block)
 	return true;
 }
 
-static bool dce_v10_0_check_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-
-	return dce_v10_0_is_display_hung(adev);
-}
-
-static int dce_v10_0_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	u32 srbm_soft_reset = 0, tmp;
-	struct amdgpu_device *adev = ip_block->adev;
-
-	if (dce_v10_0_is_display_hung(adev))
-		srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_DC_MASK;
-
-	if (srbm_soft_reset) {
-		tmp = RREG32(mmSRBM_SOFT_RESET);
-		tmp |= srbm_soft_reset;
-		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
-		WREG32(mmSRBM_SOFT_RESET, tmp);
-		tmp = RREG32(mmSRBM_SOFT_RESET);
-
-		udelay(50);
-
-		tmp &= ~srbm_soft_reset;
-		WREG32(mmSRBM_SOFT_RESET, tmp);
-		tmp = RREG32(mmSRBM_SOFT_RESET);
-
-		/* Wait a little for things to settle down */
-		udelay(50);
-	}
-	return 0;
-}
-
 static void dce_v10_0_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev,
 						     int crtc,
 						     enum amdgpu_interrupt_state state)
@@ -3332,8 +3268,6 @@ static const struct amd_ip_funcs dce_v10_0_ip_funcs = {
 	.suspend = dce_v10_0_suspend,
 	.resume = dce_v10_0_resume,
 	.is_idle = dce_v10_0_is_idle,
-	.check_soft_reset = dce_v10_0_check_soft_reset,
-	.soft_reset = dce_v10_0_soft_reset,
 	.set_clockgating_state = dce_v10_0_set_clockgating_state,
 	.set_powergating_state = dce_v10_0_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
index 58d0da5c2a74f..c68de0fe1d7d3 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
@@ -378,35 +378,6 @@ static u32 dce_v6_0_hpd_get_gpio_reg(struct amdgpu_device *adev)
 	return mmDC_GPIO_HPD_A;
 }
 
-static bool dce_v6_0_is_display_hung(struct amdgpu_device *adev)
-{
-	u32 crtc_hung = 0;
-	u32 crtc_status[6];
-	u32 i, j, tmp;
-
-	for (i = 0; i < adev->mode_info.num_crtc; i++) {
-		if (RREG32(mmCRTC_CONTROL + crtc_offsets[i]) & CRTC_CONTROL__CRTC_MASTER_EN_MASK) {
-			crtc_status[i] = RREG32(mmCRTC_STATUS_HV_COUNT + crtc_offsets[i]);
-			crtc_hung |= (1 << i);
-		}
-	}
-
-	for (j = 0; j < 10; j++) {
-		for (i = 0; i < adev->mode_info.num_crtc; i++) {
-			if (crtc_hung & (1 << i)) {
-				tmp = RREG32(mmCRTC_STATUS_HV_COUNT + crtc_offsets[i]);
-				if (tmp != crtc_status[i])
-					crtc_hung &= ~(1 << i);
-			}
-		}
-		if (crtc_hung == 0)
-			return false;
-		udelay(100);
-	}
-
-	return true;
-}
-
 static void dce_v6_0_set_vga_render_state(struct amdgpu_device *adev,
 					  bool render)
 {
@@ -2901,33 +2872,6 @@ static bool dce_v6_0_is_idle(struct amdgpu_ip_block *ip_block)
 	return true;
 }
 
-static int dce_v6_0_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	u32 srbm_soft_reset = 0, tmp;
-	struct amdgpu_device *adev = ip_block->adev;
-
-	if (dce_v6_0_is_display_hung(adev))
-		srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_DC_MASK;
-
-	if (srbm_soft_reset) {
-		tmp = RREG32(mmSRBM_SOFT_RESET);
-		tmp |= srbm_soft_reset;
-		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
-		WREG32(mmSRBM_SOFT_RESET, tmp);
-		tmp = RREG32(mmSRBM_SOFT_RESET);
-
-		udelay(50);
-
-		tmp &= ~srbm_soft_reset;
-		WREG32(mmSRBM_SOFT_RESET, tmp);
-		tmp = RREG32(mmSRBM_SOFT_RESET);
-
-		/* Wait a little for things to settle down */
-		udelay(50);
-	}
-	return 0;
-}
-
 static void dce_v6_0_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev,
 						     int crtc,
 						     enum amdgpu_interrupt_state state)
@@ -3224,7 +3168,6 @@ static const struct amd_ip_funcs dce_v6_0_ip_funcs = {
 	.suspend = dce_v6_0_suspend,
 	.resume = dce_v6_0_resume,
 	.is_idle = dce_v6_0_is_idle,
-	.soft_reset = dce_v6_0_soft_reset,
 	.set_clockgating_state = dce_v6_0_set_clockgating_state,
 	.set_powergating_state = dce_v6_0_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index 6d19f6d94d252..c3906270f25ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -362,35 +362,6 @@ static u32 dce_v8_0_hpd_get_gpio_reg(struct amdgpu_device *adev)
 	return mmDC_GPIO_HPD_A;
 }
 
-static bool dce_v8_0_is_display_hung(struct amdgpu_device *adev)
-{
-	u32 crtc_hung = 0;
-	u32 crtc_status[6];
-	u32 i, j, tmp;
-
-	for (i = 0; i < adev->mode_info.num_crtc; i++) {
-		if (RREG32(mmCRTC_CONTROL + crtc_offsets[i]) & CRTC_CONTROL__CRTC_MASTER_EN_MASK) {
-			crtc_status[i] = RREG32(mmCRTC_STATUS_HV_COUNT + crtc_offsets[i]);
-			crtc_hung |= (1 << i);
-		}
-	}
-
-	for (j = 0; j < 10; j++) {
-		for (i = 0; i < adev->mode_info.num_crtc; i++) {
-			if (crtc_hung & (1 << i)) {
-				tmp = RREG32(mmCRTC_STATUS_HV_COUNT + crtc_offsets[i]);
-				if (tmp != crtc_status[i])
-					crtc_hung &= ~(1 << i);
-			}
-		}
-		if (crtc_hung == 0)
-			return false;
-		udelay(100);
-	}
-
-	return true;
-}
-
 static void dce_v8_0_set_vga_render_state(struct amdgpu_device *adev,
 					  bool render)
 {
@@ -2873,33 +2844,6 @@ static bool dce_v8_0_is_idle(struct amdgpu_ip_block *ip_block)
 	return true;
 }
 
-static int dce_v8_0_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	u32 srbm_soft_reset = 0, tmp;
-	struct amdgpu_device *adev = ip_block->adev;
-
-	if (dce_v8_0_is_display_hung(adev))
-		srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_DC_MASK;
-
-	if (srbm_soft_reset) {
-		tmp = RREG32(mmSRBM_SOFT_RESET);
-		tmp |= srbm_soft_reset;
-		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
-		WREG32(mmSRBM_SOFT_RESET, tmp);
-		tmp = RREG32(mmSRBM_SOFT_RESET);
-
-		udelay(50);
-
-		tmp &= ~srbm_soft_reset;
-		WREG32(mmSRBM_SOFT_RESET, tmp);
-		tmp = RREG32(mmSRBM_SOFT_RESET);
-
-		/* Wait a little for things to settle down */
-		udelay(50);
-	}
-	return 0;
-}
-
 static void dce_v8_0_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev,
 						     int crtc,
 						     enum amdgpu_interrupt_state state)
@@ -3241,7 +3185,6 @@ static const struct amd_ip_funcs dce_v8_0_ip_funcs = {
 	.suspend = dce_v8_0_suspend,
 	.resume = dce_v8_0_resume,
 	.is_idle = dce_v8_0_is_idle,
-	.soft_reset = dce_v8_0_soft_reset,
 	.set_clockgating_state = dce_v8_0_set_clockgating_state,
 	.set_powergating_state = dce_v8_0_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
index a9961d5048333..a998ce77da40b 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
@@ -4080,7 +4080,7 @@ static int gfx_v10_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 
 	memset(&ib, 0, sizeof(ib));
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -4121,7 +4121,7 @@ static int gfx_v10_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err1:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -4988,12 +4988,12 @@ static int gfx_v10_0_sw_init(struct amdgpu_ip_block *ip_block)
 
 	gfx_v10_0_gpu_early_init(adev);
 
-	gfx_v10_0_alloc_ip_dump(adev);
-
 	r = amdgpu_gfx_sysfs_init(adev);
 	if (r)
 		return r;
 
+	gfx_v10_0_alloc_ip_dump(adev);
+
 	return 0;
 }
 
@@ -7861,6 +7861,8 @@ static int gfx_v10_0_early_init(struct amdgpu_ip_block *ip_block)
 	/* init rlcg reg access ctrl */
 	gfx_v10_0_init_rlcg_reg_access_ctrl(adev);
 
+	amdgpu_init_rlc_reg_funcs(adev);
+
 	return gfx_v10_0_init_microcode(adev);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
index 3b12eb27a2533..0cbbdc3694f4c 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
@@ -51,7 +51,7 @@
 #include "mes_userqueue.h"
 #include "amdgpu_userq_fence.h"
 
-#define GFX11_NUM_GFX_RINGS		1
+#define GFX11_NUM_GFX_RINGS		2
 #define GFX11_MEC_HPD_SIZE	2048
 
 #define RLCG_UCODE_LOADING_START_ADDRESS	0x00002000L
@@ -635,7 +635,7 @@ static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 
 	memset(&ib, 0, sizeof(ib));
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -676,7 +676,7 @@ static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err1:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -1625,7 +1625,10 @@ static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
 	case IP_VERSION(11, 7, 0):
 	case IP_VERSION(11, 7, 1):
 		adev->gfx.me.num_me = 1;
-		adev->gfx.me.num_pipe_per_me = 1;
+		if (adev->gfx.rs64_enable)
+			adev->gfx.me.num_pipe_per_me = 1;
+		else
+			adev->gfx.me.num_pipe_per_me = 2;
 		adev->gfx.me.num_queue_per_pipe = 2;
 		adev->gfx.mec.num_mec = 1;
 		adev->gfx.mec.num_pipe_per_mec = 4;
@@ -1917,12 +1920,17 @@ static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
 		return -EINVAL;
 	}
 
-	gfx_v11_0_alloc_ip_dump(adev);
-
 	r = amdgpu_gfx_sysfs_init(adev);
 	if (r)
 		return r;
 
+	gfx_v11_0_alloc_ip_dump(adev);
+
+	adev->gfx.me.use_mmio_for_reset = false;
+	adev->gfx.mec.use_mmio_for_reset = true;
+
+	mutex_init(&adev->gfx.mec.reset_mutex);
+
 	return 0;
 }
 
@@ -2186,6 +2194,15 @@ static void gfx_v11_0_constants_init(struct amdgpu_device *adev)
 	tmp = RREG32_SOC15(GC, 0, regTA_CNTL2);
 	adev->gfx.config.ta_cntl2_truncate_coord_mode =
 		REG_GET_FIELD(tmp, TA_CNTL2, TRUNCATE_COORD_MODE);
+	/* Program DB_RING_CONTROL for multiple GFX pipes
+	 * Default power up value is 1.
+	 * Possible values:
+	 * 0 - split occlusion counters between gfx pipes
+	 * 1 - all occlusion counters to pipe 0
+	 * 2 - all occlusion counters to pipe 1
+	 */
+	WREG32_FIELD15_PREREG(GC, 0, DB_RING_CONTROL, COUNTER_CONTROL,
+			      (adev->gfx.me.num_pipe_per_me > 1) ? 0 : 1);
 
 	/* XXX SH_MEM regs */
 	/* where to put LDS, scratch, GPUVM in FSA64 space */
@@ -4233,13 +4250,13 @@ static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m,
 	return 0;
 }
 
-static int gfx_v11_0_kgq_init_queue(struct amdgpu_ring *ring, bool reset)
+static int gfx_v11_0_kgq_init_queue(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct v11_gfx_mqd *mqd = ring->mqd_ptr;
 	int mqd_idx = ring - &adev->gfx.gfx_ring[0];
 
-	if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) {
+	if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
 		memset((void *)mqd, 0, sizeof(*mqd));
 		mutex_lock(&adev->srbm_mutex);
 		soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
@@ -4266,7 +4283,7 @@ static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
 	int r, i;
 
 	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
-		r = gfx_v11_0_kgq_init_queue(&adev->gfx.gfx_ring[i], false);
+		r = gfx_v11_0_kgq_init_queue(&adev->gfx.gfx_ring[i]);
 		if (r)
 			return r;
 	}
@@ -4603,13 +4620,13 @@ static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring)
 	return 0;
 }
 
-static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring, bool reset)
+static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct v11_compute_mqd *mqd = ring->mqd_ptr;
 	int mqd_idx = ring - &adev->gfx.compute_ring[0];
 
-	if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) {
+	if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
 		memset((void *)mqd, 0, sizeof(*mqd));
 		mutex_lock(&adev->srbm_mutex);
 		soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
@@ -4646,7 +4663,7 @@ static int gfx_v11_0_kcq_resume(struct amdgpu_device *adev)
 		gfx_v11_0_cp_compute_enable(adev, true);
 
 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
-		r = gfx_v11_0_kcq_init_queue(&adev->gfx.compute_ring[i], false);
+		r = gfx_v11_0_kcq_init_queue(&adev->gfx.compute_ring[i]);
 		if (r)
 			return r;
 	}
@@ -5265,38 +5282,12 @@ static int gfx_v11_0_soft_reset(struct amdgpu_ip_block *ip_block)
 
 	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
 
-	return gfx_v11_0_cp_resume(adev);
-}
-
-static bool gfx_v11_0_check_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	int i, r;
-	struct amdgpu_device *adev = ip_block->adev;
-	struct amdgpu_ring *ring;
-	long tmo = msecs_to_jiffies(1000);
-
-	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
-		ring = &adev->gfx.gfx_ring[i];
-		r = amdgpu_ring_test_ib(ring, tmo);
-		if (r)
-			return true;
-	}
-
-	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
-		ring = &adev->gfx.compute_ring[i];
-		r = amdgpu_ring_test_ib(ring, tmo);
-		if (r)
-			return true;
-	}
-
-	return false;
-}
+	r = gfx_v11_0_cp_resume(adev);
+	if (r)
+		return r;
 
-static int gfx_v11_0_post_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
 	/**
-	 * GFX soft reset will impact MES, need resume MES when do GFX soft reset
+	 * GFX soft reset impacts MES, resume MES after GFX soft reset is finished
 	 */
 	return amdgpu_mes_resume(adev, 0);
 }
@@ -5378,6 +5369,7 @@ static void gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
 static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
+	int r;
 
 	switch (amdgpu_user_queue) {
 	case -1:
@@ -5398,6 +5390,11 @@ static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block)
 
 	adev->gfx.funcs = &gfx_v11_0_gfx_funcs;
 
+	gfx_v11_0_set_imu_funcs(adev);
+	r = gfx_v11_0_init_microcode(adev);
+	if (r)
+		return r;
+
 	if (adev->gfx.disable_kq) {
 		/* We need one GFX ring temporarily to set up
 		 * the clear state.
@@ -5405,7 +5402,11 @@ static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block)
 		adev->gfx.num_gfx_rings = 1;
 		adev->gfx.num_compute_rings = 0;
 	} else {
-		adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS;
+		/* rs64 only supports one gfx pipe */
+		if (adev->gfx.rs64_enable)
+			adev->gfx.num_gfx_rings = 1;
+		else
+			adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS;
 		adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
 						  AMDGPU_MAX_COMPUTE_RINGS);
 	}
@@ -5416,11 +5417,12 @@ static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block)
 	gfx_v11_0_set_gds_init(adev);
 	gfx_v11_0_set_rlc_funcs(adev);
 	gfx_v11_0_set_mqd_funcs(adev);
-	gfx_v11_0_set_imu_funcs(adev);
 
 	gfx_v11_0_init_rlcg_reg_access_ctrl(adev);
 
-	return gfx_v11_0_init_microcode(adev);
+	amdgpu_init_rlc_reg_funcs(adev);
+
+	return 0;
 }
 
 static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev)
@@ -6708,22 +6710,29 @@ static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
 static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev,
 					struct amdgpu_iv_entry *entry)
 {
-	u8 me_id, pipe_id, queue_id;
-	struct amdgpu_ring *ring;
-	int i;
-
-	me_id = (entry->ring_id & 0x0c) >> 2;
-	pipe_id = (entry->ring_id & 0x03) >> 0;
-	queue_id = (entry->ring_id & 0x70) >> 4;
+	u32 doorbell_offset = entry->src_data[0] & AMDGPU_CTXID0_DOORBELL_ID_MASK;
 
+	/*
+	 * Try KQ first by ring_id (HW slot is authoritative). The
+	 * KMD compute_hqd_mask contract guarantees KCQ and user queues
+	 * never share a HW slot.
+	 */
 	if (!adev->gfx.disable_kq) {
+		u8 me_id = (entry->ring_id & 0x0c) >> 2;
+		u8 pipe_id = (entry->ring_id & 0x03) >> 0;
+		u8 queue_id = (entry->ring_id & 0x70) >> 4;
+		struct amdgpu_ring *ring;
+		int i;
+
 		switch (me_id) {
 		case 0:
 			for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 				ring = &adev->gfx.gfx_ring[i];
 				if (ring->me == me_id && ring->pipe == pipe_id &&
-				    ring->queue == queue_id)
+				    ring->queue == queue_id) {
 					drm_sched_fault(&ring->sched);
+					return;
+				}
 			}
 			break;
 		case 1:
@@ -6731,8 +6740,10 @@ static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev,
 			for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 				ring = &adev->gfx.compute_ring[i];
 				if (ring->me == me_id && ring->pipe == pipe_id &&
-				    ring->queue == queue_id)
+				    ring->queue == queue_id) {
 					drm_sched_fault(&ring->sched);
+					return;
+				}
 			}
 			break;
 		default:
@@ -6740,6 +6751,11 @@ static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev,
 			break;
 		}
 	}
+
+	/* No KQ matched: HW slot is a MES-scheduled user queue. */
+	if (adev->enable_mes && doorbell_offset)
+		amdgpu_userq_process_reset_irq(adev, entry->pasid,
+					       doorbell_offset);
 }
 
 static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev,
@@ -6846,233 +6862,14 @@ static void gfx_v11_0_emit_mem_sync(struct amdgpu_ring *ring)
 	amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */
 }
 
-static bool gfx_v11_pipe_reset_support(struct amdgpu_device *adev)
-{
-	/* Disable the pipe reset until the CPFW fully support it.*/
-	dev_warn_once(adev->dev, "The CPFW hasn't support pipe reset yet.\n");
-	return false;
-}
-
-
-static int gfx_v11_reset_gfx_pipe(struct amdgpu_ring *ring)
-{
-	struct amdgpu_device *adev = ring->adev;
-	uint32_t reset_pipe = 0, clean_pipe = 0;
-	int r;
-
-	if (!gfx_v11_pipe_reset_support(adev))
-		return -EOPNOTSUPP;
-
-	gfx_v11_0_set_safe_mode(adev, 0);
-	mutex_lock(&adev->srbm_mutex);
-	soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
-
-	switch (ring->pipe) {
-	case 0:
-		reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
-					   PFP_PIPE0_RESET, 1);
-		reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
-					   ME_PIPE0_RESET, 1);
-		clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
-					   PFP_PIPE0_RESET, 0);
-		clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
-					   ME_PIPE0_RESET, 0);
-		break;
-	case 1:
-		reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
-					   PFP_PIPE1_RESET, 1);
-		reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
-					   ME_PIPE1_RESET, 1);
-		clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
-					   PFP_PIPE1_RESET, 0);
-		clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
-					   ME_PIPE1_RESET, 0);
-		break;
-	default:
-		break;
-	}
-
-	WREG32_SOC15(GC, 0, regCP_ME_CNTL, reset_pipe);
-	WREG32_SOC15(GC, 0, regCP_ME_CNTL, clean_pipe);
-
-	r = (RREG32(SOC15_REG_OFFSET(GC, 0, regCP_GFX_RS64_INSTR_PNTR1)) << 2) -
-						RS64_FW_UC_START_ADDR_LO;
-	soc21_grbm_select(adev, 0, 0, 0, 0);
-	mutex_unlock(&adev->srbm_mutex);
-	gfx_v11_0_unset_safe_mode(adev, 0);
-
-	dev_info(adev->dev, "The ring %s pipe reset to the ME firmware start PC: %s\n", ring->name,
-			r == 0 ? "successfully" : "failed");
-	/* FIXME: Sometimes driver can't cache the ME firmware start PC correctly,
-	 * so the pipe reset status relies on the later gfx ring test result.
-	 */
-	return 0;
-}
-
 static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring,
 			       unsigned int vmid,
 			       struct amdgpu_fence *timedout_fence)
 {
 	struct amdgpu_device *adev = ring->adev;
-	bool use_mmio = false;
-	int r;
-
-	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
-
-	r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, use_mmio, 0);
-	if (r) {
-
-		dev_warn(adev->dev, "reset via MES failed and try pipe reset %d\n", r);
-		r = gfx_v11_reset_gfx_pipe(ring);
-		if (r)
-			return r;
-	}
-
-	if (use_mmio) {
-		r = gfx_v11_0_kgq_init_queue(ring, true);
-		if (r) {
-			dev_err(adev->dev, "failed to init kgq\n");
-			return r;
-		}
-
-		r = amdgpu_mes_map_legacy_queue(adev, ring, 0);
-		if (r) {
-			dev_err(adev->dev, "failed to remap kgq\n");
-			return r;
-		}
-	}
-
-	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
-}
-
-static int gfx_v11_0_reset_compute_pipe(struct amdgpu_ring *ring)
-{
-
-	struct amdgpu_device *adev = ring->adev;
-	uint32_t reset_pipe = 0, clean_pipe = 0;
-	int r;
-
-	if (!gfx_v11_pipe_reset_support(adev))
-		return -EOPNOTSUPP;
-
-	gfx_v11_0_set_safe_mode(adev, 0);
-	mutex_lock(&adev->srbm_mutex);
-	soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
-
-	reset_pipe = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
-	clean_pipe = reset_pipe;
-
-	if (adev->gfx.rs64_enable) {
-
-		switch (ring->pipe) {
-		case 0:
-			reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL,
-						   MEC_PIPE0_RESET, 1);
-			clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL,
-						   MEC_PIPE0_RESET, 0);
-			break;
-		case 1:
-			reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL,
-						   MEC_PIPE1_RESET, 1);
-			clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL,
-						   MEC_PIPE1_RESET, 0);
-			break;
-		case 2:
-			reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL,
-						   MEC_PIPE2_RESET, 1);
-			clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL,
-						   MEC_PIPE2_RESET, 0);
-			break;
-		case 3:
-			reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL,
-						   MEC_PIPE3_RESET, 1);
-			clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL,
-						   MEC_PIPE3_RESET, 0);
-			break;
-		default:
-			break;
-		}
-		WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, reset_pipe);
-		WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, clean_pipe);
-		r = (RREG32_SOC15(GC, 0, regCP_MEC_RS64_INSTR_PNTR) << 2) -
-					RS64_FW_UC_START_ADDR_LO;
-	} else {
-		if (ring->me == 1) {
-			switch (ring->pipe) {
-			case 0:
-				reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
-							   MEC_ME1_PIPE0_RESET, 1);
-				clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
-							   MEC_ME1_PIPE0_RESET, 0);
-				break;
-			case 1:
-				reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
-							   MEC_ME1_PIPE1_RESET, 1);
-				clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
-							   MEC_ME1_PIPE1_RESET, 0);
-				break;
-			case 2:
-				reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
-							   MEC_ME1_PIPE2_RESET, 1);
-				clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
-							   MEC_ME1_PIPE2_RESET, 0);
-				break;
-			case 3:
-				reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
-							   MEC_ME1_PIPE3_RESET, 1);
-				clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
-							   MEC_ME1_PIPE3_RESET, 0);
-				break;
-			default:
-				break;
-			}
-			/* mec1 fw pc: CP_MEC1_INSTR_PNTR */
-		} else {
-			switch (ring->pipe) {
-			case 0:
-				reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
-							   MEC_ME2_PIPE0_RESET, 1);
-				clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
-							   MEC_ME2_PIPE0_RESET, 0);
-				break;
-			case 1:
-				reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
-							   MEC_ME2_PIPE1_RESET, 1);
-				clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
-							   MEC_ME2_PIPE1_RESET, 0);
-				break;
-			case 2:
-				reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
-							   MEC_ME2_PIPE2_RESET, 1);
-				clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
-							   MEC_ME2_PIPE2_RESET, 0);
-				break;
-			case 3:
-				reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
-							   MEC_ME2_PIPE3_RESET, 1);
-				clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
-							   MEC_ME2_PIPE3_RESET, 0);
-				break;
-			default:
-				break;
-			}
-			/* mec2 fw pc: CP:CP_MEC2_INSTR_PNTR */
-		}
-		WREG32_SOC15(GC, 0, regCP_MEC_CNTL, reset_pipe);
-		WREG32_SOC15(GC, 0, regCP_MEC_CNTL, clean_pipe);
-		r = RREG32(SOC15_REG_OFFSET(GC, 0, regCP_MEC1_INSTR_PNTR));
-	}
-
-	soc21_grbm_select(adev, 0, 0, 0, 0);
-	mutex_unlock(&adev->srbm_mutex);
-	gfx_v11_0_unset_safe_mode(adev, 0);
+	bool use_mmio = adev->gfx.me.use_mmio_for_reset;
 
-	dev_info(adev->dev, "The ring %s pipe resets to MEC FW start PC: %s\n", ring->name,
-			r == 0 ? "successfully" : "failed");
-	/*FIXME:Sometimes driver can't cache the MEC firmware start PC correctly, so the pipe
-	 * reset status relies on the compute ring test result.
-	 */
-	return 0;
+	return amdgpu_gfx_mes_reset_queue(ring, vmid, timedout_fence, use_mmio);
 }
 
 static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring,
@@ -7080,30 +6877,8 @@ static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring,
 			       struct amdgpu_fence *timedout_fence)
 {
 	struct amdgpu_device *adev = ring->adev;
-	int r = 0;
-
-	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
-
-	r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, true, 0);
-	if (r) {
-		dev_warn(adev->dev, "fail(%d) to reset kcq and try pipe reset\n", r);
-		r = gfx_v11_0_reset_compute_pipe(ring);
-		if (r)
-			return r;
-	}
-
-	r = gfx_v11_0_kcq_init_queue(ring, true);
-	if (r) {
-		dev_err(adev->dev, "fail to init kcq\n");
-		return r;
-	}
-	r = amdgpu_mes_map_legacy_queue(adev, ring, 0);
-	if (r) {
-		dev_err(adev->dev, "failed to remap kcq\n");
-		return r;
-	}
 
-	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
+	return amdgpu_gfx_reset_mes_compute(adev, ring, timedout_fence, NULL, NULL, NULL);
 }
 
 static void gfx_v11_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p)
@@ -7281,8 +7056,6 @@ static const struct amd_ip_funcs gfx_v11_0_ip_funcs = {
 	.is_idle = gfx_v11_0_is_idle,
 	.wait_for_idle = gfx_v11_0_wait_for_idle,
 	.soft_reset = gfx_v11_0_soft_reset,
-	.check_soft_reset = gfx_v11_0_check_soft_reset,
-	.post_soft_reset = gfx_v11_0_post_soft_reset,
 	.set_clockgating_state = gfx_v11_0_set_clockgating_state,
 	.set_powergating_state = gfx_v11_0_set_powergating_state,
 	.get_clockgating_state = gfx_v11_0_get_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
index da668a8d6abdf..b3887c5262a0d 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
@@ -507,7 +507,7 @@ static int gfx_v12_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 
 	memset(&ib, 0, sizeof(ib));
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -548,7 +548,7 @@ static int gfx_v12_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err1:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -1597,12 +1597,17 @@ static int gfx_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 	if (r)
 		return r;
 
-	gfx_v12_0_alloc_ip_dump(adev);
-
 	r = amdgpu_gfx_sysfs_init(adev);
 	if (r)
 		return r;
 
+	gfx_v12_0_alloc_ip_dump(adev);
+
+	adev->gfx.me.use_mmio_for_reset = false;
+	adev->gfx.mec.use_mmio_for_reset = true;
+
+	mutex_init(&adev->gfx.mec.reset_mutex);
+
 	return 0;
 }
 
@@ -1822,6 +1827,15 @@ static void gfx_v12_0_constants_init(struct amdgpu_device *adev)
 	gfx_v12_0_get_cu_info(adev, &adev->gfx.cu_info);
 	gfx_v12_0_get_tcc_info(adev);
 	adev->gfx.config.pa_sc_tile_steering_override = 0;
+	/* Program DB_RING_CONTROL for multiple GFX pipes
+	 * Default power up value is 1.
+	 * Possible values:
+	 * 0 - split occlusion counters between gfx pipes
+	 * 1 - all occlusion counters to pipe 0
+	 * 2 - all occlusion counters to pipe 1
+	 */
+	WREG32_FIELD15_PREREG(GC, 0, DB_RING_CONTROL, COUNTER_CONTROL,
+			      (adev->gfx.me.num_pipe_per_me > 1) ? 0 : 1);
 
 	/* XXX SH_MEM regs */
 	/* where to put LDS, scratch, GPUVM in FSA64 space */
@@ -3071,13 +3085,13 @@ static int gfx_v12_0_gfx_mqd_init(struct amdgpu_device *adev, void *m,
 	return 0;
 }
 
-static int gfx_v12_0_kgq_init_queue(struct amdgpu_ring *ring, bool reset)
+static int gfx_v12_0_kgq_init_queue(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct v12_gfx_mqd *mqd = ring->mqd_ptr;
 	int mqd_idx = ring - &adev->gfx.gfx_ring[0];
 
-	if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) {
+	if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
 		memset((void *)mqd, 0, sizeof(*mqd));
 		mutex_lock(&adev->srbm_mutex);
 		soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
@@ -3104,7 +3118,7 @@ static int gfx_v12_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
 	int i, r;
 
 	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
-		r = gfx_v12_0_kgq_init_queue(&adev->gfx.gfx_ring[i], false);
+		r = gfx_v12_0_kgq_init_queue(&adev->gfx.gfx_ring[i]);
 		if (r)
 			return r;
 	}
@@ -3441,13 +3455,13 @@ static int gfx_v12_0_kiq_init_queue(struct amdgpu_ring *ring)
 	return 0;
 }
 
-static int gfx_v12_0_kcq_init_queue(struct amdgpu_ring *ring, bool reset)
+static int gfx_v12_0_kcq_init_queue(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct v12_compute_mqd *mqd = ring->mqd_ptr;
 	int mqd_idx = ring - &adev->gfx.compute_ring[0];
 
-	if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) {
+	if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
 		memset((void *)mqd, 0, sizeof(*mqd));
 		mutex_lock(&adev->srbm_mutex);
 		soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
@@ -3485,7 +3499,7 @@ static int gfx_v12_0_kcq_resume(struct amdgpu_device *adev)
 		gfx_v12_0_cp_compute_enable(adev, true);
 
 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
-		r = gfx_v12_0_kcq_init_queue(&adev->gfx.compute_ring[i], false);
+		r = gfx_v12_0_kcq_init_queue(&adev->gfx.compute_ring[i]);
 		if (r)
 			return r;
 	}
@@ -3986,6 +4000,8 @@ static int gfx_v12_0_early_init(struct amdgpu_ip_block *ip_block)
 
 	gfx_v12_0_init_rlcg_reg_access_ctrl(adev);
 
+	amdgpu_init_rlc_reg_funcs(adev);
+
 	return gfx_v12_0_init_microcode(adev);
 }
 
@@ -5025,22 +5041,30 @@ static int gfx_v12_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
 static void gfx_v12_0_handle_priv_fault(struct amdgpu_device *adev,
 					struct amdgpu_iv_entry *entry)
 {
-	u8 me_id, pipe_id, queue_id;
-	struct amdgpu_ring *ring;
-	int i;
-
-	me_id = (entry->ring_id & 0x0c) >> 2;
-	pipe_id = (entry->ring_id & 0x03) >> 0;
-	queue_id = (entry->ring_id & 0x70) >> 4;
+	u32 doorbell_offset = entry->src_data[0] & AMDGPU_CTXID0_DOORBELL_ID_MASK;
 
+	/*
+	 * Try KQ first by ring_id; UQ as fallback. KCQ and UQ never share
+	 * a HW slot (compute_hqd_mask contract).
+	 */
 	if (!adev->gfx.disable_kq) {
+		u8 me_id, pipe_id, queue_id;
+		struct amdgpu_ring *ring;
+		int i;
+
+		me_id = (entry->ring_id & 0x0c) >> 2;
+		pipe_id = (entry->ring_id & 0x03) >> 0;
+		queue_id = (entry->ring_id & 0x70) >> 4;
+
 		switch (me_id) {
 		case 0:
 			for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
 				ring = &adev->gfx.gfx_ring[i];
 				if (ring->me == me_id && ring->pipe == pipe_id &&
-				    ring->queue == queue_id)
+				    ring->queue == queue_id) {
 					drm_sched_fault(&ring->sched);
+					return;
+				}
 			}
 			break;
 		case 1:
@@ -5048,8 +5072,10 @@ static void gfx_v12_0_handle_priv_fault(struct amdgpu_device *adev,
 			for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 				ring = &adev->gfx.compute_ring[i];
 				if (ring->me == me_id && ring->pipe == pipe_id &&
-				    ring->queue == queue_id)
+				    ring->queue == queue_id) {
 					drm_sched_fault(&ring->sched);
+					return;
+				}
 			}
 			break;
 		default:
@@ -5057,6 +5083,11 @@ static void gfx_v12_0_handle_priv_fault(struct amdgpu_device *adev,
 			break;
 		}
 	}
+
+	/* No KQ matched: HW slot is a MES-scheduled user queue. */
+	if (adev->enable_mes && doorbell_offset)
+		amdgpu_userq_process_reset_irq(adev, entry->pasid,
+					       doorbell_offset);
 }
 
 static int gfx_v12_0_priv_reg_irq(struct amdgpu_device *adev,
@@ -5261,185 +5292,14 @@ static void gfx_v12_ip_dump(struct amdgpu_ip_block *ip_block)
 	amdgpu_gfx_off_ctrl(adev, true);
 }
 
-static bool gfx_v12_pipe_reset_support(struct amdgpu_device *adev)
-{
-	/* Disable the pipe reset until the CPFW fully support it.*/
-	dev_warn_once(adev->dev, "The CPFW hasn't support pipe reset yet.\n");
-	return false;
-}
-
-static int gfx_v12_reset_gfx_pipe(struct amdgpu_ring *ring)
-{
-	struct amdgpu_device *adev = ring->adev;
-	uint32_t reset_pipe = 0, clean_pipe = 0;
-	int r;
-
-	if (!gfx_v12_pipe_reset_support(adev))
-		return -EOPNOTSUPP;
-
-	gfx_v12_0_set_safe_mode(adev, 0);
-	mutex_lock(&adev->srbm_mutex);
-	soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
-
-	switch (ring->pipe) {
-	case 0:
-		reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
-					   PFP_PIPE0_RESET, 1);
-		reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
-					   ME_PIPE0_RESET, 1);
-		clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
-					   PFP_PIPE0_RESET, 0);
-		clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
-					   ME_PIPE0_RESET, 0);
-		break;
-	case 1:
-		reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
-					   PFP_PIPE1_RESET, 1);
-		reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
-					   ME_PIPE1_RESET, 1);
-		clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
-					   PFP_PIPE1_RESET, 0);
-		clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
-					   ME_PIPE1_RESET, 0);
-		break;
-	default:
-		break;
-	}
-
-	WREG32_SOC15(GC, 0, regCP_ME_CNTL, reset_pipe);
-	WREG32_SOC15(GC, 0, regCP_ME_CNTL, clean_pipe);
-
-	r = (RREG32(SOC15_REG_OFFSET(GC, 0, regCP_GFX_RS64_INSTR_PNTR1)) << 2) -
-					RS64_FW_UC_START_ADDR_LO;
-	soc24_grbm_select(adev, 0, 0, 0, 0);
-	mutex_unlock(&adev->srbm_mutex);
-	gfx_v12_0_unset_safe_mode(adev, 0);
-
-	dev_info(adev->dev, "The ring %s pipe reset: %s\n", ring->name,
-			r == 0 ? "successfully" : "failed");
-	/* Sometimes the ME start pc counter can't cache correctly, so the
-	 * PC check only as a reference and pipe reset result rely on the
-	 * later ring test.
-	 */
-	return 0;
-}
-
 static int gfx_v12_0_reset_kgq(struct amdgpu_ring *ring,
 			       unsigned int vmid,
 			       struct amdgpu_fence *timedout_fence)
 {
 	struct amdgpu_device *adev = ring->adev;
-	bool use_mmio = false;
-	int r;
-
-	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
+	bool use_mmio = adev->gfx.me.use_mmio_for_reset;
 
-	r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, use_mmio, 0);
-	if (r) {
-		dev_warn(adev->dev, "reset via MES failed and try pipe reset %d\n", r);
-		r = gfx_v12_reset_gfx_pipe(ring);
-		if (r)
-			return r;
-	}
-
-	if (use_mmio) {
-		r = gfx_v12_0_kgq_init_queue(ring, true);
-		if (r) {
-			dev_err(adev->dev, "failed to init kgq\n");
-			return r;
-		}
-
-		r = amdgpu_mes_map_legacy_queue(adev, ring, 0);
-		if (r) {
-			dev_err(adev->dev, "failed to remap kgq\n");
-			return r;
-		}
-	}
-
-	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
-}
-
-static int gfx_v12_0_reset_compute_pipe(struct amdgpu_ring *ring)
-{
-	struct amdgpu_device *adev = ring->adev;
-	uint32_t reset_pipe = 0, clean_pipe = 0;
-	int r = 0;
-
-	if (!gfx_v12_pipe_reset_support(adev))
-		return -EOPNOTSUPP;
-
-	gfx_v12_0_set_safe_mode(adev, 0);
-	mutex_lock(&adev->srbm_mutex);
-	soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
-
-	reset_pipe = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
-	clean_pipe = reset_pipe;
-
-	if (adev->gfx.rs64_enable) {
-		switch (ring->pipe) {
-		case 0:
-			reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL,
-						   MEC_PIPE0_RESET, 1);
-			clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL,
-						   MEC_PIPE0_RESET, 0);
-			break;
-		case 1:
-			reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL,
-						   MEC_PIPE1_RESET, 1);
-			clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL,
-						   MEC_PIPE1_RESET, 0);
-			break;
-		case 2:
-			reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL,
-						   MEC_PIPE2_RESET, 1);
-			clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL,
-						   MEC_PIPE2_RESET, 0);
-			break;
-		case 3:
-			reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL,
-						   MEC_PIPE3_RESET, 1);
-			clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL,
-						   MEC_PIPE3_RESET, 0);
-			break;
-		default:
-			break;
-		}
-		WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, reset_pipe);
-		WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, clean_pipe);
-		r = (RREG32_SOC15(GC, 0, regCP_MEC_RS64_INSTR_PNTR) << 2) -
-				RS64_FW_UC_START_ADDR_LO;
-	} else {
-		switch (ring->pipe) {
-		case 0:
-			reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
-							   MEC_ME1_PIPE0_RESET, 1);
-			clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
-							   MEC_ME1_PIPE0_RESET, 0);
-			break;
-		case 1:
-			reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL,
-							   MEC_ME1_PIPE1_RESET, 1);
-			clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL,
-							   MEC_ME1_PIPE1_RESET, 0);
-			break;
-		default:
-		break;
-		}
-		WREG32_SOC15(GC, 0, regCP_MEC_CNTL, reset_pipe);
-		WREG32_SOC15(GC, 0, regCP_MEC_CNTL, clean_pipe);
-		/* Doesn't find the F32 MEC instruction pointer register, and suppose
-		 * the driver won't run into the F32 mode.
-		 */
-	}
-
-	soc24_grbm_select(adev, 0, 0, 0, 0);
-	mutex_unlock(&adev->srbm_mutex);
-	gfx_v12_0_unset_safe_mode(adev, 0);
-
-	dev_info(adev->dev, "The ring %s pipe resets: %s\n", ring->name,
-			r == 0 ? "successfully" : "failed");
-	/* Need the ring test to verify the pipe reset result.*/
-	return 0;
+	return amdgpu_gfx_mes_reset_queue(ring, vmid, timedout_fence, use_mmio);
 }
 
 static int gfx_v12_0_reset_kcq(struct amdgpu_ring *ring,
@@ -5447,30 +5307,8 @@ static int gfx_v12_0_reset_kcq(struct amdgpu_ring *ring,
 			       struct amdgpu_fence *timedout_fence)
 {
 	struct amdgpu_device *adev = ring->adev;
-	int r;
-
-	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
-
-	r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, true, 0);
-	if (r) {
-		dev_warn(adev->dev, "fail(%d) to reset kcq  and try pipe reset\n", r);
-		r = gfx_v12_0_reset_compute_pipe(ring);
-		if (r)
-			return r;
-	}
-
-	r = gfx_v12_0_kcq_init_queue(ring, true);
-	if (r) {
-		dev_err(adev->dev, "failed to init kcq\n");
-		return r;
-	}
-	r = amdgpu_mes_map_legacy_queue(adev, ring, 0);
-	if (r) {
-		dev_err(adev->dev, "failed to remap kcq\n");
-		return r;
-	}
 
-	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
+	return amdgpu_gfx_reset_mes_compute(adev, ring, timedout_fence, NULL, NULL, NULL);
 }
 
 static void gfx_v12_0_ring_begin_use(struct amdgpu_ring *ring)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
index e7e9f11b97541..1f7e243d2ad99 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
@@ -69,6 +69,127 @@ MODULE_FIRMWARE("amdgpu/gc_12_1_0_rlc_1.bin");
 	 (SH_MEM_ALIGNMENT_MODE_UNALIGNED_GFX12_1_0 << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \
 	 (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))
 
+
+static const struct amdgpu_hwip_reg_entry gc_reg_list_12_1[] = {
+	SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2),
+	SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS3),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT3),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT2),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT2),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HPD_STATUS0),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_BASE),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_BASE),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_RPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_WPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_CMD_BUFSZ),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_LO),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BUFSZ),
+	SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS_2),
+	SOC15_REG_ENTRY_STR(GC, 0, regPA_CL_CNTL_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, regSQC_CACHES),
+	SOC15_REG_ENTRY_STR(GC, 0, regSQG_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL2),
+	SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS_LO32),
+	SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS_HI32),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_CNTL),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_INSTR_PNTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_INSTR_PNTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_INSTR_PNTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_RS64_INSTR_PNTR0),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_RS64_INSTR_PNTR1),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_RS64_INSTR_PNTR),
+	/* cp header registers */
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP),
+	/* SE status registers */
+	SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0),
+	SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1),
+};
+
+static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_12_1[] = {
+	/* compute registers */
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_OFFSET),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_WG_STATE_OFFSET),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_STATUS),
+	/* cp header registers */
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
+	SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP),
+};
+
 static void gfx_v12_1_xcc_disable_gpa_mode(struct amdgpu_device *adev, int xcc_id);
 static void gfx_v12_1_set_ring_funcs(struct amdgpu_device *adev);
 static void gfx_v12_1_set_irq_funcs(struct amdgpu_device *adev);
@@ -320,7 +441,7 @@ static int gfx_v12_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 
 	memset(&ib, 0, sizeof(ib));
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -361,7 +482,7 @@ static int gfx_v12_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err1:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -1149,6 +1270,155 @@ static int gfx_v12_1_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
 	return 0;
 }
 
+static void gfx_v12_1_alloc_ip_dump(struct amdgpu_device *adev)
+{
+	uint32_t reg_count = ARRAY_SIZE(gc_reg_list_12_1);
+	uint32_t *ptr, inst, num_xcc;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+
+	ptr = kcalloc(reg_count * num_xcc, sizeof(uint32_t), GFP_KERNEL);
+	if (!ptr) {
+		DRM_ERROR("Failed to allocate memory for GFX IP Dump\n");
+		adev->gfx.ip_dump_core = NULL;
+	} else {
+		adev->gfx.ip_dump_core = ptr;
+	}
+
+	/* Allocate memory for compute queue registers for all the instances */
+	reg_count = ARRAY_SIZE(gc_cp_reg_list_12_1);
+	inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
+		adev->gfx.mec.num_queue_per_pipe;
+
+	ptr = kcalloc(reg_count * inst * num_xcc, sizeof(uint32_t), GFP_KERNEL);
+	if (!ptr) {
+		DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n");
+		adev->gfx.ip_dump_compute_queues = NULL;
+	} else {
+		adev->gfx.ip_dump_compute_queues = ptr;
+	}
+}
+
+static void gfx_v12_1_ip_print(struct amdgpu_ip_block *ip_block,
+			       struct drm_printer *p)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	uint32_t i, j, k;
+	uint32_t xcc_id, xcc_offset, inst_offset;
+	uint32_t num_xcc, reg, num_inst;
+	uint32_t reg_count = ARRAY_SIZE(gc_reg_list_12_1);
+
+	if (!adev->gfx.ip_dump_core)
+		return;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	drm_printf(p, "Number of Instances:%d\n", num_xcc);
+	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
+		xcc_offset = xcc_id * reg_count;
+		drm_printf(p, "\nInstance id:%d\n", xcc_id);
+		for (i = 0; i < reg_count; i++)
+			drm_printf(p, "%-50s \t 0x%08x\n",
+				   gc_reg_list_12_1[i].reg_name,
+				   adev->gfx.ip_dump_core[xcc_offset + i]);
+	}
+
+	/* print compute queue registers for all instances */
+	if (!adev->gfx.ip_dump_compute_queues)
+		return;
+
+	reg_count = ARRAY_SIZE(gc_cp_reg_list_12_1);
+	drm_printf(p, "\nnum_xcc: %d num_mec: %d num_pipe: %d num_queue: %d\n",
+		   num_xcc,
+		   adev->gfx.mec.num_mec,
+		   adev->gfx.mec.num_pipe_per_mec,
+		   adev->gfx.mec.num_queue_per_pipe);
+
+	num_inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
+		adev->gfx.mec.num_queue_per_pipe;
+	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
+		xcc_offset = xcc_id * reg_count * num_inst;
+		inst_offset = 0;
+		for (i = 0; i < adev->gfx.mec.num_mec; i++) {
+			for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
+				for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
+					drm_printf(p,
+						   "\nxcc:%d mec:%d, pipe:%d, queue:%d\n",
+						    xcc_id, i, j, k);
+					for (reg = 0; reg < reg_count; reg++) {
+						drm_printf(p,
+							   "%-50s \t 0x%08x\n",
+							   gc_cp_reg_list_12_1[reg].reg_name,
+							   adev->gfx.ip_dump_compute_queues
+							   [xcc_offset + inst_offset +
+							    reg]);
+					}
+					inst_offset += reg_count;
+				}
+			}
+		}
+	}
+}
+
+static void gfx_v12_1_ip_dump(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	uint32_t i, j, k;
+	uint32_t num_xcc, reg, num_inst;
+	uint32_t xcc_id, xcc_offset, inst_offset;
+	uint32_t reg_count = ARRAY_SIZE(gc_reg_list_12_1);
+
+	if (!adev->gfx.ip_dump_core)
+		return;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+
+	amdgpu_gfx_off_ctrl(adev, false);
+	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
+		xcc_offset = xcc_id * reg_count;
+		for (i = 0; i < reg_count; i++)
+			adev->gfx.ip_dump_core[xcc_offset + i] =
+				RREG32(SOC15_REG_ENTRY_OFFSET_INST(gc_reg_list_12_1[i],
+								   GET_INST(GC, xcc_id)));
+	}
+	amdgpu_gfx_off_ctrl(adev, true);
+
+	/* dump compute queue registers for all instances */
+	if (!adev->gfx.ip_dump_compute_queues)
+		return;
+
+	num_inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
+		adev->gfx.mec.num_queue_per_pipe;
+	reg_count = ARRAY_SIZE(gc_cp_reg_list_12_1);
+	amdgpu_gfx_off_ctrl(adev, false);
+	mutex_lock(&adev->srbm_mutex);
+	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
+		xcc_offset = xcc_id * reg_count * num_inst;
+		inst_offset = 0;
+		for (i = 0; i < adev->gfx.mec.num_mec; i++) {
+			for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) {
+				for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) {
+					/* ME0 is for GFX so start from 1 for CP */
+					soc_v1_0_grbm_select(adev, 1 + i, j, k, 0,
+							     GET_INST(GC, xcc_id));
+
+					for (reg = 0; reg < reg_count; reg++) {
+						adev->gfx.ip_dump_compute_queues
+							[xcc_offset +
+							 inst_offset + reg] =
+							RREG32(SOC15_REG_ENTRY_OFFSET_INST(
+								       gc_cp_reg_list_12_1[reg],
+								       GET_INST(GC, xcc_id)));
+					}
+					inst_offset += reg_count;
+				}
+			}
+		}
+	}
+	soc_v1_0_grbm_select(adev, 0, 0, 0, 0, 0);
+	mutex_unlock(&adev->srbm_mutex);
+	amdgpu_gfx_off_ctrl(adev, true);
+}
+
 static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
 {
 	uint16_t major_ver, minor_ver;
@@ -1287,6 +1557,10 @@ static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
 	if (r)
 		return r;
 
+	gfx_v12_1_alloc_ip_dump(adev);
+
+	mutex_init(&adev->gfx.mec.reset_mutex);
+
 	return 0;
 }
 
@@ -1324,6 +1598,9 @@ static int gfx_v12_1_sw_fini(struct amdgpu_ip_block *ip_block)
 	gfx_v12_1_free_microcode(adev);
 	amdgpu_gfx_sysfs_fini(adev);
 
+	kfree(adev->gfx.ip_dump_core);
+	kfree(adev->gfx.ip_dump_compute_queues);
+
 	return 0;
 }
 
@@ -3004,6 +3281,8 @@ static int gfx_v12_1_early_init(struct amdgpu_ip_block *ip_block)
 
 	gfx_v12_1_init_rlcg_reg_access_ctrl(adev);
 
+	amdgpu_init_rlc_reg_funcs(adev);
+
 	return gfx_v12_1_init_microcode(adev);
 }
 
@@ -3908,6 +4187,8 @@ static const struct amd_ip_funcs gfx_v12_1_ip_funcs = {
 	.set_clockgating_state = gfx_v12_1_set_clockgating_state,
 	.set_powergating_state = gfx_v12_1_set_powergating_state,
 	.get_clockgating_state = gfx_v12_1_get_clockgating_state,
+	.dump_ip_state = gfx_v12_1_ip_dump,
+	.print_ip_state = gfx_v12_1_ip_print,
 };
 
 static const struct amdgpu_ring_funcs gfx_v12_1_ring_funcs_compute = {
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index 70ba81e6b4d43..9e0840df88497 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -876,7 +876,7 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	uint32_t tmp;
 	long r;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -917,7 +917,7 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err1:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -1487,7 +1487,14 @@ static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
 
 	/* bail if the compute ring is not ready */
 	if (!ring->sched.ready)
-		return 0;
+		return -EBUSY;
+
+	if (amdgpu_in_reset(adev)) {
+		/* Set preempt condition to execute IB */
+		amdgpu_ring_set_preempt_cond_exec(ring, true);
+		/* Flush HDP cache so the GPU can see the updated COND_EXEC value */
+		amdgpu_device_flush_hdp(adev, NULL);
+	}
 
 	tmp = RREG32(mmGB_EDC_MODE);
 	WREG32(mmGB_EDC_MODE, 0);
@@ -2028,6 +2035,11 @@ static int gfx_v8_0_sw_init(struct amdgpu_ip_block *ip_block)
 	adev->gfx.compute_supported_reset =
 		amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]);
 
+	if (!amdgpu_sriov_vf(adev) && !adev->debug_disable_ip_block_soft_reset) {
+		adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_IP_BLOCK_SOFT_RESET;
+		adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_IP_BLOCK_SOFT_RESET;
+	}
+
 	return 0;
 }
 
@@ -4703,12 +4715,14 @@ static int gfx_v8_0_cp_test_all_rings(struct amdgpu_device *adev)
 	if (r)
 		return r;
 
+	r = 0;
+
 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 		ring = &adev->gfx.compute_ring[i];
-		amdgpu_ring_test_helper(ring);
+		r |= amdgpu_ring_test_helper(ring);
 	}
 
-	return 0;
+	return r;
 }
 
 static int gfx_v8_0_cp_resume(struct amdgpu_device *adev)
@@ -4868,14 +4882,12 @@ static int gfx_v8_0_hw_fini(struct amdgpu_ip_block *ip_block)
 	}
 
 	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
-	if (!gfx_v8_0_wait_for_idle(ip_block))
-		gfx_v8_0_cp_enable(adev, false);
-	else
+	if (!amdgpu_in_reset(adev) && gfx_v8_0_wait_for_idle(ip_block))
 		pr_err("cp is busy, skip halt cp\n");
-	if (!gfx_v8_0_wait_for_rlc_idle(adev))
-		adev->gfx.rlc.funcs->stop(adev);
-	else
-		pr_err("rlc is busy, skip halt rlc\n");
+	if (!amdgpu_in_reset(adev) && gfx_v8_0_wait_for_rlc_idle(adev))
+		pr_err("rlc is busy\n");
+	gfx_v8_0_cp_enable(adev, false);
+	adev->gfx.rlc.funcs->stop(adev);
 	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
 
 	return 0;
@@ -4891,128 +4903,49 @@ static int gfx_v8_0_resume(struct amdgpu_ip_block *ip_block)
 	return gfx_v8_0_hw_init(ip_block);
 }
 
-static bool gfx_v8_0_check_soft_reset(struct amdgpu_ip_block *ip_block)
+static int gfx_v8_0_soft_reset(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
 	u32 tmp;
+	int i;
+	int r;
 
-	/* GRBM_STATUS */
-	tmp = RREG32(mmGRBM_STATUS);
-	if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
-		   GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
-		   GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
-		   GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
-		   GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
-		   GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK |
-		   GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
-		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
-						GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
-		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
-						GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
-		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
-						SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);
-	}
-
-	/* GRBM_STATUS2 */
-	tmp = RREG32(mmGRBM_STATUS2);
-	if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
-		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
-						GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
-
-	if (REG_GET_FIELD(tmp, GRBM_STATUS2, CPF_BUSY) ||
-	    REG_GET_FIELD(tmp, GRBM_STATUS2, CPC_BUSY) ||
-	    REG_GET_FIELD(tmp, GRBM_STATUS2, CPG_BUSY)) {
-		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
-						SOFT_RESET_CPF, 1);
-		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
-						SOFT_RESET_CPC, 1);
-		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
-						SOFT_RESET_CPG, 1);
-		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET,
-						SOFT_RESET_GRBM, 1);
-	}
-
-	/* SRBM_STATUS */
-	tmp = RREG32(mmSRBM_STATUS);
-	if (REG_GET_FIELD(tmp, SRBM_STATUS, GRBM_RQ_PENDING))
-		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
-						SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);
-	if (REG_GET_FIELD(tmp, SRBM_STATUS, SEM_BUSY))
-		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
-						SRBM_SOFT_RESET, SOFT_RESET_SEM, 1);
-
-	if (grbm_soft_reset || srbm_soft_reset) {
-		adev->gfx.grbm_soft_reset = grbm_soft_reset;
-		adev->gfx.srbm_soft_reset = srbm_soft_reset;
-		return true;
-	} else {
-		adev->gfx.grbm_soft_reset = 0;
-		adev->gfx.srbm_soft_reset = 0;
-		return false;
-	}
-}
-
-static int gfx_v8_0_pre_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 grbm_soft_reset = 0;
-
-	if ((!adev->gfx.grbm_soft_reset) &&
-	    (!adev->gfx.srbm_soft_reset))
-		return 0;
-
-	grbm_soft_reset = adev->gfx.grbm_soft_reset;
-
-	/* stop the rlc */
-	adev->gfx.rlc.funcs->stop(adev);
+	grbm_soft_reset =
+		REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1) |
+		REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_GFX, 1) |
+		REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CP, 1) |
+		REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CPF, 1) |
+		REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CPC, 1) |
+		REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CPG, 1);
 
-	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
-	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))
-		/* Disable GFX parsing/prefetching */
-		gfx_v8_0_cp_gfx_enable(adev, false);
+	srbm_soft_reset =
+		REG_SET_FIELD(0, SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1) |
+		REG_SET_FIELD(0, SRBM_SOFT_RESET, SOFT_RESET_SEM, 1);
 
-	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
-	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||
-	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||
-	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {
-		int i;
+	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
+		struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
 
-		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
-			struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
+		mutex_lock(&adev->srbm_mutex);
+		vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
+		gfx_v8_0_deactivate_hqd(adev, 2);
+		vi_srbm_select(adev, 0, 0, 0, 0);
+		mutex_unlock(&adev->srbm_mutex);
 
-			mutex_lock(&adev->srbm_mutex);
-			vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
-			gfx_v8_0_deactivate_hqd(adev, 2);
-			vi_srbm_select(adev, 0, 0, 0, 0);
-			mutex_unlock(&adev->srbm_mutex);
-		}
-		/* Disable MEC parsing/prefetching */
-		gfx_v8_0_cp_compute_enable(adev, false);
+		udelay(50);
 	}
 
-	return 0;
-}
-
-static int gfx_v8_0_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
-	u32 tmp;
-
-	if ((!adev->gfx.grbm_soft_reset) &&
-	    (!adev->gfx.srbm_soft_reset))
-		return 0;
-
-	grbm_soft_reset = adev->gfx.grbm_soft_reset;
-	srbm_soft_reset = adev->gfx.srbm_soft_reset;
+	ip_block->version->funcs->set_clockgating_state(ip_block, AMD_CG_STATE_UNGATE);
+	ip_block->version->funcs->set_powergating_state(ip_block, AMD_PG_STATE_UNGATE);
+	ip_block->version->funcs->suspend(ip_block);
 
 	if (grbm_soft_reset || srbm_soft_reset) {
 		tmp = RREG32(mmGMCON_DEBUG);
 		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 1);
 		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 1);
 		WREG32(mmGMCON_DEBUG, tmp);
-		udelay(50);
+
+		udelay(100);
 	}
 
 	if (grbm_soft_reset) {
@@ -5022,11 +4955,13 @@ static int gfx_v8_0_soft_reset(struct amdgpu_ip_block *ip_block)
 		WREG32(mmGRBM_SOFT_RESET, tmp);
 		tmp = RREG32(mmGRBM_SOFT_RESET);
 
-		udelay(50);
+		udelay(100);
 
 		tmp &= ~grbm_soft_reset;
 		WREG32(mmGRBM_SOFT_RESET, tmp);
 		tmp = RREG32(mmGRBM_SOFT_RESET);
+
+		udelay(100);
 	}
 
 	if (srbm_soft_reset) {
@@ -5036,11 +4971,13 @@ static int gfx_v8_0_soft_reset(struct amdgpu_ip_block *ip_block)
 		WREG32(mmSRBM_SOFT_RESET, tmp);
 		tmp = RREG32(mmSRBM_SOFT_RESET);
 
-		udelay(50);
+		udelay(100);
 
 		tmp &= ~srbm_soft_reset;
 		WREG32(mmSRBM_SOFT_RESET, tmp);
 		tmp = RREG32(mmSRBM_SOFT_RESET);
+
+		udelay(100);
 	}
 
 	if (grbm_soft_reset || srbm_soft_reset) {
@@ -5051,48 +4988,15 @@ static int gfx_v8_0_soft_reset(struct amdgpu_ip_block *ip_block)
 	}
 
 	/* Wait a little for things to settle down */
-	udelay(50);
+	udelay(100);
 
-	return 0;
-}
-
-static int gfx_v8_0_post_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 grbm_soft_reset = 0;
-
-	if ((!adev->gfx.grbm_soft_reset) &&
-	    (!adev->gfx.srbm_soft_reset))
-		return 0;
-
-	grbm_soft_reset = adev->gfx.grbm_soft_reset;
-
-	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
-	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||
-	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||
-	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {
-		int i;
-
-		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
-			struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
-
-			mutex_lock(&adev->srbm_mutex);
-			vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
-			gfx_v8_0_deactivate_hqd(adev, 2);
-			vi_srbm_select(adev, 0, 0, 0, 0);
-			mutex_unlock(&adev->srbm_mutex);
-		}
-		gfx_v8_0_kiq_resume(adev);
-		gfx_v8_0_kcq_resume(adev);
-	}
-
-	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
-	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))
-		gfx_v8_0_cp_gfx_resume(adev);
+	r = ip_block->version->funcs->resume(ip_block);
+	r |= ip_block->version->funcs->late_init(ip_block);
+	if (r)
+		return r;
 
-	gfx_v8_0_cp_test_all_rings(adev);
-
-	adev->gfx.rlc.funcs->start(adev);
+	ip_block->version->funcs->set_clockgating_state(ip_block, AMD_CG_STATE_GATE);
+	ip_block->version->funcs->set_powergating_state(ip_block, AMD_PG_STATE_GATE);
 
 	return 0;
 }
@@ -6859,10 +6763,7 @@ static const struct amd_ip_funcs gfx_v8_0_ip_funcs = {
 	.resume = gfx_v8_0_resume,
 	.is_idle = gfx_v8_0_is_idle,
 	.wait_for_idle = gfx_v8_0_wait_for_idle,
-	.check_soft_reset = gfx_v8_0_check_soft_reset,
-	.pre_soft_reset = gfx_v8_0_pre_soft_reset,
 	.soft_reset = gfx_v8_0_soft_reset,
-	.post_soft_reset = gfx_v8_0_post_soft_reset,
 	.set_clockgating_state = gfx_v8_0_set_clockgating_state,
 	.set_powergating_state = gfx_v8_0_set_powergating_state,
 	.get_clockgating_state = gfx_v8_0_get_clockgating_state,
@@ -6923,10 +6824,12 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
 	.get_wptr = gfx_v8_0_ring_get_wptr_compute,
 	.set_wptr = gfx_v8_0_ring_set_wptr_compute,
 	.emit_frame_size =
+		5 + /* gfx_v8_0_ring_emit_init_cond_exec (from amdgpu_ib_schedule) */
 		20 + /* gfx_v8_0_ring_emit_gds_switch */
 		7 + /* gfx_v8_0_ring_emit_hdp_flush */
 		5 + /* hdp_invalidate */
 		7 + /* gfx_v8_0_ring_emit_pipeline_sync */
+		5 + /* gfx_v8_0_ring_emit_init_cond_exec (from amdgpu_vm_flush) */
 		VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v8_0_ring_emit_vm_flush */
 		7 + 7 + 7 + /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */
 		7 + /* gfx_v8_0_emit_mem_sync_compute */
@@ -6947,6 +6850,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
 	.soft_recovery = gfx_v8_0_ring_soft_recovery,
 	.emit_mem_sync = gfx_v8_0_emit_mem_sync_compute,
 	.emit_wave_limit = gfx_v8_0_emit_wave_limit,
+	.init_cond_exec = gfx_v8_0_ring_emit_init_cond_exec,
 };
 
 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = {
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 3370f542e9909..1cad6dc9476e0 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -1232,7 +1232,7 @@ static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	uint32_t tmp;
 	long r;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -1273,7 +1273,7 @@ static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err1:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -2442,12 +2442,12 @@ static int gfx_v9_0_sw_init(struct amdgpu_ip_block *ip_block)
 		return -EINVAL;
 	}
 
-	gfx_v9_0_alloc_ip_dump(adev);
-
 	r = amdgpu_gfx_sysfs_init(adev);
 	if (r)
 		return r;
 
+	gfx_v9_0_alloc_ip_dump(adev);
+
 	return 0;
 }
 
@@ -4267,7 +4267,7 @@ static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
 	BUG_ON(!ring->funcs->emit_rreg);
 
 	spin_lock_irqsave(&kiq->ring_lock, flags);
-	if (amdgpu_device_wb_get(adev, &reg_val_offs)) {
+	if (amdgpu_wb_get(adev, &reg_val_offs)) {
 		pr_err("critical bug! too many kiq readers\n");
 		goto failed_unlock;
 	}
@@ -4315,7 +4315,7 @@ static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
 	mb();
 	value = (uint64_t)adev->wb.wb[reg_val_offs] |
 		(uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL;
-	amdgpu_device_wb_free(adev, reg_val_offs);
+	amdgpu_wb_free(adev, reg_val_offs);
 	return value;
 
 failed_undo:
@@ -4324,7 +4324,7 @@ static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
 	spin_unlock_irqrestore(&kiq->ring_lock, flags);
 failed_kiq_read:
 	if (reg_val_offs)
-		amdgpu_device_wb_free(adev, reg_val_offs);
+		amdgpu_wb_free(adev, reg_val_offs);
 	pr_err("failed to read gpu clock\n");
 	return ~0;
 }
@@ -4875,6 +4875,8 @@ static int gfx_v9_0_early_init(struct amdgpu_ip_block *ip_block)
 	/* init rlcg reg access ctrl */
 	gfx_v9_0_init_rlcg_reg_access_ctrl(adev);
 
+	amdgpu_init_rlc_reg_funcs(adev);
+
 	return gfx_v9_0_init_microcode(adev);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
index 2a36647b975a9..9512fef81d849 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
@@ -39,7 +39,6 @@
 #include "gfx_v9_4_3.h"
 #include "gfx_v9_4_3_cleaner_shader.h"
 #include "amdgpu_xcp.h"
-#include "amdgpu_aca.h"
 
 MODULE_FIRMWARE("amdgpu/gc_9_4_3_mec.bin");
 MODULE_FIRMWARE("amdgpu/gc_9_4_4_mec.bin");
@@ -459,7 +458,7 @@ static int gfx_v9_4_3_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	uint32_t tmp;
 	long r;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -500,7 +499,7 @@ static int gfx_v9_4_3_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err1:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -851,73 +850,6 @@ static const struct amdgpu_gfx_funcs gfx_v9_4_3_gfx_funcs = {
 	.get_hdp_flush_mask = &amdgpu_gfx_get_hdp_flush_mask,
 };
 
-static int gfx_v9_4_3_aca_bank_parser(struct aca_handle *handle,
-				      struct aca_bank *bank, enum aca_smu_type type,
-				      void *data)
-{
-	struct aca_bank_info info;
-	u64 misc0;
-	u32 instlo;
-	int ret;
-
-	ret = aca_bank_info_decode(bank, &info);
-	if (ret)
-		return ret;
-
-	/* NOTE: overwrite info.die_id with xcd id for gfx */
-	instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
-	instlo &= GENMASK(31, 1);
-	info.die_id = instlo == mmSMNAID_XCD0_MCA_SMU ? 0 : 1;
-
-	misc0 = bank->regs[ACA_REG_IDX_MISC0];
-
-	switch (type) {
-	case ACA_SMU_TYPE_UE:
-		bank->aca_err_type = ACA_ERROR_TYPE_UE;
-		ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, 1ULL);
-		break;
-	case ACA_SMU_TYPE_CE:
-		bank->aca_err_type = ACA_ERROR_TYPE_CE;
-		ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type,
-						     ACA_REG__MISC0__ERRCNT(misc0));
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-static bool gfx_v9_4_3_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
-					 enum aca_smu_type type, void *data)
-{
-	u32 instlo;
-
-	instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
-	instlo &= GENMASK(31, 1);
-	switch (instlo) {
-	case mmSMNAID_XCD0_MCA_SMU:
-	case mmSMNAID_XCD1_MCA_SMU:
-	case mmSMNXCD_XCD0_MCA_SMU:
-		return true;
-	default:
-		break;
-	}
-
-	return false;
-}
-
-static const struct aca_bank_ops gfx_v9_4_3_aca_bank_ops = {
-	.aca_bank_parser = gfx_v9_4_3_aca_bank_parser,
-	.aca_bank_is_valid = gfx_v9_4_3_aca_bank_is_valid,
-};
-
-static const struct aca_info gfx_v9_4_3_aca_info = {
-	.hwip = ACA_HWIP_TYPE_SMU,
-	.mask = ACA_ERROR_UE_MASK | ACA_ERROR_CE_MASK,
-	.bank_ops = &gfx_v9_4_3_aca_bank_ops,
-};
-
 static int gfx_v9_4_3_gpu_early_init(struct amdgpu_device *adev)
 {
 	adev->gfx.funcs = &gfx_v9_4_3_gfx_funcs;
@@ -1107,22 +1039,24 @@ static int gfx_v9_4_3_sw_init(struct amdgpu_ip_block *ip_block)
 	/* set up the compute queues - allocate horizontally across pipes */
 	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
 		ring_id = 0;
-		for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
-			for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
-				for (k = 0; k < adev->gfx.mec.num_pipe_per_mec;
-				     k++) {
-					if (!amdgpu_gfx_is_mec_queue_enabled(
-							adev, xcc_id, i, k, j))
-						continue;
-
-					r = gfx_v9_4_3_compute_ring_init(adev,
-								       ring_id,
-								       xcc_id,
-								       i, k, j);
-					if (r)
-						return r;
-
-					ring_id++;
+		if (!adev->gfx.disable_kq) {
+			for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
+				for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
+					for (k = 0; k < adev->gfx.mec.num_pipe_per_mec;
+					     k++) {
+						if (!amdgpu_gfx_is_mec_queue_enabled(
+							    adev, xcc_id, i, k, j))
+							continue;
+
+						r = gfx_v9_4_3_compute_ring_init(adev,
+										 ring_id,
+										 xcc_id,
+										 i, k, j);
+						if (r)
+							return r;
+
+						ring_id++;
+					}
 				}
 			}
 		}
@@ -2350,6 +2284,65 @@ static void gfx_v9_4_3_xcc_fini(struct amdgpu_device *adev, int xcc_id)
 	gfx_v9_4_3_xcc_cp_compute_enable(adev, false, xcc_id);
 }
 
+static int gfx_v9_4_3_set_userq_eop_interrupts(struct amdgpu_device *adev,
+					       bool enable)
+{
+	int num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	unsigned int irq_type;
+	int m, p, xcc_id, r;
+
+	if (adev->gfx.disable_kq) {
+		for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
+			for (m = 0; m < adev->gfx.mec.num_mec; ++m) {
+				for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) {
+					irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
+						+ (m * adev->gfx.mec.num_pipe_per_mec)
+						+ p;
+
+					if (enable)
+						r = amdgpu_irq_get(adev, &adev->gfx.eop_irq,
+								   irq_type);
+					else
+						r = amdgpu_irq_put(adev, &adev->gfx.eop_irq,
+								   irq_type);
+					if (r) {
+						if (!enable)
+							return r;
+						goto err_compute;
+					}
+				}
+			}
+		}
+	}
+
+	return 0;
+
+err_compute:
+	for (p--; p >= 0; p--) {
+		irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
+			+ (m * adev->gfx.mec.num_pipe_per_mec) + p;
+		amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+	}
+	for (m--; m >= 0; m--) {
+		for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) {
+			irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
+				+ (m * adev->gfx.mec.num_pipe_per_mec) + p;
+			amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+		}
+	}
+	for (xcc_id--; xcc_id >= 0; xcc_id--) {
+		for (m = adev->gfx.mec.num_mec - 1; m >= 0; m--) {
+			for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) {
+				irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
+					+ (m * adev->gfx.mec.num_pipe_per_mec) + p;
+				amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+			}
+		}
+	}
+
+	return r;
+}
+
 static int gfx_v9_4_3_hw_init(struct amdgpu_ip_block *ip_block)
 {
 	int r;
@@ -2382,9 +2375,14 @@ static int gfx_v9_4_3_hw_init(struct amdgpu_ip_block *ip_block)
 	r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
 	if (r)
 		goto err_bad_op;
+	r = gfx_v9_4_3_set_userq_eop_interrupts(adev, true);
+	if (r)
+		goto err_bad_eop;
 
 	return 0;
 
+err_bad_eop:
+	amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
 err_bad_op:
 	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
 err_priv_inst:
@@ -2467,6 +2465,7 @@ static int gfx_v9_4_3_hw_fini(struct amdgpu_ip_block *ip_block)
 	amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
 	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
 	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
+	gfx_v9_4_3_set_userq_eop_interrupts(adev, false);
 
 	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
 	for (i = 0; i < num_xcc; i++) {
@@ -2612,8 +2611,24 @@ static int gfx_v9_4_3_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
-	adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
-					  AMDGPU_MAX_COMPUTE_RINGS);
+	switch (amdgpu_user_queue) {
+	case -1:
+	case 0:
+	default:
+		adev->gfx.disable_kq = false;
+		adev->gfx.disable_uq = true;
+		break;
+	case 2:
+		adev->gfx.disable_kq = true;
+		adev->gfx.disable_uq = true;
+		break;
+	}
+
+	if (adev->gfx.disable_kq)
+		adev->gfx.num_compute_rings = 0;
+	else
+		adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
+						  AMDGPU_MAX_COMPUTE_RINGS);
 	gfx_v9_4_3_set_kiq_pm4_funcs(adev);
 	gfx_v9_4_3_set_ring_funcs(adev);
 	gfx_v9_4_3_set_irq_funcs(adev);
@@ -2623,6 +2638,8 @@ static int gfx_v9_4_3_early_init(struct amdgpu_ip_block *ip_block)
 	/* init rlcg reg access ctrl */
 	gfx_v9_4_3_init_rlcg_reg_access_ctrl(adev);
 
+	amdgpu_init_rlc_reg_funcs(adev);
+
 	return gfx_v9_4_3_init_microcode(adev);
 }
 
@@ -3709,872 +3726,6 @@ static int gfx_v9_4_3_reset_kcq(struct amdgpu_ring *ring,
 	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
 }
 
-enum amdgpu_gfx_cp_ras_mem_id {
-	AMDGPU_GFX_CP_MEM1 = 1,
-	AMDGPU_GFX_CP_MEM2,
-	AMDGPU_GFX_CP_MEM3,
-	AMDGPU_GFX_CP_MEM4,
-	AMDGPU_GFX_CP_MEM5,
-};
-
-enum amdgpu_gfx_gcea_ras_mem_id {
-	AMDGPU_GFX_GCEA_IOWR_CMDMEM = 4,
-	AMDGPU_GFX_GCEA_IORD_CMDMEM,
-	AMDGPU_GFX_GCEA_GMIWR_CMDMEM,
-	AMDGPU_GFX_GCEA_GMIRD_CMDMEM,
-	AMDGPU_GFX_GCEA_DRAMWR_CMDMEM,
-	AMDGPU_GFX_GCEA_DRAMRD_CMDMEM,
-	AMDGPU_GFX_GCEA_MAM_DMEM0,
-	AMDGPU_GFX_GCEA_MAM_DMEM1,
-	AMDGPU_GFX_GCEA_MAM_DMEM2,
-	AMDGPU_GFX_GCEA_MAM_DMEM3,
-	AMDGPU_GFX_GCEA_MAM_AMEM0,
-	AMDGPU_GFX_GCEA_MAM_AMEM1,
-	AMDGPU_GFX_GCEA_MAM_AMEM2,
-	AMDGPU_GFX_GCEA_MAM_AMEM3,
-	AMDGPU_GFX_GCEA_MAM_AFLUSH_BUFFER,
-	AMDGPU_GFX_GCEA_WRET_TAGMEM,
-	AMDGPU_GFX_GCEA_RRET_TAGMEM,
-	AMDGPU_GFX_GCEA_IOWR_DATAMEM,
-	AMDGPU_GFX_GCEA_GMIWR_DATAMEM,
-	AMDGPU_GFX_GCEA_DRAM_DATAMEM,
-};
-
-enum amdgpu_gfx_gc_cane_ras_mem_id {
-	AMDGPU_GFX_GC_CANE_MEM0 = 0,
-};
-
-enum amdgpu_gfx_gcutcl2_ras_mem_id {
-	AMDGPU_GFX_GCUTCL2_MEM2P512X95 = 160,
-};
-
-enum amdgpu_gfx_gds_ras_mem_id {
-	AMDGPU_GFX_GDS_MEM0 = 0,
-};
-
-enum amdgpu_gfx_lds_ras_mem_id {
-	AMDGPU_GFX_LDS_BANK0 = 0,
-	AMDGPU_GFX_LDS_BANK1,
-	AMDGPU_GFX_LDS_BANK2,
-	AMDGPU_GFX_LDS_BANK3,
-	AMDGPU_GFX_LDS_BANK4,
-	AMDGPU_GFX_LDS_BANK5,
-	AMDGPU_GFX_LDS_BANK6,
-	AMDGPU_GFX_LDS_BANK7,
-	AMDGPU_GFX_LDS_BANK8,
-	AMDGPU_GFX_LDS_BANK9,
-	AMDGPU_GFX_LDS_BANK10,
-	AMDGPU_GFX_LDS_BANK11,
-	AMDGPU_GFX_LDS_BANK12,
-	AMDGPU_GFX_LDS_BANK13,
-	AMDGPU_GFX_LDS_BANK14,
-	AMDGPU_GFX_LDS_BANK15,
-	AMDGPU_GFX_LDS_BANK16,
-	AMDGPU_GFX_LDS_BANK17,
-	AMDGPU_GFX_LDS_BANK18,
-	AMDGPU_GFX_LDS_BANK19,
-	AMDGPU_GFX_LDS_BANK20,
-	AMDGPU_GFX_LDS_BANK21,
-	AMDGPU_GFX_LDS_BANK22,
-	AMDGPU_GFX_LDS_BANK23,
-	AMDGPU_GFX_LDS_BANK24,
-	AMDGPU_GFX_LDS_BANK25,
-	AMDGPU_GFX_LDS_BANK26,
-	AMDGPU_GFX_LDS_BANK27,
-	AMDGPU_GFX_LDS_BANK28,
-	AMDGPU_GFX_LDS_BANK29,
-	AMDGPU_GFX_LDS_BANK30,
-	AMDGPU_GFX_LDS_BANK31,
-	AMDGPU_GFX_LDS_SP_BUFFER_A,
-	AMDGPU_GFX_LDS_SP_BUFFER_B,
-};
-
-enum amdgpu_gfx_rlc_ras_mem_id {
-	AMDGPU_GFX_RLC_GPMF32 = 1,
-	AMDGPU_GFX_RLC_RLCVF32,
-	AMDGPU_GFX_RLC_SCRATCH,
-	AMDGPU_GFX_RLC_SRM_ARAM,
-	AMDGPU_GFX_RLC_SRM_DRAM,
-	AMDGPU_GFX_RLC_TCTAG,
-	AMDGPU_GFX_RLC_SPM_SE,
-	AMDGPU_GFX_RLC_SPM_GRBMT,
-};
-
-enum amdgpu_gfx_sp_ras_mem_id {
-	AMDGPU_GFX_SP_SIMDID0 = 0,
-};
-
-enum amdgpu_gfx_spi_ras_mem_id {
-	AMDGPU_GFX_SPI_MEM0 = 0,
-	AMDGPU_GFX_SPI_MEM1,
-	AMDGPU_GFX_SPI_MEM2,
-	AMDGPU_GFX_SPI_MEM3,
-};
-
-enum amdgpu_gfx_sqc_ras_mem_id {
-	AMDGPU_GFX_SQC_INST_CACHE_A = 100,
-	AMDGPU_GFX_SQC_INST_CACHE_B = 101,
-	AMDGPU_GFX_SQC_INST_CACHE_TAG_A = 102,
-	AMDGPU_GFX_SQC_INST_CACHE_TAG_B = 103,
-	AMDGPU_GFX_SQC_INST_CACHE_MISS_FIFO_A = 104,
-	AMDGPU_GFX_SQC_INST_CACHE_MISS_FIFO_B = 105,
-	AMDGPU_GFX_SQC_INST_CACHE_GATCL1_MISS_FIFO_A = 106,
-	AMDGPU_GFX_SQC_INST_CACHE_GATCL1_MISS_FIFO_B = 107,
-	AMDGPU_GFX_SQC_DATA_CACHE_A = 200,
-	AMDGPU_GFX_SQC_DATA_CACHE_B = 201,
-	AMDGPU_GFX_SQC_DATA_CACHE_TAG_A = 202,
-	AMDGPU_GFX_SQC_DATA_CACHE_TAG_B = 203,
-	AMDGPU_GFX_SQC_DATA_CACHE_MISS_FIFO_A = 204,
-	AMDGPU_GFX_SQC_DATA_CACHE_MISS_FIFO_B = 205,
-	AMDGPU_GFX_SQC_DATA_CACHE_HIT_FIFO_A = 206,
-	AMDGPU_GFX_SQC_DATA_CACHE_HIT_FIFO_B = 207,
-	AMDGPU_GFX_SQC_DIRTY_BIT_A = 208,
-	AMDGPU_GFX_SQC_DIRTY_BIT_B = 209,
-	AMDGPU_GFX_SQC_WRITE_DATA_BUFFER_CU0 = 210,
-	AMDGPU_GFX_SQC_WRITE_DATA_BUFFER_CU1 = 211,
-	AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_DATA_CACHE_A = 212,
-	AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_DATA_CACHE_B = 213,
-	AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_INST_CACHE = 108,
-};
-
-enum amdgpu_gfx_sq_ras_mem_id {
-	AMDGPU_GFX_SQ_SGPR_MEM0 = 0,
-	AMDGPU_GFX_SQ_SGPR_MEM1,
-	AMDGPU_GFX_SQ_SGPR_MEM2,
-	AMDGPU_GFX_SQ_SGPR_MEM3,
-};
-
-enum amdgpu_gfx_ta_ras_mem_id {
-	AMDGPU_GFX_TA_FS_AFIFO_RAM_LO = 1,
-	AMDGPU_GFX_TA_FS_AFIFO_RAM_HI,
-	AMDGPU_GFX_TA_FS_CFIFO_RAM,
-	AMDGPU_GFX_TA_FSX_LFIFO,
-	AMDGPU_GFX_TA_FS_DFIFO_RAM,
-};
-
-enum amdgpu_gfx_tcc_ras_mem_id {
-	AMDGPU_GFX_TCC_MEM1 = 1,
-};
-
-enum amdgpu_gfx_tca_ras_mem_id {
-	AMDGPU_GFX_TCA_MEM1 = 1,
-};
-
-enum amdgpu_gfx_tci_ras_mem_id {
-	AMDGPU_GFX_TCIW_MEM = 1,
-};
-
-enum amdgpu_gfx_tcp_ras_mem_id {
-	AMDGPU_GFX_TCP_LFIFO0 = 1,
-	AMDGPU_GFX_TCP_SET0BANK0_RAM,
-	AMDGPU_GFX_TCP_SET0BANK1_RAM,
-	AMDGPU_GFX_TCP_SET0BANK2_RAM,
-	AMDGPU_GFX_TCP_SET0BANK3_RAM,
-	AMDGPU_GFX_TCP_SET1BANK0_RAM,
-	AMDGPU_GFX_TCP_SET1BANK1_RAM,
-	AMDGPU_GFX_TCP_SET1BANK2_RAM,
-	AMDGPU_GFX_TCP_SET1BANK3_RAM,
-	AMDGPU_GFX_TCP_SET2BANK0_RAM,
-	AMDGPU_GFX_TCP_SET2BANK1_RAM,
-	AMDGPU_GFX_TCP_SET2BANK2_RAM,
-	AMDGPU_GFX_TCP_SET2BANK3_RAM,
-	AMDGPU_GFX_TCP_SET3BANK0_RAM,
-	AMDGPU_GFX_TCP_SET3BANK1_RAM,
-	AMDGPU_GFX_TCP_SET3BANK2_RAM,
-	AMDGPU_GFX_TCP_SET3BANK3_RAM,
-	AMDGPU_GFX_TCP_VM_FIFO,
-	AMDGPU_GFX_TCP_DB_TAGRAM0,
-	AMDGPU_GFX_TCP_DB_TAGRAM1,
-	AMDGPU_GFX_TCP_DB_TAGRAM2,
-	AMDGPU_GFX_TCP_DB_TAGRAM3,
-	AMDGPU_GFX_TCP_UTCL1_LFIFO_PROBE0,
-	AMDGPU_GFX_TCP_UTCL1_LFIFO_PROBE1,
-	AMDGPU_GFX_TCP_CMD_FIFO,
-};
-
-enum amdgpu_gfx_td_ras_mem_id {
-	AMDGPU_GFX_TD_UTD_CS_FIFO_MEM = 1,
-	AMDGPU_GFX_TD_UTD_SS_FIFO_LO_MEM,
-	AMDGPU_GFX_TD_UTD_SS_FIFO_HI_MEM,
-};
-
-enum amdgpu_gfx_tcx_ras_mem_id {
-	AMDGPU_GFX_TCX_FIFOD0 = 0,
-	AMDGPU_GFX_TCX_FIFOD1,
-	AMDGPU_GFX_TCX_FIFOD2,
-	AMDGPU_GFX_TCX_FIFOD3,
-	AMDGPU_GFX_TCX_FIFOD4,
-	AMDGPU_GFX_TCX_FIFOD5,
-	AMDGPU_GFX_TCX_FIFOD6,
-	AMDGPU_GFX_TCX_FIFOD7,
-	AMDGPU_GFX_TCX_FIFOB0,
-	AMDGPU_GFX_TCX_FIFOB1,
-	AMDGPU_GFX_TCX_FIFOB2,
-	AMDGPU_GFX_TCX_FIFOB3,
-	AMDGPU_GFX_TCX_FIFOB4,
-	AMDGPU_GFX_TCX_FIFOB5,
-	AMDGPU_GFX_TCX_FIFOB6,
-	AMDGPU_GFX_TCX_FIFOB7,
-	AMDGPU_GFX_TCX_FIFOA0,
-	AMDGPU_GFX_TCX_FIFOA1,
-	AMDGPU_GFX_TCX_FIFOA2,
-	AMDGPU_GFX_TCX_FIFOA3,
-	AMDGPU_GFX_TCX_FIFOA4,
-	AMDGPU_GFX_TCX_FIFOA5,
-	AMDGPU_GFX_TCX_FIFOA6,
-	AMDGPU_GFX_TCX_FIFOA7,
-	AMDGPU_GFX_TCX_CFIFO0,
-	AMDGPU_GFX_TCX_CFIFO1,
-	AMDGPU_GFX_TCX_CFIFO2,
-	AMDGPU_GFX_TCX_CFIFO3,
-	AMDGPU_GFX_TCX_CFIFO4,
-	AMDGPU_GFX_TCX_CFIFO5,
-	AMDGPU_GFX_TCX_CFIFO6,
-	AMDGPU_GFX_TCX_CFIFO7,
-	AMDGPU_GFX_TCX_FIFO_ACKB0,
-	AMDGPU_GFX_TCX_FIFO_ACKB1,
-	AMDGPU_GFX_TCX_FIFO_ACKB2,
-	AMDGPU_GFX_TCX_FIFO_ACKB3,
-	AMDGPU_GFX_TCX_FIFO_ACKB4,
-	AMDGPU_GFX_TCX_FIFO_ACKB5,
-	AMDGPU_GFX_TCX_FIFO_ACKB6,
-	AMDGPU_GFX_TCX_FIFO_ACKB7,
-	AMDGPU_GFX_TCX_FIFO_ACKD0,
-	AMDGPU_GFX_TCX_FIFO_ACKD1,
-	AMDGPU_GFX_TCX_FIFO_ACKD2,
-	AMDGPU_GFX_TCX_FIFO_ACKD3,
-	AMDGPU_GFX_TCX_FIFO_ACKD4,
-	AMDGPU_GFX_TCX_FIFO_ACKD5,
-	AMDGPU_GFX_TCX_FIFO_ACKD6,
-	AMDGPU_GFX_TCX_FIFO_ACKD7,
-	AMDGPU_GFX_TCX_DST_FIFOA0,
-	AMDGPU_GFX_TCX_DST_FIFOA1,
-	AMDGPU_GFX_TCX_DST_FIFOA2,
-	AMDGPU_GFX_TCX_DST_FIFOA3,
-	AMDGPU_GFX_TCX_DST_FIFOA4,
-	AMDGPU_GFX_TCX_DST_FIFOA5,
-	AMDGPU_GFX_TCX_DST_FIFOA6,
-	AMDGPU_GFX_TCX_DST_FIFOA7,
-	AMDGPU_GFX_TCX_DST_FIFOB0,
-	AMDGPU_GFX_TCX_DST_FIFOB1,
-	AMDGPU_GFX_TCX_DST_FIFOB2,
-	AMDGPU_GFX_TCX_DST_FIFOB3,
-	AMDGPU_GFX_TCX_DST_FIFOB4,
-	AMDGPU_GFX_TCX_DST_FIFOB5,
-	AMDGPU_GFX_TCX_DST_FIFOB6,
-	AMDGPU_GFX_TCX_DST_FIFOB7,
-	AMDGPU_GFX_TCX_DST_FIFOD0,
-	AMDGPU_GFX_TCX_DST_FIFOD1,
-	AMDGPU_GFX_TCX_DST_FIFOD2,
-	AMDGPU_GFX_TCX_DST_FIFOD3,
-	AMDGPU_GFX_TCX_DST_FIFOD4,
-	AMDGPU_GFX_TCX_DST_FIFOD5,
-	AMDGPU_GFX_TCX_DST_FIFOD6,
-	AMDGPU_GFX_TCX_DST_FIFOD7,
-	AMDGPU_GFX_TCX_DST_FIFO_ACKB0,
-	AMDGPU_GFX_TCX_DST_FIFO_ACKB1,
-	AMDGPU_GFX_TCX_DST_FIFO_ACKB2,
-	AMDGPU_GFX_TCX_DST_FIFO_ACKB3,
-	AMDGPU_GFX_TCX_DST_FIFO_ACKB4,
-	AMDGPU_GFX_TCX_DST_FIFO_ACKB5,
-	AMDGPU_GFX_TCX_DST_FIFO_ACKB6,
-	AMDGPU_GFX_TCX_DST_FIFO_ACKB7,
-	AMDGPU_GFX_TCX_DST_FIFO_ACKD0,
-	AMDGPU_GFX_TCX_DST_FIFO_ACKD1,
-	AMDGPU_GFX_TCX_DST_FIFO_ACKD2,
-	AMDGPU_GFX_TCX_DST_FIFO_ACKD3,
-	AMDGPU_GFX_TCX_DST_FIFO_ACKD4,
-	AMDGPU_GFX_TCX_DST_FIFO_ACKD5,
-	AMDGPU_GFX_TCX_DST_FIFO_ACKD6,
-	AMDGPU_GFX_TCX_DST_FIFO_ACKD7,
-};
-
-enum amdgpu_gfx_atc_l2_ras_mem_id {
-	AMDGPU_GFX_ATC_L2_MEM0 = 0,
-};
-
-enum amdgpu_gfx_utcl2_ras_mem_id {
-	AMDGPU_GFX_UTCL2_MEM0 = 0,
-};
-
-enum amdgpu_gfx_vml2_ras_mem_id {
-	AMDGPU_GFX_VML2_MEM0 = 0,
-};
-
-enum amdgpu_gfx_vml2_walker_ras_mem_id {
-	AMDGPU_GFX_VML2_WALKER_MEM0 = 0,
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_cp_mem_list[] = {
-	{AMDGPU_GFX_CP_MEM1, "CP_MEM1"},
-	{AMDGPU_GFX_CP_MEM2, "CP_MEM2"},
-	{AMDGPU_GFX_CP_MEM3, "CP_MEM3"},
-	{AMDGPU_GFX_CP_MEM4, "CP_MEM4"},
-	{AMDGPU_GFX_CP_MEM5, "CP_MEM5"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_gcea_mem_list[] = {
-	{AMDGPU_GFX_GCEA_IOWR_CMDMEM, "GCEA_IOWR_CMDMEM"},
-	{AMDGPU_GFX_GCEA_IORD_CMDMEM, "GCEA_IORD_CMDMEM"},
-	{AMDGPU_GFX_GCEA_GMIWR_CMDMEM, "GCEA_GMIWR_CMDMEM"},
-	{AMDGPU_GFX_GCEA_GMIRD_CMDMEM, "GCEA_GMIRD_CMDMEM"},
-	{AMDGPU_GFX_GCEA_DRAMWR_CMDMEM, "GCEA_DRAMWR_CMDMEM"},
-	{AMDGPU_GFX_GCEA_DRAMRD_CMDMEM, "GCEA_DRAMRD_CMDMEM"},
-	{AMDGPU_GFX_GCEA_MAM_DMEM0, "GCEA_MAM_DMEM0"},
-	{AMDGPU_GFX_GCEA_MAM_DMEM1, "GCEA_MAM_DMEM1"},
-	{AMDGPU_GFX_GCEA_MAM_DMEM2, "GCEA_MAM_DMEM2"},
-	{AMDGPU_GFX_GCEA_MAM_DMEM3, "GCEA_MAM_DMEM3"},
-	{AMDGPU_GFX_GCEA_MAM_AMEM0, "GCEA_MAM_AMEM0"},
-	{AMDGPU_GFX_GCEA_MAM_AMEM1, "GCEA_MAM_AMEM1"},
-	{AMDGPU_GFX_GCEA_MAM_AMEM2, "GCEA_MAM_AMEM2"},
-	{AMDGPU_GFX_GCEA_MAM_AMEM3, "GCEA_MAM_AMEM3"},
-	{AMDGPU_GFX_GCEA_MAM_AFLUSH_BUFFER, "GCEA_MAM_AFLUSH_BUFFER"},
-	{AMDGPU_GFX_GCEA_WRET_TAGMEM, "GCEA_WRET_TAGMEM"},
-	{AMDGPU_GFX_GCEA_RRET_TAGMEM, "GCEA_RRET_TAGMEM"},
-	{AMDGPU_GFX_GCEA_IOWR_DATAMEM, "GCEA_IOWR_DATAMEM"},
-	{AMDGPU_GFX_GCEA_GMIWR_DATAMEM, "GCEA_GMIWR_DATAMEM"},
-	{AMDGPU_GFX_GCEA_DRAM_DATAMEM, "GCEA_DRAM_DATAMEM"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_gc_cane_mem_list[] = {
-	{AMDGPU_GFX_GC_CANE_MEM0, "GC_CANE_MEM0"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_gcutcl2_mem_list[] = {
-	{AMDGPU_GFX_GCUTCL2_MEM2P512X95, "GCUTCL2_MEM2P512X95"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_gds_mem_list[] = {
-	{AMDGPU_GFX_GDS_MEM0, "GDS_MEM"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_lds_mem_list[] = {
-	{AMDGPU_GFX_LDS_BANK0, "LDS_BANK0"},
-	{AMDGPU_GFX_LDS_BANK1, "LDS_BANK1"},
-	{AMDGPU_GFX_LDS_BANK2, "LDS_BANK2"},
-	{AMDGPU_GFX_LDS_BANK3, "LDS_BANK3"},
-	{AMDGPU_GFX_LDS_BANK4, "LDS_BANK4"},
-	{AMDGPU_GFX_LDS_BANK5, "LDS_BANK5"},
-	{AMDGPU_GFX_LDS_BANK6, "LDS_BANK6"},
-	{AMDGPU_GFX_LDS_BANK7, "LDS_BANK7"},
-	{AMDGPU_GFX_LDS_BANK8, "LDS_BANK8"},
-	{AMDGPU_GFX_LDS_BANK9, "LDS_BANK9"},
-	{AMDGPU_GFX_LDS_BANK10, "LDS_BANK10"},
-	{AMDGPU_GFX_LDS_BANK11, "LDS_BANK11"},
-	{AMDGPU_GFX_LDS_BANK12, "LDS_BANK12"},
-	{AMDGPU_GFX_LDS_BANK13, "LDS_BANK13"},
-	{AMDGPU_GFX_LDS_BANK14, "LDS_BANK14"},
-	{AMDGPU_GFX_LDS_BANK15, "LDS_BANK15"},
-	{AMDGPU_GFX_LDS_BANK16, "LDS_BANK16"},
-	{AMDGPU_GFX_LDS_BANK17, "LDS_BANK17"},
-	{AMDGPU_GFX_LDS_BANK18, "LDS_BANK18"},
-	{AMDGPU_GFX_LDS_BANK19, "LDS_BANK19"},
-	{AMDGPU_GFX_LDS_BANK20, "LDS_BANK20"},
-	{AMDGPU_GFX_LDS_BANK21, "LDS_BANK21"},
-	{AMDGPU_GFX_LDS_BANK22, "LDS_BANK22"},
-	{AMDGPU_GFX_LDS_BANK23, "LDS_BANK23"},
-	{AMDGPU_GFX_LDS_BANK24, "LDS_BANK24"},
-	{AMDGPU_GFX_LDS_BANK25, "LDS_BANK25"},
-	{AMDGPU_GFX_LDS_BANK26, "LDS_BANK26"},
-	{AMDGPU_GFX_LDS_BANK27, "LDS_BANK27"},
-	{AMDGPU_GFX_LDS_BANK28, "LDS_BANK28"},
-	{AMDGPU_GFX_LDS_BANK29, "LDS_BANK29"},
-	{AMDGPU_GFX_LDS_BANK30, "LDS_BANK30"},
-	{AMDGPU_GFX_LDS_BANK31, "LDS_BANK31"},
-	{AMDGPU_GFX_LDS_SP_BUFFER_A, "LDS_SP_BUFFER_A"},
-	{AMDGPU_GFX_LDS_SP_BUFFER_B, "LDS_SP_BUFFER_B"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_rlc_mem_list[] = {
-	{AMDGPU_GFX_RLC_GPMF32, "RLC_GPMF32"},
-	{AMDGPU_GFX_RLC_RLCVF32, "RLC_RLCVF32"},
-	{AMDGPU_GFX_RLC_SCRATCH, "RLC_SCRATCH"},
-	{AMDGPU_GFX_RLC_SRM_ARAM, "RLC_SRM_ARAM"},
-	{AMDGPU_GFX_RLC_SRM_DRAM, "RLC_SRM_DRAM"},
-	{AMDGPU_GFX_RLC_TCTAG, "RLC_TCTAG"},
-	{AMDGPU_GFX_RLC_SPM_SE, "RLC_SPM_SE"},
-	{AMDGPU_GFX_RLC_SPM_GRBMT, "RLC_SPM_GRBMT"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_sp_mem_list[] = {
-	{AMDGPU_GFX_SP_SIMDID0, "SP_SIMDID0"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_spi_mem_list[] = {
-	{AMDGPU_GFX_SPI_MEM0, "SPI_MEM0"},
-	{AMDGPU_GFX_SPI_MEM1, "SPI_MEM1"},
-	{AMDGPU_GFX_SPI_MEM2, "SPI_MEM2"},
-	{AMDGPU_GFX_SPI_MEM3, "SPI_MEM3"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_sqc_mem_list[] = {
-	{AMDGPU_GFX_SQC_INST_CACHE_A, "SQC_INST_CACHE_A"},
-	{AMDGPU_GFX_SQC_INST_CACHE_B, "SQC_INST_CACHE_B"},
-	{AMDGPU_GFX_SQC_INST_CACHE_TAG_A, "SQC_INST_CACHE_TAG_A"},
-	{AMDGPU_GFX_SQC_INST_CACHE_TAG_B, "SQC_INST_CACHE_TAG_B"},
-	{AMDGPU_GFX_SQC_INST_CACHE_MISS_FIFO_A, "SQC_INST_CACHE_MISS_FIFO_A"},
-	{AMDGPU_GFX_SQC_INST_CACHE_MISS_FIFO_B, "SQC_INST_CACHE_MISS_FIFO_B"},
-	{AMDGPU_GFX_SQC_INST_CACHE_GATCL1_MISS_FIFO_A, "SQC_INST_CACHE_GATCL1_MISS_FIFO_A"},
-	{AMDGPU_GFX_SQC_INST_CACHE_GATCL1_MISS_FIFO_B, "SQC_INST_CACHE_GATCL1_MISS_FIFO_B"},
-	{AMDGPU_GFX_SQC_DATA_CACHE_A, "SQC_DATA_CACHE_A"},
-	{AMDGPU_GFX_SQC_DATA_CACHE_B, "SQC_DATA_CACHE_B"},
-	{AMDGPU_GFX_SQC_DATA_CACHE_TAG_A, "SQC_DATA_CACHE_TAG_A"},
-	{AMDGPU_GFX_SQC_DATA_CACHE_TAG_B, "SQC_DATA_CACHE_TAG_B"},
-	{AMDGPU_GFX_SQC_DATA_CACHE_MISS_FIFO_A, "SQC_DATA_CACHE_MISS_FIFO_A"},
-	{AMDGPU_GFX_SQC_DATA_CACHE_MISS_FIFO_B, "SQC_DATA_CACHE_MISS_FIFO_B"},
-	{AMDGPU_GFX_SQC_DATA_CACHE_HIT_FIFO_A, "SQC_DATA_CACHE_HIT_FIFO_A"},
-	{AMDGPU_GFX_SQC_DATA_CACHE_HIT_FIFO_B, "SQC_DATA_CACHE_HIT_FIFO_B"},
-	{AMDGPU_GFX_SQC_DIRTY_BIT_A, "SQC_DIRTY_BIT_A"},
-	{AMDGPU_GFX_SQC_DIRTY_BIT_B, "SQC_DIRTY_BIT_B"},
-	{AMDGPU_GFX_SQC_WRITE_DATA_BUFFER_CU0, "SQC_WRITE_DATA_BUFFER_CU0"},
-	{AMDGPU_GFX_SQC_WRITE_DATA_BUFFER_CU1, "SQC_WRITE_DATA_BUFFER_CU1"},
-	{AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_DATA_CACHE_A, "SQC_UTCL1_MISS_LFIFO_DATA_CACHE_A"},
-	{AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_DATA_CACHE_B, "SQC_UTCL1_MISS_LFIFO_DATA_CACHE_B"},
-	{AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_INST_CACHE, "SQC_UTCL1_MISS_LFIFO_INST_CACHE"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_sq_mem_list[] = {
-	{AMDGPU_GFX_SQ_SGPR_MEM0, "SQ_SGPR_MEM0"},
-	{AMDGPU_GFX_SQ_SGPR_MEM1, "SQ_SGPR_MEM1"},
-	{AMDGPU_GFX_SQ_SGPR_MEM2, "SQ_SGPR_MEM2"},
-	{AMDGPU_GFX_SQ_SGPR_MEM3, "SQ_SGPR_MEM3"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_ta_mem_list[] = {
-	{AMDGPU_GFX_TA_FS_AFIFO_RAM_LO, "TA_FS_AFIFO_RAM_LO"},
-	{AMDGPU_GFX_TA_FS_AFIFO_RAM_HI, "TA_FS_AFIFO_RAM_HI"},
-	{AMDGPU_GFX_TA_FS_CFIFO_RAM, "TA_FS_CFIFO_RAM"},
-	{AMDGPU_GFX_TA_FSX_LFIFO, "TA_FSX_LFIFO"},
-	{AMDGPU_GFX_TA_FS_DFIFO_RAM, "TA_FS_DFIFO_RAM"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_tcc_mem_list[] = {
-	{AMDGPU_GFX_TCC_MEM1, "TCC_MEM1"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_tca_mem_list[] = {
-	{AMDGPU_GFX_TCA_MEM1, "TCA_MEM1"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_tci_mem_list[] = {
-	{AMDGPU_GFX_TCIW_MEM, "TCIW_MEM"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_tcp_mem_list[] = {
-	{AMDGPU_GFX_TCP_LFIFO0, "TCP_LFIFO0"},
-	{AMDGPU_GFX_TCP_SET0BANK0_RAM, "TCP_SET0BANK0_RAM"},
-	{AMDGPU_GFX_TCP_SET0BANK1_RAM, "TCP_SET0BANK1_RAM"},
-	{AMDGPU_GFX_TCP_SET0BANK2_RAM, "TCP_SET0BANK2_RAM"},
-	{AMDGPU_GFX_TCP_SET0BANK3_RAM, "TCP_SET0BANK3_RAM"},
-	{AMDGPU_GFX_TCP_SET1BANK0_RAM, "TCP_SET1BANK0_RAM"},
-	{AMDGPU_GFX_TCP_SET1BANK1_RAM, "TCP_SET1BANK1_RAM"},
-	{AMDGPU_GFX_TCP_SET1BANK2_RAM, "TCP_SET1BANK2_RAM"},
-	{AMDGPU_GFX_TCP_SET1BANK3_RAM, "TCP_SET1BANK3_RAM"},
-	{AMDGPU_GFX_TCP_SET2BANK0_RAM, "TCP_SET2BANK0_RAM"},
-	{AMDGPU_GFX_TCP_SET2BANK1_RAM, "TCP_SET2BANK1_RAM"},
-	{AMDGPU_GFX_TCP_SET2BANK2_RAM, "TCP_SET2BANK2_RAM"},
-	{AMDGPU_GFX_TCP_SET2BANK3_RAM, "TCP_SET2BANK3_RAM"},
-	{AMDGPU_GFX_TCP_SET3BANK0_RAM, "TCP_SET3BANK0_RAM"},
-	{AMDGPU_GFX_TCP_SET3BANK1_RAM, "TCP_SET3BANK1_RAM"},
-	{AMDGPU_GFX_TCP_SET3BANK2_RAM, "TCP_SET3BANK2_RAM"},
-	{AMDGPU_GFX_TCP_SET3BANK3_RAM, "TCP_SET3BANK3_RAM"},
-	{AMDGPU_GFX_TCP_VM_FIFO, "TCP_VM_FIFO"},
-	{AMDGPU_GFX_TCP_DB_TAGRAM0, "TCP_DB_TAGRAM0"},
-	{AMDGPU_GFX_TCP_DB_TAGRAM1, "TCP_DB_TAGRAM1"},
-	{AMDGPU_GFX_TCP_DB_TAGRAM2, "TCP_DB_TAGRAM2"},
-	{AMDGPU_GFX_TCP_DB_TAGRAM3, "TCP_DB_TAGRAM3"},
-	{AMDGPU_GFX_TCP_UTCL1_LFIFO_PROBE0, "TCP_UTCL1_LFIFO_PROBE0"},
-	{AMDGPU_GFX_TCP_UTCL1_LFIFO_PROBE1, "TCP_UTCL1_LFIFO_PROBE1"},
-	{AMDGPU_GFX_TCP_CMD_FIFO, "TCP_CMD_FIFO"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_td_mem_list[] = {
-	{AMDGPU_GFX_TD_UTD_CS_FIFO_MEM, "TD_UTD_CS_FIFO_MEM"},
-	{AMDGPU_GFX_TD_UTD_SS_FIFO_LO_MEM, "TD_UTD_SS_FIFO_LO_MEM"},
-	{AMDGPU_GFX_TD_UTD_SS_FIFO_HI_MEM, "TD_UTD_SS_FIFO_HI_MEM"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_tcx_mem_list[] = {
-	{AMDGPU_GFX_TCX_FIFOD0, "TCX_FIFOD0"},
-	{AMDGPU_GFX_TCX_FIFOD1, "TCX_FIFOD1"},
-	{AMDGPU_GFX_TCX_FIFOD2, "TCX_FIFOD2"},
-	{AMDGPU_GFX_TCX_FIFOD3, "TCX_FIFOD3"},
-	{AMDGPU_GFX_TCX_FIFOD4, "TCX_FIFOD4"},
-	{AMDGPU_GFX_TCX_FIFOD5, "TCX_FIFOD5"},
-	{AMDGPU_GFX_TCX_FIFOD6, "TCX_FIFOD6"},
-	{AMDGPU_GFX_TCX_FIFOD7, "TCX_FIFOD7"},
-	{AMDGPU_GFX_TCX_FIFOB0, "TCX_FIFOB0"},
-	{AMDGPU_GFX_TCX_FIFOB1, "TCX_FIFOB1"},
-	{AMDGPU_GFX_TCX_FIFOB2, "TCX_FIFOB2"},
-	{AMDGPU_GFX_TCX_FIFOB3, "TCX_FIFOB3"},
-	{AMDGPU_GFX_TCX_FIFOB4, "TCX_FIFOB4"},
-	{AMDGPU_GFX_TCX_FIFOB5, "TCX_FIFOB5"},
-	{AMDGPU_GFX_TCX_FIFOB6, "TCX_FIFOB6"},
-	{AMDGPU_GFX_TCX_FIFOB7, "TCX_FIFOB7"},
-	{AMDGPU_GFX_TCX_FIFOA0, "TCX_FIFOA0"},
-	{AMDGPU_GFX_TCX_FIFOA1, "TCX_FIFOA1"},
-	{AMDGPU_GFX_TCX_FIFOA2, "TCX_FIFOA2"},
-	{AMDGPU_GFX_TCX_FIFOA3, "TCX_FIFOA3"},
-	{AMDGPU_GFX_TCX_FIFOA4, "TCX_FIFOA4"},
-	{AMDGPU_GFX_TCX_FIFOA5, "TCX_FIFOA5"},
-	{AMDGPU_GFX_TCX_FIFOA6, "TCX_FIFOA6"},
-	{AMDGPU_GFX_TCX_FIFOA7, "TCX_FIFOA7"},
-	{AMDGPU_GFX_TCX_CFIFO0, "TCX_CFIFO0"},
-	{AMDGPU_GFX_TCX_CFIFO1, "TCX_CFIFO1"},
-	{AMDGPU_GFX_TCX_CFIFO2, "TCX_CFIFO2"},
-	{AMDGPU_GFX_TCX_CFIFO3, "TCX_CFIFO3"},
-	{AMDGPU_GFX_TCX_CFIFO4, "TCX_CFIFO4"},
-	{AMDGPU_GFX_TCX_CFIFO5, "TCX_CFIFO5"},
-	{AMDGPU_GFX_TCX_CFIFO6, "TCX_CFIFO6"},
-	{AMDGPU_GFX_TCX_CFIFO7, "TCX_CFIFO7"},
-	{AMDGPU_GFX_TCX_FIFO_ACKB0, "TCX_FIFO_ACKB0"},
-	{AMDGPU_GFX_TCX_FIFO_ACKB1, "TCX_FIFO_ACKB1"},
-	{AMDGPU_GFX_TCX_FIFO_ACKB2, "TCX_FIFO_ACKB2"},
-	{AMDGPU_GFX_TCX_FIFO_ACKB3, "TCX_FIFO_ACKB3"},
-	{AMDGPU_GFX_TCX_FIFO_ACKB4, "TCX_FIFO_ACKB4"},
-	{AMDGPU_GFX_TCX_FIFO_ACKB5, "TCX_FIFO_ACKB5"},
-	{AMDGPU_GFX_TCX_FIFO_ACKB6, "TCX_FIFO_ACKB6"},
-	{AMDGPU_GFX_TCX_FIFO_ACKB7, "TCX_FIFO_ACKB7"},
-	{AMDGPU_GFX_TCX_FIFO_ACKD0, "TCX_FIFO_ACKD0"},
-	{AMDGPU_GFX_TCX_FIFO_ACKD1, "TCX_FIFO_ACKD1"},
-	{AMDGPU_GFX_TCX_FIFO_ACKD2, "TCX_FIFO_ACKD2"},
-	{AMDGPU_GFX_TCX_FIFO_ACKD3, "TCX_FIFO_ACKD3"},
-	{AMDGPU_GFX_TCX_FIFO_ACKD4, "TCX_FIFO_ACKD4"},
-	{AMDGPU_GFX_TCX_FIFO_ACKD5, "TCX_FIFO_ACKD5"},
-	{AMDGPU_GFX_TCX_FIFO_ACKD6, "TCX_FIFO_ACKD6"},
-	{AMDGPU_GFX_TCX_FIFO_ACKD7, "TCX_FIFO_ACKD7"},
-	{AMDGPU_GFX_TCX_DST_FIFOA0, "TCX_DST_FIFOA0"},
-	{AMDGPU_GFX_TCX_DST_FIFOA1, "TCX_DST_FIFOA1"},
-	{AMDGPU_GFX_TCX_DST_FIFOA2, "TCX_DST_FIFOA2"},
-	{AMDGPU_GFX_TCX_DST_FIFOA3, "TCX_DST_FIFOA3"},
-	{AMDGPU_GFX_TCX_DST_FIFOA4, "TCX_DST_FIFOA4"},
-	{AMDGPU_GFX_TCX_DST_FIFOA5, "TCX_DST_FIFOA5"},
-	{AMDGPU_GFX_TCX_DST_FIFOA6, "TCX_DST_FIFOA6"},
-	{AMDGPU_GFX_TCX_DST_FIFOA7, "TCX_DST_FIFOA7"},
-	{AMDGPU_GFX_TCX_DST_FIFOB0, "TCX_DST_FIFOB0"},
-	{AMDGPU_GFX_TCX_DST_FIFOB1, "TCX_DST_FIFOB1"},
-	{AMDGPU_GFX_TCX_DST_FIFOB2, "TCX_DST_FIFOB2"},
-	{AMDGPU_GFX_TCX_DST_FIFOB3, "TCX_DST_FIFOB3"},
-	{AMDGPU_GFX_TCX_DST_FIFOB4, "TCX_DST_FIFOB4"},
-	{AMDGPU_GFX_TCX_DST_FIFOB5, "TCX_DST_FIFOB5"},
-	{AMDGPU_GFX_TCX_DST_FIFOB6, "TCX_DST_FIFOB6"},
-	{AMDGPU_GFX_TCX_DST_FIFOB7, "TCX_DST_FIFOB7"},
-	{AMDGPU_GFX_TCX_DST_FIFOD0, "TCX_DST_FIFOD0"},
-	{AMDGPU_GFX_TCX_DST_FIFOD1, "TCX_DST_FIFOD1"},
-	{AMDGPU_GFX_TCX_DST_FIFOD2, "TCX_DST_FIFOD2"},
-	{AMDGPU_GFX_TCX_DST_FIFOD3, "TCX_DST_FIFOD3"},
-	{AMDGPU_GFX_TCX_DST_FIFOD4, "TCX_DST_FIFOD4"},
-	{AMDGPU_GFX_TCX_DST_FIFOD5, "TCX_DST_FIFOD5"},
-	{AMDGPU_GFX_TCX_DST_FIFOD6, "TCX_DST_FIFOD6"},
-	{AMDGPU_GFX_TCX_DST_FIFOD7, "TCX_DST_FIFOD7"},
-	{AMDGPU_GFX_TCX_DST_FIFO_ACKB0, "TCX_DST_FIFO_ACKB0"},
-	{AMDGPU_GFX_TCX_DST_FIFO_ACKB1, "TCX_DST_FIFO_ACKB1"},
-	{AMDGPU_GFX_TCX_DST_FIFO_ACKB2, "TCX_DST_FIFO_ACKB2"},
-	{AMDGPU_GFX_TCX_DST_FIFO_ACKB3, "TCX_DST_FIFO_ACKB3"},
-	{AMDGPU_GFX_TCX_DST_FIFO_ACKB4, "TCX_DST_FIFO_ACKB4"},
-	{AMDGPU_GFX_TCX_DST_FIFO_ACKB5, "TCX_DST_FIFO_ACKB5"},
-	{AMDGPU_GFX_TCX_DST_FIFO_ACKB6, "TCX_DST_FIFO_ACKB6"},
-	{AMDGPU_GFX_TCX_DST_FIFO_ACKB7, "TCX_DST_FIFO_ACKB7"},
-	{AMDGPU_GFX_TCX_DST_FIFO_ACKD0, "TCX_DST_FIFO_ACKD0"},
-	{AMDGPU_GFX_TCX_DST_FIFO_ACKD1, "TCX_DST_FIFO_ACKD1"},
-	{AMDGPU_GFX_TCX_DST_FIFO_ACKD2, "TCX_DST_FIFO_ACKD2"},
-	{AMDGPU_GFX_TCX_DST_FIFO_ACKD3, "TCX_DST_FIFO_ACKD3"},
-	{AMDGPU_GFX_TCX_DST_FIFO_ACKD4, "TCX_DST_FIFO_ACKD4"},
-	{AMDGPU_GFX_TCX_DST_FIFO_ACKD5, "TCX_DST_FIFO_ACKD5"},
-	{AMDGPU_GFX_TCX_DST_FIFO_ACKD6, "TCX_DST_FIFO_ACKD6"},
-	{AMDGPU_GFX_TCX_DST_FIFO_ACKD7, "TCX_DST_FIFO_ACKD7"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_atc_l2_mem_list[] = {
-	{AMDGPU_GFX_ATC_L2_MEM, "ATC_L2_MEM"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_utcl2_mem_list[] = {
-	{AMDGPU_GFX_UTCL2_MEM, "UTCL2_MEM"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_vml2_mem_list[] = {
-	{AMDGPU_GFX_VML2_MEM, "VML2_MEM"},
-};
-
-static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_vml2_walker_mem_list[] = {
-	{AMDGPU_GFX_VML2_WALKER_MEM, "VML2_WALKER_MEM"},
-};
-
-static const struct amdgpu_gfx_ras_mem_id_entry gfx_v9_4_3_ras_mem_list_array[AMDGPU_GFX_MEM_TYPE_NUM] = {
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_cp_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_gcea_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_gc_cane_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_gcutcl2_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_gds_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_lds_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_rlc_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_sp_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_spi_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_sqc_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_sq_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_ta_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_tcc_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_tca_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_tci_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_tcp_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_td_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_tcx_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_atc_l2_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_utcl2_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_vml2_mem_list)
-	AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_vml2_walker_mem_list)
-};
-
-static const struct amdgpu_gfx_ras_reg_entry gfx_v9_4_3_ce_reg_list[] = {
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regRLC_CE_ERR_STATUS_LOW, regRLC_CE_ERR_STATUS_HIGH),
-	    1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "RLC"},
-	    AMDGPU_GFX_RLC_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPC_CE_ERR_STATUS_LO, regCPC_CE_ERR_STATUS_HI),
-	    1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPC"},
-	    AMDGPU_GFX_CP_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPF_CE_ERR_STATUS_LO, regCPF_CE_ERR_STATUS_HI),
-	    1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPF"},
-	    AMDGPU_GFX_CP_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPG_CE_ERR_STATUS_LO, regCPG_CE_ERR_STATUS_HI),
-	    1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPG"},
-	    AMDGPU_GFX_CP_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regGDS_CE_ERR_STATUS_LO, regGDS_CE_ERR_STATUS_HI),
-	    1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "GDS"},
-	    AMDGPU_GFX_GDS_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regGC_CANE_CE_ERR_STATUS_LO, regGC_CANE_CE_ERR_STATUS_HI),
-	    1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CANE"},
-	    AMDGPU_GFX_GC_CANE_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSPI_CE_ERR_STATUS_LO, regSPI_CE_ERR_STATUS_HI),
-	    1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SPI"},
-	    AMDGPU_GFX_SPI_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP0_CE_ERR_STATUS_LO, regSP0_CE_ERR_STATUS_HI),
-	    10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP0"},
-	    AMDGPU_GFX_SP_MEM, 4},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP1_CE_ERR_STATUS_LO, regSP1_CE_ERR_STATUS_HI),
-	    10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP1"},
-	    AMDGPU_GFX_SP_MEM, 4},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQ_CE_ERR_STATUS_LO, regSQ_CE_ERR_STATUS_HI),
-	    10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQ"},
-	    AMDGPU_GFX_SQ_MEM, 4},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQC_CE_EDC_LO, regSQC_CE_EDC_HI),
-	    5, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQC"},
-	    AMDGPU_GFX_SQC_MEM, 4},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCX_CE_ERR_STATUS_LO, regTCX_CE_ERR_STATUS_HI),
-	    2, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCX"},
-	    AMDGPU_GFX_TCX_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCC_CE_ERR_STATUS_LO, regTCC_CE_ERR_STATUS_HI),
-	    16, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCC"},
-	    AMDGPU_GFX_TCC_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTA_CE_EDC_LO, regTA_CE_EDC_HI),
-	    10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TA"},
-	    AMDGPU_GFX_TA_MEM, 4},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCI_CE_EDC_LO_REG, regTCI_CE_EDC_HI_REG),
-	    27, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCI"},
-	    AMDGPU_GFX_TCI_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCP_CE_EDC_LO_REG, regTCP_CE_EDC_HI_REG),
-	    10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCP"},
-	    AMDGPU_GFX_TCP_MEM, 4},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTD_CE_EDC_LO, regTD_CE_EDC_HI),
-	    10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TD"},
-	    AMDGPU_GFX_TD_MEM, 4},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regGCEA_CE_ERR_STATUS_LO, regGCEA_CE_ERR_STATUS_HI),
-	    16, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "GCEA"},
-	    AMDGPU_GFX_GCEA_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regLDS_CE_ERR_STATUS_LO, regLDS_CE_ERR_STATUS_HI),
-	    10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "LDS"},
-	    AMDGPU_GFX_LDS_MEM, 4},
-};
-
-static const struct amdgpu_gfx_ras_reg_entry gfx_v9_4_3_ue_reg_list[] = {
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regRLC_UE_ERR_STATUS_LOW, regRLC_UE_ERR_STATUS_HIGH),
-	    1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "RLC"},
-	    AMDGPU_GFX_RLC_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPC_UE_ERR_STATUS_LO, regCPC_UE_ERR_STATUS_HI),
-	    1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPC"},
-	    AMDGPU_GFX_CP_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPF_UE_ERR_STATUS_LO, regCPF_UE_ERR_STATUS_HI),
-	    1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPF"},
-	    AMDGPU_GFX_CP_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPG_UE_ERR_STATUS_LO, regCPG_UE_ERR_STATUS_HI),
-	    1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPG"},
-	    AMDGPU_GFX_CP_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regGDS_UE_ERR_STATUS_LO, regGDS_UE_ERR_STATUS_HI),
-	    1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "GDS"},
-	    AMDGPU_GFX_GDS_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regGC_CANE_UE_ERR_STATUS_LO, regGC_CANE_UE_ERR_STATUS_HI),
-	    1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CANE"},
-	    AMDGPU_GFX_GC_CANE_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSPI_UE_ERR_STATUS_LO, regSPI_UE_ERR_STATUS_HI),
-	    1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SPI"},
-	    AMDGPU_GFX_SPI_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP0_UE_ERR_STATUS_LO, regSP0_UE_ERR_STATUS_HI),
-	    10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP0"},
-	    AMDGPU_GFX_SP_MEM, 4},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP1_UE_ERR_STATUS_LO, regSP1_UE_ERR_STATUS_HI),
-	    10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP1"},
-	    AMDGPU_GFX_SP_MEM, 4},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQ_UE_ERR_STATUS_LO, regSQ_UE_ERR_STATUS_HI),
-	    10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQ"},
-	    AMDGPU_GFX_SQ_MEM, 4},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQC_UE_EDC_LO, regSQC_UE_EDC_HI),
-	    5, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQC"},
-	    AMDGPU_GFX_SQC_MEM, 4},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCX_UE_ERR_STATUS_LO, regTCX_UE_ERR_STATUS_HI),
-	    2, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCX"},
-	    AMDGPU_GFX_TCX_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCC_UE_ERR_STATUS_LO, regTCC_UE_ERR_STATUS_HI),
-	    16, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCC"},
-	    AMDGPU_GFX_TCC_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTA_UE_EDC_LO, regTA_UE_EDC_HI),
-	    10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TA"},
-	    AMDGPU_GFX_TA_MEM, 4},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCI_UE_EDC_LO_REG, regTCI_UE_EDC_HI_REG),
-	    27, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCI"},
-	    AMDGPU_GFX_TCI_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCP_UE_EDC_LO_REG, regTCP_UE_EDC_HI_REG),
-	    10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCP"},
-	    AMDGPU_GFX_TCP_MEM, 4},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTD_UE_EDC_LO, regTD_UE_EDC_HI),
-	    10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TD"},
-	    AMDGPU_GFX_TD_MEM, 4},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCA_UE_ERR_STATUS_LO, regTCA_UE_ERR_STATUS_HI),
-	    2, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCA"},
-	    AMDGPU_GFX_TCA_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regGCEA_UE_ERR_STATUS_LO, regGCEA_UE_ERR_STATUS_HI),
-	    16, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "GCEA"},
-	    AMDGPU_GFX_GCEA_MEM, 1},
-	{{AMDGPU_RAS_REG_ENTRY(GC, 0, regLDS_UE_ERR_STATUS_LO, regLDS_UE_ERR_STATUS_HI),
-	    10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "LDS"},
-	    AMDGPU_GFX_LDS_MEM, 4},
-};
-
-static void gfx_v9_4_3_inst_query_ras_err_count(struct amdgpu_device *adev,
-					void *ras_error_status, int xcc_id)
-{
-	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
-	unsigned long ce_count = 0, ue_count = 0;
-	uint32_t i, j, k;
-
-	/* NOTE: convert xcc_id to physical XCD ID (XCD0 or XCD1) */
-	struct amdgpu_smuio_mcm_config_info mcm_info = {
-		.socket_id = adev->smuio.funcs->get_socket_id(adev),
-		.die_id = xcc_id & 0x01 ? 1 : 0,
-	};
-
-	mutex_lock(&adev->grbm_idx_mutex);
-
-	for (i = 0; i < ARRAY_SIZE(gfx_v9_4_3_ce_reg_list); i++) {
-		for (j = 0; j < gfx_v9_4_3_ce_reg_list[i].se_num; j++) {
-			for (k = 0; k < gfx_v9_4_3_ce_reg_list[i].reg_entry.reg_inst; k++) {
-				/* no need to select if instance number is 1 */
-				if (gfx_v9_4_3_ce_reg_list[i].se_num > 1 ||
-				    gfx_v9_4_3_ce_reg_list[i].reg_entry.reg_inst > 1)
-					gfx_v9_4_3_xcc_select_se_sh(adev, j, 0, k, xcc_id);
-
-				amdgpu_ras_inst_query_ras_error_count(adev,
-					&(gfx_v9_4_3_ce_reg_list[i].reg_entry),
-					1,
-					gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ce_reg_list[i].mem_id_type].mem_id_ent,
-					gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ce_reg_list[i].mem_id_type].size,
-					GET_INST(GC, xcc_id),
-					AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE,
-					&ce_count);
-
-				amdgpu_ras_inst_query_ras_error_count(adev,
-					&(gfx_v9_4_3_ue_reg_list[i].reg_entry),
-					1,
-					gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ue_reg_list[i].mem_id_type].mem_id_ent,
-					gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ue_reg_list[i].mem_id_type].size,
-					GET_INST(GC, xcc_id),
-					AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
-					&ue_count);
-			}
-		}
-	}
-
-	/* handle extra register entries of UE */
-	for (; i < ARRAY_SIZE(gfx_v9_4_3_ue_reg_list); i++) {
-		for (j = 0; j < gfx_v9_4_3_ue_reg_list[i].se_num; j++) {
-			for (k = 0; k < gfx_v9_4_3_ue_reg_list[i].reg_entry.reg_inst; k++) {
-				/* no need to select if instance number is 1 */
-				if (gfx_v9_4_3_ue_reg_list[i].se_num > 1 ||
-					gfx_v9_4_3_ue_reg_list[i].reg_entry.reg_inst > 1)
-					gfx_v9_4_3_xcc_select_se_sh(adev, j, 0, k, xcc_id);
-
-				amdgpu_ras_inst_query_ras_error_count(adev,
-					&(gfx_v9_4_3_ue_reg_list[i].reg_entry),
-					1,
-					gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ue_reg_list[i].mem_id_type].mem_id_ent,
-					gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ue_reg_list[i].mem_id_type].size,
-					GET_INST(GC, xcc_id),
-					AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
-					&ue_count);
-			}
-		}
-	}
-
-	gfx_v9_4_3_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff,
-			xcc_id);
-	mutex_unlock(&adev->grbm_idx_mutex);
-
-	/* the caller should make sure initialize value of
-	 * err_data->ue_count and err_data->ce_count
-	 */
-	amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, ue_count);
-	amdgpu_ras_error_statistic_ce_count(err_data, &mcm_info, ce_count);
-}
-
-static void gfx_v9_4_3_inst_reset_ras_err_count(struct amdgpu_device *adev,
-					void *ras_error_status, int xcc_id)
-{
-	uint32_t i, j, k;
-
-	mutex_lock(&adev->grbm_idx_mutex);
-
-	for (i = 0; i < ARRAY_SIZE(gfx_v9_4_3_ce_reg_list); i++) {
-		for (j = 0; j < gfx_v9_4_3_ce_reg_list[i].se_num; j++) {
-			for (k = 0; k < gfx_v9_4_3_ce_reg_list[i].reg_entry.reg_inst; k++) {
-				/* no need to select if instance number is 1 */
-				if (gfx_v9_4_3_ce_reg_list[i].se_num > 1 ||
-				    gfx_v9_4_3_ce_reg_list[i].reg_entry.reg_inst > 1)
-					gfx_v9_4_3_xcc_select_se_sh(adev, j, 0, k, xcc_id);
-
-				amdgpu_ras_inst_reset_ras_error_count(adev,
-					&(gfx_v9_4_3_ce_reg_list[i].reg_entry),
-					1,
-					GET_INST(GC, xcc_id));
-
-				amdgpu_ras_inst_reset_ras_error_count(adev,
-					&(gfx_v9_4_3_ue_reg_list[i].reg_entry),
-					1,
-					GET_INST(GC, xcc_id));
-			}
-		}
-	}
-
-	/* handle extra register entries of UE */
-	for (; i < ARRAY_SIZE(gfx_v9_4_3_ue_reg_list); i++) {
-		for (j = 0; j < gfx_v9_4_3_ue_reg_list[i].se_num; j++) {
-			for (k = 0; k < gfx_v9_4_3_ue_reg_list[i].reg_entry.reg_inst; k++) {
-				/* no need to select if instance number is 1 */
-				if (gfx_v9_4_3_ue_reg_list[i].se_num > 1 ||
-					gfx_v9_4_3_ue_reg_list[i].reg_entry.reg_inst > 1)
-					gfx_v9_4_3_xcc_select_se_sh(adev, j, 0, k, xcc_id);
-
-				amdgpu_ras_inst_reset_ras_error_count(adev,
-					&(gfx_v9_4_3_ue_reg_list[i].reg_entry),
-					1,
-					GET_INST(GC, xcc_id));
-			}
-		}
-	}
-
-	gfx_v9_4_3_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff,
-			xcc_id);
-	mutex_unlock(&adev->grbm_idx_mutex);
-}
-
 static void gfx_v9_4_3_inst_enable_watchdog_timer(struct amdgpu_device *adev,
 					void *ras_error_status, int xcc_id)
 {
@@ -4607,18 +3758,6 @@ static void gfx_v9_4_3_inst_enable_watchdog_timer(struct amdgpu_device *adev,
 	mutex_unlock(&adev->grbm_idx_mutex);
 }
 
-static void gfx_v9_4_3_query_ras_error_count(struct amdgpu_device *adev,
-					void *ras_error_status)
-{
-	amdgpu_gfx_ras_error_func(adev, ras_error_status,
-			gfx_v9_4_3_inst_query_ras_err_count);
-}
-
-static void gfx_v9_4_3_reset_ras_error_count(struct amdgpu_device *adev)
-{
-	amdgpu_gfx_ras_error_func(adev, NULL, gfx_v9_4_3_inst_reset_ras_err_count);
-}
-
 static void gfx_v9_4_3_enable_watchdog_timer(struct amdgpu_device *adev)
 {
 	amdgpu_gfx_ras_error_func(adev, NULL, gfx_v9_4_3_inst_enable_watchdog_timer);
@@ -5099,37 +4238,9 @@ struct amdgpu_xcp_ip_funcs gfx_v9_4_3_xcp_funcs = {
 	.resume = &gfx_v9_4_3_xcp_resume
 };
 
-struct amdgpu_ras_block_hw_ops  gfx_v9_4_3_ras_ops = {
-	.query_ras_error_count = &gfx_v9_4_3_query_ras_error_count,
-	.reset_ras_error_count = &gfx_v9_4_3_reset_ras_error_count,
-};
-
-static int gfx_v9_4_3_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
-{
-	int r;
-
-	r = amdgpu_ras_block_late_init(adev, ras_block);
-	if (r)
-		return r;
-
-	r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__GFX,
-				&gfx_v9_4_3_aca_info,
-				NULL);
-	if (r)
-		goto late_fini;
-
-	return 0;
-
-late_fini:
-	amdgpu_ras_block_late_fini(adev, ras_block);
-
-	return r;
-}
-
 struct amdgpu_gfx_ras gfx_v9_4_3_ras = {
 	.ras_block = {
-		.hw_ops = &gfx_v9_4_3_ras_ops,
-		.ras_late_init = &gfx_v9_4_3_ras_late_init,
+		.hw_ops = NULL,
 	},
 	.enable_watchdog_timer = &gfx_v9_4_3_enable_watchdog_timer,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index a914dd8183b58..b49098931d197 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -124,7 +124,7 @@ static int gmc_v6_0_init_microcode(struct amdgpu_device *adev)
 		chip_name = "hainan";
 		break;
 	default:
-		BUG();
+		return -EINVAL;
 	}
 
 	/* this memory configuration requires special firmware */
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index c2a41fa3a396b..64ebedc595b50 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -167,44 +167,6 @@ static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev)
 	}
 }
 
-static void gmc_v8_0_mc_stop(struct amdgpu_device *adev)
-{
-	u32 blackout;
-	struct amdgpu_ip_block *ip_block;
-
-	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GMC);
-	if (!ip_block)
-		return;
-
-	gmc_v8_0_wait_for_idle(ip_block);
-
-	blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
-	if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
-		/* Block CPU access */
-		WREG32(mmBIF_FB_EN, 0);
-		/* blackout the MC */
-		blackout = REG_SET_FIELD(blackout,
-					 MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 1);
-		WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout);
-	}
-	/* wait for the MC to settle */
-	udelay(100);
-}
-
-static void gmc_v8_0_mc_resume(struct amdgpu_device *adev)
-{
-	u32 tmp;
-
-	/* unblackout the MC */
-	tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
-	tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
-	WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp);
-	/* allow CPU access */
-	tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
-	tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
-	WREG32(mmBIF_FB_EN, tmp);
-}
-
 /**
  * gmc_v8_0_init_microcode - load ucode images from disk
  *
@@ -1293,89 +1255,6 @@ static int gmc_v8_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 
 }
 
-static bool gmc_v8_0_check_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	u32 srbm_soft_reset = 0;
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 tmp = RREG32(mmSRBM_STATUS);
-
-	if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
-		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
-						SRBM_SOFT_RESET, SOFT_RESET_VMC, 1);
-
-	if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
-		   SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
-		if (!(adev->flags & AMD_IS_APU))
-			srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
-							SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
-	}
-
-	if (srbm_soft_reset) {
-		adev->gmc.srbm_soft_reset = srbm_soft_reset;
-		return true;
-	}
-
-	adev->gmc.srbm_soft_reset = 0;
-
-	return false;
-}
-
-static int gmc_v8_0_pre_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-
-	if (!adev->gmc.srbm_soft_reset)
-		return 0;
-
-	gmc_v8_0_mc_stop(adev);
-	if (gmc_v8_0_wait_for_idle(ip_block))
-		dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
-
-	return 0;
-}
-
-static int gmc_v8_0_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 srbm_soft_reset;
-
-	if (!adev->gmc.srbm_soft_reset)
-		return 0;
-	srbm_soft_reset = adev->gmc.srbm_soft_reset;
-
-	if (srbm_soft_reset) {
-		u32 tmp;
-
-		tmp = RREG32(mmSRBM_SOFT_RESET);
-		tmp |= srbm_soft_reset;
-		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
-		WREG32(mmSRBM_SOFT_RESET, tmp);
-		tmp = RREG32(mmSRBM_SOFT_RESET);
-
-		udelay(50);
-
-		tmp &= ~srbm_soft_reset;
-		WREG32(mmSRBM_SOFT_RESET, tmp);
-		tmp = RREG32(mmSRBM_SOFT_RESET);
-
-		/* Wait a little for things to settle down */
-		udelay(50);
-	}
-
-	return 0;
-}
-
-static int gmc_v8_0_post_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-
-	if (!adev->gmc.srbm_soft_reset)
-		return 0;
-
-	gmc_v8_0_mc_resume(adev);
-	return 0;
-}
-
 static int gmc_v8_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
 					     struct amdgpu_irq_src *src,
 					     unsigned int type,
@@ -1715,10 +1594,6 @@ static const struct amd_ip_funcs gmc_v8_0_ip_funcs = {
 	.resume = gmc_v8_0_resume,
 	.is_idle = gmc_v8_0_is_idle,
 	.wait_for_idle = gmc_v8_0_wait_for_idle,
-	.check_soft_reset = gmc_v8_0_check_soft_reset,
-	.pre_soft_reset = gmc_v8_0_pre_soft_reset,
-	.soft_reset = gmc_v8_0_soft_reset,
-	.post_soft_reset = gmc_v8_0_post_soft_reset,
 	.set_clockgating_state = gmc_v8_0_set_clockgating_state,
 	.set_powergating_state = gmc_v8_0_set_powergating_state,
 	.get_clockgating_state = gmc_v8_0_get_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 8a5c44810ba1e..1fcc0594fd0a9 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -57,6 +57,7 @@
 #include "umc_v6_0.h"
 #include "umc_v6_7.h"
 #include "umc_v12_0.h"
+#include "ras_umc_v12_0.h"
 #include "hdp_v4_0.h"
 #include "mca_v3_0.h"
 
@@ -1382,7 +1383,7 @@ static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
 	case IP_VERSION(12, 0, 0):
 	case IP_VERSION(12, 5, 0):
 		adev->umc.max_ras_err_cnt_per_query =
-			UMC_V12_0_TOTAL_CHANNEL_NUM(adev) * UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL;
+			UMC_V12_0_TOTAL_CHANNEL_NUM * UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL;
 		adev->umc.channel_inst_num = UMC_V12_0_CHANNEL_INSTANCE_NUM;
 		adev->umc.umc_inst_num = UMC_V12_0_UMC_INSTANCE_NUM;
 		adev->umc.node_inst_num /= UMC_V12_0_UMC_INSTANCE_NUM;
@@ -2025,11 +2026,19 @@ static int gmc_v9_0_sw_init(struct amdgpu_ip_block *ip_block)
 	 * The first KFD VMID is 8 for GPUs with graphics, 3 for
 	 * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs
 	 * for video processing.
+	 *
+	 * If kernel queues are disabled, allow KFD to use all vmids.
 	 */
-	adev->vm_manager.first_kfd_vmid =
-		(amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
-		 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
-		 amdgpu_is_multi_aid(adev)) ?
+	if (adev->gfx.disable_kq &&
+	    adev->jpeg.disable_kq &&
+	    adev->vcn.disable_kq &&
+	    adev->sdma.no_user_submission)
+		adev->vm_manager.first_kfd_vmid = 1;
+	else
+		adev->vm_manager.first_kfd_vmid =
+			(amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
+			 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
+			 amdgpu_is_multi_aid(adev)) ?
 			3 :
 			8;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c
index cffb1e6bab353..a9c00adcfe410 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c
@@ -747,7 +747,7 @@ static int jpeg_v2_0_set_interrupt_state(struct amdgpu_device *adev,
 	return 0;
 }
 
-static int jpeg_v2_0_process_interrupt(struct amdgpu_device *adev,
+int jpeg_v2_0_process_interrupt(struct amdgpu_device *adev,
 				      struct amdgpu_irq_src *source,
 				      struct amdgpu_iv_entry *entry)
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.h b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.h
index 654e43e83e2c4..4f400fb476f1a 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.h
@@ -58,6 +58,10 @@ void jpeg_v2_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
 void jpeg_v2_0_dec_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val);
 void jpeg_v2_0_dec_ring_nop(struct amdgpu_ring *ring, uint32_t count);
 
+int jpeg_v2_0_process_interrupt(struct amdgpu_device *adev,
+				struct amdgpu_irq_src *source,
+				struct amdgpu_iv_entry *entry);
+
 extern const struct amdgpu_ip_block_version jpeg_v2_0_ip_block;
 
 #endif /* __JPEG_V2_0_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c
index d0445df39d2c0..e9ccadeb76961 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c
@@ -539,25 +539,6 @@ static int jpeg_v3_0_set_interrupt_state(struct amdgpu_device *adev,
 	return 0;
 }
 
-static int jpeg_v3_0_process_interrupt(struct amdgpu_device *adev,
-				      struct amdgpu_irq_src *source,
-				      struct amdgpu_iv_entry *entry)
-{
-	DRM_DEBUG("IH: JPEG TRAP\n");
-
-	switch (entry->src_id) {
-	case VCN_2_0__SRCID__JPEG_DECODE:
-		amdgpu_fence_process(adev->jpeg.inst->ring_dec);
-		break;
-	default:
-		DRM_ERROR("Unhandled interrupt: %d %d\n",
-			  entry->src_id, entry->src_data[0]);
-		break;
-	}
-
-	return 0;
-}
-
 static int jpeg_v3_0_ring_reset(struct amdgpu_ring *ring,
 				unsigned int vmid,
 				struct amdgpu_fence *timedout_fence)
@@ -630,7 +611,7 @@ static void jpeg_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev)
 
 static const struct amdgpu_irq_src_funcs jpeg_v3_0_irq_funcs = {
 	.set = jpeg_v3_0_set_interrupt_state,
-	.process = jpeg_v3_0_process_interrupt,
+	.process = jpeg_v2_0_process_interrupt,
 };
 
 static void jpeg_v3_0_set_irq_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
index d8204fbc198d6..0fdc32b3ae91b 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
@@ -119,6 +119,19 @@ static int jpeg_v4_0_3_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
+	switch (amdgpu_user_queue) {
+	case -1:
+	case 0:
+	default:
+		adev->jpeg.disable_kq = false;
+		adev->jpeg.disable_uq = true;
+		break;
+	case 2:
+		adev->jpeg.disable_kq = true;
+		adev->jpeg.disable_uq = true;
+		break;
+	}
+
 	adev->jpeg.num_jpeg_rings = AMDGPU_MAX_JPEG_RINGS_4_0_3;
 
 	jpeg_v4_0_3_set_dec_ring_funcs(adev);
@@ -175,6 +188,10 @@ static int jpeg_v4_0_3_sw_init(struct amdgpu_ip_block *ip_block)
 		for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 			ring = &adev->jpeg.inst[i].ring_dec[j];
 			ring->use_doorbell = true;
+			if (adev->jpeg.disable_kq) {
+				ring->no_scheduler = true;
+				ring->no_user_submission = true;
+			}
 			ring->vm_hub = AMDGPU_MMHUB0(adev->jpeg.inst[i].aid_id);
 			if (!amdgpu_sriov_vf(adev)) {
 				ring->doorbell_index =
@@ -1425,72 +1442,6 @@ static const struct amdgpu_ras_block_hw_ops jpeg_v4_0_3_ras_hw_ops = {
 	.query_poison_status = jpeg_v4_0_3_query_ras_poison_status,
 };
 
-static int jpeg_v4_0_3_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
-				      enum aca_smu_type type, void *data)
-{
-	struct aca_bank_info info;
-	u64 misc0;
-	int ret;
-
-	ret = aca_bank_info_decode(bank, &info);
-	if (ret)
-		return ret;
-
-	misc0 = bank->regs[ACA_REG_IDX_MISC0];
-	switch (type) {
-	case ACA_SMU_TYPE_UE:
-		bank->aca_err_type = ACA_ERROR_TYPE_UE;
-		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE,
-						     1ULL);
-		break;
-	case ACA_SMU_TYPE_CE:
-		bank->aca_err_type = ACA_ERROR_TYPE_CE;
-		ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type,
-						     ACA_REG__MISC0__ERRCNT(misc0));
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-/* reference to smu driver if header file */
-static int jpeg_v4_0_3_err_codes[] = {
-	16, 17, 18, 19, 20, 21, 22, 23, /* JPEG[0-7][S|D] */
-	24, 25, 26, 27, 28, 29, 30, 31
-};
-
-static bool jpeg_v4_0_3_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
-					 enum aca_smu_type type, void *data)
-{
-	u32 instlo;
-
-	instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
-	instlo &= GENMASK(31, 1);
-
-	if (instlo != mmSMNAID_AID0_MCA_SMU)
-		return false;
-
-	if (aca_bank_check_error_codes(handle->adev, bank,
-				       jpeg_v4_0_3_err_codes,
-				       ARRAY_SIZE(jpeg_v4_0_3_err_codes)))
-		return false;
-
-	return true;
-}
-
-static const struct aca_bank_ops jpeg_v4_0_3_aca_bank_ops = {
-	.aca_bank_parser = jpeg_v4_0_3_aca_bank_parser,
-	.aca_bank_is_valid = jpeg_v4_0_3_aca_bank_is_valid,
-};
-
-static const struct aca_info jpeg_v4_0_3_aca_info = {
-	.hwip = ACA_HWIP_TYPE_SMU,
-	.mask = ACA_ERROR_UE_MASK,
-	.bank_ops = &jpeg_v4_0_3_aca_bank_ops,
-};
-
 static int jpeg_v4_0_3_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
 {
 	int r;
@@ -1506,11 +1457,6 @@ static int jpeg_v4_0_3_ras_late_init(struct amdgpu_device *adev, struct ras_comm
 			goto late_fini;
 	}
 
-	r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__JPEG,
-				&jpeg_v4_0_3_aca_info, NULL);
-	if (r)
-		goto late_fini;
-
 	return 0;
 
 late_fini:
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c
index 72a4b2d0676fa..1cdbe353ae400 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c
@@ -625,7 +625,7 @@ static int jpeg_v5_0_0_set_interrupt_state(struct amdgpu_device *adev,
 	return 0;
 }
 
-static int jpeg_v5_0_0_process_interrupt(struct amdgpu_device *adev,
+int jpeg_v5_0_0_process_interrupt(struct amdgpu_device *adev,
 				      struct amdgpu_irq_src *source,
 				      struct amdgpu_iv_entry *entry)
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.h b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.h
index 5abb961598145..4eeb0c14702ca 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.h
@@ -32,4 +32,8 @@
 
 extern const struct amdgpu_ip_block_version jpeg_v5_0_0_ip_block;
 
+int jpeg_v5_0_0_process_interrupt(struct amdgpu_device *adev,
+				  struct amdgpu_irq_src *source,
+				  struct amdgpu_iv_entry *entry);
+
 #endif /* __JPEG_V5_0_0_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c
index ae3afc7ab3267..db3fbe4eabb99 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c
@@ -118,6 +118,19 @@ static int jpeg_v5_0_1_early_init(struct amdgpu_ip_block *ip_block)
 	if (!adev->jpeg.num_jpeg_inst || adev->jpeg.num_jpeg_inst > AMDGPU_MAX_JPEG_INSTANCES)
 		return -ENOENT;
 
+	switch (amdgpu_user_queue) {
+	case -1:
+	case 0:
+	default:
+		adev->jpeg.disable_kq = false;
+		adev->jpeg.disable_uq = true;
+		break;
+	case 2:
+		adev->jpeg.disable_kq = true;
+		adev->jpeg.disable_uq = true;
+		break;
+	}
+
 	adev->jpeg.num_jpeg_rings = AMDGPU_MAX_JPEG_RINGS;
 	jpeg_v5_0_1_set_dec_ring_funcs(adev);
 	jpeg_v5_0_1_set_irq_funcs(adev);
@@ -172,6 +185,10 @@ static int jpeg_v5_0_1_sw_init(struct amdgpu_ip_block *ip_block)
 		for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 			ring = &adev->jpeg.inst[i].ring_dec[j];
 			ring->use_doorbell = true;
+			if (adev->jpeg.disable_kq) {
+				ring->no_scheduler = true;
+				ring->no_user_submission = true;
+			}
 			ring->vm_hub = AMDGPU_MMHUB0(adev->jpeg.inst[i].aid_id);
 			if (!amdgpu_sriov_vf(adev)) {
 				ring->doorbell_index =
@@ -755,7 +772,7 @@ static int jpeg_v5_0_1_set_ras_interrupt_state(struct amdgpu_device *adev,
 
 
 
-static int jpeg_v5_0_1_process_interrupt(struct amdgpu_device *adev,
+int jpeg_v5_0_1_process_interrupt(struct amdgpu_device *adev,
 					 struct amdgpu_irq_src *source,
 					 struct amdgpu_iv_entry *entry)
 {
@@ -871,10 +888,7 @@ static const struct amd_ip_funcs jpeg_v5_0_1_ip_funcs = {
 	.resume = jpeg_v5_0_1_resume,
 	.is_idle = jpeg_v5_0_1_is_idle,
 	.wait_for_idle = jpeg_v5_0_1_wait_for_idle,
-	.check_soft_reset = NULL,
-	.pre_soft_reset = NULL,
 	.soft_reset = NULL,
-	.post_soft_reset = NULL,
 	.set_clockgating_state = jpeg_v5_0_1_set_clockgating_state,
 	.set_powergating_state = jpeg_v5_0_1_set_powergating_state,
 	.dump_ip_state = amdgpu_jpeg_dump_ip_state,
@@ -1003,73 +1017,6 @@ static const struct amdgpu_ras_block_hw_ops jpeg_v5_0_1_ras_hw_ops = {
 	.query_poison_status = jpeg_v5_0_1_query_ras_poison_status,
 };
 
-static int jpeg_v5_0_1_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
-				      enum aca_smu_type type, void *data)
-{
-	struct aca_bank_info info;
-	u64 misc0;
-	int ret;
-
-	ret = aca_bank_info_decode(bank, &info);
-	if (ret)
-		return ret;
-
-	misc0 = bank->regs[ACA_REG_IDX_MISC0];
-	switch (type) {
-	case ACA_SMU_TYPE_UE:
-		bank->aca_err_type = ACA_ERROR_TYPE_UE;
-		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE,
-						     1ULL);
-		break;
-	case ACA_SMU_TYPE_CE:
-		bank->aca_err_type = ACA_ERROR_TYPE_CE;
-		ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type,
-						     ACA_REG__MISC0__ERRCNT(misc0));
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-/* reference to smu driver if header file */
-static int jpeg_v5_0_1_err_codes[] = {
-	16, 17, 18, 19, 20, 21, 22, 23, /* JPEG[0-9][S|D] */
-	24, 25, 26, 27, 28, 29, 30, 31,
-	48, 49, 50, 51,
-};
-
-static bool jpeg_v5_0_1_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
-					 enum aca_smu_type type, void *data)
-{
-	u32 instlo;
-
-	instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
-	instlo &= GENMASK(31, 1);
-
-	if (instlo != mmSMNAID_AID0_MCA_SMU)
-		return false;
-
-	if (aca_bank_check_error_codes(handle->adev, bank,
-				       jpeg_v5_0_1_err_codes,
-				       ARRAY_SIZE(jpeg_v5_0_1_err_codes)))
-		return false;
-
-	return true;
-}
-
-static const struct aca_bank_ops jpeg_v5_0_1_aca_bank_ops = {
-	.aca_bank_parser = jpeg_v5_0_1_aca_bank_parser,
-	.aca_bank_is_valid = jpeg_v5_0_1_aca_bank_is_valid,
-};
-
-static const struct aca_info jpeg_v5_0_1_aca_info = {
-	.hwip = ACA_HWIP_TYPE_SMU,
-	.mask = ACA_ERROR_UE_MASK,
-	.bank_ops = &jpeg_v5_0_1_aca_bank_ops,
-};
-
 static int jpeg_v5_0_1_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
 {
 	int r;
@@ -1078,11 +1025,6 @@ static int jpeg_v5_0_1_ras_late_init(struct amdgpu_device *adev, struct ras_comm
 	if (r)
 		return r;
 
-	r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__JPEG,
-				&jpeg_v5_0_1_aca_info, NULL);
-	if (r)
-		goto late_fini;
-
 	if (amdgpu_ras_is_supported(adev, ras_block->block) &&
 		adev->jpeg.inst->ras_poison_irq.funcs) {
 		r = amdgpu_irq_get(adev, &adev->jpeg.inst->ras_poison_irq, 0);
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.h b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.h
index a7e58d5fb2465..67346faecb473 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.h
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.h
@@ -108,4 +108,12 @@ enum amdgpu_jpeg_v5_0_1_sub_block {
 	AMDGPU_JPEG_V5_0_1_MAX_SUB_BLOCK,
 };
 
+struct amdgpu_irq_src;
+struct amdgpu_iv_entry;
+struct amdgpu_device;
+
+int jpeg_v5_0_1_process_interrupt(struct amdgpu_device *adev,
+					struct amdgpu_irq_src *source,
+					struct amdgpu_iv_entry *entry);
+
 #endif /* __JPEG_V5_0_1_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c
index 7a4ecea6b39a4..ea54a2b6eddb8 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c
@@ -34,6 +34,8 @@
 #include "vcn/vcn_5_0_0_sh_mask.h"
 #include "ivsrcid/vcn/irqsrcs_vcn_5_0.h"
 
+#include "jpeg_v5_0_1.h"
+
 static void jpeg_v5_0_2_set_dec_ring_funcs(struct amdgpu_device *adev);
 static void jpeg_v5_0_2_set_irq_funcs(struct amdgpu_device *adev);
 static int jpeg_v5_0_2_set_powergating_state(struct amdgpu_ip_block *ip_block,
@@ -583,65 +585,6 @@ static int jpeg_v5_0_2_set_interrupt_state(struct amdgpu_device *adev,
 	return 0;
 }
 
-static int jpeg_v5_0_2_process_interrupt(struct amdgpu_device *adev,
-					 struct amdgpu_irq_src *source,
-					 struct amdgpu_iv_entry *entry)
-{
-	u32 i, inst;
-
-	i = node_id_to_phys_map[entry->node_id];
-	DRM_DEV_DEBUG(adev->dev, "IH: JPEG TRAP\n");
-
-	for (inst = 0; inst < adev->jpeg.num_jpeg_inst; ++inst)
-		if (adev->jpeg.inst[inst].aid_id == i)
-			break;
-
-	if (inst >= adev->jpeg.num_jpeg_inst) {
-		dev_WARN_ONCE(adev->dev, 1,
-			      "Interrupt received for unknown JPEG instance %d",
-			      entry->node_id);
-		return 0;
-	}
-
-	switch (entry->src_id) {
-	case VCN_5_0__SRCID__JPEG_DECODE:
-		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[0]);
-		break;
-	case VCN_5_0__SRCID__JPEG1_DECODE:
-		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[1]);
-		break;
-	case VCN_5_0__SRCID__JPEG2_DECODE:
-		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[2]);
-		break;
-	case VCN_5_0__SRCID__JPEG3_DECODE:
-		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[3]);
-		break;
-	case VCN_5_0__SRCID__JPEG4_DECODE:
-		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[4]);
-		break;
-	case VCN_5_0__SRCID__JPEG5_DECODE:
-		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[5]);
-		break;
-	case VCN_5_0__SRCID__JPEG6_DECODE:
-		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[6]);
-		break;
-	case VCN_5_0__SRCID__JPEG7_DECODE:
-		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[7]);
-		break;
-	case VCN_5_0__SRCID__JPEG8_DECODE:
-		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[8]);
-		break;
-	case VCN_5_0__SRCID__JPEG9_DECODE:
-		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[9]);
-		break;
-	default:
-		DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n",
-			      entry->src_id, entry->src_data[0]);
-		break;
-	}
-
-	return 0;
-}
 
 static void jpeg_v5_0_2_core_stall_reset(struct amdgpu_ring *ring)
 {
@@ -690,10 +633,7 @@ static const struct amd_ip_funcs jpeg_v5_0_2_ip_funcs = {
 	.resume = jpeg_v5_0_2_resume,
 	.is_idle = jpeg_v5_0_2_is_idle,
 	.wait_for_idle = jpeg_v5_0_2_wait_for_idle,
-	.check_soft_reset = NULL,
-	.pre_soft_reset = NULL,
 	.soft_reset = NULL,
-	.post_soft_reset = NULL,
 	.set_clockgating_state = jpeg_v5_0_2_set_clockgating_state,
 	.set_powergating_state = jpeg_v5_0_2_set_powergating_state,
 	.dump_ip_state = amdgpu_jpeg_dump_ip_state,
@@ -750,7 +690,7 @@ static void jpeg_v5_0_2_set_dec_ring_funcs(struct amdgpu_device *adev)
 
 static const struct amdgpu_irq_src_funcs jpeg_v5_0_2_irq_funcs = {
 	.set = jpeg_v5_0_2_set_interrupt_state,
-	.process = jpeg_v5_0_2_process_interrupt,
+	.process = jpeg_v5_0_1_process_interrupt,
 };
 
 static void jpeg_v5_0_2_set_irq_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c
index e7546816baba3..68152b2ce015b 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c
@@ -32,6 +32,7 @@
 #include "vcn/vcn_5_3_0_offset.h"
 #include "vcn/vcn_5_3_0_sh_mask.h"
 #include "ivsrcid/vcn/irqsrcs_vcn_5_0.h"
+#include "jpeg_v5_0_0.h"
 #include "jpeg_v5_3_0.h"
 
 static void jpeg_v5_3_0_set_dec_ring_funcs(struct amdgpu_device *adev);
@@ -398,6 +399,25 @@ static void jpeg_v5_3_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
 	WREG32_SOC15(JPEG, inst_idx, regUVD_JPEG_POWER_STATUS, reg_data);
 }
 
+/**
+ * jpeg_v5_3_0_set_mmhub_eco_sec_level - set jpeg sec lvl reg
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * request psp to set secure lvl
+ */
+static int jpeg_v5_3_0_set_mmhub_eco_sec_level(struct amdgpu_device *adev)
+{
+	int r = 0;
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+		/* Request to PSP to program JPEG secure lvl */
+		r = psp_set_mmhub_eco_sec_level(adev);
+	}
+
+	return r;
+}
+
 /**
  * jpeg_v5_3_0_start - start JPEG block
  *
@@ -423,6 +443,11 @@ static int jpeg_v5_3_0_start(struct amdgpu_device *adev)
 	if (r)
 		return r;
 
+	/* program JPEG secure lvl register */
+	r = jpeg_v5_3_0_set_mmhub_eco_sec_level(adev);
+	if (r)
+		return r;
+
 	/* JPEG disable CGC */
 	jpeg_v5_3_0_disable_clock_gating(adev);
 
@@ -608,25 +633,6 @@ static int jpeg_v5_3_0_set_interrupt_state(struct amdgpu_device *adev,
 	return 0;
 }
 
-static int jpeg_v5_3_0_process_interrupt(struct amdgpu_device *adev,
-				      struct amdgpu_irq_src *source,
-				      struct amdgpu_iv_entry *entry)
-{
-	DRM_DEBUG("IH: JPEG TRAP\n");
-
-	switch (entry->src_id) {
-	case VCN_5_0__SRCID__JPEG_DECODE:
-		amdgpu_fence_process(adev->jpeg.inst->ring_dec);
-		break;
-	default:
-		DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n",
-			  entry->src_id, entry->src_data[0]);
-		break;
-	}
-
-	return 0;
-}
-
 static int jpeg_v5_3_0_ring_reset(struct amdgpu_ring *ring,
 				  unsigned int vmid,
 				  struct amdgpu_fence *timedout_fence)
@@ -697,7 +703,7 @@ static void jpeg_v5_3_0_set_dec_ring_funcs(struct amdgpu_device *adev)
 
 static const struct amdgpu_irq_src_funcs jpeg_v5_3_0_irq_funcs = {
 	.set = jpeg_v5_3_0_set_interrupt_state,
-	.process = jpeg_v5_3_0_process_interrupt,
+	.process = jpeg_v5_0_0_process_interrupt,
 };
 
 static void jpeg_v5_3_0_set_irq_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
index 16625c31bfd32..639e4a37f872d 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
@@ -70,27 +70,23 @@ mes_userq_create_wptr_mapping(struct amdgpu_device *adev,
 		ret = -EINVAL;
 		goto fail_map;
 	}
-
-	/* TODO use eviction fence instead of pinning. */
-	ret = amdgpu_bo_pin(wptr_obj->obj, AMDGPU_GEM_DOMAIN_GTT);
+	/* Keep WPTR BO under eviction-fence control instead of pinning. */
+	ret = amdgpu_evf_mgr_attach_fence(&uq_mgr_to_fpriv(uq_mgr)->evf_mgr, wptr_obj->obj);
 	if (ret) {
-		DRM_ERROR("Failed to pin wptr bo. ret %d\n", ret);
+		DRM_ERROR("Failed to attach eviction fence to wptr bo. ret %d\n", ret);
 		goto fail_map;
 	}
 
 	ret = amdgpu_ttm_alloc_gart(&wptr_obj->obj->tbo);
 	if (ret) {
-		DRM_ERROR("Failed to bind bo to GART. ret %d\n", ret);
-		goto fail_alloc_gart;
+		DRM_ERROR("Failed to bind wptr bo to GART. ret %d\n", ret);
+		goto fail_map;
 	}
 
 	queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(wptr_obj->obj);
 
 	drm_exec_fini(&exec);
 	return 0;
-
-fail_alloc_gart:
-	amdgpu_bo_unpin(wptr_obj->obj);
 fail_map:
 	amdgpu_bo_unref(&wptr_obj->obj);
 fail_lock:
@@ -121,6 +117,7 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
 	struct amdgpu_userq_obj *ctx = &queue->fw_obj;
 	struct amdgpu_mqd_prop *userq_props = queue->userq_prop;
 	struct mes_add_queue_input queue_input;
+	struct amdgpu_mes *mes = &adev->mes;
 	int r;
 
 	memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input));
@@ -133,8 +130,8 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
 	queue_input.gang_quantum = 10000;
 	queue_input.paging = false;
 
-	queue_input.process_context_addr = ctx->gpu_addr;
-	queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
+	queue_input.process_context_addr = uq_mgr->proc_ctx_obj.gpu_addr;
+	queue_input.gang_context_addr = ctx->gpu_addr;
 	queue_input.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
 	queue_input.gang_global_priority_level = convert_to_mes_priority(queue->priority);
 
@@ -146,7 +143,12 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
 	queue_input.doorbell_offset = userq_props->doorbell_index;
 	queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo);
 	queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr;
-
+	if (mes->use_rs64mem) {
+		amdgpu_mes_alloc_proc_ctx_index(mes, queue);
+		queue_input.process_context_array_index = queue->proc_ctx_array_index;
+		amdgpu_mes_alloc_gang_ctx_index(mes, queue);
+		queue_input.gang_context_array_index = queue->gang_ctx_array_index;
+	}
 	amdgpu_mes_lock(&adev->mes);
 	r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
 	amdgpu_mes_unlock(&adev->mes);
@@ -165,20 +167,82 @@ static int mes_userq_unmap(struct amdgpu_usermode_queue *queue)
 	struct amdgpu_device *adev = uq_mgr->adev;
 	struct mes_remove_queue_input queue_input;
 	struct amdgpu_userq_obj *ctx = &queue->fw_obj;
+	struct amdgpu_mes *mes = &adev->mes;
 	int r;
 
 	memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
+	queue_input.gang_context_array_index = queue->gang_ctx_array_index;
 	queue_input.doorbell_offset = queue->doorbell_index;
-	queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
+	queue_input.gang_context_addr = ctx->gpu_addr;
+	queue_input.queue_type = queue->queue_type;
 
 	amdgpu_mes_lock(&adev->mes);
 	r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);
 	amdgpu_mes_unlock(&adev->mes);
+	if (mes->use_rs64mem) {
+		amdgpu_mes_free_proc_ctx_index(mes, queue);
+		amdgpu_mes_free_gang_ctx_index(mes, queue);
+	}
 	if (r)
 		DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r);
 	return r;
 }
 
+int mes_userq_reset(struct amdgpu_usermode_queue *queue)
+{
+	struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr;
+	struct amdgpu_device *adev = uq_mgr->adev;
+	struct mes_reset_queue_input queue_input;
+	int r;
+
+	/* XXX: add a FW version check for SDMA per queue reset */
+	memset(&queue_input, 0x0, sizeof(struct mes_reset_queue_input));
+	queue_input.doorbell_offset = queue->doorbell_index;
+	queue_input.queue_type = queue->queue_type;
+
+	amdgpu_mes_lock(&adev->mes);
+	r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input);
+	amdgpu_mes_unlock(&adev->mes);
+	if (r)
+		return r;
+	return mes_userq_unmap(queue);
+}
+
+int mes_userq_reset_queue(struct amdgpu_device *adev,
+			  struct amdgpu_usermode_queue *guilty_uq,
+			  int queue_type,
+			  unsigned int pipe,
+			  unsigned int queue,
+			  unsigned int db)
+{
+	struct amdgpu_usermode_queue *uq;
+	bool use_mmio = adev->gfx.mec.use_mmio_for_reset;
+	unsigned long uq_id;
+	int r;
+
+	xa_for_each(&adev->userq_doorbell_xa, uq_id, uq) {
+		if (uq->queue_type == queue_type) {
+			if (uq == guilty_uq)
+				continue;
+			if (uq->doorbell_index == db) {
+				uq->state = AMDGPU_USERQ_STATE_HUNG;
+				if (use_mmio)
+					r = amdgpu_mes_reset_queue_mmio(adev, queue_type, 0, 1, pipe, queue, 0);
+				else
+					r = amdgpu_mes_reset_user_queue(adev, queue_type, db, 0);
+				if (r)
+					return r;
+				r = mes_userq_unmap(uq);
+				if (r)
+					return r;
+				amdgpu_userq_fence_driver_force_completion(uq);
+				break;
+			}
+		}
+	}
+	return 0;
+}
+
 static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr,
 				      struct amdgpu_usermode_queue *queue,
 				      struct drm_amdgpu_userq_in *mqd_user)
@@ -186,12 +250,8 @@ static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr,
 	struct amdgpu_userq_obj *ctx = &queue->fw_obj;
 	int r, size;
 
-	/*
-	 * The FW expects at least one page space allocated for
-	 * process ctx and gang ctx each. Create an object
-	 * for the same.
-	 */
-	size = AMDGPU_USERQ_PROC_CTX_SZ + AMDGPU_USERQ_GANG_CTX_SZ;
+	/* The FW expects at least one page space allocated for gang ctx. */
+	size = AMDGPU_USERQ_GANG_CTX_SZ;
 	r = amdgpu_bo_create_kernel(uq_mgr->adev, size, 0,
 				    AMDGPU_GEM_DOMAIN_GTT,
 				    &ctx->obj, &ctx->gpu_addr,
@@ -205,54 +265,26 @@ static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr,
 	return 0;
 }
 
-static int mes_userq_detect_and_reset(struct amdgpu_device *adev,
-				      int queue_type)
+static int mes_userq_create_proc_ctx_space(struct amdgpu_userq_mgr *uq_mgr)
 {
-	int db_array_size = amdgpu_mes_get_hung_queue_db_array_size(adev);
-	struct mes_detect_and_reset_queue_input input;
-	struct amdgpu_usermode_queue *queue;
-	unsigned int hung_db_num = 0;
-	unsigned long queue_id;
-	u32 db_array[8];
-	bool found_hung_queue = false;
-	int r, i;
-
-	if (db_array_size > 8) {
-		dev_err(adev->dev, "DB array size (%d vs 8) too small\n",
-			db_array_size);
-		return -EINVAL;
-	}
-
-	memset(&input, 0x0, sizeof(struct mes_detect_and_reset_queue_input));
-
-	input.queue_type = queue_type;
+	int r = 0;
 
-	amdgpu_mes_lock(&adev->mes);
-	r = amdgpu_mes_detect_and_reset_hung_queues(adev, queue_type, false,
-						    &hung_db_num, db_array, 0);
-	amdgpu_mes_unlock(&adev->mes);
-	if (r) {
-		dev_err(adev->dev, "Failed to detect and reset queues, err (%d)\n", r);
-	} else if (hung_db_num) {
-		xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
-			if (queue->queue_type == queue_type) {
-				for (i = 0; i < hung_db_num; i++) {
-					if (queue->doorbell_index == db_array[i]) {
-						queue->state = AMDGPU_USERQ_STATE_HUNG;
-						found_hung_queue = true;
-						atomic_inc(&adev->gpu_reset_counter);
-						amdgpu_userq_fence_driver_force_completion(queue);
-						drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, NULL);
-					}
-				}
-			}
-		}
+	mutex_lock(&uq_mgr->proc_ctx_lock);
+	/* This check is a necessary because amdgpu_bo_create_kernel()
+	 * calls helpers like amdgpu_bo_pin() and memset() unconditionally
+	 */
+	if (!uq_mgr->proc_ctx_obj.obj) {
+		r = amdgpu_bo_create_kernel(uq_mgr->adev, AMDGPU_USERQ_PROC_CTX_SZ,
+					    0, AMDGPU_GEM_DOMAIN_GTT,
+					    &uq_mgr->proc_ctx_obj.obj,
+					    &uq_mgr->proc_ctx_obj.gpu_addr,
+					    &uq_mgr->proc_ctx_obj.cpu_ptr);
+
+		if (!r)
+			memset(uq_mgr->proc_ctx_obj.cpu_ptr, 0, AMDGPU_USERQ_PROC_CTX_SZ);
 	}
 
-	if (found_hung_queue) {
-		/* Resume scheduling after hang recovery */
-		r = amdgpu_mes_resume(adev, input.xcc_id);
-	}
+	mutex_unlock(&uq_mgr->proc_ctx_lock);
 
 	return r;
 }
@@ -429,7 +461,14 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue,
 		goto free_mqd;
 	}
 
-	/* Create BO for FW operations */
+	/* Create per-process MES process context BO */
+	r = mes_userq_create_proc_ctx_space(uq_mgr);
+	if (r) {
+		DRM_ERROR("Failed to allocate MES process context space bo, error: %d\n", r);
+		goto free_mqd;
+	}
+
+	/* Create BO of a gang for FW operations */
 	r = mes_userq_create_ctx_space(uq_mgr, queue, mqd_user);
 	if (r) {
 		DRM_ERROR("Failed to allocate BO for userqueue (%d)", r);
@@ -468,9 +507,6 @@ static void mes_userq_mqd_destroy(struct amdgpu_usermode_queue *queue)
 	amdgpu_bo_free_kernel(&queue->mqd.obj, &queue->mqd.gpu_addr,
 			      &queue->mqd.cpu_ptr);
 
-	amdgpu_bo_reserve(queue->wptr_obj.obj, true);
-	amdgpu_bo_unpin(queue->wptr_obj.obj);
-	amdgpu_bo_unreserve(queue->wptr_obj.obj);
 	amdgpu_bo_unref(&queue->wptr_obj.obj);
 }
 
@@ -488,7 +524,7 @@ static int mes_userq_preempt(struct amdgpu_usermode_queue *queue)
 
 	if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
 		return 0;
-	r = amdgpu_device_wb_get(adev, &fence_offset);
+	r = amdgpu_wb_get(adev, &fence_offset);
 	if (r)
 		return r;
 
@@ -497,7 +533,7 @@ static int mes_userq_preempt(struct amdgpu_usermode_queue *queue)
 	*fence_ptr = 0;
 
 	memset(&queue_input, 0x0, sizeof(struct mes_suspend_gang_input));
-	queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
+	queue_input.gang_context_addr = ctx->gpu_addr;
 	queue_input.suspend_fence_addr = fence_gpu_addr;
 	queue_input.suspend_fence_value = 1;
 	amdgpu_mes_lock(&adev->mes);
@@ -516,7 +552,7 @@ static int mes_userq_preempt(struct amdgpu_usermode_queue *queue)
 	r = -ETIMEDOUT;
 
 out:
-	amdgpu_device_wb_free(adev, fence_offset);
+	amdgpu_wb_free(adev, fence_offset);
 	return r;
 }
 
@@ -534,7 +570,7 @@ static int mes_userq_restore(struct amdgpu_usermode_queue *queue)
 		return 0;
 
 	memset(&queue_input, 0x0, sizeof(struct mes_resume_gang_input));
-	queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
+	queue_input.gang_context_addr = ctx->gpu_addr;
 
 	amdgpu_mes_lock(&adev->mes);
 	r = adev->mes.funcs->resume_gang(&adev->mes, &queue_input);
@@ -549,7 +585,7 @@ const struct amdgpu_userq_funcs userq_mes_funcs = {
 	.mqd_destroy = mes_userq_mqd_destroy,
 	.unmap = mes_userq_unmap,
 	.map = mes_userq_map,
-	.detect_and_reset = mes_userq_detect_and_reset,
 	.preempt = mes_userq_preempt,
 	.restore = mes_userq_restore,
+	.reset = mes_userq_reset,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.h b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.h
index 090ae88977705..a473360d6a8b7 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.h
+++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.h
@@ -27,4 +27,13 @@
 #include "amdgpu_userq.h"
 
 extern const struct amdgpu_userq_funcs userq_mes_funcs;
+
+int mes_userq_reset(struct amdgpu_usermode_queue *queue);
+int mes_userq_reset_queue(struct amdgpu_device *adev,
+			  struct amdgpu_usermode_queue *guilty_uq,
+			  int queue_type,
+			  unsigned int pipe,
+			  unsigned int queue,
+			  unsigned int db);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
index 1b071a3de1735..b72da75dd8512 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
@@ -198,7 +198,7 @@ static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 		timeout = 15 * 600 * 1000;
 	}
 
-	ret = amdgpu_device_wb_get(adev, &status_offset);
+	ret = amdgpu_wb_get(adev, &status_offset);
 	if (ret)
 		return ret;
 
@@ -270,7 +270,7 @@ static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 		goto error_wb_free;
 	}
 
-	amdgpu_device_wb_free(adev, status_offset);
+	amdgpu_wb_free(adev, status_offset);
 	return 0;
 
 error_undo:
@@ -281,7 +281,7 @@ static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 	spin_unlock_irqrestore(&mes->ring_lock[0], flags);
 
 error_wb_free:
-	amdgpu_device_wb_free(adev, status_offset);
+	amdgpu_wb_free(adev, status_offset);
 	return r;
 }
 
@@ -335,8 +335,10 @@ static int mes_v11_0_add_hw_queue(struct amdgpu_mes *mes,
 	mes_add_queue_pkt.process_va_end = input->process_va_end;
 	mes_add_queue_pkt.process_quantum = input->process_quantum;
 	mes_add_queue_pkt.process_context_addr = input->process_context_addr;
+	mes_add_queue_pkt.process_context_array_index = input->process_context_array_index;
 	mes_add_queue_pkt.gang_quantum = input->gang_quantum;
 	mes_add_queue_pkt.gang_context_addr = input->gang_context_addr;
+	mes_add_queue_pkt.gang_context_array_index = input->gang_context_array_index;
 	mes_add_queue_pkt.inprocess_gang_priority =
 		convert_to_mes_priority_level(input->inprocess_gang_priority);
 	mes_add_queue_pkt.gang_global_priority_level =
@@ -387,6 +389,9 @@ static int mes_v11_0_remove_hw_queue(struct amdgpu_mes *mes,
 
 	mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
 	mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr;
+	mes_remove_queue_pkt.gang_context_array_index = input->gang_context_array_index;
+	mes_remove_queue_pkt.queue_type =
+		convert_to_mes_queue_type(input->queue_type);
 
 	if (mes_rev >= 0x60)
 		mes_remove_queue_pkt.remove_queue_after_reset = input->remove_queue_after_reset;
@@ -396,6 +401,230 @@ static int mes_v11_0_remove_hw_queue(struct amdgpu_mes *mes,
 			offsetof(union MESAPI__REMOVE_QUEUE, api_status));
 }
 
+static bool mes_v11_0_pipe_reset_support(struct amdgpu_device *adev)
+{
+	/* Disable the pipe reset until the CPFW fully support it.*/
+	dev_warn_once(adev->dev, "The CPFW hasn't support pipe reset yet.\n");
+	return false;
+}
+static int mes_v11_0_reset_gfx_pipe_mmio(struct amdgpu_device *adev,
+					 u32 me, u32 pipe, u32 queue)
+{
+	uint32_t reset_pipe = 0, clean_pipe = 0;
+	int r;
+
+	if (!mes_v11_0_pipe_reset_support(adev))
+		return -EOPNOTSUPP;
+
+	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
+	mutex_lock(&adev->srbm_mutex);
+	soc21_grbm_select(adev, me, pipe, queue, 0);
+
+	switch (pipe) {
+	case 0:
+		reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
+					   PFP_PIPE0_RESET, 1);
+		reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
+					   ME_PIPE0_RESET, 1);
+		clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
+					   PFP_PIPE0_RESET, 0);
+		clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
+					   ME_PIPE0_RESET, 0);
+		break;
+	case 1:
+		reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
+					   PFP_PIPE1_RESET, 1);
+		reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
+					   ME_PIPE1_RESET, 1);
+		clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
+					   PFP_PIPE1_RESET, 0);
+		clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
+					   ME_PIPE1_RESET, 0);
+		break;
+	default:
+		break;
+	}
+
+	WREG32_SOC15(GC, 0, regCP_ME_CNTL, reset_pipe);
+	WREG32_SOC15(GC, 0, regCP_ME_CNTL, clean_pipe);
+
+	r = (RREG32(SOC15_REG_OFFSET(GC, 0, regCP_GFX_RS64_INSTR_PNTR1)) << 2) -
+						RS64_FW_UC_START_ADDR_LO;
+	soc21_grbm_select(adev, 0, 0, 0, 0);
+	mutex_unlock(&adev->srbm_mutex);
+	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
+
+	dev_info(adev->dev, "The gfx pipe reset to the ME firmware start PC: %s\n",
+			r == 0 ? "successfully" : "failed");
+	/* FIXME: Sometimes driver can't cache the ME firmware start PC correctly,
+	 * so the pipe reset status relies on the later gfx ring test result.
+	 */
+	return 0;
+}
+
+/*
+ * With MEC pipe reset asserted, clear CP_HQD_ACTIVE / CP_HQD_DEQUEUE_REQUEST for
+ * every queue on (me, pipe). HQDs must be torn down while pipe reset stays
+ * asserted; only then clear the pipe reset bit.
+ * Caller must hold adev->srbm_mutex.
+ */
+static void mes_v11_0_clear_hqds_on_mec_pipe(struct amdgpu_device *adev, u32 me,
+					     u32 pipe)
+{
+	unsigned int q;
+
+	for (q = 0; q < adev->gfx.mec.num_queue_per_pipe; q++) {
+		soc21_grbm_select(adev, me, pipe, q, 0);
+		/* Start from a clean HQD dequeue state before forcing HQD inactive. */
+		WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0);
+		WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0);
+	}
+}
+
+static int mes_v11_0_reset_compute_pipe_mmio(struct amdgpu_device *adev,
+					     u32 me, u32 pipe, u32 queue)
+{
+	uint32_t reset_val, clean_val;
+	int r;
+
+	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
+	mutex_lock(&adev->srbm_mutex);
+	soc21_grbm_select(adev, me, pipe, queue, 0);
+
+	if (adev->gfx.rs64_enable) {
+		reset_val = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
+		clean_val = reset_val;
+
+		switch (pipe) {
+		case 0:
+			reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL,
+						  MEC_PIPE0_RESET, 1);
+			clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL,
+						  MEC_PIPE0_RESET, 0);
+			break;
+		case 1:
+			reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL,
+						  MEC_PIPE1_RESET, 1);
+			clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL,
+						  MEC_PIPE1_RESET, 0);
+			break;
+		case 2:
+			reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL,
+						  MEC_PIPE2_RESET, 1);
+			clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL,
+						  MEC_PIPE2_RESET, 0);
+			break;
+		case 3:
+			reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL,
+						  MEC_PIPE3_RESET, 1);
+			clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL,
+						  MEC_PIPE3_RESET, 0);
+			break;
+		default:
+			break;
+		}
+		WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, reset_val);
+		mes_v11_0_clear_hqds_on_mec_pipe(adev, me, pipe);
+		WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, clean_val);
+		r = (RREG32_SOC15(GC, 0, regCP_MEC_RS64_INSTR_PNTR) << 2) -
+					RS64_FW_UC_START_ADDR_LO;
+	} else {
+		reset_val = RREG32_SOC15(GC, 0, regCP_MEC_CNTL);
+		clean_val = reset_val;
+
+		if (me == 1) {
+			switch (pipe) {
+			case 0:
+				reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL,
+							  MEC_ME1_PIPE0_RESET, 1);
+				clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL,
+							  MEC_ME1_PIPE0_RESET, 0);
+				break;
+			case 1:
+				reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL,
+							  MEC_ME1_PIPE1_RESET, 1);
+				clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL,
+							  MEC_ME1_PIPE1_RESET, 0);
+				break;
+			case 2:
+				reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL,
+							  MEC_ME1_PIPE2_RESET, 1);
+				clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL,
+							  MEC_ME1_PIPE2_RESET, 0);
+				break;
+			case 3:
+				reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL,
+							  MEC_ME1_PIPE3_RESET, 1);
+				clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL,
+							  MEC_ME1_PIPE3_RESET, 0);
+				break;
+			default:
+				break;
+			}
+			/* mec1 fw pc: CP_MEC1_INSTR_PNTR */
+		} else {
+			switch (pipe) {
+			case 0:
+				reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL,
+							  MEC_ME2_PIPE0_RESET, 1);
+				clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL,
+							  MEC_ME2_PIPE0_RESET, 0);
+				break;
+			case 1:
+				reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL,
+							  MEC_ME2_PIPE1_RESET, 1);
+				clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL,
+							  MEC_ME2_PIPE1_RESET, 0);
+				break;
+			case 2:
+				reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL,
+							  MEC_ME2_PIPE2_RESET, 1);
+				clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL,
+							  MEC_ME2_PIPE2_RESET, 0);
+				break;
+			case 3:
+				reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL,
+							  MEC_ME2_PIPE3_RESET, 1);
+				clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL,
+							  MEC_ME2_PIPE3_RESET, 0);
+				break;
+			default:
+				break;
+			}
+			/* mec2 fw pc: CP:CP_MEC2_INSTR_PNTR */
+		}
+		WREG32_SOC15(GC, 0, regCP_MEC_CNTL, reset_val);
+		mes_v11_0_clear_hqds_on_mec_pipe(adev, me, pipe);
+		WREG32_SOC15(GC, 0, regCP_MEC_CNTL, clean_val);
+		r = RREG32(SOC15_REG_OFFSET(GC, 0, regCP_MEC1_INSTR_PNTR));
+	}
+
+	soc21_grbm_select(adev, 0, 0, 0, 0);
+	mutex_unlock(&adev->srbm_mutex);
+	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
+
+	dev_dbg(adev->dev, "MEC pipe me%u pipe%u queue%u resets to MEC FW start PC: %s\n",
+		me, pipe, queue, r == 0 ? "successfully" : "failed");
+	/*FIXME:Sometimes driver can't cache the MEC firmware start PC correctly, so the pipe
+	 * reset status relies on the compute ring test result.
+	 */
+	return 0;
+}
+
+static int mes_v11_0_reset_pipe_mmio(struct amdgpu_mes *mes, uint32_t queue_type,
+				     uint32_t me_id, uint32_t pipe_id,
+				     uint32_t queue_id, uint32_t vmid)
+{
+	struct amdgpu_device *adev = mes->adev;
+
+	if (queue_type == AMDGPU_RING_TYPE_GFX)
+		return mes_v11_0_reset_gfx_pipe_mmio(adev, me_id, pipe_id, queue_id);
+	else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
+		return mes_v11_0_reset_compute_pipe_mmio(adev, me_id, pipe_id, queue_id);
+	else
+		return -EOPNOTSUPP;
+}
+
 static int mes_v11_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_type,
 				      uint32_t me_id, uint32_t pipe_id,
 				      uint32_t queue_id, uint32_t vmid)
@@ -543,6 +772,11 @@ static int mes_v11_0_unmap_legacy_queue(struct amdgpu_mes *mes,
 			convert_to_mes_queue_type(input->queue_type);
 	}
 
+	if (input->action == RESET_QUEUES &&
+	   (mes->sched_version & AMDGPU_MES_VERSION_MASK) >= 0x60)
+		mes_remove_queue_pkt.remove_queue_after_reset = 1;
+
+
 	return mes_v11_0_submit_pkt_and_poll_completion(mes,
 			&mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
 			offsetof(union MESAPI__REMOVE_QUEUE, api_status));
@@ -605,6 +839,58 @@ static int mes_v11_0_query_sched_status(struct amdgpu_mes *mes)
 			offsetof(union MESAPI__QUERY_MES_STATUS, api_status));
 }
 
+/*
+ * QUERY_SCHEDULER_STATUS: get proc/gang context array sizes
+ */
+static int mes_v11_0_query_ctx_array_sizes(struct amdgpu_mes *mes)
+{
+	struct amdgpu_device *adev = mes->adev;
+	union MESAPI__QUERY_MES_STATUS mes_query_pkt;
+	int r;
+
+	if (!mes->ctx_array_size_cpu_ptr)
+		return -EINVAL;
+
+	/* Clear the output buffer */
+	mes->ctx_array_size_cpu_ptr[0] = 0;  /* proc_ctx_array_size */
+	mes->ctx_array_size_cpu_ptr[1] = 0;  /* gang_ctx_array_size */
+
+	memset(&mes_query_pkt, 0, sizeof(mes_query_pkt));
+
+	mes_query_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	mes_query_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
+	mes_query_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+	mes_query_pkt.subopcode = MES_API_QUERY_MES__GET_CTX_ARRAY_SIZE;
+
+	/*
+	 * MES FW will write the array sizes to these GPU addresses:
+	 *   proc_ctx_array_size_addr -> uint32_t N (e.g., 50)
+	 *   gang_ctx_array_size_addr -> uint32_t M (e.g., 300)
+	 */
+	mes_query_pkt.ctx_array_size.proc_ctx_array_size_addr =
+		mes->ctx_array_size_gpu_addr;
+	mes_query_pkt.ctx_array_size.gang_ctx_array_size_addr =
+		mes->ctx_array_size_gpu_addr + sizeof(uint32_t);
+
+	mes_query_pkt.api_status.api_completion_fence_addr =
+		mes->ring[0].fence_drv.gpu_addr;
+	mes_query_pkt.api_status.api_completion_fence_value =
+		++mes->ring[0].fence_drv.sync_seq;
+
+	r = mes_v11_0_submit_pkt_and_poll_completion(mes,
+			&mes_query_pkt, sizeof(mes_query_pkt),
+			offsetof(union MESAPI__QUERY_MES_STATUS, api_status));
+	if (r) {
+		dev_err(adev->dev,
+			"MES QUERY_SCHEDULER_STATUS (GET_CTX_ARRAY_SIZE) failed, r=%d\n", r);
+		return r;
+	}
+
+	/* MES has written the sizes - now set up bitmaps */
+	return amdgpu_mes_rs64mem_setup_bitmaps(mes);
+}
+
 static int mes_v11_0_misc_op(struct amdgpu_mes *mes,
 			     struct mes_misc_op_input *input)
 {
@@ -770,10 +1056,16 @@ static int mes_v11_0_reset_hw_queue(struct amdgpu_mes *mes,
 {
 	union MESAPI__RESET mes_reset_queue_pkt;
 
-	if (input->use_mmio)
-		return mes_v11_0_reset_queue_mmio(mes, input->queue_type,
-						  input->me_id, input->pipe_id,
-						  input->queue_id, input->vmid);
+	if (input->use_mmio) {
+		int r = mes_v11_0_reset_queue_mmio(mes, input->queue_type,
+						   input->me_id, input->pipe_id,
+						   input->queue_id, input->vmid);
+		if (r)
+			return mes_v11_0_reset_pipe_mmio(mes, input->queue_type,
+							 input->me_id, input->pipe_id,
+							 input->queue_id, input->vmid);
+		return 0;
+	}
 
 	memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
 
@@ -1672,10 +1964,33 @@ static int mes_v11_0_hw_init(struct amdgpu_ip_block *ip_block)
 	if (r)
 		goto failure;
 
+	/* Allocate GPU buffer for array size query results */
+	r = amdgpu_mes_rs64mem_init(&adev->mes);
+	if (r)
+		dev_warn(adev->dev,
+			 "RS64 local memory init failed (%d),"
+			 "falling back to system memory path\n", r);
+
 	r = mes_v11_0_set_hw_resources(&adev->mes);
+
 	if (r)
 		goto failure;
 
+	/*
+	 * QUERY_SCHEDULER_STATUS to get array sizes (N, M).
+	 * MES writes the sizes to the GPU buffer, then we allocate bitmaps.
+	 */
+	if (adev->mes.use_rs64mem) {
+		r = mes_v11_0_query_ctx_array_sizes(&adev->mes);
+		if (r) {
+			dev_warn(adev->dev,
+				 "Failed to query ctx array sizes (%d),"
+				 "disabling RS64 local memory\n", r);
+			/* Continue without optimization - not fatal */
+		}
+	}
+
+
 	if ((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x52) {
 		r = mes_v11_0_set_hw_resources_1(&adev->mes);
 		if (r) {
@@ -1713,6 +2028,10 @@ static int mes_v11_0_hw_init(struct amdgpu_ip_block *ip_block)
 
 static int mes_v11_0_hw_fini(struct amdgpu_ip_block *ip_block)
 {
+	struct amdgpu_device *adev = ip_block->adev;
+
+	if (adev->mes.use_rs64mem)
+		amdgpu_mes_rs64mem_fini(&adev->mes);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
index b6cbc25e1ab47..04465804c2544 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
@@ -26,7 +26,7 @@
 #include "amdgpu.h"
 #include "gfx_v12_0.h"
 #include "soc15_common.h"
-#include "soc21.h"
+#include "soc24.h"
 #include "gc/gc_12_0_0_offset.h"
 #include "gc/gc_12_0_0_sh_mask.h"
 #include "gc/gc_11_0_0_default.h"
@@ -175,7 +175,7 @@ static int mes_v12_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 		timeout = 15 * 600 * 1000;
 	}
 
-	ret = amdgpu_device_wb_get(adev, &status_offset);
+	ret = amdgpu_wb_get(adev, &status_offset);
 	if (ret)
 		return ret;
 
@@ -253,7 +253,7 @@ static int mes_v12_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 		goto error_wb_free;
 	}
 
-	amdgpu_device_wb_free(adev, status_offset);
+	amdgpu_wb_free(adev, status_offset);
 	return 0;
 
 error_undo:
@@ -264,7 +264,7 @@ static int mes_v12_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 	spin_unlock_irqrestore(ring_lock, flags);
 
 error_wb_free:
-	amdgpu_device_wb_free(adev, status_offset);
+	amdgpu_wb_free(adev, status_offset);
 	return r;
 }
 
@@ -371,6 +371,8 @@ static int mes_v12_0_remove_hw_queue(struct amdgpu_mes *mes,
 
 	mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
 	mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr;
+	mes_remove_queue_pkt.queue_type =
+		convert_to_mes_queue_type(input->queue_type);
 
 	if (mes_rev >= 0x5a)
 		mes_remove_queue_pkt.remove_queue_after_reset = input->remove_queue_after_reset;
@@ -413,6 +415,171 @@ int gfx_v12_0_request_gfx_index_mutex(struct amdgpu_device *adev,
 	return 0;
 }
 
+static bool mes_v12_0_pipe_reset_support(struct amdgpu_device *adev)
+{
+	/* Disable the pipe reset until the CPFW fully support it.*/
+	dev_warn_once(adev->dev, "The CPFW hasn't support pipe reset yet.\n");
+	return false;
+}
+
+static int mes_v12_0_reset_gfx_pipe_mmio(struct amdgpu_device *adev,
+					 u32 me, u32 pipe, u32 queue)
+{
+	uint32_t reset_pipe = 0, clean_pipe = 0;
+	int r;
+
+	if (!mes_v12_0_pipe_reset_support(adev))
+		return -EOPNOTSUPP;
+
+	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
+	mutex_lock(&adev->srbm_mutex);
+	soc24_grbm_select(adev, me, pipe, queue, 0);
+
+	switch (pipe) {
+	case 0:
+		reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
+					   PFP_PIPE0_RESET, 1);
+		reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
+					   ME_PIPE0_RESET, 1);
+		clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
+					   PFP_PIPE0_RESET, 0);
+		clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
+					   ME_PIPE0_RESET, 0);
+		break;
+	case 1:
+		reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
+					   PFP_PIPE1_RESET, 1);
+		reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL,
+					   ME_PIPE1_RESET, 1);
+		clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
+					   PFP_PIPE1_RESET, 0);
+		clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL,
+					   ME_PIPE1_RESET, 0);
+		break;
+	default:
+		break;
+	}
+
+	WREG32_SOC15(GC, 0, regCP_ME_CNTL, reset_pipe);
+	WREG32_SOC15(GC, 0, regCP_ME_CNTL, clean_pipe);
+
+	r = (RREG32(SOC15_REG_OFFSET(GC, 0, regCP_GFX_RS64_INSTR_PNTR1)) << 2) -
+					RS64_FW_UC_START_ADDR_LO;
+	soc24_grbm_select(adev, 0, 0, 0, 0);
+	mutex_unlock(&adev->srbm_mutex);
+	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
+
+	dev_info(adev->dev, "The gfx pipe reset: %s\n",
+			r == 0 ? "successfully" : "failed");
+	/* Sometimes the ME start pc counter can't cache correctly, so the
+	 * PC check only as a reference and pipe reset result rely on the
+	 * later ring test.
+	 */
+	return 0;
+}
+
+/*
+ * With MEC pipe reset asserted, clear CP_HQD_ACTIVE / CP_HQD_DEQUEUE_REQUEST for
+ * every queue on (me, pipe). HQDs must be torn down while pipe reset stays
+ * asserted; only then clear the pipe reset bit.
+ * Caller must hold adev->srbm_mutex.
+ */
+static void mes_v12_0_clear_hqds_on_mec_pipe(struct amdgpu_device *adev, u32 me,
+					     u32 pipe)
+{
+	unsigned int q;
+
+	for (q = 0; q < adev->gfx.mec.num_queue_per_pipe; q++) {
+		soc24_grbm_select(adev, me, pipe, q, 0);
+		/* Start from a clean HQD dequeue state before forcing HQD inactive. */
+		WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0);
+		WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0);
+	}
+}
+
+static int mes_v12_0_reset_compute_pipe_mmio(struct amdgpu_device *adev,
+					     u32 me, u32 pipe, u32 queue)
+{
+	uint32_t reset_val, clean_val;
+	int r = 0;
+
+	amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
+	mutex_lock(&adev->srbm_mutex);
+	soc24_grbm_select(adev, me, pipe, queue, 0);
+	if (adev->gfx.rs64_enable) {
+		reset_val = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
+		clean_val = reset_val;
+
+		switch (pipe) {
+		case 0:
+			reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL,
+						  MEC_PIPE0_RESET, 1);
+			clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL,
+						  MEC_PIPE0_RESET, 0);
+			break;
+		case 1:
+			reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL,
+						  MEC_PIPE1_RESET, 1);
+			clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL,
+						  MEC_PIPE1_RESET, 0);
+			break;
+		case 2:
+			reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL,
+						  MEC_PIPE2_RESET, 1);
+			clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL,
+						  MEC_PIPE2_RESET, 0);
+			break;
+		case 3:
+			reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL,
+						  MEC_PIPE3_RESET, 1);
+			clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL,
+						  MEC_PIPE3_RESET, 0);
+			break;
+		default:
+			break;
+		}
+		WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, reset_val);
+		mes_v12_0_clear_hqds_on_mec_pipe(adev, me, pipe);
+		soc24_grbm_select(adev, me, pipe, queue, 0);
+		WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, clean_val);
+		r = (RREG32_SOC15(GC, 0, regCP_MEC_RS64_INSTR_PNTR) << 2) -
+				RS64_FW_UC_START_ADDR_LO;
+	} else {
+		reset_val = RREG32_SOC15(GC, 0, regCP_MEC_CNTL);
+		clean_val = reset_val;
+
+		switch (pipe) {
+		case 0:
+			reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL,
+						  MEC_ME1_PIPE0_RESET, 1);
+			clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL,
+						  MEC_ME1_PIPE0_RESET, 0);
+			break;
+		case 1:
+			reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL,
+						  MEC_ME1_PIPE1_RESET, 1);
+			clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL,
+						  MEC_ME1_PIPE1_RESET, 0);
+			break;
+		default:
+			break;
+		}
+
+		WREG32_SOC15(GC, 0, regCP_MEC_CNTL, reset_val);
+		mes_v12_0_clear_hqds_on_mec_pipe(adev, me, pipe);
+		soc24_grbm_select(adev, me, pipe, queue, 0);
+		WREG32_SOC15(GC, 0, regCP_MEC_CNTL, clean_val);
+	}
+
+	soc24_grbm_select(adev, 0, 0, 0, 0);
+	mutex_unlock(&adev->srbm_mutex);
+	amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
+
+	dev_dbg(adev->dev, "MEC pipe me%u pipe%u queue%u resets to MEC FW start PC: %s\n",
+		me, pipe, queue, r == 0 ? "successfully" : "failed");
+	return 0;
+}
+
 static int mes_v12_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_type,
 				      uint32_t me_id, uint32_t pipe_id,
 				      uint32_t queue_id, uint32_t vmid)
@@ -442,7 +609,7 @@ static int mes_v12_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_typ
 		mutex_unlock(&adev->gfx.reset_sem_mutex);
 
 		mutex_lock(&adev->srbm_mutex);
-		soc21_grbm_select(adev, me_id, pipe_id, queue_id, 0);
+		soc24_grbm_select(adev, me_id, pipe_id, queue_id, 0);
 		/* wait till dequeue take effects */
 		for (i = 0; i < adev->usec_timeout; i++) {
 			if (!(RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE) & 1))
@@ -454,13 +621,13 @@ static int mes_v12_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_typ
 			r = -ETIMEDOUT;
 		}
 
-		soc21_grbm_select(adev, 0, 0, 0, 0);
+		soc24_grbm_select(adev, 0, 0, 0, 0);
 		mutex_unlock(&adev->srbm_mutex);
 	} else if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
 		dev_info(adev->dev, "reset compute queue (%d:%d:%d)\n",
 			 me_id, pipe_id, queue_id);
 		mutex_lock(&adev->srbm_mutex);
-		soc21_grbm_select(adev, me_id, pipe_id, queue_id, 0);
+		soc24_grbm_select(adev, me_id, pipe_id, queue_id, 0);
 		WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2);
 		WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1);
 
@@ -474,7 +641,7 @@ static int mes_v12_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_typ
 			dev_err(adev->dev, "failed to wait on hqd deactivate\n");
 			r = -ETIMEDOUT;
 		}
-		soc21_grbm_select(adev, 0, 0, 0, 0);
+		soc24_grbm_select(adev, 0, 0, 0, 0);
 		mutex_unlock(&adev->srbm_mutex);
 	} else if (queue_type == AMDGPU_RING_TYPE_SDMA) {
 		dev_info(adev->dev, "reset sdma queue (%d:%d:%d)\n",
@@ -507,6 +674,20 @@ static int mes_v12_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_typ
 	return r;
 }
 
+static int mes_v12_0_reset_pipe_mmio(struct amdgpu_mes *mes, uint32_t queue_type,
+				     uint32_t me_id, uint32_t pipe_id,
+				     uint32_t queue_id, uint32_t vmid)
+{
+	struct amdgpu_device *adev = mes->adev;
+
+	if (queue_type == AMDGPU_RING_TYPE_GFX)
+		return mes_v12_0_reset_gfx_pipe_mmio(adev, me_id, pipe_id, queue_id);
+	else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
+		return mes_v12_0_reset_compute_pipe_mmio(adev, me_id, pipe_id, queue_id);
+	else
+		return -EOPNOTSUPP;
+}
+
 static int mes_v12_0_map_legacy_queue(struct amdgpu_mes *mes,
 				      struct mes_map_legacy_queue_input *input)
 {
@@ -528,10 +709,15 @@ static int mes_v12_0_map_legacy_queue(struct amdgpu_mes *mes,
 		convert_to_mes_queue_type(input->queue_type);
 	mes_add_queue_pkt.map_legacy_kq = 1;
 
-	if (mes->adev->enable_uni_mes)
-		pipe = AMDGPU_MES_KIQ_PIPE;
-	else
+	if (mes->adev->enable_uni_mes) {
+		/* Keep scheduler queue on KIQ pipe; map all other kernel queues on sched pipe. */
+		if (input->queue_type == AMDGPU_RING_TYPE_MES)
+			pipe = AMDGPU_MES_KIQ_PIPE;
+		else
+			pipe = AMDGPU_MES_SCHED_PIPE;
+	} else {
 		pipe = AMDGPU_MES_SCHED_PIPE;
+	}
 
 	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 			&mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
@@ -565,12 +751,28 @@ static int mes_v12_0_unmap_legacy_queue(struct amdgpu_mes *mes,
 		mes_remove_queue_pkt.unmap_legacy_queue = 1;
 		mes_remove_queue_pkt.queue_type =
 			convert_to_mes_queue_type(input->queue_type);
+		/*
+		 * A reset-time unmap: the queue was already reset via MMIO while
+		 * gangs are suspended and it is on the MES hung/fail list. Tell
+		 * MES to just drop its internal state for it. Without this flag
+		 * MES asks CP to unmap the already-reset (still wedged) queue
+		 * again, which times out and forces a GPU reset.
+		 */
+		if (input->action == RESET_QUEUES &&
+		    (mes->sched_version & AMDGPU_MES_VERSION_MASK) >= 0x5a)
+			mes_remove_queue_pkt.remove_queue_after_reset = 1;
+
 	}
 
-	if (mes->adev->enable_uni_mes)
-		pipe = AMDGPU_MES_KIQ_PIPE;
-	else
+	if (mes->adev->enable_uni_mes) {
+		/* Keep scheduler queue on KIQ pipe; unmap all other kernel queues on sched pipe. */
+		if (input->queue_type == AMDGPU_RING_TYPE_MES)
+			pipe = AMDGPU_MES_KIQ_PIPE;
+		else
+			pipe = AMDGPU_MES_SCHED_PIPE;
+	} else {
 		pipe = AMDGPU_MES_SCHED_PIPE;
+	}
 
 	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 			&mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
@@ -888,10 +1090,16 @@ static int mes_v12_0_reset_hw_queue(struct amdgpu_mes *mes,
 	union MESAPI__RESET mes_reset_queue_pkt;
 	int pipe;
 
-	if (input->use_mmio)
-		return mes_v12_0_reset_queue_mmio(mes, input->queue_type,
-						  input->me_id, input->pipe_id,
-						  input->queue_id, input->vmid);
+	if (input->use_mmio) {
+		int r = mes_v12_0_reset_queue_mmio(mes, input->queue_type,
+						   input->me_id, input->pipe_id,
+						   input->queue_id, input->vmid);
+		if (r)
+			return mes_v12_0_reset_pipe_mmio(mes, input->queue_type,
+							 input->me_id, input->pipe_id,
+							 input->queue_id, input->vmid);
+		return 0;
+	}
 
 	memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
 
@@ -915,10 +1123,7 @@ static int mes_v12_0_reset_hw_queue(struct amdgpu_mes *mes,
 		mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset;
 	}
 
-	if (input->is_kq)
-		pipe = AMDGPU_MES_KIQ_PIPE;
-	else
-		pipe = AMDGPU_MES_SCHED_PIPE;
+	pipe = AMDGPU_MES_SCHED_PIPE;
 
 	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
 			&mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
@@ -1094,7 +1299,7 @@ static void mes_v12_0_enable(struct amdgpu_device *adev, bool enable)
 	if (enable) {
 		mutex_lock(&adev->srbm_mutex);
 		for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
-			soc21_grbm_select(adev, 3, pipe, 0, 0);
+			soc24_grbm_select(adev, 3, pipe, 0, 0);
 			if (amdgpu_mes_log_enable) {
 				u32 log_size = AMDGPU_MES_LOG_BUFFER_SIZE + AMDGPU_MES_MSCRATCH_SIZE;
 				/* In case uni mes is not enabled, only program for pipe 0 */
@@ -1133,7 +1338,7 @@ static void mes_v12_0_enable(struct amdgpu_device *adev, bool enable)
 			WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
 
 		}
-		soc21_grbm_select(adev, 0, 0, 0, 0);
+		soc24_grbm_select(adev, 0, 0, 0, 0);
 		mutex_unlock(&adev->srbm_mutex);
 
 		if (amdgpu_emu_mode)
@@ -1165,7 +1370,7 @@ static void mes_v12_0_set_ucode_start_addr(struct amdgpu_device *adev)
 	mutex_lock(&adev->srbm_mutex);
 	for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
 		/* me=3, queue=0 */
-		soc21_grbm_select(adev, 3, pipe, 0, 0);
+		soc24_grbm_select(adev, 3, pipe, 0, 0);
 
 		/* set ucode start address */
 		ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
@@ -1174,7 +1379,7 @@ static void mes_v12_0_set_ucode_start_addr(struct amdgpu_device *adev)
 		WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,
 				upper_32_bits(ucode_addr));
 
-		soc21_grbm_select(adev, 0, 0, 0, 0);
+		soc24_grbm_select(adev, 0, 0, 0, 0);
 	}
 	mutex_unlock(&adev->srbm_mutex);
 }
@@ -1203,7 +1408,7 @@ static int mes_v12_0_load_microcode(struct amdgpu_device *adev,
 
 	mutex_lock(&adev->srbm_mutex);
 	/* me=3, pipe=0, queue=0 */
-	soc21_grbm_select(adev, 3, pipe, 0, 0);
+	soc24_grbm_select(adev, 3, pipe, 0, 0);
 
 	WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_CNTL, 0);
 
@@ -1238,7 +1443,7 @@ static int mes_v12_0_load_microcode(struct amdgpu_device *adev,
 		WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data);
 	}
 
-	soc21_grbm_select(adev, 0, 0, 0, 0);
+	soc24_grbm_select(adev, 0, 0, 0, 0);
 	mutex_unlock(&adev->srbm_mutex);
 
 	return 0;
@@ -1385,7 +1590,7 @@ static void mes_v12_0_queue_init_register(struct amdgpu_ring *ring)
 	uint32_t data = 0;
 
 	mutex_lock(&adev->srbm_mutex);
-	soc21_grbm_select(adev, 3, ring->pipe, 0, 0);
+	soc24_grbm_select(adev, 3, ring->pipe, 0, 0);
 
 	/* set CP_HQD_VMID.VMID = 0. */
 	data = RREG32_SOC15(GC, 0, regCP_HQD_VMID);
@@ -1436,7 +1641,7 @@ static void mes_v12_0_queue_init_register(struct amdgpu_ring *ring)
 	/* set CP_HQD_ACTIVE.ACTIVE=1 */
 	WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, mqd->cp_hqd_active);
 
-	soc21_grbm_select(adev, 0, 0, 0, 0);
+	soc24_grbm_select(adev, 0, 0, 0, 0);
 	mutex_unlock(&adev->srbm_mutex);
 }
 
@@ -1502,14 +1707,14 @@ static int mes_v12_0_queue_init(struct amdgpu_device *adev,
 	    ((pipe == AMDGPU_MES_KIQ_PIPE) && !adev->mes.kiq_version)) {
 		/* get MES scheduler/KIQ versions */
 		mutex_lock(&adev->srbm_mutex);
-		soc21_grbm_select(adev, 3, pipe, 0, 0);
+		soc24_grbm_select(adev, 3, pipe, 0, 0);
 
 		if (pipe == AMDGPU_MES_SCHED_PIPE)
 			adev->mes.sched_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);
 		else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq)
 			adev->mes.kiq_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);
 
-		soc21_grbm_select(adev, 0, 0, 0, 0);
+		soc24_grbm_select(adev, 0, 0, 0, 0);
 		mutex_unlock(&adev->srbm_mutex);
 	}
 
@@ -1697,7 +1902,7 @@ static void mes_v12_0_kiq_dequeue_sched(struct amdgpu_device *adev)
 	int i;
 
 	mutex_lock(&adev->srbm_mutex);
-	soc21_grbm_select(adev, 3, AMDGPU_MES_SCHED_PIPE, 0, 0);
+	soc24_grbm_select(adev, 3, AMDGPU_MES_SCHED_PIPE, 0, 0);
 
 	/* disable the queue if it's active */
 	if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) {
@@ -1721,7 +1926,7 @@ static void mes_v12_0_kiq_dequeue_sched(struct amdgpu_device *adev)
 	WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 0);
 	WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, 0);
 
-	soc21_grbm_select(adev, 0, 0, 0, 0);
+	soc24_grbm_select(adev, 0, 0, 0, 0);
 	mutex_unlock(&adev->srbm_mutex);
 
 	adev->mes.ring[0].sched.ready = false;
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
index e13535d94c516..dbdb9ba02a5fc 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
@@ -43,7 +43,6 @@ static int mes_v12_1_xcc_hw_init(struct amdgpu_ip_block *ip_block, int xcc_id);
 static int mes_v12_1_hw_fini(struct amdgpu_ip_block *ip_block);
 static int mes_v12_1_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id);
 static int mes_v12_1_kiq_hw_fini(struct amdgpu_device *adev, uint32_t xcc_id);
-static int mes_v12_1_self_test(struct amdgpu_device *adev, int xcc_id);
 static int mes_v12_1_setup_coop_mode(struct amdgpu_device *adev, int xcc_id);
 
 #define MES_EOP_SIZE   2048
@@ -178,7 +177,7 @@ static int mes_v12_1_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 		timeout = 15 * 600 * 1000;
 	}
 
-	ret = amdgpu_device_wb_get(adev, &status_offset);
+	ret = amdgpu_wb_get(adev, &status_offset);
 	if (ret)
 		return ret;
 
@@ -253,7 +252,7 @@ static int mes_v12_1_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 		goto error_wb_free;
 	}
 
-	amdgpu_device_wb_free(adev, status_offset);
+	amdgpu_wb_free(adev, status_offset);
 	return 0;
 
 error_undo:
@@ -264,7 +263,7 @@ static int mes_v12_1_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 	spin_unlock_irqrestore(ring_lock, flags);
 
 error_wb_free:
-	amdgpu_device_wb_free(adev, status_offset);
+	amdgpu_wb_free(adev, status_offset);
 	return r;
 }
 
@@ -362,6 +361,8 @@ static int mes_v12_1_remove_hw_queue(struct amdgpu_mes *mes,
 
 	mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
 	mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr;
+	mes_remove_queue_pkt.queue_type =
+		convert_to_mes_queue_type(input->queue_type);
 
 	return mes_v12_1_submit_pkt_and_poll_completion(mes,
 			xcc_id, AMDGPU_MES_SCHED_PIPE,
@@ -417,10 +418,15 @@ static int mes_v12_1_map_legacy_queue(struct amdgpu_mes *mes,
 		convert_to_mes_queue_type(input->queue_type);
 	mes_add_queue_pkt.map_legacy_kq = 1;
 
-	if (mes->adev->enable_uni_mes)
-		pipe = AMDGPU_MES_KIQ_PIPE;
-	else
+	if (mes->adev->enable_uni_mes) {
+		/* Keep scheduler queue on KIQ pipe; map all other kernel queues on sched pipe. */
+		if (input->queue_type == AMDGPU_RING_TYPE_MES)
+			pipe = AMDGPU_MES_KIQ_PIPE;
+		else
+			pipe = AMDGPU_MES_SCHED_PIPE;
+	} else {
 		pipe = AMDGPU_MES_SCHED_PIPE;
+	}
 
 	return mes_v12_1_submit_pkt_and_poll_completion(mes,
 			input->xcc_id, pipe,
@@ -457,10 +463,15 @@ static int mes_v12_1_unmap_legacy_queue(struct amdgpu_mes *mes,
 			convert_to_mes_queue_type(input->queue_type);
 	}
 
-	if (mes->adev->enable_uni_mes)
-		pipe = AMDGPU_MES_KIQ_PIPE;
-	else
+	if (mes->adev->enable_uni_mes) {
+		/* Keep scheduler queue on KIQ pipe; map all other kernel queues on sched pipe. */
+		if (input->queue_type == AMDGPU_RING_TYPE_MES)
+			pipe = AMDGPU_MES_KIQ_PIPE;
+		else
+			pipe = AMDGPU_MES_SCHED_PIPE;
+	} else {
 		pipe = AMDGPU_MES_SCHED_PIPE;
+	}
 
 	return mes_v12_1_submit_pkt_and_poll_completion(mes,
 			input->xcc_id, pipe,
@@ -1982,31 +1993,10 @@ static int mes_v12_1_early_init(struct amdgpu_ip_block *ip_block)
 	return 0;
 }
 
-static int mes_v12_1_late_init(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	int xcc_id, num_xcc = NUM_XCC(adev->gfx.xcc_mask);
-
-	/* TODO: remove it if issue fixed. */
-	if (adev->mes.enable_coop_mode)
-		return 0;
-
-	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
-		/* for COOP mode, only test master xcc. */
-		if (adev->mes.enable_coop_mode &&
-		    adev->mes.master_xcc_ids[xcc_id] != xcc_id)
-			continue;
-
-		mes_v12_1_self_test(adev, xcc_id);
-	}
-
-	return 0;
-}
-
 static const struct amd_ip_funcs mes_v12_1_ip_funcs = {
 	.name = "mes_v12_1",
 	.early_init = mes_v12_1_early_init,
-	.late_init = mes_v12_1_late_init,
+	.late_init = NULL,
 	.sw_init = mes_v12_1_sw_init,
 	.sw_fini = mes_v12_1_sw_fini,
 	.hw_init = mes_v12_1_hw_init,
@@ -2022,312 +2012,3 @@ const struct amdgpu_ip_block_version mes_v12_1_ip_block = {
 	.rev = 0,
 	.funcs = &mes_v12_1_ip_funcs,
 };
-
-static int mes_v12_1_alloc_test_buf(struct amdgpu_device *adev,
-				    struct amdgpu_bo **bo, uint64_t *addr,
-				    void **ptr, int size)
-{
-	amdgpu_bo_create_kernel(adev, size, PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
-				bo, addr, ptr);
-	if (!*bo) {
-		dev_err(adev->dev, "failed to allocate test buffer bo\n");
-		return -ENOMEM;
-	}
-	memset(*ptr, 0, size);
-	return 0;
-}
-
-static int mes_v12_1_map_test_bo(struct amdgpu_device *adev,
-				 struct amdgpu_bo *bo, struct amdgpu_vm *vm,
-				 struct amdgpu_bo_va **bo_va, u64 va, int size)
-{
-	struct amdgpu_sync sync;
-	int r;
-
-	r = amdgpu_map_static_csa(adev, vm, bo, bo_va, va, size);
-	if (r)
-		return r;
-
-	amdgpu_sync_create(&sync);
-
-	r = amdgpu_vm_bo_update(adev, *bo_va, false);
-	if (r) {
-		dev_err(adev->dev, "failed to do vm_bo_update on meta data\n");
-		goto error;
-	}
-	amdgpu_sync_fence(&sync, (*bo_va)->last_pt_update, GFP_KERNEL);
-
-	r = amdgpu_vm_update_pdes(adev, vm, false);
-	if (r) {
-		dev_err(adev->dev, "failed to update pdes on meta data\n");
-		goto error;
-	}
-	amdgpu_sync_fence(&sync, vm->last_update, GFP_KERNEL);
-	amdgpu_sync_wait(&sync, false);
-
-error:
-	amdgpu_sync_free(&sync);
-	return r;
-}
-
-static int mes_v12_1_test_ring(struct amdgpu_device *adev, int xcc_id,
-			       u32 *queue_ptr, u64 fence_gpu_addr,
-			       void *fence_cpu_ptr, void *wptr_cpu_addr,
-			       u64 doorbell_idx, int queue_type)
-{
-	volatile uint32_t *cpu_ptr = fence_cpu_ptr;
-	int num_xcc = NUM_XCC(adev->gfx.xcc_mask);
-	int sdma_ring_align = 0x10, compute_ring_align = 0x100;
-	uint32_t tmp, xcc_offset;
-	int r = 0, i, j, wptr = 0;
-
-	if (queue_type == AMDGPU_RING_TYPE_COMPUTE) {
-		if (!adev->mes.enable_coop_mode) {
-			WREG32_SOC15(GC, GET_INST(GC, xcc_id),
-				     regSCRATCH_REG0, 0xCAFEDEAD);
-		} else {
-			for (i = 0; i < num_xcc; i++) {
-				if (adev->mes.master_xcc_ids[i] == xcc_id)
-					WREG32_SOC15(GC, GET_INST(GC, i),
-					       regSCRATCH_REG0, 0xCAFEDEAD);
-			}
-		}
-
-		xcc_offset = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0);
-		queue_ptr[wptr++] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
-		queue_ptr[wptr++] = xcc_offset - PACKET3_SET_UCONFIG_REG_START;
-		queue_ptr[wptr++] = 0xDEADBEEF;
-
-		for (i = wptr; i < compute_ring_align; i++)
-			queue_ptr[wptr++] = PACKET3(PACKET3_NOP, 0x3FFF);
-
-	}  else if (queue_type == AMDGPU_RING_TYPE_SDMA) {
-		*cpu_ptr = 0xCAFEDEAD;
-
-		queue_ptr[wptr++] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
-			SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
-		queue_ptr[wptr++] = lower_32_bits(fence_gpu_addr);
-		queue_ptr[wptr++] = upper_32_bits(fence_gpu_addr);
-		queue_ptr[wptr++] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
-		queue_ptr[wptr++] = 0xDEADBEEF;
-
-		for (i = wptr; i < sdma_ring_align; i++)
-			queue_ptr[wptr++] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-
-		wptr <<= 2;
-	}
-
-	atomic64_set((atomic64_t *)wptr_cpu_addr, wptr);
-	WDOORBELL64(doorbell_idx, wptr);
-
-	for (i = 0; i < adev->usec_timeout; i++) {
-		if (queue_type == AMDGPU_RING_TYPE_SDMA) {
-			tmp = le32_to_cpu(*cpu_ptr);
-		} else {
-			if (!adev->mes.enable_coop_mode) {
-				tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
-						   regSCRATCH_REG0);
-			} else {
-				for (j = 0; j < num_xcc; j++) {
-					if (xcc_id != adev->mes.master_xcc_ids[j])
-						continue;
-
-					tmp = RREG32_SOC15(GC, GET_INST(GC, j),
-							   regSCRATCH_REG0);
-					if (tmp != 0xDEADBEEF)
-						break;
-				}
-			}
-		}
-
-		if (tmp == 0xDEADBEEF)
-			break;
-
-		if (amdgpu_emu_mode == 1)
-			msleep(1);
-		else
-			udelay(1);
-	}
-
-	if (i >= adev->usec_timeout) {
-		dev_err(adev->dev, "xcc%d: mes self test (%s) failed\n", xcc_id,
-		      queue_type == AMDGPU_RING_TYPE_SDMA ? "sdma" : "compute");
-
-		while (halt_if_hws_hang)
-			schedule();
-
-		r = -ETIMEDOUT;
-	} else {
-		dev_info(adev->dev, "xcc%d: mes self test (%s) pass\n", xcc_id,
-		      queue_type == AMDGPU_RING_TYPE_SDMA ? "sdma" : "compute");
-	}
-
-	return r;
-}
-
-#define USER_CTX_SIZE (PAGE_SIZE * 2)
-#define USER_CTX_VA AMDGPU_VA_RESERVED_BOTTOM
-#define RING_OFFSET(addr) ((addr))
-#define EOP_OFFSET(addr)  ((addr) + PAGE_SIZE)
-#define WPTR_OFFSET(addr) ((addr) + USER_CTX_SIZE - sizeof(u64))
-#define RPTR_OFFSET(addr) ((addr) + USER_CTX_SIZE - sizeof(u64) * 2)
-#define FENCE_OFFSET(addr) ((addr) + USER_CTX_SIZE - sizeof(u64) * 3)
-
-static int mes_v12_1_test_queue(struct amdgpu_device *adev, int xcc_id,
-		     int pasid, struct amdgpu_vm *vm, u64 meta_gpu_addr,
-		     u64 queue_gpu_addr, void *ctx_ptr, int queue_type)
-{
-	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
-	struct amdgpu_mqd *mqd_mgr = &adev->mqds[queue_type];
-	struct amdgpu_mqd_prop mqd_prop = {0};
-	struct mes_add_queue_input add_queue = {0};
-	struct mes_remove_queue_input remove_queue = {0};
-	struct amdgpu_bo *mqd_bo = NULL;
-	int num_xcc = NUM_XCC(adev->gfx.xcc_mask);
-	int i, r, off, mqd_size, mqd_count = 1;
-	void *mqd_ptr = NULL;
-	u64 mqd_gpu_addr, doorbell_idx;
-
-	/* extra one page size padding for mes fw */
-	mqd_size = mqd_mgr->mqd_size + PAGE_SIZE;
-
-	if (queue_type == AMDGPU_RING_TYPE_SDMA) {
-		doorbell_idx = adev->mes.db_start_dw_offset +	\
-			adev->doorbell_index.sdma_engine[0];
-	} else {
-		doorbell_idx = adev->mes.db_start_dw_offset + \
-			adev->doorbell_index.userqueue_start;
-	}
-
-	if (adev->mes.enable_coop_mode &&
-	    queue_type == AMDGPU_RING_TYPE_COMPUTE) {
-		for (i = 0, mqd_count = 0; i < num_xcc; i++) {
-			if (adev->mes.master_xcc_ids[i] == xcc_id)
-				mqd_count++;
-		}
-		mqd_size *= mqd_count;
-	}
-
-	r = mes_v12_1_alloc_test_buf(adev, &mqd_bo, &mqd_gpu_addr,
-				     &mqd_ptr, mqd_size * mqd_count);
-	if (r < 0)
-		return r;
-
-	mqd_prop.mqd_gpu_addr = mqd_gpu_addr;
-	mqd_prop.hqd_base_gpu_addr = RING_OFFSET(USER_CTX_VA);
-	mqd_prop.eop_gpu_addr = EOP_OFFSET(USER_CTX_VA);
-	mqd_prop.wptr_gpu_addr = WPTR_OFFSET(USER_CTX_VA);
-	mqd_prop.rptr_gpu_addr = RPTR_OFFSET(USER_CTX_VA);
-	mqd_prop.doorbell_index = doorbell_idx;
-	mqd_prop.queue_size = PAGE_SIZE;
-	mqd_prop.mqd_stride_size = mqd_size;
-	mqd_prop.use_doorbell = true;
-	mqd_prop.hqd_active = false;
-
-	mqd_mgr->init_mqd(adev, mqd_ptr, &mqd_prop);
-	if (mqd_count > 1) {
-		for (i = 1; i < mqd_count; i++) {
-			off = mqd_size * i;
-			mqd_prop.mqd_gpu_addr = mqd_gpu_addr + off;
-			mqd_mgr->init_mqd(adev, (char *)mqd_ptr + off,
-					  &mqd_prop);
-		}
-	}
-
-	add_queue.xcc_id = xcc_id;
-	add_queue.process_id = pasid;
-	add_queue.page_table_base_addr = adev->vm_manager.vram_base_offset +
-		amdgpu_bo_gpu_offset(vm->root.bo) - adev->gmc.vram_start;
-	add_queue.process_va_start = 0;
-	add_queue.process_va_end = adev->vm_manager.max_pfn - 1;
-	add_queue.process_context_addr = meta_gpu_addr;
-	add_queue.gang_context_addr = meta_gpu_addr + AMDGPU_MES_PROC_CTX_SIZE;
-	add_queue.doorbell_offset = doorbell_idx;
-	add_queue.mqd_addr = mqd_gpu_addr;
-	add_queue.wptr_addr = mqd_prop.wptr_gpu_addr;
-	add_queue.wptr_mc_addr = WPTR_OFFSET(queue_gpu_addr);
-	add_queue.queue_type = queue_type;
-	add_queue.vm_cntx_cntl = hub->vm_cntx_cntl;
-
-	r = mes_v12_1_add_hw_queue(&adev->mes, &add_queue);
-	if (r)
-		goto error;
-
-	mes_v12_1_test_ring(adev, xcc_id, (u32 *)RING_OFFSET((char *)ctx_ptr),
-			    FENCE_OFFSET(USER_CTX_VA),
-			    FENCE_OFFSET((char *)ctx_ptr),
-			    WPTR_OFFSET((char *)ctx_ptr),
-			    doorbell_idx, queue_type);
-
-	remove_queue.xcc_id = xcc_id;
-	remove_queue.doorbell_offset = doorbell_idx;
-	remove_queue.gang_context_addr = add_queue.gang_context_addr;
-	r = mes_v12_1_remove_hw_queue(&adev->mes, &remove_queue);
-
-error:
-	amdgpu_bo_free_kernel(&mqd_bo, &mqd_gpu_addr, &mqd_ptr);
-	return r;
-}
-
-static int mes_v12_1_self_test(struct amdgpu_device *adev, int xcc_id)
-{
-	int queue_types[] = { AMDGPU_RING_TYPE_COMPUTE,
-		              /* AMDGPU_RING_TYPE_SDMA */ };
-	struct amdgpu_bo_va *bo_va = NULL;
-	struct amdgpu_vm *vm = NULL;
-	struct amdgpu_bo *meta_bo = NULL, *ctx_bo = NULL;
-	void *meta_ptr = NULL, *ctx_ptr = NULL;
-	u64 meta_gpu_addr, ctx_gpu_addr;
-	int size, i, r, pasid;
-
-	pasid = amdgpu_pasid_alloc(16);
-	if (pasid < 0)
-		pasid = 0;
-
-	size = AMDGPU_MES_PROC_CTX_SIZE + AMDGPU_MES_GANG_CTX_SIZE;
-	r = mes_v12_1_alloc_test_buf(adev, &meta_bo, &meta_gpu_addr,
-				     &meta_ptr, size);
-	if (r < 0)
-		goto err2;
-
-	r = mes_v12_1_alloc_test_buf(adev, &ctx_bo, &ctx_gpu_addr,
-				     &ctx_ptr, USER_CTX_SIZE);
-	if (r < 0)
-		goto err2;
-
-	vm = kzalloc(sizeof(*vm), GFP_KERNEL);
-	if (!vm) {
-		r = -ENOMEM;
-		goto err2;
-	}
-
-	r = amdgpu_vm_init(adev, vm, -1, pasid);
-	if (r)
-		goto err1;
-
-	r = mes_v12_1_map_test_bo(adev, ctx_bo, vm, &bo_va,
-				  USER_CTX_VA, USER_CTX_SIZE);
-	if (r)
-		goto err0;
-
-	for (i = 0; i < ARRAY_SIZE(queue_types); i++) {
-		memset(ctx_ptr, 0, USER_CTX_SIZE);
-
-		r = mes_v12_1_test_queue(adev, xcc_id, pasid, vm, meta_gpu_addr,
-					 ctx_gpu_addr, ctx_ptr, queue_types[i]);
-		if (r)
-			break;
-	}
-
-	amdgpu_unmap_static_csa(adev, vm, ctx_bo, bo_va, USER_CTX_VA);
-err0:
-	amdgpu_vm_fini(adev, vm);
-err1:
-	kfree(vm);
-err2:
-	amdgpu_bo_free_kernel(&meta_bo, &meta_gpu_addr, &meta_ptr);
-	amdgpu_bo_free_kernel(&ctx_bo, &ctx_gpu_addr, &ctx_ptr);
-	amdgpu_pasid_free(pasid);
-	return r;
-}
-
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c
index cc688ae79e84b..47d07cd25fc41 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_8.c
@@ -29,7 +29,6 @@
 
 #include "soc15_common.h"
 #include "soc15.h"
-#include "amdgpu_ras.h"
 #include "amdgpu_psp.h"
 
 #define regVM_L2_CNTL3_DEFAULT	0x80100007
@@ -636,236 +635,8 @@ const struct amdgpu_mmhub_funcs mmhub_v1_8_funcs = {
 	.get_clockgating = mmhub_v1_8_get_clockgating,
 };
 
-static const struct amdgpu_ras_err_status_reg_entry mmhub_v1_8_ce_reg_list[] = {
-	{AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA0_CE_ERR_STATUS_LO, regMMEA0_CE_ERR_STATUS_HI),
-	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA0"},
-	{AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA1_CE_ERR_STATUS_LO, regMMEA1_CE_ERR_STATUS_HI),
-	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA1"},
-	{AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA2_CE_ERR_STATUS_LO, regMMEA2_CE_ERR_STATUS_HI),
-	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA2"},
-	{AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA3_CE_ERR_STATUS_LO, regMMEA3_CE_ERR_STATUS_HI),
-	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA3"},
-	{AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA4_CE_ERR_STATUS_LO, regMMEA4_CE_ERR_STATUS_HI),
-	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA4"},
-	{AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMM_CANE_CE_ERR_STATUS_LO, regMM_CANE_CE_ERR_STATUS_HI),
-	1, 0, "MM_CANE"},
-};
-
-static const struct amdgpu_ras_err_status_reg_entry mmhub_v1_8_ue_reg_list[] = {
-	{AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA0_UE_ERR_STATUS_LO, regMMEA0_UE_ERR_STATUS_HI),
-	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA0"},
-	{AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA1_UE_ERR_STATUS_LO, regMMEA1_UE_ERR_STATUS_HI),
-	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA1"},
-	{AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA2_UE_ERR_STATUS_LO, regMMEA2_UE_ERR_STATUS_HI),
-	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA2"},
-	{AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA3_UE_ERR_STATUS_LO, regMMEA3_UE_ERR_STATUS_HI),
-	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA3"},
-	{AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA4_UE_ERR_STATUS_LO, regMMEA4_UE_ERR_STATUS_HI),
-	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA4"},
-	{AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMM_CANE_UE_ERR_STATUS_LO, regMM_CANE_UE_ERR_STATUS_HI),
-	1, 0, "MM_CANE"},
-};
-
-static const struct amdgpu_ras_memory_id_entry mmhub_v1_8_ras_memory_list[] = {
-	{AMDGPU_MMHUB_WGMI_PAGEMEM, "MMEA_WGMI_PAGEMEM"},
-	{AMDGPU_MMHUB_RGMI_PAGEMEM, "MMEA_RGMI_PAGEMEM"},
-	{AMDGPU_MMHUB_WDRAM_PAGEMEM, "MMEA_WDRAM_PAGEMEM"},
-	{AMDGPU_MMHUB_RDRAM_PAGEMEM, "MMEA_RDRAM_PAGEMEM"},
-	{AMDGPU_MMHUB_WIO_CMDMEM, "MMEA_WIO_CMDMEM"},
-	{AMDGPU_MMHUB_RIO_CMDMEM, "MMEA_RIO_CMDMEM"},
-	{AMDGPU_MMHUB_WGMI_CMDMEM, "MMEA_WGMI_CMDMEM"},
-	{AMDGPU_MMHUB_RGMI_CMDMEM, "MMEA_RGMI_CMDMEM"},
-	{AMDGPU_MMHUB_WDRAM_CMDMEM, "MMEA_WDRAM_CMDMEM"},
-	{AMDGPU_MMHUB_RDRAM_CMDMEM, "MMEA_RDRAM_CMDMEM"},
-	{AMDGPU_MMHUB_MAM_DMEM0, "MMEA_MAM_DMEM0"},
-	{AMDGPU_MMHUB_MAM_DMEM1, "MMEA_MAM_DMEM1"},
-	{AMDGPU_MMHUB_MAM_DMEM2, "MMEA_MAM_DMEM2"},
-	{AMDGPU_MMHUB_MAM_DMEM3, "MMEA_MAM_DMEM3"},
-	{AMDGPU_MMHUB_WRET_TAGMEM, "MMEA_WRET_TAGMEM"},
-	{AMDGPU_MMHUB_RRET_TAGMEM, "MMEA_RRET_TAGMEM"},
-	{AMDGPU_MMHUB_WIO_DATAMEM, "MMEA_WIO_DATAMEM"},
-	{AMDGPU_MMHUB_WGMI_DATAMEM, "MMEA_WGMI_DATAMEM"},
-	{AMDGPU_MMHUB_WDRAM_DATAMEM, "MMEA_WDRAM_DATAMEM"},
-};
-
-static void mmhub_v1_8_inst_query_ras_error_count(struct amdgpu_device *adev,
-						  uint32_t mmhub_inst,
-						  void *ras_err_status)
-{
-	struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status;
-	unsigned long ue_count = 0, ce_count = 0;
-
-	/* NOTE: mmhub is converted by aid_mask and the range is 0-3,
-	 * which can be used as die ID directly */
-	struct amdgpu_smuio_mcm_config_info mcm_info = {
-		.socket_id = adev->smuio.funcs->get_socket_id(adev),
-		.die_id = mmhub_inst,
-	};
-
-	amdgpu_ras_inst_query_ras_error_count(adev,
-					mmhub_v1_8_ce_reg_list,
-					ARRAY_SIZE(mmhub_v1_8_ce_reg_list),
-					mmhub_v1_8_ras_memory_list,
-					ARRAY_SIZE(mmhub_v1_8_ras_memory_list),
-					mmhub_inst,
-					AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE,
-					&ce_count);
-	amdgpu_ras_inst_query_ras_error_count(adev,
-					mmhub_v1_8_ue_reg_list,
-					ARRAY_SIZE(mmhub_v1_8_ue_reg_list),
-					mmhub_v1_8_ras_memory_list,
-					ARRAY_SIZE(mmhub_v1_8_ras_memory_list),
-					mmhub_inst,
-					AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
-					&ue_count);
-
-	amdgpu_ras_error_statistic_ce_count(err_data, &mcm_info, ce_count);
-	amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, ue_count);
-}
-
-static void mmhub_v1_8_query_ras_error_count(struct amdgpu_device *adev,
-					     void *ras_err_status)
-{
-	uint32_t inst_mask;
-	uint32_t i;
-
-	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__MMHUB)) {
-		dev_warn(adev->dev, "MMHUB RAS is not supported\n");
-		return;
-	}
-
-	inst_mask = adev->aid_mask;
-	for_each_inst(i, inst_mask)
-		mmhub_v1_8_inst_query_ras_error_count(adev, i, ras_err_status);
-}
-
-static void mmhub_v1_8_inst_reset_ras_error_count(struct amdgpu_device *adev,
-						  uint32_t mmhub_inst)
-{
-	amdgpu_ras_inst_reset_ras_error_count(adev,
-					mmhub_v1_8_ce_reg_list,
-					ARRAY_SIZE(mmhub_v1_8_ce_reg_list),
-					mmhub_inst);
-	amdgpu_ras_inst_reset_ras_error_count(adev,
-					mmhub_v1_8_ue_reg_list,
-					ARRAY_SIZE(mmhub_v1_8_ue_reg_list),
-					mmhub_inst);
-}
-
-static void mmhub_v1_8_reset_ras_error_count(struct amdgpu_device *adev)
-{
-	uint32_t inst_mask;
-	uint32_t i;
-
-	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__MMHUB)) {
-		dev_warn(adev->dev, "MMHUB RAS is not supported\n");
-		return;
-	}
-
-	inst_mask = adev->aid_mask;
-	for_each_inst(i, inst_mask)
-		mmhub_v1_8_inst_reset_ras_error_count(adev, i);
-}
-
-static const struct amdgpu_ras_block_hw_ops mmhub_v1_8_ras_hw_ops = {
-	.query_ras_error_count = mmhub_v1_8_query_ras_error_count,
-	.reset_ras_error_count = mmhub_v1_8_reset_ras_error_count,
-};
-
-static int mmhub_v1_8_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
-				      enum aca_smu_type type, void *data)
-{
-	struct aca_bank_info info;
-	u64 misc0;
-	int ret;
-
-	ret = aca_bank_info_decode(bank, &info);
-	if (ret)
-		return ret;
-
-	misc0 = bank->regs[ACA_REG_IDX_MISC0];
-	switch (type) {
-	case ACA_SMU_TYPE_UE:
-		bank->aca_err_type = ACA_ERROR_TYPE_UE;
-		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE,
-						     1ULL);
-		break;
-	case ACA_SMU_TYPE_CE:
-		bank->aca_err_type = ACA_ERROR_TYPE_CE;
-		ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type,
-						     ACA_REG__MISC0__ERRCNT(misc0));
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-/* reference to smu driver if header file */
-static int mmhub_v1_8_err_codes[] = {
-	0, 1, 2, 3, 4, /* CODE_DAGB0 - 4 */
-	5, 6, 7, 8, 9, /* CODE_EA0 - 4 */
-	10, /* CODE_UTCL2_ROUTER */
-	11, /* CODE_VML2 */
-	12, /* CODE_VML2_WALKER */
-	13, /* CODE_MMCANE */
-};
-
-static bool mmhub_v1_8_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
-					 enum aca_smu_type type, void *data)
-{
-	u32 instlo;
-
-	instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
-	instlo &= GENMASK(31, 1);
-
-	if (instlo != mmSMNAID_AID0_MCA_SMU)
-		return false;
-
-	if (aca_bank_check_error_codes(handle->adev, bank,
-				       mmhub_v1_8_err_codes,
-				       ARRAY_SIZE(mmhub_v1_8_err_codes)))
-		return false;
-
-	return true;
-}
-
-static const struct aca_bank_ops mmhub_v1_8_aca_bank_ops = {
-	.aca_bank_parser = mmhub_v1_8_aca_bank_parser,
-	.aca_bank_is_valid = mmhub_v1_8_aca_bank_is_valid,
-};
-
-static const struct aca_info mmhub_v1_8_aca_info = {
-	.hwip = ACA_HWIP_TYPE_SMU,
-	.mask = ACA_ERROR_UE_MASK,
-	.bank_ops = &mmhub_v1_8_aca_bank_ops,
-};
-
-static int mmhub_v1_8_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
-{
-	int r;
-
-	r = amdgpu_ras_block_late_init(adev, ras_block);
-	if (r)
-		return r;
-
-	r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__MMHUB,
-				&mmhub_v1_8_aca_info, NULL);
-	if (r)
-		goto late_fini;
-
-	return 0;
-
-late_fini:
-	amdgpu_ras_block_late_fini(adev, ras_block);
-
-	return r;
-}
-
 struct amdgpu_mmhub_ras mmhub_v1_8_ras = {
 	.ras_block = {
-		.hw_ops = &mmhub_v1_8_ras_hw_ops,
-		.ras_late_init = mmhub_v1_8_ras_late_init,
+		.hw_ops = NULL,
 	},
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c
index 49b7f16a941f6..5827c758b373d 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c
@@ -36,40 +36,41 @@
 
 static const char *mmhub_client_ids_v4_2_0[][2] = {
 	[0][0] = "VMC",
-	[4][0] = "DCEDMC",
-	[5][0] = "DCEVGA",
-	[6][0] = "MP0",
-	[7][0] = "MP1",
+	[2][0] = "MPNHT",
+	[7][0] = "MPIFOE",
 	[8][0] = "MPIO",
-	[16][0] = "HDP",
-	[17][0] = "LSDMA",
-	[18][0] = "JPEG",
-	[19][0] = "VCNU0",
-	[21][0] = "VSCH",
-	[22][0] = "VCNU1",
-	[23][0] = "VCN1",
-	[32+20][0] = "VCN0",
-	[2][1] = "DBGUNBIO",
-	[3][1] = "DCEDWB",
-	[4][1] = "DCEDMC",
-	[5][1] = "DCEVGA",
-	[6][1] = "MP0",
-	[7][1] = "MP1",
+	[11][0] = "JPEG0",
+	[12][0] = "VCN0",
+	[13][0] = "VCNU0",
+	[14][0] = "VSCH0",
+	[15][0] = "LSDMA",
+	[32+5][0] = "MPRAS",
+	[32+6][0] = "MP1",
+	[32+7][0] = "MP0",
+	[32+11][0] = "JPEG1",
+	[32+12][0] = "VCN1",
+	[32+13][0] = "VCNU1",
+	[32+14][0] = "VSCH1",
+	[2][1] = "MPNHT",
+	[3][1] = "DBGU0",
+	[7][1] = "MPIFOE",
 	[8][1] = "MPIO",
-	[10][1] = "DBGU0",
-	[11][1] = "DBGU1",
-	[12][1] = "DBGU2",
-	[13][1] = "DBGU3",
-	[14][1] = "XDP",
-	[15][1] = "OSSSYS",
-	[16][1] = "HDP",
-	[17][1] = "LSDMA",
-	[18][1] = "JPEG",
-	[19][1] = "VCNU0",
-	[20][1] = "VCN0",
-	[21][1] = "VSCH",
-	[22][1] = "VCNU1",
-	[23][1] = "VCN1",
+	[10][1] = "UTCL2_NHT",
+	[11][1] = "JPEG0",
+	[12][1] = "VCN0",
+	[13][1] = "VCNU0",
+	[14][1] = "VSCH0",
+	[15][1] = "LSDMA",
+	[32+3][1] = "DBGU1",
+	[32+4][1] = "DBGU2",
+	[32+5][1] = "MPRAS",
+	[32+6][1] = "MP1",
+	[32+7][1] = "MP0",
+	[32+8][1] = "IH",
+	[32+11][1] = "JPEG1",
+	[32+12][1] = "VCN1",
+	[32+13][1] = "VCNU1",
+	[32+14][1] = "VSCH1",
 };
 
 static int mmhub_v4_2_0_get_xgmi_info(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/nv.c b/drivers/gpu/drm/amd/amdgpu/nv.c
index 72edf5326b05b..96faaf8c5737e 100644
--- a/drivers/gpu/drm/amd/amdgpu/nv.c
+++ b/drivers/gpu/drm/amd/amdgpu/nv.c
@@ -36,6 +36,7 @@
 #include "amdgpu_psp.h"
 #include "atom.h"
 #include "amd_pcie.h"
+#include "amdgpu_video_codecs.h"
 
 #include "gc/gc_10_1_0_offset.h"
 #include "gc/gc_10_1_0_sh_mask.h"
@@ -507,11 +508,6 @@ void nv_set_virt_ops(struct amdgpu_device *adev)
 	adev->virt.ops = &xgpu_nv_virt_ops;
 }
 
-static bool nv_need_full_reset(struct amdgpu_device *adev)
-{
-	return true;
-}
-
 static bool nv_need_reset_on_init(struct amdgpu_device *adev)
 {
 	u32 sol_reg;
@@ -595,7 +591,6 @@ static const struct amdgpu_asic_funcs nv_asic_funcs = {
 	.set_vce_clocks = &nv_set_vce_clocks,
 	.get_config_memsize = &nv_get_config_memsize,
 	.init_doorbell_index = &nv_init_doorbell_index,
-	.need_full_reset = &nv_need_full_reset,
 	.need_reset_on_init = &nv_need_reset_on_init,
 	.get_pcie_replay_count = &amdgpu_nbio_get_pcie_replay_count,
 	.supports_baco = &amdgpu_dpm_is_baco_supported,
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
index ac34bac3c8391..ad618b78685f6 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
+++ b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
@@ -110,6 +110,7 @@ enum psp_gfx_cmd_id
     GFX_CMD_ID_PERF_HW            = 0x0000004C,   /* performance monitor */
     GFX_CMD_ID_FB_FW_RESERV_ADDR  = 0x00000050,  /* Query FW reservation addr */
     GFX_CMD_ID_FB_FW_RESERV_EXT_ADDR = 0x00000051,  /* Query FW reservation extended addr */
+    GFX_CMD_ID_SET_MMHUB_ECO_SEC_LEVEL = 0x0000005D,  /* Set MMHUB ECO sec lvls on VCN block */
 };
 
 /* PSP boot config sub-commands */
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index 93ec52c1f3678..c7253b908351c 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -138,7 +138,7 @@ static int sdma_v2_4_init_microcode(struct amdgpu_device *adev)
 		chip_name = "topaz";
 		break;
 	default:
-		BUG();
+		return -EINVAL;
 	}
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
@@ -536,7 +536,7 @@ static int sdma_v2_4_ring_test_ring(struct amdgpu_ring *ring)
 	u32 tmp;
 	u64 gpu_addr;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -567,7 +567,7 @@ static int sdma_v2_4_ring_test_ring(struct amdgpu_ring *ring)
 		r = -ETIMEDOUT;
 
 error_free_wb:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -590,7 +590,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	u64 gpu_addr;
 	long r;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -635,7 +635,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 3fde9be746900..05cce4bde73eb 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -298,7 +298,8 @@ static int sdma_v3_0_init_microcode(struct amdgpu_device *adev)
 	case CHIP_STONEY:
 		chip_name = "stoney";
 		break;
-	default: BUG();
+	default:
+		return -EINVAL;
 	}
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
@@ -810,7 +811,7 @@ static int sdma_v3_0_ring_test_ring(struct amdgpu_ring *ring)
 	u32 tmp;
 	u64 gpu_addr;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -841,7 +842,7 @@ static int sdma_v3_0_ring_test_ring(struct amdgpu_ring *ring)
 		r = -ETIMEDOUT;
 
 error_free_wb:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -864,7 +865,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	u64 gpu_addr;
 	long r;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -908,7 +909,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -1237,65 +1238,6 @@ static int sdma_v3_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 	return -ETIMEDOUT;
 }
 
-static bool sdma_v3_0_check_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 srbm_soft_reset = 0;
-	u32 tmp = RREG32(mmSRBM_STATUS2);
-
-	if ((tmp & SRBM_STATUS2__SDMA_BUSY_MASK) ||
-	    (tmp & SRBM_STATUS2__SDMA1_BUSY_MASK)) {
-		srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA_MASK;
-		srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA1_MASK;
-	}
-
-	if (srbm_soft_reset) {
-		adev->sdma.srbm_soft_reset = srbm_soft_reset;
-		return true;
-	} else {
-		adev->sdma.srbm_soft_reset = 0;
-		return false;
-	}
-}
-
-static int sdma_v3_0_pre_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 srbm_soft_reset = 0;
-
-	if (!adev->sdma.srbm_soft_reset)
-		return 0;
-
-	srbm_soft_reset = adev->sdma.srbm_soft_reset;
-
-	if (REG_GET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_SDMA) ||
-	    REG_GET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_SDMA1)) {
-		sdma_v3_0_ctx_switch_enable(adev, false);
-		sdma_v3_0_enable(adev, false);
-	}
-
-	return 0;
-}
-
-static int sdma_v3_0_post_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 srbm_soft_reset = 0;
-
-	if (!adev->sdma.srbm_soft_reset)
-		return 0;
-
-	srbm_soft_reset = adev->sdma.srbm_soft_reset;
-
-	if (REG_GET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_SDMA) ||
-	    REG_GET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_SDMA1)) {
-		sdma_v3_0_gfx_resume(adev);
-		sdma_v3_0_rlc_resume(adev);
-	}
-
-	return 0;
-}
-
 static int sdma_v3_0_soft_reset(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -1552,9 +1494,6 @@ static const struct amd_ip_funcs sdma_v3_0_ip_funcs = {
 	.resume = sdma_v3_0_resume,
 	.is_idle = sdma_v3_0_is_idle,
 	.wait_for_idle = sdma_v3_0_wait_for_idle,
-	.check_soft_reset = sdma_v3_0_check_soft_reset,
-	.pre_soft_reset = sdma_v3_0_pre_soft_reset,
-	.post_soft_reset = sdma_v3_0_post_soft_reset,
 	.soft_reset = sdma_v3_0_soft_reset,
 	.set_clockgating_state = sdma_v3_0_set_clockgating_state,
 	.set_powergating_state = sdma_v3_0_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index cb64d17000df0..20c8ebf0e1591 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -1468,7 +1468,7 @@ static int sdma_v4_0_ring_test_ring(struct amdgpu_ring *ring)
 	u32 tmp;
 	u64 gpu_addr;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -1499,7 +1499,7 @@ static int sdma_v4_0_ring_test_ring(struct amdgpu_ring *ring)
 		r = -ETIMEDOUT;
 
 error_free_wb:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -1522,7 +1522,7 @@ static int sdma_v4_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	u32 tmp = 0;
 	u64 gpu_addr;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -1567,7 +1567,7 @@ static int sdma_v4_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
index 8652928861ad7..6fe3d3d56c406 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
@@ -95,8 +95,6 @@ static const struct amdgpu_hwip_reg_entry sdma_reg_list_4_4_2[] = {
 	SOC15_REG_ENTRY_STR(GC, 0, regSDMA_VM_CNTL)
 };
 
-#define mmSMNAID_AID0_MCA_SMU 0x03b30400
-
 #define WREG32_SDMA(instance, offset, value) \
 	WREG32(sdma_v4_4_2_get_reg_offset(adev, (instance), (offset)), value)
 #define RREG32_SDMA(instance, offset) \
@@ -1064,7 +1062,7 @@ static int sdma_v4_4_2_ring_test_ring(struct amdgpu_ring *ring)
 	u32 tmp;
 	u64 gpu_addr;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -1095,7 +1093,7 @@ static int sdma_v4_4_2_ring_test_ring(struct amdgpu_ring *ring)
 		r = -ETIMEDOUT;
 
 error_free_wb:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -1118,7 +1116,7 @@ static int sdma_v4_4_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	u32 tmp = 0;
 	u64 gpu_addr;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -1163,7 +1161,7 @@ static int sdma_v4_4_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -1359,6 +1357,19 @@ static int sdma_v4_4_2_early_init(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	int r;
 
+	switch (amdgpu_user_queue) {
+	case -1:
+	case 0:
+	default:
+		adev->sdma.no_user_submission = false;
+		adev->sdma.disable_uq = true;
+		break;
+	case 2:
+		adev->sdma.no_user_submission = true;
+		adev->sdma.disable_uq = true;
+		break;
+	}
+
 	r = sdma_v4_4_2_init_microcode(adev);
 	if (r)
 		return r;
@@ -1478,6 +1489,7 @@ static int sdma_v4_4_2_sw_init(struct amdgpu_ip_block *ip_block)
 		/* doorbell size is 2 dwords, get DWORD offset */
 		ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1;
 		ring->vm_hub = AMDGPU_MMHUB0(aid_id);
+		ring->no_user_submission = adev->sdma.no_user_submission;
 
 		sprintf(ring->name, "sdma%d.%d", aid_id,
 				i % adev->sdma.num_inst_per_aid);
@@ -2404,187 +2416,9 @@ struct amdgpu_xcp_ip_funcs sdma_v4_4_2_xcp_funcs = {
 	.resume = &sdma_v4_4_2_xcp_resume
 };
 
-static const struct amdgpu_ras_err_status_reg_entry sdma_v4_2_2_ue_reg_list[] = {
-	{AMDGPU_RAS_REG_ENTRY(SDMA0, 0, regSDMA_UE_ERR_STATUS_LO, regSDMA_UE_ERR_STATUS_HI),
-	1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SDMA"},
-};
-
-static const struct amdgpu_ras_memory_id_entry sdma_v4_4_2_ras_memory_list[] = {
-	{AMDGPU_SDMA_MBANK_DATA_BUF0, "SDMA_MBANK_DATA_BUF0"},
-	{AMDGPU_SDMA_MBANK_DATA_BUF1, "SDMA_MBANK_DATA_BUF1"},
-	{AMDGPU_SDMA_MBANK_DATA_BUF2, "SDMA_MBANK_DATA_BUF2"},
-	{AMDGPU_SDMA_MBANK_DATA_BUF3, "SDMA_MBANK_DATA_BUF3"},
-	{AMDGPU_SDMA_MBANK_DATA_BUF4, "SDMA_MBANK_DATA_BUF4"},
-	{AMDGPU_SDMA_MBANK_DATA_BUF5, "SDMA_MBANK_DATA_BUF5"},
-	{AMDGPU_SDMA_MBANK_DATA_BUF6, "SDMA_MBANK_DATA_BUF6"},
-	{AMDGPU_SDMA_MBANK_DATA_BUF7, "SDMA_MBANK_DATA_BUF7"},
-	{AMDGPU_SDMA_MBANK_DATA_BUF8, "SDMA_MBANK_DATA_BUF8"},
-	{AMDGPU_SDMA_MBANK_DATA_BUF9, "SDMA_MBANK_DATA_BUF9"},
-	{AMDGPU_SDMA_MBANK_DATA_BUF10, "SDMA_MBANK_DATA_BUF10"},
-	{AMDGPU_SDMA_MBANK_DATA_BUF11, "SDMA_MBANK_DATA_BUF11"},
-	{AMDGPU_SDMA_MBANK_DATA_BUF12, "SDMA_MBANK_DATA_BUF12"},
-	{AMDGPU_SDMA_MBANK_DATA_BUF13, "SDMA_MBANK_DATA_BUF13"},
-	{AMDGPU_SDMA_MBANK_DATA_BUF14, "SDMA_MBANK_DATA_BUF14"},
-	{AMDGPU_SDMA_MBANK_DATA_BUF15, "SDMA_MBANK_DATA_BUF15"},
-	{AMDGPU_SDMA_UCODE_BUF, "SDMA_UCODE_BUF"},
-	{AMDGPU_SDMA_RB_CMD_BUF, "SDMA_RB_CMD_BUF"},
-	{AMDGPU_SDMA_IB_CMD_BUF, "SDMA_IB_CMD_BUF"},
-	{AMDGPU_SDMA_UTCL1_RD_FIFO, "SDMA_UTCL1_RD_FIFO"},
-	{AMDGPU_SDMA_UTCL1_RDBST_FIFO, "SDMA_UTCL1_RDBST_FIFO"},
-	{AMDGPU_SDMA_UTCL1_WR_FIFO, "SDMA_UTCL1_WR_FIFO"},
-	{AMDGPU_SDMA_DATA_LUT_FIFO, "SDMA_DATA_LUT_FIFO"},
-	{AMDGPU_SDMA_SPLIT_DAT_BUF, "SDMA_SPLIT_DAT_BUF"},
-};
-
-static void sdma_v4_4_2_inst_query_ras_error_count(struct amdgpu_device *adev,
-						   uint32_t sdma_inst,
-						   void *ras_err_status)
-{
-	struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status;
-	uint32_t sdma_dev_inst = GET_INST(SDMA0, sdma_inst);
-	unsigned long ue_count = 0;
-	struct amdgpu_smuio_mcm_config_info mcm_info = {
-		.socket_id = adev->smuio.funcs->get_socket_id(adev),
-		.die_id = adev->sdma.instance[sdma_inst].aid_id,
-	};
-
-	/* sdma v4_4_2 doesn't support query ce counts */
-	amdgpu_ras_inst_query_ras_error_count(adev,
-					sdma_v4_2_2_ue_reg_list,
-					ARRAY_SIZE(sdma_v4_2_2_ue_reg_list),
-					sdma_v4_4_2_ras_memory_list,
-					ARRAY_SIZE(sdma_v4_4_2_ras_memory_list),
-					sdma_dev_inst,
-					AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
-					&ue_count);
-
-	amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, ue_count);
-}
-
-static void sdma_v4_4_2_query_ras_error_count(struct amdgpu_device *adev,
-					      void *ras_err_status)
-{
-	uint32_t inst_mask;
-	int i = 0;
-
-	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
-	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
-		for_each_inst(i, inst_mask)
-			sdma_v4_4_2_inst_query_ras_error_count(adev, i, ras_err_status);
-	} else {
-		dev_warn(adev->dev, "SDMA RAS is not supported\n");
-	}
-}
-
-static void sdma_v4_4_2_inst_reset_ras_error_count(struct amdgpu_device *adev,
-						   uint32_t sdma_inst)
-{
-	uint32_t sdma_dev_inst = GET_INST(SDMA0, sdma_inst);
-
-	amdgpu_ras_inst_reset_ras_error_count(adev,
-					sdma_v4_2_2_ue_reg_list,
-					ARRAY_SIZE(sdma_v4_2_2_ue_reg_list),
-					sdma_dev_inst);
-}
-
-static void sdma_v4_4_2_reset_ras_error_count(struct amdgpu_device *adev)
-{
-	uint32_t inst_mask;
-	int i = 0;
-
-	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
-	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
-		for_each_inst(i, inst_mask)
-			sdma_v4_4_2_inst_reset_ras_error_count(adev, i);
-	} else {
-		dev_warn(adev->dev, "SDMA RAS is not supported\n");
-	}
-}
-
-static const struct amdgpu_ras_block_hw_ops sdma_v4_4_2_ras_hw_ops = {
-	.query_ras_error_count = sdma_v4_4_2_query_ras_error_count,
-	.reset_ras_error_count = sdma_v4_4_2_reset_ras_error_count,
-};
-
-static int sdma_v4_4_2_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
-				       enum aca_smu_type type, void *data)
-{
-	struct aca_bank_info info;
-	u64 misc0;
-	int ret;
-
-	ret = aca_bank_info_decode(bank, &info);
-	if (ret)
-		return ret;
-
-	misc0 = bank->regs[ACA_REG_IDX_MISC0];
-	switch (type) {
-	case ACA_SMU_TYPE_UE:
-		bank->aca_err_type = ACA_ERROR_TYPE_UE;
-		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE,
-						     1ULL);
-		break;
-	case ACA_SMU_TYPE_CE:
-		bank->aca_err_type = ACA_ERROR_TYPE_CE;
-		ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type,
-						     ACA_REG__MISC0__ERRCNT(misc0));
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-/* CODE_SDMA0 - CODE_SDMA4, reference to smu driver if header file */
-static int sdma_v4_4_2_err_codes[] = { 33, 34, 35, 36 };
-
-static bool sdma_v4_4_2_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
-					  enum aca_smu_type type, void *data)
-{
-	u32 instlo;
-
-	instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
-	instlo &= GENMASK(31, 1);
-
-	if (instlo != mmSMNAID_AID0_MCA_SMU)
-		return false;
-
-	if (aca_bank_check_error_codes(handle->adev, bank,
-				       sdma_v4_4_2_err_codes,
-				       ARRAY_SIZE(sdma_v4_4_2_err_codes)))
-		return false;
-
-	return true;
-}
-
-static const struct aca_bank_ops sdma_v4_4_2_aca_bank_ops = {
-	.aca_bank_parser = sdma_v4_4_2_aca_bank_parser,
-	.aca_bank_is_valid = sdma_v4_4_2_aca_bank_is_valid,
-};
-
-static const struct aca_info sdma_v4_4_2_aca_info = {
-	.hwip = ACA_HWIP_TYPE_SMU,
-	.mask = ACA_ERROR_UE_MASK,
-	.bank_ops = &sdma_v4_4_2_aca_bank_ops,
-};
-
-static int sdma_v4_4_2_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
-{
-	int r;
-
-	r = amdgpu_sdma_ras_late_init(adev, ras_block);
-	if (r)
-		return r;
-
-	return amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__SDMA,
-				   &sdma_v4_4_2_aca_info, NULL);
-}
-
 static struct amdgpu_sdma_ras sdma_v4_4_2_ras = {
 	.ras_block = {
-		.hw_ops = &sdma_v4_4_2_ras_hw_ops,
-		.ras_late_init = sdma_v4_4_2_ras_late_init,
+		.hw_ops = NULL,
 	},
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
index b809942b1eb79..1ca0e7f65442e 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
@@ -1019,7 +1019,7 @@ static int sdma_v5_0_ring_test_ring(struct amdgpu_ring *ring)
 
 	tmp = 0xCAFEDEAD;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
 		return r;
@@ -1031,7 +1031,7 @@ static int sdma_v5_0_ring_test_ring(struct amdgpu_ring *ring)
 	r = amdgpu_ring_alloc(ring, 20);
 	if (r) {
 		drm_err(adev_to_drm(adev), "dma failed to lock ring %d (%d).\n", ring->idx, r);
-		amdgpu_device_wb_free(adev, index);
+		amdgpu_wb_free(adev, index);
 		return r;
 	}
 
@@ -1056,7 +1056,7 @@ static int sdma_v5_0_ring_test_ring(struct amdgpu_ring *ring)
 	if (i >= adev->usec_timeout)
 		r = -ETIMEDOUT;
 
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 
 	return r;
 }
@@ -1083,7 +1083,7 @@ static int sdma_v5_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	tmp = 0xCAFEDEAD;
 	memset(&ib, 0, sizeof(ib));
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
 		return r;
@@ -1135,7 +1135,7 @@ static int sdma_v5_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
index 87c1e29fd298d..81f1e9882177b 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
@@ -919,7 +919,7 @@ static int sdma_v5_2_ring_test_ring(struct amdgpu_ring *ring)
 
 	tmp = 0xCAFEDEAD;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
 		return r;
@@ -931,7 +931,7 @@ static int sdma_v5_2_ring_test_ring(struct amdgpu_ring *ring)
 	r = amdgpu_ring_alloc(ring, 20);
 	if (r) {
 		drm_err(adev_to_drm(adev), "dma failed to lock ring %d (%d).\n", ring->idx, r);
-		amdgpu_device_wb_free(adev, index);
+		amdgpu_wb_free(adev, index);
 		return r;
 	}
 
@@ -956,7 +956,7 @@ static int sdma_v5_2_ring_test_ring(struct amdgpu_ring *ring)
 	if (i >= adev->usec_timeout)
 		r = -ETIMEDOUT;
 
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 
 	return r;
 }
@@ -983,7 +983,7 @@ static int sdma_v5_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	tmp = 0xCAFEDEAD;
 	memset(&ib, 0, sizeof(ib));
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
 		return r;
@@ -1034,7 +1034,7 @@ static int sdma_v5_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
index d7537888e60cc..cf3d2997fff88 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
@@ -793,23 +793,6 @@ static int sdma_v6_0_soft_reset(struct amdgpu_ip_block *ip_block)
 	return sdma_v6_0_start(adev);
 }
 
-static bool sdma_v6_0_check_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	struct amdgpu_ring *ring;
-	int i, r;
-	long tmo = msecs_to_jiffies(1000);
-
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		ring = &adev->sdma.instance[i].ring;
-		r = amdgpu_ring_test_ib(ring, tmo);
-		if (r)
-			return true;
-	}
-
-	return false;
-}
-
 /**
  * sdma_v6_0_start - setup and start the async dma engines
  *
@@ -927,7 +910,7 @@ static int sdma_v6_0_ring_test_ring(struct amdgpu_ring *ring)
 
 	tmp = 0xCAFEDEAD;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
 		return r;
@@ -939,7 +922,7 @@ static int sdma_v6_0_ring_test_ring(struct amdgpu_ring *ring)
 	r = amdgpu_ring_alloc(ring, 5);
 	if (r) {
 		drm_err(adev_to_drm(adev), "dma failed to lock ring %d (%d).\n", ring->idx, r);
-		amdgpu_device_wb_free(adev, index);
+		amdgpu_wb_free(adev, index);
 		return r;
 	}
 
@@ -964,7 +947,7 @@ static int sdma_v6_0_ring_test_ring(struct amdgpu_ring *ring)
 	if (i >= adev->usec_timeout)
 		r = -ETIMEDOUT;
 
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 
 	return r;
 }
@@ -991,7 +974,7 @@ static int sdma_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	tmp = 0xCAFEDEAD;
 	memset(&ib, 0, sizeof(ib));
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
 		return r;
@@ -1042,7 +1025,7 @@ static int sdma_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -1747,7 +1730,6 @@ const struct amd_ip_funcs sdma_v6_0_ip_funcs = {
 	.is_idle = sdma_v6_0_is_idle,
 	.wait_for_idle = sdma_v6_0_wait_for_idle,
 	.soft_reset = sdma_v6_0_soft_reset,
-	.check_soft_reset = sdma_v6_0_check_soft_reset,
 	.set_clockgating_state = sdma_v6_0_set_clockgating_state,
 	.set_powergating_state = sdma_v6_0_set_powergating_state,
 	.get_clockgating_state = sdma_v6_0_get_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
index 49c57a38151bc..69cb89298a3e0 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
@@ -784,23 +784,6 @@ static int sdma_v7_0_soft_reset(struct amdgpu_ip_block *ip_block)
 	return sdma_v7_0_start(adev);
 }
 
-static bool sdma_v7_0_check_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	struct amdgpu_ring *ring;
-	int i, r;
-	long tmo = msecs_to_jiffies(1000);
-
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		ring = &adev->sdma.instance[i].ring;
-		r = amdgpu_ring_test_ib(ring, tmo);
-		if (r)
-			return true;
-	}
-
-	return false;
-}
-
 static int sdma_v7_0_reset_queue(struct amdgpu_ring *ring,
 				 unsigned int vmid,
 				 struct amdgpu_fence *timedout_fence)
@@ -942,7 +925,7 @@ static int sdma_v7_0_ring_test_ring(struct amdgpu_ring *ring)
 
 	tmp = 0xCAFEDEAD;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
 		return r;
@@ -954,7 +937,7 @@ static int sdma_v7_0_ring_test_ring(struct amdgpu_ring *ring)
 	r = amdgpu_ring_alloc(ring, 5);
 	if (r) {
 		drm_err(adev_to_drm(adev), "dma failed to lock ring %d (%d).\n", ring->idx, r);
-		amdgpu_device_wb_free(adev, index);
+		amdgpu_wb_free(adev, index);
 		return r;
 	}
 
@@ -979,7 +962,7 @@ static int sdma_v7_0_ring_test_ring(struct amdgpu_ring *ring)
 	if (i >= adev->usec_timeout)
 		r = -ETIMEDOUT;
 
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 
 	return r;
 }
@@ -1006,7 +989,7 @@ static int sdma_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	tmp = 0xCAFEDEAD;
 	memset(&ib, 0, sizeof(ib));
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
 		return r;
@@ -1057,7 +1040,7 @@ static int sdma_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -1679,7 +1662,6 @@ const struct amd_ip_funcs sdma_v7_0_ip_funcs = {
 	.is_idle = sdma_v7_0_is_idle,
 	.wait_for_idle = sdma_v7_0_wait_for_idle,
 	.soft_reset = sdma_v7_0_soft_reset,
-	.check_soft_reset = sdma_v7_0_check_soft_reset,
 	.set_clockgating_state = sdma_v7_0_set_clockgating_state,
 	.set_powergating_state = sdma_v7_0_set_powergating_state,
 	.get_clockgating_state = sdma_v7_0_get_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
index b06001f6b5363..18366e16ef3fa 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
@@ -775,23 +775,6 @@ static int sdma_v7_1_soft_reset(struct amdgpu_ip_block *ip_block)
 	return sdma_v7_1_inst_start(adev, inst_mask);
 }
 
-static bool sdma_v7_1_check_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	struct amdgpu_ring *ring;
-	int i, r;
-	long tmo = msecs_to_jiffies(1000);
-
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		ring = &adev->sdma.instance[i].ring;
-		r = amdgpu_ring_test_ib(ring, tmo);
-		if (r)
-			return true;
-	}
-
-	return false;
-}
-
 static int sdma_v7_1_reset_queue(struct amdgpu_ring *ring,
 				 unsigned int vmid,
 				 struct amdgpu_fence *timedout_fence)
@@ -932,7 +915,7 @@ static int sdma_v7_1_ring_test_ring(struct amdgpu_ring *ring)
 
 	tmp = 0xCAFEDEAD;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
 		return r;
@@ -944,7 +927,7 @@ static int sdma_v7_1_ring_test_ring(struct amdgpu_ring *ring)
 	r = amdgpu_ring_alloc(ring, 5);
 	if (r) {
 		DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
-		amdgpu_device_wb_free(adev, index);
+		amdgpu_wb_free(adev, index);
 		return r;
 	}
 
@@ -969,7 +952,7 @@ static int sdma_v7_1_ring_test_ring(struct amdgpu_ring *ring)
 	if (i >= adev->usec_timeout)
 		r = -ETIMEDOUT;
 
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 
 	return r;
 }
@@ -996,7 +979,7 @@ static int sdma_v7_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	tmp = 0xCAFEDEAD;
 	memset(&ib, 0, sizeof(ib));
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
 		return r;
@@ -1047,7 +1030,7 @@ static int sdma_v7_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -1644,7 +1627,6 @@ const struct amd_ip_funcs sdma_v7_1_ip_funcs = {
 	.is_idle = sdma_v7_1_is_idle,
 	.wait_for_idle = sdma_v7_1_wait_for_idle,
 	.soft_reset = sdma_v7_1_soft_reset,
-	.check_soft_reset = sdma_v7_1_check_soft_reset,
 	.set_clockgating_state = sdma_v7_1_set_clockgating_state,
 	.set_powergating_state = sdma_v7_1_set_powergating_state,
 	.get_clockgating_state = sdma_v7_1_get_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c
index c26cb3e8bff6d..7100f30ffeffa 100644
--- a/drivers/gpu/drm/amd/amdgpu/si.c
+++ b/drivers/gpu/drm/amd/amdgpu/si.c
@@ -35,6 +35,7 @@
 #include "amdgpu_vce.h"
 #include "atom.h"
 #include "amd_pcie.h"
+#include "amdgpu_video_codecs.h"
 
 #include "si_dpm.h"
 #include "sid.h"
@@ -1509,12 +1510,6 @@ static void si_invalidate_hdp(struct amdgpu_device *adev,
 	}
 }
 
-static bool si_need_full_reset(struct amdgpu_device *adev)
-{
-	/* change this when we support soft reset */
-	return true;
-}
-
 static bool si_need_reset_on_init(struct amdgpu_device *adev)
 {
 	return false;
@@ -2019,7 +2014,6 @@ static const struct amdgpu_asic_funcs si_asic_funcs =
 	.get_config_memsize = &si_get_config_memsize,
 	.flush_hdp = &si_flush_hdp,
 	.invalidate_hdp = &si_invalidate_hdp,
-	.need_full_reset = &si_need_full_reset,
 	.get_pcie_usage = &si_get_pcie_usage,
 	.need_reset_on_init = &si_need_reset_on_init,
 	.get_pcie_replay_count = &si_get_pcie_replay_count,
@@ -2224,7 +2218,7 @@ static void si_init_golden_registers(struct amdgpu_device *adev)
 
 
 	default:
-		BUG();
+		break;
 	}
 }
 
@@ -2741,7 +2735,7 @@ int si_set_ip_blocks(struct amdgpu_device *adev)
 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
 		break;
 	default:
-		BUG();
+		break;
 	}
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c
index 549708075eb48..47f1d325bd1af 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dma.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c
@@ -213,7 +213,7 @@ static int si_dma_ring_test_ring(struct amdgpu_ring *ring)
 	u32 tmp;
 	u64 gpu_addr;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -242,7 +242,7 @@ static int si_dma_ring_test_ring(struct amdgpu_ring *ring)
 		r = -ETIMEDOUT;
 
 error_free_wb:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
@@ -265,7 +265,7 @@ static int si_dma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	u64 gpu_addr;
 	long r;
 
-	r = amdgpu_device_wb_get(adev, &index);
+	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
 
@@ -304,7 +304,7 @@ static int si_dma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
-	amdgpu_device_wb_free(adev, index);
+	amdgpu_wb_free(adev, index);
 	return r;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 87b398dd0769b..3d4573d937422 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -77,6 +77,7 @@
 #include "mxgpu_ai.h"
 #include "amdgpu_ras.h"
 #include "amdgpu_xgmi.h"
+#include "amdgpu_video_codecs.h"
 #include <uapi/linux/kfd_ioctl.h>
 
 #define mmMP0_MISC_CGTT_CTRL0                                                                   0x01b9
@@ -721,12 +722,6 @@ void soc15_set_virt_ops(struct amdgpu_device *adev)
 	soc15_reg_base_init(adev);
 }
 
-static bool soc15_need_full_reset(struct amdgpu_device *adev)
-{
-	/* change this when we implement soft reset */
-	return true;
-}
-
 static void soc15_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
 				 uint64_t *count1)
 {
@@ -878,7 +873,6 @@ static const struct amdgpu_asic_funcs soc15_asic_funcs =
 	.set_uvd_clocks = &soc15_set_uvd_clocks,
 	.set_vce_clocks = &soc15_set_vce_clocks,
 	.get_config_memsize = &soc15_get_config_memsize,
-	.need_full_reset = &soc15_need_full_reset,
 	.init_doorbell_index = &vega10_doorbell_index_init,
 	.get_pcie_usage = &soc15_get_pcie_usage,
 	.need_reset_on_init = &soc15_need_reset_on_init,
@@ -899,7 +893,6 @@ static const struct amdgpu_asic_funcs vega20_asic_funcs =
 	.set_uvd_clocks = &soc15_set_uvd_clocks,
 	.set_vce_clocks = &soc15_set_vce_clocks,
 	.get_config_memsize = &soc15_get_config_memsize,
-	.need_full_reset = &soc15_need_full_reset,
 	.init_doorbell_index = &vega20_doorbell_index_init,
 	.get_pcie_usage = &vega20_get_pcie_usage,
 	.need_reset_on_init = &soc15_need_reset_on_init,
@@ -920,7 +913,6 @@ static const struct amdgpu_asic_funcs aqua_vanjaram_asic_funcs =
 	.set_uvd_clocks = &soc15_set_uvd_clocks,
 	.set_vce_clocks = &soc15_set_vce_clocks,
 	.get_config_memsize = &soc15_get_config_memsize,
-	.need_full_reset = &soc15_need_full_reset,
 	.init_doorbell_index = &aqua_vanjaram_doorbell_index_init,
 	.need_reset_on_init = &soc15_need_reset_on_init,
 	.get_pcie_replay_count = &amdgpu_nbio_get_pcie_replay_count,
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15_common.h b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
index a7b5a95ebebbe..47e0329b6f3f4 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15_common.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
@@ -38,30 +38,30 @@
 	(adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + (reg)+(offset))
 
 #define __WREG32_SOC15_RLC__(reg, value, flag, hwip, inst) \
-	((amdgpu_sriov_vf(adev) && adev->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported) ? \
-	 amdgpu_sriov_wreg(adev, reg, value, flag, hwip, inst) : \
-	 WREG32(reg, value))
+	adev->gfx.rlc.reg_funcs->wreg32(adev, reg, value, flag, hwip, inst)
 
 #define __RREG32_SOC15_RLC__(reg, flag, hwip, inst) \
-	((amdgpu_sriov_vf(adev) && adev->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported) ? \
-	 amdgpu_sriov_rreg(adev, reg, flag, hwip, inst) : \
-	 RREG32(reg))
-
-#define WREG32_FIELD15(ip, idx, reg, field, val)	\
-	 __WREG32_SOC15_RLC__(adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg,	\
-				(__RREG32_SOC15_RLC__( \
-					adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg, \
-					0, ip##_HWIP, idx) & \
-				~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field), \
-			      0, ip##_HWIP, idx)
-
-#define WREG32_FIELD15_PREREG(ip, idx, reg_name, field, val)        \
-	__WREG32_SOC15_RLC__(adev->reg_offset[ip##_HWIP][idx][reg##reg_name##_BASE_IDX] + reg##reg_name,   \
-			(__RREG32_SOC15_RLC__( \
-					adev->reg_offset[ip##_HWIP][idx][reg##reg_name##_BASE_IDX] + reg##reg_name, \
-					0, ip##_HWIP, idx) & \
-					~REG_FIELD_MASK(reg_name, field)) | (val) << REG_FIELD_SHIFT(reg_name, field), \
-			0, ip##_HWIP, idx)
+	adev->gfx.rlc.reg_funcs->rreg32(adev, reg, flag, hwip, inst)
+
+#define WREG32_FIELD15(ip, idx, reg_name, field, val) \
+do { \
+	u32 reg__ = adev->reg_offset[ip##_HWIP][idx][mm##reg_name##_BASE_IDX] + mm##reg_name; \
+	u32 val__ = __RREG32_SOC15_RLC__(reg__, 0, ip##_HWIP, idx); \
+\
+	val__ &= ~REG_FIELD_MASK(reg_name, field); \
+	val__ |= (val) << REG_FIELD_SHIFT(reg_name, field); \
+	__WREG32_SOC15_RLC__(reg__, val__, 0, ip##_HWIP, idx); \
+} while (0)
+
+#define WREG32_FIELD15_PREREG(ip, idx, reg_name, field, val) \
+do { \
+	u32 reg__ = adev->reg_offset[ip##_HWIP][idx][reg##reg_name##_BASE_IDX] + reg##reg_name; \
+	u32 val__ = __RREG32_SOC15_RLC__(reg__, 0, ip##_HWIP, idx); \
+\
+	val__ &= ~REG_FIELD_MASK(reg_name, field); \
+	val__ |= (val) << REG_FIELD_SHIFT(reg_name, field); \
+	__WREG32_SOC15_RLC__(reg__, val__, 0, ip##_HWIP, idx); \
+} while (0)
 
 #define RREG32_SOC15(ip, inst, reg) \
 	__RREG32_SOC15_RLC__(adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg, \
@@ -181,12 +181,15 @@
 			WREG32_RLC_EX(prefix, target_reg, value, inst); \
 	} while (0)
 
-#define WREG32_FIELD15_RLC(ip, idx, reg, field, val)   \
-	__WREG32_SOC15_RLC__((adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg), \
-			     (__RREG32_SOC15_RLC__(adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg, \
-						   AMDGPU_REGS_RLC, ip##_HWIP, idx) & \
-			      ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field), \
-			     AMDGPU_REGS_RLC, ip##_HWIP, idx)
+#define WREG32_FIELD15_RLC(ip, idx, reg_name, field, val) \
+do { \
+	u32 reg__ = adev->reg_offset[ip##_HWIP][idx][mm##reg_name##_BASE_IDX] + mm##reg_name; \
+	u32 val__ = __RREG32_SOC15_RLC__(reg__, AMDGPU_REGS_RLC, ip##_HWIP, idx); \
+\
+	val__ &= ~REG_FIELD_MASK(reg_name, field); \
+	val__ |= (val) << REG_FIELD_SHIFT(reg_name, field); \
+	__WREG32_SOC15_RLC__(reg__, val__, AMDGPU_REGS_RLC, ip##_HWIP, idx); \
+} while (0)
 
 #define WREG32_SOC15_OFFSET_RLC(ip, inst, reg, offset, value) \
 	__WREG32_SOC15_RLC__((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) + offset, value, AMDGPU_REGS_RLC, ip##_HWIP, inst)
@@ -207,10 +210,4 @@
 			amdgpu_reg_get_smn_base64(adev, ip##_HWIP, inst),   \
 		value)
 
-#define RREG64_MCA(smn_base, mca_base, idx) \
-	RREG64_PCIE_EXT(smn_base + mca_base + (idx * 8))
-
-#define WREG64_MCA(smn_base, mca_base, idx, val) \
-	WREG64_PCIE_EXT(smn_base + mca_base + (idx * 8), val)
-
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/soc21.c b/drivers/gpu/drm/amd/amdgpu/soc21.c
index e0b80abcd0750..713746a75d1f5 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc21.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc21.c
@@ -33,6 +33,7 @@
 #include "amdgpu_ucode.h"
 #include "amdgpu_psp.h"
 #include "amdgpu_smu.h"
+#include "amdgpu_video_codecs.h"
 #include "atom.h"
 #include "amd_pcie.h"
 
@@ -462,17 +463,6 @@ const struct amdgpu_ip_block_version soc21_common_ip_block = {
 	.funcs = &soc21_common_ip_funcs,
 };
 
-static bool soc21_need_full_reset(struct amdgpu_device *adev)
-{
-	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
-	case IP_VERSION(11, 0, 0):
-	case IP_VERSION(11, 0, 2):
-	case IP_VERSION(11, 0, 3):
-	default:
-		return true;
-	}
-}
-
 static bool soc21_need_reset_on_init(struct amdgpu_device *adev)
 {
 	u32 sol_reg;
@@ -551,7 +541,6 @@ static const struct amdgpu_asic_funcs soc21_asic_funcs = {
 	.set_vce_clocks = &soc21_set_vce_clocks,
 	.get_config_memsize = &soc21_get_config_memsize,
 	.init_doorbell_index = &soc21_init_doorbell_index,
-	.need_full_reset = &soc21_need_full_reset,
 	.need_reset_on_init = &soc21_need_reset_on_init,
 	.get_pcie_replay_count = &amdgpu_nbio_get_pcie_replay_count,
 	.supports_baco = &amdgpu_dpm_is_baco_supported,
diff --git a/drivers/gpu/drm/amd/amdgpu/soc24.c b/drivers/gpu/drm/amd/amdgpu/soc24.c
index 9dce30d2bb8d3..9a658e4837f48 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc24.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc24.c
@@ -34,6 +34,7 @@
 #include "amdgpu_smu.h"
 #include "atom.h"
 #include "amd_pcie.h"
+#include "amdgpu_video_codecs.h"
 
 #include "gc/gc_12_0_0_offset.h"
 #include "gc/gc_12_0_0_sh_mask.h"
@@ -238,16 +239,6 @@ const struct amdgpu_ip_block_version soc24_common_ip_block = {
 	.funcs = &soc24_common_ip_funcs,
 };
 
-static bool soc24_need_full_reset(struct amdgpu_device *adev)
-{
-	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
-	case IP_VERSION(12, 0, 0):
-	case IP_VERSION(12, 0, 1):
-	default:
-		return true;
-	}
-}
-
 static bool soc24_need_reset_on_init(struct amdgpu_device *adev)
 {
 	u32 sol_reg;
@@ -330,7 +321,6 @@ static const struct amdgpu_asic_funcs soc24_asic_funcs = {
 	.get_xclk = &soc24_get_xclk,
 	.get_config_memsize = &soc24_get_config_memsize,
 	.init_doorbell_index = &soc24_init_doorbell_index,
-	.need_full_reset = &soc24_need_full_reset,
 	.need_reset_on_init = &soc24_need_reset_on_init,
 	.get_pcie_replay_count = &soc24_get_pcie_replay_count,
 	.supports_baco = &amdgpu_dpm_is_baco_supported,
diff --git a/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c
index 5f05c8e68297b..43e92e2a85e89 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c
@@ -29,6 +29,7 @@
 #include "gfxhub_v12_1.h"
 #include "sdma_v7_1.h"
 #include "gfx_v12_1.h"
+#include "amdgpu_video_codecs.h"
 
 #include "gc/gc_12_1_0_offset.h"
 #include "gc/gc_12_1_0_sh_mask.h"
@@ -223,15 +224,6 @@ static int soc_v1_0_read_register(struct amdgpu_device *adev,
 	return -EINVAL;
 }
 
-static bool soc_v1_0_need_full_reset(struct amdgpu_device *adev)
-{
-	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
-	case IP_VERSION(12, 1, 0):
-	default:
-		return true;
-	}
-}
-
 static bool soc_v1_0_need_reset_on_init(struct amdgpu_device *adev)
 {
 
@@ -271,7 +263,6 @@ static const struct amdgpu_asic_funcs soc_v1_0_asic_funcs = {
 	.read_register = &soc_v1_0_read_register,
 	.get_config_memsize = &soc_v1_0_get_config_memsize,
 	.get_xclk = &soc_v1_0_get_xclk,
-	.need_full_reset = &soc_v1_0_need_full_reset,
 	.init_doorbell_index = &soc_v1_0_doorbell_index_init,
 	.need_reset_on_init = &soc_v1_0_need_reset_on_init,
 	.encode_ext_smn_addressing = &soc_v1_0_encode_ext_smn_addressing,
@@ -600,8 +591,10 @@ static int soc_v1_0_get_xcp_res_info(struct amdgpu_xcp_mgr *xcp_mgr,
 	xcp_cfg->num_res = ARRAY_SIZE(max_res);
 
 	for (i = 0; i < xcp_cfg->num_res; i++) {
-		res_lt_xcp = max_res[i] < num_xcp;
 		xcp_cfg->xcp_res[i].id = i;
+		if (!max_res[i])
+			continue;
+		res_lt_xcp = max_res[i] < num_xcp;
 		xcp_cfg->xcp_res[i].num_inst =
 			res_lt_xcp ? 1 : max_res[i] / num_xcp;
 		xcp_cfg->xcp_res[i].num_inst =
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
index ee8038df17e3b..a3e883f6f0992 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
@@ -390,43 +390,6 @@ static int tonga_ih_wait_for_idle(struct amdgpu_ip_block *ip_block)
 	return -ETIMEDOUT;
 }
 
-static bool tonga_ih_check_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 srbm_soft_reset = 0;
-	u32 tmp = RREG32(mmSRBM_STATUS);
-
-	if (tmp & SRBM_STATUS__IH_BUSY_MASK)
-		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET,
-						SOFT_RESET_IH, 1);
-
-	if (srbm_soft_reset) {
-		adev->irq.srbm_soft_reset = srbm_soft_reset;
-		return true;
-	} else {
-		adev->irq.srbm_soft_reset = 0;
-		return false;
-	}
-}
-
-static int tonga_ih_pre_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	if (!ip_block->adev->irq.srbm_soft_reset)
-		return 0;
-
-	return tonga_ih_hw_fini(ip_block);
-}
-
-static int tonga_ih_post_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-
-	if (!adev->irq.srbm_soft_reset)
-		return 0;
-
-	return tonga_ih_hw_init(ip_block);
-}
-
 static int tonga_ih_soft_reset(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -481,10 +444,7 @@ static const struct amd_ip_funcs tonga_ih_ip_funcs = {
 	.resume = tonga_ih_resume,
 	.is_idle = tonga_ih_is_idle,
 	.wait_for_idle = tonga_ih_wait_for_idle,
-	.check_soft_reset = tonga_ih_check_soft_reset,
-	.pre_soft_reset = tonga_ih_pre_soft_reset,
 	.soft_reset = tonga_ih_soft_reset,
-	.post_soft_reset = tonga_ih_post_soft_reset,
 	.set_clockgating_state = tonga_ih_set_clockgating_state,
 	.set_powergating_state = tonga_ih_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/umc_v12_0.c
index 14092150336a5..99d19de42525f 100644
--- a/drivers/gpu/drm/amd/amdgpu/umc_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/umc_v12_0.c
@@ -28,72 +28,8 @@
 #include "umc/umc_12_0_0_sh_mask.h"
 #include "mp/mp_13_0_6_sh_mask.h"
 
-#define MAX_ECC_NUM_PER_RETIREMENT  32
-#define DELAYED_TIME_FOR_GPU_RESET  1000  //ms
-
-static inline uint64_t get_umc_v12_0_reg_offset(struct amdgpu_device *adev,
-					    uint32_t node_inst,
-					    uint32_t umc_inst,
-					    uint32_t ch_inst)
-{
-	uint32_t index = umc_inst * adev->umc.channel_inst_num + ch_inst;
-	uint64_t cross_node_offset = (node_inst == 0) ? 0 : UMC_V12_0_CROSS_NODE_OFFSET;
-
-	umc_inst = index / 4;
-	ch_inst = index % 4;
-
-	return adev->umc.channel_offs * ch_inst + UMC_V12_0_INST_DIST * umc_inst +
-		UMC_V12_0_NODE_DIST * node_inst + cross_node_offset;
-}
-
-static int umc_v12_0_reset_error_count_per_channel(struct amdgpu_device *adev,
-					uint32_t node_inst, uint32_t umc_inst,
-					uint32_t ch_inst, void *data)
-{
-	uint64_t odecc_err_cnt_addr;
-	uint64_t umc_reg_offset =
-		get_umc_v12_0_reg_offset(adev, node_inst, umc_inst, ch_inst);
-
-	odecc_err_cnt_addr =
-		SOC15_REG_OFFSET(UMC, 0, regUMCCH0_OdEccErrCnt);
-
-	/* clear error count */
-	WREG32_PCIE_EXT((odecc_err_cnt_addr + umc_reg_offset) * 4,
-			UMC_V12_0_CE_CNT_INIT);
-
-	return 0;
-}
-
-static void umc_v12_0_reset_error_count(struct amdgpu_device *adev)
-{
-	amdgpu_umc_loop_channels(adev,
-		umc_v12_0_reset_error_count_per_channel, NULL);
-}
-
-bool umc_v12_0_is_deferred_error(struct amdgpu_device *adev, uint64_t mc_umc_status)
-{
-	dev_dbg(adev->dev,
-		"MCA_UMC_STATUS(0x%llx): Val:%llu, Poison:%llu, Deferred:%llu, PCC:%llu, UC:%llu, TCC:%llu\n",
-		mc_umc_status,
-		REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val),
-		REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Poison),
-		REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Deferred),
-		REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, PCC),
-		REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UC),
-		REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, TCC)
-	);
-
-	return (amdgpu_ras_is_poison_mode_supported(adev) &&
-		(REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1) &&
-		((REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Deferred) == 1) ||
-		(REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Poison) == 1)));
-}
-
 bool umc_v12_0_is_uncorrectable_error(struct amdgpu_device *adev, uint64_t mc_umc_status)
 {
-	if (umc_v12_0_is_deferred_error(adev, mc_umc_status))
-		return false;
-
 	return ((REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1) &&
 		(REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, PCC) == 1 ||
 		REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UC) == 1 ||
@@ -102,9 +38,6 @@ bool umc_v12_0_is_uncorrectable_error(struct amdgpu_device *adev, uint64_t mc_um
 
 bool umc_v12_0_is_correctable_error(struct amdgpu_device *adev, uint64_t mc_umc_status)
 {
-	if (umc_v12_0_is_deferred_error(adev, mc_umc_status))
-		return false;
-
 	return (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
 		(REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1 ||
 		(REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 &&
@@ -115,676 +48,9 @@ bool umc_v12_0_is_correctable_error(struct amdgpu_device *adev, uint64_t mc_umc_
 		!(umc_v12_0_is_uncorrectable_error(adev, mc_umc_status)))));
 }
 
-static void umc_v12_0_query_error_count_per_type(struct amdgpu_device *adev,
-						   uint64_t umc_reg_offset,
-						   unsigned long *error_count,
-						   check_error_type_func error_type_func)
-{
-	uint64_t mc_umc_status;
-	uint64_t mc_umc_status_addr;
-
-	mc_umc_status_addr =
-		SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0);
-
-	/* Check MCUMC_STATUS */
-	mc_umc_status =
-		RREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4);
-
-	if (error_type_func(adev, mc_umc_status))
-		*error_count += 1;
-}
-
-static int umc_v12_0_query_error_count(struct amdgpu_device *adev,
-					uint32_t node_inst, uint32_t umc_inst,
-					uint32_t ch_inst, void *data)
-{
-	struct ras_err_data *err_data = (struct ras_err_data *)data;
-	unsigned long ue_count = 0, ce_count = 0, de_count = 0;
-
-	/* NOTE: node_inst is converted by adev->umc.active_mask and the range is [0-3],
-	 * which can be used as die ID directly */
-	struct amdgpu_smuio_mcm_config_info mcm_info = {
-		.socket_id = adev->smuio.funcs->get_socket_id(adev),
-		.die_id = node_inst,
-	};
-
-	uint64_t umc_reg_offset =
-		get_umc_v12_0_reg_offset(adev, node_inst, umc_inst, ch_inst);
-
-	umc_v12_0_query_error_count_per_type(adev, umc_reg_offset,
-					    &ce_count, umc_v12_0_is_correctable_error);
-	umc_v12_0_query_error_count_per_type(adev, umc_reg_offset,
-					    &ue_count, umc_v12_0_is_uncorrectable_error);
-	umc_v12_0_query_error_count_per_type(adev, umc_reg_offset,
-					    &de_count, umc_v12_0_is_deferred_error);
-
-	amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, ue_count);
-	amdgpu_ras_error_statistic_ce_count(err_data, &mcm_info, ce_count);
-	amdgpu_ras_error_statistic_de_count(err_data, &mcm_info, de_count);
-
-	return 0;
-}
-
-static void umc_v12_0_query_ras_error_count(struct amdgpu_device *adev,
-					   void *ras_error_status)
-{
-	amdgpu_umc_loop_channels(adev,
-		umc_v12_0_query_error_count, ras_error_status);
-
-	umc_v12_0_reset_error_count(adev);
-}
-
-static void umc_v12_0_get_retire_flip_bits(struct amdgpu_device *adev)
-{
-	enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE;
-	uint32_t vram_type = adev->gmc.vram_type;
-	struct amdgpu_umc_flip_bits *flip_bits = &(adev->umc.flip_bits);
-
-	if (adev->gmc.gmc_funcs->query_mem_partition_mode)
-		nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
-
-	if (adev->gmc.num_umc == 16) {
-		/* default setting */
-		flip_bits->flip_bits_in_pa[0] = UMC_V12_0_PA_C2_BIT;
-		flip_bits->flip_bits_in_pa[1] = UMC_V12_0_PA_C3_BIT;
-		flip_bits->flip_bits_in_pa[2] = UMC_V12_0_PA_C4_BIT;
-		flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R13_BIT;
-		flip_bits->flip_row_bit = 13;
-		flip_bits->bit_num = 4;
-		flip_bits->r13_in_pa = UMC_V12_0_PA_R13_BIT;
-
-		if (nps == AMDGPU_NPS2_PARTITION_MODE) {
-			flip_bits->flip_bits_in_pa[0] = UMC_V12_0_PA_CH5_BIT;
-			flip_bits->flip_bits_in_pa[1] = UMC_V12_0_PA_C2_BIT;
-			flip_bits->flip_bits_in_pa[2] = UMC_V12_0_PA_B1_BIT;
-			flip_bits->r13_in_pa = UMC_V12_0_PA_R12_BIT;
-		} else if (nps == AMDGPU_NPS4_PARTITION_MODE) {
-			flip_bits->flip_bits_in_pa[0] = UMC_V12_0_PA_CH4_BIT;
-			flip_bits->flip_bits_in_pa[1] = UMC_V12_0_PA_CH5_BIT;
-			flip_bits->flip_bits_in_pa[2] = UMC_V12_0_PA_B0_BIT;
-			flip_bits->r13_in_pa = UMC_V12_0_PA_R11_BIT;
-		}
-
-		switch (vram_type) {
-		case AMDGPU_VRAM_TYPE_HBM:
-			/* other nps modes are taken as nps1 */
-			if (nps == AMDGPU_NPS2_PARTITION_MODE)
-				flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R12_BIT;
-			else if (nps == AMDGPU_NPS4_PARTITION_MODE)
-				flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R11_BIT;
-
-			break;
-		case AMDGPU_VRAM_TYPE_HBM3E:
-			flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R12_BIT;
-			flip_bits->flip_row_bit = 12;
-
-			if (nps == AMDGPU_NPS2_PARTITION_MODE)
-				flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R11_BIT;
-			else if (nps == AMDGPU_NPS4_PARTITION_MODE)
-				flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R10_BIT;
-
-			break;
-		default:
-			dev_warn(adev->dev,
-				"Unknown HBM type, set RAS retire flip bits to the value in NPS1 mode.\n");
-			break;
-		}
-	} else if (adev->gmc.num_umc == 8) {
-		/* default setting */
-		flip_bits->flip_bits_in_pa[0] = UMC_V12_0_PA_CH5_BIT;
-		flip_bits->flip_bits_in_pa[1] = UMC_V12_0_PA_C2_BIT;
-		flip_bits->flip_bits_in_pa[2] = UMC_V12_0_PA_B1_BIT;
-		flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R11_BIT;
-		flip_bits->flip_row_bit = 12;
-		flip_bits->bit_num = 4;
-		flip_bits->r13_in_pa = UMC_V12_0_PA_R12_BIT;
-
-		if (nps == AMDGPU_NPS2_PARTITION_MODE) {
-			flip_bits->flip_bits_in_pa[0] = UMC_V12_0_PA_CH4_BIT;
-			flip_bits->flip_bits_in_pa[1] = UMC_V12_0_PA_CH5_BIT;
-			flip_bits->flip_bits_in_pa[2] = UMC_V12_0_PA_B0_BIT;
-			flip_bits->r13_in_pa = UMC_V12_0_PA_R11_BIT;
-		}
-
-		switch (vram_type) {
-		case AMDGPU_VRAM_TYPE_HBM:
-			flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R12_BIT;
-
-			/* other nps modes are taken as nps1 */
-			if (nps == AMDGPU_NPS2_PARTITION_MODE)
-				flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R11_BIT;
-
-			break;
-		case AMDGPU_VRAM_TYPE_HBM3E:
-			flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R11_BIT;
-			flip_bits->flip_row_bit = 12;
-
-			if (nps == AMDGPU_NPS2_PARTITION_MODE)
-				flip_bits->flip_bits_in_pa[3] = UMC_V12_0_PA_R10_BIT;
-
-			break;
-		default:
-			dev_warn(adev->dev,
-				"Unknown HBM type, set RAS retire flip bits to the value in NPS1 mode.\n");
-			break;
-		}
-	} else {
-		dev_warn(adev->dev,
-			"Unsupported UMC number(%d), failed to set RAS flip bits.\n",
-			adev->gmc.num_umc);
-
-		return;
-	}
-
-	adev->umc.retire_unit = 0x1 << flip_bits->bit_num;
-}
-
-static int umc_v12_0_convert_error_address(struct amdgpu_device *adev,
-					struct ras_err_data *err_data,
-					struct ta_ras_query_address_input *addr_in,
-					struct ta_ras_query_address_output *addr_out,
-					bool dump_addr)
-{
-	uint32_t row = 0, row_lower = 0, row_high = 0;
-	uint32_t col = 0, col_lower = 0, bank = 0;
-	uint32_t channel_index = 0, umc_inst = 0;
-	uint32_t i, bit_num, retire_unit, *flip_bits;
-	uint64_t soc_pa, column, err_addr;
-	struct ta_ras_query_address_output addr_out_tmp;
-	struct ta_ras_query_address_output *paddr_out;
-	int ret = 0;
-
-	if (!addr_out)
-		paddr_out = &addr_out_tmp;
-	else
-		paddr_out = addr_out;
-
-	err_addr = bank = 0;
-	if (addr_in) {
-		err_addr = addr_in->ma.err_addr;
-		addr_in->addr_type = TA_RAS_MCA_TO_PA;
-		ret = psp_ras_query_address(&adev->psp, addr_in, paddr_out);
-		if (ret) {
-			dev_warn(adev->dev, "Failed to query RAS physical address for 0x%llx",
-				err_addr);
-
-			goto out;
-		}
-
-		bank = paddr_out->pa.bank;
-		/* no need to care about umc inst if addr_in is NULL */
-		umc_inst = addr_in->ma.umc_inst;
-	}
-
-	flip_bits = adev->umc.flip_bits.flip_bits_in_pa;
-	bit_num = adev->umc.flip_bits.bit_num;
-	retire_unit = adev->umc.retire_unit;
-
-	soc_pa = paddr_out->pa.pa;
-	channel_index = paddr_out->pa.channel_idx;
-	/* clear loop bits in soc physical address */
-	for (i = 0; i < bit_num; i++)
-		soc_pa &= ~BIT_ULL(flip_bits[i]);
-
-	paddr_out->pa.pa = soc_pa;
-	/* get column bit 0 and 1 in mca address */
-	col_lower = (err_addr >> 1) & 0x3ULL;
-	/* extra row bit will be handled later */
-	row_lower = (err_addr >> UMC_V12_0_MA_R0_BIT) & 0x1fffULL;
-	row_lower &= ~BIT_ULL(adev->umc.flip_bits.flip_row_bit);
-
-	if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(9, 5, 0)) {
-		row_high = (soc_pa >> adev->umc.flip_bits.r13_in_pa) & 0x3ULL;
-		/* it's 2.25GB in each channel, from MCA address to PA
-		 * [R14 R13] is converted if the two bits value are 0x3,
-		 * get them from PA instead of MCA address.
-		 */
-		row_lower |= (row_high << 13);
-	}
-
-	if (!err_data && !dump_addr)
-		goto out;
-
-	/* loop for all possibilities of retired bits */
-	for (column = 0; column < retire_unit; column++) {
-		soc_pa = paddr_out->pa.pa;
-		for (i = 0; i < bit_num; i++)
-			soc_pa |= (((column >> i) & 0x1ULL) << flip_bits[i]);
-
-		col = ((column & 0x7) << 2) | col_lower;
-		/* handle extra row bit */
-		if (bit_num == RETIRE_FLIP_BITS_NUM)
-			row = ((column >> 3) << adev->umc.flip_bits.flip_row_bit) |
-					row_lower;
-
-		if (dump_addr)
-			dev_info(adev->dev,
-				"Error Address(PA):0x%-10llx Row:0x%-4x Col:0x%-2x Bank:0x%x Channel:0x%x\n",
-				soc_pa, row, col, bank, channel_index);
-
-		if (err_data)
-			amdgpu_umc_fill_error_record(err_data, err_addr,
-				soc_pa, channel_index, umc_inst);
-	}
-
-out:
-	return ret;
-}
-
-static int umc_v12_0_query_error_address(struct amdgpu_device *adev,
-					uint32_t node_inst, uint32_t umc_inst,
-					uint32_t ch_inst, void *data)
-{
-	struct ras_err_data *err_data = (struct ras_err_data *)data;
-	struct ta_ras_query_address_input addr_in;
-	uint64_t mc_umc_status_addr;
-	uint64_t mc_umc_status, err_addr;
-	uint64_t mc_umc_addrt0;
-	uint64_t umc_reg_offset =
-		get_umc_v12_0_reg_offset(adev, node_inst, umc_inst, ch_inst);
-
-	mc_umc_status_addr =
-		SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0);
-
-	mc_umc_status = RREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4);
-
-	if (mc_umc_status == 0)
-		return 0;
-
-	if (!err_data->err_addr) {
-		/* clear umc status */
-		WREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
-
-		return 0;
-	}
-
-	/* calculate error address if ue error is detected */
-	if (umc_v12_0_is_uncorrectable_error(adev, mc_umc_status) ||
-	    umc_v12_0_is_deferred_error(adev, mc_umc_status)) {
-		mc_umc_addrt0 =
-			SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_ADDRT0);
-
-		err_addr = RREG64_PCIE_EXT((mc_umc_addrt0 + umc_reg_offset) * 4);
-
-		err_addr = REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr);
-
-		if (!adev->aid_mask &&
-		    adev->smuio.funcs &&
-		    adev->smuio.funcs->get_socket_id)
-			addr_in.ma.socket_id = adev->smuio.funcs->get_socket_id(adev);
-		else
-			addr_in.ma.socket_id = 0;
-
-		addr_in.ma.err_addr = err_addr;
-		addr_in.ma.ch_inst = ch_inst;
-		addr_in.ma.umc_inst = umc_inst;
-		addr_in.ma.node_inst = node_inst;
-
-		umc_v12_0_convert_error_address(adev, err_data, &addr_in, NULL, true);
-	}
-
-	/* clear umc status */
-	WREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
-
-	return 0;
-}
-
-static void umc_v12_0_query_ras_error_address(struct amdgpu_device *adev,
-					     void *ras_error_status)
-{
-	amdgpu_umc_loop_channels(adev,
-		umc_v12_0_query_error_address, ras_error_status);
-}
-
-static int umc_v12_0_err_cnt_init_per_channel(struct amdgpu_device *adev,
-					uint32_t node_inst, uint32_t umc_inst,
-					uint32_t ch_inst, void *data)
-{
-	uint32_t odecc_cnt_sel;
-	uint64_t odecc_cnt_sel_addr, odecc_err_cnt_addr;
-	uint64_t umc_reg_offset =
-		get_umc_v12_0_reg_offset(adev, node_inst, umc_inst, ch_inst);
-
-	odecc_cnt_sel_addr =
-		SOC15_REG_OFFSET(UMC, 0, regUMCCH0_OdEccCntSel);
-	odecc_err_cnt_addr =
-		SOC15_REG_OFFSET(UMC, 0, regUMCCH0_OdEccErrCnt);
-
-	odecc_cnt_sel = RREG32_PCIE_EXT((odecc_cnt_sel_addr + umc_reg_offset) * 4);
-
-	/* set ce error interrupt type to APIC based interrupt */
-	odecc_cnt_sel = REG_SET_FIELD(odecc_cnt_sel, UMCCH0_OdEccCntSel,
-					OdEccErrInt, 0x1);
-	WREG32_PCIE_EXT((odecc_cnt_sel_addr + umc_reg_offset) * 4, odecc_cnt_sel);
-
-	/* set error count to initial value */
-	WREG32_PCIE_EXT((odecc_err_cnt_addr + umc_reg_offset) * 4, UMC_V12_0_CE_CNT_INIT);
-
-	return 0;
-}
-
-static bool umc_v12_0_check_ecc_err_status(struct amdgpu_device *adev,
-			enum amdgpu_mca_error_type type, void *ras_error_status)
-{
-	uint64_t mc_umc_status = *(uint64_t *)ras_error_status;
-
-	switch (type) {
-	case AMDGPU_MCA_ERROR_TYPE_UE:
-		return umc_v12_0_is_uncorrectable_error(adev, mc_umc_status);
-	case AMDGPU_MCA_ERROR_TYPE_CE:
-		return umc_v12_0_is_correctable_error(adev, mc_umc_status);
-	case AMDGPU_MCA_ERROR_TYPE_DE:
-		return umc_v12_0_is_deferred_error(adev, mc_umc_status);
-	default:
-		return false;
-	}
-
-	return false;
-}
-
-static void umc_v12_0_err_cnt_init(struct amdgpu_device *adev)
-{
-	amdgpu_umc_loop_channels(adev,
-		umc_v12_0_err_cnt_init_per_channel, NULL);
-}
-
-static bool umc_v12_0_query_ras_poison_mode(struct amdgpu_device *adev)
-{
-	/*
-	 * Force return true, because regUMCCH0_EccCtrl
-	 * is not accessible from host side
-	 */
-	return true;
-}
-
-const struct amdgpu_ras_block_hw_ops umc_v12_0_ras_hw_ops = {
-	.query_ras_error_count = umc_v12_0_query_ras_error_count,
-	.query_ras_error_address = umc_v12_0_query_ras_error_address,
-};
-
-static int umc_v12_0_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
-				     enum aca_smu_type type, void *data)
-{
-	struct amdgpu_device *adev = handle->adev;
-	struct aca_bank_info info;
-	enum aca_error_type err_type;
-	u64 status, count;
-	u32 ext_error_code;
-	int ret;
-
-	status = bank->regs[ACA_REG_IDX_STATUS];
-	if (umc_v12_0_is_deferred_error(adev, status))
-		err_type = ACA_ERROR_TYPE_DEFERRED;
-	else if (umc_v12_0_is_uncorrectable_error(adev, status))
-		err_type = ACA_ERROR_TYPE_UE;
-	else if (umc_v12_0_is_correctable_error(adev, status))
-		err_type = ACA_ERROR_TYPE_CE;
-	else
-		return 0;
-	bank->aca_err_type = err_type;
-
-	ret = aca_bank_info_decode(bank, &info);
-	if (ret)
-		return ret;
-
-	amdgpu_umc_update_ecc_status(adev,
-		bank->regs[ACA_REG_IDX_STATUS],
-		bank->regs[ACA_REG_IDX_IPID],
-		bank->regs[ACA_REG_IDX_ADDR]);
-
-	ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status);
-	if (umc_v12_0_is_deferred_error(adev, status))
-		count = ext_error_code == 0 ?
-			adev->umc.err_addr_cnt / adev->umc.retire_unit : 1ULL;
-	else
-		count = ext_error_code == 0 ?
-			ACA_REG__MISC0__ERRCNT(bank->regs[ACA_REG_IDX_MISC0]) : 1ULL;
-
-	return aca_error_cache_log_bank_error(handle, &info, err_type, count);
-}
-
-static const struct aca_bank_ops umc_v12_0_aca_bank_ops = {
-	.aca_bank_parser = umc_v12_0_aca_bank_parser,
-};
-
-const struct aca_info umc_v12_0_aca_info = {
-	.hwip = ACA_HWIP_TYPE_UMC,
-	.mask = ACA_ERROR_UE_MASK | ACA_ERROR_CE_MASK | ACA_ERROR_DEFERRED_MASK,
-	.bank_ops = &umc_v12_0_aca_bank_ops,
-};
-
-static int umc_v12_0_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
-{
-	int ret;
-
-	ret = amdgpu_umc_ras_late_init(adev, ras_block);
-	if (ret)
-		return ret;
-
-	ret = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__UMC,
-				  &umc_v12_0_aca_info, NULL);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static int umc_v12_0_update_ecc_status(struct amdgpu_device *adev,
-			uint64_t status, uint64_t ipid, uint64_t addr)
-{
-	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-	uint16_t hwid, mcatype;
-	uint64_t page_pfn[UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL];
-	uint64_t err_addr, pa_addr = 0;
-	struct ras_ecc_err *ecc_err;
-	struct ta_ras_query_address_output addr_out;
-	uint32_t shift_bit = adev->umc.flip_bits.flip_bits_in_pa[2];
-	int count, ret, i;
-
-	hwid = REG_GET_FIELD(ipid, MCMP1_IPIDT0, HardwareID);
-	mcatype = REG_GET_FIELD(ipid, MCMP1_IPIDT0, McaType);
-
-	/* The IP block decode of consumption is SMU */
-	if (hwid != MCA_UMC_HWID_V12_0 || mcatype != MCA_UMC_MCATYPE_V12_0) {
-		con->umc_ecc_log.consumption_q_count++;
-		return 0;
-	}
-
-	if (!status)
-		return 0;
-
-	if (!umc_v12_0_is_deferred_error(adev, status))
-		return 0;
-
-	err_addr = REG_GET_FIELD(addr,
-				MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr);
-
-	dev_dbg(adev->dev,
-		"UMC:IPID:0x%llx, socket:%llu, aid:%llu, inst:%llu, ch:%llu, err_addr:0x%llx\n",
-		ipid,
-		MCA_IPID_2_SOCKET_ID(ipid),
-		MCA_IPID_2_DIE_ID(ipid),
-		MCA_IPID_2_UMC_INST(ipid),
-		MCA_IPID_2_UMC_CH(ipid),
-		err_addr);
-
-	ret = amdgpu_umc_mca_to_addr(adev,
-			err_addr, MCA_IPID_2_UMC_CH(ipid),
-			MCA_IPID_2_UMC_INST(ipid), MCA_IPID_2_DIE_ID(ipid),
-			MCA_IPID_2_SOCKET_ID(ipid), &addr_out, true);
-	if (ret)
-		return ret;
-
-	ecc_err = kzalloc_obj(*ecc_err);
-	if (!ecc_err)
-		return -ENOMEM;
-
-	pa_addr = addr_out.pa.pa;
-	ecc_err->status = status;
-	ecc_err->ipid = ipid;
-	ecc_err->addr = addr;
-	ecc_err->pa_pfn = pa_addr >> AMDGPU_GPU_PAGE_SHIFT;
-	ecc_err->channel_idx = addr_out.pa.channel_idx;
-
-	/* If converted pa_pfn is 0, use pa C4 pfn. */
-	if (!ecc_err->pa_pfn)
-		ecc_err->pa_pfn = BIT_ULL(shift_bit) >> AMDGPU_GPU_PAGE_SHIFT;
-
-	ret = amdgpu_umc_logs_ecc_err(adev, &con->umc_ecc_log.de_page_tree, ecc_err);
-	if (ret) {
-		if (ret == -EEXIST)
-			con->umc_ecc_log.de_queried_count++;
-		else
-			dev_err(adev->dev, "Fail to log ecc error! ret:%d\n", ret);
-
-		kfree(ecc_err);
-		return ret;
-	}
-
-	con->umc_ecc_log.de_queried_count++;
-
-	memset(page_pfn, 0, sizeof(page_pfn));
-	count = amdgpu_umc_lookup_bad_pages_in_a_row(adev,
-				pa_addr,
-				page_pfn, ARRAY_SIZE(page_pfn));
-	if (count <= 0) {
-		dev_warn(adev->dev, "Fail to convert error address! count:%d\n", count);
-		return 0;
-	}
-
-	/* Reserve memory */
-	for (i = 0; i < count; i++)
-		amdgpu_ras_reserve_page(adev, page_pfn[i]);
-
-	/* The problem case is as follows:
-	 * 1. GPU A triggers a gpu ras reset, and GPU A drives
-	 *    GPU B to also perform a gpu ras reset.
-	 * 2. After gpu B ras reset started, gpu B queried a DE
-	 *    data. Since the DE data was queried in the ras reset
-	 *    thread instead of the page retirement thread, bad
-	 *    page retirement work would not be triggered. Then
-	 *    even if all gpu resets are completed, the bad pages
-	 *    will be cached in RAM until GPU B's bad page retirement
-	 *    work is triggered again and then saved to eeprom.
-	 * Trigger delayed work to save the bad pages to eeprom in time
-	 * after gpu ras reset is completed.
-	 */
-	if (amdgpu_ras_in_recovery(adev))
-		schedule_delayed_work(&con->page_retirement_dwork,
-			msecs_to_jiffies(DELAYED_TIME_FOR_GPU_RESET));
-
-	return 0;
-}
-
-static int umc_v12_0_fill_error_record(struct amdgpu_device *adev,
-				struct ras_ecc_err *ecc_err, void *ras_error_status)
-{
-	struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
-	uint64_t page_pfn[UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL];
-	int ret, i, count;
-
-	if (!err_data || !ecc_err)
-		return -EINVAL;
-
-	memset(page_pfn, 0, sizeof(page_pfn));
-	count = amdgpu_umc_lookup_bad_pages_in_a_row(adev,
-				ecc_err->pa_pfn << AMDGPU_GPU_PAGE_SHIFT,
-				page_pfn, ARRAY_SIZE(page_pfn));
-
-	for (i = 0; i < count; i++) {
-		ret = amdgpu_umc_fill_error_record(err_data,
-				ecc_err->addr,
-				page_pfn[i] << AMDGPU_GPU_PAGE_SHIFT,
-				ecc_err->channel_idx,
-				MCA_IPID_2_UMC_INST(ecc_err->ipid));
-		if (ret)
-			break;
-	}
-
-	err_data->de_count++;
-
-	return ret;
-}
-
-static void umc_v12_0_query_ras_ecc_err_addr(struct amdgpu_device *adev,
-					void *ras_error_status)
-{
-	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-	struct ras_ecc_err *entries[MAX_ECC_NUM_PER_RETIREMENT];
-	struct radix_tree_root *ecc_tree;
-	int new_detected, ret, i;
-
-	ecc_tree = &con->umc_ecc_log.de_page_tree;
-
-	mutex_lock(&con->umc_ecc_log.lock);
-	new_detected = radix_tree_gang_lookup_tag(ecc_tree, (void **)entries,
-			0, ARRAY_SIZE(entries), UMC_ECC_NEW_DETECTED_TAG);
-	for (i = 0; i < new_detected; i++) {
-		if (!entries[i])
-			continue;
-
-		ret = umc_v12_0_fill_error_record(adev, entries[i], ras_error_status);
-		if (ret) {
-			dev_err(adev->dev, "Fail to fill umc error record, ret:%d\n", ret);
-			break;
-		}
-		radix_tree_tag_clear(ecc_tree,
-				entries[i]->pa_pfn, UMC_ECC_NEW_DETECTED_TAG);
-	}
-	mutex_unlock(&con->umc_ecc_log.lock);
-}
-
-static uint32_t umc_v12_0_get_die_id(struct amdgpu_device *adev,
-		uint64_t mca_addr, uint64_t retired_page)
-{
-	uint32_t die = 0;
-
-	/* we only calculate die id for nps1 mode right now */
-	die += ((((retired_page >> 12) & 0x1ULL)^
-	    ((retired_page >> 20) & 0x1ULL) ^
-	    ((retired_page >> 27) & 0x1ULL) ^
-	    ((retired_page >> 34) & 0x1ULL) ^
-	    ((retired_page >> 41) & 0x1ULL)) << 0);
-
-	/* the original PA_C4 and PA_R13 may be cleared in retired_page, so
-	 * get them from mca_addr.
-	 */
-	die += ((((retired_page >> 13) & 0x1ULL) ^
-	    ((mca_addr >> 5) & 0x1ULL) ^
-	    ((retired_page >> 28) & 0x1ULL) ^
-	    ((mca_addr >> 23) & 0x1ULL) ^
-	    ((retired_page >> 42) & 0x1ULL)) << 1);
-	die &= 3;
-
-	return die;
-}
-
-static void umc_v12_0_mca_ipid_parse(struct amdgpu_device *adev, uint64_t ipid,
-		uint32_t *did, uint32_t *ch, uint32_t *umc_inst, uint32_t *sid)
-{
-	if (did)
-		*did = MCA_IPID_2_DIE_ID(ipid);
-	if (ch)
-		*ch = MCA_IPID_2_UMC_CH(ipid);
-	if (umc_inst)
-		*umc_inst = MCA_IPID_2_UMC_INST(ipid);
-	if (sid)
-		*sid = MCA_IPID_2_SOCKET_ID(ipid);
-}
-
 struct amdgpu_umc_ras umc_v12_0_ras = {
 	.ras_block = {
-		.hw_ops = &umc_v12_0_ras_hw_ops,
-		.ras_late_init = umc_v12_0_ras_late_init,
+		.hw_ops = NULL,
 	},
-	.err_cnt_init = umc_v12_0_err_cnt_init,
-	.query_ras_poison_mode = umc_v12_0_query_ras_poison_mode,
-	.ecc_info_query_ras_error_address = umc_v12_0_query_ras_ecc_err_addr,
-	.check_ecc_err_status = umc_v12_0_check_ecc_err_status,
-	.update_ecc_status = umc_v12_0_update_ecc_status,
-	.convert_ras_err_addr = umc_v12_0_convert_error_address,
-	.get_die_id_from_pa = umc_v12_0_get_die_id,
-	.get_retire_flip_bits = umc_v12_0_get_retire_flip_bits,
-	.mca_ipid_parse = umc_v12_0_mca_ipid_parse,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v12_0.h b/drivers/gpu/drm/amd/amdgpu/umc_v12_0.h
index 63b7e7254526c..906dc7fa10080 100644
--- a/drivers/gpu/drm/amd/amdgpu/umc_v12_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/umc_v12_0.h
@@ -26,31 +26,6 @@
 #include "soc15_common.h"
 #include "amdgpu.h"
 
-#define UMC_V12_0_NODE_DIST		0x40000000
-#define UMC_V12_0_INST_DIST		0x40000
-
-/* UMC register per channel offset */
-#define UMC_V12_0_PER_CHANNEL_OFFSET	0x400
-
-/* UMC cross node offset */
-#define UMC_V12_0_CROSS_NODE_OFFSET		0x100000000
-
-/* OdEccErrCnt max value */
-#define UMC_V12_0_CE_CNT_MAX		0xffff
-/* umc ce interrupt threshold */
-#define UMC_V12_0_CE_INT_THRESHOLD	0xffff
-/* umc ce count initial value */
-#define UMC_V12_0_CE_CNT_INIT	(UMC_V12_0_CE_CNT_MAX - UMC_V12_0_CE_INT_THRESHOLD)
-
-/* number of umc channel instance with memory map register access */
-#define UMC_V12_0_CHANNEL_INSTANCE_NUM		8
-/* number of umc instance with memory map register access */
-#define UMC_V12_0_UMC_INSTANCE_NUM		4
-
-/* Total channel instances for all available umc nodes */
-#define UMC_V12_0_TOTAL_CHANNEL_NUM(adev) \
-	(UMC_V12_0_CHANNEL_INSTANCE_NUM * (adev)->gmc.num_umc)
-
 /* one piece of normalized address is mapped to 8 pieces of physical address */
 #define UMC_V12_0_NA_MAP_PA_NUM        8
 /* R13 bit shift should be considered, double the number */
@@ -75,9 +50,6 @@
 /* row bits in MCA address */
 #define UMC_V12_0_MA_R0_BIT 10
 
-#define MCA_UMC_HWID_V12_0     0x96
-#define MCA_UMC_MCATYPE_V12_0  0x0
-
 #define MCA_IPID_LO_2_UMC_CH(_ipid_lo) (((((_ipid_lo) >> 20) & 0x1) * 4) + \
 			(((_ipid_lo) >> 12) & 0xF))
 #define MCA_IPID_LO_2_UMC_INST(_ipid_lo) (((_ipid_lo) >> 21) & 0x7)
@@ -94,12 +66,9 @@
 	(((REG_GET_FIELD(ipid, MCMP1_IPIDT0, InstanceIdLo) & 0x1) << 2) | \
 	 (REG_GET_FIELD(ipid, MCMP1_IPIDT0, InstanceIdHi) & 0x03))
 
-bool umc_v12_0_is_deferred_error(struct amdgpu_device *adev, uint64_t mc_umc_status);
 bool umc_v12_0_is_uncorrectable_error(struct amdgpu_device *adev, uint64_t mc_umc_status);
 bool umc_v12_0_is_correctable_error(struct amdgpu_device *adev, uint64_t mc_umc_status);
 
-typedef bool (*check_error_type_func)(struct amdgpu_device *adev, uint64_t mc_umc_status);
-
 extern struct amdgpu_umc_ras umc_v12_0_ras;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index ecd7ead7a60b1..bbd0e05d39e51 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -217,8 +217,9 @@ static int uvd_v6_0_enc_get_create_msg(struct amdgpu_ring *ring, uint32_t handle
 	int i, r;
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, ib_size_dw * 4,
-				     AMDGPU_IB_POOL_DIRECT, &job,
-				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
@@ -282,8 +283,9 @@ static int uvd_v6_0_enc_get_destroy_msg(struct amdgpu_ring *ring,
 	int i, r;
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, ib_size_dw * 4,
-				     AMDGPU_IB_POOL_DIRECT, &job,
-				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
@@ -1165,36 +1167,6 @@ static int uvd_v6_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 }
 
 #define AMDGPU_UVD_STATUS_BUSY_MASK    0xfd
-static bool uvd_v6_0_check_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 srbm_soft_reset = 0;
-	u32 tmp = RREG32(mmSRBM_STATUS);
-
-	if (REG_GET_FIELD(tmp, SRBM_STATUS, UVD_RQ_PENDING) ||
-	    REG_GET_FIELD(tmp, SRBM_STATUS, UVD_BUSY) ||
-	    (RREG32(mmUVD_STATUS) & AMDGPU_UVD_STATUS_BUSY_MASK))
-		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_UVD, 1);
-
-	if (srbm_soft_reset) {
-		adev->uvd.inst->srbm_soft_reset = srbm_soft_reset;
-		return true;
-	} else {
-		adev->uvd.inst->srbm_soft_reset = 0;
-		return false;
-	}
-}
-
-static int uvd_v6_0_pre_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-
-	if (!adev->uvd.inst->srbm_soft_reset)
-		return 0;
-
-	uvd_v6_0_stop(adev);
-	return 0;
-}
 
 static int uvd_v6_0_soft_reset(struct amdgpu_ip_block *ip_block)
 {
@@ -1227,18 +1199,6 @@ static int uvd_v6_0_soft_reset(struct amdgpu_ip_block *ip_block)
 	return 0;
 }
 
-static int uvd_v6_0_post_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-
-	if (!adev->uvd.inst->srbm_soft_reset)
-		return 0;
-
-	mdelay(5);
-
-	return uvd_v6_0_start(adev);
-}
-
 static int uvd_v6_0_set_interrupt_state(struct amdgpu_device *adev,
 					struct amdgpu_irq_src *source,
 					unsigned type,
@@ -1538,10 +1498,7 @@ static const struct amd_ip_funcs uvd_v6_0_ip_funcs = {
 	.resume = uvd_v6_0_resume,
 	.is_idle = uvd_v6_0_is_idle,
 	.wait_for_idle = uvd_v6_0_wait_for_idle,
-	.check_soft_reset = uvd_v6_0_check_soft_reset,
-	.pre_soft_reset = uvd_v6_0_pre_soft_reset,
 	.soft_reset = uvd_v6_0_soft_reset,
-	.post_soft_reset = uvd_v6_0_post_soft_reset,
 	.set_clockgating_state = uvd_v6_0_set_clockgating_state,
 	.set_powergating_state = uvd_v6_0_set_powergating_state,
 	.get_clockgating_state = uvd_v6_0_get_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
index df2c83348315d..46222fc30be65 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
@@ -225,8 +225,9 @@ static int uvd_v7_0_enc_get_create_msg(struct amdgpu_ring *ring, u32 handle,
 	int i, r;
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, ib_size_dw * 4,
-				     AMDGPU_IB_POOL_DIRECT, &job,
-				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
@@ -289,8 +290,9 @@ static int uvd_v7_0_enc_get_destroy_msg(struct amdgpu_ring *ring, u32 handle,
 	int i, r;
 
 	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, ib_size_dw * 4,
-				     AMDGPU_IB_POOL_DIRECT, &job,
-				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST);
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_VCN_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
index c69f7d82060f0..9f4e88440c0a7 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
@@ -631,47 +631,6 @@ static int vce_v3_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 #define  AMDGPU_VCE_STATUS_BUSY_MASK (VCE_STATUS_VCPU_REPORT_AUTO_BUSY_MASK | \
 				      VCE_STATUS_VCPU_REPORT_RB0_BUSY_MASK)
 
-static bool vce_v3_0_check_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 srbm_soft_reset = 0;
-
-	/* According to VCE team , we should use VCE_STATUS instead
-	 * SRBM_STATUS.VCE_BUSY bit for busy status checking.
-	 * GRBM_GFX_INDEX.INSTANCE_INDEX is used to specify which VCE
-	 * instance's registers are accessed
-	 * (0 for 1st instance, 10 for 2nd instance).
-	 *
-	 *VCE_STATUS
-	 *|UENC|ACPI|AUTO ACTIVE|RB1 |RB0 |RB2 |          |FW_LOADED|JOB |
-	 *|----+----+-----------+----+----+----+----------+---------+----|
-	 *|bit8|bit7|    bit6   |bit5|bit4|bit3|   bit2   |  bit1   |bit0|
-	 *
-	 * VCE team suggest use bit 3--bit 6 for busy status check
-	 */
-	mutex_lock(&adev->grbm_idx_mutex);
-	WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(0));
-	if (RREG32(mmVCE_STATUS) & AMDGPU_VCE_STATUS_BUSY_MASK) {
-		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE0, 1);
-		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE1, 1);
-	}
-	WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(1));
-	if (RREG32(mmVCE_STATUS) & AMDGPU_VCE_STATUS_BUSY_MASK) {
-		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE0, 1);
-		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE1, 1);
-	}
-	WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(0));
-	mutex_unlock(&adev->grbm_idx_mutex);
-
-	if (srbm_soft_reset) {
-		adev->vce.srbm_soft_reset = srbm_soft_reset;
-		return true;
-	} else {
-		adev->vce.srbm_soft_reset = 0;
-		return false;
-	}
-}
-
 static int vce_v3_0_soft_reset(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -703,31 +662,6 @@ static int vce_v3_0_soft_reset(struct amdgpu_ip_block *ip_block)
 	return 0;
 }
 
-static int vce_v3_0_pre_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-
-	if (!adev->vce.srbm_soft_reset)
-		return 0;
-
-	mdelay(5);
-
-	return vce_v3_0_suspend(ip_block);
-}
-
-
-static int vce_v3_0_post_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-
-	if (!adev->vce.srbm_soft_reset)
-		return 0;
-
-	mdelay(5);
-
-	return vce_v3_0_resume(ip_block);
-}
-
 static int vce_v3_0_set_interrupt_state(struct amdgpu_device *adev,
 					struct amdgpu_irq_src *source,
 					unsigned type,
@@ -909,10 +843,7 @@ static const struct amd_ip_funcs vce_v3_0_ip_funcs = {
 	.resume = vce_v3_0_resume,
 	.is_idle = vce_v3_0_is_idle,
 	.wait_for_idle = vce_v3_0_wait_for_idle,
-	.check_soft_reset = vce_v3_0_check_soft_reset,
-	.pre_soft_reset = vce_v3_0_pre_soft_reset,
 	.soft_reset = vce_v3_0_soft_reset,
-	.post_soft_reset = vce_v3_0_post_soft_reset,
 	.set_clockgating_state = vce_v3_0_set_clockgating_state,
 	.set_powergating_state = vce_v3_0_set_powergating_state,
 	.get_clockgating_state = vce_v3_0_get_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
index 8b8184fe6764b..0d8a3cea63ee2 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
@@ -159,9 +159,8 @@ static void vcn_v2_5_ring_begin_use(struct amdgpu_ring *ring)
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_vcn_inst *v = &adev->vcn.inst[ring->me];
 
-	atomic_inc(&adev->vcn.inst[0].total_submission_cnt);
-
-	cancel_delayed_work_sync(&adev->vcn.inst[0].idle_work);
+	if (!atomic_fetch_inc(&adev->vcn.inst[0].total_submission_cnt))
+		cancel_delayed_work_sync(&adev->vcn.inst[0].idle_work);
 
 	/* We can safely return early here because we've cancelled the
 	 * the delayed work so there is no one else to set it to false
@@ -207,10 +206,9 @@ static void vcn_v2_5_ring_end_use(struct amdgpu_ring *ring)
 	    !adev->vcn.inst[ring->me].using_unified_queue)
 		atomic_dec(&adev->vcn.inst[ring->me].dpg_enc_submission_cnt);
 
-	atomic_dec(&adev->vcn.inst[0].total_submission_cnt);
-
-	schedule_delayed_work(&adev->vcn.inst[0].idle_work,
-			      VCN_IDLE_TIMEOUT);
+	if (atomic_dec_and_test(&adev->vcn.inst[0].total_submission_cnt))
+		schedule_delayed_work(&adev->vcn.inst[0].idle_work,
+				      VCN_IDLE_TIMEOUT);
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
index 894780669f9cf..0cce78b205a85 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
@@ -1995,7 +1995,7 @@ static int vcn_v4_0_ring_reset(struct amdgpu_ring *ring,
 	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
 }
 
-static struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs = {
+static const struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_VCN_ENC,
 	.align_mask = 0x3f,
 	.nop = VCN_ENC_CMD_NO_OP,
@@ -2028,6 +2028,40 @@ static struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs = {
 	.reset = vcn_v4_0_ring_reset,
 };
 
+static const struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs_secure = {
+	.type = AMDGPU_RING_TYPE_VCN_ENC,
+	.align_mask = 0x3f,
+	.nop = VCN_ENC_CMD_NO_OP,
+	.secure_submission_supported = true,
+	.no_user_fence = true,
+	.extra_bytes = sizeof(struct amdgpu_vcn_rb_metadata),
+	.get_rptr = vcn_v4_0_unified_ring_get_rptr,
+	.get_wptr = vcn_v4_0_unified_ring_get_wptr,
+	.set_wptr = vcn_v4_0_unified_ring_set_wptr,
+	.patch_cs_in_place = vcn_v4_0_ring_patch_cs_in_place,
+	.emit_frame_size =
+		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
+		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
+		4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
+		5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
+		1, /* vcn_v2_0_enc_ring_insert_end */
+	.emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
+	.emit_ib = vcn_v2_0_enc_ring_emit_ib,
+	.emit_fence = vcn_v2_0_enc_ring_emit_fence,
+	.emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
+	.test_ring = amdgpu_vcn_enc_ring_test_ring,
+	.test_ib = amdgpu_vcn_unified_ring_test_ib,
+	.insert_nop = amdgpu_ring_insert_nop,
+	.insert_end = vcn_v2_0_enc_ring_insert_end,
+	.pad_ib = amdgpu_ring_generic_pad_ib,
+	.begin_use = amdgpu_vcn_ring_begin_use,
+	.end_use = amdgpu_vcn_ring_end_use,
+	.emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
+	.emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
+	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
+	.reset = vcn_v4_0_ring_reset,
+};
+
 /**
  * vcn_v4_0_set_unified_ring_funcs - set unified ring functions
  *
@@ -2044,10 +2078,11 @@ static void vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device *adev)
 			continue;
 
 		if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 2))
-			vcn_v4_0_unified_ring_vm_funcs.secure_submission_supported = true;
-
-		adev->vcn.inst[i].ring_enc[0].funcs =
-		       (const struct amdgpu_ring_funcs *)&vcn_v4_0_unified_ring_vm_funcs;
+			adev->vcn.inst[i].ring_enc[0].funcs =
+				&vcn_v4_0_unified_ring_vm_funcs_secure;
+		else
+			adev->vcn.inst[i].ring_enc[0].funcs =
+				&vcn_v4_0_unified_ring_vm_funcs;
 		adev->vcn.inst[i].ring_enc[0].me = i;
 	}
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
index 7f001c32e9113..179b892fb410e 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
@@ -115,6 +115,19 @@ static int vcn_v4_0_3_early_init(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	int i, r;
 
+	switch (amdgpu_user_queue) {
+	case -1:
+	case 0:
+	default:
+		adev->vcn.disable_kq = false;
+		adev->vcn.disable_uq = true;
+		break;
+	case 2:
+		adev->vcn.disable_kq = true;
+		adev->vcn.disable_uq = true;
+		break;
+	}
+
 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i)
 		/* re-use enc ring as unified ring */
 		adev->vcn.inst[i].num_enc_rings = 1;
@@ -217,6 +230,10 @@ static int vcn_v4_0_3_sw_init(struct amdgpu_ip_block *ip_block)
 
 		ring = &adev->vcn.inst[i].ring_enc[0];
 		ring->use_doorbell = true;
+		if (adev->vcn.disable_kq) {
+			ring->no_scheduler = true;
+			ring->no_user_submission = true;
+		}
 
 		if (!amdgpu_sriov_vf(adev))
 			ring->doorbell_index =
@@ -2146,71 +2163,6 @@ static const struct amdgpu_ras_block_hw_ops vcn_v4_0_3_ras_hw_ops = {
 	.query_poison_status = vcn_v4_0_3_query_poison_status,
 };
 
-static int vcn_v4_0_3_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
-				      enum aca_smu_type type, void *data)
-{
-	struct aca_bank_info info;
-	u64 misc0;
-	int ret;
-
-	ret = aca_bank_info_decode(bank, &info);
-	if (ret)
-		return ret;
-
-	misc0 = bank->regs[ACA_REG_IDX_MISC0];
-	switch (type) {
-	case ACA_SMU_TYPE_UE:
-		bank->aca_err_type = ACA_ERROR_TYPE_UE;
-		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE,
-						     1ULL);
-		break;
-	case ACA_SMU_TYPE_CE:
-		bank->aca_err_type = ACA_ERROR_TYPE_CE;
-		ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type,
-						     ACA_REG__MISC0__ERRCNT(misc0));
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-/* reference to smu driver if header file */
-static int vcn_v4_0_3_err_codes[] = {
-	14, 15, /* VCN */
-};
-
-static bool vcn_v4_0_3_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
-					 enum aca_smu_type type, void *data)
-{
-	u32 instlo;
-
-	instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
-	instlo &= GENMASK(31, 1);
-
-	if (instlo != mmSMNAID_AID0_MCA_SMU)
-		return false;
-
-	if (aca_bank_check_error_codes(handle->adev, bank,
-				       vcn_v4_0_3_err_codes,
-				       ARRAY_SIZE(vcn_v4_0_3_err_codes)))
-		return false;
-
-	return true;
-}
-
-static const struct aca_bank_ops vcn_v4_0_3_aca_bank_ops = {
-	.aca_bank_parser = vcn_v4_0_3_aca_bank_parser,
-	.aca_bank_is_valid = vcn_v4_0_3_aca_bank_is_valid,
-};
-
-static const struct aca_info vcn_v4_0_3_aca_info = {
-	.hwip = ACA_HWIP_TYPE_SMU,
-	.mask = ACA_ERROR_UE_MASK,
-	.bank_ops = &vcn_v4_0_3_aca_bank_ops,
-};
-
 static int vcn_v4_0_3_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
 {
 	int r;
@@ -2226,11 +2178,6 @@ static int vcn_v4_0_3_ras_late_init(struct amdgpu_device *adev, struct ras_commo
 			goto late_fini;
 	}
 
-	r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__VCN,
-				&vcn_v4_0_3_aca_info, NULL);
-	if (r)
-		goto late_fini;
-
 	return 0;
 
 late_fini:
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
index 1571cc5a148c8..c8879a6e5297d 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
@@ -1479,10 +1479,11 @@ static int vcn_v4_0_5_ring_reset(struct amdgpu_ring *ring,
 	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
 }
 
-static struct amdgpu_ring_funcs vcn_v4_0_5_unified_ring_vm_funcs = {
+static const struct amdgpu_ring_funcs vcn_v4_0_5_unified_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_VCN_ENC,
 	.align_mask = 0x3f,
 	.nop = VCN_ENC_CMD_NO_OP,
+	.secure_submission_supported = true,
 	.no_user_fence = true,
 	.get_rptr = vcn_v4_0_5_unified_ring_get_rptr,
 	.get_wptr = vcn_v4_0_5_unified_ring_get_wptr,
@@ -1525,9 +1526,6 @@ static void vcn_v4_0_5_set_unified_ring_funcs(struct amdgpu_device *adev)
 		if (adev->vcn.harvest_config & (1 << i))
 			continue;
 
-		if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 5))
-			vcn_v4_0_5_unified_ring_vm_funcs.secure_submission_supported = true;
-
 		adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v4_0_5_unified_ring_vm_funcs;
 		adev->vcn.inst[i].ring_enc[0].me = i;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
index 45580e9c4e0cd..886e2c2074a9e 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
@@ -795,6 +795,27 @@ static int vcn_v5_0_0_start_dpg_mode(struct amdgpu_vcn_inst *vinst,
 	return 0;
 }
 
+/**
+ * vcn_v5_0_0_set_mmhub_eco_sec_level - set vcn sec lvl reg
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * request psp to set sec lvl
+ */
+static int vcn_v5_0_0_set_mmhub_eco_sec_level(struct amdgpu_device *adev)
+{
+	int r = 0;
+
+	if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(5, 3, 0)) {
+		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+			/* Request to PSP to program VCN secure lvl */
+			r = psp_set_mmhub_eco_sec_level(adev);
+		}
+	}
+
+	return r;
+}
+
 /**
  * vcn_v5_0_0_start - VCN start
  *
@@ -819,6 +840,11 @@ static int vcn_v5_0_0_start(struct amdgpu_vcn_inst *vinst)
 
 	fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
 
+	/* program VCN secure lvl register */
+	r = vcn_v5_0_0_set_mmhub_eco_sec_level(adev);
+	if (r)
+		return r;
+
 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
 		return vcn_v5_0_0_start_dpg_mode(vinst, adev->vcn.inst[i].indirect_sram);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
index d3db0494341e3..1a07c3bf44253 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
@@ -94,6 +94,19 @@ static int vcn_v5_0_1_early_init(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	int i, r;
 
+	switch (amdgpu_user_queue) {
+	case -1:
+	case 0:
+	default:
+		adev->vcn.disable_kq = false;
+		adev->vcn.disable_uq = true;
+		break;
+	case 2:
+		adev->vcn.disable_kq = true;
+		adev->vcn.disable_uq = true;
+		break;
+	}
+
 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i)
 		/* re-use enc ring as unified ring */
 		adev->vcn.inst[i].num_enc_rings = 1;
@@ -188,6 +201,10 @@ static int vcn_v5_0_1_sw_init(struct amdgpu_ip_block *ip_block)
 
 		ring = &adev->vcn.inst[i].ring_enc[0];
 		ring->use_doorbell = true;
+		if (adev->vcn.disable_kq) {
+			ring->no_scheduler = true;
+			ring->no_user_submission = true;
+		}
 		if (!amdgpu_sriov_vf(adev))
 			ring->doorbell_index =
 				(adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
@@ -1657,10 +1674,7 @@ static const struct amd_ip_funcs vcn_v5_0_1_ip_funcs = {
 	.resume = vcn_v5_0_1_resume,
 	.is_idle = vcn_v5_0_1_is_idle,
 	.wait_for_idle = vcn_v5_0_1_wait_for_idle,
-	.check_soft_reset = NULL,
-	.pre_soft_reset = NULL,
 	.soft_reset = NULL,
-	.post_soft_reset = NULL,
 	.set_clockgating_state = vcn_v5_0_1_set_clockgating_state,
 	.set_powergating_state = vcn_set_powergating_state,
 	.dump_ip_state = amdgpu_vcn_dump_ip_state,
@@ -1713,71 +1727,6 @@ static const struct amdgpu_ras_block_hw_ops vcn_v5_0_1_ras_hw_ops = {
 	.query_poison_status = vcn_v5_0_1_query_poison_status,
 };
 
-static int vcn_v5_0_1_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
-				      enum aca_smu_type type, void *data)
-{
-	struct aca_bank_info info;
-	u64 misc0;
-	int ret;
-
-	ret = aca_bank_info_decode(bank, &info);
-	if (ret)
-		return ret;
-
-	misc0 = bank->regs[ACA_REG_IDX_MISC0];
-	switch (type) {
-	case ACA_SMU_TYPE_UE:
-		bank->aca_err_type = ACA_ERROR_TYPE_UE;
-		ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE,
-						     1ULL);
-		break;
-	case ACA_SMU_TYPE_CE:
-		bank->aca_err_type = ACA_ERROR_TYPE_CE;
-		ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type,
-						     ACA_REG__MISC0__ERRCNT(misc0));
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-/* reference to smu driver if header file */
-static int vcn_v5_0_1_err_codes[] = {
-	14, 15, 47, /* VCN [D|V|S] */
-};
-
-static bool vcn_v5_0_1_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank,
-					 enum aca_smu_type type, void *data)
-{
-	u32 instlo;
-
-	instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]);
-	instlo &= GENMASK(31, 1);
-
-	if (instlo != mmSMNAID_AID0_MCA_SMU)
-		return false;
-
-	if (aca_bank_check_error_codes(handle->adev, bank,
-				       vcn_v5_0_1_err_codes,
-				       ARRAY_SIZE(vcn_v5_0_1_err_codes)))
-		return false;
-
-	return true;
-}
-
-static const struct aca_bank_ops vcn_v5_0_1_aca_bank_ops = {
-	.aca_bank_parser = vcn_v5_0_1_aca_bank_parser,
-	.aca_bank_is_valid = vcn_v5_0_1_aca_bank_is_valid,
-};
-
-static const struct aca_info vcn_v5_0_1_aca_info = {
-	.hwip = ACA_HWIP_TYPE_SMU,
-	.mask = ACA_ERROR_UE_MASK,
-	.bank_ops = &vcn_v5_0_1_aca_bank_ops,
-};
-
 static int vcn_v5_0_1_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
 {
 	int r;
@@ -1786,11 +1735,6 @@ static int vcn_v5_0_1_ras_late_init(struct amdgpu_device *adev, struct ras_commo
 	if (r)
 		return r;
 
-	r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__VCN,
-				&vcn_v5_0_1_aca_info, NULL);
-	if (r)
-		goto late_fini;
-
 	if (amdgpu_ras_is_supported(adev, ras_block->block) &&
 		adev->vcn.inst->ras_poison_irq.funcs) {
 		r = amdgpu_irq_get(adev, &adev->vcn.inst->ras_poison_irq, 0);
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
index bbc172db91a11..b9f6ae75ea727 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
@@ -1203,10 +1203,7 @@ static const struct amd_ip_funcs vcn_v5_0_2_ip_funcs = {
 	.resume = vcn_v5_0_2_resume,
 	.is_idle = vcn_v5_0_2_is_idle,
 	.wait_for_idle = vcn_v5_0_2_wait_for_idle,
-	.check_soft_reset = NULL,
-	.pre_soft_reset = NULL,
 	.soft_reset = NULL,
-	.post_soft_reset = NULL,
 	.set_clockgating_state = vcn_v5_0_2_set_clockgating_state,
 	.set_powergating_state = vcn_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index a256320b92f33..514715793f785 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -34,6 +34,7 @@
 #include "amdgpu_ucode.h"
 #include "atom.h"
 #include "amd_pcie.h"
+#include "amdgpu_video_codecs.h"
 
 #include "gmc/gmc_8_1_d.h"
 #include "gmc/gmc_8_1_sh_mask.h"
@@ -1328,27 +1329,6 @@ static void vi_invalidate_hdp(struct amdgpu_device *adev,
 	}
 }
 
-static bool vi_need_full_reset(struct amdgpu_device *adev)
-{
-	switch (adev->asic_type) {
-	case CHIP_CARRIZO:
-	case CHIP_STONEY:
-		/* CZ has hang issues with full reset at the moment */
-		return false;
-	case CHIP_FIJI:
-	case CHIP_TONGA:
-		/* XXX: soft reset should work on fiji and tonga */
-		return true;
-	case CHIP_POLARIS10:
-	case CHIP_POLARIS11:
-	case CHIP_POLARIS12:
-	case CHIP_TOPAZ:
-	default:
-		/* change this when we support soft reset */
-		return true;
-	}
-}
-
 static void vi_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
 			      uint64_t *count1)
 {
@@ -1437,7 +1417,6 @@ static const struct amdgpu_asic_funcs vi_asic_funcs =
 	.get_config_memsize = &vi_get_config_memsize,
 	.flush_hdp = &vi_flush_hdp,
 	.invalidate_hdp = &vi_invalidate_hdp,
-	.need_full_reset = &vi_need_full_reset,
 	.init_doorbell_index = &legacy_doorbell_index_init,
 	.get_pcie_usage = &vi_get_pcie_usage,
 	.need_reset_on_init = &vi_need_reset_on_init,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index c7edebd2fd8a4..2c1a936459ac8 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -1783,9 +1783,6 @@ static int kfd_ptl_control(struct kfd_process_device *pdd, bool enable)
 	uint32_t ptl_state = enable ? 1 : 0;
 	int ret;
 
-	if (!ptl->hw_supported)
-		return -EOPNOTSUPP;
-
 	if (!pdd->dev->kfd2kgd || !pdd->dev->kfd2kgd->ptl_ctrl)
 		return -EOPNOTSUPP;
 
@@ -1804,6 +1801,9 @@ int kfd_ptl_disable_request(struct kfd_process_device *pdd,
 	struct amdgpu_ptl *ptl = &adev->psp.ptl;
 	int ret = 0;
 
+	if (!ptl->hw_supported)
+		return -EOPNOTSUPP;
+
 	mutex_lock(&ptl->mutex);
 
 	if (pdd->ptl_disable_req)
@@ -1833,6 +1833,9 @@ int kfd_ptl_disable_release(struct kfd_process_device *pdd,
 	struct amdgpu_ptl *ptl = &adev->psp.ptl;
 	int ret = 0;
 
+	if (!ptl->hw_supported)
+		return -EOPNOTSUPP;
+
 	mutex_lock(&ptl->mutex);
 
 	if (!pdd->ptl_disable_req)
@@ -3300,6 +3303,11 @@ static int kfd_ioctl_create_process(struct file *filep, struct kfd_process *p, v
 	}
 
 	filep->private_data = process;
+	ret = kfd_debugfs_add_process(process);
+	if (ret)
+		pr_warn("Failed to create debugfs entry for the kfd_process, ret = %d\n",
+			ret);
+
 	mutex_unlock(&kfd_processes_mutex);
 
 	ret = kfd_create_process_sysfs(process);
@@ -3734,7 +3742,8 @@ static int kfd_mmap(struct file *filep, struct vm_area_struct *vma)
 		return kfd_doorbell_mmap(dev, process, vma);
 
 	case KFD_MMAP_TYPE_EVENTS:
-		return kfd_event_mmap(process, vma);
+		pr_warn("KFD_MMAP_TYPE_EVENTS is no longer supported\n");
+		return -EINVAL;
 
 	case KFD_MMAP_TYPE_RESERVED_MEM:
 		pr_warn("KFD_MMAP_TYPE_RESERVED_MEM is no longer supported\n");
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
index 2a239f45fc249..6e0df685503d0 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
@@ -1412,6 +1412,15 @@ int kfd_parse_crat_table(void *crat_image, struct list_head *device_list,
 			break;
 		}
 
+		/* Validate subtype fits within remaining image */
+		if ((char *)sub_type_hdr + sub_type_hdr->length >
+		    (char *)crat_image + image_len) {
+			pr_warn("CRAT subtype length %u exceeds image bounds\n",
+				sub_type_hdr->length);
+			ret = -EINVAL;
+			break;
+		}
+
 		if (sub_type_hdr->flags & CRAT_SUBTYPE_FLAGS_ENABLED) {
 			ret = kfd_parse_subtype(sub_type_hdr, device_list);
 			if (ret)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c b/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c
index 7d4e07452cdbc..33e905d59dd95 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c
@@ -33,6 +33,7 @@ static struct list_head procs;
 struct debugfs_proc_entry {
 	struct list_head list;
 	struct dentry *proc_dentry;
+	struct kfd_process *process;
 	pid_t pid;
 };
 
@@ -140,34 +141,93 @@ static const struct file_operations kfd_debugfs_pasid_fops = {
 	.read = kfd_debugfs_pasid_read,
 };
 
-void kfd_debugfs_add_process(struct kfd_process *p)
+/* This helper locates the debugfs entry of a kfd process */
+static struct debugfs_proc_entry *kfd_debugfs_find_process_entry(struct kfd_process *p)
 {
+	struct debugfs_proc_entry *entry;
+
+	list_for_each_entry(entry, &procs, list) {
+		if (entry->process == p)
+			return entry;
+	}
+
+	return NULL;
+}
+
+/* This helper creates pasid file of a kfd process under debugfs */
+static void kfd_debugfs_create_pasid_files(struct kfd_process *p,
+					   struct dentry *dir)
+{
+	char name[MAX_DEBUGFS_FILENAME_LEN];
+	struct kfd_process_device *pdd;
 	int i;
+
+	/* create pasid file for each GPU */
+	for (i = 0; i < p->n_pdds; i++) {
+		pdd = p->pdds[i];
+		snprintf(name, MAX_DEBUGFS_FILENAME_LEN, "pasid_%u", pdd->dev->id);
+		debugfs_create_file((const char *)name, S_IFREG | 0444,
+				    dir, pdd, &kfd_debugfs_pasid_fops);
+	}
+}
+
+int kfd_debugfs_add_process(struct kfd_process *p)
+{
+	struct debugfs_proc_entry *primary_entry;
 	char name[MAX_DEBUGFS_FILENAME_LEN];
+	struct kfd_process *primary_process;
 	struct debugfs_proc_entry *entry;
+	int ret;
 
 	entry = kzalloc_obj(*entry);
 	if (!entry)
-		return;
+		return -ENOMEM;
 
-	list_add(&entry->list, &procs);
+	entry->process = p;
 	entry->pid = p->lead_thread->pid;
-	snprintf(name, MAX_DEBUGFS_FILENAME_LEN, "%d",
-		 (int)entry->pid);
-	entry->proc_dentry = debugfs_create_dir(name, debugfs_proc);
 
-	/* Create debugfs files for each GPU:
-	 * - proc/<pid>/pasid_<gpuid>
-	 */
-	for (i = 0; i < p->n_pdds; i++) {
-		struct kfd_process_device *pdd = p->pdds[i];
+	if (p->context_id == KFD_CONTEXT_ID_PRIMARY) {
+		snprintf(name, MAX_DEBUGFS_FILENAME_LEN, "%d",
+			 (int)entry->pid);
+		entry->proc_dentry = debugfs_create_dir(name, debugfs_proc);
+	} else {
+		primary_process = kfd_lookup_process_by_mm(p->lead_thread->mm);
+		if (!primary_process) {
+			ret = -ESRCH;
+			goto err_free_entry;
+		}
 
-		snprintf(name, MAX_DEBUGFS_FILENAME_LEN, "pasid_%u",
-			 pdd->dev->id);
-		debugfs_create_file((const char *)name, S_IFREG | 0444,
-				    entry->proc_dentry, pdd,
-				    &kfd_debugfs_pasid_fops);
+		primary_entry = kfd_debugfs_find_process_entry(primary_process);
+		kfd_unref_process(primary_process);
+		if (!primary_entry) {
+			pr_warn("Failed to find the primary debugfs entry for pid %d\n",
+				entry->pid);
+			ret = -ENOENT;
+			goto err_free_entry;
+		}
+
+		snprintf(name, MAX_DEBUGFS_FILENAME_LEN, "context_%u",
+			 p->context_id);
+		entry->proc_dentry = debugfs_create_dir(name,
+							primary_entry->proc_dentry);
 	}
+
+	list_add(&entry->list, &procs);
+	kfd_debugfs_create_pasid_files(p, entry->proc_dentry);
+
+	return 0;
+
+err_free_entry:
+	kfree(entry);
+	return ret;
+}
+
+/* This helper removes a debugfs entry and its sub-entries */
+static void kfd_debugfs_remove_entry(struct debugfs_proc_entry *entry)
+{
+	debugfs_remove(entry->proc_dentry);
+	list_del(&entry->list);
+	kfree(entry);
 }
 
 void kfd_debugfs_remove_process(struct kfd_process *p)
@@ -175,13 +235,22 @@ void kfd_debugfs_remove_process(struct kfd_process *p)
 	struct debugfs_proc_entry *entry, *next;
 
 	mutex_lock(&kfd_processes_mutex);
+	if (p->context_id == KFD_CONTEXT_ID_PRIMARY) {
+		/* remove entries of secondary contexts */
+		list_for_each_entry_safe(entry, next, &procs, list) {
+			if (entry->pid != p->lead_thread->pid || entry->process == p)
+				continue;
+
+			kfd_debugfs_remove_entry(entry);
+		}
+	}
+
 	list_for_each_entry_safe(entry, next, &procs, list) {
-		if (entry->pid != p->lead_thread->pid)
+		if (entry->process != p)
 			continue;
 
-		debugfs_remove_recursive(entry->proc_dentry);
-		list_del(&entry->list);
-		kfree(entry);
+		kfd_debugfs_remove_entry(entry);
 	}
+
 	mutex_unlock(&kfd_processes_mutex);
 }
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 008a0719fe1f2..586e640f13dce 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -1806,6 +1806,30 @@ void kgd2kfd_teardown_processes(struct amdgpu_device *adev)
 		cond_resched();
 }
 
+int kgd2kfd_reset_mes_queue(struct kfd_dev *kfd, uint32_t node_id,
+			    int queue_type, int pipe, int queue,
+			    unsigned int db)
+{
+	struct kfd_node *node;
+	int ret;
+
+	if (!kfd->init_complete)
+		return 0;
+
+	if (node_id >= kfd->num_nodes) {
+		dev_warn(kfd->adev->dev, "Invalid node ID: %u exceeds %u\n",
+			 node_id, kfd->num_nodes - 1);
+		return -EINVAL;
+	}
+	node = kfd->nodes[node_id];
+
+	ret = kfd_reset_queue_mes(node->dqm, queue_type, pipe, queue, db);
+	if (ret)
+		dev_err(kfd_device, "Error resetting queue\n");
+
+	return ret;
+}
+
 #if defined(CONFIG_DEBUG_FS)
 
 /* This function will send a package to HIQ to hang the HWS
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 678ec611a4f26..51ee9c39104b1 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -37,7 +37,8 @@
 #include "amdgpu_amdkfd.h"
 #include "amdgpu_reset.h"
 #include "amdgpu_sdma.h"
-#include "mes_v11_api_def.h"
+#include "amdgpu_ring.h"
+#include "amdgpu_mes.h"
 #include "kfd_debug.h"
 
 /* Size of the per-pipe EOP queue */
@@ -71,12 +72,11 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm,
 				struct queue *q, const uint32_t *restore_sdma_id);
 
 static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma);
-static int resume_all_queues_mes(struct device_queue_manager *dqm);
-static int suspend_all_queues_mes(struct device_queue_manager *dqm);
 static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm,
 						   u32 doorbell_offset);
 static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q,
 			       struct qcm_process_device *qpd);
+static int reset_queues_mes(struct device_queue_manager *dqm, struct queue *q);
 
 static inline
 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
@@ -184,24 +184,24 @@ static void kfd_hws_hang(struct device_queue_manager *dqm)
 	amdgpu_amdkfd_gpu_reset(dqm->dev->adev);
 }
 
-static int convert_to_mes_queue_type(int queue_type)
+static int convert_to_amdgpu_ring_type(int queue_type)
 {
-	int mes_queue_type;
+	int amdgpu_ring_type;
 
 	switch (queue_type) {
 	case KFD_QUEUE_TYPE_COMPUTE:
-		mes_queue_type = MES_QUEUE_TYPE_COMPUTE;
+		amdgpu_ring_type = AMDGPU_RING_TYPE_COMPUTE;
 		break;
 	case KFD_QUEUE_TYPE_SDMA:
-		mes_queue_type = MES_QUEUE_TYPE_SDMA;
+		amdgpu_ring_type = AMDGPU_RING_TYPE_SDMA;
 		break;
 	default:
 		WARN(1, "Invalid queue type %d", queue_type);
-		mes_queue_type = -EINVAL;
+		amdgpu_ring_type = -EINVAL;
 		break;
 	}
 
-	return mes_queue_type;
+	return amdgpu_ring_type;
 }
 
 static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
@@ -251,7 +251,7 @@ static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
 						(qpd->pqm->process->debug_trap_enabled ||
 						 kfd_dbg_has_ttmps_always_setup(q->device));
 
-	queue_type = convert_to_mes_queue_type(q->properties.type);
+	queue_type = convert_to_amdgpu_ring_type(q->properties.type);
 	if (queue_type < 0) {
 		dev_err(adev->dev, "Queue type not supported with MES, queue:%d\n",
 			q->properties.type);
@@ -300,6 +300,7 @@ static int remove_queue_mes_on_reset_option(struct device_queue_manager *dqm, st
 	memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
 	queue_input.doorbell_offset = q->properties.doorbell_off;
 	queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
+	queue_input.queue_type = convert_to_amdgpu_ring_type(q->properties.type);
 	queue_input.remove_queue_after_reset = flush_mes_queue;
 	queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1;
 
@@ -308,14 +309,14 @@ static int remove_queue_mes_on_reset_option(struct device_queue_manager *dqm, st
 	amdgpu_mes_unlock(&adev->mes);
 	up_read(&adev->reset_domain->sem);
 
-	if (is_for_reset)
+	/* If is_for_reset set, it is a mes internal cleanup */
+	if (!r || is_for_reset)
 		return r;
 
-	if (r) {
-		if (!suspend_all_queues_mes(dqm))
-			return resume_all_queues_mes(dqm);
-
-		dev_err(adev->dev, "failed to remove hardware queue from MES, doorbell=0x%x\n",
+	/* remove_hw_queue failure indicates a queue hang. reset the queue */
+	r = reset_queues_mes(dqm, q);
+	if (r && amdgpu_gpu_recovery) {
+		dev_err(adev->dev, "failed to remove queue from MES, doorbell=0x%x\n",
 			q->properties.doorbell_off);
 		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
 		kfd_hws_hang(dqm);
@@ -407,16 +408,53 @@ static int add_all_kfd_queues_mes(struct device_queue_manager *dqm)
 	return retval;
 }
 
-static int reset_queues_mes(struct device_queue_manager *dqm)
+static int reset_queue_mes(struct device_queue_manager *dqm, struct queue *q,
+			   int queue_type, int pipe, int queue, unsigned int db)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
-	int hqd_info_size = adev->mes.hung_queue_hqd_info_offset;
-	int num_hung = 0, r = 0, i, pipe, queue, queue_type;
-	u32 *hung_array = dqm->hung_db_array;
-	struct amdgpu_mes_hung_queue_hqd_info *hqd_info = dqm->hqd_info;
 	struct kfd_process_device *pdd;
+	bool use_mmio = adev->gfx.mec.use_mmio_for_reset;
+	int r;
+
+	pdd = kfd_get_process_device_data(q->device, q->process);
+	if (!pdd)
+		return -ENODEV;
+
+	if (use_mmio)
+		r = amdgpu_mes_reset_queue_mmio(adev, queue_type, 0, 1, pipe, queue,
+						ffs(dqm->dev->xcc_mask) - 1);
+	else
+		r = amdgpu_mes_reset_user_queue(adev, queue_type, db,
+						ffs(dqm->dev->xcc_mask) - 1);
+	if (r)
+		return r;
+	/* Proceed remove_queue with reset=true */
+	remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, true);
+	set_queue_as_reset(dqm, q, &pdd->qpd);
+	return 0;
+}
+
+int kfd_reset_queue_mes(struct device_queue_manager *dqm, int queue_type,
+			int pipe, int queue, unsigned int db)
+{
 	struct queue *q;
 
+	q = find_queue_by_doorbell_offset(dqm, db);
+	if (!q)
+		return 0;
+	return reset_queue_mes(dqm, q, queue_type, pipe, queue, db);
+}
+
+static int reset_queues_mes(struct device_queue_manager *dqm, struct queue *q)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
+	struct drm_wedge_task_info *info = NULL;
+	struct amdgpu_task_info *ti = NULL;
+	struct kfd_process_device *pdd;
+	unsigned int num_hung = 0;
+	int r = 0;
+	struct mes_remove_queue_input queue_input;
+
 	if (!amdgpu_mes_queue_reset_by_mes_supported(adev)) {
 		r = -ENOTRECOVERABLE;
 		goto fail;
@@ -431,111 +469,43 @@ static int reset_queues_mes(struct device_queue_manager *dqm)
 		goto fail;
 	}
 
-	if (!hung_array || !hqd_info) {
-		r = -ENOMEM;
+	memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
+	queue_input.doorbell_offset = q->properties.doorbell_off;
+	queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
+	queue_input.queue_type = convert_to_amdgpu_ring_type(q->properties.type);
+	queue_input.remove_queue_after_reset = false;
+	queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1;
+	/* pass the known bad queue info to the reset function */
+	r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL, NULL, &num_hung, &queue_input);
+	if (r)
 		goto fail;
+	pdd = kfd_get_process_device_data(q->device, q->process);
+	if (pdd) {
+		ti = amdgpu_vm_get_task_info_pasid(adev, pdd->pasid);
+		if (ti) {
+			amdgpu_vm_print_task_info(adev, ti);
+			info = &ti->task;
+		}
 	}
 
-	memset(hqd_info, 0, hqd_info_size * sizeof(struct amdgpu_mes_hung_queue_hqd_info));
-
-	/*
-	 * AMDGPU_RING_TYPE_COMPUTE parameter does not matter if called
-	 * post suspend_all as reset & detect will return all hung queue types.
-	 *
-	 * Passed parameter is for targeting queues not scheduled by MES add_queue.
+	dqm->detect_hang_count = num_hung;
+	/* When MES doesn't detect any queue hang, no reset happens. Don't signal reset
+	 * event.
 	 */
-	r =  amdgpu_mes_detect_and_reset_hung_queues(adev, AMDGPU_RING_TYPE_COMPUTE,
-		false, &num_hung, hung_array, ffs(dqm->dev->xcc_mask) - 1);
-
-	if (!num_hung || r) {
-		r = -ENOTRECOVERABLE;
-		goto fail;
-	}
-
-	/* MES resets queue/pipe and cleans up internally */
-	for (i = 0; i < num_hung; i++) {
-		hqd_info[i].bit0_31 = hung_array[i + hqd_info_size];
-		pipe = hqd_info[i].pipe_index;
-		queue = hqd_info[i].queue_index;
-		queue_type = hqd_info[i].queue_type;
-
-		if (queue_type != MES_QUEUE_TYPE_COMPUTE &&
-		    queue_type != MES_QUEUE_TYPE_SDMA) {
-			pr_warn("Unsupported hung queue reset type: %d\n", queue_type);
-			hung_array[i] = AMDGPU_MES_INVALID_DB_OFFSET;
-			continue;
-		}
-
-		q = find_queue_by_doorbell_offset(dqm, hung_array[i]);
-		if (!q) {
-			r = -ENOTRECOVERABLE;
-			goto fail;
-		}
-
-		pdd = kfd_get_process_device_data(q->device, q->process);
-		if (!pdd) {
-			r = -ENODEV;
-			goto fail;
+	if (dqm->detect_hang_count) {
+		kfd_signal_reset_event(dqm->dev);
+		if (pdd && pdd->has_reset_queue) {
+			atomic_inc(&adev->gpu_reset_counter);
+			drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, info);
 		}
-
-		pr_warn("Hang detected doorbell %x pipe %d queue %d type %d\n",
-				hung_array[i], pipe, queue, queue_type);
-		/* Proceed remove_queue with reset=true */
-		remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, false);
-		set_queue_as_reset(dqm, q, &pdd->qpd);
 	}
-
-	dqm->detect_hang_count = num_hung;
-	kfd_signal_reset_event(dqm->dev);
+	amdgpu_vm_put_task_info(ti);
 
 fail:
 	dqm->detect_hang_count = 0;
 	return r;
 }
 
-static int suspend_all_queues_mes(struct device_queue_manager *dqm)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
-	int r = 0;
-
-	if (!down_read_trylock(&adev->reset_domain->sem))
-		return -EIO;
-
-	r = amdgpu_mes_suspend(adev, ffs(dqm->dev->xcc_mask) - 1);
-	up_read(&adev->reset_domain->sem);
-
-	if (r) {
-		if (!reset_queues_mes(dqm))
-			return 0;
-
-		dev_err(adev->dev, "failed to suspend gangs from MES\n");
-		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
-		kfd_hws_hang(dqm);
-	}
-
-	return r;
-}
-
-static int resume_all_queues_mes(struct device_queue_manager *dqm)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
-	int r = 0;
-
-	if (!down_read_trylock(&adev->reset_domain->sem))
-		return -EIO;
-
-	r = amdgpu_mes_resume(adev, ffs(dqm->dev->xcc_mask) - 1);
-	up_read(&adev->reset_domain->sem);
-
-	if (r) {
-		dev_err(adev->dev, "failed to resume gangs from MES\n");
-		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
-		kfd_hws_hang(dqm);
-	}
-
-	return r;
-}
-
 static void increment_queue_count(struct device_queue_manager *dqm,
 				  struct qcm_process_device *qpd,
 				  struct queue *q)
@@ -1064,8 +1034,15 @@ static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
 	/* Get the SDMA queue stats */
 	if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
 	    (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
-		retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr,
-							&sdma_val);
+		if (dqm->dev->kfd2kgd->hqd_sdma_get_counter)
+			retval = dqm->dev->kfd2kgd->hqd_sdma_get_counter(
+					dqm->dev->adev, q->mqd,
+					dqm->dev->kfd->device_info.num_sdma_queues_per_engine,
+					&sdma_val);
+		else
+			retval = read_sdma_queue_counter(
+					(uint64_t __user *)q->properties.read_ptr,
+					&sdma_val);
 		if (retval)
 			dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n",
 				q->properties.queue_id);
@@ -2703,8 +2680,16 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
 	/* Get the SDMA queue stats */
 	if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
 	    (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
-		retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr,
-							&sdma_val);
+		if (dqm->dev->kfd2kgd->hqd_sdma_get_counter)
+			retval = dqm->dev->kfd2kgd->hqd_sdma_get_counter(
+					dqm->dev->adev, q->mqd,
+					dqm->dev->kfd->device_info.num_sdma_queues_per_engine,
+					&sdma_val);
+		else
+			retval = read_sdma_queue_counter(
+					(uint64_t __user *)q->properties.read_ptr,
+					&sdma_val);
+
 		if (retval)
 			dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n",
 				q->properties.queue_id);
@@ -3103,6 +3088,7 @@ static void deallocate_hiq_sdma_mqd(struct kfd_node *dev,
 struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
 {
 	struct device_queue_manager *dqm;
+	int i;
 
 	pr_debug("Loading device queue manager\n");
 
@@ -3231,6 +3217,9 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
 		deallocate_hiq_sdma_mqd(dev, &dqm->hiq_sdma_mqd);
 
 out_free:
+	for (i = 0; i < KFD_MQD_TYPE_MAX; i++)
+		kfree(dqm->mqd_mgrs[i]);
+
 	kfree(dqm);
 	return NULL;
 }
@@ -3244,12 +3233,12 @@ void device_queue_manager_uninit(struct device_queue_manager *dqm)
 	kfree(dqm);
 }
 
+/* bad queue notified by interrupt from CP */
 int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbell_id)
 {
 	struct kfd_process_device *pdd = NULL;
 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, &pdd);
 	struct device_queue_manager *dqm = knode->dqm;
-	struct device *dev = dqm->dev->adev->dev;
 	struct qcm_process_device *qpd;
 	struct queue *q = NULL;
 	int ret = 0;
@@ -3264,19 +3253,10 @@ int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbel
 
 		list_for_each_entry(q, &qpd->queues_list, list) {
 			if (q->doorbell_id == doorbell_id && q->properties.is_active) {
-				/* suspend all queues will save any good queues and mark the rest as bad */
-				suspend_all_queues_mes(dqm);
-
+				reset_queues_mes(dqm, q);
 				q->properties.is_evicted = true;
 				q->properties.is_active = false;
 				decrement_queue_count(dqm, qpd, q);
-
-				/* this will remove the bad queue and sched a GPU reset if needed */
-				ret = remove_queue_mes(dqm, q, qpd);
-				if (ret)
-					dev_err(dev, "Removing bad queue failed");
-				/* resume the good queues */
-				resume_all_queues_mes(dqm);
 				break;
 			}
 		}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index 641b8ada82a03..c9f9f7a871117 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -333,6 +333,8 @@ int debug_refresh_runlist(struct device_queue_manager *dqm);
 bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm,
 				 struct qcm_process_device *qpd,
 				 int doorbell_off, u32 *queue_format);
+int kfd_reset_queue_mes(struct device_queue_manager *dqm, int queue_type,
+			int pipe, int queue, unsigned int db);
 size_t mqd_size_from_queue_type(struct device_queue_manager *dqm,
 				enum kfd_queue_type type);
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
index 2e97da597b3da..f5fd78f0df7d2 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
@@ -29,10 +29,12 @@
 #include <linux/uaccess.h>
 #include <linux/mman.h>
 #include <linux/memory.h>
+#include <linux/workqueue.h>
 #include "kfd_priv.h"
 #include "kfd_events.h"
 #include "kfd_device_queue_manager.h"
 #include <linux/device.h>
+#include <drm/amdgpu_drm.h>
 
 /*
  * Wrapper around wait_queue_entry_t
@@ -44,67 +46,19 @@ struct kfd_event_waiter {
 	bool event_age_enabled;  /* set to true when last_event_age is non-zero */
 };
 
-/*
- * Each signal event needs a 64-bit signal slot where the signaler will write
- * a 1 before sending an interrupt. (This is needed because some interrupts
- * do not contain enough spare data bits to identify an event.)
- * We get whole pages and map them to the process VA.
- * Individual signal events use their event_id as slot index.
- */
-struct kfd_signal_page {
-	uint64_t *kernel_address;
-	uint64_t __user *user_address;
-	bool need_to_free_pages;
-};
-
-static uint64_t *page_slots(struct kfd_signal_page *page)
-{
-	return page->kernel_address;
-}
-
-static struct kfd_signal_page *allocate_signal_page(struct kfd_process *p)
-{
-	void *backing_store;
-	struct kfd_signal_page *page;
-
-	page = kzalloc_obj(*page);
-	if (!page)
-		return NULL;
-
-	backing_store = (void *) __get_free_pages(GFP_KERNEL,
-					get_order(KFD_SIGNAL_EVENT_LIMIT * 8));
-	if (!backing_store)
-		goto fail_alloc_signal_store;
-
-	/* Initialize all events to unsignaled */
-	memset(backing_store, (uint8_t) UNSIGNALED_EVENT_SLOT,
-	       KFD_SIGNAL_EVENT_LIMIT * 8);
-
-	page->kernel_address = backing_store;
-	page->need_to_free_pages = true;
-	pr_debug("Allocated new event signal page at %p, for process %p\n",
-			page, p);
-
-	return page;
-
-fail_alloc_signal_store:
-	kfree(page);
-	return NULL;
-}
-
 static int allocate_event_notification_slot(struct kfd_process *p,
 					    struct kfd_event *ev,
 					    const int *restore_id)
 {
 	int id;
 
-	if (!p->signal_page) {
-		p->signal_page = allocate_signal_page(p);
-		if (!p->signal_page)
-			return -ENOMEM;
-		/* Oldest user mode expects 256 event slots */
-		p->signal_mapped_size = 256*8;
-	}
+	/*
+	 * The signal page is allocated in user mode and mapped to the kernel
+	 * via the event_page_offset of the create event IOCTL. Without it no
+	 * signal events can be created.
+	 */
+	if (!p->signal_page)
+		return -ENOMEM;
 
 	if (restore_id) {
 		if (*restore_id >= KFD_SIGNAL_EVENT_LIMIT)
@@ -126,7 +80,7 @@ static int allocate_event_notification_slot(struct kfd_process *p,
 		return id;
 
 	ev->event_id = id;
-	page_slots(p->signal_page)[id] = UNSIGNALED_EVENT_SLOT;
+	p->signal_page[id] = UNSIGNALED_EVENT_SLOT;
 
 	return 0;
 }
@@ -172,7 +126,7 @@ static struct kfd_event *lookup_signaled_event_by_partial_id(
 	 */
 	if (bits > 31 || (1U << bits) >= KFD_SIGNAL_EVENT_LIMIT) {
 		if (signal_mailbox_updated &&
-		    page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT)
+		    p->signal_page[id] == UNSIGNALED_EVENT_SLOT)
 			return NULL;
 
 		return idr_find(&p->event_idr, id);
@@ -182,7 +136,7 @@ static struct kfd_event *lookup_signaled_event_by_partial_id(
 	 * and find the first one that has signaled.
 	 */
 	for (ev = NULL; id < KFD_SIGNAL_EVENT_LIMIT && !ev; id += 1U << bits) {
-		if (page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT)
+		if (p->signal_page[id] == UNSIGNALED_EVENT_SLOT)
 			continue;
 
 		ev = idr_find(&p->event_idr, id);
@@ -213,10 +167,8 @@ static int create_signal_event(struct file *devkfd, struct kfd_process *p,
 
 	p->signal_event_count++;
 
-	ev->user_signal_address = &p->signal_page->user_address[ev->event_id];
-	pr_debug("Signal event number %zu created with id %d, address %p\n",
-			p->signal_event_count, ev->event_id,
-			ev->user_signal_address);
+	pr_debug("Signal event number %zu created with id %d\n",
+			p->signal_event_count, ev->event_id);
 
 	return 0;
 }
@@ -296,26 +248,9 @@ static void destroy_events(struct kfd_process *p)
 	mutex_destroy(&p->event_mutex);
 }
 
-/*
- * We assume that the process is being destroyed and there is no need to
- * unmap the pages or keep bookkeeping data in order.
- */
-static void shutdown_signal_page(struct kfd_process *p)
-{
-	struct kfd_signal_page *page = p->signal_page;
-
-	if (page) {
-		if (page->need_to_free_pages)
-			free_pages((unsigned long)page->kernel_address,
-				   get_order(KFD_SIGNAL_EVENT_LIMIT * 8));
-		kfree(page);
-	}
-}
-
 void kfd_event_free_process(struct kfd_process *p)
 {
 	destroy_events(p);
-	shutdown_signal_page(p);
 }
 
 static bool event_can_be_gpu_signaled(const struct kfd_event *ev)
@@ -332,8 +267,6 @@ static bool event_can_be_cpu_signaled(const struct kfd_event *ev)
 static int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
 		       uint64_t size, uint64_t user_handle)
 {
-	struct kfd_signal_page *page;
-
 	if (p->signal_page)
 		return -EBUSY;
 
@@ -343,17 +276,11 @@ static int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
 		return -EINVAL;
 	}
 
-	page = kzalloc_obj(*page);
-	if (!page)
-		return -ENOMEM;
-
 	/* Initialize all events to unsignaled */
 	memset(kernel_address, (uint8_t) UNSIGNALED_EVENT_SLOT,
 	       KFD_SIGNAL_EVENT_LIMIT * 8);
 
-	page->kernel_address = kernel_address;
-
-	p->signal_page = page;
+	p->signal_page = kernel_address;
 	p->signal_mapped_size = size;
 	p->signal_handle = user_handle;
 	return 0;
@@ -390,7 +317,8 @@ int kfd_kmap_event_page(struct kfd_process *p, uint64_t event_page_offset)
 		return -EINVAL;
 	}
 
-	err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(mem, &kern_addr, &size);
+	err = amdgpu_amdkfd_gpuvm_map_bo_to_kernel(mem, &kern_addr, &size,
+						   AMDGPU_GEM_DOMAIN_GTT);
 	if (err) {
 		pr_err("Failed to map event page to kernel\n");
 		return err;
@@ -399,7 +327,7 @@ int kfd_kmap_event_page(struct kfd_process *p, uint64_t event_page_offset)
 	err = kfd_event_page_set(p, kern_addr, size, event_page_offset);
 	if (err) {
 		pr_err("Failed to set event page\n");
-		amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
+		amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem);
 		return err;
 	}
 	return err;
@@ -720,7 +648,7 @@ int kfd_reset_event(struct kfd_process *p, uint32_t event_id)
 
 static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev)
 {
-	WRITE_ONCE(page_slots(p->signal_page)[ev->event_id], UNSIGNALED_EVENT_SLOT);
+	WRITE_ONCE(p->signal_page[ev->event_id], UNSIGNALED_EVENT_SLOT);
 }
 
 static void set_event_from_interrupt(struct kfd_process *p,
@@ -763,7 +691,7 @@ void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
 		 * in the interrupt payload was invalid and do an
 		 * exhaustive search of signaled events.
 		 */
-		uint64_t *slots = page_slots(p->signal_page);
+		uint64_t *slots = p->signal_page;
 		uint32_t id;
 
 		if (valid_id_bits)
@@ -1071,51 +999,6 @@ int kfd_wait_on_events(struct kfd_process *p,
 	return ret;
 }
 
-int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)
-{
-	unsigned long pfn;
-	struct kfd_signal_page *page;
-	int ret;
-
-	/* check required size doesn't exceed the allocated size */
-	if (get_order(KFD_SIGNAL_EVENT_LIMIT * 8) <
-			get_order(vma->vm_end - vma->vm_start)) {
-		pr_err("Event page mmap requested illegal size\n");
-		return -EINVAL;
-	}
-
-	page = p->signal_page;
-	if (!page) {
-		/* Probably KFD bug, but mmap is user-accessible. */
-		pr_debug("Signal page could not be found\n");
-		return -EINVAL;
-	}
-
-	pfn = __pa(page->kernel_address);
-	pfn >>= PAGE_SHIFT;
-
-	vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE
-		       | VM_DONTDUMP | VM_PFNMAP);
-
-	pr_debug("Mapping signal page\n");
-	pr_debug("     start user address  == 0x%08lx\n", vma->vm_start);
-	pr_debug("     end user address    == 0x%08lx\n", vma->vm_end);
-	pr_debug("     pfn                 == 0x%016lX\n", pfn);
-	pr_debug("     vm_flags            == 0x%08lX\n", vma->vm_flags);
-	pr_debug("     size                == 0x%08lX\n",
-			vma->vm_end - vma->vm_start);
-
-	page->user_address = (uint64_t __user *)vma->vm_start;
-
-	/* mapping the page to user process */
-	ret = remap_pfn_range(vma, vma->vm_start, pfn,
-			vma->vm_end - vma->vm_start, vma->vm_page_prot);
-	if (!ret)
-		p->signal_mapped_size = vma->vm_end - vma->vm_start;
-
-	return ret;
-}
-
 /*
  * Assumes that p is not going away.
  */
@@ -1341,6 +1224,71 @@ void kfd_signal_reset_event(struct kfd_node *dev)
 	srcu_read_unlock(&kfd_processes_srcu, idx);
 }
 
+/*
+ * Per-process opt-in for poison-consumption SIGBUS handling.
+ *
+ * Default: kernel sends SIGBUS to the process immediately when poison is
+ * consumed, in addition to delivering the KFD HW/MEMORY exception events.
+ *
+ * Userspace (ROCr) can opt-in per-process via the
+ * DRM_IOCTL_AMDGPU_PROC_OPTIONS / AMDGPU_PROC_OPTIONS_OP_KFD_SIGBUS_DELAY
+ * option. This lets the app's registered system-event callback handle the
+ * RAS error first, instead of being killed by SIGBUS.
+ *
+ * Encoded value (stored on the kfd_process):
+ *   0          - default: SIGBUS immediately (no opt-in)
+ *   0xFFFFFFFF - opt-in, never escalate to SIGBUS
+ *   N (other)  - opt-in, escalate to SIGBUS after N ms if app does not
+ *                handle the error in time (safety timeout)
+ */
+
+void kfd_signal_sigbus_delayed_fn(struct work_struct *work)
+{
+	struct kfd_process *p = container_of(to_delayed_work(work),
+				struct kfd_process, signal_work);
+
+	if (p->lead_thread)
+		send_sig(SIGBUS, p->lead_thread, 0);
+
+	kfd_unref_process(p);
+}
+
+static void kfd_signal_sigbus_with_delay(struct kfd_node *dev,
+					 struct kfd_process *p)
+{
+	u32 delay_ms = atomic_read(&p->kfd_sigbus_delay_ms);
+
+	if (delay_ms == AMDGPU_PROC_OPTIONS_KFD_SIGBUS_DELAY_DISABLED) {
+		dev_info(dev->adev->dev,
+			 "SIGBUS suppressed for process %s(pid:%d): app opted in to handle RAS error\n",
+			 p->lead_thread->comm, p->lead_thread->pid);
+		return;
+	}
+
+	if (delay_ms == 0)
+		goto send_now;
+
+	/*
+	 * Take an extra reference for the delayed worker. If the work is
+	 * already pending (e.g. another device of this process consumed poison
+	 * just before), drop the reference and skip rescheduling - the process
+	 * only needs to be notified once.
+	 */
+	kref_get(&p->ref);
+	if (!schedule_delayed_work(&p->signal_work, msecs_to_jiffies(delay_ms))) {
+		kfd_unref_process(p);
+		return;
+	}
+
+	dev_info(dev->adev->dev,
+		 "Deferring SIGBUS to process %s(pid:%d) by %u ms (RAS error opt-in safety timeout)\n",
+		 p->lead_thread->comm, p->lead_thread->pid, delay_ms);
+	return;
+
+send_now:
+	send_sig(SIGBUS, p->lead_thread, 0);
+}
+
 void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid)
 {
 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, NULL);
@@ -1395,7 +1343,7 @@ void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid)
 	rcu_read_unlock();
 
 	/* user application will handle SIGBUS signal */
-	send_sig(SIGBUS, p->lead_thread, 0);
+	kfd_signal_sigbus_with_delay(dev, p);
 
 	kfd_unref_process(p);
 }
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.h b/drivers/gpu/drm/amd/amdkfd/kfd_events.h
index 1dc21c13833b7..827a2c7d7721b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.h
@@ -49,7 +49,6 @@
 #define UNSIGNALED_EVENT_SLOT ((uint64_t)-1)
 
 struct kfd_event_waiter;
-struct signal_page;
 
 struct kfd_event {
 	u32 event_id;
@@ -63,9 +62,6 @@ struct kfd_event {
 	spinlock_t lock;
 	wait_queue_head_t wq; /* List of event waiters. */
 
-	/* Only for signal events. */
-	uint64_t __user *user_signal_address;
-
 	/* type specific data */
 	union {
 		struct kfd_hsa_memory_exception_data memory_exception_data;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 7cd236c1ff75f..ed3649a813324 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -68,8 +68,8 @@ svm_migrate_gart_map(struct amdgpu_ring *ring,
 				     AMDGPU_FENCE_OWNER_UNDEFINED,
 				     num_dw * 4 + num_bytes,
 				     AMDGPU_IB_POOL_DELAYED,
-				     &job,
-				     AMDGPU_KERNEL_JOB_ID_KFD_GART_MAP);
+				     AMDGPU_KERNEL_JOB_ID_KFD_GART_MAP,
+				     &job);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
index 723b725d20b86..9b7859a77950e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
@@ -315,3 +315,19 @@ bool kfd_check_hiq_mqd_doorbell_id(struct kfd_node *node, uint32_t doorbell_id,
 
 	return false;
 }
+
+bool mqd_on_vram(struct amdgpu_device *adev)
+{
+	if (adev->apu_prefer_gtt)
+		return false;
+
+	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
+	case IP_VERSION(9, 4, 2):
+	case IP_VERSION(9, 4, 3):
+	case IP_VERSION(9, 4, 4):
+	case IP_VERSION(9, 5, 0):
+		return true;
+	default:
+		return false;
+	}
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
index 63ea70e5c0e65..59eff3389d394 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
@@ -202,4 +202,6 @@ uint64_t kfd_mqd_stride(struct mqd_manager *mm,
 			struct queue_properties *q);
 bool kfd_check_hiq_mqd_doorbell_id(struct kfd_node *node, uint32_t doorbell_id,
 				   uint32_t inst);
+bool mqd_on_vram(struct amdgpu_device *adev);
+
 #endif /* KFD_MQD_MANAGER_H_ */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index be99f0d53b188..75e5a9f67d508 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -116,20 +116,6 @@ static void set_priority(struct v9_mqd *m, struct queue_properties *q)
 	m->cp_hqd_pipe_priority = pipe_priority_map[q->priority];
 }
 
-static bool mqd_on_vram(struct amdgpu_device *adev)
-{
-	if (adev->apu_prefer_gtt)
-		return false;
-
-	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
-	case IP_VERSION(9, 4, 3):
-	case IP_VERSION(9, 5, 0):
-		return true;
-	default:
-		return false;
-	}
-}
-
 static struct kfd_mem_obj *allocate_mqd(struct mqd_manager *mm,
 		struct queue_properties *q)
 {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
index b1a6eb349bb30..85bf1790a6c82 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
@@ -269,9 +269,19 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 	}
 	pm->is_over_subscription = !!is_over_subscription;
 
-	for (i = 0; i < alloc_size_bytes / sizeof(uint32_t); i++)
-		pr_debug("0x%2X ", rl_buffer[i]);
-	pr_debug("\n");
+	pr_debug("Runlist dump:");
+	for (i = 0; i < alloc_size_bytes / sizeof(uint32_t); i += 8) {
+		char buf[128];
+		int j, len = 0;
+
+		/* Dump 8 entries per line with an index for each line */
+		len += scnprintf(buf + len, sizeof(buf) - len, "%4u:", i);
+
+		for (j = 0; j < 8 && (i + j) < alloc_size_bytes / sizeof(uint32_t); j++)
+			len += scnprintf(buf + len, sizeof(buf) - len, " 0x%08x", rl_buffer[i + j]);
+
+		pr_debug("%s\n", buf);
+	}
 
 	return retval;
 }
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 4d65f94da4d81..88191a4c16579 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -32,6 +32,7 @@
 #include <linux/atomic.h>
 #include <linux/workqueue.h>
 #include <linux/spinlock.h>
+#include <linux/iosys-map.h>
 #include <uapi/linux/kfd_ioctl.h>
 #include <linux/idr.h>
 #include <linux/kfifo.h>
@@ -711,7 +712,7 @@ struct qcm_process_device {
 
 	/* CWSR memory */
 	struct kgd_mem *cwsr_mem;
-	void *cwsr_kaddr;
+	struct iosys_map cwsr_map;
 	uint64_t cwsr_base;
 	uint64_t tba_addr;
 	uint64_t tma_addr;
@@ -953,11 +954,32 @@ struct kfd_process {
 	struct idr event_idr;
 	/* Event page */
 	u64 signal_handle;
-	struct kfd_signal_page *signal_page;
+	/*
+	 * Each signal event needs a 64-bit signal slot where the signaler will
+	 * write a 1 before sending an interrupt. (This is needed because some
+	 * interrupts do not contain enough spare data bits to identify an
+	 * event.) The signal page is allocated in user mode and mapped to the
+	 * kernel; individual signal events use their event_id as slot index.
+	 */
+	uint64_t *signal_page;
 	size_t signal_mapped_size;
 	size_t signal_event_count;
 	bool signal_event_limit_reached;
 
+	/**
+	 * @kfd_sigbus_delay_ms: Per-process KFD SIGBUS delivery option for
+	 * poison/RAS events (set via DRM_IOCTL_AMDGPU_PROC_OPTIONS /
+	 * AMDGPU_PROC_OPTIONS_OP_KFD_SIGBUS_DELAY).
+	 *
+	 *   0          - send SIGBUS immediately (default)
+	 *   0xFFFFFFFF - suppress SIGBUS delivery
+	 *   other      - delay SIGBUS delivery by this many milliseconds
+	 */
+	atomic_t kfd_sigbus_delay_ms;
+
+	/* Delayed signal delivery to user */
+	struct delayed_work signal_work;
+
 	/* Information used for memory eviction */
 	void *kgd_process_info;
 	/* Eviction fence that is attached to all the BOs of this process. The
@@ -1526,7 +1548,6 @@ extern const struct kfd_device_global_init_class device_global_init_class_cik;
 
 int kfd_event_init_process(struct kfd_process *p);
 void kfd_event_free_process(struct kfd_process *p);
-int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma);
 int kfd_wait_on_events(struct kfd_process *p,
 		       uint32_t num_events, void __user *data,
 		       bool all, uint32_t *user_timeout_ms,
@@ -1555,6 +1576,7 @@ void kfd_signal_vm_fault_event(struct kfd_process_device *pdd,
 void kfd_signal_reset_event(struct kfd_node *dev);
 
 void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid);
+void kfd_signal_sigbus_delayed_fn(struct work_struct *work);
 void kfd_signal_process_terminate_event(struct kfd_process *p);
 
 static inline void kfd_flush_tlb(struct kfd_process_device *pdd)
@@ -1629,14 +1651,14 @@ int kfd_debugfs_hang_hws(struct kfd_node *dev);
 int pm_debugfs_hang_hws(struct packet_manager *pm);
 int dqm_debugfs_hang_hws(struct device_queue_manager *dqm);
 
-void kfd_debugfs_add_process(struct kfd_process *p);
+int kfd_debugfs_add_process(struct kfd_process *p);
 void kfd_debugfs_remove_process(struct kfd_process *p);
 
 #else
 
 static inline void kfd_debugfs_init(void) {}
 static inline void kfd_debugfs_fini(void) {}
-static inline void kfd_debugfs_add_process(struct kfd_process *p) {}
+static inline int kfd_debugfs_add_process(struct kfd_process *p) { return 0; }
 static inline void kfd_debugfs_remove_process(struct kfd_process *p) {}
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 5fb3679e4e858..eb508fe3ded78 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -33,6 +33,7 @@
 #include <linux/mman.h>
 #include <linux/file.h>
 #include <linux/pm_runtime.h>
+#include <drm/ttm/ttm_bo.h>
 #include "amdgpu_amdkfd.h"
 #include "amdgpu.h"
 #include "amdgpu_reset.h"
@@ -153,6 +154,21 @@ static void kfd_sdma_activity_worker(struct work_struct *work)
 		    (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
 			continue;
 
+		if (dqm->dev->kfd2kgd->hqd_sdma_get_counter) {
+			val = 0;
+			ret = dqm->dev->kfd2kgd->hqd_sdma_get_counter(
+					dqm->dev->adev, q->mqd,
+					dqm->dev->kfd->device_info.num_sdma_queues_per_engine,
+					&val);
+
+			if (ret)
+				pr_debug("Failed to read SDMA queue active counter %i\n", ret);
+			else
+				workarea->sdma_activity_counter += val;
+
+			continue;
+		}
+
 		sdma_q = kzalloc_obj(struct temp_sdma_queue_list);
 		if (!sdma_q) {
 			dqm_unlock(dqm);
@@ -171,7 +187,7 @@ static void kfd_sdma_activity_worker(struct work_struct *work)
 	 * count
 	 */
 	if (list_empty(&sdma_q_list.list)) {
-		workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;
+		workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
 		dqm_unlock(dqm);
 		return;
 	}
@@ -721,7 +737,7 @@ static void kfd_process_free_gpuvm(struct kgd_mem *mem,
 	struct kfd_node *dev = pdd->dev;
 
 	if (kptr && *kptr) {
-		amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
+		amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem);
 		*kptr = NULL;
 	}
 
@@ -730,6 +746,22 @@ static void kfd_process_free_gpuvm(struct kgd_mem *mem,
 					       NULL);
 }
 
+static void kfd_process_free_gpuvm_map(struct kgd_mem *mem,
+				       struct kfd_process_device *pdd,
+				       struct iosys_map *map)
+{
+	struct kfd_node *dev = pdd->dev;
+
+	if (map && !iosys_map_is_null(map)) {
+		amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem);
+		iosys_map_clear(map);
+	}
+
+	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);
+	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,
+					       NULL);
+}
+
 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
  *	This function should be only called right after the process
  *	is created and when kfd_processes_mutex is still being held
@@ -761,10 +793,16 @@ static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
 	}
 
 	if (kptr) {
-		err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(
-				(struct kgd_mem *)*mem, kptr, NULL);
+		u32 domain;
+
+		if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
+			domain = AMDGPU_GEM_DOMAIN_VRAM;
+		else
+			domain = AMDGPU_GEM_DOMAIN_GTT;
+		err = amdgpu_amdkfd_gpuvm_map_bo_to_kernel((struct kgd_mem *)*mem,
+							   kptr, NULL, domain);
 		if (err) {
-			pr_debug("Map GTT BO to kernel failed\n");
+			pr_debug("Map BO to kernel failed err %d\n", err);
 			goto sync_memory_failed;
 		}
 	}
@@ -975,7 +1013,10 @@ struct kfd_process *kfd_create_process(struct task_struct *thread)
 		if (ret)
 			pr_warn("Failed to create sysfs entry for the kfd_process");
 
-		kfd_debugfs_add_process(process);
+		ret = kfd_debugfs_add_process(process);
+		if (ret)
+			pr_warn("Failed to create debugfs entry for the kfd_process, ret = %d\n",
+				ret);
 
 		init_waitqueue_head(&process->wait_irq_drain);
 	}
@@ -986,6 +1027,33 @@ struct kfd_process *kfd_create_process(struct task_struct *thread)
 	return process;
 }
 
+/**
+ * amdgpu_amdkfd_set_sigbus_delay - Set per-process KFD SIGBUS delay
+ * @task: task in the target process
+ * @ms:   encoded delay value (0 = immediate, 0xFFFFFFFF = suppress,
+ *        otherwise delay in milliseconds)
+ *
+ * Stores the SIGBUS delivery option on the kfd_process associated with
+ * @task. If the calling process has not opened /dev/kfd yet (no
+ * kfd_process exists), this is a no-op - the option only applies to
+ * processes that actually use KFD.
+ */
+int amdgpu_amdkfd_set_sigbus_delay(struct task_struct *task, u32 ms)
+{
+	struct kfd_process *p;
+
+	if (!task->mm)
+		return -EINVAL;
+
+	p = kfd_lookup_process_by_mm(task->mm);
+	if (!p)
+		return 0;
+
+	atomic_set(&p->kfd_sigbus_delay_ms, ms);
+	kfd_unref_process(p);
+	return 0;
+}
+
 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
 {
 	struct kfd_process *process;
@@ -1092,7 +1160,7 @@ static void kfd_process_kunmap_signal_bo(struct kfd_process *p)
 	if (!mem)
 		goto out;
 
-	amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
+	amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem);
 
 out:
 	mutex_unlock(&p->mutex);
@@ -1138,9 +1206,9 @@ static void kfd_process_destroy_pdds(struct kfd_process *p)
 		if (pdd->drm_file)
 			fput(pdd->drm_file);
 
-		if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
-			free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
-				get_order(KFD_CWSR_TBA_TMA_SIZE));
+		if (!iosys_map_is_null(&pdd->qpd.cwsr_map) && !pdd->qpd.cwsr_base)
+			free_pages((unsigned long)pdd->qpd.cwsr_map.vaddr,
+				   get_order(KFD_CWSR_TBA_TMA_SIZE));
 
 		idr_destroy(&pdd->alloc_idr);
 
@@ -1330,6 +1398,11 @@ void kfd_process_notifier_release_internal(struct kfd_process *p)
 	kfd_process_table_remove(p);
 	cancel_delayed_work_sync(&p->eviction_work);
 	cancel_delayed_work_sync(&p->restore_work);
+	/*
+	 * If work pending, cancel it and drop the extra ref
+	 */
+	if (cancel_delayed_work_sync(&p->signal_work))
+		kfd_unref_process(p);
 
 	/*
 	 * Dequeue and destroy user queues, it is not safe for GPU to access
@@ -1436,34 +1509,49 @@ static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
 {
 	struct kfd_node *dev = pdd->dev;
 	struct qcm_process_device *qpd = &pdd->qpd;
-	uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT
-			| KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
+	u32 flags = KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
 			| KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
 	struct kgd_mem *mem;
 	void *kaddr;
 	int ret;
 
-	if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
+	if (!dev->kfd->cwsr_enabled || !iosys_map_is_null(&qpd->cwsr_map) || !qpd->cwsr_base)
 		return 0;
 
-	/* cwsr_base is only set for dGPU */
+	if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 4, 2) && !dev->adev->apu_prefer_gtt)
+		flags |= KFD_IOC_ALLOC_MEM_FLAGS_VRAM;
+	else
+		flags |= KFD_IOC_ALLOC_MEM_FLAGS_GTT;
+
+	/* Allocate CWSR TBA/TMA buffers */
 	ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,
 				      KFD_CWSR_TBA_TMA_SIZE, flags, &mem, &kaddr);
 	if (ret)
 		return ret;
 
 	qpd->cwsr_mem = mem;
-	qpd->cwsr_kaddr = kaddr;
+
+	/* Set up iosys_map based on whether memory is MMIO or system memory */
+	if (mem->bo->kmap.bo_kmap_type & TTM_BO_MAP_IOMEM_MASK)
+		iosys_map_set_vaddr_iomem(&qpd->cwsr_map, kaddr);
+	else
+		iosys_map_set_vaddr(&qpd->cwsr_map, kaddr);
+
 	qpd->tba_addr = qpd->cwsr_base;
 
-	memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
+	/* Copy CWSR ISA to buffer using appropriate accessor */
+	iosys_map_memcpy_to(&qpd->cwsr_map, 0, dev->kfd->cwsr_isa,
+			    dev->kfd->cwsr_isa_size);
 
 	kfd_process_set_trap_debug_flag(&pdd->qpd,
 					pdd->process->debug_trap_enabled);
 
 	qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
-	pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
-		 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
+	pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_map:%s at %p for pqm.\n",
+		 qpd->tba_addr, qpd->tma_addr,
+		 qpd->cwsr_map.is_iomem ? "iomem" : "system",
+		 qpd->cwsr_map.is_iomem ? (void *)qpd->cwsr_map.vaddr_iomem :
+					  qpd->cwsr_map.vaddr);
 
 	return 0;
 }
@@ -1473,24 +1561,24 @@ static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)
 	struct kfd_node *dev = pdd->dev;
 	struct qcm_process_device *qpd = &pdd->qpd;
 
-	if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)
+	if (!dev->kfd->cwsr_enabled || iosys_map_is_null(&qpd->cwsr_map) || !qpd->cwsr_base)
 		return;
 
-	kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr);
+	kfd_process_free_gpuvm_map(qpd->cwsr_mem, pdd, &qpd->cwsr_map);
 }
 
 void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
 				  uint64_t tba_addr,
 				  uint64_t tma_addr)
 {
-	if (qpd->cwsr_kaddr) {
+	if (!iosys_map_is_null(&qpd->cwsr_map)) {
 		/* KFD trap handler is bound, record as second-level TBA/TMA
 		 * in first-level TMA. First-level trap will jump to second.
 		 */
-		uint64_t *tma =
-			(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
-		tma[0] = tba_addr;
-		tma[1] = tma_addr;
+		iosys_map_wr(&qpd->cwsr_map, KFD_CWSR_TMA_OFFSET,
+			     uint64_t, tba_addr);
+		iosys_map_wr(&qpd->cwsr_map, KFD_CWSR_TMA_OFFSET + sizeof(uint64_t),
+			     uint64_t, tma_addr);
 	} else {
 		/* No trap handler bound, bind as first-level TBA/TMA. */
 		qpd->tba_addr = tba_addr;
@@ -1556,10 +1644,10 @@ bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
 void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
 				     bool enabled)
 {
-	if (qpd->cwsr_kaddr) {
-		uint64_t *tma =
-			(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
-		tma[2] = enabled;
+	if (!iosys_map_is_null(&qpd->cwsr_map)) {
+		iosys_map_wr(&qpd->cwsr_map,
+			     KFD_CWSR_TMA_OFFSET + 2 * sizeof(uint64_t),
+			     uint64_t, enabled);
 	}
 }
 
@@ -1586,6 +1674,7 @@ struct kfd_process *create_process(const struct task_struct *thread, bool primar
 
 	INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
 	INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
+	INIT_DELAYED_WORK(&process->signal_work, kfd_signal_sigbus_delayed_fn);
 	process->last_restore_timestamp = get_jiffies_64();
 	err = kfd_event_init_process(process);
 	if (err)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index 9ccbc6e5b27b3..97e77a5a35e51 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -1055,7 +1055,7 @@ int kfd_criu_restore_queue(struct kfd_process *p,
 	ctl_stack = mqd + q_data->mqd_size;
 
 	memset(&qp, 0, sizeof(qp));
-	set_queue_properties_from_criu(&qp, q_data, NUM_XCC(pdd->dev->adev->gfx.xcc_mask));
+	set_queue_properties_from_criu(&qp, q_data, NUM_XCC(pdd->dev->xcc_mask));
 
 	print_queue_properties(&qp);
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
index 28354a4e5dd5d..98a5512b701b1 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
@@ -23,6 +23,7 @@
  */
 
 #include <linux/slab.h>
+#include <linux/overflow.h>
 #include "kfd_priv.h"
 #include "kfd_topology.h"
 #include "kfd_svm.h"
@@ -235,7 +236,7 @@ int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_prope
 	struct kfd_topology_device *topo_dev;
 	u64 expected_queue_size;
 	struct amdgpu_vm *vm;
-	u32 total_cwsr_size;
+	u64 total_cwsr_size;
 	int err;
 
 	topo_dev = kfd_topology_device_by_id(pdd->dev->id);
@@ -308,8 +309,14 @@ int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_prope
 		goto out_err_unreserve;
 	}
 
-	total_cwsr_size = (properties->ctx_save_restore_area_size +
-			   topo_dev->node_props.debug_memory_size) * NUM_XCC(pdd->dev->xcc_mask);
+	total_cwsr_size = (u64)properties->ctx_save_restore_area_size +
+			  topo_dev->node_props.debug_memory_size;
+	if (check_mul_overflow(total_cwsr_size,
+			       NUM_XCC(pdd->dev->xcc_mask),
+			       &total_cwsr_size)) {
+		err = -EINVAL;
+		goto out_err_unreserve;
+	}
 	total_cwsr_size = ALIGN(total_cwsr_size, PAGE_SIZE);
 
 	err = kfd_queue_buffer_get(vm, (void *)properties->ctx_save_restore_area_address,
@@ -344,7 +351,7 @@ int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_prope
 int kfd_queue_release_buffers(struct kfd_process_device *pdd, struct queue_properties *properties)
 {
 	struct kfd_topology_device *topo_dev;
-	u32 total_cwsr_size;
+	u64 total_cwsr_size;
 
 	kfd_queue_buffer_put(&properties->wptr_bo);
 	kfd_queue_buffer_put(&properties->rptr_bo);
@@ -355,8 +362,12 @@ int kfd_queue_release_buffers(struct kfd_process_device *pdd, struct queue_prope
 	topo_dev = kfd_topology_device_by_id(pdd->dev->id);
 	if (!topo_dev)
 		return -EINVAL;
-	total_cwsr_size = (properties->ctx_save_restore_area_size +
-			   topo_dev->node_props.debug_memory_size) * NUM_XCC(pdd->dev->xcc_mask);
+	total_cwsr_size = (u64)properties->ctx_save_restore_area_size +
+			  topo_dev->node_props.debug_memory_size;
+	if (check_mul_overflow(total_cwsr_size,
+			       NUM_XCC(pdd->dev->xcc_mask),
+			       &total_cwsr_size))
+		return -EINVAL;
 	total_cwsr_size = ALIGN(total_cwsr_size, PAGE_SIZE);
 
 	kfd_queue_buffer_svm_put(pdd, properties->ctx_save_restore_area_address, total_cwsr_size);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c
index e659cd50eb0bd..6a7b4d959541b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c
@@ -224,10 +224,10 @@ static void kfd_smi_event_add(struct task_struct *task, struct kfd_node *dev,
 
 	pid = kfd_smi_task_to_pid(task);
 
-	len = snprintf(fifo_in, sizeof(fifo_in), "%x ", event);
+	len = scnprintf(fifo_in, sizeof(fifo_in), "%x ", event);
 
 	va_start(args, fmt);
-	len += vsnprintf(fifo_in + len, sizeof(fifo_in) - len, fmt, args);
+	len += vscnprintf(fifo_in + len, sizeof(fifo_in) - len, fmt, args);
 	va_end(args);
 
 	add_event_to_kfifo(pid, dev, event, fifo_in, len);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 0900bb23349ef..30ad10bbd47ee 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -3473,7 +3473,13 @@ svm_range_is_valid(struct kfd_process *p, uint64_t start, uint64_t size)
 	unsigned long start_unchg = start;
 
 	start <<= PAGE_SHIFT;
-	end = start + (size << PAGE_SHIFT);
+
+	if (size == 0)
+		return -EINVAL;
+
+	if (check_add_overflow(start, size << PAGE_SHIFT, &end))
+		return -EOVERFLOW;
+
 	do {
 		vma = vma_lookup(p->mm, start);
 		if (!vma || (vma->vm_flags & device_vma))
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 00517c3d0e6a0..35b3abe57b800 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -2013,12 +2013,21 @@ static void kfd_topology_set_capabilities(struct kfd_topology_device *dev)
 			dev->node_props.capability |=
 				HSA_CAP_TRAP_DEBUG_PRECISE_MEMORY_OPERATIONS_SUPPORTED;
 
-		if (!amdgpu_sriov_vf(dev->gpu->adev))
+		if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(9, 4, 3) &&
+		    !amdgpu_sriov_vf(dev->gpu->adev))
 			dev->node_props.capability |= HSA_CAP_PER_QUEUE_RESET_SUPPORTED;
 
 	} else {
 		dev->node_props.debug_prop |= HSA_DBG_WATCH_ADDR_MASK_LO_BIT_GFX10 |
 					HSA_DBG_WATCH_ADDR_MASK_HI_BIT;
+		/* gfx11 dGPU and gfx12.0 */
+		if ((KFD_GC_VERSION(dev->gpu) == IP_VERSION(11, 0, 0) ||
+		     KFD_GC_VERSION(dev->gpu) == IP_VERSION(11, 0, 2) ||
+		     KFD_GC_VERSION(dev->gpu) == IP_VERSION(11, 0, 3) ||
+		     KFD_GC_VERSION(dev->gpu) == IP_VERSION(12, 0, 0) ||
+		     KFD_GC_VERSION(dev->gpu) == IP_VERSION(12, 0, 1)) &&
+		     !amdgpu_sriov_vf(dev->gpu->adev))
+			dev->node_props.capability |= HSA_CAP_PER_QUEUE_RESET_SUPPORTED;
 
 		if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(12, 0, 0))
 			dev->node_props.capability |=
diff --git a/drivers/gpu/drm/amd/amdxcp/amdgpu_xcp_drv.c b/drivers/gpu/drm/amd/amdxcp/amdgpu_xcp_drv.c
index 995cae6be144a..84e13537a7bac 100644
--- a/drivers/gpu/drm/amd/amdxcp/amdgpu_xcp_drv.c
+++ b/drivers/gpu/drm/amd/amdxcp/amdgpu_xcp_drv.c
@@ -45,7 +45,7 @@ static const struct drm_driver amdgpu_xcp_driver = {
 	.minor = 0,
 };
 
-static int8_t pdev_num;
+static u8 pdev_num;
 static struct xcp_device *xcp_dev[MAX_XCP_PLATFORM_DEVICE];
 static DEFINE_MUTEX(xcp_mutex);
 
@@ -56,6 +56,11 @@ int amdgpu_xcp_drm_dev_alloc(struct drm_device **ddev)
 	char *dev_name;
 	int ret, i;
 
+	if (!ddev)
+		return -EINVAL;
+
+	BUILD_BUG_ON(MAX_XCP_PLATFORM_DEVICE >= U8_MAX);
+
 	guard(mutex)(&xcp_mutex);
 
 	if (pdev_num >= MAX_XCP_PLATFORM_DEVICE)
@@ -105,7 +110,7 @@ int amdgpu_xcp_drm_dev_alloc(struct drm_device **ddev)
 }
 EXPORT_SYMBOL(amdgpu_xcp_drm_dev_alloc);
 
-static void free_xcp_dev(int8_t index)
+static void free_xcp_dev(uint8_t index)
 {
 	if ((index < MAX_XCP_PLATFORM_DEVICE) && (xcp_dev[index])) {
 		struct platform_device *pdev = xcp_dev[index]->pdev;
@@ -114,17 +119,18 @@ static void free_xcp_dev(int8_t index)
 		platform_device_unregister(pdev);
 
 		xcp_dev[index] = NULL;
-		pdev_num--;
+		if (pdev_num > 0)
+			pdev_num--;
 	}
 }
 
 void amdgpu_xcp_drm_dev_free(struct drm_device *ddev)
 {
-	int8_t i;
+	uint8_t i;
 
 	guard(mutex)(&xcp_mutex);
 
-	for (i = 0; i < MAX_XCP_PLATFORM_DEVICE; i++) {
+	for (i = 0; pdev_num && i < MAX_XCP_PLATFORM_DEVICE; i++) {
 		if ((xcp_dev[i]) && (&xcp_dev[i]->drm == ddev)) {
 			free_xcp_dev(i);
 			break;
@@ -135,7 +141,7 @@ EXPORT_SYMBOL(amdgpu_xcp_drm_dev_free);
 
 void amdgpu_xcp_drv_release(void)
 {
-	int8_t i;
+	uint8_t i;
 
 	guard(mutex)(&xcp_mutex);
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/Makefile b/drivers/gpu/drm/amd/display/amdgpu_dm/Makefile
index 54a93e4255b36..d83878e35b613 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/Makefile
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/Makefile
@@ -41,7 +41,11 @@ AMDGPUDM = \
 	amdgpu_dm_quirks.o \
 	amdgpu_dm_wb.o \
 	amdgpu_dm_colorop.o \
-	amdgpu_dm_ism.o
+	amdgpu_dm_ism.o \
+	amdgpu_dm_backlight.o \
+	amdgpu_dm_audio.o \
+	amdgpu_dm_dmub.o \
+	amdgpu_dm_connector.o
 
 ifdef CONFIG_DRM_AMD_DC_FP
 AMDGPUDM += dc_fpu.o
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
index 18145d78334fd..e7080880b2211 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
@@ -66,6 +66,11 @@
 #endif
 #include "amdgpu_dm_psr.h"
 #include "amdgpu_dm_replay.h"
+#include "amdgpu_dm_backlight.h"
+#include "amdgpu_dm_audio.h"
+#include "amdgpu_dm_dmub.h"
+#include "amdgpu_dm_connector.h"
+#include "amdgpu_dm_kunit_helpers.h"
 
 #include "ivsrcid/ivsrcid_vislands30.h"
 
@@ -94,7 +99,6 @@
 #include <drm/drm_mode.h>
 #include <drm/drm_utils.h>
 #include <drm/drm_vblank.h>
-#include <drm/drm_audio_component.h>
 #include <drm/drm_colorop.h>
 #include <drm/drm_gem_atomic_helper.h>
 
@@ -107,60 +111,9 @@
 #include "modules/inc/mod_power.h"
 #include "modules/power/power_helpers.h"
 
-static_assert(AMDGPU_DMUB_NOTIFICATION_MAX == DMUB_NOTIFICATION_MAX, "AMDGPU_DMUB_NOTIFICATION_MAX mismatch");
-
-#define FIRMWARE_RENOIR_DMUB "amdgpu/renoir_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_RENOIR_DMUB);
-#define FIRMWARE_SIENNA_CICHLID_DMUB "amdgpu/sienna_cichlid_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_SIENNA_CICHLID_DMUB);
-#define FIRMWARE_NAVY_FLOUNDER_DMUB "amdgpu/navy_flounder_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_NAVY_FLOUNDER_DMUB);
-#define FIRMWARE_GREEN_SARDINE_DMUB "amdgpu/green_sardine_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_GREEN_SARDINE_DMUB);
-#define FIRMWARE_VANGOGH_DMUB "amdgpu/vangogh_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_VANGOGH_DMUB);
-#define FIRMWARE_DIMGREY_CAVEFISH_DMUB "amdgpu/dimgrey_cavefish_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_DIMGREY_CAVEFISH_DMUB);
-#define FIRMWARE_BEIGE_GOBY_DMUB "amdgpu/beige_goby_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_BEIGE_GOBY_DMUB);
-#define FIRMWARE_YELLOW_CARP_DMUB "amdgpu/yellow_carp_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_YELLOW_CARP_DMUB);
-#define FIRMWARE_DCN_314_DMUB "amdgpu/dcn_3_1_4_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_DCN_314_DMUB);
-#define FIRMWARE_DCN_315_DMUB "amdgpu/dcn_3_1_5_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_DCN_315_DMUB);
-#define FIRMWARE_DCN316_DMUB "amdgpu/dcn_3_1_6_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_DCN316_DMUB);
-
-#define FIRMWARE_DCN_V3_2_0_DMCUB "amdgpu/dcn_3_2_0_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_DCN_V3_2_0_DMCUB);
-#define FIRMWARE_DCN_V3_2_1_DMCUB "amdgpu/dcn_3_2_1_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_DCN_V3_2_1_DMCUB);
-
-#define FIRMWARE_RAVEN_DMCU		"amdgpu/raven_dmcu.bin"
 MODULE_FIRMWARE(FIRMWARE_RAVEN_DMCU);
-
-#define FIRMWARE_NAVI12_DMCU            "amdgpu/navi12_dmcu.bin"
 MODULE_FIRMWARE(FIRMWARE_NAVI12_DMCU);
 
-#define FIRMWARE_DCN_35_DMUB "amdgpu/dcn_3_5_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_DCN_35_DMUB);
-
-#define FIRMWARE_DCN_351_DMUB "amdgpu/dcn_3_5_1_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_DCN_351_DMUB);
-
-#define FIRMWARE_DCN_36_DMUB "amdgpu/dcn_3_6_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_DCN_36_DMUB);
-
-#define FIRMWARE_DCN_401_DMUB "amdgpu/dcn_4_0_1_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_DCN_401_DMUB);
-
-#define FIRMWARE_DCN_42_DMUB "amdgpu/dcn_4_2_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_DCN_42_DMUB);
-
-#define FIRMWARE_DCN_42B_DMUB "amdgpu/dcn_4_2_1_dmcub.bin"
-MODULE_FIRMWARE(FIRMWARE_DCN_42B_DMUB);
-
 /**
  * DOC: overview
  *
@@ -174,46 +127,7 @@ MODULE_FIRMWARE(FIRMWARE_DCN_42B_DMUB);
 /* basic init/fini API */
 static int amdgpu_dm_init(struct amdgpu_device *adev);
 static void amdgpu_dm_fini(struct amdgpu_device *adev);
-static bool is_freesync_video_mode(const struct drm_display_mode *mode, struct amdgpu_dm_connector *aconnector);
 static void reset_freesync_config_for_crtc(struct dm_crtc_state *new_crtc_state);
-static struct amdgpu_i2c_adapter *
-create_i2c(struct ddc_service *ddc_service, bool oem);
-
-static enum drm_mode_subconnector get_subconnector_type(struct dc_link *link)
-{
-	switch (link->dpcd_caps.dongle_type) {
-	case DISPLAY_DONGLE_NONE:
-		return DRM_MODE_SUBCONNECTOR_Native;
-	case DISPLAY_DONGLE_DP_VGA_CONVERTER:
-		return DRM_MODE_SUBCONNECTOR_VGA;
-	case DISPLAY_DONGLE_DP_DVI_CONVERTER:
-	case DISPLAY_DONGLE_DP_DVI_DONGLE:
-		return DRM_MODE_SUBCONNECTOR_DVID;
-	case DISPLAY_DONGLE_DP_HDMI_CONVERTER:
-	case DISPLAY_DONGLE_DP_HDMI_DONGLE:
-		return DRM_MODE_SUBCONNECTOR_HDMIA;
-	case DISPLAY_DONGLE_DP_HDMI_MISMATCHED_DONGLE:
-	default:
-		return DRM_MODE_SUBCONNECTOR_Unknown;
-	}
-}
-
-static void update_subconnector_property(struct amdgpu_dm_connector *aconnector)
-{
-	struct dc_link *link = aconnector->dc_link;
-	struct drm_connector *connector = &aconnector->base;
-	enum drm_mode_subconnector subconnector = DRM_MODE_SUBCONNECTOR_Unknown;
-
-	if (connector->connector_type != DRM_MODE_CONNECTOR_DisplayPort)
-		return;
-
-	if (aconnector->dc_sink)
-		subconnector = get_subconnector_type(link);
-
-	drm_object_property_set_value(&connector->base,
-			connector->dev->mode_config.dp_subconnector_property,
-			subconnector);
-}
 
 /*
  * initializes drm_device display related structures, based on the information
@@ -226,32 +140,23 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev);
 /* removes and deallocates the drm structures, created by the above function */
 static void amdgpu_dm_destroy_drm_device(struct amdgpu_display_manager *dm);
 
-static int amdgpu_dm_connector_init(struct amdgpu_display_manager *dm,
-				    struct amdgpu_dm_connector *amdgpu_dm_connector,
-				    u32 link_index,
-				    struct amdgpu_encoder *amdgpu_encoder);
-static int amdgpu_dm_encoder_init(struct drm_device *dev,
-				  struct amdgpu_encoder *aencoder,
-				  uint32_t link_index);
-
-static int amdgpu_dm_connector_get_modes(struct drm_connector *connector);
-
 static int amdgpu_dm_atomic_setup_commit(struct drm_atomic_commit *state);
 static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_commit *state);
+static void dm_enable_per_frame_crtc_master_sync(struct dc_state *context);
 
 static int amdgpu_dm_atomic_check(struct drm_device *dev,
 				  struct drm_atomic_commit *state);
 
-static void handle_hpd_irq_helper(struct amdgpu_dm_connector *aconnector);
-static void handle_hpd_rx_irq(void *param);
-
-static void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm,
-					 int bl_idx,
-					 u32 user_brightness);
-
-static bool
+STATIC_IFN_KUNIT bool
 is_timing_unchanged_for_freesync(struct drm_crtc_state *old_crtc_state,
 				 struct drm_crtc_state *new_crtc_state);
+
+static inline void amdgpu_dm_exit_ips_for_hw_access(struct dc *dc)
+{
+	if (dc->ctx->dmub_srv && !dc->ctx->dmub_srv->idle_exit_counter)
+		dc_exit_ips_for_hw_access(dc);
+}
+
 /*
  * dm_vblank_get_counter
  *
@@ -332,40 +237,14 @@ static int dm_wait_for_idle(struct amdgpu_ip_block *ip_block)
 	return 0;
 }
 
-static bool dm_check_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	return false;
-}
-
 static int dm_soft_reset(struct amdgpu_ip_block *ip_block)
 {
 	/* XXX todo */
 	return 0;
 }
 
-static struct amdgpu_crtc *
-get_crtc_by_otg_inst(struct amdgpu_device *adev,
-		     int otg_inst)
-{
-	struct drm_device *dev = adev_to_drm(adev);
-	struct drm_crtc *crtc;
-	struct amdgpu_crtc *amdgpu_crtc;
-
-	if (WARN_ON(otg_inst == -1))
-		return adev->mode_info.crtcs[0];
-
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-		amdgpu_crtc = to_amdgpu_crtc(crtc);
-
-		if (amdgpu_crtc->otg_inst == otg_inst)
-			return amdgpu_crtc;
-	}
-
-	return NULL;
-}
-
-static inline bool is_dc_timing_adjust_needed(struct dm_crtc_state *old_state,
-					      struct dm_crtc_state *new_state)
+STATIC_IFN_KUNIT bool is_dc_timing_adjust_needed(struct dm_crtc_state *old_state,
+						 struct dm_crtc_state *new_state)
 {
 	if (new_state->stream->adjust.timing_adjust_pending)
 		return true;
@@ -376,13 +255,14 @@ static inline bool is_dc_timing_adjust_needed(struct dm_crtc_state *old_state,
 	else
 		return false;
 }
+EXPORT_IF_KUNIT(is_dc_timing_adjust_needed);
 
 /*
  * DC will program planes with their z-order determined by their ordering
  * in the dc_surface_updates array. This comparator is used to sort them
  * by descending zpos.
  */
-static int dm_plane_layer_index_cmp(const void *a, const void *b)
+STATIC_IFN_KUNIT int dm_plane_layer_index_cmp(const void *a, const void *b)
 {
 	const struct dc_surface_update *sa = (struct dc_surface_update *)a;
 	const struct dc_surface_update *sb = (struct dc_surface_update *)b;
@@ -390,6 +270,7 @@ static int dm_plane_layer_index_cmp(const void *a, const void *b)
 	/* Sort by descending dc_plane layer_index (i.e. normalized_zpos) */
 	return sb->surface->layer_index - sa->surface->layer_index;
 }
+EXPORT_IF_KUNIT(dm_plane_layer_index_cmp);
 
 /**
  * update_planes_and_stream_adapter() - Send planes to be updated in DC
@@ -430,8406 +311,3144 @@ static inline bool update_planes_and_stream_adapter(struct dc *dc,
 					   stream_update);
 }
 
-/**
- * dm_pflip_high_irq() - Handle pageflip interrupt
- * @interrupt_params: ignored
- *
- * Handles the pageflip interrupt by notifying all interested parties
- * that the pageflip has been completed.
- */
-static void dm_pflip_high_irq(void *interrupt_params)
+static int dm_set_clockgating_state(struct amdgpu_ip_block *ip_block,
+		  enum amd_clockgating_state state)
 {
-	struct amdgpu_crtc *amdgpu_crtc;
-	struct common_irq_params *irq_params = interrupt_params;
-	struct amdgpu_device *adev = irq_params->adev;
-	struct drm_device *dev = adev_to_drm(adev);
-	unsigned long flags;
-	struct drm_pending_vblank_event *e;
-	u32 vpos, hpos, v_blank_start, v_blank_end;
-	bool vrr_active;
+	return 0;
+}
 
-	amdgpu_crtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_PFLIP);
+static int dm_set_powergating_state(struct amdgpu_ip_block *ip_block,
+		  enum amd_powergating_state state)
+{
+	return 0;
+}
 
-	/* IRQ could occur when in initial stage */
-	/* TODO work and BO cleanup */
-	if (amdgpu_crtc == NULL) {
-		drm_dbg_state(dev, "CRTC is null, returning.\n");
-		return;
-	}
+/* Prototypes of private functions */
+static int dm_early_init(struct amdgpu_ip_block *ip_block);
 
-	spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
+/* Allocate memory for FBC compressed data  */
+static void mmhub_read_system_context(struct amdgpu_device *adev, struct dc_phy_addr_space_config *pa_config)
+{
+	u64 pt_base;
+	u32 logical_addr_low;
+	u32 logical_addr_high;
+	u32 agp_base, agp_bot, agp_top;
+	PHYSICAL_ADDRESS_LOC page_table_start, page_table_end, page_table_base;
 
-	if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED) {
-		drm_dbg_state(dev,
-			      "amdgpu_crtc->pflip_status = %d != AMDGPU_FLIP_SUBMITTED(%d) on crtc:%d[%p]\n",
-			      amdgpu_crtc->pflip_status, AMDGPU_FLIP_SUBMITTED,
-			      amdgpu_crtc->crtc_id, amdgpu_crtc);
-		spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
-		return;
-	}
+	memset(pa_config, 0, sizeof(*pa_config));
 
-	/* page flip completed. */
-	e = amdgpu_crtc->event;
-	amdgpu_crtc->event = NULL;
+	agp_base = 0;
+	agp_bot = adev->gmc.agp_start >> 24;
+	agp_top = adev->gmc.agp_end >> 24;
 
-	WARN_ON(!e);
+	/* AGP aperture is disabled */
+	if (agp_bot > agp_top) {
+		logical_addr_low = adev->gmc.fb_start >> 18;
+		if (adev->apu_flags & (AMD_APU_IS_RAVEN2 |
+				       AMD_APU_IS_RENOIR |
+				       AMD_APU_IS_GREEN_SARDINE))
+			/*
+			 * Raven2 has a HW issue that it is unable to use the vram which
+			 * is out of MC_VM_SYSTEM_APERTURE_HIGH_ADDR. So here is the
+			 * workaround that increase system aperture high address (add 1)
+			 * to get rid of the VM fault and hardware hang.
+			 */
+			logical_addr_high = (adev->gmc.fb_end >> 18) + 0x1;
+		else
+			logical_addr_high = adev->gmc.fb_end >> 18;
+	} else {
+		logical_addr_low = min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18;
+		if (adev->apu_flags & (AMD_APU_IS_RAVEN2 |
+				       AMD_APU_IS_RENOIR |
+				       AMD_APU_IS_GREEN_SARDINE))
+			/*
+			 * Raven2 has a HW issue that it is unable to use the vram which
+			 * is out of MC_VM_SYSTEM_APERTURE_HIGH_ADDR. So here is the
+			 * workaround that increase system aperture high address (add 1)
+			 * to get rid of the VM fault and hardware hang.
+			 */
+			logical_addr_high = max((adev->gmc.fb_end >> 18) + 0x1, adev->gmc.agp_end >> 18);
+		else
+			logical_addr_high = max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18;
+	}
 
-	vrr_active = amdgpu_dm_crtc_vrr_active_irq(amdgpu_crtc);
+	pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
 
-	/* Fixed refresh rate, or VRR scanout position outside front-porch? */
-	if (!vrr_active ||
-	    !dc_stream_get_scanoutpos(amdgpu_crtc->dm_irq_params.stream, &v_blank_start,
-				      &v_blank_end, &hpos, &vpos) ||
-	    (vpos < v_blank_start)) {
-		/* Update to correct count and vblank timestamp if racing with
-		 * vblank irq. This also updates to the correct vblank timestamp
-		 * even in VRR mode, as scanout is past the front-porch atm.
-		 */
-		drm_crtc_accurate_vblank_count(&amdgpu_crtc->base);
+	page_table_start.high_part = upper_32_bits(adev->gmc.gart_start >>
+						   AMDGPU_GPU_PAGE_SHIFT);
+	page_table_start.low_part = lower_32_bits(adev->gmc.gart_start >>
+						  AMDGPU_GPU_PAGE_SHIFT);
+	page_table_end.high_part = upper_32_bits(adev->gmc.gart_end >>
+						 AMDGPU_GPU_PAGE_SHIFT);
+	page_table_end.low_part = lower_32_bits(adev->gmc.gart_end >>
+						AMDGPU_GPU_PAGE_SHIFT);
+	page_table_base.high_part = upper_32_bits(pt_base);
+	page_table_base.low_part = lower_32_bits(pt_base);
 
-		/* Wake up userspace by sending the pageflip event with proper
-		 * count and timestamp of vblank of flip completion.
-		 */
-		if (e) {
-			drm_crtc_send_vblank_event(&amdgpu_crtc->base, e);
-
-			/* Event sent, so done with vblank for this flip */
-			drm_crtc_vblank_put(&amdgpu_crtc->base);
-		}
-	} else if (e) {
-		/* VRR active and inside front-porch: vblank count and
-		 * timestamp for pageflip event will only be up to date after
-		 * drm_crtc_handle_vblank() has been executed from late vblank
-		 * irq handler after start of back-porch (vline 0). We queue the
-		 * pageflip event for send-out by drm_crtc_handle_vblank() with
-		 * updated timestamp and count, once it runs after us.
-		 *
-		 * We need to open-code this instead of using the helper
-		 * drm_crtc_arm_vblank_event(), as that helper would
-		 * call drm_crtc_accurate_vblank_count(), which we must
-		 * not call in VRR mode while we are in front-porch!
-		 */
+	pa_config->system_aperture.start_addr = (uint64_t)logical_addr_low << 18;
+	pa_config->system_aperture.end_addr = (uint64_t)logical_addr_high << 18;
 
-		/* sequence will be replaced by real count during send-out. */
-		e->sequence = drm_crtc_vblank_count(&amdgpu_crtc->base);
-		e->pipe = amdgpu_crtc->crtc_id;
+	pa_config->system_aperture.agp_base = (uint64_t)agp_base << 24;
+	pa_config->system_aperture.agp_bot = (uint64_t)agp_bot << 24;
+	pa_config->system_aperture.agp_top = (uint64_t)agp_top << 24;
 
-		list_add_tail(&e->base.link, &adev_to_drm(adev)->vblank_event_list);
-		e = NULL;
-	}
+	pa_config->system_aperture.fb_base = adev->gmc.fb_start;
+	pa_config->system_aperture.fb_offset = adev->vm_manager.vram_base_offset;
+	pa_config->system_aperture.fb_top = adev->gmc.fb_end;
 
-	/* Keep track of vblank of this flip for flip throttling. We use the
-	 * cooked hw counter, as that one incremented at start of this vblank
-	 * of pageflip completion, so last_flip_vblank is the forbidden count
-	 * for queueing new pageflips if vsync + VRR is enabled.
-	 */
-	amdgpu_crtc->dm_irq_params.last_flip_vblank =
-		amdgpu_get_vblank_counter_kms(&amdgpu_crtc->base);
+	pa_config->gart_config.page_table_start_addr = page_table_start.quad_part << 12;
+	pa_config->gart_config.page_table_end_addr = page_table_end.quad_part << 12;
+	pa_config->gart_config.page_table_base_addr = page_table_base.quad_part;
 
-	amdgpu_crtc->pflip_status = AMDGPU_FLIP_NONE;
-	spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
+	pa_config->is_hvm_enabled = adev->mode_info.gpu_vm_support;
 
-	drm_dbg_state(dev,
-		      "crtc:%d[%p], pflip_stat:AMDGPU_FLIP_NONE, vrr[%d]-fp %d\n",
-		      amdgpu_crtc->crtc_id, amdgpu_crtc, vrr_active, (int)!e);
 }
 
-static void dm_handle_vmin_vmax_update(struct work_struct *offload_work)
-{
-	struct vupdate_offload_work *work = container_of(offload_work, struct vupdate_offload_work, work);
-	struct amdgpu_device *adev = work->adev;
-	struct dc_stream_state *stream = work->stream;
-	struct dc_crtc_timing_adjust *adjust = work->adjust;
-
-	mutex_lock(&adev->dm.dc_lock);
-	dc_stream_adjust_vmin_vmax(adev->dm.dc, stream, adjust);
-	mutex_unlock(&adev->dm.dc_lock);
+struct amdgpu_stutter_quirk {
+	u16 chip_vendor;
+	u16 chip_device;
+	u16 subsys_vendor;
+	u16 subsys_device;
+	u8 revision;
+};
 
-	dc_stream_release(stream);
-	kfree(work->adjust);
-	kfree(work);
-}
+static const struct amdgpu_stutter_quirk amdgpu_stutter_quirk_list[] = {
+	/* https://bugzilla.kernel.org/show_bug.cgi?id=214417 */
+	{ 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
+	{ 0, 0, 0, 0, 0 },
+};
 
-static void schedule_dc_vmin_vmax(struct amdgpu_device *adev,
-	struct dc_stream_state *stream,
-	struct dc_crtc_timing_adjust *adjust)
+static bool dm_should_disable_stutter(struct pci_dev *pdev)
 {
-	struct vupdate_offload_work *offload_work = kzalloc_obj(*offload_work,
-								GFP_NOWAIT);
-	if (!offload_work) {
-		drm_dbg_driver(adev_to_drm(adev), "Failed to allocate vupdate_offload_work\n");
-		return;
-	}
+	const struct amdgpu_stutter_quirk *p = amdgpu_stutter_quirk_list;
 
-	struct dc_crtc_timing_adjust *adjust_copy = kzalloc_obj(*adjust_copy,
-								GFP_NOWAIT);
-	if (!adjust_copy) {
-		drm_dbg_driver(adev_to_drm(adev), "Failed to allocate adjust_copy\n");
-		kfree(offload_work);
-		return;
+	while (p && p->chip_device != 0) {
+		if (pdev->vendor == p->chip_vendor &&
+		    pdev->device == p->chip_device &&
+		    pdev->subsystem_vendor == p->subsys_vendor &&
+		    pdev->subsystem_device == p->subsys_device &&
+		    pdev->revision == p->revision) {
+			return true;
+		}
+		++p;
 	}
-
-	dc_stream_retain(stream);
-	memcpy(adjust_copy, adjust, sizeof(*adjust_copy));
-
-	INIT_WORK(&offload_work->work, dm_handle_vmin_vmax_update);
-	offload_work->adev = adev;
-	offload_work->stream = stream;
-	offload_work->adjust = adjust_copy;
-
-	queue_work(system_percpu_wq, &offload_work->work);
+	return false;
 }
 
-static void dm_vupdate_high_irq(void *interrupt_params)
-{
-	struct common_irq_params *irq_params = interrupt_params;
-	struct amdgpu_device *adev = irq_params->adev;
-	struct amdgpu_crtc *acrtc;
-	struct drm_device *drm_dev;
-	struct drm_vblank_crtc *vblank;
-	ktime_t frame_duration_ns, previous_timestamp;
-	unsigned long flags;
-	int vrr_active;
 
-	acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VUPDATE);
+void*
+dm_allocate_gpu_mem(
+		struct amdgpu_device *adev,
+		enum dc_gpu_mem_alloc_type type,
+		size_t size,
+		long long *addr)
+{
+	struct dal_allocation *da;
+	u32 domain = (type == DC_MEM_ALLOC_TYPE_GART) ?
+		AMDGPU_GEM_DOMAIN_GTT : AMDGPU_GEM_DOMAIN_VRAM;
+	int ret;
 
-	if (acrtc) {
-		vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc);
-		drm_dev = acrtc->base.dev;
-		vblank = drm_crtc_vblank_crtc(&acrtc->base);
-		previous_timestamp = atomic64_read(&irq_params->previous_timestamp);
-		frame_duration_ns = vblank->time - previous_timestamp;
+	da = kzalloc_obj(struct dal_allocation);
+	if (!da)
+		return NULL;
 
-		if (frame_duration_ns > 0) {
-			trace_amdgpu_refresh_rate_track(acrtc->base.index,
-						frame_duration_ns,
-						ktime_divns(NSEC_PER_SEC, frame_duration_ns));
-			atomic64_set(&irq_params->previous_timestamp, vblank->time);
-		}
+	ret = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE,
+				      domain, &da->bo,
+				      &da->gpu_addr, &da->cpu_ptr);
 
-		drm_dbg_vbl(drm_dev,
-			    "crtc:%d, vupdate-vrr:%d\n", acrtc->crtc_id,
-			    vrr_active);
+	*addr = da->gpu_addr;
 
-		/* Core vblank handling is done here after end of front-porch in
-		 * vrr mode, as vblank timestamping will give valid results
-		 * while now done after front-porch. This will also deliver
-		 * page-flip completion events that have been queued to us
-		 * if a pageflip happened inside front-porch.
-		 */
-		if (vrr_active && acrtc->dm_irq_params.stream) {
-			bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled;
-			bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled;
-			bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state
-				== VRR_STATE_ACTIVE_VARIABLE;
-
-			amdgpu_dm_crtc_handle_vblank(acrtc);
-
-			/* BTR processing for pre-DCE12 ASICs */
-			if (adev->family < AMDGPU_FAMILY_AI) {
-				spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
-				mod_freesync_handle_v_update(
-				    adev->dm.freesync_module,
-				    acrtc->dm_irq_params.stream,
-				    &acrtc->dm_irq_params.vrr_params);
-
-				if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) {
-					schedule_dc_vmin_vmax(adev,
-						acrtc->dm_irq_params.stream,
-						&acrtc->dm_irq_params.vrr_params.adjust);
-				}
-				spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
-			}
-		}
+	if (ret) {
+		kfree(da);
+		return NULL;
 	}
+
+	/* add da to list in dm */
+	list_add(&da->list, &adev->dm.da_list);
+
+	return da->cpu_ptr;
 }
 
-/**
- * dm_crtc_high_irq() - Handles CRTC interrupt
- * @interrupt_params: used for determining the CRTC instance
- *
- * Handles the CRTC/VSYNC interrupt by notfying DRM's VBLANK
- * event handler.
- */
-static void dm_crtc_high_irq(void *interrupt_params)
+void
+dm_free_gpu_mem(
+		struct amdgpu_device *adev,
+		enum dc_gpu_mem_alloc_type type,
+		void *pvMem)
 {
-	struct common_irq_params *irq_params = interrupt_params;
-	struct amdgpu_device *adev = irq_params->adev;
-	struct drm_writeback_job *job;
-	struct amdgpu_crtc *acrtc;
-	unsigned long flags;
-	int vrr_active;
-
-	acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VBLANK);
-	if (!acrtc)
-		return;
+	struct dal_allocation *da;
 
-	if (acrtc->wb_conn) {
-		spin_lock_irqsave(&acrtc->wb_conn->job_lock, flags);
-
-		if (acrtc->wb_pending) {
-			job = list_first_entry_or_null(&acrtc->wb_conn->job_queue,
-						       struct drm_writeback_job,
-						       list_entry);
-			acrtc->wb_pending = false;
-			spin_unlock_irqrestore(&acrtc->wb_conn->job_lock, flags);
-
-			if (job) {
-				unsigned int v_total, refresh_hz;
-				struct dc_stream_state *stream = acrtc->dm_irq_params.stream;
-
-				v_total = stream->adjust.v_total_max ?
-					  stream->adjust.v_total_max : stream->timing.v_total;
-				refresh_hz = div_u64((uint64_t) stream->timing.pix_clk_100hz *
-					     100LL, (v_total * stream->timing.h_total));
-				mdelay(1000 / refresh_hz);
-
-				drm_writeback_signal_completion(acrtc->wb_conn, 0);
-				dc_stream_fc_disable_writeback(adev->dm.dc,
-							       acrtc->dm_irq_params.stream, 0);
-			}
-		} else
-			spin_unlock_irqrestore(&acrtc->wb_conn->job_lock, flags);
+	/* walk the da list in DM */
+	list_for_each_entry(da, &adev->dm.da_list, list) {
+		if (pvMem == da->cpu_ptr) {
+			amdgpu_bo_free_kernel(&da->bo, &da->gpu_addr, &da->cpu_ptr);
+			list_del(&da->list);
+			kfree(da);
+			break;
+		}
 	}
 
-	vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc);
-
-	drm_dbg_vbl(adev_to_drm(adev),
-		    "crtc:%d, vupdate-vrr:%d, planes:%d\n", acrtc->crtc_id,
-		    vrr_active, acrtc->dm_irq_params.active_planes);
-
-	/**
-	 * Core vblank handling at start of front-porch is only possible
-	 * in non-vrr mode, as only there vblank timestamping will give
-	 * valid results while done in front-porch. Otherwise defer it
-	 * to dm_vupdate_high_irq after end of front-porch.
-	 */
-	if (!vrr_active)
-		amdgpu_dm_crtc_handle_vblank(acrtc);
-
-	/**
-	 * Following stuff must happen at start of vblank, for crc
-	 * computation and below-the-range btr support in vrr mode.
-	 */
-	amdgpu_dm_crtc_handle_crc_irq(&acrtc->base);
-
-	/* BTR updates need to happen before VUPDATE on Vega and above. */
-	if (adev->family < AMDGPU_FAMILY_AI)
-		return;
-
-	spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
-
-	if (acrtc->dm_irq_params.stream &&
-		acrtc->dm_irq_params.vrr_params.supported) {
-		bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled;
-		bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled;
-		bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state == VRR_STATE_ACTIVE_VARIABLE;
+}
 
-		mod_freesync_handle_v_update(adev->dm.freesync_module,
-					     acrtc->dm_irq_params.stream,
-					     &acrtc->dm_irq_params.vrr_params);
+static int amdgpu_dm_init_power_module(struct amdgpu_display_manager *dm)
+{
+	struct mod_power_init_params init_data[MAX_NUM_EDP];
 
-		/* update vmin_vmax only if freesync is enabled, or only if PSR and REPLAY are disabled */
-		if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) {
-			schedule_dc_vmin_vmax(adev, acrtc->dm_irq_params.stream,
-					&acrtc->dm_irq_params.vrr_params.adjust);
-		}
+	if (dm->num_of_edps == 0) {
+		drm_dbg_driver(
+			dm->ddev,
+			"amdgpu: No eDP detected, skip initializing power module\n");
+		return 0;
 	}
 
-	/*
-	 * If there aren't any active_planes then DCH HUBP may be clock-gated.
-	 * In that case, pageflip completion interrupts won't fire and pageflip
-	 * completion events won't get delivered. Prevent this by sending
-	 * pending pageflip events from here if a flip is still pending.
-	 *
-	 * If any planes are enabled, use dm_pflip_high_irq() instead, to
-	 * avoid race conditions between flip programming and completion,
-	 * which could cause too early flip completion events.
-	 */
-	if (adev->family >= AMDGPU_FAMILY_RV &&
-	    acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED &&
-	    acrtc->dm_irq_params.active_planes == 0) {
-		if (acrtc->event) {
-			drm_crtc_send_vblank_event(&acrtc->base, acrtc->event);
-			acrtc->event = NULL;
-			drm_crtc_vblank_put(&acrtc->base);
-		}
-		acrtc->pflip_status = AMDGPU_FLIP_NONE;
-	}
-
-	spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
-}
-
-#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
-/**
- * dm_dcn_vertical_interrupt0_high_irq() - Handles OTG Vertical interrupt0 for
- * DCN generation ASICs
- * @interrupt_params: interrupt parameters
- *
- * Used to set crc window/read out crc value at vertical line 0 position
- */
-static void dm_dcn_vertical_interrupt0_high_irq(void *interrupt_params)
-{
-	struct common_irq_params *irq_params = interrupt_params;
-	struct amdgpu_device *adev = irq_params->adev;
-	struct amdgpu_crtc *acrtc;
+	/* Initialize all the power module parameters */
+	for (int i = 0; i < dm->num_of_edps; i++) {
+		init_data[i].allow_psr_smu_optimizations =
+			!!(amdgpu_dc_feature_mask & DC_PSR_ALLOW_SMU_OPT);
+		init_data[i].allow_psr_multi_disp_optimizations =
+			!!(amdgpu_dc_feature_mask & DC_PSR_ALLOW_MULTI_DISP_OPT);
+		/* See dm_late_init */
+		init_data[i].backlight_ramping_override = false;
+		init_data[i].backlight_ramping_start = 0xCCCC;
+		init_data[i].backlight_ramping_reduction = 0xCCCCCCCC;
+		init_data[i].def_varibright_level = 0;
+		init_data[i].abm_config_setting = 0;
+		init_data[i].num_backlight_levels = 101;
+		init_data[i].use_nits_based_brightness = false;
+		init_data[i].panel_max_millinits = 0;
+		init_data[i].panel_min_millinits = 0;
+		init_data[i].disable_fractional_pwm =
+			!(amdgpu_dc_feature_mask & DC_DISABLE_FRACTIONAL_PWM_MASK);
+		init_data[i].use_custom_backlight_caps = false;
+		init_data[i].custom_backlight_caps_config_no = 0;
+		init_data[i].use_linear_backlight_curve = false;
+		init_data[i].def_varibright_enable = 0;
+		init_data[i].varibright_level = 0;
+		/*
+		 * Power module uses 16-bit backlight levels (0xFFFF max) rather
+		 * than 8-bit(0XFF max)
+		 */
+		init_data[i].min_backlight_pwm =
+			dm->backlight_caps[i].min_input_signal * 0x101;
+		init_data[i].max_backlight_pwm =
+			dm->backlight_caps[i].max_input_signal * 0x101;
+		init_data[i].min_abm_backlight =
+			dm->backlight_caps[i].min_input_signal * 0x101;
 
-	acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VLINE0);
+		/* Min backlight level after ABM reduction,  Don't allow below 1%
+		 * 0xFFFF x 0.01 = 0x28F
+		 */
+		init_data[i].min_abm_backlight = (init_data[i].min_abm_backlight < 0x28F) ?
+			0x28F : init_data[i].min_abm_backlight;
+	}
 
-	if (!acrtc)
-		return;
+	dm->power_module = mod_power_create(dm->dc, init_data, dm->num_of_edps);
+	if (!dm->power_module) {
+		drm_err(dm->ddev, "amdgpu: Error allocating memory for power module\n");
+		return -ENOMEM;
+	}
 
-	amdgpu_dm_crtc_handle_crc_window_irq(&acrtc->base);
-}
-#endif /* CONFIG_DRM_AMD_SECURE_DISPLAY */
+	mod_power_hw_init(dm->power_module);
+	drm_dbg_driver(dm->ddev, "amdgpu: Power module init done\n");
 
-/**
- * dmub_aux_setconfig_callback - Callback for AUX or SET_CONFIG command.
- * @adev: amdgpu_device pointer
- * @notify: dmub notification structure
- *
- * Dmub AUX or SET_CONFIG command completion processing callback
- * Copies dmub notification to DM which is to be read by AUX command.
- * issuing thread and also signals the event to wake up the thread.
- */
-static void dmub_aux_setconfig_callback(struct amdgpu_device *adev,
-					struct dmub_notification *notify)
-{
-	if (adev->dm.dmub_notify)
-		memcpy(adev->dm.dmub_notify, notify, sizeof(struct dmub_notification));
-	if (notify->type == DMUB_NOTIFICATION_AUX_REPLY)
-		complete(&adev->dm.dmub_aux_transfer_done);
+	return 0;
 }
 
-static void dmub_aux_fused_io_callback(struct amdgpu_device *adev,
-					struct dmub_notification *notify)
+static int amdgpu_dm_init(struct amdgpu_device *adev)
 {
-	if (!adev || !notify) {
-		ASSERT(false);
-		return;
-	}
-
-	const struct dmub_cmd_fused_request *req = &notify->fused_request;
-	const uint8_t ddc_line = req->u.aux.ddc_line;
-
-	if (ddc_line >= ARRAY_SIZE(adev->dm.fused_io)) {
-		ASSERT(false);
-		return;
-	}
+	struct dc_init_data init_data;
+	struct dc_callback_init init_params;
+	int r;
 
-	struct fused_io_sync *sync = &adev->dm.fused_io[ddc_line];
+	adev->dm.ddev = adev_to_drm(adev);
+	adev->dm.adev = adev;
 
-	static_assert(sizeof(*req) <= sizeof(sync->reply_data), "Size mismatch");
-	memcpy(sync->reply_data, req, sizeof(*req));
-	complete(&sync->replied);
-}
+	/* Zero all the fields */
+	memset(&init_data, 0, sizeof(init_data));
+	memset(&init_params, 0, sizeof(init_params));
 
-/**
- * dmub_hpd_callback - DMUB HPD interrupt processing callback.
- * @adev: amdgpu_device pointer
- * @notify: dmub notification structure
- *
- * Dmub Hpd interrupt processing callback. Gets displayindex through the
- * ink index and calls helper to do the processing.
- */
-static void dmub_hpd_callback(struct amdgpu_device *adev,
-			      struct dmub_notification *notify)
-{
-	struct amdgpu_dm_connector *aconnector;
-	struct amdgpu_dm_connector *hpd_aconnector = NULL;
-	struct drm_connector *connector;
-	struct drm_connector_list_iter iter;
-	struct dc_link *link;
-	u8 link_index = 0;
-	struct drm_device *dev;
+	mutex_init(&adev->dm.dpia_aux_lock);
+	mutex_init(&adev->dm.dc_lock);
+	mutex_init(&adev->dm.audio_lock);
 
-	if (adev == NULL)
-		return;
+	spin_lock_init(&adev->dm.dmub_lock);
 
-	if (notify == NULL) {
-		drm_err(adev_to_drm(adev), "DMUB HPD callback notification was NULL");
-		return;
+	if (amdgpu_dm_irq_init(adev)) {
+		drm_err(adev_to_drm(adev), "failed to initialize DM IRQ support.\n");
+		goto error;
 	}
 
-	if (notify->link_index > adev->dm.dc->link_count) {
-		drm_err(adev_to_drm(adev), "DMUB HPD index (%u)is abnormal", notify->link_index);
-		return;
-	}
+	/* special handling for early revisions of GC 11.5.4 */
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 4))
+		init_data.asic_id.chip_family = AMDGPU_FAMILY_GC_11_5_4;
+	else
+		init_data.asic_id.chip_family = adev->family;
 
-	/* Skip DMUB HPD IRQ in suspend/resume. We will probe them later. */
-	if (notify->type == DMUB_NOTIFICATION_HPD && adev->in_suspend) {
-		drm_info(adev_to_drm(adev), "Skip DMUB HPD IRQ callback in suspend/resume\n");
-		return;
-	}
+	init_data.asic_id.pci_revision_id = adev->pdev->revision;
+	init_data.asic_id.hw_internal_rev = adev->external_rev_id;
+	init_data.asic_id.chip_id = adev->pdev->device;
 
-	link_index = notify->link_index;
-	link = adev->dm.dc->links[link_index];
-	dev = adev->dm.ddev;
+	init_data.asic_id.vram_width = adev->gmc.vram_width;
+	/* TODO: initialize init_data.asic_id.vram_type here!!!! */
+	init_data.asic_id.atombios_base_address =
+		adev->mode_info.atom_context->bios;
 
-	drm_connector_list_iter_begin(dev, &iter);
-	drm_for_each_connector_iter(connector, &iter) {
+	init_data.driver = adev;
 
-		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
-			continue;
+	/* cgs_device was created in dm_sw_init() */
+	init_data.cgs_device = adev->dm.cgs_device;
 
-		aconnector = to_amdgpu_dm_connector(connector);
-		if (link && aconnector->dc_link == link) {
-			if (notify->type == DMUB_NOTIFICATION_HPD)
-				drm_info(adev_to_drm(adev), "DMUB HPD IRQ callback: link_index=%u\n", link_index);
-			else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ)
-				drm_info(adev_to_drm(adev), "DMUB HPD RX IRQ callback: link_index=%u\n", link_index);
-			else
-				drm_warn(adev_to_drm(adev), "DMUB Unknown HPD callback type %d, link_index=%u\n",
-						notify->type, link_index);
+	init_data.dce_environment = DCE_ENV_PRODUCTION_DRV;
 
-			hpd_aconnector = aconnector;
+	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
+	case IP_VERSION(2, 1, 0):
+		switch (adev->dm.dmcub_fw_version) {
+		case 0: /* development */
+		case 0x1: /* linux-firmware.git hash 6d9f399 */
+		case 0x01000000: /* linux-firmware.git hash 9a0b0f4 */
+			init_data.flags.disable_dmcu = false;
 			break;
+		default:
+			init_data.flags.disable_dmcu = true;
 		}
+		break;
+	case IP_VERSION(2, 0, 3):
+		init_data.flags.disable_dmcu = true;
+		break;
+	default:
+		break;
 	}
-	drm_connector_list_iter_end(&iter);
 
-	if (hpd_aconnector) {
-		if (notify->type == DMUB_NOTIFICATION_HPD) {
-			if (hpd_aconnector->dc_link->hpd_status == (notify->hpd_status == DP_HPD_PLUG))
-				drm_warn(adev_to_drm(adev), "DMUB reported hpd status unchanged. link_index=%u\n", link_index);
-			handle_hpd_irq_helper(hpd_aconnector);
-		} else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ) {
-			handle_hpd_rx_irq(hpd_aconnector);
-		}
-	}
-}
+	/* APU support S/G display by default except:
+	 * ASICs before Carrizo,
+	 * RAVEN1 (Users reported stability issue)
+	 */
 
-/**
- * dmub_hpd_sense_callback - DMUB HPD sense processing callback.
- * @adev: amdgpu_device pointer
- * @notify: dmub notification structure
- *
- * HPD sense changes can occur during low power states and need to be
- * notified from firmware to driver.
- */
-static void dmub_hpd_sense_callback(struct amdgpu_device *adev,
-			      struct dmub_notification *notify)
-{
-	drm_dbg_driver(adev_to_drm(adev), "DMUB HPD SENSE callback.\n");
-}
+	if (adev->asic_type < CHIP_CARRIZO) {
+		init_data.flags.gpu_vm_support = false;
+	} else if (adev->asic_type == CHIP_RAVEN) {
+		if (adev->apu_flags & AMD_APU_IS_RAVEN)
+			init_data.flags.gpu_vm_support = false;
+		else
+			init_data.flags.gpu_vm_support = (amdgpu_sg_display != 0);
+	} else {
+		if (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(2, 0, 3))
+			init_data.flags.gpu_vm_support = (amdgpu_sg_display == 1);
+		else
+			init_data.flags.gpu_vm_support =
+				(amdgpu_sg_display != 0) && (adev->flags & AMD_IS_APU);
+	}
 
-/**
- * register_dmub_notify_callback - Sets callback for DMUB notify
- * @adev: amdgpu_device pointer
- * @type: Type of dmub notification
- * @callback: Dmub interrupt callback function
- * @dmub_int_thread_offload: offload indicator
- *
- * API to register a dmub callback handler for a dmub notification
- * Also sets indicator whether callback processing to be offloaded.
- * to dmub interrupt handling thread
- * Return: true if successfully registered, false if there is existing registration
- */
-static bool register_dmub_notify_callback(struct amdgpu_device *adev,
-					  enum dmub_notification_type type,
-					  dmub_notify_interrupt_callback_t callback,
-					  bool dmub_int_thread_offload)
-{
-	if (callback != NULL && type < ARRAY_SIZE(adev->dm.dmub_thread_offload)) {
-		adev->dm.dmub_callback[type] = callback;
-		adev->dm.dmub_thread_offload[type] = dmub_int_thread_offload;
-	} else
-		return false;
+	adev->mode_info.gpu_vm_support = init_data.flags.gpu_vm_support;
 
-	return true;
-}
+	if (amdgpu_dc_feature_mask & DC_FBC_MASK)
+		init_data.flags.fbc_support = true;
 
-static void dm_handle_hpd_work(struct work_struct *work)
-{
-	struct dmub_hpd_work *dmub_hpd_wrk;
+	if (amdgpu_dc_feature_mask & DC_MULTI_MON_PP_MCLK_SWITCH_MASK)
+		init_data.flags.multi_mon_pp_mclk_switch = true;
 
-	dmub_hpd_wrk = container_of(work, struct dmub_hpd_work, handle_hpd_work);
+	if (amdgpu_dc_feature_mask & DC_DISABLE_FRACTIONAL_PWM_MASK)
+		init_data.flags.disable_fractional_pwm = true;
 
-	if (!dmub_hpd_wrk->dmub_notify) {
-		drm_err(adev_to_drm(dmub_hpd_wrk->adev), "dmub_hpd_wrk dmub_notify is NULL");
-		return;
-	}
+	if (amdgpu_dc_feature_mask & DC_EDP_NO_POWER_SEQUENCING)
+		init_data.flags.edp_no_power_sequencing = true;
 
-	if (dmub_hpd_wrk->dmub_notify->type < ARRAY_SIZE(dmub_hpd_wrk->adev->dm.dmub_callback)) {
-		dmub_hpd_wrk->adev->dm.dmub_callback[dmub_hpd_wrk->dmub_notify->type](dmub_hpd_wrk->adev,
-		dmub_hpd_wrk->dmub_notify);
-	}
+	if (amdgpu_dc_feature_mask & DC_DISABLE_LTTPR_DP1_4A)
+		init_data.flags.allow_lttpr_non_transparent_mode.bits.DP1_4A = true;
+	if (amdgpu_dc_feature_mask & DC_DISABLE_LTTPR_DP2_0)
+		init_data.flags.allow_lttpr_non_transparent_mode.bits.DP2_0 = true;
 
-	kfree(dmub_hpd_wrk->dmub_notify);
-	kfree(dmub_hpd_wrk);
+	if (amdgpu_dc_feature_mask & DC_FRL_MASK)
+		init_data.flags.enable_frl = true;
 
-}
+	init_data.flags.seamless_boot_edp_requested = false;
 
-static const char *dmub_notification_type_str(enum dmub_notification_type e)
-{
-	switch (e) {
-	case DMUB_NOTIFICATION_NO_DATA:
-		return "NO_DATA";
-	case DMUB_NOTIFICATION_AUX_REPLY:
-		return "AUX_REPLY";
-	case DMUB_NOTIFICATION_HPD:
-		return "HPD";
-	case DMUB_NOTIFICATION_HPD_IRQ:
-		return "HPD_IRQ";
-	case DMUB_NOTIFICATION_SET_CONFIG_REPLY:
-		return "SET_CONFIG_REPLY";
-	case DMUB_NOTIFICATION_DPIA_NOTIFICATION:
-		return "DPIA_NOTIFICATION";
-	case DMUB_NOTIFICATION_HPD_SENSE_NOTIFY:
-		return "HPD_SENSE_NOTIFY";
-	case DMUB_NOTIFICATION_FUSED_IO:
-		return "FUSED_IO";
-	default:
-		return "<unknown>";
+	if (amdgpu_device_seamless_boot_supported(adev)) {
+		init_data.flags.seamless_boot_edp_requested = true;
+		init_data.flags.allow_seamless_boot_optimization = true;
+		drm_dbg(adev->dm.ddev, "Seamless boot requested\n");
 	}
-}
-
-#define DMUB_TRACE_MAX_READ 64
-/**
- * dm_dmub_outbox1_low_irq() - Handles Outbox interrupt
- * @interrupt_params: used for determining the Outbox instance
- *
- * Handles the Outbox Interrupt
- * event handler.
- */
-static void dm_dmub_outbox1_low_irq(void *interrupt_params)
-{
-	struct dmub_notification notify = {0};
-	struct common_irq_params *irq_params = interrupt_params;
-	struct amdgpu_device *adev = irq_params->adev;
-	struct amdgpu_display_manager *dm = &adev->dm;
-	struct dmcub_trace_buf_entry entry = { 0 };
-	u32 count = 0;
-	struct dmub_hpd_work *dmub_hpd_wrk;
 
-	do {
-		if (dc_dmub_srv_get_dmub_outbox0_msg(dm->dc, &entry)) {
-			trace_amdgpu_dmub_trace_high_irq(entry.trace_code, entry.tick_count,
-							entry.param0, entry.param1);
+	init_data.flags.enable_mipi_converter_optimization = true;
 
-			drm_dbg_driver(adev_to_drm(adev), "trace_code:%u, tick_count:%u, param0:%u, param1:%u\n",
-				 entry.trace_code, entry.tick_count, entry.param0, entry.param1);
-		} else
-			break;
+	init_data.dcn_reg_offsets = adev->reg_offset[DCE_HWIP][0];
+	init_data.nbio_reg_offsets = adev->reg_offset[NBIO_HWIP][0];
+	init_data.clk_reg_offsets = adev->reg_offset[CLK_HWIP][0];
 
-		count++;
+	if (amdgpu_dc_debug_mask & DC_DISABLE_IPS)
+		init_data.flags.disable_ips = DMUB_IPS_DISABLE_ALL;
+	else if (amdgpu_dc_debug_mask & DC_DISABLE_IPS_DYNAMIC)
+		init_data.flags.disable_ips = DMUB_IPS_DISABLE_DYNAMIC;
+	else if (amdgpu_dc_debug_mask & DC_DISABLE_IPS2_DYNAMIC)
+		init_data.flags.disable_ips = DMUB_IPS_RCG_IN_ACTIVE_IPS2_IN_OFF;
+	else if (amdgpu_dc_debug_mask & DC_FORCE_IPS_ENABLE)
+		init_data.flags.disable_ips = DMUB_IPS_ENABLE;
+	else
+		init_data.flags.disable_ips = dm_get_default_ips_mode(adev);
 
-	} while (count <= DMUB_TRACE_MAX_READ);
+	init_data.flags.disable_ips_in_vpb = 0;
 
-	if (count > DMUB_TRACE_MAX_READ)
-		drm_dbg_driver(adev_to_drm(adev), "Warning : count > DMUB_TRACE_MAX_READ");
+	/* DCN35 and above supports dynamic DTBCLK switch */
+	if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 5, 0))
+		init_data.flags.allow_0_dtb_clk = true;
 
-	if (dc_enable_dmub_notifications(adev->dm.dc) &&
-		irq_params->irq_src == DC_IRQ_SOURCE_DMCUB_OUTBOX) {
+	/* Enable DWB for tested platforms only */
+	if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0))
+		init_data.num_virtual_links = 1;
 
-		do {
-			dc_stat_get_dmub_notification(adev->dm.dc, &notify);
-			if (notify.type >= ARRAY_SIZE(dm->dmub_thread_offload)) {
-				drm_err(adev_to_drm(adev), "DM: notify type %d invalid!", notify.type);
-				continue;
-			}
-			if (!dm->dmub_callback[notify.type]) {
-				drm_warn(adev_to_drm(adev), "DMUB notification skipped due to no handler: type=%s\n",
-					dmub_notification_type_str(notify.type));
-				continue;
-			}
-			if (dm->dmub_thread_offload[notify.type] == true) {
-				dmub_hpd_wrk = kzalloc_obj(*dmub_hpd_wrk,
-							   GFP_ATOMIC);
-				if (!dmub_hpd_wrk) {
-					drm_err(adev_to_drm(adev), "Failed to allocate dmub_hpd_wrk");
-					return;
-				}
-				dmub_hpd_wrk->dmub_notify = kmemdup(&notify, sizeof(struct dmub_notification),
-								    GFP_ATOMIC);
-				if (!dmub_hpd_wrk->dmub_notify) {
-					kfree(dmub_hpd_wrk);
-					drm_err(adev_to_drm(adev), "Failed to allocate dmub_hpd_wrk->dmub_notify");
-					return;
-				}
-				INIT_WORK(&dmub_hpd_wrk->handle_hpd_work, dm_handle_hpd_work);
-				dmub_hpd_wrk->adev = adev;
-				queue_work(adev->dm.delayed_hpd_wq, &dmub_hpd_wrk->handle_hpd_work);
-			} else {
-				dm->dmub_callback[notify.type](adev, &notify);
-			}
-		} while (notify.pending_notification);
+	/* DCN42 and above dpia switch to unified link training path */
+	if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(4, 2, 0)) {
+		init_data.flags.consolidated_dpia_dp_lt = true;
+		init_data.flags.enable_dpia_pre_training = true;
+		init_data.flags.unify_link_enc_assignment = true;
+		init_data.flags.usb4_bw_alloc_support = true;
 	}
-}
+	retrieve_dmi_info(&adev->dm);
+	if (adev->dm.edp0_on_dp1_quirk)
+		init_data.flags.support_edp0_on_dp1 = true;
 
-static int dm_set_clockgating_state(struct amdgpu_ip_block *ip_block,
-		  enum amd_clockgating_state state)
-{
-	return 0;
-}
-
-static int dm_set_powergating_state(struct amdgpu_ip_block *ip_block,
-		  enum amd_powergating_state state)
-{
-	return 0;
-}
-
-/* Prototypes of private functions */
-static int dm_early_init(struct amdgpu_ip_block *ip_block);
-
-/* Allocate memory for FBC compressed data  */
-static void amdgpu_dm_fbc_init(struct drm_connector *connector)
-{
-	struct amdgpu_device *adev = drm_to_adev(connector->dev);
-	struct dm_compressor_info *compressor = &adev->dm.compressor;
-	struct amdgpu_dm_connector *aconn = to_amdgpu_dm_connector(connector);
-	struct drm_display_mode *mode;
-	unsigned long max_size = 0;
-
-	if (adev->dm.dc->fbc_compressor == NULL)
-		return;
-
-	if (aconn->dc_link->connector_signal != SIGNAL_TYPE_EDP)
-		return;
-
-	if (compressor->bo_ptr)
-		return;
+	if (adev->dm.bb_from_dmub)
+		init_data.bb_from_dmub = adev->dm.bb_from_dmub;
+	else
+		init_data.bb_from_dmub = NULL;
 
+	/* Display Core create. */
+	adev->dm.dc = dc_create(&init_data);
 
-	list_for_each_entry(mode, &connector->modes, head) {
-		if (max_size < (unsigned long) mode->htotal * mode->vtotal)
-			max_size = (unsigned long) mode->htotal * mode->vtotal;
+	if (adev->dm.dc) {
+		drm_info(adev_to_drm(adev), "Display Core v%s initialized on %s\n", DC_VER,
+			 dce_version_to_string(adev->dm.dc->ctx->dce_version));
+	} else {
+		drm_info(adev_to_drm(adev), "Display Core failed to initialize with v%s!\n", DC_VER);
+		goto error;
 	}
 
-	if (max_size) {
-		int r = amdgpu_bo_create_kernel(adev, max_size * 4, PAGE_SIZE,
-			    AMDGPU_GEM_DOMAIN_GTT, &compressor->bo_ptr,
-			    &compressor->gpu_addr, &compressor->cpu_addr);
-
-		if (r)
-			drm_err(adev_to_drm(adev), "DM: Failed to initialize FBC\n");
-		else {
-			adev->dm.dc->ctx->fbc_gpu_addr = compressor->gpu_addr;
-			drm_info(adev_to_drm(adev), "DM: FBC alloc %lu\n", max_size*4);
-		}
-
+	if (amdgpu_dc_debug_mask & DC_DISABLE_PIPE_SPLIT) {
+		adev->dm.dc->debug.force_single_disp_pipe_split = false;
+		adev->dm.dc->debug.pipe_split_policy = MPC_SPLIT_AVOID;
 	}
 
-}
-
-static int amdgpu_dm_audio_component_get_eld(struct device *kdev, int port,
-					  int pipe, bool *enabled,
-					  unsigned char *buf, int max_bytes)
-{
-	struct drm_device *dev = dev_get_drvdata(kdev);
-	struct amdgpu_device *adev = drm_to_adev(dev);
-	struct drm_connector *connector;
-	struct drm_connector_list_iter conn_iter;
-	struct amdgpu_dm_connector *aconnector;
-	int ret = 0;
-
-	*enabled = false;
-
-	mutex_lock(&adev->dm.audio_lock);
+	if (adev->asic_type != CHIP_CARRIZO && adev->asic_type != CHIP_STONEY)
+		adev->dm.dc->debug.disable_stutter = amdgpu_pp_feature_mask & PP_STUTTER_MODE ? false : true;
+	if (dm_should_disable_stutter(adev->pdev))
+		adev->dm.dc->debug.disable_stutter = true;
 
-	drm_connector_list_iter_begin(dev, &conn_iter);
-	drm_for_each_connector_iter(connector, &conn_iter) {
+	if (amdgpu_dc_debug_mask & DC_DISABLE_STUTTER)
+		adev->dm.dc->debug.disable_stutter = true;
 
-		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
-			continue;
+	if (amdgpu_dc_debug_mask & DC_DISABLE_DSC)
+		adev->dm.dc->debug.disable_dsc = true;
 
-		aconnector = to_amdgpu_dm_connector(connector);
-		if (aconnector->audio_inst != port)
-			continue;
+	if (amdgpu_dc_debug_mask & DC_DISABLE_CLOCK_GATING)
+		adev->dm.dc->debug.disable_clock_gate = true;
 
-		*enabled = true;
-		mutex_lock(&connector->eld_mutex);
-		ret = drm_eld_size(connector->eld);
-		memcpy(buf, connector->eld, min(max_bytes, ret));
-		mutex_unlock(&connector->eld_mutex);
+	if (amdgpu_dc_debug_mask & DC_FORCE_SUBVP_MCLK_SWITCH)
+		adev->dm.dc->debug.force_subvp_mclk_switch = true;
 
-		break;
+	if (amdgpu_dc_debug_mask & DC_DISABLE_SUBVP_FAMS) {
+		adev->dm.dc->debug.force_disable_subvp = true;
+		adev->dm.dc->debug.fams2_config.bits.enable = false;
 	}
-	drm_connector_list_iter_end(&conn_iter);
-
-	mutex_unlock(&adev->dm.audio_lock);
-
-	drm_dbg_kms(adev_to_drm(adev), "Get ELD : idx=%d ret=%d en=%d\n", port, ret, *enabled);
-
-	return ret;
-}
-
-static const struct drm_audio_component_ops amdgpu_dm_audio_component_ops = {
-	.get_eld = amdgpu_dm_audio_component_get_eld,
-};
 
-static int amdgpu_dm_audio_component_bind(struct device *kdev,
-				       struct device *hda_kdev, void *data)
-{
-	struct drm_device *dev = dev_get_drvdata(kdev);
-	struct amdgpu_device *adev = drm_to_adev(dev);
-	struct drm_audio_component *acomp = data;
+	if (amdgpu_dc_debug_mask & DC_ENABLE_DML2) {
+		adev->dm.dc->debug.using_dml2 = true;
+		adev->dm.dc->debug.using_dml21 = true;
+	}
 
-	acomp->ops = &amdgpu_dm_audio_component_ops;
-	acomp->dev = kdev;
-	adev->dm.audio_component = acomp;
+	if (amdgpu_dc_debug_mask & DC_HDCP_LC_FORCE_FW_ENABLE)
+		adev->dm.dc->debug.hdcp_lc_force_fw_enable = true;
 
-	return 0;
-}
+	if (amdgpu_dc_debug_mask & DC_HDCP_LC_ENABLE_SW_FALLBACK)
+		adev->dm.dc->debug.hdcp_lc_enable_sw_fallback = true;
 
-static void amdgpu_dm_audio_component_unbind(struct device *kdev,
-					  struct device *hda_kdev, void *data)
-{
-	struct amdgpu_device *adev = drm_to_adev(dev_get_drvdata(kdev));
-	struct drm_audio_component *acomp = data;
+	if (amdgpu_dc_debug_mask & DC_SKIP_DETECTION_LT)
+		adev->dm.dc->debug.skip_detection_link_training = true;
 
-	acomp->ops = NULL;
-	acomp->dev = NULL;
-	adev->dm.audio_component = NULL;
-}
+	adev->dm.dc->debug.visual_confirm = amdgpu_dc_visual_confirm;
 
-static const struct component_ops amdgpu_dm_audio_component_bind_ops = {
-	.bind	= amdgpu_dm_audio_component_bind,
-	.unbind	= amdgpu_dm_audio_component_unbind,
-};
+	/* TODO: Remove after DP2 receiver gets proper support of Cable ID feature */
+	adev->dm.dc->debug.ignore_cable_id = true;
 
-static int amdgpu_dm_audio_init(struct amdgpu_device *adev)
-{
-	int i, ret;
+	if (adev->dm.dc->caps.dp_hdmi21_pcon_support)
+		drm_info(adev_to_drm(adev), "DP-HDMI FRL PCON supported\n");
 
-	if (!amdgpu_audio)
-		return 0;
+	r = dm_dmub_hw_init(adev);
+	if (r) {
+		drm_err(adev_to_drm(adev), "DMUB interface failed to initialize: status=%d\n", r);
+		goto error;
+	}
 
-	adev->mode_info.audio.enabled = true;
+	dc_hardware_init(adev->dm.dc);
 
-	adev->mode_info.audio.num_pins = adev->dm.dc->res_pool->audio_count;
+	/* GOP/vBIOS may leave an OPTC enabled for a display present at power-on
+	 * but no longer driven (e.g. an external DP unplugged at boot). Such a
+	 * dangling pipe keeps DCN out of idle and blocks s0i3. Power it down
+	 * here when nothing needs to be preserved.
+	 */
+	if (adev->flags & AMD_IS_APU)
+		dc_disable_dangling_timing_generators(adev->dm.dc);
 
-	for (i = 0; i < adev->mode_info.audio.num_pins; i++) {
-		adev->mode_info.audio.pin[i].channels = -1;
-		adev->mode_info.audio.pin[i].rate = -1;
-		adev->mode_info.audio.pin[i].bits_per_sample = -1;
-		adev->mode_info.audio.pin[i].status_bits = 0;
-		adev->mode_info.audio.pin[i].category_code = 0;
-		adev->mode_info.audio.pin[i].connected = false;
-		adev->mode_info.audio.pin[i].id =
-			adev->dm.dc->res_pool->audios[i]->inst;
-		adev->mode_info.audio.pin[i].offset = 0;
+	adev->dm.hpd_rx_offload_wq = amdgpu_dm_hpd_rx_irq_create_workqueue(adev);
+	if (!adev->dm.hpd_rx_offload_wq) {
+		drm_err(adev_to_drm(adev), "failed to create hpd rx offload workqueue.\n");
+		goto error;
 	}
 
-	ret = component_add(adev->dev, &amdgpu_dm_audio_component_bind_ops);
-	if (ret < 0)
-		return ret;
+	if ((adev->flags & AMD_IS_APU) && (adev->asic_type >= CHIP_CARRIZO)) {
+		struct dc_phy_addr_space_config pa_config;
 
-	adev->dm.audio_registered = true;
+		mmhub_read_system_context(adev, &pa_config);
 
-	return 0;
-}
+		// Call the DC init_memory func
+		dc_setup_system_context(adev->dm.dc, &pa_config);
+	}
 
-static void amdgpu_dm_audio_fini(struct amdgpu_device *adev)
-{
-	if (!amdgpu_audio)
-		return;
+	adev->dm.freesync_module = mod_freesync_create(adev->dm.dc);
+	if (!adev->dm.freesync_module) {
+		drm_err(adev_to_drm(adev),
+		"failed to initialize freesync_module.\n");
+	} else
+		drm_dbg_driver(adev_to_drm(adev), "freesync_module init done %p.\n",
+				adev->dm.freesync_module);
 
-	if (!adev->mode_info.audio.enabled)
-		return;
+	amdgpu_dm_init_color_mod();
 
-	if (adev->dm.audio_registered) {
-		component_del(adev->dev, &amdgpu_dm_audio_component_bind_ops);
-		adev->dm.audio_registered = false;
+	if (adev->dm.dc->caps.max_links > 0) {
+		adev->dm.vblank_control_workqueue =
+			create_singlethread_workqueue("dm_vblank_control_workqueue");
+		if (!adev->dm.vblank_control_workqueue)
+			drm_err(adev_to_drm(adev), "failed to initialize vblank_workqueue.\n");
 	}
 
-	/* TODO: Disable audio? */
-
-	adev->mode_info.audio.enabled = false;
-}
+	if (adev->dm.dc->caps.ips_support &&
+	    adev->dm.dc->config.disable_ips != DMUB_IPS_DISABLE_ALL)
+		adev->dm.idle_workqueue = idle_create_workqueue(adev);
 
-static  void amdgpu_dm_audio_eld_notify(struct amdgpu_device *adev, int pin)
-{
-	struct drm_audio_component *acomp = adev->dm.audio_component;
+	if (adev->dm.dc->caps.max_links > 0 && adev->family >= AMDGPU_FAMILY_RV) {
+		adev->dm.hdcp_workqueue = hdcp_create_workqueue(adev, &init_params.cp_psp, adev->dm.dc);
 
-	if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify) {
-		drm_dbg_kms(adev_to_drm(adev), "Notify ELD: %d\n", pin);
+		if (!adev->dm.hdcp_workqueue)
+			drm_err(adev_to_drm(adev), "failed to initialize hdcp_workqueue.\n");
+		else
+			drm_dbg_driver(adev_to_drm(adev),
+				       "hdcp_workqueue init done %p.\n",
+				       adev->dm.hdcp_workqueue);
 
-		acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr,
-						 pin, -1);
+		dc_init_callbacks(adev->dm.dc, &init_params);
 	}
-}
+	if (adev->dm.dc->caps.max_links > 0) {
+		adev->dm.hdmi_frl_status_polling_wq =
+			create_singlethread_workqueue("hdmi_frl_status_polling_workqueue");
+		if (!adev->dm.hdmi_frl_status_polling_wq)
+			drm_err(adev_to_drm(adev), "failed to initialize hdmi_frl_status_polling_workqueue\n");
+	}
+	if (dc_is_dmub_outbox_supported(adev->dm.dc)) {
+		init_completion(&adev->dm.dmub_aux_transfer_done);
+		adev->dm.dmub_notify = kzalloc_obj(struct dmub_notification);
+		if (!adev->dm.dmub_notify) {
+			drm_info(adev_to_drm(adev), "fail to allocate adev->dm.dmub_notify");
+			goto error;
+		}
 
-static int dm_dmub_hw_init(struct amdgpu_device *adev)
-{
-	const struct dmcub_firmware_header_v1_0 *hdr;
-	struct dmub_srv *dmub_srv = adev->dm.dmub_srv;
-	struct dmub_srv_fb_info *fb_info = adev->dm.dmub_fb_info;
-	const struct firmware *dmub_fw = adev->dm.dmub_fw;
-	struct dc *dc = adev->dm.dc;
-	struct dmcu *dmcu = adev->dm.dc->res_pool->dmcu;
-	struct abm *abm = adev->dm.dc->res_pool->abm;
-	struct dc_context *ctx = adev->dm.dc->ctx;
-	struct dmub_srv_hw_params hw_params;
-	enum dmub_status status;
-	const unsigned char *fw_inst_const, *fw_bss_data;
-	u32 i, fw_inst_const_size, fw_bss_data_size;
-	bool has_hw_support;
-
-	if (!dmub_srv)
-		/* DMUB isn't supported on the ASIC. */
-		return 0;
+		adev->dm.delayed_hpd_wq = create_singlethread_workqueue("amdgpu_dm_hpd_wq");
+		if (!adev->dm.delayed_hpd_wq) {
+			drm_err(adev_to_drm(adev), "failed to create hpd offload workqueue.\n");
+			goto error;
+		}
 
-	if (!fb_info) {
-		drm_err(adev_to_drm(adev), "No framebuffer info for DMUB service.\n");
-		return -EINVAL;
-	}
+		amdgpu_dm_outbox_init(adev);
+		if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_AUX_REPLY,
+			dm_dmub_aux_setconfig_callback, false)) {
+			drm_err(adev_to_drm(adev), "fail to register dmub aux callback");
+			goto error;
+		}
 
-	if (!dmub_fw) {
-		/* Firmware required for DMUB support. */
-		drm_err(adev_to_drm(adev), "No firmware provided for DMUB.\n");
-		return -EINVAL;
-	}
+		for (size_t i = 0; i < ARRAY_SIZE(adev->dm.fused_io); i++)
+			init_completion(&adev->dm.fused_io[i].replied);
 
-	/* initialize register offsets for ASICs with runtime initialization available */
-	if (dmub_srv->hw_funcs.init_reg_offsets)
-		dmub_srv->hw_funcs.init_reg_offsets(dmub_srv, ctx);
+		if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_FUSED_IO,
+			dm_dmub_aux_fused_io_callback, false)) {
+			drm_err(adev_to_drm(adev), "fail to register dmub fused io callback");
+			goto error;
+		}
+		/* Enable outbox notification only after IRQ handlers are registered and DMUB is alive.
+		 * It is expected that DMUB will resend any pending notifications at this point. Note
+		 * that hpd and hpd_irq handler registration are deferred to
+		 * amdgpu_dm_register_hpd_handlers() to align legacy interface initialization
+		 * sequence. Connection status will be proactivly detected once in the
+		 * amdgpu_dm_initialize_drm_device.
+		 */
+		dc_enable_dmub_outbox(adev->dm.dc);
 
-	status = dmub_srv_has_hw_support(dmub_srv, &has_hw_support);
-	if (status != DMUB_STATUS_OK) {
-		drm_err(adev_to_drm(adev), "Error checking HW support for DMUB: %d\n", status);
-		return -EINVAL;
+		/* DPIA trace goes to dmesg logs only if outbox is enabled */
+		if (amdgpu_dc_debug_mask & DC_ENABLE_DPIA_TRACE)
+			dc_dmub_srv_enable_dpia_trace(adev->dm.dc);
 	}
 
-	if (!has_hw_support) {
-		drm_info(adev_to_drm(adev), "DMUB unsupported on ASIC\n");
-		return 0;
+	if (amdgpu_dm_initialize_drm_device(adev)) {
+		drm_err(adev_to_drm(adev),
+		"failed to initialize sw for display support.\n");
+		goto error;
 	}
 
-	/* Reset DMCUB if it was previously running - before we overwrite its memory. */
-	status = dmub_srv_hw_reset(dmub_srv);
-	if (status != DMUB_STATUS_OK)
-		drm_warn(adev_to_drm(adev), "Error resetting DMUB HW: %d\n", status);
-
-	hdr = (const struct dmcub_firmware_header_v1_0 *)dmub_fw->data;
-
-	fw_inst_const = dmub_fw->data +
-			le32_to_cpu(hdr->header.ucode_array_offset_bytes) +
-			PSP_HEADER_BYTES_256;
+	if (amdgpu_dm_init_power_module(&adev->dm))
+		goto error;
 
-	fw_bss_data = dmub_fw->data +
-		      le32_to_cpu(hdr->header.ucode_array_offset_bytes) +
-		      le32_to_cpu(hdr->inst_const_bytes);
+	/* create fake encoders for MST */
+	dm_dp_create_fake_mst_encoders(adev);
 
-	/* Copy firmware and bios info into FB memory. */
-	fw_inst_const_size = adev->dm.fw_inst_size;
+	/* TODO: Add_display_info? */
 
-	fw_bss_data_size = le32_to_cpu(hdr->bss_data_bytes);
+	/* TODO use dynamic cursor width */
+	adev_to_drm(adev)->mode_config.cursor_width = adev->dm.dc->caps.max_cursor_size;
+	adev_to_drm(adev)->mode_config.cursor_height = adev->dm.dc->caps.max_cursor_size;
 
-	/* if adev->firmware.load_type == AMDGPU_FW_LOAD_PSP,
-	 * amdgpu_ucode_init_single_fw will load dmub firmware
-	 * fw_inst_const part to cw0; otherwise, the firmware back door load
-	 * will be done by dm_dmub_hw_init
-	 */
-	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
-		memcpy(fb_info->fb[DMUB_WINDOW_0_INST_CONST].cpu_addr, fw_inst_const,
-				fw_inst_const_size);
+	if (drm_vblank_init(adev_to_drm(adev), adev->dm.display_indexes_num)) {
+		drm_err(adev_to_drm(adev),
+		"failed to initialize vblank for display support.\n");
+		goto error;
 	}
 
-	if (fw_bss_data_size)
-		memcpy(fb_info->fb[DMUB_WINDOW_2_BSS_DATA].cpu_addr,
-		       fw_bss_data, fw_bss_data_size);
+#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
+	amdgpu_dm_crtc_secure_display_create_contexts(adev);
+	if (!adev->dm.secure_display_ctx.crtc_ctx)
+		drm_err(adev_to_drm(adev), "failed to initialize secure display contexts.\n");
 
-	/* Copy firmware bios info into FB memory. */
-	memcpy(fb_info->fb[DMUB_WINDOW_3_VBIOS].cpu_addr, adev->bios,
-	       adev->bios_size);
+	if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(4, 0, 1))
+		adev->dm.secure_display_ctx.support_mul_roi = true;
 
-	/* Reset regions that need to be reset. */
-	memset(fb_info->fb[DMUB_WINDOW_4_MAILBOX].cpu_addr, 0,
-	fb_info->fb[DMUB_WINDOW_4_MAILBOX].size);
+#endif
 
-	memset(fb_info->fb[DMUB_WINDOW_5_TRACEBUFF].cpu_addr, 0,
-	       fb_info->fb[DMUB_WINDOW_5_TRACEBUFF].size);
+	drm_dbg_driver(adev_to_drm(adev), "KMS initialized.\n");
 
-	memset(fb_info->fb[DMUB_WINDOW_6_FW_STATE].cpu_addr, 0,
-	       fb_info->fb[DMUB_WINDOW_6_FW_STATE].size);
+	return 0;
+error:
+	amdgpu_dm_fini(adev);
 
-	memset(fb_info->fb[DMUB_WINDOW_SHARED_STATE].cpu_addr, 0,
-	       fb_info->fb[DMUB_WINDOW_SHARED_STATE].size);
+	return -EINVAL;
+}
 
-	/* Initialize hardware. */
-	memset(&hw_params, 0, sizeof(hw_params));
-	hw_params.soc_fb_info.fb_base = adev->gmc.fb_start;
-	hw_params.soc_fb_info.fb_offset = adev->vm_manager.vram_base_offset;
+static int amdgpu_dm_early_fini(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
 
-	/* backdoor load firmware and trigger dmub running */
-	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
-		hw_params.load_inst_const = true;
+	amdgpu_dm_audio_fini(adev);
 
-	if (dmcu)
-		hw_params.psp_version = dmcu->psp_version;
+	return 0;
+}
 
-	for (i = 0; i < fb_info->num_fb; ++i)
-		hw_params.fb[i] = &fb_info->fb[i];
+static void amdgpu_dm_fini(struct amdgpu_device *adev)
+{
+	int i;
 
-	/* Enable usb4 dpia in the FW APU */
-	if (dc->caps.is_apu &&
-		dc->res_pool->usb4_dpia_count != 0 &&
-		!dc->debug.dpia_debug.bits.disable_dpia) {
-		hw_params.dpia_supported = true;
-		hw_params.disable_dpia = dc->debug.dpia_debug.bits.disable_dpia;
-		hw_params.dpia_hpd_int_enable_supported = false;
-		hw_params.enable_non_transparent_setconfig = dc->config.consolidated_dpia_dp_lt;
-		hw_params.disable_dpia_bw_allocation = !dc->config.usb4_bw_alloc_support;
+	if (adev->dm.vblank_control_workqueue) {
+		destroy_workqueue(adev->dm.vblank_control_workqueue);
+		adev->dm.vblank_control_workqueue = NULL;
 	}
 
-	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
-	case IP_VERSION(3, 5, 0):
-	case IP_VERSION(3, 5, 1):
-	case IP_VERSION(3, 6, 0):
-	case IP_VERSION(4, 2, 0):
-	case IP_VERSION(4, 2, 1):
-		hw_params.ips_sequential_ono = adev->external_rev_id > 0x10;
-		hw_params.lower_hbr3_phy_ssc = true;
-		break;
-	default:
-		break;
+	if (adev->dm.idle_workqueue) {
+		if (adev->dm.idle_workqueue->running) {
+			adev->dm.idle_workqueue->enable = false;
+			flush_work(&adev->dm.idle_workqueue->work);
+		}
+
+		kfree(adev->dm.idle_workqueue);
+		adev->dm.idle_workqueue = NULL;
 	}
 
-	status = dmub_srv_hw_init(dmub_srv, &hw_params);
-	if (status != DMUB_STATUS_OK) {
-		drm_err(adev_to_drm(adev), "Error initializing DMUB HW: %d\n", status);
-		return -EINVAL;
-	}
+	/*
+	 * Disable ISM before dc_destroy() invalidates dm->dc.
+	 *
+	 * Quiesce workers first without dc_lock (they take dc_lock
+	 * themselves, so syncing under it would deadlock), then drive the
+	 * FSM back to FULL_POWER_RUNNING under dc_lock.
+	 */
+	amdgpu_dm_ism_disable(&adev->dm);
+	scoped_guard(mutex, &adev->dm.dc_lock)
+		amdgpu_dm_ism_force_full_power(&adev->dm);
 
-	/* Wait for firmware load to finish. */
-	status = dmub_srv_wait_for_auto_load(dmub_srv, 100000);
-	if (status != DMUB_STATUS_OK)
-		drm_warn(adev_to_drm(adev), "Wait for DMUB auto-load failed: %d\n", status);
+	amdgpu_dm_destroy_drm_device(&adev->dm);
 
-	/* Init DMCU and ABM if available. */
-	if (dmcu && abm) {
-		dmcu->funcs->dmcu_init(dmcu);
-		abm->dmcu_is_running = dmcu->funcs->is_dmcu_initialized(dmcu);
+#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
+	if (adev->dm.secure_display_ctx.crtc_ctx) {
+		for (i = 0; i < adev->mode_info.num_crtc; i++) {
+			if (adev->dm.secure_display_ctx.crtc_ctx[i].crtc) {
+				flush_work(&adev->dm.secure_display_ctx.crtc_ctx[i].notify_ta_work);
+				flush_work(&adev->dm.secure_display_ctx.crtc_ctx[i].forward_roi_work);
+			}
+		}
+		kfree(adev->dm.secure_display_ctx.crtc_ctx);
+		adev->dm.secure_display_ctx.crtc_ctx = NULL;
+	}
+#endif
+	if (adev->dm.hdcp_workqueue) {
+		hdcp_destroy(&adev->dev->kobj, adev->dm.hdcp_workqueue);
+		adev->dm.hdcp_workqueue = NULL;
 	}
 
-	if (!adev->dm.dc->ctx->dmub_srv)
-		adev->dm.dc->ctx->dmub_srv = dc_dmub_srv_create(adev->dm.dc, dmub_srv);
-	if (!adev->dm.dc->ctx->dmub_srv) {
-		drm_err(adev_to_drm(adev), "Couldn't allocate DC DMUB server!\n");
-		return -ENOMEM;
+	if (adev->dm.dc) {
+		dc_deinit_callbacks(adev->dm.dc);
+		dc_dmub_srv_destroy(&adev->dm.dc->ctx->dmub_srv);
+		if (dc_enable_dmub_notifications(adev->dm.dc)) {
+			kfree(adev->dm.dmub_notify);
+			adev->dm.dmub_notify = NULL;
+			destroy_workqueue(adev->dm.delayed_hpd_wq);
+			adev->dm.delayed_hpd_wq = NULL;
+		}
 	}
 
-	drm_info(adev_to_drm(adev), "DMUB hardware initialized: version=0x%08X\n",
-		 adev->dm.dmcub_fw_version);
+	if (adev->dm.dmub_bo)
+		amdgpu_bo_free_kernel(&adev->dm.dmub_bo,
+				      &adev->dm.dmub_bo_gpu_addr,
+				      &adev->dm.dmub_bo_cpu_addr);
 
-	/* Keeping sanity checks off if
-	 * DCN31 >= 4.0.59.0
-	 * DCN314 >= 8.0.16.0
-	 * Otherwise, turn on sanity checks
-	 */
-	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
-	case IP_VERSION(3, 1, 2):
-	case IP_VERSION(3, 1, 3):
-		if (adev->dm.dmcub_fw_version &&
-			adev->dm.dmcub_fw_version >= DMUB_FW_VERSION(4, 0, 0) &&
-			adev->dm.dmcub_fw_version < DMUB_FW_VERSION(4, 0, 59))
-				adev->dm.dc->debug.sanity_checks = true;
-		break;
-	case IP_VERSION(3, 1, 4):
-		if (adev->dm.dmcub_fw_version &&
-			adev->dm.dmcub_fw_version >= DMUB_FW_VERSION(4, 0, 0) &&
-			adev->dm.dmcub_fw_version < DMUB_FW_VERSION(8, 0, 16))
-				adev->dm.dc->debug.sanity_checks = true;
-		break;
-	default:
-		break;
+	if (adev->dm.boot_time_crc_info.bo_ptr)
+		amdgpu_bo_free_kernel(&adev->dm.boot_time_crc_info.bo_ptr,
+					&adev->dm.boot_time_crc_info.gpu_addr,
+					&adev->dm.boot_time_crc_info.cpu_addr);
+
+	if (adev->dm.hpd_rx_offload_wq && adev->dm.dc) {
+		for (i = 0; i < adev->dm.dc->caps.max_links; i++) {
+			if (adev->dm.hpd_rx_offload_wq[i].wq) {
+				destroy_workqueue(adev->dm.hpd_rx_offload_wq[i].wq);
+				adev->dm.hpd_rx_offload_wq[i].wq = NULL;
+			}
+		}
+
+		kfree(adev->dm.hpd_rx_offload_wq);
+		adev->dm.hpd_rx_offload_wq = NULL;
 	}
 
-	return 0;
-}
+	amdgpu_dm_irq_fini(adev);
 
-static void dm_dmub_hw_resume(struct amdgpu_device *adev)
-{
-	struct dmub_srv *dmub_srv = adev->dm.dmub_srv;
-	enum dmub_status status;
-	bool init;
-	int r;
+	/* DC Destroy TODO: Replace destroy DAL */
+	if (adev->dm.dc)
+		dc_destroy(&adev->dm.dc);
 
-	if (!dmub_srv) {
-		/* DMUB isn't supported on the ASIC. */
-		return;
+	if (adev->dm.cgs_device) {
+		amdgpu_cgs_destroy_device(adev->dm.cgs_device);
+		adev->dm.cgs_device = NULL;
+	}
+	if (adev->dm.freesync_module) {
+		mod_freesync_destroy(adev->dm.freesync_module);
+		adev->dm.freesync_module = NULL;
 	}
 
-	status = dmub_srv_is_hw_init(dmub_srv, &init);
-	if (status != DMUB_STATUS_OK)
-		drm_warn(adev_to_drm(adev), "DMUB hardware init check failed: %d\n", status);
-
-	if (status == DMUB_STATUS_OK && init) {
-		/* Wait for firmware load to finish. */
-		status = dmub_srv_wait_for_auto_load(dmub_srv, 100000);
-		if (status != DMUB_STATUS_OK)
-			drm_warn(adev_to_drm(adev), "Wait for DMUB auto-load failed: %d\n", status);
-	} else {
-		/* Perform the full hardware initialization. */
-		r = dm_dmub_hw_init(adev);
-		if (r)
-			drm_err(adev_to_drm(adev), "DMUB interface failed to initialize: status=%d\n", r);
+	if (adev->dm.power_module) {
+		mod_power_destroy(adev->dm.power_module);
+		adev->dm.power_module = NULL;
 	}
+	mutex_destroy(&adev->dm.audio_lock);
+	mutex_destroy(&adev->dm.dc_lock);
+	mutex_destroy(&adev->dm.dpia_aux_lock);
 }
 
-static void mmhub_read_system_context(struct amdgpu_device *adev, struct dc_phy_addr_space_config *pa_config)
+static int load_dmcu_fw(struct amdgpu_device *adev)
 {
-	u64 pt_base;
-	u32 logical_addr_low;
-	u32 logical_addr_high;
-	u32 agp_base, agp_bot, agp_top;
-	PHYSICAL_ADDRESS_LOC page_table_start, page_table_end, page_table_base;
-
-	memset(pa_config, 0, sizeof(*pa_config));
-
-	agp_base = 0;
-	agp_bot = adev->gmc.agp_start >> 24;
-	agp_top = adev->gmc.agp_end >> 24;
+	const char *fw_name_dmcu = NULL;
+	int r;
+	const struct dmcu_firmware_header_v1_0 *hdr;
 
-	/* AGP aperture is disabled */
-	if (agp_bot > agp_top) {
-		logical_addr_low = adev->gmc.fb_start >> 18;
-		if (adev->apu_flags & (AMD_APU_IS_RAVEN2 |
-				       AMD_APU_IS_RENOIR |
-				       AMD_APU_IS_GREEN_SARDINE))
-			/*
-			 * Raven2 has a HW issue that it is unable to use the vram which
-			 * is out of MC_VM_SYSTEM_APERTURE_HIGH_ADDR. So here is the
-			 * workaround that increase system aperture high address (add 1)
-			 * to get rid of the VM fault and hardware hang.
-			 */
-			logical_addr_high = (adev->gmc.fb_end >> 18) + 0x1;
-		else
-			logical_addr_high = adev->gmc.fb_end >> 18;
-	} else {
-		logical_addr_low = min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18;
-		if (adev->apu_flags & (AMD_APU_IS_RAVEN2 |
-				       AMD_APU_IS_RENOIR |
-				       AMD_APU_IS_GREEN_SARDINE))
-			/*
-			 * Raven2 has a HW issue that it is unable to use the vram which
-			 * is out of MC_VM_SYSTEM_APERTURE_HIGH_ADDR. So here is the
-			 * workaround that increase system aperture high address (add 1)
-			 * to get rid of the VM fault and hardware hang.
-			 */
-			logical_addr_high = max((adev->gmc.fb_end >> 18) + 0x1, adev->gmc.agp_end >> 18);
+	switch (adev->asic_type) {
+#if defined(CONFIG_DRM_AMD_DC_SI)
+	case CHIP_TAHITI:
+	case CHIP_PITCAIRN:
+	case CHIP_VERDE:
+	case CHIP_OLAND:
+#endif
+	case CHIP_BONAIRE:
+	case CHIP_HAWAII:
+	case CHIP_KAVERI:
+	case CHIP_KABINI:
+	case CHIP_MULLINS:
+	case CHIP_TONGA:
+	case CHIP_FIJI:
+	case CHIP_CARRIZO:
+	case CHIP_STONEY:
+	case CHIP_POLARIS11:
+	case CHIP_POLARIS10:
+	case CHIP_POLARIS12:
+	case CHIP_VEGAM:
+	case CHIP_VEGA10:
+	case CHIP_VEGA12:
+	case CHIP_VEGA20:
+		return 0;
+	case CHIP_NAVI12:
+		fw_name_dmcu = FIRMWARE_NAVI12_DMCU;
+		break;
+	case CHIP_RAVEN:
+		if (ASICREV_IS_PICASSO(adev->external_rev_id))
+			fw_name_dmcu = FIRMWARE_RAVEN_DMCU;
+		else if (ASICREV_IS_RAVEN2(adev->external_rev_id))
+			fw_name_dmcu = FIRMWARE_RAVEN_DMCU;
 		else
-			logical_addr_high = max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18;
+			return 0;
+		break;
+	default:
+		switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
+		case IP_VERSION(2, 0, 2):
+		case IP_VERSION(2, 0, 3):
+		case IP_VERSION(2, 0, 0):
+		case IP_VERSION(2, 1, 0):
+		case IP_VERSION(3, 0, 0):
+		case IP_VERSION(3, 0, 2):
+		case IP_VERSION(3, 0, 3):
+		case IP_VERSION(3, 0, 1):
+		case IP_VERSION(3, 1, 2):
+		case IP_VERSION(3, 1, 3):
+		case IP_VERSION(3, 1, 4):
+		case IP_VERSION(3, 1, 5):
+		case IP_VERSION(3, 1, 6):
+		case IP_VERSION(3, 2, 0):
+		case IP_VERSION(3, 2, 1):
+		case IP_VERSION(3, 5, 0):
+		case IP_VERSION(3, 5, 1):
+		case IP_VERSION(3, 6, 0):
+		case IP_VERSION(4, 0, 1):
+		case IP_VERSION(4, 2, 0):
+		case IP_VERSION(4, 2, 1):
+			return 0;
+		default:
+			break;
+		}
+		drm_err(adev_to_drm(adev), "Unsupported ASIC type: 0x%X\n", adev->asic_type);
+		return -EINVAL;
 	}
 
-	pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
-
-	page_table_start.high_part = upper_32_bits(adev->gmc.gart_start >>
-						   AMDGPU_GPU_PAGE_SHIFT);
-	page_table_start.low_part = lower_32_bits(adev->gmc.gart_start >>
-						  AMDGPU_GPU_PAGE_SHIFT);
-	page_table_end.high_part = upper_32_bits(adev->gmc.gart_end >>
-						 AMDGPU_GPU_PAGE_SHIFT);
-	page_table_end.low_part = lower_32_bits(adev->gmc.gart_end >>
-						AMDGPU_GPU_PAGE_SHIFT);
-	page_table_base.high_part = upper_32_bits(pt_base);
-	page_table_base.low_part = lower_32_bits(pt_base);
+	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
+		drm_dbg_kms(adev_to_drm(adev), "dm: DMCU firmware not supported on direct or SMU loading\n");
+		return 0;
+	}
 
-	pa_config->system_aperture.start_addr = (uint64_t)logical_addr_low << 18;
-	pa_config->system_aperture.end_addr = (uint64_t)logical_addr_high << 18;
+	r = amdgpu_ucode_request(adev, &adev->dm.fw_dmcu, AMDGPU_UCODE_REQUIRED,
+				 "%s", fw_name_dmcu);
+	if (r == -ENODEV) {
+		/* DMCU firmware is not necessary, so don't raise a fuss if it's missing */
+		drm_dbg_kms(adev_to_drm(adev), "dm: DMCU firmware not found\n");
+		adev->dm.fw_dmcu = NULL;
+		return 0;
+	}
+	if (r) {
+		drm_err(adev_to_drm(adev), "amdgpu_dm: Can't validate firmware \"%s\"\n",
+			fw_name_dmcu);
+		amdgpu_ucode_release(&adev->dm.fw_dmcu);
+		return r;
+	}
 
-	pa_config->system_aperture.agp_base = (uint64_t)agp_base << 24;
-	pa_config->system_aperture.agp_bot = (uint64_t)agp_bot << 24;
-	pa_config->system_aperture.agp_top = (uint64_t)agp_top << 24;
+	hdr = (const struct dmcu_firmware_header_v1_0 *)adev->dm.fw_dmcu->data;
+	adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_ERAM].ucode_id = AMDGPU_UCODE_ID_DMCU_ERAM;
+	adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_ERAM].fw = adev->dm.fw_dmcu;
+	adev->firmware.fw_size +=
+		ALIGN(le32_to_cpu(hdr->header.ucode_size_bytes) - le32_to_cpu(hdr->intv_size_bytes), PAGE_SIZE);
 
-	pa_config->system_aperture.fb_base = adev->gmc.fb_start;
-	pa_config->system_aperture.fb_offset = adev->vm_manager.vram_base_offset;
-	pa_config->system_aperture.fb_top = adev->gmc.fb_end;
+	adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_INTV].ucode_id = AMDGPU_UCODE_ID_DMCU_INTV;
+	adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_INTV].fw = adev->dm.fw_dmcu;
+	adev->firmware.fw_size +=
+		ALIGN(le32_to_cpu(hdr->intv_size_bytes), PAGE_SIZE);
 
-	pa_config->gart_config.page_table_start_addr = page_table_start.quad_part << 12;
-	pa_config->gart_config.page_table_end_addr = page_table_end.quad_part << 12;
-	pa_config->gart_config.page_table_base_addr = page_table_base.quad_part;
+	adev->dm.dmcu_fw_version = le32_to_cpu(hdr->header.ucode_version);
 
-	pa_config->is_hvm_enabled = adev->mode_info.gpu_vm_support;
+	drm_dbg_kms(adev_to_drm(adev), "PSP loading DMCU firmware\n");
 
+	return 0;
 }
 
-static void force_connector_state(
-	struct amdgpu_dm_connector *aconnector,
-	enum drm_connector_force force_state)
+static int dm_sw_init(struct amdgpu_ip_block *ip_block)
 {
-	struct drm_connector *connector = &aconnector->base;
-
-	mutex_lock(&connector->dev->mode_config.mutex);
-	aconnector->base.force = force_state;
-	mutex_unlock(&connector->dev->mode_config.mutex);
-
-	mutex_lock(&aconnector->hpd_lock);
-	drm_kms_helper_connector_hotplug_event(connector);
-	mutex_unlock(&aconnector->hpd_lock);
-}
-
-static void dm_handle_hpd_rx_offload_work(struct work_struct *work)
-{
-	struct hpd_rx_irq_offload_work *offload_work;
-	struct amdgpu_dm_connector *aconnector;
-	struct dc_link *dc_link;
-	struct amdgpu_device *adev;
-	enum dc_connection_type new_connection_type = dc_connection_none;
-	unsigned long flags;
-	union test_response test_response;
-
-	memset(&test_response, 0, sizeof(test_response));
+	struct amdgpu_device *adev = ip_block->adev;
+	int r;
 
-	offload_work = container_of(work, struct hpd_rx_irq_offload_work, work);
-	aconnector = offload_work->offload_wq->aconnector;
-	adev = offload_work->adev;
+	adev->dm.cgs_device = amdgpu_cgs_create_device(adev);
 
-	if (!aconnector) {
-		drm_err(adev_to_drm(adev), "Can't retrieve aconnector in hpd_rx_irq_offload_work");
-		goto skip;
+	if (!adev->dm.cgs_device) {
+		drm_err(adev_to_drm(adev), "failed to create cgs device.\n");
+		return -EINVAL;
 	}
 
-	dc_link = aconnector->dc_link;
+	/* Moved from dm init since we need to use allocations for storing bounding box data */
+	INIT_LIST_HEAD(&adev->dm.da_list);
 
-	mutex_lock(&aconnector->hpd_lock);
-	if (!dc_link_detect_connection_type(dc_link, &new_connection_type))
-		drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n");
-	mutex_unlock(&aconnector->hpd_lock);
+	r = dm_dmub_sw_init(adev);
+	if (r)
+		return r;
 
-	if (new_connection_type == dc_connection_none)
-		goto skip;
+	return load_dmcu_fw(adev);
+}
 
-	if (amdgpu_in_reset(adev))
-		goto skip;
+static int dm_sw_fini(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	struct dal_allocation *da;
 
-	if (offload_work->data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY ||
-		offload_work->data.bytes.device_service_irq.bits.DOWN_REP_MSG_RDY) {
-		dm_handle_mst_sideband_msg_ready_event(&aconnector->mst_mgr, DOWN_OR_UP_MSG_RDY_EVENT);
-		spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags);
-		offload_work->offload_wq->is_handling_mst_msg_rdy_event = false;
-		spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags);
-		goto skip;
+	list_for_each_entry(da, &adev->dm.da_list, list) {
+		if (adev->dm.bb_from_dmub == (void *) da->cpu_ptr) {
+			amdgpu_bo_free_kernel(&da->bo, &da->gpu_addr, &da->cpu_ptr);
+			list_del(&da->list);
+			kfree(da);
+			adev->dm.bb_from_dmub = NULL;
+			break;
+		}
 	}
 
-	mutex_lock(&adev->dm.dc_lock);
-	if (offload_work->data.bytes.device_service_irq.bits.AUTOMATED_TEST) {
-		dc_link_dp_handle_automated_test(dc_link);
-
-		if (aconnector->timing_changed) {
-			/* force connector disconnect and reconnect */
-			force_connector_state(aconnector, DRM_FORCE_OFF);
-			msleep(100);
-			force_connector_state(aconnector, DRM_FORCE_UNSPECIFIED);
-		}
-
-		test_response.bits.ACK = 1;
-
-		core_link_write_dpcd(
-		dc_link,
-		DP_TEST_RESPONSE,
-		&test_response.raw,
-		sizeof(test_response));
-	} else if ((dc_link->connector_signal != SIGNAL_TYPE_EDP) &&
-			dc_link_check_link_loss_status(dc_link, &offload_work->data) &&
-			dc_link_dp_allow_hpd_rx_irq(dc_link)) {
-		/* offload_work->data is from handle_hpd_rx_irq->
-		 * schedule_hpd_rx_offload_work.this is defer handle
-		 * for hpd short pulse. upon here, link status may be
-		 * changed, need get latest link status from dpcd
-		 * registers. if link status is good, skip run link
-		 * training again.
-		 */
-		union hpd_irq_data irq_data;
-
-		memset(&irq_data, 0, sizeof(irq_data));
 
-		/* before dc_link_dp_handle_link_loss, allow new link lost handle
-		 * request be added to work queue if link lost at end of dc_link_
-		 * dp_handle_link_loss
-		 */
-		spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags);
-		offload_work->offload_wq->is_handling_link_loss = false;
-		spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags);
+	kfree(adev->dm.dmub_fb_info);
+	adev->dm.dmub_fb_info = NULL;
 
-		if ((dc_link_dp_read_hpd_rx_irq_data(dc_link, &irq_data) == DC_OK) &&
-			dc_link_check_link_loss_status(dc_link, &irq_data))
-			dc_link_dp_handle_link_loss(dc_link);
+	if (adev->dm.dmub_srv) {
+		dmub_srv_destroy(adev->dm.dmub_srv);
+		kfree(adev->dm.dmub_srv);
+		adev->dm.dmub_srv = NULL;
 	}
-	mutex_unlock(&adev->dm.dc_lock);
 
-skip:
-	kfree(offload_work);
+	amdgpu_ucode_release(&adev->dm.dmub_fw);
+	amdgpu_ucode_release(&adev->dm.fw_dmcu);
 
+	return 0;
 }
 
-static struct hpd_rx_irq_offload_work_queue *hpd_rx_irq_create_workqueue(struct amdgpu_device *adev)
-{
-	struct dc *dc = adev->dm.dc;
-	int max_caps = dc->caps.max_links;
-	int i = 0;
-	struct hpd_rx_irq_offload_work_queue *hpd_rx_offload_wq = NULL;
-
-	hpd_rx_offload_wq = kzalloc_objs(*hpd_rx_offload_wq, max_caps);
-
-	if (!hpd_rx_offload_wq)
-		return NULL;
 
+static void amdgpu_dm_boot_time_crc_init(struct amdgpu_device *adev)
+{
+	struct dm_boot_time_crc_info *bootcrc_info = NULL;
+	struct dmub_srv *dmub = NULL;
+	union dmub_fw_boot_options option = {0};
+	int ret = 0;
+	const uint32_t fb_size = 3 * 1024 * 1024;	/* 3MB for DCC pattern */
 
-	for (i = 0; i < max_caps; i++) {
-		hpd_rx_offload_wq[i].wq =
-				    create_singlethread_workqueue("amdgpu_dm_hpd_rx_offload_wq");
+	if (!adev || !adev->dm.dc || !adev->dm.dc->ctx ||
+		!adev->dm.dc->ctx->dmub_srv) {
+		return;
+	}
 
-		if (hpd_rx_offload_wq[i].wq == NULL) {
-			drm_err(adev_to_drm(adev), "create amdgpu_dm_hpd_rx_offload_wq fail!");
-			goto out_err;
-		}
+	dmub = adev->dm.dc->ctx->dmub_srv->dmub;
+	bootcrc_info = &adev->dm.boot_time_crc_info;
 
-		spin_lock_init(&hpd_rx_offload_wq[i].offload_lock);
+	if (!dmub || !dmub->hw_funcs.get_fw_boot_option) {
+		drm_dbg(adev_to_drm(adev), "failed to init boot time crc buffer\n");
+		return;
 	}
 
-	return hpd_rx_offload_wq;
+	option = dmub->hw_funcs.get_fw_boot_option(dmub);
 
-out_err:
-	for (i = 0; i < max_caps; i++) {
-		if (hpd_rx_offload_wq[i].wq)
-			destroy_workqueue(hpd_rx_offload_wq[i].wq);
-	}
-	kfree(hpd_rx_offload_wq);
-	return NULL;
-}
+	/* Return if boot time CRC is not enabled */
+	if (option.bits.bootcrc_en_at_S0i3 == 0)
+		return;
 
-struct amdgpu_stutter_quirk {
-	u16 chip_vendor;
-	u16 chip_device;
-	u16 subsys_vendor;
-	u16 subsys_device;
-	u8 revision;
-};
+	/* Create a buffer for boot time CRC */
+	ret = amdgpu_bo_create_kernel(adev, fb_size, PAGE_SIZE,
+		AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT,
+		&bootcrc_info->bo_ptr,
+		&bootcrc_info->gpu_addr,
+		&bootcrc_info->cpu_addr);
 
-static const struct amdgpu_stutter_quirk amdgpu_stutter_quirk_list[] = {
-	/* https://bugzilla.kernel.org/show_bug.cgi?id=214417 */
-	{ 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
-	{ 0, 0, 0, 0, 0 },
-};
+	if (ret) {
+		drm_dbg(adev_to_drm(adev), "failed to create boot time crc buffer\n");
+	} else {
+		bootcrc_info->size = fb_size;
 
-static bool dm_should_disable_stutter(struct pci_dev *pdev)
-{
-	const struct amdgpu_stutter_quirk *p = amdgpu_stutter_quirk_list;
+		drm_dbg(adev_to_drm(adev), "boot time crc buffer created addr 0x%llx, size %u\n",
+			bootcrc_info->gpu_addr, bootcrc_info->size);
 
-	while (p && p->chip_device != 0) {
-		if (pdev->vendor == p->chip_vendor &&
-		    pdev->device == p->chip_device &&
-		    pdev->subsystem_vendor == p->subsys_vendor &&
-		    pdev->subsystem_device == p->subsys_device &&
-		    pdev->revision == p->revision) {
-			return true;
-		}
-		++p;
+		/* Send the buffer info to DMUB */
+		dc_dmub_srv_boot_time_crc_init(adev->dm.dc,
+			bootcrc_info->gpu_addr, bootcrc_info->size);
 	}
-	return false;
 }
 
-
-void*
-dm_allocate_gpu_mem(
-		struct amdgpu_device *adev,
-		enum dc_gpu_mem_alloc_type type,
-		size_t size,
-		long long *addr)
+static int dm_late_init(struct amdgpu_ip_block *ip_block)
 {
-	struct dal_allocation *da;
-	u32 domain = (type == DC_MEM_ALLOC_TYPE_GART) ?
-		AMDGPU_GEM_DOMAIN_GTT : AMDGPU_GEM_DOMAIN_VRAM;
-	int ret;
-
-	da = kzalloc_obj(struct dal_allocation);
-	if (!da)
-		return NULL;
+	struct amdgpu_device *adev = ip_block->adev;
 
-	ret = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE,
-				      domain, &da->bo,
-				      &da->gpu_addr, &da->cpu_ptr);
+	struct dmcu_iram_parameters params;
+	unsigned int linear_lut[16];
+	int i;
+	struct dmcu *dmcu = NULL;
 
-	*addr = da->gpu_addr;
+	dmcu = adev->dm.dc->res_pool->dmcu;
 
-	if (ret) {
-		kfree(da);
-		return NULL;
-	}
+	/* Init the boot time CRC (skip in resume) */
+	if ((adev->in_suspend == 0) &&
+		(amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(3, 6, 0)))
+		amdgpu_dm_boot_time_crc_init(adev);
 
-	/* add da to list in dm */
-	list_add(&da->list, &adev->dm.da_list);
+	for (i = 0; i < 16; i++)
+		linear_lut[i] = 0xFFFF * i / 15;
 
-	return da->cpu_ptr;
-}
+	params.set = 0;
+	params.backlight_ramping_override = false;
+	params.backlight_ramping_start = 0xCCCC;
+	params.backlight_ramping_reduction = 0xCCCCCCCC;
+	params.backlight_lut_array_size = 16;
+	params.backlight_lut_array = linear_lut;
 
-void
-dm_free_gpu_mem(
-		struct amdgpu_device *adev,
-		enum dc_gpu_mem_alloc_type type,
-		void *pvMem)
-{
-	struct dal_allocation *da;
+	/* Min backlight level after ABM reduction,  Don't allow below 1%
+	 * 0xFFFF x 0.01 = 0x28F
+	 */
+	params.min_abm_backlight = 0x28F;
+	/* In the case where abm is implemented on dmcub,
+	 * dmcu object will be null.
+	 * ABM 2.4 and up are implemented on dmcub.
+	 */
+	if (dmcu) {
+		if (!dmcu_load_iram(dmcu, params))
+			return -EINVAL;
+	} else if (adev->dm.dc->ctx->dmub_srv) {
+		struct dc_link *edp_links[MAX_NUM_EDP];
+		int edp_num;
 
-	/* walk the da list in DM */
-	list_for_each_entry(da, &adev->dm.da_list, list) {
-		if (pvMem == da->cpu_ptr) {
-			amdgpu_bo_free_kernel(&da->bo, &da->gpu_addr, &da->cpu_ptr);
-			list_del(&da->list);
-			kfree(da);
-			break;
+		dc_get_edp_links(adev->dm.dc, edp_links, &edp_num);
+		for (i = 0; i < edp_num; i++) {
+			if (!dmub_init_abm_config(adev->dm.dc->res_pool, params, i))
+				return -EINVAL;
 		}
 	}
 
+	return amdgpu_dm_detect_mst_link_for_all_connectors(adev_to_drm(adev));
 }
 
-static enum dmub_status
-dm_dmub_send_vbios_gpint_command(struct amdgpu_device *adev,
-				 enum dmub_gpint_command command_code,
-				 uint16_t param,
-				 uint32_t timeout_us)
+static void resume_mst_branch_status(struct drm_dp_mst_topology_mgr *mgr)
 {
-	union dmub_gpint_data_register reg, test;
-	uint32_t i;
-
-	/* Assume that VBIOS DMUB is ready to take commands */
-
-	reg.bits.status = 1;
-	reg.bits.command_code = command_code;
-	reg.bits.param = param;
-
-	cgs_write_register(adev->dm.cgs_device, 0x34c0 + 0x01f8, reg.all);
-
-	for (i = 0; i < timeout_us; ++i) {
-		udelay(1);
+	u8 buf[UUID_SIZE];
+	guid_t guid;
+	int ret;
 
-		/* Check if our GPINT got acked */
-		reg.bits.status = 0;
-		test = (union dmub_gpint_data_register)
-			cgs_read_register(adev->dm.cgs_device, 0x34c0 + 0x01f8);
+	mutex_lock(&mgr->lock);
+	if (!mgr->mst_primary)
+		goto out_fail;
 
-		if (test.all == reg.all)
-			return DMUB_STATUS_OK;
+	if (drm_dp_read_dpcd_caps(mgr->aux, mgr->dpcd) < 0) {
+		drm_dbg_kms(mgr->dev, "dpcd read failed - undocked during suspend?\n");
+		goto out_fail;
 	}
 
-	return DMUB_STATUS_TIMEOUT;
-}
-
-static void *dm_dmub_get_vbios_bounding_box(struct amdgpu_device *adev)
-{
-	void *bb;
-	long long addr;
-	unsigned int bb_size;
-	int i = 0;
-	uint16_t chunk;
-	enum dmub_gpint_command send_addrs[] = {
-		DMUB_GPINT__SET_BB_ADDR_WORD0,
-		DMUB_GPINT__SET_BB_ADDR_WORD1,
-		DMUB_GPINT__SET_BB_ADDR_WORD2,
-		DMUB_GPINT__SET_BB_ADDR_WORD3,
-	};
-	enum dmub_status ret;
+	ret = drm_dp_dpcd_writeb(mgr->aux, DP_MSTM_CTRL,
+				 DP_MST_EN |
+				 DP_UP_REQ_EN |
+				 DP_UPSTREAM_IS_SRC);
+	if (ret < 0) {
+		drm_dbg_kms(mgr->dev, "mst write failed - undocked during suspend?\n");
+		goto out_fail;
+	}
 
-	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
-	case IP_VERSION(4, 0, 1):
-		bb_size = sizeof(struct dml2_soc_bb);
-		break;
-	case IP_VERSION(4, 2, 0):
-	case IP_VERSION(4, 2, 1):
-		bb_size = sizeof(struct dml2_soc_bb);
-		break;
-	default:
-		return NULL;
+	/* Some hubs forget their guids after they resume */
+	ret = drm_dp_dpcd_read(mgr->aux, DP_GUID, buf, sizeof(buf));
+	if (ret != sizeof(buf)) {
+		drm_dbg_kms(mgr->dev, "dpcd read failed - undocked during suspend?\n");
+		goto out_fail;
 	}
 
-	bb =  dm_allocate_gpu_mem(adev,
-				  DC_MEM_ALLOC_TYPE_GART,
-				  bb_size,
-				  &addr);
-	if (!bb)
-		return NULL;
+	import_guid(&guid, buf);
 
-	for (i = 0; i < 4; i++) {
-		/* Extract 16-bit chunk */
-		chunk = ((uint64_t) addr >> (i * 16)) & 0xFFFF;
-		/* Send the chunk */
-		ret = dm_dmub_send_vbios_gpint_command(adev, send_addrs[i], chunk, 30000);
-		if (ret != DMUB_STATUS_OK)
-			goto free_bb;
-	}
+	if (guid_is_null(&guid)) {
+		guid_gen(&guid);
+		export_guid(buf, &guid);
 
-	/* Now ask DMUB to copy the bb */
-	ret = dm_dmub_send_vbios_gpint_command(adev, DMUB_GPINT__BB_COPY, 1, 200000);
-	if (ret != DMUB_STATUS_OK)
-		goto free_bb;
+		ret = drm_dp_dpcd_write(mgr->aux, DP_GUID, buf, sizeof(buf));
 
-	return bb;
+		if (ret != sizeof(buf)) {
+			drm_dbg_kms(mgr->dev, "check mstb guid failed - undocked during suspend?\n");
+			goto out_fail;
+		}
+	}
 
-free_bb:
-	dm_free_gpu_mem(adev, DC_MEM_ALLOC_TYPE_GART, (void *) bb);
-	return NULL;
+	guid_copy(&mgr->mst_primary->guid, &guid);
 
+out_fail:
+	mutex_unlock(&mgr->lock);
 }
 
-static enum dmub_ips_disable_type dm_get_default_ips_mode(
-	struct amdgpu_device *adev)
+static void s3_handle_mst(struct drm_device *dev, bool suspend)
 {
-	enum dmub_ips_disable_type ret = DMUB_IPS_ENABLE;
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_connector *connector;
+	struct drm_connector_list_iter iter;
+	struct drm_dp_mst_topology_mgr *mgr;
 
-	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
-	case IP_VERSION(3, 5, 0):
-	case IP_VERSION(3, 6, 0):
-	case IP_VERSION(3, 5, 1):
-		ret =  DMUB_IPS_RCG_IN_ACTIVE_IPS2_IN_OFF;
-		break;
-	case IP_VERSION(4, 2, 0):
-	case IP_VERSION(4, 2, 1):
-		ret =  DMUB_IPS_ENABLE;
-		break;
-	default:
-		/* ASICs older than DCN35 do not have IPSs */
-		if (amdgpu_ip_version(adev, DCE_HWIP, 0) < IP_VERSION(3, 5, 0))
-			ret = DMUB_IPS_DISABLE_ALL;
-		break;
-	}
+	drm_connector_list_iter_begin(dev, &iter);
+	drm_for_each_connector_iter(connector, &iter) {
 
-	return ret;
-}
+		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
+			continue;
 
-static int amdgpu_dm_init_power_module(struct amdgpu_display_manager *dm)
-{
-	struct mod_power_init_params init_data[MAX_NUM_EDP];
+		aconnector = to_amdgpu_dm_connector(connector);
+		if (aconnector->dc_link->type != dc_connection_mst_branch ||
+		    aconnector->mst_root)
+			continue;
 
-	if (dm->num_of_edps == 0) {
-		drm_dbg_driver(
-			dm->ddev,
-			"amdgpu: No eDP detected, skip initializing power module\n");
-		return 0;
-	}
+		mgr = &aconnector->mst_mgr;
 
-	/* Initialize all the power module parameters */
-	for (int i = 0; i < dm->num_of_edps; i++) {
-		init_data[i].allow_psr_smu_optimizations =
-			!!(amdgpu_dc_feature_mask & DC_PSR_ALLOW_SMU_OPT);
-		init_data[i].allow_psr_multi_disp_optimizations =
-			!!(amdgpu_dc_feature_mask & DC_PSR_ALLOW_MULTI_DISP_OPT);
-		/* See dm_late_init */
-		init_data[i].backlight_ramping_override = false;
-		init_data[i].backlight_ramping_start = 0xCCCC;
-		init_data[i].backlight_ramping_reduction = 0xCCCCCCCC;
-		init_data[i].def_varibright_level = 0;
-		init_data[i].abm_config_setting = 0;
-		init_data[i].num_backlight_levels = 101;
-		init_data[i].use_nits_based_brightness = false;
-		init_data[i].panel_max_millinits = 0;
-		init_data[i].panel_min_millinits = 0;
-		init_data[i].disable_fractional_pwm =
-			!(amdgpu_dc_feature_mask & DC_DISABLE_FRACTIONAL_PWM_MASK);
-		init_data[i].use_custom_backlight_caps = false;
-		init_data[i].custom_backlight_caps_config_no = 0;
-		init_data[i].use_linear_backlight_curve = false;
-		init_data[i].def_varibright_enable = 0;
-		init_data[i].varibright_level = 0;
-		/*
-		 * Power module uses 16-bit backlight levels (0xFFFF max) rather
-		 * than 8-bit(0XFF max)
-		 */
-		init_data[i].min_backlight_pwm =
-			dm->backlight_caps[i].min_input_signal * 0x101;
-		init_data[i].max_backlight_pwm =
-			dm->backlight_caps[i].max_input_signal * 0x101;
-		init_data[i].min_abm_backlight =
-			dm->backlight_caps[i].min_input_signal * 0x101;
+		if (suspend) {
+			drm_dp_mst_topology_mgr_suspend(mgr);
+		} else {
+			/* if extended timeout is supported in hardware,
+			 * default to LTTPR timeout (3.2ms) first as a W/A for DP link layer
+			 * CTS 4.2.1.1 regression introduced by CTS specs requirement update.
+			 */
+			try_to_configure_aux_timeout(aconnector->dc_link->ddc, LINK_AUX_DEFAULT_LTTPR_TIMEOUT_PERIOD);
+			if (!dp_is_lttpr_present(aconnector->dc_link))
+				try_to_configure_aux_timeout(aconnector->dc_link->ddc, LINK_AUX_DEFAULT_TIMEOUT_PERIOD);
 
-		/* Min backlight level after ABM reduction,  Don't allow below 1%
-		 * 0xFFFF x 0.01 = 0x28F
-		 */
-		init_data[i].min_abm_backlight = (init_data[i].min_abm_backlight < 0x28F) ?
-			0x28F : init_data[i].min_abm_backlight;
+			/* TODO: move resume_mst_branch_status() into drm mst resume again
+			 * once topology probing work is pulled out from mst resume into mst
+			 * resume 2nd step. mst resume 2nd step should be called after old
+			 * state getting restored (i.e. drm_atomic_helper_resume()).
+			 */
+			resume_mst_branch_status(mgr);
+		}
 	}
+	drm_connector_list_iter_end(&iter);
+}
 
-	dm->power_module = mod_power_create(dm->dc, init_data, dm->num_of_edps);
-	if (!dm->power_module) {
-		drm_err(dm->ddev, "amdgpu: Error allocating memory for power module\n");
-		return -ENOMEM;
+static int amdgpu_dm_smu_write_watermarks_table(struct amdgpu_device *adev)
+{
+	int ret = 0;
+
+	/* This interface is for dGPU Navi1x.Linux dc-pplib interface depends
+	 * on window driver dc implementation.
+	 * For Navi1x, clock settings of dcn watermarks are fixed. the settings
+	 * should be passed to smu during boot up and resume from s3.
+	 * boot up: dc calculate dcn watermark clock settings within dc_create,
+	 * dcn20_resource_construct
+	 * then call pplib functions below to pass the settings to smu:
+	 * smu_set_watermarks_for_clock_ranges
+	 * smu_set_watermarks_table
+	 * navi10_set_watermarks_table
+	 * smu_write_watermarks_table
+	 *
+	 * For Renoir, clock settings of dcn watermark are also fixed values.
+	 * dc has implemented different flow for window driver:
+	 * dc_hardware_init / dc_set_power_state
+	 * dcn10_init_hw
+	 * notify_wm_ranges
+	 * set_wm_ranges
+	 * -- Linux
+	 * smu_set_watermarks_for_clock_ranges
+	 * renoir_set_watermarks_table
+	 * smu_write_watermarks_table
+	 *
+	 * For Linux,
+	 * dc_hardware_init -> amdgpu_dm_init
+	 * dc_set_power_state --> dm_resume
+	 *
+	 * therefore, this function apply to navi10/12/14 but not Renoir
+	 * *
+	 */
+	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
+	case IP_VERSION(2, 0, 2):
+	case IP_VERSION(2, 0, 0):
+		break;
+	default:
+		return 0;
 	}
 
-	mod_power_hw_init(dm->power_module);
-	drm_dbg_driver(dm->ddev, "amdgpu: Power module init done\n");
+	ret = amdgpu_dpm_write_watermarks_table(adev);
+	if (ret) {
+		drm_err(adev_to_drm(adev), "Failed to update WMTABLE!\n");
+		return ret;
+	}
 
 	return 0;
 }
 
-static void hdmi_frl_status_polling_work(struct work_struct *work)
+static int dm_oem_i2c_hw_init(struct amdgpu_device *adev)
 {
-	struct amdgpu_display_manager *dm =
-		container_of(to_delayed_work(work), struct amdgpu_display_manager,
-				hdmi_frl_status_polling_work);
-	struct dc *dc = dm->dc;
-	struct dc_link *dc_link;
-	bool link_update = false;
-
-	for (int i = 0; i < MAX_LINKS; i++) {
-		dc_link = dc->links[i];
-
-		if (!dc_link || !dc_link->local_sink)
-			continue;
-
-		if (!dc_is_hdmi_signal(dc_link->connector_signal))
-			continue;
+	struct amdgpu_display_manager *dm = &adev->dm;
+	struct amdgpu_i2c_adapter *oem_i2c;
+	struct ddc_service *oem_ddc_service;
+	int r;
 
-		if (dc_link->connector_signal != SIGNAL_TYPE_HDMI_FRL)
-			continue;
+	oem_ddc_service = dc_get_oem_i2c_device(adev->dm.dc);
+	if (oem_ddc_service) {
+		oem_i2c = amdgpu_dm_create_i2c(oem_ddc_service, true);
+		if (!oem_i2c) {
+			drm_info(adev_to_drm(adev), "Failed to create oem i2c adapter data\n");
+			return -ENOMEM;
+		}
 
-		link_update = dc_link_frl_poll_status_flag(dc_link);
-		if (link_update) {
-			mutex_lock(&dm->dc_lock);
-			dc_link_detect(dc_link, DETECT_REASON_RETRAIN);
-			mutex_unlock(&dm->dc_lock);
+		r = devm_i2c_add_adapter(adev->dev, &oem_i2c->base);
+		if (r) {
+			drm_info(adev_to_drm(adev), "Failed to register oem i2c\n");
+			kfree(oem_i2c);
+			return r;
 		}
+		dm->oem_i2c = oem_i2c;
 	}
 
-	queue_delayed_work(dm->hdmi_frl_status_polling_wq,
-			   &dm->hdmi_frl_status_polling_work,
-			   msecs_to_jiffies(dm->hdmi_frl_status_polling_delay_ms));
+	return 0;
 }
 
-static int amdgpu_dm_init(struct amdgpu_device *adev)
+/**
+ * dm_hw_init() - Initialize DC device
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ *
+ * Initialize the &struct amdgpu_display_manager device. This involves calling
+ * the initializers of each DM component, then populating the struct with them.
+ *
+ * Although the function implies hardware initialization, both hardware and
+ * software are initialized here. Splitting them out to their relevant init
+ * hooks is a future TODO item.
+ *
+ * Some notable things that are initialized here:
+ *
+ * - Display Core, both software and hardware
+ * - DC modules that we need (freesync and color management)
+ * - DRM software states
+ * - Interrupt sources and handlers
+ * - Vblank support
+ * - Debug FS entries, if enabled
+ */
+static int dm_hw_init(struct amdgpu_ip_block *ip_block)
 {
-	struct dc_init_data init_data;
-	struct dc_callback_init init_params;
+	struct amdgpu_device *adev = ip_block->adev;
 	int r;
 
-	adev->dm.ddev = adev_to_drm(adev);
-	adev->dm.adev = adev;
+	adev->dm.i2c_devres_group = devres_open_group(adev->dev, NULL, GFP_KERNEL);
+	if (!adev->dm.i2c_devres_group)
+		return -ENOMEM;
 
-	/* Zero all the fields */
-	memset(&init_data, 0, sizeof(init_data));
-	memset(&init_params, 0, sizeof(init_params));
+	/* Create DAL display manager */
+	r = amdgpu_dm_init(adev);
+	if (r)
+		goto err_release_i2c;
+	amdgpu_dm_hpd_init(adev);
 
-	mutex_init(&adev->dm.dpia_aux_lock);
-	mutex_init(&adev->dm.dc_lock);
-	mutex_init(&adev->dm.audio_lock);
+	r = dm_oem_i2c_hw_init(adev);
+	if (r)
+		drm_info(adev_to_drm(adev), "Failed to add OEM i2c bus\n");
 
-	spin_lock_init(&adev->dm.dmub_lock);
+	devres_close_group(adev->dev, adev->dm.i2c_devres_group);
+	return 0;
 
-	if (amdgpu_dm_irq_init(adev)) {
-		drm_err(adev_to_drm(adev), "failed to initialize DM IRQ support.\n");
-		goto error;
-	}
+err_release_i2c:
+	devres_release_group(adev->dev, adev->dm.i2c_devres_group);
+	return r;
+}
 
-	/* special handling for early revisions of GC 11.5.4 */
-	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 4))
-		init_data.asic_id.chip_family = AMDGPU_FAMILY_GC_11_5_4;
-	else
-		init_data.asic_id.chip_family = adev->family;
+/**
+ * dm_hw_fini() - Teardown DC device
+ * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
+ *
+ * Teardown components within &struct amdgpu_display_manager that require
+ * cleanup. This involves cleaning up the DRM device, DC, and any modules that
+ * were loaded. Also flush IRQ workqueues and disable them.
+ */
+static int dm_hw_fini(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
 
-	init_data.asic_id.pci_revision_id = adev->pdev->revision;
-	init_data.asic_id.hw_internal_rev = adev->external_rev_id;
-	init_data.asic_id.chip_id = adev->pdev->device;
+	if (adev->dm.i2c_devres_group)
+		devres_release_group(adev->dev, adev->dm.i2c_devres_group);
 
-	init_data.asic_id.vram_width = adev->gmc.vram_width;
-	/* TODO: initialize init_data.asic_id.vram_type here!!!! */
-	init_data.asic_id.atombios_base_address =
-		adev->mode_info.atom_context->bios;
+	amdgpu_dm_hpd_fini(adev);
 
-	init_data.driver = adev;
+	amdgpu_dm_fini(adev);
+	return 0;
+}
 
-	/* cgs_device was created in dm_sw_init() */
-	init_data.cgs_device = adev->dm.cgs_device;
 
-	init_data.dce_environment = DCE_ENV_PRODUCTION_DRV;
+static void dm_gpureset_toggle_interrupts(struct amdgpu_device *adev,
+				 struct dc_state *state, bool enable)
+{
+	enum dc_irq_source irq_source;
+	struct amdgpu_crtc *acrtc;
+	int rc = -EBUSY;
+	int i = 0;
 
-	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
-	case IP_VERSION(2, 1, 0):
-		switch (adev->dm.dmcub_fw_version) {
-		case 0: /* development */
-		case 0x1: /* linux-firmware.git hash 6d9f399 */
-		case 0x01000000: /* linux-firmware.git hash 9a0b0f4 */
-			init_data.flags.disable_dmcu = false;
-			break;
-		default:
-			init_data.flags.disable_dmcu = true;
-		}
-		break;
-	case IP_VERSION(2, 0, 3):
-		init_data.flags.disable_dmcu = true;
-		break;
-	default:
-		break;
-	}
+	for (i = 0; i < state->stream_count; i++) {
+		acrtc = amdgpu_dm_get_crtc_by_otg_inst(
+				adev, state->stream_status[i].primary_otg_inst);
 
-	/* APU support S/G display by default except:
-	 * ASICs before Carrizo,
-	 * RAVEN1 (Users reported stability issue)
-	 */
+		if (acrtc && state->stream_status[i].plane_count != 0 &&
+		    amdgpu_ip_version(adev, DCE_HWIP, 0) == 0) {
+			irq_source = IRQ_TYPE_PFLIP + acrtc->otg_inst;
+			rc = dc_interrupt_set(adev->dm.dc, irq_source, enable) ? 0 : -EBUSY;
+			if (rc)
+				drm_warn(adev_to_drm(adev), "Failed to %s pflip interrupts\n",
+					 enable ? "enable" : "disable");
 
-	if (adev->asic_type < CHIP_CARRIZO) {
-		init_data.flags.gpu_vm_support = false;
-	} else if (adev->asic_type == CHIP_RAVEN) {
-		if (adev->apu_flags & AMD_APU_IS_RAVEN)
-			init_data.flags.gpu_vm_support = false;
-		else
-			init_data.flags.gpu_vm_support = (amdgpu_sg_display != 0);
-	} else {
-		if (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(2, 0, 3))
-			init_data.flags.gpu_vm_support = (amdgpu_sg_display == 1);
-		else
-			init_data.flags.gpu_vm_support =
-				(amdgpu_sg_display != 0) && (adev->flags & AMD_IS_APU);
-	}
+			if (dc_supports_vrr(adev->dm.dc->ctx->dce_version)) {
+				if (enable) {
+					if (amdgpu_dm_crtc_vrr_active(
+							to_dm_crtc_state(acrtc->base.state)))
+						rc = amdgpu_dm_crtc_set_vupdate_irq(
+							&acrtc->base, true);
+				} else
+					rc = amdgpu_dm_crtc_set_vupdate_irq(
+							&acrtc->base, false);
 
-	adev->mode_info.gpu_vm_support = init_data.flags.gpu_vm_support;
+				if (rc)
+					drm_warn(adev_to_drm(adev), "Failed to %sable vupdate interrupt\n",
+						enable ? "en" : "dis");
+			}
 
-	if (amdgpu_dc_feature_mask & DC_FBC_MASK)
-		init_data.flags.fbc_support = true;
+			irq_source = IRQ_TYPE_VBLANK + acrtc->otg_inst;
+			/* During gpu-reset we disable and then enable vblank irq, so
+			 * don't use amdgpu_irq_get/put() to avoid refcount change.
+			 */
+			if (!dc_interrupt_set(adev->dm.dc, irq_source, enable))
+				drm_warn(adev_to_drm(adev), "Failed to %sable vblank interrupt\n", enable ? "en" : "dis");
 
-	if (amdgpu_dc_feature_mask & DC_MULTI_MON_PP_MCLK_SWITCH_MASK)
-		init_data.flags.multi_mon_pp_mclk_switch = true;
+		} else if (acrtc && state->stream_status[i].plane_count != 0) {
+			/* DCN only needs to toggle VUPDATE_NO_LOCK */
+			rc = amdgpu_dm_crtc_set_vupdate_irq(&acrtc->base, enable);
+			if (rc)
+				drm_warn(adev_to_drm(adev), "Failed to %sable vupdate interrupt\n",
+					 enable ? "en" : "dis");
+		}
+	}
 
-	if (amdgpu_dc_feature_mask & DC_DISABLE_FRACTIONAL_PWM_MASK)
-		init_data.flags.disable_fractional_pwm = true;
+}
 
-	if (amdgpu_dc_feature_mask & DC_EDP_NO_POWER_SEQUENCING)
-		init_data.flags.edp_no_power_sequencing = true;
+DEFINE_FREE(state_release, struct dc_state *, if (_T) dc_state_release(_T))
 
-	if (amdgpu_dc_feature_mask & DC_DISABLE_LTTPR_DP1_4A)
-		init_data.flags.allow_lttpr_non_transparent_mode.bits.DP1_4A = true;
-	if (amdgpu_dc_feature_mask & DC_DISABLE_LTTPR_DP2_0)
-		init_data.flags.allow_lttpr_non_transparent_mode.bits.DP2_0 = true;
+static enum dc_status amdgpu_dm_commit_zero_streams(struct dc *dc)
+{
+	struct dc_state *context __free(state_release) = NULL;
+	int i;
+	struct dc_stream_state *del_streams[MAX_PIPES];
+	int del_streams_count = 0;
+	struct dc_commit_streams_params params = {};
 
-	if (amdgpu_dc_feature_mask & DC_FRL_MASK)
-		init_data.flags.enable_frl = true;
+	memset(del_streams, 0, sizeof(del_streams));
 
-	init_data.flags.seamless_boot_edp_requested = false;
+	context = dc_state_create_current_copy(dc);
+	if (context == NULL)
+		return DC_ERROR_UNEXPECTED;
 
-	if (amdgpu_device_seamless_boot_supported(adev)) {
-		init_data.flags.seamless_boot_edp_requested = true;
-		init_data.flags.allow_seamless_boot_optimization = true;
-		drm_dbg(adev->dm.ddev, "Seamless boot requested\n");
+	/* First remove from context all streams */
+	for (i = 0; i < context->stream_count; i++) {
+		struct dc_stream_state *stream = context->streams[i];
+
+		del_streams[del_streams_count++] = stream;
 	}
 
-	init_data.flags.enable_mipi_converter_optimization = true;
+	/* Remove all planes for removed streams and then remove the streams */
+	for (i = 0; i < del_streams_count; i++) {
+		enum dc_status res;
 
-	init_data.dcn_reg_offsets = adev->reg_offset[DCE_HWIP][0];
-	init_data.nbio_reg_offsets = adev->reg_offset[NBIO_HWIP][0];
-	init_data.clk_reg_offsets = adev->reg_offset[CLK_HWIP][0];
+		if (!dc_state_rem_all_planes_for_stream(dc, del_streams[i], context))
+			return DC_FAIL_DETACH_SURFACES;
 
-	if (amdgpu_dc_debug_mask & DC_DISABLE_IPS)
-		init_data.flags.disable_ips = DMUB_IPS_DISABLE_ALL;
-	else if (amdgpu_dc_debug_mask & DC_DISABLE_IPS_DYNAMIC)
-		init_data.flags.disable_ips = DMUB_IPS_DISABLE_DYNAMIC;
-	else if (amdgpu_dc_debug_mask & DC_DISABLE_IPS2_DYNAMIC)
-		init_data.flags.disable_ips = DMUB_IPS_RCG_IN_ACTIVE_IPS2_IN_OFF;
-	else if (amdgpu_dc_debug_mask & DC_FORCE_IPS_ENABLE)
-		init_data.flags.disable_ips = DMUB_IPS_ENABLE;
-	else
-		init_data.flags.disable_ips = dm_get_default_ips_mode(adev);
+		res = dc_state_remove_stream(dc, context, del_streams[i]);
+		if (res != DC_OK)
+			return res;
+	}
 
-	init_data.flags.disable_ips_in_vpb = 0;
+	params.streams = context->streams;
+	params.stream_count = context->stream_count;
 
-	/* DCN35 and above supports dynamic DTBCLK switch */
-	if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 5, 0))
-		init_data.flags.allow_0_dtb_clk = true;
+	return dc_commit_streams(dc, &params);
+}
 
-	/* Enable DWB for tested platforms only */
-	if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0))
-		init_data.num_virtual_links = 1;
+static int dm_cache_state(struct amdgpu_device *adev)
+{
+	int r;
 
-	/* DCN42 and above dpia switch to unified link training path */
-	if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(4, 2, 0)) {
-		init_data.flags.consolidated_dpia_dp_lt = true;
-		init_data.flags.enable_dpia_pre_training = true;
-		init_data.flags.unify_link_enc_assignment = true;
-		init_data.flags.usb4_bw_alloc_support = true;
+	adev->dm.cached_state = drm_atomic_helper_suspend(adev_to_drm(adev));
+	if (IS_ERR(adev->dm.cached_state)) {
+		r = PTR_ERR(adev->dm.cached_state);
+		adev->dm.cached_state = NULL;
 	}
-	retrieve_dmi_info(&adev->dm);
-	if (adev->dm.edp0_on_dp1_quirk)
-		init_data.flags.support_edp0_on_dp1 = true;
 
-	if (adev->dm.bb_from_dmub)
-		init_data.bb_from_dmub = adev->dm.bb_from_dmub;
-	else
-		init_data.bb_from_dmub = NULL;
+	return adev->dm.cached_state ? 0 : r;
+}
 
-	/* Display Core create. */
-	adev->dm.dc = dc_create(&init_data);
+static void dm_destroy_cached_state(struct amdgpu_device *adev)
+{
+	struct amdgpu_display_manager *dm = &adev->dm;
+	struct drm_device *ddev = adev_to_drm(adev);
+	struct dm_plane_state *dm_new_plane_state;
+	struct drm_plane_state *new_plane_state;
+	struct dm_crtc_state *dm_new_crtc_state;
+	struct drm_crtc_state *new_crtc_state;
+	struct drm_plane *plane;
+	struct drm_crtc *crtc;
+	int i;
 
-	if (adev->dm.dc) {
-		drm_info(adev_to_drm(adev), "Display Core v%s initialized on %s\n", DC_VER,
-			 dce_version_to_string(adev->dm.dc->ctx->dce_version));
-	} else {
-		drm_info(adev_to_drm(adev), "Display Core failed to initialize with v%s!\n", DC_VER);
-		goto error;
+	if (!dm->cached_state)
+		return;
+
+	/* Force mode set in atomic commit */
+	for_each_new_crtc_in_state(dm->cached_state, crtc, new_crtc_state, i) {
+		new_crtc_state->active_changed = true;
+		dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
+		reset_freesync_config_for_crtc(dm_new_crtc_state);
 	}
 
-	if (amdgpu_dc_debug_mask & DC_DISABLE_PIPE_SPLIT) {
-		adev->dm.dc->debug.force_single_disp_pipe_split = false;
-		adev->dm.dc->debug.pipe_split_policy = MPC_SPLIT_AVOID;
+	/*
+	 * atomic_check is expected to create the dc states. We need to release
+	 * them here, since they were duplicated as part of the suspend
+	 * procedure.
+	 */
+	for_each_new_crtc_in_state(dm->cached_state, crtc, new_crtc_state, i) {
+		dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
+		if (dm_new_crtc_state->stream) {
+			WARN_ON(kref_read(&dm_new_crtc_state->stream->refcount) > 1);
+			dc_stream_release(dm_new_crtc_state->stream);
+			dm_new_crtc_state->stream = NULL;
+		}
+		dm_new_crtc_state->base.color_mgmt_changed = true;
 	}
 
-	if (adev->asic_type != CHIP_CARRIZO && adev->asic_type != CHIP_STONEY)
-		adev->dm.dc->debug.disable_stutter = amdgpu_pp_feature_mask & PP_STUTTER_MODE ? false : true;
-	if (dm_should_disable_stutter(adev->pdev))
-		adev->dm.dc->debug.disable_stutter = true;
+	for_each_new_plane_in_state(dm->cached_state, plane, new_plane_state, i) {
+		dm_new_plane_state = to_dm_plane_state(new_plane_state);
+		if (dm_new_plane_state->dc_state) {
+			WARN_ON(kref_read(&dm_new_plane_state->dc_state->refcount) > 1);
+			dc_plane_state_release(dm_new_plane_state->dc_state);
+			dm_new_plane_state->dc_state = NULL;
+		}
+	}
 
-	if (amdgpu_dc_debug_mask & DC_DISABLE_STUTTER)
-		adev->dm.dc->debug.disable_stutter = true;
+	drm_atomic_helper_resume(ddev, dm->cached_state);
 
-	if (amdgpu_dc_debug_mask & DC_DISABLE_DSC)
-		adev->dm.dc->debug.disable_dsc = true;
+	dm->cached_state = NULL;
+}
 
-	if (amdgpu_dc_debug_mask & DC_DISABLE_CLOCK_GATING)
-		adev->dm.dc->debug.disable_clock_gate = true;
+static int dm_suspend(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	struct amdgpu_display_manager *dm = &adev->dm;
 
-	if (amdgpu_dc_debug_mask & DC_FORCE_SUBVP_MCLK_SWITCH)
-		adev->dm.dc->debug.force_subvp_mclk_switch = true;
+	if (amdgpu_in_reset(adev)) {
+		enum dc_status res;
 
-	if (amdgpu_dc_debug_mask & DC_DISABLE_SUBVP_FAMS) {
-		adev->dm.dc->debug.force_disable_subvp = true;
-		adev->dm.dc->debug.fams2_config.bits.enable = false;
-	}
+		/* Quiesce ISM workers before taking dc_lock (workers take
+		 * dc_lock themselves; syncing under it would deadlock).
+		 */
+		amdgpu_dm_ism_disable(dm);
 
-	if (amdgpu_dc_debug_mask & DC_ENABLE_DML2) {
-		adev->dm.dc->debug.using_dml2 = true;
-		adev->dm.dc->debug.using_dml21 = true;
-	}
+		mutex_lock(&dm->dc_lock);
 
-	if (amdgpu_dc_debug_mask & DC_HDCP_LC_FORCE_FW_ENABLE)
-		adev->dm.dc->debug.hdcp_lc_force_fw_enable = true;
+		amdgpu_dm_ism_force_full_power(dm);
+		dc_allow_idle_optimizations(adev->dm.dc, false);
 
-	if (amdgpu_dc_debug_mask & DC_HDCP_LC_ENABLE_SW_FALLBACK)
-		adev->dm.dc->debug.hdcp_lc_enable_sw_fallback = true;
+		dm->cached_dc_state = dc_state_create_copy(dm->dc->current_state);
 
-	if (amdgpu_dc_debug_mask & DC_SKIP_DETECTION_LT)
-		adev->dm.dc->debug.skip_detection_link_training = true;
+		if (dm->cached_dc_state)
+			dm_gpureset_toggle_interrupts(adev, dm->cached_dc_state, false);
 
-	adev->dm.dc->debug.visual_confirm = amdgpu_dc_visual_confirm;
+		res = amdgpu_dm_commit_zero_streams(dm->dc);
+		if (res != DC_OK) {
+			drm_err(adev_to_drm(adev), "Failed to commit zero streams: %d\n", res);
+			return -EINVAL;
+		}
 
-	/* TODO: Remove after DP2 receiver gets proper support of Cable ID feature */
-	adev->dm.dc->debug.ignore_cable_id = true;
+		amdgpu_dm_irq_suspend(adev);
 
-	if (adev->dm.dc->caps.dp_hdmi21_pcon_support)
-		drm_info(adev_to_drm(adev), "DP-HDMI FRL PCON supported\n");
+		amdgpu_dm_hpd_rx_irq_work_suspend(dm);
 
-	r = dm_dmub_hw_init(adev);
-	if (r) {
-		drm_err(adev_to_drm(adev), "DMUB interface failed to initialize: status=%d\n", r);
-		goto error;
+		return 0;
 	}
 
-	dc_hardware_init(adev->dm.dc);
+	if (!adev->dm.cached_state) {
+		int r = dm_cache_state(adev);
 
-	adev->dm.hpd_rx_offload_wq = hpd_rx_irq_create_workqueue(adev);
-	if (!adev->dm.hpd_rx_offload_wq) {
-		drm_err(adev_to_drm(adev), "failed to create hpd rx offload workqueue.\n");
-		goto error;
+		if (r)
+			return r;
 	}
 
-	if ((adev->flags & AMD_IS_APU) && (adev->asic_type >= CHIP_CARRIZO)) {
-		struct dc_phy_addr_space_config pa_config;
-
-		mmhub_read_system_context(adev, &pa_config);
-
-		// Call the DC init_memory func
-		dc_setup_system_context(adev->dm.dc, &pa_config);
-	}
+	amdgpu_dm_s3_handle_hdmi_cec(adev_to_drm(adev), true);
 
-	adev->dm.freesync_module = mod_freesync_create(adev->dm.dc);
-	if (!adev->dm.freesync_module) {
-		drm_err(adev_to_drm(adev),
-		"failed to initialize freesync_module.\n");
-	} else
-		drm_dbg_driver(adev_to_drm(adev), "freesync_module init done %p.\n",
-				adev->dm.freesync_module);
+	s3_handle_mst(adev_to_drm(adev), true);
 
-	amdgpu_dm_init_color_mod();
+	amdgpu_dm_irq_suspend(adev);
 
-	if (adev->dm.dc->caps.max_links > 0) {
-		adev->dm.vblank_control_workqueue =
-			create_singlethread_workqueue("dm_vblank_control_workqueue");
-		if (!adev->dm.vblank_control_workqueue)
-			drm_err(adev_to_drm(adev), "failed to initialize vblank_workqueue.\n");
-	}
+	/*
+	 * Quiesce ISM workers before taking dc_lock (workers take dc_lock
+	 * themselves; syncing under it would deadlock).
+	 */
+	amdgpu_dm_ism_disable(dm);
+	scoped_guard(mutex, &dm->dc_lock)
+		amdgpu_dm_ism_force_full_power(dm);
 
-	if (adev->dm.dc->caps.ips_support &&
-	    adev->dm.dc->config.disable_ips != DMUB_IPS_DISABLE_ALL)
-		adev->dm.idle_workqueue = idle_create_workqueue(adev);
+	amdgpu_dm_hpd_rx_irq_work_suspend(dm);
 
-	if (adev->dm.dc->caps.max_links > 0 && adev->family >= AMDGPU_FAMILY_RV) {
-		adev->dm.hdcp_workqueue = hdcp_create_workqueue(adev, &init_params.cp_psp, adev->dm.dc);
+	dc_set_power_state(dm->dc, DC_ACPI_CM_POWER_STATE_D3);
 
-		if (!adev->dm.hdcp_workqueue)
-			drm_err(adev_to_drm(adev), "failed to initialize hdcp_workqueue.\n");
-		else
-			drm_dbg_driver(adev_to_drm(adev),
-				       "hdcp_workqueue init done %p.\n",
-				       adev->dm.hdcp_workqueue);
+	if (dm->dc->caps.ips_support && adev->in_s0ix)
+		dc_allow_idle_optimizations(dm->dc, true);
 
-		dc_init_callbacks(adev->dm.dc, &init_params);
-	}
-	if (adev->dm.dc->caps.max_links > 0) {
-		adev->dm.hdmi_frl_status_polling_wq =
-			create_singlethread_workqueue("hdmi_frl_status_polling_workqueue");
-		if (!adev->dm.hdmi_frl_status_polling_wq)
-			drm_err(adev_to_drm(adev), "failed to initialize hdmi_frl_status_polling_workqueue\n");
-		adev->dm.hdmi_frl_status_polling_delay_ms = 200;
-		INIT_DELAYED_WORK(&adev->dm.hdmi_frl_status_polling_work, hdmi_frl_status_polling_work);
-	}
-	if (dc_is_dmub_outbox_supported(adev->dm.dc)) {
-		init_completion(&adev->dm.dmub_aux_transfer_done);
-		adev->dm.dmub_notify = kzalloc_obj(struct dmub_notification);
-		if (!adev->dm.dmub_notify) {
-			drm_info(adev_to_drm(adev), "fail to allocate adev->dm.dmub_notify");
-			goto error;
-		}
+	dc_dmub_srv_set_power_state(dm->dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D3);
 
-		adev->dm.delayed_hpd_wq = create_singlethread_workqueue("amdgpu_dm_hpd_wq");
-		if (!adev->dm.delayed_hpd_wq) {
-			drm_err(adev_to_drm(adev), "failed to create hpd offload workqueue.\n");
-			goto error;
-		}
+	return 0;
+}
 
-		amdgpu_dm_outbox_init(adev);
-		if (!register_dmub_notify_callback(adev, DMUB_NOTIFICATION_AUX_REPLY,
-			dmub_aux_setconfig_callback, false)) {
-			drm_err(adev_to_drm(adev), "fail to register dmub aux callback");
-			goto error;
-		}
+void amdgpu_dm_emulated_link_detect(struct dc_link *link)
+{
+	struct dc_sink_init_data sink_init_data = { 0 };
+	struct display_sink_capability sink_caps = { 0 };
+	enum dc_edid_status edid_status;
+	struct dc_context *dc_ctx = link->ctx;
+	struct drm_device *dev = adev_to_drm(dc_ctx->driver_context);
+	struct dc_sink *sink = NULL;
+	struct dc_sink *prev_sink = NULL;
 
-		for (size_t i = 0; i < ARRAY_SIZE(adev->dm.fused_io); i++)
-			init_completion(&adev->dm.fused_io[i].replied);
+	link->type = dc_connection_none;
+	prev_sink = link->local_sink;
 
-		if (!register_dmub_notify_callback(adev, DMUB_NOTIFICATION_FUSED_IO,
-			dmub_aux_fused_io_callback, false)) {
-			drm_err(adev_to_drm(adev), "fail to register dmub fused io callback");
-			goto error;
-		}
-		/* Enable outbox notification only after IRQ handlers are registered and DMUB is alive.
-		 * It is expected that DMUB will resend any pending notifications at this point. Note
-		 * that hpd and hpd_irq handler registration are deferred to register_hpd_handlers() to
-		 * align legacy interface initialization sequence. Connection status will be proactivly
-		 * detected once in the amdgpu_dm_initialize_drm_device.
-		 */
-		dc_enable_dmub_outbox(adev->dm.dc);
+	if (prev_sink)
+		dc_sink_release(prev_sink);
 
-		/* DPIA trace goes to dmesg logs only if outbox is enabled */
-		if (amdgpu_dc_debug_mask & DC_ENABLE_DPIA_TRACE)
-			dc_dmub_srv_enable_dpia_trace(adev->dm.dc);
+	switch (link->connector_signal) {
+	case SIGNAL_TYPE_HDMI_TYPE_A: {
+		sink_caps.transaction_type = DDC_TRANSACTION_TYPE_I2C;
+		sink_caps.signal = SIGNAL_TYPE_HDMI_TYPE_A;
+		break;
 	}
 
-	if (amdgpu_dm_initialize_drm_device(adev)) {
-		drm_err(adev_to_drm(adev),
-		"failed to initialize sw for display support.\n");
-		goto error;
+	case SIGNAL_TYPE_DVI_SINGLE_LINK: {
+		sink_caps.transaction_type = DDC_TRANSACTION_TYPE_I2C;
+		sink_caps.signal = SIGNAL_TYPE_DVI_SINGLE_LINK;
+		break;
 	}
 
-	if (amdgpu_dm_init_power_module(&adev->dm))
-		goto error;
+	case SIGNAL_TYPE_DVI_DUAL_LINK: {
+		sink_caps.transaction_type = DDC_TRANSACTION_TYPE_I2C;
+		sink_caps.signal = SIGNAL_TYPE_DVI_DUAL_LINK;
+		break;
+	}
 
-	/* create fake encoders for MST */
-	dm_dp_create_fake_mst_encoders(adev);
+	case SIGNAL_TYPE_LVDS: {
+		sink_caps.transaction_type = DDC_TRANSACTION_TYPE_I2C;
+		sink_caps.signal = SIGNAL_TYPE_LVDS;
+		break;
+	}
 
-	/* TODO: Add_display_info? */
+	case SIGNAL_TYPE_EDP: {
+		sink_caps.transaction_type =
+			DDC_TRANSACTION_TYPE_I2C_OVER_AUX;
+		sink_caps.signal = SIGNAL_TYPE_EDP;
+		break;
+	}
 
-	/* TODO use dynamic cursor width */
-	adev_to_drm(adev)->mode_config.cursor_width = adev->dm.dc->caps.max_cursor_size;
-	adev_to_drm(adev)->mode_config.cursor_height = adev->dm.dc->caps.max_cursor_size;
+	case SIGNAL_TYPE_DISPLAY_PORT: {
+		sink_caps.transaction_type =
+			DDC_TRANSACTION_TYPE_I2C_OVER_AUX;
+		sink_caps.signal = SIGNAL_TYPE_VIRTUAL;
+		break;
+	}
 
-	if (drm_vblank_init(adev_to_drm(adev), adev->dm.display_indexes_num)) {
-		drm_err(adev_to_drm(adev),
-		"failed to initialize vblank for display support.\n");
-		goto error;
+	default:
+		drm_err(dev, "Invalid connector type! signal:%d\n",
+			link->connector_signal);
+		return;
 	}
 
-#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
-	amdgpu_dm_crtc_secure_display_create_contexts(adev);
-	if (!adev->dm.secure_display_ctx.crtc_ctx)
-		drm_err(adev_to_drm(adev), "failed to initialize secure display contexts.\n");
+	sink_init_data.link = link;
+	sink_init_data.sink_signal = sink_caps.signal;
 
-	if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(4, 0, 1))
-		adev->dm.secure_display_ctx.support_mul_roi = true;
+	sink = dc_sink_create(&sink_init_data);
+	if (!sink) {
+		drm_err(dev, "Failed to create sink!\n");
+		return;
+	}
 
-#endif
+	/* dc_sink_create returns a new reference */
+	link->local_sink = sink;
 
-	drm_dbg_driver(adev_to_drm(adev), "KMS initialized.\n");
+	edid_status = dm_helpers_read_local_edid(
+			link->ctx,
+			link,
+			sink);
 
-	return 0;
-error:
-	amdgpu_dm_fini(adev);
+	if (edid_status != EDID_OK)
+		drm_err(dev, "Failed to read EDID\n");
 
-	return -EINVAL;
 }
 
-static int amdgpu_dm_early_fini(struct amdgpu_ip_block *ip_block)
+static void dm_gpureset_commit_state(struct dc_state *dc_state,
+				     struct amdgpu_display_manager *dm)
 {
-	struct amdgpu_device *adev = ip_block->adev;
-
-	amdgpu_dm_audio_fini(adev);
-
-	return 0;
-}
+	struct {
+		struct dc_surface_update surface_updates[MAX_SURFACES];
+		struct dc_plane_info plane_infos[MAX_SURFACES];
+		struct dc_scaling_info scaling_infos[MAX_SURFACES];
+		struct dc_flip_addrs flip_addrs[MAX_SURFACES];
+		struct dc_stream_update stream_update;
+	} *bundle __free(kfree);
+	int k, m;
 
-static void amdgpu_dm_fini(struct amdgpu_device *adev)
-{
-	int i;
+	bundle = kzalloc_obj(*bundle);
 
-	if (adev->dm.vblank_control_workqueue) {
-		destroy_workqueue(adev->dm.vblank_control_workqueue);
-		adev->dm.vblank_control_workqueue = NULL;
+	if (!bundle) {
+		drm_err(dm->ddev, "Failed to allocate update bundle\n");
+		return;
 	}
 
-	if (adev->dm.idle_workqueue) {
-		if (adev->dm.idle_workqueue->running) {
-			adev->dm.idle_workqueue->enable = false;
-			flush_work(&adev->dm.idle_workqueue->work);
+	for (k = 0; k < dc_state->stream_count; k++) {
+		bundle->stream_update.stream = dc_state->streams[k];
+
+		for (m = 0; m < dc_state->stream_status[k].plane_count; m++) {
+			bundle->surface_updates[m].surface =
+				dc_state->stream_status[k].plane_states[m];
+			bundle->surface_updates[m].surface->force_full_update =
+				true;
 		}
 
-		kfree(adev->dm.idle_workqueue);
-		adev->dm.idle_workqueue = NULL;
+		update_planes_and_stream_adapter(dm->dc,
+					 UPDATE_TYPE_FULL,
+					 dc_state->stream_status[k].plane_count,
+					 dc_state->streams[k],
+					 &bundle->stream_update,
+					 bundle->surface_updates);
 	}
+}
 
-	/*
-	 * Disable ISM before dc_destroy() invalidates dm->dc.
-	 *
-	 * Quiesce workers first without dc_lock (they take dc_lock
-	 * themselves, so syncing under it would deadlock), then drive the
-	 * FSM back to FULL_POWER_RUNNING under dc_lock.
-	 */
-	amdgpu_dm_ism_disable(&adev->dm);
-	scoped_guard(mutex, &adev->dm.dc_lock)
-		amdgpu_dm_ism_force_full_power(&adev->dm);
+void amdgpu_dm_apply_delay_after_dpcd_poweroff(struct amdgpu_device *adev,
+											   struct dc_sink *sink)
+{
+	struct dc_panel_patch *ppatch = NULL;
 
-	amdgpu_dm_destroy_drm_device(&adev->dm);
+	if (!sink)
+		return;
 
-#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
-	if (adev->dm.secure_display_ctx.crtc_ctx) {
-		for (i = 0; i < adev->mode_info.num_crtc; i++) {
-			if (adev->dm.secure_display_ctx.crtc_ctx[i].crtc) {
-				flush_work(&adev->dm.secure_display_ctx.crtc_ctx[i].notify_ta_work);
-				flush_work(&adev->dm.secure_display_ctx.crtc_ctx[i].forward_roi_work);
-			}
-		}
-		kfree(adev->dm.secure_display_ctx.crtc_ctx);
-		adev->dm.secure_display_ctx.crtc_ctx = NULL;
-	}
-#endif
-	if (adev->dm.hdcp_workqueue) {
-		hdcp_destroy(&adev->dev->kobj, adev->dm.hdcp_workqueue);
-		adev->dm.hdcp_workqueue = NULL;
-	}
-
-	if (adev->dm.dc) {
-		dc_deinit_callbacks(adev->dm.dc);
-		dc_dmub_srv_destroy(&adev->dm.dc->ctx->dmub_srv);
-		if (dc_enable_dmub_notifications(adev->dm.dc)) {
-			kfree(adev->dm.dmub_notify);
-			adev->dm.dmub_notify = NULL;
-			destroy_workqueue(adev->dm.delayed_hpd_wq);
-			adev->dm.delayed_hpd_wq = NULL;
-		}
-	}
-
-	if (adev->dm.dmub_bo)
-		amdgpu_bo_free_kernel(&adev->dm.dmub_bo,
-				      &adev->dm.dmub_bo_gpu_addr,
-				      &adev->dm.dmub_bo_cpu_addr);
-
-	if (adev->dm.boot_time_crc_info.bo_ptr)
-		amdgpu_bo_free_kernel(&adev->dm.boot_time_crc_info.bo_ptr,
-					&adev->dm.boot_time_crc_info.gpu_addr,
-					&adev->dm.boot_time_crc_info.cpu_addr);
-
-	if (adev->dm.hpd_rx_offload_wq && adev->dm.dc) {
-		for (i = 0; i < adev->dm.dc->caps.max_links; i++) {
-			if (adev->dm.hpd_rx_offload_wq[i].wq) {
-				destroy_workqueue(adev->dm.hpd_rx_offload_wq[i].wq);
-				adev->dm.hpd_rx_offload_wq[i].wq = NULL;
-			}
-		}
-
-		kfree(adev->dm.hpd_rx_offload_wq);
-		adev->dm.hpd_rx_offload_wq = NULL;
-	}
-
-	/* DC Destroy TODO: Replace destroy DAL */
-	if (adev->dm.dc)
-		dc_destroy(&adev->dm.dc);
-	/*
-	 * TODO: pageflip, vlank interrupt
-	 *
-	 * amdgpu_dm_irq_fini(adev);
-	 */
-
-	if (adev->dm.cgs_device) {
-		amdgpu_cgs_destroy_device(adev->dm.cgs_device);
-		adev->dm.cgs_device = NULL;
-	}
-	if (adev->dm.freesync_module) {
-		mod_freesync_destroy(adev->dm.freesync_module);
-		adev->dm.freesync_module = NULL;
-	}
-
-	if (adev->dm.power_module) {
-		mod_power_destroy(adev->dm.power_module);
-		adev->dm.power_module = NULL;
-	}
-	mutex_destroy(&adev->dm.audio_lock);
-	mutex_destroy(&adev->dm.dc_lock);
-	mutex_destroy(&adev->dm.dpia_aux_lock);
-}
-
-static int load_dmcu_fw(struct amdgpu_device *adev)
-{
-	const char *fw_name_dmcu = NULL;
-	int r;
-	const struct dmcu_firmware_header_v1_0 *hdr;
-
-	switch (adev->asic_type) {
-#if defined(CONFIG_DRM_AMD_DC_SI)
-	case CHIP_TAHITI:
-	case CHIP_PITCAIRN:
-	case CHIP_VERDE:
-	case CHIP_OLAND:
-#endif
-	case CHIP_BONAIRE:
-	case CHIP_HAWAII:
-	case CHIP_KAVERI:
-	case CHIP_KABINI:
-	case CHIP_MULLINS:
-	case CHIP_TONGA:
-	case CHIP_FIJI:
-	case CHIP_CARRIZO:
-	case CHIP_STONEY:
-	case CHIP_POLARIS11:
-	case CHIP_POLARIS10:
-	case CHIP_POLARIS12:
-	case CHIP_VEGAM:
-	case CHIP_VEGA10:
-	case CHIP_VEGA12:
-	case CHIP_VEGA20:
-		return 0;
-	case CHIP_NAVI12:
-		fw_name_dmcu = FIRMWARE_NAVI12_DMCU;
-		break;
-	case CHIP_RAVEN:
-		if (ASICREV_IS_PICASSO(adev->external_rev_id))
-			fw_name_dmcu = FIRMWARE_RAVEN_DMCU;
-		else if (ASICREV_IS_RAVEN2(adev->external_rev_id))
-			fw_name_dmcu = FIRMWARE_RAVEN_DMCU;
-		else
-			return 0;
-		break;
-	default:
-		switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
-		case IP_VERSION(2, 0, 2):
-		case IP_VERSION(2, 0, 3):
-		case IP_VERSION(2, 0, 0):
-		case IP_VERSION(2, 1, 0):
-		case IP_VERSION(3, 0, 0):
-		case IP_VERSION(3, 0, 2):
-		case IP_VERSION(3, 0, 3):
-		case IP_VERSION(3, 0, 1):
-		case IP_VERSION(3, 1, 2):
-		case IP_VERSION(3, 1, 3):
-		case IP_VERSION(3, 1, 4):
-		case IP_VERSION(3, 1, 5):
-		case IP_VERSION(3, 1, 6):
-		case IP_VERSION(3, 2, 0):
-		case IP_VERSION(3, 2, 1):
-		case IP_VERSION(3, 5, 0):
-		case IP_VERSION(3, 5, 1):
-		case IP_VERSION(3, 6, 0):
-		case IP_VERSION(4, 0, 1):
-		case IP_VERSION(4, 2, 0):
-		case IP_VERSION(4, 2, 1):
-			return 0;
-		default:
-			break;
-		}
-		drm_err(adev_to_drm(adev), "Unsupported ASIC type: 0x%X\n", adev->asic_type);
-		return -EINVAL;
-	}
-
-	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
-		drm_dbg_kms(adev_to_drm(adev), "dm: DMCU firmware not supported on direct or SMU loading\n");
-		return 0;
-	}
-
-	r = amdgpu_ucode_request(adev, &adev->dm.fw_dmcu, AMDGPU_UCODE_REQUIRED,
-				 "%s", fw_name_dmcu);
-	if (r == -ENODEV) {
-		/* DMCU firmware is not necessary, so don't raise a fuss if it's missing */
-		drm_dbg_kms(adev_to_drm(adev), "dm: DMCU firmware not found\n");
-		adev->dm.fw_dmcu = NULL;
-		return 0;
-	}
-	if (r) {
-		drm_err(adev_to_drm(adev), "amdgpu_dm: Can't validate firmware \"%s\"\n",
-			fw_name_dmcu);
-		amdgpu_ucode_release(&adev->dm.fw_dmcu);
-		return r;
-	}
-
-	hdr = (const struct dmcu_firmware_header_v1_0 *)adev->dm.fw_dmcu->data;
-	adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_ERAM].ucode_id = AMDGPU_UCODE_ID_DMCU_ERAM;
-	adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_ERAM].fw = adev->dm.fw_dmcu;
-	adev->firmware.fw_size +=
-		ALIGN(le32_to_cpu(hdr->header.ucode_size_bytes) - le32_to_cpu(hdr->intv_size_bytes), PAGE_SIZE);
-
-	adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_INTV].ucode_id = AMDGPU_UCODE_ID_DMCU_INTV;
-	adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_INTV].fw = adev->dm.fw_dmcu;
-	adev->firmware.fw_size +=
-		ALIGN(le32_to_cpu(hdr->intv_size_bytes), PAGE_SIZE);
-
-	adev->dm.dmcu_fw_version = le32_to_cpu(hdr->header.ucode_version);
-
-	drm_dbg_kms(adev_to_drm(adev), "PSP loading DMCU firmware\n");
-
-	return 0;
-}
-
-static uint32_t amdgpu_dm_dmub_reg_read(void *ctx, uint32_t address)
-{
-	struct amdgpu_device *adev = ctx;
-
-	return dm_read_reg(adev->dm.dc->ctx, address);
-}
-
-static void amdgpu_dm_dmub_reg_write(void *ctx, uint32_t address,
-				     uint32_t value)
-{
-	struct amdgpu_device *adev = ctx;
-
-	return dm_write_reg(adev->dm.dc->ctx, address, value);
-}
-
-static int dm_dmub_sw_init(struct amdgpu_device *adev)
-{
-	struct dmub_srv_create_params create_params;
-	struct dmub_srv_fw_meta_info_params fw_meta_info_params;
-	struct dmub_srv_region_params region_params;
-	struct dmub_srv_region_info region_info;
-	struct dmub_srv_memory_params memory_params;
-	struct dmub_fw_meta_info fw_info;
-	struct dmub_srv_fb_info *fb_info;
-	struct dmub_srv *dmub_srv;
-	const struct dmcub_firmware_header_v1_0 *hdr;
-	enum dmub_asic dmub_asic;
-	enum dmub_status status;
-	static enum dmub_window_memory_type window_memory_type[DMUB_WINDOW_TOTAL] = {
-		DMUB_WINDOW_MEMORY_TYPE_FB,		//DMUB_WINDOW_0_INST_CONST
-		DMUB_WINDOW_MEMORY_TYPE_FB,		//DMUB_WINDOW_1_STACK
-		DMUB_WINDOW_MEMORY_TYPE_FB,		//DMUB_WINDOW_2_BSS_DATA
-		DMUB_WINDOW_MEMORY_TYPE_FB,		//DMUB_WINDOW_3_VBIOS
-		DMUB_WINDOW_MEMORY_TYPE_FB,		//DMUB_WINDOW_4_MAILBOX
-		DMUB_WINDOW_MEMORY_TYPE_FB,		//DMUB_WINDOW_5_TRACEBUFF
-		DMUB_WINDOW_MEMORY_TYPE_FB,		//DMUB_WINDOW_6_FW_STATE
-		DMUB_WINDOW_MEMORY_TYPE_FB,		//DMUB_WINDOW_7_SCRATCH_MEM
-		DMUB_WINDOW_MEMORY_TYPE_FB,		//DMUB_WINDOW_IB_MEM
-		DMUB_WINDOW_MEMORY_TYPE_FB,		//DMUB_WINDOW_SHARED_STATE
-		DMUB_WINDOW_MEMORY_TYPE_FB,		//DMUB_WINDOW_LSDMA_BUFFER
-		DMUB_WINDOW_MEMORY_TYPE_FB,		//DMUB_WINDOW_CURSOR_OFFLOAD
-	};
-	int r;
-
-	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
-	case IP_VERSION(2, 1, 0):
-		dmub_asic = DMUB_ASIC_DCN21;
-		break;
-	case IP_VERSION(3, 0, 0):
-		dmub_asic = DMUB_ASIC_DCN30;
-		break;
-	case IP_VERSION(3, 0, 1):
-		dmub_asic = DMUB_ASIC_DCN301;
-		break;
-	case IP_VERSION(3, 0, 2):
-		dmub_asic = DMUB_ASIC_DCN302;
-		break;
-	case IP_VERSION(3, 0, 3):
-		dmub_asic = DMUB_ASIC_DCN303;
-		break;
-	case IP_VERSION(3, 1, 2):
-	case IP_VERSION(3, 1, 3):
-		dmub_asic = (adev->external_rev_id == YELLOW_CARP_B0) ? DMUB_ASIC_DCN31B : DMUB_ASIC_DCN31;
-		break;
-	case IP_VERSION(3, 1, 4):
-		dmub_asic = DMUB_ASIC_DCN314;
-		break;
-	case IP_VERSION(3, 1, 5):
-		dmub_asic = DMUB_ASIC_DCN315;
-		break;
-	case IP_VERSION(3, 1, 6):
-		dmub_asic = DMUB_ASIC_DCN316;
-		break;
-	case IP_VERSION(3, 2, 0):
-		dmub_asic = DMUB_ASIC_DCN32;
-		break;
-	case IP_VERSION(3, 2, 1):
-		dmub_asic = DMUB_ASIC_DCN321;
-		break;
-	case IP_VERSION(3, 5, 0):
-	case IP_VERSION(3, 5, 1):
-		dmub_asic = DMUB_ASIC_DCN35;
-		break;
-	case IP_VERSION(3, 6, 0):
-		dmub_asic = DMUB_ASIC_DCN36;
-		break;
-	case IP_VERSION(4, 0, 1):
-		dmub_asic = DMUB_ASIC_DCN401;
-		break;
-	case IP_VERSION(4, 2, 0):
-		dmub_asic = DMUB_ASIC_DCN42;
-		break;
-	case IP_VERSION(4, 2, 1):
-		dmub_asic = DMUB_ASIC_DCN42B;
-		break;
-	default:
-		/* ASIC doesn't support DMUB. */
-		return 0;
-	}
-
-	hdr = (const struct dmcub_firmware_header_v1_0 *)adev->dm.dmub_fw->data;
-	adev->dm.dmcub_fw_version = le32_to_cpu(hdr->header.ucode_version);
-
-	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
-		adev->firmware.ucode[AMDGPU_UCODE_ID_DMCUB].ucode_id =
-			AMDGPU_UCODE_ID_DMCUB;
-		adev->firmware.ucode[AMDGPU_UCODE_ID_DMCUB].fw =
-			adev->dm.dmub_fw;
-		adev->firmware.fw_size +=
-			ALIGN(le32_to_cpu(hdr->inst_const_bytes), PAGE_SIZE);
-
-		drm_info(adev_to_drm(adev), "Loading DMUB firmware via PSP: version=0x%08X\n",
-			 adev->dm.dmcub_fw_version);
-	}
-
-
-	adev->dm.dmub_srv = kzalloc_obj(*adev->dm.dmub_srv);
-	dmub_srv = adev->dm.dmub_srv;
-
-	if (!dmub_srv) {
-		drm_err(adev_to_drm(adev), "Failed to allocate DMUB service!\n");
-		return -ENOMEM;
-	}
-
-	memset(&create_params, 0, sizeof(create_params));
-	create_params.user_ctx = adev;
-	create_params.funcs.reg_read = amdgpu_dm_dmub_reg_read;
-	create_params.funcs.reg_write = amdgpu_dm_dmub_reg_write;
-	create_params.asic = dmub_asic;
-
-	/* Create the DMUB service. */
-	status = dmub_srv_create(dmub_srv, &create_params);
-	if (status != DMUB_STATUS_OK) {
-		drm_err(adev_to_drm(adev), "Error creating DMUB service: %d\n", status);
-		return -EINVAL;
-	}
-
-	/* Extract the FW meta info. */
-	memset(&fw_meta_info_params, 0, sizeof(fw_meta_info_params));
-
-	fw_meta_info_params.inst_const_size = le32_to_cpu(hdr->inst_const_bytes) -
-					      PSP_HEADER_BYTES_256;
-	fw_meta_info_params.bss_data_size = le32_to_cpu(hdr->bss_data_bytes);
-	fw_meta_info_params.fw_inst_const = adev->dm.dmub_fw->data +
-					    le32_to_cpu(hdr->header.ucode_array_offset_bytes) +
-					    PSP_HEADER_BYTES_256;
-	fw_meta_info_params.fw_bss_data = fw_meta_info_params.bss_data_size ? adev->dm.dmub_fw->data +
-					  le32_to_cpu(hdr->header.ucode_array_offset_bytes) +
-					  le32_to_cpu(hdr->inst_const_bytes) : NULL;
-	fw_meta_info_params.custom_psp_footer_size = 0;
-
-	status = dmub_srv_get_fw_meta_info_from_raw_fw(&fw_meta_info_params, &fw_info);
-	if (status != DMUB_STATUS_OK) {
-		/* Skip returning early, just log the error. */
-		drm_err(adev_to_drm(adev), "Error getting DMUB FW meta info: %d\n", status);
-		// return -EINVAL;
-	}
-
-	/* Calculate the size of all the regions for the DMUB service. */
-	memset(&region_params, 0, sizeof(region_params));
-
-	region_params.inst_const_size = fw_meta_info_params.inst_const_size;
-	region_params.bss_data_size = fw_meta_info_params.bss_data_size;
-	region_params.vbios_size = adev->bios_size;
-	region_params.fw_bss_data = fw_meta_info_params.fw_bss_data;
-	region_params.fw_inst_const = fw_meta_info_params.fw_inst_const;
-	region_params.window_memory_type = window_memory_type;
-	region_params.fw_info = (status == DMUB_STATUS_OK) ? &fw_info : NULL;
-
-	status = dmub_srv_calc_region_info(dmub_srv, &region_params,
-					   &region_info);
-
-	if (status != DMUB_STATUS_OK) {
-		drm_err(adev_to_drm(adev), "Error calculating DMUB region info: %d\n", status);
-		return -EINVAL;
-	}
-
-	/*
-	 * Allocate a framebuffer based on the total size of all the regions.
-	 * TODO: Move this into GART.
-	 */
-	r = amdgpu_bo_create_kernel(adev, region_info.fb_size, PAGE_SIZE,
-				    AMDGPU_GEM_DOMAIN_VRAM |
-				    AMDGPU_GEM_DOMAIN_GTT,
-				    &adev->dm.dmub_bo,
-				    &adev->dm.dmub_bo_gpu_addr,
-				    &adev->dm.dmub_bo_cpu_addr);
-	if (r)
-		return r;
-
-	/* Rebase the regions on the framebuffer address. */
-	memset(&memory_params, 0, sizeof(memory_params));
-	memory_params.cpu_fb_addr = adev->dm.dmub_bo_cpu_addr;
-	memory_params.gpu_fb_addr = adev->dm.dmub_bo_gpu_addr;
-	memory_params.region_info = &region_info;
-	memory_params.window_memory_type = window_memory_type;
-
-	adev->dm.dmub_fb_info = kzalloc_obj(*adev->dm.dmub_fb_info);
-	fb_info = adev->dm.dmub_fb_info;
-
-	if (!fb_info) {
-		drm_err(adev_to_drm(adev),
-			"Failed to allocate framebuffer info for DMUB service!\n");
-		return -ENOMEM;
-	}
-
-	status = dmub_srv_calc_mem_info(dmub_srv, &memory_params, fb_info);
-	if (status != DMUB_STATUS_OK) {
-		drm_err(adev_to_drm(adev), "Error calculating DMUB FB info: %d\n", status);
-		return -EINVAL;
-	}
-
-	adev->dm.bb_from_dmub = dm_dmub_get_vbios_bounding_box(adev);
-	adev->dm.fw_inst_size = fw_meta_info_params.inst_const_size;
-
-	return 0;
-}
-
-static int dm_sw_init(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	int r;
-
-	adev->dm.cgs_device = amdgpu_cgs_create_device(adev);
-
-	if (!adev->dm.cgs_device) {
-		drm_err(adev_to_drm(adev), "failed to create cgs device.\n");
-		return -EINVAL;
-	}
-
-	/* Moved from dm init since we need to use allocations for storing bounding box data */
-	INIT_LIST_HEAD(&adev->dm.da_list);
-
-	r = dm_dmub_sw_init(adev);
-	if (r)
-		return r;
-
-	return load_dmcu_fw(adev);
-}
-
-static int dm_sw_fini(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	struct dal_allocation *da;
-
-	list_for_each_entry(da, &adev->dm.da_list, list) {
-		if (adev->dm.bb_from_dmub == (void *) da->cpu_ptr) {
-			amdgpu_bo_free_kernel(&da->bo, &da->gpu_addr, &da->cpu_ptr);
-			list_del(&da->list);
-			kfree(da);
-			adev->dm.bb_from_dmub = NULL;
-			break;
-		}
-	}
-
-
-	kfree(adev->dm.dmub_fb_info);
-	adev->dm.dmub_fb_info = NULL;
-
-	if (adev->dm.dmub_srv) {
-		dmub_srv_destroy(adev->dm.dmub_srv);
-		kfree(adev->dm.dmub_srv);
-		adev->dm.dmub_srv = NULL;
-	}
-
-	amdgpu_ucode_release(&adev->dm.dmub_fw);
-	amdgpu_ucode_release(&adev->dm.fw_dmcu);
-
-	return 0;
-}
-
-static int detect_mst_link_for_all_connectors(struct drm_device *dev)
-{
-	struct amdgpu_dm_connector *aconnector;
-	struct drm_connector *connector;
-	struct drm_connector_list_iter iter;
-	int ret = 0;
-
-	drm_connector_list_iter_begin(dev, &iter);
-	drm_for_each_connector_iter(connector, &iter) {
-
-		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
-			continue;
-
-		aconnector = to_amdgpu_dm_connector(connector);
-		if (aconnector->dc_link->type == dc_connection_mst_branch &&
-		    aconnector->mst_mgr.aux) {
-			drm_dbg_kms(dev, "DM_MST: starting TM on aconnector: %p [id: %d]\n",
-					 aconnector,
-					 aconnector->base.base.id);
-
-			ret = drm_dp_mst_topology_mgr_set_mst(&aconnector->mst_mgr, true);
-			if (ret < 0) {
-				drm_err(dev, "DM_MST: Failed to start MST\n");
-				aconnector->dc_link->type =
-					dc_connection_single;
-				ret = dm_helpers_dp_mst_stop_top_mgr(aconnector->dc_link->ctx,
-								     aconnector->dc_link);
-				break;
-			}
-		}
-	}
-	drm_connector_list_iter_end(&iter);
-
-	return ret;
-}
-
-static void amdgpu_dm_boot_time_crc_init(struct amdgpu_device *adev)
-{
-	struct dm_boot_time_crc_info *bootcrc_info = NULL;
-	struct dmub_srv *dmub = NULL;
-	union dmub_fw_boot_options option = {0};
-	int ret = 0;
-	const uint32_t fb_size = 3 * 1024 * 1024;	/* 3MB for DCC pattern */
-
-	if (!adev || !adev->dm.dc || !adev->dm.dc->ctx ||
-		!adev->dm.dc->ctx->dmub_srv) {
-		return;
-	}
-
-	dmub = adev->dm.dc->ctx->dmub_srv->dmub;
-	bootcrc_info = &adev->dm.boot_time_crc_info;
-
-	if (!dmub || !dmub->hw_funcs.get_fw_boot_option) {
-		drm_dbg(adev_to_drm(adev), "failed to init boot time crc buffer\n");
-		return;
-	}
-
-	option = dmub->hw_funcs.get_fw_boot_option(dmub);
-
-	/* Return if boot time CRC is not enabled */
-	if (option.bits.bootcrc_en_at_S0i3 == 0)
-		return;
-
-	/* Create a buffer for boot time CRC */
-	ret = amdgpu_bo_create_kernel(adev, fb_size, PAGE_SIZE,
-		AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT,
-		&bootcrc_info->bo_ptr,
-		&bootcrc_info->gpu_addr,
-		&bootcrc_info->cpu_addr);
-
-	if (ret) {
-		drm_dbg(adev_to_drm(adev), "failed to create boot time crc buffer\n");
-	} else {
-		bootcrc_info->size = fb_size;
-
-		drm_dbg(adev_to_drm(adev), "boot time crc buffer created addr 0x%llx, size %u\n",
-			bootcrc_info->gpu_addr, bootcrc_info->size);
-
-		/* Send the buffer info to DMUB */
-		dc_dmub_srv_boot_time_crc_init(adev->dm.dc,
-			bootcrc_info->gpu_addr, bootcrc_info->size);
-	}
-}
-
-static int dm_late_init(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-
-	struct dmcu_iram_parameters params;
-	unsigned int linear_lut[16];
-	int i;
-	struct dmcu *dmcu = NULL;
-
-	dmcu = adev->dm.dc->res_pool->dmcu;
-
-	/* Init the boot time CRC (skip in resume) */
-	if ((adev->in_suspend == 0) &&
-		(amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(3, 6, 0)))
-		amdgpu_dm_boot_time_crc_init(adev);
-
-	for (i = 0; i < 16; i++)
-		linear_lut[i] = 0xFFFF * i / 15;
-
-	params.set = 0;
-	params.backlight_ramping_override = false;
-	params.backlight_ramping_start = 0xCCCC;
-	params.backlight_ramping_reduction = 0xCCCCCCCC;
-	params.backlight_lut_array_size = 16;
-	params.backlight_lut_array = linear_lut;
-
-	/* Min backlight level after ABM reduction,  Don't allow below 1%
-	 * 0xFFFF x 0.01 = 0x28F
-	 */
-	params.min_abm_backlight = 0x28F;
-	/* In the case where abm is implemented on dmcub,
-	 * dmcu object will be null.
-	 * ABM 2.4 and up are implemented on dmcub.
-	 */
-	if (dmcu) {
-		if (!dmcu_load_iram(dmcu, params))
-			return -EINVAL;
-	} else if (adev->dm.dc->ctx->dmub_srv) {
-		struct dc_link *edp_links[MAX_NUM_EDP];
-		int edp_num;
-
-		dc_get_edp_links(adev->dm.dc, edp_links, &edp_num);
-		for (i = 0; i < edp_num; i++) {
-			if (!dmub_init_abm_config(adev->dm.dc->res_pool, params, i))
-				return -EINVAL;
-		}
-	}
-
-	return detect_mst_link_for_all_connectors(adev_to_drm(adev));
-}
-
-static void resume_mst_branch_status(struct drm_dp_mst_topology_mgr *mgr)
-{
-	u8 buf[UUID_SIZE];
-	guid_t guid;
-	int ret;
-
-	mutex_lock(&mgr->lock);
-	if (!mgr->mst_primary)
-		goto out_fail;
-
-	if (drm_dp_read_dpcd_caps(mgr->aux, mgr->dpcd) < 0) {
-		drm_dbg_kms(mgr->dev, "dpcd read failed - undocked during suspend?\n");
-		goto out_fail;
-	}
-
-	ret = drm_dp_dpcd_writeb(mgr->aux, DP_MSTM_CTRL,
-				 DP_MST_EN |
-				 DP_UP_REQ_EN |
-				 DP_UPSTREAM_IS_SRC);
-	if (ret < 0) {
-		drm_dbg_kms(mgr->dev, "mst write failed - undocked during suspend?\n");
-		goto out_fail;
-	}
-
-	/* Some hubs forget their guids after they resume */
-	ret = drm_dp_dpcd_read(mgr->aux, DP_GUID, buf, sizeof(buf));
-	if (ret != sizeof(buf)) {
-		drm_dbg_kms(mgr->dev, "dpcd read failed - undocked during suspend?\n");
-		goto out_fail;
-	}
-
-	import_guid(&guid, buf);
-
-	if (guid_is_null(&guid)) {
-		guid_gen(&guid);
-		export_guid(buf, &guid);
-
-		ret = drm_dp_dpcd_write(mgr->aux, DP_GUID, buf, sizeof(buf));
-
-		if (ret != sizeof(buf)) {
-			drm_dbg_kms(mgr->dev, "check mstb guid failed - undocked during suspend?\n");
-			goto out_fail;
-		}
-	}
-
-	guid_copy(&mgr->mst_primary->guid, &guid);
-
-out_fail:
-	mutex_unlock(&mgr->lock);
-}
-
-void hdmi_cec_unset_edid(struct amdgpu_dm_connector *aconnector)
-{
-	struct cec_notifier *n = aconnector->notifier;
-
-	if (!n)
-		return;
-
-	cec_notifier_phys_addr_invalidate(n);
-}
-
-void hdmi_cec_set_edid(struct amdgpu_dm_connector *aconnector)
-{
-	struct drm_connector *connector = &aconnector->base;
-	struct cec_notifier *n = aconnector->notifier;
-
-	if (!n)
-		return;
-
-	cec_notifier_set_phys_addr(n,
-				   connector->display_info.source_physical_address);
-}
-
-static void s3_handle_hdmi_cec(struct drm_device *ddev, bool suspend)
-{
-	struct amdgpu_dm_connector *aconnector;
-	struct drm_connector *connector;
-	struct drm_connector_list_iter conn_iter;
-
-	drm_connector_list_iter_begin(ddev, &conn_iter);
-	drm_for_each_connector_iter(connector, &conn_iter) {
-		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
-			continue;
-
-		aconnector = to_amdgpu_dm_connector(connector);
-		if (suspend)
-			hdmi_cec_unset_edid(aconnector);
-		else
-			hdmi_cec_set_edid(aconnector);
-	}
-	drm_connector_list_iter_end(&conn_iter);
-}
-
-static void s3_handle_mst(struct drm_device *dev, bool suspend)
-{
-	struct amdgpu_dm_connector *aconnector;
-	struct drm_connector *connector;
-	struct drm_connector_list_iter iter;
-	struct drm_dp_mst_topology_mgr *mgr;
-
-	drm_connector_list_iter_begin(dev, &iter);
-	drm_for_each_connector_iter(connector, &iter) {
-
-		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
-			continue;
-
-		aconnector = to_amdgpu_dm_connector(connector);
-		if (aconnector->dc_link->type != dc_connection_mst_branch ||
-		    aconnector->mst_root)
-			continue;
-
-		mgr = &aconnector->mst_mgr;
-
-		if (suspend) {
-			drm_dp_mst_topology_mgr_suspend(mgr);
-		} else {
-			/* if extended timeout is supported in hardware,
-			 * default to LTTPR timeout (3.2ms) first as a W/A for DP link layer
-			 * CTS 4.2.1.1 regression introduced by CTS specs requirement update.
-			 */
-			try_to_configure_aux_timeout(aconnector->dc_link->ddc, LINK_AUX_DEFAULT_LTTPR_TIMEOUT_PERIOD);
-			if (!dp_is_lttpr_present(aconnector->dc_link))
-				try_to_configure_aux_timeout(aconnector->dc_link->ddc, LINK_AUX_DEFAULT_TIMEOUT_PERIOD);
-
-			/* TODO: move resume_mst_branch_status() into drm mst resume again
-			 * once topology probing work is pulled out from mst resume into mst
-			 * resume 2nd step. mst resume 2nd step should be called after old
-			 * state getting restored (i.e. drm_atomic_helper_resume()).
-			 */
-			resume_mst_branch_status(mgr);
-		}
-	}
-	drm_connector_list_iter_end(&iter);
-}
-
-static int amdgpu_dm_smu_write_watermarks_table(struct amdgpu_device *adev)
-{
-	int ret = 0;
-
-	/* This interface is for dGPU Navi1x.Linux dc-pplib interface depends
-	 * on window driver dc implementation.
-	 * For Navi1x, clock settings of dcn watermarks are fixed. the settings
-	 * should be passed to smu during boot up and resume from s3.
-	 * boot up: dc calculate dcn watermark clock settings within dc_create,
-	 * dcn20_resource_construct
-	 * then call pplib functions below to pass the settings to smu:
-	 * smu_set_watermarks_for_clock_ranges
-	 * smu_set_watermarks_table
-	 * navi10_set_watermarks_table
-	 * smu_write_watermarks_table
-	 *
-	 * For Renoir, clock settings of dcn watermark are also fixed values.
-	 * dc has implemented different flow for window driver:
-	 * dc_hardware_init / dc_set_power_state
-	 * dcn10_init_hw
-	 * notify_wm_ranges
-	 * set_wm_ranges
-	 * -- Linux
-	 * smu_set_watermarks_for_clock_ranges
-	 * renoir_set_watermarks_table
-	 * smu_write_watermarks_table
-	 *
-	 * For Linux,
-	 * dc_hardware_init -> amdgpu_dm_init
-	 * dc_set_power_state --> dm_resume
-	 *
-	 * therefore, this function apply to navi10/12/14 but not Renoir
-	 * *
-	 */
-	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
-	case IP_VERSION(2, 0, 2):
-	case IP_VERSION(2, 0, 0):
-		break;
-	default:
-		return 0;
-	}
-
-	ret = amdgpu_dpm_write_watermarks_table(adev);
-	if (ret) {
-		drm_err(adev_to_drm(adev), "Failed to update WMTABLE!\n");
-		return ret;
-	}
-
-	return 0;
-}
-
-static int dm_oem_i2c_hw_init(struct amdgpu_device *adev)
-{
-	struct amdgpu_display_manager *dm = &adev->dm;
-	struct amdgpu_i2c_adapter *oem_i2c;
-	struct ddc_service *oem_ddc_service;
-	int r;
-
-	oem_ddc_service = dc_get_oem_i2c_device(adev->dm.dc);
-	if (oem_ddc_service) {
-		oem_i2c = create_i2c(oem_ddc_service, true);
-		if (!oem_i2c) {
-			drm_info(adev_to_drm(adev), "Failed to create oem i2c adapter data\n");
-			return -ENOMEM;
-		}
-
-		r = devm_i2c_add_adapter(adev->dev, &oem_i2c->base);
-		if (r) {
-			drm_info(adev_to_drm(adev), "Failed to register oem i2c\n");
-			kfree(oem_i2c);
-			return r;
-		}
-		dm->oem_i2c = oem_i2c;
-	}
-
-	return 0;
-}
-
-/**
- * dm_hw_init() - Initialize DC device
- * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
- *
- * Initialize the &struct amdgpu_display_manager device. This involves calling
- * the initializers of each DM component, then populating the struct with them.
- *
- * Although the function implies hardware initialization, both hardware and
- * software are initialized here. Splitting them out to their relevant init
- * hooks is a future TODO item.
- *
- * Some notable things that are initialized here:
- *
- * - Display Core, both software and hardware
- * - DC modules that we need (freesync and color management)
- * - DRM software states
- * - Interrupt sources and handlers
- * - Vblank support
- * - Debug FS entries, if enabled
- */
-static int dm_hw_init(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	int r;
-
-	/* Create DAL display manager */
-	r = amdgpu_dm_init(adev);
-	if (r)
-		return r;
-	amdgpu_dm_hpd_init(adev);
-
-	r = dm_oem_i2c_hw_init(adev);
-	if (r)
-		drm_info(adev_to_drm(adev), "Failed to add OEM i2c bus\n");
-
-	return 0;
-}
-
-/**
- * dm_hw_fini() - Teardown DC device
- * @ip_block: Pointer to the amdgpu_ip_block for this hw instance.
- *
- * Teardown components within &struct amdgpu_display_manager that require
- * cleanup. This involves cleaning up the DRM device, DC, and any modules that
- * were loaded. Also flush IRQ workqueues and disable them.
- */
-static int dm_hw_fini(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-
-	amdgpu_dm_hpd_fini(adev);
-
-	amdgpu_dm_irq_fini(adev);
-	amdgpu_dm_fini(adev);
-	return 0;
-}
-
-
-static void dm_gpureset_toggle_interrupts(struct amdgpu_device *adev,
-				 struct dc_state *state, bool enable)
-{
-	enum dc_irq_source irq_source;
-	struct amdgpu_crtc *acrtc;
-	int rc = -EBUSY;
-	int i = 0;
-
-	for (i = 0; i < state->stream_count; i++) {
-		acrtc = get_crtc_by_otg_inst(
-				adev, state->stream_status[i].primary_otg_inst);
-
-		if (acrtc && state->stream_status[i].plane_count != 0) {
-			irq_source = IRQ_TYPE_PFLIP + acrtc->otg_inst;
-			rc = dc_interrupt_set(adev->dm.dc, irq_source, enable) ? 0 : -EBUSY;
-			if (rc)
-				drm_warn(adev_to_drm(adev), "Failed to %s pflip interrupts\n",
-					 enable ? "enable" : "disable");
-
-			if (dc_supports_vrr(adev->dm.dc->ctx->dce_version)) {
-				if (enable) {
-					if (amdgpu_dm_crtc_vrr_active(
-							to_dm_crtc_state(acrtc->base.state)))
-						rc = amdgpu_dm_crtc_set_vupdate_irq(
-							&acrtc->base, true);
-				} else
-					rc = amdgpu_dm_crtc_set_vupdate_irq(
-							&acrtc->base, false);
-
-				if (rc)
-					drm_warn(adev_to_drm(adev), "Failed to %sable vupdate interrupt\n",
-						enable ? "en" : "dis");
-			}
-
-			irq_source = IRQ_TYPE_VBLANK + acrtc->otg_inst;
-			/* During gpu-reset we disable and then enable vblank irq, so
-			 * don't use amdgpu_irq_get/put() to avoid refcount change.
-			 */
-			if (!dc_interrupt_set(adev->dm.dc, irq_source, enable))
-				drm_warn(adev_to_drm(adev), "Failed to %sable vblank interrupt\n", enable ? "en" : "dis");
-		}
-	}
-
-}
-
-DEFINE_FREE(state_release, struct dc_state *, if (_T) dc_state_release(_T))
-
-static enum dc_status amdgpu_dm_commit_zero_streams(struct dc *dc)
-{
-	struct dc_state *context __free(state_release) = NULL;
-	int i;
-	struct dc_stream_state *del_streams[MAX_PIPES];
-	int del_streams_count = 0;
-	struct dc_commit_streams_params params = {};
-
-	memset(del_streams, 0, sizeof(del_streams));
-
-	context = dc_state_create_current_copy(dc);
-	if (context == NULL)
-		return DC_ERROR_UNEXPECTED;
-
-	/* First remove from context all streams */
-	for (i = 0; i < context->stream_count; i++) {
-		struct dc_stream_state *stream = context->streams[i];
-
-		del_streams[del_streams_count++] = stream;
-	}
-
-	/* Remove all planes for removed streams and then remove the streams */
-	for (i = 0; i < del_streams_count; i++) {
-		enum dc_status res;
-
-		if (!dc_state_rem_all_planes_for_stream(dc, del_streams[i], context))
-			return DC_FAIL_DETACH_SURFACES;
-
-		res = dc_state_remove_stream(dc, context, del_streams[i]);
-		if (res != DC_OK)
-			return res;
-	}
-
-	params.streams = context->streams;
-	params.stream_count = context->stream_count;
-
-	return dc_commit_streams(dc, &params);
-}
-
-static void hpd_rx_irq_work_suspend(struct amdgpu_display_manager *dm)
-{
-	int i;
-
-	if (dm->hpd_rx_offload_wq) {
-		for (i = 0; i < dm->dc->caps.max_links; i++)
-			flush_workqueue(dm->hpd_rx_offload_wq[i].wq);
-	}
-}
-
-static int dm_cache_state(struct amdgpu_device *adev)
-{
-	int r;
-
-	adev->dm.cached_state = drm_atomic_helper_suspend(adev_to_drm(adev));
-	if (IS_ERR(adev->dm.cached_state)) {
-		r = PTR_ERR(adev->dm.cached_state);
-		adev->dm.cached_state = NULL;
-	}
-
-	return adev->dm.cached_state ? 0 : r;
-}
-
-static void dm_destroy_cached_state(struct amdgpu_device *adev)
-{
-	struct amdgpu_display_manager *dm = &adev->dm;
-	struct drm_device *ddev = adev_to_drm(adev);
-	struct dm_plane_state *dm_new_plane_state;
-	struct drm_plane_state *new_plane_state;
-	struct dm_crtc_state *dm_new_crtc_state;
-	struct drm_crtc_state *new_crtc_state;
-	struct drm_plane *plane;
-	struct drm_crtc *crtc;
-	int i;
-
-	if (!dm->cached_state)
-		return;
-
-	/* Force mode set in atomic commit */
-	for_each_new_crtc_in_state(dm->cached_state, crtc, new_crtc_state, i) {
-		new_crtc_state->active_changed = true;
-		dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
-		reset_freesync_config_for_crtc(dm_new_crtc_state);
-	}
-
-	/*
-	 * atomic_check is expected to create the dc states. We need to release
-	 * them here, since they were duplicated as part of the suspend
-	 * procedure.
-	 */
-	for_each_new_crtc_in_state(dm->cached_state, crtc, new_crtc_state, i) {
-		dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
-		if (dm_new_crtc_state->stream) {
-			WARN_ON(kref_read(&dm_new_crtc_state->stream->refcount) > 1);
-			dc_stream_release(dm_new_crtc_state->stream);
-			dm_new_crtc_state->stream = NULL;
-		}
-		dm_new_crtc_state->base.color_mgmt_changed = true;
-	}
-
-	for_each_new_plane_in_state(dm->cached_state, plane, new_plane_state, i) {
-		dm_new_plane_state = to_dm_plane_state(new_plane_state);
-		if (dm_new_plane_state->dc_state) {
-			WARN_ON(kref_read(&dm_new_plane_state->dc_state->refcount) > 1);
-			dc_plane_state_release(dm_new_plane_state->dc_state);
-			dm_new_plane_state->dc_state = NULL;
-		}
-	}
-
-	drm_atomic_helper_resume(ddev, dm->cached_state);
-
-	dm->cached_state = NULL;
-}
-
-static int dm_suspend(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	struct amdgpu_display_manager *dm = &adev->dm;
-
-	if (amdgpu_in_reset(adev)) {
-		enum dc_status res;
-
-		/* Quiesce ISM workers before taking dc_lock (workers take
-		 * dc_lock themselves; syncing under it would deadlock).
-		 */
-		amdgpu_dm_ism_disable(dm);
-
-		mutex_lock(&dm->dc_lock);
-
-		amdgpu_dm_ism_force_full_power(dm);
-		dc_allow_idle_optimizations(adev->dm.dc, false);
-
-		dm->cached_dc_state = dc_state_create_copy(dm->dc->current_state);
-
-		if (dm->cached_dc_state)
-			dm_gpureset_toggle_interrupts(adev, dm->cached_dc_state, false);
-
-		res = amdgpu_dm_commit_zero_streams(dm->dc);
-		if (res != DC_OK) {
-			drm_err(adev_to_drm(adev), "Failed to commit zero streams: %d\n", res);
-			return -EINVAL;
-		}
-
-		amdgpu_dm_irq_suspend(adev);
-
-		hpd_rx_irq_work_suspend(dm);
-
-		return 0;
-	}
-
-	if (!adev->dm.cached_state) {
-		int r = dm_cache_state(adev);
-
-		if (r)
-			return r;
-	}
-
-	s3_handle_hdmi_cec(adev_to_drm(adev), true);
-
-	s3_handle_mst(adev_to_drm(adev), true);
-
-	amdgpu_dm_irq_suspend(adev);
-
-	/*
-	 * Quiesce ISM workers before taking dc_lock (workers take dc_lock
-	 * themselves; syncing under it would deadlock).
-	 */
-	amdgpu_dm_ism_disable(dm);
-	scoped_guard(mutex, &dm->dc_lock)
-		amdgpu_dm_ism_force_full_power(dm);
-
-	hpd_rx_irq_work_suspend(dm);
-
-	dc_set_power_state(dm->dc, DC_ACPI_CM_POWER_STATE_D3);
-
-	if (dm->dc->caps.ips_support && adev->in_s0ix)
-		dc_allow_idle_optimizations(dm->dc, true);
-
-	dc_dmub_srv_set_power_state(dm->dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D3);
-
-	return 0;
-}
-
-struct drm_connector *
-amdgpu_dm_find_first_crtc_matching_connector(struct drm_atomic_commit *state,
-					     struct drm_crtc *crtc)
-{
-	u32 i;
-	struct drm_connector_state *new_con_state;
-	struct drm_connector *connector;
-	struct drm_crtc *crtc_from_state;
-
-	for_each_new_connector_in_state(state, connector, new_con_state, i) {
-		crtc_from_state = new_con_state->crtc;
-
-		if (crtc_from_state == crtc)
-			return connector;
-	}
-
-	return NULL;
-}
-
-static void emulated_link_detect(struct dc_link *link)
-{
-	struct dc_sink_init_data sink_init_data = { 0 };
-	struct display_sink_capability sink_caps = { 0 };
-	enum dc_edid_status edid_status;
-	struct dc_context *dc_ctx = link->ctx;
-	struct drm_device *dev = adev_to_drm(dc_ctx->driver_context);
-	struct dc_sink *sink = NULL;
-	struct dc_sink *prev_sink = NULL;
-
-	link->type = dc_connection_none;
-	prev_sink = link->local_sink;
-
-	if (prev_sink)
-		dc_sink_release(prev_sink);
-
-	switch (link->connector_signal) {
-	case SIGNAL_TYPE_HDMI_TYPE_A: {
-		sink_caps.transaction_type = DDC_TRANSACTION_TYPE_I2C;
-		sink_caps.signal = SIGNAL_TYPE_HDMI_TYPE_A;
-		break;
-	}
-
-	case SIGNAL_TYPE_DVI_SINGLE_LINK: {
-		sink_caps.transaction_type = DDC_TRANSACTION_TYPE_I2C;
-		sink_caps.signal = SIGNAL_TYPE_DVI_SINGLE_LINK;
-		break;
-	}
-
-	case SIGNAL_TYPE_DVI_DUAL_LINK: {
-		sink_caps.transaction_type = DDC_TRANSACTION_TYPE_I2C;
-		sink_caps.signal = SIGNAL_TYPE_DVI_DUAL_LINK;
-		break;
-	}
-
-	case SIGNAL_TYPE_LVDS: {
-		sink_caps.transaction_type = DDC_TRANSACTION_TYPE_I2C;
-		sink_caps.signal = SIGNAL_TYPE_LVDS;
-		break;
-	}
-
-	case SIGNAL_TYPE_EDP: {
-		sink_caps.transaction_type =
-			DDC_TRANSACTION_TYPE_I2C_OVER_AUX;
-		sink_caps.signal = SIGNAL_TYPE_EDP;
-		break;
-	}
-
-	case SIGNAL_TYPE_DISPLAY_PORT: {
-		sink_caps.transaction_type =
-			DDC_TRANSACTION_TYPE_I2C_OVER_AUX;
-		sink_caps.signal = SIGNAL_TYPE_VIRTUAL;
-		break;
-	}
-
-	default:
-		drm_err(dev, "Invalid connector type! signal:%d\n",
-			link->connector_signal);
-		return;
-	}
-
-	sink_init_data.link = link;
-	sink_init_data.sink_signal = sink_caps.signal;
-
-	sink = dc_sink_create(&sink_init_data);
-	if (!sink) {
-		drm_err(dev, "Failed to create sink!\n");
-		return;
-	}
-
-	/* dc_sink_create returns a new reference */
-	link->local_sink = sink;
-
-	edid_status = dm_helpers_read_local_edid(
-			link->ctx,
-			link,
-			sink);
-
-	if (edid_status != EDID_OK)
-		drm_err(dev, "Failed to read EDID\n");
-
-}
-
-static void dm_gpureset_commit_state(struct dc_state *dc_state,
-				     struct amdgpu_display_manager *dm)
-{
-	struct {
-		struct dc_surface_update surface_updates[MAX_SURFACES];
-		struct dc_plane_info plane_infos[MAX_SURFACES];
-		struct dc_scaling_info scaling_infos[MAX_SURFACES];
-		struct dc_flip_addrs flip_addrs[MAX_SURFACES];
-		struct dc_stream_update stream_update;
-	} *bundle __free(kfree);
-	int k, m;
-
-	bundle = kzalloc_obj(*bundle);
-
-	if (!bundle) {
-		drm_err(dm->ddev, "Failed to allocate update bundle\n");
-		return;
-	}
-
-	for (k = 0; k < dc_state->stream_count; k++) {
-		bundle->stream_update.stream = dc_state->streams[k];
-
-		for (m = 0; m < dc_state->stream_status[k].plane_count; m++) {
-			bundle->surface_updates[m].surface =
-				dc_state->stream_status[k].plane_states[m];
-			bundle->surface_updates[m].surface->force_full_update =
-				true;
-		}
-
-		update_planes_and_stream_adapter(dm->dc,
-					 UPDATE_TYPE_FULL,
-					 dc_state->stream_status[k].plane_count,
-					 dc_state->streams[k],
-					 &bundle->stream_update,
-					 bundle->surface_updates);
-	}
-}
-
-static void apply_delay_after_dpcd_poweroff(struct amdgpu_device *adev,
-					    struct dc_sink *sink)
-{
-	struct dc_panel_patch *ppatch = NULL;
-
-	if (!sink)
-		return;
-
-	ppatch = &sink->edid_caps.panel_patch;
-	if (ppatch->wait_after_dpcd_poweroff_ms) {
-		msleep(ppatch->wait_after_dpcd_poweroff_ms);
-		drm_dbg_driver(adev_to_drm(adev),
-			       "%s: adding a %ds delay as w/a for panel\n",
-			       __func__,
-			       ppatch->wait_after_dpcd_poweroff_ms / 1000);
-	}
-}
-
-/**
- * amdgpu_dm_dump_links_and_sinks - Debug dump of all DC links and their sinks
- * @adev: amdgpu device pointer
- *
- * Iterates through all DC links and dumps information about local and remote
- * (MST) sinks. Should be called after connector detection is complete to see
- * the final state of all links.
- */
-static void amdgpu_dm_dump_links_and_sinks(struct amdgpu_device *adev)
-{
-	struct dc *dc = adev->dm.dc;
-	struct drm_device *dev = adev_to_drm(adev);
-	int li;
-
-	if (!dc)
-		return;
-
-	for (li = 0; li < dc->link_count; li++) {
-		struct dc_link *l = dc->links[li];
-		const char *name = NULL;
-		int rs;
-
-		if (!l)
-			continue;
-		if (l->local_sink && l->local_sink->edid_caps.display_name[0])
-			name = l->local_sink->edid_caps.display_name;
-		else
-			name = "n/a";
-
-		drm_dbg_kms(dev,
-			"LINK_DUMP[%d]: local_sink=%p type=%d sink_signal=%d sink_count=%u edid_name=%s mst_capable=%d mst_alloc_streams=%d\n",
-			li,
-			l->local_sink,
-			l->type,
-			l->local_sink ? l->local_sink->sink_signal : SIGNAL_TYPE_NONE,
-			l->sink_count,
-			name,
-			l->dpcd_caps.is_mst_capable,
-			l->mst_stream_alloc_table.stream_count);
-
-		/* Dump remote (MST) sinks if any */
-		for (rs = 0; rs < l->sink_count; rs++) {
-			struct dc_sink *rsink = l->remote_sinks[rs];
-			const char *rname = NULL;
-
-			if (!rsink)
-				continue;
-			if (rsink->edid_caps.display_name[0])
-				rname = rsink->edid_caps.display_name;
-			else
-				rname = "n/a";
-			drm_dbg_kms(dev,
-				"  REMOTE_SINK[%d:%d]: sink=%p signal=%d edid_name=%s\n",
-				li, rs,
-				rsink,
-				rsink->sink_signal,
-				rname);
-		}
-	}
-}
-
-static int dm_resume(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	struct drm_device *ddev = adev_to_drm(adev);
-	struct amdgpu_display_manager *dm = &adev->dm;
-	struct amdgpu_dm_connector *aconnector;
-	struct drm_connector *connector;
-	struct drm_connector_list_iter iter;
-	struct dm_atomic_state *dm_state = to_dm_atomic_state(dm->atomic_obj.state);
-	enum dc_connection_type new_connection_type = dc_connection_none;
-	struct dc_state *dc_state;
-	int i, r, j;
-	struct dc_commit_streams_params commit_params = {};
-
-	if (dm->dc->caps.ips_support) {
-		if (!amdgpu_in_reset(adev))
-			mutex_lock(&dm->dc_lock);
-
-		/* Need to set POWER_STATE_D0 first or it will not execute
-		 * idle_power_optimizations command to DMUB.
-		 */
-		dc_dmub_srv_set_power_state(dm->dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D0);
-		dc_dmub_srv_apply_idle_power_optimizations(dm->dc, false);
-
-		if (!amdgpu_in_reset(adev))
-			mutex_unlock(&dm->dc_lock);
-	}
-
-	if (amdgpu_in_reset(adev)) {
-		dc_state = dm->cached_dc_state;
-
-		/*
-		 * The dc->current_state is backed up into dm->cached_dc_state
-		 * before we commit 0 streams.
-		 *
-		 * DC will clear link encoder assignments on the real state
-		 * but the changes won't propagate over to the copy we made
-		 * before the 0 streams commit.
-		 *
-		 * DC expects that link encoder assignments are *not* valid
-		 * when committing a state, so as a workaround we can copy
-		 * off of the current state.
-		 *
-		 * We lose the previous assignments, but we had already
-		 * commit 0 streams anyway.
-		 */
-		link_enc_cfg_copy(adev->dm.dc->current_state, dc_state);
-
-		r = dm_dmub_hw_init(adev);
-		if (r) {
-			drm_err(adev_to_drm(adev), "DMUB interface failed to initialize: status=%d\n", r);
-			return r;
-		}
-
-		dc_dmub_srv_set_power_state(dm->dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D0);
-		dc_set_power_state(dm->dc, DC_ACPI_CM_POWER_STATE_D0);
-
-		dc_resume(dm->dc);
-
-		amdgpu_dm_ism_enable(dm);
-		amdgpu_dm_irq_resume_early(adev);
-
-		for (i = 0; i < dc_state->stream_count; i++) {
-			dc_state->streams[i]->mode_changed = true;
-			for (j = 0; j < dc_state->stream_status[i].plane_count; j++) {
-				dc_state->stream_status[i].plane_states[j]->update_flags.raw
-					= 0xffffffff;
-			}
-		}
-
-		if (dc_is_dmub_outbox_supported(adev->dm.dc)) {
-			amdgpu_dm_outbox_init(adev);
-			dc_enable_dmub_outbox(adev->dm.dc);
-		}
-
-		commit_params.streams = dc_state->streams;
-		commit_params.stream_count = dc_state->stream_count;
-		dc_exit_ips_for_hw_access(dm->dc);
-		WARN_ON(!dc_commit_streams(dm->dc, &commit_params));
-
-		dm_gpureset_commit_state(dm->cached_dc_state, dm);
-
-		dm_gpureset_toggle_interrupts(adev, dm->cached_dc_state, true);
-
-		dc_state_release(dm->cached_dc_state);
-		dm->cached_dc_state = NULL;
-
-		amdgpu_dm_irq_resume_late(adev);
-
-		mutex_unlock(&dm->dc_lock);
-
-		/* set the backlight after a reset */
-		for (i = 0; i < dm->num_of_edps; i++) {
-			if (dm->backlight_dev[i])
-				amdgpu_dm_backlight_set_level(dm, i, dm->brightness[i]);
-		}
-
-		return 0;
-	}
-	/* Recreate dc_state - DC invalidates it when setting power state to S3. */
-	dc_state_release(dm_state->context);
-	dm_state->context = dc_state_create(dm->dc, NULL);
-	/* TODO: Remove dc_state->dccg, use dc->dccg directly. */
-
-	/* Before powering on DC we need to re-initialize DMUB. */
-	dm_dmub_hw_resume(adev);
-
-	/* Re-enable outbox interrupts for DPIA. */
-	if (dc_is_dmub_outbox_supported(adev->dm.dc)) {
-		amdgpu_dm_outbox_init(adev);
-		dc_enable_dmub_outbox(adev->dm.dc);
-	}
-
-	/* power on hardware */
-	dc_dmub_srv_set_power_state(dm->dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D0);
-	dc_set_power_state(dm->dc, DC_ACPI_CM_POWER_STATE_D0);
-
-	/* program HPD filter */
-	dc_resume(dm->dc);
-
-	scoped_guard(mutex, &dm->dc_lock)
-		amdgpu_dm_ism_enable(dm);
-
-	/*
-	 * early enable HPD Rx IRQ, should be done before set mode as short
-	 * pulse interrupts are used for MST
-	 */
-	amdgpu_dm_irq_resume_early(adev);
-
-	s3_handle_hdmi_cec(ddev, false);
-
-	/* On resume we need to rewrite the MSTM control bits to enable MST*/
-	s3_handle_mst(ddev, false);
-
-	/* Do detection*/
-	drm_connector_list_iter_begin(ddev, &iter);
-	drm_for_each_connector_iter(connector, &iter) {
-		bool ret;
-
-		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
-			continue;
-
-		aconnector = to_amdgpu_dm_connector(connector);
-
-		if (!aconnector->dc_link)
-			continue;
-
-		/*
-		 * this is the case when traversing through already created end sink
-		 * MST connectors, should be skipped
-		 */
-		if (aconnector->mst_root)
-			continue;
-
-		/* Skip eDP detection, when there is no sink present */
-		if (aconnector->dc_link->connector_signal == SIGNAL_TYPE_EDP &&
-		    !aconnector->dc_link->edp_sink_present)
-			continue;
-
-		guard(mutex)(&aconnector->hpd_lock);
-		if (!dc_link_detect_connection_type(aconnector->dc_link, &new_connection_type))
-			drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n");
-
-		if (aconnector->base.force && new_connection_type == dc_connection_none) {
-			emulated_link_detect(aconnector->dc_link);
-		} else {
-			guard(mutex)(&dm->dc_lock);
-			dc_exit_ips_for_hw_access(dm->dc);
-			ret = dc_link_detect(aconnector->dc_link, DETECT_REASON_RESUMEFROMS3S4);
-			if (ret) {
-				/* w/a delay for certain panels */
-				apply_delay_after_dpcd_poweroff(adev, aconnector->dc_sink);
-			}
-		}
-
-		if (aconnector->fake_enable && aconnector->dc_link->local_sink)
-			aconnector->fake_enable = false;
-
-		if (aconnector->dc_sink)
-			dc_sink_release(aconnector->dc_sink);
-		aconnector->dc_sink = NULL;
-		amdgpu_dm_update_connector_after_detect(aconnector);
-	}
-	drm_connector_list_iter_end(&iter);
-
-	dm_destroy_cached_state(adev);
-
-	/* Do mst topology probing after resuming cached state*/
-	drm_connector_list_iter_begin(ddev, &iter);
-	drm_for_each_connector_iter(connector, &iter) {
-		bool init = false;
-
-		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
-			continue;
-
-		aconnector = to_amdgpu_dm_connector(connector);
-		if (aconnector->dc_link->type != dc_connection_mst_branch ||
-		    aconnector->mst_root)
-			continue;
-
-		scoped_guard(mutex, &aconnector->mst_mgr.lock) {
-			init = !aconnector->mst_mgr.mst_primary;
-		}
-		if (init)
-			dm_helpers_dp_mst_start_top_mgr(aconnector->dc_link->ctx,
-				aconnector->dc_link, false);
-		else
-			drm_dp_mst_topology_queue_probe(&aconnector->mst_mgr);
-	}
-	drm_connector_list_iter_end(&iter);
-
-	/* Debug dump: list all DC links and their associated sinks after detection
-	 * is complete for all connectors. This provides a comprehensive view of the
-	 * final state without repeating the dump for each connector.
-	 */
-	amdgpu_dm_dump_links_and_sinks(adev);
-
-	amdgpu_dm_irq_resume_late(adev);
-
-	amdgpu_dm_smu_write_watermarks_table(adev);
-
-	drm_kms_helper_hotplug_event(ddev);
-
-	return 0;
-}
-
-/**
- * DOC: DM Lifecycle
- *
- * DM (and consequently DC) is registered in the amdgpu base driver as a IP
- * block. When CONFIG_DRM_AMD_DC is enabled, the DM device IP block is added to
- * the base driver's device list to be initialized and torn down accordingly.
- *
- * The functions to do so are provided as hooks in &struct amd_ip_funcs.
- */
-
-static const struct amd_ip_funcs amdgpu_dm_funcs = {
-	.name = "dm",
-	.early_init = dm_early_init,
-	.late_init = dm_late_init,
-	.sw_init = dm_sw_init,
-	.sw_fini = dm_sw_fini,
-	.early_fini = amdgpu_dm_early_fini,
-	.hw_init = dm_hw_init,
-	.hw_fini = dm_hw_fini,
-	.suspend = dm_suspend,
-	.resume = dm_resume,
-	.is_idle = dm_is_idle,
-	.wait_for_idle = dm_wait_for_idle,
-	.check_soft_reset = dm_check_soft_reset,
-	.soft_reset = dm_soft_reset,
-	.set_clockgating_state = dm_set_clockgating_state,
-	.set_powergating_state = dm_set_powergating_state,
-};
-
-const struct amdgpu_ip_block_version dm_ip_block = {
-	.type = AMD_IP_BLOCK_TYPE_DCE,
-	.major = 1,
-	.minor = 0,
-	.rev = 0,
-	.funcs = &amdgpu_dm_funcs,
-};
-
-
-/**
- * DOC: atomic
- *
- * *WIP*
- */
-
-static const struct drm_mode_config_funcs amdgpu_dm_mode_funcs = {
-	.fb_create = amdgpu_display_user_framebuffer_create,
-	.get_format_info = amdgpu_dm_plane_get_format_info,
-	.atomic_check = amdgpu_dm_atomic_check,
-	.atomic_commit = drm_atomic_helper_commit,
-};
-
-static struct drm_mode_config_helper_funcs amdgpu_dm_mode_config_helperfuncs = {
-	.atomic_commit_tail = amdgpu_dm_atomic_commit_tail,
-	.atomic_commit_setup = amdgpu_dm_atomic_setup_commit,
-};
-
-#define DDC_MANUFACTURERNAME_SAMSUNG 0x2D4C
-
-static void dm_set_panel_type(struct amdgpu_dm_connector *aconnector)
-{
-	struct drm_connector *connector = &aconnector->base;
-	struct drm_display_info *display_info = &connector->display_info;
-	struct dc_link *link = aconnector->dc_link;
-	struct amdgpu_device *adev;
-
-	adev = drm_to_adev(connector->dev);
-
-	link->panel_type = PANEL_TYPE_NONE;
-
-	switch (display_info->amd_vsdb.panel_type) {
-	case AMD_VSDB_PANEL_TYPE_OLED:
-		link->panel_type = PANEL_TYPE_OLED;
-		break;
-	case AMD_VSDB_PANEL_TYPE_MINILED:
-		link->panel_type = PANEL_TYPE_MINILED;
-		break;
-	}
-
-	/* If VSDB didn't determine panel type, check DPCD ext caps */
-	if (link->panel_type == PANEL_TYPE_NONE) {
-		if (link->dpcd_sink_ext_caps.bits.miniled == 1)
-			link->panel_type = PANEL_TYPE_MINILED;
-		if (link->dpcd_sink_ext_caps.bits.oled == 1)
-			link->panel_type = PANEL_TYPE_OLED;
-	}
-
-	/*
-	 * TODO: get panel type from DID2 that has device technology field
-	 * to specify if it's OLED or not. But we need to wait for DID2
-	 * support in DC and EDID parser to be able to use it here.
-	 */
-
-	if (link->panel_type == PANEL_TYPE_NONE) {
-		struct drm_amd_vsdb_info *vsdb = &display_info->amd_vsdb;
-		u32 lum1_max = vsdb->luminance_range1.max_luminance;
-		u32 lum2_max = vsdb->luminance_range2.max_luminance;
-
-		if (vsdb->version && link->local_sink &&
-		    link->local_sink->edid_caps.manufacturer_id ==
-		    DDC_MANUFACTURERNAME_SAMSUNG &&
-		    lum1_max >= ((lum2_max * 3) / 2))
-			link->panel_type = PANEL_TYPE_MINILED;
-	}
-
-	if (link->panel_type == PANEL_TYPE_OLED)
-		drm_object_property_set_value(&connector->base,
-		    adev_to_drm(adev)->mode_config.panel_type_property,
-		    DRM_MODE_PANEL_TYPE_OLED);
-	else
-		drm_object_property_set_value(&connector->base,
-		    adev_to_drm(adev)->mode_config.panel_type_property,
-		    DRM_MODE_PANEL_TYPE_UNKNOWN);
-
-	drm_dbg_kms(aconnector->base.dev, "Panel type: %d\n", link->panel_type);
-}
-
-static void update_connector_ext_caps(struct amdgpu_dm_connector *aconnector)
-{
-	const struct drm_panel_backlight_quirk *panel_backlight_quirk;
-	struct amdgpu_dm_backlight_caps *caps;
-	struct drm_connector *conn_base;
-	struct amdgpu_device *adev;
-	struct drm_luminance_range_info *luminance_range;
-	struct drm_device *drm;
-
-	if (aconnector->bl_idx == -1 ||
-	    aconnector->dc_link->connector_signal != SIGNAL_TYPE_EDP)
-		return;
-
-	conn_base = &aconnector->base;
-	drm = conn_base->dev;
-	adev = drm_to_adev(drm);
-
-	caps = &adev->dm.backlight_caps[aconnector->bl_idx];
-	caps->ext_caps = &aconnector->dc_link->dpcd_sink_ext_caps;
-	caps->aux_support = false;
-
-	if (caps->ext_caps->bits.oled == 1
-	    /*
-	     * ||
-	     * caps->ext_caps->bits.sdr_aux_backlight_control == 1 ||
-	     * caps->ext_caps->bits.hdr_aux_backlight_control == 1
-	     */)
-		caps->aux_support = true;
-
-	if (amdgpu_backlight == 0)
-		caps->aux_support = false;
-	else if (amdgpu_backlight == 1)
-		caps->aux_support = true;
-	if (caps->aux_support)
-		aconnector->dc_link->backlight_control_type = BACKLIGHT_CONTROL_AMD_AUX;
-
-	luminance_range = &conn_base->display_info.luminance_range;
-
-	if (luminance_range->max_luminance)
-		caps->aux_max_input_signal = luminance_range->max_luminance;
-	else
-		caps->aux_max_input_signal = 512;
-
-	if (luminance_range->min_luminance)
-		caps->aux_min_input_signal = luminance_range->min_luminance;
-	else
-		caps->aux_min_input_signal = 1;
-
-	panel_backlight_quirk =
-		drm_get_panel_backlight_quirk(aconnector->drm_edid);
-	if (!IS_ERR_OR_NULL(panel_backlight_quirk)) {
-		if (panel_backlight_quirk->min_brightness) {
-			caps->min_input_signal =
-				panel_backlight_quirk->min_brightness - 1;
-			drm_info(drm,
-				 "Applying panel backlight quirk, min_brightness: %d\n",
-				 caps->min_input_signal);
-		}
-		if (panel_backlight_quirk->brightness_mask) {
-			drm_info(drm,
-				 "Applying panel backlight quirk, brightness_mask: 0x%X\n",
-				 panel_backlight_quirk->brightness_mask);
-			caps->brightness_mask =
-				panel_backlight_quirk->brightness_mask;
-		}
-	}
-}
-
-DEFINE_FREE(sink_release, struct dc_sink *, if (_T) dc_sink_release(_T))
-
-void amdgpu_dm_update_connector_after_detect(
-		struct amdgpu_dm_connector *aconnector)
-{
-	struct drm_connector *connector = &aconnector->base;
-	struct dc_sink *sink __free(sink_release) = NULL;
-	struct drm_device *dev = connector->dev;
-
-	/* MST handled by drm_mst framework */
-	if (aconnector->mst_mgr.mst_state == true)
-		return;
-
-	sink = aconnector->dc_link->local_sink;
-	if (sink)
-		dc_sink_retain(sink);
-
-	/*
-	 * Edid mgmt connector gets first update only in mode_valid hook and then
-	 * the connector sink is set to either fake or physical sink depends on link status.
-	 * Skip if already done during boot.
-	 */
-	if (aconnector->base.force != DRM_FORCE_UNSPECIFIED
-			&& aconnector->dc_em_sink) {
-
-		/*
-		 * For S3 resume with headless use eml_sink to fake stream
-		 * because on resume connector->sink is set to NULL
-		 */
-		guard(mutex)(&dev->mode_config.mutex);
-
-		if (sink) {
-			if (aconnector->dc_sink) {
-				amdgpu_dm_update_freesync_caps(connector, NULL, true);
-				/*
-				 * retain and release below are used to
-				 * bump up refcount for sink because the link doesn't point
-				 * to it anymore after disconnect, so on next crtc to connector
-				 * reshuffle by UMD we will get into unwanted dc_sink release
-				 */
-				dc_sink_release(aconnector->dc_sink);
-			}
-			aconnector->dc_sink = sink;
-			dc_sink_retain(aconnector->dc_sink);
-			amdgpu_dm_update_freesync_caps(connector,
-					aconnector->drm_edid, true);
-		} else {
-			amdgpu_dm_update_freesync_caps(connector, NULL, true);
-			if (!aconnector->dc_sink) {
-				aconnector->dc_sink = aconnector->dc_em_sink;
-				dc_sink_retain(aconnector->dc_sink);
-			}
-		}
-
-		return;
-	}
-
-	/*
-	 * TODO: temporary guard to look for proper fix
-	 * if this sink is MST sink, we should not do anything
-	 */
-	if (sink && sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT_MST)
-		return;
-
-	if (aconnector->dc_sink == sink) {
-		/*
-		 * We got a DP short pulse (Link Loss, DP CTS, etc...).
-		 * Do nothing!!
-		 */
-		drm_dbg_kms(dev, "DCHPD: connector_id=%d: dc_sink didn't change.\n",
-				 aconnector->connector_id);
-		return;
-	}
-
-	drm_dbg_kms(dev, "DCHPD: connector_id=%d: Old sink=%p New sink=%p\n",
-		    aconnector->connector_id, aconnector->dc_sink, sink);
-
-	/* When polling, DRM has already locked the mutex for us. */
-	if (!drm_kms_helper_is_poll_worker())
-		mutex_lock(&dev->mode_config.mutex);
-
-	/*
-	 * 1. Update status of the drm connector
-	 * 2. Send an event and let userspace tell us what to do
-	 */
-	if (sink) {
-		/*
-		 * TODO: check if we still need the S3 mode update workaround.
-		 * If yes, put it here.
-		 */
-		if (aconnector->dc_sink) {
-			amdgpu_dm_update_freesync_caps(connector, NULL, true);
-			dc_sink_release(aconnector->dc_sink);
-		}
-
-		aconnector->dc_sink = sink;
-		dc_sink_retain(aconnector->dc_sink);
-		drm_edid_free(aconnector->drm_edid);
-		aconnector->drm_edid = NULL;
-		if (sink->dc_edid.length == 0) {
-			hdmi_cec_unset_edid(aconnector);
-			if (aconnector->dc_link->aux_mode) {
-				drm_dp_cec_unset_edid(&aconnector->dm_dp_aux.aux);
-			}
-		} else {
-			const struct edid *edid = (const struct edid *)sink->dc_edid.raw_edid;
-
-			aconnector->drm_edid = drm_edid_alloc(edid, sink->dc_edid.length);
-			drm_edid_connector_update(connector, aconnector->drm_edid);
-
-			hdmi_cec_set_edid(aconnector);
-			if (aconnector->dc_link->aux_mode)
-				drm_dp_cec_attach(&aconnector->dm_dp_aux.aux,
-						  connector->display_info.source_physical_address);
-		}
-
-		if (!aconnector->timing_requested) {
-			aconnector->timing_requested =
-				kzalloc_obj(struct dc_crtc_timing);
-			if (!aconnector->timing_requested)
-				drm_err(dev,
-					"failed to create aconnector->requested_timing\n");
-		}
-
-		amdgpu_dm_update_freesync_caps(connector, aconnector->drm_edid, true);
-		update_connector_ext_caps(aconnector);
-		dm_set_panel_type(aconnector);
-	} else {
-		hdmi_cec_unset_edid(aconnector);
-		drm_dp_cec_unset_edid(&aconnector->dm_dp_aux.aux);
-		amdgpu_dm_update_freesync_caps(connector, NULL, true);
-		aconnector->num_modes = 0;
-		dc_sink_release(aconnector->dc_sink);
-		aconnector->dc_sink = NULL;
-		drm_edid_free(aconnector->drm_edid);
-		aconnector->drm_edid = NULL;
-		kfree(aconnector->timing_requested);
-		aconnector->timing_requested = NULL;
-		/* Set CP to DESIRED if it was ENABLED, so we can re-enable it again on hotplug */
-		if (connector->state->content_protection == DRM_MODE_CONTENT_PROTECTION_ENABLED)
-			connector->state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED;
-	}
-
-	update_subconnector_property(aconnector);
-
-	/* When polling, the mutex will be unlocked for us by DRM. */
-	if (!drm_kms_helper_is_poll_worker())
-		mutex_unlock(&dev->mode_config.mutex);
-}
-
-static bool are_sinks_equal(const struct dc_sink *sink1, const struct dc_sink *sink2)
-{
-	if (!sink1 || !sink2)
-		return false;
-	if (sink1->sink_signal != sink2->sink_signal)
-		return false;
-
-	if (sink1->dc_edid.length != sink2->dc_edid.length)
-		return false;
-
-	if (memcmp(sink1->dc_edid.raw_edid, sink2->dc_edid.raw_edid,
-		   sink1->dc_edid.length) != 0)
-		return false;
-	return true;
-}
-
-
-/**
- * DOC: hdmi_hpd_debounce_work
- *
- * HDMI HPD debounce delay in milliseconds. When an HDMI display toggles HPD
- * (such as during power save transitions), this delay determines how long to
- * wait before processing the HPD event. This allows distinguishing between a
- * physical unplug (>hdmi_hpd_debounce_delay)
- * and a spontaneous RX HPD toggle (<hdmi_hpd_debounce_delay).
- *
- * If the toggle is less than this delay, the driver compares sink capabilities
- * and permits a hotplug event if they changed.
- *
- * The default value of 1500ms was chosen based on experimental testing with
- * various monitors that exhibit spontaneous HPD toggling behavior.
- */
-static void hdmi_hpd_debounce_work(struct work_struct *work)
-{
-	struct amdgpu_dm_connector *aconnector =
-		container_of(to_delayed_work(work), struct amdgpu_dm_connector,
-			     hdmi_hpd_debounce_work);
-	struct drm_connector *connector = &aconnector->base;
-	struct drm_device *dev = connector->dev;
-	struct amdgpu_device *adev = drm_to_adev(dev);
-	struct dc *dc = aconnector->dc_link->ctx->dc;
-	bool fake_reconnect = false;
-	bool reallow_idle = false;
-	bool ret = false;
-	guard(mutex)(&aconnector->hpd_lock);
-
-	/* Re-detect the display */
-	scoped_guard(mutex, &adev->dm.dc_lock) {
-		if (dc->caps.ips_support && dc->ctx->dmub_srv->idle_allowed) {
-			dc_allow_idle_optimizations(dc, false);
-			reallow_idle = true;
-		}
-		ret = dc_link_detect(aconnector->dc_link, DETECT_REASON_HPD);
-	}
-
-	if (ret) {
-		/* Apply workaround delay for certain panels */
-		apply_delay_after_dpcd_poweroff(adev, aconnector->dc_sink);
-		/* Compare sinks to determine if this was a spontaneous HPD toggle */
-		if (are_sinks_equal(aconnector->dc_link->local_sink, aconnector->hdmi_prev_sink)) {
-			/*
-			* Sinks match - this was a spontaneous HDMI HPD toggle.
-			*/
-			drm_dbg_kms(dev, "HDMI HPD: Sink unchanged after debounce, internal re-enable\n");
-			fake_reconnect = true;
-		}
-
-		/* Update connector state */
-		amdgpu_dm_update_connector_after_detect(aconnector);
-
-		drm_modeset_lock_all(dev);
-		dm_restore_drm_connector_state(dev, connector);
-		drm_modeset_unlock_all(dev);
-
-		/* Only notify OS if sink actually changed */
-		if (!fake_reconnect && aconnector->base.force == DRM_FORCE_UNSPECIFIED)
-			drm_kms_helper_hotplug_event(dev);
-	}
-
-	/* Release the cached sink reference */
-	if (aconnector->hdmi_prev_sink) {
-		dc_sink_release(aconnector->hdmi_prev_sink);
-		aconnector->hdmi_prev_sink = NULL;
-	}
-
-	scoped_guard(mutex, &adev->dm.dc_lock) {
-		if (reallow_idle && dc->caps.ips_support)
-			dc_allow_idle_optimizations(dc, true);
-	}
-}
-
-static void handle_hpd_irq_helper(struct amdgpu_dm_connector *aconnector)
-{
-	struct drm_connector *connector = &aconnector->base;
-	struct drm_device *dev = connector->dev;
-	enum dc_connection_type new_connection_type = dc_connection_none;
-	struct amdgpu_device *adev = drm_to_adev(dev);
-	struct dm_connector_state *dm_con_state = to_dm_connector_state(connector->state);
-	struct dc *dc = aconnector->dc_link->ctx->dc;
-	bool ret = false;
-	bool debounce_required = false;
-
-	if (adev->dm.disable_hpd_irq)
-		return;
-
-	/*
-	 * In case of failure or MST no need to update connector status or notify the OS
-	 * since (for MST case) MST does this in its own context.
-	 */
-	guard(mutex)(&aconnector->hpd_lock);
-
-	if (adev->dm.hdcp_workqueue) {
-		hdcp_reset_display(adev->dm.hdcp_workqueue, aconnector->dc_link->link_index);
-		dm_con_state->update_hdcp = true;
-	}
-	if (aconnector->fake_enable)
-		aconnector->fake_enable = false;
-
-	aconnector->timing_changed = false;
-
-	if (!dc_link_detect_connection_type(aconnector->dc_link, &new_connection_type))
-		drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n");
-
-	/*
-	 * Check for HDMI disconnect with debounce enabled.
-	 */
-	debounce_required = (aconnector->hdmi_hpd_debounce_delay_ms > 0 &&
-			      dc_is_hdmi_signal(aconnector->dc_link->connector_signal) &&
-			      new_connection_type == dc_connection_none &&
-			      aconnector->dc_link->local_sink != NULL);
-
-	if (aconnector->base.force && new_connection_type == dc_connection_none) {
-		emulated_link_detect(aconnector->dc_link);
-
-		drm_modeset_lock_all(dev);
-		dm_restore_drm_connector_state(dev, connector);
-		drm_modeset_unlock_all(dev);
-
-		if (aconnector->base.force == DRM_FORCE_UNSPECIFIED)
-			drm_kms_helper_connector_hotplug_event(connector);
-	} else if (debounce_required) {
-		/*
-		 * HDMI disconnect detected - schedule delayed work instead of
-		 * processing immediately. This allows us to coalesce spurious
-		 * HDMI signals from physical unplugs.
-		 */
-		drm_dbg_kms(dev, "HDMI HPD: Disconnect detected, scheduling debounce work (%u ms)\n",
-			    aconnector->hdmi_hpd_debounce_delay_ms);
-
-		/* Cache the current sink for later comparison */
-		if (aconnector->hdmi_prev_sink)
-			dc_sink_release(aconnector->hdmi_prev_sink);
-		aconnector->hdmi_prev_sink = aconnector->dc_link->local_sink;
-		if (aconnector->hdmi_prev_sink)
-			dc_sink_retain(aconnector->hdmi_prev_sink);
-
-		/* Schedule delayed detection. */
-		if (mod_delayed_work(system_percpu_wq,
-				 &aconnector->hdmi_hpd_debounce_work,
-				 msecs_to_jiffies(aconnector->hdmi_hpd_debounce_delay_ms)))
-			drm_dbg_kms(dev, "HDMI HPD: Re-scheduled debounce work\n");
-
-	} else {
-
-		/* If the aconnector->hdmi_hpd_debounce_work is scheduled, exit early */
-		if (delayed_work_pending(&aconnector->hdmi_hpd_debounce_work))
-			return;
-
-		scoped_guard(mutex, &adev->dm.dc_lock) {
-			dc_exit_ips_for_hw_access(dc);
-			ret = dc_link_detect(aconnector->dc_link, DETECT_REASON_HPD);
-		}
-		if (ret) {
-			/* w/a delay for certain panels */
-			apply_delay_after_dpcd_poweroff(adev, aconnector->dc_sink);
-			amdgpu_dm_update_connector_after_detect(aconnector);
-
-			drm_modeset_lock_all(dev);
-			dm_restore_drm_connector_state(dev, connector);
-			drm_modeset_unlock_all(dev);
-
-			if (aconnector->base.force == DRM_FORCE_UNSPECIFIED)
-				drm_kms_helper_connector_hotplug_event(connector);
-		}
-	}
-}
-
-static void handle_hpd_irq(void *param)
-{
-	struct amdgpu_dm_connector *aconnector = (struct amdgpu_dm_connector *)param;
-
-	handle_hpd_irq_helper(aconnector);
-
-}
-
-static void schedule_hpd_rx_offload_work(struct amdgpu_device *adev, struct hpd_rx_irq_offload_work_queue *offload_wq,
-							union hpd_irq_data hpd_irq_data)
-{
-	struct hpd_rx_irq_offload_work *offload_work = kzalloc_obj(*offload_work);
-
-	if (!offload_work) {
-		drm_err(adev_to_drm(adev), "Failed to allocate hpd_rx_irq_offload_work.\n");
-		return;
-	}
-
-	INIT_WORK(&offload_work->work, dm_handle_hpd_rx_offload_work);
-	offload_work->data = hpd_irq_data;
-	offload_work->offload_wq = offload_wq;
-	offload_work->adev = adev;
-
-	queue_work(offload_wq->wq, &offload_work->work);
-	drm_dbg_kms(adev_to_drm(adev), "queue work to handle hpd_rx offload work");
-}
-
-static void handle_hpd_rx_irq(void *param)
-{
-	struct amdgpu_dm_connector *aconnector = (struct amdgpu_dm_connector *)param;
-	struct drm_connector *connector = &aconnector->base;
-	struct drm_device *dev = connector->dev;
-	struct dc_link *dc_link = aconnector->dc_link;
-	bool is_mst_root_connector = aconnector->mst_mgr.mst_state;
-	bool result = false;
-	enum dc_connection_type new_connection_type = dc_connection_none;
-	struct amdgpu_device *adev = drm_to_adev(dev);
-	union hpd_irq_data hpd_irq_data;
-	bool link_loss = false;
-	bool has_left_work = false;
-	int idx = dc_link->link_index;
-	struct hpd_rx_irq_offload_work_queue *offload_wq = &adev->dm.hpd_rx_offload_wq[idx];
-	struct dc *dc = aconnector->dc_link->ctx->dc;
-
-	memset(&hpd_irq_data, 0, sizeof(hpd_irq_data));
-
-	if (adev->dm.disable_hpd_irq)
-		return;
-
-	/*
-	 * TODO:Temporary add mutex to protect hpd interrupt not have a gpio
-	 * conflict, after implement i2c helper, this mutex should be
-	 * retired.
-	 */
-	mutex_lock(&aconnector->hpd_lock);
-
-	result = dc_link_handle_hpd_rx_irq(dc_link, &hpd_irq_data,
-						&link_loss, true, &has_left_work);
-
-	if (!has_left_work)
-		goto out;
-
-	if (hpd_irq_data.bytes.device_service_irq.bits.AUTOMATED_TEST) {
-		schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data);
-		goto out;
-	}
-
-	if (dc_link_dp_allow_hpd_rx_irq(dc_link)) {
-		if (hpd_irq_data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY ||
-			hpd_irq_data.bytes.device_service_irq.bits.DOWN_REP_MSG_RDY) {
-			bool skip = false;
-
-			/*
-			 * DOWN_REP_MSG_RDY is also handled by polling method
-			 * mgr->cbs->poll_hpd_irq()
-			 */
-			spin_lock(&offload_wq->offload_lock);
-			skip = offload_wq->is_handling_mst_msg_rdy_event;
-
-			if (!skip)
-				offload_wq->is_handling_mst_msg_rdy_event = true;
-
-			spin_unlock(&offload_wq->offload_lock);
-
-			if (!skip)
-				schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data);
-
-			goto out;
-		}
-
-		if (link_loss) {
-			bool skip = false;
-
-			spin_lock(&offload_wq->offload_lock);
-			skip = offload_wq->is_handling_link_loss;
-
-			if (!skip)
-				offload_wq->is_handling_link_loss = true;
-
-			spin_unlock(&offload_wq->offload_lock);
-
-			if (!skip)
-				schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data);
-
-			goto out;
-		}
-	}
-
-out:
-	if (result && !is_mst_root_connector) {
-		/* Downstream Port status changed. */
-		if (!dc_link_detect_connection_type(dc_link, &new_connection_type))
-			drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n");
-
-		if (aconnector->base.force && new_connection_type == dc_connection_none) {
-			emulated_link_detect(dc_link);
-
-			if (aconnector->fake_enable)
-				aconnector->fake_enable = false;
-
-			amdgpu_dm_update_connector_after_detect(aconnector);
-
-
-			drm_modeset_lock_all(dev);
-			dm_restore_drm_connector_state(dev, connector);
-			drm_modeset_unlock_all(dev);
-
-			drm_kms_helper_connector_hotplug_event(connector);
-		} else {
-			bool ret = false;
-
-			mutex_lock(&adev->dm.dc_lock);
-			dc_exit_ips_for_hw_access(dc);
-			ret = dc_link_detect(dc_link, DETECT_REASON_HPDRX);
-			mutex_unlock(&adev->dm.dc_lock);
-
-			if (ret) {
-				if (aconnector->fake_enable)
-					aconnector->fake_enable = false;
-
-				amdgpu_dm_update_connector_after_detect(aconnector);
-
-				drm_modeset_lock_all(dev);
-				dm_restore_drm_connector_state(dev, connector);
-				drm_modeset_unlock_all(dev);
-
-				drm_kms_helper_connector_hotplug_event(connector);
-			}
-		}
-	}
-	if (hpd_irq_data.bytes.device_service_irq.bits.CP_IRQ) {
-		if (adev->dm.hdcp_workqueue)
-			hdcp_handle_cpirq(adev->dm.hdcp_workqueue,  aconnector->base.index);
-	}
-
-	if (dc_link->type != dc_connection_mst_branch)
-		drm_dp_cec_irq(&aconnector->dm_dp_aux.aux);
-
-	mutex_unlock(&aconnector->hpd_lock);
-}
-
-static int register_hpd_handlers(struct amdgpu_device *adev)
-{
-	struct drm_device *dev = adev_to_drm(adev);
-	struct drm_connector *connector;
-	struct amdgpu_dm_connector *aconnector;
-	const struct dc_link *dc_link;
-	struct dc_interrupt_params int_params = {0};
-
-	int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
-	int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
-
-	if (dc_is_dmub_outbox_supported(adev->dm.dc)) {
-		if (!register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD,
-			dmub_hpd_callback, true)) {
-			drm_err(adev_to_drm(adev), "fail to register dmub hpd callback");
-			return -EINVAL;
-		}
-
-		if (!register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD_IRQ,
-			dmub_hpd_callback, true)) {
-			drm_err(adev_to_drm(adev), "fail to register dmub hpd callback");
-			return -EINVAL;
-		}
-
-		if (!register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD_SENSE_NOTIFY,
-			dmub_hpd_sense_callback, true)) {
-			drm_err(adev_to_drm(adev), "fail to register dmub hpd sense callback");
-			return -EINVAL;
-		}
-	}
-
-	list_for_each_entry(connector,
-			&dev->mode_config.connector_list, head)	{
-
-		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
-			continue;
-
-		aconnector = to_amdgpu_dm_connector(connector);
-		dc_link = aconnector->dc_link;
-
-		if (dc_link->irq_source_hpd != DC_IRQ_SOURCE_INVALID) {
-			int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
-			int_params.irq_source = dc_link->irq_source_hpd;
-
-			if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
-				int_params.irq_source  < DC_IRQ_SOURCE_HPD1 ||
-				int_params.irq_source  > DC_IRQ_SOURCE_HPD6) {
-				drm_err(adev_to_drm(adev), "Failed to register hpd irq!\n");
-				return -EINVAL;
-			}
-
-			if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
-				handle_hpd_irq, (void *) aconnector))
-				return -ENOMEM;
-		}
-
-		if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) {
-
-			/* Also register for DP short pulse (hpd_rx). */
-			int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
-			int_params.irq_source =	dc_link->irq_source_hpd_rx;
-
-			if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
-				int_params.irq_source  < DC_IRQ_SOURCE_HPD1RX ||
-				int_params.irq_source  > DC_IRQ_SOURCE_HPD6RX) {
-				drm_err(adev_to_drm(adev), "Failed to register hpd rx irq!\n");
-				return -EINVAL;
-			}
-
-			if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
-				handle_hpd_rx_irq, (void *) aconnector))
-				return -ENOMEM;
-		}
-	}
-	return 0;
-}
-
-/* Register IRQ sources and initialize IRQ callbacks */
-static int dce110_register_irq_handlers(struct amdgpu_device *adev)
-{
-	struct dc *dc = adev->dm.dc;
-	struct common_irq_params *c_irq_params;
-	struct dc_interrupt_params int_params = {0};
-	int r;
-	int i;
-	unsigned int src_id;
-	unsigned int client_id = AMDGPU_IRQ_CLIENTID_LEGACY;
-	/* Use different interrupts for VBLANK on DCE 6 vs. newer. */
-	const unsigned int vblank_d1 =
-		adev->dm.dc->ctx->dce_version >= DCE_VERSION_8_0
-		? VISLANDS30_IV_SRCID_D1_VERTICAL_INTERRUPT0 : 1;
-
-	if (adev->family >= AMDGPU_FAMILY_AI)
-		client_id = SOC15_IH_CLIENTID_DCE;
-
-	int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
-	int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
-
-	/*
-	 * Actions of amdgpu_irq_add_id():
-	 * 1. Register a set() function with base driver.
-	 *    Base driver will call set() function to enable/disable an
-	 *    interrupt in DC hardware.
-	 * 2. Register amdgpu_dm_irq_handler().
-	 *    Base driver will call amdgpu_dm_irq_handler() for ALL interrupts
-	 *    coming from DC hardware.
-	 *    amdgpu_dm_irq_handler() will re-direct the interrupt to DC
-	 *    for acknowledging and handling.
-	 */
-
-	/* Use VBLANK interrupt */
-	for (i = 0; i < adev->mode_info.num_crtc; i++) {
-		src_id = vblank_d1 + i;
-		r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->crtc_irq);
-		if (r) {
-			drm_err(adev_to_drm(adev), "Failed to add crtc irq id!\n");
-			return r;
-		}
-
-		int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
-		int_params.irq_source =
-			dc_interrupt_to_irq_source(dc, src_id, 0);
-
-		if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
-			int_params.irq_source  < DC_IRQ_SOURCE_VBLANK1 ||
-			int_params.irq_source  > DC_IRQ_SOURCE_VBLANK6) {
-			drm_err(adev_to_drm(adev), "Failed to register vblank irq!\n");
-			return -EINVAL;
-		}
-
-		c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1];
-
-		c_irq_params->adev = adev;
-		c_irq_params->irq_src = int_params.irq_source;
-
-		if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
-			dm_crtc_high_irq, c_irq_params))
-			return -ENOMEM;
-	}
-
-	if (dc_supports_vrr(adev->dm.dc->ctx->dce_version)) {
-		/* Use VUPDATE interrupt */
-		for (i = 0; i < adev->mode_info.num_crtc; i++) {
-			src_id = VISLANDS30_IV_SRCID_D1_V_UPDATE_INT + i * 2;
-			r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->vupdate_irq);
-			if (r) {
-				drm_err(adev_to_drm(adev), "Failed to add vupdate irq id!\n");
-				return r;
-			}
-
-			int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
-			int_params.irq_source =
-				dc_interrupt_to_irq_source(dc, src_id, 0);
-
-			if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
-				int_params.irq_source  < DC_IRQ_SOURCE_VUPDATE1 ||
-				int_params.irq_source  > DC_IRQ_SOURCE_VUPDATE6) {
-				drm_err(adev_to_drm(adev), "Failed to register vupdate irq!\n");
-				return -EINVAL;
-			}
-
-			c_irq_params = &adev->dm.vupdate_params[
-				int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1];
-			c_irq_params->adev = adev;
-			c_irq_params->irq_src = int_params.irq_source;
-
-			if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
-				dm_vupdate_high_irq, c_irq_params))
-				return -ENOMEM;
-		}
-	}
-
-	/* Use GRPH_PFLIP interrupt */
-	for (i = VISLANDS30_IV_SRCID_D1_GRPH_PFLIP;
-			i <= VISLANDS30_IV_SRCID_D6_GRPH_PFLIP; i += 2) {
-		r = amdgpu_irq_add_id(adev, client_id, i, &adev->pageflip_irq);
-		if (r) {
-			drm_err(adev_to_drm(adev), "Failed to add page flip irq id!\n");
-			return r;
-		}
-
-		int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
-		int_params.irq_source =
-			dc_interrupt_to_irq_source(dc, i, 0);
-
-		if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
-			int_params.irq_source  < DC_IRQ_SOURCE_PFLIP_FIRST ||
-			int_params.irq_source  > DC_IRQ_SOURCE_PFLIP_LAST) {
-			drm_err(adev_to_drm(adev), "Failed to register pflip irq!\n");
-			return -EINVAL;
-		}
-
-		c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST];
-
-		c_irq_params->adev = adev;
-		c_irq_params->irq_src = int_params.irq_source;
-
-		if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
-			dm_pflip_high_irq, c_irq_params))
-			return -ENOMEM;
-	}
-
-	/* HPD */
-	r = amdgpu_irq_add_id(adev, client_id,
-			VISLANDS30_IV_SRCID_HOTPLUG_DETECT_A, &adev->hpd_irq);
-	if (r) {
-		drm_err(adev_to_drm(adev), "Failed to add hpd irq id!\n");
-		return r;
-	}
-
-	r = register_hpd_handlers(adev);
-
-	return r;
-}
-
-/* Register IRQ sources and initialize IRQ callbacks */
-static int dcn10_register_irq_handlers(struct amdgpu_device *adev)
-{
-	struct dc *dc = adev->dm.dc;
-	struct common_irq_params *c_irq_params;
-	struct dc_interrupt_params int_params = {0};
-	int r;
-	int i;
-#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
-	static const unsigned int vrtl_int_srcid[] = {
-		DCN_1_0__SRCID__OTG1_VERTICAL_INTERRUPT0_CONTROL,
-		DCN_1_0__SRCID__OTG2_VERTICAL_INTERRUPT0_CONTROL,
-		DCN_1_0__SRCID__OTG3_VERTICAL_INTERRUPT0_CONTROL,
-		DCN_1_0__SRCID__OTG4_VERTICAL_INTERRUPT0_CONTROL,
-		DCN_1_0__SRCID__OTG5_VERTICAL_INTERRUPT0_CONTROL,
-		DCN_1_0__SRCID__OTG6_VERTICAL_INTERRUPT0_CONTROL
-	};
-#endif
-
-	int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
-	int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
-
-	/*
-	 * Actions of amdgpu_irq_add_id():
-	 * 1. Register a set() function with base driver.
-	 *    Base driver will call set() function to enable/disable an
-	 *    interrupt in DC hardware.
-	 * 2. Register amdgpu_dm_irq_handler().
-	 *    Base driver will call amdgpu_dm_irq_handler() for ALL interrupts
-	 *    coming from DC hardware.
-	 *    amdgpu_dm_irq_handler() will re-direct the interrupt to DC
-	 *    for acknowledging and handling.
-	 */
-
-	/* Use VSTARTUP interrupt */
-	for (i = DCN_1_0__SRCID__DC_D1_OTG_VSTARTUP;
-			i <= DCN_1_0__SRCID__DC_D1_OTG_VSTARTUP + adev->mode_info.num_crtc - 1;
-			i++) {
-		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->crtc_irq);
-
-		if (r) {
-			drm_err(adev_to_drm(adev), "Failed to add crtc irq id!\n");
-			return r;
-		}
-
-		int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
-		int_params.irq_source =
-			dc_interrupt_to_irq_source(dc, i, 0);
-
-		if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
-			int_params.irq_source  < DC_IRQ_SOURCE_VBLANK1 ||
-			int_params.irq_source  > DC_IRQ_SOURCE_VBLANK6) {
-			drm_err(adev_to_drm(adev), "Failed to register vblank irq!\n");
-			return -EINVAL;
-		}
-
-		c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1];
-
-		c_irq_params->adev = adev;
-		c_irq_params->irq_src = int_params.irq_source;
-
-		if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
-			dm_crtc_high_irq, c_irq_params))
-			return -ENOMEM;
-	}
-
-	/* Use otg vertical line interrupt */
-#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
-	for (i = 0; i <= adev->mode_info.num_crtc - 1; i++) {
-		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE,
-				vrtl_int_srcid[i], &adev->vline0_irq);
-
-		if (r) {
-			drm_err(adev_to_drm(adev), "Failed to add vline0 irq id!\n");
-			return r;
-		}
-
-		int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
-		int_params.irq_source =
-			dc_interrupt_to_irq_source(dc, vrtl_int_srcid[i], 0);
-
-		if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
-			int_params.irq_source < DC_IRQ_SOURCE_DC1_VLINE0 ||
-			int_params.irq_source > DC_IRQ_SOURCE_DC6_VLINE0) {
-			drm_err(adev_to_drm(adev), "Failed to register vline0 irq!\n");
-			return -EINVAL;
-		}
-
-		c_irq_params = &adev->dm.vline0_params[int_params.irq_source
-					- DC_IRQ_SOURCE_DC1_VLINE0];
-
-		c_irq_params->adev = adev;
-		c_irq_params->irq_src = int_params.irq_source;
-
-		if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
-			dm_dcn_vertical_interrupt0_high_irq,
-			c_irq_params))
-			return -ENOMEM;
-	}
-#endif
-
-	/* Use VUPDATE_NO_LOCK interrupt on DCN, which seems to correspond to
-	 * the regular VUPDATE interrupt on DCE. We want DC_IRQ_SOURCE_VUPDATEx
-	 * to trigger at end of each vblank, regardless of state of the lock,
-	 * matching DCE behaviour.
-	 */
-	for (i = DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT;
-	     i <= DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT + adev->mode_info.num_crtc - 1;
-	     i++) {
-		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->vupdate_irq);
-
-		if (r) {
-			drm_err(adev_to_drm(adev), "Failed to add vupdate irq id!\n");
-			return r;
-		}
-
-		int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
-		int_params.irq_source =
-			dc_interrupt_to_irq_source(dc, i, 0);
-
-		if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
-			int_params.irq_source  < DC_IRQ_SOURCE_VUPDATE1 ||
-			int_params.irq_source  > DC_IRQ_SOURCE_VUPDATE6) {
-			drm_err(adev_to_drm(adev), "Failed to register vupdate irq!\n");
-			return -EINVAL;
-		}
-
-		c_irq_params = &adev->dm.vupdate_params[int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1];
-
-		c_irq_params->adev = adev;
-		c_irq_params->irq_src = int_params.irq_source;
-
-		if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
-			dm_vupdate_high_irq, c_irq_params))
-			return -ENOMEM;
-	}
-
-	/* Use GRPH_PFLIP interrupt */
-	for (i = DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT;
-			i <= DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT + dc->caps.max_otg_num - 1;
-			i++) {
-		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->pageflip_irq);
-		if (r) {
-			drm_err(adev_to_drm(adev), "Failed to add page flip irq id!\n");
-			return r;
-		}
-
-		int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
-		int_params.irq_source =
-			dc_interrupt_to_irq_source(dc, i, 0);
-
-		if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
-			int_params.irq_source  < DC_IRQ_SOURCE_PFLIP_FIRST ||
-			int_params.irq_source  > DC_IRQ_SOURCE_PFLIP_LAST) {
-			drm_err(adev_to_drm(adev), "Failed to register pflip irq!\n");
-			return -EINVAL;
-		}
-
-		c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST];
-
-		c_irq_params->adev = adev;
-		c_irq_params->irq_src = int_params.irq_source;
-
-		if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
-			dm_pflip_high_irq, c_irq_params))
-			return -ENOMEM;
-	}
-
-	/* HPD */
-	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DC_HPD1_INT,
-			&adev->hpd_irq);
-	if (r) {
-		drm_err(adev_to_drm(adev), "Failed to add hpd irq id!\n");
-		return r;
-	}
-
-	r = register_hpd_handlers(adev);
-
-	return r;
-}
-/* Register Outbox IRQ sources and initialize IRQ callbacks */
-static int register_outbox_irq_handlers(struct amdgpu_device *adev)
-{
-	struct dc *dc = adev->dm.dc;
-	struct common_irq_params *c_irq_params;
-	struct dc_interrupt_params int_params = {0};
-	int r, i;
-
-	int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
-	int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
-
-	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT,
-			&adev->dmub_outbox_irq);
-	if (r) {
-		drm_err(adev_to_drm(adev), "Failed to add outbox irq id!\n");
-		return r;
-	}
-
-	if (dc->ctx->dmub_srv) {
-		i = DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT;
-		int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
-		int_params.irq_source =
-		dc_interrupt_to_irq_source(dc, i, 0);
-
-		c_irq_params = &adev->dm.dmub_outbox_params[0];
-
-		c_irq_params->adev = adev;
-		c_irq_params->irq_src = int_params.irq_source;
-
-		if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
-			dm_dmub_outbox1_low_irq, c_irq_params))
-			return -ENOMEM;
-	}
-
-	return 0;
-}
-
-/*
- * Acquires the lock for the atomic state object and returns
- * the new atomic state.
- *
- * This should only be called during atomic check.
- */
-int dm_atomic_get_state(struct drm_atomic_commit *state,
-			struct dm_atomic_state **dm_state)
-{
-	struct drm_device *dev = state->dev;
-	struct amdgpu_device *adev = drm_to_adev(dev);
-	struct amdgpu_display_manager *dm = &adev->dm;
-	struct drm_private_state *priv_state;
-
-	if (*dm_state)
-		return 0;
-
-	priv_state = drm_atomic_get_private_obj_state(state, &dm->atomic_obj);
-	if (IS_ERR(priv_state))
-		return PTR_ERR(priv_state);
-
-	*dm_state = to_dm_atomic_state(priv_state);
-
-	return 0;
-}
-
-static struct dm_atomic_state *
-dm_atomic_get_new_state(struct drm_atomic_commit *state)
-{
-	struct drm_device *dev = state->dev;
-	struct amdgpu_device *adev = drm_to_adev(dev);
-	struct amdgpu_display_manager *dm = &adev->dm;
-	struct drm_private_obj *obj;
-	struct drm_private_state *new_obj_state;
-	int i;
-
-	for_each_new_private_obj_in_state(state, obj, new_obj_state, i) {
-		if (obj->funcs == dm->atomic_obj.funcs)
-			return to_dm_atomic_state(new_obj_state);
-	}
-
-	return NULL;
-}
-
-static struct drm_private_state *
-dm_atomic_duplicate_state(struct drm_private_obj *obj)
-{
-	struct dm_atomic_state *old_state, *new_state;
-
-	new_state = kzalloc_obj(*new_state);
-	if (!new_state)
-		return NULL;
-
-	__drm_atomic_helper_private_obj_duplicate_state(obj, &new_state->base);
-
-	old_state = to_dm_atomic_state(obj->state);
-
-	if (old_state && old_state->context)
-		new_state->context = dc_state_create_copy(old_state->context);
-
-	if (!new_state->context) {
-		kfree(new_state);
-		return NULL;
-	}
-
-	return &new_state->base;
-}
-
-static void dm_atomic_destroy_state(struct drm_private_obj *obj,
-				    struct drm_private_state *state)
-{
-	struct dm_atomic_state *dm_state = to_dm_atomic_state(state);
-
-	if (dm_state && dm_state->context)
-		dc_state_release(dm_state->context);
-
-	kfree(dm_state);
-}
-
-static struct drm_private_state *
-dm_atomic_create_state(struct drm_private_obj *obj)
-{
-	struct amdgpu_device *adev = drm_to_adev(obj->dev);
-	struct dm_atomic_state *dm_state;
-	struct dc_state *context;
-
-	dm_state = kzalloc_obj(*dm_state);
-	if (!dm_state)
-		return ERR_PTR(-ENOMEM);
-
-	context = dc_state_create_current_copy(adev->dm.dc);
-	if (!context) {
-		kfree(dm_state);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	__drm_atomic_helper_private_obj_create_state(obj, &dm_state->base);
-	dm_state->context = context;
-
-	return &dm_state->base;
-}
-
-static struct drm_private_state_funcs dm_atomic_state_funcs = {
-	.atomic_create_state = dm_atomic_create_state,
-	.atomic_duplicate_state = dm_atomic_duplicate_state,
-	.atomic_destroy_state = dm_atomic_destroy_state,
-};
-
-static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev)
-{
-	int r;
-
-	adev->mode_info.mode_config_initialized = true;
-
-	adev_to_drm(adev)->mode_config.funcs = (void *)&amdgpu_dm_mode_funcs;
-	adev_to_drm(adev)->mode_config.helper_private = &amdgpu_dm_mode_config_helperfuncs;
-
-	adev_to_drm(adev)->mode_config.max_width = 16384;
-	adev_to_drm(adev)->mode_config.max_height = 16384;
-
-	adev_to_drm(adev)->mode_config.preferred_depth = 24;
-	if (adev->asic_type == CHIP_HAWAII)
-		/* disable prefer shadow for now due to hibernation issues */
-		adev_to_drm(adev)->mode_config.prefer_shadow = 0;
-	else
-		adev_to_drm(adev)->mode_config.prefer_shadow = 1;
-	/* indicates support for immediate flip */
-	adev_to_drm(adev)->mode_config.async_page_flip = true;
-
-	drm_atomic_private_obj_init(adev_to_drm(adev),
-				    &adev->dm.atomic_obj,
-				    &dm_atomic_state_funcs);
-
-	r = amdgpu_display_modeset_create_props(adev);
-	if (r)
-		return r;
-
-#ifdef AMD_PRIVATE_COLOR
-	if (amdgpu_dm_create_color_properties(adev))
-		return -ENOMEM;
-#endif
-
-	r = amdgpu_dm_audio_init(adev);
-	if (r)
-		return r;
-
-	return 0;
-}
-
-#define AMDGPU_DM_DEFAULT_MIN_BACKLIGHT 12
-#define AMDGPU_DM_DEFAULT_MAX_BACKLIGHT 255
-#define AMDGPU_DM_MIN_SPREAD ((AMDGPU_DM_DEFAULT_MAX_BACKLIGHT - AMDGPU_DM_DEFAULT_MIN_BACKLIGHT) / 2)
-#define AUX_BL_DEFAULT_TRANSITION_TIME_MS 50
-
-void amdgpu_dm_update_backlight_caps(struct amdgpu_display_manager *dm,
-				     int bl_idx)
-{
-	struct amdgpu_dm_backlight_caps *caps = &dm->backlight_caps[bl_idx];
-
-	if (caps->caps_valid)
-		return;
-
-#if defined(CONFIG_ACPI)
-	amdgpu_acpi_get_backlight_caps(caps);
-
-	/* validate the firmware value is sane */
-	if (caps->caps_valid) {
-		int spread = caps->max_input_signal - caps->min_input_signal;
-
-		if (caps->max_input_signal > AMDGPU_DM_DEFAULT_MAX_BACKLIGHT ||
-		    caps->min_input_signal < 0 ||
-		    spread > AMDGPU_DM_DEFAULT_MAX_BACKLIGHT ||
-		    spread < AMDGPU_DM_MIN_SPREAD) {
-			drm_dbg_kms(adev_to_drm(dm->adev), "DM: Invalid backlight caps: min=%d, max=%d\n",
-				      caps->min_input_signal, caps->max_input_signal);
-			caps->caps_valid = false;
-		}
-	}
-
-	if (!caps->caps_valid) {
-		caps->min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT;
-		caps->max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
-		caps->caps_valid = true;
-	}
-#else
-	if (caps->aux_support)
-		return;
-
-	caps->min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT;
-	caps->max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
-	caps->caps_valid = true;
-#endif
-}
-
-static int get_brightness_range(const struct amdgpu_dm_backlight_caps *caps,
-				unsigned int *min, unsigned int *max)
-{
-	if (!caps)
-		return 0;
-
-	if (caps->aux_support) {
-		// Firmware limits are in nits, DC API wants millinits.
-		*max = 1000 * caps->aux_max_input_signal;
-		*min = 1000 * caps->aux_min_input_signal;
-	} else {
-		// Firmware limits are 8-bit, PWM control is 16-bit.
-		*max = 0x101 * caps->max_input_signal;
-		*min = 0x101 * caps->min_input_signal;
-	}
-	return 1;
-}
-
-/* Rescale from [min..max] to [0..AMDGPU_MAX_BL_LEVEL] */
-static inline u32 scale_input_to_fw(int min, int max, u64 input)
-{
-	return DIV_ROUND_CLOSEST_ULL(input * AMDGPU_MAX_BL_LEVEL, max - min);
-}
-
-/* Rescale from [0..AMDGPU_MAX_BL_LEVEL] to [min..max] */
-static inline u32 scale_fw_to_input(int min, int max, u64 input)
-{
-	return min + DIV_ROUND_CLOSEST_ULL(input * (max - min), AMDGPU_MAX_BL_LEVEL);
-}
-
-static void convert_custom_brightness(const struct amdgpu_dm_backlight_caps *caps,
-				      unsigned int min, unsigned int max,
-				      uint32_t *user_brightness)
-{
-	u32 brightness = scale_input_to_fw(min, max, *user_brightness);
-	u8 lower_signal, upper_signal, upper_lum, lower_lum, lum;
-	int left, right;
-
-	if (amdgpu_dc_debug_mask & DC_DISABLE_CUSTOM_BRIGHTNESS_CURVE)
-		return;
-
-	if (!caps->data_points)
-		return;
-
-	/*
-	 * Handle the case where brightness is below the first data point
-	 * Interpolate between (0,0) and (first_signal, first_lum)
-	 */
-	if (brightness < caps->luminance_data[0].input_signal) {
-		lum = DIV_ROUND_CLOSEST(caps->luminance_data[0].luminance * brightness,
-					caps->luminance_data[0].input_signal);
-		goto scale;
-	}
-
-	left = 0;
-	right = caps->data_points - 1;
-	while (left <= right) {
-		int mid = left + (right - left) / 2;
-		u8 signal = caps->luminance_data[mid].input_signal;
-
-		/* Exact match found */
-		if (signal == brightness) {
-			lum = caps->luminance_data[mid].luminance;
-			goto scale;
-		}
-
-		if (signal < brightness)
-			left = mid + 1;
-		else
-			right = mid - 1;
-	}
-
-	/* verify bound */
-	if (left >= caps->data_points)
-		left = caps->data_points - 1;
-
-	/* At this point, left > right */
-	lower_signal = caps->luminance_data[right].input_signal;
-	upper_signal = caps->luminance_data[left].input_signal;
-	lower_lum = caps->luminance_data[right].luminance;
-	upper_lum = caps->luminance_data[left].luminance;
-
-	/* interpolate */
-	if (right == left || !lower_lum)
-		lum = upper_lum;
-	else
-		lum = lower_lum + DIV_ROUND_CLOSEST((upper_lum - lower_lum) *
-						    (brightness - lower_signal),
-						    upper_signal - lower_signal);
-scale:
-	*user_brightness = scale_fw_to_input(min, max,
-					     DIV_ROUND_CLOSEST(lum * brightness, 101));
-}
-
-static u32 convert_brightness_from_user(const struct amdgpu_dm_backlight_caps *caps,
-					uint32_t brightness)
-{
-	unsigned int min, max;
-
-	if (!get_brightness_range(caps, &min, &max))
-		return brightness;
-
-	convert_custom_brightness(caps, min, max, &brightness);
-
-	// Rescale 0..max to min..max
-	return min + DIV_ROUND_CLOSEST_ULL((u64)(max - min) * brightness, max);
-}
-
-static u32 convert_brightness_to_user(const struct amdgpu_dm_backlight_caps *caps,
-				      uint32_t brightness)
-{
-	unsigned int min, max;
-
-	if (!get_brightness_range(caps, &min, &max))
-		return brightness;
-
-	if (brightness < min)
-		return 0;
-	// Rescale min..max to 0..max
-	return DIV_ROUND_CLOSEST_ULL((u64)max * (brightness - min),
-				 max - min);
-}
-
-static struct dc_stream_state *dm_find_stream_with_link(
-	struct amdgpu_display_manager *dm,
-	struct dc_link *link)
-{
-	struct dc_state *cur_dc_state = dm->dc->current_state;
-	struct dc_stream_state *stream = NULL;
-	int i;
-
-	for (i = 0; i < cur_dc_state->stream_count; i++) {
-		stream = cur_dc_state->streams[i];
-		if (stream->link == link)
-			return stream;
-	}
-
-	return NULL;
-}
-
-static void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm,
-					 int bl_idx,
-					 u32 user_brightness)
-{
-	struct amdgpu_dm_backlight_caps *caps;
-	struct dc_link *link;
-	u32 brightness = 0;
-	bool rc = false, reallow_idle = false;
-	struct drm_connector *connector;
-	struct dc_stream_state *stream;
-	unsigned int min, max;
-
-	list_for_each_entry(connector, &dm->ddev->mode_config.connector_list, head) {
-		struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
-
-		if (aconnector->bl_idx != bl_idx)
-			continue;
-
-		/* if connector is off, save the brightness for next time it's on */
-		if (!aconnector->base.encoder) {
-			dm->brightness[bl_idx] = user_brightness;
-			dm->actual_brightness[bl_idx] = 0;
-			return;
-		}
-	}
-
-	amdgpu_dm_update_backlight_caps(dm, bl_idx);
-	caps = &dm->backlight_caps[bl_idx];
-
-	dm->brightness[bl_idx] = user_brightness;
-	/* update scratch register */
-	if (bl_idx == 0)
-		amdgpu_atombios_scratch_regs_set_backlight_level(dm->adev, dm->brightness[bl_idx]);
-	brightness = convert_brightness_from_user(caps, dm->brightness[bl_idx]);
-	link = (struct dc_link *)dm->backlight_link[bl_idx];
-
-	/* Apply brightness quirk */
-	if (caps->brightness_mask)
-		brightness |= caps->brightness_mask;
-
-	if (trace_amdgpu_dm_brightness_enabled()) {
-		trace_amdgpu_dm_brightness(__builtin_return_address(0),
-					   user_brightness,
-					   brightness,
-					   caps->aux_support,
-					   power_supply_is_system_supplied() > 0);
-	}
-
-	stream = dm_find_stream_with_link(dm, link);
-	if (!stream)
-		return;
-
-	mutex_lock(&dm->dc_lock);
-	if (dm->dc->caps.ips_support && dm->dc->ctx->dmub_srv->idle_allowed) {
-		dc_allow_idle_optimizations(dm->dc, false);
-		reallow_idle = true;
-	}
-
-	if (caps->aux_support) {
-		rc = mod_power_set_backlight_nits(dm->power_module, stream, brightness,
-			AUX_BL_DEFAULT_TRANSITION_TIME_MS, false, true);
-	} else {
-		/* power module uses millipercent */
-		get_brightness_range(caps, &min, &max);
-		brightness = DIV_ROUND_CLOSEST(brightness * 100, (max - min)) * 1000;
-		rc = mod_power_set_backlight_percent(dm->power_module, stream,
-						     brightness, 0, false);
-	}
-
-	/*
-	 * Some kms clients create a ramped backlight transition effect
-	 * by rapidly changing the backlight. Yet we must wait on dmcub
-	 * fw to exit psr/replay before programming backlight. To
-	 * prevent lag, keep disable psr/replay and let the next atomic
-	 * flip clear the event.
-	 *
-	 * ToDo: use ISM to handle rapidly backlight change
-	 *
-	 * Rapidly backlight change is similar to rapidly cursor events,
-	 * which is now handled by ISM. ISM can delay the event until system
-	 * is really idle, so we may use ISM to handle backlight change as well.
-	 */
-	amdgpu_dm_psr_set_event(dm, stream, true,
-		psr_event_hw_programming, true);
-	amdgpu_dm_replay_set_event(dm, stream, true,
-		replay_event_hw_programming, true);
-
-	if (dm->dc->caps.ips_support && reallow_idle)
-		dc_allow_idle_optimizations(dm->dc, true);
-
-	mutex_unlock(&dm->dc_lock);
-
-	if (rc)
-		dm->actual_brightness[bl_idx] = user_brightness;
-}
-
-static int amdgpu_dm_backlight_update_status(struct backlight_device *bd)
-{
-	struct amdgpu_display_manager *dm = bl_get_data(bd);
-	int i;
-
-	for (i = 0; i < dm->num_of_edps; i++) {
-		if (bd == dm->backlight_dev[i])
-			break;
-	}
-	if (i >= AMDGPU_DM_MAX_NUM_EDP)
-		i = 0;
-	amdgpu_dm_backlight_set_level(dm, i, bd->props.brightness);
-
-	return 0;
-}
-
-static u32 amdgpu_dm_backlight_get_level(struct amdgpu_display_manager *dm,
-					 int bl_idx)
-{
-	int ret;
-	struct amdgpu_dm_backlight_caps caps;
-	struct dc_link *link = (struct dc_link *)dm->backlight_link[bl_idx];
-
-	amdgpu_dm_update_backlight_caps(dm, bl_idx);
-	caps = dm->backlight_caps[bl_idx];
-
-	if (caps.aux_support) {
-		u32 avg, peak;
-
-		if (!dc_link_get_backlight_level_nits(link, &avg, &peak))
-			return dm->brightness[bl_idx];
-		return convert_brightness_to_user(&caps, avg);
-	}
-
-	ret = dc_link_get_backlight_level(link);
-
-	if (ret == DC_ERROR_UNEXPECTED)
-		return dm->brightness[bl_idx];
-
-	return convert_brightness_to_user(&caps, ret);
-}
-
-static int amdgpu_dm_backlight_get_brightness(struct backlight_device *bd)
-{
-	struct amdgpu_display_manager *dm = bl_get_data(bd);
-	int i;
-
-	for (i = 0; i < dm->num_of_edps; i++) {
-		if (bd == dm->backlight_dev[i])
-			break;
-	}
-	if (i >= AMDGPU_DM_MAX_NUM_EDP)
-		i = 0;
-	return amdgpu_dm_backlight_get_level(dm, i);
-}
-
-static const struct backlight_ops amdgpu_dm_backlight_ops = {
-	.options = BL_CORE_SUSPENDRESUME,
-	.get_brightness = amdgpu_dm_backlight_get_brightness,
-	.update_status	= amdgpu_dm_backlight_update_status,
-};
-
-static void
-amdgpu_dm_register_backlight_device(struct amdgpu_dm_connector *aconnector)
-{
-	struct drm_device *drm = aconnector->base.dev;
-	struct amdgpu_display_manager *dm = &drm_to_adev(drm)->dm;
-	struct backlight_properties props = { 0 };
-	struct amdgpu_dm_backlight_caps *caps;
-	char bl_name[16];
-	int min, max;
-	int real_brightness;
-	int init_brightness;
-
-	if (aconnector->bl_idx == -1)
-		return;
-
-	if (!acpi_video_backlight_use_native()) {
-		drm_info(drm, "Skipping amdgpu DM backlight registration\n");
-		/* Try registering an ACPI video backlight device instead. */
-		acpi_video_register_backlight();
-		return;
-	}
-
-	caps = &dm->backlight_caps[aconnector->bl_idx];
-	if (get_brightness_range(caps, &min, &max)) {
-		if (power_supply_is_system_supplied() > 0)
-			props.brightness = DIV_ROUND_CLOSEST((max - min) * caps->ac_level, 100);
-		else
-			props.brightness = DIV_ROUND_CLOSEST((max - min) * caps->dc_level, 100);
-		/* min is zero, so max needs to be adjusted */
-		props.max_brightness = max - min;
-		drm_dbg(drm, "Backlight caps: min: %d, max: %d, ac %d, dc %d\n", min, max,
-			caps->ac_level, caps->dc_level);
-	} else
-		props.brightness = props.max_brightness = MAX_BACKLIGHT_LEVEL;
-
-	init_brightness = props.brightness;
-
-	if (caps->data_points && !(amdgpu_dc_debug_mask & DC_DISABLE_CUSTOM_BRIGHTNESS_CURVE)) {
-		drm_info(drm, "Using custom brightness curve\n");
-		props.scale = BACKLIGHT_SCALE_NON_LINEAR;
-	} else
-		props.scale = BACKLIGHT_SCALE_LINEAR;
-	props.type = BACKLIGHT_RAW;
-
-	snprintf(bl_name, sizeof(bl_name), "amdgpu_bl%d",
-		 drm->primary->index + aconnector->bl_idx);
-
-	dm->backlight_dev[aconnector->bl_idx] =
-		backlight_device_register(bl_name, aconnector->base.kdev, dm,
-					  &amdgpu_dm_backlight_ops, &props);
-	dm->brightness[aconnector->bl_idx] = props.brightness;
-
-	if (IS_ERR(dm->backlight_dev[aconnector->bl_idx])) {
-		drm_err(drm, "DM: Backlight registration failed!\n");
-		dm->backlight_dev[aconnector->bl_idx] = NULL;
-	} else {
-		/*
-		 * dm->brightness[x] can be inconsistent just after startup until
-		 * ops.get_brightness is called.
-		 */
-		real_brightness =
-			amdgpu_dm_backlight_ops.get_brightness(dm->backlight_dev[aconnector->bl_idx]);
-
-		if (real_brightness != init_brightness) {
-			dm->actual_brightness[aconnector->bl_idx] = real_brightness;
-			dm->brightness[aconnector->bl_idx] = real_brightness;
-		}
-		drm_dbg_driver(drm, "DM: Registered Backlight device: %s\n", bl_name);
-	}
-}
-
-static int initialize_plane(struct amdgpu_display_manager *dm,
-			    struct amdgpu_mode_info *mode_info, int plane_id,
-			    enum drm_plane_type plane_type,
-			    const struct dc_plane_cap *plane_cap)
-{
-	struct drm_plane *plane;
-	unsigned long possible_crtcs;
-	int ret = 0;
-
-	plane = kzalloc_obj(struct drm_plane);
-	if (!plane) {
-		drm_err(adev_to_drm(dm->adev), "KMS: Failed to allocate plane\n");
-		return -ENOMEM;
-	}
-	plane->type = plane_type;
-
-	/*
-	 * HACK: IGT tests expect that the primary plane for a CRTC
-	 * can only have one possible CRTC. Only expose support for
-	 * any CRTC if they're not going to be used as a primary plane
-	 * for a CRTC - like overlay or underlay planes.
-	 */
-	possible_crtcs = 1 << plane_id;
-	if (plane_id >= dm->dc->caps.max_streams)
-		possible_crtcs = 0xff;
-
-	ret = amdgpu_dm_plane_init(dm, plane, possible_crtcs, plane_cap);
-
-	if (ret) {
-		drm_err(adev_to_drm(dm->adev), "KMS: Failed to initialize plane\n");
-		kfree(plane);
-		return ret;
-	}
-
-	if (mode_info)
-		mode_info->planes[plane_id] = plane;
-
-	return ret;
-}
-
-
-static void setup_backlight_device(struct amdgpu_display_manager *dm,
-				   struct amdgpu_dm_connector *aconnector)
-{
-	struct amdgpu_dm_backlight_caps *caps;
-	struct dc_link *link = aconnector->dc_link;
-	int bl_idx = dm->num_of_edps;
-
-	if (!(link->connector_signal & (SIGNAL_TYPE_EDP | SIGNAL_TYPE_LVDS)) ||
-	    link->type == dc_connection_none)
-		return;
-
-	if (dm->num_of_edps >= AMDGPU_DM_MAX_NUM_EDP) {
-		drm_warn(adev_to_drm(dm->adev), "Too much eDP connections, skipping backlight setup for additional eDPs\n");
-		return;
-	}
-
-	aconnector->bl_idx = bl_idx;
-
-	amdgpu_dm_update_backlight_caps(dm, bl_idx);
-	dm->backlight_link[bl_idx] = link;
-	dm->num_of_edps++;
-
-	update_connector_ext_caps(aconnector);
-	caps = &dm->backlight_caps[aconnector->bl_idx];
-
-	/* Only offer ABM property when non-OLED and user didn't turn off by module parameter */
-	if (caps->ext_caps && !caps->ext_caps->bits.oled && amdgpu_dm_abm_level < 0)
-		drm_object_attach_property(&aconnector->base.base,
-					   dm->adev->mode_info.abm_level_property,
-					   ABM_SYSFS_CONTROL);
-}
-
-static void amdgpu_set_panel_orientation(struct drm_connector *connector);
-
-
-
-/*
- * In this architecture, the association
- * connector -> encoder -> crtc
- * id not really requried. The crtc and connector will hold the
- * display_index as an abstraction to use with DAL component
- *
- * Returns 0 on success
- */
-static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
-{
-	struct amdgpu_display_manager *dm = &adev->dm;
-	s32 i;
-	struct amdgpu_dm_connector *aconnector = NULL;
-	struct amdgpu_encoder *aencoder = NULL;
-	struct amdgpu_mode_info *mode_info = &adev->mode_info;
-	u32 link_cnt;
-	s32 primary_planes;
-	enum dc_connection_type new_connection_type = dc_connection_none;
-	const struct dc_plane_cap *plane;
-	bool psr_feature_enabled = false;
-	bool replay_feature_enabled = false;
-	int max_overlay = dm->dc->caps.max_slave_planes;
-
-	dm->display_indexes_num = dm->dc->caps.max_streams;
-	/* Update the actual used number of crtc */
-	adev->mode_info.num_crtc = adev->dm.display_indexes_num;
-
-	amdgpu_dm_set_irq_funcs(adev);
-
-	link_cnt = dm->dc->caps.max_links;
-	if (amdgpu_dm_mode_config_init(dm->adev)) {
-		drm_err(adev_to_drm(adev), "DM: Failed to initialize mode config\n");
-		return -EINVAL;
-	}
-
-	/* There is one primary plane per CRTC */
-	primary_planes = dm->dc->caps.max_streams;
-	if (primary_planes > AMDGPU_MAX_PLANES) {
-		drm_err(adev_to_drm(adev), "DM: Plane nums out of 6 planes\n");
-		return -EINVAL;
-	}
-
-	/*
-	 * Initialize primary planes, implicit planes for legacy IOCTLS.
-	 * Order is reversed to match iteration order in atomic check.
-	 */
-	for (i = (primary_planes - 1); i >= 0; i--) {
-		plane = &dm->dc->caps.planes[i];
-
-		if (initialize_plane(dm, mode_info, i,
-				     DRM_PLANE_TYPE_PRIMARY, plane)) {
-			drm_err(adev_to_drm(adev), "KMS: Failed to initialize primary plane\n");
-			goto fail;
-		}
-	}
-
-	/*
-	 * Initialize overlay planes, index starting after primary planes.
-	 * These planes have a higher DRM index than the primary planes since
-	 * they should be considered as having a higher z-order.
-	 * Order is reversed to match iteration order in atomic check.
-	 *
-	 * Only support DCN for now, and only expose one so we don't encourage
-	 * userspace to use up all the pipes.
-	 */
-	for (i = 0; i < dm->dc->caps.max_planes; ++i) {
-		struct dc_plane_cap *plane = &dm->dc->caps.planes[i];
-
-		/* Do not create overlay if MPO disabled */
-		if (amdgpu_dc_debug_mask & DC_DISABLE_MPO)
-			break;
-
-		if (plane->type != DC_PLANE_TYPE_DCN_UNIVERSAL)
-			continue;
-
-		if (!plane->pixel_format_support.argb8888)
-			continue;
-
-		if (max_overlay-- == 0)
-			break;
-
-		if (initialize_plane(dm, NULL, primary_planes + i,
-				     DRM_PLANE_TYPE_OVERLAY, plane)) {
-			drm_err(adev_to_drm(adev), "KMS: Failed to initialize overlay plane\n");
-			goto fail;
-		}
-	}
-
-	for (i = 0; i < dm->dc->caps.max_streams; i++)
-		if (amdgpu_dm_crtc_init(dm, mode_info->planes[i], i)) {
-			drm_err(adev_to_drm(adev), "KMS: Failed to initialize crtc\n");
-			goto fail;
-		}
-
-	/* Use Outbox interrupt */
-	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
-	case IP_VERSION(3, 0, 0):
-	case IP_VERSION(3, 1, 2):
-	case IP_VERSION(3, 1, 3):
-	case IP_VERSION(3, 1, 4):
-	case IP_VERSION(3, 1, 5):
-	case IP_VERSION(3, 1, 6):
-	case IP_VERSION(3, 2, 0):
-	case IP_VERSION(3, 2, 1):
-	case IP_VERSION(2, 1, 0):
-	case IP_VERSION(3, 5, 0):
-	case IP_VERSION(3, 5, 1):
-	case IP_VERSION(3, 6, 0):
-	case IP_VERSION(4, 0, 1):
-	case IP_VERSION(4, 2, 0):
-	case IP_VERSION(4, 2, 1):
-		if (register_outbox_irq_handlers(dm->adev)) {
-			drm_err(adev_to_drm(adev), "DM: Failed to initialize IRQ\n");
-			goto fail;
-		}
-		break;
-	default:
-		drm_dbg_kms(adev_to_drm(adev), "Unsupported DCN IP version for outbox: 0x%X\n",
-			      amdgpu_ip_version(adev, DCE_HWIP, 0));
-	}
-
-	/* Determine whether to enable PSR support by default. */
-	if (!(amdgpu_dc_debug_mask & DC_DISABLE_PSR)) {
-		switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
-		case IP_VERSION(3, 1, 2):
-		case IP_VERSION(3, 1, 3):
-		case IP_VERSION(3, 1, 4):
-		case IP_VERSION(3, 1, 5):
-		case IP_VERSION(3, 1, 6):
-		case IP_VERSION(3, 2, 0):
-		case IP_VERSION(3, 2, 1):
-		case IP_VERSION(3, 5, 0):
-		case IP_VERSION(3, 5, 1):
-		case IP_VERSION(3, 6, 0):
-		case IP_VERSION(4, 0, 1):
-		case IP_VERSION(4, 2, 0):
-		case IP_VERSION(4, 2, 1):
-			psr_feature_enabled = true;
-			break;
-		default:
-			psr_feature_enabled = amdgpu_dc_feature_mask & DC_PSR_MASK;
-			break;
-		}
-	}
-
-	/* Determine whether to enable Replay support by default. */
-	if (!(amdgpu_dc_debug_mask & DC_DISABLE_REPLAY)) {
-		switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
-		case IP_VERSION(3, 1, 4):
-		case IP_VERSION(3, 2, 0):
-		case IP_VERSION(3, 2, 1):
-		case IP_VERSION(3, 5, 0):
-		case IP_VERSION(3, 5, 1):
-		case IP_VERSION(3, 6, 0):
-		case IP_VERSION(4, 2, 0):
-		case IP_VERSION(4, 2, 1):
-			replay_feature_enabled = true;
-			break;
-
-		default:
-			replay_feature_enabled = amdgpu_dc_feature_mask & DC_REPLAY_MASK;
-			break;
-		}
-	}
-
-	if (link_cnt > MAX_LINKS) {
-		drm_err(adev_to_drm(adev),
-			"KMS: Cannot support more than %d display indexes\n",
-				MAX_LINKS);
-		goto fail;
-	}
-
-	/* loops over all connectors on the board */
-	for (i = 0; i < link_cnt; i++) {
-		struct dc_link *link = NULL;
-
-		link = dc_get_link_at_index(dm->dc, i);
-
-		if (link->connector_signal == SIGNAL_TYPE_VIRTUAL) {
-			struct amdgpu_dm_wb_connector *wbcon = kzalloc_obj(*wbcon);
-
-			if (!wbcon) {
-				drm_err(adev_to_drm(adev), "KMS: Failed to allocate writeback connector\n");
-				continue;
-			}
-
-			if (amdgpu_dm_wb_connector_init(dm, wbcon, i)) {
-				drm_err(adev_to_drm(adev), "KMS: Failed to initialize writeback connector\n");
-				kfree(wbcon);
-				continue;
-			}
-
-			link->psr_settings.psr_feature_enabled = false;
-			link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED;
-
-			continue;
-		}
-
-		aconnector = kzalloc_obj(*aconnector);
-		if (!aconnector)
-			goto fail;
-
-		aencoder = kzalloc_obj(*aencoder);
-		if (!aencoder)
-			goto fail;
-
-		if (amdgpu_dm_encoder_init(dm->ddev, aencoder, i)) {
-			drm_err(adev_to_drm(adev), "KMS: Failed to initialize encoder\n");
-			goto fail;
-		}
-
-		if (amdgpu_dm_connector_init(dm, aconnector, i, aencoder)) {
-			drm_err(adev_to_drm(adev), "KMS: Failed to initialize connector\n");
-			goto fail;
-		}
-
-		if (dm->hpd_rx_offload_wq)
-			dm->hpd_rx_offload_wq[aconnector->base.index].aconnector =
-				aconnector;
-
-		if (!dc_link_detect_connection_type(link, &new_connection_type))
-			drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n");
-
-		if (aconnector->base.force && new_connection_type == dc_connection_none) {
-			emulated_link_detect(link);
-			amdgpu_dm_update_connector_after_detect(aconnector);
-		} else {
-			bool ret = false;
-
-			mutex_lock(&dm->dc_lock);
-			dc_exit_ips_for_hw_access(dm->dc);
-			ret = dc_link_detect(link, DETECT_REASON_BOOT);
-			mutex_unlock(&dm->dc_lock);
-
-			if (ret) {
-				amdgpu_dm_update_connector_after_detect(aconnector);
-				setup_backlight_device(dm, aconnector);
-
-				/* Disable PSR if Replay can be enabled */
-				if (replay_feature_enabled)
-					if (amdgpu_dm_set_replay_caps(link, aconnector))
-						psr_feature_enabled = false;
-
-				if (psr_feature_enabled) {
-					amdgpu_dm_set_psr_caps(link, aconnector);
-					drm_info(adev_to_drm(adev), "%s: PSR support %d, DC PSR ver %d, sink PSR ver %d DPCD caps 0x%x su_y_granularity %d\n",
-						 aconnector->base.name,
-						 link->psr_settings.psr_feature_enabled,
-						 link->psr_settings.psr_version,
-						 link->dpcd_caps.psr_info.psr_version,
-						 link->dpcd_caps.psr_info.psr_dpcd_caps.raw,
-						 link->dpcd_caps.psr_info.psr2_su_y_granularity_cap);
-				}
-			}
-		}
-		amdgpu_set_panel_orientation(&aconnector->base);
-	}
-
-	/* Debug dump: list all DC links and their associated sinks after detection
-	 * is complete for all connectors. This provides a comprehensive view of the
-	 * final state without repeating the dump for each connector.
-	 */
-	amdgpu_dm_dump_links_and_sinks(adev);
-
-	/* Software is initialized. Now we can register interrupt handlers. */
-	switch (adev->asic_type) {
-#if defined(CONFIG_DRM_AMD_DC_SI)
-	case CHIP_TAHITI:
-	case CHIP_PITCAIRN:
-	case CHIP_VERDE:
-	case CHIP_OLAND:
-#endif
-	case CHIP_BONAIRE:
-	case CHIP_HAWAII:
-	case CHIP_KAVERI:
-	case CHIP_KABINI:
-	case CHIP_MULLINS:
-	case CHIP_TONGA:
-	case CHIP_FIJI:
-	case CHIP_CARRIZO:
-	case CHIP_STONEY:
-	case CHIP_POLARIS11:
-	case CHIP_POLARIS10:
-	case CHIP_POLARIS12:
-	case CHIP_VEGAM:
-	case CHIP_VEGA10:
-	case CHIP_VEGA12:
-	case CHIP_VEGA20:
-		if (dce110_register_irq_handlers(dm->adev)) {
-			drm_err(adev_to_drm(adev), "DM: Failed to initialize IRQ\n");
-			goto fail;
-		}
-		break;
-	default:
-		switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
-		case IP_VERSION(1, 0, 0):
-		case IP_VERSION(1, 0, 1):
-		case IP_VERSION(2, 0, 2):
-		case IP_VERSION(2, 0, 3):
-		case IP_VERSION(2, 0, 0):
-		case IP_VERSION(2, 1, 0):
-		case IP_VERSION(3, 0, 0):
-		case IP_VERSION(3, 0, 2):
-		case IP_VERSION(3, 0, 3):
-		case IP_VERSION(3, 0, 1):
-		case IP_VERSION(3, 1, 2):
-		case IP_VERSION(3, 1, 3):
-		case IP_VERSION(3, 1, 4):
-		case IP_VERSION(3, 1, 5):
-		case IP_VERSION(3, 1, 6):
-		case IP_VERSION(3, 2, 0):
-		case IP_VERSION(3, 2, 1):
-		case IP_VERSION(3, 5, 0):
-		case IP_VERSION(3, 5, 1):
-		case IP_VERSION(3, 6, 0):
-		case IP_VERSION(4, 0, 1):
-		case IP_VERSION(4, 2, 0):
-		case IP_VERSION(4, 2, 1):
-			if (dcn10_register_irq_handlers(dm->adev)) {
-				drm_err(adev_to_drm(adev), "DM: Failed to initialize IRQ\n");
-				goto fail;
-			}
-			break;
-		default:
-			drm_err(adev_to_drm(adev), "Unsupported DCE IP versions: 0x%X\n",
-					amdgpu_ip_version(adev, DCE_HWIP, 0));
-			goto fail;
-		}
-		break;
-	}
-
-	return 0;
-fail:
-	kfree(aencoder);
-	kfree(aconnector);
-
-	return -EINVAL;
-}
-
-static void amdgpu_dm_destroy_drm_device(struct amdgpu_display_manager *dm)
-{
-	if (dm->atomic_obj.state)
-		drm_atomic_private_obj_fini(&dm->atomic_obj);
-}
-
-/******************************************************************************
- * amdgpu_display_funcs functions
- *****************************************************************************/
-
-/*
- * dm_bandwidth_update - program display watermarks
- *
- * @adev: amdgpu_device pointer
- *
- * Calculate and program the display watermarks and line buffer allocation.
- */
-static void dm_bandwidth_update(struct amdgpu_device *adev)
-{
-	/* TODO: implement later */
-}
-
-static const struct amdgpu_display_funcs dm_display_funcs = {
-	.bandwidth_update = dm_bandwidth_update, /* called unconditionally */
-	.vblank_get_counter = dm_vblank_get_counter,/* called unconditionally */
-	.backlight_set_level = NULL, /* never called for DC */
-	.backlight_get_level = NULL, /* never called for DC */
-	.hpd_sense = NULL,/* called unconditionally */
-	.hpd_set_polarity = NULL, /* called unconditionally */
-	.hpd_get_gpio_reg = NULL, /* VBIOS parsing. DAL does it. */
-	.page_flip_get_scanoutpos =
-		dm_crtc_get_scanoutpos,/* called unconditionally */
-	.add_encoder = NULL, /* VBIOS parsing. DAL does it. */
-	.add_connector = NULL, /* VBIOS parsing. DAL does it. */
-};
-
-#if defined(CONFIG_DEBUG_KERNEL_DC)
-
-static ssize_t s3_debug_store(struct device *device,
-			      struct device_attribute *attr,
-			      const char *buf,
-			      size_t count)
-{
-	int ret;
-	int s3_state;
-	struct drm_device *drm_dev = dev_get_drvdata(device);
-	struct amdgpu_device *adev = drm_to_adev(drm_dev);
-	struct amdgpu_ip_block *ip_block;
-
-	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_DCE);
-	if (!ip_block)
-		return -EINVAL;
-
-	ret = kstrtoint(buf, 0, &s3_state);
-
-	if (ret == 0) {
-		if (s3_state) {
-			dm_resume(ip_block);
-			drm_kms_helper_hotplug_event(adev_to_drm(adev));
-		} else
-			dm_suspend(ip_block);
-	}
-
-	return ret == 0 ? count : 0;
-}
-
-DEVICE_ATTR_WO(s3_debug);
-
-#endif
-
-static int dm_init_microcode(struct amdgpu_device *adev)
-{
-	char *fw_name_dmub;
-	int r;
-
-	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
-	case IP_VERSION(2, 1, 0):
-		fw_name_dmub = FIRMWARE_RENOIR_DMUB;
-		if (ASICREV_IS_GREEN_SARDINE(adev->external_rev_id))
-			fw_name_dmub = FIRMWARE_GREEN_SARDINE_DMUB;
-		break;
-	case IP_VERSION(3, 0, 0):
-		if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(10, 3, 0))
-			fw_name_dmub = FIRMWARE_SIENNA_CICHLID_DMUB;
-		else
-			fw_name_dmub = FIRMWARE_NAVY_FLOUNDER_DMUB;
-		break;
-	case IP_VERSION(3, 0, 1):
-		fw_name_dmub = FIRMWARE_VANGOGH_DMUB;
-		break;
-	case IP_VERSION(3, 0, 2):
-		fw_name_dmub = FIRMWARE_DIMGREY_CAVEFISH_DMUB;
-		break;
-	case IP_VERSION(3, 0, 3):
-		fw_name_dmub = FIRMWARE_BEIGE_GOBY_DMUB;
-		break;
-	case IP_VERSION(3, 1, 2):
-	case IP_VERSION(3, 1, 3):
-		fw_name_dmub = FIRMWARE_YELLOW_CARP_DMUB;
-		break;
-	case IP_VERSION(3, 1, 4):
-		fw_name_dmub = FIRMWARE_DCN_314_DMUB;
-		break;
-	case IP_VERSION(3, 1, 5):
-		fw_name_dmub = FIRMWARE_DCN_315_DMUB;
-		break;
-	case IP_VERSION(3, 1, 6):
-		fw_name_dmub = FIRMWARE_DCN316_DMUB;
-		break;
-	case IP_VERSION(3, 2, 0):
-		fw_name_dmub = FIRMWARE_DCN_V3_2_0_DMCUB;
-		break;
-	case IP_VERSION(3, 2, 1):
-		fw_name_dmub = FIRMWARE_DCN_V3_2_1_DMCUB;
-		break;
-	case IP_VERSION(3, 5, 0):
-		fw_name_dmub = FIRMWARE_DCN_35_DMUB;
-		break;
-	case IP_VERSION(3, 5, 1):
-		fw_name_dmub = FIRMWARE_DCN_351_DMUB;
-		break;
-	case IP_VERSION(3, 6, 0):
-		fw_name_dmub = FIRMWARE_DCN_36_DMUB;
-		break;
-	case IP_VERSION(4, 0, 1):
-		fw_name_dmub = FIRMWARE_DCN_401_DMUB;
-		break;
-	case IP_VERSION(4, 2, 0):
-		fw_name_dmub = FIRMWARE_DCN_42_DMUB;
-		break;
-	case IP_VERSION(4, 2, 1):
-		fw_name_dmub = FIRMWARE_DCN_42B_DMUB;
-		break;
-	default:
-		/* ASIC doesn't support DMUB. */
-		return 0;
-	}
-	r = amdgpu_ucode_request(adev, &adev->dm.dmub_fw, AMDGPU_UCODE_REQUIRED,
-				 "%s", fw_name_dmub);
-	return r;
-}
-
-static int dm_early_init(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	struct amdgpu_mode_info *mode_info = &adev->mode_info;
-	struct atom_context *ctx = mode_info->atom_context;
-	int index = GetIndexIntoMasterTable(DATA, Object_Header);
-	u16 data_offset;
-
-	/* if there is no object header, skip DM */
-	if (!amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
-		adev->harvest_ip_mask |= AMD_HARVEST_IP_DMU_MASK;
-		drm_info(adev_to_drm(adev), "No object header, skipping DM\n");
-		return -ENOENT;
-	}
-
-	switch (adev->asic_type) {
-#if defined(CONFIG_DRM_AMD_DC_SI)
-	case CHIP_TAHITI:
-	case CHIP_PITCAIRN:
-	case CHIP_VERDE:
-		adev->mode_info.num_crtc = 6;
-		adev->mode_info.num_hpd = 6;
-		adev->mode_info.num_dig = 6;
-		break;
-	case CHIP_OLAND:
-		adev->mode_info.num_crtc = 2;
-		adev->mode_info.num_hpd = 2;
-		adev->mode_info.num_dig = 2;
-		break;
-#endif
-	case CHIP_BONAIRE:
-	case CHIP_HAWAII:
-		adev->mode_info.num_crtc = 6;
-		adev->mode_info.num_hpd = 6;
-		adev->mode_info.num_dig = 6;
-		break;
-	case CHIP_KAVERI:
-		adev->mode_info.num_crtc = 4;
-		adev->mode_info.num_hpd = 6;
-		adev->mode_info.num_dig = 7;
-		break;
-	case CHIP_KABINI:
-	case CHIP_MULLINS:
-		adev->mode_info.num_crtc = 2;
-		adev->mode_info.num_hpd = 6;
-		adev->mode_info.num_dig = 6;
-		break;
-	case CHIP_FIJI:
-	case CHIP_TONGA:
-		adev->mode_info.num_crtc = 6;
-		adev->mode_info.num_hpd = 6;
-		adev->mode_info.num_dig = 7;
-		break;
-	case CHIP_CARRIZO:
-		adev->mode_info.num_crtc = 3;
-		adev->mode_info.num_hpd = 6;
-		adev->mode_info.num_dig = 9;
-		break;
-	case CHIP_STONEY:
-		adev->mode_info.num_crtc = 2;
-		adev->mode_info.num_hpd = 6;
-		adev->mode_info.num_dig = 9;
-		break;
-	case CHIP_POLARIS11:
-	case CHIP_POLARIS12:
-		adev->mode_info.num_crtc = 5;
-		adev->mode_info.num_hpd = 5;
-		adev->mode_info.num_dig = 5;
-		break;
-	case CHIP_POLARIS10:
-	case CHIP_VEGAM:
-		adev->mode_info.num_crtc = 6;
-		adev->mode_info.num_hpd = 6;
-		adev->mode_info.num_dig = 6;
-		break;
-	case CHIP_VEGA10:
-	case CHIP_VEGA12:
-	case CHIP_VEGA20:
-		adev->mode_info.num_crtc = 6;
-		adev->mode_info.num_hpd = 6;
-		adev->mode_info.num_dig = 6;
-		break;
-	default:
-
-		switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
-		case IP_VERSION(2, 0, 2):
-		case IP_VERSION(3, 0, 0):
-			adev->mode_info.num_crtc = 6;
-			adev->mode_info.num_hpd = 6;
-			adev->mode_info.num_dig = 6;
-			break;
-		case IP_VERSION(2, 0, 0):
-		case IP_VERSION(3, 0, 2):
-			adev->mode_info.num_crtc = 5;
-			adev->mode_info.num_hpd = 5;
-			adev->mode_info.num_dig = 5;
-			break;
-		case IP_VERSION(2, 0, 3):
-		case IP_VERSION(3, 0, 3):
-			adev->mode_info.num_crtc = 2;
-			adev->mode_info.num_hpd = 2;
-			adev->mode_info.num_dig = 2;
-			break;
-		case IP_VERSION(1, 0, 0):
-		case IP_VERSION(1, 0, 1):
-		case IP_VERSION(3, 0, 1):
-		case IP_VERSION(2, 1, 0):
-		case IP_VERSION(3, 1, 2):
-		case IP_VERSION(3, 1, 3):
-		case IP_VERSION(3, 1, 4):
-		case IP_VERSION(3, 1, 5):
-		case IP_VERSION(3, 1, 6):
-		case IP_VERSION(3, 2, 0):
-		case IP_VERSION(3, 2, 1):
-		case IP_VERSION(3, 5, 0):
-		case IP_VERSION(3, 5, 1):
-		case IP_VERSION(3, 6, 0):
-		case IP_VERSION(4, 0, 1):
-		case IP_VERSION(4, 2, 0):
-		case IP_VERSION(4, 2, 1):
-			adev->mode_info.num_crtc = 4;
-			adev->mode_info.num_hpd = 4;
-			adev->mode_info.num_dig = 4;
-			break;
-		default:
-			drm_err(adev_to_drm(adev), "Unsupported DCE IP versions: 0x%x\n",
-					amdgpu_ip_version(adev, DCE_HWIP, 0));
-			return -EINVAL;
-		}
-		break;
-	}
-
-	if (adev->mode_info.funcs == NULL)
-		adev->mode_info.funcs = &dm_display_funcs;
-
-	/*
-	 * Note: Do NOT change adev->reg.audio_endpt.rreg and
-	 * adev->reg.audio_endpt.wreg because they are initialised in
-	 * amdgpu_device_init()
-	 */
-#if defined(CONFIG_DEBUG_KERNEL_DC)
-	device_create_file(
-		adev_to_drm(adev)->dev,
-		&dev_attr_s3_debug);
-#endif
-	adev->dc_enabled = true;
-
-	return dm_init_microcode(adev);
-}
-
-static bool modereset_required(struct drm_crtc_state *crtc_state)
-{
-	return !crtc_state->active && drm_atomic_crtc_needs_modeset(crtc_state);
-}
-
-static void amdgpu_dm_encoder_destroy(struct drm_encoder *encoder)
-{
-	drm_encoder_cleanup(encoder);
-	kfree(encoder);
-}
-
-static const struct drm_encoder_funcs amdgpu_dm_encoder_funcs = {
-	.destroy = amdgpu_dm_encoder_destroy,
-};
-
-static int
-fill_plane_color_attributes(const struct drm_plane_state *plane_state,
-			    const enum surface_pixel_format format,
-			    enum dc_color_space *color_space)
-{
-	bool full_range;
-
-	*color_space = COLOR_SPACE_SRGB;
-
-	/* Ignore properties when DRM_CLIENT_CAP_PLANE_COLOR_PIPELINE is set */
-	if (plane_state->state && plane_state->state->plane_color_pipeline)
-		return 0;
-
-	/* DRM color properties only affect non-RGB formats. */
-	if (format < SURFACE_PIXEL_FORMAT_VIDEO_BEGIN)
-		return 0;
-
-	full_range = (plane_state->color_range == DRM_COLOR_YCBCR_FULL_RANGE);
-
-	switch (plane_state->color_encoding) {
-	case DRM_COLOR_YCBCR_BT601:
-		if (full_range)
-			*color_space = COLOR_SPACE_YCBCR601;
-		else
-			*color_space = COLOR_SPACE_YCBCR601_LIMITED;
-		break;
-
-	case DRM_COLOR_YCBCR_BT709:
-		if (full_range)
-			*color_space = COLOR_SPACE_YCBCR709;
-		else
-			*color_space = COLOR_SPACE_YCBCR709_LIMITED;
-		break;
-
-	case DRM_COLOR_YCBCR_BT2020:
-		if (full_range)
-			*color_space = COLOR_SPACE_2020_YCBCR_FULL;
-		else
-			*color_space = COLOR_SPACE_2020_YCBCR_LIMITED;
-		break;
-
-	default:
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int
-fill_dc_plane_info_and_addr(struct amdgpu_device *adev,
-			    const struct drm_plane_state *plane_state,
-			    const u64 tiling_flags,
-			    struct dc_plane_info *plane_info,
-			    struct dc_plane_address *address,
-			    bool tmz_surface)
-{
-	const struct drm_framebuffer *fb = plane_state->fb;
-	const struct amdgpu_framebuffer *afb =
-		to_amdgpu_framebuffer(plane_state->fb);
-	int ret;
-
-	memset(plane_info, 0, sizeof(*plane_info));
-
-	switch (fb->format->format) {
-	case DRM_FORMAT_C8:
-		plane_info->format =
-			SURFACE_PIXEL_FORMAT_GRPH_PALETA_256_COLORS;
-		break;
-	case DRM_FORMAT_RGB565:
-		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_RGB565;
-		break;
-	case DRM_FORMAT_XRGB8888:
-	case DRM_FORMAT_ARGB8888:
-		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB8888;
-		break;
-	case DRM_FORMAT_XRGB2101010:
-	case DRM_FORMAT_ARGB2101010:
-		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB2101010;
-		break;
-	case DRM_FORMAT_XBGR2101010:
-	case DRM_FORMAT_ABGR2101010:
-		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR2101010;
-		break;
-	case DRM_FORMAT_XBGR8888:
-	case DRM_FORMAT_ABGR8888:
-		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR8888;
-		break;
-	case DRM_FORMAT_NV21:
-		plane_info->format = SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr;
-		break;
-	case DRM_FORMAT_NV12:
-		plane_info->format = SURFACE_PIXEL_FORMAT_VIDEO_420_YCrCb;
-		break;
-	case DRM_FORMAT_P010:
-		plane_info->format = SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb;
-		break;
-	case DRM_FORMAT_XRGB16161616F:
-	case DRM_FORMAT_ARGB16161616F:
-		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616F;
-		break;
-	case DRM_FORMAT_XBGR16161616F:
-	case DRM_FORMAT_ABGR16161616F:
-		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F;
-		break;
-	case DRM_FORMAT_XRGB16161616:
-	case DRM_FORMAT_ARGB16161616:
-		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616;
-		break;
-	case DRM_FORMAT_XBGR16161616:
-	case DRM_FORMAT_ABGR16161616:
-		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616;
-		break;
-	default:
-		drm_err(adev_to_drm(adev),
-			"Unsupported screen format %p4cc\n",
-			&fb->format->format);
-		return -EINVAL;
-	}
-
-	switch (plane_state->rotation & DRM_MODE_ROTATE_MASK) {
-	case DRM_MODE_ROTATE_0:
-		plane_info->rotation = ROTATION_ANGLE_0;
-		break;
-	case DRM_MODE_ROTATE_90:
-		plane_info->rotation = ROTATION_ANGLE_90;
-		break;
-	case DRM_MODE_ROTATE_180:
-		plane_info->rotation = ROTATION_ANGLE_180;
-		break;
-	case DRM_MODE_ROTATE_270:
-		plane_info->rotation = ROTATION_ANGLE_270;
-		break;
-	default:
-		plane_info->rotation = ROTATION_ANGLE_0;
-		break;
+	ppatch = &sink->edid_caps.panel_patch;
+	if (ppatch->wait_after_dpcd_poweroff_ms) {
+		msleep(ppatch->wait_after_dpcd_poweroff_ms);
+		drm_dbg_driver(adev_to_drm(adev),
+			       "%s: adding a %ds delay as w/a for panel\n",
+			       __func__,
+			       ppatch->wait_after_dpcd_poweroff_ms / 1000);
 	}
-
-
-	plane_info->visible = true;
-	plane_info->stereo_format = PLANE_STEREO_FORMAT_NONE;
-
-	plane_info->layer_index = plane_state->normalized_zpos;
-
-	ret = fill_plane_color_attributes(plane_state, plane_info->format,
-					  &plane_info->color_space);
-	if (ret)
-		return ret;
-
-	ret = amdgpu_dm_plane_fill_plane_buffer_attributes(adev, afb, plane_info->format,
-					   plane_info->rotation, tiling_flags,
-					   &plane_info->tiling_info,
-					   &plane_info->plane_size,
-					   &plane_info->dcc, address,
-					   tmz_surface);
-	if (ret)
-		return ret;
-
-	amdgpu_dm_plane_fill_blending_from_plane_state(
-		plane_state, &plane_info->per_pixel_alpha, &plane_info->pre_multiplied_alpha,
-		&plane_info->global_alpha, &plane_info->global_alpha_value);
-
-	return 0;
-}
-
-static int fill_dc_plane_attributes(struct amdgpu_device *adev,
-				    struct dc_plane_state *dc_plane_state,
-				    struct drm_plane_state *plane_state,
-				    struct drm_crtc_state *crtc_state)
-{
-	struct dm_crtc_state *dm_crtc_state = to_dm_crtc_state(crtc_state);
-	struct amdgpu_framebuffer *afb = (struct amdgpu_framebuffer *)plane_state->fb;
-	struct dc_scaling_info scaling_info;
-	struct dc_plane_info plane_info;
-	int ret;
-
-	ret = amdgpu_dm_plane_fill_dc_scaling_info(adev, plane_state, &scaling_info);
-	if (ret)
-		return ret;
-
-	dc_plane_state->src_rect = scaling_info.src_rect;
-	dc_plane_state->dst_rect = scaling_info.dst_rect;
-	dc_plane_state->clip_rect = scaling_info.clip_rect;
-	dc_plane_state->scaling_quality = scaling_info.scaling_quality;
-
-	ret = fill_dc_plane_info_and_addr(adev, plane_state,
-					  afb->tiling_flags,
-					  &plane_info,
-					  &dc_plane_state->address,
-					  afb->tmz_surface);
-	if (ret)
-		return ret;
-
-	dc_plane_state->format = plane_info.format;
-	dc_plane_state->color_space = plane_info.color_space;
-	dc_plane_state->format = plane_info.format;
-	dc_plane_state->plane_size = plane_info.plane_size;
-	dc_plane_state->rotation = plane_info.rotation;
-	dc_plane_state->horizontal_mirror = plane_info.horizontal_mirror;
-	dc_plane_state->stereo_format = plane_info.stereo_format;
-	dc_plane_state->tiling_info = plane_info.tiling_info;
-	dc_plane_state->visible = plane_info.visible;
-	dc_plane_state->per_pixel_alpha = plane_info.per_pixel_alpha;
-	dc_plane_state->pre_multiplied_alpha = plane_info.pre_multiplied_alpha;
-	dc_plane_state->global_alpha = plane_info.global_alpha;
-	dc_plane_state->global_alpha_value = plane_info.global_alpha_value;
-	dc_plane_state->dcc = plane_info.dcc;
-	dc_plane_state->layer_index = plane_info.layer_index;
-	dc_plane_state->flip_int_enabled = true;
-
-	/*
-	 * Always set input transfer function, since plane state is refreshed
-	 * every time.
-	 */
-	ret = amdgpu_dm_update_plane_color_mgmt(dm_crtc_state,
-						plane_state,
-						dc_plane_state);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static inline void fill_dc_dirty_rect(struct drm_plane *plane,
-				      struct rect *dirty_rect, int32_t x,
-				      s32 y, s32 width, s32 height,
-				      int *i, bool ffu)
-{
-	WARN_ON(*i >= DC_MAX_DIRTY_RECTS);
-
-	dirty_rect->x = x;
-	dirty_rect->y = y;
-	dirty_rect->width = width;
-	dirty_rect->height = height;
-
-	if (ffu)
-		drm_dbg(plane->dev,
-			"[PLANE:%d] PSR FFU dirty rect size (%d, %d)\n",
-			plane->base.id, width, height);
-	else
-		drm_dbg(plane->dev,
-			"[PLANE:%d] PSR SU dirty rect at (%d, %d) size (%d, %d)",
-			plane->base.id, x, y, width, height);
-
-	(*i)++;
 }
 
 /**
- * fill_dc_dirty_rects() - Fill DC dirty regions for PSR selective updates
- *
- * @plane: DRM plane containing dirty regions that need to be flushed to the eDP
- *         remote fb
- * @old_plane_state: Old state of @plane
- * @new_plane_state: New state of @plane
- * @crtc_state: New state of CRTC connected to the @plane
- * @flip_addrs: DC flip tracking struct, which also tracts dirty rects
- * @is_psr_su: Flag indicating whether Panel Self Refresh Selective Update (PSR SU) is enabled.
- *             If PSR SU is enabled and damage clips are available, only the regions of the screen
- *             that have changed will be updated. If PSR SU is not enabled,
- *             or if damage clips are not available, the entire screen will be updated.
- * @dirty_regions_changed: dirty regions changed
- *
- * For PSR SU, DC informs the DMUB uController of dirty rectangle regions
- * (referred to as "damage clips" in DRM nomenclature) that require updating on
- * the eDP remote buffer. The responsibility of specifying the dirty regions is
- * amdgpu_dm's.
+ * amdgpu_dm_dump_links_and_sinks - Debug dump of all DC links and their sinks
+ * @adev: amdgpu device pointer
  *
- * A damage-aware DRM client should fill the FB_DAMAGE_CLIPS property on the
- * plane with regions that require flushing to the eDP remote buffer. In
- * addition, certain use cases - such as cursor and multi-plane overlay (MPO) -
- * implicitly provide damage clips without any client support via the plane
- * bounds.
- */
-static void fill_dc_dirty_rects(struct drm_plane *plane,
-				struct drm_plane_state *old_plane_state,
-				struct drm_plane_state *new_plane_state,
-				struct drm_crtc_state *crtc_state,
-				struct dc_flip_addrs *flip_addrs,
-				bool is_psr_su,
-				bool *dirty_regions_changed)
+ * Iterates through all DC links and dumps information about local and remote
+ * (MST) sinks. Should be called after connector detection is complete to see
+ * the final state of all links.
+ */
+static void amdgpu_dm_dump_links_and_sinks(struct amdgpu_device *adev)
 {
-	struct dm_crtc_state *dm_crtc_state = to_dm_crtc_state(crtc_state);
-	struct rect *dirty_rects = flip_addrs->dirty_rects;
-	u32 num_clips = 0;
-	struct drm_mode_rect *clips = NULL;
-	bool bb_changed;
-	bool fb_changed;
-	u32 i = 0;
-	*dirty_regions_changed = false;
+	struct dc *dc = adev->dm.dc;
+	struct drm_device *dev = adev_to_drm(adev);
+	int li;
 
-	/*
-	 * Cursor plane has it's own dirty rect update interface. See
-	 * dcn10_dmub_update_cursor_data and dmub_cmd_update_cursor_info_data
-	 */
-	if (plane->type == DRM_PLANE_TYPE_CURSOR)
+	if (!dc)
 		return;
 
-	if (new_plane_state->rotation != DRM_MODE_ROTATE_0)
-		goto ffu;
+	for (li = 0; li < dc->link_count; li++) {
+		struct dc_link *l = dc->links[li];
+		const char *name = NULL;
+		int rs;
 
-	if (!new_plane_state->ignore_damage_clips) {
-		num_clips = drm_plane_get_damage_clips_count(new_plane_state);
-		clips = drm_plane_get_damage_clips(new_plane_state);
+		if (!l)
+			continue;
+		if (l->local_sink && l->local_sink->edid_caps.display_name[0])
+			name = l->local_sink->edid_caps.display_name;
+		else
+			name = "n/a";
+
+		drm_dbg_kms(dev,
+			"LINK_DUMP[%d]: local_sink=%p type=%d sink_signal=%d sink_count=%u edid_name=%s mst_capable=%d mst_alloc_streams=%d\n",
+			li,
+			l->local_sink,
+			l->type,
+			l->local_sink ? l->local_sink->sink_signal : SIGNAL_TYPE_NONE,
+			l->sink_count,
+			name,
+			l->dpcd_caps.is_mst_capable,
+			l->mst_stream_alloc_table.stream_count);
+
+		/* Dump remote (MST) sinks if any */
+		for (rs = 0; rs < l->sink_count; rs++) {
+			struct dc_sink *rsink = l->remote_sinks[rs];
+			const char *rname = NULL;
+
+			if (!rsink)
+				continue;
+			if (rsink->edid_caps.display_name[0])
+				rname = rsink->edid_caps.display_name;
+			else
+				rname = "n/a";
+			drm_dbg_kms(dev,
+				"  REMOTE_SINK[%d:%d]: sink=%p signal=%d edid_name=%s\n",
+				li, rs,
+				rsink,
+				rsink->sink_signal,
+				rname);
+		}
 	}
+}
 
-	if (num_clips && (!amdgpu_damage_clips || (amdgpu_damage_clips < 0 &&
-						   is_psr_su)))
-		goto ffu;
+static int dm_resume(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	struct drm_device *ddev = adev_to_drm(adev);
+	struct amdgpu_display_manager *dm = &adev->dm;
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_connector *connector;
+	struct drm_connector_list_iter iter;
+	struct dm_atomic_state *dm_state = to_dm_atomic_state(dm->atomic_obj.state);
+	enum dc_connection_type new_connection_type = dc_connection_none;
+	struct dc_state *dc_state;
+	int i, r, j;
+	struct dc_commit_streams_params commit_params = {};
 
-	if (!dm_crtc_state->mpo_requested) {
-		if (!num_clips || num_clips > DC_MAX_DIRTY_RECTS)
-			goto ffu;
+	if (dm->dc->caps.ips_support) {
+		if (!amdgpu_in_reset(adev))
+			mutex_lock(&dm->dc_lock);
 
-		for (; flip_addrs->dirty_rect_count < num_clips; clips++)
-			fill_dc_dirty_rect(new_plane_state->plane,
-					   &dirty_rects[flip_addrs->dirty_rect_count],
-					   clips->x1, clips->y1,
-					   clips->x2 - clips->x1, clips->y2 - clips->y1,
-					   &flip_addrs->dirty_rect_count,
-					   false);
-		return;
+		/* Need to set POWER_STATE_D0 first or it will not execute
+		 * idle_power_optimizations command to DMUB.
+		 */
+		dc_dmub_srv_set_power_state(dm->dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D0);
+		dc_dmub_srv_apply_idle_power_optimizations(dm->dc, false);
+
+		if (!amdgpu_in_reset(adev))
+			mutex_unlock(&dm->dc_lock);
 	}
 
-	/*
-	 * MPO is requested. Add entire plane bounding box to dirty rects if
-	 * flipped to or damaged.
-	 *
-	 * If plane is moved or resized, also add old bounding box to dirty
-	 * rects.
-	 */
-	fb_changed = old_plane_state->fb->base.id !=
-		     new_plane_state->fb->base.id;
-	bb_changed = (old_plane_state->crtc_x != new_plane_state->crtc_x ||
-		      old_plane_state->crtc_y != new_plane_state->crtc_y ||
-		      old_plane_state->crtc_w != new_plane_state->crtc_w ||
-		      old_plane_state->crtc_h != new_plane_state->crtc_h);
+	if (amdgpu_in_reset(adev)) {
+		dc_state = dm->cached_dc_state;
 
-	drm_dbg(plane->dev,
-		"[PLANE:%d] PSR bb_changed:%d fb_changed:%d num_clips:%d\n",
-		new_plane_state->plane->base.id,
-		bb_changed, fb_changed, num_clips);
+		/*
+		 * The dc->current_state is backed up into dm->cached_dc_state
+		 * before we commit 0 streams.
+		 *
+		 * DC will clear link encoder assignments on the real state
+		 * but the changes won't propagate over to the copy we made
+		 * before the 0 streams commit.
+		 *
+		 * DC expects that link encoder assignments are *not* valid
+		 * when committing a state, so as a workaround we can copy
+		 * off of the current state.
+		 *
+		 * We lose the previous assignments, but we had already
+		 * commit 0 streams anyway.
+		 */
+		link_enc_cfg_copy(adev->dm.dc->current_state, dc_state);
 
-	*dirty_regions_changed = bb_changed;
+		r = dm_dmub_hw_init(adev);
+		if (r) {
+			drm_err(adev_to_drm(adev), "DMUB interface failed to initialize: status=%d\n", r);
+			return r;
+		}
 
-	if ((num_clips + (bb_changed ? 2 : 0)) > DC_MAX_DIRTY_RECTS)
-		goto ffu;
+		dc_dmub_srv_set_power_state(dm->dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D0);
+		dc_set_power_state(dm->dc, DC_ACPI_CM_POWER_STATE_D0);
 
-	if (bb_changed) {
-		fill_dc_dirty_rect(new_plane_state->plane, &dirty_rects[i],
-				   new_plane_state->crtc_x,
-				   new_plane_state->crtc_y,
-				   new_plane_state->crtc_w,
-				   new_plane_state->crtc_h, &i, false);
+		dc_resume(dm->dc);
 
-		/* Add old plane bounding-box if plane is moved or resized */
-		fill_dc_dirty_rect(new_plane_state->plane, &dirty_rects[i],
-				   old_plane_state->crtc_x,
-				   old_plane_state->crtc_y,
-				   old_plane_state->crtc_w,
-				   old_plane_state->crtc_h, &i, false);
-	}
+		amdgpu_dm_ism_enable(dm);
+		amdgpu_dm_irq_resume_early(adev);
 
-	if (num_clips) {
-		for (; i < num_clips; clips++)
-			fill_dc_dirty_rect(new_plane_state->plane,
-					   &dirty_rects[i], clips->x1,
-					   clips->y1, clips->x2 - clips->x1,
-					   clips->y2 - clips->y1, &i, false);
-	} else if (fb_changed && !bb_changed) {
-		fill_dc_dirty_rect(new_plane_state->plane, &dirty_rects[i],
-				   new_plane_state->crtc_x,
-				   new_plane_state->crtc_y,
-				   new_plane_state->crtc_w,
-				   new_plane_state->crtc_h, &i, false);
-	}
+		for (i = 0; i < dc_state->stream_count; i++) {
+			dc_state->streams[i]->mode_changed = true;
+			for (j = 0; j < dc_state->stream_status[i].plane_count; j++) {
+				dc_pipe_update_bits_set_full(&dc_state->stream_status[i].plane_states[j]->update_bits);
+			}
+		}
 
-	flip_addrs->dirty_rect_count = i;
-	return;
+		if (dc_is_dmub_outbox_supported(adev->dm.dc)) {
+			amdgpu_dm_outbox_init(adev);
+			dc_enable_dmub_outbox(adev->dm.dc);
+		}
 
-ffu:
-	fill_dc_dirty_rect(new_plane_state->plane, &dirty_rects[0], 0, 0,
-			   dm_crtc_state->base.mode.crtc_hdisplay,
-			   dm_crtc_state->base.mode.crtc_vdisplay,
-			   &flip_addrs->dirty_rect_count, true);
-}
+		commit_params.streams = dc_state->streams;
+		commit_params.stream_count = dc_state->stream_count;
+		dc_exit_ips_for_hw_access(dm->dc);
+		WARN_ON(!dc_commit_streams(dm->dc, &commit_params));
 
-static void update_stream_scaling_settings(struct drm_device *dev,
-					   const struct drm_display_mode *mode,
-					   const struct dm_connector_state *dm_state,
-					   struct dc_stream_state *stream)
-{
-	enum amdgpu_rmx_type rmx_type;
+		dm_gpureset_commit_state(dm->cached_dc_state, dm);
 
-	struct rect src = { 0 }; /* viewport in composition space*/
-	struct rect dst = { 0 }; /* stream addressable area */
+		dm_gpureset_toggle_interrupts(adev, dm->cached_dc_state, true);
 
-	/* no mode. nothing to be done */
-	if (!mode)
-		return;
+		dc_state_release(dm->cached_dc_state);
+		dm->cached_dc_state = NULL;
 
-	/* Full screen scaling by default */
-	src.width = mode->hdisplay;
-	src.height = mode->vdisplay;
-	dst.width = stream->timing.h_addressable;
-	dst.height = stream->timing.v_addressable;
+		amdgpu_dm_irq_resume_late(adev);
 
-	if (dm_state) {
-		rmx_type = dm_state->scaling;
-		if (rmx_type == RMX_ASPECT || rmx_type == RMX_OFF) {
-			if (src.width * dst.height <
-					src.height * dst.width) {
-				/* height needs less upscaling/more downscaling */
-				dst.width = src.width *
-						dst.height / src.height;
-			} else {
-				/* width needs less upscaling/more downscaling */
-				dst.height = src.height *
-						dst.width / src.width;
-			}
-		} else if (rmx_type == RMX_CENTER) {
-			dst = src;
+		mutex_unlock(&dm->dc_lock);
+
+		/* set the backlight after a reset */
+		for (i = 0; i < dm->num_of_edps; i++) {
+			if (dm->backlight_dev[i])
+				amdgpu_dm_backlight_set_level(dm, i, dm->brightness[i]);
 		}
 
-		dst.x = (stream->timing.h_addressable - dst.width) / 2;
-		dst.y = (stream->timing.v_addressable - dst.height) / 2;
+		return 0;
+	}
+	/* Recreate dc_state - DC invalidates it when setting power state to S3. */
+	dc_state_release(dm_state->context);
+	dm_state->context = dc_state_create(dm->dc, NULL);
+	/* TODO: Remove dc_state->dccg, use dc->dccg directly. */
+
+	/* Before powering on DC we need to re-initialize DMUB. */
+	dm_dmub_hw_resume(adev);
+
+	/* Re-enable outbox interrupts for DPIA. */
+	if (dc_is_dmub_outbox_supported(adev->dm.dc)) {
+		amdgpu_dm_outbox_init(adev);
+		dc_enable_dmub_outbox(adev->dm.dc);
+	}
+
+	/* power on hardware */
+	dc_dmub_srv_set_power_state(dm->dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D0);
+	dc_set_power_state(dm->dc, DC_ACPI_CM_POWER_STATE_D0);
 
-		if (dm_state->underscan_enable) {
-			dst.x += dm_state->underscan_hborder / 2;
-			dst.y += dm_state->underscan_vborder / 2;
-			dst.width -= dm_state->underscan_hborder;
-			dst.height -= dm_state->underscan_vborder;
-		}
-	}
+	/* program HPD filter */
+	dc_resume(dm->dc);
 
-	stream->src = src;
-	stream->dst = dst;
+	scoped_guard(mutex, &dm->dc_lock)
+		amdgpu_dm_ism_enable(dm);
 
-	drm_dbg_kms(dev, "Destination Rectangle x:%d  y:%d  width:%d  height:%d\n",
-		    dst.x, dst.y, dst.width, dst.height);
+	/*
+	 * early enable HPD Rx IRQ, should be done before set mode as short
+	 * pulse interrupts are used for MST
+	 */
+	amdgpu_dm_irq_resume_early(adev);
 
-}
+	amdgpu_dm_s3_handle_hdmi_cec(ddev, false);
 
-static enum dc_color_depth
-convert_color_depth_from_display_info(const struct drm_connector *connector,
-				      bool is_y420, int requested_bpc)
-{
-	u8 bpc;
-
-	if (is_y420) {
-		bpc = 8;
-
-		/* Cap display bpc based on HDMI 2.0 HF-VSDB */
-		if (connector->display_info.hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_48)
-			bpc = 16;
-		else if (connector->display_info.hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_36)
-			bpc = 12;
-		else if (connector->display_info.hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_30)
-			bpc = 10;
-	} else {
-		bpc = (uint8_t)connector->display_info.bpc;
-		/* Assume 8 bpc by default if no bpc is specified. */
-		bpc = bpc ? bpc : 8;
-	}
+	/* On resume we need to rewrite the MSTM control bits to enable MST*/
+	s3_handle_mst(ddev, false);
 
-	if (requested_bpc > 0) {
-		/*
-		 * Cap display bpc based on the user requested value.
-		 *
-		 * The value for state->max_bpc may not correctly updated
-		 * depending on when the connector gets added to the state
-		 * or if this was called outside of atomic check, so it
-		 * can't be used directly.
-		 */
-		bpc = min_t(u8, bpc, requested_bpc);
+	/* Do detection*/
+	drm_connector_list_iter_begin(ddev, &iter);
+	drm_for_each_connector_iter(connector, &iter) {
+		bool ret;
 
-		/* Round down to the nearest even number. */
-		bpc = bpc - (bpc & 1);
-	}
+		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
+			continue;
+
+		aconnector = to_amdgpu_dm_connector(connector);
+
+		if (!aconnector->dc_link)
+			continue;
 
-	switch (bpc) {
-	case 0:
 		/*
-		 * Temporary Work around, DRM doesn't parse color depth for
-		 * EDID revision before 1.4
-		 * TODO: Fix edid parsing
+		 * this is the case when traversing through already created end sink
+		 * MST connectors, should be skipped
 		 */
-		return COLOR_DEPTH_888;
-	case 6:
-		return COLOR_DEPTH_666;
-	case 8:
-		return COLOR_DEPTH_888;
-	case 10:
-		return COLOR_DEPTH_101010;
-	case 12:
-		return COLOR_DEPTH_121212;
-	case 14:
-		return COLOR_DEPTH_141414;
-	case 16:
-		return COLOR_DEPTH_161616;
-	default:
-		return COLOR_DEPTH_UNDEFINED;
-	}
-}
+		if (aconnector->mst_root)
+			continue;
 
-static enum dc_aspect_ratio
-get_aspect_ratio(const struct drm_display_mode *mode_in)
-{
-	/* 1-1 mapping, since both enums follow the HDMI spec. */
-	return (enum dc_aspect_ratio) mode_in->picture_aspect_ratio;
-}
+		/* Skip eDP detection, when there is no sink present */
+		if (aconnector->dc_link->connector_signal == SIGNAL_TYPE_EDP &&
+		    !aconnector->dc_link->edp_sink_present)
+			continue;
 
-static enum dc_color_space
-get_output_color_space(const struct dc_crtc_timing *dc_crtc_timing,
-		       const struct drm_connector_state *connector_state)
-{
-	enum dc_color_space color_space = COLOR_SPACE_SRGB;
+		guard(mutex)(&aconnector->hpd_lock);
+		if (!dc_link_detect_connection_type(aconnector->dc_link, &new_connection_type))
+			drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n");
 
-	switch (connector_state->colorspace) {
-	case DRM_MODE_COLORIMETRY_BT601_YCC:
-		if (dc_crtc_timing->flags.Y_ONLY)
-			color_space = COLOR_SPACE_YCBCR601_LIMITED;
-		else
-			color_space = COLOR_SPACE_YCBCR601;
-		break;
-	case DRM_MODE_COLORIMETRY_BT709_YCC:
-		if (dc_crtc_timing->flags.Y_ONLY)
-			color_space = COLOR_SPACE_YCBCR709_LIMITED;
-		else
-			color_space = COLOR_SPACE_YCBCR709;
-		break;
-	case DRM_MODE_COLORIMETRY_OPRGB:
-		color_space = COLOR_SPACE_ADOBERGB;
-		break;
-	case DRM_MODE_COLORIMETRY_BT2020_RGB:
-	case DRM_MODE_COLORIMETRY_BT2020_YCC:
-		if (dc_crtc_timing->pixel_encoding == PIXEL_ENCODING_RGB)
-			color_space = COLOR_SPACE_2020_RGB_FULLRANGE;
-		else
-			color_space = COLOR_SPACE_2020_YCBCR_LIMITED;
-		break;
-	case DRM_MODE_COLORIMETRY_DEFAULT: // ITU601
-	default:
-		if (dc_crtc_timing->pixel_encoding == PIXEL_ENCODING_RGB) {
-			color_space = COLOR_SPACE_SRGB;
-			if (connector_state->hdmi.broadcast_rgb == DRM_HDMI_BROADCAST_RGB_LIMITED)
-				color_space = COLOR_SPACE_SRGB_LIMITED;
-		/*
-		 * 27030khz is the separation point between HDTV and SDTV
-		 * according to HDMI spec, we use YCbCr709 and YCbCr601
-		 * respectively
-		 */
-		} else if (dc_crtc_timing->pix_clk_100hz > 270300) {
-			if (dc_crtc_timing->flags.Y_ONLY)
-				color_space =
-					COLOR_SPACE_YCBCR709_LIMITED;
-			else
-				color_space = COLOR_SPACE_YCBCR709;
+		if (aconnector->base.force && new_connection_type == dc_connection_none) {
+			amdgpu_dm_emulated_link_detect(aconnector->dc_link);
 		} else {
-			if (dc_crtc_timing->flags.Y_ONLY)
-				color_space =
-					COLOR_SPACE_YCBCR601_LIMITED;
-			else
-				color_space = COLOR_SPACE_YCBCR601;
+			guard(mutex)(&dm->dc_lock);
+			dc_exit_ips_for_hw_access(dm->dc);
+			ret = dc_link_detect(aconnector->dc_link, DETECT_REASON_RESUMEFROMS3S4);
+			if (ret) {
+				/* w/a delay for certain panels */
+				amdgpu_dm_apply_delay_after_dpcd_poweroff(adev, aconnector->dc_sink);
+			}
 		}
-		break;
-	}
 
-	return color_space;
-}
+		if (aconnector->fake_enable && aconnector->dc_link->local_sink)
+			aconnector->fake_enable = false;
 
-static enum display_content_type
-get_output_content_type(const struct drm_connector_state *connector_state)
-{
-	switch (connector_state->content_type) {
-	default:
-	case DRM_MODE_CONTENT_TYPE_NO_DATA:
-		return DISPLAY_CONTENT_TYPE_NO_DATA;
-	case DRM_MODE_CONTENT_TYPE_GRAPHICS:
-		return DISPLAY_CONTENT_TYPE_GRAPHICS;
-	case DRM_MODE_CONTENT_TYPE_PHOTO:
-		return DISPLAY_CONTENT_TYPE_PHOTO;
-	case DRM_MODE_CONTENT_TYPE_CINEMA:
-		return DISPLAY_CONTENT_TYPE_CINEMA;
-	case DRM_MODE_CONTENT_TYPE_GAME:
-		return DISPLAY_CONTENT_TYPE_GAME;
+		if (aconnector->dc_sink)
+			dc_sink_release(aconnector->dc_sink);
+		aconnector->dc_sink = NULL;
+		amdgpu_dm_update_connector_after_detect(aconnector);
 	}
-}
+	drm_connector_list_iter_end(&iter);
 
-static bool adjust_colour_depth_from_display_info(
-	struct dc_crtc_timing *timing_out,
-	const struct drm_display_info *info)
-{
-	enum dc_color_depth depth = timing_out->display_color_depth;
-	int normalized_clk;
-
-	do {
-		normalized_clk = timing_out->pix_clk_100hz / 10;
-		/* YCbCr 4:2:0 requires additional adjustment of 1/2 */
-		if (timing_out->pixel_encoding == PIXEL_ENCODING_YCBCR420)
-			normalized_clk /= 2;
-		/* Adjusting pix clock following on HDMI spec based on colour depth */
-		switch (depth) {
-		case COLOR_DEPTH_888:
-			break;
-		case COLOR_DEPTH_101010:
-			normalized_clk = (normalized_clk * 30) / 24;
-			break;
-		case COLOR_DEPTH_121212:
-			normalized_clk = (normalized_clk * 36) / 24;
-			break;
-		case COLOR_DEPTH_161616:
-			normalized_clk = (normalized_clk * 48) / 24;
-			break;
-		default:
-			/* The above depths are the only ones valid for HDMI. */
-			return false;
-		}
-		if (normalized_clk <= info->max_tmds_clock) {
-			timing_out->display_color_depth = depth;
-			return true;
-		}
-	} while (--depth > COLOR_DEPTH_666);
-	return false;
-}
+	dm_destroy_cached_state(adev);
 
-static void fill_stream_properties_from_drm_display_mode(
-	struct dc_stream_state *stream,
-	const struct drm_display_mode *mode_in,
-	const struct drm_connector *connector,
-	const struct drm_connector_state *connector_state,
-	const struct dc_stream_state *old_stream,
-	int requested_bpc)
-{
-	struct dc_crtc_timing *timing_out = &stream->timing;
-	const struct drm_display_info *info = &connector->display_info;
-	struct amdgpu_dm_connector *aconnector = NULL;
-	struct hdmi_vendor_infoframe hv_frame;
-	struct hdmi_avi_infoframe avi_frame;
-	ssize_t err;
+	/* Do mst topology probing after resuming cached state*/
+	drm_connector_list_iter_begin(ddev, &iter);
+	drm_for_each_connector_iter(connector, &iter) {
+		bool init = false;
+
+		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
+			continue;
 
-	if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK)
 		aconnector = to_amdgpu_dm_connector(connector);
+		if (aconnector->dc_link->type != dc_connection_mst_branch ||
+		    aconnector->mst_root)
+			continue;
 
-	memset(&hv_frame, 0, sizeof(hv_frame));
-	memset(&avi_frame, 0, sizeof(avi_frame));
-
-	timing_out->h_border_left = 0;
-	timing_out->h_border_right = 0;
-	timing_out->v_border_top = 0;
-	timing_out->v_border_bottom = 0;
-	/* TODO: un-hardcode */
-	if (drm_mode_is_420_only(info, mode_in)
-			&& (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A ||
-			    stream->signal == SIGNAL_TYPE_HDMI_FRL)
-			&& aconnector
-			&& aconnector->force_yuv_pixel_format == PIXEL_ENCODING_YCBCR420)
-		timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR420;
-	else if (drm_mode_is_420_also(info, mode_in)
-			&& aconnector
-			&& (aconnector->force_yuv_pixel_format == PIXEL_ENCODING_YCBCR420
-			|| aconnector->force_yuv420_output))
-		timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR420;
-	else if ((connector->display_info.color_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422))
-			&& aconnector
-			&& (aconnector->force_yuv_pixel_format == PIXEL_ENCODING_YCBCR422
-			|| aconnector->force_yuv422_output))
-		timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR422;
-	else if ((connector->display_info.color_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444))
-			&& (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A ||
-			    stream->signal == SIGNAL_TYPE_HDMI_FRL)
-			&& aconnector
-			&& aconnector->force_yuv_pixel_format == PIXEL_ENCODING_YCBCR444)
-		timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR444;
-	else
-		timing_out->pixel_encoding = PIXEL_ENCODING_RGB;
-
-	timing_out->timing_3d_format = TIMING_3D_FORMAT_NONE;
-	timing_out->display_color_depth = convert_color_depth_from_display_info(
-		connector,
-		(timing_out->pixel_encoding == PIXEL_ENCODING_YCBCR420),
-		requested_bpc);
-	timing_out->scan_type = SCANNING_TYPE_NODATA;
-	timing_out->hdmi_vic = 0;
-
-	if (old_stream) {
-		timing_out->vic = old_stream->timing.vic;
-		timing_out->flags.HSYNC_POSITIVE_POLARITY = old_stream->timing.flags.HSYNC_POSITIVE_POLARITY;
-		timing_out->flags.VSYNC_POSITIVE_POLARITY = old_stream->timing.flags.VSYNC_POSITIVE_POLARITY;
-	} else {
-		timing_out->vic = drm_match_cea_mode(mode_in);
-		if (mode_in->flags & DRM_MODE_FLAG_PHSYNC)
-			timing_out->flags.HSYNC_POSITIVE_POLARITY = 1;
-		if (mode_in->flags & DRM_MODE_FLAG_PVSYNC)
-			timing_out->flags.VSYNC_POSITIVE_POLARITY = 1;
-	}
-
-	if (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A ||
-		stream->signal == SIGNAL_TYPE_HDMI_FRL) {
-		err = drm_hdmi_avi_infoframe_from_display_mode(&avi_frame,
-							       (struct drm_connector *)connector,
-							       mode_in);
-		if (err < 0)
-			drm_warn_once(connector->dev, "Failed to setup avi infoframe on connector %s: %zd\n",
-				      connector->name, err);
-		timing_out->vic = avi_frame.video_code;
-		err = drm_hdmi_vendor_infoframe_from_display_mode(&hv_frame,
-								  (struct drm_connector *)connector,
-								  mode_in);
-		if (err < 0)
-			drm_warn_once(connector->dev, "Failed to setup vendor infoframe on connector %s: %zd\n",
-				      connector->name, err);
-		timing_out->hdmi_vic = hv_frame.vic;
-	}
-
-	if (aconnector && is_freesync_video_mode(mode_in, aconnector)) {
-		timing_out->h_addressable = mode_in->hdisplay;
-		timing_out->h_total = mode_in->htotal;
-		timing_out->h_sync_width = mode_in->hsync_end - mode_in->hsync_start;
-		timing_out->h_front_porch = mode_in->hsync_start - mode_in->hdisplay;
-		timing_out->v_total = mode_in->vtotal;
-		timing_out->v_addressable = mode_in->vdisplay;
-		timing_out->v_front_porch = mode_in->vsync_start - mode_in->vdisplay;
-		timing_out->v_sync_width = mode_in->vsync_end - mode_in->vsync_start;
-		timing_out->pix_clk_100hz = mode_in->clock * 10;
-	} else {
-		timing_out->h_addressable = mode_in->crtc_hdisplay;
-		timing_out->h_total = mode_in->crtc_htotal;
-		timing_out->h_sync_width = mode_in->crtc_hsync_end - mode_in->crtc_hsync_start;
-		timing_out->h_front_porch = mode_in->crtc_hsync_start - mode_in->crtc_hdisplay;
-		timing_out->v_total = mode_in->crtc_vtotal;
-		timing_out->v_addressable = mode_in->crtc_vdisplay;
-		timing_out->v_front_porch = mode_in->crtc_vsync_start - mode_in->crtc_vdisplay;
-		timing_out->v_sync_width = mode_in->crtc_vsync_end - mode_in->crtc_vsync_start;
-		timing_out->pix_clk_100hz = mode_in->crtc_clock * 10;
+		scoped_guard(mutex, &aconnector->mst_mgr.lock) {
+			init = !aconnector->mst_mgr.mst_primary;
+		}
+		if (init)
+			dm_helpers_dp_mst_start_top_mgr(aconnector->dc_link->ctx,
+				aconnector->dc_link, false);
+		else
+			drm_dp_mst_topology_queue_probe(&aconnector->mst_mgr);
 	}
+	drm_connector_list_iter_end(&iter);
+
+	/* Debug dump: list all DC links and their associated sinks after detection
+	 * is complete for all connectors. This provides a comprehensive view of the
+	 * final state without repeating the dump for each connector.
+	 */
+	amdgpu_dm_dump_links_and_sinks(adev);
 
-	timing_out->aspect_ratio = get_aspect_ratio(mode_in);
+	amdgpu_dm_irq_resume_late(adev);
 
-	stream->out_transfer_func.type = TF_TYPE_PREDEFINED;
-	stream->out_transfer_func.tf = TRANSFER_FUNCTION_SRGB;
-	if (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A) {
-		if (!adjust_colour_depth_from_display_info(timing_out, info) &&
-		    drm_mode_is_420_also(info, mode_in) &&
-		    timing_out->pixel_encoding != PIXEL_ENCODING_YCBCR420) {
-			timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR420;
-			adjust_colour_depth_from_display_info(timing_out, info);
-		}
-	}
+	amdgpu_dm_smu_write_watermarks_table(adev);
 
-	stream->output_color_space = get_output_color_space(timing_out, connector_state);
-	stream->content_type = get_output_content_type(connector_state);
+	drm_kms_helper_hotplug_event(ddev);
+
+	return 0;
 }
 
-static void fill_audio_info(struct audio_info *audio_info,
-			    const struct drm_connector *drm_connector,
-			    const struct dc_sink *dc_sink)
-{
-	int i = 0;
-	int cea_revision = 0;
-	const struct dc_edid_caps *edid_caps = &dc_sink->edid_caps;
+/**
+ * DOC: DM Lifecycle
+ *
+ * DM (and consequently DC) is registered in the amdgpu base driver as a IP
+ * block. When CONFIG_DRM_AMD_DC is enabled, the DM device IP block is added to
+ * the base driver's device list to be initialized and torn down accordingly.
+ *
+ * The functions to do so are provided as hooks in &struct amd_ip_funcs.
+ */
 
-	audio_info->manufacture_id = edid_caps->manufacturer_id;
-	audio_info->product_id = edid_caps->product_id;
+static const struct amd_ip_funcs amdgpu_dm_funcs = {
+	.name = "dm",
+	.early_init = dm_early_init,
+	.late_init = dm_late_init,
+	.sw_init = dm_sw_init,
+	.sw_fini = dm_sw_fini,
+	.early_fini = amdgpu_dm_early_fini,
+	.hw_init = dm_hw_init,
+	.hw_fini = dm_hw_fini,
+	.suspend = dm_suspend,
+	.resume = dm_resume,
+	.is_idle = dm_is_idle,
+	.wait_for_idle = dm_wait_for_idle,
+	.soft_reset = dm_soft_reset,
+	.set_clockgating_state = dm_set_clockgating_state,
+	.set_powergating_state = dm_set_powergating_state,
+};
 
-	cea_revision = drm_connector->display_info.cea_rev;
+const struct amdgpu_ip_block_version dm_ip_block = {
+	.type = AMD_IP_BLOCK_TYPE_DCE,
+	.major = 1,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &amdgpu_dm_funcs,
+};
 
-	strscpy(audio_info->display_name,
-		edid_caps->display_name,
-		AUDIO_INFO_DISPLAY_NAME_SIZE_IN_CHARS);
 
-	if (cea_revision >= 3) {
-		audio_info->mode_count = edid_caps->audio_mode_count;
+/**
+ * DOC: atomic
+ *
+ * *WIP*
+ */
 
-		for (i = 0; i < audio_info->mode_count; ++i) {
-			audio_info->modes[i].format_code =
-					(enum audio_format_code)
-					(edid_caps->audio_modes[i].format_code);
-			audio_info->modes[i].channel_count =
-					edid_caps->audio_modes[i].channel_count;
-			audio_info->modes[i].sample_rates.all =
-					edid_caps->audio_modes[i].sample_rate;
-			audio_info->modes[i].sample_size =
-					edid_caps->audio_modes[i].sample_size;
-		}
-	}
+static const struct drm_mode_config_funcs amdgpu_dm_mode_funcs = {
+	.fb_create = amdgpu_display_user_framebuffer_create,
+	.get_format_info = amdgpu_dm_plane_get_format_info,
+	.atomic_check = amdgpu_dm_atomic_check,
+	.atomic_commit = drm_atomic_helper_commit,
+};
 
-	audio_info->flags.all = edid_caps->speaker_flags;
+static struct drm_mode_config_helper_funcs amdgpu_dm_mode_config_helperfuncs = {
+	.atomic_commit_tail = amdgpu_dm_atomic_commit_tail,
+	.atomic_commit_setup = amdgpu_dm_atomic_setup_commit,
+};
 
-	/* TODO: We only check for the progressive mode, check for interlace mode too */
-	if (drm_connector->latency_present[0]) {
-		audio_info->video_latency = drm_connector->video_latency[0];
-		audio_info->audio_latency = drm_connector->audio_latency[0];
-	}
+/*
+ * Acquires the lock for the atomic state object and returns
+ * the new atomic state.
+ *
+ * This should only be called during atomic check.
+ */
+int dm_atomic_get_state(struct drm_atomic_commit *state,
+			struct dm_atomic_state **dm_state)
+{
+	struct drm_device *dev = state->dev;
+	struct amdgpu_device *adev = drm_to_adev(dev);
+	struct amdgpu_display_manager *dm = &adev->dm;
+	struct drm_private_state *priv_state;
 
-	/* TODO: For DP, video and audio latency should be calculated from DPCD caps */
+	if (*dm_state)
+		return 0;
 
-}
+	priv_state = drm_atomic_get_private_obj_state(state, &dm->atomic_obj);
+	if (IS_ERR(priv_state))
+		return PTR_ERR(priv_state);
 
-static void
-copy_crtc_timing_for_drm_display_mode(const struct drm_display_mode *src_mode,
-				      struct drm_display_mode *dst_mode)
-{
-	dst_mode->crtc_hdisplay = src_mode->crtc_hdisplay;
-	dst_mode->crtc_vdisplay = src_mode->crtc_vdisplay;
-	dst_mode->crtc_clock = src_mode->crtc_clock;
-	dst_mode->crtc_hblank_start = src_mode->crtc_hblank_start;
-	dst_mode->crtc_hblank_end = src_mode->crtc_hblank_end;
-	dst_mode->crtc_hsync_start =  src_mode->crtc_hsync_start;
-	dst_mode->crtc_hsync_end = src_mode->crtc_hsync_end;
-	dst_mode->crtc_htotal = src_mode->crtc_htotal;
-	dst_mode->crtc_hskew = src_mode->crtc_hskew;
-	dst_mode->crtc_vblank_start = src_mode->crtc_vblank_start;
-	dst_mode->crtc_vblank_end = src_mode->crtc_vblank_end;
-	dst_mode->crtc_vsync_start = src_mode->crtc_vsync_start;
-	dst_mode->crtc_vsync_end = src_mode->crtc_vsync_end;
-	dst_mode->crtc_vtotal = src_mode->crtc_vtotal;
+	*dm_state = to_dm_atomic_state(priv_state);
+
+	return 0;
 }
 
-static void
-decide_crtc_timing_for_drm_display_mode(struct drm_display_mode *drm_mode,
-					const struct drm_display_mode *native_mode,
-					bool scale_enabled)
+static struct dm_atomic_state *
+dm_atomic_get_new_state(struct drm_atomic_commit *state)
 {
-	if (scale_enabled || (
-	    native_mode->clock == drm_mode->clock &&
-	    native_mode->htotal == drm_mode->htotal &&
-	    native_mode->vtotal == drm_mode->vtotal)) {
-		if (native_mode->crtc_clock)
-			copy_crtc_timing_for_drm_display_mode(native_mode, drm_mode);
-	} else {
-		/* no scaling nor amdgpu inserted, no need to patch */
+	struct drm_device *dev = state->dev;
+	struct amdgpu_device *adev = drm_to_adev(dev);
+	struct amdgpu_display_manager *dm = &adev->dm;
+	struct drm_private_obj *obj;
+	struct drm_private_state *new_obj_state;
+	int i;
+
+	for_each_new_private_obj_in_state(state, obj, new_obj_state, i) {
+		if (obj->funcs == dm->atomic_obj.funcs)
+			return to_dm_atomic_state(new_obj_state);
 	}
+
+	return NULL;
 }
 
-static struct dc_sink *
-create_fake_sink(struct drm_device *dev, struct dc_link *link)
+static struct drm_private_state *
+dm_atomic_duplicate_state(struct drm_private_obj *obj)
 {
-	struct dc_sink_init_data sink_init_data = { 0 };
-	struct dc_sink *sink = NULL;
+	struct dm_atomic_state *old_state, *new_state;
 
-	sink_init_data.link = link;
-	sink_init_data.sink_signal = link->connector_signal;
+	new_state = kzalloc_obj(*new_state);
+	if (!new_state)
+		return NULL;
 
-	sink = dc_sink_create(&sink_init_data);
-	if (!sink) {
-		drm_err(dev, "Failed to create sink!\n");
+	__drm_atomic_helper_private_obj_duplicate_state(obj, &new_state->base);
+
+	old_state = to_dm_atomic_state(obj->state);
+
+	if (old_state && old_state->context)
+		new_state->context = dc_state_create_copy(old_state->context);
+
+	if (!new_state->context) {
+		kfree(new_state);
 		return NULL;
 	}
-	sink->sink_signal = SIGNAL_TYPE_VIRTUAL;
 
-	return sink;
+	return &new_state->base;
 }
 
-static void set_multisync_trigger_params(
-		struct dc_stream_state *stream)
+static void dm_atomic_destroy_state(struct drm_private_obj *obj,
+				    struct drm_private_state *state)
 {
-	struct dc_stream_state *master = NULL;
+	struct dm_atomic_state *dm_state = to_dm_atomic_state(state);
 
-	if (stream->triggered_crtc_reset.enabled) {
-		master = stream->triggered_crtc_reset.event_source;
-		stream->triggered_crtc_reset.event =
-			master->timing.flags.VSYNC_POSITIVE_POLARITY ?
-			CRTC_EVENT_VSYNC_RISING : CRTC_EVENT_VSYNC_FALLING;
-		stream->triggered_crtc_reset.delay = TRIGGER_DELAY_NEXT_PIXEL;
-	}
+	if (dm_state && dm_state->context)
+		dc_state_release(dm_state->context);
+
+	kfree(dm_state);
 }
 
-static void set_master_stream(struct dc_stream_state *stream_set[],
-			      int stream_count)
+static struct drm_private_state *
+dm_atomic_create_state(struct drm_private_obj *obj)
 {
-	int j, highest_rfr = 0, master_stream = 0;
+	struct amdgpu_device *adev = drm_to_adev(obj->dev);
+	struct dm_atomic_state *dm_state;
+	struct dc_state *context;
 
-	for (j = 0;  j < stream_count; j++) {
-		if (stream_set[j] && stream_set[j]->triggered_crtc_reset.enabled) {
-			int refresh_rate = 0;
+	dm_state = kzalloc_obj(*dm_state);
+	if (!dm_state)
+		return ERR_PTR(-ENOMEM);
 
-			refresh_rate = (stream_set[j]->timing.pix_clk_100hz*100)/
-				(stream_set[j]->timing.h_total*stream_set[j]->timing.v_total);
-			if (refresh_rate > highest_rfr) {
-				highest_rfr = refresh_rate;
-				master_stream = j;
-			}
-		}
-	}
-	for (j = 0;  j < stream_count; j++) {
-		if (stream_set[j])
-			stream_set[j]->triggered_crtc_reset.event_source = stream_set[master_stream];
+	context = dc_state_create_current_copy(adev->dm.dc);
+	if (!context) {
+		kfree(dm_state);
+		return ERR_PTR(-ENOMEM);
 	}
+
+	__drm_atomic_helper_private_obj_create_state(obj, &dm_state->base);
+	dm_state->context = context;
+
+	return &dm_state->base;
 }
 
-static void dm_enable_per_frame_crtc_master_sync(struct dc_state *context)
+static struct drm_private_state_funcs dm_atomic_state_funcs = {
+	.atomic_create_state = dm_atomic_create_state,
+	.atomic_duplicate_state = dm_atomic_duplicate_state,
+	.atomic_destroy_state = dm_atomic_destroy_state,
+};
+
+static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev)
 {
-	int i = 0;
-	struct dc_stream_state *stream;
+	int r;
 
-	if (context->stream_count < 2)
-		return;
-	for (i = 0; i < context->stream_count ; i++) {
-		if (!context->streams[i])
-			continue;
-		/*
-		 * TODO: add a function to read AMD VSDB bits and set
-		 * crtc_sync_master.multi_sync_enabled flag
-		 * For now it's set to false
-		 */
-	}
+	adev->mode_info.mode_config_initialized = true;
 
-	set_master_stream(context->streams, context->stream_count);
+	adev_to_drm(adev)->mode_config.funcs = (void *)&amdgpu_dm_mode_funcs;
+	adev_to_drm(adev)->mode_config.helper_private = &amdgpu_dm_mode_config_helperfuncs;
 
-	for (i = 0; i < context->stream_count ; i++) {
-		stream = context->streams[i];
+	adev_to_drm(adev)->mode_config.max_width = 16384;
+	adev_to_drm(adev)->mode_config.max_height = 16384;
 
-		if (!stream)
-			continue;
+	adev_to_drm(adev)->mode_config.preferred_depth = 24;
+	if (adev->asic_type == CHIP_HAWAII)
+		/* disable prefer shadow for now due to hibernation issues */
+		adev_to_drm(adev)->mode_config.prefer_shadow = 0;
+	else
+		adev_to_drm(adev)->mode_config.prefer_shadow = 1;
+	/* indicates support for immediate flip */
+	adev_to_drm(adev)->mode_config.async_page_flip = true;
 
-		set_multisync_trigger_params(stream);
-	}
-}
+	drm_atomic_private_obj_init(adev_to_drm(adev),
+				    &adev->dm.atomic_obj,
+				    &dm_atomic_state_funcs);
 
-/**
- * DOC: FreeSync Video
- *
- * When a userspace application wants to play a video, the content follows a
- * standard format definition that usually specifies the FPS for that format.
- * The below list illustrates some video format and the expected FPS,
- * respectively:
- *
- * - TV/NTSC (23.976 FPS)
- * - Cinema (24 FPS)
- * - TV/PAL (25 FPS)
- * - TV/NTSC (29.97 FPS)
- * - TV/NTSC (30 FPS)
- * - Cinema HFR (48 FPS)
- * - TV/PAL (50 FPS)
- * - Commonly used (60 FPS)
- * - Multiples of 24 (48,72,96 FPS)
- *
- * The list of standards video format is not huge and can be added to the
- * connector modeset list beforehand. With that, userspace can leverage
- * FreeSync to extends the front porch in order to attain the target refresh
- * rate. Such a switch will happen seamlessly, without screen blanking or
- * reprogramming of the output in any other way. If the userspace requests a
- * modesetting change compatible with FreeSync modes that only differ in the
- * refresh rate, DC will skip the full update and avoid blink during the
- * transition. For example, the video player can change the modesetting from
- * 60Hz to 30Hz for playing TV/NTSC content when it goes full screen without
- * causing any display blink. This same concept can be applied to a mode
- * setting change.
- */
-static struct drm_display_mode *
-get_highest_refresh_rate_mode(struct amdgpu_dm_connector *aconnector,
-		bool use_probed_modes)
-{
-	struct drm_display_mode *m, *m_pref = NULL;
-	u16 current_refresh, highest_refresh;
-	struct list_head *list_head = use_probed_modes ?
-		&aconnector->base.probed_modes :
-		&aconnector->base.modes;
+	r = amdgpu_display_modeset_create_props(adev);
+	if (r)
+		return r;
 
-	if (aconnector->base.connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
-		return NULL;
+#ifdef AMD_PRIVATE_COLOR
+	if (amdgpu_dm_create_color_properties(adev))
+		return -ENOMEM;
+#endif
 
-	if (aconnector->freesync_vid_base.clock != 0)
-		return &aconnector->freesync_vid_base;
+	r = amdgpu_dm_audio_init(adev);
+	if (r)
+		return r;
 
-	/* Find the preferred mode */
-	list_for_each_entry(m, list_head, head) {
-		if (m->type & DRM_MODE_TYPE_PREFERRED) {
-			m_pref = m;
-			break;
-		}
-	}
+	return 0;
+}
 
-	if (!m_pref) {
-		/* Probably an EDID with no preferred mode. Fallback to first entry */
-		m_pref = list_first_entry_or_null(
-				&aconnector->base.modes, struct drm_display_mode, head);
-		if (!m_pref) {
-			drm_dbg_driver(aconnector->base.dev, "No preferred mode found in EDID\n");
-			return NULL;
-		}
-	}
+static int initialize_plane(struct amdgpu_display_manager *dm,
+			    struct amdgpu_mode_info *mode_info, int plane_id,
+			    enum drm_plane_type plane_type,
+			    const struct dc_plane_cap *plane_cap)
+{
+	struct drm_plane *plane;
+	unsigned long possible_crtcs;
+	int ret = 0;
 
-	highest_refresh = drm_mode_vrefresh(m_pref);
+	plane = kzalloc_obj(struct drm_plane);
+	if (!plane) {
+		drm_err(adev_to_drm(dm->adev), "KMS: Failed to allocate plane\n");
+		return -ENOMEM;
+	}
+	plane->type = plane_type;
 
 	/*
-	 * Find the mode with highest refresh rate with same resolution.
-	 * For some monitors, preferred mode is not the mode with highest
-	 * supported refresh rate.
+	 * HACK: IGT tests expect that the primary plane for a CRTC
+	 * can only have one possible CRTC. Only expose support for
+	 * any CRTC if they're not going to be used as a primary plane
+	 * for a CRTC - like overlay or underlay planes.
 	 */
-	list_for_each_entry(m, list_head, head) {
-		current_refresh  = drm_mode_vrefresh(m);
+	possible_crtcs = 1 << plane_id;
+	if (plane_id >= dm->dc->caps.max_streams)
+		possible_crtcs = 0xff;
 
-		if (m->hdisplay == m_pref->hdisplay &&
-		    m->vdisplay == m_pref->vdisplay &&
-		    highest_refresh < current_refresh) {
-			highest_refresh = current_refresh;
-			m_pref = m;
-		}
+	ret = amdgpu_dm_plane_init(dm, plane, possible_crtcs, plane_cap);
+
+	if (ret) {
+		drm_err(adev_to_drm(dm->adev), "KMS: Failed to initialize plane\n");
+		kfree(plane);
+		return ret;
 	}
 
-	drm_mode_copy(&aconnector->freesync_vid_base, m_pref);
-	return m_pref;
+	if (mode_info)
+		mode_info->planes[plane_id] = plane;
+
+	return ret;
 }
 
-static bool is_freesync_video_mode(const struct drm_display_mode *mode,
-		struct amdgpu_dm_connector *aconnector)
-{
-	struct drm_display_mode *high_mode;
-	int timing_diff;
 
-	high_mode = get_highest_refresh_rate_mode(aconnector, false);
-	if (!high_mode || !mode)
-		return false;
+/*
+ * In this architecture, the association
+ * connector -> encoder -> crtc
+ * id not really requried. The crtc and connector will hold the
+ * display_index as an abstraction to use with DAL component
+ *
+ * Returns 0 on success
+ */
+static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
+{
+	struct amdgpu_display_manager *dm = &adev->dm;
+	s32 i;
+	struct amdgpu_dm_connector *aconnector = NULL;
+	struct amdgpu_encoder *aencoder = NULL;
+	struct amdgpu_mode_info *mode_info = &adev->mode_info;
+	u32 link_cnt;
+	s32 primary_planes;
+	enum dc_connection_type new_connection_type = dc_connection_none;
+	const struct dc_plane_cap *plane;
+	bool psr_feature_enabled = false;
+	bool replay_feature_enabled = false;
+	int max_overlay = dm->dc->caps.max_slave_planes;
 
-	timing_diff = high_mode->vtotal - mode->vtotal;
-
-	if (high_mode->clock == 0 || high_mode->clock != mode->clock ||
-	    high_mode->hdisplay != mode->hdisplay ||
-	    high_mode->vdisplay != mode->vdisplay ||
-	    high_mode->hsync_start != mode->hsync_start ||
-	    high_mode->hsync_end != mode->hsync_end ||
-	    high_mode->htotal != mode->htotal ||
-	    high_mode->hskew != mode->hskew ||
-	    high_mode->vscan != mode->vscan ||
-	    high_mode->vsync_start - mode->vsync_start != timing_diff ||
-	    high_mode->vsync_end - mode->vsync_end != timing_diff)
-		return false;
-	else
-		return true;
-}
+	dm->display_indexes_num = dm->dc->caps.max_streams;
+	/* Update the actual used number of crtc */
+	adev->mode_info.num_crtc = adev->dm.display_indexes_num;
 
-#if defined(CONFIG_DRM_AMD_DC_FP)
-static void update_dsc_caps(struct amdgpu_dm_connector *aconnector,
-			    struct dc_sink *sink, struct dc_stream_state *stream,
-			    struct dsc_dec_dpcd_caps *dsc_caps)
-{
-	stream->timing.flags.DSC = 0;
-	dsc_caps->is_dsc_supported = false;
-
-	if (aconnector->dc_link && (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT ||
-	    sink->sink_signal == SIGNAL_TYPE_EDP)) {
-		if (sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_NONE)
-			dc_dsc_parse_dsc_dpcd(aconnector->dc_link->ctx->dc,
-				aconnector->dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.raw,
-				aconnector->dc_link->dpcd_caps.dsc_caps.dsc_branch_decoder_caps.raw,
-				dsc_caps);
-		else if (sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_DP_HDMI_CONVERTER) {
-			if (aconnector->dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_PASSTHROUGH_SUPPORT &&
-					!aconnector->dsc_settings.dsc_force_disable_passthrough &&
-					aconnector->dc_link->dpcd_caps.dongle_caps.dp_hdmi_frl_max_link_bw_in_kbps > 0 &&
-					sink->edid_caps.frl_dsc_support &&
-					sink->edid_caps.max_frl_rate > 0 &&
-					sink->edid_caps.frl_dsc_max_frl_rate > 0)
-				dc_dsc_parse_dsc_edid(aconnector->dc_link->ctx->dc, &sink->edid_caps, dsc_caps);
-			else
-				dc_dsc_parse_dsc_dpcd(aconnector->dc_link->ctx->dc,
-				      aconnector->dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.raw,
-				      aconnector->dc_link->dpcd_caps.dsc_caps.dsc_branch_decoder_caps.raw,
-				      dsc_caps);
-		}
-	} else if (aconnector->dc_link && sink->sink_signal == SIGNAL_TYPE_HDMI_FRL) {
-		if (sink->edid_caps.frl_dsc_support &&
-				sink->edid_caps.max_frl_rate > 0 &&
-				sink->edid_caps.frl_dsc_max_frl_rate > 0)
-			dc_dsc_parse_dsc_edid(aconnector->dc_link->ctx->dc, &sink->edid_caps, dsc_caps);
-	}
-}
+	amdgpu_dm_set_irq_funcs(adev);
 
-static void apply_dsc_policy_for_edp(struct amdgpu_dm_connector *aconnector,
-				    struct dc_sink *sink, struct dc_stream_state *stream,
-				    struct dsc_dec_dpcd_caps *dsc_caps,
-				    uint32_t max_dsc_target_bpp_limit_override)
-{
-	const struct dc_link_settings *verified_link_cap = NULL;
-	u32 link_bw_in_kbps;
-	u32 edp_min_bpp_x16, edp_max_bpp_x16;
-	struct dc *dc = sink->ctx->dc;
-	struct dc_dsc_bw_range bw_range = {0};
-	struct dc_dsc_config dsc_cfg = {0};
-	struct dc_dsc_config_options dsc_options = {0};
-
-	dc_dsc_get_default_config_option(dc, &dsc_options);
-	dsc_options.max_target_bpp_limit_override_x16 = max_dsc_target_bpp_limit_override * 16;
-
-	verified_link_cap = dc_link_get_link_cap(stream->link);
-	link_bw_in_kbps = dc_link_bandwidth_kbps(stream->link, verified_link_cap);
-	edp_min_bpp_x16 = 8 * 16;
-	edp_max_bpp_x16 = 8 * 16;
-
-	if (edp_max_bpp_x16 > dsc_caps->edp_max_bits_per_pixel)
-		edp_max_bpp_x16 = dsc_caps->edp_max_bits_per_pixel;
-
-	if (edp_max_bpp_x16 < edp_min_bpp_x16)
-		edp_min_bpp_x16 = edp_max_bpp_x16;
-
-	if (dc_dsc_compute_bandwidth_range(dc->res_pool->dscs[0],
-				dc->debug.dsc_min_slice_height_override,
-				edp_min_bpp_x16, edp_max_bpp_x16,
-				dsc_caps,
-				&stream->timing,
-				dc_link_get_highest_encoding_format(aconnector->dc_link),
-				&bw_range)) {
-
-		if (bw_range.max_kbps < link_bw_in_kbps) {
-			if (dc_dsc_compute_config(dc->res_pool->dscs[0],
-					dsc_caps,
-					&dsc_options,
-					0,
-					&stream->timing,
-					dc_link_get_highest_encoding_format(aconnector->dc_link),
-					&dsc_cfg)) {
-				stream->timing.dsc_cfg = dsc_cfg;
-				stream->timing.flags.DSC = 1;
-				stream->timing.dsc_cfg.bits_per_pixel = edp_max_bpp_x16;
-			}
-			return;
-		}
+	link_cnt = dm->dc->caps.max_links;
+	if (amdgpu_dm_mode_config_init(dm->adev)) {
+		drm_err(adev_to_drm(adev), "DM: Failed to initialize mode config\n");
+		return -EINVAL;
 	}
 
-	if (dc_dsc_compute_config(dc->res_pool->dscs[0],
-				dsc_caps,
-				&dsc_options,
-				link_bw_in_kbps,
-				&stream->timing,
-				dc_link_get_highest_encoding_format(aconnector->dc_link),
-				&dsc_cfg)) {
-		stream->timing.dsc_cfg = dsc_cfg;
-		stream->timing.flags.DSC = 1;
+	/* There is one primary plane per CRTC */
+	primary_planes = dm->dc->caps.max_streams;
+	if (primary_planes > AMDGPU_MAX_PLANES) {
+		drm_err(adev_to_drm(adev), "DM: Plane nums out of 6 planes\n");
+		return -EINVAL;
 	}
-}
 
-static void apply_dsc_policy_for_stream(struct amdgpu_dm_connector *aconnector,
-					struct dc_sink *sink, struct dc_stream_state *stream,
-					struct dsc_dec_dpcd_caps *dsc_caps)
-{
-	struct drm_connector *drm_connector = &aconnector->base;
-	u32 link_bandwidth_kbps;
-	struct dc *dc = sink->ctx->dc;
-	const struct dc_hdmi_frl_link_settings *frl_verified_link_cap = NULL;
-	u32 converter_bw_in_kbps;
-	u32 sink_bw_in_kbps;
-	u32 dsc_sink_bw_in_kbps;
-	u32 max_supported_bw_in_kbps, timing_bw_in_kbps;
-	u32 dsc_max_supported_bw_in_kbps;
-	u32 max_dsc_target_bpp_limit_override =
-		drm_connector->display_info.max_dsc_bpp;
-	struct dc_dsc_config_options dsc_options = {0};
-
-	dc_dsc_get_default_config_option(dc, &dsc_options);
-	dsc_options.max_target_bpp_limit_override_x16 = max_dsc_target_bpp_limit_override * 16;
-
-	link_bandwidth_kbps = dc_link_bandwidth_kbps(aconnector->dc_link,
-							dc_link_get_link_cap(aconnector->dc_link));
-
-	/* Set DSC policy according to dsc_clock_en */
-	dc_dsc_policy_set_enable_dsc_when_not_needed(
-		aconnector->dsc_settings.dsc_force_enable == DSC_CLK_FORCE_ENABLE);
-
-	if (sink->sink_signal == SIGNAL_TYPE_EDP &&
-	    !aconnector->dc_link->panel_config.dsc.disable_dsc_edp &&
-	    dc->caps.edp_dsc_support && aconnector->dsc_settings.dsc_force_enable != DSC_CLK_FORCE_DISABLE) {
-
-		apply_dsc_policy_for_edp(aconnector, sink, stream, dsc_caps, max_dsc_target_bpp_limit_override);
-
-	} else if (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT) {
-		if (sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_NONE) {
-			if (dc_dsc_compute_config(aconnector->dc_link->ctx->dc->res_pool->dscs[0],
-						dsc_caps,
-						&dsc_options,
-						link_bandwidth_kbps,
-						&stream->timing,
-						dc_link_get_highest_encoding_format(aconnector->dc_link),
-						&stream->timing.dsc_cfg)) {
-				stream->timing.flags.DSC = 1;
-				drm_dbg_driver(drm_connector->dev, "%s: SST_DSC [%s] DSC is selected from SST RX\n",
-							__func__, drm_connector->name);
-			}
-		} else if (sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_DP_HDMI_CONVERTER) {
-			timing_bw_in_kbps = dc_bandwidth_in_kbps_from_timing(&stream->timing,
-					dc_link_get_highest_encoding_format(aconnector->dc_link));
-			converter_bw_in_kbps = aconnector->dc_link->dpcd_caps.dongle_caps.dp_hdmi_frl_max_link_bw_in_kbps;
-			sink_bw_in_kbps = dc_link_bw_kbps_from_raw_frl_link_rate_data(dc, sink->edid_caps.max_frl_rate);
-			dsc_sink_bw_in_kbps = dc_link_bw_kbps_from_raw_frl_link_rate_data(dc, sink->edid_caps.frl_dsc_max_frl_rate);
-
-			if (dsc_caps->is_frl) {
-				max_supported_bw_in_kbps = min(link_bandwidth_kbps, converter_bw_in_kbps);
-				max_supported_bw_in_kbps = min(max_supported_bw_in_kbps, sink_bw_in_kbps);
-				dsc_max_supported_bw_in_kbps = min(max_supported_bw_in_kbps, dsc_sink_bw_in_kbps);
-			} else {
-				max_supported_bw_in_kbps = link_bandwidth_kbps;
-				dsc_max_supported_bw_in_kbps = link_bandwidth_kbps;
-			}
+	/*
+	 * Initialize primary planes, implicit planes for legacy IOCTLS.
+	 * Order is reversed to match iteration order in atomic check.
+	 */
+	for (i = (primary_planes - 1); i >= 0; i--) {
+		plane = &dm->dc->caps.planes[i];
 
-			if (timing_bw_in_kbps > max_supported_bw_in_kbps &&
-					max_supported_bw_in_kbps > 0 &&
-					dsc_max_supported_bw_in_kbps > 0)
-				if (dc_dsc_compute_config(aconnector->dc_link->ctx->dc->res_pool->dscs[0],
-						dsc_caps,
-						&dsc_options,
-						dsc_max_supported_bw_in_kbps,
-						&stream->timing,
-						dc_link_get_highest_encoding_format(aconnector->dc_link),
-						&stream->timing.dsc_cfg)) {
-					stream->timing.flags.DSC = 1;
-					drm_dbg_driver(drm_connector->dev, "%s: SST_DSC [%s] DSC is selected from %s\n",
-							__func__, drm_connector->name,
-							(dsc_caps->is_frl == 1) ? "HDMI FRL RX" : "DP-HDMI PCON");
-				}
-		}
-	}
-	else if (aconnector->dc_link && sink->sink_signal == SIGNAL_TYPE_HDMI_FRL) {
-		struct dc_dsc_policy dsc_policy = {0};
-
-		frl_verified_link_cap = dc_link_get_frl_link_cap(stream->link);
-		if (frl_verified_link_cap->frl_link_rate != HDMI_FRL_LINK_RATE_DISABLE &&
-			aconnector->dc_link->frl_flags.force_frl_dsc) {
-			dc_dsc_policy_set_enable_dsc_when_not_needed(true);
-			dc_dsc_get_policy_for_timing(&stream->timing, 0, &dsc_policy, dc_link_get_highest_encoding_format(stream->link));
-		}
-
-		timing_bw_in_kbps = dc_bandwidth_in_kbps_from_timing(&stream->timing, DC_LINK_ENCODING_HDMI_FRL);
-		link_bandwidth_kbps = dc_link_frl_bandwidth_kbps(stream->link, frl_verified_link_cap->frl_link_rate);
-		dsc_sink_bw_in_kbps = dc_link_bw_kbps_from_raw_frl_link_rate_data(dc, sink->edid_caps.frl_dsc_max_frl_rate);
-
-		if ((timing_bw_in_kbps > link_bandwidth_kbps && dsc_sink_bw_in_kbps > 0) ||
-		    (dsc_policy.enable_dsc_when_not_needed || dsc_options.force_dsc_when_not_needed)) {
-			if (dc_dsc_compute_config(aconnector->dc_link->ctx->dc->res_pool->dscs[0],
-					dsc_caps,
-					&dsc_options,
-					dsc_sink_bw_in_kbps,
-					&stream->timing,
-					dc_link_get_highest_encoding_format(aconnector->dc_link),
-					&stream->timing.dsc_cfg)) {
-				stream->timing.flags.DSC = 1;
-				drm_dbg_driver(drm_connector->dev, "%s: HDMI_FRL_DSC [%s] DSC is selected from HDMI FRL RX\n",
-						__func__, drm_connector->name);
-			}
+		if (initialize_plane(dm, mode_info, i,
+				     DRM_PLANE_TYPE_PRIMARY, plane)) {
+			drm_err(adev_to_drm(adev), "KMS: Failed to initialize primary plane\n");
+			goto fail;
 		}
 	}
 
-	/* Overwrite the stream flag if DSC is enabled through debugfs */
-	if (aconnector->dsc_settings.dsc_force_enable == DSC_CLK_FORCE_ENABLE)
-		stream->timing.flags.DSC = 1;
-
-	if (stream->timing.flags.DSC && aconnector->dsc_settings.dsc_num_slices_h)
-		stream->timing.dsc_cfg.num_slices_h = aconnector->dsc_settings.dsc_num_slices_h;
-
-	if (stream->timing.flags.DSC && aconnector->dsc_settings.dsc_num_slices_v)
-		stream->timing.dsc_cfg.num_slices_v = aconnector->dsc_settings.dsc_num_slices_v;
-
-	if (stream->timing.flags.DSC && aconnector->dsc_settings.dsc_bits_per_pixel)
-		stream->timing.dsc_cfg.bits_per_pixel = aconnector->dsc_settings.dsc_bits_per_pixel;
-}
-#endif
-
-static struct dc_stream_state *
-create_stream_for_sink(struct drm_connector *connector,
-		       const struct drm_display_mode *drm_mode,
-		       const struct dm_connector_state *dm_state,
-		       const struct dc_stream_state *old_stream,
-		       int requested_bpc)
-{
-	struct drm_device *dev = connector->dev;
-	struct amdgpu_dm_connector *aconnector = NULL;
-	struct drm_display_mode *preferred_mode = NULL;
-	const struct drm_connector_state *con_state = &dm_state->base;
-	struct dc_stream_state *stream = NULL;
-	struct drm_display_mode mode;
-	struct drm_display_mode saved_mode;
-	struct drm_display_mode *freesync_mode = NULL;
-	bool native_mode_found = false;
-	bool recalculate_timing = false;
-	bool scale = dm_state->scaling != RMX_OFF;
-	int mode_refresh;
-	int preferred_refresh = 0;
-	enum color_transfer_func tf = TRANSFER_FUNC_UNKNOWN;
-#if defined(CONFIG_DRM_AMD_DC_FP)
-	struct dsc_dec_dpcd_caps dsc_caps = {0};
-#endif
-	struct dc_link *link = NULL;
-	struct dc_sink *sink = NULL;
-
-	drm_mode_init(&mode, drm_mode);
-	memset(&saved_mode, 0, sizeof(saved_mode));
-
-	if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) {
-		aconnector = NULL;
-		aconnector = to_amdgpu_dm_connector(connector);
-		link = aconnector->dc_link;
-	} else {
-		struct drm_writeback_connector *wbcon = NULL;
-		struct amdgpu_dm_wb_connector *dm_wbcon = NULL;
+	/*
+	 * Initialize overlay planes, index starting after primary planes.
+	 * These planes have a higher DRM index than the primary planes since
+	 * they should be considered as having a higher z-order.
+	 * Order is reversed to match iteration order in atomic check.
+	 *
+	 * Only support DCN for now, and only expose one so we don't encourage
+	 * userspace to use up all the pipes.
+	 */
+	for (i = 0; i < dm->dc->caps.max_planes; ++i) {
+		struct dc_plane_cap *plane = &dm->dc->caps.planes[i];
 
-		wbcon = drm_connector_to_writeback(connector);
-		dm_wbcon = to_amdgpu_dm_wb_connector(wbcon);
-		link = dm_wbcon->link;
-	}
+		/* Do not create overlay if MPO disabled */
+		if (amdgpu_dc_debug_mask & DC_DISABLE_MPO)
+			break;
 
-	if (!aconnector || !aconnector->dc_sink) {
-		sink = create_fake_sink(dev, link);
-		if (!sink)
-			return stream;
+		if (plane->type != DC_PLANE_TYPE_DCN_UNIVERSAL)
+			continue;
 
-	} else {
-		sink = aconnector->dc_sink;
-		dc_sink_retain(sink);
-	}
+		if (!plane->pixel_format_support.argb8888)
+			continue;
 
-	stream = dc_create_stream_for_sink(sink);
+		if (max_overlay-- == 0)
+			break;
 
-	if (stream == NULL) {
-		drm_err(dev, "Failed to create stream for sink!\n");
-		goto finish;
+		if (initialize_plane(dm, NULL, primary_planes + i,
+				     DRM_PLANE_TYPE_OVERLAY, plane)) {
+			drm_err(adev_to_drm(adev), "KMS: Failed to initialize overlay plane\n");
+			goto fail;
+		}
 	}
 
-	/* We leave this NULL for writeback connectors */
-	stream->dm_stream_context = aconnector;
+	for (i = 0; i < dm->dc->caps.max_streams; i++)
+		if (amdgpu_dm_crtc_init(dm, mode_info->planes[i], i)) {
+			drm_err(adev_to_drm(adev), "KMS: Failed to initialize crtc\n");
+			goto fail;
+		}
 
-	stream->timing.flags.LTE_340MCSC_SCRAMBLE =
-		connector->display_info.hdmi.scdc.scrambling.low_rates;
+	/* Use Outbox interrupt */
+	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
+	case IP_VERSION(3, 0, 0):
+	case IP_VERSION(3, 1, 2):
+	case IP_VERSION(3, 1, 3):
+	case IP_VERSION(3, 1, 4):
+	case IP_VERSION(3, 1, 5):
+	case IP_VERSION(3, 1, 6):
+	case IP_VERSION(3, 2, 0):
+	case IP_VERSION(3, 2, 1):
+	case IP_VERSION(2, 1, 0):
+	case IP_VERSION(3, 5, 0):
+	case IP_VERSION(3, 5, 1):
+	case IP_VERSION(3, 6, 0):
+	case IP_VERSION(4, 0, 1):
+	case IP_VERSION(4, 2, 0):
+	case IP_VERSION(4, 2, 1):
+		if (amdgpu_dm_register_outbox_irq_handlers(dm->adev)) {
+			drm_err(adev_to_drm(adev), "DM: Failed to initialize IRQ\n");
+			goto fail;
+		}
+		break;
+	default:
+		drm_dbg_kms(adev_to_drm(adev), "Unsupported DCN IP version for outbox: 0x%X\n",
+			      amdgpu_ip_version(adev, DCE_HWIP, 0));
+	}
 
-	list_for_each_entry(preferred_mode, &connector->modes, head) {
-		/* Search for preferred mode */
-		if (preferred_mode->type & DRM_MODE_TYPE_PREFERRED) {
-			native_mode_found = true;
+	/* Determine whether to enable PSR support by default. */
+	if (!(amdgpu_dc_debug_mask & DC_DISABLE_PSR)) {
+		switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
+		case IP_VERSION(3, 1, 2):
+		case IP_VERSION(3, 1, 3):
+		case IP_VERSION(3, 1, 4):
+		case IP_VERSION(3, 1, 5):
+		case IP_VERSION(3, 1, 6):
+		case IP_VERSION(3, 2, 0):
+		case IP_VERSION(3, 2, 1):
+		case IP_VERSION(3, 5, 0):
+		case IP_VERSION(3, 5, 1):
+		case IP_VERSION(3, 6, 0):
+		case IP_VERSION(4, 0, 1):
+		case IP_VERSION(4, 2, 0):
+		case IP_VERSION(4, 2, 1):
+			psr_feature_enabled = true;
+			break;
+		default:
+			psr_feature_enabled = amdgpu_dc_feature_mask & DC_PSR_MASK;
 			break;
 		}
 	}
-	if (!native_mode_found)
-		preferred_mode = list_first_entry_or_null(
-				&connector->modes,
-				struct drm_display_mode,
-				head);
-
-	mode_refresh = drm_mode_vrefresh(&mode);
 
-	if (preferred_mode == NULL) {
-		/*
-		 * This may not be an error, the use case is when we have no
-		 * usermode calls to reset and set mode upon hotplug. In this
-		 * case, we call set mode ourselves to restore the previous mode
-		 * and the modelist may not be filled in time.
-		 */
-		drm_dbg_driver(dev, "No preferred mode found\n");
-	} else if (aconnector) {
-		recalculate_timing = amdgpu_freesync_vid_mode &&
-				 is_freesync_video_mode(&mode, aconnector);
-		if (recalculate_timing) {
-			freesync_mode = get_highest_refresh_rate_mode(aconnector, false);
-			drm_mode_copy(&saved_mode, &mode);
-			saved_mode.picture_aspect_ratio = mode.picture_aspect_ratio;
-			drm_mode_copy(&mode, freesync_mode);
-			mode.picture_aspect_ratio = saved_mode.picture_aspect_ratio;
-		} else {
-			decide_crtc_timing_for_drm_display_mode(
-					&mode, preferred_mode, scale);
+	/* Determine whether to enable Replay support by default. */
+	if (!(amdgpu_dc_debug_mask & DC_DISABLE_REPLAY)) {
+		switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
+		case IP_VERSION(3, 1, 4):
+		case IP_VERSION(3, 2, 0):
+		case IP_VERSION(3, 2, 1):
+		case IP_VERSION(3, 5, 0):
+		case IP_VERSION(3, 5, 1):
+		case IP_VERSION(3, 6, 0):
+		case IP_VERSION(4, 2, 0):
+		case IP_VERSION(4, 2, 1):
+			replay_feature_enabled = true;
+			break;
 
-			preferred_refresh = drm_mode_vrefresh(preferred_mode);
+		default:
+			replay_feature_enabled = amdgpu_dc_feature_mask & DC_REPLAY_MASK;
+			break;
 		}
 	}
 
-	if (recalculate_timing)
-		drm_mode_set_crtcinfo(&saved_mode, 0);
-
-	/*
-	 * If scaling is enabled and refresh rate didn't change
-	 * we copy the vic and polarities of the old timings
-	 */
-	if (!scale || mode_refresh != preferred_refresh)
-		fill_stream_properties_from_drm_display_mode(
-			stream, &mode, connector, con_state, NULL,
-			requested_bpc);
-	else
-		fill_stream_properties_from_drm_display_mode(
-			stream, &mode, connector, con_state, old_stream,
-			requested_bpc);
-
-	/* The rest isn't needed for writeback connectors */
-	if (!aconnector)
-		goto finish;
-
-	if (aconnector->timing_changed) {
-		drm_dbg(aconnector->base.dev,
-			"overriding timing for automated test, bpc %d, changing to %d\n",
-			stream->timing.display_color_depth,
-			aconnector->timing_requested->display_color_depth);
-		stream->timing = *aconnector->timing_requested;
+	if (link_cnt > MAX_LINKS) {
+		drm_err(adev_to_drm(adev),
+			"KMS: Cannot support more than %d display indexes\n",
+				MAX_LINKS);
+		goto fail;
 	}
 
-#if defined(CONFIG_DRM_AMD_DC_FP)
-	/* SST DSC determination policy */
-	update_dsc_caps(aconnector, sink, stream, &dsc_caps);
-	if (aconnector->dsc_settings.dsc_force_enable != DSC_CLK_FORCE_DISABLE && dsc_caps.is_dsc_supported)
-		apply_dsc_policy_for_stream(aconnector, sink, stream, &dsc_caps);
-#endif
+	/* loops over all connectors on the board */
+	for (i = 0; i < link_cnt; i++) {
+		struct dc_link *link = NULL;
 
-	update_stream_scaling_settings(dev, &mode, dm_state, stream);
+		link = dc_get_link_at_index(dm->dc, i);
 
-	fill_audio_info(
-		&stream->audio_info,
-		connector,
-		sink);
+		if (link->connector_signal == SIGNAL_TYPE_VIRTUAL) {
+			struct amdgpu_dm_wb_connector *wbcon = kzalloc_obj(*wbcon);
 
-	update_stream_signal(stream, sink);
+			if (!wbcon) {
+				drm_err(adev_to_drm(adev), "KMS: Failed to allocate writeback connector\n");
+				continue;
+			}
 
-	if (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A ||
-	    stream->signal == SIGNAL_TYPE_HDMI_FRL)
-		mod_build_hf_vsif_infopacket(stream, &stream->vsp_infopacket, false, false);
+			if (amdgpu_dm_wb_connector_init(dm, wbcon, i)) {
+				drm_err(adev_to_drm(adev), "KMS: Failed to initialize writeback connector\n");
+				kfree(wbcon);
+				continue;
+			}
 
-	if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT ||
-	    stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST ||
-	    stream->signal == SIGNAL_TYPE_EDP) {
-		const struct dc_edid_caps *edid_caps;
-		unsigned int disable_colorimetry = 0;
+			link->psr_settings.psr_feature_enabled = false;
+			link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED;
 
-		if (aconnector->dc_sink) {
-			edid_caps = &aconnector->dc_sink->edid_caps;
-			disable_colorimetry = edid_caps->panel_patch.disable_colorimetry;
+			continue;
 		}
 
-		//
-		// should decide stream support vsc sdp colorimetry capability
-		// before building vsc info packet
-		//
-		stream->use_vsc_sdp_for_colorimetry = stream->link->dpcd_caps.dpcd_rev.raw >= 0x14 &&
-						      stream->link->dpcd_caps.dprx_feature.bits.VSC_SDP_COLORIMETRY_SUPPORTED &&
-						      !disable_colorimetry;
+		aconnector = kzalloc_obj(*aconnector);
+		if (!aconnector)
+			goto fail;
 
-		if (stream->out_transfer_func.tf == TRANSFER_FUNCTION_GAMMA22)
-			tf = TRANSFER_FUNC_GAMMA_22;
-		mod_build_vsc_infopacket(stream, &stream->vsc_infopacket, stream->output_color_space, tf);
-		aconnector->sr_skip_count = AMDGPU_DM_PSR_ENTRY_DELAY;
+		aencoder = kzalloc_obj(*aencoder);
+		if (!aencoder)
+			goto fail;
 
-	}
-finish:
-	dc_sink_release(sink);
+		if (amdgpu_dm_encoder_init(dm->ddev, aencoder, i)) {
+			drm_err(adev_to_drm(adev), "KMS: Failed to initialize encoder\n");
+			goto fail;
+		}
 
-	return stream;
-}
+		if (amdgpu_dm_connector_init(dm, aconnector, i, aencoder)) {
+			drm_err(adev_to_drm(adev), "KMS: Failed to initialize connector\n");
+			goto fail;
+		}
 
-/**
- * amdgpu_dm_connector_poll - Poll a connector to see if it's connected to a display
- * @aconnector: DM connector to poll (owns @base drm_connector and @dc_link)
- * @force: if true, force polling even when DAC load detection was used
- *
- * Used for connectors that don't support HPD (hotplug detection) to
- * periodically check whether the connector is connected to a display.
- *
- * When connection was determined via DAC load detection, we avoid
- * re-running it on normal polls to prevent visible glitches, unless
- * @force is set.
- *
- * Return: The probed connector status (connected/disconnected/unknown).
- */
-static enum drm_connector_status
-amdgpu_dm_connector_poll(struct amdgpu_dm_connector *aconnector, bool force)
-{
-	struct drm_connector *connector = &aconnector->base;
-	struct drm_device *dev = connector->dev;
-	struct amdgpu_device *adev = drm_to_adev(dev);
-	struct dc_link *link = aconnector->dc_link;
-	enum dc_connection_type conn_type = dc_connection_none;
-	enum drm_connector_status status = connector_status_disconnected;
-
-	/* When we determined the connection using DAC load detection,
-	 * do NOT poll the connector do detect disconnect because
-	 * that would run DAC load detection again which can cause
-	 * visible visual glitches.
-	 *
-	 * Only allow to poll such a connector again when forcing.
-	 */
-	if (!force && link->local_sink && link->type == dc_connection_analog_load)
-		return connector->status;
+		if (dm->hpd_rx_offload_wq)
+			dm->hpd_rx_offload_wq[aconnector->base.index].aconnector =
+				aconnector;
 
-	mutex_lock(&aconnector->hpd_lock);
+		if (!dc_link_detect_connection_type(link, &new_connection_type))
+			drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n");
 
-	if (dc_link_detect_connection_type(aconnector->dc_link, &conn_type) &&
-	    conn_type != dc_connection_none) {
-		mutex_lock(&adev->dm.dc_lock);
+		if (aconnector->base.force && new_connection_type == dc_connection_none) {
+			amdgpu_dm_emulated_link_detect(link);
+			amdgpu_dm_update_connector_after_detect(aconnector);
+		} else {
+			bool ret = false;
 
-		/* Only call full link detection when a sink isn't created yet,
-		 * ie. just when the display is plugged in, otherwise we risk flickering.
-		 */
-		if (link->local_sink ||
-			dc_link_detect(link, DETECT_REASON_HPD))
-			status = connector_status_connected;
+			mutex_lock(&dm->dc_lock);
+			dc_exit_ips_for_hw_access(dm->dc);
+			ret = dc_link_detect(link, DETECT_REASON_BOOT);
+			mutex_unlock(&dm->dc_lock);
 
-		mutex_unlock(&adev->dm.dc_lock);
-	}
+			if (ret) {
+				amdgpu_dm_update_connector_after_detect(aconnector);
+				amdgpu_dm_setup_backlight_device(dm, aconnector);
 
-	if (connector->status != status) {
-		if (status == connector_status_disconnected) {
-			if (link->local_sink)
-				dc_sink_release(link->local_sink);
+				/* Disable PSR if Replay can be enabled */
+				if (replay_feature_enabled)
+					if (amdgpu_dm_set_replay_caps(link, aconnector))
+						psr_feature_enabled = false;
 
-			link->local_sink = NULL;
-			link->dpcd_sink_count = 0;
-			link->type = dc_connection_none;
+				if (psr_feature_enabled) {
+					amdgpu_dm_set_psr_caps(link, aconnector);
+					drm_info(adev_to_drm(adev), "%s: PSR support %d, DC PSR ver %d, sink PSR ver %d DPCD caps 0x%x su_y_granularity %d\n",
+						 aconnector->base.name,
+						 link->psr_settings.psr_feature_enabled,
+						 link->psr_settings.psr_version,
+						 link->dpcd_caps.psr_info.psr_version,
+						 link->dpcd_caps.psr_info.psr_dpcd_caps.raw,
+						 link->dpcd_caps.psr_info.psr2_su_y_granularity_cap);
+				}
+			}
 		}
-
-		amdgpu_dm_update_connector_after_detect(aconnector);
+		amdgpu_set_panel_orientation(&aconnector->base);
 	}
 
-	mutex_unlock(&aconnector->hpd_lock);
-	return status;
-}
-
-/**
- * amdgpu_dm_connector_detect() - Detect whether a DRM connector is connected to a display
- *
- * A connector is considered connected when it has a sink that is not NULL.
- * For connectors that support HPD (hotplug detection), the connection is
- * handled in the HPD interrupt.
- * For connectors that may not support HPD, such as analog connectors,
- * DRM will call this function repeatedly to poll them.
- *
- * Notes:
- * 1. This interface is NOT called in context of HPD irq.
- * 2. This interface *is called* in context of user-mode ioctl. Which
- *    makes it a bad place for *any* MST-related activity.
- *
- * @connector: The DRM connector we are checking. We convert it to
- *             amdgpu_dm_connector so we can read the DC link and state.
- * @force:     If true, do a full detect again. This is used even when
- *             a lighter check would normally be used to avoid flicker.
- *
- * Return: The connector status (connected, disconnected, or unknown).
- *
- */
-static enum drm_connector_status
-amdgpu_dm_connector_detect(struct drm_connector *connector, bool force)
-{
-	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
-
-	update_subconnector_property(aconnector);
-
-	if (aconnector->base.force == DRM_FORCE_ON ||
-		aconnector->base.force == DRM_FORCE_ON_DIGITAL)
-		return connector_status_connected;
-	else if (aconnector->base.force == DRM_FORCE_OFF)
-		return connector_status_disconnected;
-
-	/* Poll analog connectors and only when either
-	 * disconnected or connected to an analog display.
+	/* Debug dump: list all DC links and their associated sinks after detection
+	 * is complete for all connectors. This provides a comprehensive view of the
+	 * final state without repeating the dump for each connector.
 	 */
-	if (drm_kms_helper_is_poll_worker() &&
-		dc_connector_supports_analog(aconnector->dc_link->link_id.id) &&
-		(!aconnector->dc_sink || aconnector->dc_sink->edid_caps.analog))
-		return amdgpu_dm_connector_poll(aconnector, force);
-
-	return (aconnector->dc_sink ? connector_status_connected :
-			connector_status_disconnected);
-}
-
-int amdgpu_dm_connector_atomic_set_property(struct drm_connector *connector,
-					    struct drm_connector_state *connector_state,
-					    struct drm_property *property,
-					    uint64_t val)
-{
-	struct drm_device *dev = connector->dev;
-	struct amdgpu_device *adev = drm_to_adev(dev);
-	struct dm_connector_state *dm_old_state =
-		to_dm_connector_state(connector->state);
-	struct dm_connector_state *dm_new_state =
-		to_dm_connector_state(connector_state);
-
-	int ret = -EINVAL;
-
-	if (property == dev->mode_config.scaling_mode_property) {
-		enum amdgpu_rmx_type rmx_type;
+	amdgpu_dm_dump_links_and_sinks(adev);
 
-		switch (val) {
-		case DRM_MODE_SCALE_CENTER:
-			rmx_type = RMX_CENTER;
-			break;
-		case DRM_MODE_SCALE_ASPECT:
-			rmx_type = RMX_ASPECT;
-			break;
-		case DRM_MODE_SCALE_FULLSCREEN:
-			rmx_type = RMX_FULL;
-			break;
-		case DRM_MODE_SCALE_NONE:
-		default:
-			rmx_type = RMX_OFF;
-			break;
+	/* Software is initialized. Now we can register interrupt handlers. */
+	switch (adev->asic_type) {
+#if defined(CONFIG_DRM_AMD_DC_SI)
+	case CHIP_TAHITI:
+	case CHIP_PITCAIRN:
+	case CHIP_VERDE:
+	case CHIP_OLAND:
+#endif
+	case CHIP_BONAIRE:
+	case CHIP_HAWAII:
+	case CHIP_KAVERI:
+	case CHIP_KABINI:
+	case CHIP_MULLINS:
+	case CHIP_TONGA:
+	case CHIP_FIJI:
+	case CHIP_CARRIZO:
+	case CHIP_STONEY:
+	case CHIP_POLARIS11:
+	case CHIP_POLARIS10:
+	case CHIP_POLARIS12:
+	case CHIP_VEGAM:
+	case CHIP_VEGA10:
+	case CHIP_VEGA12:
+	case CHIP_VEGA20:
+		if (amdgpu_dm_dce110_register_irq_handlers(dm->adev)) {
+			drm_err(adev_to_drm(adev), "DM: Failed to initialize IRQ\n");
+			goto fail;
 		}
-
-		if (dm_old_state->scaling == rmx_type)
-			return 0;
-
-		dm_new_state->scaling = rmx_type;
-		ret = 0;
-	} else if (property == adev->mode_info.underscan_hborder_property) {
-		dm_new_state->underscan_hborder = val;
-		ret = 0;
-	} else if (property == adev->mode_info.underscan_vborder_property) {
-		dm_new_state->underscan_vborder = val;
-		ret = 0;
-	} else if (property == adev->mode_info.underscan_property) {
-		dm_new_state->underscan_enable = val;
-		ret = 0;
-	} else if (property == adev->mode_info.abm_level_property) {
-		switch (val) {
-		case ABM_SYSFS_CONTROL:
-			dm_new_state->abm_sysfs_forbidden = false;
-			break;
-		case ABM_LEVEL_OFF:
-			dm_new_state->abm_sysfs_forbidden = true;
-			dm_new_state->abm_level = ABM_LEVEL_IMMEDIATE_DISABLE;
+		break;
+	default:
+		switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
+		case IP_VERSION(1, 0, 0):
+		case IP_VERSION(1, 0, 1):
+		case IP_VERSION(2, 0, 2):
+		case IP_VERSION(2, 0, 3):
+		case IP_VERSION(2, 0, 0):
+		case IP_VERSION(2, 1, 0):
+		case IP_VERSION(3, 0, 0):
+		case IP_VERSION(3, 0, 2):
+		case IP_VERSION(3, 0, 3):
+		case IP_VERSION(3, 0, 1):
+		case IP_VERSION(3, 1, 2):
+		case IP_VERSION(3, 1, 3):
+		case IP_VERSION(3, 1, 4):
+		case IP_VERSION(3, 1, 5):
+		case IP_VERSION(3, 1, 6):
+		case IP_VERSION(3, 2, 0):
+		case IP_VERSION(3, 2, 1):
+		case IP_VERSION(3, 5, 0):
+		case IP_VERSION(3, 5, 1):
+		case IP_VERSION(3, 6, 0):
+		case IP_VERSION(4, 0, 1):
+		case IP_VERSION(4, 2, 0):
+		case IP_VERSION(4, 2, 1):
+			if (amdgpu_dm_dcn10_register_irq_handlers(dm->adev)) {
+				drm_err(adev_to_drm(adev), "DM: Failed to initialize IRQ\n");
+				goto fail;
+			}
 			break;
 		default:
-			dm_new_state->abm_sysfs_forbidden = true;
-			dm_new_state->abm_level = val;
+			drm_err(adev_to_drm(adev), "Unsupported DCE IP versions: 0x%X\n",
+					amdgpu_ip_version(adev, DCE_HWIP, 0));
+			goto fail;
 		}
-		ret = 0;
+		break;
 	}
 
-	return ret;
+	return 0;
+fail:
+	kfree(aencoder);
+	kfree(aconnector);
+
+	return -EINVAL;
 }
 
-int amdgpu_dm_connector_atomic_get_property(struct drm_connector *connector,
-					    const struct drm_connector_state *state,
-					    struct drm_property *property,
-					    uint64_t *val)
+static void amdgpu_dm_destroy_drm_device(struct amdgpu_display_manager *dm)
 {
-	struct drm_device *dev = connector->dev;
-	struct amdgpu_device *adev = drm_to_adev(dev);
-	struct dm_connector_state *dm_state =
-		to_dm_connector_state(state);
-	int ret = -EINVAL;
-
-	if (property == dev->mode_config.scaling_mode_property) {
-		switch (dm_state->scaling) {
-		case RMX_CENTER:
-			*val = DRM_MODE_SCALE_CENTER;
-			break;
-		case RMX_ASPECT:
-			*val = DRM_MODE_SCALE_ASPECT;
-			break;
-		case RMX_FULL:
-			*val = DRM_MODE_SCALE_FULLSCREEN;
-			break;
-		case RMX_OFF:
-		default:
-			*val = DRM_MODE_SCALE_NONE;
-			break;
-		}
-		ret = 0;
-	} else if (property == adev->mode_info.underscan_hborder_property) {
-		*val = dm_state->underscan_hborder;
-		ret = 0;
-	} else if (property == adev->mode_info.underscan_vborder_property) {
-		*val = dm_state->underscan_vborder;
-		ret = 0;
-	} else if (property == adev->mode_info.underscan_property) {
-		*val = dm_state->underscan_enable;
-		ret = 0;
-	} else if (property == adev->mode_info.abm_level_property) {
-		if (!dm_state->abm_sysfs_forbidden)
-			*val = ABM_SYSFS_CONTROL;
-		else
-			*val = (dm_state->abm_level != ABM_LEVEL_IMMEDIATE_DISABLE) ?
-				dm_state->abm_level : 0;
-		ret = 0;
-	}
-
-	return ret;
+	if (dm->atomic_obj.state)
+		drm_atomic_private_obj_fini(&dm->atomic_obj);
 }
 
-/**
- * DOC: panel power savings
- *
- * The display manager allows you to set your desired **panel power savings**
- * level (between 0-4, with 0 representing off), e.g. using the following::
+/******************************************************************************
+ * amdgpu_display_funcs functions
+ *****************************************************************************/
+
+/*
+ * dm_bandwidth_update - program display watermarks
  *
- *   # echo 3 > /sys/class/drm/card0-eDP-1/amdgpu/panel_power_savings
+ * @adev: amdgpu_device pointer
  *
- * Modifying this value can have implications on color accuracy, so tread
- * carefully.
+ * Calculate and program the display watermarks and line buffer allocation.
  */
-
-static ssize_t panel_power_savings_show(struct device *device,
-					struct device_attribute *attr,
-					char *buf)
-{
-	struct drm_connector *connector = dev_get_drvdata(device);
-	struct drm_device *dev = connector->dev;
-	u8 val;
-
-	drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
-	val = to_dm_connector_state(connector->state)->abm_level ==
-		ABM_LEVEL_IMMEDIATE_DISABLE ? 0 :
-		to_dm_connector_state(connector->state)->abm_level;
-	drm_modeset_unlock(&dev->mode_config.connection_mutex);
-
-	return sysfs_emit(buf, "%u\n", val);
-}
-
-static ssize_t panel_power_savings_store(struct device *device,
-					 struct device_attribute *attr,
-					 const char *buf, size_t count)
+static void dm_bandwidth_update(struct amdgpu_device *adev)
 {
-	struct drm_connector *connector = dev_get_drvdata(device);
-	struct drm_device *dev = connector->dev;
-	long val;
-	int ret;
-
-	ret = kstrtol(buf, 0, &val);
-
-	if (ret)
-		return ret;
-
-	if (val < 0 || val > 4)
-		return -EINVAL;
-
-	drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
-	if (to_dm_connector_state(connector->state)->abm_sysfs_forbidden)
-		ret = -EBUSY;
-	else
-		to_dm_connector_state(connector->state)->abm_level = val ?:
-			ABM_LEVEL_IMMEDIATE_DISABLE;
-	drm_modeset_unlock(&dev->mode_config.connection_mutex);
-
-	if (ret)
-		return ret;
-
-	drm_kms_helper_hotplug_event(dev);
-
-	return count;
+	/* TODO: implement later */
 }
 
-static DEVICE_ATTR_RW(panel_power_savings);
-
-static struct attribute *amdgpu_attrs[] = {
-	&dev_attr_panel_power_savings.attr,
-	NULL
+static const struct amdgpu_display_funcs dm_display_funcs = {
+	.bandwidth_update = dm_bandwidth_update, /* called unconditionally */
+	.vblank_get_counter = dm_vblank_get_counter,/* called unconditionally */
+	.backlight_set_level = NULL, /* never called for DC */
+	.backlight_get_level = NULL, /* never called for DC */
+	.hpd_sense = NULL,/* called unconditionally */
+	.hpd_set_polarity = NULL, /* called unconditionally */
+	.hpd_get_gpio_reg = NULL, /* VBIOS parsing. DAL does it. */
+	.page_flip_get_scanoutpos =
+		dm_crtc_get_scanoutpos,/* called unconditionally */
+	.add_encoder = NULL, /* VBIOS parsing. DAL does it. */
+	.add_connector = NULL, /* VBIOS parsing. DAL does it. */
 };
 
-static const struct attribute_group amdgpu_group = {
-	.name = "amdgpu",
-	.attrs = amdgpu_attrs
-};
+#if defined(CONFIG_DEBUG_KERNEL_DC)
 
-static bool
-amdgpu_dm_should_create_sysfs(struct amdgpu_dm_connector *amdgpu_dm_connector)
+static ssize_t s3_debug_store(struct device *device,
+			      struct device_attribute *attr,
+			      const char *buf,
+			      size_t count)
 {
-	if (amdgpu_dm_abm_level >= 0)
-		return false;
+	int ret;
+	int s3_state;
+	struct drm_device *drm_dev = dev_get_drvdata(device);
+	struct amdgpu_device *adev = drm_to_adev(drm_dev);
+	struct amdgpu_ip_block *ip_block;
 
-	if (amdgpu_dm_connector->base.connector_type != DRM_MODE_CONNECTOR_eDP)
-		return false;
+	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_DCE);
+	if (!ip_block)
+		return -EINVAL;
 
-	/* check for OLED panels */
-	if (amdgpu_dm_connector->bl_idx >= 0) {
-		struct drm_device *drm = amdgpu_dm_connector->base.dev;
-		struct amdgpu_display_manager *dm = &drm_to_adev(drm)->dm;
-		struct amdgpu_dm_backlight_caps *caps;
+	ret = kstrtoint(buf, 0, &s3_state);
 
-		caps = &dm->backlight_caps[amdgpu_dm_connector->bl_idx];
-		if (caps->aux_support)
-			return false;
+	if (ret == 0) {
+		if (s3_state) {
+			dm_resume(ip_block);
+			drm_kms_helper_hotplug_event(adev_to_drm(adev));
+		} else
+			dm_suspend(ip_block);
 	}
 
-	return true;
-}
-
-static void amdgpu_dm_connector_unregister(struct drm_connector *connector)
-{
-	struct amdgpu_dm_connector *amdgpu_dm_connector = to_amdgpu_dm_connector(connector);
-
-	if (amdgpu_dm_should_create_sysfs(amdgpu_dm_connector))
-		sysfs_remove_group(&connector->kdev->kobj, &amdgpu_group);
-
-	cec_notifier_conn_unregister(amdgpu_dm_connector->notifier);
-	drm_dp_aux_unregister(&amdgpu_dm_connector->dm_dp_aux.aux);
+	return ret == 0 ? count : 0;
 }
 
-static void amdgpu_dm_connector_destroy(struct drm_connector *connector)
-{
-	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
-	struct amdgpu_device *adev = drm_to_adev(connector->dev);
-	struct amdgpu_display_manager *dm = &adev->dm;
-
-	/*
-	 * Call only if mst_mgr was initialized before since it's not done
-	 * for all connector types.
-	 */
-	if (aconnector->mst_mgr.dev)
-		drm_dp_mst_topology_mgr_destroy(&aconnector->mst_mgr);
-
-	/* Cancel and flush any pending HDMI HPD debounce work */
-	if (aconnector->hdmi_hpd_debounce_delay_ms) {
-		cancel_delayed_work_sync(&aconnector->hdmi_hpd_debounce_work);
-		if (aconnector->hdmi_prev_sink) {
-			dc_sink_release(aconnector->hdmi_prev_sink);
-			aconnector->hdmi_prev_sink = NULL;
-		}
-	}
-
-	if (aconnector->bl_idx != -1) {
-		backlight_device_unregister(dm->backlight_dev[aconnector->bl_idx]);
-		dm->backlight_dev[aconnector->bl_idx] = NULL;
-	}
-
-	if (aconnector->dc_em_sink)
-		dc_sink_release(aconnector->dc_em_sink);
-	aconnector->dc_em_sink = NULL;
-	if (aconnector->dc_sink)
-		dc_sink_release(aconnector->dc_sink);
-	aconnector->dc_sink = NULL;
-
-	drm_dp_cec_unregister_connector(&aconnector->dm_dp_aux.aux);
-	drm_connector_unregister(connector);
-	drm_connector_cleanup(connector);
-	kfree(aconnector->dm_dp_aux.aux.name);
+DEVICE_ATTR_WO(s3_debug);
 
-	kfree(connector);
-}
+#endif
 
-void amdgpu_dm_connector_funcs_reset(struct drm_connector *connector)
+static int dm_early_init(struct amdgpu_ip_block *ip_block)
 {
-	struct dm_connector_state *state =
-		to_dm_connector_state(connector->state);
-
-	if (connector->state)
-		__drm_atomic_helper_connector_destroy_state(connector->state);
-
-	kfree(state);
-
-	state = kzalloc_obj(*state);
-
-	if (state) {
-		state->scaling = RMX_OFF;
-		state->underscan_enable = false;
-		state->underscan_hborder = 0;
-		state->underscan_vborder = 0;
-		state->base.max_requested_bpc = 8;
-		state->vcpi_slots = 0;
-		state->pbn = 0;
-
-		if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) {
-			if (amdgpu_dm_abm_level <= 0)
-				state->abm_level = ABM_LEVEL_IMMEDIATE_DISABLE;
-			else
-				state->abm_level = amdgpu_dm_abm_level;
-		}
+	struct amdgpu_device *adev = ip_block->adev;
+	struct amdgpu_mode_info *mode_info = &adev->mode_info;
+	struct atom_context *ctx = mode_info->atom_context;
+	int index = GetIndexIntoMasterTable(DATA, Object_Header);
+	u16 data_offset;
 
-		__drm_atomic_helper_connector_reset(connector, &state->base);
+	/* if there is no object header, skip DM */
+	if (!amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) {
+		adev->harvest_ip_mask |= AMD_HARVEST_IP_DMU_MASK;
+		drm_info(adev_to_drm(adev), "No object header, skipping DM\n");
+		return -ENOENT;
 	}
-}
-
-struct drm_connector_state *
-amdgpu_dm_connector_atomic_duplicate_state(struct drm_connector *connector)
-{
-	struct dm_connector_state *state =
-		to_dm_connector_state(connector->state);
-
-	struct dm_connector_state *new_state =
-			kmemdup(state, sizeof(*state), GFP_KERNEL);
-
-	if (!new_state)
-		return NULL;
-
-	__drm_atomic_helper_connector_duplicate_state(connector, &new_state->base);
-
-	new_state->freesync_capable = state->freesync_capable;
-	new_state->abm_level = state->abm_level;
-	new_state->scaling = state->scaling;
-	new_state->underscan_enable = state->underscan_enable;
-	new_state->underscan_hborder = state->underscan_hborder;
-	new_state->underscan_vborder = state->underscan_vborder;
-	new_state->vcpi_slots = state->vcpi_slots;
-	new_state->pbn = state->pbn;
-	return &new_state->base;
-}
 
-static int
-amdgpu_dm_connector_late_register(struct drm_connector *connector)
-{
-	struct amdgpu_dm_connector *amdgpu_dm_connector =
-		to_amdgpu_dm_connector(connector);
-	int r;
+	switch (adev->asic_type) {
+#if defined(CONFIG_DRM_AMD_DC_SI)
+	case CHIP_TAHITI:
+	case CHIP_PITCAIRN:
+	case CHIP_VERDE:
+		adev->mode_info.num_crtc = 6;
+		adev->mode_info.num_hpd = 6;
+		adev->mode_info.num_dig = 6;
+		break;
+	case CHIP_OLAND:
+		adev->mode_info.num_crtc = 2;
+		adev->mode_info.num_hpd = 2;
+		adev->mode_info.num_dig = 2;
+		break;
+#endif
+	case CHIP_BONAIRE:
+	case CHIP_HAWAII:
+		adev->mode_info.num_crtc = 6;
+		adev->mode_info.num_hpd = 6;
+		adev->mode_info.num_dig = 6;
+		break;
+	case CHIP_KAVERI:
+		adev->mode_info.num_crtc = 4;
+		adev->mode_info.num_hpd = 6;
+		adev->mode_info.num_dig = 7;
+		break;
+	case CHIP_KABINI:
+	case CHIP_MULLINS:
+		adev->mode_info.num_crtc = 2;
+		adev->mode_info.num_hpd = 6;
+		adev->mode_info.num_dig = 6;
+		break;
+	case CHIP_FIJI:
+	case CHIP_TONGA:
+		adev->mode_info.num_crtc = 6;
+		adev->mode_info.num_hpd = 6;
+		adev->mode_info.num_dig = 7;
+		break;
+	case CHIP_CARRIZO:
+		adev->mode_info.num_crtc = 3;
+		adev->mode_info.num_hpd = 6;
+		adev->mode_info.num_dig = 9;
+		break;
+	case CHIP_STONEY:
+		adev->mode_info.num_crtc = 2;
+		adev->mode_info.num_hpd = 6;
+		adev->mode_info.num_dig = 9;
+		break;
+	case CHIP_POLARIS11:
+	case CHIP_POLARIS12:
+		adev->mode_info.num_crtc = 5;
+		adev->mode_info.num_hpd = 5;
+		adev->mode_info.num_dig = 5;
+		break;
+	case CHIP_POLARIS10:
+	case CHIP_VEGAM:
+		adev->mode_info.num_crtc = 6;
+		adev->mode_info.num_hpd = 6;
+		adev->mode_info.num_dig = 6;
+		break;
+	case CHIP_VEGA10:
+	case CHIP_VEGA12:
+	case CHIP_VEGA20:
+		adev->mode_info.num_crtc = 6;
+		adev->mode_info.num_hpd = 6;
+		adev->mode_info.num_dig = 6;
+		break;
+	default:
 
-	if (amdgpu_dm_should_create_sysfs(amdgpu_dm_connector)) {
-		r = sysfs_create_group(&connector->kdev->kobj,
-				       &amdgpu_group);
-		if (r)
-			return r;
+		switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
+		case IP_VERSION(2, 0, 2):
+		case IP_VERSION(3, 0, 0):
+			adev->mode_info.num_crtc = 6;
+			adev->mode_info.num_hpd = 6;
+			adev->mode_info.num_dig = 6;
+			break;
+		case IP_VERSION(2, 0, 0):
+		case IP_VERSION(3, 0, 2):
+			adev->mode_info.num_crtc = 5;
+			adev->mode_info.num_hpd = 5;
+			adev->mode_info.num_dig = 5;
+			break;
+		case IP_VERSION(2, 0, 3):
+		case IP_VERSION(3, 0, 3):
+			adev->mode_info.num_crtc = 2;
+			adev->mode_info.num_hpd = 2;
+			adev->mode_info.num_dig = 2;
+			break;
+		case IP_VERSION(1, 0, 0):
+		case IP_VERSION(1, 0, 1):
+		case IP_VERSION(3, 0, 1):
+		case IP_VERSION(2, 1, 0):
+		case IP_VERSION(3, 1, 2):
+		case IP_VERSION(3, 1, 3):
+		case IP_VERSION(3, 1, 4):
+		case IP_VERSION(3, 1, 5):
+		case IP_VERSION(3, 1, 6):
+		case IP_VERSION(3, 2, 0):
+		case IP_VERSION(3, 2, 1):
+		case IP_VERSION(3, 5, 0):
+		case IP_VERSION(3, 5, 1):
+		case IP_VERSION(3, 6, 0):
+		case IP_VERSION(4, 0, 1):
+		case IP_VERSION(4, 2, 0):
+		case IP_VERSION(4, 2, 1):
+			adev->mode_info.num_crtc = 4;
+			adev->mode_info.num_hpd = 4;
+			adev->mode_info.num_dig = 4;
+			break;
+		default:
+			drm_err(adev_to_drm(adev), "Unsupported DCE IP versions: 0x%x\n",
+					amdgpu_ip_version(adev, DCE_HWIP, 0));
+			return -EINVAL;
+		}
+		break;
 	}
 
-	amdgpu_dm_register_backlight_device(amdgpu_dm_connector);
-
-	if ((connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort) ||
-	    (connector->connector_type == DRM_MODE_CONNECTOR_eDP)) {
-		amdgpu_dm_connector->dm_dp_aux.aux.dev = connector->kdev;
-		r = drm_dp_aux_register(&amdgpu_dm_connector->dm_dp_aux.aux);
-		if (r)
-			return r;
-	}
+	if (adev->mode_info.funcs == NULL)
+		adev->mode_info.funcs = &dm_display_funcs;
 
-#if defined(CONFIG_DEBUG_FS)
-	connector_debugfs_init(amdgpu_dm_connector);
+	/*
+	 * Note: Do NOT change adev->reg.audio_endpt.rreg and
+	 * adev->reg.audio_endpt.wreg because they are initialised in
+	 * amdgpu_device_init()
+	 */
+#if defined(CONFIG_DEBUG_KERNEL_DC)
+	device_create_file(
+		adev_to_drm(adev)->dev,
+		&dev_attr_s3_debug);
 #endif
+	adev->dc_enabled = true;
 
-	return 0;
+	return dm_init_microcode(adev);
 }
 
-static void amdgpu_dm_connector_funcs_force(struct drm_connector *connector)
+STATIC_IFN_KUNIT bool modereset_required(struct drm_crtc_state *crtc_state)
 {
-	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
-	struct dc_link *dc_link = aconnector->dc_link;
-	struct dc_sink *dc_em_sink = aconnector->dc_em_sink;
-	const struct drm_edid *drm_edid;
-	struct i2c_adapter *ddc;
-	struct drm_device *dev = connector->dev;
-
-	if (dc_link && dc_link->aux_mode)
-		ddc = &aconnector->dm_dp_aux.aux.ddc;
-	else
-		ddc = &aconnector->i2c->base;
-
-	drm_edid = drm_edid_read_ddc(connector, ddc);
-	drm_edid_connector_update(connector, drm_edid);
-	if (!drm_edid) {
-		drm_err(dev, "No EDID found on connector: %s.\n", connector->name);
-		return;
-	}
-
-	aconnector->drm_edid = drm_edid;
-	/* Update emulated (virtual) sink's EDID */
-	if (dc_em_sink && dc_link) {
-		// FIXME: Get rid of drm_edid_raw()
-		const struct edid *edid = drm_edid_raw(drm_edid);
-
-		memset(&dc_em_sink->edid_caps, 0, sizeof(struct dc_edid_caps));
-		memmove(dc_em_sink->dc_edid.raw_edid, edid,
-			(edid->extensions + 1) * EDID_LENGTH);
-		dm_helpers_parse_edid_caps(
-			dc_link,
-			&dc_em_sink->dc_edid,
-			&dc_em_sink->edid_caps);
-	}
+	return !crtc_state->active && drm_atomic_crtc_needs_modeset(crtc_state);
 }
+EXPORT_IF_KUNIT(modereset_required);
 
-static const struct drm_connector_funcs amdgpu_dm_connector_funcs = {
-	.reset = amdgpu_dm_connector_funcs_reset,
-	.detect = amdgpu_dm_connector_detect,
-	.fill_modes = drm_helper_probe_single_connector_modes,
-	.destroy = amdgpu_dm_connector_destroy,
-	.atomic_duplicate_state = amdgpu_dm_connector_atomic_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
-	.atomic_set_property = amdgpu_dm_connector_atomic_set_property,
-	.atomic_get_property = amdgpu_dm_connector_atomic_get_property,
-	.late_register = amdgpu_dm_connector_late_register,
-	.early_unregister = amdgpu_dm_connector_unregister,
-	.force = amdgpu_dm_connector_funcs_force
-};
-
-static int get_modes(struct drm_connector *connector)
+STATIC_IFN_KUNIT int
+fill_plane_color_attributes(const struct drm_plane_state *plane_state,
+			    const enum surface_pixel_format format,
+			    enum dc_color_space *color_space)
 {
-	return amdgpu_dm_connector_get_modes(connector);
-}
+	bool full_range;
 
-static void create_eml_sink(struct amdgpu_dm_connector *aconnector)
-{
-	struct drm_connector *connector = &aconnector->base;
-	struct dc_link *dc_link = aconnector->dc_link;
-	struct dc_sink_init_data init_params = {
-			.link = aconnector->dc_link,
-			.sink_signal = SIGNAL_TYPE_VIRTUAL
-	};
-	const struct drm_edid *drm_edid;
-	const struct edid *edid;
-	struct i2c_adapter *ddc;
-
-	if (dc_link && dc_link->aux_mode)
-		ddc = &aconnector->dm_dp_aux.aux.ddc;
-	else
-		ddc = &aconnector->i2c->base;
+	*color_space = COLOR_SPACE_SRGB;
 
-	drm_edid = drm_edid_read_ddc(connector, ddc);
-	drm_edid_connector_update(connector, drm_edid);
-	if (!drm_edid) {
-		drm_err(connector->dev, "No EDID found on connector: %s.\n", connector->name);
-		return;
-	}
+	/* Ignore properties when DRM_CLIENT_CAP_PLANE_COLOR_PIPELINE is set */
+	if (plane_state->state && plane_state->state->plane_color_pipeline)
+		return 0;
 
-	if (connector->display_info.is_hdmi)
-		init_params.sink_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	/* DRM color properties only affect non-RGB formats. */
+	if (format < SURFACE_PIXEL_FORMAT_VIDEO_BEGIN)
+		return 0;
 
-	aconnector->drm_edid = drm_edid;
+	full_range = (plane_state->color_range == DRM_COLOR_YCBCR_FULL_RANGE);
 
-	edid = drm_edid_raw(drm_edid); // FIXME: Get rid of drm_edid_raw()
-	aconnector->dc_em_sink = dc_link_add_remote_sink(
-		aconnector->dc_link,
-		(uint8_t *)edid,
-		(edid->extensions + 1) * EDID_LENGTH,
-		&init_params);
+	switch (plane_state->color_encoding) {
+	case DRM_COLOR_YCBCR_BT601:
+		if (full_range)
+			*color_space = COLOR_SPACE_YCBCR601;
+		else
+			*color_space = COLOR_SPACE_YCBCR601_LIMITED;
+		break;
 
-	if (aconnector->base.force == DRM_FORCE_ON) {
-		aconnector->dc_sink = aconnector->dc_link->local_sink ?
-		aconnector->dc_link->local_sink :
-		aconnector->dc_em_sink;
-		if (aconnector->dc_sink)
-			dc_sink_retain(aconnector->dc_sink);
-	}
-}
+	case DRM_COLOR_YCBCR_BT709:
+		if (full_range)
+			*color_space = COLOR_SPACE_YCBCR709;
+		else
+			*color_space = COLOR_SPACE_YCBCR709_LIMITED;
+		break;
 
-static void handle_edid_mgmt(struct amdgpu_dm_connector *aconnector)
-{
-	struct dc_link *link = (struct dc_link *)aconnector->dc_link;
+	case DRM_COLOR_YCBCR_BT2020:
+		if (full_range)
+			*color_space = COLOR_SPACE_2020_YCBCR_FULL;
+		else
+			*color_space = COLOR_SPACE_2020_YCBCR_LIMITED;
+		break;
 
-	/*
-	 * In case of headless boot with force on for DP managed connector
-	 * Those settings have to be != 0 to get initial modeset
-	 */
-	if (link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT) {
-		link->verified_link_cap.lane_count = LANE_COUNT_FOUR;
-		link->verified_link_cap.link_rate = LINK_RATE_HIGH2;
+	default:
+		return -EINVAL;
 	}
 
-	create_eml_sink(aconnector);
-}
-
-static enum dc_status dm_validate_stream_and_context(struct dc *dc,
-						struct dc_stream_state *stream)
-{
-	enum dc_status dc_result = DC_ERROR_UNEXPECTED;
-	struct dc_plane_state *dc_plane_state = NULL;
-	struct dc_state *dc_state = NULL;
-
-	if (!stream)
-		goto cleanup;
-
-	dc_plane_state = dc_create_plane_state(dc);
-	if (!dc_plane_state)
-		goto cleanup;
-
-	dc_state = dc_state_create(dc, NULL);
-	if (!dc_state)
-		goto cleanup;
-
-	/* populate stream to plane */
-	dc_plane_state->src_rect.height  = stream->src.height;
-	dc_plane_state->src_rect.width   = stream->src.width;
-	dc_plane_state->dst_rect.height  = stream->src.height;
-	dc_plane_state->dst_rect.width   = stream->src.width;
-	dc_plane_state->clip_rect.height = stream->src.height;
-	dc_plane_state->clip_rect.width  = stream->src.width;
-	dc_plane_state->plane_size.surface_pitch = ((stream->src.width + 255) / 256) * 256;
-	dc_plane_state->plane_size.surface_size.height = stream->src.height;
-	dc_plane_state->plane_size.surface_size.width  = stream->src.width;
-	dc_plane_state->plane_size.chroma_size.height  = stream->src.height;
-	dc_plane_state->plane_size.chroma_size.width   = stream->src.width;
-	dc_plane_state->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB8888;
-	dc_plane_state->tiling_info.gfx9.swizzle = DC_SW_UNKNOWN;
-	dc_plane_state->rotation = ROTATION_ANGLE_0;
-	dc_plane_state->is_tiling_rotated = false;
-	dc_plane_state->tiling_info.gfx8.array_mode = DC_ARRAY_LINEAR_GENERAL;
-
-	dc_result = dc_validate_stream(dc, stream);
-	if (dc_result == DC_OK)
-		dc_result = dc_validate_plane(dc, dc_plane_state);
-
-	if (dc_result == DC_OK)
-		dc_result = dc_state_add_stream(dc, dc_state, stream);
-
-	if (dc_result == DC_OK && !dc_state_add_plane(
-						dc,
-						stream,
-						dc_plane_state,
-						dc_state))
-		dc_result = DC_FAIL_ATTACH_SURFACES;
-
-	if (dc_result == DC_OK)
-		dc_result = dc_validate_global_state(dc, dc_state, DC_VALIDATE_MODE_ONLY);
-
-cleanup:
-	if (dc_state)
-		dc_state_release(dc_state);
-
-	if (dc_plane_state)
-		dc_plane_state_release(dc_plane_state);
-
-	return dc_result;
+	return 0;
 }
+EXPORT_IF_KUNIT(fill_plane_color_attributes);
 
-struct dc_stream_state *
-create_validate_stream_for_sink(struct drm_connector *connector,
-				const struct drm_display_mode *drm_mode,
-				const struct dm_connector_state *dm_state,
-				const struct dc_stream_state *old_stream)
+static int
+fill_dc_plane_info_and_addr(struct amdgpu_device *adev,
+			    const struct drm_plane_state *plane_state,
+			    const u64 tiling_flags,
+			    struct dc_plane_info *plane_info,
+			    struct dc_plane_address *address,
+			    bool tmz_surface)
 {
-	struct amdgpu_dm_connector *aconnector = NULL;
-	struct amdgpu_device *adev = drm_to_adev(connector->dev);
-	struct dc_stream_state *stream;
-	const struct drm_connector_state *drm_state = dm_state ? &dm_state->base : NULL;
-	int requested_bpc = drm_state ? drm_state->max_requested_bpc : 8;
-	enum dc_status dc_result = DC_OK;
-	uint8_t bpc_limit = 6;
-
-	if (!dm_state)
-		return NULL;
-
-	if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK)
-		aconnector = to_amdgpu_dm_connector(connector);
-
-	if (aconnector &&
-	    (aconnector->dc_link->connector_signal == SIGNAL_TYPE_HDMI_TYPE_A ||
-	     aconnector->dc_link->connector_signal == SIGNAL_TYPE_HDMI_FRL ||
-	     aconnector->dc_link->dpcd_caps.dongle_type == DISPLAY_DONGLE_DP_HDMI_CONVERTER))
-		bpc_limit = 8;
-
-	do {
-		drm_dbg_kms(connector->dev, "Trying with %d bpc\n", requested_bpc);
-		stream = create_stream_for_sink(connector, drm_mode,
-						dm_state, old_stream,
-						requested_bpc);
-		if (stream == NULL) {
-			drm_err(adev_to_drm(adev), "Failed to create stream for sink!\n");
-			break;
-		}
-
-		dc_result = dc_validate_stream(adev->dm.dc, stream);
-
-		if (!aconnector) /* writeback connector */
-			return stream;
-
-		if (dc_result == DC_OK && stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST)
-			dc_result = dm_dp_mst_is_port_support_mode(aconnector, stream);
-
-		if (dc_result == DC_OK)
-			dc_result = dm_validate_stream_and_context(adev->dm.dc, stream);
-
-		if (dc_result != DC_OK) {
-			drm_dbg_kms(connector->dev, "Pruned mode %d x %d (clk %d) %s %s -- %s\n",
-				      drm_mode->hdisplay,
-				      drm_mode->vdisplay,
-				      drm_mode->clock,
-				      dc_pixel_encoding_to_str(stream->timing.pixel_encoding),
-				      dc_color_depth_to_str(stream->timing.display_color_depth),
-				      dc_status_to_str(dc_result));
+	const struct drm_framebuffer *fb = plane_state->fb;
+	const struct amdgpu_framebuffer *afb =
+		to_amdgpu_framebuffer(plane_state->fb);
+	int ret;
 
-			dc_stream_release(stream);
-			stream = NULL;
-			requested_bpc -= 2; /* lower bpc to retry validation */
-		}
+	memset(plane_info, 0, sizeof(*plane_info));
 
-	} while (stream == NULL && requested_bpc >= bpc_limit);
+	switch (fb->format->format) {
+	case DRM_FORMAT_C8:
+		plane_info->format =
+			SURFACE_PIXEL_FORMAT_GRPH_PALETA_256_COLORS;
+		break;
+	case DRM_FORMAT_RGB565:
+		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_RGB565;
+		break;
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_ARGB8888:
+		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB8888;
+		break;
+	case DRM_FORMAT_XRGB2101010:
+	case DRM_FORMAT_ARGB2101010:
+		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB2101010;
+		break;
+	case DRM_FORMAT_XBGR2101010:
+	case DRM_FORMAT_ABGR2101010:
+		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR2101010;
+		break;
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_ABGR8888:
+		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR8888;
+		break;
+	case DRM_FORMAT_NV21:
+		plane_info->format = SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr;
+		break;
+	case DRM_FORMAT_NV12:
+		plane_info->format = SURFACE_PIXEL_FORMAT_VIDEO_420_YCrCb;
+		break;
+	case DRM_FORMAT_P010:
+		plane_info->format = SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb;
+		break;
+	case DRM_FORMAT_XRGB16161616F:
+	case DRM_FORMAT_ARGB16161616F:
+		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616F;
+		break;
+	case DRM_FORMAT_XBGR16161616F:
+	case DRM_FORMAT_ABGR16161616F:
+		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F;
+		break;
+	case DRM_FORMAT_XRGB16161616:
+	case DRM_FORMAT_ARGB16161616:
+		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616;
+		break;
+	case DRM_FORMAT_XBGR16161616:
+	case DRM_FORMAT_ABGR16161616:
+		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616;
+		break;
+	default:
+		drm_err(adev_to_drm(adev),
+			"Unsupported screen format %p4cc\n",
+			&fb->format->format);
+		return -EINVAL;
+	}
 
-	switch (dc_result) {
-	/*
-	 * If we failed to validate DP bandwidth stream with the requested RGB color depth,
-	 * we try to fallback and configure in order:
-	 * YUV422 (8bpc, 6bpc)
-	 * YUV420 (8bpc, 6bpc)
-	 */
-	case DC_FAIL_ENC_VALIDATE:
-	case DC_EXCEED_DONGLE_CAP:
-	case DC_NO_DP_LINK_BANDWIDTH:
-		/* recursively entered twice and already tried both YUV422 and YUV420 */
-		if (aconnector->force_yuv422_output && aconnector->force_yuv420_output)
-			break;
-		/* first failure; try YUV422 */
-		if (!aconnector->force_yuv422_output) {
-			drm_dbg_kms(connector->dev, "%s:%d Validation failed with %d, retrying w/ YUV422\n",
-				    __func__, __LINE__, dc_result);
-			aconnector->force_yuv422_output = true;
-		/* recursively entered and YUV422 failed, try YUV420 */
-		} else if (!aconnector->force_yuv420_output) {
-			drm_dbg_kms(connector->dev, "%s:%d Validation failed with %d, retrying w/ YUV420\n",
-				    __func__, __LINE__, dc_result);
-			aconnector->force_yuv420_output = true;
-		}
-		stream = create_validate_stream_for_sink(connector, drm_mode,
-							 dm_state, old_stream);
-		aconnector->force_yuv422_output = false;
-		aconnector->force_yuv420_output = false;
+	switch (plane_state->rotation & DRM_MODE_ROTATE_MASK) {
+	case DRM_MODE_ROTATE_0:
+		plane_info->rotation = ROTATION_ANGLE_0;
+		break;
+	case DRM_MODE_ROTATE_90:
+		plane_info->rotation = ROTATION_ANGLE_90;
+		break;
+	case DRM_MODE_ROTATE_180:
+		plane_info->rotation = ROTATION_ANGLE_180;
 		break;
-	case DC_OK:
+	case DRM_MODE_ROTATE_270:
+		plane_info->rotation = ROTATION_ANGLE_270;
 		break;
 	default:
-		drm_dbg_kms(connector->dev, "%s:%d Unhandled validation failure %d\n",
-			    __func__, __LINE__, dc_result);
+		plane_info->rotation = ROTATION_ANGLE_0;
 		break;
 	}
 
-	return stream;
-}
-
-enum drm_mode_status amdgpu_dm_connector_mode_valid(struct drm_connector *connector,
-				   const struct drm_display_mode *mode)
-{
-	int result = MODE_ERROR;
-	struct dc_sink *dc_sink;
-	struct drm_display_mode *test_mode;
-	/* TODO: Unhardcode stream count */
-	struct dc_stream_state *stream;
-	/* we always have an amdgpu_dm_connector here since we got
-	 * here via the amdgpu_dm_connector_helper_funcs
-	 */
-	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
-
-	if ((mode->flags & DRM_MODE_FLAG_INTERLACE) ||
-			(mode->flags & DRM_MODE_FLAG_DBLSCAN))
-		return result;
 
-	/*
-	 * Only run this the first time mode_valid is called to initilialize
-	 * EDID mgmt
-	 */
-	if (aconnector->base.force != DRM_FORCE_UNSPECIFIED &&
-		!aconnector->dc_em_sink)
-		handle_edid_mgmt(aconnector);
-
-	dc_sink = to_amdgpu_dm_connector(connector)->dc_sink;
+	plane_info->visible = true;
+	plane_info->stereo_format = PLANE_STEREO_FORMAT_NONE;
 
-	if (dc_sink == NULL && aconnector->base.force != DRM_FORCE_ON_DIGITAL &&
-				aconnector->base.force != DRM_FORCE_ON) {
-		drm_err(connector->dev, "dc_sink is NULL!\n");
-		goto fail;
-	}
+	plane_info->layer_index = plane_state->normalized_zpos;
 
-	test_mode = drm_mode_duplicate(connector->dev, mode);
-	if (!test_mode)
-		goto fail;
+	ret = fill_plane_color_attributes(plane_state, plane_info->format,
+					  &plane_info->color_space);
+	if (ret)
+		return ret;
 
-	drm_mode_set_crtcinfo(test_mode, 0);
+	ret = amdgpu_dm_plane_fill_plane_buffer_attributes(adev, afb, plane_info->format,
+					   plane_info->rotation, tiling_flags,
+					   &plane_info->tiling_info,
+					   &plane_info->plane_size,
+					   &plane_info->dcc, address,
+					   tmz_surface);
+	if (ret)
+		return ret;
 
-	stream = create_validate_stream_for_sink(connector, test_mode,
-						 to_dm_connector_state(connector->state),
-						 NULL);
-	drm_mode_destroy(connector->dev, test_mode);
-	if (stream) {
-		dc_stream_release(stream);
-		result = MODE_OK;
-	}
+	amdgpu_dm_plane_fill_blending_from_plane_state(
+		plane_state, &plane_info->per_pixel_alpha, &plane_info->pre_multiplied_alpha,
+		&plane_info->global_alpha, &plane_info->global_alpha_value);
 
-fail:
-	/* TODO: error handling*/
-	return result;
+	return 0;
 }
 
-static int fill_hdr_info_packet(const struct drm_connector_state *state,
-				struct dc_info_packet *out)
+static int fill_dc_plane_attributes(struct amdgpu_device *adev,
+				    struct dc_plane_state *dc_plane_state,
+				    struct drm_plane_state *plane_state,
+				    struct drm_crtc_state *crtc_state)
 {
-	struct hdmi_drm_infoframe frame;
-	unsigned char buf[30]; /* 26 + 4 */
-	ssize_t len;
-	int ret, i;
+	struct dm_crtc_state *dm_crtc_state = to_dm_crtc_state(crtc_state);
+	struct amdgpu_framebuffer *afb = (struct amdgpu_framebuffer *)plane_state->fb;
+	struct dc_scaling_info scaling_info;
+	struct dc_plane_info plane_info;
+	int ret;
 
-	memset(out, 0, sizeof(*out));
+	ret = amdgpu_dm_plane_fill_dc_scaling_info(adev, plane_state, &scaling_info);
+	if (ret)
+		return ret;
 
-	if (!state->hdr_output_metadata)
-		return 0;
+	dc_plane_state->src_rect = scaling_info.src_rect;
+	dc_plane_state->dst_rect = scaling_info.dst_rect;
+	dc_plane_state->clip_rect = scaling_info.clip_rect;
+	dc_plane_state->scaling_quality = scaling_info.scaling_quality;
 
-	ret = drm_hdmi_infoframe_set_hdr_metadata(&frame, state);
+	ret = fill_dc_plane_info_and_addr(adev, plane_state,
+					  afb->tiling_flags,
+					  &plane_info,
+					  &dc_plane_state->address,
+					  afb->tmz_surface);
 	if (ret)
 		return ret;
 
-	len = hdmi_drm_infoframe_pack_only(&frame, buf, sizeof(buf));
-	if (len < 0)
-		return (int)len;
-
-	/* Static metadata is a fixed 26 bytes + 4 byte header. */
-	if (len != 30)
-		return -EINVAL;
+	dc_plane_state->format = plane_info.format;
+	dc_plane_state->color_space = plane_info.color_space;
+	dc_plane_state->format = plane_info.format;
+	dc_plane_state->plane_size = plane_info.plane_size;
+	dc_plane_state->rotation = plane_info.rotation;
+	dc_plane_state->horizontal_mirror = plane_info.horizontal_mirror;
+	dc_plane_state->stereo_format = plane_info.stereo_format;
+	dc_plane_state->tiling_info = plane_info.tiling_info;
+	dc_plane_state->visible = plane_info.visible;
+	dc_plane_state->per_pixel_alpha = plane_info.per_pixel_alpha;
+	dc_plane_state->pre_multiplied_alpha = plane_info.pre_multiplied_alpha;
+	dc_plane_state->global_alpha = plane_info.global_alpha;
+	dc_plane_state->global_alpha_value = plane_info.global_alpha_value;
+	dc_plane_state->dcc = plane_info.dcc;
+	dc_plane_state->layer_index = plane_info.layer_index;
+	dc_plane_state->flip_int_enabled = true;
 
-	/* Prepare the infopacket for DC. */
-	switch (state->connector->connector_type) {
-	case DRM_MODE_CONNECTOR_HDMIA:
-		out->hb0 = 0x87; /* type */
-		out->hb1 = 0x01; /* version */
-		out->hb2 = 0x1A; /* length */
-		out->sb[0] = buf[3]; /* checksum */
-		i = 1;
-		break;
+	/*
+	 * Always set input transfer function, since plane state is refreshed
+	 * every time.
+	 */
+	ret = amdgpu_dm_update_plane_color_mgmt(dm_crtc_state,
+						plane_state,
+						dc_plane_state);
+	if (ret)
+		return ret;
 
-	case DRM_MODE_CONNECTOR_DisplayPort:
-	case DRM_MODE_CONNECTOR_eDP:
-		out->hb0 = 0x00; /* sdp id, zero */
-		out->hb1 = 0x87; /* type */
-		out->hb2 = 0x1D; /* payload len - 1 */
-		out->hb3 = (0x13 << 2); /* sdp version */
-		out->sb[0] = 0x01; /* version */
-		out->sb[1] = 0x1A; /* length */
-		i = 2;
-		break;
+	return 0;
+}
 
-	default:
-		return -EINVAL;
-	}
+static inline void fill_dc_dirty_rect(struct drm_plane *plane,
+				      struct rect *dirty_rect, int32_t x,
+				      s32 y, s32 width, s32 height,
+				      int *i, bool ffu)
+{
+	WARN_ON(*i >= DC_MAX_DIRTY_RECTS);
 
-	memcpy(&out->sb[i], &buf[4], 26);
-	out->valid = true;
+	dirty_rect->x = x;
+	dirty_rect->y = y;
+	dirty_rect->width = width;
+	dirty_rect->height = height;
 
-	print_hex_dump(KERN_DEBUG, "HDR SB:", DUMP_PREFIX_NONE, 16, 1, out->sb,
-		       sizeof(out->sb), false);
+	if (ffu)
+		drm_dbg(plane->dev,
+			"[PLANE:%d] PSR FFU dirty rect size (%d, %d)\n",
+			plane->base.id, width, height);
+	else
+		drm_dbg(plane->dev,
+			"[PLANE:%d] PSR SU dirty rect at (%d, %d) size (%d, %d)",
+			plane->base.id, x, y, width, height);
 
-	return 0;
+	(*i)++;
 }
 
-static int
-amdgpu_dm_connector_atomic_check(struct drm_connector *conn,
-				 struct drm_atomic_commit *state)
+/**
+ * fill_dc_dirty_rects() - Fill DC dirty regions for PSR selective updates
+ *
+ * @plane: DRM plane containing dirty regions that need to be flushed to the eDP
+ *         remote fb
+ * @old_plane_state: Old state of @plane
+ * @new_plane_state: New state of @plane
+ * @crtc_state: New state of CRTC connected to the @plane
+ * @flip_addrs: DC flip tracking struct, which also tracts dirty rects
+ * @is_psr_su: Flag indicating whether Panel Self Refresh Selective Update (PSR SU) is enabled.
+ *             If PSR SU is enabled and damage clips are available, only the regions of the screen
+ *             that have changed will be updated. If PSR SU is not enabled,
+ *             or if damage clips are not available, the entire screen will be updated.
+ * @dirty_regions_changed: dirty regions changed
+ *
+ * For PSR SU, DC informs the DMUB uController of dirty rectangle regions
+ * (referred to as "damage clips" in DRM nomenclature) that require updating on
+ * the eDP remote buffer. The responsibility of specifying the dirty regions is
+ * amdgpu_dm's.
+ *
+ * A damage-aware DRM client should fill the FB_DAMAGE_CLIPS property on the
+ * plane with regions that require flushing to the eDP remote buffer. In
+ * addition, certain use cases - such as cursor and multi-plane overlay (MPO) -
+ * implicitly provide damage clips without any client support via the plane
+ * bounds.
+ */
+static void fill_dc_dirty_rects(struct drm_plane *plane,
+				struct drm_plane_state *old_plane_state,
+				struct drm_plane_state *new_plane_state,
+				struct drm_crtc_state *crtc_state,
+				struct dc_flip_addrs *flip_addrs,
+				bool is_psr_su,
+				bool *dirty_regions_changed)
 {
-	struct drm_connector_state *new_con_state =
-		drm_atomic_get_new_connector_state(state, conn);
-	struct drm_connector_state *old_con_state =
-		drm_atomic_get_old_connector_state(state, conn);
-	struct drm_crtc *crtc = new_con_state->crtc;
-	struct drm_crtc_state *new_crtc_state;
-	struct amdgpu_dm_connector *aconn = to_amdgpu_dm_connector(conn);
-	int ret;
+	struct dm_crtc_state *dm_crtc_state = to_dm_crtc_state(crtc_state);
+	struct rect *dirty_rects = flip_addrs->dirty_rects;
+	u32 num_clips = 0;
+	struct drm_mode_rect *clips = NULL;
+	bool bb_changed;
+	bool fb_changed;
+	u32 i = 0;
+	*dirty_regions_changed = false;
 
-	if (WARN_ON(unlikely(!old_con_state || !new_con_state)))
-		return -EINVAL;
+	/*
+	 * Cursor plane has it's own dirty rect update interface. See
+	 * dcn10_dmub_update_cursor_data and dmub_cmd_update_cursor_info_data
+	 */
+	if (plane->type == DRM_PLANE_TYPE_CURSOR)
+		return;
 
-	trace_amdgpu_dm_connector_atomic_check(new_con_state);
+	if (new_plane_state->rotation != DRM_MODE_ROTATE_0)
+		goto ffu;
 
-	if (conn->connector_type == DRM_MODE_CONNECTOR_DisplayPort) {
-		ret = drm_dp_mst_root_conn_atomic_check(new_con_state, &aconn->mst_mgr);
-		if (ret < 0)
-			return ret;
+	if (!new_plane_state->ignore_damage_clips) {
+		num_clips = drm_plane_get_damage_clips_count(new_plane_state);
+		clips = drm_plane_get_damage_clips(new_plane_state);
 	}
 
-	if (!crtc)
-		return 0;
-
-	if (new_con_state->privacy_screen_sw_state != old_con_state->privacy_screen_sw_state) {
-		new_crtc_state = drm_atomic_get_crtc_state(state, crtc);
-		if (IS_ERR(new_crtc_state))
-			return PTR_ERR(new_crtc_state);
-
-		new_crtc_state->mode_changed = true;
-	}
+	if (num_clips && (!amdgpu_damage_clips || (amdgpu_damage_clips < 0 &&
+						   is_psr_su)))
+		goto ffu;
 
-	if (new_con_state->colorspace != old_con_state->colorspace) {
-		new_crtc_state = drm_atomic_get_crtc_state(state, crtc);
-		if (IS_ERR(new_crtc_state))
-			return PTR_ERR(new_crtc_state);
+	if (!dm_crtc_state->mpo_requested) {
+		if (!num_clips || num_clips > DC_MAX_DIRTY_RECTS)
+			goto ffu;
 
-		new_crtc_state->mode_changed = true;
+		for (; flip_addrs->dirty_rect_count < num_clips; clips++)
+			fill_dc_dirty_rect(new_plane_state->plane,
+					   &dirty_rects[flip_addrs->dirty_rect_count],
+					   clips->x1, clips->y1,
+					   clips->x2 - clips->x1, clips->y2 - clips->y1,
+					   &flip_addrs->dirty_rect_count,
+					   false);
+		return;
 	}
 
-	if (new_con_state->content_type != old_con_state->content_type) {
-		new_crtc_state = drm_atomic_get_crtc_state(state, crtc);
-		if (IS_ERR(new_crtc_state))
-			return PTR_ERR(new_crtc_state);
+	/*
+	 * MPO is requested. Add entire plane bounding box to dirty rects if
+	 * flipped to or damaged.
+	 *
+	 * If plane is moved or resized, also add old bounding box to dirty
+	 * rects.
+	 */
+	fb_changed = old_plane_state->fb->base.id !=
+		     new_plane_state->fb->base.id;
+	bb_changed = (old_plane_state->crtc_x != new_plane_state->crtc_x ||
+		      old_plane_state->crtc_y != new_plane_state->crtc_y ||
+		      old_plane_state->crtc_w != new_plane_state->crtc_w ||
+		      old_plane_state->crtc_h != new_plane_state->crtc_h);
 
-		new_crtc_state->mode_changed = true;
-	}
+	drm_dbg(plane->dev,
+		"[PLANE:%d] PSR bb_changed:%d fb_changed:%d num_clips:%d\n",
+		new_plane_state->plane->base.id,
+		bb_changed, fb_changed, num_clips);
 
-	if (!drm_connector_atomic_hdr_metadata_equal(old_con_state, new_con_state)) {
-		struct dc_info_packet hdr_infopacket;
+	*dirty_regions_changed = bb_changed;
 
-		ret = fill_hdr_info_packet(new_con_state, &hdr_infopacket);
-		if (ret)
-			return ret;
+	if ((num_clips + (bb_changed ? 2 : 0)) > DC_MAX_DIRTY_RECTS)
+		goto ffu;
 
-		new_crtc_state = drm_atomic_get_crtc_state(state, crtc);
-		if (IS_ERR(new_crtc_state))
-			return PTR_ERR(new_crtc_state);
+	if (bb_changed) {
+		fill_dc_dirty_rect(new_plane_state->plane, &dirty_rects[i],
+				   new_plane_state->crtc_x,
+				   new_plane_state->crtc_y,
+				   new_plane_state->crtc_w,
+				   new_plane_state->crtc_h, &i, false);
 
-		/*
-		 * DC considers the stream backends changed if the
-		 * static metadata changes. Forcing the modeset also
-		 * gives a simple way for userspace to switch from
-		 * 8bpc to 10bpc when setting the metadata to enter
-		 * or exit HDR.
-		 *
-		 * Changing the static metadata after it's been
-		 * set is permissible, however. So only force a
-		 * modeset if we're entering or exiting HDR.
-		 */
-		new_crtc_state->mode_changed = new_crtc_state->mode_changed ||
-			!old_con_state->hdr_output_metadata ||
-			!new_con_state->hdr_output_metadata;
+		/* Add old plane bounding-box if plane is moved or resized */
+		fill_dc_dirty_rect(new_plane_state->plane, &dirty_rects[i],
+				   old_plane_state->crtc_x,
+				   old_plane_state->crtc_y,
+				   old_plane_state->crtc_w,
+				   old_plane_state->crtc_h, &i, false);
 	}
 
-	return 0;
-}
-
-static const struct drm_connector_helper_funcs
-amdgpu_dm_connector_helper_funcs = {
-	/*
-	 * If hotplugging a second bigger display in FB Con mode, bigger resolution
-	 * modes will be filtered by drm_mode_validate_size(), and those modes
-	 * are missing after user start lightdm. So we need to renew modes list.
-	 * in get_modes call back, not just return the modes count
-	 */
-	.get_modes = get_modes,
-	.mode_valid = amdgpu_dm_connector_mode_valid,
-	.atomic_check = amdgpu_dm_connector_atomic_check,
-};
+	if (num_clips) {
+		for (; i < num_clips; clips++)
+			fill_dc_dirty_rect(new_plane_state->plane,
+					   &dirty_rects[i], clips->x1,
+					   clips->y1, clips->x2 - clips->x1,
+					   clips->y2 - clips->y1, &i, false);
+	} else if (fb_changed && !bb_changed) {
+		fill_dc_dirty_rect(new_plane_state->plane, &dirty_rects[i],
+				   new_plane_state->crtc_x,
+				   new_plane_state->crtc_y,
+				   new_plane_state->crtc_w,
+				   new_plane_state->crtc_h, &i, false);
+	}
 
-static void dm_encoder_helper_disable(struct drm_encoder *encoder)
-{
+	flip_addrs->dirty_rect_count = i;
+	return;
 
+ffu:
+	fill_dc_dirty_rect(new_plane_state->plane, &dirty_rects[0], 0, 0,
+			   dm_crtc_state->base.mode.crtc_hdisplay,
+			   dm_crtc_state->base.mode.crtc_vdisplay,
+			   &flip_addrs->dirty_rect_count, true);
 }
 
-int convert_dc_color_depth_into_bpc(enum dc_color_depth display_color_depth)
+void amdgpu_dm_update_stream_scaling_settings(struct drm_device *dev,
+					   const struct drm_display_mode *mode,
+					   const struct dm_connector_state *dm_state,
+					   struct dc_stream_state *stream)
 {
-	switch (display_color_depth) {
-	case COLOR_DEPTH_666:
-		return 6;
-	case COLOR_DEPTH_888:
-		return 8;
-	case COLOR_DEPTH_101010:
-		return 10;
-	case COLOR_DEPTH_121212:
-		return 12;
-	case COLOR_DEPTH_141414:
-		return 14;
-	case COLOR_DEPTH_161616:
-		return 16;
-	default:
-		break;
-	}
-	return 0;
-}
+	enum amdgpu_rmx_type rmx_type;
 
-static int dm_encoder_helper_atomic_check(struct drm_encoder *encoder,
-					  struct drm_crtc_state *crtc_state,
-					  struct drm_connector_state *conn_state)
-{
-	struct drm_atomic_commit *state = crtc_state->state;
-	struct drm_connector *connector = conn_state->connector;
-	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
-	struct dm_connector_state *dm_new_connector_state = to_dm_connector_state(conn_state);
-	const struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
-	struct drm_dp_mst_topology_mgr *mst_mgr;
-	struct drm_dp_mst_port *mst_port;
-	struct drm_dp_mst_topology_state *mst_state;
-	enum dc_color_depth color_depth;
-	int clock, bpp = 0;
-	bool is_y420 = false;
-
-	if ((connector->connector_type == DRM_MODE_CONNECTOR_eDP) ||
-	    (connector->connector_type == DRM_MODE_CONNECTOR_LVDS)) {
-		struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder);
-		struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode;
-		enum drm_mode_status result;
-
-		result = drm_crtc_helper_mode_valid_fixed(encoder->crtc, adjusted_mode, native_mode);
-		if (result != MODE_OK && dm_new_connector_state->scaling == RMX_OFF) {
-			drm_dbg_driver(encoder->dev,
-				       "mode %dx%d@%dHz is not native, enabling scaling\n",
-				       adjusted_mode->hdisplay, adjusted_mode->vdisplay,
-				       drm_mode_vrefresh(adjusted_mode));
-			dm_new_connector_state->scaling = RMX_ASPECT;
-		}
-		return 0;
-	}
+	struct rect src = { 0 }; /* viewport in composition space*/
+	struct rect dst = { 0 }; /* stream addressable area */
 
-	if (!aconnector->mst_output_port)
-		return 0;
+	/* no mode. nothing to be done */
+	if (!mode)
+		return;
 
-	mst_port = aconnector->mst_output_port;
-	mst_mgr = &aconnector->mst_root->mst_mgr;
+	/* Full screen scaling by default */
+	src.width = mode->hdisplay;
+	src.height = mode->vdisplay;
+	dst.width = stream->timing.h_addressable;
+	dst.height = stream->timing.v_addressable;
 
-	if (!crtc_state->connectors_changed && !crtc_state->mode_changed)
-		return 0;
+	if (dm_state) {
+		rmx_type = dm_state->scaling;
+		if (rmx_type == RMX_ASPECT || rmx_type == RMX_OFF) {
+			if (src.width * dst.height <
+					src.height * dst.width) {
+				/* height needs less upscaling/more downscaling */
+				dst.width = src.width *
+						dst.height / src.height;
+			} else {
+				/* width needs less upscaling/more downscaling */
+				dst.height = src.height *
+						dst.width / src.width;
+			}
+		} else if (rmx_type == RMX_CENTER) {
+			dst = src;
+		}
 
-	mst_state = drm_atomic_get_mst_topology_state(state, mst_mgr);
-	if (IS_ERR(mst_state))
-		return PTR_ERR(mst_state);
+		dst.x = (stream->timing.h_addressable - dst.width) / 2;
+		dst.y = (stream->timing.v_addressable - dst.height) / 2;
 
-	mst_state->pbn_div.full = dm_mst_get_pbn_divider(aconnector->mst_root->dc_link);
+		if (dm_state->underscan_enable) {
+			dst.x += dm_state->underscan_hborder / 2;
+			dst.y += dm_state->underscan_vborder / 2;
+			dst.width -= dm_state->underscan_hborder;
+			dst.height -= dm_state->underscan_vborder;
+		}
+	}
 
-	if (!state->duplicated) {
-		int max_bpc = conn_state->max_requested_bpc;
+	stream->src = src;
+	stream->dst = dst;
 
-		is_y420 = drm_mode_is_420_also(&connector->display_info, adjusted_mode) &&
-			  aconnector->force_yuv420_output;
-		color_depth = convert_color_depth_from_display_info(connector,
-								    is_y420,
-								    max_bpc);
-		bpp = convert_dc_color_depth_into_bpc(color_depth) * 3;
-		clock = adjusted_mode->clock;
-		dm_new_connector_state->pbn = drm_dp_calc_pbn_mode(clock, bpp << 4);
-	}
+	drm_dbg_kms(dev, "Destination Rectangle x:%d  y:%d  width:%d  height:%d\n",
+		    dst.x, dst.y, dst.width, dst.height);
 
-	dm_new_connector_state->vcpi_slots =
-		drm_dp_atomic_find_time_slots(state, mst_mgr, mst_port,
-					      dm_new_connector_state->pbn);
-	if (dm_new_connector_state->vcpi_slots < 0) {
-		drm_dbg_atomic(connector->dev, "failed finding vcpi slots: %d\n", (int)dm_new_connector_state->vcpi_slots);
-		return dm_new_connector_state->vcpi_slots;
-	}
-	return 0;
 }
 
-const struct drm_encoder_helper_funcs amdgpu_dm_encoder_helper_funcs = {
-	.disable = dm_encoder_helper_disable,
-	.atomic_check = dm_encoder_helper_atomic_check
-};
-
 static int dm_update_mst_vcpi_slots_for_dsc(struct drm_atomic_commit *state,
 					    struct dc_state *dc_state,
 					    struct dsc_mst_fairness_vars *vars)
@@ -8907,753 +3526,6 @@ static int dm_update_mst_vcpi_slots_for_dsc(struct drm_atomic_commit *state,
 	return 0;
 }
 
-static int to_drm_connector_type(enum signal_type st, uint32_t connector_id)
-{
-	switch (st) {
-	case SIGNAL_TYPE_HDMI_TYPE_A:
-		return DRM_MODE_CONNECTOR_HDMIA;
-	case SIGNAL_TYPE_EDP:
-		return DRM_MODE_CONNECTOR_eDP;
-	case SIGNAL_TYPE_LVDS:
-		return DRM_MODE_CONNECTOR_LVDS;
-	case SIGNAL_TYPE_RGB:
-		return DRM_MODE_CONNECTOR_VGA;
-	case SIGNAL_TYPE_DISPLAY_PORT:
-	case SIGNAL_TYPE_DISPLAY_PORT_MST:
-		/* External DP bridges have a different connector type. */
-		if (connector_id == CONNECTOR_ID_VGA)
-			return DRM_MODE_CONNECTOR_VGA;
-		else if (connector_id == CONNECTOR_ID_LVDS)
-			return DRM_MODE_CONNECTOR_LVDS;
-
-		return DRM_MODE_CONNECTOR_DisplayPort;
-	case SIGNAL_TYPE_DVI_DUAL_LINK:
-	case SIGNAL_TYPE_DVI_SINGLE_LINK:
-		if (connector_id == CONNECTOR_ID_SINGLE_LINK_DVII ||
-			connector_id == CONNECTOR_ID_DUAL_LINK_DVII)
-			return DRM_MODE_CONNECTOR_DVII;
-
-		return DRM_MODE_CONNECTOR_DVID;
-	case SIGNAL_TYPE_VIRTUAL:
-		return DRM_MODE_CONNECTOR_VIRTUAL;
-
-	default:
-		return DRM_MODE_CONNECTOR_Unknown;
-	}
-}
-
-static struct drm_encoder *amdgpu_dm_connector_to_encoder(struct drm_connector *connector)
-{
-	struct drm_encoder *encoder;
-
-	/* There is only one encoder per connector */
-	drm_connector_for_each_possible_encoder(connector, encoder)
-		return encoder;
-
-	return NULL;
-}
-
-static void amdgpu_dm_get_native_mode(struct drm_connector *connector)
-{
-	struct drm_encoder *encoder;
-	struct amdgpu_encoder *amdgpu_encoder;
-
-	encoder = amdgpu_dm_connector_to_encoder(connector);
-
-	if (encoder == NULL)
-		return;
-
-	amdgpu_encoder = to_amdgpu_encoder(encoder);
-
-	amdgpu_encoder->native_mode.clock = 0;
-
-	if (!list_empty(&connector->probed_modes)) {
-		struct drm_display_mode *preferred_mode = NULL;
-
-		list_for_each_entry(preferred_mode,
-				    &connector->probed_modes,
-				    head) {
-			if (preferred_mode->type & DRM_MODE_TYPE_PREFERRED)
-				amdgpu_encoder->native_mode = *preferred_mode;
-
-			break;
-		}
-
-	}
-}
-
-static struct drm_display_mode *
-amdgpu_dm_create_common_mode(struct drm_encoder *encoder,
-			     const char *name,
-			     int hdisplay, int vdisplay)
-{
-	struct drm_device *dev = encoder->dev;
-	struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder);
-	struct drm_display_mode *mode = NULL;
-	struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode;
-
-	mode = drm_mode_duplicate(dev, native_mode);
-
-	if (mode == NULL)
-		return NULL;
-
-	mode->hdisplay = hdisplay;
-	mode->vdisplay = vdisplay;
-	mode->type &= ~DRM_MODE_TYPE_PREFERRED;
-	strscpy(mode->name, name, DRM_DISPLAY_MODE_LEN);
-
-	return mode;
-
-}
-
-static const struct amdgpu_dm_mode_size {
-	char name[DRM_DISPLAY_MODE_LEN];
-	int w;
-	int h;
-} common_modes[] = {
-	{  "640x480",  640,  480},
-	{  "800x600",  800,  600},
-	{ "1024x768", 1024,  768},
-	{ "1280x720", 1280,  720},
-	{ "1280x800", 1280,  800},
-	{"1280x1024", 1280, 1024},
-	{ "1440x900", 1440,  900},
-	{"1680x1050", 1680, 1050},
-	{"1600x1200", 1600, 1200},
-	{"1920x1080", 1920, 1080},
-	{"1920x1200", 1920, 1200}
-};
-
-static void amdgpu_dm_connector_add_common_modes(struct drm_encoder *encoder,
-						 struct drm_connector *connector)
-{
-	struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder);
-	struct drm_display_mode *mode = NULL;
-	struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode;
-	struct amdgpu_dm_connector *amdgpu_dm_connector =
-				to_amdgpu_dm_connector(connector);
-	int i;
-	int n;
-
-	if ((connector->connector_type != DRM_MODE_CONNECTOR_eDP) &&
-	    (connector->connector_type != DRM_MODE_CONNECTOR_LVDS))
-		return;
-
-	n = ARRAY_SIZE(common_modes);
-
-	for (i = 0; i < n; i++) {
-		struct drm_display_mode *curmode = NULL;
-		bool mode_existed = false;
-
-		if (common_modes[i].w > native_mode->hdisplay ||
-		    common_modes[i].h > native_mode->vdisplay ||
-		   (common_modes[i].w == native_mode->hdisplay &&
-		    common_modes[i].h == native_mode->vdisplay))
-			continue;
-
-		list_for_each_entry(curmode, &connector->probed_modes, head) {
-			if (common_modes[i].w == curmode->hdisplay &&
-			    common_modes[i].h == curmode->vdisplay) {
-				mode_existed = true;
-				break;
-			}
-		}
-
-		if (mode_existed)
-			continue;
-
-		mode = amdgpu_dm_create_common_mode(encoder,
-				common_modes[i].name, common_modes[i].w,
-				common_modes[i].h);
-		if (!mode)
-			continue;
-
-		drm_mode_probed_add(connector, mode);
-		amdgpu_dm_connector->num_modes++;
-	}
-}
-
-static void amdgpu_set_panel_orientation(struct drm_connector *connector)
-{
-	struct drm_encoder *encoder;
-	struct amdgpu_encoder *amdgpu_encoder;
-	const struct drm_display_mode *native_mode;
-
-	if (connector->connector_type != DRM_MODE_CONNECTOR_eDP &&
-	    connector->connector_type != DRM_MODE_CONNECTOR_LVDS)
-		return;
-
-	mutex_lock(&connector->dev->mode_config.mutex);
-	amdgpu_dm_connector_get_modes(connector);
-	mutex_unlock(&connector->dev->mode_config.mutex);
-
-	encoder = amdgpu_dm_connector_to_encoder(connector);
-	if (!encoder)
-		return;
-
-	amdgpu_encoder = to_amdgpu_encoder(encoder);
-
-	native_mode = &amdgpu_encoder->native_mode;
-	if (native_mode->hdisplay == 0 || native_mode->vdisplay == 0)
-		return;
-
-	drm_connector_set_panel_orientation_with_quirk(connector,
-						       DRM_MODE_PANEL_ORIENTATION_UNKNOWN,
-						       native_mode->hdisplay,
-						       native_mode->vdisplay);
-}
-
-static void amdgpu_dm_connector_ddc_get_modes(struct drm_connector *connector,
-					      const struct drm_edid *drm_edid)
-{
-	struct amdgpu_dm_connector *amdgpu_dm_connector =
-			to_amdgpu_dm_connector(connector);
-
-	if (drm_edid) {
-		/* empty probed_modes */
-		INIT_LIST_HEAD(&connector->probed_modes);
-		amdgpu_dm_connector->num_modes =
-				drm_edid_connector_add_modes(connector);
-
-		/* sorting the probed modes before calling function
-		 * amdgpu_dm_get_native_mode() since EDID can have
-		 * more than one preferred mode. The modes that are
-		 * later in the probed mode list could be of higher
-		 * and preferred resolution. For example, 3840x2160
-		 * resolution in base EDID preferred timing and 4096x2160
-		 * preferred resolution in DID extension block later.
-		 */
-		drm_mode_sort(&connector->probed_modes);
-		amdgpu_dm_get_native_mode(connector);
-
-		/* Freesync capabilities are reset by calling
-		 * drm_edid_connector_add_modes() and need to be
-		 * restored here.
-		 */
-		amdgpu_dm_update_freesync_caps(connector, drm_edid, false);
-	} else {
-		amdgpu_dm_connector->num_modes = 0;
-	}
-}
-
-static bool is_duplicate_mode(struct amdgpu_dm_connector *aconnector,
-			      struct drm_display_mode *mode)
-{
-	struct drm_display_mode *m;
-
-	list_for_each_entry(m, &aconnector->base.probed_modes, head) {
-		if (drm_mode_equal(m, mode))
-			return true;
-	}
-
-	return false;
-}
-
-static uint add_fs_modes(struct amdgpu_dm_connector *aconnector)
-{
-	const struct drm_display_mode *m;
-	struct drm_display_mode *new_mode;
-	uint i;
-	u32 new_modes_count = 0;
-
-	/* Standard FPS values
-	 *
-	 * 23.976       - TV/NTSC
-	 * 24           - Cinema
-	 * 25           - TV/PAL
-	 * 29.97        - TV/NTSC
-	 * 30           - TV/NTSC
-	 * 48           - Cinema HFR
-	 * 50           - TV/PAL
-	 * 60           - Commonly used
-	 * 48,72,96,120 - Multiples of 24
-	 */
-	static const u32 common_rates[] = {
-		23976, 24000, 25000, 29970, 30000,
-		48000, 50000, 60000, 72000, 96000, 120000
-	};
-
-	/*
-	 * Find mode with highest refresh rate with the same resolution
-	 * as the preferred mode. Some monitors report a preferred mode
-	 * with lower resolution than the highest refresh rate supported.
-	 */
-
-	m = get_highest_refresh_rate_mode(aconnector, true);
-	if (!m)
-		return 0;
-
-	for (i = 0; i < ARRAY_SIZE(common_rates); i++) {
-		u64 target_vtotal, target_vtotal_diff;
-		u64 num, den;
-
-		if (drm_mode_vrefresh(m) * 1000 < common_rates[i])
-			continue;
-
-		if (common_rates[i] < aconnector->min_vfreq * 1000 ||
-		    common_rates[i] > aconnector->max_vfreq * 1000)
-			continue;
-
-		num = (unsigned long long)m->clock * 1000 * 1000;
-		den = common_rates[i] * (unsigned long long)m->htotal;
-		target_vtotal = div_u64(num, den);
-		target_vtotal_diff = target_vtotal - m->vtotal;
-
-		/* Check for illegal modes */
-		if (m->vsync_start + target_vtotal_diff < m->vdisplay ||
-		    m->vsync_end + target_vtotal_diff < m->vsync_start ||
-		    m->vtotal + target_vtotal_diff < m->vsync_end)
-			continue;
-
-		new_mode = drm_mode_duplicate(aconnector->base.dev, m);
-		if (!new_mode)
-			goto out;
-
-		new_mode->vtotal += (u16)target_vtotal_diff;
-		new_mode->vsync_start += (u16)target_vtotal_diff;
-		new_mode->vsync_end += (u16)target_vtotal_diff;
-		new_mode->type &= ~DRM_MODE_TYPE_PREFERRED;
-		new_mode->type |= DRM_MODE_TYPE_DRIVER;
-
-		if (!is_duplicate_mode(aconnector, new_mode)) {
-			drm_mode_probed_add(&aconnector->base, new_mode);
-			new_modes_count += 1;
-		} else
-			drm_mode_destroy(aconnector->base.dev, new_mode);
-	}
- out:
-	return new_modes_count;
-}
-
-static void amdgpu_dm_connector_add_freesync_modes(struct drm_connector *connector,
-						   const struct drm_edid *drm_edid)
-{
-	struct amdgpu_dm_connector *amdgpu_dm_connector =
-		to_amdgpu_dm_connector(connector);
-
-	if (!(amdgpu_freesync_vid_mode && drm_edid))
-		return;
-
-	if (!amdgpu_dm_connector->dc_sink || !amdgpu_dm_connector->dc_link)
-		return;
-
-	if (!dc_supports_vrr(amdgpu_dm_connector->dc_sink->ctx->dce_version))
-		return;
-
-	if (dc_connector_supports_analog(amdgpu_dm_connector->dc_link->link_id.id) &&
-	    amdgpu_dm_connector->dc_sink->edid_caps.analog)
-		return;
-
-	if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10)
-		amdgpu_dm_connector->num_modes +=
-			add_fs_modes(amdgpu_dm_connector);
-}
-
-static int amdgpu_dm_connector_get_modes(struct drm_connector *connector)
-{
-	struct amdgpu_dm_connector *amdgpu_dm_connector =
-			to_amdgpu_dm_connector(connector);
-	struct dc_link *dc_link = amdgpu_dm_connector->dc_link;
-	struct drm_encoder *encoder;
-	const struct drm_edid *drm_edid = amdgpu_dm_connector->drm_edid;
-	struct dc_link_settings *verified_link_cap = &dc_link->verified_link_cap;
-	const struct dc *dc = dc_link->dc;
-
-	encoder = amdgpu_dm_connector_to_encoder(connector);
-
-	if (!drm_edid) {
-		amdgpu_dm_connector->num_modes =
-				drm_add_modes_noedid(connector, 640, 480);
-		if (dc->link_srv->dp_get_encoding_format(verified_link_cap) == DP_128b_132b_ENCODING)
-			amdgpu_dm_connector->num_modes +=
-				drm_add_modes_noedid(connector, 1920, 1080);
-
-		if (amdgpu_dm_connector->dc_sink &&
-		    amdgpu_dm_connector->dc_sink->edid_caps.analog &&
-		    dc_connector_supports_analog(dc_link->link_id.id)) {
-			/* Analog monitor connected by DAC load detection.
-			 * Add common modes. It will be up to the user to select one that works.
-			 */
-			for (int i = 0; i < ARRAY_SIZE(common_modes); i++)
-				amdgpu_dm_connector->num_modes += drm_add_modes_noedid(
-					connector, common_modes[i].w, common_modes[i].h);
-		}
-	} else {
-		amdgpu_dm_connector_ddc_get_modes(connector, drm_edid);
-		if (encoder)
-			amdgpu_dm_connector_add_common_modes(encoder, connector);
-		amdgpu_dm_connector_add_freesync_modes(connector, drm_edid);
-	}
-	amdgpu_dm_fbc_init(connector);
-
-	return amdgpu_dm_connector->num_modes;
-}
-
-static const u32 supported_colorspaces =
-	BIT(DRM_MODE_COLORIMETRY_BT709_YCC) |
-	BIT(DRM_MODE_COLORIMETRY_OPRGB) |
-	BIT(DRM_MODE_COLORIMETRY_BT2020_RGB) |
-	BIT(DRM_MODE_COLORIMETRY_BT2020_YCC);
-
-void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm,
-				     struct amdgpu_dm_connector *aconnector,
-				     int connector_type,
-				     struct dc_link *link,
-				     int link_index)
-{
-	struct amdgpu_device *adev = drm_to_adev(dm->ddev);
-
-	/*
-	 * Some of the properties below require access to state, like bpc.
-	 * Allocate some default initial connector state with our reset helper.
-	 */
-	if (aconnector->base.funcs->reset)
-		aconnector->base.funcs->reset(&aconnector->base);
-
-	aconnector->connector_id = link_index;
-	aconnector->bl_idx = -1;
-	aconnector->dc_link = link;
-	aconnector->base.interlace_allowed = false;
-	aconnector->base.doublescan_allowed = false;
-	aconnector->base.stereo_allowed = false;
-	aconnector->base.dpms = DRM_MODE_DPMS_OFF;
-	aconnector->hpd.hpd = AMDGPU_HPD_NONE; /* not used */
-	aconnector->audio_inst = -1;
-	aconnector->pack_sdp_v1_3 = false;
-	aconnector->as_type = ADAPTIVE_SYNC_TYPE_NONE;
-	memset(&aconnector->vsdb_info, 0, sizeof(aconnector->vsdb_info));
-	mutex_init(&aconnector->hpd_lock);
-	mutex_init(&aconnector->handle_mst_msg_ready);
-
-	/*
-	 * If HDMI HPD debounce delay is set, use the minimum between selected
-	 * value and AMDGPU_DM_MAX_HDMI_HPD_DEBOUNCE_MS
-	 */
-	if (amdgpu_hdmi_hpd_debounce_delay_ms) {
-		aconnector->hdmi_hpd_debounce_delay_ms = min(amdgpu_hdmi_hpd_debounce_delay_ms,
-							     AMDGPU_DM_MAX_HDMI_HPD_DEBOUNCE_MS);
-		INIT_DELAYED_WORK(&aconnector->hdmi_hpd_debounce_work, hdmi_hpd_debounce_work);
-		aconnector->hdmi_prev_sink = NULL;
-	} else {
-		aconnector->hdmi_hpd_debounce_delay_ms = 0;
-	}
-
-	/*
-	 * configure support HPD hot plug connector_>polled default value is 0
-	 * which means HPD hot plug not supported
-	 */
-	switch (connector_type) {
-	case DRM_MODE_CONNECTOR_HDMIA:
-		aconnector->base.polled = DRM_CONNECTOR_POLL_HPD;
-		aconnector->base.ycbcr_420_allowed =
-			link->link_enc->features.hdmi_ycbcr420_supported ? true : false;
-		break;
-	case DRM_MODE_CONNECTOR_DisplayPort:
-		aconnector->base.polled = DRM_CONNECTOR_POLL_HPD;
-		link->link_enc = link_enc_cfg_get_link_enc(link);
-		ASSERT(link->link_enc);
-		if (link->link_enc)
-			aconnector->base.ycbcr_420_allowed =
-			link->link_enc->features.dp_ycbcr420_supported ? true : false;
-		break;
-	case DRM_MODE_CONNECTOR_DVID:
-		aconnector->base.polled = DRM_CONNECTOR_POLL_HPD;
-		break;
-	case DRM_MODE_CONNECTOR_DVII:
-	case DRM_MODE_CONNECTOR_VGA:
-		aconnector->base.polled =
-			DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT;
-		break;
-	default:
-		break;
-	}
-
-	drm_object_attach_property(&aconnector->base.base,
-				dm->ddev->mode_config.scaling_mode_property,
-				DRM_MODE_SCALE_NONE);
-
-	if (connector_type == DRM_MODE_CONNECTOR_HDMIA
-		|| (connector_type == DRM_MODE_CONNECTOR_DisplayPort && !aconnector->mst_root))
-		drm_connector_attach_broadcast_rgb_property(&aconnector->base);
-
-	drm_object_attach_property(&aconnector->base.base,
-				adev->mode_info.underscan_property,
-				UNDERSCAN_OFF);
-	drm_object_attach_property(&aconnector->base.base,
-				adev->mode_info.underscan_hborder_property,
-				0);
-	drm_object_attach_property(&aconnector->base.base,
-				adev->mode_info.underscan_vborder_property,
-				0);
-
-	if (!aconnector->mst_root)
-		drm_connector_attach_max_bpc_property(&aconnector->base, 8, 16);
-
-	aconnector->base.state->max_bpc = 16;
-	aconnector->base.state->max_requested_bpc = aconnector->base.state->max_bpc;
-
-	if (connector_type == DRM_MODE_CONNECTOR_HDMIA) {
-		/* Content Type is currently only implemented for HDMI. */
-		drm_connector_attach_content_type_property(&aconnector->base);
-	}
-
-	if (connector_type == DRM_MODE_CONNECTOR_HDMIA) {
-		if (!drm_mode_create_hdmi_colorspace_property(&aconnector->base, supported_colorspaces))
-			drm_connector_attach_colorspace_property(&aconnector->base);
-	} else if ((connector_type == DRM_MODE_CONNECTOR_DisplayPort && !aconnector->mst_root) ||
-		   connector_type == DRM_MODE_CONNECTOR_eDP) {
-		if (!drm_mode_create_dp_colorspace_property(&aconnector->base, supported_colorspaces))
-			drm_connector_attach_colorspace_property(&aconnector->base);
-	}
-
-	if (connector_type == DRM_MODE_CONNECTOR_HDMIA ||
-	    connector_type == DRM_MODE_CONNECTOR_DisplayPort ||
-	    connector_type == DRM_MODE_CONNECTOR_eDP) {
-		drm_connector_attach_hdr_output_metadata_property(&aconnector->base);
-
-		if (!aconnector->mst_root)
-			drm_connector_attach_vrr_capable_property(&aconnector->base);
-
-		if (adev->dm.hdcp_workqueue)
-			drm_connector_attach_content_protection_property(&aconnector->base, true);
-	}
-
-	if (connector_type == DRM_MODE_CONNECTOR_eDP) {
-		struct drm_privacy_screen *privacy_screen;
-
-		drm_connector_attach_panel_type_property(&aconnector->base);
-
-		privacy_screen = drm_privacy_screen_get(adev_to_drm(adev)->dev, NULL);
-		if (!IS_ERR(privacy_screen)) {
-			drm_connector_attach_privacy_screen_provider(&aconnector->base,
-								     privacy_screen);
-		} else if (PTR_ERR(privacy_screen) != -ENODEV) {
-			drm_warn(adev_to_drm(adev), "Error getting privacy-screen\n");
-		}
-	}
-}
-
-static int amdgpu_dm_i2c_xfer(struct i2c_adapter *i2c_adap,
-			      struct i2c_msg *msgs, int num)
-{
-	struct amdgpu_i2c_adapter *i2c = i2c_get_adapdata(i2c_adap);
-	struct ddc_service *ddc_service = i2c->ddc_service;
-	struct i2c_command cmd;
-	int i;
-	int result = -EIO;
-
-	if (!ddc_service->ddc_pin)
-		return result;
-
-	cmd.payloads = kzalloc_objs(struct i2c_payload, num);
-
-	if (!cmd.payloads)
-		return result;
-
-	cmd.number_of_payloads = num;
-	cmd.engine = I2C_COMMAND_ENGINE_DEFAULT;
-	cmd.speed = 100;
-
-	for (i = 0; i < num; i++) {
-		cmd.payloads[i].write = !(msgs[i].flags & I2C_M_RD);
-		cmd.payloads[i].address = msgs[i].addr;
-		cmd.payloads[i].length = msgs[i].len;
-		cmd.payloads[i].data = msgs[i].buf;
-	}
-
-	if (i2c->oem) {
-		if (dc_submit_i2c_oem(
-			    ddc_service->ctx->dc,
-			    &cmd))
-			result = num;
-	} else {
-		if (dc_submit_i2c(
-			    ddc_service->ctx->dc,
-			    ddc_service->link->link_index,
-			    &cmd))
-			result = num;
-	}
-
-	kfree(cmd.payloads);
-	return result;
-}
-
-static u32 amdgpu_dm_i2c_func(struct i2c_adapter *adap)
-{
-	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
-}
-
-static const struct i2c_algorithm amdgpu_dm_i2c_algo = {
-	.master_xfer = amdgpu_dm_i2c_xfer,
-	.functionality = amdgpu_dm_i2c_func,
-};
-
-static struct amdgpu_i2c_adapter *
-create_i2c(struct ddc_service *ddc_service, bool oem)
-{
-	struct amdgpu_device *adev = ddc_service->ctx->driver_context;
-	struct amdgpu_i2c_adapter *i2c;
-
-	i2c = kzalloc_obj(struct amdgpu_i2c_adapter);
-	if (!i2c)
-		return NULL;
-	i2c->base.owner = THIS_MODULE;
-	i2c->base.dev.parent = &adev->pdev->dev;
-	i2c->base.algo = &amdgpu_dm_i2c_algo;
-	if (oem)
-		snprintf(i2c->base.name, sizeof(i2c->base.name), "AMDGPU DM i2c OEM bus");
-	else
-		snprintf(i2c->base.name, sizeof(i2c->base.name), "AMDGPU DM i2c hw bus %d",
-			 ddc_service->link->link_index);
-	i2c_set_adapdata(&i2c->base, i2c);
-	i2c->ddc_service = ddc_service;
-	i2c->oem = oem;
-
-	return i2c;
-}
-
-int amdgpu_dm_initialize_hdmi_connector(struct amdgpu_dm_connector *aconnector)
-{
-	struct cec_connector_info conn_info;
-	struct drm_device *ddev = aconnector->base.dev;
-	struct device *hdmi_dev = ddev->dev;
-
-	if (amdgpu_dc_debug_mask & DC_DISABLE_HDMI_CEC) {
-		drm_info(ddev, "HDMI-CEC feature masked\n");
-		return -EINVAL;
-	}
-
-	cec_fill_conn_info_from_drm(&conn_info, &aconnector->base);
-	aconnector->notifier =
-		cec_notifier_conn_register(hdmi_dev, NULL, &conn_info);
-	if (!aconnector->notifier) {
-		drm_err(ddev, "Failed to create cec notifier\n");
-		return -ENOMEM;
-	}
-
-	return 0;
-}
-
-/*
- * Note: this function assumes that dc_link_detect() was called for the
- * dc_link which will be represented by this aconnector.
- */
-static int amdgpu_dm_connector_init(struct amdgpu_display_manager *dm,
-				    struct amdgpu_dm_connector *aconnector,
-				    u32 link_index,
-				    struct amdgpu_encoder *aencoder)
-{
-	int res = 0;
-	int connector_type;
-	struct dc *dc = dm->dc;
-	struct dc_link *link = dc_get_link_at_index(dc, link_index);
-	struct amdgpu_i2c_adapter *i2c;
-
-	/* Not needed for writeback connector */
-	link->priv = aconnector;
-
-
-	i2c = create_i2c(link->ddc, false);
-	if (!i2c) {
-		drm_err(adev_to_drm(dm->adev), "Failed to create i2c adapter data\n");
-		return -ENOMEM;
-	}
-
-	aconnector->i2c = i2c;
-	res = devm_i2c_add_adapter(dm->adev->dev, &i2c->base);
-
-	if (res) {
-		drm_err(adev_to_drm(dm->adev), "Failed to register hw i2c %d\n", link->link_index);
-		goto out_free;
-	}
-
-	connector_type = to_drm_connector_type(link->connector_signal, link->link_id.id);
-
-	res = drm_connector_init_with_ddc(
-			dm->ddev,
-			&aconnector->base,
-			&amdgpu_dm_connector_funcs,
-			connector_type,
-			&i2c->base);
-
-	if (res) {
-		drm_err(adev_to_drm(dm->adev), "connector_init failed\n");
-		aconnector->connector_id = -1;
-		goto out_free;
-	}
-
-	drm_connector_helper_add(
-			&aconnector->base,
-			&amdgpu_dm_connector_helper_funcs);
-
-	amdgpu_dm_connector_init_helper(
-		dm,
-		aconnector,
-		connector_type,
-		link,
-		link_index);
-
-	drm_connector_attach_encoder(
-		&aconnector->base, &aencoder->base);
-
-	if (connector_type == DRM_MODE_CONNECTOR_HDMIA ||
-	    connector_type == DRM_MODE_CONNECTOR_HDMIB)
-		amdgpu_dm_initialize_hdmi_connector(aconnector);
-
-	if (dc_is_dp_signal(link->connector_signal))
-		amdgpu_dm_initialize_dp_connector(dm, aconnector, link->link_index);
-
-out_free:
-	if (res) {
-		kfree(i2c);
-		aconnector->i2c = NULL;
-	}
-	return res;
-}
-
-int amdgpu_dm_get_encoder_crtc_mask(struct amdgpu_device *adev)
-{
-	switch (adev->mode_info.num_crtc) {
-	case 1:
-		return 0x1;
-	case 2:
-		return 0x3;
-	case 3:
-		return 0x7;
-	case 4:
-		return 0xf;
-	case 5:
-		return 0x1f;
-	case 6:
-	default:
-		return 0x3f;
-	}
-}
-
-static int amdgpu_dm_encoder_init(struct drm_device *dev,
-				  struct amdgpu_encoder *aencoder,
-				  uint32_t link_index)
-{
-	struct amdgpu_device *adev = drm_to_adev(dev);
-
-	int res = drm_encoder_init(dev,
-				   &aencoder->base,
-				   &amdgpu_dm_encoder_funcs,
-				   DRM_MODE_ENCODER_TMDS,
-				   NULL);
-
-	aencoder->base.possible_crtcs = amdgpu_dm_get_encoder_crtc_mask(adev);
-
-	if (!res)
-		aencoder->encoder_id = link_index;
-	else
-		aencoder->encoder_id = -1;
-
-	drm_encoder_helper_add(&aencoder->base, &amdgpu_dm_encoder_helper_funcs);
-
-	return res;
-}
-
 static void manage_dm_interrupts(struct amdgpu_device *adev,
 				 struct amdgpu_crtc *acrtc,
 				 struct dm_crtc_state *acrtc_state)
@@ -9673,22 +3545,10 @@ static void manage_dm_interrupts(struct amdgpu_device *adev,
 
 	if (acrtc_state) {
 		timing = &acrtc_state->stream->timing;
-
-		if (amdgpu_ip_version(adev, DCE_HWIP, 0) >=
-		      IP_VERSION(3, 2, 0) &&
-		      !(adev->flags & AMD_IS_APU)) {
-			/*
-			 * DGPUs NV3x and newer that support idle optimizations
-			 * experience intermittent flip-done timeouts on cursor
-			 * updates. Restore 5s offdelay behavior for now.
-			 *
-			 * Discussion on the issue:
-			 * https://lore.kernel.org/amd-gfx/20260217191632.1243826-1-sysdadmin@m1k.cloud/
-			 */
-			config.offdelay_ms = 5000;
-			config.disable_immediate = false;
-		} else if (amdgpu_ip_version(adev, DCE_HWIP, 0) <
-			     IP_VERSION(3, 5, 0)) {
+
+		if (amdgpu_ip_version(adev, DCE_HWIP, 0) <
+			   IP_VERSION(3, 5, 0) ||
+			   !(adev->flags & AMD_IS_APU)) {
 			/*
 			 * Older HW and DGPU have issues with instant off;
 			 * use a 2 frame offdelay.
@@ -9707,14 +3567,22 @@ static void manage_dm_interrupts(struct amdgpu_device *adev,
 
 		drm_crtc_vblank_on_config(&acrtc->base,
 					  &config);
-		/* Allow RX6xxx, RX7700, RX7800 GPUs to call amdgpu_irq_get.*/
+		/*
+		 * Since pflip_high_irq is no longer registered for DCN, grab an
+		 * extra reference to vupdate irq instead to workaround this
+		 * issue:
+		 * https://gitlab.freedesktop.org/drm/amd/-/work_items/3936
+		 *
+		 * The callbacks to drm_vblank_on/off should really take care of
+		 * this though.
+		 */
 		switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
 		case IP_VERSION(3, 0, 0):
 		case IP_VERSION(3, 0, 2):
 		case IP_VERSION(3, 0, 3):
 		case IP_VERSION(3, 2, 0):
-			if (amdgpu_irq_get(adev, &adev->pageflip_irq, irq_type))
-				drm_err(dev, "DM_IRQ: Cannot get pageflip irq!\n");
+			if (amdgpu_irq_get(adev, &adev->vupdate_irq, irq_type))
+				drm_err(dev, "DM_IRQ: Cannot get vupdate irq!\n");
 #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
 			if (amdgpu_irq_get(adev, &adev->vline0_irq, irq_type))
 				drm_err(dev, "DM_IRQ: Cannot get vline0 irq!\n");
@@ -9732,8 +3600,8 @@ static void manage_dm_interrupts(struct amdgpu_device *adev,
 			if (amdgpu_irq_put(adev, &adev->vline0_irq, irq_type))
 				drm_err(dev, "DM_IRQ: Cannot put vline0 irq!\n");
 #endif
-			if (amdgpu_irq_put(adev, &adev->pageflip_irq, irq_type))
-				drm_err(dev, "DM_IRQ: Cannot put pageflip irq!\n");
+			if (amdgpu_irq_put(adev, &adev->vupdate_irq, irq_type))
+				drm_err(dev, "DM_IRQ: Cannot put vupdate irq!\n");
 		}
 
 		drm_crtc_vblank_off(&acrtc->base);
@@ -9746,6 +3614,10 @@ static void dm_update_pflip_irq_state(struct amdgpu_device *adev,
 	int irq_type =
 		amdgpu_display_crtc_idx_to_irq_type(adev, acrtc->crtc_id);
 
+	/* GRPH_PFLIP is not used on DCN; nothing to reapply. */
+	if (amdgpu_ip_version(adev, DCE_HWIP, 0) != 0)
+		return;
+
 	/**
 	 * This reads the current state for the IRQ and force reapplies
 	 * the setting to hardware.
@@ -9753,7 +3625,7 @@ static void dm_update_pflip_irq_state(struct amdgpu_device *adev,
 	amdgpu_irq_update(adev, &adev->pageflip_irq, irq_type);
 }
 
-static bool
+STATIC_IFN_KUNIT bool
 is_scaling_state_different(const struct dm_connector_state *dm_state,
 			   const struct dm_connector_state *old_dm_state)
 {
@@ -9770,6 +3642,7 @@ is_scaling_state_different(const struct dm_connector_state *dm_state,
 		return true;
 	return false;
 }
+EXPORT_IF_KUNIT(is_scaling_state_different);
 
 static bool is_content_protection_different(struct drm_crtc_state *new_crtc_state,
 					    struct drm_crtc_state *old_crtc_state,
@@ -10078,9 +3951,13 @@ static void amdgpu_dm_handle_vrr_transition(struct amdgpu_display_manager *dm,
 					    struct dm_crtc_state *old_state,
 					    struct dm_crtc_state *new_state)
 {
+	struct amdgpu_device *adev = dm->adev;
 	bool old_vrr_active = amdgpu_dm_crtc_vrr_active(old_state);
 	bool new_vrr_active = amdgpu_dm_crtc_vrr_active(new_state);
 
+	/* Only DCE gates vupdate on VRR, keep it enabled for DCN */
+	bool vrr_gates_vupdate = amdgpu_ip_version(adev, DCE_HWIP, 0) == 0;
+
 	if (!old_vrr_active && new_vrr_active) {
 		/* Transition VRR inactive -> active:
 		 * While VRR is active, we must not disable vblank irq, as a
@@ -10090,7 +3967,8 @@ static void amdgpu_dm_handle_vrr_transition(struct amdgpu_display_manager *dm,
 		 * We also need vupdate irq for the actual core vblank handling
 		 * at end of vblank.
 		 */
-		WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, true) != 0);
+		if (vrr_gates_vupdate)
+			WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, true) != 0);
 		WARN_ON(drm_crtc_vblank_get(new_state->base.crtc) != 0);
 		drm_dbg_driver(new_state->base.crtc->dev, "%s: crtc=%u VRR off->on: Get vblank ref\n",
 				 __func__, new_state->base.crtc->base.id);
@@ -10106,7 +3984,8 @@ static void amdgpu_dm_handle_vrr_transition(struct amdgpu_display_manager *dm,
 		/* Transition VRR active -> inactive:
 		 * Allow vblank irq disable again for fixed refresh rate.
 		 */
-		WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, false) != 0);
+		if (vrr_gates_vupdate)
+			WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, false) != 0);
 		drm_crtc_vblank_put(new_state->base.crtc);
 		drm_dbg_driver(new_state->base.crtc->dev, "%s: crtc=%u VRR on->off: Drop vblank ref\n",
 				 __func__, new_state->base.crtc->base.id);
@@ -10255,6 +4134,28 @@ static void amdgpu_dm_enable_self_refresh(struct amdgpu_display_manager *dm,
 	}
 }
 
+static void dm_arm_vblank_event(struct amdgpu_crtc *acrtc,
+				struct dm_crtc_state *acrtc_state,
+				bool pflip_update,
+				bool cursor_update)
+{
+	assert_spin_locked(&acrtc->base.dev->event_lock);
+
+	if (!acrtc->base.state->event || acrtc_state->active_planes == 0)
+		return;
+
+	if (pflip_update) {
+		drm_crtc_vblank_get(&acrtc->base);
+		WARN_ON(acrtc->pflip_status != AMDGPU_FLIP_NONE);
+		/* Arm flip completion handling and event delivery after programming. */
+		prepare_flip_isr(acrtc);
+	} else if (cursor_update) {
+		drm_crtc_vblank_get(&acrtc->base);
+		acrtc->event = acrtc->base.state->event;
+		acrtc->base.state->event = NULL;
+	}
+}
+
 static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 				    struct drm_device *dev,
 				    struct amdgpu_display_manager *dm,
@@ -10277,6 +4178,8 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 	bool vrr_active = amdgpu_dm_crtc_vrr_active(acrtc_state);
 	bool cursor_update = false;
 	bool pflip_present = false;
+	bool immediate_flip = false;
+	bool flip_latched_during_prog = false;
 	bool dirty_rects_changed = false;
 	bool updated_planes_and_streams = false;
 	struct {
@@ -10353,14 +4256,12 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 			continue;
 
 		bundle->surface_updates[planes_count].surface = dc_plane;
-		if (new_pcrtc_state->color_mgmt_changed || new_plane_state->color_mgmt_changed) {
+		if (new_pcrtc_state->color_mgmt_changed) {
 			bundle->surface_updates[planes_count].gamma = &dc_plane->gamma_correction;
 			bundle->surface_updates[planes_count].in_transfer_func = &dc_plane->in_transfer_func;
 			bundle->surface_updates[planes_count].gamut_remap_matrix = &dc_plane->gamut_remap_matrix;
 			bundle->surface_updates[planes_count].hdr_mult = dc_plane->hdr_mult;
-			bundle->surface_updates[planes_count].func_shaper = &dc_plane->in_shaper_func;
-			bundle->surface_updates[planes_count].lut3d_func = &dc_plane->lut3d_func;
-			bundle->surface_updates[planes_count].blend_tf = &dc_plane->blend_tf;
+			bundle->surface_updates[planes_count].cm = &dc_plane->cm;
 		}
 
 		amdgpu_dm_plane_fill_dc_scaling_info(dm->adev, new_plane_state,
@@ -10441,6 +4342,8 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 			acrtc_state->update_type == UPDATE_TYPE_FAST &&
 			get_mem_type(old_plane_state->fb) == get_mem_type(fb);
 
+		immediate_flip |= bundle->flip_addrs[planes_count].flip_immediate;
+
 		timestamp_ns = ktime_get_ns();
 		bundle->flip_addrs[planes_count].flip_timestamp_in_us = div_u64(timestamp_ns, 1000);
 		bundle->surface_updates[planes_count].flip_addr = &bundle->flip_addrs[planes_count];
@@ -10509,39 +4412,24 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 			usleep_range(1000, 1100);
 		}
 
-		/**
-		 * Prepare the flip event for the pageflip interrupt to handle.
-		 *
-		 * This only works in the case where we've already turned on the
-		 * appropriate hardware blocks (eg. HUBP) so in the transition case
-		 * from 0 -> n planes we have to skip a hardware generated event
-		 * and rely on sending it from software.
-		 */
-		if (acrtc_attach->base.state->event &&
-		    acrtc_state->active_planes > 0) {
-			drm_crtc_vblank_get(pcrtc);
-
-			spin_lock_irqsave(&pcrtc->dev->event_lock, flags);
-
-			WARN_ON(acrtc_attach->pflip_status != AMDGPU_FLIP_NONE);
-			prepare_flip_isr(acrtc_attach);
-
-			spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags);
-		}
-
 		if (acrtc_state->stream) {
 			if (acrtc_state->freesync_vrr_info_changed)
 				bundle->stream_update.vrr_infopacket =
 					&acrtc_state->stream->vrr_infopacket;
 		}
-	} else if (cursor_update && acrtc_state->active_planes > 0) {
-		spin_lock_irqsave(&pcrtc->dev->event_lock, flags);
-		if (acrtc_attach->base.state->event) {
-			drm_crtc_vblank_get(pcrtc);
-			acrtc_attach->event = acrtc_attach->base.state->event;
-			acrtc_attach->base.state->event = NULL;
+	}
+
+	/*
+	 * DCE depends on a combination of GRPH_FLIP, VLINE0, and VUPDATE for
+	 * event delivery. Only GRPH_FLIP handler can send pflip events, and it
+	 * only fires if HW latched to the flip. Maintain legacy behavior by
+	 * arming event before programming.
+	 */
+	if (amdgpu_ip_version(dm->adev, DCE_HWIP, 0) == 0) {
+		scoped_guard(spinlock_irqsave, &pcrtc->dev->event_lock) {
+			dm_arm_vblank_event(acrtc_attach, acrtc_state,
+					pflip_present, cursor_update);
 		}
-		spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags);
 	}
 
 	/* Update the planes if changed or disable if we don't have any. */
@@ -10633,89 +4521,117 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 	    acrtc_state->cursor_mode == DM_CURSOR_NATIVE_MODE)
 		amdgpu_dm_commit_cursors(state);
 
-cleanup:
-	kfree(bundle);
-}
-
-static void amdgpu_dm_commit_audio(struct drm_device *dev,
-				   struct drm_atomic_commit *state)
-{
-	struct amdgpu_device *adev = drm_to_adev(dev);
-	struct amdgpu_dm_connector *aconnector;
-	struct drm_connector *connector;
-	struct drm_connector_state *old_con_state, *new_con_state;
-	struct drm_crtc_state *new_crtc_state;
-	struct dm_crtc_state *new_dm_crtc_state;
-	const struct dc_stream_status *status;
-	int i, inst;
+	/*
+	 * DCN specific vblank handling
+	 * ============================
+	 *
+	 * With the event_lock held, arm the vblank event, and determine whether
+	 * deliver it immediately, or in VUPDATE_NO_LOCK IRQ (i.e. HW latch
+	 * point) handler. Do this *after* programming so that the IRQ handler
+	 * will not deliver the event before HW laches onto the programmed
+	 * values:
+	 *
+	 *     Commit thread      IRQ handler                       HW
+	 *     -----------------------------------------------------------------
+	 *     arm_vblank_event()
+	 *                                                          vupdate()
+	 *                        vupdate_handler()
+	 *                          cook_timestamp()
+	 *                          # prev flip already latched,
+	 *                          # so flip_latched == true.
+	 *                          if event_armed && flip_latched:
+	 *                            send_vblank_event()
+	 *                            # sent before latch, **BAD!**
+	 *     hw_program()
+	 *                                                          vupdate()
+	 *                                                          **latch**
+	 *
+	 * There's a consequence of arming after: it's possible for HW to latch
+	 * between start of HW programming and acrtc->event/pflip_status arming.
+	 * When this happens, the IRQ handler will send the event on the next
+	 * immediate latch point, even though HW has already latched. This is
+	 * handled by optimistically checking for HW latch after programming,
+	 * and if latched, send the event immediately:
+	 *
+	 *     Commit thread             IRQ handler                HW
+	 *     -----------------------------------------------------------------
+	 *     hw_program()
+	 *                                                          vupdate()
+	 *                                                          **latch**
+	 *                               vupdate_handler()
+	 *                                 cook_timestamp()
+	 *                                 # event_armed == false
+	 *                                 # **no event sent!**
+	 *     arm_vblank_event()
+	 *     if flip_latched:
+	 *       **send_vblank_event()**
+	 *       disarm_vblank_event()
+	 *
+	 * The IRQ handler is expected to cook the timestamp, but we need to
+	 * cook the timestamp before optimistic sending as well. That's because
+	 * the following sequence is possible:
+	 *
+	 *     Commit thread              IRQ handler              HW
+	 *     -----------------------------------------------------------------
+	 *     hw_program()
+	 *     arm_vblank_event()
+	 *                                                         vupdate()
+	 *                                                         **latch**
+	 *     if flip_latched:
+	 *       # Need cook before send!
+	 *       **cook_timestamp()**
+	 *       send_vblank_event()
+	 *       disarm_vblank_event()
+	 *                                vupdate_handler()
+	 *                                  cook_timestamp()
+	 *                                  # event_armed == false
+	 *                                  # no event sent!
+	 *
+	 * Cooking twice is OK, since DRM scanout accurate timestamps report A)
+	 * the previous vactive start if currently in vactive, or B) the next
+	 * vactive start if currently in vblank (see &get_vblank_counter). 'A)'
+	 * is what we want for the optimistic send, and for 'B)', we'll cook a
+	 * timestamp no later than the next IRQ handler run.
+	 *
+	 * The more correct fix is to wrap programming and arming with the
+	 * event_lock and thus serializing it with the IRQ handler. However,
+	 * there are various sleep-waits within
+	 * update_planes_and_stream_adapter() that makes spin locking illegal.
+	 * And on full updates, it can take 1-2 frame-times to return (see
+	 * commit_planes_for_stream).
+	 *
+	 * On DCE, GRPH_PFLIP IRQ is used and takes care of this.
+	 */
+	if (amdgpu_ip_version(dm->adev, DCE_HWIP, 0) != 0) {
+		spin_lock_irqsave(&pcrtc->dev->event_lock, flags);
 
-	/* Notify device removals. */
-	for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) {
-		if (old_con_state->crtc != new_con_state->crtc) {
-			/* CRTC changes require notification. */
-			goto notify;
+		if (updated_planes_and_streams) {
+			flip_latched_during_prog =
+				!dc_get_flip_pending_on_otg(dm->dc, acrtc_attach->otg_inst);
 		}
 
-		if (!new_con_state->crtc)
-			continue;
-
-		new_crtc_state = drm_atomic_get_new_crtc_state(
-			state, new_con_state->crtc);
-
-		if (!new_crtc_state)
-			continue;
-
-		if (!drm_atomic_crtc_needs_modeset(new_crtc_state))
-			continue;
-
-notify:
-		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
-			continue;
-
-		aconnector = to_amdgpu_dm_connector(connector);
-
-		mutex_lock(&adev->dm.audio_lock);
-		inst = aconnector->audio_inst;
-		aconnector->audio_inst = -1;
-		mutex_unlock(&adev->dm.audio_lock);
+		dm_arm_vblank_event(acrtc_attach, acrtc_state,
+				    pflip_present, cursor_update);
 
-		amdgpu_dm_audio_eld_notify(adev, inst);
+		/*
+		 * Deliver the event immediately on immediate flip, or on a
+		 * update that has already latched.
+		 */
+		if ((immediate_flip || flip_latched_during_prog) &&
+		    acrtc_attach->pflip_status == AMDGPU_FLIP_SUBMITTED &&
+		    acrtc_attach->event) {
+			drm_crtc_accurate_vblank_count(&acrtc_attach->base);
+			drm_crtc_send_vblank_event(&acrtc_attach->base,
+						   acrtc_attach->event);
+			acrtc_attach->event = NULL;
+			drm_crtc_vblank_put(&acrtc_attach->base);
+			acrtc_attach->pflip_status = AMDGPU_FLIP_NONE;
+		}
+		spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags);
 	}
 
-	/* Notify audio device additions. */
-	for_each_new_connector_in_state(state, connector, new_con_state, i) {
-		if (!new_con_state->crtc)
-			continue;
-
-		new_crtc_state = drm_atomic_get_new_crtc_state(
-			state, new_con_state->crtc);
-
-		if (!new_crtc_state)
-			continue;
-
-		if (!drm_atomic_crtc_needs_modeset(new_crtc_state))
-			continue;
-
-		new_dm_crtc_state = to_dm_crtc_state(new_crtc_state);
-		if (!new_dm_crtc_state->stream)
-			continue;
-
-		status = dc_stream_get_status(new_dm_crtc_state->stream);
-		if (!status)
-			continue;
-
-		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
-			continue;
-
-		aconnector = to_amdgpu_dm_connector(connector);
-
-		mutex_lock(&adev->dm.audio_lock);
-		inst = status->audio_inst;
-		aconnector->audio_inst = inst;
-		mutex_unlock(&adev->dm.audio_lock);
-
-		amdgpu_dm_audio_eld_notify(adev, inst);
-	}
+cleanup:
+	kfree(bundle);
 }
 
 /*
@@ -10732,10 +4648,55 @@ static void amdgpu_dm_crtc_copy_transient_flags(struct drm_crtc_state *crtc_stat
 	stream_state->mode_changed = drm_atomic_crtc_needs_modeset(crtc_state);
 }
 
+/**
+ * amdgpu_dm_crtc_complete_writeback - finish a pending writeback job
+ * @acrtc: the CRTC whose pending writeback should be completed
+ *
+ * Clears the pending state, signals the writeback out fence and releases the
+ * vblank reference taken in dm_set_writeback() while the writeback was armed.
+ * The pending flag is tested and cleared under the writeback job lock, so this
+ * is safe to call concurrently from the completion vblank IRQ
+ * (dm_crtc_high_irq()) and from the writeback teardown path
+ * (dm_clear_writeback()); only the caller that observes the pending job
+ * performs the completion.
+ *
+ * Return: true if a pending writeback job was completed by this call.
+ */
+bool amdgpu_dm_crtc_complete_writeback(struct amdgpu_crtc *acrtc)
+{
+	unsigned long flags;
+	bool pending;
+
+	if (!acrtc->wb_conn)
+		return false;
+
+	spin_lock_irqsave(&acrtc->wb_conn->job_lock, flags);
+	pending = acrtc->wb_pending;
+	acrtc->wb_pending = false;
+	spin_unlock_irqrestore(&acrtc->wb_conn->job_lock, flags);
+
+	if (!pending)
+		return false;
+
+	drm_writeback_signal_completion(acrtc->wb_conn, 0);
+	drm_crtc_vblank_put(&acrtc->base);
+
+	return true;
+}
+
 static void dm_clear_writeback(struct amdgpu_display_manager *dm,
+			      struct amdgpu_crtc *acrtc,
 			      struct dm_crtc_state *crtc_state)
 {
 	dc_stream_remove_writeback(dm->dc, crtc_state->stream, 0);
+
+	/*
+	 * If the writeback is still pending when it is torn down (its
+	 * completion vblank IRQ never fired), signal the out fence so a
+	 * waiting client does not stall and release the vblank reference
+	 * taken in dm_set_writeback().
+	 */
+	amdgpu_dm_crtc_complete_writeback(acrtc);
 }
 
 /**
@@ -10888,7 +4849,7 @@ static void amdgpu_dm_commit_streams(struct drm_atomic_commit *state,
 
 		dm_old_crtc_state = to_dm_crtc_state(old_crtc_state);
 
-		dm_clear_writeback(dm, dm_old_crtc_state);
+		dm_clear_writeback(dm, acrtc, dm_old_crtc_state);
 		acrtc->wb_enabled = false;
 	}
 
@@ -11162,9 +5123,24 @@ static void dm_set_writeback(struct amdgpu_display_manager *dm,
 
 	dc_stream_add_writeback(dm->dc, crtc_state->stream, wb_info);
 
-	acrtc->wb_pending = true;
 	acrtc->wb_conn = wb_conn;
 	drm_writeback_queue_job(wb_conn, new_con_state);
+
+	/*
+	 * Writeback completion is detected in the CRTC vblank IRQ
+	 * (dm_crtc_high_irq()). Take a vblank reference so the vblank interrupt
+	 * stays enabled while the writeback is pending; otherwise a
+	 * writeback-only commit right after drm_crtc_vblank_on() (e.g.
+	 * re-enabling a CRTC that was disabled) has no other vblank reference,
+	 * the IRQ never fires and the out fence times out. The matching put
+	 * happens once completion is signalled in dm_crtc_high_irq(), or when
+	 * the writeback is torn down in dm_clear_writeback().
+	 *
+	 * Arm wb_pending only after the reference is held so the completion IRQ
+	 * cannot run its matching vblank_put before this get.
+	 */
+	WARN_ON(drm_crtc_vblank_get(&acrtc->base));
+	acrtc->wb_pending = true;
 }
 
 static void amdgpu_dm_update_hdcp(struct drm_atomic_commit *state)
@@ -11315,6 +5291,74 @@ static int amdgpu_dm_atomic_setup_commit(struct drm_atomic_commit *state)
 	return 0;
 }
 
+STATIC_IFN_KUNIT void set_multisync_trigger_params(
+		struct dc_stream_state *stream)
+{
+	struct dc_stream_state *master = NULL;
+
+	if (stream->triggered_crtc_reset.enabled) {
+		master = stream->triggered_crtc_reset.event_source;
+		stream->triggered_crtc_reset.event =
+			master->timing.flags.VSYNC_POSITIVE_POLARITY ?
+			CRTC_EVENT_VSYNC_RISING : CRTC_EVENT_VSYNC_FALLING;
+		stream->triggered_crtc_reset.delay = TRIGGER_DELAY_NEXT_PIXEL;
+	}
+}
+EXPORT_IF_KUNIT(set_multisync_trigger_params);
+
+STATIC_IFN_KUNIT void set_master_stream(struct dc_stream_state *stream_set[],
+					int stream_count)
+{
+	int j, highest_rfr = 0, master_stream = 0;
+
+	for (j = 0;  j < stream_count; j++) {
+		if (stream_set[j] && stream_set[j]->triggered_crtc_reset.enabled) {
+			int refresh_rate = 0;
+
+			refresh_rate = (stream_set[j]->timing.pix_clk_100hz*100)/
+				(stream_set[j]->timing.h_total*stream_set[j]->timing.v_total);
+			if (refresh_rate > highest_rfr) {
+				highest_rfr = refresh_rate;
+				master_stream = j;
+			}
+		}
+	}
+	for (j = 0;  j < stream_count; j++) {
+		if (stream_set[j])
+			stream_set[j]->triggered_crtc_reset.event_source = stream_set[master_stream];
+	}
+}
+EXPORT_IF_KUNIT(set_master_stream);
+
+static void dm_enable_per_frame_crtc_master_sync(struct dc_state *context)
+{
+	int i = 0;
+	struct dc_stream_state *stream;
+
+	if (context->stream_count < 2)
+		return;
+	for (i = 0; i < context->stream_count ; i++) {
+		if (!context->streams[i])
+			continue;
+		/*
+		 * TODO: add a function to read AMD VSDB bits and set
+		 * crtc_sync_master.multi_sync_enabled flag
+		 * For now it's set to false
+		 */
+	}
+
+	set_master_stream(context->streams, context->stream_count);
+
+	for (i = 0; i < context->stream_count ; i++) {
+		stream = context->streams[i];
+
+		if (!stream)
+			continue;
+
+		set_multisync_trigger_params(stream);
+	}
+}
+
 /**
  * amdgpu_dm_atomic_commit_tail() - AMDgpu DM's commit tail implementation.
  * @state: The atomic state to commit
@@ -11383,7 +5427,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_commit *state)
 
 		if ((new_con_state->hdmi.broadcast_rgb != old_con_state->hdmi.broadcast_rgb) &&
 			(dm_old_crtc_state->stream->output_color_space !=
-				get_output_color_space(&dm_new_crtc_state->stream->timing, new_con_state)))
+				amdgpu_dm_get_output_color_space(&dm_new_crtc_state->stream->timing, new_con_state)))
 			output_color_space_changed = true;
 
 		abm_changed = dm_new_crtc_state->abm_level !=
@@ -11397,7 +5441,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_commit *state)
 
 		stream_update.stream = dm_new_crtc_state->stream;
 		if (scaling_changed) {
-			update_stream_scaling_settings(dev, &dm_new_con_state->base.crtc->mode,
+			amdgpu_dm_update_stream_scaling_settings(dev, &dm_new_con_state->base.crtc->mode,
 					dm_new_con_state, dm_new_crtc_state->stream);
 
 			stream_update.src = dm_new_crtc_state->stream->src;
@@ -11406,7 +5450,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_commit *state)
 
 		if (output_color_space_changed) {
 			dm_new_crtc_state->stream->output_color_space
-				= get_output_color_space(&dm_new_crtc_state->stream->timing, new_con_state);
+				= amdgpu_dm_get_output_color_space(&dm_new_crtc_state->stream->timing, new_con_state);
 
 			stream_update.output_color_space = &dm_new_crtc_state->stream->output_color_space;
 		}
@@ -11418,7 +5462,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_commit *state)
 		}
 
 		if (hdr_changed) {
-			fill_hdr_info_packet(new_con_state, &hdr_packet);
+			amdgpu_dm_fill_hdr_info_packet(new_con_state, &hdr_packet);
 			stream_update.hdr_static_metadata = &hdr_packet;
 		}
 
@@ -11584,144 +5628,46 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_commit *state)
 	 * send vblank event on all events not handled in flip and
 	 * mark consumed event for drm_atomic_helper_commit_hw_done
 	 */
-	spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
-	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
-
-		if (new_crtc_state->event)
-			drm_send_event_locked(dev, &new_crtc_state->event->base);
-
-		new_crtc_state->event = NULL;
-	}
-	spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
-
-	/* Signal HW programming completion */
-	drm_atomic_helper_commit_hw_done(state);
-
-	if (wait_for_vblank)
-		drm_atomic_helper_wait_for_flip_done(dev, state);
-
-	drm_atomic_helper_cleanup_planes(dev, state);
-
-	/* Don't free the memory if we are hitting this as part of suspend.
-	 * This way we don't free any memory during suspend; see
-	 * amdgpu_bo_free_kernel().  The memory will be freed in the first
-	 * non-suspend modeset or when the driver is torn down.
-	 */
-	if (!adev->in_suspend) {
-		/* return the stolen vga memory back to VRAM */
-		if (!adev->mman.keep_stolen_vga_memory)
-			amdgpu_ttm_unmark_vram_reserved(adev, AMDGPU_RESV_STOLEN_VGA);
-		amdgpu_ttm_unmark_vram_reserved(adev, AMDGPU_RESV_STOLEN_EXTENDED);
-	}
-
-	/*
-	 * Finally, drop a runtime PM reference for each newly disabled CRTC,
-	 * so we can put the GPU into runtime suspend if we're not driving any
-	 * displays anymore
-	 */
-	for (i = 0; i < crtc_disable_count; i++)
-		pm_runtime_put_autosuspend(dev->dev);
-	pm_runtime_mark_last_busy(dev->dev);
-
-	trace_amdgpu_dm_atomic_commit_tail_finish(state);
-}
-
-static int dm_force_atomic_commit(struct drm_connector *connector)
-{
-	int ret = 0;
-	struct drm_device *ddev = connector->dev;
-	struct drm_atomic_commit *state = drm_atomic_commit_alloc(ddev);
-	struct amdgpu_crtc *disconnected_acrtc = to_amdgpu_crtc(connector->encoder->crtc);
-	struct drm_plane *plane = disconnected_acrtc->base.primary;
-	struct drm_connector_state *conn_state;
-	struct drm_crtc_state *crtc_state;
-	struct drm_plane_state *plane_state;
-
-	if (!state)
-		return -ENOMEM;
-
-	state->acquire_ctx = ddev->mode_config.acquire_ctx;
-
-	/* Construct an atomic state to restore previous display setting */
-
-	/*
-	 * Attach connectors to drm_atomic_commit
-	 */
-	conn_state = drm_atomic_get_connector_state(state, connector);
-
-	/* Check for error in getting connector state */
-	if (IS_ERR(conn_state)) {
-		ret = PTR_ERR(conn_state);
-		goto out;
-	}
-
-	/* Attach crtc to drm_atomic_commit*/
-	crtc_state = drm_atomic_get_crtc_state(state, &disconnected_acrtc->base);
-
-	/* Check for error in getting crtc state */
-	if (IS_ERR(crtc_state)) {
-		ret = PTR_ERR(crtc_state);
-		goto out;
-	}
-
-	/* force a restore */
-	crtc_state->mode_changed = true;
-
-	/* Attach plane to drm_atomic_commit */
-	plane_state = drm_atomic_get_plane_state(state, plane);
-
-	/* Check for error in getting plane state */
-	if (IS_ERR(plane_state)) {
-		ret = PTR_ERR(plane_state);
-		goto out;
-	}
-
-	/* Call commit internally with the state we just constructed */
-	ret = drm_atomic_commit(state);
-
-out:
-	drm_atomic_commit_put(state);
-	if (ret)
-		drm_err(ddev, "Restoring old state failed with %i\n", ret);
-
-	return ret;
-}
+	spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
+	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
 
-/*
- * This function handles all cases when set mode does not come upon hotplug.
- * This includes when a display is unplugged then plugged back into the
- * same port and when running without usermode desktop manager supprot
- */
-void dm_restore_drm_connector_state(struct drm_device *dev,
-				    struct drm_connector *connector)
-{
-	struct amdgpu_dm_connector *aconnector;
-	struct amdgpu_crtc *disconnected_acrtc;
-	struct dm_crtc_state *acrtc_state;
+		if (new_crtc_state->event)
+			drm_send_event_locked(dev, &new_crtc_state->event->base);
 
-	if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
-		return;
+		new_crtc_state->event = NULL;
+	}
+	spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
 
-	aconnector = to_amdgpu_dm_connector(connector);
+	/* Signal HW programming completion */
+	drm_atomic_helper_commit_hw_done(state);
 
-	if (!aconnector->dc_sink || !connector->state || !connector->encoder)
-		return;
+	if (wait_for_vblank)
+		drm_atomic_helper_wait_for_flip_done(dev, state);
 
-	disconnected_acrtc = to_amdgpu_crtc(connector->encoder->crtc);
-	if (!disconnected_acrtc)
-		return;
+	drm_atomic_helper_cleanup_planes(dev, state);
 
-	acrtc_state = to_dm_crtc_state(disconnected_acrtc->base.state);
-	if (!acrtc_state->stream)
-		return;
+	/* Don't free the memory if we are hitting this as part of suspend.
+	 * This way we don't free any memory during suspend; see
+	 * amdgpu_bo_free_kernel().  The memory will be freed in the first
+	 * non-suspend modeset or when the driver is torn down.
+	 */
+	if (!adev->in_suspend) {
+		/* return the stolen vga memory back to VRAM */
+		if (!adev->mman.keep_stolen_vga_memory)
+			amdgpu_ttm_unmark_vram_reserved(adev, AMDGPU_RESV_STOLEN_VGA);
+		amdgpu_ttm_unmark_vram_reserved(adev, AMDGPU_RESV_STOLEN_EXTENDED);
+	}
 
 	/*
-	 * If the previous sink is not released and different from the current,
-	 * we deduce we are in a state where we can not rely on usermode call
-	 * to turn on the display, so we do it here
+	 * Finally, drop a runtime PM reference for each newly disabled CRTC,
+	 * so we can put the GPU into runtime suspend if we're not driving any
+	 * displays anymore
 	 */
-	if (acrtc_state->stream->sink != aconnector->dc_sink)
-		dm_force_atomic_commit(&aconnector->base);
+	for (i = 0; i < crtc_disable_count; i++)
+		pm_runtime_put_autosuspend(dev->dev);
+	pm_runtime_mark_last_busy(dev->dev);
+
+	trace_amdgpu_dm_atomic_commit_tail_finish(state);
 }
 
 /*
@@ -11828,7 +5774,7 @@ static void reset_freesync_config_for_crtc(
 	       sizeof(new_crtc_state->vrr_infopacket));
 }
 
-static bool
+STATIC_IFN_KUNIT bool
 is_timing_unchanged_for_freesync(struct drm_crtc_state *old_crtc_state,
 				 struct drm_crtc_state *new_crtc_state)
 {
@@ -11857,8 +5803,9 @@ is_timing_unchanged_for_freesync(struct drm_crtc_state *old_crtc_state,
 
 	return false;
 }
+EXPORT_IF_KUNIT(is_timing_unchanged_for_freesync);
 
-static void set_freesync_fixed_config(struct dm_crtc_state *dm_new_crtc_state)
+STATIC_IFN_KUNIT void set_freesync_fixed_config(struct dm_crtc_state *dm_new_crtc_state)
 {
 	u64 num, den, res;
 	struct drm_crtc_state *new_crtc_state = &dm_new_crtc_state->base;
@@ -11872,6 +5819,7 @@ static void set_freesync_fixed_config(struct dm_crtc_state *dm_new_crtc_state)
 	res = div_u64(num, den);
 	dm_new_crtc_state->freesync_config.fixed_refresh_in_uhz = res;
 }
+EXPORT_IF_KUNIT(set_freesync_fixed_config);
 
 static int dm_update_crtc_state(struct amdgpu_display_manager *dm,
 			 struct drm_atomic_commit *state,
@@ -11925,7 +5873,7 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm,
 		if (!drm_atomic_crtc_needs_modeset(new_crtc_state))
 			goto skip_modeset;
 
-		new_stream = create_validate_stream_for_sink(connector,
+		new_stream = amdgpu_dm_create_validate_stream_for_sink(connector,
 							     &new_crtc_state->mode,
 							     dm_new_conn_state,
 							     dm_old_crtc_state->stream);
@@ -11953,7 +5901,7 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm,
 
 		dm_new_crtc_state->abm_level = dm_new_conn_state->abm_level;
 
-		ret = fill_hdr_info_packet(drm_new_conn_state,
+		ret = amdgpu_dm_fill_hdr_info_packet(drm_new_conn_state,
 					   &new_stream->hdr_static_metadata);
 		if (ret)
 			goto fail;
@@ -12022,11 +5970,11 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm,
 
 			goto skip_modeset;
 		} else if (amdgpu_freesync_vid_mode && aconnector &&
-			   is_freesync_video_mode(&new_crtc_state->mode,
+			   amdgpu_dm_is_freesync_video_mode(&new_crtc_state->mode,
 						  aconnector)) {
 			struct drm_display_mode *high_mode;
 
-			high_mode = get_highest_refresh_rate_mode(aconnector, false);
+			high_mode = amdgpu_dm_get_highest_refresh_rate_mode(aconnector, false);
 			if (!drm_mode_equal(&new_crtc_state->mode, high_mode))
 				set_freesync_fixed_config(dm_new_crtc_state);
 		}
@@ -12098,6 +6046,7 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm,
 	/* Release extra reference */
 	if (new_stream)
 		dc_stream_release(new_stream);
+	new_stream = NULL;
 
 	/*
 	 * We want to do dc stream updates that do not require a
@@ -12118,7 +6067,7 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm,
 	/* Scaling or underscan settings */
 	if (is_scaling_state_different(dm_old_conn_state, dm_new_conn_state) ||
 				drm_atomic_crtc_needs_modeset(new_crtc_state))
-		update_stream_scaling_settings(adev_to_drm(adev),
+		amdgpu_dm_update_stream_scaling_settings(adev_to_drm(adev),
 			&new_crtc_state->mode, dm_new_conn_state, dm_new_crtc_state->stream);
 
 	/* ABM settings */
@@ -12214,10 +6163,6 @@ static bool should_reset_plane(struct drm_atomic_commit *state,
 	if (new_crtc_state->color_mgmt_changed)
 		return true;
 
-	/* Plane color pipeline or its colorop changes. */
-	if (new_plane_state->color_mgmt_changed)
-		return true;
-
 	/*
 	 * On zpos change, planes need to be reordered by removing and re-adding
 	 * them one by one to the dc state, in order of descending zpos.
@@ -12592,7 +6537,7 @@ static int dm_update_plane_state(struct dc *dc,
 		/* Tell DC to do a full surface update every time there
 		 * is a plane change. Inefficient, but works for now.
 		 */
-		dm_new_plane_state->dc_state->update_flags.bits.full_update = 1;
+		dm_new_plane_state->dc_state->update_bits.full_update = 1;
 
 		*lock_and_validation_needed = true;
 	}
@@ -12611,8 +6556,8 @@ static int dm_update_plane_state(struct dc *dc,
 	return ret;
 }
 
-static void dm_get_oriented_plane_size(struct drm_plane_state *plane_state,
-				       int *src_w, int *src_h)
+STATIC_IFN_KUNIT void dm_get_oriented_plane_size(struct drm_plane_state *plane_state,
+					 int *src_w, int *src_h)
 {
 	switch (plane_state->rotation & DRM_MODE_ROTATE_MASK) {
 	case DRM_MODE_ROTATE_90:
@@ -12628,8 +6573,9 @@ static void dm_get_oriented_plane_size(struct drm_plane_state *plane_state,
 		break;
 	}
 }
+EXPORT_IF_KUNIT(dm_get_oriented_plane_size);
 
-static void
+STATIC_IFN_KUNIT void
 dm_get_plane_scale(struct drm_plane_state *plane_state,
 		   int *out_plane_scale_w, int *out_plane_scale_h)
 {
@@ -12639,6 +6585,7 @@ dm_get_plane_scale(struct drm_plane_state *plane_state,
 	*out_plane_scale_w = plane_src_w ? plane_state->crtc_w * 1000 / plane_src_w : 0;
 	*out_plane_scale_h = plane_src_h ? plane_state->crtc_h * 1000 / plane_src_h : 0;
 }
+EXPORT_IF_KUNIT(dm_get_plane_scale);
 
 /*
  * The normalized_zpos value cannot be used by this iterator directly. It's only
@@ -12814,10 +6761,15 @@ static int dm_crtc_get_cursor_mode(struct amdgpu_device *adev,
 	/* Overlay cursor not supported on HW before DCN
 	 * DCN401/420 does not have the cursor-on-scaled-plane or cursor-on-yuv-plane restrictions
 	 * as previous DCN generations, so enable native mode on DCN401/420
+	 *
+	 * Always set native cursor mode when the CRTC is disabled,
+	 * to make sure it doesn't cause atomic commits to fail when
+	 * they are trying to disable the CRTC.
 	 */
 	if (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1) ||
 	    amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 2, 0) ||
-	    amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 2, 1)) {
+	    amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 2, 1) ||
+	    !dm_crtc_state->base.enable) {
 		*cursor_mode = DM_CURSOR_NATIVE_MODE;
 		return 0;
 	}
@@ -12853,6 +6805,31 @@ static int dm_crtc_get_cursor_mode(struct amdgpu_device *adev,
 			break;
 		}
 
+		/*
+		 * A non-cursor plane moving or resizing (without a scale change)
+		 * changes how much of the CRTC it covers. This can create or
+		 * remove a hole under the cursor and thus flip the required
+		 * cursor mode (native vs overlay), so its destination rect must
+		 * be re-evaluated too.
+		 *
+		 * The cursor plane itself is deliberately excluded: the cursor
+		 * mode depends on the underlying planes' coverage, not on the
+		 * cursor's position (see the entire_crtc_covered logic below).
+		 * Triggering on cursor movement would force every legacy cursor
+		 * update off its fast path, and in a cursor-only commit - where
+		 * the underlying planes are not part of the state - the coverage
+		 * loop would see no covering plane and misevaluate the mode as
+		 * overlay, regressing flip-vs-cursor-legacy.
+		 */
+		if (plane->type != DRM_PLANE_TYPE_CURSOR &&
+		    (old_plane_state->crtc_x != plane_state->crtc_x ||
+		     old_plane_state->crtc_y != plane_state->crtc_y ||
+		     old_plane_state->crtc_w != plane_state->crtc_w ||
+		     old_plane_state->crtc_h != plane_state->crtc_h)) {
+			consider_mode_change = true;
+			break;
+		}
+
 		if (dm_plane_color_pipeline_active(state, plane, true) !=
 		    dm_plane_color_pipeline_active(state, plane, false)) {
 			consider_mode_change = true;
@@ -13008,7 +6985,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 
 	ret = drm_atomic_helper_check_modeset(dev, state);
 	if (ret) {
-		drm_dbg_atomic(dev, "drm_atomic_helper_check_modeset() failed\n");
+		drm_dbg_atomic(dev, "drm_atomic_helper_check_modeset() failed: %pe\n", ERR_PTR(ret));
 		goto fail;
 	}
 
@@ -13023,7 +7000,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 
 		new_crtc_state = drm_atomic_get_crtc_state(state, new_con_state->crtc);
 		if (IS_ERR(new_crtc_state)) {
-			drm_dbg_atomic(dev, "drm_atomic_get_crtc_state() failed\n");
+			drm_dbg_atomic(dev, "drm_atomic_get_crtc_state() failed: %pe\n", new_crtc_state);
 			ret = PTR_ERR(new_crtc_state);
 			goto fail;
 		}
@@ -13043,7 +7020,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 			if (drm_atomic_crtc_needs_modeset(new_crtc_state)) {
 				ret = add_affected_mst_dsc_crtcs(state, crtc);
 				if (ret) {
-					drm_dbg_atomic(dev, "add_affected_mst_dsc_crtcs() failed\n");
+					drm_dbg_atomic(dev, "add_affected_mst_dsc_crtcs() failed: %pe\n", ERR_PTR(ret));
 					goto fail;
 				}
 			}
@@ -13060,7 +7037,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 
 		ret = amdgpu_dm_verify_lut_sizes(new_crtc_state);
 		if (ret) {
-			drm_dbg_atomic(dev, "amdgpu_dm_verify_lut_sizes() failed\n");
+			drm_dbg_atomic(dev, "amdgpu_dm_verify_lut_sizes() failed: %pe\n", ERR_PTR(ret));
 			goto fail;
 		}
 
@@ -13069,13 +7046,13 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 
 		ret = drm_atomic_add_affected_connectors(state, crtc);
 		if (ret) {
-			drm_dbg_atomic(dev, "drm_atomic_add_affected_connectors() failed\n");
+			drm_dbg_atomic(dev, "drm_atomic_add_affected_connectors() failed: %pe\n", ERR_PTR(ret));
 			goto fail;
 		}
 
 		ret = drm_atomic_add_affected_planes(state, crtc);
 		if (ret) {
-			drm_dbg_atomic(dev, "drm_atomic_add_affected_planes() failed\n");
+			drm_dbg_atomic(dev, "drm_atomic_add_affected_planes() failed: %pe\n", ERR_PTR(ret));
 			goto fail;
 		}
 
@@ -13114,7 +7091,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 
 			if (IS_ERR(new_plane_state)) {
 				ret = PTR_ERR(new_plane_state);
-				drm_dbg_atomic(dev, "new_plane_state is BAD\n");
+				drm_dbg_atomic(dev, "new_plane_state is BAD: %pe\n", new_plane_state);
 				goto fail;
 			}
 		}
@@ -13128,7 +7105,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 	 */
 	ret = drm_atomic_normalize_zpos(dev, state);
 	if (ret) {
-		drm_dbg(dev, "drm_atomic_normalize_zpos() failed\n");
+		drm_dbg(dev, "drm_atomic_normalize_zpos() failed: %pe\n", ERR_PTR(ret));
 		goto fail;
 	}
 
@@ -13142,7 +7119,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 		ret = dm_crtc_get_cursor_mode(adev, state, dm_new_crtc_state,
 					      &dm_new_crtc_state->cursor_mode);
 		if (ret) {
-			drm_dbg(dev, "Failed to determine cursor mode\n");
+			drm_dbg(dev, "Failed to determine cursor mode: %pe\n", ERR_PTR(ret));
 			goto fail;
 		}
 
@@ -13174,7 +7151,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 					    &lock_and_validation_needed,
 					    &is_top_most_overlay);
 		if (ret) {
-			drm_dbg_atomic(dev, "dm_update_plane_state() failed\n");
+			drm_dbg_atomic(dev, "dm_update_plane_state() failed: %pe\n", ERR_PTR(ret));
 			goto fail;
 		}
 	}
@@ -13187,7 +7164,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 					   false,
 					   &lock_and_validation_needed);
 		if (ret) {
-			drm_dbg_atomic(dev, "DISABLE: dm_update_crtc_state() failed\n");
+			drm_dbg_atomic(dev, "DISABLE: dm_update_crtc_state() failed: %pe\n", ERR_PTR(ret));
 			goto fail;
 		}
 	}
@@ -13200,7 +7177,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 					   true,
 					   &lock_and_validation_needed);
 		if (ret) {
-			drm_dbg_atomic(dev, "ENABLE: dm_update_crtc_state() failed\n");
+			drm_dbg_atomic(dev, "ENABLE: dm_update_crtc_state() failed: %pe\n", ERR_PTR(ret));
 			goto fail;
 		}
 	}
@@ -13214,7 +7191,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 					    &lock_and_validation_needed,
 					    &is_top_most_overlay);
 		if (ret) {
-			drm_dbg_atomic(dev, "dm_update_plane_state() failed\n");
+			drm_dbg_atomic(dev, "dm_update_plane_state() failed: %pe\n", ERR_PTR(ret));
 			goto fail;
 		}
 	}
@@ -13230,7 +7207,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 	/* Run this here since we want to validate the streams we created */
 	ret = drm_atomic_helper_check_planes(dev, state);
 	if (ret) {
-		drm_dbg_atomic(dev, "drm_atomic_helper_check_planes() failed\n");
+		drm_dbg_atomic(dev, "drm_atomic_helper_check_planes() failed: %pe\n", ERR_PTR(ret));
 		goto fail;
 	}
 
@@ -13368,13 +7345,13 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 	if (lock_and_validation_needed) {
 		ret = dm_atomic_get_state(state, &dm_state);
 		if (ret) {
-			drm_dbg_atomic(dev, "dm_atomic_get_state() failed\n");
+			drm_dbg_atomic(dev, "dm_atomic_get_state() failed: %pe\n", ERR_PTR(ret));
 			goto fail;
 		}
 
 		ret = do_aquire_global_lock(dev, state);
 		if (ret) {
-			drm_dbg_atomic(dev, "do_aquire_global_lock() failed\n");
+			drm_dbg_atomic(dev, "do_aquire_global_lock() failed: %pe\n", ERR_PTR(ret));
 			goto fail;
 		}
 
@@ -13382,7 +7359,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 		if (dc_resource_is_dsc_encoding_supported(dc)) {
 			ret = compute_mst_dsc_configs_for_state(state, dm_state->context, vars);
 			if (ret) {
-				drm_dbg_atomic(dev, "MST_DSC compute_mst_dsc_configs_for_state() failed\n");
+				drm_dbg_atomic(dev, "MST_DSC compute_mst_dsc_configs_for_state() failed: %pe\n", ERR_PTR(ret));
 				ret = -EINVAL;
 				goto fail;
 			}
@@ -13391,7 +7368,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 
 		ret = dm_update_mst_vcpi_slots_for_dsc(state, dm_state->context, vars);
 		if (ret) {
-			drm_dbg_atomic(dev, "dm_update_mst_vcpi_slots_for_dsc() failed\n");
+			drm_dbg_atomic(dev, "dm_update_mst_vcpi_slots_for_dsc() failed: %pe\n", ERR_PTR(ret));
 			goto fail;
 		}
 
@@ -13403,7 +7380,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 		 */
 		ret = drm_dp_mst_atomic_check(state);
 		if (ret) {
-			drm_dbg_atomic(dev, "MST drm_dp_mst_atomic_check() failed\n");
+			drm_dbg_atomic(dev, "MST drm_dp_mst_atomic_check() failed: %pe\n", ERR_PTR(ret));
 			goto fail;
 		}
 		status = dc_validate_global_state(dc, dm_state->context, DC_VALIDATE_MODE_ONLY);
@@ -13492,380 +7469,13 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 	else if (ret == -EINTR || ret == -EAGAIN || ret == -ERESTARTSYS)
 		drm_dbg_atomic(dev, "Atomic check stopped due to signal.\n");
 	else
-		drm_dbg_atomic(dev, "Atomic check failed with err: %d\n", ret);
+		drm_dbg_atomic(dev, "Atomic check failed: %pe\n", ERR_PTR(ret));
 
 	trace_amdgpu_dm_atomic_check_finish(state, ret);
 
 	return ret;
 }
 
-static bool dm_edid_parser_send_cea(struct amdgpu_display_manager *dm,
-		unsigned int offset,
-		unsigned int total_length,
-		u8 *data,
-		unsigned int length,
-		struct amdgpu_hdmi_vsdb_info *vsdb)
-{
-	bool res;
-	union dmub_rb_cmd cmd;
-	struct dmub_cmd_send_edid_cea *input;
-	struct dmub_cmd_edid_cea_output *output;
-
-	if (length > DMUB_EDID_CEA_DATA_CHUNK_BYTES)
-		return false;
-
-	memset(&cmd, 0, sizeof(cmd));
-
-	input = &cmd.edid_cea.data.input;
-
-	cmd.edid_cea.header.type = DMUB_CMD__EDID_CEA;
-	cmd.edid_cea.header.sub_type = 0;
-	cmd.edid_cea.header.payload_bytes =
-		sizeof(cmd.edid_cea) - sizeof(cmd.edid_cea.header);
-	input->offset = offset;
-	input->length = length;
-	input->cea_total_length = total_length;
-	memcpy(input->payload, data, length);
-
-	res = dc_wake_and_execute_dmub_cmd(dm->dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY);
-	if (!res) {
-		drm_err(adev_to_drm(dm->adev), "EDID CEA parser failed\n");
-		return false;
-	}
-
-	output = &cmd.edid_cea.data.output;
-
-	if (output->type == DMUB_CMD__EDID_CEA_ACK) {
-		if (!output->ack.success) {
-			drm_err(adev_to_drm(dm->adev), "EDID CEA ack failed at offset %d\n",
-					output->ack.offset);
-		}
-	} else if (output->type == DMUB_CMD__EDID_CEA_AMD_VSDB) {
-		if (!output->amd_vsdb.vsdb_found)
-			return false;
-
-		vsdb->freesync_supported = output->amd_vsdb.freesync_supported;
-		vsdb->amd_vsdb_version = output->amd_vsdb.amd_vsdb_version;
-		vsdb->min_refresh_rate_hz = output->amd_vsdb.min_frame_rate;
-		vsdb->max_refresh_rate_hz = output->amd_vsdb.max_frame_rate;
-		vsdb->freesync_mccs_vcp_code = output->amd_vsdb.freesync_mccs_vcp_code;
-	} else {
-		drm_warn(adev_to_drm(dm->adev), "Unknown EDID CEA parser results\n");
-		return false;
-	}
-
-	return true;
-}
-
-static bool parse_edid_cea_dmcu(struct amdgpu_display_manager *dm,
-		u8 *edid_ext, int len,
-		struct amdgpu_hdmi_vsdb_info *vsdb_info)
-{
-	int i;
-
-	/* send extension block to DMCU for parsing */
-	for (i = 0; i < len; i += 8) {
-		bool res;
-		int offset;
-
-		/* send 8 bytes a time */
-		if (!dc_edid_parser_send_cea(dm->dc, i, len, &edid_ext[i], 8))
-			return false;
-
-		if (i+8 == len) {
-			/* EDID block sent completed, expect result */
-			int version, min_rate, max_rate;
-
-			res = dc_edid_parser_recv_amd_vsdb(dm->dc, &version, &min_rate, &max_rate);
-			if (res) {
-				/* amd vsdb found */
-				vsdb_info->freesync_supported = 1;
-				vsdb_info->amd_vsdb_version = version;
-				vsdb_info->min_refresh_rate_hz = min_rate;
-				vsdb_info->max_refresh_rate_hz = max_rate;
-				/* Not enabled on DMCU*/
-				vsdb_info->freesync_mccs_vcp_code = 0;
-				return true;
-			}
-			/* not amd vsdb */
-			return false;
-		}
-
-		/* check for ack*/
-		res = dc_edid_parser_recv_cea_ack(dm->dc, &offset);
-		if (!res)
-			return false;
-	}
-
-	return false;
-}
-
-static bool parse_edid_cea_dmub(struct amdgpu_display_manager *dm,
-		u8 *edid_ext, int len,
-		struct amdgpu_hdmi_vsdb_info *vsdb_info)
-{
-	int i;
-
-	/* send extension block to DMCU for parsing */
-	for (i = 0; i < len; i += 8) {
-		/* send 8 bytes a time */
-		if (!dm_edid_parser_send_cea(dm, i, len, &edid_ext[i], 8, vsdb_info))
-			return false;
-	}
-
-	return vsdb_info->freesync_supported;
-}
-
-static bool parse_edid_cea(struct amdgpu_dm_connector *aconnector,
-		u8 *edid_ext, int len,
-		struct amdgpu_hdmi_vsdb_info *vsdb_info)
-{
-	struct amdgpu_device *adev = drm_to_adev(aconnector->base.dev);
-	bool ret;
-
-	mutex_lock(&adev->dm.dc_lock);
-	if (adev->dm.dmub_srv)
-		ret = parse_edid_cea_dmub(&adev->dm, edid_ext, len, vsdb_info);
-	else
-		ret = parse_edid_cea_dmcu(&adev->dm, edid_ext, len, vsdb_info);
-	mutex_unlock(&adev->dm.dc_lock);
-	return ret;
-}
-
-static void parse_edid_displayid_vrr(struct drm_connector *connector,
-				     const struct edid *edid)
-{
-	u8 *edid_ext = NULL;
-	int i;
-	int j = 0;
-	u16 min_vfreq;
-	u16 max_vfreq;
-
-	if (!edid || !edid->extensions)
-		return;
-
-	/* Find DisplayID extension */
-	for (i = 0; i < edid->extensions; i++) {
-		edid_ext = (void *)(edid + (i + 1));
-		if (edid_ext[0] == DISPLAYID_EXT)
-			break;
-	}
-
-	if (i == edid->extensions)
-		return;
-
-	while (j < EDID_LENGTH) {
-		/* Get dynamic video timing range from DisplayID if available */
-		if (EDID_LENGTH - j > 13 && edid_ext[j] == 0x25	&&
-		    (edid_ext[j+1] & 0xFE) == 0 && (edid_ext[j+2] == 9)) {
-			min_vfreq = edid_ext[j+9];
-			if (edid_ext[j+1] & 7)
-				max_vfreq = edid_ext[j+10] + ((edid_ext[j+11] & 3) << 8);
-			else
-				max_vfreq = edid_ext[j+10];
-
-			if (max_vfreq && min_vfreq) {
-				connector->display_info.monitor_range.max_vfreq = max_vfreq;
-				connector->display_info.monitor_range.min_vfreq = min_vfreq;
-
-				return;
-			}
-		}
-		j++;
-	}
-}
-
-static int get_amd_vsdb(struct amdgpu_dm_connector *aconnector,
-			struct amdgpu_hdmi_vsdb_info *vsdb_info)
-{
-	struct drm_connector *connector = &aconnector->base;
-
-	vsdb_info->replay_mode = connector->display_info.amd_vsdb.replay_mode;
-	vsdb_info->amd_vsdb_version = connector->display_info.amd_vsdb.version;
-
-	return connector->display_info.amd_vsdb.version != 0;
-}
-
-static int parse_hdmi_amd_vsdb(struct amdgpu_dm_connector *aconnector,
-			       const struct edid *edid,
-			       struct amdgpu_hdmi_vsdb_info *vsdb_info)
-{
-	u8 *edid_ext = NULL;
-	int i;
-	bool valid_vsdb_found = false;
-
-	/*----- drm_find_cea_extension() -----*/
-	/* No EDID or EDID extensions */
-	if (edid == NULL || edid->extensions == 0)
-		return -ENODEV;
-
-	/* Find CEA extension */
-	for (i = 0; i < edid->extensions; i++) {
-		edid_ext = (uint8_t *)edid + EDID_LENGTH * (i + 1);
-		if (edid_ext[0] == CEA_EXT)
-			break;
-	}
-
-	if (i == edid->extensions)
-		return -ENODEV;
-
-	/*----- cea_db_offsets() -----*/
-	if (edid_ext[0] != CEA_EXT)
-		return -ENODEV;
-
-	valid_vsdb_found = parse_edid_cea(aconnector, edid_ext, EDID_LENGTH, vsdb_info);
-
-	return valid_vsdb_found ? i : -ENODEV;
-}
-
-/**
- * amdgpu_dm_update_freesync_caps - Update Freesync capabilities
- *
- * @connector: Connector to query.
- * @drm_edid: DRM EDID from monitor
- * @do_mccs: Controls whether MCCS (Monitor Control Command Set) over
- *	      DDC (Display Data Channel) transactions are performed. When true,
- *	      the driver queries the monitor to get or update additional FreeSync
- *	      capability information. When false, these transactions are skipped.
- *
- * Amdgpu supports Freesync in DP and HDMI displays, and it is required to keep
- * track of some of the display information in the internal data struct used by
- * amdgpu_dm. This function checks which type of connector we need to set the
- * FreeSync parameters.
- */
-void amdgpu_dm_update_freesync_caps(struct drm_connector *connector,
-				    const struct drm_edid *drm_edid, bool do_mccs)
-{
-	int i = 0;
-	struct amdgpu_dm_connector *amdgpu_dm_connector =
-			to_amdgpu_dm_connector(connector);
-	struct dm_connector_state *dm_con_state = NULL;
-	struct dc_sink *sink;
-	struct amdgpu_device *adev = drm_to_adev(connector->dev);
-	struct amdgpu_hdmi_vsdb_info vsdb_info = {0};
-	const struct edid *edid;
-	bool freesync_capable = false;
-	enum adaptive_sync_type as_type = ADAPTIVE_SYNC_TYPE_NONE;
-
-	if (!connector->state) {
-		drm_err(adev_to_drm(adev), "%s - Connector has no state", __func__);
-		goto update;
-	}
-
-	sink = amdgpu_dm_connector->dc_sink ?
-		amdgpu_dm_connector->dc_sink :
-		amdgpu_dm_connector->dc_em_sink;
-
-	drm_edid_connector_update(connector, drm_edid);
-
-	if (!drm_edid || !sink) {
-		dm_con_state = to_dm_connector_state(connector->state);
-
-		amdgpu_dm_connector->min_vfreq = 0;
-		amdgpu_dm_connector->max_vfreq = 0;
-		freesync_capable = false;
-
-		goto update;
-	}
-
-	dm_con_state = to_dm_connector_state(connector->state);
-
-	if (!adev->dm.freesync_module || !dc_supports_vrr(sink->ctx->dce_version))
-		goto update;
-
-	edid = drm_edid_raw(drm_edid); // FIXME: Get rid of drm_edid_raw()
-
-	/* Some eDP panels only have the refresh rate range info in DisplayID */
-	if ((connector->display_info.monitor_range.min_vfreq == 0 ||
-	     connector->display_info.monitor_range.max_vfreq == 0))
-		parse_edid_displayid_vrr(connector, edid);
-
-	if (edid && (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT ||
-		     sink->sink_signal == SIGNAL_TYPE_EDP)) {
-		if (amdgpu_dm_connector->dc_link &&
-		    amdgpu_dm_connector->dc_link->dpcd_caps.allow_invalid_MSA_timing_param) {
-			amdgpu_dm_connector->min_vfreq = connector->display_info.monitor_range.min_vfreq;
-			amdgpu_dm_connector->max_vfreq = connector->display_info.monitor_range.max_vfreq;
-			if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10)
-				freesync_capable = true;
-		}
-
-		get_amd_vsdb(amdgpu_dm_connector, &vsdb_info);
-
-		if (vsdb_info.replay_mode) {
-			amdgpu_dm_connector->vsdb_info.replay_mode = vsdb_info.replay_mode;
-			amdgpu_dm_connector->vsdb_info.amd_vsdb_version = vsdb_info.amd_vsdb_version;
-			amdgpu_dm_connector->as_type = ADAPTIVE_SYNC_TYPE_EDP;
-		}
-
-	} else if (drm_edid && sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A) {
-		i = parse_hdmi_amd_vsdb(amdgpu_dm_connector, edid, &vsdb_info);
-		if (i >= 0) {
-			amdgpu_dm_connector->vsdb_info = vsdb_info;
-			sink->edid_caps.freesync_vcp_code = vsdb_info.freesync_mccs_vcp_code;
-
-			if (vsdb_info.freesync_supported) {
-				amdgpu_dm_connector->min_vfreq = vsdb_info.min_refresh_rate_hz;
-				amdgpu_dm_connector->max_vfreq = vsdb_info.max_refresh_rate_hz;
-				if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10)
-					freesync_capable = true;
-
-				connector->display_info.monitor_range.min_vfreq = vsdb_info.min_refresh_rate_hz;
-				connector->display_info.monitor_range.max_vfreq = vsdb_info.max_refresh_rate_hz;
-			}
-		}
-	}
-
-	if (amdgpu_dm_connector->dc_link)
-		as_type = dm_get_adaptive_sync_support_type(amdgpu_dm_connector->dc_link);
-
-	if (as_type == FREESYNC_TYPE_PCON_IN_WHITELIST) {
-		i = parse_hdmi_amd_vsdb(amdgpu_dm_connector, edid, &vsdb_info);
-		if (i >= 0) {
-			amdgpu_dm_connector->vsdb_info = vsdb_info;
-			sink->edid_caps.freesync_vcp_code = vsdb_info.freesync_mccs_vcp_code;
-
-			if (vsdb_info.freesync_supported && vsdb_info.amd_vsdb_version > 0) {
-				amdgpu_dm_connector->pack_sdp_v1_3 = true;
-				amdgpu_dm_connector->as_type = as_type;
-
-				amdgpu_dm_connector->min_vfreq = vsdb_info.min_refresh_rate_hz;
-				amdgpu_dm_connector->max_vfreq = vsdb_info.max_refresh_rate_hz;
-				if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10)
-					freesync_capable = true;
-
-				connector->display_info.monitor_range.min_vfreq = vsdb_info.min_refresh_rate_hz;
-				connector->display_info.monitor_range.max_vfreq = vsdb_info.max_refresh_rate_hz;
-			}
-		}
-	}
-
-	/* Handle MCCS */
-	if (do_mccs) {
-		dm_helpers_read_mccs_caps(adev->dm.dc->ctx, amdgpu_dm_connector->dc_link, sink);
-
-		if (sink->edid_caps.freesync_vcp_code && !sink->mccs_caps.freesync_supported)
-			freesync_capable = false;
-
-		if (sink->mccs_caps.freesync_supported && freesync_capable)
-			dm_helpers_mccs_vcp_set(adev->dm.dc->ctx, amdgpu_dm_connector->dc_link, sink);
-	}
-
-update:
-	if (dm_con_state)
-		dm_con_state->freesync_capable = freesync_capable;
-
-	if (connector->state && amdgpu_dm_connector->dc_link && !freesync_capable &&
-	    amdgpu_dm_connector->dc_link->replay_settings.config.replay_supported) {
-		amdgpu_dm_connector->dc_link->replay_settings.config.replay_supported = false;
-		amdgpu_dm_connector->dc_link->replay_settings.replay_feature_enabled = false;
-	}
-
-	if (connector->vrr_capable_property)
-		drm_connector_set_vrr_capable_property(connector,
-						       freesync_capable);
-}
-
 void amdgpu_dm_trigger_timing_sync(struct drm_device *dev)
 {
 	struct amdgpu_device *adev = drm_to_adev(dev);
@@ -13885,12 +7495,6 @@ void amdgpu_dm_trigger_timing_sync(struct drm_device *dev)
 	mutex_unlock(&adev->dm.dc_lock);
 }
 
-static inline void amdgpu_dm_exit_ips_for_hw_access(struct dc *dc)
-{
-	if (dc->ctx->dmub_srv && !dc->ctx->dmub_srv->idle_exit_counter)
-		dc_exit_ips_for_hw_access(dc);
-}
-
 void dm_write_reg_func(const struct dc_context *ctx, uint32_t address,
 		       u32 value, const char *func_name)
 {
@@ -13919,13 +7523,6 @@ uint32_t dm_read_reg_func(const struct dc_context *ctx, uint32_t address,
 	}
 #endif
 
-	if (ctx->dmub_srv &&
-	    ctx->dmub_srv->reg_helper_offload.gather_in_progress &&
-	    !ctx->dmub_srv->reg_helper_offload.should_burst_write) {
-		ASSERT(false);
-		return 0;
-	}
-
 	amdgpu_dm_exit_ips_for_hw_access(ctx->dc);
 
 	value = cgs_read_register(ctx->cgs_device, address);
@@ -13935,179 +7532,6 @@ uint32_t dm_read_reg_func(const struct dc_context *ctx, uint32_t address,
 	return value;
 }
 
-int amdgpu_dm_process_dmub_aux_transfer_sync(
-		struct dc_context *ctx,
-		unsigned int link_index,
-		struct aux_payload *payload,
-		enum aux_return_code_type *operation_result)
-{
-	struct amdgpu_device *adev = ctx->driver_context;
-	struct dmub_notification *p_notify = adev->dm.dmub_notify;
-	int ret = -1;
-
-	mutex_lock(&adev->dm.dpia_aux_lock);
-	if (!dc_process_dmub_aux_transfer_async(ctx->dc, link_index, payload)) {
-		*operation_result = AUX_RET_ERROR_ENGINE_ACQUIRE;
-		goto out;
-	}
-
-	if (!wait_for_completion_timeout(&adev->dm.dmub_aux_transfer_done, 10 * HZ)) {
-		drm_err(adev_to_drm(adev), "wait_for_completion_timeout timeout!");
-		*operation_result = AUX_RET_ERROR_TIMEOUT;
-		goto out;
-	}
-
-	if (p_notify->result != AUX_RET_SUCCESS) {
-		/*
-		 * Transient states before tunneling is enabled could
-		 * lead to this error. We can ignore this for now.
-		 */
-		if (p_notify->result == AUX_RET_ERROR_PROTOCOL_ERROR) {
-			drm_warn(adev_to_drm(adev), "DPIA AUX failed on 0x%x(%d), error %d\n",
-					payload->address, payload->length,
-					p_notify->result);
-		}
-		*operation_result = p_notify->result;
-		goto out;
-	}
-
-	payload->reply[0] = adev->dm.dmub_notify->aux_reply.command & 0xF;
-	if (adev->dm.dmub_notify->aux_reply.command & 0xF0)
-		/* The reply is stored in the top nibble of the command. */
-		payload->reply[0] = (adev->dm.dmub_notify->aux_reply.command >> 4) & 0xF;
-
-	/*write req may receive a byte indicating partially written number as well*/
-	if (p_notify->aux_reply.length)
-		memcpy(payload->data, p_notify->aux_reply.data,
-				p_notify->aux_reply.length);
-
-	/* success */
-	ret = p_notify->aux_reply.length;
-	*operation_result = p_notify->result;
-out:
-	reinit_completion(&adev->dm.dmub_aux_transfer_done);
-	mutex_unlock(&adev->dm.dpia_aux_lock);
-	return ret;
-}
-
-static void abort_fused_io(
-		struct dc_context *ctx,
-		const struct dmub_cmd_fused_request *request
-)
-{
-	union dmub_rb_cmd command = { 0 };
-	struct dmub_rb_cmd_fused_io *io = &command.fused_io;
-
-	io->header.type = DMUB_CMD__FUSED_IO;
-	io->header.sub_type = DMUB_CMD__FUSED_IO_ABORT;
-	io->header.payload_bytes = sizeof(*io) - sizeof(io->header);
-	io->request = *request;
-	dm_execute_dmub_cmd(ctx, &command, DM_DMUB_WAIT_TYPE_NO_WAIT);
-}
-
-static bool execute_fused_io(
-		struct amdgpu_device *dev,
-		struct dc_context *ctx,
-		union dmub_rb_cmd *commands,
-		uint8_t count,
-		uint32_t timeout_us
-)
-{
-	const uint8_t ddc_line = commands[0].fused_io.request.u.aux.ddc_line;
-
-	if (ddc_line >= ARRAY_SIZE(dev->dm.fused_io))
-		return false;
-
-	struct fused_io_sync *sync = &dev->dm.fused_io[ddc_line];
-	struct dmub_rb_cmd_fused_io *first = &commands[0].fused_io;
-	const bool result = dm_execute_dmub_cmd_list(ctx, count, commands, DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY)
-			&& first->header.ret_status
-			&& first->request.status == FUSED_REQUEST_STATUS_SUCCESS;
-
-	if (!result)
-		return false;
-
-	while (wait_for_completion_timeout(&sync->replied, usecs_to_jiffies(timeout_us))) {
-		reinit_completion(&sync->replied);
-
-		struct dmub_cmd_fused_request *reply = (struct dmub_cmd_fused_request *) sync->reply_data;
-
-		static_assert(sizeof(*reply) <= sizeof(sync->reply_data), "Size mismatch");
-
-		if (reply->identifier == first->request.identifier) {
-			first->request = *reply;
-			return true;
-		}
-	}
-
-	reinit_completion(&sync->replied);
-	first->request.status = FUSED_REQUEST_STATUS_TIMEOUT;
-	abort_fused_io(ctx, &first->request);
-	return false;
-}
-
-bool amdgpu_dm_execute_fused_io(
-		struct amdgpu_device *dev,
-		struct dc_link *link,
-		union dmub_rb_cmd *commands,
-		uint8_t count,
-		uint32_t timeout_us)
-{
-	struct amdgpu_display_manager *dm = &dev->dm;
-
-	mutex_lock(&dm->dpia_aux_lock);
-
-	const bool result = execute_fused_io(dev, link->ctx, commands, count, timeout_us);
-
-	mutex_unlock(&dm->dpia_aux_lock);
-	return result;
-}
-
-int amdgpu_dm_process_dmub_set_config_sync(
-		struct dc_context *ctx,
-		unsigned int link_index,
-		struct set_config_cmd_payload *payload,
-		enum set_config_status *operation_result)
-{
-	struct amdgpu_device *adev = ctx->driver_context;
-	bool is_cmd_complete;
-	int ret;
-
-	mutex_lock(&adev->dm.dpia_aux_lock);
-	is_cmd_complete = dc_process_dmub_set_config_async(ctx->dc,
-			link_index, payload, adev->dm.dmub_notify);
-
-	if (is_cmd_complete || wait_for_completion_timeout(&adev->dm.dmub_aux_transfer_done, 10 * HZ)) {
-		ret = 0;
-		*operation_result = adev->dm.dmub_notify->sc_status;
-	} else {
-		drm_err(adev_to_drm(adev), "wait_for_completion_timeout timeout!");
-		ret = -1;
-		*operation_result = SET_CONFIG_UNKNOWN_ERROR;
-	}
-
-	if (!is_cmd_complete)
-		reinit_completion(&adev->dm.dmub_aux_transfer_done);
-	mutex_unlock(&adev->dm.dpia_aux_lock);
-	return ret;
-}
-
-bool dm_execute_dmub_cmd(const struct dc_context *ctx, union dmub_rb_cmd *cmd, enum dm_dmub_wait_type wait_type)
-{
-	struct amdgpu_device *adev = ctx->driver_context;
-
-	guard(spinlock_irqsave)(&adev->dm.dmub_lock);
-	return dc_dmub_srv_cmd_run(ctx->dmub_srv, cmd, wait_type);
-}
-
-bool dm_execute_dmub_cmd_list(const struct dc_context *ctx, unsigned int count, union dmub_rb_cmd *cmd, enum dm_dmub_wait_type wait_type)
-{
-	struct amdgpu_device *adev = ctx->driver_context;
-
-	guard(spinlock_irqsave)(&adev->dm.dmub_lock);
-	return dc_dmub_srv_cmd_run_list(ctx->dmub_srv, count, cmd, wait_type);
-}
-
 void dm_acpi_process_phy_transition_interlock(
 	const struct dc_context *ctx,
 	struct dm_process_phy_transition_init_params process_phy_transition_init_params)
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
index dd199e0b79226..f396803d14854 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
@@ -324,6 +324,8 @@ struct hpd_rx_irq_offload_work {
  * @ddev: DRM base driver structure
  * @display_indexes_num: Max number of display streams supported
  * @irq_handler_list_table_lock: Synchronizes access to IRQ tables
+ * @irq_wq: Dedicated high-priority unbound workqueue for deferred IRQ work
+ * @vmin_vmax_wq: Dedicated unbound workqueue for deferred vmin/vmax updates
  * @backlight_dev: Backlight control device
  * @backlight_link: Link on which to control backlight
  * @backlight_caps: Capabilities of the backlight device
@@ -563,6 +565,8 @@ struct amdgpu_display_manager {
 	dmub_outbox_params[1];
 
 	spinlock_t irq_handler_list_table_lock;
+	struct workqueue_struct *irq_wq;
+	struct workqueue_struct *vmin_vmax_wq;
 
 	struct backlight_device *backlight_dev[AMDGPU_DM_MAX_NUM_EDP];
 
@@ -688,6 +692,13 @@ struct amdgpu_display_manager {
 	 */
 	void *bb_from_dmub;
 
+	/**
+	 * @i2c_devres_group:
+	 *
+	 * Devres group for DM i2c adapter lifetime management.
+	 */
+	void *i2c_devres_group;
+
 	/**
 	 * @oem_i2c:
 	 *
@@ -877,6 +888,9 @@ struct amdgpu_dm_connector {
 	unsigned int hdmi_hpd_debounce_delay_ms;
 	struct delayed_work hdmi_hpd_debounce_work;
 	struct dc_sink *hdmi_prev_sink;
+
+	/* HDMI compliance automation */
+	bool hdmi_comp_auto;
 };
 
 static inline void amdgpu_dm_set_mst_status(uint8_t *status,
@@ -1061,35 +1075,7 @@ struct dm_connector_state {
 #define to_dm_connector_state(x)\
 	container_of((x), struct dm_connector_state, base)
 
-void amdgpu_dm_connector_funcs_reset(struct drm_connector *connector);
-struct drm_connector_state *
-amdgpu_dm_connector_atomic_duplicate_state(struct drm_connector *connector);
-int amdgpu_dm_connector_atomic_set_property(struct drm_connector *connector,
-					    struct drm_connector_state *state,
-					    struct drm_property *property,
-					    uint64_t val);
-
-int amdgpu_dm_connector_atomic_get_property(struct drm_connector *connector,
-					    const struct drm_connector_state *state,
-					    struct drm_property *property,
-					    uint64_t *val);
-
-int amdgpu_dm_get_encoder_crtc_mask(struct amdgpu_device *adev);
-
-void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm,
-				     struct amdgpu_dm_connector *aconnector,
-				     int connector_type,
-				     struct dc_link *link,
-				     int link_index);
-
-enum drm_mode_status amdgpu_dm_connector_mode_valid(struct drm_connector *connector,
-				   const struct drm_display_mode *mode);
-
-void dm_restore_drm_connector_state(struct drm_device *dev,
-				    struct drm_connector *connector);
-
-void amdgpu_dm_update_freesync_caps(struct drm_connector *connector,
-				    const struct drm_edid *drm_edid, bool do_mccs);
+#include "amdgpu_dm_connector.h"
 
 void amdgpu_dm_trigger_timing_sync(struct drm_device *dev);
 
@@ -1113,14 +1099,9 @@ int amdgpu_dm_update_plane_color_mgmt(struct dm_crtc_state *crtc,
 				      struct drm_plane_state *plane_state,
 				      struct dc_plane_state *dc_plane_state);
 
-void amdgpu_dm_update_connector_after_detect(
-		struct amdgpu_dm_connector *aconnector);
-
 void populate_hdmi_info_from_connector(bool enable_frl, struct drm_hdmi_info *info,
 				      struct dc_edid_caps *edid_caps);
 
-extern const struct drm_encoder_helper_funcs amdgpu_dm_encoder_helper_funcs;
-
 int amdgpu_dm_process_dmub_aux_transfer_sync(struct dc_context *ctx, unsigned int link_index,
 					struct aux_payload *payload, enum aux_return_code_type *operation_result);
 
@@ -1135,20 +1116,9 @@ bool amdgpu_dm_execute_fused_io(
 int amdgpu_dm_process_dmub_set_config_sync(struct dc_context *ctx, unsigned int link_index,
 					struct set_config_cmd_payload *payload, enum set_config_status *operation_result);
 
-struct dc_stream_state *
-	create_validate_stream_for_sink(struct drm_connector *connector,
-					const struct drm_display_mode *drm_mode,
-					const struct dm_connector_state *dm_state,
-					const struct dc_stream_state *old_stream);
-
 int dm_atomic_get_state(struct drm_atomic_commit *state,
 			struct dm_atomic_state **dm_state);
 
-struct drm_connector *
-amdgpu_dm_find_first_crtc_matching_connector(struct drm_atomic_commit *state,
-					     struct drm_crtc *crtc);
-
-int convert_dc_color_depth_into_bpc(enum dc_color_depth display_color_depth);
 struct idle_workqueue *idle_create_workqueue(struct amdgpu_device *adev);
 
 void *dm_allocate_gpu_mem(struct amdgpu_device *adev,
@@ -1161,11 +1131,33 @@ void dm_free_gpu_mem(struct amdgpu_device *adev,
 
 bool amdgpu_dm_is_headless(struct amdgpu_device *adev);
 
-void hdmi_cec_set_edid(struct amdgpu_dm_connector *aconnector);
-void hdmi_cec_unset_edid(struct amdgpu_dm_connector *aconnector);
-int amdgpu_dm_initialize_hdmi_connector(struct amdgpu_dm_connector *aconnector);
+bool amdgpu_dm_crtc_complete_writeback(struct amdgpu_crtc *acrtc);
 
 void retrieve_dmi_info(struct amdgpu_display_manager *dm);
 
-void amdgpu_dm_update_backlight_caps(struct amdgpu_display_manager *dm, int bl_idx);
+void amdgpu_dm_emulated_link_detect(struct dc_link *link);
+void amdgpu_dm_apply_delay_after_dpcd_poweroff(struct amdgpu_device *adev,
+											   struct dc_sink *sink);
+
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+int dm_plane_layer_index_cmp(const void *a, const void *b);
+int fill_plane_color_attributes(const struct drm_plane_state *plane_state,
+				const enum surface_pixel_format format,
+				enum dc_color_space *color_space);
+bool modereset_required(struct drm_crtc_state *crtc_state);
+void dm_get_oriented_plane_size(struct drm_plane_state *plane_state,
+				int *src_w, int *src_h);
+void dm_get_plane_scale(struct drm_plane_state *plane_state,
+			int *out_plane_scale_w, int *out_plane_scale_h);
+bool is_scaling_state_different(const struct dm_connector_state *dm_state,
+				const struct dm_connector_state *old_dm_state);
+bool is_timing_unchanged_for_freesync(struct drm_crtc_state *old_crtc_state,
+				      struct drm_crtc_state *new_crtc_state);
+void set_freesync_fixed_config(struct dm_crtc_state *dm_new_crtc_state);
+bool is_dc_timing_adjust_needed(struct dm_crtc_state *old_state,
+				struct dm_crtc_state *new_state);
+void set_multisync_trigger_params(struct dc_stream_state *stream);
+void set_master_stream(struct dc_stream_state *stream_set[], int stream_count);
+#endif
+
 #endif /* __AMDGPU_DM_H__ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_audio.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_audio.c
new file mode 100644
index 0000000000000..e97ce5c2c0e09
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_audio.c
@@ -0,0 +1,337 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_dm.h"
+#include "amdgpu_dm_audio.h"
+#include "amdgpu_dm_kunit_helpers.h"
+#include "dc.h"
+
+#include <linux/component.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_audio_component.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_eld.h>
+
+#include "dc/inc/core_types.h"
+
+static int amdgpu_dm_audio_component_get_eld(struct device *kdev, int port,
+					  int pipe, bool *enabled,
+					  unsigned char *buf, int max_bytes)
+{
+	struct drm_device *dev = dev_get_drvdata(kdev);
+	struct amdgpu_device *adev = drm_to_adev(dev);
+	struct drm_connector *connector;
+	struct drm_connector_list_iter conn_iter;
+	struct amdgpu_dm_connector *aconnector;
+	int ret = 0;
+
+	*enabled = false;
+
+	mutex_lock(&adev->dm.audio_lock);
+
+	drm_connector_list_iter_begin(dev, &conn_iter);
+	drm_for_each_connector_iter(connector, &conn_iter) {
+
+		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
+			continue;
+
+		aconnector = to_amdgpu_dm_connector(connector);
+		if (aconnector->audio_inst != port)
+			continue;
+
+		*enabled = true;
+		mutex_lock(&connector->eld_mutex);
+		ret = drm_eld_size(connector->eld);
+		memcpy(buf, connector->eld, min(max_bytes, ret));
+		mutex_unlock(&connector->eld_mutex);
+
+		break;
+	}
+	drm_connector_list_iter_end(&conn_iter);
+
+	mutex_unlock(&adev->dm.audio_lock);
+
+	drm_dbg_kms(adev_to_drm(adev), "Get ELD : idx=%d ret=%d en=%d\n", port, ret, *enabled);
+
+	return ret;
+}
+
+static const struct drm_audio_component_ops amdgpu_dm_audio_component_ops = {
+	.get_eld = amdgpu_dm_audio_component_get_eld,
+};
+
+STATIC_IFN_KUNIT int amdgpu_dm_audio_component_bind(struct device *kdev,
+				       struct device *hda_kdev, void *data)
+{
+	struct drm_device *dev = dev_get_drvdata(kdev);
+	struct amdgpu_device *adev = drm_to_adev(dev);
+	struct drm_audio_component *acomp = data;
+
+	acomp->ops = &amdgpu_dm_audio_component_ops;
+	acomp->dev = kdev;
+	adev->dm.audio_component = acomp;
+
+	return 0;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_audio_component_bind);
+
+STATIC_IFN_KUNIT void amdgpu_dm_audio_component_unbind(struct device *kdev,
+					  struct device *hda_kdev, void *data)
+{
+	struct amdgpu_device *adev = drm_to_adev(dev_get_drvdata(kdev));
+	struct drm_audio_component *acomp = data;
+
+	acomp->ops = NULL;
+	acomp->dev = NULL;
+	adev->dm.audio_component = NULL;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_audio_component_unbind);
+
+static const struct component_ops amdgpu_dm_audio_component_bind_ops = {
+	.bind	= amdgpu_dm_audio_component_bind,
+	.unbind	= amdgpu_dm_audio_component_unbind,
+};
+
+STATIC_IFN_KUNIT
+void amdgpu_dm_audio_init_pins(struct amdgpu_device *adev, int audio_count,
+			       const unsigned int *inst_array)
+{
+	int i;
+
+	adev->mode_info.audio.num_pins = audio_count;
+
+	for (i = 0; i < audio_count; i++) {
+		adev->mode_info.audio.pin[i].channels = -1;
+		adev->mode_info.audio.pin[i].rate = -1;
+		adev->mode_info.audio.pin[i].bits_per_sample = -1;
+		adev->mode_info.audio.pin[i].status_bits = 0;
+		adev->mode_info.audio.pin[i].category_code = 0;
+		adev->mode_info.audio.pin[i].connected = false;
+		adev->mode_info.audio.pin[i].id = inst_array[i];
+		adev->mode_info.audio.pin[i].offset = 0;
+	}
+}
+EXPORT_IF_KUNIT(amdgpu_dm_audio_init_pins);
+
+int amdgpu_dm_audio_init(struct amdgpu_device *adev)
+{
+	unsigned int inst_array[MAX_AUDIOS];
+	int audio_count;
+	int i, ret;
+
+	if (!amdgpu_audio)
+		return 0;
+
+	adev->mode_info.audio.enabled = true;
+
+	audio_count = adev->dm.dc->res_pool->audio_count;
+	for (i = 0; i < audio_count; i++)
+		inst_array[i] = adev->dm.dc->res_pool->audios[i]->inst;
+	amdgpu_dm_audio_init_pins(adev, audio_count, inst_array);
+
+	ret = component_add(adev->dev, &amdgpu_dm_audio_component_bind_ops);
+	if (ret < 0)
+		return ret;
+
+	adev->dm.audio_registered = true;
+
+	return 0;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_audio_init);
+
+void amdgpu_dm_audio_fini(struct amdgpu_device *adev)
+{
+	if (!amdgpu_audio)
+		return;
+
+	if (!adev->mode_info.audio.enabled)
+		return;
+
+	if (adev->dm.audio_registered) {
+		component_del(adev->dev, &amdgpu_dm_audio_component_bind_ops);
+		adev->dm.audio_registered = false;
+	}
+
+	/* TODO: Disable audio? */
+
+	adev->mode_info.audio.enabled = false;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_audio_fini);
+
+void amdgpu_dm_audio_eld_notify(struct amdgpu_device *adev, int pin)
+{
+	struct drm_audio_component *acomp = adev->dm.audio_component;
+
+	if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify) {
+		drm_dbg_kms(adev_to_drm(adev), "Notify ELD: %d\n", pin);
+
+		acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr,
+						 pin, -1);
+	}
+}
+EXPORT_IF_KUNIT(amdgpu_dm_audio_eld_notify);
+
+void amdgpu_dm_fill_audio_info(struct audio_info *audio_info,
+		     const struct drm_connector *drm_connector,
+		     const struct dc_sink *dc_sink)
+{
+	int i = 0;
+	int cea_revision = 0;
+	const struct dc_edid_caps *edid_caps = &dc_sink->edid_caps;
+
+	audio_info->manufacture_id = edid_caps->manufacturer_id;
+	audio_info->product_id = edid_caps->product_id;
+
+	cea_revision = drm_connector->display_info.cea_rev;
+
+	strscpy(audio_info->display_name,
+		edid_caps->display_name,
+		AUDIO_INFO_DISPLAY_NAME_SIZE_IN_CHARS);
+
+	if (cea_revision >= 3) {
+		audio_info->mode_count = edid_caps->audio_mode_count;
+
+		for (i = 0; i < audio_info->mode_count; ++i) {
+			audio_info->modes[i].format_code =
+					(enum audio_format_code)
+					(edid_caps->audio_modes[i].format_code);
+			audio_info->modes[i].channel_count =
+					edid_caps->audio_modes[i].channel_count;
+			audio_info->modes[i].sample_rates.all =
+					edid_caps->audio_modes[i].sample_rate;
+			audio_info->modes[i].sample_size =
+					edid_caps->audio_modes[i].sample_size;
+		}
+	}
+
+	audio_info->flags.all = edid_caps->speaker_flags;
+
+	/* TODO: We only check for the progressive mode, check for interlace mode too */
+	if (drm_connector->latency_present[0]) {
+		audio_info->video_latency = drm_connector->video_latency[0];
+		audio_info->audio_latency = drm_connector->audio_latency[0];
+	}
+
+	/* TODO: For DP, video and audio latency should be calculated from DPCD caps */
+
+}
+EXPORT_IF_KUNIT(amdgpu_dm_fill_audio_info);
+
+void amdgpu_dm_commit_audio(struct drm_device *dev,
+			    struct drm_atomic_commit *state)
+{
+	struct amdgpu_device *adev = drm_to_adev(dev);
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_connector *connector;
+	struct drm_connector_state *old_con_state, *new_con_state;
+	struct drm_crtc_state *new_crtc_state;
+	struct dm_crtc_state *new_dm_crtc_state;
+	const struct dc_stream_status *status;
+	int i, inst;
+
+	/* Notify device removals. */
+	for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) {
+		if (old_con_state->crtc != new_con_state->crtc) {
+			/* CRTC changes require notification. */
+			goto notify;
+		}
+
+		if (!new_con_state->crtc)
+			continue;
+
+		new_crtc_state = drm_atomic_get_new_crtc_state(
+			state, new_con_state->crtc);
+
+		if (!new_crtc_state)
+			continue;
+
+		if (!drm_atomic_crtc_needs_modeset(new_crtc_state))
+			continue;
+
+notify:
+		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
+			continue;
+
+		aconnector = to_amdgpu_dm_connector(connector);
+
+		mutex_lock(&adev->dm.audio_lock);
+		inst = aconnector->audio_inst;
+		aconnector->audio_inst = -1;
+		mutex_unlock(&adev->dm.audio_lock);
+
+		amdgpu_dm_audio_eld_notify(adev, inst);
+	}
+
+	/* Notify audio device additions. */
+	for_each_new_connector_in_state(state, connector, new_con_state, i) {
+		if (!new_con_state->crtc)
+			continue;
+
+		new_crtc_state = drm_atomic_get_new_crtc_state(
+			state, new_con_state->crtc);
+
+		if (!new_crtc_state)
+			continue;
+
+		if (!drm_atomic_crtc_needs_modeset(new_crtc_state))
+			continue;
+
+		new_dm_crtc_state = to_dm_crtc_state(new_crtc_state);
+		if (!new_dm_crtc_state->stream)
+			continue;
+
+		status = dc_stream_get_status(new_dm_crtc_state->stream);
+		if (!status)
+			continue;
+
+		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
+			continue;
+
+		aconnector = to_amdgpu_dm_connector(connector);
+
+		mutex_lock(&adev->dm.audio_lock);
+		inst = status->audio_inst;
+		aconnector->audio_inst = inst;
+		mutex_unlock(&adev->dm.audio_lock);
+
+		amdgpu_dm_audio_eld_notify(adev, inst);
+	}
+}
+
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+int amdgpu_dm_audio_get_param(void)
+{
+	return amdgpu_audio;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_audio_get_param);
+
+void amdgpu_dm_audio_set_param(int val)
+{
+	amdgpu_audio = val;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_audio_set_param);
+#endif
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_audio.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_audio.h
new file mode 100644
index 0000000000000..e57e2ed20d566
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_audio.h
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ */
+
+#ifndef __AMDGPU_DM_AUDIO_H__
+#define __AMDGPU_DM_AUDIO_H__
+
+struct amdgpu_device;
+struct drm_device;
+struct drm_atomic_state;
+struct drm_connector;
+struct audio_info;
+struct dc_sink;
+
+int amdgpu_dm_audio_init(struct amdgpu_device *adev);
+void amdgpu_dm_audio_fini(struct amdgpu_device *adev);
+void amdgpu_dm_commit_audio(struct drm_device *dev,
+			    struct drm_atomic_commit *state);
+void amdgpu_dm_fill_audio_info(struct audio_info *audio_info,
+		     const struct drm_connector *drm_connector,
+		     const struct dc_sink *dc_sink);
+void amdgpu_dm_audio_eld_notify(struct amdgpu_device *adev, int pin);
+
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+struct device;
+
+int amdgpu_dm_audio_component_bind(struct device *kdev,
+				   struct device *hda_kdev, void *data);
+void amdgpu_dm_audio_component_unbind(struct device *kdev,
+				      struct device *hda_kdev, void *data);
+int amdgpu_dm_audio_get_param(void);
+void amdgpu_dm_audio_set_param(int val);
+void amdgpu_dm_audio_init_pins(struct amdgpu_device *adev, int audio_count,
+			       const unsigned int *inst_array);
+#endif
+
+#endif /* __AMDGPU_DM_AUDIO_H__ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.c
new file mode 100644
index 0000000000000..4eaf5e303850b
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.c
@@ -0,0 +1,737 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ */
+
+#include "dc.h"
+#include "dc/dc_dmub_srv.h"
+#include "dc/dc_state.h"
+#include "dc/dc_stat.h"
+
+#include "amdgpu.h"
+#include "amdgpu_display.h"
+#include "amdgpu_dm.h"
+#include "amdgpu_dm_backlight.h"
+#include "amdgpu_dm_psr.h"
+#include "amdgpu_dm_replay.h"
+#include "amdgpu_atombios.h"
+
+#include "modules/inc/mod_power.h"
+
+#include <linux/backlight.h>
+#include <linux/power_supply.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_utils.h>
+
+#include <acpi/video.h>
+
+#include "amdgpu_dm_trace.h"
+#include "amd_shared.h"
+#include "amdgpu_dm_kunit_helpers.h"
+
+void amdgpu_dm_update_backlight_caps(struct amdgpu_display_manager *dm,
+				     int bl_idx)
+{
+	struct amdgpu_dm_backlight_caps *caps = &dm->backlight_caps[bl_idx];
+
+	if (caps->caps_valid)
+		return;
+
+#if defined(CONFIG_ACPI)
+	amdgpu_acpi_get_backlight_caps(caps);
+
+	/* validate the firmware value is sane */
+	if (caps->caps_valid) {
+		int spread = caps->max_input_signal - caps->min_input_signal;
+
+		if (caps->max_input_signal > AMDGPU_DM_DEFAULT_MAX_BACKLIGHT ||
+		    caps->min_input_signal < 0 ||
+		    spread > AMDGPU_DM_DEFAULT_MAX_BACKLIGHT ||
+		    spread < AMDGPU_DM_MIN_SPREAD) {
+			drm_dbg_kms(adev_to_drm(dm->adev), "DM: Invalid backlight caps: min=%d, max=%d\n",
+				      caps->min_input_signal, caps->max_input_signal);
+			caps->caps_valid = false;
+		}
+	}
+#else
+	if (caps->aux_support)
+		return;
+#endif
+	if (!caps->caps_valid) {
+		caps->min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT;
+		caps->max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+		caps->ac_level = caps->dc_level = 50;
+		caps->caps_valid = true;
+	}
+}
+EXPORT_IF_KUNIT(amdgpu_dm_update_backlight_caps);
+
+STATIC_IFN_KUNIT
+int get_brightness_range(const struct amdgpu_dm_backlight_caps *caps,
+			 unsigned int *min, unsigned int *max)
+{
+	if (!caps)
+		return 0;
+
+	if (caps->aux_support) {
+		/* Firmware limits are in nits, DC API wants millinits. */
+		*max = 1000 * caps->aux_max_input_signal;
+		*min = 1000 * caps->aux_min_input_signal;
+	} else {
+		/* Firmware limits are 8-bit, PWM control is 16-bit. */
+		*max = 0x101 * caps->max_input_signal;
+		*min = 0x101 * caps->min_input_signal;
+	}
+	return 1;
+}
+EXPORT_IF_KUNIT(get_brightness_range);
+
+/* Rescale from [min..max] to [0..AMDGPU_MAX_BL_LEVEL] */
+static inline u32 scale_input_to_fw(int min, int max, u64 input)
+{
+	return DIV_ROUND_CLOSEST_ULL(input * AMDGPU_MAX_BL_LEVEL, max - min);
+}
+
+/* Rescale from [0..AMDGPU_MAX_BL_LEVEL] to [min..max] */
+static inline u32 scale_fw_to_input(int min, int max, u64 input)
+{
+	return min + DIV_ROUND_CLOSEST_ULL(input * (max - min), AMDGPU_MAX_BL_LEVEL);
+}
+
+STATIC_IFN_KUNIT
+void convert_custom_brightness(const struct amdgpu_dm_backlight_caps *caps,
+			       unsigned int min, unsigned int max,
+			       uint32_t *user_brightness)
+{
+	u32 brightness = scale_input_to_fw(min, max, *user_brightness);
+	u8 lower_signal, upper_signal, upper_lum, lower_lum, lum;
+	int left, right;
+
+	if (amdgpu_dc_debug_mask & DC_DISABLE_CUSTOM_BRIGHTNESS_CURVE)
+		return;
+
+	if (!caps->data_points)
+		return;
+
+	/*
+	 * Handle the case where brightness is below the first data point
+	 * Interpolate between (0,0) and (first_signal, first_lum)
+	 */
+	if (brightness < caps->luminance_data[0].input_signal) {
+		lum = DIV_ROUND_CLOSEST(caps->luminance_data[0].luminance * brightness,
+					caps->luminance_data[0].input_signal);
+		goto scale;
+	}
+
+	left = 0;
+	right = caps->data_points - 1;
+	while (left <= right) {
+		int mid = left + (right - left) / 2;
+		u8 signal = caps->luminance_data[mid].input_signal;
+
+		/* Exact match found */
+		if (signal == brightness) {
+			lum = caps->luminance_data[mid].luminance;
+			goto scale;
+		}
+
+		if (signal < brightness)
+			left = mid + 1;
+		else
+			right = mid - 1;
+	}
+
+	/* verify bound */
+	if (left >= caps->data_points)
+		left = caps->data_points - 1;
+
+	/* At this point, left > right */
+	lower_signal = caps->luminance_data[right].input_signal;
+	upper_signal = caps->luminance_data[left].input_signal;
+	lower_lum = caps->luminance_data[right].luminance;
+	upper_lum = caps->luminance_data[left].luminance;
+
+	/* interpolate */
+	if (right == left || !lower_lum)
+		lum = upper_lum;
+	else
+		lum = lower_lum + DIV_ROUND_CLOSEST((upper_lum - lower_lum) *
+						    (brightness - lower_signal),
+						    upper_signal - lower_signal);
+scale:
+	*user_brightness = scale_fw_to_input(min, max,
+					     DIV_ROUND_CLOSEST(lum * brightness, 101));
+}
+
+EXPORT_IF_KUNIT(convert_custom_brightness);
+
+STATIC_IFN_KUNIT
+u32 convert_brightness_from_user(const struct amdgpu_dm_backlight_caps *caps,
+				uint32_t brightness)
+{
+	unsigned int min, max;
+
+	if (!get_brightness_range(caps, &min, &max))
+		return brightness;
+
+	convert_custom_brightness(caps, min, max, &brightness);
+
+	/* Rescale 0..max to min..max */
+	return min + DIV_ROUND_CLOSEST_ULL((u64)(max - min) * brightness, max);
+}
+
+EXPORT_IF_KUNIT(convert_brightness_from_user);
+
+STATIC_IFN_KUNIT
+u32 convert_brightness_to_user(const struct amdgpu_dm_backlight_caps *caps,
+			      uint32_t brightness)
+{
+	unsigned int min, max;
+
+	if (!get_brightness_range(caps, &min, &max))
+		return brightness;
+
+	if (brightness < min)
+		return 0;
+	/* Rescale min..max to 0..max */
+	return DIV_ROUND_CLOSEST_ULL((u64)max * (brightness - min),
+				 max - min);
+}
+EXPORT_IF_KUNIT(convert_brightness_to_user);
+
+STATIC_IFN_KUNIT
+struct dc_stream_state *dm_find_stream_with_link(
+	struct amdgpu_display_manager *dm,
+	struct dc_link *link)
+{
+	struct dc_state *cur_dc_state = dm->dc->current_state;
+	struct dc_stream_state *stream = NULL;
+	int i;
+
+	for (i = 0; i < cur_dc_state->stream_count; i++) {
+		stream = cur_dc_state->streams[i];
+		if (stream->link == link)
+			return stream;
+	}
+
+	return NULL;
+}
+EXPORT_IF_KUNIT(dm_find_stream_with_link);
+
+STATIC_IFN_KUNIT
+int amdgpu_dm_backlight_get_device_index(struct amdgpu_display_manager *dm,
+					 struct backlight_device *bd)
+{
+	int i;
+
+	for (i = 0; i < dm->num_of_edps; i++) {
+		if (bd == dm->backlight_dev[i])
+			return i;
+	}
+
+	return 0;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_backlight_get_device_index);
+
+void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm,
+				   int bl_idx,
+				   u32 user_brightness)
+{
+	struct amdgpu_dm_backlight_caps *caps;
+	struct dc_link *link;
+	u32 brightness = 0;
+	bool rc = false, reallow_idle = false;
+	struct drm_connector *connector;
+	struct dc_stream_state *stream;
+	unsigned int min, max;
+
+	list_for_each_entry(connector, &dm->ddev->mode_config.connector_list, head) {
+		struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
+
+		if (aconnector->bl_idx != bl_idx)
+			continue;
+
+		/* if connector is off, save the brightness for next time it's on */
+		if (!aconnector->base.encoder) {
+			dm->brightness[bl_idx] = user_brightness;
+			dm->actual_brightness[bl_idx] = 0;
+			return;
+		}
+	}
+
+	amdgpu_dm_update_backlight_caps(dm, bl_idx);
+	caps = &dm->backlight_caps[bl_idx];
+
+	dm->brightness[bl_idx] = user_brightness;
+	/* update scratch register */
+	if (bl_idx == 0)
+		amdgpu_atombios_scratch_regs_set_backlight_level(dm->adev, dm->brightness[bl_idx]);
+	brightness = convert_brightness_from_user(caps, dm->brightness[bl_idx]);
+	link = (struct dc_link *)dm->backlight_link[bl_idx];
+
+	/* Apply brightness quirk */
+	if (caps->brightness_mask)
+		brightness |= caps->brightness_mask;
+
+	if (trace_amdgpu_dm_brightness_enabled()) {
+		trace_amdgpu_dm_brightness(__builtin_return_address(0),
+					   user_brightness,
+					   brightness,
+					   caps->aux_support,
+					   power_supply_is_system_supplied() > 0);
+	}
+
+	stream = dm_find_stream_with_link(dm, link);
+	if (!stream)
+		return;
+
+	mutex_lock(&dm->dc_lock);
+	if (dm->dc->caps.ips_support && dm->dc->ctx->dmub_srv->idle_allowed) {
+		dc_allow_idle_optimizations(dm->dc, false);
+		reallow_idle = true;
+	}
+
+	if (caps->aux_support) {
+		rc = mod_power_set_backlight_nits(dm->power_module, stream, brightness,
+			AUX_BL_DEFAULT_TRANSITION_TIME_MS, false, true);
+	} else {
+		/* power module uses millipercent */
+		get_brightness_range(caps, &min, &max);
+		brightness = DIV_ROUND_CLOSEST(brightness * 100, (max - min)) * 1000;
+		rc = mod_power_set_backlight_percent(dm->power_module, stream,
+						     brightness, 0, false);
+	}
+
+	/*
+	 * Some kms clients create a ramped backlight transition effect
+	 * by rapidly changing the backlight. Yet we must wait on dmcub
+	 * fw to exit psr/replay before programming backlight. To
+	 * prevent lag, keep disable psr/replay and let the next atomic
+	 * flip clear the event.
+	 *
+	 * ToDo: use ISM to handle rapidly backlight change
+	 *
+	 * Rapidly backlight change is similar to rapidly cursor events,
+	 * which is now handled by ISM. ISM can delay the event until system
+	 * is really idle, so we may use ISM to handle backlight change as well.
+	 */
+	amdgpu_dm_psr_set_event(dm, stream, true,
+		psr_event_hw_programming, true);
+	amdgpu_dm_replay_set_event(dm, stream, true,
+		replay_event_hw_programming, true);
+
+	if (dm->dc->caps.ips_support && reallow_idle)
+		dc_allow_idle_optimizations(dm->dc, true);
+
+	mutex_unlock(&dm->dc_lock);
+
+	if (rc)
+		dm->actual_brightness[bl_idx] = user_brightness;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_backlight_set_level);
+
+STATIC_IFN_KUNIT
+int amdgpu_dm_backlight_update_status(struct backlight_device *bd)
+{
+	struct amdgpu_display_manager *dm = bl_get_data(bd);
+	int i = amdgpu_dm_backlight_get_device_index(dm, bd);
+
+	amdgpu_dm_backlight_set_level(dm, i, bd->props.brightness);
+
+	return 0;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_backlight_update_status);
+
+STATIC_IFN_KUNIT
+u32 amdgpu_dm_backlight_get_level(struct amdgpu_display_manager *dm, int bl_idx)
+{
+	int ret;
+	struct amdgpu_dm_backlight_caps caps;
+	struct dc_link *link = (struct dc_link *)dm->backlight_link[bl_idx];
+
+	amdgpu_dm_update_backlight_caps(dm, bl_idx);
+	caps = dm->backlight_caps[bl_idx];
+
+	if (caps.aux_support) {
+		u32 avg, peak;
+
+		if (!dc_link_get_backlight_level_nits(link, &avg, &peak))
+			return dm->brightness[bl_idx];
+		return convert_brightness_to_user(&caps, avg);
+	}
+
+	ret = dc_link_get_backlight_level(link);
+
+	if (ret == DC_ERROR_UNEXPECTED)
+		return dm->brightness[bl_idx];
+
+	return convert_brightness_to_user(&caps, ret);
+}
+EXPORT_IF_KUNIT(amdgpu_dm_backlight_get_level);
+
+STATIC_IFN_KUNIT
+int amdgpu_dm_backlight_get_brightness(struct backlight_device *bd)
+{
+	struct amdgpu_display_manager *dm = bl_get_data(bd);
+	int i = amdgpu_dm_backlight_get_device_index(dm, bd);
+
+	return amdgpu_dm_backlight_get_level(dm, i);
+}
+EXPORT_IF_KUNIT(amdgpu_dm_backlight_get_brightness);
+
+static const struct backlight_ops amdgpu_dm_backlight_ops = {
+	.options = BL_CORE_SUSPENDRESUME,
+	.get_brightness = amdgpu_dm_backlight_get_brightness,
+	.update_status	= amdgpu_dm_backlight_update_status,
+};
+
+STATIC_IFN_KUNIT
+void amdgpu_dm_backlight_fill_props(const struct amdgpu_dm_backlight_caps *caps,
+				    bool is_system_supplied,
+				    bool custom_curve_enabled,
+				    struct backlight_properties *props)
+{
+	unsigned int min, max;
+
+	if (get_brightness_range(caps, &min, &max)) {
+		if (is_system_supplied)
+			props->brightness = DIV_ROUND_CLOSEST(max * caps->ac_level,
+							       100);
+		else
+			props->brightness = DIV_ROUND_CLOSEST(max * caps->dc_level,
+							       100);
+		props->max_brightness = max;
+	} else {
+		props->brightness = MAX_BACKLIGHT_LEVEL;
+		props->max_brightness = MAX_BACKLIGHT_LEVEL;
+	}
+
+	if (caps && caps->data_points && custom_curve_enabled)
+		props->scale = BACKLIGHT_SCALE_NON_LINEAR;
+	else
+		props->scale = BACKLIGHT_SCALE_LINEAR;
+	props->type = BACKLIGHT_RAW;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_backlight_fill_props);
+
+void
+amdgpu_dm_register_backlight_device(struct amdgpu_dm_connector *aconnector)
+{
+	struct drm_device *drm = aconnector->base.dev;
+	struct amdgpu_display_manager *dm = &drm_to_adev(drm)->dm;
+	struct backlight_properties props = { 0 };
+	struct amdgpu_dm_backlight_caps *caps;
+	char bl_name[16];
+	int real_brightness;
+	int init_brightness;
+
+	if (aconnector->bl_idx == -1)
+		return;
+
+	if (!acpi_video_backlight_use_native()) {
+		drm_info(drm, "Skipping amdgpu DM backlight registration\n");
+		/* Try registering an ACPI video backlight device instead. */
+		acpi_video_register_backlight();
+		return;
+	}
+
+	caps = &dm->backlight_caps[aconnector->bl_idx];
+	amdgpu_dm_backlight_fill_props(caps, power_supply_is_system_supplied() > 0,
+				       !(amdgpu_dc_debug_mask &
+					 DC_DISABLE_CUSTOM_BRIGHTNESS_CURVE),
+				       &props);
+	drm_dbg(drm, "Backlight caps: max_brightness: %d, ac %d, dc %d\n",
+		props.max_brightness, caps->ac_level, caps->dc_level);
+
+	init_brightness = props.brightness;
+
+	if (props.scale == BACKLIGHT_SCALE_NON_LINEAR)
+		drm_info(drm, "Using custom brightness curve\n");
+
+	snprintf(bl_name, sizeof(bl_name), "amdgpu_bl%d",
+		 drm->primary->index + aconnector->bl_idx);
+
+	dm->backlight_dev[aconnector->bl_idx] =
+		backlight_device_register(bl_name, aconnector->base.kdev, dm,
+					  &amdgpu_dm_backlight_ops, &props);
+	dm->brightness[aconnector->bl_idx] = props.brightness;
+
+	if (IS_ERR(dm->backlight_dev[aconnector->bl_idx])) {
+		drm_err(drm, "DM: Backlight registration failed!\n");
+		dm->backlight_dev[aconnector->bl_idx] = NULL;
+	} else {
+		/*
+		 * dm->brightness[x] can be inconsistent just after startup until
+		 * ops.get_brightness is called.
+		 */
+		real_brightness =
+			amdgpu_dm_backlight_ops.get_brightness(dm->backlight_dev[aconnector->bl_idx]);
+
+		if (real_brightness != init_brightness) {
+			dm->actual_brightness[aconnector->bl_idx] = real_brightness;
+			dm->brightness[aconnector->bl_idx] = real_brightness;
+		}
+		drm_dbg_driver(drm, "DM: Registered Backlight device: %s\n", bl_name);
+	}
+}
+EXPORT_IF_KUNIT(amdgpu_dm_register_backlight_device);
+
+void amdgpu_dm_update_connector_ext_caps(struct amdgpu_dm_connector *aconnector)
+{
+	const struct drm_panel_backlight_quirk *panel_backlight_quirk;
+	struct amdgpu_dm_backlight_caps *caps;
+	struct drm_connector *conn_base;
+	struct amdgpu_device *adev;
+	struct drm_luminance_range_info *luminance_range;
+	struct drm_device *drm;
+
+	if (aconnector->bl_idx == -1 ||
+	    aconnector->dc_link->connector_signal != SIGNAL_TYPE_EDP)
+		return;
+
+	conn_base = &aconnector->base;
+	drm = conn_base->dev;
+	adev = drm_to_adev(drm);
+
+	caps = &adev->dm.backlight_caps[aconnector->bl_idx];
+	caps->ext_caps = &aconnector->dc_link->dpcd_sink_ext_caps;
+	caps->aux_support = false;
+
+	panel_backlight_quirk = drm_get_panel_backlight_quirk(aconnector->drm_edid);
+
+	if (caps->ext_caps->bits.oled == 1
+	    /*
+	     * ||
+	     * caps->ext_caps->bits.sdr_aux_backlight_control == 1 ||
+	     * caps->ext_caps->bits.hdr_aux_backlight_control == 1
+	     */)
+		caps->aux_support = true;
+
+	if (amdgpu_backlight == 0)
+		caps->aux_support = false;
+	else if (amdgpu_backlight == 1)
+		caps->aux_support = true;
+	else if (!IS_ERR_OR_NULL(panel_backlight_quirk) &&
+		 panel_backlight_quirk->force_pwm)
+		caps->aux_support = false;
+	if (caps->aux_support)
+		aconnector->dc_link->backlight_control_type = BACKLIGHT_CONTROL_AMD_AUX;
+
+	luminance_range = &conn_base->display_info.luminance_range;
+
+	if (luminance_range->max_luminance)
+		caps->aux_max_input_signal = luminance_range->max_luminance;
+	else
+		caps->aux_max_input_signal = 512;
+
+	if (luminance_range->min_luminance)
+		caps->aux_min_input_signal = luminance_range->min_luminance;
+	else
+		caps->aux_min_input_signal = 1;
+
+	if (!IS_ERR_OR_NULL(panel_backlight_quirk)) {
+		if (panel_backlight_quirk->min_brightness) {
+			caps->min_input_signal =
+				panel_backlight_quirk->min_brightness - 1;
+			drm_info(drm,
+				 "Applying panel backlight quirk, min_brightness: %d\n",
+				 caps->min_input_signal);
+		}
+		if (panel_backlight_quirk->brightness_mask) {
+			drm_info(drm,
+				 "Applying panel backlight quirk, brightness_mask: 0x%X\n",
+				 panel_backlight_quirk->brightness_mask);
+			caps->brightness_mask =
+				panel_backlight_quirk->brightness_mask;
+		}
+	}
+}
+EXPORT_IF_KUNIT(amdgpu_dm_update_connector_ext_caps);
+
+void amdgpu_dm_setup_backlight_device(struct amdgpu_display_manager *dm,
+			    struct amdgpu_dm_connector *aconnector)
+{
+	struct dc_link *link = aconnector->dc_link;
+	int bl_idx = dm->num_of_edps;
+
+	if (!(link->connector_signal & (SIGNAL_TYPE_EDP | SIGNAL_TYPE_LVDS)) ||
+	    link->type == dc_connection_none)
+		return;
+
+	if (dm->num_of_edps >= AMDGPU_DM_MAX_NUM_EDP) {
+		drm_warn(adev_to_drm(dm->adev), "Too much eDP connections, skipping backlight setup for additional eDPs\n");
+		return;
+	}
+
+	aconnector->bl_idx = bl_idx;
+
+	amdgpu_dm_update_backlight_caps(dm, bl_idx);
+	dm->backlight_link[bl_idx] = link;
+	dm->num_of_edps++;
+
+	amdgpu_dm_update_connector_ext_caps(aconnector);
+
+	/* Offer ABM property when user didn't turn off by module parameter.
+	 * OLED panels are included to support CACP (Content Adaptive
+	 * Contrast and Power) feature via set_abm_level.
+	 */
+	if (amdgpu_dm_abm_level < 0)
+		drm_object_attach_property(&aconnector->base.base,
+					   dm->adev->mode_info.abm_level_property,
+					   ABM_SYSFS_CONTROL);
+}
+EXPORT_IF_KUNIT(amdgpu_dm_setup_backlight_device);
+
+/**
+ * DOC: panel power savings
+ *
+ * The display manager allows you to set your desired **panel power savings**
+ * level (between 0-4, with 0 representing off), e.g. using the following::
+ *
+ *   # echo 3 > /sys/class/drm/card0-eDP-1/amdgpu/panel_power_savings
+ *
+ * Modifying this value can have implications on color accuracy, so tread
+ * carefully.
+ */
+
+STATIC_IFN_KUNIT
+ssize_t panel_power_savings_show(struct device *device,
+				 struct device_attribute *attr,
+				 char *buf)
+{
+	struct drm_connector *connector = dev_get_drvdata(device);
+	struct drm_device *dev = connector->dev;
+	u8 val;
+
+	drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
+	val = to_dm_connector_state(connector->state)->abm_level ==
+		ABM_LEVEL_IMMEDIATE_DISABLE ? 0 :
+		to_dm_connector_state(connector->state)->abm_level;
+	drm_modeset_unlock(&dev->mode_config.connection_mutex);
+
+	return sysfs_emit(buf, "%u\n", val);
+}
+EXPORT_IF_KUNIT(panel_power_savings_show);
+
+STATIC_IFN_KUNIT
+ssize_t panel_power_savings_store(struct device *device,
+				 struct device_attribute *attr,
+				 const char *buf, size_t count)
+{
+	struct drm_connector *connector = dev_get_drvdata(device);
+	struct drm_device *dev = connector->dev;
+	long val;
+	int ret;
+
+	ret = kstrtol(buf, 0, &val);
+
+	if (ret)
+		return ret;
+
+	if (val < 0 || val > 4)
+		return -EINVAL;
+
+	drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
+	if (to_dm_connector_state(connector->state)->abm_sysfs_forbidden)
+		ret = -EBUSY;
+	else
+		to_dm_connector_state(connector->state)->abm_level = val ?:
+			ABM_LEVEL_IMMEDIATE_DISABLE;
+	drm_modeset_unlock(&dev->mode_config.connection_mutex);
+
+	if (ret)
+		return ret;
+
+	drm_kms_helper_hotplug_event(dev);
+
+	return count;
+}
+EXPORT_IF_KUNIT(panel_power_savings_store);
+
+static DEVICE_ATTR_RW(panel_power_savings);
+
+static struct attribute *amdgpu_attrs[] = {
+	&dev_attr_panel_power_savings.attr,
+	NULL
+};
+
+const struct attribute_group amdgpu_group = {
+	.name = "amdgpu",
+	.attrs = amdgpu_attrs
+};
+
+bool
+amdgpu_dm_should_create_sysfs(struct amdgpu_dm_connector *amdgpu_dm_connector)
+{
+	struct dc_link *link = amdgpu_dm_connector->dc_link;
+
+	if (amdgpu_dm_abm_level >= 0)
+		return false;
+
+	if (amdgpu_dm_connector->base.connector_type != DRM_MODE_CONNECTOR_eDP)
+		return false;
+
+	if (link->panel_type != PANEL_TYPE_LCD && !link->panel_config.cacp.cacp_supported)
+		return false;
+
+	return true;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_should_create_sysfs);
+
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+uint amdgpu_dm_get_dc_debug_mask(void)
+{
+	return amdgpu_dc_debug_mask;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_get_dc_debug_mask);
+
+void amdgpu_dm_set_dc_debug_mask(uint val)
+{
+	amdgpu_dc_debug_mask = val;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_set_dc_debug_mask);
+
+int amdgpu_dm_get_abm_level_param(void)
+{
+	return amdgpu_dm_abm_level;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_get_abm_level_param);
+
+void amdgpu_dm_set_abm_level_param(int val)
+{
+	amdgpu_dm_abm_level = val;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_set_abm_level_param);
+
+int amdgpu_dm_get_backlight_param(void)
+{
+	return amdgpu_backlight;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_get_backlight_param);
+
+void amdgpu_dm_set_backlight_param(int val)
+{
+	amdgpu_backlight = val;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_set_backlight_param);
+#endif
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.h
new file mode 100644
index 0000000000000..07b75064847c7
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.h
@@ -0,0 +1,90 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __AMDGPU_DM_BACKLIGHT_H__
+#define __AMDGPU_DM_BACKLIGHT_H__
+
+struct amdgpu_display_manager;
+struct amdgpu_dm_connector;
+struct backlight_device;
+struct backlight_properties;
+struct dc_link;
+struct dc_stream_state;
+struct device;
+struct device_attribute;
+struct drm_connector;
+struct attribute_group;
+
+#define AMDGPU_DM_DEFAULT_MIN_BACKLIGHT 12
+#define AMDGPU_DM_DEFAULT_MAX_BACKLIGHT 255
+#define AMDGPU_DM_MIN_SPREAD ((AMDGPU_DM_DEFAULT_MAX_BACKLIGHT - AMDGPU_DM_DEFAULT_MIN_BACKLIGHT) / 2)
+#define AUX_BL_DEFAULT_TRANSITION_TIME_MS 50
+
+void amdgpu_dm_update_backlight_caps(struct amdgpu_display_manager *dm,
+				     int bl_idx);
+void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm,
+				   int bl_idx, u32 user_brightness);
+void amdgpu_dm_register_backlight_device(struct amdgpu_dm_connector *aconnector);
+void amdgpu_dm_setup_backlight_device(struct amdgpu_display_manager *dm,
+			    struct amdgpu_dm_connector *aconnector);
+void amdgpu_dm_update_connector_ext_caps(struct amdgpu_dm_connector *aconnector);
+bool amdgpu_dm_should_create_sysfs(struct amdgpu_dm_connector *aconnector);
+
+extern const struct attribute_group amdgpu_group;
+
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+struct dc_stream_state *dm_find_stream_with_link(struct amdgpu_display_manager *dm,
+						 struct dc_link *link);
+int amdgpu_dm_backlight_update_status(struct backlight_device *bd);
+u32 amdgpu_dm_backlight_get_level(struct amdgpu_display_manager *dm, int bl_idx);
+int amdgpu_dm_backlight_get_brightness(struct backlight_device *bd);
+ssize_t panel_power_savings_show(struct device *device,
+				 struct device_attribute *attr,
+				 char *buf);
+ssize_t panel_power_savings_store(struct device *device,
+				 struct device_attribute *attr,
+				 const char *buf, size_t count);
+int get_brightness_range(const struct amdgpu_dm_backlight_caps *caps,
+			 unsigned int *min, unsigned int *max);
+void convert_custom_brightness(const struct amdgpu_dm_backlight_caps *caps,
+			       unsigned int min, unsigned int max,
+			       uint32_t *user_brightness);
+u32 convert_brightness_from_user(const struct amdgpu_dm_backlight_caps *caps,
+				 uint32_t brightness);
+u32 convert_brightness_to_user(const struct amdgpu_dm_backlight_caps *caps,
+			       uint32_t brightness);
+int amdgpu_dm_backlight_get_device_index(struct amdgpu_display_manager *dm,
+					 struct backlight_device *bd);
+void amdgpu_dm_backlight_fill_props(const struct amdgpu_dm_backlight_caps *caps,
+				    bool is_system_supplied,
+				    bool custom_curve_enabled,
+				    struct backlight_properties *props);
+uint amdgpu_dm_get_dc_debug_mask(void);
+void amdgpu_dm_set_dc_debug_mask(uint val);
+int amdgpu_dm_get_abm_level_param(void);
+void amdgpu_dm_set_abm_level_param(int val);
+int amdgpu_dm_get_backlight_param(void);
+void amdgpu_dm_set_backlight_param(int val);
+#endif
+
+#endif /* __AMDGPU_DM_BACKLIGHT_H__ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
index 86086d10c5435..357c7c5c85cff 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
@@ -1051,26 +1051,28 @@ EXPORT_IF_KUNIT(__drm_3dlut32_to_dc_3dlut);
 /* amdgpu_dm_atomic_lut3d - set DRM 3D LUT to DC stream
  * @drm_lut3d: user 3D LUT
  * @drm_lut3d_size: size of 3D LUT
- * @lut3d: DC 3D LUT
+ * @cm: DC Color Manager (includes 3D LUT)
  *
  * Map user 3D LUT data to DC 3D LUT and all necessary bits to program it
  * on DCN accordingly.
  */
 STATIC_IFN_KUNIT void amdgpu_dm_atomic_lut3d(const struct drm_color_lut *drm_lut3d,
 				     uint32_t drm_lut3d_size,
-				     struct dc_3dlut *lut)
+				     struct dc_plane_cm *cm)
 {
 	if (!drm_lut3d_size) {
-		lut->state.bits.initialized = 0;
+		cm->lut3d_func.state.bits.initialized = 0;
+		cm->flags.bits.lut3d_enable = 0;
 	} else {
 		/* Stride and bit depth are not programmable by API yet.
 		 * Therefore, only supports 17x17x17 3D LUT (12-bit).
 		 */
-		lut->lut_3d.use_tetrahedral_9 = false;
-		lut->lut_3d.use_12bits = true;
-		lut->state.bits.initialized = 1;
-		__drm_3dlut_to_dc_3dlut(drm_lut3d, drm_lut3d_size, &lut->lut_3d,
-					lut->lut_3d.use_tetrahedral_9,
+		cm->lut3d_func.lut_3d.use_tetrahedral_9 = false;
+		cm->lut3d_func.lut_3d.use_12bits = true;
+		cm->lut3d_func.state.bits.initialized = 1;
+		cm->flags.bits.lut3d_enable = 1;
+		__drm_3dlut_to_dc_3dlut(drm_lut3d, drm_lut3d_size, &cm->lut3d_func.lut_3d,
+					cm->lut3d_func.lut_3d.use_tetrahedral_9,
 					MAX_COLOR_3DLUT_BITDEPTH);
 	}
 }
@@ -1080,7 +1082,7 @@ STATIC_IFN_KUNIT int amdgpu_dm_atomic_shaper_lut(const struct drm_color_lut *sha
 				       bool has_rom,
 				       enum dc_transfer_func_predefined tf,
 				       uint32_t shaper_size,
-				       struct dc_transfer_func *func_shaper)
+				       struct dc_plane_cm *cm)
 {
 	int ret = 0;
 
@@ -1089,10 +1091,13 @@ STATIC_IFN_KUNIT int amdgpu_dm_atomic_shaper_lut(const struct drm_color_lut *sha
 		 * If user shaper LUT is set, we assume a linear color space
 		 * (linearized by degamma 1D LUT or not).
 		 */
-		__set_tf_distributed_points(func_shaper, tf);
-		ret = __set_output_tf(func_shaper, shaper_lut, shaper_size, has_rom);
+		__set_tf_distributed_points(&cm->shaper_func, tf);
+		cm->flags.bits.shaper_enable = 1;
+
+		ret = __set_output_tf(&cm->shaper_func, shaper_lut, shaper_size, has_rom);
 	} else {
-		__set_tf_bypass(func_shaper);
+		__set_tf_bypass(&cm->shaper_func);
+		cm->flags.bits.shaper_enable = 0;
 	}
 
 	return ret;
@@ -1103,7 +1108,7 @@ STATIC_IFN_KUNIT int amdgpu_dm_atomic_blend_lut(const struct drm_color_lut *blen
 				       bool has_rom,
 				       enum dc_transfer_func_predefined tf,
 				       uint32_t blend_size,
-				       struct dc_transfer_func *func_blend)
+				       struct dc_plane_cm *cm)
 {
 	int ret = 0;
 
@@ -1115,10 +1120,13 @@ STATIC_IFN_KUNIT int amdgpu_dm_atomic_blend_lut(const struct drm_color_lut *blen
 		 * module to fill the parameters that will be translated to HW
 		 * points.
 		 */
-		__set_tf_distributed_points(func_blend, tf);
-		ret = __set_input_tf(NULL, func_blend, blend_lut, blend_size);
+		__set_tf_distributed_points(&cm->blend_func, tf);
+		cm->flags.bits.blend_enable = 1;
+
+		ret = __set_input_tf(NULL, &cm->blend_func, blend_lut, blend_size);
 	} else {
-		__set_tf_bypass(func_blend);
+		__set_tf_bypass(&cm->blend_func);
+		cm->flags.bits.blend_enable = 0;
 	}
 
 	return ret;
@@ -1461,7 +1469,7 @@ __set_dm_plane_degamma(struct drm_plane_state *plane_state,
 	const struct drm_color_lut *degamma_lut;
 	enum amdgpu_transfer_function tf = AMDGPU_TRANSFER_FUNCTION_DEFAULT;
 	uint32_t degamma_size;
-	bool has_degamma_lut;
+	bool has_degamma_lut, is_subsampled_format;
 	int ret;
 
 	degamma_lut = __extract_blob_lut(dm_plane_state->degamma_lut,
@@ -1491,12 +1499,20 @@ __set_dm_plane_degamma(struct drm_plane_state *plane_state,
 		if (ret)
 			return ret;
        } else {
-		dc_plane_state->in_transfer_func.type =
-			TF_TYPE_PREDEFINED;
+	       /* Check if format requires post-scale color processing (subsampled formats) */
+		is_subsampled_format = (dc_plane_state->format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN &&
+					dc_plane_state->format < SURFACE_PIXEL_FORMAT_SUBSAMPLE_END);
+
+		dc_plane_state->in_transfer_func.type = TF_TYPE_PREDEFINED;
 
 		if (!mod_color_calculate_degamma_params(color_caps,
-		    &dc_plane_state->in_transfer_func, NULL, false))
+							&dc_plane_state->in_transfer_func,
+							NULL,
+							is_subsampled_format)) {
+			drm_err(plane_state->state->dev,
+				"Failed to calculate degamma params.\n");
 			return -ENOMEM;
+		}
 	}
 	return 0;
 }
@@ -1632,63 +1648,64 @@ __set_dm_plane_colorop_shaper(struct drm_plane_state *plane_state,
 			      struct drm_colorop *colorop)
 {
 	struct drm_colorop *old_colorop;
-	struct drm_colorop_state *colorop_state = NULL, *new_colorop_state;
+	struct drm_colorop_state *new_colorop_state;
+	struct drm_colorop_state *tf_state = NULL, *lut_state = NULL;
 	struct drm_atomic_commit *state = plane_state->state;
+	struct drm_colorop *lut_colorop;
 	enum dc_transfer_func_predefined default_tf = TRANSFER_FUNCTION_LINEAR;
-	struct dc_transfer_func *tf = &dc_plane_state->in_shaper_func;
+	struct dc_transfer_func *tf = &dc_plane_state->cm.shaper_func;
 	const struct drm_color_lut32 *shaper_lut;
 	struct drm_device *dev = colorop->dev;
 	bool enabled = false;
 	u32 shaper_size;
 	int i = 0, ret = 0;
 
-	/* 1D Curve - SHAPER TF */
+	/* 1D Curve - SHAPER TF: find state */
 	old_colorop = colorop;
 	for_each_new_colorop_in_state(state, colorop, new_colorop_state, i) {
 		if (new_colorop_state->colorop == old_colorop &&
 		    (BIT(new_colorop_state->curve_1d_type) & amdgpu_dm_supported_shaper_tfs)) {
-			colorop_state = new_colorop_state;
+			tf_state = new_colorop_state;
 			break;
 		}
 	}
 
-	if (colorop_state && !colorop_state->bypass && colorop->type == DRM_COLOROP_1D_CURVE) {
-		drm_dbg(dev, "Shaper TF colorop with ID: %d\n", colorop->base.id);
-		tf->type = TF_TYPE_DISTRIBUTED_POINTS;
-		tf->tf = default_tf = amdgpu_colorop_tf_to_dc_tf(colorop_state->curve_1d_type);
-		tf->sdr_ref_white_level = SDR_WHITE_LEVEL_INIT_VALUE;
-		ret = __set_output_tf(tf, 0, 0, false);
-		if (ret)
-			return ret;
-		enabled = true;
-	}
-
-	/* 1D LUT - SHAPER LUT */
-	colorop = old_colorop->next;
-	if (!colorop) {
+	/* 1D LUT - SHAPER LUT: find state */
+	lut_colorop = old_colorop->next;
+	if (!lut_colorop) {
 		drm_dbg(dev, "no Shaper LUT colorop found\n");
 		return -EINVAL;
 	}
 
-	old_colorop = colorop;
 	for_each_new_colorop_in_state(state, colorop, new_colorop_state, i) {
-		if (new_colorop_state->colorop == old_colorop &&
+		if (new_colorop_state->colorop == lut_colorop &&
 		    new_colorop_state->colorop->type == DRM_COLOROP_1D_LUT) {
-			colorop_state = new_colorop_state;
+			lut_state = new_colorop_state;
 			break;
 		}
 	}
 
-	if (colorop_state && !colorop_state->bypass && colorop->type == DRM_COLOROP_1D_LUT) {
-		drm_dbg(dev, "Shaper LUT colorop with ID: %d\n", colorop->base.id);
+	if (tf_state && !tf_state->bypass) {
+		drm_dbg(dev, "Shaper TF colorop with ID: %d\n", old_colorop->base.id);
+		tf->type = TF_TYPE_DISTRIBUTED_POINTS;
+		tf->tf = default_tf = amdgpu_colorop_tf_to_dc_tf(tf_state->curve_1d_type);
+		tf->sdr_ref_white_level = SDR_WHITE_LEVEL_INIT_VALUE;
+		ret = __set_output_tf(tf, 0, 0, false);
+		if (ret)
+			return ret;
+		enabled = true;
+	}
+
+	if (lut_state && !lut_state->bypass) {
+		drm_dbg(dev, "Shaper LUT colorop with ID: %d\n", lut_colorop->base.id);
 		tf->type = TF_TYPE_DISTRIBUTED_POINTS;
 		tf->tf = default_tf;
 		tf->sdr_ref_white_level = SDR_WHITE_LEVEL_INIT_VALUE;
-		shaper_lut = __extract_blob_lut32(colorop_state->data, &shaper_size);
+		shaper_lut = __extract_blob_lut32(lut_state->data, &shaper_size);
 		shaper_size = shaper_lut != NULL ? shaper_size : 0;
 
 		/* Custom LUT size must be the same as supported size */
-		if (shaper_size == colorop->size) {
+		if (shaper_size == lut_colorop->size) {
 			ret = __set_output_tf_32(tf, shaper_lut, shaper_size, false);
 			if (ret)
 				return ret;
@@ -1696,8 +1713,12 @@ __set_dm_plane_colorop_shaper(struct drm_plane_state *plane_state,
 		}
 	}
 
-	if (!enabled)
+	if (!enabled) {
 		tf->type = TF_TYPE_BYPASS;
+		dc_plane_state->cm.flags.bits.shaper_enable = 0;
+	} else {
+		dc_plane_state->cm.flags.bits.shaper_enable = 1;
+	}
 
 	return 0;
 }
@@ -1741,7 +1762,7 @@ __set_dm_plane_colorop_3dlut(struct drm_plane_state *plane_state,
 {
 	struct drm_colorop *old_colorop;
 	struct drm_colorop_state *colorop_state = NULL, *new_colorop_state;
-	struct dc_transfer_func *tf = &dc_plane_state->in_shaper_func;
+	struct dc_transfer_func *tf = &dc_plane_state->cm.shaper_func;
 	struct drm_atomic_commit *state = plane_state->state;
 	const struct amdgpu_device *adev = drm_to_adev(colorop->dev);
 	bool has_3dlut = adev->dm.dc->caps.color.dpp.hw_3d_lut || adev->dm.dc->caps.color.mpc.preblend;
@@ -1769,13 +1790,15 @@ __set_dm_plane_colorop_3dlut(struct drm_plane_state *plane_state,
 		drm_dbg(dev, "3D LUT colorop with ID: %d\n", colorop->base.id);
 		lut3d = __extract_blob_lut32(colorop_state->data, &lut3d_size);
 		lut3d_size = lut3d != NULL ? lut3d_size : 0;
-		ret = __set_colorop_3dlut(lut3d, lut3d_size, &dc_plane_state->lut3d_func);
+		ret = __set_colorop_3dlut(lut3d, lut3d_size, &dc_plane_state->cm.lut3d_func);
 		if (ret) {
 			drm_dbg(dev, "3D LUT colorop with ID: %d has LUT size = %d\n",
 				colorop->base.id, lut3d_size);
 			return ret;
 		}
 
+		dc_plane_state->cm.flags.bits.lut3d_enable = 1;
+
 		/* 3D LUT requires shaper. If shaper colorop is bypassed, enable shaper curve
 		 * with TRANSFER_FUNCTION_LINEAR
 		 */
@@ -1785,6 +1808,8 @@ __set_dm_plane_colorop_3dlut(struct drm_plane_state *plane_state,
 			tf->sdr_ref_white_level = SDR_WHITE_LEVEL_INIT_VALUE;
 			ret = __set_output_tf_32(tf, NULL, 0, false);
 		}
+	} else {
+		dc_plane_state->cm.flags.bits.lut3d_enable = 0;
 	}
 
 	return ret;
@@ -1796,61 +1821,64 @@ __set_dm_plane_colorop_blend(struct drm_plane_state *plane_state,
 			     struct drm_colorop *colorop)
 {
 	struct drm_colorop *old_colorop;
-	struct drm_colorop_state *colorop_state = NULL, *new_colorop_state;
+	struct drm_colorop_state *new_colorop_state;
+	struct drm_colorop_state *tf_state = NULL, *lut_state = NULL;
 	struct drm_atomic_commit *state = plane_state->state;
+	struct drm_colorop *lut_colorop;
 	enum dc_transfer_func_predefined default_tf = TRANSFER_FUNCTION_LINEAR;
-	struct dc_transfer_func *tf = &dc_plane_state->blend_tf;
+	struct dc_transfer_func *tf = &dc_plane_state->cm.blend_func;
 	const struct drm_color_lut32 *blend_lut = NULL;
 	struct drm_device *dev = colorop->dev;
 	uint32_t blend_size = 0;
 	int i = 0;
 
-	/* 1D Curve - BLND TF */
+	dc_plane_state->cm.flags.bits.blend_enable = 0;
+
+	/* 1D Curve - BLND TF: find state */
 	old_colorop = colorop;
 	for_each_new_colorop_in_state(state, colorop, new_colorop_state, i) {
 		if (new_colorop_state->colorop == old_colorop &&
 		    (BIT(new_colorop_state->curve_1d_type) & amdgpu_dm_supported_blnd_tfs)) {
-			colorop_state = new_colorop_state;
+			tf_state = new_colorop_state;
 			break;
 		}
 	}
 
-	if (colorop_state && !colorop_state->bypass && colorop->type == DRM_COLOROP_1D_CURVE &&
-	    (BIT(colorop_state->curve_1d_type) & amdgpu_dm_supported_blnd_tfs)) {
-		drm_dbg(dev, "Blend TF colorop with ID: %d\n", colorop->base.id);
-		tf->type = TF_TYPE_DISTRIBUTED_POINTS;
-		tf->tf = default_tf = amdgpu_colorop_tf_to_dc_tf(colorop_state->curve_1d_type);
-		tf->sdr_ref_white_level = SDR_WHITE_LEVEL_INIT_VALUE;
-		__set_input_tf_32(NULL, tf, blend_lut, blend_size);
-	}
-
-	/* 1D Curve - BLND LUT */
-	colorop = old_colorop->next;
-	if (!colorop) {
+	/* 1D LUT - BLND LUT: find state */
+	lut_colorop = old_colorop->next;
+	if (!lut_colorop) {
 		drm_dbg(dev, "no Blend LUT colorop found\n");
 		return -EINVAL;
 	}
 
-	old_colorop = colorop;
 	for_each_new_colorop_in_state(state, colorop, new_colorop_state, i) {
-		if (new_colorop_state->colorop == old_colorop &&
+		if (new_colorop_state->colorop == lut_colorop &&
 		    new_colorop_state->colorop->type == DRM_COLOROP_1D_LUT) {
-			colorop_state = new_colorop_state;
+			lut_state = new_colorop_state;
 			break;
 		}
 	}
 
-	if (colorop_state && !colorop_state->bypass && colorop->type == DRM_COLOROP_1D_LUT &&
-	    (BIT(colorop_state->curve_1d_type) & amdgpu_dm_supported_blnd_tfs)) {
-		drm_dbg(dev, "Blend LUT colorop with ID: %d\n", colorop->base.id);
+	if (tf_state && !tf_state->bypass) {
+		drm_dbg(dev, "Blend TF colorop with ID: %d\n", old_colorop->base.id);
+		tf->type = TF_TYPE_DISTRIBUTED_POINTS;
+		tf->tf = default_tf = amdgpu_colorop_tf_to_dc_tf(tf_state->curve_1d_type);
+		tf->sdr_ref_white_level = SDR_WHITE_LEVEL_INIT_VALUE;
+		dc_plane_state->cm.flags.bits.blend_enable = 1;
+		__set_input_tf_32(NULL, tf, blend_lut, blend_size);
+	}
+
+	if (lut_state && !lut_state->bypass) {
+		drm_dbg(dev, "Blend LUT colorop with ID: %d\n", lut_colorop->base.id);
 		tf->type = TF_TYPE_DISTRIBUTED_POINTS;
 		tf->tf = default_tf;
 		tf->sdr_ref_white_level = SDR_WHITE_LEVEL_INIT_VALUE;
-		blend_lut = __extract_blob_lut32(colorop_state->data, &blend_size);
+		dc_plane_state->cm.flags.bits.blend_enable = 1;
+		blend_lut = __extract_blob_lut32(lut_state->data, &blend_size);
 		blend_size = blend_lut != NULL ? blend_size : 0;
 
 		/* Custom LUT size must be the same as supported size */
-		if (blend_size == colorop->size)
+		if (blend_size == lut_colorop->size)
 			__set_input_tf_32(NULL, tf, blend_lut, blend_size);
 	}
 
@@ -1876,11 +1904,11 @@ amdgpu_dm_plane_set_color_properties(struct drm_plane_state *plane_state,
 	lut3d = __extract_blob_lut(dm_plane_state->lut3d, &lut3d_size);
 	lut3d_size = lut3d != NULL ? lut3d_size : 0;
 
-	amdgpu_dm_atomic_lut3d(lut3d, lut3d_size, &dc_plane_state->lut3d_func);
+	amdgpu_dm_atomic_lut3d(lut3d, lut3d_size, &dc_plane_state->cm);
 	ret = amdgpu_dm_atomic_shaper_lut(shaper_lut, false,
 					  amdgpu_tf_to_dc_tf(shaper_tf),
 					  shaper_size,
-					  &dc_plane_state->in_shaper_func);
+					  &dc_plane_state->cm);
 	if (ret) {
 		drm_dbg_kms(plane_state->plane->dev,
 			    "setting plane %d shaper LUT failed.\n",
@@ -1895,7 +1923,8 @@ amdgpu_dm_plane_set_color_properties(struct drm_plane_state *plane_state,
 
 	ret = amdgpu_dm_atomic_blend_lut(blend_lut, false,
 					 amdgpu_tf_to_dc_tf(blend_tf),
-					 blend_size, &dc_plane_state->blend_tf);
+					 blend_size, &dc_plane_state->cm);
+
 	if (ret) {
 		drm_dbg_kms(plane_state->plane->dev,
 			    "setting plane %d gamma lut failed.\n",
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.h
index e4f53b7bc7539..8dbbcb3ab1561 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.h
@@ -87,10 +87,10 @@ void __drm_3dlut32_to_dc_3dlut(const struct drm_color_lut32 *lut,
 				struct tetrahedral_params *params,
 				bool use_tetrahedral_9,
 				int bit_depth);
-struct dc_3dlut;
+struct dc_plane_cm;
 void amdgpu_dm_atomic_lut3d(const struct drm_color_lut *drm_lut3d,
 			     uint32_t drm_lut3d_size,
-			     struct dc_3dlut *lut);
+			     struct dc_plane_cm *cm);
 int __set_colorop_3dlut(const struct drm_color_lut32 *drm_lut3d,
 			uint32_t drm_lut3d_size,
 			struct dc_3dlut *lut);
@@ -105,12 +105,12 @@ int amdgpu_dm_atomic_shaper_lut(const struct drm_color_lut *shaper_lut,
 				bool has_rom,
 				enum dc_transfer_func_predefined tf,
 				uint32_t shaper_size,
-				struct dc_transfer_func *func_shaper);
+				struct dc_plane_cm *cm);
 int amdgpu_dm_atomic_blend_lut(const struct drm_color_lut *blend_lut,
 			       bool has_rom,
 			       enum dc_transfer_func_predefined tf,
 			       uint32_t blend_size,
-			       struct dc_transfer_func *func_blend);
+			       struct dc_plane_cm *cm);
 int __set_colorop_in_tf_1d_curve(struct dc_plane_state *dc_plane_state,
 				 struct drm_colorop_state *colorop_state);
 #endif
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c
index 48f5c431eaf91..056a76b88f438 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c
@@ -235,3 +235,4 @@ int amdgpu_dm_initialize_default_pipeline(struct drm_plane *plane, struct drm_pr
 
 	return amdgpu_dm_build_default_pipeline(dev, plane, hw_3d_lut, list);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_initialize_default_pipeline);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.c
new file mode 100644
index 0000000000000..5c5f9c3e6d778
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.c
@@ -0,0 +1,3715 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services_types.h"
+#include "dc.h"
+#include "dc/dc_dmub_srv.h"
+#include "dc/dc_edid_parser.h"
+#include "dc/dc_stat.h"
+#include "dc/dc_state.h"
+#include "dc/dc_stream.h"
+#include "dc/inc/core_types.h"
+#include "link_enc_cfg.h"
+#include "link/protocols/link_dpcd.h"
+#include "link_service_types.h"
+#include "link/protocols/link_dp_capability.h"
+#include "link/protocols/link_ddc.h"
+
+#include "amdgpu.h"
+#include "amdgpu_display.h"
+#include "amdgpu_dm.h"
+#include "amdgpu_dm_connector.h"
+#include "amdgpu_dm_kunit_helpers.h"
+#include "amdgpu_dm_plane.h"
+#include "amdgpu_dm_crtc.h"
+#include "amdgpu_dm_wb.h"
+#include "amdgpu_dm_mst_types.h"
+#if defined(CONFIG_DEBUG_FS)
+#include "amdgpu_dm_debugfs.h"
+#endif
+#include "amdgpu_dm_backlight.h"
+#include "amdgpu_dm_audio.h"
+#include "amdgpu_dm_irq.h"
+#include "amdgpu_dm_psr.h"
+#include "dm_helpers.h"
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_uapi.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_eld.h>
+#include <drm/drm_fixed.h>
+#include <drm/drm_mode.h>
+#include <drm/drm_probe_helper.h>
+#include <drm/drm_utils.h>
+#include <drm/display/drm_dp_mst_helper.h>
+#include <drm/display/drm_hdmi_helper.h>
+#include <drm/drm_privacy_screen_consumer.h>
+#include <drm/display/drm_hdcp_helper.h>
+
+#include <linux/backlight.h>
+
+#include <media/cec-notifier.h>
+
+#include "modules/inc/mod_freesync.h"
+#include "modules/inc/mod_power.h"
+
+#include "amdgpu_dm_trace.h"
+
+/* Encoder functions */
+
+static void amdgpu_dm_encoder_destroy(struct drm_encoder *encoder)
+{
+	drm_encoder_cleanup(encoder);
+	kfree(encoder);
+}
+
+static const struct drm_encoder_funcs amdgpu_dm_encoder_funcs = {
+	.destroy = amdgpu_dm_encoder_destroy,
+};
+
+static void dm_encoder_helper_disable(struct drm_encoder *encoder)
+{
+}
+
+static int dm_encoder_helper_atomic_check(struct drm_encoder *encoder,
+					  struct drm_crtc_state *crtc_state,
+					  struct drm_connector_state *conn_state)
+{
+	struct drm_atomic_commit *state = crtc_state->state;
+	struct drm_connector *connector = conn_state->connector;
+	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
+	struct dm_connector_state *dm_new_connector_state = to_dm_connector_state(conn_state);
+	const struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
+	struct drm_dp_mst_topology_mgr *mst_mgr;
+	struct drm_dp_mst_port *mst_port;
+	struct drm_dp_mst_topology_state *mst_state;
+	enum dc_color_depth color_depth;
+	int clock, bpp = 0;
+	bool is_y420 = false;
+
+	if ((connector->connector_type == DRM_MODE_CONNECTOR_eDP) ||
+	    (connector->connector_type == DRM_MODE_CONNECTOR_LVDS)) {
+		struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder);
+		struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode;
+		enum drm_mode_status result;
+
+		result = drm_crtc_helper_mode_valid_fixed(encoder->crtc, adjusted_mode, native_mode);
+		if (result != MODE_OK && dm_new_connector_state->scaling == RMX_OFF) {
+			drm_dbg_driver(encoder->dev,
+				       "mode %dx%d@%dHz is not native, enabling scaling\n",
+				       adjusted_mode->hdisplay, adjusted_mode->vdisplay,
+				       drm_mode_vrefresh(adjusted_mode));
+			dm_new_connector_state->scaling = RMX_ASPECT;
+		}
+		return 0;
+	}
+
+	if (!aconnector->mst_output_port)
+		return 0;
+
+	mst_port = aconnector->mst_output_port;
+	mst_mgr = &aconnector->mst_root->mst_mgr;
+
+	if (!crtc_state->connectors_changed && !crtc_state->mode_changed)
+		return 0;
+
+	mst_state = drm_atomic_get_mst_topology_state(state, mst_mgr);
+	if (IS_ERR(mst_state))
+		return PTR_ERR(mst_state);
+
+	mst_state->pbn_div.full = dm_mst_get_pbn_divider(aconnector->mst_root->dc_link);
+
+	if (!state->duplicated) {
+		int max_bpc = conn_state->max_requested_bpc;
+
+		is_y420 = drm_mode_is_420_also(&connector->display_info, adjusted_mode) &&
+			  aconnector->force_yuv420_output;
+		color_depth = amdgpu_dm_convert_color_depth_from_display_info(connector,
+								    is_y420,
+								    max_bpc);
+		bpp = amdgpu_dm_convert_dc_color_depth_into_bpc(color_depth) * 3;
+		clock = adjusted_mode->clock;
+		dm_new_connector_state->pbn = drm_dp_calc_pbn_mode(clock, bpp << 4);
+	}
+
+	dm_new_connector_state->vcpi_slots =
+		drm_dp_atomic_find_time_slots(state, mst_mgr, mst_port,
+					      dm_new_connector_state->pbn);
+	if (dm_new_connector_state->vcpi_slots < 0) {
+		drm_dbg_atomic(connector->dev, "failed finding vcpi slots: %d\n", (int)dm_new_connector_state->vcpi_slots);
+		return dm_new_connector_state->vcpi_slots;
+	}
+	return 0;
+}
+
+const struct drm_encoder_helper_funcs amdgpu_dm_encoder_helper_funcs = {
+	.disable = dm_encoder_helper_disable,
+	.atomic_check = dm_encoder_helper_atomic_check
+};
+
+int amdgpu_dm_get_encoder_crtc_mask(struct amdgpu_device *adev)
+{
+	switch (adev->mode_info.num_crtc) {
+	case 1:
+		return 0x1;
+	case 2:
+		return 0x3;
+	case 3:
+		return 0x7;
+	case 4:
+		return 0xf;
+	case 5:
+		return 0x1f;
+	case 6:
+	default:
+		return 0x3f;
+	}
+}
+EXPORT_IF_KUNIT(amdgpu_dm_get_encoder_crtc_mask);
+
+int amdgpu_dm_encoder_init(struct drm_device *dev,
+			   struct amdgpu_encoder *aencoder,
+			   uint32_t link_index)
+{
+	struct amdgpu_device *adev = drm_to_adev(dev);
+
+	int res = drm_encoder_init(dev,
+				   &aencoder->base,
+				   &amdgpu_dm_encoder_funcs,
+				   DRM_MODE_ENCODER_TMDS,
+				   NULL);
+
+	aencoder->base.possible_crtcs = amdgpu_dm_get_encoder_crtc_mask(adev);
+
+	if (!res)
+		aencoder->encoder_id = link_index;
+	else
+		aencoder->encoder_id = -1;
+
+	drm_encoder_helper_add(&aencoder->base, &amdgpu_dm_encoder_helper_funcs);
+
+	return res;
+}
+
+STATIC_IFN_KUNIT enum drm_mode_subconnector get_subconnector_type(struct dc_link *link)
+{
+	switch (link->dpcd_caps.dongle_type) {
+	case DISPLAY_DONGLE_NONE:
+		return DRM_MODE_SUBCONNECTOR_Native;
+	case DISPLAY_DONGLE_DP_VGA_CONVERTER:
+		return DRM_MODE_SUBCONNECTOR_VGA;
+	case DISPLAY_DONGLE_DP_DVI_CONVERTER:
+	case DISPLAY_DONGLE_DP_DVI_DONGLE:
+		return DRM_MODE_SUBCONNECTOR_DVID;
+	case DISPLAY_DONGLE_DP_HDMI_CONVERTER:
+	case DISPLAY_DONGLE_DP_HDMI_DONGLE:
+		return DRM_MODE_SUBCONNECTOR_HDMIA;
+	case DISPLAY_DONGLE_DP_HDMI_MISMATCHED_DONGLE:
+	default:
+		return DRM_MODE_SUBCONNECTOR_Unknown;
+	}
+}
+EXPORT_IF_KUNIT(get_subconnector_type);
+
+STATIC_IFN_KUNIT void update_subconnector_property(struct amdgpu_dm_connector *aconnector)
+{
+	struct dc_link *link = aconnector->dc_link;
+	struct drm_connector *connector = &aconnector->base;
+	enum drm_mode_subconnector subconnector = DRM_MODE_SUBCONNECTOR_Unknown;
+
+	if (connector->connector_type != DRM_MODE_CONNECTOR_DisplayPort)
+		return;
+
+	if (aconnector->dc_sink)
+		subconnector = get_subconnector_type(link);
+
+	drm_object_property_set_value(&connector->base,
+			connector->dev->mode_config.dp_subconnector_property,
+			subconnector);
+}
+EXPORT_IF_KUNIT(update_subconnector_property);
+
+static int amdgpu_dm_connector_get_modes(struct drm_connector *connector);
+
+STATIC_IFN_KUNIT void amdgpu_dm_fbc_init(struct drm_connector *connector)
+{
+	struct amdgpu_device *adev = drm_to_adev(connector->dev);
+	struct dm_compressor_info *compressor = &adev->dm.compressor;
+	struct amdgpu_dm_connector *aconn = to_amdgpu_dm_connector(connector);
+	struct drm_display_mode *mode;
+	unsigned long max_size = 0;
+
+	if (adev->dm.dc->fbc_compressor == NULL)
+		return;
+
+	if (aconn->dc_link->connector_signal != SIGNAL_TYPE_EDP)
+		return;
+
+	if (compressor->bo_ptr)
+		return;
+
+
+	list_for_each_entry(mode, &connector->modes, head) {
+		if (max_size < (unsigned long) mode->htotal * mode->vtotal)
+			max_size = (unsigned long) mode->htotal * mode->vtotal;
+	}
+
+	if (max_size) {
+		int r = amdgpu_bo_create_kernel(adev, max_size * 4, PAGE_SIZE,
+			    AMDGPU_GEM_DOMAIN_GTT, &compressor->bo_ptr,
+			    &compressor->gpu_addr, &compressor->cpu_addr);
+
+		if (r)
+			drm_err(adev_to_drm(adev), "DM: Failed to initialize FBC\n");
+		else {
+			adev->dm.dc->ctx->fbc_gpu_addr = compressor->gpu_addr;
+			drm_info(adev_to_drm(adev), "DM: FBC alloc %lu\n", max_size*4);
+		}
+
+	}
+
+}
+EXPORT_IF_KUNIT(amdgpu_dm_fbc_init);
+
+
+int amdgpu_dm_detect_mst_link_for_all_connectors(struct drm_device *dev)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_connector *connector;
+	struct drm_connector_list_iter iter;
+	int ret = 0;
+
+	drm_connector_list_iter_begin(dev, &iter);
+	drm_for_each_connector_iter(connector, &iter) {
+
+		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
+			continue;
+
+		aconnector = to_amdgpu_dm_connector(connector);
+		if (aconnector->dc_link->type == dc_connection_mst_branch &&
+		    aconnector->mst_mgr.aux) {
+			drm_dbg_kms(dev, "DM_MST: starting TM on aconnector: %p [id: %d]\n",
+					 aconnector,
+					 aconnector->base.base.id);
+
+			ret = drm_dp_mst_topology_mgr_set_mst(&aconnector->mst_mgr, true);
+			if (ret < 0) {
+				drm_err(dev, "DM_MST: Failed to start MST\n");
+				aconnector->dc_link->type =
+					dc_connection_single;
+				ret = dm_helpers_dp_mst_stop_top_mgr(aconnector->dc_link->ctx,
+								     aconnector->dc_link);
+				break;
+			}
+		}
+	}
+	drm_connector_list_iter_end(&iter);
+
+	return ret;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_detect_mst_link_for_all_connectors);
+
+static void hdmi_cec_unset_edid(struct amdgpu_dm_connector *aconnector)
+{
+	struct cec_notifier *n = aconnector->notifier;
+
+	if (!n)
+		return;
+
+	cec_notifier_phys_addr_invalidate(n);
+}
+
+void amdgpu_dm_hdmi_cec_set_edid(struct amdgpu_dm_connector *aconnector)
+{
+	struct drm_connector *connector = &aconnector->base;
+	struct cec_notifier *n = aconnector->notifier;
+
+	if (!n)
+		return;
+
+	cec_notifier_set_phys_addr(n,
+				   connector->display_info.source_physical_address);
+}
+
+void amdgpu_dm_s3_handle_hdmi_cec(struct drm_device *ddev, bool suspend)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_connector *connector;
+	struct drm_connector_list_iter conn_iter;
+
+	drm_connector_list_iter_begin(ddev, &conn_iter);
+	drm_for_each_connector_iter(connector, &conn_iter) {
+		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
+			continue;
+
+		aconnector = to_amdgpu_dm_connector(connector);
+		if (suspend)
+			hdmi_cec_unset_edid(aconnector);
+		else
+			amdgpu_dm_hdmi_cec_set_edid(aconnector);
+	}
+	drm_connector_list_iter_end(&conn_iter);
+}
+
+
+struct drm_connector *
+amdgpu_dm_find_first_crtc_matching_connector(struct drm_atomic_commit *state,
+					     struct drm_crtc *crtc)
+{
+	u32 i;
+	struct drm_connector_state *new_con_state;
+	struct drm_connector *connector;
+	struct drm_crtc *crtc_from_state;
+
+	for_each_new_connector_in_state(state, connector, new_con_state, i) {
+		crtc_from_state = new_con_state->crtc;
+
+		if (crtc_from_state == crtc)
+			return connector;
+	}
+
+	return NULL;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_find_first_crtc_matching_connector);
+
+STATIC_IFN_KUNIT void amdgpu_dm_set_panel_type(struct amdgpu_dm_connector *aconnector)
+{
+	struct drm_connector *connector = &aconnector->base;
+	struct drm_display_info *display_info = &connector->display_info;
+	struct dc_link *link = aconnector->dc_link;
+	struct amdgpu_device *adev;
+	enum dc_panel_type panel_type = PANEL_TYPE_NONE;
+
+	adev = drm_to_adev(connector->dev);
+
+	switch (display_info->amd_vsdb.panel_type) {
+	case AMD_VSDB_PANEL_TYPE_OLED:
+		panel_type = PANEL_TYPE_OLED;
+		break;
+	case AMD_VSDB_PANEL_TYPE_MINILED:
+		panel_type = PANEL_TYPE_MINILED;
+		break;
+	}
+
+	/* If VSDB didn't determine panel type, check DPCD ext caps */
+	if (panel_type == PANEL_TYPE_NONE) {
+		if (link->dpcd_sink_ext_caps.bits.miniled == 1)
+			panel_type = PANEL_TYPE_MINILED;
+		if (link->dpcd_sink_ext_caps.bits.oled == 1)
+			panel_type = PANEL_TYPE_OLED;
+	}
+
+	/* If VSDB and DPCD didn't determine panel type, check DID */
+	if (panel_type == PANEL_TYPE_NONE) {
+		if (display_info->panel_type == DRM_MODE_PANEL_TYPE_LCD)
+			panel_type = PANEL_TYPE_LCD;
+		else if (display_info->panel_type == DRM_MODE_PANEL_TYPE_OLED)
+			panel_type = PANEL_TYPE_OLED;
+	}
+
+	if (panel_type == PANEL_TYPE_NONE) {
+		struct drm_amd_vsdb_info *vsdb = &display_info->amd_vsdb;
+		u32 lum1_max = vsdb->luminance_range1.max_luminance;
+		u32 lum2_max = vsdb->luminance_range2.max_luminance;
+
+		if (vsdb->version && link->local_sink &&
+		    link->local_sink->edid_caps.manufacturer_id ==
+		    DDC_MANUFACTURERNAME_SAMSUNG &&
+		    lum1_max >= ((lum2_max * 3) / 2))
+			panel_type = PANEL_TYPE_MINILED;
+	}
+
+	if (panel_type != PANEL_TYPE_NONE)
+		link->panel_type = panel_type;
+	else
+		link->panel_type = PANEL_TYPE_LCD;
+
+	if (link->panel_type == PANEL_TYPE_OLED)
+		drm_object_property_set_value(&connector->base,
+		    adev_to_drm(adev)->mode_config.panel_type_property,
+		    DRM_MODE_PANEL_TYPE_OLED);
+	else if (link->panel_type == PANEL_TYPE_LCD)
+		drm_object_property_set_value(&connector->base,
+		    adev_to_drm(adev)->mode_config.panel_type_property,
+		    DRM_MODE_PANEL_TYPE_LCD);
+	else
+		drm_object_property_set_value(&connector->base,
+		    adev_to_drm(adev)->mode_config.panel_type_property,
+		    DRM_MODE_PANEL_TYPE_UNKNOWN);
+
+	drm_dbg_kms(aconnector->base.dev, "Panel type: %d\n", link->panel_type);
+}
+EXPORT_IF_KUNIT(amdgpu_dm_set_panel_type);
+
+STATIC_IFN_KUNIT void amdgpu_dm_update_cacp_caps(struct amdgpu_dm_connector *aconnector)
+{
+	struct amdgpu_device *adev = drm_to_adev(aconnector->base.dev);
+	struct dc_link *link = aconnector->dc_link;
+
+	link->panel_config.cacp.cacp_supported = true;
+
+	if (amdgpu_ip_version(adev, DCE_HWIP, 0) < IP_VERSION(3, 1, 4) ||
+	    amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(3, 1, 6)) {
+		link->panel_config.cacp.cacp_supported = false;
+		return;
+	}
+
+	if (link->connector_signal != SIGNAL_TYPE_EDP &&
+	    link->connector_signal != SIGNAL_TYPE_LVDS) {
+		link->panel_config.cacp.cacp_supported = false;
+		return;
+	}
+
+	if (link->panel_type == PANEL_TYPE_LCD)
+		link->panel_config.cacp.cacp_supported = false;
+
+	drm_dbg_kms(aconnector->base.dev, "cacp_supported: %d\n",
+		    link->panel_config.cacp.cacp_supported);
+}
+EXPORT_IF_KUNIT(amdgpu_dm_update_cacp_caps);
+
+DEFINE_FREE(sink_release, struct dc_sink *, if (_T) dc_sink_release(_T))
+
+void amdgpu_dm_update_connector_after_detect(
+		struct amdgpu_dm_connector *aconnector)
+{
+	struct drm_connector *connector = &aconnector->base;
+	struct dc_sink *sink __free(sink_release) = NULL;
+	struct drm_device *dev = connector->dev;
+	struct amdgpu_device *adev = drm_to_adev(dev);
+	int inst;
+
+	/* MST handled by drm_mst framework */
+	if (aconnector->mst_mgr.mst_state)
+		return;
+
+	sink = aconnector->dc_link->local_sink;
+	if (sink)
+		dc_sink_retain(sink);
+
+	/*
+	 * Edid mgmt connector gets first update only in mode_valid hook and then
+	 * the connector sink is set to either fake or physical sink depends on link status.
+	 * Skip if already done during boot.
+	 */
+	if (aconnector->base.force != DRM_FORCE_UNSPECIFIED
+			&& aconnector->dc_em_sink) {
+
+		/*
+		 * For S3 resume with headless use eml_sink to fake stream
+		 * because on resume connector->sink is set to NULL
+		 */
+		guard(mutex)(&dev->mode_config.mutex);
+
+		if (sink) {
+			if (aconnector->dc_sink) {
+				amdgpu_dm_update_freesync_caps(connector, NULL, true);
+				/*
+				 * retain and release below are used to
+				 * bump up refcount for sink because the link doesn't point
+				 * to it anymore after disconnect, so on next crtc to connector
+				 * reshuffle by UMD we will get into unwanted dc_sink release
+				 */
+				dc_sink_release(aconnector->dc_sink);
+			}
+			aconnector->dc_sink = sink;
+			dc_sink_retain(aconnector->dc_sink);
+			amdgpu_dm_update_freesync_caps(connector,
+					aconnector->drm_edid, true);
+		} else {
+			amdgpu_dm_update_freesync_caps(connector, NULL, true);
+			if (!aconnector->dc_sink) {
+				aconnector->dc_sink = aconnector->dc_em_sink;
+				dc_sink_retain(aconnector->dc_sink);
+			}
+		}
+
+		return;
+	}
+
+	/*
+	 * TODO: temporary guard to look for proper fix
+	 * if this sink is MST sink, we should not do anything
+	 */
+	if (sink && sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT_MST)
+		return;
+
+	if (aconnector->dc_sink == sink) {
+		/*
+		 * We got a DP short pulse (Link Loss, DP CTS, etc...).
+		 * Do nothing!!
+		 */
+		drm_dbg_kms(dev, "DCHPD: connector_id=%d: dc_sink didn't change.\n",
+				 aconnector->connector_id);
+		return;
+	}
+
+	drm_dbg_kms(dev, "DCHPD: connector_id=%d: Old sink=%p New sink=%p\n",
+		    aconnector->connector_id, aconnector->dc_sink, sink);
+
+	/* When polling, DRM has already locked the mutex for us. */
+	if (!drm_kms_helper_is_poll_worker())
+		mutex_lock(&dev->mode_config.mutex);
+
+	/*
+	 * 1. Update status of the drm connector
+	 * 2. Send an event and let userspace tell us what to do
+	 */
+	if (sink) {
+		/*
+		 * TODO: check if we still need the S3 mode update workaround.
+		 * If yes, put it here.
+		 */
+		if (aconnector->dc_sink) {
+			amdgpu_dm_update_freesync_caps(connector, NULL, true);
+			dc_sink_release(aconnector->dc_sink);
+		}
+
+		aconnector->dc_sink = sink;
+		dc_sink_retain(aconnector->dc_sink);
+		drm_edid_free(aconnector->drm_edid);
+		aconnector->drm_edid = NULL;
+		if (sink->dc_edid.length == 0) {
+			hdmi_cec_unset_edid(aconnector);
+			if (aconnector->dc_link->aux_mode)
+				drm_dp_cec_unset_edid(&aconnector->dm_dp_aux.aux);
+		} else {
+			const struct edid *edid = (const struct edid *)sink->dc_edid.raw_edid;
+
+			aconnector->drm_edid = drm_edid_alloc(edid, sink->dc_edid.length);
+			drm_edid_connector_update(connector, aconnector->drm_edid);
+
+			amdgpu_dm_hdmi_cec_set_edid(aconnector);
+			if (aconnector->dc_link->aux_mode)
+				drm_dp_cec_attach(&aconnector->dm_dp_aux.aux,
+						  connector->display_info.source_physical_address);
+		}
+
+		if (!aconnector->timing_requested) {
+			aconnector->timing_requested =
+				kzalloc_obj(struct dc_crtc_timing);
+			if (!aconnector->timing_requested)
+				drm_err(dev,
+					"failed to create aconnector->requested_timing\n");
+		}
+
+		amdgpu_dm_update_freesync_caps(connector, aconnector->drm_edid, true);
+		amdgpu_dm_update_connector_ext_caps(aconnector);
+		amdgpu_dm_set_panel_type(aconnector);
+		amdgpu_dm_update_cacp_caps(aconnector);
+
+		if (aconnector->hdmi_comp_auto) {
+			if (sink->sink_signal != SIGNAL_TYPE_HDMI_FRL)
+				sink->sink_signal = SIGNAL_TYPE_HDMI_FRL;
+		}
+	} else {
+		hdmi_cec_unset_edid(aconnector);
+		drm_dp_cec_unset_edid(&aconnector->dm_dp_aux.aux);
+		amdgpu_dm_update_freesync_caps(connector, NULL, true);
+		aconnector->num_modes = 0;
+		dc_sink_release(aconnector->dc_sink);
+		aconnector->dc_sink = NULL;
+		drm_edid_free(aconnector->drm_edid);
+		aconnector->drm_edid = NULL;
+		kfree(aconnector->timing_requested);
+		aconnector->timing_requested = NULL;
+		/* Set CP to DESIRED if it was ENABLED, so we can re-enable it again on hotplug */
+		if (connector->state->content_protection == DRM_MODE_CONTENT_PROTECTION_ENABLED)
+			connector->state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED;
+
+		mutex_lock(&adev->dm.audio_lock);
+		inst = aconnector->audio_inst;
+		aconnector->audio_inst = -1;
+		mutex_unlock(&adev->dm.audio_lock);
+		if (inst != -1)
+			amdgpu_dm_audio_eld_notify(adev, inst);
+	}
+
+	update_subconnector_property(aconnector);
+
+	/* When polling, the mutex will be unlocked for us by DRM. */
+	if (!drm_kms_helper_is_poll_worker())
+		mutex_unlock(&dev->mode_config.mutex);
+}
+
+enum dc_color_depth
+amdgpu_dm_convert_color_depth_from_display_info(const struct drm_connector *connector,
+						bool is_y420, int requested_bpc)
+{
+	u8 bpc;
+
+	if (is_y420) {
+		bpc = 8;
+
+		/* Cap display bpc based on HDMI 2.0 HF-VSDB */
+		if (connector->display_info.hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_48)
+			bpc = 16;
+		else if (connector->display_info.hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_36)
+			bpc = 12;
+		else if (connector->display_info.hdmi.y420_dc_modes & DRM_EDID_YCBCR420_DC_30)
+			bpc = 10;
+	} else {
+		bpc = (uint8_t)connector->display_info.bpc;
+		/* Assume 8 bpc by default if no bpc is specified. */
+		bpc = bpc ? bpc : 8;
+	}
+
+	if (requested_bpc > 0) {
+		/*
+		 * Cap display bpc based on the user requested value.
+		 *
+		 * The value for state->max_bpc may not correctly updated
+		 * depending on when the connector gets added to the state
+		 * or if this was called outside of atomic check, so it
+		 * can't be used directly.
+		 */
+		bpc = min_t(u8, bpc, requested_bpc);
+
+		/* Round down to the nearest even number. */
+		bpc = bpc - (bpc & 1);
+	}
+
+	switch (bpc) {
+	case 0:
+		/*
+		 * Temporary Work around, DRM doesn't parse color depth for
+		 * EDID revision before 1.4
+		 * TODO: Fix edid parsing
+		 */
+		return COLOR_DEPTH_888;
+	case 6:
+		return COLOR_DEPTH_666;
+	case 8:
+		return COLOR_DEPTH_888;
+	case 10:
+		return COLOR_DEPTH_101010;
+	case 12:
+		return COLOR_DEPTH_121212;
+	case 14:
+		return COLOR_DEPTH_141414;
+	case 16:
+		return COLOR_DEPTH_161616;
+	default:
+		return COLOR_DEPTH_UNDEFINED;
+	}
+}
+EXPORT_IF_KUNIT(amdgpu_dm_convert_color_depth_from_display_info);
+
+STATIC_IFN_KUNIT enum dc_aspect_ratio
+get_aspect_ratio(const struct drm_display_mode *mode_in)
+{
+	/* 1-1 mapping, since both enums follow the HDMI spec. */
+	return (enum dc_aspect_ratio) mode_in->picture_aspect_ratio;
+}
+EXPORT_IF_KUNIT(get_aspect_ratio);
+
+enum dc_color_space
+amdgpu_dm_get_output_color_space(const struct dc_crtc_timing *dc_crtc_timing,
+				 const struct drm_connector_state *connector_state)
+{
+	enum dc_color_space color_space = COLOR_SPACE_SRGB;
+
+	switch (connector_state->colorspace) {
+	case DRM_MODE_COLORIMETRY_BT601_YCC:
+		if (dc_crtc_timing->flags.Y_ONLY)
+			color_space = COLOR_SPACE_YCBCR601_LIMITED;
+		else
+			color_space = COLOR_SPACE_YCBCR601;
+		break;
+	case DRM_MODE_COLORIMETRY_BT709_YCC:
+		if (dc_crtc_timing->flags.Y_ONLY)
+			color_space = COLOR_SPACE_YCBCR709_LIMITED;
+		else
+			color_space = COLOR_SPACE_YCBCR709;
+		break;
+	case DRM_MODE_COLORIMETRY_OPRGB:
+		color_space = COLOR_SPACE_ADOBERGB;
+		break;
+	case DRM_MODE_COLORIMETRY_BT2020_RGB:
+	case DRM_MODE_COLORIMETRY_BT2020_YCC:
+		if (dc_crtc_timing->pixel_encoding == PIXEL_ENCODING_RGB)
+			color_space = COLOR_SPACE_2020_RGB_FULLRANGE;
+		else
+			color_space = COLOR_SPACE_2020_YCBCR_LIMITED;
+		break;
+	case DRM_MODE_COLORIMETRY_DEFAULT: /* ITU601 */
+	default:
+		if (dc_crtc_timing->pixel_encoding == PIXEL_ENCODING_RGB) {
+			color_space = COLOR_SPACE_SRGB;
+			if (connector_state->hdmi.broadcast_rgb == DRM_HDMI_BROADCAST_RGB_LIMITED)
+				color_space = COLOR_SPACE_SRGB_LIMITED;
+		/*
+		 * 27030khz is the separation point between HDTV and SDTV
+		 * according to HDMI spec, we use YCbCr709 and YCbCr601
+		 * respectively
+		 */
+		} else if (dc_crtc_timing->pix_clk_100hz > 270300) {
+			if (dc_crtc_timing->flags.Y_ONLY)
+				color_space =
+					COLOR_SPACE_YCBCR709_LIMITED;
+			else
+				color_space = COLOR_SPACE_YCBCR709;
+		} else {
+			if (dc_crtc_timing->flags.Y_ONLY)
+				color_space =
+					COLOR_SPACE_YCBCR601_LIMITED;
+			else
+				color_space = COLOR_SPACE_YCBCR601;
+		}
+		break;
+	}
+
+	return color_space;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_get_output_color_space);
+
+STATIC_IFN_KUNIT enum display_content_type
+get_output_content_type(const struct drm_connector_state *connector_state)
+{
+	switch (connector_state->content_type) {
+	default:
+	case DRM_MODE_CONTENT_TYPE_NO_DATA:
+		return DISPLAY_CONTENT_TYPE_NO_DATA;
+	case DRM_MODE_CONTENT_TYPE_GRAPHICS:
+		return DISPLAY_CONTENT_TYPE_GRAPHICS;
+	case DRM_MODE_CONTENT_TYPE_PHOTO:
+		return DISPLAY_CONTENT_TYPE_PHOTO;
+	case DRM_MODE_CONTENT_TYPE_CINEMA:
+		return DISPLAY_CONTENT_TYPE_CINEMA;
+	case DRM_MODE_CONTENT_TYPE_GAME:
+		return DISPLAY_CONTENT_TYPE_GAME;
+	}
+}
+EXPORT_IF_KUNIT(get_output_content_type);
+
+STATIC_IFN_KUNIT bool adjust_colour_depth_from_display_info(
+	struct dc_crtc_timing *timing_out,
+	const struct drm_display_info *info)
+{
+	enum dc_color_depth depth = timing_out->display_color_depth;
+	int normalized_clk;
+
+	do {
+		normalized_clk = timing_out->pix_clk_100hz / 10;
+		/* YCbCr 4:2:0 requires additional adjustment of 1/2 */
+		if (timing_out->pixel_encoding == PIXEL_ENCODING_YCBCR420)
+			normalized_clk /= 2;
+		/* Adjusting pix clock following on HDMI spec based on colour depth */
+		switch (depth) {
+		case COLOR_DEPTH_888:
+			break;
+		case COLOR_DEPTH_101010:
+			normalized_clk = (normalized_clk * 30) / 24;
+			break;
+		case COLOR_DEPTH_121212:
+			normalized_clk = (normalized_clk * 36) / 24;
+			break;
+		case COLOR_DEPTH_161616:
+			normalized_clk = (normalized_clk * 48) / 24;
+			break;
+		default:
+			/* The above depths are the only ones valid for HDMI. */
+			return false;
+		}
+		if (normalized_clk <= info->max_tmds_clock) {
+			timing_out->display_color_depth = depth;
+			return true;
+		}
+	} while (--depth > COLOR_DEPTH_666);
+	return false;
+}
+EXPORT_IF_KUNIT(adjust_colour_depth_from_display_info);
+
+STATIC_IFN_KUNIT void fill_stream_properties_from_drm_display_mode(
+	struct dc_stream_state *stream,
+	const struct drm_display_mode *mode_in,
+	const struct drm_connector *connector,
+	const struct drm_connector_state *connector_state,
+	const struct dc_stream_state *old_stream,
+	int requested_bpc)
+{
+	bool is_dp_or_hdmi = dc_is_hdmi_signal(stream->signal) || dc_is_dp_signal(stream->signal);
+	struct dc_crtc_timing *timing_out = &stream->timing;
+	const struct drm_display_info *info = &connector->display_info;
+	struct amdgpu_dm_connector *aconnector = NULL;
+	struct hdmi_vendor_infoframe hv_frame;
+	struct hdmi_avi_infoframe avi_frame;
+	bool want_420;
+	bool want_422;
+	ssize_t err;
+
+	if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK)
+		aconnector = to_amdgpu_dm_connector(connector);
+
+	memset(&hv_frame, 0, sizeof(hv_frame));
+	memset(&avi_frame, 0, sizeof(avi_frame));
+
+	timing_out->h_border_left = 0;
+	timing_out->h_border_right = 0;
+	timing_out->v_border_top = 0;
+	timing_out->v_border_bottom = 0;
+
+	want_420 = (aconnector && aconnector->force_yuv_pixel_format == PIXEL_ENCODING_YCBCR420) ||
+		   (connector_state->color_format == DRM_CONNECTOR_COLOR_FORMAT_YCBCR420);
+	want_422 = (aconnector && aconnector->force_yuv_pixel_format == PIXEL_ENCODING_YCBCR422) ||
+		   (connector_state->color_format == DRM_CONNECTOR_COLOR_FORMAT_YCBCR422);
+
+	if (drm_mode_is_420_only(info, mode_in) &&
+	    (want_420 || connector_state->color_format == DRM_CONNECTOR_COLOR_FORMAT_AUTO)) {
+		timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR420;
+	} else if (drm_mode_is_420_also(info, mode_in) && want_420) {
+		timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR420;
+	} else if ((info->color_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422)) &&
+		   want_422 && is_dp_or_hdmi) {
+		timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR422;
+	} else if (connector_state->color_format == DRM_CONNECTOR_COLOR_FORMAT_YCBCR444 &&
+		   (info->color_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444)) &&
+		   is_dp_or_hdmi) {
+		timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR444;
+	} else if (connector_state->color_format == DRM_CONNECTOR_COLOR_FORMAT_RGB444 ||
+		   connector_state->color_format == DRM_CONNECTOR_COLOR_FORMAT_AUTO) {
+		timing_out->pixel_encoding = PIXEL_ENCODING_RGB;
+	} else {
+		/*
+		 * If a format was explicitly requested but the requested format
+		 * can't be satisfied, set it to an invalid value so that an
+		 * error bubbles up to userspace. This way, userspace knows it
+		 * needs to make a better choice.
+		 */
+		if (connector_state->color_format != DRM_CONNECTOR_COLOR_FORMAT_AUTO)
+			timing_out->pixel_encoding = PIXEL_ENCODING_UNDEFINED;
+		else if (drm_mode_is_420_only(info, mode_in))
+			timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR420;
+		else
+			timing_out->pixel_encoding = PIXEL_ENCODING_RGB;
+	}
+
+	timing_out->timing_3d_format = TIMING_3D_FORMAT_NONE;
+	timing_out->display_color_depth = amdgpu_dm_convert_color_depth_from_display_info(
+		connector,
+		(timing_out->pixel_encoding == PIXEL_ENCODING_YCBCR420),
+		requested_bpc);
+	timing_out->scan_type = SCANNING_TYPE_NODATA;
+	timing_out->hdmi_vic = 0;
+
+	if (old_stream) {
+		timing_out->vic = old_stream->timing.vic;
+		timing_out->flags.HSYNC_POSITIVE_POLARITY = old_stream->timing.flags.HSYNC_POSITIVE_POLARITY;
+		timing_out->flags.VSYNC_POSITIVE_POLARITY = old_stream->timing.flags.VSYNC_POSITIVE_POLARITY;
+	} else {
+		timing_out->vic = drm_match_cea_mode(mode_in);
+		if (mode_in->flags & DRM_MODE_FLAG_PHSYNC)
+			timing_out->flags.HSYNC_POSITIVE_POLARITY = 1;
+		if (mode_in->flags & DRM_MODE_FLAG_PVSYNC)
+			timing_out->flags.VSYNC_POSITIVE_POLARITY = 1;
+	}
+
+	if (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A ||
+		stream->signal == SIGNAL_TYPE_HDMI_FRL) {
+		err = drm_hdmi_avi_infoframe_from_display_mode(&avi_frame,
+							       (struct drm_connector *)connector,
+							       mode_in);
+		if (err < 0)
+			drm_warn_once(connector->dev, "Failed to setup avi infoframe on connector %s: %zd\n",
+				      connector->name, err);
+		timing_out->vic = avi_frame.video_code;
+		err = drm_hdmi_vendor_infoframe_from_display_mode(&hv_frame,
+								  (struct drm_connector *)connector,
+								  mode_in);
+		if (err < 0)
+			drm_warn_once(connector->dev, "Failed to setup vendor infoframe on connector %s: %zd\n",
+				      connector->name, err);
+		timing_out->hdmi_vic = hv_frame.vic;
+	}
+
+	if (aconnector && amdgpu_dm_is_freesync_video_mode(mode_in, aconnector)) {
+		timing_out->h_addressable = mode_in->hdisplay;
+		timing_out->h_total = mode_in->htotal;
+		timing_out->h_sync_width = mode_in->hsync_end - mode_in->hsync_start;
+		timing_out->h_front_porch = mode_in->hsync_start - mode_in->hdisplay;
+		timing_out->v_total = mode_in->vtotal;
+		timing_out->v_addressable = mode_in->vdisplay;
+		timing_out->v_front_porch = mode_in->vsync_start - mode_in->vdisplay;
+		timing_out->v_sync_width = mode_in->vsync_end - mode_in->vsync_start;
+		timing_out->pix_clk_100hz = mode_in->clock * 10;
+	} else {
+		timing_out->h_addressable = mode_in->crtc_hdisplay;
+		timing_out->h_total = mode_in->crtc_htotal;
+		timing_out->h_sync_width = mode_in->crtc_hsync_end - mode_in->crtc_hsync_start;
+		timing_out->h_front_porch = mode_in->crtc_hsync_start - mode_in->crtc_hdisplay;
+		timing_out->v_total = mode_in->crtc_vtotal;
+		timing_out->v_addressable = mode_in->crtc_vdisplay;
+		timing_out->v_front_porch = mode_in->crtc_vsync_start - mode_in->crtc_vdisplay;
+		timing_out->v_sync_width = mode_in->crtc_vsync_end - mode_in->crtc_vsync_start;
+		timing_out->pix_clk_100hz = mode_in->crtc_clock * 10;
+	}
+
+	timing_out->aspect_ratio = get_aspect_ratio(mode_in);
+
+	stream->out_transfer_func.type = TF_TYPE_PREDEFINED;
+	stream->out_transfer_func.tf = TRANSFER_FUNCTION_SRGB;
+	if (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A) {
+		if (!adjust_colour_depth_from_display_info(timing_out, info) &&
+		    drm_mode_is_420_also(info, mode_in) &&
+		    timing_out->pixel_encoding != PIXEL_ENCODING_YCBCR420) {
+			timing_out->pixel_encoding = PIXEL_ENCODING_YCBCR420;
+			adjust_colour_depth_from_display_info(timing_out, info);
+		}
+	}
+
+	stream->output_color_space = amdgpu_dm_get_output_color_space(timing_out, connector_state);
+	stream->content_type = get_output_content_type(connector_state);
+}
+EXPORT_IF_KUNIT(fill_stream_properties_from_drm_display_mode);
+
+STATIC_IFN_KUNIT void
+copy_crtc_timing_for_drm_display_mode(const struct drm_display_mode *src_mode,
+				      struct drm_display_mode *dst_mode)
+{
+	dst_mode->crtc_hdisplay = src_mode->crtc_hdisplay;
+	dst_mode->crtc_vdisplay = src_mode->crtc_vdisplay;
+	dst_mode->crtc_clock = src_mode->crtc_clock;
+	dst_mode->crtc_hblank_start = src_mode->crtc_hblank_start;
+	dst_mode->crtc_hblank_end = src_mode->crtc_hblank_end;
+	dst_mode->crtc_hsync_start =  src_mode->crtc_hsync_start;
+	dst_mode->crtc_hsync_end = src_mode->crtc_hsync_end;
+	dst_mode->crtc_htotal = src_mode->crtc_htotal;
+	dst_mode->crtc_hskew = src_mode->crtc_hskew;
+	dst_mode->crtc_vblank_start = src_mode->crtc_vblank_start;
+	dst_mode->crtc_vblank_end = src_mode->crtc_vblank_end;
+	dst_mode->crtc_vsync_start = src_mode->crtc_vsync_start;
+	dst_mode->crtc_vsync_end = src_mode->crtc_vsync_end;
+	dst_mode->crtc_vtotal = src_mode->crtc_vtotal;
+}
+EXPORT_IF_KUNIT(copy_crtc_timing_for_drm_display_mode);
+
+STATIC_IFN_KUNIT void
+decide_crtc_timing_for_drm_display_mode(struct drm_display_mode *drm_mode,
+					const struct drm_display_mode *native_mode,
+					bool scale_enabled)
+{
+	if (scale_enabled || (
+	    native_mode->clock == drm_mode->clock &&
+	    native_mode->htotal == drm_mode->htotal &&
+	    native_mode->vtotal == drm_mode->vtotal)) {
+		if (native_mode->crtc_clock)
+			copy_crtc_timing_for_drm_display_mode(native_mode, drm_mode);
+	} else {
+		/* no scaling nor amdgpu inserted, no need to patch */
+	}
+}
+EXPORT_IF_KUNIT(decide_crtc_timing_for_drm_display_mode);
+
+static struct dc_sink *
+create_fake_sink(struct drm_device *dev, struct dc_link *link)
+{
+	struct dc_sink_init_data sink_init_data = { 0 };
+	struct dc_sink *sink = NULL;
+
+	sink_init_data.link = link;
+	sink_init_data.sink_signal = link->connector_signal;
+
+	sink = dc_sink_create(&sink_init_data);
+	if (!sink) {
+		drm_err(dev, "Failed to create sink!\n");
+		return NULL;
+	}
+	sink->sink_signal = SIGNAL_TYPE_VIRTUAL;
+
+	return sink;
+}
+
+/**
+ * DOC: FreeSync Video
+ *
+ * When a userspace application wants to play a video, the content follows a
+ * standard format definition that usually specifies the FPS for that format.
+ * The below list illustrates some video format and the expected FPS,
+ * respectively:
+ *
+ * - TV/NTSC (23.976 FPS)
+ * - Cinema (24 FPS)
+ * - TV/PAL (25 FPS)
+ * - TV/NTSC (29.97 FPS)
+ * - TV/NTSC (30 FPS)
+ * - Cinema HFR (48 FPS)
+ * - TV/PAL (50 FPS)
+ * - Commonly used (60 FPS)
+ * - Multiples of 24 (48,72,96 FPS)
+ *
+ * The list of standards video format is not huge and can be added to the
+ * connector modeset list beforehand. With that, userspace can leverage
+ * FreeSync to extends the front porch in order to attain the target refresh
+ * rate. Such a switch will happen seamlessly, without screen blanking or
+ * reprogramming of the output in any other way. If the userspace requests a
+ * modesetting change compatible with FreeSync modes that only differ in the
+ * refresh rate, DC will skip the full update and avoid blink during the
+ * transition. For example, the video player can change the modesetting from
+ * 60Hz to 30Hz for playing TV/NTSC content when it goes full screen without
+ * causing any display blink. This same concept can be applied to a mode
+ * setting change.
+ */
+struct drm_display_mode *
+amdgpu_dm_get_highest_refresh_rate_mode(struct amdgpu_dm_connector *aconnector,
+		bool use_probed_modes)
+{
+	struct drm_display_mode *m, *m_pref = NULL;
+	u16 current_refresh, highest_refresh;
+	struct list_head *list_head = use_probed_modes ?
+		&aconnector->base.probed_modes :
+		&aconnector->base.modes;
+
+	if (aconnector->base.connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
+		return NULL;
+
+	if (aconnector->freesync_vid_base.clock != 0)
+		return &aconnector->freesync_vid_base;
+
+	/* Find the preferred mode */
+	list_for_each_entry(m, list_head, head) {
+		if (m->type & DRM_MODE_TYPE_PREFERRED) {
+			m_pref = m;
+			break;
+		}
+	}
+
+	if (!m_pref) {
+		/* Probably an EDID with no preferred mode. Fallback to first entry */
+		m_pref = list_first_entry_or_null(
+				&aconnector->base.modes, struct drm_display_mode, head);
+		if (!m_pref) {
+			drm_dbg_driver(aconnector->base.dev, "No preferred mode found in EDID\n");
+			return NULL;
+		}
+	}
+
+	highest_refresh = drm_mode_vrefresh(m_pref);
+
+	/*
+	 * Find the mode with highest refresh rate with same resolution.
+	 * For some monitors, preferred mode is not the mode with highest
+	 * supported refresh rate.
+	 */
+	list_for_each_entry(m, list_head, head) {
+		current_refresh  = drm_mode_vrefresh(m);
+
+		if (m->hdisplay == m_pref->hdisplay &&
+		    m->vdisplay == m_pref->vdisplay &&
+		    highest_refresh < current_refresh) {
+			highest_refresh = current_refresh;
+			m_pref = m;
+		}
+	}
+
+	drm_mode_copy(&aconnector->freesync_vid_base, m_pref);
+	return m_pref;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_get_highest_refresh_rate_mode);
+
+bool amdgpu_dm_is_freesync_video_mode(const struct drm_display_mode *mode,
+		struct amdgpu_dm_connector *aconnector)
+{
+	struct drm_display_mode *high_mode;
+	int timing_diff;
+
+	high_mode = amdgpu_dm_get_highest_refresh_rate_mode(aconnector, false);
+	if (!high_mode || !mode)
+		return false;
+
+	timing_diff = high_mode->vtotal - mode->vtotal;
+
+	if (high_mode->clock == 0 || high_mode->clock != mode->clock ||
+	    high_mode->hdisplay != mode->hdisplay ||
+	    high_mode->vdisplay != mode->vdisplay ||
+	    high_mode->hsync_start != mode->hsync_start ||
+	    high_mode->hsync_end != mode->hsync_end ||
+	    high_mode->htotal != mode->htotal ||
+	    high_mode->hskew != mode->hskew ||
+	    high_mode->vscan != mode->vscan ||
+	    high_mode->vsync_start - mode->vsync_start != timing_diff ||
+	    high_mode->vsync_end - mode->vsync_end != timing_diff)
+		return false;
+	else
+		return true;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_is_freesync_video_mode);
+
+#if defined(CONFIG_DRM_AMD_DC_FP)
+static void update_dsc_caps(struct amdgpu_dm_connector *aconnector,
+			    struct dc_sink *sink, struct dc_stream_state *stream,
+			    struct dsc_dec_dpcd_caps *dsc_caps)
+{
+	stream->timing.flags.DSC = 0;
+	dsc_caps->is_dsc_supported = false;
+
+	if (aconnector->dc_link && (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT ||
+	    sink->sink_signal == SIGNAL_TYPE_EDP)) {
+		if (sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_NONE)
+			dc_dsc_parse_dsc_dpcd(aconnector->dc_link->ctx->dc,
+				aconnector->dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.raw,
+				aconnector->dc_link->dpcd_caps.dsc_caps.dsc_branch_decoder_caps.raw,
+				dsc_caps);
+		else if (sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_DP_HDMI_CONVERTER) {
+			if (aconnector->dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_PASSTHROUGH_SUPPORT &&
+					!aconnector->dsc_settings.dsc_force_disable_passthrough &&
+					aconnector->dc_link->dpcd_caps.dongle_caps.dp_hdmi_frl_max_link_bw_in_kbps > 0 &&
+					sink->edid_caps.frl_dsc_support &&
+					sink->edid_caps.max_frl_rate > 0 &&
+					sink->edid_caps.frl_dsc_max_frl_rate > 0)
+				dc_dsc_parse_dsc_edid(aconnector->dc_link->ctx->dc, &sink->edid_caps, dsc_caps);
+			else
+				dc_dsc_parse_dsc_dpcd(aconnector->dc_link->ctx->dc,
+				      aconnector->dc_link->dpcd_caps.dsc_caps.dsc_basic_caps.raw,
+				      aconnector->dc_link->dpcd_caps.dsc_caps.dsc_branch_decoder_caps.raw,
+				      dsc_caps);
+		}
+	} else if (aconnector->dc_link && sink->sink_signal == SIGNAL_TYPE_HDMI_FRL) {
+		if (sink->edid_caps.frl_dsc_support &&
+				sink->edid_caps.max_frl_rate > 0 &&
+				sink->edid_caps.frl_dsc_max_frl_rate > 0)
+			dc_dsc_parse_dsc_edid(aconnector->dc_link->ctx->dc, &sink->edid_caps, dsc_caps);
+	}
+}
+
+static void apply_dsc_policy_for_edp(struct amdgpu_dm_connector *aconnector,
+				    struct dc_sink *sink, struct dc_stream_state *stream,
+				    struct dsc_dec_dpcd_caps *dsc_caps,
+				    uint32_t max_dsc_target_bpp_limit_override)
+{
+	const struct dc_link_settings *verified_link_cap = NULL;
+	u32 link_bw_in_kbps;
+	u32 edp_min_bpp_x16, edp_max_bpp_x16;
+	struct dc *dc = sink->ctx->dc;
+	struct dc_dsc_bw_range bw_range = {0};
+	struct dc_dsc_config dsc_cfg = {0};
+	struct dc_dsc_config_options dsc_options = {0};
+
+	dc_dsc_get_default_config_option(dc, &dsc_options);
+	dsc_options.max_target_bpp_limit_override_x16 = max_dsc_target_bpp_limit_override * 16;
+
+	verified_link_cap = dc_link_get_link_cap(stream->link);
+	link_bw_in_kbps = dc_link_bandwidth_kbps(stream->link, verified_link_cap);
+	edp_min_bpp_x16 = 8 * 16;
+	edp_max_bpp_x16 = 8 * 16;
+
+	if (edp_max_bpp_x16 > dsc_caps->edp_max_bits_per_pixel)
+		edp_max_bpp_x16 = dsc_caps->edp_max_bits_per_pixel;
+
+	if (edp_max_bpp_x16 < edp_min_bpp_x16)
+		edp_min_bpp_x16 = edp_max_bpp_x16;
+
+	if (dc_dsc_compute_bandwidth_range(dc->res_pool->dscs[0],
+				dc->debug.dsc_min_slice_height_override,
+				edp_min_bpp_x16, edp_max_bpp_x16,
+				dsc_caps,
+				&stream->timing,
+				dc_link_get_highest_encoding_format(aconnector->dc_link),
+				&bw_range)) {
+
+		if (bw_range.max_kbps < link_bw_in_kbps) {
+			if (dc_dsc_compute_config(dc->res_pool->dscs[0],
+					dsc_caps,
+					&dsc_options,
+					0,
+					&stream->timing,
+					dc_link_get_highest_encoding_format(aconnector->dc_link),
+					&dsc_cfg)) {
+				stream->timing.dsc_cfg = dsc_cfg;
+				stream->timing.flags.DSC = 1;
+				stream->timing.dsc_cfg.bits_per_pixel = edp_max_bpp_x16;
+			}
+			return;
+		}
+	}
+
+	if (dc_dsc_compute_config(dc->res_pool->dscs[0],
+				dsc_caps,
+				&dsc_options,
+				link_bw_in_kbps,
+				&stream->timing,
+				dc_link_get_highest_encoding_format(aconnector->dc_link),
+				&dsc_cfg)) {
+		stream->timing.dsc_cfg = dsc_cfg;
+		stream->timing.flags.DSC = 1;
+	}
+}
+
+static void apply_dsc_policy_for_stream(struct amdgpu_dm_connector *aconnector,
+					struct dc_sink *sink, struct dc_stream_state *stream,
+					struct dsc_dec_dpcd_caps *dsc_caps)
+{
+	struct drm_connector *drm_connector = &aconnector->base;
+	u32 link_bandwidth_kbps;
+	struct dc *dc = sink->ctx->dc;
+	const struct dc_hdmi_frl_link_settings *frl_verified_link_cap = NULL;
+	u32 converter_bw_in_kbps;
+	u32 sink_bw_in_kbps;
+	u32 dsc_sink_bw_in_kbps;
+	u32 max_supported_bw_in_kbps, timing_bw_in_kbps;
+	u32 dsc_max_supported_bw_in_kbps;
+	u32 max_dsc_target_bpp_limit_override =
+		drm_connector->display_info.max_dsc_bpp;
+	struct dc_dsc_config_options dsc_options = {0};
+
+	if (!aconnector->dc_link)
+		return;
+
+	dc_dsc_get_default_config_option(dc, &dsc_options);
+	dsc_options.max_target_bpp_limit_override_x16 = max_dsc_target_bpp_limit_override * 16;
+
+	link_bandwidth_kbps = dc_link_bandwidth_kbps(aconnector->dc_link,
+							dc_link_get_link_cap(aconnector->dc_link));
+
+	/* Set DSC policy according to dsc_clock_en */
+	dc_dsc_policy_set_enable_dsc_when_not_needed(
+		aconnector->dsc_settings.dsc_force_enable == DSC_CLK_FORCE_ENABLE);
+
+	if (sink->sink_signal == SIGNAL_TYPE_EDP &&
+	    !aconnector->dc_link->panel_config.dsc.disable_dsc_edp &&
+	    dc->caps.edp_dsc_support && aconnector->dsc_settings.dsc_force_enable != DSC_CLK_FORCE_DISABLE) {
+
+		apply_dsc_policy_for_edp(aconnector, sink, stream, dsc_caps, max_dsc_target_bpp_limit_override);
+
+	} else if (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT) {
+		if (sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_NONE) {
+			if (dc_dsc_compute_config(aconnector->dc_link->ctx->dc->res_pool->dscs[0],
+						dsc_caps,
+						&dsc_options,
+						link_bandwidth_kbps,
+						&stream->timing,
+						dc_link_get_highest_encoding_format(aconnector->dc_link),
+						&stream->timing.dsc_cfg)) {
+				stream->timing.flags.DSC = 1;
+				drm_dbg_driver(drm_connector->dev, "%s: SST_DSC [%s] DSC is selected from SST RX\n",
+							__func__, drm_connector->name);
+			}
+		} else if (sink->link->dpcd_caps.dongle_type == DISPLAY_DONGLE_DP_HDMI_CONVERTER) {
+			timing_bw_in_kbps = dc_bandwidth_in_kbps_from_timing(&stream->timing,
+					dc_link_get_highest_encoding_format(aconnector->dc_link));
+			converter_bw_in_kbps = aconnector->dc_link->dpcd_caps.dongle_caps.dp_hdmi_frl_max_link_bw_in_kbps;
+			sink_bw_in_kbps = dc_link_bw_kbps_from_raw_frl_link_rate_data(dc, sink->edid_caps.max_frl_rate);
+			dsc_sink_bw_in_kbps = dc_link_bw_kbps_from_raw_frl_link_rate_data(dc, sink->edid_caps.frl_dsc_max_frl_rate);
+
+			if (dsc_caps->is_frl) {
+				max_supported_bw_in_kbps = min(link_bandwidth_kbps, converter_bw_in_kbps);
+				max_supported_bw_in_kbps = min(max_supported_bw_in_kbps, sink_bw_in_kbps);
+				dsc_max_supported_bw_in_kbps = min(max_supported_bw_in_kbps, dsc_sink_bw_in_kbps);
+			} else {
+				max_supported_bw_in_kbps = link_bandwidth_kbps;
+				dsc_max_supported_bw_in_kbps = link_bandwidth_kbps;
+			}
+
+			if (timing_bw_in_kbps > max_supported_bw_in_kbps &&
+					max_supported_bw_in_kbps > 0 &&
+					dsc_max_supported_bw_in_kbps > 0)
+				if (dc_dsc_compute_config(aconnector->dc_link->ctx->dc->res_pool->dscs[0],
+						dsc_caps,
+						&dsc_options,
+						dsc_max_supported_bw_in_kbps,
+						&stream->timing,
+						dc_link_get_highest_encoding_format(aconnector->dc_link),
+						&stream->timing.dsc_cfg)) {
+					stream->timing.flags.DSC = 1;
+					drm_dbg_driver(drm_connector->dev, "%s: SST_DSC [%s] DSC is selected from %s\n",
+							__func__, drm_connector->name,
+							(dsc_caps->is_frl == 1) ? "HDMI FRL RX" : "DP-HDMI PCON");
+				}
+		}
+	} else if (sink->sink_signal == SIGNAL_TYPE_HDMI_FRL) {
+		struct dc_dsc_policy dsc_policy = {0};
+
+		frl_verified_link_cap = dc_link_get_frl_link_cap(stream->link);
+		if (frl_verified_link_cap->frl_link_rate != HDMI_FRL_LINK_RATE_DISABLE &&
+			aconnector->dc_link->frl_flags.force_frl_dsc) {
+			dc_dsc_policy_set_enable_dsc_when_not_needed(true);
+			dc_dsc_get_policy_for_timing(&stream->timing, 0, &dsc_policy, dc_link_get_highest_encoding_format(stream->link));
+		}
+
+		timing_bw_in_kbps = dc_bandwidth_in_kbps_from_timing(&stream->timing, DC_LINK_ENCODING_HDMI_FRL);
+		link_bandwidth_kbps = dc_link_frl_bandwidth_kbps(stream->link, frl_verified_link_cap->frl_link_rate);
+		dsc_sink_bw_in_kbps = dc_link_bw_kbps_from_raw_frl_link_rate_data(dc, sink->edid_caps.frl_dsc_max_frl_rate);
+
+		if ((timing_bw_in_kbps > link_bandwidth_kbps && dsc_sink_bw_in_kbps > 0) ||
+		    (dsc_policy.enable_dsc_when_not_needed || dsc_options.force_dsc_when_not_needed)) {
+			if (dc_dsc_compute_config(aconnector->dc_link->ctx->dc->res_pool->dscs[0],
+					dsc_caps,
+					&dsc_options,
+					dsc_sink_bw_in_kbps,
+					&stream->timing,
+					dc_link_get_highest_encoding_format(aconnector->dc_link),
+					&stream->timing.dsc_cfg)) {
+				stream->timing.flags.DSC = 1;
+				drm_dbg_driver(drm_connector->dev, "%s: HDMI_FRL_DSC [%s] DSC is selected from HDMI FRL RX\n",
+						__func__, drm_connector->name);
+			}
+		}
+	}
+
+	/* Overwrite the stream flag if DSC is enabled through debugfs */
+	if (aconnector->dsc_settings.dsc_force_enable == DSC_CLK_FORCE_ENABLE)
+		stream->timing.flags.DSC = 1;
+
+	if (stream->timing.flags.DSC && aconnector->dsc_settings.dsc_num_slices_h)
+		stream->timing.dsc_cfg.num_slices_h = aconnector->dsc_settings.dsc_num_slices_h;
+
+	if (stream->timing.flags.DSC && aconnector->dsc_settings.dsc_num_slices_v)
+		stream->timing.dsc_cfg.num_slices_v = aconnector->dsc_settings.dsc_num_slices_v;
+
+	if (stream->timing.flags.DSC && aconnector->dsc_settings.dsc_bits_per_pixel)
+		stream->timing.dsc_cfg.bits_per_pixel = aconnector->dsc_settings.dsc_bits_per_pixel;
+}
+#endif
+
+static struct dc_stream_state *
+create_stream_for_sink(struct drm_connector *connector,
+		       const struct drm_display_mode *drm_mode,
+		       const struct dm_connector_state *dm_state,
+		       const struct dc_stream_state *old_stream,
+		       int requested_bpc)
+{
+	struct drm_device *dev = connector->dev;
+	struct amdgpu_dm_connector *aconnector = NULL;
+	struct drm_display_mode *preferred_mode = NULL;
+	const struct drm_connector_state *con_state = &dm_state->base;
+	struct dc_stream_state *stream = NULL;
+	struct drm_display_mode mode;
+	struct drm_display_mode saved_mode;
+	struct drm_display_mode *freesync_mode = NULL;
+	bool native_mode_found = false;
+	bool recalculate_timing = false;
+	bool scale = dm_state->scaling != RMX_OFF;
+	int mode_refresh;
+	int preferred_refresh = 0;
+	enum color_transfer_func tf = TRANSFER_FUNC_UNKNOWN;
+#if defined(CONFIG_DRM_AMD_DC_FP)
+	struct dsc_dec_dpcd_caps dsc_caps = {0};
+#endif
+	struct dc_link *link = NULL;
+	struct dc_sink *sink = NULL;
+
+	drm_mode_init(&mode, drm_mode);
+	memset(&saved_mode, 0, sizeof(saved_mode));
+
+	if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK) {
+		aconnector = NULL;
+		aconnector = to_amdgpu_dm_connector(connector);
+		link = aconnector->dc_link;
+	} else {
+		struct drm_writeback_connector *wbcon = NULL;
+		struct amdgpu_dm_wb_connector *dm_wbcon = NULL;
+
+		wbcon = drm_connector_to_writeback(connector);
+		dm_wbcon = to_amdgpu_dm_wb_connector(wbcon);
+		link = dm_wbcon->link;
+	}
+
+	if (!aconnector || !aconnector->dc_sink) {
+		sink = create_fake_sink(dev, link);
+		if (!sink)
+			return stream;
+
+	} else {
+		sink = aconnector->dc_sink;
+		dc_sink_retain(sink);
+	}
+
+	stream = dc_create_stream_for_sink(sink);
+
+	if (stream == NULL) {
+		drm_err(dev, "Failed to create stream for sink!\n");
+		goto finish;
+	}
+
+	/* We leave this NULL for writeback connectors */
+	stream->dm_stream_context = aconnector;
+
+	stream->timing.flags.LTE_340MCSC_SCRAMBLE =
+		connector->display_info.hdmi.scdc.scrambling.low_rates;
+
+	list_for_each_entry(preferred_mode, &connector->modes, head) {
+		/* Search for preferred mode */
+		if (preferred_mode->type & DRM_MODE_TYPE_PREFERRED) {
+			native_mode_found = true;
+			break;
+		}
+	}
+	if (!native_mode_found)
+		preferred_mode = list_first_entry_or_null(
+				&connector->modes,
+				struct drm_display_mode,
+				head);
+
+	mode_refresh = drm_mode_vrefresh(&mode);
+
+	if (preferred_mode == NULL) {
+		/*
+		 * This may not be an error, the use case is when we have no
+		 * usermode calls to reset and set mode upon hotplug. In this
+		 * case, we call set mode ourselves to restore the previous mode
+		 * and the modelist may not be filled in time.
+		 */
+		drm_dbg_driver(dev, "No preferred mode found\n");
+	} else if (aconnector) {
+		recalculate_timing = amdgpu_freesync_vid_mode &&
+				 amdgpu_dm_is_freesync_video_mode(&mode, aconnector);
+		if (recalculate_timing) {
+			freesync_mode = amdgpu_dm_get_highest_refresh_rate_mode(aconnector, false);
+			drm_mode_copy(&saved_mode, &mode);
+			saved_mode.picture_aspect_ratio = mode.picture_aspect_ratio;
+			drm_mode_copy(&mode, freesync_mode);
+			mode.picture_aspect_ratio = saved_mode.picture_aspect_ratio;
+		} else {
+			decide_crtc_timing_for_drm_display_mode(
+					&mode, preferred_mode, scale);
+
+			preferred_refresh = drm_mode_vrefresh(preferred_mode);
+		}
+	}
+
+	if (recalculate_timing)
+		drm_mode_set_crtcinfo(&saved_mode, 0);
+
+	/*
+	 * If scaling is enabled and refresh rate didn't change
+	 * we copy the vic and polarities of the old timings
+	 */
+	if (!scale || mode_refresh != preferred_refresh)
+		fill_stream_properties_from_drm_display_mode(
+			stream, &mode, connector, con_state, NULL,
+			requested_bpc);
+	else
+		fill_stream_properties_from_drm_display_mode(
+			stream, &mode, connector, con_state, old_stream,
+			requested_bpc);
+
+	/* The rest isn't needed for writeback connectors */
+	if (!aconnector)
+		goto finish;
+
+	if (aconnector->timing_changed) {
+		drm_dbg(aconnector->base.dev,
+			"overriding timing for automated test, bpc %d, changing to %d\n",
+			stream->timing.display_color_depth,
+			aconnector->timing_requested->display_color_depth);
+		stream->timing = *aconnector->timing_requested;
+	}
+
+#if defined(CONFIG_DRM_AMD_DC_FP)
+	/* SST DSC determination policy */
+	update_dsc_caps(aconnector, sink, stream, &dsc_caps);
+	if (aconnector->dsc_settings.dsc_force_enable != DSC_CLK_FORCE_DISABLE && dsc_caps.is_dsc_supported)
+		apply_dsc_policy_for_stream(aconnector, sink, stream, &dsc_caps);
+#endif
+
+	amdgpu_dm_update_stream_scaling_settings(dev, &mode, dm_state, stream);
+
+	amdgpu_dm_fill_audio_info(
+		&stream->audio_info,
+		connector,
+		sink);
+
+	update_stream_signal(stream, sink);
+
+	if (stream->signal == SIGNAL_TYPE_HDMI_TYPE_A ||
+	    stream->signal == SIGNAL_TYPE_HDMI_FRL)
+		mod_build_hf_vsif_infopacket(stream, &stream->vsp_infopacket, false, false);
+
+	if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT ||
+	    stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST ||
+	    stream->signal == SIGNAL_TYPE_EDP) {
+		const struct dc_edid_caps *edid_caps;
+		unsigned int disable_colorimetry = 0;
+
+		if (aconnector->dc_sink) {
+			edid_caps = &aconnector->dc_sink->edid_caps;
+			disable_colorimetry = edid_caps->panel_patch.disable_colorimetry;
+		}
+
+		/*
+		 * should decide stream support vsc sdp colorimetry capability
+		 * before building vsc info packet
+		 */
+		stream->use_vsc_sdp_for_colorimetry = stream->link->dpcd_caps.dpcd_rev.raw >= 0x14 &&
+						      stream->link->dpcd_caps.dprx_feature.bits.VSC_SDP_COLORIMETRY_SUPPORTED &&
+						      !disable_colorimetry;
+
+		if (stream->out_transfer_func.tf == TRANSFER_FUNCTION_GAMMA22)
+			tf = TRANSFER_FUNC_GAMMA_22;
+		mod_build_vsc_infopacket(stream, &stream->vsc_infopacket, stream->output_color_space, tf);
+		aconnector->sr_skip_count = AMDGPU_DM_PSR_ENTRY_DELAY;
+
+	}
+finish:
+	dc_sink_release(sink);
+
+	return stream;
+}
+
+/**
+ * amdgpu_dm_connector_poll - Poll a connector to see if it's connected to a display
+ * @aconnector: DM connector to poll (owns @base drm_connector and @dc_link)
+ * @force: if true, force polling even when DAC load detection was used
+ *
+ * Used for connectors that don't support HPD (hotplug detection) to
+ * periodically check whether the connector is connected to a display.
+ *
+ * When connection was determined via DAC load detection, we avoid
+ * re-running it on normal polls to prevent visible glitches, unless
+ * @force is set.
+ *
+ * Return: The probed connector status (connected/disconnected/unknown).
+ */
+static enum drm_connector_status
+amdgpu_dm_connector_poll(struct amdgpu_dm_connector *aconnector, bool force)
+{
+	struct drm_connector *connector = &aconnector->base;
+	struct drm_device *dev = connector->dev;
+	struct amdgpu_device *adev = drm_to_adev(dev);
+	struct dc_link *link = aconnector->dc_link;
+	enum dc_connection_type conn_type = dc_connection_none;
+	enum drm_connector_status status = connector_status_disconnected;
+
+	/* When we determined the connection using DAC load detection,
+	 * do NOT poll the connector do detect disconnect because
+	 * that would run DAC load detection again which can cause
+	 * visible visual glitches.
+	 *
+	 * Only allow to poll such a connector again when forcing.
+	 */
+	if (!force && link->local_sink && link->type == dc_connection_analog_load)
+		return connector->status;
+
+	mutex_lock(&aconnector->hpd_lock);
+
+	if (dc_link_detect_connection_type(aconnector->dc_link, &conn_type) &&
+	    conn_type != dc_connection_none) {
+		mutex_lock(&adev->dm.dc_lock);
+
+		/* Only call full link detection when a sink isn't created yet,
+		 * ie. just when the display is plugged in, otherwise we risk flickering.
+		 */
+		if (link->local_sink ||
+			dc_link_detect(link, DETECT_REASON_HPD))
+			status = connector_status_connected;
+
+		mutex_unlock(&adev->dm.dc_lock);
+	}
+
+	if (connector->status != status) {
+		if (status == connector_status_disconnected) {
+			if (link->local_sink)
+				dc_sink_release(link->local_sink);
+
+			link->local_sink = NULL;
+			link->dpcd_sink_count = 0;
+			link->type = dc_connection_none;
+		}
+
+		amdgpu_dm_update_connector_after_detect(aconnector);
+	}
+
+	mutex_unlock(&aconnector->hpd_lock);
+	return status;
+}
+
+/**
+ * amdgpu_dm_connector_detect() - Detect whether a DRM connector is connected to a display
+ *
+ * A connector is considered connected when it has a sink that is not NULL.
+ * For connectors that support HPD (hotplug detection), the connection is
+ * handled in the HPD interrupt.
+ * For connectors that may not support HPD, such as analog connectors,
+ * DRM will call this function repeatedly to poll them.
+ *
+ * Notes:
+ * 1. This interface is NOT called in context of HPD irq.
+ * 2. This interface *is called* in context of user-mode ioctl. Which
+ *    makes it a bad place for *any* MST-related activity.
+ *
+ * @connector: The DRM connector we are checking. We convert it to
+ *             amdgpu_dm_connector so we can read the DC link and state.
+ * @force:     If true, do a full detect again. This is used even when
+ *             a lighter check would normally be used to avoid flicker.
+ *
+ * Return: The connector status (connected, disconnected, or unknown).
+ *
+ */
+static enum drm_connector_status
+amdgpu_dm_connector_detect(struct drm_connector *connector, bool force)
+{
+	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
+
+	update_subconnector_property(aconnector);
+
+	if (aconnector->base.force == DRM_FORCE_ON ||
+		aconnector->base.force == DRM_FORCE_ON_DIGITAL)
+		return connector_status_connected;
+	else if (aconnector->base.force == DRM_FORCE_OFF)
+		return connector_status_disconnected;
+
+	/* Poll analog connectors and only when either
+	 * disconnected or connected to an analog display.
+	 */
+	if (drm_kms_helper_is_poll_worker() &&
+		dc_connector_supports_analog(aconnector->dc_link->link_id.id) &&
+		(!aconnector->dc_sink || aconnector->dc_sink->edid_caps.analog))
+		return amdgpu_dm_connector_poll(aconnector, force);
+
+	return (aconnector->dc_sink ? connector_status_connected :
+			connector_status_disconnected);
+}
+
+int amdgpu_dm_connector_atomic_set_property(struct drm_connector *connector,
+					    struct drm_connector_state *connector_state,
+					    struct drm_property *property,
+					    uint64_t val)
+{
+	struct drm_device *dev = connector->dev;
+	struct amdgpu_device *adev = drm_to_adev(dev);
+	struct dm_connector_state *dm_old_state =
+		to_dm_connector_state(connector->state);
+	struct dm_connector_state *dm_new_state =
+		to_dm_connector_state(connector_state);
+
+	int ret = -EINVAL;
+
+	if (property == dev->mode_config.scaling_mode_property) {
+		enum amdgpu_rmx_type rmx_type;
+
+		switch (val) {
+		case DRM_MODE_SCALE_CENTER:
+			rmx_type = RMX_CENTER;
+			break;
+		case DRM_MODE_SCALE_ASPECT:
+			rmx_type = RMX_ASPECT;
+			break;
+		case DRM_MODE_SCALE_FULLSCREEN:
+			rmx_type = RMX_FULL;
+			break;
+		case DRM_MODE_SCALE_NONE:
+		default:
+			rmx_type = RMX_OFF;
+			break;
+		}
+
+		if (dm_old_state->scaling == rmx_type)
+			return 0;
+
+		dm_new_state->scaling = rmx_type;
+		ret = 0;
+	} else if (property == adev->mode_info.underscan_hborder_property) {
+		dm_new_state->underscan_hborder = val;
+		ret = 0;
+	} else if (property == adev->mode_info.underscan_vborder_property) {
+		dm_new_state->underscan_vborder = val;
+		ret = 0;
+	} else if (property == adev->mode_info.underscan_property) {
+		dm_new_state->underscan_enable = val;
+		ret = 0;
+	} else if (property == adev->mode_info.abm_level_property) {
+		switch (val) {
+		case ABM_SYSFS_CONTROL:
+			dm_new_state->abm_sysfs_forbidden = false;
+			break;
+		case ABM_LEVEL_OFF:
+			dm_new_state->abm_sysfs_forbidden = true;
+			dm_new_state->abm_level = ABM_LEVEL_IMMEDIATE_DISABLE;
+			break;
+		default:
+			dm_new_state->abm_sysfs_forbidden = true;
+			dm_new_state->abm_level = val;
+		}
+		ret = 0;
+	}
+
+	return ret;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_connector_atomic_set_property);
+
+int amdgpu_dm_connector_atomic_get_property(struct drm_connector *connector,
+					    const struct drm_connector_state *state,
+					    struct drm_property *property,
+					    uint64_t *val)
+{
+	struct drm_device *dev = connector->dev;
+	struct amdgpu_device *adev = drm_to_adev(dev);
+	struct dm_connector_state *dm_state =
+		to_dm_connector_state(state);
+	int ret = -EINVAL;
+
+	if (property == dev->mode_config.scaling_mode_property) {
+		switch (dm_state->scaling) {
+		case RMX_CENTER:
+			*val = DRM_MODE_SCALE_CENTER;
+			break;
+		case RMX_ASPECT:
+			*val = DRM_MODE_SCALE_ASPECT;
+			break;
+		case RMX_FULL:
+			*val = DRM_MODE_SCALE_FULLSCREEN;
+			break;
+		case RMX_OFF:
+		default:
+			*val = DRM_MODE_SCALE_NONE;
+			break;
+		}
+		ret = 0;
+	} else if (property == adev->mode_info.underscan_hborder_property) {
+		*val = dm_state->underscan_hborder;
+		ret = 0;
+	} else if (property == adev->mode_info.underscan_vborder_property) {
+		*val = dm_state->underscan_vborder;
+		ret = 0;
+	} else if (property == adev->mode_info.underscan_property) {
+		*val = dm_state->underscan_enable;
+		ret = 0;
+	} else if (property == adev->mode_info.abm_level_property) {
+		if (!dm_state->abm_sysfs_forbidden)
+			*val = ABM_SYSFS_CONTROL;
+		else
+			*val = (dm_state->abm_level != ABM_LEVEL_IMMEDIATE_DISABLE) ?
+				dm_state->abm_level : 0;
+		ret = 0;
+	}
+
+	return ret;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_connector_atomic_get_property);
+
+static void amdgpu_dm_connector_unregister(struct drm_connector *connector)
+{
+	struct amdgpu_dm_connector *amdgpu_dm_connector = to_amdgpu_dm_connector(connector);
+
+	if (amdgpu_dm_should_create_sysfs(amdgpu_dm_connector))
+		sysfs_remove_group(&connector->kdev->kobj, &amdgpu_group);
+
+	cec_notifier_conn_unregister(amdgpu_dm_connector->notifier);
+	drm_dp_aux_unregister(&amdgpu_dm_connector->dm_dp_aux.aux);
+}
+
+static void amdgpu_dm_connector_destroy(struct drm_connector *connector)
+{
+	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
+	struct amdgpu_device *adev = drm_to_adev(connector->dev);
+	struct amdgpu_display_manager *dm = &adev->dm;
+
+	/*
+	 * Call only if mst_mgr was initialized before since it's not done
+	 * for all connector types.
+	 */
+	if (aconnector->mst_mgr.dev)
+		drm_dp_mst_topology_mgr_destroy(&aconnector->mst_mgr);
+
+	/* Cancel and flush any pending HDMI HPD debounce work */
+	if (aconnector->hdmi_hpd_debounce_delay_ms) {
+		cancel_delayed_work_sync(&aconnector->hdmi_hpd_debounce_work);
+		if (aconnector->hdmi_prev_sink) {
+			dc_sink_release(aconnector->hdmi_prev_sink);
+			aconnector->hdmi_prev_sink = NULL;
+		}
+	}
+
+	if (aconnector->bl_idx != -1) {
+		backlight_device_unregister(dm->backlight_dev[aconnector->bl_idx]);
+		dm->backlight_dev[aconnector->bl_idx] = NULL;
+	}
+
+	if (aconnector->dc_em_sink)
+		dc_sink_release(aconnector->dc_em_sink);
+	aconnector->dc_em_sink = NULL;
+	if (aconnector->dc_sink)
+		dc_sink_release(aconnector->dc_sink);
+	aconnector->dc_sink = NULL;
+
+	drm_dp_cec_unregister_connector(&aconnector->dm_dp_aux.aux);
+	drm_connector_unregister(connector);
+	drm_connector_cleanup(connector);
+	kfree(aconnector->dm_dp_aux.aux.name);
+
+	kfree(connector);
+}
+
+void amdgpu_dm_connector_funcs_reset(struct drm_connector *connector)
+{
+	struct dm_connector_state *old_state =
+		to_dm_connector_state(connector->state);
+	struct dm_connector_state *state;
+
+	state = kzalloc_obj(*state);
+	if (!state)
+		return;
+
+	if (connector->state)
+		__drm_atomic_helper_connector_destroy_state(connector->state);
+
+	kfree(old_state);
+
+	__drm_atomic_helper_connector_reset(connector, &state->base);
+
+	state->scaling = RMX_OFF;
+	state->underscan_enable = false;
+	state->underscan_hborder = 0;
+	state->underscan_vborder = 0;
+	state->base.max_requested_bpc = 8;
+	state->vcpi_slots = 0;
+	state->pbn = 0;
+
+	if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) {
+		if (amdgpu_dm_abm_level <= 0)
+			state->abm_level = ABM_LEVEL_IMMEDIATE_DISABLE;
+		else
+			state->abm_level = amdgpu_dm_abm_level;
+	}
+}
+EXPORT_IF_KUNIT(amdgpu_dm_connector_funcs_reset);
+
+struct drm_connector_state *
+amdgpu_dm_connector_atomic_duplicate_state(struct drm_connector *connector)
+{
+	struct dm_connector_state *state =
+		to_dm_connector_state(connector->state);
+
+	struct dm_connector_state *new_state =
+			kmemdup(state, sizeof(*state), GFP_KERNEL);
+
+	if (!new_state)
+		return NULL;
+
+	__drm_atomic_helper_connector_duplicate_state(connector, &new_state->base);
+
+	new_state->freesync_capable = state->freesync_capable;
+	new_state->abm_level = state->abm_level;
+	new_state->scaling = state->scaling;
+	new_state->underscan_enable = state->underscan_enable;
+	new_state->underscan_hborder = state->underscan_hborder;
+	new_state->underscan_vborder = state->underscan_vborder;
+	new_state->vcpi_slots = state->vcpi_slots;
+	new_state->pbn = state->pbn;
+	return &new_state->base;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_connector_atomic_duplicate_state);
+
+static int
+amdgpu_dm_connector_late_register(struct drm_connector *connector)
+{
+	struct amdgpu_dm_connector *amdgpu_dm_connector =
+		to_amdgpu_dm_connector(connector);
+	int r;
+
+	if (amdgpu_dm_should_create_sysfs(amdgpu_dm_connector)) {
+		r = sysfs_create_group(&connector->kdev->kobj,
+				       &amdgpu_group);
+		if (r)
+			return r;
+	}
+
+	amdgpu_dm_register_backlight_device(amdgpu_dm_connector);
+
+	if ((connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort) ||
+	    (connector->connector_type == DRM_MODE_CONNECTOR_eDP)) {
+		amdgpu_dm_connector->dm_dp_aux.aux.dev = connector->kdev;
+		r = drm_dp_aux_register(&amdgpu_dm_connector->dm_dp_aux.aux);
+		if (r)
+			return r;
+	}
+
+#if defined(CONFIG_DEBUG_FS)
+	connector_debugfs_init(amdgpu_dm_connector);
+#endif
+
+	return 0;
+}
+
+static void amdgpu_dm_connector_funcs_force(struct drm_connector *connector)
+{
+	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
+	struct dc_link *dc_link = aconnector->dc_link;
+	struct dc_sink *dc_em_sink = aconnector->dc_em_sink;
+	const struct drm_edid *drm_edid;
+	struct i2c_adapter *ddc;
+	struct drm_device *dev = connector->dev;
+
+	if (dc_link && dc_link->aux_mode)
+		ddc = &aconnector->dm_dp_aux.aux.ddc;
+	else
+		ddc = &aconnector->i2c->base;
+
+	drm_edid = drm_edid_read_ddc(connector, ddc);
+	drm_edid_connector_update(connector, drm_edid);
+	if (!drm_edid) {
+		drm_err(dev, "No EDID found on connector: %s.\n", connector->name);
+		return;
+	}
+
+	aconnector->drm_edid = drm_edid;
+	/* Update emulated (virtual) sink's EDID */
+	if (dc_em_sink && dc_link) {
+		/* FIXME: Get rid of drm_edid_raw() */
+		const struct edid *edid = drm_edid_raw(drm_edid);
+
+		memset(&dc_em_sink->edid_caps, 0, sizeof(struct dc_edid_caps));
+		memmove(dc_em_sink->dc_edid.raw_edid, edid,
+			(edid->extensions + 1) * EDID_LENGTH);
+		dm_helpers_parse_edid_caps(
+			dc_link,
+			&dc_em_sink->dc_edid,
+			&dc_em_sink->edid_caps);
+	}
+}
+
+static const struct drm_connector_funcs amdgpu_dm_connector_funcs = {
+	.reset = amdgpu_dm_connector_funcs_reset,
+	.detect = amdgpu_dm_connector_detect,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = amdgpu_dm_connector_destroy,
+	.atomic_duplicate_state = amdgpu_dm_connector_atomic_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+	.atomic_set_property = amdgpu_dm_connector_atomic_set_property,
+	.atomic_get_property = amdgpu_dm_connector_atomic_get_property,
+	.late_register = amdgpu_dm_connector_late_register,
+	.early_unregister = amdgpu_dm_connector_unregister,
+	.force = amdgpu_dm_connector_funcs_force
+};
+
+static int get_modes(struct drm_connector *connector)
+{
+	return amdgpu_dm_connector_get_modes(connector);
+}
+
+static void create_eml_sink(struct amdgpu_dm_connector *aconnector)
+{
+	struct drm_connector *connector = &aconnector->base;
+	struct dc_link *dc_link = aconnector->dc_link;
+	struct dc_sink_init_data init_params = {
+			.link = aconnector->dc_link,
+			.sink_signal = SIGNAL_TYPE_VIRTUAL
+	};
+	const struct drm_edid *drm_edid;
+	const struct edid *edid;
+	struct i2c_adapter *ddc;
+
+	if (dc_link && dc_link->aux_mode)
+		ddc = &aconnector->dm_dp_aux.aux.ddc;
+	else
+		ddc = &aconnector->i2c->base;
+
+	drm_edid = drm_edid_read_ddc(connector, ddc);
+	drm_edid_connector_update(connector, drm_edid);
+	if (!drm_edid) {
+		drm_err(connector->dev, "No EDID found on connector: %s.\n", connector->name);
+		return;
+	}
+
+	if (connector->display_info.is_hdmi)
+		init_params.sink_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+
+	aconnector->drm_edid = drm_edid;
+
+	/* FIXME: Get rid of drm_edid_raw() */
+	edid = drm_edid_raw(drm_edid);
+	aconnector->dc_em_sink = dc_link_add_remote_sink(
+		aconnector->dc_link,
+		(uint8_t *)edid,
+		(edid->extensions + 1) * EDID_LENGTH,
+		&init_params);
+
+	if (aconnector->base.force == DRM_FORCE_ON) {
+		aconnector->dc_sink = aconnector->dc_link->local_sink ?
+		aconnector->dc_link->local_sink :
+		aconnector->dc_em_sink;
+		if (aconnector->dc_sink)
+			dc_sink_retain(aconnector->dc_sink);
+	}
+}
+
+static void handle_edid_mgmt(struct amdgpu_dm_connector *aconnector)
+{
+	struct dc_link *link = (struct dc_link *)aconnector->dc_link;
+
+	/*
+	 * In case of headless boot with force on for DP managed connector
+	 * Those settings have to be != 0 to get initial modeset
+	 */
+	if (link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT) {
+		link->verified_link_cap.lane_count = LANE_COUNT_FOUR;
+		link->verified_link_cap.link_rate = LINK_RATE_HIGH2;
+	}
+
+	create_eml_sink(aconnector);
+}
+
+static enum dc_status dm_validate_stream_and_context(struct dc *dc,
+						struct dc_stream_state *stream)
+{
+	enum dc_status dc_result = DC_ERROR_UNEXPECTED;
+	struct dc_plane_state *dc_plane_state = NULL;
+	struct dc_state *dc_state = NULL;
+
+	if (!stream)
+		goto cleanup;
+
+	dc_plane_state = dc_create_plane_state(dc);
+	if (!dc_plane_state)
+		goto cleanup;
+
+	dc_state = dc_state_create(dc, NULL);
+	if (!dc_state)
+		goto cleanup;
+
+	/* populate stream to plane */
+	dc_plane_state->src_rect.height  = stream->src.height;
+	dc_plane_state->src_rect.width   = stream->src.width;
+	dc_plane_state->dst_rect.height  = stream->src.height;
+	dc_plane_state->dst_rect.width   = stream->src.width;
+	dc_plane_state->clip_rect.height = stream->src.height;
+	dc_plane_state->clip_rect.width  = stream->src.width;
+	dc_plane_state->plane_size.surface_pitch = ((stream->src.width + 255) / 256) * 256;
+	dc_plane_state->plane_size.surface_size.height = stream->src.height;
+	dc_plane_state->plane_size.surface_size.width  = stream->src.width;
+	dc_plane_state->plane_size.chroma_size.height  = stream->src.height;
+	dc_plane_state->plane_size.chroma_size.width   = stream->src.width;
+	dc_plane_state->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB8888;
+	dc_plane_state->tiling_info.gfx9.swizzle = DC_SW_UNKNOWN;
+	dc_plane_state->rotation = ROTATION_ANGLE_0;
+	dc_plane_state->is_tiling_rotated = false;
+	dc_plane_state->tiling_info.gfx8.array_mode = DC_ARRAY_LINEAR_GENERAL;
+
+	dc_result = dc_validate_stream(dc, stream);
+	if (dc_result == DC_OK)
+		dc_result = dc_validate_plane(dc, dc_plane_state);
+
+	if (dc_result == DC_OK)
+		dc_result = dc_state_add_stream(dc, dc_state, stream);
+
+	if (dc_result == DC_OK && !dc_state_add_plane(
+						dc,
+						stream,
+						dc_plane_state,
+						dc_state))
+		dc_result = DC_FAIL_ATTACH_SURFACES;
+
+	if (dc_result == DC_OK)
+		dc_result = dc_validate_global_state(dc, dc_state, DC_VALIDATE_MODE_ONLY);
+
+cleanup:
+	if (dc_state)
+		dc_state_release(dc_state);
+
+	if (dc_plane_state)
+		dc_plane_state_release(dc_plane_state);
+
+	return dc_result;
+}
+
+static enum dc_status
+dm_validate_stream_color_format(const struct drm_connector_state *drm_state,
+				const struct dc_stream_state *stream)
+{
+	enum dc_pixel_encoding encoding;
+
+	if (!drm_state->color_format)
+		return DC_OK;
+
+	switch (drm_state->color_format) {
+	case DRM_CONNECTOR_COLOR_FORMAT_AUTO:
+	case DRM_CONNECTOR_COLOR_FORMAT_RGB444:
+		encoding = PIXEL_ENCODING_RGB;
+		break;
+	case DRM_CONNECTOR_COLOR_FORMAT_YCBCR444:
+		encoding = PIXEL_ENCODING_YCBCR444;
+		break;
+	case DRM_CONNECTOR_COLOR_FORMAT_YCBCR422:
+		encoding = PIXEL_ENCODING_YCBCR422;
+		break;
+	case DRM_CONNECTOR_COLOR_FORMAT_YCBCR420:
+		encoding = PIXEL_ENCODING_YCBCR420;
+		break;
+	default:
+		encoding = PIXEL_ENCODING_UNDEFINED;
+		break;
+	}
+
+	return encoding == stream->timing.pixel_encoding ?
+		DC_OK : DC_UNSUPPORTED_VALUE;
+}
+
+struct dc_stream_state *
+amdgpu_dm_create_validate_stream_for_sink(struct drm_connector *connector,
+					  const struct drm_display_mode *drm_mode,
+					  const struct dm_connector_state *dm_state,
+					  const struct dc_stream_state *old_stream)
+{
+	struct amdgpu_dm_connector *aconnector = NULL;
+	struct amdgpu_device *adev = drm_to_adev(connector->dev);
+	struct dc_stream_state *stream;
+	const struct drm_connector_state *drm_state = dm_state ? &dm_state->base : NULL;
+	int requested_bpc = drm_state ? drm_state->max_requested_bpc : 8;
+	enum dc_status dc_result = DC_OK;
+	uint8_t bpc_limit = 6;
+
+	if (!dm_state)
+		return NULL;
+
+	if (connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK)
+		aconnector = to_amdgpu_dm_connector(connector);
+
+	if (aconnector &&
+	    (aconnector->dc_link->connector_signal == SIGNAL_TYPE_HDMI_TYPE_A ||
+	     aconnector->dc_link->connector_signal == SIGNAL_TYPE_HDMI_FRL ||
+	     aconnector->dc_link->dpcd_caps.dongle_type == DISPLAY_DONGLE_DP_HDMI_CONVERTER))
+		bpc_limit = 8;
+
+	do {
+		drm_dbg_kms(connector->dev, "Trying with %d bpc\n", requested_bpc);
+		stream = create_stream_for_sink(connector, drm_mode,
+						dm_state, old_stream,
+						requested_bpc);
+		if (stream == NULL) {
+			drm_err(adev_to_drm(adev), "Failed to create stream for sink!\n");
+			break;
+		}
+
+		dc_result = dc_validate_stream(adev->dm.dc, stream);
+
+		if (!aconnector) /* writeback connector */
+			return stream;
+
+		if (dc_result == DC_OK && stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST)
+			dc_result = dm_dp_mst_is_port_support_mode(aconnector, stream);
+
+		if (dc_result == DC_OK)
+			dc_result = dm_validate_stream_and_context(adev->dm.dc, stream);
+
+		if (dc_result == DC_OK)
+			dc_result = dm_validate_stream_color_format(drm_state, stream);
+
+		if (dc_result != DC_OK) {
+			drm_dbg_kms(connector->dev, "Pruned mode %d x %d (clk %d) %s %s -- %s\n",
+				      drm_mode->hdisplay,
+				      drm_mode->vdisplay,
+				      drm_mode->clock,
+				      dc_pixel_encoding_to_str(stream->timing.pixel_encoding),
+				      dc_color_depth_to_str(stream->timing.display_color_depth),
+				      dc_status_to_str(dc_result));
+
+			dc_stream_release(stream);
+			stream = NULL;
+			requested_bpc -= 2; /* lower bpc to retry validation */
+		}
+
+	} while (stream == NULL && requested_bpc >= bpc_limit);
+
+	switch (dc_result) {
+	/*
+	 * If we failed to validate DP bandwidth stream with the requested RGB color depth,
+	 * we try to fallback and configure in order:
+	 * YUV422 (8bpc, 6bpc)
+	 * YUV420 (8bpc, 6bpc)
+	 */
+	case DC_FAIL_ENC_VALIDATE:
+	case DC_EXCEED_DONGLE_CAP:
+	case DC_NO_DP_LINK_BANDWIDTH:
+		/* recursively entered twice and already tried both YUV422 and YUV420 */
+		if (aconnector->force_yuv422_output && aconnector->force_yuv420_output)
+			break;
+		/* first failure; try YUV422 */
+		if (!aconnector->force_yuv422_output) {
+			drm_dbg_kms(connector->dev, "%s:%d Validation failed with %d, retrying w/ YUV422\n",
+				    __func__, __LINE__, dc_result);
+			aconnector->force_yuv422_output = true;
+		/* recursively entered and YUV422 failed, try YUV420 */
+		} else if (!aconnector->force_yuv420_output) {
+			drm_dbg_kms(connector->dev, "%s:%d Validation failed with %d, retrying w/ YUV420\n",
+				    __func__, __LINE__, dc_result);
+			aconnector->force_yuv420_output = true;
+		}
+		stream = amdgpu_dm_create_validate_stream_for_sink(connector, drm_mode,
+							 dm_state, old_stream);
+		aconnector->force_yuv422_output = false;
+		aconnector->force_yuv420_output = false;
+		break;
+	case DC_OK:
+		break;
+	default:
+		drm_dbg_kms(connector->dev, "%s:%d Unhandled validation failure %d\n",
+			    __func__, __LINE__, dc_result);
+		break;
+	}
+
+	return stream;
+}
+
+enum drm_mode_status amdgpu_dm_connector_mode_valid(struct drm_connector *connector,
+				   const struct drm_display_mode *mode)
+{
+	int result = MODE_ERROR;
+	struct dc_sink *dc_sink;
+	struct drm_display_mode *test_mode;
+	/* TODO: Unhardcode stream count */
+	struct dc_stream_state *stream;
+	/* we always have an amdgpu_dm_connector here since we got
+	 * here via the amdgpu_dm_connector_helper_funcs
+	 */
+	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
+
+	if ((mode->flags & DRM_MODE_FLAG_INTERLACE) ||
+			(mode->flags & DRM_MODE_FLAG_DBLSCAN))
+		return result;
+
+	/*
+	 * Only run this the first time mode_valid is called to initilialize
+	 * EDID mgmt
+	 */
+	if (aconnector->base.force != DRM_FORCE_UNSPECIFIED &&
+		!aconnector->dc_em_sink)
+		handle_edid_mgmt(aconnector);
+
+	dc_sink = to_amdgpu_dm_connector(connector)->dc_sink;
+
+	if (dc_sink == NULL && aconnector->base.force != DRM_FORCE_ON_DIGITAL &&
+				aconnector->base.force != DRM_FORCE_ON) {
+		drm_err(connector->dev, "dc_sink is NULL!\n");
+		goto fail;
+	}
+
+	test_mode = drm_mode_duplicate(connector->dev, mode);
+	if (!test_mode)
+		goto fail;
+
+	drm_mode_set_crtcinfo(test_mode, 0);
+
+	stream = amdgpu_dm_create_validate_stream_for_sink(connector, test_mode,
+						 to_dm_connector_state(connector->state),
+						 NULL);
+	drm_mode_destroy(connector->dev, test_mode);
+	if (stream) {
+		dc_stream_release(stream);
+		result = MODE_OK;
+	}
+
+fail:
+	/* TODO: error handling*/
+	return result;
+}
+
+int amdgpu_dm_fill_hdr_info_packet(const struct drm_connector_state *state,
+				   struct dc_info_packet *out)
+{
+	struct hdmi_drm_infoframe frame;
+	unsigned char buf[30]; /* 26 + 4 */
+	ssize_t len;
+	int ret, i;
+
+	memset(out, 0, sizeof(*out));
+
+	if (!state->hdr_output_metadata)
+		return 0;
+
+	ret = drm_hdmi_infoframe_set_hdr_metadata(&frame, state);
+	if (ret)
+		return ret;
+
+	len = hdmi_drm_infoframe_pack_only(&frame, buf, sizeof(buf));
+	if (len < 0)
+		return (int)len;
+
+	/* Static metadata is a fixed 26 bytes + 4 byte header. */
+	if (len != 30)
+		return -EINVAL;
+
+	/* Prepare the infopacket for DC. */
+	switch (state->connector->connector_type) {
+	case DRM_MODE_CONNECTOR_HDMIA:
+		out->hb0 = 0x87; /* type */
+		out->hb1 = 0x01; /* version */
+		out->hb2 = 0x1A; /* length */
+		out->sb[0] = buf[3]; /* checksum */
+		i = 1;
+		break;
+
+	case DRM_MODE_CONNECTOR_DisplayPort:
+	case DRM_MODE_CONNECTOR_eDP:
+		out->hb0 = 0x00; /* sdp id, zero */
+		out->hb1 = 0x87; /* type */
+		out->hb2 = 0x1D; /* payload len - 1 */
+		out->hb3 = (0x13 << 2); /* sdp version */
+		out->sb[0] = 0x01; /* version */
+		out->sb[1] = 0x1A; /* length */
+		i = 2;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	memcpy(&out->sb[i], &buf[4], 26);
+	out->valid = true;
+
+	print_hex_dump(KERN_DEBUG, "HDR SB:", DUMP_PREFIX_NONE, 16, 1, out->sb,
+		       sizeof(out->sb), false);
+
+	return 0;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_fill_hdr_info_packet);
+
+static int
+amdgpu_dm_connector_atomic_check(struct drm_connector *conn,
+				 struct drm_atomic_commit *state)
+{
+	struct drm_connector_state *new_con_state =
+		drm_atomic_get_new_connector_state(state, conn);
+	struct drm_connector_state *old_con_state =
+		drm_atomic_get_old_connector_state(state, conn);
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *new_crtc_state;
+	struct amdgpu_dm_connector *aconn = to_amdgpu_dm_connector(conn);
+	int ret;
+
+	if (WARN_ON(unlikely(!old_con_state || !new_con_state)))
+		return -EINVAL;
+
+	trace_amdgpu_dm_connector_atomic_check(new_con_state);
+
+	if (conn->connector_type == DRM_MODE_CONNECTOR_DisplayPort) {
+		ret = drm_dp_mst_root_conn_atomic_check(new_con_state, &aconn->mst_mgr);
+		if (ret < 0)
+			return ret;
+	}
+
+	crtc = new_con_state->crtc;
+	if (!crtc)
+		return 0;
+
+	if (new_con_state->privacy_screen_sw_state != old_con_state->privacy_screen_sw_state) {
+		new_crtc_state = drm_atomic_get_crtc_state(state, crtc);
+		if (IS_ERR(new_crtc_state))
+			return PTR_ERR(new_crtc_state);
+
+		new_crtc_state->mode_changed = true;
+	}
+
+	if (new_con_state->colorspace != old_con_state->colorspace) {
+		new_crtc_state = drm_atomic_get_crtc_state(state, crtc);
+		if (IS_ERR(new_crtc_state))
+			return PTR_ERR(new_crtc_state);
+
+		new_crtc_state->mode_changed = true;
+	}
+
+	if (new_con_state->content_type != old_con_state->content_type) {
+		new_crtc_state = drm_atomic_get_crtc_state(state, crtc);
+		if (IS_ERR(new_crtc_state))
+			return PTR_ERR(new_crtc_state);
+
+		new_crtc_state->mode_changed = true;
+	}
+
+	if (!drm_connector_atomic_hdr_metadata_equal(old_con_state, new_con_state)) {
+		struct dc_info_packet hdr_infopacket;
+
+		ret = amdgpu_dm_fill_hdr_info_packet(new_con_state, &hdr_infopacket);
+		if (ret)
+			return ret;
+
+		new_crtc_state = drm_atomic_get_crtc_state(state, crtc);
+		if (IS_ERR(new_crtc_state))
+			return PTR_ERR(new_crtc_state);
+
+		/*
+		 * DC considers the stream backends changed if the
+		 * static metadata changes. Forcing the modeset also
+		 * gives a simple way for userspace to switch from
+		 * 8bpc to 10bpc when setting the metadata to enter
+		 * or exit HDR.
+		 *
+		 * Changing the static metadata after it's been
+		 * set is permissible, however. So only force a
+		 * modeset if we're entering or exiting HDR.
+		 */
+		new_crtc_state->mode_changed = new_crtc_state->mode_changed ||
+			!old_con_state->hdr_output_metadata ||
+			!new_con_state->hdr_output_metadata;
+	}
+
+	return 0;
+}
+
+static const struct drm_connector_helper_funcs
+amdgpu_dm_connector_helper_funcs = {
+	/*
+	 * If hotplugging a second bigger display in FB Con mode, bigger resolution
+	 * modes will be filtered by drm_mode_validate_size(), and those modes
+	 * are missing after user start lightdm. So we need to renew modes list.
+	 * in get_modes call back, not just return the modes count
+	 */
+	.get_modes = get_modes,
+	.mode_valid = amdgpu_dm_connector_mode_valid,
+	.atomic_check = amdgpu_dm_connector_atomic_check,
+};
+
+int amdgpu_dm_convert_dc_color_depth_into_bpc(enum dc_color_depth display_color_depth)
+{
+	switch (display_color_depth) {
+	case COLOR_DEPTH_666:
+		return 6;
+	case COLOR_DEPTH_888:
+		return 8;
+	case COLOR_DEPTH_101010:
+		return 10;
+	case COLOR_DEPTH_121212:
+		return 12;
+	case COLOR_DEPTH_141414:
+		return 14;
+	case COLOR_DEPTH_161616:
+		return 16;
+	default:
+		break;
+	}
+	return 0;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_convert_dc_color_depth_into_bpc);
+
+STATIC_IFN_KUNIT int to_drm_connector_type(enum signal_type st, uint32_t connector_id)
+{
+	switch (st) {
+	case SIGNAL_TYPE_HDMI_TYPE_A:
+		return DRM_MODE_CONNECTOR_HDMIA;
+	case SIGNAL_TYPE_EDP:
+		return DRM_MODE_CONNECTOR_eDP;
+	case SIGNAL_TYPE_LVDS:
+		return DRM_MODE_CONNECTOR_LVDS;
+	case SIGNAL_TYPE_RGB:
+		return DRM_MODE_CONNECTOR_VGA;
+	case SIGNAL_TYPE_DISPLAY_PORT:
+	case SIGNAL_TYPE_DISPLAY_PORT_MST:
+		/* External DP bridges have a different connector type. */
+		if (connector_id == CONNECTOR_ID_VGA)
+			return DRM_MODE_CONNECTOR_VGA;
+		else if (connector_id == CONNECTOR_ID_LVDS)
+			return DRM_MODE_CONNECTOR_LVDS;
+
+		return DRM_MODE_CONNECTOR_DisplayPort;
+	case SIGNAL_TYPE_DVI_DUAL_LINK:
+	case SIGNAL_TYPE_DVI_SINGLE_LINK:
+		if (connector_id == CONNECTOR_ID_SINGLE_LINK_DVII ||
+			connector_id == CONNECTOR_ID_DUAL_LINK_DVII)
+			return DRM_MODE_CONNECTOR_DVII;
+
+		return DRM_MODE_CONNECTOR_DVID;
+	case SIGNAL_TYPE_VIRTUAL:
+		return DRM_MODE_CONNECTOR_VIRTUAL;
+
+	default:
+		return DRM_MODE_CONNECTOR_Unknown;
+	}
+}
+EXPORT_IF_KUNIT(to_drm_connector_type);
+
+static struct drm_encoder *amdgpu_dm_connector_to_encoder(struct drm_connector *connector)
+{
+	struct drm_encoder *encoder;
+
+	/* There is only one encoder per connector */
+	drm_connector_for_each_possible_encoder(connector, encoder)
+		return encoder;
+
+	return NULL;
+}
+
+static void amdgpu_dm_get_native_mode(struct drm_connector *connector)
+{
+	struct drm_encoder *encoder;
+	struct amdgpu_encoder *amdgpu_encoder;
+
+	encoder = amdgpu_dm_connector_to_encoder(connector);
+
+	if (encoder == NULL)
+		return;
+
+	amdgpu_encoder = to_amdgpu_encoder(encoder);
+
+	amdgpu_encoder->native_mode.clock = 0;
+
+	if (!list_empty(&connector->probed_modes)) {
+		struct drm_display_mode *preferred_mode = NULL;
+
+		list_for_each_entry(preferred_mode,
+				    &connector->probed_modes,
+				    head) {
+			if (preferred_mode->type & DRM_MODE_TYPE_PREFERRED)
+				amdgpu_encoder->native_mode = *preferred_mode;
+
+			break;
+		}
+
+	}
+}
+
+static struct drm_display_mode *
+amdgpu_dm_create_common_mode(struct drm_encoder *encoder,
+			     const char *name,
+			     int hdisplay, int vdisplay)
+{
+	struct drm_device *dev = encoder->dev;
+	struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder);
+	struct drm_display_mode *mode = NULL;
+	struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode;
+
+	mode = drm_mode_duplicate(dev, native_mode);
+
+	if (mode == NULL)
+		return NULL;
+
+	mode->hdisplay = hdisplay;
+	mode->vdisplay = vdisplay;
+	mode->type &= ~DRM_MODE_TYPE_PREFERRED;
+	strscpy(mode->name, name, DRM_DISPLAY_MODE_LEN);
+
+	return mode;
+
+}
+
+static const struct amdgpu_dm_mode_size {
+	char name[DRM_DISPLAY_MODE_LEN];
+	int w;
+	int h;
+} common_modes[] = {
+	{  "640x480",  640,  480},
+	{  "800x600",  800,  600},
+	{ "1024x768", 1024,  768},
+	{ "1280x720", 1280,  720},
+	{ "1280x800", 1280,  800},
+	{"1280x1024", 1280, 1024},
+	{ "1440x900", 1440,  900},
+	{"1680x1050", 1680, 1050},
+	{"1600x1200", 1600, 1200},
+	{"1920x1080", 1920, 1080},
+	{"1920x1200", 1920, 1200}
+};
+
+static void amdgpu_dm_connector_add_common_modes(struct drm_encoder *encoder,
+						 struct drm_connector *connector)
+{
+	struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder);
+	struct drm_display_mode *mode = NULL;
+	struct drm_display_mode *native_mode = &amdgpu_encoder->native_mode;
+	struct amdgpu_dm_connector *amdgpu_dm_connector =
+				to_amdgpu_dm_connector(connector);
+	int i;
+	int n;
+
+	if ((connector->connector_type != DRM_MODE_CONNECTOR_eDP) &&
+	    (connector->connector_type != DRM_MODE_CONNECTOR_LVDS))
+		return;
+
+	n = ARRAY_SIZE(common_modes);
+
+	for (i = 0; i < n; i++) {
+		struct drm_display_mode *curmode = NULL;
+		bool mode_existed = false;
+
+		if (common_modes[i].w > native_mode->hdisplay ||
+		    common_modes[i].h > native_mode->vdisplay ||
+		   (common_modes[i].w == native_mode->hdisplay &&
+		    common_modes[i].h == native_mode->vdisplay))
+			continue;
+
+		list_for_each_entry(curmode, &connector->probed_modes, head) {
+			if (common_modes[i].w == curmode->hdisplay &&
+			    common_modes[i].h == curmode->vdisplay) {
+				mode_existed = true;
+				break;
+			}
+		}
+
+		if (mode_existed)
+			continue;
+
+		mode = amdgpu_dm_create_common_mode(encoder,
+				common_modes[i].name, common_modes[i].w,
+				common_modes[i].h);
+		if (!mode)
+			continue;
+
+		drm_mode_probed_add(connector, mode);
+		amdgpu_dm_connector->num_modes++;
+	}
+}
+
+void amdgpu_set_panel_orientation(struct drm_connector *connector)
+{
+	struct drm_encoder *encoder;
+	struct amdgpu_encoder *amdgpu_encoder;
+	const struct drm_display_mode *native_mode;
+
+	if (connector->connector_type != DRM_MODE_CONNECTOR_eDP &&
+	    connector->connector_type != DRM_MODE_CONNECTOR_LVDS)
+		return;
+
+	mutex_lock(&connector->dev->mode_config.mutex);
+	amdgpu_dm_connector_get_modes(connector);
+	mutex_unlock(&connector->dev->mode_config.mutex);
+
+	encoder = amdgpu_dm_connector_to_encoder(connector);
+	if (!encoder)
+		return;
+
+	amdgpu_encoder = to_amdgpu_encoder(encoder);
+
+	native_mode = &amdgpu_encoder->native_mode;
+	if (native_mode->hdisplay == 0 || native_mode->vdisplay == 0)
+		return;
+
+	drm_connector_set_panel_orientation_with_quirk(connector,
+						       DRM_MODE_PANEL_ORIENTATION_UNKNOWN,
+						       native_mode->hdisplay,
+						       native_mode->vdisplay);
+}
+
+static void amdgpu_dm_connector_ddc_get_modes(struct drm_connector *connector,
+					      const struct drm_edid *drm_edid)
+{
+	struct amdgpu_dm_connector *amdgpu_dm_connector =
+			to_amdgpu_dm_connector(connector);
+
+	if (drm_edid) {
+		/* empty probed_modes */
+		INIT_LIST_HEAD(&connector->probed_modes);
+		amdgpu_dm_connector->num_modes =
+				drm_edid_connector_add_modes(connector);
+
+		/* sorting the probed modes before calling function
+		 * amdgpu_dm_get_native_mode() since EDID can have
+		 * more than one preferred mode. The modes that are
+		 * later in the probed mode list could be of higher
+		 * and preferred resolution. For example, 3840x2160
+		 * resolution in base EDID preferred timing and 4096x2160
+		 * preferred resolution in DID extension block later.
+		 */
+		drm_mode_sort(&connector->probed_modes);
+		amdgpu_dm_get_native_mode(connector);
+
+		/* Freesync capabilities are reset by calling
+		 * drm_edid_connector_add_modes() and need to be
+		 * restored here.
+		 */
+		amdgpu_dm_update_freesync_caps(connector, drm_edid, false);
+	} else {
+		amdgpu_dm_connector->num_modes = 0;
+	}
+}
+
+STATIC_IFN_KUNIT bool is_duplicate_mode(struct amdgpu_dm_connector *aconnector,
+			      struct drm_display_mode *mode)
+{
+	struct drm_display_mode *m;
+
+	list_for_each_entry(m, &aconnector->base.probed_modes, head) {
+		if (drm_mode_equal(m, mode))
+			return true;
+	}
+
+	return false;
+}
+EXPORT_IF_KUNIT(is_duplicate_mode);
+
+static uint add_fs_modes(struct amdgpu_dm_connector *aconnector)
+{
+	const struct drm_display_mode *m;
+	struct drm_display_mode *new_mode;
+	uint i;
+	u32 new_modes_count = 0;
+
+	/* Standard FPS values
+	 *
+	 * 23.976       - TV/NTSC
+	 * 24           - Cinema
+	 * 25           - TV/PAL
+	 * 29.97        - TV/NTSC
+	 * 30           - TV/NTSC
+	 * 48           - Cinema HFR
+	 * 50           - TV/PAL
+	 * 60           - Commonly used
+	 * 48,72,96,120 - Multiples of 24
+	 */
+	static const u32 common_rates[] = {
+		23976, 24000, 25000, 29970, 30000,
+		48000, 50000, 60000, 72000, 96000, 120000
+	};
+
+	/*
+	 * Find mode with highest refresh rate with the same resolution
+	 * as the preferred mode. Some monitors report a preferred mode
+	 * with lower resolution than the highest refresh rate supported.
+	 */
+
+	m = amdgpu_dm_get_highest_refresh_rate_mode(aconnector, true);
+	if (!m)
+		return 0;
+
+	for (i = 0; i < ARRAY_SIZE(common_rates); i++) {
+		u64 target_vtotal, target_vtotal_diff;
+		u64 num, den;
+
+		if (drm_mode_vrefresh(m) * 1000 < common_rates[i])
+			continue;
+
+		if (common_rates[i] < aconnector->min_vfreq * 1000 ||
+		    common_rates[i] > aconnector->max_vfreq * 1000)
+			continue;
+
+		num = (unsigned long long)m->clock * 1000 * 1000;
+		den = common_rates[i] * (unsigned long long)m->htotal;
+		target_vtotal = div_u64(num, den);
+		target_vtotal_diff = target_vtotal - m->vtotal;
+
+		/* Check for illegal modes */
+		if (m->vsync_start + target_vtotal_diff < m->vdisplay ||
+		    m->vsync_end + target_vtotal_diff < m->vsync_start ||
+		    m->vtotal + target_vtotal_diff < m->vsync_end)
+			continue;
+
+		new_mode = drm_mode_duplicate(aconnector->base.dev, m);
+		if (!new_mode)
+			goto out;
+
+		new_mode->vtotal += (u16)target_vtotal_diff;
+		new_mode->vsync_start += (u16)target_vtotal_diff;
+		new_mode->vsync_end += (u16)target_vtotal_diff;
+		new_mode->type &= ~DRM_MODE_TYPE_PREFERRED;
+		new_mode->type |= DRM_MODE_TYPE_DRIVER;
+
+		if (!is_duplicate_mode(aconnector, new_mode)) {
+			drm_mode_probed_add(&aconnector->base, new_mode);
+			new_modes_count += 1;
+		} else
+			drm_mode_destroy(aconnector->base.dev, new_mode);
+	}
+ out:
+	return new_modes_count;
+}
+
+static void amdgpu_dm_connector_add_freesync_modes(struct drm_connector *connector,
+						   const struct drm_edid *drm_edid)
+{
+	struct amdgpu_dm_connector *amdgpu_dm_connector =
+		to_amdgpu_dm_connector(connector);
+
+	if (!(amdgpu_freesync_vid_mode && drm_edid))
+		return;
+
+	if (!amdgpu_dm_connector->dc_sink || !amdgpu_dm_connector->dc_link)
+		return;
+
+	if (!dc_supports_vrr(amdgpu_dm_connector->dc_sink->ctx->dce_version))
+		return;
+
+	if (dc_connector_supports_analog(amdgpu_dm_connector->dc_link->link_id.id) &&
+	    amdgpu_dm_connector->dc_sink->edid_caps.analog)
+		return;
+
+	if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10)
+		amdgpu_dm_connector->num_modes +=
+			add_fs_modes(amdgpu_dm_connector);
+}
+
+static int amdgpu_dm_connector_get_modes(struct drm_connector *connector)
+{
+	struct amdgpu_dm_connector *amdgpu_dm_connector =
+			to_amdgpu_dm_connector(connector);
+	struct dc_link *dc_link = amdgpu_dm_connector->dc_link;
+	struct drm_encoder *encoder;
+	const struct drm_edid *drm_edid = amdgpu_dm_connector->drm_edid;
+	struct dc_link_settings *verified_link_cap = &dc_link->verified_link_cap;
+	const struct dc *dc = dc_link->dc;
+
+	encoder = amdgpu_dm_connector_to_encoder(connector);
+
+	if (!drm_edid) {
+		amdgpu_dm_connector->num_modes =
+				drm_add_modes_noedid(connector, 640, 480);
+		if (dc->link_srv->dp_get_encoding_format(verified_link_cap) == DP_128b_132b_ENCODING)
+			amdgpu_dm_connector->num_modes +=
+				drm_add_modes_noedid(connector, 1920, 1080);
+
+		if (amdgpu_dm_connector->dc_sink &&
+		    amdgpu_dm_connector->dc_sink->edid_caps.analog &&
+		    dc_connector_supports_analog(dc_link->link_id.id)) {
+			/* Analog monitor connected by DAC load detection.
+			 * Add common modes. It will be up to the user to select one that works.
+			 */
+			for (int i = 0; i < ARRAY_SIZE(common_modes); i++)
+				amdgpu_dm_connector->num_modes += drm_add_modes_noedid(
+					connector, common_modes[i].w, common_modes[i].h);
+		}
+	} else {
+		amdgpu_dm_connector_ddc_get_modes(connector, drm_edid);
+		if (encoder)
+			amdgpu_dm_connector_add_common_modes(encoder, connector);
+		amdgpu_dm_connector_add_freesync_modes(connector, drm_edid);
+	}
+	amdgpu_dm_fbc_init(connector);
+
+	return amdgpu_dm_connector->num_modes;
+}
+
+static const u32 supported_colorspaces =
+	BIT(DRM_MODE_COLORIMETRY_BT709_YCC) |
+	BIT(DRM_MODE_COLORIMETRY_OPRGB) |
+	BIT(DRM_MODE_COLORIMETRY_BT2020_RGB) |
+	BIT(DRM_MODE_COLORIMETRY_BT2020_YCC);
+
+static const u32 supported_colorformats =
+	BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444) |
+	BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444) |
+	BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422) |
+	BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420);
+
+static void hdmi_frl_status_polling_work(struct work_struct *work)
+{
+	struct amdgpu_display_manager *dm =
+		container_of(to_delayed_work(work), struct amdgpu_display_manager,
+				hdmi_frl_status_polling_work);
+	struct dc *dc = dm->dc;
+	struct dc_link *dc_link;
+	bool link_update = false;
+
+	for (int i = 0; i < MAX_LINKS; i++) {
+		dc_link = dc->links[i];
+
+
+		if (!dc_link || !dc_link->local_sink)
+			continue;
+
+		if (!dc_is_hdmi_signal(dc_link->connector_signal))
+			continue;
+
+		if (dc_link->connector_signal != SIGNAL_TYPE_HDMI_FRL)
+			continue;
+
+		link_update = dc_link_frl_poll_status_flag(dc_link);
+		if (link_update) {
+			mutex_lock(&dm->dc_lock);
+			dc_link_detect(dc_link, DETECT_REASON_RETRAIN);
+			mutex_unlock(&dm->dc_lock);
+		}
+	}
+
+	queue_delayed_work(dm->hdmi_frl_status_polling_wq,
+			   &dm->hdmi_frl_status_polling_work,
+			   msecs_to_jiffies(dm->hdmi_frl_status_polling_delay_ms));
+}
+
+void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm,
+				     struct amdgpu_dm_connector *aconnector,
+				     int connector_type,
+				     struct dc_link *link,
+				     int link_index)
+{
+	struct amdgpu_device *adev = drm_to_adev(dm->ddev);
+
+	/*
+	 * Some of the properties below require access to state, like bpc.
+	 * Allocate some default initial connector state with our reset helper.
+	 */
+	if (aconnector->base.funcs->reset)
+		aconnector->base.funcs->reset(&aconnector->base);
+
+	aconnector->connector_id = link_index;
+	aconnector->bl_idx = -1;
+	aconnector->dc_link = link;
+	aconnector->base.interlace_allowed = false;
+	aconnector->base.doublescan_allowed = false;
+	aconnector->base.stereo_allowed = false;
+	aconnector->base.dpms = DRM_MODE_DPMS_OFF;
+	aconnector->hpd.hpd = AMDGPU_HPD_NONE; /* not used */
+	aconnector->audio_inst = -1;
+	aconnector->pack_sdp_v1_3 = false;
+	aconnector->as_type = ADAPTIVE_SYNC_TYPE_NONE;
+	memset(&aconnector->vsdb_info, 0, sizeof(aconnector->vsdb_info));
+	mutex_init(&aconnector->hpd_lock);
+	mutex_init(&aconnector->handle_mst_msg_ready);
+
+	/*
+	 * If HDMI HPD debounce delay is set, use the minimum between selected
+	 * value and AMDGPU_DM_MAX_HDMI_HPD_DEBOUNCE_MS
+	 */
+	if (amdgpu_hdmi_hpd_debounce_delay_ms) {
+		aconnector->hdmi_hpd_debounce_delay_ms = min(amdgpu_hdmi_hpd_debounce_delay_ms,
+							     AMDGPU_DM_MAX_HDMI_HPD_DEBOUNCE_MS);
+		INIT_DELAYED_WORK(&aconnector->hdmi_hpd_debounce_work, amdgpu_dm_hdmi_hpd_debounce_work);
+		aconnector->hdmi_prev_sink = NULL;
+	} else {
+		aconnector->hdmi_hpd_debounce_delay_ms = 0;
+	}
+
+	dm->hdmi_frl_status_polling_delay_ms = 200;
+	INIT_DELAYED_WORK(&dm->hdmi_frl_status_polling_work, hdmi_frl_status_polling_work);
+	/*
+	 * configure support HPD hot plug connector_>polled default value is 0
+	 * which means HPD hot plug not supported
+	 */
+	switch (connector_type) {
+	case DRM_MODE_CONNECTOR_HDMIA:
+		aconnector->base.polled = DRM_CONNECTOR_POLL_HPD;
+		aconnector->base.ycbcr_420_allowed =
+			link->link_enc->features.hdmi_ycbcr420_supported ? true : false;
+		break;
+	case DRM_MODE_CONNECTOR_DisplayPort:
+		aconnector->base.polled = DRM_CONNECTOR_POLL_HPD;
+		link->link_enc = link_enc_cfg_get_link_enc(link);
+		ASSERT(link->link_enc);
+		if (link->link_enc)
+			aconnector->base.ycbcr_420_allowed =
+			link->link_enc->features.dp_ycbcr420_supported ? true : false;
+		break;
+	case DRM_MODE_CONNECTOR_DVID:
+		aconnector->base.polled = DRM_CONNECTOR_POLL_HPD;
+		break;
+	case DRM_MODE_CONNECTOR_DVII:
+	case DRM_MODE_CONNECTOR_VGA:
+		aconnector->base.polled =
+			DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT;
+		break;
+	default:
+		break;
+	}
+
+	drm_object_attach_property(&aconnector->base.base,
+				dm->ddev->mode_config.scaling_mode_property,
+				DRM_MODE_SCALE_NONE);
+
+	if (connector_type == DRM_MODE_CONNECTOR_HDMIA
+		|| (connector_type == DRM_MODE_CONNECTOR_DisplayPort && !aconnector->mst_root))
+		drm_connector_attach_broadcast_rgb_property(&aconnector->base);
+
+	drm_object_attach_property(&aconnector->base.base,
+				adev->mode_info.underscan_property,
+				UNDERSCAN_OFF);
+	drm_object_attach_property(&aconnector->base.base,
+				adev->mode_info.underscan_hborder_property,
+				0);
+	drm_object_attach_property(&aconnector->base.base,
+				adev->mode_info.underscan_vborder_property,
+				0);
+
+	if (!aconnector->mst_root)
+		drm_connector_attach_max_bpc_property(&aconnector->base, 8, 16);
+
+	aconnector->base.state->max_bpc = 16;
+	aconnector->base.state->max_requested_bpc = aconnector->base.state->max_bpc;
+
+	if (connector_type == DRM_MODE_CONNECTOR_HDMIA) {
+		/* Content Type is currently only implemented for HDMI. */
+		drm_connector_attach_content_type_property(&aconnector->base);
+	}
+
+	if (connector_type == DRM_MODE_CONNECTOR_HDMIA) {
+		if (!drm_mode_create_hdmi_colorspace_property(&aconnector->base, supported_colorspaces))
+			drm_connector_attach_colorspace_property(&aconnector->base);
+	} else if ((connector_type == DRM_MODE_CONNECTOR_DisplayPort && !aconnector->mst_root) ||
+		   connector_type == DRM_MODE_CONNECTOR_eDP) {
+		if (!drm_mode_create_dp_colorspace_property(&aconnector->base, supported_colorspaces))
+			drm_connector_attach_colorspace_property(&aconnector->base);
+	}
+
+	if (connector_type == DRM_MODE_CONNECTOR_HDMIA ||
+	    connector_type == DRM_MODE_CONNECTOR_DisplayPort ||
+	    connector_type == DRM_MODE_CONNECTOR_eDP) {
+		drm_connector_attach_hdr_output_metadata_property(&aconnector->base);
+
+		if (!aconnector->mst_root) {
+			drm_connector_attach_vrr_capable_property(&aconnector->base);
+			drm_connector_attach_color_format_property(&aconnector->base,
+								   supported_colorformats);
+		}
+
+
+		if (adev->dm.hdcp_workqueue)
+			drm_connector_attach_content_protection_property(&aconnector->base, true);
+	}
+
+	if (connector_type == DRM_MODE_CONNECTOR_eDP) {
+		struct drm_privacy_screen *privacy_screen;
+
+		drm_connector_attach_panel_type_property(&aconnector->base);
+
+		privacy_screen = drm_privacy_screen_get(adev_to_drm(adev)->dev, NULL);
+		if (!IS_ERR(privacy_screen)) {
+			drm_connector_attach_privacy_screen_provider(&aconnector->base,
+								     privacy_screen);
+		} else if (PTR_ERR(privacy_screen) != -ENODEV) {
+			drm_warn(adev_to_drm(adev), "Error getting privacy-screen\n");
+		}
+	}
+}
+
+static int amdgpu_dm_i2c_xfer(struct i2c_adapter *i2c_adap,
+			      struct i2c_msg *msgs, int num)
+{
+	struct amdgpu_i2c_adapter *i2c = i2c_get_adapdata(i2c_adap);
+	struct ddc_service *ddc_service = i2c->ddc_service;
+	struct i2c_command cmd;
+	int i;
+	int result = -EIO;
+
+	if (!ddc_service->ddc_pin)
+		return result;
+
+	cmd.payloads = kzalloc_objs(struct i2c_payload, num);
+
+	if (!cmd.payloads)
+		return result;
+
+	cmd.number_of_payloads = num;
+	cmd.engine = I2C_COMMAND_ENGINE_DEFAULT;
+	cmd.speed = 100;
+
+	for (i = 0; i < num; i++) {
+		cmd.payloads[i].write = !(msgs[i].flags & I2C_M_RD);
+		cmd.payloads[i].address = msgs[i].addr;
+		cmd.payloads[i].length = msgs[i].len;
+		cmd.payloads[i].data = msgs[i].buf;
+	}
+
+	if (i2c->oem) {
+		if (dc_submit_i2c_oem(
+			    ddc_service->ctx->dc,
+			    &cmd))
+			result = num;
+	} else {
+		if (dc_submit_i2c(
+			    ddc_service->ctx->dc,
+			    ddc_service->link->link_index,
+			    &cmd))
+			result = num;
+	}
+
+	kfree(cmd.payloads);
+	return result;
+}
+
+static u32 amdgpu_dm_i2c_func(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+static const struct i2c_algorithm amdgpu_dm_i2c_algo = {
+	.master_xfer = amdgpu_dm_i2c_xfer,
+	.functionality = amdgpu_dm_i2c_func,
+};
+
+struct amdgpu_i2c_adapter *
+amdgpu_dm_create_i2c(struct ddc_service *ddc_service, bool oem)
+{
+	struct amdgpu_device *adev = ddc_service->ctx->driver_context;
+	struct amdgpu_i2c_adapter *i2c;
+
+	i2c = kzalloc_obj(struct amdgpu_i2c_adapter);
+	if (!i2c)
+		return NULL;
+	i2c->base.owner = THIS_MODULE;
+	i2c->base.dev.parent = &adev->pdev->dev;
+	i2c->base.algo = &amdgpu_dm_i2c_algo;
+	if (oem)
+		snprintf(i2c->base.name, sizeof(i2c->base.name), "AMDGPU DM i2c OEM bus");
+	else
+		snprintf(i2c->base.name, sizeof(i2c->base.name), "AMDGPU DM i2c hw bus %d",
+			 ddc_service->link->link_index);
+	i2c_set_adapdata(&i2c->base, i2c);
+	i2c->ddc_service = ddc_service;
+	i2c->oem = oem;
+
+	return i2c;
+}
+
+int amdgpu_dm_initialize_hdmi_connector(struct amdgpu_dm_connector *aconnector)
+{
+	struct cec_connector_info conn_info;
+	struct drm_device *ddev = aconnector->base.dev;
+	struct device *hdmi_dev = ddev->dev;
+
+	if (amdgpu_dc_debug_mask & DC_DISABLE_HDMI_CEC) {
+		drm_info(ddev, "HDMI-CEC feature masked\n");
+		return -EINVAL;
+	}
+
+	cec_fill_conn_info_from_drm(&conn_info, &aconnector->base);
+	aconnector->notifier =
+		cec_notifier_conn_register(hdmi_dev, NULL, &conn_info);
+	if (!aconnector->notifier) {
+		drm_err(ddev, "Failed to create cec notifier\n");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/*
+ * Note: this function assumes that dc_link_detect() was called for the
+ * dc_link which will be represented by this aconnector.
+ */
+int amdgpu_dm_connector_init(struct amdgpu_display_manager *dm,
+			     struct amdgpu_dm_connector *aconnector,
+			     u32 link_index,
+			     struct amdgpu_encoder *aencoder)
+{
+	int res = 0;
+	int connector_type;
+	struct dc *dc = dm->dc;
+	struct dc_link *link = dc_get_link_at_index(dc, link_index);
+	struct amdgpu_i2c_adapter *i2c;
+
+	/* Not needed for writeback connector */
+	link->priv = aconnector;
+
+
+	i2c = amdgpu_dm_create_i2c(link->ddc, false);
+	if (!i2c) {
+		drm_err(adev_to_drm(dm->adev), "Failed to create i2c adapter data\n");
+		return -ENOMEM;
+	}
+
+	aconnector->i2c = i2c;
+	res = devm_i2c_add_adapter(dm->adev->dev, &i2c->base);
+
+	if (res) {
+		drm_err(adev_to_drm(dm->adev), "Failed to register hw i2c %d\n", link->link_index);
+		goto out_free;
+	}
+
+	connector_type = to_drm_connector_type(link->connector_signal, link->link_id.id);
+
+	res = drm_connector_init_with_ddc(
+			dm->ddev,
+			&aconnector->base,
+			&amdgpu_dm_connector_funcs,
+			connector_type,
+			&i2c->base);
+
+	if (res) {
+		drm_err(adev_to_drm(dm->adev), "connector_init failed\n");
+		aconnector->connector_id = -1;
+		goto out_free;
+	}
+
+	drm_connector_helper_add(
+			&aconnector->base,
+			&amdgpu_dm_connector_helper_funcs);
+
+	amdgpu_dm_connector_init_helper(
+		dm,
+		aconnector,
+		connector_type,
+		link,
+		link_index);
+
+	drm_connector_attach_encoder(
+		&aconnector->base, &aencoder->base);
+
+	if (connector_type == DRM_MODE_CONNECTOR_HDMIA ||
+	    connector_type == DRM_MODE_CONNECTOR_HDMIB)
+		amdgpu_dm_initialize_hdmi_connector(aconnector);
+
+	if (dc_is_dp_signal(link->connector_signal))
+		amdgpu_dm_initialize_dp_connector(dm, aconnector, link->link_index);
+
+out_free:
+	if (res) {
+		kfree(i2c);
+		aconnector->i2c = NULL;
+	}
+	return res;
+}
+
+static int dm_force_atomic_commit(struct drm_connector *connector)
+{
+	int ret = 0;
+	struct drm_device *ddev = connector->dev;
+	struct drm_atomic_commit *state = drm_atomic_commit_alloc(ddev);
+	struct amdgpu_crtc *disconnected_acrtc = to_amdgpu_crtc(connector->encoder->crtc);
+	struct drm_plane *plane = disconnected_acrtc->base.primary;
+	struct drm_connector_state *conn_state;
+	struct drm_crtc_state *crtc_state;
+	struct drm_plane_state *plane_state;
+
+	if (!state)
+		return -ENOMEM;
+
+	state->acquire_ctx = ddev->mode_config.acquire_ctx;
+
+	/* Construct an atomic state to restore previous display setting */
+
+	/*
+	 * Attach connectors to drm_atomic_commit
+	 */
+	conn_state = drm_atomic_get_connector_state(state, connector);
+
+	/* Check for error in getting connector state */
+	if (IS_ERR(conn_state)) {
+		ret = PTR_ERR(conn_state);
+		goto out;
+	}
+
+	/* Attach crtc to drm_atomic_commit*/
+	crtc_state = drm_atomic_get_crtc_state(state, &disconnected_acrtc->base);
+
+	/* Check for error in getting crtc state */
+	if (IS_ERR(crtc_state)) {
+		ret = PTR_ERR(crtc_state);
+		goto out;
+	}
+
+	/* force a restore */
+	crtc_state->mode_changed = true;
+
+	/* Attach plane to drm_atomic_commit */
+	plane_state = drm_atomic_get_plane_state(state, plane);
+
+	/* Check for error in getting plane state */
+	if (IS_ERR(plane_state)) {
+		ret = PTR_ERR(plane_state);
+		goto out;
+	}
+
+	/* Call commit internally with the state we just constructed */
+	ret = drm_atomic_commit(state);
+
+out:
+	drm_atomic_commit_put(state);
+	if (ret)
+		drm_err(ddev, "Restoring old state failed with %i\n", ret);
+
+	return ret;
+}
+
+/*
+ * This function handles all cases when set mode does not come upon hotplug.
+ * This includes when a display is unplugged then plugged back into the
+ * same port and when running without usermode desktop manager support
+ */
+void dm_restore_drm_connector_state(struct drm_device *dev,
+				    struct drm_connector *connector)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_crtc *disconnected_acrtc;
+	struct dm_crtc_state *acrtc_state;
+
+	if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
+		return;
+
+	aconnector = to_amdgpu_dm_connector(connector);
+
+	if (!aconnector->dc_sink || !connector->state || !connector->encoder)
+		return;
+
+	disconnected_acrtc = to_amdgpu_crtc(connector->encoder->crtc);
+	if (!disconnected_acrtc)
+		return;
+
+	acrtc_state = to_dm_crtc_state(disconnected_acrtc->base.state);
+	if (!acrtc_state->stream)
+		return;
+
+	/*
+	 * If the previous sink is not released and different from the current,
+	 * we deduce we are in a state where we can not rely on usermode call
+	 * to turn on the display, so we do it here
+	 */
+	if (acrtc_state->stream->sink != aconnector->dc_sink)
+		dm_force_atomic_commit(&aconnector->base);
+}
+
+static bool dm_edid_parser_send_cea(struct amdgpu_display_manager *dm,
+		unsigned int offset,
+		unsigned int total_length,
+		u8 *data,
+		unsigned int length,
+		struct amdgpu_hdmi_vsdb_info *vsdb)
+{
+	bool res;
+	union dmub_rb_cmd cmd;
+	struct dmub_cmd_send_edid_cea *input;
+	struct dmub_cmd_edid_cea_output *output;
+
+	if (length > DMUB_EDID_CEA_DATA_CHUNK_BYTES)
+		return false;
+
+	memset(&cmd, 0, sizeof(cmd));
+
+	input = &cmd.edid_cea.data.input;
+
+	cmd.edid_cea.header.type = DMUB_CMD__EDID_CEA;
+	cmd.edid_cea.header.sub_type = 0;
+	cmd.edid_cea.header.payload_bytes =
+		sizeof(cmd.edid_cea) - sizeof(cmd.edid_cea.header);
+	input->offset = offset;
+	input->length = length;
+	input->cea_total_length = total_length;
+	memcpy(input->payload, data, length);
+
+	res = dc_wake_and_execute_dmub_cmd(dm->dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY);
+	if (!res) {
+		drm_err(adev_to_drm(dm->adev), "EDID CEA parser failed\n");
+		return false;
+	}
+
+	output = &cmd.edid_cea.data.output;
+
+	if (output->type == DMUB_CMD__EDID_CEA_ACK) {
+		if (!output->ack.success) {
+			drm_err(adev_to_drm(dm->adev), "EDID CEA ack failed at offset %d\n",
+					output->ack.offset);
+		}
+	} else if (output->type == DMUB_CMD__EDID_CEA_AMD_VSDB) {
+		if (!output->amd_vsdb.vsdb_found)
+			return false;
+
+		vsdb->freesync_supported = output->amd_vsdb.freesync_supported;
+		vsdb->amd_vsdb_version = output->amd_vsdb.amd_vsdb_version;
+		vsdb->min_refresh_rate_hz = output->amd_vsdb.min_frame_rate;
+		vsdb->max_refresh_rate_hz = output->amd_vsdb.max_frame_rate;
+		vsdb->freesync_mccs_vcp_code = output->amd_vsdb.freesync_mccs_vcp_code;
+	} else {
+		drm_warn(adev_to_drm(dm->adev), "Unknown EDID CEA parser results\n");
+		return false;
+	}
+
+	return true;
+}
+
+static bool parse_edid_cea_dmcu(struct amdgpu_display_manager *dm,
+		u8 *edid_ext, int len,
+		struct amdgpu_hdmi_vsdb_info *vsdb_info)
+{
+	int i;
+
+	/* send extension block to DMCU for parsing */
+	for (i = 0; i < len; i += 8) {
+		bool res;
+		int offset;
+
+		/* send 8 bytes a time */
+		if (!dc_edid_parser_send_cea(dm->dc, i, len, &edid_ext[i], 8))
+			return false;
+
+		if (i+8 == len) {
+			/* EDID block sent completed, expect result */
+			int version, min_rate, max_rate;
+
+			res = dc_edid_parser_recv_amd_vsdb(dm->dc, &version, &min_rate, &max_rate);
+			if (res) {
+				/* amd vsdb found */
+				vsdb_info->freesync_supported = 1;
+				vsdb_info->amd_vsdb_version = version;
+				vsdb_info->min_refresh_rate_hz = min_rate;
+				vsdb_info->max_refresh_rate_hz = max_rate;
+				/* Not enabled on DMCU*/
+				vsdb_info->freesync_mccs_vcp_code = 0;
+				return true;
+			}
+			/* not amd vsdb */
+			return false;
+		}
+
+		/* check for ack*/
+		res = dc_edid_parser_recv_cea_ack(dm->dc, &offset);
+		if (!res)
+			return false;
+	}
+
+	return false;
+}
+
+static bool parse_edid_cea_dmub(struct amdgpu_display_manager *dm,
+		u8 *edid_ext, int len,
+		struct amdgpu_hdmi_vsdb_info *vsdb_info)
+{
+	int i;
+
+	/* send extension block to DMCU for parsing */
+	for (i = 0; i < len; i += 8) {
+		/* send 8 bytes a time */
+		if (!dm_edid_parser_send_cea(dm, i, len, &edid_ext[i], 8, vsdb_info))
+			return false;
+	}
+
+	return vsdb_info->freesync_supported;
+}
+
+static bool parse_edid_cea(struct amdgpu_dm_connector *aconnector,
+		u8 *edid_ext, int len,
+		struct amdgpu_hdmi_vsdb_info *vsdb_info)
+{
+	struct amdgpu_device *adev = drm_to_adev(aconnector->base.dev);
+	bool ret;
+
+	mutex_lock(&adev->dm.dc_lock);
+	if (adev->dm.dmub_srv)
+		ret = parse_edid_cea_dmub(&adev->dm, edid_ext, len, vsdb_info);
+	else
+		ret = parse_edid_cea_dmcu(&adev->dm, edid_ext, len, vsdb_info);
+	mutex_unlock(&adev->dm.dc_lock);
+	return ret;
+}
+
+static void parse_edid_displayid_vrr(struct drm_connector *connector,
+				     const struct edid *edid)
+{
+	u8 *edid_ext = NULL;
+	int i;
+	int j = 0;
+	u16 min_vfreq;
+	u16 max_vfreq;
+
+	if (!edid || !edid->extensions)
+		return;
+
+	/* Find DisplayID extension */
+	for (i = 0; i < edid->extensions; i++) {
+		edid_ext = (void *)(edid + (i + 1));
+		if (edid_ext[0] == DISPLAYID_EXT)
+			break;
+	}
+
+	if (i == edid->extensions)
+		return;
+
+	while (j < EDID_LENGTH) {
+		/* Get dynamic video timing range from DisplayID if available */
+		if (EDID_LENGTH - j > 13 && edid_ext[j] == 0x25	&&
+		    (edid_ext[j+1] & 0xFE) == 0 && (edid_ext[j+2] == 9)) {
+			min_vfreq = edid_ext[j+9];
+			if (edid_ext[j+1] & 7)
+				max_vfreq = edid_ext[j+10] + ((edid_ext[j+11] & 3) << 8);
+			else
+				max_vfreq = edid_ext[j+10];
+
+			if (max_vfreq && min_vfreq) {
+				connector->display_info.monitor_range.max_vfreq = max_vfreq;
+				connector->display_info.monitor_range.min_vfreq = min_vfreq;
+
+				return;
+			}
+		}
+		j++;
+	}
+}
+
+static int get_amd_vsdb(struct amdgpu_dm_connector *aconnector,
+			struct amdgpu_hdmi_vsdb_info *vsdb_info)
+{
+	struct drm_connector *connector = &aconnector->base;
+
+	vsdb_info->replay_mode = connector->display_info.amd_vsdb.replay_mode;
+	vsdb_info->amd_vsdb_version = connector->display_info.amd_vsdb.version;
+
+	return connector->display_info.amd_vsdb.version != 0;
+}
+
+static int parse_hdmi_amd_vsdb(struct amdgpu_dm_connector *aconnector,
+			       const struct edid *edid,
+			       struct amdgpu_hdmi_vsdb_info *vsdb_info)
+{
+	u8 *edid_ext = NULL;
+	int i;
+	bool valid_vsdb_found = false;
+
+	/*----- drm_find_cea_extension() -----*/
+	/* No EDID or EDID extensions */
+	if (edid == NULL || edid->extensions == 0)
+		return -ENODEV;
+
+	/* Find CEA extension */
+	for (i = 0; i < edid->extensions; i++) {
+		edid_ext = (uint8_t *)edid + EDID_LENGTH * (i + 1);
+		if (edid_ext[0] == CEA_EXT)
+			break;
+	}
+
+	if (i == edid->extensions)
+		return -ENODEV;
+
+	/*----- cea_db_offsets() -----*/
+	if (edid_ext[0] != CEA_EXT)
+		return -ENODEV;
+
+	valid_vsdb_found = parse_edid_cea(aconnector, edid_ext, EDID_LENGTH, vsdb_info);
+
+	return valid_vsdb_found ? i : -ENODEV;
+}
+
+/**
+ * amdgpu_dm_update_freesync_caps - Update Freesync capabilities
+ *
+ * @connector: Connector to query.
+ * @drm_edid: DRM EDID from monitor
+ * @do_mccs: Controls whether MCCS (Monitor Control Command Set) over
+ *	      DDC (Display Data Channel) transactions are performed. When true,
+ *	      the driver queries the monitor to get or update additional FreeSync
+ *	      capability information. When false, these transactions are skipped.
+ *
+ * Amdgpu supports Freesync in DP and HDMI displays, and it is required to keep
+ * track of some of the display information in the internal data struct used by
+ * amdgpu_dm. This function checks which type of connector we need to set the
+ * FreeSync parameters.
+ */
+void amdgpu_dm_update_freesync_caps(struct drm_connector *connector,
+				    const struct drm_edid *drm_edid, bool do_mccs)
+{
+	int i = 0;
+	struct amdgpu_dm_connector *amdgpu_dm_connector =
+			to_amdgpu_dm_connector(connector);
+	struct dm_connector_state *dm_con_state = NULL;
+	struct dc_sink *sink;
+	struct amdgpu_device *adev = drm_to_adev(connector->dev);
+	struct amdgpu_hdmi_vsdb_info vsdb_info = {0};
+	const struct edid *edid;
+	bool freesync_capable = false;
+	enum adaptive_sync_type as_type = ADAPTIVE_SYNC_TYPE_NONE;
+
+	if (!connector->state) {
+		drm_err(adev_to_drm(adev), "%s - Connector has no state", __func__);
+		goto update;
+	}
+
+	sink = amdgpu_dm_connector->dc_sink ?
+		amdgpu_dm_connector->dc_sink :
+		amdgpu_dm_connector->dc_em_sink;
+
+	drm_edid_connector_update(connector, drm_edid);
+
+	if (!drm_edid || !sink) {
+		dm_con_state = to_dm_connector_state(connector->state);
+
+		amdgpu_dm_connector->min_vfreq = 0;
+		amdgpu_dm_connector->max_vfreq = 0;
+		freesync_capable = false;
+
+		goto update;
+	}
+
+	dm_con_state = to_dm_connector_state(connector->state);
+
+	if (!adev->dm.freesync_module || !dc_supports_vrr(sink->ctx->dce_version))
+		goto update;
+
+	/* FIXME: Get rid of drm_edid_raw() */
+	edid = drm_edid_raw(drm_edid);
+
+	/* Some eDP panels only have the refresh rate range info in DisplayID */
+	if ((connector->display_info.monitor_range.min_vfreq == 0 ||
+	     connector->display_info.monitor_range.max_vfreq == 0))
+		parse_edid_displayid_vrr(connector, edid);
+
+	if (edid && (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT ||
+		     sink->sink_signal == SIGNAL_TYPE_EDP)) {
+		if (amdgpu_dm_connector->dc_link &&
+		    amdgpu_dm_connector->dc_link->dpcd_caps.allow_invalid_MSA_timing_param) {
+			amdgpu_dm_connector->min_vfreq = connector->display_info.monitor_range.min_vfreq;
+			amdgpu_dm_connector->max_vfreq = connector->display_info.monitor_range.max_vfreq;
+			if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10)
+				freesync_capable = true;
+		}
+
+		get_amd_vsdb(amdgpu_dm_connector, &vsdb_info);
+
+		if (vsdb_info.replay_mode) {
+			amdgpu_dm_connector->vsdb_info.replay_mode = vsdb_info.replay_mode;
+			amdgpu_dm_connector->vsdb_info.amd_vsdb_version = vsdb_info.amd_vsdb_version;
+			amdgpu_dm_connector->as_type = ADAPTIVE_SYNC_TYPE_EDP;
+		}
+
+	} else if (drm_edid && sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A) {
+		i = parse_hdmi_amd_vsdb(amdgpu_dm_connector, edid, &vsdb_info);
+		if (i >= 0) {
+			amdgpu_dm_connector->vsdb_info = vsdb_info;
+			sink->edid_caps.freesync_vcp_code = vsdb_info.freesync_mccs_vcp_code;
+
+			if (vsdb_info.freesync_supported) {
+				amdgpu_dm_connector->min_vfreq = vsdb_info.min_refresh_rate_hz;
+				amdgpu_dm_connector->max_vfreq = vsdb_info.max_refresh_rate_hz;
+				if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10)
+					freesync_capable = true;
+
+				connector->display_info.monitor_range.min_vfreq = vsdb_info.min_refresh_rate_hz;
+				connector->display_info.monitor_range.max_vfreq = vsdb_info.max_refresh_rate_hz;
+			}
+		}
+	}
+
+	if (amdgpu_dm_connector->dc_link)
+		as_type = dm_get_adaptive_sync_support_type(amdgpu_dm_connector->dc_link);
+
+	if (as_type == FREESYNC_TYPE_PCON_IN_WHITELIST) {
+		i = parse_hdmi_amd_vsdb(amdgpu_dm_connector, edid, &vsdb_info);
+		if (i >= 0) {
+			amdgpu_dm_connector->vsdb_info = vsdb_info;
+			sink->edid_caps.freesync_vcp_code = vsdb_info.freesync_mccs_vcp_code;
+
+			if (vsdb_info.freesync_supported && vsdb_info.amd_vsdb_version > 0) {
+				amdgpu_dm_connector->pack_sdp_v1_3 = true;
+				amdgpu_dm_connector->as_type = as_type;
+
+				amdgpu_dm_connector->min_vfreq = vsdb_info.min_refresh_rate_hz;
+				amdgpu_dm_connector->max_vfreq = vsdb_info.max_refresh_rate_hz;
+				if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10)
+					freesync_capable = true;
+
+				connector->display_info.monitor_range.min_vfreq = vsdb_info.min_refresh_rate_hz;
+				connector->display_info.monitor_range.max_vfreq = vsdb_info.max_refresh_rate_hz;
+			}
+		}
+	}
+
+	/* Handle MCCS */
+	if (do_mccs)
+		dm_helpers_read_mccs_caps(adev->dm.dc->ctx, amdgpu_dm_connector->dc_link, sink);
+
+	if ((sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A ||
+		as_type == FREESYNC_TYPE_PCON_IN_WHITELIST) &&
+		(!sink->edid_caps.freesync_vcp_code ||
+		(sink->edid_caps.freesync_vcp_code && !sink->mccs_caps.freesync_supported)))
+		freesync_capable = false;
+
+	if (do_mccs && sink->mccs_caps.freesync_supported && freesync_capable)
+		dm_helpers_mccs_vcp_set(adev->dm.dc->ctx, amdgpu_dm_connector->dc_link, sink);
+
+update:
+	if (dm_con_state)
+		dm_con_state->freesync_capable = freesync_capable;
+
+	if (connector->state && amdgpu_dm_connector->dc_link && !freesync_capable &&
+	    amdgpu_dm_connector->dc_link->replay_settings.config.replay_supported) {
+		amdgpu_dm_connector->dc_link->replay_settings.config.replay_supported = false;
+		amdgpu_dm_connector->dc_link->replay_settings.replay_feature_enabled = false;
+	}
+
+	if (connector->vrr_capable_property)
+		drm_connector_set_vrr_capable_property(connector,
+						       freesync_capable);
+}
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.h
new file mode 100644
index 0000000000000..60f7ab63b782a
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.h
@@ -0,0 +1,177 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __AMDGPU_DM_CONNECTOR_H__
+#define __AMDGPU_DM_CONNECTOR_H__
+
+struct amdgpu_device;
+struct amdgpu_dm_connector;
+struct amdgpu_display_manager;
+struct amdgpu_encoder;
+struct amdgpu_i2c_adapter;
+struct dc_crtc_timing;
+struct dc_link;
+enum signal_type;
+struct dc_state;
+struct dc_stream_state;
+struct ddc_service;
+struct dm_connector_state;
+struct drm_atomic_commit;
+struct drm_device;
+struct drm_encoder_helper_funcs;
+struct drm_connector;
+struct drm_connector_state;
+struct drm_crtc;
+struct drm_device;
+struct drm_display_mode;
+struct drm_edid;
+struct drm_property;
+
+void amdgpu_dm_connector_funcs_reset(struct drm_connector *connector);
+
+struct drm_connector_state *
+amdgpu_dm_connector_atomic_duplicate_state(struct drm_connector *connector);
+
+int amdgpu_dm_connector_atomic_set_property(struct drm_connector *connector,
+					    struct drm_connector_state *connector_state,
+					    struct drm_property *property,
+					    uint64_t val);
+
+int amdgpu_dm_connector_atomic_get_property(struct drm_connector *connector,
+					    const struct drm_connector_state *state,
+					    struct drm_property *property,
+					    uint64_t *val);
+
+void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm,
+				     struct amdgpu_dm_connector *aconnector,
+				     int connector_type,
+				     struct dc_link *link,
+				     int link_index);
+
+enum drm_mode_status amdgpu_dm_connector_mode_valid(struct drm_connector *connector,
+						    const struct drm_display_mode *mode);
+
+void dm_restore_drm_connector_state(struct drm_device *dev,
+				    struct drm_connector *connector);
+
+void amdgpu_dm_update_freesync_caps(struct drm_connector *connector,
+				    const struct drm_edid *drm_edid,
+				    bool do_mccs);
+
+void amdgpu_dm_update_connector_after_detect(
+		struct amdgpu_dm_connector *aconnector);
+
+void amdgpu_dm_hdmi_cec_set_edid(struct amdgpu_dm_connector *aconnector);
+int amdgpu_dm_initialize_hdmi_connector(struct amdgpu_dm_connector *aconnector);
+
+struct drm_connector *
+amdgpu_dm_find_first_crtc_matching_connector(struct drm_atomic_commit *state,
+					     struct drm_crtc *crtc);
+
+int amdgpu_dm_convert_dc_color_depth_into_bpc(enum dc_color_depth display_color_depth);
+
+struct dc_stream_state *
+amdgpu_dm_create_validate_stream_for_sink(struct drm_connector *connector,
+				const struct drm_display_mode *drm_mode,
+				const struct dm_connector_state *dm_state,
+				const struct dc_stream_state *old_stream);
+
+int amdgpu_dm_connector_init(struct amdgpu_display_manager *dm,
+			     struct amdgpu_dm_connector *amdgpu_dm_connector,
+			     u32 link_index,
+			     struct amdgpu_encoder *amdgpu_encoder);
+
+void amdgpu_dm_s3_handle_hdmi_cec(struct drm_device *ddev, bool suspend);
+
+int amdgpu_dm_detect_mst_link_for_all_connectors(struct drm_device *dev);
+
+void amdgpu_set_panel_orientation(struct drm_connector *connector);
+
+enum dc_color_depth
+amdgpu_dm_convert_color_depth_from_display_info(const struct drm_connector *connector,
+				      bool is_y420, int requested_bpc);
+
+void amdgpu_dm_update_stream_scaling_settings(struct drm_device *dev,
+				    const struct drm_display_mode *mode,
+				    const struct dm_connector_state *dm_state,
+				    struct dc_stream_state *stream);
+
+bool amdgpu_dm_is_freesync_video_mode(const struct drm_display_mode *mode,
+			    struct amdgpu_dm_connector *aconnector);
+
+int amdgpu_dm_fill_hdr_info_packet(const struct drm_connector_state *state,
+			 struct dc_info_packet *out);
+
+enum dc_color_space
+amdgpu_dm_get_output_color_space(const struct dc_crtc_timing *dc_crtc_timing,
+		       const struct drm_connector_state *connector_state);
+
+struct drm_display_mode *
+amdgpu_dm_get_highest_refresh_rate_mode(struct amdgpu_dm_connector *aconnector,
+			      bool use_probed_modes);
+
+struct amdgpu_i2c_adapter *
+amdgpu_dm_create_i2c(struct ddc_service *ddc_service, bool oem);
+
+#define DDC_MANUFACTURERNAME_SAMSUNG 0x2D4C
+
+/* Encoder functions */
+extern const struct drm_encoder_helper_funcs amdgpu_dm_encoder_helper_funcs;
+int amdgpu_dm_get_encoder_crtc_mask(struct amdgpu_device *adev);
+int amdgpu_dm_encoder_init(struct drm_device *dev,
+			   struct amdgpu_encoder *aencoder,
+			   uint32_t link_index);
+
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+enum drm_mode_subconnector get_subconnector_type(struct dc_link *link);
+void update_subconnector_property(struct amdgpu_dm_connector *aconnector);
+void amdgpu_dm_fbc_init(struct drm_connector *connector);
+void amdgpu_dm_set_panel_type(struct amdgpu_dm_connector *aconnector);
+void amdgpu_dm_update_cacp_caps(struct amdgpu_dm_connector *aconnector);
+void fill_stream_properties_from_drm_display_mode(
+	struct dc_stream_state *stream,
+	const struct drm_display_mode *mode_in,
+	const struct drm_connector *connector,
+	const struct drm_connector_state *connector_state,
+	const struct dc_stream_state *old_stream,
+	int requested_bpc);
+enum display_content_type
+get_output_content_type(const struct drm_connector_state *connector_state);
+bool adjust_colour_depth_from_display_info(struct dc_crtc_timing *timing_out,
+					   const struct drm_display_info *info);
+
+int to_drm_connector_type(enum signal_type st, uint32_t connector_id);
+bool is_duplicate_mode(struct amdgpu_dm_connector *aconnector, struct drm_display_mode *mode);
+enum dc_aspect_ratio get_aspect_ratio(const struct drm_display_mode *mode_in);
+void copy_crtc_timing_for_drm_display_mode(const struct drm_display_mode *src_mode,
+					   struct drm_display_mode *dst_mode);
+void decide_crtc_timing_for_drm_display_mode(struct drm_display_mode *drm_mode,
+					     const struct drm_display_mode *native_mode,
+					     bool scale_enabled);
+void amdgpu_dm_set_panel_type(struct amdgpu_dm_connector *aconnector);
+void amdgpu_dm_update_cacp_caps(struct amdgpu_dm_connector *aconnector);
+#endif
+#endif /* __AMDGPU_DM_CONNECTOR_H__ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c
index 88f7cfea56240..970490c401e92 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c
@@ -87,6 +87,65 @@ bool dm_need_crc_dither(enum amdgpu_dm_pipe_crc_source src)
 }
 EXPORT_IF_KUNIT(dm_need_crc_dither);
 
+/**
+ * dm_need_dp_aux() - Does this source transition require the DP AUX handle?
+ * @source: Requested CRC source.
+ * @cur_crc_src: Current CRC source.
+ *
+ * Returns true when either the new source is DPRX-based (starting DPRX CRC),
+ * or the current source is DPRX-based and the new source is NONE (stopping it).
+ *
+ * Return: true if the DP AUX handle is needed, false otherwise.
+ */
+STATIC_IFN_KUNIT
+bool dm_need_dp_aux(enum amdgpu_dm_pipe_crc_source source,
+		    enum amdgpu_dm_pipe_crc_source cur_crc_src)
+{
+	return dm_is_crc_source_dprx(source) ||
+	       (source == AMDGPU_DM_PIPE_CRC_SOURCE_NONE && dm_is_crc_source_dprx(cur_crc_src));
+}
+EXPORT_IF_KUNIT(dm_need_dp_aux);
+
+/**
+ * dm_crc_source_should_start_dprx() - Should drm_dp_start_crc() be called?
+ * @source: Requested CRC source.
+ * @cur_crc_src: Current CRC source.
+ *
+ * True when CRC is transitioning from off to a DPRX source
+ * (!enabled && enable && is_dprx(@source)).
+ *
+ * Return: true if drm_dp_start_crc() should be called, false otherwise.
+ */
+STATIC_IFN_KUNIT
+bool dm_crc_source_should_start_dprx(enum amdgpu_dm_pipe_crc_source source,
+				      enum amdgpu_dm_pipe_crc_source cur_crc_src)
+{
+	return !amdgpu_dm_is_valid_crc_source(cur_crc_src) &&
+	       amdgpu_dm_is_valid_crc_source(source) &&
+	       dm_is_crc_source_dprx(source);
+}
+EXPORT_IF_KUNIT(dm_crc_source_should_start_dprx);
+
+/**
+ * dm_crc_source_should_stop_dprx() - Should drm_dp_stop_crc() be called?
+ * @source: Requested CRC source.
+ * @cur_crc_src: Current CRC source.
+ *
+ * True when CRC is transitioning from a DPRX source to off
+ * (enabled && !enable && is_dprx(@cur_crc_src)).
+ *
+ * Return: true if drm_dp_stop_crc() should be called, false otherwise.
+ */
+STATIC_IFN_KUNIT
+bool dm_crc_source_should_stop_dprx(enum amdgpu_dm_pipe_crc_source source,
+				     enum amdgpu_dm_pipe_crc_source cur_crc_src)
+{
+	return amdgpu_dm_is_valid_crc_source(cur_crc_src) &&
+	       !amdgpu_dm_is_valid_crc_source(source) &&
+	       dm_is_crc_source_dprx(cur_crc_src);
+}
+EXPORT_IF_KUNIT(dm_crc_source_should_stop_dprx);
+
 const char *const *amdgpu_dm_crtc_get_crc_sources(struct drm_crtc *crtc,
 						  size_t *count)
 {
@@ -496,7 +555,7 @@ amdgpu_dm_crtc_verify_crc_source(struct drm_crtc *crtc, const char *src_name,
 {
 	enum amdgpu_dm_pipe_crc_source source = dm_parse_crc_source(src_name);
 
-	if (source < 0) {
+	if (source == AMDGPU_DM_PIPE_CRC_SOURCE_INVALID) {
 		DRM_DEBUG_DRIVER("Unknown CRC source %s for CRTC%d\n",
 				 src_name, crtc->index);
 		return -EINVAL;
@@ -595,7 +654,7 @@ int amdgpu_dm_crtc_set_crc_source(struct drm_crtc *crtc, const char *src_name)
 	bool enabled = false;
 	int ret = 0;
 
-	if (source < 0) {
+	if (source == AMDGPU_DM_PIPE_CRC_SOURCE_INVALID) {
 		DRM_DEBUG_DRIVER("Unknown CRC source %s for CRTC%d\n",
 				 src_name, crtc->index);
 		return -EINVAL;
@@ -622,7 +681,7 @@ int amdgpu_dm_crtc_set_crc_source(struct drm_crtc *crtc, const char *src_name)
 		 */
 		ret = wait_for_completion_interruptible_timeout(
 			&commit->hw_done, 10 * HZ);
-		if (ret < 0)
+		if (ret < 0 && ret != -ERESTARTSYS)
 			goto cleanup;
 
 		if (ret == 0) {
@@ -650,9 +709,7 @@ int amdgpu_dm_crtc_set_crc_source(struct drm_crtc *crtc, const char *src_name)
 	 * CRTC DITHER  | XXXX        | Enable CRTC CRC, set dither
 	 * DPRX DITHER  | XXXX        | Enable DPRX CRC, need 'aux', set dither
 	 */
-	if (dm_is_crc_source_dprx(source) ||
-	    (source == AMDGPU_DM_PIPE_CRC_SOURCE_NONE &&
-	     dm_is_crc_source_dprx(cur_crc_src))) {
+	if (dm_need_dp_aux(source, cur_crc_src)) {
 		struct amdgpu_dm_connector *aconn = NULL;
 		struct drm_connector *connector;
 		struct drm_connector_list_iter conn_iter;
@@ -714,23 +771,24 @@ int amdgpu_dm_crtc_set_crc_source(struct drm_crtc *crtc, const char *src_name)
 		goto cleanup;
 	}
 
-	if (!enabled && enable) {
-		if (dm_is_crc_source_dprx(source)) {
-			if (drm_dp_start_crc(aux, crtc)) {
-				DRM_DEBUG_DRIVER("dp start crc failed\n");
-				ret = -EINVAL;
-				goto cleanup;
-			}
+	if (dm_crc_source_should_start_dprx(source, cur_crc_src)) {
+		/* !enabled && enable && is_dprx(source): CRC off → DPRX on */
+		if (drm_dp_start_crc(aux, crtc)) {
+			DRM_DEBUG_DRIVER("dp start crc failed\n");
+			ret = -EINVAL;
+			goto cleanup;
 		}
-	} else if (enabled && !enable) {
+	} else if (dm_crc_source_should_stop_dprx(source, cur_crc_src)) {
+		/* enabled && !enable && is_dprx(cur_crc_src): DPRX on → CRC off */
 		drm_crtc_vblank_put(crtc);
-		if (dm_is_crc_source_dprx(source)) {
-			if (drm_dp_stop_crc(aux)) {
-				DRM_DEBUG_DRIVER("dp stop crc failed\n");
-				ret = -EINVAL;
-				goto cleanup;
-			}
+		if (drm_dp_stop_crc(aux)) {
+			DRM_DEBUG_DRIVER("dp stop crc failed\n");
+			ret = -EINVAL;
+			goto cleanup;
 		}
+	} else if (enabled && !enable) {
+		/* Non-DPRX source (e.g. CRTC) turning off: release vblank ref */
+		drm_crtc_vblank_put(crtc);
 	}
 
 	spin_lock_irq(&drm_dev->event_lock);
@@ -767,9 +825,9 @@ void amdgpu_dm_crtc_handle_crc_irq(struct drm_crtc *crtc)
 {
 	struct dm_crtc_state *crtc_state;
 	struct dc_stream_state *stream_state;
-	struct drm_device *drm_dev = NULL;
+	struct drm_device *drm_dev;
 	enum amdgpu_dm_pipe_crc_source cur_crc_src;
-	struct amdgpu_crtc *acrtc = NULL;
+	struct amdgpu_crtc *acrtc;
 	uint32_t crcs[3];
 	unsigned long flags;
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.h
index c9aa0c82038fe..8bb8a6f6c1480 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.h
@@ -156,6 +156,12 @@ enum amdgpu_dm_pipe_crc_source dm_parse_crc_source(const char *source);
 bool dm_is_crc_source_crtc(enum amdgpu_dm_pipe_crc_source src);
 bool dm_is_crc_source_dprx(enum amdgpu_dm_pipe_crc_source src);
 bool dm_need_crc_dither(enum amdgpu_dm_pipe_crc_source src);
+bool dm_need_dp_aux(enum amdgpu_dm_pipe_crc_source source,
+		    enum amdgpu_dm_pipe_crc_source cur_crc_src);
+bool dm_crc_source_should_start_dprx(enum amdgpu_dm_pipe_crc_source source,
+				      enum amdgpu_dm_pipe_crc_source cur_crc_src);
+bool dm_crc_source_should_stop_dprx(enum amdgpu_dm_pipe_crc_source source,
+				     enum amdgpu_dm_pipe_crc_source cur_crc_src);
 #endif
 
 #endif /* AMD_DAL_DEV_AMDGPU_DM_AMDGPU_DM_CRC_H_ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c
index 3dcedaa67ed8e..d63688b9d93dd 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c
@@ -34,6 +34,7 @@
 #include "amdgpu_dm_plane.h"
 #include "amdgpu_dm_trace.h"
 #include "amdgpu_dm_debugfs.h"
+#include "amdgpu_dm_kunit_helpers.h"
 #include "modules/inc/mod_power.h"
 
 #define HPD_DETECTION_PERIOD_uS 2000000
@@ -65,6 +66,7 @@ bool amdgpu_dm_crtc_modeset_required(struct drm_crtc_state *crtc_state,
 {
 	return crtc_state->active && drm_atomic_crtc_needs_modeset(crtc_state);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_crtc_modeset_required);
 
 bool amdgpu_dm_crtc_vrr_active_irq(struct amdgpu_crtc *acrtc)
 
@@ -74,6 +76,7 @@ bool amdgpu_dm_crtc_vrr_active_irq(struct amdgpu_crtc *acrtc)
 	       acrtc->dm_irq_params.freesync_config.state ==
 		       VRR_STATE_ACTIVE_FIXED;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_crtc_vrr_active_irq);
 
 int amdgpu_dm_crtc_set_vupdate_irq(struct drm_crtc *crtc, bool enable)
 {
@@ -93,12 +96,14 @@ int amdgpu_dm_crtc_set_vupdate_irq(struct drm_crtc *crtc, bool enable)
 		      acrtc->crtc_id, enable ? "en" : "dis", rc);
 	return rc;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_crtc_set_vupdate_irq);
 
 bool amdgpu_dm_crtc_vrr_active(const struct dm_crtc_state *dm_state)
 {
 	return dm_state->freesync_config.state == VRR_STATE_ACTIVE_VARIABLE ||
 	       dm_state->freesync_config.state == VRR_STATE_ACTIVE_FIXED;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_crtc_vrr_active);
 
 /**
  * amdgpu_dm_crtc_set_static_screen_optimze() - Toggle static screen optimizations.
@@ -156,6 +161,7 @@ bool amdgpu_dm_is_headless(struct amdgpu_device *adev)
 	drm_connector_list_iter_end(&iter);
 	return is_headless;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_is_headless);
 
 static void amdgpu_dm_idle_worker(struct work_struct *work)
 {
@@ -207,6 +213,7 @@ struct idle_workqueue *idle_create_workqueue(struct amdgpu_device *adev)
 
 	return idle_work;
 }
+EXPORT_IF_KUNIT(idle_create_workqueue);
 
 static void amdgpu_dm_crtc_vblank_control_worker(struct work_struct *work)
 {
@@ -274,7 +281,14 @@ static inline int amdgpu_dm_crtc_set_vblank(struct drm_crtc *crtc, bool enable)
 			drm_crtc_vblank_restore(crtc);
 	}
 
-	if (dc_supports_vrr(dm->dc->ctx->dce_version)) {
+	/*
+	 * On DCN, VUPDATE_NO_LOCK is the single OTG interrupt used to deliver
+	 * vblank and pageflip completion events, so enable it whenever vblank
+	 * is enabled. On DCE, vupdate is only needed in VRR mode.
+	 */
+	if (amdgpu_ip_version(adev, DCE_HWIP, 0) != 0) {
+		rc = amdgpu_dm_crtc_set_vupdate_irq(crtc, enable);
+	} else if (dc_supports_vrr(dm->dc->ctx->dce_version)) {
 		if (enable) {
 			/* vblank irq on -> Only need vupdate irq in vrr mode */
 			if (amdgpu_dm_crtc_vrr_active(acrtc_state))
@@ -285,39 +299,46 @@ static inline int amdgpu_dm_crtc_set_vblank(struct drm_crtc *crtc, bool enable)
 		}
 	}
 
-	if (rc)
-		return rc;
-
-	/* crtc vblank or vstartup interrupt */
-	if (enable) {
-		rc = amdgpu_irq_get(adev, &adev->crtc_irq, irq_type);
-		drm_dbg_vbl(crtc->dev, "Get crtc_irq ret=%d\n", rc);
-	} else {
-		rc = amdgpu_irq_put(adev, &adev->crtc_irq, irq_type);
-		drm_dbg_vbl(crtc->dev, "Put crtc_irq ret=%d\n", rc);
-	}
-
 	if (rc)
 		return rc;
 
 	/*
-	 * hubp surface flip interrupt
-	 *
-	 * We have no guarantee that the frontend index maps to the same
-	 * backend index - some even map to more than one.
-	 *
-	 * TODO: Use a different interrupt or check DC itself for the mapping.
+	 * VLINE0 (crtc_irq) and GRPH_PFLIP (pageflip_irq) are only used on
+	 * DCE. On DCN, vblank and pageflip completion are delivered from
+	 * VUPDATE_NO_LOCK (enabled above), so don't touch them here.
 	 */
-	if (enable) {
-		rc = amdgpu_irq_get(adev, &adev->pageflip_irq, irq_type);
-		drm_dbg_vbl(crtc->dev, "Get pageflip_irq ret=%d\n", rc);
-	} else {
-		rc = amdgpu_irq_put(adev, &adev->pageflip_irq, irq_type);
-		drm_dbg_vbl(crtc->dev, "Put pageflip_irq ret=%d\n", rc);
-	}
+	if (amdgpu_ip_version(adev, DCE_HWIP, 0) == 0) {
+		/* crtc vblank or vstartup interrupt */
+		if (enable) {
+			rc = amdgpu_irq_get(adev, &adev->crtc_irq, irq_type);
+			drm_dbg_vbl(crtc->dev, "Get crtc_irq ret=%d\n", rc);
+		} else {
+			rc = amdgpu_irq_put(adev, &adev->crtc_irq, irq_type);
+			drm_dbg_vbl(crtc->dev, "Put crtc_irq ret=%d\n", rc);
+		}
 
-	if (rc)
-		return rc;
+		if (rc)
+			return rc;
+
+		/*
+		 * hubp surface flip interrupt
+		 *
+		 * We have no guarantee that the frontend index maps to the same
+		 * backend index - some even map to more than one.
+		 *
+		 * TODO: Use a different interrupt or check DC itself for the mapping.
+		 */
+		if (enable) {
+			rc = amdgpu_irq_get(adev, &adev->pageflip_irq, irq_type);
+			drm_dbg_vbl(crtc->dev, "Get pageflip_irq ret=%d\n", rc);
+		} else {
+			rc = amdgpu_irq_put(adev, &adev->pageflip_irq, irq_type);
+			drm_dbg_vbl(crtc->dev, "Put pageflip_irq ret=%d\n", rc);
+		}
+
+		if (rc)
+			return rc;
+	}
 
 #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
 	/* crtc vline0 interrupt, only available on DCN+ */
@@ -437,13 +458,13 @@ static void amdgpu_dm_crtc_reset_state(struct drm_crtc *crtc)
 {
 	struct dm_crtc_state *state;
 
-	if (crtc->state)
-		amdgpu_dm_crtc_destroy_state(crtc, crtc->state);
-
 	state = kzalloc_obj(*state);
-	if (WARN_ON(!state))
+	if (!state)
 		return;
 
+	if (crtc->state)
+		amdgpu_dm_crtc_destroy_state(crtc, crtc->state);
+
 	__drm_atomic_helper_crtc_reset(crtc, &state->base);
 }
 
@@ -595,12 +616,13 @@ static void amdgpu_dm_crtc_update_crtc_active_planes(struct drm_crtc *crtc,
 		amdgpu_dm_crtc_count_crtc_active_planes(new_crtc_state);
 }
 
-static bool amdgpu_dm_crtc_helper_mode_fixup(struct drm_crtc *crtc,
-				      const struct drm_display_mode *mode,
-				      struct drm_display_mode *adjusted_mode)
+STATIC_IFN_KUNIT bool amdgpu_dm_crtc_helper_mode_fixup(struct drm_crtc *crtc,
+					       const struct drm_display_mode *mode,
+					       struct drm_display_mode *adjusted_mode)
 {
 	return true;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_crtc_helper_mode_fixup);
 
 static int amdgpu_dm_crtc_helper_atomic_check(struct drm_crtc *crtc,
 					      struct drm_atomic_commit *state)
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h
index e9fb52f0e66d9..d8b004f613abc 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h
@@ -42,6 +42,12 @@ int amdgpu_dm_crtc_set_vupdate_irq(struct drm_crtc *crtc, bool enable);
 
 bool amdgpu_dm_crtc_vrr_active_irq(struct amdgpu_crtc *acrtc);
 
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+bool amdgpu_dm_crtc_helper_mode_fixup(struct drm_crtc *crtc,
+				      const struct drm_display_mode *mode,
+				      struct drm_display_mode *adjusted_mode);
+#endif
+
 bool amdgpu_dm_crtc_vrr_active(const struct dm_crtc_state *dm_state);
 
 int amdgpu_dm_crtc_enable_vblank(struct drm_crtc *crtc);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
index 7db38ad3f8487..830cf8da06b46 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
@@ -606,7 +606,7 @@ static int dp_lttpr_status_show(struct seq_file *m, void *unused)
 		break;
 	}
 
-	seq_puts(m, "\n");
+	seq_putc(m, '\n');
 	return 0;
 }
 
@@ -1081,7 +1081,7 @@ static int psr_capability_show(struct seq_file *m, void *data)
 	seq_printf(m, "Driver support: %s", str_yes_no(link->psr_settings.psr_feature_enabled));
 	if (link->psr_settings.psr_version)
 		seq_printf(m, " [0x%02x]", link->psr_settings.psr_version);
-	seq_puts(m, "\n");
+	seq_putc(m, '\n');
 
 	return 0;
 }
@@ -1266,7 +1266,7 @@ static int hdcp_sink_capability_show(struct seq_file *m, void *data)
 	if (!hdcp_cap && !hdcp2_cap)
 		seq_printf(m, "%s ", "None");
 
-	seq_puts(m, "\n");
+	seq_putc(m, '\n');
 
 	return 0;
 }
@@ -2710,11 +2710,10 @@ static int ips_status_show(struct seq_file *m, void *unused)
 		rcg_count = ips_fw->rcg_exit_count;
 		ips1_count = ips_fw->ips1_exit_count;
 		ips2_count = ips_fw->ips2_exit_count;
-		seq_printf(m, "exit counts: rcg=%u ips1=%u ips2=%u",
+		seq_printf(m, "exit counts: rcg=%u ips1=%u ips2=%u\n",
 			   rcg_count,
 			   ips1_count,
 			   ips2_count);
-		seq_puts(m, "\n");
 	}
 	return 0;
 }
@@ -2971,7 +2970,7 @@ static ssize_t hdmi_cec_state_write(struct file *f, const char __user *buf,
 		ret = amdgpu_dm_initialize_hdmi_connector(aconnector);
 		if (ret)
 			return ret;
-		hdmi_cec_set_edid(aconnector);
+		amdgpu_dm_hdmi_cec_set_edid(aconnector);
 	} else {
 		if (!aconnector->notifier)
 			return -EINVAL;
@@ -2982,6 +2981,64 @@ static ssize_t hdmi_cec_state_write(struct file *f, const char __user *buf,
 	return size;
 }
 
+/**
+ * hdmi_automation_enable - Enable/Disable HDMI automation feature
+ * @f: file structure.
+ * @buf: userspace buffer. set to '1' to enable; '0' to disable automation feature.
+ * @size: size of buffer from userpsace.
+ * @pos: unused.
+ *
+ * Return size on success, error code on failure
+ */
+static ssize_t hdmi_automation_enable(struct file *f, const char __user *buf,
+	size_t size, loff_t *pos)
+{
+	struct amdgpu_dm_connector *aconnector = file_inode(f)->i_private;
+	char *wr_buf = NULL;
+	const uint32_t wr_buf_size = 40;
+	int max_param_num = 1;
+	uint8_t param_nums = 0;
+	long param[2];
+	bool hdmi_comp_auto;
+
+	if (size == 0)
+		return -EINVAL;
+
+	wr_buf = kcalloc(wr_buf_size, sizeof(char), GFP_KERNEL);
+	if (!wr_buf)
+		return -ENOSPC;
+
+	if (parse_write_buffer_into_params(wr_buf, wr_buf_size,
+					   (long *)param, buf,
+					   max_param_num,
+					   &param_nums)) {
+		kfree(wr_buf);
+		return -EINVAL;
+	}
+
+	if (param_nums <= 0) {
+		kfree(wr_buf);
+		DRM_DEBUG_DRIVER("user data not be read\n");
+		return -EINVAL;
+	}
+
+	switch (param[0]) {
+	case 0:
+		hdmi_comp_auto = false;
+		break;
+	case 1:
+	default:
+		hdmi_comp_auto = true;
+		break;
+	}
+
+	/* Persist setting across sink re-detection/hotplug. */
+	aconnector->hdmi_comp_auto = hdmi_comp_auto;
+
+	kfree(wr_buf);
+	return size;
+}
+
 DEFINE_SHOW_ATTRIBUTE(dp_dsc_fec_support);
 DEFINE_SHOW_ATTRIBUTE(dmub_fw_state);
 DEFINE_SHOW_ATTRIBUTE(dmub_tracebuffer);
@@ -3099,6 +3156,12 @@ static const struct file_operations dp_mst_link_settings_debugfs_fops = {
 	.llseek = default_llseek
 };
 
+static const struct file_operations hdmi_automation_debugfs_fops = {
+	.owner = THIS_MODULE,
+	.write = hdmi_automation_enable,
+	.llseek = default_llseek
+};
+
 static const struct {
 	char *name;
 	const struct file_operations *fops;
@@ -3131,7 +3194,8 @@ static const struct {
 	const struct file_operations *fops;
 } hdmi_debugfs_entries[] = {
 		{"hdcp_sink_capability", &hdcp_sink_capability_fops},
-		{"hdmi_cec_state", &hdmi_cec_state_fops}
+		{"hdmi_cec_state", &hdmi_cec_state_fops},
+		{"hdmi_automation", &hdmi_automation_debugfs_fops}
 };
 
 /*
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.c
new file mode 100644
index 0000000000000..c17488e57eae1
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.c
@@ -0,0 +1,949 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services_types.h"
+#include "dc.h"
+#include "dc/inc/core_types.h"
+#include "dc/dc_dmub_srv.h"
+#include "dmub/dmub_srv.h"
+#include "dc/inc/hw/dmcu.h"
+#include "dc/inc/hw/abm.h"
+#include "dal_asic_id.h"
+
+#include "amdgpu.h"
+#include "amdgpu_display.h"
+#include "amdgpu_ucode.h"
+#include "amdgpu_dm.h"
+#include "amdgpu_dm_dmub.h"
+#include "amdgpu_dm_kunit_helpers.h"
+#include <linux/component.h>
+#include <linux/firmware.h>
+
+static_assert(AMDGPU_DMUB_NOTIFICATION_MAX == DMUB_NOTIFICATION_MAX, "AMDGPU_DMUB_NOTIFICATION_MAX mismatch");
+
+MODULE_FIRMWARE(FIRMWARE_RENOIR_DMUB);
+MODULE_FIRMWARE(FIRMWARE_SIENNA_CICHLID_DMUB);
+MODULE_FIRMWARE(FIRMWARE_NAVY_FLOUNDER_DMUB);
+MODULE_FIRMWARE(FIRMWARE_GREEN_SARDINE_DMUB);
+MODULE_FIRMWARE(FIRMWARE_VANGOGH_DMUB);
+MODULE_FIRMWARE(FIRMWARE_DIMGREY_CAVEFISH_DMUB);
+MODULE_FIRMWARE(FIRMWARE_BEIGE_GOBY_DMUB);
+MODULE_FIRMWARE(FIRMWARE_YELLOW_CARP_DMUB);
+MODULE_FIRMWARE(FIRMWARE_DCN_314_DMUB);
+MODULE_FIRMWARE(FIRMWARE_DCN_315_DMUB);
+MODULE_FIRMWARE(FIRMWARE_DCN316_DMUB);
+MODULE_FIRMWARE(FIRMWARE_DCN_V3_2_0_DMCUB);
+MODULE_FIRMWARE(FIRMWARE_DCN_V3_2_1_DMCUB);
+MODULE_FIRMWARE(FIRMWARE_DCN_35_DMUB);
+MODULE_FIRMWARE(FIRMWARE_DCN_351_DMUB);
+MODULE_FIRMWARE(FIRMWARE_DCN_36_DMUB);
+MODULE_FIRMWARE(FIRMWARE_DCN_401_DMUB);
+MODULE_FIRMWARE(FIRMWARE_DCN_42_DMUB);
+MODULE_FIRMWARE(FIRMWARE_DCN_42B_DMUB);
+
+/**
+ * dm_dmub_aux_setconfig_callback - Callback for AUX or SET_CONFIG command.
+ * @adev: amdgpu_device pointer
+ * @notify: dmub notification structure
+ *
+ * Dmub AUX or SET_CONFIG command completion processing callback
+ * Copies dmub notification to DM which is to be read by AUX command.
+ * issuing thread and also signals the event to wake up the thread.
+ */
+void dm_dmub_aux_setconfig_callback(struct amdgpu_device *adev,
+				    struct dmub_notification *notify)
+{
+	if (adev->dm.dmub_notify)
+		memcpy(adev->dm.dmub_notify, notify, sizeof(struct dmub_notification));
+	if (notify->type == DMUB_NOTIFICATION_AUX_REPLY)
+		complete(&adev->dm.dmub_aux_transfer_done);
+}
+EXPORT_IF_KUNIT(dm_dmub_aux_setconfig_callback);
+
+void dm_dmub_aux_fused_io_callback(struct amdgpu_device *adev,
+				   struct dmub_notification *notify)
+{
+	if (!adev || !notify) {
+		ASSERT(false);
+		return;
+	}
+
+	const struct dmub_cmd_fused_request *req = &notify->fused_request;
+	const uint8_t ddc_line = req->u.aux.ddc_line;
+
+	if (ddc_line >= ARRAY_SIZE(adev->dm.fused_io)) {
+		ASSERT(false);
+		return;
+	}
+
+	struct fused_io_sync *sync = &adev->dm.fused_io[ddc_line];
+
+	static_assert(sizeof(*req) <= sizeof(sync->reply_data), "Size mismatch");
+	memcpy(sync->reply_data, req, sizeof(*req));
+	complete(&sync->replied);
+}
+EXPORT_IF_KUNIT(dm_dmub_aux_fused_io_callback);
+
+/**
+ * dm_register_dmub_notify_callback - Sets callback for DMUB notify
+ * @adev: amdgpu_device pointer
+ * @type: Type of dmub notification
+ * @callback: Dmub interrupt callback function
+ * @dmub_int_thread_offload: offload indicator
+ *
+ * API to register a dmub callback handler for a dmub notification
+ * Also sets indicator whether callback processing to be offloaded.
+ * to dmub interrupt handling thread
+ * Return: true if successfully registered, false if there is existing registration
+ */
+bool dm_register_dmub_notify_callback(struct amdgpu_device *adev,
+				      enum dmub_notification_type type,
+				      dmub_notify_interrupt_callback_t callback,
+				      bool dmub_int_thread_offload)
+{
+	if (!callback || type >= ARRAY_SIZE(adev->dm.dmub_thread_offload))
+		return false;
+
+	adev->dm.dmub_callback[type] = callback;
+	adev->dm.dmub_thread_offload[type] = dmub_int_thread_offload;
+
+	return true;
+}
+EXPORT_IF_KUNIT(dm_register_dmub_notify_callback);
+
+int dm_dmub_hw_init(struct amdgpu_device *adev)
+{
+	const struct dmcub_firmware_header_v1_0 *hdr;
+	struct dmub_srv *dmub_srv = adev->dm.dmub_srv;
+	struct dmub_srv_fb_info *fb_info = adev->dm.dmub_fb_info;
+	const struct firmware *dmub_fw = adev->dm.dmub_fw;
+	struct dc *dc = adev->dm.dc;
+	struct dmcu *dmcu = adev->dm.dc->res_pool->dmcu;
+	struct abm *abm = adev->dm.dc->res_pool->abm;
+	struct dc_context *ctx = adev->dm.dc->ctx;
+	struct dmub_srv_hw_params hw_params;
+	enum dmub_status status;
+	const unsigned char *fw_inst_const, *fw_bss_data;
+	u32 fw_inst_const_size, fw_bss_data_size;
+	bool has_hw_support;
+
+	if (!dmub_srv)
+		/* DMUB isn't supported on the ASIC. */
+		return 0;
+
+	if (!fb_info) {
+		drm_err(adev_to_drm(adev), "No framebuffer info for DMUB service.\n");
+		return -EINVAL;
+	}
+
+	if (!dmub_fw) {
+		/* Firmware required for DMUB support. */
+		drm_err(adev_to_drm(adev), "No firmware provided for DMUB.\n");
+		return -EINVAL;
+	}
+
+	/* initialize register offsets for ASICs with runtime initialization available */
+	if (dmub_srv->hw_funcs.init_reg_offsets)
+		dmub_srv->hw_funcs.init_reg_offsets(dmub_srv, ctx);
+
+	status = dmub_srv_has_hw_support(dmub_srv, &has_hw_support);
+	if (status != DMUB_STATUS_OK) {
+		drm_err(adev_to_drm(adev), "Error checking HW support for DMUB: %d\n", status);
+		return -EINVAL;
+	}
+
+	if (!has_hw_support) {
+		drm_info(adev_to_drm(adev), "DMUB unsupported on ASIC\n");
+		return 0;
+	}
+
+	/* Reset DMCUB if it was previously running - before we overwrite its memory. */
+	status = dmub_srv_hw_reset(dmub_srv);
+	if (status != DMUB_STATUS_OK)
+		drm_warn(adev_to_drm(adev), "Error resetting DMUB HW: %d\n", status);
+
+	hdr = (const struct dmcub_firmware_header_v1_0 *)dmub_fw->data;
+
+	fw_inst_const = dmub_fw->data +
+			le32_to_cpu(hdr->header.ucode_array_offset_bytes) +
+			PSP_HEADER_BYTES_256;
+
+	fw_bss_data = dmub_fw->data +
+		      le32_to_cpu(hdr->header.ucode_array_offset_bytes) +
+		      le32_to_cpu(hdr->inst_const_bytes);
+
+	/* Copy firmware and bios info into FB memory. */
+	fw_inst_const_size = adev->dm.fw_inst_size;
+
+	fw_bss_data_size = le32_to_cpu(hdr->bss_data_bytes);
+
+	/* if adev->firmware.load_type == AMDGPU_FW_LOAD_PSP,
+	 * amdgpu_ucode_init_single_fw will load dmub firmware
+	 * fw_inst_const part to cw0; otherwise, the firmware back door load
+	 * will be done by dm_dmub_hw_init
+	 */
+	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
+		memcpy(fb_info->fb[DMUB_WINDOW_0_INST_CONST].cpu_addr, fw_inst_const,
+				fw_inst_const_size);
+	}
+
+	if (fw_bss_data_size)
+		memcpy(fb_info->fb[DMUB_WINDOW_2_BSS_DATA].cpu_addr,
+		       fw_bss_data, fw_bss_data_size);
+
+	/* Copy firmware bios info into FB memory. */
+	memcpy(fb_info->fb[DMUB_WINDOW_3_VBIOS].cpu_addr, adev->bios,
+	       adev->bios_size);
+
+	/* Reset regions that need to be reset. */
+	memset(fb_info->fb[DMUB_WINDOW_4_MAILBOX].cpu_addr, 0,
+	fb_info->fb[DMUB_WINDOW_4_MAILBOX].size);
+
+	memset(fb_info->fb[DMUB_WINDOW_5_TRACEBUFF].cpu_addr, 0,
+	       fb_info->fb[DMUB_WINDOW_5_TRACEBUFF].size);
+
+	memset(fb_info->fb[DMUB_WINDOW_6_FW_STATE].cpu_addr, 0,
+	       fb_info->fb[DMUB_WINDOW_6_FW_STATE].size);
+
+	memset(fb_info->fb[DMUB_WINDOW_SHARED_STATE].cpu_addr, 0,
+	       fb_info->fb[DMUB_WINDOW_SHARED_STATE].size);
+
+	/* Initialize hardware. */
+	memset(&hw_params, 0, sizeof(hw_params));
+	hw_params.soc_fb_info.fb_base = adev->gmc.fb_start;
+	hw_params.soc_fb_info.fb_offset = adev->vm_manager.vram_base_offset;
+
+	/* backdoor load firmware and trigger dmub running */
+	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
+		hw_params.load_inst_const = true;
+
+	if (dmcu)
+		hw_params.psp_version = dmcu->psp_version;
+
+	hw_params.fb_info = fb_info;
+
+	/* Enable usb4 dpia in the FW APU */
+	if (dc->caps.is_apu &&
+		dc->res_pool->usb4_dpia_count != 0 &&
+		!dc->debug.dpia_debug.bits.disable_dpia) {
+		hw_params.dpia_supported = true;
+		hw_params.disable_dpia = dc->debug.dpia_debug.bits.disable_dpia;
+		hw_params.dpia_hpd_int_enable_supported = false;
+		hw_params.enable_non_transparent_setconfig = dc->config.consolidated_dpia_dp_lt;
+		hw_params.disable_dpia_bw_allocation = !dc->config.usb4_bw_alloc_support;
+	}
+
+	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
+	case IP_VERSION(3, 5, 0):
+	case IP_VERSION(3, 5, 1):
+	case IP_VERSION(3, 6, 0):
+	case IP_VERSION(4, 2, 0):
+	case IP_VERSION(4, 2, 1):
+		hw_params.ips_sequential_ono = adev->external_rev_id > 0x10;
+		hw_params.lower_hbr3_phy_ssc = true;
+		break;
+	default:
+		break;
+	}
+
+	status = dmub_srv_hw_init(dmub_srv, &hw_params);
+	if (status != DMUB_STATUS_OK) {
+		drm_err(adev_to_drm(adev), "Error initializing DMUB HW: %d\n", status);
+		return -EINVAL;
+	}
+
+	/* Wait for firmware load to finish. */
+	status = dmub_srv_wait_for_auto_load(dmub_srv, 100000);
+	if (status != DMUB_STATUS_OK)
+		drm_warn(adev_to_drm(adev), "Wait for DMUB auto-load failed: %d\n", status);
+
+	/* Init DMCU and ABM if available. */
+	if (dmcu && abm) {
+		dmcu->funcs->dmcu_init(dmcu);
+		abm->dmcu_is_running = dmcu->funcs->is_dmcu_initialized(dmcu);
+	}
+
+	if (!adev->dm.dc->ctx->dmub_srv)
+		adev->dm.dc->ctx->dmub_srv = dc_dmub_srv_create(adev->dm.dc, dmub_srv);
+	if (!adev->dm.dc->ctx->dmub_srv) {
+		drm_err(adev_to_drm(adev), "Couldn't allocate DC DMUB server!\n");
+		return -ENOMEM;
+	}
+
+	drm_info(adev_to_drm(adev), "DMUB hardware initialized: version=0x%08X\n",
+		 adev->dm.dmcub_fw_version);
+
+	/* Keeping sanity checks off if
+	 * DCN31 >= 4.0.59.0
+	 * DCN314 >= 8.0.16.0
+	 * Otherwise, turn on sanity checks
+	 */
+	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
+	case IP_VERSION(3, 1, 2):
+	case IP_VERSION(3, 1, 3):
+		if (adev->dm.dmcub_fw_version &&
+		    adev->dm.dmcub_fw_version >= DMUB_FW_VERSION(4, 0, 0) &&
+		    adev->dm.dmcub_fw_version < DMUB_FW_VERSION(4, 0, 59))
+			adev->dm.dc->debug.sanity_checks = true;
+		break;
+	case IP_VERSION(3, 1, 4):
+		if (adev->dm.dmcub_fw_version &&
+		    adev->dm.dmcub_fw_version >= DMUB_FW_VERSION(4, 0, 0) &&
+		    adev->dm.dmcub_fw_version < DMUB_FW_VERSION(8, 0, 16))
+			adev->dm.dc->debug.sanity_checks = true;
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+EXPORT_IF_KUNIT(dm_dmub_hw_init);
+
+void dm_dmub_hw_resume(struct amdgpu_device *adev)
+{
+	struct dmub_srv *dmub_srv = adev->dm.dmub_srv;
+	enum dmub_status status;
+	bool init;
+	int r;
+
+	if (!dmub_srv) {
+		/* DMUB isn't supported on the ASIC. */
+		return;
+	}
+
+	status = dmub_srv_is_hw_init(dmub_srv, &init);
+	if (status != DMUB_STATUS_OK)
+		drm_warn(adev_to_drm(adev), "DMUB hardware init check failed: %d\n", status);
+
+	if (status == DMUB_STATUS_OK && init) {
+		/* Wait for firmware load to finish. */
+		status = dmub_srv_wait_for_auto_load(dmub_srv, 100000);
+		if (status != DMUB_STATUS_OK)
+			drm_warn(adev_to_drm(adev), "Wait for DMUB auto-load failed: %d\n", status);
+	} else {
+		/* Perform the full hardware initialization. */
+		r = dm_dmub_hw_init(adev);
+		if (r)
+			drm_err(adev_to_drm(adev), "DMUB interface failed to initialize: status=%d\n", r);
+	}
+}
+EXPORT_IF_KUNIT(dm_dmub_hw_resume);
+
+static enum dmub_status
+dm_dmub_send_vbios_gpint_command(struct amdgpu_device *adev,
+				 enum dmub_gpint_command command_code,
+				 uint16_t param,
+				 uint32_t timeout_us)
+{
+	union dmub_gpint_data_register reg, test;
+	uint32_t i;
+
+	/* Assume that VBIOS DMUB is ready to take commands */
+
+	reg.bits.status = 1;
+	reg.bits.command_code = command_code;
+	reg.bits.param = param;
+
+	cgs_write_register(adev->dm.cgs_device, 0x34c0 + 0x01f8, reg.all);
+
+	for (i = 0; i < timeout_us; ++i) {
+		udelay(1);
+
+		/* Check if our GPINT got acked */
+		reg.bits.status = 0;
+		test = (union dmub_gpint_data_register)
+			cgs_read_register(adev->dm.cgs_device, 0x34c0 + 0x01f8);
+
+		if (test.all == reg.all)
+			return DMUB_STATUS_OK;
+	}
+
+	return DMUB_STATUS_TIMEOUT;
+}
+
+static void *dm_dmub_get_vbios_bounding_box(struct amdgpu_device *adev)
+{
+	void *bb;
+	long long addr;
+	unsigned int bb_size;
+	int i = 0;
+	uint16_t chunk;
+	enum dmub_gpint_command send_addrs[] = {
+		DMUB_GPINT__SET_BB_ADDR_WORD0,
+		DMUB_GPINT__SET_BB_ADDR_WORD1,
+		DMUB_GPINT__SET_BB_ADDR_WORD2,
+		DMUB_GPINT__SET_BB_ADDR_WORD3,
+	};
+	enum dmub_status ret;
+
+	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
+	case IP_VERSION(4, 0, 1):
+		bb_size = sizeof(struct dml2_soc_bb);
+		break;
+	case IP_VERSION(4, 2, 0):
+	case IP_VERSION(4, 2, 1):
+		bb_size = sizeof(struct dml2_soc_bb);
+		break;
+	default:
+		return NULL;
+	}
+
+	bb =  dm_allocate_gpu_mem(adev,
+				  DC_MEM_ALLOC_TYPE_GART,
+				  bb_size,
+				  &addr);
+	if (!bb)
+		return NULL;
+
+	for (i = 0; i < 4; i++) {
+		/* Extract 16-bit chunk */
+		chunk = ((uint64_t) addr >> (i * 16)) & 0xFFFF;
+		/* Send the chunk */
+		ret = dm_dmub_send_vbios_gpint_command(adev, send_addrs[i], chunk, 30000);
+		if (ret != DMUB_STATUS_OK)
+			goto free_bb;
+	}
+
+	/* Now ask DMUB to copy the bb */
+	ret = dm_dmub_send_vbios_gpint_command(adev, DMUB_GPINT__BB_COPY, 1, 200000);
+	if (ret != DMUB_STATUS_OK)
+		goto free_bb;
+
+	return bb;
+
+free_bb:
+	dm_free_gpu_mem(adev, DC_MEM_ALLOC_TYPE_GART, (void *) bb);
+	return NULL;
+
+}
+
+enum dmub_ips_disable_type dm_get_default_ips_mode(
+	struct amdgpu_device *adev)
+{
+	enum dmub_ips_disable_type ret = DMUB_IPS_ENABLE;
+
+	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
+	case IP_VERSION(3, 5, 0):
+	case IP_VERSION(3, 6, 0):
+	case IP_VERSION(3, 5, 1):
+		ret =  DMUB_IPS_RCG_IN_ACTIVE_IPS2_IN_OFF;
+		break;
+	default:
+		/* ASICs older than DCN35 do not have IPSs */
+		if (amdgpu_ip_version(adev, DCE_HWIP, 0) < IP_VERSION(3, 5, 0))
+			ret = DMUB_IPS_DISABLE_ALL;
+		break;
+	}
+
+	return ret;
+}
+EXPORT_IF_KUNIT(dm_get_default_ips_mode);
+
+static uint32_t amdgpu_dm_dmub_reg_read(void *ctx, uint32_t address)
+{
+	struct amdgpu_device *adev = ctx;
+
+	return dm_read_reg(adev->dm.dc->ctx, address);
+}
+
+static void amdgpu_dm_dmub_reg_write(void *ctx, uint32_t address,
+				     uint32_t value)
+{
+	struct amdgpu_device *adev = ctx;
+
+	return dm_write_reg(adev->dm.dc->ctx, address, value);
+}
+
+int dm_dmub_sw_init(struct amdgpu_device *adev)
+{
+	struct dmub_srv_create_params create_params;
+	struct dmub_srv_fw_meta_info_params fw_meta_info_params;
+	struct dmub_srv_region_params region_params;
+	struct dmub_srv_region_info region_info;
+	struct dmub_srv_memory_params memory_params;
+	struct dmub_fw_meta_info fw_info;
+	struct dmub_srv_fb_info *fb_info;
+	struct dmub_srv *dmub_srv;
+	const struct dmcub_firmware_header_v1_0 *hdr;
+	enum dmub_asic dmub_asic;
+	enum dmub_status status;
+	static enum dmub_window_memory_type window_memory_type[DMUB_WINDOW_TOTAL] = {
+		DMUB_WINDOW_MEMORY_TYPE_FB,		/* DMUB_WINDOW_0_INST_CONST */
+		DMUB_WINDOW_MEMORY_TYPE_FB,		/* DMUB_WINDOW_1_STACK */
+		DMUB_WINDOW_MEMORY_TYPE_FB,		/* DMUB_WINDOW_2_BSS_DATA */
+		DMUB_WINDOW_MEMORY_TYPE_FB,		/* DMUB_WINDOW_3_VBIOS */
+		DMUB_WINDOW_MEMORY_TYPE_FB,		/* DMUB_WINDOW_4_MAILBOX */
+		DMUB_WINDOW_MEMORY_TYPE_FB,		/* DMUB_WINDOW_5_TRACEBUFF */
+		DMUB_WINDOW_MEMORY_TYPE_FB,		/* DMUB_WINDOW_6_FW_STATE */
+		DMUB_WINDOW_MEMORY_TYPE_FB,		/* DMUB_WINDOW_7_SCRATCH_MEM */
+		DMUB_WINDOW_MEMORY_TYPE_FB,		/* DMUB_WINDOW_IB_MEM */
+		DMUB_WINDOW_MEMORY_TYPE_FB,		/* DMUB_WINDOW_SHARED_STATE */
+		DMUB_WINDOW_MEMORY_TYPE_FB,		/* DMUB_WINDOW_LSDMA_BUFFER */
+		DMUB_WINDOW_MEMORY_TYPE_FB,		/* DMUB_WINDOW_CURSOR_OFFLOAD */
+	};
+	int r;
+	int mem_domain = AMDGPU_GEM_DOMAIN_GTT;
+
+	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
+	case IP_VERSION(2, 1, 0):
+		dmub_asic = DMUB_ASIC_DCN21;
+		break;
+	case IP_VERSION(3, 0, 0):
+		dmub_asic = DMUB_ASIC_DCN30;
+		break;
+	case IP_VERSION(3, 0, 1):
+		dmub_asic = DMUB_ASIC_DCN301;
+		break;
+	case IP_VERSION(3, 0, 2):
+		dmub_asic = DMUB_ASIC_DCN302;
+		break;
+	case IP_VERSION(3, 0, 3):
+		dmub_asic = DMUB_ASIC_DCN303;
+		break;
+	case IP_VERSION(3, 1, 2):
+	case IP_VERSION(3, 1, 3):
+		dmub_asic = (adev->external_rev_id == YELLOW_CARP_B0) ? DMUB_ASIC_DCN31B : DMUB_ASIC_DCN31;
+		break;
+	case IP_VERSION(3, 1, 4):
+		dmub_asic = DMUB_ASIC_DCN314;
+		break;
+	case IP_VERSION(3, 1, 5):
+		dmub_asic = DMUB_ASIC_DCN315;
+		break;
+	case IP_VERSION(3, 1, 6):
+		dmub_asic = DMUB_ASIC_DCN316;
+		break;
+	case IP_VERSION(3, 2, 0):
+		dmub_asic = DMUB_ASIC_DCN32;
+		break;
+	case IP_VERSION(3, 2, 1):
+		dmub_asic = DMUB_ASIC_DCN321;
+		break;
+	case IP_VERSION(3, 5, 0):
+	case IP_VERSION(3, 5, 1):
+		dmub_asic = DMUB_ASIC_DCN35;
+		break;
+	case IP_VERSION(3, 6, 0):
+		dmub_asic = DMUB_ASIC_DCN36;
+		break;
+	case IP_VERSION(4, 0, 1):
+		dmub_asic = DMUB_ASIC_DCN401;
+		break;
+	case IP_VERSION(4, 2, 0):
+		dmub_asic = DMUB_ASIC_DCN42;
+		break;
+	case IP_VERSION(4, 2, 1):
+		dmub_asic = DMUB_ASIC_DCN42B;
+		break;
+	default:
+		/* ASIC doesn't support DMUB. */
+		return 0;
+	}
+
+	hdr = (const struct dmcub_firmware_header_v1_0 *)adev->dm.dmub_fw->data;
+	adev->dm.dmcub_fw_version = le32_to_cpu(hdr->header.ucode_version);
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+		adev->firmware.ucode[AMDGPU_UCODE_ID_DMCUB].ucode_id =
+			AMDGPU_UCODE_ID_DMCUB;
+		adev->firmware.ucode[AMDGPU_UCODE_ID_DMCUB].fw =
+			adev->dm.dmub_fw;
+		adev->firmware.fw_size +=
+			ALIGN(le32_to_cpu(hdr->inst_const_bytes), PAGE_SIZE);
+
+		drm_info(adev_to_drm(adev), "Loading DMUB firmware via PSP: version=0x%08X\n",
+			 adev->dm.dmcub_fw_version);
+	}
+
+
+	adev->dm.dmub_srv = kzalloc_obj(*adev->dm.dmub_srv);
+	dmub_srv = adev->dm.dmub_srv;
+
+	if (!dmub_srv) {
+		drm_err(adev_to_drm(adev), "Failed to allocate DMUB service!\n");
+		return -ENOMEM;
+	}
+
+	memset(&create_params, 0, sizeof(create_params));
+	create_params.user_ctx = adev;
+	create_params.funcs.reg_read = amdgpu_dm_dmub_reg_read;
+	create_params.funcs.reg_write = amdgpu_dm_dmub_reg_write;
+	create_params.asic = dmub_asic;
+
+	/* Create the DMUB service. */
+	status = dmub_srv_create(dmub_srv, &create_params);
+	if (status != DMUB_STATUS_OK) {
+		drm_err(adev_to_drm(adev), "Error creating DMUB service: %d\n", status);
+		return -EINVAL;
+	}
+
+	/* Extract the FW meta info. */
+	memset(&fw_meta_info_params, 0, sizeof(fw_meta_info_params));
+
+	fw_meta_info_params.inst_const_size = le32_to_cpu(hdr->inst_const_bytes) -
+					      PSP_HEADER_BYTES_256;
+	fw_meta_info_params.bss_data_size = le32_to_cpu(hdr->bss_data_bytes);
+	fw_meta_info_params.fw_inst_const = adev->dm.dmub_fw->data +
+					    le32_to_cpu(hdr->header.ucode_array_offset_bytes) +
+					    PSP_HEADER_BYTES_256;
+	fw_meta_info_params.fw_bss_data = fw_meta_info_params.bss_data_size ? adev->dm.dmub_fw->data +
+					  le32_to_cpu(hdr->header.ucode_array_offset_bytes) +
+					  le32_to_cpu(hdr->inst_const_bytes) : NULL;
+	fw_meta_info_params.custom_psp_footer_size = 0;
+
+	status = dmub_srv_get_fw_meta_info_from_raw_fw(&fw_meta_info_params, &fw_info);
+	if (status != DMUB_STATUS_OK) {
+		/* Skip returning early, just log the error. */
+		drm_err(adev_to_drm(adev), "Error getting DMUB FW meta info: %d\n", status);
+	}
+
+	/* Calculate the size of all the regions for the DMUB service. */
+	memset(&region_params, 0, sizeof(region_params));
+
+	region_params.inst_const_size = fw_meta_info_params.inst_const_size;
+	region_params.bss_data_size = fw_meta_info_params.bss_data_size;
+	region_params.vbios_size = adev->bios_size;
+	region_params.fw_bss_data = fw_meta_info_params.fw_bss_data;
+	region_params.fw_inst_const = fw_meta_info_params.fw_inst_const;
+	region_params.window_memory_type = window_memory_type;
+	region_params.fw_info = (status == DMUB_STATUS_OK) ? &fw_info : NULL;
+
+	status = dmub_srv_calc_region_info(dmub_srv, &region_params,
+					   &region_info);
+
+	if (status != DMUB_STATUS_OK) {
+		drm_err(adev_to_drm(adev), "Error calculating DMUB region info: %d\n", status);
+		return -EINVAL;
+	}
+
+	/* Limit to allocate dmub to GTT on DCN32/1
+	 * TODO: Other asics with GDDR7 may have worse latency
+	 */
+	if (dmub_asic != DMUB_ASIC_DCN32 &&
+	    dmub_asic != DMUB_ASIC_DCN321)
+		mem_domain |= AMDGPU_GEM_DOMAIN_VRAM;
+
+	/*
+	 * Allocate a framebuffer based on the total size of all the regions.
+	 * TODO: Move this into GART.
+	 */
+	r = amdgpu_bo_create_kernel(adev, region_info.fb_size, PAGE_SIZE,
+				    mem_domain,
+				    &adev->dm.dmub_bo,
+				    &adev->dm.dmub_bo_gpu_addr,
+				    &adev->dm.dmub_bo_cpu_addr);
+	if (r)
+		return r;
+
+	/* Rebase the regions on the framebuffer address. */
+	memset(&memory_params, 0, sizeof(memory_params));
+	memory_params.cpu_fb_addr = adev->dm.dmub_bo_cpu_addr;
+	memory_params.gpu_fb_addr = adev->dm.dmub_bo_gpu_addr;
+	memory_params.region_info = &region_info;
+	memory_params.window_memory_type = window_memory_type;
+
+	adev->dm.dmub_fb_info = kzalloc_obj(*adev->dm.dmub_fb_info);
+	fb_info = adev->dm.dmub_fb_info;
+
+	if (!fb_info) {
+		drm_err(adev_to_drm(adev),
+			"Failed to allocate framebuffer info for DMUB service!\n");
+		return -ENOMEM;
+	}
+
+	status = dmub_srv_calc_mem_info(dmub_srv, &memory_params, fb_info);
+	if (status != DMUB_STATUS_OK) {
+		drm_err(adev_to_drm(adev), "Error calculating DMUB FB info: %d\n", status);
+		return -EINVAL;
+	}
+
+	adev->dm.bb_from_dmub = dm_dmub_get_vbios_bounding_box(adev);
+	adev->dm.fw_inst_size = fw_meta_info_params.inst_const_size;
+
+	return 0;
+}
+EXPORT_IF_KUNIT(dm_dmub_sw_init);
+
+int dm_init_microcode(struct amdgpu_device *adev)
+{
+	char *fw_name_dmub;
+	int r;
+
+	switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
+	case IP_VERSION(2, 1, 0):
+		fw_name_dmub = FIRMWARE_RENOIR_DMUB;
+		if (ASICREV_IS_GREEN_SARDINE(adev->external_rev_id))
+			fw_name_dmub = FIRMWARE_GREEN_SARDINE_DMUB;
+		break;
+	case IP_VERSION(3, 0, 0):
+		if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(10, 3, 0))
+			fw_name_dmub = FIRMWARE_SIENNA_CICHLID_DMUB;
+		else
+			fw_name_dmub = FIRMWARE_NAVY_FLOUNDER_DMUB;
+		break;
+	case IP_VERSION(3, 0, 1):
+		fw_name_dmub = FIRMWARE_VANGOGH_DMUB;
+		break;
+	case IP_VERSION(3, 0, 2):
+		fw_name_dmub = FIRMWARE_DIMGREY_CAVEFISH_DMUB;
+		break;
+	case IP_VERSION(3, 0, 3):
+		fw_name_dmub = FIRMWARE_BEIGE_GOBY_DMUB;
+		break;
+	case IP_VERSION(3, 1, 2):
+	case IP_VERSION(3, 1, 3):
+		fw_name_dmub = FIRMWARE_YELLOW_CARP_DMUB;
+		break;
+	case IP_VERSION(3, 1, 4):
+		fw_name_dmub = FIRMWARE_DCN_314_DMUB;
+		break;
+	case IP_VERSION(3, 1, 5):
+		fw_name_dmub = FIRMWARE_DCN_315_DMUB;
+		break;
+	case IP_VERSION(3, 1, 6):
+		fw_name_dmub = FIRMWARE_DCN316_DMUB;
+		break;
+	case IP_VERSION(3, 2, 0):
+		fw_name_dmub = FIRMWARE_DCN_V3_2_0_DMCUB;
+		break;
+	case IP_VERSION(3, 2, 1):
+		fw_name_dmub = FIRMWARE_DCN_V3_2_1_DMCUB;
+		break;
+	case IP_VERSION(3, 5, 0):
+		fw_name_dmub = FIRMWARE_DCN_35_DMUB;
+		break;
+	case IP_VERSION(3, 5, 1):
+		fw_name_dmub = FIRMWARE_DCN_351_DMUB;
+		break;
+	case IP_VERSION(3, 6, 0):
+		fw_name_dmub = FIRMWARE_DCN_36_DMUB;
+		break;
+	case IP_VERSION(4, 0, 1):
+		fw_name_dmub = FIRMWARE_DCN_401_DMUB;
+		break;
+	case IP_VERSION(4, 2, 0):
+		fw_name_dmub = FIRMWARE_DCN_42_DMUB;
+		break;
+	case IP_VERSION(4, 2, 1):
+		fw_name_dmub = FIRMWARE_DCN_42B_DMUB;
+		break;
+	default:
+		/* ASIC doesn't support DMUB. */
+		return 0;
+	}
+	r = amdgpu_ucode_request(adev, &adev->dm.dmub_fw, AMDGPU_UCODE_REQUIRED,
+				 "%s", fw_name_dmub);
+	return r;
+}
+EXPORT_IF_KUNIT(dm_init_microcode);
+
+int amdgpu_dm_process_dmub_aux_transfer_sync(
+		struct dc_context *ctx,
+		unsigned int link_index,
+		struct aux_payload *payload,
+		enum aux_return_code_type *operation_result)
+{
+	struct amdgpu_device *adev = ctx->driver_context;
+	struct dmub_notification *p_notify = adev->dm.dmub_notify;
+	int ret = -1;
+
+	mutex_lock(&adev->dm.dpia_aux_lock);
+	if (!dc_process_dmub_aux_transfer_async(ctx->dc, link_index, payload)) {
+		*operation_result = AUX_RET_ERROR_ENGINE_ACQUIRE;
+		goto out;
+	}
+
+	if (!wait_for_completion_timeout(&adev->dm.dmub_aux_transfer_done, 10 * HZ)) {
+		drm_err(adev_to_drm(adev), "wait_for_completion_timeout timeout!");
+		*operation_result = AUX_RET_ERROR_TIMEOUT;
+		goto out;
+	}
+
+	if (p_notify->result != AUX_RET_SUCCESS) {
+		/*
+		 * Transient states before tunneling is enabled could
+		 * lead to this error. We can ignore this for now.
+		 */
+		if (p_notify->result == AUX_RET_ERROR_PROTOCOL_ERROR) {
+			drm_warn(adev_to_drm(adev), "DPIA AUX failed on 0x%x(%d), error %d\n",
+					payload->address, payload->length,
+					p_notify->result);
+		}
+		*operation_result = p_notify->result;
+		goto out;
+	}
+
+	payload->reply[0] = adev->dm.dmub_notify->aux_reply.command & 0xF;
+	if (adev->dm.dmub_notify->aux_reply.command & 0xF0)
+		/* The reply is stored in the top nibble of the command. */
+		payload->reply[0] = (adev->dm.dmub_notify->aux_reply.command >> 4) & 0xF;
+
+	/*write req may receive a byte indicating partially written number as well*/
+	if (p_notify->aux_reply.length && payload->data) {
+		/* Bound the reply to the scratch buffer it was read into. */
+		ret = min_t(uint32_t, p_notify->aux_reply.length,
+			    sizeof(p_notify->aux_reply.data));
+
+		/*
+		 * During a write-status-update retry the caller zeroes
+		 * payload->length while still expecting the partial-write
+		 * status byte in payload->data (see dce_aux_transfer_with_retries),
+		 * so only clamp to payload->length for regular transfers.
+		 */
+		if (!payload->write_status_update)
+			ret = min_t(int, ret, payload->length);
+
+		memcpy(payload->data, p_notify->aux_reply.data, ret);
+	} else {
+		/* success */
+		ret = p_notify->aux_reply.length;
+	}
+
+	*operation_result = p_notify->result;
+out:
+	reinit_completion(&adev->dm.dmub_aux_transfer_done);
+	mutex_unlock(&adev->dm.dpia_aux_lock);
+	return ret;
+}
+
+static void abort_fused_io(
+		struct dc_context *ctx,
+		const struct dmub_cmd_fused_request *request
+)
+{
+	union dmub_rb_cmd command = { 0 };
+	struct dmub_rb_cmd_fused_io *io = &command.fused_io;
+
+	io->header.type = DMUB_CMD__FUSED_IO;
+	io->header.sub_type = DMUB_CMD__FUSED_IO_ABORT;
+	io->header.payload_bytes = sizeof(*io) - sizeof(io->header);
+	io->request = *request;
+	dm_execute_dmub_cmd(ctx, &command, DM_DMUB_WAIT_TYPE_NO_WAIT);
+}
+
+static bool execute_fused_io(
+		struct amdgpu_device *dev,
+		struct dc_context *ctx,
+		union dmub_rb_cmd *commands,
+		uint8_t count,
+		uint32_t timeout_us
+)
+{
+	const uint8_t ddc_line = commands[0].fused_io.request.u.aux.ddc_line;
+
+	if (ddc_line >= ARRAY_SIZE(dev->dm.fused_io))
+		return false;
+
+	struct fused_io_sync *sync = &dev->dm.fused_io[ddc_line];
+	struct dmub_rb_cmd_fused_io *first = &commands[0].fused_io;
+	const bool result = dm_execute_dmub_cmd_list(ctx, count, commands, DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY)
+			&& first->header.ret_status
+			&& first->request.status == FUSED_REQUEST_STATUS_SUCCESS;
+
+	if (!result)
+		return false;
+
+	while (wait_for_completion_timeout(&sync->replied, usecs_to_jiffies(timeout_us))) {
+		reinit_completion(&sync->replied);
+
+		struct dmub_cmd_fused_request *reply = (struct dmub_cmd_fused_request *) sync->reply_data;
+
+		static_assert(sizeof(*reply) <= sizeof(sync->reply_data), "Size mismatch");
+
+		if (reply->identifier == first->request.identifier) {
+			first->request = *reply;
+			return true;
+		}
+	}
+
+	reinit_completion(&sync->replied);
+	first->request.status = FUSED_REQUEST_STATUS_TIMEOUT;
+	abort_fused_io(ctx, &first->request);
+	return false;
+}
+
+bool amdgpu_dm_execute_fused_io(
+		struct amdgpu_device *dev,
+		struct dc_link *link,
+		union dmub_rb_cmd *commands,
+		uint8_t count,
+		uint32_t timeout_us)
+{
+	struct amdgpu_display_manager *dm = &dev->dm;
+
+	mutex_lock(&dm->dpia_aux_lock);
+
+	const bool result = execute_fused_io(dev, link->ctx, commands, count, timeout_us);
+
+	mutex_unlock(&dm->dpia_aux_lock);
+	return result;
+}
+
+int amdgpu_dm_process_dmub_set_config_sync(
+		struct dc_context *ctx,
+		unsigned int link_index,
+		struct set_config_cmd_payload *payload,
+		enum set_config_status *operation_result)
+{
+	struct amdgpu_device *adev = ctx->driver_context;
+	bool is_cmd_complete;
+	int ret;
+
+	mutex_lock(&adev->dm.dpia_aux_lock);
+	is_cmd_complete = dc_process_dmub_set_config_async(ctx->dc,
+			link_index, payload, adev->dm.dmub_notify);
+
+	if (is_cmd_complete || wait_for_completion_timeout(&adev->dm.dmub_aux_transfer_done, 10 * HZ)) {
+		ret = 0;
+		*operation_result = adev->dm.dmub_notify->sc_status;
+	} else {
+		drm_err(adev_to_drm(adev), "wait_for_completion_timeout timeout!");
+		ret = -1;
+		*operation_result = SET_CONFIG_UNKNOWN_ERROR;
+	}
+
+	if (!is_cmd_complete)
+		reinit_completion(&adev->dm.dmub_aux_transfer_done);
+	mutex_unlock(&adev->dm.dpia_aux_lock);
+	return ret;
+}
+
+bool dm_execute_dmub_cmd(const struct dc_context *ctx, union dmub_rb_cmd *cmd, enum dm_dmub_wait_type wait_type)
+{
+	struct amdgpu_device *adev = ctx->driver_context;
+
+	guard(spinlock_irqsave)(&adev->dm.dmub_lock);
+	return dc_dmub_srv_cmd_run(ctx->dmub_srv, cmd, wait_type);
+}
+
+bool dm_execute_dmub_cmd_list(const struct dc_context *ctx, unsigned int count, union dmub_rb_cmd *cmd, enum dm_dmub_wait_type wait_type)
+{
+	struct amdgpu_device *adev = ctx->driver_context;
+
+	guard(spinlock_irqsave)(&adev->dm.dmub_lock);
+	return dc_dmub_srv_cmd_run_list(ctx->dmub_srv, count, cmd, wait_type);
+}
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.h
new file mode 100644
index 0000000000000..a4a03e40ec373
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.h
@@ -0,0 +1,68 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef AMDGPU_DM_AMDGPU_DM_DMUB_H_
+#define AMDGPU_DM_AMDGPU_DM_DMUB_H_
+
+#include "amdgpu.h"
+
+void dm_dmub_aux_setconfig_callback(struct amdgpu_device *adev,
+				    struct dmub_notification *notify);
+void dm_dmub_aux_fused_io_callback(struct amdgpu_device *adev,
+				   struct dmub_notification *notify);
+bool dm_register_dmub_notify_callback(struct amdgpu_device *adev,
+				      enum dmub_notification_type type,
+				      dmub_notify_interrupt_callback_t callback,
+				      bool dmub_int_thread_offload);
+int dm_dmub_hw_init(struct amdgpu_device *adev);
+void dm_dmub_hw_resume(struct amdgpu_device *adev);
+enum dmub_ips_disable_type dm_get_default_ips_mode(struct amdgpu_device *adev);
+int dm_dmub_sw_init(struct amdgpu_device *adev);
+int dm_init_microcode(struct amdgpu_device *adev);
+
+#define FIRMWARE_RENOIR_DMUB		"amdgpu/renoir_dmcub.bin"
+#define FIRMWARE_SIENNA_CICHLID_DMUB	"amdgpu/sienna_cichlid_dmcub.bin"
+#define FIRMWARE_NAVY_FLOUNDER_DMUB	"amdgpu/navy_flounder_dmcub.bin"
+#define FIRMWARE_GREEN_SARDINE_DMUB	"amdgpu/green_sardine_dmcub.bin"
+#define FIRMWARE_VANGOGH_DMUB		"amdgpu/vangogh_dmcub.bin"
+#define FIRMWARE_DIMGREY_CAVEFISH_DMUB	"amdgpu/dimgrey_cavefish_dmcub.bin"
+#define FIRMWARE_BEIGE_GOBY_DMUB	"amdgpu/beige_goby_dmcub.bin"
+#define FIRMWARE_YELLOW_CARP_DMUB	"amdgpu/yellow_carp_dmcub.bin"
+#define FIRMWARE_DCN_314_DMUB		"amdgpu/dcn_3_1_4_dmcub.bin"
+#define FIRMWARE_DCN_315_DMUB		"amdgpu/dcn_3_1_5_dmcub.bin"
+#define FIRMWARE_DCN316_DMUB		"amdgpu/dcn_3_1_6_dmcub.bin"
+#define FIRMWARE_DCN_V3_2_0_DMCUB	"amdgpu/dcn_3_2_0_dmcub.bin"
+#define FIRMWARE_DCN_V3_2_1_DMCUB	"amdgpu/dcn_3_2_1_dmcub.bin"
+#define FIRMWARE_DCN_35_DMUB		"amdgpu/dcn_3_5_dmcub.bin"
+#define FIRMWARE_DCN_351_DMUB		"amdgpu/dcn_3_5_1_dmcub.bin"
+#define FIRMWARE_DCN_36_DMUB		"amdgpu/dcn_3_6_dmcub.bin"
+#define FIRMWARE_DCN_401_DMUB		"amdgpu/dcn_4_0_1_dmcub.bin"
+#define FIRMWARE_DCN_42_DMUB		"amdgpu/dcn_4_2_dmcub.bin"
+#define FIRMWARE_DCN_42B_DMUB		"amdgpu/dcn_4_2_1_dmcub.bin"
+#define FIRMWARE_RAVEN_DMCU		"amdgpu/raven_dmcu.bin"
+#define FIRMWARE_NAVI12_DMCU		"amdgpu/navi12_dmcu.bin"
+
+#endif /* AMDGPU_DM_AMDGPU_DM_DMUB_H_ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c
index 4c164ae4a4f9c..5f43f6a74ffd8 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c
@@ -182,6 +182,70 @@ void process_output(struct hdcp_workqueue *hdcp_work)
 }
 EXPORT_IF_KUNIT(process_output);
 
+STATIC_IFN_KUNIT
+bool hdcp_get_content_protection_from_status(
+	unsigned int hdcp_content_type,
+	enum mod_hdcp_encryption_status encryption_status,
+	unsigned int *content_protection)
+{
+	if (encryption_status == MOD_HDCP_ENCRYPTION_STATUS_HDCP_OFF) {
+		*content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED;
+		return true;
+	}
+
+	if (hdcp_content_type == DRM_MODE_HDCP_CONTENT_TYPE0 &&
+	    encryption_status <= MOD_HDCP_ENCRYPTION_STATUS_HDCP2_TYPE0_ON) {
+		*content_protection = DRM_MODE_CONTENT_PROTECTION_ENABLED;
+		return true;
+	}
+
+	if (hdcp_content_type == DRM_MODE_HDCP_CONTENT_TYPE1 &&
+	    encryption_status == MOD_HDCP_ENCRYPTION_STATUS_HDCP2_TYPE1_ON) {
+		*content_protection = DRM_MODE_CONTENT_PROTECTION_ENABLED;
+		return true;
+	}
+
+	return false;
+}
+EXPORT_IF_KUNIT(hdcp_get_content_protection_from_status);
+
+STATIC_IFN_KUNIT
+void hdcp_get_link_display_adjustments(
+	bool enable_encryption,
+	u8 content_type,
+	bool fused_io_supported,
+	bool hdcp_lc_force_fw_enable,
+	bool hdcp_lc_enable_sw_fallback,
+	struct mod_hdcp_link_adjustment *link_adjust,
+	struct mod_hdcp_display_adjustment *display_adjust)
+{
+	memset(link_adjust, 0, sizeof(*link_adjust));
+	memset(display_adjust, 0, sizeof(*display_adjust));
+
+	if (!enable_encryption) {
+		display_adjust->disable =
+			MOD_HDCP_DISPLAY_DISABLE_AUTHENTICATION;
+		return;
+	}
+
+	display_adjust->disable = MOD_HDCP_DISPLAY_NOT_DISABLE;
+	link_adjust->auth_delay = 2;
+	link_adjust->retry_limit = MAX_NUM_OF_ATTEMPTS;
+
+	if (content_type == DRM_MODE_HDCP_CONTENT_TYPE0) {
+		link_adjust->hdcp2.force_type = MOD_HDCP_FORCE_TYPE_0;
+	} else if (content_type == DRM_MODE_HDCP_CONTENT_TYPE1) {
+		link_adjust->hdcp1.disable = 1;
+		link_adjust->hdcp2.force_type = MOD_HDCP_FORCE_TYPE_1;
+	}
+
+	link_adjust->hdcp2.use_fw_locality_check =
+		fused_io_supported || hdcp_lc_force_fw_enable;
+	link_adjust->hdcp2.use_sw_locality_fallback =
+		hdcp_lc_enable_sw_fallback;
+}
+EXPORT_IF_KUNIT(hdcp_get_link_display_adjustments);
+
 static void link_lock(struct hdcp_workqueue *work, bool lock)
 {
 	int i = 0;
@@ -194,6 +258,30 @@ static void link_lock(struct hdcp_workqueue *work, bool lock)
 	}
 }
 
+STATIC_IFN_KUNIT
+void hdcp_update_display_encryption_control(struct hdcp_workqueue *hdcp_work,
+					    struct hdcp_workqueue *hdcp_w,
+					    unsigned int conn_index,
+					    bool enable_encryption)
+{
+	if (enable_encryption) {
+		/* Explicitly set the saved SRM as sysfs call will be after we already enabled hdcp
+		 * (s3 resume case)
+		 */
+		if (hdcp_work->srm_size > 0)
+			psp_set_srm(hdcp_work->hdcp.config.psp.handle, hdcp_work->srm,
+				    hdcp_work->srm_size,
+				    &hdcp_work->srm_version);
+
+		schedule_delayed_work(&hdcp_w->property_validate_dwork,
+				      msecs_to_jiffies(DRM_HDCP_CHECK_PERIOD_MS));
+	} else {
+		hdcp_w->encryption_status[conn_index] = MOD_HDCP_ENCRYPTION_STATUS_HDCP_OFF;
+		cancel_delayed_work(&hdcp_w->property_validate_dwork);
+	}
+}
+EXPORT_IF_KUNIT(hdcp_update_display_encryption_control);
+
 void hdcp_update_display(struct hdcp_workqueue *hdcp_work,
 			 unsigned int link_index,
 			 struct amdgpu_dm_connector *aconnector,
@@ -212,40 +300,13 @@ void hdcp_update_display(struct hdcp_workqueue *hdcp_work,
 		drm_connector_put(&hdcp_w->aconnector[conn_index]->base);
 	hdcp_w->aconnector[conn_index] = aconnector;
 
-	memset(&link_adjust, 0, sizeof(link_adjust));
-	memset(&display_adjust, 0, sizeof(display_adjust));
-
-	if (enable_encryption) {
-		/* Explicitly set the saved SRM as sysfs call will be after we already enabled hdcp
-		 * (s3 resume case)
-		 */
-		if (hdcp_work->srm_size > 0)
-			psp_set_srm(hdcp_work->hdcp.config.psp.handle, hdcp_work->srm,
-				    hdcp_work->srm_size,
-				    &hdcp_work->srm_version);
-
-		display_adjust.disable = MOD_HDCP_DISPLAY_NOT_DISABLE;
-
-		link_adjust.auth_delay = 2;
-		link_adjust.retry_limit = MAX_NUM_OF_ATTEMPTS;
-
-		if (content_type == DRM_MODE_HDCP_CONTENT_TYPE0) {
-			link_adjust.hdcp2.force_type = MOD_HDCP_FORCE_TYPE_0;
-		} else if (content_type == DRM_MODE_HDCP_CONTENT_TYPE1) {
-			link_adjust.hdcp1.disable = 1;
-			link_adjust.hdcp2.force_type = MOD_HDCP_FORCE_TYPE_1;
-		}
-		link_adjust.hdcp2.use_fw_locality_check =
-				(dc->caps.fused_io_supported || dc->debug.hdcp_lc_force_fw_enable);
-		link_adjust.hdcp2.use_sw_locality_fallback = dc->debug.hdcp_lc_enable_sw_fallback;
-
-		schedule_delayed_work(&hdcp_w->property_validate_dwork,
-				      msecs_to_jiffies(DRM_HDCP_CHECK_PERIOD_MS));
-	} else {
-		display_adjust.disable = MOD_HDCP_DISPLAY_DISABLE_AUTHENTICATION;
-		hdcp_w->encryption_status[conn_index] = MOD_HDCP_ENCRYPTION_STATUS_HDCP_OFF;
-		cancel_delayed_work(&hdcp_w->property_validate_dwork);
-	}
+	hdcp_get_link_display_adjustments(enable_encryption, content_type,
+			dc->caps.fused_io_supported,
+			dc->debug.hdcp_lc_force_fw_enable,
+			dc->debug.hdcp_lc_enable_sw_fallback,
+			&link_adjust, &display_adjust);
+	hdcp_update_display_encryption_control(hdcp_work, hdcp_w, conn_index,
+					      enable_encryption);
 
 	mod_hdcp_update_display(&hdcp_w->hdcp, conn_index, &link_adjust, &display_adjust, &hdcp_w->output);
 
@@ -312,6 +373,7 @@ void hdcp_handle_cpirq(struct hdcp_workqueue *hdcp_work, unsigned int link_index
 
 	schedule_work(&hdcp_w->cpirq_work);
 }
+EXPORT_IF_KUNIT(hdcp_handle_cpirq);
 
 static void event_callback(struct work_struct *work)
 {
@@ -330,12 +392,14 @@ static void event_callback(struct work_struct *work)
 	process_output(hdcp_work);
 }
 
-static void event_property_update(struct work_struct *work)
+STATIC_IFN_KUNIT
+void event_property_update(struct work_struct *work)
 {
 	struct hdcp_workqueue *hdcp_work = container_of(work, struct hdcp_workqueue,
 							property_update_work);
 	struct amdgpu_dm_connector *aconnector = NULL;
 	struct drm_device *dev;
+	unsigned int content_protection;
 	long ret;
 	unsigned int conn_index;
 	struct drm_connector *connector;
@@ -375,30 +439,20 @@ static void event_property_update(struct work_struct *work)
 					MOD_HDCP_ENCRYPTION_STATUS_HDCP_OFF;
 			}
 		}
-		if (hdcp_work->encryption_status[conn_index] !=
-			MOD_HDCP_ENCRYPTION_STATUS_HDCP_OFF) {
-			if (conn_state->hdcp_content_type ==
-				DRM_MODE_HDCP_CONTENT_TYPE0 &&
-				hdcp_work->encryption_status[conn_index] <=
-				MOD_HDCP_ENCRYPTION_STATUS_HDCP2_TYPE0_ON) {
+		if (hdcp_get_content_protection_from_status(conn_state->hdcp_content_type,
+							    hdcp_work->encryption_status[conn_index],
+							    &content_protection)) {
+			if (content_protection == DRM_MODE_CONTENT_PROTECTION_ENABLED)
 				DRM_DEBUG_DRIVER("[HDCP_DM] DRM_MODE_CONTENT_PROTECTION_ENABLED\n");
-				drm_hdcp_update_content_protection(connector,
-								   DRM_MODE_CONTENT_PROTECTION_ENABLED);
-			} else if (conn_state->hdcp_content_type ==
-					DRM_MODE_HDCP_CONTENT_TYPE1 &&
-					hdcp_work->encryption_status[conn_index] ==
-					MOD_HDCP_ENCRYPTION_STATUS_HDCP2_TYPE1_ON) {
-				drm_hdcp_update_content_protection(connector,
-								   DRM_MODE_CONTENT_PROTECTION_ENABLED);
-			}
-		} else {
-			DRM_DEBUG_DRIVER("[HDCP_DM] DRM_MODE_CONTENT_PROTECTION_DESIRED\n");
-			drm_hdcp_update_content_protection(connector,
-							   DRM_MODE_CONTENT_PROTECTION_DESIRED);
+			else
+				DRM_DEBUG_DRIVER("[HDCP_DM] DRM_MODE_CONTENT_PROTECTION_DESIRED\n");
+
+			drm_hdcp_update_content_protection(connector, content_protection);
 		}
 		drm_modeset_unlock(&dev->mode_config.connection_mutex);
 	}
 }
+EXPORT_IF_KUNIT(event_property_update);
 
 static void event_property_validate(struct work_struct *work)
 {
@@ -535,15 +589,27 @@ static bool enable_assr(void *handle, struct dc_link *link)
 static void update_config(void *handle, struct cp_psp_stream_config *config)
 {
 	struct hdcp_workqueue *hdcp_work = handle;
-	struct amdgpu_dm_connector *aconnector = config->dm_stream_ctx;
-	int link_index = aconnector->dc_link->link_index;
-	unsigned int conn_index = aconnector->base.index;
-	struct mod_hdcp_display *display = &hdcp_work[link_index].display;
-	struct mod_hdcp_link *link = &hdcp_work[link_index].link;
-	struct hdcp_workqueue *hdcp_w = &hdcp_work[link_index];
+	struct amdgpu_dm_connector *aconnector;
+	const struct dc *dc;
+	int link_index;
+	unsigned int conn_index;
+	struct mod_hdcp_display *display;
+	struct mod_hdcp_link *link;
+	struct hdcp_workqueue *hdcp_w;
 	struct dc_sink *sink = NULL;
 	bool link_is_hdcp14 = false;
-	const struct dc *dc = aconnector->dc_link->dc;
+
+	aconnector = config->dm_stream_ctx;
+	if (!aconnector || !aconnector->dc_link)
+		return;
+
+	link_index = aconnector->dc_link->link_index;
+	display = &hdcp_work[link_index].display;
+	link = &hdcp_work[link_index].link;
+	hdcp_w = &hdcp_work[link_index];
+
+	conn_index = aconnector->base.index;
+	dc = aconnector->dc_link->dc;
 
 	if (config->dpms_off) {
 		hdcp_remove_display(hdcp_work, link_index, aconnector);
@@ -581,7 +647,7 @@ static void update_config(void *handle, struct cp_psp_stream_config *config)
 	link->dp.mst_enabled = config->mst_enabled;
 	link->dp.dp2_enabled = config->dp2_enabled;
 	link->dp.usb4_enabled = config->usb4_enabled;
-	if (aconnector->dc_sink->sink_signal == SIGNAL_TYPE_HDMI_FRL)
+	if (sink && sink->sink_signal == SIGNAL_TYPE_HDMI_FRL)
 		link->hdmi.frl_enabled = config->frl_enabled;
 	display->adjust.disable = MOD_HDCP_DISPLAY_DISABLE_AUTHENTICATION;
 	link->adjust.auth_delay = 2;
@@ -831,4 +897,5 @@ struct hdcp_workqueue *hdcp_create_workqueue(struct amdgpu_device *adev,
 
 	return NULL;
 }
+EXPORT_IF_KUNIT(hdcp_create_workqueue);
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h
index 90b18c450ca61..a82a20b805185 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h
@@ -96,6 +96,23 @@ struct hdcp_workqueue *hdcp_create_workqueue(struct amdgpu_device *adev, struct
 
 #if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
 void process_output(struct hdcp_workqueue *hdcp_work);
+bool hdcp_get_content_protection_from_status(
+	unsigned int hdcp_content_type,
+	enum mod_hdcp_encryption_status encryption_status,
+	unsigned int *content_protection);
+void hdcp_get_link_display_adjustments(
+	bool enable_encryption,
+	u8 content_type,
+	bool fused_io_supported,
+	bool hdcp_lc_force_fw_enable,
+	bool hdcp_lc_enable_sw_fallback,
+	struct mod_hdcp_link_adjustment *link_adjust,
+	struct mod_hdcp_display_adjustment *display_adjust);
+void hdcp_update_display_encryption_control(struct hdcp_workqueue *hdcp_work,
+					    struct hdcp_workqueue *hdcp_w,
+					    unsigned int conn_index,
+					    bool enable_encryption);
+void event_property_update(struct work_struct *work);
 #endif
 
 #endif /* AMDGPU_DM_AMDGPU_DM_HDCP_H_ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
index c6f94eb71ffad..30ac1f32fad51 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
@@ -48,6 +48,8 @@
 #include "dm_helpers.h"
 #include "ddc_service_types.h"
 #include "clk_mgr.h"
+#include "amdgpu_dm_kunit_helpers.h"
+#include "amdgpu_dm_helpers.h"
 
 #define MCCS_DEST_ADDR (0x6E >> 1)
 #define MCCS_SRC_ADDR	0x51
@@ -88,14 +90,15 @@ union vcp_reply {
 	unsigned char raw[11];
 };
 
-static u32 edid_extract_panel_id(struct edid *edid)
+STATIC_IFN_KUNIT u32 edid_extract_panel_id(struct edid *edid)
 {
 	return (u32)edid->mfg_id[0] << 24   |
 	       (u32)edid->mfg_id[1] << 16   |
 	       (u32)EDID_PRODUCT_ID(edid);
 }
+EXPORT_IF_KUNIT(edid_extract_panel_id);
 
-static void apply_edid_quirks(struct dc_link *link, struct edid *edid,
+STATIC_IFN_KUNIT void apply_edid_quirks(struct dc_link *link, struct edid *edid,
 			      struct dc_edid_caps *edid_caps)
 {
 	struct amdgpu_dm_connector *aconnector = link->priv;
@@ -138,6 +141,7 @@ static void apply_edid_quirks(struct dc_link *link, struct edid *edid,
 		return;
 	}
 }
+EXPORT_IF_KUNIT(apply_edid_quirks);
 
 /**
  * dm_helpers_parse_edid_caps() - Parse edid caps
@@ -193,6 +197,12 @@ enum dc_edid_status dm_helpers_parse_edid_caps(
 					__func__, connector->name, edid_caps->frl_dsc_10bpc, edid_caps->frl_dsc_12bpc, \
 					edid_caps->frl_dsc_all_bpp, edid_caps->frl_dsc_native_420, edid_caps->frl_dsc_max_slices, \
 					edid_caps->frl_dsc_max_frl_rate, edid_caps->frl_dsc_total_chunk_kbytes);
+		if (aconnector->hdmi_comp_auto) {
+			edid_caps->panel_patch.hdmi_comp_auto = true;
+			link->ctx->dc->debug.force_frl_max = true;
+			link->ctx->dc->debug.force_frl_dsc = true;
+			drm_dbg_driver(connector->dev, "%s: HDMI_FRL [%s] hdmi_comp_auto --> enabled\n", __func__, connector->name);
+		}
 	}
 
 	apply_edid_quirks(link, edid_buf, edid_caps);
@@ -228,8 +238,9 @@ enum dc_edid_status dm_helpers_parse_edid_caps(
 
 	return result;
 }
+EXPORT_IF_KUNIT(dm_helpers_parse_edid_caps);
 
-static void
+STATIC_IFN_KUNIT void
 fill_dc_mst_payload_table_from_drm(struct dc_link *link,
 				   bool enable,
 				   struct drm_dp_mst_atomic_payload *target_payload,
@@ -279,13 +290,15 @@ fill_dc_mst_payload_table_from_drm(struct dc_link *link,
 	/* Overwrite the old table */
 	*table = new_table;
 }
+EXPORT_IF_KUNIT(fill_dc_mst_payload_table_from_drm);
 
 void dm_helpers_dp_update_branch_info(
 	struct dc_context *ctx,
 	const struct dc_link *link)
 {}
+EXPORT_IF_KUNIT(dm_helpers_dp_update_branch_info);
 
-static void dm_helpers_construct_old_payload(
+STATIC_IFN_KUNIT void dm_helpers_construct_old_payload(
 			struct drm_dp_mst_topology_mgr *mgr,
 			struct drm_dp_mst_topology_state *mst_state,
 			struct drm_dp_mst_atomic_payload *new_payload,
@@ -316,6 +329,7 @@ static void dm_helpers_construct_old_payload(
 	old_payload->time_slots = allocated_time_slots;
 	old_payload->pbn = allocated_time_slots * pbn_per_slot;
 }
+EXPORT_IF_KUNIT(dm_helpers_construct_old_payload);
 
 /*
  * Writes payload allocation table in immediate downstream device.
@@ -368,6 +382,7 @@ bool dm_helpers_dp_mst_write_payload_allocation_table(
 
 	return true;
 }
+EXPORT_IF_KUNIT(dm_helpers_dp_mst_write_payload_allocation_table);
 
 /*
  * poll pending down reply
@@ -376,6 +391,7 @@ void dm_helpers_dp_mst_poll_pending_down_reply(
 	struct dc_context *ctx,
 	const struct dc_link *link)
 {}
+EXPORT_IF_KUNIT(dm_helpers_dp_mst_poll_pending_down_reply);
 
 /*
  * Clear payload allocation table before enable MST DP link.
@@ -384,6 +400,7 @@ void dm_helpers_dp_mst_clear_payload_allocation_table(
 	struct dc_context *ctx,
 	const struct dc_link *link)
 {}
+EXPORT_IF_KUNIT(dm_helpers_dp_mst_clear_payload_allocation_table);
 
 /*
  * Polls for ACT (allocation change trigger) handled and sends
@@ -414,6 +431,7 @@ enum act_return_status dm_helpers_dp_mst_poll_for_allocation_change_trigger(
 
 	return ACT_SUCCESS;
 }
+EXPORT_IF_KUNIT(dm_helpers_dp_mst_poll_for_allocation_change_trigger);
 
 void dm_helpers_dp_mst_send_payload_allocation(
 		struct dc_context *ctx,
@@ -448,6 +466,7 @@ void dm_helpers_dp_mst_send_payload_allocation(
 			clr_flag, false);
 	}
 }
+EXPORT_IF_KUNIT(dm_helpers_dp_mst_send_payload_allocation);
 
 void dm_helpers_dp_mst_update_mst_mgr_for_deallocation(
 		struct dc_context *ctx,
@@ -476,6 +495,7 @@ void dm_helpers_dp_mst_update_mst_mgr_for_deallocation(
 	amdgpu_dm_set_mst_status(&aconnector->mst_status, set_flag, true);
 	amdgpu_dm_set_mst_status(&aconnector->mst_status, clr_flag, false);
  }
+EXPORT_IF_KUNIT(dm_helpers_dp_mst_update_mst_mgr_for_deallocation);
 
 void dm_dtn_log_begin(struct dc_context *ctx,
 	struct dc_log_buffer_ctx *log_ctx)
@@ -489,6 +509,7 @@ void dm_dtn_log_begin(struct dc_context *ctx,
 
 	dm_dtn_log_append_v(ctx, log_ctx, "%s", msg);
 }
+EXPORT_IF_KUNIT(dm_dtn_log_begin);
 
 __printf(3, 4)
 void dm_dtn_log_append_v(struct dc_context *ctx,
@@ -551,6 +572,7 @@ void dm_dtn_log_append_v(struct dc_context *ctx,
 	if (n > 0)
 		log_ctx->pos += n;
 }
+EXPORT_IF_KUNIT(dm_dtn_log_append_v);
 
 void dm_dtn_log_end(struct dc_context *ctx,
 	struct dc_log_buffer_ctx *log_ctx)
@@ -564,6 +586,7 @@ void dm_dtn_log_end(struct dc_context *ctx,
 
 	dm_dtn_log_append_v(ctx, log_ctx, "%s", msg);
 }
+EXPORT_IF_KUNIT(dm_dtn_log_end);
 
 bool dm_helpers_dp_mst_start_top_mgr(
 		struct dc_context *ctx,
@@ -598,6 +621,7 @@ bool dm_helpers_dp_mst_start_top_mgr(
 
 	return true;
 }
+EXPORT_IF_KUNIT(dm_helpers_dp_mst_start_top_mgr);
 
 bool dm_helpers_dp_mst_stop_top_mgr(
 		struct dc_context *ctx,
@@ -620,6 +644,7 @@ bool dm_helpers_dp_mst_stop_top_mgr(
 
 	return false;
 }
+EXPORT_IF_KUNIT(dm_helpers_dp_mst_stop_top_mgr);
 
 bool dm_helpers_dp_read_dpcd(
 		struct dc_context *ctx,
@@ -637,6 +662,7 @@ bool dm_helpers_dp_read_dpcd(
 	return drm_dp_dpcd_read(&aconnector->dm_dp_aux.aux, address, data,
 				size) == size;
 }
+EXPORT_IF_KUNIT(dm_helpers_dp_read_dpcd);
 
 bool dm_helpers_dp_write_dpcd(
 		struct dc_context *ctx,
@@ -653,6 +679,7 @@ bool dm_helpers_dp_write_dpcd(
 	return drm_dp_dpcd_write(&aconnector->dm_dp_aux.aux,
 			address, (uint8_t *)data, size) > 0;
 }
+EXPORT_IF_KUNIT(dm_helpers_dp_write_dpcd);
 
 bool dm_helpers_submit_i2c(
 		struct dc_context *ctx,
@@ -688,6 +715,7 @@ bool dm_helpers_submit_i2c(
 
 	return result;
 }
+EXPORT_IF_KUNIT(dm_helpers_submit_i2c);
 
 bool dm_helpers_execute_fused_io(
 		struct dc_context *ctx,
@@ -701,8 +729,9 @@ bool dm_helpers_execute_fused_io(
 
 	return amdgpu_dm_execute_fused_io(dev, link, commands, count, timeout_us);
 }
+EXPORT_IF_KUNIT(dm_helpers_execute_fused_io);
 
-static bool execute_synaptics_rc_command(struct drm_dp_aux *aux,
+STATIC_IFN_KUNIT bool execute_synaptics_rc_command(struct drm_dp_aux *aux,
 		bool is_write_cmd,
 		unsigned char cmd,
 		unsigned int length,
@@ -774,8 +803,9 @@ static bool execute_synaptics_rc_command(struct drm_dp_aux *aux,
 	DRM_ERROR("%s: write cmd ..., err = %d\n",  __func__, ret);
 	return false;
 }
+EXPORT_IF_KUNIT(execute_synaptics_rc_command);
 
-static void apply_synaptics_fifo_reset_wa(struct drm_dp_aux *aux)
+STATIC_IFN_KUNIT void apply_synaptics_fifo_reset_wa(struct drm_dp_aux *aux)
 {
 	unsigned char data[16] = {0};
 
@@ -839,11 +869,12 @@ static void apply_synaptics_fifo_reset_wa(struct drm_dp_aux *aux)
 
 	drm_dbg_dp(aux->drm_dev, "Done\n");
 }
+EXPORT_IF_KUNIT(apply_synaptics_fifo_reset_wa);
 
 /* MST Dock */
 static const uint8_t SYNAPTICS_DEVICE_ID[] = "SYNA";
 
-static uint8_t write_dsc_enable_synaptics_non_virtual_dpcd_mst(
+STATIC_IFN_KUNIT uint8_t write_dsc_enable_synaptics_non_virtual_dpcd_mst(
 		struct drm_dp_aux *aux,
 		const struct dc_stream_state *stream,
 		bool enable)
@@ -879,6 +910,7 @@ static uint8_t write_dsc_enable_synaptics_non_virtual_dpcd_mst(
 
 	return ret;
 }
+EXPORT_IF_KUNIT(write_dsc_enable_synaptics_non_virtual_dpcd_mst);
 
 bool dm_helpers_dp_write_dsc_enable(
 		struct dc_context *ctx,
@@ -962,12 +994,28 @@ bool dm_helpers_dp_write_dsc_enable(
 
 	return ret;
 }
+EXPORT_IF_KUNIT(dm_helpers_dp_write_dsc_enable);
+
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+uint dm_helpers_get_dc_debug_mask(void)
+{
+	return amdgpu_dc_debug_mask;
+}
+EXPORT_IF_KUNIT(dm_helpers_get_dc_debug_mask);
+
+void dm_helpers_set_dc_debug_mask(uint debug_mask)
+{
+	amdgpu_dc_debug_mask = debug_mask;
+}
+EXPORT_IF_KUNIT(dm_helpers_set_dc_debug_mask);
+#endif
 
 bool dm_helpers_dp_write_hblank_reduction(struct dc_context *ctx, const struct dc_stream_state *stream)
 {
 	// TODO
 	return false;
 }
+EXPORT_IF_KUNIT(dm_helpers_dp_write_hblank_reduction);
 
 bool dm_helpers_is_dp_sink_present(struct dc_link *link)
 {
@@ -984,8 +1032,9 @@ bool dm_helpers_is_dp_sink_present(struct dc_link *link)
 	mutex_unlock(&aconnector->dm_dp_aux.aux.hw_mutex);
 	return dp_sink_present;
 }
+EXPORT_IF_KUNIT(dm_helpers_is_dp_sink_present);
 
-static int
+STATIC_IFN_KUNIT int
 dm_helpers_probe_acpi_edid(void *data, u8 *buf, unsigned int block, size_t len)
 {
 	struct drm_connector *connector = data;
@@ -1023,8 +1072,9 @@ dm_helpers_probe_acpi_edid(void *data, u8 *buf, unsigned int block, size_t len)
 
 	return r;
 }
+EXPORT_IF_KUNIT(dm_helpers_probe_acpi_edid);
 
-static const struct drm_edid *
+STATIC_IFN_KUNIT const struct drm_edid *
 dm_helpers_read_acpi_edid(struct amdgpu_dm_connector *aconnector)
 {
 	struct drm_connector *connector = &aconnector->base;
@@ -1045,8 +1095,9 @@ dm_helpers_read_acpi_edid(struct amdgpu_dm_connector *aconnector)
 
 	return drm_edid_read_custom(connector, dm_helpers_probe_acpi_edid, connector);
 }
+EXPORT_IF_KUNIT(dm_helpers_read_acpi_edid);
 
-static const struct drm_edid *
+STATIC_IFN_KUNIT const struct drm_edid *
 dm_helpers_read_vbios_hardcoded_edid(struct dc_link *link, struct amdgpu_dm_connector *aconnector)
 {
 	struct dc_bios *bios = link->ctx->dc_bios;
@@ -1084,8 +1135,9 @@ dm_helpers_read_vbios_hardcoded_edid(struct dc_link *link, struct amdgpu_dm_conn
 
 	return edid;
 }
+EXPORT_IF_KUNIT(dm_helpers_read_vbios_hardcoded_edid);
 
-static uint8_t get_max_frl_rate(uint8_t max_lanes, uint8_t max_rate_per_lane)
+STATIC_IFN_KUNIT uint8_t get_max_frl_rate(uint8_t max_lanes, uint8_t max_rate_per_lane)
 {
 	uint8_t max_frl_rate;
 
@@ -1106,8 +1158,9 @@ static uint8_t get_max_frl_rate(uint8_t max_lanes, uint8_t max_rate_per_lane)
 
 	return max_frl_rate;
 }
+EXPORT_IF_KUNIT(get_max_frl_rate);
 
-static uint8_t get_dsc_max_slices(uint8_t max_slices, int clk_per_slice)
+STATIC_IFN_KUNIT uint8_t get_dsc_max_slices(uint8_t max_slices, int clk_per_slice)
 {
 	uint8_t dsc_max_slices;
 
@@ -1130,6 +1183,7 @@ static uint8_t get_dsc_max_slices(uint8_t max_slices, int clk_per_slice)
 
 	return dsc_max_slices;
 }
+EXPORT_IF_KUNIT(get_dsc_max_slices);
 
 void populate_hdmi_info_from_connector(bool enable_frl, struct drm_hdmi_info *hdmi, struct dc_edid_caps *edid_caps)
 {
@@ -1150,6 +1204,7 @@ void populate_hdmi_info_from_connector(bool enable_frl, struct drm_hdmi_info *hd
 		}
 	}
 }
+EXPORT_IF_KUNIT(populate_hdmi_info_from_connector);
 
 enum dc_edid_status dm_helpers_read_local_edid(
 		struct dc_context *ctx,
@@ -1201,11 +1256,25 @@ enum dc_edid_status dm_helpers_read_local_edid(
 			continue;
 
 		edid = drm_edid_raw(drm_edid); // FIXME: Get rid of drm_edid_raw()
-		if (!edid ||
-		    edid->extensions >= sizeof(sink->dc_edid.raw_edid) / EDID_LENGTH)
+		/*
+		 * Use the length of the EDID property blob populated by
+		 * drm_edid_connector_update() above. It reflects the true number
+		 * of EDID blocks, including any HDMI Forum EDID Extension Override
+		 * Data Block (HF-EEODB) count, which the raw byte 0x7e extension
+		 * count can hide (e.g. HDMI 8K sinks).
+		 */
+		if (!edid || !connector->edid_blob_ptr ||
+		    connector->edid_blob_ptr->length > sizeof(sink->dc_edid.raw_edid))
 			return EDID_BAD_INPUT;
 
-		sink->dc_edid.length = EDID_LENGTH * (edid->extensions + 1);
+		/*
+		 * FIXME: amdgpu_dm today does not consider the HF-EEODB, which
+		 * may contain additional mode info for sinks. This is a
+		 * workaround until dc_edid is refactored out from DC into
+		 * amdgpu_dm's ownership, allowing amdgpu_dm to use drm_edid
+		 * directly
+		 */
+		sink->dc_edid.length = connector->edid_blob_ptr->length;
 		memmove(sink->dc_edid.raw_edid, (uint8_t *)edid, sink->dc_edid.length);
 
 		/* We don't need the original edid anymore */
@@ -1267,6 +1336,7 @@ int dm_helper_dmub_aux_transfer_sync(
 	return amdgpu_dm_process_dmub_aux_transfer_sync(ctx, link->link_index, payload,
 			operation_result);
 }
+EXPORT_IF_KUNIT(dm_helper_dmub_aux_transfer_sync);
 
 int dm_helpers_dmub_set_config_sync(struct dc_context *ctx,
 		const struct dc_link *link,
@@ -1281,18 +1351,21 @@ void dm_set_dcn_clocks(struct dc_context *ctx, struct dc_clocks *clks)
 {
 	/* TODO: something */
 }
+EXPORT_IF_KUNIT(dm_set_dcn_clocks);
 
 void dm_helpers_dmu_timeout(struct dc_context *ctx)
 {
 	// TODO:
 	//amdgpu_device_gpu_recover(dc_context->driver-context, NULL);
 }
+EXPORT_IF_KUNIT(dm_helpers_dmu_timeout);
 
 void dm_helpers_smu_timeout(struct dc_context *ctx, unsigned int msg_id, unsigned int param, unsigned int timeout_us)
 {
 	// TODO:
 	//amdgpu_device_gpu_recover(dc_context->driver-context, NULL);
 }
+EXPORT_IF_KUNIT(dm_helpers_smu_timeout);
 
 void dm_helpers_init_panel_settings(
 	struct dc_context *ctx,
@@ -1311,6 +1384,7 @@ void dm_helpers_init_panel_settings(
 	panel_config->dsc.disable_dsc_edp = false;
 	panel_config->dsc.force_dsc_edp_policy = 0;
 }
+EXPORT_IF_KUNIT(dm_helpers_init_panel_settings);
 
 void dm_helpers_override_panel_settings(
 	struct dc_context *ctx,
@@ -1328,6 +1402,7 @@ void dm_helpers_override_panel_settings(
 		link->panel_config.psr.disallow_replay = true;
 	}
 }
+EXPORT_IF_KUNIT(dm_helpers_override_panel_settings);
 
 void *dm_helpers_allocate_gpu_mem(
 		struct dc_context *ctx,
@@ -1363,6 +1438,7 @@ bool dm_helpers_dmub_outbox_interrupt_control(struct dc_context *ctx, bool enabl
 			 enable ? "en" : "dis", ret);
 	return ret;
 }
+EXPORT_IF_KUNIT(dm_helpers_dmub_outbox_interrupt_control);
 
 void dm_helpers_mst_enable_stream_features(const struct dc_stream_state *stream)
 {
@@ -1388,6 +1464,7 @@ void dm_helpers_mst_enable_stream_features(const struct dc_stream_state *stream)
 					 &new_downspread.raw,
 					 sizeof(new_downspread));
 }
+EXPORT_IF_KUNIT(dm_helpers_mst_enable_stream_features);
 
 bool dm_helpers_dp_handle_test_pattern_request(
 		struct dc_context *ctx,
@@ -1526,11 +1603,13 @@ bool dm_helpers_dp_handle_test_pattern_request(
 
 	return false;
 }
+EXPORT_IF_KUNIT(dm_helpers_dp_handle_test_pattern_request);
 
 void dm_set_phyd32clk(struct dc_context *ctx, int freq_khz)
 {
        // TODO
 }
+EXPORT_IF_KUNIT(dm_set_phyd32clk);
 
 void dm_helpers_enable_periodic_detection(struct dc_context *ctx, bool enable)
 {
@@ -1542,6 +1621,7 @@ void dm_helpers_enable_periodic_detection(struct dc_context *ctx, bool enable)
 			schedule_work(&adev->dm.idle_workqueue->work);
 	}
 }
+EXPORT_IF_KUNIT(dm_helpers_enable_periodic_detection);
 
 void dm_helpers_dp_mst_update_branch_bandwidth(
 		struct dc_context *ctx,
@@ -1549,25 +1629,34 @@ void dm_helpers_dp_mst_update_branch_bandwidth(
 {
 	// TODO
 }
+EXPORT_IF_KUNIT(dm_helpers_dp_mst_update_branch_bandwidth);
+
+STATIC_IFN_KUNIT const uint32_t dm_freesync_pcon_whitelist[] = {
+	DP_BRANCH_DEVICE_ID_0060AD,
+	DP_BRANCH_DEVICE_ID_00E04C,
+	DP_BRANCH_DEVICE_ID_90CC24,
+	DP_BRANCH_DEVICE_ID_001CF8,
+	DP_BRANCH_DEVICE_ID_001FF2,
+};
+EXPORT_IF_KUNIT(dm_freesync_pcon_whitelist);
 
-static bool dm_is_freesync_pcon_whitelist(const uint32_t branch_dev_id)
+STATIC_IFN_KUNIT uint32_t dm_freesync_pcon_whitelist_count(void)
 {
-	bool ret_val = false;
-
-	switch (branch_dev_id) {
-	case DP_BRANCH_DEVICE_ID_0060AD:
-	case DP_BRANCH_DEVICE_ID_00E04C:
-	case DP_BRANCH_DEVICE_ID_90CC24:
-	case DP_BRANCH_DEVICE_ID_001CF8:
-	case DP_BRANCH_DEVICE_ID_001FF2:
-		ret_val = true;
-		break;
-	default:
-		break;
-	}
+	return ARRAY_SIZE(dm_freesync_pcon_whitelist);
+}
+EXPORT_IF_KUNIT(dm_freesync_pcon_whitelist_count);
 
-	return ret_val;
+STATIC_IFN_KUNIT bool dm_is_freesync_pcon_whitelist(const uint32_t branch_dev_id)
+{
+	u32 i;
+
+	for (i = 0; i < dm_freesync_pcon_whitelist_count(); i++)
+		if (dm_freesync_pcon_whitelist[i] == branch_dev_id)
+			return true;
+
+	return false;
 }
+EXPORT_IF_KUNIT(dm_is_freesync_pcon_whitelist);
 
 enum adaptive_sync_type dm_get_adaptive_sync_support_type(struct dc_link *link)
 {
@@ -1587,18 +1676,21 @@ enum adaptive_sync_type dm_get_adaptive_sync_support_type(struct dc_link *link)
 
 	return as_type;
 }
+EXPORT_IF_KUNIT(dm_get_adaptive_sync_support_type);
 
 bool dm_helpers_is_fullscreen(struct dc_context *ctx, struct dc_stream_state *stream)
 {
 	// TODO
 	return false;
 }
+EXPORT_IF_KUNIT(dm_helpers_is_fullscreen);
 
 bool dm_helpers_is_hdr_on(struct dc_context *ctx, struct dc_stream_state *stream)
 {
 	// TODO
 	return false;
 }
+EXPORT_IF_KUNIT(dm_helpers_is_hdr_on);
 
 static int mccs_operation_vcp_request(unsigned int vcp_code, struct dc_link *link,
 				union vcp_reply *reply)
@@ -1724,6 +1816,7 @@ void dm_helpers_read_mccs_caps(struct dc_context *ctx, struct dc_link *link,
 		}
 	}
 }
+EXPORT_IF_KUNIT(dm_helpers_read_mccs_caps);
 
 static int mccs_operation_vcp_set(unsigned int vcp_code, struct dc_link *link, uint16_t value)
 {
@@ -1798,4 +1891,5 @@ void dm_helpers_mccs_vcp_set(struct dc_context *ctx, struct dc_link *link,
 		drm_dbg_driver(dev, "%s: Failed to set VCP code %d", __func__,
 				sink->edid_caps.freesync_vcp_code);
 }
+EXPORT_IF_KUNIT(dm_helpers_mccs_vcp_set);
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.h
new file mode 100644
index 0000000000000..0eab820757c13
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.h
@@ -0,0 +1,61 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#ifndef __AMDGPU_DM_HELPERS_H__
+#define __AMDGPU_DM_HELPERS_H__
+
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+#include <drm/drm_edid.h>
+
+struct amdgpu_dm_connector;
+struct dc_link;
+struct dc_context;
+struct dc_stream_state;
+struct dc_edid_caps;
+struct dc_dp_mst_stream_allocation_table;
+struct drm_dp_aux;
+struct drm_dp_mst_atomic_payload;
+struct drm_dp_mst_topology_mgr;
+struct drm_dp_mst_topology_state;
+
+/* Exported for KUnit testing */
+u32 edid_extract_panel_id(struct edid *edid);
+void apply_edid_quirks(struct dc_link *link, struct edid *edid,
+		       struct dc_edid_caps *edid_caps);
+uint8_t get_max_frl_rate(uint8_t max_lanes, uint8_t max_rate_per_lane);
+uint8_t get_dsc_max_slices(uint8_t max_slices, int clk_per_slice);
+bool dm_is_freesync_pcon_whitelist(const uint32_t branch_dev_id);
+extern const uint32_t dm_freesync_pcon_whitelist[];
+uint32_t dm_freesync_pcon_whitelist_count(void);
+void fill_dc_mst_payload_table_from_drm(struct dc_link *link,
+					 bool enable,
+					 struct drm_dp_mst_atomic_payload *target_payload,
+					 struct dc_dp_mst_stream_allocation_table *table);
+void dm_helpers_construct_old_payload(struct drm_dp_mst_topology_mgr *mgr,
+				      struct drm_dp_mst_topology_state *mst_state,
+				      struct drm_dp_mst_atomic_payload *new_payload,
+				      struct drm_dp_mst_atomic_payload *old_payload);
+bool dm_helpers_dp_write_dsc_enable(struct dc_context *ctx,
+				    const struct dc_stream_state *stream,
+				    bool enable);
+uint dm_helpers_get_dc_debug_mask(void);
+void dm_helpers_set_dc_debug_mask(uint debug_mask);
+int dm_helpers_probe_acpi_edid(void *data, u8 *buf, unsigned int block, size_t len);
+const struct drm_edid *dm_helpers_read_acpi_edid(struct amdgpu_dm_connector *aconnector);
+const struct drm_edid *dm_helpers_read_vbios_hardcoded_edid(struct dc_link *link,
+							    struct amdgpu_dm_connector *aconnector);
+bool execute_synaptics_rc_command(struct drm_dp_aux *aux,
+				  bool is_write_cmd,
+				  unsigned char cmd,
+				  unsigned int length,
+				  unsigned int offset,
+				  unsigned char *data);
+void apply_synaptics_fifo_reset_wa(struct drm_dp_aux *aux);
+uint8_t write_dsc_enable_synaptics_non_virtual_dpcd_mst(struct drm_dp_aux *aux,
+							const struct dc_stream_state *stream,
+							bool enable);
+#endif /* CONFIG_DRM_AMD_DC_KUNIT_TEST */
+
+#endif /* __AMDGPU_DM_HELPERS_H__ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c
index e49803a90edad..ae3f81f53074f 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c
@@ -26,10 +26,25 @@
 
 #include "dm_services_types.h"
 #include "dc.h"
+#include "dc/dc_dmub_srv.h"
+#include "dc/dc_stat.h"
 
 #include "amdgpu.h"
+#include "amdgpu_display.h"
 #include "amdgpu_dm.h"
 #include "amdgpu_dm_irq.h"
+#include "amdgpu_dm_kunit_helpers.h"
+#include "amdgpu_dm_crtc.h"
+#include "amdgpu_dm_hdcp.h"
+#include "amdgpu_dm_mst_types.h"
+#include "amdgpu_dm_dmub.h"
+#include "amdgpu_dm_trace.h"
+#include "link/protocols/link_dpcd.h"
+#include "link_service_types.h"
+#include "ivsrcid/ivsrcid_vislands30.h"
+#include "ivsrcid/dcn/irqsrcs_dcn_1_0.h"
+#include "modules/inc/mod_freesync.h"
+#include <drm/drm_vblank.h>
 
 /**
  * DOC: overview
@@ -55,7 +70,8 @@
  * are all set to the DM generic handler amdgpu_dm_irq_handler(), which looks up
  * DM's IRQ tables. However, in order for base driver to recognize this hook, DM
  * still needs to register the IRQ with the base driver. See
- * dce110_register_irq_handlers() and dcn10_register_irq_handlers().
+ * amdgpu_dm_dce110_register_irq_handlers() and
+ * amdgpu_dm_dcn10_register_irq_handlers().
  *
  * To expose DC's hardware interrupt toggle to the base driver, DM implements
  * &amdgpu_irq_src_funcs.set hooks. Base driver calls it through
@@ -172,13 +188,16 @@ static struct list_head *remove_irq_handler(struct amdgpu_device *adev,
 
 	DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
 
-	if (handler_removed == false) {
+	if (!handler_removed) {
 		/* Not necessarily an error - caller may not
 		 * know the context.
 		 */
 		return NULL;
 	}
 
+	if (int_params->int_context == INTERRUPT_LOW_IRQ_CONTEXT)
+		cancel_work_sync(&handler->work);
+
 	kfree(handler);
 
 	DRM_DEBUG_KMS(
@@ -188,55 +207,6 @@ static struct list_head *remove_irq_handler(struct amdgpu_device *adev,
 	return hnd_list;
 }
 
-/**
- * unregister_all_irq_handlers() - Cleans up handlers from the DM IRQ table
- * @adev: The base driver device containing the DM device
- *
- * Go through low and high context IRQ tables and deallocate handlers.
- */
-static void unregister_all_irq_handlers(struct amdgpu_device *adev)
-{
-	struct list_head *hnd_list_low;
-	struct list_head *hnd_list_high;
-	struct list_head *entry, *tmp;
-	struct amdgpu_dm_irq_handler_data *handler;
-	unsigned long irq_table_flags;
-	int i;
-
-	DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
-
-	for (i = 0; i < DAL_IRQ_SOURCES_NUMBER; i++) {
-		hnd_list_low = &adev->dm.irq_handler_list_low_tab[i];
-		hnd_list_high = &adev->dm.irq_handler_list_high_tab[i];
-
-		list_for_each_safe(entry, tmp, hnd_list_low) {
-
-			handler = list_entry(entry, struct amdgpu_dm_irq_handler_data,
-					     list);
-
-			if (handler == NULL || handler->handler == NULL)
-				continue;
-
-			list_del(&handler->list);
-			kfree(handler);
-		}
-
-		list_for_each_safe(entry, tmp, hnd_list_high) {
-
-			handler = list_entry(entry, struct amdgpu_dm_irq_handler_data,
-					     list);
-
-			if (handler == NULL || handler->handler == NULL)
-				continue;
-
-			list_del(&handler->list);
-			kfree(handler);
-		}
-	}
-
-	DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
-}
-
 static bool
 validate_irq_registration_params(struct dc_interrupt_params *int_params,
 				 void (*ih)(void *))
@@ -310,7 +280,7 @@ void *amdgpu_dm_irq_register_interrupt(struct amdgpu_device *adev,
 	unsigned long irq_table_flags;
 	enum dc_irq_source irq_source;
 
-	if (false == validate_irq_registration_params(int_params, ih))
+	if (!validate_irq_registration_params(int_params, ih))
 		return DAL_INVALID_IRQ_HANDLER_IDX;
 
 	handler_data = kzalloc_obj(*handler_data);
@@ -357,6 +327,7 @@ void *amdgpu_dm_irq_register_interrupt(struct amdgpu_device *adev,
 
 	return handler_data;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_irq_register_interrupt);
 
 /**
  * amdgpu_dm_irq_unregister_interrupt() - Remove a handler from the DM IRQ table
@@ -375,7 +346,7 @@ void amdgpu_dm_irq_unregister_interrupt(struct amdgpu_device *adev,
 	struct dc_interrupt_params int_params;
 	int i;
 
-	if (false == validate_irq_unregistration_params(irq_source, ih))
+	if (!validate_irq_unregistration_params(irq_source, ih))
 		return;
 
 	memset(&int_params, 0, sizeof(int_params));
@@ -401,6 +372,7 @@ void amdgpu_dm_irq_unregister_interrupt(struct amdgpu_device *adev,
 			ih, irq_source);
 	}
 }
+EXPORT_IF_KUNIT(amdgpu_dm_irq_unregister_interrupt);
 
 /**
  * amdgpu_dm_irq_init() - Initialize DM IRQ management
@@ -425,6 +397,21 @@ int amdgpu_dm_irq_init(struct amdgpu_device *adev)
 
 	spin_lock_init(&adev->dm.irq_handler_list_table_lock);
 
+	adev->dm.irq_wq = alloc_workqueue("amdgpu_dm_irq",
+		WQ_UNBOUND | WQ_HIGHPRI, 0);
+
+	if (!adev->dm.irq_wq)
+		return -ENOMEM;
+
+	adev->dm.vmin_vmax_wq = alloc_workqueue("amdgpu_dm_vmin_vmax",
+		WQ_UNBOUND, 0);
+
+	if (!adev->dm.vmin_vmax_wq) {
+		destroy_workqueue(adev->dm.irq_wq);
+		adev->dm.irq_wq = NULL;
+		return -ENOMEM;
+	}
+
 	for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++) {
 		/* low context handler list init */
 		lh = &adev->dm.irq_handler_list_low_tab[src];
@@ -435,50 +422,94 @@ int amdgpu_dm_irq_init(struct amdgpu_device *adev)
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_irq_init);
 
 /**
  * amdgpu_dm_irq_fini() - Tear down DM IRQ management
  * @adev: The base driver device containing the DM device
  *
- * Flush all work within the low context IRQ table.
+ * Removes all handlers from the IRQ tables under the spinlock, cancels
+ * pending work items, and deallocates all handler data.
  */
 void amdgpu_dm_irq_fini(struct amdgpu_device *adev)
 {
 	int src;
-	struct list_head *lh;
+	LIST_HEAD(low_handlers);
+	LIST_HEAD(high_handlers);
 	struct list_head *entry, *tmp;
 	struct amdgpu_dm_irq_handler_data *handler;
 	unsigned long irq_table_flags;
 
 	DRM_DEBUG_KMS("DM_IRQ: releasing resources.\n");
+
 	for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++) {
 		DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
-		/* The handler was removed from the table,
-		 * it means it is safe to flush all the 'work'
-		 * (because no code can schedule a new one).
+
+		/*
+		 * Move all handlers from the low and high context tables to
+		 * temporary lists under the lock. This prevents the ISR from
+		 * finding them while we process them outside the lock.
 		 */
-		lh = &adev->dm.irq_handler_list_low_tab[src];
+		list_splice_init(&adev->dm.irq_handler_list_low_tab[src],
+				 &low_handlers);
+		list_splice_init(&adev->dm.irq_handler_list_high_tab[src],
+				 &high_handlers);
+
 		DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
 
-		if (!list_empty(lh)) {
-			list_for_each_safe(entry, tmp, lh) {
-				handler = list_entry(
-					entry,
-					struct amdgpu_dm_irq_handler_data,
-					list);
-				flush_work(&handler->work);
-			}
+		/*
+		 * Cancel all pending work for the low-context handlers
+		 * outside the lock. cancel_work_sync() may sleep and waits
+		 * until any running work completes, preventing UAF.
+		 */
+		list_for_each_safe(entry, tmp, &low_handlers) {
+			handler = list_entry(entry,
+					    struct amdgpu_dm_irq_handler_data,
+					    list);
+			cancel_work_sync(&handler->work);
 		}
+
+		/*
+		 * High-context handlers are executed synchronously within ISR
+		 * context (see amdgpu_dm_irq_immediate_work()) and have no
+		 * work_struct, so there is no pending work to cancel here.
+		 * They will be freed along with low_handlers after the loop.
+		 */
+	}
+
+	/* Deallocate all handlers. */
+	list_for_each_safe(entry, tmp, &low_handlers) {
+		handler = list_entry(entry,
+				     struct amdgpu_dm_irq_handler_data,
+				     list);
+		list_del(&handler->list);
+		kfree(handler);
+	}
+
+	list_for_each_safe(entry, tmp, &high_handlers) {
+		handler = list_entry(entry,
+				     struct amdgpu_dm_irq_handler_data,
+				     list);
+		list_del(&handler->list);
+		kfree(handler);
+	}
+
+	if (adev->dm.vmin_vmax_wq) {
+		destroy_workqueue(adev->dm.vmin_vmax_wq);
+		adev->dm.vmin_vmax_wq = NULL;
+	}
+
+	if (adev->dm.irq_wq) {
+		destroy_workqueue(adev->dm.irq_wq);
+		adev->dm.irq_wq = NULL;
 	}
-	/* Deallocate handlers from the table. */
-	unregister_all_irq_handlers(adev);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_irq_fini);
 
 void amdgpu_dm_irq_suspend(struct amdgpu_device *adev)
 {
 	struct drm_device *dev = adev_to_drm(adev);
 	int src;
-	struct list_head *hnd_list_h;
 	struct list_head *hnd_list_l;
 	unsigned long irq_table_flags;
 	struct list_head *entry, *tmp;
@@ -491,12 +522,15 @@ void amdgpu_dm_irq_suspend(struct amdgpu_device *adev)
 	/**
 	 * Disable HW interrupt  for HPD and HPDRX only since FLIP and VBLANK
 	 * will be disabled from manage_dm_interrupts on disable CRTC.
+	 *
+	 * Disable the HW interrupt first, then flush any pending work. Since
+	 * the HW interrupt is disabled under the lock, no new IRQ can be
+	 * generated after the disable completes. Any work already queued by an
+	 * in-flight ISR will be flushed below.
 	 */
 	for (src = DC_IRQ_SOURCE_HPD1; src <= DC_IRQ_SOURCE_HPD6RX; src++) {
 		hnd_list_l = &adev->dm.irq_handler_list_low_tab[src];
-		hnd_list_h = &adev->dm.irq_handler_list_high_tab[src];
-		if (!list_empty(hnd_list_l) || !list_empty(hnd_list_h))
-			dc_interrupt_set(adev->dm.dc, src, false);
+		dc_interrupt_set(adev->dm.dc, src, false);
 
 		DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
 
@@ -517,6 +551,7 @@ void amdgpu_dm_irq_suspend(struct amdgpu_device *adev)
 	if (dev->mode_config.poll_enabled)
 		drm_kms_helper_poll_disable(dev);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_irq_suspend);
 
 void amdgpu_dm_irq_resume_early(struct amdgpu_device *adev)
 {
@@ -538,6 +573,7 @@ void amdgpu_dm_irq_resume_early(struct amdgpu_device *adev)
 
 	DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_irq_resume_early);
 
 void amdgpu_dm_irq_resume_late(struct amdgpu_device *adev)
 {
@@ -566,23 +602,27 @@ void amdgpu_dm_irq_resume_late(struct amdgpu_device *adev)
 	if (dev->mode_config.poll_enabled)
 		drm_kms_helper_poll_enable(dev);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_irq_resume_late);
 
 /*
  * amdgpu_dm_irq_schedule_work - schedule all work items registered for the
  * "irq_source".
  */
-static void amdgpu_dm_irq_schedule_work(struct amdgpu_device *adev,
+STATIC_IFN_KUNIT void amdgpu_dm_irq_schedule_work(struct amdgpu_device *adev,
 					enum dc_irq_source irq_source)
 {
 	struct  list_head *handler_list = &adev->dm.irq_handler_list_low_tab[irq_source];
 	struct  amdgpu_dm_irq_handler_data *handler_data;
 	bool    work_queued = false;
+	unsigned long irq_table_flags;
+
+	DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
 
 	if (list_empty(handler_list))
-		return;
+		goto out_unlock;
 
 	list_for_each_entry(handler_data, handler_list, list) {
-		if (queue_work(system_highpri_wq, &handler_data->work)) {
+		if (queue_work(adev->dm.irq_wq, &handler_data->work)) {
 			work_queued = true;
 			break;
 		}
@@ -597,7 +637,7 @@ static void amdgpu_dm_irq_schedule_work(struct amdgpu_device *adev,
 		handler_data_add = kzalloc_obj(*handler_data, GFP_ATOMIC);
 		if (!handler_data_add) {
 			DRM_ERROR("DM_IRQ: failed to allocate irq handler!\n");
-			return;
+			goto out_unlock;
 		}
 
 		/*copy new amdgpu_dm_irq_handler_data members from handler_data*/
@@ -610,7 +650,7 @@ static void amdgpu_dm_irq_schedule_work(struct amdgpu_device *adev,
 
 		INIT_WORK(&handler_data_add->work, dm_irq_work_func);
 
-		if (queue_work(system_highpri_wq, &handler_data_add->work))
+		if (queue_work(adev->dm.irq_wq, &handler_data_add->work))
 			DRM_DEBUG("Queued work for handling interrupt from "
 				  "display for IRQ source %d\n",
 				  irq_source);
@@ -619,13 +659,17 @@ static void amdgpu_dm_irq_schedule_work(struct amdgpu_device *adev,
 				  "from display for IRQ source %d\n",
 				  irq_source);
 	}
+
+out_unlock:
+	DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_irq_schedule_work);
 
 /*
  * amdgpu_dm_irq_immediate_work
  * Callback high irq work immediately, don't send to work queue
  */
-static void amdgpu_dm_irq_immediate_work(struct amdgpu_device *adev,
+STATIC_IFN_KUNIT void amdgpu_dm_irq_immediate_work(struct amdgpu_device *adev,
 					 enum dc_irq_source irq_source)
 {
 	struct amdgpu_dm_irq_handler_data *handler_data;
@@ -644,6 +688,7 @@ static void amdgpu_dm_irq_immediate_work(struct amdgpu_device *adev,
 
 	DM_IRQ_TABLE_UNLOCK(adev, irq_table_flags);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_irq_immediate_work);
 
 /**
  * amdgpu_dm_irq_handler - Generic DM IRQ handler
@@ -654,7 +699,7 @@ static void amdgpu_dm_irq_immediate_work(struct amdgpu_device *adev,
  * Calls all registered high irq work immediately, and schedules work for low
  * irq. The DM IRQ table is used to find the corresponding handlers.
  */
-static int amdgpu_dm_irq_handler(struct amdgpu_device *adev,
+STATIC_IFN_KUNIT int amdgpu_dm_irq_handler(struct amdgpu_device *adev,
 				 struct amdgpu_irq_src *source,
 				 struct amdgpu_iv_entry *entry)
 {
@@ -674,8 +719,9 @@ static int amdgpu_dm_irq_handler(struct amdgpu_device *adev,
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_irq_handler);
 
-static enum dc_irq_source amdgpu_dm_hpd_to_dal_irq_source(unsigned int type)
+STATIC_IFN_KUNIT enum dc_irq_source amdgpu_dm_hpd_to_dal_irq_source(unsigned int type)
 {
 	switch (type) {
 	case AMDGPU_HPD_1:
@@ -694,8 +740,9 @@ static enum dc_irq_source amdgpu_dm_hpd_to_dal_irq_source(unsigned int type)
 		return DC_IRQ_SOURCE_INVALID;
 	}
 }
+EXPORT_IF_KUNIT(amdgpu_dm_hpd_to_dal_irq_source);
 
-static int amdgpu_dm_set_hpd_irq_state(struct amdgpu_device *adev,
+STATIC_IFN_KUNIT int amdgpu_dm_set_hpd_irq_state(struct amdgpu_device *adev,
 				       struct amdgpu_irq_src *source,
 				       unsigned int type,
 				       enum amdgpu_interrupt_state state)
@@ -706,6 +753,7 @@ static int amdgpu_dm_set_hpd_irq_state(struct amdgpu_device *adev,
 	dc_interrupt_set(adev->dm.dc, src, st);
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_set_hpd_irq_state);
 
 static inline int dm_irq_state(struct amdgpu_device *adev,
 			       struct amdgpu_irq_src *source,
@@ -741,7 +789,7 @@ static inline int dm_irq_state(struct amdgpu_device *adev,
 	return 0;
 }
 
-static int amdgpu_dm_set_pflip_irq_state(struct amdgpu_device *adev,
+STATIC_IFN_KUNIT int amdgpu_dm_set_pflip_irq_state(struct amdgpu_device *adev,
 					 struct amdgpu_irq_src *source,
 					 unsigned int crtc_id,
 					 enum amdgpu_interrupt_state state)
@@ -754,8 +802,9 @@ static int amdgpu_dm_set_pflip_irq_state(struct amdgpu_device *adev,
 		IRQ_TYPE_PFLIP,
 		__func__);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_set_pflip_irq_state);
 
-static int amdgpu_dm_set_crtc_irq_state(struct amdgpu_device *adev,
+STATIC_IFN_KUNIT int amdgpu_dm_set_crtc_irq_state(struct amdgpu_device *adev,
 					struct amdgpu_irq_src *source,
 					unsigned int crtc_id,
 					enum amdgpu_interrupt_state state)
@@ -768,8 +817,9 @@ static int amdgpu_dm_set_crtc_irq_state(struct amdgpu_device *adev,
 		IRQ_TYPE_VBLANK,
 		__func__);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_set_crtc_irq_state);
 
-static int amdgpu_dm_set_vline0_irq_state(struct amdgpu_device *adev,
+STATIC_IFN_KUNIT int amdgpu_dm_set_vline0_irq_state(struct amdgpu_device *adev,
 					struct amdgpu_irq_src *source,
 					unsigned int crtc_id,
 					enum amdgpu_interrupt_state state)
@@ -782,8 +832,9 @@ static int amdgpu_dm_set_vline0_irq_state(struct amdgpu_device *adev,
 		IRQ_TYPE_VLINE0,
 		__func__);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_set_vline0_irq_state);
 
-static int amdgpu_dm_set_dmub_outbox_irq_state(struct amdgpu_device *adev,
+STATIC_IFN_KUNIT int amdgpu_dm_set_dmub_outbox_irq_state(struct amdgpu_device *adev,
 					struct amdgpu_irq_src *source,
 					unsigned int crtc_id,
 					enum amdgpu_interrupt_state state)
@@ -794,8 +845,9 @@ static int amdgpu_dm_set_dmub_outbox_irq_state(struct amdgpu_device *adev,
 	dc_interrupt_set(adev->dm.dc, irq_source, st);
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_set_dmub_outbox_irq_state);
 
-static int amdgpu_dm_set_vupdate_irq_state(struct amdgpu_device *adev,
+STATIC_IFN_KUNIT int amdgpu_dm_set_vupdate_irq_state(struct amdgpu_device *adev,
 					   struct amdgpu_irq_src *source,
 					   unsigned int crtc_id,
 					   enum amdgpu_interrupt_state state)
@@ -808,8 +860,9 @@ static int amdgpu_dm_set_vupdate_irq_state(struct amdgpu_device *adev,
 		IRQ_TYPE_VUPDATE,
 		__func__);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_set_vupdate_irq_state);
 
-static int amdgpu_dm_set_dmub_trace_irq_state(struct amdgpu_device *adev,
+STATIC_IFN_KUNIT int amdgpu_dm_set_dmub_trace_irq_state(struct amdgpu_device *adev,
 					   struct amdgpu_irq_src *source,
 					   unsigned int type,
 					   enum amdgpu_interrupt_state state)
@@ -820,6 +873,7 @@ static int amdgpu_dm_set_dmub_trace_irq_state(struct amdgpu_device *adev,
 	dc_interrupt_set(adev->dm.dc, irq_source, st);
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_set_dmub_trace_irq_state);
 
 static const struct amdgpu_irq_src_funcs dm_crtc_irq_funcs = {
 	.set = amdgpu_dm_set_crtc_irq_state,
@@ -879,12 +933,15 @@ void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev)
 	adev->hpd_irq.num_types = adev->mode_info.num_hpd;
 	adev->hpd_irq.funcs = &dm_hpd_irq_funcs;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_set_irq_funcs);
+
 void amdgpu_dm_outbox_init(struct amdgpu_device *adev)
 {
 	dc_interrupt_set(adev->dm.dc,
 		DC_IRQ_SOURCE_DMCUB_OUTBOX,
 		true);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_outbox_init);
 
 /**
  * amdgpu_dm_hpd_init - hpd setup callback.
@@ -967,6 +1024,7 @@ void amdgpu_dm_hpd_init(struct amdgpu_device *adev)
 	if (use_polling)
 		drm_kms_helper_poll_init(dev);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_hpd_init);
 
 /**
  * amdgpu_dm_hpd_fini - hpd tear down callback.
@@ -1020,3 +1078,1531 @@ void amdgpu_dm_hpd_fini(struct amdgpu_device *adev)
 	if (dev->mode_config.poll_enabled)
 		drm_kms_helper_poll_fini(dev);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_hpd_fini);
+
+/* ========== HPD handling ========== */
+static void force_connector_state(
+	struct amdgpu_dm_connector *aconnector,
+	enum drm_connector_force force_state)
+{
+	struct drm_connector *connector = &aconnector->base;
+
+	mutex_lock(&connector->dev->mode_config.mutex);
+	aconnector->base.force = force_state;
+	mutex_unlock(&connector->dev->mode_config.mutex);
+
+	mutex_lock(&aconnector->hpd_lock);
+	drm_kms_helper_connector_hotplug_event(connector);
+	mutex_unlock(&aconnector->hpd_lock);
+}
+
+STATIC_IFN_KUNIT void dm_handle_hpd_rx_offload_work(struct work_struct *work)
+{
+	struct hpd_rx_irq_offload_work *offload_work;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *dc_link;
+	struct amdgpu_device *adev;
+	enum dc_connection_type new_connection_type = dc_connection_none;
+	unsigned long flags;
+	union test_response test_response;
+
+	memset(&test_response, 0, sizeof(test_response));
+
+	offload_work = container_of(work, struct hpd_rx_irq_offload_work, work);
+	aconnector = offload_work->offload_wq->aconnector;
+	adev = offload_work->adev;
+
+	if (!aconnector) {
+		drm_err(adev_to_drm(adev), "Can't retrieve aconnector in hpd_rx_irq_offload_work");
+		goto skip;
+	}
+
+	dc_link = aconnector->dc_link;
+
+	mutex_lock(&aconnector->hpd_lock);
+	if (!dc_link_detect_connection_type(dc_link, &new_connection_type))
+		drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n");
+	mutex_unlock(&aconnector->hpd_lock);
+
+	if (new_connection_type == dc_connection_none)
+		goto skip;
+
+	if (amdgpu_in_reset(adev))
+		goto skip;
+
+	if (offload_work->data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY ||
+		offload_work->data.bytes.device_service_irq.bits.DOWN_REP_MSG_RDY) {
+		dm_handle_mst_sideband_msg_ready_event(&aconnector->mst_mgr, DOWN_OR_UP_MSG_RDY_EVENT);
+		spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags);
+		offload_work->offload_wq->is_handling_mst_msg_rdy_event = false;
+		spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags);
+		goto skip;
+	}
+
+	mutex_lock(&adev->dm.dc_lock);
+	if (offload_work->data.bytes.device_service_irq.bits.AUTOMATED_TEST) {
+		dc_link_dp_handle_automated_test(dc_link);
+
+		if (aconnector->timing_changed) {
+			/* force connector disconnect and reconnect */
+			force_connector_state(aconnector, DRM_FORCE_OFF);
+			msleep(100);
+			force_connector_state(aconnector, DRM_FORCE_UNSPECIFIED);
+		}
+
+		test_response.bits.ACK = 1;
+
+		core_link_write_dpcd(
+		dc_link,
+		DP_TEST_RESPONSE,
+		&test_response.raw,
+		sizeof(test_response));
+	} else if ((dc_link->connector_signal != SIGNAL_TYPE_EDP) &&
+			dc_link_check_link_loss_status(dc_link, &offload_work->data) &&
+			dc_link_dp_allow_hpd_rx_irq(dc_link)) {
+		/* offload_work->data is from handle_hpd_rx_irq->
+		 * schedule_hpd_rx_offload_work.this is defer handle
+		 * for hpd short pulse. upon here, link status may be
+		 * changed, need get latest link status from dpcd
+		 * registers. if link status is good, skip run link
+		 * training again.
+		 */
+		union hpd_irq_data irq_data;
+
+		memset(&irq_data, 0, sizeof(irq_data));
+
+		/* before dc_link_dp_handle_link_loss, allow new link lost handle
+		 * request be added to work queue if link lost at end of dc_link_
+		 * dp_handle_link_loss
+		 */
+		spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags);
+		offload_work->offload_wq->is_handling_link_loss = false;
+		spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags);
+
+		if ((dc_link_dp_read_hpd_rx_irq_data(dc_link, &irq_data) == DC_OK) &&
+			dc_link_check_link_loss_status(dc_link, &irq_data))
+			dc_link_dp_handle_link_loss(dc_link);
+	}
+	mutex_unlock(&adev->dm.dc_lock);
+
+skip:
+	kfree(offload_work);
+
+}
+EXPORT_IF_KUNIT(dm_handle_hpd_rx_offload_work);
+
+struct hpd_rx_irq_offload_work_queue *amdgpu_dm_hpd_rx_irq_create_workqueue(struct amdgpu_device *adev)
+{
+	struct dc *dc = adev->dm.dc;
+	int max_caps = dc->caps.max_links;
+	int i = 0;
+	struct hpd_rx_irq_offload_work_queue *hpd_rx_offload_wq = NULL;
+
+	hpd_rx_offload_wq = kzalloc_objs(*hpd_rx_offload_wq, max_caps);
+
+	if (!hpd_rx_offload_wq)
+		return NULL;
+
+
+	for (i = 0; i < max_caps; i++) {
+		hpd_rx_offload_wq[i].wq =
+				    create_singlethread_workqueue("amdgpu_dm_hpd_rx_offload_wq");
+
+		if (hpd_rx_offload_wq[i].wq == NULL) {
+			drm_err(adev_to_drm(adev), "create amdgpu_dm_hpd_rx_offload_wq fail!");
+			goto out_err;
+		}
+
+		spin_lock_init(&hpd_rx_offload_wq[i].offload_lock);
+	}
+
+	return hpd_rx_offload_wq;
+
+out_err:
+	for (i = 0; i < max_caps; i++) {
+		if (hpd_rx_offload_wq[i].wq)
+			destroy_workqueue(hpd_rx_offload_wq[i].wq);
+	}
+	kfree(hpd_rx_offload_wq);
+	return NULL;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_hpd_rx_irq_create_workqueue);
+
+void amdgpu_dm_hpd_rx_irq_work_suspend(struct amdgpu_display_manager *dm)
+{
+	int i;
+
+	if (dm->hpd_rx_offload_wq) {
+		for (i = 0; i < dm->dc->caps.max_links; i++)
+			flush_workqueue(dm->hpd_rx_offload_wq[i].wq);
+	}
+}
+EXPORT_IF_KUNIT(amdgpu_dm_hpd_rx_irq_work_suspend);
+
+STATIC_IFN_KUNIT bool are_sinks_equal(const struct dc_sink *sink1, const struct dc_sink *sink2)
+{
+	if (!sink1 || !sink2)
+		return false;
+	if (sink1->sink_signal != sink2->sink_signal)
+		return false;
+
+	if (sink1->dc_edid.length != sink2->dc_edid.length)
+		return false;
+
+	if (memcmp(sink1->dc_edid.raw_edid, sink2->dc_edid.raw_edid,
+		   sink1->dc_edid.length) != 0)
+		return false;
+	return true;
+}
+EXPORT_IF_KUNIT(are_sinks_equal);
+
+
+/**
+ * DOC: amdgpu_dm_hdmi_hpd_debounce_work
+ *
+ * HDMI HPD debounce delay in milliseconds. When an HDMI display toggles HPD
+ * (such as during power save transitions), this delay determines how long to
+ * wait before processing the HPD event. This allows distinguishing between a
+ * physical unplug (>hdmi_hpd_debounce_delay)
+ * and a spontaneous RX HPD toggle (<hdmi_hpd_debounce_delay).
+ *
+ * If the toggle is less than this delay, the driver compares sink capabilities
+ * and permits a hotplug event if they changed.
+ *
+ * The default value of 1500ms was chosen based on experimental testing with
+ * various monitors that exhibit spontaneous HPD toggling behavior.
+ */
+void amdgpu_dm_hdmi_hpd_debounce_work(struct work_struct *work)
+{
+	struct amdgpu_dm_connector *aconnector =
+		container_of(to_delayed_work(work), struct amdgpu_dm_connector,
+			     hdmi_hpd_debounce_work);
+	struct drm_connector *connector = &aconnector->base;
+	struct drm_device *dev = connector->dev;
+	struct amdgpu_device *adev = drm_to_adev(dev);
+	struct dc *dc = aconnector->dc_link->ctx->dc;
+	bool fake_reconnect = false;
+	bool reallow_idle = false;
+	bool ret = false;
+
+	guard(mutex)(&aconnector->hpd_lock);
+
+	/* Re-detect the display */
+	scoped_guard(mutex, &adev->dm.dc_lock) {
+		if (dc->caps.ips_support && dc->ctx->dmub_srv->idle_allowed) {
+			dc_allow_idle_optimizations(dc, false);
+			reallow_idle = true;
+		}
+		ret = dc_link_detect(aconnector->dc_link, DETECT_REASON_HPD);
+	}
+
+	if (ret) {
+		/* Apply workaround delay for certain panels */
+		amdgpu_dm_apply_delay_after_dpcd_poweroff(adev, aconnector->dc_sink);
+		/* Compare sinks to determine if this was a spontaneous HPD toggle */
+		if (are_sinks_equal(aconnector->dc_link->local_sink, aconnector->hdmi_prev_sink)) {
+			/*
+			 * Sinks match - this was a spontaneous HDMI HPD toggle.
+			 */
+			drm_dbg_kms(dev, "HDMI HPD: Sink unchanged after debounce, internal re-enable\n");
+			fake_reconnect = true;
+		}
+
+		/* Update connector state */
+		amdgpu_dm_update_connector_after_detect(aconnector);
+
+		drm_modeset_lock_all(dev);
+		dm_restore_drm_connector_state(dev, connector);
+		drm_modeset_unlock_all(dev);
+
+		/* Only notify OS if sink actually changed */
+		if (!fake_reconnect && aconnector->base.force == DRM_FORCE_UNSPECIFIED)
+			drm_kms_helper_hotplug_event(dev);
+	}
+
+	/* Release the cached sink reference */
+	if (aconnector->hdmi_prev_sink) {
+		dc_sink_release(aconnector->hdmi_prev_sink);
+		aconnector->hdmi_prev_sink = NULL;
+	}
+
+	scoped_guard(mutex, &adev->dm.dc_lock) {
+		if (reallow_idle && dc->caps.ips_support)
+			dc_allow_idle_optimizations(dc, true);
+	}
+}
+EXPORT_IF_KUNIT(amdgpu_dm_hdmi_hpd_debounce_work);
+
+STATIC_IFN_KUNIT void handle_hpd_irq_helper(struct amdgpu_dm_connector *aconnector,
+				  enum dc_detect_reason reason)
+{
+	struct drm_connector *connector = &aconnector->base;
+	struct drm_device *dev = connector->dev;
+	enum dc_connection_type new_connection_type = dc_connection_none;
+	struct amdgpu_device *adev = drm_to_adev(dev);
+	struct dm_connector_state *dm_con_state = to_dm_connector_state(connector->state);
+	struct dc *dc = aconnector->dc_link->ctx->dc;
+	bool ret = false;
+	bool debounce_required = false;
+
+	if (adev->dm.disable_hpd_irq)
+		return;
+
+	/*
+	 * In case of failure or MST no need to update connector status or notify the OS
+	 * since (for MST case) MST does this in its own context.
+	 */
+	guard(mutex)(&aconnector->hpd_lock);
+
+	if (adev->dm.hdcp_workqueue) {
+		hdcp_reset_display(adev->dm.hdcp_workqueue, aconnector->dc_link->link_index);
+		dm_con_state->update_hdcp = true;
+	}
+	if (aconnector->fake_enable)
+		aconnector->fake_enable = false;
+
+	aconnector->timing_changed = false;
+
+	if (!dc_link_detect_connection_type(aconnector->dc_link, &new_connection_type))
+		drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n");
+
+	/*
+	 * Check for HDMI disconnect with debounce enabled.
+	 */
+	debounce_required = (aconnector->hdmi_hpd_debounce_delay_ms > 0 &&
+			      dc_is_hdmi_signal(aconnector->dc_link->connector_signal) &&
+			      new_connection_type == dc_connection_none &&
+			      aconnector->dc_link->local_sink != NULL);
+
+	if (aconnector->base.force && new_connection_type == dc_connection_none) {
+		amdgpu_dm_emulated_link_detect(aconnector->dc_link);
+
+		drm_modeset_lock_all(dev);
+		dm_restore_drm_connector_state(dev, connector);
+		drm_modeset_unlock_all(dev);
+
+		if (aconnector->base.force == DRM_FORCE_UNSPECIFIED ||
+		    reason == DETECT_REASON_HPDRX)
+			drm_kms_helper_connector_hotplug_event(connector);
+	} else if (debounce_required) {
+		/*
+		 * HDMI disconnect detected - schedule delayed work instead of
+		 * processing immediately. This allows us to coalesce spurious
+		 * HDMI signals from physical unplugs.
+		 */
+		drm_dbg_kms(dev, "HDMI HPD: Disconnect detected, scheduling debounce work (%u ms)\n",
+			    aconnector->hdmi_hpd_debounce_delay_ms);
+
+		/* Cache the current sink for later comparison */
+		if (aconnector->hdmi_prev_sink)
+			dc_sink_release(aconnector->hdmi_prev_sink);
+		aconnector->hdmi_prev_sink = aconnector->dc_link->local_sink;
+		if (aconnector->hdmi_prev_sink)
+			dc_sink_retain(aconnector->hdmi_prev_sink);
+
+		/* Schedule delayed detection. */
+		if (mod_delayed_work(system_percpu_wq,
+				 &aconnector->hdmi_hpd_debounce_work,
+				 msecs_to_jiffies(aconnector->hdmi_hpd_debounce_delay_ms)))
+			drm_dbg_kms(dev, "HDMI HPD: Re-scheduled debounce work\n");
+
+	} else {
+
+		/* If the aconnector->hdmi_hpd_debounce_work is scheduled, exit early */
+		if (delayed_work_pending(&aconnector->hdmi_hpd_debounce_work))
+			return;
+
+		scoped_guard(mutex, &adev->dm.dc_lock) {
+			dc_exit_ips_for_hw_access(dc);
+			ret = dc_link_detect(aconnector->dc_link, reason);
+		}
+		if (ret) {
+			/* w/a delay for certain panels */
+			amdgpu_dm_apply_delay_after_dpcd_poweroff(adev, aconnector->dc_sink);
+			amdgpu_dm_update_connector_after_detect(aconnector);
+
+			drm_modeset_lock_all(dev);
+			dm_restore_drm_connector_state(dev, connector);
+			drm_modeset_unlock_all(dev);
+
+			if (aconnector->base.force == DRM_FORCE_UNSPECIFIED ||
+			    reason == DETECT_REASON_HPDRX)
+				drm_kms_helper_connector_hotplug_event(connector);
+		}
+	}
+}
+EXPORT_IF_KUNIT(handle_hpd_irq_helper);
+
+STATIC_IFN_KUNIT void handle_hpd_irq(void *param)
+{
+	struct amdgpu_dm_connector *aconnector = (struct amdgpu_dm_connector *)param;
+
+	handle_hpd_irq_helper(aconnector, DETECT_REASON_HPD);
+
+}
+EXPORT_IF_KUNIT(handle_hpd_irq);
+
+STATIC_IFN_KUNIT void schedule_hpd_rx_offload_work(struct amdgpu_device *adev, struct hpd_rx_irq_offload_work_queue *offload_wq,
+							union hpd_irq_data hpd_irq_data)
+{
+	struct hpd_rx_irq_offload_work *offload_work = kzalloc_obj(*offload_work);
+
+	if (!offload_work) {
+		drm_err(adev_to_drm(adev), "Failed to allocate hpd_rx_irq_offload_work.\n");
+		return;
+	}
+
+	INIT_WORK(&offload_work->work, dm_handle_hpd_rx_offload_work);
+	offload_work->data = hpd_irq_data;
+	offload_work->offload_wq = offload_wq;
+	offload_work->adev = adev;
+
+	queue_work(offload_wq->wq, &offload_work->work);
+	drm_dbg_kms(adev_to_drm(adev), "queue work to handle hpd_rx offload work");
+}
+EXPORT_IF_KUNIT(schedule_hpd_rx_offload_work);
+
+STATIC_IFN_KUNIT void handle_hpd_rx_irq(void *param)
+{
+	struct amdgpu_dm_connector *aconnector = (struct amdgpu_dm_connector *)param;
+	struct drm_connector *connector = &aconnector->base;
+	struct drm_device *dev = connector->dev;
+	struct dc_link *dc_link = aconnector->dc_link;
+	bool is_mst_root_connector = aconnector->mst_mgr.mst_state;
+	bool result = false;
+	enum dc_connection_type new_connection_type = dc_connection_none;
+	struct amdgpu_device *adev = drm_to_adev(dev);
+	union hpd_irq_data hpd_irq_data;
+	bool link_loss = false;
+	bool has_left_work = false;
+	int idx = dc_link->link_index;
+	struct hpd_rx_irq_offload_work_queue *offload_wq = &adev->dm.hpd_rx_offload_wq[idx];
+	struct dc *dc = aconnector->dc_link->ctx->dc;
+
+	memset(&hpd_irq_data, 0, sizeof(hpd_irq_data));
+
+	if (adev->dm.disable_hpd_irq)
+		return;
+
+	/*
+	 * TODO:Temporary add mutex to protect hpd interrupt not have a gpio
+	 * conflict, after implement i2c helper, this mutex should be
+	 * retired.
+	 */
+	mutex_lock(&aconnector->hpd_lock);
+
+	result = dc_link_handle_hpd_rx_irq(dc_link, &hpd_irq_data,
+						&link_loss, true, &has_left_work);
+
+	if (!has_left_work)
+		goto out;
+
+	if (hpd_irq_data.bytes.device_service_irq.bits.AUTOMATED_TEST) {
+		schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data);
+		goto out;
+	}
+
+	if (dc_link_dp_allow_hpd_rx_irq(dc_link)) {
+		if (hpd_irq_data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY ||
+			hpd_irq_data.bytes.device_service_irq.bits.DOWN_REP_MSG_RDY) {
+			bool skip = false;
+
+			/*
+			 * DOWN_REP_MSG_RDY is also handled by polling method
+			 * mgr->cbs->poll_hpd_irq()
+			 */
+			spin_lock(&offload_wq->offload_lock);
+			skip = offload_wq->is_handling_mst_msg_rdy_event;
+
+			if (!skip)
+				offload_wq->is_handling_mst_msg_rdy_event = true;
+
+			spin_unlock(&offload_wq->offload_lock);
+
+			if (!skip)
+				schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data);
+
+			goto out;
+		}
+
+		if (link_loss) {
+			bool skip = false;
+
+			spin_lock(&offload_wq->offload_lock);
+			skip = offload_wq->is_handling_link_loss;
+
+			if (!skip)
+				offload_wq->is_handling_link_loss = true;
+
+			spin_unlock(&offload_wq->offload_lock);
+
+			if (!skip)
+				schedule_hpd_rx_offload_work(adev, offload_wq, hpd_irq_data);
+
+			goto out;
+		}
+	}
+
+out:
+	if (result && !is_mst_root_connector) {
+		/* Downstream Port status changed. */
+		if (!dc_link_detect_connection_type(dc_link, &new_connection_type))
+			drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n");
+
+		if (aconnector->base.force && new_connection_type == dc_connection_none) {
+			amdgpu_dm_emulated_link_detect(dc_link);
+
+			if (aconnector->fake_enable)
+				aconnector->fake_enable = false;
+
+			amdgpu_dm_update_connector_after_detect(aconnector);
+
+
+			drm_modeset_lock_all(dev);
+			dm_restore_drm_connector_state(dev, connector);
+			drm_modeset_unlock_all(dev);
+
+			drm_kms_helper_connector_hotplug_event(connector);
+		} else {
+			bool ret = false;
+
+			mutex_lock(&adev->dm.dc_lock);
+			dc_exit_ips_for_hw_access(dc);
+			ret = dc_link_detect(dc_link, DETECT_REASON_HPDRX);
+			mutex_unlock(&adev->dm.dc_lock);
+
+			if (ret) {
+				if (aconnector->fake_enable)
+					aconnector->fake_enable = false;
+
+				amdgpu_dm_update_connector_after_detect(aconnector);
+
+				drm_modeset_lock_all(dev);
+				dm_restore_drm_connector_state(dev, connector);
+				drm_modeset_unlock_all(dev);
+
+				drm_kms_helper_connector_hotplug_event(connector);
+			}
+		}
+	}
+	if (hpd_irq_data.bytes.device_service_irq.bits.CP_IRQ) {
+		if (adev->dm.hdcp_workqueue)
+			hdcp_handle_cpirq(adev->dm.hdcp_workqueue,  aconnector->base.index);
+	}
+
+	if (dc_link->type != dc_connection_mst_branch)
+		drm_dp_cec_irq(&aconnector->dm_dp_aux.aux);
+
+	mutex_unlock(&aconnector->hpd_lock);
+}
+EXPORT_IF_KUNIT(handle_hpd_rx_irq);
+
+/**
+ * dmub_hpd_callback - DMUB HPD interrupt processing callback.
+ * @adev: amdgpu_device pointer
+ * @notify: dmub notification structure
+ *
+ * Dmub Hpd interrupt processing callback. Gets displayindex through the
+ * ink index and calls helper to do the processing.
+ */
+STATIC_IFN_KUNIT void dmub_hpd_callback(struct amdgpu_device *adev,
+			      struct dmub_notification *notify)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_dm_connector *hpd_aconnector = NULL;
+	struct drm_connector *connector;
+	struct drm_connector_list_iter iter;
+	struct dc_link *link;
+	u8 link_index = 0;
+	struct drm_device *dev;
+
+	if (adev == NULL)
+		return;
+
+	if (notify == NULL) {
+		drm_err(adev_to_drm(adev), "DMUB HPD callback notification was NULL");
+		return;
+	}
+
+	if (notify->link_index > adev->dm.dc->link_count) {
+		drm_err(adev_to_drm(adev), "DMUB HPD index (%u)is abnormal", notify->link_index);
+		return;
+	}
+
+	/* Skip DMUB HPD IRQ in suspend/resume. We will probe them later. */
+	if (notify->type == DMUB_NOTIFICATION_HPD && adev->in_suspend) {
+		drm_info(adev_to_drm(adev), "Skip DMUB HPD IRQ callback in suspend/resume\n");
+		return;
+	}
+
+	link_index = notify->link_index;
+	link = adev->dm.dc->links[link_index];
+	dev = adev->dm.ddev;
+
+	drm_connector_list_iter_begin(dev, &iter);
+	drm_for_each_connector_iter(connector, &iter) {
+
+		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
+			continue;
+
+		aconnector = to_amdgpu_dm_connector(connector);
+		if (link && aconnector->dc_link == link) {
+			if (notify->type == DMUB_NOTIFICATION_HPD)
+				drm_info(adev_to_drm(adev), "DMUB HPD IRQ callback: link_index=%u\n", link_index);
+			else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ)
+				drm_info(adev_to_drm(adev), "DMUB HPD RX IRQ callback: link_index=%u\n", link_index);
+			else
+				drm_warn(adev_to_drm(adev), "DMUB Unknown HPD callback type %d, link_index=%u\n",
+						notify->type, link_index);
+
+			hpd_aconnector = aconnector;
+			break;
+		}
+	}
+	drm_connector_list_iter_end(&iter);
+
+	if (hpd_aconnector) {
+		if (notify->type == DMUB_NOTIFICATION_HPD) {
+			if (hpd_aconnector->dc_link->hpd_status == (notify->hpd_status == DP_HPD_PLUG))
+				drm_warn(adev_to_drm(adev), "DMUB reported hpd status unchanged. link_index=%u\n", link_index);
+			handle_hpd_irq_helper(hpd_aconnector, DETECT_REASON_HPD);
+		} else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ) {
+			handle_hpd_rx_irq(hpd_aconnector);
+		}
+	}
+}
+EXPORT_IF_KUNIT(dmub_hpd_callback);
+
+/**
+ * dmub_hpd_sense_callback - DMUB HPD sense processing callback.
+ * @adev: amdgpu_device pointer
+ * @notify: dmub notification structure
+ *
+ * HPD sense changes can occur during low power states and need to be
+ * notified from firmware to driver.
+ */
+STATIC_IFN_KUNIT void dmub_hpd_sense_callback(struct amdgpu_device *adev,
+			      struct dmub_notification *notify)
+{
+	drm_dbg_driver(adev_to_drm(adev), "DMUB HPD SENSE callback.\n");
+}
+EXPORT_IF_KUNIT(dmub_hpd_sense_callback);
+
+int amdgpu_dm_register_hpd_handlers(struct amdgpu_device *adev)
+{
+	struct drm_device *dev = adev_to_drm(adev);
+	struct drm_connector *connector;
+	struct amdgpu_dm_connector *aconnector;
+	const struct dc_link *dc_link;
+	struct dc_interrupt_params int_params = {0};
+
+	int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
+	int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
+
+	if (dc_is_dmub_outbox_supported(adev->dm.dc)) {
+		if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD,
+			dmub_hpd_callback, true)) {
+			drm_err(adev_to_drm(adev), "fail to register dmub hpd callback");
+			return -EINVAL;
+		}
+
+		if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD_IRQ,
+			dmub_hpd_callback, true)) {
+			drm_err(adev_to_drm(adev), "fail to register dmub hpd callback");
+			return -EINVAL;
+		}
+
+		if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_HPD_SENSE_NOTIFY,
+			dmub_hpd_sense_callback, true)) {
+			drm_err(adev_to_drm(adev), "fail to register dmub hpd sense callback");
+			return -EINVAL;
+		}
+	}
+
+	list_for_each_entry(connector,
+			&dev->mode_config.connector_list, head)	{
+
+		if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
+			continue;
+
+		aconnector = to_amdgpu_dm_connector(connector);
+		dc_link = aconnector->dc_link;
+
+		if (dc_link->irq_source_hpd != DC_IRQ_SOURCE_INVALID) {
+			int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+			int_params.irq_source = dc_link->irq_source_hpd;
+
+			if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
+				int_params.irq_source  < DC_IRQ_SOURCE_HPD1 ||
+				int_params.irq_source  > DC_IRQ_SOURCE_HPD6) {
+				drm_err(adev_to_drm(adev), "Failed to register hpd irq!\n");
+				return -EINVAL;
+			}
+
+			if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
+				handle_hpd_irq, (void *) aconnector))
+				return -ENOMEM;
+		}
+
+		if (dc_link->irq_source_hpd_rx != DC_IRQ_SOURCE_INVALID) {
+
+			/* Also register for DP short pulse (hpd_rx). */
+			int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+			int_params.irq_source =	dc_link->irq_source_hpd_rx;
+
+			if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
+				int_params.irq_source  < DC_IRQ_SOURCE_HPD1RX ||
+				int_params.irq_source  > DC_IRQ_SOURCE_HPD6RX) {
+				drm_err(adev_to_drm(adev), "Failed to register hpd rx irq!\n");
+				return -EINVAL;
+			}
+
+			if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
+				handle_hpd_rx_irq, (void *) aconnector))
+				return -ENOMEM;
+		}
+	}
+	return 0;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_register_hpd_handlers);
+
+/* ========== IRQ handlers ========== */
+struct amdgpu_crtc *
+amdgpu_dm_get_crtc_by_otg_inst(struct amdgpu_device *adev,
+		     int otg_inst)
+{
+	struct drm_device *dev = adev_to_drm(adev);
+	struct drm_crtc *crtc;
+	struct amdgpu_crtc *amdgpu_crtc;
+
+	if (WARN_ON(otg_inst == -1))
+		return adev->mode_info.crtcs[0];
+
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+		amdgpu_crtc = to_amdgpu_crtc(crtc);
+
+		if (amdgpu_crtc->otg_inst == otg_inst)
+			return amdgpu_crtc;
+	}
+
+	return NULL;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_get_crtc_by_otg_inst);
+
+/**
+ * dm_pflip_high_irq() - Handle pageflip interrupt
+ * @interrupt_params: ignored
+ *
+ * Handles the pageflip interrupt by notifying all interested parties
+ * that the pageflip has been completed.
+ */
+STATIC_IFN_KUNIT void dm_pflip_high_irq(void *interrupt_params)
+{
+	struct amdgpu_crtc *amdgpu_crtc;
+	struct common_irq_params *irq_params = interrupt_params;
+	struct amdgpu_device *adev = irq_params->adev;
+	struct drm_device *dev = adev_to_drm(adev);
+	unsigned long flags;
+	struct drm_pending_vblank_event *e;
+	u32 vpos, hpos, v_blank_start, v_blank_end;
+	bool vrr_active;
+
+	amdgpu_crtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_PFLIP);
+
+	/* IRQ could occur when in initial stage */
+	/* TODO work and BO cleanup */
+	if (amdgpu_crtc == NULL) {
+		drm_dbg_state(dev, "CRTC is null, returning.\n");
+		return;
+	}
+
+	spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
+
+	if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED) {
+		drm_dbg_state(dev,
+			      "amdgpu_crtc->pflip_status = %d != AMDGPU_FLIP_SUBMITTED(%d) on crtc:%d[%p]\n",
+			      amdgpu_crtc->pflip_status, AMDGPU_FLIP_SUBMITTED,
+			      amdgpu_crtc->crtc_id, amdgpu_crtc);
+		spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
+		return;
+	}
+
+	/* page flip completed. */
+	e = amdgpu_crtc->event;
+	amdgpu_crtc->event = NULL;
+
+	WARN_ON(!e);
+
+	vrr_active = amdgpu_dm_crtc_vrr_active_irq(amdgpu_crtc);
+
+	/* Fixed refresh rate, or VRR scanout position outside front-porch? */
+	if (!vrr_active ||
+	    !dc_stream_get_scanoutpos(amdgpu_crtc->dm_irq_params.stream, &v_blank_start,
+				      &v_blank_end, &hpos, &vpos) ||
+	    (vpos < v_blank_start)) {
+		/* Update to correct count and vblank timestamp if racing with
+		 * vblank irq. This also updates to the correct vblank timestamp
+		 * even in VRR mode, as scanout is past the front-porch atm.
+		 */
+		drm_crtc_accurate_vblank_count(&amdgpu_crtc->base);
+
+		/* Wake up userspace by sending the pageflip event with proper
+		 * count and timestamp of vblank of flip completion.
+		 */
+		if (e) {
+			drm_crtc_send_vblank_event(&amdgpu_crtc->base, e);
+
+			/* Event sent, so done with vblank for this flip */
+			drm_crtc_vblank_put(&amdgpu_crtc->base);
+		}
+	} else if (e) {
+		/* VRR active and inside front-porch: vblank count and
+		 * timestamp for pageflip event will only be up to date after
+		 * drm_crtc_handle_vblank() has been executed from late vblank
+		 * irq handler after start of back-porch (vline 0). We queue the
+		 * pageflip event for send-out by drm_crtc_handle_vblank() with
+		 * updated timestamp and count, once it runs after us.
+		 *
+		 * We need to open-code this instead of using the helper
+		 * drm_crtc_arm_vblank_event(), as that helper would
+		 * call drm_crtc_accurate_vblank_count(), which we must
+		 * not call in VRR mode while we are in front-porch!
+		 */
+
+		/* sequence will be replaced by real count during send-out. */
+		e->sequence = drm_crtc_vblank_count(&amdgpu_crtc->base);
+		e->pipe = amdgpu_crtc->crtc_id;
+
+		list_add_tail(&e->base.link, &adev_to_drm(adev)->vblank_event_list);
+		e = NULL;
+	}
+
+	/* Keep track of vblank of this flip for flip throttling. We use the
+	 * cooked hw counter, as that one incremented at start of this vblank
+	 * of pageflip completion, so last_flip_vblank is the forbidden count
+	 * for queueing new pageflips if vsync + VRR is enabled.
+	 */
+	amdgpu_crtc->dm_irq_params.last_flip_vblank =
+		amdgpu_get_vblank_counter_kms(&amdgpu_crtc->base);
+
+	amdgpu_crtc->pflip_status = AMDGPU_FLIP_NONE;
+	spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
+
+	drm_dbg_state(dev,
+		      "crtc:%d[%p], pflip_stat:AMDGPU_FLIP_NONE, vrr[%d]-fp %d\n",
+		      amdgpu_crtc->crtc_id, amdgpu_crtc, vrr_active, (int)!e);
+}
+EXPORT_IF_KUNIT(dm_pflip_high_irq);
+
+STATIC_IFN_KUNIT void dm_handle_vmin_vmax_update(struct work_struct *offload_work)
+{
+	struct vupdate_offload_work *work = container_of(offload_work, struct vupdate_offload_work, work);
+	struct amdgpu_device *adev = work->adev;
+	struct dc_stream_state *stream = work->stream;
+	struct dc_crtc_timing_adjust *adjust = work->adjust;
+
+	mutex_lock(&adev->dm.dc_lock);
+	dc_stream_adjust_vmin_vmax(adev->dm.dc, stream, adjust);
+	mutex_unlock(&adev->dm.dc_lock);
+
+	dc_stream_release(stream);
+	kfree(work->adjust);
+	kfree(work);
+}
+EXPORT_IF_KUNIT(dm_handle_vmin_vmax_update);
+
+static void schedule_dc_vmin_vmax(struct amdgpu_device *adev,
+	struct dc_stream_state *stream,
+	struct dc_crtc_timing_adjust *adjust)
+{
+	struct vupdate_offload_work *offload_work = kzalloc_obj(*offload_work,
+								GFP_NOWAIT);
+	if (!offload_work) {
+		drm_dbg_driver(adev_to_drm(adev), "Failed to allocate vupdate_offload_work\n");
+		return;
+	}
+
+	struct dc_crtc_timing_adjust *adjust_copy = kzalloc_obj(*adjust_copy,
+								GFP_NOWAIT);
+	if (!adjust_copy) {
+		drm_dbg_driver(adev_to_drm(adev), "Failed to allocate adjust_copy\n");
+		kfree(offload_work);
+		return;
+	}
+
+	dc_stream_retain(stream);
+	memcpy(adjust_copy, adjust, sizeof(*adjust_copy));
+
+	INIT_WORK(&offload_work->work, dm_handle_vmin_vmax_update);
+	offload_work->adev = adev;
+	offload_work->stream = stream;
+	offload_work->adjust = adjust_copy;
+
+	queue_work(adev->dm.vmin_vmax_wq, &offload_work->work);
+}
+
+/**
+ * dm_crtc_high_irq_handler() - Common OTG vblank/flip event handling
+ * @adev: amdgpu device
+ * @acrtc: the CRTC to service
+ * Performs writeback completion, vblank event handling, CRC processing, VRR BTR
+ * updates and pageflip completion delivery.
+ *
+ * On DCN this is driven by VUPDATE_NO_LOCK (the register latch point) from
+ * dm_vupdate_high_irq(); on DCE it is driven by VLINE0 at the start of vblank
+ * from dm_crtc_high_irq().
+ */
+static void dm_crtc_high_irq_handler(struct amdgpu_device *adev,
+				     struct amdgpu_crtc *acrtc)
+{
+	unsigned long flags;
+	int vrr_active;
+	bool is_dcn = amdgpu_ip_version(adev, DCE_HWIP, 0) != 0;
+
+	if (acrtc->wb_conn && acrtc->wb_pending) {
+		struct dc_stream_state *stream = acrtc->dm_irq_params.stream;
+		unsigned int v_total, refresh_hz;
+
+		v_total = stream->adjust.v_total_max ?
+			  stream->adjust.v_total_max : stream->timing.v_total;
+		refresh_hz = div_u64((uint64_t) stream->timing.pix_clk_100hz *
+			     100LL, (v_total * stream->timing.h_total));
+		mdelay(1000 / refresh_hz);
+
+		/*
+		 * Completion (signalling the out fence and releasing the vblank
+		 * reference taken in dm_set_writeback()) is handled by the shared
+		 * helper, which is also used by the teardown path.
+		 */
+		if (amdgpu_dm_crtc_complete_writeback(acrtc))
+			dc_stream_fc_disable_writeback(adev->dm.dc,
+						       acrtc->dm_irq_params.stream, 0);
+	}
+
+	vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc);
+
+	drm_dbg_vbl(adev_to_drm(adev),
+		    "crtc:%d, vupdate-vrr:%d, planes:%d\n", acrtc->crtc_id,
+		    vrr_active, acrtc->dm_irq_params.active_planes);
+
+	/**
+	 * Core vblank handling.
+	 *
+	 * On DCN this handler runs at VUPDATE_NO_LOCK, the register latch
+	 * point, which is the correct place to timestamp both VRR and non-VRR
+	 * vblanks.
+	 *
+	 * On DCE this handler runs at the start of front-porch, where only
+	 * non-VRR timestamping is valid; VRR vblank is deferred to
+	 * dm_vupdate_high_irq() after end of front-porch.
+	 */
+	if (is_dcn || !vrr_active)
+		amdgpu_dm_crtc_handle_vblank(acrtc);
+
+	/**
+	 * Following stuff must happen at start of vblank, for crc
+	 * computation and below-the-range btr support in vrr mode.
+	 */
+	amdgpu_dm_crtc_handle_crc_irq(&acrtc->base);
+
+	/* BTR updates need to happen before VUPDATE on Vega and above. */
+	if (adev->family < AMDGPU_FAMILY_AI)
+		return;
+
+	spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
+
+	if (acrtc->dm_irq_params.stream &&
+		acrtc->dm_irq_params.vrr_params.supported) {
+		bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled;
+		bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled;
+		bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state == VRR_STATE_ACTIVE_VARIABLE;
+
+		mod_freesync_handle_v_update(adev->dm.freesync_module,
+					     acrtc->dm_irq_params.stream,
+					     &acrtc->dm_irq_params.vrr_params);
+
+		/* update vmin_vmax only if freesync is enabled, or only if PSR and REPLAY are disabled */
+		if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) {
+			schedule_dc_vmin_vmax(adev, acrtc->dm_irq_params.stream,
+					&acrtc->dm_irq_params.vrr_params.adjust);
+		}
+	}
+
+	/*
+	 * Deliver pageflip completion events (DCN only).
+	 *
+	 * Since GRPH_PFLIP is not used, VUPDATE_NO_LOCK is the flip latch
+	 * point. Deliver any pending pageflip completion event from here,
+	 * once HW has consumed the new address (the OTG no longer reports a
+	 * pending flip).
+	 *
+	 * Also handle the case here where there aren't any active planes and
+	 * DCN HUBP may be clock-gated, so the flip-pending status may be
+	 * undefined.
+	 */
+	if (is_dcn && acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED &&
+	    acrtc->event) {
+
+		if (!dc_get_flip_pending_on_otg(adev->dm.dc, acrtc->otg_inst)) {
+			drm_crtc_send_vblank_event(&acrtc->base, acrtc->event);
+			acrtc->event = NULL;
+			drm_crtc_vblank_put(&acrtc->base);
+			acrtc->pflip_status = AMDGPU_FLIP_NONE;
+		}
+		/*
+		 * If the flip is still pending, leave it armed and
+		 * retry on the next vupdate.
+		 */
+	} else if (is_dcn && acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED &&
+		   acrtc->dm_irq_params.active_planes == 0) {
+
+		if (acrtc->event) {
+			drm_crtc_send_vblank_event(&acrtc->base, acrtc->event);
+			acrtc->event = NULL;
+			drm_crtc_vblank_put(&acrtc->base);
+		}
+		acrtc->pflip_status = AMDGPU_FLIP_NONE;
+	}
+
+	spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
+}
+
+STATIC_IFN_KUNIT void dm_vupdate_high_irq(void *interrupt_params)
+{
+	struct common_irq_params *irq_params = interrupt_params;
+	struct amdgpu_device *adev = irq_params->adev;
+	struct amdgpu_crtc *acrtc;
+	struct drm_device *drm_dev;
+	struct drm_vblank_crtc *vblank;
+	ktime_t frame_duration_ns, previous_timestamp;
+	unsigned long flags;
+	int vrr_active;
+
+	acrtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VUPDATE);
+	if (!acrtc)
+		return;
+
+	vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc);
+	drm_dev = acrtc->base.dev;
+	vblank = drm_crtc_vblank_crtc(&acrtc->base);
+	previous_timestamp = atomic64_read(&irq_params->previous_timestamp);
+	frame_duration_ns = vblank->time - previous_timestamp;
+
+	if (frame_duration_ns > 0) {
+		trace_amdgpu_refresh_rate_track(acrtc->base.index,
+					frame_duration_ns,
+					ktime_divns(NSEC_PER_SEC, frame_duration_ns));
+		atomic64_set(&irq_params->previous_timestamp, vblank->time);
+	}
+
+	drm_dbg_vbl(drm_dev,
+		    "crtc:%d, vupdate-vrr:%d\n", acrtc->crtc_id,
+		    vrr_active);
+
+	/*
+	 * On DCN, VUPDATE_NO_LOCK is the single OTG interrupt used to deliver
+	 * vblank and pageflip completion events; VSTARTUP and GRPH_PFLIP are
+	 * not used. Run the full handler here.
+	 */
+	if (amdgpu_ip_version(adev, DCE_HWIP, 0) != 0) {
+		dm_crtc_high_irq_handler(adev, acrtc);
+		return;
+	}
+
+	/* DCE only below. */
+
+	/* Core vblank handling is done here after end of front-porch in
+	 * vrr mode, as vblank timestamping will give valid results
+	 * while now done after front-porch. This will also deliver
+	 * page-flip completion events that have been queued to us
+	 * if a pageflip happened inside front-porch.
+	 */
+	if (vrr_active && acrtc->dm_irq_params.stream) {
+		bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled;
+		bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled;
+		bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state
+			== VRR_STATE_ACTIVE_VARIABLE;
+
+		amdgpu_dm_crtc_handle_vblank(acrtc);
+
+		/* BTR processing for pre-DCE12 ASICs */
+		if (adev->family < AMDGPU_FAMILY_AI) {
+			spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
+			mod_freesync_handle_v_update(
+				adev->dm.freesync_module,
+				acrtc->dm_irq_params.stream,
+				&acrtc->dm_irq_params.vrr_params);
+
+			if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) {
+				schedule_dc_vmin_vmax(adev,
+					acrtc->dm_irq_params.stream,
+					&acrtc->dm_irq_params.vrr_params.adjust);
+			}
+			spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
+		}
+	}
+}
+EXPORT_IF_KUNIT(dm_vupdate_high_irq);
+
+/**
+ * dm_crtc_high_irq() - Handles CRTC interrupt
+ * @interrupt_params: used for determining the CRTC instance
+ *
+ * Handles the CRTC/VSYNC interrupt by notifying DRM's VBLANK event handler.
+ *
+ * Used on DCE (VLINE0, set to vblank start). On DCN the equivalent handling is
+ * driven by VUPDATE_NO_LOCK in dm_vupdate_high_irq().
+ */
+STATIC_IFN_KUNIT void dm_crtc_high_irq(void *interrupt_params)
+{
+	struct common_irq_params *irq_params = interrupt_params;
+	struct amdgpu_device *adev = irq_params->adev;
+	struct amdgpu_crtc *acrtc;
+
+	acrtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VBLANK);
+	if (!acrtc)
+		return;
+
+	dm_crtc_high_irq_handler(adev, acrtc);
+}
+EXPORT_IF_KUNIT(dm_crtc_high_irq);
+
+#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
+/**
+ * dm_dcn_vertical_interrupt0_high_irq() - Handles OTG Vertical interrupt0 for
+ * DCN generation ASICs
+ * @interrupt_params: interrupt parameters
+ *
+ * Used to set crc window/read out crc value at vertical line 0 position
+ */
+static void dm_dcn_vertical_interrupt0_high_irq(void *interrupt_params)
+{
+	struct common_irq_params *irq_params = interrupt_params;
+	struct amdgpu_device *adev = irq_params->adev;
+	struct amdgpu_crtc *acrtc;
+
+	acrtc = amdgpu_dm_get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VLINE0);
+
+	if (!acrtc)
+		return;
+
+	amdgpu_dm_crtc_handle_crc_window_irq(&acrtc->base);
+}
+#endif /* CONFIG_DRM_AMD_SECURE_DISPLAY */
+
+STATIC_IFN_KUNIT void dm_handle_hpd_work(struct work_struct *work)
+{
+	struct dmub_hpd_work *dmub_hpd_wrk;
+
+	dmub_hpd_wrk = container_of(work, struct dmub_hpd_work, handle_hpd_work);
+
+	if (!dmub_hpd_wrk->dmub_notify) {
+		drm_err(adev_to_drm(dmub_hpd_wrk->adev), "dmub_hpd_wrk dmub_notify is NULL");
+		return;
+	}
+
+	if (dmub_hpd_wrk->dmub_notify->type < ARRAY_SIZE(dmub_hpd_wrk->adev->dm.dmub_callback)) {
+		dmub_hpd_wrk->adev->dm.dmub_callback[dmub_hpd_wrk->dmub_notify->type](dmub_hpd_wrk->adev,
+		dmub_hpd_wrk->dmub_notify);
+	}
+
+	kfree(dmub_hpd_wrk->dmub_notify);
+	kfree(dmub_hpd_wrk);
+
+}
+EXPORT_IF_KUNIT(dm_handle_hpd_work);
+
+STATIC_IFN_KUNIT const char *dmub_notification_type_str(enum dmub_notification_type e)
+{
+	switch (e) {
+	case DMUB_NOTIFICATION_NO_DATA:
+		return "NO_DATA";
+	case DMUB_NOTIFICATION_AUX_REPLY:
+		return "AUX_REPLY";
+	case DMUB_NOTIFICATION_HPD:
+		return "HPD";
+	case DMUB_NOTIFICATION_HPD_IRQ:
+		return "HPD_IRQ";
+	case DMUB_NOTIFICATION_SET_CONFIG_REPLY:
+		return "SET_CONFIG_REPLY";
+	case DMUB_NOTIFICATION_DPIA_NOTIFICATION:
+		return "DPIA_NOTIFICATION";
+	case DMUB_NOTIFICATION_HPD_SENSE_NOTIFY:
+		return "HPD_SENSE_NOTIFY";
+	case DMUB_NOTIFICATION_FUSED_IO:
+		return "FUSED_IO";
+	default:
+		return "<unknown>";
+	}
+}
+EXPORT_IF_KUNIT(dmub_notification_type_str);
+
+#define DMUB_TRACE_MAX_READ 64
+/**
+ * dm_dmub_outbox1_low_irq() - Handles Outbox interrupt
+ * @interrupt_params: used for determining the Outbox instance
+ *
+ * Handles the Outbox Interrupt
+ * event handler.
+ */
+STATIC_IFN_KUNIT void dm_dmub_outbox1_low_irq(void *interrupt_params)
+{
+	struct dmub_notification notify = {0};
+	struct common_irq_params *irq_params = interrupt_params;
+	struct amdgpu_device *adev = irq_params->adev;
+	struct amdgpu_display_manager *dm = &adev->dm;
+	struct dmcub_trace_buf_entry entry = { 0 };
+	u32 count = 0;
+	struct dmub_hpd_work *dmub_hpd_wrk;
+
+	do {
+		if (dc_dmub_srv_get_dmub_outbox0_msg(dm->dc, &entry)) {
+			trace_amdgpu_dmub_trace_high_irq(entry.trace_code, entry.tick_count,
+							entry.param0, entry.param1);
+
+			drm_dbg_driver(adev_to_drm(adev), "trace_code:%u, tick_count:%u, param0:%u, param1:%u\n",
+				 entry.trace_code, entry.tick_count, entry.param0, entry.param1);
+		} else
+			break;
+
+		count++;
+
+	} while (count <= DMUB_TRACE_MAX_READ);
+
+	if (count > DMUB_TRACE_MAX_READ)
+		drm_dbg_driver(adev_to_drm(adev), "Warning : count > DMUB_TRACE_MAX_READ");
+
+	if (dc_enable_dmub_notifications(adev->dm.dc) &&
+		irq_params->irq_src == DC_IRQ_SOURCE_DMCUB_OUTBOX) {
+
+		do {
+			dc_stat_get_dmub_notification(adev->dm.dc, &notify);
+			if (notify.type >= ARRAY_SIZE(dm->dmub_thread_offload)) {
+				drm_err(adev_to_drm(adev), "DM: notify type %d invalid!", notify.type);
+				continue;
+			}
+			if (!dm->dmub_callback[notify.type]) {
+				drm_warn(adev_to_drm(adev), "DMUB notification skipped due to no handler: type=%s\n",
+					dmub_notification_type_str(notify.type));
+				continue;
+			}
+			if (dm->dmub_thread_offload[notify.type]) {
+				dmub_hpd_wrk = kzalloc_obj(*dmub_hpd_wrk,
+							   GFP_ATOMIC);
+				if (!dmub_hpd_wrk) {
+					drm_err(adev_to_drm(adev), "Failed to allocate dmub_hpd_wrk");
+					return;
+				}
+				dmub_hpd_wrk->dmub_notify = kmemdup(&notify, sizeof(struct dmub_notification),
+								    GFP_ATOMIC);
+				if (!dmub_hpd_wrk->dmub_notify) {
+					kfree(dmub_hpd_wrk);
+					drm_err(adev_to_drm(adev), "Failed to allocate dmub_hpd_wrk->dmub_notify");
+					return;
+				}
+				INIT_WORK(&dmub_hpd_wrk->handle_hpd_work, dm_handle_hpd_work);
+				dmub_hpd_wrk->adev = adev;
+				queue_work(adev->dm.delayed_hpd_wq, &dmub_hpd_wrk->handle_hpd_work);
+			} else {
+				dm->dmub_callback[notify.type](adev, &notify);
+			}
+		} while (notify.pending_notification);
+	}
+}
+EXPORT_IF_KUNIT(dm_dmub_outbox1_low_irq);
+
+/* Register IRQ sources and initialize IRQ callbacks */
+int amdgpu_dm_dce110_register_irq_handlers(struct amdgpu_device *adev)
+{
+	struct dc *dc = adev->dm.dc;
+	struct common_irq_params *c_irq_params;
+	struct dc_interrupt_params int_params = {0};
+	int r;
+	int i;
+	unsigned int src_id;
+	unsigned int client_id = AMDGPU_IRQ_CLIENTID_LEGACY;
+	/* Use different interrupts for VBLANK on DCE 6 vs. newer. */
+	const unsigned int vblank_d1 =
+		adev->dm.dc->ctx->dce_version >= DCE_VERSION_8_0
+		? VISLANDS30_IV_SRCID_D1_VERTICAL_INTERRUPT0 : 1;
+
+	if (adev->family >= AMDGPU_FAMILY_AI)
+		client_id = SOC15_IH_CLIENTID_DCE;
+
+	int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
+	int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
+
+	/*
+	 * Actions of amdgpu_irq_add_id():
+	 * 1. Register a set() function with base driver.
+	 *    Base driver will call set() function to enable/disable an
+	 *    interrupt in DC hardware.
+	 * 2. Register amdgpu_dm_irq_handler().
+	 *    Base driver will call amdgpu_dm_irq_handler() for ALL interrupts
+	 *    coming from DC hardware.
+	 *    amdgpu_dm_irq_handler() will re-direct the interrupt to DC
+	 *    for acknowledging and handling.
+	 */
+
+	/* Use VBLANK interrupt */
+	for (i = 0; i < adev->mode_info.num_crtc; i++) {
+		src_id = vblank_d1 + i;
+		r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->crtc_irq);
+		if (r) {
+			drm_err(adev_to_drm(adev), "Failed to add crtc irq id!\n");
+			return r;
+		}
+
+		int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
+		int_params.irq_source =
+			dc_interrupt_to_irq_source(dc, src_id, 0);
+
+		if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
+			int_params.irq_source  < DC_IRQ_SOURCE_VBLANK1 ||
+			int_params.irq_source  > DC_IRQ_SOURCE_VBLANK6) {
+			drm_err(adev_to_drm(adev), "Failed to register vblank irq!\n");
+			return -EINVAL;
+		}
+
+		c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1];
+
+		c_irq_params->adev = adev;
+		c_irq_params->irq_src = int_params.irq_source;
+
+		if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
+			dm_crtc_high_irq, c_irq_params))
+			return -ENOMEM;
+	}
+
+	if (dc_supports_vrr(adev->dm.dc->ctx->dce_version)) {
+		/* Use VUPDATE interrupt */
+		for (i = 0; i < adev->mode_info.num_crtc; i++) {
+			src_id = VISLANDS30_IV_SRCID_D1_V_UPDATE_INT + i * 2;
+			r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->vupdate_irq);
+			if (r) {
+				drm_err(adev_to_drm(adev), "Failed to add vupdate irq id!\n");
+				return r;
+			}
+
+			int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
+			int_params.irq_source =
+				dc_interrupt_to_irq_source(dc, src_id, 0);
+
+			if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
+				int_params.irq_source  < DC_IRQ_SOURCE_VUPDATE1 ||
+				int_params.irq_source  > DC_IRQ_SOURCE_VUPDATE6) {
+				drm_err(adev_to_drm(adev), "Failed to register vupdate irq!\n");
+				return -EINVAL;
+			}
+
+			c_irq_params = &adev->dm.vupdate_params[
+				int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1];
+			c_irq_params->adev = adev;
+			c_irq_params->irq_src = int_params.irq_source;
+
+			if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
+				dm_vupdate_high_irq, c_irq_params))
+				return -ENOMEM;
+		}
+	}
+
+	/* Use GRPH_PFLIP interrupt */
+	for (i = VISLANDS30_IV_SRCID_D1_GRPH_PFLIP;
+			i <= VISLANDS30_IV_SRCID_D6_GRPH_PFLIP; i += 2) {
+		r = amdgpu_irq_add_id(adev, client_id, i, &adev->pageflip_irq);
+		if (r) {
+			drm_err(adev_to_drm(adev), "Failed to add page flip irq id!\n");
+			return r;
+		}
+
+		int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
+		int_params.irq_source =
+			dc_interrupt_to_irq_source(dc, i, 0);
+
+		if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
+			int_params.irq_source  < DC_IRQ_SOURCE_PFLIP_FIRST ||
+			int_params.irq_source  > DC_IRQ_SOURCE_PFLIP_LAST) {
+			drm_err(adev_to_drm(adev), "Failed to register pflip irq!\n");
+			return -EINVAL;
+		}
+
+		c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST];
+
+		c_irq_params->adev = adev;
+		c_irq_params->irq_src = int_params.irq_source;
+
+		if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
+			dm_pflip_high_irq, c_irq_params))
+			return -ENOMEM;
+	}
+
+	/* HPD */
+	r = amdgpu_irq_add_id(adev, client_id,
+			VISLANDS30_IV_SRCID_HOTPLUG_DETECT_A, &adev->hpd_irq);
+	if (r) {
+		drm_err(adev_to_drm(adev), "Failed to add hpd irq id!\n");
+		return r;
+	}
+
+	r = amdgpu_dm_register_hpd_handlers(adev);
+
+	return r;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_dce110_register_irq_handlers);
+
+/* Register IRQ sources and initialize IRQ callbacks */
+int amdgpu_dm_dcn10_register_irq_handlers(struct amdgpu_device *adev)
+{
+	struct dc *dc = adev->dm.dc;
+	struct common_irq_params *c_irq_params;
+	struct dc_interrupt_params int_params = {0};
+	int r;
+	int i;
+#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
+	static const unsigned int vrtl_int_srcid[] = {
+		DCN_1_0__SRCID__OTG1_VERTICAL_INTERRUPT0_CONTROL,
+		DCN_1_0__SRCID__OTG2_VERTICAL_INTERRUPT0_CONTROL,
+		DCN_1_0__SRCID__OTG3_VERTICAL_INTERRUPT0_CONTROL,
+		DCN_1_0__SRCID__OTG4_VERTICAL_INTERRUPT0_CONTROL,
+		DCN_1_0__SRCID__OTG5_VERTICAL_INTERRUPT0_CONTROL,
+		DCN_1_0__SRCID__OTG6_VERTICAL_INTERRUPT0_CONTROL
+	};
+#endif
+
+	int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
+	int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
+
+	/*
+	 * Actions of amdgpu_irq_add_id():
+	 * 1. Register a set() function with base driver.
+	 *    Base driver will call set() function to enable/disable an
+	 *    interrupt in DC hardware.
+	 * 2. Register amdgpu_dm_irq_handler().
+	 *    Base driver will call amdgpu_dm_irq_handler() for ALL interrupts
+	 *    coming from DC hardware.
+	 *    amdgpu_dm_irq_handler() will re-direct the interrupt to DC
+	 *    for acknowledging and handling.
+	 */
+
+	/* Use otg vertical line interrupt */
+#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
+	for (i = 0; i <= adev->mode_info.num_crtc - 1; i++) {
+		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE,
+				vrtl_int_srcid[i], &adev->vline0_irq);
+
+		if (r) {
+			drm_err(adev_to_drm(adev), "Failed to add vline0 irq id!\n");
+			return r;
+		}
+
+		int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
+		int_params.irq_source =
+			dc_interrupt_to_irq_source(dc, vrtl_int_srcid[i], 0);
+
+		if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
+			int_params.irq_source < DC_IRQ_SOURCE_DC1_VLINE0 ||
+			int_params.irq_source > DC_IRQ_SOURCE_DC6_VLINE0) {
+			drm_err(adev_to_drm(adev), "Failed to register vline0 irq!\n");
+			return -EINVAL;
+		}
+
+		c_irq_params = &adev->dm.vline0_params[int_params.irq_source
+					- DC_IRQ_SOURCE_DC1_VLINE0];
+
+		c_irq_params->adev = adev;
+		c_irq_params->irq_src = int_params.irq_source;
+
+		if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
+			dm_dcn_vertical_interrupt0_high_irq,
+			c_irq_params))
+			return -ENOMEM;
+	}
+#endif
+
+	/* Use VUPDATE_NO_LOCK interrupt on DCN, which seems to correspond to
+	 * the regular VUPDATE interrupt on DCE. We want DC_IRQ_SOURCE_VUPDATEx
+	 * to trigger at end of each vblank, regardless of state of the lock,
+	 * matching DCE behaviour.
+	 */
+	for (i = DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT;
+	     i <= DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT + adev->mode_info.num_crtc - 1;
+	     i++) {
+		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->vupdate_irq);
+
+		if (r) {
+			drm_err(adev_to_drm(adev), "Failed to add vupdate irq id!\n");
+			return r;
+		}
+
+		int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
+		int_params.irq_source =
+			dc_interrupt_to_irq_source(dc, i, 0);
+
+		if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
+			int_params.irq_source  < DC_IRQ_SOURCE_VUPDATE1 ||
+			int_params.irq_source  > DC_IRQ_SOURCE_VUPDATE6) {
+			drm_err(adev_to_drm(adev), "Failed to register vupdate irq!\n");
+			return -EINVAL;
+		}
+
+		c_irq_params = &adev->dm.vupdate_params[int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1];
+
+		c_irq_params->adev = adev;
+		c_irq_params->irq_src = int_params.irq_source;
+
+		if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
+			dm_vupdate_high_irq, c_irq_params))
+			return -ENOMEM;
+	}
+
+	/* HPD */
+	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DC_HPD1_INT,
+			&adev->hpd_irq);
+	if (r) {
+		drm_err(adev_to_drm(adev), "Failed to add hpd irq id!\n");
+		return r;
+	}
+
+	r = amdgpu_dm_register_hpd_handlers(adev);
+
+	return r;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_dcn10_register_irq_handlers);
+
+/* Register Outbox IRQ sources and initialize IRQ callbacks */
+int amdgpu_dm_register_outbox_irq_handlers(struct amdgpu_device *adev)
+{
+	struct dc *dc = adev->dm.dc;
+	struct common_irq_params *c_irq_params;
+	struct dc_interrupt_params int_params = {0};
+	int r, i;
+
+	int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
+	int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
+
+	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT,
+			&adev->dmub_outbox_irq);
+	if (r) {
+		drm_err(adev_to_drm(adev), "Failed to add outbox irq id!\n");
+		return r;
+	}
+
+	if (dc->ctx->dmub_srv) {
+		i = DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT;
+		int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+		int_params.irq_source =
+		dc_interrupt_to_irq_source(dc, i, 0);
+
+		c_irq_params = &adev->dm.dmub_outbox_params[0];
+
+		c_irq_params->adev = adev;
+		c_irq_params->irq_src = int_params.irq_source;
+
+		if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
+			dm_dmub_outbox1_low_irq, c_irq_params))
+			return -ENOMEM;
+	}
+
+	return 0;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_register_outbox_irq_handlers);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.h
index 4f6b58f4f90d7..4c200a9614a77 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.h
@@ -27,6 +27,14 @@
 
 #include "irq_types.h" /* DAL irq definitions */
 
+struct amdgpu_device;
+struct amdgpu_crtc;
+struct amdgpu_display_manager;
+struct dc_sink;
+struct hpd_rx_irq_offload_work_queue;
+struct work_struct;
+enum dmub_notification_type;
+
 /*
  * Display Manager IRQ-related interfaces (for use by DAL).
  */
@@ -101,4 +109,80 @@ void amdgpu_dm_irq_suspend(struct amdgpu_device *adev);
 void amdgpu_dm_irq_resume_early(struct amdgpu_device *adev);
 void amdgpu_dm_irq_resume_late(struct amdgpu_device *adev);
 
+/* HPD handling */
+struct hpd_rx_irq_offload_work_queue *amdgpu_dm_hpd_rx_irq_create_workqueue(struct amdgpu_device *adev);
+void amdgpu_dm_hpd_rx_irq_work_suspend(struct amdgpu_display_manager *dm);
+int amdgpu_dm_register_hpd_handlers(struct amdgpu_device *adev);
+void amdgpu_dm_hdmi_hpd_debounce_work(struct work_struct *work);
+
+/* IRQ handlers */
+struct amdgpu_crtc *amdgpu_dm_get_crtc_by_otg_inst(struct amdgpu_device *adev,
+						    int otg_inst);
+int amdgpu_dm_dce110_register_irq_handlers(struct amdgpu_device *adev);
+int amdgpu_dm_dcn10_register_irq_handlers(struct amdgpu_device *adev);
+int amdgpu_dm_register_outbox_irq_handlers(struct amdgpu_device *adev);
+
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+struct amdgpu_irq_src;
+struct amdgpu_iv_entry;
+enum amdgpu_interrupt_state;
+
+enum dc_irq_source amdgpu_dm_hpd_to_dal_irq_source(unsigned int type);
+bool are_sinks_equal(const struct dc_sink *sink1, const struct dc_sink *sink2);
+const char *dmub_notification_type_str(enum dmub_notification_type e);
+int amdgpu_dm_set_hpd_irq_state(struct amdgpu_device *adev,
+				struct amdgpu_irq_src *source,
+				unsigned int type,
+				enum amdgpu_interrupt_state state);
+int amdgpu_dm_set_dmub_outbox_irq_state(struct amdgpu_device *adev,
+					struct amdgpu_irq_src *source,
+					unsigned int crtc_id,
+					enum amdgpu_interrupt_state state);
+int amdgpu_dm_set_dmub_trace_irq_state(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *source,
+				       unsigned int type,
+				       enum amdgpu_interrupt_state state);
+int amdgpu_dm_set_pflip_irq_state(struct amdgpu_device *adev,
+				  struct amdgpu_irq_src *source,
+				  unsigned int crtc_id,
+				  enum amdgpu_interrupt_state state);
+int amdgpu_dm_set_crtc_irq_state(struct amdgpu_device *adev,
+				 struct amdgpu_irq_src *source,
+				 unsigned int crtc_id,
+				 enum amdgpu_interrupt_state state);
+int amdgpu_dm_set_vline0_irq_state(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   unsigned int crtc_id,
+				   enum amdgpu_interrupt_state state);
+int amdgpu_dm_set_vupdate_irq_state(struct amdgpu_device *adev,
+				    struct amdgpu_irq_src *source,
+				    unsigned int crtc_id,
+				    enum amdgpu_interrupt_state state);
+void amdgpu_dm_irq_schedule_work(struct amdgpu_device *adev,
+				 enum dc_irq_source irq_source);
+void amdgpu_dm_irq_immediate_work(struct amdgpu_device *adev,
+				  enum dc_irq_source irq_source);
+void dm_handle_hpd_rx_offload_work(struct work_struct *work);
+void handle_hpd_irq_helper(struct amdgpu_dm_connector *aconnector,
+			   enum dc_detect_reason reason);
+void handle_hpd_irq(void *param);
+void schedule_hpd_rx_offload_work(struct amdgpu_device *adev,
+				  struct hpd_rx_irq_offload_work_queue *offload_wq,
+				  union hpd_irq_data hpd_irq_data);
+void handle_hpd_rx_irq(void *param);
+void dmub_hpd_callback(struct amdgpu_device *adev,
+		       struct dmub_notification *notify);
+void dmub_hpd_sense_callback(struct amdgpu_device *adev,
+			     struct dmub_notification *notify);
+void dm_pflip_high_irq(void *interrupt_params);
+void dm_vupdate_high_irq(void *interrupt_params);
+void dm_crtc_high_irq(void *interrupt_params);
+void dm_handle_hpd_work(struct work_struct *work);
+void dm_dmub_outbox1_low_irq(void *interrupt_params);
+int amdgpu_dm_irq_handler(struct amdgpu_device *adev,
+			  struct amdgpu_irq_src *source,
+			  struct amdgpu_iv_entry *entry);
+void dm_handle_vmin_vmax_update(struct work_struct *offload_work);
+#endif
+
 #endif /* __AMDGPU_DM_IRQ_H__ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c
index 32391b56097ea..a9575bf25fc21 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c
@@ -502,6 +502,7 @@ void amdgpu_dm_ism_commit_event(struct amdgpu_dm_ism *ism,
 
 	} while (next_event < DM_ISM_NUM_EVENTS);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_ism_commit_event);
 
 
 static void dm_ism_delayed_work_func(struct work_struct *work)
@@ -568,6 +569,7 @@ void amdgpu_dm_ism_disable(struct amdgpu_display_manager *dm)
 		disable_delayed_work_sync(&ism->sso_delayed_work);
 	}
 }
+EXPORT_IF_KUNIT(amdgpu_dm_ism_disable);
 
 /**
  * amdgpu_dm_ism_force_full_power - Force every CRTC's ISM FSM to FULL_POWER
@@ -603,6 +605,7 @@ void amdgpu_dm_ism_force_full_power(struct amdgpu_display_manager *dm)
 					   DM_ISM_EVENT_EXIT_IDLE_REQUESTED);
 	}
 }
+EXPORT_IF_KUNIT(amdgpu_dm_ism_force_full_power);
 
 /**
  * amdgpu_dm_ism_enable - enable the ISM
@@ -626,6 +629,7 @@ void amdgpu_dm_ism_enable(struct amdgpu_display_manager *dm)
 		enable_delayed_work(&ism->sso_delayed_work);
 	}
 }
+EXPORT_IF_KUNIT(amdgpu_dm_ism_enable);
 
 void amdgpu_dm_ism_init(struct amdgpu_dm_ism *ism,
 			struct amdgpu_dm_ism_config *config)
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_kunit_helpers.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_kunit_helpers.h
index 4b2864375105b..1f910a6a00c0f 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_kunit_helpers.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_kunit_helpers.h
@@ -10,6 +10,7 @@
 #define STATIC_IFN_KUNIT
 #define INLINE_IFN_KUNIT inline
 #define EXPORT_IF_KUNIT(symbol) EXPORT_SYMBOL(symbol)
+
 #else
 #define STATIC_IFN_KUNIT static
 #define INLINE_IFN_KUNIT
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
index b3af7445b4571..a45d8d79f3b2d 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
@@ -34,16 +34,17 @@
 #include "dm_services.h"
 #include "amdgpu.h"
 #include "amdgpu_dm.h"
+#include "dmub_cmd.h"
 #include "amdgpu_dm_mst_types.h"
 #include "amdgpu_dm_hdcp.h"
 
 #include "dc.h"
 #include "dm_helpers.h"
+#include "amdgpu_dm_kunit_helpers.h"
 
 #include "ddc_service_types.h"
 #include "dpcd_defs.h"
 
-#include "dmub_cmd.h"
 #if defined(CONFIG_DEBUG_FS)
 #include "amdgpu_dm_debugfs.h"
 #endif
@@ -52,11 +53,54 @@
 
 #define PEAK_FACTOR_X1000 1006
 
+/*
+ * Translate a failed AUX transaction's operation result into an errno-style
+ * return value. @result is returned unchanged for AUX_RET_SUCCESS.
+ */
+STATIC_IFN_KUNIT ssize_t dm_dp_aux_transfer_result(ssize_t result,
+						   enum aux_return_code_type operation_result)
+{
+	switch (operation_result) {
+	case AUX_RET_SUCCESS:
+		break;
+	case AUX_RET_ERROR_HPD_DISCON:
+	case AUX_RET_ERROR_UNKNOWN:
+	case AUX_RET_ERROR_INVALID_OPERATION:
+	case AUX_RET_ERROR_PROTOCOL_ERROR:
+		result = -EIO;
+		break;
+	case AUX_RET_ERROR_INVALID_REPLY:
+	case AUX_RET_ERROR_ENGINE_ACQUIRE:
+		result = -EBUSY;
+		break;
+	case AUX_RET_ERROR_TIMEOUT:
+		result = -ETIMEDOUT;
+		break;
+	}
+
+	return result;
+}
+EXPORT_IF_KUNIT(dm_dp_aux_transfer_result);
+
+/*
+ * Derive the AUX payload transaction flags from a DP AUX request field.
+ */
+STATIC_IFN_KUNIT void dm_dp_aux_fill_payload_flags(u8 request,
+						   struct aux_payload *payload)
+{
+	payload->i2c_over_aux = (request & DP_AUX_NATIVE_WRITE) == 0;
+	payload->write = (request & DP_AUX_I2C_READ) == 0;
+	payload->mot = (request & DP_AUX_I2C_MOT) != 0;
+	payload->write_status_update =
+			(request & DP_AUX_I2C_WRITE_STATUS_UPDATE) != 0;
+}
+EXPORT_IF_KUNIT(dm_dp_aux_fill_payload_flags);
+
 /*
  * This function handles both native AUX and I2C-Over-AUX transactions.
  */
-static ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux,
-				  struct drm_dp_aux_msg *msg)
+STATIC_IFN_KUNIT ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux,
+					    struct drm_dp_aux_msg *msg)
 {
 	ssize_t result = 0;
 	struct aux_payload payload;
@@ -72,11 +116,7 @@ static ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux,
 	payload.data = msg->buffer;
 	payload.length = msg->size;
 	payload.reply = &msg->reply;
-	payload.i2c_over_aux = (msg->request & DP_AUX_NATIVE_WRITE) == 0;
-	payload.write = (msg->request & DP_AUX_I2C_READ) == 0;
-	payload.mot = (msg->request & DP_AUX_I2C_MOT) != 0;
-	payload.write_status_update =
-			(msg->request & DP_AUX_I2C_WRITE_STATUS_UPDATE) != 0;
+	dm_dp_aux_fill_payload_flags(msg->request, &payload);
 	payload.defer_delay = 0;
 
 	if (payload.write) {
@@ -116,23 +156,7 @@ static ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux,
 	}
 
 	if (result < 0) {
-		switch (operation_result) {
-		case AUX_RET_SUCCESS:
-			break;
-		case AUX_RET_ERROR_HPD_DISCON:
-		case AUX_RET_ERROR_UNKNOWN:
-		case AUX_RET_ERROR_INVALID_OPERATION:
-		case AUX_RET_ERROR_PROTOCOL_ERROR:
-			result = -EIO;
-			break;
-		case AUX_RET_ERROR_INVALID_REPLY:
-		case AUX_RET_ERROR_ENGINE_ACQUIRE:
-			result = -EBUSY;
-			break;
-		case AUX_RET_ERROR_TIMEOUT:
-			result = -ETIMEDOUT;
-			break;
-		}
+		result = dm_dp_aux_transfer_result(result, operation_result);
 
 		drm_dbg_dp(adev_to_drm(adev), "DP AUX transfer fail:%d\n", operation_result);
 	}
@@ -143,6 +167,7 @@ static ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux,
 
 	return result;
 }
+EXPORT_IF_KUNIT(dm_dp_aux_transfer);
 
 static void
 dm_dp_mst_connector_destroy(struct drm_connector *connector)
@@ -183,7 +208,7 @@ amdgpu_dm_mst_connector_late_register(struct drm_connector *connector)
 }
 
 
-static inline void
+STATIC_IFN_KUNIT void
 amdgpu_dm_mst_reset_mst_connector_setting(struct amdgpu_dm_connector *aconnector)
 {
 	aconnector->drm_edid = NULL;
@@ -192,6 +217,7 @@ amdgpu_dm_mst_reset_mst_connector_setting(struct amdgpu_dm_connector *aconnector
 	aconnector->mst_local_bw = 0;
 	aconnector->vc_full_pbn = 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_mst_reset_mst_connector_setting);
 
 static void
 amdgpu_dm_mst_connector_early_unregister(struct drm_connector *connector)
@@ -248,6 +274,7 @@ bool needs_dsc_aux_workaround(struct dc_link *link)
 
 	return false;
 }
+EXPORT_IF_KUNIT(needs_dsc_aux_workaround);
 
 #if defined(CONFIG_DRM_AMD_DC_FP)
 static bool is_synaptics_cascaded_panamera(struct dc_link *link, struct drm_dp_mst_port *port)
@@ -311,7 +338,7 @@ static bool validate_dsc_caps_on_connector(struct amdgpu_dm_connector *aconnecto
 }
 #endif
 
-static bool retrieve_downstream_port_device(struct amdgpu_dm_connector *aconnector)
+STATIC_IFN_KUNIT bool retrieve_downstream_port_device(struct amdgpu_dm_connector *aconnector)
 {
 	union dp_downstream_port_present ds_port_present;
 
@@ -329,8 +356,9 @@ static bool retrieve_downstream_port_device(struct amdgpu_dm_connector *aconnect
 
 	return true;
 }
+EXPORT_IF_KUNIT(retrieve_downstream_port_device);
 
-static bool retrieve_branch_specific_data(struct amdgpu_dm_connector *aconnector)
+STATIC_IFN_KUNIT bool retrieve_branch_specific_data(struct amdgpu_dm_connector *aconnector)
 {
 	struct drm_connector *connector = &aconnector->base;
 	struct drm_dp_mst_port *port = aconnector->mst_output_port;
@@ -357,15 +385,13 @@ static bool retrieve_branch_specific_data(struct amdgpu_dm_connector *aconnector
 
 	return true;
 }
+EXPORT_IF_KUNIT(retrieve_branch_specific_data);
 
 static int dm_dp_mst_get_modes(struct drm_connector *connector)
 {
 	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
 	int ret = 0;
 
-	if (!aconnector)
-		return drm_add_edid_modes(connector, NULL);
-
 	if (!aconnector->drm_edid) {
 		const struct drm_edid *drm_edid;
 
@@ -456,7 +482,7 @@ static int dm_dp_mst_get_modes(struct drm_connector *connector)
 		 * plugged back with same display index, its hdcp properties
 		 * will be retrieved from hdcp_work within dm_dp_mst_get_modes
 		 */
-		if (aconnector->dc_sink && connector->state) {
+		if (connector->state) {
 			struct drm_device *dev = connector->dev;
 			struct amdgpu_device *adev = drm_to_adev(dev);
 
@@ -472,20 +498,18 @@ static int dm_dp_mst_get_modes(struct drm_connector *connector)
 			}
 		}
 
-		if (aconnector->dc_sink) {
-			amdgpu_dm_update_freesync_caps(
-					connector, aconnector->drm_edid, true);
+		amdgpu_dm_update_freesync_caps(
+				connector, aconnector->drm_edid, true);
 
 #if defined(CONFIG_DRM_AMD_DC_FP)
-			if (!validate_dsc_caps_on_connector(aconnector))
-				memset(&aconnector->dc_sink->dsc_caps,
-				       0, sizeof(aconnector->dc_sink->dsc_caps));
+		if (!validate_dsc_caps_on_connector(aconnector))
+			memset(&aconnector->dc_sink->dsc_caps,
+			       0, sizeof(aconnector->dc_sink->dsc_caps));
 #endif
 
-			if (!retrieve_downstream_port_device(aconnector))
-				memset(&aconnector->mst_downstream_port_present,
-					0, sizeof(aconnector->mst_downstream_port_present));
-		}
+		if (!retrieve_downstream_port_device(aconnector))
+			memset(&aconnector->mst_downstream_port_present,
+				0, sizeof(aconnector->mst_downstream_port_present));
 	}
 
 	drm_edid_connector_update(&aconnector->base, aconnector->drm_edid);
@@ -495,7 +519,7 @@ static int dm_dp_mst_get_modes(struct drm_connector *connector)
 	return ret;
 }
 
-static struct drm_encoder *
+STATIC_IFN_KUNIT struct drm_encoder *
 dm_mst_atomic_best_encoder(struct drm_connector *connector,
 			   struct drm_atomic_commit *state)
 {
@@ -506,8 +530,9 @@ dm_mst_atomic_best_encoder(struct drm_connector *connector,
 
 	return &adev->dm.mst_encoders[acrtc->crtc_id].base;
 }
+EXPORT_IF_KUNIT(dm_mst_atomic_best_encoder);
 
-static int
+STATIC_IFN_KUNIT int
 dm_dp_mst_detect(struct drm_connector *connector,
 		 struct drm_modeset_acquire_ctx *ctx, bool force)
 {
@@ -577,9 +602,10 @@ dm_dp_mst_detect(struct drm_connector *connector,
 
 	return connection_status;
 }
+EXPORT_IF_KUNIT(dm_dp_mst_detect);
 
-static int dm_dp_mst_atomic_check(struct drm_connector *connector,
-				  struct drm_atomic_commit *state)
+STATIC_IFN_KUNIT int dm_dp_mst_atomic_check(struct drm_connector *connector,
+					    struct drm_atomic_commit *state)
 {
 	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
 	struct drm_dp_mst_topology_mgr *mst_mgr = &aconnector->mst_root->mst_mgr;
@@ -587,6 +613,7 @@ static int dm_dp_mst_atomic_check(struct drm_connector *connector,
 
 	return drm_dp_atomic_release_time_slots(state, mst_mgr, mst_port);
 }
+EXPORT_IF_KUNIT(dm_dp_mst_atomic_check);
 
 static const struct drm_connector_helper_funcs dm_dp_mst_connector_helper_funcs = {
 	.get_modes = dm_dp_mst_get_modes,
@@ -627,6 +654,7 @@ dm_dp_create_fake_mst_encoders(struct amdgpu_device *adev)
 		drm_encoder_helper_add(encoder, &amdgpu_dm_encoder_helper_funcs);
 	}
 }
+EXPORT_IF_KUNIT(dm_dp_create_fake_mst_encoders);
 
 static struct drm_connector *
 dm_dp_add_mst_connector(struct drm_dp_mst_topology_mgr *mgr,
@@ -711,6 +739,44 @@ dm_dp_add_mst_connector(struct drm_dp_mst_topology_mgr *mgr,
 	return connector;
 }
 
+/*
+ * Select the ESI[1] mask used to filter the MST sideband ready bits for a
+ * given message-ready event type.
+ */
+STATIC_IFN_KUNIT u8 dm_mst_msg_ready_mask(enum mst_msg_ready_type msg_rdy_type)
+{
+	switch (msg_rdy_type) {
+	case DOWN_REP_MSG_RDY_EVENT:
+		/* Only handle DOWN_REP_MSG_RDY case*/
+		return DP_DOWN_REP_MSG_RDY;
+	case UP_REQ_MSG_RDY_EVENT:
+		/* Only handle UP_REQ_MSG_RDY case*/
+		return DP_UP_REQ_MSG_RDY;
+	default:
+		/* Handle both cases*/
+		return DP_DOWN_REP_MSG_RDY | DP_UP_REQ_MSG_RDY;
+	}
+}
+EXPORT_IF_KUNIT(dm_mst_msg_ready_mask);
+
+/*
+ * Select the DPCD ESI address and read length based on the DPCD revision.
+ */
+STATIC_IFN_KUNIT void dm_mst_select_esi_dpcd(u8 dpcd_rev, int *dpcd_addr,
+					     u8 *dpcd_bytes_to_read)
+{
+	if (dpcd_rev < 0x12) {
+		*dpcd_bytes_to_read = DP_LANE0_1_STATUS - DP_SINK_COUNT;
+		/* DPCD 0x200 - 0x201 for downstream IRQ */
+		*dpcd_addr = DP_SINK_COUNT;
+	} else {
+		*dpcd_bytes_to_read = DP_PSR_ERROR_STATUS - DP_SINK_COUNT_ESI;
+		/* DPCD 0x2002 - 0x2005 for downstream IRQ */
+		*dpcd_addr = DP_SINK_COUNT_ESI;
+	}
+}
+EXPORT_IF_KUNIT(dm_mst_select_esi_dpcd);
+
 void dm_handle_mst_sideband_msg_ready_event(
 	struct drm_dp_mst_topology_mgr *mgr,
 	enum mst_msg_ready_type msg_rdy_type)
@@ -729,15 +795,8 @@ void dm_handle_mst_sideband_msg_ready_event(
 
 	const struct dc_link_status *link_status = dc_link_get_status(aconnector->dc_link);
 
-	if (link_status->dpcd_caps->dpcd_rev.raw < 0x12) {
-		dpcd_bytes_to_read = DP_LANE0_1_STATUS - DP_SINK_COUNT;
-		/* DPCD 0x200 - 0x201 for downstream IRQ */
-		dpcd_addr = DP_SINK_COUNT;
-	} else {
-		dpcd_bytes_to_read = DP_PSR_ERROR_STATUS - DP_SINK_COUNT_ESI;
-		/* DPCD 0x2002 - 0x2005 for downstream IRQ */
-		dpcd_addr = DP_SINK_COUNT_ESI;
-	}
+	dm_mst_select_esi_dpcd(link_status->dpcd_caps->dpcd_rev.raw, &dpcd_addr,
+			       &dpcd_bytes_to_read);
 
 	mutex_lock(&aconnector->handle_mst_msg_ready);
 
@@ -759,20 +818,7 @@ void dm_handle_mst_sideband_msg_ready_event(
 
 		DRM_DEBUG_DRIVER("ESI %02x %02x %02x\n", esi[0], esi[1], esi[2]);
 
-		switch (msg_rdy_type) {
-		case DOWN_REP_MSG_RDY_EVENT:
-			/* Only handle DOWN_REP_MSG_RDY case*/
-			esi[1] &= DP_DOWN_REP_MSG_RDY;
-			break;
-		case UP_REQ_MSG_RDY_EVENT:
-			/* Only handle UP_REQ_MSG_RDY case*/
-			esi[1] &= DP_UP_REQ_MSG_RDY;
-			break;
-		default:
-			/* Handle both cases*/
-			esi[1] &= (DP_DOWN_REP_MSG_RDY | DP_UP_REQ_MSG_RDY);
-			break;
-		}
+		esi[1] &= dm_mst_msg_ready_mask(msg_rdy_type);
 
 		if (!esi[1])
 			break;
@@ -814,11 +860,13 @@ void dm_handle_mst_sideband_msg_ready_event(
 	if (process_count == max_process_count)
 		DRM_DEBUG_DRIVER("Loop exceeded max iterations\n");
 }
+EXPORT_IF_KUNIT(dm_handle_mst_sideband_msg_ready_event);
 
-static void dm_handle_mst_down_rep_msg_ready(struct drm_dp_mst_topology_mgr *mgr)
+STATIC_IFN_KUNIT void dm_handle_mst_down_rep_msg_ready(struct drm_dp_mst_topology_mgr *mgr)
 {
 	dm_handle_mst_sideband_msg_ready_event(mgr, DOWN_REP_MSG_RDY_EVENT);
 }
+EXPORT_IF_KUNIT(dm_handle_mst_down_rep_msg_ready);
 
 static const struct drm_dp_mst_topology_cbs dm_mst_cbs = {
 	.add_connector = dm_dp_add_mst_connector,
@@ -853,6 +901,7 @@ void amdgpu_dm_initialize_dp_connector(struct amdgpu_display_manager *dm,
 
 	drm_connector_attach_dp_subconnector_property(&aconnector->base);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_initialize_dp_connector);
 
 uint32_t dm_mst_get_pbn_divider(struct dc_link *link)
 {
@@ -869,6 +918,7 @@ uint32_t dm_mst_get_pbn_divider(struct dc_link *link)
 
 	return dfixed_const(pbn_div_x100) / 100;
 }
+EXPORT_IF_KUNIT(dm_mst_get_pbn_divider);
 
 struct dsc_mst_fairness_params {
 	struct dc_crtc_timing *timing;
@@ -1787,7 +1837,7 @@ int pre_validate_dsc(struct drm_atomic_commit *state,
 			dm_old_crtc_state = to_dm_crtc_state(state->crtcs[ind].old_state);
 
 			local_dc_state->streams[i] =
-				create_validate_stream_for_sink(connector,
+				amdgpu_dm_create_validate_stream_for_sink(connector,
 								&state->crtcs[ind].new_state->mode,
 								dm_new_conn_state,
 								dm_old_crtc_state->stream);
@@ -2066,3 +2116,4 @@ enum dc_status dm_dp_mst_is_port_support_mode(
 #endif
 	return DC_OK;
 }
+EXPORT_IF_KUNIT(dm_dp_mst_is_port_support_mode);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.h
index 0e8eef5bdb743..2ae9d4b88888f 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.h
@@ -57,8 +57,15 @@ enum mst_msg_ready_type {
 	DOWN_OR_UP_MSG_RDY_EVENT = 3
 };
 
+struct amdgpu_device;
 struct amdgpu_display_manager;
 struct amdgpu_dm_connector;
+struct aux_payload;
+struct dc_state;
+struct dc_stream_state;
+struct dm_atomic_state;
+struct drm_atomic_commit;
+struct drm_dp_mst_topology_mgr;
 
 uint32_t dm_mst_get_pbn_divider(struct dc_link *link);
 
@@ -94,4 +101,23 @@ enum dc_status dm_dp_mst_is_port_support_mode(
 	struct amdgpu_dm_connector *aconnector,
 	struct dc_stream_state *stream);
 
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+void amdgpu_dm_mst_reset_mst_connector_setting(struct amdgpu_dm_connector *aconnector);
+bool retrieve_downstream_port_device(struct amdgpu_dm_connector *aconnector);
+bool retrieve_branch_specific_data(struct amdgpu_dm_connector *aconnector);
+ssize_t dm_dp_aux_transfer_result(ssize_t result,
+				  enum aux_return_code_type operation_result);
+void dm_dp_aux_fill_payload_flags(u8 request, struct aux_payload *payload);
+ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg);
+u8 dm_mst_msg_ready_mask(enum mst_msg_ready_type msg_rdy_type);
+void dm_mst_select_esi_dpcd(u8 dpcd_rev, int *dpcd_addr, u8 *dpcd_bytes_to_read);
+void dm_handle_mst_down_rep_msg_ready(struct drm_dp_mst_topology_mgr *mgr);
+struct drm_encoder *dm_mst_atomic_best_encoder(struct drm_connector *connector,
+						 struct drm_atomic_commit *state);
+int dm_dp_mst_atomic_check(struct drm_connector *connector,
+				   struct drm_atomic_commit *state);
+int dm_dp_mst_detect(struct drm_connector *connector,
+			     struct drm_modeset_acquire_ctx *ctx, bool force);
+#endif
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
index c7f8e08feaf4b..26f35434a92f0 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
@@ -37,6 +37,7 @@
 #include "amdgpu_display.h"
 #include "amdgpu_dm_trace.h"
 #include "amdgpu_dm_plane.h"
+#include "amdgpu_dm_kunit_helpers.h"
 #include "amdgpu_dm_colorop.h"
 #include "gc/gc_11_0_0_offset.h"
 #include "gc/gc_11_0_0_sh_mask.h"
@@ -97,6 +98,7 @@ const struct drm_format_info *amdgpu_dm_plane_get_format_info(u32 pixel_format,
 {
 	return amdgpu_lookup_format_info(pixel_format, modifier);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_get_format_info);
 
 void amdgpu_dm_plane_fill_blending_from_plane_state(const struct drm_plane_state *plane_state,
 			       bool *per_pixel_alpha, bool *pre_multiplied_alpha,
@@ -135,12 +137,24 @@ void amdgpu_dm_plane_fill_blending_from_plane_state(const struct drm_plane_state
 	}
 
 	if (plane_state->alpha < 0xffff) {
+		struct amdgpu_device *adev = drm_to_adev(plane_state->plane->dev);
 		*global_alpha = true;
-		*global_alpha_value = plane_state->alpha >> 8;
+		/*
+		 * DCN 4.2 uses a 12-bit MPCC_GLOBAL_ALPHA field, while
+		 * other ASICs use an 8-bit field. The DRM plane alpha is
+		 * 16-bit, so scale it down to the width the hardware expects.
+		 */
+		if (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 2, 0))
+			*global_alpha_value = plane_state->alpha >> 4;
+		else
+			*global_alpha_value = plane_state->alpha >> 8;
+
 	}
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_fill_blending_from_plane_state);
 
-static void amdgpu_dm_plane_add_modifier(uint64_t **mods, uint64_t *size, uint64_t *cap, uint64_t mod)
+STATIC_IFN_KUNIT void amdgpu_dm_plane_add_modifier(uint64_t **mods, uint64_t *size,
+						   uint64_t *cap, uint64_t mod)
 {
 	if (!*mods)
 		return;
@@ -164,27 +178,29 @@ static void amdgpu_dm_plane_add_modifier(uint64_t **mods, uint64_t *size, uint64
 	(*mods)[*size] = mod;
 	*size += 1;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_add_modifier);
 
-static bool amdgpu_dm_plane_modifier_has_dcc(uint64_t modifier)
+STATIC_IFN_KUNIT bool amdgpu_dm_plane_modifier_has_dcc(uint64_t modifier)
 {
 	return IS_AMD_FMT_MOD(modifier) && AMD_FMT_MOD_GET(DCC, modifier);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_modifier_has_dcc);
 
-static unsigned int amdgpu_dm_plane_modifier_gfx9_swizzle_mode(uint64_t modifier)
+STATIC_IFN_KUNIT unsigned int amdgpu_dm_plane_modifier_gfx9_swizzle_mode(uint64_t modifier)
 {
 	if (modifier == DRM_FORMAT_MOD_LINEAR)
 		return 0;
 
 	return AMD_FMT_MOD_GET(TILE, modifier);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_modifier_gfx9_swizzle_mode);
 
-static void amdgpu_dm_plane_fill_gfx8_tiling_info_from_flags(struct dc_tiling_info *tiling_info,
-							     uint64_t tiling_flags)
+STATIC_IFN_KUNIT void amdgpu_dm_plane_fill_gfx8_tiling_info_from_flags(struct dc_tiling_info *tiling_info,
+								       uint64_t tiling_flags)
 {
 	/* Fill GFX8 params */
 	if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == DC_ARRAY_2D_TILED_THIN1) {
 		unsigned int bankw, bankh, mtaspect, tile_split, num_banks;
-
 		bankw = AMDGPU_TILING_GET(tiling_flags, BANK_WIDTH);
 		bankh = AMDGPU_TILING_GET(tiling_flags, BANK_HEIGHT);
 		mtaspect = AMDGPU_TILING_GET(tiling_flags, MACRO_TILE_ASPECT);
@@ -210,9 +226,10 @@ static void amdgpu_dm_plane_fill_gfx8_tiling_info_from_flags(struct dc_tiling_in
 	tiling_info->gfx8.pipe_config =
 			AMDGPU_TILING_GET(tiling_flags, PIPE_CONFIG);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_fill_gfx8_tiling_info_from_flags);
 
-static void amdgpu_dm_plane_fill_gfx9_tiling_info_from_device(const struct amdgpu_device *adev,
-							      struct dc_tiling_info *tiling_info)
+STATIC_IFN_KUNIT void amdgpu_dm_plane_fill_gfx9_tiling_info_from_device(const struct amdgpu_device *adev,
+									struct dc_tiling_info *tiling_info)
 {
 	/* Fill GFX9 params */
 	tiling_info->gfx9.num_pipes =
@@ -231,10 +248,11 @@ static void amdgpu_dm_plane_fill_gfx9_tiling_info_from_device(const struct amdgp
 	if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 3, 0))
 		tiling_info->gfx9.num_pkrs = adev->gfx.config.gb_addr_config_fields.num_pkrs;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_fill_gfx9_tiling_info_from_device);
 
-static void amdgpu_dm_plane_fill_gfx9_tiling_info_from_modifier(const struct amdgpu_device *adev,
-								struct dc_tiling_info *tiling_info,
-								uint64_t modifier)
+STATIC_IFN_KUNIT void amdgpu_dm_plane_fill_gfx9_tiling_info_from_modifier(const struct amdgpu_device *adev,
+									  struct dc_tiling_info *tiling_info,
+									  uint64_t modifier)
 {
 	unsigned int mod_bank_xor_bits = AMD_FMT_MOD_GET(BANK_XOR_BITS, modifier);
 	unsigned int mod_pipe_xor_bits = AMD_FMT_MOD_GET(PIPE_XOR_BITS, modifier);
@@ -259,14 +277,15 @@ static void amdgpu_dm_plane_fill_gfx9_tiling_info_from_modifier(const struct amd
 		/* for DCC we know it isn't rb aligned, so rb_per_se doesn't matter. */
 	}
 }
-
-static int amdgpu_dm_plane_validate_dcc(struct amdgpu_device *adev,
-					const enum surface_pixel_format format,
-					const enum dc_rotation_angle rotation,
-					const struct dc_tiling_info *tiling_info,
-					const struct dc_plane_dcc_param *dcc,
-					const struct dc_plane_address *address,
-					const struct plane_size *plane_size)
+EXPORT_IF_KUNIT(amdgpu_dm_plane_fill_gfx9_tiling_info_from_modifier);
+
+STATIC_IFN_KUNIT int amdgpu_dm_plane_validate_dcc(struct amdgpu_device *adev,
+						  const enum surface_pixel_format format,
+						  const enum dc_rotation_angle rotation,
+						  const struct dc_tiling_info *tiling_info,
+						  const struct dc_plane_dcc_param *dcc,
+						  const struct dc_plane_address *address,
+						  const struct plane_size *plane_size)
 {
 	struct dc *dc = adev->dm.dc;
 	struct dc_dcc_surface_param input;
@@ -307,15 +326,16 @@ static int amdgpu_dm_plane_validate_dcc(struct amdgpu_device *adev,
 
 	return 0;
 }
-
-static int amdgpu_dm_plane_fill_gfx9_plane_attributes_from_modifiers(struct amdgpu_device *adev,
-								     const struct amdgpu_framebuffer *afb,
-								     const enum surface_pixel_format format,
-								     const enum dc_rotation_angle rotation,
-								     const struct plane_size *plane_size,
-								     struct dc_tiling_info *tiling_info,
-								     struct dc_plane_dcc_param *dcc,
-								     struct dc_plane_address *address)
+EXPORT_IF_KUNIT(amdgpu_dm_plane_validate_dcc);
+
+STATIC_IFN_KUNIT int amdgpu_dm_plane_fill_gfx9_attrs_from_modifiers(struct amdgpu_device *adev,
+									       const struct amdgpu_framebuffer *afb,
+									       const enum surface_pixel_format format,
+									       const enum dc_rotation_angle rotation,
+									       const struct plane_size *plane_size,
+									       struct dc_tiling_info *tiling_info,
+									       struct dc_plane_dcc_param *dcc,
+									       struct dc_plane_address *address)
 {
 	const uint64_t modifier = afb->base.modifier;
 	int ret = 0;
@@ -354,19 +374,20 @@ static int amdgpu_dm_plane_fill_gfx9_plane_attributes_from_modifiers(struct amdg
 
 	ret = amdgpu_dm_plane_validate_dcc(adev, format, rotation, tiling_info, dcc, address, plane_size);
 	if (ret)
-		drm_dbg_kms(adev_to_drm(adev), "amdgpu_dm_plane_validate_dcc: returned error: %d\n", ret);
+		drm_dbg_kms(adev_to_drm(adev), "amdgpu_dm_plane_validate_dcc: returned error: %pe\n", ERR_PTR(ret));
 
 	return ret;
 }
-
-static int amdgpu_dm_plane_fill_gfx12_plane_attributes_from_modifiers(struct amdgpu_device *adev,
-								      const struct amdgpu_framebuffer *afb,
-								      const enum surface_pixel_format format,
-								      const enum dc_rotation_angle rotation,
-								      const struct plane_size *plane_size,
-								      struct dc_tiling_info *tiling_info,
-								      struct dc_plane_dcc_param *dcc,
-								      struct dc_plane_address *address)
+EXPORT_IF_KUNIT(amdgpu_dm_plane_fill_gfx9_attrs_from_modifiers);
+
+STATIC_IFN_KUNIT int amdgpu_dm_plane_fill_gfx12_attrs_from_modifiers(struct amdgpu_device *adev,
+										const struct amdgpu_framebuffer *afb,
+										const enum surface_pixel_format format,
+										const enum dc_rotation_angle rotation,
+										const struct plane_size *plane_size,
+										struct dc_tiling_info *tiling_info,
+										struct dc_plane_dcc_param *dcc,
+										struct dc_plane_address *address)
 {
 	const uint64_t modifier = afb->base.modifier;
 	int ret = 0;
@@ -394,10 +415,11 @@ static int amdgpu_dm_plane_fill_gfx12_plane_attributes_from_modifiers(struct amd
 	/* TODO: This seems wrong because there is no DCC plane on GFX12. */
 	ret = amdgpu_dm_plane_validate_dcc(adev, format, rotation, tiling_info, dcc, address, plane_size);
 	if (ret)
-		drm_dbg_kms(adev_to_drm(adev), "amdgpu_dm_plane_validate_dcc: returned error: %d\n", ret);
+		drm_dbg_kms(adev_to_drm(adev), "amdgpu_dm_plane_validate_dcc: returned: %pe\n", ERR_PTR(ret));
 
 	return ret;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_fill_gfx12_attrs_from_modifiers);
 
 static void amdgpu_dm_plane_add_gfx10_1_modifiers(const struct amdgpu_device *adev,
 						  uint64_t **mods,
@@ -724,7 +746,7 @@ static void amdgpu_dm_plane_add_gfx12_modifiers(struct amdgpu_device *adev,
 
 }
 
-static int amdgpu_dm_plane_get_plane_modifiers(struct amdgpu_device *adev, unsigned int plane_type, uint64_t **mods)
+STATIC_IFN_KUNIT int amdgpu_dm_plane_get_plane_modifiers(struct amdgpu_device *adev, unsigned int plane_type, uint64_t **mods)
 {
 	uint64_t size = 0, capacity = 128;
 	*mods = NULL;
@@ -777,10 +799,11 @@ static int amdgpu_dm_plane_get_plane_modifiers(struct amdgpu_device *adev, unsig
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_get_plane_modifiers);
 
-static int amdgpu_dm_plane_get_plane_formats(const struct drm_plane *plane,
-					     const struct dc_plane_cap *plane_cap,
-					     uint32_t *formats, int max_formats)
+STATIC_IFN_KUNIT int amdgpu_dm_plane_get_plane_formats(const struct drm_plane *plane,
+						       const struct dc_plane_cap *plane_cap,
+						       uint32_t *formats, int max_formats)
 {
 	int i, num_formats = 0;
 
@@ -836,6 +859,7 @@ static int amdgpu_dm_plane_get_plane_formats(const struct drm_plane *plane,
 
 	return num_formats;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_get_plane_formats);
 
 int amdgpu_dm_plane_fill_plane_buffer_attributes(struct amdgpu_device *adev,
 			     const struct amdgpu_framebuffer *afb,
@@ -903,14 +927,14 @@ int amdgpu_dm_plane_fill_plane_buffer_attributes(struct amdgpu_device *adev,
 	}
 
 	if (adev->family == AMDGPU_FAMILY_GC_12_0_0) {
-		ret = amdgpu_dm_plane_fill_gfx12_plane_attributes_from_modifiers(adev, afb, format,
+		ret = amdgpu_dm_plane_fill_gfx12_attrs_from_modifiers(adev, afb, format,
 										 rotation, plane_size,
 										 tiling_info, dcc,
 										 address);
 		if (ret)
 			return ret;
 	} else if (adev->family >= AMDGPU_FAMILY_AI) {
-		ret = amdgpu_dm_plane_fill_gfx9_plane_attributes_from_modifiers(adev, afb, format,
+		ret = amdgpu_dm_plane_fill_gfx9_attrs_from_modifiers(adev, afb, format,
 										rotation, plane_size,
 										tiling_info, dcc,
 										address);
@@ -922,6 +946,7 @@ int amdgpu_dm_plane_fill_plane_buffer_attributes(struct amdgpu_device *adev,
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_fill_plane_buffer_attributes);
 
 static int amdgpu_dm_plane_helper_prepare_fb(struct drm_plane *plane,
 					     struct drm_plane_state *new_state)
@@ -950,13 +975,15 @@ static int amdgpu_dm_plane_helper_prepare_fb(struct drm_plane *plane,
 	adev = amdgpu_ttm_adev(rbo->tbo.bdev);
 	r = amdgpu_bo_reserve(rbo, true);
 	if (r) {
-		drm_err(adev_to_drm(adev), "fail to reserve bo (%d)\n", r);
+		drm_err(adev_to_drm(adev), "fail to reserve bo: %pe\n", ERR_PTR(r));
 		return r;
 	}
 
 	r = dma_resv_reserve_fences(rbo->tbo.base.resv, TTM_NUM_MOVE_FENCES);
-	if (r)
+	if (r) {
+		drm_err(adev_to_drm(adev), "reserving fence slot failed: %pe\n", ERR_PTR(r));
 		goto error_unlock;
+	}
 
 	if (plane->type != DRM_PLANE_TYPE_CURSOR)
 		domain = amdgpu_display_supported_domains(adev, rbo->flags);
@@ -967,13 +994,13 @@ static int amdgpu_dm_plane_helper_prepare_fb(struct drm_plane *plane,
 	r = amdgpu_bo_pin(rbo, domain);
 	if (unlikely(r != 0)) {
 		if (r != -ERESTARTSYS)
-			DRM_ERROR("Failed to pin framebuffer with error %d\n", r);
+			DRM_ERROR("Failed to pin framebuffer: %pe\n", ERR_PTR(r));
 		goto error_unlock;
 	}
 
 	r = amdgpu_ttm_alloc_gart(&rbo->tbo);
 	if (unlikely(r != 0)) {
-		DRM_ERROR("%p bind failed\n", rbo);
+		DRM_ERROR("%p bind failed: %pe\n", rbo, ERR_PTR(r));
 		goto error_unpin;
 	}
 
@@ -1033,7 +1060,7 @@ static void amdgpu_dm_plane_helper_cleanup_fb(struct drm_plane *plane,
 	rbo = gem_to_amdgpu_bo(old_state->fb->obj[0]);
 	r = amdgpu_bo_reserve(rbo, false);
 	if (unlikely(r)) {
-		DRM_ERROR("failed to reserve rbo before unpin\n");
+		DRM_ERROR("failed to reserve rbo before unpin: %pe\n", ERR_PTR(r));
 		return;
 	}
 
@@ -1042,9 +1069,9 @@ static void amdgpu_dm_plane_helper_cleanup_fb(struct drm_plane *plane,
 	amdgpu_bo_unref(&rbo);
 }
 
-static void amdgpu_dm_plane_get_min_max_dc_plane_scaling(struct drm_device *dev,
-					 struct drm_framebuffer *fb,
-					 int *min_downscale, int *max_upscale)
+STATIC_IFN_KUNIT void amdgpu_dm_plane_get_min_max_dc_plane_scaling(struct drm_device *dev,
+								   struct drm_framebuffer *fb,
+								   int *min_downscale, int *max_upscale)
 {
 	struct amdgpu_device *adev = drm_to_adev(dev);
 	struct dc *dc = adev->dm.dc;
@@ -1088,6 +1115,7 @@ static void amdgpu_dm_plane_get_min_max_dc_plane_scaling(struct drm_device *dev,
 	if (*min_downscale == 1)
 		*min_downscale = 1000;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_get_min_max_dc_plane_scaling);
 
 int amdgpu_dm_plane_helper_check_state(struct drm_plane_state *state,
 				       struct drm_crtc_state *new_crtc_state)
@@ -1142,6 +1170,7 @@ int amdgpu_dm_plane_helper_check_state(struct drm_plane_state *state,
 	return drm_atomic_helper_check_plane_state(
 		state, new_crtc_state, min_scale, max_scale, true, true);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_helper_check_state);
 
 int amdgpu_dm_plane_fill_dc_scaling_info(struct amdgpu_device *adev,
 				const struct drm_plane_state *state,
@@ -1225,9 +1254,10 @@ int amdgpu_dm_plane_fill_dc_scaling_info(struct amdgpu_device *adev,
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_fill_dc_scaling_info);
 
-static int amdgpu_dm_plane_atomic_check(struct drm_plane *plane,
-					struct drm_atomic_commit *state)
+STATIC_IFN_KUNIT int amdgpu_dm_plane_atomic_check(struct drm_plane *plane,
+						  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -1272,9 +1302,10 @@ static int amdgpu_dm_plane_atomic_check(struct drm_plane *plane,
 
 	return -EINVAL;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_atomic_check);
 
-static int amdgpu_dm_plane_atomic_async_check(struct drm_plane *plane,
-					      struct drm_atomic_commit *state, bool flip)
+STATIC_IFN_KUNIT int amdgpu_dm_plane_atomic_async_check(struct drm_plane *plane,
+							struct drm_atomic_commit *state, bool flip)
 {
 	struct drm_crtc_state *new_crtc_state;
 	struct drm_plane_state *new_plane_state;
@@ -1296,6 +1327,7 @@ static int amdgpu_dm_plane_atomic_async_check(struct drm_plane *plane,
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_atomic_async_check);
 
 int amdgpu_dm_plane_get_cursor_position(struct drm_plane *plane, struct drm_crtc *crtc,
 					struct dc_cursor_position *position)
@@ -1343,6 +1375,7 @@ int amdgpu_dm_plane_get_cursor_position(struct drm_plane *plane, struct drm_crtc
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_get_cursor_position);
 
 void amdgpu_dm_plane_handle_cursor_update(struct drm_plane *plane,
 				 struct drm_plane_state *old_plane_state)
@@ -1454,7 +1487,7 @@ static void amdgpu_dm_plane_atomic_async_update(struct drm_plane *plane,
 	amdgpu_dm_plane_handle_cursor_update(plane, old_state);
 }
 
-static void amdgpu_dm_plane_panic_flush(struct drm_plane *plane)
+STATIC_IFN_KUNIT void amdgpu_dm_plane_panic_flush(struct drm_plane *plane)
 {
 	struct dm_plane_state *dm_plane_state = to_dm_plane_state(plane->state);
 	struct drm_framebuffer *fb = plane->state->fb;
@@ -1467,6 +1500,7 @@ static void amdgpu_dm_plane_panic_flush(struct drm_plane *plane)
 
 	dc_plane_force_dcc_and_tiling_disable(dc_plane_state, fb->modifier ? true : false);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_panic_flush);
 
 static const struct drm_plane_helper_funcs dm_plane_helper_funcs = {
 	.prepare_fb = amdgpu_dm_plane_helper_prepare_fb,
@@ -1486,27 +1520,27 @@ static const struct drm_plane_helper_funcs dm_primary_plane_helper_funcs = {
 	.panic_flush = amdgpu_dm_plane_panic_flush,
 };
 
-static void amdgpu_dm_plane_drm_plane_reset(struct drm_plane *plane)
+STATIC_IFN_KUNIT void amdgpu_dm_plane_drm_plane_reset(struct drm_plane *plane)
 {
-	struct dm_plane_state *amdgpu_state = NULL;
-
-	if (plane->state)
-		plane->funcs->atomic_destroy_state(plane, plane->state);
+	struct dm_plane_state *amdgpu_state;
 
 	amdgpu_state = kzalloc_obj(*amdgpu_state);
-	WARN_ON(amdgpu_state == NULL);
-
 	if (!amdgpu_state)
 		return;
 
+	if (plane->state)
+		plane->funcs->atomic_destroy_state(plane, plane->state);
+
 	__drm_atomic_helper_plane_reset(plane, &amdgpu_state->base);
 	amdgpu_state->degamma_tf = AMDGPU_TRANSFER_FUNCTION_DEFAULT;
 	amdgpu_state->hdr_mult = AMDGPU_HDR_MULT_DEFAULT;
 	amdgpu_state->shaper_tf = AMDGPU_TRANSFER_FUNCTION_DEFAULT;
 	amdgpu_state->blend_tf = AMDGPU_TRANSFER_FUNCTION_DEFAULT;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_drm_plane_reset);
 
-static struct drm_plane_state *amdgpu_dm_plane_drm_plane_duplicate_state(struct drm_plane *plane)
+STATIC_IFN_KUNIT struct drm_plane_state *
+amdgpu_dm_plane_drm_plane_duplicate_state(struct drm_plane *plane)
 {
 	struct dm_plane_state *dm_plane_state, *old_dm_plane_state;
 
@@ -1545,10 +1579,11 @@ static struct drm_plane_state *amdgpu_dm_plane_drm_plane_duplicate_state(struct
 
 	return &dm_plane_state->base;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_drm_plane_duplicate_state);
 
-static bool amdgpu_dm_plane_format_mod_supported(struct drm_plane *plane,
-						 uint32_t format,
-						 uint64_t modifier)
+STATIC_IFN_KUNIT bool amdgpu_dm_plane_format_mod_supported(struct drm_plane *plane,
+							   uint32_t format,
+							   uint64_t modifier)
 {
 	struct amdgpu_device *adev = drm_to_adev(plane->dev);
 	const struct drm_format_info *info = drm_format_info(format);
@@ -1607,9 +1642,10 @@ static bool amdgpu_dm_plane_format_mod_supported(struct drm_plane *plane,
 
 	return true;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_format_mod_supported);
 
-static void amdgpu_dm_plane_drm_plane_destroy_state(struct drm_plane *plane,
-						    struct drm_plane_state *state)
+STATIC_IFN_KUNIT void amdgpu_dm_plane_drm_plane_destroy_state(struct drm_plane *plane,
+							      struct drm_plane_state *state)
 {
 	struct dm_plane_state *dm_plane_state = to_dm_plane_state(state);
 
@@ -1629,6 +1665,7 @@ static void amdgpu_dm_plane_drm_plane_destroy_state(struct drm_plane *plane,
 
 	drm_atomic_helper_plane_destroy_state(plane, state);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_drm_plane_destroy_state);
 
 #ifdef AMD_PRIVATE_COLOR
 static void
@@ -1833,8 +1870,8 @@ dm_plane_init_colorops(struct drm_plane *plane)
 	if (dc->ctx->dce_version >= DCN_VERSION_3_0) {
 		ret = amdgpu_dm_initialize_default_pipeline(plane, &pipelines[len]);
 		if (ret) {
-			drm_err(plane->dev, "Failed to create color pipeline for plane %d: %d\n",
-				plane->base.id, ret);
+			drm_err(plane->dev, "Failed to create color pipeline for plane %d: %pe\n",
+				plane->base.id, ERR_PTR(ret));
 			goto out;
 		}
 		len++;
@@ -1982,4 +2019,5 @@ bool amdgpu_dm_plane_is_video_format(uint32_t format)
 
 	return false;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_is_video_format);
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.h
index ea2619b507db7..365e306a52e78 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.h
@@ -28,6 +28,8 @@
 #define __AMDGPU_DM_PLANE_H__
 
 #include "dc.h"
+#include <drm/drm_plane.h>
+#include "amdgpu.h"
 
 int amdgpu_dm_plane_get_cursor_position(struct drm_plane *plane, struct drm_crtc *crtc,
 					struct dc_cursor_position *position);
@@ -65,4 +67,62 @@ void amdgpu_dm_plane_fill_blending_from_plane_state(const struct drm_plane_state
 				    bool *global_alpha, int *global_alpha_value);
 
 bool amdgpu_dm_plane_is_video_format(uint32_t format);
+
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+void amdgpu_dm_plane_add_modifier(uint64_t **mods, uint64_t *size,
+				  uint64_t *cap, uint64_t mod);
+void amdgpu_dm_plane_fill_gfx8_tiling_info_from_flags(struct dc_tiling_info *tiling_info,
+						      uint64_t tiling_flags);
+void amdgpu_dm_plane_fill_gfx9_tiling_info_from_device(const struct amdgpu_device *adev,
+						       struct dc_tiling_info *tiling_info);
+void amdgpu_dm_plane_fill_gfx9_tiling_info_from_modifier(const struct amdgpu_device *adev,
+							 struct dc_tiling_info *tiling_info,
+							 uint64_t modifier);
+int amdgpu_dm_plane_validate_dcc(struct amdgpu_device *adev,
+				 const enum surface_pixel_format format,
+				 const enum dc_rotation_angle rotation,
+				 const struct dc_tiling_info *tiling_info,
+				 const struct dc_plane_dcc_param *dcc,
+				 const struct dc_plane_address *address,
+				 const struct plane_size *plane_size);
+bool amdgpu_dm_plane_modifier_has_dcc(uint64_t modifier);
+unsigned int amdgpu_dm_plane_modifier_gfx9_swizzle_mode(uint64_t modifier);
+int amdgpu_dm_plane_get_plane_modifiers(struct amdgpu_device *adev,
+					unsigned int plane_type, uint64_t **mods);
+int amdgpu_dm_plane_get_plane_formats(const struct drm_plane *plane,
+				      const struct dc_plane_cap *plane_cap,
+				      uint32_t *formats, int max_formats);
+int amdgpu_dm_plane_fill_gfx9_attrs_from_modifiers(struct amdgpu_device *adev,
+							      const struct amdgpu_framebuffer *afb,
+							      const enum surface_pixel_format format,
+							      const enum dc_rotation_angle rotation,
+							      const struct plane_size *plane_size,
+							      struct dc_tiling_info *tiling_info,
+							      struct dc_plane_dcc_param *dcc,
+							      struct dc_plane_address *address);
+int amdgpu_dm_plane_fill_gfx12_attrs_from_modifiers(struct amdgpu_device *adev,
+							       const struct amdgpu_framebuffer *afb,
+							       const enum surface_pixel_format format,
+							       const enum dc_rotation_angle rotation,
+							       const struct plane_size *plane_size,
+							       struct dc_tiling_info *tiling_info,
+							       struct dc_plane_dcc_param *dcc,
+							       struct dc_plane_address *address);
+bool amdgpu_dm_plane_format_mod_supported(struct drm_plane *plane,
+					  uint32_t format,
+					  uint64_t modifier);
+void amdgpu_dm_plane_get_min_max_dc_plane_scaling(struct drm_device *dev,
+						  struct drm_framebuffer *fb,
+						  int *min_downscale,
+						  int *max_upscale);
+int amdgpu_dm_plane_atomic_async_check(struct drm_plane *plane,
+				       struct drm_atomic_commit *state, bool flip);
+int amdgpu_dm_plane_atomic_check(struct drm_plane *plane,
+				 struct drm_atomic_commit *state);
+void amdgpu_dm_plane_panic_flush(struct drm_plane *plane);
+void amdgpu_dm_plane_drm_plane_reset(struct drm_plane *plane);
+struct drm_plane_state *amdgpu_dm_plane_drm_plane_duplicate_state(struct drm_plane *plane);
+void amdgpu_dm_plane_drm_plane_destroy_state(struct drm_plane *plane,
+					     struct drm_plane_state *state);
+#endif
 #endif
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.c
index 2cdb8fea504a0..0d2e5294d0625 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.c
@@ -33,73 +33,67 @@
 #include "amdgpu_dm_irq.h"
 #include "amdgpu_pm.h"
 #include "dm_pp_smu.h"
+#include "amdgpu_dm_kunit_helpers.h"
+#include "amdgpu_dm_pp_smu.h"
 
-bool dm_pp_apply_display_requirements(
-		const struct dc_context *ctx,
+STATIC_IFN_KUNIT void build_pm_display_cfg(
+		struct amd_pp_display_configuration *pm_display_cfg,
 		const struct dm_pp_display_configuration *pp_display_cfg)
 {
-	struct amdgpu_device *adev = ctx->driver_context;
 	int i;
 
-	if (adev->pm.dpm_enabled) {
+	memset(pm_display_cfg, 0, sizeof(*pm_display_cfg));
 
-		memset(&adev->pm.pm_display_cfg, 0,
-				sizeof(adev->pm.pm_display_cfg));
+	pm_display_cfg->cpu_cc6_disable = pp_display_cfg->cpu_cc6_disable;
+	pm_display_cfg->cpu_pstate_disable = pp_display_cfg->cpu_pstate_disable;
+	pm_display_cfg->cpu_pstate_separation_time = pp_display_cfg->cpu_pstate_separation_time;
+	pm_display_cfg->nb_pstate_switch_disable = pp_display_cfg->nb_pstate_switch_disable;
 
-		adev->pm.pm_display_cfg.cpu_cc6_disable =
-			pp_display_cfg->cpu_cc6_disable;
+	pm_display_cfg->num_display = pp_display_cfg->display_count;
+	pm_display_cfg->num_path_including_non_display = pp_display_cfg->display_count;
 
-		adev->pm.pm_display_cfg.cpu_pstate_disable =
-			pp_display_cfg->cpu_pstate_disable;
+	pm_display_cfg->min_core_set_clock = pp_display_cfg->min_engine_clock_khz/10;
+	pm_display_cfg->min_core_set_clock_in_sr =
+			pp_display_cfg->min_engine_clock_deep_sleep_khz/10;
+	pm_display_cfg->min_mem_set_clock = pp_display_cfg->min_memory_clock_khz/10;
 
-		adev->pm.pm_display_cfg.cpu_pstate_separation_time =
-			pp_display_cfg->cpu_pstate_separation_time;
+	pm_display_cfg->min_dcef_deep_sleep_set_clk =
+			pp_display_cfg->min_engine_clock_deep_sleep_khz/10;
+	pm_display_cfg->min_dcef_set_clk = pp_display_cfg->min_dcfclock_khz/10;
 
-		adev->pm.pm_display_cfg.nb_pstate_switch_disable =
-			pp_display_cfg->nb_pstate_switch_disable;
+	pm_display_cfg->multi_monitor_in_sync = pp_display_cfg->all_displays_in_sync;
+	pm_display_cfg->min_vblank_time = pp_display_cfg->avail_mclk_switch_time_us;
 
-		adev->pm.pm_display_cfg.num_display =
-				pp_display_cfg->display_count;
-		adev->pm.pm_display_cfg.num_path_including_non_display =
-				pp_display_cfg->display_count;
+	pm_display_cfg->display_clk = pp_display_cfg->disp_clk_khz/10;
 
-		adev->pm.pm_display_cfg.min_core_set_clock =
-				pp_display_cfg->min_engine_clock_khz/10;
-		adev->pm.pm_display_cfg.min_core_set_clock_in_sr =
-				pp_display_cfg->min_engine_clock_deep_sleep_khz/10;
-		adev->pm.pm_display_cfg.min_mem_set_clock =
-				pp_display_cfg->min_memory_clock_khz/10;
+	pm_display_cfg->dce_tolerable_mclk_in_active_latency =
+			pp_display_cfg->avail_mclk_switch_time_in_disp_active_us;
 
-		adev->pm.pm_display_cfg.min_dcef_deep_sleep_set_clk =
-				pp_display_cfg->min_engine_clock_deep_sleep_khz/10;
-		adev->pm.pm_display_cfg.min_dcef_set_clk =
-				pp_display_cfg->min_dcfclock_khz/10;
+	pm_display_cfg->crtc_index = pp_display_cfg->crtc_index;
+	pm_display_cfg->line_time_in_us = pp_display_cfg->line_time_in_us;
 
-		adev->pm.pm_display_cfg.multi_monitor_in_sync =
-				pp_display_cfg->all_displays_in_sync;
-		adev->pm.pm_display_cfg.min_vblank_time =
-				pp_display_cfg->avail_mclk_switch_time_us;
+	pm_display_cfg->vrefresh = pp_display_cfg->disp_configs[0].v_refresh;
+	pm_display_cfg->crossfire_display_index = -1;
+	pm_display_cfg->min_bus_bandwidth = 0;
 
-		adev->pm.pm_display_cfg.display_clk =
-				pp_display_cfg->disp_clk_khz/10;
-
-		adev->pm.pm_display_cfg.dce_tolerable_mclk_in_active_latency =
-				pp_display_cfg->avail_mclk_switch_time_in_disp_active_us;
+	for (i = 0; i < pp_display_cfg->display_count; i++) {
+		const struct dm_pp_single_disp_config *dc_cfg =
+					&pp_display_cfg->disp_configs[i];
+		pm_display_cfg->displays[i].controller_id = dc_cfg->pipe_idx + 1;
+		pm_display_cfg->displays[i].pixel_clock = dc_cfg->pixel_clock;
+	}
+}
+EXPORT_IF_KUNIT(build_pm_display_cfg);
 
-		adev->pm.pm_display_cfg.crtc_index = pp_display_cfg->crtc_index;
-		adev->pm.pm_display_cfg.line_time_in_us =
-				pp_display_cfg->line_time_in_us;
+bool dm_pp_apply_display_requirements(
+		const struct dc_context *ctx,
+		const struct dm_pp_display_configuration *pp_display_cfg)
+{
+	struct amdgpu_device *adev = ctx->driver_context;
 
-		adev->pm.pm_display_cfg.vrefresh = pp_display_cfg->disp_configs[0].v_refresh;
-		adev->pm.pm_display_cfg.crossfire_display_index = -1;
-		adev->pm.pm_display_cfg.min_bus_bandwidth = 0;
+	if (adev->pm.dpm_enabled) {
 
-		for (i = 0; i < pp_display_cfg->display_count; i++) {
-			const struct dm_pp_single_disp_config *dc_cfg =
-						&pp_display_cfg->disp_configs[i];
-			adev->pm.pm_display_cfg.displays[i].controller_id = dc_cfg->pipe_idx + 1;
-			adev->pm.pm_display_cfg.displays[i].pixel_clock = dc_cfg->pixel_clock;
-		}
+		build_pm_display_cfg(&adev->pm.pm_display_cfg, pp_display_cfg);
 
 		amdgpu_dpm_display_configuration_change(adev, &adev->pm.pm_display_cfg);
 
@@ -108,8 +102,9 @@ bool dm_pp_apply_display_requirements(
 
 	return true;
 }
+EXPORT_IF_KUNIT(dm_pp_apply_display_requirements);
 
-static void get_default_clock_levels(
+STATIC_IFN_KUNIT void get_default_clock_levels(
 		enum dm_pp_clock_type clk_type,
 		struct dm_pp_clock_levels *clks)
 {
@@ -140,8 +135,9 @@ static void get_default_clock_levels(
 		break;
 	}
 }
+EXPORT_IF_KUNIT(get_default_clock_levels);
 
-static enum amd_pp_clock_type dc_to_pp_clock_type(
+STATIC_IFN_KUNIT enum amd_pp_clock_type dc_to_pp_clock_type(
 		enum dm_pp_clock_type dm_pp_clk_type)
 {
 	enum amd_pp_clock_type amd_pp_clk_type = 0;
@@ -182,8 +178,9 @@ static enum amd_pp_clock_type dc_to_pp_clock_type(
 
 	return amd_pp_clk_type;
 }
+EXPORT_IF_KUNIT(dc_to_pp_clock_type);
 
-static void pp_to_dc_clock_levels(
+STATIC_IFN_KUNIT void pp_to_dc_clock_levels(
 		const struct amd_pp_clocks *pp_clks,
 		struct dm_pp_clock_levels *dc_clks,
 		enum dm_pp_clock_type dc_clk_type)
@@ -208,8 +205,9 @@ static void pp_to_dc_clock_levels(
 		dc_clks->clocks_in_khz[i] = pp_clks->clock[i];
 	}
 }
+EXPORT_IF_KUNIT(pp_to_dc_clock_levels);
 
-static void pp_to_dc_clock_levels_with_latency(
+STATIC_IFN_KUNIT void pp_to_dc_clock_levels_with_latency(
 		const struct pp_clock_levels_with_latency *pp_clks,
 		struct dm_pp_clock_levels_with_latency *clk_level_info,
 		enum dm_pp_clock_type dc_clk_type)
@@ -235,8 +233,9 @@ static void pp_to_dc_clock_levels_with_latency(
 		clk_level_info->data[i].latency_in_us = pp_clks->data[i].latency_in_us;
 	}
 }
+EXPORT_IF_KUNIT(pp_to_dc_clock_levels_with_latency);
 
-static void pp_to_dc_clock_levels_with_voltage(
+STATIC_IFN_KUNIT void pp_to_dc_clock_levels_with_voltage(
 		const struct pp_clock_levels_with_voltage *pp_clks,
 		struct dm_pp_clock_levels_with_voltage *clk_level_info,
 		enum dm_pp_clock_type dc_clk_type)
@@ -263,6 +262,41 @@ static void pp_to_dc_clock_levels_with_voltage(
 		clk_level_info->data[i].voltage_in_mv = pp_clks->data[i].voltage_in_mv;
 	}
 }
+EXPORT_IF_KUNIT(pp_to_dc_clock_levels_with_voltage);
+
+STATIC_IFN_KUNIT void cap_clock_levels_to_validation(
+		struct dm_pp_clock_levels *dc_clks,
+		enum dm_pp_clock_type clk_type,
+		const struct amd_pp_simple_clock_info *validation_clks)
+{
+	uint32_t i;
+
+	/* Determine the highest non-boosted level from the Validation Clocks */
+	if (clk_type == DM_PP_CLOCK_TYPE_ENGINE_CLK) {
+		for (i = 0; i < dc_clks->num_levels; i++) {
+			if (dc_clks->clocks_in_khz[i] > validation_clks->engine_max_clock) {
+				/* This clock is higher the validation clock.
+				 * Than means the previous one is the highest
+				 * non-boosted one.
+				 */
+				DRM_INFO("DM_PPLIB: reducing engine clock level from %d to %d\n",
+						dc_clks->num_levels, i);
+				dc_clks->num_levels = i > 0 ? i : 1;
+				break;
+			}
+		}
+	} else if (clk_type == DM_PP_CLOCK_TYPE_MEMORY_CLK) {
+		for (i = 0; i < dc_clks->num_levels; i++) {
+			if (dc_clks->clocks_in_khz[i] > validation_clks->memory_max_clock) {
+				DRM_INFO("DM_PPLIB: reducing memory clock level from %d to %d\n",
+						dc_clks->num_levels, i);
+				dc_clks->num_levels = i > 0 ? i : 1;
+				break;
+			}
+		}
+	}
+}
+EXPORT_IF_KUNIT(cap_clock_levels_to_validation);
 
 bool dm_pp_get_clock_levels_by_type(
 		const struct dc_context *ctx,
@@ -272,7 +306,6 @@ bool dm_pp_get_clock_levels_by_type(
 	struct amdgpu_device *adev = ctx->driver_context;
 	struct amd_pp_clocks pp_clks = { 0 };
 	struct amd_pp_simple_clock_info validation_clks = { 0 };
-	uint32_t i;
 
 	if (amdgpu_dpm_get_clock_by_type(adev,
 		dc_to_pp_clock_type(clk_type), &pp_clks)) {
@@ -300,33 +333,11 @@ bool dm_pp_get_clock_levels_by_type(
 	validation_clks.engine_max_clock *= 10;
 	validation_clks.memory_max_clock *= 10;
 
-	/* Determine the highest non-boosted level from the Validation Clocks */
-	if (clk_type == DM_PP_CLOCK_TYPE_ENGINE_CLK) {
-		for (i = 0; i < dc_clks->num_levels; i++) {
-			if (dc_clks->clocks_in_khz[i] > validation_clks.engine_max_clock) {
-				/* This clock is higher the validation clock.
-				 * Than means the previous one is the highest
-				 * non-boosted one.
-				 */
-				DRM_INFO("DM_PPLIB: reducing engine clock level from %d to %d\n",
-						dc_clks->num_levels, i);
-				dc_clks->num_levels = i > 0 ? i : 1;
-				break;
-			}
-		}
-	} else if (clk_type == DM_PP_CLOCK_TYPE_MEMORY_CLK) {
-		for (i = 0; i < dc_clks->num_levels; i++) {
-			if (dc_clks->clocks_in_khz[i] > validation_clks.memory_max_clock) {
-				DRM_INFO("DM_PPLIB: reducing memory clock level from %d to %d\n",
-						dc_clks->num_levels, i);
-				dc_clks->num_levels = i > 0 ? i : 1;
-				break;
-			}
-		}
-	}
+	cap_clock_levels_to_validation(dc_clks, clk_type, &validation_clks);
 
 	return true;
 }
+EXPORT_IF_KUNIT(dm_pp_get_clock_levels_by_type);
 
 bool dm_pp_get_clock_levels_by_type_with_latency(
 	const struct dc_context *ctx,
@@ -347,6 +358,7 @@ bool dm_pp_get_clock_levels_by_type_with_latency(
 
 	return true;
 }
+EXPORT_IF_KUNIT(dm_pp_get_clock_levels_by_type_with_latency);
 
 bool dm_pp_get_clock_levels_by_type_with_voltage(
 	const struct dc_context *ctx,
@@ -367,6 +379,7 @@ bool dm_pp_get_clock_levels_by_type_with_voltage(
 
 	return true;
 }
+EXPORT_IF_KUNIT(dm_pp_get_clock_levels_by_type_with_voltage);
 
 bool dm_pp_notify_wm_clock_changes(
 	const struct dc_context *ctx,
@@ -386,6 +399,7 @@ bool dm_pp_notify_wm_clock_changes(
 
 	return false;
 }
+EXPORT_IF_KUNIT(dm_pp_notify_wm_clock_changes);
 
 bool dm_pp_apply_clock_for_voltage_request(
 	const struct dc_context *ctx,
@@ -407,26 +421,26 @@ bool dm_pp_apply_clock_for_voltage_request(
 
 	return true;
 }
+EXPORT_IF_KUNIT(dm_pp_apply_clock_for_voltage_request);
 
-static void pp_rv_set_wm_ranges(struct pp_smu *pp,
-		struct pp_smu_wm_range_sets *ranges)
+STATIC_IFN_KUNIT void build_wm_clock_ranges_soc15(
+		const struct pp_smu_wm_range_sets *ranges,
+		struct dm_pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges)
 {
-	const struct dc_context *ctx = pp->dm;
-	struct amdgpu_device *adev = ctx->driver_context;
-	struct dm_pp_wm_sets_with_clock_ranges_soc15 wm_with_clock_ranges;
-	struct dm_pp_clock_range_for_dmif_wm_set_soc15 *wm_dce_clocks = wm_with_clock_ranges.wm_dmif_clocks_ranges;
-	struct dm_pp_clock_range_for_mcif_wm_set_soc15 *wm_soc_clocks = wm_with_clock_ranges.wm_mcif_clocks_ranges;
+	struct dm_pp_clock_range_for_dmif_wm_set_soc15 *wm_dce_clocks =
+			wm_with_clock_ranges->wm_dmif_clocks_ranges;
+	struct dm_pp_clock_range_for_mcif_wm_set_soc15 *wm_soc_clocks =
+			wm_with_clock_ranges->wm_mcif_clocks_ranges;
 	int32_t i;
 
-	wm_with_clock_ranges.num_wm_dmif_sets = ranges->num_reader_wm_sets;
-	wm_with_clock_ranges.num_wm_mcif_sets = ranges->num_writer_wm_sets;
+	wm_with_clock_ranges->num_wm_dmif_sets = ranges->num_reader_wm_sets;
+	wm_with_clock_ranges->num_wm_mcif_sets = ranges->num_writer_wm_sets;
 
-	for (i = 0; i < wm_with_clock_ranges.num_wm_dmif_sets; i++) {
+	for (i = 0; i < wm_with_clock_ranges->num_wm_dmif_sets; i++) {
 		if (ranges->reader_wm_sets[i].wm_inst > 3)
 			wm_dce_clocks[i].wm_set_id = WM_SET_A;
 		else
-			wm_dce_clocks[i].wm_set_id =
-					ranges->reader_wm_sets[i].wm_inst;
+			wm_dce_clocks[i].wm_set_id = ranges->reader_wm_sets[i].wm_inst;
 		wm_dce_clocks[i].wm_max_dcfclk_clk_in_khz =
 				ranges->reader_wm_sets[i].max_drain_clk_mhz * 1000;
 		wm_dce_clocks[i].wm_min_dcfclk_clk_in_khz =
@@ -437,12 +451,11 @@ static void pp_rv_set_wm_ranges(struct pp_smu *pp,
 				ranges->reader_wm_sets[i].min_fill_clk_mhz * 1000;
 	}
 
-	for (i = 0; i < wm_with_clock_ranges.num_wm_mcif_sets; i++) {
+	for (i = 0; i < wm_with_clock_ranges->num_wm_mcif_sets; i++) {
 		if (ranges->writer_wm_sets[i].wm_inst > 3)
 			wm_soc_clocks[i].wm_set_id = WM_SET_A;
 		else
-			wm_soc_clocks[i].wm_set_id =
-					ranges->writer_wm_sets[i].wm_inst;
+			wm_soc_clocks[i].wm_set_id = ranges->writer_wm_sets[i].wm_inst;
 		wm_soc_clocks[i].wm_max_socclk_clk_in_khz =
 				ranges->writer_wm_sets[i].max_fill_clk_mhz * 1000;
 		wm_soc_clocks[i].wm_min_socclk_clk_in_khz =
@@ -452,52 +465,69 @@ static void pp_rv_set_wm_ranges(struct pp_smu *pp,
 		wm_soc_clocks[i].wm_min_mem_clk_in_khz =
 				ranges->writer_wm_sets[i].min_drain_clk_mhz * 1000;
 	}
+}
+EXPORT_IF_KUNIT(build_wm_clock_ranges_soc15);
+
+STATIC_IFN_KUNIT void pp_rv_set_wm_ranges(struct pp_smu *pp,
+		struct pp_smu_wm_range_sets *ranges)
+{
+	const struct dc_context *ctx = pp->dm;
+	struct amdgpu_device *adev = ctx->driver_context;
+	struct dm_pp_wm_sets_with_clock_ranges_soc15 wm_with_clock_ranges;
+
+	build_wm_clock_ranges_soc15(ranges, &wm_with_clock_ranges);
 
 	amdgpu_dpm_set_watermarks_for_clocks_ranges(adev,
 						    &wm_with_clock_ranges);
 }
+EXPORT_IF_KUNIT(pp_rv_set_wm_ranges);
 
-static void pp_rv_set_pme_wa_enable(struct pp_smu *pp)
+STATIC_IFN_KUNIT void pp_rv_set_pme_wa_enable(struct pp_smu *pp)
 {
 	const struct dc_context *ctx = pp->dm;
 	struct amdgpu_device *adev = ctx->driver_context;
 
 	amdgpu_dpm_notify_smu_enable_pwe(adev);
 }
+EXPORT_IF_KUNIT(pp_rv_set_pme_wa_enable);
 
-static void pp_rv_set_active_display_count(struct pp_smu *pp, int count)
+STATIC_IFN_KUNIT void pp_rv_set_active_display_count(struct pp_smu *pp, int count)
 {
 	const struct dc_context *ctx = pp->dm;
 	struct amdgpu_device *adev = ctx->driver_context;
 
 	amdgpu_dpm_set_active_display_count(adev, count);
 }
+EXPORT_IF_KUNIT(pp_rv_set_active_display_count);
 
-static void pp_rv_set_min_deep_sleep_dcfclk(struct pp_smu *pp, int clock)
+STATIC_IFN_KUNIT void pp_rv_set_min_deep_sleep_dcfclk(struct pp_smu *pp, int clock)
 {
 	const struct dc_context *ctx = pp->dm;
 	struct amdgpu_device *adev = ctx->driver_context;
 
 	amdgpu_dpm_set_min_deep_sleep_dcefclk(adev, clock);
 }
+EXPORT_IF_KUNIT(pp_rv_set_min_deep_sleep_dcfclk);
 
-static void pp_rv_set_hard_min_dcefclk_by_freq(struct pp_smu *pp, int clock)
+STATIC_IFN_KUNIT void pp_rv_set_hard_min_dcefclk_by_freq(struct pp_smu *pp, int clock)
 {
 	const struct dc_context *ctx = pp->dm;
 	struct amdgpu_device *adev = ctx->driver_context;
 
 	amdgpu_dpm_set_hard_min_dcefclk_by_freq(adev, clock);
 }
+EXPORT_IF_KUNIT(pp_rv_set_hard_min_dcefclk_by_freq);
 
-static void pp_rv_set_hard_min_fclk_by_freq(struct pp_smu *pp, int mhz)
+STATIC_IFN_KUNIT void pp_rv_set_hard_min_fclk_by_freq(struct pp_smu *pp, int mhz)
 {
 	const struct dc_context *ctx = pp->dm;
 	struct amdgpu_device *adev = ctx->driver_context;
 
 	amdgpu_dpm_set_hard_min_fclk_by_freq(adev, mhz);
 }
+EXPORT_IF_KUNIT(pp_rv_set_hard_min_fclk_by_freq);
 
-static enum pp_smu_status pp_nv_set_wm_ranges(struct pp_smu *pp,
+STATIC_IFN_KUNIT enum pp_smu_status pp_nv_set_wm_ranges(struct pp_smu *pp,
 		struct pp_smu_wm_range_sets *ranges)
 {
 	const struct dc_context *ctx = pp->dm;
@@ -507,8 +537,9 @@ static enum pp_smu_status pp_nv_set_wm_ranges(struct pp_smu *pp,
 
 	return PP_SMU_RESULT_OK;
 }
+EXPORT_IF_KUNIT(pp_nv_set_wm_ranges);
 
-static enum pp_smu_status pp_nv_set_display_count(struct pp_smu *pp, int count)
+STATIC_IFN_KUNIT enum pp_smu_status pp_nv_set_display_count(struct pp_smu *pp, int count)
 {
 	const struct dc_context *ctx = pp->dm;
 	struct amdgpu_device *adev = ctx->driver_context;
@@ -523,8 +554,9 @@ static enum pp_smu_status pp_nv_set_display_count(struct pp_smu *pp, int count)
 
 	return PP_SMU_RESULT_OK;
 }
+EXPORT_IF_KUNIT(pp_nv_set_display_count);
 
-static enum pp_smu_status
+STATIC_IFN_KUNIT enum pp_smu_status
 pp_nv_set_min_deep_sleep_dcfclk(struct pp_smu *pp, int mhz)
 {
 	const struct dc_context *ctx = pp->dm;
@@ -540,8 +572,9 @@ pp_nv_set_min_deep_sleep_dcfclk(struct pp_smu *pp, int mhz)
 
 	return PP_SMU_RESULT_OK;
 }
+EXPORT_IF_KUNIT(pp_nv_set_min_deep_sleep_dcfclk);
 
-static enum pp_smu_status pp_nv_set_hard_min_dcefclk_by_freq(
+STATIC_IFN_KUNIT enum pp_smu_status pp_nv_set_hard_min_dcefclk_by_freq(
 		struct pp_smu *pp, int mhz)
 {
 	const struct dc_context *ctx = pp->dm;
@@ -563,8 +596,9 @@ static enum pp_smu_status pp_nv_set_hard_min_dcefclk_by_freq(
 
 	return PP_SMU_RESULT_OK;
 }
+EXPORT_IF_KUNIT(pp_nv_set_hard_min_dcefclk_by_freq);
 
-static enum pp_smu_status
+STATIC_IFN_KUNIT enum pp_smu_status
 pp_nv_set_hard_min_uclk_by_freq(struct pp_smu *pp, int mhz)
 {
 	const struct dc_context *ctx = pp->dm;
@@ -586,8 +620,9 @@ pp_nv_set_hard_min_uclk_by_freq(struct pp_smu *pp, int mhz)
 
 	return PP_SMU_RESULT_OK;
 }
+EXPORT_IF_KUNIT(pp_nv_set_hard_min_uclk_by_freq);
 
-static enum pp_smu_status pp_nv_set_pstate_handshake_support(
+STATIC_IFN_KUNIT enum pp_smu_status pp_nv_set_pstate_handshake_support(
 	struct pp_smu *pp, bool pstate_handshake_supported)
 {
 	const struct dc_context *ctx = pp->dm;
@@ -599,28 +634,40 @@ static enum pp_smu_status pp_nv_set_pstate_handshake_support(
 
 	return PP_SMU_RESULT_OK;
 }
+EXPORT_IF_KUNIT(pp_nv_set_pstate_handshake_support);
 
-static enum pp_smu_status pp_nv_set_voltage_by_freq(struct pp_smu *pp,
-		enum pp_smu_nv_clock_id clock_id, int mhz)
+STATIC_IFN_KUNIT bool pp_smu_nv_clock_id_to_pp(enum pp_smu_nv_clock_id clock_id,
+		enum amd_pp_clock_type *clock_type)
 {
-	const struct dc_context *ctx = pp->dm;
-	struct amdgpu_device *adev = ctx->driver_context;
-	struct pp_display_clock_request clock_req;
-	int ret = 0;
-
 	switch (clock_id) {
 	case PP_SMU_NV_DISPCLK:
-		clock_req.clock_type = amd_pp_disp_clock;
+		*clock_type = amd_pp_disp_clock;
 		break;
 	case PP_SMU_NV_PHYCLK:
-		clock_req.clock_type = amd_pp_phy_clock;
+		*clock_type = amd_pp_phy_clock;
 		break;
 	case PP_SMU_NV_PIXELCLK:
-		clock_req.clock_type = amd_pp_pixel_clock;
+		*clock_type = amd_pp_pixel_clock;
 		break;
 	default:
-		break;
+		return false;
 	}
+
+	return true;
+}
+EXPORT_IF_KUNIT(pp_smu_nv_clock_id_to_pp);
+
+STATIC_IFN_KUNIT enum pp_smu_status pp_nv_set_voltage_by_freq(struct pp_smu *pp,
+		enum pp_smu_nv_clock_id clock_id, int mhz)
+{
+	const struct dc_context *ctx = pp->dm;
+	struct amdgpu_device *adev = ctx->driver_context;
+	struct pp_display_clock_request clock_req;
+	int ret = 0;
+
+	if (!pp_smu_nv_clock_id_to_pp(clock_id, &clock_req.clock_type))
+		return PP_SMU_RESULT_FAIL;
+
 	clock_req.clock_freq_in_khz = mhz * 1000;
 
 	/* 0: successful or smu.ppt_funcs->display_clock_voltage_request = NULL
@@ -634,8 +681,9 @@ static enum pp_smu_status pp_nv_set_voltage_by_freq(struct pp_smu *pp,
 
 	return PP_SMU_RESULT_OK;
 }
+EXPORT_IF_KUNIT(pp_nv_set_voltage_by_freq);
 
-static enum pp_smu_status pp_nv_get_maximum_sustainable_clocks(
+STATIC_IFN_KUNIT enum pp_smu_status pp_nv_get_maximum_sustainable_clocks(
 		struct pp_smu *pp, struct pp_smu_nv_clock_table *max_clocks)
 {
 	const struct dc_context *ctx = pp->dm;
@@ -651,8 +699,9 @@ static enum pp_smu_status pp_nv_get_maximum_sustainable_clocks(
 
 	return PP_SMU_RESULT_OK;
 }
+EXPORT_IF_KUNIT(pp_nv_get_maximum_sustainable_clocks);
 
-static enum pp_smu_status pp_nv_get_uclk_dpm_states(struct pp_smu *pp,
+STATIC_IFN_KUNIT enum pp_smu_status pp_nv_get_uclk_dpm_states(struct pp_smu *pp,
 		unsigned int *clock_values_in_khz, unsigned int *num_states)
 {
 	const struct dc_context *ctx = pp->dm;
@@ -669,8 +718,9 @@ static enum pp_smu_status pp_nv_get_uclk_dpm_states(struct pp_smu *pp,
 
 	return PP_SMU_RESULT_OK;
 }
+EXPORT_IF_KUNIT(pp_nv_get_uclk_dpm_states);
 
-static enum pp_smu_status pp_rn_get_dpm_clock_table(
+STATIC_IFN_KUNIT enum pp_smu_status pp_rn_get_dpm_clock_table(
 		struct pp_smu *pp, struct dpm_clocks *clock_table)
 {
 	const struct dc_context *ctx = pp->dm;
@@ -685,17 +735,7 @@ static enum pp_smu_status pp_rn_get_dpm_clock_table(
 
 	return PP_SMU_RESULT_OK;
 }
-
-static enum pp_smu_status pp_rn_set_wm_ranges(struct pp_smu *pp,
-		struct pp_smu_wm_range_sets *ranges)
-{
-	const struct dc_context *ctx = pp->dm;
-	struct amdgpu_device *adev = ctx->driver_context;
-
-	amdgpu_dpm_set_watermarks_for_clocks_ranges(adev, ranges);
-
-	return PP_SMU_RESULT_OK;
-}
+EXPORT_IF_KUNIT(pp_rn_get_dpm_clock_table);
 
 void dm_pp_get_funcs(
 		struct dc_context *ctx,
@@ -743,7 +783,7 @@ void dm_pp_get_funcs(
 	case DCN_VERSION_2_1:
 		funcs->ctx.ver = PP_SMU_VER_RN;
 		funcs->rn_funcs.pp_smu.dm = ctx;
-		funcs->rn_funcs.set_wm_ranges = pp_rn_set_wm_ranges;
+		funcs->rn_funcs.set_wm_ranges = pp_nv_set_wm_ranges;
 		funcs->rn_funcs.get_dpm_clock_table = pp_rn_get_dpm_clock_table;
 		break;
 	default:
@@ -751,3 +791,4 @@ void dm_pp_get_funcs(
 		break;
 	}
 }
+EXPORT_IF_KUNIT(dm_pp_get_funcs);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.h
new file mode 100644
index 0000000000000..f918eb71f0d1d
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.h
@@ -0,0 +1,61 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#ifndef __AMDGPU_DM_PP_SMU_H__
+#define __AMDGPU_DM_PP_SMU_H__
+
+#include "dm_pp_interface.h"
+
+struct amd_pp_display_configuration;
+struct pp_smu_wm_range_sets;
+struct dm_pp_wm_sets_with_clock_ranges_soc15;
+
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+void build_pm_display_cfg(struct amd_pp_display_configuration *pm_display_cfg,
+			  const struct dm_pp_display_configuration *pp_display_cfg);
+void build_wm_clock_ranges_soc15(const struct pp_smu_wm_range_sets *ranges,
+		struct dm_pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges);
+void get_default_clock_levels(enum dm_pp_clock_type clk_type, struct dm_pp_clock_levels *clks);
+enum amd_pp_clock_type dc_to_pp_clock_type(enum dm_pp_clock_type dm_pp_clk_type);
+void pp_to_dc_clock_levels(const struct amd_pp_clocks *pp_clks,
+			   struct dm_pp_clock_levels *dc_clks,
+			   enum dm_pp_clock_type dc_clk_type);
+void pp_to_dc_clock_levels_with_latency(const struct pp_clock_levels_with_latency *pp_clks,
+					struct dm_pp_clock_levels_with_latency *clk_level_info,
+					enum dm_pp_clock_type dc_clk_type);
+void pp_to_dc_clock_levels_with_voltage(const struct pp_clock_levels_with_voltage *pp_clks,
+					struct dm_pp_clock_levels_with_voltage *clk_level_info,
+					enum dm_pp_clock_type dc_clk_type);
+void cap_clock_levels_to_validation(struct dm_pp_clock_levels *dc_clks,
+				    enum dm_pp_clock_type clk_type,
+				    const struct amd_pp_simple_clock_info *validation_clks);
+bool pp_smu_nv_clock_id_to_pp(enum pp_smu_nv_clock_id clock_id,
+			      enum amd_pp_clock_type *clock_type);
+void pp_rv_set_wm_ranges(struct pp_smu *pp, struct pp_smu_wm_range_sets *ranges);
+void pp_rv_set_pme_wa_enable(struct pp_smu *pp);
+void pp_rv_set_active_display_count(struct pp_smu *pp, int count);
+void pp_rv_set_min_deep_sleep_dcfclk(struct pp_smu *pp, int clock);
+void pp_rv_set_hard_min_dcefclk_by_freq(struct pp_smu *pp, int clock);
+void pp_rv_set_hard_min_fclk_by_freq(struct pp_smu *pp, int mhz);
+enum pp_smu_status pp_nv_set_wm_ranges(struct pp_smu *pp,
+					struct pp_smu_wm_range_sets *ranges);
+enum pp_smu_status pp_nv_set_display_count(struct pp_smu *pp, int count);
+enum pp_smu_status pp_nv_set_min_deep_sleep_dcfclk(struct pp_smu *pp, int mhz);
+enum pp_smu_status pp_nv_set_hard_min_dcefclk_by_freq(struct pp_smu *pp, int mhz);
+enum pp_smu_status pp_nv_set_hard_min_uclk_by_freq(struct pp_smu *pp, int mhz);
+enum pp_smu_status pp_nv_set_pstate_handshake_support(struct pp_smu *pp,
+						      bool pstate_handshake_supported);
+enum pp_smu_status pp_nv_set_voltage_by_freq(struct pp_smu *pp,
+					     enum pp_smu_nv_clock_id clock_id, int mhz);
+enum pp_smu_status pp_nv_get_maximum_sustainable_clocks(struct pp_smu *pp,
+							struct pp_smu_nv_clock_table *max_clocks);
+enum pp_smu_status pp_nv_get_uclk_dpm_states(struct pp_smu *pp,
+					     unsigned int *clock_values_in_khz,
+					     unsigned int *num_states);
+enum pp_smu_status pp_rn_get_dpm_clock_table(struct pp_smu *pp,
+					     struct dpm_clocks *clock_table);
+#endif
+
+#endif /* __AMDGPU_DM_PP_SMU_H__ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c
index 0dadc0bb214ff..f87de3d18ac0e 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c
@@ -32,8 +32,8 @@
 #include "modules/power/power_helpers.h"
 #include "amdgpu_dm_kunit_helpers.h"
 
-
-static bool link_supports_psrsu(struct dc_link *link)
+STATIC_IFN_KUNIT
+bool link_supports_psrsu(struct dc_link *link)
 {
 	struct dc *dc = link->ctx->dc;
 
@@ -60,6 +60,7 @@ static bool link_supports_psrsu(struct dc_link *link)
 	/* Temporarily disable PSR-SU to avoid glitches */
 	return false;
 }
+EXPORT_IF_KUNIT(link_supports_psrsu);
 
 STATIC_IFN_KUNIT
 void amdgpu_dm_psr_fill_caps(struct dc_link *link, struct psr_caps *caps)
@@ -134,6 +135,7 @@ bool amdgpu_dm_set_psr_caps(struct dc_link *link, struct amdgpu_dm_connector *ac
 	amdgpu_dm_psr_fill_caps(link, &aconnector->psr_caps);
 	return true;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_set_psr_caps);
 
 /*
  * amdgpu_dm_psr_is_active_allowed() - check if psr is allowed on any stream
@@ -157,6 +159,7 @@ bool amdgpu_dm_psr_is_active_allowed(struct amdgpu_display_manager *dm)
 	}
 	return false;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_psr_is_active_allowed);
 
 /*
  * amdgpu_dm_psr_set_event() - set or clear PSR event for stream
@@ -190,3 +193,47 @@ bool amdgpu_dm_psr_set_event(struct amdgpu_display_manager *dm, struct dc_stream
 				       set_event, event, wait_for_disable);
 }
 EXPORT_IF_KUNIT(amdgpu_dm_psr_set_event);
+
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+/**
+ * amdgpu_dm_psr_get_dc_feature_mask() - Get DC feature mask for KUnit tests.
+ *
+ * Return: Current value of amdgpu_dc_feature_mask.
+ */
+unsigned int amdgpu_dm_psr_get_dc_feature_mask(void)
+{
+	return amdgpu_dc_feature_mask;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_psr_get_dc_feature_mask);
+
+/**
+ * amdgpu_dm_psr_set_dc_feature_mask() - Set DC feature mask for KUnit tests.
+ * @feature_mask: DC feature mask to set while testing amdgpu_dm_psr_fill_caps().
+ */
+void amdgpu_dm_psr_set_dc_feature_mask(unsigned int feature_mask)
+{
+	amdgpu_dc_feature_mask = feature_mask;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_psr_set_dc_feature_mask);
+
+/**
+ * amdgpu_dm_psr_get_dc_debug_mask() - Get DC debug mask for KUnit tests.
+ *
+ * Return: Current value of amdgpu_dc_debug_mask.
+ */
+unsigned int amdgpu_dm_psr_get_dc_debug_mask(void)
+{
+	return amdgpu_dc_debug_mask;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_psr_get_dc_debug_mask);
+
+/**
+ * amdgpu_dm_psr_set_dc_debug_mask() - Set DC debug mask for KUnit tests.
+ * @debug_mask: DC debug mask to set while testing link_supports_psrsu().
+ */
+void amdgpu_dm_psr_set_dc_debug_mask(unsigned int debug_mask)
+{
+	amdgpu_dc_debug_mask = debug_mask;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_psr_set_dc_debug_mask);
+#endif
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.h
index 40a09b5dc606f..e442e7ed82ec1 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.h
@@ -43,7 +43,12 @@ bool amdgpu_dm_psr_set_event(struct amdgpu_display_manager *dm,
 		bool wait_for_disable);
 
 #if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+bool link_supports_psrsu(struct dc_link *link);
 void amdgpu_dm_psr_fill_caps(struct dc_link *link, struct psr_caps *caps);
+unsigned int amdgpu_dm_psr_get_dc_feature_mask(void);
+void amdgpu_dm_psr_set_dc_feature_mask(unsigned int feature_mask);
+unsigned int amdgpu_dm_psr_get_dc_debug_mask(void);
+void amdgpu_dm_psr_set_dc_debug_mask(unsigned int debug_mask);
 #endif
 
 #endif /* AMDGPU_DM_AMDGPU_DM_PSR_H_ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_quirks.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_quirks.c
index 1da07ebf9217c..cf28d50c3b5ee 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_quirks.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_quirks.c
@@ -28,6 +28,7 @@
 
 #include "amdgpu.h"
 #include "amdgpu_dm.h"
+#include "amdgpu_dm_kunit_helpers.h"
 
 struct amdgpu_dm_quirks {
 	bool aux_hpd_discon;
@@ -176,3 +177,4 @@ void retrieve_dmi_info(struct amdgpu_display_manager *dm)
 		drm_info(dev, "support_edp0_on_dp1 attached\n");
 	}
 }
+EXPORT_IF_KUNIT(retrieve_dmi_info);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_replay.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_replay.c
index 22aa4305d2af9..42e17119461dc 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_replay.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_replay.c
@@ -121,14 +121,14 @@ bool amdgpu_dm_set_replay_caps(struct dc_link *link, struct amdgpu_dm_connector
 
 	debug_flags = (union replay_debug_flags *)&pr_config.debug_flags;
 	debug_flags->u32All = 0;
-	debug_flags->bitfields.visual_confirm =
-		link->ctx->dc->debug.visual_confirm == VISUAL_CONFIRM_REPLAY;
+	debug_flags->bitfields.visual_confirm =	dc->debug.visual_confirm == VISUAL_CONFIRM_REPLAY;
 	debug_flags->bitfields.skip_crtc_disabled = dc->debug.replay_skip_crtc_disabled;
 
 	init_replay_config(link, &pr_config);
 
 	return true;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_set_replay_caps);
 
 /*
  * amdgpu_dm_link_setup_replay() - config replay settings
@@ -144,7 +144,6 @@ bool amdgpu_dm_link_setup_replay(struct dc_stream_state *stream,
 {
 	struct dc_link *link;
 	unsigned int static_coasting_vtotal;
-	unsigned int nom_coasting_vtotal;
 
 	if (!stream || !stream->link || !vrr_params)
 		return false;
@@ -159,16 +158,16 @@ bool amdgpu_dm_link_setup_replay(struct dc_stream_state *stream,
 	calculate_replay_link_off_frame_count(link, stream->timing.v_total,
 			stream->timing.h_total);
 
-	nom_coasting_vtotal = stream->timing.v_total;
 	static_coasting_vtotal = mod_freesync_calc_v_total_from_refresh(stream,
 			vrr_params->min_refresh_in_uhz);
 
 	set_replay_coasting_vtotal(link, PR_COASTING_TYPE_NOM,
-			nom_coasting_vtotal);
+			stream->timing.v_total);
 	set_replay_coasting_vtotal(link, PR_COASTING_TYPE_STATIC,
 			static_coasting_vtotal);
 	return true;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_link_setup_replay);
 
 /*
  * amdgpu_dm_replay_set_event() - set or clear replay event for a stream
@@ -208,3 +207,4 @@ bool amdgpu_dm_replay_set_event(struct amdgpu_display_manager *dm,
 	return mod_power_set_replay_event(dm->power_module, stream,
 					 set_event, event, wait_for_disable);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_replay_set_event);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
index 84dcb573d98f5..6c0464754ed8c 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
@@ -32,9 +32,11 @@
 #include "dm_services.h"
 #include "amdgpu.h"
 #include "amdgpu_dm.h"
+#include "amdgpu_dm_backlight.h"
 #include "amdgpu_dm_irq.h"
 #include "amdgpu_pm.h"
 #include "amdgpu_dm_trace.h"
+#include "amdgpu_dm_kunit_helpers.h"
 
 	unsigned long long
 	dm_get_elapse_time_in_ns(struct dc_context *ctx,
@@ -43,6 +45,7 @@
 {
 	return current_time_stamp - last_time_stamp;
 }
+EXPORT_IF_KUNIT(dm_get_elapse_time_in_ns);
 
 void dm_perf_trace_timestamp(const char *func_name, unsigned int line, struct dc_context *ctx)
 {
@@ -52,14 +55,17 @@ void dm_perf_trace_timestamp(const char *func_name, unsigned int line, struct dc
 				    &ctx->perf_trace->last_entry_write,
 				    func_name, line);
 }
+EXPORT_IF_KUNIT(dm_perf_trace_timestamp);
 
 void dm_trace_smu_enter(uint32_t msg_id, uint32_t param_in, unsigned int delay, struct dc_context *ctx)
 {
 }
+EXPORT_IF_KUNIT(dm_trace_smu_enter);
 
 void dm_trace_smu_exit(bool success, uint32_t response, struct dc_context *ctx)
 {
 }
+EXPORT_IF_KUNIT(dm_trace_smu_exit);
 
 /**** power component interfaces ****/
 
@@ -89,3 +95,4 @@ bool dm_query_extended_brightness_caps(struct dc_context *ctx,
 			sizeof(struct dm_bl_data_point) * pCaps->num_data_points);
 	return true;
 }
+EXPORT_IF_KUNIT(dm_query_extended_brightness_caps);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.c
index 110f0173eee67..9e7bad4d6ed0c 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.c
@@ -29,6 +29,7 @@
 #include "amdgpu.h"
 #include "amdgpu_dm.h"
 #include "amdgpu_dm_wb.h"
+#include "amdgpu_dm_kunit_helpers.h"
 #include "amdgpu_display.h"
 #include "dc.h"
 
@@ -40,7 +41,7 @@ static const u32 amdgpu_dm_wb_formats[] = {
 	DRM_FORMAT_XRGB2101010,
 };
 
-static int amdgpu_dm_wb_encoder_atomic_check(struct drm_encoder *encoder,
+STATIC_IFN_KUNIT int amdgpu_dm_wb_encoder_atomic_check(struct drm_encoder *encoder,
 					struct drm_crtc_state *crtc_state,
 					struct drm_connector_state *conn_state)
 {
@@ -59,9 +60,11 @@ static int amdgpu_dm_wb_encoder_atomic_check(struct drm_encoder *encoder,
 		return -EINVAL;
 	}
 
-	for (i = 0; i < sizeof(amdgpu_dm_wb_formats) / sizeof(u32); i++) {
-		if (fb->format->format == amdgpu_dm_wb_formats[i])
+	for (i = 0; i < ARRAY_SIZE(amdgpu_dm_wb_formats); i++) {
+		if (fb->format->format == amdgpu_dm_wb_formats[i]) {
 			found = true;
+			break;
+		}
 	}
 
 	if (!found) {
@@ -72,15 +75,17 @@ static int amdgpu_dm_wb_encoder_atomic_check(struct drm_encoder *encoder,
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_wb_encoder_atomic_check);
 
 
-static int amdgpu_dm_wb_connector_get_modes(struct drm_connector *connector)
+STATIC_IFN_KUNIT int amdgpu_dm_wb_connector_get_modes(struct drm_connector *connector)
 {
 	/* Maximum resolution supported by DWB */
 	return drm_add_modes_noedid(connector, 3840, 2160);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_wb_connector_get_modes);
 
-static int amdgpu_dm_wb_prepare_job(struct drm_writeback_connector *wb_connector,
+STATIC_IFN_KUNIT int amdgpu_dm_wb_prepare_job(struct drm_writeback_connector *wb_connector,
 			       struct drm_writeback_job *job)
 {
 	struct amdgpu_framebuffer *afb;
@@ -102,13 +107,15 @@ static int amdgpu_dm_wb_prepare_job(struct drm_writeback_connector *wb_connector
 
 	r = amdgpu_bo_reserve(rbo, true);
 	if (r) {
-		drm_err(adev_to_drm(adev), "fail to reserve bo (%d)\n", r);
+		drm_err(adev_to_drm(adev), "fail to reserve bo: %pe\n", ERR_PTR(r));
 		return r;
 	}
 
 	r = dma_resv_reserve_fences(rbo->tbo.base.resv, TTM_NUM_MOVE_FENCES);
-	if (r)
+	if (r) {
+		drm_err(adev_to_drm(adev), "reserving fence slot failed: %pe\n", ERR_PTR(r));
 		goto error_unlock;
+	}
 
 	domain = amdgpu_display_supported_domains(adev, rbo->flags);
 
@@ -116,13 +123,13 @@ static int amdgpu_dm_wb_prepare_job(struct drm_writeback_connector *wb_connector
 	r = amdgpu_bo_pin(rbo, domain);
 	if (unlikely(r != 0)) {
 		if (r != -ERESTARTSYS)
-			DRM_ERROR("Failed to pin framebuffer with error %d\n", r);
+			DRM_ERROR("Failed to pin framebuffer: %pe\n", ERR_PTR(r));
 		goto error_unlock;
 	}
 
 	r = amdgpu_ttm_alloc_gart(&rbo->tbo);
 	if (unlikely(r != 0)) {
-		DRM_ERROR("%p bind failed\n", rbo);
+		DRM_ERROR("%p bind failed: %pe\n", rbo, ERR_PTR(r));
 		goto error_unpin;
 	}
 
@@ -141,8 +148,9 @@ static int amdgpu_dm_wb_prepare_job(struct drm_writeback_connector *wb_connector
 	amdgpu_bo_unreserve(rbo);
 	return r;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_wb_prepare_job);
 
-static void amdgpu_dm_wb_cleanup_job(struct drm_writeback_connector *connector,
+STATIC_IFN_KUNIT void amdgpu_dm_wb_cleanup_job(struct drm_writeback_connector *connector,
 				struct drm_writeback_job *job)
 {
 	struct amdgpu_bo *rbo;
@@ -154,7 +162,7 @@ static void amdgpu_dm_wb_cleanup_job(struct drm_writeback_connector *connector,
 	rbo = gem_to_amdgpu_bo(job->fb->obj[0]);
 	r = amdgpu_bo_reserve(rbo, false);
 	if (unlikely(r)) {
-		DRM_ERROR("failed to reserve rbo before unpin\n");
+		DRM_ERROR("failed to reserve rbo before unpin: %pe\n", ERR_PTR(r));
 		return;
 	}
 
@@ -162,6 +170,7 @@ static void amdgpu_dm_wb_cleanup_job(struct drm_writeback_connector *connector,
 	amdgpu_bo_unreserve(rbo);
 	amdgpu_bo_unref(&rbo);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_wb_cleanup_job);
 
 static const struct drm_encoder_helper_funcs amdgpu_dm_wb_encoder_helper_funcs = {
 	.atomic_check = amdgpu_dm_wb_encoder_atomic_check,
@@ -187,7 +196,7 @@ int amdgpu_dm_wb_connector_init(struct amdgpu_display_manager *dm,
 {
 	struct dc *dc = dm->dc;
 	struct dc_link *link = dc_get_link_at_index(dc, link_index);
-	int res = 0;
+	int res;
 
 	wbcon->link = link;
 
@@ -211,3 +220,4 @@ int amdgpu_dm_wb_connector_init(struct amdgpu_display_manager *dm,
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_wb_connector_init);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.h
index 13d31c857dee4..5fd616bc43b5c 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.h
@@ -29,8 +29,25 @@
 
 #include <drm/drm_writeback.h>
 
+struct amdgpu_display_manager;
+struct amdgpu_dm_wb_connector;
+
 int amdgpu_dm_wb_connector_init(struct amdgpu_display_manager *dm,
 				struct amdgpu_dm_wb_connector *dm_wbcon,
 				uint32_t link_index);
 
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+#include <drm/drm_connector.h>
+#include <drm/drm_crtc.h>
+
+int amdgpu_dm_wb_encoder_atomic_check(struct drm_encoder *encoder,
+				      struct drm_crtc_state *crtc_state,
+				      struct drm_connector_state *conn_state);
+int amdgpu_dm_wb_connector_get_modes(struct drm_connector *connector);
+int amdgpu_dm_wb_prepare_job(struct drm_writeback_connector *wb_connector,
+			     struct drm_writeback_job *job);
+void amdgpu_dm_wb_cleanup_job(struct drm_writeback_connector *connector,
+			      struct drm_writeback_job *job);
+#endif
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/.kunitconfig b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/.kunitconfig
index bd1bf8d959f93..c7c8527dbb106 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/.kunitconfig
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/.kunitconfig
@@ -15,6 +15,12 @@ CONFIG_I2C=y
 CONFIG_POWER_SUPPLY=y
 CONFIG_CRC16=y
 
+# Limit stack size to 1280
+CONFIG_FRAME_WARN=1280
+
+# Treat warnings as errors
+CONFIG_WERROR=y
+
 # GCOV Coverage - see tools/testing/kunit/configs/coverage_uml.config
 CONFIG_DEBUG_KERNEL=y
 CONFIG_DEBUG_INFO=y
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/Makefile b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/Makefile
index 768f9bbc50e14..1592e8dae1a90 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/Makefile
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/Makefile
@@ -8,11 +8,29 @@ ccflags-y += -I$(src)/../../include
 ccflags-y += -I$(src)/../../modules/inc
 ccflags-y += -I$(src)/../../dc
 ccflags-y += -I$(src)/../../../amdgpu
+ccflags-y += -I$(src)/../../../amdkfd
+ccflags-y += -I$(src)/../../../include
 
+obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_kunit_helpers.o
 obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_crc_test.o
 obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_hdcp_test.o
+obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_audio_test.o
 obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_color_test.o
 obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_colorop_test.o
+obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_connector_test.o
+obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_backlight_test.o
+obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_dmub_test.o
 obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_psr_test.o
 obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_replay_test.o
 obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_ism_test.o
+obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_irq_test.o
+obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_wb_test.o
+obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_plane_test.o
+obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_mst_types_test.o
+obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_pp_smu_test.o
+obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_test.o
+obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_crtc_test.o
+obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_services_test.o
+obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_helpers_test.o
+obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_quirks_test.o
+obj-$(CONFIG_DRM_AMD_DC_KUNIT_TEST) += amdgpu_dm_plane_test.o
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_audio_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_audio_test.c
new file mode 100644
index 0000000000000..3f0108e9a951d
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_audio_test.c
@@ -0,0 +1,608 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * KUnit tests for amdgpu_dm_audio.c
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#include <kunit/test.h>
+#include <linux/device.h>
+
+#include <drm/drm_audio_component.h>
+
+#include "dc.h"
+#include "dc/inc/core_types.h"
+#include "dc/inc/hw/audio.h"
+#include "amdgpu.h"
+#include "amdgpu_mode.h"
+#include "amdgpu_dm.h"
+#include "amdgpu_dm_audio.h"
+
+/* Tests for amdgpu_dm_audio_init() */
+
+/**
+ * dm_test_audio_init_disabled - Test audio init exits when audio is disabled
+ * @test: The KUnit test context
+ */
+static void dm_test_audio_init_disabled(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	int saved_audio = amdgpu_dm_audio_get_param();
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	amdgpu_dm_audio_set_param(0);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_audio_init(adev), 0);
+	KUNIT_EXPECT_FALSE(test, adev->mode_info.audio.enabled);
+	KUNIT_EXPECT_FALSE(test, adev->dm.audio_registered);
+
+	amdgpu_dm_audio_set_param(saved_audio);
+}
+
+/**
+ * dm_test_audio_init_enabled_success - Test init deeper path when audio is enabled
+ * @test: The KUnit test context
+ */
+static void dm_test_audio_init_enabled_success(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct resource_pool *res_pool;
+	struct audio *audio0;
+	struct audio *audio1;
+	struct device *dev;
+	int saved_audio = amdgpu_dm_audio_get_param();
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	res_pool = kunit_kzalloc(test, sizeof(*res_pool), GFP_KERNEL);
+	audio0 = kunit_kzalloc(test, sizeof(*audio0), GFP_KERNEL);
+	audio1 = kunit_kzalloc(test, sizeof(*audio1), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, res_pool);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, audio0);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, audio1);
+
+	dev = root_device_register("kunit-dm-audio-init");
+	KUNIT_ASSERT_FALSE(test, IS_ERR(dev));
+
+	audio0->inst = 2;
+	audio1->inst = 6;
+	res_pool->audio_count = 2;
+	res_pool->audios[0] = audio0;
+	res_pool->audios[1] = audio1;
+	dc->res_pool = res_pool;
+	adev->dm.dc = dc;
+	adev->dev = dev;
+
+	amdgpu_dm_audio_set_param(1);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_audio_init(adev), 0);
+	KUNIT_EXPECT_TRUE(test, adev->mode_info.audio.enabled);
+	KUNIT_EXPECT_TRUE(test, adev->dm.audio_registered);
+	KUNIT_EXPECT_EQ(test, adev->mode_info.audio.num_pins, 2);
+	KUNIT_EXPECT_EQ(test, adev->mode_info.audio.pin[0].id, 2U);
+	KUNIT_EXPECT_EQ(test, adev->mode_info.audio.pin[1].id, 6U);
+
+	amdgpu_dm_audio_fini(adev);
+	root_device_unregister(dev);
+	amdgpu_dm_audio_set_param(saved_audio);
+}
+
+/* Tests for amdgpu_dm_audio_fini() */
+
+/**
+ * dm_test_audio_fini_without_enabled_audio - Test fini exits when audio is not enabled
+ * @test: The KUnit test context
+ */
+static void dm_test_audio_fini_without_enabled_audio(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	int saved_audio = amdgpu_dm_audio_get_param();
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	amdgpu_dm_audio_set_param(1);
+	adev->mode_info.audio.enabled = false;
+	adev->dm.audio_registered = true;
+
+	amdgpu_dm_audio_fini(adev);
+
+	KUNIT_EXPECT_FALSE(test, adev->mode_info.audio.enabled);
+	KUNIT_EXPECT_TRUE(test, adev->dm.audio_registered);
+
+	amdgpu_dm_audio_set_param(saved_audio);
+}
+
+/* Tests for amdgpu_dm_fill_audio_info() */
+
+/**
+ * dm_test_fill_audio_info_ids_name_flags - Test Fill audio info ids name flags
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_audio_info_ids_name_flags(struct kunit *test)
+{
+	struct audio_info *audio_info;
+	struct drm_connector *connector;
+	struct dc_sink *dc_sink;
+	const char *name = "DM-AUDIO-PANEL";
+
+	audio_info = kunit_kzalloc(test, sizeof(*audio_info), GFP_KERNEL);
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	dc_sink = kunit_kzalloc(test, sizeof(*dc_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, audio_info);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc_sink);
+
+	dc_sink->edid_caps.manufacturer_id = 0x1234;
+	dc_sink->edid_caps.product_id = 0xABCD;
+	dc_sink->edid_caps.speaker_flags = 0x5;
+	strscpy(dc_sink->edid_caps.display_name, name,
+		AUDIO_INFO_DISPLAY_NAME_SIZE_IN_CHARS);
+
+	connector->display_info.cea_rev = 1;
+
+	amdgpu_dm_fill_audio_info(audio_info, connector, dc_sink);
+
+	KUNIT_EXPECT_EQ(test, audio_info->manufacture_id, 0x1234U);
+	KUNIT_EXPECT_EQ(test, audio_info->product_id, 0xABCDU);
+	KUNIT_EXPECT_EQ(test, audio_info->flags.all, 0x5U);
+	KUNIT_EXPECT_STREQ(test, audio_info->display_name, name);
+}
+
+/**
+ * dm_test_fill_audio_info_cea_lt_3_skips_modes - Test Fill audio info cea lt 3 skips modes
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_audio_info_cea_lt_3_skips_modes(struct kunit *test)
+{
+	struct audio_info *audio_info;
+	struct drm_connector *connector;
+	struct dc_sink *dc_sink;
+
+	audio_info = kunit_kzalloc(test, sizeof(*audio_info), GFP_KERNEL);
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	dc_sink = kunit_kzalloc(test, sizeof(*dc_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, audio_info);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc_sink);
+
+	connector->display_info.cea_rev = 2;
+	dc_sink->edid_caps.audio_mode_count = 2;
+	dc_sink->edid_caps.audio_modes[0].format_code = 1;
+	dc_sink->edid_caps.audio_modes[0].channel_count = 2;
+	dc_sink->edid_caps.audio_modes[0].sample_rate = 0x07;
+	dc_sink->edid_caps.audio_modes[0].sample_size = 16;
+
+	amdgpu_dm_fill_audio_info(audio_info, connector, dc_sink);
+
+	KUNIT_EXPECT_EQ(test, audio_info->mode_count, 0U);
+}
+
+/**
+ * dm_test_fill_audio_info_cea_ge_3_copies_modes - Test Fill audio info cea ge 3 copies modes
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_audio_info_cea_ge_3_copies_modes(struct kunit *test)
+{
+	struct audio_info *audio_info;
+	struct drm_connector *connector;
+	struct dc_sink *dc_sink;
+
+	audio_info = kunit_kzalloc(test, sizeof(*audio_info), GFP_KERNEL);
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	dc_sink = kunit_kzalloc(test, sizeof(*dc_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, audio_info);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc_sink);
+
+	connector->display_info.cea_rev = 3;
+	dc_sink->edid_caps.audio_mode_count = 2;
+
+	dc_sink->edid_caps.audio_modes[0].format_code = 1;
+	dc_sink->edid_caps.audio_modes[0].channel_count = 2;
+	dc_sink->edid_caps.audio_modes[0].sample_rate = 0x07;
+	dc_sink->edid_caps.audio_modes[0].sample_size = 16;
+
+	dc_sink->edid_caps.audio_modes[1].format_code = 11;
+	dc_sink->edid_caps.audio_modes[1].channel_count = 6;
+	dc_sink->edid_caps.audio_modes[1].sample_rate = 0x1F;
+	dc_sink->edid_caps.audio_modes[1].sample_size = 24;
+
+	amdgpu_dm_fill_audio_info(audio_info, connector, dc_sink);
+
+	KUNIT_EXPECT_EQ(test, audio_info->mode_count, 2U);
+
+	KUNIT_EXPECT_EQ(test, (int)audio_info->modes[0].format_code, 1);
+	KUNIT_EXPECT_EQ(test, audio_info->modes[0].channel_count, 2);
+	KUNIT_EXPECT_EQ(test, audio_info->modes[0].sample_rates.all, 0x07U);
+	KUNIT_EXPECT_EQ(test, audio_info->modes[0].sample_size, 16);
+
+	KUNIT_EXPECT_EQ(test, (int)audio_info->modes[1].format_code, 11);
+	KUNIT_EXPECT_EQ(test, audio_info->modes[1].channel_count, 6);
+	KUNIT_EXPECT_EQ(test, audio_info->modes[1].sample_rates.all, 0x1FU);
+	KUNIT_EXPECT_EQ(test, audio_info->modes[1].sample_size, 24);
+}
+
+/**
+ * dm_test_fill_audio_info_latency_present - Test Fill audio info latency present
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_audio_info_latency_present(struct kunit *test)
+{
+	struct audio_info *audio_info;
+	struct drm_connector *connector;
+	struct dc_sink *dc_sink;
+
+	audio_info = kunit_kzalloc(test, sizeof(*audio_info), GFP_KERNEL);
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	dc_sink = kunit_kzalloc(test, sizeof(*dc_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, audio_info);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc_sink);
+
+	connector->display_info.cea_rev = 3;
+	connector->latency_present[0] = true;
+	connector->video_latency[0] = 11;
+	connector->audio_latency[0] = 22;
+
+	amdgpu_dm_fill_audio_info(audio_info, connector, dc_sink);
+
+	KUNIT_EXPECT_EQ(test, audio_info->video_latency, 11U);
+	KUNIT_EXPECT_EQ(test, audio_info->audio_latency, 22U);
+}
+
+/**
+ * dm_test_fill_audio_info_latency_absent_keeps_zero - Test Fill audio info latency absent keeps zero
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_audio_info_latency_absent_keeps_zero(struct kunit *test)
+{
+	struct audio_info *audio_info;
+	struct drm_connector *connector;
+	struct dc_sink *dc_sink;
+
+	audio_info = kunit_kzalloc(test, sizeof(*audio_info), GFP_KERNEL);
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	dc_sink = kunit_kzalloc(test, sizeof(*dc_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, audio_info);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc_sink);
+
+	connector->display_info.cea_rev = 3;
+	connector->latency_present[0] = false;
+	connector->video_latency[0] = 99;
+	connector->audio_latency[0] = 88;
+
+	amdgpu_dm_fill_audio_info(audio_info, connector, dc_sink);
+
+	KUNIT_EXPECT_EQ(test, audio_info->video_latency, 0U);
+	KUNIT_EXPECT_EQ(test, audio_info->audio_latency, 0U);
+}
+
+/**
+ * dm_test_fill_audio_info_cea_ge_3_zero_modes - Test cea >= 3 with zero modes
+ * @test: The KUnit test context
+ *
+ * When cea_rev >= 3 but the sink reports no audio modes, mode_count must be
+ * copied as 0 and no mode entries should be populated.
+ */
+static void dm_test_fill_audio_info_cea_ge_3_zero_modes(struct kunit *test)
+{
+	struct audio_info *audio_info;
+	struct drm_connector *connector;
+	struct dc_sink *dc_sink;
+
+	audio_info = kunit_kzalloc(test, sizeof(*audio_info), GFP_KERNEL);
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	dc_sink = kunit_kzalloc(test, sizeof(*dc_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, audio_info);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc_sink);
+
+	connector->display_info.cea_rev = 3;
+	dc_sink->edid_caps.audio_mode_count = 0;
+
+	amdgpu_dm_fill_audio_info(audio_info, connector, dc_sink);
+
+	KUNIT_EXPECT_EQ(test, audio_info->mode_count, 0U);
+	KUNIT_EXPECT_EQ(test, (int)audio_info->modes[0].format_code, 0);
+}
+
+/* Tests for amdgpu_dm_audio_component_bind()/unbind() */
+
+/**
+ * dm_test_audio_component_bind_sets_fields - Test bind wires up audio component
+ * @test: The KUnit test context
+ *
+ * Binding must publish the DRM audio component ops, record the kernel device,
+ * and store the component pointer in the display manager.
+ */
+static void dm_test_audio_component_bind_sets_fields(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct device *kdev;
+	struct drm_audio_component *acomp;
+	int ret;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	kdev = kunit_kzalloc(test, sizeof(*kdev), GFP_KERNEL);
+	acomp = kunit_kzalloc(test, sizeof(*acomp), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, kdev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acomp);
+
+	dev_set_drvdata(kdev, &adev->ddev);
+
+	ret = amdgpu_dm_audio_component_bind(kdev, NULL, acomp);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_NOT_NULL(test, acomp->ops);
+	KUNIT_EXPECT_PTR_EQ(test, acomp->dev, kdev);
+	KUNIT_EXPECT_PTR_EQ(test, adev->dm.audio_component, acomp);
+}
+
+/**
+ * dm_test_audio_component_unbind_clears_fields - Test unbind tears down component
+ * @test: The KUnit test context
+ *
+ * Unbinding must clear the component ops, the kernel device, and the display
+ * manager's stored component pointer.
+ */
+static void dm_test_audio_component_unbind_clears_fields(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct device *kdev;
+	struct drm_audio_component *acomp;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	kdev = kunit_kzalloc(test, sizeof(*kdev), GFP_KERNEL);
+	acomp = kunit_kzalloc(test, sizeof(*acomp), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, kdev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acomp);
+
+	dev_set_drvdata(kdev, &adev->ddev);
+
+	/* Pretend a prior bind already happened. */
+	acomp->dev = kdev;
+	adev->dm.audio_component = acomp;
+
+	amdgpu_dm_audio_component_unbind(kdev, NULL, acomp);
+
+	KUNIT_EXPECT_NULL(test, acomp->ops);
+	KUNIT_EXPECT_NULL(test, acomp->dev);
+	KUNIT_EXPECT_NULL(test, adev->dm.audio_component);
+}
+
+/* Tests for amdgpu_dm_audio_eld_notify() */
+
+static int dm_test_eld_notify_count;
+static int dm_test_eld_notify_port;
+static void *dm_test_eld_notify_ptr;
+
+static void dm_test_pin_eld_notify(void *audio_ptr, int port, int pipe)
+{
+	dm_test_eld_notify_count++;
+	dm_test_eld_notify_port = port;
+	dm_test_eld_notify_ptr = audio_ptr;
+}
+
+/**
+ * dm_test_eld_notify_invokes_callback - Test ELD notify forwards to hda driver
+ * @test: The KUnit test context
+ *
+ * When a component with a pin_eld_notify callback is registered, the notify
+ * helper must invoke it with the audio pointer and the requested pin.
+ */
+static void dm_test_eld_notify_invokes_callback(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_audio_component *acomp;
+	struct drm_audio_component_audio_ops *audio_ops;
+	int marker = 0;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	acomp = kunit_kzalloc(test, sizeof(*acomp), GFP_KERNEL);
+	audio_ops = kunit_kzalloc(test, sizeof(*audio_ops), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acomp);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, audio_ops);
+
+	audio_ops->audio_ptr = &marker;
+	audio_ops->pin_eld_notify = dm_test_pin_eld_notify;
+	acomp->audio_ops = audio_ops;
+	adev->dm.audio_component = acomp;
+
+	dm_test_eld_notify_count = 0;
+	dm_test_eld_notify_port = -100;
+	dm_test_eld_notify_ptr = NULL;
+
+	amdgpu_dm_audio_eld_notify(adev, 7);
+
+	KUNIT_EXPECT_EQ(test, dm_test_eld_notify_count, 1);
+	KUNIT_EXPECT_EQ(test, dm_test_eld_notify_port, 7);
+	KUNIT_EXPECT_PTR_EQ(test, dm_test_eld_notify_ptr, (void *)&marker);
+}
+
+/**
+ * dm_test_eld_notify_no_component - Test ELD notify is a no-op without component
+ * @test: The KUnit test context
+ *
+ * With no registered audio component, the notify helper must return without
+ * invoking any callback.
+ */
+static void dm_test_eld_notify_no_component(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	adev->dm.audio_component = NULL;
+
+	dm_test_eld_notify_count = 0;
+
+	amdgpu_dm_audio_eld_notify(adev, 3);
+
+	KUNIT_EXPECT_EQ(test, dm_test_eld_notify_count, 0);
+}
+
+/**
+ * dm_test_eld_notify_null_audio_ops - Test ELD notify is a no-op without audio_ops
+ * @test: The KUnit test context
+ *
+ * A component without audio_ops must not trigger any callback.
+ */
+static void dm_test_eld_notify_null_audio_ops(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_audio_component *acomp;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	acomp = kunit_kzalloc(test, sizeof(*acomp), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acomp);
+
+	acomp->audio_ops = NULL;
+	adev->dm.audio_component = acomp;
+
+	dm_test_eld_notify_count = 0;
+
+	amdgpu_dm_audio_eld_notify(adev, 3);
+
+	KUNIT_EXPECT_EQ(test, dm_test_eld_notify_count, 0);
+}
+
+/**
+ * dm_test_eld_notify_null_callback - Test ELD notify is a no-op without callback
+ * @test: The KUnit test context
+ *
+ * audio_ops present but with a NULL pin_eld_notify must not crash or call
+ * anything.
+ */
+static void dm_test_eld_notify_null_callback(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_audio_component *acomp;
+	struct drm_audio_component_audio_ops *audio_ops;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	acomp = kunit_kzalloc(test, sizeof(*acomp), GFP_KERNEL);
+	audio_ops = kunit_kzalloc(test, sizeof(*audio_ops), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acomp);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, audio_ops);
+
+	audio_ops->pin_eld_notify = NULL;
+	acomp->audio_ops = audio_ops;
+	adev->dm.audio_component = acomp;
+
+	dm_test_eld_notify_count = 0;
+
+	amdgpu_dm_audio_eld_notify(adev, 3);
+
+	KUNIT_EXPECT_EQ(test, dm_test_eld_notify_count, 0);
+}
+
+/* Tests for amdgpu_dm_audio_init_pins() */
+
+/**
+ * dm_test_audio_init_pins_sets_defaults - pin entries are initialised to default values
+ * @test: The KUnit test context
+ *
+ * amdgpu_dm_audio_init_pins() must set num_pins from audio_count, reset every
+ * pin to the sentinel defaults (-1 for rate/channels/bits, 0 for the rest)
+ * and copy each pin's hardware instance index from res_pool->audios[i]->inst.
+ */
+static void dm_test_audio_init_pins_sets_defaults(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	const unsigned int inst_array[] = {3, 7};
+	int i;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	amdgpu_dm_audio_init_pins(adev, 2, inst_array);
+
+	KUNIT_EXPECT_EQ(test, adev->mode_info.audio.num_pins, 2);
+
+	for (i = 0; i < 2; i++) {
+		KUNIT_EXPECT_EQ(test, adev->mode_info.audio.pin[i].channels, -1);
+		KUNIT_EXPECT_EQ(test, adev->mode_info.audio.pin[i].rate, -1);
+		KUNIT_EXPECT_EQ(test, adev->mode_info.audio.pin[i].bits_per_sample, -1);
+		KUNIT_EXPECT_EQ(test, adev->mode_info.audio.pin[i].status_bits, 0);
+		KUNIT_EXPECT_EQ(test, adev->mode_info.audio.pin[i].category_code, 0);
+		KUNIT_EXPECT_FALSE(test, adev->mode_info.audio.pin[i].connected);
+		KUNIT_EXPECT_EQ(test, adev->mode_info.audio.pin[i].offset, 0);
+	}
+	KUNIT_EXPECT_EQ(test, adev->mode_info.audio.pin[0].id, 3U);
+	KUNIT_EXPECT_EQ(test, adev->mode_info.audio.pin[1].id, 7U);
+}
+
+/**
+ * dm_test_audio_init_pins_zero_count - zero audio_count leaves num_pins at zero
+ * @test: The KUnit test context
+ *
+ * When res_pool->audio_count is 0, num_pins must be 0 and no pins touched.
+ */
+static void dm_test_audio_init_pins_zero_count(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	/* Pre-fill a sentinel so we can confirm the loop never ran. */
+	adev->mode_info.audio.pin[0].channels = 99;
+
+	amdgpu_dm_audio_init_pins(adev, 0, NULL);
+
+	KUNIT_EXPECT_EQ(test, adev->mode_info.audio.num_pins, 0);
+	KUNIT_EXPECT_EQ(test, adev->mode_info.audio.pin[0].channels, 99);
+}
+
+/* End of tests for amdgpu_dm_audio_init_pins() */
+
+static struct kunit_case dm_audio_test_cases[] = {
+	/* amdgpu_dm_audio_init */
+	KUNIT_CASE(dm_test_audio_init_disabled),
+	KUNIT_CASE(dm_test_audio_init_enabled_success),
+	/* amdgpu_dm_audio_init_pins */
+	KUNIT_CASE(dm_test_audio_init_pins_sets_defaults),
+	KUNIT_CASE(dm_test_audio_init_pins_zero_count),
+	/* amdgpu_dm_audio_fini */
+	KUNIT_CASE(dm_test_audio_fini_without_enabled_audio),
+	/* amdgpu_dm_fill_audio_info */
+	KUNIT_CASE(dm_test_fill_audio_info_ids_name_flags),
+	KUNIT_CASE(dm_test_fill_audio_info_cea_lt_3_skips_modes),
+	KUNIT_CASE(dm_test_fill_audio_info_cea_ge_3_copies_modes),
+	KUNIT_CASE(dm_test_fill_audio_info_cea_ge_3_zero_modes),
+	KUNIT_CASE(dm_test_fill_audio_info_latency_present),
+	KUNIT_CASE(dm_test_fill_audio_info_latency_absent_keeps_zero),
+	/* amdgpu_dm_audio_component_bind/unbind */
+	KUNIT_CASE(dm_test_audio_component_bind_sets_fields),
+	KUNIT_CASE(dm_test_audio_component_unbind_clears_fields),
+	/* amdgpu_dm_audio_eld_notify */
+	KUNIT_CASE(dm_test_eld_notify_invokes_callback),
+	KUNIT_CASE(dm_test_eld_notify_no_component),
+	KUNIT_CASE(dm_test_eld_notify_null_audio_ops),
+	KUNIT_CASE(dm_test_eld_notify_null_callback),
+	{}
+};
+
+static struct kunit_suite dm_audio_test_suite = {
+	.name = "amdgpu_dm_audio",
+	.test_cases = dm_audio_test_cases,
+};
+
+kunit_test_suite(dm_audio_test_suite);
+
+MODULE_LICENSE("Dual MIT/GPL");
+MODULE_DESCRIPTION("KUnit tests for amdgpu_dm_audio");
+MODULE_AUTHOR("AMD");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_backlight_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_backlight_test.c
new file mode 100644
index 0000000000000..f55f93747df77
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_backlight_test.c
@@ -0,0 +1,1950 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * KUnit tests for amdgpu_dm_backlight.c
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#include <kunit/test.h>
+#include <linux/backlight.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_mode_config.h>
+#include <drm/drm_property.h>
+
+#include "dc.h"
+#include "dc_dmub_srv.h"
+#include "amdgpu.h"
+#include "amdgpu_display.h"
+#include "amdgpu_mode.h"
+#include "amdgpu_dm.h"
+#include "amdgpu_dm_backlight.h"
+#include "amdgpu_dm_kunit_test_helpers.h"
+#include "amd_shared.h"
+#include "link_service.h"
+#include "dc/inc/hw/panel_cntl.h"
+
+struct dm_backlight_connector_fixture {
+	struct amdgpu_device *adev;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+};
+
+static const struct drm_connector_funcs dm_backlight_test_connector_funcs = {
+	.reset = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static void setup_test_connector(struct kunit *test,
+				 struct dm_backlight_connector_fixture *fixture,
+				 int bl_idx, enum signal_type signal)
+{
+	fixture->adev = kunit_kzalloc(test, sizeof(*fixture->adev), GFP_KERNEL);
+	fixture->aconnector = kunit_kzalloc(test, sizeof(*fixture->aconnector), GFP_KERNEL);
+	fixture->link = kunit_kzalloc(test, sizeof(*fixture->link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fixture->adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fixture->aconnector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fixture->link);
+
+	fixture->aconnector->bl_idx = bl_idx;
+	fixture->aconnector->dc_link = fixture->link;
+	fixture->aconnector->base.dev = &fixture->adev->ddev;
+	fixture->link->connector_signal = signal;
+}
+
+static void setup_test_dm_ddev(struct kunit *test, struct amdgpu_display_manager *dm)
+{
+	struct drm_device *ddev;
+
+	ddev = kunit_kzalloc(test, sizeof(*ddev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ddev);
+
+	INIT_LIST_HEAD(&ddev->mode_config.connector_list);
+	spin_lock_init(&ddev->mode_config.connector_list_lock);
+	dm->ddev = ddev;
+}
+
+/* Tests for dm_find_stream_with_link() */
+
+/**
+ * dm_test_find_stream_with_link_returns_match - Test matching stream lookup
+ * @test: The KUnit test context
+ */
+static void dm_test_find_stream_with_link_returns_match(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct dc_link *other_link = dm_kunit_alloc_link(test);
+	struct dc_link *target_link = dm_kunit_alloc_link(test);
+	struct dc_stream_state *stream;
+
+	dm_kunit_add_stream_to_state(test, dm->dc->current_state, 0, other_link);
+	dm_kunit_add_stream_to_state(test, dm->dc->current_state, 1, target_link);
+	stream = dm_find_stream_with_link(dm, target_link);
+
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+	KUNIT_EXPECT_PTR_EQ(test, stream->link, target_link);
+}
+
+/**
+ * dm_test_find_stream_with_link_missing - Test missing stream lookup
+ * @test: The KUnit test context
+ */
+static void dm_test_find_stream_with_link_missing(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct dc_link *stream_link = dm_kunit_alloc_link(test);
+	struct dc_link *missing_link = dm_kunit_alloc_link(test);
+
+	dm_kunit_add_stream_to_state(test, dm->dc->current_state, 0, stream_link);
+
+	KUNIT_EXPECT_NULL(test, dm_find_stream_with_link(dm, missing_link));
+}
+
+/* Tests for amdgpu_dm_backlight_set_level() */
+
+/**
+ * dm_test_backlight_set_level_connector_off - Test connector-off cache path
+ * @test: The KUnit test context
+ *
+ * If the matching connector has no encoder, set_level() must cache the
+ * requested brightness and return before touching DC or backlight hardware.
+ */
+static void dm_test_backlight_set_level_connector_off(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct amdgpu_dm_connector *aconnector;
+
+	setup_test_dm_ddev(test, dm);
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconnector);
+	INIT_LIST_HEAD(&aconnector->base.head);
+	aconnector->bl_idx = 1;
+	aconnector->base.encoder = NULL;
+	list_add_tail(&aconnector->base.head, &dm->ddev->mode_config.connector_list);
+
+	amdgpu_dm_backlight_set_level(dm, 1, 1234);
+
+	KUNIT_EXPECT_EQ(test, dm->brightness[1], 1234U);
+	KUNIT_EXPECT_EQ(test, dm->actual_brightness[1], 0U);
+}
+
+/**
+ * dm_test_backlight_set_level_no_stream - Test no-stream early return
+ * @test: The KUnit test context
+ *
+ * With no stream for the backlight link, set_level() records the requested
+ * brightness and exits before calling the power-module programming path.
+ */
+static void dm_test_backlight_set_level_no_stream(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
+
+	setup_test_dm_ddev(test, dm);
+	dm->backlight_caps[1].caps_valid = true;
+	dm->backlight_caps[1].min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT;
+	dm->backlight_caps[1].max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+	dm->backlight_link[1] = link;
+
+	amdgpu_dm_backlight_set_level(dm, 1, 2000);
+
+	KUNIT_EXPECT_EQ(test, dm->brightness[1], 2000U);
+	KUNIT_EXPECT_EQ(test, dm->actual_brightness[1], 0U);
+}
+
+/**
+ * dm_test_backlight_set_level_aux_programs_power_module - Test AUX programming path
+ * @test: The KUnit test context
+ *
+ * With a matching stream present, set_level() walks into the DC programming
+ * path. A NULL power_module makes mod_power_set_backlight_nits() a safe
+ * early-false, and ips_support disabled leaves idle optimizations untouched.
+ * A non-matching connector exercises the connector-list skip, and a non-zero
+ * brightness_mask exercises the quirk-OR path.
+ */
+static void dm_test_backlight_set_level_aux_programs_power_module(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
+	struct amdgpu_dm_connector *other;
+
+	setup_test_dm_ddev(test, dm);
+	mutex_init(&dm->dc_lock);
+	dm->power_module = NULL;
+
+	/* Non-matching connector exercises the bl_idx skip (continue). */
+	other = kunit_kzalloc(test, sizeof(*other), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, other);
+	INIT_LIST_HEAD(&other->base.head);
+	other->bl_idx = 0;
+	list_add_tail(&other->base.head, &dm->ddev->mode_config.connector_list);
+
+	dm->backlight_caps[1].caps_valid = true;
+	dm->backlight_caps[1].aux_support = true;
+	dm->backlight_caps[1].brightness_mask = 0x3;
+	dm->backlight_caps[1].aux_min_input_signal = 1;
+	dm->backlight_caps[1].aux_max_input_signal = 512;
+	dm->backlight_caps[1].min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT;
+	dm->backlight_caps[1].max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+	dm->backlight_link[1] = link;
+	dm_kunit_add_stream_to_state(test, dm->dc->current_state, 0, link);
+
+	amdgpu_dm_backlight_set_level(dm, 1, 2000);
+
+	/* power_module is NULL so programming fails; actual stays unchanged. */
+	KUNIT_EXPECT_EQ(test, dm->brightness[1], 2000U);
+	KUNIT_EXPECT_EQ(test, dm->actual_brightness[1], 0U);
+}
+
+/**
+ * dm_test_backlight_set_level_pwm_programs_power_module - Test PWM programming path
+ * @test: The KUnit test context
+ *
+ * With aux_support cleared, set_level() takes the millipercent branch:
+ * get_brightness_range() + mod_power_set_backlight_percent(). A NULL
+ * power_module keeps the call a safe early-false.
+ */
+static void dm_test_backlight_set_level_pwm_programs_power_module(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
+
+	setup_test_dm_ddev(test, dm);
+	mutex_init(&dm->dc_lock);
+	dm->power_module = NULL;
+
+	dm->backlight_caps[1].caps_valid = true;
+	dm->backlight_caps[1].aux_support = false;
+	dm->backlight_caps[1].min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT;
+	dm->backlight_caps[1].max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+	dm->backlight_link[1] = link;
+	dm_kunit_add_stream_to_state(test, dm->dc->current_state, 0, link);
+
+	amdgpu_dm_backlight_set_level(dm, 1, 2000);
+
+	KUNIT_EXPECT_EQ(test, dm->brightness[1], 2000U);
+	KUNIT_EXPECT_EQ(test, dm->actual_brightness[1], 0U);
+}
+
+/**
+ * dm_test_backlight_set_level_reallows_idle - Test idle-optimization toggle path
+ * @test: The KUnit test context
+ *
+ * When ips_support is set and dmub idle is allowed, set_level() disables idle
+ * optimizations around the programming call and re-enables them afterwards.
+ * disable_idle_power_optimizations keeps dc_allow_idle_optimizations() a safe
+ * early return, and ctx->logger is wired because DC_LOG_* dereferences it.
+ */
+static void dm_test_backlight_set_level_reallows_idle(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
+	struct dc_dmub_srv *dmub_srv;
+	struct dal_logger *logger;
+	struct dc_context *ctx;
+
+	setup_test_dm_ddev(test, dm);
+	mutex_init(&dm->dc_lock);
+	dm->power_module = NULL;
+
+	/* dm_kunit_alloc_dm() leaves dc->ctx NULL; the idle path dereferences it. */
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+	dm->dc->ctx = ctx;
+
+	logger = kunit_kzalloc(test, sizeof(*logger), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, logger);
+	logger->dev = &adev->ddev;
+	dm->dc->ctx->logger = logger;
+
+	dmub_srv = kunit_kzalloc(test, sizeof(*dmub_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dmub_srv);
+	dmub_srv->idle_allowed = true;
+	dm->dc->ctx->dmub_srv = dmub_srv;
+	dm->dc->caps.ips_support = true;
+	/* Keep dc_allow_idle_optimizations() a safe early return. */
+	dm->dc->debug.disable_idle_power_optimizations = true;
+
+	dm->backlight_caps[1].caps_valid = true;
+	dm->backlight_caps[1].aux_support = true;
+	dm->backlight_caps[1].aux_min_input_signal = 1;
+	dm->backlight_caps[1].aux_max_input_signal = 512;
+	dm->backlight_caps[1].min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT;
+	dm->backlight_caps[1].max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+	dm->backlight_link[1] = link;
+	dm_kunit_add_stream_to_state(test, dm->dc->current_state, 0, link);
+
+	amdgpu_dm_backlight_set_level(dm, 1, 2000);
+
+	KUNIT_EXPECT_EQ(test, dm->brightness[1], 2000U);
+}
+
+/**
+ * dm_test_backlight_update_status_no_stream - Test update_status wrapper
+ * @test: The KUnit test context
+ */
+static void dm_test_backlight_update_status_no_stream(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct backlight_device *bd;
+
+	setup_test_dm_ddev(test, dm);
+	bd = kunit_kzalloc(test, sizeof(*bd), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bd);
+	dev_set_drvdata(&bd->dev, dm);
+	bd->props.brightness = 3456;
+	dm->num_of_edps = 2;
+	dm->backlight_dev[1] = bd;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_backlight_update_status(bd), 0);
+	KUNIT_EXPECT_EQ(test, dm->brightness[1], 3456U);
+}
+
+static void setup_test_link_service(struct kunit *test, struct dc_link *link)
+{
+	struct link_service *link_srv;
+	struct dc_context *ctx;
+	struct dc *dc;
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
+
+	dc->ctx = ctx;
+	dc->link_srv = link_srv;
+	ctx->dc = dc;
+	link->dc = dc;
+	link->ctx = ctx;
+}
+
+static int dm_test_get_backlight_level_mid(const struct dc_link *link)
+{
+	return (0x101 * AMDGPU_DM_DEFAULT_MIN_BACKLIGHT) + 1000;
+}
+
+static int dm_test_get_backlight_level_error(const struct dc_link *link)
+{
+	return DC_ERROR_UNEXPECTED;
+}
+
+static bool dm_test_get_backlight_level_nits(struct dc_link *link,
+					     uint32_t *avg,
+					     uint32_t *peak)
+{
+	*avg = 250000;
+	*peak = 300000;
+
+	return true;
+}
+
+static bool dm_test_get_backlight_level_nits_fail(struct dc_link *link,
+						  uint32_t *avg,
+						  uint32_t *peak)
+{
+	return false;
+}
+
+/* Tests for amdgpu_dm_backlight_get_level()/get_brightness() */
+
+/**
+ * dm_test_backlight_get_level_pwm_success - Test PWM brightness readback
+ * @test: The KUnit test context
+ */
+static void dm_test_backlight_get_level_pwm_success(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
+	u32 hw_level = dm_test_get_backlight_level_mid(link);
+
+	setup_test_link_service(test, link);
+	link->dc->link_srv->edp_get_backlight_level = dm_test_get_backlight_level_mid;
+	dm->backlight_link[0] = link;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_backlight_get_level(dm, 0),
+			 convert_brightness_to_user(&dm->backlight_caps[0], hw_level));
+}
+
+/**
+ * dm_test_backlight_get_level_pwm_error - Test PWM readback fallback
+ * @test: The KUnit test context
+ */
+static void dm_test_backlight_get_level_pwm_error(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
+
+	setup_test_link_service(test, link);
+	link->dc->link_srv->edp_get_backlight_level = dm_test_get_backlight_level_error;
+	dm->brightness[0] = 4321;
+	dm->backlight_link[0] = link;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_backlight_get_level(dm, 0), 4321U);
+}
+
+/**
+ * dm_test_backlight_get_level_aux_success - Test AUX brightness readback
+ * @test: The KUnit test context
+ */
+static void dm_test_backlight_get_level_aux_success(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
+	struct amdgpu_dm_backlight_caps *caps = &dm->backlight_caps[0];
+
+	setup_test_link_service(test, link);
+	link->dc->link_srv->edp_get_backlight_level_nits = dm_test_get_backlight_level_nits;
+	dm->backlight_link[0] = link;
+	caps->caps_valid = true;
+	caps->aux_support = true;
+	caps->aux_min_input_signal = 1;
+	caps->aux_max_input_signal = 512;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_backlight_get_level(dm, 0),
+			 convert_brightness_to_user(caps, 250000));
+}
+
+/**
+ * dm_test_backlight_get_level_aux_error - Test AUX readback fallback
+ * @test: The KUnit test context
+ */
+static void dm_test_backlight_get_level_aux_error(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
+	struct amdgpu_dm_backlight_caps *caps = &dm->backlight_caps[0];
+
+	setup_test_link_service(test, link);
+	link->dc->link_srv->edp_get_backlight_level_nits = dm_test_get_backlight_level_nits_fail;
+	dm->brightness[0] = 6789;
+	dm->backlight_link[0] = link;
+	caps->caps_valid = true;
+	caps->aux_support = true;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_backlight_get_level(dm, 0), 6789U);
+}
+
+/**
+ * dm_test_backlight_get_brightness_uses_device_index - Test get_brightness wrapper
+ * @test: The KUnit test context
+ */
+static void dm_test_backlight_get_brightness_uses_device_index(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
+	struct backlight_device *bd;
+
+	setup_test_link_service(test, link);
+	link->dc->link_srv->edp_get_backlight_level = dm_test_get_backlight_level_error;
+	bd = kunit_kzalloc(test, sizeof(*bd), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bd);
+	dev_set_drvdata(&bd->dev, dm);
+	dm->num_of_edps = 2;
+	dm->backlight_dev[1] = bd;
+	dm->brightness[1] = 2468;
+	dm->backlight_link[1] = link;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_backlight_get_brightness(bd), 2468);
+}
+
+/* Tests for amdgpu_dm_register_backlight_device() */
+
+/**
+ * dm_test_register_backlight_device_negative_index - Test invalid index no-op
+ * @test: The KUnit test context
+ */
+static void dm_test_register_backlight_device_negative_index(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = dm_kunit_alloc_connector(test, adev, NULL);
+	aconnector->bl_idx = -1;
+
+	amdgpu_dm_register_backlight_device(aconnector);
+	KUNIT_EXPECT_NULL(test, adev->dm.backlight_dev[0]);
+}
+
+static struct drm_connector *setup_panel_power_savings_connector(struct kunit *test,
+							 struct device **device_out,
+							 struct dm_connector_state **state_out)
+{
+	struct dm_connector_state *state;
+	struct drm_connector *connector;
+	struct amdgpu_device *adev;
+	struct device *device;
+	int ret;
+
+	adev = dm_kunit_alloc_adev(test);
+	ret = drmm_mode_config_init(&adev->ddev);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	device = kunit_kzalloc(test, sizeof(*device), GFP_KERNEL);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, device);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+	connector->dev = &adev->ddev;
+	connector->state = &state->base;
+	dev_set_drvdata(device, connector);
+	*device_out = device;
+	*state_out = state;
+
+	return connector;
+}
+
+static void dm_test_free_sysfs_buf(void *data)
+{
+	free_page((unsigned long)data);
+}
+
+static char *dm_test_alloc_sysfs_buf(struct kunit *test)
+{
+	char *buf;
+
+	buf = (char *)get_zeroed_page(GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, buf);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_free_sysfs_buf, buf), 0);
+
+	return buf;
+}
+
+/* Tests for panel_power_savings_show()/panel_power_savings_store() */
+
+/**
+ * dm_test_panel_power_savings_show_maps_disable_to_zero - Test show output
+ * @test: The KUnit test context
+ */
+static void dm_test_panel_power_savings_show_maps_disable_to_zero(struct kunit *test)
+{
+	struct dm_connector_state *state;
+	struct device *device;
+	char *buf;
+
+	setup_panel_power_savings_connector(test, &device, &state);
+	buf = dm_test_alloc_sysfs_buf(test);
+	state->abm_level = ABM_LEVEL_IMMEDIATE_DISABLE;
+
+	KUNIT_EXPECT_EQ(test, panel_power_savings_show(device, NULL, buf), 2);
+	KUNIT_EXPECT_STREQ(test, buf, "0\n");
+}
+
+/**
+ * dm_test_panel_power_savings_show_reports_level - Test show output for active level
+ * @test: The KUnit test context
+ *
+ * When abm_level is not the immediate-disable sentinel, show() reports the
+ * raw level value.
+ */
+static void dm_test_panel_power_savings_show_reports_level(struct kunit *test)
+{
+	struct dm_connector_state *state;
+	struct device *device;
+	char *buf;
+
+	setup_panel_power_savings_connector(test, &device, &state);
+	buf = dm_test_alloc_sysfs_buf(test);
+	state->abm_level = 3;
+
+	KUNIT_EXPECT_EQ(test, panel_power_savings_show(device, NULL, buf), 2);
+	KUNIT_EXPECT_STREQ(test, buf, "3\n");
+}
+
+/**
+ * dm_test_panel_power_savings_store_sets_disable - Test zero maps to disable
+ * @test: The KUnit test context
+ */
+static void dm_test_panel_power_savings_store_sets_disable(struct kunit *test)
+{
+	struct dm_connector_state *state;
+	struct device *device;
+	size_t count = strlen("0");
+
+	setup_panel_power_savings_connector(test, &device, &state);
+
+	KUNIT_EXPECT_EQ(test, panel_power_savings_store(device, NULL, "0", count),
+			 (ssize_t)count);
+	KUNIT_EXPECT_EQ(test, state->abm_level, ABM_LEVEL_IMMEDIATE_DISABLE);
+}
+
+/**
+ * dm_test_panel_power_savings_store_forbidden - Test forbidden update
+ * @test: The KUnit test context
+ */
+static void dm_test_panel_power_savings_store_forbidden(struct kunit *test)
+{
+	struct dm_connector_state *state;
+	struct device *device;
+
+	setup_panel_power_savings_connector(test, &device, &state);
+	state->abm_sysfs_forbidden = true;
+
+	KUNIT_EXPECT_EQ(test, panel_power_savings_store(device, NULL, "1", 1), -EBUSY);
+}
+
+/**
+ * dm_test_panel_power_savings_store_rejects_invalid_text - Test parse failure
+ * @test: The KUnit test context
+ */
+static void dm_test_panel_power_savings_store_rejects_invalid_text(struct kunit *test)
+{
+	struct drm_connector *connector;
+	struct drm_device *drm;
+	struct device *device;
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	drm = kunit_kzalloc(test, sizeof(*drm), GFP_KERNEL);
+	device = kunit_kzalloc(test, sizeof(*device), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, device);
+
+	connector->dev = drm;
+	dev_set_drvdata(device, connector);
+
+	KUNIT_EXPECT_LT(test, panel_power_savings_store(device, NULL, "bad", 3), 0);
+}
+
+/**
+ * dm_test_panel_power_savings_store_rejects_out_of_range - Test range failure
+ * @test: The KUnit test context
+ */
+static void dm_test_panel_power_savings_store_rejects_out_of_range(struct kunit *test)
+{
+	struct drm_connector *connector;
+	struct drm_device *drm;
+	struct device *device;
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	drm = kunit_kzalloc(test, sizeof(*drm), GFP_KERNEL);
+	device = kunit_kzalloc(test, sizeof(*device), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, device);
+
+	connector->dev = drm;
+	dev_set_drvdata(device, connector);
+
+	KUNIT_EXPECT_EQ(test, panel_power_savings_store(device, NULL, "5", 1), -EINVAL);
+}
+
+/* Tests for amdgpu_dm_backlight_get_device_index() */
+
+/**
+ * dm_test_backlight_device_index_matches_second - Test matching second backlight device
+ * @test: The KUnit test context
+ */
+static void dm_test_backlight_device_index_matches_second(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct backlight_device *bd0;
+	struct backlight_device *bd1;
+
+	bd0 = kunit_kzalloc(test, sizeof(*bd0), GFP_KERNEL);
+	bd1 = kunit_kzalloc(test, sizeof(*bd1), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, bd0);
+	KUNIT_ASSERT_NOT_NULL(test, bd1);
+
+	dm->num_of_edps = 2;
+	dm->backlight_dev[0] = bd0;
+	dm->backlight_dev[1] = bd1;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_backlight_get_device_index(dm, bd1), 1);
+}
+
+/**
+ * dm_test_backlight_device_index_missing_fallback - Test missing backlight device fallback
+ * @test: The KUnit test context
+ */
+static void dm_test_backlight_device_index_missing_fallback(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct backlight_device *known_bd;
+	struct backlight_device *unknown_bd;
+
+	known_bd = kunit_kzalloc(test, sizeof(*known_bd), GFP_KERNEL);
+	unknown_bd = kunit_kzalloc(test, sizeof(*unknown_bd), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, known_bd);
+	KUNIT_ASSERT_NOT_NULL(test, unknown_bd);
+
+	dm->num_of_edps = 1;
+	dm->backlight_dev[0] = known_bd;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_backlight_get_device_index(dm, unknown_bd), 0);
+}
+
+/* Tests for amdgpu_dm_update_backlight_caps() */
+
+/**
+ * dm_test_backlight_caps_valid_short_circuit - Test Backlight caps valid short circuit
+ * @test: The KUnit test context
+ */
+static void dm_test_backlight_caps_valid_short_circuit(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct amdgpu_dm_backlight_caps *caps = &dm->backlight_caps[0];
+
+	caps->caps_valid = true;
+	caps->aux_support = false;
+	caps->min_input_signal = 42;
+	caps->max_input_signal = 199;
+
+	amdgpu_dm_update_backlight_caps(dm, 0);
+
+	KUNIT_EXPECT_TRUE(test, caps->caps_valid);
+	KUNIT_EXPECT_EQ(test, caps->min_input_signal, 42);
+	KUNIT_EXPECT_EQ(test, caps->max_input_signal, 199);
+}
+
+#if !defined(CONFIG_ACPI)
+
+/**
+ * dm_test_backlight_caps_aux_support_noop - Test Backlight caps aux support noop
+ * @test: The KUnit test context
+ */
+static void dm_test_backlight_caps_aux_support_noop(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct amdgpu_dm_backlight_caps *caps = &dm->backlight_caps[0];
+
+	caps->caps_valid = false;
+	caps->aux_support = true;
+	caps->min_input_signal = 11;
+	caps->max_input_signal = 222;
+
+	amdgpu_dm_update_backlight_caps(dm, 0);
+
+	KUNIT_EXPECT_FALSE(test, caps->caps_valid);
+	KUNIT_EXPECT_EQ(test, caps->min_input_signal, 11);
+	KUNIT_EXPECT_EQ(test, caps->max_input_signal, 222);
+}
+
+/**
+ * dm_test_backlight_caps_non_aux_sets_defaults - Test Backlight caps non aux sets defaults
+ * @test: The KUnit test context
+ */
+static void dm_test_backlight_caps_non_aux_sets_defaults(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct amdgpu_dm_backlight_caps *caps = &dm->backlight_caps[0];
+
+	caps->caps_valid = false;
+	caps->aux_support = false;
+	caps->min_input_signal = 0;
+	caps->max_input_signal = 0;
+
+	amdgpu_dm_update_backlight_caps(dm, 0);
+
+	KUNIT_EXPECT_TRUE(test, caps->caps_valid);
+	KUNIT_EXPECT_EQ(test, caps->min_input_signal, AMDGPU_DM_DEFAULT_MIN_BACKLIGHT);
+	KUNIT_EXPECT_EQ(test, caps->max_input_signal, AMDGPU_DM_DEFAULT_MAX_BACKLIGHT);
+}
+#endif
+
+/* Tests for get_brightness_range() */
+
+/**
+ * dm_test_brightness_range_null_caps - Test Brightness range null caps
+ * @test: The KUnit test context
+ */
+static void dm_test_brightness_range_null_caps(struct kunit *test)
+{
+	unsigned int min = 99, max = 99;
+
+	KUNIT_EXPECT_EQ(test, get_brightness_range(NULL, &min, &max), 0);
+	/* min/max should remain untouched */
+	KUNIT_EXPECT_EQ(test, min, 99U);
+	KUNIT_EXPECT_EQ(test, max, 99U);
+}
+
+/**
+ * dm_test_brightness_range_pwm - Test Brightness range pwm
+ * @test: The KUnit test context
+ */
+static void dm_test_brightness_range_pwm(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	unsigned int min, max;
+
+	caps.aux_support = false;
+	caps.min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT;
+	caps.max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+
+	KUNIT_EXPECT_EQ(test, get_brightness_range(&caps, &min, &max), 1);
+	/* 0x101 * AMDGPU_DM_DEFAULT_MIN_BACKLIGHT, 0x101 * AMDGPU_DM_DEFAULT_MAX_BACKLIGHT */
+	KUNIT_EXPECT_EQ(test, min, 0x101U * AMDGPU_DM_DEFAULT_MIN_BACKLIGHT);
+	KUNIT_EXPECT_EQ(test, max, 0x101U * AMDGPU_DM_DEFAULT_MAX_BACKLIGHT);
+}
+
+/**
+ * dm_test_brightness_range_aux - Test Brightness range aux
+ * @test: The KUnit test context
+ */
+static void dm_test_brightness_range_aux(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	unsigned int min, max;
+
+	caps.aux_support = true;
+	caps.aux_min_input_signal = 1;
+	caps.aux_max_input_signal = 512;
+
+	KUNIT_EXPECT_EQ(test, get_brightness_range(&caps, &min, &max), 1);
+	/* millinits: 1000 * value */
+	KUNIT_EXPECT_EQ(test, min, 1000U);
+	KUNIT_EXPECT_EQ(test, max, 512000U);
+}
+
+/* Tests for convert_brightness_to_user() */
+
+/**
+ * dm_test_brightness_to_user_null_caps - Test Brightness to user null caps
+ * @test: The KUnit test context
+ */
+static void dm_test_brightness_to_user_null_caps(struct kunit *test)
+{
+	/*
+	 * With NULL caps, get_brightness_range fails → passthrough.
+	 * We simulate this by passing a zeroed caps struct where
+	 * max_input_signal=0 makes max=0 and the function hits
+	 * get_brightness_range returning 0 since caps is NULL.
+	 */
+	KUNIT_EXPECT_EQ(test, convert_brightness_to_user(NULL, 42), 42U);
+}
+
+/**
+ * dm_test_brightness_to_user_below_min - Test Brightness to user below min
+ * @test: The KUnit test context
+ */
+static void dm_test_brightness_to_user_below_min(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+
+	caps.aux_support = false;
+	caps.min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT;
+	caps.max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+
+	/* brightness < min (0x101*AMDGPU_DM_DEFAULT_MIN_BACKLIGHT), should return 0 */
+	KUNIT_EXPECT_EQ(test, convert_brightness_to_user(&caps, 100), 0U);
+}
+
+/**
+ * dm_test_brightness_to_user_at_max - Test Brightness to user at max
+ * @test: The KUnit test context
+ */
+static void dm_test_brightness_to_user_at_max(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	unsigned int min, max;
+
+	caps.aux_support = false;
+	caps.min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT;
+	caps.max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+
+	get_brightness_range(&caps, &min, &max);
+
+	/* At max → should return max */
+	KUNIT_EXPECT_EQ(test, convert_brightness_to_user(&caps, max), max);
+}
+
+/**
+ * dm_test_brightness_to_user_at_min - Test Brightness to user at min
+ * @test: The KUnit test context
+ */
+static void dm_test_brightness_to_user_at_min(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	unsigned int min, max;
+
+	caps.aux_support = false;
+	caps.min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT;
+	caps.max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+
+	get_brightness_range(&caps, &min, &max);
+
+	/* At min → should return 0 */
+	KUNIT_EXPECT_EQ(test, convert_brightness_to_user(&caps, min), 0U);
+}
+
+/**
+ * dm_test_brightness_to_user_midpoint_pwm - Test Brightness to user midpoint pwm
+ * @test: The KUnit test context
+ */
+static void dm_test_brightness_to_user_midpoint_pwm(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	unsigned int min, max, mid_hw, result;
+	u64 expected;
+
+	caps.aux_support = false;
+	caps.min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT;
+	caps.max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+
+	get_brightness_range(&caps, &min, &max);
+
+	/* midpoint of hw range */
+	mid_hw = min + (max - min) / 2;
+	/* expected = DIV_ROUND_CLOSEST_ULL((u64)max * (mid_hw - min), max - min) */
+	expected = DIV_ROUND_CLOSEST_ULL((u64)max * (mid_hw - min), max - min);
+	result = convert_brightness_to_user(&caps, mid_hw);
+
+	KUNIT_EXPECT_EQ(test, result, (u32)expected);
+}
+
+/* Tests for convert_brightness_from_user() — no custom curve */
+
+/**
+ * dm_test_brightness_from_user_null_caps - Test Brightness from user null caps
+ * @test: The KUnit test context
+ */
+static void dm_test_brightness_from_user_null_caps(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, convert_brightness_from_user(NULL, 100), 100U);
+}
+
+/**
+ * dm_test_brightness_from_user_zero - Test Brightness from user zero
+ * @test: The KUnit test context
+ */
+static void dm_test_brightness_from_user_zero(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	unsigned int min, max;
+
+	caps.aux_support = false;
+	caps.min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT;
+	caps.max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+	/* no custom curve */
+	caps.data_points = 0;
+
+	get_brightness_range(&caps, &min, &max);
+
+	/* brightness=0 → min + 0 = min */
+	KUNIT_EXPECT_EQ(test, convert_brightness_from_user(&caps, 0), (u32)min);
+}
+
+/**
+ * dm_test_brightness_from_user_max - Test Brightness from user max
+ * @test: The KUnit test context
+ */
+static void dm_test_brightness_from_user_max(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	unsigned int min, max;
+
+	caps.aux_support = false;
+	caps.min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT;
+	caps.max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+	caps.data_points = 0;
+
+	get_brightness_range(&caps, &min, &max);
+
+	/*
+	 * brightness=max → min + DIV_ROUND_CLOSEST((max-min)*max, max)
+	 *               = min + (max - min) = max
+	 */
+	KUNIT_EXPECT_EQ(test, convert_brightness_from_user(&caps, max), (u32)max);
+}
+
+/**
+ * dm_test_brightness_from_user_aux - Test Brightness from user aux
+ * @test: The KUnit test context
+ */
+static void dm_test_brightness_from_user_aux(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	unsigned int min, max;
+
+	caps.aux_support = true;
+	caps.aux_min_input_signal = 1;
+	caps.aux_max_input_signal = 512;
+	caps.data_points = 0;
+
+	get_brightness_range(&caps, &min, &max);
+
+	/* brightness=0 → min */
+	KUNIT_EXPECT_EQ(test, convert_brightness_from_user(&caps, 0), (u32)min);
+	/* brightness=max → max */
+	KUNIT_EXPECT_EQ(test, convert_brightness_from_user(&caps, max), (u32)max);
+}
+
+/* Tests for convert_custom_brightness() */
+
+/**
+ * dm_test_custom_brightness_no_data_points - Test Custom brightness no data points
+ * @test: The KUnit test context
+ */
+static void dm_test_custom_brightness_no_data_points(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	uint32_t brightness = 128;
+	uint32_t saved = brightness;
+
+	caps.data_points = 0;
+
+	convert_custom_brightness(&caps, 3084, 65535, &brightness);
+
+	/* No data points → no-op */
+	KUNIT_EXPECT_EQ(test, brightness, saved);
+}
+
+/**
+ * dm_test_custom_brightness_debug_mask_disables - Test Custom brightness debug mask disables
+ * @test: The KUnit test context
+ */
+static void dm_test_custom_brightness_debug_mask_disables(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	uint32_t brightness = 128;
+	uint32_t saved = brightness;
+	uint saved_mask = amdgpu_dm_get_dc_debug_mask();
+
+	caps.data_points = 3;
+	caps.luminance_data[0].input_signal = 50;
+	caps.luminance_data[0].luminance = 10;
+
+	/* Set the disable flag */
+	amdgpu_dm_set_dc_debug_mask(amdgpu_dm_get_dc_debug_mask() | DC_DISABLE_CUSTOM_BRIGHTNESS_CURVE);
+
+	convert_custom_brightness(&caps, 3084, 65535, &brightness);
+
+	/* Should be no-op due to debug mask */
+	KUNIT_EXPECT_EQ(test, brightness, saved);
+
+	amdgpu_dm_set_dc_debug_mask(saved_mask);
+}
+
+/**
+ * dm_test_custom_brightness_exact_match - Test Custom brightness exact match
+ * @test: The KUnit test context
+ */
+static void dm_test_custom_brightness_exact_match(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	uint32_t brightness;
+	unsigned int min, max;
+	uint saved_mask = amdgpu_dm_get_dc_debug_mask();
+
+	amdgpu_dm_set_dc_debug_mask(amdgpu_dm_get_dc_debug_mask() & ~DC_DISABLE_CUSTOM_BRIGHTNESS_CURVE);
+
+	caps.aux_support = false;
+	caps.min_input_signal = 0;
+	caps.max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+	caps.data_points = 3;
+	caps.luminance_data[0].input_signal = 50;
+	caps.luminance_data[0].luminance = 20;
+	caps.luminance_data[1].input_signal = 128;
+	caps.luminance_data[1].luminance = 50;
+	caps.luminance_data[2].input_signal = 200;
+	caps.luminance_data[2].luminance = 90;
+
+	get_brightness_range(&caps, &min, &max);
+
+	/*
+	 * Set brightness so that scale_input_to_fw yields exactly 128.
+	 * scale_input_to_fw(min, max, x) = DIV_ROUND_CLOSEST(x * 255, max - min)
+	 * With min=0, max=0x101*255=65535:
+	 * We need x such that DIV_ROUND_CLOSEST(x * 255, 65535) = 128
+	 * → x = 128 * 65535 / 255 = 32896
+	 */
+	brightness = 32896;
+
+	convert_custom_brightness(&caps, min, max, &brightness);
+
+	/*
+	 * Exact match: lum=50, brightness_scaled=128
+	 * result = scale_fw_to_input(min, max, DIV_ROUND_CLOSEST(50*128, 101))
+	 *        = scale_fw_to_input(0, 65535, DIV_ROUND_CLOSEST(6400, 101))
+	 *        = scale_fw_to_input(0, 65535, 63)
+	 *        = 0 + DIV_ROUND_CLOSEST(63 * 65535, 255) = 16191 (approx)
+	 */
+	KUNIT_EXPECT_TRUE(test, brightness != 32896);
+	KUNIT_EXPECT_TRUE(test, brightness < 32896);
+
+	amdgpu_dm_set_dc_debug_mask(saved_mask);
+}
+
+/**
+ * dm_test_custom_brightness_below_first - Test Custom brightness below first
+ * @test: The KUnit test context
+ */
+static void dm_test_custom_brightness_below_first(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	uint32_t brightness;
+	unsigned int min, max;
+	uint saved_mask = amdgpu_dm_get_dc_debug_mask();
+
+	amdgpu_dm_set_dc_debug_mask(amdgpu_dm_get_dc_debug_mask() & ~DC_DISABLE_CUSTOM_BRIGHTNESS_CURVE);
+
+	caps.aux_support = false;
+	caps.min_input_signal = 0;
+	caps.max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+	caps.data_points = 2;
+	caps.luminance_data[0].input_signal = 100;
+	caps.luminance_data[0].luminance = 40;
+	caps.luminance_data[1].input_signal = 200;
+	caps.luminance_data[1].luminance = 80;
+
+	get_brightness_range(&caps, &min, &max);
+
+	/*
+	 * Set brightness low enough that scaled value < 100.
+	 * scale_input_to_fw(0, 65535, x) = DIV_ROUND_CLOSEST(x*255, 65535)
+	 * For result=50: x = 50*65535/255 = 12850
+	 */
+	brightness = 12850;
+
+	convert_custom_brightness(&caps, min, max, &brightness);
+
+	/*
+	 * Below first data point: lum = DIV_ROUND_CLOSEST(40 * 50, 100) = 20
+	 * Then: scale_fw_to_input(0, 65535, DIV_ROUND_CLOSEST(20 * 50, 101))
+	 *      = scale_fw_to_input(0, 65535, DIV_ROUND_CLOSEST(1000, 101))
+	 *      = scale_fw_to_input(0, 65535, 10)
+	 * The output should be significantly less than input.
+	 */
+	KUNIT_EXPECT_TRUE(test, brightness < 12850);
+
+	amdgpu_dm_set_dc_debug_mask(saved_mask);
+}
+
+/**
+ * dm_test_custom_brightness_interpolation - Test Custom brightness interpolation
+ * @test: The KUnit test context
+ */
+static void dm_test_custom_brightness_interpolation(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	uint32_t brightness;
+	unsigned int min, max;
+	uint saved_mask = amdgpu_dm_get_dc_debug_mask();
+
+	amdgpu_dm_set_dc_debug_mask(amdgpu_dm_get_dc_debug_mask() & ~DC_DISABLE_CUSTOM_BRIGHTNESS_CURVE);
+
+	caps.aux_support = false;
+	caps.min_input_signal = 0;
+	caps.max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+	caps.data_points = 2;
+	caps.luminance_data[0].input_signal = 50;
+	caps.luminance_data[0].luminance = 20;
+	caps.luminance_data[1].input_signal = 200;
+	caps.luminance_data[1].luminance = 80;
+
+	get_brightness_range(&caps, &min, &max);
+
+	/*
+	 * Choose a value between data points 50 and 200.
+	 * scale_input_to_fw(0, 65535, x) = 125 when x = 125*65535/255 = 32125
+	 */
+	brightness = 32125;
+
+	convert_custom_brightness(&caps, min, max, &brightness);
+
+	/*
+	 * The function should interpolate between data points and produce
+	 * a remapped value different from the input.
+	 */
+	KUNIT_EXPECT_TRUE(test, brightness != 32125);
+
+	amdgpu_dm_set_dc_debug_mask(saved_mask);
+}
+
+/**
+ * dm_test_custom_brightness_above_last - Test Custom brightness above last data point
+ * @test: The KUnit test context
+ */
+static void dm_test_custom_brightness_above_last(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	uint32_t brightness;
+	unsigned int min, max;
+	uint saved_mask = amdgpu_dm_get_dc_debug_mask();
+
+	amdgpu_dm_set_dc_debug_mask(amdgpu_dm_get_dc_debug_mask() & ~DC_DISABLE_CUSTOM_BRIGHTNESS_CURVE);
+
+	caps.aux_support = false;
+	caps.min_input_signal = 0;
+	caps.max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+	caps.data_points = 2;
+	caps.luminance_data[0].input_signal = 50;
+	caps.luminance_data[0].luminance = 20;
+	caps.luminance_data[1].input_signal = 150;
+	caps.luminance_data[1].luminance = 60;
+
+	get_brightness_range(&caps, &min, &max);
+
+	/*
+	 * Choose brightness above the last data point (150).
+	 * scale_input_to_fw(0, 65535, x) = 220 when x = 220*65535/255 = 56533
+	 * After binary search, left >= data_points, clamped → right==left,
+	 * so lum = upper_lum = 60.
+	 */
+	brightness = 56533;
+
+	convert_custom_brightness(&caps, min, max, &brightness);
+
+	/* Output should differ from input (remapped via curve) */
+	KUNIT_EXPECT_TRUE(test, brightness != 56533);
+	KUNIT_EXPECT_TRUE(test, brightness < 56533);
+
+	amdgpu_dm_set_dc_debug_mask(saved_mask);
+}
+
+/**
+ * dm_test_custom_brightness_single_data_point - Test Custom brightness with single data point
+ * @test: The KUnit test context
+ */
+static void dm_test_custom_brightness_single_data_point(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	uint32_t brightness;
+	unsigned int min, max;
+	uint saved_mask = amdgpu_dm_get_dc_debug_mask();
+
+	amdgpu_dm_set_dc_debug_mask(amdgpu_dm_get_dc_debug_mask() & ~DC_DISABLE_CUSTOM_BRIGHTNESS_CURVE);
+
+	caps.aux_support = false;
+	caps.min_input_signal = 0;
+	caps.max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+	caps.data_points = 1;
+	caps.luminance_data[0].input_signal = 128;
+	caps.luminance_data[0].luminance = 50;
+
+	get_brightness_range(&caps, &min, &max);
+
+	/*
+	 * Brightness below the single data point triggers the
+	 * "below first" path: lum = DIV_ROUND_CLOSEST(50 * scaled, 128).
+	 * scale_input_to_fw(0, 65535, x) = 64 when x = 64*65535/255 = 16448
+	 */
+	brightness = 16448;
+
+	convert_custom_brightness(&caps, min, max, &brightness);
+
+	KUNIT_EXPECT_TRUE(test, brightness < 16448);
+
+	amdgpu_dm_set_dc_debug_mask(saved_mask);
+}
+
+/**
+ * dm_test_custom_brightness_lower_lum_zero - Test Custom brightness with zero lower luminance
+ * @test: The KUnit test context
+ */
+static void dm_test_custom_brightness_lower_lum_zero(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	uint32_t brightness;
+	unsigned int min, max;
+	uint saved_mask = amdgpu_dm_get_dc_debug_mask();
+
+	amdgpu_dm_set_dc_debug_mask(amdgpu_dm_get_dc_debug_mask() & ~DC_DISABLE_CUSTOM_BRIGHTNESS_CURVE);
+
+	caps.aux_support = false;
+	caps.min_input_signal = 0;
+	caps.max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+	caps.data_points = 2;
+	caps.luminance_data[0].input_signal = 50;
+	caps.luminance_data[0].luminance = 0;	/* zero lower luminance */
+	caps.luminance_data[1].input_signal = 200;
+	caps.luminance_data[1].luminance = 80;
+
+	get_brightness_range(&caps, &min, &max);
+
+	/*
+	 * Choose brightness between data points to trigger interpolation.
+	 * scale_input_to_fw(0, 65535, x) = 125 when x = 125*65535/255 = 32125
+	 * With lower_lum == 0, code takes shortcut: lum = upper_lum = 80.
+	 */
+	brightness = 32125;
+
+	convert_custom_brightness(&caps, min, max, &brightness);
+
+	/* Should remap; result should differ from input */
+	KUNIT_EXPECT_TRUE(test, brightness != 32125);
+
+	amdgpu_dm_set_dc_debug_mask(saved_mask);
+}
+
+/**
+ * dm_test_brightness_to_user_above_max - Test Brightness to user above max
+ * @test: The KUnit test context
+ */
+static void dm_test_brightness_to_user_above_max(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	unsigned int min, max, result;
+
+	caps.aux_support = false;
+	caps.min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT;
+	caps.max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+
+	get_brightness_range(&caps, &min, &max);
+
+	/* brightness above max → result > max (linear extrapolation) */
+	result = convert_brightness_to_user(&caps, max + 1000);
+
+	KUNIT_EXPECT_GT(test, result, max);
+}
+
+/**
+ * dm_test_brightness_from_user_midrange - Test Brightness from user mid-range value
+ * @test: The KUnit test context
+ */
+static void dm_test_brightness_from_user_midrange(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	unsigned int min, max;
+	u32 result;
+
+	caps.aux_support = false;
+	caps.min_input_signal = AMDGPU_DM_DEFAULT_MIN_BACKLIGHT;
+	caps.max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+	caps.data_points = 0;
+
+	get_brightness_range(&caps, &min, &max);
+
+	/* Mid-range brightness should map to between min and max */
+	result = convert_brightness_from_user(&caps, max / 2);
+
+	KUNIT_EXPECT_GE(test, result, min);
+	KUNIT_EXPECT_LE(test, result, max);
+}
+
+/**
+ * dm_test_brightness_from_user_with_curve - Test Brightness from user with custom curve active
+ * @test: The KUnit test context
+ */
+static void dm_test_brightness_from_user_with_curve(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	unsigned int min, max;
+	u32 with_curve, without_curve;
+	uint saved_mask = amdgpu_dm_get_dc_debug_mask();
+
+	amdgpu_dm_set_dc_debug_mask(amdgpu_dm_get_dc_debug_mask() & ~DC_DISABLE_CUSTOM_BRIGHTNESS_CURVE);
+
+	caps.aux_support = false;
+	caps.min_input_signal = 0;
+	caps.max_input_signal = AMDGPU_DM_DEFAULT_MAX_BACKLIGHT;
+	caps.data_points = 2;
+	caps.luminance_data[0].input_signal = 50;
+	caps.luminance_data[0].luminance = 20;
+	caps.luminance_data[1].input_signal = 200;
+	caps.luminance_data[1].luminance = 80;
+
+	get_brightness_range(&caps, &min, &max);
+
+	with_curve = convert_brightness_from_user(&caps, max / 2);
+
+	/* Now disable the curve and compare */
+	amdgpu_dm_set_dc_debug_mask(amdgpu_dm_get_dc_debug_mask() | DC_DISABLE_CUSTOM_BRIGHTNESS_CURVE);
+	without_curve = convert_brightness_from_user(&caps, max / 2);
+
+	/* Custom curve should produce a different mapping */
+	KUNIT_EXPECT_NE(test, with_curve, without_curve);
+
+	amdgpu_dm_set_dc_debug_mask(saved_mask);
+}
+
+/**
+ * dm_test_brightness_range_zero_signals - Test Brightness range with zero min and max signals
+ * @test: The KUnit test context
+ */
+static void dm_test_brightness_range_zero_signals(struct kunit *test)
+{
+	struct amdgpu_dm_backlight_caps caps = {};
+	unsigned int min = 99, max = 99;
+
+	caps.aux_support = false;
+	caps.min_input_signal = 0;
+	caps.max_input_signal = 0;
+
+	/* Both signals zero → min=max=0 */
+	KUNIT_EXPECT_EQ(test, get_brightness_range(&caps, &min, &max), 1);
+	KUNIT_EXPECT_EQ(test, min, 0U);
+	KUNIT_EXPECT_EQ(test, max, 0U);
+}
+
+/* Tests for amdgpu_dm_backlight_fill_props() */
+
+/**
+ * dm_test_backlight_fill_props_ac_linear - Test AC brightness and linear scale
+ * @test: The KUnit test context
+ */
+static void dm_test_backlight_fill_props_ac_linear(struct kunit *test)
+{
+	struct backlight_properties props = {};
+	struct amdgpu_dm_backlight_caps caps = {};
+	unsigned int min, max;
+
+	caps.min_input_signal = 12;
+	caps.max_input_signal = 255;
+	caps.ac_level = 40;
+	caps.dc_level = 20;
+
+	get_brightness_range(&caps, &min, &max);
+	amdgpu_dm_backlight_fill_props(&caps, true, false, &props);
+
+	KUNIT_EXPECT_EQ(test, props.brightness,
+			 DIV_ROUND_CLOSEST(max * caps.ac_level, 100));
+	KUNIT_EXPECT_EQ(test, props.max_brightness, max);
+	KUNIT_EXPECT_EQ(test, props.scale, BACKLIGHT_SCALE_LINEAR);
+	KUNIT_EXPECT_EQ(test, props.type, BACKLIGHT_RAW);
+}
+
+/**
+ * dm_test_backlight_fill_props_dc_nonlinear - Test DC brightness and non-linear scale
+ * @test: The KUnit test context
+ */
+static void dm_test_backlight_fill_props_dc_nonlinear(struct kunit *test)
+{
+	struct backlight_properties props = {};
+	struct amdgpu_dm_backlight_caps caps = {};
+	unsigned int min, max;
+
+	caps.min_input_signal = 12;
+	caps.max_input_signal = 255;
+	caps.ac_level = 40;
+	caps.dc_level = 20;
+	caps.data_points = 2;
+
+	get_brightness_range(&caps, &min, &max);
+	amdgpu_dm_backlight_fill_props(&caps, false, true, &props);
+
+	KUNIT_EXPECT_EQ(test, props.brightness,
+			 DIV_ROUND_CLOSEST(max * caps.dc_level, 100));
+	KUNIT_EXPECT_EQ(test, props.max_brightness, max);
+	KUNIT_EXPECT_EQ(test, props.scale, BACKLIGHT_SCALE_NON_LINEAR);
+	KUNIT_EXPECT_EQ(test, props.type, BACKLIGHT_RAW);
+}
+
+/**
+ * dm_test_backlight_fill_props_default_range - Test default properties without caps
+ * @test: The KUnit test context
+ */
+static void dm_test_backlight_fill_props_default_range(struct kunit *test)
+{
+	struct backlight_properties props = {};
+
+	amdgpu_dm_backlight_fill_props(NULL, false, true, &props);
+
+	KUNIT_EXPECT_EQ(test, props.brightness, MAX_BACKLIGHT_LEVEL);
+	KUNIT_EXPECT_EQ(test, props.max_brightness, MAX_BACKLIGHT_LEVEL);
+	KUNIT_EXPECT_EQ(test, props.scale, BACKLIGHT_SCALE_LINEAR);
+	KUNIT_EXPECT_EQ(test, props.type, BACKLIGHT_RAW);
+}
+
+/* Tests for amdgpu_dm_update_connector_ext_caps() */
+
+/**
+ * dm_test_update_connector_ext_caps_negative_bl_idx - Test negative backlight index early return
+ * @test: The KUnit test context
+ */
+static void dm_test_update_connector_ext_caps_negative_bl_idx(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconnector);
+
+	aconnector->bl_idx = -1;
+
+	amdgpu_dm_update_connector_ext_caps(aconnector);
+
+	KUNIT_SUCCEED(test);
+}
+
+/**
+ * dm_test_update_connector_ext_caps_non_edp - Test non-eDP connector early return
+ * @test: The KUnit test context
+ */
+static void dm_test_update_connector_ext_caps_non_edp(struct kunit *test)
+{
+	struct dm_backlight_connector_fixture fixture = {};
+
+	setup_test_connector(test, &fixture, 0, SIGNAL_TYPE_HDMI_TYPE_A);
+	fixture.adev->dm.backlight_caps[0].aux_support = true;
+
+	amdgpu_dm_update_connector_ext_caps(fixture.aconnector);
+
+	KUNIT_EXPECT_TRUE(test, fixture.adev->dm.backlight_caps[0].aux_support);
+	KUNIT_EXPECT_PTR_EQ(test, fixture.adev->dm.backlight_caps[0].ext_caps, NULL);
+}
+
+/**
+ * dm_test_update_connector_ext_caps_oled_defaults - Test OLED eDP defaults to AUX backlight
+ * @test: The KUnit test context
+ */
+static void dm_test_update_connector_ext_caps_oled_defaults(struct kunit *test)
+{
+	struct dm_backlight_connector_fixture fixture = {};
+	int saved_backlight = amdgpu_dm_get_backlight_param();
+
+	amdgpu_dm_set_backlight_param(-1);
+	setup_test_connector(test, &fixture, 0, SIGNAL_TYPE_EDP);
+	fixture.link->dpcd_sink_ext_caps.bits.oled = 1;
+
+	amdgpu_dm_update_connector_ext_caps(fixture.aconnector);
+
+	KUNIT_EXPECT_PTR_EQ(test, fixture.adev->dm.backlight_caps[0].ext_caps,
+			    &fixture.link->dpcd_sink_ext_caps);
+	KUNIT_EXPECT_TRUE(test, fixture.adev->dm.backlight_caps[0].aux_support);
+	KUNIT_EXPECT_EQ(test, fixture.link->backlight_control_type,
+			BACKLIGHT_CONTROL_AMD_AUX);
+	KUNIT_EXPECT_EQ(test, fixture.adev->dm.backlight_caps[0].aux_max_input_signal, 512);
+	KUNIT_EXPECT_EQ(test, fixture.adev->dm.backlight_caps[0].aux_min_input_signal, 1);
+
+	amdgpu_dm_set_backlight_param(saved_backlight);
+}
+
+/**
+ * dm_test_update_connector_ext_caps_luminance_values - Test luminance range copy
+ * @test: The KUnit test context
+ */
+static void dm_test_update_connector_ext_caps_luminance_values(struct kunit *test)
+{
+	struct dm_backlight_connector_fixture fixture = {};
+	int saved_backlight = amdgpu_dm_get_backlight_param();
+
+	amdgpu_dm_set_backlight_param(-1);
+	setup_test_connector(test, &fixture, 0, SIGNAL_TYPE_EDP);
+	fixture.aconnector->base.display_info.luminance_range.min_luminance = 2;
+	fixture.aconnector->base.display_info.luminance_range.max_luminance = 400;
+
+	amdgpu_dm_update_connector_ext_caps(fixture.aconnector);
+
+	KUNIT_EXPECT_FALSE(test, fixture.adev->dm.backlight_caps[0].aux_support);
+	KUNIT_EXPECT_EQ(test, fixture.adev->dm.backlight_caps[0].aux_max_input_signal, 400);
+	KUNIT_EXPECT_EQ(test, fixture.adev->dm.backlight_caps[0].aux_min_input_signal, 2);
+
+	amdgpu_dm_set_backlight_param(saved_backlight);
+}
+
+/**
+ * dm_test_update_connector_ext_caps_force_aux - Test module parameter forces AUX backlight
+ * @test: The KUnit test context
+ */
+static void dm_test_update_connector_ext_caps_force_aux(struct kunit *test)
+{
+	struct dm_backlight_connector_fixture fixture = {};
+	int saved_backlight = amdgpu_dm_get_backlight_param();
+
+	amdgpu_dm_set_backlight_param(1);
+	setup_test_connector(test, &fixture, 0, SIGNAL_TYPE_EDP);
+
+	amdgpu_dm_update_connector_ext_caps(fixture.aconnector);
+
+	KUNIT_EXPECT_TRUE(test, fixture.adev->dm.backlight_caps[0].aux_support);
+	KUNIT_EXPECT_EQ(test, fixture.link->backlight_control_type,
+			BACKLIGHT_CONTROL_AMD_AUX);
+
+	amdgpu_dm_set_backlight_param(saved_backlight);
+}
+
+/**
+ * dm_test_update_connector_ext_caps_force_pwm - Test module parameter forces PWM backlight
+ * @test: The KUnit test context
+ */
+static void dm_test_update_connector_ext_caps_force_pwm(struct kunit *test)
+{
+	struct dm_backlight_connector_fixture fixture = {};
+	int saved_backlight = amdgpu_dm_get_backlight_param();
+
+	amdgpu_dm_set_backlight_param(0);
+	setup_test_connector(test, &fixture, 0, SIGNAL_TYPE_EDP);
+	fixture.link->dpcd_sink_ext_caps.bits.oled = 1;
+
+	amdgpu_dm_update_connector_ext_caps(fixture.aconnector);
+
+	KUNIT_EXPECT_FALSE(test, fixture.adev->dm.backlight_caps[0].aux_support);
+	KUNIT_EXPECT_NE(test, fixture.link->backlight_control_type,
+			BACKLIGHT_CONTROL_AMD_AUX);
+
+	amdgpu_dm_set_backlight_param(saved_backlight);
+}
+
+/* Tests for amdgpu_dm_should_create_sysfs() */
+
+/**
+ * dm_test_should_create_sysfs_abm_forced - Test forced ABM disables sysfs
+ * @test: The KUnit test context
+ */
+static void dm_test_should_create_sysfs_abm_forced(struct kunit *test)
+{
+	struct dm_backlight_connector_fixture fixture = {};
+	int saved_abm_level = amdgpu_dm_get_abm_level_param();
+
+	amdgpu_dm_set_abm_level_param(1);
+	setup_test_connector(test, &fixture, 0, SIGNAL_TYPE_EDP);
+	fixture.aconnector->base.connector_type = DRM_MODE_CONNECTOR_eDP;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_should_create_sysfs(fixture.aconnector));
+
+	amdgpu_dm_set_abm_level_param(saved_abm_level);
+}
+
+/**
+ * dm_test_should_create_sysfs_non_edp - Test non-eDP connector disables sysfs
+ * @test: The KUnit test context
+ */
+static void dm_test_should_create_sysfs_non_edp(struct kunit *test)
+{
+	struct dm_backlight_connector_fixture fixture = {};
+	int saved_abm_level = amdgpu_dm_get_abm_level_param();
+
+	amdgpu_dm_set_abm_level_param(-1);
+	setup_test_connector(test, &fixture, 0, SIGNAL_TYPE_HDMI_TYPE_A);
+	fixture.aconnector->base.connector_type = DRM_MODE_CONNECTOR_HDMIA;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_should_create_sysfs(fixture.aconnector));
+
+	amdgpu_dm_set_abm_level_param(saved_abm_level);
+}
+
+/**
+ * dm_test_should_create_sysfs_no_backlight_index - Test eDP without backlight index enables sysfs
+ * @test: The KUnit test context
+ */
+static void dm_test_should_create_sysfs_no_backlight_index(struct kunit *test)
+{
+	struct dm_backlight_connector_fixture fixture = {};
+	int saved_abm_level = amdgpu_dm_get_abm_level_param();
+
+	amdgpu_dm_set_abm_level_param(-1);
+	setup_test_connector(test, &fixture, -1, SIGNAL_TYPE_EDP);
+	fixture.aconnector->base.connector_type = DRM_MODE_CONNECTOR_eDP;
+	fixture.link->panel_type = PANEL_TYPE_LCD;
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_should_create_sysfs(fixture.aconnector));
+
+	amdgpu_dm_set_abm_level_param(saved_abm_level);
+}
+
+/**
+ * dm_test_should_create_sysfs_oled_no_cacp - Test OLED without CACP disables sysfs
+ * @test: The KUnit test context
+ *
+ * A non-LCD panel that does not support CACP must not expose the sysfs
+ * backlight interface.
+ */
+static void dm_test_should_create_sysfs_oled_no_cacp(struct kunit *test)
+{
+	struct dm_backlight_connector_fixture fixture = {};
+	int saved_abm_level = amdgpu_dm_get_abm_level_param();
+
+	amdgpu_dm_set_abm_level_param(-1);
+	setup_test_connector(test, &fixture, 0, SIGNAL_TYPE_EDP);
+	fixture.aconnector->base.connector_type = DRM_MODE_CONNECTOR_eDP;
+	fixture.link->panel_type = PANEL_TYPE_OLED;
+	fixture.link->panel_config.cacp.cacp_supported = false;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_should_create_sysfs(fixture.aconnector));
+
+	amdgpu_dm_set_abm_level_param(saved_abm_level);
+}
+
+/**
+ * dm_test_should_create_sysfs_oled_cacp - Test OLED with CACP enables sysfs
+ * @test: The KUnit test context
+ *
+ * An OLED panel that supports CACP must expose the sysfs backlight
+ * interface so the ABM/CACP level can be controlled.
+ */
+static void dm_test_should_create_sysfs_oled_cacp(struct kunit *test)
+{
+	struct dm_backlight_connector_fixture fixture = {};
+	int saved_abm_level = amdgpu_dm_get_abm_level_param();
+
+	amdgpu_dm_set_abm_level_param(-1);
+	setup_test_connector(test, &fixture, 0, SIGNAL_TYPE_EDP);
+	fixture.aconnector->base.connector_type = DRM_MODE_CONNECTOR_eDP;
+	fixture.link->panel_type = PANEL_TYPE_OLED;
+	fixture.link->panel_config.cacp.cacp_supported = true;
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_should_create_sysfs(fixture.aconnector));
+
+	amdgpu_dm_set_abm_level_param(saved_abm_level);
+}
+
+/**
+ * dm_test_should_create_sysfs_lcd_panel - Test LCD eDP panel enables sysfs
+ * @test: The KUnit test context
+ */
+static void dm_test_should_create_sysfs_lcd_panel(struct kunit *test)
+{
+	struct dm_backlight_connector_fixture fixture = {};
+	int saved_abm_level = amdgpu_dm_get_abm_level_param();
+
+	amdgpu_dm_set_abm_level_param(-1);
+	setup_test_connector(test, &fixture, 0, SIGNAL_TYPE_EDP);
+	fixture.aconnector->base.connector_type = DRM_MODE_CONNECTOR_eDP;
+	fixture.link->panel_type = PANEL_TYPE_LCD;
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_should_create_sysfs(fixture.aconnector));
+
+	amdgpu_dm_set_abm_level_param(saved_abm_level);
+}
+
+/* Tests for amdgpu_dm_setup_backlight_device() */
+
+/**
+ * dm_test_setup_backlight_device_non_edp - Test non-eDP/LVDS link is skipped
+ * @test: The KUnit test context
+ */
+static void dm_test_setup_backlight_device_non_edp(struct kunit *test)
+{
+	struct dm_backlight_connector_fixture fixture = {};
+	struct amdgpu_display_manager *dm;
+
+	setup_test_connector(test, &fixture, -1, SIGNAL_TYPE_HDMI_TYPE_A);
+	fixture.link->type = dc_connection_single;
+	dm = &fixture.adev->dm;
+	dm->adev = fixture.adev;
+	dm->num_of_edps = 0;
+
+	amdgpu_dm_setup_backlight_device(dm, fixture.aconnector);
+
+	/* Non-eDP/LVDS signal → no backlight setup */
+	KUNIT_EXPECT_EQ(test, dm->num_of_edps, 0);
+	KUNIT_EXPECT_EQ(test, fixture.aconnector->bl_idx, -1);
+}
+
+/**
+ * dm_test_setup_backlight_device_connection_none - Test disconnected link is skipped
+ * @test: The KUnit test context
+ */
+static void dm_test_setup_backlight_device_connection_none(struct kunit *test)
+{
+	struct dm_backlight_connector_fixture fixture = {};
+	struct amdgpu_display_manager *dm;
+
+	setup_test_connector(test, &fixture, -1, SIGNAL_TYPE_EDP);
+	fixture.link->type = dc_connection_none;
+	dm = &fixture.adev->dm;
+	dm->adev = fixture.adev;
+	dm->num_of_edps = 0;
+
+	amdgpu_dm_setup_backlight_device(dm, fixture.aconnector);
+
+	/* Disconnected link → no backlight setup */
+	KUNIT_EXPECT_EQ(test, dm->num_of_edps, 0);
+	KUNIT_EXPECT_EQ(test, fixture.aconnector->bl_idx, -1);
+}
+
+/**
+ * dm_test_setup_backlight_device_max_edps - Test setup is skipped when at eDP limit
+ * @test: The KUnit test context
+ */
+static void dm_test_setup_backlight_device_max_edps(struct kunit *test)
+{
+	struct dm_backlight_connector_fixture fixture = {};
+	struct amdgpu_display_manager *dm;
+
+	setup_test_connector(test, &fixture, -1, SIGNAL_TYPE_EDP);
+	fixture.link->type = dc_connection_single;
+	dm = &fixture.adev->dm;
+	dm->adev = fixture.adev;
+	dm->num_of_edps = AMDGPU_DM_MAX_NUM_EDP;
+
+	amdgpu_dm_setup_backlight_device(dm, fixture.aconnector);
+
+	/* Already at the eDP limit → no additional setup */
+	KUNIT_EXPECT_EQ(test, dm->num_of_edps, AMDGPU_DM_MAX_NUM_EDP);
+	KUNIT_EXPECT_EQ(test, fixture.aconnector->bl_idx, -1);
+}
+
+/**
+ * dm_test_setup_backlight_device_oled_success - Test successful eDP backlight setup
+ * @test: The KUnit test context
+ */
+static void dm_test_setup_backlight_device_oled_success(struct kunit *test)
+{
+	struct dm_backlight_connector_fixture fixture = {};
+	struct amdgpu_display_manager *dm;
+	int saved_backlight = amdgpu_dm_get_backlight_param();
+	int saved_abm_level = amdgpu_dm_get_abm_level_param();
+
+	amdgpu_dm_set_backlight_param(-1);
+	/* Skip ABM property attach (requires full DRM object setup) */
+	amdgpu_dm_set_abm_level_param(0);
+	setup_test_connector(test, &fixture, -1, SIGNAL_TYPE_EDP);
+	fixture.link->type = dc_connection_single;
+	fixture.link->dpcd_sink_ext_caps.bits.oled = 1;
+	dm = &fixture.adev->dm;
+	dm->adev = fixture.adev;
+	dm->num_of_edps = 0;
+
+	amdgpu_dm_setup_backlight_device(dm, fixture.aconnector);
+
+	KUNIT_EXPECT_EQ(test, dm->num_of_edps, 1);
+	KUNIT_EXPECT_EQ(test, fixture.aconnector->bl_idx, 0);
+	KUNIT_EXPECT_PTR_EQ(test, (void *)dm->backlight_link[0],
+			    (void *)fixture.link);
+	KUNIT_EXPECT_TRUE(test, dm->backlight_caps[0].aux_support);
+
+	amdgpu_dm_set_backlight_param(saved_backlight);
+	amdgpu_dm_set_abm_level_param(saved_abm_level);
+}
+
+/**
+ * dm_test_setup_backlight_device_attaches_abm_property - Test ABM property path
+ * @test: The KUnit test context
+ */
+static void dm_test_setup_backlight_device_attaches_abm_property(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_display_manager *dm;
+	struct amdgpu_device *adev;
+	struct drm_property *prop;
+	struct dc_link *link;
+	int saved_abm_level = amdgpu_dm_get_abm_level_param();
+	int saved_backlight = amdgpu_dm_get_backlight_param();
+	int old_count;
+	int ret;
+
+	amdgpu_dm_set_abm_level_param(-1);
+	amdgpu_dm_set_backlight_param(-1);
+	adev = dm_kunit_alloc_adev(test);
+	ret = drmm_mode_config_init(&adev->ddev);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	prop = drm_property_create_range(&adev->ddev, 0, "abm level", 0, 4);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, prop);
+	adev->mode_info.abm_level_property = prop;
+
+	aconnector = dm_kunit_alloc_connector(test, adev, NULL);
+	ret = drmm_connector_init(&adev->ddev, &aconnector->base,
+				   &dm_backlight_test_connector_funcs,
+				   DRM_MODE_CONNECTOR_eDP, NULL);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	link = dm_kunit_alloc_link(test);
+	link->connector_signal = SIGNAL_TYPE_EDP;
+	link->type = dc_connection_single;
+	aconnector->dc_link = link;
+	aconnector->bl_idx = -1;
+	dm = &adev->dm;
+	dm->adev = adev;
+	dm->ddev = &adev->ddev;
+	old_count = aconnector->base.base.properties->count;
+
+	amdgpu_dm_setup_backlight_device(dm, aconnector);
+
+	KUNIT_EXPECT_EQ(test, dm->num_of_edps, 1);
+	KUNIT_EXPECT_EQ(test, aconnector->bl_idx, 0);
+	KUNIT_EXPECT_EQ(test, aconnector->base.base.properties->count, old_count + 1);
+	KUNIT_EXPECT_PTR_EQ(test, aconnector->base.base.properties->properties[old_count], prop);
+	KUNIT_EXPECT_EQ(test, aconnector->base.base.properties->values[old_count],
+			 (uint64_t)ABM_SYSFS_CONTROL);
+
+	amdgpu_dm_set_backlight_param(saved_backlight);
+	amdgpu_dm_set_abm_level_param(saved_abm_level);
+}
+
+static struct kunit_case dm_backlight_test_cases[] = {
+	/* dm_find_stream_with_link */
+	KUNIT_CASE(dm_test_find_stream_with_link_returns_match),
+	KUNIT_CASE(dm_test_find_stream_with_link_missing),
+	/* amdgpu_dm_backlight_set_level / update_status */
+	KUNIT_CASE(dm_test_backlight_set_level_connector_off),
+	KUNIT_CASE(dm_test_backlight_set_level_no_stream),
+	KUNIT_CASE(dm_test_backlight_set_level_aux_programs_power_module),
+	KUNIT_CASE(dm_test_backlight_set_level_pwm_programs_power_module),
+	KUNIT_CASE(dm_test_backlight_set_level_reallows_idle),
+	KUNIT_CASE(dm_test_backlight_update_status_no_stream),
+	/* amdgpu_dm_backlight_get_level / get_brightness */
+	KUNIT_CASE(dm_test_backlight_get_level_pwm_success),
+	KUNIT_CASE(dm_test_backlight_get_level_pwm_error),
+	KUNIT_CASE(dm_test_backlight_get_level_aux_success),
+	KUNIT_CASE(dm_test_backlight_get_level_aux_error),
+	KUNIT_CASE(dm_test_backlight_get_brightness_uses_device_index),
+	/* amdgpu_dm_register_backlight_device */
+	KUNIT_CASE(dm_test_register_backlight_device_negative_index),
+	/* panel_power_savings_show / store */
+	KUNIT_CASE(dm_test_panel_power_savings_show_maps_disable_to_zero),
+	KUNIT_CASE(dm_test_panel_power_savings_show_reports_level),
+	KUNIT_CASE(dm_test_panel_power_savings_store_sets_disable),
+	KUNIT_CASE(dm_test_panel_power_savings_store_forbidden),
+	KUNIT_CASE(dm_test_panel_power_savings_store_rejects_invalid_text),
+	KUNIT_CASE(dm_test_panel_power_savings_store_rejects_out_of_range),
+	/* amdgpu_dm_backlight_get_device_index */
+	KUNIT_CASE(dm_test_backlight_device_index_matches_second),
+	KUNIT_CASE(dm_test_backlight_device_index_missing_fallback),
+	KUNIT_CASE(dm_test_backlight_caps_valid_short_circuit),
+#if !defined(CONFIG_ACPI)
+	KUNIT_CASE(dm_test_backlight_caps_aux_support_noop),
+	KUNIT_CASE(dm_test_backlight_caps_non_aux_sets_defaults),
+#endif
+	/* get_brightness_range */
+	KUNIT_CASE(dm_test_brightness_range_null_caps),
+	KUNIT_CASE(dm_test_brightness_range_pwm),
+	KUNIT_CASE(dm_test_brightness_range_aux),
+	/* convert_brightness_to_user */
+	KUNIT_CASE(dm_test_brightness_to_user_null_caps),
+	KUNIT_CASE(dm_test_brightness_to_user_below_min),
+	KUNIT_CASE(dm_test_brightness_to_user_at_max),
+	KUNIT_CASE(dm_test_brightness_to_user_at_min),
+	KUNIT_CASE(dm_test_brightness_to_user_midpoint_pwm),
+	/* convert_brightness_from_user */
+	KUNIT_CASE(dm_test_brightness_from_user_null_caps),
+	KUNIT_CASE(dm_test_brightness_from_user_zero),
+	KUNIT_CASE(dm_test_brightness_from_user_max),
+	KUNIT_CASE(dm_test_brightness_from_user_aux),
+	/* convert_custom_brightness */
+	KUNIT_CASE(dm_test_custom_brightness_no_data_points),
+	KUNIT_CASE(dm_test_custom_brightness_debug_mask_disables),
+	KUNIT_CASE(dm_test_custom_brightness_exact_match),
+	KUNIT_CASE(dm_test_custom_brightness_below_first),
+	KUNIT_CASE(dm_test_custom_brightness_interpolation),
+	KUNIT_CASE(dm_test_custom_brightness_above_last),
+	KUNIT_CASE(dm_test_custom_brightness_single_data_point),
+	KUNIT_CASE(dm_test_custom_brightness_lower_lum_zero),
+	KUNIT_CASE(dm_test_brightness_to_user_above_max),
+	KUNIT_CASE(dm_test_brightness_from_user_midrange),
+	KUNIT_CASE(dm_test_brightness_from_user_with_curve),
+	KUNIT_CASE(dm_test_brightness_range_zero_signals),
+	/* amdgpu_dm_backlight_fill_props */
+	KUNIT_CASE(dm_test_backlight_fill_props_ac_linear),
+	KUNIT_CASE(dm_test_backlight_fill_props_dc_nonlinear),
+	KUNIT_CASE(dm_test_backlight_fill_props_default_range),
+	/* amdgpu_dm_update_connector_ext_caps */
+	KUNIT_CASE(dm_test_update_connector_ext_caps_negative_bl_idx),
+	KUNIT_CASE(dm_test_update_connector_ext_caps_non_edp),
+	KUNIT_CASE(dm_test_update_connector_ext_caps_oled_defaults),
+	KUNIT_CASE(dm_test_update_connector_ext_caps_luminance_values),
+	KUNIT_CASE(dm_test_update_connector_ext_caps_force_aux),
+	KUNIT_CASE(dm_test_update_connector_ext_caps_force_pwm),
+	/* amdgpu_dm_should_create_sysfs */
+	KUNIT_CASE(dm_test_should_create_sysfs_abm_forced),
+	KUNIT_CASE(dm_test_should_create_sysfs_non_edp),
+	KUNIT_CASE(dm_test_should_create_sysfs_no_backlight_index),
+	KUNIT_CASE(dm_test_should_create_sysfs_oled_no_cacp),
+	KUNIT_CASE(dm_test_should_create_sysfs_oled_cacp),
+	KUNIT_CASE(dm_test_should_create_sysfs_lcd_panel),
+	/* amdgpu_dm_setup_backlight_device */
+	KUNIT_CASE(dm_test_setup_backlight_device_non_edp),
+	KUNIT_CASE(dm_test_setup_backlight_device_connection_none),
+	KUNIT_CASE(dm_test_setup_backlight_device_max_edps),
+	KUNIT_CASE(dm_test_setup_backlight_device_oled_success),
+	KUNIT_CASE(dm_test_setup_backlight_device_attaches_abm_property),
+	{}
+};
+
+static struct kunit_suite dm_backlight_test_suite = {
+	.name = "amdgpu_dm_backlight",
+	.test_cases = dm_backlight_test_cases,
+};
+
+kunit_test_suite(dm_backlight_test_suite);
+
+MODULE_LICENSE("Dual MIT/GPL");
+MODULE_DESCRIPTION("KUnit tests for amdgpu_dm_backlight");
+MODULE_AUTHOR("AMD");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_color_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_color_test.c
index f943361b70e82..d64c7da20f2cd 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_color_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_color_test.c
@@ -1159,19 +1159,19 @@ static void dm_test_verify_lut_sizes_invalid_degamma_valid_gamma(struct kunit *t
  */
 static void dm_test_atomic_lut3d_zero_size(struct kunit *test)
 {
-	struct dc_3dlut *lut;
+	struct dc_plane_cm *cm;
 	u32 initialized;
 
-	lut = kunit_kzalloc(test, sizeof(*lut), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, lut);
+	cm = kunit_kzalloc(test, sizeof(*cm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, cm);
 
 	/* Pre-set initialized so we can confirm it is cleared */
-	lut->state.bits.initialized = 1;
+	cm->lut3d_func.state.bits.initialized = 1;
 
-	amdgpu_dm_atomic_lut3d(NULL, 0, lut);
+	amdgpu_dm_atomic_lut3d(NULL, 0, cm);
 
 	/* Copy bit-field: typeof cannot be applied to a bit-field */
-	initialized = lut->state.bits.initialized;
+	initialized = cm->lut3d_func.state.bits.initialized;
 	KUNIT_EXPECT_EQ(test, initialized, 0U);
 }
 
@@ -1183,22 +1183,22 @@ static void dm_test_atomic_lut3d_nonzero_state_bits(struct kunit *test)
 {
 	const uint32_t lut3d_size = 5;
 	struct drm_color_lut *lut_data;
-	struct dc_3dlut *lut;
+	struct dc_plane_cm *cm;
 	u32 initialized;
 
 	lut_data = kunit_kcalloc(test, lut3d_size, sizeof(*lut_data), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, lut_data);
 
-	lut = kunit_kzalloc(test, sizeof(*lut), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, lut);
+	cm = kunit_kzalloc(test, sizeof(*cm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, cm);
 
-	amdgpu_dm_atomic_lut3d(lut_data, lut3d_size, lut);
+	amdgpu_dm_atomic_lut3d(lut_data, lut3d_size, cm);
 
 	/* Copy bit-field: typeof cannot be applied to a bit-field */
-	initialized = lut->state.bits.initialized;
+	initialized = cm->lut3d_func.state.bits.initialized;
 	KUNIT_EXPECT_EQ(test, initialized, 1U);
-	KUNIT_EXPECT_FALSE(test, lut->lut_3d.use_tetrahedral_9);
-	KUNIT_EXPECT_TRUE(test, lut->lut_3d.use_12bits);
+	KUNIT_EXPECT_FALSE(test, cm->lut3d_func.lut_3d.use_tetrahedral_9);
+	KUNIT_EXPECT_TRUE(test, cm->lut3d_func.lut_3d.use_12bits);
 }
 
 /**
@@ -1209,29 +1209,29 @@ static void dm_test_atomic_lut3d_data_forwarded(struct kunit *test)
 {
 	const uint32_t lut3d_size = 5;
 	struct drm_color_lut *lut_data;
-	struct dc_3dlut *lut;
+	struct dc_plane_cm *cm;
 
 	lut_data = kunit_kcalloc(test, lut3d_size, sizeof(*lut_data), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, lut_data);
 
-	lut = kunit_kzalloc(test, sizeof(*lut), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, lut);
+	cm = kunit_kzalloc(test, sizeof(*cm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, cm);
 
 	lut_data[0].red   = 0xFFFF;
 	lut_data[0].green = 0x8000;
 	lut_data[0].blue  = 0x4000;
 
-	amdgpu_dm_atomic_lut3d(lut_data, lut3d_size, lut);
+	amdgpu_dm_atomic_lut3d(lut_data, lut3d_size, cm);
 
 	/*
 	 * use_tetrahedral_9 == false → data goes into tetrahedral_17.
 	 * lut[0] maps to lut0[0] (first element of the first group).
 	 */
-	KUNIT_EXPECT_EQ(test, lut->lut_3d.tetrahedral_17.lut0[0].red,
+	KUNIT_EXPECT_EQ(test, cm->lut3d_func.lut_3d.tetrahedral_17.lut0[0].red,
 			drm_color_lut_extract(0xFFFF, MAX_COLOR_3DLUT_BITDEPTH));
-	KUNIT_EXPECT_EQ(test, lut->lut_3d.tetrahedral_17.lut0[0].green,
+	KUNIT_EXPECT_EQ(test, cm->lut3d_func.lut_3d.tetrahedral_17.lut0[0].green,
 			drm_color_lut_extract(0x8000, MAX_COLOR_3DLUT_BITDEPTH));
-	KUNIT_EXPECT_EQ(test, lut->lut_3d.tetrahedral_17.lut0[0].blue,
+	KUNIT_EXPECT_EQ(test, cm->lut3d_func.lut_3d.tetrahedral_17.lut0[0].blue,
 			drm_color_lut_extract(0x4000, MAX_COLOR_3DLUT_BITDEPTH));
 }
 
@@ -1398,19 +1398,19 @@ static void dm_test_set_atomic_regamma_bypass(struct kunit *test)
  */
 static void dm_test_atomic_shaper_lut_bypass(struct kunit *test)
 {
-	struct dc_transfer_func *func_shaper;
+	struct dc_plane_cm *cm;
 
-	func_shaper = kunit_kzalloc(test, sizeof(*func_shaper), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, func_shaper);
+	cm = kunit_kzalloc(test, sizeof(*cm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, cm);
 
 	/* size=0 and tf=LINEAR: must take the bypass branch */
 	KUNIT_EXPECT_EQ(test,
 		amdgpu_dm_atomic_shaper_lut(NULL, false,
 					    TRANSFER_FUNCTION_LINEAR,
-					    0, func_shaper),
+					    0, cm),
 		0);
-	KUNIT_EXPECT_EQ(test, (int)func_shaper->type, (int)TF_TYPE_BYPASS);
-	KUNIT_EXPECT_EQ(test, (int)func_shaper->tf, (int)TRANSFER_FUNCTION_LINEAR);
+	KUNIT_EXPECT_EQ(test, (int)cm->shaper_func.type, (int)TF_TYPE_BYPASS);
+	KUNIT_EXPECT_EQ(test, (int)cm->shaper_func.tf, (int)TRANSFER_FUNCTION_LINEAR);
 }
 
 /**
@@ -1419,19 +1419,19 @@ static void dm_test_atomic_shaper_lut_bypass(struct kunit *test)
  */
 static void dm_test_atomic_blend_lut_bypass(struct kunit *test)
 {
-	struct dc_transfer_func *func_blend;
+	struct dc_plane_cm *cm;
 
-	func_blend = kunit_kzalloc(test, sizeof(*func_blend), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, func_blend);
+	cm = kunit_kzalloc(test, sizeof(*cm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, cm);
 
 	/* size=0 and tf=LINEAR: must take the bypass branch */
 	KUNIT_EXPECT_EQ(test,
 		amdgpu_dm_atomic_blend_lut(NULL, false,
 					   TRANSFER_FUNCTION_LINEAR,
-					   0, func_blend),
+					   0, cm),
 		0);
-	KUNIT_EXPECT_EQ(test, (int)func_blend->type, (int)TF_TYPE_BYPASS);
-	KUNIT_EXPECT_EQ(test, (int)func_blend->tf, (int)TRANSFER_FUNCTION_LINEAR);
+	KUNIT_EXPECT_EQ(test, (int)cm->blend_func.type, (int)TF_TYPE_BYPASS);
+	KUNIT_EXPECT_EQ(test, (int)cm->blend_func.tf, (int)TRANSFER_FUNCTION_LINEAR);
 }
 
 /* ---- Tests for __set_colorop_in_tf_1d_curve ---- */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_colorop_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_colorop_test.c
index fa270ff28c6aa..2e557ff66818b 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_colorop_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_colorop_test.c
@@ -9,7 +9,10 @@
 #include <drm/drm_colorop.h>
 #include <drm/drm_kunit_helpers.h>
 
+#include "dc.h"
+#include "amdgpu.h"
 #include "amdgpu_dm_colorop.h"
+#include "amdgpu_dm_kunit_test_helpers.h"
 
 /* Tests for amdgpu_dm_supported_degam_tfs */
 
@@ -133,6 +136,30 @@ static void kunit_colorop_pipeline_destroy(void *drm)
 	drm_colorop_pipeline_destroy((struct drm_device *)drm);
 }
 
+static void dm_expect_colorop_pipeline(struct kunit *test, struct drm_device *drm,
+				       const struct drm_prop_enum_list *list,
+				       const enum drm_colorop_type *expected,
+				       int expected_count)
+{
+	struct drm_colorop *op, *first = NULL;
+	int i = 0;
+
+	drm_for_each_colorop(op, drm) {
+		if (op->base.id == (uint32_t)list->type) {
+			first = op;
+			break;
+		}
+	}
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, first);
+
+	for (op = first; op; op = op->next, i++) {
+		KUNIT_ASSERT_LT(test, i, expected_count);
+		KUNIT_EXPECT_EQ(test, op->type, expected[i]);
+		KUNIT_EXPECT_NOT_NULL(test, op->bypass_property);
+	}
+	KUNIT_EXPECT_EQ(test, i, expected_count);
+}
+
 /**
  * dm_test_initialize_default_pipeline() - Verify amdgpu_dm_build_default_pipeline()
  *   produces the expected colorop chain with all ops bypassable.
@@ -154,8 +181,6 @@ static void dm_test_initialize_default_pipeline(struct kunit *test)
 	struct drm_device *drm;
 	struct drm_plane *plane;
 	struct drm_prop_enum_list list = {};
-	struct drm_colorop *op, *first = NULL;
-	int i = 0;
 	int ret;
 
 	dev = drm_kunit_helper_alloc_device(test);
@@ -185,20 +210,102 @@ static void dm_test_initialize_default_pipeline(struct kunit *test)
 	KUNIT_ASSERT_EQ(test, ret, 0);
 	kfree(list.name);
 
-	drm_for_each_colorop(op, drm) {
-		if (op->base.id == (uint32_t)list.type) {
-			first = op;
-			break;
-		}
-	}
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, first);
+	dm_expect_colorop_pipeline(test, drm, &list, expected, ARRAY_SIZE(expected));
+}
 
-	for (op = first; op; op = op->next, i++) {
-		KUNIT_ASSERT_LT(test, i, (int)ARRAY_SIZE(expected));
-		KUNIT_EXPECT_EQ(test, op->type, expected[i]);
-		KUNIT_EXPECT_NOT_NULL(test, op->bypass_property);
-	}
-	KUNIT_EXPECT_EQ(test, i, (int)ARRAY_SIZE(expected));
+static void dm_test_initialize_default_pipeline_caps(struct kunit *test,
+					     bool dpp_hw_3d_lut,
+					     bool mpc_preblend,
+					     const enum drm_colorop_type *expected,
+					     int expected_count)
+{
+	struct drm_prop_enum_list list = {};
+	struct amdgpu_device *adev;
+	struct drm_device *drm;
+	struct drm_plane *plane;
+	struct dc *dc;
+	int ret;
+
+	adev = dm_kunit_alloc_adev(test);
+	drm = &adev->ddev;
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	adev->dm.dc = dc;
+	adev->dm.dc->caps.color.dpp.hw_3d_lut = dpp_hw_3d_lut;
+	adev->dm.dc->caps.color.mpc.preblend = mpc_preblend;
+
+	plane = drm_kunit_helper_create_primary_plane(test, drm,
+						       NULL, NULL, NULL, 0, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, plane);
+
+	kunit_add_action(test, kunit_colorop_pipeline_destroy, drm);
+
+	ret = amdgpu_dm_initialize_default_pipeline(plane, &list);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	kfree(list.name);
+
+	dm_expect_colorop_pipeline(test, drm, &list, expected, expected_count);
+}
+
+/**
+ * dm_test_initialize_default_pipeline_dpp_3d_lut() - Test DPP 3D LUT cap.
+ * @test: KUnit test context.
+ */
+static void dm_test_initialize_default_pipeline_dpp_3d_lut(struct kunit *test)
+{
+	static const enum drm_colorop_type expected[] = {
+		DRM_COLOROP_1D_CURVE,
+		DRM_COLOROP_MULTIPLIER,
+		DRM_COLOROP_CTM_3X4,
+		DRM_COLOROP_1D_CURVE,
+		DRM_COLOROP_1D_LUT,
+		DRM_COLOROP_3D_LUT,
+		DRM_COLOROP_1D_CURVE,
+		DRM_COLOROP_1D_LUT,
+	};
+
+	dm_test_initialize_default_pipeline_caps(test, true, false,
+						 expected, ARRAY_SIZE(expected));
+}
+
+/**
+ * dm_test_initialize_default_pipeline_mpc_preblend() - Test MPC preblend cap.
+ * @test: KUnit test context.
+ */
+static void dm_test_initialize_default_pipeline_mpc_preblend(struct kunit *test)
+{
+	static const enum drm_colorop_type expected[] = {
+		DRM_COLOROP_1D_CURVE,
+		DRM_COLOROP_MULTIPLIER,
+		DRM_COLOROP_CTM_3X4,
+		DRM_COLOROP_1D_CURVE,
+		DRM_COLOROP_1D_LUT,
+		DRM_COLOROP_3D_LUT,
+		DRM_COLOROP_1D_CURVE,
+		DRM_COLOROP_1D_LUT,
+	};
+
+	dm_test_initialize_default_pipeline_caps(test, false, true,
+						 expected, ARRAY_SIZE(expected));
+}
+
+/**
+ * dm_test_initialize_default_pipeline_no_3d_lut() - Test no 3D LUT caps.
+ * @test: KUnit test context.
+ */
+static void dm_test_initialize_default_pipeline_no_3d_lut(struct kunit *test)
+{
+	static const enum drm_colorop_type expected[] = {
+		DRM_COLOROP_1D_CURVE,
+		DRM_COLOROP_MULTIPLIER,
+		DRM_COLOROP_CTM_3X4,
+		DRM_COLOROP_1D_CURVE,
+		DRM_COLOROP_1D_LUT,
+	};
+
+	dm_test_initialize_default_pipeline_caps(test, false, false,
+						 expected, ARRAY_SIZE(expected));
 }
 
 static struct kunit_case dm_colorop_test_cases[] = {
@@ -224,6 +331,9 @@ static struct kunit_case dm_colorop_test_cases[] = {
 	KUNIT_CASE(dm_test_degam_and_blnd_tfs_match),
 	/* amdgpu_dm_initialize_default_pipeline */
 	KUNIT_CASE(dm_test_initialize_default_pipeline),
+	KUNIT_CASE(dm_test_initialize_default_pipeline_dpp_3d_lut),
+	KUNIT_CASE(dm_test_initialize_default_pipeline_mpc_preblend),
+	KUNIT_CASE(dm_test_initialize_default_pipeline_no_3d_lut),
 	{}
 };
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_connector_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_connector_test.c
new file mode 100644
index 0000000000000..0a5d439f66c36
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_connector_test.c
@@ -0,0 +1,3813 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * KUnit tests for amdgpu_dm_connector.c
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#include <kunit/test.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_state_helper.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_kunit_helpers.h>
+#include <drm/drm_mode_object.h>
+#include <drm/drm_property.h>
+#include <linux/hdmi.h>
+
+#include "dc.h"
+#include "amdgpu.h"
+#include "amdgpu_mode.h"
+#include "amdgpu_display.h"
+#include "amdgpu_dm.h"
+#include "amdgpu_dm_connector.h"
+#include "amdgpu_dm_backlight.h"
+#include "include/grph_object_id.h"
+
+/* Tests for get_subconnector_type() */
+
+/**
+ * dm_test_subconnector_type_none - Test Subconnector type none
+ * @test: The KUnit test context
+ */
+static void dm_test_subconnector_type_none(struct kunit *test)
+{
+	struct dc_link *link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link->dpcd_caps.dongle_type = DISPLAY_DONGLE_NONE;
+	KUNIT_EXPECT_EQ(test, (int)get_subconnector_type(link), (int)DRM_MODE_SUBCONNECTOR_Native);
+}
+
+/**
+ * dm_test_subconnector_type_vga - Test Subconnector type vga
+ * @test: The KUnit test context
+ */
+static void dm_test_subconnector_type_vga(struct kunit *test)
+{
+	struct dc_link *link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_VGA_CONVERTER;
+	KUNIT_EXPECT_EQ(test, (int)get_subconnector_type(link), (int)DRM_MODE_SUBCONNECTOR_VGA);
+}
+
+/**
+ * dm_test_subconnector_type_dvi_converter - Test Subconnector type dvi converter
+ * @test: The KUnit test context
+ */
+static void dm_test_subconnector_type_dvi_converter(struct kunit *test)
+{
+	struct dc_link *link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_DVI_CONVERTER;
+	KUNIT_EXPECT_EQ(test, (int)get_subconnector_type(link), (int)DRM_MODE_SUBCONNECTOR_DVID);
+}
+
+/**
+ * dm_test_subconnector_type_dvi_dongle - Test Subconnector type dvi dongle
+ * @test: The KUnit test context
+ */
+static void dm_test_subconnector_type_dvi_dongle(struct kunit *test)
+{
+	struct dc_link *link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_DVI_DONGLE;
+	KUNIT_EXPECT_EQ(test, (int)get_subconnector_type(link), (int)DRM_MODE_SUBCONNECTOR_DVID);
+}
+
+/**
+ * dm_test_subconnector_type_hdmi_converter - Test Subconnector type hdmi converter
+ * @test: The KUnit test context
+ */
+static void dm_test_subconnector_type_hdmi_converter(struct kunit *test)
+{
+	struct dc_link *link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_HDMI_CONVERTER;
+	KUNIT_EXPECT_EQ(test, (int)get_subconnector_type(link), (int)DRM_MODE_SUBCONNECTOR_HDMIA);
+}
+
+/**
+ * dm_test_subconnector_type_hdmi_dongle - Test Subconnector type hdmi dongle
+ * @test: The KUnit test context
+ */
+static void dm_test_subconnector_type_hdmi_dongle(struct kunit *test)
+{
+	struct dc_link *link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_HDMI_DONGLE;
+	KUNIT_EXPECT_EQ(test, (int)get_subconnector_type(link), (int)DRM_MODE_SUBCONNECTOR_HDMIA);
+}
+
+/**
+ * dm_test_subconnector_type_mismatched - Test Subconnector type mismatched
+ * @test: The KUnit test context
+ */
+static void dm_test_subconnector_type_mismatched(struct kunit *test)
+{
+	struct dc_link *link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_HDMI_MISMATCHED_DONGLE;
+	KUNIT_EXPECT_EQ(test, (int)get_subconnector_type(link), (int)DRM_MODE_SUBCONNECTOR_Unknown);
+}
+
+/**
+ * dm_test_subconnector_type_default_unknown - Test Subconnector type default unknown
+ * @test: The KUnit test context
+ */
+static void dm_test_subconnector_type_default_unknown(struct kunit *test)
+{
+	struct dc_link *link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link->dpcd_caps.dongle_type = (typeof(link->dpcd_caps.dongle_type))0x7f;
+	KUNIT_EXPECT_EQ(test, (int)get_subconnector_type(link), (int)DRM_MODE_SUBCONNECTOR_Unknown);
+}
+
+/* Tests for get_output_content_type() */
+
+/**
+ * dm_test_content_type_no_data - Test Content type no data
+ * @test: The KUnit test context
+ */
+static void dm_test_content_type_no_data(struct kunit *test)
+{
+	struct drm_connector_state state = {};
+
+	state.content_type = DRM_MODE_CONTENT_TYPE_NO_DATA;
+	KUNIT_EXPECT_EQ(test, (int)get_output_content_type(&state), (int)DISPLAY_CONTENT_TYPE_NO_DATA);
+}
+
+/**
+ * dm_test_content_type_graphics - Test Content type graphics
+ * @test: The KUnit test context
+ */
+static void dm_test_content_type_graphics(struct kunit *test)
+{
+	struct drm_connector_state state = {};
+
+	state.content_type = DRM_MODE_CONTENT_TYPE_GRAPHICS;
+	KUNIT_EXPECT_EQ(test, (int)get_output_content_type(&state), (int)DISPLAY_CONTENT_TYPE_GRAPHICS);
+}
+
+/**
+ * dm_test_content_type_photo - Test Content type photo
+ * @test: The KUnit test context
+ */
+static void dm_test_content_type_photo(struct kunit *test)
+{
+	struct drm_connector_state state = {};
+
+	state.content_type = DRM_MODE_CONTENT_TYPE_PHOTO;
+	KUNIT_EXPECT_EQ(test, (int)get_output_content_type(&state), (int)DISPLAY_CONTENT_TYPE_PHOTO);
+}
+
+/**
+ * dm_test_content_type_cinema - Test Content type cinema
+ * @test: The KUnit test context
+ */
+static void dm_test_content_type_cinema(struct kunit *test)
+{
+	struct drm_connector_state state = {};
+
+	state.content_type = DRM_MODE_CONTENT_TYPE_CINEMA;
+	KUNIT_EXPECT_EQ(test, (int)get_output_content_type(&state), (int)DISPLAY_CONTENT_TYPE_CINEMA);
+}
+
+/**
+ * dm_test_content_type_game - Test Content type game
+ * @test: The KUnit test context
+ */
+static void dm_test_content_type_game(struct kunit *test)
+{
+	struct drm_connector_state state = {};
+
+	state.content_type = DRM_MODE_CONTENT_TYPE_GAME;
+	KUNIT_EXPECT_EQ(test, (int)get_output_content_type(&state), (int)DISPLAY_CONTENT_TYPE_GAME);
+}
+
+/**
+ * dm_test_content_type_unknown_defaults_no_data - Test unknown content type defaults to no data
+ * @test: The KUnit test context
+ */
+static void dm_test_content_type_unknown_defaults_no_data(struct kunit *test)
+{
+	struct drm_connector_state state = {};
+
+	state.content_type = 0x7f;
+	KUNIT_EXPECT_EQ(test, (int)get_output_content_type(&state),
+			(int)DISPLAY_CONTENT_TYPE_NO_DATA);
+}
+
+/* Tests for adjust_colour_depth_from_display_info() */
+
+/**
+ * dm_test_adjust_colour_depth_fits_at_888 - Test Adjust colour depth fits at 888
+ * @test: The KUnit test context
+ */
+static void dm_test_adjust_colour_depth_fits_at_888(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_display_info info = {};
+
+	/* 1080p @ 148500 KHz = 1485000 in 100Hz units */
+	timing.pix_clk_100hz = 1485000;
+	timing.display_color_depth = COLOR_DEPTH_888;
+	timing.pixel_encoding = PIXEL_ENCODING_RGB;
+	info.max_tmds_clock = 150000; /* 150 MHz */
+
+	KUNIT_EXPECT_TRUE(test, adjust_colour_depth_from_display_info(&timing, &info));
+	KUNIT_EXPECT_EQ(test, (int)timing.display_color_depth, (int)COLOR_DEPTH_888);
+}
+
+/**
+ * dm_test_adjust_colour_depth_reduces_to_888 - Test Adjust colour depth reduces to 888
+ * @test: The KUnit test context
+ */
+static void dm_test_adjust_colour_depth_reduces_to_888(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_display_info info = {};
+
+	/* Request 10bpc but TMDS limit only allows 8bpc */
+	timing.pix_clk_100hz = 1485000;
+	timing.display_color_depth = COLOR_DEPTH_101010;
+	timing.pixel_encoding = PIXEL_ENCODING_RGB;
+	/* 10bpc would need 148500*30/24 = 185625 KHz, exceeds limit */
+	info.max_tmds_clock = 160000;
+
+	KUNIT_EXPECT_TRUE(test, adjust_colour_depth_from_display_info(&timing, &info));
+	KUNIT_EXPECT_EQ(test, (int)timing.display_color_depth, (int)COLOR_DEPTH_888);
+}
+
+/**
+ * dm_test_adjust_colour_depth_10bpc_passes - Test Adjust colour depth 10bpc passes
+ * @test: The KUnit test context
+ */
+static void dm_test_adjust_colour_depth_10bpc_passes(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_display_info info = {};
+
+	timing.pix_clk_100hz = 1485000;
+	timing.display_color_depth = COLOR_DEPTH_101010;
+	timing.pixel_encoding = PIXEL_ENCODING_RGB;
+	/* 10bpc needs 185625 KHz, allow it */
+	info.max_tmds_clock = 200000;
+
+	KUNIT_EXPECT_TRUE(test, adjust_colour_depth_from_display_info(&timing, &info));
+	KUNIT_EXPECT_EQ(test, (int)timing.display_color_depth, (int)COLOR_DEPTH_101010);
+}
+
+/**
+ * dm_test_adjust_colour_depth_420_halves_clk - Test Adjust colour depth 420 halves clk
+ * @test: The KUnit test context
+ */
+static void dm_test_adjust_colour_depth_420_halves_clk(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_display_info info = {};
+
+	/* 4K @ 594000 KHz = 5940000 in 100Hz units */
+	timing.pix_clk_100hz = 5940000;
+	timing.display_color_depth = COLOR_DEPTH_101010;
+	timing.pixel_encoding = PIXEL_ENCODING_YCBCR420;
+	/* With 420: effective = 594000/2 = 297000, 10bpc = 297000*30/24 = 371250 */
+	info.max_tmds_clock = 400000;
+
+	KUNIT_EXPECT_TRUE(test, adjust_colour_depth_from_display_info(&timing, &info));
+	KUNIT_EXPECT_EQ(test, (int)timing.display_color_depth, (int)COLOR_DEPTH_101010);
+}
+
+/**
+ * dm_test_adjust_colour_depth_420_reduces - Test Adjust colour depth 420 reduces
+ * @test: The KUnit test context
+ */
+static void dm_test_adjust_colour_depth_420_reduces(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_display_info info = {};
+
+	/* 4K @ 594000 KHz = 5940000 in 100Hz units */
+	timing.pix_clk_100hz = 5940000;
+	timing.display_color_depth = COLOR_DEPTH_121212;
+	timing.pixel_encoding = PIXEL_ENCODING_YCBCR420;
+	/*
+	 * With 420: effective = 594000/2 = 297000.
+	 * 12bpc = 297000*36/24 = 445500 (exceeds limit),
+	 * 10bpc = 297000*30/24 = 371250 (fits).
+	 */
+	info.max_tmds_clock = 400000;
+
+	KUNIT_EXPECT_TRUE(test, adjust_colour_depth_from_display_info(&timing, &info));
+	KUNIT_EXPECT_EQ(test, (int)timing.display_color_depth, (int)COLOR_DEPTH_101010);
+}
+
+/**
+ * dm_test_adjust_colour_depth_reduces_12bpc_to_10bpc - Test Adjust colour
+ * depth reduces 12bpc to 10bpc
+ * @test: The KUnit test context
+ */
+static void dm_test_adjust_colour_depth_reduces_12bpc_to_10bpc(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_display_info info = {};
+
+	timing.pix_clk_100hz = 1485000;
+	timing.display_color_depth = COLOR_DEPTH_121212;
+	timing.pixel_encoding = PIXEL_ENCODING_RGB;
+	info.max_tmds_clock = 190000;
+
+	KUNIT_EXPECT_TRUE(test, adjust_colour_depth_from_display_info(&timing, &info));
+	KUNIT_EXPECT_EQ(test, (int)timing.display_color_depth, (int)COLOR_DEPTH_101010);
+}
+
+/**
+ * dm_test_adjust_colour_depth_16bpc_no_fallback - Test Adjust colour depth
+ * 16bpc cannot fall back
+ * @test: The KUnit test context
+ */
+static void dm_test_adjust_colour_depth_16bpc_no_fallback(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_display_info info = {};
+
+	/* 16bpc that exceeds limit cannot reduce because the next enum
+	 * value (COLOR_DEPTH_141414) is not a valid HDMI depth.
+	 */
+	timing.pix_clk_100hz = 1485000;
+	timing.display_color_depth = COLOR_DEPTH_161616;
+	timing.pixel_encoding = PIXEL_ENCODING_RGB;
+	info.max_tmds_clock = 230000;
+
+	KUNIT_EXPECT_FALSE(test, adjust_colour_depth_from_display_info(&timing, &info));
+}
+
+/**
+ * dm_test_adjust_colour_depth_none_fits - Test Adjust colour depth none fits
+ * @test: The KUnit test context
+ */
+static void dm_test_adjust_colour_depth_none_fits(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_display_info info = {};
+
+	/* Even 8bpc doesn't fit */
+	timing.pix_clk_100hz = 1485000;
+	timing.display_color_depth = COLOR_DEPTH_888;
+	timing.pixel_encoding = PIXEL_ENCODING_RGB;
+	info.max_tmds_clock = 100000; /* Too low */
+
+	KUNIT_EXPECT_FALSE(test, adjust_colour_depth_from_display_info(&timing, &info));
+}
+
+/**
+ * dm_test_adjust_colour_depth_invalid_depth - Test Adjust colour depth invalid depth
+ * @test: The KUnit test context
+ */
+static void dm_test_adjust_colour_depth_invalid_depth(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_display_info info = {};
+
+	timing.pix_clk_100hz = 1485000;
+	timing.display_color_depth = COLOR_DEPTH_141414;
+	timing.pixel_encoding = PIXEL_ENCODING_RGB;
+	info.max_tmds_clock = 400000;
+
+	KUNIT_EXPECT_FALSE(test, adjust_colour_depth_from_display_info(&timing, &info));
+	KUNIT_EXPECT_EQ(test, (int)timing.display_color_depth, (int)COLOR_DEPTH_141414);
+}
+
+/* Tests for amdgpu_dm_get_output_color_space() */
+
+/**
+ * dm_test_output_color_space_default_rgb_full - Test Output color space default rgb full
+ * @test: The KUnit test context
+ */
+static void dm_test_output_color_space_default_rgb_full(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_connector_state state = {};
+
+	timing.pixel_encoding = PIXEL_ENCODING_RGB;
+	state.colorspace = DRM_MODE_COLORIMETRY_DEFAULT;
+	state.hdmi.broadcast_rgb = DRM_HDMI_BROADCAST_RGB_AUTO;
+
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state),
+			(int)COLOR_SPACE_SRGB);
+}
+
+/**
+ * dm_test_output_color_space_default_rgb_limited - Test Output color space default rgb limited
+ * @test: The KUnit test context
+ */
+static void dm_test_output_color_space_default_rgb_limited(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_connector_state state = {};
+
+	timing.pixel_encoding = PIXEL_ENCODING_RGB;
+	state.colorspace = DRM_MODE_COLORIMETRY_DEFAULT;
+	state.hdmi.broadcast_rgb = DRM_HDMI_BROADCAST_RGB_LIMITED;
+
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state),
+			(int)COLOR_SPACE_SRGB_LIMITED);
+}
+
+/**
+ * dm_test_output_color_space_default_ycbcr709 - Test Output color space default ycbcr709
+ * @test: The KUnit test context
+ */
+static void dm_test_output_color_space_default_ycbcr709(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_connector_state state = {};
+
+	timing.pixel_encoding = PIXEL_ENCODING_YCBCR444;
+	timing.pix_clk_100hz = 300000;
+	timing.flags.Y_ONLY = 0;
+	state.colorspace = DRM_MODE_COLORIMETRY_DEFAULT;
+
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state),
+			(int)COLOR_SPACE_YCBCR709);
+}
+
+/**
+ * dm_test_output_color_space_default_ycbcr601_limited - Test Output color space
+ * default ycbcr601 limited
+ * @test: The KUnit test context
+ */
+static void dm_test_output_color_space_default_ycbcr601_limited(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_connector_state state = {};
+
+	timing.pixel_encoding = PIXEL_ENCODING_YCBCR444;
+	timing.pix_clk_100hz = 270300;
+	timing.flags.Y_ONLY = 1;
+	state.colorspace = DRM_MODE_COLORIMETRY_DEFAULT;
+
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state),
+			(int)COLOR_SPACE_YCBCR601_LIMITED);
+}
+
+/**
+ * dm_test_output_color_space_bt601_y_only - Test Output color space bt601 y only
+ * @test: The KUnit test context
+ */
+static void dm_test_output_color_space_bt601_y_only(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_connector_state state = {};
+
+	timing.flags.Y_ONLY = 1;
+	state.colorspace = DRM_MODE_COLORIMETRY_BT601_YCC;
+
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state),
+			(int)COLOR_SPACE_YCBCR601_LIMITED);
+}
+
+/**
+ * dm_test_output_color_space_bt601 - Test Output color space bt601
+ * @test: The KUnit test context
+ */
+static void dm_test_output_color_space_bt601(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_connector_state state = {};
+
+	timing.flags.Y_ONLY = 0;
+	state.colorspace = DRM_MODE_COLORIMETRY_BT601_YCC;
+
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state),
+			(int)COLOR_SPACE_YCBCR601);
+}
+
+/**
+ * dm_test_output_color_space_bt709 - Test Output color space bt709
+ * @test: The KUnit test context
+ */
+static void dm_test_output_color_space_bt709(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_connector_state state = {};
+
+	timing.flags.Y_ONLY = 0;
+	state.colorspace = DRM_MODE_COLORIMETRY_BT709_YCC;
+
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state),
+			(int)COLOR_SPACE_YCBCR709);
+}
+
+/**
+ * dm_test_output_color_space_bt709_y_only - Test Output color space bt709 y only
+ * @test: The KUnit test context
+ */
+static void dm_test_output_color_space_bt709_y_only(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_connector_state state = {};
+
+	timing.flags.Y_ONLY = 1;
+	state.colorspace = DRM_MODE_COLORIMETRY_BT709_YCC;
+
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state),
+			(int)COLOR_SPACE_YCBCR709_LIMITED);
+}
+
+/**
+ * dm_test_output_color_space_oprgb - Test Output color space oprgb
+ * @test: The KUnit test context
+ */
+static void dm_test_output_color_space_oprgb(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_connector_state state = {};
+
+	state.colorspace = DRM_MODE_COLORIMETRY_OPRGB;
+
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state),
+			(int)COLOR_SPACE_ADOBERGB);
+}
+
+/**
+ * dm_test_output_color_space_bt2020_rgb - Test Output color space bt2020 rgb
+ * @test: The KUnit test context
+ */
+static void dm_test_output_color_space_bt2020_rgb(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_connector_state state = {};
+
+	timing.pixel_encoding = PIXEL_ENCODING_RGB;
+	state.colorspace = DRM_MODE_COLORIMETRY_BT2020_RGB;
+
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state),
+			(int)COLOR_SPACE_2020_RGB_FULLRANGE);
+}
+
+/**
+ * dm_test_output_color_space_bt2020_ycc - Test Output color space bt2020 ycc
+ * @test: The KUnit test context
+ */
+static void dm_test_output_color_space_bt2020_ycc(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_connector_state state = {};
+
+	timing.pixel_encoding = PIXEL_ENCODING_YCBCR422;
+	state.colorspace = DRM_MODE_COLORIMETRY_BT2020_YCC;
+
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state),
+			(int)COLOR_SPACE_2020_YCBCR_LIMITED);
+}
+
+/**
+ * dm_test_output_color_space_default_ycbcr709_y_only - Test Output color space
+ * default ycbcr709 limited via Y_ONLY at high pixel clock
+ * @test: The KUnit test context
+ */
+static void dm_test_output_color_space_default_ycbcr709_y_only(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_connector_state state = {};
+
+	timing.pixel_encoding = PIXEL_ENCODING_YCBCR444;
+	timing.pix_clk_100hz = 300000;
+	timing.flags.Y_ONLY = 1;
+	state.colorspace = DRM_MODE_COLORIMETRY_DEFAULT;
+
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state),
+			(int)COLOR_SPACE_YCBCR709_LIMITED);
+}
+
+/**
+ * dm_test_output_color_space_default_ycbcr601 - Test Output color space default
+ * ycbcr601 full range at low pixel clock
+ * @test: The KUnit test context
+ */
+static void dm_test_output_color_space_default_ycbcr601(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_connector_state state = {};
+
+	timing.pixel_encoding = PIXEL_ENCODING_YCBCR444;
+	timing.pix_clk_100hz = 270300;
+	timing.flags.Y_ONLY = 0;
+	state.colorspace = DRM_MODE_COLORIMETRY_DEFAULT;
+
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state),
+			(int)COLOR_SPACE_YCBCR601);
+}
+
+/**
+ * dm_test_output_color_space_bt2020_ycc_rgb_encoding - Test Output color space
+ * bt2020 ycc with rgb pixel encoding falls back to full range rgb
+ * @test: The KUnit test context
+ */
+static void dm_test_output_color_space_bt2020_ycc_rgb_encoding(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_connector_state state = {};
+
+	timing.pixel_encoding = PIXEL_ENCODING_RGB;
+	state.colorspace = DRM_MODE_COLORIMETRY_BT2020_YCC;
+
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state),
+			(int)COLOR_SPACE_2020_RGB_FULLRANGE);
+}
+
+/**
+ * dm_test_output_color_space_bt2020_rgb_ycc_encoding - Test Output color space
+ * bt2020 rgb with non-rgb pixel encoding falls back to limited ycbcr
+ * @test: The KUnit test context
+ */
+static void dm_test_output_color_space_bt2020_rgb_ycc_encoding(struct kunit *test)
+{
+	struct dc_crtc_timing timing = {};
+	struct drm_connector_state state = {};
+
+	timing.pixel_encoding = PIXEL_ENCODING_YCBCR444;
+	state.colorspace = DRM_MODE_COLORIMETRY_BT2020_RGB;
+
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state),
+			(int)COLOR_SPACE_2020_YCBCR_LIMITED);
+}
+
+/* Tests for amdgpu_dm_convert_dc_color_depth_into_bpc() */
+
+/**
+ * dm_test_convert_color_depth_bpc_mappings - Test Convert color depth bpc mappings
+ * @test: The KUnit test context
+ */
+static void dm_test_convert_color_depth_bpc_mappings(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_convert_dc_color_depth_into_bpc(COLOR_DEPTH_666), 6);
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_convert_dc_color_depth_into_bpc(COLOR_DEPTH_888), 8);
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_convert_dc_color_depth_into_bpc(COLOR_DEPTH_101010), 10);
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_convert_dc_color_depth_into_bpc(COLOR_DEPTH_121212), 12);
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_convert_dc_color_depth_into_bpc(COLOR_DEPTH_141414), 14);
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_convert_dc_color_depth_into_bpc(COLOR_DEPTH_161616), 16);
+}
+
+/**
+ * dm_test_convert_color_depth_bpc_unknown - Test Convert color depth bpc unknown
+ * @test: The KUnit test context
+ */
+static void dm_test_convert_color_depth_bpc_unknown(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_convert_dc_color_depth_into_bpc(COLOR_DEPTH_UNDEFINED), 0);
+}
+
+/* Tests for amdgpu_dm_convert_color_depth_from_display_info() */
+
+/**
+ * dm_test_color_depth_from_info_bpc8 - Test Color depth from info bpc8
+ * @test: The KUnit test context
+ */
+static void dm_test_color_depth_from_info_bpc8(struct kunit *test)
+{
+	struct drm_connector *connector;
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+
+	connector->display_info.bpc = 8;
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_convert_color_depth_from_display_info(connector, false, 0),
+			(int)COLOR_DEPTH_888);
+}
+
+/**
+ * dm_test_color_depth_from_info_bpc10 - Test Color depth from info bpc10
+ * @test: The KUnit test context
+ */
+static void dm_test_color_depth_from_info_bpc10(struct kunit *test)
+{
+	struct drm_connector *connector;
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+
+	connector->display_info.bpc = 10;
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_convert_color_depth_from_display_info(connector, false, 0),
+			(int)COLOR_DEPTH_101010);
+}
+
+/**
+ * dm_test_color_depth_from_info_zero_bpc_defaults_888 - Test Color depth from
+ * info zero bpc defaults 888
+ * @test: The KUnit test context
+ */
+static void dm_test_color_depth_from_info_zero_bpc_defaults_888(struct kunit *test)
+{
+	struct drm_connector *connector;
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+
+	connector->display_info.bpc = 0;
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_convert_color_depth_from_display_info(connector, false, 0),
+			(int)COLOR_DEPTH_888);
+}
+
+/**
+ * dm_test_color_depth_from_info_requested_bpc_caps - Test Color depth from info requested bpc caps
+ * @test: The KUnit test context
+ */
+static void dm_test_color_depth_from_info_requested_bpc_caps(struct kunit *test)
+{
+	struct drm_connector *connector;
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+
+	/* Display supports 12bpc but user requests max 10 */
+	connector->display_info.bpc = 12;
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_convert_color_depth_from_display_info(connector, false, 10),
+			(int)COLOR_DEPTH_101010);
+}
+
+/**
+ * dm_test_color_depth_from_info_y420_default - Test Color depth from info y420 default
+ * @test: The KUnit test context
+ */
+static void dm_test_color_depth_from_info_y420_default(struct kunit *test)
+{
+	struct drm_connector *connector;
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+
+	/* No Y420 DC modes set → 8bpc */
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_convert_color_depth_from_display_info(connector, true, 0),
+			(int)COLOR_DEPTH_888);
+}
+
+/**
+ * dm_test_color_depth_from_info_y420_10bpc - Test Color depth from info y420 10bpc
+ * @test: The KUnit test context
+ */
+static void dm_test_color_depth_from_info_y420_10bpc(struct kunit *test)
+{
+	struct drm_connector *connector;
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+
+	connector->display_info.hdmi.y420_dc_modes = DRM_EDID_YCBCR420_DC_30;
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_convert_color_depth_from_display_info(connector, true, 0),
+			(int)COLOR_DEPTH_101010);
+}
+
+/**
+ * dm_test_color_depth_from_info_y420_12bpc - Test Color depth from info y420 12bpc
+ * @test: The KUnit test context
+ */
+static void dm_test_color_depth_from_info_y420_12bpc(struct kunit *test)
+{
+	struct drm_connector *connector;
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+
+	connector->display_info.hdmi.y420_dc_modes = DRM_EDID_YCBCR420_DC_36;
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_convert_color_depth_from_display_info(connector, true, 0),
+			(int)COLOR_DEPTH_121212);
+}
+
+/**
+ * dm_test_color_depth_from_info_y420_16bpc - Test Color depth from info y420 16bpc
+ * @test: The KUnit test context
+ */
+static void dm_test_color_depth_from_info_y420_16bpc(struct kunit *test)
+{
+	struct drm_connector *connector;
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+
+	connector->display_info.hdmi.y420_dc_modes = DRM_EDID_YCBCR420_DC_48;
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_convert_color_depth_from_display_info(connector, true, 0),
+			(int)COLOR_DEPTH_161616);
+}
+
+/**
+ * dm_test_color_depth_from_info_requested_odd_bpc - Test Color depth from info requested odd bpc
+ * @test: The KUnit test context
+ */
+static void dm_test_color_depth_from_info_requested_odd_bpc(struct kunit *test)
+{
+	struct drm_connector *connector;
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+
+	connector->display_info.bpc = 12;
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_convert_color_depth_from_display_info(connector, false, 11),
+			(int)COLOR_DEPTH_101010);
+}
+
+/**
+ * dm_test_color_depth_from_info_unsupported_bpc - Test Color depth from info unsupported bpc
+ * @test: The KUnit test context
+ */
+static void dm_test_color_depth_from_info_unsupported_bpc(struct kunit *test)
+{
+	struct drm_connector *connector;
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, connector);
+
+	connector->display_info.bpc = 9;
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_convert_color_depth_from_display_info(connector, false, 0),
+			(int)COLOR_DEPTH_UNDEFINED);
+}
+
+/* Tests for to_drm_connector_type() */
+
+/**
+ * dm_test_to_connector_type_hdmi - Test To connector type hdmi
+ * @test: The KUnit test context
+ */
+static void dm_test_to_connector_type_hdmi(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, to_drm_connector_type(SIGNAL_TYPE_HDMI_TYPE_A, 0),
+			DRM_MODE_CONNECTOR_HDMIA);
+}
+
+/**
+ * dm_test_to_connector_type_edp - Test To connector type edp
+ * @test: The KUnit test context
+ */
+static void dm_test_to_connector_type_edp(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, to_drm_connector_type(SIGNAL_TYPE_EDP, 0),
+			DRM_MODE_CONNECTOR_eDP);
+}
+
+/**
+ * dm_test_to_connector_type_lvds - Test To connector type lvds
+ * @test: The KUnit test context
+ */
+static void dm_test_to_connector_type_lvds(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, to_drm_connector_type(SIGNAL_TYPE_LVDS, 0),
+			DRM_MODE_CONNECTOR_LVDS);
+}
+
+/**
+ * dm_test_to_connector_type_rgb - Test To connector type rgb
+ * @test: The KUnit test context
+ */
+static void dm_test_to_connector_type_rgb(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, to_drm_connector_type(SIGNAL_TYPE_RGB, 0),
+			DRM_MODE_CONNECTOR_VGA);
+}
+
+/**
+ * dm_test_to_connector_type_dp - Test To connector type dp
+ * @test: The KUnit test context
+ */
+static void dm_test_to_connector_type_dp(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, to_drm_connector_type(SIGNAL_TYPE_DISPLAY_PORT, 0),
+			DRM_MODE_CONNECTOR_DisplayPort);
+}
+
+/**
+ * dm_test_to_connector_type_dp_mst - Test To connector type dp mst
+ * @test: The KUnit test context
+ */
+static void dm_test_to_connector_type_dp_mst(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, to_drm_connector_type(SIGNAL_TYPE_DISPLAY_PORT_MST, 0),
+			DRM_MODE_CONNECTOR_DisplayPort);
+}
+
+/**
+ * dm_test_to_connector_type_dvi_dvii - Test To connector type dvi dvii
+ * @test: The KUnit test context
+ */
+static void dm_test_to_connector_type_dvi_dvii(struct kunit *test)
+{
+	int type = to_drm_connector_type(SIGNAL_TYPE_DVI_SINGLE_LINK, CONNECTOR_ID_SINGLE_LINK_DVII);
+
+	KUNIT_EXPECT_EQ(test, type, DRM_MODE_CONNECTOR_DVII);
+}
+
+/**
+ * dm_test_to_connector_type_dual_link_dvii - Test To connector type dual link dvii
+ * @test: The KUnit test context
+ */
+static void dm_test_to_connector_type_dual_link_dvii(struct kunit *test)
+{
+	int type = to_drm_connector_type(SIGNAL_TYPE_DVI_DUAL_LINK, CONNECTOR_ID_DUAL_LINK_DVII);
+
+	KUNIT_EXPECT_EQ(test, type, DRM_MODE_CONNECTOR_DVII);
+}
+
+/**
+ * dm_test_to_connector_type_dvi_dvid - Test To connector type dvi dvid
+ * @test: The KUnit test context
+ */
+static void dm_test_to_connector_type_dvi_dvid(struct kunit *test)
+{
+	int type = to_drm_connector_type(SIGNAL_TYPE_DVI_SINGLE_LINK, CONNECTOR_ID_SINGLE_LINK_DVID);
+
+	KUNIT_EXPECT_EQ(test, type, DRM_MODE_CONNECTOR_DVID);
+}
+
+/**
+ * dm_test_to_connector_type_dual_link_dvid - Test To connector type dual link dvid
+ * @test: The KUnit test context
+ */
+static void dm_test_to_connector_type_dual_link_dvid(struct kunit *test)
+{
+	int type = to_drm_connector_type(SIGNAL_TYPE_DVI_DUAL_LINK, CONNECTOR_ID_DUAL_LINK_DVID);
+
+	KUNIT_EXPECT_EQ(test, type, DRM_MODE_CONNECTOR_DVID);
+}
+
+/**
+ * dm_test_to_connector_type_virtual - Test To connector type virtual
+ * @test: The KUnit test context
+ */
+static void dm_test_to_connector_type_virtual(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, to_drm_connector_type(SIGNAL_TYPE_VIRTUAL, 0),
+			DRM_MODE_CONNECTOR_VIRTUAL);
+}
+
+/**
+ * dm_test_to_connector_type_unknown - Test To connector type unknown
+ * @test: The KUnit test context
+ */
+static void dm_test_to_connector_type_unknown(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, to_drm_connector_type(SIGNAL_TYPE_NONE, 0),
+			DRM_MODE_CONNECTOR_Unknown);
+}
+
+/* Tests for is_duplicate_mode() */
+
+/**
+ * dm_test_is_duplicate_mode_empty_list - Test Is duplicate mode empty list
+ * @test: The KUnit test context
+ */
+static void dm_test_is_duplicate_mode_empty_list(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_display_mode mode = {};
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconnector);
+
+	INIT_LIST_HEAD(&aconnector->base.probed_modes);
+	mode.hdisplay = 1920;
+	mode.vdisplay = 1080;
+
+	KUNIT_EXPECT_FALSE(test, is_duplicate_mode(aconnector, &mode));
+}
+
+/**
+ * dm_test_is_duplicate_mode_match - Test Is duplicate mode match
+ * @test: The KUnit test context
+ */
+static void dm_test_is_duplicate_mode_match(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_display_mode existing = {};
+	struct drm_display_mode candidate = {};
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconnector);
+
+	INIT_LIST_HEAD(&aconnector->base.probed_modes);
+	existing.hdisplay = 1920;
+	existing.vdisplay = 1080;
+	existing.clock = 148500;
+	list_add_tail(&existing.head, &aconnector->base.probed_modes);
+
+	candidate.hdisplay = 1920;
+	candidate.vdisplay = 1080;
+	candidate.clock = 148500;
+
+	KUNIT_EXPECT_TRUE(test, is_duplicate_mode(aconnector, &candidate));
+}
+
+/**
+ * dm_test_is_duplicate_mode_no_match - Test Is duplicate mode no match
+ * @test: The KUnit test context
+ */
+static void dm_test_is_duplicate_mode_no_match(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_display_mode existing = {};
+	struct drm_display_mode candidate = {};
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconnector);
+
+	INIT_LIST_HEAD(&aconnector->base.probed_modes);
+	existing.hdisplay = 1920;
+	existing.vdisplay = 1080;
+	existing.clock = 148500;
+	list_add_tail(&existing.head, &aconnector->base.probed_modes);
+
+	candidate.hdisplay = 2560;
+	candidate.vdisplay = 1440;
+	candidate.clock = 241500;
+
+	KUNIT_EXPECT_FALSE(test, is_duplicate_mode(aconnector, &candidate));
+}
+
+/**
+ * dm_test_is_duplicate_mode_same_size_different_clock - Test Is duplicate mode
+ * same size different clock
+ * @test: The KUnit test context
+ */
+static void dm_test_is_duplicate_mode_same_size_different_clock(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_display_mode existing = {};
+	struct drm_display_mode candidate = {};
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconnector);
+
+	INIT_LIST_HEAD(&aconnector->base.probed_modes);
+	existing.hdisplay = 1920;
+	existing.vdisplay = 1080;
+	existing.clock = 148500;
+	list_add_tail(&existing.head, &aconnector->base.probed_modes);
+
+	candidate.hdisplay = 1920;
+	candidate.vdisplay = 1080;
+	candidate.clock = 74250;
+
+	KUNIT_EXPECT_FALSE(test, is_duplicate_mode(aconnector, &candidate));
+}
+
+/* Tests for amdgpu_dm_get_encoder_crtc_mask() */
+
+/**
+ * dm_test_encoder_crtc_mask_1 - Test Encoder crtc mask 1
+ * @test: The KUnit test context
+ */
+static void dm_test_encoder_crtc_mask_1(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	adev->mode_info.num_crtc = 1;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_get_encoder_crtc_mask(adev), 0x1);
+}
+
+/**
+ * dm_test_encoder_crtc_mask_2 - Test Encoder crtc mask 2
+ * @test: The KUnit test context
+ */
+static void dm_test_encoder_crtc_mask_2(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	adev->mode_info.num_crtc = 2;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_get_encoder_crtc_mask(adev), 0x3);
+}
+
+/**
+ * dm_test_encoder_crtc_mask_3 - Test Encoder crtc mask 3
+ * @test: The KUnit test context
+ */
+static void dm_test_encoder_crtc_mask_3(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	adev->mode_info.num_crtc = 3;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_get_encoder_crtc_mask(adev), 0x7);
+}
+
+/**
+ * dm_test_encoder_crtc_mask_4 - Test Encoder crtc mask 4
+ * @test: The KUnit test context
+ */
+static void dm_test_encoder_crtc_mask_4(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	adev->mode_info.num_crtc = 4;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_get_encoder_crtc_mask(adev), 0xf);
+}
+
+/**
+ * dm_test_encoder_crtc_mask_5 - Test Encoder crtc mask 5
+ * @test: The KUnit test context
+ */
+static void dm_test_encoder_crtc_mask_5(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	adev->mode_info.num_crtc = 5;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_get_encoder_crtc_mask(adev), 0x1f);
+}
+
+/**
+ * dm_test_encoder_crtc_mask_6 - Test Encoder crtc mask 6
+ * @test: The KUnit test context
+ */
+static void dm_test_encoder_crtc_mask_6(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	adev->mode_info.num_crtc = 6;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_get_encoder_crtc_mask(adev), 0x3f);
+}
+
+/**
+ * dm_test_encoder_crtc_mask_default - Test Encoder crtc mask default
+ * @test: The KUnit test context
+ */
+static void dm_test_encoder_crtc_mask_default(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	/* Values > 6 use the default case */
+	adev->mode_info.num_crtc = 8;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_get_encoder_crtc_mask(adev), 0x3f);
+}
+
+/* Tests for get_aspect_ratio() */
+
+/**
+ * dm_test_aspect_ratio_no_data - Test Aspect ratio no data
+ * @test: The KUnit test context
+ */
+static void dm_test_aspect_ratio_no_data(struct kunit *test)
+{
+	struct drm_display_mode mode = {};
+
+	mode.picture_aspect_ratio = HDMI_PICTURE_ASPECT_NONE;
+	KUNIT_EXPECT_EQ(test, (int)get_aspect_ratio(&mode), (int)ASPECT_RATIO_NO_DATA);
+}
+
+/**
+ * dm_test_aspect_ratio_4_3 - Test Aspect ratio 4 3
+ * @test: The KUnit test context
+ */
+static void dm_test_aspect_ratio_4_3(struct kunit *test)
+{
+	struct drm_display_mode mode = {};
+
+	mode.picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3;
+	KUNIT_EXPECT_EQ(test, (int)get_aspect_ratio(&mode), (int)ASPECT_RATIO_4_3);
+}
+
+/**
+ * dm_test_aspect_ratio_16_9 - Test Aspect ratio 16 9
+ * @test: The KUnit test context
+ */
+static void dm_test_aspect_ratio_16_9(struct kunit *test)
+{
+	struct drm_display_mode mode = {};
+
+	mode.picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9;
+	KUNIT_EXPECT_EQ(test, (int)get_aspect_ratio(&mode), (int)ASPECT_RATIO_16_9);
+}
+
+/**
+ * dm_test_aspect_ratio_64_27 - Test Aspect ratio 64 27
+ * @test: The KUnit test context
+ */
+static void dm_test_aspect_ratio_64_27(struct kunit *test)
+{
+	struct drm_display_mode mode = {};
+
+	mode.picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27;
+	KUNIT_EXPECT_EQ(test, (int)get_aspect_ratio(&mode), (int)ASPECT_RATIO_64_27);
+}
+
+/**
+ * dm_test_aspect_ratio_256_135 - Test Aspect ratio 256 135
+ * @test: The KUnit test context
+ */
+static void dm_test_aspect_ratio_256_135(struct kunit *test)
+{
+	struct drm_display_mode mode = {};
+
+	mode.picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135;
+	KUNIT_EXPECT_EQ(test, (int)get_aspect_ratio(&mode), (int)ASPECT_RATIO_256_135);
+}
+
+/* Tests for copy_crtc_timing_for_drm_display_mode() */
+
+/**
+ * dm_test_copy_crtc_timing_copies_all_fields - Test all crtc timing fields copied
+ * @test: The KUnit test context
+ */
+static void dm_test_copy_crtc_timing_copies_all_fields(struct kunit *test)
+{
+	struct drm_display_mode src = {};
+	struct drm_display_mode dst = {};
+
+	src.crtc_hdisplay = 1920;
+	src.crtc_vdisplay = 1080;
+	src.crtc_clock = 148500;
+	src.crtc_hblank_start = 1920;
+	src.crtc_hblank_end = 2200;
+	src.crtc_hsync_start = 2008;
+	src.crtc_hsync_end = 2052;
+	src.crtc_htotal = 2200;
+	src.crtc_hskew = 1;
+	src.crtc_vblank_start = 1080;
+	src.crtc_vblank_end = 1125;
+	src.crtc_vsync_start = 1084;
+	src.crtc_vsync_end = 1089;
+	src.crtc_vtotal = 1125;
+
+	copy_crtc_timing_for_drm_display_mode(&src, &dst);
+
+	KUNIT_EXPECT_EQ(test, dst.crtc_hdisplay, 1920);
+	KUNIT_EXPECT_EQ(test, dst.crtc_vdisplay, 1080);
+	KUNIT_EXPECT_EQ(test, dst.crtc_clock, 148500);
+	KUNIT_EXPECT_EQ(test, dst.crtc_hblank_start, 1920);
+	KUNIT_EXPECT_EQ(test, dst.crtc_hblank_end, 2200);
+	KUNIT_EXPECT_EQ(test, dst.crtc_hsync_start, 2008);
+	KUNIT_EXPECT_EQ(test, dst.crtc_hsync_end, 2052);
+	KUNIT_EXPECT_EQ(test, dst.crtc_htotal, 2200);
+	KUNIT_EXPECT_EQ(test, dst.crtc_hskew, 1);
+	KUNIT_EXPECT_EQ(test, dst.crtc_vblank_start, 1080);
+	KUNIT_EXPECT_EQ(test, dst.crtc_vblank_end, 1125);
+	KUNIT_EXPECT_EQ(test, dst.crtc_vsync_start, 1084);
+	KUNIT_EXPECT_EQ(test, dst.crtc_vsync_end, 1089);
+	KUNIT_EXPECT_EQ(test, dst.crtc_vtotal, 1125);
+}
+
+/**
+ * dm_test_copy_crtc_timing_leaves_non_crtc_fields - Test non-crtc fields untouched
+ * @test: The KUnit test context
+ */
+static void dm_test_copy_crtc_timing_leaves_non_crtc_fields(struct kunit *test)
+{
+	struct drm_display_mode src = {};
+	struct drm_display_mode dst = {};
+
+	src.crtc_hdisplay = 1280;
+	src.crtc_vdisplay = 720;
+
+	/* Non-crtc geometry on dst must be preserved by the copy */
+	dst.hdisplay = 1920;
+	dst.vdisplay = 1080;
+	dst.clock = 148500;
+
+	copy_crtc_timing_for_drm_display_mode(&src, &dst);
+
+	KUNIT_EXPECT_EQ(test, dst.crtc_hdisplay, 1280);
+	KUNIT_EXPECT_EQ(test, dst.crtc_vdisplay, 720);
+	KUNIT_EXPECT_EQ(test, dst.hdisplay, 1920);
+	KUNIT_EXPECT_EQ(test, dst.vdisplay, 1080);
+	KUNIT_EXPECT_EQ(test, dst.clock, 148500);
+}
+
+/* Tests for decide_crtc_timing_for_drm_display_mode() */
+
+/**
+ * dm_test_decide_crtc_timing_scale_enabled - Test Decide crtc timing scale enabled
+ * @test: The KUnit test context
+ */
+static void dm_test_decide_crtc_timing_scale_enabled(struct kunit *test)
+{
+	struct drm_display_mode drm_mode = {};
+	struct drm_display_mode native_mode = {};
+
+	native_mode.crtc_clock = 148500;
+	native_mode.crtc_hdisplay = 1920;
+	native_mode.crtc_vdisplay = 1080;
+	native_mode.crtc_htotal = 2200;
+	native_mode.crtc_vtotal = 1125;
+	native_mode.crtc_hsync_start = 2008;
+	native_mode.crtc_hsync_end = 2052;
+	native_mode.crtc_vsync_start = 1084;
+	native_mode.crtc_vsync_end = 1089;
+
+	/* Different clock/htotal/vtotal, but scale_enabled forces copy */
+	drm_mode.clock = 74250;
+	drm_mode.htotal = 1650;
+	drm_mode.vtotal = 750;
+
+	decide_crtc_timing_for_drm_display_mode(&drm_mode, &native_mode, true);
+
+	KUNIT_EXPECT_EQ(test, drm_mode.crtc_clock, 148500);
+	KUNIT_EXPECT_EQ(test, drm_mode.crtc_hdisplay, 1920);
+	KUNIT_EXPECT_EQ(test, drm_mode.crtc_vdisplay, 1080);
+	KUNIT_EXPECT_EQ(test, drm_mode.crtc_htotal, 2200);
+	KUNIT_EXPECT_EQ(test, drm_mode.crtc_vtotal, 1125);
+}
+
+/**
+ * dm_test_decide_crtc_timing_matching_mode - Test Decide crtc timing matching mode
+ * @test: The KUnit test context
+ */
+static void dm_test_decide_crtc_timing_matching_mode(struct kunit *test)
+{
+	struct drm_display_mode drm_mode = {};
+	struct drm_display_mode native_mode = {};
+
+	native_mode.clock = 148500;
+	native_mode.htotal = 2200;
+	native_mode.vtotal = 1125;
+	native_mode.crtc_clock = 148500;
+	native_mode.crtc_hdisplay = 1920;
+	native_mode.crtc_vdisplay = 1080;
+	native_mode.crtc_htotal = 2200;
+	native_mode.crtc_vtotal = 1125;
+
+	/* Matching clock/htotal/vtotal triggers copy */
+	drm_mode.clock = 148500;
+	drm_mode.htotal = 2200;
+	drm_mode.vtotal = 1125;
+
+	decide_crtc_timing_for_drm_display_mode(&drm_mode, &native_mode, false);
+
+	KUNIT_EXPECT_EQ(test, drm_mode.crtc_clock, 148500);
+	KUNIT_EXPECT_EQ(test, drm_mode.crtc_hdisplay, 1920);
+	KUNIT_EXPECT_EQ(test, drm_mode.crtc_vtotal, 1125);
+}
+
+/**
+ * dm_test_decide_crtc_timing_no_copy - Test Decide crtc timing no copy
+ * @test: The KUnit test context
+ */
+static void dm_test_decide_crtc_timing_no_copy(struct kunit *test)
+{
+	struct drm_display_mode drm_mode = {};
+	struct drm_display_mode native_mode = {};
+
+	native_mode.clock = 148500;
+	native_mode.htotal = 2200;
+	native_mode.vtotal = 1125;
+	native_mode.crtc_clock = 148500;
+	native_mode.crtc_hdisplay = 1920;
+
+	/* Different timings, no scaling → no copy */
+	drm_mode.clock = 74250;
+	drm_mode.htotal = 1650;
+	drm_mode.vtotal = 750;
+
+	decide_crtc_timing_for_drm_display_mode(&drm_mode, &native_mode, false);
+
+	KUNIT_EXPECT_EQ(test, drm_mode.crtc_clock, 0);
+	KUNIT_EXPECT_EQ(test, drm_mode.crtc_hdisplay, 0);
+}
+
+/**
+ * dm_test_decide_crtc_timing_no_crtc_clock - Test Decide crtc timing no crtc clock
+ * @test: The KUnit test context
+ */
+static void dm_test_decide_crtc_timing_no_crtc_clock(struct kunit *test)
+{
+	struct drm_display_mode drm_mode = {};
+	struct drm_display_mode native_mode = {};
+
+	/* Matching timings but native crtc_clock is 0 → no copy */
+	native_mode.clock = 148500;
+	native_mode.htotal = 2200;
+	native_mode.vtotal = 1125;
+	native_mode.crtc_clock = 0;
+	native_mode.crtc_hdisplay = 1920;
+
+	drm_mode.clock = 148500;
+	drm_mode.htotal = 2200;
+	drm_mode.vtotal = 1125;
+
+	decide_crtc_timing_for_drm_display_mode(&drm_mode, &native_mode, false);
+
+	KUNIT_EXPECT_EQ(test, drm_mode.crtc_clock, 0);
+	KUNIT_EXPECT_EQ(test, drm_mode.crtc_hdisplay, 0);
+}
+
+/* Tests for amdgpu_dm_connector_funcs_reset() */
+
+static const struct drm_connector_funcs dm_test_connector_funcs = {
+	.reset = amdgpu_dm_connector_funcs_reset,
+	.atomic_duplicate_state = amdgpu_dm_connector_atomic_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+/**
+ * dm_test_funcs_reset_sets_defaults - Test funcs_reset sets defaults
+ * @test: The KUnit test context
+ */
+static void dm_test_funcs_reset_sets_defaults(struct kunit *test)
+{
+	struct device *dev;
+	struct drm_device *drm;
+	struct drm_connector *connector;
+	struct dm_connector_state *dm_state;
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						   sizeof(*drm), 0,
+						   DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, connector);
+
+	drmm_connector_init(drm, connector, &dm_test_connector_funcs,
+			    DRM_MODE_CONNECTOR_DisplayPort, NULL);
+
+	amdgpu_dm_connector_funcs_reset(connector);
+
+	KUNIT_ASSERT_NOT_NULL(test, connector->state);
+	dm_state = to_dm_connector_state(connector->state);
+	KUNIT_EXPECT_EQ(test, (int)dm_state->scaling, (int)RMX_OFF);
+	KUNIT_EXPECT_FALSE(test, dm_state->underscan_enable);
+	KUNIT_EXPECT_EQ(test, (int)dm_state->underscan_hborder, 0);
+	KUNIT_EXPECT_EQ(test, (int)dm_state->underscan_vborder, 0);
+	KUNIT_EXPECT_EQ(test, (int)dm_state->base.max_requested_bpc, 8);
+	KUNIT_EXPECT_EQ(test, dm_state->vcpi_slots, 0);
+	KUNIT_EXPECT_EQ(test, (int)dm_state->pbn, 0);
+}
+
+/**
+ * dm_test_funcs_reset_edp_abm_level - Test funcs_reset eDP sets ABM
+ * @test: The KUnit test context
+ */
+static void dm_test_funcs_reset_edp_abm_level(struct kunit *test)
+{
+	struct device *dev;
+	struct drm_device *drm;
+	struct drm_connector *connector;
+	struct dm_connector_state *dm_state;
+	int saved_abm_level = amdgpu_dm_get_abm_level_param();
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						   sizeof(*drm), 0,
+						   DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, connector);
+
+	drmm_connector_init(drm, connector, &dm_test_connector_funcs,
+			    DRM_MODE_CONNECTOR_eDP, NULL);
+
+	/* Test with abm_level > 0 */
+	amdgpu_dm_set_abm_level_param(3);
+	amdgpu_dm_connector_funcs_reset(connector);
+
+	KUNIT_ASSERT_NOT_NULL(test, connector->state);
+	dm_state = to_dm_connector_state(connector->state);
+	KUNIT_EXPECT_EQ(test, (int)dm_state->abm_level, 3);
+
+	amdgpu_dm_set_abm_level_param(saved_abm_level);
+}
+
+/**
+ * dm_test_funcs_reset_edp_abm_disabled - Test funcs_reset eDP ABM
+ * disabled
+ * @test: The KUnit test context
+ */
+static void dm_test_funcs_reset_edp_abm_disabled(struct kunit *test)
+{
+	struct device *dev;
+	struct drm_device *drm;
+	struct drm_connector *connector;
+	struct dm_connector_state *dm_state;
+	int saved_abm_level = amdgpu_dm_get_abm_level_param();
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						   sizeof(*drm), 0,
+						   DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, connector);
+
+	drmm_connector_init(drm, connector, &dm_test_connector_funcs,
+			    DRM_MODE_CONNECTOR_eDP, NULL);
+
+	/* Test with abm_level <= 0 → immediate disable */
+	amdgpu_dm_set_abm_level_param(-1);
+	amdgpu_dm_connector_funcs_reset(connector);
+
+	KUNIT_ASSERT_NOT_NULL(test, connector->state);
+	dm_state = to_dm_connector_state(connector->state);
+	KUNIT_EXPECT_EQ(test, (int)dm_state->abm_level,
+			(int)ABM_LEVEL_IMMEDIATE_DISABLE);
+
+	amdgpu_dm_set_abm_level_param(saved_abm_level);
+}
+
+/* Tests for amdgpu_dm_connector_atomic_duplicate_state() */
+
+/**
+ * dm_test_atomic_dup_state_copies_fields - Test atomic_duplicate copies
+ * fields
+ * @test: The KUnit test context
+ */
+static void dm_test_atomic_dup_state_copies_fields(struct kunit *test)
+{
+	struct device *dev;
+	struct drm_device *drm;
+	struct drm_connector *connector;
+	struct dm_connector_state *dm_state;
+	struct dm_connector_state *new_dm_state;
+	struct drm_connector_state *new_state;
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						   sizeof(*drm), 0,
+						   DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, connector);
+
+	drmm_connector_init(drm, connector, &dm_test_connector_funcs,
+			    DRM_MODE_CONNECTOR_HDMIA, NULL);
+
+	amdgpu_dm_connector_funcs_reset(connector);
+	KUNIT_ASSERT_NOT_NULL(test, connector->state);
+
+	/* Modify original state fields */
+	dm_state = to_dm_connector_state(connector->state);
+	dm_state->scaling = RMX_CENTER;
+	dm_state->underscan_enable = true;
+	dm_state->underscan_hborder = 10;
+	dm_state->underscan_vborder = 20;
+	dm_state->freesync_capable = true;
+	dm_state->abm_level = 2;
+	dm_state->vcpi_slots = 4;
+	dm_state->pbn = 1234;
+
+	/* Duplicate */
+	new_state = amdgpu_dm_connector_atomic_duplicate_state(connector);
+	KUNIT_ASSERT_NOT_NULL(test, new_state);
+	new_dm_state = to_dm_connector_state(new_state);
+
+	/* Verify all fields copied */
+	KUNIT_EXPECT_EQ(test, (int)new_dm_state->scaling, (int)RMX_CENTER);
+	KUNIT_EXPECT_TRUE(test, new_dm_state->underscan_enable);
+	KUNIT_EXPECT_EQ(test, (int)new_dm_state->underscan_hborder, 10);
+	KUNIT_EXPECT_EQ(test, (int)new_dm_state->underscan_vborder, 20);
+	KUNIT_EXPECT_TRUE(test, new_dm_state->freesync_capable);
+	KUNIT_EXPECT_EQ(test, (int)new_dm_state->abm_level, 2);
+	KUNIT_EXPECT_EQ(test, new_dm_state->vcpi_slots, 4);
+	KUNIT_EXPECT_EQ(test, (int)new_dm_state->pbn, 1234);
+
+	kfree(new_dm_state);
+}
+
+/* Tests for amdgpu_dm_fill_hdr_info_packet() */
+
+/**
+ * dm_test_fill_hdr_null_metadata - Test fill_hdr returns 0 with no
+ * metadata
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_hdr_null_metadata(struct kunit *test)
+{
+	struct drm_connector_state state = {};
+	struct dc_info_packet out = {};
+
+	/* No hdr_output_metadata → early return 0, out stays zeroed */
+	state.hdr_output_metadata = NULL;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_fill_hdr_info_packet(&state, &out), 0);
+	KUNIT_EXPECT_FALSE(test, out.valid);
+}
+
+/**
+ * dm_test_fill_hdr_zeroes_output - Test fill_hdr zeroes output with no
+ * metadata
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_hdr_zeroes_output(struct kunit *test)
+{
+	struct drm_connector_state state = {};
+	struct dc_info_packet out;
+
+	/* Pre-fill out with nonzero to verify memset(0) */
+	memset(&out, 0xAA, sizeof(out));
+
+	state.hdr_output_metadata = NULL;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_fill_hdr_info_packet(&state, &out), 0);
+	KUNIT_EXPECT_FALSE(test, out.valid);
+	KUNIT_EXPECT_EQ(test, (int)out.hb0, 0);
+	KUNIT_EXPECT_EQ(test, (int)out.hb1, 0);
+	KUNIT_EXPECT_EQ(test, (int)out.hb2, 0);
+	KUNIT_EXPECT_EQ(test, (int)out.hb3, 0);
+}
+
+/* Tests for amdgpu_dm_connector_atomic_set_property() */
+
+/*
+ * Build a connector wired to a kunit-allocated amdgpu_device so that
+ * drm_to_adev() resolves correctly, together with old/new dm states and
+ * the set of properties used by the get/set property handlers.
+ */
+struct dm_test_prop_ctx {
+	struct amdgpu_device *adev;
+	struct drm_connector *connector;
+	struct dm_connector_state *old_state;
+	struct dm_connector_state *new_state;
+	struct drm_property *scaling_prop;
+	struct drm_property *hborder_prop;
+	struct drm_property *vborder_prop;
+	struct drm_property *underscan_prop;
+	struct drm_property *abm_prop;
+};
+
+static struct dm_test_prop_ctx *dm_test_prop_ctx_alloc(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	ctx->adev = kunit_kzalloc(test, sizeof(*ctx->adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->adev);
+	ctx->connector = kunit_kzalloc(test, sizeof(*ctx->connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->connector);
+	ctx->old_state = kunit_kzalloc(test, sizeof(*ctx->old_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->old_state);
+	ctx->new_state = kunit_kzalloc(test, sizeof(*ctx->new_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->new_state);
+	ctx->scaling_prop = kunit_kzalloc(test, sizeof(*ctx->scaling_prop), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->scaling_prop);
+	ctx->hborder_prop = kunit_kzalloc(test, sizeof(*ctx->hborder_prop), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->hborder_prop);
+	ctx->vborder_prop = kunit_kzalloc(test, sizeof(*ctx->vborder_prop), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->vborder_prop);
+	ctx->underscan_prop = kunit_kzalloc(test, sizeof(*ctx->underscan_prop), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->underscan_prop);
+	ctx->abm_prop = kunit_kzalloc(test, sizeof(*ctx->abm_prop), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->abm_prop);
+
+	ctx->connector->dev = &ctx->adev->ddev;
+	ctx->connector->state = &ctx->old_state->base;
+
+	ctx->adev->ddev.mode_config.scaling_mode_property = ctx->scaling_prop;
+	ctx->adev->mode_info.underscan_hborder_property = ctx->hborder_prop;
+	ctx->adev->mode_info.underscan_vborder_property = ctx->vborder_prop;
+	ctx->adev->mode_info.underscan_property = ctx->underscan_prop;
+	ctx->adev->mode_info.abm_level_property = ctx->abm_prop;
+
+	return ctx;
+}
+
+/**
+ * dm_test_set_property_scaling_center - Test set scaling property to center
+ * @test: The KUnit test context
+ */
+static void dm_test_set_property_scaling_center(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_set_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->scaling_prop, DRM_MODE_SCALE_CENTER), 0);
+	KUNIT_EXPECT_EQ(test, (int)ctx->new_state->scaling, (int)RMX_CENTER);
+}
+
+/**
+ * dm_test_set_property_scaling_aspect - Test set scaling property to aspect
+ * @test: The KUnit test context
+ */
+static void dm_test_set_property_scaling_aspect(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_set_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->scaling_prop, DRM_MODE_SCALE_ASPECT), 0);
+	KUNIT_EXPECT_EQ(test, (int)ctx->new_state->scaling, (int)RMX_ASPECT);
+}
+
+/**
+ * dm_test_set_property_scaling_fullscreen - Test set scaling property to full
+ * @test: The KUnit test context
+ */
+static void dm_test_set_property_scaling_fullscreen(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_set_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->scaling_prop, DRM_MODE_SCALE_FULLSCREEN), 0);
+	KUNIT_EXPECT_EQ(test, (int)ctx->new_state->scaling, (int)RMX_FULL);
+}
+
+/**
+ * dm_test_set_property_scaling_none - Test set scaling property to none
+ * @test: The KUnit test context
+ */
+static void dm_test_set_property_scaling_none(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+
+	/* old scaling is RMX_CENTER so RMX_OFF is a real change */
+	ctx->old_state->scaling = RMX_CENTER;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_set_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->scaling_prop, DRM_MODE_SCALE_NONE), 0);
+	KUNIT_EXPECT_EQ(test, (int)ctx->new_state->scaling, (int)RMX_OFF);
+}
+
+/**
+ * dm_test_set_property_scaling_unchanged - Test set scaling property unchanged
+ * @test: The KUnit test context
+ */
+static void dm_test_set_property_scaling_unchanged(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+
+	/* old already RMX_OFF, requesting NONE/OFF returns 0 without write */
+	ctx->old_state->scaling = RMX_OFF;
+	ctx->new_state->scaling = RMX_CENTER;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_set_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->scaling_prop, DRM_MODE_SCALE_NONE), 0);
+	/* new_state untouched because of early return */
+	KUNIT_EXPECT_EQ(test, (int)ctx->new_state->scaling, (int)RMX_CENTER);
+}
+
+/**
+ * dm_test_set_property_underscan_hborder - Test set underscan hborder
+ * @test: The KUnit test context
+ */
+static void dm_test_set_property_underscan_hborder(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_set_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->hborder_prop, 42), 0);
+	KUNIT_EXPECT_EQ(test, (int)ctx->new_state->underscan_hborder, 42);
+}
+
+/**
+ * dm_test_set_property_underscan_vborder - Test set underscan vborder
+ * @test: The KUnit test context
+ */
+static void dm_test_set_property_underscan_vborder(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_set_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->vborder_prop, 24), 0);
+	KUNIT_EXPECT_EQ(test, (int)ctx->new_state->underscan_vborder, 24);
+}
+
+/**
+ * dm_test_set_property_underscan_enable - Test set underscan enable
+ * @test: The KUnit test context
+ */
+static void dm_test_set_property_underscan_enable(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_set_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->underscan_prop, 1), 0);
+	KUNIT_EXPECT_TRUE(test, ctx->new_state->underscan_enable);
+}
+
+/**
+ * dm_test_set_property_abm_sysfs_control - Test set abm sysfs control
+ * @test: The KUnit test context
+ */
+static void dm_test_set_property_abm_sysfs_control(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+
+	ctx->new_state->abm_sysfs_forbidden = true;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_set_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->abm_prop, ABM_SYSFS_CONTROL), 0);
+	KUNIT_EXPECT_FALSE(test, ctx->new_state->abm_sysfs_forbidden);
+}
+
+/**
+ * dm_test_set_property_abm_level_off - Test set abm level off
+ * @test: The KUnit test context
+ */
+static void dm_test_set_property_abm_level_off(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_set_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->abm_prop, ABM_LEVEL_OFF), 0);
+	KUNIT_EXPECT_TRUE(test, ctx->new_state->abm_sysfs_forbidden);
+	KUNIT_EXPECT_EQ(test, (int)ctx->new_state->abm_level,
+			(int)ABM_LEVEL_IMMEDIATE_DISABLE);
+}
+
+/**
+ * dm_test_set_property_abm_level_value - Test set abm level to a value
+ * @test: The KUnit test context
+ */
+static void dm_test_set_property_abm_level_value(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_set_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->abm_prop, 3), 0);
+	KUNIT_EXPECT_TRUE(test, ctx->new_state->abm_sysfs_forbidden);
+	KUNIT_EXPECT_EQ(test, (int)ctx->new_state->abm_level, 3);
+}
+
+/**
+ * dm_test_set_property_unknown - Test set unknown property returns -EINVAL
+ * @test: The KUnit test context
+ */
+static void dm_test_set_property_unknown(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+	struct drm_property *other;
+
+	other = kunit_kzalloc(test, sizeof(*other), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, other);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_set_property(
+				ctx->connector, &ctx->new_state->base,
+				other, 0), -EINVAL);
+}
+
+/* Tests for amdgpu_dm_connector_atomic_get_property() */
+
+/**
+ * dm_test_get_property_scaling_center - Test get scaling property center
+ * @test: The KUnit test context
+ */
+static void dm_test_get_property_scaling_center(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+	uint64_t val = 0;
+
+	ctx->new_state->scaling = RMX_CENTER;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_get_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->scaling_prop, &val), 0);
+	KUNIT_EXPECT_EQ(test, (int)val, (int)DRM_MODE_SCALE_CENTER);
+}
+
+/**
+ * dm_test_get_property_scaling_aspect - Test get scaling property aspect
+ * @test: The KUnit test context
+ */
+static void dm_test_get_property_scaling_aspect(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+	uint64_t val = 0;
+
+	ctx->new_state->scaling = RMX_ASPECT;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_get_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->scaling_prop, &val), 0);
+	KUNIT_EXPECT_EQ(test, (int)val, (int)DRM_MODE_SCALE_ASPECT);
+}
+
+/**
+ * dm_test_get_property_scaling_full - Test get scaling property fullscreen
+ * @test: The KUnit test context
+ */
+static void dm_test_get_property_scaling_full(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+	uint64_t val = 0;
+
+	ctx->new_state->scaling = RMX_FULL;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_get_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->scaling_prop, &val), 0);
+	KUNIT_EXPECT_EQ(test, (int)val, (int)DRM_MODE_SCALE_FULLSCREEN);
+}
+
+/**
+ * dm_test_get_property_scaling_off - Test get scaling property off/none
+ * @test: The KUnit test context
+ */
+static void dm_test_get_property_scaling_off(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+	uint64_t val = 0;
+
+	ctx->new_state->scaling = RMX_OFF;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_get_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->scaling_prop, &val), 0);
+	KUNIT_EXPECT_EQ(test, (int)val, (int)DRM_MODE_SCALE_NONE);
+}
+
+/**
+ * dm_test_get_property_underscan_borders - Test get underscan borders/enable
+ * @test: The KUnit test context
+ */
+static void dm_test_get_property_underscan_borders(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+	uint64_t val = 0;
+
+	ctx->new_state->underscan_hborder = 12;
+	ctx->new_state->underscan_vborder = 34;
+	ctx->new_state->underscan_enable = true;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_get_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->hborder_prop, &val), 0);
+	KUNIT_EXPECT_EQ(test, (int)val, 12);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_get_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->vborder_prop, &val), 0);
+	KUNIT_EXPECT_EQ(test, (int)val, 34);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_get_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->underscan_prop, &val), 0);
+	KUNIT_EXPECT_EQ(test, (int)val, 1);
+}
+
+/**
+ * dm_test_get_property_abm_sysfs_allowed - Test get abm returns sysfs control
+ * @test: The KUnit test context
+ */
+static void dm_test_get_property_abm_sysfs_allowed(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+	uint64_t val = 0;
+
+	ctx->new_state->abm_sysfs_forbidden = false;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_get_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->abm_prop, &val), 0);
+	KUNIT_EXPECT_EQ(test, (int)val, (int)ABM_SYSFS_CONTROL);
+}
+
+/**
+ * dm_test_get_property_abm_level - Test get abm returns level when forbidden
+ * @test: The KUnit test context
+ */
+static void dm_test_get_property_abm_level(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+	uint64_t val = 0;
+
+	ctx->new_state->abm_sysfs_forbidden = true;
+	ctx->new_state->abm_level = 2;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_get_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->abm_prop, &val), 0);
+	KUNIT_EXPECT_EQ(test, (int)val, 2);
+}
+
+/**
+ * dm_test_get_property_abm_disabled_zero - Test get abm returns 0 when disabled
+ * @test: The KUnit test context
+ */
+static void dm_test_get_property_abm_disabled_zero(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+	uint64_t val = 0xdead;
+
+	ctx->new_state->abm_sysfs_forbidden = true;
+	ctx->new_state->abm_level = ABM_LEVEL_IMMEDIATE_DISABLE;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_get_property(
+				ctx->connector, &ctx->new_state->base,
+				ctx->abm_prop, &val), 0);
+	KUNIT_EXPECT_EQ(test, (int)val, 0);
+}
+
+/**
+ * dm_test_get_property_unknown - Test get unknown property returns -EINVAL
+ * @test: The KUnit test context
+ */
+static void dm_test_get_property_unknown(struct kunit *test)
+{
+	struct dm_test_prop_ctx *ctx = dm_test_prop_ctx_alloc(test);
+	struct drm_property *other;
+	uint64_t val = 0;
+
+	other = kunit_kzalloc(test, sizeof(*other), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, other);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_connector_atomic_get_property(
+				ctx->connector, &ctx->new_state->base,
+				other, &val), -EINVAL);
+}
+
+/* Tests for amdgpu_dm_get_highest_refresh_rate_mode() */
+
+/**
+ * dm_test_highest_refresh_writeback_null - Test writeback connector returns NULL
+ * @test: The KUnit test context
+ */
+static void dm_test_highest_refresh_writeback_null(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+
+	aconnector->base.connector_type = DRM_MODE_CONNECTOR_WRITEBACK;
+	KUNIT_EXPECT_NULL(test, amdgpu_dm_get_highest_refresh_rate_mode(aconnector, false));
+}
+
+/**
+ * dm_test_highest_refresh_cached_base - Test cached freesync_vid_base is returned
+ * @test: The KUnit test context
+ */
+static void dm_test_highest_refresh_cached_base(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+
+	aconnector->base.connector_type = DRM_MODE_CONNECTOR_HDMIA;
+	aconnector->freesync_vid_base.clock = 148500;
+
+	KUNIT_EXPECT_PTR_EQ(test, amdgpu_dm_get_highest_refresh_rate_mode(aconnector, false),
+			    &aconnector->freesync_vid_base);
+}
+
+/**
+ * dm_test_highest_refresh_preferred_mode - Test preferred mode is selected
+ * @test: The KUnit test context
+ */
+static void dm_test_highest_refresh_preferred_mode(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_display_mode *mode;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	mode = kunit_kzalloc(test, sizeof(*mode), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, mode);
+
+	aconnector->base.connector_type = DRM_MODE_CONNECTOR_HDMIA;
+	INIT_LIST_HEAD(&aconnector->base.modes);
+
+	mode->type = DRM_MODE_TYPE_PREFERRED;
+	mode->clock = 148500;
+	mode->hdisplay = 1920;
+	mode->vdisplay = 1080;
+	mode->htotal = 2200;
+	mode->vtotal = 1125;
+	list_add_tail(&mode->head, &aconnector->base.modes);
+
+	KUNIT_EXPECT_PTR_EQ(test, amdgpu_dm_get_highest_refresh_rate_mode(aconnector, false),
+			    mode);
+}
+
+/* Tests for amdgpu_dm_is_freesync_video_mode() */
+
+/**
+ * dm_test_is_freesync_video_mode_null_mode - Test NULL mode returns false
+ * @test: The KUnit test context
+ */
+static void dm_test_is_freesync_video_mode_null_mode(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+
+	aconnector->base.connector_type = DRM_MODE_CONNECTOR_HDMIA;
+	aconnector->freesync_vid_base.clock = 148500;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_is_freesync_video_mode(NULL, aconnector));
+}
+
+/**
+ * dm_test_is_freesync_video_mode_match - Test matching mode returns true
+ * @test: The KUnit test context
+ */
+static void dm_test_is_freesync_video_mode_match(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_display_mode candidate = {};
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+
+	/* Cached high mode acts as reference */
+	aconnector->base.connector_type = DRM_MODE_CONNECTOR_HDMIA;
+	aconnector->freesync_vid_base.clock = 148500;
+	aconnector->freesync_vid_base.hdisplay = 1920;
+	aconnector->freesync_vid_base.vdisplay = 1080;
+	aconnector->freesync_vid_base.hsync_start = 2008;
+	aconnector->freesync_vid_base.hsync_end = 2052;
+	aconnector->freesync_vid_base.htotal = 2200;
+	aconnector->freesync_vid_base.vsync_start = 1084;
+	aconnector->freesync_vid_base.vsync_end = 1089;
+	aconnector->freesync_vid_base.vtotal = 1125;
+
+	candidate.clock = 148500;
+	candidate.hdisplay = 1920;
+	candidate.vdisplay = 1080;
+	candidate.hsync_start = 2008;
+	candidate.hsync_end = 2052;
+	candidate.htotal = 2200;
+	candidate.vsync_start = 1084;
+	candidate.vsync_end = 1089;
+	candidate.vtotal = 1125;
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_is_freesync_video_mode(&candidate, aconnector));
+}
+
+/**
+ * dm_test_is_freesync_video_mode_no_match - Test mismatched mode returns false
+ * @test: The KUnit test context
+ */
+static void dm_test_is_freesync_video_mode_no_match(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_display_mode candidate = {};
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+
+	aconnector->base.connector_type = DRM_MODE_CONNECTOR_HDMIA;
+	aconnector->freesync_vid_base.clock = 148500;
+	aconnector->freesync_vid_base.hdisplay = 1920;
+	aconnector->freesync_vid_base.vdisplay = 1080;
+	aconnector->freesync_vid_base.htotal = 2200;
+	aconnector->freesync_vid_base.vtotal = 1125;
+
+	/* Different resolution → not a freesync video mode */
+	candidate.clock = 148500;
+	candidate.hdisplay = 1280;
+	candidate.vdisplay = 720;
+	candidate.htotal = 1650;
+	candidate.vtotal = 750;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_is_freesync_video_mode(&candidate, aconnector));
+}
+
+/* Tests for amdgpu_dm_update_cacp_caps() */
+
+struct dm_cacp_fixture {
+	struct amdgpu_device *adev;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+};
+
+static void setup_cacp_fixture(struct kunit *test,
+			       struct dm_cacp_fixture *fixture,
+			       enum signal_type signal,
+			       enum dc_panel_type panel_type)
+{
+	fixture->adev = kunit_kzalloc(test, sizeof(*fixture->adev), GFP_KERNEL);
+	fixture->aconnector = kunit_kzalloc(test, sizeof(*fixture->aconnector),
+					    GFP_KERNEL);
+	fixture->link = kunit_kzalloc(test, sizeof(*fixture->link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fixture->adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fixture->aconnector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fixture->link);
+
+	fixture->aconnector->dc_link = fixture->link;
+	fixture->aconnector->base.dev = &fixture->adev->ddev;
+	fixture->link->connector_signal = signal;
+	fixture->link->panel_type = panel_type;
+}
+
+/**
+ * dm_test_cacp_caps_unsupported_ip - Test CACP disabled on old DCE IP
+ * @test: The KUnit test context
+ *
+ * A DCE IP version below 3.1.4 does not support CACP, so cacp_supported
+ * must remain false regardless of signal or panel type.
+ */
+static void dm_test_cacp_caps_unsupported_ip(struct kunit *test)
+{
+	struct dm_cacp_fixture fixture = {};
+
+	setup_cacp_fixture(test, &fixture, SIGNAL_TYPE_EDP, PANEL_TYPE_OLED);
+	fixture.adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 1, 2);
+
+	amdgpu_dm_update_cacp_caps(fixture.aconnector);
+
+	KUNIT_EXPECT_FALSE(test, fixture.link->panel_config.cacp.cacp_supported);
+}
+
+/**
+ * dm_test_cacp_caps_excluded_ip_316 - Test CACP disabled on DCE IP 3.1.6
+ * @test: The KUnit test context
+ *
+ * DCE IP version 3.1.6 is explicitly excluded from CACP support.
+ */
+static void dm_test_cacp_caps_excluded_ip_316(struct kunit *test)
+{
+	struct dm_cacp_fixture fixture = {};
+
+	setup_cacp_fixture(test, &fixture, SIGNAL_TYPE_EDP, PANEL_TYPE_OLED);
+	fixture.adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 1, 6);
+
+	amdgpu_dm_update_cacp_caps(fixture.aconnector);
+
+	KUNIT_EXPECT_FALSE(test, fixture.link->panel_config.cacp.cacp_supported);
+}
+
+/**
+ * dm_test_cacp_caps_edp_oled_supported - Test CACP enabled on eDP OLED
+ * @test: The KUnit test context
+ *
+ * A supported DCE IP version on an eDP OLED panel must enable CACP.
+ */
+static void dm_test_cacp_caps_edp_oled_supported(struct kunit *test)
+{
+	struct dm_cacp_fixture fixture = {};
+
+	setup_cacp_fixture(test, &fixture, SIGNAL_TYPE_EDP, PANEL_TYPE_OLED);
+	fixture.adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 1, 4);
+
+	amdgpu_dm_update_cacp_caps(fixture.aconnector);
+
+	KUNIT_EXPECT_TRUE(test, fixture.link->panel_config.cacp.cacp_supported);
+}
+
+/**
+ * dm_test_cacp_caps_lvds_oled_supported - Test CACP enabled on LVDS OLED
+ * @test: The KUnit test context
+ *
+ * LVDS is an accepted connector signal for CACP support.
+ */
+static void dm_test_cacp_caps_lvds_oled_supported(struct kunit *test)
+{
+	struct dm_cacp_fixture fixture = {};
+
+	setup_cacp_fixture(test, &fixture, SIGNAL_TYPE_LVDS, PANEL_TYPE_OLED);
+	fixture.adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 5, 0);
+
+	amdgpu_dm_update_cacp_caps(fixture.aconnector);
+
+	KUNIT_EXPECT_TRUE(test, fixture.link->panel_config.cacp.cacp_supported);
+}
+
+/**
+ * dm_test_cacp_caps_non_edp_signal - Test CACP disabled on non-eDP/LVDS signal
+ * @test: The KUnit test context
+ *
+ * External DisplayPort is neither eDP nor LVDS, so CACP must be disabled.
+ */
+static void dm_test_cacp_caps_non_edp_signal(struct kunit *test)
+{
+	struct dm_cacp_fixture fixture = {};
+
+	setup_cacp_fixture(test, &fixture, SIGNAL_TYPE_DISPLAY_PORT,
+			   PANEL_TYPE_OLED);
+	fixture.adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 1, 4);
+
+	amdgpu_dm_update_cacp_caps(fixture.aconnector);
+
+	KUNIT_EXPECT_FALSE(test, fixture.link->panel_config.cacp.cacp_supported);
+}
+
+/**
+ * dm_test_cacp_caps_lcd_panel - Test CACP disabled on LCD panel
+ * @test: The KUnit test context
+ *
+ * Plain LCD panels do not benefit from CACP, so it must be disabled even
+ * on a supported IP version and eDP signal.
+ */
+static void dm_test_cacp_caps_lcd_panel(struct kunit *test)
+{
+	struct dm_cacp_fixture fixture = {};
+
+	setup_cacp_fixture(test, &fixture, SIGNAL_TYPE_EDP, PANEL_TYPE_LCD);
+	fixture.adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 1, 4);
+
+	amdgpu_dm_update_cacp_caps(fixture.aconnector);
+
+	KUNIT_EXPECT_FALSE(test, fixture.link->panel_config.cacp.cacp_supported);
+}
+
+/* Tests for amdgpu_dm_set_panel_type() */
+
+struct dm_panel_type_fixture {
+	struct amdgpu_device *adev;
+	struct drm_device *drm;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+	struct dc_sink *sink;
+};
+
+static void setup_panel_type_fixture(struct kunit *test,
+				     struct dm_panel_type_fixture *fixture)
+{
+	struct device *dev;
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	fixture->drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+							   sizeof(*fixture->adev),
+							   offsetof(struct amdgpu_device, ddev),
+							   DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fixture->drm);
+	fixture->adev = drm_to_adev(fixture->drm);
+
+	fixture->aconnector = kunit_kzalloc(test, sizeof(*fixture->aconnector),
+					    GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fixture->aconnector);
+	fixture->link = kunit_kzalloc(test, sizeof(*fixture->link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fixture->link);
+	fixture->sink = kunit_kzalloc(test, sizeof(*fixture->sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fixture->sink);
+
+	fixture->aconnector->dc_link = fixture->link;
+	drmm_connector_init(fixture->drm, &fixture->aconnector->base,
+			    &dm_test_connector_funcs, DRM_MODE_CONNECTOR_eDP,
+			    NULL);
+}
+
+/**
+ * dm_test_set_panel_type_vsdb_oled - Test VSDB OLED maps to PANEL_TYPE_OLED
+ * @test: The KUnit test context
+ */
+static void dm_test_set_panel_type_vsdb_oled(struct kunit *test)
+{
+	struct dm_panel_type_fixture fixture = {};
+
+	setup_panel_type_fixture(test, &fixture);
+	fixture.aconnector->base.display_info.amd_vsdb.panel_type =
+		AMD_VSDB_PANEL_TYPE_OLED;
+
+	amdgpu_dm_set_panel_type(fixture.aconnector);
+
+	KUNIT_EXPECT_EQ(test, (int)fixture.link->panel_type,
+			(int)PANEL_TYPE_OLED);
+}
+
+/**
+ * dm_test_set_panel_type_vsdb_miniled - Test VSDB MINILED maps to PANEL_TYPE_MINILED
+ * @test: The KUnit test context
+ */
+static void dm_test_set_panel_type_vsdb_miniled(struct kunit *test)
+{
+	struct dm_panel_type_fixture fixture = {};
+
+	setup_panel_type_fixture(test, &fixture);
+	fixture.aconnector->base.display_info.amd_vsdb.panel_type =
+		AMD_VSDB_PANEL_TYPE_MINILED;
+
+	amdgpu_dm_set_panel_type(fixture.aconnector);
+
+	KUNIT_EXPECT_EQ(test, (int)fixture.link->panel_type,
+			(int)PANEL_TYPE_MINILED);
+}
+
+/**
+ * dm_test_set_panel_type_dpcd_oled - Test DPCD oled bit maps to PANEL_TYPE_OLED
+ * @test: The KUnit test context
+ */
+static void dm_test_set_panel_type_dpcd_oled(struct kunit *test)
+{
+	struct dm_panel_type_fixture fixture = {};
+
+	setup_panel_type_fixture(test, &fixture);
+	fixture.link->dpcd_sink_ext_caps.bits.oled = 1;
+
+	amdgpu_dm_set_panel_type(fixture.aconnector);
+
+	KUNIT_EXPECT_EQ(test, (int)fixture.link->panel_type,
+			(int)PANEL_TYPE_OLED);
+}
+
+/**
+ * dm_test_set_panel_type_dpcd_miniled - Test DPCD miniled bit maps to PANEL_TYPE_MINILED
+ * @test: The KUnit test context
+ */
+static void dm_test_set_panel_type_dpcd_miniled(struct kunit *test)
+{
+	struct dm_panel_type_fixture fixture = {};
+
+	setup_panel_type_fixture(test, &fixture);
+	fixture.link->dpcd_sink_ext_caps.bits.miniled = 1;
+
+	amdgpu_dm_set_panel_type(fixture.aconnector);
+
+	KUNIT_EXPECT_EQ(test, (int)fixture.link->panel_type,
+			(int)PANEL_TYPE_MINILED);
+}
+
+/**
+ * dm_test_set_panel_type_did_oled - Test DID OLED maps to PANEL_TYPE_OLED
+ * @test: The KUnit test context
+ *
+ * When VSDB and DPCD do not identify the panel, a DID panel type of
+ * DRM_MODE_PANEL_TYPE_OLED must map to PANEL_TYPE_OLED.
+ */
+static void dm_test_set_panel_type_did_oled(struct kunit *test)
+{
+	struct dm_panel_type_fixture fixture = {};
+
+	setup_panel_type_fixture(test, &fixture);
+	fixture.aconnector->base.display_info.panel_type =
+		DRM_MODE_PANEL_TYPE_OLED;
+
+	amdgpu_dm_set_panel_type(fixture.aconnector);
+
+	KUNIT_EXPECT_EQ(test, (int)fixture.link->panel_type,
+			(int)PANEL_TYPE_OLED);
+}
+
+/**
+ * dm_test_set_panel_type_did_lcd - Test DID LCD maps to PANEL_TYPE_LCD
+ * @test: The KUnit test context
+ *
+ * When VSDB and DPCD do not identify the panel, a DID panel type of
+ * DRM_MODE_PANEL_TYPE_LCD must map to PANEL_TYPE_LCD.
+ */
+static void dm_test_set_panel_type_did_lcd(struct kunit *test)
+{
+	struct dm_panel_type_fixture fixture = {};
+
+	setup_panel_type_fixture(test, &fixture);
+	fixture.aconnector->base.display_info.panel_type =
+		DRM_MODE_PANEL_TYPE_LCD;
+
+	amdgpu_dm_set_panel_type(fixture.aconnector);
+
+	KUNIT_EXPECT_EQ(test, (int)fixture.link->panel_type,
+			(int)PANEL_TYPE_LCD);
+}
+
+/**
+ * dm_test_set_panel_type_vendor_lum_heuristic - Test vendor luminance heuristic maps to MINILED
+ * @test: The KUnit test context
+ *
+ * A panel from the specific vendor whose first luminance range is at least
+ * 1.5x the second is treated as a mini-LED panel.
+ */
+static void dm_test_set_panel_type_vendor_lum_heuristic(struct kunit *test)
+{
+	struct dm_panel_type_fixture fixture = {};
+	struct drm_amd_vsdb_info *vsdb;
+
+	setup_panel_type_fixture(test, &fixture);
+	fixture.link->local_sink = fixture.sink;
+	fixture.sink->edid_caps.manufacturer_id = DDC_MANUFACTURERNAME_SAMSUNG;
+
+	vsdb = &fixture.aconnector->base.display_info.amd_vsdb;
+	vsdb->version = 1;
+	vsdb->luminance_range1.max_luminance = 3000;
+	vsdb->luminance_range2.max_luminance = 1000;
+
+	amdgpu_dm_set_panel_type(fixture.aconnector);
+
+	KUNIT_EXPECT_EQ(test, (int)fixture.link->panel_type,
+			(int)PANEL_TYPE_MINILED);
+}
+
+/**
+ * dm_test_set_panel_type_defaults_to_lcd - Test undetermined panel defaults to LCD
+ * @test: The KUnit test context
+ *
+ * When no source identifies the panel, the type now defaults to
+ * PANEL_TYPE_LCD instead of remaining PANEL_TYPE_NONE.
+ */
+static void dm_test_set_panel_type_defaults_to_lcd(struct kunit *test)
+{
+	struct dm_panel_type_fixture fixture = {};
+
+	setup_panel_type_fixture(test, &fixture);
+
+	amdgpu_dm_set_panel_type(fixture.aconnector);
+
+	KUNIT_EXPECT_EQ(test, (int)fixture.link->panel_type,
+			(int)PANEL_TYPE_LCD);
+}
+
+/* Tests for update_subconnector_property() */
+
+/**
+ * dm_test_update_subconnector_dp_with_sink - Test subconnector property is set
+ * from the dongle type for a DisplayPort connector with a sink
+ * @test: The KUnit test context
+ */
+static void dm_test_update_subconnector_dp_with_sink(struct kunit *test)
+{
+	struct device *dev;
+	struct drm_device *drm;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+	uint64_t val = 0;
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						   sizeof(*drm), 0,
+						   DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	drmm_connector_init(drm, &aconnector->base, &dm_test_connector_funcs,
+			    DRM_MODE_CONNECTOR_DisplayPort, NULL);
+	drm_connector_attach_dp_subconnector_property(&aconnector->base);
+
+	link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_VGA_CONVERTER;
+	aconnector->dc_link = link;
+	/* Any non-NULL sink enables dongle-type resolution */
+	aconnector->dc_sink = kunit_kzalloc(test, sizeof(*aconnector->dc_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector->dc_sink);
+
+	update_subconnector_property(aconnector);
+
+	KUNIT_EXPECT_EQ(test, drm_object_property_get_value(&aconnector->base.base,
+				aconnector->base.dev->mode_config.dp_subconnector_property,
+				&val), 0);
+	KUNIT_EXPECT_EQ(test, (int)val, (int)DRM_MODE_SUBCONNECTOR_VGA);
+}
+
+/**
+ * dm_test_update_subconnector_dp_no_sink - Test subconnector property stays
+ * unknown for a DisplayPort connector without a sink
+ * @test: The KUnit test context
+ */
+static void dm_test_update_subconnector_dp_no_sink(struct kunit *test)
+{
+	struct device *dev;
+	struct drm_device *drm;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+	uint64_t val = 0;
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						   sizeof(*drm), 0,
+						   DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	drmm_connector_init(drm, &aconnector->base, &dm_test_connector_funcs,
+			    DRM_MODE_CONNECTOR_DisplayPort, NULL);
+	drm_connector_attach_dp_subconnector_property(&aconnector->base);
+
+	/* Dongle type is set, but no sink means it must not be consulted */
+	link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_HDMI_CONVERTER;
+	aconnector->dc_link = link;
+	aconnector->dc_sink = NULL;
+
+	update_subconnector_property(aconnector);
+
+	KUNIT_EXPECT_EQ(test, drm_object_property_get_value(&aconnector->base.base,
+				aconnector->base.dev->mode_config.dp_subconnector_property,
+				&val), 0);
+	KUNIT_EXPECT_EQ(test, (int)val, (int)DRM_MODE_SUBCONNECTOR_Unknown);
+}
+
+/**
+ * dm_test_update_subconnector_non_dp_noop - Test non-DisplayPort connector is
+ * left untouched (early return)
+ * @test: The KUnit test context
+ */
+static void dm_test_update_subconnector_non_dp_noop(struct kunit *test)
+{
+	struct device *dev;
+	struct drm_device *drm;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+	uint64_t val = 0;
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						   sizeof(*drm), 0,
+						   DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	drmm_connector_init(drm, &aconnector->base, &dm_test_connector_funcs,
+			    DRM_MODE_CONNECTOR_HDMIA, NULL);
+	drm_connector_attach_dp_subconnector_property(&aconnector->base);
+
+	/* Pre-seed the property to a non-default value */
+	drm_object_property_set_value(&aconnector->base.base,
+			aconnector->base.dev->mode_config.dp_subconnector_property,
+			DRM_MODE_SUBCONNECTOR_VGA);
+
+	link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_HDMI_CONVERTER;
+	aconnector->dc_link = link;
+	aconnector->dc_sink = kunit_kzalloc(test, sizeof(*aconnector->dc_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector->dc_sink);
+
+	update_subconnector_property(aconnector);
+
+	/* Non-DP connector: value must remain what we seeded */
+	KUNIT_EXPECT_EQ(test, drm_object_property_get_value(&aconnector->base.base,
+				aconnector->base.dev->mode_config.dp_subconnector_property,
+				&val), 0);
+	KUNIT_EXPECT_EQ(test, (int)val, (int)DRM_MODE_SUBCONNECTOR_VGA);
+}
+
+/* Tests for amdgpu_dm_fbc_init() */
+
+/*
+ * Build an amdgpu_dm_connector wired to a kunit-allocated amdgpu_device so
+ * that drm_to_adev() and to_amdgpu_dm_connector() resolve correctly, with a
+ * dc, dc_link and an empty modes list ready for amdgpu_dm_fbc_init().
+ */
+struct dm_test_fbc_ctx {
+	struct amdgpu_device *adev;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc *dc;
+	struct dc_link *link;
+};
+
+static struct dm_test_fbc_ctx *dm_test_fbc_ctx_alloc(struct kunit *test)
+{
+	struct dm_test_fbc_ctx *ctx;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	ctx->adev = kunit_kzalloc(test, sizeof(*ctx->adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->adev);
+	ctx->aconnector = kunit_kzalloc(test, sizeof(*ctx->aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector);
+	ctx->dc = kunit_kzalloc(test, sizeof(*ctx->dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->dc);
+	ctx->link = kunit_kzalloc(test, sizeof(*ctx->link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link);
+
+	ctx->aconnector->base.dev = &ctx->adev->ddev;
+	INIT_LIST_HEAD(&ctx->aconnector->base.modes);
+	ctx->adev->dm.dc = ctx->dc;
+	ctx->aconnector->dc_link = ctx->link;
+
+	/* Default to the fully-enabled path so each test only flips one knob */
+	ctx->link->connector_signal = SIGNAL_TYPE_EDP;
+	ctx->dc->fbc_compressor =
+		(struct compressor *)kunit_kzalloc(test, sizeof(void *), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->dc->fbc_compressor);
+
+	return ctx;
+}
+
+/**
+ * dm_test_fbc_init_no_compressor - Test fbc_init is a no-op without a compressor
+ * @test: The KUnit test context
+ */
+static void dm_test_fbc_init_no_compressor(struct kunit *test)
+{
+	struct dm_test_fbc_ctx *ctx = dm_test_fbc_ctx_alloc(test);
+
+	ctx->dc->fbc_compressor = NULL;
+
+	amdgpu_dm_fbc_init(&ctx->aconnector->base);
+
+	KUNIT_EXPECT_NULL(test, ctx->adev->dm.compressor.bo_ptr);
+}
+
+/**
+ * dm_test_fbc_init_non_edp - Test fbc_init is a no-op for non-eDP links
+ * @test: The KUnit test context
+ */
+static void dm_test_fbc_init_non_edp(struct kunit *test)
+{
+	struct dm_test_fbc_ctx *ctx = dm_test_fbc_ctx_alloc(test);
+
+	ctx->link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+
+	amdgpu_dm_fbc_init(&ctx->aconnector->base);
+
+	KUNIT_EXPECT_NULL(test, ctx->adev->dm.compressor.bo_ptr);
+}
+
+/**
+ * dm_test_fbc_init_already_allocated - Test fbc_init keeps an existing buffer
+ * @test: The KUnit test context
+ */
+static void dm_test_fbc_init_already_allocated(struct kunit *test)
+{
+	struct dm_test_fbc_ctx *ctx = dm_test_fbc_ctx_alloc(test);
+	struct amdgpu_bo *existing;
+
+	existing = kunit_kzalloc(test, sizeof(void *), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, existing);
+	ctx->adev->dm.compressor.bo_ptr = existing;
+
+	amdgpu_dm_fbc_init(&ctx->aconnector->base);
+
+	/* Buffer already present → left untouched, no reallocation */
+	KUNIT_EXPECT_PTR_EQ(test, ctx->adev->dm.compressor.bo_ptr, existing);
+}
+
+/**
+ * dm_test_fbc_init_no_modes - Test fbc_init skips allocation with no modes
+ * @test: The KUnit test context
+ */
+static void dm_test_fbc_init_no_modes(struct kunit *test)
+{
+	struct dm_test_fbc_ctx *ctx = dm_test_fbc_ctx_alloc(test);
+
+	/* All prerequisites met but the modes list is empty → max_size 0 */
+	amdgpu_dm_fbc_init(&ctx->aconnector->base);
+
+	KUNIT_EXPECT_NULL(test, ctx->adev->dm.compressor.bo_ptr);
+}
+
+/* Tests for amdgpu_dm_detect_mst_link_for_all_connectors() */
+
+/* Allocate a bare drm_device suitable for registering connectors against. */
+static struct drm_device *dm_test_alloc_drm(struct kunit *test)
+{
+	struct device *dev;
+	struct drm_device *drm;
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	drm = __drm_kunit_helper_alloc_drm_device(test, dev, sizeof(*drm), 0,
+						  DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
+
+	return drm;
+}
+
+/*
+ * Allocate an amdgpu_dm_connector and register its embedded drm_connector with
+ * @drm so that drm_for_each_connector_iter() and to_amdgpu_dm_connector() both
+ * resolve to it.
+ */
+static struct amdgpu_dm_connector *dm_test_add_connector(struct kunit *test,
+		struct drm_device *drm, int connector_type)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+
+	KUNIT_ASSERT_EQ(test,
+		drmm_connector_init(drm, &aconnector->base,
+				    &dm_test_connector_funcs, connector_type,
+				    NULL), 0);
+
+	return aconnector;
+}
+
+/**
+ * dm_test_detect_mst_no_connectors - Test the no-op path on an empty device
+ * @test: The KUnit test context
+ */
+static void dm_test_detect_mst_no_connectors(struct kunit *test)
+{
+	struct drm_device *drm = dm_test_alloc_drm(test);
+
+	/* No connectors registered → iteration body never runs */
+	KUNIT_EXPECT_EQ(test,
+		amdgpu_dm_detect_mst_link_for_all_connectors(drm), 0);
+}
+
+/**
+ * dm_test_detect_mst_skips_writeback - Test writeback connectors are skipped
+ * @test: The KUnit test context
+ *
+ * A writeback connector is hit by the early ``continue`` before its dc_link is
+ * ever dereferenced, so leaving dc_link NULL must not crash.
+ */
+static void dm_test_detect_mst_skips_writeback(struct kunit *test)
+{
+	struct drm_device *drm = dm_test_alloc_drm(test);
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = dm_test_add_connector(test, drm,
+					   DRM_MODE_CONNECTOR_WRITEBACK);
+	/* dc_link intentionally left NULL: it must not be touched */
+	aconnector->dc_link = NULL;
+
+	KUNIT_EXPECT_EQ(test,
+		amdgpu_dm_detect_mst_link_for_all_connectors(drm), 0);
+}
+
+/**
+ * dm_test_detect_mst_non_mst_link - Test a non-MST link starts no topology
+ * @test: The KUnit test context
+ */
+static void dm_test_detect_mst_non_mst_link(struct kunit *test)
+{
+	struct drm_device *drm = dm_test_alloc_drm(test);
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+
+	aconnector = dm_test_add_connector(test, drm,
+					   DRM_MODE_CONNECTOR_DisplayPort);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	/* Not an MST branch → the topology manager is never started */
+	link->type = dc_connection_single;
+	aconnector->dc_link = link;
+
+	KUNIT_EXPECT_EQ(test,
+		amdgpu_dm_detect_mst_link_for_all_connectors(drm), 0);
+}
+
+/**
+ * dm_test_detect_mst_branch_without_aux - Test an MST branch with no aux is
+ * skipped
+ * @test: The KUnit test context
+ *
+ * The condition short-circuits on a NULL mst_mgr.aux, so the real
+ * drm_dp_mst_topology_mgr_set_mst() path is never reached.
+ */
+static void dm_test_detect_mst_branch_without_aux(struct kunit *test)
+{
+	struct drm_device *drm = dm_test_alloc_drm(test);
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+
+	aconnector = dm_test_add_connector(test, drm,
+					   DRM_MODE_CONNECTOR_DisplayPort);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	link->type = dc_connection_mst_branch;
+	aconnector->dc_link = link;
+	/* mst_mgr.aux is NULL (kzalloc) → second half of the && is false */
+	KUNIT_ASSERT_NULL(test, aconnector->mst_mgr.aux);
+
+	KUNIT_EXPECT_EQ(test,
+		amdgpu_dm_detect_mst_link_for_all_connectors(drm), 0);
+}
+
+/* Tests for amdgpu_dm_find_first_crtc_matching_connector() */
+
+/*
+ * Build a minimal drm_atomic_commit holding @count connector slots. The
+ * function under test only reads num_connector, connectors[i].ptr and
+ * connectors[i].new_state, so a hand-rolled state is sufficient.
+ */
+static struct drm_atomic_commit *
+dm_test_alloc_atomic_state(struct kunit *test, int count)
+{
+	struct drm_atomic_commit *state;
+
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+
+	state->num_connector = count;
+	if (count) {
+		state->connectors = kunit_kcalloc(test, count,
+						  sizeof(*state->connectors),
+						  GFP_KERNEL);
+		KUNIT_ASSERT_NOT_NULL(test, state->connectors);
+	}
+
+	return state;
+}
+
+/**
+ * dm_test_find_first_crtc_match - Test find_first_crtc returns matching connector
+ * @test: The KUnit test context
+ */
+static void dm_test_find_first_crtc_match(struct kunit *test)
+{
+	struct drm_atomic_commit *state = dm_test_alloc_atomic_state(test, 1);
+	struct drm_connector *connector;
+	struct drm_connector_state *con_state;
+	struct drm_crtc *crtc;
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, connector);
+	con_state = kunit_kzalloc(test, sizeof(*con_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, con_state);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+
+	con_state->crtc = crtc;
+	state->connectors[0].ptr = connector;
+	state->connectors[0].new_state = con_state;
+
+	KUNIT_EXPECT_PTR_EQ(test,
+		amdgpu_dm_find_first_crtc_matching_connector(state, crtc),
+		connector);
+}
+
+/**
+ * dm_test_find_first_crtc_no_match - Test find_first_crtc returns NULL when no crtc matches
+ * @test: The KUnit test context
+ */
+static void dm_test_find_first_crtc_no_match(struct kunit *test)
+{
+	struct drm_atomic_commit *state = dm_test_alloc_atomic_state(test, 1);
+	struct drm_connector *connector;
+	struct drm_connector_state *con_state;
+	struct drm_crtc *crtc;
+	struct drm_crtc *other_crtc;
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, connector);
+	con_state = kunit_kzalloc(test, sizeof(*con_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, con_state);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+	other_crtc = kunit_kzalloc(test, sizeof(*other_crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, other_crtc);
+
+	con_state->crtc = other_crtc;
+	state->connectors[0].ptr = connector;
+	state->connectors[0].new_state = con_state;
+
+	KUNIT_EXPECT_NULL(test,
+		amdgpu_dm_find_first_crtc_matching_connector(state, crtc));
+}
+
+/**
+ * dm_test_find_first_crtc_empty_state - Test find_first_crtc returns NULL with no connectors
+ * @test: The KUnit test context
+ */
+static void dm_test_find_first_crtc_empty_state(struct kunit *test)
+{
+	struct drm_atomic_commit *state = dm_test_alloc_atomic_state(test, 0);
+	struct drm_crtc *crtc;
+
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+
+	KUNIT_EXPECT_NULL(test,
+		amdgpu_dm_find_first_crtc_matching_connector(state, crtc));
+}
+
+/**
+ * dm_test_find_first_crtc_skips_null_ptr - Test find_first_crtc skips empty connector slots
+ * @test: The KUnit test context
+ */
+static void dm_test_find_first_crtc_skips_null_ptr(struct kunit *test)
+{
+	struct drm_atomic_commit *state = dm_test_alloc_atomic_state(test, 2);
+	struct drm_connector *connector;
+	struct drm_connector_state *con_state;
+	struct drm_crtc *crtc;
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, connector);
+	con_state = kunit_kzalloc(test, sizeof(*con_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, con_state);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+
+	/* Slot 0 has no connector (ptr == NULL) and must be skipped. */
+	con_state->crtc = crtc;
+	state->connectors[1].ptr = connector;
+	state->connectors[1].new_state = con_state;
+
+	KUNIT_EXPECT_PTR_EQ(test,
+		amdgpu_dm_find_first_crtc_matching_connector(state, crtc),
+		connector);
+}
+
+/**
+ * dm_test_find_first_crtc_returns_first - Test find_first_crtc returns the first match
+ * @test: The KUnit test context
+ */
+static void dm_test_find_first_crtc_returns_first(struct kunit *test)
+{
+	struct drm_atomic_commit *state = dm_test_alloc_atomic_state(test, 2);
+	struct drm_connector *first;
+	struct drm_connector *second;
+	struct drm_connector_state *first_state;
+	struct drm_connector_state *second_state;
+	struct drm_crtc *crtc;
+
+	first = kunit_kzalloc(test, sizeof(*first), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, first);
+	second = kunit_kzalloc(test, sizeof(*second), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, second);
+	first_state = kunit_kzalloc(test, sizeof(*first_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, first_state);
+	second_state = kunit_kzalloc(test, sizeof(*second_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, second_state);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+
+	/* Both connectors target the same crtc; the first one must win. */
+	first_state->crtc = crtc;
+	second_state->crtc = crtc;
+	state->connectors[0].ptr = first;
+	state->connectors[0].new_state = first_state;
+	state->connectors[1].ptr = second;
+	state->connectors[1].new_state = second_state;
+
+	KUNIT_EXPECT_PTR_EQ(test,
+		amdgpu_dm_find_first_crtc_matching_connector(state, crtc),
+		first);
+}
+
+/* Tests for amdgpu_dm_set_panel_type() */
+
+/*
+ * Build an amdgpu_dm_connector registered against a real kunit drm_device that
+ * is embedded in an amdgpu_device, so drm_to_adev()/adev_to_drm() resolve and
+ * the panel_type property can be created, attached and updated for real.
+ */
+struct dm_test_panel_ctx {
+	struct amdgpu_device *adev;
+	struct drm_device *drm;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+	struct drm_display_info *display_info;
+};
+
+static struct dm_test_panel_ctx *dm_test_panel_ctx_alloc(struct kunit *test)
+{
+	struct dm_test_panel_ctx *ctx;
+	struct drm_property *prop;
+	struct device *dev;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	ctx->drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						       sizeof(*ctx->adev),
+						       offsetof(struct amdgpu_device, ddev),
+						       DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->drm);
+	ctx->adev = drm_to_adev(ctx->drm);
+
+	/* The function under test writes through this property. */
+	prop = drm_property_create_range(ctx->drm, DRM_MODE_PROP_IMMUTABLE,
+					 "panel_type", 0, 0xff);
+	KUNIT_ASSERT_NOT_NULL(test, prop);
+	ctx->drm->mode_config.panel_type_property = prop;
+
+	ctx->aconnector = kunit_kzalloc(test, sizeof(*ctx->aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector);
+	KUNIT_ASSERT_EQ(test,
+		drmm_connector_init(ctx->drm, &ctx->aconnector->base,
+				    &dm_test_connector_funcs,
+				    DRM_MODE_CONNECTOR_eDP, NULL), 0);
+	drm_object_attach_property(&ctx->aconnector->base.base, prop,
+				   DRM_MODE_PANEL_TYPE_UNKNOWN);
+
+	ctx->link = kunit_kzalloc(test, sizeof(*ctx->link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link);
+	ctx->aconnector->dc_link = ctx->link;
+
+	ctx->display_info = &ctx->aconnector->base.display_info;
+
+	return ctx;
+}
+
+static uint64_t dm_test_panel_prop_value(struct kunit *test,
+					 struct dm_test_panel_ctx *ctx)
+{
+	uint64_t val = ~0ULL;
+
+	KUNIT_EXPECT_EQ(test,
+		drm_object_property_get_value(&ctx->aconnector->base.base,
+			ctx->drm->mode_config.panel_type_property, &val), 0);
+	return val;
+}
+
+/**
+ * dm_test_set_panel_type_samsung_miniled - Test Samsung luminance heuristic
+ * @test: The KUnit test context
+ *
+ * When no VSDB or DPCD hint is present, a Samsung sink whose first luminance
+ * range is at least 1.5x the second is treated as mini-LED.
+ */
+static void dm_test_set_panel_type_samsung_miniled(struct kunit *test)
+{
+	struct dm_test_panel_ctx *ctx = dm_test_panel_ctx_alloc(test);
+	struct dc_sink *sink;
+
+	sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+	sink->edid_caps.manufacturer_id = DDC_MANUFACTURERNAME_SAMSUNG;
+	ctx->link->local_sink = sink;
+
+	ctx->display_info->amd_vsdb.version = 1;
+	ctx->display_info->amd_vsdb.luminance_range1.max_luminance = 1500;
+	ctx->display_info->amd_vsdb.luminance_range2.max_luminance = 1000;
+
+	amdgpu_dm_set_panel_type(ctx->aconnector);
+
+	KUNIT_EXPECT_EQ(test, (int)ctx->link->panel_type, (int)PANEL_TYPE_MINILED);
+}
+
+/**
+ * dm_test_set_panel_type_samsung_below_threshold - Test Samsung sink below the
+ * mini-LED luminance threshold falls back to LCD
+ * @test: The KUnit test context
+ */
+static void dm_test_set_panel_type_samsung_below_threshold(struct kunit *test)
+{
+	struct dm_test_panel_ctx *ctx = dm_test_panel_ctx_alloc(test);
+	struct dc_sink *sink;
+
+	sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+	sink->edid_caps.manufacturer_id = DDC_MANUFACTURERNAME_SAMSUNG;
+	ctx->link->local_sink = sink;
+
+	ctx->display_info->amd_vsdb.version = 1;
+	ctx->display_info->amd_vsdb.luminance_range1.max_luminance = 1000;
+	ctx->display_info->amd_vsdb.luminance_range2.max_luminance = 1000;
+
+	amdgpu_dm_set_panel_type(ctx->aconnector);
+
+	KUNIT_EXPECT_EQ(test, (int)ctx->link->panel_type, (int)PANEL_TYPE_LCD);
+}
+
+/**
+ * dm_test_set_panel_type_default_lcd - Test default fallback is LCD
+ * @test: The KUnit test context
+ *
+ * With no VSDB, DPCD or DID hints the panel type defaults to LCD.
+ */
+static void dm_test_set_panel_type_default_lcd(struct kunit *test)
+{
+	struct dm_test_panel_ctx *ctx = dm_test_panel_ctx_alloc(test);
+
+	amdgpu_dm_set_panel_type(ctx->aconnector);
+
+	KUNIT_EXPECT_EQ(test, (int)ctx->link->panel_type, (int)PANEL_TYPE_LCD);
+	KUNIT_EXPECT_EQ(test, dm_test_panel_prop_value(test, ctx),
+			(uint64_t)DRM_MODE_PANEL_TYPE_LCD);
+}
+
+/* Tests for amdgpu_dm_update_cacp_caps() */
+
+/*
+ * Build an amdgpu_dm_connector wired to a real kunit drm_device embedded in an
+ * amdgpu_device, so drm_to_adev() resolves and drm_dbg_kms() has a valid
+ * device. Defaults are seeded to the fully-supported configuration so each
+ * test only flips a single knob.
+ */
+struct dm_test_cacp_ctx {
+	struct amdgpu_device *adev;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+};
+
+static struct dm_test_cacp_ctx *dm_test_cacp_ctx_alloc(struct kunit *test)
+{
+	struct dm_test_cacp_ctx *ctx;
+	struct drm_device *drm;
+	struct device *dev;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						  sizeof(*ctx->adev),
+						  offsetof(struct amdgpu_device, ddev),
+						  DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
+	ctx->adev = drm_to_adev(drm);
+
+	ctx->aconnector = kunit_kzalloc(test, sizeof(*ctx->aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector);
+	ctx->aconnector->base.dev = drm;
+
+	ctx->link = kunit_kzalloc(test, sizeof(*ctx->link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link);
+	ctx->aconnector->dc_link = ctx->link;
+
+	/* Fully-supported defaults: new enough DCE, eDP, non-LCD panel. */
+	ctx->adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 1, 4);
+	ctx->link->connector_signal = SIGNAL_TYPE_EDP;
+	ctx->link->panel_type = PANEL_TYPE_OLED;
+
+	return ctx;
+}
+
+/**
+ * dm_test_cacp_caps_edp_supported - Test CACP supported on a new eDP panel
+ * @test: The KUnit test context
+ */
+static void dm_test_cacp_caps_edp_supported(struct kunit *test)
+{
+	struct dm_test_cacp_ctx *ctx = dm_test_cacp_ctx_alloc(test);
+
+	amdgpu_dm_update_cacp_caps(ctx->aconnector);
+
+	KUNIT_EXPECT_TRUE(test, ctx->link->panel_config.cacp.cacp_supported);
+}
+
+/**
+ * dm_test_cacp_caps_lvds_supported - Test CACP supported on an LVDS panel
+ * @test: The KUnit test context
+ */
+static void dm_test_cacp_caps_lvds_supported(struct kunit *test)
+{
+	struct dm_test_cacp_ctx *ctx = dm_test_cacp_ctx_alloc(test);
+
+	ctx->link->connector_signal = SIGNAL_TYPE_LVDS;
+
+	amdgpu_dm_update_cacp_caps(ctx->aconnector);
+
+	KUNIT_EXPECT_TRUE(test, ctx->link->panel_config.cacp.cacp_supported);
+}
+
+/**
+ * dm_test_cacp_caps_old_ip_unsupported - Test CACP unsupported on old DCE
+ * @test: The KUnit test context
+ *
+ * DCE versions older than 3.1.4 do not support CACP.
+ */
+static void dm_test_cacp_caps_old_ip_unsupported(struct kunit *test)
+{
+	struct dm_test_cacp_ctx *ctx = dm_test_cacp_ctx_alloc(test);
+
+	ctx->adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 1, 3);
+
+	amdgpu_dm_update_cacp_caps(ctx->aconnector);
+
+	KUNIT_EXPECT_FALSE(test, ctx->link->panel_config.cacp.cacp_supported);
+}
+
+/**
+ * dm_test_cacp_caps_ip_3_1_6_unsupported - Test CACP unsupported on DCE 3.1.6
+ * @test: The KUnit test context
+ *
+ * DCE 3.1.6 is explicitly excluded even though it is newer than 3.1.4.
+ */
+static void dm_test_cacp_caps_ip_3_1_6_unsupported(struct kunit *test)
+{
+	struct dm_test_cacp_ctx *ctx = dm_test_cacp_ctx_alloc(test);
+
+	ctx->adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 1, 6);
+
+	amdgpu_dm_update_cacp_caps(ctx->aconnector);
+
+	KUNIT_EXPECT_FALSE(test, ctx->link->panel_config.cacp.cacp_supported);
+}
+
+/**
+ * dm_test_cacp_caps_non_edp_lvds_unsupported - Test CACP unsupported on a
+ * non-eDP/LVDS signal
+ * @test: The KUnit test context
+ */
+static void dm_test_cacp_caps_non_edp_lvds_unsupported(struct kunit *test)
+{
+	struct dm_test_cacp_ctx *ctx = dm_test_cacp_ctx_alloc(test);
+
+	ctx->link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+
+	amdgpu_dm_update_cacp_caps(ctx->aconnector);
+
+	KUNIT_EXPECT_FALSE(test, ctx->link->panel_config.cacp.cacp_supported);
+}
+
+/**
+ * dm_test_cacp_caps_lcd_unsupported - Test CACP unsupported on an LCD panel
+ * @test: The KUnit test context
+ */
+static void dm_test_cacp_caps_lcd_unsupported(struct kunit *test)
+{
+	struct dm_test_cacp_ctx *ctx = dm_test_cacp_ctx_alloc(test);
+
+	ctx->link->panel_type = PANEL_TYPE_LCD;
+
+	amdgpu_dm_update_cacp_caps(ctx->aconnector);
+
+	KUNIT_EXPECT_FALSE(test, ctx->link->panel_config.cacp.cacp_supported);
+}
+
+/* Tests for fill_stream_properties_from_drm_display_mode() */
+
+/*
+ * Build the inputs for fill_stream_properties_from_drm_display_mode(). The
+ * connector is registered against a real kunit drm_device so that
+ * to_amdgpu_dm_connector(), connector->display_info and the drm debug helpers
+ * all resolve. Large structs are heap-allocated to keep the stack small.
+ *
+ * The stream signal defaults to DisplayPort so the HDMI infoframe paths are
+ * skipped, keeping the exercised behaviour deterministic.
+ */
+struct dm_test_fill_ctx {
+	struct drm_device *drm;
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_connector_state *conn_state;
+	struct dc_stream_state *stream;
+	struct drm_display_mode *mode;
+};
+
+static struct dm_test_fill_ctx *dm_test_fill_ctx_alloc(struct kunit *test)
+{
+	struct dm_test_fill_ctx *ctx;
+	struct device *dev;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	ctx->drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						       sizeof(*ctx->drm), 0,
+						       DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->drm);
+
+	ctx->aconnector = kunit_kzalloc(test, sizeof(*ctx->aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector);
+	KUNIT_ASSERT_EQ(test,
+		drmm_connector_init(ctx->drm, &ctx->aconnector->base,
+				    &dm_test_connector_funcs,
+				    DRM_MODE_CONNECTOR_DisplayPort, NULL), 0);
+
+	ctx->conn_state = kunit_kzalloc(test, sizeof(*ctx->conn_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->conn_state);
+	ctx->stream = kunit_kzalloc(test, sizeof(*ctx->stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->stream);
+	ctx->mode = kunit_kzalloc(test, sizeof(*ctx->mode), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->mode);
+
+	ctx->stream->signal = SIGNAL_TYPE_DISPLAY_PORT;
+
+	return ctx;
+}
+
+/**
+ * dm_test_fill_stream_borders_zeroed - Test the timing borders are cleared
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_stream_borders_zeroed(struct kunit *test)
+{
+	struct dm_test_fill_ctx *ctx = dm_test_fill_ctx_alloc(test);
+	struct dc_crtc_timing *timing = &ctx->stream->timing;
+
+	/* Pre-seed nonzero borders to prove they get reset. */
+	timing->h_border_left = 5;
+	timing->h_border_right = 6;
+	timing->v_border_top = 7;
+	timing->v_border_bottom = 8;
+
+	fill_stream_properties_from_drm_display_mode(ctx->stream, ctx->mode,
+		&ctx->aconnector->base, ctx->conn_state, NULL, 8);
+
+	KUNIT_EXPECT_EQ(test, (int)timing->h_border_left, 0);
+	KUNIT_EXPECT_EQ(test, (int)timing->h_border_right, 0);
+	KUNIT_EXPECT_EQ(test, (int)timing->v_border_top, 0);
+	KUNIT_EXPECT_EQ(test, (int)timing->v_border_bottom, 0);
+}
+
+/**
+ * dm_test_fill_stream_rgb_defaults - Test the default RGB/sRGB output
+ * @test: The KUnit test context
+ *
+ * A plain DisplayPort sink with no YCbCr color formats produces RGB encoding
+ * and the sRGB color space, with a predefined sRGB transfer function.
+ */
+static void dm_test_fill_stream_rgb_defaults(struct kunit *test)
+{
+	struct dm_test_fill_ctx *ctx = dm_test_fill_ctx_alloc(test);
+	struct dc_crtc_timing *timing = &ctx->stream->timing;
+
+	fill_stream_properties_from_drm_display_mode(ctx->stream, ctx->mode,
+		&ctx->aconnector->base, ctx->conn_state, NULL, 8);
+
+	KUNIT_EXPECT_EQ(test, (int)timing->pixel_encoding, (int)PIXEL_ENCODING_RGB);
+	KUNIT_EXPECT_EQ(test, (int)timing->timing_3d_format,
+			(int)TIMING_3D_FORMAT_NONE);
+	KUNIT_EXPECT_EQ(test, (int)timing->scan_type, (int)SCANNING_TYPE_NODATA);
+	KUNIT_EXPECT_EQ(test, (int)ctx->stream->output_color_space,
+			(int)COLOR_SPACE_SRGB);
+	KUNIT_EXPECT_EQ(test, (int)ctx->stream->out_transfer_func.type,
+			(int)TF_TYPE_PREDEFINED);
+	KUNIT_EXPECT_EQ(test, (int)ctx->stream->out_transfer_func.tf,
+			(int)TRANSFER_FUNCTION_SRGB);
+}
+
+/**
+ * dm_test_fill_stream_sync_polarity_positive - Test sync polarity from mode flags
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_stream_sync_polarity_positive(struct kunit *test)
+{
+	struct dm_test_fill_ctx *ctx = dm_test_fill_ctx_alloc(test);
+	struct dc_crtc_timing *timing = &ctx->stream->timing;
+
+	ctx->mode->flags = DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC;
+
+	fill_stream_properties_from_drm_display_mode(ctx->stream, ctx->mode,
+		&ctx->aconnector->base, ctx->conn_state, NULL, 8);
+
+	KUNIT_EXPECT_EQ(test, (int)timing->flags.HSYNC_POSITIVE_POLARITY, 1);
+	KUNIT_EXPECT_EQ(test, (int)timing->flags.VSYNC_POSITIVE_POLARITY, 1);
+}
+
+/**
+ * dm_test_fill_stream_sync_polarity_negative - Test negative sync polarity default
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_stream_sync_polarity_negative(struct kunit *test)
+{
+	struct dm_test_fill_ctx *ctx = dm_test_fill_ctx_alloc(test);
+	struct dc_crtc_timing *timing = &ctx->stream->timing;
+
+	/* No sync flags set on the mode → polarity stays negative (0). */
+	ctx->mode->flags = 0;
+
+	fill_stream_properties_from_drm_display_mode(ctx->stream, ctx->mode,
+		&ctx->aconnector->base, ctx->conn_state, NULL, 8);
+
+	KUNIT_EXPECT_EQ(test, (int)timing->flags.HSYNC_POSITIVE_POLARITY, 0);
+	KUNIT_EXPECT_EQ(test, (int)timing->flags.VSYNC_POSITIVE_POLARITY, 0);
+}
+
+/**
+ * dm_test_fill_stream_inherits_old_stream - Test vic/polarity copied from old stream
+ * @test: The KUnit test context
+ *
+ * When an old stream is supplied its vic and sync polarities are reused instead
+ * of being derived from the mode.
+ */
+static void dm_test_fill_stream_inherits_old_stream(struct kunit *test)
+{
+	struct dm_test_fill_ctx *ctx = dm_test_fill_ctx_alloc(test);
+	struct dc_crtc_timing *timing = &ctx->stream->timing;
+	struct dc_stream_state *old_stream;
+
+	old_stream = kunit_kzalloc(test, sizeof(*old_stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_stream);
+	old_stream->timing.vic = 16;
+	old_stream->timing.flags.HSYNC_POSITIVE_POLARITY = 1;
+	old_stream->timing.flags.VSYNC_POSITIVE_POLARITY = 0;
+
+	/* Mode flags would force positive polarity if the old stream were ignored. */
+	ctx->mode->flags = DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC;
+
+	fill_stream_properties_from_drm_display_mode(ctx->stream, ctx->mode,
+		&ctx->aconnector->base, ctx->conn_state, old_stream, 8);
+
+	KUNIT_EXPECT_EQ(test, (int)timing->vic, 16);
+	KUNIT_EXPECT_EQ(test, (int)timing->flags.HSYNC_POSITIVE_POLARITY, 1);
+	KUNIT_EXPECT_EQ(test, (int)timing->flags.VSYNC_POSITIVE_POLARITY, 0);
+}
+
+/**
+ * dm_test_fill_stream_timing_from_crtc - Test timing taken from crtc_* fields
+ * @test: The KUnit test context
+ *
+ * Without a freesync video match the function uses the mode's crtc_* timing
+ * fields and scales the pixel clock to 100Hz units.
+ */
+static void dm_test_fill_stream_timing_from_crtc(struct kunit *test)
+{
+	struct dm_test_fill_ctx *ctx = dm_test_fill_ctx_alloc(test);
+	struct dc_crtc_timing *timing = &ctx->stream->timing;
+
+	ctx->mode->crtc_hdisplay = 1920;
+	ctx->mode->crtc_htotal = 2200;
+	ctx->mode->crtc_hsync_start = 2008;
+	ctx->mode->crtc_hsync_end = 2052;
+	ctx->mode->crtc_vdisplay = 1080;
+	ctx->mode->crtc_vtotal = 1125;
+	ctx->mode->crtc_vsync_start = 1084;
+	ctx->mode->crtc_vsync_end = 1089;
+	ctx->mode->crtc_clock = 148500;
+
+	fill_stream_properties_from_drm_display_mode(ctx->stream, ctx->mode,
+		&ctx->aconnector->base, ctx->conn_state, NULL, 8);
+
+	KUNIT_EXPECT_EQ(test, (int)timing->h_addressable, 1920);
+	KUNIT_EXPECT_EQ(test, (int)timing->h_total, 2200);
+	KUNIT_EXPECT_EQ(test, (int)timing->h_sync_width, 44);
+	KUNIT_EXPECT_EQ(test, (int)timing->h_front_porch, 88);
+	KUNIT_EXPECT_EQ(test, (int)timing->v_addressable, 1080);
+	KUNIT_EXPECT_EQ(test, (int)timing->v_total, 1125);
+	KUNIT_EXPECT_EQ(test, (int)timing->v_sync_width, 5);
+	KUNIT_EXPECT_EQ(test, (int)timing->v_front_porch, 4);
+	KUNIT_EXPECT_EQ(test, (int)timing->pix_clk_100hz, 1485000);
+}
+
+/**
+ * dm_test_fill_stream_color_depth_requested_bpc - Test bpc capping
+ * @test: The KUnit test context
+ *
+ * The requested bpc caps the display bpc and is rounded down to an even value.
+ */
+static void dm_test_fill_stream_color_depth_requested_bpc(struct kunit *test)
+{
+	struct dm_test_fill_ctx *ctx = dm_test_fill_ctx_alloc(test);
+	struct dc_crtc_timing *timing = &ctx->stream->timing;
+
+	ctx->aconnector->base.display_info.bpc = 12;
+
+	fill_stream_properties_from_drm_display_mode(ctx->stream, ctx->mode,
+		&ctx->aconnector->base, ctx->conn_state, NULL, 10);
+
+	KUNIT_EXPECT_EQ(test, (int)timing->display_color_depth,
+			(int)COLOR_DEPTH_101010);
+}
+
+/**
+ * dm_test_fill_stream_content_type - Test content type forwarded from state
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_stream_content_type(struct kunit *test)
+{
+	struct dm_test_fill_ctx *ctx = dm_test_fill_ctx_alloc(test);
+
+	ctx->conn_state->content_type = DRM_MODE_CONTENT_TYPE_GRAPHICS;
+
+	fill_stream_properties_from_drm_display_mode(ctx->stream, ctx->mode,
+		&ctx->aconnector->base, ctx->conn_state, NULL, 8);
+
+	KUNIT_EXPECT_EQ(test, (int)ctx->stream->content_type,
+			(int)DISPLAY_CONTENT_TYPE_GRAPHICS);
+}
+
+/**
+ * dm_test_fill_stream_aspect_ratio - Test aspect ratio mapped from the mode
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_stream_aspect_ratio(struct kunit *test)
+{
+	struct dm_test_fill_ctx *ctx = dm_test_fill_ctx_alloc(test);
+	struct dc_crtc_timing *timing = &ctx->stream->timing;
+
+	ctx->mode->picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9;
+
+	fill_stream_properties_from_drm_display_mode(ctx->stream, ctx->mode,
+		&ctx->aconnector->base, ctx->conn_state, NULL, 8);
+
+	KUNIT_EXPECT_EQ(test, (int)timing->aspect_ratio,
+			(int)ASPECT_RATIO_16_9);
+}
+
+static struct kunit_case amdgpu_dm_connector_tests[] = {
+	/* get_subconnector_type */
+	KUNIT_CASE(dm_test_subconnector_type_none),
+	KUNIT_CASE(dm_test_subconnector_type_vga),
+	KUNIT_CASE(dm_test_subconnector_type_dvi_converter),
+	KUNIT_CASE(dm_test_subconnector_type_dvi_dongle),
+	KUNIT_CASE(dm_test_subconnector_type_hdmi_converter),
+	KUNIT_CASE(dm_test_subconnector_type_hdmi_dongle),
+	KUNIT_CASE(dm_test_subconnector_type_mismatched),
+	KUNIT_CASE(dm_test_subconnector_type_default_unknown),
+	/* get_output_content_type */
+	KUNIT_CASE(dm_test_content_type_no_data),
+	KUNIT_CASE(dm_test_content_type_graphics),
+	KUNIT_CASE(dm_test_content_type_photo),
+	KUNIT_CASE(dm_test_content_type_cinema),
+	KUNIT_CASE(dm_test_content_type_game),
+	KUNIT_CASE(dm_test_content_type_unknown_defaults_no_data),
+	/* adjust_colour_depth_from_display_info */
+	KUNIT_CASE(dm_test_adjust_colour_depth_fits_at_888),
+	KUNIT_CASE(dm_test_adjust_colour_depth_reduces_to_888),
+	KUNIT_CASE(dm_test_adjust_colour_depth_10bpc_passes),
+	KUNIT_CASE(dm_test_adjust_colour_depth_420_halves_clk),
+	KUNIT_CASE(dm_test_adjust_colour_depth_420_reduces),
+	KUNIT_CASE(dm_test_adjust_colour_depth_reduces_12bpc_to_10bpc),
+	KUNIT_CASE(dm_test_adjust_colour_depth_16bpc_no_fallback),
+	KUNIT_CASE(dm_test_adjust_colour_depth_none_fits),
+	KUNIT_CASE(dm_test_adjust_colour_depth_invalid_depth),
+	/* amdgpu_dm_get_output_color_space */
+	KUNIT_CASE(dm_test_output_color_space_default_rgb_full),
+	KUNIT_CASE(dm_test_output_color_space_default_rgb_limited),
+	KUNIT_CASE(dm_test_output_color_space_default_ycbcr709),
+	KUNIT_CASE(dm_test_output_color_space_default_ycbcr601_limited),
+	KUNIT_CASE(dm_test_output_color_space_bt601_y_only),
+	KUNIT_CASE(dm_test_output_color_space_bt601),
+	KUNIT_CASE(dm_test_output_color_space_bt709),
+	KUNIT_CASE(dm_test_output_color_space_bt709_y_only),
+	KUNIT_CASE(dm_test_output_color_space_oprgb),
+	KUNIT_CASE(dm_test_output_color_space_bt2020_rgb),
+	KUNIT_CASE(dm_test_output_color_space_bt2020_ycc),
+	KUNIT_CASE(dm_test_output_color_space_default_ycbcr709_y_only),
+	KUNIT_CASE(dm_test_output_color_space_default_ycbcr601),
+	KUNIT_CASE(dm_test_output_color_space_bt2020_ycc_rgb_encoding),
+	KUNIT_CASE(dm_test_output_color_space_bt2020_rgb_ycc_encoding),
+	/* Tests for amdgpu_dm_convert_dc_color_depth_into_bpc */
+	KUNIT_CASE(dm_test_convert_color_depth_bpc_mappings),
+	KUNIT_CASE(dm_test_convert_color_depth_bpc_unknown),
+	/* amdgpu_dm_convert_color_depth_from_display_info */
+	KUNIT_CASE(dm_test_color_depth_from_info_bpc8),
+	KUNIT_CASE(dm_test_color_depth_from_info_bpc10),
+	KUNIT_CASE(dm_test_color_depth_from_info_zero_bpc_defaults_888),
+	KUNIT_CASE(dm_test_color_depth_from_info_requested_bpc_caps),
+	KUNIT_CASE(dm_test_color_depth_from_info_y420_default),
+	KUNIT_CASE(dm_test_color_depth_from_info_y420_10bpc),
+	KUNIT_CASE(dm_test_color_depth_from_info_y420_12bpc),
+	KUNIT_CASE(dm_test_color_depth_from_info_y420_16bpc),
+	KUNIT_CASE(dm_test_color_depth_from_info_requested_odd_bpc),
+	KUNIT_CASE(dm_test_color_depth_from_info_unsupported_bpc),
+	/* to_drm_connector_type */
+	KUNIT_CASE(dm_test_to_connector_type_hdmi),
+	KUNIT_CASE(dm_test_to_connector_type_edp),
+	KUNIT_CASE(dm_test_to_connector_type_lvds),
+	KUNIT_CASE(dm_test_to_connector_type_rgb),
+	KUNIT_CASE(dm_test_to_connector_type_dp),
+	KUNIT_CASE(dm_test_to_connector_type_dp_mst),
+	KUNIT_CASE(dm_test_to_connector_type_dvi_dvii),
+	KUNIT_CASE(dm_test_to_connector_type_dual_link_dvii),
+	KUNIT_CASE(dm_test_to_connector_type_dvi_dvid),
+	KUNIT_CASE(dm_test_to_connector_type_dual_link_dvid),
+	KUNIT_CASE(dm_test_to_connector_type_virtual),
+	KUNIT_CASE(dm_test_to_connector_type_unknown),
+	/* is_duplicate_mode */
+	KUNIT_CASE(dm_test_is_duplicate_mode_empty_list),
+	KUNIT_CASE(dm_test_is_duplicate_mode_match),
+	KUNIT_CASE(dm_test_is_duplicate_mode_no_match),
+	KUNIT_CASE(dm_test_is_duplicate_mode_same_size_different_clock),
+	/* amdgpu_dm_get_encoder_crtc_mask */
+	KUNIT_CASE(dm_test_encoder_crtc_mask_1),
+	KUNIT_CASE(dm_test_encoder_crtc_mask_2),
+	KUNIT_CASE(dm_test_encoder_crtc_mask_3),
+	KUNIT_CASE(dm_test_encoder_crtc_mask_4),
+	KUNIT_CASE(dm_test_encoder_crtc_mask_5),
+	KUNIT_CASE(dm_test_encoder_crtc_mask_6),
+	KUNIT_CASE(dm_test_encoder_crtc_mask_default),
+	/* get_aspect_ratio */
+	KUNIT_CASE(dm_test_aspect_ratio_no_data),
+	KUNIT_CASE(dm_test_aspect_ratio_4_3),
+	KUNIT_CASE(dm_test_aspect_ratio_16_9),
+	KUNIT_CASE(dm_test_aspect_ratio_64_27),
+	KUNIT_CASE(dm_test_aspect_ratio_256_135),
+	/* copy_crtc_timing_for_drm_display_mode */
+	KUNIT_CASE(dm_test_copy_crtc_timing_copies_all_fields),
+	KUNIT_CASE(dm_test_copy_crtc_timing_leaves_non_crtc_fields),
+	/* decide_crtc_timing_for_drm_display_mode */
+	KUNIT_CASE(dm_test_decide_crtc_timing_scale_enabled),
+	KUNIT_CASE(dm_test_decide_crtc_timing_matching_mode),
+	KUNIT_CASE(dm_test_decide_crtc_timing_no_copy),
+	KUNIT_CASE(dm_test_decide_crtc_timing_no_crtc_clock),
+	/* amdgpu_dm_connector_funcs_reset */
+	KUNIT_CASE(dm_test_funcs_reset_sets_defaults),
+	KUNIT_CASE(dm_test_funcs_reset_edp_abm_level),
+	KUNIT_CASE(dm_test_funcs_reset_edp_abm_disabled),
+	/* amdgpu_dm_connector_atomic_duplicate_state */
+	KUNIT_CASE(dm_test_atomic_dup_state_copies_fields),
+	/* amdgpu_dm_fill_hdr_info_packet */
+	KUNIT_CASE(dm_test_fill_hdr_null_metadata),
+	KUNIT_CASE(dm_test_fill_hdr_zeroes_output),
+	/* amdgpu_dm_connector_atomic_set_property */
+	KUNIT_CASE(dm_test_set_property_scaling_center),
+	KUNIT_CASE(dm_test_set_property_scaling_aspect),
+	KUNIT_CASE(dm_test_set_property_scaling_fullscreen),
+	KUNIT_CASE(dm_test_set_property_scaling_none),
+	KUNIT_CASE(dm_test_set_property_scaling_unchanged),
+	KUNIT_CASE(dm_test_set_property_underscan_hborder),
+	KUNIT_CASE(dm_test_set_property_underscan_vborder),
+	KUNIT_CASE(dm_test_set_property_underscan_enable),
+	KUNIT_CASE(dm_test_set_property_abm_sysfs_control),
+	KUNIT_CASE(dm_test_set_property_abm_level_off),
+	KUNIT_CASE(dm_test_set_property_abm_level_value),
+	KUNIT_CASE(dm_test_set_property_unknown),
+	/* amdgpu_dm_connector_atomic_get_property */
+	KUNIT_CASE(dm_test_get_property_scaling_center),
+	KUNIT_CASE(dm_test_get_property_scaling_aspect),
+	KUNIT_CASE(dm_test_get_property_scaling_full),
+	KUNIT_CASE(dm_test_get_property_scaling_off),
+	KUNIT_CASE(dm_test_get_property_underscan_borders),
+	KUNIT_CASE(dm_test_get_property_abm_sysfs_allowed),
+	KUNIT_CASE(dm_test_get_property_abm_level),
+	KUNIT_CASE(dm_test_get_property_abm_disabled_zero),
+	KUNIT_CASE(dm_test_get_property_unknown),
+	/* amdgpu_dm_get_highest_refresh_rate_mode */
+	KUNIT_CASE(dm_test_highest_refresh_writeback_null),
+	KUNIT_CASE(dm_test_highest_refresh_cached_base),
+	KUNIT_CASE(dm_test_highest_refresh_preferred_mode),
+	/* amdgpu_dm_is_freesync_video_mode */
+	KUNIT_CASE(dm_test_is_freesync_video_mode_null_mode),
+	KUNIT_CASE(dm_test_is_freesync_video_mode_match),
+	KUNIT_CASE(dm_test_is_freesync_video_mode_no_match),
+	/* update_subconnector_property */
+	KUNIT_CASE(dm_test_update_subconnector_dp_with_sink),
+	KUNIT_CASE(dm_test_update_subconnector_dp_no_sink),
+	KUNIT_CASE(dm_test_update_subconnector_non_dp_noop),
+	/* amdgpu_dm_update_cacp_caps */
+	KUNIT_CASE(dm_test_cacp_caps_unsupported_ip),
+	KUNIT_CASE(dm_test_cacp_caps_excluded_ip_316),
+	KUNIT_CASE(dm_test_cacp_caps_edp_oled_supported),
+	KUNIT_CASE(dm_test_cacp_caps_lvds_oled_supported),
+	KUNIT_CASE(dm_test_cacp_caps_non_edp_signal),
+	KUNIT_CASE(dm_test_cacp_caps_lcd_panel),
+	/* amdgpu_dm_set_panel_type */
+	KUNIT_CASE(dm_test_set_panel_type_vsdb_oled),
+	KUNIT_CASE(dm_test_set_panel_type_vsdb_miniled),
+	KUNIT_CASE(dm_test_set_panel_type_dpcd_oled),
+	KUNIT_CASE(dm_test_set_panel_type_dpcd_miniled),
+	KUNIT_CASE(dm_test_set_panel_type_did_oled),
+	KUNIT_CASE(dm_test_set_panel_type_did_lcd),
+	KUNIT_CASE(dm_test_set_panel_type_vendor_lum_heuristic),
+	KUNIT_CASE(dm_test_set_panel_type_defaults_to_lcd),
+	/* amdgpu_dm_fbc_init */
+	KUNIT_CASE(dm_test_fbc_init_no_compressor),
+	KUNIT_CASE(dm_test_fbc_init_non_edp),
+	KUNIT_CASE(dm_test_fbc_init_already_allocated),
+	KUNIT_CASE(dm_test_fbc_init_no_modes),
+	/* amdgpu_dm_detect_mst_link_for_all_connectors */
+	KUNIT_CASE(dm_test_detect_mst_no_connectors),
+	KUNIT_CASE(dm_test_detect_mst_skips_writeback),
+	KUNIT_CASE(dm_test_detect_mst_non_mst_link),
+	KUNIT_CASE(dm_test_detect_mst_branch_without_aux),
+	/* amdgpu_dm_find_first_crtc_matching_connector */
+	KUNIT_CASE(dm_test_find_first_crtc_match),
+	KUNIT_CASE(dm_test_find_first_crtc_no_match),
+	KUNIT_CASE(dm_test_find_first_crtc_empty_state),
+	KUNIT_CASE(dm_test_find_first_crtc_skips_null_ptr),
+	KUNIT_CASE(dm_test_find_first_crtc_returns_first),
+	/* amdgpu_dm_set_panel_type */
+	KUNIT_CASE(dm_test_set_panel_type_samsung_miniled),
+	KUNIT_CASE(dm_test_set_panel_type_samsung_below_threshold),
+	KUNIT_CASE(dm_test_set_panel_type_default_lcd),
+	/* amdgpu_dm_update_cacp_caps */
+	KUNIT_CASE(dm_test_cacp_caps_edp_supported),
+	KUNIT_CASE(dm_test_cacp_caps_lvds_supported),
+	KUNIT_CASE(dm_test_cacp_caps_old_ip_unsupported),
+	KUNIT_CASE(dm_test_cacp_caps_ip_3_1_6_unsupported),
+	KUNIT_CASE(dm_test_cacp_caps_non_edp_lvds_unsupported),
+	KUNIT_CASE(dm_test_cacp_caps_lcd_unsupported),
+	/* fill_stream_properties_from_drm_display_mode */
+	KUNIT_CASE(dm_test_fill_stream_borders_zeroed),
+	KUNIT_CASE(dm_test_fill_stream_rgb_defaults),
+	KUNIT_CASE(dm_test_fill_stream_sync_polarity_positive),
+	KUNIT_CASE(dm_test_fill_stream_sync_polarity_negative),
+	KUNIT_CASE(dm_test_fill_stream_inherits_old_stream),
+	KUNIT_CASE(dm_test_fill_stream_timing_from_crtc),
+	KUNIT_CASE(dm_test_fill_stream_color_depth_requested_bpc),
+	KUNIT_CASE(dm_test_fill_stream_content_type),
+	KUNIT_CASE(dm_test_fill_stream_aspect_ratio),
+	{}
+};
+
+static struct kunit_suite amdgpu_dm_connector_test_suite = {
+	.name = "amdgpu_dm_connector",
+	.test_cases = amdgpu_dm_connector_tests,
+};
+
+kunit_test_suite(amdgpu_dm_connector_test_suite);
+
+MODULE_AUTHOR("AMD");
+MODULE_DESCRIPTION("KUnit tests for amdgpu_dm_connector");
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_crc_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_crc_test.c
index bba8b1a8fa1c0..a6fd3a6fd803a 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_crc_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_crc_test.c
@@ -95,17 +95,139 @@ static void dm_test_is_valid_crc_source(struct kunit *test)
 	KUNIT_EXPECT_FALSE(test, amdgpu_dm_is_valid_crc_source(AMDGPU_DM_PIPE_CRC_SOURCE_INVALID));
 }
 
+/**
+ * dm_test_need_dp_aux() - Test dm_need_dp_aux().
+ * @test: KUnit test context.
+ *
+ * Verifies that dm_need_dp_aux() returns true when the transition starts or
+ * stops a DPRX CRC source (requiring the DP AUX handle), and false for
+ * non-DPRX transitions such as CRTC or NONE→NONE.
+ */
+static void dm_test_need_dp_aux(struct kunit *test)
+{
+	/* Starting a DPRX source always needs AUX, regardless of current source */
+	KUNIT_EXPECT_TRUE(test, dm_need_dp_aux(AMDGPU_DM_PIPE_CRC_SOURCE_DPRX,
+					       AMDGPU_DM_PIPE_CRC_SOURCE_NONE));
+	KUNIT_EXPECT_TRUE(test, dm_need_dp_aux(AMDGPU_DM_PIPE_CRC_SOURCE_DPRX,
+					       AMDGPU_DM_PIPE_CRC_SOURCE_CRTC));
+	KUNIT_EXPECT_TRUE(test, dm_need_dp_aux(AMDGPU_DM_PIPE_CRC_SOURCE_DPRX_DITHER,
+					       AMDGPU_DM_PIPE_CRC_SOURCE_NONE));
+
+	/* Stopping a DPRX source (NONE requested, DPRX was active) needs AUX */
+	KUNIT_EXPECT_TRUE(test, dm_need_dp_aux(AMDGPU_DM_PIPE_CRC_SOURCE_NONE,
+					       AMDGPU_DM_PIPE_CRC_SOURCE_DPRX));
+	KUNIT_EXPECT_TRUE(test, dm_need_dp_aux(AMDGPU_DM_PIPE_CRC_SOURCE_NONE,
+					       AMDGPU_DM_PIPE_CRC_SOURCE_DPRX_DITHER));
+
+	/* CRTC transitions do not need AUX */
+	KUNIT_EXPECT_FALSE(test, dm_need_dp_aux(AMDGPU_DM_PIPE_CRC_SOURCE_CRTC,
+						AMDGPU_DM_PIPE_CRC_SOURCE_NONE));
+	KUNIT_EXPECT_FALSE(test, dm_need_dp_aux(AMDGPU_DM_PIPE_CRC_SOURCE_NONE,
+						AMDGPU_DM_PIPE_CRC_SOURCE_CRTC));
+	KUNIT_EXPECT_FALSE(test, dm_need_dp_aux(AMDGPU_DM_PIPE_CRC_SOURCE_NONE,
+						AMDGPU_DM_PIPE_CRC_SOURCE_NONE));
+}
+
+/**
+ * dm_test_crc_source_should_start_dprx() - Test dm_crc_source_should_start_dprx().
+ * @test: KUnit test context.
+ *
+ * Verifies that dm_crc_source_should_start_dprx() returns true only when CRC
+ * is transitioning from off (!enabled) to a DPRX source (enable &&
+ * is_dprx(source)), and false for all other combinations including
+ * already-enabled or non-DPRX targets.
+ */
+static void dm_test_crc_source_should_start_dprx(struct kunit *test)
+{
+	/* CRC off → DPRX: should start */
+	KUNIT_EXPECT_TRUE(test,
+		dm_crc_source_should_start_dprx(AMDGPU_DM_PIPE_CRC_SOURCE_DPRX,
+						AMDGPU_DM_PIPE_CRC_SOURCE_NONE));
+	KUNIT_EXPECT_TRUE(test,
+		dm_crc_source_should_start_dprx(AMDGPU_DM_PIPE_CRC_SOURCE_DPRX_DITHER,
+						AMDGPU_DM_PIPE_CRC_SOURCE_NONE));
+
+	/* CRC already on (any source) → DPRX: should NOT start (already enabled) */
+	KUNIT_EXPECT_FALSE(test,
+		dm_crc_source_should_start_dprx(AMDGPU_DM_PIPE_CRC_SOURCE_DPRX,
+						AMDGPU_DM_PIPE_CRC_SOURCE_CRTC));
+	KUNIT_EXPECT_FALSE(test,
+		dm_crc_source_should_start_dprx(AMDGPU_DM_PIPE_CRC_SOURCE_DPRX,
+						AMDGPU_DM_PIPE_CRC_SOURCE_DPRX));
+
+	/* CRC off → CRTC: not a DPRX start */
+	KUNIT_EXPECT_FALSE(test,
+		dm_crc_source_should_start_dprx(AMDGPU_DM_PIPE_CRC_SOURCE_CRTC,
+						AMDGPU_DM_PIPE_CRC_SOURCE_NONE));
+
+	/* Disabling: should not start */
+	KUNIT_EXPECT_FALSE(test,
+		dm_crc_source_should_start_dprx(AMDGPU_DM_PIPE_CRC_SOURCE_NONE,
+						AMDGPU_DM_PIPE_CRC_SOURCE_DPRX));
+}
+
+/**
+ * dm_test_crc_source_should_stop_dprx() - Test dm_crc_source_should_stop_dprx().
+ * @test: KUnit test context.
+ *
+ * Verifies that dm_crc_source_should_stop_dprx() returns true only when CRC
+ * is transitioning from a DPRX source (enabled && is_dprx(cur_crc_src)) to
+ * off (!enable), and false for non-DPRX disables, DPRX starts, and no-op
+ * transitions.
+ */
+static void dm_test_crc_source_should_stop_dprx(struct kunit *test)
+{
+	/* DPRX → off: should stop */
+	KUNIT_EXPECT_TRUE(test,
+		dm_crc_source_should_stop_dprx(AMDGPU_DM_PIPE_CRC_SOURCE_NONE,
+					       AMDGPU_DM_PIPE_CRC_SOURCE_DPRX));
+	KUNIT_EXPECT_TRUE(test,
+		dm_crc_source_should_stop_dprx(AMDGPU_DM_PIPE_CRC_SOURCE_NONE,
+					       AMDGPU_DM_PIPE_CRC_SOURCE_DPRX_DITHER));
+
+	/* CRTC → off: not a DPRX stop */
+	KUNIT_EXPECT_FALSE(test,
+		dm_crc_source_should_stop_dprx(AMDGPU_DM_PIPE_CRC_SOURCE_NONE,
+					       AMDGPU_DM_PIPE_CRC_SOURCE_CRTC));
+
+	/* off → DPRX: not a stop */
+	KUNIT_EXPECT_FALSE(test,
+		dm_crc_source_should_stop_dprx(AMDGPU_DM_PIPE_CRC_SOURCE_DPRX,
+					       AMDGPU_DM_PIPE_CRC_SOURCE_NONE));
+
+	/* DPRX → DPRX: no transition, not a stop */
+	KUNIT_EXPECT_FALSE(test,
+		dm_crc_source_should_stop_dprx(AMDGPU_DM_PIPE_CRC_SOURCE_DPRX,
+					       AMDGPU_DM_PIPE_CRC_SOURCE_DPRX));
+
+	/* off → off: not a stop */
+	KUNIT_EXPECT_FALSE(test,
+		dm_crc_source_should_stop_dprx(AMDGPU_DM_PIPE_CRC_SOURCE_NONE,
+					       AMDGPU_DM_PIPE_CRC_SOURCE_NONE));
+}
+
 static struct kunit_case dm_crc_test_cases[] = {
+	/* dm_parse_crc_source() */
 	KUNIT_CASE(dm_test_parse_crc_source_none),
 	KUNIT_CASE(dm_test_parse_crc_source_crtc),
 	KUNIT_CASE(dm_test_parse_crc_source_dprx),
 	KUNIT_CASE(dm_test_parse_crc_source_crtc_dither),
 	KUNIT_CASE(dm_test_parse_crc_source_dprx_dither),
 	KUNIT_CASE(dm_test_parse_crc_source_invalid),
+	/* dm_is_crc_source_crtc() */
 	KUNIT_CASE(dm_test_is_crc_source_crtc),
+	/* dm_is_crc_source_dprx() */
 	KUNIT_CASE(dm_test_is_crc_source_dprx),
+	/* dm_need_crc_dither() */
 	KUNIT_CASE(dm_test_need_crc_dither),
+	/* amdgpu_dm_is_valid_crc_source() */
 	KUNIT_CASE(dm_test_is_valid_crc_source),
+	/* dm_need_dp_aux() */
+	KUNIT_CASE(dm_test_need_dp_aux),
+	/* dm_crc_source_should_start_dprx() */
+	KUNIT_CASE(dm_test_crc_source_should_start_dprx),
+	/* dm_crc_source_should_stop_dprx() */
+	KUNIT_CASE(dm_test_crc_source_should_stop_dprx),
 	{}
 };
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_crtc_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_crtc_test.c
new file mode 100644
index 0000000000000..0edaf969f16b0
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_crtc_test.c
@@ -0,0 +1,523 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * KUnit tests for amdgpu_dm_crtc.c
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#include <kunit/test.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_kunit_helpers.h>
+
+#include "dc.h"
+#include "amdgpu.h"
+#include "amdgpu_mode.h"
+#include "amdgpu_dm.h"
+#include "amdgpu_dm_crtc.h"
+#include "amdgpu_dm_kunit_test_helpers.h"
+#include "amdgpu_dm_irq_params.h"
+
+/* Tests for amdgpu_dm_crtc_modeset_required() */
+
+/**
+ * dm_test_crtc_modeset_required_active_mode_changed - Test Crtc modeset required active mode changed
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_modeset_required_active_mode_changed(struct kunit *test)
+{
+	struct drm_crtc_state state = {};
+
+	state.active = true;
+	state.mode_changed = true;
+
+	KUNIT_EXPECT_TRUE(test,
+			  amdgpu_dm_crtc_modeset_required(&state, NULL, NULL));
+}
+
+/**
+ * dm_test_crtc_modeset_required_active_active_changed - Test Crtc modeset required active active changed
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_modeset_required_active_active_changed(struct kunit *test)
+{
+	struct drm_crtc_state state = {};
+
+	state.active = true;
+	state.active_changed = true;
+
+	KUNIT_EXPECT_TRUE(test,
+			  amdgpu_dm_crtc_modeset_required(&state, NULL, NULL));
+}
+
+/**
+ * dm_test_crtc_modeset_required_active_connectors_changed - Test Crtc modeset required active connectors changed
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_modeset_required_active_connectors_changed(struct kunit *test)
+{
+	struct drm_crtc_state state = {};
+
+	state.active = true;
+	state.connectors_changed = true;
+
+	KUNIT_EXPECT_TRUE(test,
+			  amdgpu_dm_crtc_modeset_required(&state, NULL, NULL));
+}
+
+/**
+ * dm_test_crtc_modeset_required_inactive - Test Crtc modeset required inactive
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_modeset_required_inactive(struct kunit *test)
+{
+	struct drm_crtc_state state = {};
+
+	state.active = false;
+	state.mode_changed = true;
+
+	KUNIT_EXPECT_FALSE(test,
+			   amdgpu_dm_crtc_modeset_required(&state, NULL, NULL));
+}
+
+/**
+ * dm_test_crtc_modeset_required_no_changes - Test Crtc modeset required no changes
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_modeset_required_no_changes(struct kunit *test)
+{
+	struct drm_crtc_state state = {};
+
+	state.active = true;
+	state.mode_changed = false;
+	state.active_changed = false;
+	state.connectors_changed = false;
+
+	KUNIT_EXPECT_FALSE(test,
+			   amdgpu_dm_crtc_modeset_required(&state, NULL, NULL));
+}
+
+/* Tests for amdgpu_dm_crtc_vrr_active_irq() */
+
+/**
+ * dm_test_crtc_vrr_active_irq_variable - Test Crtc vrr active irq variable
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_vrr_active_irq_variable(struct kunit *test)
+{
+	struct amdgpu_crtc *acrtc = kunit_kzalloc(test, sizeof(*acrtc),
+						  GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	acrtc->dm_irq_params.freesync_config.state = VRR_STATE_ACTIVE_VARIABLE;
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_crtc_vrr_active_irq(acrtc));
+}
+
+/**
+ * dm_test_crtc_vrr_active_irq_fixed - Test Crtc vrr active irq fixed
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_vrr_active_irq_fixed(struct kunit *test)
+{
+	struct amdgpu_crtc *acrtc = kunit_kzalloc(test, sizeof(*acrtc),
+						  GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	acrtc->dm_irq_params.freesync_config.state = VRR_STATE_ACTIVE_FIXED;
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_crtc_vrr_active_irq(acrtc));
+}
+
+/**
+ * dm_test_crtc_vrr_active_irq_inactive - Test Crtc vrr active irq inactive
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_vrr_active_irq_inactive(struct kunit *test)
+{
+	struct amdgpu_crtc *acrtc = kunit_kzalloc(test, sizeof(*acrtc),
+						  GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	acrtc->dm_irq_params.freesync_config.state = VRR_STATE_INACTIVE;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_crtc_vrr_active_irq(acrtc));
+}
+
+/**
+ * dm_test_crtc_vrr_active_irq_disabled - Test Crtc vrr active irq disabled
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_vrr_active_irq_disabled(struct kunit *test)
+{
+	struct amdgpu_crtc *acrtc = kunit_kzalloc(test, sizeof(*acrtc),
+						  GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	acrtc->dm_irq_params.freesync_config.state = VRR_STATE_DISABLED;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_crtc_vrr_active_irq(acrtc));
+}
+
+/**
+ * dm_test_crtc_vrr_active_irq_unsupported - Test Crtc vrr active irq unsupported
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_vrr_active_irq_unsupported(struct kunit *test)
+{
+	struct amdgpu_crtc *acrtc = kunit_kzalloc(test, sizeof(*acrtc),
+						  GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	acrtc->dm_irq_params.freesync_config.state = VRR_STATE_UNSUPPORTED;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_crtc_vrr_active_irq(acrtc));
+}
+
+/* Tests for amdgpu_dm_crtc_vrr_active() */
+
+/**
+ * dm_test_crtc_vrr_active_variable - Test Crtc vrr active variable
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_vrr_active_variable(struct kunit *test)
+{
+	struct dm_crtc_state *dm_state = kunit_kzalloc(test,
+						       sizeof(*dm_state),
+						       GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm_state);
+
+	dm_state->freesync_config.state = VRR_STATE_ACTIVE_VARIABLE;
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_crtc_vrr_active(dm_state));
+}
+
+/**
+ * dm_test_crtc_vrr_active_fixed - Test Crtc vrr active fixed
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_vrr_active_fixed(struct kunit *test)
+{
+	struct dm_crtc_state *dm_state = kunit_kzalloc(test,
+						       sizeof(*dm_state),
+						       GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm_state);
+
+	dm_state->freesync_config.state = VRR_STATE_ACTIVE_FIXED;
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_crtc_vrr_active(dm_state));
+}
+
+/**
+ * dm_test_crtc_vrr_active_inactive - Test Crtc vrr active inactive
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_vrr_active_inactive(struct kunit *test)
+{
+	struct dm_crtc_state *dm_state = kunit_kzalloc(test,
+						       sizeof(*dm_state),
+						       GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm_state);
+
+	dm_state->freesync_config.state = VRR_STATE_INACTIVE;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_crtc_vrr_active(dm_state));
+}
+
+/**
+ * dm_test_crtc_vrr_active_disabled - Test Crtc vrr active disabled
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_vrr_active_disabled(struct kunit *test)
+{
+	struct dm_crtc_state *dm_state = kunit_kzalloc(test,
+						       sizeof(*dm_state),
+						       GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm_state);
+
+	dm_state->freesync_config.state = VRR_STATE_DISABLED;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_crtc_vrr_active(dm_state));
+}
+
+/**
+ * dm_test_crtc_vrr_active_unsupported - Test Crtc vrr active unsupported
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_vrr_active_unsupported(struct kunit *test)
+{
+	struct dm_crtc_state *dm_state = kunit_kzalloc(test,
+						       sizeof(*dm_state),
+						       GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm_state);
+
+	dm_state->freesync_config.state = VRR_STATE_UNSUPPORTED;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_crtc_vrr_active(dm_state));
+}
+
+/* Tests for amdgpu_dm_is_headless() */
+
+static void dm_test_add_connector(struct drm_device *dev,
+				  struct drm_connector *connector,
+				  int connector_type,
+				  enum drm_connector_status status)
+{
+	INIT_LIST_HEAD(&connector->head);
+	kref_init(&connector->base.refcount);
+	connector->connector_type = connector_type;
+	connector->status = status;
+	list_add_tail(&connector->head, &dev->mode_config.connector_list);
+}
+
+/**
+ * dm_test_crtc_is_headless_null_adev - Test Crtc is headless null adev
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_is_headless_null_adev(struct kunit *test)
+{
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_is_headless(NULL));
+}
+
+/**
+ * dm_test_crtc_is_headless_no_connectors - Test Crtc is headless no connectors
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_is_headless_no_connectors(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct drm_device *dev = kunit_kzalloc(test, sizeof(*dev), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	INIT_LIST_HEAD(&dev->mode_config.connector_list);
+	spin_lock_init(&dev->mode_config.connector_list_lock);
+	adev->dm.ddev = dev;
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_is_headless(adev));
+}
+
+/**
+ * dm_test_crtc_is_headless_writeback_only - Test Crtc is headless writeback only
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_is_headless_writeback_only(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct drm_device *dev = kunit_kzalloc(test, sizeof(*dev), GFP_KERNEL);
+	struct drm_connector *wb = kunit_kzalloc(test, sizeof(*wb), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, wb);
+
+	INIT_LIST_HEAD(&dev->mode_config.connector_list);
+	spin_lock_init(&dev->mode_config.connector_list_lock);
+	adev->dm.ddev = dev;
+
+	dm_test_add_connector(dev, wb, DRM_MODE_CONNECTOR_WRITEBACK,
+			      connector_status_connected);
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_is_headless(adev));
+}
+
+/**
+ * dm_test_crtc_is_headless_disconnected_display - Test Crtc is headless disconnected display
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_is_headless_disconnected_display(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct drm_device *dev = kunit_kzalloc(test, sizeof(*dev), GFP_KERNEL);
+	struct drm_connector *display = kunit_kzalloc(test, sizeof(*display), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, display);
+
+	INIT_LIST_HEAD(&dev->mode_config.connector_list);
+	spin_lock_init(&dev->mode_config.connector_list_lock);
+	adev->dm.ddev = dev;
+
+	dm_test_add_connector(dev, display, DRM_MODE_CONNECTOR_HDMIA,
+			      connector_status_disconnected);
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_is_headless(adev));
+}
+
+/**
+ * dm_test_crtc_is_headless_connected_display - Test Crtc is headless connected display
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_is_headless_connected_display(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct drm_device *dev = kunit_kzalloc(test, sizeof(*dev), GFP_KERNEL);
+	struct drm_connector *display = kunit_kzalloc(test, sizeof(*display), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, display);
+
+	INIT_LIST_HEAD(&dev->mode_config.connector_list);
+	spin_lock_init(&dev->mode_config.connector_list_lock);
+	adev->dm.ddev = dev;
+
+	dm_test_add_connector(dev, display, DRM_MODE_CONNECTOR_HDMIA,
+			      connector_status_connected);
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_is_headless(adev));
+}
+
+/**
+ * dm_test_crtc_is_headless_mixed_connectors - Test headless skips WB and finds display
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_is_headless_mixed_connectors(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct drm_device *dev = kunit_kzalloc(test, sizeof(*dev), GFP_KERNEL);
+	struct drm_connector *wb = kunit_kzalloc(test, sizeof(*wb), GFP_KERNEL);
+	struct drm_connector *display = kunit_kzalloc(test, sizeof(*display), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, wb);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, display);
+
+	INIT_LIST_HEAD(&dev->mode_config.connector_list);
+	spin_lock_init(&dev->mode_config.connector_list_lock);
+	adev->dm.ddev = dev;
+
+	dm_test_add_connector(dev, wb, DRM_MODE_CONNECTOR_WRITEBACK,
+			      connector_status_connected);
+	dm_test_add_connector(dev, display, DRM_MODE_CONNECTOR_DisplayPort,
+			      connector_status_connected);
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_is_headless(adev));
+}
+
+/* Tests for amdgpu_dm_crtc_helper_mode_fixup() */
+
+/**
+ * dm_test_crtc_helper_mode_fixup_returns_true - Test mode_fixup accepts mode
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_helper_mode_fixup_returns_true(struct kunit *test)
+{
+	struct drm_display_mode mode = { 0 };
+	struct drm_display_mode adjusted_mode = { 0 };
+
+	KUNIT_EXPECT_TRUE(test,
+			  amdgpu_dm_crtc_helper_mode_fixup(NULL, &mode, &adjusted_mode));
+}
+
+/* Tests for amdgpu_dm_crtc_set_vupdate_irq() */
+
+/**
+ * dm_test_crtc_set_vupdate_irq_no_otg - Test vupdate irq with unassigned OTG
+ * @test: The KUnit test context
+ *
+ * When the CRTC has no OTG instance assigned (otg_inst == -1) the function
+ * must return 0 immediately without touching the DC interrupt state.
+ */
+static void dm_test_crtc_set_vupdate_irq_no_otg(struct kunit *test)
+{
+	struct amdgpu_crtc *acrtc;
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	acrtc->base.dev = &adev->ddev;
+	acrtc->otg_inst = -1;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_crtc_set_vupdate_irq(&acrtc->base, true), 0);
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_crtc_set_vupdate_irq(&acrtc->base, false), 0);
+}
+
+/* Tests for idle_create_workqueue() */
+
+/**
+ * dm_test_idle_create_workqueue - Test idle workqueue creation
+ * @test: The KUnit test context
+ *
+ * Verify that idle_create_workqueue() allocates an idle workqueue tied to the
+ * device's display manager and initializes it in a disabled, non-running state.
+ */
+static void dm_test_idle_create_workqueue(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct idle_workqueue *idle_work;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	idle_work = idle_create_workqueue(adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, idle_work);
+
+	KUNIT_EXPECT_PTR_EQ(test, idle_work->dm, &adev->dm);
+	KUNIT_EXPECT_FALSE(test, idle_work->enable);
+	KUNIT_EXPECT_FALSE(test, idle_work->running);
+
+	kfree(idle_work);
+}
+
+static struct kunit_case amdgpu_dm_crtc_tests[] = {
+	/* amdgpu_dm_crtc_modeset_required */
+	KUNIT_CASE(dm_test_crtc_modeset_required_active_mode_changed),
+	KUNIT_CASE(dm_test_crtc_modeset_required_active_active_changed),
+	KUNIT_CASE(dm_test_crtc_modeset_required_active_connectors_changed),
+	KUNIT_CASE(dm_test_crtc_modeset_required_inactive),
+	KUNIT_CASE(dm_test_crtc_modeset_required_no_changes),
+	/* amdgpu_dm_crtc_vrr_active_irq */
+	KUNIT_CASE(dm_test_crtc_vrr_active_irq_variable),
+	KUNIT_CASE(dm_test_crtc_vrr_active_irq_fixed),
+	KUNIT_CASE(dm_test_crtc_vrr_active_irq_inactive),
+	KUNIT_CASE(dm_test_crtc_vrr_active_irq_disabled),
+	KUNIT_CASE(dm_test_crtc_vrr_active_irq_unsupported),
+	/* amdgpu_dm_crtc_vrr_active */
+	KUNIT_CASE(dm_test_crtc_vrr_active_variable),
+	KUNIT_CASE(dm_test_crtc_vrr_active_fixed),
+	KUNIT_CASE(dm_test_crtc_vrr_active_inactive),
+	KUNIT_CASE(dm_test_crtc_vrr_active_disabled),
+	KUNIT_CASE(dm_test_crtc_vrr_active_unsupported),
+	/* amdgpu_dm_is_headless */
+	KUNIT_CASE(dm_test_crtc_is_headless_null_adev),
+	KUNIT_CASE(dm_test_crtc_is_headless_no_connectors),
+	KUNIT_CASE(dm_test_crtc_is_headless_writeback_only),
+	KUNIT_CASE(dm_test_crtc_is_headless_disconnected_display),
+	KUNIT_CASE(dm_test_crtc_is_headless_connected_display),
+	KUNIT_CASE(dm_test_crtc_is_headless_mixed_connectors),
+	/* amdgpu_dm_crtc_helper_mode_fixup */
+	KUNIT_CASE(dm_test_crtc_helper_mode_fixup_returns_true),
+	/* amdgpu_dm_crtc_set_vupdate_irq */
+	KUNIT_CASE(dm_test_crtc_set_vupdate_irq_no_otg),
+	/* idle_create_workqueue */
+	KUNIT_CASE(dm_test_idle_create_workqueue),
+	{}
+};
+
+static struct kunit_suite amdgpu_dm_crtc_test_suite = {
+	.name = "amdgpu_dm_crtc",
+	.test_cases = amdgpu_dm_crtc_tests,
+};
+
+kunit_test_suite(amdgpu_dm_crtc_test_suite);
+
+MODULE_AUTHOR("AMD");
+MODULE_DESCRIPTION("KUnit tests for amdgpu_dm_crtc");
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_dmub_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_dmub_test.c
new file mode 100644
index 0000000000000..bf90ccfbf4316
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_dmub_test.c
@@ -0,0 +1,582 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * KUnit tests for amdgpu_dm_dmub.c
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#include <kunit/test.h>
+
+#include "dc.h"
+#include "dc/inc/core_types.h"
+#include "amdgpu_mode.h"
+#include "amdgpu_dm.h"
+#include "dmub/dmub_srv.h"
+#include "amdgpu_dm_dmub.h"
+
+/* Tests for dm_register_dmub_notify_callback() */
+
+static void dummy_callback(struct amdgpu_device *adev,
+			   struct dmub_notification *notify)
+{
+}
+
+/**
+ * dm_test_register_dmub_notify_callback_null_callback - Test null callback is rejected
+ * @test: The KUnit test context
+ */
+static void dm_test_register_dmub_notify_callback_null_callback(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	KUNIT_EXPECT_FALSE(test, dm_register_dmub_notify_callback(adev,
+		DMUB_NOTIFICATION_AUX_REPLY, NULL, false));
+}
+
+/**
+ * dm_test_register_dmub_notify_callback_type_out_of_range - Test out-of-range type is rejected
+ * @test: The KUnit test context
+ */
+static void dm_test_register_dmub_notify_callback_type_out_of_range(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	KUNIT_EXPECT_FALSE(test, dm_register_dmub_notify_callback(adev,
+		AMDGPU_DMUB_NOTIFICATION_MAX, dummy_callback, false));
+}
+
+/**
+ * dm_test_register_dmub_notify_callback_valid - Test Register dmub notify callback valid
+ * @test: The KUnit test context
+ */
+static void dm_test_register_dmub_notify_callback_valid(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	KUNIT_EXPECT_TRUE(test, dm_register_dmub_notify_callback(adev,
+		DMUB_NOTIFICATION_AUX_REPLY, dummy_callback, true));
+
+	KUNIT_EXPECT_TRUE(test,
+		adev->dm.dmub_callback[DMUB_NOTIFICATION_AUX_REPLY] == dummy_callback);
+	KUNIT_EXPECT_TRUE(test,
+		adev->dm.dmub_thread_offload[DMUB_NOTIFICATION_AUX_REPLY]);
+}
+
+/**
+ * dm_test_register_dmub_notify_callback_offload_false - Test registration with offload disabled
+ * @test: The KUnit test context
+ */
+static void dm_test_register_dmub_notify_callback_offload_false(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	KUNIT_EXPECT_TRUE(test, dm_register_dmub_notify_callback(adev,
+		DMUB_NOTIFICATION_HPD, dummy_callback, false));
+
+	KUNIT_EXPECT_TRUE(test,
+		adev->dm.dmub_callback[DMUB_NOTIFICATION_HPD] == dummy_callback);
+	KUNIT_EXPECT_FALSE(test,
+		adev->dm.dmub_thread_offload[DMUB_NOTIFICATION_HPD]);
+}
+
+/* Tests for dm_dmub_aux_setconfig_callback() */
+
+/**
+ * dm_test_dmub_aux_setconfig_callback_copies_and_completes - Test copy and complete on AUX reply
+ * @test: The KUnit test context
+ */
+static void dm_test_dmub_aux_setconfig_callback_copies_and_completes(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dmub_notification *dm_notify;
+	struct dmub_notification notify = {};
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	dm_notify = kunit_kzalloc(test, sizeof(*dm_notify), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm_notify);
+
+	init_completion(&adev->dm.dmub_aux_transfer_done);
+	adev->dm.dmub_notify = dm_notify;
+
+	notify.type = DMUB_NOTIFICATION_AUX_REPLY;
+	notify.result = AUX_RET_SUCCESS;
+	notify.aux_reply.command = 0xA5;
+	notify.aux_reply.length = 3;
+	notify.aux_reply.data[0] = 0x11;
+	notify.aux_reply.data[1] = 0x22;
+	notify.aux_reply.data[2] = 0x33;
+
+	dm_dmub_aux_setconfig_callback(adev, &notify);
+
+	KUNIT_EXPECT_EQ(test, dm_notify->type, notify.type);
+	KUNIT_EXPECT_EQ(test, dm_notify->result, notify.result);
+	KUNIT_EXPECT_EQ(test, dm_notify->aux_reply.command, notify.aux_reply.command);
+	KUNIT_EXPECT_EQ(test, dm_notify->aux_reply.length, notify.aux_reply.length);
+	KUNIT_EXPECT_EQ(test, dm_notify->aux_reply.data[0], notify.aux_reply.data[0]);
+	KUNIT_EXPECT_EQ(test, dm_notify->aux_reply.data[1], notify.aux_reply.data[1]);
+	KUNIT_EXPECT_EQ(test, dm_notify->aux_reply.data[2], notify.aux_reply.data[2]);
+	KUNIT_EXPECT_TRUE(test, completion_done(&adev->dm.dmub_aux_transfer_done));
+}
+
+/**
+ * dm_test_dmub_aux_setconfig_callback_non_aux_no_complete - Test non-AUX type skips completion
+ * @test: The KUnit test context
+ */
+static void dm_test_dmub_aux_setconfig_callback_non_aux_no_complete(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dmub_notification *dm_notify;
+	struct dmub_notification notify = {};
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	dm_notify = kunit_kzalloc(test, sizeof(*dm_notify), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm_notify);
+
+	init_completion(&adev->dm.dmub_aux_transfer_done);
+	adev->dm.dmub_notify = dm_notify;
+
+	notify.type = DMUB_NOTIFICATION_HPD;
+	notify.result = AUX_RET_ERROR_TIMEOUT;
+
+	dm_dmub_aux_setconfig_callback(adev, &notify);
+
+	KUNIT_EXPECT_EQ(test, dm_notify->type, notify.type);
+	KUNIT_EXPECT_FALSE(test, completion_done(&adev->dm.dmub_aux_transfer_done));
+}
+
+/**
+ * dm_test_dmub_aux_setconfig_callback_aux_with_null_dm_notify - Test AUX with NULL dm_notify
+ * @test: The KUnit test context
+ */
+static void dm_test_dmub_aux_setconfig_callback_aux_with_null_dm_notify(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dmub_notification notify = {};
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	init_completion(&adev->dm.dmub_aux_transfer_done);
+	adev->dm.dmub_notify = NULL;
+
+	notify.type = DMUB_NOTIFICATION_AUX_REPLY;
+
+	dm_dmub_aux_setconfig_callback(adev, &notify);
+
+	KUNIT_EXPECT_TRUE(test, completion_done(&adev->dm.dmub_aux_transfer_done));
+}
+
+/**
+ * dm_test_dmub_aux_setconfig_callback_set_config_reply - Test SET_CONFIG reply copies status
+ * @test: The KUnit test context
+ */
+static void dm_test_dmub_aux_setconfig_callback_set_config_reply(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dmub_notification *dm_notify;
+	struct dmub_notification notify = {};
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	dm_notify = kunit_kzalloc(test, sizeof(*dm_notify), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm_notify);
+
+	init_completion(&adev->dm.dmub_aux_transfer_done);
+	adev->dm.dmub_notify = dm_notify;
+
+	notify.type = DMUB_NOTIFICATION_SET_CONFIG_REPLY;
+	notify.sc_status = SET_CONFIG_RX_TIMEOUT;
+
+	dm_dmub_aux_setconfig_callback(adev, &notify);
+
+	KUNIT_EXPECT_EQ(test, dm_notify->type, notify.type);
+	KUNIT_EXPECT_EQ(test, dm_notify->sc_status, notify.sc_status);
+	KUNIT_EXPECT_FALSE(test, completion_done(&adev->dm.dmub_aux_transfer_done));
+}
+
+/* Tests for dm_dmub_aux_fused_io_callback() */
+
+/**
+ * dm_test_dmub_aux_fused_io_callback_copies_reply_and_completes - Test copy and complete
+ * @test: The KUnit test context
+ */
+static void dm_test_dmub_aux_fused_io_callback_copies_reply_and_completes(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dmub_notification notify = {};
+	struct dmub_cmd_fused_request *reply;
+	u32 reply_ddc_line;
+	u32 notify_ddc_line;
+	u32 reply_address;
+	u32 notify_address;
+	u32 reply_length;
+	u32 notify_length;
+	uint8_t ddc_line = 2;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	init_completion(&adev->dm.fused_io[ddc_line].replied);
+
+	notify.fused_request.identifier = 0x34;
+	notify.fused_request.status = FUSED_REQUEST_STATUS_SUCCESS;
+	notify.fused_request.u.aux.ddc_line = ddc_line;
+	notify.fused_request.u.aux.address = 0x50;
+	notify.fused_request.u.aux.length = 4;
+
+	dm_dmub_aux_fused_io_callback(adev, &notify);
+
+	KUNIT_EXPECT_TRUE(test, completion_done(&adev->dm.fused_io[ddc_line].replied));
+
+	reply = (struct dmub_cmd_fused_request *)adev->dm.fused_io[ddc_line].reply_data;
+	reply_ddc_line = reply->u.aux.ddc_line;
+	notify_ddc_line = notify.fused_request.u.aux.ddc_line;
+	reply_address = reply->u.aux.address;
+	notify_address = notify.fused_request.u.aux.address;
+	reply_length = reply->u.aux.length;
+	notify_length = notify.fused_request.u.aux.length;
+
+	KUNIT_EXPECT_EQ(test, reply->identifier, notify.fused_request.identifier);
+	KUNIT_EXPECT_EQ(test, reply->status, notify.fused_request.status);
+	KUNIT_EXPECT_EQ(test, reply_ddc_line, notify_ddc_line);
+	KUNIT_EXPECT_EQ(test, reply_address, notify_address);
+	KUNIT_EXPECT_EQ(test, reply_length, notify_length);
+}
+
+/**
+ * dm_test_dmub_aux_fused_io_callback_max_ddc_line - Test Dmub aux fused io callback max ddc line
+ * @test: The KUnit test context
+ */
+static void dm_test_dmub_aux_fused_io_callback_max_ddc_line(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dmub_notification notify = {};
+	struct dmub_cmd_fused_request *reply;
+	u32 reply_ddc_line;
+	u32 notify_ddc_line;
+	uint8_t ddc_line;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	ddc_line = ARRAY_SIZE(adev->dm.fused_io) - 1;
+	init_completion(&adev->dm.fused_io[ddc_line].replied);
+
+	notify.fused_request.identifier = 0x56;
+	notify.fused_request.status = FUSED_REQUEST_STATUS_SUCCESS;
+	notify.fused_request.u.aux.ddc_line = ddc_line;
+	notify.fused_request.u.aux.address = 0x50;
+	notify.fused_request.u.aux.length = 1;
+
+	dm_dmub_aux_fused_io_callback(adev, &notify);
+
+	KUNIT_EXPECT_TRUE(test, completion_done(&adev->dm.fused_io[ddc_line].replied));
+
+	reply = (struct dmub_cmd_fused_request *)adev->dm.fused_io[ddc_line].reply_data;
+	reply_ddc_line = reply->u.aux.ddc_line;
+	notify_ddc_line = notify.fused_request.u.aux.ddc_line;
+
+	KUNIT_EXPECT_EQ(test, reply->identifier, notify.fused_request.identifier);
+	KUNIT_EXPECT_EQ(test, reply_ddc_line, notify_ddc_line);
+}
+
+/* Tests for dm_get_default_ips_mode() */
+
+/**
+ * dm_test_get_default_ips_mode_dcn35 - Test Get default ips mode dcn35
+ * @test: The KUnit test context
+ */
+static void dm_test_get_default_ips_mode_dcn35(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 5, 0);
+
+	KUNIT_EXPECT_EQ(test, dm_get_default_ips_mode(adev),
+			DMUB_IPS_RCG_IN_ACTIVE_IPS2_IN_OFF);
+}
+
+/**
+ * dm_test_get_default_ips_mode_dcn351 - Test Get default ips mode dcn351
+ * @test: The KUnit test context
+ */
+static void dm_test_get_default_ips_mode_dcn351(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 5, 1);
+
+	KUNIT_EXPECT_EQ(test, dm_get_default_ips_mode(adev),
+			DMUB_IPS_RCG_IN_ACTIVE_IPS2_IN_OFF);
+}
+
+/**
+ * dm_test_get_default_ips_mode_dcn36 - Test Get default ips mode dcn36
+ * @test: The KUnit test context
+ */
+static void dm_test_get_default_ips_mode_dcn36(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 6, 0);
+
+	KUNIT_EXPECT_EQ(test, dm_get_default_ips_mode(adev),
+			DMUB_IPS_RCG_IN_ACTIVE_IPS2_IN_OFF);
+}
+
+/**
+ * dm_test_get_default_ips_mode_older_than_dcn35 - Test Get default ips mode older than dcn35
+ * @test: The KUnit test context
+ */
+static void dm_test_get_default_ips_mode_older_than_dcn35(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 2, 0);
+
+	KUNIT_EXPECT_EQ(test, dm_get_default_ips_mode(adev),
+			DMUB_IPS_DISABLE_ALL);
+}
+
+/**
+ * dm_test_get_default_ips_mode_newer_default - Test Get default ips mode newer default
+ * @test: The KUnit test context
+ */
+static void dm_test_get_default_ips_mode_newer_default(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	/* DCN 4.0.1 is >= 3.5 but has no explicit case, returns ENABLE */
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(4, 0, 1);
+
+	KUNIT_EXPECT_EQ(test, dm_get_default_ips_mode(adev),
+			DMUB_IPS_ENABLE);
+}
+
+/* Tests for dm_dmub_hw_init() */
+
+/*
+ * Build an amdgpu_device with the minimal dc/res_pool pointers that
+ * dm_dmub_hw_init() and dm_dmub_hw_resume() dereference before their
+ * early-return checks.
+ */
+static struct amdgpu_device *dm_test_alloc_adev_with_dc(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct resource_pool *res_pool;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+
+	res_pool = kunit_kzalloc(test, sizeof(*res_pool), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, res_pool);
+
+	dc->res_pool = res_pool;
+	adev->dm.dc = dc;
+
+	return adev;
+}
+
+/**
+ * dm_test_dmub_hw_init_no_dmub_srv - Test hw init returns 0 when DMUB unsupported
+ * @test: The KUnit test context
+ *
+ * When adev->dm.dmub_srv is NULL the ASIC does not support DMUB and
+ * dm_dmub_hw_init() should return 0 without touching the hardware.
+ */
+static void dm_test_dmub_hw_init_no_dmub_srv(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev_with_dc(test);
+
+	adev->dm.dmub_srv = NULL;
+
+	KUNIT_EXPECT_EQ(test, dm_dmub_hw_init(adev), 0);
+}
+
+/**
+ * dm_test_dmub_hw_init_no_fb_info - Test hw init fails without framebuffer info
+ * @test: The KUnit test context
+ *
+ * With a DMUB service present but no framebuffer info, dm_dmub_hw_init()
+ * should return -EINVAL.
+ */
+static void dm_test_dmub_hw_init_no_fb_info(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev_with_dc(test);
+	struct dmub_srv *dmub_srv;
+
+	dmub_srv = kunit_kzalloc(test, sizeof(*dmub_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dmub_srv);
+
+	adev->dm.dmub_srv = dmub_srv;
+	adev->dm.dmub_fb_info = NULL;
+
+	KUNIT_EXPECT_EQ(test, dm_dmub_hw_init(adev), -EINVAL);
+}
+
+/**
+ * dm_test_dmub_hw_init_no_firmware - Test hw init fails without firmware
+ * @test: The KUnit test context
+ *
+ * With a DMUB service and framebuffer info present but no firmware,
+ * dm_dmub_hw_init() should return -EINVAL.
+ */
+static void dm_test_dmub_hw_init_no_firmware(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev_with_dc(test);
+	struct dmub_srv *dmub_srv;
+	struct dmub_srv_fb_info *fb_info;
+
+	dmub_srv = kunit_kzalloc(test, sizeof(*dmub_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dmub_srv);
+
+	fb_info = kunit_kzalloc(test, sizeof(*fb_info), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fb_info);
+
+	adev->dm.dmub_srv = dmub_srv;
+	adev->dm.dmub_fb_info = fb_info;
+	adev->dm.dmub_fw = NULL;
+
+	KUNIT_EXPECT_EQ(test, dm_dmub_hw_init(adev), -EINVAL);
+}
+
+/* Tests for dm_dmub_hw_resume() */
+
+/**
+ * dm_test_dmub_hw_resume_no_dmub_srv - Test hw resume is a no-op when DMUB unsupported
+ * @test: The KUnit test context
+ *
+ * When adev->dm.dmub_srv is NULL, dm_dmub_hw_resume() should return early
+ * without dereferencing the (absent) DMUB service.
+ */
+static void dm_test_dmub_hw_resume_no_dmub_srv(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev_with_dc(test);
+
+	adev->dm.dmub_srv = NULL;
+
+	/* Must not crash. */
+	dm_dmub_hw_resume(adev);
+}
+
+/* Tests for dm_dmub_sw_init() */
+
+/**
+ * dm_test_dmub_sw_init_unsupported_asic - Test sw init returns 0 for unsupported ASIC
+ * @test: The KUnit test context
+ *
+ * For an IP version with no DMUB support, dm_dmub_sw_init() should return 0
+ * before attempting to access the firmware.
+ */
+static void dm_test_dmub_sw_init_unsupported_asic(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(1, 0, 0);
+
+	KUNIT_EXPECT_EQ(test, dm_dmub_sw_init(adev), 0);
+}
+
+/* Tests for dm_init_microcode() */
+
+/**
+ * dm_test_init_microcode_unsupported_asic - Test microcode init returns 0 for unsupported ASIC
+ * @test: The KUnit test context
+ *
+ * For an IP version with no DMUB support, dm_init_microcode() should return 0
+ * without requesting any firmware.
+ */
+static void dm_test_init_microcode_unsupported_asic(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(1, 0, 0);
+
+	KUNIT_EXPECT_EQ(test, dm_init_microcode(adev), 0);
+}
+
+static struct kunit_case amdgpu_dm_dmub_tests[] = {
+	/* dm_register_dmub_notify_callback() */
+	KUNIT_CASE(dm_test_register_dmub_notify_callback_null_callback),
+	KUNIT_CASE(dm_test_register_dmub_notify_callback_type_out_of_range),
+	KUNIT_CASE(dm_test_register_dmub_notify_callback_valid),
+	KUNIT_CASE(dm_test_register_dmub_notify_callback_offload_false),
+	/* dm_dmub_aux_setconfig_callback() */
+	KUNIT_CASE(dm_test_dmub_aux_setconfig_callback_copies_and_completes),
+	KUNIT_CASE(dm_test_dmub_aux_setconfig_callback_non_aux_no_complete),
+	KUNIT_CASE(dm_test_dmub_aux_setconfig_callback_aux_with_null_dm_notify),
+	KUNIT_CASE(dm_test_dmub_aux_setconfig_callback_set_config_reply),
+	/* dm_dmub_aux_fused_io_callback() */
+	KUNIT_CASE(dm_test_dmub_aux_fused_io_callback_copies_reply_and_completes),
+	KUNIT_CASE(dm_test_dmub_aux_fused_io_callback_max_ddc_line),
+	/* dm_get_default_ips_mode() */
+	KUNIT_CASE(dm_test_get_default_ips_mode_dcn35),
+	KUNIT_CASE(dm_test_get_default_ips_mode_dcn351),
+	KUNIT_CASE(dm_test_get_default_ips_mode_dcn36),
+	KUNIT_CASE(dm_test_get_default_ips_mode_older_than_dcn35),
+	KUNIT_CASE(dm_test_get_default_ips_mode_newer_default),
+	/* dm_dmub_hw_init() */
+	KUNIT_CASE(dm_test_dmub_hw_init_no_dmub_srv),
+	KUNIT_CASE(dm_test_dmub_hw_init_no_fb_info),
+	KUNIT_CASE(dm_test_dmub_hw_init_no_firmware),
+	/* dm_dmub_hw_resume() */
+	KUNIT_CASE(dm_test_dmub_hw_resume_no_dmub_srv),
+	/* dm_dmub_sw_init() */
+	KUNIT_CASE(dm_test_dmub_sw_init_unsupported_asic),
+	/* dm_init_microcode() */
+	KUNIT_CASE(dm_test_init_microcode_unsupported_asic),
+	{}
+};
+
+static struct kunit_suite amdgpu_dm_dmub_test_suite = {
+	.name = "amdgpu_dm_dmub",
+	.test_cases = amdgpu_dm_dmub_tests,
+};
+
+kunit_test_suite(amdgpu_dm_dmub_test_suite);
+
+MODULE_AUTHOR("AMD");
+MODULE_DESCRIPTION("KUnit tests for amdgpu_dm_dmub");
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_hdcp_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_hdcp_test.c
index d03b606d27bc7..dbcff92672e69 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_hdcp_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_hdcp_test.c
@@ -12,11 +12,241 @@
 
 static void dummy_work_fn(struct work_struct *work) {}
 
+/* Tests for hdcp_get_content_protection_from_status() */
+
+/**
+ * dm_test_hdcp_get_cp_disabled_returns_desired - HDCP off maps to DESIRED
+ * @test: KUnit test context
+ *
+ * When encryption status is HDCP_OFF, content_protection should be set
+ * to DESIRED and the function should return true to indicate an update.
+ */
+static void dm_test_hdcp_get_cp_disabled_returns_desired(struct kunit *test)
+{
+	unsigned int content_protection = DRM_MODE_CONTENT_PROTECTION_UNDESIRED;
+	bool update;
+
+	update = hdcp_get_content_protection_from_status(
+		DRM_MODE_HDCP_CONTENT_TYPE0,
+		MOD_HDCP_ENCRYPTION_STATUS_HDCP_OFF,
+		&content_protection);
+
+	KUNIT_EXPECT_TRUE(test, update);
+	KUNIT_EXPECT_EQ(test, content_protection,
+			DRM_MODE_CONTENT_PROTECTION_DESIRED);
+}
+
+/**
+ * dm_test_hdcp_get_cp_type0_returns_enabled - TYPE0 with TYPE0_ON maps to ENABLED
+ * @test: KUnit test context
+ *
+ * When content type is TYPE0 and encryption status is at or below
+ * HDCP2_TYPE0_ON, content_protection should be set to ENABLED.
+ */
+static void dm_test_hdcp_get_cp_type0_returns_enabled(struct kunit *test)
+{
+	unsigned int content_protection = DRM_MODE_CONTENT_PROTECTION_UNDESIRED;
+	bool update;
+
+	update = hdcp_get_content_protection_from_status(
+		DRM_MODE_HDCP_CONTENT_TYPE0,
+		MOD_HDCP_ENCRYPTION_STATUS_HDCP2_TYPE0_ON,
+		&content_protection);
+
+	KUNIT_EXPECT_TRUE(test, update);
+	KUNIT_EXPECT_EQ(test, content_protection,
+			DRM_MODE_CONTENT_PROTECTION_ENABLED);
+}
+
+/**
+ * dm_test_hdcp_get_cp_type1_returns_enabled - TYPE1 with TYPE1_ON maps to ENABLED
+ * @test: KUnit test context
+ *
+ * When content type is TYPE1 and encryption status is exactly
+ * HDCP2_TYPE1_ON, content_protection should be set to ENABLED.
+ */
+static void dm_test_hdcp_get_cp_type1_returns_enabled(struct kunit *test)
+{
+	unsigned int content_protection = DRM_MODE_CONTENT_PROTECTION_UNDESIRED;
+	bool update;
+
+	update = hdcp_get_content_protection_from_status(
+		DRM_MODE_HDCP_CONTENT_TYPE1,
+		MOD_HDCP_ENCRYPTION_STATUS_HDCP2_TYPE1_ON,
+		&content_protection);
+
+	KUNIT_EXPECT_TRUE(test, update);
+	KUNIT_EXPECT_EQ(test, content_protection,
+			DRM_MODE_CONTENT_PROTECTION_ENABLED);
+}
+
+/**
+ * dm_test_hdcp_get_cp_type1_rejects_type0_status - TYPE1 rejects TYPE0_ON
+ * @test: KUnit test context
+ *
+ * When content type is TYPE1 but encryption status is only TYPE0_ON,
+ * the function should return false and leave content_protection unchanged.
+ */
+static void dm_test_hdcp_get_cp_type1_rejects_type0_status(struct kunit *test)
+{
+	unsigned int content_protection = DRM_MODE_CONTENT_PROTECTION_UNDESIRED;
+	bool update;
+
+	update = hdcp_get_content_protection_from_status(
+		DRM_MODE_HDCP_CONTENT_TYPE1,
+		MOD_HDCP_ENCRYPTION_STATUS_HDCP2_TYPE0_ON,
+		&content_protection);
+
+	KUNIT_EXPECT_FALSE(test, update);
+	KUNIT_EXPECT_EQ(test, content_protection,
+			DRM_MODE_CONTENT_PROTECTION_UNDESIRED);
+}
+
+/**
+ * dm_test_hdcp_get_cp_type0_rejects_type1_status - TYPE0 rejects TYPE1_ON
+ * @test: KUnit test context
+ *
+ * When content type is TYPE0 but encryption status exceeds the TYPE0_ON
+ * boundary (TYPE1_ON), the function should return false.
+ */
+static void dm_test_hdcp_get_cp_type0_rejects_type1_status(struct kunit *test)
+{
+	unsigned int content_protection = DRM_MODE_CONTENT_PROTECTION_UNDESIRED;
+	bool update;
+
+	update = hdcp_get_content_protection_from_status(
+		DRM_MODE_HDCP_CONTENT_TYPE0,
+		MOD_HDCP_ENCRYPTION_STATUS_HDCP2_TYPE1_ON,
+		&content_protection);
+
+	KUNIT_EXPECT_FALSE(test, update);
+	KUNIT_EXPECT_EQ(test, content_protection,
+			DRM_MODE_CONTENT_PROTECTION_UNDESIRED);
+}
+
+/* Tests for hdcp_get_link_display_adjustments() */
+
+/**
+ * dm_test_hdcp_get_adjustments_disable_authentication - disable path zeroes adjustments
+ * @test: KUnit test context
+ *
+ * When enable_encryption is false, display_adjust should disable
+ * authentication and all link_adjust fields should remain zeroed.
+ */
+static void dm_test_hdcp_get_adjustments_disable_authentication(struct kunit *test)
+{
+	struct mod_hdcp_link_adjustment link_adjust;
+	struct mod_hdcp_display_adjustment display_adjust;
+	unsigned int disable;
+	unsigned int hdcp1_disable;
+	unsigned int force_type;
+
+	hdcp_get_link_display_adjustments(false, DRM_MODE_HDCP_CONTENT_TYPE0,
+		false, false, false, &link_adjust, &display_adjust);
+	disable = display_adjust.disable;
+	hdcp1_disable = link_adjust.hdcp1.disable;
+	force_type = link_adjust.hdcp2.force_type;
+
+	KUNIT_EXPECT_EQ(test, disable,
+			MOD_HDCP_DISPLAY_DISABLE_AUTHENTICATION);
+	KUNIT_EXPECT_EQ(test, link_adjust.auth_delay, 0);
+	KUNIT_EXPECT_EQ(test, link_adjust.retry_limit, 0);
+	KUNIT_EXPECT_EQ(test, hdcp1_disable, 0);
+	KUNIT_EXPECT_EQ(test, force_type, 0);
+}
+
+/**
+ * dm_test_hdcp_get_adjustments_type0_policy - TYPE0 enables HDCP1 and forces TYPE0
+ * @test: KUnit test context
+ *
+ * When encryption is enabled with content TYPE0, hdcp1 should remain
+ * enabled, force_type should be TYPE_0, and sw_locality_fallback should
+ * be propagated from the input parameter.
+ */
+static void dm_test_hdcp_get_adjustments_type0_policy(struct kunit *test)
+{
+	struct mod_hdcp_link_adjustment link_adjust;
+	struct mod_hdcp_display_adjustment display_adjust;
+	unsigned int disable;
+	unsigned int hdcp1_disable;
+	unsigned int force_type;
+
+	hdcp_get_link_display_adjustments(true, DRM_MODE_HDCP_CONTENT_TYPE0,
+		false, false, true, &link_adjust, &display_adjust);
+	disable = display_adjust.disable;
+	hdcp1_disable = link_adjust.hdcp1.disable;
+	force_type = link_adjust.hdcp2.force_type;
+
+	KUNIT_EXPECT_EQ(test, disable,
+			MOD_HDCP_DISPLAY_NOT_DISABLE);
+	KUNIT_EXPECT_EQ(test, link_adjust.auth_delay, 2);
+	KUNIT_EXPECT_EQ(test, link_adjust.retry_limit, MAX_NUM_OF_ATTEMPTS);
+	KUNIT_EXPECT_EQ(test, hdcp1_disable, 0);
+	KUNIT_EXPECT_EQ(test, force_type,
+			MOD_HDCP_FORCE_TYPE_0);
+	KUNIT_EXPECT_FALSE(test, link_adjust.hdcp2.use_fw_locality_check);
+	KUNIT_EXPECT_TRUE(test, link_adjust.hdcp2.use_sw_locality_fallback);
+}
+
+/**
+ * dm_test_hdcp_get_adjustments_type1_policy - TYPE1 disables HDCP1 and forces TYPE1
+ * @test: KUnit test context
+ *
+ * When encryption is enabled with content TYPE1, hdcp1 should be
+ * disabled, force_type should be TYPE_1, and fw_locality_check should
+ * be enabled when hdcp_lc_force_fw_enable is set.
+ */
+static void dm_test_hdcp_get_adjustments_type1_policy(struct kunit *test)
+{
+	struct mod_hdcp_link_adjustment link_adjust;
+	struct mod_hdcp_display_adjustment display_adjust;
+	unsigned int disable;
+	unsigned int hdcp1_disable;
+	unsigned int force_type;
+
+	hdcp_get_link_display_adjustments(true, DRM_MODE_HDCP_CONTENT_TYPE1,
+		false, true, false, &link_adjust, &display_adjust);
+	disable = display_adjust.disable;
+	hdcp1_disable = link_adjust.hdcp1.disable;
+	force_type = link_adjust.hdcp2.force_type;
+
+	KUNIT_EXPECT_EQ(test, disable,
+			MOD_HDCP_DISPLAY_NOT_DISABLE);
+	KUNIT_EXPECT_EQ(test, link_adjust.auth_delay, 2);
+	KUNIT_EXPECT_EQ(test, link_adjust.retry_limit, MAX_NUM_OF_ATTEMPTS);
+	KUNIT_EXPECT_EQ(test, hdcp1_disable, 1);
+	KUNIT_EXPECT_EQ(test, force_type,
+			MOD_HDCP_FORCE_TYPE_1);
+	KUNIT_EXPECT_TRUE(test, link_adjust.hdcp2.use_fw_locality_check);
+	KUNIT_EXPECT_FALSE(test, link_adjust.hdcp2.use_sw_locality_fallback);
+}
+
+/**
+ * dm_test_hdcp_get_adjustments_fused_io_enables_fw_check - fused_io enables FW locality check
+ * @test: KUnit test context
+ *
+ * When fused_io_supported is true, use_fw_locality_check should be
+ * enabled regardless of hdcp_lc_force_fw_enable.
+ */
+static void dm_test_hdcp_get_adjustments_fused_io_enables_fw_check(struct kunit *test)
+{
+	struct mod_hdcp_link_adjustment link_adjust;
+	struct mod_hdcp_display_adjustment display_adjust;
+
+	hdcp_get_link_display_adjustments(true, DRM_MODE_HDCP_CONTENT_TYPE0,
+		true, false, false, &link_adjust, &display_adjust);
+
+	KUNIT_EXPECT_TRUE(test, link_adjust.hdcp2.use_fw_locality_check);
+}
+
 /* Tests for process_output() */
 
-/*
- * Helper: allocate and initialise a minimal hdcp_workqueue sufficient for
- * process_output() testing.  Only the three delayed works accessed by
+/**
+ * alloc_test_workqueue - allocate a minimal hdcp_workqueue for testing
+ * @test: KUnit test context for managed allocation
+ *
+ * Allocates and initialises a minimal hdcp_workqueue sufficient for
+ * process_output() testing. Only the three delayed works accessed by
  * process_output() are initialised; everything else is zeroed.
  */
 static struct hdcp_workqueue *alloc_test_workqueue(struct kunit *test)
@@ -33,9 +263,12 @@ static struct hdcp_workqueue *alloc_test_workqueue(struct kunit *test)
 	return work;
 }
 
-/*
+/**
+ * dm_test_process_output_property_validate_always_scheduled - validate_dwork always queued
+ * @test: KUnit test context
+ *
  * process_output() always schedules property_validate_dwork with delay=0,
- * which queues the work item directly (bypassing the timer).  Use
+ * which queues the work item directly (bypassing the timer). Uses
  * work_pending() rather than delayed_work_pending() to detect this.
  */
 static void dm_test_process_output_property_validate_always_scheduled(struct kunit *test)
@@ -52,8 +285,12 @@ static void dm_test_process_output_property_validate_always_scheduled(struct kun
 	cancel_delayed_work_sync(&work->property_validate_dwork);
 }
 
-/*
- * output.callback_needed=true must schedule callback_dwork.
+/**
+ * dm_test_process_output_callback_needed - callback_needed schedules callback_dwork
+ * @test: KUnit test context
+ *
+ * When output.callback_needed is true, process_output() must schedule
+ * callback_dwork with the specified delay.
  */
 static void dm_test_process_output_callback_needed(struct kunit *test)
 {
@@ -70,8 +307,12 @@ static void dm_test_process_output_callback_needed(struct kunit *test)
 	cancel_delayed_work_sync(&work->property_validate_dwork);
 }
 
-/*
- * output.callback_stop=true must cancel a previously scheduled callback_dwork.
+/**
+ * dm_test_process_output_callback_stop - callback_stop cancels callback_dwork
+ * @test: KUnit test context
+ *
+ * When output.callback_stop is true, process_output() must cancel a
+ * previously scheduled callback_dwork.
  */
 static void dm_test_process_output_callback_stop(struct kunit *test)
 {
@@ -90,8 +331,12 @@ static void dm_test_process_output_callback_stop(struct kunit *test)
 	cancel_delayed_work_sync(&work->property_validate_dwork);
 }
 
-/*
- * output.watchdog_timer_needed=true must schedule watchdog_timer_dwork.
+/**
+ * dm_test_process_output_watchdog_needed - watchdog_needed schedules watchdog_dwork
+ * @test: KUnit test context
+ *
+ * When output.watchdog_timer_needed is true, process_output() must
+ * schedule watchdog_timer_dwork with the specified delay.
  */
 static void dm_test_process_output_watchdog_needed(struct kunit *test)
 {
@@ -108,9 +353,12 @@ static void dm_test_process_output_watchdog_needed(struct kunit *test)
 	cancel_delayed_work_sync(&work->property_validate_dwork);
 }
 
-/*
- * output.watchdog_timer_stop=true must cancel a previously scheduled
- * watchdog_timer_dwork.
+/**
+ * dm_test_process_output_watchdog_stop - watchdog_stop cancels watchdog_dwork
+ * @test: KUnit test context
+ *
+ * When output.watchdog_timer_stop is true, process_output() must cancel
+ * a previously scheduled watchdog_timer_dwork.
  */
 static void dm_test_process_output_watchdog_stop(struct kunit *test)
 {
@@ -129,9 +377,12 @@ static void dm_test_process_output_watchdog_stop(struct kunit *test)
 	cancel_delayed_work_sync(&work->property_validate_dwork);
 }
 
-/*
- * Both callback_needed and watchdog_timer_needed set: both dworks are
- * scheduled independently.
+/**
+ * dm_test_process_output_callback_and_watchdog_needed - both dworks scheduled independently
+ * @test: KUnit test context
+ *
+ * When both callback_needed and watchdog_timer_needed are set,
+ * process_output() must schedule both dworks independently.
  */
 static void dm_test_process_output_callback_and_watchdog_needed(struct kunit *test)
 {
@@ -151,15 +402,267 @@ static void dm_test_process_output_callback_and_watchdog_needed(struct kunit *te
 	cancel_delayed_work_sync(&work->watchdog_timer_dwork);
 	cancel_delayed_work_sync(&work->property_validate_dwork);
 }
+
+/**
+ * dm_test_process_output_callback_stop_and_needed_requeues - stop+needed requeues callback work
+ * @test: KUnit test context
+ *
+ * When callback_stop and callback_needed are both set, process_output()
+ * should cancel the previous callback_dwork and queue it again.
+ */
+static void dm_test_process_output_callback_stop_and_needed_requeues(struct kunit *test)
+{
+	struct hdcp_workqueue *work = alloc_test_workqueue(test);
+
+	schedule_delayed_work(&work->callback_dwork, msecs_to_jiffies(10000));
+	KUNIT_ASSERT_TRUE(test, delayed_work_pending(&work->callback_dwork));
+
+	work->output.callback_stop = true;
+	work->output.callback_needed = true;
+	work->output.callback_delay = 300;
+
+	process_output(work);
+
+	KUNIT_EXPECT_TRUE(test, delayed_work_pending(&work->callback_dwork));
+
+	cancel_delayed_work_sync(&work->callback_dwork);
+	cancel_delayed_work_sync(&work->property_validate_dwork);
+}
+
+/**
+ * dm_test_process_output_watchdog_stop_and_needed_requeues - stop+needed requeues watchdog work
+ * @test: KUnit test context
+ *
+ * When watchdog_timer_stop and watchdog_timer_needed are both set,
+ * process_output() should cancel previous watchdog work and queue it again.
+ */
+static void dm_test_process_output_watchdog_stop_and_needed_requeues(struct kunit *test)
+{
+	struct hdcp_workqueue *work = alloc_test_workqueue(test);
+
+	schedule_delayed_work(&work->watchdog_timer_dwork, msecs_to_jiffies(10000));
+	KUNIT_ASSERT_TRUE(test, delayed_work_pending(&work->watchdog_timer_dwork));
+
+	work->output.watchdog_timer_stop = true;
+	work->output.watchdog_timer_needed = true;
+	work->output.watchdog_timer_delay = 700;
+
+	process_output(work);
+
+	KUNIT_EXPECT_TRUE(test, delayed_work_pending(&work->watchdog_timer_dwork));
+
+	cancel_delayed_work_sync(&work->watchdog_timer_dwork);
+	cancel_delayed_work_sync(&work->property_validate_dwork);
+}
 /* End of tests for process_output() */
 
+/* Tests for event_property_update() */
+
+/**
+ * alloc_test_workqueue_for_property_update - allocate minimal workqueue for callback tests
+ * @test: KUnit test context for managed allocation
+ *
+ * Allocates a minimal hdcp_workqueue with property_update_work initialised
+ * so event_property_update() can resolve container_of() safely.
+ */
+static struct hdcp_workqueue *alloc_test_workqueue_for_property_update(struct kunit *test)
+{
+	struct hdcp_workqueue *work;
+
+	work = kunit_kzalloc(test, sizeof(*work), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, work);
+
+	INIT_WORK(&work->property_update_work, dummy_work_fn);
+
+	return work;
+}
+
+/**
+ * dm_test_event_property_update_skips_null_connector - null connector is ignored
+ * @test: KUnit test context
+ *
+ * If aconnector entry is NULL, event_property_update() should skip it
+ * without modifying encryption_status.
+ */
+static void dm_test_event_property_update_skips_null_connector(struct kunit *test)
+{
+	struct hdcp_workqueue *work = alloc_test_workqueue_for_property_update(test);
+	enum mod_hdcp_encryption_status before;
+
+	work->encryption_status[0] = MOD_HDCP_ENCRYPTION_STATUS_HDCP2_TYPE0_ON;
+	before = work->encryption_status[0];
+
+	event_property_update(&work->property_update_work);
+
+	KUNIT_EXPECT_EQ(test, work->encryption_status[0], before);
+}
+
+/* End of tests for event_property_update() */
+
+/* Tests for hdcp_handle_cpirq() */
+
+/**
+ * dm_test_hdcp_handle_cpirq_schedules_work - cpirq handler queues cpirq_work
+ * @test: KUnit test context
+ *
+ * hdcp_handle_cpirq() should schedule cpirq_work for the selected link.
+ */
+static void dm_test_hdcp_handle_cpirq_schedules_work(struct kunit *test)
+{
+	struct hdcp_workqueue *work;
+
+	work = kunit_kzalloc(test, sizeof(*work), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, work);
+
+	INIT_WORK(&work->cpirq_work, dummy_work_fn);
+
+	hdcp_handle_cpirq(work, 0);
+
+	KUNIT_EXPECT_TRUE(test, work_pending(&work->cpirq_work));
+
+	cancel_work_sync(&work->cpirq_work);
+}
+
+/**
+ * dm_test_hdcp_handle_cpirq_selects_link_index - only selected link work is queued
+ * @test: KUnit test context
+ *
+ * hdcp_handle_cpirq() should schedule cpirq_work for the selected index
+ * and not queue unrelated links.
+ */
+static void dm_test_hdcp_handle_cpirq_selects_link_index(struct kunit *test)
+{
+	struct hdcp_workqueue *work;
+
+	work = kunit_kcalloc(test, 2, sizeof(*work), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, work);
+
+	INIT_WORK(&work[0].cpirq_work, dummy_work_fn);
+	INIT_WORK(&work[1].cpirq_work, dummy_work_fn);
+
+	hdcp_handle_cpirq(work, 1);
+
+	KUNIT_EXPECT_FALSE(test, work_pending(&work[0].cpirq_work));
+	KUNIT_EXPECT_TRUE(test, work_pending(&work[1].cpirq_work));
+
+	cancel_work_sync(&work[1].cpirq_work);
+}
+
+/* End of tests for hdcp_handle_cpirq() */
+
+/* Tests for hdcp_update_display() helper logic */
+
+/**
+ * dm_test_hdcp_update_display_enable_schedules_property_validate - enable path queues validate work
+ * @test: KUnit test context
+ *
+ * hdcp_update_display() should schedule property_validate_dwork when
+ * encryption is enabled.
+ */
+static void dm_test_hdcp_update_display_enable_schedules_property_validate(struct kunit *test)
+{
+	struct hdcp_workqueue *work;
+
+	work = kunit_kzalloc(test, sizeof(*work), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, work);
+
+	INIT_DELAYED_WORK(&work->property_validate_dwork, dummy_work_fn);
+	work->srm_size = 0;
+
+	hdcp_update_display_encryption_control(work, work, 0, true);
+
+	KUNIT_EXPECT_TRUE(test, delayed_work_pending(&work->property_validate_dwork));
+
+	cancel_delayed_work_sync(&work->property_validate_dwork);
+}
+
+/**
+ * dm_test_hdcp_update_display_disable_resets_status_and_cancels_validate - disable path state update
+ * @test: KUnit test context
+ *
+ * hdcp_update_display() should set encryption_status to HDCP_OFF and
+ * cancel property_validate_dwork when encryption is disabled.
+ */
+static void dm_test_hdcp_update_display_disable_resets_status_and_cancels_validate(struct kunit *test)
+{
+	struct hdcp_workqueue *work;
+
+	work = kunit_kzalloc(test, sizeof(*work), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, work);
+
+	INIT_DELAYED_WORK(&work->property_validate_dwork, dummy_work_fn);
+	schedule_delayed_work(&work->property_validate_dwork, msecs_to_jiffies(10000));
+	KUNIT_ASSERT_TRUE(test, delayed_work_pending(&work->property_validate_dwork));
+
+	work->encryption_status[3] = MOD_HDCP_ENCRYPTION_STATUS_HDCP2_TYPE1_ON;
+
+	hdcp_update_display_encryption_control(work, work, 3, false);
+
+	KUNIT_EXPECT_EQ(test, work->encryption_status[3],
+			MOD_HDCP_ENCRYPTION_STATUS_HDCP_OFF);
+	KUNIT_EXPECT_FALSE(test, delayed_work_pending(&work->property_validate_dwork));
+}
+
+/* End of tests for hdcp_update_display() helper logic */
+
+/* Tests for hdcp_create_workqueue() */
+
+/**
+ * dm_test_hdcp_create_workqueue_zero_max_links_returns_null - zero-link creation fails early
+ * @test: KUnit test context
+ *
+ * When dc->caps.max_links is zero, hdcp_create_workqueue() should fail
+ * the initial allocation path and return NULL.
+ */
+static void dm_test_hdcp_create_workqueue_zero_max_links_returns_null(struct kunit *test)
+{
+	struct cp_psp cp_psp = {0};
+	struct dc *dc;
+	struct hdcp_workqueue *work;
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+
+	dc->caps.max_links = 0;
+
+	work = hdcp_create_workqueue(NULL, &cp_psp, dc);
+
+	KUNIT_EXPECT_PTR_EQ(test, work, NULL);
+}
+
+/* End of tests for hdcp_create_workqueue() */
+
 static struct kunit_case dm_hdcp_test_cases[] = {
+	/* hdcp_get_content_protection_from_status() */
+	KUNIT_CASE(dm_test_hdcp_get_cp_disabled_returns_desired),
+	KUNIT_CASE(dm_test_hdcp_get_cp_type0_returns_enabled),
+	KUNIT_CASE(dm_test_hdcp_get_cp_type1_returns_enabled),
+	KUNIT_CASE(dm_test_hdcp_get_cp_type1_rejects_type0_status),
+	KUNIT_CASE(dm_test_hdcp_get_cp_type0_rejects_type1_status),
+	/* hdcp_get_link_display_adjustments() */
+	KUNIT_CASE(dm_test_hdcp_get_adjustments_disable_authentication),
+	KUNIT_CASE(dm_test_hdcp_get_adjustments_type0_policy),
+	KUNIT_CASE(dm_test_hdcp_get_adjustments_type1_policy),
+	KUNIT_CASE(dm_test_hdcp_get_adjustments_fused_io_enables_fw_check),
+	/* process_output() */
 	KUNIT_CASE(dm_test_process_output_property_validate_always_scheduled),
 	KUNIT_CASE(dm_test_process_output_callback_needed),
 	KUNIT_CASE(dm_test_process_output_callback_stop),
 	KUNIT_CASE(dm_test_process_output_watchdog_needed),
 	KUNIT_CASE(dm_test_process_output_watchdog_stop),
 	KUNIT_CASE(dm_test_process_output_callback_and_watchdog_needed),
+	KUNIT_CASE(dm_test_process_output_callback_stop_and_needed_requeues),
+	KUNIT_CASE(dm_test_process_output_watchdog_stop_and_needed_requeues),
+	/* event_property_update() */
+	KUNIT_CASE(dm_test_event_property_update_skips_null_connector),
+	/* hdcp_handle_cpirq() */
+	KUNIT_CASE(dm_test_hdcp_handle_cpirq_schedules_work),
+	KUNIT_CASE(dm_test_hdcp_handle_cpirq_selects_link_index),
+	/* hdcp_update_display() helper logic */
+	KUNIT_CASE(dm_test_hdcp_update_display_enable_schedules_property_validate),
+	KUNIT_CASE(dm_test_hdcp_update_display_disable_resets_status_and_cancels_validate),
+	/* hdcp_create_workqueue() */
+	KUNIT_CASE(dm_test_hdcp_create_workqueue_zero_max_links_returns_null),
 	{}
 };
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_helpers_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_helpers_test.c
new file mode 100644
index 0000000000000..058e1ad15dfef
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_helpers_test.c
@@ -0,0 +1,3907 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * KUnit tests for amdgpu_dm_helpers.c
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#include <kunit/test.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_kunit_helpers.h>
+#include <drm/display/drm_dp_mst_helper.h>
+
+#include "dc.h"
+#include "core_types.h"
+#include "amdgpu.h"
+#include "amdgpu_mode.h"
+#include "amdgpu_dm.h"
+#include "amdgpu_dm_mst_types.h"
+#include "dc_bios_types.h"
+#include "dm_helpers.h"
+#include "ddc_service_types.h"
+#include "dmub_cmd.h"
+#include "amdgpu_dm_helpers.h"
+#include "amdgpu_dm_kunit_test_helpers.h"
+
+/* Tests for edid_extract_panel_id() */
+
+/**
+ * dm_test_edid_extract_panel_id_basic - Test Edid extract panel id basic
+ * @test: The KUnit test context
+ */
+static void dm_test_edid_extract_panel_id_basic(struct kunit *test)
+{
+	struct edid *edid;
+	u32 panel_id;
+
+	edid = kunit_kzalloc(test, sizeof(*edid), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, edid);
+
+	edid->mfg_id[0] = 0x12;
+	edid->mfg_id[1] = 0x34;
+	edid->prod_code[0] = 0xAB;
+	edid->prod_code[1] = 0xCD;
+
+	panel_id = edid_extract_panel_id(edid);
+
+	/*
+	 * Expected: (0x12 << 24) | (0x34 << 16) | EDID_PRODUCT_ID(edid)
+	 * EDID_PRODUCT_ID = prod_code[0] | (prod_code[1] << 8) = 0xAB | 0xCD00 = 0xCDAB
+	 * Result: 0x12340000 | 0x0000CDAB = 0x1234CDAB
+	 */
+	KUNIT_EXPECT_EQ(test, panel_id, (u32)0x1234CDAB);
+}
+
+/**
+ * dm_test_edid_extract_panel_id_zeros - Test Edid extract panel id zeros
+ * @test: The KUnit test context
+ */
+static void dm_test_edid_extract_panel_id_zeros(struct kunit *test)
+{
+	struct edid *edid;
+
+	edid = kunit_kzalloc(test, sizeof(*edid), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, edid);
+
+	KUNIT_EXPECT_EQ(test, edid_extract_panel_id(edid), 0U);
+}
+
+/* Tests for apply_edid_quirks() */
+
+/*
+ * Build an EDID whose extracted panel id equals @panel_id. Inverse of
+ * edid_extract_panel_id(): mfg_id holds the top 16 bits, prod_code the
+ * low 16 bits (prod_code[0] | prod_code[1] << 8).
+ */
+static struct edid *dm_test_edid_with_panel_id(struct kunit *test, u32 panel_id)
+{
+	struct edid *edid;
+
+	edid = kunit_kzalloc(test, sizeof(*edid), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, edid);
+
+	edid->mfg_id[0] = (panel_id >> 24) & 0xff;
+	edid->mfg_id[1] = (panel_id >> 16) & 0xff;
+	edid->prod_code[0] = panel_id & 0xff;
+	edid->prod_code[1] = (panel_id >> 8) & 0xff;
+
+	return edid;
+}
+
+/*
+ * Wire a connector-backed link so apply_edid_quirks() can resolve
+ * link->priv->base.dev for its drm_dbg_driver() calls.
+ */
+static struct dc_link *dm_test_quirk_link(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_device *adev;
+	struct dc_link *link;
+
+	adev = dm_kunit_alloc_adev(test);
+	link = dm_kunit_alloc_link(test);
+	aconnector = dm_kunit_alloc_connector(test, adev, NULL);
+	link->priv = aconnector;
+
+	return link;
+}
+
+/**
+ * dm_test_apply_edid_quirks_dpcd_poweroff_delay - Test GBT 0x3215 delay quirk
+ * @test: The KUnit test context
+ */
+static void dm_test_apply_edid_quirks_dpcd_poweroff_delay(struct kunit *test)
+{
+	struct dc_edid_caps edid_caps = {0};
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct edid *edid = dm_test_edid_with_panel_id(test,
+			drm_edid_encode_panel_id('G', 'B', 'T', 0x3215));
+
+	apply_edid_quirks(link, edid, &edid_caps);
+
+	KUNIT_EXPECT_EQ(test, edid_caps.panel_patch.wait_after_dpcd_poweroff_ms, 10000U);
+}
+
+/**
+ * dm_test_apply_edid_quirks_disable_fams - Test SAM panel FAMS-disable quirk
+ * @test: The KUnit test context
+ */
+static void dm_test_apply_edid_quirks_disable_fams(struct kunit *test)
+{
+	struct dc_edid_caps edid_caps = {0};
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct edid *edid = dm_test_edid_with_panel_id(test,
+			drm_edid_encode_panel_id('S', 'A', 'M', 0x0E5E));
+
+	apply_edid_quirks(link, edid, &edid_caps);
+
+	KUNIT_EXPECT_TRUE(test, edid_caps.panel_patch.disable_fams);
+}
+
+/**
+ * dm_test_apply_edid_quirks_remove_sink_ext_caps - Test AUO 0x317 clear quirk
+ * @test: The KUnit test context
+ */
+static void dm_test_apply_edid_quirks_remove_sink_ext_caps(struct kunit *test)
+{
+	struct dc_edid_caps edid_caps = {0};
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct edid *edid = dm_test_edid_with_panel_id(test,
+			drm_edid_encode_panel_id('A', 'U', 'O', 0xA7AB));
+
+	apply_edid_quirks(link, edid, &edid_caps);
+
+	KUNIT_EXPECT_TRUE(test, edid_caps.panel_patch.remove_sink_ext_caps);
+}
+
+/**
+ * dm_test_apply_edid_quirks_disable_colorimetry - Test SDC VSC-disable quirk
+ * @test: The KUnit test context
+ */
+static void dm_test_apply_edid_quirks_disable_colorimetry(struct kunit *test)
+{
+	struct dc_edid_caps edid_caps = {0};
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct edid *edid = dm_test_edid_with_panel_id(test,
+			drm_edid_encode_panel_id('S', 'D', 'C', 0x4154));
+
+	apply_edid_quirks(link, edid, &edid_caps);
+
+	KUNIT_EXPECT_TRUE(test, edid_caps.panel_patch.disable_colorimetry);
+}
+
+/**
+ * dm_test_apply_edid_quirks_skip_phy_ssc - Test DEL 0x4147 PHY SSC quirk
+ * @test: The KUnit test context
+ */
+static void dm_test_apply_edid_quirks_skip_phy_ssc(struct kunit *test)
+{
+	struct dc_edid_caps edid_caps = {0};
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct edid *edid = dm_test_edid_with_panel_id(test,
+			drm_edid_encode_panel_id('D', 'E', 'L', 0x4147));
+
+	apply_edid_quirks(link, edid, &edid_caps);
+
+	KUNIT_EXPECT_TRUE(test, link->wa_flags.skip_phy_ssc_reduction);
+}
+
+/**
+ * dm_test_apply_edid_quirks_unknown_noop - Test unknown panel id is a no-op
+ * @test: The KUnit test context
+ */
+static void dm_test_apply_edid_quirks_unknown_noop(struct kunit *test)
+{
+	struct dc_edid_caps edid_caps = {0};
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct edid *edid = dm_test_edid_with_panel_id(test,
+			drm_edid_encode_panel_id('X', 'Y', 'Z', 0x0000));
+
+	apply_edid_quirks(link, edid, &edid_caps);
+
+	/* default: branch leaves every quirk field untouched */
+	KUNIT_EXPECT_EQ(test, edid_caps.panel_patch.wait_after_dpcd_poweroff_ms, 0U);
+	KUNIT_EXPECT_FALSE(test, edid_caps.panel_patch.disable_fams);
+	KUNIT_EXPECT_FALSE(test, edid_caps.panel_patch.remove_sink_ext_caps);
+	KUNIT_EXPECT_FALSE(test, edid_caps.panel_patch.disable_colorimetry);
+	KUNIT_EXPECT_FALSE(test, link->wa_flags.skip_phy_ssc_reduction);
+}
+
+/* Tests for dm_helpers_parse_edid_caps() */
+
+/*
+ * Build a minimal valid base EDID block into @dc_edid. When @good_checksum is
+ * false the final checksum byte is corrupted so drm_edid_is_valid() fails.
+ *
+ *   manufacturer_id = 0xAC10, product_id = 0x1234, serial = 0x12345678,
+ *   week = 10, year (raw) = 30, digital input, no CEA extension.
+ */
+static void dm_test_fill_base_edid(struct dc_edid *dc_edid, bool good_checksum)
+{
+	static const u8 header[8] = {
+		0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00
+	};
+	u8 *raw = dc_edid->raw_edid;
+	int sum = 0;
+	int i;
+
+	memset(raw, 0, EDID_LENGTH);
+	memcpy(raw, header, sizeof(header));
+
+	raw[8] = 0x10;	raw[9] = 0xAC;		/* mfg_id    -> 0xAC10 */
+	raw[10] = 0x34;	raw[11] = 0x12;		/* prod_code -> 0x1234 */
+	raw[12] = 0x78;	raw[13] = 0x56;		/* serial    -> 0x12345678 */
+	raw[14] = 0x34;	raw[15] = 0x12;
+	raw[16] = 10;				/* mfg_week */
+	raw[17] = 30;				/* mfg_year (raw) */
+	raw[18] = 1;				/* version */
+	raw[19] = 4;				/* revision */
+	raw[20] = DRM_EDID_INPUT_DIGITAL;	/* digital input */
+	raw[126] = 0;				/* no extensions */
+
+	for (i = 0; i < EDID_LENGTH - 1; i++)
+		sum += raw[i];
+	raw[127] = (256 - (sum % 256)) % 256;
+	if (!good_checksum)
+		raw[127] ^= 0xFF;		/* corrupt checksum */
+
+	dc_edid->length = EDID_LENGTH;
+}
+
+/**
+ * dm_test_parse_edid_caps_null_edid - Test NULL edid returns EDID_BAD_INPUT
+ * @test: The KUnit test context
+ */
+static void dm_test_parse_edid_caps_null_edid(struct kunit *test)
+{
+	struct dc_edid_caps edid_caps = {0};
+	struct dc_link *link = dm_test_quirk_link(test);
+
+	KUNIT_EXPECT_EQ(test, dm_helpers_parse_edid_caps(link, NULL, &edid_caps), EDID_BAD_INPUT);
+}
+
+/**
+ * dm_test_parse_edid_caps_null_caps - Test NULL edid_caps returns EDID_BAD_INPUT
+ * @test: The KUnit test context
+ */
+static void dm_test_parse_edid_caps_null_caps(struct kunit *test)
+{
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct dc_edid *dc_edid;
+
+	dc_edid = kunit_kzalloc(test, sizeof(*dc_edid), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc_edid);
+	dm_test_fill_base_edid(dc_edid, true);
+
+	KUNIT_EXPECT_EQ(test, dm_helpers_parse_edid_caps(link, dc_edid, NULL), EDID_BAD_INPUT);
+}
+
+/**
+ * dm_test_parse_edid_caps_valid - Test field extraction from a valid EDID
+ * @test: The KUnit test context
+ */
+static void dm_test_parse_edid_caps_valid(struct kunit *test)
+{
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct dc_edid_caps *edid_caps;
+	struct dc_edid *dc_edid;
+
+	dc_edid = kunit_kzalloc(test, sizeof(*dc_edid), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc_edid);
+	edid_caps = kunit_kzalloc(test, sizeof(*edid_caps), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, edid_caps);
+
+	dm_test_fill_base_edid(dc_edid, true);
+
+	KUNIT_EXPECT_EQ(test, dm_helpers_parse_edid_caps(link, dc_edid, edid_caps), EDID_OK);
+	KUNIT_EXPECT_EQ(test, edid_caps->manufacturer_id, 0xAC10);
+	KUNIT_EXPECT_EQ(test, edid_caps->product_id, 0x1234);
+	KUNIT_EXPECT_EQ(test, edid_caps->serial_number, 0x12345678U);
+	KUNIT_EXPECT_EQ(test, edid_caps->manufacture_week, 10);
+	KUNIT_EXPECT_EQ(test, edid_caps->manufacture_year, 30);
+	KUNIT_EXPECT_FALSE(test, edid_caps->analog);
+}
+
+/**
+ * dm_test_parse_edid_caps_bad_checksum - Test bad checksum still parses fields
+ * @test: The KUnit test context
+ */
+static void dm_test_parse_edid_caps_bad_checksum(struct kunit *test)
+{
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct dc_edid_caps *edid_caps;
+	struct dc_edid *dc_edid;
+
+	dc_edid = kunit_kzalloc(test, sizeof(*dc_edid), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc_edid);
+	edid_caps = kunit_kzalloc(test, sizeof(*edid_caps), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, edid_caps);
+
+	dm_test_fill_base_edid(dc_edid, false);
+
+	/* Invalid checksum -> EDID_BAD_CHECKSUM but fields are still parsed */
+	KUNIT_EXPECT_EQ(test,
+			dm_helpers_parse_edid_caps(link, dc_edid, edid_caps),
+			EDID_BAD_CHECKSUM);
+	KUNIT_EXPECT_EQ(test, edid_caps->manufacturer_id, 0xAC10);
+	KUNIT_EXPECT_EQ(test, edid_caps->product_id, 0x1234);
+}
+
+/**
+ * dm_test_parse_edid_caps_hdmi_frl - Test HDMI/FRL branch via connector info
+ * @test: The KUnit test context
+ *
+ * Drives the edid_caps->edid_hdmi path by marking the connector as HDMI and
+ * providing a fake dc with FRL enabled, so populate_hdmi_info_from_connector()
+ * is exercised inside dm_helpers_parse_edid_caps().
+ */
+static void dm_test_parse_edid_caps_hdmi_frl(struct kunit *test)
+{
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct amdgpu_dm_connector *aconnector = link->priv;
+	struct drm_connector *connector = &aconnector->base;
+	struct dc_edid_caps *edid_caps;
+	struct dc_edid *dc_edid;
+	struct dc *dc;
+
+	dc_edid = kunit_kzalloc(test, sizeof(*dc_edid), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc_edid);
+	edid_caps = kunit_kzalloc(test, sizeof(*edid_caps), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, edid_caps);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+
+	link->dc = dc;
+	dc->config.enable_frl = true;
+
+	dm_test_fill_base_edid(dc_edid, true);
+
+	/* Drive the HDMI/FRL branch */
+	connector->display_info.is_hdmi = true;
+	connector->display_info.hdmi.scdc.supported = true;
+	connector->display_info.hdmi.max_lanes = 4;
+	connector->display_info.hdmi.max_frl_rate_per_lane = 12;
+
+	KUNIT_EXPECT_EQ(test, dm_helpers_parse_edid_caps(link, dc_edid, edid_caps), EDID_OK);
+	KUNIT_EXPECT_TRUE(test, edid_caps->edid_hdmi);
+	KUNIT_EXPECT_TRUE(test, edid_caps->scdc_present);
+	/* max_lanes 4 + max_frl_rate_per_lane 12 -> rate index 6 */
+	KUNIT_EXPECT_EQ(test, edid_caps->max_frl_rate, 6);
+}
+
+/**
+ * dm_test_parse_edid_caps_hdmi_frl_dsc - Test HDMI FRL DSC sub-branch
+ * @test: The KUnit test context
+ *
+ * Sets the connector's HDMI DSC capability so populate_hdmi_info_from_connector()
+ * reports frl_dsc_support, exercising the frl_dsc_support log path.
+ */
+static void dm_test_parse_edid_caps_hdmi_frl_dsc(struct kunit *test)
+{
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct amdgpu_dm_connector *aconnector = link->priv;
+	struct drm_connector *connector = &aconnector->base;
+	struct dc_edid_caps *edid_caps;
+	struct dc_edid *dc_edid;
+	struct dc *dc;
+
+	dc_edid = kunit_kzalloc(test, sizeof(*dc_edid), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc_edid);
+	edid_caps = kunit_kzalloc(test, sizeof(*edid_caps), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, edid_caps);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+
+	link->dc = dc;
+	dc->config.enable_frl = true;
+
+	dm_test_fill_base_edid(dc_edid, true);
+
+	connector->display_info.is_hdmi = true;
+	/* Drive the frl_dsc_support sub-branch */
+	connector->display_info.hdmi.dsc_cap.v_1p2 = true;
+	connector->display_info.hdmi.dsc_cap.bpc_supported = 10;
+
+	KUNIT_EXPECT_EQ(test, dm_helpers_parse_edid_caps(link, dc_edid, edid_caps), EDID_OK);
+	KUNIT_EXPECT_TRUE(test, edid_caps->frl_dsc_support);
+	KUNIT_EXPECT_TRUE(test, edid_caps->frl_dsc_10bpc);
+}
+
+/*
+ * Build a 2-block EDID: a valid base block plus a CTA-861 extension block
+ * carrying one LPCM Short Audio Descriptor and, when @with_speaker is set, a
+ * Speaker Allocation Data Block, so the audio/speaker parsing code is
+ * exercised.
+ */
+static void dm_test_fill_cea_edid(struct dc_edid *dc_edid, bool with_speaker)
+{
+	u8 *raw = dc_edid->raw_edid;
+	u8 *ext = raw + EDID_LENGTH;
+	u8 dtd_offset;
+	int sum = 0;
+	int i;
+
+	/* Base block, flagged as having one extension */
+	dm_test_fill_base_edid(dc_edid, true);
+	raw[126] = 1;
+	for (i = 0; i < EDID_LENGTH - 1; i++)
+		sum += raw[i];
+	raw[127] = (256 - (sum % 256)) % 256;
+
+	/* Audio DB spans [4,7]; optional Speaker Alloc DB spans [8,11] */
+	dtd_offset = with_speaker ? 12 : 8;
+
+	/* CTA-861 extension block */
+	memset(ext, 0, EDID_LENGTH);
+	ext[0] = 0x02;			/* CTA extension tag */
+	ext[1] = 0x03;			/* revision 3 */
+	ext[2] = dtd_offset;		/* DTD offset; end of data blocks */
+	ext[3] = 0x00;			/* no native DTDs / feature flags */
+	/* Audio Data Block: tag 1, length 3, one LPCM SAD */
+	ext[4] = (1 << 5) | 3;
+	ext[5] = (1 << 3) | 1;		/* format 1 (LPCM), 2 channels */
+	ext[6] = 0x07;			/* 32 / 44.1 / 48 kHz */
+	ext[7] = 0x07;			/* 16 / 20 / 24-bit */
+	if (with_speaker) {
+		/* Speaker Allocation Data Block: tag 4, length 3 */
+		ext[8] = (4 << 5) | 3;
+		ext[9] = 0x01;		/* front left / front right */
+	}
+
+	sum = 0;
+	for (i = 0; i < EDID_LENGTH - 1; i++)
+		sum += ext[i];
+	ext[127] = (256 - (sum % 256)) % 256;
+
+	dc_edid->length = 2 * EDID_LENGTH;
+}
+
+/**
+ * dm_test_parse_edid_caps_cea_audio - Test CEA audio/speaker block parsing
+ * @test: The KUnit test context
+ */
+static void dm_test_parse_edid_caps_cea_audio(struct kunit *test)
+{
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct dc_edid_caps *edid_caps;
+	struct dc_edid *dc_edid;
+
+	dc_edid = kunit_kzalloc(test, sizeof(*dc_edid), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc_edid);
+	edid_caps = kunit_kzalloc(test, sizeof(*edid_caps), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, edid_caps);
+
+	dm_test_fill_cea_edid(dc_edid, true);
+
+	KUNIT_EXPECT_EQ(test, dm_helpers_parse_edid_caps(link, dc_edid, edid_caps), EDID_OK);
+	KUNIT_EXPECT_EQ(test, edid_caps->audio_mode_count, 1);
+	KUNIT_EXPECT_EQ(test, edid_caps->audio_modes[0].format_code, 1);
+	KUNIT_EXPECT_EQ(test, edid_caps->audio_modes[0].channel_count, 2);
+	KUNIT_EXPECT_EQ(test, edid_caps->audio_modes[0].sample_rate, 0x07);
+	KUNIT_EXPECT_EQ(test, edid_caps->audio_modes[0].sample_size, 0x07);
+	KUNIT_EXPECT_EQ(test, edid_caps->speaker_flags, 0x01);
+}
+
+/**
+ * dm_test_parse_edid_caps_cea_no_speaker - Test default speaker flags path
+ * @test: The KUnit test context
+ *
+ * Audio data block present but no Speaker Allocation Data Block, so
+ * speaker_flags falls back to DEFAULT_SPEAKER_LOCATION.
+ */
+static void dm_test_parse_edid_caps_cea_no_speaker(struct kunit *test)
+{
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct dc_edid_caps *edid_caps;
+	struct dc_edid *dc_edid;
+
+	dc_edid = kunit_kzalloc(test, sizeof(*dc_edid), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc_edid);
+	edid_caps = kunit_kzalloc(test, sizeof(*edid_caps), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, edid_caps);
+
+	dm_test_fill_cea_edid(dc_edid, false);
+
+	KUNIT_EXPECT_EQ(test, dm_helpers_parse_edid_caps(link, dc_edid, edid_caps), EDID_OK);
+	KUNIT_EXPECT_EQ(test, edid_caps->audio_mode_count, 1);
+	KUNIT_EXPECT_EQ(test, edid_caps->speaker_flags, DEFAULT_SPEAKER_LOCATION);
+}
+
+/* Tests for ACPI and VBIOS EDID readers */
+
+/**
+ * dm_test_probe_acpi_edid_no_companion - Test ACPI probe without companion
+ * @test: The KUnit test context
+ */
+static void dm_test_probe_acpi_edid_no_companion(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_device *adev;
+	u8 buf[EDID_LENGTH] = {0};
+
+	adev = dm_kunit_alloc_adev(test);
+	aconnector = dm_kunit_alloc_connector(test, adev, NULL);
+
+	KUNIT_EXPECT_EQ(test,
+			dm_helpers_probe_acpi_edid(&aconnector->base, buf, 0, sizeof(buf)),
+			-ENODEV);
+}
+
+/**
+ * dm_test_read_acpi_edid_debug_mask_disabled - Test debug mask early return
+ * @test: The KUnit test context
+ */
+static void dm_test_read_acpi_edid_debug_mask_disabled(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_device *adev;
+	uint old_debug_mask;
+
+	adev = dm_kunit_alloc_adev(test);
+	aconnector = dm_kunit_alloc_connector(test, adev, NULL);
+	old_debug_mask = dm_helpers_get_dc_debug_mask();
+
+	dm_helpers_set_dc_debug_mask(old_debug_mask | DC_DISABLE_ACPI_EDID);
+	KUNIT_EXPECT_NULL(test, dm_helpers_read_acpi_edid(aconnector));
+	dm_helpers_set_dc_debug_mask(old_debug_mask);
+}
+
+/**
+ * dm_test_read_acpi_edid_non_panel_connector - Test non-panel connector skip
+ * @test: The KUnit test context
+ */
+static void dm_test_read_acpi_edid_non_panel_connector(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+	aconnector = dm_kunit_alloc_connector(test, adev, NULL);
+	aconnector->base.connector_type = DRM_MODE_CONNECTOR_HDMIA;
+
+	KUNIT_EXPECT_NULL(test, dm_helpers_read_acpi_edid(aconnector));
+}
+
+/**
+ * dm_test_read_acpi_edid_force_off - Test forced-off connector skip
+ * @test: The KUnit test context
+ */
+static void dm_test_read_acpi_edid_force_off(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+	aconnector = dm_kunit_alloc_connector(test, adev, NULL);
+	aconnector->base.connector_type = DRM_MODE_CONNECTOR_eDP;
+	aconnector->base.force = DRM_FORCE_OFF;
+
+	KUNIT_EXPECT_NULL(test, dm_helpers_read_acpi_edid(aconnector));
+}
+
+struct dm_test_vbios_edid {
+	struct dc_bios bios;
+	enum bp_result result;
+	const u8 *fake_edid;
+	u16 fake_edid_size;
+	u16 width_mm;
+	u16 height_mm;
+};
+
+static enum bp_result dm_test_get_embedded_panel_info(struct dc_bios *bios,
+						      struct embedded_panel_info *info)
+{
+	struct dm_test_vbios_edid *fixture;
+
+	fixture = container_of(bios, struct dm_test_vbios_edid, bios);
+	if (fixture->result != BP_RESULT_OK)
+		return fixture->result;
+
+	info->fake_edid = fixture->fake_edid;
+	info->fake_edid_size = fixture->fake_edid_size;
+	info->panel_width_mm = fixture->width_mm;
+	info->panel_height_mm = fixture->height_mm;
+
+	return BP_RESULT_OK;
+}
+
+static const struct dc_vbios_funcs dm_test_vbios_edid_funcs = {
+	.get_embedded_panel_info = dm_test_get_embedded_panel_info,
+};
+
+static void dm_test_setup_vbios_link(struct kunit *test,
+				     struct dc_link **link_out,
+				     struct amdgpu_dm_connector **aconnector_out,
+				     struct dm_test_vbios_edid **fixture_out)
+{
+	struct dm_test_vbios_edid *fixture;
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_link *link;
+
+	adev = dm_kunit_alloc_adev(test);
+	link = dm_kunit_alloc_link(test);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture);
+	aconnector = dm_kunit_alloc_connector(test, adev, link);
+
+	fixture->bios.funcs = &dm_test_vbios_edid_funcs;
+	fixture->result = BP_RESULT_OK;
+	ctx->dc_bios = &fixture->bios;
+	link->ctx = ctx;
+	link->priv = aconnector;
+	link->connector_signal = SIGNAL_TYPE_EDP;
+
+	*link_out = link;
+	*aconnector_out = aconnector;
+	*fixture_out = fixture;
+}
+
+/**
+ * dm_test_read_vbios_edid_non_embedded - Test non-embedded signal skip
+ * @test: The KUnit test context
+ */
+static void dm_test_read_vbios_edid_non_embedded(struct kunit *test)
+{
+	struct dm_test_vbios_edid *fixture;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+
+	dm_test_setup_vbios_link(test, &link, &aconnector, &fixture);
+	link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+
+	KUNIT_EXPECT_NULL(test,
+			  dm_helpers_read_vbios_hardcoded_edid(link, aconnector));
+}
+
+/**
+ * dm_test_read_vbios_edid_missing_callback - Test missing callback skip
+ * @test: The KUnit test context
+ */
+static void dm_test_read_vbios_edid_missing_callback(struct kunit *test)
+{
+	static const struct dc_vbios_funcs empty_funcs;
+	struct dm_test_vbios_edid *fixture;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+
+	dm_test_setup_vbios_link(test, &link, &aconnector, &fixture);
+	fixture->bios.funcs = &empty_funcs;
+
+	KUNIT_EXPECT_NULL(test,
+			  dm_helpers_read_vbios_hardcoded_edid(link, aconnector));
+}
+
+/**
+ * dm_test_read_vbios_edid_callback_error - Test callback failure skip
+ * @test: The KUnit test context
+ */
+static void dm_test_read_vbios_edid_callback_error(struct kunit *test)
+{
+	struct dm_test_vbios_edid *fixture;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+
+	dm_test_setup_vbios_link(test, &link, &aconnector, &fixture);
+	fixture->result = BP_RESULT_BADINPUT;
+
+	KUNIT_EXPECT_NULL(test,
+			  dm_helpers_read_vbios_hardcoded_edid(link, aconnector));
+}
+
+/**
+ * dm_test_read_vbios_edid_missing_fake_edid - Test missing fake EDID skip
+ * @test: The KUnit test context
+ */
+static void dm_test_read_vbios_edid_missing_fake_edid(struct kunit *test)
+{
+	struct dm_test_vbios_edid *fixture;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+
+	dm_test_setup_vbios_link(test, &link, &aconnector, &fixture);
+	fixture->fake_edid = NULL;
+	fixture->fake_edid_size = 0;
+
+	KUNIT_EXPECT_NULL(test,
+			  dm_helpers_read_vbios_hardcoded_edid(link, aconnector));
+}
+
+/**
+ * dm_test_read_vbios_edid_invalid_fake_edid - Test invalid fake EDID skip
+ * @test: The KUnit test context
+ */
+static void dm_test_read_vbios_edid_invalid_fake_edid(struct kunit *test)
+{
+	struct dm_test_vbios_edid *fixture;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_edid *dc_edid;
+	struct dc_link *link;
+
+	dm_test_setup_vbios_link(test, &link, &aconnector, &fixture);
+	dc_edid = kunit_kzalloc(test, sizeof(*dc_edid), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc_edid);
+	dm_test_fill_base_edid(dc_edid, false);
+	fixture->fake_edid = dc_edid->raw_edid;
+	fixture->fake_edid_size = EDID_LENGTH;
+
+	KUNIT_EXPECT_NULL(test,
+			  dm_helpers_read_vbios_hardcoded_edid(link, aconnector));
+}
+
+/**
+ * dm_test_read_vbios_edid_valid - Test valid fake EDID updates display size
+ * @test: The KUnit test context
+ */
+static void dm_test_read_vbios_edid_valid(struct kunit *test)
+{
+	struct dm_test_vbios_edid *fixture;
+	struct amdgpu_dm_connector *aconnector;
+	const struct drm_edid *edid;
+	struct dc_edid *dc_edid;
+	struct dc_link *link;
+
+	dm_test_setup_vbios_link(test, &link, &aconnector, &fixture);
+	dc_edid = kunit_kzalloc(test, sizeof(*dc_edid), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc_edid);
+	dm_test_fill_base_edid(dc_edid, true);
+	fixture->fake_edid = dc_edid->raw_edid;
+	fixture->fake_edid_size = EDID_LENGTH;
+	fixture->width_mm = 301;
+	fixture->height_mm = 201;
+
+	edid = dm_helpers_read_vbios_hardcoded_edid(link, aconnector);
+
+	KUNIT_ASSERT_NOT_NULL(test, edid);
+	KUNIT_EXPECT_EQ(test, aconnector->base.display_info.width_mm, 301U);
+	KUNIT_EXPECT_EQ(test, aconnector->base.display_info.height_mm, 201U);
+	drm_edid_free(edid);
+}
+
+/* Tests for dm_is_freesync_pcon_whitelist() */
+
+/**
+ * dm_test_freesync_pcon_whitelist_all_known - Test all known Freesync Pcon whitelist entries
+ * @test: The KUnit test context
+ *
+ * Iterates over the driver's whitelist table directly so that any ID added
+ * to dm_freesync_pcon_whitelist[] is automatically covered by this test.
+ */
+static void dm_test_freesync_pcon_whitelist_all_known(struct kunit *test)
+{
+	u32 i;
+
+	for (i = 0; i < dm_freesync_pcon_whitelist_count(); i++)
+		KUNIT_EXPECT_TRUE(test,
+				  dm_is_freesync_pcon_whitelist(dm_freesync_pcon_whitelist[i]));
+}
+
+/**
+ * dm_test_freesync_pcon_whitelist_not_in_list - Test Freesync pcon whitelist not in list
+ * @test: The KUnit test context
+ */
+static void dm_test_freesync_pcon_whitelist_not_in_list(struct kunit *test)
+{
+	/* 0xFFFFFF is not a known whitelist device */
+	KUNIT_EXPECT_FALSE(test, dm_is_freesync_pcon_whitelist(0xFFFFFF));
+}
+
+/**
+ * dm_test_freesync_pcon_whitelist_zero - Test Freesync pcon whitelist zero
+ * @test: The KUnit test context
+ */
+static void dm_test_freesync_pcon_whitelist_zero(struct kunit *test)
+{
+	KUNIT_EXPECT_FALSE(test, dm_is_freesync_pcon_whitelist(0));
+}
+
+/* Tests for populate_hdmi_info_from_connector() */
+
+/**
+ * dm_test_populate_hdmi_scdc_present_true - Test Populate hdmi scdc present true
+ * @test: The KUnit test context
+ */
+static void dm_test_populate_hdmi_scdc_present_true(struct kunit *test)
+{
+	struct drm_hdmi_info *hdmi;
+	struct dc_edid_caps *caps;
+
+	hdmi = kunit_kzalloc(test, sizeof(*hdmi), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, hdmi);
+	caps = kunit_kzalloc(test, sizeof(*caps), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, caps);
+
+	hdmi->scdc.supported = true;
+
+	populate_hdmi_info_from_connector(true, hdmi, caps);
+
+	KUNIT_EXPECT_TRUE(test, caps->scdc_present);
+}
+
+/**
+ * dm_test_populate_hdmi_scdc_present_false - Test Populate hdmi scdc present false
+ * @test: The KUnit test context
+ */
+static void dm_test_populate_hdmi_scdc_present_false(struct kunit *test)
+{
+	struct drm_hdmi_info *hdmi;
+	struct dc_edid_caps *caps;
+
+	hdmi = kunit_kzalloc(test, sizeof(*hdmi), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, hdmi);
+	caps = kunit_kzalloc(test, sizeof(*caps), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, caps);
+
+	hdmi->scdc.supported = false;
+	caps->scdc_present = true; /* pre-set to confirm it gets cleared */
+
+	populate_hdmi_info_from_connector(true, hdmi, caps);
+
+	KUNIT_EXPECT_FALSE(test, caps->scdc_present);
+}
+
+/**
+ * dm_test_populate_hdmi_frl_dsc_10bpc - Test HDMI FRL DSC 10 bpc caps
+ * @test: The KUnit test context
+ */
+static void dm_test_populate_hdmi_frl_dsc_10bpc(struct kunit *test)
+{
+	struct drm_hdmi_info *hdmi;
+	struct dc_edid_caps *caps;
+
+	hdmi = kunit_kzalloc(test, sizeof(*hdmi), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, hdmi);
+	caps = kunit_kzalloc(test, sizeof(*caps), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, caps);
+
+	hdmi->max_lanes = 4;
+	hdmi->max_frl_rate_per_lane = 12;
+	hdmi->dsc_cap.v_1p2 = true;
+	hdmi->dsc_cap.bpc_supported = 10;
+	hdmi->dsc_cap.all_bpp = true;
+	hdmi->dsc_cap.native_420 = true;
+	hdmi->dsc_cap.max_slices = 8;
+	hdmi->dsc_cap.clk_per_slice = 400;
+	hdmi->dsc_cap.max_lanes = 4;
+	hdmi->dsc_cap.max_frl_rate_per_lane = 10;
+	hdmi->dsc_cap.total_chunk_kbytes = 7;
+
+	populate_hdmi_info_from_connector(true, hdmi, caps);
+
+	KUNIT_EXPECT_EQ(test, caps->max_frl_rate, 6);
+	KUNIT_EXPECT_TRUE(test, caps->frl_dsc_support);
+	KUNIT_EXPECT_TRUE(test, caps->frl_dsc_10bpc);
+	KUNIT_EXPECT_FALSE(test, caps->frl_dsc_12bpc);
+	KUNIT_EXPECT_TRUE(test, caps->frl_dsc_all_bpp);
+	KUNIT_EXPECT_TRUE(test, caps->frl_dsc_native_420);
+	KUNIT_EXPECT_EQ(test, caps->frl_dsc_max_slices, 5);
+	KUNIT_EXPECT_EQ(test, caps->frl_dsc_max_frl_rate, 5);
+	KUNIT_EXPECT_EQ(test, caps->frl_dsc_total_chunk_kbytes, 7);
+}
+
+/**
+ * dm_test_populate_hdmi_frl_dsc_12bpc - Test HDMI FRL DSC 12 bpc caps
+ * @test: The KUnit test context
+ */
+static void dm_test_populate_hdmi_frl_dsc_12bpc(struct kunit *test)
+{
+	struct drm_hdmi_info *hdmi;
+	struct dc_edid_caps *caps;
+
+	hdmi = kunit_kzalloc(test, sizeof(*hdmi), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, hdmi);
+	caps = kunit_kzalloc(test, sizeof(*caps), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, caps);
+
+	hdmi->max_lanes = 3;
+	hdmi->max_frl_rate_per_lane = 6;
+	hdmi->dsc_cap.v_1p2 = true;
+	hdmi->dsc_cap.bpc_supported = 12;
+	hdmi->dsc_cap.max_slices = 16;
+	hdmi->dsc_cap.clk_per_slice = 400;
+	hdmi->dsc_cap.max_lanes = 3;
+	hdmi->dsc_cap.max_frl_rate_per_lane = 3;
+
+	populate_hdmi_info_from_connector(true, hdmi, caps);
+
+	KUNIT_EXPECT_EQ(test, caps->max_frl_rate, 2);
+	KUNIT_EXPECT_TRUE(test, caps->frl_dsc_support);
+	KUNIT_EXPECT_FALSE(test, caps->frl_dsc_10bpc);
+	KUNIT_EXPECT_TRUE(test, caps->frl_dsc_12bpc);
+	KUNIT_EXPECT_EQ(test, caps->frl_dsc_max_slices, 7);
+	KUNIT_EXPECT_EQ(test, caps->frl_dsc_max_frl_rate, 1);
+}
+
+/**
+ * dm_test_populate_hdmi_frl_dsc_unknown_values - Test HDMI FRL DSC unknown values
+ * @test: The KUnit test context
+ */
+static void dm_test_populate_hdmi_frl_dsc_unknown_values(struct kunit *test)
+{
+	struct drm_hdmi_info *hdmi;
+	struct dc_edid_caps *caps;
+
+	hdmi = kunit_kzalloc(test, sizeof(*hdmi), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, hdmi);
+	caps = kunit_kzalloc(test, sizeof(*caps), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, caps);
+
+	hdmi->max_lanes = 2;
+	hdmi->max_frl_rate_per_lane = 3;
+	hdmi->dsc_cap.v_1p2 = true;
+	hdmi->dsc_cap.bpc_supported = 8;
+	hdmi->dsc_cap.max_slices = 3;
+	hdmi->dsc_cap.clk_per_slice = 340;
+	hdmi->dsc_cap.max_lanes = 2;
+	hdmi->dsc_cap.max_frl_rate_per_lane = 12;
+
+	populate_hdmi_info_from_connector(true, hdmi, caps);
+
+	KUNIT_EXPECT_EQ(test, caps->max_frl_rate, 0);
+	KUNIT_EXPECT_TRUE(test, caps->frl_dsc_support);
+	KUNIT_EXPECT_FALSE(test, caps->frl_dsc_10bpc);
+	KUNIT_EXPECT_FALSE(test, caps->frl_dsc_12bpc);
+	KUNIT_EXPECT_EQ(test, caps->frl_dsc_max_slices, 0);
+	KUNIT_EXPECT_EQ(test, caps->frl_dsc_max_frl_rate, 0);
+}
+
+/* Tests for dm_get_adaptive_sync_support_type() */
+
+/**
+ * dm_test_adaptive_sync_type_none_default - Test Adaptive sync type none default
+ * @test: The KUnit test context
+ */
+static void dm_test_adaptive_sync_type_none_default(struct kunit *test)
+{
+	struct dc_link *link;
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	/* dongle_type = 0 (DISPLAY_DONGLE_NONE) → default case → TYPE_NONE */
+	KUNIT_EXPECT_EQ(test,
+			(int)dm_get_adaptive_sync_support_type(link),
+			(int)ADAPTIVE_SYNC_TYPE_NONE);
+}
+
+/**
+ * dm_test_adaptive_sync_type_converter_no_conditions - Converter without caps
+ * @test: The KUnit test context
+ */
+static void dm_test_adaptive_sync_type_converter_no_conditions(struct kunit *test)
+{
+	struct dc_link *link;
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	/* HDMI converter but no adaptive sync cap → still NONE */
+	link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_HDMI_CONVERTER;
+
+	KUNIT_EXPECT_EQ(test,
+			(int)dm_get_adaptive_sync_support_type(link),
+			(int)ADAPTIVE_SYNC_TYPE_NONE);
+}
+
+/**
+ * dm_test_adaptive_sync_type_converter_partial_conditions - Partial caps
+ * @test: The KUnit test context
+ */
+static void dm_test_adaptive_sync_type_converter_partial_conditions(struct kunit *test)
+{
+	struct dc_link *link;
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	/* Cap set and whitelist ID, but allow_invalid_MSA_timing_param = false */
+	link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_HDMI_CONVERTER;
+	link->dpcd_caps.adaptive_sync_caps.dp_adap_sync_caps.bits.ADAPTIVE_SYNC_SDP_SUPPORT = 1;
+	link->dpcd_caps.allow_invalid_MSA_timing_param = false;
+	link->dpcd_caps.branch_dev_id = DP_BRANCH_DEVICE_ID_0060AD;
+
+	KUNIT_EXPECT_EQ(test,
+			(int)dm_get_adaptive_sync_support_type(link),
+			(int)ADAPTIVE_SYNC_TYPE_NONE);
+}
+
+/**
+ * dm_test_adaptive_sync_type_pcon_whitelist - Test Adaptive sync type pcon whitelist
+ * @test: The KUnit test context
+ */
+static void dm_test_adaptive_sync_type_pcon_whitelist(struct kunit *test)
+{
+	struct dc_link *link;
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	/* All conditions met → FREESYNC_TYPE_PCON_IN_WHITELIST */
+	link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_HDMI_CONVERTER;
+	link->dpcd_caps.adaptive_sync_caps.dp_adap_sync_caps.bits.ADAPTIVE_SYNC_SDP_SUPPORT = 1;
+	link->dpcd_caps.allow_invalid_MSA_timing_param = true;
+	link->dpcd_caps.branch_dev_id = DP_BRANCH_DEVICE_ID_0060AD;
+
+	KUNIT_EXPECT_EQ(test,
+			(int)dm_get_adaptive_sync_support_type(link),
+			(int)FREESYNC_TYPE_PCON_IN_WHITELIST);
+}
+
+/**
+ * dm_test_adaptive_sync_type_converter_nonwhitelist - Converter not whitelisted
+ * @test: The KUnit test context
+ */
+static void dm_test_adaptive_sync_type_converter_nonwhitelist(struct kunit *test)
+{
+	struct dc_link *link;
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	/* All conditions met but branch_dev_id not in whitelist → NONE */
+	link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_HDMI_CONVERTER;
+	link->dpcd_caps.adaptive_sync_caps.dp_adap_sync_caps.bits.ADAPTIVE_SYNC_SDP_SUPPORT = 1;
+	link->dpcd_caps.allow_invalid_MSA_timing_param = true;
+	link->dpcd_caps.branch_dev_id = 0xFFFFFF;
+
+	KUNIT_EXPECT_EQ(test,
+			(int)dm_get_adaptive_sync_support_type(link),
+			(int)ADAPTIVE_SYNC_TYPE_NONE);
+}
+
+/* Tests for dm_helpers_is_fullscreen() and dm_helpers_is_hdr_on() */
+
+/**
+ * dm_test_helpers_is_fullscreen_returns_false - Test Helpers is fullscreen returns false
+ * @test: The KUnit test context
+ */
+static void dm_test_helpers_is_fullscreen_returns_false(struct kunit *test)
+{
+	/* Stub — always returns false */
+	KUNIT_EXPECT_FALSE(test, dm_helpers_is_fullscreen(NULL, NULL));
+}
+
+/**
+ * dm_test_helpers_is_hdr_on_returns_false - Test Helpers is hdr on returns false
+ * @test: The KUnit test context
+ */
+static void dm_test_helpers_is_hdr_on_returns_false(struct kunit *test)
+{
+	/* Stub — always returns false */
+	KUNIT_EXPECT_FALSE(test, dm_helpers_is_hdr_on(NULL, NULL));
+}
+
+/* Tests for get_max_frl_rate() */
+
+/**
+ * dm_test_get_max_frl_rate_3lanes_3gbps - Test Get max frl rate 3lanes 3gbps
+ * @test: The KUnit test context
+ */
+static void dm_test_get_max_frl_rate_3lanes_3gbps(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, get_max_frl_rate(3, 3), 1);
+}
+
+/**
+ * dm_test_get_max_frl_rate_3lanes_6gbps - Test Get max frl rate 3lanes 6gbps
+ * @test: The KUnit test context
+ */
+static void dm_test_get_max_frl_rate_3lanes_6gbps(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, get_max_frl_rate(3, 6), 2);
+}
+
+/**
+ * dm_test_get_max_frl_rate_4lanes_6gbps - Test Get max frl rate 4lanes 6gbps
+ * @test: The KUnit test context
+ */
+static void dm_test_get_max_frl_rate_4lanes_6gbps(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, get_max_frl_rate(4, 6), 3);
+}
+
+/**
+ * dm_test_get_max_frl_rate_4lanes_8gbps - Test Get max frl rate 4lanes 8gbps
+ * @test: The KUnit test context
+ */
+static void dm_test_get_max_frl_rate_4lanes_8gbps(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, get_max_frl_rate(4, 8), 4);
+}
+
+/**
+ * dm_test_get_max_frl_rate_4lanes_10gbps - Test Get max frl rate 4lanes 10gbps
+ * @test: The KUnit test context
+ */
+static void dm_test_get_max_frl_rate_4lanes_10gbps(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, get_max_frl_rate(4, 10), 5);
+}
+
+/**
+ * dm_test_get_max_frl_rate_4lanes_12gbps - Test Get max frl rate 4lanes 12gbps
+ * @test: The KUnit test context
+ */
+static void dm_test_get_max_frl_rate_4lanes_12gbps(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, get_max_frl_rate(4, 12), 6);
+}
+
+/**
+ * dm_test_get_max_frl_rate_unknown - Test Get max frl rate unknown
+ * @test: The KUnit test context
+ */
+static void dm_test_get_max_frl_rate_unknown(struct kunit *test)
+{
+	/* Unknown lane/rate combination → 0 */
+	KUNIT_EXPECT_EQ(test, get_max_frl_rate(2, 3), 0);
+}
+
+/* Tests for dm_dtn_log_begin() / dm_dtn_log_append_v() / dm_dtn_log_end() */
+
+/**
+ * dm_test_dtn_log_buffer_accumulates - Test DTN log buffer accumulation
+ * @test: The KUnit test context
+ */
+static void dm_test_dtn_log_buffer_accumulates(struct kunit *test)
+{
+	struct dc_log_buffer_ctx log_ctx = {0};
+
+	dm_dtn_log_begin(NULL, &log_ctx);
+	dm_dtn_log_append_v(NULL, &log_ctx, "x=%d\n", 7);
+	dm_dtn_log_end(NULL, &log_ctx);
+
+	KUNIT_ASSERT_NOT_NULL(test, log_ctx.buf);
+	KUNIT_EXPECT_STREQ(test, log_ctx.buf, "[dtn begin]\nx=7\n[dtn end]\n");
+	KUNIT_EXPECT_EQ(test, log_ctx.pos, strlen("[dtn begin]\nx=7\n[dtn end]\n"));
+
+	kvfree(log_ctx.buf);
+}
+
+/**
+ * dm_test_dtn_log_null_ctx_no_crash - Test DTN log helpers with NULL log buffer
+ * @test: The KUnit test context
+ */
+static void dm_test_dtn_log_null_ctx_no_crash(struct kunit *test)
+{
+	/* NULL log_ctx redirects to dmesg and must not dereference a buffer */
+	dm_dtn_log_begin(NULL, NULL);
+	dm_dtn_log_append_v(NULL, NULL, "value %d\n", 1);
+	dm_dtn_log_end(NULL, NULL);
+
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/* Tests for dm_helpers_dp_read_dpcd() / dm_helpers_dp_write_dpcd() */
+
+/**
+ * dm_test_dp_read_dpcd_null_priv - Test DPCD read returns false without connector
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_read_dpcd_null_priv(struct kunit *test)
+{
+	struct dc_link *link;
+	uint8_t data = 0;
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	/* link->priv (aconnector) is NULL → early return false */
+	KUNIT_EXPECT_FALSE(test,
+			   dm_helpers_dp_read_dpcd(NULL, link, 0, &data, sizeof(data)));
+}
+
+/**
+ * dm_test_dp_write_dpcd_null_priv - Test DPCD write returns false without connector
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_write_dpcd_null_priv(struct kunit *test)
+{
+	struct dc_link *link;
+	uint8_t data = 0;
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	/* link->priv (aconnector) is NULL → early return false */
+	KUNIT_EXPECT_FALSE(test,
+			   dm_helpers_dp_write_dpcd(NULL, link, 0, &data, sizeof(data)));
+}
+
+/*
+ * Stub AUX transfer that ACKs every transaction (zero-filling reads), so
+ * drm_dp_dpcd_read()/drm_dp_dpcd_write() report the full transfer size.
+ */
+static ssize_t dm_test_dpcd_ack_transfer(struct drm_dp_aux *aux,
+					 struct drm_dp_aux_msg *msg)
+{
+	if ((msg->request & ~DP_AUX_I2C_MOT) == DP_AUX_NATIVE_READ)
+		memset(msg->buffer, 0, msg->size);
+	msg->reply = DP_AUX_NATIVE_REPLY_ACK;
+	return msg->size;
+}
+
+/*
+ * Wire a connector-backed link with a working AUX channel so the DPCD
+ * read/write helpers can complete a real transaction.
+ */
+static struct dc_link *dm_test_dpcd_link(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_device *adev;
+	struct dc_link *link;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	link = dm_kunit_alloc_link(test);
+	aconnector = dm_kunit_alloc_connector(test, adev, NULL);
+
+	aconnector->dm_dp_aux.aux.drm_dev = &adev->ddev;
+	aconnector->dm_dp_aux.aux.transfer = dm_test_dpcd_ack_transfer;
+	drm_dp_aux_init(&aconnector->dm_dp_aux.aux);
+
+	link->priv = aconnector;
+
+	return link;
+}
+
+/**
+ * dm_test_dp_read_dpcd_success - Test DPCD read returns true on ACKed transfer
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_read_dpcd_success(struct kunit *test)
+{
+	struct dc_link *link = dm_test_dpcd_link(test);
+	uint8_t data = 0;
+
+	KUNIT_EXPECT_TRUE(test,
+			  dm_helpers_dp_read_dpcd(NULL, link, 0, &data, sizeof(data)));
+}
+
+/**
+ * dm_test_dp_write_dpcd_success - Test DPCD write returns true on ACKed transfer
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_write_dpcd_success(struct kunit *test)
+{
+	struct dc_link *link = dm_test_dpcd_link(test);
+	uint8_t data = 0;
+
+	KUNIT_EXPECT_TRUE(test,
+			  dm_helpers_dp_write_dpcd(NULL, link, 0, &data, sizeof(data)));
+}
+
+/* Tests for dm_helpers_execute_fused_io() */
+
+/**
+ * dm_test_execute_fused_io_null_dmub_srv - Test fused IO fails without DMUB service
+ * @test: The KUnit test context
+ */
+static void dm_test_execute_fused_io_null_dmub_srv(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	union dmub_rb_cmd *commands;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	mutex_init(&adev->dm.dpia_aux_lock);
+	spin_lock_init(&adev->dm.dmub_lock);
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+	commands = kunit_kzalloc(test, sizeof(*commands), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, commands);
+
+	ctx->driver_context = adev;
+	link->ctx = ctx;
+	commands[0].fused_io.request.u.aux.ddc_line = 0;
+
+	KUNIT_EXPECT_FALSE(test, dm_helpers_execute_fused_io(ctx, link, commands, 1, 1));
+}
+
+struct dm_test_synaptics_aux {
+	struct drm_dp_aux aux;
+	u32 fail_address;
+	u32 last_dpcd_write_address;
+	u8 rc_result;
+	u8 dpcd_read_value;
+	u8 dpcd_write_value;
+	u8 last_rc_data[16];
+	u8 read_rc_data[16];
+	u8 last_rc_command;
+	u8 rc_commands[32];
+	u8 dsc_enable_values[8];
+	u32 last_rc_offset;
+	u32 last_rc_length;
+	unsigned int rc_data_writes;
+	unsigned int rc_data_reads;
+	unsigned int rc_command_reads;
+	unsigned int rc_result_reads;
+	unsigned int rc_command_count;
+	unsigned int downspread_reads;
+	unsigned int downspread_writes;
+	unsigned int dsc_enable_writes;
+};
+
+static ssize_t dm_test_synaptics_aux_transfer(struct drm_dp_aux *aux,
+					      struct drm_dp_aux_msg *msg)
+{
+	struct dm_test_synaptics_aux *fixture;
+	u8 request;
+	u8 *buffer;
+	size_t copy_size;
+	unsigned int index;
+
+	fixture = container_of(aux, struct dm_test_synaptics_aux, aux);
+	request = msg->request & ~DP_AUX_I2C_MOT;
+	buffer = msg->buffer;
+
+	if (fixture->fail_address == msg->address)
+		return -EIO;
+
+	if (request == DP_AUX_NATIVE_WRITE) {
+		switch (msg->address) {
+		case DP_DOWNSPREAD_CTRL:
+			fixture->last_dpcd_write_address = msg->address;
+			if (msg->size)
+				fixture->dpcd_write_value = buffer[0];
+			fixture->downspread_writes++;
+			break;
+		case SYNAPTICS_RC_DATA:
+			copy_size = min_t(size_t, msg->size, sizeof(fixture->last_rc_data));
+			memset(fixture->last_rc_data, 0, sizeof(fixture->last_rc_data));
+			memcpy(fixture->last_rc_data, buffer, copy_size);
+			fixture->rc_data_writes++;
+			break;
+		case SYNAPTICS_RC_OFFSET:
+			if (msg->size >= 4)
+				fixture->last_rc_offset = buffer[0] | buffer[1] << 8 |
+							  buffer[2] << 16 | buffer[3] << 24;
+			break;
+		case SYNAPTICS_RC_LENGTH:
+			if (msg->size >= 2)
+				fixture->last_rc_length = buffer[0] | buffer[1] << 8;
+			break;
+		case SYNAPTICS_RC_COMMAND:
+			fixture->last_rc_command = buffer[0];
+			if (fixture->rc_command_count < ARRAY_SIZE(fixture->rc_commands)) {
+				fixture->rc_commands[fixture->rc_command_count] = buffer[0] & 0x7f;
+				fixture->rc_command_count++;
+			}
+			break;
+		case DP_DSC_ENABLE:
+			if (fixture->dsc_enable_writes < ARRAY_SIZE(fixture->dsc_enable_values)) {
+				fixture->dsc_enable_values[fixture->dsc_enable_writes] = buffer[0];
+				fixture->dsc_enable_writes++;
+			}
+			break;
+		}
+		msg->reply = DP_AUX_NATIVE_REPLY_ACK;
+		return msg->size;
+	}
+
+	if (request == DP_AUX_NATIVE_READ) {
+		memset(buffer, 0, msg->size);
+		switch (msg->address) {
+		case DP_DOWNSPREAD_CTRL:
+			if (msg->size)
+				buffer[0] = fixture->dpcd_read_value;
+			fixture->downspread_reads++;
+			break;
+		case SYNAPTICS_RC_COMMAND:
+			if (msg->size)
+				buffer[0] = fixture->last_rc_command & 0x7f;
+			fixture->rc_command_reads++;
+			break;
+		case SYNAPTICS_RC_RESULT:
+			if (msg->size)
+				buffer[0] = fixture->rc_result;
+			fixture->rc_result_reads++;
+			break;
+		case SYNAPTICS_RC_DATA:
+			copy_size = min_t(size_t, msg->size, sizeof(fixture->read_rc_data));
+			for (index = 0; index < copy_size; index++)
+				buffer[index] = fixture->read_rc_data[index];
+			fixture->rc_data_reads++;
+			break;
+		}
+		msg->reply = DP_AUX_NATIVE_REPLY_ACK;
+		return msg->size;
+	}
+
+	msg->reply = DP_AUX_NATIVE_REPLY_ACK;
+	return msg->size;
+}
+
+static struct dm_test_synaptics_aux *dm_test_current_aux_recorder;
+
+static ssize_t dm_test_current_aux_transfer(struct drm_dp_aux *aux,
+					    struct drm_dp_aux_msg *msg)
+{
+	return dm_test_synaptics_aux_transfer(&dm_test_current_aux_recorder->aux, msg);
+}
+
+static struct dm_test_synaptics_aux *dm_test_alloc_synaptics_aux_with_dev(struct kunit *test,
+								 struct drm_device *drm_dev)
+{
+	struct dm_test_synaptics_aux *fixture;
+	unsigned int index;
+
+	fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture);
+
+	for (index = 0; index < ARRAY_SIZE(fixture->read_rc_data); index++)
+		fixture->read_rc_data[index] = 0x03;
+
+	fixture->rc_result = 0;
+	fixture->aux.drm_dev = drm_dev;
+	fixture->aux.transfer = dm_test_synaptics_aux_transfer;
+	drm_dp_aux_init(&fixture->aux);
+
+	return fixture;
+}
+
+static struct dm_test_synaptics_aux *dm_test_alloc_synaptics_aux(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	return dm_test_alloc_synaptics_aux_with_dev(test, &adev->ddev);
+}
+
+static void dm_test_expect_synaptics_commands(struct kunit *test,
+					      struct dm_test_synaptics_aux *fixture,
+					      const u8 *expected_commands,
+					      unsigned int expected_count)
+{
+	unsigned int index;
+
+	KUNIT_ASSERT_EQ(test, fixture->rc_command_count, expected_count);
+
+	for (index = 0; index < expected_count; index++)
+		KUNIT_EXPECT_EQ(test, fixture->rc_commands[index], expected_commands[index]);
+}
+
+static void dm_test_setup_synaptics_stream(struct dc_stream_state *stream,
+						  struct dc_link *link)
+{
+	stream->link = link;
+	stream->signal = SIGNAL_TYPE_DISPLAY_PORT_MST;
+	link->dpcd_caps.branch_dev_id = DP_BRANCH_DEVICE_ID_90CC24;
+	link->dpcd_caps.dpcd_rev.raw = DP_DPCD_REV_14;
+	link->dpcd_caps.sink_count.bits.SINK_COUNT = 2;
+	memcpy(link->dpcd_caps.branch_dev_name, "SYNA", 4);
+}
+
+/**
+ * dm_test_execute_synaptics_rc_command_write_success - Test RC write success
+ * @test: The KUnit test context
+ */
+static void dm_test_execute_synaptics_rc_command_write_success(struct kunit *test)
+{
+	struct dm_test_synaptics_aux *fixture;
+	u8 data[5] = { 'P', 'R', 'I', 'U', 'S' };
+
+	fixture = dm_test_alloc_synaptics_aux(test);
+
+	KUNIT_EXPECT_TRUE(test, execute_synaptics_rc_command(&fixture->aux, true,
+							     0x01, sizeof(data), 0x123456,
+							     data));
+	KUNIT_EXPECT_EQ(test, memcmp(fixture->last_rc_data, data, sizeof(data)), 0);
+	KUNIT_EXPECT_EQ(test, fixture->last_rc_offset, 0x123456U);
+	KUNIT_EXPECT_EQ(test, fixture->last_rc_length, (u32)sizeof(data));
+	KUNIT_EXPECT_EQ(test, fixture->last_rc_command, (u8)0x81);
+	KUNIT_EXPECT_EQ(test, fixture->rc_command_reads, 1U);
+	KUNIT_EXPECT_EQ(test, fixture->rc_result_reads, 1U);
+}
+
+/**
+ * dm_test_execute_synaptics_rc_command_read_success - Test RC read success
+ * @test: The KUnit test context
+ */
+static void dm_test_execute_synaptics_rc_command_read_success(struct kunit *test)
+{
+	struct dm_test_synaptics_aux *fixture;
+	u8 data[4] = { 0 };
+	u8 expected[4] = { 0xa5, 0x5a, 0xc3, 0x3c };
+
+	fixture = dm_test_alloc_synaptics_aux(test);
+	memcpy(fixture->read_rc_data, expected, sizeof(expected));
+
+	KUNIT_EXPECT_TRUE(test, execute_synaptics_rc_command(&fixture->aux, false,
+							     0x31, sizeof(data), 0x220998,
+							     data));
+	KUNIT_EXPECT_EQ(test, memcmp(data, expected, sizeof(expected)), 0);
+	KUNIT_EXPECT_EQ(test, fixture->rc_data_writes, 0U);
+	KUNIT_EXPECT_EQ(test, fixture->rc_data_reads, 1U);
+	KUNIT_EXPECT_EQ(test, fixture->last_rc_offset, 0x220998U);
+	KUNIT_EXPECT_EQ(test, fixture->last_rc_length, (u32)sizeof(data));
+}
+
+/**
+ * dm_test_execute_synaptics_rc_command_write_fail - Test RC write failure
+ * @test: The KUnit test context
+ */
+static void dm_test_execute_synaptics_rc_command_write_fail(struct kunit *test)
+{
+	struct dm_test_synaptics_aux *fixture;
+	u8 data = 0;
+
+	fixture = dm_test_alloc_synaptics_aux(test);
+	fixture->fail_address = SYNAPTICS_RC_LENGTH;
+
+	KUNIT_EXPECT_FALSE(test, execute_synaptics_rc_command(&fixture->aux, true,
+							      0x01, sizeof(data), 0, &data));
+	KUNIT_EXPECT_EQ(test, fixture->rc_command_count, 0U);
+}
+
+/**
+ * dm_test_apply_synaptics_fifo_reset_wa_full - Test full FIFO reset sequence
+ * @test: The KUnit test context
+ */
+static void dm_test_apply_synaptics_fifo_reset_wa_full(struct kunit *test)
+{
+	static const u8 expected_commands[] = {
+		0x01, 0x31, 0x21, 0x31, 0x21, 0x31, 0x21,
+		0x31, 0x21, 0x31, 0x31, 0x21, 0x02,
+	};
+	struct dm_test_synaptics_aux *fixture;
+
+	fixture = dm_test_alloc_synaptics_aux(test);
+
+	apply_synaptics_fifo_reset_wa(&fixture->aux);
+
+	dm_test_expect_synaptics_commands(test, fixture, expected_commands,
+					  ARRAY_SIZE(expected_commands));
+	KUNIT_EXPECT_EQ(test, fixture->rc_result_reads, (unsigned int)ARRAY_SIZE(expected_commands));
+}
+
+/**
+ * dm_test_apply_synaptics_fifo_reset_wa_first_fail - Test FIFO reset early exit
+ * @test: The KUnit test context
+ */
+static void dm_test_apply_synaptics_fifo_reset_wa_first_fail(struct kunit *test)
+{
+	struct dm_test_synaptics_aux *fixture;
+
+	fixture = dm_test_alloc_synaptics_aux(test);
+	fixture->rc_result = 0xff;
+
+	apply_synaptics_fifo_reset_wa(&fixture->aux);
+
+	KUNIT_EXPECT_EQ(test, fixture->rc_command_count, 1U);
+	KUNIT_EXPECT_EQ(test, fixture->rc_commands[0], (u8)0x01);
+}
+
+static void dm_test_write_dsc_enable_synaptics(struct kunit *test,
+					       bool link_active,
+					       bool enable,
+					       bool synaptics_branch,
+					       unsigned int expected_dsc_writes,
+					       unsigned int expected_rc_commands)
+{
+	struct dm_test_synaptics_aux *fixture;
+	struct dc_stream_state *stream;
+	struct dc_link *link;
+	u8 ret;
+
+	fixture = dm_test_alloc_synaptics_aux(test);
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+	link = dm_kunit_alloc_link(test);
+
+	dm_test_setup_synaptics_stream(stream, link);
+	link->link_status.link_active = link_active;
+	if (!synaptics_branch)
+		memcpy(link->dpcd_caps.branch_dev_name, "ABCD", 4);
+
+	ret = write_dsc_enable_synaptics_non_virtual_dpcd_mst(&fixture->aux, stream, enable);
+
+	KUNIT_EXPECT_EQ(test, ret, expected_dsc_writes ? 1 : 0);
+	KUNIT_EXPECT_EQ(test, fixture->dsc_enable_writes, expected_dsc_writes);
+	KUNIT_EXPECT_EQ(test, fixture->rc_command_count, expected_rc_commands);
+	if (expected_dsc_writes)
+		KUNIT_EXPECT_EQ(test, fixture->dsc_enable_values[0], enable ? 1 : 0);
+}
+
+/**
+ * dm_test_write_dsc_enable_synaptics_enable_inactive - Test enable plus FIFO reset
+ * @test: The KUnit test context
+ */
+static void dm_test_write_dsc_enable_synaptics_enable_inactive(struct kunit *test)
+{
+	dm_test_write_dsc_enable_synaptics(test, false, true, true, 1, 13);
+}
+
+/**
+ * dm_test_write_dsc_enable_synaptics_enable_active - Test enable skips FIFO reset
+ * @test: The KUnit test context
+ */
+static void dm_test_write_dsc_enable_synaptics_enable_active(struct kunit *test)
+{
+	dm_test_write_dsc_enable_synaptics(test, true, true, true, 1, 0);
+}
+
+/**
+ * dm_test_write_dsc_enable_synaptics_disable_inactive - Test inactive disable writes DPCD
+ * @test: The KUnit test context
+ */
+static void dm_test_write_dsc_enable_synaptics_disable_inactive(struct kunit *test)
+{
+	dm_test_write_dsc_enable_synaptics(test, false, false, true, 1, 0);
+}
+
+/**
+ * dm_test_write_dsc_enable_synaptics_disable_active - Test active disable skips DPCD
+ * @test: The KUnit test context
+ */
+static void dm_test_write_dsc_enable_synaptics_disable_active(struct kunit *test)
+{
+	dm_test_write_dsc_enable_synaptics(test, true, false, true, 0, 0);
+}
+
+/**
+ * dm_test_write_dsc_enable_synaptics_enable_non_synaptics - Test non-Synaptics enable
+ * @test: The KUnit test context
+ */
+static void dm_test_write_dsc_enable_synaptics_enable_non_synaptics(struct kunit *test)
+{
+	dm_test_write_dsc_enable_synaptics(test, false, true, false, 1, 0);
+}
+
+/**
+ * dm_test_dp_write_dsc_enable_routes_synaptics - Test public DSC helper workaround route
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_write_dsc_enable_routes_synaptics(struct kunit *test)
+{
+	struct dm_test_synaptics_aux *fixture;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_stream_state *stream;
+	struct dc_link *link;
+
+	fixture = dm_test_alloc_synaptics_aux(test);
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+	link = dm_kunit_alloc_link(test);
+
+	dm_test_setup_synaptics_stream(stream, link);
+	stream->dm_stream_context = aconnector;
+	aconnector->dc_link = link;
+	aconnector->dsc_aux = &fixture->aux;
+
+	KUNIT_EXPECT_TRUE(test, dm_helpers_dp_write_dsc_enable(NULL, stream, true));
+	KUNIT_EXPECT_EQ(test, fixture->dsc_enable_writes, 1U);
+	KUNIT_EXPECT_EQ(test, fixture->dsc_enable_values[0], (u8)1);
+	KUNIT_EXPECT_EQ(test, fixture->rc_command_count, 13U);
+}
+
+/* Tests for dm_helpers_dp_mst_start_top_mgr() / dm_helpers_dp_mst_stop_top_mgr() */
+
+/**
+ * dm_test_mst_start_top_mgr_null_priv - Test MST start returns false without connector
+ * @test: The KUnit test context
+ */
+static void dm_test_mst_start_top_mgr_null_priv(struct kunit *test)
+{
+	struct dc_link *link;
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	KUNIT_EXPECT_FALSE(test, dm_helpers_dp_mst_start_top_mgr(NULL, link, false));
+}
+
+/**
+ * dm_test_mst_stop_top_mgr_null_priv - Test MST stop returns false without connector
+ * @test: The KUnit test context
+ */
+static void dm_test_mst_stop_top_mgr_null_priv(struct kunit *test)
+{
+	struct dc_link *link;
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	KUNIT_EXPECT_FALSE(test, dm_helpers_dp_mst_stop_top_mgr(NULL, link));
+}
+
+/**
+ * dm_test_mst_start_top_mgr_boot - Test MST start boot path on a connector-backed link
+ * @test: The KUnit test context
+ *
+ * Uses the DRM KUnit mock device to back the connector so the link is a
+ * realistic connector-backed link. The boot path short-circuits and returns
+ * true without touching the MST topology manager.
+ */
+static void dm_test_mst_start_top_mgr_boot(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_device *adev;
+	struct dc_link *link;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	link = dm_kunit_alloc_link(test);
+
+	aconnector = dm_kunit_alloc_connector(test, adev, NULL);
+
+	link->priv = aconnector;
+
+	KUNIT_EXPECT_TRUE(test, dm_helpers_dp_mst_start_top_mgr(NULL, link, true));
+}
+
+/* Tests for dm_helpers_dp_write_hblank_reduction() */
+
+/**
+ * dm_test_dp_write_hblank_reduction_false - Test hblank reduction stub returns false
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_write_hblank_reduction_false(struct kunit *test)
+{
+	KUNIT_EXPECT_FALSE(test, dm_helpers_dp_write_hblank_reduction(NULL, NULL));
+}
+
+/* Tests for get_dsc_max_slices() */
+
+/**
+ * dm_test_get_dsc_max_slices_1_340 - Test 1 slice at 340 MHz
+ * @test: The KUnit test context
+ */
+static void dm_test_get_dsc_max_slices_1_340(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, get_dsc_max_slices(1, 340), 1);
+}
+
+/**
+ * dm_test_get_dsc_max_slices_2_340 - Test 2 slices at 340 MHz
+ * @test: The KUnit test context
+ */
+static void dm_test_get_dsc_max_slices_2_340(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, get_dsc_max_slices(2, 340), 2);
+}
+
+/**
+ * dm_test_get_dsc_max_slices_4_340 - Test 4 slices at 340 MHz
+ * @test: The KUnit test context
+ */
+static void dm_test_get_dsc_max_slices_4_340(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, get_dsc_max_slices(4, 340), 3);
+}
+
+/**
+ * dm_test_get_dsc_max_slices_8_340 - Test 8 slices at 340 MHz
+ * @test: The KUnit test context
+ */
+static void dm_test_get_dsc_max_slices_8_340(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, get_dsc_max_slices(8, 340), 4);
+}
+
+/**
+ * dm_test_get_dsc_max_slices_8_400 - Test 8 slices at 400 MHz
+ * @test: The KUnit test context
+ */
+static void dm_test_get_dsc_max_slices_8_400(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, get_dsc_max_slices(8, 400), 5);
+}
+
+/**
+ * dm_test_get_dsc_max_slices_12_400 - Test 12 slices at 400 MHz
+ * @test: The KUnit test context
+ */
+static void dm_test_get_dsc_max_slices_12_400(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, get_dsc_max_slices(12, 400), 6);
+}
+
+/**
+ * dm_test_get_dsc_max_slices_16_400 - Test 16 slices at 400 MHz
+ * @test: The KUnit test context
+ */
+static void dm_test_get_dsc_max_slices_16_400(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, get_dsc_max_slices(16, 400), 7);
+}
+
+/**
+ * dm_test_get_dsc_max_slices_unknown - Test unknown combination returns 0
+ * @test: The KUnit test context
+ */
+static void dm_test_get_dsc_max_slices_unknown(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, get_dsc_max_slices(3, 340), 0);
+	KUNIT_EXPECT_EQ(test, get_dsc_max_slices(1, 400), 0);
+}
+
+/* Tests for dm_helpers_init_panel_settings() */
+
+/**
+ * dm_test_init_panel_settings_pps - Test panel power sequence settings init
+ * @test: The KUnit test context
+ */
+static void dm_test_init_panel_settings_pps(struct kunit *test)
+{
+	struct dc_panel_config panel_config = {0};
+	struct dc_sink *sink;
+
+	sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+
+	sink->edid_caps.panel_patch.extra_t3_ms = 100;
+	sink->edid_caps.panel_patch.extra_t7_ms = 200;
+	sink->edid_caps.panel_patch.extra_delay_backlight_off = 50;
+	sink->edid_caps.panel_patch.extra_t12_ms = 300;
+
+	dm_helpers_init_panel_settings(NULL, &panel_config, sink);
+
+	KUNIT_EXPECT_EQ(test, panel_config.pps.extra_t3_ms, 100U);
+	KUNIT_EXPECT_EQ(test, panel_config.pps.extra_t7_ms, 200U);
+	KUNIT_EXPECT_EQ(test, panel_config.pps.extra_delay_backlight_off, 50U);
+	KUNIT_EXPECT_EQ(test, panel_config.pps.extra_post_t7_ms, 0U);
+	KUNIT_EXPECT_EQ(test, panel_config.pps.extra_pre_t11_ms, 0U);
+	KUNIT_EXPECT_EQ(test, panel_config.pps.extra_t12_ms, 300U);
+	KUNIT_EXPECT_EQ(test, panel_config.pps.extra_post_OUI_ms, 0U);
+}
+
+/**
+ * dm_test_init_panel_settings_dsc - Test DSC defaults in panel settings init
+ * @test: The KUnit test context
+ */
+static void dm_test_init_panel_settings_dsc(struct kunit *test)
+{
+	struct dc_panel_config panel_config;
+	struct dc_sink *sink;
+
+	memset(&panel_config, 0xFF, sizeof(panel_config));
+
+	sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+
+	dm_helpers_init_panel_settings(NULL, &panel_config, sink);
+
+	KUNIT_EXPECT_FALSE(test, panel_config.dsc.disable_dsc_edp);
+	KUNIT_EXPECT_EQ(test, panel_config.dsc.force_dsc_edp_policy, 0U);
+}
+
+/* Tests for dm_helpers_override_panel_settings() */
+
+/**
+ * dm_test_override_panel_settings_debug_mask_disables_dsc - Test DSC mask
+ * @test: The KUnit test context
+ */
+static void dm_test_override_panel_settings_debug_mask_disables_dsc(struct kunit *test)
+{
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc *dc;
+	uint old_debug_mask;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	link = dm_kunit_alloc_link(test);
+	ctx->dc = dc;
+	link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+
+	old_debug_mask = dm_helpers_get_dc_debug_mask();
+	dm_helpers_set_dc_debug_mask(old_debug_mask | DC_DISABLE_DSC);
+	dm_helpers_override_panel_settings(ctx, link);
+	dm_helpers_set_dc_debug_mask(old_debug_mask);
+
+	KUNIT_EXPECT_TRUE(test, link->panel_config.dsc.disable_dsc_edp);
+}
+
+/**
+ * dm_test_override_panel_settings_second_edp_disables_psr - Test eDP index 1
+ * @test: The KUnit test context
+ */
+static void dm_test_override_panel_settings_second_edp_disables_psr(struct kunit *test)
+{
+	struct dc_context *ctx;
+	struct dc_link *first_link;
+	struct dc_link *second_link;
+	struct dc *dc;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	first_link = dm_kunit_alloc_link(test);
+	second_link = dm_kunit_alloc_link(test);
+
+	ctx->dc = dc;
+	dc->link_count = 2;
+	dc->links[0] = first_link;
+	dc->links[1] = second_link;
+	first_link->connector_signal = SIGNAL_TYPE_EDP;
+	second_link->connector_signal = SIGNAL_TYPE_EDP;
+
+	dm_helpers_override_panel_settings(ctx, second_link);
+
+	KUNIT_EXPECT_TRUE(test, second_link->panel_config.psr.disable_psr);
+	KUNIT_EXPECT_TRUE(test, second_link->panel_config.psr.disallow_psrsu);
+	KUNIT_EXPECT_TRUE(test, second_link->panel_config.psr.disallow_replay);
+}
+
+/* Tests for fill_dc_mst_payload_table_from_drm() */
+
+/**
+ * dm_test_fill_mst_payload_table_enable - Test payload table fill on enable
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_mst_payload_table_enable(struct kunit *test)
+{
+	struct dc_link *link;
+	struct drm_dp_mst_atomic_payload payload = {0};
+	struct dc_dp_mst_stream_allocation_table table = {0};
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	/* Pre-existing allocation in the link table */
+	link->mst_stream_alloc_table.stream_count = 1;
+	link->mst_stream_alloc_table.stream_allocations[0].vcp_id = 1;
+	link->mst_stream_alloc_table.stream_allocations[0].slot_count = 4;
+
+	/* New payload to add */
+	payload.vcpi = 2;
+	payload.time_slots = 8;
+
+	fill_dc_mst_payload_table_from_drm(link, true, &payload, &table);
+
+	/* Should contain both the pre-existing and new allocation */
+	KUNIT_EXPECT_EQ(test, table.stream_count, 2);
+	KUNIT_EXPECT_EQ(test, table.stream_allocations[0].vcp_id, 1);
+	KUNIT_EXPECT_EQ(test, table.stream_allocations[0].slot_count, 4);
+	KUNIT_EXPECT_EQ(test, table.stream_allocations[1].vcp_id, 2);
+	KUNIT_EXPECT_EQ(test, table.stream_allocations[1].slot_count, 8);
+}
+
+/**
+ * dm_test_fill_mst_payload_table_disable - Test payload table fill on disable
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_mst_payload_table_disable(struct kunit *test)
+{
+	struct dc_link *link;
+	struct drm_dp_mst_atomic_payload payload = {0};
+	struct dc_dp_mst_stream_allocation_table table = {0};
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	/* Two existing allocations in the link table */
+	link->mst_stream_alloc_table.stream_count = 2;
+	link->mst_stream_alloc_table.stream_allocations[0].vcp_id = 1;
+	link->mst_stream_alloc_table.stream_allocations[0].slot_count = 4;
+	link->mst_stream_alloc_table.stream_allocations[1].vcp_id = 2;
+	link->mst_stream_alloc_table.stream_allocations[1].slot_count = 8;
+
+	/* Remove vcp_id 1 */
+	payload.vcpi = 1;
+	payload.time_slots = 4;
+
+	fill_dc_mst_payload_table_from_drm(link, false, &payload, &table);
+
+	/* Only vcp_id 2 should remain */
+	KUNIT_EXPECT_EQ(test, table.stream_count, 1);
+	KUNIT_EXPECT_EQ(test, table.stream_allocations[0].vcp_id, 2);
+	KUNIT_EXPECT_EQ(test, table.stream_allocations[0].slot_count, 8);
+}
+
+/**
+ * dm_test_fill_mst_payload_table_empty - Test payload table fill when empty
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_mst_payload_table_empty(struct kunit *test)
+{
+	struct dc_link *link;
+	struct drm_dp_mst_atomic_payload payload = {0};
+	struct dc_dp_mst_stream_allocation_table table = {0};
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	/* Enable on an empty table */
+	payload.vcpi = 5;
+	payload.time_slots = 12;
+
+	fill_dc_mst_payload_table_from_drm(link, true, &payload, &table);
+
+	KUNIT_EXPECT_EQ(test, table.stream_count, 1);
+	KUNIT_EXPECT_EQ(test, table.stream_allocations[0].vcp_id, 5);
+	KUNIT_EXPECT_EQ(test, table.stream_allocations[0].slot_count, 12);
+}
+
+/* Tests for dm_helpers_submit_i2c() */
+
+/**
+ * dm_test_submit_i2c_null_priv - Test i2c submit returns false without connector
+ * @test: The KUnit test context
+ */
+static void dm_test_submit_i2c_null_priv(struct kunit *test)
+{
+	struct dc_link *link;
+	struct i2c_command cmd = {0};
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	KUNIT_EXPECT_FALSE(test, dm_helpers_submit_i2c(NULL, link, &cmd));
+}
+
+struct dm_test_i2c_adapter {
+	struct i2c_adapter base;
+	struct kunit *test;
+	struct i2c_payload *expected_payloads;
+	int expected_num;
+	int ret;
+	int calls;
+};
+
+static void dm_test_i2c_lock_bus(struct i2c_adapter *adapter,
+					 unsigned int flags)
+{
+	rt_mutex_lock(&adapter->bus_lock);
+}
+
+static int dm_test_i2c_trylock_bus(struct i2c_adapter *adapter,
+					   unsigned int flags)
+{
+	return rt_mutex_trylock(&adapter->bus_lock);
+}
+
+static void dm_test_i2c_unlock_bus(struct i2c_adapter *adapter,
+					   unsigned int flags)
+{
+	rt_mutex_unlock(&adapter->bus_lock);
+}
+
+static const struct i2c_lock_operations dm_test_i2c_lock_ops = {
+	.lock_bus = dm_test_i2c_lock_bus,
+	.trylock_bus = dm_test_i2c_trylock_bus,
+	.unlock_bus = dm_test_i2c_unlock_bus,
+};
+
+static int dm_test_i2c_master_xfer(struct i2c_adapter *adapter,
+					   struct i2c_msg *msgs,
+					   int num)
+{
+	struct dm_test_i2c_adapter *fake;
+	int i;
+
+	fake = container_of(adapter, struct dm_test_i2c_adapter, base);
+	fake->calls++;
+
+	KUNIT_EXPECT_EQ(fake->test, num, fake->expected_num);
+
+	for (i = 0; i < num; i++) {
+		KUNIT_EXPECT_EQ(fake->test, msgs[i].flags,
+				 fake->expected_payloads[i].write ? 0 : I2C_M_RD);
+		KUNIT_EXPECT_EQ(fake->test, msgs[i].addr,
+				 (u16)fake->expected_payloads[i].address);
+		KUNIT_EXPECT_EQ(fake->test, msgs[i].len,
+				 (u16)fake->expected_payloads[i].length);
+		KUNIT_EXPECT_PTR_EQ(fake->test, msgs[i].buf,
+				    fake->expected_payloads[i].data);
+	}
+
+	return fake->ret;
+}
+
+static const struct i2c_algorithm dm_test_i2c_algorithm = {
+	.master_xfer = dm_test_i2c_master_xfer,
+};
+
+struct dm_test_mccs_i2c_adapter {
+	struct i2c_adapter base;
+	u8 write_data[16];
+	u8 read_reply[11];
+	int write_ret;
+	int read_ret;
+	unsigned int write_len;
+	unsigned int writes;
+	unsigned int reads;
+};
+
+static int dm_test_mccs_i2c_master_xfer(struct i2c_adapter *adapter,
+						struct i2c_msg *msgs,
+						int num)
+{
+	struct dm_test_mccs_i2c_adapter *fake;
+	struct i2c_msg *msg = msgs;
+	size_t copy_len;
+
+	fake = container_of(adapter, struct dm_test_mccs_i2c_adapter, base);
+
+	if (num != 1)
+		return 0;
+
+	if (msg->flags & I2C_M_RD) {
+		fake->reads++;
+		if (fake->read_ret != 1)
+			return fake->read_ret;
+
+		copy_len = min_t(size_t, msg->len, sizeof(fake->read_reply));
+		memcpy(msg->buf, fake->read_reply, copy_len);
+		return 1;
+	}
+
+	fake->writes++;
+	if (fake->write_ret != 1)
+		return fake->write_ret;
+
+	copy_len = min_t(size_t, msg->len, sizeof(fake->write_data));
+	memcpy(fake->write_data, msg->buf, copy_len);
+	fake->write_len = copy_len;
+
+	return 1;
+}
+
+static const struct i2c_algorithm dm_test_mccs_i2c_algorithm = {
+	.master_xfer = dm_test_mccs_i2c_master_xfer,
+};
+
+static struct dm_test_mccs_i2c_adapter *dm_test_alloc_mccs_i2c(struct kunit *test)
+{
+	struct dm_test_mccs_i2c_adapter *fake;
+
+	fake = kunit_kzalloc(test, sizeof(*fake), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fake);
+
+	fake->base.algo = &dm_test_mccs_i2c_algorithm;
+	fake->base.lock_ops = &dm_test_i2c_lock_ops;
+	fake->write_ret = 1;
+	fake->read_ret = 1;
+	rt_mutex_init(&fake->base.bus_lock);
+	rt_mutex_init(&fake->base.mux_lock);
+
+	return fake;
+}
+
+static u8 dm_test_mccs_checksum(const u8 *data, unsigned int len)
+{
+	u8 checksum = 0x6e;
+	unsigned int i;
+
+	for (i = 0; i < len; i++)
+		checksum ^= data[i];
+
+	return checksum;
+}
+
+static void dm_test_submit_i2c_transfer(struct kunit *test,
+					bool full_transfer)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct dm_test_i2c_adapter *fake;
+	struct dc_link *link;
+	u8 write_data[2] = { 0x12, 0x34 };
+	u8 read_data[3] = { 0 };
+	struct i2c_payload payloads[] = {
+		{ true, 0x50, sizeof(write_data), write_data },
+		{ false, 0x51, sizeof(read_data), read_data },
+	};
+	struct i2c_command cmd = {
+		.payloads = payloads,
+		.number_of_payloads = ARRAY_SIZE(payloads),
+	};
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	fake = kunit_kzalloc(test, sizeof(*fake), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fake);
+
+	fake->test = test;
+	fake->expected_payloads = payloads;
+	fake->expected_num = ARRAY_SIZE(payloads);
+	fake->ret = full_transfer ? ARRAY_SIZE(payloads) : 1;
+	fake->base.algo = &dm_test_i2c_algorithm;
+	fake->base.lock_ops = &dm_test_i2c_lock_ops;
+	rt_mutex_init(&fake->base.bus_lock);
+	rt_mutex_init(&fake->base.mux_lock);
+
+	aconnector->i2c = (struct amdgpu_i2c_adapter *)fake;
+	link->priv = aconnector;
+
+	KUNIT_EXPECT_EQ(test, dm_helpers_submit_i2c(NULL, link, &cmd), full_transfer);
+	KUNIT_EXPECT_EQ(test, fake->calls, 1);
+}
+
+/**
+ * dm_test_submit_i2c_success - Test payloads are mapped to i2c_msg array
+ * @test: The KUnit test context
+ */
+static void dm_test_submit_i2c_success(struct kunit *test)
+{
+	dm_test_submit_i2c_transfer(test, true);
+}
+
+/**
+ * dm_test_submit_i2c_partial_transfer - Test short i2c transfer returns false
+ * @test: The KUnit test context
+ */
+static void dm_test_submit_i2c_partial_transfer(struct kunit *test)
+{
+	dm_test_submit_i2c_transfer(test, false);
+}
+
+/* Tests for dm_helper_dmub_aux_transfer_sync() */
+
+/**
+ * dm_test_dmub_aux_transfer_sync_hpd_discon - Test aux transfer with HPD disconnected
+ * @test: The KUnit test context
+ */
+static void dm_test_dmub_aux_transfer_sync_hpd_discon(struct kunit *test)
+{
+	struct dc_link *link;
+	struct dc_context *ctx;
+	struct aux_payload payload = {0};
+	enum aux_return_code_type result = AUX_RET_SUCCESS;
+	int ret;
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	link->hpd_status = false;
+
+	ret = dm_helper_dmub_aux_transfer_sync(ctx, link, &payload, &result);
+
+	KUNIT_EXPECT_EQ(test, ret, -1);
+	KUNIT_EXPECT_EQ(test, (int)result, (int)AUX_RET_ERROR_HPD_DISCON);
+}
+
+/* Tests for empty stub functions (must not crash) */
+
+/**
+ * dm_test_dp_update_branch_info_no_crash - Test empty stub does not crash
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_update_branch_info_no_crash(struct kunit *test)
+{
+	dm_helpers_dp_update_branch_info(NULL, NULL);
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/**
+ * dm_test_mst_poll_pending_down_reply_no_crash - Test empty stub does not crash
+ * @test: The KUnit test context
+ */
+static void dm_test_mst_poll_pending_down_reply_no_crash(struct kunit *test)
+{
+	dm_helpers_dp_mst_poll_pending_down_reply(NULL, NULL);
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/**
+ * dm_test_mst_clear_payload_alloc_table_no_crash - Test empty stub does not crash
+ * @test: The KUnit test context
+ */
+static void dm_test_mst_clear_payload_alloc_table_no_crash(struct kunit *test)
+{
+	dm_helpers_dp_mst_clear_payload_allocation_table(NULL, NULL);
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/**
+ * dm_test_set_dcn_clocks_no_crash - Test empty stub does not crash
+ * @test: The KUnit test context
+ */
+static void dm_test_set_dcn_clocks_no_crash(struct kunit *test)
+{
+	dm_set_dcn_clocks(NULL, NULL);
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/**
+ * dm_test_dmu_timeout_no_crash - Test empty stub does not crash
+ * @test: The KUnit test context
+ */
+static void dm_test_dmu_timeout_no_crash(struct kunit *test)
+{
+	dm_helpers_dmu_timeout(NULL);
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/**
+ * dm_test_smu_timeout_no_crash - Test empty stub does not crash
+ * @test: The KUnit test context
+ */
+static void dm_test_smu_timeout_no_crash(struct kunit *test)
+{
+	dm_helpers_smu_timeout(NULL, 0, 0, 0);
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/**
+ * dm_test_set_phyd32clk_no_crash - Test empty stub does not crash
+ * @test: The KUnit test context
+ */
+static void dm_test_set_phyd32clk_no_crash(struct kunit *test)
+{
+	dm_set_phyd32clk(NULL, 0);
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/**
+ * dm_test_mst_update_branch_bandwidth_no_crash - Test empty stub does not crash
+ * @test: The KUnit test context
+ */
+static void dm_test_mst_update_branch_bandwidth_no_crash(struct kunit *test)
+{
+	dm_helpers_dp_mst_update_branch_bandwidth(NULL, NULL);
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/* Tests for MST functions null-connector early returns */
+
+/**
+ * dm_test_mst_write_payload_alloc_table_null_ctx - Test null connector returns false
+ * @test: The KUnit test context
+ */
+static void dm_test_mst_write_payload_alloc_table_null_ctx(struct kunit *test)
+{
+	struct dc_stream_state *stream;
+
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+
+	/* dm_stream_context is NULL → aconnector is NULL → return false */
+	KUNIT_EXPECT_FALSE(test,
+			   dm_helpers_dp_mst_write_payload_allocation_table(NULL, stream, NULL, true));
+}
+
+/**
+ * dm_test_mst_poll_for_act_null_ctx - Test null connector returns ACT_FAILED
+ * @test: The KUnit test context
+ */
+static void dm_test_mst_poll_for_act_null_ctx(struct kunit *test)
+{
+	struct dc_stream_state *stream;
+
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+
+	KUNIT_EXPECT_EQ(test,
+			(int)dm_helpers_dp_mst_poll_for_allocation_change_trigger(NULL, stream),
+			(int)ACT_FAILED);
+}
+
+/**
+ * dm_test_mst_send_payload_alloc_null_ctx - Test null connector does not crash
+ * @test: The KUnit test context
+ */
+static void dm_test_mst_send_payload_alloc_null_ctx(struct kunit *test)
+{
+	struct dc_stream_state *stream;
+
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+
+	/* Should early-return without crash */
+	dm_helpers_dp_mst_send_payload_allocation(NULL, stream);
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/**
+ * dm_test_mst_update_mgr_dealloc_null_ctx - Test null connector does not crash
+ * @test: The KUnit test context
+ */
+static void dm_test_mst_update_mgr_dealloc_null_ctx(struct kunit *test)
+{
+	struct dc_stream_state *stream;
+
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+
+	dm_helpers_dp_mst_update_mst_mgr_for_deallocation(NULL, stream);
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/**
+ * dm_test_is_dp_sink_present_null_priv - Test null connector returns true
+ * @test: The KUnit test context
+ */
+static void dm_test_is_dp_sink_present_null_priv(struct kunit *test)
+{
+	struct dc_link *link;
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	/* NULL priv → DRM_ERROR + return true */
+	KUNIT_EXPECT_TRUE(test, dm_helpers_is_dp_sink_present(link));
+}
+
+/* Tests for dm_helpers_dmub_outbox_interrupt_control() */
+
+/**
+ * dm_test_dmub_outbox_interrupt_control_null_dc - Test outbox irq control with NULL dc
+ * @test: The KUnit test context
+ *
+ * dc_interrupt_set() is NULL-safe and returns false when dc is NULL, so the
+ * helper returns false without touching real interrupt hardware.
+ */
+static void dm_test_dmub_outbox_interrupt_control_null_dc(struct kunit *test)
+{
+	struct dc_context *ctx;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	/* ctx->dc is NULL → dc_interrupt_set returns false */
+	KUNIT_EXPECT_FALSE(test, dm_helpers_dmub_outbox_interrupt_control(ctx, true));
+	KUNIT_EXPECT_FALSE(test, dm_helpers_dmub_outbox_interrupt_control(ctx, false));
+}
+
+/* Tests for dm_helpers_mst_enable_stream_features() */
+
+/**
+ * dm_test_mst_enable_stream_features_aux_disabled - Test early return when aux disabled
+ * @test: The KUnit test context
+ */
+static void dm_test_mst_enable_stream_features_aux_disabled(struct kunit *test)
+{
+	struct dc_stream_state *stream;
+	struct dc_link *link;
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+
+	stream->link = link;
+	link->aux_access_disabled = true;
+
+	/* aux_access_disabled → early return without DPCD access */
+	dm_helpers_mst_enable_stream_features(stream);
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/**
+ * dm_test_mst_enable_stream_features_writes_downspread - Test MSA ignore write
+ * @test: The KUnit test context
+ */
+static void dm_test_mst_enable_stream_features_writes_downspread(struct kunit *test)
+{
+	struct dm_test_synaptics_aux *fixture;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_stream_state *stream;
+	struct amdgpu_device *adev;
+	struct dc_link *link;
+
+	adev = dm_kunit_alloc_adev(test);
+	fixture = dm_test_alloc_synaptics_aux_with_dev(test, &adev->ddev);
+	aconnector = dm_kunit_alloc_connector(test, adev, NULL);
+	link = dm_kunit_alloc_link(test);
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+
+	dm_test_current_aux_recorder = fixture;
+	aconnector->dm_dp_aux.aux.drm_dev = &adev->ddev;
+	aconnector->dm_dp_aux.aux.transfer = dm_test_current_aux_transfer;
+	drm_dp_aux_init(&aconnector->dm_dp_aux.aux);
+	link->priv = aconnector;
+	stream->link = link;
+	stream->ignore_msa_timing_param = true;
+
+	dm_helpers_mst_enable_stream_features(stream);
+
+	KUNIT_EXPECT_EQ(test, fixture->downspread_reads, 1U);
+	KUNIT_EXPECT_EQ(test, fixture->downspread_writes, 1U);
+	KUNIT_EXPECT_EQ(test, fixture->last_dpcd_write_address,
+			 DP_DOWNSPREAD_CTRL);
+	KUNIT_EXPECT_EQ(test, fixture->dpcd_write_value, (u8)BIT(7));
+}
+
+/* Tests for dm_helpers_enable_periodic_detection() */
+
+struct dm_test_idle_work {
+	struct idle_workqueue base;
+	bool ran;
+};
+
+static void dm_test_idle_work_func(struct work_struct *work)
+{
+	struct dm_test_idle_work *idle_work;
+
+	idle_work = container_of(work, struct dm_test_idle_work, base.work);
+	idle_work->ran = true;
+}
+
+/**
+ * dm_test_enable_periodic_detection_no_workqueue - Test no-op without idle workqueue
+ * @test: The KUnit test context
+ */
+static void dm_test_enable_periodic_detection_no_workqueue(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	ctx->driver_context = adev;
+
+	/* adev->dm.idle_workqueue is NULL → no-op, no crash */
+	dm_helpers_enable_periodic_detection(ctx, true);
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/**
+ * dm_test_enable_periodic_detection_updates_enable - Test idle work enable flag
+ * @test: The KUnit test context
+ *
+ * Keeps idle_workqueue->running set so the helper only updates the enable flag
+ * and does not queue the idle worker.
+ */
+static void dm_test_enable_periodic_detection_updates_enable(struct kunit *test)
+{
+	struct idle_workqueue *idle_work;
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+
+	adev = dm_kunit_alloc_adev(test);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	idle_work = kunit_kzalloc(test, sizeof(*idle_work), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, idle_work);
+
+	ctx->driver_context = adev;
+	adev->dm.idle_workqueue = idle_work;
+	idle_work->running = true;
+
+	dm_helpers_enable_periodic_detection(ctx, true);
+	KUNIT_EXPECT_TRUE(test, idle_work->enable);
+
+	dm_helpers_enable_periodic_detection(ctx, false);
+	KUNIT_EXPECT_FALSE(test, idle_work->enable);
+}
+
+/**
+ * dm_test_enable_periodic_detection_schedules_work - Test headless schedule path
+ * @test: The KUnit test context
+ */
+static void dm_test_enable_periodic_detection_schedules_work(struct kunit *test)
+{
+	struct dm_test_idle_work *idle_work;
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+
+	adev = dm_kunit_alloc_adev(test);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	idle_work = kunit_kzalloc(test, sizeof(*idle_work), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, idle_work);
+
+	ctx->driver_context = adev;
+	adev->dm.ddev = &adev->ddev;
+	adev->dm.idle_workqueue = &idle_work->base;
+	INIT_WORK(&idle_work->base.work, dm_test_idle_work_func);
+
+	dm_helpers_enable_periodic_detection(ctx, true);
+	flush_work(&idle_work->base.work);
+
+	KUNIT_EXPECT_TRUE(test, idle_work->base.enable);
+	KUNIT_EXPECT_TRUE(test, idle_work->ran);
+}
+
+/* Tests for dm_helpers_read_mccs_caps() */
+
+/**
+ * dm_test_read_mccs_caps_null_ctx - Test early return with NULL context
+ * @test: The KUnit test context
+ */
+static void dm_test_read_mccs_caps_null_ctx(struct kunit *test)
+{
+	/* NULL ctx → early return, no crash */
+	dm_helpers_read_mccs_caps(NULL, NULL, NULL);
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/**
+ * dm_test_read_mccs_caps_null_link - Test early return with NULL link
+ * @test: The KUnit test context
+ */
+static void dm_test_read_mccs_caps_null_link(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_sink *sink;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	ctx->driver_context = adev;
+	sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+
+	/* link is NULL → drm_dbg_driver + return */
+	dm_helpers_read_mccs_caps(ctx, NULL, sink);
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/**
+ * dm_test_read_mccs_caps_no_vcp_code - Test no-vcp-code path clears freesync support
+ * @test: The KUnit test context
+ *
+ * With freesync_vcp_code == 0 the i2c/MCCS path is skipped entirely and the
+ * function only clears sink->mccs_caps.freesync_supported.
+ */
+static void dm_test_read_mccs_caps_no_vcp_code(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc_sink *sink;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	ctx->driver_context = adev;
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+	sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+
+	sink->edid_caps.freesync_vcp_code = 0;
+	sink->mccs_caps.freesync_supported = true; /* should be cleared */
+
+	dm_helpers_read_mccs_caps(ctx, link, sink);
+
+	KUNIT_EXPECT_FALSE(test, sink->mccs_caps.freesync_supported);
+}
+
+/*
+ * Allocate and wire the adev/ctx/link/sink/connector/i2c objects shared by the
+ * MCCS read/set tests so each test only configures the fields it exercises.
+ */
+struct dm_test_mccs_fixture {
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc_sink *sink;
+	struct amdgpu_dm_connector *aconnector;
+	struct dm_test_mccs_i2c_adapter *fake;
+};
+
+static struct dm_test_mccs_fixture dm_test_alloc_mccs_fixture(struct kunit *test)
+{
+	struct dm_test_mccs_fixture fixture;
+
+	fixture.adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.adev);
+	fixture.ctx = kunit_kzalloc(test, sizeof(*fixture.ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.ctx);
+	fixture.link = kunit_kzalloc(test, sizeof(*fixture.link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.link);
+	fixture.sink = kunit_kzalloc(test, sizeof(*fixture.sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.sink);
+	fixture.aconnector = kunit_kzalloc(test, sizeof(*fixture.aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.aconnector);
+	fixture.fake = dm_test_alloc_mccs_i2c(test);
+
+	fixture.ctx->driver_context = fixture.adev;
+	fixture.aconnector->i2c = (struct amdgpu_i2c_adapter *)fixture.fake;
+	fixture.link->priv = fixture.aconnector;
+
+	return fixture;
+}
+
+/**
+ * dm_test_read_mccs_caps_i2c_vcp_request - Test MCCS VCP request packet
+ * @test: The KUnit test context
+ */
+static void dm_test_read_mccs_caps_i2c_vcp_request(struct kunit *test)
+{
+	static const u8 expected_prefix[] = { 0x51, 0x82, 0x01, 0xe3 };
+	struct dm_test_mccs_fixture fixture = dm_test_alloc_mccs_fixture(test);
+	struct dm_test_mccs_i2c_adapter *fake = fixture.fake;
+	struct dc_context *ctx = fixture.ctx;
+	struct dc_link *link = fixture.link;
+	struct dc_sink *sink = fixture.sink;
+
+	link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+	link->dpcd_caps.dpcd_rev.raw = DP_DPCD_REV_14;
+	link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_HDMI_CONVERTER;
+	link->dpcd_caps.branch_dev_id = DP_BRANCH_DEVICE_ID_0060AD;
+	link->dpcd_caps.adaptive_sync_caps.dp_adap_sync_caps.bits.ADAPTIVE_SYNC_SDP_SUPPORT = 1;
+	sink->edid_caps.freesync_vcp_code = 0xe3;
+	fake->read_reply[1] = 0x82;
+	fake->read_reply[9] = 0x01;
+
+	dm_helpers_read_mccs_caps(ctx, link, sink);
+
+	KUNIT_EXPECT_TRUE(test, sink->mccs_caps.freesync_supported);
+	KUNIT_EXPECT_EQ(test, fake->writes, 1U);
+	KUNIT_EXPECT_EQ(test, fake->reads, 1U);
+	KUNIT_EXPECT_EQ(test, fake->write_len, (unsigned int)sizeof(expected_prefix) + 1);
+	KUNIT_EXPECT_EQ(test, memcmp(fake->write_data, expected_prefix,
+					     sizeof(expected_prefix)), 0);
+	KUNIT_EXPECT_EQ(test, fake->write_data[4],
+			dm_test_mccs_checksum(expected_prefix, sizeof(expected_prefix)));
+}
+
+/**
+ * dm_test_read_mccs_caps_hdmi_vcp_request - Test local HDMI MCCS path
+ * @test: The KUnit test context
+ */
+static void dm_test_read_mccs_caps_hdmi_vcp_request(struct kunit *test)
+{
+	struct dm_test_mccs_fixture fixture = dm_test_alloc_mccs_fixture(test);
+	struct dm_test_mccs_i2c_adapter *fake = fixture.fake;
+	struct dc_context *ctx = fixture.ctx;
+	struct dc_link *link = fixture.link;
+	struct dc_sink *sink = fixture.sink;
+
+	link->connector_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	sink->edid_caps.freesync_vcp_code = 0xe3;
+	fake->read_reply[1] = 0x82;
+	fake->read_reply[9] = 0x01;
+
+	dm_helpers_read_mccs_caps(ctx, link, sink);
+
+	KUNIT_EXPECT_TRUE(test, sink->mccs_caps.freesync_supported);
+	KUNIT_EXPECT_EQ(test, fake->writes, 1U);
+	KUNIT_EXPECT_EQ(test, fake->reads, 1U);
+}
+
+/**
+ * dm_test_read_mccs_caps_legacy_pcon_vcp_request - Test legacy PCON path
+ * @test: The KUnit test context
+ */
+static void dm_test_read_mccs_caps_legacy_pcon_vcp_request(struct kunit *test)
+{
+	struct dm_test_mccs_fixture fixture = dm_test_alloc_mccs_fixture(test);
+	struct dm_test_mccs_i2c_adapter *fake = fixture.fake;
+	struct dc_context *ctx = fixture.ctx;
+	struct dc_link *link = fixture.link;
+	struct dc_sink *sink = fixture.sink;
+
+	link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+	link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_DVI_CONVERTER;
+	sink->edid_caps.freesync_vcp_code = 0xe3;
+	fake->read_reply[1] = 0x82;
+	fake->read_reply[9] = 0x01;
+
+	dm_helpers_read_mccs_caps(ctx, link, sink);
+
+	KUNIT_EXPECT_TRUE(test, sink->mccs_caps.freesync_supported);
+	KUNIT_EXPECT_EQ(test, fake->writes, 1U);
+	KUNIT_EXPECT_EQ(test, fake->reads, 1U);
+}
+
+/**
+ * dm_test_read_mccs_caps_i2c_failure - Test VCP request retry failure
+ * @test: The KUnit test context
+ */
+static void dm_test_read_mccs_caps_i2c_failure(struct kunit *test)
+{
+	struct dm_test_mccs_fixture fixture = dm_test_alloc_mccs_fixture(test);
+	struct dm_test_mccs_i2c_adapter *fake = fixture.fake;
+	struct dc_context *ctx = fixture.ctx;
+	struct dc_link *link = fixture.link;
+	struct dc_sink *sink = fixture.sink;
+
+	fixture.aconnector->base.dev = &fixture.adev->ddev;
+	link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+	link->dpcd_caps.dpcd_rev.raw = DP_DPCD_REV_14;
+	link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_HDMI_CONVERTER;
+	link->dpcd_caps.branch_dev_id = DP_BRANCH_DEVICE_ID_0060AD;
+	link->dpcd_caps.adaptive_sync_caps.dp_adap_sync_caps.bits.ADAPTIVE_SYNC_SDP_SUPPORT = 1;
+	sink->edid_caps.freesync_vcp_code = 0xe3;
+	fake->write_ret = 0;
+
+	dm_helpers_read_mccs_caps(ctx, link, sink);
+
+	KUNIT_EXPECT_FALSE(test, sink->mccs_caps.freesync_supported);
+	KUNIT_EXPECT_EQ(test, fake->writes, 5U);
+	KUNIT_EXPECT_EQ(test, fake->reads, 0U);
+}
+
+/* Tests for dm_helpers_mccs_vcp_set() */
+
+/**
+ * dm_test_mccs_vcp_set_null_ctx - Test early return with NULL context
+ * @test: The KUnit test context
+ */
+static void dm_test_mccs_vcp_set_null_ctx(struct kunit *test)
+{
+	/* NULL ctx → early return, no crash */
+	dm_helpers_mccs_vcp_set(NULL, NULL, NULL);
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/**
+ * dm_test_mccs_vcp_set_not_supported - Test early return when freesync unsupported
+ * @test: The KUnit test context
+ */
+static void dm_test_mccs_vcp_set_not_supported(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc_sink *sink;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	ctx->driver_context = adev;
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+	sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+
+	sink->mccs_caps.freesync_supported = false;
+
+	/* freesync not supported → early return without i2c */
+	dm_helpers_mccs_vcp_set(ctx, link, sink);
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/**
+ * dm_test_mccs_vcp_set_null_link - Test early return with NULL link
+ * @test: The KUnit test context
+ */
+static void dm_test_mccs_vcp_set_null_link(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_sink *sink;
+
+	adev = dm_kunit_alloc_adev(test);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+
+	ctx->driver_context = adev;
+	dm_helpers_mccs_vcp_set(ctx, NULL, sink);
+	KUNIT_EXPECT_TRUE(test, true);
+}
+
+/**
+ * dm_test_mccs_vcp_set_i2c_packet - Test MCCS VCP set packet
+ * @test: The KUnit test context
+ */
+static void dm_test_mccs_vcp_set_i2c_packet(struct kunit *test)
+{
+	static const u8 expected_prefix[] = {
+		0x51, 0x84, 0x03, 0xe3, 0x01, 0x01,
+	};
+	struct dm_test_mccs_fixture fixture = dm_test_alloc_mccs_fixture(test);
+	struct dm_test_mccs_i2c_adapter *fake = fixture.fake;
+	struct dc_context *ctx = fixture.ctx;
+	struct dc_link *link = fixture.link;
+	struct dc_sink *sink = fixture.sink;
+
+	sink->mccs_caps.freesync_supported = true;
+	sink->edid_caps.freesync_vcp_code = 0xe3;
+
+	dm_helpers_mccs_vcp_set(ctx, link, sink);
+
+	KUNIT_EXPECT_EQ(test, fake->writes, 1U);
+	KUNIT_EXPECT_EQ(test, fake->reads, 0U);
+	KUNIT_EXPECT_EQ(test, fake->write_len, (unsigned int)sizeof(expected_prefix) + 1);
+	KUNIT_EXPECT_EQ(test, memcmp(fake->write_data, expected_prefix,
+					     sizeof(expected_prefix)), 0);
+	KUNIT_EXPECT_EQ(test, fake->write_data[6],
+			dm_test_mccs_checksum(expected_prefix, sizeof(expected_prefix)));
+}
+
+/**
+ * dm_test_mccs_vcp_set_i2c_failure - Test VCP set retry failure path
+ * @test: The KUnit test context
+ */
+static void dm_test_mccs_vcp_set_i2c_failure(struct kunit *test)
+{
+	struct dm_test_mccs_fixture fixture = dm_test_alloc_mccs_fixture(test);
+	struct dm_test_mccs_i2c_adapter *fake = fixture.fake;
+	struct dc_context *ctx = fixture.ctx;
+	struct dc_link *link = fixture.link;
+	struct dc_sink *sink = fixture.sink;
+
+	sink->mccs_caps.freesync_supported = true;
+	sink->edid_caps.freesync_vcp_code = 0xe3;
+	fake->write_ret = 0;
+
+	dm_helpers_mccs_vcp_set(ctx, link, sink);
+
+	KUNIT_EXPECT_EQ(test, fake->writes, 5U);
+	KUNIT_EXPECT_EQ(test, fake->reads, 0U);
+}
+
+/* Tests for dm_helpers_construct_old_payload() */
+
+/**
+ * dm_test_construct_old_payload_empty_list - Test PBN/time-slot calc, empty list
+ * @test: The KUnit test context
+ *
+ * With no other payloads, next_payload_vc_start stays at mgr->next_start_slot,
+ * so allocated time_slots = next_start_slot - vc_start_slot.
+ */
+static void dm_test_construct_old_payload_empty_list(struct kunit *test)
+{
+	struct drm_dp_mst_topology_mgr *mgr;
+	struct drm_dp_mst_topology_state *mst_state;
+	struct drm_dp_mst_atomic_payload *new_payload;
+	struct drm_dp_mst_atomic_payload *old_payload;
+
+	mgr = kunit_kzalloc(test, sizeof(*mgr), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, mgr);
+	mst_state = kunit_kzalloc(test, sizeof(*mst_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, mst_state);
+	new_payload = kunit_kzalloc(test, sizeof(*new_payload), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, new_payload);
+	old_payload = kunit_kzalloc(test, sizeof(*old_payload), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_payload);
+
+	INIT_LIST_HEAD(&mst_state->payloads);
+	mgr->next_start_slot = 10;
+	mst_state->pbn_div.full = 5 << 12;	/* dfixed_trunc → 5 PBN/slot */
+	new_payload->vc_start_slot = 3;
+
+	dm_helpers_construct_old_payload(mgr, mst_state, new_payload,
+					 old_payload);
+
+	/* 10 - 3 = 7 slots, 7 * 5 = 35 PBN */
+	KUNIT_EXPECT_EQ(test, old_payload->time_slots, 7);
+	KUNIT_EXPECT_EQ(test, old_payload->pbn, 35);
+}
+
+/**
+ * dm_test_construct_old_payload_intervening - Test calc with an intervening payload
+ * @test: The KUnit test context
+ *
+ * A payload whose vc_start_slot falls between the new payload and the manager's
+ * next_start_slot narrows the allocated time-slot window.
+ */
+static void dm_test_construct_old_payload_intervening(struct kunit *test)
+{
+	struct drm_dp_mst_topology_mgr *mgr;
+	struct drm_dp_mst_topology_state *mst_state;
+	struct drm_dp_mst_atomic_payload *new_payload;
+	struct drm_dp_mst_atomic_payload *other_payload;
+	struct drm_dp_mst_atomic_payload *old_payload;
+
+	mgr = kunit_kzalloc(test, sizeof(*mgr), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, mgr);
+	mst_state = kunit_kzalloc(test, sizeof(*mst_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, mst_state);
+	new_payload = kunit_kzalloc(test, sizeof(*new_payload), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, new_payload);
+	other_payload = kunit_kzalloc(test, sizeof(*other_payload), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, other_payload);
+	old_payload = kunit_kzalloc(test, sizeof(*old_payload), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_payload);
+
+	INIT_LIST_HEAD(&mst_state->payloads);
+	mgr->next_start_slot = 10;
+	mst_state->pbn_div.full = 5 << 12;
+	new_payload->vc_start_slot = 3;
+
+	/* other payload at slot 6 (between 3 and 10) narrows window to 6 */
+	other_payload->vc_start_slot = 6;
+	list_add_tail(&other_payload->next, &mst_state->payloads);
+
+	dm_helpers_construct_old_payload(mgr, mst_state, new_payload,
+					 old_payload);
+
+	/* 6 - 3 = 3 slots, 3 * 5 = 15 PBN */
+	KUNIT_EXPECT_EQ(test, old_payload->time_slots, 3);
+	KUNIT_EXPECT_EQ(test, old_payload->pbn, 15);
+}
+
+/* Tests for dm_helpers_dp_mst_write_payload_allocation_table() success path */
+
+/**
+ * dm_test_write_payload_alloc_table_success - Exercise the MST success path
+ * @test: The KUnit test context
+ * @enable: true for the add-payload path, false for the remove-payload path
+ *
+ * Builds a minimal MST topology-state fixture so the helper traverses past the
+ * early NULL checks and runs the real DRM MST payload helpers
+ * (drm_dp_add_payload_part1 / drm_dp_remove_payload_part1). With
+ * mgr->mst_primary == NULL the topology walk fails gracefully, so no remote
+ * DPCD/AUX traffic is generated, and the helper still returns true.
+ */
+static void dm_test_write_payload_alloc_table_success(struct kunit *test,
+						      bool enable)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_mst_topology_mgr *mgr;
+	struct drm_dp_mst_topology_state *mst_state;
+	struct drm_dp_mst_atomic_payload *payload;
+	struct drm_dp_mst_port *port;
+	struct dc_stream_state *stream;
+	struct dc_link *link;
+	struct dc_dp_mst_stream_allocation_table table = { 0 };
+	bool ret;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	mst_state = kunit_kzalloc(test, sizeof(*mst_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, mst_state);
+	payload = kunit_kzalloc(test, sizeof(*payload), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, payload);
+	port = kunit_kzalloc(test, sizeof(*port), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, port);
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	/* aconnector acts as its own MST root for this fixture */
+	aconnector->mst_root = aconnector;
+	aconnector->mst_output_port = port;
+
+	mgr = &aconnector->mst_mgr;
+	mutex_init(&mgr->lock);
+	mgr->dev = &adev->ddev;
+	mgr->mst_primary = NULL;		/* topology walk fails gracefully */
+	mgr->base.state = &mst_state->base;
+
+	INIT_LIST_HEAD(&mst_state->payloads);
+	mst_state->pbn_div.full = 5 << 12;	/* dfixed_trunc → 5 PBN/slot */
+
+	/* payload found by drm_atomic_get_mst_payload_state via matching port */
+	payload->port = port;
+	payload->vcpi = 1;
+	payload->vc_start_slot = 1;
+	payload->time_slots = 2;
+	list_add_tail(&payload->next, &mst_state->payloads);
+
+	/* pre-existing HW allocation so the disable path finds a VCPI to clear */
+	link->mst_stream_alloc_table.stream_count = 1;
+	link->mst_stream_alloc_table.stream_allocations[0].vcp_id = 1;
+	link->mst_stream_alloc_table.stream_allocations[0].slot_count = 2;
+
+	stream->dm_stream_context = aconnector;
+	stream->link = link;
+
+	ret = dm_helpers_dp_mst_write_payload_allocation_table(NULL, stream,
+							       &table, enable);
+
+	KUNIT_EXPECT_TRUE(test, ret);
+
+	if (enable)
+		/* add path keeps the old entry and appends the new payload */
+		KUNIT_EXPECT_EQ(test, table.stream_count, 2);
+	else
+		/* remove path clears the only entry */
+		KUNIT_EXPECT_EQ(test, table.stream_count, 0);
+}
+
+/**
+ * dm_test_write_payload_alloc_table_enable - Test add-payload success path
+ * @test: The KUnit test context
+ */
+static void dm_test_write_payload_alloc_table_enable(struct kunit *test)
+{
+	dm_test_write_payload_alloc_table_success(test, true);
+}
+
+/**
+ * dm_test_write_payload_alloc_table_disable - Test remove-payload success path
+ * @test: The KUnit test context
+ */
+static void dm_test_write_payload_alloc_table_disable(struct kunit *test)
+{
+	dm_test_write_payload_alloc_table_success(test, false);
+}
+
+/* Tests for dm_helpers_dp_mst_poll_for_allocation_change_trigger() */
+
+/*
+ * Stub AUX transfer that ACKs a DPCD read of the payload-table update status
+ * with the ACT-handled bit set, so drm_dp_check_act_status() returns 0.
+ */
+static ssize_t dm_test_act_aux_transfer_handled(struct drm_dp_aux *aux,
+						struct drm_dp_aux_msg *msg)
+{
+	if ((msg->request & ~DP_AUX_I2C_MOT) == DP_AUX_NATIVE_READ) {
+		memset(msg->buffer, 0, msg->size);
+		if (msg->size > 0)
+			((u8 *)msg->buffer)[0] = DP_PAYLOAD_ACT_HANDLED;
+	}
+	msg->reply = DP_AUX_NATIVE_REPLY_ACK;
+	return msg->size;
+}
+
+/*
+ * Stub AUX transfer that fails every transaction, so the ACT status read
+ * returns an error and drm_dp_check_act_status() returns non-zero.
+ */
+static ssize_t dm_test_act_aux_transfer_fail(struct drm_dp_aux *aux,
+					     struct drm_dp_aux_msg *msg)
+{
+	return -EIO;
+}
+
+/**
+ * dm_test_mst_start_top_mgr_set_mst_fail - Test MST start failure path
+ * @test: The KUnit test context
+ *
+ * With a connector-backed link and a failing AUX channel, the non-boot
+ * path calls drm_dp_mst_topology_mgr_set_mst(true), which fails to read the
+ * DPCD caps and returns a negative error, so the helper returns false.
+ */
+static void dm_test_mst_start_top_mgr_set_mst_fail(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_aux *aux;
+	struct dc_link *link;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	aux = kunit_kzalloc(test, sizeof(*aux), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aux);
+	link = dm_kunit_alloc_link(test);
+
+	aux->drm_dev = &adev->ddev;
+	aux->transfer = dm_test_act_aux_transfer_fail;
+	drm_dp_aux_init(aux);
+
+	mutex_init(&aconnector->mst_mgr.lock);
+	aconnector->mst_mgr.dev = &adev->ddev;
+	aconnector->mst_mgr.aux = aux;
+	link->priv = aconnector;
+
+	KUNIT_EXPECT_FALSE(test, dm_helpers_dp_mst_start_top_mgr(NULL, link, false));
+}
+
+/**
+ * dm_test_mst_stop_top_mgr_active - Test MST stop on an active topology manager
+ * @test: The KUnit test context
+ *
+ * With mst_state set, the helper calls drm_dp_mst_topology_mgr_set_mst(false)
+ * to disable MST and clears the link lane count. The helper always returns
+ * false.
+ */
+static void dm_test_mst_stop_top_mgr_active(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_aux *aux;
+	struct dc_link *link;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	aux = kunit_kzalloc(test, sizeof(*aux), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aux);
+	link = dm_kunit_alloc_link(test);
+
+	aux->drm_dev = &adev->ddev;
+	aux->transfer = dm_test_act_aux_transfer_handled;
+	drm_dp_aux_init(aux);
+
+	mutex_init(&aconnector->mst_mgr.lock);
+	aconnector->mst_mgr.dev = &adev->ddev;
+	aconnector->mst_mgr.aux = aux;
+	aconnector->mst_mgr.mst_state = true;
+	link->cur_link_settings.lane_count = 4;
+	link->priv = aconnector;
+
+	KUNIT_EXPECT_FALSE(test, dm_helpers_dp_mst_stop_top_mgr(NULL, link));
+	KUNIT_EXPECT_EQ(test, link->cur_link_settings.lane_count, 0);
+}
+
+/**
+ * dm_test_poll_for_act_no_mst_state - Test ACT poll bails when MST not started
+ * @test: The KUnit test context
+ *
+ * With mst_root set but mst_mgr->mst_state false, the helper returns
+ * ACT_FAILED before touching the AUX channel.
+ */
+static void dm_test_poll_for_act_no_mst_state(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_stream_state *stream;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+
+	aconnector->mst_root = aconnector;
+	aconnector->mst_mgr.mst_state = false;
+	stream->dm_stream_context = aconnector;
+
+	KUNIT_EXPECT_EQ(test,
+			(int)dm_helpers_dp_mst_poll_for_allocation_change_trigger(NULL, stream),
+			(int)ACT_FAILED);
+}
+
+/**
+ * dm_test_poll_for_act_success - Test ACT poll success path
+ * @test: The KUnit test context
+ *
+ * With MST started and the AUX channel reporting ACT handled,
+ * drm_dp_check_act_status() returns 0 and the helper returns ACT_SUCCESS.
+ */
+static void dm_test_poll_for_act_success(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_aux *aux;
+	struct dc_stream_state *stream;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	aux = kunit_kzalloc(test, sizeof(*aux), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aux);
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+
+	aux->drm_dev = &adev->ddev;
+	aux->transfer = dm_test_act_aux_transfer_handled;
+	drm_dp_aux_init(aux);
+
+	aconnector->mst_root = aconnector;
+	aconnector->mst_mgr.mst_state = true;
+	aconnector->mst_mgr.aux = aux;
+	stream->dm_stream_context = aconnector;
+
+	KUNIT_EXPECT_EQ(test,
+			(int)dm_helpers_dp_mst_poll_for_allocation_change_trigger(NULL, stream),
+			(int)ACT_SUCCESS);
+}
+
+/**
+ * dm_test_poll_for_act_status_failed - Test ACT poll failure path
+ * @test: The KUnit test context
+ *
+ * With MST started but the AUX channel failing, drm_dp_check_act_status()
+ * returns non-zero and the helper returns ACT_FAILED.
+ */
+static void dm_test_poll_for_act_status_failed(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_aux *aux;
+	struct dc_stream_state *stream;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	aux = kunit_kzalloc(test, sizeof(*aux), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aux);
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+
+	aux->drm_dev = &adev->ddev;
+	aux->transfer = dm_test_act_aux_transfer_fail;
+	drm_dp_aux_init(aux);
+
+	aconnector->mst_root = aconnector;
+	aconnector->mst_mgr.mst_state = true;
+	aconnector->mst_mgr.aux = aux;
+	stream->dm_stream_context = aconnector;
+
+	KUNIT_EXPECT_EQ(test,
+			(int)dm_helpers_dp_mst_poll_for_allocation_change_trigger(NULL, stream),
+			(int)ACT_FAILED);
+}
+
+/* Tests for dm_helpers_dp_mst_send_payload_allocation() */
+
+/**
+ * dm_test_mst_send_payload_alloc_part2_fail - Exercise the failure branch
+ * @test: The KUnit test context
+ *
+ * Builds a minimal MST topology-state fixture so the helper runs past the early
+ * NULL checks and calls drm_dp_add_payload_part2(). The payload's allocation
+ * status is left at its default (not DRM_DP_MST_PAYLOAD_ALLOCATION_DFP), so
+ * drm_dp_add_payload_part2() returns -EIO without any remote DPCD/AUX traffic.
+ * The non-zero return drives the failure branch, which clears the
+ * MST_ALLOCATE_NEW_PAYLOAD status bit.
+ */
+static void dm_test_mst_send_payload_alloc_part2_fail(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_mst_topology_mgr *mgr;
+	struct drm_dp_mst_topology_state *mst_state;
+	struct drm_dp_mst_atomic_payload *payload;
+	struct drm_dp_mst_port *port;
+	struct dc_stream_state *stream;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	mst_state = kunit_kzalloc(test, sizeof(*mst_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, mst_state);
+	payload = kunit_kzalloc(test, sizeof(*payload), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, payload);
+	port = kunit_kzalloc(test, sizeof(*port), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, port);
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+
+	/* aconnector acts as its own MST root for this fixture */
+	aconnector->mst_root = aconnector;
+	aconnector->mst_output_port = port;
+	/* drm_dp_add_payload_part2() logs port->connector->name on failure */
+	port->connector = &aconnector->base;
+
+	mgr = &aconnector->mst_mgr;
+	mgr->dev = &adev->ddev;
+	mgr->base.state = &mst_state->base;
+
+	INIT_LIST_HEAD(&mst_state->payloads);
+
+	/* payload found by drm_atomic_get_mst_payload_state via matching port */
+	payload->port = port;
+	list_add_tail(&payload->next, &mst_state->payloads);
+
+	/* pre-set the bit so we can observe it being cleared on failure */
+	aconnector->mst_status = MST_ALLOCATE_NEW_PAYLOAD;
+
+	stream->dm_stream_context = aconnector;
+	dm_helpers_dp_mst_send_payload_allocation(NULL, stream);
+
+	KUNIT_EXPECT_EQ(test,
+			aconnector->mst_status & MST_ALLOCATE_NEW_PAYLOAD, 0);
+}
+
+/* Tests for dm_helpers_dp_mst_update_mst_mgr_for_deallocation() */
+
+/**
+ * dm_test_mst_update_mgr_dealloc_success - Exercise the deallocation path
+ * @test: The KUnit test context
+ *
+ * Builds a minimal MST topology-state fixture so the helper runs past the early
+ * NULL checks through dm_helpers_construct_old_payload() and
+ * drm_dp_remove_payload_part2(), both of which are pure list/slot math with no
+ * remote DPCD/AUX traffic. Afterwards MST_CLEAR_ALLOCATED_PAYLOAD must be set,
+ * MST_ALLOCATE_NEW_PAYLOAD cleared, and the payload's slot released.
+ */
+static void dm_test_mst_update_mgr_dealloc_success(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_mst_topology_mgr *mgr;
+	struct drm_dp_mst_topology_state *mst_state;
+	struct drm_dp_mst_atomic_payload *payload;
+	struct drm_dp_mst_port *port;
+	struct dc_stream_state *stream;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	mst_state = kunit_kzalloc(test, sizeof(*mst_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, mst_state);
+	payload = kunit_kzalloc(test, sizeof(*payload), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, payload);
+	port = kunit_kzalloc(test, sizeof(*port), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, port);
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+
+	/* aconnector acts as its own MST root for this fixture */
+	aconnector->mst_root = aconnector;
+	aconnector->mst_output_port = port;
+
+	mgr = &aconnector->mst_mgr;
+	mgr->base.state = &mst_state->base;
+	mgr->next_start_slot = 10;
+	mgr->payload_count = 1;
+
+	INIT_LIST_HEAD(&mst_state->payloads);
+	mst_state->pbn_div.full = 5 << 12;	/* dfixed_trunc → 5 PBN/slot */
+
+	/* payload found by drm_atomic_get_mst_payload_state via matching port */
+	payload->port = port;
+	payload->vc_start_slot = 3;
+	payload->time_slots = 7;
+	list_add_tail(&payload->next, &mst_state->payloads);
+
+	/* pre-set the bit so we can observe it being cleared */
+	aconnector->mst_status = MST_ALLOCATE_NEW_PAYLOAD;
+
+	stream->dm_stream_context = aconnector;
+	dm_helpers_dp_mst_update_mst_mgr_for_deallocation(NULL, stream);
+
+	KUNIT_EXPECT_EQ(test,
+			aconnector->mst_status & MST_CLEAR_ALLOCATED_PAYLOAD,
+			(int)MST_CLEAR_ALLOCATED_PAYLOAD);
+	KUNIT_EXPECT_EQ(test,
+			aconnector->mst_status & MST_ALLOCATE_NEW_PAYLOAD, 0);
+	/* drm_dp_remove_payload_part2() releases the payload's slot */
+	KUNIT_EXPECT_EQ(test, payload->vc_start_slot, -1);
+}
+
+/* Tests for dm_helpers_dp_write_dsc_enable() */
+
+/**
+ * dm_test_dp_write_dsc_enable_mst_no_aux - Test MST early return without dsc_aux
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_write_dsc_enable_mst_no_aux(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_stream_state *stream;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+
+	stream->dm_stream_context = aconnector;
+	stream->signal = SIGNAL_TYPE_DISPLAY_PORT_MST;
+	aconnector->dsc_aux = NULL;
+
+	/* MST signal with NULL dsc_aux → return false */
+	KUNIT_EXPECT_FALSE(test, dm_helpers_dp_write_dsc_enable(NULL, stream, true));
+}
+
+/**
+ * dm_test_dp_write_dsc_enable_non_dp - Test non-DP signal returns false
+ * @test: The KUnit test context
+ *
+ * For an HDMI signal neither the MST nor the DP/eDP block runs, so ret stays 0.
+ */
+static void dm_test_dp_write_dsc_enable_non_dp(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_stream_state *stream;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+
+	stream->dm_stream_context = aconnector;
+	stream->signal = SIGNAL_TYPE_HDMI_TYPE_A;
+
+	/* Non-DP/MST signal → no DPCD write, ret stays 0 (false) */
+	KUNIT_EXPECT_FALSE(test, dm_helpers_dp_write_dsc_enable(NULL, stream, true));
+}
+
+struct dm_test_dsc_aux_pair {
+	struct dm_test_synaptics_aux *main;
+	struct dm_test_synaptics_aux *passthrough;
+};
+
+static struct amdgpu_dm_connector *dm_test_alloc_dsc_connector(struct kunit *test,
+						       struct dc_link *link)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	aconnector->dc_link = link;
+
+	return aconnector;
+}
+
+static struct dc_stream_state *dm_test_alloc_dsc_stream(struct kunit *test,
+						       struct dc_link *link,
+						       enum signal_type signal)
+{
+	struct dc_stream_state *stream;
+	struct dc_sink *sink;
+
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+	sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+
+	stream->link = link;
+	stream->signal = signal;
+	stream->sink = sink;
+	sink->link = link;
+
+	return stream;
+}
+
+static struct dm_test_dsc_aux_pair dm_test_dp_write_dsc_enable_mst(struct kunit *test,
+							   bool enable,
+							   bool passthrough)
+{
+	struct dm_test_dsc_aux_pair aux_pair;
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_mst_port *port;
+	struct dc_stream_state *stream;
+	struct dc_link *link;
+	bool ret;
+
+	link = dm_kunit_alloc_link(test);
+	aconnector = dm_test_alloc_dsc_connector(test, link);
+	stream = dm_test_alloc_dsc_stream(test, link, SIGNAL_TYPE_DISPLAY_PORT_MST);
+	port = kunit_kzalloc(test, sizeof(*port), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, port);
+	aux_pair.main = dm_test_alloc_synaptics_aux_with_dev(test, NULL);
+	aux_pair.passthrough = dm_test_alloc_synaptics_aux_with_dev(test, NULL);
+
+	stream->dm_stream_context = aconnector;
+	aconnector->dsc_aux = &aux_pair.main->aux;
+	aconnector->mst_output_port = port;
+	if (passthrough)
+		port->passthrough_aux = &aux_pair.passthrough->aux;
+
+	ret = dm_helpers_dp_write_dsc_enable(NULL, stream, enable);
+
+	KUNIT_EXPECT_TRUE(test, ret);
+	KUNIT_EXPECT_EQ(test, aux_pair.main->dsc_enable_writes, 1U);
+	KUNIT_EXPECT_EQ(test, aux_pair.main->dsc_enable_values[0], enable ? 1 : 0);
+	KUNIT_EXPECT_EQ(test, aux_pair.passthrough->dsc_enable_writes,
+			 passthrough ? 1U : 0U);
+	if (passthrough)
+		KUNIT_EXPECT_EQ(test, aux_pair.passthrough->dsc_enable_values[0], enable ? 2 : 0);
+
+	return aux_pair;
+}
+
+/**
+ * dm_test_dp_write_dsc_enable_mst_enable_decode_only - Test MST enable decoding write
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_write_dsc_enable_mst_enable_decode_only(struct kunit *test)
+{
+	dm_test_dp_write_dsc_enable_mst(test, true, false);
+}
+
+/**
+ * dm_test_dp_write_dsc_enable_mst_enable_passthrough - Test MST enable passthrough write
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_write_dsc_enable_mst_enable_passthrough(struct kunit *test)
+{
+	dm_test_dp_write_dsc_enable_mst(test, true, true);
+}
+
+/**
+ * dm_test_dp_write_dsc_enable_mst_disable_decode_only - Test MST disable decoding write
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_write_dsc_enable_mst_disable_decode_only(struct kunit *test)
+{
+	dm_test_dp_write_dsc_enable_mst(test, false, false);
+}
+
+/**
+ * dm_test_dp_write_dsc_enable_mst_disable_passthrough - Test MST disable passthrough write
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_write_dsc_enable_mst_disable_passthrough(struct kunit *test)
+{
+	dm_test_dp_write_dsc_enable_mst(test, false, true);
+}
+
+static void dm_test_dp_write_dsc_enable_sst(struct kunit *test,
+					   enum display_dongle_type dongle_type,
+					   bool enable,
+					   u8 expected_value)
+{
+	struct dm_test_synaptics_aux *fixture;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_stream_state *stream;
+	struct dc_link *link;
+	bool ret;
+
+	link = dm_kunit_alloc_link(test);
+	aconnector = dm_test_alloc_dsc_connector(test, link);
+	stream = dm_test_alloc_dsc_stream(test, link, SIGNAL_TYPE_DISPLAY_PORT);
+	fixture = dm_test_alloc_synaptics_aux_with_dev(test, NULL);
+
+	stream->dm_stream_context = aconnector;
+	link->priv = aconnector;
+	link->dpcd_caps.dongle_type = dongle_type;
+	dm_test_current_aux_recorder = fixture;
+	aconnector->dm_dp_aux.aux.transfer = dm_test_current_aux_transfer;
+	drm_dp_aux_init(&aconnector->dm_dp_aux.aux);
+
+	ret = dm_helpers_dp_write_dsc_enable(NULL, stream, enable);
+
+	KUNIT_EXPECT_TRUE(test, ret);
+	KUNIT_EXPECT_EQ(test, fixture->dsc_enable_writes, 1U);
+	KUNIT_EXPECT_EQ(test, fixture->dsc_enable_values[0], expected_value);
+}
+
+/**
+ * dm_test_dp_write_dsc_enable_sst_rx_enable - Test SST RX enable DPCD write
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_write_dsc_enable_sst_rx_enable(struct kunit *test)
+{
+	dm_test_dp_write_dsc_enable_sst(test, DISPLAY_DONGLE_NONE, true, 1);
+}
+
+/**
+ * dm_test_dp_write_dsc_enable_sst_rx_disable - Test SST RX disable DPCD write
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_write_dsc_enable_sst_rx_disable(struct kunit *test)
+{
+	dm_test_dp_write_dsc_enable_sst(test, DISPLAY_DONGLE_NONE, false, 0);
+}
+
+/**
+ * dm_test_dp_write_dsc_enable_pcon_enable - Test DP-HDMI PCON enable DPCD write
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_write_dsc_enable_pcon_enable(struct kunit *test)
+{
+	dm_test_dp_write_dsc_enable_sst(test, DISPLAY_DONGLE_DP_HDMI_CONVERTER, true, 1);
+}
+
+/**
+ * dm_test_dp_write_dsc_enable_pcon_disable - Test DP-HDMI PCON disable DPCD write
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_write_dsc_enable_pcon_disable(struct kunit *test)
+{
+	dm_test_dp_write_dsc_enable_sst(test, DISPLAY_DONGLE_DP_HDMI_CONVERTER, false, 0);
+}
+
+/* Tests for dm_helpers_dp_handle_test_pattern_request() */
+
+/**
+ * dm_test_dp_handle_test_pattern_no_pipe - Test no matching pipe returns false
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_handle_test_pattern_no_pipe(struct kunit *test)
+{
+	union link_test_pattern test_pattern = {0};
+	union test_misc test_params = {0};
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_state *state;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	link = dm_kunit_alloc_link(test);
+	aconnector = dm_kunit_alloc_connector(test, adev, link);
+
+	ctx->dc = dc;
+	dc->current_state = state;
+	link->dc = dc;
+	link->priv = aconnector;
+
+	KUNIT_EXPECT_FALSE(test,
+			dm_helpers_dp_handle_test_pattern_request(ctx, link,
+							       test_pattern,
+							       test_params));
+}
+
+static struct kunit_case amdgpu_dm_helpers_test_cases[] = {
+	/* edid_extract_panel_id */
+	KUNIT_CASE(dm_test_edid_extract_panel_id_basic),
+	KUNIT_CASE(dm_test_edid_extract_panel_id_zeros),
+	/* apply_edid_quirks */
+	KUNIT_CASE(dm_test_apply_edid_quirks_dpcd_poweroff_delay),
+	KUNIT_CASE(dm_test_apply_edid_quirks_disable_fams),
+	KUNIT_CASE(dm_test_apply_edid_quirks_remove_sink_ext_caps),
+	KUNIT_CASE(dm_test_apply_edid_quirks_disable_colorimetry),
+	KUNIT_CASE(dm_test_apply_edid_quirks_skip_phy_ssc),
+	KUNIT_CASE(dm_test_apply_edid_quirks_unknown_noop),
+	/* dm_helpers_parse_edid_caps */
+	KUNIT_CASE(dm_test_parse_edid_caps_null_edid),
+	KUNIT_CASE(dm_test_parse_edid_caps_null_caps),
+	KUNIT_CASE(dm_test_parse_edid_caps_valid),
+	KUNIT_CASE(dm_test_parse_edid_caps_bad_checksum),
+	KUNIT_CASE(dm_test_parse_edid_caps_hdmi_frl),
+	KUNIT_CASE(dm_test_parse_edid_caps_hdmi_frl_dsc),
+	KUNIT_CASE(dm_test_parse_edid_caps_cea_audio),
+	KUNIT_CASE(dm_test_parse_edid_caps_cea_no_speaker),
+	/* ACPI / VBIOS / local EDID readers */
+	KUNIT_CASE(dm_test_probe_acpi_edid_no_companion),
+	KUNIT_CASE(dm_test_read_acpi_edid_debug_mask_disabled),
+	KUNIT_CASE(dm_test_read_acpi_edid_non_panel_connector),
+	KUNIT_CASE(dm_test_read_acpi_edid_force_off),
+	KUNIT_CASE(dm_test_read_vbios_edid_non_embedded),
+	KUNIT_CASE(dm_test_read_vbios_edid_missing_callback),
+	KUNIT_CASE(dm_test_read_vbios_edid_callback_error),
+	KUNIT_CASE(dm_test_read_vbios_edid_missing_fake_edid),
+	KUNIT_CASE(dm_test_read_vbios_edid_invalid_fake_edid),
+	KUNIT_CASE(dm_test_read_vbios_edid_valid),
+	/* dm_is_freesync_pcon_whitelist */
+	KUNIT_CASE(dm_test_freesync_pcon_whitelist_all_known),
+	KUNIT_CASE(dm_test_freesync_pcon_whitelist_not_in_list),
+	KUNIT_CASE(dm_test_freesync_pcon_whitelist_zero),
+	/* populate_hdmi_info_from_connector */
+	KUNIT_CASE(dm_test_populate_hdmi_scdc_present_true),
+	KUNIT_CASE(dm_test_populate_hdmi_scdc_present_false),
+	KUNIT_CASE(dm_test_populate_hdmi_frl_dsc_10bpc),
+	KUNIT_CASE(dm_test_populate_hdmi_frl_dsc_12bpc),
+	KUNIT_CASE(dm_test_populate_hdmi_frl_dsc_unknown_values),
+	/* dm_get_adaptive_sync_support_type */
+	KUNIT_CASE(dm_test_adaptive_sync_type_none_default),
+	KUNIT_CASE(dm_test_adaptive_sync_type_converter_no_conditions),
+	KUNIT_CASE(dm_test_adaptive_sync_type_converter_partial_conditions),
+	KUNIT_CASE(dm_test_adaptive_sync_type_pcon_whitelist),
+	KUNIT_CASE(dm_test_adaptive_sync_type_converter_nonwhitelist),
+	/* dm_helpers_is_fullscreen / dm_helpers_is_hdr_on */
+	KUNIT_CASE(dm_test_helpers_is_fullscreen_returns_false),
+	KUNIT_CASE(dm_test_helpers_is_hdr_on_returns_false),
+	/* get_max_frl_rate */
+	KUNIT_CASE(dm_test_get_max_frl_rate_3lanes_3gbps),
+	KUNIT_CASE(dm_test_get_max_frl_rate_3lanes_6gbps),
+	KUNIT_CASE(dm_test_get_max_frl_rate_4lanes_6gbps),
+	KUNIT_CASE(dm_test_get_max_frl_rate_4lanes_8gbps),
+	KUNIT_CASE(dm_test_get_max_frl_rate_4lanes_10gbps),
+	KUNIT_CASE(dm_test_get_max_frl_rate_4lanes_12gbps),
+	KUNIT_CASE(dm_test_get_max_frl_rate_unknown),
+	/* dm_dtn_log_begin / dm_dtn_log_append_v / dm_dtn_log_end */
+	KUNIT_CASE(dm_test_dtn_log_buffer_accumulates),
+	KUNIT_CASE(dm_test_dtn_log_null_ctx_no_crash),
+	/* dm_helpers_dp_read_dpcd / dm_helpers_dp_write_dpcd */
+	KUNIT_CASE(dm_test_dp_read_dpcd_null_priv),
+	KUNIT_CASE(dm_test_dp_write_dpcd_null_priv),
+	KUNIT_CASE(dm_test_dp_read_dpcd_success),
+	KUNIT_CASE(dm_test_dp_write_dpcd_success),
+	/* dm_helpers_execute_fused_io */
+	KUNIT_CASE(dm_test_execute_fused_io_null_dmub_srv),
+	/* Synaptics RC/FIFO/DSC helpers */
+	KUNIT_CASE(dm_test_execute_synaptics_rc_command_write_success),
+	KUNIT_CASE(dm_test_execute_synaptics_rc_command_read_success),
+	KUNIT_CASE(dm_test_execute_synaptics_rc_command_write_fail),
+	KUNIT_CASE(dm_test_apply_synaptics_fifo_reset_wa_full),
+	KUNIT_CASE(dm_test_apply_synaptics_fifo_reset_wa_first_fail),
+	KUNIT_CASE(dm_test_write_dsc_enable_synaptics_enable_inactive),
+	KUNIT_CASE(dm_test_write_dsc_enable_synaptics_enable_active),
+	KUNIT_CASE(dm_test_write_dsc_enable_synaptics_disable_inactive),
+	KUNIT_CASE(dm_test_write_dsc_enable_synaptics_disable_active),
+	KUNIT_CASE(dm_test_write_dsc_enable_synaptics_enable_non_synaptics),
+	KUNIT_CASE(dm_test_dp_write_dsc_enable_routes_synaptics),
+	/* dm_helpers_dp_mst_start_top_mgr / dm_helpers_dp_mst_stop_top_mgr */
+	KUNIT_CASE(dm_test_mst_start_top_mgr_null_priv),
+	KUNIT_CASE(dm_test_mst_stop_top_mgr_null_priv),
+	KUNIT_CASE(dm_test_mst_start_top_mgr_boot),
+	KUNIT_CASE(dm_test_mst_start_top_mgr_set_mst_fail),
+	KUNIT_CASE(dm_test_mst_stop_top_mgr_active),
+	/* dm_helpers_dp_write_hblank_reduction */
+	KUNIT_CASE(dm_test_dp_write_hblank_reduction_false),
+	/* get_dsc_max_slices */
+	KUNIT_CASE(dm_test_get_dsc_max_slices_1_340),
+	KUNIT_CASE(dm_test_get_dsc_max_slices_2_340),
+	KUNIT_CASE(dm_test_get_dsc_max_slices_4_340),
+	KUNIT_CASE(dm_test_get_dsc_max_slices_8_340),
+	KUNIT_CASE(dm_test_get_dsc_max_slices_8_400),
+	KUNIT_CASE(dm_test_get_dsc_max_slices_12_400),
+	KUNIT_CASE(dm_test_get_dsc_max_slices_16_400),
+	KUNIT_CASE(dm_test_get_dsc_max_slices_unknown),
+	/* dm_helpers_init_panel_settings */
+	KUNIT_CASE(dm_test_init_panel_settings_pps),
+	KUNIT_CASE(dm_test_init_panel_settings_dsc),
+	/* dm_helpers_override_panel_settings */
+	KUNIT_CASE(dm_test_override_panel_settings_debug_mask_disables_dsc),
+	KUNIT_CASE(dm_test_override_panel_settings_second_edp_disables_psr),
+	/* fill_dc_mst_payload_table_from_drm */
+	KUNIT_CASE(dm_test_fill_mst_payload_table_enable),
+	KUNIT_CASE(dm_test_fill_mst_payload_table_disable),
+	KUNIT_CASE(dm_test_fill_mst_payload_table_empty),
+	/* dm_helpers_submit_i2c */
+	KUNIT_CASE(dm_test_submit_i2c_null_priv),
+	KUNIT_CASE(dm_test_submit_i2c_success),
+	KUNIT_CASE(dm_test_submit_i2c_partial_transfer),
+	/* dm_helper_dmub_aux_transfer_sync */
+	KUNIT_CASE(dm_test_dmub_aux_transfer_sync_hpd_discon),
+	/* Empty stub functions */
+	KUNIT_CASE(dm_test_dp_update_branch_info_no_crash),
+	KUNIT_CASE(dm_test_mst_poll_pending_down_reply_no_crash),
+	KUNIT_CASE(dm_test_mst_clear_payload_alloc_table_no_crash),
+	KUNIT_CASE(dm_test_set_dcn_clocks_no_crash),
+	KUNIT_CASE(dm_test_dmu_timeout_no_crash),
+	KUNIT_CASE(dm_test_smu_timeout_no_crash),
+	KUNIT_CASE(dm_test_set_phyd32clk_no_crash),
+	KUNIT_CASE(dm_test_mst_update_branch_bandwidth_no_crash),
+	/* dm_helpers_dp_mst_write_payload_allocation_table success path */
+	KUNIT_CASE(dm_test_write_payload_alloc_table_enable),
+	KUNIT_CASE(dm_test_write_payload_alloc_table_disable),
+	/* MST null-connector early returns */
+	KUNIT_CASE(dm_test_mst_write_payload_alloc_table_null_ctx),
+	KUNIT_CASE(dm_test_mst_poll_for_act_null_ctx),
+	/* dm_helpers_dp_mst_poll_for_allocation_change_trigger success/fail */
+	KUNIT_CASE(dm_test_poll_for_act_no_mst_state),
+	KUNIT_CASE(dm_test_poll_for_act_success),
+	KUNIT_CASE(dm_test_poll_for_act_status_failed),
+	KUNIT_CASE(dm_test_mst_send_payload_alloc_null_ctx),
+	KUNIT_CASE(dm_test_mst_update_mgr_dealloc_null_ctx),
+	/* dm_helpers_dp_mst_send_payload_allocation failure path */
+	KUNIT_CASE(dm_test_mst_send_payload_alloc_part2_fail),
+	/* dm_helpers_dp_mst_update_mst_mgr_for_deallocation success path */
+	KUNIT_CASE(dm_test_mst_update_mgr_dealloc_success),
+	/* dm_helpers_is_dp_sink_present */
+	KUNIT_CASE(dm_test_is_dp_sink_present_null_priv),
+	/* dm_helpers_dmub_outbox_interrupt_control */
+	KUNIT_CASE(dm_test_dmub_outbox_interrupt_control_null_dc),
+	/* dm_helpers_mst_enable_stream_features */
+	KUNIT_CASE(dm_test_mst_enable_stream_features_aux_disabled),
+	KUNIT_CASE(dm_test_mst_enable_stream_features_writes_downspread),
+	/* dm_helpers_enable_periodic_detection */
+	KUNIT_CASE(dm_test_enable_periodic_detection_no_workqueue),
+	KUNIT_CASE(dm_test_enable_periodic_detection_updates_enable),
+	KUNIT_CASE(dm_test_enable_periodic_detection_schedules_work),
+	/* dm_helpers_read_mccs_caps */
+	KUNIT_CASE(dm_test_read_mccs_caps_null_ctx),
+	KUNIT_CASE(dm_test_read_mccs_caps_null_link),
+	KUNIT_CASE(dm_test_read_mccs_caps_no_vcp_code),
+	KUNIT_CASE(dm_test_read_mccs_caps_i2c_vcp_request),
+	KUNIT_CASE(dm_test_read_mccs_caps_hdmi_vcp_request),
+	KUNIT_CASE(dm_test_read_mccs_caps_legacy_pcon_vcp_request),
+	KUNIT_CASE(dm_test_read_mccs_caps_i2c_failure),
+	/* dm_helpers_mccs_vcp_set */
+	KUNIT_CASE(dm_test_mccs_vcp_set_null_ctx),
+	KUNIT_CASE(dm_test_mccs_vcp_set_not_supported),
+	KUNIT_CASE(dm_test_mccs_vcp_set_null_link),
+	KUNIT_CASE(dm_test_mccs_vcp_set_i2c_packet),
+	KUNIT_CASE(dm_test_mccs_vcp_set_i2c_failure),
+	/* dm_helpers_construct_old_payload */
+	KUNIT_CASE(dm_test_construct_old_payload_empty_list),
+	KUNIT_CASE(dm_test_construct_old_payload_intervening),
+	/* dm_helpers_dp_write_dsc_enable */
+	KUNIT_CASE(dm_test_dp_write_dsc_enable_mst_no_aux),
+	KUNIT_CASE(dm_test_dp_write_dsc_enable_non_dp),
+	KUNIT_CASE(dm_test_dp_write_dsc_enable_mst_enable_decode_only),
+	KUNIT_CASE(dm_test_dp_write_dsc_enable_mst_enable_passthrough),
+	KUNIT_CASE(dm_test_dp_write_dsc_enable_mst_disable_decode_only),
+	KUNIT_CASE(dm_test_dp_write_dsc_enable_mst_disable_passthrough),
+	KUNIT_CASE(dm_test_dp_write_dsc_enable_sst_rx_enable),
+	KUNIT_CASE(dm_test_dp_write_dsc_enable_sst_rx_disable),
+	KUNIT_CASE(dm_test_dp_write_dsc_enable_pcon_enable),
+	KUNIT_CASE(dm_test_dp_write_dsc_enable_pcon_disable),
+	/* dm_helpers_dp_handle_test_pattern_request */
+	KUNIT_CASE(dm_test_dp_handle_test_pattern_no_pipe),
+	{}
+};
+
+static struct kunit_suite amdgpu_dm_helpers_test_suite = {
+	.name = "amdgpu_dm_helpers",
+	.test_cases = amdgpu_dm_helpers_test_cases,
+};
+
+kunit_test_suite(amdgpu_dm_helpers_test_suite);
+
+MODULE_DESCRIPTION("KUnit tests for amdgpu_dm_helpers");
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_irq_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_irq_test.c
new file mode 100644
index 0000000000000..ed20e278742d2
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_irq_test.c
@@ -0,0 +1,4096 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * KUnit tests for amdgpu_dm_irq.c
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#include <kunit/test.h>
+#include <drm/drm_kunit_helpers.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_probe_helper.h>
+
+#include "dc.h"
+#include "inc/core_types.h"
+#include "irq/irq_service.h"
+#include "amdgpu.h"
+#include "amdgpu_mode.h"
+#include "amdgpu_dm.h"
+#include "amdgpu_dm_irq.h"
+#include "amdgpu_dm_kunit_test_helpers.h"
+#include "dc_dmub_srv.h"
+#include "ivsrcid/ivsrcid_vislands30.h"
+#include "ivsrcid/dcn/irqsrcs_dcn_1_0.h"
+#include "link_service.h"
+#include "dmub/dmub_srv.h"
+#include "dal_asic_id.h"
+#include "../../amdgpu/amdgpu_reset.h"
+
+static void dm_test_irq_handler(void *arg)
+{
+}
+
+static void dm_test_irq_handler_alt(void *arg)
+{
+}
+
+static void dm_test_irq_handler_count(void *arg)
+{
+	int *count = arg;
+
+	if (count)
+		(*count)++;
+}
+
+static bool dm_test_detect_connection_none(struct dc_link *link,
+					   enum dc_connection_type *type)
+{
+	*type = dc_connection_none;
+
+	return true;
+}
+
+static bool dm_test_detect_link_false(struct dc_link *link,
+				      enum dc_detect_reason reason)
+{
+	return false;
+}
+
+static bool dm_test_detect_connection_single(struct dc_link *link,
+					     enum dc_connection_type *type)
+{
+	*type = dc_connection_single;
+
+	return true;
+}
+
+/* Recording stubs for the dm_handle_hpd_rx_offload_work() DP-IRQ branches. */
+static int dm_test_automated_test_count;
+static int dm_test_handle_link_loss_count;
+
+static void dm_test_dp_handle_automated_test(struct dc_link *link)
+{
+	dm_test_automated_test_count++;
+}
+
+static void dm_test_dp_handle_link_loss(struct dc_link *link)
+{
+	dm_test_handle_link_loss_count++;
+}
+
+static bool dm_test_dp_parse_link_loss_true(struct dc_link *link,
+					    union hpd_irq_data *hpd_irq_dpcd_data)
+{
+	return true;
+}
+
+static bool dm_test_dp_should_allow_hpd_rx_irq_true(const struct dc_link *link)
+{
+	return true;
+}
+
+static enum dc_status dm_test_dp_read_hpd_rx_irq_data_ok(struct dc_link *link,
+							 union hpd_irq_data *irq_data)
+{
+	return DC_OK;
+}
+
+/*
+ * Allocate a refcounted dc_sink for tests without pulling in the DC-core
+ * dc_sink_create()/dc_sink_release() symbols. Production code under test still
+ * uses dc_sink_retain()/dc_sink_release() (resolved inside the amdgpu module).
+ * Use kzalloc (not kunit_kzalloc) so the final kref_put frees it exactly once.
+ */
+static struct dc_sink *dm_test_sink_create(struct dc_link *link)
+{
+	struct dc_sink *sink = kzalloc(sizeof(*sink), GFP_KERNEL);
+
+	if (!sink)
+		return NULL;
+
+	sink->link = link;
+	sink->ctx = link->ctx;
+	kref_init(&sink->refcount);
+
+	return sink;
+}
+
+static void dm_test_sink_free(struct kref *kref)
+{
+	struct dc_sink *sink = container_of(kref, struct dc_sink, refcount);
+
+	kfree(sink->dc_container_id);
+	kfree(sink);
+}
+
+static void dm_test_sink_release(struct dc_sink *sink)
+{
+	kref_put(&sink->refcount, dm_test_sink_free);
+}
+
+static bool dm_test_handle_hpd_rx_no_work(struct dc_link *link,
+					  union hpd_irq_data *hpd_irq_data,
+					  bool *link_loss,
+					  bool defer_handling,
+					  bool *has_left_work)
+{
+	*link_loss = false;
+	*has_left_work = false;
+
+	return false;
+}
+
+static bool dm_test_handle_hpd_rx_automated(struct dc_link *link,
+					    union hpd_irq_data *hpd_irq_data,
+					    bool *link_loss,
+					    bool defer_handling,
+					    bool *has_left_work)
+{
+	*link_loss = false;
+	*has_left_work = true;
+	hpd_irq_data->bytes.device_service_irq.bits.AUTOMATED_TEST = 1;
+
+	return false;
+}
+
+static bool dm_test_handle_hpd_rx_msg_rdy(struct dc_link *link,
+					  union hpd_irq_data *hpd_irq_data,
+					  bool *link_loss,
+					  bool defer_handling,
+					  bool *has_left_work)
+{
+	*link_loss = false;
+	*has_left_work = true;
+	hpd_irq_data->bytes.device_service_irq.bits.UP_REQ_MSG_RDY = 1;
+
+	return false;
+}
+
+static bool dm_test_handle_hpd_rx_link_loss(struct dc_link *link,
+					    union hpd_irq_data *hpd_irq_data,
+					    bool *link_loss,
+					    bool defer_handling,
+					    bool *has_left_work)
+{
+	*link_loss = true;
+	*has_left_work = true;
+
+	return false;
+}
+
+static bool dm_test_allow_hpd_rx_irq_true(const struct dc_link *link)
+{
+	return true;
+}
+
+
+static uint32_t dm_test_dmub_get_outbox0_wptr(struct dmub_srv *dmub)
+{
+	return 0;
+}
+
+static uint32_t dm_test_dmub_get_outbox1_wptr(struct dmub_srv *dmub)
+{
+	return 0;
+}
+
+static int dm_test_dmub_notify_count;
+
+static void dm_test_dmub_notify_callback(struct amdgpu_device *adev,
+					 struct dmub_notification *notify)
+{
+	dm_test_dmub_notify_count++;
+}
+
+static struct dc *dm_test_alloc_dc_with_ctx(struct kunit *test)
+{
+	struct dc_context *ctx;
+	struct dc *dc;
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+
+	dc->ctx = ctx;
+	ctx->dc = dc;
+
+	return dc;
+}
+
+static enum dc_irq_source dm_test_to_dal_irq_source_dce110(
+		struct irq_service *irq_service,
+		uint32_t src_id,
+		uint32_t ext_id)
+{
+	switch (src_id) {
+	case VISLANDS30_IV_SRCID_D1_VERTICAL_INTERRUPT0:
+		return DC_IRQ_SOURCE_VBLANK1;
+	case VISLANDS30_IV_SRCID_D1_V_UPDATE_INT:
+		return DC_IRQ_SOURCE_VUPDATE1;
+	case VISLANDS30_IV_SRCID_D1_GRPH_PFLIP:
+		return DC_IRQ_SOURCE_PFLIP1;
+	case VISLANDS30_IV_SRCID_D1_GRPH_PFLIP + 2:
+		return DC_IRQ_SOURCE_PFLIP2;
+	case VISLANDS30_IV_SRCID_D1_GRPH_PFLIP + 4:
+		return DC_IRQ_SOURCE_PFLIP3;
+	case VISLANDS30_IV_SRCID_D1_GRPH_PFLIP + 6:
+		return DC_IRQ_SOURCE_PFLIP4;
+	case VISLANDS30_IV_SRCID_D1_GRPH_PFLIP + 8:
+		return DC_IRQ_SOURCE_PFLIP5;
+	case VISLANDS30_IV_SRCID_D1_GRPH_PFLIP + 10:
+		return DC_IRQ_SOURCE_PFLIP6;
+	default:
+		return DC_IRQ_SOURCE_INVALID;
+	}
+}
+
+static const struct irq_service_funcs dm_test_irq_service_funcs_dce110 = {
+	.to_dal_irq_source = dm_test_to_dal_irq_source_dce110
+};
+
+static enum dc_irq_source dm_test_to_dal_irq_source_dcn10(
+		struct irq_service *irq_service,
+		uint32_t src_id,
+		uint32_t ext_id)
+{
+	switch (src_id) {
+	case DCN_1_0__SRCID__DC_D1_OTG_VSTARTUP:
+		return DC_IRQ_SOURCE_VBLANK1;
+	case DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+		return DC_IRQ_SOURCE_VUPDATE1;
+	case DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT:
+		return DC_IRQ_SOURCE_PFLIP1;
+	case DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT:
+		return DC_IRQ_SOURCE_DMCUB_OUTBOX;
+	default:
+		return DC_IRQ_SOURCE_INVALID;
+	}
+}
+
+static const struct irq_service_funcs dm_test_irq_service_funcs_dcn10 = {
+	.to_dal_irq_source = dm_test_to_dal_irq_source_dcn10
+};
+
+static bool dm_test_irq_src_set(struct irq_service *irq_service,
+				const struct irq_source_info *info, bool enable)
+{
+	return true;
+}
+
+static bool dm_test_irq_src_ack(struct irq_service *irq_service,
+				const struct irq_source_info *info)
+{
+	return true;
+}
+
+/* Per-source funcs let dc_interrupt_set() succeed without register access. */
+static struct irq_source_info_funcs dm_test_irq_src_funcs = {
+	.set = dm_test_irq_src_set,
+	.ack = dm_test_irq_src_ack,
+};
+
+static struct dc *dm_test_alloc_dc_with_irq_service(struct kunit *test,
+						    const struct irq_service_funcs *funcs)
+{
+	struct irq_source_info *info;
+	struct resource_pool *res_pool;
+	struct irq_service *irqs;
+	struct dc *dc;
+	int i;
+
+	dc = dm_test_alloc_dc_with_ctx(test);
+	res_pool = kunit_kzalloc(test, sizeof(*res_pool), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, res_pool);
+	irqs = kunit_kzalloc(test, sizeof(*irqs), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, irqs);
+
+	/*
+	 * Populate the per-source info table so dc_interrupt_set()/_ack()
+	 * succeed without touching hardware registers.
+	 */
+	info = kunit_kzalloc(test, sizeof(*info) * DAL_IRQ_SOURCES_NUMBER,
+			     GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, info);
+	for (i = 0; i < DAL_IRQ_SOURCES_NUMBER; i++)
+		info[i].funcs = &dm_test_irq_src_funcs;
+
+	irqs->funcs = funcs;
+	irqs->info = info;
+	res_pool->irqs = irqs;
+	dc->res_pool = res_pool;
+
+	return dc;
+}
+
+static void dm_test_free_irq_sources(void *data)
+{
+	struct amdgpu_device *adev = data;
+	int i;
+
+	for (i = 0; i < AMDGPU_IRQ_CLIENTID_MAX; i++) {
+		kfree(adev->irq.client[i].sources);
+		adev->irq.client[i].sources = NULL;
+	}
+
+	kfree(adev->crtc_irq.enabled_types);
+	adev->crtc_irq.enabled_types = NULL;
+	kfree(adev->vline0_irq.enabled_types);
+	adev->vline0_irq.enabled_types = NULL;
+	kfree(adev->vupdate_irq.enabled_types);
+	adev->vupdate_irq.enabled_types = NULL;
+	kfree(adev->pageflip_irq.enabled_types);
+	adev->pageflip_irq.enabled_types = NULL;
+	kfree(adev->dmub_outbox_irq.enabled_types);
+	adev->dmub_outbox_irq.enabled_types = NULL;
+	kfree(adev->dmub_trace_irq.enabled_types);
+	adev->dmub_trace_irq.enabled_types = NULL;
+	kfree(adev->hpd_irq.enabled_types);
+	adev->hpd_irq.enabled_types = NULL;
+}
+
+static void dm_test_crtc_list_del(void *data)
+{
+	struct amdgpu_crtc *acrtc = data;
+
+	list_del_init(&acrtc->base.head);
+}
+
+struct dm_test_hpd_rx_wq_ctx {
+	struct hpd_rx_irq_offload_work_queue *wq;
+	int count;
+};
+
+static void dm_test_destroy_hpd_rx_wq(void *data)
+{
+	struct dm_test_hpd_rx_wq_ctx *ctx = data;
+	int i;
+
+	for (i = 0; i < ctx->count; i++)
+		if (ctx->wq[i].wq)
+			destroy_workqueue(ctx->wq[i].wq);
+	kfree(ctx->wq);
+}
+
+static const struct drm_connector_funcs dm_test_connector_funcs = {
+	.reset = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = drm_connector_cleanup,
+};
+
+static void dm_test_connector_cleanup(void *data)
+{
+	drm_connector_cleanup(data);
+}
+
+/* Tests for amdgpu_dm_hpd_to_dal_irq_source() */
+
+/**
+ * dm_test_hpd_to_dal_irq_source_hpd1 - Test Hpd to dal irq source hpd1
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_to_dal_irq_source_hpd1(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_hpd_to_dal_irq_source(AMDGPU_HPD_1),
+			(int)DC_IRQ_SOURCE_HPD1);
+}
+
+/**
+ * dm_test_hpd_to_dal_irq_source_hpd2 - Test Hpd to dal irq source hpd2
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_to_dal_irq_source_hpd2(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_hpd_to_dal_irq_source(AMDGPU_HPD_2),
+			(int)DC_IRQ_SOURCE_HPD2);
+}
+
+/**
+ * dm_test_hpd_to_dal_irq_source_hpd3 - Test Hpd to dal irq source hpd3
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_to_dal_irq_source_hpd3(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_hpd_to_dal_irq_source(AMDGPU_HPD_3),
+			(int)DC_IRQ_SOURCE_HPD3);
+}
+
+/**
+ * dm_test_hpd_to_dal_irq_source_hpd4 - Test Hpd to dal irq source hpd4
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_to_dal_irq_source_hpd4(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_hpd_to_dal_irq_source(AMDGPU_HPD_4),
+			(int)DC_IRQ_SOURCE_HPD4);
+}
+
+/**
+ * dm_test_hpd_to_dal_irq_source_hpd5 - Test Hpd to dal irq source hpd5
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_to_dal_irq_source_hpd5(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_hpd_to_dal_irq_source(AMDGPU_HPD_5),
+			(int)DC_IRQ_SOURCE_HPD5);
+}
+
+/**
+ * dm_test_hpd_to_dal_irq_source_hpd6 - Test Hpd to dal irq source hpd6
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_to_dal_irq_source_hpd6(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_hpd_to_dal_irq_source(AMDGPU_HPD_6),
+			(int)DC_IRQ_SOURCE_HPD6);
+}
+
+/**
+ * dm_test_hpd_to_dal_irq_source_invalid - Test Hpd to dal irq source invalid
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_to_dal_irq_source_invalid(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_hpd_to_dal_irq_source(AMDGPU_HPD_NONE),
+			(int)DC_IRQ_SOURCE_INVALID);
+}
+
+/**
+ * dm_test_hpd_to_dal_irq_source_out_of_range - Test Hpd to dal irq source out of range
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_to_dal_irq_source_out_of_range(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_hpd_to_dal_irq_source(99),
+			(int)DC_IRQ_SOURCE_INVALID);
+}
+
+/* Tests for are_sinks_equal() */
+
+/**
+ * dm_test_are_sinks_equal_both_null - Test Are sinks equal both null
+ * @test: The KUnit test context
+ */
+static void dm_test_are_sinks_equal_both_null(struct kunit *test)
+{
+	KUNIT_EXPECT_FALSE(test, are_sinks_equal(NULL, NULL));
+}
+
+/**
+ * dm_test_are_sinks_equal_first_null - Test Are sinks equal first null
+ * @test: The KUnit test context
+ */
+static void dm_test_are_sinks_equal_first_null(struct kunit *test)
+{
+	struct dc_sink *sink2;
+
+	sink2 = kunit_kzalloc(test, sizeof(*sink2), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink2);
+
+	KUNIT_EXPECT_FALSE(test, are_sinks_equal(NULL, sink2));
+}
+
+/**
+ * dm_test_are_sinks_equal_second_null - Test Are sinks equal second null
+ * @test: The KUnit test context
+ */
+static void dm_test_are_sinks_equal_second_null(struct kunit *test)
+{
+	struct dc_sink *sink1;
+
+	sink1 = kunit_kzalloc(test, sizeof(*sink1), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink1);
+
+	KUNIT_EXPECT_FALSE(test, are_sinks_equal(sink1, NULL));
+}
+
+/**
+ * dm_test_are_sinks_equal_different_signal - Test Are sinks equal different signal
+ * @test: The KUnit test context
+ */
+static void dm_test_are_sinks_equal_different_signal(struct kunit *test)
+{
+	struct dc_sink *sink1, *sink2;
+
+	sink1 = kunit_kzalloc(test, sizeof(*sink1), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink1);
+	sink2 = kunit_kzalloc(test, sizeof(*sink2), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink2);
+
+	sink1->sink_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	sink2->sink_signal = SIGNAL_TYPE_DISPLAY_PORT;
+
+	KUNIT_EXPECT_FALSE(test, are_sinks_equal(sink1, sink2));
+}
+
+/**
+ * dm_test_are_sinks_equal_different_edid_length - Test Are sinks equal different edid length
+ * @test: The KUnit test context
+ */
+static void dm_test_are_sinks_equal_different_edid_length(struct kunit *test)
+{
+	struct dc_sink *sink1, *sink2;
+
+	sink1 = kunit_kzalloc(test, sizeof(*sink1), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink1);
+	sink2 = kunit_kzalloc(test, sizeof(*sink2), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink2);
+
+	sink1->sink_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	sink2->sink_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	sink1->dc_edid.length = 128;
+	sink2->dc_edid.length = 256;
+
+	KUNIT_EXPECT_FALSE(test, are_sinks_equal(sink1, sink2));
+}
+
+/**
+ * dm_test_are_sinks_equal_different_edid_data - Test Are sinks equal different edid data
+ * @test: The KUnit test context
+ */
+static void dm_test_are_sinks_equal_different_edid_data(struct kunit *test)
+{
+	struct dc_sink *sink1, *sink2;
+
+	sink1 = kunit_kzalloc(test, sizeof(*sink1), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink1);
+	sink2 = kunit_kzalloc(test, sizeof(*sink2), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink2);
+
+	sink1->sink_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	sink2->sink_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	sink1->dc_edid.length = 4;
+	sink2->dc_edid.length = 4;
+	memset(sink1->dc_edid.raw_edid, 0xAA, 4);
+	memset(sink2->dc_edid.raw_edid, 0xBB, 4);
+
+	KUNIT_EXPECT_FALSE(test, are_sinks_equal(sink1, sink2));
+}
+
+/**
+ * dm_test_are_sinks_equal_identical - Test Are sinks equal identical
+ * @test: The KUnit test context
+ */
+static void dm_test_are_sinks_equal_identical(struct kunit *test)
+{
+	struct dc_sink *sink1, *sink2;
+
+	sink1 = kunit_kzalloc(test, sizeof(*sink1), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink1);
+	sink2 = kunit_kzalloc(test, sizeof(*sink2), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink2);
+
+	sink1->sink_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	sink2->sink_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	sink1->dc_edid.length = 4;
+	sink2->dc_edid.length = 4;
+	memset(sink1->dc_edid.raw_edid, 0xAA, 4);
+	memset(sink2->dc_edid.raw_edid, 0xAA, 4);
+
+	KUNIT_EXPECT_TRUE(test, are_sinks_equal(sink1, sink2));
+}
+
+/**
+ * dm_test_are_sinks_equal_zero_length - Test Are sinks equal zero length
+ * @test: The KUnit test context
+ */
+static void dm_test_are_sinks_equal_zero_length(struct kunit *test)
+{
+	struct dc_sink *sink1, *sink2;
+
+	sink1 = kunit_kzalloc(test, sizeof(*sink1), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink1);
+	sink2 = kunit_kzalloc(test, sizeof(*sink2), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink2);
+
+	sink1->sink_signal = SIGNAL_TYPE_DISPLAY_PORT;
+	sink2->sink_signal = SIGNAL_TYPE_DISPLAY_PORT;
+	sink1->dc_edid.length = 0;
+	sink2->dc_edid.length = 0;
+
+	KUNIT_EXPECT_TRUE(test, are_sinks_equal(sink1, sink2));
+}
+
+/**
+ * dm_test_are_sinks_equal_full_edid_identical - Test Are sinks equal full edid identical
+ * @test: The KUnit test context
+ */
+static void dm_test_are_sinks_equal_full_edid_identical(struct kunit *test)
+{
+	struct dc_sink *sink1, *sink2;
+
+	sink1 = kunit_kzalloc(test, sizeof(*sink1), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink1);
+	sink2 = kunit_kzalloc(test, sizeof(*sink2), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink2);
+
+	sink1->sink_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	sink2->sink_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	sink1->dc_edid.length = 128;
+	sink2->dc_edid.length = 128;
+	memset(sink1->dc_edid.raw_edid, 0x5A, 128);
+	memset(sink2->dc_edid.raw_edid, 0x5A, 128);
+
+	KUNIT_EXPECT_TRUE(test, are_sinks_equal(sink1, sink2));
+}
+
+/**
+ * dm_test_are_sinks_equal_full_edid_last_byte_differs - Test Are sinks equal last byte differs
+ * @test: The KUnit test context
+ */
+static void dm_test_are_sinks_equal_full_edid_last_byte_differs(struct kunit *test)
+{
+	struct dc_sink *sink1, *sink2;
+
+	sink1 = kunit_kzalloc(test, sizeof(*sink1), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink1);
+	sink2 = kunit_kzalloc(test, sizeof(*sink2), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink2);
+
+	sink1->sink_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	sink2->sink_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	sink1->dc_edid.length = 128;
+	sink2->dc_edid.length = 128;
+	memset(sink1->dc_edid.raw_edid, 0x5A, 128);
+	memset(sink2->dc_edid.raw_edid, 0x5A, 128);
+	sink2->dc_edid.raw_edid[127] = 0x5B;
+
+	KUNIT_EXPECT_FALSE(test, are_sinks_equal(sink1, sink2));
+}
+
+/* Tests for dmub_notification_type_str() */
+
+/**
+ * dm_test_notification_str_no_data - Test Notification str no data
+ * @test: The KUnit test context
+ */
+static void dm_test_notification_str_no_data(struct kunit *test)
+{
+	KUNIT_EXPECT_STREQ(test, dmub_notification_type_str(DMUB_NOTIFICATION_NO_DATA), "NO_DATA");
+}
+
+/**
+ * dm_test_notification_str_aux_reply - Test Notification str aux reply
+ * @test: The KUnit test context
+ */
+static void dm_test_notification_str_aux_reply(struct kunit *test)
+{
+	KUNIT_EXPECT_STREQ(test, dmub_notification_type_str(DMUB_NOTIFICATION_AUX_REPLY), "AUX_REPLY");
+}
+
+/**
+ * dm_test_notification_str_hpd - Test Notification str hpd
+ * @test: The KUnit test context
+ */
+static void dm_test_notification_str_hpd(struct kunit *test)
+{
+	KUNIT_EXPECT_STREQ(test, dmub_notification_type_str(DMUB_NOTIFICATION_HPD), "HPD");
+}
+
+/**
+ * dm_test_notification_str_hpd_irq - Test Notification str hpd irq
+ * @test: The KUnit test context
+ */
+static void dm_test_notification_str_hpd_irq(struct kunit *test)
+{
+	KUNIT_EXPECT_STREQ(test, dmub_notification_type_str(DMUB_NOTIFICATION_HPD_IRQ), "HPD_IRQ");
+}
+
+/**
+ * dm_test_notification_str_set_config - Test Notification str set config
+ * @test: The KUnit test context
+ */
+static void dm_test_notification_str_set_config(struct kunit *test)
+{
+	KUNIT_EXPECT_STREQ(test, dmub_notification_type_str(DMUB_NOTIFICATION_SET_CONFIG_REPLY),
+			   "SET_CONFIG_REPLY");
+}
+
+/**
+ * dm_test_notification_str_dpia - Test Notification str dpia
+ * @test: The KUnit test context
+ */
+static void dm_test_notification_str_dpia(struct kunit *test)
+{
+	KUNIT_EXPECT_STREQ(test, dmub_notification_type_str(DMUB_NOTIFICATION_DPIA_NOTIFICATION),
+			   "DPIA_NOTIFICATION");
+}
+
+/**
+ * dm_test_notification_str_hpd_sense - Test Notification str hpd sense
+ * @test: The KUnit test context
+ */
+static void dm_test_notification_str_hpd_sense(struct kunit *test)
+{
+	KUNIT_EXPECT_STREQ(test, dmub_notification_type_str(DMUB_NOTIFICATION_HPD_SENSE_NOTIFY),
+			   "HPD_SENSE_NOTIFY");
+}
+
+/**
+ * dm_test_notification_str_fused_io - Test Notification str fused io
+ * @test: The KUnit test context
+ */
+static void dm_test_notification_str_fused_io(struct kunit *test)
+{
+	KUNIT_EXPECT_STREQ(test, dmub_notification_type_str(DMUB_NOTIFICATION_FUSED_IO),
+			   "FUSED_IO");
+}
+
+/**
+ * dm_test_notification_str_unknown - Test Notification str unknown
+ * @test: The KUnit test context
+ */
+static void dm_test_notification_str_unknown(struct kunit *test)
+{
+	KUNIT_EXPECT_STREQ(test, dmub_notification_type_str(DMUB_NOTIFICATION_MAX), "<unknown>");
+}
+
+/* Tests for amdgpu_dm_irq_init() */
+
+/**
+ * dm_test_irq_init_initializes_lists - Test irq init initializes list heads
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_init_initializes_lists(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	int src;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++) {
+		KUNIT_EXPECT_TRUE(test,
+				  list_empty(&adev->dm.irq_handler_list_low_tab[src]));
+		KUNIT_EXPECT_TRUE(test,
+				  list_empty(&adev->dm.irq_handler_list_high_tab[src]));
+	}
+}
+
+/* Tests for amdgpu_dm_irq_register_interrupt() */
+
+/**
+ * dm_test_irq_register_rejects_null_params - Test register rejects null params
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_register_rejects_null_params(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_interrupt_params int_params = { 0 };
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_HPD1;
+
+	KUNIT_EXPECT_NULL(test,
+		amdgpu_dm_irq_register_interrupt(adev, NULL,
+						   dm_test_irq_handler, NULL));
+	KUNIT_EXPECT_NULL(test,
+		amdgpu_dm_irq_register_interrupt(adev, &int_params, NULL, NULL));
+}
+
+/**
+ * dm_test_irq_register_rejects_invalid_context - Test register rejects context
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_register_rejects_invalid_context(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_interrupt_params int_params = { 0 };
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	int_params.int_context = INTERRUPT_CONTEXT_NUMBER;
+	int_params.irq_source = DC_IRQ_SOURCE_HPD1;
+
+	KUNIT_EXPECT_NULL(test,
+		amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						   dm_test_irq_handler, NULL));
+}
+
+/**
+ * dm_test_irq_register_rejects_invalid_source - Test register rejects source
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_register_rejects_invalid_source(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_interrupt_params int_params = { 0 };
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_INVALID;
+
+	KUNIT_EXPECT_NULL(test,
+		amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						   dm_test_irq_handler, NULL));
+}
+
+/**
+ * dm_test_irq_register_adds_low_context_handler - Test register adds low handler
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_register_adds_low_context_handler(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_interrupt_params int_params = { 0 };
+	void *handler;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_HPD1;
+
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						    dm_test_irq_handler, adev);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+	KUNIT_EXPECT_FALSE(test,
+			   list_empty(&adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_HPD1]));
+	KUNIT_EXPECT_TRUE(test,
+			  list_empty(&adev->dm.irq_handler_list_high_tab[DC_IRQ_SOURCE_HPD1]));
+
+	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD1,
+						dm_test_irq_handler);
+	KUNIT_EXPECT_TRUE(test,
+			  list_empty(&adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_HPD1]));
+}
+
+/**
+ * dm_test_irq_register_adds_high_context_handler - Test register adds high handler
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_register_adds_high_context_handler(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_interrupt_params int_params = { 0 };
+	void *handler;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_HPD2;
+
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						    dm_test_irq_handler, adev);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+	KUNIT_EXPECT_FALSE(test,
+			   list_empty(&adev->dm.irq_handler_list_high_tab[DC_IRQ_SOURCE_HPD2]));
+	KUNIT_EXPECT_TRUE(test,
+			  list_empty(&adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_HPD2]));
+
+	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD2,
+						dm_test_irq_handler);
+	KUNIT_EXPECT_TRUE(test,
+			  list_empty(&adev->dm.irq_handler_list_high_tab[DC_IRQ_SOURCE_HPD2]));
+}
+
+/**
+ * dm_test_irq_register_multiple_handlers - Test register keeps multiple handlers
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_register_multiple_handlers(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_interrupt_params int_params = { 0 };
+	struct list_head *hnd_list;
+	void *handler1, *handler2;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_HPD1;
+
+	handler1 = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						    dm_test_irq_handler, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler1);
+	handler2 = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						    dm_test_irq_handler_alt, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler2);
+
+	hnd_list = &adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_HPD1];
+	KUNIT_EXPECT_EQ(test, list_count_nodes(hnd_list), 2);
+
+	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD1,
+						dm_test_irq_handler);
+	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD1,
+						dm_test_irq_handler_alt);
+	KUNIT_EXPECT_TRUE(test, list_empty(hnd_list));
+}
+
+/**
+ * dm_test_irq_register_separate_contexts - Test register same source in two contexts
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_register_separate_contexts(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_interrupt_params int_params = { 0 };
+	void *handler;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	int_params.irq_source = DC_IRQ_SOURCE_HPD5;
+
+	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						    dm_test_irq_handler, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+
+	int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						    dm_test_irq_handler, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+
+	KUNIT_EXPECT_FALSE(test,
+			   list_empty(&adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_HPD5]));
+	KUNIT_EXPECT_FALSE(test,
+			   list_empty(&adev->dm.irq_handler_list_high_tab[DC_IRQ_SOURCE_HPD5]));
+
+	/*
+	 * A single unregister call stops at the first context where the handler
+	 * is found (low context), leaving the high context handler in place.
+	 */
+	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD5,
+						dm_test_irq_handler);
+
+	KUNIT_EXPECT_TRUE(test,
+			  list_empty(&adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_HPD5]));
+	KUNIT_EXPECT_FALSE(test,
+			   list_empty(&adev->dm.irq_handler_list_high_tab[DC_IRQ_SOURCE_HPD5]));
+
+	/* A second call removes the remaining high context handler. */
+	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD5,
+						dm_test_irq_handler);
+
+	KUNIT_EXPECT_TRUE(test,
+			  list_empty(&adev->dm.irq_handler_list_high_tab[DC_IRQ_SOURCE_HPD5]));
+}
+
+/* Tests for amdgpu_dm_irq_unregister_interrupt() */
+
+/**
+ * dm_test_irq_unregister_rejects_invalid_source - Test unregister rejects source
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_unregister_rejects_invalid_source(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_INVALID,
+						dm_test_irq_handler);
+
+	KUNIT_EXPECT_TRUE(test,
+			  list_empty(&adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_HPD1]));
+	KUNIT_EXPECT_TRUE(test,
+			  list_empty(&adev->dm.irq_handler_list_high_tab[DC_IRQ_SOURCE_HPD1]));
+}
+
+/**
+ * dm_test_irq_unregister_rejects_null_handler - Test unregister rejects handler
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_unregister_rejects_null_handler(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD1,
+						DAL_INVALID_IRQ_HANDLER_IDX);
+
+	KUNIT_EXPECT_TRUE(test,
+			  list_empty(&adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_HPD1]));
+	KUNIT_EXPECT_TRUE(test,
+			  list_empty(&adev->dm.irq_handler_list_high_tab[DC_IRQ_SOURCE_HPD1]));
+}
+
+/**
+ * dm_test_irq_unregister_handler_not_found - Test unregister keeps unmatched handler
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_unregister_handler_not_found(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_interrupt_params int_params = { 0 };
+	void *handler;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_HPD1;
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						    dm_test_irq_handler, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+
+	/* Unregister a handler that was never registered for this source. */
+	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD1,
+						dm_test_irq_handler_alt);
+
+	/* The originally registered handler must still be present. */
+	KUNIT_EXPECT_FALSE(test,
+			   list_empty(&adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_HPD1]));
+
+	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD1,
+						dm_test_irq_handler);
+	KUNIT_EXPECT_TRUE(test,
+			  list_empty(&adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_HPD1]));
+}
+
+/* Tests for amdgpu_dm_irq_fini() */
+
+/**
+ * dm_test_irq_fini_removes_registered_handlers - Test fini removes handlers
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_fini_removes_registered_handlers(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_interrupt_params int_params = { 0 };
+	void *handler;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_HPD3;
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						    dm_test_irq_handler, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+
+	int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_HPD4;
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						    dm_test_irq_handler, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+
+	amdgpu_dm_irq_fini(adev);
+
+	KUNIT_EXPECT_TRUE(test,
+			  list_empty(&adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_HPD3]));
+	KUNIT_EXPECT_TRUE(test,
+			  list_empty(&adev->dm.irq_handler_list_high_tab[DC_IRQ_SOURCE_HPD4]));
+}
+
+/**
+ * dm_test_irq_fini_on_empty_tables - Test fini on tables with no handlers
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_fini_on_empty_tables(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	int src;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	amdgpu_dm_irq_fini(adev);
+
+	for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++) {
+		KUNIT_EXPECT_TRUE(test,
+				  list_empty(&adev->dm.irq_handler_list_low_tab[src]));
+		KUNIT_EXPECT_TRUE(test,
+				  list_empty(&adev->dm.irq_handler_list_high_tab[src]));
+	}
+}
+
+/* Tests for amdgpu_dm_get_crtc_by_otg_inst() */
+
+/**
+ * dm_test_get_crtc_by_otg_inst_returns_match - Test CRTC lookup by OTG instance
+ * @test: The KUnit test context
+ */
+static void dm_test_get_crtc_by_otg_inst_returns_match(struct kunit *test)
+{
+	struct amdgpu_crtc *acrtc_a, *acrtc_b;
+	struct amdgpu_device *adev;
+	struct drm_device *drm;
+
+	adev = dm_kunit_alloc_adev(test);
+	drm = &adev->ddev;
+
+	acrtc_a = kunit_kzalloc(test, sizeof(*acrtc_a), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc_a);
+	acrtc_b = kunit_kzalloc(test, sizeof(*acrtc_b), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc_b);
+
+	INIT_LIST_HEAD(&acrtc_a->base.head);
+	INIT_LIST_HEAD(&acrtc_b->base.head);
+	acrtc_a->otg_inst = 1;
+	acrtc_b->otg_inst = 3;
+
+	list_add_tail(&acrtc_a->base.head, &drm->mode_config.crtc_list);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_crtc_list_del, acrtc_a), 0);
+	list_add_tail(&acrtc_b->base.head, &drm->mode_config.crtc_list);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_crtc_list_del, acrtc_b), 0);
+
+	KUNIT_EXPECT_PTR_EQ(test, amdgpu_dm_get_crtc_by_otg_inst(adev, 3), acrtc_b);
+}
+
+/**
+ * dm_test_get_crtc_by_otg_inst_returns_null - Test CRTC lookup misses unknown OTG
+ * @test: The KUnit test context
+ */
+static void dm_test_get_crtc_by_otg_inst_returns_null(struct kunit *test)
+{
+	struct amdgpu_crtc *acrtc;
+	struct amdgpu_device *adev;
+	struct drm_device *drm;
+
+	adev = dm_kunit_alloc_adev(test);
+	drm = &adev->ddev;
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	INIT_LIST_HEAD(&acrtc->base.head);
+	acrtc->otg_inst = 2;
+
+	list_add_tail(&acrtc->base.head, &drm->mode_config.crtc_list);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_crtc_list_del, acrtc), 0);
+
+	KUNIT_EXPECT_NULL(test, amdgpu_dm_get_crtc_by_otg_inst(adev, 5));
+}
+
+/**
+ * dm_test_get_crtc_by_otg_inst_empty_list - Test CRTC lookup on empty CRTC list
+ * @test: The KUnit test context
+ */
+static void dm_test_get_crtc_by_otg_inst_empty_list(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	KUNIT_EXPECT_NULL(test, amdgpu_dm_get_crtc_by_otg_inst(adev, 0));
+}
+
+/* Tests for amdgpu_dm_set_irq_funcs() */
+
+/**
+ * dm_test_set_irq_funcs - Test irq src funcs and counts are populated
+ * @test: The KUnit test context
+ */
+static void dm_test_set_irq_funcs(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	adev->mode_info.num_crtc = 6;
+	adev->mode_info.num_hpd = 4;
+
+	amdgpu_dm_set_irq_funcs(adev);
+
+	KUNIT_EXPECT_EQ(test, adev->crtc_irq.num_types, 6);
+	KUNIT_EXPECT_EQ(test, adev->vline0_irq.num_types, 6);
+	KUNIT_EXPECT_EQ(test, adev->vupdate_irq.num_types, 6);
+	KUNIT_EXPECT_EQ(test, adev->pageflip_irq.num_types, 6);
+	KUNIT_EXPECT_EQ(test, adev->dmub_outbox_irq.num_types, 1);
+	KUNIT_EXPECT_EQ(test, adev->dmub_trace_irq.num_types, 1);
+	KUNIT_EXPECT_EQ(test, adev->hpd_irq.num_types, 4);
+
+	KUNIT_EXPECT_TRUE(test, adev->crtc_irq.funcs != NULL);
+	KUNIT_EXPECT_TRUE(test, adev->vline0_irq.funcs != NULL);
+	KUNIT_EXPECT_TRUE(test, adev->dmub_outbox_irq.funcs != NULL);
+	KUNIT_EXPECT_TRUE(test, adev->vupdate_irq.funcs != NULL);
+	KUNIT_EXPECT_TRUE(test, adev->dmub_trace_irq.funcs != NULL);
+	KUNIT_EXPECT_TRUE(test, adev->pageflip_irq.funcs != NULL);
+	KUNIT_EXPECT_TRUE(test, adev->hpd_irq.funcs != NULL);
+}
+
+/* Tests for amdgpu_dm_irq_suspend()/resume_early()/resume_late() */
+
+/**
+ * dm_test_irq_suspend_empty - Test suspend walks empty handler tables safely
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_suspend_empty(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	int src;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	/*
+	 * With no registered handlers the HW dc_interrupt_set() calls are
+	 * skipped, so suspend must complete without touching the (absent) DC.
+	 */
+	amdgpu_dm_irq_suspend(adev);
+
+	for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++) {
+		KUNIT_EXPECT_TRUE(test, list_empty(&adev->dm.irq_handler_list_low_tab[src]));
+		KUNIT_EXPECT_TRUE(test, list_empty(&adev->dm.irq_handler_list_high_tab[src]));
+	}
+}
+
+/**
+ * dm_test_irq_resume_early_empty - Test early resume walks empty tables safely
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_resume_early_empty(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	int src;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	amdgpu_dm_irq_resume_early(adev);
+
+	for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++)
+		KUNIT_EXPECT_TRUE(test, list_empty(&adev->dm.irq_handler_list_high_tab[src]));
+}
+
+/**
+ * dm_test_irq_resume_late_empty - Test late resume walks empty tables safely
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_resume_late_empty(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	int src;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	amdgpu_dm_irq_resume_late(adev);
+
+	for (src = 0; src < DAL_IRQ_SOURCES_NUMBER; src++)
+		KUNIT_EXPECT_TRUE(test, list_empty(&adev->dm.irq_handler_list_low_tab[src]));
+}
+
+/**
+ * dm_test_irq_suspend_registered - Test suspend reaches the dc_interrupt_set path
+ * @test: The KUnit test context
+ *
+ * Registers a low-context HPD handler so the handler list is non-empty,
+ * forcing amdgpu_dm_irq_suspend() to call dc_interrupt_set() (NULL-safe with
+ * no DC) and flush_work() on the registered handler.
+ */
+static void dm_test_irq_suspend_registered(struct kunit *test)
+{
+	struct dc_interrupt_params int_params = { 0 };
+	struct amdgpu_device *adev;
+	void *handler;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_HPD1;
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						   dm_test_irq_handler, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+
+	amdgpu_dm_irq_suspend(adev);
+
+	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD1,
+					   dm_test_irq_handler);
+}
+
+/**
+ * dm_test_irq_suspend_disables_polling - Test suspend disables KMS polling
+ * @test: The KUnit test context
+ *
+ * With KMS polling active, suspend must take the poll-disable branch.
+ * drm_kms_helper_poll_disable() only cancels the poll work; it leaves the
+ * poll_enabled flag set (cleared later by drm_kms_helper_poll_fini()).
+ */
+static void dm_test_irq_suspend_disables_polling(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	/* Enable KMS polling so suspend takes the poll-disable branch. */
+	drm_kms_helper_poll_init(&adev->ddev);
+	KUNIT_ASSERT_TRUE(test, adev->ddev.mode_config.poll_enabled);
+
+	amdgpu_dm_irq_suspend(adev);
+
+	KUNIT_EXPECT_TRUE(test, adev->ddev.mode_config.poll_enabled);
+
+	drm_kms_helper_poll_fini(&adev->ddev);
+}
+
+/**
+ * dm_test_irq_resume_early_registered - Test early resume reaches dc_interrupt_set
+ * @test: The KUnit test context
+ *
+ * Registers a low-context HPD RX handler so early resume calls
+ * dc_interrupt_set() for the short-pulse interrupt source.
+ */
+static void dm_test_irq_resume_early_registered(struct kunit *test)
+{
+	struct dc_interrupt_params int_params = { 0 };
+	struct amdgpu_device *adev;
+	void *handler;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_HPD1RX;
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						   dm_test_irq_handler, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+
+	amdgpu_dm_irq_resume_early(adev);
+
+	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD1RX,
+					   dm_test_irq_handler);
+}
+
+/**
+ * dm_test_irq_resume_late_registered - Test late resume reaches dc_interrupt_set
+ * @test: The KUnit test context
+ *
+ * Registers a low-context HPD handler so late resume calls dc_interrupt_set()
+ * for the HPD interrupt source.
+ */
+static void dm_test_irq_resume_late_registered(struct kunit *test)
+{
+	struct dc_interrupt_params int_params = { 0 };
+	struct amdgpu_device *adev;
+	void *handler;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_HPD1;
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						   dm_test_irq_handler, adev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+
+	amdgpu_dm_irq_resume_late(adev);
+
+	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD1,
+					   dm_test_irq_handler);
+}
+
+/**
+ * dm_test_irq_resume_late_enables_polling - Test late resume re-enables polling
+ * @test: The KUnit test context
+ *
+ * With KMS polling active, late resume must take the poll-enable branch and
+ * leave polling enabled.
+ */
+static void dm_test_irq_resume_late_enables_polling(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	/* Enable KMS polling so resume_late takes the poll-enable branch. */
+	drm_kms_helper_poll_init(&adev->ddev);
+	KUNIT_ASSERT_TRUE(test, adev->ddev.mode_config.poll_enabled);
+
+	amdgpu_dm_irq_resume_late(adev);
+
+	KUNIT_EXPECT_TRUE(test, adev->ddev.mode_config.poll_enabled);
+
+	drm_kms_helper_poll_fini(&adev->ddev);
+}
+
+/* Tests for amdgpu_dm_hpd_rx_irq_create_workqueue() */
+
+/**
+ * dm_test_hpd_rx_irq_create_workqueue - Test workqueue array creation
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_rx_irq_create_workqueue(struct kunit *test)
+{
+	struct dm_test_hpd_rx_wq_ctx *ctx;
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	int i;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	dc->caps.max_links = 4;
+	adev->dm.dc = dc;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+
+	ctx->wq = amdgpu_dm_hpd_rx_irq_create_workqueue(adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->wq);
+	ctx->count = dc->caps.max_links;
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_destroy_hpd_rx_wq, ctx), 0);
+
+	for (i = 0; i < dc->caps.max_links; i++)
+		KUNIT_EXPECT_TRUE(test, ctx->wq[i].wq != NULL);
+}
+
+/* Tests for amdgpu_dm_hpd_rx_irq_work_suspend() */
+
+/**
+ * dm_test_hpd_rx_irq_work_suspend_null - Test suspend with no work queue
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_rx_irq_work_suspend_null(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm;
+
+	dm = kunit_kzalloc(test, sizeof(*dm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm);
+
+	/* A NULL hpd_rx_offload_wq must be a safe no-op (DC untouched). */
+	amdgpu_dm_hpd_rx_irq_work_suspend(dm);
+}
+
+/**
+ * dm_test_hpd_rx_irq_work_suspend_flushes - Test suspend flushes queues
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_rx_irq_work_suspend_flushes(struct kunit *test)
+{
+	struct dm_test_hpd_rx_wq_ctx *ctx;
+	struct amdgpu_device *adev;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	dc->caps.max_links = 2;
+	adev->dm.dc = dc;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+
+	ctx->wq = amdgpu_dm_hpd_rx_irq_create_workqueue(adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->wq);
+	ctx->count = dc->caps.max_links;
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_destroy_hpd_rx_wq, ctx), 0);
+
+	adev->dm.hpd_rx_offload_wq = ctx->wq;
+
+	amdgpu_dm_hpd_rx_irq_work_suspend(&adev->dm);
+}
+
+/* Tests for CRTC-based irq state callbacks (no-CRTC early return) */
+
+/**
+ * dm_test_set_crtc_irq_state_no_crtc - Test crtc irq state with missing CRTC
+ * @test: The KUnit test context
+ */
+static void dm_test_set_crtc_irq_state_no_crtc(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	/* mode_info.crtcs[0] is NULL -> returns 0 without dereferencing DC. */
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_set_crtc_irq_state(adev, NULL, 0, AMDGPU_IRQ_STATE_ENABLE), 0);
+}
+
+/**
+ * dm_test_set_pflip_irq_state_no_crtc - Test pflip irq state with missing CRTC
+ * @test: The KUnit test context
+ */
+static void dm_test_set_pflip_irq_state_no_crtc(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_set_pflip_irq_state(adev, NULL, 0, AMDGPU_IRQ_STATE_DISABLE), 0);
+}
+
+/**
+ * dm_test_set_vline0_irq_state_no_crtc - Test vline0 irq state with missing CRTC
+ * @test: The KUnit test context
+ */
+static void dm_test_set_vline0_irq_state_no_crtc(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_set_vline0_irq_state(adev, NULL, 0, AMDGPU_IRQ_STATE_ENABLE), 0);
+}
+
+/**
+ * dm_test_set_vupdate_irq_state_no_crtc - Test vupdate irq state with missing CRTC
+ * @test: The KUnit test context
+ */
+static void dm_test_set_vupdate_irq_state_no_crtc(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_set_vupdate_irq_state(adev, NULL, 0, AMDGPU_IRQ_STATE_ENABLE), 0);
+}
+
+/* Tests for CRTC-based irq state callbacks (dm_irq_state happy path) */
+
+/**
+ * dm_test_set_crtc_irq_state_otg_disabled - Test crtc irq state with disabled OTG
+ * @test: The KUnit test context
+ */
+static void dm_test_set_crtc_irq_state_otg_disabled(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	/* otg_inst == -1 short-circuits before computing the irq source. */
+	acrtc->otg_inst = -1;
+	adev->mode_info.crtcs[0] = acrtc;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_set_crtc_irq_state(adev, NULL, 0, AMDGPU_IRQ_STATE_ENABLE), 0);
+}
+
+/**
+ * dm_test_set_crtc_irq_state_enable - Test crtc irq state reaches DC (enable)
+ * @test: The KUnit test context
+ */
+static void dm_test_set_crtc_irq_state_enable(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	/*
+	 * otg_inst >= 0 computes the irq source and reaches the NULL-safe
+	 * dc_interrupt_set(); the ips_support branch is skipped (dc == NULL).
+	 */
+	acrtc->otg_inst = 3;
+	adev->mode_info.crtcs[0] = acrtc;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_set_crtc_irq_state(adev, NULL, 0, AMDGPU_IRQ_STATE_ENABLE), 0);
+}
+
+/**
+ * dm_test_set_pflip_irq_state_disable - Test pflip irq state reaches DC (disable)
+ * @test: The KUnit test context
+ */
+static void dm_test_set_pflip_irq_state_disable(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	/* The disable state exercises the st == false path. */
+	acrtc->otg_inst = 1;
+	adev->mode_info.crtcs[0] = acrtc;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_set_pflip_irq_state(adev, NULL, 0, AMDGPU_IRQ_STATE_DISABLE), 0);
+}
+
+/**
+ * dm_test_set_vline0_irq_state_enable - Test vline0 irq state reaches DC (enable)
+ * @test: The KUnit test context
+ */
+static void dm_test_set_vline0_irq_state_enable(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	acrtc->otg_inst = 0;
+	adev->mode_info.crtcs[0] = acrtc;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_set_vline0_irq_state(adev, NULL, 0, AMDGPU_IRQ_STATE_ENABLE), 0);
+}
+
+/**
+ * dm_test_set_vupdate_irq_state_enable - Test vupdate irq state reaches DC (enable)
+ * @test: The KUnit test context
+ */
+static void dm_test_set_vupdate_irq_state_enable(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	acrtc->otg_inst = 2;
+	adev->mode_info.crtcs[0] = acrtc;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_set_vupdate_irq_state(adev, NULL, 0, AMDGPU_IRQ_STATE_ENABLE), 0);
+}
+
+/**
+ * dm_test_set_crtc_irq_state_allows_idle - Test the idle-optimization branch
+ * @test: The KUnit test context
+ *
+ * With a non-NULL DC that advertises IPS support and currently allows idle
+ * optimizations, dm_irq_state() must call dc_allow_idle_optimizations() before
+ * dc_interrupt_set(). disable_idle_power_optimizations makes that call a safe
+ * early return, and per-source stub funcs let dc_interrupt_set() succeed.
+ */
+static void dm_test_set_crtc_irq_state_allows_idle(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+	struct dal_logger *logger;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	dc = dm_test_alloc_dc_with_irq_service(test, &dm_test_irq_service_funcs_dcn10);
+
+	/* DC_LOG_* dereferences ctx->logger->dev, so wire a real drm device. */
+	logger = kunit_kzalloc(test, sizeof(*logger), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, logger);
+	logger->dev = &adev->ddev;
+	dc->ctx->logger = logger;
+
+	dc->caps.ips_support = true;
+	dc->idle_optimizations_allowed = true;
+	/* Keep dc_allow_idle_optimizations() a safe early return. */
+	dc->debug.disable_idle_power_optimizations = true;
+	adev->dm.dc = dc;
+
+	acrtc->otg_inst = 0;
+	adev->mode_info.crtcs[0] = acrtc;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_set_crtc_irq_state(adev, NULL, 0, AMDGPU_IRQ_STATE_ENABLE), 0);
+}
+
+/* Tests for amdgpu_dm_irq_immediate_work() */
+
+/**
+ * dm_test_irq_immediate_work_empty - Test immediate work on empty high table
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_immediate_work_empty(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	/* No registered high-context handlers: must be a safe no-op. */
+	amdgpu_dm_irq_immediate_work(adev, DC_IRQ_SOURCE_HPD1);
+}
+
+/**
+ * dm_test_irq_immediate_work_invokes_handler - Test immediate work calls handler
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_immediate_work_invokes_handler(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_interrupt_params int_params = { 0 };
+	int count = 0;
+	void *handler;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_HPD1;
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						   dm_test_irq_handler_count, &count);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+
+	/* High-context handlers are invoked synchronously, in-place. */
+	amdgpu_dm_irq_immediate_work(adev, DC_IRQ_SOURCE_HPD1);
+	KUNIT_EXPECT_EQ(test, count, 1);
+
+	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD1, dm_test_irq_handler_count);
+}
+
+/**
+ * dm_test_irq_immediate_work_invokes_all - Test immediate work calls all handlers
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_immediate_work_invokes_all(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_interrupt_params int_params = { 0 };
+	int count = 0;
+	void *handler;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_HPD2;
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						   dm_test_irq_handler_count, &count);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						   dm_test_irq_handler_count, &count);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+
+	/* Both registered high-context handlers must run. */
+	amdgpu_dm_irq_immediate_work(adev, DC_IRQ_SOURCE_HPD2);
+	KUNIT_EXPECT_EQ(test, count, 2);
+
+	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD2, dm_test_irq_handler_count);
+	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD2, dm_test_irq_handler_count);
+}
+
+/* Tests for amdgpu_dm_irq_schedule_work() */
+
+/**
+ * dm_test_irq_schedule_work_empty - Test schedule work on empty low table
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_schedule_work_empty(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	/* Empty handler list: schedule_work returns immediately. */
+	amdgpu_dm_irq_schedule_work(adev, DC_IRQ_SOURCE_HPD1);
+}
+
+/**
+ * dm_test_irq_schedule_work_queues_handler - Test schedule work runs handler
+ * @test: The KUnit test context
+ */
+static void dm_test_irq_schedule_work_queues_handler(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_interrupt_params int_params = { 0 };
+	int count = 0;
+	void *handler;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_HPD1;
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						   dm_test_irq_handler_count, &count);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+
+	amdgpu_dm_irq_schedule_work(adev, DC_IRQ_SOURCE_HPD1);
+
+	/*
+	 * Low-context work runs asynchronously on system_highpri_wq.
+	 * amdgpu_dm_irq_fini() flushes each pending work item before freeing
+	 * the handlers, so the handler is guaranteed to have run afterwards.
+	 */
+	amdgpu_dm_irq_fini(adev);
+	KUNIT_EXPECT_EQ(test, count, 1);
+}
+
+/**
+ * dm_test_irq_schedule_work_requeue_fallback - Test the re-queue fallback path
+ * @test: The KUnit test context
+ *
+ * The first schedule queues the handler's work item. Issuing a second
+ * schedule before the work has run makes queue_work() fail for the
+ * still-pending item, forcing amdgpu_dm_irq_schedule_work() into the fallback
+ * that allocates and queues a fresh handler copy. Both work items run when
+ * amdgpu_dm_irq_fini() flushes the queue, so the handler fires twice.
+ */
+static void dm_test_irq_schedule_work_requeue_fallback(struct kunit *test)
+{
+	struct dc_interrupt_params int_params = { 0 };
+	struct amdgpu_device *adev;
+	int count = 0;
+	void *handler;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_HPD1;
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						   dm_test_irq_handler_count, &count);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+
+	amdgpu_dm_irq_schedule_work(adev, DC_IRQ_SOURCE_HPD1);
+	amdgpu_dm_irq_schedule_work(adev, DC_IRQ_SOURCE_HPD1);
+
+	amdgpu_dm_irq_fini(adev);
+	KUNIT_EXPECT_EQ(test, count, 2);
+}
+
+/* Tests for amdgpu_dm_set_hpd_irq_state() */
+
+/**
+ * dm_test_set_hpd_irq_state_null_dc - Test HPD irq state with no DC
+ * @test: The KUnit test context
+ */
+static void dm_test_set_hpd_irq_state_null_dc(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	/* dc_interrupt_set() is a no-op when dc is NULL, so both states
+	 * return 0 without dereferencing the (absent) DC.
+	 */
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_set_hpd_irq_state(adev, NULL, AMDGPU_HPD_1,
+							  AMDGPU_IRQ_STATE_ENABLE), 0);
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_set_hpd_irq_state(adev, NULL, AMDGPU_HPD_1,
+							  AMDGPU_IRQ_STATE_DISABLE), 0);
+}
+
+/* Tests for amdgpu_dm_set_dmub_outbox_irq_state() */
+
+/**
+ * dm_test_set_dmub_outbox_irq_state_null_dc - Test outbox irq state with no DC
+ * @test: The KUnit test context
+ */
+static void dm_test_set_dmub_outbox_irq_state_null_dc(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_set_dmub_outbox_irq_state(adev, NULL, 0,
+								  AMDGPU_IRQ_STATE_ENABLE), 0);
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_set_dmub_outbox_irq_state(adev, NULL, 0,
+								  AMDGPU_IRQ_STATE_DISABLE), 0);
+}
+
+/* Tests for amdgpu_dm_set_dmub_trace_irq_state() */
+
+/**
+ * dm_test_set_dmub_trace_irq_state_null_dc - Test trace irq state with no DC
+ * @test: The KUnit test context
+ */
+static void dm_test_set_dmub_trace_irq_state_null_dc(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_set_dmub_trace_irq_state(adev, NULL, 0,
+								 AMDGPU_IRQ_STATE_ENABLE), 0);
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_set_dmub_trace_irq_state(adev, NULL, 0,
+								 AMDGPU_IRQ_STATE_DISABLE), 0);
+}
+
+/* Tests for amdgpu_dm_outbox_init() */
+
+/**
+ * dm_test_outbox_init_null_dc - Test outbox init is a safe no-op with no DC
+ * @test: The KUnit test context
+ */
+static void dm_test_outbox_init_null_dc(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	/* Single dc_interrupt_set() call must be skipped when dc is NULL. */
+	amdgpu_dm_outbox_init(adev);
+}
+
+/* Tests for amdgpu_dm_hpd_init()/amdgpu_dm_hpd_fini() */
+
+/**
+ * dm_test_hpd_init_empty_connectors - Test HPD init with no connectors
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_init_empty_connectors(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	/*
+	 * With an empty connector list the per-connector loop is skipped and
+	 * the initial clear loop relies on dc_interrupt_set() being a no-op
+	 * for a NULL dc, so init must complete without touching the DC.
+	 */
+	amdgpu_dm_hpd_init(adev);
+}
+
+/**
+ * dm_test_hpd_fini_empty_connectors - Test HPD fini with no connectors
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_fini_empty_connectors(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	/* Empty connector list and disabled polling: fini is a safe no-op. */
+	amdgpu_dm_hpd_fini(adev);
+}
+
+/**
+ * dm_test_hpd_init_fini_with_connectors - Test HPD init/fini walk connectors
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_init_fini_with_connectors(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct drm_connector *wbconn;
+	struct amdgpu_device *adev;
+	struct dc_link *link;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	/*
+	 * num_hpd = 0 forces irq_type >= num_hpd so the loop takes the HW
+	 * fallback (dc_interrupt_set()) instead of amdgpu_irq_get(); with a
+	 * NULL dc that fallback is a safe no-op.
+	 */
+	adev->mode_info.num_hpd = 0;
+
+	/* A writeback connector must be skipped by the per-connector loop. */
+	wbconn = kunit_kzalloc(test, sizeof(*wbconn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, wbconn);
+	KUNIT_ASSERT_EQ(test, drm_connector_init(&adev->ddev, wbconn, &dm_test_connector_funcs,
+						 DRM_MODE_CONNECTOR_WRITEBACK), 0);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_connector_cleanup, wbconn), 0);
+
+	/* A DisplayPort connector with a dc_link exercises the loop body. */
+	aconn = kunit_kzalloc(test, sizeof(*aconn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn);
+	KUNIT_ASSERT_EQ(test, drm_connector_init(&adev->ddev, &aconn->base,
+						 &dm_test_connector_funcs,
+						 DRM_MODE_CONNECTOR_DisplayPort), 0);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_connector_cleanup, &aconn->base), 0);
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+	link->irq_source_hpd = DC_IRQ_SOURCE_HPD1;
+	link->irq_source_hpd_rx = DC_IRQ_SOURCE_HPD1RX;
+	aconn->dc_link = link;
+
+	/* DC is absent (NULL), so the HW writes are NULL-safe no-ops. */
+	amdgpu_dm_hpd_init(adev);
+	amdgpu_dm_hpd_fini(adev);
+}
+
+/**
+ * dm_test_hpd_init_fini_analog_connector - Test HPD init/fini analog polling path
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_init_fini_analog_connector(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct amdgpu_device *adev;
+	struct dc_link *link;
+
+	adev = dm_kunit_alloc_adev(test);
+	adev->mode_info.num_hpd = 0;
+
+	aconn = kunit_kzalloc(test, sizeof(*aconn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn);
+	KUNIT_ASSERT_EQ(test, drm_connector_init(&adev->ddev, &aconn->base,
+						 &dm_test_connector_funcs,
+						 DRM_MODE_CONNECTOR_VGA), 0);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_connector_cleanup, &aconn->base), 0);
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+	link->irq_source_hpd = DC_IRQ_SOURCE_HPD1;
+	link->irq_source_hpd_rx = DC_IRQ_SOURCE_HPD1RX;
+	/* An analog connector id makes the loop request polling. */
+	link->link_id.id = CONNECTOR_ID_VGA;
+	aconn->dc_link = link;
+
+	/* use_polling becomes true, so init must enable KMS polling. */
+	amdgpu_dm_hpd_init(adev);
+	KUNIT_EXPECT_TRUE(test, adev->ddev.mode_config.poll_enabled);
+
+	/* fini must tear the polling back down. */
+	amdgpu_dm_hpd_fini(adev);
+	KUNIT_EXPECT_FALSE(test, adev->ddev.mode_config.poll_enabled);
+}
+
+/**
+ * dm_test_hpd_init_fini_irq_ref - Test HPD init/fini base-driver irq ref path
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_init_fini_irq_ref(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct amdgpu_device *adev;
+	struct dc_link *link;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	/*
+	 * num_hpd >= 1 makes irq_type (0) < num_hpd, so the loop takes the
+	 * amdgpu_irq_get()/amdgpu_irq_put() branch instead of the
+	 * dc_interrupt_set() fallback. The mock device has irq.installed ==
+	 * false, so both calls fail early with -ENOENT (logging an error)
+	 * without touching the base-driver irq state.
+	 */
+	adev->mode_info.num_hpd = 1;
+
+	aconn = kunit_kzalloc(test, sizeof(*aconn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn);
+	KUNIT_ASSERT_EQ(test, drm_connector_init(&adev->ddev, &aconn->base,
+						 &dm_test_connector_funcs,
+						 DRM_MODE_CONNECTOR_DisplayPort), 0);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_connector_cleanup, &aconn->base), 0);
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+	link->irq_source_hpd = DC_IRQ_SOURCE_HPD1;
+	link->irq_source_hpd_rx = DC_IRQ_SOURCE_INVALID;
+	aconn->dc_link = link;
+
+	amdgpu_dm_hpd_init(adev);
+	amdgpu_dm_hpd_fini(adev);
+}
+
+/* Tests for dm_handle_hpd_rx_offload_work() */
+
+/**
+ * dm_test_hpd_rx_offload_work_no_connector - Test missing connector early exit
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_rx_offload_work_no_connector(struct kunit *test)
+{
+	struct hpd_rx_irq_offload_work_queue *offload_wq;
+	struct hpd_rx_irq_offload_work *offload_work;
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	offload_wq = kunit_kzalloc(test, sizeof(*offload_wq), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_wq);
+
+	offload_work = kzalloc(sizeof(*offload_work), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_work);
+	offload_work->offload_wq = offload_wq;
+	offload_work->adev = adev;
+	INIT_WORK(&offload_work->work, dm_handle_hpd_rx_offload_work);
+
+	dm_handle_hpd_rx_offload_work(&offload_work->work);
+}
+
+/**
+ * dm_test_hpd_rx_offload_work_no_connection - Test no connection early exit
+ * @test: The KUnit test context
+ */
+static void dm_test_hpd_rx_offload_work_no_connection(struct kunit *test)
+{
+	struct hpd_rx_irq_offload_work_queue *offload_wq;
+	struct hpd_rx_irq_offload_work *offload_work;
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+	struct amdgpu_device *adev;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	mutex_init(&adev->dm.dc_lock);
+
+	offload_wq = kunit_kzalloc(test, sizeof(*offload_wq), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_wq);
+
+	aconn = dm_kunit_alloc_connector(test, adev, NULL);
+	mutex_init(&aconn->hpd_lock);
+	offload_wq->aconnector = aconn;
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
+	link_srv->detect_connection_type = dm_test_detect_connection_none;
+	dc->link_srv = link_srv;
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+	link->dc = dc;
+	aconn->dc_link = link;
+
+	offload_work = kzalloc(sizeof(*offload_work), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_work);
+	offload_work->offload_wq = offload_wq;
+	offload_work->adev = adev;
+	INIT_WORK(&offload_work->work, dm_handle_hpd_rx_offload_work);
+
+	dm_handle_hpd_rx_offload_work(&offload_work->work);
+}
+
+/**
+ * dm_test_hpd_rx_offload_work_automated_test - Test AUTOMATED_TEST branch
+ * @test: The KUnit test context
+ *
+ * With a present connection and the AUTOMATED_TEST service-IRQ bit set, the
+ * worker runs dc_link_dp_handle_automated_test() (stubbed via link_srv) and
+ * writes the test response with core_link_write_dpcd(). timing_changed is left
+ * false so force_connector_state() (which needs a registered DRM device) is
+ * skipped, and aux_access_disabled makes the DPCD write a safe no-op.
+ */
+static void dm_test_hpd_rx_offload_work_automated_test(struct kunit *test)
+{
+	struct hpd_rx_irq_offload_work_queue *offload_wq;
+	struct hpd_rx_irq_offload_work *offload_work;
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc *dc;
+
+	dm_test_automated_test_count = 0;
+
+	adev = dm_kunit_alloc_adev(test);
+	mutex_init(&adev->dm.dc_lock);
+	adev->reset_domain = kunit_kzalloc(test, sizeof(*adev->reset_domain),
+					   GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev->reset_domain);
+
+	offload_wq = kunit_kzalloc(test, sizeof(*offload_wq), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_wq);
+	spin_lock_init(&offload_wq->offload_lock);
+
+	aconn = dm_kunit_alloc_connector(test, adev, NULL);
+	mutex_init(&aconn->hpd_lock);
+	aconn->timing_changed = false;
+	offload_wq->aconnector = aconn;
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link_srv->detect_connection_type = dm_test_detect_connection_single;
+	link_srv->dp_handle_automated_test = dm_test_dp_handle_automated_test;
+	dc->link_srv = link_srv;
+	dc->ctx = ctx;
+	ctx->dc = dc;
+	link->dc = dc;
+	link->ctx = ctx;
+	link->aux_access_disabled = true;
+	link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+	aconn->dc_link = link;
+
+	offload_work = kzalloc(sizeof(*offload_work), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_work);
+	offload_work->offload_wq = offload_wq;
+	offload_work->adev = adev;
+	offload_work->data.bytes.device_service_irq.bits.AUTOMATED_TEST = 1;
+	INIT_WORK(&offload_work->work, dm_handle_hpd_rx_offload_work);
+
+	dm_handle_hpd_rx_offload_work(&offload_work->work);
+
+	KUNIT_EXPECT_EQ(test, dm_test_automated_test_count, 1);
+}
+
+/**
+ * dm_test_hpd_rx_offload_work_link_loss - Test link-loss branch
+ * @test: The KUnit test context
+ *
+ * With a present non-eDP connection and no service-IRQ bits set, the worker
+ * takes the else-if link-loss path: dc_link_check_link_loss_status() and
+ * dc_link_dp_allow_hpd_rx_irq() gate entry, then dc_link_dp_read_hpd_rx_irq_data()
+ * returns DC_OK and a second link-loss check triggers dc_link_dp_handle_link_loss().
+ * All four are stubbed via link_srv.
+ */
+static void dm_test_hpd_rx_offload_work_link_loss(struct kunit *test)
+{
+	struct hpd_rx_irq_offload_work_queue *offload_wq;
+	struct hpd_rx_irq_offload_work *offload_work;
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc *dc;
+
+	dm_test_handle_link_loss_count = 0;
+
+	adev = dm_kunit_alloc_adev(test);
+	mutex_init(&adev->dm.dc_lock);
+	adev->reset_domain = kunit_kzalloc(test, sizeof(*adev->reset_domain),
+					   GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev->reset_domain);
+
+	offload_wq = kunit_kzalloc(test, sizeof(*offload_wq), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_wq);
+	spin_lock_init(&offload_wq->offload_lock);
+
+	aconn = dm_kunit_alloc_connector(test, adev, NULL);
+	mutex_init(&aconn->hpd_lock);
+	offload_wq->aconnector = aconn;
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link_srv->detect_connection_type = dm_test_detect_connection_single;
+	link_srv->dp_parse_link_loss_status = dm_test_dp_parse_link_loss_true;
+	link_srv->dp_should_allow_hpd_rx_irq = dm_test_dp_should_allow_hpd_rx_irq_true;
+	link_srv->dp_read_hpd_rx_irq_data = dm_test_dp_read_hpd_rx_irq_data_ok;
+	link_srv->dp_handle_link_loss = dm_test_dp_handle_link_loss;
+	dc->link_srv = link_srv;
+	dc->ctx = ctx;
+	ctx->dc = dc;
+	link->dc = dc;
+	link->ctx = ctx;
+	link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+	aconn->dc_link = link;
+
+	offload_work = kzalloc(sizeof(*offload_work), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_work);
+	offload_work->offload_wq = offload_wq;
+	offload_work->adev = adev;
+	INIT_WORK(&offload_work->work, dm_handle_hpd_rx_offload_work);
+
+	dm_handle_hpd_rx_offload_work(&offload_work->work);
+
+	KUNIT_EXPECT_EQ(test, dm_test_handle_link_loss_count, 1);
+}
+
+/* Tests for amdgpu_dm_hdmi_hpd_debounce_work() */
+
+/**
+ * dm_test_hdmi_hpd_debounce_detect_false - Test debounce false detect path
+ * @test: The KUnit test context
+ */
+static void dm_test_hdmi_hpd_debounce_detect_false(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	mutex_init(&adev->dm.dc_lock);
+
+	aconn = dm_kunit_alloc_connector(test, adev, NULL);
+	mutex_init(&aconn->hpd_lock);
+	INIT_DELAYED_WORK(&aconn->hdmi_hpd_debounce_work,
+			  amdgpu_dm_hdmi_hpd_debounce_work);
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link_srv->detect_link = dm_test_detect_link_false;
+	dc->ctx = ctx;
+	dc->link_srv = link_srv;
+	ctx->dc = dc;
+	link->dc = dc;
+	link->ctx = ctx;
+	aconn->dc_link = link;
+
+	amdgpu_dm_hdmi_hpd_debounce_work(&aconn->hdmi_hpd_debounce_work.work);
+	KUNIT_EXPECT_NULL(test, aconn->hdmi_prev_sink);
+}
+
+/**
+ * dm_test_hdmi_hpd_debounce_reallow_idle - Test debounce idle/sink-release tail
+ * @test: The KUnit test context
+ *
+ * With detection stubbed to return false, the if (ret) block is skipped, but
+ * the function tail is still exercised: IPS support plus an idle-allowed dmub
+ * makes reallow_idle true so dc_allow_idle_optimizations() runs on both entry
+ * and exit (disable_idle_power_optimizations keeps those safe early returns),
+ * and a cached hdmi_prev_sink is released and cleared.
+ */
+static void dm_test_hdmi_hpd_debounce_reallow_idle(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+	struct dc_dmub_srv *dmub_srv;
+	struct amdgpu_device *adev;
+	struct dal_logger *logger;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	mutex_init(&adev->dm.dc_lock);
+
+	aconn = dm_kunit_alloc_connector(test, adev, NULL);
+	mutex_init(&aconn->hpd_lock);
+	INIT_DELAYED_WORK(&aconn->hdmi_hpd_debounce_work,
+			  amdgpu_dm_hdmi_hpd_debounce_work);
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+	logger = kunit_kzalloc(test, sizeof(*logger), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, logger);
+	dmub_srv = kunit_kzalloc(test, sizeof(*dmub_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dmub_srv);
+
+	link_srv->detect_link = dm_test_detect_link_false;
+	dc->ctx = ctx;
+	dc->link_srv = link_srv;
+	ctx->dc = dc;
+	link->dc = dc;
+	link->ctx = ctx;
+	aconn->dc_link = link;
+
+	/* DC_LOG_* dereferences ctx->logger->dev, so wire a real drm device. */
+	logger->dev = &adev->ddev;
+	dc->ctx->logger = logger;
+
+	/* Make reallow_idle true and keep dc_allow_idle*() a safe early return. */
+	dmub_srv->idle_allowed = true;
+	dc->ctx->dmub_srv = dmub_srv;
+	dc->caps.ips_support = true;
+	dc->debug.disable_idle_power_optimizations = true;
+
+	/* A cached previous sink must be released and cleared. */
+	aconn->hdmi_prev_sink = dm_test_sink_create(link);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn->hdmi_prev_sink);
+
+	amdgpu_dm_hdmi_hpd_debounce_work(&aconn->hdmi_hpd_debounce_work.work);
+	KUNIT_EXPECT_NULL(test, aconn->hdmi_prev_sink);
+}
+
+/* Tests for handle_hpd_irq()/handle_hpd_irq_helper() */
+
+/**
+ * dm_test_handle_hpd_irq_disabled - Test HPD helper returns when disabled
+ * @test: The KUnit test context
+ */
+static void dm_test_handle_hpd_irq_disabled(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct dm_connector_state *state;
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	adev->dm.disable_hpd_irq = true;
+
+	aconn = dm_kunit_alloc_connector(test, adev, NULL);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+	aconn->base.state = &state->base;
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	dc->ctx = ctx;
+	ctx->dc = dc;
+	link->ctx = ctx;
+	aconn->dc_link = link;
+	aconn->fake_enable = true;
+
+	handle_hpd_irq_helper(aconn, DETECT_REASON_HPD);
+	KUNIT_EXPECT_TRUE(test, aconn->fake_enable);
+
+	handle_hpd_irq(aconn);
+	KUNIT_EXPECT_TRUE(test, aconn->fake_enable);
+}
+
+/**
+ * dm_test_handle_hpd_irq_helper_debounce_schedule - Test HDMI debounce branch
+ * @test: The KUnit test context
+ *
+ * An HDMI link reporting a disconnect (connection type none) with a non-zero
+ * debounce delay and a cached local_sink must take the debounce branch: it
+ * caches local_sink in hdmi_prev_sink and schedules the delayed debounce work
+ * instead of detecting immediately.
+ */
+static void dm_test_handle_hpd_irq_helper_debounce_schedule(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct dm_connector_state *state;
+	struct link_service *link_srv;
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	aconn = dm_kunit_alloc_connector(test, adev, NULL);
+	mutex_init(&aconn->hpd_lock);
+	INIT_DELAYED_WORK(&aconn->hdmi_hpd_debounce_work,
+			  amdgpu_dm_hdmi_hpd_debounce_work);
+
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+	aconn->base.state = &state->base;
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link_srv->detect_connection_type = dm_test_detect_connection_none;
+	dc->ctx = ctx;
+	dc->link_srv = link_srv;
+	ctx->dc = dc;
+	link->dc = dc;
+	link->ctx = ctx;
+	aconn->dc_link = link;
+
+	/* HDMI signal + debounce delay + cached sink -> debounce branch. */
+	aconn->hdmi_hpd_debounce_delay_ms = 100;
+	link->connector_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	link->local_sink = dm_test_sink_create(link);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link->local_sink);
+
+	handle_hpd_irq_helper(aconn, DETECT_REASON_HPD);
+
+	/* local_sink is cached for later comparison by the debounce work. */
+	KUNIT_EXPECT_PTR_EQ(test, aconn->hdmi_prev_sink, link->local_sink);
+
+	cancel_delayed_work_sync(&aconn->hdmi_hpd_debounce_work);
+	dm_test_sink_release(aconn->hdmi_prev_sink);
+	dm_test_sink_release(link->local_sink);
+}
+
+/**
+ * dm_test_handle_hpd_irq_helper_debounce_release_prev - Test stale prev_sink
+ * @test: The KUnit test context
+ *
+ * When the debounce branch is taken and a stale hdmi_prev_sink is already
+ * cached from a previous HPD, it must be released before caching the current
+ * local_sink. This exercises the dc_sink_release() of the previous sink.
+ */
+static void dm_test_handle_hpd_irq_helper_debounce_release_prev(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct dm_connector_state *state;
+	struct link_service *link_srv;
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	aconn = dm_kunit_alloc_connector(test, adev, NULL);
+	mutex_init(&aconn->hpd_lock);
+	INIT_DELAYED_WORK(&aconn->hdmi_hpd_debounce_work,
+			  amdgpu_dm_hdmi_hpd_debounce_work);
+
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+	aconn->base.state = &state->base;
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link_srv->detect_connection_type = dm_test_detect_connection_none;
+	dc->ctx = ctx;
+	dc->link_srv = link_srv;
+	ctx->dc = dc;
+	link->dc = dc;
+	link->ctx = ctx;
+	aconn->dc_link = link;
+
+	/* HDMI signal + debounce delay + cached sink -> debounce branch. */
+	aconn->hdmi_hpd_debounce_delay_ms = 100;
+	link->connector_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	link->local_sink = dm_test_sink_create(link);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link->local_sink);
+
+	/* Stale sink from a previous HPD must be released by the helper. */
+	aconn->hdmi_prev_sink = dm_test_sink_create(link);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn->hdmi_prev_sink);
+	KUNIT_ASSERT_PTR_NE(test, aconn->hdmi_prev_sink, link->local_sink);
+
+	handle_hpd_irq_helper(aconn, DETECT_REASON_HPD);
+
+	/* Stale sink replaced by the current local_sink. */
+	KUNIT_EXPECT_PTR_EQ(test, aconn->hdmi_prev_sink, link->local_sink);
+
+	cancel_delayed_work_sync(&aconn->hdmi_hpd_debounce_work);
+	dm_test_sink_release(aconn->hdmi_prev_sink);
+	dm_test_sink_release(link->local_sink);
+}
+
+/**
+ * dm_test_handle_hpd_irq_helper_detect_false - Test immediate detect branch
+ * @test: The KUnit test context
+ *
+ * With no force, no debounce, and detection stubbed to report no connection,
+ * the helper takes the else branch: dc_exit_ips_for_hw_access() is a safe
+ * no-op (no IPS support) and dc_link_detect() returns false, so the connected
+ * if (ret) block is skipped. fake_enable must still be cleared.
+ */
+static void dm_test_handle_hpd_irq_helper_detect_false(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct dm_connector_state *state;
+	struct link_service *link_srv;
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	mutex_init(&adev->dm.dc_lock);
+
+	aconn = dm_kunit_alloc_connector(test, adev, NULL);
+	mutex_init(&aconn->hpd_lock);
+	INIT_DELAYED_WORK(&aconn->hdmi_hpd_debounce_work,
+			  amdgpu_dm_hdmi_hpd_debounce_work);
+
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+	aconn->base.state = &state->base;
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link_srv->detect_connection_type = dm_test_detect_connection_none;
+	link_srv->detect_link = dm_test_detect_link_false;
+	dc->ctx = ctx;
+	dc->link_srv = link_srv;
+	ctx->dc = dc;
+	link->dc = dc;
+	link->ctx = ctx;
+	aconn->dc_link = link;
+	aconn->fake_enable = true;
+
+	/* No debounce delay and no force -> immediate-detect else branch. */
+	handle_hpd_irq_helper(aconn, DETECT_REASON_HPD);
+
+	KUNIT_EXPECT_FALSE(test, aconn->fake_enable);
+}
+
+/* Tests for handle_hpd_rx_irq()/schedule_hpd_rx_offload_work() */
+
+/**
+ * dm_test_handle_hpd_rx_irq_disabled - Test HPDRX handler returns when disabled
+ * @test: The KUnit test context
+ */
+static void dm_test_handle_hpd_rx_irq_disabled(struct kunit *test)
+{
+	struct hpd_rx_irq_offload_work_queue *offload_wq;
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	adev->dm.disable_hpd_irq = true;
+
+	offload_wq = kunit_kzalloc(test, sizeof(*offload_wq), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_wq);
+	adev->dm.hpd_rx_offload_wq = offload_wq;
+
+	aconn = dm_kunit_alloc_connector(test, adev, NULL);
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link_srv->dp_handle_hpd_rx_irq = dm_test_handle_hpd_rx_no_work;
+	dc->ctx = ctx;
+	dc->link_srv = link_srv;
+	ctx->dc = dc;
+	link->dc = dc;
+	link->ctx = ctx;
+	link->link_index = 0;
+	aconn->dc_link = link;
+
+	handle_hpd_rx_irq(aconn);
+}
+
+/**
+ * dm_test_schedule_hpd_rx_offload_work - Test offload work is queued
+ * @test: The KUnit test context
+ */
+static void dm_test_schedule_hpd_rx_offload_work(struct kunit *test)
+{
+	struct hpd_rx_irq_offload_work_queue *offload_wq;
+	struct amdgpu_device *adev;
+	union hpd_irq_data data = { 0 };
+
+	adev = dm_kunit_alloc_adev(test);
+
+	offload_wq = kunit_kzalloc(test, sizeof(*offload_wq), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_wq);
+	offload_wq->wq = create_singlethread_workqueue("dm_irq_test_hpd_rx");
+	KUNIT_ASSERT_NOT_NULL(test, offload_wq->wq);
+
+	schedule_hpd_rx_offload_work(adev, offload_wq, data);
+	flush_workqueue(offload_wq->wq);
+	destroy_workqueue(offload_wq->wq);
+}
+
+static void dm_test_destroy_offload_wq(void *data)
+{
+	struct workqueue_struct *wq = data;
+
+	flush_workqueue(wq);
+	destroy_workqueue(wq);
+}
+
+/*
+ * Build an aconnector wired for handle_hpd_rx_irq(): HPD enabled, an MST-root
+ * connector on an MST-branch link so the post-detect block and drm_dp_cec_irq
+ * are skipped, and a flushed offload work queue. Caller sets the link_srv
+ * stubs (dp_handle_hpd_rx_irq and, if needed, dp_should_allow_hpd_rx_irq).
+ */
+static struct amdgpu_dm_connector *dm_test_setup_hpd_rx_irq(struct kunit *test,
+							    struct link_service **link_srv_out)
+{
+	struct hpd_rx_irq_offload_work_queue *offload_wq;
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	mutex_init(&adev->dm.dc_lock);
+
+	offload_wq = kunit_kzalloc(test, sizeof(*offload_wq), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_wq);
+	spin_lock_init(&offload_wq->offload_lock);
+	offload_wq->wq = create_singlethread_workqueue("dm_irq_test_hpd_rx");
+	KUNIT_ASSERT_NOT_NULL(test, offload_wq->wq);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_destroy_offload_wq,
+							offload_wq->wq), 0);
+	adev->dm.hpd_rx_offload_wq = offload_wq;
+
+	aconn = dm_kunit_alloc_connector(test, adev, NULL);
+	mutex_init(&aconn->hpd_lock);
+	aconn->mst_mgr.mst_state = true;
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	dc->ctx = ctx;
+	dc->link_srv = link_srv;
+	ctx->dc = dc;
+	link->dc = dc;
+	link->ctx = ctx;
+	link->link_index = 0;
+	link->type = dc_connection_mst_branch;
+	aconn->dc_link = link;
+
+	*link_srv_out = link_srv;
+
+	return aconn;
+}
+
+/**
+ * dm_test_handle_hpd_rx_irq_no_left_work - Test HPDRX early out (no left work)
+ * @test: The KUnit test context
+ *
+ * When dc_link_handle_hpd_rx_irq() reports no left-over work, the handler
+ * jumps to out and returns without scheduling any offload work.
+ */
+static void dm_test_handle_hpd_rx_irq_no_left_work(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+
+	aconn = dm_test_setup_hpd_rx_irq(test, &link_srv);
+	link_srv->dp_handle_hpd_rx_irq = dm_test_handle_hpd_rx_no_work;
+
+	handle_hpd_rx_irq(aconn);
+}
+
+/**
+ * dm_test_handle_hpd_rx_irq_automated_test - Test HPDRX automated-test path
+ * @test: The KUnit test context
+ *
+ * The AUTOMATED_TEST device-service bit must schedule offload work and jump to
+ * out before the allow-hpd-rx-irq checks.
+ */
+static void dm_test_handle_hpd_rx_irq_automated_test(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+
+	aconn = dm_test_setup_hpd_rx_irq(test, &link_srv);
+	link_srv->dp_handle_hpd_rx_irq = dm_test_handle_hpd_rx_automated;
+
+	handle_hpd_rx_irq(aconn);
+}
+
+/**
+ * dm_test_handle_hpd_rx_irq_msg_rdy - Test HPDRX MST message-ready path
+ * @test: The KUnit test context
+ *
+ * With HPD RX IRQs allowed and an UP_REQ_MSG_RDY bit set, the handler must take
+ * the MST message-ready branch, mark is_handling_mst_msg_rdy_event and schedule
+ * offload work.
+ */
+static void dm_test_handle_hpd_rx_irq_msg_rdy(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+
+	aconn = dm_test_setup_hpd_rx_irq(test, &link_srv);
+	link_srv->dp_handle_hpd_rx_irq = dm_test_handle_hpd_rx_msg_rdy;
+	link_srv->dp_should_allow_hpd_rx_irq = dm_test_allow_hpd_rx_irq_true;
+
+	handle_hpd_rx_irq(aconn);
+
+	KUNIT_EXPECT_TRUE(test, drm_to_adev(aconn->base.dev)
+				->dm.hpd_rx_offload_wq->is_handling_mst_msg_rdy_event);
+}
+
+/**
+ * dm_test_handle_hpd_rx_irq_link_loss - Test HPDRX link-loss path
+ * @test: The KUnit test context
+ *
+ * With HPD RX IRQs allowed and a reported link loss (no message-ready bits),
+ * the handler must take the link-loss branch, mark is_handling_link_loss and
+ * schedule offload work.
+ */
+static void dm_test_handle_hpd_rx_irq_link_loss(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+
+	aconn = dm_test_setup_hpd_rx_irq(test, &link_srv);
+	link_srv->dp_handle_hpd_rx_irq = dm_test_handle_hpd_rx_link_loss;
+	link_srv->dp_should_allow_hpd_rx_irq = dm_test_allow_hpd_rx_irq_true;
+
+	handle_hpd_rx_irq(aconn);
+
+	KUNIT_EXPECT_TRUE(test, drm_to_adev(aconn->base.dev)
+				->dm.hpd_rx_offload_wq->is_handling_link_loss);
+}
+
+/* Tests for dmub_hpd_callback()/dmub_hpd_sense_callback() */
+
+/**
+ * dm_test_dmub_hpd_callback_null_inputs - Test DMUB callback null inputs
+ * @test: The KUnit test context
+ */
+static void dm_test_dmub_hpd_callback_null_inputs(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	dmub_hpd_callback(NULL, NULL);
+	dmub_hpd_callback(adev, NULL);
+}
+
+/**
+ * dm_test_dmub_hpd_callback_invalid_index - Test DMUB callback index check
+ * @test: The KUnit test context
+ */
+static void dm_test_dmub_hpd_callback_invalid_index(struct kunit *test)
+{
+	struct dmub_notification notify = { 0 };
+	struct amdgpu_device *adev;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	adev->dm.dc = dc;
+	notify.link_index = 1;
+
+	dmub_hpd_callback(adev, &notify);
+}
+
+/**
+ * dm_test_dmub_hpd_callback_empty_connectors - Test DMUB callback without match
+ * @test: The KUnit test context
+ */
+static void dm_test_dmub_hpd_callback_empty_connectors(struct kunit *test)
+{
+	struct dmub_notification notify = { 0 };
+	struct amdgpu_device *adev;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	adev->dm.dc = dc;
+	adev->dm.ddev = &adev->ddev;
+	dc->link_count = 1;
+	notify.type = DMUB_NOTIFICATION_HPD_SENSE_NOTIFY;
+
+	dmub_hpd_callback(adev, &notify);
+	dmub_hpd_sense_callback(adev, &notify);
+}
+
+/**
+ * dm_test_dmub_hpd_callback_unknown_type_match - Test DMUB callback matched
+ *						   connector, unknown type
+ * @test: The KUnit test context
+ *
+ * A writeback connector must be skipped (continue) and a non-writeback
+ * connector whose dc_link matches the notification link must be selected.
+ * An unrecognized notification type takes the "unknown" warn branch, and the
+ * post-loop dispatch is a no-op (neither HPD nor HPD_IRQ), so no real DC/MST
+ * handler runs.
+ */
+static void dm_test_dmub_hpd_callback_unknown_type_match(struct kunit *test)
+{
+	struct dmub_notification notify = { 0 };
+	struct amdgpu_dm_connector *aconn;
+	struct drm_connector *wbconn;
+	struct amdgpu_device *adev;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	adev->dm.dc = dc;
+	adev->dm.ddev = &adev->ddev;
+	dc->link_count = 1;
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+	dc->links[0] = link;
+
+	/* A writeback connector must be skipped by the loop. */
+	wbconn = kunit_kzalloc(test, sizeof(*wbconn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, wbconn);
+	KUNIT_ASSERT_EQ(test, drm_connector_init(&adev->ddev, wbconn, &dm_test_connector_funcs,
+						 DRM_MODE_CONNECTOR_WRITEBACK), 0);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_connector_cleanup, wbconn), 0);
+
+	/* A DisplayPort connector whose dc_link matches drives the match branch. */
+	aconn = kunit_kzalloc(test, sizeof(*aconn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn);
+	KUNIT_ASSERT_EQ(test, drm_connector_init(&adev->ddev, &aconn->base,
+						 &dm_test_connector_funcs,
+						 DRM_MODE_CONNECTOR_DisplayPort), 0);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_connector_cleanup, &aconn->base), 0);
+	aconn->dc_link = link;
+
+	notify.link_index = 0;
+	notify.type = DMUB_NOTIFICATION_HPD_SENSE_NOTIFY;
+
+	dmub_hpd_callback(adev, &notify);
+}
+
+/**
+ * dm_test_dmub_hpd_callback_hpd_type - Test DMUB callback HPD type dispatch
+ * @test: The KUnit test context
+ *
+ * A matched connector with notification type DMUB_NOTIFICATION_HPD logs the
+ * HPD callback and dispatches to handle_hpd_irq_helper(). Detection is stubbed
+ * to report no connection and link detect to fail, so the connected branches
+ * are skipped and no real DC/DRM hotplug path runs.
+ */
+static void dm_test_dmub_hpd_callback_hpd_type(struct kunit *test)
+{
+	struct dmub_notification notify = { 0 };
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	mutex_init(&adev->dm.dc_lock);
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link_srv->detect_connection_type = dm_test_detect_connection_none;
+	link_srv->detect_link = dm_test_detect_link_false;
+	dc->ctx = ctx;
+	dc->link_srv = link_srv;
+	ctx->dc = dc;
+	link->dc = dc;
+	link->ctx = ctx;
+
+	adev->dm.dc = dc;
+	adev->dm.ddev = &adev->ddev;
+	dc->link_count = 1;
+	dc->links[0] = link;
+
+	aconn = kunit_kzalloc(test, sizeof(*aconn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn);
+	KUNIT_ASSERT_EQ(test, drm_connector_init(&adev->ddev, &aconn->base,
+						 &dm_test_connector_funcs,
+						 DRM_MODE_CONNECTOR_DisplayPort), 0);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_connector_cleanup, &aconn->base), 0);
+	mutex_init(&aconn->hpd_lock);
+	INIT_DELAYED_WORK(&aconn->hdmi_hpd_debounce_work,
+			  amdgpu_dm_hdmi_hpd_debounce_work);
+	aconn->dc_link = link;
+
+	notify.link_index = 0;
+	notify.type = DMUB_NOTIFICATION_HPD;
+
+	dmub_hpd_callback(adev, &notify);
+}
+
+/**
+ * dm_test_dmub_hpd_callback_hpd_irq_type - Test DMUB callback HPD_IRQ dispatch
+ * @test: The KUnit test context
+ *
+ * A matched connector with notification type DMUB_NOTIFICATION_HPD_IRQ logs
+ * the HPD RX callback and dispatches to handle_hpd_rx_irq(). The HPD RX handler
+ * stub reports no left work, so the function returns through the early goto
+ * without scheduling offload work; an MST-branch link skips the trailing
+ * drm_dp_cec_irq().
+ */
+static void dm_test_dmub_hpd_callback_hpd_irq_type(struct kunit *test)
+{
+	struct hpd_rx_irq_offload_work_queue *offload_wq;
+	struct dmub_notification notify = { 0 };
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	mutex_init(&adev->dm.dc_lock);
+
+	offload_wq = kunit_kzalloc(test, sizeof(*offload_wq), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_wq);
+	spin_lock_init(&offload_wq->offload_lock);
+	adev->dm.hpd_rx_offload_wq = offload_wq;
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link_srv->dp_handle_hpd_rx_irq = dm_test_handle_hpd_rx_no_work;
+	dc->ctx = ctx;
+	dc->link_srv = link_srv;
+	ctx->dc = dc;
+	link->dc = dc;
+	link->ctx = ctx;
+	link->link_index = 0;
+	link->type = dc_connection_mst_branch;
+
+	adev->dm.dc = dc;
+	adev->dm.ddev = &adev->ddev;
+	dc->link_count = 1;
+	dc->links[0] = link;
+
+	aconn = kunit_kzalloc(test, sizeof(*aconn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn);
+	KUNIT_ASSERT_EQ(test, drm_connector_init(&adev->ddev, &aconn->base,
+						 &dm_test_connector_funcs,
+						 DRM_MODE_CONNECTOR_DisplayPort), 0);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_connector_cleanup, &aconn->base), 0);
+	mutex_init(&aconn->hpd_lock);
+	aconn->mst_mgr.mst_state = true;
+	aconn->dc_link = link;
+
+	notify.link_index = 0;
+	notify.type = DMUB_NOTIFICATION_HPD_IRQ;
+
+	dmub_hpd_callback(adev, &notify);
+}
+
+/* Tests for amdgpu_dm_register_hpd_handlers() */
+
+/**
+ * dm_test_register_hpd_handlers_empty - Test HPD registration with no connectors
+ * @test: The KUnit test context
+ */
+static void dm_test_register_hpd_handlers_empty(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	adev->dm.dc = dc;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_register_hpd_handlers(adev), 0);
+}
+
+/**
+ * dm_test_register_hpd_handlers_valid - Test HPD/HPDRX registration succeeds
+ * @test: The KUnit test context
+ */
+static void dm_test_register_hpd_handlers_valid(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct amdgpu_device *adev;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	adev->dm.dc = dc;
+
+	aconn = kunit_kzalloc(test, sizeof(*aconn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn);
+	KUNIT_ASSERT_EQ(test, drm_connector_init(&adev->ddev, &aconn->base,
+						 &dm_test_connector_funcs,
+						 DRM_MODE_CONNECTOR_DisplayPort), 0);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_connector_cleanup, &aconn->base), 0);
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+	link->irq_source_hpd = DC_IRQ_SOURCE_HPD1;
+	link->irq_source_hpd_rx = DC_IRQ_SOURCE_HPD1RX;
+	aconn->dc_link = link;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_register_hpd_handlers(adev), 0);
+	KUNIT_EXPECT_FALSE(test,
+			   list_empty(&adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_HPD1]));
+	KUNIT_EXPECT_FALSE(test,
+			   list_empty(&adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_HPD1RX]));
+
+	amdgpu_dm_irq_fini(adev);
+}
+
+/**
+ * dm_test_register_hpd_handlers_invalid_hpd - Test invalid HPD source fails
+ * @test: The KUnit test context
+ */
+static void dm_test_register_hpd_handlers_invalid_hpd(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct amdgpu_device *adev;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	adev->dm.dc = dc;
+
+	aconn = kunit_kzalloc(test, sizeof(*aconn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn);
+	KUNIT_ASSERT_EQ(test, drm_connector_init(&adev->ddev, &aconn->base,
+						 &dm_test_connector_funcs,
+						 DRM_MODE_CONNECTOR_DisplayPort), 0);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_connector_cleanup, &aconn->base), 0);
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+	link->irq_source_hpd = DC_IRQ_SOURCE_HPD1RX;
+	link->irq_source_hpd_rx = DC_IRQ_SOURCE_INVALID;
+	aconn->dc_link = link;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_register_hpd_handlers(adev), -EINVAL);
+	amdgpu_dm_irq_fini(adev);
+}
+
+/**
+ * dm_test_register_hpd_handlers_invalid_hpd_rx - Test invalid HPDRX source fails
+ * @test: The KUnit test context
+ */
+static void dm_test_register_hpd_handlers_invalid_hpd_rx(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct amdgpu_device *adev;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	adev->dm.dc = dc;
+
+	aconn = kunit_kzalloc(test, sizeof(*aconn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn);
+	KUNIT_ASSERT_EQ(test, drm_connector_init(&adev->ddev, &aconn->base,
+						 &dm_test_connector_funcs,
+						 DRM_MODE_CONNECTOR_DisplayPort), 0);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_connector_cleanup, &aconn->base), 0);
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+	link->irq_source_hpd = DC_IRQ_SOURCE_INVALID;
+	link->irq_source_hpd_rx = DC_IRQ_SOURCE_HPD1;
+	aconn->dc_link = link;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_register_hpd_handlers(adev), -EINVAL);
+	amdgpu_dm_irq_fini(adev);
+}
+
+/**
+ * dm_test_register_hpd_handlers_dmub_outbox - Test DMUB outbox callbacks register
+ * @test: The KUnit test context
+ *
+ * Enables DMUB outbox support so dc_is_dmub_outbox_supported() returns true,
+ * exercising the dm_register_dmub_notify_callback() block. Asserts that the
+ * HPD, HPD_IRQ and HPD_SENSE_NOTIFY callbacks are registered.
+ */
+static void dm_test_register_hpd_handlers_dmub_outbox(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+	dc = dm_test_alloc_dc_with_ctx(test);
+	adev->dm.dc = dc;
+
+	/* Make dc_is_dmub_outbox_supported() return true. */
+	dc->caps.dmcub_support = true;
+	dc->ctx->asic_id.chip_family = AMDGPU_FAMILY_GC_11_0_1;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_register_hpd_handlers(adev), 0);
+	KUNIT_EXPECT_PTR_EQ(test, adev->dm.dmub_callback[DMUB_NOTIFICATION_HPD],
+			    (dmub_notify_interrupt_callback_t)dmub_hpd_callback);
+	KUNIT_EXPECT_PTR_EQ(test, adev->dm.dmub_callback[DMUB_NOTIFICATION_HPD_IRQ],
+			    (dmub_notify_interrupt_callback_t)dmub_hpd_callback);
+	KUNIT_EXPECT_PTR_EQ(test,
+			    adev->dm.dmub_callback[DMUB_NOTIFICATION_HPD_SENSE_NOTIFY],
+			    (dmub_notify_interrupt_callback_t)dmub_hpd_sense_callback);
+
+	amdgpu_dm_irq_fini(adev);
+}
+
+/* Tests for CRTC/pflip/vupdate high IRQ callbacks */
+
+/**
+ * dm_test_pflip_high_irq_no_crtc - Test pflip high IRQ with no matching CRTC
+ * @test: The KUnit test context
+ */
+static void dm_test_pflip_high_irq_no_crtc(struct kunit *test)
+{
+	struct common_irq_params params = { 0 };
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+	params.adev = adev;
+	params.irq_src = (enum dc_irq_source)IRQ_TYPE_PFLIP;
+
+	dm_pflip_high_irq(&params);
+}
+
+/**
+ * dm_test_pflip_high_irq_not_submitted - Test pflip high IRQ early status exit
+ * @test: The KUnit test context
+ */
+static void dm_test_pflip_high_irq_not_submitted(struct kunit *test)
+{
+	struct common_irq_params params = { 0 };
+	struct amdgpu_crtc *acrtc;
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	INIT_LIST_HEAD(&acrtc->base.head);
+	acrtc->base.dev = &adev->ddev;
+	acrtc->otg_inst = 0;
+	acrtc->pflip_status = AMDGPU_FLIP_NONE;
+	list_add_tail(&acrtc->base.head, &adev->ddev.mode_config.crtc_list);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test,
+				dm_test_crtc_list_del, acrtc), 0);
+
+	params.adev = adev;
+	params.irq_src = (enum dc_irq_source)IRQ_TYPE_PFLIP;
+
+	dm_pflip_high_irq(&params);
+	KUNIT_EXPECT_EQ(test, acrtc->pflip_status, AMDGPU_FLIP_NONE);
+}
+
+/**
+ * dm_test_vupdate_high_irq_no_crtc - Test vupdate high IRQ with no CRTC
+ * @test: The KUnit test context
+ */
+static void dm_test_vupdate_high_irq_no_crtc(struct kunit *test)
+{
+	struct common_irq_params params = { 0 };
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+	params.adev = adev;
+	params.irq_src = (enum dc_irq_source)IRQ_TYPE_VUPDATE;
+
+	dm_vupdate_high_irq(&params);
+}
+
+/**
+ * dm_test_crtc_high_irq_no_crtc - Test crtc high IRQ with no CRTC
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_high_irq_no_crtc(struct kunit *test)
+{
+	struct common_irq_params params = { 0 };
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+	params.adev = adev;
+	params.irq_src = (enum dc_irq_source)IRQ_TYPE_VBLANK;
+
+	dm_crtc_high_irq(&params);
+}
+
+/**
+ * dm_test_crtc_high_irq_vrr_pre_ai - Test crtc high IRQ VRR path on pre-AI ASIC
+ * @test: The KUnit test context
+ *
+ * With a matching CRTC, no writeback, VRR active (so the !vrr_active vblank
+ * handler is skipped) and a pre-AI family, the handler runs through CRC
+ * handling and returns before the freesync section.
+ */
+static void dm_test_crtc_high_irq_vrr_pre_ai(struct kunit *test)
+{
+	struct common_irq_params params = { 0 };
+	struct amdgpu_crtc *acrtc;
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	INIT_LIST_HEAD(&acrtc->base.head);
+	acrtc->base.dev = &adev->ddev;
+	acrtc->otg_inst = 0;
+	/* VRR active so the !vrr_active vblank handler is skipped. */
+	acrtc->dm_irq_params.freesync_config.state = VRR_STATE_ACTIVE_VARIABLE;
+	list_add_tail(&acrtc->base.head, &adev->ddev.mode_config.crtc_list);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test,
+				dm_test_crtc_list_del, acrtc), 0);
+
+	/* Pre-AI family returns right after CRC handling. */
+	adev->family = AMDGPU_FAMILY_SI;
+
+	params.adev = adev;
+	params.irq_src = (enum dc_irq_source)IRQ_TYPE_VBLANK;
+
+	dm_crtc_high_irq(&params);
+}
+
+/**
+ * dm_test_crtc_high_irq_vrr_ai_no_stream - Test crtc high IRQ AI path, no stream
+ * @test: The KUnit test context
+ *
+ * On an AI+ family the handler runs the post-CRC freesync section. With no
+ * stream and no pending flip, both inner blocks are skipped and the handler
+ * completes through the event-lock critical section, leaving pflip_status
+ * untouched.
+ */
+static void dm_test_crtc_high_irq_vrr_ai_no_stream(struct kunit *test)
+{
+	struct common_irq_params params = { 0 };
+	struct amdgpu_crtc *acrtc;
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	INIT_LIST_HEAD(&acrtc->base.head);
+	acrtc->base.dev = &adev->ddev;
+	acrtc->otg_inst = 0;
+	acrtc->pflip_status = AMDGPU_FLIP_NONE;
+	/* VRR active so the !vrr_active vblank handler is skipped. */
+	acrtc->dm_irq_params.freesync_config.state = VRR_STATE_ACTIVE_VARIABLE;
+	list_add_tail(&acrtc->base.head, &adev->ddev.mode_config.crtc_list);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test,
+				dm_test_crtc_list_del, acrtc), 0);
+
+	/* AI+ family runs the freesync section; no stream skips it. */
+	adev->family = AMDGPU_FAMILY_AI;
+
+	params.adev = adev;
+	params.irq_src = (enum dc_irq_source)IRQ_TYPE_VBLANK;
+
+	dm_crtc_high_irq(&params);
+	KUNIT_EXPECT_EQ(test, acrtc->pflip_status, AMDGPU_FLIP_NONE);
+}
+
+/* Tests for dm_handle_hpd_work() */
+
+/**
+ * dm_test_handle_hpd_work_out_of_range - Test HPD work frees unknown notification
+ * @test: The KUnit test context
+ */
+static void dm_test_handle_hpd_work_out_of_range(struct kunit *test)
+{
+	struct dmub_hpd_work *hpd_work;
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+	hpd_work = kzalloc(sizeof(*hpd_work), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hpd_work);
+	hpd_work->dmub_notify = kzalloc(sizeof(*hpd_work->dmub_notify), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hpd_work->dmub_notify);
+	hpd_work->dmub_notify->type =
+		(enum dmub_notification_type)ARRAY_SIZE(adev->dm.dmub_callback);
+	hpd_work->adev = adev;
+	INIT_WORK(&hpd_work->handle_hpd_work, dm_handle_hpd_work);
+
+	dm_handle_hpd_work(&hpd_work->handle_hpd_work);
+}
+
+/* Tests for dm_dmub_outbox1_low_irq() */
+
+/**
+ * dm_test_dmub_outbox1_low_irq_empty - Test outbox low IRQ with empty trace queue
+ * @test: The KUnit test context
+ */
+static void dm_test_dmub_outbox1_low_irq_empty(struct kunit *test)
+{
+	struct common_irq_params params = { 0 };
+	struct dc_dmub_srv *dc_dmub_srv;
+	struct amdgpu_device *adev;
+	struct dmub_srv *dmub;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	dc = dm_test_alloc_dc_with_ctx(test);
+	dc_dmub_srv = kunit_kzalloc(test, sizeof(*dc_dmub_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc_dmub_srv);
+	dmub = kunit_kzalloc(test, sizeof(*dmub), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dmub);
+
+	dmub->hw_funcs.get_outbox0_wptr = dm_test_dmub_get_outbox0_wptr;
+	dc_dmub_srv->dmub = dmub;
+	dc->ctx->dmub_srv = dc_dmub_srv;
+	adev->dm.dc = dc;
+	params.adev = adev;
+	params.irq_src = DC_IRQ_SOURCE_DMCUB_OUTBOX;
+
+	dm_dmub_outbox1_low_irq(&params);
+}
+
+/*
+ * dm_test_alloc_adev_outbox_notify - Build an adev wired for DMUB outbox
+ * notification handling.
+ *
+ * Configures dc/dc_dmub_srv/dmub so that the trace queue is empty and
+ * dc_enable_dmub_notifications() returns true, allowing the notification
+ * handling block of dm_dmub_outbox1_low_irq() to execute. The outbox1 ring
+ * buffer is left empty, so each notification read returns
+ * DMUB_NOTIFICATION_NO_DATA with no pending notification.
+ */
+static struct amdgpu_device *dm_test_alloc_adev_outbox_notify(struct kunit *test)
+{
+	struct dc_dmub_srv *dc_dmub_srv;
+	struct amdgpu_device *adev;
+	struct dmub_srv *dmub;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	dc = dm_test_alloc_dc_with_ctx(test);
+	dc_dmub_srv = kunit_kzalloc(test, sizeof(*dc_dmub_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc_dmub_srv);
+	dmub = kunit_kzalloc(test, sizeof(*dmub), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dmub);
+
+	dmub->hw_init = true;
+	dmub->hw_funcs.get_outbox0_wptr = dm_test_dmub_get_outbox0_wptr;
+	dmub->hw_funcs.get_outbox1_wptr = dm_test_dmub_get_outbox1_wptr;
+	dc_dmub_srv->dmub = dmub;
+	dc->ctx->dmub_srv = dc_dmub_srv;
+
+	/* Make dc_enable_dmub_notifications() return true. */
+	dc->caps.dmcub_support = true;
+	dc->ctx->asic_id.chip_family = AMDGPU_FAMILY_GC_11_0_1;
+
+	adev->dm.dc = dc;
+
+	return adev;
+}
+
+/**
+ * dm_test_dmub_outbox1_low_irq_no_handler - Test notification with no handler
+ * @test: The KUnit test context
+ *
+ * Notifications are enabled but no callback is registered for the returned
+ * notification type, exercising the skip-with-warning path.
+ */
+static void dm_test_dmub_outbox1_low_irq_no_handler(struct kunit *test)
+{
+	struct common_irq_params params = { 0 };
+	struct amdgpu_device *adev;
+
+	adev = dm_test_alloc_adev_outbox_notify(test);
+	params.adev = adev;
+	params.irq_src = DC_IRQ_SOURCE_DMCUB_OUTBOX;
+
+	dm_dmub_outbox1_low_irq(&params);
+}
+
+/**
+ * dm_test_dmub_outbox1_low_irq_direct_callback - Test direct callback dispatch
+ * @test: The KUnit test context
+ *
+ * Notifications are enabled with a registered callback and thread offload
+ * disabled, so the callback is invoked directly from the IRQ handler.
+ */
+static void dm_test_dmub_outbox1_low_irq_direct_callback(struct kunit *test)
+{
+	struct common_irq_params params = { 0 };
+	struct amdgpu_device *adev;
+
+	adev = dm_test_alloc_adev_outbox_notify(test);
+	adev->dm.dmub_callback[DMUB_NOTIFICATION_NO_DATA] = dm_test_dmub_notify_callback;
+	adev->dm.dmub_thread_offload[DMUB_NOTIFICATION_NO_DATA] = false;
+	params.adev = adev;
+	params.irq_src = DC_IRQ_SOURCE_DMCUB_OUTBOX;
+
+	dm_test_dmub_notify_count = 0;
+
+	dm_dmub_outbox1_low_irq(&params);
+
+	KUNIT_EXPECT_EQ(test, dm_test_dmub_notify_count, 1);
+}
+
+/**
+ * dm_test_dmub_outbox1_low_irq_offload - Test offloaded callback dispatch
+ * @test: The KUnit test context
+ *
+ * Notifications are enabled with a registered callback and thread offload
+ * enabled, so the callback is dispatched via the delayed HPD work queue.
+ */
+static void dm_test_dmub_outbox1_low_irq_offload(struct kunit *test)
+{
+	struct common_irq_params params = { 0 };
+	struct amdgpu_device *adev;
+
+	adev = dm_test_alloc_adev_outbox_notify(test);
+	adev->dm.dmub_callback[DMUB_NOTIFICATION_NO_DATA] = dm_test_dmub_notify_callback;
+	adev->dm.dmub_thread_offload[DMUB_NOTIFICATION_NO_DATA] = true;
+	adev->dm.delayed_hpd_wq = create_singlethread_workqueue("dm_irq_test_outbox");
+	KUNIT_ASSERT_NOT_NULL(test, adev->dm.delayed_hpd_wq);
+	params.adev = adev;
+	params.irq_src = DC_IRQ_SOURCE_DMCUB_OUTBOX;
+
+	dm_test_dmub_notify_count = 0;
+
+	dm_dmub_outbox1_low_irq(&params);
+
+	flush_workqueue(adev->dm.delayed_hpd_wq);
+	destroy_workqueue(adev->dm.delayed_hpd_wq);
+
+	KUNIT_EXPECT_EQ(test, dm_test_dmub_notify_count, 1);
+}
+
+
+/**
+ * dm_test_dce110_register_irq_handlers_rejects_uninitialized_sources - Test DCE110 error
+ * @test: The KUnit test context
+ */
+static void dm_test_dce110_register_irq_handlers_rejects_uninitialized_sources(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	dc = dm_test_alloc_dc_with_ctx(test);
+	adev->dm.dc = dc;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_dce110_register_irq_handlers(adev), -EINVAL);
+}
+
+/**
+ * dm_test_dce110_register_irq_handlers_one_crtc - Test DCE110 with 1 CRTC
+ * @test: The KUnit test context
+ *
+ * Exercises the VBLANK, VUPDATE, PFLIP and HPD for-loop bodies with a
+ * fake IRQ service that maps source IDs to DC IRQ sources.
+ */
+static void dm_test_dce110_register_irq_handlers_one_crtc(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_free_irq_sources,
+							 adev), 0);
+	dc = dm_test_alloc_dc_with_irq_service(test, &dm_test_irq_service_funcs_dce110);
+	dc->ctx->dce_version = DCE_VERSION_11_0;
+	adev->dm.dc = dc;
+	adev->mode_info.num_crtc = 1;
+	adev->mode_info.num_hpd = 1;
+	amdgpu_dm_set_irq_funcs(adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_dce110_register_irq_handlers(adev), 0);
+
+	/* Verify VBLANK params were populated */
+	KUNIT_EXPECT_EQ(test, (int)adev->dm.vblank_params[0].irq_src,
+			(int)DC_IRQ_SOURCE_VBLANK1);
+	KUNIT_EXPECT_PTR_EQ(test, adev->dm.vblank_params[0].adev, adev);
+
+	/* Verify VUPDATE params were populated (VRR supported on DCE 11) */
+	KUNIT_EXPECT_EQ(test, (int)adev->dm.vupdate_params[0].irq_src,
+			(int)DC_IRQ_SOURCE_VUPDATE1);
+
+	/* Verify PFLIP params were populated (6 fixed entries) */
+	KUNIT_EXPECT_EQ(test, (int)adev->dm.pflip_params[0].irq_src,
+			(int)DC_IRQ_SOURCE_PFLIP1);
+	KUNIT_EXPECT_EQ(test, (int)adev->dm.pflip_params[5].irq_src,
+			(int)DC_IRQ_SOURCE_PFLIP6);
+
+	amdgpu_dm_irq_fini(adev);
+}
+
+/**
+ * dm_test_dcn10_register_irq_handlers_zero_crtc - Test DCN10 zero-CRTC registration
+ * @test: The KUnit test context
+ */
+static void dm_test_dcn10_register_irq_handlers_zero_crtc(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_free_irq_sources,
+							 adev), 0);
+	dc = dm_test_alloc_dc_with_ctx(test);
+	adev->dm.dc = dc;
+	adev->mode_info.num_hpd = 1;
+	amdgpu_dm_set_irq_funcs(adev);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_dcn10_register_irq_handlers(adev), 0);
+	KUNIT_ASSERT_NOT_NULL(test, adev->irq.client[SOC15_IH_CLIENTID_DCE].sources);
+	KUNIT_EXPECT_PTR_EQ(test,
+		adev->irq.client[SOC15_IH_CLIENTID_DCE].sources[DCN_1_0__SRCID__DC_HPD1_INT],
+		&adev->hpd_irq);
+}
+
+/**
+ * dm_test_dcn10_register_irq_handlers_one_crtc - Test DCN10 with 1 CRTC
+ * @test: The KUnit test context
+ *
+ * Exercises the VUPDATE_NO_LOCK for-loop bodies with a fake IRQ service, and
+ * that it's the only one populated (other than VLINE0, for secure display).
+ */
+static void dm_test_dcn10_register_irq_handlers_one_crtc(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_free_irq_sources,
+							 adev), 0);
+	dc = dm_test_alloc_dc_with_irq_service(test, &dm_test_irq_service_funcs_dcn10);
+	adev->dm.dc = dc;
+	adev->mode_info.num_crtc = 1;
+	adev->mode_info.num_hpd = 1;
+	dc->caps.max_otg_num = 1;
+	amdgpu_dm_set_irq_funcs(adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_dcn10_register_irq_handlers(adev), 0);
+
+	/* Verify VBLANK params were *not* populated */
+	KUNIT_EXPECT_EQ(test, (int)adev->dm.vblank_params[0].irq_src, 0);
+	KUNIT_EXPECT_NULL(test, adev->dm.vblank_params[0].adev);
+
+	/* Verify VUPDATE params were populated */
+	KUNIT_EXPECT_EQ(test, (int)adev->dm.vupdate_params[0].irq_src,
+			(int)DC_IRQ_SOURCE_VUPDATE1);
+
+	/* Verify PFLIP params were *not* populated */
+	KUNIT_EXPECT_EQ(test, (int)adev->dm.pflip_params[0].irq_src, 0);
+
+	amdgpu_dm_irq_fini(adev);
+}
+
+/**
+ * dm_test_register_outbox_irq_handlers_without_dmub - Test outbox registration without DMUB
+ * @test: The KUnit test context
+ */
+static void dm_test_register_outbox_irq_handlers_without_dmub(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_free_irq_sources,
+							 adev), 0);
+	dc = dm_test_alloc_dc_with_ctx(test);
+	adev->dm.dc = dc;
+	amdgpu_dm_set_irq_funcs(adev);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_register_outbox_irq_handlers(adev), 0);
+	KUNIT_ASSERT_NOT_NULL(test, adev->irq.client[SOC15_IH_CLIENTID_DCE].sources);
+	KUNIT_EXPECT_PTR_EQ(test,
+		adev->irq.client[SOC15_IH_CLIENTID_DCE].sources[
+			DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT],
+		&adev->dmub_outbox_irq);
+}
+
+/**
+ * dm_test_register_outbox_irq_handlers_with_dmub - Test outbox registration with DMUB
+ * @test: The KUnit test context
+ *
+ * Exercises the dc->ctx->dmub_srv branch which maps the outbox source and
+ * registers dm_dmub_outbox1_low_irq in the low IRQ context table.
+ */
+static void dm_test_register_outbox_irq_handlers_with_dmub(struct kunit *test)
+{
+	struct dc_dmub_srv *dc_dmub_srv;
+	struct amdgpu_device *adev;
+	struct list_head *hnd_list;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_free_irq_sources,
+							 adev), 0);
+	dc = dm_test_alloc_dc_with_irq_service(test, &dm_test_irq_service_funcs_dcn10);
+	dc_dmub_srv = kunit_kzalloc(test, sizeof(*dc_dmub_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc_dmub_srv);
+	dc->ctx->dmub_srv = dc_dmub_srv;
+	adev->dm.dc = dc;
+	amdgpu_dm_set_irq_funcs(adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_register_outbox_irq_handlers(adev), 0);
+
+	/* Verify the outbox IRQ params were populated */
+	KUNIT_EXPECT_EQ(test, (int)adev->dm.dmub_outbox_params[0].irq_src,
+			(int)DC_IRQ_SOURCE_DMCUB_OUTBOX);
+	KUNIT_EXPECT_PTR_EQ(test, adev->dm.dmub_outbox_params[0].adev, adev);
+
+	/* Verify a low-context handler was registered for the outbox source */
+	hnd_list = &adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_DMCUB_OUTBOX];
+	KUNIT_EXPECT_FALSE(test, list_empty(hnd_list));
+
+	amdgpu_dm_irq_fini(adev);
+}
+
+/* Tests for amdgpu_dm_irq_handler() */
+
+/**
+ * dm_test_irq_handler_dispatches_work - Test the top-level IRQ handler
+ * @test: The KUnit test context
+ *
+ * amdgpu_dm_irq_handler() translates the hardware IRQ entry to a DC IRQ
+ * source, acknowledges it, then dispatches to the high-context (immediate)
+ * and low-context (scheduled) handler lists. A fake dc with a stubbed
+ * irq_service maps the source id to DC_IRQ_SOURCE_VBLANK1 and lets the ack
+ * succeed without touching hardware registers.
+ */
+static void dm_test_irq_handler_dispatches_work(struct kunit *test)
+{
+	struct dc_interrupt_params int_params = { 0 };
+	struct amdgpu_iv_entry entry = { 0 };
+	struct amdgpu_device *adev;
+	int high_count = 0;
+	int low_count = 0;
+	void *handler;
+	struct dc *dc;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	dc = dm_test_alloc_dc_with_irq_service(test,
+					       &dm_test_irq_service_funcs_dce110);
+	adev->dm.dc = dc;
+
+	/* High-context (immediate) handler on VBLANK1. */
+	int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_VBLANK1;
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						   dm_test_irq_handler_count,
+						   &high_count);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+
+	/* Low-context (scheduled) handler on VBLANK1. */
+	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						   dm_test_irq_handler_count,
+						   &low_count);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+
+	/* src_id maps to DC_IRQ_SOURCE_VBLANK1 via the dce110 stub. */
+	entry.src_id = VISLANDS30_IV_SRCID_D1_VERTICAL_INTERRUPT0;
+	entry.src_data[0] = 0;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_irq_handler(adev, NULL, &entry), 0);
+
+	/* High-context handler runs synchronously in-place. */
+	KUNIT_EXPECT_EQ(test, high_count, 1);
+
+	/*
+	 * Low-context work runs asynchronously; amdgpu_dm_irq_fini() flushes
+	 * each pending work item before freeing, so it has run afterwards.
+	 */
+	amdgpu_dm_irq_fini(adev);
+	KUNIT_EXPECT_EQ(test, low_count, 1);
+}
+
+/* Tests for dm_handle_vmin_vmax_update() */
+
+/**
+ * dm_test_handle_vmin_vmax_update - Test the deferred vmin/vmax worker
+ * @test: The KUnit test context
+ *
+ * The worker applies the cached timing adjust to the stream via
+ * dc_stream_adjust_vmin_vmax(), drops the stream reference taken when the
+ * work was scheduled, and frees the work and its adjust copy. A fake dc with
+ * a current_state lets the adjust walk an empty pipe list without touching
+ * hardware. The work and adjust are kmalloc'd because the worker frees them.
+ */
+static void dm_test_handle_vmin_vmax_update(struct kunit *test)
+{
+	struct dc_crtc_timing_adjust *adjust;
+	struct vupdate_offload_work *work;
+	struct dc_stream_state *stream;
+	struct amdgpu_device *adev;
+	struct dc *dc;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	mutex_init(&adev->dm.dc_lock);
+
+	dc = dm_test_alloc_dc_with_ctx(test);
+	dc->current_state = kunit_kzalloc(test, sizeof(*dc->current_state),
+					  GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc->current_state);
+	adev->dm.dc = dc;
+
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+	/*
+	 * Start at two references: the worker's dc_stream_release() drops one,
+	 * leaving the kunit-managed allocation intact (no kfree).
+	 */
+	kref_init(&stream->refcount);
+	kref_get(&stream->refcount);
+
+	/* The worker kfree()s both, so they must come from the slab. */
+	work = kzalloc(sizeof(*work), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, work);
+	adjust = kzalloc(sizeof(*adjust), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adjust);
+
+	work->adev = adev;
+	work->stream = stream;
+	work->adjust = adjust;
+	adjust->v_total_min = 1000;
+	adjust->v_total_max = 1100;
+
+	dm_handle_vmin_vmax_update(&work->work);
+
+	/* The adjust was applied and one stream reference was dropped. */
+	KUNIT_EXPECT_EQ(test, stream->adjust.v_total_min, 1000);
+	KUNIT_EXPECT_EQ(test, stream->adjust.v_total_max, 1100);
+	KUNIT_EXPECT_EQ(test, kref_read(&stream->refcount), 1);
+}
+
+static struct kunit_case amdgpu_dm_irq_tests[] = {
+	/* amdgpu_dm_hpd_to_dal_irq_source */
+	KUNIT_CASE(dm_test_hpd_to_dal_irq_source_hpd1),
+	KUNIT_CASE(dm_test_hpd_to_dal_irq_source_hpd2),
+	KUNIT_CASE(dm_test_hpd_to_dal_irq_source_hpd3),
+	KUNIT_CASE(dm_test_hpd_to_dal_irq_source_hpd4),
+	KUNIT_CASE(dm_test_hpd_to_dal_irq_source_hpd5),
+	KUNIT_CASE(dm_test_hpd_to_dal_irq_source_hpd6),
+	KUNIT_CASE(dm_test_hpd_to_dal_irq_source_invalid),
+	KUNIT_CASE(dm_test_hpd_to_dal_irq_source_out_of_range),
+	/* are_sinks_equal */
+	KUNIT_CASE(dm_test_are_sinks_equal_both_null),
+	KUNIT_CASE(dm_test_are_sinks_equal_first_null),
+	KUNIT_CASE(dm_test_are_sinks_equal_second_null),
+	KUNIT_CASE(dm_test_are_sinks_equal_different_signal),
+	KUNIT_CASE(dm_test_are_sinks_equal_different_edid_length),
+	KUNIT_CASE(dm_test_are_sinks_equal_different_edid_data),
+	KUNIT_CASE(dm_test_are_sinks_equal_identical),
+	KUNIT_CASE(dm_test_are_sinks_equal_zero_length),
+	KUNIT_CASE(dm_test_are_sinks_equal_full_edid_identical),
+	KUNIT_CASE(dm_test_are_sinks_equal_full_edid_last_byte_differs),
+	/* dmub_notification_type_str */
+	KUNIT_CASE(dm_test_notification_str_no_data),
+	KUNIT_CASE(dm_test_notification_str_aux_reply),
+	KUNIT_CASE(dm_test_notification_str_hpd),
+	KUNIT_CASE(dm_test_notification_str_hpd_irq),
+	KUNIT_CASE(dm_test_notification_str_set_config),
+	KUNIT_CASE(dm_test_notification_str_dpia),
+	KUNIT_CASE(dm_test_notification_str_hpd_sense),
+	KUNIT_CASE(dm_test_notification_str_fused_io),
+	KUNIT_CASE(dm_test_notification_str_unknown),
+	/* amdgpu_dm_irq_init */
+	KUNIT_CASE(dm_test_irq_init_initializes_lists),
+	/* amdgpu_dm_irq_register_interrupt */
+	KUNIT_CASE(dm_test_irq_register_rejects_null_params),
+	KUNIT_CASE(dm_test_irq_register_rejects_invalid_context),
+	KUNIT_CASE(dm_test_irq_register_rejects_invalid_source),
+	KUNIT_CASE(dm_test_irq_register_adds_low_context_handler),
+	KUNIT_CASE(dm_test_irq_register_adds_high_context_handler),
+	KUNIT_CASE(dm_test_irq_register_multiple_handlers),
+	KUNIT_CASE(dm_test_irq_register_separate_contexts),
+	/* amdgpu_dm_irq_unregister_interrupt */
+	KUNIT_CASE(dm_test_irq_unregister_rejects_invalid_source),
+	KUNIT_CASE(dm_test_irq_unregister_rejects_null_handler),
+	KUNIT_CASE(dm_test_irq_unregister_handler_not_found),
+	/* amdgpu_dm_irq_fini */
+	KUNIT_CASE(dm_test_irq_fini_removes_registered_handlers),
+	KUNIT_CASE(dm_test_irq_fini_on_empty_tables),
+	/* amdgpu_dm_get_crtc_by_otg_inst */
+	KUNIT_CASE(dm_test_get_crtc_by_otg_inst_returns_match),
+	KUNIT_CASE(dm_test_get_crtc_by_otg_inst_returns_null),
+	KUNIT_CASE(dm_test_get_crtc_by_otg_inst_empty_list),
+	/* amdgpu_dm_set_irq_funcs */
+	KUNIT_CASE(dm_test_set_irq_funcs),
+	/* amdgpu_dm_irq_suspend/resume_early/resume_late */
+	KUNIT_CASE(dm_test_irq_suspend_empty),
+	KUNIT_CASE(dm_test_irq_resume_early_empty),
+	KUNIT_CASE(dm_test_irq_resume_late_empty),
+	KUNIT_CASE(dm_test_irq_suspend_registered),
+	KUNIT_CASE(dm_test_irq_suspend_disables_polling),
+	KUNIT_CASE(dm_test_irq_resume_early_registered),
+	KUNIT_CASE(dm_test_irq_resume_late_registered),
+	KUNIT_CASE(dm_test_irq_resume_late_enables_polling),
+	/* amdgpu_dm_hpd_rx_irq_create_workqueue */
+	KUNIT_CASE(dm_test_hpd_rx_irq_create_workqueue),
+	/* amdgpu_dm_hpd_rx_irq_work_suspend */
+	KUNIT_CASE(dm_test_hpd_rx_irq_work_suspend_null),
+	KUNIT_CASE(dm_test_hpd_rx_irq_work_suspend_flushes),
+	/* CRTC-based irq state callbacks (no-CRTC early return) */
+	KUNIT_CASE(dm_test_set_crtc_irq_state_no_crtc),
+	KUNIT_CASE(dm_test_set_pflip_irq_state_no_crtc),
+	KUNIT_CASE(dm_test_set_vline0_irq_state_no_crtc),
+	KUNIT_CASE(dm_test_set_vupdate_irq_state_no_crtc),
+	/* CRTC-based irq state callbacks (dm_irq_state happy path) */
+	KUNIT_CASE(dm_test_set_crtc_irq_state_otg_disabled),
+	KUNIT_CASE(dm_test_set_crtc_irq_state_enable),
+	KUNIT_CASE(dm_test_set_pflip_irq_state_disable),
+	KUNIT_CASE(dm_test_set_vline0_irq_state_enable),
+	KUNIT_CASE(dm_test_set_vupdate_irq_state_enable),
+	KUNIT_CASE(dm_test_set_crtc_irq_state_allows_idle),
+	/* amdgpu_dm_irq_immediate_work */
+	KUNIT_CASE(dm_test_irq_immediate_work_empty),
+	KUNIT_CASE(dm_test_irq_immediate_work_invokes_handler),
+	KUNIT_CASE(dm_test_irq_immediate_work_invokes_all),
+	/* amdgpu_dm_irq_schedule_work */
+	KUNIT_CASE(dm_test_irq_schedule_work_empty),
+	KUNIT_CASE(dm_test_irq_schedule_work_queues_handler),
+	KUNIT_CASE(dm_test_irq_schedule_work_requeue_fallback),
+	/* amdgpu_dm_set_hpd_irq_state */
+	KUNIT_CASE(dm_test_set_hpd_irq_state_null_dc),
+	/* amdgpu_dm_set_dmub_outbox_irq_state */
+	KUNIT_CASE(dm_test_set_dmub_outbox_irq_state_null_dc),
+	/* amdgpu_dm_set_dmub_trace_irq_state */
+	KUNIT_CASE(dm_test_set_dmub_trace_irq_state_null_dc),
+	/* amdgpu_dm_outbox_init */
+	KUNIT_CASE(dm_test_outbox_init_null_dc),
+	/* amdgpu_dm_hpd_init/amdgpu_dm_hpd_fini */
+	KUNIT_CASE(dm_test_hpd_init_empty_connectors),
+	KUNIT_CASE(dm_test_hpd_fini_empty_connectors),
+	KUNIT_CASE(dm_test_hpd_init_fini_with_connectors),
+	KUNIT_CASE(dm_test_hpd_init_fini_analog_connector),
+	KUNIT_CASE(dm_test_hpd_init_fini_irq_ref),
+	/* dm_handle_hpd_rx_offload_work */
+	KUNIT_CASE(dm_test_hpd_rx_offload_work_no_connector),
+	KUNIT_CASE(dm_test_hpd_rx_offload_work_no_connection),
+	KUNIT_CASE(dm_test_hpd_rx_offload_work_automated_test),
+	KUNIT_CASE(dm_test_hpd_rx_offload_work_link_loss),
+	/* amdgpu_dm_hdmi_hpd_debounce_work */
+	KUNIT_CASE(dm_test_hdmi_hpd_debounce_detect_false),
+	KUNIT_CASE(dm_test_hdmi_hpd_debounce_reallow_idle),
+	/* handle_hpd_irq/handle_hpd_irq_helper */
+	KUNIT_CASE(dm_test_handle_hpd_irq_disabled),
+	KUNIT_CASE(dm_test_handle_hpd_irq_helper_debounce_schedule),
+	KUNIT_CASE(dm_test_handle_hpd_irq_helper_debounce_release_prev),
+	KUNIT_CASE(dm_test_handle_hpd_irq_helper_detect_false),
+	/* handle_hpd_rx_irq/schedule_hpd_rx_offload_work */
+	KUNIT_CASE(dm_test_handle_hpd_rx_irq_disabled),
+	KUNIT_CASE(dm_test_handle_hpd_rx_irq_no_left_work),
+	KUNIT_CASE(dm_test_handle_hpd_rx_irq_automated_test),
+	KUNIT_CASE(dm_test_handle_hpd_rx_irq_msg_rdy),
+	KUNIT_CASE(dm_test_handle_hpd_rx_irq_link_loss),
+	KUNIT_CASE(dm_test_schedule_hpd_rx_offload_work),
+	/* dmub_hpd_callback/dmub_hpd_sense_callback */
+	KUNIT_CASE(dm_test_dmub_hpd_callback_null_inputs),
+	KUNIT_CASE(dm_test_dmub_hpd_callback_invalid_index),
+	KUNIT_CASE(dm_test_dmub_hpd_callback_empty_connectors),
+	KUNIT_CASE(dm_test_dmub_hpd_callback_unknown_type_match),
+	KUNIT_CASE(dm_test_dmub_hpd_callback_hpd_type),
+	KUNIT_CASE(dm_test_dmub_hpd_callback_hpd_irq_type),
+	/* amdgpu_dm_register_hpd_handlers */
+	KUNIT_CASE(dm_test_register_hpd_handlers_empty),
+	KUNIT_CASE(dm_test_register_hpd_handlers_valid),
+	KUNIT_CASE(dm_test_register_hpd_handlers_invalid_hpd),
+	KUNIT_CASE(dm_test_register_hpd_handlers_invalid_hpd_rx),
+	KUNIT_CASE(dm_test_register_hpd_handlers_dmub_outbox),
+	/* pflip/vupdate/crtc high IRQ callbacks */
+	KUNIT_CASE(dm_test_pflip_high_irq_no_crtc),
+	KUNIT_CASE(dm_test_pflip_high_irq_not_submitted),
+	KUNIT_CASE(dm_test_vupdate_high_irq_no_crtc),
+	KUNIT_CASE(dm_test_crtc_high_irq_no_crtc),
+	KUNIT_CASE(dm_test_crtc_high_irq_vrr_pre_ai),
+	KUNIT_CASE(dm_test_crtc_high_irq_vrr_ai_no_stream),
+	/* dm_handle_hpd_work */
+	KUNIT_CASE(dm_test_handle_hpd_work_out_of_range),
+	/* dm_dmub_outbox1_low_irq */
+	KUNIT_CASE(dm_test_dmub_outbox1_low_irq_empty),
+	KUNIT_CASE(dm_test_dmub_outbox1_low_irq_no_handler),
+	KUNIT_CASE(dm_test_dmub_outbox1_low_irq_direct_callback),
+	KUNIT_CASE(dm_test_dmub_outbox1_low_irq_offload),
+	/* IRQ handler registration helpers */
+	KUNIT_CASE(dm_test_dce110_register_irq_handlers_rejects_uninitialized_sources),
+	KUNIT_CASE(dm_test_dce110_register_irq_handlers_one_crtc),
+	KUNIT_CASE(dm_test_dcn10_register_irq_handlers_zero_crtc),
+	KUNIT_CASE(dm_test_dcn10_register_irq_handlers_one_crtc),
+	KUNIT_CASE(dm_test_register_outbox_irq_handlers_without_dmub),
+	KUNIT_CASE(dm_test_register_outbox_irq_handlers_with_dmub),
+	/* amdgpu_dm_irq_handler */
+	KUNIT_CASE(dm_test_irq_handler_dispatches_work),
+	/* dm_handle_vmin_vmax_update */
+	KUNIT_CASE(dm_test_handle_vmin_vmax_update),
+	{}
+};
+
+static struct kunit_suite amdgpu_dm_irq_test_suite = {
+	.name = "amdgpu_dm_irq",
+	.test_cases = amdgpu_dm_irq_tests,
+};
+
+kunit_test_suite(amdgpu_dm_irq_test_suite);
+
+MODULE_AUTHOR("AMD");
+MODULE_DESCRIPTION("KUnit tests for amdgpu_dm_irq");
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_ism_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_ism_test.c
index f3b3f77aafd53..6394b967ac3ae 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_ism_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_ism_test.c
@@ -8,21 +8,11 @@
 #include <kunit/test.h>
 
 #include "dc.h"
+#include "amdgpu.h"
+#include "amdgpu_mode.h"
+#include "amdgpu_dm.h"
 #include "amdgpu_dm_ism.h"
-
-/*
- * Helper: allocate and zero-initialise a dc_stream_state for timing tests.
- * Only the timing sub-struct is accessed by the functions under test.
- */
-static struct dc_stream_state *alloc_test_stream(struct kunit *test)
-{
-	struct dc_stream_state *stream;
-
-	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, stream);
-
-	return stream;
-}
+#include "amdgpu_dm_kunit_test_helpers.h"
 
 /*
  * Helper: allocate and zero-initialise an ISM instance.
@@ -275,7 +265,7 @@ static void dm_test_ism_sso_delay_null_stream(struct kunit *test)
 static void dm_test_ism_sso_delay_zero_frames(struct kunit *test)
 {
 	struct amdgpu_dm_ism *ism = alloc_test_ism(test);
-	struct dc_stream_state *stream = alloc_test_stream(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
 
 	stream->timing.v_total = 1125;
 	stream->timing.h_total = 2200;
@@ -288,7 +278,7 @@ static void dm_test_ism_sso_delay_zero_frames(struct kunit *test)
 static void dm_test_ism_sso_delay_1080p60_3frames(struct kunit *test)
 {
 	struct amdgpu_dm_ism *ism = alloc_test_ism(test);
-	struct dc_stream_state *stream = alloc_test_stream(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
 	uint64_t expected_one_frame_ns, expected;
 
 	/*
@@ -311,7 +301,7 @@ static void dm_test_ism_sso_delay_1080p60_3frames(struct kunit *test)
 static void dm_test_ism_sso_delay_4k60_1frame(struct kunit *test)
 {
 	struct amdgpu_dm_ism *ism = alloc_test_ism(test);
-	struct dc_stream_state *stream = alloc_test_stream(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
 	uint64_t expected_one_frame_ns;
 
 	/*
@@ -347,7 +337,7 @@ static void dm_test_ism_idle_delay_null_stream(struct kunit *test)
 static void dm_test_ism_idle_delay_zero_filter_frames(struct kunit *test)
 {
 	struct amdgpu_dm_ism *ism = alloc_test_ism(test);
-	struct dc_stream_state *stream = alloc_test_stream(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
 
 	stream->timing.v_total = 1125;
 	stream->timing.h_total = 2200;
@@ -361,7 +351,7 @@ static void dm_test_ism_idle_delay_zero_filter_frames(struct kunit *test)
 static void dm_test_ism_idle_delay_zero_entry_count(struct kunit *test)
 {
 	struct amdgpu_dm_ism *ism = alloc_test_ism(test);
-	struct dc_stream_state *stream = alloc_test_stream(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
 
 	stream->timing.v_total = 1125;
 	stream->timing.h_total = 2200;
@@ -376,7 +366,7 @@ static void dm_test_ism_idle_delay_zero_entry_count(struct kunit *test)
 static void dm_test_ism_idle_delay_zero_delay_frames(struct kunit *test)
 {
 	struct amdgpu_dm_ism *ism = alloc_test_ism(test);
-	struct dc_stream_state *stream = alloc_test_stream(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
 
 	stream->timing.v_total = 1125;
 	stream->timing.h_total = 2200;
@@ -392,7 +382,7 @@ static void dm_test_ism_idle_delay_zero_delay_frames(struct kunit *test)
 static void dm_test_ism_idle_delay_no_short_idles(struct kunit *test)
 {
 	struct amdgpu_dm_ism *ism = alloc_test_ism(test);
-	struct dc_stream_state *stream = alloc_test_stream(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
 	uint64_t one_frame_ns;
 
 	/*
@@ -426,7 +416,7 @@ static void dm_test_ism_idle_delay_no_short_idles(struct kunit *test)
 static void dm_test_ism_idle_delay_enough_short_idles(struct kunit *test)
 {
 	struct amdgpu_dm_ism *ism = alloc_test_ism(test);
-	struct dc_stream_state *stream = alloc_test_stream(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
 	uint64_t one_frame_ns, expected;
 
 	/*
@@ -461,7 +451,7 @@ static void dm_test_ism_idle_delay_enough_short_idles(struct kunit *test)
 static void dm_test_ism_idle_delay_wraps_around_buffer(struct kunit *test)
 {
 	struct amdgpu_dm_ism *ism = alloc_test_ism(test);
-	struct dc_stream_state *stream = alloc_test_stream(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
 	uint64_t one_frame_ns, expected;
 
 	/*
@@ -497,7 +487,7 @@ static void dm_test_ism_idle_delay_wraps_around_buffer(struct kunit *test)
 static void dm_test_ism_idle_delay_old_history_cutoff(struct kunit *test)
 {
 	struct amdgpu_dm_ism *ism = alloc_test_ism(test);
-	struct dc_stream_state *stream = alloc_test_stream(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
 	uint64_t one_frame_ns;
 
 	/*
@@ -545,7 +535,7 @@ static void dm_test_ism_idle_delay_old_history_cutoff(struct kunit *test)
 static void dm_test_ism_idle_delay_mixed_durations(struct kunit *test)
 {
 	struct amdgpu_dm_ism *ism = alloc_test_ism(test);
-	struct dc_stream_state *stream = alloc_test_stream(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
 	uint64_t one_frame_ns;
 
 	/*
@@ -586,7 +576,7 @@ static void dm_test_ism_idle_delay_mixed_durations(struct kunit *test)
 static void dm_test_ism_idle_delay_entry_count_exceeds_history_size(struct kunit *test)
 {
 	struct amdgpu_dm_ism *ism = alloc_test_ism(test);
-	struct dc_stream_state *stream = alloc_test_stream(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
 	uint64_t one_frame_ns, expected;
 
 	/*
@@ -861,6 +851,345 @@ static void dm_test_dispatch_next_event_no_action_state(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, (int)result, (int)DM_ISM_NUM_EVENTS);
 }
 
+/*
+ * Helper: allocate an amdgpu_crtc (which embeds the ISM) wired up to a
+ * minimally-populated amdgpu_device so that the container_of()/drm_to_adev()
+ * lookups inside the ISM event machinery resolve correctly.
+ *
+ * The amdgpu_device is large, so it must be heap-allocated via kunit_kzalloc.
+ * acrtc->base.dev points at the embedded &adev->ddev, which is what
+ * drm_to_adev() expects (it is a container_of of ddev).
+ */
+static struct amdgpu_crtc *alloc_test_acrtc(struct kunit *test,
+					    struct amdgpu_device **adev_out)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+	struct dc *dc;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+
+	adev->dm.dc = dc;
+	adev->dm.ddev = &adev->ddev;
+	mutex_init(&adev->dm.dc_lock);
+
+	acrtc->base.dev = &adev->ddev;
+
+	if (adev_out)
+		*adev_out = adev;
+
+	return acrtc;
+}
+
+/*
+ * Helper: register an already-allocated amdgpu_crtc into the DRM device's
+ * crtc_list so that drm_for_each_crtc() iterates it. Only the list linkage is
+ * required for the ISM enable/disable/force-full-power helpers.
+ */
+static void register_test_acrtc(struct amdgpu_device *adev,
+				struct amdgpu_crtc *acrtc)
+{
+	INIT_LIST_HEAD(&adev->ddev.mode_config.crtc_list);
+	INIT_LIST_HEAD(&acrtc->base.head);
+	list_add_tail(&acrtc->base.head, &adev->ddev.mode_config.crtc_list);
+}
+
+/* ===== Tests for amdgpu_dm_ism_commit_event ===== */
+
+/**
+ * dm_test_ism_commit_event_no_state - commit_event returns early without a crtc state
+ * @test: KUnit test context
+ *
+ * When the CRTC has no atomic state (base.state == NULL) the function takes
+ * the NO_STATE early-return path and the FSM is left untouched.
+ */
+static void dm_test_ism_commit_event_no_state(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc = alloc_test_acrtc(test, &adev);
+	struct amdgpu_dm_ism_config config = { 0 };
+
+	amdgpu_dm_ism_init(&acrtc->ism, &config);
+	acrtc->base.state = NULL;
+
+	guard(mutex)(&adev->dm.dc_lock);
+	amdgpu_dm_ism_commit_event(&acrtc->ism,
+				   DM_ISM_EVENT_BEGIN_CURSOR_UPDATE);
+
+	KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+			(int)DM_ISM_STATE_FULL_POWER_RUNNING);
+
+	amdgpu_dm_ism_fini(&acrtc->ism);
+}
+
+/**
+ * dm_test_ism_commit_event_cursor_transition - cursor begin/end drive FSM without DC work
+ * @test: KUnit test context
+ *
+ * BEGIN_CURSOR_UPDATE then END_CURSOR_UPDATE from FULL_POWER_RUNNING traverse
+ * the FULL_POWER_BUSY state. Neither transition reaches the idle-optimization
+ * commit path, so no DC hardware access occurs and the FSM returns to
+ * FULL_POWER_RUNNING.
+ */
+static void dm_test_ism_commit_event_cursor_transition(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc = alloc_test_acrtc(test, &adev);
+	struct dm_crtc_state *dm_state;
+	struct amdgpu_dm_ism_config config = { 0 };
+
+	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+
+	amdgpu_dm_ism_init(&acrtc->ism, &config);
+	acrtc->base.state = &dm_state->base;
+
+	guard(mutex)(&adev->dm.dc_lock);
+	amdgpu_dm_ism_commit_event(&acrtc->ism,
+				   DM_ISM_EVENT_BEGIN_CURSOR_UPDATE);
+	KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+			(int)DM_ISM_STATE_FULL_POWER_BUSY);
+
+	amdgpu_dm_ism_commit_event(&acrtc->ism,
+				   DM_ISM_EVENT_END_CURSOR_UPDATE);
+	KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+			(int)DM_ISM_STATE_FULL_POWER_RUNNING);
+
+	amdgpu_dm_ism_fini(&acrtc->ism);
+}
+
+/**
+ * dm_test_ism_commit_event_invalid_event - invalid event leaves FSM unchanged
+ * @test: KUnit test context
+ *
+ * EXIT_IDLE_REQUESTED is not a valid event from FULL_POWER_RUNNING, so the
+ * FSM does not transition and no power-state dispatch occurs.
+ */
+static void dm_test_ism_commit_event_invalid_event(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc = alloc_test_acrtc(test, &adev);
+	struct dm_crtc_state *dm_state;
+	struct amdgpu_dm_ism_config config = { 0 };
+
+	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+
+	amdgpu_dm_ism_init(&acrtc->ism, &config);
+	acrtc->base.state = &dm_state->base;
+
+	guard(mutex)(&adev->dm.dc_lock);
+	amdgpu_dm_ism_commit_event(&acrtc->ism,
+				   DM_ISM_EVENT_EXIT_IDLE_REQUESTED);
+
+	KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+			(int)DM_ISM_STATE_FULL_POWER_RUNNING);
+
+	amdgpu_dm_ism_fini(&acrtc->ism);
+}
+
+/* ===== Tests for amdgpu_dm_ism_force_full_power ===== */
+
+/**
+ * dm_test_ism_force_full_power - force-full-power sends EXIT_IDLE to every CRTC
+ * @test: KUnit test context
+ *
+ * From the initial FULL_POWER_RUNNING state EXIT_IDLE_REQUESTED is a no-op, so
+ * the FSM remains in FULL_POWER_RUNNING and no DC work is scheduled.
+ */
+static void dm_test_ism_force_full_power(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc = alloc_test_acrtc(test, &adev);
+	struct dm_crtc_state *dm_state;
+	struct amdgpu_dm_ism_config config = { 0 };
+
+	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+
+	amdgpu_dm_ism_init(&acrtc->ism, &config);
+	acrtc->base.state = &dm_state->base;
+	register_test_acrtc(adev, acrtc);
+
+	guard(mutex)(&adev->dm.dc_lock);
+	amdgpu_dm_ism_force_full_power(&adev->dm);
+
+	KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+			(int)DM_ISM_STATE_FULL_POWER_RUNNING);
+
+	amdgpu_dm_ism_fini(&acrtc->ism);
+}
+
+/* ===== Tests for amdgpu_dm_ism_disable / amdgpu_dm_ism_enable ===== */
+
+/**
+ * dm_test_ism_disable_enable_cycle - disable then enable quiesces and re-arms work
+ * @test: KUnit test context
+ *
+ * Walks every CRTC's ISM, disabling its delayed work (synchronously) and then
+ * re-enabling it. With no work ever scheduled both calls complete without
+ * touching the FSM state. disable must be called without dc_lock held.
+ */
+static void dm_test_ism_disable_enable_cycle(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc = alloc_test_acrtc(test, &adev);
+	struct amdgpu_dm_ism_config config = { 0 };
+
+	amdgpu_dm_ism_init(&acrtc->ism, &config);
+	register_test_acrtc(adev, acrtc);
+
+	amdgpu_dm_ism_disable(&adev->dm);
+	amdgpu_dm_ism_enable(&adev->dm);
+
+	KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+			(int)DM_ISM_STATE_FULL_POWER_RUNNING);
+
+	amdgpu_dm_ism_fini(&acrtc->ism);
+}
+
+/* ===== Tests for dm_ism_dispatch_power_state (via commit_event) ===== */
+
+/*
+ * Build a config + history that makes dm_ism_get_idle_allow_delay() return a
+ * non-zero hysteresis delay. A non-zero delay keeps the FSM parked in
+ * HYSTERESIS_WAITING (the dispatcher returns DM_ISM_NUM_EVENTS instead of an
+ * immediate follow-up event), preventing the cascade into the DC-dependent
+ * OPTIMIZED_IDLE / *_SSO states.
+ */
+static void setup_idle_delay_history(struct amdgpu_dm_ism *ism,
+				     struct dc_stream_state *stream)
+{
+	uint64_t one_frame_ns;
+
+	stream->timing.v_total = 1125;
+	stream->timing.h_total = 2200;
+	stream->timing.pix_clk_100hz = 1485000;
+
+	one_frame_ns = div64_u64((uint64_t)1125 * 2200 * 10000000ULL, 1485000);
+
+	for (int i = 0; i < 8; i++) {
+		ism->records[i].duration_ns = one_frame_ns;
+		ism->records[i].timestamp_ns = 0;
+	}
+	ism->next_record_idx = 8;
+}
+
+/**
+ * dm_test_ism_dispatch_hysteresis_schedule_and_cancel - cover the HYSTERESIS_WAITING dispatch arms
+ * @test: KUnit test context
+ *
+ * ENTER_IDLE_REQUESTED moves FULL_POWER_RUNNING -> HYSTERESIS_WAITING. The
+ * current-state arm of dm_ism_dispatch_power_state() then records the idle
+ * timestamp, computes a (non-zero) idle-allow delay and schedules the delayed
+ * worker. A subsequent BEGIN_CURSOR_UPDATE (-> HYSTERESIS_BUSY) exercises the
+ * previous-state arm that cancels that pending worker. Neither arm reaches the
+ * DC-dependent idle-optimization commit path.
+ */
+static void dm_test_ism_dispatch_hysteresis_schedule_and_cancel(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc = alloc_test_acrtc(test, &adev);
+	struct dm_crtc_state *dm_state;
+	struct dc_stream_state *stream;
+	struct amdgpu_dm_ism_config config = {
+		.filter_num_frames = 5,
+		.filter_entry_count = 3,
+		.activation_num_delay_frames = 10,
+		.filter_history_size = 8,
+		.filter_old_history_threshold = 0,
+		.sso_num_frames = 0,
+	};
+
+	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+	stream = dm_kunit_alloc_stream(test, NULL);
+
+	amdgpu_dm_ism_init(&acrtc->ism, &config);
+	setup_idle_delay_history(&acrtc->ism, stream);
+	dm_state->stream = stream;
+	acrtc->base.state = &dm_state->base;
+
+	scoped_guard(mutex, &adev->dm.dc_lock) {
+		/* Enter HYSTERESIS_WAITING: schedules the idle-allow worker. */
+		amdgpu_dm_ism_commit_event(&acrtc->ism,
+					   DM_ISM_EVENT_ENTER_IDLE_REQUESTED);
+		KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+				(int)DM_ISM_STATE_HYSTERESIS_WAITING);
+
+		/* Cursor update cancels the pending worker (prev-state arm). */
+		amdgpu_dm_ism_commit_event(&acrtc->ism,
+					   DM_ISM_EVENT_BEGIN_CURSOR_UPDATE);
+		KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+				(int)DM_ISM_STATE_HYSTERESIS_BUSY);
+	}
+
+	amdgpu_dm_ism_fini(&acrtc->ism);
+}
+
+/**
+ * dm_test_ism_dispatch_optimized_idle_defers_sso - cover OPTIMIZED_IDLE dispatch without DC
+ * @test: KUnit test context
+ *
+ * With sso_num_frames < filter_num_frames the OPTIMIZED_IDLE current-state arm
+ * skips the idle-optimization commit and only schedules the SSO worker. Driving
+ * HYSTERESIS_WAITING -> OPTIMIZED_IDLE via TIMER_ELAPSED exercises that arm
+ * (get_sso_delay + the skip branch + mod_delayed_work) without any DC access.
+ */
+static void dm_test_ism_dispatch_optimized_idle_defers_sso(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc = alloc_test_acrtc(test, &adev);
+	struct dm_crtc_state *dm_state;
+	struct dc_stream_state *stream;
+	struct amdgpu_dm_ism_config config = {
+		.filter_num_frames = 5,
+		.filter_entry_count = 3,
+		.activation_num_delay_frames = 10,
+		.filter_history_size = 8,
+		.filter_old_history_threshold = 0,
+		.sso_num_frames = 2,
+	};
+
+	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+	stream = dm_kunit_alloc_stream(test, NULL);
+
+	amdgpu_dm_ism_init(&acrtc->ism, &config);
+	setup_idle_delay_history(&acrtc->ism, stream);
+	dm_state->stream = stream;
+	acrtc->base.state = &dm_state->base;
+
+	scoped_guard(mutex, &adev->dm.dc_lock) {
+		amdgpu_dm_ism_commit_event(&acrtc->ism,
+					   DM_ISM_EVENT_ENTER_IDLE_REQUESTED);
+		KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+				(int)DM_ISM_STATE_HYSTERESIS_WAITING);
+
+		/*
+		 * Timer fires: HYSTERESIS_WAITING -> OPTIMIZED_IDLE. sso_delay
+		 * is non-zero and sso_num_frames < filter_num_frames, so the
+		 * commit is deferred to the SSO worker and the FSM parks here.
+		 */
+		amdgpu_dm_ism_commit_event(&acrtc->ism,
+					   DM_ISM_EVENT_TIMER_ELAPSED);
+		KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+				(int)DM_ISM_STATE_OPTIMIZED_IDLE);
+
+		/* Cancel the scheduled SSO worker while still holding dc_lock. */
+		cancel_delayed_work(&acrtc->ism.sso_delayed_work);
+	}
+
+	amdgpu_dm_ism_fini(&acrtc->ism);
+}
+
 static struct kunit_case dm_ism_test_cases[] = {
 	/* dm_ism_next_state — FULL_POWER_RUNNING */
 	KUNIT_CASE(dm_test_ism_next_state_running_enter_idle),
@@ -923,6 +1252,17 @@ static struct kunit_case dm_ism_test_cases[] = {
 	KUNIT_CASE(dm_test_dispatch_next_event_opt_idle_with_sso_delay),
 	KUNIT_CASE(dm_test_dispatch_next_event_timer_aborted),
 	KUNIT_CASE(dm_test_dispatch_next_event_no_action_state),
+	/* amdgpu_dm_ism_commit_event */
+	KUNIT_CASE(dm_test_ism_commit_event_no_state),
+	KUNIT_CASE(dm_test_ism_commit_event_cursor_transition),
+	KUNIT_CASE(dm_test_ism_commit_event_invalid_event),
+	/* amdgpu_dm_ism_force_full_power */
+	KUNIT_CASE(dm_test_ism_force_full_power),
+	/* amdgpu_dm_ism_disable / amdgpu_dm_ism_enable */
+	KUNIT_CASE(dm_test_ism_disable_enable_cycle),
+	/* dm_ism_dispatch_power_state (via commit_event) */
+	KUNIT_CASE(dm_test_ism_dispatch_hysteresis_schedule_and_cancel),
+	KUNIT_CASE(dm_test_ism_dispatch_optimized_idle_defers_sso),
 	{}
 };
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_kunit_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_kunit_helpers.c
new file mode 100644
index 0000000000000..58615cdbe854f
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_kunit_helpers.c
@@ -0,0 +1,142 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * KUnit test helpers for amdgpu_dm tests.
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#include <kunit/test.h>
+#include <linux/module.h>
+#include <drm/drm_kunit_helpers.h>
+
+#include "dc.h"
+#include "core_types.h"
+#include "amdgpu.h"
+#include "amdgpu_mode.h"
+#include "amdgpu_dm.h"
+#include "amdgpu_dm_kunit_test_helpers.h"
+
+struct amdgpu_device *dm_kunit_alloc_adev(struct kunit *test)
+{
+	struct drm_device *drm;
+	struct device *dev;
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						   sizeof(struct amdgpu_device),
+						   offsetof(struct amdgpu_device, ddev),
+						   DRIVER_MODESET | DRIVER_ATOMIC);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
+
+	return drm_to_adev(drm);
+}
+EXPORT_SYMBOL(dm_kunit_alloc_adev);
+
+struct dc_link *dm_kunit_alloc_link(struct kunit *test)
+{
+	struct dc_link *link;
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	return link;
+}
+EXPORT_SYMBOL(dm_kunit_alloc_link);
+
+struct dc_link *dm_kunit_alloc_link_with_ctx(struct kunit *test)
+{
+	struct dc_link *link;
+	struct dc_context *ctx;
+	struct dc *dc;
+
+	link = dm_kunit_alloc_link(test);
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+
+	link->ctx = ctx;
+	ctx->dc = dc;
+	dc->ctx = ctx;
+
+	return link;
+}
+EXPORT_SYMBOL(dm_kunit_alloc_link_with_ctx);
+
+struct amdgpu_display_manager *dm_kunit_alloc_dm(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm;
+	struct dc *dc;
+	struct dc_state *state;
+
+	dm = kunit_kzalloc(test, sizeof(*dm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm);
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+
+	dm->dc = dc;
+	dc->current_state = state;
+
+	return dm;
+}
+EXPORT_SYMBOL(dm_kunit_alloc_dm);
+
+struct dc_stream_state *dm_kunit_alloc_stream(struct kunit *test,
+					      struct dc_link *link)
+{
+	struct dc_stream_state *stream;
+
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+
+	stream->link = link;
+	kref_init(&stream->refcount);
+
+	return stream;
+}
+EXPORT_SYMBOL(dm_kunit_alloc_stream);
+
+void dm_kunit_add_stream_to_state(struct kunit *test, struct dc_state *state,
+				  unsigned int index, struct dc_link *link)
+{
+	struct dc_stream_state *stream;
+
+	KUNIT_ASSERT_LT(test, index, (unsigned int)MAX_PIPES);
+
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+
+	stream->link = link;
+	state->streams[index] = stream;
+	if (state->stream_count <= index)
+		state->stream_count = index + 1;
+}
+EXPORT_SYMBOL(dm_kunit_add_stream_to_state);
+
+struct amdgpu_dm_connector *dm_kunit_alloc_connector(struct kunit *test,
+						     struct amdgpu_device *adev,
+						     struct dc_link *link)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+
+	if (adev)
+		aconnector->base.dev = &adev->ddev;
+	aconnector->dc_link = link;
+
+	return aconnector;
+}
+EXPORT_SYMBOL(dm_kunit_alloc_connector);
+
+MODULE_LICENSE("Dual MIT/GPL");
+MODULE_DESCRIPTION("KUnit test helpers for amdgpu_dm tests");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_kunit_test_helpers.h b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_kunit_test_helpers.h
new file mode 100644
index 0000000000000..0f1c48fa21281
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_kunit_test_helpers.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/*
+ * KUnit test helpers for amdgpu_dm tests.
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#ifndef AMDGPU_DM_KUNIT_TEST_HELPERS_H
+#define AMDGPU_DM_KUNIT_TEST_HELPERS_H
+
+#include <kunit/test.h>
+
+struct amdgpu_device;
+struct amdgpu_display_manager;
+struct amdgpu_dm_connector;
+struct dc_link;
+struct dc_state;
+struct dc_stream_state;
+
+struct amdgpu_device *dm_kunit_alloc_adev(struct kunit *test);
+struct dc_link *dm_kunit_alloc_link(struct kunit *test);
+struct dc_link *dm_kunit_alloc_link_with_ctx(struct kunit *test);
+struct amdgpu_display_manager *dm_kunit_alloc_dm(struct kunit *test);
+struct dc_stream_state *dm_kunit_alloc_stream(struct kunit *test,
+					      struct dc_link *link);
+void dm_kunit_add_stream_to_state(struct kunit *test, struct dc_state *state,
+				  unsigned int index, struct dc_link *link);
+struct amdgpu_dm_connector *dm_kunit_alloc_connector(struct kunit *test,
+						     struct amdgpu_device *adev,
+						     struct dc_link *link);
+
+#endif /* AMDGPU_DM_KUNIT_TEST_HELPERS_H */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_mst_types_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_mst_types_test.c
new file mode 100644
index 0000000000000..caedad4197bd9
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_mst_types_test.c
@@ -0,0 +1,1429 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * KUnit tests for amdgpu_dm_mst_types.c
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#include <kunit/test.h>
+
+#include <drm/drm_drv.h>
+#include <drm/drm_fixed.h>
+#include <drm/drm_kunit_helpers.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_mode_config.h>
+#include <drm/display/drm_dp.h>
+#include <drm/display/drm_dp_helper.h>
+#include <drm/display/drm_dp_mst_helper.h>
+
+#include "dc.h"
+#include "dpcd_defs.h"
+#include "dmub_cmd.h"
+#include "amdgpu.h"
+#include "amdgpu_mode.h"
+#include "amdgpu_dm.h"
+#include "amdgpu_dm_mst_types.h"
+#include "amdgpu_dm_kunit_test_helpers.h"
+#include "inc/link_service.h"
+
+/*
+ * Minimal mock DPCD backing store and AUX transfer callback used to exercise
+ * the DPCD read paths without real hardware.
+ */
+static u8 dm_mst_test_dpcd[0x10];
+static u8 dm_mst_test_desc_dpcd[0x10];
+static struct aux_payload dm_mst_test_last_payload;
+static int dm_mst_test_aux_transfer_raw_result;
+static u8 dm_mst_test_aux_transfer_raw_reply;
+static enum aux_return_code_type dm_mst_test_aux_transfer_raw_operation_result;
+static ssize_t dm_mst_test_aux_transfer_override;
+
+static int dm_mst_test_aux_transfer_raw(struct ddc_service *ddc,
+						struct aux_payload *payload,
+						enum aux_return_code_type *operation_result)
+{
+	size_t i;
+
+	dm_mst_test_last_payload = *payload;
+	*operation_result = dm_mst_test_aux_transfer_raw_operation_result;
+	payload->reply[0] = dm_mst_test_aux_transfer_raw_reply;
+
+	if (dm_mst_test_aux_transfer_raw_result)
+		return dm_mst_test_aux_transfer_raw_result;
+
+	if (payload->write)
+		return 0;
+
+	for (i = 0; i < payload->length; i++)
+		payload->data[i] = dm_mst_test_dpcd[(payload->address + i) & 0xf];
+
+	return payload->length;
+}
+
+static void dm_mst_test_setup_dm_aux(struct amdgpu_dm_dp_aux *dm_aux,
+					    struct ddc_service *ddc,
+					    struct dc_link *link,
+					    struct dc *dc,
+					    struct link_service *link_srv,
+					    struct dc_context *ctx,
+					    struct amdgpu_device *adev)
+{
+	memset(&dm_mst_test_last_payload, 0, sizeof(dm_mst_test_last_payload));
+	dm_mst_test_aux_transfer_raw_result = 0;
+	dm_mst_test_aux_transfer_raw_reply = 0;
+	dm_mst_test_aux_transfer_raw_operation_result = AUX_RET_SUCCESS;
+	link_srv->aux_transfer_raw = dm_mst_test_aux_transfer_raw;
+	dc->link_srv = link_srv;
+	link->dc = dc;
+	ctx->driver_context = adev;
+	ddc->link = link;
+	ddc->ctx = ctx;
+	dm_aux->ddc_service = ddc;
+	dm_aux->aux.name = "dm_mst_test_dm_aux";
+	dm_aux->aux.transfer = dm_dp_aux_transfer;
+	drm_dp_aux_init(&dm_aux->aux);
+	drm_dp_dpcd_set_probe(&dm_aux->aux, false);
+}
+
+static const struct dc_link_status *dm_mst_test_get_status(const struct dc_link *link)
+{
+	return &link->link_status;
+}
+
+static ssize_t dm_mst_test_aux_transfer(struct drm_dp_aux *aux,
+					struct drm_dp_aux_msg *msg)
+{
+	size_t i;
+	ssize_t ret;
+
+	ret = dm_mst_test_aux_transfer_override;
+	if (ret)
+		return ret;
+
+	switch (msg->request & ~DP_AUX_I2C_MOT) {
+	case DP_AUX_NATIVE_READ:
+		for (i = 0; i < msg->size; i++)
+			((u8 *)msg->buffer)[i] =
+				dm_mst_test_dpcd[(msg->address + i) & 0xf];
+		msg->reply = DP_AUX_NATIVE_REPLY_ACK;
+		return msg->size;
+	case DP_AUX_NATIVE_WRITE:
+		msg->reply = DP_AUX_NATIVE_REPLY_ACK;
+		return msg->size;
+	default:
+		return -EINVAL;
+	}
+}
+
+static struct amdgpu_dm_connector *dm_mst_test_alloc_sideband_connector(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct link_service *link_srv;
+	struct dc_link *link;
+	struct dc *dc;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+
+	mutex_init(&aconnector->handle_mst_msg_ready);
+	link_srv->get_status = dm_mst_test_get_status;
+	dc->link_srv = link_srv;
+	link->dc = dc;
+	link->dpcd_caps.dpcd_rev.raw = DPCD_REV_14;
+	link->link_status.dpcd_caps = &link->dpcd_caps;
+	aconnector->dc_link = link;
+	aconnector->dm_dp_aux.aux.name = "dm_mst_test_sideband_aux";
+	aconnector->dm_dp_aux.aux.transfer = dm_mst_test_aux_transfer;
+	drm_dp_aux_init(&aconnector->dm_dp_aux.aux);
+	drm_dp_dpcd_set_probe(&aconnector->dm_dp_aux.aux, false);
+
+	memset(dm_mst_test_dpcd, 0, sizeof(dm_mst_test_dpcd));
+	dm_mst_test_aux_transfer_override = 0;
+
+	return aconnector;
+}
+
+static uint32_t dm_mst_test_dp_link_bandwidth_kbps(
+	const struct dc_link *link,
+	const struct dc_link_settings *link_settings)
+{
+	return 4320000;
+}
+
+static const struct dc_link_settings *dm_mst_test_dp_get_verified_link_cap(
+	const struct dc_link *link)
+{
+	return &link->verified_link_cap;
+}
+
+static ssize_t dm_mst_test_desc_aux_transfer(struct drm_dp_aux *aux,
+					     struct drm_dp_aux_msg *msg)
+{
+	size_t i;
+
+	if ((msg->request & ~DP_AUX_I2C_MOT) != DP_AUX_NATIVE_READ)
+		return -EINVAL;
+
+	for (i = 0; i < msg->size; i++)
+		((u8 *)msg->buffer)[i] = dm_mst_test_desc_dpcd[msg->address + i - DP_BRANCH_OUI];
+
+	msg->reply = DP_AUX_NATIVE_REPLY_ACK;
+	return msg->size;
+}
+
+/* Tests for needs_dsc_aux_workaround */
+
+/**
+ * dm_mst_test_needs_dsc_aux_workaround_match - Test workaround triggers for matching device
+ * @test: KUnit test context
+ *
+ * Verify that needs_dsc_aux_workaround() returns true when the link has
+ * the specific branch device ID, DPCD rev 1.4, and sink count >= 2.
+ */
+static void dm_mst_test_needs_dsc_aux_workaround_match(struct kunit *test)
+{
+	struct dc_link *link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link->dpcd_caps.branch_dev_id = DP_BRANCH_DEVICE_ID_90CC24;
+	link->dpcd_caps.dpcd_rev.raw = DPCD_REV_14;
+	link->dpcd_caps.sink_count.bits.SINK_COUNT = 2;
+
+	KUNIT_EXPECT_TRUE(test, needs_dsc_aux_workaround(link));
+}
+
+/**
+ * dm_mst_test_needs_dsc_aux_workaround_rev12 - Test workaround triggers for DPCD rev 1.2
+ * @test: KUnit test context
+ *
+ * Verify that needs_dsc_aux_workaround() returns true when the link has
+ * the specific branch device ID, DPCD rev 1.2, and sink count >= 2.
+ */
+static void dm_mst_test_needs_dsc_aux_workaround_rev12(struct kunit *test)
+{
+	struct dc_link *link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link->dpcd_caps.branch_dev_id = DP_BRANCH_DEVICE_ID_90CC24;
+	link->dpcd_caps.dpcd_rev.raw = DPCD_REV_12;
+	link->dpcd_caps.sink_count.bits.SINK_COUNT = 3;
+
+	KUNIT_EXPECT_TRUE(test, needs_dsc_aux_workaround(link));
+}
+
+/**
+ * dm_mst_test_needs_dsc_aux_workaround_wrong_dev_id - Test workaround skipped for wrong device
+ * @test: KUnit test context
+ *
+ * Verify that needs_dsc_aux_workaround() returns false when the branch
+ * device ID does not match DP_BRANCH_DEVICE_ID_90CC24.
+ */
+static void dm_mst_test_needs_dsc_aux_workaround_wrong_dev_id(struct kunit *test)
+{
+	struct dc_link *link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link->dpcd_caps.branch_dev_id = 0x123456;
+	link->dpcd_caps.dpcd_rev.raw = DPCD_REV_14;
+	link->dpcd_caps.sink_count.bits.SINK_COUNT = 2;
+
+	KUNIT_EXPECT_FALSE(test, needs_dsc_aux_workaround(link));
+}
+
+/**
+ * dm_mst_test_needs_dsc_aux_workaround_wrong_rev - Test workaround skipped for unsupported rev
+ * @test: KUnit test context
+ *
+ * Verify that needs_dsc_aux_workaround() returns false when the DPCD
+ * revision is neither 1.2 nor 1.4.
+ */
+static void dm_mst_test_needs_dsc_aux_workaround_wrong_rev(struct kunit *test)
+{
+	struct dc_link *link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link->dpcd_caps.branch_dev_id = DP_BRANCH_DEVICE_ID_90CC24;
+	link->dpcd_caps.dpcd_rev.raw = 0x11; /* DPCD 1.1 */
+	link->dpcd_caps.sink_count.bits.SINK_COUNT = 2;
+
+	KUNIT_EXPECT_FALSE(test, needs_dsc_aux_workaround(link));
+}
+
+/**
+ * dm_mst_test_needs_dsc_aux_workaround_low_sink_count - Test workaround skipped for single sink
+ * @test: KUnit test context
+ *
+ * Verify that needs_dsc_aux_workaround() returns false when the sink
+ * count is less than 2, even if device ID and DPCD rev match.
+ */
+static void dm_mst_test_needs_dsc_aux_workaround_low_sink_count(struct kunit *test)
+{
+	struct dc_link *link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link->dpcd_caps.branch_dev_id = DP_BRANCH_DEVICE_ID_90CC24;
+	link->dpcd_caps.dpcd_rev.raw = DPCD_REV_14;
+	link->dpcd_caps.sink_count.bits.SINK_COUNT = 1;
+
+	KUNIT_EXPECT_FALSE(test, needs_dsc_aux_workaround(link));
+}
+
+/**
+ * dm_mst_test_needs_dsc_aux_workaround_zero_sink_count - Test workaround skipped for zero sinks
+ * @test: KUnit test context
+ *
+ * Verify that needs_dsc_aux_workaround() returns false when the sink
+ * count is zero, even if device ID and DPCD rev match.
+ */
+static void dm_mst_test_needs_dsc_aux_workaround_zero_sink_count(struct kunit *test)
+{
+	struct dc_link *link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+
+	link->dpcd_caps.branch_dev_id = DP_BRANCH_DEVICE_ID_90CC24;
+	link->dpcd_caps.dpcd_rev.raw = DPCD_REV_14;
+	link->dpcd_caps.sink_count.bits.SINK_COUNT = 0;
+
+	KUNIT_EXPECT_FALSE(test, needs_dsc_aux_workaround(link));
+}
+
+/* Tests for dm_mst_get_pbn_divider */
+
+/**
+ * dm_mst_test_pbn_divider_null_link - Test pbn_divider with NULL link
+ * @test: KUnit test context
+ *
+ * Verify that dm_mst_get_pbn_divider() returns 0 when passed a NULL
+ * link pointer without crashing.
+ */
+static void dm_mst_test_pbn_divider_null_link(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, dm_mst_get_pbn_divider(NULL), 0U);
+}
+
+/**
+ * dm_mst_test_pbn_divider_uses_link_bandwidth - Test pbn_divider with link cap
+ * @test: KUnit test context
+ *
+ * Verify that dm_mst_get_pbn_divider() uses the DC link service to derive the
+ * fixed-point PBN divider when a link is present.
+ */
+static void dm_mst_test_pbn_divider_uses_link_bandwidth(struct kunit *test)
+{
+	struct link_service *link_srv;
+	struct dc_link *link;
+	struct dc *dc;
+
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+
+	link_srv->dp_get_verified_link_cap = dm_mst_test_dp_get_verified_link_cap;
+	link_srv->dp_link_bandwidth_kbps = dm_mst_test_dp_link_bandwidth_kbps;
+	dc->link_srv = link_srv;
+	link->dc = dc;
+
+	KUNIT_EXPECT_EQ(test, dm_mst_get_pbn_divider(link),
+			 (uint32_t)(dfixed_const(1000) / 100));
+}
+
+/* Tests for amdgpu_dm_mst_reset_mst_connector_setting */
+
+/**
+ * dm_mst_test_reset_connector_setting - Test MST connector setting reset
+ * @test: KUnit test context
+ *
+ * Verify that amdgpu_dm_mst_reset_mst_connector_setting() clears the cached
+ * EDID, DSC AUX, passthrough AUX, local bandwidth, and VC PBN state.
+ */
+static void dm_mst_test_reset_connector_setting(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_mst_port *port;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	port = kunit_kzalloc(test, sizeof(*port), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	KUNIT_ASSERT_NOT_NULL(test, port);
+
+	aconnector->drm_edid = (const struct drm_edid *)test;
+	aconnector->dsc_aux = (struct drm_dp_aux *)test;
+	aconnector->mst_output_port = port;
+	aconnector->mst_output_port->passthrough_aux = (struct drm_dp_aux *)test;
+	aconnector->mst_local_bw = 12345;
+	aconnector->vc_full_pbn = 678;
+
+	amdgpu_dm_mst_reset_mst_connector_setting(aconnector);
+
+	KUNIT_EXPECT_TRUE(test, aconnector->drm_edid == NULL);
+	KUNIT_EXPECT_TRUE(test, aconnector->dsc_aux == NULL);
+	KUNIT_EXPECT_TRUE(test, aconnector->mst_output_port->passthrough_aux == NULL);
+	KUNIT_EXPECT_EQ(test, aconnector->mst_local_bw, 0U);
+	KUNIT_EXPECT_EQ(test, aconnector->vc_full_pbn, 0U);
+}
+
+/* Tests for retrieve_downstream_port_device */
+
+/**
+ * dm_mst_test_retrieve_downstream_no_aux - Test retrieval bails out without AUX
+ * @test: KUnit test context
+ *
+ * Verify that retrieve_downstream_port_device() returns false when the
+ * connector has no DSC AUX channel and therefore cannot read DPCD.
+ */
+static void dm_mst_test_retrieve_downstream_no_aux(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+
+	aconnector->dsc_aux = NULL;
+
+	KUNIT_EXPECT_FALSE(test, retrieve_downstream_port_device(aconnector));
+}
+
+/**
+ * dm_mst_test_retrieve_downstream_present - Test retrieval parses DPCD 0x05
+ * @test: KUnit test context
+ *
+ * Verify that retrieve_downstream_port_device() reads DP_DOWNSTREAMPORT_PRESENT
+ * over a mock AUX channel and caches the parsed downstream port fields.
+ */
+static void dm_mst_test_retrieve_downstream_present(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_aux *aux;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	aux = kunit_kzalloc(test, sizeof(*aux), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	KUNIT_ASSERT_NOT_NULL(test, aux);
+
+	memset(dm_mst_test_dpcd, 0, sizeof(dm_mst_test_dpcd));
+	/* PORT_PRESENT = 1, PORT_TYPE = 2 (0b101) */
+	dm_mst_test_dpcd[DP_DOWNSTREAMPORT_PRESENT] = 0x05;
+	dm_mst_test_aux_transfer_override = 0;
+
+	aux->name = "dm_mst_test_aux";
+	aux->transfer = dm_mst_test_aux_transfer;
+	drm_dp_aux_init(aux);
+	drm_dp_dpcd_set_probe(aux, false);
+	aconnector->dsc_aux = aux;
+
+	KUNIT_EXPECT_TRUE(test, retrieve_downstream_port_device(aconnector));
+	KUNIT_EXPECT_EQ(test,
+			(int)aconnector->mst_downstream_port_present.fields.PORT_PRESENT, 1);
+	KUNIT_EXPECT_EQ(test,
+			(int)aconnector->mst_downstream_port_present.fields.PORT_TYPE, 2);
+}
+
+/**
+ * dm_mst_test_retrieve_downstream_aux_error - Test downstream read failure
+ * @test: KUnit test context
+ *
+ * Verify that retrieve_downstream_port_device() returns false when the AUX
+ * DPCD read fails.
+ */
+static void dm_mst_test_retrieve_downstream_aux_error(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_aux *aux;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	aux = kunit_kzalloc(test, sizeof(*aux), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	KUNIT_ASSERT_NOT_NULL(test, aux);
+
+	dm_mst_test_aux_transfer_override = -EIO;
+	aux->name = "dm_mst_test_aux";
+	aux->transfer = dm_mst_test_aux_transfer;
+	drm_dp_aux_init(aux);
+	drm_dp_dpcd_set_probe(aux, false);
+	aconnector->dsc_aux = aux;
+
+	KUNIT_EXPECT_FALSE(test, retrieve_downstream_port_device(aconnector));
+
+	dm_mst_test_aux_transfer_override = 0;
+}
+
+/* Tests for retrieve_branch_specific_data */
+
+/**
+ * dm_mst_test_retrieve_branch_no_parent - Test branch lookup needs a parent port
+ * @test: KUnit test context
+ *
+ * Verify that retrieve_branch_specific_data() returns false when the MST
+ * output port has no parent branch device to query.
+ */
+static void dm_mst_test_retrieve_branch_no_parent(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_mst_port *port;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	port = kunit_kzalloc(test, sizeof(*port), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	KUNIT_ASSERT_NOT_NULL(test, port);
+
+	port->parent = NULL;
+	aconnector->mst_output_port = port;
+
+	KUNIT_EXPECT_FALSE(test, retrieve_branch_specific_data(aconnector));
+}
+
+/**
+ * dm_mst_test_retrieve_branch_reads_oui - Test branch OUI parsing
+ * @test: KUnit test context
+ *
+ * Verify that retrieve_branch_specific_data() reads the immediate upstream
+ * branch descriptor and caches its IEEE OUI value on the connector.
+ */
+static void dm_mst_test_retrieve_branch_reads_oui(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_mst_topology_mgr *mgr;
+	struct drm_dp_mst_branch *branch;
+	struct drm_dp_mst_port *port;
+	struct drm_dp_aux *aux;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	mgr = kunit_kzalloc(test, sizeof(*mgr), GFP_KERNEL);
+	branch = kunit_kzalloc(test, sizeof(*branch), GFP_KERNEL);
+	port = kunit_kzalloc(test, sizeof(*port), GFP_KERNEL);
+	aux = kunit_kzalloc(test, sizeof(*aux), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	KUNIT_ASSERT_NOT_NULL(test, mgr);
+	KUNIT_ASSERT_NOT_NULL(test, branch);
+	KUNIT_ASSERT_NOT_NULL(test, port);
+	KUNIT_ASSERT_NOT_NULL(test, aux);
+
+	memset(dm_mst_test_desc_dpcd, 0, sizeof(dm_mst_test_desc_dpcd));
+	dm_mst_test_desc_dpcd[0] = 0x12;
+	dm_mst_test_desc_dpcd[1] = 0x34;
+	dm_mst_test_desc_dpcd[2] = 0x56;
+
+	aux->name = "dm_mst_test_desc_aux";
+	aux->transfer = dm_mst_test_desc_aux_transfer;
+	drm_dp_aux_init(aux);
+	drm_dp_dpcd_set_probe(aux, false);
+	mgr->aux = aux;
+	port->parent = branch;
+	port->mgr = mgr;
+	port->aux.drm_dev = NULL;
+	aconnector->mst_output_port = port;
+
+	KUNIT_EXPECT_TRUE(test, retrieve_branch_specific_data(aconnector));
+	KUNIT_EXPECT_EQ(test, aconnector->branch_ieee_oui, 0x123456U);
+}
+
+/**
+ * dm_mst_test_aux_result_success - AUX_RET_SUCCESS preserves the input result.
+ * @test: KUnit test context.
+ *
+ * On success the original (negative) transfer result must be returned unchanged.
+ */
+static void dm_mst_test_aux_result_success(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, dm_dp_aux_transfer_result(-5, AUX_RET_SUCCESS), (ssize_t)-5);
+	KUNIT_EXPECT_EQ(test, dm_dp_aux_transfer_result(3, AUX_RET_SUCCESS), (ssize_t)3);
+}
+
+/**
+ * dm_mst_test_aux_result_eio - HPD/unknown/protocol errors map to -EIO.
+ * @test: KUnit test context.
+ *
+ * AUX_RET_ERROR_HPD_DISCON, AUX_RET_ERROR_UNKNOWN,
+ * AUX_RET_ERROR_INVALID_OPERATION and AUX_RET_ERROR_PROTOCOL_ERROR all map to -EIO.
+ */
+static void dm_mst_test_aux_result_eio(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, dm_dp_aux_transfer_result(-1, AUX_RET_ERROR_HPD_DISCON),
+			(ssize_t)-EIO);
+	KUNIT_EXPECT_EQ(test, dm_dp_aux_transfer_result(-1, AUX_RET_ERROR_UNKNOWN),
+			(ssize_t)-EIO);
+	KUNIT_EXPECT_EQ(test, dm_dp_aux_transfer_result(-1, AUX_RET_ERROR_INVALID_OPERATION),
+			(ssize_t)-EIO);
+	KUNIT_EXPECT_EQ(test, dm_dp_aux_transfer_result(-1, AUX_RET_ERROR_PROTOCOL_ERROR),
+			(ssize_t)-EIO);
+}
+
+/**
+ * dm_mst_test_aux_result_ebusy - invalid reply / engine acquire map to -EBUSY.
+ * @test: KUnit test context.
+ *
+ * AUX_RET_ERROR_INVALID_REPLY and AUX_RET_ERROR_ENGINE_ACQUIRE map to -EBUSY.
+ */
+static void dm_mst_test_aux_result_ebusy(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, dm_dp_aux_transfer_result(-1, AUX_RET_ERROR_INVALID_REPLY),
+			(ssize_t)-EBUSY);
+	KUNIT_EXPECT_EQ(test, dm_dp_aux_transfer_result(-1, AUX_RET_ERROR_ENGINE_ACQUIRE),
+			(ssize_t)-EBUSY);
+}
+
+/**
+ * dm_mst_test_aux_result_timeout - AUX_RET_ERROR_TIMEOUT maps to -ETIMEDOUT.
+ * @test: KUnit test context.
+ */
+static void dm_mst_test_aux_result_timeout(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, dm_dp_aux_transfer_result(-1, AUX_RET_ERROR_TIMEOUT),
+			(ssize_t)-ETIMEDOUT);
+}
+
+/**
+ * dm_mst_test_aux_transfer_native_read - native AUX read through DM callback.
+ * @test: KUnit test context.
+ *
+ * The DM AUX transfer callback should build a read payload, call the DC link
+ * service, and return the number of bytes provided by the fake backend.
+ */
+static void dm_mst_test_aux_transfer_native_read(struct kunit *test)
+{
+	struct amdgpu_dm_dp_aux *dm_aux;
+	struct amdgpu_device *adev;
+	struct ddc_service *ddc;
+	struct dc_link *link;
+	struct dc *dc;
+	struct link_service *link_srv;
+	struct dc_context *ctx;
+	u8 buffer[3] = { 0 };
+	ssize_t ret;
+
+	dm_aux = kunit_kzalloc(test, sizeof(*dm_aux), GFP_KERNEL);
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	ddc = kunit_kzalloc(test, sizeof(*ddc), GFP_KERNEL);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_aux);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ddc);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	memset(dm_mst_test_dpcd, 0, sizeof(dm_mst_test_dpcd));
+	dm_mst_test_dpcd[4] = 0xaa;
+	dm_mst_test_dpcd[5] = 0xbb;
+	dm_mst_test_dpcd[6] = 0xcc;
+	dm_mst_test_setup_dm_aux(dm_aux, ddc, link, dc, link_srv, ctx, adev);
+
+	ret = drm_dp_dpcd_read(&dm_aux->aux, 4, buffer, sizeof(buffer));
+
+	KUNIT_EXPECT_EQ(test, ret, (ssize_t)sizeof(buffer));
+	KUNIT_EXPECT_EQ(test, buffer[0], (u8)0xaa);
+	KUNIT_EXPECT_EQ(test, buffer[1], (u8)0xbb);
+	KUNIT_EXPECT_EQ(test, buffer[2], (u8)0xcc);
+	KUNIT_EXPECT_FALSE(test, dm_mst_test_last_payload.write);
+	KUNIT_EXPECT_FALSE(test, dm_mst_test_last_payload.i2c_over_aux);
+	KUNIT_EXPECT_EQ(test, dm_mst_test_last_payload.address, 4U);
+}
+
+/**
+ * dm_mst_test_aux_transfer_native_write - native AUX write through DM callback.
+ * @test: KUnit test context.
+ *
+ * A successful write with an ACK reply should report the requested write size
+ * and pass a write payload into the fake DC link service.
+ */
+static void dm_mst_test_aux_transfer_native_write(struct kunit *test)
+{
+	struct amdgpu_dm_dp_aux *dm_aux;
+	struct amdgpu_device *adev;
+	struct ddc_service *ddc;
+	struct dc_link *link;
+	struct dc *dc;
+	struct link_service *link_srv;
+	struct dc_context *ctx;
+	u8 buffer[2] = { 0x11, 0x22 };
+	ssize_t ret;
+
+	dm_aux = kunit_kzalloc(test, sizeof(*dm_aux), GFP_KERNEL);
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	ddc = kunit_kzalloc(test, sizeof(*ddc), GFP_KERNEL);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_aux);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ddc);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dm_mst_test_setup_dm_aux(dm_aux, ddc, link, dc, link_srv, ctx, adev);
+
+	ret = drm_dp_dpcd_write(&dm_aux->aux, 7, buffer, sizeof(buffer));
+
+	KUNIT_EXPECT_EQ(test, ret, (ssize_t)sizeof(buffer));
+	KUNIT_EXPECT_TRUE(test, dm_mst_test_last_payload.write);
+	KUNIT_EXPECT_FALSE(test, dm_mst_test_last_payload.i2c_over_aux);
+	KUNIT_EXPECT_EQ(test, dm_mst_test_last_payload.address, 7U);
+	KUNIT_EXPECT_EQ(test, dm_mst_test_last_payload.length,
+			(u32)sizeof(buffer));
+}
+
+/**
+ * dm_mst_test_aux_transfer_partial_write - partial write reports byte count.
+ * @test: KUnit test context.
+ *
+ * A positive write result from the DC link service should be interpreted as a
+ * partial write and replaced with the first payload byte.
+ */
+static void dm_mst_test_aux_transfer_partial_write(struct kunit *test)
+{
+	struct amdgpu_dm_dp_aux *dm_aux;
+	struct amdgpu_device *adev;
+	struct ddc_service *ddc;
+	struct dc_link *link;
+	struct dc *dc;
+	struct link_service *link_srv;
+	struct dc_context *ctx;
+	u8 buffer[2] = { 1, 0xaa };
+	struct drm_dp_aux_msg msg = {
+		.address = 7,
+		.request = DP_AUX_NATIVE_WRITE,
+		.buffer = buffer,
+		.size = sizeof(buffer),
+	};
+	ssize_t ret;
+
+	dm_aux = kunit_kzalloc(test, sizeof(*dm_aux), GFP_KERNEL);
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	ddc = kunit_kzalloc(test, sizeof(*ddc), GFP_KERNEL);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_aux);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ddc);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dm_mst_test_setup_dm_aux(dm_aux, ddc, link, dc, link_srv, ctx, adev);
+	dm_mst_test_aux_transfer_raw_result = 1;
+
+	ret = dm_dp_aux_transfer(&dm_aux->aux, &msg);
+
+	KUNIT_EXPECT_EQ(test, ret, (ssize_t)buffer[0]);
+	KUNIT_EXPECT_TRUE(test, dm_mst_test_last_payload.write);
+	KUNIT_EXPECT_EQ(test, dm_mst_test_last_payload.address, 7U);
+}
+
+/**
+ * dm_mst_test_aux_transfer_error_result - transfer errors are remapped.
+ * @test: KUnit test context.
+ *
+ * A negative DC link service result should be converted through
+ * dm_dp_aux_transfer_result() using the returned AUX operation result.
+ */
+static void dm_mst_test_aux_transfer_error_result(struct kunit *test)
+{
+	struct amdgpu_dm_dp_aux *dm_aux;
+	struct amdgpu_device *adev;
+	struct ddc_service *ddc;
+	struct dc_link *link;
+	struct dc *dc;
+	struct link_service *link_srv;
+	struct dc_context *ctx;
+	u8 buffer[2] = { 0 };
+	ssize_t ret;
+
+	dm_aux = kunit_kzalloc(test, sizeof(*dm_aux), GFP_KERNEL);
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	ddc = kunit_kzalloc(test, sizeof(*ddc), GFP_KERNEL);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_aux);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ddc);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dm_mst_test_setup_dm_aux(dm_aux, ddc, link, dc, link_srv, ctx, adev);
+	dm_mst_test_aux_transfer_raw_result = -EIO;
+	dm_mst_test_aux_transfer_raw_operation_result = AUX_RET_ERROR_TIMEOUT;
+
+	ret = drm_dp_dpcd_read(&dm_aux->aux, 4, buffer, sizeof(buffer));
+
+	KUNIT_EXPECT_EQ(test, ret, (ssize_t)-ETIMEDOUT);
+	KUNIT_EXPECT_FALSE(test, dm_mst_test_last_payload.write);
+	KUNIT_EXPECT_EQ(test, dm_mst_test_last_payload.address, 4U);
+}
+
+/**
+ * dm_mst_test_aux_transfer_hpd_discon_quirk - HPD disconnect quirk succeeds.
+ * @test: KUnit test context.
+ *
+ * AUX_RET_ERROR_HPD_DISCON on the sideband down request address should be
+ * treated as a successful transfer when the platform quirk is enabled.
+ */
+static void dm_mst_test_aux_transfer_hpd_discon_quirk(struct kunit *test)
+{
+	struct amdgpu_dm_dp_aux *dm_aux;
+	struct amdgpu_device *adev;
+	struct ddc_service *ddc;
+	struct dc_link *link;
+	struct dc *dc;
+	struct link_service *link_srv;
+	struct dc_context *ctx;
+	u8 buffer[2] = { 2, 0 };
+	ssize_t ret;
+
+	dm_aux = kunit_kzalloc(test, sizeof(*dm_aux), GFP_KERNEL);
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	ddc = kunit_kzalloc(test, sizeof(*ddc), GFP_KERNEL);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_aux);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ddc);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dm_mst_test_setup_dm_aux(dm_aux, ddc, link, dc, link_srv, ctx, adev);
+	adev->dm.aux_hpd_discon_quirk = true;
+	dm_mst_test_aux_transfer_raw_result = -EIO;
+	dm_mst_test_aux_transfer_raw_operation_result = AUX_RET_ERROR_HPD_DISCON;
+
+	ret = drm_dp_dpcd_write(&dm_aux->aux, DP_SIDEBAND_MSG_DOWN_REQ_BASE,
+					 buffer, sizeof(buffer));
+
+	KUNIT_EXPECT_EQ(test, ret, (ssize_t)sizeof(buffer));
+	KUNIT_EXPECT_TRUE(test, dm_mst_test_last_payload.write);
+	KUNIT_EXPECT_EQ(test, dm_mst_test_last_payload.address,
+			DP_SIDEBAND_MSG_DOWN_REQ_BASE);
+}
+
+/**
+ * dm_mst_test_aux_transfer_non_ack_reply - non-ACK AUX reply is logged.
+ * @test: KUnit test context.
+ *
+ * A successful read with a nonzero reply byte should still return the backend
+ * byte count while exercising the non-ACK reply handling path.
+ */
+static void dm_mst_test_aux_transfer_non_ack_reply(struct kunit *test)
+{
+	struct amdgpu_dm_dp_aux *dm_aux;
+	struct amdgpu_device *adev;
+	struct ddc_service *ddc;
+	struct dc_link *link;
+	struct dc *dc;
+	struct link_service *link_srv;
+	struct dc_context *ctx;
+	u8 buffer[2] = { 0 };
+	struct drm_dp_aux_msg msg = {
+		.address = 4,
+		.request = DP_AUX_NATIVE_READ,
+		.buffer = buffer,
+		.size = sizeof(buffer),
+	};
+	ssize_t ret;
+
+	dm_aux = kunit_kzalloc(test, sizeof(*dm_aux), GFP_KERNEL);
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	ddc = kunit_kzalloc(test, sizeof(*ddc), GFP_KERNEL);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_aux);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ddc);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dm_mst_test_setup_dm_aux(dm_aux, ddc, link, dc, link_srv, ctx, adev);
+	dm_mst_test_aux_transfer_raw_reply = DP_AUX_NATIVE_REPLY_NACK;
+
+	ret = dm_dp_aux_transfer(&dm_aux->aux, &msg);
+
+	KUNIT_EXPECT_EQ(test, ret, (ssize_t)sizeof(buffer));
+	KUNIT_EXPECT_EQ(test, dm_mst_test_last_payload.address, 4U);
+}
+
+/**
+ * dm_mst_test_fill_payload_flags_native_write - native write request decode.
+ * @test: KUnit test context.
+ *
+ * DP_AUX_NATIVE_WRITE clears i2c_over_aux and sets write; no I2C bits set.
+ */
+static void dm_mst_test_fill_payload_flags_native_write(struct kunit *test)
+{
+	struct aux_payload payload = { 0 };
+
+	dm_dp_aux_fill_payload_flags(DP_AUX_NATIVE_WRITE, &payload);
+
+	KUNIT_EXPECT_FALSE(test, payload.i2c_over_aux);
+	KUNIT_EXPECT_TRUE(test, payload.write);
+	KUNIT_EXPECT_FALSE(test, payload.mot);
+	KUNIT_EXPECT_FALSE(test, payload.write_status_update);
+}
+
+/**
+ * dm_mst_test_fill_payload_flags_native_read - native read request decode.
+ * @test: KUnit test context.
+ *
+ * DP_AUX_NATIVE_READ keeps i2c_over_aux clear; the I2C_READ bit clears write.
+ */
+static void dm_mst_test_fill_payload_flags_native_read(struct kunit *test)
+{
+	struct aux_payload payload = { 0 };
+
+	dm_dp_aux_fill_payload_flags(DP_AUX_NATIVE_READ, &payload);
+
+	KUNIT_EXPECT_FALSE(test, payload.i2c_over_aux);
+	KUNIT_EXPECT_FALSE(test, payload.write);
+	KUNIT_EXPECT_FALSE(test, payload.mot);
+}
+
+/**
+ * dm_mst_test_fill_payload_flags_i2c_read_mot - I2C read with MOT request decode.
+ * @test: KUnit test context.
+ *
+ * DP_AUX_I2C_READ sets i2c_over_aux and clears write; DP_AUX_I2C_MOT sets mot.
+ */
+static void dm_mst_test_fill_payload_flags_i2c_read_mot(struct kunit *test)
+{
+	struct aux_payload payload = { 0 };
+
+	dm_dp_aux_fill_payload_flags(DP_AUX_I2C_READ | DP_AUX_I2C_MOT, &payload);
+
+	KUNIT_EXPECT_TRUE(test, payload.i2c_over_aux);
+	KUNIT_EXPECT_FALSE(test, payload.write);
+	KUNIT_EXPECT_TRUE(test, payload.mot);
+}
+
+/**
+ * dm_mst_test_fill_payload_flags_write_status - write status update decode.
+ * @test: KUnit test context.
+ *
+ * DP_AUX_I2C_WRITE_STATUS_UPDATE sets write_status_update.
+ */
+static void dm_mst_test_fill_payload_flags_write_status(struct kunit *test)
+{
+	struct aux_payload payload = { 0 };
+
+	dm_dp_aux_fill_payload_flags(DP_AUX_I2C_WRITE | DP_AUX_I2C_WRITE_STATUS_UPDATE,
+				     &payload);
+
+	KUNIT_EXPECT_TRUE(test, payload.i2c_over_aux);
+	KUNIT_EXPECT_TRUE(test, payload.write_status_update);
+}
+
+/**
+ * dm_mst_test_msg_ready_mask - ESI mask selection per message-ready type.
+ * @test: KUnit test context.
+ *
+ * DOWN_REP and UP_REQ each select their single bit; other types select both.
+ */
+static void dm_mst_test_msg_ready_mask(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, dm_mst_msg_ready_mask(DOWN_REP_MSG_RDY_EVENT),
+			(u8)DP_DOWN_REP_MSG_RDY);
+	KUNIT_EXPECT_EQ(test, dm_mst_msg_ready_mask(UP_REQ_MSG_RDY_EVENT),
+			(u8)DP_UP_REQ_MSG_RDY);
+	KUNIT_EXPECT_EQ(test, dm_mst_msg_ready_mask(DOWN_OR_UP_MSG_RDY_EVENT),
+			(u8)(DP_DOWN_REP_MSG_RDY | DP_UP_REQ_MSG_RDY));
+	KUNIT_EXPECT_EQ(test, dm_mst_msg_ready_mask(NONE_MSG_RDY_EVENT),
+			(u8)(DP_DOWN_REP_MSG_RDY | DP_UP_REQ_MSG_RDY));
+}
+
+/**
+ * dm_mst_test_select_esi_dpcd_legacy - pre-1.2 DPCD ESI address/length.
+ * @test: KUnit test context.
+ *
+ * For DPCD rev < 0x12 the legacy DP_SINK_COUNT address/length pair is selected.
+ */
+static void dm_mst_test_select_esi_dpcd_legacy(struct kunit *test)
+{
+	int dpcd_addr = -1;
+	u8 dpcd_bytes_to_read = 0;
+
+	dm_mst_select_esi_dpcd(0x11, &dpcd_addr, &dpcd_bytes_to_read);
+
+	KUNIT_EXPECT_EQ(test, dpcd_addr, DP_SINK_COUNT);
+	KUNIT_EXPECT_EQ(test, (int)dpcd_bytes_to_read,
+			(int)(DP_LANE0_1_STATUS - DP_SINK_COUNT));
+}
+
+/**
+ * dm_mst_test_select_esi_dpcd_esi - 1.2+ DPCD ESI address/length.
+ * @test: KUnit test context.
+ *
+ * For DPCD rev >= 0x12 the ESI DP_SINK_COUNT_ESI address/length pair is selected.
+ */
+static void dm_mst_test_select_esi_dpcd_esi(struct kunit *test)
+{
+	int dpcd_addr = -1;
+	u8 dpcd_bytes_to_read = 0;
+
+	dm_mst_select_esi_dpcd(0x14, &dpcd_addr, &dpcd_bytes_to_read);
+
+	KUNIT_EXPECT_EQ(test, dpcd_addr, DP_SINK_COUNT_ESI);
+	KUNIT_EXPECT_EQ(test, (int)dpcd_bytes_to_read,
+			(int)(DP_PSR_ERROR_STATUS - DP_SINK_COUNT_ESI));
+}
+
+/**
+ * dm_mst_test_sideband_msg_ready_no_ready_bits - Test idle sideband event
+ * @test: KUnit test context
+ *
+ * Verify that dm_handle_mst_sideband_msg_ready_event() returns cleanly when
+ * the ESI read succeeds but no DOWN_REP/UP_REQ ready bits are set.
+ */
+static void dm_mst_test_sideband_msg_ready_no_ready_bits(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = dm_mst_test_alloc_sideband_connector(test);
+
+	dm_handle_mst_sideband_msg_ready_event(&aconnector->mst_mgr,
+					       DOWN_REP_MSG_RDY_EVENT);
+
+	KUNIT_EXPECT_EQ(test, dm_mst_test_dpcd[1], (u8)0);
+}
+
+/**
+ * dm_mst_test_sideband_msg_ready_read_error - Test ESI read failure path
+ * @test: KUnit test context
+ *
+ * Verify that dm_handle_mst_sideband_msg_ready_event() returns cleanly when
+ * the DPCD read fails before a ready bit can be handled.
+ */
+static void dm_mst_test_sideband_msg_ready_read_error(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = dm_mst_test_alloc_sideband_connector(test);
+	dm_mst_test_aux_transfer_override = -EIO;
+
+	dm_handle_mst_sideband_msg_ready_event(&aconnector->mst_mgr,
+					       DOWN_REP_MSG_RDY_EVENT);
+
+	KUNIT_EXPECT_EQ(test, dm_mst_test_dpcd[1], (u8)0);
+	dm_mst_test_aux_transfer_override = 0;
+}
+
+/**
+ * dm_mst_test_sideband_msg_ready_without_mst_state - Test ready bit no-op path
+ * @test: KUnit test context
+ *
+ * Verify that a DOWN_REP ready bit is filtered and then ignored when the MST
+ * topology manager is not enabled.
+ */
+static void dm_mst_test_sideband_msg_ready_without_mst_state(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = dm_mst_test_alloc_sideband_connector(test);
+	dm_mst_test_dpcd[(DP_SINK_COUNT_ESI + 1) & 0xf] = DP_DOWN_REP_MSG_RDY;
+
+	dm_handle_mst_sideband_msg_ready_event(&aconnector->mst_mgr,
+					       DOWN_REP_MSG_RDY_EVENT);
+
+	KUNIT_EXPECT_EQ(test, dm_mst_test_dpcd[(DP_SINK_COUNT_ESI + 1) & 0xf],
+			 DP_DOWN_REP_MSG_RDY);
+}
+
+/**
+ * dm_mst_test_down_rep_msg_ready_wrapper - Test DOWN_REP wrapper
+ * @test: KUnit test context
+ *
+ * Verify that dm_handle_mst_down_rep_msg_ready() forwards to the generic MST
+ * sideband handler with the DOWN_REP event selection.
+ */
+static void dm_mst_test_down_rep_msg_ready_wrapper(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = dm_mst_test_alloc_sideband_connector(test);
+
+	dm_handle_mst_down_rep_msg_ready(&aconnector->mst_mgr);
+
+	KUNIT_EXPECT_EQ(test, dm_mst_test_dpcd[1], (u8)0);
+}
+
+/**
+ * dm_mst_test_initialize_dp_connector_edp - Test eDP initialization path
+ * @test: KUnit test context
+ *
+ * Verify that amdgpu_dm_initialize_dp_connector() initializes the DP AUX state
+ * and exits before MST topology setup for eDP connectors.
+ */
+static void dm_mst_test_initialize_dp_connector_edp(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_device *adev;
+	struct ddc_service *ddc;
+	struct dc_link *link;
+
+	adev = dm_kunit_alloc_adev(test);
+	link = dm_kunit_alloc_link(test);
+	ddc = kunit_kzalloc(test, sizeof(*ddc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ddc);
+
+	adev->dm.adev = adev;
+	adev->dm.ddev = &adev->ddev;
+	link->ddc = ddc;
+	aconnector = dm_kunit_alloc_connector(test, adev, link);
+	aconnector->base.connector_type = DRM_MODE_CONNECTOR_eDP;
+
+	amdgpu_dm_initialize_dp_connector(&adev->dm, aconnector, 5);
+
+	KUNIT_EXPECT_TRUE(test, aconnector->dm_dp_aux.aux.transfer == dm_dp_aux_transfer);
+	KUNIT_EXPECT_PTR_EQ(test, aconnector->dm_dp_aux.aux.drm_dev, &adev->ddev);
+	KUNIT_EXPECT_PTR_EQ(test, aconnector->dm_dp_aux.ddc_service, ddc);
+	KUNIT_EXPECT_PTR_EQ(test, aconnector->mst_mgr.dev, NULL);
+	KUNIT_EXPECT_NOT_NULL(test, aconnector->dm_dp_aux.aux.name);
+	if (aconnector->dm_dp_aux.aux.name)
+		KUNIT_EXPECT_NOT_NULL(test, strstr(aconnector->dm_dp_aux.aux.name, "5"));
+
+	drm_dp_cec_unregister_connector(&aconnector->dm_dp_aux.aux);
+	kfree(aconnector->dm_dp_aux.aux.name);
+}
+
+static bool dm_mst_test_dp_get_max_link_enc_cap(const struct dc_link *link,
+						struct dc_link_settings *cap)
+{
+	return true;
+}
+
+static void dm_mst_test_connector_destroy(struct drm_connector *connector)
+{
+}
+
+static const struct drm_connector_funcs dm_mst_test_connector_funcs = {
+	.reset = drm_atomic_helper_connector_reset,
+	.destroy = dm_mst_test_connector_destroy,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+/**
+ * dm_mst_test_initialize_dp_connector_mst - Test MST root initialization path
+ * @test: KUnit test context
+ *
+ * Verify that amdgpu_dm_initialize_dp_connector() initializes the MST topology
+ * manager for a non-eDP DisplayPort connector. This exercises the path past the
+ * eDP early return, including dc_link_dp_get_max_link_enc_cap() and
+ * drm_dp_mst_topology_mgr_init(). A fully initialized DRM mode config and
+ * connector are required because the topology manager registers a private
+ * atomic object and the subconnector property is attached to the connector.
+ */
+static void dm_mst_test_initialize_dp_connector_mst(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_device *adev;
+	struct link_service *link_srv;
+	struct ddc_service *ddc;
+	struct dc_link *link;
+	struct dc *dc;
+	int ret;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	ret = drmm_mode_config_init(&adev->ddev);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	ddc = kunit_kzalloc(test, sizeof(*ddc), GFP_KERNEL);
+	link = dm_kunit_alloc_link(test);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ddc);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+
+	link_srv->dp_get_max_link_enc_cap = dm_mst_test_dp_get_max_link_enc_cap;
+	dc->link_srv = link_srv;
+	link->dc = dc;
+	link->ddc = ddc;
+
+	adev->dm.adev = adev;
+	adev->dm.ddev = &adev->ddev;
+
+	aconnector = dm_kunit_alloc_connector(test, adev, link);
+
+	ret = drm_connector_init(&adev->ddev, &aconnector->base,
+				 &dm_mst_test_connector_funcs,
+				 DRM_MODE_CONNECTOR_DisplayPort);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	amdgpu_dm_initialize_dp_connector(&adev->dm, aconnector, 7);
+
+	KUNIT_EXPECT_TRUE(test, aconnector->dm_dp_aux.aux.transfer == dm_dp_aux_transfer);
+	KUNIT_EXPECT_PTR_EQ(test, aconnector->mst_mgr.dev, &adev->ddev);
+	KUNIT_EXPECT_PTR_EQ(test, aconnector->mst_mgr.aux, &aconnector->dm_dp_aux.aux);
+	KUNIT_EXPECT_EQ(test, aconnector->mst_mgr.max_payloads, 4);
+	KUNIT_EXPECT_TRUE(test, aconnector->mst_mgr.cbs != NULL);
+
+	drm_dp_mst_topology_mgr_destroy(&aconnector->mst_mgr);
+	drm_dp_cec_unregister_connector(&aconnector->dm_dp_aux.aux);
+	kfree(aconnector->dm_dp_aux.aux.name);
+	drm_connector_cleanup(&aconnector->base);
+}
+
+/**
+ * dm_mst_test_atomic_best_encoder - Test MST encoder selection
+ * @test: KUnit test context
+ *
+ * Verify that dm_mst_atomic_best_encoder() selects the MST encoder indexed by
+ * the CRTC ID in the connector's new atomic state. This uses structural DRM
+ * mocks only; registering connector/CRTC objects is unnecessary for this helper.
+ */
+static void dm_mst_test_atomic_best_encoder(struct kunit *test)
+{
+	struct drm_connector_state connector_state = { 0 };
+	struct drm_atomic_commit state = { 0 };
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+	unsigned int connector_index = 3;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+
+	aconnector->base.dev = &adev->ddev;
+	aconnector->base.index = connector_index;
+	acrtc->crtc_id = 2;
+	connector_state.connector = &aconnector->base;
+	connector_state.crtc = &acrtc->base;
+	state.num_connector = connector_index + 1;
+	state.connectors = kunit_kzalloc(test,
+					 sizeof(*state.connectors) * state.num_connector,
+					 GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state.connectors);
+	state.connectors[connector_index].ptr = &aconnector->base;
+	state.connectors[connector_index].new_state = &connector_state;
+
+	KUNIT_EXPECT_PTR_EQ(test, dm_mst_atomic_best_encoder(&aconnector->base, &state),
+			     &adev->dm.mst_encoders[2].base);
+}
+
+/**
+ * dm_mst_test_create_fake_mst_encoders - Test fake MST encoder setup
+ * @test: KUnit test context
+ *
+ * Verify that dm_dp_create_fake_mst_encoders() initializes the requested MST
+ * encoders as DPMST encoders with the CRTC mask derived from the device state.
+ */
+static void dm_mst_test_create_fake_mst_encoders(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_device *drm;
+	int i;
+
+	adev = dm_kunit_alloc_adev(test);
+	drm = &adev->ddev;
+	adev->dm.display_indexes_num = 3;
+	adev->mode_info.num_crtc = 3;
+
+	dm_dp_create_fake_mst_encoders(adev);
+
+	for (i = 0; i < adev->dm.display_indexes_num; i++) {
+		struct drm_encoder *encoder = &adev->dm.mst_encoders[i].base;
+
+		KUNIT_EXPECT_PTR_EQ(test, encoder->dev, drm);
+		KUNIT_EXPECT_EQ(test, encoder->encoder_type, DRM_MODE_ENCODER_DPMST);
+		KUNIT_EXPECT_EQ(test, encoder->possible_crtcs, 0x7U);
+		KUNIT_EXPECT_TRUE(test, encoder->helper_private != NULL);
+	}
+}
+
+/**
+ * dm_mst_test_atomic_check_no_old_crtc - Test atomic check no-op path
+ * @test: KUnit test context
+ *
+ * Verify that dm_dp_mst_atomic_check() returns success when the MST port's old
+ * connector state has no CRTC, before MST topology state is required.
+ */
+static void dm_mst_test_atomic_check_no_old_crtc(struct kunit *test)
+{
+	struct drm_connector_state *old_conn_state;
+	struct drm_connector_state *new_conn_state;
+	struct drm_atomic_commit *state;
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_dm_connector *root;
+	struct drm_dp_mst_port *port;
+	unsigned int connector_index = 2;
+
+	old_conn_state = kunit_kzalloc(test, sizeof(*old_conn_state), GFP_KERNEL);
+	new_conn_state = kunit_kzalloc(test, sizeof(*new_conn_state), GFP_KERNEL);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	root = kunit_kzalloc(test, sizeof(*root), GFP_KERNEL);
+	port = kunit_kzalloc(test, sizeof(*port), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_conn_state);
+	KUNIT_ASSERT_NOT_NULL(test, new_conn_state);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	KUNIT_ASSERT_NOT_NULL(test, root);
+	KUNIT_ASSERT_NOT_NULL(test, port);
+
+	aconnector->base.index = connector_index;
+	aconnector->mst_root = root;
+	aconnector->mst_output_port = port;
+	port->connector = &aconnector->base;
+	old_conn_state->connector = &aconnector->base;
+	new_conn_state->connector = &aconnector->base;
+	state->num_connector = connector_index + 1;
+	state->connectors = kunit_kzalloc(test,
+					 sizeof(*state->connectors) * state->num_connector,
+					 GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state->connectors);
+	state->connectors[connector_index].ptr = &aconnector->base;
+	state->connectors[connector_index].old_state = old_conn_state;
+	state->connectors[connector_index].new_state = new_conn_state;
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_atomic_check(&aconnector->base, state), 0);
+}
+
+/**
+ * dm_mst_test_detect_unregistered - Test detect skips unregistered connector
+ * @test: KUnit test context
+ *
+ * Verify that dm_dp_mst_detect() returns disconnected for an unregistered
+ * connector before calling into the MST topology helper.
+ */
+static void dm_mst_test_detect_unregistered(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+
+	aconnector->base.registration_state = DRM_CONNECTOR_UNREGISTERED;
+
+	KUNIT_EXPECT_EQ(test,
+			dm_dp_mst_detect(&aconnector->base, NULL, false),
+			(int)connector_status_disconnected);
+}
+
+#if !defined(CONFIG_DRM_AMD_DC_FP)
+/**
+ * dm_mst_test_fp_guarded_public_stubs - Test FP-off public fallbacks
+ * @test: KUnit test context
+ *
+ * When CONFIG_DRM_AMD_DC_FP is disabled, the public DSC validation helper
+ * has no FP body and must return DC_OK without touching its arguments.
+ */
+static void dm_mst_test_fp_guarded_public_stubs(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_is_port_support_mode(NULL, NULL),
+			(enum dc_status)DC_OK);
+}
+#endif
+
+static struct kunit_case dm_mst_types_test_cases[] = {
+	/* needs_dsc_aux_workaround tests */
+	KUNIT_CASE(dm_mst_test_needs_dsc_aux_workaround_match),
+	KUNIT_CASE(dm_mst_test_needs_dsc_aux_workaround_rev12),
+	KUNIT_CASE(dm_mst_test_needs_dsc_aux_workaround_wrong_dev_id),
+	KUNIT_CASE(dm_mst_test_needs_dsc_aux_workaround_wrong_rev),
+	KUNIT_CASE(dm_mst_test_needs_dsc_aux_workaround_low_sink_count),
+	KUNIT_CASE(dm_mst_test_needs_dsc_aux_workaround_zero_sink_count),
+	/* dm_mst_get_pbn_divider tests */
+	KUNIT_CASE(dm_mst_test_pbn_divider_null_link),
+	KUNIT_CASE(dm_mst_test_pbn_divider_uses_link_bandwidth),
+	/* amdgpu_dm_mst_reset_mst_connector_setting tests */
+	KUNIT_CASE(dm_mst_test_reset_connector_setting),
+	/* retrieve_downstream_port_device tests */
+	KUNIT_CASE(dm_mst_test_retrieve_downstream_no_aux),
+	KUNIT_CASE(dm_mst_test_retrieve_downstream_present),
+	KUNIT_CASE(dm_mst_test_retrieve_downstream_aux_error),
+	/* retrieve_branch_specific_data tests */
+	KUNIT_CASE(dm_mst_test_retrieve_branch_no_parent),
+	KUNIT_CASE(dm_mst_test_retrieve_branch_reads_oui),
+	/* dm_dp_aux_transfer_result tests */
+	KUNIT_CASE(dm_mst_test_aux_result_success),
+	KUNIT_CASE(dm_mst_test_aux_result_eio),
+	KUNIT_CASE(dm_mst_test_aux_result_ebusy),
+	KUNIT_CASE(dm_mst_test_aux_result_timeout),
+	KUNIT_CASE(dm_mst_test_aux_transfer_native_read),
+	KUNIT_CASE(dm_mst_test_aux_transfer_native_write),
+	KUNIT_CASE(dm_mst_test_aux_transfer_partial_write),
+	KUNIT_CASE(dm_mst_test_aux_transfer_error_result),
+	KUNIT_CASE(dm_mst_test_aux_transfer_hpd_discon_quirk),
+	KUNIT_CASE(dm_mst_test_aux_transfer_non_ack_reply),
+	/* dm_dp_aux_fill_payload_flags tests */
+	KUNIT_CASE(dm_mst_test_fill_payload_flags_native_write),
+	KUNIT_CASE(dm_mst_test_fill_payload_flags_native_read),
+	KUNIT_CASE(dm_mst_test_fill_payload_flags_i2c_read_mot),
+	KUNIT_CASE(dm_mst_test_fill_payload_flags_write_status),
+	/* dm_mst_msg_ready_mask tests */
+	KUNIT_CASE(dm_mst_test_msg_ready_mask),
+	/* dm_mst_select_esi_dpcd tests */
+	KUNIT_CASE(dm_mst_test_select_esi_dpcd_legacy),
+	KUNIT_CASE(dm_mst_test_select_esi_dpcd_esi),
+	/* dm_handle_mst_sideband_msg_ready_event tests */
+	KUNIT_CASE(dm_mst_test_sideband_msg_ready_no_ready_bits),
+	KUNIT_CASE(dm_mst_test_sideband_msg_ready_read_error),
+	KUNIT_CASE(dm_mst_test_sideband_msg_ready_without_mst_state),
+	KUNIT_CASE(dm_mst_test_down_rep_msg_ready_wrapper),
+	/* amdgpu_dm_initialize_dp_connector tests */
+	KUNIT_CASE(dm_mst_test_initialize_dp_connector_edp),
+	KUNIT_CASE(dm_mst_test_initialize_dp_connector_mst),
+	/* dm_mst_atomic_best_encoder tests */
+	KUNIT_CASE(dm_mst_test_atomic_best_encoder),
+	/* dm_dp_create_fake_mst_encoders tests */
+	KUNIT_CASE(dm_mst_test_create_fake_mst_encoders),
+	/* dm_dp_mst_atomic_check tests */
+	KUNIT_CASE(dm_mst_test_atomic_check_no_old_crtc),
+	/* dm_dp_mst_detect tests */
+	KUNIT_CASE(dm_mst_test_detect_unregistered),
+	/* CONFIG_DRM_AMD_DC_FP disabled public paths */
+#if !defined(CONFIG_DRM_AMD_DC_FP)
+	KUNIT_CASE(dm_mst_test_fp_guarded_public_stubs),
+#endif
+	{}
+};
+
+static struct kunit_suite dm_mst_types_test_suite = {
+	.name = "amdgpu_dm_mst_types",
+	.test_cases = dm_mst_types_test_cases,
+};
+
+kunit_test_suite(dm_mst_types_test_suite);
+
+MODULE_LICENSE("Dual MIT/GPL");
+MODULE_DESCRIPTION("KUnit tests for amdgpu_dm_mst_types");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_plane_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_plane_test.c
new file mode 100644
index 0000000000000..a751e67cadf1d
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_plane_test.c
@@ -0,0 +1,3284 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * KUnit tests for amdgpu_dm_plane.c
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#include <kunit/test.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_blend.h>
+#include "link_enc_cfg.h"
+#include "amdgpu_dm_plane.h"
+#include "amdgpu_rlc.h"
+#include "gc/gc_11_0_0_offset.h"
+#include "gc/gc_11_0_0_sh_mask.h"
+#include <drm/amdgpu_drm.h>
+#include <drm/drm_plane.h>
+
+struct dm_test_dcc_cap_ctx {
+	bool callback_ret;
+	bool capable;
+	bool output_independent_64b_blks;
+	bool called;
+	struct dc_dcc_surface_param captured_input;
+};
+
+static struct dm_test_dcc_cap_ctx *dm_test_dcc_ctx;
+
+struct dm_test_gfx11_reg_ctx {
+	u32 gb_addr_config;
+	u32 gc_reg_offsets[1];
+	u32 expected_reg;
+	u32 captured_reg;
+	u32 captured_acc_flags;
+	u32 captured_hwip;
+	u32 captured_xcc_id;
+	bool called;
+};
+
+static struct dm_test_gfx11_reg_ctx *dm_test_gfx11_reg_ctx;
+
+static u32 dm_test_gfx11_rreg32(struct amdgpu_device *adev,
+				u32 reg, u32 acc_flags, u32 hwip,
+				u32 xcc_id)
+{
+	if (!dm_test_gfx11_reg_ctx)
+		return 0;
+
+	dm_test_gfx11_reg_ctx->called = true;
+	dm_test_gfx11_reg_ctx->captured_reg = reg;
+	dm_test_gfx11_reg_ctx->captured_acc_flags = acc_flags;
+	dm_test_gfx11_reg_ctx->captured_hwip = hwip;
+	dm_test_gfx11_reg_ctx->captured_xcc_id = xcc_id;
+
+	return dm_test_gfx11_reg_ctx->gb_addr_config;
+}
+
+static const struct amdgpu_rlc_reg_funcs dm_test_gfx11_reg_funcs = {
+	.rreg32 = dm_test_gfx11_rreg32,
+};
+
+static bool dm_test_get_dcc_compression_cap(const struct dc *dc,
+					    const struct dc_dcc_surface_param *input,
+					    struct dc_surface_dcc_cap *output)
+{
+	if (!dm_test_dcc_ctx)
+		return false;
+
+	dm_test_dcc_ctx->called = true;
+	dm_test_dcc_ctx->captured_input = *input;
+	output->capable = dm_test_dcc_ctx->capable;
+	output->grph.rgb.independent_64b_blks = dm_test_dcc_ctx->output_independent_64b_blks;
+
+	return dm_test_dcc_ctx->callback_ret;
+}
+
+static void dm_test_init_validate_dcc_inputs(struct amdgpu_device **adev,
+					     struct dc **dc,
+					     struct dc_tiling_info *tiling_info,
+					     struct dc_plane_dcc_param *dcc,
+					     struct dc_plane_address *address,
+					     struct plane_size *plane_size,
+					     struct kunit *test)
+{
+	*adev = kunit_kzalloc(test, sizeof(**adev), GFP_KERNEL);
+	*dc = kunit_kzalloc(test, sizeof(**dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, *adev);
+	KUNIT_ASSERT_NOT_NULL(test, *dc);
+
+	(*adev)->dm.dc = *dc;
+	(*adev)->family = AMDGPU_FAMILY_NV;
+
+	tiling_info->gfx9.swizzle = 9;
+	dcc->enable = 1;
+	dcc->independent_64b_blks = 1;
+	plane_size->surface_size.width = 1920;
+	plane_size->surface_size.height = 1080;
+
+	(void)address;
+}
+
+
+/**
+ * dm_test_plane_is_video_format_known_video() - Verify known video formats.
+ * @test: KUnit test context.
+ *
+ * Verify if NV12, NV21, and P010 are treated as video formats.
+ */
+static void dm_test_plane_is_video_format_known_video(struct kunit *test)
+{
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_plane_is_video_format(DRM_FORMAT_NV12));
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_plane_is_video_format(DRM_FORMAT_NV21));
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_plane_is_video_format(DRM_FORMAT_P010));
+}
+
+/**
+ * dm_test_fill_blending_defaults() - Verify default blending output values.
+ * @test: KUnit test context.
+ *
+ * Verify if default blending output values are used for opaque alpha and no
+ * per-pixel blending.
+ */
+static void dm_test_fill_blending_defaults(struct kunit *test)
+{
+	struct drm_plane_state state = { 0 };
+	bool per_pixel_alpha;
+	bool pre_multiplied_alpha;
+	bool global_alpha;
+	int global_alpha_value;
+
+	state.pixel_blend_mode = DRM_MODE_BLEND_PIXEL_NONE;
+	state.alpha = 0xffff;
+
+	amdgpu_dm_plane_fill_blending_from_plane_state(&state,
+						       &per_pixel_alpha,
+						       &pre_multiplied_alpha,
+						       &global_alpha,
+						       &global_alpha_value);
+
+	KUNIT_EXPECT_FALSE(test, per_pixel_alpha);
+	KUNIT_EXPECT_TRUE(test, pre_multiplied_alpha);
+	KUNIT_EXPECT_FALSE(test, global_alpha);
+	KUNIT_EXPECT_EQ(test, global_alpha_value, 0xff);
+}
+
+/**
+ * dm_test_fill_blending_premulti_alpha_format() - Verify premultiplied alpha path.
+ * @test: KUnit test context.
+ *
+ * Verify if premultiplied mode enables per-pixel alpha for ARGB8888.
+ */
+static void dm_test_fill_blending_premulti_alpha_format(struct kunit *test)
+{
+	struct drm_plane_state state = { 0 };
+	struct drm_framebuffer fb = { 0 };
+	bool per_pixel_alpha;
+	bool pre_multiplied_alpha;
+	bool global_alpha;
+	int global_alpha_value;
+
+	fb.format = drm_format_info(DRM_FORMAT_ARGB8888);
+	KUNIT_ASSERT_NOT_NULL(test, fb.format);
+
+	state.fb = &fb;
+	state.pixel_blend_mode = DRM_MODE_BLEND_PREMULTI;
+	state.alpha = 0xffff;
+
+	amdgpu_dm_plane_fill_blending_from_plane_state(&state,
+						       &per_pixel_alpha,
+						       &pre_multiplied_alpha,
+						       &global_alpha,
+						       &global_alpha_value);
+
+	KUNIT_EXPECT_TRUE(test, per_pixel_alpha);
+	KUNIT_EXPECT_TRUE(test, pre_multiplied_alpha);
+	KUNIT_EXPECT_FALSE(test, global_alpha);
+	KUNIT_EXPECT_EQ(test, global_alpha_value, 0xff);
+}
+
+/**
+ * dm_test_fill_blending_coverage_alpha_format() - Verify coverage mode behavior.
+ * @test: KUnit test context.
+ *
+ * Verify if coverage mode sets per-pixel alpha and disables
+ * pre_multiplied_alpha for ARGB8888.
+ */
+static void dm_test_fill_blending_coverage_alpha_format(struct kunit *test)
+{
+	struct drm_plane_state state = { 0 };
+	struct drm_framebuffer fb = { 0 };
+	bool per_pixel_alpha;
+	bool pre_multiplied_alpha;
+	bool global_alpha;
+	int global_alpha_value;
+
+	fb.format = drm_format_info(DRM_FORMAT_ARGB8888);
+	KUNIT_ASSERT_NOT_NULL(test, fb.format);
+
+	state.fb = &fb;
+	state.pixel_blend_mode = DRM_MODE_BLEND_COVERAGE;
+	state.alpha = 0xffff;
+
+	amdgpu_dm_plane_fill_blending_from_plane_state(&state,
+						       &per_pixel_alpha,
+						       &pre_multiplied_alpha,
+						       &global_alpha,
+						       &global_alpha_value);
+
+	KUNIT_EXPECT_TRUE(test, per_pixel_alpha);
+	KUNIT_EXPECT_FALSE(test, pre_multiplied_alpha);
+	KUNIT_EXPECT_FALSE(test, global_alpha);
+	KUNIT_EXPECT_EQ(test, global_alpha_value, 0xff);
+}
+
+/**
+ * dm_test_fill_blending_global_alpha() - Verify global alpha conversion to 8 bits.
+ * @test: KUnit test context.
+ *
+ * Verify if global alpha is enabled and converted from 16-bit to 8-bit.
+ */
+static void dm_test_fill_blending_global_alpha(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_plane *plane;
+	struct drm_plane_state *state;
+	bool per_pixel_alpha;
+	bool pre_multiplied_alpha;
+	bool global_alpha;
+	int global_alpha_value;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+
+	plane->dev = &adev->ddev;
+	state->plane = plane;
+	state->pixel_blend_mode = DRM_MODE_BLEND_PIXEL_NONE;
+	state->alpha = 0x8000;
+
+	amdgpu_dm_plane_fill_blending_from_plane_state(state,
+						       &per_pixel_alpha,
+						       &pre_multiplied_alpha,
+						       &global_alpha,
+						       &global_alpha_value);
+
+	KUNIT_EXPECT_FALSE(test, per_pixel_alpha);
+	KUNIT_EXPECT_TRUE(test, pre_multiplied_alpha);
+	KUNIT_EXPECT_TRUE(test, global_alpha);
+	KUNIT_EXPECT_EQ(test, global_alpha_value, 0x80);
+}
+
+/**
+ * dm_test_modifier_has_dcc() - Verify helper detects AMD DCC modifiers.
+ * @test: KUnit test context.
+ *
+ * Verify if DCC detection works for linear and AMD DCC modifiers.
+ */
+static void dm_test_modifier_has_dcc(struct kunit *test)
+{
+	uint64_t dcc_mod = AMD_FMT_MOD | AMD_FMT_MOD_SET(DCC, 1);
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_plane_modifier_has_dcc(DRM_FORMAT_MOD_LINEAR));
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_plane_modifier_has_dcc(dcc_mod));
+}
+
+/**
+ * dm_test_modifier_gfx9_swizzle_mode() - Verify swizzle helper for linear and AMD modifiers.
+ * @test: KUnit test context.
+ *
+ * Verify if swizzle mode decoding works for linear and AMD tiled modifiers.
+ */
+static void dm_test_modifier_gfx9_swizzle_mode(struct kunit *test)
+{
+	uint64_t mod = AMD_FMT_MOD | AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_S_X);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_modifier_gfx9_swizzle_mode(DRM_FORMAT_MOD_LINEAR), 0U);
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_modifier_gfx9_swizzle_mode(mod),
+			(unsigned int)AMD_FMT_MOD_TILE_GFX9_64K_S_X);
+}
+
+/**
+ * dm_test_get_plane_formats() - Verify plane format counts for key plane types.
+ * @test: KUnit test context.
+ *
+ * Verify if returned format counts match primary, overlay, and cursor planes.
+ */
+static void dm_test_get_plane_formats(struct kunit *test)
+{
+	struct drm_plane *plane;
+	struct dc_plane_cap *cap;
+	uint32_t formats[32] = {0};
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	cap = kunit_kzalloc(test, sizeof(*cap), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, cap);
+
+	plane->type = DRM_PLANE_TYPE_PRIMARY;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_get_plane_formats(plane, NULL, formats, 32), 14);
+
+	cap->pixel_format_support.nv12 = true;
+	cap->pixel_format_support.p010 = true;
+	cap->pixel_format_support.fp16 = true;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_get_plane_formats(plane, cap, formats, 32), 20);
+
+	plane->type = DRM_PLANE_TYPE_OVERLAY;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_get_plane_formats(plane, NULL, formats, 32), 9);
+
+	plane->type = DRM_PLANE_TYPE_CURSOR;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_get_plane_formats(plane, NULL, formats, 32), 1);
+}
+
+/**
+ * dm_test_get_plane_modifiers() - Verify early-return and cursor modifier list.
+ * @test: KUnit test context.
+ *
+ * Verify if modifier list handling works for unsupported families and cursor planes.
+ */
+static void dm_test_get_plane_modifiers(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	uint64_t *mods = NULL;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	adev->family = AMDGPU_FAMILY_SI;
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_get_plane_modifiers(adev, DRM_PLANE_TYPE_PRIMARY, &mods),
+			0);
+	KUNIT_EXPECT_PTR_EQ(test, mods, NULL);
+
+	adev->family = AMDGPU_FAMILY_NV;
+	KUNIT_ASSERT_EQ(test,
+			amdgpu_dm_plane_get_plane_modifiers(adev, DRM_PLANE_TYPE_CURSOR, &mods),
+			0);
+	KUNIT_ASSERT_NOT_NULL(test, mods);
+	KUNIT_EXPECT_EQ(test, mods[0], DRM_FORMAT_MOD_LINEAR);
+	KUNIT_EXPECT_EQ(test, mods[1], DRM_FORMAT_MOD_INVALID);
+	kfree(mods);
+}
+
+/**
+ * dm_test_fill_dc_scaling_info() - Verify basic error and success paths.
+ * @test: KUnit test context.
+ *
+ * Verify if scaling info rejects invalid sizes and accepts valid sizes.
+ */
+static void dm_test_fill_dc_scaling_info(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_plane_state state = {0};
+	struct dc_scaling_info info = {0};
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	state.src_w = 0;
+	state.src_h = 100 << 16;
+	state.crtc_w = 100;
+	state.crtc_h = 100;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_fill_dc_scaling_info(adev, &state, &info), -EINVAL);
+
+	state.src_w = 100 << 16;
+	state.src_h = 100 << 16;
+	state.crtc_w = 100;
+	state.crtc_h = 100;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_fill_dc_scaling_info(adev, &state, &info), 0);
+}
+
+/**
+ * dm_test_get_min_max_dc_plane_scaling() - Verify format-specific cap selection and 1->1000 conversion.
+ * @test: KUnit test context.
+ *
+ * Verify if min/max scaling values are correct for NV12 and XRGB8888 formats.
+ */
+static void dm_test_get_min_max_dc_plane_scaling(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct drm_framebuffer *fb;
+	int min_downscale = 0;
+	int max_upscale = 0;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+
+	adev->dm.dc = dc;
+	dc->caps.planes[0].max_upscale_factor.nv12 = 1;
+	dc->caps.planes[0].max_downscale_factor.nv12 = 1;
+	dc->caps.planes[0].max_upscale_factor.argb8888 = 1600;
+	dc->caps.planes[0].max_downscale_factor.argb8888 = 250;
+
+	fb->format = drm_format_info(DRM_FORMAT_NV12);
+	KUNIT_ASSERT_NOT_NULL(test, fb->format);
+	amdgpu_dm_plane_get_min_max_dc_plane_scaling(&adev->ddev, fb, &min_downscale, &max_upscale);
+	KUNIT_EXPECT_EQ(test, min_downscale, 1000);
+	KUNIT_EXPECT_EQ(test, max_upscale, 1000);
+
+	fb->format = drm_format_info(DRM_FORMAT_XRGB8888);
+	KUNIT_ASSERT_NOT_NULL(test, fb->format);
+	amdgpu_dm_plane_get_min_max_dc_plane_scaling(&adev->ddev, fb, &min_downscale, &max_upscale);
+	KUNIT_EXPECT_EQ(test, min_downscale, 250);
+	KUNIT_EXPECT_EQ(test, max_upscale, 1600);
+}
+
+/**
+ * dm_test_fill_plane_buffer_attributes_gfx8() - Verify graphics path and GFX8 tiling fill.
+ * @test: KUnit test context.
+ *
+ * Verify if GFX8 plane buffer attributes and tiling fields are filled correctly.
+ */
+static void dm_test_fill_plane_buffer_attributes_gfx8(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_framebuffer *afb;
+	struct dc_tiling_info *tiling_info;
+	struct plane_size *plane_size;
+	struct dc_plane_dcc_param *dcc;
+	struct dc_plane_address *address;
+	uint64_t tiling_flags = 0;
+	int ret;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	tiling_info = kunit_kzalloc(test, sizeof(*tiling_info), GFP_KERNEL);
+	plane_size = kunit_kzalloc(test, sizeof(*plane_size), GFP_KERNEL);
+	dcc = kunit_kzalloc(test, sizeof(*dcc), GFP_KERNEL);
+	address = kunit_kzalloc(test, sizeof(*address), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+	KUNIT_ASSERT_NOT_NULL(test, tiling_info);
+	KUNIT_ASSERT_NOT_NULL(test, plane_size);
+	KUNIT_ASSERT_NOT_NULL(test, dcc);
+	KUNIT_ASSERT_NOT_NULL(test, address);
+
+	adev->family = AMDGPU_FAMILY_SI;
+	afb->address = 0x12345000ULL;
+	afb->base.width = 1920;
+	afb->base.height = 1080;
+	afb->base.offsets[0] = 0x1000;
+	afb->base.pitches[0] = 7680;
+	afb->base.format = drm_format_info(DRM_FORMAT_XRGB8888);
+	KUNIT_ASSERT_NOT_NULL(test, afb->base.format);
+
+	tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, DC_ARRAY_1D_TILED_THIN1);
+	tiling_flags |= AMDGPU_TILING_SET(PIPE_CONFIG, 5);
+
+	ret = amdgpu_dm_plane_fill_plane_buffer_attributes(adev, afb,
+		SURFACE_PIXEL_FORMAT_GRPH_ARGB8888, ROTATION_ANGLE_0,
+		tiling_flags, tiling_info, plane_size, dcc, address, true);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, plane_size->surface_size.width, 1920);
+	KUNIT_EXPECT_EQ(test, plane_size->surface_size.height, 1080);
+	KUNIT_EXPECT_EQ(test, plane_size->surface_pitch, 1920);
+	KUNIT_EXPECT_EQ(test, address->type, (int)PLN_ADDR_TYPE_GRAPHICS);
+	KUNIT_EXPECT_TRUE(test, address->tmz_surface);
+	KUNIT_EXPECT_EQ(test, (int)tiling_info->gfx8.array_mode, (int)DC_ARRAY_1D_TILED_THIN1);
+	KUNIT_EXPECT_EQ(test, tiling_info->gfx8.pipe_config, 5U);
+}
+
+/**
+ * dm_test_get_cursor_position() - Verify cursor clipping and off-screen handling.
+ * @test: KUnit test context.
+ *
+ * Verify if cursor clipping, hotspot adjustment, and off-screen disable behavior work.
+ */
+static void dm_test_get_cursor_position(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *amdgpu_crtc;
+	struct drm_plane *plane;
+	struct drm_plane_state *state;
+	struct drm_framebuffer *fb;
+	struct dc_cursor_position position = {0};
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	amdgpu_crtc = kunit_kzalloc(test, sizeof(*amdgpu_crtc), GFP_KERNEL);
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, amdgpu_crtc);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(4, 0, 0);
+	amdgpu_crtc->max_cursor_width = 64;
+	amdgpu_crtc->max_cursor_height = 64;
+
+	plane->dev = &adev->ddev;
+	plane->state = state;
+	state->fb = fb;
+	state->crtc_x = -5;
+	state->crtc_y = -7;
+	state->crtc_w = 32;
+	state->crtc_h = 32;
+
+	KUNIT_ASSERT_EQ(test,
+			amdgpu_dm_plane_get_cursor_position(plane, &amdgpu_crtc->base, &position),
+			0);
+	KUNIT_EXPECT_TRUE(test, position.enable);
+	KUNIT_EXPECT_EQ(test, position.x, 0);
+	KUNIT_EXPECT_EQ(test, position.y, 0);
+	KUNIT_EXPECT_EQ(test, position.x_hotspot, 5);
+	KUNIT_EXPECT_EQ(test, position.y_hotspot, 7);
+	KUNIT_EXPECT_TRUE(test, position.translate_by_source);
+
+	memset(&position, 0, sizeof(position));
+	state->crtc_x = -64;
+	state->crtc_y = 0;
+	KUNIT_ASSERT_EQ(test,
+			amdgpu_dm_plane_get_cursor_position(plane, &amdgpu_crtc->base, &position),
+			0);
+	KUNIT_EXPECT_FALSE(test, position.enable);
+}
+
+/**
+ * dm_test_format_mod_supported() - Verify key format/modifier acceptance and rejection paths.
+ * @test: KUnit test context.
+ *
+ * Verify if format-modifier support checks match accepted and rejected cases.
+ */
+static void dm_test_format_mod_supported(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_plane *plane;
+	uint64_t listed_mod;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+
+	adev->family = AMDGPU_FAMILY_NV;
+	plane->dev = &adev->ddev;
+
+	KUNIT_EXPECT_TRUE(test,
+			  amdgpu_dm_plane_format_mod_supported(plane, DRM_FORMAT_XRGB8888,
+							       DRM_FORMAT_MOD_LINEAR));
+	KUNIT_EXPECT_TRUE(test,
+			  amdgpu_dm_plane_format_mod_supported(plane, DRM_FORMAT_XRGB8888,
+							       DRM_FORMAT_MOD_INVALID));
+
+	KUNIT_EXPECT_FALSE(test,
+			   amdgpu_dm_plane_format_mod_supported(plane, DRM_FORMAT_XRGB8888,
+								DRM_FORMAT_MOD_VENDOR_AMD));
+
+	listed_mod = AMD_FMT_MOD |
+		     AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_S_X) |
+		     AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX9) |
+		     AMD_FMT_MOD_SET(DCC, 1);
+	plane->modifiers = &listed_mod;
+	plane->modifier_count = 1;
+
+	KUNIT_EXPECT_FALSE(test,
+			   amdgpu_dm_plane_format_mod_supported(plane, DRM_FORMAT_NV12, listed_mod));
+}
+
+/**
+ * dm_test_fill_gfx12_plane_attributes_from_modifiers() - Verify GFX12 DCC mapping path.
+ * @test: KUnit test context.
+ *
+ * Verify if GFX12 modifier parsing enables DCC and sets expected DCC block mode.
+ */
+static void dm_test_fill_gfx12_plane_attributes_from_modifiers(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct amdgpu_framebuffer *afb;
+	struct plane_size plane_size = {0};
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	struct dm_test_dcc_cap_ctx ctx = {
+		.callback_ret = true,
+		.capable = true,
+		.output_independent_64b_blks = false,
+	};
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+
+	adev->family = AMDGPU_FAMILY_GC_12_0_0;
+	adev->dm.dc = dc;
+	adev->gfx.config.gb_addr_config_fields.num_pipes = 2;
+	adev->gfx.config.gb_addr_config_fields.num_banks = 4;
+	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 256;
+	adev->gfx.config.gb_addr_config_fields.num_se = 1;
+	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 2;
+	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1;
+	dc->cap_funcs.get_dcc_compression_cap = dm_test_get_dcc_compression_cap;
+	dm_test_dcc_ctx = &ctx;
+
+	afb->base.modifier = AMD_FMT_MOD |
+			     AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX12_64K_2D) |
+			     AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX12) |
+			     AMD_FMT_MOD_SET(DCC, 1) |
+			     AMD_FMT_MOD_SET(DCC_MAX_COMPRESSED_BLOCK, 1);
+	plane_size.surface_size.width = 1920;
+	plane_size.surface_size.height = 1080;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_fill_gfx12_attrs_from_modifiers(
+			adev, afb, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888,
+			ROTATION_ANGLE_0, &plane_size, &tiling_info, &dcc, &address),
+			0);
+	KUNIT_EXPECT_EQ(test, (int)tiling_info.gfxversion, (int)DcGfxAddr3);
+	KUNIT_EXPECT_TRUE(test, dcc.enable);
+	KUNIT_EXPECT_EQ(test, (int)dcc.dcc_ind_blk, (int)hubp_ind_block_128b);
+
+	dm_test_dcc_ctx = NULL;
+}
+
+/**
+ * dm_test_fill_gfx9_plane_attributes_from_modifiers() - Verify basic GFX9 linear modifier path.
+ * @test: KUnit test context.
+ *
+ * Verify if GFX9 linear modifier handling keeps DCC disabled.
+ */
+static void dm_test_fill_gfx9_plane_attributes_from_modifiers(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_framebuffer *afb;
+	struct plane_size plane_size = {0};
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+
+	adev->family = AMDGPU_FAMILY_NV;
+	adev->gfx.config.gb_addr_config_fields.num_pipes = 2;
+	adev->gfx.config.gb_addr_config_fields.num_banks = 4;
+	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 256;
+	adev->gfx.config.gb_addr_config_fields.num_se = 1;
+	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 2;
+	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1;
+	adev->gfx.config.gb_addr_config_fields.num_pkrs = 2;
+	adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 3, 0);
+
+	afb->base.modifier = DRM_FORMAT_MOD_LINEAR;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_fill_gfx9_attrs_from_modifiers(
+			adev, afb, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888,
+			ROTATION_ANGLE_0, &plane_size, &tiling_info, &dcc, &address),
+			0);
+	KUNIT_EXPECT_EQ(test, (int)tiling_info.gfxversion, (int)DcGfxVersion9);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.swizzle, 0U);
+	KUNIT_EXPECT_FALSE(test, dcc.enable);
+}
+
+/**
+ * dm_test_helper_check_state_viewport_reject() - Verify viewport outside screen rejects state.
+ * @test: KUnit test context.
+ *
+ * Verify if plane state is rejected when the viewport is outside display bounds.
+ */
+static void dm_test_helper_check_state_viewport_reject(struct kunit *test)
+{
+	struct drm_plane *plane;
+	struct drm_plane_state *state;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *new_crtc_state;
+	struct drm_framebuffer *fb;
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	new_crtc_state = kunit_kzalloc(test, sizeof(*new_crtc_state), GFP_KERNEL);
+	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+	KUNIT_ASSERT_NOT_NULL(test, new_crtc_state);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+
+	plane->type = DRM_PLANE_TYPE_OVERLAY;
+	state->plane = plane;
+	state->fb = fb;
+	state->crtc = crtc;
+	state->crtc_x = 200;
+	state->crtc_y = 0;
+	state->crtc_w = 100;
+	state->crtc_h = 100;
+	new_crtc_state->mode.crtc_hdisplay = 100;
+	new_crtc_state->mode.crtc_vdisplay = 100;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_helper_check_state(state, new_crtc_state), -EINVAL);
+}
+
+/**
+ * dm_test_validate_dcc_disabled_returns_success() - Verify disabled DCC is accepted.
+ * @test: KUnit test context.
+ *
+ * Verify if DCC validation succeeds when DCC is disabled.
+ */
+static void dm_test_validate_dcc_disabled_returns_success(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	struct plane_size plane_size = {0};
+
+	dm_test_init_validate_dcc_inputs(&adev, &dc, &tiling_info, &dcc, &address,
+					 &plane_size, test);
+	dcc.enable = 0;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_validate_dcc(adev, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888,
+			ROTATION_ANGLE_0, &tiling_info, &dcc,
+			&address, &plane_size),
+			0);
+}
+
+/**
+ * dm_test_validate_dcc_video_non_gfx12_fails() - Verify video format restriction on pre-GFX12.
+ * @test: KUnit test context.
+ *
+ * Verify if video format DCC validation fails on non-GFX12 devices.
+ */
+static void dm_test_validate_dcc_video_non_gfx12_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	struct plane_size plane_size = {0};
+
+	dm_test_init_validate_dcc_inputs(&adev, &dc, &tiling_info, &dcc, &address,
+					 &plane_size, test);
+	adev->family = AMDGPU_FAMILY_NV;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_validate_dcc(adev, SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr,
+			ROTATION_ANGLE_0, &tiling_info, &dcc,
+			&address, &plane_size),
+			-EINVAL);
+}
+
+/**
+ * dm_test_validate_dcc_missing_cap_func_fails() - Verify missing capability callback fails.
+ * @test: KUnit test context.
+ *
+ * Verify if validation fails when DCC capability callback is not provided.
+ */
+static void dm_test_validate_dcc_missing_cap_func_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	struct plane_size plane_size = {0};
+
+	dm_test_init_validate_dcc_inputs(&adev, &dc, &tiling_info, &dcc, &address,
+					 &plane_size, test);
+	dc->cap_funcs.get_dcc_compression_cap = NULL;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_validate_dcc(adev, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888,
+			ROTATION_ANGLE_0, &tiling_info, &dcc,
+			&address, &plane_size),
+			-EINVAL);
+}
+
+/**
+ * dm_test_validate_dcc_cap_callback_fails() - Verify callback failure path.
+ * @test: KUnit test context.
+ *
+ * Verify if validation fails when the DCC capability callback returns false.
+ */
+static void dm_test_validate_dcc_cap_callback_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	struct plane_size plane_size = {0};
+	enum surface_pixel_format format = SURFACE_PIXEL_FORMAT_GRPH_ARGB8888;
+	enum dc_rotation_angle rotation = ROTATION_ANGLE_0;
+	struct dm_test_dcc_cap_ctx ctx = {
+		.callback_ret = false,
+		.capable = true,
+	};
+	int ret;
+
+	dm_test_init_validate_dcc_inputs(&adev, &dc, &tiling_info, &dcc, &address,
+					 &plane_size, test);
+	dc->cap_funcs.get_dcc_compression_cap = dm_test_get_dcc_compression_cap;
+	dm_test_dcc_ctx = &ctx;
+
+	ret = amdgpu_dm_plane_validate_dcc(adev, format, rotation, &tiling_info,
+					   &dcc, &address, &plane_size);
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+	KUNIT_EXPECT_TRUE(test, ctx.called);
+
+	dm_test_dcc_ctx = NULL;
+}
+
+/**
+ * dm_test_validate_dcc_not_capable_fails() - Verify not-capable callback output.
+ * @test: KUnit test context.
+ *
+ * Verify if validation fails when the DCC capability callback reports that the
+ * surface is not DCC capable.
+ */
+static void dm_test_validate_dcc_not_capable_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	struct plane_size plane_size = {0};
+	enum surface_pixel_format format = SURFACE_PIXEL_FORMAT_GRPH_ARGB8888;
+	enum dc_rotation_angle rotation = ROTATION_ANGLE_0;
+	struct dm_test_dcc_cap_ctx ctx = {
+		.callback_ret = true,
+		.capable = false,
+	};
+	int ret;
+
+	dm_test_init_validate_dcc_inputs(&adev, &dc, &tiling_info, &dcc, &address,
+					 &plane_size, test);
+	dc->cap_funcs.get_dcc_compression_cap = dm_test_get_dcc_compression_cap;
+	dm_test_dcc_ctx = &ctx;
+
+	ret = amdgpu_dm_plane_validate_dcc(adev, format, rotation, &tiling_info,
+					   &dcc, &address, &plane_size);
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+	KUNIT_EXPECT_TRUE(test, ctx.called);
+
+	dm_test_dcc_ctx = NULL;
+}
+
+/**
+ * dm_test_validate_dcc_success_and_scan_mapping() - Verify success path and rotation-to-scan mapping.
+ * @test: KUnit test context.
+ *
+ * Verify if DCC validation succeeds and rotation-to-scan mapping is correct.
+ */
+static void dm_test_validate_dcc_success_and_scan_mapping(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	struct plane_size plane_size = {0};
+	struct dm_test_dcc_cap_ctx ctx = {
+		.callback_ret = true,
+		.capable = true,
+		.output_independent_64b_blks = true,
+	};
+
+	dm_test_init_validate_dcc_inputs(&adev, &dc, &tiling_info, &dcc, &address,
+					 &plane_size, test);
+	dc->cap_funcs.get_dcc_compression_cap = dm_test_get_dcc_compression_cap;
+	dm_test_dcc_ctx = &ctx;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_validate_dcc(adev, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888,
+			ROTATION_ANGLE_90, &tiling_info, &dcc,
+			&address, &plane_size),
+			0);
+	KUNIT_EXPECT_TRUE(test, ctx.called);
+	KUNIT_EXPECT_EQ(test, (int)ctx.captured_input.scan, (int)SCAN_DIRECTION_VERTICAL);
+	KUNIT_EXPECT_EQ(test, (int)ctx.captured_input.format,
+			(int)SURFACE_PIXEL_FORMAT_GRPH_ARGB8888);
+
+	dm_test_dcc_ctx = NULL;
+}
+
+/**
+ * dm_test_validate_dcc_independent_64b_mismatch_fails() - Verify 64B compatibility check.
+ * @test: KUnit test context.
+ *
+ * Verify if validation fails when independent_64b_blks values do not match.
+ */
+static void dm_test_validate_dcc_independent_64b_mismatch_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	struct plane_size plane_size = {0};
+	struct dm_test_dcc_cap_ctx ctx = {
+		.callback_ret = true,
+		.capable = true,
+		.output_independent_64b_blks = true,
+	};
+
+	dm_test_init_validate_dcc_inputs(&adev, &dc, &tiling_info, &dcc, &address,
+					 &plane_size, test);
+	dcc.independent_64b_blks = 0;
+	dc->cap_funcs.get_dcc_compression_cap = dm_test_get_dcc_compression_cap;
+	dm_test_dcc_ctx = &ctx;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_validate_dcc(adev, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888,
+			ROTATION_ANGLE_0, &tiling_info, &dcc,
+			&address, &plane_size),
+			-EINVAL);
+
+	dm_test_dcc_ctx = NULL;
+}
+
+/**
+ * dm_test_add_modifier_appends_value() - Verify one modifier append.
+ * @test: KUnit test context.
+ *
+ * Verify if a modifier is appended and size is updated.
+ */
+static void dm_test_add_modifier_appends_value(struct kunit *test)
+{
+	uint64_t size = 0;
+	uint64_t cap = 2;
+	uint64_t *mods = kmalloc_array(cap, sizeof(*mods), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, mods);
+
+	amdgpu_dm_plane_add_modifier(&mods, &size, &cap, 0x1234ULL);
+
+	KUNIT_ASSERT_NOT_NULL(test, mods);
+	KUNIT_EXPECT_EQ(test, size, 1ULL);
+	KUNIT_EXPECT_EQ(test, cap, 2ULL);
+	KUNIT_EXPECT_EQ(test, mods[0], 0x1234ULL);
+
+	kfree(mods);
+}
+
+/**
+ * dm_test_add_modifier_grows_capacity() - Verify add triggers growth and preserves old data.
+ * @test: KUnit test context.
+ *
+ * Verify if modifier array growth keeps old data and appends new data.
+ */
+static void dm_test_add_modifier_grows_capacity(struct kunit *test)
+{
+	uint64_t size = 1;
+	uint64_t cap = 1;
+	uint64_t *mods = kmalloc_array(cap, sizeof(*mods), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, mods);
+	mods[0] = 0xAAULL;
+
+	amdgpu_dm_plane_add_modifier(&mods, &size, &cap, 0xBBULL);
+
+	KUNIT_ASSERT_NOT_NULL(test, mods);
+	KUNIT_EXPECT_EQ(test, cap, 2ULL);
+	KUNIT_EXPECT_EQ(test, size, 2ULL);
+	KUNIT_EXPECT_EQ(test, mods[0], 0xAAULL);
+	KUNIT_EXPECT_EQ(test, mods[1], 0xBBULL);
+
+	kfree(mods);
+}
+
+/**
+ * dm_test_add_modifier_noop_when_mods_null() - Verify helper is a no-op on NULL mods list.
+ * @test: KUnit test context.
+ *
+ * Verify if add_modifier does nothing when the modifier list is NULL.
+ */
+static void dm_test_add_modifier_noop_when_mods_null(struct kunit *test)
+{
+	uint64_t size = 3;
+	uint64_t cap = 7;
+	uint64_t *mods = NULL;
+
+	amdgpu_dm_plane_add_modifier(&mods, &size, &cap, 0x55ULL);
+
+	KUNIT_EXPECT_PTR_EQ(test, mods, NULL);
+	KUNIT_EXPECT_EQ(test, size, 3ULL);
+	KUNIT_EXPECT_EQ(test, cap, 7ULL);
+}
+
+/**
+ * dm_test_fill_gfx8_tiling_info_2d_tiled() - Verify GFX8 2D tiled flag parsing.
+ * @test: KUnit test context.
+ *
+ * Verify if 2D tiled GFX8 flags populate expected tiling fields.
+ */
+static void dm_test_fill_gfx8_tiling_info_2d_tiled(struct kunit *test)
+{
+	struct dc_tiling_info tiling_info = {0};
+	uint64_t tiling_flags = 0;
+
+	tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, DC_ARRAY_2D_TILED_THIN1);
+	tiling_flags |= AMDGPU_TILING_SET(BANK_WIDTH, 2);
+	tiling_flags |= AMDGPU_TILING_SET(BANK_HEIGHT, 1);
+	tiling_flags |= AMDGPU_TILING_SET(MACRO_TILE_ASPECT, 3);
+	tiling_flags |= AMDGPU_TILING_SET(TILE_SPLIT, 4);
+	tiling_flags |= AMDGPU_TILING_SET(NUM_BANKS, 2);
+	tiling_flags |= AMDGPU_TILING_SET(PIPE_CONFIG, 7);
+
+	amdgpu_dm_plane_fill_gfx8_tiling_info_from_flags(&tiling_info, tiling_flags);
+
+	KUNIT_EXPECT_EQ(test, (int)tiling_info.gfxversion, (int)DcGfxVersion8);
+	KUNIT_EXPECT_EQ(test, (int)tiling_info.gfx8.array_mode, (int)DC_ARRAY_2D_TILED_THIN1);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.bank_width, 2U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.bank_height, 1U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.tile_aspect, 3U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.tile_split, 4U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.num_banks, 2U);
+	KUNIT_EXPECT_EQ(test, (int)tiling_info.gfx8.tile_mode,
+			(int)DC_ADDR_SURF_MICRO_TILING_DISPLAY);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.pipe_config, 7U);
+}
+
+/**
+ * dm_test_fill_gfx8_tiling_info_1d_tiled() - Verify GFX8 1D tiled flag parsing.
+ * @test: KUnit test context.
+ *
+ * Verify if 1D tiled GFX8 flags populate array mode and pipe config.
+ */
+static void dm_test_fill_gfx8_tiling_info_1d_tiled(struct kunit *test)
+{
+	struct dc_tiling_info tiling_info = {0};
+	uint64_t tiling_flags = 0;
+
+	tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, DC_ARRAY_1D_TILED_THIN1);
+	tiling_flags |= AMDGPU_TILING_SET(PIPE_CONFIG, 5);
+
+	amdgpu_dm_plane_fill_gfx8_tiling_info_from_flags(&tiling_info, tiling_flags);
+
+	KUNIT_EXPECT_EQ(test, (int)tiling_info.gfx8.array_mode, (int)DC_ARRAY_1D_TILED_THIN1);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.pipe_config, 5U);
+}
+
+/**
+ * dm_test_fill_gfx8_tiling_info_other_mode() - Verify non-1D/non-2D mode handling.
+ * @test: KUnit test context.
+ *
+ * Verify if unsupported array mode keeps preset fields and updates pipe config.
+ */
+static void dm_test_fill_gfx8_tiling_info_other_mode(struct kunit *test)
+{
+	struct dc_tiling_info tiling_info = {0};
+	uint64_t tiling_flags = 0;
+
+	tiling_info.gfxversion = 0x7f;
+	tiling_info.gfx8.array_mode = 0x7f;
+	tiling_info.gfx8.tile_mode = 0x7f;
+	tiling_info.gfx8.num_banks = 0x7f;
+
+	tiling_flags |= AMDGPU_TILING_SET(PIPE_CONFIG, 6);
+
+	amdgpu_dm_plane_fill_gfx8_tiling_info_from_flags(&tiling_info, tiling_flags);
+
+	KUNIT_EXPECT_EQ(test, tiling_info.gfxversion, 0x7f);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.array_mode, 0x7f);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.tile_mode, 0x7f);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.num_banks, 0x7f);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.pipe_config, 6U);
+}
+
+/**
+ * dm_test_fill_gfx9_tiling_info_from_device_pre_10_3() - Verify GFX9 field copy before 10.3.
+ * @test: KUnit test context.
+ *
+ * Verify if pre-10.3 device fields are copied and existing num_pkrs is kept.
+ */
+static void dm_test_fill_gfx9_tiling_info_from_device_pre_10_3(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_tiling_info tiling_info = {0};
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	adev->gfx.config.gb_addr_config_fields.num_pipes = 4;
+	adev->gfx.config.gb_addr_config_fields.num_banks = 8;
+	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 256;
+	adev->gfx.config.gb_addr_config_fields.num_se = 2;
+	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1;
+	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 2;
+	adev->gfx.config.gb_addr_config_fields.num_pkrs = 3;
+	adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 2, 9);
+
+	tiling_info.gfx9.num_pkrs = 0x5a;
+
+	amdgpu_dm_plane_fill_gfx9_tiling_info_from_device(adev, &tiling_info);
+
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_pipes, 4U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_banks, 8U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.pipe_interleave, 256U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_shader_engines, 2U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.max_compressed_frags, 1U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_rb_per_se, 2U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.shaderEnable, 1U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_pkrs, 0x5aU);
+}
+
+/**
+ * dm_test_fill_gfx9_tiling_info_from_device_10_3_plus() - Verify num_pkrs update on 10.3+.
+ * @test: KUnit test context.
+ *
+ * Verify if 10.3+ device fields are copied and num_pkrs is updated.
+ */
+static void dm_test_fill_gfx9_tiling_info_from_device_10_3_plus(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_tiling_info tiling_info = {0};
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	adev->gfx.config.gb_addr_config_fields.num_pipes = 2;
+	adev->gfx.config.gb_addr_config_fields.num_banks = 4;
+	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 128;
+	adev->gfx.config.gb_addr_config_fields.num_se = 1;
+	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 2;
+	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1;
+	adev->gfx.config.gb_addr_config_fields.num_pkrs = 6;
+	adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 3, 0);
+
+	amdgpu_dm_plane_fill_gfx9_tiling_info_from_device(adev, &tiling_info);
+
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_pipes, 2U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_banks, 4U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.pipe_interleave, 128U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_shader_engines, 1U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.max_compressed_frags, 2U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_rb_per_se, 1U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.shaderEnable, 1U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_pkrs, 6U);
+}
+
+/**
+ * dm_test_fill_gfx9_tiling_info_from_modifier_linear() - Verify non-AMD modifier keeps device values.
+ * @test: KUnit test context.
+ *
+ * Verify if linear modifier path keeps values from device configuration.
+ */
+static void dm_test_fill_gfx9_tiling_info_from_modifier_linear(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_tiling_info tiling_info = {0};
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	adev->family = AMDGPU_FAMILY_NV;
+	adev->gfx.config.gb_addr_config_fields.num_pipes = 4;
+	adev->gfx.config.gb_addr_config_fields.num_banks = 8;
+	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 256;
+	adev->gfx.config.gb_addr_config_fields.num_se = 2;
+	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1;
+	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 2;
+	adev->gfx.config.gb_addr_config_fields.num_pkrs = 3;
+	adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 3, 0);
+
+	amdgpu_dm_plane_fill_gfx9_tiling_info_from_modifier(adev, &tiling_info,
+							    DRM_FORMAT_MOD_LINEAR);
+
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_pipes, 4U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_banks, 8U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.pipe_interleave, 256U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_shader_engines, 2U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.max_compressed_frags, 1U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_rb_per_se, 2U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.shaderEnable, 1U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_pkrs, 3U);
+}
+
+/**
+ * dm_test_fill_gfx9_tiling_info_from_modifier_pre_nv() - Verify AMD modifier updates banks on pre-NV.
+ * @test: KUnit test context.
+ *
+ * Verify if AMD modifier updates pre-NV pipe, engine, and bank fields.
+ */
+static void dm_test_fill_gfx9_tiling_info_from_modifier_pre_nv(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_tiling_info tiling_info = {0};
+	uint64_t modifier;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	adev->family = AMDGPU_FAMILY_RV;
+	adev->gfx.config.gb_addr_config_fields.num_pipes = 4;
+	adev->gfx.config.gb_addr_config_fields.num_banks = 16;
+	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 256;
+	adev->gfx.config.gb_addr_config_fields.num_se = 2;
+	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1;
+	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 2;
+	adev->gfx.config.gb_addr_config_fields.num_pkrs = 7;
+	adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 2, 9);
+
+	tiling_info.gfx9.num_pkrs = 0x5a;
+
+	modifier = AMD_FMT_MOD |
+		    AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_S_X) |
+		    AMD_FMT_MOD_SET(PIPE_XOR_BITS, 7) |
+		    AMD_FMT_MOD_SET(BANK_XOR_BITS, 3) |
+		    AMD_FMT_MOD_SET(PACKERS, 2);
+
+	amdgpu_dm_plane_fill_gfx9_tiling_info_from_modifier(adev, &tiling_info, modifier);
+
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_pipes, 32U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_shader_engines, 4U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_banks, 8U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_pkrs, 0x5aU);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.shaderEnable, 1U);
+}
+
+/**
+ * dm_test_fill_gfx9_tiling_info_from_modifier_nv() - Verify AMD modifier updates packers on NV+.
+ * @test: KUnit test context.
+ *
+ * Verify if AMD modifier updates NV+ pipe, engine, and packer fields.
+ */
+static void dm_test_fill_gfx9_tiling_info_from_modifier_nv(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_tiling_info tiling_info = {0};
+	uint64_t modifier;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	adev->family = AMDGPU_FAMILY_NV;
+	adev->gfx.config.gb_addr_config_fields.num_pipes = 2;
+	adev->gfx.config.gb_addr_config_fields.num_banks = 9;
+	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 128;
+	adev->gfx.config.gb_addr_config_fields.num_se = 1;
+	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 2;
+	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1;
+	adev->gfx.config.gb_addr_config_fields.num_pkrs = 2;
+	adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 3, 0);
+
+	modifier = AMD_FMT_MOD |
+		    AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_S_X) |
+		    AMD_FMT_MOD_SET(PIPE_XOR_BITS, 6) |
+		    AMD_FMT_MOD_SET(BANK_XOR_BITS, 2) |
+		    AMD_FMT_MOD_SET(PACKERS, 3);
+
+	amdgpu_dm_plane_fill_gfx9_tiling_info_from_modifier(adev, &tiling_info, modifier);
+
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_pipes, 32U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_shader_engines, 2U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_banks, 9U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_pkrs, 8U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.shaderEnable, 1U);
+}
+
+/**
+ * dm_test_get_format_info() - Verify modifier-based format info lookup.
+ * @test: KUnit test context.
+ *
+ * Verify if non-AMD modifiers return NULL and AMD DCC modifiers resolve a
+ * dedicated format info structure.
+ */
+static void dm_test_get_format_info(struct kunit *test)
+{
+	u64 dcc_mod = AMD_FMT_MOD |
+		      AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX9) |
+		      AMD_FMT_MOD_SET(DCC, 1);
+	const struct drm_format_info *format_info;
+
+	KUNIT_EXPECT_PTR_EQ(test,
+			    (void *)amdgpu_dm_plane_get_format_info(DRM_FORMAT_XRGB8888,
+								    DRM_FORMAT_MOD_LINEAR),
+			    NULL);
+	format_info = amdgpu_dm_plane_get_format_info(DRM_FORMAT_XRGB8888, dcc_mod);
+	KUNIT_EXPECT_NOT_NULL(test, format_info);
+}
+
+/**
+ * dm_test_fill_blending_global_alpha_dcn42() - Verify DCN 4.2 alpha scaling.
+ * @test: KUnit test context.
+ *
+ * Verify if DCN 4.2 scales the 16-bit DRM alpha down by 4 bits instead of 8.
+ */
+static void dm_test_fill_blending_global_alpha_dcn42(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_plane *plane;
+	struct drm_plane_state *state;
+	bool per_pixel_alpha;
+	bool pre_multiplied_alpha;
+	bool global_alpha;
+	int global_alpha_value;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(4, 2, 0);
+	plane->dev = &adev->ddev;
+	state->plane = plane;
+	state->pixel_blend_mode = DRM_MODE_BLEND_PIXEL_NONE;
+	state->alpha = 0x8000;
+
+	amdgpu_dm_plane_fill_blending_from_plane_state(state,
+						       &per_pixel_alpha,
+						       &pre_multiplied_alpha,
+						       &global_alpha,
+						       &global_alpha_value);
+
+	KUNIT_EXPECT_TRUE(test, global_alpha);
+	KUNIT_EXPECT_EQ(test, global_alpha_value, 0x800);
+}
+
+static void dm_test_expect_mods_terminated(struct kunit *test, struct amdgpu_device *adev)
+{
+	u64 *mods = NULL;
+	int ret;
+	int i;
+
+	ret = amdgpu_dm_plane_get_plane_modifiers(adev, DRM_PLANE_TYPE_PRIMARY, &mods);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	KUNIT_ASSERT_NOT_NULL(test, mods);
+
+	for (i = 0; mods[i] != DRM_FORMAT_MOD_INVALID; i++)
+		;
+
+	KUNIT_EXPECT_GT(test, i, 0);
+	KUNIT_EXPECT_EQ(test, mods[i - 1], DRM_FORMAT_MOD_LINEAR);
+	kfree(mods);
+}
+
+static bool dm_test_mods_contain(const u64 *mods, u64 expected)
+{
+	int i;
+
+	for (i = 0; mods[i] != DRM_FORMAT_MOD_INVALID; i++) {
+		if (mods[i] == expected)
+			return true;
+	}
+
+	return false;
+}
+
+static u64 *dm_test_get_primary_mods(struct kunit *test, struct amdgpu_device *adev)
+{
+	u64 *mods = NULL;
+	int ret;
+
+	ret = amdgpu_dm_plane_get_plane_modifiers(adev, DRM_PLANE_TYPE_PRIMARY, &mods);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	KUNIT_ASSERT_NOT_NULL(test, mods);
+
+	return mods;
+}
+
+static int dm_test_gfx9_attrs(struct amdgpu_device *adev,
+			      const struct amdgpu_framebuffer *afb,
+				      const struct plane_size *plane_size,
+				      struct dc_tiling_info *tiling_info,
+				      struct dc_plane_dcc_param *dcc,
+				      struct dc_plane_address *address)
+{
+	return amdgpu_dm_plane_fill_gfx9_attrs_from_modifiers(adev,
+		afb, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888, ROTATION_ANGLE_0,
+		plane_size, tiling_info, dcc, address);
+}
+
+static int dm_test_gfx12_attrs(struct amdgpu_device *adev,
+			       const struct amdgpu_framebuffer *afb,
+				       const struct plane_size *plane_size,
+				       struct dc_tiling_info *tiling_info,
+				       struct dc_plane_dcc_param *dcc,
+				       struct dc_plane_address *address)
+{
+	return amdgpu_dm_plane_fill_gfx12_attrs_from_modifiers(adev,
+		afb, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888, ROTATION_ANGLE_0,
+		plane_size, tiling_info, dcc, address);
+}
+
+static int dm_test_plane_attrs(struct amdgpu_device *adev,
+			       const struct amdgpu_framebuffer *afb,
+			       enum surface_pixel_format format,
+			       struct dc_tiling_info *tiling_info,
+			       struct plane_size *plane_size,
+			       struct dc_plane_dcc_param *dcc,
+			       struct dc_plane_address *address)
+{
+	return amdgpu_dm_plane_fill_plane_buffer_attributes(adev, afb, format,
+		ROTATION_ANGLE_0, 0, tiling_info, plane_size, dcc, address,
+		false);
+}
+
+static int dm_test_video_attrs(struct amdgpu_device *adev,
+			       const struct amdgpu_framebuffer *afb,
+			       struct dc_tiling_info *tiling_info,
+			       struct plane_size *plane_size,
+			       struct dc_plane_dcc_param *dcc,
+			       struct dc_plane_address *address)
+{
+	return dm_test_plane_attrs(adev, afb, SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr,
+		tiling_info, plane_size, dcc, address);
+}
+
+static int dm_test_graphics_attrs(struct amdgpu_device *adev,
+				  const struct amdgpu_framebuffer *afb,
+				  struct dc_tiling_info *tiling_info,
+				  struct plane_size *plane_size,
+				  struct dc_plane_dcc_param *dcc,
+				  struct dc_plane_address *address)
+{
+	return dm_test_plane_attrs(adev, afb, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888,
+		tiling_info, plane_size, dcc, address);
+}
+
+static void dm_test_setup_gfx9_dcc_device(struct amdgpu_device *adev,
+					  struct dc *dc,
+					  struct dm_test_dcc_cap_ctx *ctx,
+					  bool output_independent_64b_blks)
+{
+	adev->family = AMDGPU_FAMILY_NV;
+	adev->dm.dc = dc;
+	adev->gfx.config.gb_addr_config_fields.num_pipes = 2;
+	adev->gfx.config.gb_addr_config_fields.num_pkrs = 2;
+	adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 3, 0);
+	dc->cap_funcs.get_dcc_compression_cap = dm_test_get_dcc_compression_cap;
+	ctx->callback_ret = true;
+	ctx->capable = true;
+	ctx->output_independent_64b_blks = output_independent_64b_blks;
+	dm_test_dcc_ctx = ctx;
+}
+
+static u64 dm_test_gfx9_dcc_modifier(u64 tile_version, bool independent_64b_blks,
+				     bool independent_128b_blks)
+{
+	return AMD_FMT_MOD |
+	       AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_S_X) |
+	       AMD_FMT_MOD_SET(TILE_VERSION, tile_version) |
+	       AMD_FMT_MOD_SET(DCC, 1) |
+	       AMD_FMT_MOD_SET(DCC_INDEPENDENT_64B, independent_64b_blks) |
+	       AMD_FMT_MOD_SET(DCC_INDEPENDENT_128B, independent_128b_blks);
+}
+
+static void dm_test_setup_gfx11_device(struct amdgpu_device *adev,
+				       struct dm_test_gfx11_reg_ctx *ctx,
+				       u32 num_pkrs_log2, u32 num_pipes_log2)
+{
+	ctx->gb_addr_config =
+		REG_SET_FIELD(0, GB_ADDR_CONFIG, NUM_PKRS, num_pkrs_log2) |
+		REG_SET_FIELD(0, GB_ADDR_CONFIG, NUM_PIPES, num_pipes_log2);
+	ctx->gc_reg_offsets[regGB_ADDR_CONFIG_BASE_IDX] = 0;
+	ctx->expected_reg = ctx->gc_reg_offsets[regGB_ADDR_CONFIG_BASE_IDX] +
+			    regGB_ADDR_CONFIG;
+	dm_test_gfx11_reg_ctx = ctx;
+
+	adev->family = AMDGPU_FAMILY_GC_11_0_0;
+	adev->reg_offset[GC_HWIP][0] = ctx->gc_reg_offsets;
+	adev->gfx.rlc.reg_funcs = &dm_test_gfx11_reg_funcs;
+}
+
+static u64 dm_test_gfx11_dcc_best_modifier(u32 pipe_xor_bits, u32 pkrs, u32 tile)
+{
+	return AMD_FMT_MOD |
+	       AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX11) |
+	       AMD_FMT_MOD_SET(PIPE_XOR_BITS, pipe_xor_bits) |
+	       AMD_FMT_MOD_SET(TILE, tile) |
+	       AMD_FMT_MOD_SET(PACKERS, pkrs) |
+	       AMD_FMT_MOD_SET(DCC, 1) |
+	       AMD_FMT_MOD_SET(DCC_INDEPENDENT_128B, 1) |
+	       AMD_FMT_MOD_SET(DCC_MAX_COMPRESSED_BLOCK, AMD_FMT_MOD_DCC_BLOCK_128B);
+}
+
+static u64 dm_test_gfx11_dcc_4k_modifier(u32 pipe_xor_bits, u32 pkrs, u32 tile)
+{
+	return AMD_FMT_MOD |
+	       AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX11) |
+	       AMD_FMT_MOD_SET(PIPE_XOR_BITS, pipe_xor_bits) |
+	       AMD_FMT_MOD_SET(TILE, tile) |
+	       AMD_FMT_MOD_SET(PACKERS, pkrs) |
+	       AMD_FMT_MOD_SET(DCC, 1) |
+	       AMD_FMT_MOD_SET(DCC_INDEPENDENT_64B, 1) |
+	       AMD_FMT_MOD_SET(DCC_INDEPENDENT_128B, 1) |
+	       AMD_FMT_MOD_SET(DCC_MAX_COMPRESSED_BLOCK, AMD_FMT_MOD_DCC_BLOCK_64B);
+}
+
+/**
+ * dm_test_get_plane_formats_overlay_universal_cap() - Verify universal overlay.
+ * @test: KUnit test context.
+ *
+ * Verify if an overlay plane with a DCN universal plane cap reports the RGB
+ * format list instead of the overlay-only list.
+ */
+static void dm_test_get_plane_formats_overlay_universal_cap(struct kunit *test)
+{
+	struct drm_plane *plane;
+	struct dc_plane_cap *cap;
+	u32 formats[32] = {0};
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	cap = kunit_kzalloc(test, sizeof(*cap), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, cap);
+
+	plane->type = DRM_PLANE_TYPE_OVERLAY;
+	cap->type = DC_PLANE_TYPE_DCN_UNIVERSAL;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_get_plane_formats(plane, cap, formats, 32),
+			14);
+}
+
+/**
+ * dm_test_get_plane_modifiers_gfx9() - Verify GFX9 modifier list generation.
+ * @test: KUnit test context.
+ *
+ * Verify if the GFX9 family produces a non-empty modifier list terminated by
+ * LINEAR and INVALID entries.
+ */
+static void dm_test_get_plane_modifiers_gfx9(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	adev->family = AMDGPU_FAMILY_AI;
+	adev->gfx.config.gb_addr_config_fields.num_pipes = 4;
+	adev->gfx.config.gb_addr_config_fields.num_banks = 8;
+	adev->gfx.config.gb_addr_config_fields.num_se = 2;
+	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 2;
+
+	dm_test_expect_mods_terminated(test, adev);
+}
+
+/**
+ * dm_test_get_plane_modifiers_rv() - Verify RV modifier list generation.
+ * @test: KUnit test context.
+ *
+ * Verify if pre-Raven2 RV devices add RV-specific S-swizzle modifiers and
+ * non-constant-encode DCC modifiers.
+ */
+static void dm_test_get_plane_modifiers_rv(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	u64 *mods;
+	u64 dcc_mod;
+	u64 s_x_mod;
+	u64 s_mod;
+	int pipes = 2;
+	int pipe_xor_bits = 3;
+	int bank_xor_bits = 2;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	adev->family = AMDGPU_FAMILY_RV;
+	adev->asic_type = CHIP_RAVEN;
+	adev->external_rev_id = 0x80;
+	adev->gfx.config.gb_addr_config_fields.num_pipes = 4;
+	adev->gfx.config.gb_addr_config_fields.num_banks = 4;
+	adev->gfx.config.gb_addr_config_fields.num_se = 2;
+	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 2;
+
+	mods = dm_test_get_primary_mods(test, adev);
+	dcc_mod = AMD_FMT_MOD |
+		  AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_S_X) |
+		  AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX9) |
+		  AMD_FMT_MOD_SET(PIPE_XOR_BITS, pipe_xor_bits) |
+		  AMD_FMT_MOD_SET(BANK_XOR_BITS, bank_xor_bits) |
+		  AMD_FMT_MOD_SET(DCC, 1) |
+		  AMD_FMT_MOD_SET(DCC_INDEPENDENT_64B, 1) |
+		  AMD_FMT_MOD_SET(DCC_MAX_COMPRESSED_BLOCK, AMD_FMT_MOD_DCC_BLOCK_64B) |
+		  AMD_FMT_MOD_SET(DCC_CONSTANT_ENCODE, 0);
+	s_x_mod = AMD_FMT_MOD |
+		  AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_S_X) |
+		  AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX9) |
+		  AMD_FMT_MOD_SET(PIPE_XOR_BITS, pipe_xor_bits) |
+		  AMD_FMT_MOD_SET(BANK_XOR_BITS, bank_xor_bits);
+	s_mod = AMD_FMT_MOD |
+		AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_S) |
+		AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX9);
+
+	KUNIT_EXPECT_TRUE(test, dm_test_mods_contain(mods, dcc_mod));
+	KUNIT_EXPECT_TRUE(test, dm_test_mods_contain(mods, dcc_mod |
+							     AMD_FMT_MOD_SET(DCC_RETILE, 1) |
+							     AMD_FMT_MOD_SET(RB, 2) |
+							     AMD_FMT_MOD_SET(PIPE, pipes)));
+	KUNIT_EXPECT_TRUE(test, dm_test_mods_contain(mods, s_x_mod));
+	KUNIT_EXPECT_TRUE(test, dm_test_mods_contain(mods, s_mod));
+
+	kfree(mods);
+}
+
+/**
+ * dm_test_get_plane_modifiers_rv_constant_encode() - Verify Raven2+ modifiers.
+ * @test: KUnit test context.
+ *
+ * Verify if Raven2 and later RV devices add the constant-encode modifier
+ * variants.
+ */
+static void dm_test_get_plane_modifiers_rv_constant_encode(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	u64 *mods;
+	u64 dcc_mod;
+	int pipes = 2;
+	int pipe_xor_bits = 3;
+	int bank_xor_bits = 2;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	adev->family = AMDGPU_FAMILY_RV;
+	adev->asic_type = CHIP_RAVEN;
+	adev->external_rev_id = 0x81;
+	adev->gfx.config.gb_addr_config_fields.num_pipes = 4;
+	adev->gfx.config.gb_addr_config_fields.num_banks = 4;
+	adev->gfx.config.gb_addr_config_fields.num_se = 2;
+	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 2;
+
+	mods = dm_test_get_primary_mods(test, adev);
+	dcc_mod = AMD_FMT_MOD |
+		  AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_S_X) |
+		  AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX9) |
+		  AMD_FMT_MOD_SET(PIPE_XOR_BITS, pipe_xor_bits) |
+		  AMD_FMT_MOD_SET(BANK_XOR_BITS, bank_xor_bits) |
+		  AMD_FMT_MOD_SET(DCC, 1) |
+		  AMD_FMT_MOD_SET(DCC_INDEPENDENT_64B, 1) |
+		  AMD_FMT_MOD_SET(DCC_MAX_COMPRESSED_BLOCK, AMD_FMT_MOD_DCC_BLOCK_64B) |
+		  AMD_FMT_MOD_SET(DCC_CONSTANT_ENCODE, 1);
+
+	KUNIT_EXPECT_TRUE(test, dm_test_mods_contain(mods, dcc_mod));
+	KUNIT_EXPECT_TRUE(test, dm_test_mods_contain(mods, dcc_mod |
+							     AMD_FMT_MOD_SET(DCC_RETILE, 1) |
+							     AMD_FMT_MOD_SET(RB, 2) |
+							     AMD_FMT_MOD_SET(PIPE, pipes)));
+
+	kfree(mods);
+}
+
+/**
+ * dm_test_get_plane_modifiers_gfx10_1() - Verify GFX10.1 modifier list generation.
+ * @test: KUnit test context.
+ *
+ * Verify if a pre-10.3 NV family device dispatches to the GFX10.1 modifier
+ * builder and produces a terminated list.
+ */
+static void dm_test_get_plane_modifiers_gfx10_1(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	adev->family = AMDGPU_FAMILY_NV;
+	adev->gfx.config.gb_addr_config_fields.num_pipes = 4;
+	adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 1, 0);
+
+	dm_test_expect_mods_terminated(test, adev);
+}
+
+/**
+ * dm_test_get_plane_modifiers_gfx10_3() - Verify GFX10.3 modifier list generation.
+ * @test: KUnit test context.
+ *
+ * Verify if a 10.3+ NV family device dispatches to the GFX10.3 modifier
+ * builder and produces a terminated list.
+ */
+static void dm_test_get_plane_modifiers_gfx10_3(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	adev->family = AMDGPU_FAMILY_NV;
+	adev->gfx.config.gb_addr_config_fields.num_pipes = 4;
+	adev->gfx.config.gb_addr_config_fields.num_pkrs = 2;
+	adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 3, 0);
+
+	dm_test_expect_mods_terminated(test, adev);
+}
+
+/**
+ * dm_test_get_plane_modifiers_gfx11_64k_first() - Verify GFX11 small-pipe order.
+ * @test: KUnit test context.
+ *
+ * Verify if GFX11 modifier generation reads GB_ADDR_CONFIG through the RLC
+ * register callback and prefers 64K_R_X when the pipe count is 16 or lower.
+ */
+static void dm_test_get_plane_modifiers_gfx11_64k_first(struct kunit *test)
+{
+	struct dm_test_gfx11_reg_ctx ctx = {0};
+	struct amdgpu_device *adev;
+	u64 *mods;
+	u32 pipe_xor_bits = 4;
+	u32 pkrs = 1;
+	u32 tile = AMD_FMT_MOD_TILE_GFX9_64K_R_X;
+	u64 dcc_best;
+	u64 dcc_4k;
+	u64 d_mod;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	dm_test_setup_gfx11_device(adev, &ctx, pkrs, pipe_xor_bits);
+
+	mods = dm_test_get_primary_mods(test, adev);
+	dcc_best = dm_test_gfx11_dcc_best_modifier(pipe_xor_bits, pkrs, tile);
+	dcc_4k = dm_test_gfx11_dcc_4k_modifier(pipe_xor_bits, pkrs, tile);
+	d_mod = AMD_FMT_MOD |
+		AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX11) |
+		AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_D);
+
+	KUNIT_EXPECT_TRUE(test, ctx.called);
+	KUNIT_EXPECT_EQ(test, ctx.captured_reg, ctx.expected_reg);
+	KUNIT_EXPECT_EQ(test, ctx.captured_acc_flags, 0U);
+	KUNIT_EXPECT_EQ(test, ctx.captured_hwip, (u32)GC_HWIP);
+	KUNIT_EXPECT_EQ(test, ctx.captured_xcc_id, 0U);
+	KUNIT_EXPECT_EQ(test, mods[0], dcc_best);
+	KUNIT_EXPECT_TRUE(test, dm_test_mods_contain(mods, dcc_4k));
+	KUNIT_EXPECT_TRUE(test,
+			  dm_test_mods_contain(mods,
+					       dcc_best | AMD_FMT_MOD_SET(DCC_RETILE, 1)));
+	KUNIT_EXPECT_TRUE(test, dm_test_mods_contain(mods, d_mod));
+
+	dm_test_gfx11_reg_ctx = NULL;
+	kfree(mods);
+}
+
+/**
+ * dm_test_get_plane_modifiers_gfx11_256k_first() - Verify GFX11 large-pipe order.
+ * @test: KUnit test context.
+ *
+ * Verify if GFX11 modifier generation prefers 256K_R_X when more than 16 pipes
+ * are reported by GB_ADDR_CONFIG.
+ */
+static void dm_test_get_plane_modifiers_gfx11_256k_first(struct kunit *test)
+{
+	struct dm_test_gfx11_reg_ctx ctx = {0};
+	struct amdgpu_device *adev;
+	u64 *mods;
+	u32 pipe_xor_bits = 5;
+	u32 pkrs = 2;
+	u32 tile = AMD_FMT_MOD_TILE_GFX11_256K_R_X;
+	u32 fallback_tile = AMD_FMT_MOD_TILE_GFX9_64K_R_X;
+	u64 dcc_best;
+	u64 fallback_dcc_best = dm_test_gfx11_dcc_best_modifier(pipe_xor_bits, pkrs,
+								  fallback_tile);
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	dm_test_setup_gfx11_device(adev, &ctx, pkrs, pipe_xor_bits);
+
+	mods = dm_test_get_primary_mods(test, adev);
+	dcc_best = dm_test_gfx11_dcc_best_modifier(pipe_xor_bits, pkrs, tile);
+
+	KUNIT_EXPECT_TRUE(test, ctx.called);
+	KUNIT_EXPECT_EQ(test, ctx.captured_reg, ctx.expected_reg);
+	KUNIT_EXPECT_EQ(test, mods[0], dcc_best);
+	KUNIT_EXPECT_TRUE(test, dm_test_mods_contain(mods, fallback_dcc_best));
+
+	dm_test_gfx11_reg_ctx = NULL;
+	kfree(mods);
+}
+
+/**
+ * dm_test_get_plane_modifiers_gfx12() - Verify GFX12 modifier list generation.
+ * @test: KUnit test context.
+ *
+ * Verify if the GFX12 family dispatches to the GFX12 modifier builder and
+ * produces a terminated list.
+ */
+static void dm_test_get_plane_modifiers_gfx12(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	adev->family = AMDGPU_FAMILY_GC_12_0_0;
+
+	dm_test_expect_mods_terminated(test, adev);
+}
+
+/**
+ * dm_test_fill_gfx9_plane_attributes_dcc() - Verify GFX9 DCC modifier path.
+ * @test: KUnit test context.
+ *
+ * Verify if a GFX10-RBPLUS DCC modifier enables DCC and selects the 64B
+ * independent block mode.
+ */
+static void dm_test_fill_gfx9_plane_attributes_dcc(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct amdgpu_framebuffer *afb;
+	struct plane_size plane_size = {0};
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	struct dm_test_dcc_cap_ctx ctx = {
+		.callback_ret = true,
+		.capable = true,
+		.output_independent_64b_blks = true,
+	};
+	int ret;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+
+	adev->family = AMDGPU_FAMILY_NV;
+	adev->dm.dc = dc;
+	adev->gfx.config.gb_addr_config_fields.num_pipes = 2;
+	adev->gfx.config.gb_addr_config_fields.num_pkrs = 2;
+	adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 3, 0);
+	dc->cap_funcs.get_dcc_compression_cap = dm_test_get_dcc_compression_cap;
+	dm_test_dcc_ctx = &ctx;
+
+	afb->base.pitches[1] = 256;
+	afb->base.modifier = AMD_FMT_MOD |
+			     AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_S_X) |
+			     AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX10_RBPLUS) |
+			     AMD_FMT_MOD_SET(DCC, 1) |
+			     AMD_FMT_MOD_SET(DCC_INDEPENDENT_64B, 1);
+	plane_size.surface_size.width = 1920;
+	plane_size.surface_size.height = 1080;
+
+	ret = dm_test_gfx9_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
+				 &address);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, (int)tiling_info.gfxversion, (int)DcGfxVersion9);
+	KUNIT_EXPECT_TRUE(test, dcc.enable);
+	KUNIT_EXPECT_EQ(test, (int)dcc.dcc_ind_blk, (int)hubp_ind_block_64b);
+	KUNIT_EXPECT_EQ(test, dcc.meta_pitch, 256U);
+
+	dm_test_dcc_ctx = NULL;
+}
+
+/**
+ * dm_test_fill_gfx9_plane_attributes_validate_fails() - Verify GFX9 error path.
+ * @test: KUnit test context.
+ *
+ * Verify if GFX9 modifier parsing returns validation errors from the shared DCC
+ * validation helper.
+ */
+static void dm_test_fill_gfx9_plane_attributes_validate_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct amdgpu_framebuffer *afb;
+	struct plane_size plane_size = {0};
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	u64 tile_version = AMD_FMT_MOD_TILE_VER_GFX10_RBPLUS;
+	int ret;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+
+	adev->family = AMDGPU_FAMILY_NV;
+	adev->dm.dc = dc;
+	adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 3, 0);
+	afb->base.modifier = dm_test_gfx9_dcc_modifier(tile_version, true, false);
+	plane_size.surface_size.width = 1920;
+	plane_size.surface_size.height = 1080;
+
+	ret = dm_test_gfx9_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
+				 &address);
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+}
+
+/**
+ * dm_test_fill_gfx9_plane_attributes_dcc_rbplus_64b_no_128bcl() - Verify block mode.
+ * @test: KUnit test context.
+ *
+ * Verify if a GFX10-RBPLUS modifier with both 64B and 128B independent block
+ * bits selects the 64B-no-128BCL block mode.
+ */
+static void dm_test_fill_gfx9_plane_attributes_dcc_rbplus_64b_no_128bcl(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct amdgpu_framebuffer *afb;
+	struct plane_size plane_size = {0};
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	struct dm_test_dcc_cap_ctx ctx = {0};
+	u64 tile_version = AMD_FMT_MOD_TILE_VER_GFX10_RBPLUS;
+	int ret;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+
+	dm_test_setup_gfx9_dcc_device(adev, dc, &ctx, true);
+	afb->base.modifier = dm_test_gfx9_dcc_modifier(tile_version, true, true);
+	plane_size.surface_size.width = 1920;
+	plane_size.surface_size.height = 1080;
+
+	ret = dm_test_gfx9_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
+				 &address);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_TRUE(test, dcc.enable);
+	KUNIT_EXPECT_EQ(test, (int)dcc.dcc_ind_blk,
+			(int)hubp_ind_block_64b_no_128bcl);
+
+	dm_test_dcc_ctx = NULL;
+}
+
+/**
+ * dm_test_fill_gfx9_plane_attributes_dcc_rbplus_128b() - Verify 128B block mode.
+ * @test: KUnit test context.
+ *
+ * Verify if a GFX10-RBPLUS modifier with only the 128B independent block bit
+ * selects the 128B block mode.
+ */
+static void dm_test_fill_gfx9_plane_attributes_dcc_rbplus_128b(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct amdgpu_framebuffer *afb;
+	struct plane_size plane_size = {0};
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	struct dm_test_dcc_cap_ctx ctx = {0};
+	u64 tile_version = AMD_FMT_MOD_TILE_VER_GFX10_RBPLUS;
+	int ret;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+
+	dm_test_setup_gfx9_dcc_device(adev, dc, &ctx, false);
+	afb->base.modifier = dm_test_gfx9_dcc_modifier(tile_version, false, true);
+	plane_size.surface_size.width = 1920;
+	plane_size.surface_size.height = 1080;
+
+	ret = dm_test_gfx9_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
+				 &address);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_TRUE(test, dcc.enable);
+	KUNIT_EXPECT_EQ(test, (int)dcc.dcc_ind_blk, (int)hubp_ind_block_128b);
+
+	dm_test_dcc_ctx = NULL;
+}
+
+/**
+ * dm_test_fill_gfx9_plane_attributes_dcc_rbplus_unconstrained() - Verify block mode.
+ * @test: KUnit test context.
+ *
+ * Verify if a GFX10-RBPLUS modifier without independent block bits selects the
+ * unconstrained block mode.
+ */
+static void dm_test_fill_gfx9_plane_attributes_dcc_rbplus_unconstrained(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct amdgpu_framebuffer *afb;
+	struct plane_size plane_size = {0};
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	struct dm_test_dcc_cap_ctx ctx = {0};
+	u64 tile_version = AMD_FMT_MOD_TILE_VER_GFX10_RBPLUS;
+	int ret;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+
+	dm_test_setup_gfx9_dcc_device(adev, dc, &ctx, false);
+	afb->base.modifier = dm_test_gfx9_dcc_modifier(tile_version, false, false);
+	plane_size.surface_size.width = 1920;
+	plane_size.surface_size.height = 1080;
+
+	ret = dm_test_gfx9_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
+				 &address);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_TRUE(test, dcc.enable);
+	KUNIT_EXPECT_EQ(test, (int)dcc.dcc_ind_blk,
+			(int)hubp_ind_block_unconstrained);
+
+	dm_test_dcc_ctx = NULL;
+}
+
+/**
+ * dm_test_fill_gfx9_plane_attributes_dcc_gfx9_64b() - Verify legacy 64B mode.
+ * @test: KUnit test context.
+ *
+ * Verify if a pre-RBPLUS GFX9 modifier with the 64B independent block bit
+ * selects the 64B block mode.
+ */
+static void dm_test_fill_gfx9_plane_attributes_dcc_gfx9_64b(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct amdgpu_framebuffer *afb;
+	struct plane_size plane_size = {0};
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	struct dm_test_dcc_cap_ctx ctx = {0};
+	int ret;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+
+	dm_test_setup_gfx9_dcc_device(adev, dc, &ctx, true);
+	afb->base.modifier = dm_test_gfx9_dcc_modifier(AMD_FMT_MOD_TILE_VER_GFX9,
+						       true, false);
+	plane_size.surface_size.width = 1920;
+	plane_size.surface_size.height = 1080;
+
+	ret = dm_test_gfx9_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
+				 &address);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_TRUE(test, dcc.enable);
+	KUNIT_EXPECT_EQ(test, (int)dcc.dcc_ind_blk, (int)hubp_ind_block_64b);
+
+	dm_test_dcc_ctx = NULL;
+}
+
+/**
+ * dm_test_fill_gfx9_plane_attributes_dcc_gfx9_unconstrained() - Verify legacy mode.
+ * @test: KUnit test context.
+ *
+ * Verify if a pre-RBPLUS GFX9 modifier without the 64B independent block bit
+ * selects the unconstrained block mode.
+ */
+static void dm_test_fill_gfx9_plane_attributes_dcc_gfx9_unconstrained(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct amdgpu_framebuffer *afb;
+	struct plane_size plane_size = {0};
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	struct dm_test_dcc_cap_ctx ctx = {0};
+	int ret;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+
+	dm_test_setup_gfx9_dcc_device(adev, dc, &ctx, false);
+	afb->base.modifier = dm_test_gfx9_dcc_modifier(AMD_FMT_MOD_TILE_VER_GFX9,
+						       false, false);
+	plane_size.surface_size.width = 1920;
+	plane_size.surface_size.height = 1080;
+
+	ret = dm_test_gfx9_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
+				 &address);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_TRUE(test, dcc.enable);
+	KUNIT_EXPECT_EQ(test, (int)dcc.dcc_ind_blk,
+			(int)hubp_ind_block_unconstrained);
+
+	dm_test_dcc_ctx = NULL;
+}
+
+/**
+ * dm_test_fill_gfx12_plane_attributes_block0() - Verify GFX12 64B max-compressed-block path.
+ * @test: KUnit test context.
+ *
+ * Verify if a zero max-compressed-block modifier selects the 64B independent
+ * block mode on GFX12.
+ */
+static void dm_test_fill_gfx12_plane_attributes_block0(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct amdgpu_framebuffer *afb;
+	struct plane_size plane_size = {0};
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	struct dm_test_dcc_cap_ctx ctx = {
+		.callback_ret = true,
+		.capable = true,
+		.output_independent_64b_blks = false,
+	};
+	int ret;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+
+	adev->family = AMDGPU_FAMILY_GC_12_0_0;
+	adev->dm.dc = dc;
+	dc->cap_funcs.get_dcc_compression_cap = dm_test_get_dcc_compression_cap;
+	dm_test_dcc_ctx = &ctx;
+
+	afb->base.modifier = AMD_FMT_MOD |
+			     AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX12_64K_2D) |
+			     AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX12) |
+			     AMD_FMT_MOD_SET(DCC, 1) |
+			     AMD_FMT_MOD_SET(DCC_MAX_COMPRESSED_BLOCK, 0);
+	plane_size.surface_size.width = 1920;
+	plane_size.surface_size.height = 1080;
+
+	ret = dm_test_gfx12_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
+				  &address);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_TRUE(test, dcc.enable);
+	KUNIT_EXPECT_TRUE(test, dcc.independent_64b_blks);
+	KUNIT_EXPECT_EQ(test, (int)dcc.dcc_ind_blk, (int)hubp_ind_block_64b);
+
+	dm_test_dcc_ctx = NULL;
+}
+
+/**
+ * dm_test_fill_gfx12_plane_attributes_block_unconstrained() - Verify block path.
+ * @test: KUnit test context.
+ *
+ * Verify if a max-compressed-block value above one selects the unconstrained
+ * independent block mode on GFX12.
+ */
+static void dm_test_fill_gfx12_plane_attributes_block_unconstrained(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct amdgpu_framebuffer *afb;
+	struct plane_size plane_size = {0};
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	struct dm_test_dcc_cap_ctx ctx = {
+		.callback_ret = true,
+		.capable = true,
+		.output_independent_64b_blks = false,
+	};
+	int ret;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+
+	adev->family = AMDGPU_FAMILY_GC_12_0_0;
+	adev->dm.dc = dc;
+	dc->cap_funcs.get_dcc_compression_cap = dm_test_get_dcc_compression_cap;
+	dm_test_dcc_ctx = &ctx;
+
+	afb->base.modifier = AMD_FMT_MOD |
+			     AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX12_64K_2D) |
+			     AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX12) |
+			     AMD_FMT_MOD_SET(DCC, 1) |
+			     AMD_FMT_MOD_SET(DCC_MAX_COMPRESSED_BLOCK, 2);
+	plane_size.surface_size.width = 1920;
+	plane_size.surface_size.height = 1080;
+
+	ret = dm_test_gfx12_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
+				  &address);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_TRUE(test, dcc.enable);
+	KUNIT_EXPECT_FALSE(test, dcc.independent_64b_blks);
+	KUNIT_EXPECT_EQ(test, (int)dcc.dcc_ind_blk,
+			(int)hubp_ind_block_unconstrained);
+
+	dm_test_dcc_ctx = NULL;
+}
+
+/**
+ * dm_test_fill_gfx12_plane_attributes_validate_fails() - Verify GFX12 error path.
+ * @test: KUnit test context.
+ *
+ * Verify if GFX12 modifier parsing returns validation errors from the shared
+ * DCC validation helper.
+ */
+static void dm_test_fill_gfx12_plane_attributes_validate_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct amdgpu_framebuffer *afb;
+	struct plane_size plane_size = {0};
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	int ret;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+
+	adev->family = AMDGPU_FAMILY_GC_12_0_0;
+	adev->dm.dc = dc;
+	afb->base.modifier = AMD_FMT_MOD |
+			     AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX12_64K_2D) |
+			     AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX12) |
+			     AMD_FMT_MOD_SET(DCC, 1) |
+			     AMD_FMT_MOD_SET(DCC_MAX_COMPRESSED_BLOCK, 1);
+	plane_size.surface_size.width = 1920;
+	plane_size.surface_size.height = 1080;
+
+	ret = dm_test_gfx12_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
+				  &address);
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+}
+
+/**
+ * dm_test_fill_plane_buffer_attributes_video() - Verify NV12 attributes.
+ * @test: KUnit test context.
+ *
+ * Verify if a video pixel format fills chroma plane size and the progressive
+ * video address type on a GFX9 family device.
+ */
+static void dm_test_fill_plane_buffer_attributes_video(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_framebuffer *afb;
+	struct dc_tiling_info tiling_info;
+	struct plane_size plane_size;
+	struct dc_plane_dcc_param dcc;
+	struct dc_plane_address address;
+	int ret;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	tiling_info = (struct dc_tiling_info){0};
+	plane_size = (struct plane_size){0};
+	dcc = (struct dc_plane_dcc_param){0};
+	address = (struct dc_plane_address){0};
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+
+	adev->family = AMDGPU_FAMILY_NV;
+	adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 3, 0);
+	afb->address = 0x80000000ULL;
+	afb->base.width = 1920;
+	afb->base.height = 1080;
+	afb->base.offsets[0] = 0;
+	afb->base.offsets[1] = 0x200000;
+	afb->base.pitches[0] = 1920;
+	afb->base.pitches[1] = 1920;
+	afb->base.format = drm_format_info(DRM_FORMAT_NV12);
+	afb->base.modifier = DRM_FORMAT_MOD_LINEAR;
+	KUNIT_ASSERT_NOT_NULL(test, afb->base.format);
+
+	ret = dm_test_video_attrs(adev, afb, &tiling_info, &plane_size, &dcc,
+				  &address);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, plane_size.surface_size.width, 1920);
+	KUNIT_EXPECT_EQ(test, plane_size.chroma_size.width, 960U);
+	KUNIT_EXPECT_EQ(test, plane_size.chroma_size.height, 540U);
+	KUNIT_EXPECT_EQ(test, address.type,
+			(int)PLN_ADDR_TYPE_VIDEO_PROGRESSIVE);
+	KUNIT_EXPECT_EQ(test, (int)tiling_info.gfxversion, (int)DcGfxVersion9);
+}
+
+/**
+ * dm_test_fill_plane_buffer_attributes_gfx12() - Verify GFX12 dispatch path.
+ * @test: KUnit test context.
+ *
+ * Verify if a GFX12 family device fills graphics attributes via the GFX12
+ * modifier path and reports the GFX addr3 version.
+ */
+static void dm_test_fill_plane_buffer_attributes_gfx12(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct amdgpu_framebuffer *afb;
+	struct dc_tiling_info tiling_info;
+	struct plane_size plane_size;
+	struct dc_plane_dcc_param dcc;
+	struct dc_plane_address address;
+	int ret;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	tiling_info = (struct dc_tiling_info){0};
+	plane_size = (struct plane_size){0};
+	dcc = (struct dc_plane_dcc_param){0};
+	address = (struct dc_plane_address){0};
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+
+	adev->family = AMDGPU_FAMILY_GC_12_0_0;
+	adev->dm.dc = dc;
+	afb->address = 0x80000000ULL;
+	afb->base.width = 1920;
+	afb->base.height = 1080;
+	afb->base.pitches[0] = 7680;
+	afb->base.format = drm_format_info(DRM_FORMAT_XRGB8888);
+	afb->base.modifier = DRM_FORMAT_MOD_LINEAR;
+	KUNIT_ASSERT_NOT_NULL(test, afb->base.format);
+
+	ret = dm_test_graphics_attrs(adev, afb, &tiling_info, &plane_size, &dcc,
+				     &address);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, address.type, (int)PLN_ADDR_TYPE_GRAPHICS);
+	KUNIT_EXPECT_EQ(test, (int)tiling_info.gfxversion, (int)DcGfxAddr3);
+}
+
+/**
+ * dm_test_get_min_max_dc_plane_scaling_fp16() - Verify fp16 cap selection.
+ * @test: KUnit test context.
+ *
+ * Verify if 64bpp fp16 formats use the fp16 scaling caps.
+ */
+static void dm_test_get_min_max_dc_plane_scaling_fp16(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct drm_framebuffer *fb;
+	int min_downscale = 0;
+	int max_upscale = 0;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+
+	adev->dm.dc = dc;
+	dc->caps.planes[0].max_upscale_factor.fp16 = 2000;
+	dc->caps.planes[0].max_downscale_factor.fp16 = 500;
+
+	fb->format = drm_format_info(DRM_FORMAT_ARGB16161616F);
+	KUNIT_ASSERT_NOT_NULL(test, fb->format);
+	amdgpu_dm_plane_get_min_max_dc_plane_scaling(&adev->ddev, fb,
+						     &min_downscale, &max_upscale);
+	KUNIT_EXPECT_EQ(test, min_downscale, 500);
+	KUNIT_EXPECT_EQ(test, max_upscale, 2000);
+}
+
+/**
+ * dm_test_helper_check_state_small_viewport_width() - Verify width rejection.
+ * @test: KUnit test context.
+ *
+ * Verify if a viewport width below the minimum pipe-split width is rejected.
+ */
+static void dm_test_helper_check_state_small_viewport_width(struct kunit *test)
+{
+	struct drm_plane *plane;
+	struct drm_plane_state *state;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *new_crtc_state;
+	struct drm_framebuffer *fb;
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	new_crtc_state = kunit_kzalloc(test, sizeof(*new_crtc_state), GFP_KERNEL);
+	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+	KUNIT_ASSERT_NOT_NULL(test, new_crtc_state);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+
+	plane->type = DRM_PLANE_TYPE_OVERLAY;
+	state->plane = plane;
+	state->fb = fb;
+	state->crtc = crtc;
+	state->crtc_x = 0;
+	state->crtc_y = 0;
+	state->crtc_w = 10;
+	state->crtc_h = 100;
+	new_crtc_state->mode.crtc_hdisplay = 1920;
+	new_crtc_state->mode.crtc_vdisplay = 1080;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_helper_check_state(state, new_crtc_state),
+			-EINVAL);
+}
+
+/**
+ * dm_test_helper_check_state_small_viewport_height() - Verify height rejection.
+ * @test: KUnit test context.
+ *
+ * Verify if a negative-offset viewport with a too-small height is rejected.
+ */
+static void dm_test_helper_check_state_small_viewport_height(struct kunit *test)
+{
+	struct drm_plane *plane;
+	struct drm_plane_state *state;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *new_crtc_state;
+	struct drm_framebuffer *fb;
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	new_crtc_state = kunit_kzalloc(test, sizeof(*new_crtc_state), GFP_KERNEL);
+	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+	KUNIT_ASSERT_NOT_NULL(test, new_crtc_state);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+
+	plane->type = DRM_PLANE_TYPE_OVERLAY;
+	state->plane = plane;
+	state->fb = fb;
+	state->crtc = crtc;
+	state->crtc_x = -2;
+	state->crtc_y = -95;
+	state->crtc_w = 100;
+	state->crtc_h = 100;
+	new_crtc_state->mode.crtc_hdisplay = 1920;
+	new_crtc_state->mode.crtc_vdisplay = 1080;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_helper_check_state(state, new_crtc_state),
+			-EINVAL);
+}
+
+/**
+ * dm_test_helper_check_state_bottom_clipped_height() - Verify bottom clipping.
+ * @test: KUnit test context.
+ *
+ * Verify if a viewport clipped by the bottom edge to below the minimum height
+ * is rejected.
+ */
+static void dm_test_helper_check_state_bottom_clipped_height(struct kunit *test)
+{
+	struct drm_plane *plane;
+	struct drm_plane_state *state;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *new_crtc_state;
+	struct drm_framebuffer *fb;
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	new_crtc_state = kunit_kzalloc(test, sizeof(*new_crtc_state), GFP_KERNEL);
+	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+	KUNIT_ASSERT_NOT_NULL(test, new_crtc_state);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+
+	plane->type = DRM_PLANE_TYPE_OVERLAY;
+	state->plane = plane;
+	state->fb = fb;
+	state->crtc = crtc;
+	state->crtc_x = 0;
+	state->crtc_y = 95;
+	state->crtc_w = 100;
+	state->crtc_h = 100;
+	new_crtc_state->mode.crtc_hdisplay = 1920;
+	new_crtc_state->mode.crtc_vdisplay = 100;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_helper_check_state(state, new_crtc_state),
+			-EINVAL);
+}
+
+/**
+ * dm_test_helper_check_state_scaling_caps() - Verify DC scaling caps are applied.
+ * @test: KUnit test context.
+ *
+ * Verify if helper_check_state converts DC plane scaling caps to DRM scale
+ * limits and rejects scaling outside those limits.
+ */
+static void dm_test_helper_check_state_scaling_caps(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct drm_plane *plane;
+	struct drm_plane_state *state;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *new_crtc_state;
+	struct drm_framebuffer *fb;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	new_crtc_state = kunit_kzalloc(test, sizeof(*new_crtc_state), GFP_KERNEL);
+	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+	KUNIT_ASSERT_NOT_NULL(test, new_crtc_state);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+
+	adev->dm.dc = dc;
+	dc->caps.planes[0].max_upscale_factor.argb8888 = 1;
+	dc->caps.planes[0].max_downscale_factor.argb8888 = 1;
+
+	plane->type = DRM_PLANE_TYPE_OVERLAY;
+	plane->dev = &adev->ddev;
+	crtc->dev = &adev->ddev;
+	fb->width = 100;
+	fb->height = 100;
+	fb->format = drm_format_info(DRM_FORMAT_XRGB8888);
+	KUNIT_ASSERT_NOT_NULL(test, fb->format);
+
+	state->plane = plane;
+	state->fb = fb;
+	state->crtc = crtc;
+	state->src_w = 100 << 16;
+	state->src_h = 100 << 16;
+	state->crtc_w = 200;
+	state->crtc_h = 200;
+	new_crtc_state->crtc = crtc;
+	new_crtc_state->mode.crtc_hdisplay = 1920;
+	new_crtc_state->mode.crtc_vdisplay = 1080;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_helper_check_state(state, new_crtc_state),
+			-ERANGE);
+}
+
+/**
+ * dm_test_fill_dc_scaling_info_nv12_dcn1x() - Verify NV12 DCN1x rejection.
+ * @test: KUnit test context.
+ *
+ * Verify if a non-zero NV12 source origin is rejected on DCN 1.0 to avoid the
+ * known DCN1x hang.
+ */
+static void dm_test_fill_dc_scaling_info_nv12_dcn1x(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_plane_state state = {0};
+	struct drm_framebuffer fb = {0};
+	struct dc_scaling_info info = {0};
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(1, 0, 0);
+	fb.format = drm_format_info(DRM_FORMAT_NV12);
+	KUNIT_ASSERT_NOT_NULL(test, fb.format);
+
+	state.fb = &fb;
+	state.src_x = 10 << 16;
+	state.src_y = 0;
+	state.src_w = 100 << 16;
+	state.src_h = 100 << 16;
+	state.crtc_w = 100;
+	state.crtc_h = 100;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_fill_dc_scaling_info(adev, &state, &info),
+			-EINVAL);
+
+	state.src_x = 0;
+	state.src_y = 10 << 16;
+	memset(&info, 0, sizeof(info));
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_fill_dc_scaling_info(adev, &state, &info),
+			-EINVAL);
+}
+
+/**
+ * dm_test_fill_dc_scaling_info_plane_caps() - Verify scaling caps path.
+ * @test: KUnit test context.
+ *
+ * Verify if scaling info uses plane caps when the state references a plane,
+ * device, and framebuffer.
+ */
+static void dm_test_fill_dc_scaling_info_plane_caps(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct drm_plane *plane;
+	struct drm_plane_state *state;
+	struct drm_framebuffer *fb;
+	struct dc_scaling_info info = {0};
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+
+	adev->dm.dc = dc;
+	dc->caps.planes[0].max_upscale_factor.argb8888 = 16000;
+	dc->caps.planes[0].max_downscale_factor.argb8888 = 250;
+
+	plane->dev = &adev->ddev;
+	fb->format = drm_format_info(DRM_FORMAT_XRGB8888);
+	KUNIT_ASSERT_NOT_NULL(test, fb->format);
+
+	state->plane = plane;
+	state->fb = fb;
+	state->src_w = 100 << 16;
+	state->src_h = 100 << 16;
+	state->crtc_w = 100;
+	state->crtc_h = 100;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_fill_dc_scaling_info(adev, state, &info),
+			0);
+}
+
+/**
+ * dm_test_get_cursor_position_bad_size() - Verify oversized cursor rejection.
+ * @test: KUnit test context.
+ *
+ * Verify if a cursor larger than the CRTC maximum is rejected.
+ */
+static void dm_test_get_cursor_position_bad_size(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *amdgpu_crtc;
+	struct drm_plane *plane;
+	struct drm_plane_state *state;
+	struct drm_framebuffer *fb;
+	struct dc_cursor_position position = {0};
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	amdgpu_crtc = kunit_kzalloc(test, sizeof(*amdgpu_crtc), GFP_KERNEL);
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, amdgpu_crtc);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+
+	amdgpu_crtc->max_cursor_width = 64;
+	amdgpu_crtc->max_cursor_height = 64;
+	plane->dev = &adev->ddev;
+	plane->state = state;
+	state->fb = fb;
+	state->crtc_w = 128;
+	state->crtc_h = 32;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_get_cursor_position(plane, &amdgpu_crtc->base, &position),
+			-EINVAL);
+}
+
+/**
+ * dm_test_format_mod_supported_d_swizzle_reject() - Verify D swizzle rejection.
+ * @test: KUnit test context.
+ *
+ * Verify if a D micro-swizzle modifier is rejected for formats narrower than
+ * 8 bytes per pixel.
+ */
+static void dm_test_format_mod_supported_d_swizzle_reject(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_plane *plane;
+	u64 listed_mod;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+
+	adev->family = AMDGPU_FAMILY_RV;
+	plane->dev = &adev->ddev;
+
+	listed_mod = AMD_FMT_MOD |
+		     AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_D) |
+		     AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX9);
+	plane->modifiers = &listed_mod;
+	plane->modifier_count = 1;
+
+	KUNIT_EXPECT_FALSE(test,
+			   amdgpu_dm_plane_format_mod_supported(plane, DRM_FORMAT_XRGB8888,
+								listed_mod));
+}
+
+/**
+ * dm_test_atomic_async_check_rejects() - Verify async check rejections.
+ * @test: KUnit test context.
+ *
+ * Verify if async flip on non-overlay planes and async cursor update on
+ * non-cursor planes are rejected.
+ */
+static void dm_test_atomic_async_check_rejects(struct kunit *test)
+{
+	struct drm_plane *plane;
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+
+	plane->type = DRM_PLANE_TYPE_PRIMARY;
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_atomic_async_check(plane, NULL, true),
+			-EINVAL);
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_atomic_async_check(plane, NULL, false),
+			-EINVAL);
+}
+
+/**
+ * dm_test_atomic_async_check_overlay_cursor() - Verify overlay cursor rejection.
+ * @test: KUnit test context.
+ *
+ * Verify if async cursor updates are rejected while the CRTC is using an
+ * overlay cursor mode.
+ */
+static void dm_test_atomic_async_check_overlay_cursor(struct kunit *test)
+{
+	struct drm_atomic_commit *state;
+	struct __drm_planes_state *planes;
+	struct __drm_crtcs_state *crtcs;
+	struct drm_plane *plane;
+	struct drm_plane_state *plane_state;
+	struct drm_crtc *crtc;
+	struct dm_crtc_state *dm_crtc_state;
+
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	planes = kunit_kzalloc(test, sizeof(*planes), GFP_KERNEL);
+	crtcs = kunit_kzalloc(test, sizeof(*crtcs), GFP_KERNEL);
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	plane_state = kunit_kzalloc(test, sizeof(*plane_state), GFP_KERNEL);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	dm_crtc_state = kunit_kzalloc(test, sizeof(*dm_crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	KUNIT_ASSERT_NOT_NULL(test, planes);
+	KUNIT_ASSERT_NOT_NULL(test, crtcs);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, plane_state);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+	KUNIT_ASSERT_NOT_NULL(test, dm_crtc_state);
+
+	plane->type = DRM_PLANE_TYPE_CURSOR;
+	plane->index = 0;
+	crtc->index = 0;
+	plane_state->crtc = crtc;
+	dm_crtc_state->cursor_mode = DM_CURSOR_OVERLAY_MODE;
+	state->planes = planes;
+	state->crtcs = crtcs;
+	state->planes[0].new_state = plane_state;
+	state->crtcs[0].new_state = &dm_crtc_state->base;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_atomic_async_check(plane, state, false),
+			-EINVAL);
+
+	dm_crtc_state->cursor_mode = DM_CURSOR_NATIVE_MODE;
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_atomic_async_check(plane, state, false),
+			0);
+}
+
+static struct amdgpu_device *dm_test_init_atomic_check_state(struct kunit *test,
+							     struct drm_atomic_commit **state,
+							     struct drm_plane **plane,
+							     struct dm_plane_state **dm_plane_state,
+							     struct drm_crtc_state **new_crtc_state,
+							     struct drm_framebuffer **fb)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	struct __drm_planes_state *planes;
+	struct __drm_crtcs_state *crtcs;
+	struct dc_plane_state *dc_plane_state;
+	struct drm_crtc *crtc;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	*state = kunit_kzalloc(test, sizeof(**state), GFP_KERNEL);
+	planes = kunit_kzalloc(test, sizeof(*planes), GFP_KERNEL);
+	crtcs = kunit_kzalloc(test, sizeof(*crtcs), GFP_KERNEL);
+	*plane = kunit_kzalloc(test, sizeof(**plane), GFP_KERNEL);
+	*dm_plane_state = kunit_kzalloc(test, sizeof(**dm_plane_state), GFP_KERNEL);
+	dc_plane_state = kunit_kzalloc(test, sizeof(*dc_plane_state), GFP_KERNEL);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	*new_crtc_state = kunit_kzalloc(test, sizeof(**new_crtc_state), GFP_KERNEL);
+	*fb = kunit_kzalloc(test, sizeof(**fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, *state);
+	KUNIT_ASSERT_NOT_NULL(test, planes);
+	KUNIT_ASSERT_NOT_NULL(test, crtcs);
+	KUNIT_ASSERT_NOT_NULL(test, *plane);
+	KUNIT_ASSERT_NOT_NULL(test, *dm_plane_state);
+	KUNIT_ASSERT_NOT_NULL(test, dc_plane_state);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+	KUNIT_ASSERT_NOT_NULL(test, *new_crtc_state);
+	KUNIT_ASSERT_NOT_NULL(test, *fb);
+
+	adev->dm.dc = dc;
+	dc->caps.planes[0].max_upscale_factor.argb8888 = 1000;
+	dc->caps.planes[0].max_downscale_factor.argb8888 = 1000;
+	dc->caps.planes[0].max_upscale_factor.nv12 = 1000;
+	dc->caps.planes[0].max_downscale_factor.nv12 = 1000;
+
+	(*plane)->dev = &adev->ddev;
+	(*plane)->index = 0;
+	(*plane)->type = DRM_PLANE_TYPE_OVERLAY;
+	(*plane)->name = "kunit-plane";
+	crtc->dev = &adev->ddev;
+	crtc->index = 0;
+	(*fb)->width = 100;
+	(*fb)->height = 100;
+	(*fb)->format = drm_format_info(DRM_FORMAT_XRGB8888);
+	KUNIT_ASSERT_NOT_NULL(test, (*fb)->format);
+
+	(*dm_plane_state)->base.plane = *plane;
+	(*dm_plane_state)->base.state = *state;
+	(*dm_plane_state)->base.crtc = crtc;
+	(*dm_plane_state)->base.fb = *fb;
+	(*dm_plane_state)->base.src_w = 100 << 16;
+	(*dm_plane_state)->base.src_h = 100 << 16;
+	(*dm_plane_state)->base.crtc_w = 100;
+	(*dm_plane_state)->base.crtc_h = 100;
+	(*dm_plane_state)->dc_state = dc_plane_state;
+
+	(*new_crtc_state)->crtc = crtc;
+	(*new_crtc_state)->enable = true;
+	(*new_crtc_state)->mode.crtc_hdisplay = 1920;
+	(*new_crtc_state)->mode.crtc_vdisplay = 1080;
+
+	(*state)->planes = planes;
+	(*state)->crtcs = crtcs;
+	(*state)->planes[0].new_state = &(*dm_plane_state)->base;
+	(*state)->crtcs[0].new_state = *new_crtc_state;
+
+	return adev;
+}
+
+/**
+ * dm_test_atomic_check_no_dc_state() - Verify missing DC plane state succeeds.
+ * @test: KUnit test context.
+ *
+ * Verify if atomic_check exits before deeper validation when the DM plane state
+ * has no DC plane state attached.
+ */
+static void dm_test_atomic_check_no_dc_state(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_atomic_commit *state;
+	struct __drm_planes_state *planes;
+	struct drm_plane *plane;
+	struct dm_plane_state *dm_plane_state;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	planes = kunit_kzalloc(test, sizeof(*planes), GFP_KERNEL);
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	dm_plane_state = kunit_kzalloc(test, sizeof(*dm_plane_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	KUNIT_ASSERT_NOT_NULL(test, planes);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, dm_plane_state);
+
+	plane->dev = &adev->ddev;
+	plane->index = 0;
+	dm_plane_state->base.plane = plane;
+	state->planes = planes;
+	state->planes[0].new_state = &dm_plane_state->base;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_atomic_check(plane, state), 0);
+}
+
+/**
+ * dm_test_atomic_check_missing_crtc_state() - Verify missing CRTC state fails.
+ * @test: KUnit test context.
+ *
+ * Verify if atomic_check rejects a plane with DC state when the atomic CRTC
+ * state is absent.
+ */
+static void dm_test_atomic_check_missing_crtc_state(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_atomic_commit *state;
+	struct __drm_planes_state *planes;
+	struct __drm_crtcs_state *crtcs;
+	struct drm_plane *plane;
+	struct dm_plane_state *dm_plane_state;
+	struct dc_plane_state *dc_plane_state;
+	struct drm_crtc *crtc;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	planes = kunit_kzalloc(test, sizeof(*planes), GFP_KERNEL);
+	crtcs = kunit_kzalloc(test, sizeof(*crtcs), GFP_KERNEL);
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	dm_plane_state = kunit_kzalloc(test, sizeof(*dm_plane_state), GFP_KERNEL);
+	dc_plane_state = kunit_kzalloc(test, sizeof(*dc_plane_state), GFP_KERNEL);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	KUNIT_ASSERT_NOT_NULL(test, planes);
+	KUNIT_ASSERT_NOT_NULL(test, crtcs);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, dm_plane_state);
+	KUNIT_ASSERT_NOT_NULL(test, dc_plane_state);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+
+	plane->dev = &adev->ddev;
+	plane->index = 0;
+	crtc->index = 0;
+	dm_plane_state->base.plane = plane;
+	dm_plane_state->base.crtc = crtc;
+	dm_plane_state->dc_state = dc_plane_state;
+	state->planes = planes;
+	state->crtcs = crtcs;
+	state->planes[0].new_state = &dm_plane_state->base;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_atomic_check(plane, state), -EINVAL);
+}
+
+/**
+ * dm_test_atomic_check_helper_failure() - Verify helper-check failures return.
+ * @test: KUnit test context.
+ *
+ * Verify if atomic_check returns before DC validation when the DRM helper state
+ * validation rejects the plane.
+ */
+static void dm_test_atomic_check_helper_failure(struct kunit *test)
+{
+	struct drm_atomic_commit *state;
+	struct drm_plane *plane;
+	struct dm_plane_state *dm_plane_state;
+	struct drm_crtc_state *new_crtc_state;
+	struct drm_framebuffer *fb;
+
+	dm_test_init_atomic_check_state(test, &state, &plane, &dm_plane_state,
+					&new_crtc_state, &fb);
+	dm_plane_state->base.crtc_w = 10;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_atomic_check(plane, state), -EINVAL);
+}
+
+/**
+ * dm_test_atomic_check_color_pipeline_conflict() - Verify color conflict rejection.
+ * @test: KUnit test context.
+ *
+ * Verify if atomic_check rejects use of both plane COLOR_PIPELINE and CRTC
+ * DEGAMMA_LUT before DC validation.
+ */
+static void dm_test_atomic_check_color_pipeline_conflict(struct kunit *test)
+{
+	struct drm_atomic_commit *state;
+	struct drm_plane *plane;
+	struct dm_plane_state *dm_plane_state;
+	struct drm_crtc_state *new_crtc_state;
+	struct drm_framebuffer *fb;
+	void *color_pipeline;
+	void *degamma_lut;
+
+	dm_test_init_atomic_check_state(test, &state, &plane, &dm_plane_state,
+					&new_crtc_state, &fb);
+	color_pipeline = kunit_kzalloc(test, 1, GFP_KERNEL);
+	degamma_lut = kunit_kzalloc(test, 1, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, color_pipeline);
+	KUNIT_ASSERT_NOT_NULL(test, degamma_lut);
+
+	dm_plane_state->base.color_pipeline = color_pipeline;
+	new_crtc_state->degamma_lut = degamma_lut;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_atomic_check(plane, state), -EINVAL);
+}
+
+/**
+ * dm_test_atomic_check_scaling_failure() - Verify scaling-info failures return.
+ * @test: KUnit test context.
+ *
+ * Verify if atomic_check returns the scaling-info error before DC validation.
+ */
+static void dm_test_atomic_check_scaling_failure(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_atomic_commit *state;
+	struct drm_plane *plane;
+	struct dm_plane_state *dm_plane_state;
+	struct drm_crtc_state *new_crtc_state;
+	struct drm_framebuffer *fb;
+
+	adev = dm_test_init_atomic_check_state(test, &state, &plane, &dm_plane_state,
+					       &new_crtc_state, &fb);
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(1, 0, 0);
+	fb->width = 200;
+	fb->format = drm_format_info(DRM_FORMAT_NV12);
+	KUNIT_ASSERT_NOT_NULL(test, fb->format);
+	dm_plane_state->base.src_x = 1 << 16;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_atomic_check(plane, state), -EINVAL);
+}
+
+/**
+ * dm_test_panic_flush_no_dc_state() - Verify panic flush exits without DC state.
+ * @test: KUnit test context.
+ *
+ * Verify if panic_flush returns without dereferencing DC state when the current
+ * plane state has no DC plane state attached.
+ */
+static void dm_test_panic_flush_no_dc_state(struct kunit *test)
+{
+	struct drm_plane *plane;
+	struct dm_plane_state *dm_plane_state;
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	dm_plane_state = kunit_kzalloc(test, sizeof(*dm_plane_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, dm_plane_state);
+
+	plane->state = &dm_plane_state->base;
+
+	amdgpu_dm_plane_panic_flush(plane);
+}
+
+static const struct drm_plane_funcs dm_test_plane_reset_funcs = {
+	.atomic_destroy_state = amdgpu_dm_plane_drm_plane_destroy_state,
+};
+
+/**
+ * dm_test_plane_reset_initializes_state() - Verify reset installs default state.
+ * @test: KUnit test context.
+ *
+ * Verify amdgpu_dm_plane_drm_plane_reset() destroys the existing plane state,
+ * allocates a fresh dm_plane_state, and initializes the AMD-specific transfer
+ * function and HDR multiplier defaults.
+ */
+static void dm_test_plane_reset_initializes_state(struct kunit *test)
+{
+	struct dm_plane_state *old_state;
+	struct dm_plane_state *new_state;
+	struct drm_plane *plane;
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+
+	/*
+	 * Provide an existing state plus a funcs table so reset exercises the
+	 * destroy-existing-state path. The destroy hook frees this state, so it
+	 * must be a plain (non-KUnit-managed) allocation.
+	 */
+	old_state = kzalloc(sizeof(*old_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_state);
+	plane->funcs = &dm_test_plane_reset_funcs;
+	plane->state = &old_state->base;
+
+	amdgpu_dm_plane_drm_plane_reset(plane);
+
+	KUNIT_ASSERT_NOT_NULL(test, plane->state);
+	new_state = to_dm_plane_state(plane->state);
+	KUNIT_EXPECT_EQ(test, new_state->degamma_tf, AMDGPU_TRANSFER_FUNCTION_DEFAULT);
+	KUNIT_EXPECT_EQ(test, new_state->hdr_mult, AMDGPU_HDR_MULT_DEFAULT);
+	KUNIT_EXPECT_EQ(test, new_state->shaper_tf, AMDGPU_TRANSFER_FUNCTION_DEFAULT);
+	KUNIT_EXPECT_EQ(test, new_state->blend_tf, AMDGPU_TRANSFER_FUNCTION_DEFAULT);
+
+	kfree(new_state);
+}
+
+/**
+ * dm_test_plane_duplicate_state_copies_fields() - Verify state duplication.
+ * @test: KUnit test context.
+ *
+ * Verify amdgpu_dm_plane_drm_plane_duplicate_state() allocates a new state and
+ * copies the transfer-function and HDR-multiplier fields from the current
+ * plane state when no DC state or color blob is attached.
+ */
+static void dm_test_plane_duplicate_state_copies_fields(struct kunit *test)
+{
+	struct dm_plane_state *old_state;
+	struct drm_plane_state *dup_base;
+	struct dm_plane_state *dup_state;
+	struct drm_plane *plane;
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	old_state = kunit_kzalloc(test, sizeof(*old_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, old_state);
+
+	old_state->degamma_tf = AMDGPU_TRANSFER_FUNCTION_PQ_EOTF;
+	old_state->hdr_mult = 0x123456789ULL;
+	old_state->shaper_tf = AMDGPU_TRANSFER_FUNCTION_IDENTITY;
+	old_state->blend_tf = AMDGPU_TRANSFER_FUNCTION_SRGB_EOTF;
+	plane->state = &old_state->base;
+
+	dup_base = amdgpu_dm_plane_drm_plane_duplicate_state(plane);
+	KUNIT_ASSERT_NOT_NULL(test, dup_base);
+
+	dup_state = to_dm_plane_state(dup_base);
+	KUNIT_EXPECT_EQ(test, dup_state->degamma_tf, AMDGPU_TRANSFER_FUNCTION_PQ_EOTF);
+	KUNIT_EXPECT_EQ(test, dup_state->hdr_mult, 0x123456789ULL);
+	KUNIT_EXPECT_EQ(test, dup_state->shaper_tf, AMDGPU_TRANSFER_FUNCTION_IDENTITY);
+	KUNIT_EXPECT_EQ(test, dup_state->blend_tf, AMDGPU_TRANSFER_FUNCTION_SRGB_EOTF);
+	KUNIT_EXPECT_NULL(test, dup_state->dc_state);
+
+	kfree(dup_state);
+}
+
+/**
+ * dm_test_plane_destroy_state_minimal() - Verify destroy of a minimal state.
+ * @test: KUnit test context.
+ *
+ * Verify amdgpu_dm_plane_drm_plane_destroy_state() tears down a plane state
+ * that has no color blobs or DC plane state attached without dereferencing
+ * NULL resources.
+ */
+static void dm_test_plane_destroy_state_minimal(struct kunit *test)
+{
+	struct dm_plane_state *dm_plane_state;
+	struct drm_plane *plane;
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+
+	/* destroy_state frees the state itself, so use a plain allocation. */
+	dm_plane_state = kzalloc(sizeof(*dm_plane_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_plane_state);
+
+	amdgpu_dm_plane_drm_plane_destroy_state(plane, &dm_plane_state->base);
+}
+
+static struct kunit_case amdgpu_dm_plane_test_cases[] = {
+	/* amdgpu_dm_plane_is_video_format() */
+	KUNIT_CASE(dm_test_plane_is_video_format_known_video),
+	/* amdgpu_dm_plane_get_format_info() */
+	KUNIT_CASE(dm_test_get_format_info),
+	/* amdgpu_dm_plane_fill_blending_from_plane_state() */
+	KUNIT_CASE(dm_test_fill_blending_defaults),
+	KUNIT_CASE(dm_test_fill_blending_premulti_alpha_format),
+	KUNIT_CASE(dm_test_fill_blending_coverage_alpha_format),
+	KUNIT_CASE(dm_test_fill_blending_global_alpha),
+	KUNIT_CASE(dm_test_fill_blending_global_alpha_dcn42),
+	/* amdgpu_dm_plane_modifier_* helpers() */
+	KUNIT_CASE(dm_test_modifier_has_dcc),
+	KUNIT_CASE(dm_test_modifier_gfx9_swizzle_mode),
+	/* amdgpu_dm_plane_get_plane_formats() */
+	KUNIT_CASE(dm_test_get_plane_formats),
+	KUNIT_CASE(dm_test_get_plane_formats_overlay_universal_cap),
+	/* amdgpu_dm_plane_get_plane_modifiers() */
+	KUNIT_CASE(dm_test_get_plane_modifiers),
+	KUNIT_CASE(dm_test_get_plane_modifiers_gfx9),
+	KUNIT_CASE(dm_test_get_plane_modifiers_rv),
+	KUNIT_CASE(dm_test_get_plane_modifiers_rv_constant_encode),
+	KUNIT_CASE(dm_test_get_plane_modifiers_gfx10_1),
+	KUNIT_CASE(dm_test_get_plane_modifiers_gfx10_3),
+	KUNIT_CASE(dm_test_get_plane_modifiers_gfx11_64k_first),
+	KUNIT_CASE(dm_test_get_plane_modifiers_gfx11_256k_first),
+	KUNIT_CASE(dm_test_get_plane_modifiers_gfx12),
+	/* amdgpu_dm_plane_fill_dc_scaling_info() */
+	KUNIT_CASE(dm_test_fill_dc_scaling_info),
+	KUNIT_CASE(dm_test_fill_dc_scaling_info_nv12_dcn1x),
+	KUNIT_CASE(dm_test_fill_dc_scaling_info_plane_caps),
+	/* amdgpu_dm_plane_get_min_max_dc_plane_scaling() */
+	KUNIT_CASE(dm_test_get_min_max_dc_plane_scaling),
+	KUNIT_CASE(dm_test_get_min_max_dc_plane_scaling_fp16),
+	/* amdgpu_dm_plane_fill_plane_buffer_attributes() */
+	KUNIT_CASE(dm_test_fill_plane_buffer_attributes_gfx8),
+	KUNIT_CASE(dm_test_fill_plane_buffer_attributes_video),
+	KUNIT_CASE(dm_test_fill_plane_buffer_attributes_gfx12),
+	/* amdgpu_dm_plane_get_cursor_position() */
+	KUNIT_CASE(dm_test_get_cursor_position),
+	KUNIT_CASE(dm_test_get_cursor_position_bad_size),
+	/* amdgpu_dm_plane_format_mod_supported() */
+	KUNIT_CASE(dm_test_format_mod_supported),
+	KUNIT_CASE(dm_test_format_mod_supported_d_swizzle_reject),
+	/* amdgpu_dm_plane_fill_gfx12_attrs_from_modifiers() */
+	KUNIT_CASE(dm_test_fill_gfx12_plane_attributes_from_modifiers),
+	KUNIT_CASE(dm_test_fill_gfx12_plane_attributes_block0),
+	KUNIT_CASE(dm_test_fill_gfx12_plane_attributes_block_unconstrained),
+	KUNIT_CASE(dm_test_fill_gfx12_plane_attributes_validate_fails),
+	/* amdgpu_dm_plane_fill_gfx9_attrs_from_modifiers() */
+	KUNIT_CASE(dm_test_fill_gfx9_plane_attributes_from_modifiers),
+	KUNIT_CASE(dm_test_fill_gfx9_plane_attributes_dcc),
+	KUNIT_CASE(dm_test_fill_gfx9_plane_attributes_validate_fails),
+	KUNIT_CASE(dm_test_fill_gfx9_plane_attributes_dcc_rbplus_64b_no_128bcl),
+	KUNIT_CASE(dm_test_fill_gfx9_plane_attributes_dcc_rbplus_128b),
+	KUNIT_CASE(dm_test_fill_gfx9_plane_attributes_dcc_rbplus_unconstrained),
+	KUNIT_CASE(dm_test_fill_gfx9_plane_attributes_dcc_gfx9_64b),
+	KUNIT_CASE(dm_test_fill_gfx9_plane_attributes_dcc_gfx9_unconstrained),
+	/* amdgpu_dm_plane_helper_check_state() */
+	KUNIT_CASE(dm_test_helper_check_state_viewport_reject),
+	KUNIT_CASE(dm_test_helper_check_state_small_viewport_width),
+	KUNIT_CASE(dm_test_helper_check_state_small_viewport_height),
+	KUNIT_CASE(dm_test_helper_check_state_bottom_clipped_height),
+	KUNIT_CASE(dm_test_helper_check_state_scaling_caps),
+	/* amdgpu_dm_plane_atomic_async_check() */
+	KUNIT_CASE(dm_test_atomic_async_check_rejects),
+	KUNIT_CASE(dm_test_atomic_async_check_overlay_cursor),
+	/* amdgpu_dm_plane_atomic_check() */
+	KUNIT_CASE(dm_test_atomic_check_no_dc_state),
+	KUNIT_CASE(dm_test_atomic_check_missing_crtc_state),
+	KUNIT_CASE(dm_test_atomic_check_helper_failure),
+	KUNIT_CASE(dm_test_atomic_check_color_pipeline_conflict),
+	KUNIT_CASE(dm_test_atomic_check_scaling_failure),
+	/* amdgpu_dm_plane_panic_flush() */
+	KUNIT_CASE(dm_test_panic_flush_no_dc_state),
+	/* amdgpu_dm_plane_drm_plane_reset() */
+	KUNIT_CASE(dm_test_plane_reset_initializes_state),
+	/* amdgpu_dm_plane_drm_plane_duplicate_state() */
+	KUNIT_CASE(dm_test_plane_duplicate_state_copies_fields),
+	/* amdgpu_dm_plane_drm_plane_destroy_state() */
+	KUNIT_CASE(dm_test_plane_destroy_state_minimal),
+	/* amdgpu_dm_plane_add_modifier() */
+	KUNIT_CASE(dm_test_add_modifier_appends_value),
+	KUNIT_CASE(dm_test_add_modifier_grows_capacity),
+	KUNIT_CASE(dm_test_add_modifier_noop_when_mods_null),
+	/* amdgpu_dm_plane_fill_gfx8_tiling_info_from_flags() */
+	KUNIT_CASE(dm_test_fill_gfx8_tiling_info_2d_tiled),
+	KUNIT_CASE(dm_test_fill_gfx8_tiling_info_1d_tiled),
+	KUNIT_CASE(dm_test_fill_gfx8_tiling_info_other_mode),
+	/* amdgpu_dm_plane_fill_gfx9_tiling_info_from_device() */
+	KUNIT_CASE(dm_test_fill_gfx9_tiling_info_from_device_pre_10_3),
+	KUNIT_CASE(dm_test_fill_gfx9_tiling_info_from_device_10_3_plus),
+	/* amdgpu_dm_plane_fill_gfx9_tiling_info_from_modifier() */
+	KUNIT_CASE(dm_test_fill_gfx9_tiling_info_from_modifier_linear),
+	KUNIT_CASE(dm_test_fill_gfx9_tiling_info_from_modifier_pre_nv),
+	KUNIT_CASE(dm_test_fill_gfx9_tiling_info_from_modifier_nv),
+	/* amdgpu_dm_plane_validate_dcc() */
+	KUNIT_CASE(dm_test_validate_dcc_disabled_returns_success),
+	KUNIT_CASE(dm_test_validate_dcc_video_non_gfx12_fails),
+	KUNIT_CASE(dm_test_validate_dcc_missing_cap_func_fails),
+	KUNIT_CASE(dm_test_validate_dcc_cap_callback_fails),
+	KUNIT_CASE(dm_test_validate_dcc_not_capable_fails),
+	KUNIT_CASE(dm_test_validate_dcc_success_and_scan_mapping),
+	KUNIT_CASE(dm_test_validate_dcc_independent_64b_mismatch_fails),
+	{}
+};
+
+static struct kunit_suite amdgpu_dm_plane_test_suite = {
+	.name = "amdgpu_dm_plane",
+	.test_cases = amdgpu_dm_plane_test_cases,
+};
+
+kunit_test_suite(amdgpu_dm_plane_test_suite);
+
+MODULE_DESCRIPTION("KUnit tests for amdgpu_dm_plane");
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_pp_smu_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_pp_smu_test.c
new file mode 100644
index 0000000000000..8d1d26bfcc16c
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_pp_smu_test.c
@@ -0,0 +1,2454 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * KUnit tests for amdgpu_dm_pp_smu.c
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#include <kunit/test.h>
+#include <linux/types.h>
+#include <linux/mutex.h>
+
+#include "dc.h"
+#include "dm_services.h"
+#include "dm_pp_smu.h"
+#include "amdgpu.h"
+#include "amdgpu_mode.h"
+#include "amdgpu_dm.h"
+#include "amdgpu_dm_pp_smu.h"
+
+/* ---- Stub DPM layer ---- */
+
+/**
+ * struct stub_dpm_context - Tracks stub DPM callback invocations
+ * @ret_val: Return value for the next DPM callback
+ * @get_current_clocks_info: Clock info returned by stub get_current_clocks
+ * @get_clock_by_type_clocks: Clocks returned by stub get_clock_by_type
+ * @get_validation_clks: Validation clocks returned by stub
+ * @get_clock_by_type_with_latency_clks: Returned by stub with_latency
+ * @get_clock_by_type_with_voltage_clks: Returned by stub with_voltage
+ * @set_watermarks_ret: Return value for set_watermarks
+ * @display_clock_voltage_ret: Return value for display_clock_voltage_request
+ * @display_disable_memory_clock_switch_ret: Return for disable_memory_clock
+ * @get_max_sustainable_ret: Return for get_max_sustainable_clocks_by_dc
+ * @get_uclk_dpm_ret: Return for get_uclk_dpm_states
+ * @get_dpm_clock_table_ret: Return for get_dpm_clock_table
+ * @set_active_display_count_ret: Return for set_active_display_count
+ * @set_min_deep_sleep_dcefclk_ret: Return for set_min_deep_sleep_dcefclk
+ * @get_validation_clks_ret: Return for get_display_mode_validation_clocks
+ */
+struct stub_dpm_context {
+	int ret_val;
+	struct amd_pp_clock_info get_current_clocks_info;
+	struct amd_pp_clocks get_clock_by_type_clocks;
+	struct amd_pp_simple_clock_info get_validation_clks;
+	int get_validation_clks_ret;
+	struct pp_clock_levels_with_latency get_clock_by_type_with_latency_clks;
+	struct pp_clock_levels_with_voltage get_clock_by_type_with_voltage_clks;
+	int set_watermarks_ret;
+	int display_clock_voltage_ret;
+	int display_disable_memory_clock_switch_ret;
+	int get_max_sustainable_ret;
+	int get_uclk_dpm_ret;
+	int get_dpm_clock_table_ret;
+	int set_active_display_count_ret;
+	int set_min_deep_sleep_dcefclk_ret;
+};
+
+static struct stub_dpm_context *stub_dpm_ctx;
+
+static int stub_get_current_clocks(void *handle, struct amd_pp_clock_info *clocks)
+{
+	if (stub_dpm_ctx->ret_val)
+		return stub_dpm_ctx->ret_val;
+	*clocks = stub_dpm_ctx->get_current_clocks_info;
+	return 0;
+}
+
+static int stub_get_clock_by_type(void *handle, enum amd_pp_clock_type type,
+				  struct amd_pp_clocks *clocks)
+{
+	if (stub_dpm_ctx->ret_val)
+		return stub_dpm_ctx->ret_val;
+	*clocks = stub_dpm_ctx->get_clock_by_type_clocks;
+	return 0;
+}
+
+static int stub_get_display_mode_validation_clocks(void *handle,
+						   struct amd_pp_simple_clock_info *clocks)
+{
+	if (stub_dpm_ctx->get_validation_clks_ret)
+		return stub_dpm_ctx->get_validation_clks_ret;
+	*clocks = stub_dpm_ctx->get_validation_clks;
+	return 0;
+}
+
+static int stub_get_clock_by_type_with_latency(void *handle,
+					       enum amd_pp_clock_type type,
+					       struct pp_clock_levels_with_latency *clocks)
+{
+	if (stub_dpm_ctx->ret_val)
+		return stub_dpm_ctx->ret_val;
+	*clocks = stub_dpm_ctx->get_clock_by_type_with_latency_clks;
+	return 0;
+}
+
+static int stub_get_clock_by_type_with_voltage(void *handle,
+					       enum amd_pp_clock_type type,
+					       struct pp_clock_levels_with_voltage *clocks)
+{
+	if (stub_dpm_ctx->ret_val)
+		return stub_dpm_ctx->ret_val;
+	*clocks = stub_dpm_ctx->get_clock_by_type_with_voltage_clks;
+	return 0;
+}
+
+static void stub_display_configuration_change(void *handle)
+{
+	/* No-op: satisfies display_configuration_changed callback */
+}
+
+static void stub_pm_compute_clocks(void *handle)
+{
+	/* No-op: satisfies pm_compute_clocks callback */
+}
+
+static int stub_set_watermarks_for_clocks_ranges(void *handle, void *clock_ranges)
+{
+	return stub_dpm_ctx->set_watermarks_ret;
+}
+
+static int stub_display_clock_voltage_request(void *handle,
+					      struct pp_display_clock_request *clock)
+{
+	return stub_dpm_ctx->display_clock_voltage_ret;
+}
+
+static int stub_set_active_display_count(void *handle, uint32_t count)
+{
+	return stub_dpm_ctx->set_active_display_count_ret;
+}
+
+static int stub_set_min_deep_sleep_dcefclk(void *handle, uint32_t clock)
+{
+	return stub_dpm_ctx->set_min_deep_sleep_dcefclk_ret;
+}
+
+static int stub_set_hard_min_dcefclk_by_freq(void *handle, uint32_t clock)
+{
+	return 0;
+}
+
+static int stub_set_hard_min_fclk_by_freq(void *handle, uint32_t clock)
+{
+	return 0;
+}
+
+static int stub_notify_smu_enable_pwe(void *handle)
+{
+	return 0;
+}
+
+static int stub_display_disable_memory_clock_switch(void *handle,
+						    bool disable_memory_clock_switch)
+{
+	return stub_dpm_ctx->display_disable_memory_clock_switch_ret;
+}
+
+static int stub_get_max_sustainable_clocks_by_dc(void *handle,
+						 struct pp_smu_nv_clock_table *max_clocks)
+{
+	return stub_dpm_ctx->get_max_sustainable_ret;
+}
+
+static int stub_get_uclk_dpm_states(void *handle,
+				    unsigned int *clock_values_in_khz,
+				    unsigned int *num_states)
+{
+	return stub_dpm_ctx->get_uclk_dpm_ret;
+}
+
+static int stub_get_dpm_clock_table(void *handle, struct dpm_clocks *clock_table)
+{
+	return stub_dpm_ctx->get_dpm_clock_table_ret;
+}
+
+static const struct amd_pm_funcs stub_pp_funcs = {
+	.get_current_clocks = stub_get_current_clocks,
+	.get_clock_by_type = stub_get_clock_by_type,
+	.get_display_mode_validation_clocks = stub_get_display_mode_validation_clocks,
+	.get_clock_by_type_with_latency = stub_get_clock_by_type_with_latency,
+	.get_clock_by_type_with_voltage = stub_get_clock_by_type_with_voltage,
+	.display_configuration_changed = stub_display_configuration_change,
+	.pm_compute_clocks = stub_pm_compute_clocks,
+	.set_watermarks_for_clocks_ranges = stub_set_watermarks_for_clocks_ranges,
+	.display_clock_voltage_request = stub_display_clock_voltage_request,
+	.set_active_display_count = stub_set_active_display_count,
+	.set_min_deep_sleep_dcefclk = stub_set_min_deep_sleep_dcefclk,
+	.set_hard_min_dcefclk_by_freq = stub_set_hard_min_dcefclk_by_freq,
+	.set_hard_min_fclk_by_freq = stub_set_hard_min_fclk_by_freq,
+	.notify_smu_enable_pwe = stub_notify_smu_enable_pwe,
+	.display_disable_memory_clock_switch = stub_display_disable_memory_clock_switch,
+	.get_max_sustainable_clocks_by_dc = stub_get_max_sustainable_clocks_by_dc,
+	.get_uclk_dpm_states = stub_get_uclk_dpm_states,
+	.get_dpm_clock_table = stub_get_dpm_clock_table,
+};
+
+/**
+ * setup_stub_dpm - Initialize a stub DPM environment for testing
+ * @test: KUnit test context
+ * @adev: Pointer to amdgpu_device to configure
+ *
+ * Sets up adev->powerplay.pp_funcs and initializes adev->pm.mutex so that
+ * amdgpu_dpm_* functions can be safely called with stub callbacks.
+ */
+static void setup_stub_dpm(struct kunit *test, struct amdgpu_device *adev)
+{
+	stub_dpm_ctx = kunit_kzalloc(test, sizeof(*stub_dpm_ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stub_dpm_ctx);
+
+	adev->powerplay.pp_funcs = &stub_pp_funcs;
+	adev->powerplay.pp_handle = adev;
+	mutex_init(&adev->pm.mutex);
+}
+
+/* ---- Tests for get_default_clock_levels ---- */
+
+/**
+ * dm_test_default_clock_levels_display - Test display clock default levels
+ * @test: KUnit test context
+ *
+ * Verify that get_default_clock_levels populates 6 display clock levels
+ * with the expected frequencies in kHz.
+ */
+static void dm_test_default_clock_levels_display(struct kunit *test)
+{
+	struct dm_pp_clock_levels clks = { 0 };
+	uint32_t expected[] = { 300000, 400000, 496560, 626090, 685720, 757900 };
+	int i;
+
+	get_default_clock_levels(DM_PP_CLOCK_TYPE_DISPLAY_CLK, &clks);
+
+	KUNIT_EXPECT_EQ(test, clks.num_levels, 6U);
+	for (i = 0; i < 6; i++)
+		KUNIT_EXPECT_EQ(test, clks.clocks_in_khz[i], expected[i]);
+}
+
+/**
+ * dm_test_default_clock_levels_engine - Test engine clock default levels
+ * @test: KUnit test context
+ *
+ * Verify that get_default_clock_levels populates 6 engine clock levels
+ * with the expected frequencies in kHz.
+ */
+static void dm_test_default_clock_levels_engine(struct kunit *test)
+{
+	struct dm_pp_clock_levels clks = { 0 };
+	uint32_t expected[] = { 300000, 360000, 423530, 514290, 626090, 720000 };
+	int i;
+
+	get_default_clock_levels(DM_PP_CLOCK_TYPE_ENGINE_CLK, &clks);
+
+	KUNIT_EXPECT_EQ(test, clks.num_levels, 6U);
+	for (i = 0; i < 6; i++)
+		KUNIT_EXPECT_EQ(test, clks.clocks_in_khz[i], expected[i]);
+}
+
+/**
+ * dm_test_default_clock_levels_memory - Test memory clock default levels
+ * @test: KUnit test context
+ *
+ * Verify that get_default_clock_levels populates 2 memory clock levels
+ * with the expected frequencies in kHz.
+ */
+static void dm_test_default_clock_levels_memory(struct kunit *test)
+{
+	struct dm_pp_clock_levels clks = { 0 };
+
+	get_default_clock_levels(DM_PP_CLOCK_TYPE_MEMORY_CLK, &clks);
+
+	KUNIT_EXPECT_EQ(test, clks.num_levels, 2U);
+	KUNIT_EXPECT_EQ(test, clks.clocks_in_khz[0], 333000U);
+	KUNIT_EXPECT_EQ(test, clks.clocks_in_khz[1], 800000U);
+}
+
+/**
+ * dm_test_default_clock_levels_unknown - Test unknown clock type default
+ * @test: KUnit test context
+ *
+ * Verify that get_default_clock_levels sets num_levels to 0 for an
+ * unrecognized clock type.
+ */
+static void dm_test_default_clock_levels_unknown(struct kunit *test)
+{
+	struct dm_pp_clock_levels clks = { 0 };
+
+	get_default_clock_levels(DM_PP_CLOCK_TYPE_FCLK, &clks);
+
+	KUNIT_EXPECT_EQ(test, clks.num_levels, 0U);
+}
+
+/* ---- Tests for dc_to_pp_clock_type ---- */
+
+/**
+ * dm_test_dc_to_pp_clock_type_display - Test display clock type mapping
+ * @test: KUnit test context
+ *
+ * Verify DM_PP_CLOCK_TYPE_DISPLAY_CLK maps to amd_pp_disp_clock.
+ */
+static void dm_test_dc_to_pp_clock_type_display(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)dc_to_pp_clock_type(DM_PP_CLOCK_TYPE_DISPLAY_CLK),
+			(int)amd_pp_disp_clock);
+}
+
+/**
+ * dm_test_dc_to_pp_clock_type_engine - Test engine clock type mapping
+ * @test: KUnit test context
+ *
+ * Verify DM_PP_CLOCK_TYPE_ENGINE_CLK maps to amd_pp_sys_clock.
+ */
+static void dm_test_dc_to_pp_clock_type_engine(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)dc_to_pp_clock_type(DM_PP_CLOCK_TYPE_ENGINE_CLK),
+			(int)amd_pp_sys_clock);
+}
+
+/**
+ * dm_test_dc_to_pp_clock_type_memory - Test memory clock type mapping
+ * @test: KUnit test context
+ *
+ * Verify DM_PP_CLOCK_TYPE_MEMORY_CLK maps to amd_pp_mem_clock.
+ */
+static void dm_test_dc_to_pp_clock_type_memory(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)dc_to_pp_clock_type(DM_PP_CLOCK_TYPE_MEMORY_CLK),
+			(int)amd_pp_mem_clock);
+}
+
+/**
+ * dm_test_dc_to_pp_clock_type_dcefclk - Test DCEF clock type mapping
+ * @test: KUnit test context
+ *
+ * Verify DM_PP_CLOCK_TYPE_DCEFCLK maps to amd_pp_dcef_clock.
+ */
+static void dm_test_dc_to_pp_clock_type_dcefclk(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)dc_to_pp_clock_type(DM_PP_CLOCK_TYPE_DCEFCLK),
+			(int)amd_pp_dcef_clock);
+}
+
+/**
+ * dm_test_dc_to_pp_clock_type_dcfclk - Test DCF clock type mapping
+ * @test: KUnit test context
+ *
+ * Verify DM_PP_CLOCK_TYPE_DCFCLK maps to amd_pp_dcf_clock.
+ */
+static void dm_test_dc_to_pp_clock_type_dcfclk(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)dc_to_pp_clock_type(DM_PP_CLOCK_TYPE_DCFCLK),
+			(int)amd_pp_dcf_clock);
+}
+
+/**
+ * dm_test_dc_to_pp_clock_type_pixelclk - Test pixel clock type mapping
+ * @test: KUnit test context
+ *
+ * Verify DM_PP_CLOCK_TYPE_PIXELCLK maps to amd_pp_pixel_clock.
+ */
+static void dm_test_dc_to_pp_clock_type_pixelclk(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)dc_to_pp_clock_type(DM_PP_CLOCK_TYPE_PIXELCLK),
+			(int)amd_pp_pixel_clock);
+}
+
+/**
+ * dm_test_dc_to_pp_clock_type_fclk - Test FCLK type mapping
+ * @test: KUnit test context
+ *
+ * Verify DM_PP_CLOCK_TYPE_FCLK maps to amd_pp_f_clock.
+ */
+static void dm_test_dc_to_pp_clock_type_fclk(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)dc_to_pp_clock_type(DM_PP_CLOCK_TYPE_FCLK),
+			(int)amd_pp_f_clock);
+}
+
+/**
+ * dm_test_dc_to_pp_clock_type_phyclk - Test display PHY clock type mapping
+ * @test: KUnit test context
+ *
+ * Verify DM_PP_CLOCK_TYPE_DISPLAYPHYCLK maps to amd_pp_phy_clock.
+ */
+static void dm_test_dc_to_pp_clock_type_phyclk(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)dc_to_pp_clock_type(DM_PP_CLOCK_TYPE_DISPLAYPHYCLK),
+			(int)amd_pp_phy_clock);
+}
+
+/**
+ * dm_test_dc_to_pp_clock_type_dppclk - Test DPP clock type mapping
+ * @test: KUnit test context
+ *
+ * Verify DM_PP_CLOCK_TYPE_DPPCLK maps to amd_pp_dpp_clock.
+ */
+static void dm_test_dc_to_pp_clock_type_dppclk(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)dc_to_pp_clock_type(DM_PP_CLOCK_TYPE_DPPCLK),
+			(int)amd_pp_dpp_clock);
+}
+
+/**
+ * dm_test_dc_to_pp_clock_type_invalid - Test invalid clock type mapping
+ * @test: KUnit test context
+ *
+ * Verify that an invalid clock type value maps to 0.
+ */
+static void dm_test_dc_to_pp_clock_type_invalid(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, (int)dc_to_pp_clock_type(0), 0);
+}
+
+/* ---- Tests for pp_to_dc_clock_levels ---- */
+
+/**
+ * dm_test_pp_to_dc_clock_levels_within_limit - Test normal copy within limit
+ * @test: KUnit test context
+ *
+ * Verify that pp_to_dc_clock_levels correctly copies clock values when the
+ * count is within DM_PP_MAX_CLOCK_LEVELS.
+ */
+static void dm_test_pp_to_dc_clock_levels_within_limit(struct kunit *test)
+{
+	struct amd_pp_clocks pp_clks = {};
+	struct dm_pp_clock_levels dc_clks = {};
+
+	pp_clks.count = 3;
+	pp_clks.clock[0] = 300000;
+	pp_clks.clock[1] = 500000;
+	pp_clks.clock[2] = 700000;
+
+	pp_to_dc_clock_levels(&pp_clks, &dc_clks, DM_PP_CLOCK_TYPE_ENGINE_CLK);
+
+	KUNIT_EXPECT_EQ(test, dc_clks.num_levels, 3U);
+	KUNIT_EXPECT_EQ(test, dc_clks.clocks_in_khz[0], 300000U);
+	KUNIT_EXPECT_EQ(test, dc_clks.clocks_in_khz[1], 500000U);
+	KUNIT_EXPECT_EQ(test, dc_clks.clocks_in_khz[2], 700000U);
+}
+
+/**
+ * dm_test_pp_to_dc_clock_levels_caps_at_max - Test count capping at max
+ * @test: KUnit test context
+ *
+ * Verify that pp_to_dc_clock_levels caps num_levels at DM_PP_MAX_CLOCK_LEVELS
+ * when the input count exceeds the maximum.
+ */
+static void dm_test_pp_to_dc_clock_levels_caps_at_max(struct kunit *test)
+{
+	struct amd_pp_clocks pp_clks = {};
+	struct dm_pp_clock_levels dc_clks = {};
+	uint32_t i;
+
+	pp_clks.count = DM_PP_MAX_CLOCK_LEVELS + 1;
+	for (i = 0; i < DM_PP_MAX_CLOCK_LEVELS; i++)
+		pp_clks.clock[i] = (i + 1) * 100000;
+
+	pp_to_dc_clock_levels(&pp_clks, &dc_clks, DM_PP_CLOCK_TYPE_ENGINE_CLK);
+
+	KUNIT_EXPECT_EQ(test, dc_clks.num_levels, (uint32_t)DM_PP_MAX_CLOCK_LEVELS);
+}
+
+/* ---- Tests for pp_to_dc_clock_levels_with_latency ---- */
+
+/**
+ * dm_test_pp_to_dc_clock_levels_latency_within_limit - Test normal copy
+ * @test: KUnit test context
+ *
+ * Verify that pp_to_dc_clock_levels_with_latency correctly copies clock
+ * and latency values when count is within limits.
+ */
+static void dm_test_pp_to_dc_clock_levels_latency_within_limit(struct kunit *test)
+{
+	struct pp_clock_levels_with_latency pp_clks = {};
+	struct dm_pp_clock_levels_with_latency dc_clks = {};
+
+	pp_clks.num_levels = 2;
+	pp_clks.data[0].clocks_in_khz = 400000;
+	pp_clks.data[0].latency_in_us = 10;
+	pp_clks.data[1].clocks_in_khz = 800000;
+	pp_clks.data[1].latency_in_us = 20;
+
+	pp_to_dc_clock_levels_with_latency(&pp_clks, &dc_clks,
+					   DM_PP_CLOCK_TYPE_ENGINE_CLK);
+
+	KUNIT_EXPECT_EQ(test, dc_clks.num_levels, 2U);
+	KUNIT_EXPECT_EQ(test, dc_clks.data[0].clocks_in_khz, 400000U);
+	KUNIT_EXPECT_EQ(test, dc_clks.data[0].latency_in_us, 10U);
+	KUNIT_EXPECT_EQ(test, dc_clks.data[1].clocks_in_khz, 800000U);
+	KUNIT_EXPECT_EQ(test, dc_clks.data[1].latency_in_us, 20U);
+}
+
+/**
+ * dm_test_pp_to_dc_clock_levels_latency_caps_at_max - Test count capping
+ * @test: KUnit test context
+ *
+ * Verify that pp_to_dc_clock_levels_with_latency caps num_levels at
+ * DM_PP_MAX_CLOCK_LEVELS when input exceeds the maximum.
+ */
+static void dm_test_pp_to_dc_clock_levels_latency_caps_at_max(struct kunit *test)
+{
+	struct pp_clock_levels_with_latency pp_clks = {};
+	struct dm_pp_clock_levels_with_latency dc_clks = {};
+
+	pp_clks.num_levels = DM_PP_MAX_CLOCK_LEVELS + 1;
+
+	pp_to_dc_clock_levels_with_latency(&pp_clks, &dc_clks,
+					   DM_PP_CLOCK_TYPE_ENGINE_CLK);
+
+	KUNIT_EXPECT_EQ(test, dc_clks.num_levels, (uint32_t)DM_PP_MAX_CLOCK_LEVELS);
+}
+
+/* ---- Tests for pp_to_dc_clock_levels_with_voltage ---- */
+
+/**
+ * dm_test_pp_to_dc_clock_levels_voltage_within_limit - Test normal copy
+ * @test: KUnit test context
+ *
+ * Verify that pp_to_dc_clock_levels_with_voltage correctly copies clock
+ * and voltage values when count is within limits.
+ */
+static void dm_test_pp_to_dc_clock_levels_voltage_within_limit(struct kunit *test)
+{
+	struct pp_clock_levels_with_voltage pp_clks = {};
+	struct dm_pp_clock_levels_with_voltage dc_clks = {};
+
+	pp_clks.num_levels = 2;
+	pp_clks.data[0].clocks_in_khz = 300000;
+	pp_clks.data[0].voltage_in_mv = 800;
+	pp_clks.data[1].clocks_in_khz = 600000;
+	pp_clks.data[1].voltage_in_mv = 950;
+
+	pp_to_dc_clock_levels_with_voltage(&pp_clks, &dc_clks,
+					   DM_PP_CLOCK_TYPE_MEMORY_CLK);
+
+	KUNIT_EXPECT_EQ(test, dc_clks.num_levels, 2U);
+	KUNIT_EXPECT_EQ(test, dc_clks.data[0].clocks_in_khz, 300000U);
+	KUNIT_EXPECT_EQ(test, dc_clks.data[0].voltage_in_mv, 800U);
+	KUNIT_EXPECT_EQ(test, dc_clks.data[1].clocks_in_khz, 600000U);
+	KUNIT_EXPECT_EQ(test, dc_clks.data[1].voltage_in_mv, 950U);
+}
+
+/**
+ * dm_test_pp_to_dc_clock_levels_voltage_caps_at_max - Test count capping
+ * @test: KUnit test context
+ *
+ * Verify that pp_to_dc_clock_levels_with_voltage caps num_levels at
+ * DM_PP_MAX_CLOCK_LEVELS when input exceeds the maximum.
+ */
+static void dm_test_pp_to_dc_clock_levels_voltage_caps_at_max(struct kunit *test)
+{
+	struct pp_clock_levels_with_voltage pp_clks = {};
+	struct dm_pp_clock_levels_with_voltage dc_clks = {};
+
+	pp_clks.num_levels = DM_PP_MAX_CLOCK_LEVELS + 1;
+
+	pp_to_dc_clock_levels_with_voltage(&pp_clks, &dc_clks,
+					   DM_PP_CLOCK_TYPE_MEMORY_CLK);
+
+	KUNIT_EXPECT_EQ(test, dc_clks.num_levels, (uint32_t)DM_PP_MAX_CLOCK_LEVELS);
+}
+
+/* ---- Tests for dm_pp_get_funcs ---- */
+
+/**
+ * dm_test_get_funcs_rv - Test Raven PP SMU function table setup
+ * @test: KUnit test context
+ *
+ * Verify that DCN 1.0 initializes the Raven SMU function table and stores
+ * the DC context in the PP SMU handle.
+ */
+static void dm_test_get_funcs_rv(struct kunit *test)
+{
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu_funcs *funcs = kunit_kzalloc(test, sizeof(*funcs), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	KUNIT_ASSERT_NOT_NULL(test, funcs);
+
+	ctx->dce_version = DCN_VERSION_1_0;
+
+	dm_pp_get_funcs(ctx, funcs);
+
+	KUNIT_EXPECT_EQ(test, funcs->ctx.ver, PP_SMU_VER_RV);
+	KUNIT_EXPECT_PTR_EQ(test, funcs->rv_funcs.pp_smu.dm, ctx);
+	KUNIT_EXPECT_TRUE(test, funcs->rv_funcs.set_wm_ranges != NULL);
+	KUNIT_EXPECT_TRUE(test, funcs->rv_funcs.set_pme_wa_enable != NULL);
+	KUNIT_EXPECT_TRUE(test, funcs->rv_funcs.set_display_count != NULL);
+	KUNIT_EXPECT_TRUE(test, funcs->rv_funcs.set_min_deep_sleep_dcfclk != NULL);
+	KUNIT_EXPECT_TRUE(test, funcs->rv_funcs.set_hard_min_dcfclk_by_freq != NULL);
+	KUNIT_EXPECT_TRUE(test, funcs->rv_funcs.set_hard_min_fclk_by_freq != NULL);
+	KUNIT_EXPECT_FALSE(test, funcs->rv_funcs.set_hard_min_socclk_by_freq != NULL);
+}
+
+/**
+ * dm_test_get_funcs_rv_101 - Test DCN 1.01 Raven PP SMU setup
+ * @test: KUnit test context
+ *
+ * Verify that DCN 1.01 uses the same Raven SMU function table as DCN 1.0.
+ */
+static void dm_test_get_funcs_rv_101(struct kunit *test)
+{
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu_funcs *funcs = kunit_kzalloc(test, sizeof(*funcs), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	KUNIT_ASSERT_NOT_NULL(test, funcs);
+
+	ctx->dce_version = DCN_VERSION_1_01;
+
+	dm_pp_get_funcs(ctx, funcs);
+
+	KUNIT_EXPECT_EQ(test, funcs->ctx.ver, PP_SMU_VER_RV);
+	KUNIT_EXPECT_PTR_EQ(test, funcs->rv_funcs.pp_smu.dm, ctx);
+	KUNIT_EXPECT_TRUE(test, funcs->rv_funcs.set_display_count != NULL);
+}
+
+/**
+ * dm_test_get_funcs_nv - Test Navi PP SMU function table setup
+ * @test: KUnit test context
+ *
+ * Verify that DCN 2.0 initializes the Navi SMU function table and leaves the
+ * unsupported PME workaround callback unset.
+ */
+static void dm_test_get_funcs_nv(struct kunit *test)
+{
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu_funcs *funcs = kunit_kzalloc(test, sizeof(*funcs), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	KUNIT_ASSERT_NOT_NULL(test, funcs);
+
+	ctx->dce_version = DCN_VERSION_2_0;
+
+	dm_pp_get_funcs(ctx, funcs);
+
+	KUNIT_EXPECT_EQ(test, funcs->ctx.ver, PP_SMU_VER_NV);
+	KUNIT_EXPECT_PTR_EQ(test, funcs->nv_funcs.pp_smu.dm, ctx);
+	KUNIT_EXPECT_TRUE(test, funcs->nv_funcs.set_display_count != NULL);
+	KUNIT_EXPECT_TRUE(test, funcs->nv_funcs.set_hard_min_dcfclk_by_freq != NULL);
+	KUNIT_EXPECT_TRUE(test, funcs->nv_funcs.set_min_deep_sleep_dcfclk != NULL);
+	KUNIT_EXPECT_TRUE(test, funcs->nv_funcs.set_voltage_by_freq != NULL);
+	KUNIT_EXPECT_TRUE(test, funcs->nv_funcs.set_wm_ranges != NULL);
+	KUNIT_EXPECT_FALSE(test, funcs->nv_funcs.set_pme_wa_enable != NULL);
+	KUNIT_EXPECT_TRUE(test, funcs->nv_funcs.set_hard_min_uclk_by_freq != NULL);
+	KUNIT_EXPECT_TRUE(test, funcs->nv_funcs.get_maximum_sustainable_clocks != NULL);
+	KUNIT_EXPECT_TRUE(test, funcs->nv_funcs.get_uclk_dpm_states != NULL);
+	KUNIT_EXPECT_TRUE(test, funcs->nv_funcs.set_pstate_handshake_support != NULL);
+}
+
+/**
+ * dm_test_get_funcs_rn - Test Renoir PP SMU function table setup
+ * @test: KUnit test context
+ *
+ * Verify that DCN 2.1 initializes the Renoir SMU function table.
+ */
+static void dm_test_get_funcs_rn(struct kunit *test)
+{
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu_funcs *funcs = kunit_kzalloc(test, sizeof(*funcs), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	KUNIT_ASSERT_NOT_NULL(test, funcs);
+
+	ctx->dce_version = DCN_VERSION_2_1;
+
+	dm_pp_get_funcs(ctx, funcs);
+
+	KUNIT_EXPECT_EQ(test, funcs->ctx.ver, PP_SMU_VER_RN);
+	KUNIT_EXPECT_PTR_EQ(test, funcs->rn_funcs.pp_smu.dm, ctx);
+	KUNIT_EXPECT_TRUE(test, funcs->rn_funcs.set_wm_ranges != NULL);
+	KUNIT_EXPECT_TRUE(test, funcs->rn_funcs.get_dpm_clock_table != NULL);
+}
+
+/**
+ * dm_test_get_funcs_unsupported - Test unsupported DCE version handling
+ * @test: KUnit test context
+ *
+ * Verify that unsupported DCE versions do not initialize a PP SMU version or
+ * function table callbacks.
+ */
+static void dm_test_get_funcs_unsupported(struct kunit *test)
+{
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu_funcs *funcs = kunit_kzalloc(test, sizeof(*funcs), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	KUNIT_ASSERT_NOT_NULL(test, funcs);
+
+	ctx->dce_version = DCE_VERSION_MAX;
+
+	dm_pp_get_funcs(ctx, funcs);
+
+	KUNIT_EXPECT_EQ(test, funcs->ctx.ver, PP_SMU_UNSUPPORTED);
+	KUNIT_EXPECT_FALSE(test, funcs->rv_funcs.set_wm_ranges != NULL);
+}
+
+/* ---- Tests for amdgpu_device-backed entry points ---- */
+
+/**
+ * dm_test_apply_display_requirements_dpm_disabled - Test DPM-disabled path
+ * @test: KUnit test context
+ *
+ * Verify that dm_pp_apply_display_requirements returns true without touching
+ * the display configuration when DPM is disabled.
+ */
+static void dm_test_apply_display_requirements_dpm_disabled(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct dm_pp_display_configuration cfg = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	adev->pm.dpm_enabled = false;
+	ctx->driver_context = adev;
+
+	KUNIT_EXPECT_TRUE(test, dm_pp_apply_display_requirements(ctx, &cfg));
+}
+
+/**
+ * dm_test_apply_clock_for_voltage_invalid_type - Test invalid clock type path
+ * @test: KUnit test context
+ *
+ * Verify that dm_pp_apply_clock_for_voltage_request returns false for a clock
+ * type that does not map to a valid PP clock type, taking the early-return
+ * path before any SMU request is issued.
+ */
+static void dm_test_apply_clock_for_voltage_invalid_type(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct dm_pp_clock_for_voltage_req req = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	ctx->driver_context = adev;
+	req.clk_type = (enum dm_pp_clock_type)0xffff;
+	req.clocks_in_khz = 500000;
+
+	KUNIT_EXPECT_FALSE(test, dm_pp_apply_clock_for_voltage_request(ctx, &req));
+}
+
+/* ---- Tests for build_pm_display_cfg ---- */
+
+/**
+ * dm_test_build_pm_display_cfg_scalar_fields - Test scalar field translation
+ * @test: KUnit test context
+ *
+ * Verify that build_pm_display_cfg copies the pass-through fields and applies
+ * the /10 (10 kHz) scaling, and sets the fixed constants.
+ */
+static void dm_test_build_pm_display_cfg_scalar_fields(struct kunit *test)
+{
+	struct amd_pp_display_configuration *pm =
+		kunit_kzalloc(test, sizeof(*pm), GFP_KERNEL);
+	struct dm_pp_display_configuration *pp =
+		kunit_kzalloc(test, sizeof(*pp), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, pm);
+	KUNIT_ASSERT_NOT_NULL(test, pp);
+
+	pp->cpu_cc6_disable = true;
+	pp->cpu_pstate_disable = true;
+	pp->cpu_pstate_separation_time = 7;
+	pp->nb_pstate_switch_disable = true;
+	pp->display_count = 2;
+	pp->min_engine_clock_khz = 300000;
+	pp->min_engine_clock_deep_sleep_khz = 50000;
+	pp->min_memory_clock_khz = 800000;
+	pp->min_dcfclock_khz = 600000;
+	pp->all_displays_in_sync = true;
+	pp->avail_mclk_switch_time_us = 11;
+	pp->disp_clk_khz = 400000;
+	pp->avail_mclk_switch_time_in_disp_active_us = 13;
+	pp->crtc_index = 3;
+	pp->line_time_in_us = 17;
+	pp->disp_configs[0].v_refresh = 60;
+
+	build_pm_display_cfg(pm, pp);
+
+	KUNIT_EXPECT_TRUE(test, pm->cpu_cc6_disable);
+	KUNIT_EXPECT_TRUE(test, pm->cpu_pstate_disable);
+	KUNIT_EXPECT_EQ(test, pm->cpu_pstate_separation_time, 7);
+	KUNIT_EXPECT_TRUE(test, pm->nb_pstate_switch_disable);
+	KUNIT_EXPECT_EQ(test, pm->num_display, 2);
+	KUNIT_EXPECT_EQ(test, pm->num_path_including_non_display, 2);
+	KUNIT_EXPECT_EQ(test, pm->min_core_set_clock, 30000);
+	KUNIT_EXPECT_EQ(test, pm->min_core_set_clock_in_sr, 5000);
+	KUNIT_EXPECT_EQ(test, pm->min_mem_set_clock, 80000);
+	KUNIT_EXPECT_EQ(test, pm->min_dcef_deep_sleep_set_clk, 5000);
+	KUNIT_EXPECT_EQ(test, pm->min_dcef_set_clk, 60000);
+	KUNIT_EXPECT_TRUE(test, pm->multi_monitor_in_sync);
+	KUNIT_EXPECT_EQ(test, pm->min_vblank_time, 11);
+	KUNIT_EXPECT_EQ(test, pm->display_clk, 40000);
+	KUNIT_EXPECT_EQ(test, pm->dce_tolerable_mclk_in_active_latency, 13);
+	KUNIT_EXPECT_EQ(test, pm->crtc_index, 3);
+	KUNIT_EXPECT_EQ(test, pm->line_time_in_us, 17);
+	KUNIT_EXPECT_EQ(test, pm->vrefresh, 60);
+	KUNIT_EXPECT_EQ(test, pm->crossfire_display_index, -1);
+	KUNIT_EXPECT_EQ(test, pm->min_bus_bandwidth, 0);
+}
+
+/**
+ * dm_test_build_pm_display_cfg_per_display - Test per-display translation
+ * @test: KUnit test context
+ *
+ * Verify that build_pm_display_cfg maps each display config, applying the
+ * controller_id = pipe_idx + 1 offset and copying the pixel clock.
+ */
+static void dm_test_build_pm_display_cfg_per_display(struct kunit *test)
+{
+	struct amd_pp_display_configuration *pm =
+		kunit_kzalloc(test, sizeof(*pm), GFP_KERNEL);
+	struct dm_pp_display_configuration *pp =
+		kunit_kzalloc(test, sizeof(*pp), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, pm);
+	KUNIT_ASSERT_NOT_NULL(test, pp);
+
+	pp->display_count = 2;
+	pp->disp_configs[0].pipe_idx = 0;
+	pp->disp_configs[0].pixel_clock = 148500;
+	pp->disp_configs[1].pipe_idx = 4;
+	pp->disp_configs[1].pixel_clock = 297000;
+
+	build_pm_display_cfg(pm, pp);
+
+	KUNIT_EXPECT_EQ(test, pm->displays[0].controller_id, 1);
+	KUNIT_EXPECT_EQ(test, pm->displays[0].pixel_clock, 148500);
+	KUNIT_EXPECT_EQ(test, pm->displays[1].controller_id, 5);
+	KUNIT_EXPECT_EQ(test, pm->displays[1].pixel_clock, 297000);
+}
+
+/* ---- Tests for build_wm_clock_ranges_soc15 ---- */
+
+/**
+ * dm_test_build_wm_clock_ranges_dmif - Test reader (DMIF) watermark sets
+ * @test: KUnit test context
+ *
+ * Verify that build_wm_clock_ranges_soc15 copies the reader set count,
+ * maps wm_inst to wm_set_id (clamping instances > 3 to WM_SET_A), and
+ * converts every clock from MHz to kHz (x1000) into the DMIF clock ranges.
+ */
+static void dm_test_build_wm_clock_ranges_dmif(struct kunit *test)
+{
+	struct pp_smu_wm_range_sets *ranges =
+		kunit_kzalloc(test, sizeof(*ranges), GFP_KERNEL);
+	struct dm_pp_wm_sets_with_clock_ranges_soc15 *wm =
+		kunit_kzalloc(test, sizeof(*wm), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, ranges);
+	KUNIT_ASSERT_NOT_NULL(test, wm);
+
+	ranges->num_reader_wm_sets = 2;
+	/* set 0: wm_inst within range -> preserved */
+	ranges->reader_wm_sets[0].wm_inst = 2;
+	ranges->reader_wm_sets[0].max_drain_clk_mhz = 600;
+	ranges->reader_wm_sets[0].min_drain_clk_mhz = 300;
+	ranges->reader_wm_sets[0].max_fill_clk_mhz = 800;
+	ranges->reader_wm_sets[0].min_fill_clk_mhz = 400;
+	/* set 1: wm_inst > 3 -> clamped to WM_SET_A */
+	ranges->reader_wm_sets[1].wm_inst = 5;
+	ranges->reader_wm_sets[1].max_drain_clk_mhz = 700;
+	ranges->reader_wm_sets[1].min_drain_clk_mhz = 350;
+	ranges->reader_wm_sets[1].max_fill_clk_mhz = 900;
+	ranges->reader_wm_sets[1].min_fill_clk_mhz = 450;
+
+	build_wm_clock_ranges_soc15(ranges, wm);
+
+	KUNIT_EXPECT_EQ(test, wm->num_wm_dmif_sets, 2U);
+	KUNIT_EXPECT_EQ(test, wm->num_wm_mcif_sets, 0U);
+
+	KUNIT_EXPECT_EQ(test, wm->wm_dmif_clocks_ranges[0].wm_set_id, WM_SET_C);
+	KUNIT_EXPECT_EQ(test, wm->wm_dmif_clocks_ranges[0].wm_max_dcfclk_clk_in_khz, 600000U);
+	KUNIT_EXPECT_EQ(test, wm->wm_dmif_clocks_ranges[0].wm_min_dcfclk_clk_in_khz, 300000U);
+	KUNIT_EXPECT_EQ(test, wm->wm_dmif_clocks_ranges[0].wm_max_mem_clk_in_khz, 800000U);
+	KUNIT_EXPECT_EQ(test, wm->wm_dmif_clocks_ranges[0].wm_min_mem_clk_in_khz, 400000U);
+
+	KUNIT_EXPECT_EQ(test, wm->wm_dmif_clocks_ranges[1].wm_set_id, WM_SET_A);
+	KUNIT_EXPECT_EQ(test, wm->wm_dmif_clocks_ranges[1].wm_max_dcfclk_clk_in_khz, 700000U);
+	KUNIT_EXPECT_EQ(test, wm->wm_dmif_clocks_ranges[1].wm_min_dcfclk_clk_in_khz, 350000U);
+	KUNIT_EXPECT_EQ(test, wm->wm_dmif_clocks_ranges[1].wm_max_mem_clk_in_khz, 900000U);
+	KUNIT_EXPECT_EQ(test, wm->wm_dmif_clocks_ranges[1].wm_min_mem_clk_in_khz, 450000U);
+}
+
+/**
+ * dm_test_build_wm_clock_ranges_mcif - Test writer (MCIF) watermark sets
+ * @test: KUnit test context
+ *
+ * Verify that build_wm_clock_ranges_soc15 copies the writer set count and
+ * maps the writer clocks into the MCIF ranges: fill clocks become socclk
+ * and drain clocks become mem clk, each converted from MHz to kHz.
+ */
+static void dm_test_build_wm_clock_ranges_mcif(struct kunit *test)
+{
+	struct pp_smu_wm_range_sets *ranges =
+		kunit_kzalloc(test, sizeof(*ranges), GFP_KERNEL);
+	struct dm_pp_wm_sets_with_clock_ranges_soc15 *wm =
+		kunit_kzalloc(test, sizeof(*wm), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, ranges);
+	KUNIT_ASSERT_NOT_NULL(test, wm);
+
+	ranges->num_writer_wm_sets = 2;
+	ranges->writer_wm_sets[0].wm_inst = 1;
+	ranges->writer_wm_sets[0].max_fill_clk_mhz = 1200;
+	ranges->writer_wm_sets[0].min_fill_clk_mhz = 600;
+	ranges->writer_wm_sets[0].max_drain_clk_mhz = 1000;
+	ranges->writer_wm_sets[0].min_drain_clk_mhz = 500;
+	/* set 1: wm_inst > 3 -> clamped to WM_SET_A */
+	ranges->writer_wm_sets[1].wm_inst = 5;
+	ranges->writer_wm_sets[1].max_fill_clk_mhz = 1400;
+	ranges->writer_wm_sets[1].min_fill_clk_mhz = 700;
+	ranges->writer_wm_sets[1].max_drain_clk_mhz = 1100;
+	ranges->writer_wm_sets[1].min_drain_clk_mhz = 550;
+
+	build_wm_clock_ranges_soc15(ranges, wm);
+
+	KUNIT_EXPECT_EQ(test, wm->num_wm_dmif_sets, 0U);
+	KUNIT_EXPECT_EQ(test, wm->num_wm_mcif_sets, 2U);
+
+	KUNIT_EXPECT_EQ(test, wm->wm_mcif_clocks_ranges[0].wm_set_id, WM_SET_B);
+	KUNIT_EXPECT_EQ(test, wm->wm_mcif_clocks_ranges[0].wm_max_socclk_clk_in_khz, 1200000U);
+	KUNIT_EXPECT_EQ(test, wm->wm_mcif_clocks_ranges[0].wm_min_socclk_clk_in_khz, 600000U);
+	KUNIT_EXPECT_EQ(test, wm->wm_mcif_clocks_ranges[0].wm_max_mem_clk_in_khz, 1000000U);
+	KUNIT_EXPECT_EQ(test, wm->wm_mcif_clocks_ranges[0].wm_min_mem_clk_in_khz, 500000U);
+
+	KUNIT_EXPECT_EQ(test, wm->wm_mcif_clocks_ranges[1].wm_set_id, WM_SET_A);
+	KUNIT_EXPECT_EQ(test, wm->wm_mcif_clocks_ranges[1].wm_max_socclk_clk_in_khz, 1400000U);
+	KUNIT_EXPECT_EQ(test, wm->wm_mcif_clocks_ranges[1].wm_min_socclk_clk_in_khz, 700000U);
+	KUNIT_EXPECT_EQ(test, wm->wm_mcif_clocks_ranges[1].wm_max_mem_clk_in_khz, 1100000U);
+	KUNIT_EXPECT_EQ(test, wm->wm_mcif_clocks_ranges[1].wm_min_mem_clk_in_khz, 550000U);
+}
+
+/* ---- Tests for cap_clock_levels_to_validation ---- */
+
+/**
+ * dm_test_cap_clock_levels_engine_caps - Test engine clock level capping
+ * @test: KUnit test context
+ *
+ * Verify that for engine clocks, num_levels is reduced to the index of the
+ * first level whose frequency exceeds the engine validation clock.
+ */
+static void dm_test_cap_clock_levels_engine_caps(struct kunit *test)
+{
+	struct dm_pp_clock_levels clks = { 0 };
+	struct amd_pp_simple_clock_info validation = {
+		.engine_max_clock = 450000,
+		.memory_max_clock = 800000,
+	};
+
+	clks.num_levels = 3;
+	clks.clocks_in_khz[0] = 300000;
+	clks.clocks_in_khz[1] = 400000;
+	clks.clocks_in_khz[2] = 500000;
+
+	cap_clock_levels_to_validation(&clks, DM_PP_CLOCK_TYPE_ENGINE_CLK, &validation);
+
+	KUNIT_EXPECT_EQ(test, clks.num_levels, 2U);
+}
+
+/**
+ * dm_test_cap_clock_levels_engine_first_exceeds - Test floor of one level
+ * @test: KUnit test context
+ *
+ * Verify that when the very first engine clock level already exceeds the
+ * validation clock, num_levels is clamped to 1 rather than 0.
+ */
+static void dm_test_cap_clock_levels_engine_first_exceeds(struct kunit *test)
+{
+	struct dm_pp_clock_levels clks = { 0 };
+	struct amd_pp_simple_clock_info validation = {
+		.engine_max_clock = 100000,
+		.memory_max_clock = 800000,
+	};
+
+	clks.num_levels = 3;
+	clks.clocks_in_khz[0] = 300000;
+	clks.clocks_in_khz[1] = 400000;
+	clks.clocks_in_khz[2] = 500000;
+
+	cap_clock_levels_to_validation(&clks, DM_PP_CLOCK_TYPE_ENGINE_CLK, &validation);
+
+	KUNIT_EXPECT_EQ(test, clks.num_levels, 1U);
+}
+
+/**
+ * dm_test_cap_clock_levels_memory_caps - Test memory clock level capping
+ * @test: KUnit test context
+ *
+ * Verify that for memory clocks, num_levels is reduced based on the memory
+ * validation clock (and is unaffected by the engine validation clock).
+ */
+static void dm_test_cap_clock_levels_memory_caps(struct kunit *test)
+{
+	struct dm_pp_clock_levels clks = { 0 };
+	struct amd_pp_simple_clock_info validation = {
+		.engine_max_clock = 100000,
+		.memory_max_clock = 700000,
+	};
+
+	clks.num_levels = 2;
+	clks.clocks_in_khz[0] = 333000;
+	clks.clocks_in_khz[1] = 800000;
+
+	cap_clock_levels_to_validation(&clks, DM_PP_CLOCK_TYPE_MEMORY_CLK, &validation);
+
+	KUNIT_EXPECT_EQ(test, clks.num_levels, 1U);
+}
+
+/**
+ * dm_test_cap_clock_levels_within_limit - Test no capping when within limit
+ * @test: KUnit test context
+ *
+ * Verify that num_levels is left unchanged when no level exceeds the
+ * validation clock.
+ */
+static void dm_test_cap_clock_levels_within_limit(struct kunit *test)
+{
+	struct dm_pp_clock_levels clks = { 0 };
+	struct amd_pp_simple_clock_info validation = {
+		.engine_max_clock = 999000,
+		.memory_max_clock = 999000,
+	};
+
+	clks.num_levels = 3;
+	clks.clocks_in_khz[0] = 300000;
+	clks.clocks_in_khz[1] = 400000;
+	clks.clocks_in_khz[2] = 500000;
+
+	cap_clock_levels_to_validation(&clks, DM_PP_CLOCK_TYPE_ENGINE_CLK, &validation);
+
+	KUNIT_EXPECT_EQ(test, clks.num_levels, 3U);
+}
+
+/**
+ * dm_test_cap_clock_levels_other_type - Test non-engine/memory types ignored
+ * @test: KUnit test context
+ *
+ * Verify that for clock types other than engine or memory, num_levels is
+ * left unchanged regardless of the validation clocks.
+ */
+static void dm_test_cap_clock_levels_other_type(struct kunit *test)
+{
+	struct dm_pp_clock_levels clks = { 0 };
+	struct amd_pp_simple_clock_info validation = {
+		.engine_max_clock = 1,
+		.memory_max_clock = 1,
+	};
+
+	clks.num_levels = 3;
+	clks.clocks_in_khz[0] = 300000;
+	clks.clocks_in_khz[1] = 400000;
+	clks.clocks_in_khz[2] = 500000;
+
+	cap_clock_levels_to_validation(&clks, DM_PP_CLOCK_TYPE_DISPLAY_CLK, &validation);
+
+	KUNIT_EXPECT_EQ(test, clks.num_levels, 3U);
+}
+
+/* ---- Tests for pp_smu_nv_clock_id_to_pp ---- */
+
+/**
+ * dm_test_nv_clock_id_dispclk - Test DISPCLK id mapping
+ * @test: KUnit test context
+ *
+ * Verify that PP_SMU_NV_DISPCLK maps to amd_pp_disp_clock and returns true.
+ */
+static void dm_test_nv_clock_id_dispclk(struct kunit *test)
+{
+	enum amd_pp_clock_type clock_type = amd_pp_mem_clock;
+
+	KUNIT_EXPECT_TRUE(test, pp_smu_nv_clock_id_to_pp(PP_SMU_NV_DISPCLK, &clock_type));
+	KUNIT_EXPECT_EQ(test, clock_type, amd_pp_disp_clock);
+}
+
+/**
+ * dm_test_nv_clock_id_phyclk - Test PHYCLK id mapping
+ * @test: KUnit test context
+ *
+ * Verify that PP_SMU_NV_PHYCLK maps to amd_pp_phy_clock and returns true.
+ */
+static void dm_test_nv_clock_id_phyclk(struct kunit *test)
+{
+	enum amd_pp_clock_type clock_type = amd_pp_mem_clock;
+
+	KUNIT_EXPECT_TRUE(test, pp_smu_nv_clock_id_to_pp(PP_SMU_NV_PHYCLK, &clock_type));
+	KUNIT_EXPECT_EQ(test, clock_type, amd_pp_phy_clock);
+}
+
+/**
+ * dm_test_nv_clock_id_pixelclk - Test PIXELCLK id mapping
+ * @test: KUnit test context
+ *
+ * Verify that PP_SMU_NV_PIXELCLK maps to amd_pp_pixel_clock and returns true.
+ */
+static void dm_test_nv_clock_id_pixelclk(struct kunit *test)
+{
+	enum amd_pp_clock_type clock_type = amd_pp_mem_clock;
+
+	KUNIT_EXPECT_TRUE(test, pp_smu_nv_clock_id_to_pp(PP_SMU_NV_PIXELCLK, &clock_type));
+	KUNIT_EXPECT_EQ(test, clock_type, amd_pp_pixel_clock);
+}
+
+/**
+ * dm_test_nv_clock_id_invalid - Test unknown id is rejected
+ * @test: KUnit test context
+ *
+ * Verify that an unknown clock id returns false and leaves the output
+ * clock_type untouched, guarding against the previously uninitialized path.
+ */
+static void dm_test_nv_clock_id_invalid(struct kunit *test)
+{
+	enum amd_pp_clock_type clock_type = amd_pp_dcef_clock;
+
+	KUNIT_EXPECT_FALSE(test, pp_smu_nv_clock_id_to_pp((enum pp_smu_nv_clock_id)0xff,
+							  &clock_type));
+	KUNIT_EXPECT_EQ(test, clock_type, amd_pp_dcef_clock);
+}
+
+/* ---- Tests using stub DPM layer ---- */
+
+/**
+ * dm_test_apply_display_requirements_dpm_enabled - Test DPM-enabled path
+ * @test: KUnit test context
+ *
+ * Verify that dm_pp_apply_display_requirements calls build_pm_display_cfg
+ * and the DPM callbacks when DPM is enabled, and returns true.
+ */
+static void dm_test_apply_display_requirements_dpm_enabled(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct dm_pp_display_configuration cfg = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	adev->pm.dpm_enabled = true;
+
+	cfg.display_count = 1;
+	cfg.min_engine_clock_khz = 300000;
+	cfg.disp_configs[0].v_refresh = 60;
+
+	KUNIT_EXPECT_TRUE(test, dm_pp_apply_display_requirements(ctx, &cfg));
+	KUNIT_EXPECT_EQ(test, adev->pm.pm_display_cfg.min_core_set_clock, 30000);
+	KUNIT_EXPECT_EQ(test, adev->pm.pm_display_cfg.vrefresh, 60);
+}
+
+/**
+ * dm_test_get_clock_levels_by_type_dpm_error - Test DPM error fallback
+ * @test: KUnit test context
+ *
+ * Verify that dm_pp_get_clock_levels_by_type falls back to default clock
+ * levels when amdgpu_dpm_get_clock_by_type returns an error.
+ */
+static void dm_test_get_clock_levels_by_type_dpm_error(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct dm_pp_clock_levels dc_clks = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	stub_dpm_ctx->ret_val = -EINVAL;
+
+	KUNIT_EXPECT_TRUE(test, dm_pp_get_clock_levels_by_type(ctx,
+				DM_PP_CLOCK_TYPE_DISPLAY_CLK, &dc_clks));
+	KUNIT_EXPECT_EQ(test, dc_clks.num_levels, 6U);
+	KUNIT_EXPECT_EQ(test, dc_clks.clocks_in_khz[0], 300000U);
+}
+
+/**
+ * dm_test_get_clock_levels_by_type_success - Test successful clock query
+ * @test: KUnit test context
+ *
+ * Verify that dm_pp_get_clock_levels_by_type returns the queried clocks
+ * capped by validation clocks.
+ */
+static void dm_test_get_clock_levels_by_type_success(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct dm_pp_clock_levels dc_clks = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+
+	stub_dpm_ctx->get_clock_by_type_clocks.count = 3;
+	stub_dpm_ctx->get_clock_by_type_clocks.clock[0] = 300000;
+	stub_dpm_ctx->get_clock_by_type_clocks.clock[1] = 500000;
+	stub_dpm_ctx->get_clock_by_type_clocks.clock[2] = 700000;
+
+	/* validation at 60000 * 10 = 600000 kHz → caps to 2 levels */
+	stub_dpm_ctx->get_validation_clks.engine_max_clock = 60000;
+	stub_dpm_ctx->get_validation_clks.memory_max_clock = 80000;
+
+	KUNIT_EXPECT_TRUE(test, dm_pp_get_clock_levels_by_type(ctx,
+				DM_PP_CLOCK_TYPE_ENGINE_CLK, &dc_clks));
+	KUNIT_EXPECT_EQ(test, dc_clks.num_levels, 2U);
+	KUNIT_EXPECT_EQ(test, dc_clks.clocks_in_khz[0], 300000U);
+	KUNIT_EXPECT_EQ(test, dc_clks.clocks_in_khz[1], 500000U);
+}
+
+/**
+ * dm_test_get_clock_levels_by_type_validation_fallback - Test validation error
+ * @test: KUnit test context
+ *
+ * Verify that dm_pp_get_clock_levels_by_type uses default validation clocks
+ * (engine=720000, memory=800000 kHz) when get_display_mode_validation_clocks
+ * returns an error, capping levels accordingly.
+ */
+static void dm_test_get_clock_levels_by_type_validation_fallback(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct dm_pp_clock_levels dc_clks = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+
+	/* get_clock_by_type succeeds with 3 engine clock levels */
+	stub_dpm_ctx->get_clock_by_type_clocks.count = 3;
+	stub_dpm_ctx->get_clock_by_type_clocks.clock[0] = 300000;
+	stub_dpm_ctx->get_clock_by_type_clocks.clock[1] = 500000;
+	stub_dpm_ctx->get_clock_by_type_clocks.clock[2] = 800000;
+
+	/* Force validation clocks to fail → triggers default path */
+	stub_dpm_ctx->get_validation_clks_ret = -EINVAL;
+
+	KUNIT_EXPECT_TRUE(test, dm_pp_get_clock_levels_by_type(ctx,
+				DM_PP_CLOCK_TYPE_ENGINE_CLK, &dc_clks));
+	/*
+	 * Default validation: engine_max_clock = 72000 * 10 = 720000 kHz.
+	 * Clocks 300000 and 500000 are within limit, 800000 exceeds it,
+	 * so num_levels is capped to 2.
+	 */
+	KUNIT_EXPECT_EQ(test, dc_clks.num_levels, 2U);
+	KUNIT_EXPECT_EQ(test, dc_clks.clocks_in_khz[0], 300000U);
+	KUNIT_EXPECT_EQ(test, dc_clks.clocks_in_khz[1], 500000U);
+}
+
+/**
+ * dm_test_get_clock_levels_with_latency_success - Test latency clock query
+ * @test: KUnit test context
+ *
+ * Verify dm_pp_get_clock_levels_by_type_with_latency returns true and
+ * copies the clock/latency data from the DPM backend.
+ */
+static void dm_test_get_clock_levels_with_latency_success(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct dm_pp_clock_levels_with_latency info = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+
+	stub_dpm_ctx->get_clock_by_type_with_latency_clks.num_levels = 1;
+	stub_dpm_ctx->get_clock_by_type_with_latency_clks.data[0].clocks_in_khz = 600000;
+	stub_dpm_ctx->get_clock_by_type_with_latency_clks.data[0].latency_in_us = 15;
+
+	KUNIT_EXPECT_TRUE(test, dm_pp_get_clock_levels_by_type_with_latency(ctx,
+				DM_PP_CLOCK_TYPE_ENGINE_CLK, &info));
+	KUNIT_EXPECT_EQ(test, info.num_levels, 1U);
+	KUNIT_EXPECT_EQ(test, info.data[0].clocks_in_khz, 600000U);
+	KUNIT_EXPECT_EQ(test, info.data[0].latency_in_us, 15U);
+}
+
+/**
+ * dm_test_get_clock_levels_with_latency_failure - Test latency query error
+ * @test: KUnit test context
+ *
+ * Verify dm_pp_get_clock_levels_by_type_with_latency returns false on DPM error.
+ */
+static void dm_test_get_clock_levels_with_latency_failure(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct dm_pp_clock_levels_with_latency info = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	stub_dpm_ctx->ret_val = -EINVAL;
+
+	KUNIT_EXPECT_FALSE(test, dm_pp_get_clock_levels_by_type_with_latency(ctx,
+				DM_PP_CLOCK_TYPE_ENGINE_CLK, &info));
+}
+
+/**
+ * dm_test_get_clock_levels_with_voltage_success - Test voltage clock query
+ * @test: KUnit test context
+ *
+ * Verify dm_pp_get_clock_levels_by_type_with_voltage returns true and
+ * copies the clock/voltage data from the DPM backend.
+ */
+static void dm_test_get_clock_levels_with_voltage_success(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct dm_pp_clock_levels_with_voltage info = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+
+	stub_dpm_ctx->get_clock_by_type_with_voltage_clks.num_levels = 1;
+	stub_dpm_ctx->get_clock_by_type_with_voltage_clks.data[0].clocks_in_khz = 400000;
+	stub_dpm_ctx->get_clock_by_type_with_voltage_clks.data[0].voltage_in_mv = 900;
+
+	KUNIT_EXPECT_TRUE(test, dm_pp_get_clock_levels_by_type_with_voltage(ctx,
+				DM_PP_CLOCK_TYPE_MEMORY_CLK, &info));
+	KUNIT_EXPECT_EQ(test, info.num_levels, 1U);
+	KUNIT_EXPECT_EQ(test, info.data[0].clocks_in_khz, 400000U);
+	KUNIT_EXPECT_EQ(test, info.data[0].voltage_in_mv, 900U);
+}
+
+/**
+ * dm_test_get_clock_levels_with_voltage_failure - Test voltage query error
+ * @test: KUnit test context
+ *
+ * Verify dm_pp_get_clock_levels_by_type_with_voltage returns false on DPM error.
+ */
+static void dm_test_get_clock_levels_with_voltage_failure(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct dm_pp_clock_levels_with_voltage info = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	stub_dpm_ctx->ret_val = -EINVAL;
+
+	KUNIT_EXPECT_FALSE(test, dm_pp_get_clock_levels_by_type_with_voltage(ctx,
+				DM_PP_CLOCK_TYPE_MEMORY_CLK, &info));
+}
+
+/**
+ * dm_test_notify_wm_clock_changes_polaris - Test Polaris watermark path
+ * @test: KUnit test context
+ *
+ * Verify dm_pp_notify_wm_clock_changes returns true for Polaris ASICs
+ * when the DPM set_watermarks call succeeds.
+ */
+static void dm_test_notify_wm_clock_changes_polaris(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct dm_pp_wm_sets_with_clock_ranges wm = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	adev->asic_type = CHIP_POLARIS10;
+	stub_dpm_ctx->set_watermarks_ret = 0;
+
+	KUNIT_EXPECT_TRUE(test, dm_pp_notify_wm_clock_changes(ctx, &wm));
+}
+
+/**
+ * dm_test_notify_wm_clock_changes_non_polaris - Test non-Polaris path
+ * @test: KUnit test context
+ *
+ * Verify dm_pp_notify_wm_clock_changes returns false for non-Polaris ASICs.
+ */
+static void dm_test_notify_wm_clock_changes_non_polaris(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct dm_pp_wm_sets_with_clock_ranges wm = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	adev->asic_type = CHIP_NAVI10;
+
+	KUNIT_EXPECT_FALSE(test, dm_pp_notify_wm_clock_changes(ctx, &wm));
+}
+
+/**
+ * dm_test_apply_clock_for_voltage_success - Test successful voltage request
+ * @test: KUnit test context
+ *
+ * Verify dm_pp_apply_clock_for_voltage_request returns true when the DPM
+ * callback succeeds for a valid clock type.
+ */
+static void dm_test_apply_clock_for_voltage_success(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct dm_pp_clock_for_voltage_req req = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	stub_dpm_ctx->display_clock_voltage_ret = 0;
+
+	req.clk_type = DM_PP_CLOCK_TYPE_ENGINE_CLK;
+	req.clocks_in_khz = 500000;
+
+	KUNIT_EXPECT_TRUE(test, dm_pp_apply_clock_for_voltage_request(ctx, &req));
+}
+
+/**
+ * dm_test_apply_clock_for_voltage_eopnotsupp - Test EOPNOTSUPP treated as success
+ * @test: KUnit test context
+ *
+ * Verify dm_pp_apply_clock_for_voltage_request returns true when the DPM
+ * callback returns -EOPNOTSUPP (not supported is non-fatal).
+ */
+static void dm_test_apply_clock_for_voltage_eopnotsupp(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct dm_pp_clock_for_voltage_req req = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	stub_dpm_ctx->display_clock_voltage_ret = -EOPNOTSUPP;
+
+	req.clk_type = DM_PP_CLOCK_TYPE_ENGINE_CLK;
+	req.clocks_in_khz = 500000;
+
+	KUNIT_EXPECT_TRUE(test, dm_pp_apply_clock_for_voltage_request(ctx, &req));
+}
+
+/**
+ * dm_test_apply_clock_for_voltage_fail - Test DPM error returns false
+ * @test: KUnit test context
+ *
+ * Verify dm_pp_apply_clock_for_voltage_request returns false when the DPM
+ * callback fails with an error other than -EOPNOTSUPP.
+ */
+static void dm_test_apply_clock_for_voltage_fail(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct dm_pp_clock_for_voltage_req req = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	stub_dpm_ctx->display_clock_voltage_ret = -EIO;
+
+	req.clk_type = DM_PP_CLOCK_TYPE_ENGINE_CLK;
+	req.clocks_in_khz = 500000;
+
+	KUNIT_EXPECT_FALSE(test, dm_pp_apply_clock_for_voltage_request(ctx, &req));
+}
+
+/* ---- Tests for pp_nv_set_display_count ---- */
+
+/**
+ * dm_test_nv_set_display_count_ok - Test successful display count set
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_set_display_count returns PP_SMU_RESULT_OK on success.
+ */
+static void dm_test_nv_set_display_count_ok(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->set_active_display_count_ret = 0;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_nv_set_display_count(&pp_smu, 2),
+			(int)PP_SMU_RESULT_OK);
+}
+
+/**
+ * dm_test_nv_set_display_count_unsupported - Test EOPNOTSUPP mapping
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_set_display_count returns PP_SMU_RESULT_UNSUPPORTED when
+ * the DPM callback returns -EOPNOTSUPP.
+ */
+static void dm_test_nv_set_display_count_unsupported(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->set_active_display_count_ret = -EOPNOTSUPP;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_nv_set_display_count(&pp_smu, 2),
+			(int)PP_SMU_RESULT_UNSUPPORTED);
+}
+
+/**
+ * dm_test_nv_set_display_count_fail - Test generic error mapping
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_set_display_count returns PP_SMU_RESULT_FAIL on a generic
+ * DPM error.
+ */
+static void dm_test_nv_set_display_count_fail(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->set_active_display_count_ret = -EIO;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_nv_set_display_count(&pp_smu, 2),
+			(int)PP_SMU_RESULT_FAIL);
+}
+
+/* ---- Tests for pp_nv_set_voltage_by_freq ---- */
+
+/**
+ * dm_test_nv_set_voltage_by_freq_ok - Test successful voltage-by-freq
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_set_voltage_by_freq returns PP_SMU_RESULT_OK on success.
+ */
+static void dm_test_nv_set_voltage_by_freq_ok(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->display_clock_voltage_ret = 0;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_nv_set_voltage_by_freq(&pp_smu, PP_SMU_NV_DISPCLK, 600),
+			(int)PP_SMU_RESULT_OK);
+}
+
+/**
+ * dm_test_nv_set_voltage_by_freq_invalid_id - Test invalid clock id
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_set_voltage_by_freq returns PP_SMU_RESULT_FAIL for an
+ * unrecognized clock id without calling DPM.
+ */
+static void dm_test_nv_set_voltage_by_freq_invalid_id(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+
+	KUNIT_EXPECT_EQ(test,
+			(int)pp_nv_set_voltage_by_freq(&pp_smu, (enum pp_smu_nv_clock_id)0xff, 600),
+			(int)PP_SMU_RESULT_FAIL);
+}
+
+/* ---- Tests for pp_nv_set_pstate_handshake_support ---- */
+
+/**
+ * dm_test_nv_pstate_handshake_ok - Test successful pstate handshake
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_set_pstate_handshake_support returns PP_SMU_RESULT_OK
+ * when the DPM callback succeeds.
+ */
+static void dm_test_nv_pstate_handshake_ok(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->display_disable_memory_clock_switch_ret = 0;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_nv_set_pstate_handshake_support(&pp_smu, true),
+			(int)PP_SMU_RESULT_OK);
+}
+
+/**
+ * dm_test_nv_pstate_handshake_fail - Test failed pstate handshake
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_set_pstate_handshake_support returns PP_SMU_RESULT_FAIL
+ * when the DPM callback returns non-zero.
+ */
+static void dm_test_nv_pstate_handshake_fail(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->display_disable_memory_clock_switch_ret = -EIO;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_nv_set_pstate_handshake_support(&pp_smu, true),
+			(int)PP_SMU_RESULT_FAIL);
+}
+
+/* ---- Tests for pp_rn_get_dpm_clock_table ---- */
+
+/**
+ * dm_test_rn_get_dpm_clock_table_ok - Test successful DPM clock table
+ * @test: KUnit test context
+ *
+ * Verify pp_rn_get_dpm_clock_table returns PP_SMU_RESULT_OK on success.
+ */
+static void dm_test_rn_get_dpm_clock_table_ok(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+	struct dpm_clocks clock_table = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->get_dpm_clock_table_ret = 0;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_rn_get_dpm_clock_table(&pp_smu, &clock_table),
+			(int)PP_SMU_RESULT_OK);
+}
+
+/**
+ * dm_test_rn_get_dpm_clock_table_unsupported - Test EOPNOTSUPP mapping
+ * @test: KUnit test context
+ *
+ * Verify pp_rn_get_dpm_clock_table returns PP_SMU_RESULT_UNSUPPORTED.
+ */
+static void dm_test_rn_get_dpm_clock_table_unsupported(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+	struct dpm_clocks clock_table = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->get_dpm_clock_table_ret = -EOPNOTSUPP;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_rn_get_dpm_clock_table(&pp_smu, &clock_table),
+			(int)PP_SMU_RESULT_UNSUPPORTED);
+}
+
+/**
+ * dm_test_rn_get_dpm_clock_table_fail - Test generic error mapping
+ * @test: KUnit test context
+ *
+ * Verify pp_rn_get_dpm_clock_table returns PP_SMU_RESULT_FAIL.
+ */
+static void dm_test_rn_get_dpm_clock_table_fail(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+	struct dpm_clocks clock_table = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->get_dpm_clock_table_ret = -EIO;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_rn_get_dpm_clock_table(&pp_smu, &clock_table),
+			(int)PP_SMU_RESULT_FAIL);
+}
+
+/* ---- Tests for pp_rv_set_wm_ranges ---- */
+
+/**
+ * dm_test_rv_set_wm_ranges - Test Raven watermark range forwarding
+ * @test: KUnit test context
+ *
+ * Verify pp_rv_set_wm_ranges converts watermark ranges via
+ * build_wm_clock_ranges_soc15 and forwards them to DPM without crashing.
+ */
+static void dm_test_rv_set_wm_ranges(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+	struct pp_smu_wm_range_sets ranges = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+
+	ranges.num_reader_wm_sets = 1;
+	ranges.reader_wm_sets[0].wm_inst = 0;
+	ranges.reader_wm_sets[0].max_drain_clk_mhz = 600;
+	ranges.reader_wm_sets[0].min_drain_clk_mhz = 300;
+
+	pp_rv_set_wm_ranges(&pp_smu, &ranges);
+
+	/* Reaching here without crash confirms coverage */
+	KUNIT_SUCCEED(test);
+}
+
+/* ---- Tests for pp_rv_set_pme_wa_enable ---- */
+
+/**
+ * dm_test_rv_set_pme_wa_enable - Test Raven PME workaround enable
+ * @test: KUnit test context
+ *
+ * Verify pp_rv_set_pme_wa_enable forwards the call to DPM without crashing.
+ */
+static void dm_test_rv_set_pme_wa_enable(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+
+	pp_rv_set_pme_wa_enable(&pp_smu);
+
+	KUNIT_SUCCEED(test);
+}
+
+/* ---- Tests for pp_rv_set_active_display_count ---- */
+
+/**
+ * dm_test_rv_set_active_display_count - Test Raven display count forwarding
+ * @test: KUnit test context
+ *
+ * Verify pp_rv_set_active_display_count forwards the count to DPM without
+ * crashing.
+ */
+static void dm_test_rv_set_active_display_count(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+
+	pp_rv_set_active_display_count(&pp_smu, 2);
+
+	KUNIT_SUCCEED(test);
+}
+
+/* ---- Tests for pp_rv_set_min_deep_sleep_dcfclk ---- */
+
+/**
+ * dm_test_rv_set_min_deep_sleep_dcfclk - Test Raven deep sleep clock
+ * @test: KUnit test context
+ *
+ * Verify pp_rv_set_min_deep_sleep_dcfclk forwards the clock value to DPM
+ * without crashing.
+ */
+static void dm_test_rv_set_min_deep_sleep_dcfclk(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+
+	pp_rv_set_min_deep_sleep_dcfclk(&pp_smu, 300);
+
+	KUNIT_SUCCEED(test);
+}
+
+/* ---- Tests for pp_rv_set_hard_min_dcefclk_by_freq ---- */
+
+/**
+ * dm_test_rv_set_hard_min_dcefclk_by_freq - Test Raven hard min DCEFCLK
+ * @test: KUnit test context
+ *
+ * Verify pp_rv_set_hard_min_dcefclk_by_freq forwards the frequency to DPM
+ * without crashing.
+ */
+static void dm_test_rv_set_hard_min_dcefclk_by_freq(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+
+	pp_rv_set_hard_min_dcefclk_by_freq(&pp_smu, 600);
+
+	KUNIT_SUCCEED(test);
+}
+
+/* ---- Tests for pp_rv_set_hard_min_fclk_by_freq ---- */
+
+/**
+ * dm_test_rv_set_hard_min_fclk_by_freq - Test Raven hard min FCLK
+ * @test: KUnit test context
+ *
+ * Verify pp_rv_set_hard_min_fclk_by_freq forwards the frequency to DPM
+ * without crashing.
+ */
+static void dm_test_rv_set_hard_min_fclk_by_freq(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+
+	pp_rv_set_hard_min_fclk_by_freq(&pp_smu, 800);
+
+	KUNIT_SUCCEED(test);
+}
+
+/* ---- Tests for pp_nv_set_wm_ranges ---- */
+
+/**
+ * dm_test_nv_set_wm_ranges - Test Navi watermark range forwarding
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_set_wm_ranges forwards ranges to DPM and unconditionally
+ * returns PP_SMU_RESULT_OK.
+ */
+static void dm_test_nv_set_wm_ranges(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+	struct pp_smu_wm_range_sets ranges = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+
+	ranges.num_reader_wm_sets = 1;
+	ranges.reader_wm_sets[0].wm_inst = 0;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_nv_set_wm_ranges(&pp_smu, &ranges),
+			(int)PP_SMU_RESULT_OK);
+}
+
+/* ---- Tests for pp_nv_set_min_deep_sleep_dcfclk ---- */
+
+/**
+ * dm_test_nv_set_min_deep_sleep_dcfclk_ok - Test successful deep sleep set
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_set_min_deep_sleep_dcfclk returns PP_SMU_RESULT_OK on success.
+ */
+static void dm_test_nv_set_min_deep_sleep_dcfclk_ok(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->set_min_deep_sleep_dcefclk_ret = 0;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_nv_set_min_deep_sleep_dcfclk(&pp_smu, 300),
+			(int)PP_SMU_RESULT_OK);
+}
+
+/**
+ * dm_test_nv_set_min_deep_sleep_dcfclk_unsupported - Test EOPNOTSUPP mapping
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_set_min_deep_sleep_dcfclk returns PP_SMU_RESULT_UNSUPPORTED.
+ */
+static void dm_test_nv_set_min_deep_sleep_dcfclk_unsupported(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->set_min_deep_sleep_dcefclk_ret = -EOPNOTSUPP;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_nv_set_min_deep_sleep_dcfclk(&pp_smu, 300),
+			(int)PP_SMU_RESULT_UNSUPPORTED);
+}
+
+/**
+ * dm_test_nv_set_min_deep_sleep_dcfclk_fail - Test generic error mapping
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_set_min_deep_sleep_dcfclk returns PP_SMU_RESULT_FAIL.
+ */
+static void dm_test_nv_set_min_deep_sleep_dcfclk_fail(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->set_min_deep_sleep_dcefclk_ret = -EIO;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_nv_set_min_deep_sleep_dcfclk(&pp_smu, 300),
+			(int)PP_SMU_RESULT_FAIL);
+}
+
+/* ---- Tests for pp_nv_set_hard_min_dcefclk_by_freq ---- */
+
+/**
+ * dm_test_nv_set_hard_min_dcefclk_ok - Test successful hard min DCEFCLK
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_set_hard_min_dcefclk_by_freq returns PP_SMU_RESULT_OK.
+ */
+static void dm_test_nv_set_hard_min_dcefclk_ok(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->display_clock_voltage_ret = 0;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_nv_set_hard_min_dcefclk_by_freq(&pp_smu, 600),
+			(int)PP_SMU_RESULT_OK);
+}
+
+/**
+ * dm_test_nv_set_hard_min_dcefclk_unsupported - Test EOPNOTSUPP mapping
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_set_hard_min_dcefclk_by_freq returns PP_SMU_RESULT_UNSUPPORTED.
+ */
+static void dm_test_nv_set_hard_min_dcefclk_unsupported(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->display_clock_voltage_ret = -EOPNOTSUPP;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_nv_set_hard_min_dcefclk_by_freq(&pp_smu, 600),
+			(int)PP_SMU_RESULT_UNSUPPORTED);
+}
+
+/**
+ * dm_test_nv_set_hard_min_dcefclk_fail - Test generic error mapping
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_set_hard_min_dcefclk_by_freq returns PP_SMU_RESULT_FAIL.
+ */
+static void dm_test_nv_set_hard_min_dcefclk_fail(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->display_clock_voltage_ret = -EIO;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_nv_set_hard_min_dcefclk_by_freq(&pp_smu, 600),
+			(int)PP_SMU_RESULT_FAIL);
+}
+
+/* ---- Tests for pp_nv_set_hard_min_uclk_by_freq ---- */
+
+/**
+ * dm_test_nv_set_hard_min_uclk_ok - Test successful hard min UCLK
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_set_hard_min_uclk_by_freq returns PP_SMU_RESULT_OK.
+ */
+static void dm_test_nv_set_hard_min_uclk_ok(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->display_clock_voltage_ret = 0;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_nv_set_hard_min_uclk_by_freq(&pp_smu, 800),
+			(int)PP_SMU_RESULT_OK);
+}
+
+/**
+ * dm_test_nv_set_hard_min_uclk_unsupported - Test EOPNOTSUPP mapping
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_set_hard_min_uclk_by_freq returns PP_SMU_RESULT_UNSUPPORTED.
+ */
+static void dm_test_nv_set_hard_min_uclk_unsupported(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->display_clock_voltage_ret = -EOPNOTSUPP;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_nv_set_hard_min_uclk_by_freq(&pp_smu, 800),
+			(int)PP_SMU_RESULT_UNSUPPORTED);
+}
+
+/**
+ * dm_test_nv_set_hard_min_uclk_fail - Test generic error mapping
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_set_hard_min_uclk_by_freq returns PP_SMU_RESULT_FAIL.
+ */
+static void dm_test_nv_set_hard_min_uclk_fail(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->display_clock_voltage_ret = -EIO;
+
+	KUNIT_EXPECT_EQ(test, (int)pp_nv_set_hard_min_uclk_by_freq(&pp_smu, 800),
+			(int)PP_SMU_RESULT_FAIL);
+}
+
+/* ---- Tests for pp_nv_get_maximum_sustainable_clocks ---- */
+
+/**
+ * dm_test_nv_get_max_sustainable_clocks_ok - Test successful query
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_get_maximum_sustainable_clocks returns PP_SMU_RESULT_OK.
+ */
+static void dm_test_nv_get_max_sustainable_clocks_ok(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+	struct pp_smu_nv_clock_table max_clocks = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->get_max_sustainable_ret = 0;
+
+	KUNIT_EXPECT_EQ(test,
+			(int)pp_nv_get_maximum_sustainable_clocks(&pp_smu, &max_clocks),
+			(int)PP_SMU_RESULT_OK);
+}
+
+/**
+ * dm_test_nv_get_max_sustainable_clocks_unsupported - Test EOPNOTSUPP
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_get_maximum_sustainable_clocks returns PP_SMU_RESULT_UNSUPPORTED.
+ */
+static void dm_test_nv_get_max_sustainable_clocks_unsupported(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+	struct pp_smu_nv_clock_table max_clocks = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->get_max_sustainable_ret = -EOPNOTSUPP;
+
+	KUNIT_EXPECT_EQ(test,
+			(int)pp_nv_get_maximum_sustainable_clocks(&pp_smu, &max_clocks),
+			(int)PP_SMU_RESULT_UNSUPPORTED);
+}
+
+/**
+ * dm_test_nv_get_max_sustainable_clocks_fail - Test generic error
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_get_maximum_sustainable_clocks returns PP_SMU_RESULT_FAIL.
+ */
+static void dm_test_nv_get_max_sustainable_clocks_fail(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+	struct pp_smu_nv_clock_table max_clocks = {};
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->get_max_sustainable_ret = -EIO;
+
+	KUNIT_EXPECT_EQ(test,
+			(int)pp_nv_get_maximum_sustainable_clocks(&pp_smu, &max_clocks),
+			(int)PP_SMU_RESULT_FAIL);
+}
+
+/* ---- Tests for pp_nv_get_uclk_dpm_states ---- */
+
+/**
+ * dm_test_nv_get_uclk_dpm_states_ok - Test successful DPM states query
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_get_uclk_dpm_states returns PP_SMU_RESULT_OK.
+ */
+static void dm_test_nv_get_uclk_dpm_states_ok(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+	unsigned int clock_values[4] = {};
+	unsigned int num_states = 0;
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->get_uclk_dpm_ret = 0;
+
+	KUNIT_EXPECT_EQ(test,
+			(int)pp_nv_get_uclk_dpm_states(&pp_smu, clock_values, &num_states),
+			(int)PP_SMU_RESULT_OK);
+}
+
+/**
+ * dm_test_nv_get_uclk_dpm_states_unsupported - Test EOPNOTSUPP mapping
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_get_uclk_dpm_states returns PP_SMU_RESULT_UNSUPPORTED.
+ */
+static void dm_test_nv_get_uclk_dpm_states_unsupported(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+	unsigned int clock_values[4] = {};
+	unsigned int num_states = 0;
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->get_uclk_dpm_ret = -EOPNOTSUPP;
+
+	KUNIT_EXPECT_EQ(test,
+			(int)pp_nv_get_uclk_dpm_states(&pp_smu, clock_values, &num_states),
+			(int)PP_SMU_RESULT_UNSUPPORTED);
+}
+
+/**
+ * dm_test_nv_get_uclk_dpm_states_fail - Test generic error mapping
+ * @test: KUnit test context
+ *
+ * Verify pp_nv_get_uclk_dpm_states returns PP_SMU_RESULT_FAIL.
+ */
+static void dm_test_nv_get_uclk_dpm_states_fail(struct kunit *test)
+{
+	struct amdgpu_device *adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct pp_smu pp_smu = {};
+	unsigned int clock_values[4] = {};
+	unsigned int num_states = 0;
+
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	setup_stub_dpm(test, adev);
+	ctx->driver_context = adev;
+	pp_smu.dm = ctx;
+	stub_dpm_ctx->get_uclk_dpm_ret = -EIO;
+
+	KUNIT_EXPECT_EQ(test,
+			(int)pp_nv_get_uclk_dpm_states(&pp_smu, clock_values, &num_states),
+			(int)PP_SMU_RESULT_FAIL);
+}
+
+static struct kunit_case dm_pp_smu_test_cases[] = {
+	/* get_default_clock_levels */
+	KUNIT_CASE(dm_test_default_clock_levels_display),
+	KUNIT_CASE(dm_test_default_clock_levels_engine),
+	KUNIT_CASE(dm_test_default_clock_levels_memory),
+	KUNIT_CASE(dm_test_default_clock_levels_unknown),
+	/* dc_to_pp_clock_type */
+	KUNIT_CASE(dm_test_dc_to_pp_clock_type_display),
+	KUNIT_CASE(dm_test_dc_to_pp_clock_type_engine),
+	KUNIT_CASE(dm_test_dc_to_pp_clock_type_memory),
+	KUNIT_CASE(dm_test_dc_to_pp_clock_type_dcefclk),
+	KUNIT_CASE(dm_test_dc_to_pp_clock_type_dcfclk),
+	KUNIT_CASE(dm_test_dc_to_pp_clock_type_pixelclk),
+	KUNIT_CASE(dm_test_dc_to_pp_clock_type_fclk),
+	KUNIT_CASE(dm_test_dc_to_pp_clock_type_phyclk),
+	KUNIT_CASE(dm_test_dc_to_pp_clock_type_dppclk),
+	KUNIT_CASE(dm_test_dc_to_pp_clock_type_invalid),
+	/* pp_to_dc_clock_levels */
+	KUNIT_CASE(dm_test_pp_to_dc_clock_levels_within_limit),
+	KUNIT_CASE(dm_test_pp_to_dc_clock_levels_caps_at_max),
+	/* pp_to_dc_clock_levels_with_latency */
+	KUNIT_CASE(dm_test_pp_to_dc_clock_levels_latency_within_limit),
+	KUNIT_CASE(dm_test_pp_to_dc_clock_levels_latency_caps_at_max),
+	/* pp_to_dc_clock_levels_with_voltage */
+	KUNIT_CASE(dm_test_pp_to_dc_clock_levels_voltage_within_limit),
+	KUNIT_CASE(dm_test_pp_to_dc_clock_levels_voltage_caps_at_max),
+	/* dm_pp_get_funcs */
+	KUNIT_CASE(dm_test_get_funcs_rv),
+	KUNIT_CASE(dm_test_get_funcs_rv_101),
+	KUNIT_CASE(dm_test_get_funcs_nv),
+	KUNIT_CASE(dm_test_get_funcs_rn),
+	KUNIT_CASE(dm_test_get_funcs_unsupported),
+	/* amdgpu_device-backed entry points */
+	KUNIT_CASE(dm_test_apply_display_requirements_dpm_disabled),
+	KUNIT_CASE(dm_test_apply_clock_for_voltage_invalid_type),
+	/* build_pm_display_cfg */
+	KUNIT_CASE(dm_test_build_pm_display_cfg_scalar_fields),
+	KUNIT_CASE(dm_test_build_pm_display_cfg_per_display),
+	/* build_wm_clock_ranges_soc15 */
+	KUNIT_CASE(dm_test_build_wm_clock_ranges_dmif),
+	KUNIT_CASE(dm_test_build_wm_clock_ranges_mcif),
+	/* cap_clock_levels_to_validation */
+	KUNIT_CASE(dm_test_cap_clock_levels_engine_caps),
+	KUNIT_CASE(dm_test_cap_clock_levels_engine_first_exceeds),
+	KUNIT_CASE(dm_test_cap_clock_levels_memory_caps),
+	KUNIT_CASE(dm_test_cap_clock_levels_within_limit),
+	KUNIT_CASE(dm_test_cap_clock_levels_other_type),
+	/* pp_smu_nv_clock_id_to_pp */
+	KUNIT_CASE(dm_test_nv_clock_id_dispclk),
+	KUNIT_CASE(dm_test_nv_clock_id_phyclk),
+	KUNIT_CASE(dm_test_nv_clock_id_pixelclk),
+	KUNIT_CASE(dm_test_nv_clock_id_invalid),
+	/* dm_pp_apply_display_requirements (DPM enabled) */
+	KUNIT_CASE(dm_test_apply_display_requirements_dpm_enabled),
+	/* dm_pp_get_clock_levels_by_type */
+	KUNIT_CASE(dm_test_get_clock_levels_by_type_dpm_error),
+	KUNIT_CASE(dm_test_get_clock_levels_by_type_success),
+	KUNIT_CASE(dm_test_get_clock_levels_by_type_validation_fallback),
+	/* dm_pp_get_clock_levels_by_type_with_latency */
+	KUNIT_CASE(dm_test_get_clock_levels_with_latency_success),
+	KUNIT_CASE(dm_test_get_clock_levels_with_latency_failure),
+	/* dm_pp_get_clock_levels_by_type_with_voltage */
+	KUNIT_CASE(dm_test_get_clock_levels_with_voltage_success),
+	KUNIT_CASE(dm_test_get_clock_levels_with_voltage_failure),
+	/* dm_pp_notify_wm_clock_changes */
+	KUNIT_CASE(dm_test_notify_wm_clock_changes_polaris),
+	KUNIT_CASE(dm_test_notify_wm_clock_changes_non_polaris),
+	/* dm_pp_apply_clock_for_voltage_request (with DPM) */
+	KUNIT_CASE(dm_test_apply_clock_for_voltage_success),
+	KUNIT_CASE(dm_test_apply_clock_for_voltage_eopnotsupp),
+	KUNIT_CASE(dm_test_apply_clock_for_voltage_fail),
+	/* pp_nv_set_display_count */
+	KUNIT_CASE(dm_test_nv_set_display_count_ok),
+	KUNIT_CASE(dm_test_nv_set_display_count_unsupported),
+	KUNIT_CASE(dm_test_nv_set_display_count_fail),
+	/* pp_nv_set_voltage_by_freq */
+	KUNIT_CASE(dm_test_nv_set_voltage_by_freq_ok),
+	KUNIT_CASE(dm_test_nv_set_voltage_by_freq_invalid_id),
+	/* pp_nv_set_pstate_handshake_support */
+	KUNIT_CASE(dm_test_nv_pstate_handshake_ok),
+	KUNIT_CASE(dm_test_nv_pstate_handshake_fail),
+	/* pp_rn_get_dpm_clock_table */
+	KUNIT_CASE(dm_test_rn_get_dpm_clock_table_ok),
+	KUNIT_CASE(dm_test_rn_get_dpm_clock_table_unsupported),
+	KUNIT_CASE(dm_test_rn_get_dpm_clock_table_fail),
+	/* pp_rv_set_wm_ranges */
+	KUNIT_CASE(dm_test_rv_set_wm_ranges),
+	/* pp_rv_set_pme_wa_enable */
+	KUNIT_CASE(dm_test_rv_set_pme_wa_enable),
+	/* pp_rv_set_active_display_count */
+	KUNIT_CASE(dm_test_rv_set_active_display_count),
+	/* pp_rv_set_min_deep_sleep_dcfclk */
+	KUNIT_CASE(dm_test_rv_set_min_deep_sleep_dcfclk),
+	/* pp_rv_set_hard_min_dcefclk_by_freq */
+	KUNIT_CASE(dm_test_rv_set_hard_min_dcefclk_by_freq),
+	/* pp_rv_set_hard_min_fclk_by_freq */
+	KUNIT_CASE(dm_test_rv_set_hard_min_fclk_by_freq),
+	/* pp_nv_set_wm_ranges */
+	KUNIT_CASE(dm_test_nv_set_wm_ranges),
+	/* pp_nv_set_min_deep_sleep_dcfclk */
+	KUNIT_CASE(dm_test_nv_set_min_deep_sleep_dcfclk_ok),
+	KUNIT_CASE(dm_test_nv_set_min_deep_sleep_dcfclk_unsupported),
+	KUNIT_CASE(dm_test_nv_set_min_deep_sleep_dcfclk_fail),
+	/* pp_nv_set_hard_min_dcefclk_by_freq */
+	KUNIT_CASE(dm_test_nv_set_hard_min_dcefclk_ok),
+	KUNIT_CASE(dm_test_nv_set_hard_min_dcefclk_unsupported),
+	KUNIT_CASE(dm_test_nv_set_hard_min_dcefclk_fail),
+	/* pp_nv_set_hard_min_uclk_by_freq */
+	KUNIT_CASE(dm_test_nv_set_hard_min_uclk_ok),
+	KUNIT_CASE(dm_test_nv_set_hard_min_uclk_unsupported),
+	KUNIT_CASE(dm_test_nv_set_hard_min_uclk_fail),
+	/* pp_nv_get_maximum_sustainable_clocks */
+	KUNIT_CASE(dm_test_nv_get_max_sustainable_clocks_ok),
+	KUNIT_CASE(dm_test_nv_get_max_sustainable_clocks_unsupported),
+	KUNIT_CASE(dm_test_nv_get_max_sustainable_clocks_fail),
+	/* pp_nv_get_uclk_dpm_states */
+	KUNIT_CASE(dm_test_nv_get_uclk_dpm_states_ok),
+	KUNIT_CASE(dm_test_nv_get_uclk_dpm_states_unsupported),
+	KUNIT_CASE(dm_test_nv_get_uclk_dpm_states_fail),
+	{}
+};
+
+static struct kunit_suite dm_pp_smu_test_suite = {
+	.name = "amdgpu_dm_pp_smu",
+	.test_cases = dm_pp_smu_test_cases,
+};
+
+kunit_test_suite(dm_pp_smu_test_suite);
+
+MODULE_LICENSE("Dual MIT/GPL");
+MODULE_DESCRIPTION("KUnit tests for amdgpu_dm_pp_smu");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_psr_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_psr_test.c
index 09084f70a4052..09bd98e930471 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_psr_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_psr_test.c
@@ -7,29 +7,331 @@
 
 #include <kunit/test.h>
 
+#include "dc.h"
+#include "core_types.h"
+#include "amdgpu_mode.h"
+#include "amdgpu_dm.h"
 #include "amdgpu_dm_psr.h"
+#include "amdgpu_dm_kunit_test_helpers.h"
+#include "power_helpers.h"
 
-/*
- * Helper: allocate and zero-initialise a dc_link sufficient for
- * amdgpu_dm_psr_fill_caps() testing.  The function only accesses
- * embedded members (dpcd_caps, psr_settings) so no pointer fields
- * need to be wired up.
- */
-static struct dc_link *alloc_test_link(struct kunit *test)
+static struct dc_stream_state *alloc_test_psr_stream(struct kunit *test)
 {
 	struct dc_link *link;
 
-	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, link);
+	link = dm_kunit_alloc_link(test);
+	link->psr_settings.psr_feature_enabled = true;
+
+	return dm_kunit_alloc_stream(test, link);
+}
+
+static struct core_power *create_test_power_module(struct kunit *test,
+		struct dc_stream_state *stream, struct psr_caps *caps)
+{
+	struct core_power *core_power;
+
+	core_power = kunit_kzalloc(test, sizeof(*core_power), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, core_power);
+
+	core_power->map = kunit_kzalloc(test, sizeof(*core_power->map), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, core_power->map);
+
+	core_power->map[0].stream = stream;
+	core_power->map[0].caps = caps;
+	core_power->map[0].psr_events = psr_event_vsync;
+	core_power->num_entities = 1;
+
+	return core_power;
+}
+
+static struct dc_link *alloc_test_psrsu_link(struct kunit *test)
+{
+	struct dc_link *link = dm_kunit_alloc_link(test);
+	struct dc_context *ctx;
+	struct dc *dc;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+
+	link->ctx = ctx;
+	ctx->dc = dc;
+	dc->ctx = ctx;
+	dc->caps.dmcub_support = true;
+	ctx->dce_version = DCN_VERSION_3_1;
+	link->dpcd_caps.edp_rev = DP_EDP_14;
+	link->dpcd_caps.psr_info.psr_version = DP_PSR2_WITH_Y_COORD_ET_SUPPORTED;
+	link->dpcd_caps.alpm_caps.bits.AUX_WAKE_ALPM_CAP = 1;
+	link->dpcd_caps.psr_info.psr_dpcd_caps.bits.Y_COORDINATE_REQUIRED = 1;
+
+	return link;
+}
+
+static struct dc_link *alloc_test_psr_caps_link(struct kunit *test)
+{
+	struct dc_link *link = alloc_test_psrsu_link(test);
+
+	link->ctx->dc->caps.dmub_caps.psr = true;
+	link->connector_signal = SIGNAL_TYPE_EDP;
+	link->type = dc_connection_single;
 
 	return link;
 }
 
+static struct amdgpu_dm_connector *alloc_test_aconnector(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+
+	return aconnector;
+}
+
+/* Tests for link_supports_psrsu() */
+
+/**
+ * dm_test_link_supports_psrsu_no_dmcub() - DMCUB support is required.
+ * @test: KUnit test context.
+ */
+static void dm_test_link_supports_psrsu_no_dmcub(struct kunit *test)
+{
+	struct dc_link *link = alloc_test_psrsu_link(test);
+
+	link->ctx->dc->caps.dmcub_support = false;
+
+	KUNIT_EXPECT_FALSE(test, link_supports_psrsu(link));
+}
+
+/**
+ * dm_test_link_supports_psrsu_old_dcn() - DCN version 3.1 or newer is required.
+ * @test: KUnit test context.
+ */
+static void dm_test_link_supports_psrsu_old_dcn(struct kunit *test)
+{
+	struct dc_link *link = alloc_test_psrsu_link(test);
+
+	link->ctx->dce_version = DCN_VERSION_3_0;
+
+	KUNIT_EXPECT_FALSE(test, link_supports_psrsu(link));
+}
+
+/**
+ * dm_test_link_supports_psrsu_panel_unsupported() - Panel PSR-SU caps are required.
+ * @test: KUnit test context.
+ */
+static void dm_test_link_supports_psrsu_panel_unsupported(struct kunit *test)
+{
+	struct dc_link *link = alloc_test_psrsu_link(test);
+
+	link->dpcd_caps.psr_info.psr_version = 0;
+
+	KUNIT_EXPECT_FALSE(test, link_supports_psrsu(link));
+}
+
+/**
+ * dm_test_link_supports_psrsu_missing_alpm() - AUX wake ALPM is required.
+ * @test: KUnit test context.
+ */
+static void dm_test_link_supports_psrsu_missing_alpm(struct kunit *test)
+{
+	struct dc_link *link = alloc_test_psrsu_link(test);
+
+	link->dpcd_caps.alpm_caps.bits.AUX_WAKE_ALPM_CAP = 0;
+
+	KUNIT_EXPECT_FALSE(test, link_supports_psrsu(link));
+}
+
+/**
+ * dm_test_link_supports_psrsu_missing_y_coordinate() - Y coordinate support is required.
+ * @test: KUnit test context.
+ */
+static void dm_test_link_supports_psrsu_missing_y_coordinate(struct kunit *test)
+{
+	struct dc_link *link = alloc_test_psrsu_link(test);
+
+	link->dpcd_caps.psr_info.psr_dpcd_caps.bits.Y_COORDINATE_REQUIRED = 0;
+
+	KUNIT_EXPECT_FALSE(test, link_supports_psrsu(link));
+}
+
+/**
+ * dm_test_link_supports_psrsu_missing_granularity() - Required granularity must
+ * be reported by the panel.
+ * @test: KUnit test context.
+ */
+static void dm_test_link_supports_psrsu_missing_granularity(struct kunit *test)
+{
+	struct dc_link *link = alloc_test_psrsu_link(test);
+
+	link->dpcd_caps.psr_info.psr_dpcd_caps.bits.SU_GRANULARITY_REQUIRED = 1;
+	link->dpcd_caps.psr_info.psr2_su_y_granularity_cap = 0;
+
+	KUNIT_EXPECT_FALSE(test, link_supports_psrsu(link));
+}
+
+/**
+ * dm_test_link_supports_psrsu_debug_mask_disabled() - Debug mask disables PSR-SU.
+ * @test: KUnit test context.
+ */
+static void dm_test_link_supports_psrsu_debug_mask_disabled(struct kunit *test)
+{
+	struct dc_link *link = alloc_test_psrsu_link(test);
+	unsigned int old_debug_mask;
+
+	old_debug_mask = amdgpu_dm_psr_get_dc_debug_mask();
+	amdgpu_dm_psr_set_dc_debug_mask(old_debug_mask | DC_DISABLE_PSR_SU);
+
+	KUNIT_EXPECT_FALSE(test, link_supports_psrsu(link));
+	amdgpu_dm_psr_set_dc_debug_mask(old_debug_mask);
+}
+
+/**
+ * dm_test_link_supports_psrsu_temporarily_disabled() - Supported panels still
+ * return false while PSR-SU is temporarily disabled.
+ * @test: KUnit test context.
+ */
+static void dm_test_link_supports_psrsu_temporarily_disabled(struct kunit *test)
+{
+	struct dc_link *link = alloc_test_psrsu_link(test);
+	unsigned int old_debug_mask;
+
+	old_debug_mask = amdgpu_dm_psr_get_dc_debug_mask();
+	amdgpu_dm_psr_set_dc_debug_mask(old_debug_mask & ~DC_DISABLE_PSR_SU);
+
+	KUNIT_EXPECT_FALSE(test, link_supports_psrsu(link));
+	amdgpu_dm_psr_set_dc_debug_mask(old_debug_mask);
+}
+
+/* End of tests for link_supports_psrsu() */
+
+/* Tests for amdgpu_dm_set_psr_caps() */
+
+/**
+ * dm_test_set_psr_caps_null_link() - NULL link is rejected.
+ * @test: KUnit test context.
+ */
+static void dm_test_set_psr_caps_null_link(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector = alloc_test_aconnector(test);
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_set_psr_caps(NULL, aconnector));
+}
+
+/**
+ * dm_test_set_psr_caps_null_connector() - NULL connector is rejected.
+ * @test: KUnit test context.
+ */
+static void dm_test_set_psr_caps_null_connector(struct kunit *test)
+{
+	struct dc_link *link = alloc_test_psr_caps_link(test);
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_set_psr_caps(link, NULL));
+}
+
+/**
+ * dm_test_set_psr_caps_no_dmub_psr() - DMUB PSR capability is required.
+ * @test: KUnit test context.
+ */
+static void dm_test_set_psr_caps_no_dmub_psr(struct kunit *test)
+{
+	struct dc_link *link = alloc_test_psr_caps_link(test);
+	struct amdgpu_dm_connector *aconnector = alloc_test_aconnector(test);
+
+	link->psr_settings.psr_version = DC_PSR_VERSION_1;
+	link->ctx->dc->caps.dmub_caps.psr = false;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_set_psr_caps(link, aconnector));
+	KUNIT_EXPECT_EQ(test, link->psr_settings.psr_version,
+			DC_PSR_VERSION_UNSUPPORTED);
+}
+
+/**
+ * dm_test_set_psr_caps_non_edp() - Only eDP links can enable PSR.
+ * @test: KUnit test context.
+ */
+static void dm_test_set_psr_caps_non_edp(struct kunit *test)
+{
+	struct dc_link *link = alloc_test_psr_caps_link(test);
+	struct amdgpu_dm_connector *aconnector = alloc_test_aconnector(test);
+
+	link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_set_psr_caps(link, aconnector));
+}
+
+/**
+ * dm_test_set_psr_caps_disconnected() - Disconnected links cannot enable PSR.
+ * @test: KUnit test context.
+ */
+static void dm_test_set_psr_caps_disconnected(struct kunit *test)
+{
+	struct dc_link *link = alloc_test_psr_caps_link(test);
+	struct amdgpu_dm_connector *aconnector = alloc_test_aconnector(test);
+
+	link->type = dc_connection_none;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_set_psr_caps(link, aconnector));
+}
+
+/**
+ * dm_test_set_psr_caps_no_dpcd_psr() - DPCD PSR version is required.
+ * @test: KUnit test context.
+ */
+static void dm_test_set_psr_caps_no_dpcd_psr(struct kunit *test)
+{
+	struct dc_link *link = alloc_test_psr_caps_link(test);
+	struct amdgpu_dm_connector *aconnector = alloc_test_aconnector(test);
+
+	link->dpcd_caps.psr_info.psr_version = 0;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_set_psr_caps(link, aconnector));
+}
+
+/**
+ * dm_test_set_psr_caps_edp1_disabled() - eDP panel instance 1 is blocked.
+ * @test: KUnit test context.
+ */
+static void dm_test_set_psr_caps_edp1_disabled(struct kunit *test)
+{
+	struct dc_link *link = alloc_test_psr_caps_link(test);
+	struct dc_link *edp0 = dm_kunit_alloc_link(test);
+	struct amdgpu_dm_connector *aconnector = alloc_test_aconnector(test);
+	struct dc *dc = link->ctx->dc;
+
+	edp0->connector_signal = SIGNAL_TYPE_EDP;
+	dc->links[0] = edp0;
+	dc->links[1] = link;
+	dc->link_count = 2;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_set_psr_caps(link, aconnector));
+}
+
+/**
+ * dm_test_set_psr_caps_success_psr1() - Valid eDP link enables PSR1 caps.
+ * @test: KUnit test context.
+ */
+static void dm_test_set_psr_caps_success_psr1(struct kunit *test)
+{
+	struct dc_link *link = alloc_test_psr_caps_link(test);
+	struct amdgpu_dm_connector *aconnector = alloc_test_aconnector(test);
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_set_psr_caps(link, aconnector));
+	KUNIT_EXPECT_EQ(test, link->psr_settings.psr_version, DC_PSR_VERSION_1);
+	KUNIT_EXPECT_EQ(test, (int)aconnector->psr_caps.psr_version, 1);
+	KUNIT_EXPECT_EQ(test, (int)aconnector->psr_caps.support_ver,
+			DP_PSR2_WITH_Y_COORD_ET_SUPPORTED);
+}
+
+/* End of tests for amdgpu_dm_set_psr_caps() */
+
 /* Tests for amdgpu_dm_psr_fill_caps() — PSR version mapping */
 
 static void dm_test_psr_fill_caps_version_1(struct kunit *test)
 {
-	struct dc_link *link = alloc_test_link(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
 	struct psr_caps caps;
 
 	memset(&caps, 0, sizeof(caps));
@@ -42,7 +344,7 @@ static void dm_test_psr_fill_caps_version_1(struct kunit *test)
 
 static void dm_test_psr_fill_caps_version_su1(struct kunit *test)
 {
-	struct dc_link *link = alloc_test_link(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
 	struct psr_caps caps;
 
 	memset(&caps, 0, sizeof(caps));
@@ -55,7 +357,7 @@ static void dm_test_psr_fill_caps_version_su1(struct kunit *test)
 
 static void dm_test_psr_fill_caps_version_unsupported(struct kunit *test)
 {
-	struct dc_link *link = alloc_test_link(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
 	struct psr_caps caps;
 
 	memset(&caps, 0, sizeof(caps));
@@ -74,7 +376,7 @@ static void dm_test_psr_fill_caps_version_unsupported(struct kunit *test)
 
 static void dm_test_psr_fill_caps_setup_time_zero(struct kunit *test)
 {
-	struct dc_link *link = alloc_test_link(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
 	struct psr_caps caps;
 
 	memset(&caps, 0, sizeof(caps));
@@ -88,7 +390,7 @@ static void dm_test_psr_fill_caps_setup_time_zero(struct kunit *test)
 
 static void dm_test_psr_fill_caps_setup_time_mid(struct kunit *test)
 {
-	struct dc_link *link = alloc_test_link(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
 	struct psr_caps caps;
 
 	memset(&caps, 0, sizeof(caps));
@@ -102,7 +404,7 @@ static void dm_test_psr_fill_caps_setup_time_mid(struct kunit *test)
 
 static void dm_test_psr_fill_caps_setup_time_max(struct kunit *test)
 {
-	struct dc_link *link = alloc_test_link(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
 	struct psr_caps caps;
 
 	memset(&caps, 0, sizeof(caps));
@@ -118,7 +420,7 @@ static void dm_test_psr_fill_caps_setup_time_max(struct kunit *test)
 
 static void dm_test_psr_fill_caps_link_training_required(struct kunit *test)
 {
-	struct dc_link *link = alloc_test_link(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
 	struct psr_caps caps;
 
 	memset(&caps, 0, sizeof(caps));
@@ -131,7 +433,7 @@ static void dm_test_psr_fill_caps_link_training_required(struct kunit *test)
 
 static void dm_test_psr_fill_caps_link_training_not_required(struct kunit *test)
 {
-	struct dc_link *link = alloc_test_link(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
 	struct psr_caps caps;
 
 	memset(&caps, 0, sizeof(caps));
@@ -146,7 +448,7 @@ static void dm_test_psr_fill_caps_link_training_not_required(struct kunit *test)
 
 static void dm_test_psr_fill_caps_dpcd_fields(struct kunit *test)
 {
-	struct dc_link *link = alloc_test_link(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
 	struct psr_caps caps;
 
 	memset(&caps, 0, sizeof(caps));
@@ -172,7 +474,7 @@ static void dm_test_psr_fill_caps_dpcd_fields(struct kunit *test)
 
 static void dm_test_psr_fill_caps_dpcd_fields_unset(struct kunit *test)
 {
-	struct dc_link *link = alloc_test_link(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
 	struct psr_caps caps;
 
 	memset(&caps, 0xFF, sizeof(caps));
@@ -193,7 +495,7 @@ static void dm_test_psr_fill_caps_dpcd_fields_unset(struct kunit *test)
 
 static void dm_test_psr_fill_caps_rate_control_always_zero(struct kunit *test)
 {
-	struct dc_link *link = alloc_test_link(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
 	struct psr_caps caps;
 
 	/* Pre-fill caps with non-zero to verify overwrite */
@@ -206,7 +508,7 @@ static void dm_test_psr_fill_caps_rate_control_always_zero(struct kunit *test)
 
 static void dm_test_psr_fill_caps_power_opts_z10_always_set(struct kunit *test)
 {
-	struct dc_link *link = alloc_test_link(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
 	struct psr_caps caps;
 
 	memset(&caps, 0, sizeof(caps));
@@ -221,6 +523,24 @@ static void dm_test_psr_fill_caps_power_opts_z10_always_set(struct kunit *test)
 			  (caps.psr_power_opt_flag &
 			   psr_power_opt_z10_static_screen) != 0);
 }
+
+static void dm_test_psr_fill_caps_power_opts_smu_opt_set(struct kunit *test)
+{
+	struct dc_link *link = dm_kunit_alloc_link(test);
+	struct psr_caps caps;
+	unsigned int old_feature_mask;
+
+	memset(&caps, 0, sizeof(caps));
+	old_feature_mask = amdgpu_dm_psr_get_dc_feature_mask();
+	amdgpu_dm_psr_set_dc_feature_mask(old_feature_mask | DC_PSR_ALLOW_SMU_OPT);
+
+	amdgpu_dm_psr_fill_caps(link, &caps);
+	amdgpu_dm_psr_set_dc_feature_mask(old_feature_mask);
+
+	KUNIT_EXPECT_TRUE(test,
+			  (caps.psr_power_opt_flag &
+			   psr_power_opt_smu_opt_static_screen) != 0);
+}
 /* End of tests for amdgpu_dm_psr_fill_caps() */
 
 /* Tests for amdgpu_dm_psr_set_event() — early-exit validation guards */
@@ -258,9 +578,155 @@ static void dm_test_psr_set_event_psr_not_enabled(struct kunit *test)
 
 	KUNIT_EXPECT_FALSE(test, amdgpu_dm_psr_set_event(NULL, stream, true, psr_event_vsync, false));
 }
+
+/**
+ * dm_test_psr_set_event_get_event_fails() - Failed power event read returns false.
+ * @test: KUnit test context.
+ */
+static void dm_test_psr_set_event_get_event_fails(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct dc_stream_state *stream = alloc_test_psr_stream(test);
+
+	dm->power_module = NULL;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_psr_set_event(dm, stream, true, psr_event_vsync, false));
+}
+
+/**
+ * dm_test_psr_set_event_already_set() - Already set event returns true.
+ * @test: KUnit test context.
+ */
+static void dm_test_psr_set_event_already_set(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct dc_stream_state *stream = alloc_test_psr_stream(test);
+	struct psr_caps caps = {0};
+	struct core_power *core_power;
+
+	caps.psr_version = 1;
+	core_power = create_test_power_module(test, stream, &caps);
+	dm->power_module = &core_power->mod_public;
+
+	KUNIT_EXPECT_TRUE(test,
+			  amdgpu_dm_psr_set_event(dm, stream, true, psr_event_vsync, false));
+	KUNIT_EXPECT_EQ(test, core_power->map[0].psr_events,
+			(unsigned int)psr_event_vsync);
+}
+
+/**
+ * dm_test_psr_set_event_updates_event() - Changed event delegates to mod_power.
+ * @test: KUnit test context.
+ */
+static void dm_test_psr_set_event_updates_event(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct dc_stream_state *stream = alloc_test_psr_stream(test);
+	struct psr_caps caps = {0};
+	struct core_power *core_power;
+
+	caps.psr_version = 1;
+	core_power = create_test_power_module(test, stream, &caps);
+	dm->power_module = &core_power->mod_public;
+
+	KUNIT_EXPECT_TRUE(test,
+			  amdgpu_dm_psr_set_event(dm, stream, true, psr_event_full_screen, false));
+	KUNIT_EXPECT_EQ(test, core_power->map[0].psr_events,
+			(unsigned int)(psr_event_vsync | psr_event_full_screen));
+}
 /* End of tests for amdgpu_dm_psr_set_event() */
 
+/* Tests for amdgpu_dm_psr_is_active_allowed() */
+
+/**
+ * dm_test_psr_is_active_allowed_no_streams() - Empty DC state disallows PSR.
+ * @test: KUnit test context.
+ */
+static void dm_test_psr_is_active_allowed_no_streams(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_psr_is_active_allowed(dm));
+}
+
+/**
+ * dm_test_psr_is_active_allowed_null_link() - Streams without links are skipped.
+ * @test: KUnit test context.
+ */
+static void dm_test_psr_is_active_allowed_null_link(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct dc_state *state = dm->dc->current_state;
+
+	dm_kunit_add_stream_to_state(test, state, 0, NULL);
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_psr_is_active_allowed(dm));
+}
+
+/**
+ * dm_test_psr_is_active_allowed_requires_enabled_and_allowed() - Both link flags
+ * must be set before PSR active is allowed.
+ * @test: KUnit test context.
+ */
+static void dm_test_psr_is_active_allowed_requires_enabled_and_allowed(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct dc_state *state = dm->dc->current_state;
+	struct dc_link *link = dm_kunit_alloc_link(test);
+
+	dm_kunit_add_stream_to_state(test, state, 0, link);
+	link->psr_settings.psr_allow_active = true;
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_psr_is_active_allowed(dm));
+
+	link->psr_settings.psr_allow_active = false;
+	link->psr_settings.psr_feature_enabled = true;
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_psr_is_active_allowed(dm));
+}
+
+/**
+ * dm_test_psr_is_active_allowed_any_stream() - Any enabled and allowed stream
+ * permits active PSR.
+ * @test: KUnit test context.
+ */
+static void dm_test_psr_is_active_allowed_any_stream(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
+	struct dc_state *state = dm->dc->current_state;
+	struct dc_link *disabled_link = dm_kunit_alloc_link(test);
+	struct dc_link *allowed_link = dm_kunit_alloc_link(test);
+
+	disabled_link->psr_settings.psr_allow_active = true;
+	allowed_link->psr_settings.psr_feature_enabled = true;
+	allowed_link->psr_settings.psr_allow_active = true;
+
+	dm_kunit_add_stream_to_state(test, state, 0, disabled_link);
+	dm_kunit_add_stream_to_state(test, state, 1, allowed_link);
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_psr_is_active_allowed(dm));
+}
+
+/* End of tests for amdgpu_dm_psr_is_active_allowed() */
+
 static struct kunit_case dm_psr_test_cases[] = {
+	/* link_supports_psrsu */
+	KUNIT_CASE(dm_test_link_supports_psrsu_no_dmcub),
+	KUNIT_CASE(dm_test_link_supports_psrsu_old_dcn),
+	KUNIT_CASE(dm_test_link_supports_psrsu_panel_unsupported),
+	KUNIT_CASE(dm_test_link_supports_psrsu_missing_alpm),
+	KUNIT_CASE(dm_test_link_supports_psrsu_missing_y_coordinate),
+	KUNIT_CASE(dm_test_link_supports_psrsu_missing_granularity),
+	KUNIT_CASE(dm_test_link_supports_psrsu_debug_mask_disabled),
+	KUNIT_CASE(dm_test_link_supports_psrsu_temporarily_disabled),
+	/* amdgpu_dm_set_psr_caps */
+	KUNIT_CASE(dm_test_set_psr_caps_null_link),
+	KUNIT_CASE(dm_test_set_psr_caps_null_connector),
+	KUNIT_CASE(dm_test_set_psr_caps_no_dmub_psr),
+	KUNIT_CASE(dm_test_set_psr_caps_non_edp),
+	KUNIT_CASE(dm_test_set_psr_caps_disconnected),
+	KUNIT_CASE(dm_test_set_psr_caps_no_dpcd_psr),
+	KUNIT_CASE(dm_test_set_psr_caps_edp1_disabled),
+	KUNIT_CASE(dm_test_set_psr_caps_success_psr1),
+	/* amdgpu_dm_psr_fill_caps */
 	KUNIT_CASE(dm_test_psr_fill_caps_version_1),
 	KUNIT_CASE(dm_test_psr_fill_caps_version_su1),
 	KUNIT_CASE(dm_test_psr_fill_caps_version_unsupported),
@@ -273,9 +739,19 @@ static struct kunit_case dm_psr_test_cases[] = {
 	KUNIT_CASE(dm_test_psr_fill_caps_dpcd_fields_unset),
 	KUNIT_CASE(dm_test_psr_fill_caps_rate_control_always_zero),
 	KUNIT_CASE(dm_test_psr_fill_caps_power_opts_z10_always_set),
+	KUNIT_CASE(dm_test_psr_fill_caps_power_opts_smu_opt_set),
+	/* amdgpu_dm_psr_set_event */
 	KUNIT_CASE(dm_test_psr_set_event_null_stream),
 	KUNIT_CASE(dm_test_psr_set_event_null_link),
 	KUNIT_CASE(dm_test_psr_set_event_psr_not_enabled),
+	KUNIT_CASE(dm_test_psr_set_event_get_event_fails),
+	KUNIT_CASE(dm_test_psr_set_event_already_set),
+	KUNIT_CASE(dm_test_psr_set_event_updates_event),
+	/* amdgpu_dm_psr_is_active_allowed */
+	KUNIT_CASE(dm_test_psr_is_active_allowed_no_streams),
+	KUNIT_CASE(dm_test_psr_is_active_allowed_null_link),
+	KUNIT_CASE(dm_test_psr_is_active_allowed_requires_enabled_and_allowed),
+	KUNIT_CASE(dm_test_psr_is_active_allowed_any_stream),
 	{}
 };
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_quirks_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_quirks_test.c
new file mode 100644
index 0000000000000..a09f31ee0a2a0
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_quirks_test.c
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * KUnit tests for amdgpu_dm_quirks.c
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#include <kunit/test.h>
+
+#include "dc.h"
+#include "amdgpu_mode.h"
+#include "amdgpu_dm.h"
+
+/* Tests for retrieve_dmi_info() */
+
+/*
+ * Verify that retrieve_dmi_info() always initialises aux_hpd_discon_quirk to
+ * false, even when the caller had previously set it to true.
+ */
+/**
+ * dm_test_quirks_aux_hpd_discon_reset - Test Quirks aux hpd discon reset
+ * @test: The KUnit test context
+ */
+static void dm_test_quirks_aux_hpd_discon_reset(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm;
+
+	dm = kunit_kzalloc(test, sizeof(*dm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm);
+
+	dm->aux_hpd_discon_quirk = true;
+
+	retrieve_dmi_info(dm);
+
+	/*
+	 * In a KUnit / UML environment no real DMI table is present, so
+	 * dmi_check_system() returns 0 and retrieve_dmi_info() leaves the
+	 * quirk at its initialised-to-false value.
+	 */
+	KUNIT_EXPECT_FALSE(test, dm->aux_hpd_discon_quirk);
+}
+
+/*
+ * Verify that retrieve_dmi_info() always initialises edp0_on_dp1_quirk to
+ * false, even when the caller had previously set it to true.
+ */
+/**
+ * dm_test_quirks_edp0_on_dp1_reset - Test Quirks edp0 on dp1 reset
+ * @test: The KUnit test context
+ */
+static void dm_test_quirks_edp0_on_dp1_reset(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm;
+
+	dm = kunit_kzalloc(test, sizeof(*dm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm);
+
+	dm->edp0_on_dp1_quirk = true;
+
+	retrieve_dmi_info(dm);
+
+	KUNIT_EXPECT_FALSE(test, dm->edp0_on_dp1_quirk);
+}
+
+/*
+ * Verify that when no DMI match is found both quirks remain false after a
+ * fresh (zero-initialised) dm is passed to retrieve_dmi_info().
+ */
+/**
+ * dm_test_quirks_no_dmi_match_both_false - Test Quirks no dmi match both false
+ * @test: The KUnit test context
+ */
+static void dm_test_quirks_no_dmi_match_both_false(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm;
+
+	dm = kunit_kzalloc(test, sizeof(*dm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm);
+
+	retrieve_dmi_info(dm);
+
+	KUNIT_EXPECT_FALSE(test, dm->aux_hpd_discon_quirk);
+	KUNIT_EXPECT_FALSE(test, dm->edp0_on_dp1_quirk);
+}
+
+static struct kunit_case amdgpu_dm_quirks_tests[] = {
+	/* retrieve_dmi_info */
+	KUNIT_CASE(dm_test_quirks_aux_hpd_discon_reset),
+	KUNIT_CASE(dm_test_quirks_edp0_on_dp1_reset),
+	KUNIT_CASE(dm_test_quirks_no_dmi_match_both_false),
+	{}
+};
+
+static struct kunit_suite amdgpu_dm_quirks_test_suite = {
+	.name = "amdgpu_dm_quirks",
+	.test_cases = amdgpu_dm_quirks_tests,
+};
+
+kunit_test_suite(amdgpu_dm_quirks_test_suite);
+
+MODULE_AUTHOR("AMD");
+MODULE_DESCRIPTION("KUnit tests for amdgpu_dm_quirks");
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_replay_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_replay_test.c
index 28ff8bbcc0f75..6f633b1bbaca9 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_replay_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_replay_test.c
@@ -8,12 +8,13 @@
 #include <kunit/test.h>
 
 #include "dc.h"
+#include "dc_dmub_srv.h"
 #include "amdgpu_mode.h"
 #include "amdgpu_dm.h"
-
-/* Extern declaration for the function under test */
-extern bool amdgpu_dm_link_supports_replay(struct dc_link *link,
-					   struct amdgpu_dm_connector *aconnector);
+#include "amdgpu_dm_replay.h"
+#include "amdgpu_dm_kunit_test_helpers.h"
+#include "modules/power/power_helpers.h"
+#include "dmub/dmub_srv.h"
 
 /*
  * Helper: allocate a dc_link, amdgpu_dm_connector, and dm_connector_state
@@ -23,6 +24,9 @@ struct replay_test_ctx {
 	struct dc_link *link;
 	struct amdgpu_dm_connector *aconnector;
 	struct dm_connector_state *dm_state;
+	struct dc *dc;
+	struct dc_context *dc_ctx;
+	struct dc_stream_state *stream;
 };
 
 static struct replay_test_ctx *alloc_replay_ctx(struct kunit *test)
@@ -32,8 +36,9 @@ static struct replay_test_ctx *alloc_replay_ctx(struct kunit *test)
 	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, ctx);
 
-	ctx->link = kunit_kzalloc(test, sizeof(*ctx->link), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, ctx->link);
+	ctx->link = dm_kunit_alloc_link_with_ctx(test);
+	ctx->dc_ctx = ctx->link->ctx;
+	ctx->dc = ctx->dc_ctx->dc;
 
 	ctx->aconnector = kunit_kzalloc(test, sizeof(*ctx->aconnector), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector);
@@ -41,6 +46,8 @@ static struct replay_test_ctx *alloc_replay_ctx(struct kunit *test)
 	ctx->dm_state = kunit_kzalloc(test, sizeof(*ctx->dm_state), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, ctx->dm_state);
 
+	ctx->stream = dm_kunit_alloc_stream(test, ctx->link);
+
 	/* Wire connector state so to_dm_connector_state() works */
 	ctx->aconnector->base.state = &ctx->dm_state->base;
 
@@ -55,6 +62,7 @@ static void set_all_replay_caps(struct replay_test_ctx *ctx)
 {
 	ctx->dm_state->freesync_capable = true;
 	ctx->aconnector->vsdb_info.replay_mode = true;
+	ctx->link->connector_signal = SIGNAL_TYPE_EDP;
 	ctx->link->dpcd_caps.edp_rev = EDP_REVISION_13;
 	ctx->link->dpcd_caps.alpm_caps.bits.AUX_WAKE_ALPM_CAP = 1;
 	ctx->link->dpcd_caps.adaptive_sync_caps.dp_adap_sync_caps.bits.ADAPTIVE_SYNC_SDP_SUPPORT = 1;
@@ -181,7 +189,398 @@ static void dm_test_replay_both_deviations_zero(struct kunit *test)
 
 /* End of tests for amdgpu_dm_link_supports_replay() */
 
+/* Tests for amdgpu_dm_set_replay_caps() */
+
+/**
+ * dm_test_replay_set_caps_already_supported - Verify cached Replay support
+ * @test: KUnit test context
+ *
+ * When replay_supported is already set, amdgpu_dm_set_replay_caps() should
+ * return true without revalidating the link capabilities.
+ */
+static void dm_test_replay_set_caps_already_supported(struct kunit *test)
+{
+	struct replay_test_ctx *ctx = alloc_replay_ctx(test);
+
+	ctx->link->replay_settings.config.replay_supported = true;
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_set_replay_caps(ctx->link, ctx->aconnector));
+}
+
+/**
+ * dm_test_replay_set_caps_non_embedded_signal - Verify non-eDP rejection
+ * @test: KUnit test context
+ *
+ * When the link signal is not embedded, amdgpu_dm_set_replay_caps() should
+ * reject Replay even if the sink capability fields are otherwise valid.
+ */
+static void dm_test_replay_set_caps_non_embedded_signal(struct kunit *test)
+{
+	struct replay_test_ctx *ctx = alloc_replay_ctx(test);
+
+	set_all_replay_caps(ctx);
+	ctx->link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_set_replay_caps(ctx->link, ctx->aconnector));
+}
+
+/**
+ * dm_test_replay_set_caps_disallowed_by_panel - Verify panel policy rejection
+ * @test: KUnit test context
+ *
+ * When the panel configuration disallows Replay, amdgpu_dm_set_replay_caps()
+ * should return false before accepting the capability set.
+ */
+static void dm_test_replay_set_caps_disallowed_by_panel(struct kunit *test)
+{
+	struct replay_test_ctx *ctx = alloc_replay_ctx(test);
+
+	set_all_replay_caps(ctx);
+	ctx->link->panel_config.psr.disallow_replay = true;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_set_replay_caps(ctx->link, ctx->aconnector));
+}
+
+/**
+ * dm_test_replay_set_caps_link_not_supported - Verify capability rejection
+ * @test: KUnit test context
+ *
+ * When amdgpu_dm_link_supports_replay() rejects the link, the higher-level
+ * Replay setup helper should also return false.
+ */
+static void dm_test_replay_set_caps_link_not_supported(struct kunit *test)
+{
+	struct replay_test_ctx *ctx = alloc_replay_ctx(test);
+
+	set_all_replay_caps(ctx);
+	ctx->dm_state->freesync_capable = false;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_set_replay_caps(ctx->link, ctx->aconnector));
+}
+
+/**
+ * dm_test_replay_set_caps_missing_dmub_srv - Verify missing DMUB rejection
+ * @test: KUnit test context
+ *
+ * When the link and connector support Replay but no DMUB service is available,
+ * amdgpu_dm_set_replay_caps() should return false.
+ */
+static void dm_test_replay_set_caps_missing_dmub_srv(struct kunit *test)
+{
+	struct replay_test_ctx *ctx = alloc_replay_ctx(test);
+
+	set_all_replay_caps(ctx);
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_set_replay_caps(ctx->link, ctx->aconnector));
+}
+
+/**
+ * dm_test_replay_set_caps_success - Verify successful Replay configuration
+ * @test: KUnit test context
+ *
+ * When all prerequisites are met (embedded signal, panel allows replay, link
+ * supports replay, DMUB present with replay support), amdgpu_dm_set_replay_caps()
+ * should configure the link replay settings and return true.
+ */
+static void dm_test_replay_set_caps_success(struct kunit *test)
+{
+	struct replay_test_ctx *ctx = alloc_replay_ctx(test);
+	struct dc_dmub_srv *dmub_srv;
+	struct dmub_srv *dmub;
+
+	set_all_replay_caps(ctx);
+
+	dmub_srv = kunit_kzalloc(test, sizeof(*dmub_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dmub_srv);
+
+	dmub = kunit_kzalloc(test, sizeof(*dmub), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dmub);
+
+	dmub->feature_caps.replay_supported = 1;
+	dmub_srv->dmub = dmub;
+	ctx->dc_ctx->dmub_srv = dmub_srv;
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_set_replay_caps(ctx->link, ctx->aconnector));
+	KUNIT_EXPECT_TRUE(test, ctx->link->replay_settings.config.replay_supported);
+}
+
+/* Tests for amdgpu_dm_link_setup_replay() */
+
+/**
+ * dm_test_replay_link_setup_null_stream - Verify NULL stream rejection
+ * @test: KUnit test context
+ *
+ * amdgpu_dm_link_setup_replay() should return false when no stream is provided.
+ */
+static void dm_test_replay_link_setup_null_stream(struct kunit *test)
+{
+	struct mod_vrr_params vrr_params = { 0 };
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_link_setup_replay(NULL, &vrr_params));
+}
+
+/**
+ * dm_test_replay_link_setup_null_link - Verify NULL stream link rejection
+ * @test: KUnit test context
+ *
+ * amdgpu_dm_link_setup_replay() should return false when the stream has no
+ * associated link.
+ */
+static void dm_test_replay_link_setup_null_link(struct kunit *test)
+{
+	struct replay_test_ctx *ctx = alloc_replay_ctx(test);
+	struct mod_vrr_params vrr_params = { 0 };
+
+	ctx->stream->link = NULL;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_link_setup_replay(ctx->stream, &vrr_params));
+}
+
+/**
+ * dm_test_replay_link_setup_null_vrr_params - Verify NULL VRR params rejection
+ * @test: KUnit test context
+ *
+ * amdgpu_dm_link_setup_replay() should return false when VRR parameters are
+ * not supplied.
+ */
+static void dm_test_replay_link_setup_null_vrr_params(struct kunit *test)
+{
+	struct replay_test_ctx *ctx = alloc_replay_ctx(test);
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_link_setup_replay(ctx->stream, NULL));
+}
+
+/**
+ * dm_test_replay_link_setup_not_supported - Verify unsupported Replay rejection
+ * @test: KUnit test context
+ *
+ * amdgpu_dm_link_setup_replay() should return false when Replay is not marked
+ * supported on the link configuration.
+ */
+static void dm_test_replay_link_setup_not_supported(struct kunit *test)
+{
+	struct replay_test_ctx *ctx = alloc_replay_ctx(test);
+	struct mod_vrr_params vrr_params = { 0 };
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_link_setup_replay(ctx->stream, &vrr_params));
+}
+
+/**
+ * dm_test_replay_link_setup_already_enabled - Verify enabled Replay success
+ * @test: KUnit test context
+ *
+ * When Replay is already enabled, amdgpu_dm_link_setup_replay() should return
+ * true without recalculating coasting vtotal state.
+ */
+static void dm_test_replay_link_setup_already_enabled(struct kunit *test)
+{
+	struct replay_test_ctx *ctx = alloc_replay_ctx(test);
+	struct mod_vrr_params vrr_params = { 0 };
+
+	ctx->link->replay_settings.config.replay_supported = true;
+	ctx->link->replay_settings.replay_feature_enabled = true;
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_link_setup_replay(ctx->stream, &vrr_params));
+}
+
+/**
+ * dm_test_replay_link_setup_success - Verify coasting vtotal configuration
+ * @test: KUnit test context
+ *
+ * When Replay is supported but not yet enabled, amdgpu_dm_link_setup_replay()
+ * should calculate the link-off frame count and set the coasting vtotal values,
+ * then return true.
+ */
+static void dm_test_replay_link_setup_success(struct kunit *test)
+{
+	struct replay_test_ctx *ctx = alloc_replay_ctx(test);
+	struct mod_vrr_params vrr_params = { 0 };
+
+	ctx->link->replay_settings.config.replay_supported = true;
+	ctx->link->replay_settings.config.replay_version = DC_FREESYNC_REPLAY;
+
+	/* Set timing so calculate_replay_link_off_frame_count computes */
+	ctx->stream->timing.v_total = 1125;
+	ctx->stream->timing.h_total = 2200;
+	ctx->stream->timing.pix_clk_100hz = 1485000;
+	ctx->link->dpcd_caps.pr_info.pixel_deviation_per_line = 4;
+	ctx->link->dpcd_caps.pr_info.max_deviation_line = 10;
+
+	/* min_refresh_in_uhz = 0 makes calc return v_total directly */
+	vrr_params.min_refresh_in_uhz = 0;
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_link_setup_replay(ctx->stream, &vrr_params));
+
+	/* Verify coasting vtotal was set */
+	KUNIT_EXPECT_EQ(test,
+			ctx->link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_NOM],
+			(uint32_t)1125);
+	KUNIT_EXPECT_EQ(test,
+			ctx->link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_STATIC],
+			(uint32_t)1125);
+
+	/* Verify link_off_frame_count was calculated: 2200*10/(4*1125) = 4 */
+	KUNIT_EXPECT_EQ(test,
+			ctx->link->replay_settings.link_off_frame_count,
+			(uint32_t)4);
+}
+
+/* Tests for amdgpu_dm_replay_set_event() */
+
+/**
+ * dm_test_replay_set_event_null_stream - Verify NULL stream rejection
+ * @test: KUnit test context
+ *
+ * amdgpu_dm_replay_set_event() should return false when no stream is provided.
+ */
+static void dm_test_replay_set_event_null_stream(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm;
+
+	dm = kunit_kzalloc(test, sizeof(*dm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm);
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_replay_set_event(dm, NULL, true,
+							    replay_event_vsync, false));
+}
+
+/**
+ * dm_test_replay_set_event_null_link - Verify NULL stream link rejection
+ * @test: KUnit test context
+ *
+ * amdgpu_dm_replay_set_event() should return false when the stream has no
+ * associated link.
+ */
+static void dm_test_replay_set_event_null_link(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm;
+	struct replay_test_ctx *ctx = alloc_replay_ctx(test);
+
+	dm = kunit_kzalloc(test, sizeof(*dm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm);
+
+	ctx->stream->link = NULL;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_replay_set_event(dm, ctx->stream, true,
+							    replay_event_vsync, false));
+}
+
+/**
+ * dm_test_replay_set_event_feature_disabled - Verify disabled Replay rejection
+ * @test: KUnit test context
+ *
+ * amdgpu_dm_replay_set_event() should return false when Replay is not enabled
+ * on the stream link.
+ */
+static void dm_test_replay_set_event_feature_disabled(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm;
+	struct replay_test_ctx *ctx = alloc_replay_ctx(test);
+
+	dm = kunit_kzalloc(test, sizeof(*dm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm);
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_replay_set_event(dm, ctx->stream, true,
+							    replay_event_vsync, false));
+}
+
+/**
+ * dm_test_replay_set_event_missing_power_module - Verify missing power rejection
+ * @test: KUnit test context
+ *
+ * When Replay is enabled but no power module is available, the event helper
+ * should return false after failing to read the current Replay events.
+ */
+static void dm_test_replay_set_event_missing_power_module(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm;
+	struct replay_test_ctx *ctx = alloc_replay_ctx(test);
+
+	dm = kunit_kzalloc(test, sizeof(*dm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm);
+
+	ctx->link->replay_settings.replay_feature_enabled = true;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_replay_set_event(dm, ctx->stream, true,
+							    replay_event_vsync, false));
+}
+
+/**
+ * dm_test_replay_set_event_already_set - Verify no-op when event already active
+ * @test: KUnit test context
+ *
+ * When the requested event is already in the desired state, the function should
+ * return true without calling mod_power_set_replay_event().
+ */
+static void dm_test_replay_set_event_already_set(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm;
+	struct replay_test_ctx *ctx = alloc_replay_ctx(test);
+	struct core_power *core_power;
+	struct power_entity *map;
+
+	dm = kunit_kzalloc(test, sizeof(*dm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm);
+
+	core_power = kunit_kzalloc(test, sizeof(*core_power), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, core_power);
+
+	map = kunit_kzalloc(test, sizeof(*map), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, map);
+
+	/* Wire the power module so mod_power_get_replay_event() succeeds */
+	map->stream = ctx->stream;
+	map->replay_events = replay_event_vsync;
+	core_power->map = map;
+	core_power->num_entities = 1;
+	dm->power_module = &core_power->mod_public;
+
+	ctx->link->replay_settings.replay_feature_enabled = true;
+
+	/* Event already set — should return true without calling set */
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_replay_set_event(dm, ctx->stream, true,
+							   replay_event_vsync, false));
+}
+
+/**
+ * dm_test_replay_set_event_already_clear - Verify no-op when event already cleared
+ * @test: KUnit test context
+ *
+ * When clearing an event that is not currently active, the function should
+ * return true without calling mod_power_set_replay_event().
+ */
+static void dm_test_replay_set_event_already_clear(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm;
+	struct replay_test_ctx *ctx = alloc_replay_ctx(test);
+	struct core_power *core_power;
+	struct power_entity *map;
+
+	dm = kunit_kzalloc(test, sizeof(*dm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm);
+
+	core_power = kunit_kzalloc(test, sizeof(*core_power), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, core_power);
+
+	map = kunit_kzalloc(test, sizeof(*map), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, map);
+
+	/* Wire the power module — replay_events has NO vsync bit */
+	map->stream = ctx->stream;
+	map->replay_events = 0;
+	core_power->map = map;
+	core_power->num_entities = 1;
+	dm->power_module = &core_power->mod_public;
+
+	ctx->link->replay_settings.replay_feature_enabled = true;
+
+	/* Clearing an event that's already clear — should return true */
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_replay_set_event(dm, ctx->stream, false,
+							   replay_event_vsync, false));
+}
+
 static struct kunit_case dm_replay_test_cases[] = {
+	/* amdgpu_dm_link_supports_replay */
 	KUNIT_CASE(dm_test_replay_supports_all_caps),
 	KUNIT_CASE(dm_test_replay_no_freesync),
 	KUNIT_CASE(dm_test_replay_no_vsdb_replay_mode),
@@ -191,6 +590,27 @@ static struct kunit_case dm_replay_test_cases[] = {
 	KUNIT_CASE(dm_test_replay_zero_pixel_deviation),
 	KUNIT_CASE(dm_test_replay_zero_max_deviation_line),
 	KUNIT_CASE(dm_test_replay_both_deviations_zero),
+	/* amdgpu_dm_set_replay_caps */
+	KUNIT_CASE(dm_test_replay_set_caps_already_supported),
+	KUNIT_CASE(dm_test_replay_set_caps_non_embedded_signal),
+	KUNIT_CASE(dm_test_replay_set_caps_disallowed_by_panel),
+	KUNIT_CASE(dm_test_replay_set_caps_link_not_supported),
+	KUNIT_CASE(dm_test_replay_set_caps_missing_dmub_srv),
+	KUNIT_CASE(dm_test_replay_set_caps_success),
+	/* amdgpu_dm_link_setup_replay */
+	KUNIT_CASE(dm_test_replay_link_setup_null_stream),
+	KUNIT_CASE(dm_test_replay_link_setup_null_link),
+	KUNIT_CASE(dm_test_replay_link_setup_null_vrr_params),
+	KUNIT_CASE(dm_test_replay_link_setup_not_supported),
+	KUNIT_CASE(dm_test_replay_link_setup_already_enabled),
+	KUNIT_CASE(dm_test_replay_link_setup_success),
+	/* amdgpu_dm_replay_set_event */
+	KUNIT_CASE(dm_test_replay_set_event_null_stream),
+	KUNIT_CASE(dm_test_replay_set_event_null_link),
+	KUNIT_CASE(dm_test_replay_set_event_feature_disabled),
+	KUNIT_CASE(dm_test_replay_set_event_missing_power_module),
+	KUNIT_CASE(dm_test_replay_set_event_already_set),
+	KUNIT_CASE(dm_test_replay_set_event_already_clear),
 	{}
 };
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_services_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_services_test.c
new file mode 100644
index 0000000000000..e48bac7fb024c
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_services_test.c
@@ -0,0 +1,313 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * KUnit tests for amdgpu_dm_services.c
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#include <kunit/test.h>
+
+#include "dc.h"
+#include "amdgpu.h"
+#include "amdgpu_mode.h"
+#include "amdgpu_dm.h"
+#include "dm_services.h"
+#include "dm_services_types.h"
+
+/* Tests for dm_get_elapse_time_in_ns() */
+
+/**
+ * dm_test_get_elapse_time_zero_delta - Test Get elapse time zero delta
+ * @test: The KUnit test context
+ */
+static void dm_test_get_elapse_time_zero_delta(struct kunit *test)
+{
+	unsigned long long ts = 1000000ULL;
+
+	KUNIT_EXPECT_EQ(test, dm_get_elapse_time_in_ns(NULL, ts, ts), 0ULL);
+}
+
+/**
+ * dm_test_get_elapse_time_positive_delta - Test Get elapse time positive delta
+ * @test: The KUnit test context
+ */
+static void dm_test_get_elapse_time_positive_delta(struct kunit *test)
+{
+	unsigned long long current_ts = 5000000ULL;
+	unsigned long long last_ts = 1000000ULL;
+
+	KUNIT_EXPECT_EQ(test, dm_get_elapse_time_in_ns(NULL, current_ts, last_ts),
+			4000000ULL);
+}
+
+/**
+ * dm_test_get_elapse_time_large_delta - Test Get elapse time large delta
+ * @test: The KUnit test context
+ */
+static void dm_test_get_elapse_time_large_delta(struct kunit *test)
+{
+	unsigned long long current_ts = ULLONG_MAX;
+	unsigned long long last_ts = 0ULL;
+
+	KUNIT_EXPECT_EQ(test, dm_get_elapse_time_in_ns(NULL, current_ts, last_ts),
+			ULLONG_MAX);
+}
+
+/**
+ * dm_test_get_elapse_time_wraparound - Test Get elapse time wraparound
+ * @test: The KUnit test context
+ */
+static void dm_test_get_elapse_time_wraparound(struct kunit *test)
+{
+	/* Unsigned wraparound: result = ULLONG_MAX - last + current + 1 */
+	unsigned long long current_ts = 5ULL;
+	unsigned long long last_ts = ULLONG_MAX - 4ULL;
+
+	KUNIT_EXPECT_EQ(test, dm_get_elapse_time_in_ns(NULL, current_ts, last_ts),
+			10ULL);
+}
+
+/* Tests for dm_perf_trace_timestamp() */
+
+/**
+ * dm_test_perf_trace_timestamp_basic - Test Perf trace timestamp basic
+ * @test: The KUnit test context
+ *
+ * The tracepoint is a no-op without an attached probe, so this verifies the
+ * function dereferences ctx->perf_trace safely and does not crash.
+ */
+static void dm_test_perf_trace_timestamp_basic(struct kunit *test)
+{
+	struct dc_context *ctx;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	ctx->perf_trace = kunit_kzalloc(test, sizeof(*ctx->perf_trace), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->perf_trace);
+
+	ctx->perf_trace->read_count = 10;
+	ctx->perf_trace->write_count = 20;
+
+	dm_perf_trace_timestamp(__func__, __LINE__, ctx);
+}
+
+/* Tests for dm_trace_smu_enter() */
+
+/**
+ * dm_test_trace_smu_enter_null_ctx - Test Trace smu enter null ctx
+ * @test: The KUnit test context
+ */
+static void dm_test_trace_smu_enter_null_ctx(struct kunit *test)
+{
+	/* Empty stub — must not crash with NULL ctx */
+	dm_trace_smu_enter(0, 0, 0, NULL);
+}
+
+/**
+ * dm_test_trace_smu_enter_with_params - Test Trace smu enter with params
+ * @test: The KUnit test context
+ */
+static void dm_test_trace_smu_enter_with_params(struct kunit *test)
+{
+	/* Exercise non-zero msg_id, param_in, and delay */
+	dm_trace_smu_enter(0xFF, 0x12345678, 1000, NULL);
+}
+
+/* Tests for dm_trace_smu_exit() */
+
+/**
+ * dm_test_trace_smu_exit_success_null_ctx - Test Trace smu exit success null ctx
+ * @test: The KUnit test context
+ */
+static void dm_test_trace_smu_exit_success_null_ctx(struct kunit *test)
+{
+	/* Empty stub — must not crash on success path with NULL ctx */
+	dm_trace_smu_exit(true, 0x0, NULL);
+}
+
+/**
+ * dm_test_trace_smu_exit_failure_null_ctx - Test Trace smu exit failure null ctx
+ * @test: The KUnit test context
+ */
+static void dm_test_trace_smu_exit_failure_null_ctx(struct kunit *test)
+{
+	/* Empty stub — must not crash on failure path with NULL ctx */
+	dm_trace_smu_exit(false, 0x0, NULL);
+}
+
+/**
+ * dm_test_trace_smu_exit_with_response - Test Trace smu exit with response
+ * @test: The KUnit test context
+ */
+static void dm_test_trace_smu_exit_with_response(struct kunit *test)
+{
+	/* Exercise non-zero response value */
+	dm_trace_smu_exit(true, 0xDEADBEEF, NULL);
+}
+
+/* Tests for dm_query_extended_brightness_caps() */
+
+/**
+ * dm_test_query_brightness_caps_null_ctx - Test Query brightness caps null ctx
+ * @test: The KUnit test context
+ */
+static void dm_test_query_brightness_caps_null_ctx(struct kunit *test)
+{
+	struct dm_acpi_atif_backlight_caps caps = {};
+
+	KUNIT_EXPECT_FALSE(test,
+			   dm_query_extended_brightness_caps(NULL, AcpiDisplayType_LCD1, &caps));
+}
+
+/**
+ * dm_test_query_brightness_caps_null_caps - Test Query brightness caps null caps
+ * @test: The KUnit test context
+ */
+static void dm_test_query_brightness_caps_null_caps(struct kunit *test)
+{
+	struct dc_context ctx = {};
+
+	ctx.driver_context = (void *)0x1; /* non-NULL sentinel */
+
+	KUNIT_EXPECT_FALSE(test,
+			   dm_query_extended_brightness_caps(&ctx, AcpiDisplayType_LCD1, NULL));
+}
+
+/**
+ * dm_test_query_brightness_caps_null_driver_ctx - Test Query brightness caps null driver ctx
+ * @test: The KUnit test context
+ */
+static void dm_test_query_brightness_caps_null_driver_ctx(struct kunit *test)
+{
+	struct dc_context ctx = {};
+	struct dm_acpi_atif_backlight_caps caps = {};
+
+	ctx.driver_context = NULL;
+
+	KUNIT_EXPECT_FALSE(test,
+			   dm_query_extended_brightness_caps(&ctx, AcpiDisplayType_LCD1, &caps));
+}
+
+/**
+ * dm_test_query_brightness_caps_lcd2_null_ctx - Test Query brightness caps lcd2 null ctx
+ * @test: The KUnit test context
+ */
+static void dm_test_query_brightness_caps_lcd2_null_ctx(struct kunit *test)
+{
+	struct dm_acpi_atif_backlight_caps caps = {};
+
+	KUNIT_EXPECT_FALSE(test,
+			   dm_query_extended_brightness_caps(NULL, AcpiDisplayType_LCD2, &caps));
+}
+
+/**
+ * dm_test_query_brightness_caps_lcd1_success - Test Query brightness caps lcd1 success
+ * @test: The KUnit test context
+ */
+static void dm_test_query_brightness_caps_lcd1_success(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_dm_backlight_caps *source_caps;
+	struct dc_context ctx = {};
+	struct dm_acpi_atif_backlight_caps caps = {};
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	source_caps = &adev->dm.backlight_caps[0];
+	source_caps->caps_valid = true;
+	source_caps->min_input_signal = 12;
+	source_caps->max_input_signal = 240;
+	source_caps->ac_level = 80;
+	source_caps->dc_level = 40;
+	source_caps->data_points = 2;
+	source_caps->luminance_data[0].luminance = 10;
+	source_caps->luminance_data[0].input_signal = 22;
+	source_caps->luminance_data[1].luminance = 90;
+	source_caps->luminance_data[1].input_signal = 200;
+	ctx.driver_context = adev;
+
+	KUNIT_EXPECT_TRUE(test,
+			  dm_query_extended_brightness_caps(&ctx, AcpiDisplayType_LCD1, &caps));
+	KUNIT_EXPECT_EQ(test, caps.num_data_points, 2);
+	KUNIT_EXPECT_EQ(test, caps.max_input_signal, 240);
+	KUNIT_EXPECT_EQ(test, caps.min_input_signal, 12);
+	KUNIT_EXPECT_EQ(test, caps.ac_level_percentage, 80);
+	KUNIT_EXPECT_EQ(test, caps.dc_level_percentage, 40);
+	KUNIT_EXPECT_EQ(test, caps.data_points[0].luminance, 10);
+	KUNIT_EXPECT_EQ(test, caps.data_points[0].signal_level, 22);
+	KUNIT_EXPECT_EQ(test, caps.data_points[1].luminance, 90);
+	KUNIT_EXPECT_EQ(test, caps.data_points[1].signal_level, 200);
+}
+
+/**
+ * dm_test_query_brightness_caps_non_lcd1_uses_second_slot - Test Query brightness caps non lcd1 uses second slot
+ * @test: The KUnit test context
+ */
+static void dm_test_query_brightness_caps_non_lcd1_uses_second_slot(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_dm_backlight_caps *source_caps;
+	struct dc_context ctx = {};
+	struct dm_acpi_atif_backlight_caps caps = {};
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	adev->dm.backlight_caps[0].caps_valid = true;
+	adev->dm.backlight_caps[0].min_input_signal = 1;
+	adev->dm.backlight_caps[0].max_input_signal = 2;
+	source_caps = &adev->dm.backlight_caps[1];
+	source_caps->caps_valid = true;
+	source_caps->min_input_signal = 33;
+	source_caps->max_input_signal = 199;
+	source_caps->ac_level = 70;
+	source_caps->dc_level = 30;
+	source_caps->data_points = 0;
+	ctx.driver_context = adev;
+
+	KUNIT_EXPECT_TRUE(test,
+			  dm_query_extended_brightness_caps(&ctx, AcpiDisplayType_DFP1, &caps));
+	KUNIT_EXPECT_EQ(test, caps.num_data_points, 0);
+	KUNIT_EXPECT_EQ(test, caps.max_input_signal, 199);
+	KUNIT_EXPECT_EQ(test, caps.min_input_signal, 33);
+	KUNIT_EXPECT_EQ(test, caps.ac_level_percentage, 70);
+	KUNIT_EXPECT_EQ(test, caps.dc_level_percentage, 30);
+	KUNIT_EXPECT_EQ(test, caps.data_points[0].luminance, 0);
+	KUNIT_EXPECT_EQ(test, caps.data_points[0].signal_level, 0);
+}
+
+static struct kunit_case amdgpu_dm_services_test_cases[] = {
+	/* dm_get_elapse_time_in_ns */
+	KUNIT_CASE(dm_test_get_elapse_time_zero_delta),
+	KUNIT_CASE(dm_test_get_elapse_time_positive_delta),
+	KUNIT_CASE(dm_test_get_elapse_time_large_delta),
+	KUNIT_CASE(dm_test_get_elapse_time_wraparound),
+	/* dm_perf_trace_timestamp */
+	KUNIT_CASE(dm_test_perf_trace_timestamp_basic),
+	/* dm_trace_smu_enter */
+	KUNIT_CASE(dm_test_trace_smu_enter_null_ctx),
+	KUNIT_CASE(dm_test_trace_smu_enter_with_params),
+	/* dm_trace_smu_exit */
+	KUNIT_CASE(dm_test_trace_smu_exit_success_null_ctx),
+	KUNIT_CASE(dm_test_trace_smu_exit_failure_null_ctx),
+	KUNIT_CASE(dm_test_trace_smu_exit_with_response),
+	/* dm_query_extended_brightness_caps */
+	KUNIT_CASE(dm_test_query_brightness_caps_null_ctx),
+	KUNIT_CASE(dm_test_query_brightness_caps_null_caps),
+	KUNIT_CASE(dm_test_query_brightness_caps_null_driver_ctx),
+	KUNIT_CASE(dm_test_query_brightness_caps_lcd2_null_ctx),
+	KUNIT_CASE(dm_test_query_brightness_caps_lcd1_success),
+	KUNIT_CASE(dm_test_query_brightness_caps_non_lcd1_uses_second_slot),
+	{}
+};
+
+static struct kunit_suite amdgpu_dm_services_test_suite = {
+	.name = "amdgpu_dm_services",
+	.test_cases = amdgpu_dm_services_test_cases,
+};
+
+kunit_test_suite(amdgpu_dm_services_test_suite);
+
+MODULE_DESCRIPTION("KUnit tests for amdgpu_dm_services");
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_test.c
new file mode 100644
index 0000000000000..0b29bf0a7d04d
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_test.c
@@ -0,0 +1,949 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * KUnit tests for amdgpu_dm.c
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#include <kunit/test.h>
+
+#include "dc.h"
+#include "amdgpu_mode.h"
+#include "amdgpu_dm.h"
+
+/* Tests for dm_plane_layer_index_cmp() */
+
+/**
+ * dm_test_plane_layer_index_cmp_equal - Test Plane layer index cmp equal
+ * @test: The KUnit test context
+ */
+static void dm_test_plane_layer_index_cmp_equal(struct kunit *test)
+{
+	struct dc_plane_state *plane_a;
+	struct dc_plane_state *plane_b;
+	struct dc_surface_update sa, sb;
+
+	plane_a = kunit_kzalloc(test, sizeof(*plane_a), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, plane_a);
+	plane_b = kunit_kzalloc(test, sizeof(*plane_b), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, plane_b);
+
+	plane_a->layer_index = 5;
+	plane_b->layer_index = 5;
+	sa.surface = plane_a;
+	sb.surface = plane_b;
+
+	KUNIT_EXPECT_EQ(test, dm_plane_layer_index_cmp(&sa, &sb), 0);
+}
+
+/**
+ * dm_test_plane_layer_index_cmp_descending - Test Plane layer index cmp descending
+ * @test: The KUnit test context
+ */
+static void dm_test_plane_layer_index_cmp_descending(struct kunit *test)
+{
+	struct dc_plane_state *plane_a;
+	struct dc_plane_state *plane_b;
+	struct dc_surface_update sa, sb;
+
+	plane_a = kunit_kzalloc(test, sizeof(*plane_a), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, plane_a);
+	plane_b = kunit_kzalloc(test, sizeof(*plane_b), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, plane_b);
+
+	plane_a->layer_index = 3;
+	plane_b->layer_index = 7;
+	sa.surface = plane_a;
+	sb.surface = plane_b;
+
+	/* b has higher index, so cmp(a,b) = b - a > 0 (b sorts first) */
+	KUNIT_EXPECT_GT(test, dm_plane_layer_index_cmp(&sa, &sb), 0);
+}
+
+/**
+ * dm_test_plane_layer_index_cmp_ascending - Test Plane layer index cmp ascending
+ * @test: The KUnit test context
+ */
+static void dm_test_plane_layer_index_cmp_ascending(struct kunit *test)
+{
+	struct dc_plane_state *plane_a;
+	struct dc_plane_state *plane_b;
+	struct dc_surface_update sa, sb;
+
+	plane_a = kunit_kzalloc(test, sizeof(*plane_a), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, plane_a);
+	plane_b = kunit_kzalloc(test, sizeof(*plane_b), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, plane_b);
+
+	plane_a->layer_index = 9;
+	plane_b->layer_index = 2;
+	sa.surface = plane_a;
+	sb.surface = plane_b;
+
+	/* a has higher index, so cmp(a,b) = b - a < 0 (a sorts first) */
+	KUNIT_EXPECT_LT(test, dm_plane_layer_index_cmp(&sa, &sb), 0);
+}
+
+/* Tests for fill_plane_color_attributes() */
+
+/**
+ * dm_test_fill_color_attr_rgb_format - Test Fill color attr rgb format
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_color_attr_rgb_format(struct kunit *test)
+{
+	struct drm_plane_state plane_state = { 0 };
+	enum dc_color_space color_space = COLOR_SPACE_UNKNOWN;
+	int ret;
+
+	/* RGB format: should return 0 and set SRGB regardless of encoding */
+	plane_state.color_encoding = DRM_COLOR_YCBCR_BT709;
+	plane_state.color_range = DRM_COLOR_YCBCR_FULL_RANGE;
+
+	ret = fill_plane_color_attributes(&plane_state,
+					  SURFACE_PIXEL_FORMAT_GRPH_ARGB8888,
+					  &color_space);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, (int)color_space, (int)COLOR_SPACE_SRGB);
+}
+
+/**
+ * dm_test_fill_color_attr_bt601_full - Test Fill color attr bt601 full
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_color_attr_bt601_full(struct kunit *test)
+{
+	struct drm_plane_state plane_state = { 0 };
+	enum dc_color_space color_space = COLOR_SPACE_UNKNOWN;
+	int ret;
+
+	plane_state.color_encoding = DRM_COLOR_YCBCR_BT601;
+	plane_state.color_range = DRM_COLOR_YCBCR_FULL_RANGE;
+
+	ret = fill_plane_color_attributes(&plane_state,
+					  SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr,
+					  &color_space);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, (int)color_space, (int)COLOR_SPACE_YCBCR601);
+}
+
+/**
+ * dm_test_fill_color_attr_bt601_limited - Test Fill color attr bt601 limited
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_color_attr_bt601_limited(struct kunit *test)
+{
+	struct drm_plane_state plane_state = { 0 };
+	enum dc_color_space color_space = COLOR_SPACE_UNKNOWN;
+	int ret;
+
+	plane_state.color_encoding = DRM_COLOR_YCBCR_BT601;
+	plane_state.color_range = DRM_COLOR_YCBCR_LIMITED_RANGE;
+
+	ret = fill_plane_color_attributes(&plane_state,
+					  SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr,
+					  &color_space);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, (int)color_space,
+			(int)COLOR_SPACE_YCBCR601_LIMITED);
+}
+
+/**
+ * dm_test_fill_color_attr_bt709_full - Test Fill color attr bt709 full
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_color_attr_bt709_full(struct kunit *test)
+{
+	struct drm_plane_state plane_state = { 0 };
+	enum dc_color_space color_space = COLOR_SPACE_UNKNOWN;
+	int ret;
+
+	plane_state.color_encoding = DRM_COLOR_YCBCR_BT709;
+	plane_state.color_range = DRM_COLOR_YCBCR_FULL_RANGE;
+
+	ret = fill_plane_color_attributes(&plane_state,
+					  SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr,
+					  &color_space);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, (int)color_space, (int)COLOR_SPACE_YCBCR709);
+}
+
+/**
+ * dm_test_fill_color_attr_bt709_limited - Test Fill color attr bt709 limited
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_color_attr_bt709_limited(struct kunit *test)
+{
+	struct drm_plane_state plane_state = { 0 };
+	enum dc_color_space color_space = COLOR_SPACE_UNKNOWN;
+	int ret;
+
+	plane_state.color_encoding = DRM_COLOR_YCBCR_BT709;
+	plane_state.color_range = DRM_COLOR_YCBCR_LIMITED_RANGE;
+
+	ret = fill_plane_color_attributes(&plane_state,
+					  SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr,
+					  &color_space);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, (int)color_space,
+			(int)COLOR_SPACE_YCBCR709_LIMITED);
+}
+
+/**
+ * dm_test_fill_color_attr_bt2020_full - Test Fill color attr bt2020 full
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_color_attr_bt2020_full(struct kunit *test)
+{
+	struct drm_plane_state plane_state = { 0 };
+	enum dc_color_space color_space = COLOR_SPACE_UNKNOWN;
+	int ret;
+
+	plane_state.color_encoding = DRM_COLOR_YCBCR_BT2020;
+	plane_state.color_range = DRM_COLOR_YCBCR_FULL_RANGE;
+
+	ret = fill_plane_color_attributes(&plane_state,
+					  SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr,
+					  &color_space);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, (int)color_space,
+			(int)COLOR_SPACE_2020_YCBCR_FULL);
+}
+
+/**
+ * dm_test_fill_color_attr_bt2020_limited - Test Fill color attr bt2020 limited
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_color_attr_bt2020_limited(struct kunit *test)
+{
+	struct drm_plane_state plane_state = { 0 };
+	enum dc_color_space color_space = COLOR_SPACE_UNKNOWN;
+	int ret;
+
+	plane_state.color_encoding = DRM_COLOR_YCBCR_BT2020;
+	plane_state.color_range = DRM_COLOR_YCBCR_LIMITED_RANGE;
+
+	ret = fill_plane_color_attributes(&plane_state,
+					  SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr,
+					  &color_space);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, (int)color_space,
+			(int)COLOR_SPACE_2020_YCBCR_LIMITED);
+}
+
+/**
+ * dm_test_fill_color_attr_invalid_encoding - Test Fill color attr invalid encoding
+ * @test: The KUnit test context
+ */
+static void dm_test_fill_color_attr_invalid_encoding(struct kunit *test)
+{
+	struct drm_plane_state plane_state = { 0 };
+	enum dc_color_space color_space = COLOR_SPACE_UNKNOWN;
+	int ret;
+
+	plane_state.color_encoding = 99;
+	plane_state.color_range = DRM_COLOR_YCBCR_FULL_RANGE;
+
+	ret = fill_plane_color_attributes(&plane_state,
+					  SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr,
+					  &color_space);
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+}
+
+/* Tests for modereset_required() */
+
+/**
+ * dm_test_modereset_required_when_inactive_and_modeset - Test Modereset required when inactive and modeset
+ * @test: The KUnit test context
+ */
+static void dm_test_modereset_required_when_inactive_and_modeset(struct kunit *test)
+{
+	struct drm_crtc_state crtc_state = { 0 };
+
+	crtc_state.active = false;
+	crtc_state.mode_changed = true;
+
+	KUNIT_EXPECT_TRUE(test, modereset_required(&crtc_state));
+}
+
+/**
+ * dm_test_modereset_not_required_when_active_and_modeset - Test Modereset not required when active and modeset
+ * @test: The KUnit test context
+ */
+static void dm_test_modereset_not_required_when_active_and_modeset(struct kunit *test)
+{
+	struct drm_crtc_state crtc_state = { 0 };
+
+	crtc_state.active = true;
+	crtc_state.mode_changed = true;
+
+	KUNIT_EXPECT_FALSE(test, modereset_required(&crtc_state));
+}
+
+/**
+ * dm_test_modereset_not_required_when_inactive_without_modeset - Test Modereset not required when inactive without modeset
+ * @test: The KUnit test context
+ */
+static void dm_test_modereset_not_required_when_inactive_without_modeset(struct kunit *test)
+{
+	struct drm_crtc_state crtc_state = { 0 };
+
+	crtc_state.active = false;
+	crtc_state.mode_changed = false;
+
+	KUNIT_EXPECT_FALSE(test, modereset_required(&crtc_state));
+}
+
+/* Tests for dm_get_oriented_plane_size() */
+
+/**
+ * dm_test_oriented_plane_size_rotate_0 - Test Oriented plane size rotate 0
+ * @test: The KUnit test context
+ */
+static void dm_test_oriented_plane_size_rotate_0(struct kunit *test)
+{
+	struct drm_plane_state plane_state = { 0 };
+	int src_w = 0;
+	int src_h = 0;
+
+	plane_state.rotation = DRM_MODE_ROTATE_0;
+	plane_state.src_w = 1920 << 16;
+	plane_state.src_h = 1080 << 16;
+
+	dm_get_oriented_plane_size(&plane_state, &src_w, &src_h);
+
+	KUNIT_EXPECT_EQ(test, src_w, 1920);
+	KUNIT_EXPECT_EQ(test, src_h, 1080);
+}
+
+/**
+ * dm_test_oriented_plane_size_rotate_90 - Test Oriented plane size rotate 90
+ * @test: The KUnit test context
+ */
+static void dm_test_oriented_plane_size_rotate_90(struct kunit *test)
+{
+	struct drm_plane_state plane_state = { 0 };
+	int src_w = 0;
+	int src_h = 0;
+
+	plane_state.rotation = DRM_MODE_ROTATE_90;
+	plane_state.src_w = 1920 << 16;
+	plane_state.src_h = 1080 << 16;
+
+	dm_get_oriented_plane_size(&plane_state, &src_w, &src_h);
+
+	KUNIT_EXPECT_EQ(test, src_w, 1080);
+	KUNIT_EXPECT_EQ(test, src_h, 1920);
+}
+
+/**
+ * dm_test_oriented_plane_size_rotate_180 - Test Oriented plane size rotate 180
+ * @test: The KUnit test context
+ */
+static void dm_test_oriented_plane_size_rotate_180(struct kunit *test)
+{
+	struct drm_plane_state plane_state = { 0 };
+	int src_w = 0;
+	int src_h = 0;
+
+	plane_state.rotation = DRM_MODE_ROTATE_180;
+	plane_state.src_w = 1920 << 16;
+	plane_state.src_h = 1080 << 16;
+
+	dm_get_oriented_plane_size(&plane_state, &src_w, &src_h);
+
+	KUNIT_EXPECT_EQ(test, src_w, 1920);
+	KUNIT_EXPECT_EQ(test, src_h, 1080);
+}
+
+/**
+ * dm_test_oriented_plane_size_rotate_270 - Test Oriented plane size rotate 270
+ * @test: The KUnit test context
+ */
+static void dm_test_oriented_plane_size_rotate_270(struct kunit *test)
+{
+	struct drm_plane_state plane_state = { 0 };
+	int src_w = 0;
+	int src_h = 0;
+
+	plane_state.rotation = DRM_MODE_ROTATE_270;
+	plane_state.src_w = 1920 << 16;
+	plane_state.src_h = 1080 << 16;
+
+	dm_get_oriented_plane_size(&plane_state, &src_w, &src_h);
+
+	KUNIT_EXPECT_EQ(test, src_w, 1080);
+	KUNIT_EXPECT_EQ(test, src_h, 1920);
+}
+
+/* Tests for dm_get_plane_scale() */
+
+/**
+ * dm_test_get_plane_scale_identity - Test Get plane scale identity
+ * @test: The KUnit test context
+ */
+static void dm_test_get_plane_scale_identity(struct kunit *test)
+{
+	struct drm_plane_state plane_state = { 0 };
+	int scale_w = 0;
+	int scale_h = 0;
+
+	plane_state.rotation = DRM_MODE_ROTATE_0;
+	plane_state.src_w = 1920 << 16;
+	plane_state.src_h = 1080 << 16;
+	plane_state.crtc_w = 1920;
+	plane_state.crtc_h = 1080;
+
+	dm_get_plane_scale(&plane_state, &scale_w, &scale_h);
+
+	KUNIT_EXPECT_EQ(test, scale_w, 1000);
+	KUNIT_EXPECT_EQ(test, scale_h, 1000);
+}
+
+/**
+ * dm_test_get_plane_scale_rotate_90_identity - Test Get plane scale rotate 90 identity
+ * @test: The KUnit test context
+ */
+static void dm_test_get_plane_scale_rotate_90_identity(struct kunit *test)
+{
+	struct drm_plane_state plane_state = { 0 };
+	int scale_w = 0;
+	int scale_h = 0;
+
+	plane_state.rotation = DRM_MODE_ROTATE_90;
+	plane_state.src_w = 1920 << 16;
+	plane_state.src_h = 1080 << 16;
+	plane_state.crtc_w = 1080;
+	plane_state.crtc_h = 1920;
+
+	dm_get_plane_scale(&plane_state, &scale_w, &scale_h);
+
+	KUNIT_EXPECT_EQ(test, scale_w, 1000);
+	KUNIT_EXPECT_EQ(test, scale_h, 1000);
+}
+
+/**
+ * dm_test_get_plane_scale_zero_src_width - Test Get plane scale zero src width
+ * @test: The KUnit test context
+ */
+static void dm_test_get_plane_scale_zero_src_width(struct kunit *test)
+{
+	struct drm_plane_state plane_state = { 0 };
+	int scale_w = 0;
+	int scale_h = 0;
+
+	plane_state.rotation = DRM_MODE_ROTATE_0;
+	plane_state.src_w = 0;
+	plane_state.src_h = 1080 << 16;
+	plane_state.crtc_w = 100;
+	plane_state.crtc_h = 200;
+
+	dm_get_plane_scale(&plane_state, &scale_w, &scale_h);
+
+	KUNIT_EXPECT_EQ(test, scale_w, 0);
+	KUNIT_EXPECT_EQ(test, scale_h, 185);
+}
+
+/* Tests for is_scaling_state_different() */
+
+/**
+ * dm_test_scaling_state_same - Test identical scaling states compare equal
+ * @test: The KUnit test context
+ */
+static void dm_test_scaling_state_same(struct kunit *test)
+{
+	struct dm_connector_state *a;
+	struct dm_connector_state *b;
+
+	a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL);
+	b = kunit_kzalloc(test, sizeof(*b), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, a);
+	KUNIT_ASSERT_NOT_NULL(test, b);
+
+	a->scaling = RMX_FULL;
+	a->underscan_enable = false;
+	*b = *a;
+
+	KUNIT_EXPECT_FALSE(test, is_scaling_state_different(a, b));
+}
+
+/**
+ * dm_test_scaling_state_scaling_changed - Test differing scaling mode is detected
+ * @test: The KUnit test context
+ */
+static void dm_test_scaling_state_scaling_changed(struct kunit *test)
+{
+	struct dm_connector_state *a;
+	struct dm_connector_state *b;
+
+	a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL);
+	b = kunit_kzalloc(test, sizeof(*b), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, a);
+	KUNIT_ASSERT_NOT_NULL(test, b);
+
+	a->scaling = RMX_FULL;
+	b->scaling = RMX_CENTER;
+
+	KUNIT_EXPECT_TRUE(test, is_scaling_state_different(a, b));
+}
+
+/**
+ * dm_test_scaling_state_underscan_enabled - Test enabling underscan with borders differs
+ * @test: The KUnit test context
+ */
+static void dm_test_scaling_state_underscan_enabled(struct kunit *test)
+{
+	struct dm_connector_state *old_state;
+	struct dm_connector_state *new_state;
+
+	old_state = kunit_kzalloc(test, sizeof(*old_state), GFP_KERNEL);
+	new_state = kunit_kzalloc(test, sizeof(*new_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_state);
+	KUNIT_ASSERT_NOT_NULL(test, new_state);
+
+	/* new enables underscan with non-zero borders, old has it disabled */
+	new_state->underscan_enable = true;
+	new_state->underscan_hborder = 16;
+	new_state->underscan_vborder = 16;
+	old_state->underscan_enable = false;
+
+	KUNIT_EXPECT_TRUE(test, is_scaling_state_different(new_state, old_state));
+}
+
+/**
+ * dm_test_scaling_state_underscan_border_changed - Test changed underscan borders differ
+ * @test: The KUnit test context
+ */
+static void dm_test_scaling_state_underscan_border_changed(struct kunit *test)
+{
+	struct dm_connector_state *a;
+	struct dm_connector_state *b;
+
+	a = kunit_kzalloc(test, sizeof(*a), GFP_KERNEL);
+	b = kunit_kzalloc(test, sizeof(*b), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, a);
+	KUNIT_ASSERT_NOT_NULL(test, b);
+
+	a->underscan_enable = true;
+	a->underscan_hborder = 16;
+	a->underscan_vborder = 16;
+	*b = *a;
+	b->underscan_hborder = 32;
+
+	KUNIT_EXPECT_TRUE(test, is_scaling_state_different(a, b));
+}
+
+/* Tests for is_timing_unchanged_for_freesync() */
+
+/**
+ * dm_test_timing_unchanged_null_args - Test NULL crtc states return false
+ * @test: The KUnit test context
+ */
+static void dm_test_timing_unchanged_null_args(struct kunit *test)
+{
+	struct drm_crtc_state crtc_state = { 0 };
+
+	KUNIT_EXPECT_FALSE(test,
+			   is_timing_unchanged_for_freesync(NULL, &crtc_state));
+	KUNIT_EXPECT_FALSE(test,
+			   is_timing_unchanged_for_freesync(&crtc_state, NULL));
+}
+
+/**
+ * dm_test_timing_unchanged_identical_modes - Test identical modes are not "unchanged"
+ * @test: The KUnit test context
+ *
+ * The helper only returns true when vtotal/vsync shift (vrr) while the rest
+ * of the timing stays fixed, so identical modes must return false.
+ */
+static void dm_test_timing_unchanged_identical_modes(struct kunit *test)
+{
+	struct drm_crtc_state old_state = { 0 };
+	struct drm_crtc_state new_state = { 0 };
+
+	old_state.mode.clock = 148500;
+	old_state.mode.hdisplay = 1920;
+	old_state.mode.vdisplay = 1080;
+	old_state.mode.htotal = 2200;
+	old_state.mode.vtotal = 1125;
+	new_state.mode = old_state.mode;
+
+	KUNIT_EXPECT_FALSE(test,
+			   is_timing_unchanged_for_freesync(&old_state, &new_state));
+}
+
+/**
+ * dm_test_timing_unchanged_vrr_shift - Test vrr-style vtotal/vsync shift is detected
+ * @test: The KUnit test context
+ */
+static void dm_test_timing_unchanged_vrr_shift(struct kunit *test)
+{
+	struct drm_crtc_state old_state = { 0 };
+	struct drm_crtc_state new_state = { 0 };
+
+	old_state.mode.clock = 148500;
+	old_state.mode.hdisplay = 1920;
+	old_state.mode.vdisplay = 1080;
+	old_state.mode.htotal = 2200;
+	old_state.mode.vtotal = 1125;
+	old_state.mode.hsync_start = 2008;
+	old_state.mode.vsync_start = 1084;
+	old_state.mode.hsync_end = 2052;
+	old_state.mode.vsync_end = 1089;
+
+	/* Same horizontal timing, vertical totals/sync shifted by 125 lines */
+	new_state.mode = old_state.mode;
+	new_state.mode.vtotal = 1250;
+	new_state.mode.vsync_start = 1209;
+	new_state.mode.vsync_end = 1214;
+
+	KUNIT_EXPECT_TRUE(test,
+			  is_timing_unchanged_for_freesync(&old_state, &new_state));
+}
+
+/**
+ * dm_test_timing_unchanged_clock_changed - Test pixel clock change returns false
+ * @test: The KUnit test context
+ */
+static void dm_test_timing_unchanged_clock_changed(struct kunit *test)
+{
+	struct drm_crtc_state old_state = { 0 };
+	struct drm_crtc_state new_state = { 0 };
+
+	old_state.mode.clock = 148500;
+	old_state.mode.htotal = 2200;
+	old_state.mode.vtotal = 1125;
+	old_state.mode.vsync_start = 1084;
+	old_state.mode.vsync_end = 1089;
+
+	new_state.mode = old_state.mode;
+	new_state.mode.clock = 297000;
+	new_state.mode.vtotal = 1250;
+	new_state.mode.vsync_start = 1209;
+	new_state.mode.vsync_end = 1214;
+
+	KUNIT_EXPECT_FALSE(test,
+			   is_timing_unchanged_for_freesync(&old_state, &new_state));
+}
+
+/* Tests for set_freesync_fixed_config() */
+
+/**
+ * dm_test_set_freesync_fixed_config_60hz - Test fixed refresh computed for 1080p60
+ * @test: The KUnit test context
+ */
+static void dm_test_set_freesync_fixed_config_60hz(struct kunit *test)
+{
+	struct dm_crtc_state dm_crtc_state = { 0 };
+
+	dm_crtc_state.base.mode.clock = 148500;
+	dm_crtc_state.base.mode.htotal = 2200;
+	dm_crtc_state.base.mode.vtotal = 1125;
+
+	set_freesync_fixed_config(&dm_crtc_state);
+
+	KUNIT_EXPECT_EQ(test, (int)dm_crtc_state.freesync_config.state,
+			(int)VRR_STATE_ACTIVE_FIXED);
+	/* 148500 kHz / (2200 * 1125) = 60 Hz = 60000000 uHz */
+	KUNIT_EXPECT_EQ(test, dm_crtc_state.freesync_config.fixed_refresh_in_uhz,
+			60000000U);
+}
+
+/* Tests for is_dc_timing_adjust_needed() */
+
+/**
+ * dm_test_dc_timing_adjust_pending - Test a pending hw timing adjust forces true
+ * @test: The KUnit test context
+ */
+static void dm_test_dc_timing_adjust_pending(struct kunit *test)
+{
+	struct dm_crtc_state *old_state, *new_state;
+	struct dc_stream_state *stream;
+
+	old_state = kunit_kzalloc(test, sizeof(*old_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, old_state);
+	new_state = kunit_kzalloc(test, sizeof(*new_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, new_state);
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+
+	new_state->stream = stream;
+	stream->adjust.timing_adjust_pending = 1;
+
+	KUNIT_EXPECT_TRUE(test, is_dc_timing_adjust_needed(old_state, new_state));
+}
+
+/**
+ * dm_test_dc_timing_adjust_active_fixed - Test VRR active-fixed forces true
+ * @test: The KUnit test context
+ */
+static void dm_test_dc_timing_adjust_active_fixed(struct kunit *test)
+{
+	struct dm_crtc_state *old_state, *new_state;
+	struct dc_stream_state *stream;
+
+	old_state = kunit_kzalloc(test, sizeof(*old_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, old_state);
+	new_state = kunit_kzalloc(test, sizeof(*new_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, new_state);
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+
+	new_state->stream = stream;
+	new_state->freesync_config.state = VRR_STATE_ACTIVE_FIXED;
+
+	KUNIT_EXPECT_TRUE(test, is_dc_timing_adjust_needed(old_state, new_state));
+}
+
+/**
+ * dm_test_dc_timing_adjust_vrr_toggle - Test a change in vrr active state forces true
+ * @test: The KUnit test context
+ */
+static void dm_test_dc_timing_adjust_vrr_toggle(struct kunit *test)
+{
+	struct dm_crtc_state *old_state, *new_state;
+	struct dc_stream_state *stream;
+
+	old_state = kunit_kzalloc(test, sizeof(*old_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, old_state);
+	new_state = kunit_kzalloc(test, sizeof(*new_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, new_state);
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+
+	new_state->stream = stream;
+	old_state->freesync_config.state = VRR_STATE_ACTIVE_VARIABLE;
+	new_state->freesync_config.state = VRR_STATE_INACTIVE;
+
+	KUNIT_EXPECT_TRUE(test, is_dc_timing_adjust_needed(old_state, new_state));
+}
+
+/**
+ * dm_test_dc_timing_adjust_not_needed - Test steady-state timing needs no adjust
+ * @test: The KUnit test context
+ */
+static void dm_test_dc_timing_adjust_not_needed(struct kunit *test)
+{
+	struct dm_crtc_state *old_state, *new_state;
+	struct dc_stream_state *stream;
+
+	old_state = kunit_kzalloc(test, sizeof(*old_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, old_state);
+	new_state = kunit_kzalloc(test, sizeof(*new_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, new_state);
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+
+	new_state->stream = stream;
+	old_state->freesync_config.state = VRR_STATE_INACTIVE;
+	new_state->freesync_config.state = VRR_STATE_INACTIVE;
+
+	KUNIT_EXPECT_FALSE(test, is_dc_timing_adjust_needed(old_state, new_state));
+}
+
+/* Tests for set_multisync_trigger_params() */
+
+/**
+ * dm_test_multisync_trigger_disabled - Test disabled reset leaves params untouched
+ * @test: The KUnit test context
+ */
+static void dm_test_multisync_trigger_disabled(struct kunit *test)
+{
+	struct dc_stream_state *stream;
+
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+
+	stream->triggered_crtc_reset.enabled = false;
+	stream->triggered_crtc_reset.event = CRTC_EVENT_VSYNC_FALLING;
+	stream->triggered_crtc_reset.delay = TRIGGER_DELAY_NEXT_LINE;
+
+	set_multisync_trigger_params(stream);
+
+	/* Nothing should change when the reset trigger is disabled */
+	KUNIT_EXPECT_EQ(test, (int)stream->triggered_crtc_reset.event,
+			(int)CRTC_EVENT_VSYNC_FALLING);
+	KUNIT_EXPECT_EQ(test, (int)stream->triggered_crtc_reset.delay,
+			(int)TRIGGER_DELAY_NEXT_LINE);
+}
+
+/**
+ * dm_test_multisync_trigger_rising - Test positive vsync polarity selects rising edge
+ * @test: The KUnit test context
+ */
+static void dm_test_multisync_trigger_rising(struct kunit *test)
+{
+	struct dc_stream_state *stream;
+	struct dc_stream_state *master;
+
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+	master = kunit_kzalloc(test, sizeof(*master), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, master);
+
+	master->timing.flags.VSYNC_POSITIVE_POLARITY = 1;
+	stream->triggered_crtc_reset.enabled = true;
+	stream->triggered_crtc_reset.event_source = master;
+
+	set_multisync_trigger_params(stream);
+
+	KUNIT_EXPECT_EQ(test, (int)stream->triggered_crtc_reset.event,
+			(int)CRTC_EVENT_VSYNC_RISING);
+	KUNIT_EXPECT_EQ(test, (int)stream->triggered_crtc_reset.delay,
+			(int)TRIGGER_DELAY_NEXT_PIXEL);
+}
+
+/**
+ * dm_test_multisync_trigger_falling - Test negative vsync polarity selects falling edge
+ * @test: The KUnit test context
+ */
+static void dm_test_multisync_trigger_falling(struct kunit *test)
+{
+	struct dc_stream_state *stream;
+	struct dc_stream_state *master;
+
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream);
+	master = kunit_kzalloc(test, sizeof(*master), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, master);
+
+	master->timing.flags.VSYNC_POSITIVE_POLARITY = 0;
+	stream->triggered_crtc_reset.enabled = true;
+	stream->triggered_crtc_reset.event_source = master;
+
+	set_multisync_trigger_params(stream);
+
+	KUNIT_EXPECT_EQ(test, (int)stream->triggered_crtc_reset.event,
+			(int)CRTC_EVENT_VSYNC_FALLING);
+	KUNIT_EXPECT_EQ(test, (int)stream->triggered_crtc_reset.delay,
+			(int)TRIGGER_DELAY_NEXT_PIXEL);
+}
+
+/* Tests for set_master_stream() */
+
+/**
+ * dm_test_master_stream_highest_refresh - Test highest refresh-rate stream becomes master
+ * @test: The KUnit test context
+ */
+static void dm_test_master_stream_highest_refresh(struct kunit *test)
+{
+	struct dc_stream_state *stream0, *stream1;
+	struct dc_stream_state *stream_set[2];
+
+	stream0 = kunit_kzalloc(test, sizeof(*stream0), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream0);
+	stream1 = kunit_kzalloc(test, sizeof(*stream1), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream1);
+	stream_set[0] = stream0;
+	stream_set[1] = stream1;
+
+	/* stream0: 60Hz, stream1: 120Hz -> stream1 is master */
+	stream0->triggered_crtc_reset.enabled = true;
+	stream0->timing.pix_clk_100hz = 1485000;
+	stream0->timing.h_total = 2200;
+	stream0->timing.v_total = 1125;
+
+	stream1->triggered_crtc_reset.enabled = true;
+	stream1->timing.pix_clk_100hz = 2970000;
+	stream1->timing.h_total = 2200;
+	stream1->timing.v_total = 1125;
+
+	set_master_stream(stream_set, 2);
+
+	KUNIT_EXPECT_PTR_EQ(test, stream0->triggered_crtc_reset.event_source,
+			    stream1);
+	KUNIT_EXPECT_PTR_EQ(test, stream1->triggered_crtc_reset.event_source,
+			    stream1);
+}
+
+/**
+ * dm_test_master_stream_defaults_to_first - Test default master when none triggered
+ * @test: The KUnit test context
+ *
+ * When no stream has the reset trigger enabled, master_stream stays 0 and all
+ * streams point at the first stream as their event source.
+ */
+static void dm_test_master_stream_defaults_to_first(struct kunit *test)
+{
+	struct dc_stream_state *stream0, *stream1;
+	struct dc_stream_state *stream_set[2];
+
+	stream0 = kunit_kzalloc(test, sizeof(*stream0), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream0);
+	stream1 = kunit_kzalloc(test, sizeof(*stream1), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, stream1);
+	stream_set[0] = stream0;
+	stream_set[1] = stream1;
+
+	set_master_stream(stream_set, 2);
+
+	KUNIT_EXPECT_PTR_EQ(test, stream0->triggered_crtc_reset.event_source,
+			    stream0);
+	KUNIT_EXPECT_PTR_EQ(test, stream1->triggered_crtc_reset.event_source,
+			    stream0);
+}
+
+static struct kunit_case amdgpu_dm_tests[] = {
+	/* dm_plane_layer_index_cmp */
+	KUNIT_CASE(dm_test_plane_layer_index_cmp_equal),
+	KUNIT_CASE(dm_test_plane_layer_index_cmp_descending),
+	KUNIT_CASE(dm_test_plane_layer_index_cmp_ascending),
+	/* fill_plane_color_attributes */
+	KUNIT_CASE(dm_test_fill_color_attr_rgb_format),
+	KUNIT_CASE(dm_test_fill_color_attr_bt601_full),
+	KUNIT_CASE(dm_test_fill_color_attr_bt601_limited),
+	KUNIT_CASE(dm_test_fill_color_attr_bt709_full),
+	KUNIT_CASE(dm_test_fill_color_attr_bt709_limited),
+	KUNIT_CASE(dm_test_fill_color_attr_bt2020_full),
+	KUNIT_CASE(dm_test_fill_color_attr_bt2020_limited),
+	KUNIT_CASE(dm_test_fill_color_attr_invalid_encoding),
+	/* modereset_required */
+	KUNIT_CASE(dm_test_modereset_required_when_inactive_and_modeset),
+	KUNIT_CASE(dm_test_modereset_not_required_when_active_and_modeset),
+	KUNIT_CASE(dm_test_modereset_not_required_when_inactive_without_modeset),
+	/* dm_get_oriented_plane_size */
+	KUNIT_CASE(dm_test_oriented_plane_size_rotate_0),
+	KUNIT_CASE(dm_test_oriented_plane_size_rotate_90),
+	KUNIT_CASE(dm_test_oriented_plane_size_rotate_180),
+	KUNIT_CASE(dm_test_oriented_plane_size_rotate_270),
+	/* dm_get_plane_scale */
+	KUNIT_CASE(dm_test_get_plane_scale_identity),
+	KUNIT_CASE(dm_test_get_plane_scale_rotate_90_identity),
+	KUNIT_CASE(dm_test_get_plane_scale_zero_src_width),
+	/* is_scaling_state_different */
+	KUNIT_CASE(dm_test_scaling_state_same),
+	KUNIT_CASE(dm_test_scaling_state_scaling_changed),
+	KUNIT_CASE(dm_test_scaling_state_underscan_enabled),
+	KUNIT_CASE(dm_test_scaling_state_underscan_border_changed),
+	/* is_timing_unchanged_for_freesync */
+	KUNIT_CASE(dm_test_timing_unchanged_null_args),
+	KUNIT_CASE(dm_test_timing_unchanged_identical_modes),
+	KUNIT_CASE(dm_test_timing_unchanged_vrr_shift),
+	KUNIT_CASE(dm_test_timing_unchanged_clock_changed),
+	/* set_freesync_fixed_config */
+	KUNIT_CASE(dm_test_set_freesync_fixed_config_60hz),
+	/* is_dc_timing_adjust_needed */
+	KUNIT_CASE(dm_test_dc_timing_adjust_pending),
+	KUNIT_CASE(dm_test_dc_timing_adjust_active_fixed),
+	KUNIT_CASE(dm_test_dc_timing_adjust_vrr_toggle),
+	KUNIT_CASE(dm_test_dc_timing_adjust_not_needed),
+	/* set_multisync_trigger_params */
+	KUNIT_CASE(dm_test_multisync_trigger_disabled),
+	KUNIT_CASE(dm_test_multisync_trigger_rising),
+	KUNIT_CASE(dm_test_multisync_trigger_falling),
+	/* set_master_stream */
+	KUNIT_CASE(dm_test_master_stream_highest_refresh),
+	KUNIT_CASE(dm_test_master_stream_defaults_to_first),
+	{}
+};
+
+static struct kunit_suite amdgpu_dm_test_suite = {
+	.name = "amdgpu_dm",
+	.test_cases = amdgpu_dm_tests,
+};
+
+kunit_test_suite(amdgpu_dm_test_suite);
+
+MODULE_AUTHOR("AMD");
+MODULE_DESCRIPTION("KUnit tests for amdgpu_dm");
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_wb_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_wb_test.c
new file mode 100644
index 0000000000000..b43bc244487e2
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_wb_test.c
@@ -0,0 +1,436 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * KUnit tests for amdgpu_dm_wb.c
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#include <kunit/test.h>
+
+#include <drm/drm_atomic_state_helper.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_kunit_helpers.h>
+#include <drm/drm_mode.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_writeback.h>
+
+#include "dc.h"
+#include "amdgpu.h"
+#include "amdgpu_dm.h"
+#include "amdgpu_dm_wb.h"
+#include "amdgpu_dm_kunit_test_helpers.h"
+
+
+/* Helper functions */
+
+static struct drm_crtc_state *alloc_test_crtc_state(struct kunit *test,
+						    int hdisplay, int vdisplay)
+{
+	struct drm_crtc_state *crtc_state;
+
+	crtc_state = kunit_kzalloc(test, sizeof(*crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc_state);
+
+	crtc_state->mode.hdisplay = hdisplay;
+	crtc_state->mode.vdisplay = vdisplay;
+
+	return crtc_state;
+}
+
+static struct drm_connector_state *alloc_test_conn_state(struct kunit *test,
+							 int fb_width,
+							 int fb_height,
+							 u32 format)
+{
+	struct drm_connector_state *conn_state;
+	struct drm_writeback_job *job;
+	struct drm_framebuffer *fb;
+	struct drm_format_info *fmt_info;
+
+	conn_state = kunit_kzalloc(test, sizeof(*conn_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, conn_state);
+
+	job = kunit_kzalloc(test, sizeof(*job), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, job);
+
+	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+
+	fmt_info = kunit_kzalloc(test, sizeof(*fmt_info), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fmt_info);
+
+	fb->width = fb_width;
+	fb->height = fb_height;
+	fmt_info->format = format;
+	fb->format = fmt_info;
+
+	job->fb = fb;
+	conn_state->writeback_job = job;
+
+	return conn_state;
+}
+
+
+
+/* Tests for amdgpu_dm_wb_encoder_atomic_check */
+
+/**
+ * dm_test_wb_atomic_check_no_job - Verify early return when no writeback job
+ * @test: KUnit test context
+ *
+ * When conn_state->writeback_job is NULL, no writeback is requested and the
+ * function should return 0 without further validation.
+ */
+static void dm_test_wb_atomic_check_no_job(struct kunit *test)
+{
+	struct drm_crtc_state *crtc_state;
+	struct drm_connector_state *conn_state;
+	int ret;
+
+	crtc_state = alloc_test_crtc_state(test, 1920, 1080);
+	conn_state = kunit_kzalloc(test, sizeof(*conn_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, conn_state);
+
+	/* No writeback_job — should return 0 */
+	conn_state->writeback_job = NULL;
+	ret = amdgpu_dm_wb_encoder_atomic_check(NULL, crtc_state, conn_state);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+}
+
+/**
+ * dm_test_wb_atomic_check_no_fb - Verify early return when job has no framebuffer
+ * @test: KUnit test context
+ *
+ * When a writeback job exists but job->fb is NULL, the function should return 0
+ * without validating dimensions or pixel format.
+ */
+static void dm_test_wb_atomic_check_no_fb(struct kunit *test)
+{
+	struct drm_crtc_state *crtc_state;
+	struct drm_connector_state *conn_state;
+	struct drm_writeback_job *job;
+	int ret;
+
+	crtc_state = alloc_test_crtc_state(test, 1920, 1080);
+	conn_state = kunit_kzalloc(test, sizeof(*conn_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, conn_state);
+
+	job = kunit_kzalloc(test, sizeof(*job), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, job);
+
+	/* writeback_job exists but no fb — should return 0 */
+	job->fb = NULL;
+	conn_state->writeback_job = job;
+	ret = amdgpu_dm_wb_encoder_atomic_check(NULL, crtc_state, conn_state);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+}
+
+/**
+ * dm_test_wb_atomic_check_valid - Verify success with matching size and supported format
+ * @test: KUnit test context
+ *
+ * When the framebuffer dimensions match the CRTC mode and the pixel format is
+ * in the supported formats list, the function should return 0.
+ */
+static void dm_test_wb_atomic_check_valid(struct kunit *test)
+{
+	struct drm_crtc_state *crtc_state;
+	struct drm_connector_state *conn_state;
+	int ret;
+
+	crtc_state = alloc_test_crtc_state(test, 1920, 1080);
+	conn_state = alloc_test_conn_state(test, 1920, 1080,
+					   DRM_FORMAT_XRGB2101010);
+
+	ret = amdgpu_dm_wb_encoder_atomic_check(NULL, crtc_state, conn_state);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+}
+
+/**
+ * dm_test_wb_atomic_check_size_mismatch - Verify rejection when both dimensions differ
+ * @test: KUnit test context
+ *
+ * When both framebuffer width and height differ from the CRTC mode, the
+ * function should return -EINVAL.
+ */
+static void dm_test_wb_atomic_check_size_mismatch(struct kunit *test)
+{
+	struct drm_crtc_state *crtc_state;
+	struct drm_connector_state *conn_state;
+	int ret;
+
+	/* FB is 3840x2160 but mode is 1920x1080 */
+	crtc_state = alloc_test_crtc_state(test, 1920, 1080);
+	conn_state = alloc_test_conn_state(test, 3840, 2160,
+					   DRM_FORMAT_XRGB2101010);
+
+	ret = amdgpu_dm_wb_encoder_atomic_check(NULL, crtc_state, conn_state);
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+}
+
+/**
+ * dm_test_wb_atomic_check_width_mismatch - Verify rejection when width alone differs
+ * @test: KUnit test context
+ *
+ * When only the framebuffer width differs from the CRTC mode hdisplay, the
+ * function should return -EINVAL.
+ */
+static void dm_test_wb_atomic_check_width_mismatch(struct kunit *test)
+{
+	struct drm_crtc_state *crtc_state;
+	struct drm_connector_state *conn_state;
+	int ret;
+
+	/* Width doesn't match */
+	crtc_state = alloc_test_crtc_state(test, 1920, 1080);
+	conn_state = alloc_test_conn_state(test, 1280, 1080,
+					   DRM_FORMAT_XRGB2101010);
+
+	ret = amdgpu_dm_wb_encoder_atomic_check(NULL, crtc_state, conn_state);
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+}
+
+/**
+ * dm_test_wb_atomic_check_height_mismatch - Verify rejection when height alone differs
+ * @test: KUnit test context
+ *
+ * When only the framebuffer height differs from the CRTC mode vdisplay, the
+ * function should return -EINVAL.
+ */
+static void dm_test_wb_atomic_check_height_mismatch(struct kunit *test)
+{
+	struct drm_crtc_state *crtc_state;
+	struct drm_connector_state *conn_state;
+	int ret;
+
+	/* Height doesn't match */
+	crtc_state = alloc_test_crtc_state(test, 1920, 1080);
+	conn_state = alloc_test_conn_state(test, 1920, 720,
+					   DRM_FORMAT_XRGB2101010);
+
+	ret = amdgpu_dm_wb_encoder_atomic_check(NULL, crtc_state, conn_state);
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+}
+
+/**
+ * dm_test_wb_atomic_check_invalid_format - Verify rejection of unsupported pixel format
+ * @test: KUnit test context
+ *
+ * When the framebuffer dimensions match but the pixel format is not in
+ * amdgpu_dm_wb_formats[], the function should return -EINVAL.
+ */
+static void dm_test_wb_atomic_check_invalid_format(struct kunit *test)
+{
+	struct drm_crtc_state *crtc_state;
+	struct drm_connector_state *conn_state;
+	int ret;
+
+	/* Correct size but unsupported format */
+	crtc_state = alloc_test_crtc_state(test, 1920, 1080);
+	conn_state = alloc_test_conn_state(test, 1920, 1080,
+					   DRM_FORMAT_XRGB8888);
+
+	ret = amdgpu_dm_wb_encoder_atomic_check(NULL, crtc_state, conn_state);
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+}
+
+/* Tests for amdgpu_dm_wb_connector_get_modes using DRM mock */
+
+static const struct drm_connector_funcs dm_wb_test_connector_funcs = {
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.reset = drm_atomic_helper_connector_reset,
+};
+
+/**
+ * dm_test_wb_get_modes_returns_modes - Verify at least one mode is returned
+ * @test: KUnit test context
+ *
+ * Uses a DRM mock connector to verify that amdgpu_dm_wb_connector_get_modes()
+ * populates the connector with at least one display mode.
+ */
+static void dm_test_wb_get_modes_returns_modes(struct kunit *test)
+{
+	struct device *dev;
+	struct drm_device *drm;
+	struct drm_connector *connector;
+	int count;
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						   sizeof(*drm), 0,
+						   DRIVER_MODESET | DRIVER_ATOMIC);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, connector);
+
+	drmm_connector_init(drm, connector, &dm_wb_test_connector_funcs,
+			    DRM_MODE_CONNECTOR_VIRTUAL, NULL);
+
+	count = amdgpu_dm_wb_connector_get_modes(connector);
+
+	/* drm_add_modes_noedid should return at least one mode */
+	KUNIT_EXPECT_GT(test, count, 0);
+}
+
+/**
+ * dm_test_wb_get_modes_bounded_by_max - Verify all modes are within max resolution
+ * @test: KUnit test context
+ *
+ * Uses a DRM mock connector to verify that all modes returned by
+ * amdgpu_dm_wb_connector_get_modes() have hdisplay <= 3840 and
+ * vdisplay <= 2160, matching the DWB hardware maximum.
+ */
+static void dm_test_wb_get_modes_bounded_by_max(struct kunit *test)
+{
+	struct device *dev;
+	struct drm_device *drm;
+	struct drm_connector *connector;
+	struct drm_display_mode *mode;
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						   sizeof(*drm), 0,
+						   DRIVER_MODESET | DRIVER_ATOMIC);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, connector);
+
+	drmm_connector_init(drm, connector, &dm_wb_test_connector_funcs,
+			    DRM_MODE_CONNECTOR_VIRTUAL, NULL);
+
+	amdgpu_dm_wb_connector_get_modes(connector);
+
+	/* All modes must fit within 3840x2160 */
+	list_for_each_entry(mode, &connector->probed_modes, head) {
+		KUNIT_EXPECT_LE(test, mode->hdisplay, 3840);
+		KUNIT_EXPECT_LE(test, mode->vdisplay, 2160);
+	}
+}
+
+/* Tests for amdgpu_dm_wb_connector_init using DRM mock */
+
+/**
+ * dm_test_wb_connector_init_success - Verify writeback connector initialization
+ * @test: KUnit test context
+ *
+ * Uses a DRM mock device embedded in struct amdgpu_device to verify that
+ * amdgpu_dm_wb_connector_init() initializes the writeback connector, stores
+ * the DC link, installs connector state through reset, and wires the expected
+ * DRM callbacks.
+ */
+static void dm_test_wb_connector_init_success(struct kunit *test)
+{
+	struct amdgpu_dm_wb_connector *wbcon;
+	struct amdgpu_display_manager *dm;
+	struct amdgpu_device *adev;
+	struct dc_link *link;
+	struct dc *dc;
+	int ret;
+
+	adev = dm_kunit_alloc_adev(test);
+	adev->mode_info.num_crtc = 1;
+	dm = &adev->dm;
+	dm->adev = adev;
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	dc->links[0] = link;
+	dm->dc = dc;
+
+	wbcon = kunit_kzalloc(test, sizeof(*wbcon), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, wbcon);
+
+	ret = amdgpu_dm_wb_connector_init(dm, wbcon, 0);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_PTR_EQ(test, wbcon->link, link);
+	KUNIT_EXPECT_TRUE(test, wbcon->base.base.funcs != NULL);
+	KUNIT_EXPECT_TRUE(test, wbcon->base.base.helper_private != NULL);
+	KUNIT_EXPECT_TRUE(test, wbcon->base.base.state != NULL);
+	KUNIT_EXPECT_TRUE(test, wbcon->base.encoder.funcs != NULL);
+	KUNIT_EXPECT_EQ(test, wbcon->base.encoder.possible_crtcs, 0x1);
+}
+
+/* Tests for amdgpu_dm_wb_prepare_job / amdgpu_dm_wb_cleanup_job */
+
+/**
+ * dm_test_wb_prepare_job_no_fb - Verify prepare_job early return without a framebuffer
+ * @test: KUnit test context
+ *
+ * When job->fb is NULL there is nothing to pin, so amdgpu_dm_wb_prepare_job()
+ * must return 0 without touching any buffer object.
+ */
+static void dm_test_wb_prepare_job_no_fb(struct kunit *test)
+{
+	struct drm_writeback_job *job;
+	int ret;
+
+	job = kunit_kzalloc(test, sizeof(*job), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, job);
+
+	job->fb = NULL;
+	ret = amdgpu_dm_wb_prepare_job(NULL, job);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+}
+
+/**
+ * dm_test_wb_cleanup_job_no_fb - Verify cleanup_job early return without a framebuffer
+ * @test: KUnit test context
+ *
+ * When job->fb is NULL there is nothing to unpin, so amdgpu_dm_wb_cleanup_job()
+ * must return immediately.
+ */
+static void dm_test_wb_cleanup_job_no_fb(struct kunit *test)
+{
+	struct drm_writeback_job *job;
+
+	job = kunit_kzalloc(test, sizeof(*job), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, job);
+
+	job->fb = NULL;
+	/* Should return without dereferencing any buffer object. */
+	amdgpu_dm_wb_cleanup_job(NULL, job);
+}
+
+static struct kunit_case dm_wb_test_cases[] = {
+	/* amdgpu_dm_wb_encoder_atomic_check */
+	KUNIT_CASE(dm_test_wb_atomic_check_no_job),
+	KUNIT_CASE(dm_test_wb_atomic_check_no_fb),
+	KUNIT_CASE(dm_test_wb_atomic_check_valid),
+	KUNIT_CASE(dm_test_wb_atomic_check_size_mismatch),
+	KUNIT_CASE(dm_test_wb_atomic_check_width_mismatch),
+	KUNIT_CASE(dm_test_wb_atomic_check_height_mismatch),
+	KUNIT_CASE(dm_test_wb_atomic_check_invalid_format),
+	/* amdgpu_dm_wb_connector_get_modes */
+	KUNIT_CASE(dm_test_wb_get_modes_returns_modes),
+	KUNIT_CASE(dm_test_wb_get_modes_bounded_by_max),
+	/* amdgpu_dm_wb_connector_init */
+	KUNIT_CASE(dm_test_wb_connector_init_success),
+	/* amdgpu_dm_wb_prepare_job / amdgpu_dm_wb_cleanup_job */
+	KUNIT_CASE(dm_test_wb_prepare_job_no_fb),
+	KUNIT_CASE(dm_test_wb_cleanup_job_no_fb),
+	{}
+};
+
+static struct kunit_suite dm_wb_test_suite = {
+	.name = "amdgpu_dm_wb",
+	.test_cases = dm_wb_test_cases,
+};
+
+kunit_test_suite(dm_wb_test_suite);
+
+MODULE_LICENSE("Dual MIT/GPL");
+MODULE_DESCRIPTION("KUnit tests for amdgpu_dm_wb");
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.c
index 4a60c5f54a04e..6ee3da89d0582 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.c
@@ -10,6 +10,7 @@
 #include "dcn31/dcn31_clk_mgr.h"
 #include "dcn32/dcn32_clk_mgr.h"
 #include "dcn401/dcn401_clk_mgr.h"
+#include "hw_sequencer.h"
 #include "reg_helper.h"
 #include "core_types.h"
 #include "dm_helpers.h"
@@ -1085,7 +1086,8 @@ static unsigned int dcn401_build_update_display_clocks_sequence(
 		struct clk_mgr *clk_mgr_base,
 		struct dc_state *context,
 		struct dc_clocks *new_clocks,
-		bool safe_to_lower)
+		bool safe_to_lower,
+		unsigned int num_steps_start)
 {
 	struct clk_mgr_internal *clk_mgr_internal = TO_CLK_MGR_INTERNAL(clk_mgr_base);
 	struct dcn401_clk_mgr *clk_mgr401 = TO_DCN401_CLK_MGR(clk_mgr_internal);
@@ -1100,7 +1102,7 @@ static unsigned int dcn401_build_update_display_clocks_sequence(
 	bool frl_present = false;
 	unsigned int i;
 
-	unsigned int num_steps = 0;
+	unsigned int num_steps = num_steps_start;
 
 	/* CLK_MGR401_READ_CLOCKS_FROM_DENTIST */
 	if (clk_mgr_base->clks.dispclk_khz == 0 ||
@@ -1239,6 +1241,44 @@ static unsigned int dcn401_build_update_display_clocks_sequence(
 	return num_steps;
 }
 
+/*
+ * Build-for-BLS functions.
+ * These build both bandwidth and display clock sequences into the clk_mgr's
+ * internal block sequence array, then add a single CLK_MGR_UPDATE_CLOCKS step
+ * to the HWSS block sequence whose executor will call
+ * execute_clk_mgr_block_sequence to dispatch all accumulated steps.
+ */
+void dcn401_build_clock_update_for_bls(
+		struct clk_mgr *clk_mgr_base,
+		struct dc_state *context,
+		bool safe_to_lower,
+		struct block_sequence_state *seq_state)
+{
+	struct clk_mgr_internal *clk_mgr_internal = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+	struct dcn401_clk_mgr *clk_mgr401 = TO_DCN401_CLK_MGR(clk_mgr_internal);
+	unsigned int num_bw_steps;
+	unsigned int total_steps;
+
+	/* Build bandwidth clocks sequence starting at index 0 */
+	num_bw_steps = dcn401_build_update_bandwidth_clocks_sequence(clk_mgr_base,
+			context,
+			&context->bw_ctx.bw.dcn.clk,
+			safe_to_lower);
+
+	/* Build display clocks sequence appended after bandwidth steps */
+	total_steps = dcn401_build_update_display_clocks_sequence(clk_mgr_base,
+			context,
+			&context->bw_ctx.bw.dcn.clk,
+			safe_to_lower,
+			num_bw_steps);
+
+	/* Store total step count for the executor */
+	clk_mgr401->num_block_sequence_steps = total_steps;
+
+	/* Add single HWSS step that will execute all clk_mgr block sequence steps */
+	hwss_add_clk_mgr_update_clocks(seq_state, clk_mgr_base);
+}
+
 static void dcn401_update_clocks(struct clk_mgr *clk_mgr_base,
 		struct dc_state *context,
 		bool safe_to_lower)
@@ -1260,7 +1300,8 @@ static void dcn401_update_clocks(struct clk_mgr *clk_mgr_base,
 	num_steps = dcn401_build_update_display_clocks_sequence(clk_mgr_base,
 			context,
 			&context->bw_ctx.bw.dcn.clk,
-			safe_to_lower);
+			safe_to_lower,
+			0);
 
 	/* execute sequence */
 	dcn401_execute_block_sequence(clk_mgr_base,	num_steps);
@@ -1549,6 +1590,14 @@ unsigned int dcn401_get_max_clock_khz(struct clk_mgr *clk_mgr_base, enum clk_typ
 	return 0;
 }
 
+static void dcn401_execute_clk_mgr_block_sequence_bls(struct clk_mgr *clk_mgr_base)
+{
+	struct clk_mgr_internal *clk_mgr_internal = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+	struct dcn401_clk_mgr *clk_mgr401 = TO_DCN401_CLK_MGR(clk_mgr_internal);
+
+	dcn401_execute_block_sequence(clk_mgr_base, clk_mgr401->num_block_sequence_steps);
+}
+
 static struct clk_mgr_funcs dcn401_funcs = {
 		.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
 		.get_dtb_ref_clk_frequency = dcn401_get_dtb_ref_freq_khz,
@@ -1566,6 +1615,8 @@ static struct clk_mgr_funcs dcn401_funcs = {
 		.get_hard_min_fclk = dcn401_get_hard_min_fclk,
 		.is_dc_mode_present = dcn401_is_dc_mode_present,
 		.get_max_clock_khz = dcn401_get_max_clock_khz,
+		.build_clock_update_for_bls = dcn401_build_clock_update_for_bls,
+		.execute_clk_mgr_block_sequence = dcn401_execute_clk_mgr_block_sequence_bls,
 };
 
 struct clk_mgr_internal *dcn401_clk_mgr_construct(
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.h
index 370d2ddd6064c..d4cd69a5a8ddd 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.h
@@ -102,6 +102,7 @@ struct dcn401_clk_mgr {
 	struct clk_mgr_internal base;
 
 	struct dcn401_clk_mgr_block_sequence block_sequence[DCN401_CLK_MGR_MAX_SEQUENCE_SIZE];
+	unsigned int num_block_sequence_steps;
 };
 
 void dcn401_init_clocks(struct clk_mgr *clk_mgr_base);
@@ -114,4 +115,12 @@ void dcn401_clk_mgr_destroy(struct clk_mgr_internal *clk_mgr);
 
 unsigned int dcn401_get_max_clock_khz(struct clk_mgr *clk_mgr_base, enum clk_type clk_type);
 
+struct block_sequence_state;
+
+void dcn401_build_clock_update_for_bls(
+		struct clk_mgr *clk_mgr_base,
+		struct dc_state *context,
+		bool safe_to_lower,
+		struct block_sequence_state *seq_state);
+
 #endif /* __DCN401_CLK_MGR_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c
index d01e5969a6708..3baac7fa313a2 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c
@@ -999,15 +999,21 @@ void dcn42_get_smu_clocks(struct clk_mgr_internal *clk_mgr_int)
 			clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_fclk_levels = dpm_clks->NumFclkLevelsEnabled;
 			clk_mgr_base->bw_params->clk_table.num_entries = dpm_clks->NumFclkLevelsEnabled;
 
-			/* Memory Pstate table is in reverse order*/
+			/* Memory Pstate table is in reverse order for dcn42. Proper way to map this is for pmfw to provide an fclk indexed uclk key since we consume fclk matched pairs.*/
 			ASSERT(dpm_clks->NumMemPstatesEnabled <= NUM_MEM_PSTATE_LEVELS);
 			if (dpm_clks->NumMemPstatesEnabled > NUM_MEM_PSTATE_LEVELS)
 				dpm_clks->NumMemPstatesEnabled = NUM_MEM_PSTATE_LEVELS;
-			for (i = 0; i < dpm_clks->NumMemPstatesEnabled; i++) {
-				clk_mgr_base->bw_params->clk_table.entries[dpm_clks->NumMemPstatesEnabled - 1 - i].memclk_mhz = dpm_clks->MemPstateTable[i].MemClk;
-				clk_mgr_base->bw_params->clk_table.entries[dpm_clks->NumMemPstatesEnabled - 1 - i].wck_ratio = dcn42_convert_wck_ratio(dpm_clks->MemPstateTable[i].WckRatio)	;
+			for (i = 0; i < dpm_clks->NumDcfClkLevelsEnabled; i++) {
+				if (i < dpm_clks->NumMemPstatesEnabled) {
+					clk_mgr_base->bw_params->clk_table.entries[dpm_clks->NumMemPstatesEnabled - 1 - i].memclk_mhz = dpm_clks->MemPstateTable[i].MemClk;
+					clk_mgr_base->bw_params->clk_table.entries[dpm_clks->NumMemPstatesEnabled - 1 - i].wck_ratio = dcn42_convert_wck_ratio(dpm_clks->MemPstateTable[i].WckRatio);
+				} else {
+					clk_mgr_base->bw_params->clk_table.entries[i].memclk_mhz = dpm_clks->MemPstateTable[0].MemClk;
+					clk_mgr_base->bw_params->clk_table.entries[i].wck_ratio = dcn42_convert_wck_ratio(dpm_clks->MemPstateTable[0].WckRatio);
+				}
 			}
-			clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_memclk_levels = dpm_clks->NumMemPstatesEnabled;
+
+			clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_memclk_levels = dpm_clks->NumDcfClkLevelsEnabled;
 
 			/* DTBCLK*/
 			clk_mgr_base->bw_params->clk_table.entries[0].dtbclk_mhz = 600; /* Fixed on platform */
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c
index 175106cce5a4a..8d77158229f1b 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc.c
@@ -769,22 +769,23 @@ void dc_stream_set_dither_option(struct dc_stream_state *stream,
 {
 	struct bit_depth_reduction_params params;
 	struct dc_link *link = stream->link;
-	struct pipe_ctx *pipes = NULL;
+	struct resource_context *res_ctx = &link->dc->current_state->res_ctx;
+	struct pipe_ctx *otg_master;
+	struct pipe_ctx *opp_heads[MAX_PIPES];
+	int opp_cnt;
 	int i;
 
-	for (i = 0; i < MAX_PIPES; i++) {
-		if (link->dc->current_state->res_ctx.pipe_ctx[i].stream ==
-				stream) {
-			pipes = &link->dc->current_state->res_ctx.pipe_ctx[i];
-			break;
-		}
-	}
-
-	if (!pipes)
+	otg_master = resource_get_otg_master_for_stream(res_ctx, stream);
+	if (!otg_master)
 		return;
 	if (option > DITHER_OPTION_MAX)
 		return;
 
+	opp_cnt = resource_get_opp_heads_for_otg_master(otg_master, res_ctx, opp_heads);
+
+	if (opp_cnt == 0)
+		return;
+
 	dc_exit_ips_for_hw_access(stream->ctx->dc);
 
 	stream->dither_option = option;
@@ -793,16 +794,30 @@ void dc_stream_set_dither_option(struct dc_stream_state *stream,
 	resource_build_bit_depth_reduction_params(stream, &params);
 	stream->bit_depth_params = params;
 
-	if (pipes->plane_res.xfm &&
-	    pipes->plane_res.xfm->funcs->transform_set_pixel_storage_depth) {
-		pipes->plane_res.xfm->funcs->transform_set_pixel_storage_depth(
-			pipes->plane_res.xfm,
-			pipes->plane_res.scl_data.lb_params.depth,
-			&stream->bit_depth_params);
-	}
+	/*
+	 * Program bit-depth reduction (dither) on every OPP head of the
+	 * stream. Under ODM combine there is more than one OPP head and they
+	 * must all be kept in sync, otherwise (e.g. when CRC capture requests
+	 * dither off) a secondary ODM segment can keep dither enabled and
+	 * produce a different CRC than the primary segment.
+	 */
+	for (i = 0; i < opp_cnt; i++) {
+		struct pipe_ctx *opp_head = opp_heads[i];
 
-	pipes->stream_res.opp->funcs->
-		opp_program_bit_depth_reduction(pipes->stream_res.opp, &params);
+		if (opp_head->plane_res.xfm &&
+		    opp_head->plane_res.xfm->funcs->transform_set_pixel_storage_depth) {
+			opp_head->plane_res.xfm->funcs->transform_set_pixel_storage_depth(
+				opp_head->plane_res.xfm,
+				opp_head->plane_res.scl_data.lb_params.depth,
+				&stream->bit_depth_params);
+		}
+
+		if (opp_head->stream_res.opp &&
+		    opp_head->stream_res.opp->funcs->opp_program_bit_depth_reduction) {
+			opp_head->stream_res.opp->funcs->opp_program_bit_depth_reduction(
+				opp_head->stream_res.opp, &params);
+		}
+	}
 }
 
 bool dc_stream_set_gamut_remap(struct dc *dc, const struct dc_stream_state *stream)
@@ -1930,6 +1945,13 @@ bool dc_validate_boot_timing(const struct dc *dc,
 		struct display_stream_compressor *dsc = NULL;
 		struct dcn_dsc_state dsc_state = {0};
 
+		if (dc->ctx->dce_version < DCN_VERSION_4_2) {
+			/*vbios enabled eDP dsc for one of DCN315  only but it has known issue,
+			since there is no production bios update, block it there*/
+			DC_LOG_DEBUG("boot timing validation failed due to unsupported DSC on this ASIC\n");
+			return false;
+		}
+
 		/* Find DSC associated with this timing generator */
 		if (tg_inst < (unsigned int)dc->res_pool->res_cap->num_dsc) {
 			dsc = dc->res_pool->dscs[tg_inst];
@@ -2310,7 +2332,7 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c
 	for (i = 0; i < context->stream_count; i++) {
 		uint32_t prev_dsc_changed = context->streams[i]->update_flags.bits.dsc_changed;
 
-		context->streams[i]->update_flags.raw = 0xFFFFFFFF;
+		stream_update_flags_set_full(&context->streams[i]->update_flags);
 		context->streams[i]->update_flags.bits.dsc_changed = prev_dsc_changed;
 	}
 
@@ -2416,7 +2438,7 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c
 
 	/* Clear update flags that were set earlier to avoid redundant programming */
 	for (i = 0; i < context->stream_count; i++) {
-		context->streams[i]->update_flags.raw = 0x0;
+		stream_update_flags_clear(&context->streams[i]->update_flags);
 	}
 
 	old_state = dc->current_state;
@@ -2764,7 +2786,7 @@ static bool is_surface_in_context(
 
 static struct surface_update_descriptor get_plane_info_update_type(const struct dc_surface_update *u)
 {
-	union surface_update_flags *update_flags = &u->surface->update_flags;
+	struct pipe_update_bits *update_bits = &u->surface->update_bits;
 	struct surface_update_descriptor update_type = { UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_NONE };
 
 	if (!u->plane_info)
@@ -2774,37 +2796,37 @@ static struct surface_update_descriptor get_plane_info_update_type(const struct
 	elevate_update_type(&update_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 
 	if (u->plane_info->color_space != u->surface->color_space) {
-		update_flags->bits.color_space_change = 1;
+		update_bits->color_space_change = 1;
 		elevate_update_type(&update_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
 	}
 
 	if (u->plane_info->horizontal_mirror != u->surface->horizontal_mirror) {
-		update_flags->bits.horizontal_mirror_change = 1;
+		update_bits->horizontal_mirror_change = 1;
 		elevate_update_type(&update_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
 	}
 
 	if (u->plane_info->rotation != u->surface->rotation) {
-		update_flags->bits.rotation_change = 1;
+		update_bits->rotation_change = 1;
 		elevate_update_type(&update_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
 	}
 
 	if (u->plane_info->format != u->surface->format) {
-		update_flags->bits.pixel_format_change = 1;
+		update_bits->pixel_format_change = 1;
 		elevate_update_type(&update_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
 	}
 
 	if (u->plane_info->stereo_format != u->surface->stereo_format) {
-		update_flags->bits.stereo_format_change = 1;
+		update_bits->stereo_format_change = 1;
 		elevate_update_type(&update_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
 	}
 
 	if (u->plane_info->per_pixel_alpha != u->surface->per_pixel_alpha) {
-		update_flags->bits.per_pixel_alpha_change = 1;
+		update_bits->per_pixel_alpha_change = 1;
 		elevate_update_type(&update_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
 	}
 
 	if (u->plane_info->global_alpha_value != u->surface->global_alpha_value) {
-		update_flags->bits.global_alpha_change = 1;
+		update_bits->global_alpha_change = 1;
 		elevate_update_type(&update_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
 	}
 
@@ -2816,7 +2838,7 @@ static struct surface_update_descriptor get_plane_info_update_type(const struct
 		 * stutter period calculation. Triggering a full update will
 		 * recalculate stutter period.
 		 */
-		update_flags->bits.dcc_change = 1;
+		update_bits->dcc_change = 1;
 		elevate_update_type(&update_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
 	}
 
@@ -2825,25 +2847,25 @@ static struct surface_update_descriptor get_plane_info_update_type(const struct
 		/* different bytes per element will require full bandwidth
 		 * and DML calculation
 		 */
-		update_flags->bits.bpp_change = 1;
+		update_bits->bpp_change = 1;
 		elevate_update_type(&update_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
 	}
 
 	if (u->plane_info->plane_size.surface_pitch != u->surface->plane_size.surface_pitch
 			|| u->plane_info->plane_size.chroma_pitch != u->surface->plane_size.chroma_pitch) {
-		update_flags->bits.plane_size_change = 1;
+		update_bits->plane_size_change = 1;
 		elevate_update_type(&update_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
 	}
 
 	const struct dc_tiling_info *tiling = &u->plane_info->tiling_info;
 
 	if (memcmp(tiling, &u->surface->tiling_info, sizeof(*tiling)) != 0) {
-		update_flags->bits.swizzle_change = 1;
+		update_bits->swizzle_change = 1;
 
 		if (tiling->flags.avoid_full_update_on_tiling_change) {
 			elevate_update_type(&update_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
 		} else {
-			update_flags->bits.bandwidth_change = 1;
+			update_bits->bandwidth_change = 1;
 			elevate_update_type(&update_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
 		}
 	}
@@ -2853,10 +2875,10 @@ static struct surface_update_descriptor get_plane_info_update_type(const struct
 }
 
 static struct surface_update_descriptor get_scaling_info_update_type(
-		const struct dc_check_config *check_config,
-		const struct dc_surface_update *u)
+	const struct dc_check_config *check_config,
+	const struct dc_surface_update *u)
 {
-	union surface_update_flags *update_flags = &u->surface->update_flags;
+	struct pipe_update_bits *update_bits = &u->surface->update_bits;
 	struct surface_update_descriptor update_type = { UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_NONE };
 
 	if (!u->scaling_info)
@@ -2873,26 +2895,26 @@ static struct surface_update_descriptor get_scaling_info_update_type(
 			|| u->scaling_info->clip_rect.height != u->surface->clip_rect.height
 			|| u->scaling_info->scaling_quality.integer_scaling !=
 					u->surface->scaling_quality.integer_scaling) {
-		update_flags->bits.scaling_change = 1;
+		update_bits->scaling_change = 1;
 		elevate_update_type(&update_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
 
 		if (u->scaling_info->src_rect.width > u->surface->src_rect.width
 				|| u->scaling_info->src_rect.height > u->surface->src_rect.height)
 			/* Making src rect bigger requires a bandwidth change */
-			update_flags->bits.clock_change = 1;
+			update_bits->clock_change = 1;
 
 		if ((u->scaling_info->dst_rect.width < u->surface->dst_rect.width
 			|| u->scaling_info->dst_rect.height < u->surface->dst_rect.height)
 				&& (u->scaling_info->dst_rect.width < u->surface->src_rect.width
 					|| u->scaling_info->dst_rect.height < u->surface->src_rect.height))
 			/* Making dst rect smaller requires a bandwidth change */
-			update_flags->bits.bandwidth_change = 1;
+			update_bits->bandwidth_change = 1;
 
 		if (u->scaling_info->src_rect.width > (int)check_config->max_optimizable_video_width &&
 			(u->scaling_info->clip_rect.width > u->surface->clip_rect.width ||
 			 u->scaling_info->clip_rect.height > u->surface->clip_rect.height))
 			 /* Changing clip size of a large surface may result in MPC slice count change */
-			update_flags->bits.bandwidth_change = 1;
+			update_bits->bandwidth_change = 1;
 	}
 
 	if (u->scaling_info->src_rect.x != u->surface->src_rect.x
@@ -2902,7 +2924,7 @@ static struct surface_update_descriptor get_scaling_info_update_type(
 			|| u->scaling_info->dst_rect.x != u->surface->dst_rect.x
 			|| u->scaling_info->dst_rect.y != u->surface->dst_rect.y) {
 		elevate_update_type(&update_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
-		update_flags->bits.position_change = 1;
+		update_bits->position_change = 1;
 	}
 
 	return update_type;
@@ -2913,15 +2935,15 @@ static struct surface_update_descriptor det_surface_update(
 		struct dc_surface_update *u)
 {
 	struct surface_update_descriptor overall_type = { UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_NONE };
-	union surface_update_flags *update_flags = &u->surface->update_flags;
+	struct pipe_update_bits *update_bits = &u->surface->update_bits;
 
 	if (u->surface->force_full_update) {
-		update_flags->raw = 0xFFFFFFFF;
+		dc_pipe_update_bits_set_full(update_bits);
 		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
 		return overall_type;
 	}
 
-	update_flags->raw = 0; // Reset all flags
+	dc_pipe_update_bits_clear(update_bits);
 
 	struct surface_update_descriptor inner_type = get_plane_info_update_type(u);
 
@@ -2931,87 +2953,112 @@ static struct surface_update_descriptor det_surface_update(
 	elevate_update_type(&overall_type, inner_type.update_type, inner_type.lock_descriptor);
 
 	if (u->flip_addr) {
-		update_flags->bits.addr_update = 1;
+		update_bits->addr_update = 1;
 		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 
 		if (u->flip_addr->address.tmz_surface != u->surface->address.tmz_surface) {
-			update_flags->bits.tmz_changed = 1;
+			update_bits->tmz_changed = 1;
 			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
 		}
 	}
 	if (u->in_transfer_func) {
-		update_flags->bits.in_transfer_func_change = 1;
+		update_bits->in_transfer_func_change = 1;
 		elevate_update_type(&overall_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
 	}
 
 	if (u->input_csc_color_matrix) {
-		update_flags->bits.input_csc_change = 1;
+		update_bits->input_csc_change = 1;
 		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 	}
 
 	if (u->cursor_csc_color_matrix) {
-		update_flags->bits.cursor_csc_color_matrix_change = 1;
+		update_bits->cursor_csc_color_matrix_change = 1;
 		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 	}
 
 	if (u->coeff_reduction_factor) {
-		update_flags->bits.coeff_reduction_change = 1;
+		update_bits->coeff_reduction_change = 1;
 		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 	}
 
 	if (u->gamut_remap_matrix) {
-		update_flags->bits.gamut_remap_change = 1;
+		update_bits->gamut_remap_change = 1;
 		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 	}
 
-	if (u->blend_tf || (u->gamma && dce_use_lut(u->plane_info ? u->plane_info->format : u->surface->format))) {
-		update_flags->bits.gamma_change = 1;
+	if ((u->cm && u->cm->flags.bits.blend_enable) ||
+			(u->gamma && dce_use_lut(u->plane_info ? u->plane_info->format : u->surface->format))) {
+		update_bits->gamma_change = 1;
 		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 	}
 
-	if (u->lut3d_func || u->func_shaper) {
-		update_flags->bits.lut_3d = 1;
+	if (u->cm && (u->cm->flags.bits.lut3d_enable || u->cm->flags.bits.shaper_enable)) {
+		update_bits->lut_3d = 1;
 		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 	}
 
+	if (u->cm && u->cm->flags.bits.lut3d_dma_enable != u->surface->cm.flags.bits.lut3d_dma_enable &&
+			u->cm->flags.bits.lut3d_enable && u->surface->cm.flags.bits.lut3d_enable) {
+		/* Toggling 3DLUT loading between DMA and Host is illegal */
+		BREAK_TO_DEBUGGER();
+	}
+
+	if (u->cm && u->cm->flags.bits.lut3d_enable && !u->cm->flags.bits.lut3d_dma_enable) {
+		/* Host loading 3DLUT requires full update but only stream lock  */
+		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_STREAM);
+	}
+
 	if (u->hdr_mult.value)
 		if (u->hdr_mult.value != u->surface->hdr_mult.value) {
 			// TODO: Should be fast?
-			update_flags->bits.hdr_mult = 1;
+			update_bits->hdr_mult = 1;
 			elevate_update_type(&overall_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
 		}
 
 	if (u->sdr_white_level_nits)
 		if (u->sdr_white_level_nits != u->surface->sdr_white_level_nits) {
 			// TODO: Should be fast?
-			update_flags->bits.sdr_white_level_nits = 1;
+			update_bits->sdr_white_level_nits = 1;
 			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
 		}
 
 	if (u->cm_hist_control) {
-		update_flags->bits.cm_hist_change = 1;
+		update_bits->cm_hist_change = 1;
 		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 	}
-	if (u->cm2_params) {
-		if (u->cm2_params->component_settings.shaper_3dlut_setting != u->surface->mcm_shaper_3dlut_setting
-				|| u->cm2_params->component_settings.lut1d_enable != u->surface->mcm_lut1d_enable
-				|| u->cm2_params->cm2_luts.lut3d_data.lut3d_src != u->surface->mcm_luts.lut3d_data.lut3d_src) {
-			update_flags->bits.mcm_transfer_function_enable_change = 1;
+
+	if (u->cm) {
+		const union dc_plane_cm_flags blend_only_flags = {
+			.bits = {
+				.blend_enable = 1,
+			}
+		};
+
+		if (u->cm->flags.bits.shaper_enable != u->surface->cm.flags.bits.shaper_enable
+				|| u->cm->flags.bits.blend_enable != u->surface->cm.flags.bits.blend_enable
+				|| u->cm->flags.bits.lut3d_enable != u->surface->cm.flags.bits.lut3d_enable
+				|| u->cm->flags.bits.lut3d_dma_enable != u->surface->cm.flags.bits.lut3d_dma_enable) {
+			update_bits->mcm_transfer_function_enable_change = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+		}
+
+		if ((u->cm->flags.all != blend_only_flags.all && u->cm->flags.all != 0) ||
+				(u->surface->cm.flags.all != blend_only_flags.all && u->surface->cm.flags.all != 0)) {
 			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
 		}
 	}
 
-	if (update_flags->bits.lut_3d &&
-			u->surface->mcm_luts.lut3d_data.lut3d_src != DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM) {
+	if (update_bits->lut_3d &&
+			!u->surface->cm.flags.bits.lut3d_dma_enable) {
 		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
 	}
 
 	if (check_config->enable_legacy_fast_update &&
-			(update_flags->bits.gamma_change ||
-			update_flags->bits.gamut_remap_change ||
-			update_flags->bits.input_csc_change ||
-			update_flags->bits.cm_hist_change ||
-			update_flags->bits.coeff_reduction_change)) {
+			(update_bits->gamma_change ||
+			update_bits->gamut_remap_change ||
+			update_bits->input_csc_change ||
+			update_bits->cm_hist_change ||
+			update_bits->coeff_reduction_change)) {
 		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
 	}
 	return overall_type;
@@ -3036,7 +3083,7 @@ static void force_immediate_gsl_plane_flip(struct dc *dc, struct dc_surface_upda
 	if (has_flip_immediate_plane && surface_count > 1) {
 		for (i = 0; i < surface_count; i++) {
 			if (updates[i].surface->flip_immediate)
-				updates[i].surface->update_flags.bits.addr_update = 1;
+				updates[i].surface->update_bits.addr_update = 1;
 		}
 	}
 }
@@ -3191,9 +3238,9 @@ struct surface_update_descriptor dc_check_update_surfaces_for_stream(
 		struct dc_stream_update *stream_update)
 {
 	if (stream_update)
-		stream_update->stream->update_flags.raw = 0;
+		stream_update_flags_clear(&stream_update->stream->update_flags);
 	for (int i = 0; i < surface_count; i++)
-		updates[i].surface->update_flags.raw = 0;
+		dc_pipe_update_bits_clear(&updates[i].surface->update_bits);
 
 	return check_update_surfaces_for_stream(check_config, updates, surface_count, stream_update);
 }
@@ -3304,24 +3351,55 @@ static void copy_surface_update_to_plane(
 			sizeof(struct dc_transfer_func_distributed_points));
 	}
 
-	if (srf_update->cm2_params) {
-		surface->mcm_shaper_3dlut_setting = srf_update->cm2_params->component_settings.shaper_3dlut_setting;
-		surface->mcm_lut1d_enable = srf_update->cm2_params->component_settings.lut1d_enable;
-		surface->mcm_luts = srf_update->cm2_params->cm2_luts;
-	}
-
-	if (srf_update->func_shaper) {
-		memcpy(&surface->in_shaper_func, srf_update->func_shaper,
-		sizeof(surface->in_shaper_func));
+	/* Shaper, 3DLUT, 1DLUT */
+	if (srf_update->cm) {
+		struct kref refcount = surface->cm.refcount;
+
+		memcpy(&surface->cm, srf_update->cm, sizeof(surface->cm));
+		surface->cm.refcount = refcount;
+
+#ifndef TRIM_CM2
+		/* Populate mcm_luts from cm for legacy consumers (dml2, hwseq) */
+		surface->mcm_luts.lut1d_func = &surface->cm.blend_func;
+		surface->mcm_luts.shaper = &surface->cm.shaper_func;
+		if (srf_update->cm->flags.bits.lut3d_dma_enable) {
+			surface->mcm_luts.lut3d_data.lut3d_src = DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM;
+			surface->mcm_luts.lut3d_data.gpu_mem_params.addr = surface->cm.lut3d_dma.addr;
+			surface->mcm_luts.lut3d_data.gpu_mem_params.layout =
+				(surface->cm.lut3d_dma.swizzle == CM_LUT_3D_SWIZZLE_LINEAR_RGB) ?
+					DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_RGB :
+				(surface->cm.lut3d_dma.swizzle == CM_LUT_3D_SWIZZLE_LINEAR_BGR) ?
+					DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_BGR :
+					DC_CM2_GPU_MEM_LAYOUT_1D_PACKED_LINEAR;
+			surface->mcm_luts.lut3d_data.gpu_mem_params.format_params.format =
+				(surface->cm.lut3d_dma.format == CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB) ?
+					DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12MSB :
+				(surface->cm.lut3d_dma.format == CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB) ?
+					DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12LSB :
+					DC_CM2_GPU_MEM_FORMAT_16161616_FLOAT_FP1_5_10;
+			surface->mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.bias =
+				surface->cm.lut3d_dma.bias;
+			surface->mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.scale =
+				surface->cm.lut3d_dma.scale;
+			surface->mcm_luts.lut3d_data.gpu_mem_params.component_order =
+				DC_CM2_GPU_MEM_PIXEL_COMPONENT_ORDER_RGBA;
+			surface->mcm_luts.lut3d_data.gpu_mem_params.size = DC_CM2_GPU_MEM_SIZE_TRANSFORMED;
+			surface->mcm_luts.lut3d_data.mpc_3dlut_enable = (srf_update->cm->flags.bits.lut3d_enable != 0);
+		} else {
+			surface->mcm_luts.lut3d_data.lut3d_src = DC_CM2_TRANSFER_FUNC_SOURCE_SYSMEM;
+			surface->mcm_luts.lut3d_data.lut3d_func = &surface->cm.lut3d_func;
+		}
 
-		if (surface->mcm_shaper_3dlut_setting >= DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER)
-			surface->mcm_luts.shaper = &surface->in_shaper_func;
+		if (srf_update->cm->flags.bits.shaper_enable &&
+				srf_update->cm->flags.bits.lut3d_enable)
+			surface->mcm_shaper_3dlut_setting = DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER_3DLUT;
+		else if (srf_update->cm->flags.bits.shaper_enable)
+			surface->mcm_shaper_3dlut_setting = DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER;
+		else
+			surface->mcm_shaper_3dlut_setting = DC_CM2_SHAPER_3DLUT_SETTING_BYPASS_ALL;
+#endif /* TRIM_CM2 */
 	}
 
-	if (srf_update->lut3d_func)
-		memcpy(&surface->lut3d_func, srf_update->lut3d_func,
-		sizeof(surface->lut3d_func));
-
 	if (srf_update->hdr_mult.value)
 		surface->hdr_mult =
 				srf_update->hdr_mult;
@@ -3330,15 +3408,10 @@ static void copy_surface_update_to_plane(
 		surface->sdr_white_level_nits =
 				srf_update->sdr_white_level_nits;
 
-	if (srf_update->blend_tf) {
-		memcpy(&surface->blend_tf, srf_update->blend_tf,
-		sizeof(surface->blend_tf));
-
-		if (surface->mcm_lut1d_enable)
-			surface->mcm_luts.lut1d_func = &surface->blend_tf;
-	}
-
-	if (srf_update->cm2_params || srf_update->blend_tf)
+	if (srf_update->cm &&
+			(srf_update->cm->flags.bits.blend_enable ||
+			srf_update->cm->flags.bits.shaper_enable ||
+			srf_update->cm->flags.bits.lut3d_enable))
 		surface->lut_bank_a = !surface->lut_bank_a;
 
 	if (srf_update->input_csc_color_matrix)
@@ -3714,11 +3787,11 @@ static bool update_planes_and_stream_state(struct dc *dc,
 	if (update_type == UPDATE_TYPE_FULL) {
 		if (stream_update) {
 			uint32_t dsc_changed = stream_update->stream->update_flags.bits.dsc_changed;
-			stream_update->stream->update_flags.raw = 0xFFFFFFFF;
+			stream_update_flags_set_full(&stream_update->stream->update_flags);
 			stream_update->stream->update_flags.bits.dsc_changed = dsc_changed;
 		}
 		for (i = 0; i < surface_count; i++)
-			srf_updates[i].surface->update_flags.raw = 0xFFFFFFFF;
+			dc_pipe_update_bits_set_full(&srf_updates[i].surface->update_bits);
 	}
 
 	if (update_type >= update_surface_trace_level)
@@ -3767,7 +3840,7 @@ static bool update_planes_and_stream_state(struct dc *dc,
 
 		if (update_type != UPDATE_TYPE_MED)
 			continue;
-		if (surface->update_flags.bits.position_change) {
+		if (surface->update_bits.position_change) {
 			for (j = 0; j < dc->res_pool->pipe_count; j++) {
 				struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
 
@@ -4205,8 +4278,8 @@ static void commit_planes_do_stream_update_sequence(struct dc *dc,
 					hwss_add_dc_set_optimized_required(&seq_state, dc, true);
 
 				} else {
-					if (get_seamless_boot_stream_count(context) == 0)
-						hwss_add_prepare_bandwidth(&seq_state, dc, dc->current_state);
+					if (get_seamless_boot_stream_count(context) == 0 && dc->hwss.prepare_bandwidth_sequence)
+						dc->hwss.prepare_bandwidth_sequence(dc, dc->current_state, &seq_state);
 					hwss_add_link_set_dpms_on(&seq_state, dc->current_state, dpms_pipe_ctx);
 				}
 			} else if (pipe_ctx->stream->link->wa_flags.blank_stream_on_ocs_change && stream_update->output_color_space
@@ -4550,14 +4623,23 @@ static void build_dmub_update_dirty_rect(
 	}
 }
 
-static bool check_address_only_update(union surface_update_flags update_flags)
+/**
+ * dc_check_address_only_update - Check if addr_update is the sole flag set
+ *
+ * @update_bits: The pipe update bits to check
+ *
+ * Determines whether an update contains only an address change with no other
+ * pending updates.
+ *
+ * Return: %true if addr_update is the sole bit set, %false otherwise.
+ */
+bool dc_check_address_only_update(struct pipe_update_bits update_bits)
 {
-	union surface_update_flags addr_only_update_flags;
-	addr_only_update_flags.raw = 0;
-	addr_only_update_flags.bits.addr_update = 1;
+	struct pipe_update_bits check = update_bits;  /* 1. Copy all flags from input */
 
-	return update_flags.bits.addr_update &&
-			!(update_flags.raw & ~addr_only_update_flags.raw);
+	check.addr_update = 0;                        /* 2. Zero the addr_update bit in the copy */
+	return update_bits.addr_update &&             /* 3. Check addr_update was set in original */
+			!dc_pipe_update_bits_is_any_set(&check); /* 4. Check no other bits remain in the copy */
 }
 
 /**
@@ -4617,7 +4699,7 @@ static void commit_plane_for_stream_offload_fams2_flip(struct dc *dc,
 				continue;
 
 			/* update pipe context for plane */
-			if (pipe_ctx->plane_state->update_flags.bits.addr_update)
+			if (pipe_ctx->plane_state->update_bits.addr_update)
 				dc->hwss.update_plane_addr(dc, pipe_ctx);
 		}
 	}
@@ -4655,8 +4737,8 @@ static void commit_planes_for_stream_fast(struct dc *dc,
 		should_offload_fams2_flip = true;
 		for (i = 0; i < surface_count; i++) {
 			if (srf_updates[i].surface &&
-					srf_updates[i].surface->update_flags.raw &&
-					!check_address_only_update(srf_updates[i].surface->update_flags)) {
+					dc_pipe_update_bits_is_any_set(&srf_updates[i].surface->update_bits) &&
+					!dc_check_address_only_update(srf_updates[i].surface->update_bits)) {
 				/* more than address update, need to acquire FAMS2 lock */
 				should_offload_fams2_flip = false;
 				break;
@@ -4747,7 +4829,7 @@ static void commit_planes_for_stream_fast(struct dc *dc,
 	 * so no need to clear here.
 	 */
 	if (top_pipe_to_program->stream)
-		top_pipe_to_program->stream->update_flags.raw = 0;
+		stream_update_flags_clear(&top_pipe_to_program->stream->update_flags);
 }
 
 static void commit_planes_for_stream(struct dc *dc,
@@ -5073,11 +5155,9 @@ static void commit_planes_for_stream(struct dc *dc,
 				if (!should_update_pipe_for_plane(context, pipe_ctx, plane_state))
 					continue;
 
-				if (srf_updates[i].cm2_params &&
-						srf_updates[i].cm2_params->cm2_luts.lut3d_data.lut3d_src ==
-								DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM &&
-						srf_updates[i].cm2_params->component_settings.shaper_3dlut_setting ==
-								DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER_3DLUT &&
+				if (srf_updates[i].cm &&
+						srf_updates[i].cm->flags.bits.lut3d_enable &&
+						srf_updates[i].cm->flags.bits.lut3d_dma_enable &&
 						dc->hwss.trigger_3dlut_dma_load)
 					dc->hwss.trigger_3dlut_dma_load(dc, pipe_ctx);
 
@@ -5087,7 +5167,7 @@ static void commit_planes_for_stream(struct dc *dc,
 					dc->hwss.program_triplebuffer(
 						dc, pipe_ctx, pipe_ctx->plane_state->triplebuffer_flips);
 				}
-				if (pipe_ctx->plane_state->update_flags.bits.addr_update)
+				if (pipe_ctx->plane_state->update_bits.addr_update)
 					dc->hwss.update_plane_addr(dc, pipe_ctx);
 			}
 		}
@@ -5178,7 +5258,7 @@ static void commit_planes_for_stream(struct dc *dc,
 
 		if (pipe_ctx->bottom_pipe || pipe_ctx->next_odm_pipe ||
 				!pipe_ctx->stream || !should_update_pipe_for_stream(context, pipe_ctx, stream) ||
-				!pipe_ctx->plane_state->update_flags.bits.addr_update ||
+				!pipe_ctx->plane_state->update_bits.addr_update ||
 				pipe_ctx->plane_state->skip_manual_trigger)
 			continue;
 
@@ -5617,7 +5697,7 @@ static bool commit_minimal_transition_state(struct dc *dc,
 	/* force full surface update */
 	for (i = 0; i < dc->current_state->stream_count; i++) {
 		for (j = 0; j < (unsigned int)dc->current_state->stream_status[i].plane_count; j++) {
-			dc->current_state->stream_status[i].plane_states[j]->update_flags.raw = 0xFFFFFFFF;
+			dc_pipe_update_bits_set_full(&dc->current_state->stream_status[i].plane_states[j]->update_bits);
 		}
 	}
 
@@ -5792,14 +5872,9 @@ static bool full_update_required(
 				(srf_updates[i].sdr_white_level_nits &&
 				srf_updates[i].sdr_white_level_nits != srf_updates->surface->sdr_white_level_nits) ||
 				srf_updates[i].in_transfer_func ||
-				srf_updates[i].func_shaper ||
-				srf_updates[i].lut3d_func ||
 				srf_updates[i].surface->force_full_update ||
 				(srf_updates[i].flip_addr &&
-				srf_updates[i].flip_addr->address.tmz_surface != srf_updates[i].surface->address.tmz_surface) ||
-				(srf_updates[i].cm2_params &&
-				 (srf_updates[i].cm2_params->component_settings.shaper_3dlut_setting != srf_updates[i].surface->mcm_shaper_3dlut_setting ||
-				  srf_updates[i].cm2_params->component_settings.lut1d_enable != srf_updates[i].surface->mcm_lut1d_enable))))
+				srf_updates[i].flip_addr->address.tmz_surface != srf_updates[i].surface->address.tmz_surface)))
 			return true;
 	}
 
@@ -6063,17 +6138,17 @@ static bool update_planes_and_stream_v3(struct dc *dc,
 	return true;
 }
 
-static void clear_update_flags(struct dc_surface_update *srf_updates,
+static void clear_update_bits(struct dc_surface_update *srf_updates,
 	int surface_count, struct dc_stream_state *stream)
 {
 	int i;
 
 	if (stream)
-		stream->update_flags.raw = 0;
+		stream_update_flags_clear(&stream->update_flags);
 
 	for (i = 0; i < surface_count; i++)
 		if (srf_updates[i].surface)
-			srf_updates[i].surface->update_flags.raw = 0;
+			dc_pipe_update_bits_clear(&srf_updates[i].surface->update_bits);
 }
 
 bool dc_update_planes_and_stream(struct dc *dc,
@@ -6125,7 +6200,7 @@ void dc_commit_updates_for_stream(struct dc *dc,
 	}
 
 	if (ret && dc->ctx->dce_version >= DCN_VERSION_3_2)
-		clear_update_flags(srf_updates, surface_count, stream);
+		clear_update_bits(srf_updates, surface_count, stream);
 }
 
 uint8_t dc_get_current_stream_count(struct dc *dc)
@@ -6165,6 +6240,160 @@ void dc_interrupt_ack(struct dc *dc, enum dc_irq_source src)
 	dal_irq_service_ack(dc->res_pool->irqs, src);
 }
 
+/* Preserve this tg if a physical link is still lighting a present display */
+static bool should_preserve_tg(struct dc *dc, struct timing_generator *tg)
+{
+	unsigned int i, j;
+
+	/* Check if a physical link is lighting this tg */
+	for (i = 0; i < dc->link_count; i++) {
+		struct dc_link *link = dc->links[i];
+		int fe;
+
+		if (!link || link->ep_type != DISPLAY_ENDPOINT_PHY ||
+				!link->link_enc ||
+				!link->link_enc->funcs->is_dig_enabled ||
+				!link->link_enc->funcs->is_dig_enabled(link->link_enc) ||
+				!link->link_enc->funcs->get_dig_frontend)
+			continue;
+
+		/* Get the DIG front-end this link's encoder drives; skip if none */
+		fe = link->link_enc->funcs->get_dig_frontend(link->link_enc);
+		if (fe == ENGINE_ID_UNKNOWN)
+			continue;
+
+		/* Find the stream encoder bound to this link's front-end */
+		for (j = 0; j < dc->res_pool->stream_enc_count; j++) {
+			struct stream_encoder *se = dc->res_pool->stream_enc[j];
+
+			/* Skip unless this stream encoder feeds our front-end and drives this tg */
+			if (se->id != fe || !se->funcs->dig_source_otg ||
+					(int)se->funcs->dig_source_otg(se) != tg->inst)
+				continue;
+
+			/* This link drives the OTG: keep a seamless-boot eDP, or
+			 * any external link whose sink is still connected.
+			 */
+			if (link->connector_signal == SIGNAL_TYPE_EDP)
+				return true;
+			if (link->link_enc->funcs->get_hpd_state &&
+					dc->link_srv->get_hpd_state(link))
+				return true;
+		}
+	}
+
+	return false;
+}
+
+/*
+ * GOP/vBIOS may leave an OPTC enabled for a display present at power-on but no
+ * longer driven (e.g. external DP unplugged at boot). Such a dangling pipe keeps
+ * DCN out of idle and blocks s0i3. If nothing needs to survive (no committed
+ * stream or seamless-boot eDP) and no sink is still connected, power down all hw
+ * blocks.
+ */
+void dc_disable_dangling_timing_generators(struct dc *dc)
+{
+	struct dce_hwseq *hws = dc->hwseq;
+	bool any_dangling = false;
+	bool any_preserved = false;
+	bool any_connected = false;
+	unsigned int i;
+
+	/* No real hw to touch on a virtual/emulated environment */
+	if (dc->ctx->dce_environment == DCE_ENV_VIRTUAL_HW)
+		return;
+
+	/* Wake hw out of IPS before reading/touching tg state */
+	dc_exit_ips_for_hw_access(dc);
+
+	/* Classify every enabled tg as either to-preserve or dangling */
+	for (i = 0; i < dc->res_pool->timing_generator_count; i++) {
+		struct timing_generator *tg = dc->res_pool->timing_generators[i];
+
+		if (!tg || !tg->funcs->is_tg_enabled ||
+				!tg->funcs->is_tg_enabled(tg))
+			continue;
+
+		if (should_preserve_tg(dc, tg))
+			any_preserved = true;
+		else
+			any_dangling = true;
+	}
+
+	/* A physically connected sink (HPD asserted) will be re-lit by a
+	 * subsequent atomic commit. For that case we don't call the global
+	 * power_down().
+	 */
+	for (i = 0; i < dc->link_count; i++) {
+		struct dc_link *link = dc->links[i];
+
+		if (link && link->ep_type == DISPLAY_ENDPOINT_PHY &&
+				link->link_enc && link->link_enc->funcs &&
+				link->link_enc->funcs->get_hpd_state &&
+				dc->link_srv->get_hpd_state(link)) {
+			any_connected = true;
+			break;
+		}
+	}
+
+	if (!any_dangling)
+		return;
+
+	if (!any_preserved && !any_connected && hws && hws->funcs.power_down) {
+		/* Truly headless / all sinks unplugged: nothing to preserve */
+		DC_LOG_DC("%s: powering down dangling hw blocks to allow idle\n",
+				__func__);
+		hws->funcs.power_down(dc);
+		return;
+	}
+}
+
+/*
+ * dc_get_flip_pending_on_otg() - Check if a GRPH_FLIP is still pending on OTG
+ *
+ * @dc: display core context @otg_inst: OTG instance to query
+ *
+ * Reads the HUBP flip-pending status for the pipe(s) bound to @otg_inst,
+ * returning true if any of them has not yet latched its programmed surface
+ * address.
+ *
+ * Unlike dc_plane_get_status(), this does not take or mutate a dc_plane_state,
+ * so it is safe to call from interrupt context without racing a concurrent
+ * commit that may be updating plane state.
+ *
+ * Return: true if a flip is still pending on the OTG, false otherwise.
+ */
+bool dc_get_flip_pending_on_otg(struct dc *dc, int otg_inst)
+{
+	bool flip_pending = false;
+	unsigned int i;
+
+	if (!dc || !dc->current_state)
+		return false;
+
+	dc_exit_ips_for_hw_access(dc);
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
+		struct hubp *hubp = pipe_ctx->plane_res.hubp;
+
+		if (!pipe_ctx->plane_state || !pipe_ctx->stream_res.tg)
+			continue;
+
+		if (pipe_ctx->stream_res.tg->inst != otg_inst)
+			continue;
+
+		if (hubp && hubp->funcs->hubp_is_flip_pending &&
+		    hubp->funcs->hubp_is_flip_pending(hubp)) {
+			flip_pending = true;
+			break;
+		}
+	}
+
+	return flip_pending;
+}
+
 void dc_power_down_on_boot(struct dc *dc)
 {
 	if (dc->ctx->dce_environment != DCE_ENV_VIRTUAL_HW &&
@@ -7542,7 +7771,7 @@ bool dc_capture_register_software_state(struct dc *dc, struct dc_register_softwa
 			struct dc_plane_state *plane_state = pipe_ctx->plane_state;
 
 			/* MPCC blending tree and mode control - capture actual blend configuration */
-			state->mpc.mpcc_mode[i] = (plane_state->blend_tf.type != TF_TYPE_BYPASS) ? 1 : 0;
+			state->mpc.mpcc_mode[i] = (plane_state->cm.blend_func.type != TF_TYPE_BYPASS) ? 1 : 0;
 			state->mpc.mpcc_alpha_blend_mode[i] = plane_state->per_pixel_alpha ? 1 : 0;
 			state->mpc.mpcc_alpha_multiplied_mode[i] = plane_state->pre_multiplied_alpha ? 1 : 0;
 			state->mpc.mpcc_blnd_active_overlap_only[i] = 0; /* Default - no overlap restriction */
@@ -7875,7 +8104,7 @@ struct dc_update_scratch_space {
 	struct dc_stream_state *stream;
 	struct dc_stream_update *stream_update;
 	bool update_v3;
-	bool do_clear_update_flags;
+	bool do_clear_update_bits;
 	enum surface_update_type update_type;
 	struct dc_state *new_context;
 	enum update_v3_flow flow;
@@ -7918,8 +8147,8 @@ static bool update_planes_and_stream_cleanup_v2(
 		const struct dc_update_scratch_space *scratch
 )
 {
-	if (scratch->do_clear_update_flags)
-		clear_update_flags(scratch->surface_updates, scratch->surface_count, scratch->stream);
+	if (scratch->do_clear_update_bits)
+		clear_update_bits(scratch->surface_updates, scratch->surface_count, scratch->stream);
 
 	return false;
 }
@@ -8173,8 +8402,8 @@ static bool update_planes_and_stream_cleanup_v3(
 		ASSERT(false);
 	}
 
-	if (scratch->do_clear_update_flags)
-		clear_update_flags(scratch->surface_updates, scratch->surface_count, scratch->stream);
+	if (scratch->do_clear_update_bits)
+		clear_update_bits(scratch->surface_updates, scratch->surface_count, scratch->stream);
 
 	return false;
 }
@@ -8197,7 +8426,7 @@ struct dc_update_scratch_space *dc_update_planes_and_stream_init(
 		.stream = stream,
 		.stream_update = stream_update,
 		.update_v3 = version >= DCN_VERSION_4_01 || version == DCN_VERSION_3_2 || version == DCN_VERSION_3_21,
-		.do_clear_update_flags = version >= DCN_VERSION_1_0,
+		.do_clear_update_bits = version >= DCN_VERSION_1_0,
 	};
 
 	return scratch;
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c.orig b/drivers/gpu/drm/amd/display/dc/core/dc.c.orig
new file mode 100644
index 0000000000000..1ea79c722f433
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/core/dc.c.orig
@@ -0,0 +1,8352 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ */
+
+#include "dm_services.h"
+
+#include "amdgpu.h"
+
+#include "dc.h"
+
+#include "core_status.h"
+#include "core_types.h"
+#include "hw_sequencer.h"
+#include "dce/dce_hwseq.h"
+#include "hw/dccg.h"
+
+#include "resource.h"
+#include "dc_state.h"
+#include "dc_state_priv.h"
+#include "dc_plane.h"
+#include "dc_plane_priv.h"
+#include "dc_stream_priv.h"
+
+#include "gpio_service_interface.h"
+#include "clk_mgr.h"
+#include "clock_source.h"
+#include "dc_bios_types.h"
+
+#include "bios_parser_interface.h"
+#include "bios/bios_parser_helper.h"
+#include "include/irq_service_interface.h"
+#include "transform.h"
+#include "dmcu.h"
+#include "dpp.h"
+#include "timing_generator.h"
+#include "abm.h"
+#include "dio/virtual/virtual_link_encoder.h"
+#include "hubp.h"
+
+#include "link_hwss.h"
+#include "link_encoder.h"
+#include "link_enc_cfg.h"
+
+#include "link_service.h"
+#include "link/protocols/link_dp_capability.h"
+#include "dm_helpers.h"
+#include "mem_input.h"
+
+#include "dc_dmub_srv.h"
+
+#include "dsc.h"
+
+#include "vm_helper.h"
+
+#include "dce/dce_i2c.h"
+
+#include "dmub/dmub_srv.h"
+
+#include "dce/dmub_psr.h"
+
+#include "dce/dmub_hw_lock_mgr.h"
+
+#include "dc_trace.h"
+
+#include "hw_sequencer_private.h"
+
+#if defined(CONFIG_DRM_AMD_DC_FP)
+#include "dml2_0/dml2_internal_types.h"
+#include "soc_and_ip_translator.h"
+#endif
+
+#include "dce/dmub_outbox.h"
+
+#define CTX \
+	dc->ctx
+
+#define DC_LOGGER \
+	dc->ctx->logger
+
+static const char DC_BUILD_ID[] = "production-build";
+
+/**
+ * DOC: Overview
+ *
+ * DC is the OS-agnostic component of the amdgpu DC driver.
+ *
+ * DC maintains and validates a set of structs representing the state of the
+ * driver and writes that state to AMD hardware
+ *
+ * Main DC HW structs:
+ *
+ * struct dc - The central struct.  One per driver.  Created on driver load,
+ * destroyed on driver unload.
+ *
+ * struct dc_context - One per driver.
+ * Used as a backpointer by most other structs in dc.
+ *
+ * struct dc_link - One per connector (the physical DP, HDMI, miniDP, or eDP
+ * plugpoints).  Created on driver load, destroyed on driver unload.
+ *
+ * struct dc_sink - One per display.  Created on boot or hotplug.
+ * Destroyed on shutdown or hotunplug.  A dc_link can have a local sink
+ * (the display directly attached).  It may also have one or more remote
+ * sinks (in the Multi-Stream Transport case)
+ *
+ * struct resource_pool - One per driver.  Represents the hw blocks not in the
+ * main pipeline.  Not directly accessible by dm.
+ *
+ * Main dc state structs:
+ *
+ * These structs can be created and destroyed as needed.  There is a full set of
+ * these structs in dc->current_state representing the currently programmed state.
+ *
+ * struct dc_state - The global DC state to track global state information,
+ * such as bandwidth values.
+ *
+ * struct dc_stream_state - Represents the hw configuration for the pipeline from
+ * a framebuffer to a display.  Maps one-to-one with dc_sink.
+ *
+ * struct dc_plane_state - Represents a framebuffer.  Each stream has at least one,
+ * and may have more in the Multi-Plane Overlay case.
+ *
+ * struct resource_context - Represents the programmable state of everything in
+ * the resource_pool.  Not directly accessible by dm.
+ *
+ * struct pipe_ctx - A member of struct resource_context.  Represents the
+ * internal hardware pipeline components.  Each dc_plane_state has either
+ * one or two (in the pipe-split case).
+ */
+
+/* Private functions */
+
+static inline void elevate_update_type(
+		struct surface_update_descriptor *descriptor,
+		enum surface_update_type new_type,
+		enum dc_lock_descriptor new_locks
+)
+{
+	if (new_type > descriptor->update_type)
+		descriptor->update_type = new_type;
+
+	descriptor->lock_descriptor |= new_locks;
+}
+
+static void destroy_links(struct dc *dc)
+{
+	uint32_t i;
+
+	for (i = 0; i < dc->link_count; i++) {
+		if (NULL != dc->links[i])
+			dc->link_srv->destroy_link(&dc->links[i]);
+	}
+}
+
+static uint32_t get_num_of_internal_disp(struct dc_link **links, uint32_t num_links)
+{
+	uint32_t i;
+	uint32_t count = 0;
+
+	for (i = 0; i < num_links; i++) {
+		if (links[i]->connector_signal == SIGNAL_TYPE_EDP ||
+				links[i]->is_internal_display)
+			count++;
+	}
+
+	return count;
+}
+
+static int get_seamless_boot_stream_count(struct dc_state *ctx)
+{
+	uint8_t i;
+	uint8_t seamless_boot_stream_count = 0;
+
+	for (i = 0; i < ctx->stream_count; i++)
+		if (ctx->streams[i]->apply_seamless_boot_optimization)
+			seamless_boot_stream_count++;
+
+	return seamless_boot_stream_count;
+}
+
+static bool create_links(
+		struct dc *dc,
+		uint32_t num_virtual_links)
+{
+	uint32_t i;
+	int connectors_num;
+	struct dc_bios *bios = dc->ctx->dc_bios;
+
+	dc->link_count = 0;
+
+	connectors_num = bios->funcs->get_connectors_number(bios);
+
+	DC_LOG_DC("BIOS object table - number of connectors: %d", connectors_num);
+
+	if (connectors_num > ENUM_ID_COUNT) {
+		dm_error(
+			"DC: Number of connectors %d exceeds maximum of %d!\n",
+			connectors_num,
+			ENUM_ID_COUNT);
+		return false;
+	}
+
+	dm_output_to_console(
+		"DC: %s: connectors_num: physical:%d, virtual:%d\n",
+		__func__,
+		connectors_num,
+		num_virtual_links);
+
+	/* When getting the number of connectors, the VBIOS reports the number of valid indices,
+	 * but it doesn't say which indices are valid, and not every index has an actual connector.
+	 * So, if we don't find a connector on an index, that is not an error.
+	 *
+	 * - There is no guarantee that the first N indices will be valid
+	 * - VBIOS may report a higher amount of valid indices than there are actual connectors
+	 * - Some VBIOS have valid configurations for more connectors than there actually are
+	 *   on the card. This may be because the manufacturer used the same VBIOS for different
+	 *   variants of the same card.
+	 */
+	for (i = 0; dc->link_count < connectors_num && i < MAX_LINKS; i++) {
+		struct graphics_object_id connector_id = bios->funcs->get_connector_id(bios, (uint8_t)i);
+		struct link_init_data link_init_params = {0};
+		struct dc_link *link;
+
+		if (connector_id.id == CONNECTOR_ID_UNKNOWN)
+			continue;
+
+		DC_LOG_DC("BIOS object table - printing link object info for connector number: %d, link_index: %d", i, dc->link_count);
+
+		link_init_params.ctx = dc->ctx;
+		/* next BIOS object table connector */
+		link_init_params.connector_index = (uint8_t)i;
+		link_init_params.link_index = dc->link_count;
+		link_init_params.dc = dc;
+		link = dc->link_srv->create_link(&link_init_params);
+
+		if (link) {
+			dc->links[dc->link_count] = link;
+			link->dc = dc;
+			++dc->link_count;
+		}
+	}
+
+	DC_LOG_DC("BIOS object table - end");
+
+	/* Create a link for each usb4 dpia port */
+	dc->lowest_dpia_link_index = MAX_LINKS;
+	for (i = 0; i < dc->res_pool->usb4_dpia_count; i++) {
+		struct link_init_data link_init_params = {0};
+		struct dc_link *link;
+
+		link_init_params.ctx = dc->ctx;
+		link_init_params.connector_index = (uint8_t)i;
+		link_init_params.link_index = dc->link_count;
+		link_init_params.dc = dc;
+		link_init_params.is_dpia_link = true;
+
+		link = dc->link_srv->create_link(&link_init_params);
+		if (link) {
+			if (dc->lowest_dpia_link_index > dc->link_count)
+				dc->lowest_dpia_link_index = dc->link_count;
+
+			dc->links[dc->link_count] = link;
+			link->dc = dc;
+			++dc->link_count;
+		}
+	}
+
+	for (i = 0; i < num_virtual_links; i++) {
+		struct dc_link *link = kzalloc_obj(*link);
+		struct encoder_init_data enc_init = {0};
+
+		if (link == NULL) {
+			BREAK_TO_DEBUGGER();
+			goto failed_alloc;
+		}
+
+		link->link_index = dc->link_count;
+		dc->links[dc->link_count] = link;
+		dc->link_count++;
+
+		link->ctx = dc->ctx;
+		link->dc = dc;
+		link->connector_signal = SIGNAL_TYPE_VIRTUAL;
+		link->link_id.type = OBJECT_TYPE_CONNECTOR;
+		link->link_id.id = CONNECTOR_ID_VIRTUAL;
+		link->link_id.enum_id = ENUM_ID_1;
+		link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED;
+		link->replay_settings.config.replay_version = DC_REPLAY_VERSION_UNSUPPORTED;
+		link->link_enc = kzalloc_obj(*link->link_enc);
+
+		if (!link->link_enc) {
+			BREAK_TO_DEBUGGER();
+			goto failed_alloc;
+		}
+
+		link->link_status.dpcd_caps = &link->dpcd_caps;
+
+		enc_init.ctx = dc->ctx;
+		enc_init.channel = CHANNEL_ID_UNKNOWN;
+		enc_init.hpd_source = HPD_SOURCEID_UNKNOWN;
+		enc_init.transmitter = TRANSMITTER_UNKNOWN;
+		enc_init.connector = link->link_id;
+		enc_init.encoder.type = OBJECT_TYPE_ENCODER;
+		enc_init.encoder.id = ENCODER_ID_INTERNAL_VIRTUAL;
+		enc_init.encoder.enum_id = ENUM_ID_1;
+		virtual_link_encoder_construct(link->link_enc, &enc_init);
+	}
+
+	dc->caps.num_of_internal_disp = get_num_of_internal_disp(dc->links, dc->link_count);
+
+	return true;
+
+failed_alloc:
+	return false;
+}
+
+/* Create additional DIG link encoder objects if fewer than the platform
+ * supports were created during link construction. This can happen if the
+ * number of physical connectors is less than the number of DIGs.
+ */
+static bool create_link_encoders(struct dc *dc)
+{
+	bool res = true;
+	unsigned int num_usb4_dpia = dc->res_pool->res_cap->num_usb4_dpia;
+	unsigned int num_dig_link_enc = dc->res_pool->res_cap->num_dig_link_enc;
+	unsigned int i;
+
+	/* A platform without USB4 DPIA endpoints has a fixed mapping between DIG
+	 * link encoders and physical display endpoints and does not require
+	 * additional link encoder objects.
+	 */
+	if (num_usb4_dpia == 0)
+		return res;
+
+	/* Create as many link encoder objects as the platform supports. DPIA
+	 * endpoints can be programmably mapped to any DIG.
+	 */
+	if (num_dig_link_enc > dc->res_pool->dig_link_enc_count) {
+		for (i = 0; i < num_dig_link_enc; i++) {
+			struct link_encoder *link_enc = dc->res_pool->link_encoders[i];
+
+			if (!link_enc && dc->res_pool->funcs->link_enc_create_minimal) {
+				link_enc = dc->res_pool->funcs->link_enc_create_minimal(dc->ctx,
+						(enum engine_id)(ENGINE_ID_DIGA + i));
+				if (link_enc) {
+					dc->res_pool->link_encoders[i] = link_enc;
+					dc->res_pool->dig_link_enc_count++;
+				} else {
+					res = false;
+				}
+			}
+		}
+	}
+
+	return res;
+}
+
+/* Destroy any additional DIG link encoder objects created by
+ * create_link_encoders().
+ * NB: Must only be called after destroy_links().
+ */
+static void destroy_link_encoders(struct dc *dc)
+{
+	unsigned int num_usb4_dpia;
+	unsigned int num_dig_link_enc;
+	unsigned int i;
+
+	if (!dc->res_pool)
+		return;
+
+	num_usb4_dpia = dc->res_pool->res_cap->num_usb4_dpia;
+	num_dig_link_enc = dc->res_pool->res_cap->num_dig_link_enc;
+
+	/* A platform without USB4 DPIA endpoints has a fixed mapping between DIG
+	 * link encoders and physical display endpoints and does not require
+	 * additional link encoder objects.
+	 */
+	if (num_usb4_dpia == 0)
+		return;
+
+	for (i = 0; i < num_dig_link_enc; i++) {
+		struct link_encoder *link_enc = dc->res_pool->link_encoders[i];
+
+		if (link_enc) {
+			link_enc->funcs->destroy(&link_enc);
+			dc->res_pool->link_encoders[i] = NULL;
+			dc->res_pool->dig_link_enc_count--;
+		}
+	}
+}
+
+static struct dc_perf_trace *dc_perf_trace_create(void)
+{
+	return kzalloc_obj(struct dc_perf_trace);
+}
+
+static void dc_perf_trace_destroy(struct dc_perf_trace **perf_trace)
+{
+	kfree(*perf_trace);
+	*perf_trace = NULL;
+}
+
+static bool set_long_vtotal(struct dc *dc, struct dc_stream_state *stream, struct dc_crtc_timing_adjust *adjust)
+{
+	if (!dc || !stream || !adjust)
+		return false;
+
+	if (!dc->current_state)
+		return false;
+
+	int i;
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+
+		if (pipe->stream == stream && pipe->stream_res.tg) {
+			if (dc->hwss.set_long_vtotal)
+				dc->hwss.set_long_vtotal(&pipe, 1, adjust->v_total_min, adjust->v_total_max);
+
+			return true;
+		}
+	}
+
+	return false;
+}
+
+/**
+ * dc_stream_get_last_used_drr_vtotal - Looks up the pipe context of
+ * dc_stream_state and gets the last VTOTAL used by DRR (Dynamic Refresh Rate)
+ *
+ * @dc: [in] dc reference
+ * @stream: [in] Initial dc stream state
+ * @refresh_rate: [in] new refresh_rate
+ *
+ * Return: %true if the pipe context is found and there is an associated
+ *         timing_generator for the DC;
+ *         %false if the pipe context is not found or there is no
+ *         timing_generator for the DC.
+ */
+bool dc_stream_get_last_used_drr_vtotal(struct dc *dc,
+		struct dc_stream_state *stream,
+		uint32_t *refresh_rate)
+{
+	bool status = false;
+
+	int i = 0;
+
+	dc_exit_ips_for_hw_access(dc);
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+
+		if (pipe->stream == stream && pipe->stream_res.tg) {
+			/* Only execute if a function pointer has been defined for
+			 * the DC version in question
+			 */
+			if (pipe->stream_res.tg->funcs->get_last_used_drr_vtotal) {
+				pipe->stream_res.tg->funcs->get_last_used_drr_vtotal(pipe->stream_res.tg, refresh_rate);
+
+				status = true;
+
+				break;
+			}
+		}
+	}
+
+	return status;
+}
+
+#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
+static inline void
+dc_stream_forward_dmub_crc_window(struct dc_dmub_srv *dmub_srv,
+		struct rect *rect, struct otg_phy_mux *mux_mapping, bool is_stop)
+{
+	union dmub_rb_cmd cmd = {0};
+
+	cmd.secure_display.roi_info.phy_id = mux_mapping->phy_output_num;
+	cmd.secure_display.roi_info.otg_id = mux_mapping->otg_output_num;
+
+	if (is_stop) {
+		cmd.secure_display.header.type = DMUB_CMD__SECURE_DISPLAY;
+		cmd.secure_display.header.sub_type = DMUB_CMD__SECURE_DISPLAY_CRC_STOP_UPDATE;
+	} else {
+		cmd.secure_display.header.type = DMUB_CMD__SECURE_DISPLAY;
+		cmd.secure_display.header.sub_type = DMUB_CMD__SECURE_DISPLAY_CRC_WIN_NOTIFY;
+		cmd.secure_display.roi_info.x_start = rect->x;
+		cmd.secure_display.roi_info.y_start = rect->y;
+		cmd.secure_display.roi_info.x_end = rect->x + rect->width;
+		cmd.secure_display.roi_info.y_end = rect->y + rect->height;
+	}
+
+	dc_wake_and_execute_dmub_cmd(dmub_srv->ctx, &cmd, DM_DMUB_WAIT_TYPE_NO_WAIT);
+}
+
+static inline void
+dc_stream_forward_dmcu_crc_window(struct dmcu *dmcu,
+		struct rect *rect, struct otg_phy_mux *mux_mapping, bool is_stop)
+{
+	if (is_stop)
+		dmcu->funcs->stop_crc_win_update(dmcu, mux_mapping);
+	else
+		dmcu->funcs->forward_crc_window(dmcu, rect, mux_mapping);
+}
+
+bool
+dc_stream_forward_crc_window(struct dc_stream_state *stream,
+		struct rect *rect, uint8_t phy_id, bool is_stop)
+{
+	struct dmcu *dmcu;
+	struct dc_dmub_srv *dmub_srv;
+	struct otg_phy_mux mux_mapping;
+	struct pipe_ctx *pipe;
+	int i;
+	struct dc *dc = stream->ctx->dc;
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+		if (pipe->stream == stream && !pipe->top_pipe && !pipe->prev_odm_pipe)
+			break;
+	}
+
+	/* Stream not found */
+	if (i == MAX_PIPES)
+		return false;
+
+	mux_mapping.phy_output_num = phy_id;
+	mux_mapping.otg_output_num = pipe->stream_res.tg->inst;
+
+	dmcu = dc->res_pool->dmcu;
+	dmub_srv = dc->ctx->dmub_srv;
+
+	/* forward to dmub */
+	if (dmub_srv)
+		dc_stream_forward_dmub_crc_window(dmub_srv, rect, &mux_mapping, is_stop);
+	/* forward to dmcu */
+	else if (dmcu && dmcu->funcs->is_dmcu_initialized(dmcu))
+		dc_stream_forward_dmcu_crc_window(dmcu, rect, &mux_mapping, is_stop);
+	else
+		return false;
+
+	return true;
+}
+
+static void
+dc_stream_forward_dmub_multiple_crc_window(struct dc_dmub_srv *dmub_srv,
+		struct crc_window *window, struct otg_phy_mux *mux_mapping, bool stop)
+{
+	int i;
+	union dmub_rb_cmd cmd = {0};
+
+	cmd.secure_display.mul_roi_ctl.phy_id = mux_mapping->phy_output_num;
+	cmd.secure_display.mul_roi_ctl.otg_id = mux_mapping->otg_output_num;
+
+	cmd.secure_display.header.type = DMUB_CMD__SECURE_DISPLAY;
+
+	if (stop) {
+		cmd.secure_display.header.sub_type = DMUB_CMD__SECURE_DISPLAY_MULTIPLE_CRC_STOP_UPDATE;
+	} else {
+		cmd.secure_display.header.sub_type = DMUB_CMD__SECURE_DISPLAY_MULTIPLE_CRC_WIN_NOTIFY;
+		for (i = 0; i < MAX_CRC_WINDOW_NUM; i++) {
+			cmd.secure_display.mul_roi_ctl.roi_ctl[i].x_start = window[i].rect.x;
+			cmd.secure_display.mul_roi_ctl.roi_ctl[i].y_start = window[i].rect.y;
+			cmd.secure_display.mul_roi_ctl.roi_ctl[i].x_end = window[i].rect.x + window[i].rect.width;
+			cmd.secure_display.mul_roi_ctl.roi_ctl[i].y_end = window[i].rect.y + window[i].rect.height;
+			cmd.secure_display.mul_roi_ctl.roi_ctl[i].enable = window[i].enable;
+		}
+	}
+
+	dc_wake_and_execute_dmub_cmd(dmub_srv->ctx, &cmd, DM_DMUB_WAIT_TYPE_NO_WAIT);
+}
+
+bool
+dc_stream_forward_multiple_crc_window(struct dc_stream_state *stream,
+		struct crc_window *window, uint8_t phy_id, bool stop)
+{
+	struct dc_dmub_srv *dmub_srv;
+	struct otg_phy_mux mux_mapping;
+	struct pipe_ctx *pipe;
+	int i;
+	struct dc *dc = stream->ctx->dc;
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+		if (pipe->stream == stream && !pipe->top_pipe && !pipe->prev_odm_pipe)
+			break;
+	}
+
+	/* Stream not found */
+	if (i == MAX_PIPES)
+		return false;
+
+	mux_mapping.phy_output_num = phy_id;
+	mux_mapping.otg_output_num = pipe->stream_res.tg->inst;
+
+	dmub_srv = dc->ctx->dmub_srv;
+
+	/* forward to dmub only. no dmcu support*/
+	if (dmub_srv)
+		dc_stream_forward_dmub_multiple_crc_window(dmub_srv, window, &mux_mapping, stop);
+	else
+		return false;
+
+	return true;
+}
+#endif /* CONFIG_DRM_AMD_SECURE_DISPLAY */
+
+/**
+ * dc_stream_configure_crc() - Configure CRC capture for the given stream.
+ * @dc: DC Object
+ * @stream: The stream to configure CRC on.
+ * @crc_window: CRC window (x/y start/end) information
+ * @enable: Enable CRC if true, disable otherwise.
+ * @continuous: Capture CRC on every frame if true. Otherwise, only capture
+ *              once.
+ * @idx: Capture CRC on which CRC engine instance
+ * @reset: Reset CRC engine before the configuration
+ * @crc_poly_mode: CRC polynomial mode
+ *
+ * By default, the entire frame is used to calculate the CRC.
+ *
+ * Return: %false if the stream is not found or CRC capture is not supported;
+ *         %true if the stream has been configured.
+ */
+bool dc_stream_configure_crc(struct dc *dc, struct dc_stream_state *stream,
+			     struct crc_params *crc_window, bool enable, bool continuous,
+			     uint8_t idx, bool reset, enum crc_poly_mode crc_poly_mode)
+{
+	struct pipe_ctx *pipe;
+	struct crc_params param;
+	struct timing_generator *tg;
+
+	pipe = resource_get_otg_master_for_stream(
+			&dc->current_state->res_ctx, stream);
+
+	/* Stream not found */
+	if (pipe == NULL)
+		return false;
+
+	dc_exit_ips_for_hw_access(dc);
+
+	/* By default, capture the full frame */
+	param.windowa_x_start = 0;
+	param.windowa_y_start = 0;
+	param.windowa_x_end = (uint16_t)pipe->stream->timing.h_addressable;
+	param.windowa_y_end = (uint16_t)pipe->stream->timing.v_addressable;
+	param.windowb_x_start = 0;
+	param.windowb_y_start = 0;
+	param.windowb_x_end = (uint16_t)pipe->stream->timing.h_addressable;
+	param.windowb_y_end = (uint16_t)pipe->stream->timing.v_addressable;
+	param.crc_poly_mode = crc_poly_mode;
+
+	if (crc_window) {
+		param.windowa_x_start = crc_window->windowa_x_start;
+		param.windowa_y_start = crc_window->windowa_y_start;
+		param.windowa_x_end = crc_window->windowa_x_end;
+		param.windowa_y_end = crc_window->windowa_y_end;
+		param.windowb_x_start = crc_window->windowb_x_start;
+		param.windowb_y_start = crc_window->windowb_y_start;
+		param.windowb_x_end = crc_window->windowb_x_end;
+		param.windowb_y_end = crc_window->windowb_y_end;
+	}
+
+	param.dsc_mode = pipe->stream->timing.flags.DSC ? 1:0;
+	param.odm_mode = pipe->next_odm_pipe ? 1:0;
+
+	/* Default to the union of both windows */
+	param.selection = UNION_WINDOW_A_B;
+	param.continuous_mode = continuous;
+	param.enable = enable;
+
+	param.crc_eng_inst = idx;
+	param.reset = reset;
+
+	tg = pipe->stream_res.tg;
+
+	/* Only call if supported */
+	if (tg->funcs->configure_crc)
+		return tg->funcs->configure_crc(tg, &param);
+	DC_LOG_WARNING("CRC capture not supported.");
+	return false;
+}
+
+/**
+ * dc_stream_get_crc() - Get CRC values for the given stream.
+ *
+ * @dc: DC object.
+ * @stream: The DC stream state of the stream to get CRCs from.
+ * @idx: index of crc engine to get CRC from
+ * @r_cr: CRC value for the red component.
+ * @g_y:  CRC value for the green component.
+ * @b_cb: CRC value for the blue component.
+ *
+ * dc_stream_configure_crc needs to be called beforehand to enable CRCs.
+ *
+ * Return:
+ * %false if stream is not found, or if CRCs are not enabled.
+ */
+bool dc_stream_get_crc(struct dc *dc, struct dc_stream_state *stream, uint8_t idx,
+		       uint32_t *r_cr, uint32_t *g_y, uint32_t *b_cb)
+{
+	int i;
+	struct pipe_ctx *pipe = NULL;
+	struct timing_generator *tg;
+
+	dc_exit_ips_for_hw_access(dc);
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+		if (pipe->stream == stream)
+			break;
+	}
+	/* Stream not found */
+	if (i == MAX_PIPES)
+		return false;
+
+	tg = pipe->stream_res.tg;
+
+	if (tg->funcs->get_crc)
+		return tg->funcs->get_crc(tg, idx, r_cr, g_y, b_cb);
+	DC_LOG_WARNING("CRC capture not supported.");
+	return false;
+}
+
+void dc_stream_set_dyn_expansion(struct dc *dc, struct dc_stream_state *stream,
+		enum dc_dynamic_expansion option)
+{
+	/* OPP FMT dyn expansion updates*/
+	int i;
+	struct pipe_ctx *pipe_ctx;
+
+	dc_exit_ips_for_hw_access(dc);
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		if (dc->current_state->res_ctx.pipe_ctx[i].stream
+				== stream) {
+			pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
+			pipe_ctx->stream_res.opp->dyn_expansion = option;
+			pipe_ctx->stream_res.opp->funcs->opp_set_dyn_expansion(
+					pipe_ctx->stream_res.opp,
+					COLOR_SPACE_YCBCR601,
+					stream->timing.display_color_depth,
+					stream->signal);
+		}
+	}
+}
+
+void dc_stream_set_dither_option(struct dc_stream_state *stream,
+		enum dc_dither_option option)
+{
+	struct bit_depth_reduction_params params;
+	struct dc_link *link = stream->link;
+	struct resource_context *res_ctx = &link->dc->current_state->res_ctx;
+	struct pipe_ctx *otg_master;
+	struct pipe_ctx *opp_heads[MAX_PIPES];
+	int opp_cnt;
+	int i;
+
+	otg_master = resource_get_otg_master_for_stream(res_ctx, stream);
+	if (!otg_master)
+		return;
+	if (option > DITHER_OPTION_MAX)
+		return;
+
+	opp_cnt = resource_get_opp_heads_for_otg_master(otg_master, res_ctx, opp_heads);
+
+	if (opp_cnt == 0)
+		return;
+
+	dc_exit_ips_for_hw_access(stream->ctx->dc);
+
+	stream->dither_option = option;
+
+	memset(&params, 0, sizeof(params));
+	resource_build_bit_depth_reduction_params(stream, &params);
+	stream->bit_depth_params = params;
+
+	/*
+	 * Program bit-depth reduction (dither) on every OPP head of the
+	 * stream. Under ODM combine there is more than one OPP head and they
+	 * must all be kept in sync, otherwise (e.g. when CRC capture requests
+	 * dither off) a secondary ODM segment can keep dither enabled and
+	 * produce a different CRC than the primary segment.
+	 */
+	for (i = 0; i < opp_cnt; i++) {
+		struct pipe_ctx *opp_head = opp_heads[i];
+
+		if (opp_head->plane_res.xfm &&
+		    opp_head->plane_res.xfm->funcs->transform_set_pixel_storage_depth) {
+			opp_head->plane_res.xfm->funcs->transform_set_pixel_storage_depth(
+				opp_head->plane_res.xfm,
+				opp_head->plane_res.scl_data.lb_params.depth,
+				&stream->bit_depth_params);
+		}
+
+		if (opp_head->stream_res.opp &&
+		    opp_head->stream_res.opp->funcs->opp_program_bit_depth_reduction) {
+			opp_head->stream_res.opp->funcs->opp_program_bit_depth_reduction(
+				opp_head->stream_res.opp, &params);
+		}
+	}
+}
+
+bool dc_stream_set_gamut_remap(struct dc *dc, const struct dc_stream_state *stream)
+{
+	int i;
+	bool ret = false;
+	struct pipe_ctx *pipes;
+
+	dc_exit_ips_for_hw_access(dc);
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		if (dc->current_state->res_ctx.pipe_ctx[i].stream == stream) {
+			pipes = &dc->current_state->res_ctx.pipe_ctx[i];
+			dc->hwss.program_gamut_remap(pipes);
+			ret = true;
+		}
+	}
+
+	return ret;
+}
+
+bool dc_stream_program_csc_matrix(struct dc *dc, struct dc_stream_state *stream)
+{
+	int i;
+	bool ret = false;
+	struct pipe_ctx *pipes;
+
+	dc_exit_ips_for_hw_access(dc);
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		if (dc->current_state->res_ctx.pipe_ctx[i].stream
+				== stream) {
+
+			pipes = &dc->current_state->res_ctx.pipe_ctx[i];
+			dc->hwss.program_output_csc(dc,
+					pipes,
+					stream->output_color_space,
+					stream->csc_color_matrix.matrix,
+					pipes->stream_res.opp->inst);
+			ret = true;
+		}
+	}
+
+	return ret;
+}
+
+void dc_stream_set_static_screen_params(struct dc *dc,
+		struct dc_stream_state **streams,
+		int num_streams,
+		const struct dc_static_screen_params *params)
+{
+	int i, j;
+	struct pipe_ctx *pipes_affected[MAX_PIPES];
+	int num_pipes_affected = 0;
+
+	dc_exit_ips_for_hw_access(dc);
+
+	for (i = 0; i < num_streams; i++) {
+		struct dc_stream_state *stream = streams[i];
+
+		for (j = 0; j < MAX_PIPES; j++) {
+			if (dc->current_state->res_ctx.pipe_ctx[j].stream
+					== stream) {
+				pipes_affected[num_pipes_affected++] =
+						&dc->current_state->res_ctx.pipe_ctx[j];
+			}
+		}
+	}
+
+	dc->hwss.set_static_screen_control(pipes_affected, num_pipes_affected, params);
+}
+
+static void dc_destruct(struct dc *dc)
+{
+	// reset link encoder assignment table on destruct
+	if (dc->res_pool && dc->res_pool->funcs->link_encs_assign &&
+			!dc->config.unify_link_enc_assignment)
+		link_enc_cfg_init(dc, dc->current_state);
+
+	if (dc->current_state) {
+		dc_state_release(dc->current_state);
+		dc->current_state = NULL;
+	}
+
+	destroy_links(dc);
+
+	destroy_link_encoders(dc);
+
+	if (dc->clk_mgr) {
+		dc_destroy_clk_mgr(dc->clk_mgr);
+		dc->clk_mgr = NULL;
+	}
+
+	dc_destroy_resource_pool(dc);
+#ifdef CONFIG_DRM_AMD_DC_FP
+	dc_destroy_soc_and_ip_translator(&dc->soc_and_ip_translator);
+#endif
+	if (dc->link_srv)
+		link_destroy_link_service(&dc->link_srv);
+
+	if (dc->ctx) {
+		if (dc->ctx->gpio_service)
+			dal_gpio_service_destroy(&dc->ctx->gpio_service);
+
+		if (dc->ctx->created_bios)
+			dal_bios_parser_destroy(&dc->ctx->dc_bios);
+		kfree(dc->ctx->logger);
+		dc_perf_trace_destroy(&dc->ctx->perf_trace);
+
+		kfree(dc->ctx);
+		dc->ctx = NULL;
+	}
+
+	kfree(dc->bw_vbios);
+	dc->bw_vbios = NULL;
+
+	kfree(dc->bw_dceip);
+	dc->bw_dceip = NULL;
+
+	kfree(dc->dcn_soc);
+	dc->dcn_soc = NULL;
+
+	kfree(dc->dcn_ip);
+	dc->dcn_ip = NULL;
+
+	kfree(dc->vm_helper);
+	dc->vm_helper = NULL;
+
+}
+
+static bool dc_construct_ctx(struct dc *dc,
+		const struct dc_init_data *init_params)
+{
+	struct dc_context *dc_ctx;
+
+	dc_ctx = kzalloc_obj(*dc_ctx);
+	if (!dc_ctx)
+		return false;
+
+	dc_stream_init_rmcm_3dlut(dc);
+
+	dc_ctx->cgs_device = init_params->cgs_device;
+	dc_ctx->driver_context = init_params->driver;
+	dc_ctx->dc = dc;
+	dc_ctx->asic_id = init_params->asic_id;
+	dc_ctx->dc_sink_id_count = 0;
+	dc_ctx->dc_stream_id_count = 0;
+	dc_ctx->dce_environment = init_params->dce_environment;
+	dc_ctx->dcn_reg_offsets = init_params->dcn_reg_offsets;
+	dc_ctx->nbio_reg_offsets = init_params->nbio_reg_offsets;
+	dc_ctx->clk_reg_offsets = init_params->clk_reg_offsets;
+
+	/* Create logger */
+	dc_ctx->logger = kmalloc_obj(*dc_ctx->logger);
+
+	if (!dc_ctx->logger) {
+		kfree(dc_ctx);
+		return false;
+	}
+
+	dc_ctx->logger->dev = adev_to_drm(init_params->driver);
+	dc->dml.logger = dc_ctx->logger;
+
+	dc_ctx->dce_version = resource_parse_asic_id(init_params->asic_id);
+
+	dc_ctx->perf_trace = dc_perf_trace_create();
+	if (!dc_ctx->perf_trace) {
+		kfree(dc_ctx);
+		ASSERT_CRITICAL(false);
+		return false;
+	}
+
+	dc->ctx = dc_ctx;
+
+	dc->link_srv = link_create_link_service();
+	if (!dc->link_srv)
+		return false;
+
+	return true;
+}
+
+static bool dc_construct(struct dc *dc,
+		const struct dc_init_data *init_params)
+{
+	struct dc_context *dc_ctx;
+	struct bw_calcs_dceip *dc_dceip;
+	struct bw_calcs_vbios *dc_vbios;
+	struct dcn_soc_bounding_box *dcn_soc;
+	struct dcn_ip_params *dcn_ip;
+
+	dc->config = init_params->flags;
+
+	// Allocate memory for the vm_helper
+	dc->vm_helper = kzalloc_obj(struct vm_helper);
+	if (!dc->vm_helper) {
+		dm_error("%s: failed to create dc->vm_helper\n", __func__);
+		goto fail;
+	}
+
+	memcpy(&dc->bb_overrides, &init_params->bb_overrides, sizeof(dc->bb_overrides));
+
+	dc_dceip = kzalloc_obj(*dc_dceip);
+	if (!dc_dceip) {
+		dm_error("%s: failed to create dceip\n", __func__);
+		goto fail;
+	}
+
+	dc->bw_dceip = dc_dceip;
+
+	dc_vbios = kzalloc_obj(*dc_vbios);
+	if (!dc_vbios) {
+		dm_error("%s: failed to create vbios\n", __func__);
+		goto fail;
+	}
+
+	dc->bw_vbios = dc_vbios;
+	dcn_soc = kzalloc_obj(*dcn_soc);
+	if (!dcn_soc) {
+		dm_error("%s: failed to create dcn_soc\n", __func__);
+		goto fail;
+	}
+
+	dc->dcn_soc = dcn_soc;
+
+	dcn_ip = kzalloc_obj(*dcn_ip);
+	if (!dcn_ip) {
+		dm_error("%s: failed to create dcn_ip\n", __func__);
+		goto fail;
+	}
+
+	dc->dcn_ip = dcn_ip;
+
+	if (init_params->bb_from_dmub)
+		dc->dml2_options.bb_from_dmub = init_params->bb_from_dmub;
+	else
+		dc->dml2_options.bb_from_dmub = NULL;
+
+	if (!dc_construct_ctx(dc, init_params)) {
+		dm_error("%s: failed to create ctx\n", __func__);
+		goto fail;
+	}
+
+	dc_ctx = dc->ctx;
+
+	/* Resource should construct all asic specific resources.
+	 * This should be the only place where we need to parse the asic id
+	 */
+	if (init_params->vbios_override)
+		dc_ctx->dc_bios = init_params->vbios_override;
+	else {
+		/* Create BIOS parser */
+		struct bp_init_data bp_init_data;
+
+		bp_init_data.ctx = dc_ctx;
+		bp_init_data.bios = init_params->asic_id.atombios_base_address;
+
+		dc_ctx->dc_bios = dal_bios_parser_create(
+				&bp_init_data, dc_ctx->dce_version);
+
+		if (!dc_ctx->dc_bios) {
+			ASSERT_CRITICAL(false);
+			goto fail;
+		}
+
+		dc_ctx->created_bios = true;
+	}
+
+	dc->vendor_signature = init_params->vendor_signature;
+
+	/* Create GPIO service */
+	dc_ctx->gpio_service = dal_gpio_service_create(
+			dc_ctx->dce_version,
+			dc_ctx->dce_environment,
+			dc_ctx);
+
+	if (!dc_ctx->gpio_service) {
+		ASSERT_CRITICAL(false);
+		goto fail;
+	}
+
+	dc->res_pool = dc_create_resource_pool(dc, init_params, dc_ctx->dce_version);
+	if (!dc->res_pool)
+		goto fail;
+
+	/* set i2c speed if not done by the respective dcnxxx__resource.c */
+	if (dc->caps.i2c_speed_in_khz_hdcp == 0)
+		dc->caps.i2c_speed_in_khz_hdcp = dc->caps.i2c_speed_in_khz;
+	if (dc->check_config.max_optimizable_video_width == 0)
+		dc->check_config.max_optimizable_video_width = 5120;
+	dc->clk_mgr = dc_clk_mgr_create(dc->ctx, dc->res_pool->pp_smu, dc->res_pool->dccg);
+	if (!dc->clk_mgr)
+		goto fail;
+#ifdef CONFIG_DRM_AMD_DC_FP
+	dc->clk_mgr->force_smu_not_present = init_params->force_smu_not_present;
+
+	if (dc->res_pool->funcs->update_bw_bounding_box)
+		dc->res_pool->funcs->update_bw_bounding_box(dc, dc->clk_mgr->bw_params);
+	dc->soc_and_ip_translator = dc_create_soc_and_ip_translator(dc_ctx->dce_version);
+	if (!dc->soc_and_ip_translator)
+		goto fail;
+#endif
+
+	if (!create_links(dc, init_params->num_virtual_links))
+		goto fail;
+
+	/* Create additional DIG link encoder objects if fewer than the platform
+	 * supports were created during link construction.
+	 */
+	if (!create_link_encoders(dc))
+		goto fail;
+
+	/* Creation of current_state must occur after dc->dml
+	 * is initialized in dc_create_resource_pool because
+	 * on creation it copies the contents of dc->dml
+	 */
+	dc->current_state = dc_state_create(dc, NULL);
+
+	if (!dc->current_state) {
+		dm_error("%s: failed to create validate ctx\n", __func__);
+		goto fail;
+	}
+
+	return true;
+
+fail:
+	return false;
+}
+
+static void disable_all_writeback_pipes_for_stream(
+		const struct dc *dc,
+		struct dc_stream_state *stream,
+		struct dc_state *context)
+{
+	(void)dc;
+	(void)context;
+	unsigned int i;
+
+	for (i = 0; i < stream->num_wb_info; i++)
+		stream->writeback_info[i].wb_enabled = false;
+}
+
+static void apply_ctx_interdependent_lock(struct dc *dc,
+					  struct dc_state *context,
+					  struct dc_stream_state *stream,
+					  bool lock)
+{
+	(void)dc;
+	(void)context;
+	unsigned int i;
+
+	/* Checks if interdependent update function pointer is NULL or not, takes care of DCE110 case */
+	if (dc->hwss.interdependent_update_lock)
+		dc->hwss.interdependent_update_lock(dc, context, lock);
+	else {
+		for (i = 0; i < dc->res_pool->pipe_count; i++) {
+			struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
+			struct pipe_ctx *old_pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
+
+			// Copied conditions that were previously in dce110_apply_ctx_for_surface
+			if (stream == pipe_ctx->stream) {
+				if (resource_is_pipe_type(pipe_ctx, OPP_HEAD) &&
+					(pipe_ctx->plane_state || old_pipe_ctx->plane_state))
+					dc->hwss.pipe_control_lock(dc, pipe_ctx, lock);
+			}
+		}
+	}
+}
+
+static void dc_update_visual_confirm_color(struct dc *dc, struct dc_state *context, struct pipe_ctx *pipe_ctx)
+{
+	if (dc->debug.visual_confirm & VISUAL_CONFIRM_EXPLICIT) {
+		memcpy(&pipe_ctx->visual_confirm_color, &pipe_ctx->plane_state->visual_confirm_color,
+		sizeof(pipe_ctx->visual_confirm_color));
+		return;
+	}
+
+	if (dc->ctx->dce_version >= DCN_VERSION_1_0) {
+		memset(&pipe_ctx->visual_confirm_color, 0, sizeof(struct tg_color));
+
+		if (dc->debug.visual_confirm == VISUAL_CONFIRM_HDR)
+			get_hdr_visual_confirm_color(pipe_ctx, &(pipe_ctx->visual_confirm_color));
+		else if (dc->debug.visual_confirm == VISUAL_CONFIRM_SURFACE)
+			get_surface_visual_confirm_color(pipe_ctx, &(pipe_ctx->visual_confirm_color));
+		else if (dc->debug.visual_confirm == VISUAL_CONFIRM_SWIZZLE)
+			get_surface_tile_visual_confirm_color(pipe_ctx, &(pipe_ctx->visual_confirm_color));
+		else if (dc->debug.visual_confirm == VISUAL_CONFIRM_HW_CURSOR)
+			get_cursor_visual_confirm_color(pipe_ctx, &(pipe_ctx->visual_confirm_color));
+		else if (dc->debug.visual_confirm == VISUAL_CONFIRM_DCC)
+			get_dcc_visual_confirm_color(dc, pipe_ctx, &(pipe_ctx->visual_confirm_color));
+		else {
+			if (dc->ctx->dce_version < DCN_VERSION_2_0)
+				color_space_to_black_color(
+					dc, pipe_ctx->stream->output_color_space, &(pipe_ctx->visual_confirm_color));
+		}
+		if (dc->ctx->dce_version >= DCN_VERSION_2_0) {
+			if (dc->debug.visual_confirm == VISUAL_CONFIRM_MPCTREE)
+				get_mpctree_visual_confirm_color(pipe_ctx, &(pipe_ctx->visual_confirm_color));
+			else if (dc->debug.visual_confirm == VISUAL_CONFIRM_SUBVP)
+				get_subvp_visual_confirm_color(pipe_ctx, &(pipe_ctx->visual_confirm_color));
+			else if (dc->debug.visual_confirm == VISUAL_CONFIRM_MCLK_SWITCH)
+				get_mclk_switch_visual_confirm_color(pipe_ctx, &(pipe_ctx->visual_confirm_color));
+			else if (dc->debug.visual_confirm == VISUAL_CONFIRM_FAMS2)
+				get_fams2_visual_confirm_color(dc, context, pipe_ctx, &(pipe_ctx->visual_confirm_color));
+			else if (dc->debug.visual_confirm == VISUAL_CONFIRM_VABC)
+				get_vabc_visual_confirm_color(pipe_ctx, &(pipe_ctx->visual_confirm_color));
+			else if (dc->debug.visual_confirm == VISUAL_CONFIRM_BOOSTED_REFRESH_RATE)
+				get_refresh_rate_confirm_color(pipe_ctx, &(pipe_ctx->visual_confirm_color));
+		}
+	}
+}
+
+void dc_get_visual_confirm_for_stream(
+	struct dc *dc,
+	struct dc_stream_state *stream_state,
+	struct tg_color *color)
+{
+	struct dc_stream_status *stream_status = dc_stream_get_status(stream_state);
+	struct pipe_ctx *pipe_ctx;
+	int i;
+	struct dc_plane_state *plane_state = NULL;
+
+	if (!stream_status)
+		return;
+
+	switch (dc->debug.visual_confirm) {
+	case VISUAL_CONFIRM_DISABLE:
+		return;
+	case VISUAL_CONFIRM_PSR:
+	case VISUAL_CONFIRM_FAMS:
+		pipe_ctx = dc_stream_get_pipe_ctx(stream_state);
+		if (!pipe_ctx)
+			return;
+		dc_dmub_srv_get_visual_confirm_color_cmd(dc, pipe_ctx);
+		memcpy(color, &dc->ctx->dmub_srv->dmub->visual_confirm_color, sizeof(struct tg_color));
+		return;
+
+	default:
+		/* find plane with highest layer_index */
+		for (i = 0; i < stream_status->plane_count; i++) {
+			if (stream_status->plane_states[i]->visible)
+				plane_state = stream_status->plane_states[i];
+		}
+		if (!plane_state)
+			return;
+		/* find pipe that contains plane with highest layer index */
+		for (i = 0; i < MAX_PIPES; i++) {
+			struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+
+			if (pipe->plane_state == plane_state) {
+				memcpy(color, &pipe->visual_confirm_color, sizeof(struct tg_color));
+				return;
+			}
+		}
+	}
+}
+
+/**
+ *  dc_stream_adjust_vmin_vmax - look up pipe context & update parts of DRR
+ *  @dc:     dc reference
+ *  @stream: Initial dc stream state
+ *  @adjust: Updated parameters for vertical_total_min and vertical_total_max
+ *
+ *  Looks up the pipe context of dc_stream_state and updates the
+ *  vertical_total_min and vertical_total_max of the DRR, Dynamic Refresh
+ *  Rate, which is a power-saving feature that targets reducing panel
+ *  refresh rate while the screen is static
+ *
+ *  Return: %true if the pipe context is found and adjusted;
+ *          %false if the pipe context is not found.
+ */
+bool dc_stream_adjust_vmin_vmax(struct dc *dc,
+		struct dc_stream_state *stream,
+		struct dc_crtc_timing_adjust *adjust)
+{
+	int i;
+
+	/*
+	 * Don't adjust DRR while there's bandwidth optimizations pending to
+	 * avoid conflicting with firmware updates.
+	 */
+	if (dc->ctx->dce_version > DCE_VERSION_MAX) {
+		if (dc->optimized_required &&
+			(stream->adjust.v_total_max != adjust->v_total_max ||
+			stream->adjust.v_total_min != adjust->v_total_min)) {
+			stream->adjust.timing_adjust_pending = true;
+			return false;
+		}
+	}
+
+	dc_exit_ips_for_hw_access(dc);
+
+	stream->adjust.v_total_max = adjust->v_total_max;
+	stream->adjust.v_total_mid = adjust->v_total_mid;
+	stream->adjust.v_total_mid_frame_num = adjust->v_total_mid_frame_num;
+	stream->adjust.v_total_min = adjust->v_total_min;
+	stream->adjust.allow_otg_v_count_halt = adjust->allow_otg_v_count_halt;
+
+	if (dc->caps.max_v_total != 0 &&
+		(adjust->v_total_max > dc->caps.max_v_total || adjust->v_total_min > dc->caps.max_v_total)) {
+		stream->adjust.timing_adjust_pending = false;
+		if (adjust->allow_otg_v_count_halt)
+			return set_long_vtotal(dc, stream, adjust);
+		else
+			return false;
+	}
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+
+		if (pipe->stream == stream && pipe->stream_res.tg) {
+			dc->hwss.set_drr(&pipe,
+					1,
+					*adjust);
+			stream->adjust.timing_adjust_pending = false;
+
+			if (dc->debug.visual_confirm == VISUAL_CONFIRM_BOOSTED_REFRESH_RATE) {
+				if (pipe->stream && pipe->plane_state) {
+					dc_update_visual_confirm_color(dc, dc->current_state, pipe);
+					dc->hwss.update_visual_confirm_color(dc, pipe, pipe->plane_res.hubp->mpcc_id);
+
+				}
+			}
+
+			if (dc->hwss.notify_cursor_offload_drr_update)
+				dc->hwss.notify_cursor_offload_drr_update(dc, dc->current_state, stream);
+
+			return true;
+		}
+	}
+
+	return false;
+}
+
+static void disable_dangling_plane(struct dc *dc, struct dc_state *context)
+{
+	unsigned int i, j;
+	struct dc_state *dangling_context = dc_state_create_current_copy(dc);
+	struct dc_state *current_ctx;
+	struct pipe_ctx *pipe;
+	struct timing_generator *tg;
+
+	if (dangling_context == NULL)
+		return;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct dc_stream_state *old_stream =
+				dc->current_state->res_ctx.pipe_ctx[i].stream;
+		bool should_disable = true;
+		bool pipe_split_change = false;
+
+		if ((context->res_ctx.pipe_ctx[i].top_pipe) &&
+			(dc->current_state->res_ctx.pipe_ctx[i].top_pipe))
+			pipe_split_change = context->res_ctx.pipe_ctx[i].top_pipe->pipe_idx !=
+				dc->current_state->res_ctx.pipe_ctx[i].top_pipe->pipe_idx;
+		else
+			pipe_split_change = context->res_ctx.pipe_ctx[i].top_pipe !=
+				dc->current_state->res_ctx.pipe_ctx[i].top_pipe;
+
+		for (j = 0; j < context->stream_count; j++) {
+			if (old_stream == context->streams[j]) {
+				should_disable = false;
+				break;
+			}
+		}
+		if (!should_disable && pipe_split_change &&
+				dc->current_state->stream_count != context->stream_count)
+			should_disable = true;
+
+		if (old_stream && !dc->current_state->res_ctx.pipe_ctx[i].top_pipe &&
+				!dc->current_state->res_ctx.pipe_ctx[i].prev_odm_pipe) {
+			struct pipe_ctx *old_pipe, *new_pipe;
+
+			old_pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+			new_pipe = &context->res_ctx.pipe_ctx[i];
+
+			if (old_pipe->plane_state && !new_pipe->plane_state)
+				should_disable = true;
+		}
+
+		if (should_disable && old_stream) {
+			bool is_phantom = dc_state_get_stream_subvp_type(dc->current_state, old_stream) == SUBVP_PHANTOM;
+			pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+			tg = pipe->stream_res.tg;
+			/* When disabling plane for a phantom pipe, we must turn on the
+			 * phantom OTG so the disable programming gets the double buffer
+			 * update. Otherwise the pipe will be left in a partially disabled
+			 * state that can result in underflow or hang when enabling it
+			 * again for different use.
+			 */
+			if (is_phantom) {
+				if (tg->funcs->enable_crtc) {
+					if (dc->hwseq->funcs.blank_pixel_data)
+						dc->hwseq->funcs.blank_pixel_data(dc, pipe, true);
+					tg->funcs->enable_crtc(tg);
+				}
+			}
+
+			if (is_phantom)
+				dc_state_rem_all_phantom_planes_for_stream(dc, old_stream, dangling_context, true);
+			else
+				dc_state_rem_all_planes_for_stream(dc, old_stream, dangling_context);
+			disable_all_writeback_pipes_for_stream(dc, old_stream, dangling_context);
+
+			if (pipe->stream && pipe->plane_state) {
+				if (!dc->debug.using_dml2)
+					set_p_state_switch_method(dc, context, pipe);
+				dc_update_visual_confirm_color(dc, context, pipe);
+			}
+
+			if (dc->hwss.apply_ctx_for_surface) {
+				apply_ctx_interdependent_lock(dc, dc->current_state, old_stream, true);
+				dc->hwss.apply_ctx_for_surface(dc, old_stream, 0, dangling_context);
+				apply_ctx_interdependent_lock(dc, dc->current_state, old_stream, false);
+				dc->hwss.post_unlock_program_front_end(dc, dangling_context);
+			}
+
+			if (dc->res_pool->funcs->prepare_mcache_programming)
+				dc->res_pool->funcs->prepare_mcache_programming(dc, dangling_context);
+			if (dc->hwss.program_front_end_for_ctx) {
+				dc->hwss.interdependent_update_lock(dc, dc->current_state, true);
+				dc->hwss.program_front_end_for_ctx(dc, dangling_context);
+				dc->hwss.interdependent_update_lock(dc, dc->current_state, false);
+				dc->hwss.post_unlock_program_front_end(dc, dangling_context);
+			}
+			/* We need to put the phantom OTG back into it's default (disabled) state or we
+			 * can get corruption when transition from one SubVP config to a different one.
+			 * The OTG is set to disable on falling edge of VUPDATE so the plane disable
+			 * will still get it's double buffer update.
+			 */
+			if (is_phantom) {
+				if (tg->funcs->disable_phantom_crtc)
+					tg->funcs->disable_phantom_crtc(tg);
+			}
+		}
+	}
+
+	current_ctx = dc->current_state;
+	dc->current_state = dangling_context;
+	dc_state_release(current_ctx);
+}
+
+static void disable_vbios_mode_if_required(
+		struct dc *dc,
+		struct dc_state *context)
+{
+	unsigned int i, j;
+
+	/* check if timing_changed, disable stream*/
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct dc_stream_state *stream = NULL;
+		struct dc_link *link = NULL;
+		struct pipe_ctx *pipe = NULL;
+
+		pipe = &context->res_ctx.pipe_ctx[i];
+		stream = pipe->stream;
+		if (stream == NULL)
+			continue;
+
+		if (stream->apply_seamless_boot_optimization)
+			continue;
+
+		// only looking for first odm pipe
+		if (pipe->prev_odm_pipe)
+			continue;
+
+		if (stream->link->local_sink &&
+			stream->link->local_sink->sink_signal == SIGNAL_TYPE_EDP) {
+			link = stream->link;
+		}
+
+		if (link != NULL && link->link_enc->funcs->is_dig_enabled(link->link_enc)) {
+			unsigned int enc_inst, tg_inst = 0;
+			unsigned int pix_clk_100hz = 0;
+
+			enc_inst = link->link_enc->funcs->get_dig_frontend(link->link_enc);
+			if (enc_inst != ENGINE_ID_UNKNOWN) {
+				for (j = 0; j < dc->res_pool->stream_enc_count; j++) {
+					if (dc->res_pool->stream_enc[j]->id == enc_inst) {
+						tg_inst = dc->res_pool->stream_enc[j]->funcs->dig_source_otg(
+							dc->res_pool->stream_enc[j]);
+						break;
+					}
+				}
+
+				dc->res_pool->dp_clock_source->funcs->get_dp_dto_frequency_100hz(
+					dc->res_pool->dp_clock_source,
+					tg_inst, &pix_clk_100hz);
+
+				if (link->link_status.link_active) {
+					uint32_t requested_pix_clk_100hz =
+						pipe->stream_res.pix_clk_params.requested_pix_clk_100hz;
+
+					if (pix_clk_100hz != requested_pix_clk_100hz) {
+						dc->link_srv->set_dpms_off(pipe);
+						pipe->stream->dpms_off = false;
+					}
+				}
+			}
+		}
+	}
+}
+
+/* Public functions */
+
+struct dc *dc_create(const struct dc_init_data *init_params)
+{
+	struct dc *dc = kvzalloc_obj(*dc);
+	unsigned int full_pipe_count;
+
+	if (!dc)
+		return NULL;
+
+	if (init_params->dce_environment == DCE_ENV_VIRTUAL_HW) {
+		dc->caps.linear_pitch_alignment = 64;
+		if (!dc_construct_ctx(dc, init_params))
+			goto destruct_dc;
+	} else {
+		if (!dc_construct(dc, init_params))
+			goto destruct_dc;
+
+		full_pipe_count = dc->res_pool->pipe_count;
+		if (dc->res_pool->underlay_pipe_index != NO_UNDERLAY_PIPE)
+			full_pipe_count--;
+		dc->caps.max_streams = min(
+				full_pipe_count,
+				dc->res_pool->stream_enc_count);
+
+		dc->caps.max_links = dc->link_count;
+		dc->caps.max_audios = dc->res_pool->audio_count;
+		dc->caps.linear_pitch_alignment = 64;
+
+		dc->caps.max_dp_protocol_version = DP_VERSION_1_4;
+
+		dc->caps.max_otg_num = dc->res_pool->res_cap->num_timing_generator;
+
+		if (dc->res_pool->dmcu != NULL)
+			dc->versions.dmcu_version = dc->res_pool->dmcu->dmcu_version;
+	}
+
+	dc->dcn_reg_offsets = init_params->dcn_reg_offsets;
+	dc->nbio_reg_offsets = init_params->nbio_reg_offsets;
+	dc->clk_reg_offsets = init_params->clk_reg_offsets;
+
+	/* Populate versioning information */
+	dc->versions.dc_ver = DC_VER;
+
+	dc->build_id = DC_BUILD_ID;
+
+	DC_LOG_DC("Display Core initialized\n");
+
+	return dc;
+
+destruct_dc:
+	dc_destruct(dc);
+	kvfree(dc);
+	return NULL;
+}
+
+static void detect_edp_presence(struct dc *dc)
+{
+	struct dc_link *edp_links[MAX_NUM_EDP];
+	struct dc_link *edp_link = NULL;
+	enum dc_connection_type type;
+	unsigned int i, edp_num;
+
+	dc_get_edp_links(dc, edp_links, &edp_num);
+	if (!edp_num)
+		return;
+
+	for (i = 0; i < edp_num; i++) {
+		edp_link = edp_links[i];
+		if (dc->config.edp_not_connected) {
+			edp_link->edp_sink_present = false;
+		} else {
+			dc_link_detect_connection_type(edp_link, &type);
+			edp_link->edp_sink_present = (type != dc_connection_none);
+		}
+	}
+}
+
+void dc_hardware_init(struct dc *dc)
+{
+
+	detect_edp_presence(dc);
+	if (dc->ctx->dce_environment != DCE_ENV_VIRTUAL_HW)
+		dc->hwss.init_hw(dc);
+	dc_dmub_srv_notify_fw_dc_power_state(dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D0);
+}
+
+void dc_init_callbacks(struct dc *dc,
+		const struct dc_callback_init *init_params)
+{
+	dc->ctx->cp_psp = init_params->cp_psp;
+}
+
+void dc_deinit_callbacks(struct dc *dc)
+{
+	memset(&dc->ctx->cp_psp, 0, sizeof(dc->ctx->cp_psp));
+}
+
+void dc_destroy(struct dc **dc)
+{
+	dc_destruct(*dc);
+	kvfree(*dc);
+	*dc = NULL;
+}
+
+static void enable_timing_multisync(
+		struct dc *dc,
+		struct dc_state *ctx)
+{
+	int i, multisync_count = 0;
+	int pipe_count = dc->res_pool->pipe_count;
+	struct pipe_ctx *multisync_pipes[MAX_PIPES] = { NULL };
+
+	for (i = 0; i < pipe_count; i++) {
+		if (!ctx->res_ctx.pipe_ctx[i].stream ||
+				!ctx->res_ctx.pipe_ctx[i].stream->triggered_crtc_reset.enabled)
+			continue;
+		if (ctx->res_ctx.pipe_ctx[i].stream == ctx->res_ctx.pipe_ctx[i].stream->triggered_crtc_reset.event_source)
+			continue;
+		multisync_pipes[multisync_count] = &ctx->res_ctx.pipe_ctx[i];
+		multisync_count++;
+	}
+
+	if (multisync_count > 0) {
+		dc->hwss.enable_per_frame_crtc_position_reset(
+			dc, multisync_count, multisync_pipes);
+	}
+}
+
+static void program_timing_sync(
+		struct dc *dc,
+		struct dc_state *ctx)
+{
+	int i, j, k;
+	int group_index = 0;
+	int num_group = 0;
+	int pipe_count = dc->res_pool->pipe_count;
+	struct pipe_ctx *unsynced_pipes[MAX_PIPES] = { NULL };
+
+	for (i = 0; i < pipe_count; i++) {
+		if (!ctx->res_ctx.pipe_ctx[i].stream
+				|| ctx->res_ctx.pipe_ctx[i].top_pipe
+				|| ctx->res_ctx.pipe_ctx[i].prev_odm_pipe)
+			continue;
+
+		unsynced_pipes[i] = &ctx->res_ctx.pipe_ctx[i];
+	}
+
+	for (i = 0; i < pipe_count; i++) {
+		int group_size = 1;
+		enum timing_synchronization_type sync_type = NOT_SYNCHRONIZABLE;
+		struct pipe_ctx *pipe_set[MAX_PIPES];
+
+		if (!unsynced_pipes[i])
+			continue;
+
+		pipe_set[0] = unsynced_pipes[i];
+		unsynced_pipes[i] = NULL;
+
+		/* Add tg to the set, search rest of the tg's for ones with
+		 * same timing, add all tgs with same timing to the group
+		 */
+		for (j = i + 1; j < pipe_count; j++) {
+			if (!unsynced_pipes[j])
+				continue;
+			if (sync_type != TIMING_SYNCHRONIZABLE &&
+				dc->hwss.enable_vblanks_synchronization &&
+				unsynced_pipes[j]->stream_res.tg->funcs->align_vblanks &&
+				resource_are_vblanks_synchronizable(
+					unsynced_pipes[j]->stream,
+					pipe_set[0]->stream)) {
+				sync_type = VBLANK_SYNCHRONIZABLE;
+				pipe_set[group_size] = unsynced_pipes[j];
+				unsynced_pipes[j] = NULL;
+				group_size++;
+			} else
+			if (sync_type != VBLANK_SYNCHRONIZABLE &&
+				resource_are_streams_timing_synchronizable(
+					unsynced_pipes[j]->stream,
+					pipe_set[0]->stream)) {
+				sync_type = TIMING_SYNCHRONIZABLE;
+				pipe_set[group_size] = unsynced_pipes[j];
+				unsynced_pipes[j] = NULL;
+				group_size++;
+			}
+		}
+
+		/* set first unblanked pipe as master */
+		for (j = 0; j < group_size; j++) {
+			bool is_blanked;
+
+			if (pipe_set[j]->stream_res.opp->funcs->dpg_is_blanked)
+				is_blanked =
+					pipe_set[j]->stream_res.opp->funcs->dpg_is_blanked(pipe_set[j]->stream_res.opp);
+			else
+				is_blanked =
+					pipe_set[j]->stream_res.tg->funcs->is_blanked(pipe_set[j]->stream_res.tg);
+			if (!is_blanked) {
+				if (j == 0)
+					break;
+
+				swap(pipe_set[0], pipe_set[j]);
+				break;
+			}
+		}
+
+		for (k = 0; k < group_size; k++) {
+			struct dc_stream_status *status = dc_state_get_stream_status(ctx, pipe_set[k]->stream);
+
+			if (!status)
+				continue;
+
+			status->timing_sync_info.group_id = num_group;
+			status->timing_sync_info.group_size = group_size;
+			if (k == 0)
+				status->timing_sync_info.master = true;
+			else
+				status->timing_sync_info.master = false;
+
+		}
+
+		/* remove any other unblanked pipes as they have already been synced */
+		if (dc->config.use_pipe_ctx_sync_logic) {
+			/* check pipe's syncd to decide which pipe to be removed */
+			for (j = 1; j < group_size; j++) {
+				if (pipe_set[j]->pipe_idx_syncd == pipe_set[0]->pipe_idx_syncd) {
+					group_size--;
+					pipe_set[j] = pipe_set[group_size];
+					j--;
+				} else
+					/* link slave pipe's syncd with master pipe */
+					pipe_set[j]->pipe_idx_syncd = pipe_set[0]->pipe_idx_syncd;
+			}
+		} else {
+			/* remove any other pipes by checking valid plane */
+			for (j = j + 1; j < group_size; j++) {
+				bool is_blanked;
+
+				if (pipe_set[j]->stream_res.opp->funcs->dpg_is_blanked)
+					is_blanked =
+						pipe_set[j]->stream_res.opp->funcs->dpg_is_blanked(pipe_set[j]->stream_res.opp);
+				else
+					is_blanked =
+						pipe_set[j]->stream_res.tg->funcs->is_blanked(pipe_set[j]->stream_res.tg);
+				if (!is_blanked) {
+					group_size--;
+					pipe_set[j] = pipe_set[group_size];
+					j--;
+				}
+			}
+		}
+
+		if (group_size > 1) {
+			if (sync_type == TIMING_SYNCHRONIZABLE) {
+				dc->hwss.enable_timing_synchronization(
+					dc, ctx, group_index, group_size, pipe_set);
+			} else
+				if (sync_type == VBLANK_SYNCHRONIZABLE) {
+				dc->hwss.enable_vblanks_synchronization(
+					dc, group_index, group_size, pipe_set);
+				}
+			group_index++;
+		}
+		num_group++;
+	}
+}
+
+static bool streams_changed(struct dc *dc,
+			    struct dc_stream_state *streams[],
+			    uint8_t stream_count)
+{
+	uint8_t i;
+
+	if (stream_count != dc->current_state->stream_count)
+		return true;
+
+	for (i = 0; i < dc->current_state->stream_count; i++) {
+		if (dc->current_state->streams[i] != streams[i])
+			return true;
+		if (!streams[i]->link->link_state_valid)
+			return true;
+	}
+
+	return false;
+}
+
+bool dc_validate_boot_timing(const struct dc *dc,
+				const struct dc_sink *sink,
+				struct dc_crtc_timing *crtc_timing)
+{
+	struct timing_generator *tg;
+	struct stream_encoder *se = NULL;
+
+	struct dc_crtc_timing hw_crtc_timing = {0};
+
+	struct dc_link *link = sink->link;
+	unsigned int i, enc_inst;
+	unsigned int tg_inst = 0;
+
+	/* Support seamless boot on EDP displays only */
+	if (sink->sink_signal != SIGNAL_TYPE_EDP) {
+		return false;
+	}
+
+	if (dc->debug.force_odm_combine) {
+		DC_LOG_DEBUG("boot timing validation failed due to force_odm_combine\n");
+		return false;
+	}
+
+	/* Check for enabled DIG to identify enabled display */
+	if (!link->link_enc->funcs->is_dig_enabled(link->link_enc)) {
+		DC_LOG_DEBUG("boot timing validation failed due to disabled DIG\n");
+		return false;
+	}
+
+	enc_inst = link->link_enc->funcs->get_dig_frontend(link->link_enc);
+
+	if (enc_inst == ENGINE_ID_UNKNOWN) {
+		DC_LOG_DEBUG("boot timing validation failed due to unknown DIG engine ID\n");
+		return false;
+	}
+
+	for (i = 0; i < dc->res_pool->stream_enc_count; i++) {
+		if (dc->res_pool->stream_enc[i]->id == enc_inst) {
+
+			se = dc->res_pool->stream_enc[i];
+
+			tg_inst = dc->res_pool->stream_enc[i]->funcs->dig_source_otg(
+				dc->res_pool->stream_enc[i]);
+			break;
+		}
+	}
+
+	// tg_inst not found
+	if (i == dc->res_pool->stream_enc_count) {
+		DC_LOG_DEBUG("boot timing validation failed due to timing generator instance not found\n");
+		return false;
+	}
+
+	if (tg_inst >= dc->res_pool->timing_generator_count) {
+		DC_LOG_DEBUG("boot timing validation failed due to invalid timing generator count\n");
+		return false;
+	}
+
+	if (tg_inst != link->link_enc->preferred_engine) {
+		DC_LOG_DEBUG("boot timing validation failed due to non-preferred timing generator\n");
+		return false;
+	}
+
+	tg = dc->res_pool->timing_generators[tg_inst];
+
+	if (!tg->funcs->get_hw_timing) {
+		DC_LOG_DEBUG("boot timing validation failed due to missing get_hw_timing callback\n");
+		return false;
+	}
+
+	if (!tg->funcs->get_hw_timing(tg, &hw_crtc_timing)) {
+		DC_LOG_DEBUG("boot timing validation failed due to failed get_hw_timing return\n");
+		return false;
+	}
+
+	if (crtc_timing->h_total != hw_crtc_timing.h_total) {
+		DC_LOG_DEBUG("boot timing validation failed due to h_total mismatch\n");
+		return false;
+	}
+
+	if (crtc_timing->h_border_left != hw_crtc_timing.h_border_left) {
+		DC_LOG_DEBUG("boot timing validation failed due to h_border_left mismatch\n");
+		return false;
+	}
+
+	if (crtc_timing->h_addressable != hw_crtc_timing.h_addressable) {
+		DC_LOG_DEBUG("boot timing validation failed due to h_addressable mismatch\n");
+		return false;
+	}
+
+	if (crtc_timing->h_border_right != hw_crtc_timing.h_border_right) {
+		DC_LOG_DEBUG("boot timing validation failed due to h_border_right mismatch\n");
+		return false;
+	}
+
+	if (crtc_timing->h_front_porch != hw_crtc_timing.h_front_porch) {
+		DC_LOG_DEBUG("boot timing validation failed due to h_front_porch mismatch\n");
+		return false;
+	}
+
+	if (crtc_timing->h_sync_width != hw_crtc_timing.h_sync_width) {
+		DC_LOG_DEBUG("boot timing validation failed due to h_sync_width mismatch\n");
+		return false;
+	}
+
+	if (crtc_timing->v_total != hw_crtc_timing.v_total) {
+		DC_LOG_DEBUG("boot timing validation failed due to v_total mismatch\n");
+		return false;
+	}
+
+	if (crtc_timing->v_border_top != hw_crtc_timing.v_border_top) {
+		DC_LOG_DEBUG("boot timing validation failed due to v_border_top mismatch\n");
+		return false;
+	}
+
+	if (crtc_timing->v_addressable != hw_crtc_timing.v_addressable) {
+		DC_LOG_DEBUG("boot timing validation failed due to v_addressable mismatch\n");
+		return false;
+	}
+
+	if (crtc_timing->v_border_bottom != hw_crtc_timing.v_border_bottom) {
+		DC_LOG_DEBUG("boot timing validation failed due to v_border_bottom mismatch\n");
+		return false;
+	}
+
+	if (crtc_timing->v_front_porch != hw_crtc_timing.v_front_porch) {
+		DC_LOG_DEBUG("boot timing validation failed due to v_front_porch mismatch\n");
+		return false;
+	}
+
+	if (crtc_timing->v_sync_width != hw_crtc_timing.v_sync_width) {
+		DC_LOG_DEBUG("boot timing validation failed due to v_sync_width mismatch\n");
+		return false;
+	}
+
+	if (crtc_timing->flags.DSC) {
+		struct display_stream_compressor *dsc = NULL;
+		struct dcn_dsc_state dsc_state = {0};
+
+		if (dc->ctx->dce_version < DCN_VERSION_4_2) {
+			/*vbios enabled eDP dsc for one of DCN315  only but it has known issue,
+			since there is no production bios update, block it there*/
+			DC_LOG_DEBUG("boot timing validation failed due to unsupported DSC on this ASIC\n");
+			return false;
+		}
+
+		/* Find DSC associated with this timing generator */
+		if (tg_inst < (unsigned int)dc->res_pool->res_cap->num_dsc) {
+			dsc = dc->res_pool->dscs[tg_inst];
+		}
+
+		if (!dsc || !dsc->funcs->dsc_read_state) {
+			DC_LOG_DEBUG("boot timing validation failed due to no DSC resource or read function\n");
+			return false;
+		}
+
+		/* Read current DSC hardware state */
+		dsc->funcs->dsc_read_state(dsc, &dsc_state);
+
+		/* Check if DSC is actually enabled in hardware */
+		if (dsc_state.dsc_clock_en == 0) {
+			DC_LOG_DEBUG("boot timing validation failed due to DSC not enabled in hardware\n");
+			return false;
+		}
+
+		uint32_t num_slices_h = 0;
+		uint32_t num_slices_v = 0;
+
+		if (dsc_state.dsc_slice_width > 0) {
+			num_slices_h = (crtc_timing->h_addressable + dsc_state.dsc_slice_width - 1) / dsc_state.dsc_slice_width;
+		}
+
+		if (dsc_state.dsc_slice_height > 0) {
+			num_slices_v = (crtc_timing->v_addressable + dsc_state.dsc_slice_height - 1) / dsc_state.dsc_slice_height;
+		}
+
+		if (crtc_timing->dsc_cfg.num_slices_h != num_slices_h) {
+			DC_LOG_DEBUG("boot timing validation failed due to num_slices_h mismatch\n");
+			return false;
+		}
+
+		if (crtc_timing->dsc_cfg.num_slices_v != num_slices_v) {
+			DC_LOG_DEBUG("boot timing validation failed due to num_slices_v mismatch\n");
+			return false;
+		}
+
+		if (crtc_timing->dsc_cfg.bits_per_pixel != dsc_state.dsc_bits_per_pixel) {
+			DC_LOG_DEBUG("boot timing validation failed due to bits_per_pixel mismatch\n");
+			return false;
+		}
+
+		if (crtc_timing->dsc_cfg.block_pred_enable != dsc_state.dsc_block_pred_enable) {
+			DC_LOG_DEBUG("boot timing validation failed due to block_pred_enable mismatch\n");
+			return false;
+		}
+
+		if (crtc_timing->dsc_cfg.linebuf_depth != dsc_state.dsc_line_buf_depth) {
+			DC_LOG_DEBUG("boot timing validation failed due to linebuf_depth mismatch\n");
+			return false;
+		}
+
+		if (crtc_timing->dsc_cfg.version_minor != dsc_state.dsc_version_minor) {
+			DC_LOG_DEBUG("boot timing validation failed due to version_minor mismatch\n");
+			return false;
+		}
+
+		if (crtc_timing->dsc_cfg.ycbcr422_simple != dsc_state.dsc_simple_422) {
+			DC_LOG_DEBUG("boot timing validation failed due to pixel encoding mismatch\n");
+			return false;
+		}
+
+		// Skip checks for is_frl, is_dp, and rc_buffer_size which are not programmed by vbios
+		// or not necessary for seamless boot validation.
+	}
+
+	if (dc_is_dp_signal(link->connector_signal)) {
+		unsigned int pix_clk_100hz = 0;
+		uint32_t numOdmPipes = 1;
+		uint32_t id_src[4] = {0};
+
+		dc->res_pool->dp_clock_source->funcs->get_dp_dto_frequency_100hz(
+			dc->res_pool->dp_clock_source,
+			tg_inst, &pix_clk_100hz);
+
+		if (tg->funcs->get_optc_source)
+			tg->funcs->get_optc_source(tg,
+						&numOdmPipes, &id_src[0], &id_src[1]);
+
+		if (numOdmPipes == 2) {
+			pix_clk_100hz *= 2;
+		} else if (numOdmPipes == 4) {
+			pix_clk_100hz *= 4;
+		} else if (se && se->funcs->get_pixels_per_cycle) {
+			uint32_t pixels_per_cycle = se->funcs->get_pixels_per_cycle(se);
+
+			if (pixels_per_cycle != 1 && !dc->debug.enable_dp_dig_pixel_rate_div_policy) {
+				DC_LOG_DEBUG("boot timing validation failed due to pixels_per_cycle\n");
+				return false;
+			}
+
+			pix_clk_100hz *= pixels_per_cycle;
+		}
+
+		// Note: In rare cases, HW pixclk may differ from crtc's pixclk
+		// slightly due to rounding issues in 10 kHz units.
+		if (crtc_timing->pix_clk_100hz != pix_clk_100hz) {
+			DC_LOG_DEBUG("boot timing validation failed due to pix_clk_100hz mismatch\n");
+			return false;
+		}
+
+		if (!se || !se->funcs->dp_get_pixel_format) {
+			DC_LOG_DEBUG("boot timing validation failed due to missing dp_get_pixel_format\n");
+			return false;
+		}
+
+		if (!se->funcs->dp_get_pixel_format(
+			se,
+			&hw_crtc_timing.pixel_encoding,
+			&hw_crtc_timing.display_color_depth)) {
+			DC_LOG_DEBUG("boot timing validation failed due to dp_get_pixel_format failure\n");
+			return false;
+		}
+
+		if (hw_crtc_timing.display_color_depth != crtc_timing->display_color_depth) {
+			DC_LOG_DEBUG("boot timing validation failed due to display_color_depth mismatch\n");
+			return false;
+		}
+
+		if (hw_crtc_timing.pixel_encoding != crtc_timing->pixel_encoding) {
+			DC_LOG_DEBUG("boot timing validation failed due to pixel_encoding mismatch\n");
+			return false;
+		}
+	}
+
+
+	if (link->dpcd_caps.dprx_feature.bits.VSC_SDP_COLORIMETRY_SUPPORTED) {
+		DC_LOG_DEBUG("boot timing validation failed due to VSC SDP colorimetry\n");
+		return false;
+	}
+
+	if (link->dpcd_caps.channel_coding_cap.bits.DP_128b_132b_SUPPORTED) {
+		DC_LOG_DEBUG("boot timing validation failed due to DP 128b/132b\n");
+		return false;
+	}
+
+	if (dc->link_srv->edp_is_ilr_optimization_required(link, crtc_timing)) {
+		DC_LOG_EVENT_LINK_TRAINING("Seamless boot disabled to optimize eDP link rate\n");
+		return false;
+	}
+
+	return true;
+}
+
+static inline bool should_update_pipe_for_stream(
+		struct dc_state *context,
+		struct pipe_ctx *pipe_ctx,
+		struct dc_stream_state *stream)
+{
+	return (pipe_ctx->stream && pipe_ctx->stream == stream);
+}
+
+static inline bool should_update_pipe_for_plane(
+		struct dc_state *context,
+		struct pipe_ctx *pipe_ctx,
+		struct dc_plane_state *plane_state)
+{
+	return (pipe_ctx->plane_state == plane_state);
+}
+
+void dc_enable_stereo(
+	struct dc *dc,
+	struct dc_state *context,
+	struct dc_stream_state *streams[],
+	uint8_t stream_count)
+{
+	int i, j;
+	struct pipe_ctx *pipe;
+
+	dc_exit_ips_for_hw_access(dc);
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		if (context != NULL) {
+			pipe = &context->res_ctx.pipe_ctx[i];
+		} else {
+			context = dc->current_state;
+			pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+		}
+
+		for (j = 0; pipe && j < stream_count; j++)  {
+			if (should_update_pipe_for_stream(context, pipe, streams[j]) &&
+				dc->hwss.setup_stereo)
+				dc->hwss.setup_stereo(pipe, dc);
+		}
+	}
+}
+
+void dc_trigger_sync(struct dc *dc, struct dc_state *context)
+{
+	if (context->stream_count > 1 && !dc->debug.disable_timing_sync) {
+		dc_exit_ips_for_hw_access(dc);
+
+		enable_timing_multisync(dc, context);
+		program_timing_sync(dc, context);
+	}
+}
+
+static uint8_t get_stream_mask(struct dc *dc, struct dc_state *context)
+{
+	unsigned int i;
+	unsigned int stream_mask = 0;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		if (context->res_ctx.pipe_ctx[i].stream)
+			stream_mask |= 1 << i;
+	}
+
+	return (uint8_t)stream_mask;
+}
+
+void dc_z10_restore(const struct dc *dc)
+{
+	if (dc->hwss.z10_restore)
+		dc->hwss.z10_restore(dc);
+}
+
+void dc_z10_save_init(struct dc *dc)
+{
+	if (dc->hwss.z10_save_init)
+		dc->hwss.z10_save_init(dc);
+}
+
+/* Set a pipe unlock order based on the change in DET allocation and stores it in dc scratch memory
+ * Prevents over allocation of DET during unlock process
+ * e.g. 2 pipe config with different streams with a max of 20 DET segments
+ *	Before:								After:
+ *		- Pipe0: 10 DET segments			- Pipe0: 12 DET segments
+ *		- Pipe1: 10 DET segments			- Pipe1: 8 DET segments
+ * If Pipe0 gets updated first, 22 DET segments will be allocated
+ */
+static void determine_pipe_unlock_order(struct dc *dc, struct dc_state *context)
+{
+	unsigned int i = 0;
+	struct pipe_ctx *pipe = NULL;
+	struct timing_generator *tg = NULL;
+
+	if (!dc->config.set_pipe_unlock_order)
+		return;
+
+	memset(dc->scratch.pipes_to_unlock_first, 0, sizeof(dc->scratch.pipes_to_unlock_first));
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		pipe = &context->res_ctx.pipe_ctx[i];
+		tg = pipe->stream_res.tg;
+
+		if (!resource_is_pipe_type(pipe, OTG_MASTER) ||
+				!tg->funcs->is_tg_enabled(tg) ||
+				dc_state_get_pipe_subvp_type(context, pipe) == SUBVP_PHANTOM) {
+			continue;
+		}
+
+		if (resource_calculate_det_for_stream(context, pipe) <
+				resource_calculate_det_for_stream(dc->current_state, &dc->current_state->res_ctx.pipe_ctx[i])) {
+			dc->scratch.pipes_to_unlock_first[i] = true;
+		}
+	}
+}
+
+/**
+ * dc_commit_state_no_check - Apply context to the hardware
+ *
+ * @dc: DC object with the current status to be updated
+ * @context: New state that will become the current status at the end of this function
+ *
+ * Applies given context to the hardware and copy it into current context.
+ * It's up to the user to release the src context afterwards.
+ *
+ * Return: an enum dc_status result code for the operation
+ */
+static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *context)
+{
+	struct dc_bios *dcb = dc->ctx->dc_bios;
+	enum dc_status result = DC_ERROR_UNEXPECTED;
+	struct pipe_ctx *pipe;
+	unsigned int i, k, l;
+	struct dc_stream_state *dc_streams[MAX_STREAMS] = {0};
+	struct dc_state *old_state;
+	bool subvp_prev_use = false;
+
+	dc_z10_restore(dc);
+	dc_allow_idle_optimizations(dc, false);
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *old_pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+
+		/* Check old context for SubVP */
+		subvp_prev_use |= (dc_state_get_pipe_subvp_type(dc->current_state, old_pipe) == SUBVP_PHANTOM);
+		if (subvp_prev_use)
+			break;
+	}
+
+	for (i = 0; i < context->stream_count; i++)
+		dc_streams[i] =  context->streams[i];
+
+	if (!dcb->funcs->is_accelerated_mode(dcb)) {
+		disable_vbios_mode_if_required(dc, context);
+		dc->hwss.enable_accelerated_mode(dc, context);
+	} else if (get_seamless_boot_stream_count(dc->current_state) > 0) {
+		/* If the previous Stream still retains the apply seamless boot flag,
+		 * it means the OS has not actually performed a flip yet.
+		 * At this point, if we receive dc_commit_streams again, we should
+		 * once more check whether the actual HW timing matches what the OS
+		 * has provided
+		 */
+		disable_vbios_mode_if_required(dc, context);
+	}
+
+	if (dc->hwseq->funcs.wait_for_pipe_update_if_needed) {
+		for (i = 0; i < dc->res_pool->pipe_count; i++) {
+			pipe = &context->res_ctx.pipe_ctx[i];
+			//Only delay otg master for a given config
+			if (resource_is_pipe_type(pipe, OTG_MASTER)) {
+				//dc_commit_state_no_check is always a full update
+				dc->hwseq->funcs.wait_for_pipe_update_if_needed(dc, pipe, false);
+				break;
+			}
+		}
+	}
+
+	if (context->stream_count > get_seamless_boot_stream_count(context) ||
+		context->stream_count == 0)
+		dc->hwss.prepare_bandwidth(dc, context);
+
+	/* When SubVP is active, all HW programming must be done while
+	 * SubVP lock is acquired
+	 */
+	if (dc->hwss.subvp_pipe_control_lock)
+		dc->hwss.subvp_pipe_control_lock(dc, context, true, true, NULL, subvp_prev_use);
+	if (dc->hwss.dmub_hw_control_lock)
+		dc->hwss.dmub_hw_control_lock(dc, context, true);
+
+	if (dc->hwss.update_dsc_pg)
+		dc->hwss.update_dsc_pg(dc, context, false);
+
+	disable_dangling_plane(dc, context);
+	/* re-program planes for existing stream, in case we need to
+	 * free up plane resource for later use
+	 */
+	if (dc->hwss.apply_ctx_for_surface) {
+		for (i = 0; i < context->stream_count; i++) {
+			if (context->streams[i]->mode_changed)
+				continue;
+			apply_ctx_interdependent_lock(dc, context, context->streams[i], true);
+			dc->hwss.apply_ctx_for_surface(
+				dc, context->streams[i],
+				context->stream_status[i].plane_count,
+				context); /* use new pipe config in new context */
+			apply_ctx_interdependent_lock(dc, context, context->streams[i], false);
+			dc->hwss.post_unlock_program_front_end(dc, context);
+		}
+	}
+
+	/* Program hardware */
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		pipe = &context->res_ctx.pipe_ctx[i];
+		dc->hwss.wait_for_mpcc_disconnect(dc, dc->res_pool, pipe);
+	}
+
+	for (i = 0; i < dc->current_state->stream_count; i++)
+		dc_dmub_srv_control_cursor_offload(dc, dc->current_state, dc->current_state->streams[i], false);
+
+	result = dc->hwss.apply_ctx_to_hw(dc, context);
+
+	for (i = 0; i < context->stream_count; i++)
+		dc_dmub_srv_control_cursor_offload(dc, context, context->streams[i], true);
+
+	if (result != DC_OK) {
+		/* Application of dc_state to hardware stopped. */
+		dc->current_state->res_ctx.link_enc_cfg_ctx.mode = LINK_ENC_CFG_STEADY;
+		return result;
+	}
+
+	dc_trigger_sync(dc, context);
+
+	/* Full update should unconditionally be triggered when dc_commit_state_no_check is called */
+	for (i = 0; i < context->stream_count; i++) {
+		uint32_t prev_dsc_changed = context->streams[i]->update_flags.bits.dsc_changed;
+
+		stream_update_flags_set_full(&context->streams[i]->update_flags);
+		context->streams[i]->update_flags.bits.dsc_changed = prev_dsc_changed;
+	}
+
+	determine_pipe_unlock_order(dc, context);
+	/* Program all planes within new context*/
+	if (dc->res_pool->funcs->prepare_mcache_programming)
+		dc->res_pool->funcs->prepare_mcache_programming(dc, context);
+	if (dc->hwss.program_front_end_for_ctx) {
+		dc->hwss.interdependent_update_lock(dc, context, true);
+		dc->hwss.program_front_end_for_ctx(dc, context);
+
+		if (dc->hwseq->funcs.set_wait_for_update_needed_for_pipe) {
+			for (i = 0; i < dc->res_pool->pipe_count; i++) {
+				pipe = &context->res_ctx.pipe_ctx[i];
+				dc->hwseq->funcs.set_wait_for_update_needed_for_pipe(dc, pipe);
+			}
+		}
+
+		dc->hwss.interdependent_update_lock(dc, context, false);
+		dc->hwss.post_unlock_program_front_end(dc, context);
+	}
+
+	if (dc->hwss.commit_subvp_config)
+		dc->hwss.commit_subvp_config(dc, context);
+	if (dc->hwss.subvp_pipe_control_lock)
+		dc->hwss.subvp_pipe_control_lock(dc, context, false, true, NULL, subvp_prev_use);
+	if (dc->hwss.dmub_hw_control_lock)
+		dc->hwss.dmub_hw_control_lock(dc, context, false);
+
+	for (i = 0; i < context->stream_count; i++) {
+		const struct dc_link *link = context->streams[i]->link;
+
+		if (!context->streams[i]->mode_changed)
+			continue;
+
+		if (dc->hwss.apply_ctx_for_surface) {
+			apply_ctx_interdependent_lock(dc, context, context->streams[i], true);
+			dc->hwss.apply_ctx_for_surface(
+					dc, context->streams[i],
+					context->stream_status[i].plane_count,
+					context);
+			apply_ctx_interdependent_lock(dc, context, context->streams[i], false);
+			dc->hwss.post_unlock_program_front_end(dc, context);
+		}
+
+		/*
+		 * enable stereo
+		 * TODO rework dc_enable_stereo call to work with validation sets?
+		 */
+		for (k = 0; k < MAX_PIPES; k++) {
+			pipe = &context->res_ctx.pipe_ctx[k];
+
+			for (l = 0 ; pipe && l < context->stream_count; l++)  {
+				if (context->streams[l] &&
+					context->streams[l] == pipe->stream &&
+					dc->hwss.setup_stereo)
+					dc->hwss.setup_stereo(pipe, dc);
+			}
+		}
+
+		CONN_MSG_MODE(link, "{%dx%d, %dx%d@%dKhz}",
+				context->streams[i]->timing.h_addressable,
+				context->streams[i]->timing.v_addressable,
+				context->streams[i]->timing.h_total,
+				context->streams[i]->timing.v_total,
+				context->streams[i]->timing.pix_clk_100hz / 10);
+	}
+
+	dc_enable_stereo(dc, context, dc_streams, context->stream_count);
+
+	if (get_seamless_boot_stream_count(context) == 0 ||
+		context->stream_count == 0) {
+		/* Must wait for no flips to be pending before doing optimize bw */
+		hwss_wait_for_no_pipes_pending(dc, context);
+		/*
+		 * optimized dispclk depends on ODM setup. Need to wait for ODM
+		 * update pending complete before optimizing bandwidth.
+		 */
+		hwss_wait_for_odm_update_pending_complete(dc, context);
+		/* pplib is notified if disp_num changed */
+		dc->hwss.optimize_bandwidth(dc, context);
+		/* Need to do otg sync again as otg could be out of sync due to otg
+		 * workaround applied during clock update
+		 */
+		dc_trigger_sync(dc, context);
+	}
+
+	if (dc->hwss.update_dsc_pg)
+		dc->hwss.update_dsc_pg(dc, context, true);
+
+	if (dc->ctx->dce_version >= DCE_VERSION_MAX)
+		TRACE_DCN_CLOCK_STATE(&context->bw_ctx.bw.dcn.clk);
+	else
+		TRACE_DCE_CLOCK_STATE(&context->bw_ctx.bw.dce);
+
+	context->stream_mask = get_stream_mask(dc, context);
+
+	if (context->stream_mask != dc->current_state->stream_mask)
+		dc_dmub_srv_notify_stream_mask(dc->ctx->dmub_srv, context->stream_mask);
+
+	for (i = 0; i < context->stream_count; i++)
+		context->streams[i]->mode_changed = false;
+
+	/* Clear update flags that were set earlier to avoid redundant programming */
+	for (i = 0; i < context->stream_count; i++) {
+		stream_update_flags_clear(&context->streams[i]->update_flags);
+	}
+
+	old_state = dc->current_state;
+	dc->current_state = context;
+
+	dc_state_release(old_state);
+
+	dc_state_retain(dc->current_state);
+
+	return result;
+}
+
+static bool commit_minimal_transition_state(struct dc *dc,
+		struct dc_state *transition_base_context);
+
+/**
+ * dc_commit_streams - Commit current stream state
+ *
+ * @dc: DC object with the commit state to be configured in the hardware
+ * @params: Parameters for the commit, including the streams to be committed
+ *
+ * Function responsible for commit streams change to the hardware.
+ *
+ * Return:
+ * Return DC_OK if everything work as expected, otherwise, return a dc_status
+ * code.
+ */
+enum dc_status dc_commit_streams(struct dc *dc, struct dc_commit_streams_params *params)
+{
+	unsigned int i, j;
+	struct dc_state *context;
+	enum dc_status res = DC_OK;
+	struct dc_validation_set set[MAX_STREAMS] = {0};
+	struct pipe_ctx *pipe;
+	bool handle_exit_odm2to1 = false;
+
+	if (!params)
+		return DC_ERROR_UNEXPECTED;
+
+	if (dc->ctx->dce_environment == DCE_ENV_VIRTUAL_HW)
+		return res;
+
+	if (!streams_changed(dc, params->streams, params->stream_count) &&
+			dc->current_state->power_source == params->power_source)
+		return res;
+
+	dc_exit_ips_for_hw_access(dc);
+
+	DC_LOG_DC("%s: %d streams\n", __func__, params->stream_count);
+
+	for (i = 0; i < params->stream_count; i++) {
+		struct dc_stream_state *stream = params->streams[i];
+		struct dc_stream_status *status = dc_stream_get_status(stream);
+		struct dc_sink *sink = stream->sink;
+
+		/* revalidate streams */
+		if (!dc_is_virtual_signal(sink->sink_signal)) {
+			res = dc_validate_stream(dc, stream);
+			if (res != DC_OK)
+				return res;
+		}
+
+
+		dc_stream_log(dc, stream);
+
+		set[i].stream = stream;
+
+		if (status) {
+			set[i].plane_count = (uint8_t)status->plane_count;
+			for (j = 0; j < (unsigned int)status->plane_count; j++)
+				set[i].plane_states[j] = status->plane_states[j];
+		}
+	}
+
+	/* ODM Combine 2:1 power optimization is only applied for single stream
+	 * scenario, it uses extra pipes than needed to reduce power consumption
+	 * We need to switch off this feature to make room for new streams.
+	 */
+	if (params->stream_count > dc->current_state->stream_count &&
+			dc->current_state->stream_count == 1) {
+		for (i = 0; i < dc->res_pool->pipe_count; i++) {
+			pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+			if (pipe->next_odm_pipe)
+				handle_exit_odm2to1 = true;
+		}
+	}
+
+	if (handle_exit_odm2to1)
+		res = commit_minimal_transition_state(dc, dc->current_state);
+
+	context = dc_state_create_current_copy(dc);
+	if (!context)
+		goto context_alloc_fail;
+
+	context->power_source = params->power_source;
+
+	res = dc_validate_with_context(dc, set, params->stream_count, context, DC_VALIDATE_MODE_AND_PROGRAMMING);
+
+	/*
+	 * Only update link encoder to stream assignment after bandwidth validation passed.
+	 */
+	if (res == DC_OK && dc->res_pool->funcs->link_encs_assign && !dc->config.unify_link_enc_assignment)
+		dc->res_pool->funcs->link_encs_assign(
+			dc, context, context->streams, context->stream_count);
+
+	if (res != DC_OK) {
+		BREAK_TO_DEBUGGER();
+		goto fail;
+	}
+
+	/*
+	 * If not already seamless, make transition seamless by inserting intermediate minimal transition
+	 */
+	if (dc->hwss.is_pipe_topology_transition_seamless &&
+			!dc->hwss.is_pipe_topology_transition_seamless(dc, dc->current_state, context)) {
+		res = commit_minimal_transition_state(dc, context);
+		if (res != DC_OK) {
+			BREAK_TO_DEBUGGER();
+			goto fail;
+		}
+	}
+
+	res = dc_commit_state_no_check(dc, context);
+
+	for (i = 0; i < params->stream_count; i++) {
+		for (j = 0; j < context->stream_count; j++) {
+			if (params->streams[i]->stream_id == context->streams[j]->stream_id)
+				params->streams[i]->out.otg_offset = (uint8_t)context->stream_status[j].primary_otg_inst;
+
+			if (dc_is_embedded_signal(params->streams[i]->signal)) {
+				struct dc_stream_status *status = dc_state_get_stream_status(context, params->streams[i]);
+
+				if (!status)
+					continue;
+
+				if (dc->hwss.is_abm_supported)
+					status->is_abm_supported = dc->hwss.is_abm_supported(dc, context, params->streams[i]);
+				else
+					status->is_abm_supported = true;
+			}
+		}
+	}
+
+fail:
+	dc_state_release(context);
+
+context_alloc_fail:
+
+	DC_LOG_DC("%s Finished.\n", __func__);
+
+	return res;
+}
+
+bool dc_acquire_release_mpc_3dlut(
+		struct dc *dc, bool acquire,
+		struct dc_stream_state *stream,
+		struct dc_3dlut **lut,
+		struct dc_transfer_func **shaper)
+{
+	unsigned int pipe_idx;
+	bool ret = false;
+	bool found_pipe_idx = false;
+	const struct resource_pool *pool = dc->res_pool;
+	struct resource_context *res_ctx = &dc->current_state->res_ctx;
+	int mpcc_id = 0;
+
+	if (pool && res_ctx) {
+		if (acquire) {
+			/*find pipe idx for the given stream*/
+			for (pipe_idx = 0; pipe_idx < pool->pipe_count; pipe_idx++) {
+				if (res_ctx->pipe_ctx[pipe_idx].stream == stream) {
+					found_pipe_idx = true;
+					mpcc_id = res_ctx->pipe_ctx[pipe_idx].plane_res.hubp->inst;
+					break;
+				}
+			}
+		} else
+			found_pipe_idx = true;/*for release pipe_idx is not required*/
+
+		if (found_pipe_idx) {
+			if (acquire && pool->funcs->acquire_post_bldn_3dlut)
+				ret = pool->funcs->acquire_post_bldn_3dlut(res_ctx, pool, mpcc_id, lut, shaper);
+			else if (!acquire && pool->funcs->release_post_bldn_3dlut)
+				ret = pool->funcs->release_post_bldn_3dlut(res_ctx, pool, lut, shaper);
+		}
+	}
+	return ret;
+}
+
+static bool is_flip_pending_in_pipes(struct dc *dc, struct dc_state *context)
+{
+	int i;
+	struct pipe_ctx *pipe;
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		pipe = &context->res_ctx.pipe_ctx[i];
+
+		// Don't check flip pending on phantom pipes
+		if (!pipe->plane_state || (dc_state_get_pipe_subvp_type(context, pipe) == SUBVP_PHANTOM))
+			continue;
+
+		/* Must set to false to start with, due to OR in update function */
+		pipe->plane_state->status.is_flip_pending = false;
+		dc->hwss.update_pending_status(pipe);
+		if (pipe->plane_state->status.is_flip_pending)
+			return true;
+	}
+	return false;
+}
+
+/* Perform updates here which need to be deferred until next vupdate
+ *
+ * i.e. blnd lut, 3dlut, and shaper lut bypass regs are double buffered
+ * but forcing lut memory to shutdown state is immediate. This causes
+ * single frame corruption as lut gets disabled mid-frame unless shutdown
+ * is deferred until after entering bypass.
+ */
+static void process_deferred_updates(struct dc *dc)
+{
+	int i = 0;
+
+	if (dc->debug.enable_mem_low_power.bits.cm) {
+		ASSERT(dc->dcn_ip->max_num_dpp);
+		for (i = 0; i < dc->dcn_ip->max_num_dpp; i++)
+			if (dc->res_pool->dpps[i]->funcs->dpp_deferred_update)
+				dc->res_pool->dpps[i]->funcs->dpp_deferred_update(dc->res_pool->dpps[i]);
+	}
+}
+
+void dc_post_update_surfaces_to_stream(struct dc *dc)
+{
+	unsigned int i;
+	struct dc_state *context = dc->current_state;
+
+	if ((!dc->optimized_required) || get_seamless_boot_stream_count(context) > 0)
+		return;
+
+	post_surface_trace(dc);
+
+	/*
+	 * Only relevant for DCN behavior where we can guarantee the optimization
+	 * is safe to apply - retain the legacy behavior for DCE.
+	 */
+
+	if (dc->ctx->dce_version < DCE_VERSION_MAX)
+		TRACE_DCE_CLOCK_STATE(&context->bw_ctx.bw.dce);
+	else {
+		TRACE_DCN_CLOCK_STATE(&context->bw_ctx.bw.dcn.clk);
+
+		if (is_flip_pending_in_pipes(dc, context))
+			return;
+
+		for (i = 0; i < dc->res_pool->pipe_count; i++)
+			if (context->res_ctx.pipe_ctx[i].stream == NULL ||
+					context->res_ctx.pipe_ctx[i].plane_state == NULL) {
+				context->res_ctx.pipe_ctx[i].pipe_idx = (uint8_t)i;
+				dc->hwss.disable_plane(dc, context, &context->res_ctx.pipe_ctx[i]);
+			}
+
+		process_deferred_updates(dc);
+
+		dc->hwss.optimize_bandwidth(dc, context);
+
+		if (dc->hwss.update_dsc_pg)
+			dc->hwss.update_dsc_pg(dc, context, true);
+	}
+
+	dc->optimized_required = false;
+}
+
+void dc_get_default_tiling_info(const struct dc *dc, struct dc_tiling_info *tiling_info)
+{
+	if (!dc || !tiling_info)
+		return;
+	if (dc->res_pool && dc->res_pool->funcs && dc->res_pool->funcs->get_default_tiling_info) {
+		dc->res_pool->funcs->get_default_tiling_info(tiling_info);
+		return;
+	}
+}
+
+bool dc_set_generic_gpio_for_stereo(bool enable,
+		struct gpio_service *gpio_service)
+{
+	enum gpio_result gpio_result = GPIO_RESULT_NON_SPECIFIC_ERROR;
+	struct gpio_pin_info pin_info;
+	struct gpio *generic;
+	struct gpio_generic_mux_config *config = kzalloc_obj(struct gpio_generic_mux_config);
+
+	if (!config)
+		return false;
+	pin_info = dal_gpio_get_generic_pin_info(gpio_service, GPIO_ID_GENERIC, 0);
+
+	if (pin_info.mask == 0xFFFFFFFF || pin_info.offset == 0xFFFFFFFF) {
+		kfree(config);
+		return false;
+	} else {
+		generic = dal_gpio_service_create_generic_mux(
+			gpio_service,
+			pin_info.offset,
+			pin_info.mask);
+	}
+
+	if (!generic) {
+		kfree(config);
+		return false;
+	}
+
+	gpio_result = dal_gpio_open(generic, GPIO_MODE_OUTPUT);
+
+	config->enable_output_from_mux = enable;
+	config->mux_select = GPIO_SIGNAL_SOURCE_PASS_THROUGH_STEREO_SYNC;
+
+	if (gpio_result == GPIO_RESULT_OK)
+		gpio_result = dal_mux_setup_config(generic, config);
+
+	if (gpio_result == GPIO_RESULT_OK) {
+		dal_gpio_close(generic);
+		dal_gpio_destroy_generic_mux(&generic);
+		kfree(config);
+		return true;
+	} else {
+		dal_gpio_close(generic);
+		dal_gpio_destroy_generic_mux(&generic);
+		kfree(config);
+		return false;
+	}
+}
+
+static bool is_surface_in_context(
+		const struct dc_state *context,
+		const struct dc_plane_state *plane_state)
+{
+	int j;
+
+	for (j = 0; j < MAX_PIPES; j++) {
+		const struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+		if (plane_state == pipe_ctx->plane_state) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
+static struct surface_update_descriptor get_plane_info_update_type(const struct dc_surface_update *u)
+{
+	struct pipe_update_bits *update_bits = &u->surface->update_bits;
+	struct surface_update_descriptor update_type = { UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_NONE };
+
+	if (!u->plane_info)
+		return update_type;
+
+	// `plane_info` present means at least `STREAM` lock is required
+	elevate_update_type(&update_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+
+	if (u->plane_info->color_space != u->surface->color_space) {
+		update_bits->color_space_change = 1;
+		elevate_update_type(&update_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
+	}
+
+	if (u->plane_info->horizontal_mirror != u->surface->horizontal_mirror) {
+		update_bits->horizontal_mirror_change = 1;
+		elevate_update_type(&update_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
+	}
+
+	if (u->plane_info->rotation != u->surface->rotation) {
+		update_bits->rotation_change = 1;
+		elevate_update_type(&update_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+	}
+
+	if (u->plane_info->format != u->surface->format) {
+		update_bits->pixel_format_change = 1;
+		elevate_update_type(&update_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+	}
+
+	if (u->plane_info->stereo_format != u->surface->stereo_format) {
+		update_bits->stereo_format_change = 1;
+		elevate_update_type(&update_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+	}
+
+	if (u->plane_info->per_pixel_alpha != u->surface->per_pixel_alpha) {
+		update_bits->per_pixel_alpha_change = 1;
+		elevate_update_type(&update_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
+	}
+
+	if (u->plane_info->global_alpha_value != u->surface->global_alpha_value) {
+		update_bits->global_alpha_change = 1;
+		elevate_update_type(&update_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
+	}
+
+	if (u->plane_info->dcc.enable != u->surface->dcc.enable
+			|| u->plane_info->dcc.dcc_ind_blk != u->surface->dcc.dcc_ind_blk
+			|| u->plane_info->dcc.meta_pitch != u->surface->dcc.meta_pitch) {
+		/* During DCC on/off, stutter period is calculated before
+		 * DCC has fully transitioned. This results in incorrect
+		 * stutter period calculation. Triggering a full update will
+		 * recalculate stutter period.
+		 */
+		update_bits->dcc_change = 1;
+		elevate_update_type(&update_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+	}
+
+	if (resource_pixel_format_to_bpp(u->plane_info->format) !=
+			resource_pixel_format_to_bpp(u->surface->format)) {
+		/* different bytes per element will require full bandwidth
+		 * and DML calculation
+		 */
+		update_bits->bpp_change = 1;
+		elevate_update_type(&update_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+	}
+
+	if (u->plane_info->plane_size.surface_pitch != u->surface->plane_size.surface_pitch
+			|| u->plane_info->plane_size.chroma_pitch != u->surface->plane_size.chroma_pitch) {
+		update_bits->plane_size_change = 1;
+		elevate_update_type(&update_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
+	}
+
+	const struct dc_tiling_info *tiling = &u->plane_info->tiling_info;
+
+	if (memcmp(tiling, &u->surface->tiling_info, sizeof(*tiling)) != 0) {
+		update_bits->swizzle_change = 1;
+
+		if (tiling->flags.avoid_full_update_on_tiling_change) {
+			elevate_update_type(&update_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
+		} else {
+			update_bits->bandwidth_change = 1;
+			elevate_update_type(&update_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+		}
+	}
+
+	/* This should be UPDATE_TYPE_FAST if nothing has changed. */
+	return update_type;
+}
+
+static struct surface_update_descriptor get_scaling_info_update_type(
+	const struct dc_check_config *check_config,
+	const struct dc_surface_update *u)
+{
+	struct pipe_update_bits *update_bits = &u->surface->update_bits;
+	struct surface_update_descriptor update_type = { UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_NONE };
+
+	if (!u->scaling_info)
+		return update_type;
+
+	// `scaling_info` present means at least `STREAM` lock is required
+	elevate_update_type(&update_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+
+	if (u->scaling_info->src_rect.width != u->surface->src_rect.width
+			|| u->scaling_info->src_rect.height != u->surface->src_rect.height
+			|| u->scaling_info->dst_rect.width != u->surface->dst_rect.width
+			|| u->scaling_info->dst_rect.height != u->surface->dst_rect.height
+			|| u->scaling_info->clip_rect.width != u->surface->clip_rect.width
+			|| u->scaling_info->clip_rect.height != u->surface->clip_rect.height
+			|| u->scaling_info->scaling_quality.integer_scaling !=
+					u->surface->scaling_quality.integer_scaling) {
+		update_bits->scaling_change = 1;
+		elevate_update_type(&update_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+
+		if (u->scaling_info->src_rect.width > u->surface->src_rect.width
+				|| u->scaling_info->src_rect.height > u->surface->src_rect.height)
+			/* Making src rect bigger requires a bandwidth change */
+			update_bits->clock_change = 1;
+
+		if ((u->scaling_info->dst_rect.width < u->surface->dst_rect.width
+			|| u->scaling_info->dst_rect.height < u->surface->dst_rect.height)
+				&& (u->scaling_info->dst_rect.width < u->surface->src_rect.width
+					|| u->scaling_info->dst_rect.height < u->surface->src_rect.height))
+			/* Making dst rect smaller requires a bandwidth change */
+			update_bits->bandwidth_change = 1;
+
+		if (u->scaling_info->src_rect.width > (int)check_config->max_optimizable_video_width &&
+			(u->scaling_info->clip_rect.width > u->surface->clip_rect.width ||
+			 u->scaling_info->clip_rect.height > u->surface->clip_rect.height))
+			 /* Changing clip size of a large surface may result in MPC slice count change */
+			update_bits->bandwidth_change = 1;
+	}
+
+	if (u->scaling_info->src_rect.x != u->surface->src_rect.x
+			|| u->scaling_info->src_rect.y != u->surface->src_rect.y
+			|| u->scaling_info->clip_rect.x != u->surface->clip_rect.x
+			|| u->scaling_info->clip_rect.y != u->surface->clip_rect.y
+			|| u->scaling_info->dst_rect.x != u->surface->dst_rect.x
+			|| u->scaling_info->dst_rect.y != u->surface->dst_rect.y) {
+		elevate_update_type(&update_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
+		update_bits->position_change = 1;
+	}
+
+	return update_type;
+}
+
+static struct surface_update_descriptor det_surface_update(
+		const struct dc_check_config *check_config,
+		struct dc_surface_update *u)
+{
+	struct surface_update_descriptor overall_type = { UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_NONE };
+	struct pipe_update_bits *update_bits = &u->surface->update_bits;
+
+	if (u->surface->force_full_update) {
+		dc_pipe_update_bits_set_full(update_bits);
+		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+		return overall_type;
+	}
+
+	dc_pipe_update_bits_clear(update_bits);
+
+	struct surface_update_descriptor inner_type = get_plane_info_update_type(u);
+
+	elevate_update_type(&overall_type, inner_type.update_type, inner_type.lock_descriptor);
+
+	inner_type = get_scaling_info_update_type(check_config, u);
+	elevate_update_type(&overall_type, inner_type.update_type, inner_type.lock_descriptor);
+
+	if (u->flip_addr) {
+		update_bits->addr_update = 1;
+		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+
+		if (u->flip_addr->address.tmz_surface != u->surface->address.tmz_surface) {
+			update_bits->tmz_changed = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+		}
+	}
+	if (u->in_transfer_func) {
+		update_bits->in_transfer_func_change = 1;
+		elevate_update_type(&overall_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
+	}
+
+	if (u->input_csc_color_matrix) {
+		update_bits->input_csc_change = 1;
+		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+	}
+
+	if (u->cursor_csc_color_matrix) {
+		update_bits->cursor_csc_color_matrix_change = 1;
+		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+	}
+
+	if (u->coeff_reduction_factor) {
+		update_bits->coeff_reduction_change = 1;
+		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+	}
+
+	if (u->gamut_remap_matrix) {
+		update_bits->gamut_remap_change = 1;
+		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+	}
+
+	if ((u->cm && u->cm->flags.bits.blend_enable) ||
+			(u->gamma && dce_use_lut(u->plane_info ? u->plane_info->format : u->surface->format))) {
+		update_bits->gamma_change = 1;
+		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+	}
+
+	if (u->cm && (u->cm->flags.bits.lut3d_enable || u->cm->flags.bits.shaper_enable)) {
+		update_bits->lut_3d = 1;
+		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+	}
+
+	if (u->cm && u->cm->flags.bits.lut3d_dma_enable != u->surface->cm.flags.bits.lut3d_dma_enable &&
+			u->cm->flags.bits.lut3d_enable && u->surface->cm.flags.bits.lut3d_enable) {
+		/* Toggling 3DLUT loading between DMA and Host is illegal */
+		BREAK_TO_DEBUGGER();
+	}
+
+	if (u->cm && u->cm->flags.bits.lut3d_enable && !u->cm->flags.bits.lut3d_dma_enable) {
+		/* Host loading 3DLUT requires full update but only stream lock  */
+		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_STREAM);
+	}
+
+	if (u->hdr_mult.value)
+		if (u->hdr_mult.value != u->surface->hdr_mult.value) {
+			// TODO: Should be fast?
+			update_bits->hdr_mult = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_MED, LOCK_DESCRIPTOR_STREAM);
+		}
+
+	if (u->sdr_white_level_nits)
+		if (u->sdr_white_level_nits != u->surface->sdr_white_level_nits) {
+			// TODO: Should be fast?
+			update_bits->sdr_white_level_nits = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+		}
+
+	if (u->cm_hist_control) {
+		update_bits->cm_hist_change = 1;
+		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+	}
+
+	if (u->cm) {
+		const union dc_plane_cm_flags blend_only_flags = {
+			.bits = {
+				.blend_enable = 1,
+			}
+		};
+
+		if (u->cm->flags.bits.shaper_enable != u->surface->cm.flags.bits.shaper_enable
+				|| u->cm->flags.bits.blend_enable != u->surface->cm.flags.bits.blend_enable
+				|| u->cm->flags.bits.lut3d_enable != u->surface->cm.flags.bits.lut3d_enable
+				|| u->cm->flags.bits.lut3d_dma_enable != u->surface->cm.flags.bits.lut3d_dma_enable) {
+			update_bits->mcm_transfer_function_enable_change = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+		}
+
+		if ((u->cm->flags.all != blend_only_flags.all && u->cm->flags.all != 0) ||
+				(u->surface->cm.flags.all != blend_only_flags.all && u->surface->cm.flags.all != 0)) {
+			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+		}
+	}
+
+	if (update_bits->lut_3d &&
+			!u->surface->cm.flags.bits.lut3d_dma_enable) {
+		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+	}
+
+	if (check_config->enable_legacy_fast_update &&
+			(update_bits->gamma_change ||
+			update_bits->gamut_remap_change ||
+			update_bits->input_csc_change ||
+			update_bits->cm_hist_change ||
+			update_bits->coeff_reduction_change)) {
+		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+	}
+	return overall_type;
+}
+
+/* May need to flip the desktop plane in cases where MPO plane receives a flip but desktop plane doesn't
+ * while both planes are flip_immediate
+ */
+static void force_immediate_gsl_plane_flip(struct dc *dc, struct dc_surface_update *updates, int surface_count)
+{
+	(void)dc;
+	bool has_flip_immediate_plane = false;
+	int i;
+
+	for (i = 0; i < surface_count; i++) {
+		if (updates[i].surface->flip_immediate) {
+			has_flip_immediate_plane = true;
+			break;
+		}
+	}
+
+	if (has_flip_immediate_plane && surface_count > 1) {
+		for (i = 0; i < surface_count; i++) {
+			if (updates[i].surface->flip_immediate)
+				updates[i].surface->update_bits.addr_update = 1;
+		}
+	}
+}
+
+static struct surface_update_descriptor check_update_surfaces_for_stream(
+		const struct dc_check_config *check_config,
+		struct dc_surface_update *updates,
+		int surface_count,
+		struct dc_stream_update *stream_update)
+{
+	struct surface_update_descriptor overall_type = { UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_NONE };
+
+	/* When countdown finishes, promote this flip to full to trigger deferred final transition */
+	if (check_config->deferred_transition_state && !check_config->transition_countdown_to_steady_state) {
+		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+	}
+
+	if (stream_update && stream_update->pending_test_pattern) {
+		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+	}
+
+	if (stream_update && stream_update->hw_cursor_req) {
+		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+	}
+
+	/* some stream updates require passive update */
+	if (stream_update) {
+		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+
+		union stream_update_flags *su_flags = &stream_update->stream->update_flags;
+
+		if ((stream_update->src.height != 0 && stream_update->src.width != 0) ||
+			(stream_update->dst.height != 0 && stream_update->dst.width != 0) ||
+			stream_update->integer_scaling_update)
+			su_flags->bits.scaling = 1;
+
+		if (check_config->enable_legacy_fast_update && stream_update->out_transfer_func)
+			su_flags->bits.out_tf = 1;
+
+		if (stream_update->abm_level)
+			su_flags->bits.abm_level = 1;
+
+		if (stream_update->dpms_off) {
+			su_flags->bits.dpms_off = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL | LOCK_DESCRIPTOR_LINK);
+		}
+
+		if (check_config->enable_legacy_fast_update && stream_update->gamut_remap)
+			su_flags->bits.gamut_remap = 1;
+
+		if (stream_update->wb_update)
+			su_flags->bits.wb_update = 1;
+
+		if (stream_update->dsc_config)
+			su_flags->bits.dsc_changed = 1;
+
+		if (stream_update->mst_bw_update)
+			su_flags->bits.mst_bw = 1;
+
+		if (stream_update->stream->freesync_on_desktop &&
+			(stream_update->vrr_infopacket || stream_update->allow_freesync ||
+				stream_update->vrr_active_variable || stream_update->vrr_active_fixed))
+			su_flags->bits.fams_changed = 1;
+
+		if (stream_update->scaler_sharpener_update)
+			su_flags->bits.scaler_sharpener = 1;
+
+		if (stream_update->sharpening_required)
+			su_flags->bits.sharpening_required = 1;
+
+		if (stream_update->output_color_space)
+			su_flags->bits.out_csc = 1;
+
+		// TODO: Make each elevation explicit, as to not override fast stream in crct_timing_adjust
+		if (su_flags->raw)
+			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+
+		// Non-global cases
+
+		if (stream_update->gamut_remap) {
+			su_flags->bits.gamut_remap = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+		}
+
+		if ((stream_update->hdr_static_metadata && !stream_update->stream->use_dynamic_meta) ||
+				stream_update->vrr_infopacket ||
+				stream_update->vsc_infopacket ||
+				stream_update->vsp_infopacket ||
+				stream_update->hfvsif_infopacket ||
+				stream_update->adaptive_sync_infopacket ||
+				stream_update->vtem_infopacket ||
+				stream_update->avi_infopacket) {
+			su_flags->bits.info_frame = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+		}
+
+		if (stream_update->hdr_static_metadata && stream_update->stream->use_dynamic_meta) {
+			su_flags->bits.dmdata = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+		}
+
+		if (stream_update->output_csc_transform) {
+			su_flags->bits.out_csc = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+		}
+
+		if (!check_config->enable_legacy_fast_update && stream_update->out_transfer_func) {
+			su_flags->bits.out_tf = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+		}
+
+		if (stream_update->periodic_interrupt) {
+			su_flags->bits.periodic_interrupt = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+		}
+
+		if (stream_update->dither_option) {
+			su_flags->bits.dither = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+		}
+
+		if (stream_update->cursor_attributes) {
+			su_flags->bits.cursor_attr = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+		}
+
+		if (stream_update->cursor_position) {
+			su_flags->bits.cursor_pos = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+		}
+	}
+
+	for (int i = 0 ; i < surface_count; i++) {
+		struct surface_update_descriptor inner_type =
+				det_surface_update(check_config, &updates[i]);
+
+		elevate_update_type(&overall_type, inner_type.update_type, inner_type.lock_descriptor);
+	}
+
+	return overall_type;
+}
+
+/*
+ * dc_check_update_surfaces_for_stream() - Determine update type (fast, med, or full)
+ *
+ * See :c:type:`enum surface_update_type <surface_update_type>` for explanation of update types
+ */
+struct surface_update_descriptor dc_check_update_surfaces_for_stream(
+		const struct dc_check_config *check_config,
+		struct dc_surface_update *updates,
+		int surface_count,
+		struct dc_stream_update *stream_update)
+{
+	if (stream_update)
+		stream_update_flags_clear(&stream_update->stream->update_flags);
+	for (int i = 0; i < surface_count; i++)
+		dc_pipe_update_bits_clear(&updates[i].surface->update_bits);
+
+	return check_update_surfaces_for_stream(check_config, updates, surface_count, stream_update);
+}
+
+static struct dc_stream_status *stream_get_status(
+	struct dc_state *ctx,
+	struct dc_stream_state *stream)
+{
+	uint8_t i;
+
+	for (i = 0; i < ctx->stream_count; i++) {
+		if (stream == ctx->streams[i]) {
+			return &ctx->stream_status[i];
+		}
+	}
+
+	return NULL;
+}
+
+static const enum surface_update_type update_surface_trace_level = UPDATE_TYPE_FULL;
+
+static void copy_surface_update_to_plane(
+		struct dc_plane_state *surface,
+		struct dc_surface_update *srf_update)
+{
+	if (srf_update->flip_addr) {
+		surface->address = srf_update->flip_addr->address;
+		surface->flip_immediate =
+			srf_update->flip_addr->flip_immediate;
+		surface->time.time_elapsed_in_us[surface->time.index] =
+			(unsigned int)(srf_update->flip_addr->flip_timestamp_in_us -
+				surface->time.prev_update_time_in_us);
+		surface->time.prev_update_time_in_us =
+			(unsigned int)srf_update->flip_addr->flip_timestamp_in_us;
+		surface->time.index++;
+		if (surface->time.index >= DC_PLANE_UPDATE_TIMES_MAX)
+			surface->time.index = 0;
+
+		surface->triplebuffer_flips = srf_update->flip_addr->triplebuffer_flips;
+	}
+
+	if (srf_update->scaling_info) {
+		surface->scaling_quality =
+				srf_update->scaling_info->scaling_quality;
+		surface->dst_rect =
+				srf_update->scaling_info->dst_rect;
+		surface->src_rect =
+				srf_update->scaling_info->src_rect;
+		surface->clip_rect =
+				srf_update->scaling_info->clip_rect;
+	}
+
+	if (srf_update->plane_info) {
+		surface->color_space =
+				srf_update->plane_info->color_space;
+		surface->format =
+				srf_update->plane_info->format;
+		surface->plane_size =
+				srf_update->plane_info->plane_size;
+		surface->rotation =
+				srf_update->plane_info->rotation;
+		surface->horizontal_mirror =
+				srf_update->plane_info->horizontal_mirror;
+		surface->stereo_format =
+				srf_update->plane_info->stereo_format;
+		surface->tiling_info =
+				srf_update->plane_info->tiling_info;
+		surface->visible =
+				srf_update->plane_info->visible;
+		surface->per_pixel_alpha =
+				srf_update->plane_info->per_pixel_alpha;
+		surface->global_alpha =
+				srf_update->plane_info->global_alpha;
+		surface->global_alpha_value =
+				srf_update->plane_info->global_alpha_value;
+		surface->dcc =
+				srf_update->plane_info->dcc;
+		surface->layer_index =
+				srf_update->plane_info->layer_index;
+	}
+
+	if (srf_update->gamma) {
+		memcpy(&surface->gamma_correction.entries,
+			&srf_update->gamma->entries,
+			sizeof(struct dc_gamma_entries));
+		surface->gamma_correction.is_identity =
+			srf_update->gamma->is_identity;
+		surface->gamma_correction.num_entries =
+			srf_update->gamma->num_entries;
+		surface->gamma_correction.type =
+			srf_update->gamma->type;
+	}
+	if (srf_update->cm_hist_control) {
+		memcpy(&surface->cm_hist_control,
+			srf_update->cm_hist_control,
+			sizeof(surface->cm_hist_control));
+	}
+
+	if (srf_update->in_transfer_func) {
+		surface->in_transfer_func.sdr_ref_white_level =
+			srf_update->in_transfer_func->sdr_ref_white_level;
+		surface->in_transfer_func.tf =
+			srf_update->in_transfer_func->tf;
+		surface->in_transfer_func.type =
+			srf_update->in_transfer_func->type;
+		memcpy(&surface->in_transfer_func.tf_pts,
+			&srf_update->in_transfer_func->tf_pts,
+			sizeof(struct dc_transfer_func_distributed_points));
+	}
+
+	/* Shaper, 3DLUT, 1DLUT */
+	if (srf_update->cm) {
+		struct kref refcount = surface->cm.refcount;
+
+		memcpy(&surface->cm, srf_update->cm, sizeof(surface->cm));
+		surface->cm.refcount = refcount;
+
+#ifndef TRIM_CM2
+		/* Populate mcm_luts from cm for legacy consumers (dml2, hwseq) */
+		surface->mcm_luts.lut1d_func = &surface->cm.blend_func;
+		surface->mcm_luts.shaper = &surface->cm.shaper_func;
+		if (srf_update->cm->flags.bits.lut3d_dma_enable) {
+			surface->mcm_luts.lut3d_data.lut3d_src = DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM;
+			surface->mcm_luts.lut3d_data.gpu_mem_params.addr = surface->cm.lut3d_dma.addr;
+			surface->mcm_luts.lut3d_data.gpu_mem_params.layout =
+				(surface->cm.lut3d_dma.swizzle == CM_LUT_3D_SWIZZLE_LINEAR_RGB) ?
+					DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_RGB :
+				(surface->cm.lut3d_dma.swizzle == CM_LUT_3D_SWIZZLE_LINEAR_BGR) ?
+					DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_BGR :
+					DC_CM2_GPU_MEM_LAYOUT_1D_PACKED_LINEAR;
+			surface->mcm_luts.lut3d_data.gpu_mem_params.format_params.format =
+				(surface->cm.lut3d_dma.format == CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB) ?
+					DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12MSB :
+				(surface->cm.lut3d_dma.format == CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB) ?
+					DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12LSB :
+					DC_CM2_GPU_MEM_FORMAT_16161616_FLOAT_FP1_5_10;
+			surface->mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.bias =
+				surface->cm.lut3d_dma.bias;
+			surface->mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.scale =
+				surface->cm.lut3d_dma.scale;
+			surface->mcm_luts.lut3d_data.gpu_mem_params.component_order =
+				DC_CM2_GPU_MEM_PIXEL_COMPONENT_ORDER_RGBA;
+			surface->mcm_luts.lut3d_data.gpu_mem_params.size = DC_CM2_GPU_MEM_SIZE_TRANSFORMED;
+			surface->mcm_luts.lut3d_data.mpc_3dlut_enable = (srf_update->cm->flags.bits.lut3d_enable != 0);
+		} else {
+			surface->mcm_luts.lut3d_data.lut3d_src = DC_CM2_TRANSFER_FUNC_SOURCE_SYSMEM;
+			surface->mcm_luts.lut3d_data.lut3d_func = &surface->cm.lut3d_func;
+		}
+
+		if (srf_update->cm->flags.bits.shaper_enable &&
+				srf_update->cm->flags.bits.lut3d_enable)
+			surface->mcm_shaper_3dlut_setting = DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER_3DLUT;
+		else if (srf_update->cm->flags.bits.shaper_enable)
+			surface->mcm_shaper_3dlut_setting = DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER;
+		else
+			surface->mcm_shaper_3dlut_setting = DC_CM2_SHAPER_3DLUT_SETTING_BYPASS_ALL;
+#endif /* TRIM_CM2 */
+	}
+
+	if (srf_update->hdr_mult.value)
+		surface->hdr_mult =
+				srf_update->hdr_mult;
+
+	if (srf_update->sdr_white_level_nits)
+		surface->sdr_white_level_nits =
+				srf_update->sdr_white_level_nits;
+
+	if (srf_update->cm &&
+			(srf_update->cm->flags.bits.blend_enable ||
+			srf_update->cm->flags.bits.shaper_enable ||
+			srf_update->cm->flags.bits.lut3d_enable))
+		surface->lut_bank_a = !surface->lut_bank_a;
+
+	if (srf_update->input_csc_color_matrix)
+		surface->input_csc_color_matrix =
+			*srf_update->input_csc_color_matrix;
+
+	if (srf_update->coeff_reduction_factor)
+		surface->coeff_reduction_factor =
+			*srf_update->coeff_reduction_factor;
+
+	if (srf_update->gamut_remap_matrix)
+		surface->gamut_remap_matrix =
+			*srf_update->gamut_remap_matrix;
+
+	if (srf_update->cursor_csc_color_matrix)
+		surface->cursor_csc_color_matrix =
+			*srf_update->cursor_csc_color_matrix;
+
+	if (srf_update->bias_and_scale.bias_and_scale_valid)
+			surface->bias_and_scale =
+					srf_update->bias_and_scale;
+}
+
+static void copy_stream_update_to_stream(struct dc *dc,
+					 struct dc_state *context,
+					 struct dc_stream_state *stream,
+					 struct dc_stream_update *update)
+{
+	(void)context;
+	struct dc_context *dc_ctx = dc->ctx;
+
+	if (update == NULL || stream == NULL)
+		return;
+
+	if (update->src.height && update->src.width)
+		stream->src = update->src;
+
+	if (update->dst.height && update->dst.width)
+		stream->dst = update->dst;
+
+	if (update->out_transfer_func) {
+		stream->out_transfer_func.sdr_ref_white_level =
+			update->out_transfer_func->sdr_ref_white_level;
+		stream->out_transfer_func.tf = update->out_transfer_func->tf;
+		stream->out_transfer_func.type =
+			update->out_transfer_func->type;
+		memcpy(&stream->out_transfer_func.tf_pts,
+		       &update->out_transfer_func->tf_pts,
+		       sizeof(struct dc_transfer_func_distributed_points));
+	}
+
+	if (update->hdr_static_metadata)
+		stream->hdr_static_metadata = *update->hdr_static_metadata;
+
+	if (update->abm_level)
+		stream->abm_level = *update->abm_level;
+
+	if (update->periodic_interrupt)
+		stream->periodic_interrupt = *update->periodic_interrupt;
+
+	if (update->gamut_remap)
+		stream->gamut_remap_matrix = *update->gamut_remap;
+
+	/* Note: this being updated after mode set is currently not a use case
+	 * however if it arises OCSC would need to be reprogrammed at the
+	 * minimum
+	 */
+	if (update->output_color_space)
+		stream->output_color_space = *update->output_color_space;
+
+	if (update->output_csc_transform)
+		stream->csc_color_matrix = *update->output_csc_transform;
+
+	if (update->vrr_infopacket)
+		stream->vrr_infopacket = *update->vrr_infopacket;
+
+	if (update->hw_cursor_req)
+		stream->hw_cursor_req = *update->hw_cursor_req;
+
+	if (update->allow_freesync)
+		stream->allow_freesync = *update->allow_freesync;
+
+	if (update->vrr_active_variable)
+		stream->vrr_active_variable = *update->vrr_active_variable;
+
+	if (update->vrr_active_fixed)
+		stream->vrr_active_fixed = *update->vrr_active_fixed;
+
+	if (update->crtc_timing_adjust) {
+		if (stream->adjust.v_total_min != update->crtc_timing_adjust->v_total_min ||
+			stream->adjust.v_total_max != update->crtc_timing_adjust->v_total_max ||
+			stream->adjust.timing_adjust_pending)
+			update->crtc_timing_adjust->timing_adjust_pending = true;
+		stream->adjust = *update->crtc_timing_adjust;
+		update->crtc_timing_adjust->timing_adjust_pending = false;
+	}
+
+	if (update->dpms_off)
+		stream->dpms_off = *update->dpms_off;
+
+	if (update->hfvsif_infopacket)
+		stream->hfvsif_infopacket = *update->hfvsif_infopacket;
+
+	if (update->vtem_infopacket)
+		stream->vtem_infopacket = *update->vtem_infopacket;
+
+	if (update->vsc_infopacket)
+		stream->vsc_infopacket = *update->vsc_infopacket;
+
+	if (update->vsp_infopacket)
+		stream->vsp_infopacket = *update->vsp_infopacket;
+
+	if (update->adaptive_sync_infopacket)
+		stream->adaptive_sync_infopacket = *update->adaptive_sync_infopacket;
+
+	if (update->avi_infopacket)
+		stream->avi_infopacket = *update->avi_infopacket;
+
+	if (update->dither_option)
+		stream->dither_option = *update->dither_option;
+
+	if (update->pending_test_pattern)
+		stream->test_pattern = *update->pending_test_pattern;
+	/* update current stream with writeback info */
+	if (update->wb_update) {
+		unsigned int i;
+
+		stream->num_wb_info = update->wb_update->num_wb_info;
+		ASSERT(stream->num_wb_info <= MAX_DWB_PIPES);
+		for (i = 0; i < stream->num_wb_info; i++)
+			stream->writeback_info[i] =
+				update->wb_update->writeback_info[i];
+	}
+	if (update->dsc_config) {
+		struct dc_dsc_config old_dsc_cfg = stream->timing.dsc_cfg;
+		uint32_t old_dsc_enabled = stream->timing.flags.DSC;
+		uint32_t enable_dsc = (update->dsc_config->num_slices_h != 0 &&
+				       update->dsc_config->num_slices_v != 0);
+
+		/* Use temporarry context for validating new DSC config */
+		struct dc_state *dsc_validate_context = dc_state_create_copy(dc->current_state);
+
+		if (dsc_validate_context) {
+			stream->timing.dsc_cfg = *update->dsc_config;
+			stream->timing.flags.DSC = enable_dsc;
+			if (dc->res_pool->funcs->validate_bandwidth(dc, dsc_validate_context,
+				DC_VALIDATE_MODE_ONLY) != DC_OK) {
+				stream->timing.dsc_cfg = old_dsc_cfg;
+				stream->timing.flags.DSC = old_dsc_enabled;
+				update->dsc_config = NULL;
+			}
+
+			dc_state_release(dsc_validate_context);
+		} else {
+			DC_ERROR("Failed to allocate new validate context for DSC change\n");
+			update->dsc_config = NULL;
+		}
+	}
+	if (update->scaler_sharpener_update)
+		stream->scaler_sharpener_update = *update->scaler_sharpener_update;
+	if (update->sharpening_required)
+		stream->sharpening_required = *update->sharpening_required;
+
+	if (update->drr_trigger_mode) {
+		stream->drr_trigger_mode = *update->drr_trigger_mode;
+	}
+}
+
+static void backup_planes_and_stream_state(
+		struct dc_scratch_space *scratch,
+		struct dc_stream_state *stream)
+{
+	int i;
+	struct dc_stream_status *status = dc_stream_get_status(stream);
+
+	if (!status)
+		return;
+
+	for (i = 0; i < status->plane_count; i++) {
+		dc_plane_copy_config(&scratch->plane_states[i], status->plane_states[i]);
+	}
+	scratch->stream_state = *stream;
+}
+
+static void restore_planes_and_stream_state(
+		struct dc_scratch_space *scratch,
+		struct dc_stream_state *stream)
+{
+	int i;
+	struct dc_stream_status *status = dc_stream_get_status(stream);
+
+	if (!status)
+		return;
+
+	for (i = 0; i < status->plane_count; i++) {
+		dc_plane_copy_config(status->plane_states[i], &scratch->plane_states[i]);
+	}
+
+	// refcount is persistent
+	struct kref temp_refcount = stream->refcount;
+	*stream = scratch->stream_state;
+	stream->refcount = temp_refcount;
+}
+
+/**
+ * update_seamless_boot_flags() - Helper function for updating seamless boot flags
+ *
+ * @dc: Current DC state
+ * @context: New DC state to be programmed
+ * @surface_count: Number of surfaces that have an updated
+ * @stream: Corresponding stream to be updated in the current flip
+ *
+ * Updating seamless boot flags do not need to be part of the commit sequence. This
+ * helper function will update the seamless boot flags on each flip (if required)
+ * outside of the HW commit sequence (fast or slow).
+ *
+ * Return: void
+ */
+static void update_seamless_boot_flags(struct dc *dc,
+		struct dc_state *context,
+		int surface_count,
+		struct dc_stream_state *stream)
+{
+	if (get_seamless_boot_stream_count(context) > 0 && (surface_count > 0 || stream->dpms_off)) {
+		/* Optimize seamless boot flag keeps clocks and watermarks high until
+		 * first flip. After first flip, optimization is required to lower
+		 * bandwidth. Important to note that it is expected UEFI will
+		 * only light up a single display on POST, therefore we only expect
+		 * one stream with seamless boot flag set.
+		 */
+		if (stream->apply_seamless_boot_optimization) {
+			stream->apply_seamless_boot_optimization = false;
+
+			if (get_seamless_boot_stream_count(context) == 0)
+				dc->optimized_required = true;
+		}
+	}
+}
+
+static bool full_update_required_weak(
+		const struct dc *dc,
+		const struct dc_surface_update *srf_updates,
+		int surface_count,
+		const struct dc_stream_update *stream_update,
+		const struct dc_stream_state *stream);
+
+struct pipe_split_policy_backup {
+	bool dynamic_odm_policy;
+	bool subvp_policy;
+	enum pipe_split_policy mpc_policy;
+	char force_odm[MAX_PIPES];
+};
+
+static void backup_and_set_minimal_pipe_split_policy(struct dc *dc,
+		struct dc_state *context,
+		struct pipe_split_policy_backup *policy)
+{
+	int i;
+
+	if (!dc->config.is_vmin_only_asic) {
+		policy->mpc_policy = dc->debug.pipe_split_policy;
+		dc->debug.pipe_split_policy = MPC_SPLIT_AVOID;
+	}
+	policy->dynamic_odm_policy = dc->debug.enable_single_display_2to1_odm_policy;
+	dc->debug.enable_single_display_2to1_odm_policy = false;
+	policy->subvp_policy = dc->debug.force_disable_subvp;
+	dc->debug.force_disable_subvp = true;
+	for (i = 0; i < context->stream_count; i++) {
+		policy->force_odm[i] = context->streams[i]->debug.force_odm_combine_segments;
+		if (context->streams[i]->debug.allow_transition_for_forced_odm)
+			context->streams[i]->debug.force_odm_combine_segments = 0;
+	}
+}
+
+static void restore_minimal_pipe_split_policy(struct dc *dc,
+		struct dc_state *context,
+		struct pipe_split_policy_backup *policy)
+{
+	uint8_t i;
+
+	if (!dc->config.is_vmin_only_asic)
+		dc->debug.pipe_split_policy = policy->mpc_policy;
+	dc->debug.enable_single_display_2to1_odm_policy =
+			policy->dynamic_odm_policy;
+	dc->debug.force_disable_subvp = policy->subvp_policy;
+	for (i = 0; i < context->stream_count; i++)
+		context->streams[i]->debug.force_odm_combine_segments = policy->force_odm[i];
+}
+
+/**
+ * update_planes_and_stream_state() - The function takes planes and stream
+ * updates as inputs and determines the appropriate update type. If update type
+ * is FULL, the function allocates a new context, populates and validates it.
+ * Otherwise, it updates current dc context. The function will return both
+ * new_context and new_update_type back to the caller. The function also backs
+ * up both current and new contexts into corresponding dc state scratch memory.
+ * TODO: The function does too many things, and even conditionally allocates dc
+ * context memory implicitly. We should consider to break it down.
+ *
+ * @dc: Current DC state
+ * @srf_updates: an array of surface updates
+ * @surface_count: surface update count
+ * @stream: Corresponding stream to be updated
+ * @stream_update: stream update
+ * @update_descriptor: describes what plane and stream changes to apply
+ * @new_update_type: [out] determined update type by the function
+ * @new_context: [out] new context allocated and validated if update type is
+ * FULL, reference to current context if update type is less than FULL.
+ *
+ * Return: true if a valid update is populated into new_context, false
+ * otherwise.
+ */
+static bool update_planes_and_stream_state(struct dc *dc,
+		struct dc_surface_update *srf_updates, int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update,
+		enum surface_update_type *new_update_type,
+		struct dc_state **new_context)
+{
+	struct dc_state *context;
+	int i;
+	unsigned int j;
+	enum surface_update_type update_type;
+	const struct dc_stream_status *stream_status;
+	struct dc_context *dc_ctx = dc->ctx;
+
+	stream_status = dc_stream_get_status(stream);
+
+	if (!stream_status) {
+		if (surface_count) /* Only an error condition if surf_count non-zero*/
+			ASSERT(false);
+
+		return false; /* Cannot commit surface to stream that is not committed */
+	}
+
+	context = dc->current_state;
+	update_type = dc_check_update_surfaces_for_stream(
+			&dc->check_config, srf_updates, surface_count, stream_update).update_type;
+	if (full_update_required_weak(dc, srf_updates, surface_count, stream_update, stream))
+		update_type = UPDATE_TYPE_FULL;
+
+	/* It is possible to receive a flip for one plane while there are multiple flip_immediate planes in the same stream.
+	 * E.g. Desktop and MPO plane are flip_immediate but only the MPO plane received a flip
+	 * Force the other flip_immediate planes to flip so GSL doesn't wait for a flip that won't come.
+	 */
+	force_immediate_gsl_plane_flip(dc, srf_updates, surface_count);
+	if (update_type == UPDATE_TYPE_FULL)
+		backup_planes_and_stream_state(&dc->scratch.current_state, stream);
+
+	/* update current stream with the new updates */
+	copy_stream_update_to_stream(dc, context, stream, stream_update);
+
+	/* do not perform surface update if surface has invalid dimensions
+	 * (all zero) and no scaling_info is provided
+	 */
+	if (surface_count > 0) {
+		for (i = 0; i < surface_count; i++) {
+			if ((srf_updates[i].surface->src_rect.width == 0 ||
+				 srf_updates[i].surface->src_rect.height == 0 ||
+				 srf_updates[i].surface->dst_rect.width == 0 ||
+				 srf_updates[i].surface->dst_rect.height == 0) &&
+				(!srf_updates[i].scaling_info ||
+				  srf_updates[i].scaling_info->src_rect.width == 0 ||
+				  srf_updates[i].scaling_info->src_rect.height == 0 ||
+				  srf_updates[i].scaling_info->dst_rect.width == 0 ||
+				  srf_updates[i].scaling_info->dst_rect.height == 0)) {
+				DC_ERROR("Invalid src/dst rects in surface update!\n");
+				return false;
+			}
+		}
+	}
+
+	if (update_type == UPDATE_TYPE_FULL) {
+		if (stream_update) {
+			uint32_t dsc_changed = stream_update->stream->update_flags.bits.dsc_changed;
+			stream_update_flags_set_full(&stream_update->stream->update_flags);
+			stream_update->stream->update_flags.bits.dsc_changed = dsc_changed;
+		}
+		for (i = 0; i < surface_count; i++)
+			dc_pipe_update_bits_set_full(&srf_updates[i].surface->update_bits);
+	}
+
+	if (update_type >= update_surface_trace_level)
+		update_surface_trace(dc, srf_updates, surface_count);
+
+	for (i = 0; i < surface_count; i++)
+		copy_surface_update_to_plane(srf_updates[i].surface, &srf_updates[i]);
+
+	if (update_type >= UPDATE_TYPE_FULL) {
+		struct dc_plane_state *new_planes[MAX_SURFACES] = {0};
+
+		for (i = 0; i < surface_count; i++)
+			new_planes[i] = srf_updates[i].surface;
+
+		/* initialize scratch memory for building context */
+		context = dc_state_create_copy(dc->current_state);
+		if (context == NULL) {
+			DC_ERROR("Failed to allocate new validate context!\n");
+			return false;
+		}
+
+		/* For each full update, remove all existing phantom pipes first.
+		 * Ensures that we have enough pipes for newly added MPO planes
+		 */
+		dc_state_remove_phantom_streams_and_planes(dc, context);
+		dc_state_release_phantom_streams_and_planes(dc, context);
+
+		/*remove old surfaces from context */
+		if (!dc_state_rem_all_planes_for_stream(dc, stream, context)) {
+
+			BREAK_TO_DEBUGGER();
+			goto fail;
+		}
+
+		/* add surface to context */
+		if (!dc_state_add_all_planes_for_stream(dc, stream, new_planes, surface_count, context)) {
+
+			BREAK_TO_DEBUGGER();
+			goto fail;
+		}
+	}
+
+	/* save update parameters into surface */
+	for (i = 0; i < surface_count; i++) {
+		struct dc_plane_state *surface = srf_updates[i].surface;
+
+		if (update_type != UPDATE_TYPE_MED)
+			continue;
+		if (surface->update_bits.position_change) {
+			for (j = 0; j < dc->res_pool->pipe_count; j++) {
+				struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+				if (pipe_ctx->plane_state != surface)
+					continue;
+
+				resource_build_scaling_params(pipe_ctx);
+			}
+		}
+	}
+
+	if (update_type == UPDATE_TYPE_FULL) {
+		struct pipe_split_policy_backup policy;
+		bool minimize = false;
+
+		if (dc->check_config.deferred_transition_state) {
+			if (dc->check_config.transition_countdown_to_steady_state) {
+				/* During countdown, all new contexts created as minimal transition states */
+				minimize = true;
+			} else {
+				dc->check_config.deferred_transition_state = false;
+			}
+		}
+
+		if (minimize)
+			backup_and_set_minimal_pipe_split_policy(dc, context, &policy);
+
+		if (dc->res_pool->funcs->validate_bandwidth(dc, context, DC_VALIDATE_MODE_AND_PROGRAMMING) != DC_OK) {
+			if (minimize)
+				restore_minimal_pipe_split_policy(dc, context, &policy);
+			BREAK_TO_DEBUGGER();
+			goto fail;
+		}
+
+		if (minimize)
+			restore_minimal_pipe_split_policy(dc, context, &policy);
+	}
+	update_seamless_boot_flags(dc, context, surface_count, stream);
+
+	*new_context = context;
+	*new_update_type = update_type;
+	if (update_type == UPDATE_TYPE_FULL)
+		backup_planes_and_stream_state(&dc->scratch.new_state, stream);
+
+	return true;
+
+fail:
+	dc_state_release(context);
+
+	return false;
+
+}
+
+static void program_cursor_attributes_sequence(
+		struct dc *dc,
+		struct dc_stream_state *stream,
+		struct dc_state *context,
+		struct block_sequence_state *seq_state)
+{
+	int k;
+	struct pipe_ctx *pipe_to_program = NULL;
+	bool enable_cursor_offload = dc_dmub_srv_is_cursor_offload_enabled(dc);
+
+	for (k = 0; k < (int)dc->res_pool->pipe_count; k++) {
+		struct pipe_ctx *tmp_pipe = &context->res_ctx.pipe_ctx[k];
+
+		if (tmp_pipe->stream != stream)
+			continue;
+
+		if (!pipe_to_program) {
+			pipe_to_program = tmp_pipe;
+
+			if (enable_cursor_offload && dc->hwss.begin_cursor_offload_update) {
+				hwss_add_begin_cursor_offload_update(seq_state, dc, tmp_pipe);
+			} else {
+				hwss_add_cursor_lock(seq_state, dc, pipe_to_program, true);
+				if (pipe_to_program->next_odm_pipe)
+					hwss_add_cursor_lock(seq_state, dc, pipe_to_program->next_odm_pipe, true);
+			}
+		}
+
+		hwss_add_set_cursor_attribute(seq_state, dc, tmp_pipe);
+		if (dc->ctx->dmub_srv)
+			hwss_add_send_update_cursor_info_to_dmu(seq_state, tmp_pipe, k);
+		if (dc->hwss.set_cursor_sdr_white_level)
+			hwss_add_set_cursor_sdr_white_level(seq_state, dc, tmp_pipe);
+		if (enable_cursor_offload && dc->hwss.update_cursor_offload_pipe)
+			hwss_add_update_cursor_offload_pipe(seq_state, dc, tmp_pipe);
+	}
+
+	if (pipe_to_program) {
+		if (enable_cursor_offload && dc->hwss.commit_cursor_offload_update) {
+			hwss_add_commit_cursor_offload_update(seq_state, dc, pipe_to_program);
+		} else {
+			hwss_add_cursor_lock(seq_state, dc, pipe_to_program, false);
+			if (pipe_to_program->next_odm_pipe)
+				hwss_add_cursor_lock(seq_state, dc, pipe_to_program->next_odm_pipe, false);
+		}
+	}
+}
+
+static void program_cursor_position_sequence(
+		struct dc *dc,
+		struct dc_stream_state *stream,
+		struct dc_state *context,
+		struct block_sequence_state *seq_state)
+{
+	int k;
+	struct pipe_ctx *pipe_to_program = NULL;
+	bool enable_cursor_offload = dc_dmub_srv_is_cursor_offload_enabled(dc);
+
+	for (k = 0; k < (int)dc->res_pool->pipe_count; k++) {
+		struct pipe_ctx *tmp_pipe = &context->res_ctx.pipe_ctx[k];
+
+		if (tmp_pipe->stream != stream ||
+				(!tmp_pipe->plane_res.mi  && !tmp_pipe->plane_res.hubp) ||
+				!tmp_pipe->plane_state ||
+				(!tmp_pipe->plane_res.xfm && !tmp_pipe->plane_res.dpp) ||
+				(!tmp_pipe->plane_res.ipp && !tmp_pipe->plane_res.dpp))
+			continue;
+
+		if (!pipe_to_program) {
+			pipe_to_program = tmp_pipe;
+
+			if (enable_cursor_offload && dc->hwss.begin_cursor_offload_update)
+				hwss_add_begin_cursor_offload_update(seq_state, dc, tmp_pipe);
+			else
+				hwss_add_cursor_lock(seq_state, dc, pipe_to_program, true);
+		}
+
+		hwss_add_set_cursor_position(seq_state, dc, tmp_pipe);
+		if (enable_cursor_offload && dc->hwss.update_cursor_offload_pipe)
+			hwss_add_update_cursor_offload_pipe(seq_state, dc, tmp_pipe);
+
+		if (dc->ctx->dmub_srv)
+			hwss_add_send_update_cursor_info_to_dmu(seq_state, tmp_pipe, k);
+	}
+
+	if (pipe_to_program) {
+		if (enable_cursor_offload && dc->hwss.commit_cursor_offload_update)
+			hwss_add_commit_cursor_offload_update(seq_state, dc, pipe_to_program);
+		else
+			hwss_add_cursor_lock(seq_state, dc, pipe_to_program, false);
+	}
+}
+
+static void add_update_info_frame_sequence(
+		struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx)
+{
+	bool is_hdmi_tmds;
+	bool is_dp;
+	bool is_hdmi_frl;
+
+	if (!pipe_ctx || !pipe_ctx->stream)
+		return;
+
+	if (pipe_ctx->stream_res.stream_enc == NULL &&
+			pipe_ctx->stream_res.hpo_frl_stream_enc == NULL)
+		return;
+
+	is_hdmi_tmds = dc_is_hdmi_tmds_signal(pipe_ctx->stream->signal);
+	is_dp = dc_is_dp_signal(pipe_ctx->stream->signal);
+
+	is_hdmi_frl = dc_is_hdmi_frl_signal(pipe_ctx->stream->signal);
+	if (!is_hdmi_tmds && !is_dp && !is_hdmi_frl)
+		return;
+
+	if (is_hdmi_tmds) {
+		hwss_add_stream_enc_update_hdmi_info_packets(seq_state, pipe_ctx);
+		return;
+	}
+
+	if (is_hdmi_frl) {
+		hwss_add_hpo_frl_stream_enc_update_hdmi_info_packets(seq_state, pipe_ctx);
+		return;
+	}
+
+	if (is_dp) {
+		if (dp_is_128b_132b_signal(pipe_ctx)) {
+			hwss_add_hpo_dp_stream_enc_update_dp_info_packets_sdp_line_num(seq_state, pipe_ctx);
+			hwss_add_hpo_dp_stream_enc_update_dp_info_packets(seq_state, pipe_ctx);
+		} else {
+			hwss_add_stream_enc_update_dp_info_packets_sdp_line_num(seq_state, pipe_ctx);
+			hwss_add_stream_enc_update_dp_info_packets(seq_state, pipe_ctx);
+		}
+	}
+}
+
+static void add_link_update_dsc_config_sequence(
+		struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx,
+		struct dsc_config *dsc_cfg,
+		struct dsc_optc_config *dsc_optc_cfg)
+{
+	struct display_stream_compressor *dsc = pipe_ctx->stream_res.dsc;
+	struct dc_stream_state *stream = pipe_ctx->stream;
+	struct dc *dc = stream->ctx->dc;
+	struct dccg *dccg = dc->res_pool->dccg;
+	struct pipe_ctx *top_pipe = pipe_ctx;
+	struct pipe_ctx *odm_pipe = NULL;
+	int opp_cnt = 1;
+	bool should_use_dto_dscclk = false;
+	struct dsc_config dsc_pps_cfg;
+	uint8_t *dsc_packed_pps = stream->dsc_packed_pps;
+	int last_dsc_set_config_step = 0;
+
+	if (!stream->timing.flags.DSC || !dsc)
+		return;
+
+	while (top_pipe->prev_odm_pipe)
+		top_pipe = top_pipe->prev_odm_pipe;
+
+	for (odm_pipe = top_pipe->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe)
+		opp_cnt++;
+
+	memset(dsc_cfg, 0, sizeof(*dsc_cfg));
+	memset(dsc_optc_cfg, 0, sizeof(*dsc_optc_cfg));
+
+	dsc_cfg->pic_width = (stream->timing.h_addressable +
+		top_pipe->dsc_padding_params.dsc_hactive_padding +
+		stream->timing.h_border_left +
+		stream->timing.h_border_right) / opp_cnt;
+	dsc_cfg->pic_height = stream->timing.v_addressable +
+		stream->timing.v_border_top +
+		stream->timing.v_border_bottom;
+	dsc_cfg->pixel_encoding = stream->timing.pixel_encoding;
+	dsc_cfg->color_depth = stream->timing.display_color_depth;
+	dsc_cfg->is_odm = top_pipe->next_odm_pipe ? true : false;
+	dsc_cfg->dc_dsc_cfg = stream->timing.dsc_cfg;
+	ASSERT(dsc_cfg->dc_dsc_cfg.num_slices_h % opp_cnt == 0);
+	dsc_cfg->dc_dsc_cfg.num_slices_h /= opp_cnt;
+	dsc_cfg->dsc_padding = 0;
+
+	if (dccg && dccg->funcs->set_dto_dscclk &&
+			stream->timing.pix_clk_100hz > 480000)
+		should_use_dto_dscclk = true;
+
+	if (should_use_dto_dscclk)
+		hwss_add_dccg_set_dto_dscclk(seq_state, dccg, dsc->inst,
+			dsc_cfg->dc_dsc_cfg.num_slices_h);
+
+	last_dsc_set_config_step = *seq_state->num_steps;
+	hwss_add_dsc_set_config(seq_state, dsc, dsc_cfg, dsc_optc_cfg);
+	hwss_add_dsc_enable_with_opp(seq_state, top_pipe);
+
+	for (odm_pipe = top_pipe->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe) {
+		struct display_stream_compressor *odm_dsc = odm_pipe->stream_res.dsc;
+
+		if (should_use_dto_dscclk)
+			hwss_add_dccg_set_dto_dscclk(seq_state, dccg, odm_dsc->inst,
+				dsc_cfg->dc_dsc_cfg.num_slices_h);
+
+		last_dsc_set_config_step = *seq_state->num_steps;
+		hwss_add_dsc_set_config(seq_state, odm_dsc, dsc_cfg, dsc_optc_cfg);
+		hwss_add_dsc_enable_with_opp(seq_state, odm_pipe);
+	}
+
+	if (dc_is_dp_signal(stream->signal) && !dp_is_128b_132b_signal(pipe_ctx))
+		hwss_add_stream_enc_dp_set_dsc_config(seq_state,
+			pipe_ctx->stream_res.stream_enc,
+			&seq_state->steps[last_dsc_set_config_step].params.dsc_set_config_simple_params.dsc_optc_cfg);
+
+	hwss_add_tg_set_dsc_config(seq_state, top_pipe->stream_res.tg,
+		&seq_state->steps[last_dsc_set_config_step].params.dsc_set_config_simple_params.dsc_optc_cfg, true);
+
+	memset(&dsc_pps_cfg, 0, sizeof(dsc_pps_cfg));
+	dsc_pps_cfg.pic_width = stream->timing.h_addressable +
+		stream->timing.h_border_left + stream->timing.h_border_right;
+	dsc_pps_cfg.pic_height = stream->timing.v_addressable +
+		stream->timing.v_border_top + stream->timing.v_border_bottom;
+	dsc_pps_cfg.pixel_encoding = stream->timing.pixel_encoding;
+	dsc_pps_cfg.color_depth = stream->timing.display_color_depth;
+	dsc_pps_cfg.is_odm = top_pipe->next_odm_pipe ? true : false;
+	dsc_pps_cfg.dc_dsc_cfg = stream->timing.dsc_cfg;
+	dsc_pps_cfg.dsc_padding = top_pipe->dsc_padding_params.dsc_hactive_padding;
+
+	if (dsc->funcs->dsc_get_packed_pps) {
+		dsc->funcs->dsc_get_packed_pps(dsc, &dsc_pps_cfg, dsc_packed_pps);
+
+		if (dc_is_dp_signal(stream->signal)) {
+			if (dp_is_128b_132b_signal(pipe_ctx))
+				hwss_add_hpo_dp_stream_enc_dp_set_dsc_pps_info_packet(seq_state,
+					pipe_ctx->stream_res.hpo_dp_stream_enc,
+					true, dsc_packed_pps, false);
+			else
+				hwss_add_stream_enc_dp_set_dsc_pps_info_packet(seq_state,
+					pipe_ctx->stream_res.stream_enc,
+					true, dsc_packed_pps, false);
+		}
+		else if (dc_is_hdmi_frl_signal(stream->signal)) {
+			hwss_add_hpo_frl_stream_enc_set_dsc_config(seq_state,
+				pipe_ctx->stream_res.hpo_frl_stream_enc,
+				&stream->timing,
+				dsc_packed_pps);
+		}
+	}
+}
+
+static void commit_planes_do_stream_update_sequence(struct dc *dc,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update,
+		enum surface_update_type update_type,
+		struct dc_state *context,
+		struct block_sequence block_sequence[MAX_HWSS_BLOCK_SEQUENCE_SIZE],
+		unsigned int *num_steps)
+{
+	int j;
+	struct block_sequence_state seq_state = { .steps = block_sequence, .num_steps = num_steps };
+	unsigned int dsc_cfg_index = 0;
+	*num_steps = 0; // Initialize to 0
+
+	// Stream updates
+	for (j = 0; j < (int)dc->res_pool->pipe_count; j++) {
+		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+		if (resource_is_pipe_type(pipe_ctx, OTG_MASTER) && pipe_ctx->stream == stream) {
+
+			if (stream_update->periodic_interrupt && dc->hwss.setup_periodic_interrupt)
+				hwss_add_setup_periodic_interrupt(&seq_state, dc, pipe_ctx);
+
+			if ((stream_update->hdr_static_metadata && !stream->use_dynamic_meta) ||
+					stream_update->vrr_infopacket ||
+					stream_update->vsc_infopacket ||
+					stream_update->vsp_infopacket ||
+					stream_update->hfvsif_infopacket ||
+					stream_update->adaptive_sync_infopacket ||
+					stream_update->vtem_infopacket ||
+					stream_update->avi_infopacket) {
+				resource_build_info_frame(pipe_ctx);
+				add_update_info_frame_sequence(&seq_state, pipe_ctx);
+
+				if (dc_is_dp_signal(pipe_ctx->stream->signal))
+					hwss_add_dp_trace_source_sequence(&seq_state,
+							pipe_ctx->stream->link,
+							DPCD_SOURCE_SEQ_AFTER_UPDATE_INFO_FRAME);
+			}
+
+			if (stream_update->hdr_static_metadata &&
+					stream->use_dynamic_meta &&
+					dc->hwss.set_dmdata_attributes &&
+					pipe_ctx->stream->dmdata_address.quad_part != 0)
+				hwss_add_set_dmdata_attributes(&seq_state, pipe_ctx);
+
+			if (stream_update->gamut_remap)
+				hwss_add_dpp_program_gamut_remap(&seq_state, pipe_ctx);
+
+			if (stream_update->output_csc_transform)
+				hwss_add_program_output_csc(&seq_state, dc, pipe_ctx,
+					stream->output_color_space,
+					stream->csc_color_matrix.matrix,
+					pipe_ctx->stream_res.opp->inst);
+
+			if (stream_update->dither_option) {
+				struct pipe_ctx *odm_pipe = pipe_ctx->next_odm_pipe;
+				resource_build_bit_depth_reduction_params(pipe_ctx->stream,
+								&pipe_ctx->stream->bit_depth_params);
+				hwss_add_opp_program_fmt(&seq_state, pipe_ctx->stream_res.opp,
+						&stream->bit_depth_params,
+						&stream->clamping);
+				while (odm_pipe) {
+					hwss_add_opp_program_fmt(&seq_state, odm_pipe->stream_res.opp,
+							&stream->bit_depth_params,
+							&stream->clamping);
+					odm_pipe = odm_pipe->next_odm_pipe;
+				}
+			}
+
+			if (stream_update->cursor_attributes)
+				program_cursor_attributes_sequence(dc, stream, context, &seq_state);
+
+			if (stream_update->cursor_position)
+				program_cursor_position_sequence(dc, stream, context, &seq_state);
+
+			/* Full fe update*/
+			if (update_type == UPDATE_TYPE_FAST)
+				continue;
+
+			if (stream_update->dsc_config)
+				if (dsc_cfg_index < MAX_PIPES) {
+					struct dsc_config dsc_cfg;
+					struct dsc_optc_config dsc_optc_cfg;
+
+					add_link_update_dsc_config_sequence(&seq_state,
+						pipe_ctx,
+						&dsc_cfg,
+						&dsc_optc_cfg);
+				}
+
+			if (stream_update->mst_bw_update) {
+				if (stream_update->mst_bw_update->is_increase)
+					hwss_add_link_increase_mst_payload(&seq_state,
+							pipe_ctx,
+							stream_update->mst_bw_update->mst_stream_bw);
+				else
+					hwss_add_link_reduce_mst_payload(&seq_state,
+							pipe_ctx,
+							stream_update->mst_bw_update->mst_stream_bw);
+			}
+
+			if (stream_update->pending_test_pattern) {
+				/*
+				 * test pattern params depends on ODM topology
+				 * changes that we could be applying to front
+				 * end. Since at the current stage front end
+				 * changes are not yet applied. We can only
+				 * apply test pattern in hw based on current
+				 * state and populate the final test pattern
+				 * params in new state. If current and new test
+				 * pattern params are different as result of
+				 * different ODM topology being used, it will be
+				 * detected and handle during front end
+				 * programming update.
+				 */
+				hwss_add_dp_set_test_pattern(&seq_state,
+					stream->link,
+					stream->test_pattern.type,
+					stream->test_pattern.color_space,
+					stream->test_pattern.p_link_settings,
+					stream->test_pattern.p_custom_pattern,
+					stream->test_pattern.cust_pattern_size);
+				resource_build_test_pattern_params(&context->res_ctx, pipe_ctx);
+			}
+
+			if (stream_update->dpms_off) {
+				// DPMS should not use partially updated pipe context
+				struct pipe_ctx *dpms_pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[j];
+
+				if (*stream_update->dpms_off) {
+					hwss_add_link_set_dpms_off(&seq_state, dpms_pipe_ctx);
+					/* for dpms, keep acquired resources*/
+					if (dpms_pipe_ctx->stream_res.audio && !dc->debug.az_endpoint_mute_only)
+						hwss_add_disable_audio_stream(&seq_state, dpms_pipe_ctx);
+
+					hwss_add_dc_set_optimized_required(&seq_state, dc, true);
+
+				} else {
+					if (get_seamless_boot_stream_count(context) == 0 && dc->hwss.prepare_bandwidth_sequence)
+						dc->hwss.prepare_bandwidth_sequence(dc, dc->current_state, &seq_state);
+					hwss_add_link_set_dpms_on(&seq_state, dc->current_state, dpms_pipe_ctx);
+				}
+			} else if (pipe_ctx->stream->link->wa_flags.blank_stream_on_ocs_change && stream_update->output_color_space
+					&& !stream->dpms_off && dc_is_dp_signal(pipe_ctx->stream->signal)) {
+				/*
+				 * Workaround for firmware issue in some receivers where they don't pick up
+				 * correct output color space unless DP link is disabled/re-enabled
+				 */
+					hwss_add_link_set_dpms_on(&seq_state, dc->current_state, pipe_ctx);
+			}
+
+			if (stream_update->abm_level && pipe_ctx->stream_res.abm) {
+				bool should_program_abm = true;
+
+				// if otg funcs defined check if blanked before programming
+				if (pipe_ctx->stream_res.tg->funcs->is_blanked)
+					if (pipe_ctx->stream_res.tg->funcs->is_blanked(pipe_ctx->stream_res.tg))
+						should_program_abm = false;
+
+				if (should_program_abm) {
+					if (*stream_update->abm_level == ABM_LEVEL_IMMEDIATE_DISABLE) {
+						hwss_add_abm_set_immediate_disable(&seq_state, dc, pipe_ctx);
+					} else {
+						hwss_add_abm_set_level(&seq_state, pipe_ctx->stream_res.abm, stream->abm_level);
+					}
+				}
+			}
+		}
+	}
+}
+
+static void commit_planes_do_stream_update(struct dc *dc,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update,
+		enum surface_update_type update_type,
+		struct dc_state *context)
+{
+	unsigned int j;
+
+	// Check if block sequence programming is enabled
+	if (dc->debug.enable_block_sequence_programming) {
+		unsigned int num_steps = 0;
+
+		// Build the block sequence using context's pre-allocated array
+		commit_planes_do_stream_update_sequence(dc, stream, stream_update,
+				update_type, context, context->block_sequence, &num_steps);
+
+		// Execute the block sequence
+		if (num_steps > 0)
+			hwss_execute_sequence(dc, context->block_sequence, num_steps);
+
+		return;
+	}
+
+	// Legacy path (existing implementation)
+	// Stream updates
+	for (j = 0; j < dc->res_pool->pipe_count; j++) {
+		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+		if (resource_is_pipe_type(pipe_ctx, OTG_MASTER) && pipe_ctx->stream == stream) {
+
+			if (stream_update->periodic_interrupt && dc->hwss.setup_periodic_interrupt)
+				dc->hwss.setup_periodic_interrupt(dc, pipe_ctx);
+
+			if ((stream_update->hdr_static_metadata && !stream->use_dynamic_meta) ||
+					stream_update->vrr_infopacket ||
+					stream_update->vsc_infopacket ||
+					stream_update->vsp_infopacket ||
+					stream_update->hfvsif_infopacket ||
+					stream_update->adaptive_sync_infopacket ||
+					stream_update->vtem_infopacket ||
+					stream_update->avi_infopacket) {
+				resource_build_info_frame(pipe_ctx);
+				dc->hwss.update_info_frame(pipe_ctx);
+
+				if (dc_is_dp_signal(pipe_ctx->stream->signal))
+					dc->link_srv->dp_trace_source_sequence(
+							pipe_ctx->stream->link,
+							DPCD_SOURCE_SEQ_AFTER_UPDATE_INFO_FRAME);
+			}
+
+			if (stream_update->hdr_static_metadata &&
+					stream->use_dynamic_meta &&
+					dc->hwss.set_dmdata_attributes &&
+					pipe_ctx->stream->dmdata_address.quad_part != 0)
+				dc->hwss.set_dmdata_attributes(pipe_ctx);
+
+			if (stream_update->gamut_remap)
+				dc_stream_set_gamut_remap(dc, stream);
+
+			if (stream_update->output_csc_transform)
+				dc_stream_program_csc_matrix(dc, stream);
+
+			if (stream_update->dither_option) {
+				struct pipe_ctx *odm_pipe = pipe_ctx->next_odm_pipe;
+				resource_build_bit_depth_reduction_params(pipe_ctx->stream,
+									&pipe_ctx->stream->bit_depth_params);
+				pipe_ctx->stream_res.opp->funcs->opp_program_fmt(pipe_ctx->stream_res.opp,
+						&stream->bit_depth_params,
+						&stream->clamping);
+				while (odm_pipe) {
+					odm_pipe->stream_res.opp->funcs->opp_program_fmt(odm_pipe->stream_res.opp,
+							&stream->bit_depth_params,
+							&stream->clamping);
+					odm_pipe = odm_pipe->next_odm_pipe;
+				}
+			}
+
+			if (stream_update->cursor_attributes)
+				program_cursor_attributes(dc, stream);
+
+			if (stream_update->cursor_position)
+				program_cursor_position(dc, stream);
+
+			/* Full fe update*/
+			if (update_type == UPDATE_TYPE_FAST)
+				continue;
+
+			if (stream_update->dsc_config)
+				dc->link_srv->update_dsc_config(pipe_ctx);
+
+			if (stream_update->mst_bw_update) {
+				if (stream_update->mst_bw_update->is_increase)
+					dc->link_srv->increase_mst_payload(pipe_ctx,
+							stream_update->mst_bw_update->mst_stream_bw);
+				else
+					dc->link_srv->reduce_mst_payload(pipe_ctx,
+							stream_update->mst_bw_update->mst_stream_bw);
+			}
+
+			if (stream_update->pending_test_pattern) {
+				/*
+				 * test pattern params depends on ODM topology
+				 * changes that we could be applying to front
+				 * end. Since at the current stage front end
+				 * changes are not yet applied. We can only
+				 * apply test pattern in hw based on current
+				 * state and populate the final test pattern
+				 * params in new state. If current and new test
+				 * pattern params are different as result of
+				 * different ODM topology being used, it will be
+				 * detected and handle during front end
+				 * programming update.
+				 */
+				dc->link_srv->dp_set_test_pattern(stream->link,
+					stream->test_pattern.type,
+					stream->test_pattern.color_space,
+					stream->test_pattern.p_link_settings,
+					stream->test_pattern.p_custom_pattern,
+					stream->test_pattern.cust_pattern_size);
+				resource_build_test_pattern_params(&context->res_ctx, pipe_ctx);
+			}
+
+			// DPMS should not use partially updated pipe context
+			struct pipe_ctx *dpms_pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[j];
+
+			if (stream_update->dpms_off) {
+				if (*stream_update->dpms_off) {
+					dc->link_srv->set_dpms_off(dpms_pipe_ctx);
+					/* for dpms, keep acquired resources*/
+					if (dpms_pipe_ctx->stream_res.audio && !dc->debug.az_endpoint_mute_only) {
+						struct audio *audio = dpms_pipe_ctx->stream_res.audio;
+
+						audio->funcs->az_disable(audio);
+					}
+
+					dc->optimized_required = true;
+
+				} else {
+					if (get_seamless_boot_stream_count(context) == 0)
+						dc->hwss.prepare_bandwidth(dc, dc->current_state);
+					dc->link_srv->set_dpms_on(dc->current_state, dpms_pipe_ctx);
+				}
+			} else if (dpms_pipe_ctx->stream->link->wa_flags.blank_stream_on_ocs_change &&
+					stream_update->output_color_space &&
+					!stream->dpms_off && dc_is_dp_signal(dpms_pipe_ctx->stream->signal)) {
+				/*
+				 * Workaround for firmware issue in some receivers where they don't pick up
+				 * correct output color space unless DP link is disabled/re-enabled
+				 */
+				dc->link_srv->set_dpms_on(dc->current_state, dpms_pipe_ctx);
+			}
+
+			if (stream_update->abm_level && pipe_ctx->stream_res.abm) {
+				bool should_program_abm = true;
+
+				// if otg funcs defined check if blanked before programming
+				if (pipe_ctx->stream_res.tg->funcs->is_blanked)
+					if (pipe_ctx->stream_res.tg->funcs->is_blanked(pipe_ctx->stream_res.tg))
+						should_program_abm = false;
+
+				if (should_program_abm) {
+					if (*stream_update->abm_level == ABM_LEVEL_IMMEDIATE_DISABLE) {
+						dc->hwss.set_abm_immediate_disable(pipe_ctx);
+					} else {
+						pipe_ctx->stream_res.abm->funcs->set_abm_level(
+							pipe_ctx->stream_res.abm, stream->abm_level);
+					}
+				}
+			}
+		}
+	}
+}
+
+static bool dc_dmub_should_send_dirty_rect_cmd(struct dc *dc, struct dc_stream_state *stream)
+{
+	(void)dc;
+	if ((stream->link->psr_settings.psr_version == DC_PSR_VERSION_SU_1
+			|| stream->link->psr_settings.psr_version == DC_PSR_VERSION_1)
+			&& stream->ctx->dce_version >= DCN_VERSION_3_1)
+		return true;
+
+	if (stream->link->replay_settings.config.replay_supported)
+		return true;
+
+	if (stream->ctx->dce_version >= DCN_VERSION_3_5 && stream->abm_level)
+		return true;
+
+	return false;
+}
+
+void dc_dmub_update_dirty_rect(struct dc *dc,
+			       int surface_count,
+			       struct dc_stream_state *stream,
+			       const struct dc_surface_update *srf_updates,
+			       struct dc_state *context)
+{
+	union dmub_rb_cmd cmd;
+	struct dmub_cmd_update_dirty_rect_data *update_dirty_rect;
+	int i;
+	unsigned int j;
+	unsigned int panel_inst = 0;
+
+	if (!dc_dmub_should_send_dirty_rect_cmd(dc, stream))
+		return;
+
+	if (!dc->config.frame_update_cmd_version2 && !dc_get_edp_link_panel_inst(dc, stream->link, &panel_inst))
+		return;
+
+	memset(&cmd, 0x0, sizeof(cmd));
+	cmd.update_dirty_rect.header.type = DMUB_CMD__UPDATE_DIRTY_RECT;
+	cmd.update_dirty_rect.header.sub_type = 0;
+	cmd.update_dirty_rect.header.payload_bytes =
+		sizeof(cmd.update_dirty_rect) -
+		sizeof(cmd.update_dirty_rect.header);
+	update_dirty_rect = &cmd.update_dirty_rect.update_dirty_rect_data;
+	for (i = 0; i < surface_count; i++) {
+		struct dc_plane_state *plane_state = srf_updates[i].surface;
+		const struct dc_flip_addrs *flip_addr = srf_updates[i].flip_addr;
+
+		if (!srf_updates[i].surface || !flip_addr)
+			continue;
+		/* Do not send in immediate flip mode */
+		if (srf_updates[i].surface->flip_immediate)
+			continue;
+
+		if (dc->config.frame_update_cmd_version2)
+			update_dirty_rect->cmd_version = DMUB_CMD_CURSOR_UPDATE_VERSION_2;
+		else
+			update_dirty_rect->cmd_version = DMUB_CMD_CURSOR_UPDATE_VERSION_1;
+
+		update_dirty_rect->dirty_rect_count = (uint8_t)flip_addr->dirty_rect_count;
+		memcpy(update_dirty_rect->src_dirty_rects, flip_addr->dirty_rects,
+				sizeof(flip_addr->dirty_rects));
+		for (j = 0; j < dc->res_pool->pipe_count; j++) {
+			struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+			if (pipe_ctx->stream != stream)
+				continue;
+			if (pipe_ctx->plane_state != plane_state)
+				continue;
+
+			update_dirty_rect->panel_inst = (uint8_t)panel_inst;
+			update_dirty_rect->pipe_idx = (uint8_t)j;
+			update_dirty_rect->otg_inst = (uint8_t)pipe_ctx->stream_res.tg->inst;
+			dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_NO_WAIT);
+		}
+	}
+}
+
+static void build_dmub_update_dirty_rect(
+		struct dc *dc,
+		int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_surface_update *srf_updates,
+		struct dc_state *context,
+		struct dc_dmub_cmd dc_dmub_cmd[],
+		unsigned int *dmub_cmd_count)
+{
+	union dmub_rb_cmd cmd;
+	struct dmub_cmd_update_dirty_rect_data *update_dirty_rect;
+	int i;
+	unsigned int j;
+	unsigned int panel_inst = 0;
+
+	if (!dc_dmub_should_send_dirty_rect_cmd(dc, stream))
+		return;
+
+	if (!dc->config.frame_update_cmd_version2 && !dc_get_edp_link_panel_inst(dc, stream->link, &panel_inst))
+		return;
+
+	memset(&cmd, 0x0, sizeof(cmd));
+	cmd.update_dirty_rect.header.type = DMUB_CMD__UPDATE_DIRTY_RECT;
+	cmd.update_dirty_rect.header.sub_type = 0;
+	cmd.update_dirty_rect.header.payload_bytes =
+		sizeof(cmd.update_dirty_rect) -
+		sizeof(cmd.update_dirty_rect.header);
+	update_dirty_rect = &cmd.update_dirty_rect.update_dirty_rect_data;
+	for (i = 0; i < surface_count; i++) {
+		struct dc_plane_state *plane_state = srf_updates[i].surface;
+		const struct dc_flip_addrs *flip_addr = srf_updates[i].flip_addr;
+
+		if (!srf_updates[i].surface || !flip_addr)
+			continue;
+		/* Do not send in immediate flip mode */
+		if (srf_updates[i].surface->flip_immediate)
+			continue;
+
+		if (dc->config.frame_update_cmd_version2)
+			update_dirty_rect->cmd_version = DMUB_CMD_CURSOR_UPDATE_VERSION_2;
+		else
+			update_dirty_rect->cmd_version = DMUB_CMD_CURSOR_UPDATE_VERSION_1;
+
+		update_dirty_rect->dirty_rect_count = (uint8_t)flip_addr->dirty_rect_count;
+		memcpy(update_dirty_rect->src_dirty_rects, flip_addr->dirty_rects,
+				sizeof(flip_addr->dirty_rects));
+		for (j = 0; j < dc->res_pool->pipe_count; j++) {
+			struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+			if (pipe_ctx->stream != stream)
+				continue;
+			if (pipe_ctx->plane_state != plane_state)
+				continue;
+			update_dirty_rect->panel_inst = (uint8_t)panel_inst;
+			update_dirty_rect->pipe_idx = (uint8_t)j;
+			update_dirty_rect->otg_inst = (uint8_t)pipe_ctx->stream_res.tg->inst;
+			dc_dmub_cmd[*dmub_cmd_count].dmub_cmd = cmd;
+			dc_dmub_cmd[*dmub_cmd_count].wait_type = DM_DMUB_WAIT_TYPE_NO_WAIT;
+			(*dmub_cmd_count)++;
+		}
+	}
+}
+
+/**
+ * dc_check_address_only_update - Check if addr_update is the sole flag set
+ *
+ * @update_bits: The pipe update bits to check
+ *
+ * Determines whether an update contains only an address change with no other
+ * pending updates.
+ *
+ * Return: %true if addr_update is the sole bit set, %false otherwise.
+ */
+bool dc_check_address_only_update(struct pipe_update_bits update_bits)
+{
+	struct pipe_update_bits check = update_bits;  /* 1. Copy all flags from input */
+
+	check.addr_update = 0;                        /* 2. Zero the addr_update bit in the copy */
+	return update_bits.addr_update &&             /* 3. Check addr_update was set in original */
+			!dc_pipe_update_bits_is_any_set(&check); /* 4. Check no other bits remain in the copy */
+}
+
+/**
+ * build_dmub_cmd_list() - Build an array of DMCUB commands to be sent to DMCUB
+ *
+ * @dc: Current DC state
+ * @srf_updates: Array of surface updates
+ * @surface_count: Number of surfaces that have an updated
+ * @stream: Corresponding stream to be updated in the current flip
+ * @context: New DC state to be programmed
+ *
+ * @dc_dmub_cmd: Array of DMCUB commands to be sent to DMCUB
+ * @dmub_cmd_count: Count indicating the number of DMCUB commands in dc_dmub_cmd array
+ *
+ * This function builds an array of DMCUB commands to be sent to DMCUB. This function is required
+ * to build an array of commands and have them sent while the OTG lock is acquired.
+ *
+ * Return: void
+ */
+static void build_dmub_cmd_list(struct dc *dc,
+		struct dc_surface_update *srf_updates,
+		int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_state *context,
+		struct dc_dmub_cmd dc_dmub_cmd[],
+		unsigned int *dmub_cmd_count)
+{
+	// Initialize cmd count to 0
+	*dmub_cmd_count = 0;
+	build_dmub_update_dirty_rect(dc, surface_count, stream, srf_updates, context, dc_dmub_cmd, dmub_cmd_count);
+}
+
+static void commit_plane_for_stream_offload_fams2_flip(struct dc *dc,
+		struct dc_surface_update *srf_updates,
+		int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_state *context)
+{
+	int i;
+	unsigned int j;
+
+	/* update dirty rect for PSR */
+	dc_dmub_update_dirty_rect(dc, surface_count, stream,
+			srf_updates, context);
+
+	/* Perform requested Updates */
+	for (i = 0; i < surface_count; i++) {
+		struct dc_plane_state *plane_state = srf_updates[i].surface;
+
+		for (j = 0; j < dc->res_pool->pipe_count; j++) {
+			struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+			if (!should_update_pipe_for_stream(context, pipe_ctx, stream))
+				continue;
+
+			if (!should_update_pipe_for_plane(context, pipe_ctx, plane_state))
+				continue;
+
+			/* update pipe context for plane */
+			if (pipe_ctx->plane_state->update_bits.addr_update)
+				dc->hwss.update_plane_addr(dc, pipe_ctx);
+		}
+	}
+
+	/* Send commands to DMCUB */
+	dc_dmub_srv_fams2_passthrough_flip(dc,
+				context,
+				stream,
+				srf_updates,
+				surface_count);
+}
+
+static void commit_planes_for_stream_fast(struct dc *dc,
+		struct dc_surface_update *srf_updates,
+		int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update,
+		enum surface_update_type update_type,
+		struct dc_state *context)
+{
+	int i;
+	unsigned int j;
+	struct pipe_ctx *top_pipe_to_program = NULL;
+	struct dc_stream_status *stream_status = NULL;
+	bool should_offload_fams2_flip = false;
+	bool should_lock_all_pipes = (update_type != UPDATE_TYPE_FAST);
+
+	if (should_lock_all_pipes)
+		determine_pipe_unlock_order(dc, context);
+
+	if (dc->debug.fams2_config.bits.enable &&
+			dc->debug.fams2_config.bits.enable_offload_flip &&
+			dc_state_is_fams2_in_use(dc, context)) {
+		/* if not offloading to HWFQ, offload to FAMS2 if needed */
+		should_offload_fams2_flip = true;
+		for (i = 0; i < surface_count; i++) {
+			if (srf_updates[i].surface &&
+					dc_pipe_update_bits_is_any_set(&srf_updates[i].surface->update_bits) &&
+					!dc_check_address_only_update(srf_updates[i].surface->update_bits)) {
+				/* more than address update, need to acquire FAMS2 lock */
+				should_offload_fams2_flip = false;
+				break;
+			}
+		}
+		if (stream_update) {
+			/* more than address update, need to acquire FAMS2 lock */
+			should_offload_fams2_flip = false;
+		}
+	}
+
+	dc_exit_ips_for_hw_access(dc);
+
+	dc_z10_restore(dc);
+
+	top_pipe_to_program = resource_get_otg_master_for_stream(
+			&context->res_ctx,
+			stream);
+
+	if (!top_pipe_to_program)
+		return;
+
+	for (i = 0; i < (int)dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+
+		if (pipe->stream && pipe->plane_state) {
+			if (!dc->debug.using_dml2)
+				set_p_state_switch_method(dc, context, pipe);
+
+			if (dc->debug.visual_confirm)
+				dc_update_visual_confirm_color(dc, context, pipe);
+		}
+	}
+
+	for (i = 0; i < surface_count; i++) {
+		struct dc_plane_state *plane_state = srf_updates[i].surface;
+		/*set logical flag for lock/unlock use*/
+		for (j = 0; j < dc->res_pool->pipe_count; j++) {
+			struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+			if (!pipe_ctx->plane_state)
+				continue;
+			if (!should_update_pipe_for_plane(context, pipe_ctx, plane_state))
+				continue;
+
+			pipe_ctx->plane_state->triplebuffer_flips = false;
+			if (update_type == UPDATE_TYPE_FAST &&
+					dc->hwss.program_triplebuffer != NULL &&
+					!pipe_ctx->plane_state->flip_immediate && dc->debug.enable_tri_buf) {
+				/*triple buffer for VUpdate only*/
+				pipe_ctx->plane_state->triplebuffer_flips = true;
+			}
+		}
+	}
+
+	stream_status = dc_state_get_stream_status(context, stream);
+
+	if (should_offload_fams2_flip) {
+		commit_plane_for_stream_offload_fams2_flip(dc,
+				srf_updates,
+				surface_count,
+				stream,
+				context);
+	} else if (stream_status) {
+		build_dmub_cmd_list(dc,
+				srf_updates,
+				surface_count,
+				stream,
+				context,
+				context->dc_dmub_cmd,
+				&(context->dmub_cmd_count));
+		hwss_build_fast_sequence(dc,
+				context->dc_dmub_cmd,
+				context->dmub_cmd_count,
+				context->block_sequence,
+				&(context->block_sequence_steps),
+				top_pipe_to_program,
+				stream_status,
+				context);
+		hwss_execute_sequence(dc,
+				context->block_sequence,
+				context->block_sequence_steps);
+	}
+
+	/* Clear update flags so next flip doesn't have redundant programming
+	 * (if there's no stream update, the update flags are not cleared).
+	 * Surface updates are cleared unconditionally at the beginning of each flip,
+	 * so no need to clear here.
+	 */
+	if (top_pipe_to_program->stream)
+		stream_update_flags_clear(&top_pipe_to_program->stream->update_flags);
+}
+
+static void commit_planes_for_stream(struct dc *dc,
+		const struct dc_surface_update *srf_updates,
+		int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update,
+		enum surface_update_type update_type,
+		struct dc_state *context)
+{
+	int i;
+	unsigned int j, pipe_idx;
+	struct pipe_ctx *top_pipe_to_program = NULL;
+	bool should_lock_all_pipes = (update_type != UPDATE_TYPE_FAST);
+	bool subvp_prev_use = false;
+	bool subvp_curr_use = false;
+	uint8_t current_stream_mask = 0;
+
+	if (should_lock_all_pipes)
+		determine_pipe_unlock_order(dc, context);
+	// Once we apply the new subvp context to hardware it won't be in the
+	// dc->current_state anymore, so we have to cache it before we apply
+	// the new SubVP context
+	subvp_prev_use = false;
+	dc_exit_ips_for_hw_access(dc);
+
+	dc_z10_restore(dc);
+	if (update_type == UPDATE_TYPE_FULL && dc->optimized_required)
+		hwss_process_outstanding_hw_updates(dc, dc->current_state);
+
+	if (update_type != UPDATE_TYPE_FAST && dc->res_pool->funcs->prepare_mcache_programming)
+		dc->res_pool->funcs->prepare_mcache_programming(dc, context);
+
+	for (pipe_idx = 0; pipe_idx < dc->res_pool->pipe_count; pipe_idx++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[pipe_idx];
+
+		if (pipe->stream && pipe->plane_state) {
+			if (!dc->debug.using_dml2)
+				set_p_state_switch_method(dc, context, pipe);
+
+			if (dc->debug.visual_confirm)
+				dc_update_visual_confirm_color(dc, context, pipe);
+		}
+	}
+
+	if (update_type == UPDATE_TYPE_FULL) {
+		dc_allow_idle_optimizations(dc, false);
+
+		if (get_seamless_boot_stream_count(context) == 0)
+			dc->hwss.prepare_bandwidth(dc, context);
+
+		if (dc->hwss.update_dsc_pg)
+			dc->hwss.update_dsc_pg(dc, context, false);
+
+		context_clock_trace(dc, context);
+	}
+
+	if (update_type == UPDATE_TYPE_FULL)
+		hwss_wait_for_outstanding_hw_updates(dc, dc->current_state);
+
+	top_pipe_to_program = resource_get_otg_master_for_stream(
+				&context->res_ctx,
+				stream);
+	ASSERT(top_pipe_to_program != NULL);
+
+	for (pipe_idx = 0; pipe_idx < dc->res_pool->pipe_count; pipe_idx++) {
+		struct pipe_ctx *old_pipe = &dc->current_state->res_ctx.pipe_ctx[pipe_idx];
+
+		// Check old context for SubVP
+		subvp_prev_use |= (dc_state_get_pipe_subvp_type(dc->current_state, old_pipe) == SUBVP_PHANTOM);
+		if (subvp_prev_use)
+			break;
+	}
+
+	for (pipe_idx = 0; pipe_idx < dc->res_pool->pipe_count; pipe_idx++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[pipe_idx];
+
+		if (dc_state_get_pipe_subvp_type(context, pipe) == SUBVP_PHANTOM) {
+			subvp_curr_use = true;
+			break;
+		}
+	}
+
+	if (stream->test_pattern.type != DP_TEST_PATTERN_VIDEO_MODE) {
+		struct pipe_ctx *mpcc_pipe;
+		struct pipe_ctx *odm_pipe;
+
+		for (mpcc_pipe = top_pipe_to_program; mpcc_pipe; mpcc_pipe = mpcc_pipe->bottom_pipe)
+			for (odm_pipe = mpcc_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe)
+				odm_pipe->ttu_regs.min_ttu_vblank = MAX_TTU;
+	}
+
+	if ((update_type != UPDATE_TYPE_FAST) && stream->update_flags.bits.dsc_changed)
+		if (top_pipe_to_program &&
+			top_pipe_to_program->stream_res.tg->funcs->lock_doublebuffer_enable) {
+			if (should_use_dmub_inbox1_lock(dc, stream->link)) {
+				union dmub_hw_lock_flags hw_locks = { 0 };
+				struct dmub_hw_lock_inst_flags inst_flags = { 0 };
+
+				hw_locks.bits.lock_dig = 1;
+				inst_flags.dig_inst = (uint8_t)top_pipe_to_program->stream_res.tg->inst;
+
+				dmub_hw_lock_mgr_cmd(dc->ctx->dmub_srv,
+							true,
+							&hw_locks,
+							&inst_flags);
+			} else
+				top_pipe_to_program->stream_res.tg->funcs->lock_doublebuffer_enable(
+						top_pipe_to_program->stream_res.tg);
+		}
+
+	if (dc->hwss.wait_for_dcc_meta_propagation) {
+		dc->hwss.wait_for_dcc_meta_propagation(dc, top_pipe_to_program);
+	}
+
+	if (dc->hwseq->funcs.wait_for_pipe_update_if_needed)
+		dc->hwseq->funcs.wait_for_pipe_update_if_needed(dc, top_pipe_to_program, update_type < UPDATE_TYPE_FULL);
+
+	if (should_lock_all_pipes && dc->hwss.interdependent_update_lock) {
+		if (dc->hwss.subvp_pipe_control_lock)
+			dc->hwss.subvp_pipe_control_lock(dc, context, true, should_lock_all_pipes, NULL, subvp_prev_use);
+
+		if (dc->hwss.dmub_hw_control_lock)
+			dc->hwss.dmub_hw_control_lock(dc, context, true);
+
+		dc->hwss.interdependent_update_lock(dc, context, true);
+	} else {
+		if (dc->hwss.subvp_pipe_control_lock)
+			dc->hwss.subvp_pipe_control_lock(dc, context, true, should_lock_all_pipes, top_pipe_to_program, subvp_prev_use);
+
+		if (dc->hwss.dmub_hw_control_lock)
+			dc->hwss.dmub_hw_control_lock(dc, context, true);
+
+		/* Lock the top pipe while updating plane addrs, since freesync requires
+		 *  plane addr update event triggers to be synchronized.
+		 *  top_pipe_to_program is expected to never be NULL
+		 */
+		dc->hwss.pipe_control_lock(dc, top_pipe_to_program, true);
+	}
+
+	dc_dmub_update_dirty_rect(dc, surface_count, stream, srf_updates, context);
+
+	// Stream updates
+	if (stream_update)
+		commit_planes_do_stream_update(dc, stream, stream_update, update_type, context);
+
+	if (surface_count == 0) {
+		/*
+		 * In case of turning off screen, no need to program front end a second time.
+		 * just return after program blank.
+		 */
+		if (dc->hwss.apply_ctx_for_surface)
+			dc->hwss.apply_ctx_for_surface(dc, stream, 0, context);
+		if (dc->hwss.program_front_end_for_ctx)
+			dc->hwss.program_front_end_for_ctx(dc, context);
+
+		if (should_lock_all_pipes && dc->hwss.interdependent_update_lock) {
+			dc->hwss.interdependent_update_lock(dc, context, false);
+		} else {
+			dc->hwss.pipe_control_lock(dc, top_pipe_to_program, false);
+		}
+		dc->hwss.post_unlock_program_front_end(dc, context);
+
+		if (update_type != UPDATE_TYPE_FAST)
+			if (dc->hwss.commit_subvp_config)
+				dc->hwss.commit_subvp_config(dc, context);
+
+		/* Since phantom pipe programming is moved to post_unlock_program_front_end,
+		 * move the SubVP lock to after the phantom pipes have been setup
+		 */
+		if (dc->hwss.subvp_pipe_control_lock)
+			dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes,
+							 NULL, subvp_prev_use);
+
+		if (dc->hwss.dmub_hw_control_lock)
+			dc->hwss.dmub_hw_control_lock(dc, context, false);
+		return;
+	}
+
+	if (update_type != UPDATE_TYPE_FAST) {
+		for (j = 0; j < dc->res_pool->pipe_count; j++) {
+			struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+			if ((dc->debug.visual_confirm == VISUAL_CONFIRM_SUBVP ||
+				dc->debug.visual_confirm == VISUAL_CONFIRM_MCLK_SWITCH) &&
+				pipe_ctx->stream && pipe_ctx->plane_state) {
+				/* Only update visual confirm for SUBVP and Mclk switching here.
+				 * The bar appears on all pipes, so we need to update the bar on all displays,
+				 * so the information doesn't get stale.
+				 */
+				dc->hwss.update_visual_confirm_color(dc, pipe_ctx,
+						pipe_ctx->plane_res.hubp->inst);
+			}
+		}
+	}
+
+	for (i = 0; i < surface_count; i++) {
+		struct dc_plane_state *plane_state = srf_updates[i].surface;
+
+		/*set logical flag for lock/unlock use*/
+		for (j = 0; j < dc->res_pool->pipe_count; j++) {
+			struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+			if (!pipe_ctx->plane_state)
+				continue;
+			if (!should_update_pipe_for_plane(context, pipe_ctx, plane_state))
+				continue;
+			pipe_ctx->plane_state->triplebuffer_flips = false;
+			if (update_type == UPDATE_TYPE_FAST &&
+					dc->hwss.program_triplebuffer != NULL &&
+					!pipe_ctx->plane_state->flip_immediate && dc->debug.enable_tri_buf) {
+				/*triple buffer for VUpdate only*/
+				pipe_ctx->plane_state->triplebuffer_flips = true;
+			}
+		}
+		if (update_type == UPDATE_TYPE_FULL) {
+			/* force vsync flip when reconfiguring pipes to prevent underflow */
+			plane_state->flip_immediate = false;
+			plane_state->triplebuffer_flips = false;
+		}
+	}
+
+	// Update Type FULL, Surface updates
+	for (j = 0; j < dc->res_pool->pipe_count; j++) {
+		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+		if (!pipe_ctx->top_pipe &&
+			!pipe_ctx->prev_odm_pipe &&
+			should_update_pipe_for_stream(context, pipe_ctx, stream)) {
+			struct dc_stream_status *pipe_stream_status = NULL;
+
+			if (!pipe_ctx->plane_state)
+				continue;
+
+			/* Full fe update*/
+			if (update_type == UPDATE_TYPE_FAST)
+				continue;
+
+			pipe_stream_status =
+				stream_get_status(context, pipe_ctx->stream);
+
+			if (dc->hwss.apply_ctx_for_surface && pipe_stream_status)
+				dc->hwss.apply_ctx_for_surface(
+					dc, pipe_ctx->stream, pipe_stream_status->plane_count, context);
+		}
+	}
+
+	for (j = 0; j < dc->res_pool->pipe_count; j++) {
+		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+		if (!pipe_ctx->plane_state)
+			continue;
+
+		/* Full fe update*/
+		if (update_type == UPDATE_TYPE_FAST)
+			continue;
+
+		ASSERT(!pipe_ctx->plane_state->triplebuffer_flips);
+		if (dc->hwss.program_triplebuffer != NULL && dc->debug.enable_tri_buf) {
+			/*turn off triple buffer for full update*/
+			dc->hwss.program_triplebuffer(
+				dc, pipe_ctx, pipe_ctx->plane_state->triplebuffer_flips);
+		}
+	}
+
+	if (dc->hwss.program_front_end_for_ctx && update_type != UPDATE_TYPE_FAST) {
+		dc->hwss.program_front_end_for_ctx(dc, context);
+
+		//Pipe busy until some frame and line #
+		if (dc->hwseq->funcs.set_wait_for_update_needed_for_pipe && update_type == UPDATE_TYPE_FULL) {
+			for (j = 0; j < dc->res_pool->pipe_count; j++) {
+				struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+				dc->hwseq->funcs.set_wait_for_update_needed_for_pipe(dc, pipe_ctx);
+			}
+		}
+
+		if (dc->debug.validate_dml_output) {
+			for (pipe_idx = 0; pipe_idx < dc->res_pool->pipe_count; pipe_idx++) {
+				struct pipe_ctx *cur_pipe = &context->res_ctx.pipe_ctx[pipe_idx];
+				if (cur_pipe->stream == NULL)
+					continue;
+
+				cur_pipe->plane_res.hubp->funcs->validate_dml_output(
+						cur_pipe->plane_res.hubp, dc->ctx,
+						&context->res_ctx.pipe_ctx[pipe_idx].rq_regs,
+						&context->res_ctx.pipe_ctx[pipe_idx].dlg_regs,
+						&context->res_ctx.pipe_ctx[pipe_idx].ttu_regs);
+			}
+		}
+	}
+
+	// Update Type FAST, Surface updates
+	if (update_type == UPDATE_TYPE_FAST) {
+		if (dc->hwss.set_flip_control_gsl)
+			for (i = 0; i < surface_count; i++) {
+				struct dc_plane_state *plane_state = srf_updates[i].surface;
+
+				for (j = 0; j < dc->res_pool->pipe_count; j++) {
+					struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+					if (!should_update_pipe_for_stream(context, pipe_ctx, stream))
+						continue;
+
+					if (!should_update_pipe_for_plane(context, pipe_ctx, plane_state))
+						continue;
+
+					// GSL has to be used for flip immediate
+					dc->hwss.set_flip_control_gsl(pipe_ctx,
+							pipe_ctx->plane_state->flip_immediate);
+				}
+			}
+
+		/* Perform requested Updates */
+		for (i = 0; i < surface_count; i++) {
+			struct dc_plane_state *plane_state = srf_updates[i].surface;
+
+			for (j = 0; j < dc->res_pool->pipe_count; j++) {
+				struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+				if (!should_update_pipe_for_stream(context, pipe_ctx, stream))
+					continue;
+
+				if (!should_update_pipe_for_plane(context, pipe_ctx, plane_state))
+					continue;
+
+				if (srf_updates[i].cm &&
+						srf_updates[i].cm->flags.bits.lut3d_enable &&
+						srf_updates[i].cm->flags.bits.lut3d_dma_enable &&
+						dc->hwss.trigger_3dlut_dma_load)
+					dc->hwss.trigger_3dlut_dma_load(dc, pipe_ctx);
+
+				/*program triple buffer after lock based on flip type*/
+				if (dc->hwss.program_triplebuffer != NULL && dc->debug.enable_tri_buf) {
+					/*only enable triplebuffer for fast_update*/
+					dc->hwss.program_triplebuffer(
+						dc, pipe_ctx, pipe_ctx->plane_state->triplebuffer_flips);
+				}
+				if (pipe_ctx->plane_state->update_bits.addr_update)
+					dc->hwss.update_plane_addr(dc, pipe_ctx);
+			}
+		}
+	}
+
+	if (should_lock_all_pipes && dc->hwss.interdependent_update_lock) {
+		dc->hwss.interdependent_update_lock(dc, context, false);
+	} else {
+		dc->hwss.pipe_control_lock(dc, top_pipe_to_program, false);
+	}
+
+	if ((update_type != UPDATE_TYPE_FAST) && stream->update_flags.bits.dsc_changed)
+		if (top_pipe_to_program &&
+		    top_pipe_to_program->stream_res.tg->funcs->lock_doublebuffer_enable) {
+			top_pipe_to_program->stream_res.tg->funcs->wait_for_state(
+				top_pipe_to_program->stream_res.tg,
+				CRTC_STATE_VACTIVE);
+			top_pipe_to_program->stream_res.tg->funcs->wait_for_state(
+				top_pipe_to_program->stream_res.tg,
+				CRTC_STATE_VBLANK);
+			top_pipe_to_program->stream_res.tg->funcs->wait_for_state(
+				top_pipe_to_program->stream_res.tg,
+				CRTC_STATE_VACTIVE);
+
+			if (should_use_dmub_inbox1_lock(dc, stream->link)) {
+				union dmub_hw_lock_flags hw_locks = { 0 };
+				struct dmub_hw_lock_inst_flags inst_flags = { 0 };
+
+				hw_locks.bits.lock_dig = 1;
+				inst_flags.dig_inst = (uint8_t)top_pipe_to_program->stream_res.tg->inst;
+
+				dmub_hw_lock_mgr_cmd(dc->ctx->dmub_srv,
+							false,
+							&hw_locks,
+							&inst_flags);
+			} else
+				top_pipe_to_program->stream_res.tg->funcs->lock_doublebuffer_disable(
+					top_pipe_to_program->stream_res.tg);
+		}
+
+	if (subvp_curr_use) {
+		/* If enabling subvp or transitioning from subvp->subvp, enable the
+		 * phantom streams before we program front end for the phantom pipes.
+		 */
+		if (update_type != UPDATE_TYPE_FAST) {
+			if (dc->hwss.enable_phantom_streams)
+				dc->hwss.enable_phantom_streams(dc, context);
+		}
+	}
+
+	if (update_type != UPDATE_TYPE_FAST)
+		dc->hwss.post_unlock_program_front_end(dc, context);
+
+	if (subvp_prev_use && !subvp_curr_use) {
+		/* If disabling subvp, disable phantom streams after front end
+		 * programming has completed (we turn on phantom OTG in order
+		 * to complete the plane disable for phantom pipes).
+		 */
+
+		if (dc->hwss.disable_phantom_streams)
+			dc->hwss.disable_phantom_streams(dc, context);
+	}
+
+	if (update_type != UPDATE_TYPE_FAST)
+		if (dc->hwss.commit_subvp_config)
+			dc->hwss.commit_subvp_config(dc, context);
+	/* Since phantom pipe programming is moved to post_unlock_program_front_end,
+	 * move the SubVP lock to after the phantom pipes have been setup
+	 */
+	if (should_lock_all_pipes && dc->hwss.interdependent_update_lock) {
+		if (dc->hwss.subvp_pipe_control_lock)
+			dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes, NULL, subvp_prev_use);
+		if (dc->hwss.dmub_hw_control_lock)
+			dc->hwss.dmub_hw_control_lock(dc, context, false);
+	} else {
+		if (dc->hwss.subvp_pipe_control_lock)
+			dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes, top_pipe_to_program, subvp_prev_use);
+		if (dc->hwss.dmub_hw_control_lock)
+			dc->hwss.dmub_hw_control_lock(dc, context, false);
+	}
+
+	// Fire manual trigger only when bottom plane is flipped
+	for (j = 0; j < dc->res_pool->pipe_count; j++) {
+		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+		if (!pipe_ctx->plane_state)
+			continue;
+
+		if (pipe_ctx->bottom_pipe || pipe_ctx->next_odm_pipe ||
+				!pipe_ctx->stream || !should_update_pipe_for_stream(context, pipe_ctx, stream) ||
+				!pipe_ctx->plane_state->update_bits.addr_update ||
+				pipe_ctx->plane_state->skip_manual_trigger)
+			continue;
+
+		if (dc->hwss.program_cursor_offload_now)
+			dc->hwss.program_cursor_offload_now(dc, pipe_ctx);
+		if (pipe_ctx->stream_res.tg->funcs->program_manual_trigger)
+			pipe_ctx->stream_res.tg->funcs->program_manual_trigger(pipe_ctx->stream_res.tg);
+	}
+
+	current_stream_mask = get_stream_mask(dc, context);
+	if (current_stream_mask != context->stream_mask) {
+		context->stream_mask = current_stream_mask;
+		dc_dmub_srv_notify_stream_mask(dc->ctx->dmub_srv, current_stream_mask);
+	}
+}
+
+/**
+ * could_mpcc_tree_change_for_active_pipes - Check if an OPP associated with MPCC might change
+ *
+ * @dc: Used to get the current state status
+ * @stream: Target stream, which we want to remove the attached planes
+ * @srf_updates: Array of surface updates
+ * @surface_count: Number of surface update
+ * @is_plane_addition: [in] Fill out with true if it is a plane addition case
+ *
+ * DCN32x and newer support a feature named Dynamic ODM which can conflict with
+ * the MPO if used simultaneously in some specific configurations (e.g.,
+ * 4k@144). This function checks if the incoming context requires applying a
+ * transition state with unnecessary pipe splitting and ODM disabled to
+ * circumvent our hardware limitations to prevent this edge case. If the OPP
+ * associated with an MPCC might change due to plane additions, this function
+ * returns true.
+ *
+ * Return:
+ * Return true if OPP and MPCC might change, otherwise, return false.
+ */
+static bool could_mpcc_tree_change_for_active_pipes(struct dc *dc,
+		struct dc_stream_state *stream,
+		struct dc_surface_update *srf_updates,
+		int surface_count,
+		bool *is_plane_addition)
+{
+	(void)srf_updates;
+
+	struct dc_stream_status *cur_stream_status = stream_get_status(dc->current_state, stream);
+	bool force_minimal_pipe_splitting = false;
+	bool subvp_active = false;
+	uint32_t i;
+
+	*is_plane_addition = false;
+
+	if (cur_stream_status &&
+			dc->current_state->stream_count > 0 &&
+			dc->debug.pipe_split_policy != MPC_SPLIT_AVOID) {
+		/* determine if minimal transition is required due to MPC*/
+		if (surface_count > 0) {
+			if (cur_stream_status->plane_count > surface_count) {
+				force_minimal_pipe_splitting = true;
+			} else if (cur_stream_status->plane_count < surface_count) {
+				force_minimal_pipe_splitting = true;
+				*is_plane_addition = true;
+			}
+		}
+	}
+
+	if (cur_stream_status &&
+			dc->current_state->stream_count == 1 &&
+			dc->debug.enable_single_display_2to1_odm_policy) {
+		/* determine if minimal transition is required due to dynamic ODM*/
+		if (surface_count > 0) {
+			if (cur_stream_status->plane_count > 2 && cur_stream_status->plane_count > surface_count) {
+				force_minimal_pipe_splitting = true;
+			} else if (surface_count > 2 && cur_stream_status->plane_count < surface_count) {
+				force_minimal_pipe_splitting = true;
+				*is_plane_addition = true;
+			}
+		}
+	}
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+
+		if (dc_state_get_pipe_subvp_type(dc->current_state, pipe) != SUBVP_NONE) {
+			subvp_active = true;
+			break;
+		}
+	}
+
+	/* For SubVP when adding or removing planes we need to add a minimal transition
+	 * (even when disabling all planes). Whenever disabling a phantom pipe, we
+	 * must use the minimal transition path to disable the pipe correctly.
+	 *
+	 * We want to use the minimal transition whenever subvp is active, not only if
+	 * a plane is being added / removed from a subvp stream (MPO plane can be added
+	 * to a DRR pipe of SubVP + DRR config, in which case we still want to run through
+	 * a min transition to disable subvp.
+	 */
+	if (cur_stream_status && subvp_active) {
+		/* determine if minimal transition is required due to SubVP*/
+		if (cur_stream_status->plane_count > surface_count) {
+			force_minimal_pipe_splitting = true;
+		} else if (cur_stream_status->plane_count < surface_count) {
+			force_minimal_pipe_splitting = true;
+			*is_plane_addition = true;
+		}
+	}
+
+	return force_minimal_pipe_splitting;
+}
+
+
+static void release_minimal_transition_state(struct dc *dc,
+		struct dc_state *minimal_transition_context,
+		struct dc_state *base_context,
+		struct pipe_split_policy_backup *policy)
+{
+	restore_minimal_pipe_split_policy(dc, base_context, policy);
+	dc_state_release(minimal_transition_context);
+}
+
+static void force_vsync_flip_in_minimal_transition_context(struct dc_state *context)
+{
+	uint8_t i;
+	int j;
+	struct dc_stream_status *stream_status;
+
+	for (i = 0; i < context->stream_count; i++) {
+		stream_status = &context->stream_status[i];
+
+		for (j = 0; j < stream_status->plane_count; j++)
+			stream_status->plane_states[j]->flip_immediate = false;
+	}
+}
+
+static struct dc_state *create_minimal_transition_state(struct dc *dc,
+		struct dc_state *base_context, struct pipe_split_policy_backup *policy)
+{
+	struct dc_state *minimal_transition_context = NULL;
+
+	minimal_transition_context = dc_state_create_copy(base_context);
+	if (!minimal_transition_context)
+		return NULL;
+
+	backup_and_set_minimal_pipe_split_policy(dc, base_context, policy);
+	/* commit minimal state */
+	if (dc->res_pool->funcs->validate_bandwidth(dc, minimal_transition_context,
+		DC_VALIDATE_MODE_AND_PROGRAMMING) == DC_OK) {
+		/* prevent underflow and corruption when reconfiguring pipes */
+		force_vsync_flip_in_minimal_transition_context(minimal_transition_context);
+	} else {
+		/*
+		 * This should never happen, minimal transition state should
+		 * always be validated first before adding pipe split features.
+		 */
+		release_minimal_transition_state(dc, minimal_transition_context, base_context, policy);
+		BREAK_TO_DEBUGGER();
+		minimal_transition_context = NULL;
+	}
+	return minimal_transition_context;
+}
+
+static bool is_pipe_topology_transition_seamless_with_intermediate_step(
+		struct dc *dc,
+		struct dc_state *initial_state,
+		struct dc_state *intermediate_state,
+		struct dc_state *final_state)
+{
+	return dc->hwss.is_pipe_topology_transition_seamless(dc, initial_state,
+			intermediate_state) &&
+			dc->hwss.is_pipe_topology_transition_seamless(dc,
+					intermediate_state, final_state);
+}
+
+static void swap_and_release_current_context(struct dc *dc,
+		struct dc_state *new_context, struct dc_stream_state *stream)
+{
+
+	unsigned int i;
+	struct dc_state *old = dc->current_state;
+	struct pipe_ctx *pipe_ctx;
+
+	/* Since memory free requires elevated IRQ, an interrupt
+	 * request is generated by mem free. If this happens
+	 * between freeing and reassigning the context, our vsync
+	 * interrupt will call into dc and cause a memory
+	 * corruption. Hence, we first reassign the context,
+	 * then free the old context.
+	 */
+	dc->current_state = new_context;
+	dc_state_release(old);
+
+	// clear any forced full updates
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		pipe_ctx = &new_context->res_ctx.pipe_ctx[i];
+
+		if (pipe_ctx->plane_state && pipe_ctx->stream == stream)
+			pipe_ctx->plane_state->force_full_update = false;
+	}
+}
+
+static int initialize_empty_surface_updates(
+		struct dc_stream_state *stream,
+		struct dc_surface_update *srf_updates)
+{
+	struct dc_stream_status *status = dc_stream_get_status(stream);
+	int i;
+
+	if (!status)
+		return 0;
+
+	for (i = 0; i < status->plane_count; i++)
+		srf_updates[i].surface = status->plane_states[i];
+
+	return status->plane_count;
+}
+
+static bool commit_minimal_transition_based_on_new_context(struct dc *dc,
+		struct dc_state *new_context,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update,
+		struct dc_surface_update *srf_updates,
+		int surface_count)
+{
+	bool success = false;
+	struct pipe_split_policy_backup policy;
+	struct dc_state *intermediate_context =
+			create_minimal_transition_state(dc, new_context,
+					&policy);
+
+	if (intermediate_context) {
+		if (is_pipe_topology_transition_seamless_with_intermediate_step(
+				dc,
+				dc->current_state,
+				intermediate_context,
+				new_context)) {
+			DC_LOG_DC("commit minimal transition state: base = new state\n");
+			commit_planes_for_stream(dc, srf_updates,
+					surface_count, stream, stream_update,
+					UPDATE_TYPE_FULL, intermediate_context);
+			swap_and_release_current_context(
+					dc, intermediate_context, stream);
+			dc_state_retain(dc->current_state);
+			success = true;
+		}
+		release_minimal_transition_state(
+				dc, intermediate_context, new_context, &policy);
+	}
+	return success;
+}
+
+static bool commit_minimal_transition_based_on_current_context(struct dc *dc,
+		struct dc_state *new_context, struct dc_stream_state *stream)
+{
+	bool success = false;
+	struct pipe_split_policy_backup policy;
+	struct dc_state *intermediate_context;
+	struct dc_state *old_current_state = dc->current_state;
+	struct dc_surface_update srf_updates[MAX_SURFACES] = {0};
+	int surface_count;
+
+	/*
+	 * Both current and new contexts share the same stream and plane state
+	 * pointers. When new context is validated, stream and planes get
+	 * populated with new updates such as new plane addresses. This makes
+	 * the current context no longer valid because stream and planes are
+	 * modified from the original. We backup current stream and plane states
+	 * into scratch space whenever we are populating new context. So we can
+	 * restore the original values back by calling the restore function now.
+	 * This restores back the original stream and plane states associated
+	 * with the current state.
+	 */
+	restore_planes_and_stream_state(&dc->scratch.current_state, stream);
+	dc_state_retain(old_current_state);
+	intermediate_context = create_minimal_transition_state(dc,
+			old_current_state, &policy);
+
+	if (intermediate_context) {
+		if (is_pipe_topology_transition_seamless_with_intermediate_step(
+				dc,
+				dc->current_state,
+				intermediate_context,
+				new_context)) {
+			DC_LOG_DC("commit minimal transition state: base = current state\n");
+			surface_count = initialize_empty_surface_updates(
+					stream, srf_updates);
+			commit_planes_for_stream(dc, srf_updates,
+					surface_count, stream, NULL,
+					UPDATE_TYPE_FULL, intermediate_context);
+			swap_and_release_current_context(
+					dc, intermediate_context, stream);
+			dc_state_retain(dc->current_state);
+			success = true;
+		}
+		release_minimal_transition_state(dc, intermediate_context,
+				old_current_state, &policy);
+	}
+	dc_state_release(old_current_state);
+	/*
+	 * Restore stream and plane states back to the values associated with
+	 * new context.
+	 */
+	restore_planes_and_stream_state(&dc->scratch.new_state, stream);
+	return success;
+}
+
+/**
+ * commit_minimal_transition_state_in_dc_update - Commit a minimal state based
+ * on current or new context
+ *
+ * @dc: DC structure, used to get the current state
+ * @new_context: New context
+ * @stream: Stream getting the update for the flip
+ * @srf_updates: Surface updates
+ * @surface_count: Number of surfaces
+ *
+ * The function takes in current state and new state and determine a minimal
+ * transition state as the intermediate step which could make the transition
+ * between current and new states seamless. If found, it will commit the minimal
+ * transition state and update current state to this minimal transition state
+ * and return true, if not, it will return false.
+ *
+ * Return:
+ * Return True if the minimal transition succeeded, false otherwise
+ */
+static bool commit_minimal_transition_state_in_dc_update(struct dc *dc,
+		struct dc_state *new_context,
+		struct dc_stream_state *stream,
+		struct dc_surface_update *srf_updates,
+		int surface_count)
+{
+	bool success = commit_minimal_transition_based_on_new_context(
+				dc, new_context, stream, NULL,
+				srf_updates, surface_count);
+	if (!success)
+		success = commit_minimal_transition_based_on_current_context(dc,
+				new_context, stream);
+	if (!success)
+		DC_LOG_ERROR("Fail to commit a seamless minimal transition state between current and new states.\nThis pipe topology update is non-seamless!\n");
+	return success;
+}
+
+/**
+ * commit_minimal_transition_state - Create a transition pipe split state
+ *
+ * @dc: Used to get the current state status
+ * @transition_base_context: New transition state
+ *
+ * In some specific configurations, such as pipe split on multi-display with
+ * MPO and/or Dynamic ODM, removing a plane may cause unsupported pipe
+ * programming when moving to new planes. To mitigate those types of problems,
+ * this function adds a transition state that minimizes pipe usage before
+ * programming the new configuration. When adding a new plane, the current
+ * state requires the least pipes, so it is applied without splitting. When
+ * removing a plane, the new state requires the least pipes, so it is applied
+ * without splitting.
+ *
+ * Return:
+ * Return false if something is wrong in the transition state.
+ */
+static bool commit_minimal_transition_state(struct dc *dc,
+		struct dc_state *transition_base_context)
+{
+	struct dc_state *transition_context;
+	struct pipe_split_policy_backup policy;
+	enum dc_status ret = DC_ERROR_UNEXPECTED;
+	unsigned int i, j;
+	unsigned int pipe_in_use = 0;
+	bool subvp_in_use = false;
+	bool odm_in_use = false;
+
+	/* check current pipes in use*/
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &transition_base_context->res_ctx.pipe_ctx[i];
+
+		if (pipe->plane_state)
+			pipe_in_use++;
+	}
+
+	/* If SubVP is enabled and we are adding or removing planes from any main subvp
+	 * pipe, we must use the minimal transition.
+	 */
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+
+		if (pipe->stream && dc_state_get_pipe_subvp_type(dc->current_state, pipe) == SUBVP_PHANTOM) {
+			subvp_in_use = true;
+			break;
+		}
+	}
+
+	/* If ODM is enabled and we are adding or removing planes from any ODM
+	 * pipe, we must use the minimal transition.
+	 */
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &transition_base_context->res_ctx.pipe_ctx[i];
+
+		if (resource_is_pipe_type(pipe, OTG_MASTER)) {
+			odm_in_use = resource_get_odm_slice_count(pipe) > 1;
+			break;
+		}
+	}
+
+	/* When the OS add a new surface if we have been used all of pipes with odm combine
+	 * and mpc split feature, it need use commit_minimal_transition_state to transition safely.
+	 * After OS exit MPO, it will back to use odm and mpc split with all of pipes, we need
+	 * call it again. Otherwise return true to skip.
+	 *
+	 * Reduce the scenarios to use dc_commit_state_no_check in the stage of flip. Especially
+	 * enter/exit MPO when DCN still have enough resources.
+	 */
+	if (pipe_in_use != dc->res_pool->pipe_count && !subvp_in_use && !odm_in_use)
+		return true;
+
+	DC_LOG_DC("%s base = %s state, reason = %s\n", __func__,
+			dc->current_state == transition_base_context ? "current" : "new",
+			subvp_in_use ? "Subvp In Use" :
+			odm_in_use ? "ODM in Use" :
+			dc->debug.pipe_split_policy != MPC_SPLIT_AVOID ? "MPC in Use" :
+			"Unknown");
+
+	dc_state_retain(transition_base_context);
+	transition_context = create_minimal_transition_state(dc,
+			transition_base_context, &policy);
+	if (transition_context) {
+		ret = dc_commit_state_no_check(dc, transition_context);
+		release_minimal_transition_state(dc, transition_context, transition_base_context, &policy);
+	}
+	dc_state_release(transition_base_context);
+
+	if (ret != DC_OK) {
+		/* this should never happen */
+		BREAK_TO_DEBUGGER();
+		return false;
+	}
+
+	/* force full surface update */
+	for (i = 0; i < dc->current_state->stream_count; i++) {
+		for (j = 0; j < (unsigned int)dc->current_state->stream_status[i].plane_count; j++) {
+			dc_pipe_update_bits_set_full(&dc->current_state->stream_status[i].plane_states[j]->update_bits);
+		}
+	}
+
+	return true;
+}
+
+void populate_fast_updates(struct dc_fast_update *fast_update,
+		struct dc_surface_update *srf_updates,
+		int surface_count,
+		struct dc_stream_update *stream_update)
+{
+	int i = 0;
+
+	if (stream_update) {
+		fast_update[0].out_transfer_func = stream_update->out_transfer_func;
+		fast_update[0].output_csc_transform = stream_update->output_csc_transform;
+		fast_update[0].cursor_attributes = stream_update->cursor_attributes;
+		fast_update[0].cursor_position = stream_update->cursor_position;
+		fast_update[0].periodic_interrupt = stream_update->periodic_interrupt;
+		fast_update[0].dither_option = stream_update->dither_option;
+		fast_update[0].gamut_remap = stream_update->gamut_remap;
+		fast_update[0].vrr_infopacket = stream_update->vrr_infopacket;
+		fast_update[0].vsc_infopacket = stream_update->vsc_infopacket;
+		fast_update[0].vsp_infopacket = stream_update->vsp_infopacket;
+		fast_update[0].hfvsif_infopacket = stream_update->hfvsif_infopacket;
+		fast_update[0].vtem_infopacket = stream_update->vtem_infopacket;
+		fast_update[0].adaptive_sync_infopacket = stream_update->adaptive_sync_infopacket;
+		fast_update[0].avi_infopacket = stream_update->avi_infopacket;
+		fast_update[0].hdr_static_metadata = stream_update->hdr_static_metadata;
+	} else {
+		fast_update[0].out_transfer_func = NULL;
+		fast_update[0].output_csc_transform = NULL;
+		fast_update[0].cursor_attributes = NULL;
+		fast_update[0].cursor_position = NULL;
+		fast_update[0].periodic_interrupt = NULL;
+		fast_update[0].dither_option = NULL;
+		fast_update[0].gamut_remap = NULL;
+		fast_update[0].vrr_infopacket = NULL;
+		fast_update[0].vsc_infopacket = NULL;
+		fast_update[0].vsp_infopacket = NULL;
+		fast_update[0].hfvsif_infopacket = NULL;
+		fast_update[0].vtem_infopacket = NULL;
+		fast_update[0].adaptive_sync_infopacket = NULL;
+		fast_update[0].avi_infopacket = NULL;
+		fast_update[0].hdr_static_metadata = NULL;
+	}
+
+	for (i = 0; i < surface_count; i++) {
+		fast_update[i].flip_addr = srf_updates[i].flip_addr;
+		fast_update[i].gamma = srf_updates[i].gamma;
+		fast_update[i].gamut_remap_matrix = srf_updates[i].gamut_remap_matrix;
+		fast_update[i].input_csc_color_matrix = srf_updates[i].input_csc_color_matrix;
+		fast_update[i].coeff_reduction_factor = srf_updates[i].coeff_reduction_factor;
+		fast_update[i].cursor_csc_color_matrix = srf_updates[i].cursor_csc_color_matrix;
+		fast_update[i].cm_hist_control = srf_updates[i].cm_hist_control;
+	}
+}
+
+static bool fast_updates_exist(const struct dc_fast_update *fast_update, int surface_count)
+{
+	int i;
+
+	if (fast_update[0].out_transfer_func ||
+		fast_update[0].output_csc_transform ||
+		fast_update[0].cursor_attributes ||
+		fast_update[0].cursor_position ||
+		fast_update[0].periodic_interrupt ||
+		fast_update[0].dither_option ||
+		fast_update[0].gamut_remap ||
+		fast_update[0].vrr_infopacket ||
+		fast_update[0].vsc_infopacket ||
+		fast_update[0].vsp_infopacket ||
+		fast_update[0].hfvsif_infopacket ||
+		fast_update[0].vtem_infopacket ||
+		fast_update[0].adaptive_sync_infopacket ||
+		fast_update[0].avi_infopacket ||
+		fast_update[0].hdr_static_metadata)
+		return true;
+
+	for (i = 0; i < surface_count; i++) {
+		if (fast_update[i].flip_addr ||
+				fast_update[i].gamma ||
+				fast_update[i].gamut_remap_matrix ||
+				fast_update[i].input_csc_color_matrix ||
+				fast_update[i].cursor_csc_color_matrix ||
+				fast_update[i].cm_hist_control ||
+				fast_update[i].coeff_reduction_factor)
+			return true;
+	}
+
+	return false;
+}
+
+bool fast_nonaddr_updates_exist(struct dc_fast_update *fast_update, int surface_count)
+{
+	int i;
+
+	if (fast_update[0].out_transfer_func ||
+		fast_update[0].output_csc_transform ||
+		fast_update[0].gamut_remap ||
+		fast_update[0].cursor_attributes ||
+		fast_update[0].cursor_position ||
+		fast_update[0].periodic_interrupt ||
+		fast_update[0].dither_option ||
+		fast_update[0].vrr_infopacket ||
+		fast_update[0].vsc_infopacket ||
+		fast_update[0].vsp_infopacket ||
+		fast_update[0].hfvsif_infopacket ||
+		fast_update[0].vtem_infopacket ||
+		fast_update[0].adaptive_sync_infopacket ||
+		fast_update[0].avi_infopacket ||
+		fast_update[0].hdr_static_metadata)
+		return true;
+
+	for (i = 0; i < surface_count; i++) {
+		if (fast_update[i].input_csc_color_matrix ||
+				fast_update[i].gamma ||
+				fast_update[i].gamut_remap_matrix ||
+				fast_update[i].coeff_reduction_factor ||
+				fast_update[i].cm_hist_control ||
+				fast_update[i].cursor_csc_color_matrix)
+			return true;
+	}
+
+	return false;
+}
+
+static bool full_update_required_weak(
+		const struct dc *dc,
+		const struct dc_surface_update *srf_updates,
+		int surface_count,
+		const struct dc_stream_update *stream_update,
+		const struct dc_stream_state *stream)
+{
+	(void)stream_update;
+	const struct dc_state *context = dc->current_state;
+	if (srf_updates)
+		for (int i = 0; i < surface_count; i++)
+			if (!is_surface_in_context(context, srf_updates[i].surface))
+				return true;
+
+	if (stream) {
+		const struct dc_stream_status *stream_status = dc_stream_get_status_const(stream);
+		if (stream_status == NULL || stream_status->plane_count != surface_count)
+			return true;
+	}
+	if (dc->idle_optimizations_allowed)
+		return true;
+
+	if (dc_can_clear_cursor_limit(dc))
+		return true;
+
+	return false;
+}
+
+static bool full_update_required(
+		const struct dc *dc,
+		const struct dc_surface_update *srf_updates,
+		int surface_count,
+		const struct dc_stream_update *stream_update,
+		const struct dc_stream_state *stream)
+{
+	if (full_update_required_weak(dc, srf_updates, surface_count, stream_update, stream))
+		return true;
+
+	for (int i = 0; i < surface_count; i++) {
+		if (srf_updates &&
+				(srf_updates[i].plane_info ||
+				srf_updates[i].scaling_info ||
+				(srf_updates[i].hdr_mult.value &&
+				srf_updates[i].hdr_mult.value != srf_updates->surface->hdr_mult.value) ||
+				(srf_updates[i].sdr_white_level_nits &&
+				srf_updates[i].sdr_white_level_nits != srf_updates->surface->sdr_white_level_nits) ||
+				srf_updates[i].in_transfer_func ||
+				srf_updates[i].surface->force_full_update ||
+				(srf_updates[i].flip_addr &&
+				srf_updates[i].flip_addr->address.tmz_surface != srf_updates[i].surface->address.tmz_surface)))
+			return true;
+	}
+
+	if (stream_update &&
+			(((stream_update->src.height != 0 && stream_update->src.width != 0) ||
+			(stream_update->dst.height != 0 && stream_update->dst.width != 0) ||
+			stream_update->integer_scaling_update) ||
+			stream_update->abm_level ||
+			stream_update->dpms_off ||
+			stream_update->allow_freesync ||
+			stream_update->vrr_active_variable ||
+			stream_update->vrr_active_fixed ||
+			stream_update->output_color_space ||
+			stream_update->wb_update ||
+			stream_update->dsc_config ||
+			stream_update->mst_bw_update ||
+			stream_update->func_shaper ||
+			stream_update->lut3d_func ||
+			stream_update->pending_test_pattern ||
+			stream_update->crtc_timing_adjust ||
+			stream_update->scaler_sharpener_update ||
+			stream_update->hw_cursor_req))
+		return true;
+
+	return false;
+}
+
+static bool fast_update_only(
+		const struct dc *dc,
+		const struct dc_fast_update *fast_update,
+		const struct dc_surface_update *srf_updates,
+		int surface_count,
+		const struct dc_stream_update *stream_update,
+		const struct dc_stream_state *stream)
+{
+	return fast_updates_exist(fast_update, surface_count)
+			&& !full_update_required(dc, srf_updates, surface_count, stream_update, stream);
+}
+
+static bool update_planes_and_stream_v2(struct dc *dc,
+		struct dc_surface_update *srf_updates, int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update)
+{
+	struct dc_state *context;
+	enum surface_update_type update_type;
+	struct dc_fast_update fast_update[MAX_SURFACES] = {0};
+
+	/* In cases where MPO and split or ODM are used transitions can
+	 * cause underflow. Apply stream configuration with minimal pipe
+	 * split first to avoid unsupported transitions for active pipes.
+	 */
+	bool force_minimal_pipe_splitting = 0;
+	bool is_plane_addition = 0;
+	bool is_fast_update_only;
+
+	populate_fast_updates(fast_update, srf_updates, surface_count, stream_update);
+	is_fast_update_only = fast_update_only(dc, fast_update, srf_updates,
+			surface_count, stream_update, stream);
+	force_minimal_pipe_splitting = could_mpcc_tree_change_for_active_pipes(
+			dc,
+			stream,
+			srf_updates,
+			surface_count,
+			&is_plane_addition);
+
+	/* on plane addition, minimal state is the current one */
+	if (force_minimal_pipe_splitting && is_plane_addition &&
+		!commit_minimal_transition_state(dc, dc->current_state))
+		return false;
+
+	if (!update_planes_and_stream_state(
+			dc,
+			srf_updates,
+			surface_count,
+			stream,
+			stream_update,
+			&update_type,
+			&context))
+		return false;
+
+	/* on plane removal, minimal state is the new one */
+	if (force_minimal_pipe_splitting && !is_plane_addition) {
+		if (!commit_minimal_transition_state(dc, context)) {
+			dc_state_release(context);
+			return false;
+		}
+		update_type = UPDATE_TYPE_FULL;
+	}
+
+	if (dc->hwss.is_pipe_topology_transition_seamless &&
+			!dc->hwss.is_pipe_topology_transition_seamless(
+					dc, dc->current_state, context))
+		commit_minimal_transition_state_in_dc_update(dc, context, stream,
+				srf_updates, surface_count);
+
+	if (is_fast_update_only && !dc->check_config.enable_legacy_fast_update) {
+		commit_planes_for_stream_fast(dc,
+				srf_updates,
+				surface_count,
+				stream,
+				stream_update,
+				update_type,
+				context);
+	} else {
+		if (!stream_update &&
+				dc->hwss.is_pipe_topology_transition_seamless &&
+				!dc->hwss.is_pipe_topology_transition_seamless(
+						dc, dc->current_state, context)) {
+			DC_LOG_ERROR("performing non-seamless pipe topology transition with surface only update!\n");
+			BREAK_TO_DEBUGGER();
+		}
+		commit_planes_for_stream(
+				dc,
+				srf_updates,
+				surface_count,
+				stream,
+				stream_update,
+				update_type,
+				context);
+	}
+	if (dc->current_state != context)
+		swap_and_release_current_context(dc, context, stream);
+	return true;
+}
+
+static void commit_planes_and_stream_update_on_current_context(struct dc *dc,
+		struct dc_surface_update *srf_updates, int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update,
+		enum surface_update_type update_type)
+{
+	struct dc_fast_update fast_update[MAX_SURFACES] = {0};
+
+	ASSERT(update_type < UPDATE_TYPE_FULL);
+	populate_fast_updates(fast_update, srf_updates, surface_count,
+			stream_update);
+	if (fast_update_only(dc, fast_update, srf_updates, surface_count,
+			stream_update, stream) &&
+			!dc->check_config.enable_legacy_fast_update)
+		commit_planes_for_stream_fast(dc,
+				srf_updates,
+				surface_count,
+				stream,
+				stream_update,
+				update_type,
+				dc->current_state);
+	else
+		commit_planes_for_stream(
+				dc,
+				srf_updates,
+				surface_count,
+				stream,
+				stream_update,
+				update_type,
+				dc->current_state);
+}
+
+static void commit_planes_and_stream_update_with_new_context(struct dc *dc,
+		struct dc_surface_update *srf_updates, int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update,
+		enum surface_update_type update_type,
+		struct dc_state *new_context)
+{
+	bool skip_new_context = false;
+	ASSERT(update_type >= UPDATE_TYPE_FULL);
+	/*
+	 * It is required by the feature design that all pipe topologies
+	 * using extra free pipes for power saving purposes such as
+	 * dynamic ODM or SubVp shall only be enabled when it can be
+	 * transitioned seamlessly to AND from its minimal transition
+	 * state. A minimal transition state is defined as the same dc
+	 * state but with all power saving features disabled. So it uses
+	 * the minimum pipe topology. When we can't seamlessly
+	 * transition from state A to state B, we will insert the
+	 * minimal transition state A' or B' in between so seamless
+	 * transition between A and B can be made possible.
+	 *
+	 * To optimize for the time it takes to execute flips,
+	 * the transition from the minimal state to the final state is
+	 * deferred until a steady state (no more transitions) is reached.
+	 */
+	if (!dc->hwss.is_pipe_topology_transition_seamless(dc, dc->current_state, new_context)) {
+		if (!dc->debug.disable_deferred_minimal_transitions) {
+			dc->check_config.deferred_transition_state = true;
+			dc->check_config.transition_countdown_to_steady_state =
+					dc->debug.num_fast_flips_to_steady_state_override ?
+					dc->debug.num_fast_flips_to_steady_state_override :
+					NUM_FAST_FLIPS_TO_STEADY_STATE;
+
+			if (commit_minimal_transition_based_on_new_context(dc, new_context, stream, stream_update,
+					srf_updates, surface_count)) {
+				skip_new_context = true;
+				dc_state_release(new_context);
+				new_context = dc->current_state;
+			} else {
+				/*
+				 * In this case a new mpo plane is being enabled on pipes that were
+				 * previously in use, and the surface update to the existing plane
+				 * includes an alpha box where the new plane will be, so the update
+				 * from minimal to final cannot be deferred as the alpha box would
+				 * be visible to the user
+				 */
+				commit_minimal_transition_based_on_current_context(dc, new_context, stream);
+			}
+		} else {
+			commit_minimal_transition_state_in_dc_update(dc, new_context, stream,
+					srf_updates, surface_count);
+		}
+	} else if (dc->check_config.deferred_transition_state) {
+		/* reset countdown as steady state not reached */
+		dc->check_config.transition_countdown_to_steady_state =
+				dc->debug.num_fast_flips_to_steady_state_override ?
+				dc->debug.num_fast_flips_to_steady_state_override :
+				NUM_FAST_FLIPS_TO_STEADY_STATE;
+	}
+
+	if (!skip_new_context) {
+		commit_planes_for_stream(dc, srf_updates, surface_count, stream, stream_update, update_type, new_context);
+		swap_and_release_current_context(dc, new_context, stream);
+	}
+}
+
+static bool update_planes_and_stream_v3(struct dc *dc,
+		struct dc_surface_update *srf_updates, int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update)
+{
+	struct dc_state *new_context;
+	enum surface_update_type update_type;
+
+	/*
+	 * When this function returns true and new_context is not equal to
+	 * current state, the function allocates and validates a new dc state
+	 * and assigns it to new_context. The function expects that the caller
+	 * is responsible to free this memory when new_context is no longer
+	 * used. We swap current with new context and free current instead. So
+	 * new_context's memory will live until the next full update after it is
+	 * replaced by a newer context. Refer to the use of
+	 * swap_and_free_current_context below.
+	 */
+	if (!update_planes_and_stream_state(dc, srf_updates, surface_count,
+				stream, stream_update, &update_type,
+				&new_context))
+		return false;
+
+	if (new_context == dc->current_state) {
+		commit_planes_and_stream_update_on_current_context(dc,
+				srf_updates, surface_count, stream,
+				stream_update, update_type);
+
+		if (dc->check_config.transition_countdown_to_steady_state)
+			dc->check_config.transition_countdown_to_steady_state--;
+	} else {
+		commit_planes_and_stream_update_with_new_context(dc,
+				srf_updates, surface_count, stream,
+				stream_update, update_type, new_context);
+	}
+
+	return true;
+}
+
+static void clear_update_bits(struct dc_surface_update *srf_updates,
+	int surface_count, struct dc_stream_state *stream)
+{
+	int i;
+
+	if (stream)
+		stream_update_flags_clear(&stream->update_flags);
+
+	for (i = 0; i < surface_count; i++)
+		if (srf_updates[i].surface)
+			dc_pipe_update_bits_clear(&srf_updates[i].surface->update_bits);
+}
+
+bool dc_update_planes_and_stream(struct dc *dc,
+		struct dc_surface_update *srf_updates, int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update)
+{
+	struct dc_update_scratch_space *scratch = dc_update_planes_and_stream_init(
+			dc,
+			srf_updates,
+			surface_count,
+			stream,
+			stream_update
+	);
+	bool more = true;
+
+	while (more) {
+		if (!dc_update_planes_and_stream_prepare(scratch))
+			return false;
+
+		dc_update_planes_and_stream_execute(scratch);
+		more = dc_update_planes_and_stream_cleanup(scratch);
+	}
+	return true;
+}
+
+void dc_commit_updates_for_stream(struct dc *dc,
+		struct dc_surface_update *srf_updates,
+		int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update,
+		struct dc_state *state)
+{
+	(void)state;
+	bool ret = false;
+
+	dc_exit_ips_for_hw_access(dc);
+	/* TODO: Since change commit sequence can have a huge impact,
+	 * we decided to only enable it for DCN3x. However, as soon as
+	 * we get more confident about this change we'll need to enable
+	 * the new sequence for all ASICs.
+	 */
+	if (dc->ctx->dce_version >= DCN_VERSION_4_01) {
+		ret = update_planes_and_stream_v3(dc, srf_updates, surface_count,
+				stream, stream_update);
+	} else {
+		ret = update_planes_and_stream_v2(dc, srf_updates, surface_count,
+				stream, stream_update);
+	}
+
+	if (ret && dc->ctx->dce_version >= DCN_VERSION_3_2)
+		clear_update_bits(srf_updates, surface_count, stream);
+}
+
+uint8_t dc_get_current_stream_count(struct dc *dc)
+{
+	return dc->current_state->stream_count;
+}
+
+struct dc_stream_state *dc_get_stream_at_index(struct dc *dc, uint8_t i)
+{
+	if (i < dc->current_state->stream_count)
+		return dc->current_state->streams[i];
+	return NULL;
+}
+
+enum dc_irq_source dc_interrupt_to_irq_source(
+		struct dc *dc,
+		uint32_t src_id,
+		uint32_t ext_id)
+{
+	return dal_irq_service_to_irq_source(dc->res_pool->irqs, src_id, ext_id);
+}
+
+/*
+ * dc_interrupt_set() - Enable/disable an AMD hw interrupt source
+ */
+bool dc_interrupt_set(struct dc *dc, enum dc_irq_source src, bool enable)
+{
+
+	if (dc == NULL)
+		return false;
+
+	return dal_irq_service_set(dc->res_pool->irqs, src, enable);
+}
+
+void dc_interrupt_ack(struct dc *dc, enum dc_irq_source src)
+{
+	dal_irq_service_ack(dc->res_pool->irqs, src);
+}
+
+/*
+ * dc_get_flip_pending_on_otg() - Check if a GRPH_FLIP is still pending on OTG
+ *
+ * @dc: display core context @otg_inst: OTG instance to query
+ *
+ * Reads the HUBP flip-pending status for the pipe(s) bound to @otg_inst,
+ * returning true if any of them has not yet latched its programmed surface
+ * address.
+ *
+ * Unlike dc_plane_get_status(), this does not take or mutate a dc_plane_state,
+ * so it is safe to call from interrupt context without racing a concurrent
+ * commit that may be updating plane state.
+ *
+ * Return: true if a flip is still pending on the OTG, false otherwise.
+ */
+bool dc_get_flip_pending_on_otg(struct dc *dc, int otg_inst)
+{
+	bool flip_pending = false;
+	unsigned int i;
+
+	if (!dc || !dc->current_state)
+		return false;
+
+	dc_exit_ips_for_hw_access(dc);
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
+		struct hubp *hubp = pipe_ctx->plane_res.hubp;
+
+		if (!pipe_ctx->plane_state || !pipe_ctx->stream_res.tg)
+			continue;
+
+		if (pipe_ctx->stream_res.tg->inst != otg_inst)
+			continue;
+
+		if (hubp && hubp->funcs->hubp_is_flip_pending &&
+		    hubp->funcs->hubp_is_flip_pending(hubp)) {
+			flip_pending = true;
+			break;
+		}
+	}
+
+	return flip_pending;
+}
+
+void dc_power_down_on_boot(struct dc *dc)
+{
+	if (dc->ctx->dce_environment != DCE_ENV_VIRTUAL_HW &&
+	    dc->hwss.power_down_on_boot) {
+		if (dc->current_state->stream_count > 0)
+			return;
+
+		if (dc->caps.ips_support)
+			dc_exit_ips_for_hw_access(dc);
+		dc->hwss.power_down_on_boot(dc);
+	}
+}
+
+void dc_set_power_state(struct dc *dc, enum dc_acpi_cm_power_state power_state)
+{
+	if (!dc->current_state)
+		return;
+
+	dc_exit_ips_for_hw_access(dc);
+
+	switch (power_state) {
+	case DC_ACPI_CM_POWER_STATE_D0:
+		dc_state_construct(dc, dc->current_state);
+
+		dc_z10_restore(dc);
+
+		dc_dmub_srv_notify_fw_dc_power_state(dc->ctx->dmub_srv, power_state);
+
+		dc->hwss.init_hw(dc);
+
+		if (dc->hwss.init_sys_ctx != NULL &&
+			dc->vm_pa_config.valid) {
+			dc->hwss.init_sys_ctx(dc->hwseq, dc, &dc->vm_pa_config);
+		}
+		break;
+	case DC_ACPI_CM_POWER_STATE_D3:
+		if (dc->caps.ips_support)
+			dc_dmub_srv_notify_fw_dc_power_state(dc->ctx->dmub_srv, DC_ACPI_CM_POWER_STATE_D3);
+
+		if (dc->caps.ips_v2_support) {
+			if (dc->clk_mgr->funcs->set_low_power_state)
+				dc->clk_mgr->funcs->set_low_power_state(dc->clk_mgr);
+		}
+		break;
+	default:
+		ASSERT(dc->current_state->stream_count == 0);
+		dc_dmub_srv_notify_fw_dc_power_state(dc->ctx->dmub_srv, power_state);
+
+		dc_state_destruct(dc->current_state);
+
+		break;
+	}
+}
+
+void dc_resume(struct dc *dc)
+{
+	uint32_t i;
+
+	for (i = 0; i < dc->link_count; i++)
+		dc->link_srv->resume(dc->links[i]);
+}
+
+bool dc_is_dmcu_initialized(struct dc *dc)
+{
+	struct dmcu *dmcu = dc->res_pool->dmcu;
+
+	if (dmcu)
+		return dmcu->funcs->is_dmcu_initialized(dmcu);
+	return false;
+}
+
+enum dc_status dc_set_clock(struct dc *dc, enum dc_clock_type clock_type, uint32_t clk_khz, uint32_t stepping)
+{
+	if (dc->hwss.set_clock)
+		return dc->hwss.set_clock(dc, clock_type, clk_khz, stepping);
+	return DC_ERROR_UNEXPECTED;
+}
+void dc_get_clock(struct dc *dc, enum dc_clock_type clock_type, struct dc_clock_config *clock_cfg)
+{
+	if (dc->hwss.get_clock)
+		dc->hwss.get_clock(dc, clock_type, clock_cfg);
+}
+
+/* enable/disable eDP PSR without specify stream for eDP */
+bool dc_set_psr_allow_active(struct dc *dc, bool enable)
+{
+	int i;
+	bool allow_active;
+
+	for (i = 0; i < dc->current_state->stream_count ; i++) {
+		struct dc_link *link;
+		struct dc_stream_state *stream = dc->current_state->streams[i];
+
+		link = stream->link;
+		if (!link)
+			continue;
+
+		if (link->psr_settings.psr_feature_enabled) {
+			if (enable && !link->psr_settings.psr_allow_active) {
+				allow_active = true;
+				if (!dc_link_set_psr_allow_active(link, &allow_active, false, false, NULL))
+					return false;
+			} else if (!enable && link->psr_settings.psr_allow_active) {
+				allow_active = false;
+				if (!dc_link_set_psr_allow_active(link, &allow_active, true, false, NULL))
+					return false;
+			}
+		}
+	}
+
+	return true;
+}
+
+/* enable/disable eDP Replay without specify stream for eDP */
+bool dc_set_replay_allow_active(struct dc *dc, bool active)
+{
+	int i;
+	bool allow_active;
+
+	for (i = 0; i < dc->current_state->stream_count; i++) {
+		struct dc_link *link;
+		struct dc_stream_state *stream = dc->current_state->streams[i];
+
+		link = stream->link;
+		if (!link)
+			continue;
+
+		if (link->replay_settings.replay_feature_enabled) {
+			if (active && !link->replay_settings.replay_allow_active) {
+				allow_active = true;
+				if (!dc_link_set_replay_allow_active(link, &allow_active,
+					false, false, NULL))
+					return false;
+			} else if (!active && link->replay_settings.replay_allow_active) {
+				allow_active = false;
+				if (!dc_link_set_replay_allow_active(link, &allow_active,
+					true, false, NULL))
+					return false;
+			}
+		}
+	}
+
+	return true;
+}
+
+/* set IPS disable state */
+bool dc_set_ips_disable(struct dc *dc, unsigned int disable_ips)
+{
+	dc_exit_ips_for_hw_access(dc);
+
+	dc->config.disable_ips = disable_ips;
+
+	return true;
+}
+
+void dc_allow_idle_optimizations_internal(struct dc *dc, bool allow, char const *caller_name)
+{
+	int idle_fclk_khz = 0, idle_dramclk_khz = 0;
+	unsigned int i = 0;
+	enum mall_stream_type subvp_pipe_type[MAX_PIPES] = {0};
+	struct pipe_ctx *pipe = NULL;
+	struct dc_state *context = dc->current_state;
+
+	if (dc->debug.disable_idle_power_optimizations) {
+		DC_LOG_DEBUG("%s: disabled\n", __func__);
+		return;
+	}
+
+	if (allow != dc->idle_optimizations_allowed)
+		DC_LOG_IPS("%s: allow_idle old=%d new=%d (caller=%s)\n", __func__,
+			   dc->idle_optimizations_allowed, allow, caller_name);
+
+	if (dc->caps.ips_support && (dc->config.disable_ips == DMUB_IPS_DISABLE_ALL))
+		return;
+
+	if (dc->clk_mgr != NULL && dc->clk_mgr->funcs->is_smu_present)
+		if (!dc->clk_mgr->funcs->is_smu_present(dc->clk_mgr))
+			return;
+
+	if (allow == dc->idle_optimizations_allowed)
+		return;
+
+	if (dc->hwss.apply_idle_power_optimizations && dc->clk_mgr != NULL &&
+	    dc->hwss.apply_idle_power_optimizations(dc, allow)) {
+		dc->idle_optimizations_allowed = allow;
+		DC_LOG_DEBUG("%s: %s\n", __func__, allow ? "enabled" : "disabled");
+	}
+
+	// log idle clocks and sub vp pipe types at idle optimization time
+	if (dc->clk_mgr != NULL && dc->clk_mgr->funcs->get_hard_min_fclk)
+		idle_fclk_khz = dc->clk_mgr->funcs->get_hard_min_fclk(dc->clk_mgr);
+
+	if (dc->clk_mgr != NULL && dc->clk_mgr->funcs->get_hard_min_memclk)
+		idle_dramclk_khz = dc->clk_mgr->funcs->get_hard_min_memclk(dc->clk_mgr);
+
+	if (dc->res_pool && context) {
+		for (i = 0; i < dc->res_pool->pipe_count; i++) {
+			pipe = &context->res_ctx.pipe_ctx[i];
+			subvp_pipe_type[i] = dc_state_get_pipe_subvp_type(context, pipe);
+		}
+	}
+	if (!dc->caps.is_apu)
+		DC_LOG_DC("%s: allow_idle=%d\n HardMinUClk_Khz=%d HardMinDramclk_Khz=%d\n Pipe_0=%d Pipe_1=%d Pipe_2=%d Pipe_3=%d Pipe_4=%d Pipe_5=%d (caller=%s)\n",
+			__func__, allow, idle_fclk_khz, idle_dramclk_khz, subvp_pipe_type[0], subvp_pipe_type[1], subvp_pipe_type[2],
+			subvp_pipe_type[3], subvp_pipe_type[4], subvp_pipe_type[5], caller_name);
+
+}
+
+void dc_exit_ips_for_hw_access_internal(struct dc *dc, const char *caller_name)
+{
+	if (dc->caps.ips_support)
+		dc_allow_idle_optimizations_internal(dc, false, caller_name);
+}
+
+bool dc_dmub_is_ips_idle_state(struct dc *dc)
+{
+	if (dc->debug.disable_idle_power_optimizations)
+		return false;
+
+	if (!dc->caps.ips_support || (dc->config.disable_ips == DMUB_IPS_DISABLE_ALL))
+		return false;
+
+	if (!dc->ctx->dmub_srv)
+		return false;
+
+	return dc->ctx->dmub_srv->idle_allowed;
+}
+
+/* set min and max memory clock to lowest and highest DPM level, respectively */
+void dc_unlock_memory_clock_frequency(struct dc *dc)
+{
+	if (dc->clk_mgr->funcs->set_hard_min_memclk)
+		dc->clk_mgr->funcs->set_hard_min_memclk(dc->clk_mgr, false);
+
+	if (dc->clk_mgr->funcs->set_hard_max_memclk)
+		dc->clk_mgr->funcs->set_hard_max_memclk(dc->clk_mgr);
+}
+
+/* set min memory clock to the min required for current mode, max to maxDPM */
+void dc_lock_memory_clock_frequency(struct dc *dc)
+{
+	if (dc->clk_mgr->funcs->get_memclk_states_from_smu)
+		dc->clk_mgr->funcs->get_memclk_states_from_smu(dc->clk_mgr);
+
+	if (dc->clk_mgr->funcs->set_hard_min_memclk)
+		dc->clk_mgr->funcs->set_hard_min_memclk(dc->clk_mgr, true);
+
+	if (dc->clk_mgr->funcs->set_hard_max_memclk)
+		dc->clk_mgr->funcs->set_hard_max_memclk(dc->clk_mgr);
+}
+
+static void blank_and_force_memclk(struct dc *dc, bool apply, unsigned int memclk_mhz)
+{
+	(void)apply;
+	struct dc_state *context = dc->current_state;
+	struct hubp *hubp;
+	struct pipe_ctx *pipe;
+	unsigned int i;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		pipe = &context->res_ctx.pipe_ctx[i];
+
+		if (pipe->stream != NULL) {
+			dc->hwss.disable_pixel_data(dc, pipe, true);
+
+			// wait for double buffer
+			pipe->stream_res.tg->funcs->wait_for_state(pipe->stream_res.tg, CRTC_STATE_VACTIVE);
+			pipe->stream_res.tg->funcs->wait_for_state(pipe->stream_res.tg, CRTC_STATE_VBLANK);
+			pipe->stream_res.tg->funcs->wait_for_state(pipe->stream_res.tg, CRTC_STATE_VACTIVE);
+
+			hubp = pipe->plane_res.hubp;
+			hubp->funcs->set_blank_regs(hubp, true);
+		}
+	}
+	if (dc->clk_mgr->funcs->set_max_memclk)
+		dc->clk_mgr->funcs->set_max_memclk(dc->clk_mgr, memclk_mhz);
+	if (dc->clk_mgr->funcs->set_min_memclk)
+		dc->clk_mgr->funcs->set_min_memclk(dc->clk_mgr, memclk_mhz);
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		pipe = &context->res_ctx.pipe_ctx[i];
+
+		if (pipe->stream != NULL) {
+			dc->hwss.disable_pixel_data(dc, pipe, false);
+
+			hubp = pipe->plane_res.hubp;
+			hubp->funcs->set_blank_regs(hubp, false);
+		}
+	}
+}
+
+
+/**
+ * dc_enable_dcmode_clk_limit() - lower clocks in dc (battery) mode
+ * @dc: pointer to dc of the dm calling this
+ * @enable: True = transition to DC mode, false = transition back to AC mode
+ *
+ * Some SoCs define additional clock limits when in DC mode, DM should
+ * invoke this function when the platform undergoes a power source transition
+ * so DC can apply/unapply the limit. This interface may be disruptive to
+ * the onscreen content.
+ *
+ * Context: Triggered by OS through DM interface, or manually by escape calls.
+ * Need to hold a dclock when doing so.
+ *
+ * Return: none (void function)
+ *
+ */
+void dc_enable_dcmode_clk_limit(struct dc *dc, bool enable)
+{
+	unsigned int softMax = 0, maxDPM = 0, funcMin = 0, i;
+	bool p_state_change_support;
+
+	if (!dc->config.dc_mode_clk_limit_support)
+		return;
+
+	softMax = dc->clk_mgr->bw_params->dc_mode_softmax_memclk;
+	for (i = 0; i < dc->clk_mgr->bw_params->clk_table.num_entries; i++) {
+		if (dc->clk_mgr->bw_params->clk_table.entries[i].memclk_mhz > maxDPM)
+			maxDPM = dc->clk_mgr->bw_params->clk_table.entries[i].memclk_mhz;
+	}
+	funcMin = (dc->clk_mgr->clks.dramclk_khz + 999) / 1000;
+	p_state_change_support = dc->clk_mgr->clks.p_state_change_support;
+
+	if (enable && !dc->clk_mgr->dc_mode_softmax_enabled) {
+		if (p_state_change_support) {
+			if (funcMin <= softMax && dc->clk_mgr->funcs->set_max_memclk)
+				dc->clk_mgr->funcs->set_max_memclk(dc->clk_mgr, softMax);
+			// else: No-Op
+		} else {
+			if (funcMin <= softMax)
+				blank_and_force_memclk(dc, true, softMax);
+			// else: No-Op
+		}
+	} else if (!enable && dc->clk_mgr->dc_mode_softmax_enabled) {
+		if (p_state_change_support) {
+			if (funcMin <= softMax && dc->clk_mgr->funcs->set_max_memclk)
+				dc->clk_mgr->funcs->set_max_memclk(dc->clk_mgr, maxDPM);
+			// else: No-Op
+		} else {
+			if (funcMin <= softMax)
+				blank_and_force_memclk(dc, true, maxDPM);
+			// else: No-Op
+		}
+	}
+	dc->clk_mgr->dc_mode_softmax_enabled = enable;
+}
+bool dc_is_plane_eligible_for_idle_optimizations(struct dc *dc,
+		unsigned int pitch,
+		unsigned int height,
+		enum surface_pixel_format format,
+		struct dc_cursor_attributes *cursor_attr)
+{
+	if (dc->hwss.does_plane_fit_in_mall && dc->hwss.does_plane_fit_in_mall(dc, pitch, height, format, cursor_attr))
+		return true;
+	return false;
+}
+
+/* cleanup on driver unload */
+void dc_hardware_release(struct dc *dc)
+{
+	dc_mclk_switch_using_fw_based_vblank_stretch_shut_down(dc);
+
+	if (dc->hwss.hardware_release)
+		dc->hwss.hardware_release(dc);
+}
+
+void dc_mclk_switch_using_fw_based_vblank_stretch_shut_down(struct dc *dc)
+{
+	if (dc->current_state)
+		dc->current_state->bw_ctx.bw.dcn.clk.fw_based_mclk_switching_shut_down = true;
+}
+
+/**
+ * dc_is_dmub_outbox_supported - Check if DMUB firmware support outbox notification
+ *
+ * @dc: [in] dc structure
+ *
+ * Checks whether DMUB FW supports outbox notifications, if supported DM
+ * should register outbox interrupt prior to actually enabling interrupts
+ * via dc_enable_dmub_outbox
+ *
+ * Return:
+ * True if DMUB FW supports outbox notifications, False otherwise
+ */
+bool dc_is_dmub_outbox_supported(struct dc *dc)
+{
+	if (!dc->caps.dmcub_support)
+		return false;
+
+	switch (dc->ctx->asic_id.chip_family) {
+
+	case FAMILY_YELLOW_CARP:
+		/* DCN31 B0 USB4 DPIA needs dmub notifications for interrupts */
+		if (dc->ctx->asic_id.hw_internal_rev == YELLOW_CARP_B0 &&
+		    !dc->debug.dpia_debug.bits.disable_dpia)
+			return true;
+	break;
+
+	case AMDGPU_FAMILY_GC_11_0_1:
+	case AMDGPU_FAMILY_GC_11_5_0:
+	case AMDGPU_FAMILY_GC_11_5_4:
+		if (!dc->debug.dpia_debug.bits.disable_dpia)
+			return true;
+	break;
+
+	default:
+		break;
+	}
+
+	/* dmub aux needs dmub notifications to be enabled */
+	return dc->debug.enable_dmub_aux_for_legacy_ddc;
+
+}
+
+/**
+ * dc_enable_dmub_notifications - Check if dmub fw supports outbox
+ *
+ * @dc: [in] dc structure
+ *
+ * Calls dc_is_dmub_outbox_supported to check if dmub fw supports outbox
+ * notifications. All DMs shall switch to dc_is_dmub_outbox_supported.  This
+ * API shall be removed after switching.
+ *
+ * Return:
+ * True if DMUB FW supports outbox notifications, False otherwise
+ */
+bool dc_enable_dmub_notifications(struct dc *dc)
+{
+	return dc_is_dmub_outbox_supported(dc);
+}
+
+/**
+ * dc_enable_dmub_outbox - Enables DMUB unsolicited notification
+ *
+ * @dc: [in] dc structure
+ *
+ * Enables DMUB unsolicited notifications to x86 via outbox.
+ */
+void dc_enable_dmub_outbox(struct dc *dc)
+{
+	struct dc_context *dc_ctx = dc->ctx;
+
+	dmub_enable_outbox_notification(dc_ctx->dmub_srv);
+	DC_LOG_DC("%s: dmub outbox notifications enabled\n", __func__);
+}
+
+/**
+ * dc_process_dmub_aux_transfer_async - Submits aux command to dmub via inbox message
+ *                                      Sets port index appropriately for legacy DDC
+ * @dc: dc structure
+ * @link_index: link index
+ * @payload: aux payload
+ *
+ * Returns: True if successful, False if failure
+ */
+bool dc_process_dmub_aux_transfer_async(struct dc *dc,
+				uint32_t link_index,
+				struct aux_payload *payload)
+{
+	uint8_t action;
+	union dmub_rb_cmd cmd = {0};
+
+	if (link_index >= dc->link_count || !dc->links[link_index])
+		return false;
+
+	if (payload->length > sizeof(cmd.dp_aux_access.aux_control.dpaux.data))
+		return false;
+
+	cmd.dp_aux_access.header.type = DMUB_CMD__DP_AUX_ACCESS;
+	cmd.dp_aux_access.header.payload_bytes = 0;
+	/* For dpia, ddc_pin is set to NULL */
+	if (!dc->links[link_index]->ddc->ddc_pin)
+		cmd.dp_aux_access.aux_control.type = AUX_CHANNEL_DPIA;
+	else
+		cmd.dp_aux_access.aux_control.type = AUX_CHANNEL_LEGACY_DDC;
+
+	cmd.dp_aux_access.aux_control.instance = (uint8_t)dc->links[link_index]->ddc_hw_inst;
+	cmd.dp_aux_access.aux_control.sw_crc_enabled = 0;
+	cmd.dp_aux_access.aux_control.timeout = 0;
+	cmd.dp_aux_access.aux_control.dpaux.address = payload->address;
+	cmd.dp_aux_access.aux_control.dpaux.is_i2c_over_aux = payload->i2c_over_aux;
+	cmd.dp_aux_access.aux_control.dpaux.length = (uint8_t)payload->length;
+
+	/* set aux action */
+	if (payload->i2c_over_aux) {
+		if (payload->write) {
+			if (payload->mot)
+				action = DP_AUX_REQ_ACTION_I2C_WRITE_MOT;
+			else
+				action = DP_AUX_REQ_ACTION_I2C_WRITE;
+		} else {
+			if (payload->mot)
+				action = DP_AUX_REQ_ACTION_I2C_READ_MOT;
+			else
+				action = DP_AUX_REQ_ACTION_I2C_READ;
+			}
+	} else {
+		if (payload->write)
+			action = DP_AUX_REQ_ACTION_DPCD_WRITE;
+		else
+			action = DP_AUX_REQ_ACTION_DPCD_READ;
+	}
+
+	cmd.dp_aux_access.aux_control.dpaux.action = action;
+
+	if (payload->length && payload->write) {
+		memcpy(cmd.dp_aux_access.aux_control.dpaux.data,
+			payload->data,
+			payload->length
+			);
+	}
+
+	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+
+	return true;
+}
+
+bool dc_smart_power_oled_enable(const struct dc_link *link, bool enable, uint16_t peak_nits,
+			uint8_t debug_control, uint16_t fixed_CLL, uint32_t triggerline)
+{
+	bool status = false;
+	struct dc *dc = link->ctx->dc;
+	union dmub_rb_cmd cmd;
+	uint8_t otg_inst = 0;
+	unsigned int panel_inst = 0;
+	struct pipe_ctx *pipe_ctx = NULL;
+	struct resource_context *res_ctx = &link->ctx->dc->current_state->res_ctx;
+	int i = 0;
+
+	// get panel_inst
+	if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst))
+		return status;
+
+	// get otg_inst
+	for (i = 0; i < MAX_PIPES; i++) {
+		if (res_ctx &&
+			res_ctx->pipe_ctx[i].stream &&
+			res_ctx->pipe_ctx[i].stream->link &&
+			res_ctx->pipe_ctx[i].stream->link == link &&
+			res_ctx->pipe_ctx[i].stream->link->connector_signal == SIGNAL_TYPE_EDP) {
+			pipe_ctx = &res_ctx->pipe_ctx[i];
+			//TODO: refactor for multi edp support
+			break;
+		}
+	}
+
+	if (pipe_ctx)
+		otg_inst = (uint8_t)pipe_ctx->stream_res.tg->inst;
+
+	// before enable smart power OLED, we need to call set pipe for DMUB to set ABM config
+	if (enable) {
+		if (dc->hwss.set_pipe && pipe_ctx)
+			dc->hwss.set_pipe(pipe_ctx);
+	}
+
+	// fill in cmd
+	memset(&cmd, 0, sizeof(cmd));
+
+	cmd.smart_power_oled_enable.header.type = DMUB_CMD__SMART_POWER_OLED;
+	cmd.smart_power_oled_enable.header.sub_type = DMUB_CMD__SMART_POWER_OLED_ENABLE;
+	cmd.smart_power_oled_enable.header.payload_bytes =
+		sizeof(struct dmub_rb_cmd_smart_power_oled_enable_data) - sizeof(struct dmub_cmd_header);
+	cmd.smart_power_oled_enable.header.ret_status = 1;
+	cmd.smart_power_oled_enable.data.enable = enable;
+	cmd.smart_power_oled_enable.data.panel_inst = (uint8_t)panel_inst;
+	cmd.smart_power_oled_enable.data.peak_nits = peak_nits;
+	cmd.smart_power_oled_enable.data.otg_inst = otg_inst;
+	cmd.smart_power_oled_enable.data.digfe_inst = (uint8_t)link->link_enc->preferred_engine;
+	cmd.smart_power_oled_enable.data.digbe_inst = (uint8_t)link->link_enc->transmitter;
+
+	cmd.smart_power_oled_enable.data.debugcontrol = debug_control;
+	cmd.smart_power_oled_enable.data.triggerline = triggerline;
+	cmd.smart_power_oled_enable.data.fixed_max_cll = fixed_CLL;
+
+	// send cmd
+	status = dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+
+	// Update firmware_controlled_hdr_info_packet state on successful command execution
+	if (status && pipe_ctx)
+		pipe_ctx->stream->firmware_controlled_hdr_info_packet = enable;
+
+	return status;
+}
+
+bool dc_smart_power_oled_get_max_cll(const struct dc_link *link, unsigned int *pCurrent_MaxCLL)
+{
+	struct dc *dc = link->ctx->dc;
+	union dmub_rb_cmd cmd;
+	bool status = false;
+	unsigned int panel_inst = 0;
+
+	// get panel_inst
+	if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst))
+		return status;
+
+	// fill in cmd
+	memset(&cmd, 0, sizeof(cmd));
+
+	cmd.smart_power_oled_getmaxcll.header.type = DMUB_CMD__SMART_POWER_OLED;
+	cmd.smart_power_oled_getmaxcll.header.sub_type = DMUB_CMD__SMART_POWER_OLED_GETMAXCLL;
+	cmd.smart_power_oled_getmaxcll.header.payload_bytes = sizeof(cmd.smart_power_oled_getmaxcll.data);
+	cmd.smart_power_oled_getmaxcll.header.ret_status = 1;
+
+	cmd.smart_power_oled_getmaxcll.data.input.panel_inst = (uint8_t)panel_inst;
+
+	// send cmd and wait for reply
+	status = dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY);
+
+	if (status)
+		*pCurrent_MaxCLL = cmd.smart_power_oled_getmaxcll.data.output.current_max_cll;
+	else
+		*pCurrent_MaxCLL = 0;
+
+	return status;
+}
+
+uint8_t get_link_index_from_dpia_port_index(const struct dc *dc,
+					    uint8_t dpia_port_index)
+{
+	uint8_t index, link_index = 0xFF;
+
+	for (index = 0; index < dc->link_count; index++) {
+		/* ddc_hw_inst has dpia port index for dpia links
+		 * and ddc instance for legacy links
+		 */
+		if (!dc->links[index]->ddc->ddc_pin) {
+			if (dc->links[index]->ddc_hw_inst == dpia_port_index) {
+				link_index = index;
+				break;
+			}
+		}
+	}
+	ASSERT(link_index != 0xFF);
+	return link_index;
+}
+
+/**
+ * dc_process_dmub_set_config_async - Submits set_config command
+ *
+ * @dc: [in] dc structure
+ * @link_index: [in] link_index: link index
+ * @payload: [in] aux payload
+ * @notify: [out] set_config immediate reply
+ *
+ * Submits set_config command to dmub via inbox message.
+ *
+ * Return:
+ * True if successful, False if failure
+ */
+bool dc_process_dmub_set_config_async(struct dc *dc,
+				uint32_t link_index,
+				struct set_config_cmd_payload *payload,
+				struct dmub_notification *notify)
+{
+	union dmub_rb_cmd cmd = {0};
+	bool is_cmd_complete = true;
+
+	/* prepare SET_CONFIG command */
+	cmd.set_config_access.header.type = DMUB_CMD__DPIA;
+	cmd.set_config_access.header.sub_type = DMUB_CMD__DPIA_SET_CONFIG_ACCESS;
+
+	cmd.set_config_access.set_config_control.instance = (uint8_t)dc->links[link_index]->ddc_hw_inst;
+	cmd.set_config_access.set_config_control.cmd_pkt.msg_type = payload->msg_type;
+	cmd.set_config_access.set_config_control.cmd_pkt.msg_data = payload->msg_data;
+
+	if (!dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY)) {
+		/* command is not processed by dmub */
+		notify->sc_status = SET_CONFIG_UNKNOWN_ERROR;
+		return is_cmd_complete;
+	}
+
+	/* command processed by dmub, if ret_status is 1, it is completed instantly */
+	if (cmd.set_config_access.header.ret_status == 1)
+		notify->sc_status = cmd.set_config_access.set_config_control.immed_status;
+	else
+		/* cmd pending, will receive notification via outbox */
+		is_cmd_complete = false;
+
+	return is_cmd_complete;
+}
+
+/**
+ * dc_process_dmub_set_mst_slots - Submits MST solt allocation
+ *
+ * @dc: [in] dc structure
+ * @link_index: [in] link index
+ * @mst_alloc_slots: [in] mst slots to be allotted
+ * @mst_slots_in_use: [out] mst slots in use returned in failure case
+ *
+ * Submits mst slot allocation command to dmub via inbox message
+ *
+ * Return:
+ * DC_OK if successful, DC_ERROR if failure
+ */
+enum dc_status dc_process_dmub_set_mst_slots(const struct dc *dc,
+				uint32_t link_index,
+				uint8_t mst_alloc_slots,
+				uint8_t *mst_slots_in_use)
+{
+	union dmub_rb_cmd cmd = {0};
+
+	/* prepare MST_ALLOC_SLOTS command */
+	cmd.set_mst_alloc_slots.header.type = DMUB_CMD__DPIA;
+	cmd.set_mst_alloc_slots.header.sub_type = DMUB_CMD__DPIA_MST_ALLOC_SLOTS;
+
+	cmd.set_mst_alloc_slots.mst_slots_control.instance = (uint8_t)dc->links[link_index]->ddc_hw_inst;
+	cmd.set_mst_alloc_slots.mst_slots_control.mst_alloc_slots = mst_alloc_slots;
+
+	if (!dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY))
+		/* command is not processed by dmub */
+		return DC_ERROR_UNEXPECTED;
+
+	/* command processed by dmub, if ret_status is 1 */
+	if (cmd.set_config_access.header.ret_status != 1)
+		/* command processing error */
+		return DC_ERROR_UNEXPECTED;
+
+	/* command processed and we have a status of 2, mst not enabled in dpia */
+	if (cmd.set_mst_alloc_slots.mst_slots_control.immed_status == 2)
+		return DC_FAIL_UNSUPPORTED_1;
+
+	/* previously configured mst alloc and used slots did not match */
+	if (cmd.set_mst_alloc_slots.mst_slots_control.immed_status == 3) {
+		*mst_slots_in_use = cmd.set_mst_alloc_slots.mst_slots_control.mst_slots_in_use;
+		return DC_NOT_SUPPORTED;
+	}
+
+	return DC_OK;
+}
+
+/**
+ * dc_process_dmub_dpia_set_tps_notification - Submits tps notification
+ *
+ * @dc: [in] dc structure
+ * @link_index: [in] link index
+ * @tps: [in] request tps
+ *
+ * Submits set_tps_notification command to dmub via inbox message
+ */
+void dc_process_dmub_dpia_set_tps_notification(const struct dc *dc, uint32_t link_index, uint8_t tps)
+{
+	union dmub_rb_cmd cmd = {0};
+
+	cmd.set_tps_notification.header.type = DMUB_CMD__DPIA;
+	cmd.set_tps_notification.header.sub_type = DMUB_CMD__DPIA_SET_TPS_NOTIFICATION;
+	cmd.set_tps_notification.tps_notification.instance = (uint8_t)dc->links[link_index]->ddc_hw_inst;
+	cmd.set_tps_notification.tps_notification.tps = tps;
+
+	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+}
+
+/**
+ * dc_process_dmub_dpia_hpd_int_enable - Submits DPIA DPD interruption
+ *
+ * @dc: [in] dc structure
+ * @hpd_int_enable: [in] 1 for hpd int enable, 0 to disable
+ *
+ * Submits dpia hpd int enable command to dmub via inbox message
+ */
+void dc_process_dmub_dpia_hpd_int_enable(const struct dc *dc,
+				uint32_t hpd_int_enable)
+{
+	union dmub_rb_cmd cmd = {0};
+
+	cmd.dpia_hpd_int_enable.header.type = DMUB_CMD__DPIA_HPD_INT_ENABLE;
+	cmd.dpia_hpd_int_enable.enable = hpd_int_enable;
+
+	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+
+	DC_LOG_DEBUG("%s: hpd_int_enable(%d)\n", __func__, hpd_int_enable);
+}
+
+/**
+ * dc_print_dmub_diagnostic_data - Print DMUB diagnostic data for debugging
+ *
+ * @dc: [in] dc structure
+ *
+ *
+ */
+void dc_print_dmub_diagnostic_data(const struct dc *dc)
+{
+	dc_dmub_srv_log_diagnostic_data(dc->ctx->dmub_srv);
+}
+
+/**
+ * dc_disable_accelerated_mode - disable accelerated mode
+ * @dc: dc structure
+ */
+void dc_disable_accelerated_mode(struct dc *dc)
+{
+	bios_set_scratch_acc_mode_change(dc->ctx->dc_bios, 0);
+}
+
+
+/**
+ *  dc_notify_vsync_int_state - notifies vsync enable/disable state
+ *  @dc: dc structure
+ *  @stream: stream where vsync int state changed
+ *  @enable: whether vsync is enabled or disabled
+ *
+ *  Called when vsync is enabled/disabled Will notify DMUB to start/stop ABM
+ *  interrupts after steady state is reached.
+ */
+void dc_notify_vsync_int_state(struct dc *dc, struct dc_stream_state *stream, bool enable)
+{
+	unsigned int i, edp_num;
+	struct pipe_ctx *pipe = NULL;
+	struct dc_link *link = stream->sink->link;
+	struct dc_link *edp_links[MAX_NUM_EDP];
+
+
+	if (link->psr_settings.psr_feature_enabled)
+		return;
+
+	if (link->replay_settings.replay_feature_enabled)
+		return;
+
+	/*find primary pipe associated with stream*/
+	for (i = 0; i < MAX_PIPES; i++) {
+		pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+
+		if (pipe->stream == stream && pipe->stream_res.tg)
+			break;
+	}
+
+	if (i == MAX_PIPES) {
+		ASSERT(0);
+		return;
+	}
+
+	dc_get_edp_links(dc, edp_links, &edp_num);
+
+	/* Determine panel inst */
+	for (i = 0; i < edp_num; i++) {
+		if (edp_links[i] == link)
+			break;
+	}
+
+	if (i == edp_num) {
+		return;
+	}
+
+	if (pipe->stream_res.abm && pipe->stream_res.abm->funcs->set_abm_pause)
+		pipe->stream_res.abm->funcs->set_abm_pause(pipe->stream_res.abm, !enable, i, pipe->stream_res.tg->inst);
+}
+
+/*****************************************************************************
+ *  dc_abm_save_restore() - Interface to DC for save+pause and restore+un-pause
+ *                          ABM
+ *  @dc: dc structure
+ *	@stream: stream where vsync int state changed
+ *  @pData: abm hw states
+ *
+ ****************************************************************************/
+bool dc_abm_save_restore(
+		struct dc *dc,
+		struct dc_stream_state *stream,
+		struct abm_save_restore *pData)
+{
+	unsigned int i, edp_num;
+	struct pipe_ctx *pipe = NULL;
+	struct dc_link *link = stream->sink->link;
+	struct dc_link *edp_links[MAX_NUM_EDP];
+
+	if (link->replay_settings.replay_feature_enabled)
+		return false;
+
+	/*find primary pipe associated with stream*/
+	for (i = 0; i < MAX_PIPES; i++) {
+		pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+
+		if (pipe->stream == stream && pipe->stream_res.tg)
+			break;
+	}
+
+	if (i == MAX_PIPES) {
+		ASSERT(0);
+		return false;
+	}
+
+	dc_get_edp_links(dc, edp_links, &edp_num);
+
+	/* Determine panel inst */
+	for (i = 0; i < edp_num; i++)
+		if (edp_links[i] == link)
+			break;
+
+	if (i == edp_num)
+		return false;
+
+	if (pipe->stream_res.abm &&
+		pipe->stream_res.abm->funcs->save_restore)
+		return pipe->stream_res.abm->funcs->save_restore(
+				pipe->stream_res.abm,
+				i,
+				pData);
+	return false;
+}
+
+void dc_query_current_properties(struct dc *dc, struct dc_current_properties *properties)
+{
+	unsigned int i;
+	unsigned int max_cursor_size = dc->caps.max_cursor_size;
+	unsigned int stream_cursor_size;
+
+	if (dc->debug.allow_sw_cursor_fallback && dc->res_pool->funcs->get_max_hw_cursor_size) {
+		for (i = 0; i < dc->current_state->stream_count; i++) {
+			stream_cursor_size = dc->res_pool->funcs->get_max_hw_cursor_size(dc,
+					dc->current_state,
+					dc->current_state->streams[i]);
+
+			if (stream_cursor_size < max_cursor_size) {
+				max_cursor_size = stream_cursor_size;
+			}
+		}
+	}
+
+	properties->cursor_size_limit = max_cursor_size;
+}
+
+/**
+ * dc_set_edp_power() - DM controls eDP power to be ON/OFF
+ *
+ * Called when DM wants to power on/off eDP.
+ *     Only work on links with flag skip_implict_edp_power_control is set.
+ *
+ * @dc: Current DC state
+ * @edp_link: a link with eDP connector signal type
+ * @powerOn: power on/off eDP
+ *
+ * Return: void
+ */
+void dc_set_edp_power(const struct dc *dc, struct dc_link *edp_link,
+				 bool powerOn)
+{
+	(void)dc;
+	if (edp_link->connector_signal != SIGNAL_TYPE_EDP)
+		return;
+
+	if (edp_link->skip_implict_edp_power_control == false)
+		return;
+
+	edp_link->dc->link_srv->edp_set_panel_power(edp_link, powerOn);
+}
+
+/**
+ * dc_get_power_profile_for_dc_state() - extracts power profile from dc state
+ *
+ * Called when DM wants to make power policy decisions based on dc_state
+ *
+ * @context: Pointer to the dc_state from which the power profile is extracted.
+ *
+ * Return: The power profile structure containing the power level information.
+ */
+struct dc_power_profile dc_get_power_profile_for_dc_state(const struct dc_state *context)
+{
+	struct dc_power_profile profile = { 0 };
+
+	profile.power_level = !context->bw_ctx.bw.dcn.clk.p_state_change_support;
+	if (!context->clk_mgr || !context->clk_mgr->ctx || !context->clk_mgr->ctx->dc)
+		return profile;
+	struct dc *dc = context->clk_mgr->ctx->dc;
+
+	if (dc->res_pool->funcs->get_power_profile)
+		profile.power_level = dc->res_pool->funcs->get_power_profile(context);
+	return profile;
+}
+
+/**
+ * dc_get_det_buffer_size_from_state() - extracts detile buffer size from dc state
+ *
+ * This function is called to log the detile buffer size from the dc_state.
+ *
+ * @context: a pointer to the dc_state from which the detile buffer size is extracted.
+ *
+ * Return: the size of the detile buffer, or 0 if not available.
+ */
+unsigned int dc_get_det_buffer_size_from_state(const struct dc_state *context)
+{
+	struct dc *dc = context->clk_mgr->ctx->dc;
+
+	if (dc->res_pool->funcs->get_det_buffer_size)
+		return dc->res_pool->funcs->get_det_buffer_size(context);
+	else
+		return 0;
+}
+
+/**
+ * dc_get_host_router_index: Get index of host router from a dpia link
+ *
+ * This function return a host router index of the target link. If the target link is dpia link.
+ *
+ * @link: Pointer to the target link (input)
+ * @host_router_index: Pointer to store the host router index of the target link (output).
+ *
+ * Return: true if the host router index is found and valid.
+ *
+ */
+bool dc_get_host_router_index(const struct dc_link *link, unsigned int *host_router_index)
+{
+	struct dc *dc;
+
+	if (!link || !host_router_index || link->ep_type != DISPLAY_ENDPOINT_USB4_DPIA)
+		return false;
+
+	dc = link->ctx->dc;
+
+	if (link->link_index < dc->lowest_dpia_link_index)
+		return false;
+
+	*host_router_index = (link->link_index - dc->lowest_dpia_link_index) / dc->caps.num_of_dpias_per_host_router;
+	if (*host_router_index < dc->caps.num_of_host_routers)
+		return true;
+	else
+		return false;
+}
+
+bool dc_is_cursor_limit_pending(struct dc *dc)
+{
+	uint32_t i;
+
+	for (i = 0; i < dc->current_state->stream_count; i++) {
+		if (dc_stream_is_cursor_limit_pending(dc, dc->current_state->streams[i]))
+			return true;
+	}
+
+	return false;
+}
+
+bool dc_can_clear_cursor_limit(const struct dc *dc)
+{
+	uint32_t i;
+
+	for (i = 0; i < dc->current_state->stream_count; i++) {
+		if (dc_state_can_clear_stream_cursor_subvp_limit(dc->current_state->streams[i], dc->current_state))
+			return true;
+	}
+
+	return false;
+}
+
+void dc_get_underflow_debug_data_for_otg(struct dc *dc, unsigned int primary_otg_inst,
+				struct dc_underflow_debug_data *out_data)
+{
+	struct timing_generator *tg = NULL;
+
+	for (int i = 0; i < MAX_PIPES; i++) {
+		if (dc->res_pool->timing_generators[i] &&
+			dc->res_pool->timing_generators[i]->inst == primary_otg_inst) {
+				tg = dc->res_pool->timing_generators[i];
+				break;
+		}
+	}
+
+	dc_exit_ips_for_hw_access(dc);
+	if (dc->hwss.get_underflow_debug_data)
+		dc->hwss.get_underflow_debug_data(dc, tg, out_data);
+}
+
+void dc_get_power_feature_status(struct dc *dc, unsigned int primary_otg_inst,
+				struct power_features *out_data)
+{
+	(void)primary_otg_inst;
+	out_data->uclk_p_state = dc->current_state->clk_mgr->clks.p_state_change_support;
+	out_data->fams = dc->current_state->bw_ctx.bw.dcn.clk.fw_based_mclk_switching;
+}
+
+bool dc_capture_register_software_state(struct dc *dc, struct dc_register_software_state *state)
+{
+	struct dc_state *context;
+	struct resource_context *res_ctx;
+	unsigned int i;
+	const unsigned int max_pipes = MAX_PIPES;
+
+	if (!dc || !dc->current_state || !state) {
+		if (state)
+			state->state_valid = false;
+		return false;
+	}
+
+	/* Initialize the state structure */
+	memset(state, 0, sizeof(struct dc_register_software_state));
+
+	context = dc->current_state;
+	res_ctx = &context->res_ctx;
+
+	/* Count active pipes and streams */
+	state->active_pipe_count = 0;
+	state->active_stream_count = context->stream_count;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		if (res_ctx->pipe_ctx[i].stream)
+			state->active_pipe_count++;
+	}
+
+	/* Capture HUBP programming state for each pipe */
+	for (i = 0; i < max_pipes && i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
+
+		state->hubp[i].valid_stream = false;
+		if (!pipe_ctx->stream)
+			continue;
+
+		state->hubp[i].valid_stream = true;
+
+		/* HUBP register programming variables */
+		if (pipe_ctx->stream_res.tg)
+			state->hubp[i].vtg_sel = pipe_ctx->stream_res.tg->inst;
+
+		state->hubp[i].hubp_clock_enable = (pipe_ctx->plane_res.hubp != NULL) ? 1 : 0;
+
+		state->hubp[i].valid_plane_state = false;
+		if (pipe_ctx->plane_state) {
+			state->hubp[i].valid_plane_state = true;
+			state->hubp[i].surface_pixel_format = pipe_ctx->plane_state->format;
+			state->hubp[i].rotation_angle = pipe_ctx->plane_state->rotation;
+			state->hubp[i].h_mirror_en = pipe_ctx->plane_state->horizontal_mirror ? 1 : 0;
+
+			/* Surface size */
+			if (pipe_ctx->plane_state->plane_size.surface_size.width > 0) {
+				state->hubp[i].surface_size_width = pipe_ctx->plane_state->plane_size.surface_size.width;
+				state->hubp[i].surface_size_height = pipe_ctx->plane_state->plane_size.surface_size.height;
+			}
+
+			/* Viewport dimensions from scaler data */
+			if (pipe_ctx->plane_state->src_rect.width > 0) {
+				state->hubp[i].pri_viewport_width = pipe_ctx->plane_state->src_rect.width;
+				state->hubp[i].pri_viewport_height = pipe_ctx->plane_state->src_rect.height;
+				state->hubp[i].pri_viewport_x_start = pipe_ctx->plane_state->src_rect.x;
+				state->hubp[i].pri_viewport_y_start = pipe_ctx->plane_state->src_rect.y;
+			}
+
+			/* DCC settings */
+			state->hubp[i].surface_dcc_en = (pipe_ctx->plane_state->dcc.enable) ? 1 : 0;
+			state->hubp[i].surface_dcc_ind_64b_blk = pipe_ctx->plane_state->dcc.independent_64b_blks;
+			state->hubp[i].surface_dcc_ind_128b_blk = pipe_ctx->plane_state->dcc.dcc_ind_blk;
+
+			/* Surface pitch */
+			state->hubp[i].surface_pitch = pipe_ctx->plane_state->plane_size.surface_pitch;
+			state->hubp[i].meta_pitch = pipe_ctx->plane_state->dcc.meta_pitch;
+			state->hubp[i].chroma_pitch = pipe_ctx->plane_state->plane_size.chroma_pitch;
+			state->hubp[i].meta_pitch_c = pipe_ctx->plane_state->dcc.meta_pitch_c;
+
+			/* Surface addresses - primary */
+			state->hubp[i].primary_surface_address_low = pipe_ctx->plane_state->address.grph.addr.low_part;
+			state->hubp[i].primary_surface_address_high = pipe_ctx->plane_state->address.grph.addr.high_part;
+			state->hubp[i].primary_meta_surface_address_low = pipe_ctx->plane_state->address.grph.meta_addr.low_part;
+			state->hubp[i].primary_meta_surface_address_high = pipe_ctx->plane_state->address.grph.meta_addr.high_part;
+
+			/* TMZ settings */
+			state->hubp[i].primary_surface_tmz = pipe_ctx->plane_state->address.tmz_surface;
+			state->hubp[i].primary_meta_surface_tmz = pipe_ctx->plane_state->address.tmz_surface;
+
+			/* Tiling configuration */
+			state->hubp[i].min_dc_gfx_version9 = false;
+			if (pipe_ctx->plane_state->tiling_info.gfxversion >= DcGfxVersion9) {
+				state->hubp[i].min_dc_gfx_version9 = true;
+				state->hubp[i].sw_mode = pipe_ctx->plane_state->tiling_info.gfx9.swizzle;
+				state->hubp[i].num_pipes = pipe_ctx->plane_state->tiling_info.gfx9.num_pipes;
+				state->hubp[i].num_banks = pipe_ctx->plane_state->tiling_info.gfx9.num_banks;
+				state->hubp[i].pipe_interleave = pipe_ctx->plane_state->tiling_info.gfx9.pipe_interleave;
+				state->hubp[i].num_shader_engines = pipe_ctx->plane_state->tiling_info.gfx9.num_shader_engines;
+				state->hubp[i].num_rb_per_se = pipe_ctx->plane_state->tiling_info.gfx9.num_rb_per_se;
+				state->hubp[i].num_pkrs = pipe_ctx->plane_state->tiling_info.gfx9.num_pkrs;
+			}
+		}
+
+		/* DML Request Size Configuration */
+		if (pipe_ctx->rq_regs.rq_regs_l.chunk_size > 0) {
+			state->hubp[i].rq_chunk_size = pipe_ctx->rq_regs.rq_regs_l.chunk_size;
+			state->hubp[i].rq_min_chunk_size = pipe_ctx->rq_regs.rq_regs_l.min_chunk_size;
+			state->hubp[i].rq_meta_chunk_size = pipe_ctx->rq_regs.rq_regs_l.meta_chunk_size;
+			state->hubp[i].rq_min_meta_chunk_size = pipe_ctx->rq_regs.rq_regs_l.min_meta_chunk_size;
+			state->hubp[i].rq_dpte_group_size = pipe_ctx->rq_regs.rq_regs_l.dpte_group_size;
+			state->hubp[i].rq_mpte_group_size = pipe_ctx->rq_regs.rq_regs_l.mpte_group_size;
+			state->hubp[i].rq_swath_height_l = pipe_ctx->rq_regs.rq_regs_l.swath_height;
+			state->hubp[i].rq_pte_row_height_l = pipe_ctx->rq_regs.rq_regs_l.pte_row_height_linear;
+		}
+
+		/* Chroma request size configuration */
+		if (pipe_ctx->rq_regs.rq_regs_c.chunk_size > 0) {
+			state->hubp[i].rq_chunk_size_c = pipe_ctx->rq_regs.rq_regs_c.chunk_size;
+			state->hubp[i].rq_min_chunk_size_c = pipe_ctx->rq_regs.rq_regs_c.min_chunk_size;
+			state->hubp[i].rq_meta_chunk_size_c = pipe_ctx->rq_regs.rq_regs_c.meta_chunk_size;
+			state->hubp[i].rq_min_meta_chunk_size_c = pipe_ctx->rq_regs.rq_regs_c.min_meta_chunk_size;
+			state->hubp[i].rq_dpte_group_size_c = pipe_ctx->rq_regs.rq_regs_c.dpte_group_size;
+			state->hubp[i].rq_mpte_group_size_c = pipe_ctx->rq_regs.rq_regs_c.mpte_group_size;
+			state->hubp[i].rq_swath_height_c = pipe_ctx->rq_regs.rq_regs_c.swath_height;
+			state->hubp[i].rq_pte_row_height_c = pipe_ctx->rq_regs.rq_regs_c.pte_row_height_linear;
+		}
+
+		/* DML expansion modes */
+		state->hubp[i].drq_expansion_mode = pipe_ctx->rq_regs.drq_expansion_mode;
+		state->hubp[i].prq_expansion_mode = pipe_ctx->rq_regs.prq_expansion_mode;
+		state->hubp[i].mrq_expansion_mode = pipe_ctx->rq_regs.mrq_expansion_mode;
+		state->hubp[i].crq_expansion_mode = pipe_ctx->rq_regs.crq_expansion_mode;
+
+		/* DML DLG parameters - nominal */
+		state->hubp[i].dst_y_per_vm_vblank = pipe_ctx->dlg_regs.dst_y_per_vm_vblank;
+		state->hubp[i].dst_y_per_row_vblank = pipe_ctx->dlg_regs.dst_y_per_row_vblank;
+		state->hubp[i].dst_y_per_vm_flip = pipe_ctx->dlg_regs.dst_y_per_vm_flip;
+		state->hubp[i].dst_y_per_row_flip = pipe_ctx->dlg_regs.dst_y_per_row_flip;
+
+		/* DML prefetch settings */
+		state->hubp[i].dst_y_prefetch = pipe_ctx->dlg_regs.dst_y_prefetch;
+		state->hubp[i].vratio_prefetch = pipe_ctx->dlg_regs.vratio_prefetch;
+		state->hubp[i].vratio_prefetch_c = pipe_ctx->dlg_regs.vratio_prefetch_c;
+
+		/* TTU parameters */
+		state->hubp[i].qos_level_low_wm = pipe_ctx->ttu_regs.qos_level_low_wm;
+		state->hubp[i].qos_level_high_wm = pipe_ctx->ttu_regs.qos_level_high_wm;
+		state->hubp[i].qos_level_flip = pipe_ctx->ttu_regs.qos_level_flip;
+		state->hubp[i].min_ttu_vblank = pipe_ctx->ttu_regs.min_ttu_vblank;
+	}
+
+	/* Capture HUBBUB programming state */
+	if (dc->res_pool->hubbub) {
+		/* Individual DET buffer sizes - software state variables that program DET registers */
+		for (i = 0; i < 4u && i < dc->res_pool->pipe_count; i++) {
+			uint32_t det_size = res_ctx->pipe_ctx[i].det_buffer_size_kb;
+			switch (i) {
+			case 0:
+				state->hubbub.det0_size = det_size;
+				break;
+			case 1:
+				state->hubbub.det1_size = det_size;
+				break;
+			case 2:
+				state->hubbub.det2_size = det_size;
+				break;
+			case 3:
+				state->hubbub.det3_size = det_size;
+				break;
+			}
+		}
+
+		/* Compression buffer configuration - software state that programs COMPBUF_SIZE register */
+		// TODO: Handle logic for legacy DCN pre-DCN401
+		state->hubbub.compbuf_size = context->bw_ctx.bw.dcn.arb_regs.compbuf_size;
+	}
+
+	/* Capture DPP programming state for each pipe */
+	for (i = 0; i < max_pipes && i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
+
+		if (!pipe_ctx->stream)
+			continue;
+
+		state->dpp[i].dpp_clock_enable = (pipe_ctx->plane_res.dpp != NULL) ? 1 : 0;
+
+		if (pipe_ctx->plane_state && pipe_ctx->plane_res.scl_data.recout.width > 0) {
+			/* Access dscl_prog_data directly - this contains the actual software state used for register programming */
+			struct dscl_prog_data *dscl_data = &pipe_ctx->plane_res.scl_data.dscl_prog_data;
+
+			/* Recout (Rectangle of Interest) configuration - software state that programs RECOUT registers */
+			state->dpp[i].recout_start_x = dscl_data->recout.x;
+			state->dpp[i].recout_start_y = dscl_data->recout.y;
+			state->dpp[i].recout_width = dscl_data->recout.width;
+			state->dpp[i].recout_height = dscl_data->recout.height;
+
+			/* MPC (Multiple Pipe/Plane Combiner) size - software state that programs MPC_SIZE registers */
+			state->dpp[i].mpc_width = dscl_data->mpc_size.width;
+			state->dpp[i].mpc_height = dscl_data->mpc_size.height;
+
+			/* DSCL mode - software state that programs SCL_MODE registers */
+			state->dpp[i].dscl_mode = dscl_data->dscl_mode;
+
+			/* Scaler ratios - software state that programs scale ratio registers (use actual programmed ratios) */
+			state->dpp[i].horz_ratio_int = dscl_data->ratios.h_scale_ratio >> 19; // Extract integer part from programmed ratio
+			state->dpp[i].vert_ratio_int = dscl_data->ratios.v_scale_ratio >> 19; // Extract integer part from programmed ratio
+
+			/* Basic scaler taps - software state that programs tap control registers (use actual programmed taps) */
+			state->dpp[i].h_taps = dscl_data->taps.h_taps + 1; // dscl_prog_data.taps stores (taps - 1), so add 1 back
+			state->dpp[i].v_taps = dscl_data->taps.v_taps + 1; // dscl_prog_data.taps stores (taps - 1), so add 1 back
+		}
+	}
+
+	/* Capture essential clock state for underflow analysis */
+	if (dc->clk_mgr && dc->clk_mgr->clks.dispclk_khz > 0) {
+		/* Core display clocks affecting bandwidth and timing */
+		state->dccg.dispclk_khz = dc->clk_mgr->clks.dispclk_khz;
+
+		/* Per-pipe clock configuration - only capture what's essential */
+		for (i = 0; i < max_pipes && i < dc->res_pool->pipe_count; i++) {
+			struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
+			if (pipe_ctx->stream) {
+				/* Essential clocks that directly affect underflow risk */
+				state->dccg.dppclk_khz[i] = dc->clk_mgr->clks.dppclk_khz;
+				state->dccg.pixclk_khz[i] = pipe_ctx->stream->timing.pix_clk_100hz / 10;
+				state->dccg.dppclk_enable[i] = 1;
+
+				/* DP stream clock only for DP signals */
+				if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT ||
+						pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) {
+					state->dccg.dpstreamclk_enable[i] = 1;
+				} else {
+					state->dccg.dpstreamclk_enable[i] = 0;
+				}
+			} else {
+				/* Inactive pipe - no clocks */
+				state->dccg.dppclk_khz[i] = 0;
+				state->dccg.pixclk_khz[i] = 0;
+				state->dccg.dppclk_enable[i] = 0;
+				if (i < 4) {
+					state->dccg.dpstreamclk_enable[i] = 0;
+				}
+			}
+		}
+
+		/* DSC clock state - only when actually using DSC */
+		for (i = 0; i < max_pipes; i++) {
+			struct pipe_ctx *pipe_ctx = (i < dc->res_pool->pipe_count) ? &res_ctx->pipe_ctx[i] : NULL;
+			if (pipe_ctx && pipe_ctx->stream && pipe_ctx->stream->timing.dsc_cfg.num_slices_h > 0) {
+				state->dccg.dscclk_khz[i] = 400000; /* Typical DSC clock frequency */
+			} else {
+				state->dccg.dscclk_khz[i] = 0;
+			}
+		}
+
+		/* SYMCLK32 LE Control - only the essential HPO state for underflow analysis */
+		for (i = 0; i < 2; i++) {
+			state->dccg.symclk32_le_enable[i] = 0; /* Default: disabled */
+		}
+
+		/* Check for active HPO usage that affects symclk32_le */
+		for (unsigned int pipe_idx = 0; pipe_idx < MAX_PIPES && pipe_idx < dc->res_pool->pipe_count; pipe_idx++) {
+			struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[pipe_idx];
+			if (!pipe_ctx->stream)
+				continue;
+
+			/* HPO FRL (HDMI FRL) streams use symclk32_le */
+			if (pipe_ctx->stream_res.hpo_frl_stream_enc && pipe_ctx->link_res.hpo_frl_link_enc) {
+				int hpo_le_inst = pipe_ctx->link_res.hpo_frl_link_enc->inst;
+				if (hpo_le_inst >= 0 && hpo_le_inst < 2) {
+					state->dccg.symclk32_le_enable[hpo_le_inst] = 1;
+				}
+			}
+		}
+	}
+
+	/* Capture essential DSC configuration for underflow analysis */
+	for (i = 0; i < max_pipes && i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
+
+		if (pipe_ctx->stream && pipe_ctx->stream->timing.dsc_cfg.num_slices_h > 0) {
+			/* DSC is enabled - capture essential configuration */
+			state->dsc[i].dsc_clock_enable = 1;
+
+			/* DSC configuration affecting bandwidth and timing */
+			struct dc_dsc_config *dsc_cfg = &pipe_ctx->stream->timing.dsc_cfg;
+			state->dsc[i].dsc_num_slices_h = dsc_cfg->num_slices_h;
+			state->dsc[i].dsc_num_slices_v = dsc_cfg->num_slices_v;
+			state->dsc[i].dsc_bits_per_pixel = dsc_cfg->bits_per_pixel;
+
+			/* OPP pipe source for DSC forwarding */
+			if (pipe_ctx->stream_res.opp) {
+				state->dsc[i].dscrm_dsc_forward_enable = 1;
+				state->dsc[i].dscrm_dsc_opp_pipe_source = pipe_ctx->stream_res.opp->inst;
+			} else {
+				state->dsc[i].dscrm_dsc_forward_enable = 0;
+				state->dsc[i].dscrm_dsc_opp_pipe_source = 0;
+			}
+		} else {
+			/* DSC not enabled - clear all fields */
+			memset(&state->dsc[i], 0, sizeof(state->dsc[i]));
+		}
+	}
+
+	/* Capture MPC programming state - comprehensive register field coverage */
+	for (i = 0; i < max_pipes && i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
+
+		if (pipe_ctx->plane_state && pipe_ctx->stream) {
+			struct dc_plane_state *plane_state = pipe_ctx->plane_state;
+
+			/* MPCC blending tree and mode control - capture actual blend configuration */
+			state->mpc.mpcc_mode[i] = (plane_state->cm.blend_func.type != TF_TYPE_BYPASS) ? 1 : 0;
+			state->mpc.mpcc_alpha_blend_mode[i] = plane_state->per_pixel_alpha ? 1 : 0;
+			state->mpc.mpcc_alpha_multiplied_mode[i] = plane_state->pre_multiplied_alpha ? 1 : 0;
+			state->mpc.mpcc_blnd_active_overlap_only[i] = 0; /* Default - no overlap restriction */
+			state->mpc.mpcc_global_alpha[i] = plane_state->global_alpha_value;
+			state->mpc.mpcc_global_gain[i] = plane_state->global_alpha ? 255 : 0;
+			state->mpc.mpcc_bg_bpc[i] = 8; /* Standard 8-bit background */
+			state->mpc.mpcc_bot_gain_mode[i] = 0; /* Standard gain mode */
+
+			/* MPCC blending tree connections - capture tree topology */
+			if (pipe_ctx->bottom_pipe) {
+				state->mpc.mpcc_bot_sel[i] = pipe_ctx->bottom_pipe->pipe_idx;
+			} else {
+				state->mpc.mpcc_bot_sel[i] = 0xF; /* No bottom connection */
+			}
+			state->mpc.mpcc_top_sel[i] = pipe_ctx->pipe_idx; /* This pipe's DPP ID */
+
+			/* MPCC output gamma control - capture gamma programming */
+			if (plane_state->gamma_correction.type != GAMMA_CS_TFM_1D && plane_state->gamma_correction.num_entries > 0) {
+				state->mpc.mpcc_ogam_mode[i] = 1; /* Gamma enabled */
+				state->mpc.mpcc_ogam_select[i] = 0; /* Bank A selection */
+				state->mpc.mpcc_ogam_pwl_disable[i] = 0; /* PWL enabled */
+			} else {
+				state->mpc.mpcc_ogam_mode[i] = 0; /* Bypass mode */
+				state->mpc.mpcc_ogam_select[i] = 0;
+				state->mpc.mpcc_ogam_pwl_disable[i] = 1; /* PWL disabled */
+			}
+
+			/* MPCC pipe assignment and operational status */
+			if (pipe_ctx->stream_res.opp) {
+				state->mpc.mpcc_opp_id[i] = pipe_ctx->stream_res.opp->inst;
+			} else {
+				state->mpc.mpcc_opp_id[i] = 0xF; /* No OPP assignment */
+			}
+
+			/* MPCC status indicators - active pipe state */
+			state->mpc.mpcc_idle[i] = 0; /* Active pipe - not idle */
+			state->mpc.mpcc_busy[i] = 1; /* Active pipe - busy processing */
+
+		} else {
+			/* Pipe not active - set disabled/idle state for all fields */
+			state->mpc.mpcc_mode[i] = 0;
+			state->mpc.mpcc_alpha_blend_mode[i] = 0;
+			state->mpc.mpcc_alpha_multiplied_mode[i] = 0;
+			state->mpc.mpcc_blnd_active_overlap_only[i] = 0;
+			state->mpc.mpcc_global_alpha[i] = 0;
+			state->mpc.mpcc_global_gain[i] = 0;
+			state->mpc.mpcc_bg_bpc[i] = 0;
+			state->mpc.mpcc_bot_gain_mode[i] = 0;
+			state->mpc.mpcc_bot_sel[i] = 0xF; /* No bottom connection */
+			state->mpc.mpcc_top_sel[i] = 0xF; /* No top connection */
+			state->mpc.mpcc_ogam_mode[i] = 0; /* Bypass */
+			state->mpc.mpcc_ogam_select[i] = 0;
+			state->mpc.mpcc_ogam_pwl_disable[i] = 1; /* PWL disabled */
+			state->mpc.mpcc_opp_id[i] = 0xF; /* No OPP assignment */
+			state->mpc.mpcc_idle[i] = 1; /* Idle */
+			state->mpc.mpcc_busy[i] = 0; /* Not busy */
+		}
+	}
+
+	/* Capture OPP programming state for each pipe - comprehensive register field coverage */
+	for (i = 0; i < max_pipes && i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
+
+		if (!pipe_ctx->stream)
+			continue;
+
+		if (pipe_ctx->stream_res.opp) {
+			struct dc_crtc_timing *timing = &pipe_ctx->stream->timing;
+
+			/* OPP Pipe Control */
+			state->opp[i].opp_pipe_clock_enable = 1; /* Active pipe has clock enabled */
+
+			/* Display Pattern Generator (DPG) Control - 19 fields */
+			if (pipe_ctx->stream->test_pattern.type != DP_TEST_PATTERN_VIDEO_MODE) {
+				state->opp[i].dpg_enable = 1;
+			} else {
+				/* Video mode - DPG disabled */
+				state->opp[i].dpg_enable = 0;
+			}
+
+			/* Format Control (FMT) - 18 fields */
+			state->opp[i].fmt_pixel_encoding = timing->pixel_encoding;
+
+			/* Chroma subsampling mode based on pixel encoding */
+			if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR420) {
+				state->opp[i].fmt_subsampling_mode = 1; /* 4:2:0 subsampling */
+			} else if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR422) {
+				state->opp[i].fmt_subsampling_mode = 2; /* 4:2:2 subsampling */
+			} else {
+				state->opp[i].fmt_subsampling_mode = 0; /* No subsampling (4:4:4) */
+			}
+
+			state->opp[i].fmt_cbcr_bit_reduction_bypass = (timing->pixel_encoding == PIXEL_ENCODING_RGB) ? 1 : 0;
+			state->opp[i].fmt_stereosync_override = (timing->timing_3d_format != TIMING_3D_FORMAT_NONE) ? 1 : 0;
+
+			/* Dithering control based on bit depth */
+			if (timing->display_color_depth < COLOR_DEPTH_121212) {
+				state->opp[i].fmt_spatial_dither_frame_counter_max = 15; /* Typical frame counter max */
+				state->opp[i].fmt_spatial_dither_frame_counter_bit_swap = 0; /* No bit swapping */
+				state->opp[i].fmt_spatial_dither_enable = 1;
+				state->opp[i].fmt_spatial_dither_mode = 0; /* Spatial dithering mode */
+				state->opp[i].fmt_spatial_dither_depth = timing->display_color_depth;
+				state->opp[i].fmt_temporal_dither_enable = 0; /* Spatial dithering preferred */
+			} else {
+				state->opp[i].fmt_spatial_dither_frame_counter_max = 0;
+				state->opp[i].fmt_spatial_dither_frame_counter_bit_swap = 0;
+				state->opp[i].fmt_spatial_dither_enable = 0;
+				state->opp[i].fmt_spatial_dither_mode = 0;
+				state->opp[i].fmt_spatial_dither_depth = 0;
+				state->opp[i].fmt_temporal_dither_enable = 0;
+			}
+
+			/* Truncation control for bit depth reduction */
+			if (timing->display_color_depth < COLOR_DEPTH_121212) {
+				state->opp[i].fmt_truncate_enable = 1;
+				state->opp[i].fmt_truncate_depth = timing->display_color_depth;
+				state->opp[i].fmt_truncate_mode = 0; /* Round mode */
+			} else {
+				state->opp[i].fmt_truncate_enable = 0;
+				state->opp[i].fmt_truncate_depth = 0;
+				state->opp[i].fmt_truncate_mode = 0;
+			}
+
+			/* Data clamping control */
+			state->opp[i].fmt_clamp_data_enable = 1; /* Clamping typically enabled */
+			state->opp[i].fmt_clamp_color_format = timing->pixel_encoding;
+
+			/* Dynamic expansion for limited range content */
+			if (timing->pixel_encoding != PIXEL_ENCODING_RGB) {
+				state->opp[i].fmt_dynamic_exp_enable = 1; /* YCbCr typically needs expansion */
+				state->opp[i].fmt_dynamic_exp_mode = 0; /* Standard expansion */
+			} else {
+				state->opp[i].fmt_dynamic_exp_enable = 0; /* RGB typically full range */
+				state->opp[i].fmt_dynamic_exp_mode = 0;
+			}
+
+			/* Legacy field for compatibility */
+			state->opp[i].fmt_bit_depth_control = timing->display_color_depth;
+
+			/* Output Buffer (OPPBUF) Control - 6 fields */
+			state->opp[i].oppbuf_active_width = timing->h_addressable;
+			state->opp[i].oppbuf_pixel_repetition = 0; /* No pixel repetition by default */
+
+			/* Multi-Stream Output (MSO) / ODM segmentation */
+			if (pipe_ctx->next_odm_pipe) {
+				state->opp[i].oppbuf_display_segmentation = 1; /* Segmented display */
+				state->opp[i].oppbuf_overlap_pixel_num = 0; /* ODM overlap pixels */
+			} else {
+				state->opp[i].oppbuf_display_segmentation = 0; /* Single segment */
+				state->opp[i].oppbuf_overlap_pixel_num = 0;
+			}
+
+			/* 3D/Stereo control */
+			if (timing->timing_3d_format != TIMING_3D_FORMAT_NONE) {
+				state->opp[i].oppbuf_3d_vact_space1_size = 30; /* Typical stereo blanking */
+				state->opp[i].oppbuf_3d_vact_space2_size = 30;
+			} else {
+				state->opp[i].oppbuf_3d_vact_space1_size = 0;
+				state->opp[i].oppbuf_3d_vact_space2_size = 0;
+			}
+
+			/* DSC Forward Config - 3 fields */
+			if (timing->dsc_cfg.num_slices_h > 0) {
+				state->opp[i].dscrm_dsc_forward_enable = 1;
+				state->opp[i].dscrm_dsc_opp_pipe_source = pipe_ctx->stream_res.opp->inst;
+				state->opp[i].dscrm_dsc_forward_enable_status = 1; /* Status follows enable */
+			} else {
+				state->opp[i].dscrm_dsc_forward_enable = 0;
+				state->opp[i].dscrm_dsc_opp_pipe_source = 0;
+				state->opp[i].dscrm_dsc_forward_enable_status = 0;
+			}
+		} else {
+			/* No OPP resource - set all fields to disabled state */
+			memset(&state->opp[i], 0, sizeof(state->opp[i]));
+		}
+	}
+
+	/* Capture OPTC programming state for each pipe - comprehensive register field coverage */
+	for (i = 0; i < max_pipes && i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
+
+		if (!pipe_ctx->stream)
+			continue;
+
+		if (pipe_ctx->stream_res.tg) {
+			struct dc_crtc_timing *timing = &pipe_ctx->stream->timing;
+
+			state->optc[i].otg_master_inst = pipe_ctx->stream_res.tg->inst;
+
+			/* OTG_CONTROL register - 5 fields */
+			state->optc[i].otg_master_enable = 1; /* Active stream */
+			state->optc[i].otg_disable_point_cntl = 0; /* Normal operation */
+			state->optc[i].otg_start_point_cntl = 0; /* Normal start */
+			state->optc[i].otg_field_number_cntl = (timing->flags.INTERLACE) ? 1 : 0;
+			state->optc[i].otg_out_mux = 0; /* Direct output */
+
+			/* OTG Horizontal Timing - 7 fields */
+			state->optc[i].otg_h_total = timing->h_total;
+			state->optc[i].otg_h_blank_start = timing->h_addressable;
+			state->optc[i].otg_h_blank_end = timing->h_total - timing->h_front_porch;
+			state->optc[i].otg_h_sync_start = timing->h_addressable + timing->h_front_porch;
+			state->optc[i].otg_h_sync_end = timing->h_addressable + timing->h_front_porch + timing->h_sync_width;
+			state->optc[i].otg_h_sync_polarity = timing->flags.HSYNC_POSITIVE_POLARITY ? 0 : 1;
+			state->optc[i].otg_h_timing_div_mode = (pipe_ctx->next_odm_pipe) ? 1 : 0; /* ODM divide mode */
+
+			/* OTG Vertical Timing - 7 fields */
+			state->optc[i].otg_v_total = timing->v_total;
+			state->optc[i].otg_v_blank_start = timing->v_addressable;
+			state->optc[i].otg_v_blank_end = timing->v_total - timing->v_front_porch;
+			state->optc[i].otg_v_sync_start = timing->v_addressable + timing->v_front_porch;
+			state->optc[i].otg_v_sync_end = timing->v_addressable + timing->v_front_porch + timing->v_sync_width;
+			state->optc[i].otg_v_sync_polarity = timing->flags.VSYNC_POSITIVE_POLARITY ? 0 : 1;
+			state->optc[i].otg_v_sync_mode = 0; /* Normal sync mode */
+
+			/* Initialize remaining core fields with appropriate defaults */
+			// TODO: Update logic for accurate vtotal min/max
+			state->optc[i].otg_v_total_max = timing->v_total + 100; /* Typical DRR range */
+			state->optc[i].otg_v_total_min = timing->v_total - 50;
+			state->optc[i].otg_v_total_mid = timing->v_total;
+
+			/* ODM configuration */
+			// TODO: Update logic to have complete ODM mappings (e.g. 3:1 and 4:1) stored in single pipe
+			if (pipe_ctx->next_odm_pipe) {
+				state->optc[i].optc_seg0_src_sel = pipe_ctx->stream_res.opp ? pipe_ctx->stream_res.opp->inst : 0;
+				state->optc[i].optc_seg1_src_sel = pipe_ctx->next_odm_pipe->stream_res.opp ? pipe_ctx->next_odm_pipe->stream_res.opp->inst : 0;
+				state->optc[i].optc_num_of_input_segment = 1; /* 2 segments - 1 */
+			} else {
+				state->optc[i].optc_seg0_src_sel = pipe_ctx->stream_res.opp ? pipe_ctx->stream_res.opp->inst : 0;
+				state->optc[i].optc_seg1_src_sel = 0;
+				state->optc[i].optc_num_of_input_segment = 0; /* Single segment */
+			}
+
+			/* DSC configuration */
+			if (timing->dsc_cfg.num_slices_h > 0) {
+				state->optc[i].optc_dsc_mode = 1; /* DSC enabled */
+				state->optc[i].optc_dsc_bytes_per_pixel = timing->dsc_cfg.bits_per_pixel / 16; /* Convert to bytes */
+				state->optc[i].optc_dsc_slice_width = timing->h_addressable / timing->dsc_cfg.num_slices_h;
+			} else {
+				state->optc[i].optc_dsc_mode = 0;
+				state->optc[i].optc_dsc_bytes_per_pixel = 0;
+				state->optc[i].optc_dsc_slice_width = 0;
+			}
+
+			/* Essential control fields */
+			state->optc[i].otg_stereo_enable = (timing->timing_3d_format != TIMING_3D_FORMAT_NONE) ? 1 : 0;
+			state->optc[i].otg_interlace_enable = timing->flags.INTERLACE ? 1 : 0;
+			state->optc[i].otg_clock_enable = 1; /* OTG clock enabled */
+			state->optc[i].vtg0_enable = 1; /* VTG enabled for timing generation */
+
+			/* Initialize other key fields to defaults */
+			state->optc[i].optc_input_pix_clk_en = 1;
+			state->optc[i].optc_segment_width = (pipe_ctx->next_odm_pipe) ? (timing->h_addressable / 2) : timing->h_addressable;
+			state->optc[i].otg_vready_offset = 1;
+			state->optc[i].otg_vstartup_start = timing->v_addressable + 10;
+			state->optc[i].otg_vupdate_offset = 0;
+			state->optc[i].otg_vupdate_width = 5;
+		} else {
+			/* No timing generator resource - initialize all fields to 0 */
+			memset(&state->optc[i], 0, sizeof(state->optc[i]));
+		}
+	}
+
+	state->state_valid = true;
+	return true;
+}
+
+void dc_log_preos_dmcub_info(const struct dc *dc)
+{
+	dc_dmub_srv_log_preos_dmcub_info(dc->ctx->dmub_srv);
+}
+
+bool dc_get_qos_info(struct dc *dc, struct dc_qos_info *info)
+{
+	const struct dc_clocks *clk = &dc->current_state->bw_ctx.bw.dcn.clk;
+	struct dc_measured_memory_qos measured = {};
+	struct dc_requested_memory_qos requested = {};
+
+	memset(info, 0, sizeof(*info));
+
+	// Check if measurement function is available
+	if (!dc->hwss.measure_memory_qos) {
+		return false;
+	}
+
+	dc->hwss.measure_memory_qos(dc, &measured);
+
+	info->actual_peak_bw_in_mbps    = measured.peak_bw_mbps;
+	info->actual_avg_bw_in_mbps     = measured.avg_bw_mbps;
+	info->actual_min_latency_in_ns  = measured.min_latency_ns;
+	info->actual_max_latency_in_ns  = measured.max_latency_ns;
+	info->actual_avg_latency_in_ns  = measured.avg_latency_ns;
+	info->dcn_bandwidth_ub_in_mbps  = (uint32_t)(clk->fclk_khz / 1000 * 64);
+
+	if (dc->clk_mgr && dc->clk_mgr->funcs->get_requested_memory_qos) {
+		dc->clk_mgr->funcs->get_requested_memory_qos(dc->clk_mgr, &requested);
+		info->qos_bandwidth_lb_in_mbps    = requested.bandwidth_lb_in_mbps;
+		info->calculated_avg_bw_in_mbps   = requested.calculated_avg_bw_in_mbps;
+		info->qos_max_latency_ub_in_ns    = requested.max_latency_ub_in_ns;
+		info->qos_avg_latency_ub_in_ns    = requested.avg_latency_ub_in_ns;
+		info->qos_max_bw_budget_in_mbps   = requested.max_bw_budget_in_mbps;
+	}
+
+	return true;
+}
+
+unsigned int dc_override_memory_bandwidth_request(
+		struct dc *dc,
+		unsigned int bw_mbps)
+{
+	if (!dc->clk_mgr || !dc->clk_mgr->funcs)
+		return 0;
+
+	return dc->clk_mgr->funcs->override_memory_bandwidth_request(
+			dc->clk_mgr, bw_mbps * 1000) / 1000;
+}
+
+enum update_v3_flow {
+	UPDATE_V3_FLOW_INVALID,
+	UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FAST,
+	UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FULL,
+	UPDATE_V3_FLOW_NEW_CONTEXT_SEAMLESS,
+	UPDATE_V3_FLOW_NEW_CONTEXT_MINIMAL_NEW,
+	UPDATE_V3_FLOW_NEW_CONTEXT_MINIMAL_CURRENT,
+};
+
+struct dc_update_scratch_space {
+	struct dc *dc;
+	struct dc_surface_update *surface_updates;
+	int surface_count;
+	struct dc_stream_state *stream;
+	struct dc_stream_update *stream_update;
+	bool update_v3;
+	bool do_clear_update_bits;
+	enum surface_update_type update_type;
+	struct dc_state *new_context;
+	enum update_v3_flow flow;
+	struct dc_state *backup_context;
+	struct dc_state *intermediate_context;
+	struct pipe_split_policy_backup intermediate_policy;
+	struct dc_surface_update intermediate_updates[MAX_SURFACES];
+	int intermediate_count;
+};
+
+size_t dc_update_scratch_space_size(void)
+{
+	return sizeof(struct dc_update_scratch_space);
+}
+
+static bool update_planes_and_stream_prepare_v2(
+		struct dc_update_scratch_space *scratch
+)
+{
+	// v2 is too tangled to break into stages, so just execute everything under lock
+	dc_exit_ips_for_hw_access(scratch->dc);
+	return update_planes_and_stream_v2(
+		scratch->dc,
+		scratch->surface_updates,
+		scratch->surface_count,
+		scratch->stream,
+		scratch->stream_update
+	);
+}
+
+static void update_planes_and_stream_execute_v2(
+		const struct dc_update_scratch_space *scratch
+)
+{
+	// Nothing to do, see `update_planes_and_stream_prepare_v2`
+	(void) scratch;
+}
+
+static bool update_planes_and_stream_cleanup_v2(
+		const struct dc_update_scratch_space *scratch
+)
+{
+	if (scratch->do_clear_update_bits)
+		clear_update_bits(scratch->surface_updates, scratch->surface_count, scratch->stream);
+
+	return false;
+}
+
+static void update_planes_and_stream_cleanup_v3_release_minimal(
+		struct dc_update_scratch_space *scratch,
+		bool backup
+);
+
+static bool update_planes_and_stream_prepare_v3_intermediate_seamless(
+		struct dc_update_scratch_space *scratch
+)
+{
+	return is_pipe_topology_transition_seamless_with_intermediate_step(
+			scratch->dc,
+			scratch->dc->current_state,
+			scratch->intermediate_context,
+			scratch->new_context
+	);
+}
+
+static void transition_countdown_init(struct dc *dc)
+{
+	dc->check_config.transition_countdown_to_steady_state =
+			dc->debug.num_fast_flips_to_steady_state_override ?
+			dc->debug.num_fast_flips_to_steady_state_override :
+			NUM_FAST_FLIPS_TO_STEADY_STATE;
+}
+
+static bool update_planes_and_stream_prepare_v3(
+		struct dc_update_scratch_space *scratch
+)
+{
+	if (scratch->flow == UPDATE_V3_FLOW_NEW_CONTEXT_SEAMLESS) {
+		return true;
+	}
+	ASSERT(scratch->flow == UPDATE_V3_FLOW_INVALID);
+	dc_exit_ips_for_hw_access(scratch->dc);
+
+	/* HWSS path determination needs to be done prior to updating the surface and stream states. */
+	struct dc_fast_update fast_update[MAX_SURFACES] = { 0 };
+
+	populate_fast_updates(fast_update,
+			      scratch->surface_updates,
+			      scratch->surface_count,
+			      scratch->stream_update);
+
+	const bool is_hwss_fast_path_only =
+		fast_update_only(scratch->dc,
+				 fast_update,
+				 scratch->surface_updates,
+				 scratch->surface_count,
+				 scratch->stream_update,
+				 scratch->stream) &&
+		!scratch->dc->check_config.enable_legacy_fast_update;
+
+	if (!update_planes_and_stream_state(
+			scratch->dc,
+			scratch->surface_updates,
+			scratch->surface_count,
+			scratch->stream,
+			scratch->stream_update,
+			&scratch->update_type,
+			&scratch->new_context
+	)) {
+		return false;
+	}
+
+	if (scratch->new_context == scratch->dc->current_state) {
+		ASSERT(scratch->update_type < UPDATE_TYPE_FULL);
+
+		scratch->flow = is_hwss_fast_path_only
+				? UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FAST
+				: UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FULL;
+		return true;
+	}
+
+	ASSERT(scratch->update_type >= UPDATE_TYPE_FULL);
+
+	const bool seamless = scratch->dc->hwss.is_pipe_topology_transition_seamless(
+			scratch->dc,
+			scratch->dc->current_state,
+			scratch->new_context
+	);
+	if (seamless) {
+		scratch->flow = UPDATE_V3_FLOW_NEW_CONTEXT_SEAMLESS;
+		if (scratch->dc->check_config.deferred_transition_state)
+			/* reset countdown as steady state not reached */
+			transition_countdown_init(scratch->dc);
+		return true;
+	}
+
+	if (!scratch->dc->debug.disable_deferred_minimal_transitions) {
+		scratch->dc->check_config.deferred_transition_state = true;
+		transition_countdown_init(scratch->dc);
+	}
+
+	scratch->intermediate_context = create_minimal_transition_state(
+		scratch->dc,
+		scratch->new_context,
+		&scratch->intermediate_policy
+	);
+	if (scratch->intermediate_context) {
+		if (update_planes_and_stream_prepare_v3_intermediate_seamless(scratch)) {
+			scratch->flow = UPDATE_V3_FLOW_NEW_CONTEXT_MINIMAL_NEW;
+			return true;
+		}
+
+		update_planes_and_stream_cleanup_v3_release_minimal(scratch, false);
+	}
+
+	scratch->backup_context = scratch->dc->current_state;
+	restore_planes_and_stream_state(&scratch->dc->scratch.current_state, scratch->stream);
+	dc_state_retain(scratch->backup_context);
+	scratch->intermediate_context = create_minimal_transition_state(
+			scratch->dc,
+			scratch->backup_context,
+			&scratch->intermediate_policy
+	);
+	if (scratch->intermediate_context) {
+		if (update_planes_and_stream_prepare_v3_intermediate_seamless(scratch)) {
+			scratch->flow = UPDATE_V3_FLOW_NEW_CONTEXT_MINIMAL_CURRENT;
+			scratch->intermediate_count = initialize_empty_surface_updates(
+					scratch->stream, scratch->intermediate_updates
+			);
+			return true;
+		}
+
+		update_planes_and_stream_cleanup_v3_release_minimal(scratch, true);
+	}
+
+	scratch->flow = UPDATE_V3_FLOW_INVALID;
+	dc_state_release(scratch->backup_context);
+	restore_planes_and_stream_state(&scratch->dc->scratch.new_state, scratch->stream);
+	return false;
+}
+
+static void update_planes_and_stream_execute_v3_commit(
+		const struct dc_update_scratch_space *scratch,
+		bool intermediate_update,
+		bool intermediate_context,
+		bool use_stream_update
+)
+{
+	commit_planes_for_stream(
+			scratch->dc,
+			intermediate_update ? scratch->intermediate_updates : scratch->surface_updates,
+			intermediate_update ? scratch->intermediate_count : scratch->surface_count,
+			scratch->stream,
+			use_stream_update ? scratch->stream_update : NULL,
+			intermediate_context ? UPDATE_TYPE_FULL : scratch->update_type,
+			// `dc->current_state` only used in `NO_NEW_CONTEXT`, where it is equal to `new_context`
+			intermediate_context ? scratch->intermediate_context : scratch->new_context
+	);
+}
+
+static void update_planes_and_stream_execute_v3(
+		const struct dc_update_scratch_space *scratch
+)
+{
+	switch (scratch->flow) {
+	case UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FAST:
+		commit_planes_for_stream_fast(
+				scratch->dc,
+				scratch->surface_updates,
+				scratch->surface_count,
+				scratch->stream,
+				scratch->stream_update,
+				scratch->update_type,
+				scratch->new_context
+		);
+		break;
+
+	case UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FULL:
+	case UPDATE_V3_FLOW_NEW_CONTEXT_SEAMLESS:
+		update_planes_and_stream_execute_v3_commit(scratch, false, false, true);
+		break;
+
+	case UPDATE_V3_FLOW_NEW_CONTEXT_MINIMAL_NEW:
+		update_planes_and_stream_execute_v3_commit(scratch, false, true,
+				scratch->dc->check_config.deferred_transition_state);
+		break;
+
+	case UPDATE_V3_FLOW_NEW_CONTEXT_MINIMAL_CURRENT:
+		update_planes_and_stream_execute_v3_commit(scratch, true, true, false);
+		break;
+
+	case UPDATE_V3_FLOW_INVALID:
+	default:
+		ASSERT(false);
+	}
+}
+
+static void update_planes_and_stream_cleanup_v3_release_minimal(
+		struct dc_update_scratch_space *scratch,
+		bool backup
+)
+{
+	release_minimal_transition_state(
+			scratch->dc,
+			scratch->intermediate_context,
+			backup ? scratch->backup_context : scratch->new_context,
+			&scratch->intermediate_policy
+	);
+}
+
+static void update_planes_and_stream_cleanup_v3_intermediate(
+		struct dc_update_scratch_space *scratch,
+		bool backup
+)
+{
+	swap_and_release_current_context(scratch->dc, scratch->intermediate_context, scratch->stream);
+	dc_state_retain(scratch->dc->current_state);
+	update_planes_and_stream_cleanup_v3_release_minimal(scratch, backup);
+}
+
+static bool update_planes_and_stream_cleanup_v3(
+		struct dc_update_scratch_space *scratch
+)
+{
+	switch (scratch->flow) {
+	case UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FAST:
+	case UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FULL:
+		if (scratch->dc->check_config.transition_countdown_to_steady_state)
+			scratch->dc->check_config.transition_countdown_to_steady_state--;
+		break;
+
+	case UPDATE_V3_FLOW_NEW_CONTEXT_SEAMLESS:
+		swap_and_release_current_context(scratch->dc, scratch->new_context, scratch->stream);
+		break;
+
+	case UPDATE_V3_FLOW_NEW_CONTEXT_MINIMAL_NEW:
+		update_planes_and_stream_cleanup_v3_intermediate(scratch, false);
+		if (scratch->dc->check_config.deferred_transition_state) {
+			dc_state_release(scratch->new_context);
+		} else {
+			scratch->flow = UPDATE_V3_FLOW_NEW_CONTEXT_SEAMLESS;
+			return true;
+		}
+		break;
+
+	case UPDATE_V3_FLOW_NEW_CONTEXT_MINIMAL_CURRENT:
+		update_planes_and_stream_cleanup_v3_intermediate(scratch, true);
+		dc_state_release(scratch->backup_context);
+		restore_planes_and_stream_state(&scratch->dc->scratch.new_state, scratch->stream);
+		scratch->flow = UPDATE_V3_FLOW_NEW_CONTEXT_SEAMLESS;
+		return true;
+
+	case UPDATE_V3_FLOW_INVALID:
+	default:
+		ASSERT(false);
+	}
+
+	if (scratch->do_clear_update_bits)
+		clear_update_bits(scratch->surface_updates, scratch->surface_count, scratch->stream);
+
+	return false;
+}
+
+struct dc_update_scratch_space *dc_update_planes_and_stream_init(
+		struct dc *dc,
+		struct dc_surface_update *surface_updates,
+		int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update
+)
+{
+	const enum dce_version version = dc->ctx->dce_version;
+	struct dc_update_scratch_space *scratch = stream->update_scratch;
+
+	*scratch = (struct dc_update_scratch_space){
+		.dc = dc,
+		.surface_updates = surface_updates,
+		.surface_count = surface_count,
+		.stream = stream,
+		.stream_update = stream_update,
+		.update_v3 = version >= DCN_VERSION_4_01 || version == DCN_VERSION_3_2 || version == DCN_VERSION_3_21,
+		.do_clear_update_bits = version >= DCN_VERSION_1_0,
+	};
+
+	return scratch;
+}
+
+bool dc_update_planes_and_stream_prepare(
+		struct dc_update_scratch_space *scratch
+)
+{
+	return scratch->update_v3
+			? update_planes_and_stream_prepare_v3(scratch)
+			: update_planes_and_stream_prepare_v2(scratch);
+}
+
+void dc_update_planes_and_stream_execute(
+		const struct dc_update_scratch_space *scratch
+)
+{
+	scratch->update_v3
+			? update_planes_and_stream_execute_v3(scratch)
+			: update_planes_and_stream_execute_v2(scratch);
+}
+
+bool dc_update_planes_and_stream_cleanup(
+		struct dc_update_scratch_space *scratch
+)
+{
+	return scratch->update_v3
+			? update_planes_and_stream_cleanup_v3(scratch)
+			: update_planes_and_stream_cleanup_v2(scratch);
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
index 88446817a71f3..e47c8cf5d036e 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
@@ -37,6 +37,7 @@
 #include "dchubbub.h"
 #include "dccg.h"
 #include "abm.h"
+#include "clk_mgr.h"
 #include "dcn10/dcn10_hubbub.h"
 #include "dce/dmub_hw_lock_mgr.h"
 #include "link_service.h"
@@ -1028,20 +1029,20 @@ void hwss_build_fast_sequence(struct dc *dc,
 		current_mpc_pipe = current_pipe;
 		while (current_mpc_pipe) {
 			if (current_mpc_pipe->plane_state) {
-				if (dc->hwss.set_flip_control_gsl && current_mpc_pipe->plane_state->update_flags.raw) {
+				if (dc->hwss.set_flip_control_gsl && dc_pipe_update_bits_is_any_set(&current_mpc_pipe->plane_state->update_bits)) {
 					block_sequence[*num_steps].params.set_flip_control_gsl_params.hubp = current_mpc_pipe->plane_res.hubp;
 					block_sequence[*num_steps].params.set_flip_control_gsl_params.flip_immediate = current_mpc_pipe->plane_state->flip_immediate;
 					block_sequence[*num_steps].func = HUBP_SET_FLIP_CONTROL_GSL;
 					(*num_steps)++;
 				}
-				if (dc->hwss.program_triplebuffer && dc->debug.enable_tri_buf && current_mpc_pipe->plane_state->update_flags.raw) {
+				if (dc->hwss.program_triplebuffer && dc->debug.enable_tri_buf && dc_pipe_update_bits_is_any_set(&current_mpc_pipe->plane_state->update_bits)) {
 					block_sequence[*num_steps].params.program_triplebuffer_params.dc = dc;
 					block_sequence[*num_steps].params.program_triplebuffer_params.pipe_ctx = current_mpc_pipe;
 					block_sequence[*num_steps].params.program_triplebuffer_params.enableTripleBuffer = current_mpc_pipe->plane_state->triplebuffer_flips;
 					block_sequence[*num_steps].func = HUBP_PROGRAM_TRIPLEBUFFER;
 					(*num_steps)++;
 				}
-				if (dc->hwss.update_plane_addr && current_mpc_pipe->plane_state->update_flags.bits.addr_update) {
+				if (dc->hwss.update_plane_addr && current_mpc_pipe->plane_state->update_bits.addr_update) {
 					if (resource_is_pipe_type(current_mpc_pipe, OTG_MASTER) &&
 							stream_status->mall_stream_config.type == SUBVP_MAIN) {
 						block_sequence[*num_steps].params.subvp_save_surf_addr.dc_dmub_srv = dc->ctx->dmub_srv;
@@ -1057,7 +1058,7 @@ void hwss_build_fast_sequence(struct dc *dc,
 					(*num_steps)++;
 				}
 
-				if (hws->funcs.set_input_transfer_func && current_mpc_pipe->plane_state->update_flags.bits.gamma_change) {
+				if (hws->funcs.set_input_transfer_func && current_mpc_pipe->plane_state->update_bits.gamma_change) {
 					block_sequence[*num_steps].params.set_input_transfer_func_params.dc = dc;
 					block_sequence[*num_steps].params.set_input_transfer_func_params.pipe_ctx = current_mpc_pipe;
 					block_sequence[*num_steps].params.set_input_transfer_func_params.plane_state = current_mpc_pipe->plane_state;
@@ -1066,23 +1067,23 @@ void hwss_build_fast_sequence(struct dc *dc,
 				}
 
 				if (dc->hwss.program_gamut_remap &&
-						(current_mpc_pipe->plane_state->update_flags.bits.gamut_remap_change ||
+						(current_mpc_pipe->plane_state->update_bits.gamut_remap_change ||
 						 current_mpc_pipe->stream->update_flags.bits.gamut_remap)) {
 					block_sequence[*num_steps].params.program_gamut_remap_params.pipe_ctx = current_mpc_pipe;
 					block_sequence[*num_steps].func = DPP_PROGRAM_GAMUT_REMAP;
 					(*num_steps)++;
 				}
-				if (current_mpc_pipe->plane_state->update_flags.bits.input_csc_change) {
+				if (current_mpc_pipe->plane_state->update_bits.input_csc_change) {
 					block_sequence[*num_steps].params.setup_dpp_params.pipe_ctx = current_mpc_pipe;
 					block_sequence[*num_steps].func = DPP_SETUP_DPP;
 					(*num_steps)++;
 				}
-				if (current_mpc_pipe->plane_state->update_flags.bits.coeff_reduction_change) {
+				if (current_mpc_pipe->plane_state->update_bits.coeff_reduction_change) {
 					block_sequence[*num_steps].params.program_bias_and_scale_params.pipe_ctx = current_mpc_pipe;
 					block_sequence[*num_steps].func = DPP_PROGRAM_BIAS_AND_SCALE;
 					(*num_steps)++;
 				}
-				if (current_mpc_pipe->plane_state->update_flags.bits.cm_hist_change) {
+				if (current_mpc_pipe->plane_state->update_bits.cm_hist_change) {
 					block_sequence[*num_steps].params.control_cm_hist_params.dpp
 						= current_mpc_pipe->plane_res.dpp;
 					block_sequence[*num_steps].params.control_cm_hist_params.cm_hist_control
@@ -1095,7 +1096,7 @@ void hwss_build_fast_sequence(struct dc *dc,
 
 				if (current_mpc_pipe->plane_res.dpp &&
 						current_mpc_pipe->plane_res.dpp->funcs->set_cursor_matrix &&
-						current_mpc_pipe->plane_state->update_flags.bits.cursor_csc_color_matrix_change) {
+						current_mpc_pipe->plane_state->update_bits.cursor_csc_color_matrix_change) {
 					block_sequence[*num_steps].params.dpp_set_cursor_matrix_params.dpp = current_mpc_pipe->plane_res.dpp;
 					block_sequence[*num_steps].params.dpp_set_cursor_matrix_params.color_space = current_mpc_pipe->plane_state->color_space;
 					block_sequence[*num_steps].params.dpp_set_cursor_matrix_params.cursor_csc_color_matrix = &current_mpc_pipe->plane_state->cursor_csc_color_matrix;
@@ -1176,7 +1177,7 @@ void hwss_build_fast_sequence(struct dc *dc,
 		while (current_mpc_pipe) {
 			if (!current_mpc_pipe->bottom_pipe && !current_mpc_pipe->next_odm_pipe &&
 					current_mpc_pipe->stream && current_mpc_pipe->plane_state &&
-					current_mpc_pipe->plane_state->update_flags.bits.addr_update &&
+					current_mpc_pipe->plane_state->update_bits.addr_update &&
 					!current_mpc_pipe->plane_state->skip_manual_trigger) {
 				if (dc->hwss.program_cursor_offload_now) {
 					block_sequence[*num_steps].params.program_cursor_update_now_params.dc = dc;
@@ -1668,6 +1669,21 @@ void hwss_execute_sequence(struct dc *dc,
 		case LINK_SET_DPMS_ON:
 			hwss_link_set_dpms_on(params);
 			break;
+		case CLK_MGR_SET_MAX_MEMCLK:
+			hwss_clk_mgr_set_max_memclk(params);
+			break;
+		case CLK_MGR_UPDATE_CLOCKS:
+			hwss_clk_mgr_update_clocks(params);
+			break;
+		case HUBBUB_PROGRAM_WATERMARKS:
+			hwss_hubbub_program_watermarks(params);
+			break;
+		case HUBBUB_PROGRAM_ARBITER:
+			hwss_hubbub_program_arbiter(params);
+			break;
+		case HUBBUB_PROGRAM_COMPBUF_SEGMENTS:
+			hwss_hubbub_program_compbuf_segments(params);
+			break;
 		default:
 			ASSERT(false);
 			break;
@@ -3849,6 +3865,70 @@ void hwss_dsc_set_config_simple(union block_sequence_params *params)
 		dsc->funcs->dsc_set_config(dsc, dsc_cfg, dsc_optc_cfg);
 }
 
+/*
+ * Clock manager executor functions
+ */
+void hwss_clk_mgr_set_max_memclk(union block_sequence_params *params)
+{
+	struct clk_mgr *clk_mgr = params->clk_mgr_set_max_memclk_params.clk_mgr;
+	unsigned int memclk_mhz = params->clk_mgr_set_max_memclk_params.memclk_mhz;
+
+	if (clk_mgr && clk_mgr->funcs && clk_mgr->funcs->set_max_memclk)
+		clk_mgr->funcs->set_max_memclk(clk_mgr, memclk_mhz);
+}
+
+void hwss_clk_mgr_update_clocks(union block_sequence_params *params)
+{
+	struct clk_mgr *clk_mgr = params->clk_mgr_update_clocks_params.clk_mgr;
+
+	if (clk_mgr && clk_mgr->funcs && clk_mgr->funcs->execute_clk_mgr_block_sequence)
+		clk_mgr->funcs->execute_clk_mgr_block_sequence(clk_mgr);
+}
+
+/*
+ * Hubbub executor functions
+ */
+void hwss_hubbub_program_watermarks(union block_sequence_params *params)
+{
+	struct dc *dc = params->hubbub_program_watermarks_params.dc;
+	struct hubbub *hubbub = params->hubbub_program_watermarks_params.hubbub;
+	union dcn_watermark_set *watermarks = params->hubbub_program_watermarks_params.watermarks;
+	unsigned int refclk_mhz = params->hubbub_program_watermarks_params.refclk_mhz;
+	bool safe_to_lower = params->hubbub_program_watermarks_params.safe_to_lower;
+
+	if (hubbub && hubbub->funcs && hubbub->funcs->program_watermarks) {
+		bool wm_changed = hubbub->funcs->program_watermarks(hubbub, watermarks, refclk_mhz, safe_to_lower);
+
+		if (dc && !safe_to_lower)
+			dc->optimized_required |= wm_changed;
+	}
+}
+
+void hwss_hubbub_program_arbiter(union block_sequence_params *params)
+{
+	struct dc *dc = params->hubbub_program_arbiter_params.dc;
+	struct hubbub *hubbub = params->hubbub_program_arbiter_params.hubbub;
+	struct dml2_display_arb_regs *arb_regs = params->hubbub_program_arbiter_params.arb_regs;
+	bool safe_to_lower = params->hubbub_program_arbiter_params.safe_to_lower;
+
+	if (hubbub && hubbub->funcs && hubbub->funcs->program_arbiter) {
+		bool arb_changed = hubbub->funcs->program_arbiter(hubbub, arb_regs, safe_to_lower);
+
+		if (dc && !safe_to_lower)
+			dc->optimized_required |= arb_changed;
+	}
+}
+
+void hwss_hubbub_program_compbuf_segments(union block_sequence_params *params)
+{
+	struct hubbub *hubbub = params->hubbub_program_compbuf_segments_params.hubbub;
+	unsigned int compbuf_size = params->hubbub_program_compbuf_segments_params.compbuf_size;
+	bool safe_to_lower = params->hubbub_program_compbuf_segments_params.safe_to_lower;
+
+	if (hubbub && hubbub->funcs && hubbub->funcs->program_compbuf_segments)
+		hubbub->funcs->program_compbuf_segments(hubbub, compbuf_size, safe_to_lower);
+}
+
 void hwss_add_dccg_set_dto_dscclk(struct block_sequence_state *seq_state,
 		struct dccg *dccg, int inst, int num_slices_h)
 {
@@ -4909,6 +4989,9 @@ void hwss_add_hpo_dp_stream_enc_update_dp_info_packets_sdp_line_num(struct block
 	}
 }
 
+/*
+ * Clock manager helper functions
+ */
 void hwss_add_hpo_dp_stream_enc_update_dp_info_packets(struct block_sequence_state *seq_state,
 		struct pipe_ctx *pipe_ctx)
 {
@@ -4919,6 +5002,28 @@ void hwss_add_hpo_dp_stream_enc_update_dp_info_packets(struct block_sequence_sta
 	}
 }
 
+void hwss_add_clk_mgr_set_max_memclk(struct block_sequence_state *seq_state,
+		struct clk_mgr *clk_mgr,
+		unsigned int memclk_mhz)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = CLK_MGR_SET_MAX_MEMCLK;
+		seq_state->steps[*seq_state->num_steps].params.clk_mgr_set_max_memclk_params.clk_mgr = clk_mgr;
+		seq_state->steps[*seq_state->num_steps].params.clk_mgr_set_max_memclk_params.memclk_mhz = memclk_mhz;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_clk_mgr_update_clocks(struct block_sequence_state *seq_state,
+		struct clk_mgr *clk_mgr)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = CLK_MGR_UPDATE_CLOCKS;
+		seq_state->steps[*seq_state->num_steps].params.clk_mgr_update_clocks_params.clk_mgr = clk_mgr;
+		(*seq_state->num_steps)++;
+	}
+}
+
 void hwss_add_stream_enc_update_dp_info_packets_sdp_line_num(struct block_sequence_state *seq_state,
 		struct pipe_ctx *pipe_ctx)
 {
@@ -5022,6 +5127,26 @@ void hwss_add_setup_periodic_interrupt(struct block_sequence_state *seq_state,
 		(*seq_state->num_steps)++;
 	}
 }
+/*
+ * Hubbub helper functions
+ */
+void hwss_add_hubbub_program_watermarks(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct hubbub *hubbub,
+		union dcn_watermark_set *watermarks,
+		unsigned int refclk_mhz,
+		bool safe_to_lower)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = HUBBUB_PROGRAM_WATERMARKS;
+		seq_state->steps[*seq_state->num_steps].params.hubbub_program_watermarks_params.dc = dc;
+		seq_state->steps[*seq_state->num_steps].params.hubbub_program_watermarks_params.hubbub = hubbub;
+		seq_state->steps[*seq_state->num_steps].params.hubbub_program_watermarks_params.watermarks = watermarks;
+		seq_state->steps[*seq_state->num_steps].params.hubbub_program_watermarks_params.refclk_mhz = refclk_mhz;
+		seq_state->steps[*seq_state->num_steps].params.hubbub_program_watermarks_params.safe_to_lower = safe_to_lower;
+		(*seq_state->num_steps)++;
+	}
+}
 
 void hwss_add_dp_trace_source_sequence(struct block_sequence_state *seq_state,
 		struct dc_link *link,
@@ -5035,6 +5160,22 @@ void hwss_add_dp_trace_source_sequence(struct block_sequence_state *seq_state,
 	}
 }
 
+void hwss_add_hubbub_program_arbiter(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct hubbub *hubbub,
+		struct dml2_display_arb_regs *arb_regs,
+		bool safe_to_lower)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = HUBBUB_PROGRAM_ARBITER;
+		seq_state->steps[*seq_state->num_steps].params.hubbub_program_arbiter_params.dc = dc;
+		seq_state->steps[*seq_state->num_steps].params.hubbub_program_arbiter_params.hubbub = hubbub;
+		seq_state->steps[*seq_state->num_steps].params.hubbub_program_arbiter_params.arb_regs = arb_regs;
+		seq_state->steps[*seq_state->num_steps].params.hubbub_program_arbiter_params.safe_to_lower = safe_to_lower;
+		(*seq_state->num_steps)++;
+	}
+}
+
 void hwss_add_set_dmdata_attributes(struct block_sequence_state *seq_state,
 		struct pipe_ctx *pipe_ctx)
 {
@@ -5119,6 +5260,19 @@ void hwss_add_disable_audio_stream(struct block_sequence_state *seq_state,
 		(*seq_state->num_steps)++;
 	}
 }
+void hwss_add_hubbub_program_compbuf_segments(struct block_sequence_state *seq_state,
+		struct hubbub *hubbub,
+		unsigned int compbuf_size,
+		bool safe_to_lower)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = HUBBUB_PROGRAM_COMPBUF_SEGMENTS;
+		seq_state->steps[*seq_state->num_steps].params.hubbub_program_compbuf_segments_params.hubbub = hubbub;
+		seq_state->steps[*seq_state->num_steps].params.hubbub_program_compbuf_segments_params.compbuf_size = compbuf_size;
+		seq_state->steps[*seq_state->num_steps].params.hubbub_program_compbuf_segments_params.safe_to_lower = safe_to_lower;
+		(*seq_state->num_steps)++;
+	}
+}
 
 void hwss_add_prepare_bandwidth(struct block_sequence_state *seq_state,
 		struct dc *dc,
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
index 5f6cc1b1f7881..b21d41df0fab8 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
@@ -4412,14 +4412,13 @@ enum dc_status dc_validate_with_context(struct dc *dc,
 	struct dc_stream_state *unchanged_streams[MAX_PIPES] = { 0 };
 	struct dc_stream_state *del_streams[MAX_PIPES] = { 0 };
 	struct dc_stream_state *add_streams[MAX_PIPES] = { 0 };
-	int old_stream_count = context->stream_count;
+	unsigned int old_stream_count = context->stream_count;
 	enum dc_status res = DC_ERROR_UNEXPECTED;
-	int unchanged_streams_count = 0;
-	int del_streams_count = 0;
-	int add_streams_count = 0;
+	unsigned int unchanged_streams_count = 0;
+	unsigned int del_streams_count = 0;
+	unsigned int add_streams_count = 0;
 	bool found = false;
-	int i, j;
-	unsigned int k;
+	unsigned int i, j, k;
 
 	DC_LOGGER_INIT(dc->ctx->logger);
 
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
index 7666cdc78f4ed..dbc12640b01c0 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
@@ -720,20 +720,17 @@ bool dc_stream_remove_writeback(struct dc *dc,
 		return false;
 	}
 
-	/* remove writeback info for disabled writeback pipes from stream */
+	/* remove writeback info for the requested writeback pipe from stream */
 	for (i = 0, j = 0; i < stream->num_wb_info; i++) {
-		if (stream->writeback_info[i].wb_enabled) {
-
-			if (stream->writeback_info[i].dwb_pipe_inst == dwb_pipe_inst)
-				stream->writeback_info[i].wb_enabled = false;
+		/* drop every entry that targets the pipe being removed */
+		if (stream->writeback_info[i].dwb_pipe_inst == dwb_pipe_inst)
+			continue;
 
-			/* trim the array */
-			if (j < i) {
-				memcpy(&stream->writeback_info[j], &stream->writeback_info[i],
-						sizeof(struct dc_writeback_info));
-				j++;
-			}
-		}
+		/* keep this entry, compacting it down when earlier entries were removed */
+		if (j != i)
+			memcpy(&stream->writeback_info[j], &stream->writeback_info[i],
+					sizeof(struct dc_writeback_info));
+		j++;
 	}
 	stream->num_wb_info = j;
 
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
index 72845fc788f3f..88e825a6582cb 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
@@ -45,14 +45,13 @@ void dc_plane_construct(struct dc_context *ctx, struct dc_plane_state *plane_sta
 
 	plane_state->in_transfer_func.type = TF_TYPE_BYPASS;
 
-	plane_state->in_shaper_func.type = TF_TYPE_BYPASS;
-
-	plane_state->lut3d_func.state.raw = 0;
-
-	plane_state->blend_tf.type = TF_TYPE_BYPASS;
-
 	plane_state->pre_multiplied_alpha = true;
 
+	/* CM */
+	plane_state->cm.shaper_func.type = TF_TYPE_BYPASS;
+	plane_state->cm.blend_func.type = TF_TYPE_BYPASS;
+	plane_state->cm.lut3d_func.state.raw = 0;
+	plane_state->cm.flags.all = 0;
 }
 
 void dc_plane_destruct(struct dc_plane_state *plane_state)
@@ -282,6 +281,39 @@ void dc_3dlut_func_retain(struct dc_3dlut *lut)
 	kref_get(&lut->refcount);
 }
 
+static void dc_plane_cm_free(struct kref *kref)
+{
+	struct dc_plane_cm *cm = container_of(kref, struct dc_plane_cm, refcount);
+
+	kvfree(cm);
+}
+
+struct dc_plane_cm *dc_plane_cm_create(void)
+{
+	struct dc_plane_cm *cm = kvzalloc(sizeof(*cm), GFP_KERNEL);
+
+	if (cm == NULL)
+		goto alloc_fail;
+
+	kref_init(&cm->refcount);
+
+	return cm;
+
+alloc_fail:
+	return NULL;
+
+}
+
+void dc_plane_cm_release(struct dc_plane_cm *cm)
+{
+	kref_put(&cm->refcount, dc_plane_cm_free);
+}
+
+void dc_plane_cm_retain(struct dc_plane_cm *cm)
+{
+	kref_get(&cm->refcount);
+}
+
 void dc_plane_force_dcc_and_tiling_disable(struct dc_plane_state *plane_state,
 					   bool clear_tiling)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h
index 82d02ebbd8290..e1b7d359d9170 100644
--- a/drivers/gpu/drm/amd/display/dc/dc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc.h
@@ -65,7 +65,7 @@ struct dcn_dsc_reg_state;
 struct dcn_optc_reg_state;
 struct dcn_dccg_reg_state;
 
-#define DC_VER "3.2.384"
+#define DC_VER "3.2.389"
 
 /**
  * MAX_SURFACES - representative of the upper bound of surfaces that can be piped to a single CRTC
@@ -591,7 +591,6 @@ struct dc_config {
 	bool enable_mipi_converter_optimization;
 	bool enable_frl;
 	bool force_hdmi21_frl_enc_enable;
-	bool skip_frl_pretraining;
 	bool use_default_clock_table;
 	bool force_bios_enable_lttpr;
 	uint8_t force_bios_fixed_vs;
@@ -633,6 +632,7 @@ enum visual_confirm {
 	VISUAL_CONFIRM_SMARTMUX_DGPU = 10,
 	VISUAL_CONFIRM_REPLAY = 12,
 	VISUAL_CONFIRM_SUBVP = 14,
+	VISUAL_CONFIRM_ABM = 15,
 	VISUAL_CONFIRM_MCLK_SWITCH = 16,
 	VISUAL_CONFIRM_FAMS2 = 19,
 	VISUAL_CONFIRM_HW_CURSOR = 20,
@@ -1129,8 +1129,6 @@ struct dc_debug_options {
 	unsigned int force_fclk_khz;
 	bool enable_tri_buf;
 	bool ips_disallow_entry;
-	bool dmub_offload_enabled;
-	bool dmcub_emulation;
 	bool disable_idle_power_optimizations;
 	unsigned int mall_size_override;
 	unsigned int mall_additional_timer_percent;
@@ -1152,7 +1150,6 @@ struct dc_debug_options {
 	bool validate_dml_output;
 	bool enable_dmcub_surface_flip;
 	bool usbc_combo_phy_reset_wa;
-	bool force_vrr;
 	bool force_fva;
 	int max_frl_rate;
 	unsigned int  force_frl_rate;
@@ -1290,6 +1287,10 @@ struct dc_debug_options {
 	unsigned int min_deep_sleep_dcfclk_khz;
 	unsigned int force_odm2to1_for_edp_pixclk_mhz;
 	bool enable_replay_esd_recovery;
+	uint8_t iommu_mismatch_temp_wka;
+	bool disable_dynamic_expansion_for_test_pattern;
+	uint32_t dml21_custom_derate_num_dpms;
+	uint32_t dml21_custom_derate_at_dpm[DML2_MAX_NUM_DPM_LVL];
 };
 
 
@@ -1332,7 +1333,6 @@ struct dc_init_data {
 	enum dce_environment dce_environment;
 
 	struct dmub_offload_funcs *dmub_if;
-	struct dc_reg_helper_state *dmub_offload;
 
 	struct dc_config flags;
 	uint64_t log_mask;
@@ -1488,6 +1488,47 @@ struct dc_3dlut {
 	struct fixed31_32 hdr_multiplier;
 	union dc_3dlut_state state;
 };
+
+/* 3DLUT DMA (Fast Load) params */
+struct dc_3dlut_dma {
+	struct dc_plane_address addr;
+	enum dc_cm_lut_swizzle swizzle;
+	enum dc_cm_lut_pixel_format format;
+	uint16_t bias; /* FP1.5.10 */
+	uint16_t scale; /* FP1.5.10 */
+	enum dc_cm_lut_size size;
+};
+
+/* color manager */
+union dc_plane_cm_flags {
+	unsigned int all;
+	struct {
+		unsigned int shaper_enable    : 1;
+		unsigned int lut3d_enable     : 1;
+		unsigned int blend_enable     : 1;
+		/* whether legacy (lut3d_func) or DMA is valid */
+		unsigned int lut3d_dma_enable : 1;
+#if defined(CONFIG_DRM_AMD_DC_DCN4_2)
+		/* RMCM lut to be used instead of MCM */
+		unsigned int rmcm_enable	 : 1;
+		unsigned int reserved: 27;
+#else
+		unsigned int reserved: 28;
+#endif
+	} bits;
+};
+
+struct dc_plane_cm {
+	struct kref refcount;
+	struct dc_transfer_func shaper_func;
+	union {
+		struct dc_3dlut lut3d_func;
+		struct dc_3dlut_dma lut3d_dma;
+	};
+	struct dc_transfer_func blend_func;
+	union dc_plane_cm_flags flags;
+};
+
 /*
  * This structure is filled in by dc_surface_get_status and contains
  * the last requested address and the currently active address so the called
@@ -1501,47 +1542,120 @@ struct dc_plane_status {
 	struct cm_hist cm_hist;
 };
 
-union surface_update_flags {
-
-	struct {
-		uint32_t addr_update:1;
-		/* Medium updates */
-		uint32_t dcc_change:1;
-		uint32_t color_space_change:1;
-		uint32_t horizontal_mirror_change:1;
-		uint32_t per_pixel_alpha_change:1;
-		uint32_t global_alpha_change:1;
-		uint32_t hdr_mult:1;
-		uint32_t rotation_change:1;
-		uint32_t swizzle_change:1;
-		uint32_t scaling_change:1;
-		uint32_t position_change:1;
-		uint32_t in_transfer_func_change:1;
-		uint32_t input_csc_change:1;
-		uint32_t coeff_reduction_change:1;
-		uint32_t pixel_format_change:1;
-		uint32_t plane_size_change:1;
-		uint32_t gamut_remap_change:1;
-		uint32_t cursor_csc_color_matrix_change:1;
-
-		/* Full updates */
-		uint32_t new_plane:1;
-		uint32_t bpp_change:1;
-		uint32_t gamma_change:1;
-		uint32_t bandwidth_change:1;
-		uint32_t clock_change:1;
-		uint32_t stereo_format_change:1;
-		uint32_t lut_3d:1;
-		uint32_t tmz_changed:1;
-		uint32_t mcm_transfer_function_enable_change:1; /* disable or enable MCM transfer func */
-		uint32_t full_update:1;
-		uint32_t sdr_white_level_nits:1;
-		uint32_t cm_hist_change:1;
-	} bits;
-
-	uint32_t raw;
+struct pipe_update_bits {
+	uint32_t addr_update:1;
+	uint32_t dcc_change:1;
+	uint32_t color_space_change:1;
+	uint32_t horizontal_mirror_change:1;
+	uint32_t per_pixel_alpha_change:1;
+	uint32_t global_alpha_change:1;
+	uint32_t hdr_mult:1;
+	uint32_t rotation_change:1;
+	uint32_t swizzle_change:1;
+	uint32_t scaling_change:1;
+	uint32_t position_change:1;
+	uint32_t in_transfer_func_change:1;
+	uint32_t input_csc_change:1;
+	uint32_t coeff_reduction_change:1;
+	uint32_t pixel_format_change:1;
+	uint32_t plane_size_change:1;
+	uint32_t gamut_remap_change:1;
+	uint32_t cursor_csc_color_matrix_change:1;
+	uint32_t new_plane:1;
+	uint32_t bpp_change:1;
+	uint32_t gamma_change:1;
+	uint32_t bandwidth_change:1;
+	uint32_t clock_change:1;
+	uint32_t stereo_format_change:1;
+	uint32_t lut_3d:1;
+	uint32_t tmz_changed:1;
+	uint32_t mcm_transfer_function_enable_change:1; /* disable or enable MCM transfer func */
+	uint32_t full_update:1;
+	uint32_t sdr_white_level_nits:1;
+	uint32_t cm_hist_change:1;
+	/* NOTE: When adding a new field, also update:
+	 *   - dc_pipe_update_bits_set_full()
+	 *   - dc_pipe_update_bits_is_any_set()
+	 */
 };
 
+static inline void dc_pipe_update_bits_clear(struct pipe_update_bits *flags)
+{
+	/* memset ensures padding bits are zeroed */
+	memset(flags, 0, sizeof(*flags));
+}
+
+static inline void dc_pipe_update_bits_set_full(struct pipe_update_bits *flags)
+{
+	dc_pipe_update_bits_clear(flags);
+	flags->addr_update = 1;
+	flags->dcc_change = 1;
+	flags->color_space_change = 1;
+	flags->horizontal_mirror_change = 1;
+	flags->per_pixel_alpha_change = 1;
+	flags->global_alpha_change = 1;
+	flags->hdr_mult = 1;
+	flags->rotation_change = 1;
+	flags->swizzle_change = 1;
+	flags->scaling_change = 1;
+	flags->position_change = 1;
+	flags->in_transfer_func_change = 1;
+	flags->input_csc_change = 1;
+	flags->coeff_reduction_change = 1;
+	flags->pixel_format_change = 1;
+	flags->plane_size_change = 1;
+	flags->gamut_remap_change = 1;
+	flags->cursor_csc_color_matrix_change = 1;
+	flags->new_plane = 1;
+	flags->bpp_change = 1;
+	flags->gamma_change = 1;
+	flags->bandwidth_change = 1;
+	flags->clock_change = 1;
+	flags->stereo_format_change = 1;
+	flags->lut_3d = 1;
+	flags->tmz_changed = 1;
+	flags->mcm_transfer_function_enable_change = 1;
+	flags->full_update = 1;
+	flags->sdr_white_level_nits = 1;
+	flags->cm_hist_change = 1;
+}
+
+static inline bool dc_pipe_update_bits_is_any_set(const struct pipe_update_bits *flags)
+{
+	return flags->addr_update ||
+		flags->dcc_change ||
+		flags->color_space_change ||
+		flags->horizontal_mirror_change ||
+		flags->per_pixel_alpha_change ||
+		flags->global_alpha_change ||
+		flags->hdr_mult ||
+		flags->rotation_change ||
+		flags->swizzle_change ||
+		flags->scaling_change ||
+		flags->position_change ||
+		flags->in_transfer_func_change ||
+		flags->input_csc_change ||
+		flags->coeff_reduction_change ||
+		flags->pixel_format_change ||
+		flags->plane_size_change ||
+		flags->gamut_remap_change ||
+		flags->cursor_csc_color_matrix_change ||
+		flags->new_plane ||
+		flags->bpp_change ||
+		flags->gamma_change ||
+		flags->bandwidth_change ||
+		flags->clock_change ||
+		flags->stereo_format_change ||
+		flags->lut_3d ||
+		flags->tmz_changed ||
+		flags->mcm_transfer_function_enable_change ||
+		flags->full_update ||
+		flags->sdr_white_level_nits ||
+		flags->cm_hist_change;
+}
+
+bool dc_check_address_only_update(struct pipe_update_bits update_bits);
+
 #define DC_REMOVE_PLANE_POINTERS 1
 
 struct dc_plane_state {
@@ -1566,14 +1680,22 @@ struct dc_plane_state {
 	struct fixed31_32 hdr_mult;
 	struct colorspace_transform gamut_remap_matrix;
 
+	enum dc_color_space color_space;
+
+#ifndef TRIM_CM2
 	// TODO: No longer used, remove
 	struct dc_hdr_static_metadata hdr_static_ctx;
 
-	enum dc_color_space color_space;
-
 	struct dc_3dlut lut3d_func;
 	struct dc_transfer_func in_shaper_func;
 	struct dc_transfer_func blend_tf;
+	enum dc_cm2_shaper_3dlut_setting mcm_shaper_3dlut_setting;
+	bool mcm_lut1d_enable;
+	struct dc_cm2_func_luts mcm_luts;
+#endif /* TRIM_CM2 */
+	bool lut_bank_a;
+	enum mpcc_movable_cm_location mcm_location;
+	struct dc_plane_cm cm;
 
 	struct dc_transfer_func *gamcor_tf;
 	enum surface_pixel_format format;
@@ -1590,7 +1712,7 @@ struct dc_plane_state {
 	bool horizontal_mirror;
 	unsigned int layer_index;
 
-	union surface_update_flags update_flags;
+	struct pipe_update_bits update_bits;
 	bool flip_int_enabled;
 	bool skip_manual_trigger;
 
@@ -1610,11 +1732,6 @@ struct dc_plane_state {
 
 	bool is_statically_allocated;
 	enum chroma_cositing cositing;
-	enum dc_cm2_shaper_3dlut_setting mcm_shaper_3dlut_setting;
-	bool mcm_lut1d_enable;
-	struct dc_cm2_func_luts mcm_luts;
-	bool lut_bank_a;
-	enum mpcc_movable_cm_location mcm_location;
 	struct dc_csc_transform cursor_csc_color_matrix;
 	bool adaptive_sharpness_en;
 	int adaptive_sharpness_policy;
@@ -1728,6 +1845,10 @@ struct dc_scratch_space {
 	 * of ddc_pin to know which aux instance is associated with link.
 	 */
 	bool no_ddc_pin;
+	/** When set, forces all native I2C communication on this DP connector
+	 *  to use the I2C-over-AUX protocol instead of native I2C signaling.
+	 */
+	bool force_to_use_aux;
 	enum gpio_ddc_line aux_hw_inst;
 
 	enum gpio_ddc_line ddc_hw_inst;
@@ -1815,6 +1936,8 @@ struct dc_scratch_space {
 		bool dp_skip_DID2;
 		bool dp_skip_reset_segment;
 		bool dp_skip_fs_144hz;
+		/* Some DP bridges don't work with RBR and must use HBR. */
+		bool dp_skip_rbr;
 		bool dp_mot_reset_segment;
 		/* Some USB4 docks do not handle turning off MST DSC once it has been enabled. */
 		bool dpia_mst_dsc_always_on;
@@ -1978,17 +2101,7 @@ struct dc_surface_update {
 
 	const struct dc_csc_transform *input_csc_color_matrix;
 	const struct fixed31_32 *coeff_reduction_factor;
-	const struct dc_transfer_func *func_shaper;
-	const struct dc_3dlut *lut3d_func;
-	const struct dc_transfer_func *blend_tf;
 	const struct colorspace_transform *gamut_remap_matrix;
-	/*
-	 * Color Transformations for pre-blend MCM (Shaper, 3DLUT, 1DLUT)
-	 *
-	 * change cm2_params.component_settings: Full update
-	 * change cm2_params.cm2_luts: Fast update
-	 */
-	const struct dc_cm2_parameters *cm2_params;
 	const struct dc_plane_cm *cm;
 	const struct dc_csc_transform *cursor_csc_color_matrix;
 	unsigned int sdr_white_level_nits;
@@ -2034,6 +2147,10 @@ struct dc_3dlut *dc_create_3dlut_func(void);
 void dc_3dlut_func_release(struct dc_3dlut *lut);
 void dc_3dlut_func_retain(struct dc_3dlut *lut);
 
+struct dc_plane_cm *dc_plane_cm_create(void);
+void dc_plane_cm_release(struct dc_plane_cm *cm);
+void dc_plane_cm_retain(struct dc_plane_cm *cm);
+
 void dc_post_update_surfaces_to_stream(
 		struct dc *dc);
 
@@ -2881,6 +2998,7 @@ enum dc_irq_source dc_interrupt_to_irq_source(
 		uint32_t ext_id);
 bool dc_interrupt_set(struct dc *dc, enum dc_irq_source src, bool enable);
 void dc_interrupt_ack(struct dc *dc, enum dc_irq_source src);
+bool dc_get_flip_pending_on_otg(struct dc *dc, int otg_inst);
 enum dc_irq_source dc_get_hpd_irq_source_at_index(
 		struct dc *dc, uint32_t link_index);
 
@@ -2895,6 +3013,8 @@ void dc_resume(struct dc *dc);
 
 void dc_power_down_on_boot(struct dc *dc);
 
+void dc_disable_dangling_timing_generators(struct dc *dc);
+
 /*
  * HDCP Interfaces
  */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
index 0ee5c0c5545ca..80a21d7cae901 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
@@ -518,9 +518,6 @@ void dc_dmub_srv_query_caps_cmd(struct dc_dmub_srv *dc_dmub_srv)
 {
 	union dmub_rb_cmd cmd = { 0 };
 
-	if (dc_dmub_srv->ctx->dc->debug.dmcub_emulation)
-		return;
-
 	memset(&cmd, 0, sizeof(cmd));
 
 	/* Prepare fw command */
@@ -1302,9 +1299,6 @@ bool dc_dmub_srv_is_hw_pwr_up(struct dc_dmub_srv *dc_dmub_srv, bool wait)
 	if (!dc_dmub_srv || !dc_dmub_srv->dmub)
 		return true;
 
-	if (dc_dmub_srv->ctx->dc->debug.dmcub_emulation)
-		return true;
-
 	dc_ctx = dc_dmub_srv->ctx;
 
 	if (wait) {
@@ -1345,9 +1339,6 @@ static void dc_dmub_srv_notify_idle(const struct dc *dc, bool allow_idle)
 	struct dc_dmub_srv *dc_dmub_srv;
 	union dmub_rb_cmd cmd = {0};
 
-	if (dc->debug.dmcub_emulation)
-		return;
-
 	if (!dc->ctx->dmub_srv || !dc->ctx->dmub_srv->dmub)
 		return;
 
@@ -1466,9 +1457,6 @@ static void dc_dmub_srv_exit_low_power_state(const struct dc *dc)
 	struct dc_dmub_srv *dc_dmub_srv;
 	uint32_t rcg_exit_count = 0, ips1_exit_count = 0, ips2_exit_count = 0, ips1z8_exit_count = 0;
 
-	if (dc->debug.dmcub_emulation)
-		return;
-
 	if (!dc->ctx->dmub_srv || !dc->ctx->dmub_srv->dmub)
 		return;
 
@@ -1920,6 +1908,8 @@ static void dc_dmub_srv_ib_based_fams2_update_config(struct dc *dc,
 	config->global.features.bits.enable = enable && context->bw_ctx.bw.dcn.fams2_global_config.features.bits.enable;
 	config->global.features.bits.enable_ppt_check = dc->debug.fams2_config.bits.enable_ppt_check;
 
+	dmub_srv_flush_buffer_mem(dc->ctx->dmub_srv->dmub, &dc->ctx->dmub_srv->dmub->ib_mem_gart);
+
 	dm_execute_dmub_cmd_list(dc->ctx, 1, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 }
 
@@ -2085,7 +2075,7 @@ bool dc_dmub_srv_ips_query_residency_info(const struct dc_context *ctx, uint8_t
 	union dmub_rb_cmd cmd;
 	uint32_t bytes = sizeof(struct dmub_ips_residency_info);
 
-	dmub_flush_buffer_mem(&ctx->dmub_srv->dmub->scratch_mem_fb);
+	dmub_srv_flush_buffer_mem(ctx->dmub_srv->dmub, &ctx->dmub_srv->dmub->scratch_mem_fb);
 	memset(&cmd, 0, sizeof(cmd));
 
 	cmd.ips_query_residency_info.header.type = DMUB_CMD__IPS;
@@ -2137,9 +2127,7 @@ bool dmub_lsdma_init(struct dc_dmub_srv *dc_dmub_srv)
 
 bool dmub_lsdma_send_linear_copy_command(
 	struct dc_dmub_srv *dc_dmub_srv,
-	uint64_t src_addr,
-	uint64_t dst_addr,
-	uint32_t count
+	struct lsdma_linear_copy_params copy_data
 )
 {
 	struct dc_context *dc_ctx = dc_dmub_srv->ctx;
@@ -2154,11 +2142,20 @@ bool dmub_lsdma_send_linear_copy_command(
 	cmd.cmd_common.header.sub_type = DMUB_CMD__LSDMA_LINEAR_COPY;
 	wait_type                      = DM_DMUB_WAIT_TYPE_NO_WAIT;
 
-	lsdma_data->u.linear_copy_data.count   = count - 1; // LSDMA controller expects bytes to copy -1
-	lsdma_data->u.linear_copy_data.src_lo  = src_addr & 0xFFFFFFFF;
-	lsdma_data->u.linear_copy_data.src_hi  = (src_addr >> 32) & 0xFFFFFFFF;
-	lsdma_data->u.linear_copy_data.dst_lo  = dst_addr & 0xFFFFFFFF;
-	lsdma_data->u.linear_copy_data.dst_hi  = (dst_addr >> 32) & 0xFFFFFFFF;
+	lsdma_data->u.linear_copy_data.count   = copy_data.count;
+	lsdma_data->u.linear_copy_data.src_lo  = copy_data.src_lo;
+	lsdma_data->u.linear_copy_data.src_hi  = copy_data.src_hi;
+	lsdma_data->u.linear_copy_data.dst_lo  = copy_data.dst_lo;
+	lsdma_data->u.linear_copy_data.dst_hi  = copy_data.dst_hi;
+	lsdma_data->u.linear_copy_data.tmz     = copy_data.tmz;
+	lsdma_data->u.linear_copy_data.data_format = copy_data.data_format;
+	lsdma_data->u.linear_copy_data.num_type = copy_data.num_type;
+	lsdma_data->u.linear_copy_data.read_compress = copy_data.read_compress;
+	lsdma_data->u.linear_copy_data.write_compress = copy_data.write_compress;
+	lsdma_data->u.linear_copy_data.max_com = copy_data.max_com;
+	lsdma_data->u.linear_copy_data.max_uncom = copy_data.max_uncom;
+	lsdma_data->u.linear_copy_data.cache_policy_src = copy_data.cache_policy_src;
+	lsdma_data->u.linear_copy_data.cache_policy_dst = copy_data.cache_policy_dst;
 
 	result = dc_wake_and_execute_dmub_cmd(dc_ctx, &cmd, wait_type);
 
@@ -2203,6 +2200,12 @@ bool dmub_lsdma_send_linear_sub_window_copy_command(
 	lsdma_data->u.linear_sub_window_copy_data.rect_y           = copy_data.rect_y;
 	lsdma_data->u.linear_sub_window_copy_data.src_cache_policy = copy_data.src_cache_policy;
 	lsdma_data->u.linear_sub_window_copy_data.dst_cache_policy = copy_data.dst_cache_policy;
+	lsdma_data->u.linear_sub_window_copy_data.data_format      = copy_data.data_format;
+	lsdma_data->u.linear_sub_window_copy_data.num_type         = copy_data.num_type;
+	lsdma_data->u.linear_sub_window_copy_data.read_compress    = copy_data.read_compress;
+	lsdma_data->u.linear_sub_window_copy_data.write_compress   = copy_data.write_compress;
+	lsdma_data->u.linear_sub_window_copy_data.max_com          = copy_data.max_com;
+	lsdma_data->u.linear_sub_window_copy_data.max_uncom        = copy_data.max_uncom;
 
 	result = dc_wake_and_execute_dmub_cmd(dc_ctx, &cmd, wait_type);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
index ebcaf49e5961e..8bdaac0b0f98c 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
@@ -37,17 +37,8 @@ struct dc_crtc_timing;
 struct dc_state;
 struct dc_surface_update;
 
-struct dc_reg_helper_state {
-	bool gather_in_progress;
-	uint32_t same_addr_count;
-	bool should_burst_write;
-	union dmub_rb_cmd cmd_data;
-	unsigned int reg_seq_count;
-};
-
 struct dc_dmub_srv {
 	struct dmub_srv *dmub;
-	struct dc_reg_helper_state reg_helper_offload;
 
 	struct dc_context *ctx;
 	void *dm;
@@ -212,11 +203,31 @@ void dc_dmub_srv_fams2_passthrough_flip(
 		int surface_count);
 
 bool dmub_lsdma_init(struct dc_dmub_srv *dc_dmub_srv);
+
+struct lsdma_linear_copy_params {
+	uint32_t src_lo;
+	uint32_t src_hi;
+
+	uint32_t dst_lo;
+	uint32_t dst_hi;
+
+	uint32_t count            : 30;
+	uint32_t read_compress    : 2;
+
+	uint32_t tmz              : 4;
+	uint32_t cache_policy_src : 3;
+	uint32_t cache_policy_dst : 3;
+	uint32_t data_format      : 6;
+	uint32_t num_type         : 3;
+	uint32_t write_compress   : 2;
+	uint32_t max_com          : 2;
+	uint32_t max_uncom        : 1;
+	uint32_t reserved0        : 8;
+};
+
 bool dmub_lsdma_send_linear_copy_command(
 	struct dc_dmub_srv *dc_dmub_srv,
-	uint64_t src_addr,
-	uint64_t dst_addr,
-	uint32_t count);
+	struct lsdma_linear_copy_params copy_data);
 
 struct lsdma_linear_sub_window_copy_params {
 	uint32_t src_lo;
@@ -244,7 +255,13 @@ struct lsdma_linear_sub_window_copy_params {
 	uint32_t element_size     : 3;
 	uint32_t src_cache_policy : 3;
 	uint32_t dst_cache_policy : 3;
-	uint32_t padding          : 19;
+	uint32_t data_format      : 6;
+	uint32_t num_type         : 3;
+	uint32_t read_compress    : 2;
+	uint32_t write_compress   : 2;
+	uint32_t max_com          : 2;
+	uint32_t max_uncom        : 1;
+	uint32_t reserved0        : 3;
 };
 
 bool dmub_lsdma_send_linear_sub_window_copy_command(
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
index fbef0dc743ff2..d0ba9ad67a3e9 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
@@ -1146,6 +1146,7 @@ struct edp_psr_info {
 	union edp_psr_dpcd_caps psr_dpcd_caps;
 	uint8_t psr2_su_y_granularity_cap;
 	uint8_t force_psrsu_cap;
+	uint8_t psr_active_vtotal_control_cap;
 };
 
 struct replay_info {
diff --git a/drivers/gpu/drm/amd/display/dc/dc_helper.c b/drivers/gpu/drm/amd/display/dc/dc_helper.c
index 0e0165764a57d..cc7fea613d9e8 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_helper.c
@@ -39,53 +39,6 @@
 #define DC_LOGGER \
 	ctx->logger
 
-static inline void submit_dmub_read_modify_write(
-	struct dc_reg_helper_state *offload,
-	const struct dc_context *ctx)
-{
-	struct dmub_rb_cmd_read_modify_write *cmd_buf = &offload->cmd_data.read_modify_write;
-
-	offload->should_burst_write =
-			(offload->same_addr_count == (DMUB_READ_MODIFY_WRITE_SEQ__MAX - 1));
-	cmd_buf->header.payload_bytes =
-			sizeof(struct dmub_cmd_read_modify_write_sequence) * offload->reg_seq_count;
-
-	dc_wake_and_execute_dmub_cmd(ctx, &offload->cmd_data, DM_DMUB_WAIT_TYPE_NO_WAIT);
-
-	memset(cmd_buf, 0, sizeof(*cmd_buf));
-
-	offload->reg_seq_count = 0;
-	offload->same_addr_count = 0;
-}
-
-static inline void submit_dmub_burst_write(
-	struct dc_reg_helper_state *offload,
-	const struct dc_context *ctx)
-{
-	struct dmub_rb_cmd_burst_write *cmd_buf = &offload->cmd_data.burst_write;
-
-	cmd_buf->header.payload_bytes =
-			sizeof(uint32_t) * offload->reg_seq_count;
-
-	dc_wake_and_execute_dmub_cmd(ctx, &offload->cmd_data, DM_DMUB_WAIT_TYPE_NO_WAIT);
-
-	memset(cmd_buf, 0, sizeof(*cmd_buf));
-
-	offload->reg_seq_count = 0;
-}
-
-static inline void submit_dmub_reg_wait(
-		struct dc_reg_helper_state *offload,
-		const struct dc_context *ctx)
-{
-	struct dmub_rb_cmd_reg_wait *cmd_buf = &offload->cmd_data.reg_wait;
-
-	dc_wake_and_execute_dmub_cmd(ctx, &offload->cmd_data, DM_DMUB_WAIT_TYPE_NO_WAIT);
-
-	memset(cmd_buf, 0, sizeof(*cmd_buf));
-	offload->reg_seq_count = 0;
-}
-
 struct dc_reg_value_masks {
 	uint32_t value;
 	uint32_t mask;
@@ -127,98 +80,6 @@ static void set_reg_field_values(struct dc_reg_value_masks *field_value_mask,
 	}
 }
 
-static void dmub_flush_buffer_execute(
-		struct dc_reg_helper_state *offload,
-		const struct dc_context *ctx)
-{
-	submit_dmub_read_modify_write(offload, ctx);
-}
-
-static void dmub_flush_burst_write_buffer_execute(
-		struct dc_reg_helper_state *offload,
-		const struct dc_context *ctx)
-{
-	submit_dmub_burst_write(offload, ctx);
-}
-
-static bool dmub_reg_value_burst_set_pack(const struct dc_context *ctx, uint32_t addr,
-		uint32_t reg_val)
-{
-	struct dc_reg_helper_state *offload = &ctx->dmub_srv->reg_helper_offload;
-	struct dmub_rb_cmd_burst_write *cmd_buf = &offload->cmd_data.burst_write;
-
-	/* flush command if buffer is full */
-	if (offload->reg_seq_count == DMUB_BURST_WRITE_VALUES__MAX)
-		dmub_flush_burst_write_buffer_execute(offload, ctx);
-
-	if (offload->cmd_data.cmd_common.header.type == DMUB_CMD__REG_SEQ_BURST_WRITE &&
-			addr != cmd_buf->addr) {
-		dmub_flush_burst_write_buffer_execute(offload, ctx);
-		return false;
-	}
-
-	cmd_buf->header.type = DMUB_CMD__REG_SEQ_BURST_WRITE;
-	cmd_buf->header.sub_type = 0;
-	cmd_buf->addr = addr;
-	cmd_buf->write_values[offload->reg_seq_count] = reg_val;
-	offload->reg_seq_count++;
-
-	return true;
-}
-
-static uint32_t dmub_reg_value_pack(const struct dc_context *ctx, uint32_t addr,
-		struct dc_reg_value_masks *field_value_mask)
-{
-	struct dc_reg_helper_state *offload = &ctx->dmub_srv->reg_helper_offload;
-	struct dmub_rb_cmd_read_modify_write *cmd_buf = &offload->cmd_data.read_modify_write;
-	struct dmub_cmd_read_modify_write_sequence *seq;
-
-	/* flush command if buffer is full */
-	if (offload->cmd_data.cmd_common.header.type != DMUB_CMD__REG_SEQ_BURST_WRITE &&
-			offload->reg_seq_count == DMUB_READ_MODIFY_WRITE_SEQ__MAX)
-		dmub_flush_buffer_execute(offload, ctx);
-
-	if (offload->should_burst_write) {
-		if (dmub_reg_value_burst_set_pack(ctx, addr, field_value_mask->value))
-			return field_value_mask->value;
-		else
-			offload->should_burst_write = false;
-	}
-
-	/* pack commands */
-	cmd_buf->header.type = DMUB_CMD__REG_SEQ_READ_MODIFY_WRITE;
-	cmd_buf->header.sub_type = 0;
-	seq = &cmd_buf->seq[offload->reg_seq_count];
-
-	if (offload->reg_seq_count) {
-		if (cmd_buf->seq[offload->reg_seq_count - 1].addr == addr)
-			offload->same_addr_count++;
-		else
-			offload->same_addr_count = 0;
-	}
-
-	seq->addr = addr;
-	seq->modify_mask = field_value_mask->mask;
-	seq->modify_value = field_value_mask->value;
-	offload->reg_seq_count++;
-
-	return field_value_mask->value;
-}
-
-static void dmub_reg_wait_done_pack(const struct dc_context *ctx, uint32_t addr,
-		uint32_t mask, uint32_t shift, uint32_t condition_value, uint32_t time_out_us)
-{
-	struct dc_reg_helper_state *offload = &ctx->dmub_srv->reg_helper_offload;
-	struct dmub_rb_cmd_reg_wait *cmd_buf = &offload->cmd_data.reg_wait;
-
-	cmd_buf->header.type = DMUB_CMD__REG_REG_WAIT;
-	cmd_buf->header.sub_type = 0;
-	cmd_buf->reg_wait.addr = addr;
-	cmd_buf->reg_wait.condition_field_value = mask & (condition_value << shift);
-	cmd_buf->reg_wait.mask = mask;
-	cmd_buf->reg_wait.time_out_us = time_out_us;
-}
-
 uint32_t generic_reg_update_ex(const struct dc_context *ctx,
 		uint32_t addr, int n,
 		uint8_t shift1, uint32_t mask1, uint32_t field_value1,
@@ -235,11 +96,6 @@ uint32_t generic_reg_update_ex(const struct dc_context *ctx,
 
 	va_end(ap);
 
-	if (ctx->dmub_srv &&
-	    ctx->dmub_srv->reg_helper_offload.gather_in_progress)
-		return dmub_reg_value_pack(ctx, addr, &field_value_mask);
-		/* todo: return void so we can decouple code running in driver from register states */
-
 	/* mmio write directly */
 	reg_val = dm_read_reg(ctx, addr);
 	reg_val = (reg_val & ~field_value_mask.mask) | field_value_mask.value;
@@ -265,12 +121,6 @@ uint32_t generic_reg_set_ex(const struct dc_context *ctx,
 	/* mmio write directly */
 	reg_val = (reg_val & ~field_value_mask.mask) | field_value_mask.value;
 
-	if (ctx->dmub_srv &&
-	    ctx->dmub_srv->reg_helper_offload.gather_in_progress) {
-		return dmub_reg_value_burst_set_pack(ctx, addr, reg_val);
-		/* todo: return void so we can decouple code running in driver from register states */
-	}
-
 	dm_write_reg(ctx, addr, reg_val);
 	return reg_val;
 }
@@ -434,13 +284,6 @@ void generic_reg_wait(const struct dc_context *ctx,
 	uint32_t reg_val;
 	unsigned int i;
 
-	if (ctx->dmub_srv &&
-	    ctx->dmub_srv->reg_helper_offload.gather_in_progress) {
-		dmub_reg_wait_done_pack(ctx, addr, mask, shift, condition_value,
-				delay_between_poll_us * time_out_num_tries);
-		return;
-	}
-
 	/*
 	 * Something is terribly wrong if time out is > 3000ms.
 	 * 3000ms is the maximum time needed for SMU to pass values back.
@@ -491,12 +334,6 @@ uint32_t generic_read_indirect_reg(const struct dc_context *ctx,
 {
 	uint32_t value = 0;
 
-	// when reg read, there should not be any offload.
-	if (ctx->dmub_srv &&
-	    ctx->dmub_srv->reg_helper_offload.gather_in_progress) {
-		ASSERT(false);
-	}
-
 	dm_write_reg(ctx, addr_index, index);
 	value = dm_read_reg(ctx, addr_data);
 
@@ -624,69 +461,6 @@ uint32_t generic_indirect_reg_get_sync(const struct dc_context *ctx,
 	return value;
 }
 
-void reg_sequence_start_gather(const struct dc_context *ctx)
-{
-	/* if reg sequence is supported and enabled, set flag to
-	 * indicate we want to have REG_SET, REG_UPDATE macro build
-	 * reg sequence command buffer rather than MMIO directly.
-	 */
-
-	if (ctx->dmub_srv && ctx->dc->debug.dmub_offload_enabled) {
-		struct dc_reg_helper_state *offload =
-			&ctx->dmub_srv->reg_helper_offload;
-
-		/* caller sequence mismatch.  need to debug caller.  offload will not work!!! */
-		ASSERT(!offload->gather_in_progress);
-
-		offload->gather_in_progress = true;
-	}
-}
-
-void reg_sequence_start_execute(const struct dc_context *ctx)
-{
-	struct dc_reg_helper_state *offload;
-
-	if (!ctx->dmub_srv)
-		return;
-
-	offload = &ctx->dmub_srv->reg_helper_offload;
-
-	if (offload && offload->gather_in_progress) {
-		offload->gather_in_progress = false;
-		offload->should_burst_write = false;
-		switch (offload->cmd_data.cmd_common.header.type) {
-		case DMUB_CMD__REG_SEQ_READ_MODIFY_WRITE:
-			submit_dmub_read_modify_write(offload, ctx);
-			break;
-		case DMUB_CMD__REG_REG_WAIT:
-			submit_dmub_reg_wait(offload, ctx);
-			break;
-		case DMUB_CMD__REG_SEQ_BURST_WRITE:
-			submit_dmub_burst_write(offload, ctx);
-			break;
-		default:
-			return;
-		}
-	}
-}
-
-void reg_sequence_wait_done(const struct dc_context *ctx)
-{
-	/* callback to DM to poll for last submission done*/
-	struct dc_reg_helper_state *offload;
-
-	if (!ctx->dmub_srv)
-		return;
-
-	offload = &ctx->dmub_srv->reg_helper_offload;
-
-	if (offload &&
-	    ctx->dc->debug.dmub_offload_enabled &&
-	    !ctx->dc->debug.dmcub_emulation) {
-		dc_dmub_srv_wait_for_idle(ctx->dmub_srv, DM_DMUB_WAIT_TYPE_WAIT, NULL);
-	}
-}
-
 char *dce_version_to_string(const int version)
 {
 	switch (version) {
diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h
index 4154cd059562b..8b164edc9c51f 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_stream.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h
@@ -128,6 +128,35 @@ union stream_update_flags {
 	uint32_t raw;
 };
 
+static inline void stream_update_flags_clear(union stream_update_flags *flags)
+{
+	flags->raw = 0;
+}
+
+static inline void stream_update_flags_set_full(union stream_update_flags *flags)
+{
+	stream_update_flags_clear(flags);
+	flags->bits.scaling = 1;
+	flags->bits.out_tf = 1;
+	flags->bits.out_csc = 1;
+	flags->bits.abm_level = 1;
+	flags->bits.dpms_off = 1;
+	flags->bits.gamut_remap = 1;
+	flags->bits.wb_update = 1;
+	flags->bits.dsc_changed = 1;
+	flags->bits.mst_bw = 1;
+	flags->bits.crtc_timing_adjust = 1;
+	flags->bits.fams_changed = 1;
+	flags->bits.scaler_sharpener = 1;
+	flags->bits.sharpening_required = 1;
+	flags->bits.cursor_attr = 1;
+	flags->bits.cursor_pos = 1;
+	flags->bits.periodic_interrupt = 1;
+	flags->bits.info_frame = 1;
+	flags->bits.dmdata = 1;
+	flags->bits.dither = 1;
+}
+
 struct test_pattern {
 	enum dp_test_pattern type;
 	enum dp_test_pattern_color_space color_space;
diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h
index 4ed1efa172709..5ba7a2fffcf0b 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_types.h
@@ -183,6 +183,7 @@ struct dc_panel_patch {
 	unsigned int force_frl;
 	unsigned int vsdb_rcc_wa;
 	unsigned int delay_hdmi_link_training;
+	unsigned int skip_frl_pre_training;
 	unsigned int skip_avmute;
 	unsigned int skip_audio_sab_check;
 	unsigned int mst_start_top_delay;
@@ -1346,6 +1347,14 @@ struct dc_panel_config {
 	struct ilr {
 		bool optimize_edp_link_rate; /* eDP ILR */
 	} ilr;
+	/* CACP*/
+	struct cacp {
+		unsigned int cacp_supported;
+		unsigned int cacp_control_mode;
+		unsigned int strscl_valid;
+		unsigned int strscl_sdr[4];
+		unsigned int strscl_hdr[4];
+	} cacp;
 	/* Adaptive VariBright*/
 	struct adaptive_vb {
 		bool disable_adaptive_vb;
@@ -1397,6 +1406,39 @@ enum dc_hpd_enable_select {
 	HPD_EN_FOR_SECONDARY_EDP_ONLY,
 };
 
+enum dc_cm_lut_swizzle {
+	CM_LUT_3D_SWIZZLE_LINEAR_RGB,
+	CM_LUT_3D_SWIZZLE_LINEAR_BGR,
+	CM_LUT_1D_PACKED_LINEAR
+};
+
+enum dc_cm_lut_pixel_format {
+	CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB,
+#if defined(CONFIG_DRM_AMD_DC_DCN4_2)
+	CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12MSB,
+#endif
+	CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB,
+#if defined(CONFIG_DRM_AMD_DC_DCN4_2)
+	CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12LSB,
+#endif
+	CM_LUT_PIXEL_FORMAT_RGBA16161616_FLOAT_FP1_5_10,
+#if defined(CONFIG_DRM_AMD_DC_DCN4_2)
+	CM_LUT_PIXEL_FORMAT_BGRA16161616_FLOAT_FP1_5_10
+#endif
+};
+
+enum dc_cm_lut_size {
+	CM_LUT_SIZE_NONE,
+	CM_LUT_SIZE_999,
+	CM_LUT_SIZE_171717,
+#if defined(CONFIG_DRM_AMD_DC_DCN4_2)
+	CM_LUT_SIZE_333333,
+	CM_LUT_SIZE_454545,
+	CM_LUT_SIZE_656565,
+#endif
+};
+
+#ifndef TRIM_CM2
 enum dc_cm2_shaper_3dlut_setting {
 	DC_CM2_SHAPER_3DLUT_SETTING_BYPASS_ALL,
 	DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER,
@@ -1421,6 +1463,16 @@ enum dc_cm2_gpu_mem_format {
 	DC_CM2_GPU_MEM_FORMAT_16161616_FLOAT_FP1_5_10
 };
 
+enum dc_cm2_gpu_mem_size {
+	DC_CM2_GPU_MEM_SIZE_171717,
+	DC_CM2_GPU_MEM_SIZE_333333,
+	DC_CM2_GPU_MEM_SIZE_454545,
+	DC_CM2_GPU_MEM_SIZE_656565,
+	DC_CM2_GPU_MEM_SIZE_TRANSFORMED,
+};
+#endif /* TRIM_CM2 */
+
+#ifndef TRIM_CM2
 struct dc_cm2_gpu_mem_format_parameters {
 	enum dc_cm2_gpu_mem_format format;
 	union {
@@ -1432,14 +1484,6 @@ struct dc_cm2_gpu_mem_format_parameters {
 	};
 };
 
-enum dc_cm2_gpu_mem_size {
-	DC_CM2_GPU_MEM_SIZE_171717,
-	DC_CM2_GPU_MEM_SIZE_333333,
-	DC_CM2_GPU_MEM_SIZE_454545,
-	DC_CM2_GPU_MEM_SIZE_656565,
-	DC_CM2_GPU_MEM_SIZE_TRANSFORMED,
-};
-
 struct dc_cm2_gpu_mem_parameters {
 	struct dc_plane_address addr;
 	enum dc_cm2_gpu_mem_layout layout;
@@ -1448,17 +1492,16 @@ struct dc_cm2_gpu_mem_parameters {
 	enum dc_cm2_gpu_mem_size  size;
 	uint16_t bit_depth;
 };
+#endif /* TRIM_CM2 */
 
+#ifndef TRIM_CM2
 enum dc_cm2_transfer_func_source {
 	DC_CM2_TRANSFER_FUNC_SOURCE_SYSMEM,
 	DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM
 };
+#endif /* TRIM_CM2 */
 
-struct dc_cm2_component_settings {
-	enum dc_cm2_shaper_3dlut_setting shaper_3dlut_setting;
-	bool lut1d_enable;
-};
-
+#ifndef TRIM_CM2
 /*
  * All pointers in this struct must remain valid for as long as the 3DLUTs are used
  */
@@ -1478,11 +1521,7 @@ struct dc_cm2_func_luts {
 	} lut3d_data;
 	const struct dc_transfer_func *lut1d_func;
 };
-
-struct dc_cm2_parameters {
-	struct dc_cm2_component_settings component_settings;
-	struct dc_cm2_func_luts cm2_luts;
-};
+#endif /* TRIM_CM2 */
 
 enum mall_stream_type {
 	SUBVP_NONE, // subvp not in use
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c
index 483cd9ab7eb76..71903a6c63a0d 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c
@@ -572,7 +572,7 @@ static void dccg35_set_hdmistreamclk_src_new(
 	case 0:
 		REG_UPDATE_2(HDMISTREAMCLK_CNTL, HDMISTREAMCLK0_EN,
 					 (src == HDMI_STREAM_REFCLK) ? 0 : 1,
-					 DPSTREAMCLK0_SRC_SEL,
+					 HDMISTREAMCLK0_SRC_SEL,
 					 (src == HDMI_STREAM_REFCLK) ? 0 : src);
 		break;
 	default:
@@ -1654,7 +1654,7 @@ void dccg35_set_dpstreamclk_root_clock_gating(struct dccg *dccg, int dp_hpo_inst
 
 
 
-static void dccg35_set_hdmistreamclk(
+void dccg35_set_hdmistreamclk(
 		struct dccg *dccg,
 		enum streamclk_source src,
 		uint32_t otg_inst)
@@ -1680,8 +1680,11 @@ void dccg35_set_hdmistreamclk_root_clock_gating(struct dccg *dccg, bool enable)
 {
 	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
 
-	if (dccg->ctx->dc->debug.root_clock_optimization.bits.hdmistream)
-		REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, HDMISTREAMCLK0_ROOT_GATE_DISABLE, enable ? 1 : 0);
+	if (!dccg->ctx->dc->debug.root_clock_optimization.bits.hdmistream && !enable) {
+		DC_LOG_DEBUG("%s: HDMISTREAMCLK0_ROOT_GATE DISABLE = 0 bypassed", __func__);
+		return;
+	}
+	REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, HDMISTREAMCLK0_ROOT_GATE_DISABLE, enable ? 1 : 0);
 
 	DC_LOG_DEBUG("%s: HDMISTREAMCLK0_ROOT_GATE_DISABLE = %d\n", __func__, enable ? 1 : 0);
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.h b/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.h
index 554700287c1ab..58f4c1e998b3c 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.h
@@ -269,5 +269,8 @@ void dccg35_disable_dscclk(struct dccg *dccg, int inst);
 
 void dccg35_enable_symclk_se(struct dccg *dccg, uint32_t stream_enc_inst, uint32_t link_enc_inst);
 void dccg35_disable_symclk_se(struct dccg *dccg, uint32_t stream_enc_inst, uint32_t link_enc_inst);
-
+void dccg35_set_hdmistreamclk(
+		struct dccg *dccg,
+		enum streamclk_source src,
+		uint32_t otg_inst);
 #endif //__DCN35_DCCG_H__
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c
index e57242f8bc120..3b03b152da22b 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c
@@ -77,13 +77,26 @@ void dccg42_otg_drop_pixel(struct dccg *dccg,
 	}
 }
 
-void dccg42_enable_global_fgcg(struct dccg *dccg, bool value)
+void dccg42_enable_global_fgcg(struct dccg *dccg, bool enable)
 {
 	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
 
-	if (dccg->ctx->dc->debug.disable_clock_gate)
-		value = false;
-	REG_UPDATE(DCCG_GLOBAL_FGCG_REP_CNTL, DCCG_GLOBAL_FGCG_REP_DIS, !value);
+	/* Temporary workaround for IOMMU mismatch issue.
+	 * Fine grain control via bit2 of debug flag.
+	 */
+	if (dccg->ctx->dc->debug.disable_clock_gate || (dccg->ctx->dc->debug.iommu_mismatch_temp_wka & 0x4))
+		enable = false;
+
+	REG_UPDATE(DCCG_GLOBAL_FGCG_REP_CNTL, DCCG_GLOBAL_FGCG_REP_DIS, !enable);
+}
+
+bool dccg42_get_global_fgcg_status(struct dccg *dccg)
+{
+	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+	uint32_t disabled = 0;
+
+	REG_GET(DCCG_GLOBAL_FGCG_REP_CNTL, DCCG_GLOBAL_FGCG_REP_DIS, &disabled);
+	return disabled & 0x1;
 }
 
 void dccg42_set_physymclk(
@@ -265,47 +278,42 @@ void dccg42_trigger_dio_fifo_resync(struct dccg *dccg)
 
 static void dccg42_init(struct dccg *dccg)
 {
-	int otg_inst;
-	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+	unsigned int i;
+	struct resource_pool *res_pool = dccg->ctx->dc->res_pool;
 
 	/* Set HPO stream encoder to use refclk to avoid case where PHY is
 	 * disabled and SYMCLK32 for HPO SE is sourced from PHYD32CLK which
 	 * will cause DCN to hang.
 	 */
-	for (otg_inst = 0; otg_inst < 4; otg_inst++)
-		dccg35_disable_symclk32_se(dccg, otg_inst);
+	for (i = 0; i < res_pool->hpo_dp_stream_enc_count; i++)
+		dccg35_disable_symclk32_se(dccg, i);
 
 	if (dccg->ctx->dc->debug.root_clock_optimization.bits.symclk32_le) {
-		dccg401_disable_symclk32_le(dccg, 0);
-		dccg401_disable_symclk32_le(dccg, 1);
-		dccg401_disable_symclk32_le(dccg, 2);
-		dccg401_disable_symclk32_le(dccg, 3);
+		for (i = 0; i < res_pool->hpo_dp_link_enc_count; i++)
+			dccg401_disable_symclk32_le(dccg, i);
 	}
 
 	if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpstream) {
-		dccg401_disable_dpstreamclk(dccg, 0);
-		dccg401_disable_dpstreamclk(dccg, 1);
-		dccg401_disable_dpstreamclk(dccg, 2);
-		dccg401_disable_dpstreamclk(dccg, 3);
-	}
-	if (!dccg->ctx->dc->debug.root_clock_optimization.bits.physymclk) {
-		REG_UPDATE_5(DCCG_GATE_DISABLE_CNTL2,
-			PHYASYMCLK_ROOT_GATE_DISABLE, 1,
-			PHYBSYMCLK_ROOT_GATE_DISABLE, 1,
-			PHYCSYMCLK_ROOT_GATE_DISABLE, 1,
-			PHYDSYMCLK_ROOT_GATE_DISABLE, 1,
-			PHYESYMCLK_ROOT_GATE_DISABLE, 1);
+		for (i = 0; i < res_pool->hpo_dp_stream_enc_count; i++)
+			dccg401_disable_dpstreamclk(dccg, i);
 	}
+
 	dccg42_disable_hdmistreamclk(dccg);
 	if (dccg->ctx->dc->debug.root_clock_optimization.bits.hdmichar)
 		dccg42_disable_hdmicharclk(dccg, 0);
+
+	if (dccg->ctx->dc->debug.root_clock_optimization.bits.dpp) {
+		for (i = 0; i < res_pool->pipe_count; i++) {
+			dccg35_dpp_root_clock_control(dccg, i, true);
+		}
+	}
 }
 
 
 static const struct dccg_funcs dccg42_funcs = {
 	.enable_hdmicharclk = dccg401_enable_hdmicharclk,
 	.disable_hdmicharclk = dccg42_disable_hdmicharclk,
-	.set_hdmistreamclk = dccg401_set_hdmistreamclk,
+	.set_hdmistreamclk = dccg35_set_hdmistreamclk,
 	.set_hdmistreamclk_root_clock_gating = dccg35_set_hdmistreamclk_root_clock_gating,
 	.update_dpp_dto = dccg35_update_dpp_dto,
 	.dpp_root_clock_control = dccg35_dpp_root_clock_control,
@@ -340,7 +348,8 @@ static const struct dccg_funcs dccg42_funcs = {
 	.dccg_root_gate_disable_control = dccg35_root_gate_disable_control,
 	.dccg_read_reg_state = dccg31_read_reg_state,
 	.dccg_enable_global_fgcg = dccg42_enable_global_fgcg,
-	.allow_clock_gating = dccg2_allow_clock_gating
+	.allow_clock_gating = dccg2_allow_clock_gating,
+	.dccg_get_global_fgcg_status = dccg42_get_global_fgcg_status,
 };
 
 struct dccg *dccg42_create(
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h
index a2b17ed11bdb6..ebd3cec1a9779 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h
@@ -247,6 +247,7 @@ void dccg42_otg_add_pixel(struct dccg *dccg,
 void dccg42_otg_drop_pixel(struct dccg *dccg,
 		uint32_t otg_inst);
 void dccg42_enable_global_fgcg(struct dccg *dccg, bool value);
+bool dccg42_get_global_fgcg_status(struct dccg *dccg);
 
 void dccg42_set_physymclk(
 		struct dccg *dccg,
diff --git a/drivers/gpu/drm/amd/display/dc/dce/Makefile b/drivers/gpu/drm/amd/display/dc/dce/Makefile
index 986e0e7abbc2f..72bf70d89b74a 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dce/Makefile
@@ -23,14 +23,14 @@
 # Makefile for common 'dce' logic
 # HW object file under this folder follow similar pattern for HW programming
 #   - register offset and/or shift + mask stored in the dec_hw struct
-#   - register programming through common macros that look up register 
+#   - register programming through common macros that look up register
 #     offset/shift/mask stored in dce_hw struct
 
 DCE = dce_audio.o dce_stream_encoder.o dce_link_encoder.o \
 dce_mem_input.o dce_clock_source.o dce_scl_filters.o dce_transform.o \
 dce_opp.o dce_dmcu.o dce_abm.o dce_ipp.o dce_aux.o \
 dce_i2c.o dce_i2c_hw.o dce_i2c_sw.o dmub_psr.o dmub_abm.o dmub_abm_lcd.o dce_panel_cntl.o \
-dmub_hw_lock_mgr.o dmub_outbox.o dmub_replay.o
+dmub_hw_lock_mgr.o dmub_outbox.o dmub_replay.o dmub_abm_cacp.o
 
 AMD_DAL_DCE = $(addprefix $(AMDDALPATH)/dc/dce/,$(DCE))
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
index 72ad3ee3d6a57..6cb5e8152cf10 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
@@ -26,6 +26,7 @@
 #include "dm_services.h"
 #include "core_types.h"
 #include "dce_aux.h"
+#include "dce/dce_11_0_d.h"
 #include "dce/dce_11_0_sh_mask.h"
 #include "dm_event_log.h"
 #include "dm_helpers.h"
@@ -529,7 +530,7 @@ static uint32_t dce_aux_configure_timeout(struct ddc_service *ddc,
 	uint32_t prev_timeout_val = 0;
 	struct ddc *ddc_pin = ddc->ddc_pin;
 
-	if (ddc->ctx->dc->config.dp_connector_no_native_i2c && ddc->link->no_ddc_pin)
+	if (ddc->link->force_to_use_aux)
 		return dce_aux_configure_timeout_without_ddc_pin(ddc, timeout_in_us);
 
 	struct dce_aux *aux_engine = ddc->ctx->dc->res_pool->engines[ddc_pin->pin_data->en];
@@ -652,7 +653,7 @@ int dce_aux_transfer_raw(struct ddc_service *ddc,
 		struct aux_payload *payload,
 		enum aux_return_code_type *operation_result)
 {
-	if (ddc->ctx->dc->config.dp_connector_no_native_i2c && ddc->link->no_ddc_pin) {
+	if (ddc->link->force_to_use_aux) {
 		/* Check whether aux to be processed via dmub or dcn directly */
 		if (ddc->ctx->dc->debug.enable_dmub_aux_for_legacy_ddc) {
 			return dce_aux_transfer_dmub_raw(ddc, payload, operation_result);
@@ -795,7 +796,7 @@ int dce_aux_transfer_dmub_raw(struct ddc_service *ddc,
 		release_engine(aux_engine);
 	}
 
-	if (ddc->ctx->dc->config.dp_connector_no_native_i2c && ddc->link->no_ddc_pin) {
+	if (ddc->link->force_to_use_aux) {
 		struct dce_aux *aux_engine = ddc->ctx->dc->res_pool->engines[ddc->link->aux_hw_inst];
 
 		if (!acquire_aux_engine_without_ddc_pin(aux_engine, ddc_pin)) {
@@ -893,7 +894,7 @@ bool dce_aux_transfer_with_retries(struct ddc_service *ddc,
 		aux110 = FROM_AUX_ENGINE(aux_engine);
 	}
 
-	if (ddc->ctx->dc->config.dp_connector_no_native_i2c && ddc->link->no_ddc_pin) {
+	if (ddc->link->force_to_use_aux) {
 		aux_engine = ddc->ctx->dc->res_pool->engines[ddc->link->aux_hw_inst];
 		aux110 = FROM_AUX_ENGINE(aux_engine);
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
index 7c293917e6fd6..ecb8493ec5233 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
@@ -1229,9 +1229,9 @@ static bool get_dp_dto_frequency_100hz(
 			 */
 			modulo_hz = REG_READ(MODULO[inst]);
 			if (modulo_hz) {
-				temp = div_u64((uint64_t)clock_hz * dp_dto_ref_khz * 10, modulo_hz);
-				ASSERT(temp / 100 <= 0xFFFFFFFFUL);
-				*pixel_clk_100hz = (unsigned int)(temp / 100);
+				temp = clock_hz * dp_dto_ref_khz * 10;
+				ASSERT(temp <= UINT_MAX * modulo_hz * 100ULL);
+				*pixel_clk_100hz = div_u64(temp, modulo_hz * 100);
 			} else
 				*pixel_clk_100hz = 0;
 		} else {
@@ -1285,13 +1285,12 @@ static bool dcn401_get_dp_dto_frequency_100hz(const struct clock_source *clock_s
 		 *     - target pix_clk_hz = (DPDTO INTEGER * DPDTO MODULO + DPDTO PHASE)
 		 */
 		temp = (unsigned long long)dp_dto_integer * modulo_hz + phase_hz;
-
-		if (temp / 100 > 0xFFFFFFFFUL) {
+		if (temp > (UINT_MAX * 100ULL)) {
 			/* pixel rate 100hz should never be this high, if it is, throw an assert and return 0  */
 			BREAK_TO_DEBUGGER();
 			*pixel_clk_100hz = 0;
 		} else {
-			*pixel_clk_100hz = (unsigned int)(temp / 100);
+			*pixel_clk_100hz = div_u64(temp, 100);
 		}
 
 		return true;
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c
index 93550c5e4d02b..2df6e82b794fa 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c
@@ -25,6 +25,7 @@
 
 #include "dmub_abm.h"
 #include "dmub_abm_lcd.h"
+#include "dmub_abm_cacp.h"
 #include "dc.h"
 #include "core_types.h"
 #include "dmub_cmd.h"
@@ -36,6 +37,7 @@
 
 #define ABM_FEATURE_NO_SUPPORT	0
 #define ABM_LCD_SUPPORT			1
+#define ABM_CACP_SUPPORT		2
 
 static unsigned int abm_feature_support(struct abm *abm, unsigned int panel_inst)
 {
@@ -52,15 +54,48 @@ static unsigned int abm_feature_support(struct abm *abm, unsigned int panel_inst
 	}
 
 	if (i < edp_num) {
-		ret = ABM_LCD_SUPPORT;
+		if (edp_links[panel_inst]->panel_config.cacp.cacp_supported)
+			ret = ABM_CACP_SUPPORT;
+		else if ((edp_links[panel_inst]->panel_type == PANEL_TYPE_LCD) ||
+				(edp_links[panel_inst]->panel_type == PANEL_TYPE_MINILED))
+			ret = ABM_LCD_SUPPORT;
 	}
 
 	return ret;
 }
 
+static enum dc_panel_type abm_get_paneltype(struct abm *abm, unsigned int panel_inst)
+{
+	struct dc_context *dc = abm->ctx;
+	struct dc_link *edp_links[MAX_NUM_EDP];
+	unsigned int i, edp_num;
+	enum dc_panel_type ret = PANEL_TYPE_NONE;
+
+	dc_get_edp_links(dc->dc, edp_links, &edp_num);
+
+	for (i = 0; i < edp_num; i++) {
+		if (edp_links[i]->link_status.link_active
+			&& panel_inst == i)
+			break;
+	}
+
+	if (i < edp_num)
+		ret = edp_links[panel_inst]->panel_type;
+
+	return ret;
+}
+
 static void dmub_abm_init_ex(struct abm *abm, uint32_t backlight, uint32_t user_level)
 {
+	unsigned int i = 0;
+	uint8_t panel_mask = 0;
+
 	dmub_abm_init(abm, backlight, user_level);
+	for (i = 0; i < MAX_NUM_EDP; i++)
+		panel_mask |= (0x01 << i);
+
+	if (panel_mask)
+		dmub_cacp_enable_fractional_pwm(abm, panel_mask);
 }
 
 static unsigned int dmub_abm_get_current_backlight_ex(struct abm *abm)
@@ -82,17 +117,23 @@ static bool dmub_abm_set_level_ex(struct abm *abm, uint32_t level)
 	bool ret = false;
 	unsigned int feature_support, i;
 	uint8_t panel_mask0 = 0;
+	uint8_t panel_mask1 = 0;
 
 	for (i = 0; i < MAX_NUM_EDP; i++) {
 		feature_support = abm_feature_support(abm, i);
 
 		if (feature_support == ABM_LCD_SUPPORT)
 			panel_mask0 |= (0x01 << i);
+		else if (feature_support == ABM_CACP_SUPPORT)
+			panel_mask1 |= (0x01 << i);
 	}
 
 	if (panel_mask0)
 		ret = dmub_abm_set_level(abm, level, panel_mask0);
 
+	if (panel_mask1)
+		ret = dmub_cacp_set_level(abm, level, panel_mask1);
+
 	return ret;
 }
 
@@ -107,6 +148,8 @@ static bool dmub_abm_init_config_ex(struct abm *abm,
 
 	if (feature_support == ABM_LCD_SUPPORT)
 		dmub_abm_init_config(abm, src, bytes, inst);
+	else if (feature_support == ABM_CACP_SUPPORT)
+		dmub_cacp_init(abm, src, bytes, inst);
 
 	return true;
 }
@@ -120,6 +163,8 @@ static bool dmub_abm_set_pause_ex(struct abm *abm, bool pause, unsigned int pane
 
 	if (feature_support == ABM_LCD_SUPPORT)
 		ret = dmub_abm_set_pause(abm, pause, panel_inst, stream_inst);
+	else if (feature_support == ABM_CACP_SUPPORT)
+		ret = dmub_cacp_set_pause(abm, pause, panel_inst, stream_inst);
 
 	return ret;
 }
@@ -163,6 +208,25 @@ static bool dmub_abm_set_pipe_ex(struct abm *abm,
 
 	if (feature_support == ABM_LCD_SUPPORT)
 		ret = dmub_abm_set_pipe(abm, otg_inst, option, panel_inst, pwrseq_inst);
+	else if (feature_support == ABM_CACP_SUPPORT)
+		ret = dmub_cacp_set_pipe(abm, otg_inst, option, panel_inst, pwrseq_inst);
+
+	return ret;
+}
+
+static bool dmub_abm_set_event_ex(struct abm *abm, unsigned int full_screen, unsigned int trans_info,
+		unsigned int hdr_mode, unsigned int scaling_enable, unsigned int scaling_strength_map,
+		unsigned int panel_inst)
+{
+	bool ret = false;
+	unsigned int feature_support;
+
+	feature_support = abm_feature_support(abm, panel_inst);
+
+	if (feature_support == ABM_LCD_SUPPORT)
+		ret = dmub_abm_set_event(abm, scaling_enable, scaling_strength_map, panel_inst);
+	else if (feature_support == ABM_CACP_SUPPORT)
+		ret = dmub_cacp_set_event(abm, full_screen, trans_info, hdr_mode, scaling_enable, panel_inst);
 
 	return ret;
 }
@@ -176,11 +240,17 @@ static bool dmub_abm_set_backlight_level_pwm_ex(struct abm *abm,
 	(void)controller_id;
 	bool ret = false;
 	unsigned int feature_support;
+	enum dc_panel_type panel_type = PANEL_TYPE_NONE;
 
 	feature_support = abm_feature_support(abm, panel_inst);
 
 	if (feature_support == ABM_LCD_SUPPORT)
 		ret = dmub_abm_set_backlight_level(abm, backlight_pwm_u16_16, frame_ramp, panel_inst);
+	else if (feature_support == ABM_CACP_SUPPORT) {
+		panel_type = abm_get_paneltype(abm, panel_inst);
+		if (panel_type == PANEL_TYPE_MINILED)
+			ret = dmub_cacp_set_backlight_level(abm, backlight_pwm_u16_16, frame_ramp, panel_inst);
+	}
 
 	return ret;
 }
@@ -194,6 +264,7 @@ static const struct abm_funcs abm_funcs = {
 	.set_abm_pause = dmub_abm_set_pause_ex,
 	.save_restore = dmub_abm_save_restore_ex,
 	.set_pipe_ex = dmub_abm_set_pipe_ex,
+	.set_abm_event = dmub_abm_set_event_ex,
 	.set_backlight_level_pwm = dmub_abm_set_backlight_level_pwm_ex,
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_cacp.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_cacp.c
new file mode 100644
index 0000000000000..9f58fa2e25fd6
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_cacp.c
@@ -0,0 +1,213 @@
+/* Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. */
+
+#include "dmub_abm.h"
+#include "dmub_abm_cacp.h"
+#include "dce_abm.h"
+#include "dc.h"
+#include "dc_dmub_srv.h"
+#include "dmub/dmub_srv.h"
+#include "core_types.h"
+
+#define CACP_LEVEL_NUM 4
+
+void dmub_cacp_init(struct abm *abm, const char *src, unsigned int bytes, unsigned int panel_inst)
+{
+	union dmub_rb_cmd cmd;
+	struct dc_context *dc = abm->ctx;
+	uint8_t panel_mask = 0x01 << panel_inst;
+	struct dc_link *edp_links[MAX_NUM_EDP];
+	unsigned int i, edp_num;
+
+	// TODO: Optimize by only reading back final 4 bytes
+	dmub_srv_flush_buffer_mem(dc->dmub_srv->dmub, &dc->dmub_srv->dmub->scratch_mem_fb);
+
+	// Copy iramtable into cw7
+	memcpy(dc->dmub_srv->dmub->scratch_mem_fb.cpu_addr, (void *)src, bytes);
+
+	memset(&cmd, 0, sizeof(cmd));
+	// Fw will copy from cw7 to fw_state
+	cmd.cacp_init_config.header.type = DMUB_CMD__CACP;
+	cmd.cacp_init_config.header.sub_type = DMUB_CMD__CACP_INIT_CONFIG;
+	cmd.cacp_init_config.cacp_init_config_data.src.quad_part = dc->dmub_srv->dmub->scratch_mem_fb.gpu_addr;
+	cmd.cacp_init_config.cacp_init_config_data.bytes = (uint16_t)bytes;
+	cmd.cacp_init_config.cacp_init_config_data.panel_mask = panel_mask;
+	cmd.cacp_init_config.cacp_init_config_data.visual_confirm =
+		(dc->dc->debug.visual_confirm == VISUAL_CONFIRM_ABM) ? true : false;
+
+	cmd.cacp_init_config.header.payload_bytes = sizeof(struct dmub_cmd_cacp_init_config_data);
+
+	dc_get_edp_links(dc->dc, edp_links, &edp_num);
+	for (i = 0; i < edp_num; i++) {
+		if (panel_inst == i)
+			break;
+	}
+
+	if (i < edp_num) {
+		cmd.cacp_init_config.cacp_init_config_data.strscl_valid =
+			(uint8_t)edp_links[panel_inst]->panel_config.cacp.strscl_valid;
+		cmd.cacp_init_config.cacp_init_config_data.mode =
+			edp_links[panel_inst]->panel_config.cacp.cacp_control_mode ?
+			DMUB_CMD_CACP_CONTROL_MODE_1 : DMUB_CMD_CACP_CONTROL_MODE_0;
+		for (int j = 0; j < CACP_LEVEL_NUM; j++) {
+			cmd.cacp_init_config.cacp_init_config_data.strscl_sdr[j] =
+				(uint8_t)edp_links[panel_inst]->panel_config.cacp.strscl_sdr[j];
+			cmd.cacp_init_config.cacp_init_config_data.strscl_hdr[j] =
+				(uint8_t)edp_links[panel_inst]->panel_config.cacp.strscl_hdr[j];
+		}
+	}
+
+	dc_wake_and_execute_dmub_cmd(dc, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+}
+
+bool dmub_cacp_set_level(struct abm *abm, unsigned int abm_level, unsigned char panel_mask)
+{
+	union dmub_rb_cmd cmd;
+	struct dc_context *dc = abm->ctx;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.cacp_set_level.header.type = DMUB_CMD__CACP;
+	cmd.cacp_set_level.header.sub_type = DMUB_CMD__CACP_SET_LEVEL;
+
+	cmd.cacp_set_level.cacp_set_level_data.level = abm_level;
+	cmd.cacp_set_level.cacp_set_level_data.version = DMUB_CMD_CACP_CONTROL_VERSION_1;
+	cmd.cacp_set_level.cacp_set_level_data.panel_mask = panel_mask;
+
+	cmd.cacp_set_level.header.payload_bytes = sizeof(struct dmub_cmd_cacp_set_level_data);
+
+	dc_wake_and_execute_dmub_cmd(dc, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+
+	return true;
+}
+
+bool dmub_cacp_set_pipe(struct abm *abm, unsigned int otg_inst,
+		unsigned int pipe_option, unsigned int panel_inst, unsigned int pwrseq_inst)
+{
+	union dmub_rb_cmd cmd;
+	struct dc_context *dc = abm->ctx;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.cacp_set_pipe.header.type = DMUB_CMD__CACP;
+	cmd.cacp_set_pipe.header.sub_type = DMUB_CMD__CACP_SET_PIPE;
+
+	cmd.cacp_set_pipe.cacp_set_pipe_data.otg_inst = (uint8_t)otg_inst;
+	cmd.cacp_set_pipe.cacp_set_pipe_data.panel_inst = (uint8_t)panel_inst;
+	cmd.cacp_set_pipe.cacp_set_pipe_data.set_pipe_option = (uint8_t)pipe_option;
+	cmd.cacp_set_pipe.cacp_set_pipe_data.pwrseq_inst = (uint8_t)pwrseq_inst;
+	cmd.cacp_set_pipe.header.payload_bytes = sizeof(struct dmub_cmd_cacp_set_pipe_data);
+
+	dc_wake_and_execute_dmub_cmd(dc, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+
+	return true;
+}
+
+bool dmub_cacp_set_event(struct abm *abm, unsigned int full_screen, unsigned int trans_info,
+		unsigned int hdr_mode, unsigned int scaling_enable, unsigned int panel_inst)
+{
+	union dmub_rb_cmd cmd;
+	struct dc_context *dc = abm->ctx;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.cacp_set_event.header.type = DMUB_CMD__CACP;
+	cmd.cacp_set_event.header.sub_type = DMUB_CMD__CACP_SET_EVENT;
+
+	//TODO:
+	cmd.cacp_set_event.cacp_set_event_data.full_screen_mode = (uint8_t)full_screen;
+	cmd.cacp_set_event.cacp_set_event_data.trans_info = trans_info;
+	cmd.cacp_set_event.cacp_set_event_data.hdr_mode = (uint8_t)hdr_mode;
+	cmd.cacp_set_event.cacp_set_event_data.vb_scaling_enable = (uint8_t)scaling_enable;
+	cmd.cacp_set_event.cacp_set_event_data.panel_mask = (1<<panel_inst);
+
+	cmd.cacp_set_event.header.payload_bytes = sizeof(struct dmub_cmd_cacp_set_event_data);
+
+	dc_wake_and_execute_dmub_cmd(dc, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+
+	return true;
+}
+
+bool dmub_cacp_set_pause(struct abm *abm, bool pause, unsigned int panel_inst, unsigned int otg_inst)
+{
+	union dmub_rb_cmd cmd;
+	struct dc_context *dc = abm->ctx;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.cacp_pause.header.type = DMUB_CMD__CACP;
+	cmd.cacp_pause.header.sub_type = DMUB_CMD__CACP_PAUSE;
+
+	cmd.cacp_pause.cacp_pause_data.panel_mask = (1<<panel_inst);
+	cmd.cacp_pause.cacp_pause_data.otg_inst = (uint8_t)otg_inst;
+	cmd.cacp_pause.cacp_pause_data.enable = pause;
+
+	cmd.cacp_pause.header.payload_bytes = sizeof(struct dmub_cmd_cacp_pause_data);
+
+	dc_wake_and_execute_dmub_cmd(dc, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+
+	return true;
+}
+
+bool dmub_cacp_set_backlight_level(struct abm *abm,
+		unsigned int backlight_pwm_u16_16,
+		unsigned int frame_ramp,
+		unsigned int panel_inst)
+{
+	union dmub_rb_cmd cmd;
+	struct dc_context *dc = abm->ctx;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.cacp_set_backlight.header.type = DMUB_CMD__CACP;
+	cmd.cacp_set_backlight.header.sub_type = DMUB_CMD__CACP_SET_BACKLIGHT;
+	cmd.cacp_set_backlight.cacp_set_backlight_data.frame_ramp = frame_ramp;
+	cmd.cacp_set_backlight.cacp_set_backlight_data.backlight_user_level = backlight_pwm_u16_16;
+	cmd.cacp_set_backlight.cacp_set_backlight_data.version = DMUB_CMD_CACP_CONTROL_VERSION_1;
+	cmd.cacp_set_backlight.cacp_set_backlight_data.panel_mask = (0x01 << panel_inst);
+	cmd.cacp_set_backlight.header.payload_bytes = sizeof(struct dmub_cmd_cacp_set_backlight_data);
+
+	dc_wake_and_execute_dmub_cmd(dc, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+
+	return true;
+}
+
+void dmub_cacp_enable_fractional_pwm(struct abm *abm, uint8_t panel_mask)
+{
+	union dmub_rb_cmd cmd;
+	struct dc_context *dc = abm->ctx;
+	uint32_t fractional_pwm = (dc->dc->config.disable_fractional_pwm == false) ? 1 : 0;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.cacp_set_pwm_frac.header.type = DMUB_CMD__CACP;
+	cmd.cacp_set_pwm_frac.header.sub_type = DMUB_CMD__CACP_SET_PWM_FRAC;
+	cmd.cacp_set_pwm_frac.cacp_set_pwm_frac_data.fractional_pwm = fractional_pwm;
+	cmd.cacp_set_pwm_frac.cacp_set_pwm_frac_data.version = DMUB_CMD_CACP_CONTROL_VERSION_1;
+	cmd.cacp_set_pwm_frac.cacp_set_pwm_frac_data.panel_mask = panel_mask;
+	cmd.cacp_set_pwm_frac.header.payload_bytes = sizeof(struct dmub_cmd_cacp_set_pwm_frac_data);
+
+	dc_wake_and_execute_dmub_cmd(dc, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+}
+
+bool dmub_cacp_get_histogram(
+		struct dc_context *dc,
+		unsigned int panel_inst,
+		unsigned int *histogram,
+		enum dmub_abm_histogram_type histogram_type,
+		unsigned int size)
+{
+	union dmub_rb_cmd cmd;
+
+	dmub_srv_flush_buffer_mem(dc->dmub_srv->dmub, &dc->dmub_srv->dmub->scratch_mem_fb);
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.cacp_get_histogram.header.type = DMUB_CMD__CACP;
+	cmd.cacp_get_histogram.header.sub_type = DMUB_CMD__CACP_GET_HISTOGRAM;
+
+	cmd.cacp_get_histogram.dest.quad_part = dc->dmub_srv->dmub->scratch_mem_fb.gpu_addr;
+	cmd.cacp_get_histogram.bytes = (uint16_t)size;
+	cmd.cacp_get_histogram.panel_inst = (uint8_t)panel_inst;
+	cmd.cacp_get_histogram.histogram_type = histogram_type;
+	cmd.cacp_get_histogram.header.payload_bytes = sizeof(struct dmub_rb_cmd_cacp_get_histogram);
+
+	dc_wake_and_execute_dmub_cmd(dc, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+
+	memcpy((void *)histogram, dc->dmub_srv->dmub->scratch_mem_fb.cpu_addr, size);
+
+	return true;
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_cacp.h b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_cacp.h
new file mode 100644
index 0000000000000..3b3b712c12423
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_cacp.h
@@ -0,0 +1,20 @@
+/* Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. */
+
+#include "dmub_cmd.h"
+
+#ifndef __DMUB_ABM_CACP_H__
+#define __DMUB_ABM_CACP_H__
+
+void dmub_cacp_init(struct abm *abm, const char *src, unsigned int bytes, unsigned int panel_inst);
+bool dmub_cacp_set_level(struct abm *abm, unsigned int cacp_level, unsigned char panel_mask);
+bool dmub_cacp_set_pipe(struct abm *abm, unsigned int otg_inst, unsigned int pipe_option,
+						unsigned int panel_inst, unsigned int pwrseq_inst);
+bool dmub_cacp_set_event(struct abm *abm, unsigned int full_screen, unsigned int trans_info, unsigned int hdr_mode,
+						unsigned int scaling_enable, unsigned int panel_inst);
+bool dmub_cacp_set_pause(struct abm *abm, bool pause, unsigned int panel_inst, unsigned int otg_inst);
+bool dmub_cacp_set_backlight_level(struct abm *abm, unsigned int backlight_pwm_u16_16, unsigned int frame_ramp,
+						unsigned int panel_inst);
+void dmub_cacp_enable_fractional_pwm(struct abm *abm, uint8_t panel_mask);
+bool dmub_cacp_get_histogram(struct dc_context *dc, unsigned int panel_inst, unsigned int *histogram,
+		enum dmub_abm_histogram_type histogram_type, unsigned int size);
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_lcd.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_lcd.c
index 2f4d368bd3fd8..f60e43ba60604 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_lcd.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_lcd.c
@@ -166,7 +166,7 @@ void dmub_abm_init_config(struct abm *abm,
 	uint8_t panel_mask = 0x01 << inst;
 
 	// TODO: Optimize by only reading back final 4 bytes
-	dmub_flush_buffer_mem(&dc->dmub_srv->dmub->scratch_mem_fb);
+	dmub_srv_flush_buffer_mem(dc->dmub_srv->dmub, &dc->dmub_srv->dmub->scratch_mem_fb);
 
 	// Copy iramtable into cw7
 	memcpy(dc->dmub_srv->dmub->scratch_mem_fb.cpu_addr, (void *)src, bytes);
@@ -227,7 +227,7 @@ bool dmub_abm_save_restore(
 	unsigned int bytes = sizeof(struct abm_save_restore);
 
 	// TODO: Optimize by only reading back final 4 bytes
-	dmub_flush_buffer_mem(&dc->dmub_srv->dmub->scratch_mem_fb);
+	dmub_srv_flush_buffer_mem(dc->dmub_srv->dmub, &dc->dmub_srv->dmub->scratch_mem_fb);
 
 	// Copy iramtable into cw7
 	memcpy(dc->dmub_srv->dmub->scratch_mem_fb.cpu_addr, (void *)pData, bytes);
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c
index 9be578ff8c880..140c66081492f 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c
@@ -27,8 +27,6 @@
 
 #include "dce/dce_11_0_d.h"
 #include "dce/dce_11_0_sh_mask.h"
-#include "gmc/gmc_8_2_sh_mask.h"
-#include "gmc/gmc_8_2_d.h"
 
 #include "include/logger_interface.h"
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c
index b265a72eeb703..095869912c091 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c
@@ -27,8 +27,6 @@
 #include "dce/dce_11_0_d.h"
 #include "dce/dce_11_0_sh_mask.h"
 /* TODO: this needs to be looked at, used by Stella's workaround*/
-#include "gmc/gmc_8_2_d.h"
-#include "gmc/gmc_8_2_sh_mask.h"
 
 #include "include/logger_interface.h"
 #include "inc/dce_calcs.h"
diff --git a/drivers/gpu/drm/amd/display/dc/dce112/dce112_compressor.c b/drivers/gpu/drm/amd/display/dc/dce112/dce112_compressor.c
index fe97d3946cab0..4b273762f07ab 100644
--- a/drivers/gpu/drm/amd/display/dc/dce112/dce112_compressor.c
+++ b/drivers/gpu/drm/amd/display/dc/dce112/dce112_compressor.c
@@ -27,8 +27,12 @@
 
 #include "dce/dce_11_2_d.h"
 #include "dce/dce_11_2_sh_mask.h"
-#include "gmc/gmc_8_1_sh_mask.h"
-#include "gmc/gmc_8_1_d.h"
+
+#ifndef mmGMCON_LPT_TARGET
+#define mmGMCON_LPT_TARGET 0x0D53
+#define GMCON_LPT_TARGET__STCTRL_LPT_TARGET__SHIFT 0x00000000
+#define GMCON_LPT_TARGET__STCTRL_LPT_TARGET_MASK 0xffffffffL
+#endif
 
 #include "include/logger_interface.h"
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c
index bfd5515c2f4fb..66fe7f313ea34 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c
@@ -303,6 +303,190 @@ bool cm3_helper_translate_curve_to_hw_format(struct dc_context *ctx,
 	return true;
 }
 
+/* Linear interpolation of tf_pts entries, where (i >> 4) is the integer tf_pts
+ * index, (i & 0xf) is the 1/16 sub-position.
+ */
+static struct fixed31_32 interp_tf_pts(const struct fixed31_32 *output_tf_channel, int i)
+{
+	struct fixed31_32 in_plus_one, in, value;
+	uint32_t t = i & 0xf;
+
+	in_plus_one = output_tf_channel[(i >> 4) + 1];
+	in = output_tf_channel[i >> 4];
+	value = dc_fixpt_sub(in_plus_one, in);
+	value = dc_fixpt_shr(dc_fixpt_mul_int(value, t), 4);
+	value = dc_fixpt_add(in, value);
+
+	return value;
+}
+
+bool cm3_helper_translate_curve_to_degamma_hw_format(
+				const struct dc_transfer_func *output_tf,
+				struct pwl_params *lut_params)
+{
+	struct curve_points3 *corner_points;
+	struct pwl_result_data *rgb_resulted;
+	struct pwl_result_data *rgb;
+	struct pwl_result_data *rgb_plus_1;
+
+	int32_t region_start, region_end;
+	int32_t i;
+	uint32_t j, k, seg_distr[MAX_REGIONS_NUMBER], increment, start_index, hw_points;
+
+	if (output_tf == NULL || lut_params == NULL || output_tf->type == TF_TYPE_BYPASS)
+		return false;
+
+	corner_points = lut_params->corner_points;
+	rgb_resulted = lut_params->rgb_resulted;
+	hw_points = 0;
+
+	memset(lut_params, 0, sizeof(struct pwl_params));
+	memset(seg_distr, 0, sizeof(seg_distr));
+
+	if (output_tf->tf == TRANSFER_FUNCTION_PQ ||
+	    output_tf->tf == TRANSFER_FUNCTION_SRGB) {
+		/* 9 segments
+		 * segments are from 2^-9 to 0
+		 */
+		const uint8_t SEG_COUNT = 9;
+		seg_distr[0] = 0; // Since we only have one point in darkest region
+		for (k = 1; k < SEG_COUNT; k++)
+			seg_distr[k] = k - 1; // 2^(k-1) points per region; halves as k decreases
+
+		region_start = -SEG_COUNT;
+		region_end = 0;
+	} else {
+		/* 12 segments
+		 * segments are from 2^-12 to 2^0
+		 * There are less than 256 points, for optimization
+		 */
+		const uint8_t SEG_COUNT = 12;
+
+		for (i = 0; i < SEG_COUNT; i++)
+			seg_distr[i] = 4;
+
+		region_start = -SEG_COUNT;
+		region_end = 0;
+	}
+
+	for (i = region_end - region_start; i < MAX_REGIONS_NUMBER ; i++)
+		seg_distr[i] = -1;
+
+	for (k = 0; k < MAX_REGIONS_NUMBER; k++) {
+		if (seg_distr[k] != -1)
+			hw_points += (1 << seg_distr[k]);
+	}
+
+	j = 0;
+	for (k = 0; k < (region_end - region_start); k++) {
+		increment = (NUMBER_SW_SEGMENTS << 4) / (1 << seg_distr[k]);
+		start_index = (region_start + k + MAX_LOW_POINT) *
+				NUMBER_SW_SEGMENTS;
+		for (i = (start_index << 4);
+		     i < (start_index << 4) + (NUMBER_SW_SEGMENTS << 4);
+		     i += increment) {
+			if (j == hw_points - 1)
+				break;
+			if ((i >> 4) + 1 >= TRANSFER_FUNC_POINTS)
+				return false;
+
+			rgb_resulted[j].red = interp_tf_pts(output_tf->tf_pts.red, i);
+			rgb_resulted[j].green = interp_tf_pts(output_tf->tf_pts.green, i);
+			rgb_resulted[j].blue = interp_tf_pts(output_tf->tf_pts.blue, i);
+			j++;
+		}
+	}
+
+	/* last point */
+	start_index = (region_end + MAX_LOW_POINT) * NUMBER_SW_SEGMENTS;
+	rgb_resulted[hw_points - 1].red = output_tf->tf_pts.red[start_index];
+	rgb_resulted[hw_points - 1].green = output_tf->tf_pts.green[start_index];
+	rgb_resulted[hw_points - 1].blue = output_tf->tf_pts.blue[start_index];
+
+	corner_points[0].red.x = dc_fixpt_pow(dc_fixpt_from_int(2),
+					     dc_fixpt_from_int(region_start));
+	corner_points[0].green.x = corner_points[0].red.x;
+	corner_points[0].blue.x = corner_points[0].red.x;
+	corner_points[1].red.x = dc_fixpt_pow(dc_fixpt_from_int(2),
+					     dc_fixpt_from_int(region_end));
+	corner_points[1].green.x = corner_points[1].red.x;
+	corner_points[1].blue.x = corner_points[1].red.x;
+
+	corner_points[0].red.y = rgb_resulted[0].red;
+	corner_points[0].green.y = rgb_resulted[0].green;
+	corner_points[0].blue.y = rgb_resulted[0].blue;
+
+	/* see comment above, m_arrPoints[1].y should be the Y value for the
+	 * region end (m_numOfHwPoints), not last HW point(m_numOfHwPoints - 1)
+	 */
+	corner_points[1].red.y = rgb_resulted[hw_points - 1].red;
+	corner_points[1].green.y = rgb_resulted[hw_points - 1].green;
+	corner_points[1].blue.y = rgb_resulted[hw_points - 1].blue;
+	corner_points[1].red.slope = dc_fixpt_zero;
+	corner_points[1].green.slope = dc_fixpt_zero;
+	corner_points[1].blue.slope = dc_fixpt_zero;
+
+	if (output_tf->tf == TRANSFER_FUNCTION_PQ) {
+		/* for PQ, we want to have a straight line from last HW X point,
+		 * and the slope to be such that we hit 1.0 at 10000 nits.
+		 */
+		const struct fixed31_32 end_value =
+				dc_fixpt_from_int(125);
+
+		corner_points[1].red.slope = dc_fixpt_div(
+			dc_fixpt_sub(dc_fixpt_one, corner_points[1].red.y),
+			dc_fixpt_sub(end_value, corner_points[1].red.x));
+		corner_points[1].green.slope = dc_fixpt_div(
+			dc_fixpt_sub(dc_fixpt_one, corner_points[1].green.y),
+			dc_fixpt_sub(end_value, corner_points[1].green.x));
+		corner_points[1].blue.slope = dc_fixpt_div(
+			dc_fixpt_sub(dc_fixpt_one, corner_points[1].blue.y),
+			dc_fixpt_sub(end_value, corner_points[1].blue.x));
+	}
+
+	lut_params->hw_points_num = hw_points;
+
+	k = 0;
+	for (i = 1; i < MAX_REGIONS_NUMBER; i++) {
+		if (seg_distr[k] != -1) {
+			lut_params->arr_curve_points[k].segments_num =
+					seg_distr[k];
+			lut_params->arr_curve_points[i].offset =
+					lut_params->arr_curve_points[k].offset + (1 << seg_distr[k]);
+		}
+		k++;
+	}
+
+	if (seg_distr[k] != -1)
+		lut_params->arr_curve_points[k].segments_num = seg_distr[k];
+
+	rgb = rgb_resulted;
+	rgb_plus_1 = rgb_resulted + 1;
+
+	i = 1;
+	while (i != hw_points + 1) {
+		if (dc_fixpt_lt(rgb_plus_1->red, rgb->red))
+			rgb_plus_1->red = rgb->red;
+		if (dc_fixpt_lt(rgb_plus_1->green, rgb->green))
+			rgb_plus_1->green = rgb->green;
+		if (dc_fixpt_lt(rgb_plus_1->blue, rgb->blue))
+			rgb_plus_1->blue = rgb->blue;
+
+		rgb->delta_red   = dc_fixpt_sub(rgb_plus_1->red,   rgb->red);
+		rgb->delta_green = dc_fixpt_sub(rgb_plus_1->green, rgb->green);
+		rgb->delta_blue  = dc_fixpt_sub(rgb_plus_1->blue,  rgb->blue);
+
+		++rgb_plus_1;
+		++rgb;
+		++i;
+	}
+	cm3_helper_convert_to_custom_float(rgb_resulted,
+						lut_params->corner_points,
+						hw_points, false);
+
+	return true;
+}
+
 bool cm3_helper_convert_to_custom_float(
 		struct pwl_result_data *rgb_resulted,
 		struct curve_points3 *corner_points,
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.c
index 4b36c01a9e7ab..00e82d627d918 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.c
@@ -1189,8 +1189,9 @@ void dcn10_link_encoder_dp_set_phy_pattern(
 		set_dp_phy_pattern_prbs7(enc10);
 		break;
 	case DP_TEST_PATTERN_80BIT_CUSTOM:
-		set_dp_phy_pattern_80bit_custom(
-			enc10, param->custom_pattern);
+		if (param)
+			set_dp_phy_pattern_80bit_custom(
+				enc10, param->custom_pattern);
 		break;
 	case DP_TEST_PATTERN_CP2520_1:
 		set_dp_phy_pattern_hbr2_compliance_cp2520_2(enc10, 1);
@@ -1202,8 +1203,9 @@ void dcn10_link_encoder_dp_set_phy_pattern(
 		set_dp_phy_pattern_hbr2_compliance_cp2520_2(enc10, 3);
 		break;
 	case DP_TEST_PATTERN_VIDEO_MODE: {
-		set_dp_phy_pattern_passthrough_mode(
-			enc10, param->dp_panel_mode);
+		if (param)
+			set_dp_phy_pattern_passthrough_mode(
+				enc10, param->dp_panel_mode);
 		break;
 	}
 
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn31/dcn31_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn31/dcn31_dio_link_encoder.c
index bcb791d741894..f57f3ba68a02e 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn31/dcn31_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn31/dcn31_dio_link_encoder.c
@@ -516,6 +516,8 @@ void dcn31_link_encoder_construct_minimal(
 	struct dc_context *ctx,
 	const struct encoder_feature_support *enc_features,
 	const struct dcn10_link_enc_registers *link_regs,
+	const struct dcn10_link_enc_shift *link_shift,
+	const struct dcn10_link_enc_mask *link_mask,
 	enum engine_id eng_id)
 {
 	struct dcn10_link_encoder *enc10 = &enc20->enc10;
@@ -529,6 +531,8 @@ void dcn31_link_encoder_construct_minimal(
 	enc10->base.features = *enc_features;
 	enc10->base.transmitter = TRANSMITTER_UNKNOWN;
 	enc10->link_regs = link_regs;
+	enc10->link_shift = link_shift;
+	enc10->link_mask = link_mask;
 
 	enc10->base.output_signals =
 		SIGNAL_TYPE_DISPLAY_PORT |
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn31/dcn31_dio_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dio/dcn31/dcn31_dio_link_encoder.h
index 3cf5875279916..88ab9684e2073 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn31/dcn31_dio_link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn31/dcn31_dio_link_encoder.h
@@ -246,6 +246,8 @@ void dcn31_link_encoder_construct_minimal(
 	struct dc_context *ctx,
 	const struct encoder_feature_support *enc_features,
 	const struct dcn10_link_enc_registers *link_regs,
+	const struct dcn10_link_enc_shift *link_shift,
+	const struct dcn10_link_enc_mask *link_mask,
 	enum engine_id eng_id);
 
 void dcn31_link_encoder_set_dio_phy_mux(
diff --git a/drivers/gpu/drm/amd/display/dc/dm_services.h b/drivers/gpu/drm/amd/display/dc/dm_services.h
index 8b062b011fc65..2cf4bcb03cb06 100644
--- a/drivers/gpu/drm/amd/display/dc/dm_services.h
+++ b/drivers/gpu/drm/amd/display/dc/dm_services.h
@@ -127,10 +127,6 @@ uint32_t generic_reg_update_ex(const struct dc_context *ctx,
 struct dc_dmub_srv *dc_dmub_srv_create(struct dc *dc, struct dmub_srv *dmub);
 void dc_dmub_srv_destroy(struct dc_dmub_srv **dmub_srv);
 
-void reg_sequence_start_gather(const struct dc_context *ctx);
-void reg_sequence_start_execute(const struct dc_context *ctx);
-void reg_sequence_wait_done(const struct dc_context *ctx);
-
 #define FD(reg_field)	reg_field ## __SHIFT, \
 						reg_field ## _MASK
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c
index dcca23d53261e..2ad4a26356832 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c
@@ -1237,7 +1237,7 @@ bool dcn_validate_bandwidth(
 			if (pipe->plane_state) {
 				struct pipe_ctx *hsplit_pipe = pipe->bottom_pipe;
 
-				pipe->plane_state->update_flags.bits.full_update = 1;
+				pipe->plane_state->update_bits.full_update = 1;
 
 				if (v->dpp_per_plane[input_idx] == 2 ||
 					((pipe->stream->view_format ==
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
index bd14ebea11110..8064c4b3fd259 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
@@ -5337,7 +5337,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 		for (j = 0; j <= 1; ++j) {
 			double VMDataOnlyReturnBWPerState;
 			double HostVMInefficiencyFactor = 1;
-			int NextPrefetchModeState = MinPrefetchMode;
+			unsigned int NextPrefetchModeState = MinPrefetchMode;
 			bool UnboundedRequestEnabledThisState = false;
 			unsigned int CompressedBufferSizeInkByteThisState = 0;
 			double dummy;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c
index 2ea5cf37f2735..bf2dde26b98b0 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c
@@ -5421,7 +5421,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 		for (j = 0; j <= 1; ++j) {
 			double VMDataOnlyReturnBWPerState;
 			double HostVMInefficiencyFactor = 1;
-			int NextPrefetchModeState = MinPrefetchMode;
+			unsigned int NextPrefetchModeState = MinPrefetchMode;
 			bool UnboundedRequestEnabledThisState = false;
 			unsigned int CompressedBufferSizeInkByteThisState = 0;
 			double dummy;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper_fpu.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper_fpu.c
index de40d7bae252d..11fc0b1cd152e 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper_fpu.c
@@ -118,16 +118,16 @@ static void dml21_calculate_rq_and_dlg_params(const struct dc *dc, struct dc_sta
 	context->bw_ctx.bw.dcn.clk.bw_dispclk_khz = context->bw_ctx.bw.dcn.clk.dispclk_khz;
 	if (in_ctx->v21.dml_init.soc_bb.clk_table.dispclk.num_clk_values > 1) {
 		context->bw_ctx.bw.dcn.clk.max_supported_dispclk_khz =
-			in_ctx->v21.dml_init.soc_bb.clk_table.dispclk.clk_values_khz[in_ctx->v21.dml_init.soc_bb.clk_table.dispclk.num_clk_values] * 1000;
+			in_ctx->v21.dml_init.soc_bb.clk_table.dispclk.clk_values_khz[in_ctx->v21.dml_init.soc_bb.clk_table.dispclk.num_clk_values - 1];
 	} else {
-		context->bw_ctx.bw.dcn.clk.max_supported_dispclk_khz = in_ctx->v21.dml_init.soc_bb.clk_table.dispclk.clk_values_khz[0] * 1000;
+		context->bw_ctx.bw.dcn.clk.max_supported_dispclk_khz = in_ctx->v21.dml_init.soc_bb.clk_table.dispclk.clk_values_khz[0];
 	}
 
 	if (in_ctx->v21.dml_init.soc_bb.clk_table.dppclk.num_clk_values > 1) {
 		context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz =
-			in_ctx->v21.dml_init.soc_bb.clk_table.dppclk.clk_values_khz[in_ctx->v21.dml_init.soc_bb.clk_table.dppclk.num_clk_values] * 1000;
+			in_ctx->v21.dml_init.soc_bb.clk_table.dppclk.clk_values_khz[in_ctx->v21.dml_init.soc_bb.clk_table.dppclk.num_clk_values - 1];
 	} else {
-		context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz = in_ctx->v21.dml_init.soc_bb.clk_table.dppclk.clk_values_khz[0] * 1000;
+		context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz = in_ctx->v21.dml_init.soc_bb.clk_table.dppclk.clk_values_khz[0];
 	}
 
 	/* get global mall allocation */
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h
index 040d89f6de359..51cae3efe310e 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h
@@ -205,7 +205,6 @@ static const struct dml2_soc_bb dml2_socbb_dcn42 = {
 	.xtalclk_mhz = 24,
 	.pcie_refclk_mhz = 100,
 	.dchub_refclk_mhz = 50,
-	.mall_allocated_for_dcn_mbytes = 0,
 	.max_outstanding_reqs = 256,
 	.fabric_datapath_to_dcn_data_return_bytes = 32,
 	.return_bus_width_bytes = 64,
@@ -216,11 +215,6 @@ static const struct dml2_soc_bb dml2_socbb_dcn42 = {
 	.phy_downspread_percent = 0.38,
 	.dcn_downspread_percent = 0.38,
 	.dispclk_dppclk_vco_speed_mhz = 3000,
-	.do_urgent_latency_adjustment = 0,
-	.mem_word_bytes = 32,
-	.num_dcc_mcaches = 8,
-	.mcache_size_bytes = 2048,
-	.mcache_line_size_bytes = 32,
 	.max_fclk_for_uclk_dpm_khz = 2200 * 1000,
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42b_soc_bb.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42b_soc_bb.h
index ce4025591b876..60ef564198465 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42b_soc_bb.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42b_soc_bb.h
@@ -75,7 +75,7 @@ static const struct dml2_soc_bb dml2_socbb_dcn42b = {
 				.clk_values_khz = {2},
 		},
 		.uclk = {
-				.clk_values_khz = {400000},
+				.clk_values_khz = {2400000},
 				.num_clk_values = 1,
 		},
 		.fclk = {
@@ -224,4 +224,42 @@ static const struct dml2_soc_bb dml2_socbb_dcn42b = {
 	.max_fclk_for_uclk_dpm_khz = 2200 * 1000,
 };
 
+static const struct dml2_ip_capabilities dml2_dcn42b_max_ip_caps = {
+	.pipe_count = 4,
+	.otg_count = 3,
+	.num_dsc = 3,
+	.max_num_dp2p0_streams = 3,
+	.max_num_hdmi_frl_outputs = 0,
+	.max_num_dp2p0_outputs = 2,
+	.rob_buffer_size_kbytes = 64,
+	.config_return_buffer_size_in_kbytes = 1792,
+	.config_return_buffer_segment_size_in_kbytes = 64,
+	.meta_fifo_size_in_kentries = 32,
+	.compressed_buffer_segment_size_in_kbytes = 64,
+	.cursor_buffer_size = 24,
+	.max_flip_time_us = 110,
+	.max_flip_time_lines = 50,
+	.hostvm_mode = 0,
+	.subvp_drr_scheduling_margin_us = 100,
+	.subvp_prefetch_end_to_mall_start_us = 15,
+	.subvp_fw_processing_delay = 15,
+	.max_vactive_det_fill_delay_us = 400,
+
+	.fams2 = {
+		.max_allow_delay_us = 100 * 1000,
+		.scheduling_delay_us = 550,
+		.vertical_interrupt_ack_delay_us = 40,
+		.allow_programming_delay_us = 18,
+		.min_allow_width_us = 20,
+		.subvp_df_throttle_delay_us = 100,
+		.subvp_programming_delay_us = 200,
+		.subvp_prefetch_to_mall_delay_us = 18,
+		.drr_programming_delay_us = 35,
+
+		.lock_timeout_us = 5000,
+		.recovery_timeout_us = 5000,
+		.flip_programming_delay_us = 300,
+	},
+};
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_soc_parameter_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_soc_parameter_types.h
index 6152155d60735..672b96a3da746 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_soc_parameter_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_soc_parameter_types.h
@@ -71,8 +71,21 @@ enum dml2_qos_param_type {
 	dml2_qos_param_type_dcn4x
 };
 
+//Indicies mapped to DPM level
+// Unpopulated indicies should fallback to the global derate value.
+struct dml2_soc_derate_values_per_dpm {
+	unsigned int dram_derate_percent_pixel[DML_MAX_CLK_TABLE_SIZE];
+	unsigned int fclk_derate_percent[DML_MAX_CLK_TABLE_SIZE];
+	unsigned int dcfclk_derate_percent[DML_MAX_CLK_TABLE_SIZE];
+};
+
+struct dml2_soc_derates_per_dpm {
+	struct dml2_soc_derate_values_per_dpm system_active_derates_per_dpm;
+};
+
 struct dml2_soc_qos_parameters {
 	struct dml2_soc_derates derate_table;
+	struct dml2_soc_derates_per_dpm derate_table_per_dpm;
 	struct {
 		unsigned int base_latency_us;
 		unsigned int scaling_factor_us;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c
index f338e733318e1..51a66e1be7a13 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c
@@ -2701,7 +2701,8 @@ static double dml_get_return_bandwidth_available(
 	bool is_hvm_only,
 	double dcfclk_mhz,
 	double fclk_mhz,
-	double dram_bw_mbps)
+	double dram_bw_mbps,
+	unsigned int uclk_dpm_level)
 {
 	double return_bw_mbps = 0.;
 	double ideal_sdp_bandwidth = (double)soc->return_bus_width_bytes * dcfclk_mhz;
@@ -2722,9 +2723,16 @@ static double dml_get_return_bandwidth_available(
 			derate_fabric_factor = soc->qos_parameters.derate_table.dcn_mall_prefetch_average.fclk_derate_percent / 100.0;
 			derate_dram_factor = soc->qos_parameters.derate_table.dcn_mall_prefetch_average.dram_derate_percent_pixel / 100.0;
 		} else { // just assume sys_active
-			derate_sdp_factor = soc->qos_parameters.derate_table.system_active_average.dcfclk_derate_percent / 100.0;
-			derate_fabric_factor = soc->qos_parameters.derate_table.system_active_average.fclk_derate_percent / 100.0;
-			derate_dram_factor = soc->qos_parameters.derate_table.system_active_average.dram_derate_percent_pixel / 100.0;
+			// use per dpm derates if the values are populated. Otherwise use global derates
+			derate_sdp_factor = soc->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.dcfclk_derate_percent[uclk_dpm_level] != 0 ?
+				soc->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.dcfclk_derate_percent[uclk_dpm_level] / 100.0 :
+				soc->qos_parameters.derate_table.system_active_average.dcfclk_derate_percent / 100.0;
+			derate_fabric_factor = soc->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.fclk_derate_percent[uclk_dpm_level] != 0 ?
+				soc->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.fclk_derate_percent[uclk_dpm_level] / 100.0 :
+				soc->qos_parameters.derate_table.system_active_average.fclk_derate_percent / 100.0;
+			derate_dram_factor = soc->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.dram_derate_percent_pixel[uclk_dpm_level] != 0 ?
+				soc->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.dram_derate_percent_pixel[uclk_dpm_level] / 100.0 :
+				soc->qos_parameters.derate_table.system_active_average.dram_derate_percent_pixel / 100.0;
 		}
 	} else { // urgent bw
 		if (state_type == dml2_core_internal_soc_state_svp_prefetch) {
@@ -2778,6 +2786,7 @@ static double dml_get_return_bandwidth_available(
 	DML_LOG_VERBOSE("DML::%s: derate_fabric_bandwidth = %f (derate %f)\n", __func__, derate_fabric_bandwidth, derate_fabric_factor);
 	DML_LOG_VERBOSE("DML::%s: derate_dram_bandwidth = %f (derate %f)\n", __func__, derate_dram_bandwidth, derate_dram_factor);
 	DML_LOG_VERBOSE("DML::%s: return_bw_mbps = %f\n", __func__, return_bw_mbps);
+	DML_LOG_VERBOSE("DML::%s: uclk_dpm_level = %u\n", __func__, uclk_dpm_level);
 	return return_bw_mbps;
 }
 
@@ -2793,7 +2802,8 @@ static noinline_for_stack void calculate_bandwidth_available(
 	bool HostVMEnable,
 	double dcfclk_mhz,
 	double fclk_mhz,
-	double dram_bw_mbps)
+	double dram_bw_mbps,
+	unsigned int uclk_dpm_level)
 {
 	unsigned int n, m;
 
@@ -2812,9 +2822,10 @@ static noinline_for_stack void calculate_bandwidth_available(
 				0, // hvm_only
 				dcfclk_mhz,
 				fclk_mhz,
-				dram_bw_mbps);
+				dram_bw_mbps,
+				uclk_dpm_level);
 
-			urg_bandwidth_available[m][n] = dml_get_return_bandwidth_available(soc, m, n, 0, HostVMEnable, 0, dcfclk_mhz, fclk_mhz, dram_bw_mbps);
+			urg_bandwidth_available[m][n] = dml_get_return_bandwidth_available(soc, m, n, 0, HostVMEnable, 0, dcfclk_mhz, fclk_mhz, dram_bw_mbps, uclk_dpm_level);
 
 
 #ifdef __DML_VBA_DEBUG__
@@ -2824,8 +2835,8 @@ static noinline_for_stack void calculate_bandwidth_available(
 
 			// urg_bandwidth_available_vm_only is indexed by soc_state
 			if (n == dml2_core_internal_bw_dram) {
-				urg_bandwidth_available_vm_only[m] = dml_get_return_bandwidth_available(soc, m, n, 0, HostVMEnable, 1, dcfclk_mhz, fclk_mhz, dram_bw_mbps);
-				urg_bandwidth_available_pixel_and_vm[m] = dml_get_return_bandwidth_available(soc, m, n, 0, HostVMEnable, 0, dcfclk_mhz, fclk_mhz, dram_bw_mbps);
+				urg_bandwidth_available_vm_only[m] = dml_get_return_bandwidth_available(soc, m, n, 0, HostVMEnable, 1, dcfclk_mhz, fclk_mhz, dram_bw_mbps, uclk_dpm_level);
+				urg_bandwidth_available_pixel_and_vm[m] = dml_get_return_bandwidth_available(soc, m, n, 0, HostVMEnable, 0, dcfclk_mhz, fclk_mhz, dram_bw_mbps, uclk_dpm_level);
 			}
 		}
 
@@ -9483,7 +9494,8 @@ static bool dml_core_mode_support(struct dml2_core_calcs_mode_support_ex *in_out
 		display_cfg->hostvm_enable,
 		mode_lib->ms.DCFCLK,
 		mode_lib->ms.FabricClock,
-		mode_lib->ms.dram_bw_mbps);
+		mode_lib->ms.dram_bw_mbps,
+		mode_lib->ms.active_min_uclk_dpm_index);
 
 	calculate_bandwidth_available(
 		mode_lib->ms.support.avg_bandwidth_available_min,
@@ -9498,10 +9510,12 @@ static bool dml_core_mode_support(struct dml2_core_calcs_mode_support_ex *in_out
 		mode_lib->ms.MaxDCFCLK,
 		mode_lib->ms.MaxFabricClock,
 #ifdef DML_MODE_SUPPORT_USE_DPM_DRAM_BW
-		mode_lib->ms.dram_bw_mbps);
+		mode_lib->ms.dram_bw_mbps,
 #else
-		mode_lib->ms.max_dram_bw_mbps);
+		mode_lib->ms.max_dram_bw_mbps,
 #endif
+		mode_lib->ms.active_min_uclk_dpm_index);
+
 
 	// Average BW support check
 	calculate_avg_bandwidth_required(
@@ -10958,7 +10972,8 @@ static bool dml_core_mode_programming(struct dml2_core_calcs_mode_programming_ex
 		display_cfg->hostvm_enable,
 		mode_lib->mp.Dcfclk,
 		mode_lib->mp.FabricClock,
-		mode_lib->mp.dram_bw_mbps);
+		mode_lib->mp.dram_bw_mbps,
+		mode_lib->mp.active_min_uclk_dpm_index);
 
 
 	calculate_hostvm_inefficiency_factor(
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_factory.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_factory.c
index 67e307fa43108..9f1222f5a8354 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_factory.c
@@ -15,8 +15,6 @@ bool dml2_core_create(enum dml2_project_id project_id, struct dml2_core_instance
 
 	memset(out, 0, sizeof(struct dml2_core_instance));
 
-	out->project_id = project_id;
-
 	switch (project_id) {
 	case dml2_project_dcn4x_stage1:
 		result = false;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_shared_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_shared_types.h
index 11e295253f72e..e9f970794488c 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_shared_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_shared_types.h
@@ -2329,7 +2329,6 @@ struct dml2_core_calcs_mode_support_ex {
 	const struct dml2_display_cfg *in_display_cfg;
 	const struct dml2_mcg_min_clock_table *min_clk_table;
 	int min_clk_index;
-	enum dml2_project_id project_id;
 	//unsigned int in_state_index;
 	struct dml2_core_internal_mode_support_info *out_evaluation_info;
 };
@@ -2342,7 +2341,6 @@ struct dml2_core_calcs_mode_programming_ex {
 	const struct dml2_mcg_min_clock_table *min_clk_table;
 	const struct core_display_cfg_support_info *cfg_support_info;
 	int min_clk_index;
-	enum dml2_project_id project_id;
 	struct dml2_display_cfg_programming *programming;
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/inc/dml2_internal_shared_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/inc/dml2_internal_shared_types.h
index d328d92240b4e..3ae817ea2aadb 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/inc/dml2_internal_shared_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/inc/dml2_internal_shared_types.h
@@ -489,7 +489,6 @@ struct dml2_core_scratch {
 };
 
 struct dml2_core_instance {
-	enum dml2_project_id project_id;
 	struct dml2_mcg_min_clock_table *minimum_clock_table;
 	struct dml2_core_internal_state_inputs inputs;
 	struct dml2_core_internal_state_intermediates intermediates;
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp_cm.c
index 53b21adc6267d..9788628cf0ad1 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp_cm.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp_cm.c
@@ -397,8 +397,6 @@ void dpp1_cm_program_regamma_lut(struct dpp *dpp_base,
 	uint32_t i;
 	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
 
-	REG_SEQ_START();
-
 	for (i = 0 ; i < num; i++) {
 		REG_SET(CM_RGAM_LUT_DATA, 0, CM_RGAM_LUT_DATA, rgb[i].red_reg);
 		REG_SET(CM_RGAM_LUT_DATA, 0, CM_RGAM_LUT_DATA, rgb[i].green_reg);
@@ -408,9 +406,6 @@ void dpp1_cm_program_regamma_lut(struct dpp *dpp_base,
 		REG_SET(CM_RGAM_LUT_DATA, 0, CM_RGAM_LUT_DATA, rgb[i].delta_green_reg);
 		REG_SET(CM_RGAM_LUT_DATA, 0, CM_RGAM_LUT_DATA, rgb[i].delta_blue_reg);
 	}
-
-	REG_SEQ_SUBMIT();
-	REG_SEQ_WAIT_DONE();
 }
 
 void dpp1_cm_configure_regamma_lut(
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c
index 62994aecf499a..e0ee00c955bc3 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c
@@ -963,14 +963,13 @@ static void dpp401_dscl_program_isharp(struct dpp *dpp_base,
 
 	PERF_TRACE();
 	/*power on isharp_delta_mem first*/
-	if (dpp_base->ctx->dc->caps.ips_v2_support) {
-		/*HW default is LS, need to wake up*/
-		REG_UPDATE_2(ISHARP_DELTA_LUT_MEM_PWR_CTRL,
-					ISHARP_DELTA_LUT_MEM_PWR_FORCE, 0,
-					ISHARP_DELTA_LUT_MEM_PWR_DIS, 1);
-		REG_WAIT(ISHARP_DELTA_LUT_MEM_PWR_CTRL,
-			ISHARP_DELTA_LUT_MEM_PWR_STATE, 0, 1, 100);
-	}
+	REG_UPDATE_2(ISHARP_DELTA_LUT_MEM_PWR_CTRL,
+		     ISHARP_DELTA_LUT_MEM_PWR_FORCE, 0,
+		     ISHARP_DELTA_LUT_MEM_PWR_DIS, 1);
+
+	REG_WAIT(ISHARP_DELTA_LUT_MEM_PWR_CTRL,
+		ISHARP_DELTA_LUT_MEM_PWR_STATE, 0, 1, 100);
+
 	/* ISHARP_MODE */
 	REG_SET_6(ISHARP_MODE, 0,
 		ISHARP_EN, scl_data->dscl_prog_data.isharp_en,
@@ -1049,12 +1048,10 @@ static void dpp401_dscl_program_isharp(struct dpp *dpp_base,
 	}
 
 	/*power on isharp_delta_mem first*/
-	if (dpp_base->ctx->dc->caps.ips_v2_support) {
-		/*HW default is LS, need to wake up*/
-		REG_UPDATE_SEQ_2(ISHARP_DELTA_LUT_MEM_PWR_CTRL,
-					ISHARP_DELTA_LUT_MEM_PWR_FORCE, 0,
-					ISHARP_DELTA_LUT_MEM_PWR_DIS, 0);
-	}
+	REG_UPDATE_SEQ_2(ISHARP_DELTA_LUT_MEM_PWR_CTRL,
+			 ISHARP_DELTA_LUT_MEM_PWR_FORCE, 0,
+			 ISHARP_DELTA_LUT_MEM_PWR_DIS, 0);
+
 	PERF_TRACE();
 } // dpp401_dscl_program_isharp
 /**
diff --git a/drivers/gpu/drm/amd/display/dc/dwb/dcn30/dcn30_cm_common.h b/drivers/gpu/drm/amd/display/dc/dwb/dcn30/dcn30_cm_common.h
index 95f9318a54efc..c23dc1bb29bf8 100644
--- a/drivers/gpu/drm/amd/display/dc/dwb/dcn30/dcn30_cm_common.h
+++ b/drivers/gpu/drm/amd/display/dc/dwb/dcn30/dcn30_cm_common.h
@@ -63,6 +63,10 @@ bool cm3_helper_translate_curve_to_hw_format(struct dc_context *ctx,
 	const struct dc_transfer_func *output_tf,
 	struct pwl_params *lut_params, bool fixpoint);
 
+bool cm3_helper_translate_curve_to_degamma_hw_format(
+				const struct dc_transfer_func *output_tf,
+				struct pwl_params *lut_params);
+
 bool cm3_helper_convert_to_custom_float(
 		struct pwl_result_data *rgb_resulted,
 		struct curve_points3 *corner_points,
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dce80/hw_translate_dce80.c b/drivers/gpu/drm/amd/display/dc/gpio/dce80/hw_translate_dce80.c
index fabb9da504be7..19d148a85f121 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/dce80/hw_translate_dce80.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dce80/hw_translate_dce80.c
@@ -35,7 +35,10 @@
 
 #include "dce/dce_8_0_d.h"
 #include "dce/dce_8_0_sh_mask.h"
-#include "smu/smu_7_0_1_d.h"
+
+#ifndef mmGPIOPAD_A
+#define mmGPIOPAD_A 0x0183
+#endif
 
 /*
  * @brief
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn10/hw_translate_dcn10.c b/drivers/gpu/drm/amd/display/dc/gpio/dcn10/hw_translate_dcn10.c
index fecc8688048d9..000f603def584 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/dcn10/hw_translate_dcn10.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn10/hw_translate_dcn10.c
@@ -58,146 +58,180 @@
 /* macros to expend register list macro defined in HW object header file
  * end *********************/
 
+static const struct gpio_id_offset_entry gpio_offsets[] = {
+	/* GENERIC */
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_A),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_B),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_C),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_D),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_E),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_F),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_G),
+	/* HPD */
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_1),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_2),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_3),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_4),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_5),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_6),
+	/* SYNCA */
+	GPIO_MASK_ENTRY(DC_GPIO_SYNCA_A,
+		DC_GPIO_SYNCA_A__DC_GPIO_HSYNCA_A_MASK,
+		GPIO_ID_SYNC, GPIO_SYNC_HSYNC_A),
+	GPIO_MASK_ENTRY(DC_GPIO_SYNCA_A,
+		DC_GPIO_SYNCA_A__DC_GPIO_VSYNCA_A_MASK,
+		GPIO_ID_SYNC, GPIO_SYNC_VSYNC_A),
+	/* GSL */
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_GENLOCK_CLOCK),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_GENLOCK_VSYNC),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_SWAPLOCK_A),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_SWAPLOCK_B),
+};
+
+/* DDC */
+static const struct gpio_ddc_offset_entry ddc_offset_map[] = {
+	{ REG(DC_GPIO_DDC1_A), GPIO_DDC_LINE_DDC1 },
+	{ REG(DC_GPIO_DDC2_A), GPIO_DDC_LINE_DDC2 },
+	{ REG(DC_GPIO_DDC3_A), GPIO_DDC_LINE_DDC3 },
+	{ REG(DC_GPIO_DDC4_A), GPIO_DDC_LINE_DDC4 },
+	{ REG(DC_GPIO_DDC5_A), GPIO_DDC_LINE_DDC5 },
+	{ REG(DC_GPIO_DDC6_A), GPIO_DDC_LINE_DDC6 },
+	{ REG(DC_GPIO_DDCVGA_A), GPIO_DDC_LINE_DDC_VGA },
+	{ REG(DC_GPIO_I2CPAD_A), GPIO_DDC_LINE_I2C_PAD },
+};
+
+static const struct gpio_pin_entry gpio_pins[] = {
+	/* DDC */
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC1,
+		DC_GPIO_DDC1_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC2,
+		DC_GPIO_DDC2_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC3,
+		DC_GPIO_DDC3_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC4,
+		DC_GPIO_DDC4_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC5,
+		DC_GPIO_DDC5_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC6,
+		DC_GPIO_DDC6_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC_VGA,
+		DC_GPIO_DDCVGA_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_I2C_PAD,
+		DC_GPIO_I2CPAD_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC1,
+		DC_GPIO_DDC1_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC2,
+		DC_GPIO_DDC2_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC3,
+		DC_GPIO_DDC3_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC4,
+		DC_GPIO_DDC4_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC5,
+		DC_GPIO_DDC5_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC6,
+		DC_GPIO_DDC6_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC_VGA,
+		DC_GPIO_DDCVGA_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_I2C_PAD,
+		   DC_GPIO_I2CPAD_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	/* GENERIC */
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_B,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_C,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_D,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_E,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_F,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_G,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK),
+	/* HPD */
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_1,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_2,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_3,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_4,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_5,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_6,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK),
+	/* SYNCA */
+	GPIO_PIN_ENTRY(GPIO_ID_SYNC, GPIO_SYNC_HSYNC_A,
+		DC_GPIO_SYNCA_A, DC_GPIO_SYNCA_A__DC_GPIO_HSYNCA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_SYNC, GPIO_SYNC_VSYNC_A,
+		DC_GPIO_SYNCA_A, DC_GPIO_SYNCA_A__DC_GPIO_VSYNCA_A_MASK),
+	/* GSL */
+	GPIO_PIN_ENTRY(GPIO_ID_GSL, GPIO_GSL_GENLOCK_CLOCK,
+		DC_GPIO_GENLK_A, DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GSL, GPIO_GSL_GENLOCK_VSYNC,
+		DC_GPIO_GENLK_A, DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GSL, GPIO_GSL_SWAPLOCK_A,
+		DC_GPIO_GENLK_A, DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GSL, GPIO_GSL_SWAPLOCK_B,
+		DC_GPIO_GENLK_A, DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK),
+};
+
 static bool offset_to_id(
 	uint32_t offset,
 	uint32_t mask,
 	enum gpio_id *id,
 	uint32_t *en)
 {
-	switch (offset) {
-	/* GENERIC */
-	case REG(DC_GPIO_GENERIC_A):
-		*id = GPIO_ID_GENERIC;
-		switch (mask) {
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK:
-			*en = GPIO_GENERIC_A;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK:
-			*en = GPIO_GENERIC_B;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK:
-			*en = GPIO_GENERIC_C;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK:
-			*en = GPIO_GENERIC_D;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK:
-			*en = GPIO_GENERIC_E;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK:
-			*en = GPIO_GENERIC_F;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK:
-			*en = GPIO_GENERIC_G;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* HPD */
-	case REG(DC_GPIO_HPD_A):
-		*id = GPIO_ID_HPD;
-		switch (mask) {
-		case DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK:
-			*en = GPIO_HPD_1;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK:
-			*en = GPIO_HPD_2;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK:
-			*en = GPIO_HPD_3;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK:
-			*en = GPIO_HPD_4;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK:
-			*en = GPIO_HPD_5;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK:
-			*en = GPIO_HPD_6;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* SYNCA */
-	case REG(DC_GPIO_SYNCA_A):
-		*id = GPIO_ID_SYNC;
-		switch (mask) {
-		case DC_GPIO_SYNCA_A__DC_GPIO_HSYNCA_A_MASK:
-			*en = GPIO_SYNC_HSYNC_A;
-			return true;
-		case DC_GPIO_SYNCA_A__DC_GPIO_VSYNCA_A_MASK:
-			*en = GPIO_SYNC_VSYNC_A;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* REG(DC_GPIO_GENLK_MASK */
-	case REG(DC_GPIO_GENLK_A):
-		*id = GPIO_ID_GSL;
-		switch (mask) {
-		case DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK:
-			*en = GPIO_GSL_GENLOCK_CLOCK;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK:
-			*en = GPIO_GSL_GENLOCK_VSYNC;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK:
-			*en = GPIO_GSL_SWAPLOCK_A;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK:
-			*en = GPIO_GSL_SWAPLOCK_B;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* DDC */
-	/* we don't care about the GPIO_ID for DDC
-	 * in DdcHandle it will use GPIO_ID_DDC_DATA/GPIO_ID_DDC_CLOCK
-	 * directly in the create method */
-	case REG(DC_GPIO_DDC1_A):
-		*en = GPIO_DDC_LINE_DDC1;
-		return true;
-	case REG(DC_GPIO_DDC2_A):
-		*en = GPIO_DDC_LINE_DDC2;
-		return true;
-	case REG(DC_GPIO_DDC3_A):
-		*en = GPIO_DDC_LINE_DDC3;
-		return true;
-	case REG(DC_GPIO_DDC4_A):
-		*en = GPIO_DDC_LINE_DDC4;
-		return true;
-	case REG(DC_GPIO_DDC5_A):
-		*en = GPIO_DDC_LINE_DDC5;
-		return true;
-	case REG(DC_GPIO_DDC6_A):
-		*en = GPIO_DDC_LINE_DDC6;
+	if (dal_hw_translate_gpio_ddc_offset_to_id(
+			ddc_offset_map,
+			ARRAY_SIZE(ddc_offset_map),
+			offset, en))
 		return true;
-	case REG(DC_GPIO_DDCVGA_A):
-		*en = GPIO_DDC_LINE_DDC_VGA;
-		return true;
-	/* GPIO_I2CPAD */
-	case REG(DC_GPIO_I2CPAD_A):
-		*en = GPIO_DDC_LINE_I2C_PAD;
+
+	if (dal_hw_translate_gpio_offset_to_id(
+			gpio_offsets,
+			ARRAY_SIZE(gpio_offsets),
+			offset, mask, id, en))
 		return true;
-	/* Not implemented */
-	case REG(DC_GPIO_PWRSEQ_A):
-	case REG(DC_GPIO_PAD_STRENGTH_1):
-	case REG(DC_GPIO_PAD_STRENGTH_2):
-	case REG(DC_GPIO_DEBUG):
-		return false;
-	/* UNEXPECTED */
-	default:
-		ASSERT_CRITICAL(false);
-		return false;
-	}
+
+	ASSERT_CRITICAL(false);
+	return false;
 }
 
 static bool id_to_offset(
@@ -205,186 +239,14 @@ static bool id_to_offset(
 	uint32_t en,
 	struct gpio_pin_info *info)
 {
-	bool result = true;
-
-	switch (id) {
-	case GPIO_ID_DDC_DATA:
-		info->mask = DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK;
-		switch (en) {
-		case GPIO_DDC_LINE_DDC1:
-			info->offset = REG(DC_GPIO_DDC1_A);
-		break;
-		case GPIO_DDC_LINE_DDC2:
-			info->offset = REG(DC_GPIO_DDC2_A);
-		break;
-		case GPIO_DDC_LINE_DDC3:
-			info->offset = REG(DC_GPIO_DDC3_A);
-		break;
-		case GPIO_DDC_LINE_DDC4:
-			info->offset = REG(DC_GPIO_DDC4_A);
-		break;
-		case GPIO_DDC_LINE_DDC5:
-			info->offset = REG(DC_GPIO_DDC5_A);
-		break;
-		case GPIO_DDC_LINE_DDC6:
-			info->offset = REG(DC_GPIO_DDC6_A);
-		break;
-		case GPIO_DDC_LINE_DDC_VGA:
-			info->offset = REG(DC_GPIO_DDCVGA_A);
-		break;
-		case GPIO_DDC_LINE_I2C_PAD:
-			info->offset = REG(DC_GPIO_I2CPAD_A);
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_DDC_CLOCK:
-		info->mask = DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK;
-		switch (en) {
-		case GPIO_DDC_LINE_DDC1:
-			info->offset = REG(DC_GPIO_DDC1_A);
-		break;
-		case GPIO_DDC_LINE_DDC2:
-			info->offset = REG(DC_GPIO_DDC2_A);
-		break;
-		case GPIO_DDC_LINE_DDC3:
-			info->offset = REG(DC_GPIO_DDC3_A);
-		break;
-		case GPIO_DDC_LINE_DDC4:
-			info->offset = REG(DC_GPIO_DDC4_A);
-		break;
-		case GPIO_DDC_LINE_DDC5:
-			info->offset = REG(DC_GPIO_DDC5_A);
-		break;
-		case GPIO_DDC_LINE_DDC6:
-			info->offset = REG(DC_GPIO_DDC6_A);
-		break;
-		case GPIO_DDC_LINE_DDC_VGA:
-			info->offset = REG(DC_GPIO_DDCVGA_A);
-		break;
-		case GPIO_DDC_LINE_I2C_PAD:
-			info->offset = REG(DC_GPIO_I2CPAD_A);
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_GENERIC:
-		info->offset = REG(DC_GPIO_GENERIC_A);
-		switch (en) {
-		case GPIO_GENERIC_A:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK;
-		break;
-		case GPIO_GENERIC_B:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK;
-		break;
-		case GPIO_GENERIC_C:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK;
-		break;
-		case GPIO_GENERIC_D:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK;
-		break;
-		case GPIO_GENERIC_E:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK;
-		break;
-		case GPIO_GENERIC_F:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK;
-		break;
-		case GPIO_GENERIC_G:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK;
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_HPD:
-		info->offset = REG(DC_GPIO_HPD_A);
-		switch (en) {
-		case GPIO_HPD_1:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK;
-		break;
-		case GPIO_HPD_2:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK;
-		break;
-		case GPIO_HPD_3:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK;
-		break;
-		case GPIO_HPD_4:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK;
-		break;
-		case GPIO_HPD_5:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK;
-		break;
-		case GPIO_HPD_6:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK;
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_SYNC:
-		switch (en) {
-		case GPIO_SYNC_HSYNC_A:
-			info->offset = REG(DC_GPIO_SYNCA_A);
-			info->mask = DC_GPIO_SYNCA_A__DC_GPIO_HSYNCA_A_MASK;
-		break;
-		case GPIO_SYNC_VSYNC_A:
-			info->offset = REG(DC_GPIO_SYNCA_A);
-			info->mask = DC_GPIO_SYNCA_A__DC_GPIO_VSYNCA_A_MASK;
-		break;
-		case GPIO_SYNC_HSYNC_B:
-		case GPIO_SYNC_VSYNC_B:
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_GSL:
-		switch (en) {
-		case GPIO_GSL_GENLOCK_CLOCK:
-			info->offset = REG(DC_GPIO_GENLK_A);
-			info->mask = DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK;
-		break;
-		case GPIO_GSL_GENLOCK_VSYNC:
-			info->offset = REG(DC_GPIO_GENLK_A);
-			info->mask =
-				DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK;
-		break;
-		case GPIO_GSL_SWAPLOCK_A:
-			info->offset = REG(DC_GPIO_GENLK_A);
-			info->mask = DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK;
-		break;
-		case GPIO_GSL_SWAPLOCK_B:
-			info->offset = REG(DC_GPIO_GENLK_A);
-			info->mask = DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK;
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_VIP_PAD:
-	default:
-		ASSERT_CRITICAL(false);
-		result = false;
-	}
-
-	if (result) {
-		info->offset_y = info->offset + 2;
-		info->offset_en = info->offset + 1;
-		info->offset_mask = info->offset - 1;
-
-		info->mask_y = info->mask;
-		info->mask_en = info->mask;
-		info->mask_mask = info->mask;
-	}
+	if (dal_hw_translate_id_to_offset(
+			gpio_pins,
+			ARRAY_SIZE(gpio_pins),
+			id, en, info))
+		return true;
 
-	return result;
+	ASSERT_CRITICAL(false);
+	return false;
 }
 
 /* function table */
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_translate_dcn20.c b/drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_translate_dcn20.c
index 3005ee7751a03..a21df86682665 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_translate_dcn20.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn20/hw_translate_dcn20.c
@@ -62,131 +62,161 @@
  * end *********************/
 
 
+static const struct gpio_id_offset_entry gpio_offsets[] = {
+	/* GENERIC */
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_A),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_B),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_C),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_D),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_E),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_F),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_G),
+	/* HPD */
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_1),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_2),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_3),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_4),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_5),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_6),
+	/* GSL */
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_GENLOCK_CLOCK),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_GENLOCK_VSYNC),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_SWAPLOCK_A),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_SWAPLOCK_B),
+};
+
+
+/* DDC */
+static const struct gpio_ddc_offset_entry ddc_offset_map[] = {
+	{ REG(DC_GPIO_DDC1_A), GPIO_DDC_LINE_DDC1 },
+	{ REG(DC_GPIO_DDC2_A), GPIO_DDC_LINE_DDC2 },
+	{ REG(DC_GPIO_DDC3_A), GPIO_DDC_LINE_DDC3 },
+	{ REG(DC_GPIO_DDC4_A), GPIO_DDC_LINE_DDC4 },
+	{ REG(DC_GPIO_DDC5_A), GPIO_DDC_LINE_DDC5 },
+	{ REG(DC_GPIO_DDC6_A), GPIO_DDC_LINE_DDC6 },
+	{ REG(DC_GPIO_DDCVGA_A), GPIO_DDC_LINE_DDC_VGA },
+};
+
+
+/*
+ * GSL is intentionally omitted here.
+ * id_to_offset() for GSL is not implemented on this ASIC.
+ */
+static const struct gpio_pin_entry gpio_pins[] = {
+	/* DDC */
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC1,
+		DC_GPIO_DDC1_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC2,
+		DC_GPIO_DDC2_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC3,
+		DC_GPIO_DDC3_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC4,
+		DC_GPIO_DDC4_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC5,
+		DC_GPIO_DDC5_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC6,
+		DC_GPIO_DDC6_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC_VGA,
+		DC_GPIO_DDCVGA_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC1,
+		DC_GPIO_DDC1_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC2,
+		DC_GPIO_DDC2_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC3,
+		DC_GPIO_DDC3_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC4,
+		DC_GPIO_DDC4_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC5,
+		DC_GPIO_DDC5_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC6,
+		DC_GPIO_DDC6_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC_VGA,
+		DC_GPIO_DDCVGA_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	/* GENERIC */
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_B,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_C,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_D,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_E,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_F,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_G,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK),
+	/* HPD */
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_1,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_2,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_3,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_4,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_5,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_6,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK),
+};
+
+
 static bool offset_to_id(
 	uint32_t offset,
 	uint32_t mask,
 	enum gpio_id *id,
 	uint32_t *en)
 {
-	switch (offset) {
-	/* GENERIC */
-	case REG(DC_GPIO_GENERIC_A):
-		*id = GPIO_ID_GENERIC;
-		switch (mask) {
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK:
-			*en = GPIO_GENERIC_A;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK:
-			*en = GPIO_GENERIC_B;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK:
-			*en = GPIO_GENERIC_C;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK:
-			*en = GPIO_GENERIC_D;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK:
-			*en = GPIO_GENERIC_E;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK:
-			*en = GPIO_GENERIC_F;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK:
-			*en = GPIO_GENERIC_G;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* HPD */
-	case REG(DC_GPIO_HPD_A):
-		*id = GPIO_ID_HPD;
-		switch (mask) {
-		case DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK:
-			*en = GPIO_HPD_1;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK:
-			*en = GPIO_HPD_2;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK:
-			*en = GPIO_HPD_3;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK:
-			*en = GPIO_HPD_4;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK:
-			*en = GPIO_HPD_5;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK:
-			*en = GPIO_HPD_6;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* REG(DC_GPIO_GENLK_MASK */
-	case REG(DC_GPIO_GENLK_A):
-		*id = GPIO_ID_GSL;
-		switch (mask) {
-		case DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK:
-			*en = GPIO_GSL_GENLOCK_CLOCK;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK:
-			*en = GPIO_GSL_GENLOCK_VSYNC;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK:
-			*en = GPIO_GSL_SWAPLOCK_A;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK:
-			*en = GPIO_GSL_SWAPLOCK_B;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* DDC */
-	/* we don't care about the GPIO_ID for DDC
-	 * in DdcHandle it will use GPIO_ID_DDC_DATA/GPIO_ID_DDC_CLOCK
-	 * directly in the create method
-	 */
-	case REG(DC_GPIO_DDC1_A):
-		*en = GPIO_DDC_LINE_DDC1;
-		return true;
-	case REG(DC_GPIO_DDC2_A):
-		*en = GPIO_DDC_LINE_DDC2;
-		return true;
-	case REG(DC_GPIO_DDC3_A):
-		*en = GPIO_DDC_LINE_DDC3;
-		return true;
-	case REG(DC_GPIO_DDC4_A):
-		*en = GPIO_DDC_LINE_DDC4;
-		return true;
-	case REG(DC_GPIO_DDC5_A):
-		*en = GPIO_DDC_LINE_DDC5;
-		return true;
-	case REG(DC_GPIO_DDC6_A):
-		*en = GPIO_DDC_LINE_DDC6;
+	if (dal_hw_translate_gpio_ddc_offset_to_id(
+			ddc_offset_map,
+			ARRAY_SIZE(ddc_offset_map),
+			offset, en))
 		return true;
-	case REG(DC_GPIO_DDCVGA_A):
-		*en = GPIO_DDC_LINE_DDC_VGA;
+
+	if (dal_hw_translate_gpio_offset_to_id(
+			gpio_offsets,
+			ARRAY_SIZE(gpio_offsets),
+			offset, mask, id, en))
 		return true;
 
-/*
- *	case REG(DC_GPIO_I2CPAD_A): not exit
- *	case REG(DC_GPIO_PWRSEQ_A):
- *	case REG(DC_GPIO_PAD_STRENGTH_1):
- *	case REG(DC_GPIO_PAD_STRENGTH_2):
- *	case REG(DC_GPIO_DEBUG):
- */
-	/* UNEXPECTED */
-	default:
-/*	case REG(DC_GPIO_SYNCA_A): not exist */
-		ASSERT_CRITICAL(false);
-		return false;
-	}
+	ASSERT_CRITICAL(false);
+	return false;
 }
 
 static bool id_to_offset(
@@ -194,170 +224,14 @@ static bool id_to_offset(
 	uint32_t en,
 	struct gpio_pin_info *info)
 {
-	bool result = true;
-
-	switch (id) {
-	case GPIO_ID_DDC_DATA:
-		info->mask = DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK;
-		switch (en) {
-		case GPIO_DDC_LINE_DDC1:
-			info->offset = REG(DC_GPIO_DDC1_A);
-		break;
-		case GPIO_DDC_LINE_DDC2:
-			info->offset = REG(DC_GPIO_DDC2_A);
-		break;
-		case GPIO_DDC_LINE_DDC3:
-			info->offset = REG(DC_GPIO_DDC3_A);
-		break;
-		case GPIO_DDC_LINE_DDC4:
-			info->offset = REG(DC_GPIO_DDC4_A);
-		break;
-		case GPIO_DDC_LINE_DDC5:
-			info->offset = REG(DC_GPIO_DDC5_A);
-		break;
-		case GPIO_DDC_LINE_DDC6:
-			info->offset = REG(DC_GPIO_DDC6_A);
-		break;
-		case GPIO_DDC_LINE_DDC_VGA:
-			info->offset = REG(DC_GPIO_DDCVGA_A);
-		break;
-		case GPIO_DDC_LINE_I2C_PAD:
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_DDC_CLOCK:
-		info->mask = DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK;
-		switch (en) {
-		case GPIO_DDC_LINE_DDC1:
-			info->offset = REG(DC_GPIO_DDC1_A);
-		break;
-		case GPIO_DDC_LINE_DDC2:
-			info->offset = REG(DC_GPIO_DDC2_A);
-		break;
-		case GPIO_DDC_LINE_DDC3:
-			info->offset = REG(DC_GPIO_DDC3_A);
-		break;
-		case GPIO_DDC_LINE_DDC4:
-			info->offset = REG(DC_GPIO_DDC4_A);
-		break;
-		case GPIO_DDC_LINE_DDC5:
-			info->offset = REG(DC_GPIO_DDC5_A);
-		break;
-		case GPIO_DDC_LINE_DDC6:
-			info->offset = REG(DC_GPIO_DDC6_A);
-		break;
-		case GPIO_DDC_LINE_DDC_VGA:
-			info->offset = REG(DC_GPIO_DDCVGA_A);
-		break;
-		case GPIO_DDC_LINE_I2C_PAD:
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_GENERIC:
-		info->offset = REG(DC_GPIO_GENERIC_A);
-		switch (en) {
-		case GPIO_GENERIC_A:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK;
-		break;
-		case GPIO_GENERIC_B:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK;
-		break;
-		case GPIO_GENERIC_C:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK;
-		break;
-		case GPIO_GENERIC_D:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK;
-		break;
-		case GPIO_GENERIC_E:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK;
-		break;
-		case GPIO_GENERIC_F:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK;
-		break;
-		case GPIO_GENERIC_G:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK;
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_HPD:
-		info->offset = REG(DC_GPIO_HPD_A);
-		switch (en) {
-		case GPIO_HPD_1:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK;
-		break;
-		case GPIO_HPD_2:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK;
-		break;
-		case GPIO_HPD_3:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK;
-		break;
-		case GPIO_HPD_4:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK;
-		break;
-		case GPIO_HPD_5:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK;
-		break;
-		case GPIO_HPD_6:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK;
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_GSL:
-		switch (en) {
-		case GPIO_GSL_GENLOCK_CLOCK:
-				/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_GENLOCK_VSYNC:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_SWAPLOCK_A:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_SWAPLOCK_B:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_SYNC:
-	case GPIO_ID_VIP_PAD:
-	default:
-		ASSERT_CRITICAL(false);
-		result = false;
-	}
-
-	if (result) {
-		info->offset_y = info->offset + 2;
-		info->offset_en = info->offset + 1;
-		info->offset_mask = info->offset - 1;
-
-		info->mask_y = info->mask;
-		info->mask_en = info->mask;
-		info->mask_mask = info->mask;
-	}
+	if (dal_hw_translate_id_to_offset(
+			gpio_pins,
+			ARRAY_SIZE(gpio_pins),
+			id, en, info))
+		return true;
 
-	return result;
+	ASSERT_CRITICAL(false);
+	return false;
 }
 
 /* function table */
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn21/hw_translate_dcn21.c b/drivers/gpu/drm/amd/display/dc/gpio/dcn21/hw_translate_dcn21.c
index e3b11b3c1daa5..18bd4d4e32d03 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/dcn21/hw_translate_dcn21.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn21/hw_translate_dcn21.c
@@ -60,6 +60,135 @@
 /* macros to expend register list macro defined in HW object header file
  * end *********************/
 
+static const struct gpio_id_offset_entry gpio_offsets[] = {
+	/* GENERIC */
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_A),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_B),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_C),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_D),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_E),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_F),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_G),
+	/* HPD */
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_1),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_2),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_3),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_4),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_5),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_6),
+	/* GSL */
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_GENLOCK_CLOCK),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_GENLOCK_VSYNC),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_SWAPLOCK_A),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_SWAPLOCK_B),
+};
+
+
+/* DDC */
+static const struct gpio_ddc_offset_entry ddc_offset_map[] = {
+	{ REG(DC_GPIO_DDC1_A), GPIO_DDC_LINE_DDC1 },
+	{ REG(DC_GPIO_DDC2_A), GPIO_DDC_LINE_DDC2 },
+	{ REG(DC_GPIO_DDC3_A), GPIO_DDC_LINE_DDC3 },
+	{ REG(DC_GPIO_DDC4_A), GPIO_DDC_LINE_DDC4 },
+	{ REG(DC_GPIO_DDC5_A), GPIO_DDC_LINE_DDC5 },
+	{ REG(DC_GPIO_DDCVGA_A), GPIO_DDC_LINE_DDC_VGA },
+};
+
+
+/*
+ * GSL is intentionally omitted here.
+ * id_to_offset() for GSL is not implemented on this ASIC.
+ */
+static const struct gpio_pin_entry gpio_pins[] = {
+	/* DDC */
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC1,
+		DC_GPIO_DDC1_A, DC_GPIO_DDC5_A__DC_GPIO_DDC5DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC2,
+		DC_GPIO_DDC2_A, DC_GPIO_DDC5_A__DC_GPIO_DDC5DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC3,
+		DC_GPIO_DDC3_A, DC_GPIO_DDC5_A__DC_GPIO_DDC5DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC4,
+		DC_GPIO_DDC4_A, DC_GPIO_DDC5_A__DC_GPIO_DDC5DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC5,
+		DC_GPIO_DDC5_A, DC_GPIO_DDC5_A__DC_GPIO_DDC5DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC_VGA,
+		DC_GPIO_DDCVGA_A, DC_GPIO_DDC5_A__DC_GPIO_DDC5DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC1,
+		DC_GPIO_DDC1_A, DC_GPIO_DDC5_A__DC_GPIO_DDC5CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC2,
+		DC_GPIO_DDC2_A, DC_GPIO_DDC5_A__DC_GPIO_DDC5CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC3,
+		DC_GPIO_DDC3_A, DC_GPIO_DDC5_A__DC_GPIO_DDC5CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC4,
+		DC_GPIO_DDC4_A, DC_GPIO_DDC5_A__DC_GPIO_DDC5CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC5,
+		DC_GPIO_DDC5_A, DC_GPIO_DDC5_A__DC_GPIO_DDC5CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC_VGA,
+		DC_GPIO_DDCVGA_A, DC_GPIO_DDC5_A__DC_GPIO_DDC5CLK_A_MASK),
+	/* GENERIC */
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_B,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_C,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_D,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_E,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_F,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_G,
+		   DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK),
+	/* HPD */
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_1,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_2,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_3,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_4,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_5,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_6,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK),
+};
+
 
 static bool offset_to_id(
 	uint32_t offset,
@@ -67,122 +196,20 @@ static bool offset_to_id(
 	enum gpio_id *id,
 	uint32_t *en)
 {
-	switch (offset) {
-	/* GENERIC */
-	case REG(DC_GPIO_GENERIC_A):
-		*id = GPIO_ID_GENERIC;
-		switch (mask) {
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK:
-			*en = GPIO_GENERIC_A;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK:
-			*en = GPIO_GENERIC_B;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK:
-			*en = GPIO_GENERIC_C;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK:
-			*en = GPIO_GENERIC_D;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK:
-			*en = GPIO_GENERIC_E;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK:
-			*en = GPIO_GENERIC_F;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK:
-			*en = GPIO_GENERIC_G;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* HPD */
-	case REG(DC_GPIO_HPD_A):
-		*id = GPIO_ID_HPD;
-		switch (mask) {
-		case DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK:
-			*en = GPIO_HPD_1;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK:
-			*en = GPIO_HPD_2;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK:
-			*en = GPIO_HPD_3;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK:
-			*en = GPIO_HPD_4;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK:
-			*en = GPIO_HPD_5;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK:
-			*en = GPIO_HPD_6;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* REG(DC_GPIO_GENLK_MASK */
-	case REG(DC_GPIO_GENLK_A):
-		*id = GPIO_ID_GSL;
-		switch (mask) {
-		case DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK:
-			*en = GPIO_GSL_GENLOCK_CLOCK;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK:
-			*en = GPIO_GSL_GENLOCK_VSYNC;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK:
-			*en = GPIO_GSL_SWAPLOCK_A;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK:
-			*en = GPIO_GSL_SWAPLOCK_B;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* DDC */
-	/* we don't care about the GPIO_ID for DDC
-	 * in DdcHandle it will use GPIO_ID_DDC_DATA/GPIO_ID_DDC_CLOCK
-	 * directly in the create method
-	 */
-	case REG(DC_GPIO_DDC1_A):
-		*en = GPIO_DDC_LINE_DDC1;
-		return true;
-	case REG(DC_GPIO_DDC2_A):
-		*en = GPIO_DDC_LINE_DDC2;
+	if (dal_hw_translate_gpio_ddc_offset_to_id(
+			ddc_offset_map,
+			ARRAY_SIZE(ddc_offset_map),
+			offset, en))
 		return true;
-	case REG(DC_GPIO_DDC3_A):
-		*en = GPIO_DDC_LINE_DDC3;
-		return true;
-	case REG(DC_GPIO_DDC4_A):
-		*en = GPIO_DDC_LINE_DDC4;
-		return true;
-	case REG(DC_GPIO_DDC5_A):
-		*en = GPIO_DDC_LINE_DDC5;
-		return true;
-	case REG(DC_GPIO_DDCVGA_A):
-		*en = GPIO_DDC_LINE_DDC_VGA;
+
+	if (dal_hw_translate_gpio_offset_to_id(
+			gpio_offsets,
+			ARRAY_SIZE(gpio_offsets),
+			offset, mask, id, en))
 		return true;
 
-/*
- *	case REG(DC_GPIO_I2CPAD_A): not exit
- *	case REG(DC_GPIO_PWRSEQ_A):
- *	case REG(DC_GPIO_PAD_STRENGTH_1):
- *	case REG(DC_GPIO_PAD_STRENGTH_2):
- *	case REG(DC_GPIO_DEBUG):
- */
-	/* UNEXPECTED */
-	default:
-/*	case REG(DC_GPIO_SYNCA_A): not exist */
-		ASSERT_CRITICAL(false);
-		return false;
-	}
+	ASSERT_CRITICAL(false);
+	return false;
 }
 
 static bool id_to_offset(
@@ -190,164 +217,14 @@ static bool id_to_offset(
 	uint32_t en,
 	struct gpio_pin_info *info)
 {
-	bool result = true;
-
-	switch (id) {
-	case GPIO_ID_DDC_DATA:
-		info->mask = DC_GPIO_DDC5_A__DC_GPIO_DDC5DATA_A_MASK;
-		switch (en) {
-		case GPIO_DDC_LINE_DDC1:
-			info->offset = REG(DC_GPIO_DDC1_A);
-		break;
-		case GPIO_DDC_LINE_DDC2:
-			info->offset = REG(DC_GPIO_DDC2_A);
-		break;
-		case GPIO_DDC_LINE_DDC3:
-			info->offset = REG(DC_GPIO_DDC3_A);
-		break;
-		case GPIO_DDC_LINE_DDC4:
-			info->offset = REG(DC_GPIO_DDC4_A);
-		break;
-		case GPIO_DDC_LINE_DDC5:
-			info->offset = REG(DC_GPIO_DDC5_A);
-		break;
-		case GPIO_DDC_LINE_DDC_VGA:
-			info->offset = REG(DC_GPIO_DDCVGA_A);
-		break;
-		case GPIO_DDC_LINE_I2C_PAD:
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_DDC_CLOCK:
-		info->mask = DC_GPIO_DDC5_A__DC_GPIO_DDC5CLK_A_MASK;
-		switch (en) {
-		case GPIO_DDC_LINE_DDC1:
-			info->offset = REG(DC_GPIO_DDC1_A);
-		break;
-		case GPIO_DDC_LINE_DDC2:
-			info->offset = REG(DC_GPIO_DDC2_A);
-		break;
-		case GPIO_DDC_LINE_DDC3:
-			info->offset = REG(DC_GPIO_DDC3_A);
-		break;
-		case GPIO_DDC_LINE_DDC4:
-			info->offset = REG(DC_GPIO_DDC4_A);
-		break;
-		case GPIO_DDC_LINE_DDC5:
-			info->offset = REG(DC_GPIO_DDC5_A);
-		break;
-		case GPIO_DDC_LINE_DDC_VGA:
-			info->offset = REG(DC_GPIO_DDCVGA_A);
-		break;
-		case GPIO_DDC_LINE_I2C_PAD:
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_GENERIC:
-		info->offset = REG(DC_GPIO_GENERIC_A);
-		switch (en) {
-		case GPIO_GENERIC_A:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK;
-		break;
-		case GPIO_GENERIC_B:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK;
-		break;
-		case GPIO_GENERIC_C:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK;
-		break;
-		case GPIO_GENERIC_D:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK;
-		break;
-		case GPIO_GENERIC_E:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK;
-		break;
-		case GPIO_GENERIC_F:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK;
-		break;
-		case GPIO_GENERIC_G:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK;
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_HPD:
-		info->offset = REG(DC_GPIO_HPD_A);
-		switch (en) {
-		case GPIO_HPD_1:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK;
-		break;
-		case GPIO_HPD_2:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK;
-		break;
-		case GPIO_HPD_3:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK;
-		break;
-		case GPIO_HPD_4:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK;
-		break;
-		case GPIO_HPD_5:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK;
-		break;
-		case GPIO_HPD_6:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK;
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_GSL:
-		switch (en) {
-		case GPIO_GSL_GENLOCK_CLOCK:
-				/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_GENLOCK_VSYNC:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_SWAPLOCK_A:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_SWAPLOCK_B:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_SYNC:
-	case GPIO_ID_VIP_PAD:
-	default:
-		ASSERT_CRITICAL(false);
-		result = false;
-	}
-
-	if (result) {
-		info->offset_y = info->offset + 2;
-		info->offset_en = info->offset + 1;
-		info->offset_mask = info->offset - 1;
-
-		info->mask_y = info->mask;
-		info->mask_en = info->mask;
-		info->mask_mask = info->mask;
-	}
+	if (dal_hw_translate_id_to_offset(
+			gpio_pins,
+			ARRAY_SIZE(gpio_pins),
+			id, en, info))
+		return true;
 
-	return result;
+	ASSERT_CRITICAL(false);
+	return false;
 }
 
 /* function table */
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn30/hw_translate_dcn30.c b/drivers/gpu/drm/amd/display/dc/gpio/dcn30/hw_translate_dcn30.c
index 49d6250037a9a..c4225231f7252 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/dcn30/hw_translate_dcn30.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn30/hw_translate_dcn30.c
@@ -67,131 +67,161 @@
  * end *********************/
 
 
+static const struct gpio_id_offset_entry gpio_offsets[] = {
+	/* GENERIC */
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_A),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_B),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_C),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_D),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_E),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_F),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_G),
+	/* HPD */
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_1),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_2),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_3),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_4),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_5),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_6),
+	/* GSL */
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_GENLOCK_CLOCK),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_GENLOCK_VSYNC),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_SWAPLOCK_A),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_SWAPLOCK_B),
+};
+
+
+/* DDC */
+static const struct gpio_ddc_offset_entry ddc_offset_map[] = {
+	{ REG(DC_GPIO_DDC1_A), GPIO_DDC_LINE_DDC1 },
+	{ REG(DC_GPIO_DDC2_A), GPIO_DDC_LINE_DDC2 },
+	{ REG(DC_GPIO_DDC3_A), GPIO_DDC_LINE_DDC3 },
+	{ REG(DC_GPIO_DDC4_A), GPIO_DDC_LINE_DDC4 },
+	{ REG(DC_GPIO_DDC5_A), GPIO_DDC_LINE_DDC5 },
+	{ REG(DC_GPIO_DDC6_A), GPIO_DDC_LINE_DDC6 },
+	{ REG(DC_GPIO_DDCVGA_A), GPIO_DDC_LINE_DDC_VGA },
+};
+
+
+/*
+ * GSL is intentionally omitted here.
+ * id_to_offset() for GSL is not implemented on this ASIC.
+ */
+static const struct gpio_pin_entry gpio_pins[] = {
+	/* DDC */
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC1,
+		DC_GPIO_DDC1_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC2,
+		DC_GPIO_DDC2_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC3,
+		DC_GPIO_DDC3_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC4,
+		DC_GPIO_DDC4_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC5,
+		DC_GPIO_DDC5_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC6,
+		DC_GPIO_DDC6_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC_VGA,
+		DC_GPIO_DDCVGA_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC1,
+		DC_GPIO_DDC1_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC2,
+		DC_GPIO_DDC2_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC3,
+		DC_GPIO_DDC3_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC4,
+		DC_GPIO_DDC4_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC5,
+		DC_GPIO_DDC5_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC6,
+		DC_GPIO_DDC6_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC_VGA,
+		DC_GPIO_DDCVGA_A, DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK),
+	/* GENERIC */
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_B,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_C,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_D,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_E,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_F,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_G,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK),
+	/* HPD */
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_1,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_2,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_3,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_4,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_5,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_6,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK),
+};
+
+
 static bool offset_to_id(
 	uint32_t offset,
 	uint32_t mask,
 	enum gpio_id *id,
 	uint32_t *en)
 {
-	switch (offset) {
-	/* GENERIC */
-	case REG(DC_GPIO_GENERIC_A):
-		*id = GPIO_ID_GENERIC;
-		switch (mask) {
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK:
-			*en = GPIO_GENERIC_A;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK:
-			*en = GPIO_GENERIC_B;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK:
-			*en = GPIO_GENERIC_C;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK:
-			*en = GPIO_GENERIC_D;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK:
-			*en = GPIO_GENERIC_E;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK:
-			*en = GPIO_GENERIC_F;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK:
-			*en = GPIO_GENERIC_G;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* HPD */
-	case REG(DC_GPIO_HPD_A):
-		*id = GPIO_ID_HPD;
-		switch (mask) {
-		case DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK:
-			*en = GPIO_HPD_1;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK:
-			*en = GPIO_HPD_2;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK:
-			*en = GPIO_HPD_3;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK:
-			*en = GPIO_HPD_4;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK:
-			*en = GPIO_HPD_5;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK:
-			*en = GPIO_HPD_6;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* REG(DC_GPIO_GENLK_MASK */
-	case REG(DC_GPIO_GENLK_A):
-		*id = GPIO_ID_GSL;
-		switch (mask) {
-		case DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK:
-			*en = GPIO_GSL_GENLOCK_CLOCK;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK:
-			*en = GPIO_GSL_GENLOCK_VSYNC;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK:
-			*en = GPIO_GSL_SWAPLOCK_A;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK:
-			*en = GPIO_GSL_SWAPLOCK_B;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* DDC */
-	/* we don't care about the GPIO_ID for DDC
-	 * in DdcHandle it will use GPIO_ID_DDC_DATA/GPIO_ID_DDC_CLOCK
-	 * directly in the create method
-	 */
-	case REG(DC_GPIO_DDC1_A):
-		*en = GPIO_DDC_LINE_DDC1;
-		return true;
-	case REG(DC_GPIO_DDC2_A):
-		*en = GPIO_DDC_LINE_DDC2;
-		return true;
-	case REG(DC_GPIO_DDC3_A):
-		*en = GPIO_DDC_LINE_DDC3;
-		return true;
-	case REG(DC_GPIO_DDC4_A):
-		*en = GPIO_DDC_LINE_DDC4;
-		return true;
-	case REG(DC_GPIO_DDC5_A):
-		*en = GPIO_DDC_LINE_DDC5;
-		return true;
-	case REG(DC_GPIO_DDC6_A):
-		*en = GPIO_DDC_LINE_DDC6;
+	if (dal_hw_translate_gpio_ddc_offset_to_id(
+			ddc_offset_map,
+			ARRAY_SIZE(ddc_offset_map),
+			offset, en))
 		return true;
-	case REG(DC_GPIO_DDCVGA_A):
-		*en = GPIO_DDC_LINE_DDC_VGA;
+
+	if (dal_hw_translate_gpio_offset_to_id(
+			gpio_offsets,
+			ARRAY_SIZE(gpio_offsets),
+			offset, mask, id, en))
 		return true;
 
-/*
- *	case REG(DC_GPIO_I2CPAD_A): not exit
- *	case REG(DC_GPIO_PWRSEQ_A):
- *	case REG(DC_GPIO_PAD_STRENGTH_1):
- *	case REG(DC_GPIO_PAD_STRENGTH_2):
- *	case REG(DC_GPIO_DEBUG):
- */
-	/* UNEXPECTED */
-	default:
-/*	case REG(DC_GPIO_SYNCA_A): not exist */
-		ASSERT_CRITICAL(false);
-		return false;
-	}
+	ASSERT_CRITICAL(false);
+	return false;
 }
 
 static bool id_to_offset(
@@ -199,170 +229,14 @@ static bool id_to_offset(
 	uint32_t en,
 	struct gpio_pin_info *info)
 {
-	bool result = true;
-
-	switch (id) {
-	case GPIO_ID_DDC_DATA:
-		info->mask = DC_GPIO_DDC6_A__DC_GPIO_DDC6DATA_A_MASK;
-		switch (en) {
-		case GPIO_DDC_LINE_DDC1:
-			info->offset = REG(DC_GPIO_DDC1_A);
-		break;
-		case GPIO_DDC_LINE_DDC2:
-			info->offset = REG(DC_GPIO_DDC2_A);
-		break;
-		case GPIO_DDC_LINE_DDC3:
-			info->offset = REG(DC_GPIO_DDC3_A);
-		break;
-		case GPIO_DDC_LINE_DDC4:
-			info->offset = REG(DC_GPIO_DDC4_A);
-		break;
-		case GPIO_DDC_LINE_DDC5:
-			info->offset = REG(DC_GPIO_DDC5_A);
-		break;
-		case GPIO_DDC_LINE_DDC6:
-			info->offset = REG(DC_GPIO_DDC6_A);
-		break;
-		case GPIO_DDC_LINE_DDC_VGA:
-			info->offset = REG(DC_GPIO_DDCVGA_A);
-		break;
-		case GPIO_DDC_LINE_I2C_PAD:
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_DDC_CLOCK:
-		info->mask = DC_GPIO_DDC6_A__DC_GPIO_DDC6CLK_A_MASK;
-		switch (en) {
-		case GPIO_DDC_LINE_DDC1:
-			info->offset = REG(DC_GPIO_DDC1_A);
-		break;
-		case GPIO_DDC_LINE_DDC2:
-			info->offset = REG(DC_GPIO_DDC2_A);
-		break;
-		case GPIO_DDC_LINE_DDC3:
-			info->offset = REG(DC_GPIO_DDC3_A);
-		break;
-		case GPIO_DDC_LINE_DDC4:
-			info->offset = REG(DC_GPIO_DDC4_A);
-		break;
-		case GPIO_DDC_LINE_DDC5:
-			info->offset = REG(DC_GPIO_DDC5_A);
-		break;
-		case GPIO_DDC_LINE_DDC6:
-			info->offset = REG(DC_GPIO_DDC6_A);
-		break;
-		case GPIO_DDC_LINE_DDC_VGA:
-			info->offset = REG(DC_GPIO_DDCVGA_A);
-		break;
-		case GPIO_DDC_LINE_I2C_PAD:
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_GENERIC:
-		info->offset = REG(DC_GPIO_GENERIC_A);
-		switch (en) {
-		case GPIO_GENERIC_A:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK;
-		break;
-		case GPIO_GENERIC_B:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK;
-		break;
-		case GPIO_GENERIC_C:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK;
-		break;
-		case GPIO_GENERIC_D:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK;
-		break;
-		case GPIO_GENERIC_E:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK;
-		break;
-		case GPIO_GENERIC_F:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK;
-		break;
-		case GPIO_GENERIC_G:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK;
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_HPD:
-		info->offset = REG(DC_GPIO_HPD_A);
-		switch (en) {
-		case GPIO_HPD_1:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK;
-		break;
-		case GPIO_HPD_2:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK;
-		break;
-		case GPIO_HPD_3:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK;
-		break;
-		case GPIO_HPD_4:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK;
-		break;
-		case GPIO_HPD_5:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK;
-		break;
-		case GPIO_HPD_6:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK;
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_GSL:
-		switch (en) {
-		case GPIO_GSL_GENLOCK_CLOCK:
-				/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_GENLOCK_VSYNC:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_SWAPLOCK_A:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_SWAPLOCK_B:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_SYNC:
-	case GPIO_ID_VIP_PAD:
-	default:
-		ASSERT_CRITICAL(false);
-		result = false;
-	}
-
-	if (result) {
-		info->offset_y = info->offset + 2;
-		info->offset_en = info->offset + 1;
-		info->offset_mask = info->offset - 1;
-
-		info->mask_y = info->mask;
-		info->mask_en = info->mask;
-		info->mask_mask = info->mask;
-	}
+	if (dal_hw_translate_id_to_offset(
+			gpio_pins,
+			ARRAY_SIZE(gpio_pins),
+			id, en, info))
+		return true;
 
-	return result;
+	ASSERT_CRITICAL(false);
+	return false;
 }
 
 /* function table */
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn315/hw_translate_dcn315.c b/drivers/gpu/drm/amd/display/dc/gpio/dcn315/hw_translate_dcn315.c
index fbdaba57f718e..aa507f7f4ef9a 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/dcn315/hw_translate_dcn315.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn315/hw_translate_dcn315.c
@@ -62,128 +62,156 @@
  * end *********************/
 
 
+static const struct gpio_id_offset_entry gpio_offsets[] = {
+	/* GENERIC */
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_A),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_B),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_C),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_D),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_E),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_F),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_G),
+	/* HPD */
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_1),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_2),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_3),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_4),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_5),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_6),
+	/* GSL */
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_GENLOCK_CLOCK),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_GENLOCK_VSYNC),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_SWAPLOCK_A),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_SWAPLOCK_B),
+};
+
+
+/* DDC */
+static const struct gpio_ddc_offset_entry ddc_offset_map[] = {
+	{ REG(DC_GPIO_DDC1_A), GPIO_DDC_LINE_DDC1 },
+	{ REG(DC_GPIO_DDC2_A), GPIO_DDC_LINE_DDC2 },
+	{ REG(DC_GPIO_DDC3_A), GPIO_DDC_LINE_DDC3 },
+	{ REG(DC_GPIO_DDC4_A), GPIO_DDC_LINE_DDC4 },
+	{ REG(DC_GPIO_DDC5_A), GPIO_DDC_LINE_DDC5 },
+	{ REG(DC_GPIO_DDCVGA_A), GPIO_DDC_LINE_DDC_VGA },
+};
+
+
+/*
+ * GSL is intentionally omitted here.
+ * id_to_offset() for GSL is not implemented on this ASIC.
+ */
+static const struct gpio_pin_entry gpio_pins[] = {
+	/* DDC */
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC1,
+		DC_GPIO_DDC1_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC2,
+		DC_GPIO_DDC2_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC3,
+		DC_GPIO_DDC3_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC4,
+		DC_GPIO_DDC4_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC5,
+		DC_GPIO_DDC5_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC_VGA,
+		DC_GPIO_DDCVGA_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC1,
+		DC_GPIO_DDC1_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC2,
+		DC_GPIO_DDC2_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC3,
+		DC_GPIO_DDC3_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC4,
+		DC_GPIO_DDC4_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC5,
+		DC_GPIO_DDC5_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC_VGA,
+		DC_GPIO_DDCVGA_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	/* GENERIC */
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_B,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_C,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_D,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_E,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_F,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_G,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK),
+	/* HPD */
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_1,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_2,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_3,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_4,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_5,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_6,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK),
+};
+
+
 static bool offset_to_id(
 	uint32_t offset,
 	uint32_t mask,
 	enum gpio_id *id,
 	uint32_t *en)
 {
-	switch (offset) {
-	/* GENERIC */
-	case REG(DC_GPIO_GENERIC_A):
-		*id = GPIO_ID_GENERIC;
-		switch (mask) {
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK:
-			*en = GPIO_GENERIC_A;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK:
-			*en = GPIO_GENERIC_B;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK:
-			*en = GPIO_GENERIC_C;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK:
-			*en = GPIO_GENERIC_D;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK:
-			*en = GPIO_GENERIC_E;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK:
-			*en = GPIO_GENERIC_F;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK:
-			*en = GPIO_GENERIC_G;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* HPD */
-	case REG(DC_GPIO_HPD_A):
-		*id = GPIO_ID_HPD;
-		switch (mask) {
-		case DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK:
-			*en = GPIO_HPD_1;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK:
-			*en = GPIO_HPD_2;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK:
-			*en = GPIO_HPD_3;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK:
-			*en = GPIO_HPD_4;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK:
-			*en = GPIO_HPD_5;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK:
-			*en = GPIO_HPD_6;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* REG(DC_GPIO_GENLK_MASK */
-	case REG(DC_GPIO_GENLK_A):
-		*id = GPIO_ID_GSL;
-		switch (mask) {
-		case DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK:
-			*en = GPIO_GSL_GENLOCK_CLOCK;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK:
-			*en = GPIO_GSL_GENLOCK_VSYNC;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK:
-			*en = GPIO_GSL_SWAPLOCK_A;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK:
-			*en = GPIO_GSL_SWAPLOCK_B;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* DDC */
-	/* we don't care about the GPIO_ID for DDC
-	 * in DdcHandle it will use GPIO_ID_DDC_DATA/GPIO_ID_DDC_CLOCK
-	 * directly in the create method
-	 */
-	case REG(DC_GPIO_DDC1_A):
-		*en = GPIO_DDC_LINE_DDC1;
-		return true;
-	case REG(DC_GPIO_DDC2_A):
-		*en = GPIO_DDC_LINE_DDC2;
+	if (dal_hw_translate_gpio_ddc_offset_to_id(
+			ddc_offset_map,
+			ARRAY_SIZE(ddc_offset_map),
+			offset, en))
 		return true;
-	case REG(DC_GPIO_DDC3_A):
-		*en = GPIO_DDC_LINE_DDC3;
-		return true;
-	case REG(DC_GPIO_DDC4_A):
-		*en = GPIO_DDC_LINE_DDC4;
-		return true;
-	case REG(DC_GPIO_DDC5_A):
-		*en = GPIO_DDC_LINE_DDC5;
-		return true;
-	case REG(DC_GPIO_DDCVGA_A):
-		*en = GPIO_DDC_LINE_DDC_VGA;
+
+	if (dal_hw_translate_gpio_offset_to_id(
+			gpio_offsets,
+			ARRAY_SIZE(gpio_offsets),
+			offset, mask, id, en))
 		return true;
 
-/*
- *	case REG(DC_GPIO_I2CPAD_A): not exit
- *	case REG(DC_GPIO_PWRSEQ_A):
- *	case REG(DC_GPIO_PAD_STRENGTH_1):
- *	case REG(DC_GPIO_PAD_STRENGTH_2):
- *	case REG(DC_GPIO_DEBUG):
- */
-	/* UNEXPECTED */
-	default:
-/*	case REG(DC_GPIO_SYNCA_A): not exist */
-		ASSERT_CRITICAL(false);
-		return false;
-	}
+	ASSERT_CRITICAL(false);
+	return false;
 }
 
 static bool id_to_offset(
@@ -191,164 +219,14 @@ static bool id_to_offset(
 	uint32_t en,
 	struct gpio_pin_info *info)
 {
-	bool result = true;
-
-	switch (id) {
-	case GPIO_ID_DDC_DATA:
-		info->mask = DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK;
-		switch (en) {
-		case GPIO_DDC_LINE_DDC1:
-			info->offset = REG(DC_GPIO_DDC1_A);
-		break;
-		case GPIO_DDC_LINE_DDC2:
-			info->offset = REG(DC_GPIO_DDC2_A);
-		break;
-		case GPIO_DDC_LINE_DDC3:
-			info->offset = REG(DC_GPIO_DDC3_A);
-		break;
-		case GPIO_DDC_LINE_DDC4:
-			info->offset = REG(DC_GPIO_DDC4_A);
-		break;
-		case GPIO_DDC_LINE_DDC5:
-			info->offset = REG(DC_GPIO_DDC5_A);
-		break;
-		case GPIO_DDC_LINE_DDC_VGA:
-			info->offset = REG(DC_GPIO_DDCVGA_A);
-		break;
-		case GPIO_DDC_LINE_I2C_PAD:
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_DDC_CLOCK:
-		info->mask = DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK;
-		switch (en) {
-		case GPIO_DDC_LINE_DDC1:
-			info->offset = REG(DC_GPIO_DDC1_A);
-		break;
-		case GPIO_DDC_LINE_DDC2:
-			info->offset = REG(DC_GPIO_DDC2_A);
-		break;
-		case GPIO_DDC_LINE_DDC3:
-			info->offset = REG(DC_GPIO_DDC3_A);
-		break;
-		case GPIO_DDC_LINE_DDC4:
-			info->offset = REG(DC_GPIO_DDC4_A);
-		break;
-		case GPIO_DDC_LINE_DDC5:
-			info->offset = REG(DC_GPIO_DDC5_A);
-		break;
-		case GPIO_DDC_LINE_DDC_VGA:
-			info->offset = REG(DC_GPIO_DDCVGA_A);
-		break;
-		case GPIO_DDC_LINE_I2C_PAD:
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_GENERIC:
-		info->offset = REG(DC_GPIO_GENERIC_A);
-		switch (en) {
-		case GPIO_GENERIC_A:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK;
-		break;
-		case GPIO_GENERIC_B:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK;
-		break;
-		case GPIO_GENERIC_C:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK;
-		break;
-		case GPIO_GENERIC_D:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK;
-		break;
-		case GPIO_GENERIC_E:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK;
-		break;
-		case GPIO_GENERIC_F:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK;
-		break;
-		case GPIO_GENERIC_G:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICG_A_MASK;
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_HPD:
-		info->offset = REG(DC_GPIO_HPD_A);
-		switch (en) {
-		case GPIO_HPD_1:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK;
-		break;
-		case GPIO_HPD_2:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK;
-		break;
-		case GPIO_HPD_3:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK;
-		break;
-		case GPIO_HPD_4:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK;
-		break;
-		case GPIO_HPD_5:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK;
-		break;
-		case GPIO_HPD_6:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD6_A_MASK;
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_GSL:
-		switch (en) {
-		case GPIO_GSL_GENLOCK_CLOCK:
-				/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_GENLOCK_VSYNC:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_SWAPLOCK_A:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_SWAPLOCK_B:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_SYNC:
-	case GPIO_ID_VIP_PAD:
-	default:
-		ASSERT_CRITICAL(false);
-		result = false;
-	}
-
-	if (result) {
-		info->offset_y = info->offset + 2;
-		info->offset_en = info->offset + 1;
-		info->offset_mask = info->offset - 1;
-
-		info->mask_y = info->mask;
-		info->mask_en = info->mask;
-		info->mask_mask = info->mask;
-	}
+	if (dal_hw_translate_id_to_offset(
+			gpio_pins,
+			ARRAY_SIZE(gpio_pins),
+			id, en, info))
+		return true;
 
-	return result;
+	ASSERT_CRITICAL(false);
+	return false;
 }
 
 /* function table */
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn32/hw_translate_dcn32.c b/drivers/gpu/drm/amd/display/dc/gpio/dcn32/hw_translate_dcn32.c
index 8493b9981f9e8..71067a8da1216 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/dcn32/hw_translate_dcn32.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn32/hw_translate_dcn32.c
@@ -60,111 +60,145 @@
  * end *********************/
 
 
+static const struct gpio_id_offset_entry gpio_offsets[] = {
+	/* GENERIC */
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_A),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_B),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_C),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_D),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_E),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_F),
+	/* HPD */
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_1),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_2),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_3),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_4),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_5),
+	/* GSL */
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_GENLOCK_CLOCK),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_GENLOCK_VSYNC),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_SWAPLOCK_A),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_SWAPLOCK_B),
+};
+
+
+/* DDC */
+static const struct gpio_ddc_offset_entry ddc_offset_map[] = {
+	{ REG(DC_GPIO_DDC1_A), GPIO_DDC_LINE_DDC1 },
+	{ REG(DC_GPIO_DDC2_A), GPIO_DDC_LINE_DDC2 },
+	{ REG(DC_GPIO_DDC3_A), GPIO_DDC_LINE_DDC3 },
+	{ REG(DC_GPIO_DDC4_A), GPIO_DDC_LINE_DDC4 },
+	{ REG(DC_GPIO_DDC5_A), GPIO_DDC_LINE_DDC5 },
+	{ REG(DC_GPIO_DDCVGA_A), GPIO_DDC_LINE_DDC_VGA },
+};
+
+/*
+ * GSL is intentionally omitted here.
+ * id_to_offset() for GSL is not implemented on this ASIC.
+ */
+static const struct gpio_pin_entry gpio_pins[] = {
+	/* DDC */
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC1,
+		DC_GPIO_DDC1_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC2,
+		DC_GPIO_DDC2_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC3,
+		DC_GPIO_DDC3_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC4,
+		DC_GPIO_DDC4_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC5,
+		DC_GPIO_DDC5_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC_VGA,
+		DC_GPIO_DDCVGA_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC1,
+		DC_GPIO_DDC1_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC2,
+		DC_GPIO_DDC2_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC3,
+		DC_GPIO_DDC3_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC4,
+		DC_GPIO_DDC4_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC5,
+		DC_GPIO_DDC5_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC_VGA,
+		DC_GPIO_DDCVGA_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	/* GENERIC */
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_B,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_C,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_D,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_E,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_F,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK),
+	/* HPD */
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_1,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_2,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_3,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_4,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_5,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK),
+};
+
+
 static bool offset_to_id(
 	uint32_t offset,
 	uint32_t mask,
 	enum gpio_id *id,
 	uint32_t *en)
 {
-	switch (offset) {
-	/* GENERIC */
-	case REG(DC_GPIO_GENERIC_A):
-		*id = GPIO_ID_GENERIC;
-		switch (mask) {
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK:
-			*en = GPIO_GENERIC_A;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK:
-			*en = GPIO_GENERIC_B;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK:
-			*en = GPIO_GENERIC_C;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK:
-			*en = GPIO_GENERIC_D;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK:
-			*en = GPIO_GENERIC_E;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK:
-			*en = GPIO_GENERIC_F;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* HPD */
-	case REG(DC_GPIO_HPD_A):
-		*id = GPIO_ID_HPD;
-		switch (mask) {
-		case DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK:
-			*en = GPIO_HPD_1;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK:
-			*en = GPIO_HPD_2;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK:
-			*en = GPIO_HPD_3;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK:
-			*en = GPIO_HPD_4;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK:
-			*en = GPIO_HPD_5;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* REG(DC_GPIO_GENLK_MASK */
-	case REG(DC_GPIO_GENLK_A):
-		*id = GPIO_ID_GSL;
-		switch (mask) {
-		case DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK:
-			*en = GPIO_GSL_GENLOCK_CLOCK;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK:
-			*en = GPIO_GSL_GENLOCK_VSYNC;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK:
-			*en = GPIO_GSL_SWAPLOCK_A;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK:
-			*en = GPIO_GSL_SWAPLOCK_B;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* DDC */
-	/* we don't care about the GPIO_ID for DDC
-	 * in DdcHandle it will use GPIO_ID_DDC_DATA/GPIO_ID_DDC_CLOCK
-	 * directly in the create method */
-	case REG(DC_GPIO_DDC1_A):
-		*en = GPIO_DDC_LINE_DDC1;
-		return true;
-	case REG(DC_GPIO_DDC2_A):
-		*en = GPIO_DDC_LINE_DDC2;
-		return true;
-	case REG(DC_GPIO_DDC3_A):
-		*en = GPIO_DDC_LINE_DDC3;
-		return true;
-	case REG(DC_GPIO_DDC4_A):
-		*en = GPIO_DDC_LINE_DDC4;
-		return true;
-	case REG(DC_GPIO_DDC5_A):
-		*en = GPIO_DDC_LINE_DDC5;
+	if (dal_hw_translate_gpio_ddc_offset_to_id(
+			ddc_offset_map,
+			ARRAY_SIZE(ddc_offset_map),
+			offset, en))
 		return true;
-	case REG(DC_GPIO_DDCVGA_A):
-		*en = GPIO_DDC_LINE_DDC_VGA;
+
+	if (dal_hw_translate_gpio_offset_to_id(
+			gpio_offsets,
+			ARRAY_SIZE(gpio_offsets),
+			offset, mask, id, en))
 		return true;
-	default:
-		ASSERT_CRITICAL(false);
-		return false;
-	}
+
+	ASSERT_CRITICAL(false);
+	return false;
 }
 
 static bool id_to_offset(
@@ -172,158 +206,14 @@ static bool id_to_offset(
 	uint32_t en,
 	struct gpio_pin_info *info)
 {
-	bool result = true;
-
-	switch (id) {
-	case GPIO_ID_DDC_DATA:
-		info->mask = DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK;
-		switch (en) {
-		case GPIO_DDC_LINE_DDC1:
-			info->offset = REG(DC_GPIO_DDC1_A);
-		break;
-		case GPIO_DDC_LINE_DDC2:
-			info->offset = REG(DC_GPIO_DDC2_A);
-		break;
-		case GPIO_DDC_LINE_DDC3:
-			info->offset = REG(DC_GPIO_DDC3_A);
-		break;
-		case GPIO_DDC_LINE_DDC4:
-			info->offset = REG(DC_GPIO_DDC4_A);
-		break;
-		case GPIO_DDC_LINE_DDC5:
-			info->offset = REG(DC_GPIO_DDC5_A);
-		break;
-		case GPIO_DDC_LINE_DDC_VGA:
-			info->offset = REG(DC_GPIO_DDCVGA_A);
-		break;
-		case GPIO_DDC_LINE_I2C_PAD:
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_DDC_CLOCK:
-		info->mask = DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK;
-		switch (en) {
-		case GPIO_DDC_LINE_DDC1:
-			info->offset = REG(DC_GPIO_DDC1_A);
-		break;
-		case GPIO_DDC_LINE_DDC2:
-			info->offset = REG(DC_GPIO_DDC2_A);
-		break;
-		case GPIO_DDC_LINE_DDC3:
-			info->offset = REG(DC_GPIO_DDC3_A);
-		break;
-		case GPIO_DDC_LINE_DDC4:
-			info->offset = REG(DC_GPIO_DDC4_A);
-		break;
-		case GPIO_DDC_LINE_DDC5:
-			info->offset = REG(DC_GPIO_DDC5_A);
-		break;
-		case GPIO_DDC_LINE_DDC_VGA:
-			info->offset = REG(DC_GPIO_DDCVGA_A);
-		break;
-		case GPIO_DDC_LINE_I2C_PAD:
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_GENERIC:
-		info->offset = REG(DC_GPIO_GENERIC_A);
-		switch (en) {
-		case GPIO_GENERIC_A:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK;
-		break;
-		case GPIO_GENERIC_B:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK;
-		break;
-		case GPIO_GENERIC_C:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK;
-		break;
-		case GPIO_GENERIC_D:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK;
-		break;
-		case GPIO_GENERIC_E:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK;
-		break;
-		case GPIO_GENERIC_F:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK;
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_HPD:
-		info->offset = REG(DC_GPIO_HPD_A);
-		switch (en) {
-		case GPIO_HPD_1:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK;
-		break;
-		case GPIO_HPD_2:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK;
-		break;
-		case GPIO_HPD_3:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK;
-		break;
-		case GPIO_HPD_4:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK;
-		break;
-		case GPIO_HPD_5:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK;
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_GSL:
-		switch (en) {
-		case GPIO_GSL_GENLOCK_CLOCK:
-				/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_GENLOCK_VSYNC:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_SWAPLOCK_A:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_SWAPLOCK_B:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_SYNC:
-	case GPIO_ID_VIP_PAD:
-	default:
-		ASSERT_CRITICAL(false);
-		result = false;
-	}
-
-	if (result) {
-		info->offset_y = info->offset + 2;
-		info->offset_en = info->offset + 1;
-		info->offset_mask = info->offset - 1;
-
-		info->mask_y = info->mask;
-		info->mask_en = info->mask;
-		info->mask_mask = info->mask;
-	}
+	if (dal_hw_translate_id_to_offset(
+			gpio_pins,
+			ARRAY_SIZE(gpio_pins),
+			id, en, info))
+		return true;
 
-	return result;
+	ASSERT_CRITICAL(false);
+	return false;
 }
 
 /* function table */
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn401/hw_translate_dcn401.c b/drivers/gpu/drm/amd/display/dc/gpio/dcn401/hw_translate_dcn401.c
index ea416f01f888c..7aa97f09955cf 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/dcn401/hw_translate_dcn401.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn401/hw_translate_dcn401.c
@@ -35,119 +35,145 @@
  * end *********************/
 
 
+static const struct gpio_id_offset_entry gpio_offsets[] = {
+	/* GENERIC */
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_A),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_B),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_C),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_D),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_E),
+	GPIO_MASK_ENTRY(DC_GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK,
+		GPIO_ID_GENERIC, GPIO_GENERIC_F),
+	/* HPD */
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_1),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_2),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_3),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_4),
+	GPIO_MASK_ENTRY(DC_GPIO_HPD_A,
+		DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK,
+		GPIO_ID_HPD, GPIO_HPD_5),
+	/* GSL */
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_GENLOCK_CLOCK),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_GENLOCK_VSYNC),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_SWAPLOCK_A),
+	GPIO_MASK_ENTRY(DC_GPIO_GENLK_A,
+		DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK,
+		GPIO_ID_GSL, GPIO_GSL_SWAPLOCK_B),
+};
+
+
+/* DDC */
+static const struct gpio_ddc_offset_entry ddc_offset_map[] = {
+	{ REG(DC_GPIO_DDC1_A), GPIO_DDC_LINE_DDC1 },
+	{ REG(DC_GPIO_DDC2_A), GPIO_DDC_LINE_DDC2 },
+	{ REG(DC_GPIO_DDC3_A), GPIO_DDC_LINE_DDC3 },
+	{ REG(DC_GPIO_DDC4_A), GPIO_DDC_LINE_DDC4 },
+	{ REG(DC_GPIO_DDCVGA_A), GPIO_DDC_LINE_DDC_VGA },
+};
+
+
+/*
+ * GSL is intentionally omitted here.
+ * id_to_offset() for GSL is not implemented on this ASIC.
+ */
+static const struct gpio_pin_entry gpio_pins[] = {
+	/* DDC */
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC1,
+		DC_GPIO_DDC1_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC2,
+		DC_GPIO_DDC2_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC3,
+		DC_GPIO_DDC3_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC4,
+		DC_GPIO_DDC4_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	/* GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC5,
+		DC_GPIO_DDC5_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK), */
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC_VGA,
+		DC_GPIO_DDCVGA_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC1,
+		DC_GPIO_DDC1_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC2,
+		DC_GPIO_DDC2_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC3,
+		DC_GPIO_DDC3_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC4,
+		DC_GPIO_DDC4_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	/* GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC5,
+		DC_GPIO_DDC5_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK), */
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC_VGA,
+		DC_GPIO_DDCVGA_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	/* GENERIC */
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_A,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_B,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_C,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_D,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_E,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_GENERIC, GPIO_GENERIC_F,
+		DC_GPIO_GENERIC_A, DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK),
+	/* HPD */
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_1,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_2,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_3,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_4,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_HPD, GPIO_HPD_5,
+		DC_GPIO_HPD_A, DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK),
+};
+
+
 static bool offset_to_id(
 	uint32_t offset,
 	uint32_t mask,
 	enum gpio_id *id,
 	uint32_t *en)
 {
-	switch (offset) {
-	/* GENERIC */
-	case REG(DC_GPIO_GENERIC_A):
-		*id = GPIO_ID_GENERIC;
-		switch (mask) {
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK:
-			*en = GPIO_GENERIC_A;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK:
-			*en = GPIO_GENERIC_B;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK:
-			*en = GPIO_GENERIC_C;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK:
-			*en = GPIO_GENERIC_D;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK:
-			*en = GPIO_GENERIC_E;
-			return true;
-		case DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK:
-			*en = GPIO_GENERIC_F;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* HPD */
-	case REG(DC_GPIO_HPD_A):
-		*id = GPIO_ID_HPD;
-		switch (mask) {
-		case DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK:
-			*en = GPIO_HPD_1;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK:
-			*en = GPIO_HPD_2;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK:
-			*en = GPIO_HPD_3;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK:
-			*en = GPIO_HPD_4;
-			return true;
-		case DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK:
-			*en = GPIO_HPD_5;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* REG(DC_GPIO_GENLK_MASK */
-	case REG(DC_GPIO_GENLK_A):
-		*id = GPIO_ID_GSL;
-		switch (mask) {
-		case DC_GPIO_GENLK_A__DC_GPIO_GENLK_CLK_A_MASK:
-			*en = GPIO_GSL_GENLOCK_CLOCK;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_GENLK_VSYNC_A_MASK:
-			*en = GPIO_GSL_GENLOCK_VSYNC;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_A_A_MASK:
-			*en = GPIO_GSL_SWAPLOCK_A;
-			return true;
-		case DC_GPIO_GENLK_A__DC_GPIO_SWAPLOCK_B_A_MASK:
-			*en = GPIO_GSL_SWAPLOCK_B;
-			return true;
-		default:
-			ASSERT_CRITICAL(false);
-			return false;
-		}
-	break;
-	/* DDC */
-	/* we don't care about the GPIO_ID for DDC
-	 * in DdcHandle it will use GPIO_ID_DDC_DATA/GPIO_ID_DDC_CLOCK
-	 * directly in the create method
-	 */
-	case REG(DC_GPIO_DDC1_A):
-		*en = GPIO_DDC_LINE_DDC1;
+	if (dal_hw_translate_gpio_ddc_offset_to_id(
+			ddc_offset_map,
+			ARRAY_SIZE(ddc_offset_map),
+			offset, en))
 		return true;
-	case REG(DC_GPIO_DDC2_A):
-		*en = GPIO_DDC_LINE_DDC2;
-		return true;
-	case REG(DC_GPIO_DDC3_A):
-		*en = GPIO_DDC_LINE_DDC3;
-		return true;
-	case REG(DC_GPIO_DDC4_A):
-		*en = GPIO_DDC_LINE_DDC4;
-		return true;
-	case REG(DC_GPIO_DDCVGA_A):
-		*en = GPIO_DDC_LINE_DDC_VGA;
+
+	if (dal_hw_translate_gpio_offset_to_id(
+			gpio_offsets,
+			ARRAY_SIZE(gpio_offsets),
+			offset, mask, id, en))
 		return true;
 
-/*
- *	case REG(DC_GPIO_I2CPAD_A): not exit
- *	case REG(DC_GPIO_PWRSEQ_A):
- *	case REG(DC_GPIO_PAD_STRENGTH_1):
- *	case REG(DC_GPIO_PAD_STRENGTH_2):
- *	case REG(DC_GPIO_DEBUG):
- */
-	/* UNEXPECTED */
-	default:
-/*	case REG(DC_GPIO_SYNCA_A): not exist */
-		ASSERT_CRITICAL(false);
-		return false;
-	}
+	ASSERT_CRITICAL(false);
+	return false;
 }
 
 
@@ -156,158 +182,14 @@ static bool id_to_offset(
 	uint32_t en,
 	struct gpio_pin_info *info)
 {
-	bool result = true;
-
-	switch (id) {
-	case GPIO_ID_DDC_DATA:
-		info->mask = DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK;
-		switch (en) {
-		case GPIO_DDC_LINE_DDC1:
-			info->offset = REG(DC_GPIO_DDC1_A);
-		break;
-		case GPIO_DDC_LINE_DDC2:
-			info->offset = REG(DC_GPIO_DDC2_A);
-		break;
-		case GPIO_DDC_LINE_DDC3:
-			info->offset = REG(DC_GPIO_DDC3_A);
-		break;
-		case GPIO_DDC_LINE_DDC4:
-			info->offset = REG(DC_GPIO_DDC4_A);
-		break;
-/*		case GPIO_DDC_LINE_DDC5:
-			info->offset = REG(DC_GPIO_DDC5_A);
-		break; */
-		case GPIO_DDC_LINE_DDC_VGA:
-			info->offset = REG(DC_GPIO_DDCVGA_A);
-		break;
-		case GPIO_DDC_LINE_I2C_PAD:
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_DDC_CLOCK:
-		info->mask = DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK;
-		switch (en) {
-		case GPIO_DDC_LINE_DDC1:
-			info->offset = REG(DC_GPIO_DDC1_A);
-		break;
-		case GPIO_DDC_LINE_DDC2:
-			info->offset = REG(DC_GPIO_DDC2_A);
-		break;
-		case GPIO_DDC_LINE_DDC3:
-			info->offset = REG(DC_GPIO_DDC3_A);
-		break;
-		case GPIO_DDC_LINE_DDC4:
-			info->offset = REG(DC_GPIO_DDC4_A);
-		break;
-/*		case GPIO_DDC_LINE_DDC5:
-			info->offset = REG(DC_GPIO_DDC5_A);
-		break; */
-		case GPIO_DDC_LINE_DDC_VGA:
-			info->offset = REG(DC_GPIO_DDCVGA_A);
-		break;
-		case GPIO_DDC_LINE_I2C_PAD:
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_GENERIC:
-		info->offset = REG(DC_GPIO_GENERIC_A);
-		switch (en) {
-		case GPIO_GENERIC_A:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICA_A_MASK;
-		break;
-		case GPIO_GENERIC_B:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICB_A_MASK;
-		break;
-		case GPIO_GENERIC_C:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICC_A_MASK;
-		break;
-		case GPIO_GENERIC_D:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICD_A_MASK;
-		break;
-		case GPIO_GENERIC_E:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICE_A_MASK;
-		break;
-		case GPIO_GENERIC_F:
-			info->mask = DC_GPIO_GENERIC_A__DC_GPIO_GENERICF_A_MASK;
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_HPD:
-		info->offset = REG(DC_GPIO_HPD_A);
-		switch (en) {
-		case GPIO_HPD_1:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD1_A_MASK;
-		break;
-		case GPIO_HPD_2:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD2_A_MASK;
-		break;
-		case GPIO_HPD_3:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD3_A_MASK;
-		break;
-		case GPIO_HPD_4:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD4_A_MASK;
-		break;
-		case GPIO_HPD_5:
-			info->mask = DC_GPIO_HPD_A__DC_GPIO_HPD5_A_MASK;
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_GSL:
-		switch (en) {
-		case GPIO_GSL_GENLOCK_CLOCK:
-				/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_GENLOCK_VSYNC:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_SWAPLOCK_A:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-		break;
-		case GPIO_GSL_SWAPLOCK_B:
-			/*not implmented*/
-			ASSERT_CRITICAL(false);
-			result = false;
-
-		break;
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_SYNC:
-	case GPIO_ID_VIP_PAD:
-	default:
-		ASSERT_CRITICAL(false);
-		result = false;
-	}
-
-	if (result) {
-		info->offset_y = info->offset + 2;
-		info->offset_en = info->offset + 1;
-		info->offset_mask = info->offset - 1;
-
-		info->mask_y = info->mask;
-		info->mask_en = info->mask;
-		info->mask_mask = info->mask;
-	}
+	if (dal_hw_translate_id_to_offset(
+			gpio_pins,
+			ARRAY_SIZE(gpio_pins),
+			id, en, info))
+		return true;
 
-	return result;
+	ASSERT_CRITICAL(false);
+	return false;
 }
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_translate_dcn42.c b/drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_translate_dcn42.c
index e7e1d99798768..7b2c4cd424500 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_translate_dcn42.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn42/hw_translate_dcn42.c
@@ -39,62 +39,76 @@
  * end *********************/
 
 
+static const struct gpio_id_offset_entry gpio_offsets[] = {
+	/* HPD */
+	GPIO_ENTRY(HPD0_DC_HPD_INT_STATUS, GPIO_ID_HPD, GPIO_HPD_1),
+	GPIO_ENTRY(HPD1_DC_HPD_INT_STATUS, GPIO_ID_HPD, GPIO_HPD_2),
+	GPIO_ENTRY(HPD2_DC_HPD_INT_STATUS, GPIO_ID_HPD, GPIO_HPD_3),
+	GPIO_ENTRY(HPD3_DC_HPD_INT_STATUS, GPIO_ID_HPD, GPIO_HPD_4),
+	GPIO_ENTRY(HPD4_DC_HPD_INT_STATUS, GPIO_ID_HPD, GPIO_HPD_5),
+};
+
+
+/* DDC */
+static const struct gpio_ddc_offset_entry ddc_offset_map[] = {
+	{ REG(DC_GPIO_DDC1_A), GPIO_DDC_LINE_DDC1 },
+	{ REG(DC_GPIO_DDC2_A), GPIO_DDC_LINE_DDC2 },
+	{ REG(DC_GPIO_DDC3_A), GPIO_DDC_LINE_DDC3 },
+	{ REG(DC_GPIO_DDC4_A), GPIO_DDC_LINE_DDC4 },
+	{ REG(DC_GPIO_DDC5_A), GPIO_DDC_LINE_DDC5 },
+	{ REG(DC_GPIO_DDCVGA_A), GPIO_DDC_LINE_DDC_VGA },
+};
+
+
+static const struct gpio_pin_entry gpio_pins[] = {
+	/* DDC */
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC1,
+		DC_GPIO_DDC1_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC2,
+		DC_GPIO_DDC2_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC3,
+		DC_GPIO_DDC3_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC4,
+		DC_GPIO_DDC4_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC5,
+		DC_GPIO_DDC5_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_DATA, GPIO_DDC_LINE_DDC_VGA,
+		DC_GPIO_DDCVGA_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC1,
+		DC_GPIO_DDC1_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC2,
+		DC_GPIO_DDC2_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC3,
+		DC_GPIO_DDC3_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC4,
+		DC_GPIO_DDC4_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC5,
+		DC_GPIO_DDC5_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+	GPIO_PIN_ENTRY(GPIO_ID_DDC_CLOCK, GPIO_DDC_LINE_DDC_VGA,
+		DC_GPIO_DDCVGA_A, DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK),
+};
+
+
 static bool offset_to_id(
 	uint32_t offset,
 	uint32_t mask,
 	enum gpio_id *id,
 	uint32_t *en)
 {
-	(void)mask;
-	switch (offset) {
-	/* HPD */
-	case REG(HPD0_DC_HPD_INT_STATUS):
-		*id = GPIO_ID_HPD;
-		*en = GPIO_HPD_1;
-		return true;
-	case REG(HPD1_DC_HPD_INT_STATUS):
-		*id = GPIO_ID_HPD;
-		*en = GPIO_HPD_2;
+	if (dal_hw_translate_gpio_ddc_offset_to_id(
+			ddc_offset_map,
+			ARRAY_SIZE(ddc_offset_map),
+			offset, en))
 		return true;
-	case REG(HPD2_DC_HPD_INT_STATUS):
-		*id = GPIO_ID_HPD;
-		*en = GPIO_HPD_3;
-		return true;
-	case REG(HPD3_DC_HPD_INT_STATUS):
-		*id = GPIO_ID_HPD;
-		*en = GPIO_HPD_4;
-		return true;
-	case REG(HPD4_DC_HPD_INT_STATUS):
-		*id = GPIO_ID_HPD;
-		*en = GPIO_HPD_5;
-		return true;
-	/* DDC */
-	/* we don't care about the GPIO_ID for DDC
-	 * in DdcHandle it will use GPIO_ID_DDC_DATA/GPIO_ID_DDC_CLOCK
-	 * directly in the create method
-	 */
-	case REG(DC_GPIO_DDC1_A):
-		*en = GPIO_DDC_LINE_DDC1;
-		return true;
-	case REG(DC_GPIO_DDC2_A):
-		*en = GPIO_DDC_LINE_DDC2;
-		return true;
-	case REG(DC_GPIO_DDC3_A):
-		*en = GPIO_DDC_LINE_DDC3;
-		return true;
-	case REG(DC_GPIO_DDC4_A):
-		*en = GPIO_DDC_LINE_DDC4;
-		return true;
-	case REG(DC_GPIO_DDC5_A):
-		*en = GPIO_DDC_LINE_DDC5;
-		return true;
-	case REG(DC_GPIO_DDCVGA_A):
-		*en = GPIO_DDC_LINE_DDC_VGA;
+
+	if (dal_hw_translate_gpio_offset_to_id(
+			gpio_offsets,
+			ARRAY_SIZE(gpio_offsets),
+			offset, mask, id, en))
 		return true;
-	default:
-		ASSERT_CRITICAL(false);
-		return false;
-	}
+
+	ASSERT_CRITICAL(false);
+	return false;
 }
 
 
@@ -103,81 +117,14 @@ static bool id_to_offset(
 	uint32_t en,
 	struct gpio_pin_info *info)
 {
-	bool result = true;
-
-	switch (id) {
-	case GPIO_ID_DDC_DATA:
-		info->mask = DC_GPIO_DDC1_A__DC_GPIO_DDC1DATA_A_MASK;
-		switch (en) {
-		case GPIO_DDC_LINE_DDC1:
-			info->offset = REG(DC_GPIO_DDC1_A);
-		break;
-		case GPIO_DDC_LINE_DDC2:
-			info->offset = REG(DC_GPIO_DDC2_A);
-		break;
-		case GPIO_DDC_LINE_DDC3:
-			info->offset = REG(DC_GPIO_DDC3_A);
-		break;
-		case GPIO_DDC_LINE_DDC4:
-			info->offset = REG(DC_GPIO_DDC4_A);
-		break;
-		case GPIO_DDC_LINE_DDC5:
-			info->offset = REG(DC_GPIO_DDC5_A);
-		break;
-		case GPIO_DDC_LINE_DDC_VGA:
-			info->offset = REG(DC_GPIO_DDCVGA_A);
-		break;
-		case GPIO_DDC_LINE_I2C_PAD:
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_DDC_CLOCK:
-		info->mask = DC_GPIO_DDC1_A__DC_GPIO_DDC1CLK_A_MASK;
-		switch (en) {
-		case GPIO_DDC_LINE_DDC1:
-			info->offset = REG(DC_GPIO_DDC1_A);
-		break;
-		case GPIO_DDC_LINE_DDC2:
-			info->offset = REG(DC_GPIO_DDC2_A);
-		break;
-		case GPIO_DDC_LINE_DDC3:
-			info->offset = REG(DC_GPIO_DDC3_A);
-		break;
-		case GPIO_DDC_LINE_DDC4:
-			info->offset = REG(DC_GPIO_DDC4_A);
-		break;
-		case GPIO_DDC_LINE_DDC5:
-			info->offset = REG(DC_GPIO_DDC5_A);
-		break;
-		case GPIO_DDC_LINE_DDC_VGA:
-			info->offset = REG(DC_GPIO_DDCVGA_A);
-		break;
-		case GPIO_DDC_LINE_I2C_PAD:
-		default:
-			ASSERT_CRITICAL(false);
-			result = false;
-		}
-	break;
-	case GPIO_ID_SYNC:
-	case GPIO_ID_VIP_PAD:
-	default:
-		ASSERT_CRITICAL(false);
-		result = false;
-	}
-
-	if (result) {
-		info->offset_y = info->offset + 2;
-		info->offset_en = info->offset + 1;
-		info->offset_mask = info->offset - 1;
-
-		info->mask_y = info->mask;
-		info->mask_en = info->mask;
-		info->mask_mask = info->mask;
-	}
-
-	return result;
+	if (dal_hw_translate_id_to_offset(
+			gpio_pins,
+			ARRAY_SIZE(gpio_pins),
+			id, en, info))
+		return true;
+
+	ASSERT_CRITICAL(false);
+	return false;
 }
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c
index 64a5e11fce5c4..b58af86dee10e 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c
@@ -133,3 +133,89 @@ bool dal_hw_translate_init(
 		return false;
 	}
 }
+
+bool dal_hw_translate_gpio_offset_to_id(
+	const struct gpio_id_offset_entry *table,
+	uint32_t table_size,
+	uint32_t offset,
+	uint32_t mask,
+	enum gpio_id *id,
+	uint32_t *en)
+{
+	uint32_t i;
+
+	for (i = 0; i < table_size; i++) {
+		const struct gpio_id_offset_entry *entry = &table[i];
+
+		if (entry->offset != offset)
+			continue;
+
+		if (entry->check_mask && entry->mask != mask)
+			continue;
+
+		*id = entry->id;
+		*en = entry->en;
+
+		return true;
+	}
+
+	return false;
+}
+
+/* we don't care about the GPIO_ID for DDC
+ * in DdcHandle it will use GPIO_ID_DDC_DATA/GPIO_ID_DDC_CLOCK
+ * directly in the create method
+ */
+bool dal_hw_translate_gpio_ddc_offset_to_id(
+	const struct gpio_ddc_offset_entry *table,
+	uint32_t table_size,
+	uint32_t offset,
+	uint32_t *en)
+{
+	uint32_t i;
+
+	for (i = 0; i < table_size; i++) {
+		const struct gpio_ddc_offset_entry *entry = &table[i];
+
+		if (entry->offset != offset)
+			continue;
+
+		*en = entry->en;
+
+		return true;
+	}
+
+	return false;
+}
+
+bool dal_hw_translate_id_to_offset(
+	const struct gpio_pin_entry *table,
+	uint32_t table_size,
+	enum gpio_id id,
+	uint32_t en,
+	struct gpio_pin_info *info)
+{
+	uint32_t i;
+
+	for (i = 0; i < table_size; i++) {
+		const struct gpio_pin_entry *entry = &table[i];
+
+		if (entry->id != id || entry->en != en)
+			continue;
+
+		info->offset = entry->offset;
+		info->mask = entry->mask;
+
+		info->offset_y = info->offset + 2;
+		info->offset_en = info->offset + 1;
+		info->offset_mask = info->offset - 1;
+
+		info->mask_y = info->mask;
+		info->mask_en = info->mask;
+		info->mask_mask = info->mask;
+
+		return true;
+	}
+
+	return false;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.h b/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.h
index 3a7d89ca1605c..339e381f8fde3 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.h
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.h
@@ -47,4 +47,25 @@ bool dal_hw_translate_init(
 	enum dce_version dce_version,
 	enum dce_environment dce_environment);
 
+bool dal_hw_translate_gpio_offset_to_id(
+	const struct gpio_id_offset_entry *table,
+	uint32_t table_size,
+	uint32_t offset,
+	uint32_t mask,
+	enum gpio_id *id,
+	uint32_t *en);
+
+bool dal_hw_translate_gpio_ddc_offset_to_id(
+	const struct gpio_ddc_offset_entry *table,
+	uint32_t table_size,
+	uint32_t offset,
+	uint32_t *en);
+
+bool dal_hw_translate_id_to_offset(
+	const struct gpio_pin_entry *table,
+	uint32_t table_size,
+	enum gpio_id id,
+	uint32_t en,
+	struct gpio_pin_info *info);
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c
index 79cb506be5cbf..cbcd227890139 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c
@@ -138,10 +138,10 @@ static void dcn31_program_compbuf_size(struct hubbub *hubbub, unsigned int compb
 
 	if (safe_to_increase || compbuf_size_segments <= hubbub2->compbuf_size_segments) {
 		if (compbuf_size_segments > hubbub2->compbuf_size_segments) {
-			REG_WAIT(DCHUBBUB_DET0_CTRL, DET0_SIZE_CURRENT, hubbub2->det0_size, 1, 100);
-			REG_WAIT(DCHUBBUB_DET1_CTRL, DET1_SIZE_CURRENT, hubbub2->det1_size, 1, 100);
-			REG_WAIT(DCHUBBUB_DET2_CTRL, DET2_SIZE_CURRENT, hubbub2->det2_size, 1, 100);
-			REG_WAIT(DCHUBBUB_DET3_CTRL, DET3_SIZE_CURRENT, hubbub2->det3_size, 1, 100);
+			dcn31_wait_for_det_apply(hubbub, 0);
+			dcn31_wait_for_det_apply(hubbub, 1);
+			dcn31_wait_for_det_apply(hubbub, 2);
+			dcn31_wait_for_det_apply(hubbub, 3);
 		}
 		/* Should never be hit, if it is we have an erroneous hw config*/
 		ASSERT(hubbub2->det0_size + hubbub2->det1_size + hubbub2->det2_size
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn35/dcn35_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn35/dcn35_hubbub.c
index 82d4e3e0e5e8f..5e5a7a74346d6 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn35/dcn35_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn35/dcn35_hubbub.c
@@ -571,7 +571,7 @@ void dcn35_dchvm_init(struct hubbub *hubbub)
 
 	if (riommu_active) {
 		// Disable gating and memory power requests
-		REG_UPDATE(DCHVM_MEM_CTRL, HVM_GPUVMRET_PWR_REQ_DIS, 1);
+		REG_UPDATE_2(DCHVM_MEM_CTRL, HVM_GPUVMRET_PWR_REQ_DIS, 1, HVM_GPUVMRET_FORCE_REQ, 0);
 		REG_UPDATE_4(DCHVM_CLK_CTRL,
 						HVM_DISPCLK_R_GATE_DIS, 1,
 						HVM_DISPCLK_G_GATE_DIS, 1,
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c
index bf30a1bb61b75..e9ff20c02b1eb 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c
@@ -1250,7 +1250,8 @@ static const struct hubbub_funcs hubbub4_01_funcs = {
 	.program_compbuf_segments = dcn401_program_compbuf_segments,
 	.wait_for_det_update = dcn401_wait_for_det_update,
 	.program_arbiter = dcn401_program_arbiter,
-	.hubbub_read_reg_state = hubbub3_read_reg_state
+	.hubbub_read_reg_state = hubbub3_read_reg_state,
+	.get_mall_en = hubbub32_get_mall_en
 };
 
 void hubbub401_construct(struct dcn20_hubbub *hubbub2,
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.h b/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.h
index f487155444299..346bead815043 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.h
@@ -133,7 +133,9 @@
 	HUBBUB_SF(DCHUBBUB_CTRL_STATUS, ROB_OVERFLOW_STATUS, mask_sh),\
 	HUBBUB_SF(DCHUBBUB_CTRL_STATUS, ROB_OVERFLOW_CLEAR, mask_sh),\
 	HUBBUB_SF(DCHUBBUB_CTRL_STATUS, DCHUBBUB_HW_DEBUG, mask_sh),\
-	HUBBUB_SF(DCHUBBUB_CTRL_STATUS, CSTATE_SWATH_CHK_GOOD_MODE, mask_sh)
+	HUBBUB_SF(DCHUBBUB_CTRL_STATUS, CSTATE_SWATH_CHK_GOOD_MODE, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_MALL_CNTL, MALL_PREFETCH_COMPLETE, mask_sh),\
+	HUBBUB_SF(DCHUBBUB_ARB_MALL_CNTL, MALL_IN_USE, mask_sh)
 
 bool hubbub401_program_urgent_watermarks(
 		struct hubbub *hubbub,
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c
index 7c97a774141ff..d8eb5996b5774 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c
@@ -772,8 +772,7 @@ bool hubp1_is_flip_pending(struct hubp *hubp)
 	if (flip_pending)
 		return true;
 
-	if (hubp &&
-	    earliest_inuse_address.grph.addr.quad_part != hubp->request_address.grph.addr.quad_part)
+	if (earliest_inuse_address.grph.addr.quad_part != hubp->request_address.grph.addr.quad_part)
 		return true;
 
 	return false;
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.h b/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.h
index 80be394a8852d..2595c6441f004 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.h
@@ -286,7 +286,8 @@
 	type MCACHEID_MALL_PREF_2H_P0;\
 	type MCACHEID_MALL_PREF_1H_P1;\
 	type MCACHEID_MALL_PREF_2H_P1;\
-	type HUBP_FGCG_REP_DIS
+	type HUBP_FGCG_REP_DIS;\
+	type DST_Y_DELTA_DRQ_LIMIT
 
 #define DCN42_HUBP_REG_FIELD_VARIABLE_LIST(type) \
 	type HUBP_3DLUT_CROSSBAR_SEL_G;\
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c
index 3025151283580..9965cf5723548 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c
@@ -136,7 +136,7 @@ void hubp401_program_3dlut_fl_config(
 	uint32_t mpc_width = {(cfg->width == 17) ? 0 : 1};
 	uint32_t width = {cfg->width};
 
-	if (cfg->layout == DC_CM2_GPU_MEM_LAYOUT_1D_PACKED_LINEAR)
+	if (cfg->layout == CM_LUT_1D_PACKED_LINEAR)
 		width = (cfg->width == 17) ? 4916 : 35940;
 
 	REG_UPDATE_2(_3DLUT_FL_CONFIG,
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c
index e4602c3ddc662..94730f3cff423 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c
@@ -20,6 +20,12 @@ static void hubp42_set_fgcg(struct hubp *hubp, bool enable)
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
+	/* Temporary workaround for IOMMU mismatch issue.
+	 * Fine grain control via bit1 of debug flag.
+	 */
+	if (hubp->ctx->dc->debug.iommu_mismatch_temp_wka & 0x2)
+		enable = false;
+
 	REG_UPDATE(HUBP_CLK_CNTL, HUBP_FGCG_REP_DIS, !enable);
 }
 
@@ -574,7 +580,56 @@ static bool hubp42_program_surface_flip_and_addr(
 
 	return true;
 }
+void hubp42_setup_interdependent(
+		struct hubp *hubp,
+		struct dml2_dchub_per_pipe_register_set *pipe_regs)
+{
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+	REG_SET_2(PREFETCH_SETTINGS, 0,
+			DST_Y_PREFETCH, pipe_regs->dlg_regs.dst_y_prefetch,
+			VRATIO_PREFETCH, pipe_regs->dlg_regs.vratio_prefetch);
+
+	REG_SET(PREFETCH_SETTINGS_C, 0,
+			VRATIO_PREFETCH_C, pipe_regs->dlg_regs.vratio_prefetch_c);
+
+	REG_SET_2(VBLANK_PARAMETERS_0, 0,
+		DST_Y_PER_VM_VBLANK, pipe_regs->dlg_regs.dst_y_per_vm_vblank,
+		DST_Y_PER_ROW_VBLANK, pipe_regs->dlg_regs.dst_y_per_row_vblank);
 
+	REG_SET_2(FLIP_PARAMETERS_0, 0,
+		DST_Y_PER_VM_FLIP, pipe_regs->dlg_regs.dst_y_per_vm_flip,
+		DST_Y_PER_ROW_FLIP, pipe_regs->dlg_regs.dst_y_per_row_flip);
+
+	REG_SET(VBLANK_PARAMETERS_3, 0,
+		REFCYC_PER_META_CHUNK_VBLANK_L, pipe_regs->dlg_regs.refcyc_per_meta_chunk_vblank_l);
+
+	REG_SET(VBLANK_PARAMETERS_4, 0,
+		REFCYC_PER_META_CHUNK_VBLANK_C, pipe_regs->dlg_regs.refcyc_per_meta_chunk_vblank_c);
+
+	REG_SET(FLIP_PARAMETERS_2, 0,
+		REFCYC_PER_META_CHUNK_FLIP_L, pipe_regs->dlg_regs.refcyc_per_meta_chunk_flip_l);
+
+	REG_SET_2(PER_LINE_DELIVERY_PRE, 0,
+		REFCYC_PER_LINE_DELIVERY_PRE_L, pipe_regs->dlg_regs.refcyc_per_line_delivery_pre_l,
+		REFCYC_PER_LINE_DELIVERY_PRE_C, pipe_regs->dlg_regs.refcyc_per_line_delivery_pre_c);
+
+	REG_SET(DCN_SURF0_TTU_CNTL1, 0,
+		REFCYC_PER_REQ_DELIVERY_PRE,
+		pipe_regs->ttu_regs.refcyc_per_req_delivery_pre_l);
+	REG_SET(DCN_SURF1_TTU_CNTL1, 0,
+		REFCYC_PER_REQ_DELIVERY_PRE,
+		pipe_regs->ttu_regs.refcyc_per_req_delivery_pre_c);
+	REG_SET(DCN_CUR0_TTU_CNTL1, 0,
+		REFCYC_PER_REQ_DELIVERY_PRE, pipe_regs->ttu_regs.refcyc_per_req_delivery_pre_cur0);
+
+	REG_SET_2(DCN_GLOBAL_TTU_CNTL, 0,
+		MIN_TTU_VBLANK, pipe_regs->ttu_regs.min_ttu_vblank,
+		QoS_LEVEL_FLIP, pipe_regs->ttu_regs.qos_level_flip);
+
+	REG_SET(DST_Y_DELTA_DRQ_LIMIT, 0,
+		DST_Y_DELTA_DRQ_LIMIT, pipe_regs->dlg_regs.dst_y_delta_drq_limit);
+}
 struct hubp_funcs dcn42_hubp_funcs = {
 	.hubp_enable_tripleBuffer = hubp2_enable_triplebuffer,
 	.hubp_is_triplebuffer_enabled = hubp2_is_triplebuffer_enabled,
@@ -582,7 +637,7 @@ struct hubp_funcs dcn42_hubp_funcs = {
 	.hubp_program_surface_config = hubp42_program_surface_config,
 	.hubp_is_flip_pending = hubp2_is_flip_pending,
 	.hubp_setup2 = hubp42_setup,
-	.hubp_setup_interdependent2 = hubp401_setup_interdependent,
+	.hubp_setup_interdependent2 = hubp42_setup_interdependent,
 	.hubp_set_vm_system_aperture_settings = hubp3_set_vm_system_aperture_settings,
 	.set_blank = hubp2_set_blank,
 	.set_blank_regs = hubp2_set_blank_regs,
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h
index fc30afd4c9129..ba4a58c353ddf 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h
@@ -35,6 +35,7 @@
 	HUBP_SF(HUBP0_3DLUT_FL_CONFIG, HUBP0_3DLUT_FL_FORMAT, mask_sh),\
 	HUBP_SF(HUBP0_3DLUT_FL_BIAS_SCALE, HUBP0_3DLUT_FL_BIAS, mask_sh),\
 	HUBP_SF(HUBP0_3DLUT_FL_BIAS_SCALE, HUBP0_3DLUT_FL_SCALE, mask_sh),\
+	HUBP_SF(HUBPREQ0_DST_Y_DELTA_DRQ_LIMIT, DST_Y_DELTA_DRQ_LIMIT, mask_sh),\
 	HUBP_SF(CURSOR0_0_HUBP_3DLUT_CONTROL, HUBP_3DLUT_ENABLE, mask_sh),\
 	HUBP_SF(CURSOR0_0_HUBP_3DLUT_CONTROL, HUBP_3DLUT_DONE, mask_sh),\
 	HUBP_SF(CURSOR0_0_HUBP_3DLUT_CONTROL, HUBP_3DLUT_ADDRESSING_MODE, mask_sh),\
@@ -54,6 +55,7 @@
 	HUBP_SF(HUBP0_3DLUT_FL_CONFIG, HUBP0_3DLUT_FL_FORMAT, mask_sh),\
 	HUBP_SF(HUBP0_3DLUT_FL_BIAS_SCALE, HUBP0_3DLUT_FL_BIAS, mask_sh),\
 	HUBP_SF(HUBP0_3DLUT_FL_BIAS_SCALE, HUBP0_3DLUT_FL_SCALE, mask_sh),\
+	HUBP_SF(HUBPREQ0_DST_Y_DELTA_DRQ_LIMIT, DST_Y_DELTA_DRQ_LIMIT, mask_sh),\
 	HUBP_SF(CURSOR0_0_HUBP_3DLUT_CONTROL, HUBP_3DLUT_ENABLE, mask_sh),\
 	HUBP_SF(CURSOR0_0_HUBP_3DLUT_CONTROL, HUBP_3DLUT_DONE, mask_sh),\
 	HUBP_SF(CURSOR0_0_HUBP_3DLUT_CONTROL, HUBP_3DLUT_ADDRESSING_MODE, mask_sh),\
@@ -95,4 +97,7 @@ void hubp42_setup(
 		union dml2_global_sync_programming *pipe_global_sync,
 		struct dc_crtc_timing *timing);
 
+void hubp42_setup_interdependent(
+		struct hubp *hubp,
+		struct dml2_dchub_per_pipe_register_set *pipe_regs);
 #endif /* __DC_HUBP_DCN42_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
index 042602c50e350..cce4f30655758 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
@@ -1287,7 +1287,9 @@ void dce110_blank_stream(struct pipe_ctx *pipe_ctx)
 		return;
 
 	if (link->local_sink && link->local_sink->sink_signal == SIGNAL_TYPE_EDP) {
-		if (!link->skip_implict_edp_power_control && hws)
+		if (link->skip_implict_edp_power_control)
+			return;
+		if (hws)
 			hws->funcs.edp_backlight_control(link, false);
 		link->dc->hwss.set_abm_immediate_disable(pipe_ctx);
 	}
@@ -1793,7 +1795,9 @@ enum dc_status dce110_apply_single_controller_ctx_to_hw(
 			dc->link_srv->set_dsc_enable(pipe_ctx, true);
 	}
 
-	if (!stream->dpms_off)
+	if (!stream->dpms_off &&
+	    !(link->connector_signal == SIGNAL_TYPE_EDP &&
+	      link->skip_implict_edp_power_control))
 		dc->link_srv->set_dpms_on(context, pipe_ctx);
 
 	/* DCN3.1 FPGA Workaround
@@ -1812,7 +1816,9 @@ enum dc_status dce110_apply_single_controller_ctx_to_hw(
 	 * is constructed with the same sink). Make sure not to override
 	 * and link programming on the main.
 	 */
-	if (dc_state_get_pipe_subvp_type(context, pipe_ctx) != SUBVP_PHANTOM) {
+	if (dc_state_get_pipe_subvp_type(context, pipe_ctx) != SUBVP_PHANTOM &&
+	    !(link->connector_signal == SIGNAL_TYPE_EDP &&
+	      link->skip_implict_edp_power_control)) {
 		pipe_ctx->stream->link->psr_settings.psr_feature_enabled = false;
 		pipe_ctx->stream->link->replay_settings.replay_feature_enabled = false;
 	}
@@ -3166,12 +3172,12 @@ static void dce110_program_front_end_for_pipe(
 				plane_state->rotation);
 
 	/* Moved programming gamma from dc to hwss */
-	if (pipe_ctx->plane_state->update_flags.bits.full_update ||
-			pipe_ctx->plane_state->update_flags.bits.in_transfer_func_change ||
-			pipe_ctx->plane_state->update_flags.bits.gamma_change)
+	if (pipe_ctx->plane_state->update_bits.full_update ||
+			pipe_ctx->plane_state->update_bits.in_transfer_func_change ||
+			pipe_ctx->plane_state->update_bits.gamma_change)
 		hws->funcs.set_input_transfer_func(dc, pipe_ctx, pipe_ctx->plane_state);
 
-	if (pipe_ctx->plane_state->update_flags.bits.full_update)
+	if (pipe_ctx->plane_state->update_bits.full_update)
 		hws->funcs.set_output_transfer_func(dc, pipe_ctx, pipe_ctx->stream);
 
 	DC_LOG_SURFACE(
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce60/dce60_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce60/dce60_hwseq.c
index a08e9f9eec176..26aa303b8237e 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dce60/dce60_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dce60/dce60_hwseq.c
@@ -332,12 +332,12 @@ dce60_program_front_end_for_pipe(
 				plane_state->rotation);
 
 	/* Moved programming gamma from dc to hwss */
-	if (pipe_ctx->plane_state->update_flags.bits.full_update ||
-			pipe_ctx->plane_state->update_flags.bits.in_transfer_func_change ||
-			pipe_ctx->plane_state->update_flags.bits.gamma_change)
+	if (pipe_ctx->plane_state->update_bits.full_update ||
+			pipe_ctx->plane_state->update_bits.in_transfer_func_change ||
+			pipe_ctx->plane_state->update_bits.gamma_change)
 		hws->funcs.set_input_transfer_func(dc, pipe_ctx, pipe_ctx->plane_state);
 
-	if (pipe_ctx->plane_state->update_flags.bits.full_update)
+	if (pipe_ctx->plane_state->update_bits.full_update)
 		hws->funcs.set_output_transfer_func(dc, pipe_ctx, pipe_ctx->stream);
 
 	DC_LOG_SURFACE(
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
index 7112b71af9770..541cd908b341f 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
@@ -2981,7 +2981,7 @@ void dcn10_update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
 	mpcc_id = hubp->inst;
 
 	/* If there is no full update, don't need to touch MPC tree*/
-	if (!pipe_ctx->plane_state->update_flags.bits.full_update) {
+	if (!pipe_ctx->plane_state->update_bits.full_update) {
 		mpc->funcs->update_blending(mpc, &blnd_cfg, mpcc_id);
 		dc->hwss.update_visual_confirm_color(dc, pipe_ctx, mpcc_id);
 		return;
@@ -3041,7 +3041,7 @@ static void dcn10_update_dchubp_dpp(
 	/* If request max dpp clk is lower than current dispclk, no need to
 	 * divided by 2
 	 */
-	if (plane_state->update_flags.bits.full_update) {
+	if (plane_state->update_bits.full_update) {
 
 		/* new calculated dispclk, dppclk are stored in
 		 * context->bw_ctx.bw.dcn.clk.dispclk_khz / dppclk_khz. current
@@ -3096,7 +3096,7 @@ static void dcn10_update_dchubp_dpp(
 	 * VTG is within DCHUBBUB which is commond block share by each pipe HUBP.
 	 * VTG is 1:1 mapping with OTG. Each pipe HUBP will select which VTG
 	 */
-	if (plane_state->update_flags.bits.full_update) {
+	if (plane_state->update_bits.full_update) {
 		hubp->funcs->hubp_vtg_sel(hubp, pipe_ctx->stream_res.tg->inst);
 
 		hubp->funcs->hubp_setup(
@@ -3113,26 +3113,26 @@ static void dcn10_update_dchubp_dpp(
 
 	size.surface_size = pipe_ctx->plane_res.scl_data.viewport;
 
-	if (plane_state->update_flags.bits.full_update ||
-		plane_state->update_flags.bits.bpp_change)
+	if (plane_state->update_bits.full_update ||
+		plane_state->update_bits.bpp_change)
 		dcn10_update_dpp(dpp, plane_state);
 
-	if (plane_state->update_flags.bits.full_update ||
-		plane_state->update_flags.bits.per_pixel_alpha_change ||
-		plane_state->update_flags.bits.global_alpha_change)
+	if (plane_state->update_bits.full_update ||
+		plane_state->update_bits.per_pixel_alpha_change ||
+		plane_state->update_bits.global_alpha_change)
 		hws->funcs.update_mpcc(dc, pipe_ctx);
 
-	if (plane_state->update_flags.bits.full_update ||
-		plane_state->update_flags.bits.per_pixel_alpha_change ||
-		plane_state->update_flags.bits.global_alpha_change ||
-		plane_state->update_flags.bits.scaling_change ||
-		plane_state->update_flags.bits.position_change) {
+	if (plane_state->update_bits.full_update ||
+		plane_state->update_bits.per_pixel_alpha_change ||
+		plane_state->update_bits.global_alpha_change ||
+		plane_state->update_bits.scaling_change ||
+		plane_state->update_bits.position_change) {
 		update_scaler(pipe_ctx);
 	}
 
-	if (plane_state->update_flags.bits.full_update ||
-		plane_state->update_flags.bits.scaling_change ||
-		plane_state->update_flags.bits.position_change) {
+	if (plane_state->update_bits.full_update ||
+		plane_state->update_bits.scaling_change ||
+		plane_state->update_bits.position_change) {
 		hubp->funcs->mem_program_viewport(
 			hubp,
 			&pipe_ctx->plane_res.scl_data.viewport,
@@ -3150,7 +3150,7 @@ static void dcn10_update_dchubp_dpp(
 			dc->hwss.set_cursor_sdr_white_level(pipe_ctx);
 	}
 
-	if (plane_state->update_flags.bits.full_update) {
+	if (plane_state->update_bits.full_update) {
 		/*gamut remap*/
 		dc->hwss.program_gamut_remap(pipe_ctx);
 
@@ -3161,15 +3161,15 @@ static void dcn10_update_dchubp_dpp(
 				pipe_ctx->stream_res.opp->inst);
 	}
 
-	if (plane_state->update_flags.bits.full_update ||
-		plane_state->update_flags.bits.pixel_format_change ||
-		plane_state->update_flags.bits.horizontal_mirror_change ||
-		plane_state->update_flags.bits.rotation_change ||
-		plane_state->update_flags.bits.swizzle_change ||
-		plane_state->update_flags.bits.dcc_change ||
-		plane_state->update_flags.bits.bpp_change ||
-		plane_state->update_flags.bits.scaling_change ||
-		plane_state->update_flags.bits.plane_size_change) {
+	if (plane_state->update_bits.full_update ||
+		plane_state->update_bits.pixel_format_change ||
+		plane_state->update_bits.horizontal_mirror_change ||
+		plane_state->update_bits.rotation_change ||
+		plane_state->update_bits.swizzle_change ||
+		plane_state->update_bits.dcc_change ||
+		plane_state->update_bits.bpp_change ||
+		plane_state->update_bits.scaling_change ||
+		plane_state->update_bits.plane_size_change) {
 		hubp->funcs->hubp_program_surface_config(
 			hubp,
 			plane_state->format,
@@ -3278,16 +3278,16 @@ void dcn10_program_pipe(
 		hws->funcs.blank_pixel_data(dc, pipe_ctx, blank);
 	}
 
-	if (pipe_ctx->plane_state->update_flags.bits.full_update)
+	if (pipe_ctx->plane_state->update_bits.full_update)
 		dcn10_enable_plane(dc, pipe_ctx, context);
 
 	dcn10_update_dchubp_dpp(dc, pipe_ctx, context);
 
 	hws->funcs.set_hdr_multiplier(pipe_ctx);
 
-	if (pipe_ctx->plane_state->update_flags.bits.full_update ||
-			pipe_ctx->plane_state->update_flags.bits.in_transfer_func_change ||
-			pipe_ctx->plane_state->update_flags.bits.gamma_change)
+	if (pipe_ctx->plane_state->update_bits.full_update ||
+			pipe_ctx->plane_state->update_bits.in_transfer_func_change ||
+			pipe_ctx->plane_state->update_bits.gamma_change)
 		hws->funcs.set_input_transfer_func(dc, pipe_ctx, pipe_ctx->plane_state);
 
 	/* dcn10_translate_regamma_to_hw_format takes 750us to finish
@@ -3296,7 +3296,7 @@ void dcn10_program_pipe(
 	 * Always call this for now since it does memcmp inside before
 	 * doing heavy calculation and programming
 	 */
-	if (pipe_ctx->plane_state->update_flags.bits.full_update)
+	if (pipe_ctx->plane_state->update_bits.full_update)
 		hws->funcs.set_output_transfer_func(dc, pipe_ctx, pipe_ctx->stream);
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c
index e6a8206f8ce0a..95e5b6a6ba0f8 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c
@@ -1066,11 +1066,11 @@ bool dcn20_set_blend_lut(
 	bool result = true;
 	const struct pwl_params *blend_lut = NULL;
 
-	if (plane_state->blend_tf.type == TF_TYPE_HWPWL)
-		blend_lut = &plane_state->blend_tf.pwl;
-	else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) {
+	if (plane_state->cm.blend_func.type == TF_TYPE_HWPWL)
+		blend_lut = &plane_state->cm.blend_func.pwl;
+	else if (plane_state->cm.blend_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 		cm_helper_translate_curve_to_hw_format(plane_state->ctx,
-				&plane_state->blend_tf,
+				&plane_state->cm.blend_func,
 				&dpp_base->regamma_params, false);
 		blend_lut = &dpp_base->regamma_params;
 	}
@@ -1086,19 +1086,19 @@ bool dcn20_set_shaper_3dlut(
 	bool result = true;
 	const struct pwl_params *shaper_lut = NULL;
 
-	if (plane_state->in_shaper_func.type == TF_TYPE_HWPWL)
-		shaper_lut = &plane_state->in_shaper_func.pwl;
-	else if (plane_state->in_shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
+	if (plane_state->cm.shaper_func.type == TF_TYPE_HWPWL)
+		shaper_lut = &plane_state->cm.shaper_func.pwl;
+	else if (plane_state->cm.shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 		cm_helper_translate_curve_to_hw_format(plane_state->ctx,
-				&plane_state->in_shaper_func,
+				&plane_state->cm.shaper_func,
 				&dpp_base->shaper_params, true);
 		shaper_lut = &dpp_base->shaper_params;
 	}
 
 	result = dpp_base->funcs->dpp_program_shaper_lut(dpp_base, shaper_lut);
-	if (plane_state->lut3d_func.state.bits.initialized == 1)
+	if (plane_state->cm.lut3d_func.state.bits.initialized == 1)
 		result = dpp_base->funcs->dpp_program_3dlut(dpp_base,
-								&plane_state->lut3d_func.lut_3d);
+								&plane_state->cm.lut3d_func.lut_3d);
 	else
 		result = dpp_base->funcs->dpp_program_3dlut(dpp_base, NULL);
 
@@ -1733,10 +1733,10 @@ void dcn20_update_dchubp_dpp(
 
 	if (pipe_ctx->update_flags.bits.enable ||
 			pipe_ctx->update_flags.bits.plane_changed ||
-			plane_state->update_flags.bits.bpp_change ||
-			plane_state->update_flags.bits.input_csc_change ||
-			plane_state->update_flags.bits.color_space_change ||
-			plane_state->update_flags.bits.coeff_reduction_change) {
+			plane_state->update_bits.bpp_change ||
+			plane_state->update_bits.input_csc_change ||
+			plane_state->update_bits.color_space_change ||
+			plane_state->update_bits.coeff_reduction_change) {
 		struct dc_bias_and_scale bns_params = plane_state->bias_and_scale;
 
 		// program the input csc
@@ -1760,16 +1760,16 @@ void dcn20_update_dchubp_dpp(
 
 	if (pipe_ctx->update_flags.bits.mpcc
 			|| pipe_ctx->update_flags.bits.plane_changed
-			|| plane_state->update_flags.bits.global_alpha_change
-			|| plane_state->update_flags.bits.per_pixel_alpha_change) {
+			|| plane_state->update_bits.global_alpha_change
+			|| plane_state->update_bits.per_pixel_alpha_change) {
 		// MPCC inst is equal to pipe index in practice
 		hws->funcs.update_mpcc(dc, pipe_ctx);
 	}
 
 	if (pipe_ctx->update_flags.bits.scaler ||
-			plane_state->update_flags.bits.scaling_change ||
-			plane_state->update_flags.bits.position_change ||
-			plane_state->update_flags.bits.per_pixel_alpha_change ||
+			plane_state->update_bits.scaling_change ||
+			plane_state->update_bits.position_change ||
+			plane_state->update_bits.per_pixel_alpha_change ||
 			pipe_ctx->stream->update_flags.bits.scaling) {
 		pipe_ctx->plane_res.scl_data.lb_params.alpha_en = pipe_ctx->plane_state->per_pixel_alpha;
 		ASSERT(pipe_ctx->plane_res.scl_data.lb_params.depth == LB_PIXEL_DEPTH_36BPP);
@@ -1779,8 +1779,8 @@ void dcn20_update_dchubp_dpp(
 	}
 
 	if (pipe_ctx->update_flags.bits.viewport ||
-			(context == dc->current_state && plane_state->update_flags.bits.position_change) ||
-			(context == dc->current_state && plane_state->update_flags.bits.scaling_change) ||
+			(context == dc->current_state && plane_state->update_bits.position_change) ||
+			(context == dc->current_state && plane_state->update_bits.scaling_change) ||
 			(context == dc->current_state && pipe_ctx->stream->update_flags.bits.scaling)) {
 
 		hubp->funcs->mem_program_viewport(
@@ -1812,7 +1812,7 @@ void dcn20_update_dchubp_dpp(
 	if (pipe_ctx->update_flags.bits.enable || pipe_ctx->update_flags.bits.opp_changed
 			|| pipe_ctx->update_flags.bits.plane_changed
 			|| pipe_ctx->stream->update_flags.bits.gamut_remap
-			|| plane_state->update_flags.bits.gamut_remap_change
+			|| plane_state->update_bits.gamut_remap_change
 			|| pipe_ctx->stream->update_flags.bits.out_csc) {
 		/* dpp/cm gamut remap*/
 		dc->hwss.program_gamut_remap(pipe_ctx);
@@ -1828,14 +1828,14 @@ void dcn20_update_dchubp_dpp(
 	if (pipe_ctx->update_flags.bits.enable ||
 			pipe_ctx->update_flags.bits.plane_changed ||
 			pipe_ctx->update_flags.bits.opp_changed ||
-			plane_state->update_flags.bits.pixel_format_change ||
-			plane_state->update_flags.bits.horizontal_mirror_change ||
-			plane_state->update_flags.bits.rotation_change ||
-			plane_state->update_flags.bits.swizzle_change ||
-			plane_state->update_flags.bits.dcc_change ||
-			plane_state->update_flags.bits.bpp_change ||
-			plane_state->update_flags.bits.scaling_change ||
-			plane_state->update_flags.bits.plane_size_change) {
+			plane_state->update_bits.pixel_format_change ||
+			plane_state->update_bits.horizontal_mirror_change ||
+			plane_state->update_bits.rotation_change ||
+			plane_state->update_bits.swizzle_change ||
+			plane_state->update_bits.dcc_change ||
+			plane_state->update_bits.bpp_change ||
+			plane_state->update_bits.scaling_change ||
+			plane_state->update_bits.plane_size_change) {
 		struct plane_size size = plane_state->plane_size;
 
 		size.surface_size = pipe_ctx->plane_res.scl_data.viewport;
@@ -1853,7 +1853,7 @@ void dcn20_update_dchubp_dpp(
 
 	if (pipe_ctx->update_flags.bits.enable ||
 		pipe_ctx->update_flags.bits.plane_changed ||
-		plane_state->update_flags.bits.addr_update) {
+		plane_state->update_bits.addr_update) {
 		if (resource_is_pipe_type(pipe_ctx, OTG_MASTER) &&
 				pipe_mall_type == SUBVP_MAIN) {
 			union block_sequence_params params;
@@ -1969,18 +1969,18 @@ static void dcn20_program_pipe(
 	}
 
 	if (pipe_ctx->plane_state && (pipe_ctx->update_flags.raw ||
-	    pipe_ctx->plane_state->update_flags.raw ||
+	    dc_pipe_update_bits_is_any_set(&pipe_ctx->plane_state->update_bits) ||
 	    pipe_ctx->stream->update_flags.raw))
 		dcn20_update_dchubp_dpp(dc, pipe_ctx, context);
 
 	if (pipe_ctx->plane_state && (pipe_ctx->update_flags.bits.enable ||
-		pipe_ctx->plane_state->update_flags.bits.hdr_mult))
+		pipe_ctx->plane_state->update_bits.hdr_mult))
 		hws->funcs.set_hdr_multiplier(pipe_ctx);
 
 	if (pipe_ctx->plane_state &&
-		(pipe_ctx->plane_state->update_flags.bits.in_transfer_func_change ||
-			pipe_ctx->plane_state->update_flags.bits.gamma_change ||
-			pipe_ctx->plane_state->update_flags.bits.lut_3d ||
+		(pipe_ctx->plane_state->update_bits.in_transfer_func_change ||
+			pipe_ctx->plane_state->update_bits.gamma_change ||
+			pipe_ctx->plane_state->update_bits.lut_3d ||
 			pipe_ctx->update_flags.bits.enable))
 		hws->funcs.set_input_transfer_func(dc, pipe_ctx, pipe_ctx->plane_state);
 
@@ -2186,7 +2186,7 @@ void dcn20_program_front_end_for_ctx(
 		pipe = &context->res_ctx.pipe_ctx[i];
 		if (!pipe->top_pipe && !pipe->prev_odm_pipe
 			&& pipe->stream && pipe->stream->num_wb_info > 0
-			&& (pipe->update_flags.raw || (pipe->plane_state && pipe->plane_state->update_flags.raw)
+			&& (pipe->update_flags.raw || (pipe->plane_state && dc_pipe_update_bits_is_any_set(&pipe->plane_state->update_bits))
 				|| pipe->stream->update_flags.raw)
 			&& hws->funcs.program_all_writeback_pipes_in_tree)
 			hws->funcs.program_all_writeback_pipes_in_tree(dc, pipe->stream, context);
@@ -2998,7 +2998,7 @@ void dcn20_update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
 	mpcc_id = hubp->inst;
 
 	/* If there is no full update, don't need to touch MPC tree*/
-	if (!pipe_ctx->plane_state->update_flags.bits.full_update &&
+	if (!pipe_ctx->plane_state->update_bits.full_update &&
 		!pipe_ctx->update_flags.bits.mpcc) {
 		mpc->funcs->update_blending(mpc, &blnd_cfg, mpcc_id);
 		dc->hwss.update_visual_confirm_color(dc, pipe_ctx, mpcc_id);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.c
index ce18d75fd9918..7b820bdae55b4 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.c
@@ -485,7 +485,7 @@ void dcn201_update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
 	mpcc_id = dpp_id;
 
 	/* If there is no full update, don't need to touch MPC tree*/
-	if (!pipe_ctx->plane_state->update_flags.bits.full_update) {
+	if (!pipe_ctx->plane_state->update_bits.full_update) {
 		dc->hwss.update_visual_confirm_color(dc, pipe_ctx, mpcc_id);
 		mpc->funcs->update_blending(mpc, &blnd_cfg, mpcc_id);
 		return;
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.c
index 476c2112afec1..2be68f8dd0fc4 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.c
@@ -136,6 +136,26 @@ void dcn21_PLAT_58856_wa(struct dc_state *context, struct pipe_ctx *pipe_ctx)
 	pipe_ctx->stream->dpms_off = true;
 }
 
+bool dcn21_dmub_cacp_set_pipe(struct abm *abm, uint32_t otg_inst,
+		uint32_t option, uint32_t panel_inst, uint32_t pwrseq_inst)
+{
+	union dmub_rb_cmd cmd;
+	struct dc_context *dc = abm->ctx;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.cacp_set_pipe.header.type = DMUB_CMD__CACP;
+	cmd.cacp_set_pipe.header.sub_type = DMUB_CMD__CACP_SET_PIPE;
+	cmd.cacp_set_pipe.cacp_set_pipe_data.otg_inst = (uint8_t)otg_inst;
+	cmd.cacp_set_pipe.cacp_set_pipe_data.pwrseq_inst = (uint8_t)pwrseq_inst;
+	cmd.cacp_set_pipe.cacp_set_pipe_data.set_pipe_option = (uint8_t)option;
+	cmd.cacp_set_pipe.cacp_set_pipe_data.panel_inst = (uint8_t)panel_inst;
+	cmd.cacp_set_pipe.header.payload_bytes = sizeof(struct dmub_cmd_cacp_set_pipe_data);
+
+	dc_wake_and_execute_dmub_cmd(dc, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+
+	return true;
+}
+
 bool dcn21_dmub_abm_set_pipe(struct abm *abm, uint32_t otg_inst,
 		uint32_t option, uint32_t panel_inst, uint32_t pwrseq_inst)
 {
@@ -181,10 +201,12 @@ void dcn21_set_abm_immediate_disable(struct pipe_ctx *pipe_ctx)
 	uint32_t otg_inst = pipe_ctx->stream_res.tg->inst;
 	struct panel_cntl *panel_cntl = pipe_ctx->stream->link->panel_cntl;
 	struct dmcu *dmcu = pipe_ctx->stream->ctx->dc->res_pool->dmcu;
+	struct dc_link *link = pipe_ctx->stream->link;
 
 	// make a short term w/a for an issue that backlight ramping unexpectedly paused in the middle,
 	// will decouple backlight from ABM and redefine DMUB interface, then this w/a could be removed
-	if (pipe_ctx->stream->abm_level == 0 || pipe_ctx->stream->abm_level == ABM_LEVEL_IMMEDIATE_DISABLE) {
+	if ((pipe_ctx->stream->abm_level == 0 || pipe_ctx->stream->abm_level == ABM_LEVEL_IMMEDIATE_DISABLE)
+		&& (link && !link->panel_config.cacp.cacp_supported)) {
 		return;
 	}
 
@@ -197,6 +219,9 @@ void dcn21_set_abm_immediate_disable(struct pipe_ctx *pipe_ctx)
 		if (abm->funcs && abm->funcs->set_pipe_ex) {
 			abm->funcs->set_pipe_ex(abm, otg_inst, SET_ABM_PIPE_IMMEDIATELY_DISABLE,
 					panel_cntl->inst, panel_cntl->pwrseq_inst);
+		} else if (link && link->panel_config.cacp.cacp_supported) {
+			dcn21_dmub_cacp_set_pipe(abm, otg_inst, SET_CACP_PIPE_IMMEDIATELY_DISABLE,
+						panel_cntl->inst, panel_cntl->pwrseq_inst);
 		} else {
 			dcn21_dmub_abm_set_pipe(abm,
 						otg_inst,
@@ -214,6 +239,7 @@ void dcn21_set_pipe(struct pipe_ctx *pipe_ctx)
 	struct timing_generator *tg = pipe_ctx->stream_res.tg;
 	struct panel_cntl *panel_cntl = pipe_ctx->stream->link->panel_cntl;
 	struct dmcu *dmcu = pipe_ctx->stream->ctx->dc->res_pool->dmcu;
+	struct dc_link *link = pipe_ctx->stream->link;
 	uint32_t otg_inst;
 
 	if (!abm || !tg || !panel_cntl)
@@ -233,6 +259,11 @@ void dcn21_set_pipe(struct pipe_ctx *pipe_ctx)
 					panel_cntl->inst,
 					panel_cntl->pwrseq_inst);
 	} else {
+
+		if (link && link->panel_config.cacp.cacp_supported)
+			dcn21_dmub_cacp_set_pipe(abm, otg_inst, SET_CACP_PIPE_NORMAL,
+					panel_cntl->inst, panel_cntl->pwrseq_inst);
+		else
 			dcn21_dmub_abm_set_pipe(abm, otg_inst,
 				  SET_ABM_PIPE_NORMAL,
 				  panel_cntl->inst,
@@ -250,6 +281,7 @@ bool dcn21_set_backlight_level(struct pipe_ctx *pipe_ctx,
 	uint32_t otg_inst;
 	uint32_t backlight_pwm_u16_16 = backlight_level_params->backlight_pwm_u16_16;
 	uint32_t frame_ramp = backlight_level_params->frame_ramp;
+	struct dc_link *link = pipe_ctx->stream->link;
 
 	if (!abm || !tg || !panel_cntl)
 		return false;
@@ -268,6 +300,12 @@ bool dcn21_set_backlight_level(struct pipe_ctx *pipe_ctx,
 					panel_cntl->inst,
 					panel_cntl->pwrseq_inst);
 	} else {
+		if (link && link->panel_config.cacp.cacp_supported)
+			dcn21_dmub_cacp_set_pipe(abm, otg_inst,
+					SET_CACP_PIPE_NORMAL,
+					panel_cntl->inst,
+					panel_cntl->pwrseq_inst);
+		else
 			dcn21_dmub_abm_set_pipe(abm,
 				  otg_inst,
 				  SET_ABM_PIPE_NORMAL,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.h
index f72a27ac1bf1b..aae13447cfa88 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.h
@@ -47,6 +47,8 @@ void dcn21_optimize_pwr_state(
 void dcn21_PLAT_58856_wa(struct dc_state *context,
 		struct pipe_ctx *pipe_ctx);
 
+bool dcn21_dmub_cacp_set_pipe(struct abm *abm, uint32_t otg_inst,
+	uint32_t option, uint32_t panel_inst, uint32_t pwrseq_inst);
 bool dcn21_dmub_abm_set_pipe(struct abm *abm, uint32_t otg_inst,
 	uint32_t option, uint32_t panel_inst, uint32_t pwrseq_inst);
 void dcn21_set_pipe(struct pipe_ctx *pipe_ctx);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c
index a7c85a2302aba..aed9d06ec538d 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c
@@ -239,11 +239,13 @@ bool dcn30_set_blend_lut(
 	bool result = true;
 	const struct pwl_params *blend_lut = NULL;
 
-	if (plane_state->blend_tf.type == TF_TYPE_HWPWL)
-		blend_lut = &plane_state->blend_tf.pwl;
-	else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) {
+	if (plane_state->cm.blend_func.type == TF_TYPE_HWPWL)
+		blend_lut = &plane_state->cm.blend_func.pwl;
+	else if (plane_state->cm.blend_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 		result = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
-				&plane_state->blend_tf, &dpp_base->regamma_params, false);
+				&plane_state->cm.blend_func,
+				&dpp_base->regamma_params,
+				false);
 		if (!result)
 			return result;
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c
index 43e4edfe9182e..6f1ad651ed2ca 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c
@@ -54,6 +54,7 @@
 #include "dce/dce_i2c_hw.h"
 #include "dce/dmub_abm_lcd.h"
 #include "dio/dcn10/dcn10_dio.h"
+#include "dce/dmub_abm_cacp.h"
 
 #define DC_LOGGER_INIT(logger)
 
@@ -536,10 +537,12 @@ static void dcn31_reset_back_end_for_pipe(
 	}
 	ASSERT(!pipe_ctx->top_pipe);
 
-	dc->hwss.set_abm_immediate_disable(pipe_ctx);
-
 	link = pipe_ctx->stream->link;
 
+	if (!(link->connector_signal == SIGNAL_TYPE_EDP &&
+	      link->skip_implict_edp_power_control))
+		dc->hwss.set_abm_immediate_disable(pipe_ctx);
+
 	if (dc->hwseq)
 		dc->hwseq->wa_state.skip_blank_stream = false;
 
@@ -554,9 +557,11 @@ static void dcn31_reset_back_end_for_pipe(
 			pipe_ctx->stream_res.tg,
 			OPTC_DSC_DISABLED, 0, 0);
 
-	pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg);
-
-	pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false);
+	if (!(link->connector_signal == SIGNAL_TYPE_EDP &&
+	      link->skip_implict_edp_power_control)) {
+		pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg);
+		pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false);
+	}
 	if (pipe_ctx->stream_res.tg->funcs->set_odm_bypass)
 		pipe_ctx->stream_res.tg->funcs->set_odm_bypass(
 				pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing);
@@ -585,7 +590,12 @@ static void dcn31_reset_back_end_for_pipe(
 	 * screen only, the dpms_off would be true but
 	 * VBIOS lit up eDP, so check link status too.
 	 */
-	if (!pipe_ctx->stream->dpms_off || link->link_status.link_active)
+	if (link->connector_signal == SIGNAL_TYPE_EDP &&
+	    link->skip_implict_edp_power_control) {
+		/* DMSS is holding the panel across the commit; skip dpms-off. */
+		if (pipe_ctx->stream_res.audio)
+			dc->hwss.disable_audio_stream(pipe_ctx);
+	} else if (!pipe_ctx->stream->dpms_off || link->link_status.link_active)
 		dc->link_srv->set_dpms_off(pipe_ctx);
 	else if (pipe_ctx->stream_res.audio)
 		dc->hwss.disable_audio_stream(pipe_ctx);
@@ -718,6 +728,34 @@ static void dmub_abm_set_backlight(struct dc_context *dc,
 	dc_wake_and_execute_dmub_cmd(dc, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 }
 
+static bool dmub_cacp_set_backlight(struct dc_context *dc,
+	struct set_backlight_level_params *backlight_level_params,
+	unsigned int panel_inst)
+{
+	union dmub_rb_cmd cmd;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.cacp_set_backlight.header.type = DMUB_CMD__CACP;
+	cmd.cacp_set_backlight.header.sub_type = DMUB_CMD__CACP_SET_BACKLIGHT;
+	cmd.cacp_set_backlight.cacp_set_backlight_data.aux_inst = backlight_level_params->aux_inst;
+	cmd.cacp_set_backlight.cacp_set_backlight_data.frame_ramp = backlight_level_params->frame_ramp;
+	cmd.cacp_set_backlight.cacp_set_backlight_data.backlight_user_level =
+		backlight_level_params->backlight_pwm_u16_16;
+	cmd.cacp_set_backlight.cacp_set_backlight_data.backlight_control_type =
+		(enum dmub_backlight_control_type)backlight_level_params->control_type;
+	cmd.cacp_set_backlight.cacp_set_backlight_data.min_luminance = backlight_level_params->min_luminance;
+	cmd.cacp_set_backlight.cacp_set_backlight_data.max_luminance = backlight_level_params->max_luminance;
+	cmd.cacp_set_backlight.cacp_set_backlight_data.min_backlight_pwm = backlight_level_params->min_backlight_pwm;
+	cmd.cacp_set_backlight.cacp_set_backlight_data.max_backlight_pwm = backlight_level_params->max_backlight_pwm;
+	cmd.cacp_set_backlight.cacp_set_backlight_data.version = DMUB_CMD_CACP_CONTROL_VERSION_1;
+	cmd.cacp_set_backlight.cacp_set_backlight_data.panel_mask = (0x01 << panel_inst);
+	cmd.cacp_set_backlight.header.payload_bytes = sizeof(struct dmub_cmd_cacp_set_backlight_data);
+
+	dc_wake_and_execute_dmub_cmd(dc, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+
+	return true;
+}
+
 bool dcn31_set_backlight_level(struct pipe_ctx *pipe_ctx,
 	struct set_backlight_level_params *backlight_level_params)
 {
@@ -726,20 +764,38 @@ bool dcn31_set_backlight_level(struct pipe_ctx *pipe_ctx,
 	struct timing_generator *tg = pipe_ctx->stream_res.tg;
 	struct panel_cntl *panel_cntl = pipe_ctx->stream->link->panel_cntl;
 	uint32_t otg_inst;
+	struct dc_link *link = pipe_ctx->stream->link;
 
 	if (!abm || !tg || !panel_cntl)
 		return false;
 
 	otg_inst = tg->inst;
 
+	if (link && link->panel_config.cacp.cacp_supported)
+		dcn21_dmub_cacp_set_pipe(abm, otg_inst,
+			SET_CACP_PIPE_NORMAL,
+			panel_cntl->inst,
+			panel_cntl->pwrseq_inst);
+	else
 		dcn21_dmub_abm_set_pipe(abm,
 			otg_inst,
 			SET_ABM_PIPE_NORMAL,
 			panel_cntl->inst,
 			panel_cntl->pwrseq_inst);
 
-	if (backlight_level_params->control_type != BACKLIGHT_CONTROL_AMD_AUX)
+	if (link && link->panel_type == PANEL_TYPE_OLED) {
+		/* For OLED panel with AMD AUX, skip set backlight call */
+		if (backlight_level_params->control_type == BACKLIGHT_CONTROL_VESA_AUX)
+			dmub_cacp_set_backlight(dc, backlight_level_params, panel_cntl->inst);
+	} else if (link && link->panel_type == PANEL_TYPE_MINILED) {
+		/* For MiniLED panel we need to check if CACP or ABM is being used */
+		if (link->panel_config.cacp.cacp_supported)
+			dmub_cacp_set_backlight(dc, backlight_level_params, panel_cntl->inst);
+		else
+			dmub_abm_set_backlight(dc, backlight_level_params, panel_cntl->inst);
+	} else {
 		dmub_abm_set_backlight(dc, backlight_level_params, panel_cntl->inst);
+	}
 
 	return true;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
index a3242e7521a48..c2ea259277656 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
@@ -490,12 +490,12 @@ bool dcn32_set_mcm_luts(
 	const struct pwl_params *lut_params = NULL;
 
 	// 1D LUT
-	if (plane_state->blend_tf.type == TF_TYPE_HWPWL)
-		lut_params = &plane_state->blend_tf.pwl;
-	else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) {
-		result = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
-								 &plane_state->blend_tf,
-								 &dpp_base->regamma_params, false);
+	if (plane_state->cm.blend_func.type == TF_TYPE_HWPWL)
+		lut_params = &plane_state->cm.blend_func.pwl;
+	else if (plane_state->cm.blend_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
+		result = cm3_helper_translate_curve_to_degamma_hw_format(
+			&plane_state->cm.blend_func,
+			&dpp_base->regamma_params);
 		if (!result)
 			return result;
 
@@ -505,21 +505,22 @@ bool dcn32_set_mcm_luts(
 	lut_params = NULL;
 
 	// Shaper
-	if (plane_state->in_shaper_func.type == TF_TYPE_HWPWL)
-		lut_params = &plane_state->in_shaper_func.pwl;
-	else if (plane_state->in_shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
+	if (plane_state->cm.shaper_func.type == TF_TYPE_HWPWL)
+		lut_params = &plane_state->cm.shaper_func.pwl;
+	else if (plane_state->cm.shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 		// TODO: dpp_base replace
 		rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
-							&plane_state->in_shaper_func,
-							&dpp_base->shaper_params, true);
+			&plane_state->cm.shaper_func,
+			&dpp_base->shaper_params,
+			true);
 		lut_params = rval ? &dpp_base->shaper_params : NULL;
 	}
 
 	mpc->funcs->program_shaper(mpc, lut_params, mpcc_id);
 
 	// 3D
-	if (plane_state->lut3d_func.state.bits.initialized == 1)
-		result = mpc->funcs->program_3dlut(mpc, &plane_state->lut3d_func.lut_3d, mpcc_id);
+	if (plane_state->cm.lut3d_func.state.bits.initialized == 1)
+		result = mpc->funcs->program_3dlut(mpc, &plane_state->cm.lut3d_func.lut_3d, mpcc_id);
 	else
 		result = mpc->funcs->program_3dlut(mpc, NULL, mpcc_id);
 
@@ -551,9 +552,8 @@ bool dcn32_set_input_transfer_func(struct dc *dc,
 	if (plane_state->in_transfer_func.type == TF_TYPE_HWPWL)
 		params = &plane_state->in_transfer_func.pwl;
 	else if (plane_state->in_transfer_func.type == TF_TYPE_DISTRIBUTED_POINTS &&
-		cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
-							&plane_state->in_transfer_func,
-							&dpp_base->degamma_params, false))
+		cm3_helper_translate_curve_to_degamma_hw_format(&plane_state->in_transfer_func,
+								&dpp_base->degamma_params))
 		params = &dpp_base->degamma_params;
 
 	dpp_base->funcs->dpp_program_gamcor_lut(dpp_base, params);
@@ -1463,7 +1463,7 @@ void dcn32_update_phantom_vp_position(struct dc *dc,
 
 		if (pipe->stream && dc_state_get_pipe_subvp_type(context, pipe) == SUBVP_MAIN &&
 				dc_state_get_paired_subvp_stream(context, pipe->stream) == phantom_pipe->stream) {
-			if (pipe->plane_state && pipe->plane_state->update_flags.bits.position_change) {
+			if (pipe->plane_state && pipe->plane_state->update_bits.position_change) {
 
 				phantom_plane->src_rect.x = pipe->plane_state->src_rect.x;
 				phantom_plane->src_rect.y = pipe->plane_state->src_rect.y;
@@ -1471,7 +1471,7 @@ void dcn32_update_phantom_vp_position(struct dc *dc,
 				phantom_plane->dst_rect.x = pipe->plane_state->dst_rect.x;
 				phantom_plane->dst_rect.y = pipe->plane_state->dst_rect.y;
 
-				phantom_pipe->plane_state->update_flags.bits.position_change = 1;
+				phantom_pipe->plane_state->update_bits.position_change = 1;
 				resource_build_scaling_params(phantom_pipe);
 				return;
 			}
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
index 8f9038fec0f7b..4ed7480d1efa6 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
@@ -487,7 +487,7 @@ void dcn35_update_odm(struct dc *dc, struct dc_state *context, struct pipe_ctx *
 
 void dcn35_dpp_root_clock_control(struct dce_hwseq *hws, unsigned int dpp_inst, bool clock_on)
 {
-	if (!hws->ctx->dc->debug.root_clock_optimization.bits.dpp)
+	if (!hws->ctx->dc->debug.root_clock_optimization.bits.dpp && !clock_on)
 		return;
 
 	if (hws->ctx->dc->res_pool->dccg->funcs->dpp_root_clock_control) {
@@ -498,7 +498,7 @@ void dcn35_dpp_root_clock_control(struct dce_hwseq *hws, unsigned int dpp_inst,
 
 void dcn35_dpstream_root_clock_control(struct dce_hwseq *hws, unsigned int dp_hpo_inst, bool clock_on)
 {
-	if (!hws->ctx->dc->debug.root_clock_optimization.bits.dpstream)
+	if (!hws->ctx->dc->debug.root_clock_optimization.bits.dpstream && !clock_on)
 		return;
 
 	if (hws->ctx->dc->res_pool->dccg->funcs->set_dpstreamclk_root_clock_gating) {
@@ -509,7 +509,7 @@ void dcn35_dpstream_root_clock_control(struct dce_hwseq *hws, unsigned int dp_hp
 
 void dcn35_hdmistream_root_clock_control(struct dce_hwseq *hws, bool clock_on)
 {
-	if (!hws->ctx->dc->debug.root_clock_optimization.bits.hdmistream)
+	if (!hws->ctx->dc->debug.root_clock_optimization.bits.hdmistream && !clock_on)
 		return;
 
 	if (hws->ctx->dc->res_pool->dccg->funcs->set_hdmistreamclk_root_clock_gating) {
@@ -520,7 +520,7 @@ void dcn35_hdmistream_root_clock_control(struct dce_hwseq *hws, bool clock_on)
 
 void dcn35_physymclk_root_clock_control(struct dce_hwseq *hws, unsigned int phy_inst, bool clock_on)
 {
-	if (!hws->ctx->dc->debug.root_clock_optimization.bits.physymclk)
+	if (!hws->ctx->dc->debug.root_clock_optimization.bits.physymclk && !clock_on)
 		return;
 
 	if (hws->ctx->dc->res_pool->dccg->funcs->set_physymclk_root_clock_gating) {
@@ -581,9 +581,6 @@ void dcn35_power_down_on_boot(struct dc *dc)
 
 bool dcn35_apply_idle_power_optimizations(struct dc *dc, bool enable)
 {
-	if (dc->debug.dmcub_emulation)
-		return true;
-
 	if (enable) {
 		uint32_t num_active_edp = 0;
 		int i;
@@ -1821,8 +1818,11 @@ void dcn35_disable_link_output(struct dc_link *link,
 		disable_link_output_symclk_on_tx_off(link, DP_UNKNOWN_ENCODING);
 		link->phy_state.symclk_state = SYMCLK_ON_TX_OFF;
 	} else {
-		link_hwss->disable_link_output(link, link_res, signal);
-		link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
+		if (!(signal == SIGNAL_TYPE_EDP &&
+		      link->skip_implict_edp_power_control)) {
+			link_hwss->disable_link_output(link, link_res, signal);
+			link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
+		}
 	}
 	/*
 	 * Add the logic to extract BOTH power up and power down sequences
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
index 96815a92a6298..d8d3208fc1af9 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
@@ -321,7 +321,7 @@ void dcn401_init_hw(struct dc *dc)
 			user_level = link->panel_cntl->stored_backlight_registers.USER_LEVEL;
 		}
 
-		if (link->ctx->dc->config.dp_connector_no_native_i2c && link->no_ddc_pin) {
+		if (link->force_to_use_aux) {
 			struct graphics_object_i2c_info i2c_info;
 			struct ddc *ddc_pin;
 			struct gpio_ddc_hw_info hw_info;
@@ -410,37 +410,27 @@ static void dcn401_get_mcm_lut_xable_from_pipe_ctx(struct dc *dc, struct pipe_ct
 		enum MCM_LUT_XABLE *lut3d_xable,
 		enum MCM_LUT_XABLE *lut1d_xable)
 {
-	enum dc_cm2_shaper_3dlut_setting shaper_3dlut_setting = DC_CM2_SHAPER_3DLUT_SETTING_BYPASS_ALL;
-	bool lut1d_enable = false;
 	struct mpc *mpc = dc->res_pool->mpc;
 	int mpcc_id = pipe_ctx->plane_res.hubp->inst;
 
 	if (!pipe_ctx->plane_state)
 		return;
-	shaper_3dlut_setting = pipe_ctx->plane_state->mcm_shaper_3dlut_setting;
-	lut1d_enable = pipe_ctx->plane_state->mcm_lut1d_enable;
+
 	mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id);
 	pipe_ctx->plane_state->mcm_location = MPCC_MOVABLE_CM_LOCATION_BEFORE;
 
-	*lut1d_xable = lut1d_enable ? MCM_LUT_ENABLE : MCM_LUT_DISABLE;
-
-	switch (shaper_3dlut_setting) {
-	case DC_CM2_SHAPER_3DLUT_SETTING_BYPASS_ALL:
-		*lut3d_xable = *shaper_xable = MCM_LUT_DISABLE;
-		break;
-	case DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER:
-		*lut3d_xable = MCM_LUT_DISABLE;
-		*shaper_xable = MCM_LUT_ENABLE;
-		break;
-	case DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER_3DLUT:
-		*lut3d_xable = *shaper_xable = MCM_LUT_ENABLE;
-		break;
-	}
+	*lut1d_xable = pipe_ctx->plane_state->cm.flags.bits.blend_enable ?
+		MCM_LUT_ENABLE : MCM_LUT_DISABLE;
+	*shaper_xable = pipe_ctx->plane_state->cm.flags.bits.shaper_enable ?
+		MCM_LUT_ENABLE : MCM_LUT_DISABLE;
+	*lut3d_xable = (pipe_ctx->plane_state->cm.flags.bits.shaper_enable &&
+			pipe_ctx->plane_state->cm.flags.bits.lut3d_enable) ?
+		MCM_LUT_ENABLE : MCM_LUT_DISABLE;
 }
 
 void dcn401_populate_mcm_luts(struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
-		struct dc_cm2_func_luts mcm_luts,
+		const struct dc_plane_cm *cm,
 		bool lut_bank_a)
 {
 	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
@@ -448,14 +438,17 @@ void dcn401_populate_mcm_luts(struct dc *dc,
 	int mpcc_id = hubp->inst;
 	struct mpc *mpc = dc->res_pool->mpc;
 	union mcm_lut_params m_lut_params;
-	enum dc_cm2_transfer_func_source lut3d_src = mcm_luts.lut3d_data.lut3d_src;
+	const bool lut3d_dma = !!cm->flags.bits.lut3d_dma_enable;
 	enum hubp_3dlut_fl_format format = 0;
 	enum hubp_3dlut_fl_mode mode;
-	enum hubp_3dlut_fl_width width = 0;
+	/* Width was previously hard-coded to TRANSFORMED via local_mcm build,
+	 * preserve identical behavior.
+	 */
+	enum hubp_3dlut_fl_width width = hubp_3dlut_fl_width_transformed;
 	enum hubp_3dlut_fl_addressing_mode addr_mode;
-	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_y_g = 0;
-	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cb_b = 0;
-	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cr_r = 0;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_y_g;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cb_b;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cr_r;
 	enum MCM_LUT_XABLE shaper_xable = MCM_LUT_DISABLE;
 	enum MCM_LUT_XABLE lut3d_xable = MCM_LUT_DISABLE;
 	enum MCM_LUT_XABLE lut1d_xable = MCM_LUT_DISABLE;
@@ -464,13 +457,13 @@ void dcn401_populate_mcm_luts(struct dc *dc,
 	dcn401_get_mcm_lut_xable_from_pipe_ctx(dc, pipe_ctx, &shaper_xable, &lut3d_xable, &lut1d_xable);
 
 	/* 1D LUT */
-	if (mcm_luts.lut1d_func) {
+	{
 		memset(&m_lut_params, 0, sizeof(m_lut_params));
-		if (mcm_luts.lut1d_func->type == TF_TYPE_HWPWL)
-			m_lut_params.pwl = &mcm_luts.lut1d_func->pwl;
-		else if (mcm_luts.lut1d_func->type == TF_TYPE_DISTRIBUTED_POINTS) {
+		if (cm->blend_func.type == TF_TYPE_HWPWL)
+			m_lut_params.pwl = &cm->blend_func.pwl;
+		else if (cm->blend_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 			rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx,
-					mcm_luts.lut1d_func,
+					&cm->blend_func,
 					&dpp_base->regamma_params, false);
 			m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL;
 		}
@@ -483,14 +476,14 @@ void dcn401_populate_mcm_luts(struct dc *dc,
 	}
 
 	/* Shaper */
-	if (mcm_luts.shaper && mcm_luts.lut3d_data.mpc_3dlut_enable) {
+	if (cm->flags.bits.lut3d_enable) {
 		memset(&m_lut_params, 0, sizeof(m_lut_params));
-		if (mcm_luts.shaper->type == TF_TYPE_HWPWL)
-			m_lut_params.pwl = &mcm_luts.shaper->pwl;
-		else if (mcm_luts.shaper->type == TF_TYPE_DISTRIBUTED_POINTS) {
+		if (cm->shaper_func.type == TF_TYPE_HWPWL)
+			m_lut_params.pwl = &cm->shaper_func.pwl;
+		else if (cm->shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 			ASSERT(false);
 			rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx,
-					mcm_luts.shaper,
+					&cm->shaper_func,
 					&dpp_base->regamma_params, true);
 			m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL;
 		}
@@ -503,42 +496,43 @@ void dcn401_populate_mcm_luts(struct dc *dc,
 	}
 
 	/* 3DLUT */
-	switch (lut3d_src) {
-	case DC_CM2_TRANSFER_FUNC_SOURCE_SYSMEM:
+	if (!lut3d_dma) {
+		/* SYSMEM (legacy lut3d_func) */
 		memset(&m_lut_params, 0, sizeof(m_lut_params));
 		if (hubp->funcs->hubp_enable_3dlut_fl)
 			hubp->funcs->hubp_enable_3dlut_fl(hubp, false);
 
-		if (mcm_luts.lut3d_data.lut3d_func && mcm_luts.lut3d_data.lut3d_func->state.bits.initialized) {
-			m_lut_params.lut3d = &mcm_luts.lut3d_data.lut3d_func->lut_3d;
+		if (cm->lut3d_func.state.bits.initialized) {
+			m_lut_params.lut3d = &cm->lut3d_func.lut_3d;
 			if (mpc->funcs->populate_lut)
 				mpc->funcs->populate_lut(mpc, MCM_LUT_3DLUT, m_lut_params, lut_bank_a, mpcc_id);
 			if (mpc->funcs->program_lut_mode)
 				mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, lut3d_xable, lut_bank_a,
 						mpcc_id);
 		}
-		break;
-	case DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM:
-		switch (mcm_luts.lut3d_data.gpu_mem_params.size) {
-		case DC_CM2_GPU_MEM_SIZE_333333:
+	} else {
+		/* VIDMEM (3DLUT DMA Fast Load) */
+
+		/* Select width based on the requested LUT size */
+		switch (cm->lut3d_dma.size) {
+#if defined(CONFIG_DRM_AMD_DC_DCN4_2)
+		case CM_LUT_SIZE_333333:
 			if (dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_33)
 				width = hubp_3dlut_fl_width_33;
 			break;
-		case DC_CM2_GPU_MEM_SIZE_171717:
+#endif // CONFIG_DRM_AMD_DC_DCN4_2
+		case CM_LUT_SIZE_171717:
 			width = hubp_3dlut_fl_width_17;
 			break;
-		case DC_CM2_GPU_MEM_SIZE_TRANSFORMED:
-			width = hubp_3dlut_fl_width_transformed;
-			break;
 		default:
-			//TODO: handle default case
+			/* keep default hubp_3dlut_fl_width_transformed */
 			break;
 		}
 
 		//check for support
 		if (mpc->funcs->mcm.is_config_supported &&
 			!mpc->funcs->mcm.is_config_supported(width))
-			break;
+			return;
 
 		if (mpc->funcs->program_lut_read_write_control)
 			mpc->funcs->program_lut_read_write_control(mpc, MCM_LUT_3DLUT, lut_bank_a, mpcc_id);
@@ -546,21 +540,24 @@ void dcn401_populate_mcm_luts(struct dc *dc,
 			mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, lut3d_xable, lut_bank_a, mpcc_id);
 
 		if (hubp->funcs->hubp_program_3dlut_fl_addr)
-			hubp->funcs->hubp_program_3dlut_fl_addr(hubp, mcm_luts.lut3d_data.gpu_mem_params.addr);
+			hubp->funcs->hubp_program_3dlut_fl_addr(hubp, cm->lut3d_dma.addr);
 
+		/* bit_depth was previously zero-initialized in local_mcm,
+		 * preserve identical behavior.
+		 */
 		if (mpc->funcs->mcm.program_bit_depth)
-			mpc->funcs->mcm.program_bit_depth(mpc, mcm_luts.lut3d_data.gpu_mem_params.bit_depth, mpcc_id);
+			mpc->funcs->mcm.program_bit_depth(mpc, 0, mpcc_id);
 
-		switch (mcm_luts.lut3d_data.gpu_mem_params.layout) {
-		case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_RGB:
+		switch (cm->lut3d_dma.swizzle) {
+		case CM_LUT_3D_SWIZZLE_LINEAR_RGB:
 			mode = hubp_3dlut_fl_mode_native_1;
 			addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
 			break;
-		case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_BGR:
+		case CM_LUT_3D_SWIZZLE_LINEAR_BGR:
 			mode = hubp_3dlut_fl_mode_native_2;
 			addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
 			break;
-		case DC_CM2_GPU_MEM_LAYOUT_1D_PACKED_LINEAR:
+		case CM_LUT_1D_PACKED_LINEAR:
 			mode = hubp_3dlut_fl_mode_transform;
 			addr_mode = hubp_3dlut_fl_addressing_mode_simple_linear;
 			break;
@@ -575,40 +572,38 @@ void dcn401_populate_mcm_luts(struct dc *dc,
 		if (hubp->funcs->hubp_program_3dlut_fl_addressing_mode)
 			hubp->funcs->hubp_program_3dlut_fl_addressing_mode(hubp, addr_mode);
 
-		switch (mcm_luts.lut3d_data.gpu_mem_params.format_params.format) {
-		case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12MSB:
+		switch (cm->lut3d_dma.format) {
+		case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB:
 			format = hubp_3dlut_fl_format_unorm_12msb_bitslice;
 			break;
-		case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12LSB:
+		case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB:
 			format = hubp_3dlut_fl_format_unorm_12lsb_bitslice;
 			break;
-		case DC_CM2_GPU_MEM_FORMAT_16161616_FLOAT_FP1_5_10:
+		case CM_LUT_PIXEL_FORMAT_RGBA16161616_FLOAT_FP1_5_10:
 			format = hubp_3dlut_fl_format_float_fp1_5_10;
 			break;
+		default:
+			break;
 		}
 		if (hubp->funcs->hubp_program_3dlut_fl_format)
 			hubp->funcs->hubp_program_3dlut_fl_format(hubp, format);
 		if (hubp->funcs->hubp_update_3dlut_fl_bias_scale &&
 				mpc->funcs->mcm.program_bias_scale) {
 			mpc->funcs->mcm.program_bias_scale(mpc,
-				mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.bias,
-				mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.scale,
+				cm->lut3d_dma.bias,
+				cm->lut3d_dma.scale,
 				mpcc_id);
 			hubp->funcs->hubp_update_3dlut_fl_bias_scale(hubp,
-						mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.bias,
-						mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.scale);
+				cm->lut3d_dma.bias,
+				cm->lut3d_dma.scale);
 		}
 
-		//navi 4x has a bug and r and blue are swapped and need to be worked around here in
-		//TODO: need to make a method for get_xbar per asic OR do the workaround in program_crossbar for 4x
-		switch (mcm_luts.lut3d_data.gpu_mem_params.component_order) {
-		case DC_CM2_GPU_MEM_PIXEL_COMPONENT_ORDER_RGBA:
-		default:
-			crossbar_bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_0_15;
-			crossbar_bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
-			crossbar_bit_slice_cb_b = hubp_3dlut_fl_crossbar_bit_slice_32_47;
-			break;
-		}
+		/* component_order was previously hard-coded to RGBA in local_mcm,
+		 * preserve identical behavior.
+		 */
+		crossbar_bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_0_15;
+		crossbar_bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
+		crossbar_bit_slice_cb_b = hubp_3dlut_fl_crossbar_bit_slice_32_47;
 
 		if (hubp->funcs->hubp_program_3dlut_fl_crossbar)
 			hubp->funcs->hubp_program_3dlut_fl_crossbar(hubp,
@@ -634,8 +629,6 @@ void dcn401_populate_mcm_luts(struct dc *dc,
 				mpc->funcs->program_lut_mode(mpc, MCM_LUT_1DLUT, MCM_LUT_DISABLE, lut_bank_a, mpcc_id);
 			}
 		}
-		break;
-
 	}
 }
 
@@ -660,19 +653,19 @@ bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx,
 	const struct pwl_params *lut_params = NULL;
 	bool rval;
 
-	if (plane_state->mcm_luts.lut3d_data.lut3d_src == DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM) {
-		dcn401_populate_mcm_luts(dc, pipe_ctx, plane_state->mcm_luts, plane_state->lut_bank_a);
+	if (plane_state->cm.flags.bits.lut3d_dma_enable) {
+		dcn401_populate_mcm_luts(dc, pipe_ctx, &plane_state->cm, plane_state->lut_bank_a);
 		return true;
 	}
 
 	mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id);
 	pipe_ctx->plane_state->mcm_location = MPCC_MOVABLE_CM_LOCATION_BEFORE;
 	// 1D LUT
-	if (plane_state->blend_tf.type == TF_TYPE_HWPWL)
-		lut_params = &plane_state->blend_tf.pwl;
-	else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) {
+	if (plane_state->cm.blend_func.type == TF_TYPE_HWPWL)
+		lut_params = &plane_state->cm.blend_func.pwl;
+	else if (plane_state->cm.blend_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 		rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
-							       &plane_state->blend_tf,
+							       &plane_state->cm.blend_func,
 							       &dpp_base->regamma_params, false);
 		lut_params = rval ? &dpp_base->regamma_params : NULL;
 	}
@@ -680,12 +673,12 @@ bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx,
 	lut_params = NULL;
 
 	// Shaper
-	if (plane_state->in_shaper_func.type == TF_TYPE_HWPWL)
-		lut_params = &plane_state->in_shaper_func.pwl;
-	else if (plane_state->in_shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
+	if (plane_state->cm.shaper_func.type == TF_TYPE_HWPWL)
+		lut_params = &plane_state->cm.shaper_func.pwl;
+	else if (plane_state->cm.shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 		// TODO: dpp_base replace
 		rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
-							       &plane_state->in_shaper_func,
+							       &plane_state->cm.shaper_func,
 							       &dpp_base->shaper_params, true);
 		lut_params = rval ? &dpp_base->shaper_params : NULL;
 	}
@@ -693,8 +686,8 @@ bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx,
 
 	// 3D
 	if (mpc->funcs->program_3dlut) {
-		if (plane_state->lut3d_func.state.bits.initialized == 1)
-			result &= mpc->funcs->program_3dlut(mpc, &plane_state->lut3d_func.lut_3d, mpcc_id);
+		if (plane_state->cm.lut3d_func.state.bits.initialized == 1)
+			result &= mpc->funcs->program_3dlut(mpc, &plane_state->cm.lut3d_func.lut_3d, mpcc_id);
 		else
 			result &= mpc->funcs->program_3dlut(mpc, NULL, mpcc_id);
 	}
@@ -1108,8 +1101,11 @@ void dcn401_disable_link_output(struct dc_link *link,
 		disable_link_output_symclk_on_tx_off(link, DP_UNKNOWN_ENCODING);
 		link->phy_state.symclk_state = SYMCLK_ON_TX_OFF;
 	} else {
-		link_hwss->disable_link_output(link, link_res, signal);
-		link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
+		if (!(signal == SIGNAL_TYPE_EDP &&
+		      link->skip_implict_edp_power_control)) {
+			link_hwss->disable_link_output(link, link_res, signal);
+			link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
+		}
 	}
 
 	if (signal == SIGNAL_TYPE_EDP &&
@@ -1428,7 +1424,7 @@ void dcn401_wait_for_dcc_meta_propagation(const struct dc *dc,
 		if (pipe_ctx->plane_state &&
 				pipe_ctx->plane_state->dcc.enable &&
 				pipe_ctx->plane_state->flip_immediate &&
-				pipe_ctx->plane_state->update_flags.bits.addr_update) {
+				pipe_ctx->plane_state->update_bits.addr_update) {
 			is_wait_needed = true;
 			break;
 		}
@@ -1503,6 +1499,57 @@ void dcn401_prepare_bandwidth(struct dc *dc,
 	}
 }
 
+void dcn401_prepare_bandwidth_sequence(struct dc *dc,
+		struct dc_state *context,
+		struct block_sequence_state *seq_state)
+{
+	struct hubbub *hubbub = dc->res_pool->hubbub;
+	bool p_state_change_support = context->bw_ctx.bw.dcn.clk.p_state_change_support;
+	unsigned int compbuf_size = 0;
+
+	/* Any transition into P-State support should disable MCLK switching first to avoid hangs */
+	if (p_state_change_support) {
+		dc->optimized_required = true;
+		context->bw_ctx.bw.dcn.clk.p_state_change_support = false;
+	}
+
+	if (dc->clk_mgr->dc_mode_softmax_enabled)
+		if (dc->clk_mgr->clks.dramclk_khz <= (int)dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000 &&
+				context->bw_ctx.bw.dcn.clk.dramclk_khz > (int)dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000)
+			hwss_add_clk_mgr_set_max_memclk(seq_state, dc->clk_mgr,
+					dc->clk_mgr->bw_params->clk_table.entries[dc->clk_mgr->bw_params->clk_table.num_entries - 1].memclk_mhz);
+
+	/* Build bandwidth and display clocks back-to-back (SW calc + append BLS steps) */
+	if (dc->clk_mgr->funcs->build_clock_update_for_bls)
+		dc->clk_mgr->funcs->build_clock_update_for_bls(
+				dc->clk_mgr, context, false, seq_state);
+
+	hwss_add_hubbub_program_watermarks(seq_state, dc, hubbub,
+					&context->bw_ctx.bw.dcn.watermarks,
+					dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000,
+					false);
+
+	if (hubbub->funcs->program_arbiter)
+		hwss_add_hubbub_program_arbiter(seq_state, dc, hubbub,
+				&context->bw_ctx.bw.dcn.arb_regs, false);
+
+	if (hubbub->funcs->program_compbuf_segments) {
+		compbuf_size = context->bw_ctx.bw.dcn.arb_regs.compbuf_size;
+		dc->optimized_required |= (compbuf_size != dc->current_state->bw_ctx.bw.dcn.arb_regs.compbuf_size);
+
+		hwss_add_hubbub_program_compbuf_segments(seq_state, hubbub, compbuf_size, false);
+	}
+
+	if (dc->debug.fams2_config.bits.enable) {
+		dcn401_dmub_hw_control_lock(dc, context, true);
+		dcn401_fams2_update_config(dc, context, false);
+		dcn401_dmub_hw_control_lock(dc, context, false);
+	}
+
+	if (p_state_change_support != context->bw_ctx.bw.dcn.clk.p_state_change_support)
+		context->bw_ctx.bw.dcn.clk.p_state_change_support = p_state_change_support;
+}
+
 void dcn401_optimize_bandwidth(
 		struct dc *dc,
 		struct dc_state *context)
@@ -1556,6 +1603,50 @@ void dcn401_optimize_bandwidth(
 	}
 }
 
+/*
+ * optimize_bandwidth_sequence is unused for now. It will be used when
+ * dc_commit_state_no_check is moved into block sequence pattern, similar
+ * to how commit_planes_do_stream_update_sequence replaces
+ * commit_planes_do_stream_update.
+ */
+void dcn401_optimize_bandwidth_sequence(struct dc *dc,
+		struct dc_state *context,
+		struct block_sequence_state *seq_state)
+{
+	struct hubbub *hubbub = dc->res_pool->hubbub;
+
+	/* enable fams2 if needed */
+	if (dc->debug.fams2_config.bits.enable) {
+		dcn401_dmub_hw_control_lock(dc, context, true);
+		dcn401_fams2_update_config(dc, context, true);
+		dcn401_dmub_hw_control_lock(dc, context, false);
+	}
+
+	hwss_add_hubbub_program_watermarks(seq_state, dc, hubbub,
+					&context->bw_ctx.bw.dcn.watermarks,
+					dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000,
+					true);
+
+	if (hubbub->funcs->program_arbiter)
+		hwss_add_hubbub_program_arbiter(seq_state, dc, hubbub,
+				&context->bw_ctx.bw.dcn.arb_regs, true);
+
+	if (dc->clk_mgr->dc_mode_softmax_enabled)
+		if (dc->clk_mgr->clks.dramclk_khz > (int)dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000 &&
+				context->bw_ctx.bw.dcn.clk.dramclk_khz <= (int)dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000)
+			hwss_add_clk_mgr_set_max_memclk(seq_state, dc->clk_mgr,
+					dc->clk_mgr->bw_params->dc_mode_softmax_memclk);
+
+	if (hubbub->funcs->program_compbuf_segments)
+		hwss_add_hubbub_program_compbuf_segments(seq_state, hubbub,
+				context->bw_ctx.bw.dcn.arb_regs.compbuf_size, true);
+
+	/* Build bandwidth and display clocks (SW calc + append BLS steps) */
+	if (dc->clk_mgr->funcs->build_clock_update_for_bls)
+		dc->clk_mgr->funcs->build_clock_update_for_bls(
+				dc->clk_mgr, context, true, seq_state);
+}
+
 void dcn401_dmub_hw_control_lock(struct dc *dc,
 		struct dc_state *context,
 		bool lock)
@@ -1999,10 +2090,9 @@ void dcn401_perform_3dlut_wa_unlock(struct pipe_ctx *pipe_ctx)
 
 	for (odm_pipe = pipe_ctx; odm_pipe != NULL; odm_pipe = odm_pipe->next_odm_pipe) {
 		for (mpc_pipe = odm_pipe; mpc_pipe != NULL; mpc_pipe = mpc_pipe->bottom_pipe) {
-			if (mpc_pipe->plane_state && mpc_pipe->plane_state->mcm_luts.lut3d_data.lut3d_src
-						== DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM
-					&& mpc_pipe->plane_state->mcm_shaper_3dlut_setting
-						== DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER_3DLUT) {
+			if (mpc_pipe->plane_state &&
+					mpc_pipe->plane_state->cm.flags.bits.lut3d_enable &&
+					mpc_pipe->plane_state->cm.flags.bits.lut3d_dma_enable) {
 				wa_pipes[wa_pipe_ct++] = mpc_pipe;
 			}
 		}
@@ -2063,7 +2153,12 @@ void dcn401_reset_back_end_for_pipe(
 	 * screen only, the dpms_off would be true but
 	 * VBIOS lit up eDP, so check link status too.
 	 */
-	if (!pipe_ctx->stream->dpms_off || link->link_status.link_active)
+	if (link->connector_signal == SIGNAL_TYPE_EDP &&
+	    link->skip_implict_edp_power_control) {
+		/* DMSS is holding the panel across the commit; skip dpms-off. */
+		if (pipe_ctx->stream_res.audio)
+			dc->hwss.disable_audio_stream(pipe_ctx);
+	} else if (!pipe_ctx->stream->dpms_off || link->link_status.link_active)
 		dc->link_srv->set_dpms_off(pipe_ctx);
 	else if (pipe_ctx->stream_res.audio)
 		dc->hwss.disable_audio_stream(pipe_ctx);
@@ -2088,12 +2183,15 @@ void dcn401_reset_back_end_for_pipe(
 	 * parent pipe.
 	 */
 	if (pipe_ctx->top_pipe == NULL) {
+		if (!(link->connector_signal == SIGNAL_TYPE_EDP &&
+		      link->skip_implict_edp_power_control)) {
 
-		dc->hwss.set_abm_immediate_disable(pipe_ctx);
+			dc->hwss.set_abm_immediate_disable(pipe_ctx);
 
-		pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg);
+			pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg);
 
-		pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false);
+			pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false);
+		}
 		if (pipe_ctx->stream_res.tg->funcs->set_odm_bypass)
 			pipe_ctx->stream_res.tg->funcs->set_odm_bypass(
 					pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing);
@@ -2276,18 +2374,18 @@ void dcn401_program_pipe(
 	}
 
 	if (pipe_ctx->plane_state && (pipe_ctx->update_flags.raw ||
-	    pipe_ctx->plane_state->update_flags.raw ||
+	    dc_pipe_update_bits_is_any_set(&pipe_ctx->plane_state->update_bits) ||
 	    pipe_ctx->stream->update_flags.raw))
 		dc->hwss.update_dchubp_dpp(dc, pipe_ctx, context);
 
 	if (pipe_ctx->plane_state && (pipe_ctx->update_flags.bits.enable ||
-		pipe_ctx->plane_state->update_flags.bits.hdr_mult))
+		pipe_ctx->plane_state->update_bits.hdr_mult))
 		hws->funcs.set_hdr_multiplier(pipe_ctx);
 
 	if (pipe_ctx->plane_state &&
-		(pipe_ctx->plane_state->update_flags.bits.in_transfer_func_change ||
-			pipe_ctx->plane_state->update_flags.bits.gamma_change ||
-			pipe_ctx->plane_state->update_flags.bits.lut_3d ||
+		(pipe_ctx->plane_state->update_bits.in_transfer_func_change ||
+			pipe_ctx->plane_state->update_bits.gamma_change ||
+			pipe_ctx->plane_state->update_bits.lut_3d ||
 			pipe_ctx->update_flags.bits.enable))
 		hws->funcs.set_input_transfer_func(dc, pipe_ctx, pipe_ctx->plane_state);
 
@@ -2346,7 +2444,7 @@ void dcn401_program_pipe(
 			pipe_ctx->stream_res.test_pattern_params.offset);
 	}
 	if (pipe_ctx->plane_state
-		&& pipe_ctx->plane_state->update_flags.bits.cm_hist_change
+		&& pipe_ctx->plane_state->update_bits.cm_hist_change
 		&& hws->funcs.program_cm_hist)
 		hws->funcs.program_cm_hist(dc, pipe_ctx, pipe_ctx->plane_state);
 }
@@ -2427,7 +2525,7 @@ void dcn401_program_pipe_sequence(
 	}
 
 	if (pipe_ctx->plane_state && (pipe_ctx->update_flags.raw ||
-	    pipe_ctx->plane_state->update_flags.raw ||
+	    dc_pipe_update_bits_is_any_set(&pipe_ctx->plane_state->update_bits) ||
 	    pipe_ctx->stream->update_flags.raw)) {
 
 		if (dc->hwss.update_dchubp_dpp_sequence)
@@ -2435,15 +2533,15 @@ void dcn401_program_pipe_sequence(
 	}
 
 	if (pipe_ctx->plane_state && (pipe_ctx->update_flags.bits.enable ||
-		pipe_ctx->plane_state->update_flags.bits.hdr_mult)) {
+		pipe_ctx->plane_state->update_bits.hdr_mult)) {
 
 		hws->funcs.set_hdr_multiplier_sequence(pipe_ctx, seq_state);
 	}
 
 	if (pipe_ctx->plane_state &&
-		(pipe_ctx->plane_state->update_flags.bits.in_transfer_func_change ||
-			pipe_ctx->plane_state->update_flags.bits.gamma_change ||
-			pipe_ctx->plane_state->update_flags.bits.lut_3d ||
+		(pipe_ctx->plane_state->update_bits.in_transfer_func_change ||
+			pipe_ctx->plane_state->update_bits.gamma_change ||
+			pipe_ctx->plane_state->update_bits.lut_3d ||
 			pipe_ctx->update_flags.bits.enable)) {
 
 		hwss_add_dpp_set_input_transfer_func(seq_state, dc, pipe_ctx, pipe_ctx->plane_state);
@@ -2501,7 +2599,7 @@ void dcn401_program_pipe_sequence(
 	}
 
 	if (pipe_ctx->plane_state
-			&& pipe_ctx->plane_state->update_flags.bits.cm_hist_change
+			&& pipe_ctx->plane_state->update_bits.cm_hist_change
 			&& hws->funcs.program_cm_hist) {
 
 		hwss_add_dpp_program_cm_hist(seq_state, pipe_ctx->plane_res.dpp,
@@ -2655,7 +2753,7 @@ void dcn401_program_front_end_for_ctx(
 		pipe = &context->res_ctx.pipe_ctx[i];
 		if (!pipe->top_pipe && !pipe->prev_odm_pipe
 			&& pipe->stream && pipe->stream->num_wb_info > 0
-			&& (pipe->update_flags.raw || (pipe->plane_state && pipe->plane_state->update_flags.raw)
+			&& (pipe->update_flags.raw || (pipe->plane_state && dc_pipe_update_bits_is_any_set(&pipe->plane_state->update_bits))
 				|| pipe->stream->update_flags.raw)
 			&& hws->funcs.program_all_writeback_pipes_in_tree)
 			hws->funcs.program_all_writeback_pipes_in_tree(dc, pipe->stream, context);
@@ -3741,10 +3839,10 @@ void dcn401_update_dchubp_dpp_sequence(struct dc *dc,
 	/* Step 7: DPP setup - input CSC and format setup */
 	if (pipe_ctx->update_flags.bits.enable ||
 			pipe_ctx->update_flags.bits.plane_changed ||
-			plane_state->update_flags.bits.bpp_change ||
-			plane_state->update_flags.bits.input_csc_change ||
-			plane_state->update_flags.bits.color_space_change ||
-			plane_state->update_flags.bits.coeff_reduction_change) {
+			plane_state->update_bits.bpp_change ||
+			plane_state->update_bits.input_csc_change ||
+			plane_state->update_bits.color_space_change ||
+			plane_state->update_bits.coeff_reduction_change) {
 		hwss_add_dpp_setup_dpp(seq_state, pipe_ctx);
 
 		/* Step 8: DPP cursor matrix setup */
@@ -3761,8 +3859,8 @@ void dcn401_update_dchubp_dpp_sequence(struct dc *dc,
 	/* Step 10: MPCC updates */
 	if (pipe_ctx->update_flags.bits.mpcc ||
 	     pipe_ctx->update_flags.bits.plane_changed ||
-	     plane_state->update_flags.bits.global_alpha_change ||
-	     plane_state->update_flags.bits.per_pixel_alpha_change) {
+	     plane_state->update_bits.global_alpha_change ||
+	     plane_state->update_bits.per_pixel_alpha_change) {
 
 		/* Check if update_mpcc_sequence is implemented and prefer it over single MPC_UPDATE_MPCC step */
 		if (hws->funcs.update_mpcc_sequence)
@@ -3771,9 +3869,9 @@ void dcn401_update_dchubp_dpp_sequence(struct dc *dc,
 
 	/* Step 11: DPP scaler setup */
 	if (pipe_ctx->update_flags.bits.scaler ||
-			plane_state->update_flags.bits.scaling_change ||
-			plane_state->update_flags.bits.position_change ||
-			plane_state->update_flags.bits.per_pixel_alpha_change ||
+			plane_state->update_bits.scaling_change ||
+			plane_state->update_bits.position_change ||
+			plane_state->update_bits.per_pixel_alpha_change ||
 			pipe_ctx->stream->update_flags.bits.scaling) {
 		pipe_ctx->plane_res.scl_data.lb_params.alpha_en = pipe_ctx->plane_state->per_pixel_alpha;
 		ASSERT(pipe_ctx->plane_res.scl_data.lb_params.depth == LB_PIXEL_DEPTH_36BPP);
@@ -3782,8 +3880,8 @@ void dcn401_update_dchubp_dpp_sequence(struct dc *dc,
 
 	/* Step 12: HUBP viewport programming */
 	if (pipe_ctx->update_flags.bits.viewport ||
-	     (context == dc->current_state && plane_state->update_flags.bits.position_change) ||
-	     (context == dc->current_state && plane_state->update_flags.bits.scaling_change) ||
+	     (context == dc->current_state && plane_state->update_bits.position_change) ||
+	     (context == dc->current_state && plane_state->update_bits.scaling_change) ||
 	     (context == dc->current_state && pipe_ctx->stream->update_flags.bits.scaling)) {
 		hwss_add_hubp_mem_program_viewport(seq_state, hubp,
 			&pipe_ctx->plane_res.scl_data.viewport, &pipe_ctx->plane_res.scl_data.viewport_c);
@@ -3815,7 +3913,7 @@ void dcn401_update_dchubp_dpp_sequence(struct dc *dc,
 	if (pipe_ctx->update_flags.bits.enable || pipe_ctx->update_flags.bits.opp_changed ||
 			pipe_ctx->update_flags.bits.plane_changed ||
 			pipe_ctx->stream->update_flags.bits.gamut_remap ||
-			plane_state->update_flags.bits.gamut_remap_change ||
+			plane_state->update_bits.gamut_remap_change ||
 			pipe_ctx->stream->update_flags.bits.out_csc) {
 
 		/* Gamut remap */
@@ -3830,14 +3928,14 @@ void dcn401_update_dchubp_dpp_sequence(struct dc *dc,
 	if (pipe_ctx->update_flags.bits.enable ||
 			pipe_ctx->update_flags.bits.plane_changed ||
 			pipe_ctx->update_flags.bits.opp_changed ||
-			plane_state->update_flags.bits.pixel_format_change ||
-			plane_state->update_flags.bits.horizontal_mirror_change ||
-			plane_state->update_flags.bits.rotation_change ||
-			plane_state->update_flags.bits.swizzle_change ||
-			plane_state->update_flags.bits.dcc_change ||
-			plane_state->update_flags.bits.bpp_change ||
-			plane_state->update_flags.bits.scaling_change ||
-			plane_state->update_flags.bits.plane_size_change) {
+			plane_state->update_bits.pixel_format_change ||
+			plane_state->update_bits.horizontal_mirror_change ||
+			plane_state->update_bits.rotation_change ||
+			plane_state->update_bits.swizzle_change ||
+			plane_state->update_bits.dcc_change ||
+			plane_state->update_bits.bpp_change ||
+			plane_state->update_bits.scaling_change ||
+			plane_state->update_bits.plane_size_change) {
 		struct plane_size size = plane_state->plane_size;
 
 		size.surface_size = pipe_ctx->plane_res.scl_data.viewport;
@@ -3851,7 +3949,7 @@ void dcn401_update_dchubp_dpp_sequence(struct dc *dc,
 	/* Step 19: Update plane address (with SubVP support) */
 	if (pipe_ctx->update_flags.bits.enable ||
 	     pipe_ctx->update_flags.bits.plane_changed ||
-	     plane_state->update_flags.bits.addr_update) {
+	     plane_state->update_bits.addr_update) {
 
 		/* SubVP save surface address if needed */
 		if (resource_is_pipe_type(pipe_ctx, OTG_MASTER) && pipe_mall_type == SUBVP_MAIN) {
@@ -3924,7 +4022,7 @@ void dcn401_update_mpcc_sequence(struct dc *dc,
 	mpcc_id = hubp->inst;
 
 	/* Step 1: Update blending if no full update needed */
-	if (!pipe_ctx->plane_state->update_flags.bits.full_update &&
+	if (!pipe_ctx->plane_state->update_bits.full_update &&
 	    !pipe_ctx->update_flags.bits.mpcc) {
 
 		/* Update blending configuration */
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.h
index f78162ab859b4..a760050eea8c7 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.h
@@ -52,7 +52,7 @@ enum dc_status dcn401_enable_stream_timing(
 void dcn401_enable_stream(struct pipe_ctx *pipe_ctx);
 void dcn401_populate_mcm_luts(struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
-		struct dc_cm2_func_luts mcm_luts,
+		const struct dc_plane_cm *cm,
 		bool lut_bank_a);
 void dcn401_setup_hpo_hw_control(const struct dce_hwseq *hws, bool enable);
 
@@ -70,10 +70,20 @@ void dcn401_wait_for_dcc_meta_propagation(const struct dc *dc,
 void dcn401_prepare_bandwidth(struct dc *dc,
 		struct dc_state *context);
 
+struct block_sequence_state;
+
+void dcn401_prepare_bandwidth_sequence(struct dc *dc,
+		struct dc_state *context,
+		struct block_sequence_state *seq_state);
+
 void dcn401_optimize_bandwidth(
 		struct dc *dc,
 		struct dc_state *context);
 
+void dcn401_optimize_bandwidth_sequence(struct dc *dc,
+		struct dc_state *context,
+		struct block_sequence_state *seq_state);
+
 void dcn401_dmub_hw_control_lock(struct dc *dc,
 		struct dc_state *context,
 		bool lock);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_init.c
index 33b2cf344f1e6..f206e221f9268 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_init.c
@@ -44,7 +44,9 @@ static const struct hw_sequencer_funcs dcn401_funcs = {
 	.interdependent_update_lock = dcn401_interdependent_update_lock,
 	.cursor_lock = dcn10_cursor_lock,
 	.prepare_bandwidth = dcn401_prepare_bandwidth,
+	.prepare_bandwidth_sequence = dcn401_prepare_bandwidth_sequence,
 	.optimize_bandwidth = dcn401_optimize_bandwidth,
+	.optimize_bandwidth_sequence = dcn401_optimize_bandwidth_sequence,
 	.update_bandwidth = dcn401_update_bandwidth,
 	.set_drr = dcn10_set_drr,
 	.get_position = dcn10_get_position,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c
index 664004cadf10e..cc8e96ffe7d1d 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c
@@ -70,6 +70,7 @@ void dcn42_init_hw(struct dc *dc)
 	uint32_t user_level = MAX_BACKLIGHT_LEVEL;
 	bool dchub_ref_freq_changed;
 	int current_dchub_ref_freq = 0;
+	uint8_t dcfclk_gate_dis_value = 0;
 
 	if (dc->clk_mgr && dc->clk_mgr->funcs && dc->clk_mgr->funcs->init_clocks) {
 		dc->clk_mgr->funcs->init_clocks(dc->clk_mgr);
@@ -243,7 +244,13 @@ void dcn42_init_hw(struct dc *dc)
 		/* enable all DCN clock gating */
 		REG_WRITE(DCCG_GATE_DISABLE_CNTL, 0);
 
-		REG_UPDATE(DCFCLK_CNTL, DCFCLK_GATE_DIS, 0);
+		/* Temporary workaround for IOMMU mismatch issue.
+		 * Fine grain control via bit0 of debug flag.
+		 */
+		if (dc->debug.iommu_mismatch_temp_wka & 0x1)
+			dcfclk_gate_dis_value = 1;
+
+		REG_UPDATE(DCFCLK_CNTL, DCFCLK_GATE_DIS, dcfclk_gate_dis_value);
 	}
 
 	dcn401_setup_hpo_hw_control(hws, true);
@@ -274,6 +281,8 @@ void dcn42_init_hw(struct dc *dc)
 		dc->caps.dmub_caps.psr = dc->ctx->dmub_srv->dmub->feature_caps.psr;
 		dc->caps.dmub_caps.mclk_sw = dc->ctx->dmub_srv->dmub->feature_caps.fw_assisted_mclk_switch_ver > 0;
 		dc->caps.dmub_caps.fams_ver = dc->ctx->dmub_srv->dmub->feature_caps.fw_assisted_mclk_switch_ver;
+		dc->caps.dmub_caps.aux_backlight_support =
+			dc->ctx->dmub_srv->dmub->feature_caps.abm_aux_backlight_support;
 
 		/* sw and fw FAMS versions must match for support */
 		dc->debug.fams2_config.bits.enable &=
@@ -348,7 +357,7 @@ void dcn42_update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
 	mpcc_id = hubp->inst;
 
 	/* If there is no full update, don't need to touch MPC tree*/
-	if (!pipe_ctx->plane_state->update_flags.bits.full_update &&
+	if (!pipe_ctx->plane_state->update_bits.full_update &&
 		!pipe_ctx->update_flags.bits.mpcc) {
 		mpc->funcs->update_blending(mpc, &blnd_cfg, mpcc_id);
 		dc->hwss.update_visual_confirm_color(dc, pipe_ctx, mpcc_id);
@@ -394,40 +403,33 @@ void dcn42_program_cm_hist(
 }
 
 static void dc_get_lut_xbar(
-	enum dc_cm2_gpu_mem_pixel_component_order order,
 	enum hubp_3dlut_fl_crossbar_bit_slice *cr_r,
 	enum hubp_3dlut_fl_crossbar_bit_slice *y_g,
 	enum hubp_3dlut_fl_crossbar_bit_slice *cb_b)
 {
-	switch (order) {
-	case DC_CM2_GPU_MEM_PIXEL_COMPONENT_ORDER_RGBA:
-		*cr_r = hubp_3dlut_fl_crossbar_bit_slice_32_47;
-		*y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
-		*cb_b =  hubp_3dlut_fl_crossbar_bit_slice_0_15;
-		break;
-	case DC_CM2_GPU_MEM_PIXEL_COMPONENT_ORDER_BGRA:
-		*cr_r = hubp_3dlut_fl_crossbar_bit_slice_0_15;
-		*y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
-		*cb_b = hubp_3dlut_fl_crossbar_bit_slice_32_47;
-		break;
-	}
+	/* component_order was previously hard-coded to RGBA in local_mcm,
+	 * preserve identical behavior.
+	 */
+	*cr_r = hubp_3dlut_fl_crossbar_bit_slice_32_47;
+	*y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
+	*cb_b = hubp_3dlut_fl_crossbar_bit_slice_0_15;
 }
 
 static void dc_get_lut_mode(
-	enum dc_cm2_gpu_mem_layout layout,
+	enum dc_cm_lut_swizzle swizzle,
 	enum hubp_3dlut_fl_mode *mode,
 	enum hubp_3dlut_fl_addressing_mode *addr_mode)
 {
-	switch (layout) {
-	case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_RGB:
+	switch (swizzle) {
+	case CM_LUT_3D_SWIZZLE_LINEAR_RGB:
 		*mode = hubp_3dlut_fl_mode_native_1;
 		*addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
 		break;
-	case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_BGR:
+	case CM_LUT_3D_SWIZZLE_LINEAR_BGR:
 		*mode = hubp_3dlut_fl_mode_native_2;
 		*addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
 		break;
-	case DC_CM2_GPU_MEM_LAYOUT_1D_PACKED_LINEAR:
+	case CM_LUT_1D_PACKED_LINEAR:
 		*mode = hubp_3dlut_fl_mode_transform;
 		*addr_mode = hubp_3dlut_fl_addressing_mode_simple_linear;
 		break;
@@ -439,19 +441,22 @@ static void dc_get_lut_mode(
 }
 
 static void dc_get_lut_format(
-	enum dc_cm2_gpu_mem_format dc_format,
+	enum dc_cm_lut_pixel_format dc_format,
 	enum hubp_3dlut_fl_format *format)
 {
 	switch (dc_format) {
-	case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12MSB:
+	case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB:
 		*format = hubp_3dlut_fl_format_unorm_12msb_bitslice;
 		break;
-	case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12LSB:
+	case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB:
 		*format = hubp_3dlut_fl_format_unorm_12lsb_bitslice;
 		break;
-	case DC_CM2_GPU_MEM_FORMAT_16161616_FLOAT_FP1_5_10:
+	case CM_LUT_PIXEL_FORMAT_RGBA16161616_FLOAT_FP1_5_10:
 		*format = hubp_3dlut_fl_format_float_fp1_5_10;
 		break;
+	default:
+		*format = hubp_3dlut_fl_format_unorm_12msb_bitslice;
+		break;
 	}
 }
 
@@ -465,16 +470,17 @@ static bool dc_is_rmcm_3dlut_supported(struct hubp *hubp, struct mpc *mpc)
 	return false;
 }
 
-static bool is_rmcm_3dlut_fl_supported(struct dc *dc, enum dc_cm2_gpu_mem_size size)
+#if defined(CONFIG_DRM_AMD_DC_DCN4_2)
+static bool is_rmcm_3dlut_fl_supported(struct dc *dc)
 {
+	/* size was previously hard-coded to TRANSFORMED in local_mcm,
+	 * which mapped to dim_17. Preserve identical behavior.
+	 */
 	if (!dc->caps.color.mpc.rmcm_3d_lut_caps.dma_3d_lut)
 		return false;
-	if (size == DC_CM2_GPU_MEM_SIZE_171717)
-		return dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_17 != 0u;
-	else if (size == DC_CM2_GPU_MEM_SIZE_333333)
-		return dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_33 != 0u;
-	return false;
+	return dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_17 != 0u;
 }
+#endif
 
 static void dcn42_set_mcm_location_post_blend(struct dc *dc, struct pipe_ctx *pipe_ctx, bool bPostBlend)
 {
@@ -495,56 +501,45 @@ static void dcn42_get_mcm_lut_xable_from_pipe_ctx(struct dc *dc, struct pipe_ctx
 		enum MCM_LUT_XABLE *lut3d_xable,
 		enum MCM_LUT_XABLE *lut1d_xable)
 {
-	enum dc_cm2_shaper_3dlut_setting shaper_3dlut_setting = DC_CM2_SHAPER_3DLUT_SETTING_BYPASS_ALL;
-	bool lut1d_enable = false;
 	struct mpc *mpc = dc->res_pool->mpc;
 	int mpcc_id = pipe_ctx->plane_res.hubp->inst;
 
 	if (!pipe_ctx->plane_state)
 		return;
-	shaper_3dlut_setting = pipe_ctx->plane_state->mcm_shaper_3dlut_setting;
-	lut1d_enable = pipe_ctx->plane_state->mcm_lut1d_enable;
+
 	mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id);
 	pipe_ctx->plane_state->mcm_location = MPCC_MOVABLE_CM_LOCATION_BEFORE;
 
-	*lut1d_xable = lut1d_enable ? MCM_LUT_ENABLE : MCM_LUT_DISABLE;
-
-	switch (shaper_3dlut_setting) {
-	case DC_CM2_SHAPER_3DLUT_SETTING_BYPASS_ALL:
-		*lut3d_xable = *shaper_xable = MCM_LUT_DISABLE;
-		break;
-	case DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER:
-		*lut3d_xable = MCM_LUT_DISABLE;
-		*shaper_xable = MCM_LUT_ENABLE;
-		break;
-	case DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER_3DLUT:
-		*lut3d_xable = *shaper_xable = MCM_LUT_ENABLE;
-		break;
-	}
+	*lut1d_xable = pipe_ctx->plane_state->cm.flags.bits.blend_enable ?
+		MCM_LUT_ENABLE : MCM_LUT_DISABLE;
+	*shaper_xable = pipe_ctx->plane_state->cm.flags.bits.shaper_enable ?
+		MCM_LUT_ENABLE : MCM_LUT_DISABLE;
+	*lut3d_xable = (pipe_ctx->plane_state->cm.flags.bits.shaper_enable &&
+			pipe_ctx->plane_state->cm.flags.bits.lut3d_enable) ?
+		MCM_LUT_ENABLE : MCM_LUT_DISABLE;
 }
 
 static void fl_get_lut_mode(
-	enum dc_cm2_gpu_mem_layout layout,
-	enum dc_cm2_gpu_mem_size   size,
+	enum dc_cm_lut_swizzle swizzle,
 	enum hubp_3dlut_fl_mode *mode,
 	enum hubp_3dlut_fl_addressing_mode *addr_mode,
 	enum hubp_3dlut_fl_width *width)
 {
+	/* size was previously hard-coded to TRANSFORMED in local_mcm,
+	 * preserve identical behavior (transformed width).
+	 */
 	*width = hubp_3dlut_fl_width_17;
 
-	if (size == DC_CM2_GPU_MEM_SIZE_333333)
-		*width = hubp_3dlut_fl_width_33;
-
-	switch (layout) {
-	case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_RGB:
+	switch (swizzle) {
+	case CM_LUT_3D_SWIZZLE_LINEAR_RGB:
 		*mode = hubp_3dlut_fl_mode_native_1;
 		*addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
 		break;
-	case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_BGR:
+	case CM_LUT_3D_SWIZZLE_LINEAR_BGR:
 		*mode = hubp_3dlut_fl_mode_native_2;
 		*addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
 		break;
-	case DC_CM2_GPU_MEM_LAYOUT_1D_PACKED_LINEAR:
+	case CM_LUT_1D_PACKED_LINEAR:
 		*mode = hubp_3dlut_fl_mode_transform;
 		*addr_mode = hubp_3dlut_fl_addressing_mode_simple_linear;
 		break;
@@ -558,8 +553,7 @@ static void fl_get_lut_mode(
 bool dcn42_program_rmcm_luts(
 	struct hubp *hubp,
 	struct pipe_ctx *pipe_ctx,
-	enum dc_cm2_transfer_func_source lut3d_src,
-	struct dc_cm2_func_luts *mcm_luts,
+	const struct dc_plane_cm *cm,
 	struct mpc *mpc,
 	bool lut_bank_a,
 	int mpcc_id)
@@ -589,21 +583,24 @@ bool dcn42_program_rmcm_luts(
 	if (!rmcm_3dlut)
 		return false;
 
-	rmcm_3dlut->protection_bits = mcm_luts->lut3d_data.rmcm_tmz;
+	/* rmcm_tmz was previously zero-initialized in local_mcm,
+	 * preserve identical behavior.
+	 */
+	rmcm_3dlut->protection_bits = 0;
 
 	dcn42_get_mcm_lut_xable_from_pipe_ctx(dc, pipe_ctx, &shaper_xable, &lut3d_xable, &lut1d_xable);
 
 	/* Shaper */
-	if (mcm_luts->shaper) {
+	{
 		memset(&m_lut_params, 0, sizeof(m_lut_params));
 
-		if (mcm_luts->shaper->type == TF_TYPE_HWPWL) {
-			m_lut_params.pwl = &mcm_luts->shaper->pwl;
-		} else if (mcm_luts->shaper->type == TF_TYPE_DISTRIBUTED_POINTS) {
+		if (cm->shaper_func.type == TF_TYPE_HWPWL) {
+			m_lut_params.pwl = &cm->shaper_func.pwl;
+		} else if (cm->shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 			ASSERT(false);
 			cm_helper_translate_curve_to_hw_format(
 					dc->ctx,
-					mcm_luts->shaper,
+					&cm->shaper_func,
 					&dpp_base->shaper_params, true);
 			m_lut_params.pwl = &dpp_base->shaper_params;
 		}
@@ -619,15 +616,16 @@ bool dcn42_program_rmcm_luts(
 	}
 
 	/* 3DLUT */
-	switch (lut3d_src) {
-	case DC_CM2_TRANSFER_FUNC_SOURCE_SYSMEM:
+	if (!cm->flags.bits.lut3d_dma_enable) {
+		/* SYSMEM path — no DMA 3DLUT available.
+		 * Previously this was treated as a no-op for the DMA/VIDMEM
+		 * programming, preserve identical behavior.
+		 */
 		memset(&m_lut_params, 0, sizeof(m_lut_params));
-		// Don't know what to do in this case.
-		//case DC_CM2_TRANSFER_FUNC_SOURCE_SYSMEM:
-		break;
-	case DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM:
-		fl_get_lut_mode(mcm_luts->lut3d_data.gpu_mem_params.layout,
-				mcm_luts->lut3d_data.gpu_mem_params.size,
+	} else {
+		/* VIDMEM (3DLUT DMA Fast Load) */
+
+		fl_get_lut_mode(cm->lut3d_dma.swizzle,
 				&mode,
 				&addr_mode,
 				&width);
@@ -639,20 +637,19 @@ bool dcn42_program_rmcm_luts(
 			return false;
 
 		// setting native or transformed mode,
-		dc_get_lut_mode(mcm_luts->lut3d_data.gpu_mem_params.layout, &mode, &addr_mode);
+		dc_get_lut_mode(cm->lut3d_dma.swizzle, &mode, &addr_mode);
 
 		//seems to be only for the MCM
-		dc_get_lut_format(mcm_luts->lut3d_data.gpu_mem_params.format_params.format, &format);
+		dc_get_lut_format(cm->lut3d_dma.format, &format);
 
 		dc_get_lut_xbar(
-			mcm_luts->lut3d_data.gpu_mem_params.component_order,
 			&crossbar_bit_slice_cr_r,
 			&crossbar_bit_slice_y_g,
 			&crossbar_bit_slice_cb_b);
 
 		fl_config.mode					= mode;
 		fl_config.enabled				= lut3d_xable != MCM_LUT_DISABLE;
-		fl_config.address				= mcm_luts->lut3d_data.gpu_mem_params.addr;
+		fl_config.address				= cm->lut3d_dma.addr;
 		fl_config.format				= format;
 		fl_config.crossbar_bit_slice_y_g  = crossbar_bit_slice_y_g;
 		fl_config.crossbar_bit_slice_cb_b = crossbar_bit_slice_cb_b;
@@ -660,17 +657,20 @@ bool dcn42_program_rmcm_luts(
 		fl_config.width				    = width;
 		fl_config.protection_bits		= rmcm_3dlut->protection_bits;
 		fl_config.addr_mode			    = addr_mode;
-		fl_config.layout                = mcm_luts->lut3d_data.gpu_mem_params.layout;
-		fl_config.bias	= mcm_luts->lut3d_data.gpu_mem_params.format_params.float_params.bias;
-		fl_config.scale	= mcm_luts->lut3d_data.gpu_mem_params.format_params.float_params.scale;
+		fl_config.layout                = cm->lut3d_dma.swizzle;
+		fl_config.bias	= cm->lut3d_dma.bias;
+		fl_config.scale	= cm->lut3d_dma.scale;
 
 		mpc_fl_config.enabled			= fl_config.enabled;
 		mpc_fl_config.width	            = width;
 		mpc_fl_config.select_lut_bank_a = lut_bank_a;
-		mpc_fl_config.bit_depth		    = mcm_luts->lut3d_data.gpu_mem_params.bit_depth;
+		/* bit_depth was previously zero-initialized in local_mcm,
+		 * preserve identical behavior.
+		 */
+		mpc_fl_config.bit_depth		    = 0;
 		mpc_fl_config.hubp_index		= hubp->inst;
-		mpc_fl_config.bias	= mcm_luts->lut3d_data.gpu_mem_params.format_params.float_params.bias;
-		mpc_fl_config.scale	= mcm_luts->lut3d_data.gpu_mem_params.format_params.float_params.scale;
+		mpc_fl_config.bias	= cm->lut3d_dma.bias;
+		mpc_fl_config.scale	= cm->lut3d_dma.scale;
 
 		//1. power down the block
 		mpc->funcs->rmcm.power_on_shaper_3dlut(mpc, mpcc_id, false);
@@ -682,10 +682,6 @@ bool dcn42_program_rmcm_luts(
 
 		//3. power on the block
 		mpc->funcs->rmcm.power_on_shaper_3dlut(mpc, mpcc_id, true);
-
-		break;
-	default:
-		return false;
 	}
 
 	return true;
@@ -693,7 +689,7 @@ bool dcn42_program_rmcm_luts(
 
 void dcn42_populate_mcm_luts(struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
-		struct dc_cm2_func_luts mcm_luts,
+		const struct dc_plane_cm *cm,
 		bool lut_bank_a)
 {
 	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
@@ -701,14 +697,17 @@ void dcn42_populate_mcm_luts(struct dc *dc,
 	int mpcc_id = hubp->inst;
 	struct mpc *mpc = dc->res_pool->mpc;
 	union mcm_lut_params m_lut_params;
-	enum dc_cm2_transfer_func_source lut3d_src = mcm_luts.lut3d_data.lut3d_src;
+	const bool lut3d_dma = !!cm->flags.bits.lut3d_dma_enable;
 	enum hubp_3dlut_fl_format format = 0;
 	enum hubp_3dlut_fl_mode mode;
-	enum hubp_3dlut_fl_width width = 0;
+	/* Width was previously hard-coded to TRANSFORMED via local_mcm build,
+	 * preserve identical behavior.
+	 */
+	enum hubp_3dlut_fl_width width = hubp_3dlut_fl_width_transformed;
 	enum hubp_3dlut_fl_addressing_mode addr_mode;
-	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_y_g = 0;
-	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cb_b = 0;
-	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cr_r = 0;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_y_g;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cb_b;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cr_r;
 	enum MCM_LUT_XABLE shaper_xable = MCM_LUT_DISABLE;
 	enum MCM_LUT_XABLE lut3d_xable = MCM_LUT_DISABLE;
 	enum MCM_LUT_XABLE lut1d_xable = MCM_LUT_DISABLE;
@@ -717,33 +716,35 @@ void dcn42_populate_mcm_luts(struct dc *dc,
 	dcn42_get_mcm_lut_xable_from_pipe_ctx(dc, pipe_ctx, &shaper_xable, &lut3d_xable, &lut1d_xable);
 
 	//MCM - setting its location (Before/After) blender
-	//set to post blend (true)
+	//mpc_mcm_post_blend was previously zero-initialized in local_mcm,
+	//preserve identical behavior.
 	dcn42_set_mcm_location_post_blend(
 		dc,
 		pipe_ctx,
-		mcm_luts.lut3d_data.mpc_mcm_post_blend);
+		false);
 
 	//RMCM - 3dLUT+Shaper
-	if (mcm_luts.lut3d_data.rmcm_3dlut_enable &&
-		is_rmcm_3dlut_fl_supported(dc, mcm_luts.lut3d_data.gpu_mem_params.size)) {
+#if defined(CONFIG_DRM_AMD_DC_DCN4_2)
+	if (cm->flags.bits.rmcm_enable &&
+		is_rmcm_3dlut_fl_supported(dc)) {
 		dcn42_program_rmcm_luts(
 			hubp,
 			pipe_ctx,
-			lut3d_src,
-			&mcm_luts,
+			cm,
 			mpc,
 			lut_bank_a,
 			mpcc_id);
 	}
+#endif /* CONFIG_DRM_AMD_DC_DCN4_2 */
 
 	/* 1D LUT */
-	if (mcm_luts.lut1d_func) {
+	{
 		memset(&m_lut_params, 0, sizeof(m_lut_params));
-		if (mcm_luts.lut1d_func->type == TF_TYPE_HWPWL)
-			m_lut_params.pwl = &mcm_luts.lut1d_func->pwl;
-		else if (mcm_luts.lut1d_func->type == TF_TYPE_DISTRIBUTED_POINTS) {
+		if (cm->blend_func.type == TF_TYPE_HWPWL)
+			m_lut_params.pwl = &cm->blend_func.pwl;
+		else if (cm->blend_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 			rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx,
-					mcm_luts.lut1d_func,
+					&cm->blend_func,
 					&dpp_base->regamma_params, false);
 			m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL;
 		}
@@ -756,14 +757,14 @@ void dcn42_populate_mcm_luts(struct dc *dc,
 	}
 
 	/* Shaper */
-	if (mcm_luts.shaper && mcm_luts.lut3d_data.mpc_3dlut_enable) {
+	if (cm->flags.bits.lut3d_enable) {
 		memset(&m_lut_params, 0, sizeof(m_lut_params));
-		if (mcm_luts.shaper->type == TF_TYPE_HWPWL)
-			m_lut_params.pwl = &mcm_luts.shaper->pwl;
-		else if (mcm_luts.shaper->type == TF_TYPE_DISTRIBUTED_POINTS) {
+		if (cm->shaper_func.type == TF_TYPE_HWPWL)
+			m_lut_params.pwl = &cm->shaper_func.pwl;
+		else if (cm->shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 			ASSERT(false);
 			rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx,
-					mcm_luts.shaper,
+					&cm->shaper_func,
 					&dpp_base->regamma_params, true);
 			m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL;
 		}
@@ -776,41 +777,27 @@ void dcn42_populate_mcm_luts(struct dc *dc,
 	}
 
 	/* 3DLUT */
-	switch (lut3d_src) {
-	case DC_CM2_TRANSFER_FUNC_SOURCE_SYSMEM:
+	if (!lut3d_dma) {
+		/* SYSMEM (legacy lut3d_func) */
 		memset(&m_lut_params, 0, sizeof(m_lut_params));
 		if (hubp->funcs->hubp_enable_3dlut_fl)
 			hubp->funcs->hubp_enable_3dlut_fl(hubp, false);
 
-		if (mcm_luts.lut3d_data.lut3d_func && mcm_luts.lut3d_data.lut3d_func->state.bits.initialized) {
-			m_lut_params.lut3d = &mcm_luts.lut3d_data.lut3d_func->lut_3d;
+		if (cm->lut3d_func.state.bits.initialized) {
+			m_lut_params.lut3d = &cm->lut3d_func.lut_3d;
 			if (mpc->funcs->populate_lut)
 				mpc->funcs->populate_lut(mpc, MCM_LUT_3DLUT, m_lut_params, lut_bank_a, mpcc_id);
 			if (mpc->funcs->program_lut_mode)
 				mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, lut3d_xable, lut_bank_a,
 						mpcc_id);
 		}
-		break;
-	case DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM:
-		switch (mcm_luts.lut3d_data.gpu_mem_params.size) {
-		case DC_CM2_GPU_MEM_SIZE_333333:
-			width = hubp_3dlut_fl_width_33;
-			break;
-		case DC_CM2_GPU_MEM_SIZE_171717:
-			width = hubp_3dlut_fl_width_17;
-			break;
-		case DC_CM2_GPU_MEM_SIZE_TRANSFORMED:
-			width = hubp_3dlut_fl_width_transformed;
-			break;
-		default:
-			//TODO: Handle default case
-			break;
-		}
+	} else {
+		/* VIDMEM (3DLUT DMA Fast Load) */
 
 		//check for support
 		if (mpc->funcs->mcm.is_config_supported &&
 			!mpc->funcs->mcm.is_config_supported(width))
-			break;
+			return;
 
 		if (mpc->funcs->program_lut_read_write_control)
 			mpc->funcs->program_lut_read_write_control(mpc, MCM_LUT_3DLUT, lut_bank_a, mpcc_id);
@@ -818,49 +805,70 @@ void dcn42_populate_mcm_luts(struct dc *dc,
 			mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, lut3d_xable, lut_bank_a, mpcc_id);
 
 		if (hubp->funcs->hubp_program_3dlut_fl_addr)
-			hubp->funcs->hubp_program_3dlut_fl_addr(hubp, mcm_luts.lut3d_data.gpu_mem_params.addr);
+			hubp->funcs->hubp_program_3dlut_fl_addr(hubp, cm->lut3d_dma.addr);
 
+		/* bit_depth was previously zero-initialized in local_mcm,
+		 * preserve identical behavior.
+		 */
 		if (mpc->funcs->mcm.program_bit_depth)
-			mpc->funcs->mcm.program_bit_depth(mpc, mcm_luts.lut3d_data.gpu_mem_params.bit_depth, mpcc_id);
+			mpc->funcs->mcm.program_bit_depth(mpc, 0, mpcc_id);
 
-		dc_get_lut_mode(mcm_luts.lut3d_data.gpu_mem_params.layout, &mode, &addr_mode);
+		switch (cm->lut3d_dma.swizzle) {
+		case CM_LUT_3D_SWIZZLE_LINEAR_RGB:
+			mode = hubp_3dlut_fl_mode_native_1;
+			addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
+			break;
+		case CM_LUT_3D_SWIZZLE_LINEAR_BGR:
+			mode = hubp_3dlut_fl_mode_native_2;
+			addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
+			break;
+		case CM_LUT_1D_PACKED_LINEAR:
+			mode = hubp_3dlut_fl_mode_transform;
+			addr_mode = hubp_3dlut_fl_addressing_mode_simple_linear;
+			break;
+		default:
+			mode = hubp_3dlut_fl_mode_disable;
+			addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
+			break;
+		}
 		if (hubp->funcs->hubp_program_3dlut_fl_mode)
 			hubp->funcs->hubp_program_3dlut_fl_mode(hubp, mode);
 
 		if (hubp->funcs->hubp_program_3dlut_fl_addressing_mode)
 			hubp->funcs->hubp_program_3dlut_fl_addressing_mode(hubp, addr_mode);
 
-		switch (mcm_luts.lut3d_data.gpu_mem_params.format_params.format) {
-		case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12MSB:
+		switch (cm->lut3d_dma.format) {
+		case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB:
 			format = hubp_3dlut_fl_format_unorm_12msb_bitslice;
 			break;
-		case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12LSB:
+		case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB:
 			format = hubp_3dlut_fl_format_unorm_12lsb_bitslice;
 			break;
-		case DC_CM2_GPU_MEM_FORMAT_16161616_FLOAT_FP1_5_10:
+		case CM_LUT_PIXEL_FORMAT_RGBA16161616_FLOAT_FP1_5_10:
 			format = hubp_3dlut_fl_format_float_fp1_5_10;
 			break;
+		default:
+			break;
 		}
 		if (hubp->funcs->hubp_program_3dlut_fl_format)
 			hubp->funcs->hubp_program_3dlut_fl_format(hubp, format);
 		if (hubp->funcs->hubp_update_3dlut_fl_bias_scale &&
 				mpc->funcs->mcm.program_bias_scale) {
 			mpc->funcs->mcm.program_bias_scale(mpc,
-				mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.bias,
-				mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.scale,
+				cm->lut3d_dma.bias,
+				cm->lut3d_dma.scale,
 				mpcc_id);
 			hubp->funcs->hubp_update_3dlut_fl_bias_scale(hubp,
-						mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.bias,
-						mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.scale);
+				cm->lut3d_dma.bias,
+				cm->lut3d_dma.scale);
 		}
 
-		//navi 4x has a bug and r and blue are swapped and need to be worked around here in
-		//TODO: need to make a method for get_xbar per asic OR do the workaround in program_crossbar for 4x
-		dc_get_lut_xbar(
-			mcm_luts.lut3d_data.gpu_mem_params.component_order,
-			&crossbar_bit_slice_cr_r,
-			&crossbar_bit_slice_y_g,
-			&crossbar_bit_slice_cb_b);
+		/* component_order was previously hard-coded to RGBA in local_mcm,
+		 * preserve identical behavior.
+		 */
+		crossbar_bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_0_15;
+		crossbar_bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
+		crossbar_bit_slice_cb_b = hubp_3dlut_fl_crossbar_bit_slice_32_47;
 
 		if (hubp->funcs->hubp_program_3dlut_fl_crossbar)
 			hubp->funcs->hubp_program_3dlut_fl_crossbar(hubp,
@@ -886,7 +894,6 @@ void dcn42_populate_mcm_luts(struct dc *dc,
 				mpc->funcs->program_lut_mode(mpc, MCM_LUT_1DLUT, MCM_LUT_DISABLE, lut_bank_a, mpcc_id);
 			}
 		}
-		break;
 	}
 }
 
@@ -901,19 +908,19 @@ bool dcn42_set_mcm_luts(struct pipe_ctx *pipe_ctx,
 	const struct pwl_params *lut_params = NULL;
 	bool rval;
 
-	if (plane_state->mcm_luts.lut3d_data.lut3d_src == DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM) {
-		dcn42_populate_mcm_luts(dc, pipe_ctx, plane_state->mcm_luts, plane_state->lut_bank_a);
+	if (plane_state->cm.flags.bits.lut3d_dma_enable) {
+		dcn42_populate_mcm_luts(dc, pipe_ctx, &plane_state->cm, plane_state->lut_bank_a);
 		return true;
 	}
 
 	mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id);
 	pipe_ctx->plane_state->mcm_location = MPCC_MOVABLE_CM_LOCATION_BEFORE;
 	// 1D LUT
-	if (plane_state->blend_tf.type == TF_TYPE_HWPWL)
-		lut_params = &plane_state->blend_tf.pwl;
-	else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) {
+	if (plane_state->cm.blend_func.type == TF_TYPE_HWPWL)
+		lut_params = &plane_state->cm.blend_func.pwl;
+	else if (plane_state->cm.blend_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 		rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
-				&plane_state->blend_tf,
+				&plane_state->cm.blend_func,
 				&dpp_base->regamma_params, false);
 		lut_params = rval ? &dpp_base->regamma_params : NULL;
 	}
@@ -921,12 +928,12 @@ bool dcn42_set_mcm_luts(struct pipe_ctx *pipe_ctx,
 	lut_params = NULL;
 
 	// Shaper
-	if (plane_state->in_shaper_func.type == TF_TYPE_HWPWL)
-		lut_params = &plane_state->in_shaper_func.pwl;
-	else if (plane_state->in_shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
+	if (plane_state->cm.shaper_func.type == TF_TYPE_HWPWL)
+		lut_params = &plane_state->cm.shaper_func.pwl;
+	else if (plane_state->cm.shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 		// TODO: dpp_base replace
 		rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
-				&plane_state->in_shaper_func,
+				&plane_state->cm.shaper_func,
 				&dpp_base->shaper_params, true);
 		lut_params = rval ? &dpp_base->shaper_params : NULL;
 	}
@@ -934,8 +941,8 @@ bool dcn42_set_mcm_luts(struct pipe_ctx *pipe_ctx,
 
 	// 3D
 	if (mpc->funcs->program_3dlut) {
-		if (plane_state->lut3d_func.state.bits.initialized == 1)
-			result &= mpc->funcs->program_3dlut(mpc, &plane_state->lut3d_func.lut_3d, mpcc_id);
+		if (plane_state->cm.lut3d_func.state.bits.initialized == 1)
+			result &= mpc->funcs->program_3dlut(mpc, &plane_state->cm.lut3d_func.lut_3d, mpcc_id);
 		else
 			result &= mpc->funcs->program_3dlut(mpc, NULL, mpcc_id);
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h
index 0539ee0ffaee3..c469e7535114e 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h
@@ -20,14 +20,13 @@ bool dcn42_set_mcm_luts(struct pipe_ctx *pipe_ctx,
 
 void dcn42_populate_mcm_luts(struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
-		struct dc_cm2_func_luts mcm_luts,
+		const struct dc_plane_cm *cm,
 		bool lut_bank_a);
 
 bool dcn42_program_rmcm_luts(
 	struct hubp *hubp,
 	struct pipe_ctx *pipe_ctx,
-	enum dc_cm2_transfer_func_source lut3d_src,
-	struct dc_cm2_func_luts *mcm_luts,
+	const struct dc_plane_cm *cm,
 	struct mpc *mpc,
 	bool lut_bank_a,
 	int mpcc_id);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
index dfb278a9fc3e5..65df8002d3d7d 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
@@ -894,6 +894,36 @@ struct disable_audio_stream_params {
 	struct pipe_ctx *pipe_ctx;
 };
 
+struct clk_mgr_set_max_memclk_params {
+	struct clk_mgr *clk_mgr;
+	unsigned int memclk_mhz;
+};
+
+struct clk_mgr_update_clocks_params {
+	struct clk_mgr *clk_mgr;
+};
+
+struct hubbub_program_watermarks_params {
+	struct dc *dc;
+	struct hubbub *hubbub;
+	union dcn_watermark_set *watermarks;
+	unsigned int refclk_mhz;
+	bool safe_to_lower;
+};
+
+struct hubbub_program_arbiter_params {
+	struct dc *dc;
+	struct hubbub *hubbub;
+	struct dml2_display_arb_regs *arb_regs;
+	bool safe_to_lower;
+};
+
+struct hubbub_program_compbuf_segments_params {
+	struct hubbub *hubbub;
+	unsigned int compbuf_size;
+	bool safe_to_lower;
+};
+
 struct prepare_bandwidth_params {
 	struct dc *dc;
 	struct dc_state *context;
@@ -1057,6 +1087,11 @@ union block_sequence_params {
 	struct disable_audio_stream_params disable_audio_stream_params;
 	struct prepare_bandwidth_params prepare_bandwidth_params;
 	struct link_set_dpms_on_params link_set_dpms_on_params;
+	struct clk_mgr_set_max_memclk_params clk_mgr_set_max_memclk_params;
+	struct clk_mgr_update_clocks_params clk_mgr_update_clocks_params;
+	struct hubbub_program_watermarks_params hubbub_program_watermarks_params;
+	struct hubbub_program_arbiter_params hubbub_program_arbiter_params;
+	struct hubbub_program_compbuf_segments_params hubbub_program_compbuf_segments_params;
 };
 
 enum block_sequence_func {
@@ -1209,6 +1244,11 @@ enum block_sequence_func {
 	DISABLE_AUDIO_STREAM,
 	PREPARE_BANDWIDTH,
 	LINK_SET_DPMS_ON,
+	CLK_MGR_SET_MAX_MEMCLK,
+	CLK_MGR_UPDATE_CLOCKS,
+	HUBBUB_PROGRAM_WATERMARKS,
+	HUBBUB_PROGRAM_ARBITER,
+	HUBBUB_PROGRAM_COMPBUF_SEGMENTS,
 	/* This must be the last value in this enum, add new ones above */
 	HWSS_BLOCK_SEQUENCE_FUNC_COUNT
 };
@@ -1316,8 +1356,14 @@ struct hw_sequencer_funcs {
 
 	/* Bandwidth Related */
 	void (*prepare_bandwidth)(struct dc *dc, struct dc_state *context);
+	void (*prepare_bandwidth_sequence)(struct dc *dc,
+			struct dc_state *context,
+			struct block_sequence_state *seq_state);
 	bool (*update_bandwidth)(struct dc *dc, struct dc_state *context);
 	void (*optimize_bandwidth)(struct dc *dc, struct dc_state *context);
+	void (*optimize_bandwidth_sequence)(struct dc *dc,
+			struct dc_state *context,
+			struct block_sequence_state *seq_state);
 
 	/* Infopacket Related */
 	void (*set_avmute)(struct pipe_ctx *pipe_ctx, bool enable);
@@ -2475,4 +2521,40 @@ void hwss_add_link_set_dpms_on(struct block_sequence_state *seq_state,
 		struct dc_state *state,
 		struct pipe_ctx *pipe_ctx);
 
+/* Clock manager BLS executor functions */
+void hwss_clk_mgr_set_max_memclk(union block_sequence_params *params);
+void hwss_clk_mgr_update_clocks(union block_sequence_params *params);
+
+void hwss_hubbub_program_watermarks(union block_sequence_params *params);
+
+void hwss_hubbub_program_arbiter(union block_sequence_params *params);
+
+void hwss_hubbub_program_compbuf_segments(union block_sequence_params *params);
+
+/* Clock manager BLS add-helper functions */
+void hwss_add_clk_mgr_set_max_memclk(struct block_sequence_state *seq_state,
+		struct clk_mgr *clk_mgr,
+		unsigned int memclk_mhz);
+
+void hwss_add_clk_mgr_update_clocks(struct block_sequence_state *seq_state,
+		struct clk_mgr *clk_mgr);
+
+void hwss_add_hubbub_program_watermarks(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct hubbub *hubbub,
+		union dcn_watermark_set *watermarks,
+		unsigned int refclk_mhz,
+		bool safe_to_lower);
+
+void hwss_add_hubbub_program_arbiter(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct hubbub *hubbub,
+		struct dml2_display_arb_regs *arb_regs,
+		bool safe_to_lower);
+
+void hwss_add_hubbub_program_compbuf_segments(struct block_sequence_state *seq_state,
+		struct hubbub *hubbub,
+		unsigned int compbuf_size,
+		bool safe_to_lower);
+
 #endif /* __DC_HW_SEQUENCER_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h
index 63c6c841c681f..b4956893ae9a6 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h
@@ -58,6 +58,7 @@ struct dc_state;
 struct dc_stream_status;
 struct dc_writeback_info;
 struct dchub_init_data;
+struct dc_plane_cm;
 struct dc_static_screen_params;
 struct resource_pool;
 struct resource_context;
@@ -219,7 +220,7 @@ struct hwseq_private_funcs {
 			struct dc_state *context);
 	void (*populate_mcm_luts)(struct dc *dc,
 			struct pipe_ctx *pipe_ctx,
-			struct dc_cm2_func_luts mcm_luts,
+			const struct dc_plane_cm *cm,
 			bool lut_bank_a);
 	void (*perform_3dlut_wa_unlock)(struct pipe_ctx *pipe_ctx);
 	void (*wait_for_pipe_update_if_needed)(struct dc *dc, struct pipe_ctx *pipe_ctx, bool is_surface_update_only);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h b/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h
index 3f0161d646755..6ae7cf1095651 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h
@@ -66,6 +66,9 @@ struct abm_funcs {
 			unsigned int option,
 			unsigned int panel_inst,
 			unsigned int pwrseq_inst);
+	bool (*set_abm_event)(struct abm *abm, unsigned int full_screen, unsigned int trans_info,
+			unsigned int hdr_mode, unsigned int scaling_enable, unsigned int scaling_strength_map,
+			unsigned int panel_inst);
 };
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
index f829ce3f70e53..68dc2d4ba7ca2 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
@@ -30,6 +30,7 @@
 #include "dc.h"
 #include "core_types.h"
 #include "dm_pp_smu.h"
+struct utm_qos_model;
 
 /* Constants */
 #define DDR4_DRAM_WIDTH   64
@@ -311,6 +312,7 @@ struct clk_bw_params {
 	struct wm_table wm_table;
 	struct dummy_pstate_entry dummy_pstate_table[4];
 	struct clk_limit_table_entry dc_mode_limit;
+	const struct utm_qos_model *utm_qos_model;
 };
 /* Public interfaces */
 
@@ -318,6 +320,8 @@ struct clk_states {
 	uint32_t dprefclk_khz;
 };
 
+struct block_sequence_state;
+
 struct clk_mgr_funcs {
 	/*
 	 * This function should set new clocks based on the input "safe_to_lower".
@@ -337,6 +341,8 @@ struct clk_mgr_funcs {
 	void (*exit_low_power_state)(struct clk_mgr *clk_mgr);
 	bool (*is_ips_supported)(struct clk_mgr *clk_mgr);
 
+	void (*set_idle_power_optimizations)(struct clk_mgr *clk_mgr, bool enable);
+
 	void (*init_clocks)(struct clk_mgr *clk_mgr);
 
 	void (*dump_clk_registers)(struct clk_state_registers_and_bypass *regs_and_bypass,
@@ -405,6 +411,12 @@ struct clk_mgr_funcs {
 	void (*get_requested_memory_qos)(
 			struct clk_mgr *clk_mgr,
 			struct dc_requested_memory_qos *qos);
+
+	void (*build_clock_update_for_bls)(struct clk_mgr *clk_mgr,
+			struct dc_state *context, bool safe_to_lower,
+			struct block_sequence_state *seq_state);
+
+	void (*execute_clk_mgr_block_sequence)(struct clk_mgr *clk_mgr);
 };
 
 struct clk_mgr {
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
index 6db7c8753081d..e756719308abc 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
@@ -348,6 +348,7 @@ struct dccg_funcs {
 	void (*dccg_root_gate_disable_control)(struct dccg *dccg, uint32_t pipe_idx, uint32_t disable_clock_gating);
 	void (*dccg_read_reg_state)(struct dccg *dccg, struct dcn_dccg_reg_state *dccg_reg_state);
 	void (*dccg_enable_global_fgcg)(struct dccg *dccg, bool enable);
+	bool (*dccg_get_global_fgcg_status)(struct dccg *dccg);
 };
 
 #endif //__DAL_DCCG_H__
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
index 1c18898aa475a..6d6eda0e7e9dc 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
@@ -108,7 +108,7 @@ struct hubp_fl_3dlut_config {
 	uint16_t scale;
 	struct dc_plane_address address;
 	enum hubp_3dlut_fl_addressing_mode addr_mode;
-	enum dc_cm2_gpu_mem_layout layout;
+	enum dc_cm_lut_swizzle layout;
 	uint8_t protection_bits;
 	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_y_g;
 	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cb_b;
diff --git a/drivers/gpu/drm/amd/display/dc/inc/link_service.h b/drivers/gpu/drm/amd/display/dc/inc/link_service.h
index 23202c2114bb9..addeb3e3b25a2 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/link_service.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/link_service.h
@@ -193,6 +193,7 @@ struct link_service {
 			struct aux_payload *payload);
 	bool (*is_in_aux_transaction_mode)(struct ddc_service *ddc);
 	uint32_t (*get_aux_defer_delay)(struct ddc_service *ddc);
+	uint8_t (*get_ddc_aux_inst)(const struct dc_link *link);
 
 
 	/*************************** DP Capability ****************************/
diff --git a/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h b/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h
index 7a1ecb8d986f2..6d15ccdc7f878 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h
@@ -536,23 +536,4 @@ uint32_t generic_indirect_reg_update_ex_sync(const struct dc_context *ctx,
 		uint8_t shift1, uint32_t mask1, uint32_t field_value1,
 		...);
 
-/* register offload macros
- *
- * instead of MMIO to register directly, in some cases we want
- * to gather register sequence and execute the register sequence
- * from another thread so we optimize time required for lengthy ops
- */
-
-/* start gathering register sequence */
-#define REG_SEQ_START() \
-	reg_sequence_start_gather(CTX)
-
-/* start execution of register sequence gathered since REG_SEQ_START */
-#define REG_SEQ_SUBMIT() \
-	reg_sequence_start_execute(CTX)
-
-/* wait for the last REG_SEQ_SUBMIT to finish */
-#define REG_SEQ_WAIT_DONE() \
-	reg_sequence_wait_done(CTX)
-
 #endif /* DRIVERS_GPU_DRM_AMD_DC_DEV_DC_INC_REG_HELPER_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/soc_and_ip_translator.h b/drivers/gpu/drm/amd/display/dc/inc/soc_and_ip_translator.h
index 6a97a3e28bd2c..5dcb9f8f4dafe 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/soc_and_ip_translator.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/soc_and_ip_translator.h
@@ -8,26 +8,12 @@
 #include "dc.h"
 #include "dml_top_soc_parameter_types.h"
 
-/* Forward declarations — callers that dereference these structs must include
- * the full UTM model headers themselves. */
-struct utm_qos_model;
-struct utm_qos_model_dchub_v2;
-
 struct soc_and_ip_translator_funcs {
 	void (*get_soc_bb)(
 			struct dml2_soc_bb *soc_bb,
 			const struct dc *dc,
 			const struct dml2_configuration_options *config);
 	void (*get_ip_caps)(struct dml2_ip_capabilities *dml_ip_caps);
-	/**
-	 * get_utm_qos_model - Return the static UTM QoS model for this DCN
-	 *     generation. Caller provides storage for @qos_model and @dchub.
-	 * @qos_model: output — populated with SoC bounding box and SOP table
-	 * @dchub: output — populated with DCHUB client extension data
-	 */
-	void (*get_utm_qos_model)(
-			struct utm_qos_model *qos_model,
-			struct utm_qos_model_dchub_v2 *dchub);
 };
 
 struct soc_and_ip_translator {
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_detection.c b/drivers/gpu/drm/amd/display/dc/link/link_detection.c
index 7d8951fecd570..4f18b668340c0 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_detection.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_detection.c
@@ -497,6 +497,12 @@ static void query_hdcp_capability(enum signal_type signal, struct dc_link *link)
 		msg22.data = link->hdcp_caps.rx_caps.raw;
 		msg22.length = sizeof(link->hdcp_caps.rx_caps.raw);
 		msg22.msg_id = HDCP_MESSAGE_ID_RX_CAPS;
+		if (link->force_to_use_aux && (signal == SIGNAL_TYPE_HDMI_TYPE_A)) {
+			// case with passive dongle with i2c over aux
+			msg22.data = &link->hdcp_caps.rx_caps.fields.version;
+			msg22.length = sizeof(link->hdcp_caps.rx_caps.fields.version);
+			msg22.msg_id = HDCP_MESSAGE_ID_HDCP2VERSION;
+		}
 	} else {
 		msg22.data = &link->hdcp_caps.rx_caps.fields.version;
 		msg22.length = sizeof(link->hdcp_caps.rx_caps.fields.version);
@@ -597,7 +603,24 @@ static bool detect_dp(struct dc_link *link,
 	if (sink_caps->transaction_type == DDC_TRANSACTION_TYPE_I2C_OVER_AUX) {
 		sink_caps->signal = SIGNAL_TYPE_DISPLAY_PORT;
 		if (!detect_dp_sink_caps(link)) {
-			return false;
+			if (link->force_to_use_aux) {
+				sink_caps->signal = dp_passive_dongle_detection(link->ddc, sink_caps, audio_support);
+				link->dpcd_caps.dongle_type = sink_caps->dongle_type;
+				link->dpcd_caps.is_dongle_type_one = sink_caps->is_dongle_type_one;
+				link->dpcd_caps.dpcd_rev.raw = 0;
+				/* Type 1 dongles do not work with I2C over Aux and also some of
+				 * Type 2 dongles do not support I2C over Aux well, so for these
+				 * cases when native Aux transactons fails and I2C over Aux fails
+				 * report that nothing is connected, as we can't tell is it bad
+				 * DP sink or bad passive dongle.
+				 */
+				if (sink_caps->dongle_type == DISPLAY_DONGLE_NONE)
+					return false;
+				else
+					return true;
+			} else {
+				return false;
+			}
 		}
 
 		if (is_dp_branch_device(link))
@@ -623,7 +646,7 @@ static bool detect_dp(struct dc_link *link,
 		link->dpcd_caps.sink_count.bits.SINK_COUNT = 1;
 		/* NUTMEG requires that we use HBR, doesn't work with RBR. */
 		if (link->dpcd_caps.branch_dev_id == DP_BRANCH_DEVICE_ID_00001A)
-			link->preferred_link_setting.link_rate = LINK_RATE_HIGH;
+			link->wa_flags.dp_skip_rbr = true;
 	}
 
 	return true;
@@ -933,7 +956,7 @@ static bool should_verify_link_capability_destructively(struct dc_link *link,
 		destrictive = true;
 		if (is_hdmi_frl_in_use(link)) {
 			destrictive = false;
-		} else if (link->dc->config.skip_frl_pretraining) {
+		} else if (link->local_sink->edid_caps.panel_patch.skip_frl_pre_training) {
 			for (i = 0; i < MAX_PIPES; i++) {
 				if (pipes[i].stream != NULL &&
 					pipes[i].stream->link == link) {
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
index 42e7f0e9b3ce4..335ae952ef60e 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
@@ -2296,6 +2296,8 @@ static enum dc_status enable_link(
 
 static bool allocate_usb4_bandwidth_for_stream(struct dc_stream_state *stream, int stream_bw)
 {
+	ASSERT(stream->sink);
+
 	struct dc_link *link = stream->sink->link;
 	int req_bw = stream_bw;
 
@@ -2342,6 +2344,8 @@ static bool allocate_usb4_bandwidth_for_stream(struct dc_stream_state *stream, i
 
 static bool allocate_usb4_bandwidth(struct dc_stream_state *stream)
 {
+	ASSERT(stream->sink);
+
 	bool ret;
 
 	int bw = dc_bandwidth_in_kbps_from_timing(&stream->timing,
@@ -2361,38 +2365,43 @@ static bool deallocate_usb4_bandwidth(struct dc_stream_state *stream)
 	return ret;
 }
 
+static struct vpg *get_vpg(struct pipe_ctx *pipe_ctx)
+{
+	if (dc_is_hdmi_frl_signal(pipe_ctx->stream->signal))
+		return pipe_ctx->stream_res.hpo_frl_stream_enc->vpg;
+	else if (dp_is_128b_132b_signal(pipe_ctx))
+		return pipe_ctx->stream_res.hpo_dp_stream_enc->vpg;
+	else
+		return pipe_ctx->stream_res.stream_enc->vpg;
+}
+
 void link_set_dpms_off(struct pipe_ctx *pipe_ctx)
 {
-	struct dc  *dc = pipe_ctx->stream->ctx->dc;
+	DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger);
 	struct dc_stream_state *stream = pipe_ctx->stream;
-	struct dc_link *link = stream->sink->link;
-	struct vpg *vpg = pipe_ctx->stream_res.stream_enc->vpg;
+	struct dc *dc = stream->ctx->dc;
+	struct dc_link *link = stream->link;
+	struct vpg *vpg = get_vpg(pipe_ctx);
 	enum dp_panel_mode panel_mode_dp = dp_get_panel_mode(link);
 
-	DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger);
-
 	ASSERT(is_master_pipe_for_link(link, pipe_ctx));
 
-	if (dp_is_128b_132b_signal(pipe_ctx))
-		vpg = pipe_ctx->stream_res.hpo_dp_stream_enc->vpg;
-	if (dc_is_hdmi_frl_signal(pipe_ctx->stream->signal))
-		vpg = pipe_ctx->stream_res.hpo_frl_stream_enc->vpg;
-	if (dc_is_virtual_signal(pipe_ctx->stream->signal))
+	if (dc_is_virtual_signal(stream->signal))
 		return;
 
-	if (pipe_ctx->stream->sink) {
-		if (pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_VIRTUAL &&
-			pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_NONE) {
+	if (stream->sink) {
+		if (stream->sink->sink_signal != SIGNAL_TYPE_VIRTUAL &&
+			stream->sink->sink_signal != SIGNAL_TYPE_NONE) {
 			DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d sink_count=%d\n", __func__,
-			pipe_ctx->stream->sink->edid_caps.display_name,
-			pipe_ctx->stream->signal, link->link_index, link->sink_count);
+			stream->sink->edid_caps.display_name,
+			stream->signal, link->link_index, link->sink_count);
 		}
 	}
 
 	link_wait_for_unlocked(link);
 
-	if (!pipe_ctx->stream->sink->edid_caps.panel_patch.skip_avmute) {
-		if (dc_is_hdmi_signal(pipe_ctx->stream->signal))
+	if (stream->sink && !stream->sink->edid_caps.panel_patch.skip_avmute) {
+		if (dc_is_hdmi_signal(stream->signal))
 			set_avmute(pipe_ctx, true);
 	}
 
@@ -2402,15 +2411,15 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx)
 	dc->hwss.blank_stream(pipe_ctx);
 
 	if (pipe_ctx->link_config.dp_tunnel_settings.should_use_dp_bw_allocation)
-		deallocate_usb4_bandwidth(pipe_ctx->stream);
+		deallocate_usb4_bandwidth(stream);
 
-	if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST)
+	if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST)
 		deallocate_mst_payload(pipe_ctx);
-	else if (dc_is_dp_sst_signal(pipe_ctx->stream->signal) &&
+	else if (dc_is_dp_sst_signal(stream->signal) &&
 			dp_is_128b_132b_signal(pipe_ctx))
 		update_sst_payload(pipe_ctx, false);
 
-	if (dc_is_hdmi_signal(pipe_ctx->stream->signal)) {
+	if (dc_is_hdmi_signal(stream->signal)) {
 		struct ext_hdmi_settings settings = {0};
 		enum engine_id eng_id = pipe_ctx->stream_res.stream_enc->id;
 
@@ -2433,7 +2442,7 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx)
 		}
 	}
 
-	if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT &&
+	if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT &&
 			!dp_is_128b_132b_signal(pipe_ctx)) {
 
 		/* In DP1.x SST mode, our encoder will go to TPS1
@@ -2443,18 +2452,18 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx)
 		 * state machine.
 		 * In DP2 or MST mode, our encoder will stay video active
 		 */
-		disable_link(pipe_ctx->stream->link, &pipe_ctx->link_res, pipe_ctx->stream->signal);
+		disable_link(link, &pipe_ctx->link_res, stream->signal);
 		dc->hwss.disable_stream(pipe_ctx);
 	} else {
 		dc->hwss.disable_stream(pipe_ctx);
-		disable_link(pipe_ctx->stream->link, &pipe_ctx->link_res, pipe_ctx->stream->signal);
+		disable_link(link, &pipe_ctx->link_res, stream->signal);
 	}
 	edp_set_panel_assr(link, pipe_ctx, &panel_mode_dp, false);
 
-	if (pipe_ctx->stream->timing.flags.DSC) {
-		if (dc_is_dp_signal(pipe_ctx->stream->signal))
+	if (stream->timing.flags.DSC) {
+		if (dc_is_dp_signal(stream->signal))
 			link_set_dsc_enable(pipe_ctx, false);
-		else if (dc_is_hdmi_frl_signal(pipe_ctx->stream->signal))
+		else if (dc_is_hdmi_frl_signal(stream->signal))
 			link_set_dsc_on_stream(pipe_ctx, false);
 	}
 	if (dp_is_128b_132b_signal(pipe_ctx)) {
@@ -2468,7 +2477,7 @@ void link_set_dpms_off(struct pipe_ctx *pipe_ctx)
 	/* for psp not exist case */
 	if (link->connector_signal == SIGNAL_TYPE_EDP && dc->debug.psp_disabled_wa) {
 		/* reset internal save state to default since eDP is  off */
-		enum dp_panel_mode panel_mode = dp_get_panel_mode(pipe_ctx->stream->link);
+		enum dp_panel_mode panel_mode = dp_get_panel_mode(link);
 		/* since current psp not loaded, we need to reset it to default */
 		link->panel_mode = panel_mode;
 	}
@@ -2478,62 +2487,57 @@ void link_set_dpms_on(
 		struct dc_state *state,
 		struct pipe_ctx *pipe_ctx)
 {
-	struct dc *dc = pipe_ctx->stream->ctx->dc;
+	DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger);
 	struct dc_stream_state *stream = pipe_ctx->stream;
-	struct dc_link *link = stream->sink->link;
+	struct dc *dc = stream->ctx->dc;
+	struct dc_link *link = stream->link;
 	enum dc_status status;
 	struct link_encoder *link_enc = pipe_ctx->link_res.dio_link_enc;
 	enum otg_out_mux_dest otg_out_dest = OUT_MUX_DIO;
-	struct vpg *vpg = pipe_ctx->stream_res.stream_enc->vpg;
+	struct vpg *vpg = get_vpg(pipe_ctx);
 	const struct link_hwss *link_hwss = get_link_hwss(link, &pipe_ctx->link_res);
 	bool apply_edp_fast_boot_optimization =
-		pipe_ctx->stream->apply_edp_fast_boot_optimization;
-
-	DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger);
+		stream->apply_edp_fast_boot_optimization;
 
 	ASSERT(is_master_pipe_for_link(link, pipe_ctx));
 
-	if (dp_is_128b_132b_signal(pipe_ctx))
-		vpg = pipe_ctx->stream_res.hpo_dp_stream_enc->vpg;
-	if (dc_is_hdmi_frl_signal(pipe_ctx->stream->signal))
-		vpg = pipe_ctx->stream_res.hpo_frl_stream_enc->vpg;
-	if (dc_is_virtual_signal(pipe_ctx->stream->signal))
+	if (dc_is_virtual_signal(stream->signal))
 		return;
 
-	if (pipe_ctx->stream->sink) {
-		if (pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_VIRTUAL &&
-			pipe_ctx->stream->sink->sink_signal != SIGNAL_TYPE_NONE) {
+	if (stream->sink) {
+		if (stream->sink->sink_signal != SIGNAL_TYPE_VIRTUAL &&
+			stream->sink->sink_signal != SIGNAL_TYPE_NONE) {
 			DC_LOG_DC("%s pipe_ctx dispname=%s signal=%x link=%d sink_count=%d\n", __func__,
-			pipe_ctx->stream->sink->edid_caps.display_name,
-			pipe_ctx->stream->signal,
+			stream->sink->edid_caps.display_name,
+			stream->signal,
 			link->link_index,
 			link->sink_count);
 		}
 	}
 
-	link_wait_for_unlocked(stream->link);
+	link_wait_for_unlocked(link);
 	if (!dc->config.unify_link_enc_assignment)
 		link_enc = link_enc_cfg_get_link_enc(link);
 	ASSERT(link_enc);
 
-	if (!dc_is_virtual_signal(pipe_ctx->stream->signal)
-			&& !dc_is_hdmi_frl_signal(pipe_ctx->stream->signal)
+	if (!dc_is_virtual_signal(stream->signal)
+			&& !dc_is_hdmi_frl_signal(stream->signal)
 			&& !dp_is_128b_132b_signal(pipe_ctx)) {
 		if (link_enc)
 			link_enc->funcs->setup(
 				link_enc,
-				pipe_ctx->stream->signal);
+				stream->signal);
 	}
 
-	pipe_ctx->stream->link->link_state_valid = true;
+	link->link_state_valid = true;
 
-	if (dc_is_hdmi_frl_signal(pipe_ctx->stream->signal))
-		hdmi_frl_decide_link_settings(stream, &stream->link->frl_link_settings, &pipe_ctx->dsc_padding_params);
+	if (dc_is_hdmi_frl_signal(stream->signal))
+		hdmi_frl_decide_link_settings(stream, &link->frl_link_settings, &pipe_ctx->dsc_padding_params);
 
 	if (pipe_ctx->stream_res.tg->funcs->set_out_mux) {
 		if (dp_is_128b_132b_signal(pipe_ctx))
 			otg_out_dest = OUT_MUX_HPO_DP;
-		else if (dc_is_hdmi_frl_signal(pipe_ctx->stream->signal))
+		else if (dc_is_hdmi_frl_signal(stream->signal))
 			otg_out_dest = OUT_MUX_HPO_FRL;
 		else
 			otg_out_dest = OUT_MUX_DIO;
@@ -2542,7 +2546,7 @@ void link_set_dpms_on(
 
 	link_hwss->setup_stream_attribute(pipe_ctx);
 
-	pipe_ctx->stream->apply_edp_fast_boot_optimization = false;
+	stream->apply_edp_fast_boot_optimization = false;
 
 	// Enable VPG before building infoframe
 	if (vpg && vpg->funcs->vpg_poweron)
@@ -2551,15 +2555,15 @@ void link_set_dpms_on(
 	resource_build_info_frame(pipe_ctx);
 	dc->hwss.update_info_frame(pipe_ctx);
 
-	if (dc_is_dp_signal(pipe_ctx->stream->signal))
+	if (dc_is_dp_signal(stream->signal))
 		dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_UPDATE_INFO_FRAME);
 
 	/* Do not touch link on seamless boot optimization. */
-	if (pipe_ctx->stream->apply_seamless_boot_optimization) {
-		pipe_ctx->stream->dpms_off = false;
+	if (stream->apply_seamless_boot_optimization) {
+		stream->dpms_off = false;
 
 		/* Still enable stream features & audio on seamless boot for DP external displays */
-		if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT) {
+		if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT) {
 			enable_stream_features(pipe_ctx);
 			dc->hwss.enable_audio_stream(pipe_ctx);
 		}
@@ -2569,11 +2573,11 @@ void link_set_dpms_on(
 	}
 
 	/* eDP lit up by bios already, no need to enable again. */
-	if (pipe_ctx->stream->signal == SIGNAL_TYPE_EDP &&
+	if (stream->signal == SIGNAL_TYPE_EDP &&
 				apply_edp_fast_boot_optimization &&
-				!pipe_ctx->stream->timing.flags.DSC &&
+				!stream->timing.flags.DSC &&
 				!pipe_ctx->next_odm_pipe) {
-		pipe_ctx->stream->dpms_off = false;
+		stream->dpms_off = false;
 		update_psp_stream_config(pipe_ctx, false);
 
 		if (link->is_dds) {
@@ -2586,7 +2590,7 @@ void link_set_dpms_on(
 		return;
 	}
 
-	if (pipe_ctx->stream->dpms_off)
+	if (stream->dpms_off)
 		return;
 
 	/* For Dp tunneling link, a pending HPD means that we have a race condition between processing
@@ -2604,20 +2608,21 @@ void link_set_dpms_on(
 	 * will be automatically set at a later time when the video is enabled
 	 * (DP_VID_STREAM_EN = 1).
 	 */
-	if (pipe_ctx->stream->timing.flags.DSC) {
-		if (dc_is_dp_signal(pipe_ctx->stream->signal) ||
-		    dc_is_virtual_signal(pipe_ctx->stream->signal))
+	if (stream->timing.flags.DSC) {
+		if (dc_is_dp_signal(stream->signal) ||
+			dc_is_virtual_signal(stream->signal))
 			link_set_dsc_enable(pipe_ctx, true);
 	}
 
 	if (link->replay_settings.config.replay_supported && !dc_is_embedded_signal(link->connector_signal))
 		dp_setup_replay(link, stream);
 
+	// TODO: Split DPMS-on into 3 functions at this point
 	status = enable_link(state, pipe_ctx);
 
 	if (status != DC_OK) {
 		DC_LOG_WARNING("enabling link %u failed: %d\n",
-		pipe_ctx->stream->link->link_index,
+		link->link_index,
 		status);
 
 		/* Abort stream enable *unless* the failure was due to
@@ -2625,21 +2630,21 @@ void link_set_dpms_on(
 		 * show the stream anyway. But MST displays can't proceed
 		 * without link training.
 		 */
-			if ((status != DC_FAIL_DP_LINK_TRAINING &&
-					status != DC_FAIL_HDMI_FRL_LINK_TRAINING) ||
-					pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) {
-			if (false == stream->link->link_status.link_active)
-				disable_link(stream->link, &pipe_ctx->link_res,
-						pipe_ctx->stream->signal);
+		if ((status != DC_FAIL_DP_LINK_TRAINING &&
+				status != DC_FAIL_HDMI_FRL_LINK_TRAINING) ||
+				stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) {
+			if (false == link->link_status.link_active)
+				disable_link(link, &pipe_ctx->link_res,
+						stream->signal);
 			BREAK_TO_DEBUGGER();
 			return;
 		}
 	}
+	// TODO: Split DPMS-on into 3 functions at this point
 
-	if (pipe_ctx->stream->timing.flags.DSC &&
-			dc_is_hdmi_frl_signal(pipe_ctx->stream->signal))
-			//TODO: bring HDMI FRL in line with DP
-			link_set_dsc_on_stream(pipe_ctx, true);
+	if (stream->timing.flags.DSC && dc_is_hdmi_frl_signal(stream->signal))
+		//TODO: bring HDMI FRL in line with DP
+		link_set_dsc_on_stream(pipe_ctx, true);
 
 	/* turn off otg test pattern if enable */
 	if (pipe_ctx->stream_res.tg->funcs->set_test_pattern)
@@ -2651,37 +2656,35 @@ void link_set_dpms_on(
 	 * as a workaround for the incorrect value being applied
 	 * from transmitter control.
 	 */
-	if (!(dc_is_virtual_signal(pipe_ctx->stream->signal) ||
-			dc_is_hdmi_frl_signal(pipe_ctx->stream->signal) ||
+	if (!(dc_is_virtual_signal(stream->signal) ||
+			dc_is_hdmi_frl_signal(stream->signal) ||
 			dp_is_128b_132b_signal(pipe_ctx))) {
-
-			if (link_enc)
-				link_enc->funcs->setup(
+		if (link_enc)
+			link_enc->funcs->setup(
 					link_enc,
-					pipe_ctx->stream->signal);
-
-		}
+					stream->signal);
+	}
 
 	dc->hwss.enable_stream(pipe_ctx);
 
 	/* Set DPS PPS SDP (AKA "info frames") */
-	if (pipe_ctx->stream->timing.flags.DSC) {
-		if (dc_is_dp_signal(pipe_ctx->stream->signal) ||
-				dc_is_virtual_signal(pipe_ctx->stream->signal)) {
+	if (stream->timing.flags.DSC) {
+		if (dc_is_dp_signal(stream->signal) ||
+				dc_is_virtual_signal(stream->signal)) {
 			dp_set_dsc_on_rx(pipe_ctx, true);
 			link_set_dsc_pps_packet(pipe_ctx, true, true);
 		}
 	}
 
-	if (dc_is_dp_signal(pipe_ctx->stream->signal))
+	if (dc_is_dp_signal(stream->signal))
 		dp_set_hblank_reduction_on_rx(pipe_ctx);
 
 	if (pipe_ctx->link_config.dp_tunnel_settings.should_use_dp_bw_allocation)
-		allocate_usb4_bandwidth(pipe_ctx->stream);
+		allocate_usb4_bandwidth(stream);
 
-	if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST)
+	if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST)
 		allocate_mst_payload(pipe_ctx);
-	else if (dc_is_dp_sst_signal(pipe_ctx->stream->signal) &&
+	else if (dc_is_dp_sst_signal(stream->signal) &&
 			dp_is_128b_132b_signal(pipe_ctx))
 		update_sst_payload(pipe_ctx, true);
 
@@ -2690,23 +2693,22 @@ void link_set_dpms_on(
 	 * training and stream unblank resolves the corruption issue.
 	 * This is workaround.
 	 */
-	if (pipe_ctx->stream->signal == SIGNAL_TYPE_EDP &&
+	if (stream->signal == SIGNAL_TYPE_EDP &&
 			link->is_display_mux_present)
 		msleep(20);
 
 	dc->hwss.unblank_stream(pipe_ctx,
-		&pipe_ctx->stream->link->cur_link_settings);
+		&link->cur_link_settings);
 
 	if (stream->sink_patches.delay_ignore_msa > 0)
 		msleep(stream->sink_patches.delay_ignore_msa);
 
-	if (dc_is_dp_signal(pipe_ctx->stream->signal))
+	if (dc_is_dp_signal(stream->signal))
 		enable_stream_features(pipe_ctx);
 	update_psp_stream_config(pipe_ctx, false);
 
 	dc->hwss.enable_audio_stream(pipe_ctx);
 
-	if (dc_is_hdmi_signal(pipe_ctx->stream->signal)) {
+	if (dc_is_hdmi_signal(stream->signal))
 		set_avmute(pipe_ctx, false);
-	}
 }
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_factory.c b/drivers/gpu/drm/amd/display/dc/link/link_factory.c
index b6262e43ca02a..67ce8d95bbd61 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_factory.c
@@ -143,6 +143,7 @@ static void construct_link_service_ddc(struct link_service *link_srv)
 			link_aux_transfer_with_retries_no_mutex;
 	link_srv->is_in_aux_transaction_mode = link_is_in_aux_transaction_mode;
 	link_srv->get_aux_defer_delay = link_get_aux_defer_delay;
+	link_srv->get_ddc_aux_inst = link_get_ddc_aux_inst;
 }
 
 /* link dp capability implements dp specific link capability retrieval sequence.
@@ -441,7 +442,7 @@ static enum channel_id get_ddc_line(struct dc_link *link)
 
 	channel = CHANNEL_ID_UNKNOWN;
 
-	if (link->ctx->dc->config.dp_connector_no_native_i2c && link->no_ddc_pin) {
+	if (link->force_to_use_aux) {
 		channel = link->aux_hw_inst + 1;
 	} else {
 		ddc = get_ddc_pin(link->ddc);
@@ -576,6 +577,8 @@ static bool construct_phy(struct dc_link *link,
 		link->is_internal_display = (disp_connect_caps_info.INTERNAL_DISPLAY != 0);
 		DC_LOG_DC("BIOS object table - is_internal_display: %d", link->is_internal_display);
 		link->no_ddc_pin = disp_connect_caps_info.NO_DDC_PIN != 0;
+		link->force_to_use_aux = link->dc->config.dp_connector_no_native_i2c
+				&& link->no_ddc_pin;
 	}
 
 	if (link->link_id.type != OBJECT_TYPE_CONNECTOR) {
@@ -598,7 +601,7 @@ static bool construct_phy(struct dc_link *link,
 		goto ddc_create_fail;
 	}
 
-	if (link->ctx->dc->config.dp_connector_no_native_i2c && link->no_ddc_pin) {
+	if (link->force_to_use_aux) {
 		link->ddc_hw_inst = link->aux_hw_inst;
 	} else {
 		/* Embedded display connectors such as LVDS may not have DDC. */
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.c
index ead71f6d116d8..f9d5a2441e38a 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.c
@@ -120,8 +120,7 @@ static void ddc_service_construct(
 	ddc_service->link = init_data->link;
 	ddc_service->ctx = init_data->ctx;
 
-	if (ddc_service->link && ddc_service->ctx->dc->config.dp_connector_no_native_i2c &&
-		ddc_service->link->no_ddc_pin) {
+	if (ddc_service->link && ddc_service->link->force_to_use_aux) {
 		// Obtain aux instance info from i2c_info without GPIO DDC pin info
 		if (dcb->funcs->get_connector_aux_info(dcb, init_data->id, &i2c_info) == BP_RESULT_OK)
 			ddc_service->link->aux_hw_inst = (uint8_t)i2c_info.i2c_line;
@@ -252,6 +251,21 @@ static uint32_t defer_delay_converter_wa(
 
 #define DP_TRANSLATOR_DELAY 5
 
+/**
+ * link_get_ddc_aux_inst - Return the AUX/DDC hardware instance for a link.
+ * @link: the link to query
+ *
+ * Return: aux_hw_inst when I2C is forced over AUX, otherwise the DDC pin
+ * channel index.
+ */
+uint8_t link_get_ddc_aux_inst(const struct dc_link *link)
+{
+	if (link->force_to_use_aux)
+		return link->aux_hw_inst;
+	ASSERT(link->ddc->ddc_pin->hw_info.ddc_channel <= 0xFF);
+	return (uint8_t)link->ddc->ddc_pin->hw_info.ddc_channel;
+}
+
 uint32_t link_get_aux_defer_delay(struct ddc_service *ddc)
 {
 	uint32_t defer_delay = 0;
@@ -526,7 +540,7 @@ bool try_to_configure_aux_timeout(struct ddc_service *ddc,
 	if (ddc->link->ep_type != DISPLAY_ENDPOINT_PHY)
 		return true;
 
-	if (ddc->ctx->dc->config.dp_connector_no_native_i2c && ddc->link->no_ddc_pin) {
+	if (ddc->link->force_to_use_aux) {
 		if (ddc->ctx->dc->res_pool->engines[ddc->link->aux_hw_inst]->funcs->configure_timeout) {
 			ddc->ctx->dc->res_pool->engines[ddc->link->aux_hw_inst]->funcs->configure_timeout(ddc, timeout);
 			result = true;
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.h
index f2a80e12494ba..fdd8a3dce97ff 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.h
@@ -46,6 +46,8 @@ void set_ddc_transaction_type(
 		struct ddc_service *ddc,
 		enum ddc_transaction_type type);
 
+uint8_t link_get_ddc_aux_inst(const struct dc_link *link);
+
 uint32_t link_get_aux_defer_delay(struct ddc_service *ddc);
 
 bool link_is_in_aux_transaction_mode(struct ddc_service *ddc);
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
index 47abb40667092..24e3bbec15ffb 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
@@ -181,6 +181,12 @@ uint32_t link_bw_kbps_from_raw_frl_link_rate_data(uint8_t bw)
 		return 40000000;
 	case 0b110:
 		return 48000000;
+	case 0b111:
+		return 64000000;
+	case 0b1000:
+		return 80000000;
+	case 0b1001:
+		return 96000000;
 	}
 
 	return 0;
@@ -750,8 +756,10 @@ static bool decide_dp_link_settings(struct dc_link *link, struct dc_link_setting
 	if (req_bw > dp_link_bandwidth_kbps(link, &link->verified_link_cap))
 		return false;
 
-	if (link->preferred_link_setting.link_rate != LINK_RATE_UNKNOWN)
-		initial_link_setting.link_rate = link->preferred_link_setting.link_rate;
+	if (link->wa_flags.dp_skip_rbr) {
+		initial_link_setting.link_rate = LINK_RATE_HIGH;
+		current_link_setting.link_rate = LINK_RATE_HIGH;
+	}
 
 	/* search for the minimum link setting that:
 	 * 1. is supported according to the link training result
@@ -2242,10 +2250,14 @@ void detect_edp_sink_caps(struct dc_link *link)
 	/*
 	 * ALPM is only valid for eDP v1.4 or higher.
 	 */
-	if (link->dpcd_caps.dpcd_rev.raw >= DP_EDP_14)
+	if (link->dpcd_caps.dpcd_rev.raw >= DP_EDP_14) {
 		core_link_read_dpcd(link, DP_RECEIVER_ALPM_CAP,
 			&link->dpcd_caps.alpm_caps.raw,
 			sizeof(link->dpcd_caps.alpm_caps.raw));
+		core_link_read_dpcd(link, DP_SINK_PSR_ACTIVE_VTOTAL_CONTROL_CAP,
+			&link->dpcd_caps.psr_info.psr_active_vtotal_control_cap,
+			sizeof(link->dpcd_caps.psr_info.psr_active_vtotal_control_cap));
+	}
 
 	/*
 	 * Read REPLAY info
@@ -2573,7 +2585,7 @@ bool dp_is_sink_present(struct dc_link *link)
 	/* We can't perform the step below for ASICs with no Native
 	 * I2C signaling support on DP connectors, so skip it.
 	 */
-	if (link->ctx->dc->config.dp_connector_no_native_i2c && link->no_ddc_pin)
+	if (link->force_to_use_aux)
 		return present;
 
 	ddc = get_ddc_pin(link->ddc);
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c
index 54ce768ae6ad9..da679fb7d89c7 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c
@@ -39,7 +39,6 @@
 #include "link/link_dpms.h"
 #include "dm_helpers.h"
 #include "link_dp_dpia_bw.h"
-#include "link_dp_panel_replay.h"
 
 #define DC_LOGGER \
 	link->ctx->logger
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c
index d87f87a02d63e..0d4f88ff844d4 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c
@@ -25,6 +25,7 @@
 
 #include "link_dp_panel_replay.h"
 #include "link_edp_panel_control.h"
+#include "link_ddc.h"
 #include "link_dpcd.h"
 #include "dm_helpers.h"
 #include "dc/dc_dmub_srv.h"
@@ -119,7 +120,7 @@ static bool dp_setup_panel_replay(struct dc_link *link, const struct dc_stream_s
 	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
 		return false;
 
-	replay_context.aux_inst = link->ddc->ddc_pin->hw_info.ddc_channel;
+	replay_context.aux_inst = (enum channel_id) link_get_ddc_aux_inst(link);
 	replay_context.digbe_inst = link->link_enc->transmitter;
 	replay_context.digfe_inst = link->link_enc->preferred_engine;
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
index 80a372ceaa517..16951a9550f25 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
@@ -29,6 +29,7 @@
  */
 
 #include "link_edp_panel_control.h"
+#include "link_ddc.h"
 #include "link_dpcd.h"
 #include "link_dp_capability.h"
 #include "dm_helpers.h"
@@ -160,9 +161,7 @@ bool edp_set_backlight_level_nits(struct dc_link *link,
 	if (link->is_dds && !link->dpcd_caps.panel_luminance_control)
 		return true;
 
-	// use internal backlight control if dmub capabilities are not present
-	if (link->backlight_control_type == BACKLIGHT_CONTROL_VESA_AUX &&
-		!link->dc->caps.dmub_caps.aux_backlight_support) {
+	if (link->backlight_control_type == BACKLIGHT_CONTROL_VESA_AUX) {
 		uint8_t backlight_enable = 0;
 		struct target_luminance_value *target_luminance = NULL;
 
@@ -272,10 +271,11 @@ bool edp_backlight_enable_aux(struct dc_link *link, bool enable)
 
 	if (link->is_dds)
 		return true;
-	if (core_link_write_dpcd(link, DP_SOURCE_BACKLIGHT_ENABLE,
-		&backlight_enable, 1) != DC_OK)
-		return false;
-
+	if (!link->dpcd_caps.panel_luminance_control) {
+		if (core_link_write_dpcd(link, DP_SOURCE_BACKLIGHT_ENABLE,
+			&backlight_enable, 1) != DC_OK)
+			return false;
+	}
 	return true;
 }
 
@@ -788,10 +788,7 @@ bool edp_setup_psr(struct dc_link *link,
 		}
 	}
 
-	if (dc->config.dp_connector_no_native_i2c && link->no_ddc_pin)
-		psr_context->channel = (enum channel_id)link->aux_hw_inst;
-	else
-		psr_context->channel = link->ddc->ddc_pin->hw_info.ddc_channel;
+	psr_context->channel = link_get_ddc_aux_inst(link);
 	psr_context->transmitterId = link->link_enc->transmitter;
 	psr_context->engineId = link->link_enc->preferred_engine;
 
@@ -1024,7 +1021,7 @@ bool edp_setup_freesync_replay(struct dc_link *link, const struct dc_stream_stat
 	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
 		return false;
 
-	replay_context.aux_inst = link->ddc->ddc_pin->hw_info.ddc_channel;
+	replay_context.aux_inst = link_get_ddc_aux_inst(link);
 	replay_context.digbe_inst = link->link_enc->transmitter;
 	replay_context.digfe_inst = link->link_enc->preferred_engine;
 
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn20/dcn20_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn20/dcn20_mpc.c
index fa600593f4c16..0e09d073ab292 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn20/dcn20_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn20/dcn20_mpc.c
@@ -380,7 +380,6 @@ static void mpc20_program_ogam_pwl(
 	struct dcn20_mpc *mpc20 = TO_DCN20_MPC(mpc);
 
 	PERF_TRACE();
-	REG_SEQ_START();
 
 	for (i = 0 ; i < num; i++) {
 		REG_SET(MPCC_OGAM_LUT_DATA[mpcc_id], 0, MPCC_OGAM_LUT_DATA, rgb[i].red_reg);
@@ -395,9 +394,6 @@ static void mpc20_program_ogam_pwl(
 				MPCC_OGAM_LUT_DATA, rgb[i].delta_blue_reg);
 	}
 
-	REG_SEQ_SUBMIT();
-	PERF_TRACE();
-	REG_SEQ_WAIT_DONE();
 	PERF_TRACE();
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.c b/drivers/gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.c
index 83730bbe26a8c..881b8da656b27 100644
--- a/drivers/gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.c
+++ b/drivers/gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.c
@@ -89,6 +89,24 @@ void opp2_set_disp_pattern_generator(
 	break;
 	}
 
+	if (opp->ctx->dc->debug.disable_dynamic_expansion_for_test_pattern) {
+		switch (test_pattern) {
+		case CONTROLLER_DP_TEST_PATTERN_COLORSQUARES:
+		case CONTROLLER_DP_TEST_PATTERN_COLORSQUARES_CEA:
+		case CONTROLLER_DP_TEST_PATTERN_VERTICALBARS:
+		case CONTROLLER_DP_TEST_PATTERN_HORIZONTALBARS:
+		case CONTROLLER_DP_TEST_PATTERN_COLORRAMP:
+			if (color_depth == COLOR_DEPTH_121212)
+				REG_UPDATE(FMT_DYNAMIC_EXP_CNTL, FMT_DYNAMIC_EXP_EN, 0);
+		break;
+		case CONTROLLER_DP_TEST_PATTERN_VIDEOMODE:
+			REG_UPDATE(FMT_DYNAMIC_EXP_CNTL, FMT_DYNAMIC_EXP_EN, 1);
+		break;
+		default:
+		break;
+		}
+	}
+
 	/* set DPG dimentions */
 	REG_SET_2(DPG_DIMENSIONS, 0,
 		DPG_ACTIVE_WIDTH, width,
@@ -149,6 +167,9 @@ void opp2_set_disp_pattern_generator(
 		case TEST_PATTERN_COLOR_FORMAT_BPC_10:
 			dst_bpc = 10;
 		break;
+		case TEST_PATTERN_COLOR_FORMAT_BPC_12:
+			dst_bpc = 12;
+		break;
 		default:
 			dst_bpc = 8;
 		break;
@@ -192,22 +213,25 @@ void opp2_set_disp_pattern_generator(
 
 	case CONTROLLER_DP_TEST_PATTERN_COLORRAMP:
 	{
-		mode = (bit_depth ==
-			TEST_PATTERN_COLOR_FORMAT_BPC_10 ?
-			TEST_PATTERN_MODE_DUALRAMP_RGB :
-			TEST_PATTERN_MODE_SINGLERAMP_RGB);
-
 		switch (bit_depth) {
 		case TEST_PATTERN_COLOR_FORMAT_BPC_6:
+			mode = TEST_PATTERN_MODE_SINGLERAMP_RGB;
 			dst_bpc = 6;
 		break;
 		case TEST_PATTERN_COLOR_FORMAT_BPC_8:
+			mode = TEST_PATTERN_MODE_SINGLERAMP_RGB;
 			dst_bpc = 8;
 		break;
 		case TEST_PATTERN_COLOR_FORMAT_BPC_10:
+			mode = TEST_PATTERN_MODE_DUALRAMP_RGB;
 			dst_bpc = 10;
 		break;
+		case TEST_PATTERN_COLOR_FORMAT_BPC_12:
+			mode = TEST_PATTERN_MODE_DUALRAMP_RGB;
+			dst_bpc = 12;
+		break;
 		default:
+			mode = TEST_PATTERN_MODE_SINGLERAMP_RGB;
 			dst_bpc = 8;
 		break;
 		}
@@ -244,9 +268,20 @@ void opp2_set_disp_pattern_generator(
 		case TEST_PATTERN_COLOR_FORMAT_BPC_10:
 		{
 			REG_SET_3(DPG_RAMP_CONTROL, 0,
-				DPG_RAMP0_OFFSET, 384 << 6,
-				DPG_INC0, inc_base,
-				DPG_INC1, inc_base + 2);
+				DPG_RAMP0_OFFSET, 384 << inc_base, // 384 start point
+				DPG_INC0, inc_base, // step size of 1
+				DPG_INC1, inc_base + 2); // step size of 4 (1 << 2)
+			REG_UPDATE_2(DPG_CONTROL,
+				DPG_VRES, 5,
+				DPG_HRES, 8);
+		}
+		break;
+		case TEST_PATTERN_COLOR_FORMAT_BPC_12:
+		{
+			REG_SET_3(DPG_RAMP_CONTROL, 0,
+				DPG_RAMP0_OFFSET, 1920 << inc_base, // 1920 start point
+				DPG_INC0, inc_base, // step size of 1
+				DPG_INC1, inc_base + 4); // step size of 16 (1 << 4)
 			REG_UPDATE_2(DPG_CONTROL,
 				DPG_VRES, 5,
 				DPG_HRES, 8);
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.c
index e6426ccee2d88..cf8e22289d6ae 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.c
@@ -539,16 +539,11 @@ static bool optc1_enable_crtc(struct timing_generator *optc)
 	REG_UPDATE(CONTROL,
 			VTG0_ENABLE, 1);
 
-	REG_SEQ_START();
-
 	/* Enable CRTC */
 	REG_UPDATE_2(OTG_CONTROL,
 			OTG_DISABLE_POINT_CNTL, 3,
 			OTG_MASTER_EN, 1);
 
-	REG_SEQ_SUBMIT();
-	REG_SEQ_WAIT_DONE();
-
 	return true;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn20/dcn20_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn20/dcn20_optc.c
index c558b1d633f32..73cc8a713556b 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn20/dcn20_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn20/dcn20_optc.c
@@ -63,16 +63,11 @@ bool optc2_enable_crtc(struct timing_generator *optc)
 	REG_UPDATE(CONTROL,
 			VTG0_ENABLE, 1);
 
-	REG_SEQ_START();
-
 	/* Enable CRTC */
 	REG_UPDATE_2(OTG_CONTROL,
 			OTG_DISABLE_POINT_CNTL, 3,
 			OTG_MASTER_EN, 1);
 
-	REG_SEQ_SUBMIT();
-	REG_SEQ_WAIT_DONE();
-
 	return true;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.c
index 98aaa22ce81cf..3ace83e1b50f0 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.c
@@ -105,16 +105,11 @@ static bool optc31_enable_crtc(struct timing_generator *optc)
 	REG_UPDATE(CONTROL,
 			VTG0_ENABLE, 1);
 
-	REG_SEQ_START();
-
 	/* Enable CRTC */
 	REG_UPDATE_2(OTG_CONTROL,
 			OTG_DISABLE_POINT_CNTL, 2,
 			OTG_MASTER_EN, 1);
 
-	REG_SEQ_SUBMIT();
-	REG_SEQ_WAIT_DONE();
-
 	return true;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn314/dcn314_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn314/dcn314_optc.c
index a7cf34937b2f8..7250478a50925 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn314/dcn314_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn314/dcn314_optc.c
@@ -115,16 +115,11 @@ static bool optc314_enable_crtc(struct timing_generator *optc)
 	REG_UPDATE(CONTROL,
 			VTG0_ENABLE, 1);
 
-	REG_SEQ_START();
-
 	/* Enable CRTC */
 	REG_UPDATE_2(OTG_CONTROL,
 			OTG_DISABLE_POINT_CNTL, 2,
 			OTG_MASTER_EN, 1);
 
-	REG_SEQ_SUBMIT();
-	REG_SEQ_WAIT_DONE();
-
 	return true;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn32/dcn32_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn32/dcn32_optc.c
index 07895d5f4dfa0..f9e05efcad988 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn32/dcn32_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn32/dcn32_optc.c
@@ -155,16 +155,11 @@ static bool optc32_enable_crtc(struct timing_generator *optc)
 	REG_UPDATE(CONTROL,
 			VTG0_ENABLE, 1);
 
-	REG_SEQ_START();
-
 	/* Enable CRTC */
 	REG_UPDATE_2(OTG_CONTROL,
 			OTG_DISABLE_POINT_CNTL, 2,
 			OTG_MASTER_EN, 1);
 
-	REG_SEQ_SUBMIT();
-	REG_SEQ_WAIT_DONE();
-
 	return true;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c
index 62f45c156c325..9b7f9d5bbfb38 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c
@@ -122,16 +122,11 @@ static bool optc35_enable_crtc(struct timing_generator *optc)
 	REG_UPDATE(CONTROL,
 			VTG0_ENABLE, 1);
 
-	REG_SEQ_START();
-
 	/* Enable CRTC */
 	REG_UPDATE_2(OTG_CONTROL,
 			OTG_DISABLE_POINT_CNTL, 2,
 			OTG_MASTER_EN, 1);
 
-	REG_SEQ_SUBMIT();
-	REG_SEQ_WAIT_DONE();
-
 	return true;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn401/dcn401_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn401/dcn401_optc.c
index a6d76f451cf8c..5fcdd74eb4a03 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn401/dcn401_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn401/dcn401_optc.c
@@ -189,16 +189,11 @@ bool optc401_enable_crtc(struct timing_generator *optc)
 	REG_UPDATE(CONTROL,
 			VTG0_ENABLE, 1);
 
-	REG_SEQ_START();
-
 	/* Enable CRTC */
 	REG_UPDATE_2(OTG_CONTROL,
 			OTG_DISABLE_POINT_CNTL, 2,
 			OTG_MASTER_EN, 1);
 
-	REG_SEQ_SUBMIT();
-	REG_SEQ_WAIT_DONE();
-
 	return true;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c b/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c
index 729c2b653161d..78b33b2dbae84 100644
--- a/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c
+++ b/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c
@@ -22,6 +22,60 @@
 #define DC_LOGGER \
 	pg_cntl->ctx->logger
 
+/*
+ * ONO PG Workoaround: Saved FGCG repeaters states captured before powering up an ONO
+ * domain so it can be restored once the domain is powered up.
+ */
+struct dcn42_global_fgcg_rep_state {
+	uint32_t dmu_rep_fgcg;
+	uint32_t dccg_global_ono_rep_fgcg;
+	uint32_t az_rep_fgcg;
+};
+
+/* Save and disable FGCG repeaters before powering up the ONO domain. */
+static void pg_cntl42_save_and_disable_global_fgcg_rep(struct pg_cntl *pg_cntl,
+		struct dcn42_global_fgcg_rep_state *state)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
+
+	REG_GET(DMU_CLK_CNTL, LONO_FGCG_REP_DIS, &state->dmu_rep_fgcg);
+	if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_get_global_fgcg_status)
+		state->dccg_global_ono_rep_fgcg = pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_get_global_fgcg_status(pg_cntl->ctx->dc->res_pool->dccg);
+	REG_GET(AZ_CLOCK_CNTL, AZ_GLOBAL_FGCG_REP_DIS, &state->az_rep_fgcg);
+
+	REG_UPDATE(DMU_CLK_CNTL, LONO_FGCG_REP_DIS, 1);
+	if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
+			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, false);
+	REG_UPDATE(AZ_CLOCK_CNTL, AZ_GLOBAL_FGCG_REP_DIS, 1);
+}
+
+/* Restore FGCG repeaters after the ONO domains are powered up. */
+static void pg_cntl42_restore_global_fgcg_rep(struct pg_cntl *pg_cntl,
+		struct dcn42_global_fgcg_rep_state *state)
+{
+	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
+
+	REG_UPDATE(DMU_CLK_CNTL, LONO_FGCG_REP_DIS, state->dmu_rep_fgcg);
+	if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
+			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, state->dccg_global_ono_rep_fgcg);
+	REG_UPDATE(AZ_CLOCK_CNTL, AZ_GLOBAL_FGCG_REP_DIS, state->az_rep_fgcg);
+}
+
+static bool should_skip_pg_control(bool dc_in_idle_opt, bool power_on, bool block_enabled)
+{
+	if (dc_in_idle_opt)
+		return true;
+
+	if (power_on && block_enabled)
+		return true;
+
+	if (!power_on && !block_enabled)
+		return true;
+
+	return false;
+}
+
+
 static bool pg_cntl42_dsc_pg_status(struct pg_cntl *pg_cntl, unsigned int dsc_inst)
 {
 	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
@@ -54,37 +108,23 @@ void pg_cntl42_dsc_pg_control(struct pg_cntl *pg_cntl, unsigned int dsc_inst, bo
 	uint32_t power_gate = power_on ? 0 : 1;
 	uint32_t pwr_status = power_on ? 0 : 2;
 	uint32_t org_ip_request_cntl = 0;
-	bool block_enabled;
-
-	/*need to enable dscclk regardless DSC_PG*/
-	if (pg_cntl->ctx->dc->res_pool->dccg->funcs->enable_dsc && power_on)
-		pg_cntl->ctx->dc->res_pool->dccg->funcs->enable_dsc(
-				pg_cntl->ctx->dc->res_pool->dccg, dsc_inst);
+	struct dcn42_global_fgcg_rep_state fgcg_rep_state = {0};
+	bool block_pg_disabled = pg_cntl->ctx->dc->debug.ignore_pg || pg_cntl->ctx->dc->debug.disable_dsc_power_gate;
 
-    bool skip_pg = pg_cntl->ctx->dc->debug.ignore_pg ||
-			pg_cntl->ctx->dc->debug.disable_dsc_power_gate ||
-			pg_cntl->ctx->dc->idle_optimizations_allowed;
-
-	if (skip_pg && !power_on)
+	if (block_pg_disabled && !power_on)
 		return;
 
-	block_enabled = pg_cntl42_dsc_pg_status(pg_cntl, dsc_inst);
-	if (power_on) {
-		if (block_enabled)
-			return;
-	} else {
-		if (!block_enabled)
-			return;
-	}
+	bool block_enabled = pg_cntl42_dsc_pg_status(pg_cntl, dsc_inst);
+	if (should_skip_pg_control(pg_cntl->ctx->dc->idle_optimizations_allowed, power_on, block_enabled))
+		return;
 
 	REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
 	if (org_ip_request_cntl == 0)
 		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 1);
 
-	if (power_on) {
-		if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
-			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, false);
-	}
+	if (power_on)
+		pg_cntl42_save_and_disable_global_fgcg_rep(pg_cntl, &fgcg_rep_state);
+
 	switch (dsc_inst) {
 	case 0: /* DSC0 */
 		REG_UPDATE(DOMAIN16_PG_CONFIG,
@@ -123,19 +163,11 @@ void pg_cntl42_dsc_pg_control(struct pg_cntl *pg_cntl, unsigned int dsc_inst, bo
 		break;
 	}
 
-	if (power_on) {
-		if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
-			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, true);
-	}
+	if (power_on)
+		pg_cntl42_restore_global_fgcg_rep(pg_cntl, &fgcg_rep_state);
 
 	if (dsc_inst < MAX_PIPES)
 		pg_cntl->pg_pipe_res_enable[PG_DSC][dsc_inst] = power_on;
-
-	if (pg_cntl->ctx->dc->res_pool->dccg->funcs->disable_dsc && !power_on) {
-		/*this is to disable dscclk*/
-		pg_cntl->ctx->dc->res_pool->dccg->funcs->disable_dsc(
-			pg_cntl->ctx->dc->res_pool->dccg, dsc_inst);
-	}
 }
 
 static bool pg_cntl42_hubp_dpp_pg_status(struct pg_cntl *pg_cntl, unsigned int hubp_dpp_inst)
@@ -174,32 +206,24 @@ void pg_cntl42_hubp_dpp_pg_control(struct pg_cntl *pg_cntl, unsigned int hubp_dp
 	uint32_t power_gate = power_on ? 0 : 1;
 	uint32_t pwr_status = power_on ? 0 : 2;
 	uint32_t org_ip_request_cntl;
-	bool block_enabled;
-	bool skip_pg = pg_cntl->ctx->dc->debug.ignore_pg ||
-		       pg_cntl->ctx->dc->debug.disable_hubp_power_gate ||
-		       pg_cntl->ctx->dc->debug.disable_dpp_power_gate ||
-		       pg_cntl->ctx->dc->idle_optimizations_allowed;
+	struct dcn42_global_fgcg_rep_state fgcg_rep_state = {0};
+	bool block_pg_disabled = pg_cntl->ctx->dc->debug.ignore_pg ||
+			pg_cntl->ctx->dc->debug.disable_hubp_power_gate ||
+			pg_cntl->ctx->dc->debug.disable_dpp_power_gate;
 
-	if (skip_pg && !power_on)
+	if (block_pg_disabled && !power_on)
 		return;
 
-	block_enabled = pg_cntl42_hubp_dpp_pg_status(pg_cntl, hubp_dpp_inst);
-	if (power_on) {
-		if (block_enabled)
-			return;
-	} else {
-		if (!block_enabled)
-			return;
-	}
+	bool block_enabled = pg_cntl42_hubp_dpp_pg_status(pg_cntl, hubp_dpp_inst);
+	if (should_skip_pg_control(pg_cntl->ctx->dc->idle_optimizations_allowed, power_on, block_enabled))
+		return;
 
 	REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
 	if (org_ip_request_cntl == 0)
 		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 1);
 
-	if (power_on) {
-		if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
-			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, false);
-	}
+	if (power_on)
+		pg_cntl42_save_and_disable_global_fgcg_rep(pg_cntl, &fgcg_rep_state);
 
 	switch (hubp_dpp_inst) {
 	case 0:
@@ -227,10 +251,9 @@ void pg_cntl42_hubp_dpp_pg_control(struct pg_cntl *pg_cntl, unsigned int hubp_dp
 		break;
 	}
 
-	if (power_on) {
-		if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
-			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, true);
-	}
+	if (power_on)
+		pg_cntl42_restore_global_fgcg_rep(pg_cntl, &fgcg_rep_state);
+
 	DC_LOG_DEBUG("HUBP DPP instance %d, power %s", hubp_dpp_inst,
 		power_on ? "ON" : "OFF");
 
@@ -258,22 +281,18 @@ void pg_cntl42_hpo_pg_control(struct pg_cntl *pg_cntl, bool power_on)
 	uint32_t pwr_status = power_on ? 0 : 2;
 	uint32_t org_ip_request_cntl;
 	uint32_t power_forceon;
-	bool block_enabled;
+	struct dcn42_global_fgcg_rep_state fgcg_rep_state = {0};
 
-	bool skip_pg = pg_cntl->ctx->dc->debug.ignore_pg ||
-			pg_cntl->ctx->dc->debug.disable_hpo_power_gate ||
-			pg_cntl->ctx->dc->idle_optimizations_allowed;
+	bool block_pg_disabled = pg_cntl->ctx->dc->debug.ignore_pg ||
+			pg_cntl->ctx->dc->debug.disable_hpo_power_gate;
 
-	if (skip_pg && !power_on)
+	if (block_pg_disabled && !power_on)
+		return;
+
+	bool block_enabled = pg_cntl42_hpo_pg_status(pg_cntl);
+
+	if (should_skip_pg_control(pg_cntl->ctx->dc->idle_optimizations_allowed, power_on, block_enabled))
 		return;
-	block_enabled = pg_cntl42_hpo_pg_status(pg_cntl);
-	if (power_on) {
-		if (block_enabled)
-			return;
-	} else {
-		if (!block_enabled)
-			return;
-	}
 
 	REG_GET(DOMAIN25_PG_CONFIG, DOMAIN_POWER_FORCEON, &power_forceon);
 	if (power_forceon)
@@ -282,17 +301,15 @@ void pg_cntl42_hpo_pg_control(struct pg_cntl *pg_cntl, bool power_on)
 	REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
 	if (org_ip_request_cntl == 0)
 		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 1);
-	if (power_on) {
-		if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
-			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, false);
-	}
+	if (power_on)
+		pg_cntl42_save_and_disable_global_fgcg_rep(pg_cntl, &fgcg_rep_state);
+
 	REG_UPDATE(DOMAIN25_PG_CONFIG, DOMAIN_POWER_GATE, power_gate);
 	REG_WAIT(DOMAIN25_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, pwr_status, 1, 1000);
 
-	if (power_on) {
-		if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
-			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, true);
-	}
+	if (power_on)
+		pg_cntl42_restore_global_fgcg_rep(pg_cntl, &fgcg_rep_state);
+
 	pg_cntl->pg_res_enable[PG_HPO] = power_on;
 }
 
@@ -314,23 +331,17 @@ void pg_cntl42_io_clk_pg_control(struct pg_cntl *pg_cntl, bool power_on)
 	uint32_t pwr_status = power_on ? 0 : 2;
 	uint32_t org_ip_request_cntl;
 	uint32_t power_forceon;
-	bool block_enabled;
 
-	bool skip_pg = pg_cntl->ctx->dc->debug.ignore_pg ||
-			pg_cntl->ctx->dc->idle_optimizations_allowed ||
+	bool block_pg_disabled = pg_cntl->ctx->dc->debug.ignore_pg ||
 			pg_cntl->ctx->dc->debug.disable_io_clk_power_gate;
 
-	if (skip_pg && !power_on)
+	if (block_pg_disabled && !power_on)
 		return;
 
-	block_enabled = pg_cntl42_io_clk_status(pg_cntl);
-	if (power_on) {
-		if (block_enabled)
-			return;
-	} else {
-		if (!block_enabled)
-			return;
-	}
+	bool block_enabled = pg_cntl42_io_clk_status(pg_cntl);
+
+	if (should_skip_pg_control(pg_cntl->ctx->dc->idle_optimizations_allowed, power_on, block_enabled))
+		return;
 
 	REG_GET(DOMAIN22_PG_CONFIG, DOMAIN_POWER_FORCEON, &power_forceon);
 	if (power_forceon)
@@ -412,24 +423,16 @@ void pg_cntl42_mem_pg_control(struct pg_cntl *pg_cntl, bool power_on)
 	uint32_t pwr_status = power_on ? 0 : 2;
 	uint32_t org_ip_request_cntl;
 	uint32_t power_forceon;
-	bool block_enabled;
 
-	bool skip_pg = pg_cntl->ctx->dc->debug.ignore_pg ||
-			pg_cntl->ctx->dc->idle_optimizations_allowed ||
+	bool block_pg_disabled = pg_cntl->ctx->dc->debug.ignore_pg ||
 			pg_cntl->ctx->dc->debug.disable_mem_power_gate;
 
-	if (skip_pg && !power_on)
+	if (block_pg_disabled && !power_on)
 		return;
 
-	block_enabled = pg_cntl42_mem_status(pg_cntl);
-	if (power_on) {
-		if (block_enabled)
-			return;
-	} else {
-		if (!block_enabled)
-			return;
-	}
-
+	bool block_enabled = pg_cntl42_mem_status(pg_cntl);
+	if (should_skip_pg_control(pg_cntl->ctx->dc->idle_optimizations_allowed, power_on, block_enabled))
+		return;
 	REG_GET(DOMAIN23_PG_CONFIG, DOMAIN_POWER_FORCEON, &power_forceon);
 	if (power_forceon)
 		return;
@@ -466,38 +469,31 @@ void pg_cntl42_dio_pg_control(struct pg_cntl *pg_cntl, bool power_on)
 	uint32_t power_gate = power_on ? 0 : 1;
 	uint32_t pwr_status = power_on ? 0 : 2;
 	uint32_t org_ip_request_cntl;
-	bool block_enabled;
+	struct dcn42_global_fgcg_rep_state fgcg_rep_state = {0};
 
-	bool skip_pg = pg_cntl->ctx->dc->debug.ignore_pg ||
-			pg_cntl->ctx->dc->idle_optimizations_allowed ||
+	bool block_pg_disabled = pg_cntl->ctx->dc->debug.ignore_pg ||
 			pg_cntl->ctx->dc->debug.disable_dio_power_gate;
-	if (skip_pg && !power_on)
+
+	if (block_pg_disabled && !power_on)
 		return;
 
-	block_enabled = pg_cntl42_dio_pg_status(pg_cntl);
-	if (power_on) {
-		if (block_enabled)
-			return;
-	} else {
-		if (!block_enabled)
-			return;
-	}
+	bool block_enabled = pg_cntl42_dio_pg_status(pg_cntl);
+	if (should_skip_pg_control(pg_cntl->ctx->dc->idle_optimizations_allowed, power_on, block_enabled))
+		return;
 
 	REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
 	if (org_ip_request_cntl == 0)
 		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 1);
-	if (power_on) {
-		if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
-			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, false);
-	}
+	if (power_on)
+		pg_cntl42_save_and_disable_global_fgcg_rep(pg_cntl, &fgcg_rep_state);
+
 	/* DIO */
 	REG_UPDATE(DOMAIN26_PG_CONFIG, DOMAIN_POWER_GATE, power_gate);
 	REG_WAIT(DOMAIN26_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, pwr_status, 1, 1000);
 
-	if (power_on) {
-		if (pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg)
-			pg_cntl->ctx->dc->res_pool->dccg->funcs->dccg_enable_global_fgcg(pg_cntl->ctx->dc->res_pool->dccg, true);
-	}
+	if (power_on)
+		pg_cntl42_restore_global_fgcg_rep(pg_cntl, &fgcg_rep_state);
+
 	pg_cntl->pg_res_enable[PG_DIO] = power_on;
 
 }
@@ -509,23 +505,19 @@ void pg_cntl42_plane_otg_pg_control(struct pg_cntl *pg_cntl, bool power_on)
 	uint32_t pwr_status = power_on ? 0 : 2;
 	uint32_t org_ip_request_cntl;
 	unsigned int i;
-	bool block_enabled;
 	bool all_mpcc_disabled = true, all_opp_disabled = true;
 	bool all_optc_disabled = true, all_stream_disabled = true;
 
-	if (pg_cntl->ctx->dc->debug.ignore_pg ||
-		pg_cntl->ctx->dc->debug.disable_optc_power_gate ||
-		pg_cntl->ctx->dc->idle_optimizations_allowed)
+	bool block_pg_disabled = pg_cntl->ctx->dc->debug.ignore_pg ||
+			pg_cntl->ctx->dc->debug.disable_optc_power_gate;
+
+	if (block_pg_disabled && !power_on)
 		return;
 
-	block_enabled = pg_cntl42_plane_otg_status(pg_cntl);
-	if (power_on) {
-		if (block_enabled)
-			return;
-	} else {
-		if (!block_enabled)
-			return;
-	}
+	bool block_enabled = pg_cntl42_plane_otg_status(pg_cntl);
+
+	if (should_skip_pg_control(pg_cntl->ctx->dc->idle_optimizations_allowed, power_on, block_enabled))
+		return;
 
 	for (i = 0; i < pg_cntl->ctx->dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe_ctx = &pg_cntl->ctx->dc->current_state->res_ctx.pipe_ctx[i];
diff --git a/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.h b/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.h
index 7e8f4f03ae0e1..813fa5c811723 100644
--- a/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.h
+++ b/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.h
@@ -34,7 +34,9 @@
 	SR(DOMAIN24_PG_STATUS), \
 	SR(DOMAIN25_PG_STATUS), \
 	SR(DOMAIN26_PG_STATUS), \
-	SR(DC_IP_REQUEST_CNTL)
+	SR(DC_IP_REQUEST_CNTL), \
+	SR(DMU_CLK_CNTL), \
+	SR(AZ_CLOCK_CNTL)
 
 #define PG_CNTL_REG_LIST_DCN42B()\
 	SR(DOMAIN0_PG_CONFIG), \
@@ -63,7 +65,9 @@
 	SR(DOMAIN24_PG_STATUS), \
 	SR(DOMAIN25_PG_STATUS), \
 	SR(DOMAIN26_PG_STATUS), \
-	SR(DC_IP_REQUEST_CNTL)
+	SR(DC_IP_REQUEST_CNTL), \
+	SR(DMU_CLK_CNTL), \
+	SR(AZ_CLOCK_CNTL)
 
 #define PG_CNTL_SF(reg_name, field_name, post_fix)\
 	.field_name = reg_name ## __ ## field_name ## post_fix
@@ -121,7 +125,9 @@
 	PG_CNTL_SF(DOMAIN25_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
 	PG_CNTL_SF(DOMAIN26_PG_STATUS, DOMAIN_DESIRED_PWR_STATE, mask_sh), \
 	PG_CNTL_SF(DOMAIN26_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	PG_CNTL_SF(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh)
+	PG_CNTL_SF(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
+	PG_CNTL_SF(DMU_CLK_CNTL, LONO_FGCG_REP_DIS, mask_sh), \
+	PG_CNTL_SF(AZ_CLOCK_CNTL, AZ_GLOBAL_FGCG_REP_DIS, mask_sh)
 
 /* Not in DCN42B:
  * PG_CNTL_SF(DOMAIN19_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),
@@ -178,7 +184,9 @@
 	PG_CNTL_SF(DOMAIN25_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
 	PG_CNTL_SF(DOMAIN26_PG_STATUS, DOMAIN_DESIRED_PWR_STATE, mask_sh), \
 	PG_CNTL_SF(DOMAIN26_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	PG_CNTL_SF(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh)
+	PG_CNTL_SF(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
+	PG_CNTL_SF(DMU_CLK_CNTL, LONO_FGCG_REP_DIS, mask_sh), \
+	PG_CNTL_SF(AZ_CLOCK_CNTL, AZ_GLOBAL_FGCG_REP_DIS, mask_sh)
 
 struct pg_cntl_shift {
 	uint8_t IP_REQUEST_EN;
@@ -186,6 +194,8 @@ struct pg_cntl_shift {
 	uint8_t DOMAIN_POWER_GATE;
 	uint8_t DOMAIN_DESIRED_PWR_STATE;
 	uint8_t DOMAIN_PGFSM_PWR_STATUS;
+	uint8_t LONO_FGCG_REP_DIS;
+	uint8_t AZ_GLOBAL_FGCG_REP_DIS;
 };
 struct pg_cntl_mask {
 	uint32_t IP_REQUEST_EN;
@@ -193,6 +203,8 @@ struct pg_cntl_mask {
 	uint32_t DOMAIN_POWER_GATE;
 	uint32_t DOMAIN_DESIRED_PWR_STATE;
 	uint32_t DOMAIN_PGFSM_PWR_STATUS;
+	uint32_t LONO_FGCG_REP_DIS;
+	uint32_t AZ_GLOBAL_FGCG_REP_DIS;
 };
 
 struct pg_cntl_registers {
@@ -224,6 +236,8 @@ struct pg_cntl_registers {
 	uint32_t DOMAIN24_PG_STATUS;
 	uint32_t DOMAIN25_PG_STATUS;
 	uint32_t DOMAIN26_PG_STATUS;
+	uint32_t DMU_CLK_CNTL;
+	uint32_t AZ_CLOCK_CNTL;
 };
 
 struct dcn_pg_cntl {
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c
index b92d4f378d602..97ea22af5d2b5 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c
@@ -992,6 +992,11 @@ struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link(
 	for (i = 0; i < pool->stream_enc_count; i++) {
 		if (!res_ctx->is_stream_enc_acquired[i] &&
 				pool->stream_enc[i]) {
+			/* DP/MST needs a digital encoder; skip analog/no-DP encoders */
+			if (dc_is_dp_signal(stream->signal) &&
+			    (!pool->stream_enc[i]->funcs ||
+			     !pool->stream_enc[i]->funcs->dp_set_stream_attribute))
+				continue;
 			/* Store first available for MST second display
 			 * in daisy chain use case
 			 */
@@ -1014,7 +1019,7 @@ struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link(
 	 * required for non DP connectors.
 	 */
 
-	if (j >= 0 && link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT)
+	if (j >= 0 && dc_is_dp_signal(stream->signal))
 		return pool->stream_enc[j];
 
 	return NULL;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c
index 01770df63d0eb..d11ab57afcdd2 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c
@@ -2479,7 +2479,6 @@ static bool dcn30_resource_construct(
 	dc->caps.post_blend_color_processing = true;
 	dc->caps.force_dp_tps4_for_cp2520 = true;
 	dc->caps.hdmi_hpo = true;
-	dc->config.skip_frl_pretraining = true;
 	dc->caps.extended_aux_timeout_support = true;
 	dc->caps.dmcub_support = true;
 
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c
index 58add1071bc12..ae8918a4ad3e4 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c
@@ -1378,7 +1378,6 @@ static bool dcn302_resource_construct(
 	dc->caps.post_blend_color_processing = true;
 	dc->caps.force_dp_tps4_for_cp2520 = true;
 	dc->caps.hdmi_hpo = true;
-	dc->config.skip_frl_pretraining = true;
 	dc->caps.extended_aux_timeout_support = true;
 	dc->caps.dmcub_support = true;
 	dc->caps.max_v_total = (1 << 15) - 1;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c
index 6cb297cd08fd1..75e6f4e46f607 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c
@@ -1322,7 +1322,6 @@ static bool dcn303_resource_construct(
 	dc->caps.post_blend_color_processing = true;
 	dc->caps.force_dp_tps4_for_cp2520 = true;
 	dc->caps.hdmi_hpo = true;
-	dc->config.skip_frl_pretraining = true;
 	dc->caps.extended_aux_timeout_support = true;
 	dc->caps.dmcub_support = true;
 	dc->caps.max_v_total = (1 << 15) - 1;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c
index 7c6a6872688b5..e29efa452c87b 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c
@@ -1189,7 +1189,7 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 {
 	struct dcn20_link_encoder *enc20;
 
-	if (((unsigned int)eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+	if (((unsigned int)eng_id - ENGINE_ID_DIGA) >= ctx->dc->res_pool->res_cap->num_dig_link_enc)
 		return NULL;
 
 	enc20 = kzalloc_obj(struct dcn20_link_encoder);
@@ -1201,6 +1201,8 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 			ctx,
 			&link_enc_feature,
 			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
+			&le_shift,
+			&le_mask,
 			eng_id);
 
 	return &enc20->enc10.base;
@@ -2077,7 +2079,6 @@ static bool dcn31_resource_construct(
 	if (dc->config.forceHBR2CP2520)
 		dc->caps.force_dp_tps4_for_cp2520 = false;
 	dc->caps.hdmi_hpo = true;
-	dc->config.skip_frl_pretraining = true;
 	dc->caps.dp_hpo = true;
 	dc->caps.dp_hdmi21_pcon_support = true;
 	dc->caps.edp_dsc_support = true;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c
index d8096f11fb77c..f50b3250dcba2 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c
@@ -1246,7 +1246,7 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 {
 	struct dcn20_link_encoder *enc20;
 
-	if (((unsigned int)eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+	if (((unsigned int)eng_id - ENGINE_ID_DIGA) >= ctx->dc->res_pool->res_cap->num_dig_link_enc)
 		return NULL;
 
 	enc20 = kzalloc_obj(struct dcn20_link_encoder);
@@ -1258,6 +1258,8 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 			ctx,
 			&link_enc_feature,
 			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
+			&le_shift,
+			&le_mask,
 			eng_id);
 
 	return &enc20->enc10.base;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c
index 3b2e57c6970fa..8297f2f04c167 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c
@@ -1188,7 +1188,7 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 {
 	struct dcn20_link_encoder *enc20;
 
-	if (((unsigned int)eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+	if (((unsigned int)eng_id - ENGINE_ID_DIGA) >= ctx->dc->res_pool->res_cap->num_dig_link_enc)
 		return NULL;
 
 	enc20 = kzalloc_obj(struct dcn20_link_encoder);
@@ -1200,6 +1200,8 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 			ctx,
 			&link_enc_feature,
 			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
+			&le_shift,
+			&le_mask,
 			eng_id);
 
 	return &enc20->enc10.base;
@@ -2053,7 +2055,6 @@ static bool dcn315_resource_construct(
 	if (dc->config.forceHBR2CP2520)
 		dc->caps.force_dp_tps4_for_cp2520 = false;
 	dc->caps.hdmi_hpo = true;
-	dc->config.skip_frl_pretraining = true;
 	dc->caps.dp_hpo = true;
 	dc->caps.dp_hdmi21_pcon_support = true;
 	dc->caps.edp_dsc_support = true;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c
index 924b167bcd74c..046566ad1afed 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c
@@ -1181,7 +1181,7 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 {
 	struct dcn20_link_encoder *enc20;
 
-	if (((unsigned int)eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+	if (((unsigned int)eng_id - ENGINE_ID_DIGA) >= ctx->dc->res_pool->res_cap->num_dig_link_enc)
 		return NULL;
 
 	enc20 = kzalloc_obj(struct dcn20_link_encoder);
@@ -1193,6 +1193,8 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 			ctx,
 			&link_enc_feature,
 			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
+			&le_shift,
+			&le_mask,
 			eng_id);
 
 	return &enc20->enc10.base;
@@ -1927,7 +1929,6 @@ static bool dcn316_resource_construct(
 	if (dc->config.forceHBR2CP2520)
 		dc->caps.force_dp_tps4_for_cp2520 = false;
 	dc->caps.hdmi_hpo = true;
-	dc->config.skip_frl_pretraining = true;
 	dc->caps.dp_hpo = true;
 	dc->caps.dp_hdmi21_pcon_support = true;
 	dc->caps.edp_dsc_support = true;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
index a11110e304fca..004c5690f876e 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
@@ -2405,7 +2405,6 @@ static bool dcn32_resource_construct(
 	if (dc->config.forceHBR2CP2520)
 		dc->caps.force_dp_tps4_for_cp2520 = false;
 	dc->caps.hdmi_hpo = true;
-	dc->config.skip_frl_pretraining = true;
 	dc->caps.dp_hpo = true;
 	dc->caps.dp_hdmi21_pcon_support = true;
 	dc->caps.edp_dsc_support = true;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c
index d1dbcc8ddb71d..53fd322493109 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c
@@ -1897,7 +1897,6 @@ static bool dcn321_resource_construct(
 	dc->caps.post_blend_color_processing = true;
 	dc->caps.force_dp_tps4_for_cp2520 = true;
 	dc->caps.hdmi_hpo = true;
-	dc->config.skip_frl_pretraining = true;
 	dc->caps.dp_hpo = true;
 	dc->caps.dp_hdmi21_pcon_support = true;
 	dc->caps.edp_dsc_support = true;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
index 53596e790eb4e..5541b89b13501 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
@@ -819,7 +819,6 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.enable_hpo_pg_support = false,
 	.enable_single_display_2to1_odm_policy = true,
 	.disable_idle_power_optimizations = false,
-	.dmcub_emulation = false,
 	.disable_boot_optimizations = false,
 	.disable_unbounded_requesting = false,
 	.disable_mem_low_power = false,
@@ -1189,7 +1188,7 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 {
 	struct dcn20_link_encoder *enc20;
 
-	if (((unsigned int)eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+	if (((unsigned int)eng_id - ENGINE_ID_DIGA) >= ctx->dc->res_pool->res_cap->num_dig_link_enc)
 		return NULL;
 
 	enc20 = kzalloc_obj(struct dcn20_link_encoder);
@@ -1201,6 +1200,8 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 			ctx,
 			&link_enc_feature,
 			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
+			&le_shift,
+			&le_mask,
 			eng_id);
 
 	return &enc20->enc10.base;
@@ -2029,7 +2030,6 @@ static bool dcn35_resource_construct(
 	if (dc->config.forceHBR2CP2520)
 		dc->caps.force_dp_tps4_for_cp2520 = false;
 	dc->caps.hdmi_hpo = true;
-	dc->config.skip_frl_pretraining = true;
 	dc->caps.dp_hpo = true;
 	dc->caps.dp_hdmi21_pcon_support = true;
 
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
index 3e2c9cfd555de..053b4380f57e9 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
@@ -799,7 +799,6 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.enable_hpo_pg_support = false,
 	.enable_single_display_2to1_odm_policy = true,
 	.disable_idle_power_optimizations = false,
-	.dmcub_emulation = false,
 	.disable_boot_optimizations = false,
 	.disable_unbounded_requesting = false,
 	.disable_mem_low_power = false,
@@ -1169,7 +1168,7 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 {
 	struct dcn20_link_encoder *enc20;
 
-	if (((unsigned int)eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+	if (((unsigned int)eng_id - ENGINE_ID_DIGA) >= ctx->dc->res_pool->res_cap->num_dig_link_enc)
 		return NULL;
 
 	enc20 = kzalloc_obj(struct dcn20_link_encoder);
@@ -1181,6 +1180,8 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 			ctx,
 			&link_enc_feature,
 			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
+			&le_shift,
+			&le_mask,
 			eng_id);
 
 	return &enc20->enc10.base;
@@ -2002,7 +2003,6 @@ static bool dcn351_resource_construct(
 	if (dc->config.forceHBR2CP2520)
 		dc->caps.force_dp_tps4_for_cp2520 = false;
 	dc->caps.hdmi_hpo = true;
-	dc->config.skip_frl_pretraining = true;
 	dc->caps.dp_hpo = true;
 	dc->caps.dp_hdmi21_pcon_support = true;
 
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c
index 9e795130eb897..592000cf92509 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c
@@ -806,7 +806,6 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.enable_hpo_pg_support = false,
 	.enable_single_display_2to1_odm_policy = true,
 	.disable_idle_power_optimizations = false,
-	.dmcub_emulation = false,
 	.disable_boot_optimizations = false,
 	.disable_unbounded_requesting = false,
 	.disable_mem_low_power = false,
@@ -1176,7 +1175,7 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 {
 	struct dcn20_link_encoder *enc20;
 
-	if (((unsigned int)eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+	if (((unsigned int)eng_id - ENGINE_ID_DIGA) >= ctx->dc->res_pool->res_cap->num_dig_link_enc)
 		return NULL;
 
 	enc20 = kzalloc_obj(struct dcn20_link_encoder);
@@ -1188,6 +1187,8 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 			ctx,
 			&link_enc_feature,
 			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
+			&le_shift,
+			&le_mask,
 			eng_id);
 
 	return &enc20->enc10.base;
@@ -1999,7 +2000,6 @@ static bool dcn36_resource_construct(
 	if (dc->config.forceHBR2CP2520)
 		dc->caps.force_dp_tps4_for_cp2520 = false;
 	dc->caps.hdmi_hpo = true;
-	dc->config.skip_frl_pretraining = true;
 	dc->caps.dp_hpo = true;
 	dc->caps.dp_hdmi21_pcon_support = true;
 
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.h
index 47f82b8182627..ce96fe1ddb7c7 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.h
@@ -622,7 +622,8 @@ int dcn401_get_power_profile(const struct dc_state *context);
 	SR(DCHUBBUB_MEM_PWR_MODE_CTRL),                                          \
 	SR(DCHUBBUB_TIMEOUT_DETECTION_CTRL1),                                    \
 	SR(DCHUBBUB_TIMEOUT_DETECTION_CTRL2),									 \
-	SR(DCHUBBUB_CTRL_STATUS)
+	SR(DCHUBBUB_CTRL_STATUS),                                                \
+	SR(DCHUBBUB_ARB_MALL_CNTL)
 
 /* DCCG */
 
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c
index 7de12b16d7ade..547a0b8165390 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c
@@ -729,8 +729,8 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.clock_trace = true,
 	.disable_pplib_clock_request = false,
 	.ignore_pg = false,
-	.disable_dpp_power_gate = true,
-	.disable_hubp_power_gate = true,
+	.disable_dpp_power_gate = false,
+	.disable_hubp_power_gate = false,
 	.disable_optc_power_gate = true,
 	.disable_dsc_power_gate = false,
 	.disable_dio_power_gate = true,
@@ -762,7 +762,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 		.bits = {
 			.dpp = true,
 			.dsc = true,/*dscclk and dsc pg*/
-			.hdmistream = false,
+			.hdmistream = true,
 			.hdmichar = true,
 			.dpstream = true,
 			.symclk32_se = true,
@@ -801,6 +801,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.replay_skip_crtc_disabled = true,
 	.psr_skip_crtc_disable = true,
 	.force_odm2to1_for_edp_pixclk_mhz = 0, // disable the policy for now
+	.iommu_mismatch_temp_wka = 0x7,
 };
 
 static const struct dc_check_config config_defaults = {
@@ -1881,7 +1882,7 @@ static struct link_encoder *dcn42_link_enc_create_minimal(
 {
 	struct dcn20_link_encoder *enc20;
 
-	if ((unsigned int)(eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+	if ((unsigned int)(eng_id - ENGINE_ID_DIGA) >= ctx->dc->res_pool->res_cap->num_dig_link_enc)
 		return NULL;
 
 	enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
@@ -1893,6 +1894,8 @@ static struct link_encoder *dcn42_link_enc_create_minimal(
 			ctx,
 			&link_enc_feature,
 			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
+			&le_shift,
+			&le_mask,
 			eng_id);
 
 	return &enc20->enc10.base;
@@ -2142,6 +2145,7 @@ static bool dcn42_resource_construct(
 	dc->config.use_pipe_ctx_sync_logic = true;
 	dc->config.dc_mode_clk_limit_support = false;
 	dc->config.enable_windowed_mpo_odm = true;
+	dc->config.set_pipe_unlock_order = true; /* Need to ensure DET gets freed before allocating */
 	/* Use psp mailbox to enable assr */
 	dc->config.use_assr_psp_message = true;
 	/* dcn42 and afterward always support external panel replay */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.h
index 3e4d9b188b268..f2b98f1574ed5 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.h
@@ -581,6 +581,7 @@
 		SRI_ARR(DCHUBP_VMPG_CONFIG, HUBP, id),                         \
 		SRI_ARR(UCLK_PSTATE_FORCE, HUBPREQ, id),                       \
 		SRI_ARR(HUBP_3DLUT_DLG_PARAM, CURSOR0_, id),                   \
+		SRI_ARR(DST_Y_DELTA_DRQ_LIMIT, HUBPREQ, id),               \
 		HUBP_3DLUT_FL_REG_LIST_DCN401(id)
 struct dcn42_resource_pool {
 	struct resource_pool base;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c
index 527d17f29f3ba..2334bc5b75b8f 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c
@@ -22,6 +22,7 @@
 #include "dcn35/dcn35_resource.h"
 #include "dcn321/dcn321_resource.h"
 #include "dcn401/dcn401_resource.h"
+#include "dcn42/dcn42_resource.h"
 #include "dcn42/dcn42_resource_fpu.h"
 
 #include "dcn10/dcn10_ipp.h"
@@ -116,6 +117,36 @@
 #define regAPG9_APG_DBG_GEN_CONTROL             0x38ae
 #define regAPG9_APG_DBG_GEN_CONTROL_BASE_IDX    2
 
+#define regHUBP0_HUBPREQ_DEBUG_DB             0x05f8
+#define regHUBP0_HUBPREQ_DEBUG_DB_BASE_IDX    2
+#define regHUBP0_HUBPREQ_DEBUG                0x05f9
+#define regHUBP0_HUBPREQ_DEBUG_BASE_IDX       2
+#define regHUBP1_HUBPREQ_DEBUG_DB             0x06d4
+#define regHUBP1_HUBPREQ_DEBUG_DB_BASE_IDX    2
+#define regHUBP1_HUBPREQ_DEBUG                0x06d5
+#define regHUBP1_HUBPREQ_DEBUG_BASE_IDX       2
+#define regHUBP2_HUBPREQ_DEBUG_DB             0x07b0
+#define regHUBP2_HUBPREQ_DEBUG_DB_BASE_IDX    2
+#define regHUBP2_HUBPREQ_DEBUG                0x07b1
+#define regHUBP2_HUBPREQ_DEBUG_BASE_IDX       2
+#define regHUBP3_HUBPREQ_DEBUG_DB             0x088c
+#define regHUBP3_HUBPREQ_DEBUG_DB_BASE_IDX    2
+#define regHUBP3_HUBPREQ_DEBUG                0x088d
+#define regHUBP3_HUBPREQ_DEBUG_BASE_IDX       2
+
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_CONTROL                                                   0x3687
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_VID_CRC_CONTROL_BASE_IDX                                          2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_CONTROL                                                   0x375b
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_VID_CRC_CONTROL_BASE_IDX                                          2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_CONTROL                                                   0x382f
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_VID_CRC_CONTROL_BASE_IDX                                          2
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_HBLANK_CONTROL                                                    0x366b
+#define regDP_SYM32_ENC0_DP_SYM32_ENC_HBLANK_CONTROL_BASE_IDX                                           2
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_HBLANK_CONTROL                                                    0x373f
+#define regDP_SYM32_ENC1_DP_SYM32_ENC_HBLANK_CONTROL_BASE_IDX                                           2
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_HBLANK_CONTROL                                                    0x3813
+#define regDP_SYM32_ENC2_DP_SYM32_ENC_HBLANK_CONTROL_BASE_IDX                                           2
+
 enum dcn401_clk_src_array_id {
 	DCN401_CLK_SRC_PLL0,
 	DCN401_CLK_SRC_PLL1,
@@ -235,10 +266,10 @@ static struct bios_registers bios_regs;
 static struct dce110_clk_src_regs clk_src_regs[5];
 
 static const struct dce110_clk_src_shift cs_shift = {
-	CS_COMMON_MASK_SH_LIST_DCN3_2(__SHIFT)
+	CS_COMMON_MASK_SH_LIST_DCN4_0_1(__SHIFT)
 };
 static const struct dce110_clk_src_mask cs_mask = {
-	CS_COMMON_MASK_SH_LIST_DCN3_2(_MASK)
+	CS_COMMON_MASK_SH_LIST_DCN4_0_1(_MASK)
 };
 #define abm_regs_init(id) \
 	ABM_DCN42B_REG_LIST_RI(id)
@@ -333,7 +364,7 @@ static const struct dcn10_link_enc_mask le_mask = {
 	LINK_ENCODER_MASK_SH_LIST_DCN42B(_MASK)};
 
 #define hpo_dp_stream_encoder_reg_init(id) \
-	DCN42B_HPO_DP_STREAM_ENC_REG_LIST_RI(id)
+	DCN42_HPO_DP_STREAM_ENC_REG_LIST_RI(id)
 
 static struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[4];
 
@@ -461,7 +492,7 @@ static const struct dcn_optc_mask optc_mask = {
 	OPTC_COMMON_MASK_SH_LIST_DCN42B(_MASK)};
 
 #define hubp_regs_init(id) \
-	HUBP_REG_LIST_DCN42B_RI(id)
+	HUBP_REG_LIST_DCN42_RI(id)
 
 static struct dcn_hubp2_registers hubp_regs[4];
 
@@ -757,7 +788,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.underflow_assert_delay_us = 0xFFFFFFFF,
 	.dwb_fi_phase = -1, // -1 = disable,
 	.dmub_command_table = true,
-	.pstate_enabled = false,
+	.pstate_enabled = true,
 	.enable_mem_low_power = {
 		.bits = {
 			.vga = false,
@@ -783,7 +814,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 		}
 	},
 	.seamless_boot_odm_combine = DML_FAIL_SOURCE_PIXEL_FORMAT,
-	.enable_z9_disable_interface = false, /* Allow support for the PMFW interface for disable Z9*/
+	.enable_z9_disable_interface = true, /* Allow support for the PMFW interface for disable Z9*/
 	.minimum_z8_residency_time = 1, /* Always allow when other conditions are met */
 	.support_eDP1_5 = true,
 	.use_max_lb = true,
@@ -805,7 +836,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.disable_timeout = true,
 	.min_disp_clk_khz = 50000,
 	.static_screen_wait_frames = 2,
-	.disable_z10 = true,
+	.disable_z10 = false,
 	.ignore_pg = true,
 	.disable_stutter_for_wm_program = true,
 	.min_deep_sleep_dcfclk_khz = 8000,
@@ -1016,9 +1047,9 @@ static struct hubp *dcn42b_hubp_create(
 }
 static const struct dc_panel_config dcn42b_panel_config_defaults = {
 	.psr = {
-		.disable_psr = true,
+		.disable_psr = false,
 		.disallow_psrsu = true,
-		.disallow_replay = true,
+		.disallow_replay = false,
 	},
 	.ilr = {
 		.optimize_edp_link_rate = true,
@@ -1824,7 +1855,7 @@ static struct link_encoder *dcn42b_link_enc_create_minimal(
 {
 	struct dcn20_link_encoder *enc20;
 
-	if ((unsigned int)(eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+	if ((unsigned int)(eng_id - ENGINE_ID_DIGA) >= ctx->dc->res_pool->res_cap->num_dig_link_enc)
 		return NULL;
 
 	enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
@@ -1836,6 +1867,8 @@ static struct link_encoder *dcn42b_link_enc_create_minimal(
 			ctx,
 			&link_enc_feature,
 			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
+			&le_shift,
+			&le_mask,
 			eng_id);
 
 	return &enc20->enc10.base;
@@ -1882,9 +1915,7 @@ static struct resource_funcs dcn42b_res_pool_funcs = {
 	.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
 	.add_phantom_pipes = dcn32_add_phantom_pipes,
 	.calculate_mall_ways_from_bytes = dcn32_calculate_mall_ways_from_bytes,
-#ifdef CONFIG_DRM_AMD_DC_DML21
 	.prepare_mcache_programming = dcn42b_prepare_mcache_programming,
-#endif
 	.build_pipe_pix_clk_params = dcn42b_build_pipe_pix_clk_params,
 	.get_power_profile = dcn401_get_power_profile,
 	.get_vstartup_for_pipe = dcn401_get_vstartup_for_pipe,
@@ -1986,11 +2017,10 @@ static bool dcn42b_resource_construct(
 	dc->caps.dmcub_support = true;
 	dc->caps.is_apu = true;
 	dc->caps.seamless_odm = true;
-	dc->caps.zstate_support = false;
-	dc->caps.ips_support = false;
+	dc->caps.zstate_support = true;
+	dc->caps.ips_support = true;
 	dc->caps.max_v_total = (1 << 15) - 1;
 	dc->caps.vtotal_limited_by_fp2 = true;
-	dc->config.disable_ips = DMUB_IPS_DISABLE_ALL;
 
 	/* Color pipeline capabilities */
 	dc->caps.color.dpp.dcn_arch = 1;
@@ -2087,6 +2117,7 @@ static bool dcn42b_resource_construct(
 	dc->config.use_pipe_ctx_sync_logic = true;
 	dc->config.dc_mode_clk_limit_support = false;
 	dc->config.enable_windowed_mpo_odm = true;
+	dc->config.set_pipe_unlock_order = true; /* Need to ensure DET gets freed before allocating */
 	/* Use psp mailbox to enable assr */
 	dc->config.use_assr_psp_message = true;
 	/* dcn42 and afterward always support external panel replay */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c.orig b/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c.orig
new file mode 100644
index 0000000000000..eaf15022d25e0
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c.orig
@@ -0,0 +1,2422 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#include "dm_services.h"
+#include "dc.h"
+
+#include "dcn32/dcn32_init.h"
+#include "dcn42/dcn42_init.h"
+#include "dcn42b/dcn42b_init.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+
+#include "dcn42b_resource.h"
+#include "dcn20/dcn20_resource.h"
+#include "dcn30/dcn30_resource.h"
+#include "dcn31/dcn31_resource.h"
+#include "dcn32/dcn32_resource.h"
+#include "dcn35/dcn35_resource.h"
+#include "dcn321/dcn321_resource.h"
+#include "dcn401/dcn401_resource.h"
+#include "dcn42/dcn42_resource.h"
+#include "dcn42/dcn42_resource_fpu.h"
+
+#include "dcn10/dcn10_ipp.h"
+#include "dcn35/dcn35_hubbub.h"
+#include "dcn42/dcn42_hubbub.h"
+#include "dcn401/dcn401_mpc.h"
+#include "dcn42/dcn42_mpc.h"
+#include "dcn35/dcn35_hubp.h"
+#include "dcn42/dcn42_hubp.h"
+#include "irq/dcn42/irq_service_dcn42.h"
+#include "dcn42/dcn42_dpp.h"
+#include "dcn401/dcn401_dsc.h"
+#include "dcn42/dcn42_optc.h"
+#include "dcn20/dcn20_hwseq.h"
+#include "dcn30/dcn30_hwseq.h"
+#include "dce110/dce110_hwseq.h"
+#include "dcn35/dcn35_opp.h"
+#include "dcn30/dcn30_vpg.h"
+#include "dcn31/dcn31_vpg.h"
+#include "dcn42/dcn42_dio_stream_encoder.h"
+#include "dcn42/dcn42_pg_cntl.h"
+#include "dcn31/dcn31_hpo_dp_stream_encoder.h"
+#include "dcn31/dcn31_hpo_dp_link_encoder.h"
+#include "dcn32/dcn32_hpo_dp_link_encoder.h"
+#include "dcn42/dcn42_hpo_dp_link_encoder.h"
+#include "dcn31/dcn31_apg.h"
+#include "dcn31/dcn31_dio_link_encoder.h"
+#include "dcn401/dcn401_dio_link_encoder.h"
+#include "dcn10/dcn10_link_encoder.h"
+#include "dcn321/dcn321_dio_link_encoder.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "clk_mgr.h"
+#include "dio/virtual/virtual_stream_encoder.h"
+#include "dml/display_mode_vba.h"
+#include "dcn42/dcn42_dccg.h"
+#include "dcn10/dcn10_resource.h"
+#include "link_service.h"
+#include "dcn31/dcn31_panel_cntl.h"
+
+#include "dcn30/dcn30_dwb.h"
+#include "dcn42/dcn42_mmhubbub.h"
+#include "dcn42/dcn42_dio_link_encoder.h"
+
+#include "dcn/dcn_4_2_1_offset.h"
+#include "dcn/dcn_4_2_1_sh_mask.h"
+#include "dpcs/dpcs_4_0_1_offset.h"
+#include "dpcs/dpcs_4_0_1_sh_mask.h"
+
+#include "reg_helper.h"
+#include "dce/dmub_abm.h"
+#include "dce/dmub_psr.h"
+#include "dce/dmub_replay.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+
+#include "dml/dcn30/display_mode_vba_30.h"
+#include "vm_helper.h"
+#include "dcn20/dcn20_vmid.h"
+
+#include "dc_state_priv.h"
+#include "link_enc_cfg.h"
+
+#include "dml2_0/dml2_wrapper.h"
+
+#define DC_LOGGER_INIT(logger)
+
+#define regBIF_BX1_BIOS_SCRATCH_3                                                                       0x003b
+#define regBIF_BX1_BIOS_SCRATCH_3_BASE_IDX                                                              1
+#define regBIF_BX1_BIOS_SCRATCH_6                                                                       0x003e
+#define regBIF_BX1_BIOS_SCRATCH_6_BASE_IDX                                                              1
+
+#define regAPG0_APG_DBG_GEN_CONTROL             0x2075
+#define regAPG0_APG_DBG_GEN_CONTROL_BASE_IDX    2
+#define regAPG1_APG_DBG_GEN_CONTROL             0x2199
+#define regAPG1_APG_DBG_GEN_CONTROL_BASE_IDX    2
+#define regAPG2_APG_DBG_GEN_CONTROL             0x22bd
+#define regAPG2_APG_DBG_GEN_CONTROL_BASE_IDX    2
+#define regAPG3_APG_DBG_GEN_CONTROL             0x23e1
+#define regAPG3_APG_DBG_GEN_CONTROL_BASE_IDX    2
+#define regAPG4_APG_DBG_GEN_CONTROL             0x2505
+#define regAPG4_APG_DBG_GEN_CONTROL_BASE_IDX    2
+#define regAPG5_APG_DBG_GEN_CONTROL             0x0915
+#define regAPG5_APG_DBG_GEN_CONTROL_BASE_IDX    3
+#define regAPG6_APG_DBG_GEN_CONTROL             0x3632
+#define regAPG6_APG_DBG_GEN_CONTROL_BASE_IDX    2
+#define regAPG7_APG_DBG_GEN_CONTROL             0x3706
+#define regAPG7_APG_DBG_GEN_CONTROL_BASE_IDX    2
+#define regAPG8_APG_DBG_GEN_CONTROL             0x37da
+#define regAPG8_APG_DBG_GEN_CONTROL_BASE_IDX    2
+#define regAPG9_APG_DBG_GEN_CONTROL             0x38ae
+#define regAPG9_APG_DBG_GEN_CONTROL_BASE_IDX    2
+
+#define regHUBP0_HUBPREQ_DEBUG_DB             0x05f8
+#define regHUBP0_HUBPREQ_DEBUG_DB_BASE_IDX    2
+#define regHUBP0_HUBPREQ_DEBUG                0x05f9
+#define regHUBP0_HUBPREQ_DEBUG_BASE_IDX       2
+#define regHUBP1_HUBPREQ_DEBUG_DB             0x06d4
+#define regHUBP1_HUBPREQ_DEBUG_DB_BASE_IDX    2
+#define regHUBP1_HUBPREQ_DEBUG                0x06d5
+#define regHUBP1_HUBPREQ_DEBUG_BASE_IDX       2
+#define regHUBP2_HUBPREQ_DEBUG_DB             0x07b0
+#define regHUBP2_HUBPREQ_DEBUG_DB_BASE_IDX    2
+#define regHUBP2_HUBPREQ_DEBUG                0x07b1
+#define regHUBP2_HUBPREQ_DEBUG_BASE_IDX       2
+#define regHUBP3_HUBPREQ_DEBUG_DB             0x088c
+#define regHUBP3_HUBPREQ_DEBUG_DB_BASE_IDX    2
+#define regHUBP3_HUBPREQ_DEBUG                0x088d
+#define regHUBP3_HUBPREQ_DEBUG_BASE_IDX       2
+
+enum dcn401_clk_src_array_id {
+	DCN401_CLK_SRC_PLL0,
+	DCN401_CLK_SRC_PLL1,
+	DCN401_CLK_SRC_PLL2,
+	DCN401_CLK_SRC_TOTAL
+};
+
+/* begin
+ * macros to expend register list macro defined in HW object header file
+ */
+
+/* DCN */
+#define BASE_INNER(seg) ctx->dcn_reg_offsets[seg]
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)                                       \
+	REG_STRUCT.reg_name = BASE(reg##reg_name##_BASE_IDX) + \
+						  reg##reg_name
+#define SR_ARR(reg_name, id)                                   \
+	REG_STRUCT[id].reg_name = BASE(reg##reg_name##_BASE_IDX) + \
+							  reg##reg_name
+#define SR_ARR_INIT(reg_name, id, value) \
+	REG_STRUCT[id].reg_name = value
+
+#define SRI(reg_name, block, id)                                         \
+	REG_STRUCT.reg_name = BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+						  reg##block##id##_##reg_name
+
+#define SRI_ARR(reg_name, block, id)                                         \
+	REG_STRUCT[id].reg_name = BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+							  reg##block##id##_##reg_name
+
+/*
+ * Used when a reg_name would otherwise begin with an integer
+ */
+#define SRI_ARR_US(reg_name, block, id)                                   \
+	REG_STRUCT[id].reg_name = BASE(reg##block##id##reg_name##_BASE_IDX) + \
+							  reg##block##id##reg_name
+#define SR_ARR_I2C(reg_name, id) \
+	REG_STRUCT[id - 1].reg_name = BASE(reg##reg_name##_BASE_IDX) + reg##reg_name
+
+#define SRI_ARR_I2C(reg_name, block, id)                                         \
+	REG_STRUCT[id - 1].reg_name = BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+								  reg##block##id##_##reg_name
+
+#define SRI_ARR_ALPHABET(reg_name, block, index, id)                            \
+	REG_STRUCT[index].reg_name = BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+								 reg##block##id##_##reg_name
+
+#define SRI2(reg_name, block, id)                \
+	.reg_name = BASE(reg##reg_name##_BASE_IDX) + \
+				reg##reg_name
+#define SRI2_ARR(reg_name, block, id)                          \
+	REG_STRUCT[id].reg_name = BASE(reg##reg_name##_BASE_IDX) + \
+							  reg##reg_name
+
+#define SRIR(var_name, reg_name, block, id)                    \
+	.var_name = BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+				reg##block##id##_##reg_name
+
+#define SRII(reg_name, block, id)                                            \
+	REG_STRUCT.reg_name[id] = BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+							  reg##block##id##_##reg_name
+
+#define SRII_ARR_2(reg_name, block, id, inst)                                      \
+	REG_STRUCT[inst].reg_name[id] = BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+									reg##block##id##_##reg_name
+
+#define SRII_MPC_RMU(reg_name, block, id)                                  \
+	.RMU##_##reg_name[id] = BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+							reg##block##id##_##reg_name
+
+#define SRII_DWB(reg_name, temp_name, block, id)                              \
+	REG_STRUCT.reg_name[id] = \
+		BASE(reg##block##id##_##temp_name##_BASE_IDX) + \
+							  reg##block##id##_##temp_name
+
+#define DCCG_SRII(reg_name, block, id)                                                 \
+	REG_STRUCT.block##_##reg_name[id] = \
+		BASE(reg##block##id##_##reg_name##_BASE_IDX) + \
+										reg##block##id##_##reg_name
+
+#define SF_DWB2(reg_name, block, id, field_name, post_fix) \
+	.field_name = reg_name##__##field_name##post_fix
+
+#define VUPDATE_SRII(reg_name, block, id)                                    \
+	REG_STRUCT.reg_name[id] = BASE(reg##reg_name##_##block##id##_BASE_IDX) + \
+							  reg##reg_name##_##block##id
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) ctx->nbio_reg_offsets[seg]
+
+#define NBIO_BASE(seg) \
+	NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)                                               \
+	REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX1_##reg_name##_BASE_IDX) + \
+						  regBIF_BX1_##reg_name
+#define NBIO_SR_ARR(reg_name, id)                                           \
+	REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX1_##reg_name##_BASE_IDX) + \
+							  regBIF_BX1_##reg_name
+
+#define CTX ctx
+#define REG(reg_name) \
+	(ctx->dcn_reg_offsets[reg##reg_name##_BASE_IDX] + reg##reg_name)
+
+static struct bios_registers bios_regs;
+
+#define bios_regs_init()     \
+	NBIO_SR(BIOS_SCRATCH_3), \
+		NBIO_SR(BIOS_SCRATCH_6)
+
+#define clk_src_regs_init(index, pllid) \
+	CS_COMMON_REG_LIST_DCN42B_RI(index, pllid)
+
+static struct dce110_clk_src_regs clk_src_regs[5];
+
+static const struct dce110_clk_src_shift cs_shift = {
+	CS_COMMON_MASK_SH_LIST_DCN4_0_1(__SHIFT)
+};
+static const struct dce110_clk_src_mask cs_mask = {
+	CS_COMMON_MASK_SH_LIST_DCN4_0_1(_MASK)
+};
+#define abm_regs_init(id) \
+	ABM_DCN42B_REG_LIST_RI(id)
+
+static struct dce_abm_registers abm_regs[4];
+
+static const struct dce_abm_shift abm_shift = {
+	ABM_MASK_SH_LIST_DCN42(__SHIFT)};
+
+static const struct dce_abm_mask abm_mask = {
+	ABM_MASK_SH_LIST_DCN42(_MASK)};
+
+#define audio_regs_init(id) \
+	AUD_COMMON_REG_LIST_RI(id)
+
+static struct dce_audio_registers audio_regs[5];
+
+/* DTO2 not present for DCN42b */
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)                                             \
+	SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),   \
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh), \
+		SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO0_SOURCE_SEL, mask_sh),                     \
+		SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO_SEL, mask_sh),                             \
+		SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO0_USE_512FBR_DTO, mask_sh),                 \
+		SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO1_USE_512FBR_DTO, mask_sh),                 \
+		SF(DCCG_AUDIO_DTO0_MODULE, DCCG_AUDIO_DTO0_MODULE, mask_sh),                        \
+		SF(DCCG_AUDIO_DTO0_PHASE, DCCG_AUDIO_DTO0_PHASE, mask_sh),                          \
+		SF(DCCG_AUDIO_DTO1_MODULE, DCCG_AUDIO_DTO1_MODULE, mask_sh),                        \
+		SF(DCCG_AUDIO_DTO1_PHASE, DCCG_AUDIO_DTO1_PHASE, mask_sh),                          \
+		SF(AZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES, AUDIO_RATE_CAPABILITIES, mask_sh), \
+		SF(AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES, CLKSTOP, mask_sh),              \
+		SF(AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES, EPSS, mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+	DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)};
+
+static const struct dce_audio_mask audio_mask = {
+	DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)};
+
+#define vpg_regs_init(id) \
+	VPG_DCN401_REG_LIST_RI(id)
+
+static struct dcn31_vpg_registers vpg_regs[10];
+
+static const struct dcn31_vpg_shift vpg_shift = {
+	DCN31_VPG_MASK_SH_LIST(__SHIFT)};
+
+static const struct dcn31_vpg_mask vpg_mask = {
+	DCN31_VPG_MASK_SH_LIST(_MASK)};
+
+#define apg_regs_init(id) \
+	APG_DCN31_REG_LIST_RI(id)
+
+static struct dcn31_apg_registers apg_regs[10];
+
+static const struct dcn31_apg_shift apg_shift = {
+	DCN42B_APG_MASK_SH_LIST(__SHIFT)};
+
+static const struct dcn31_apg_mask apg_mask = {
+	DCN42B_APG_MASK_SH_LIST(_MASK)};
+
+#define stream_enc_regs_init(id) \
+	SE_DCN42B_REG_LIST_RI(id)
+
+static struct dcn10_stream_enc_registers stream_enc_regs[5];
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+	SE_COMMON_MASK_SH_LIST_DCN42(__SHIFT)};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+	SE_COMMON_MASK_SH_LIST_DCN42(_MASK)};
+
+#define aux_regs_init(id) \
+	DCN2_AUX_REG_LIST_RI(id)
+
+static struct dcn10_link_enc_aux_registers link_enc_aux_regs[5];
+
+#define hpd_regs_init(id) \
+	HPD_REG_LIST_RI(id)
+
+static struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[5];
+
+#define link_regs_init(id, phyid) \
+	LE_DCN401_REG_LIST_RI(id)
+
+static struct dcn10_link_enc_registers link_enc_regs[5];
+
+static const struct dcn10_link_enc_shift le_shift = {
+	LINK_ENCODER_MASK_SH_LIST_DCN42B(__SHIFT)};
+
+static const struct dcn10_link_enc_mask le_mask = {
+	LINK_ENCODER_MASK_SH_LIST_DCN42B(_MASK)};
+
+#define hpo_dp_stream_encoder_reg_init(id) \
+	DCN42_HPO_DP_STREAM_ENC_REG_LIST_RI(id)
+
+static struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[4];
+
+static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
+	DCN4_2B_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)};
+
+static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
+	DCN4_2B_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)};
+
+#define hpo_dp_link_encoder_reg_init(id) \
+	DCN42B_HPO_DP_LINK_ENC_REG_LIST_RI(id)
+/*DCN3_1_RDPCSTX_REG_LIST(0),*/
+/*DCN3_1_RDPCSTX_REG_LIST(1),*/
+/*DCN3_1_RDPCSTX_REG_LIST(2),*/
+/*DCN3_1_RDPCSTX_REG_LIST(3),*/
+
+/* DCN42B has 2 HPO DP link encoders */
+static struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[2];
+
+static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
+	DCN3_2_HPO_DP_LINK_ENC_MASK_SH_LIST(__SHIFT)};
+
+static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
+	DCN3_2_HPO_DP_LINK_ENC_MASK_SH_LIST(_MASK)};
+
+#define dpp_regs_init(id) \
+	DPP_REG_LIST_DCN42B_COMMON_RI(id)
+
+static struct dcn42_dpp_registers dpp_regs[4];
+
+static const struct dcn42_dpp_shift tf_shift = {
+	DPP_REG_LIST_SH_MASK_DCN42_COMMON(__SHIFT)};
+
+static const struct dcn42_dpp_mask tf_mask = {
+	DPP_REG_LIST_SH_MASK_DCN42_COMMON(_MASK)};
+
+#define opp_regs_init(id) \
+	OPP_REG_LIST_DCN401_RI(id)
+
+static struct dcn20_opp_registers opp_regs[4];
+
+static const struct dcn20_opp_shift opp_shift = {
+	OPP_MASK_SH_LIST_DCN20(__SHIFT)};
+
+static const struct dcn20_opp_mask opp_mask = {
+	OPP_MASK_SH_LIST_DCN20(_MASK)};
+
+#define aux_engine_regs_init(id)                                              \
+	AUX_COMMON_REG_LIST0_RI(id), SR_ARR_INIT(AUXN_IMPCAL, id, 0),             \
+		SR_ARR_INIT(AUXP_IMPCAL, id, 0),                                      \
+		SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK), \
+		SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK)
+
+static struct dce110_aux_registers aux_engine_regs[5];
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCN_AUX_MASK_SH_LIST(__SHIFT)};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCN_AUX_MASK_SH_LIST(_MASK)};
+
+#define dwbc_regs_dcn401_init(id) \
+	DWBC_COMMON_REG_LIST_DCN30_RI(id)
+
+static struct dcn30_dwbc_registers dwbc401_regs[1];
+
+static const struct dcn30_dwbc_shift dwbc401_shift = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)};
+
+static const struct dcn30_dwbc_mask dwbc401_mask = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)};
+
+#define mcif_wb_regs_dcn3_init(id) \
+	MCIF_WB_COMMON_REG_LIST_DCN3_5_RI(id)
+
+static struct dcn35_mmhubbub_registers mcif_wb35_regs[1];
+
+static const struct dcn35_mmhubbub_shift mcif_wb35_shift = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN3_5(__SHIFT)};
+
+static const struct dcn35_mmhubbub_mask mcif_wb35_mask = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN3_5(_MASK)};
+
+#define dsc_regs_init(id) \
+	DSC_REG_LIST_DCN401_RI(id)
+
+static struct dcn401_dsc_registers dsc_regs[4];
+
+static const struct dcn401_dsc_shift dsc_shift = {
+	DSC_REG_LIST_SH_MASK_DCN401(__SHIFT)};
+
+static const struct dcn401_dsc_mask dsc_mask = {
+	DSC_REG_LIST_SH_MASK_DCN401(_MASK)};
+
+static struct dcn42_mpc_registers mpc_regs;
+
+#define dcn_mpc_regs_init()                \
+	MPC_REG_LIST_DCN42(0),                 \
+		MPC_REG_LIST_DCN42(1),             \
+		MPC_REG_LIST_DCN42(2),             \
+		MPC_REG_LIST_DCN42(3),             \
+		MPC_OUT_MUX_REG_LIST_DCN3_0_RI(0), \
+		MPC_OUT_MUX_REG_LIST_DCN3_0_RI(1), \
+		MPC_OUT_MUX_REG_LIST_DCN3_0_RI(2), \
+		MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3), \
+		MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0), \
+		MPC_RMCM_REG_LIST_DCN42(0),		   \
+		MPC_RMCM_REG_LIST_DCN42(1)
+
+static const struct dcn42_mpc_shift mpc_shift = {
+	MPC_COMMON_MASK_SH_LIST_DCN42(__SHIFT)};
+
+static const struct dcn42_mpc_mask mpc_mask = {
+	MPC_COMMON_MASK_SH_LIST_DCN42(_MASK)};
+
+#define optc_regs_init(id) \
+	OPTC_COMMON_REG_LIST_DCN42B_RI(id)
+
+static struct dcn_optc_registers optc_regs[4];
+
+static const struct dcn_optc_shift optc_shift = {
+	OPTC_COMMON_MASK_SH_LIST_DCN42B(__SHIFT)};
+
+static const struct dcn_optc_mask optc_mask = {
+	OPTC_COMMON_MASK_SH_LIST_DCN42B(_MASK)};
+
+#define hubp_regs_init(id) \
+	HUBP_REG_LIST_DCN42_RI(id)
+
+static struct dcn_hubp2_registers hubp_regs[4];
+
+static const struct dcn_hubp2_shift hubp_shift = {
+	HUBP_MASK_SH_LIST_DCN42B(__SHIFT)};
+
+static const struct dcn_hubp2_mask hubp_mask = {
+	HUBP_MASK_SH_LIST_DCN42B(_MASK)};
+
+static struct dcn_hubbub_registers hubbub_reg;
+
+#define hubbub_reg_init() \
+	HUBBUB_REG_LIST_DCN42B(0)
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+	HUBBUB_MASK_SH_LIST_DCN42B(__SHIFT)};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+	HUBBUB_MASK_SH_LIST_DCN42B(_MASK)};
+
+static struct dccg_registers dccg_regs;
+
+#define dccg_regs_init() \
+	DCCG_REG_LIST_DCN42B_RI()
+
+static const struct dccg_shift dccg_shift = {
+	DCCG_MASK_SH_LIST_DCN42_COMMON(__SHIFT)};
+
+static const struct dccg_mask dccg_mask = {
+	DCCG_MASK_SH_LIST_DCN42_COMMON(_MASK)};
+
+static struct pg_cntl_registers pg_cntl_regs;
+
+#define pg_cntl_dcn42b_regs_init() \
+	PG_CNTL_REG_LIST_DCN42B()
+
+static const struct pg_cntl_shift pg_cntl_shift = {
+		PG_CNTL_MASK_SH_LIST_DCN42B(__SHIFT)
+};
+
+static const struct pg_cntl_mask pg_cntl_mask = {
+		PG_CNTL_MASK_SH_LIST_DCN42B(_MASK)
+};
+#define SRII2(reg_name_pre, reg_name_post, id)                                                       \
+	.reg_name_pre##_##reg_name_post[id] = \
+		BASE(reg##reg_name_pre##id##_##reg_name_post##_BASE_IDX) + \
+					reg##reg_name_pre##id##_##reg_name_post
+
+/* Not in DCN42B:
+ * SRII(PIXEL_RATE_CNTL, OTG, 3),
+ * SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 3),
+ * SR(DOMAIN19_PG_CONFIG),
+ * SR(DOMAIN19_PG_STATUS),
+ */
+#define HWSEQ_DCN42B_REG_LIST()                \
+		SR(DCHUBBUB_GLOBAL_TIMER_CNTL),           \
+		SR(DCHUBBUB_ARB_HOSTVM_CNTL), \
+		SR(DIO_MEM_PWR_CTRL),                 \
+		SR(ODM_MEM_PWR_CTRL3),                \
+		SR(MMHUBBUB_MEM_PWR_CNTL),            \
+		SR(DCCG_GATE_DISABLE_CNTL),           \
+		SR(DCCG_GATE_DISABLE_CNTL2),          \
+		SR(DCCG_GATE_DISABLE_CNTL4), \
+		SR(DCCG_GATE_DISABLE_CNTL5), \
+		SR(DCFCLK_CNTL),                      \
+		SR(DC_MEM_GLOBAL_PWR_REQ_CNTL),       \
+		SRII(PIXEL_RATE_CNTL, OTG, 0),        \
+		SRII(PIXEL_RATE_CNTL, OTG, 1),        \
+		SRII(PIXEL_RATE_CNTL, OTG, 2),        \
+		SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 0), \
+		SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 1), \
+		SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 2), \
+		SR(MICROSECOND_TIME_BASE_DIV),        \
+		SR(MILLISECOND_TIME_BASE_DIV),        \
+		SR(DISPCLK_FREQ_CHANGE_CNTL),         \
+		SR(RBBMIF_TIMEOUT_DIS),               \
+		SR(RBBMIF_TIMEOUT_DIS_2),             \
+		SR(DCHUBBUB_CRC_CTRL),                \
+		SR(DPP_TOP0_DPP_CRC_CTRL),            \
+		SR(DPP_TOP0_DPP_CRC_VAL_R),           \
+		SR(DPP_TOP0_DPP_CRC_VAL_G),           \
+		SR(DPP_TOP0_DPP_CRC_VAL_B),           \
+		SR(DPP_TOP0_DPP_CRC_VAL_A),           \
+		SR(MPC_CRC_CTRL),                     \
+		SR(MPC_CRC_RESULT_R),                 \
+		SR(MPC_CRC_RESULT_G),                 \
+		SR(MPC_CRC_RESULT_B),                 \
+		SR(MPC_CRC_RESULT_A),                 \
+		SR(DOMAIN0_PG_CONFIG),                \
+		SR(DOMAIN1_PG_CONFIG),                \
+		SR(DOMAIN2_PG_CONFIG),                \
+		SR(DOMAIN3_PG_CONFIG),                \
+		SR(DOMAIN16_PG_CONFIG),               \
+		SR(DOMAIN17_PG_CONFIG),               \
+		SR(DOMAIN18_PG_CONFIG),               \
+		SR(DOMAIN22_PG_CONFIG),               \
+		SR(DOMAIN23_PG_CONFIG),               \
+		SR(DOMAIN24_PG_CONFIG),               \
+		SR(DOMAIN25_PG_CONFIG),               \
+		SR(DOMAIN26_PG_CONFIG),               \
+		SR(DOMAIN0_PG_STATUS),                \
+		SR(DOMAIN1_PG_STATUS),                \
+		SR(DOMAIN2_PG_STATUS),                \
+		SR(DOMAIN3_PG_STATUS),                \
+		SR(DOMAIN16_PG_STATUS),               \
+		SR(DOMAIN17_PG_STATUS),               \
+		SR(DOMAIN18_PG_STATUS),               \
+		SR(DOMAIN22_PG_STATUS),               \
+		SR(DOMAIN23_PG_STATUS),               \
+		SR(DOMAIN24_PG_STATUS),               \
+		SR(DOMAIN25_PG_STATUS),               \
+		SR(DOMAIN26_PG_STATUS),               \
+		SR(DC_IP_REQUEST_CNTL),               \
+		SR(AZALIA_AUDIO_DTO),                 \
+		SR(HPO_TOP_HW_CONTROL),               \
+		SR(AZALIA_CONTROLLER_CLOCK_GATING),   \
+		SR(DMU_CLK_CNTL)
+
+static struct dce_hwseq_registers hwseq_reg;
+
+#define hwseq_reg_init() \
+	HWSEQ_DCN42B_REG_LIST()
+
+/*	Not in DCN42B
+	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN19_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN19_PG_STATUS, DOMAIN_DESIRED_PWR_STATE, mask_sh), \
+	HWS_SF(, DMU_CLK_CNTL, DPIACLK_ALLOW_DS_CLKSTOP, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL4, DPIASYMCLK0_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL4, DPIASYMCLK1_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL4, DPIASYMCLK2_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL4, DPIASYMCLK3_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL4, DPIASYMCLK4_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL4, DPIASYMCLK5_GATE_DISABLE, mask_sh), \
+ */
+#define HWSEQ_DCN42B_MASK_SH_LIST(mask_sh)                                            \
+	HWSEQ_DCN_MASK_SH_LIST(mask_sh),                                                 \
+		HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
+		HWS_SF(, DCHUBBUB_ARB_HOSTVM_CNTL, DISABLE_HOSTVM_FORCE_ALLOW_PSTATE, mask_sh), \
+		HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                  \
+		HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                     \
+		HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                  \
+		HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                     \
+		HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                  \
+		HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                     \
+		HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                  \
+		HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                     \
+		HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                 \
+		HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                    \
+		HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                 \
+		HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                    \
+		HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                 \
+		HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                    \
+		HWS_SF(, DOMAIN22_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                 \
+		HWS_SF(, DOMAIN22_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                    \
+		HWS_SF(, DOMAIN23_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                 \
+		HWS_SF(, DOMAIN23_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                    \
+		HWS_SF(, DOMAIN24_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                 \
+		HWS_SF(, DOMAIN24_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                    \
+		HWS_SF(, DOMAIN25_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                 \
+		HWS_SF(, DOMAIN25_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                    \
+		HWS_SF(, DOMAIN26_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh),                 \
+		HWS_SF(, DOMAIN26_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh),                    \
+		HWS_SF(, DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),               \
+		HWS_SF(, DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),               \
+		HWS_SF(, DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),               \
+		HWS_SF(, DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),               \
+		HWS_SF(, DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),              \
+		HWS_SF(, DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),              \
+		HWS_SF(, DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),              \
+		HWS_SF(, DOMAIN22_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),              \
+		HWS_SF(, DOMAIN23_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),              \
+		HWS_SF(, DOMAIN24_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),              \
+		HWS_SF(, DOMAIN25_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),              \
+		HWS_SF(, DOMAIN26_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh),              \
+		HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh),                        \
+		HWS_SF(, AZALIA_AUDIO_DTO, AZALIA_AUDIO_DTO_MODULE, mask_sh),                \
+		HWS_SF(, HPO_TOP_HW_CONTROL, HPO_IO_EN, mask_sh),                            \
+		HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_UNASSIGNED_PWR_MODE, mask_sh),           \
+		HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_VBLANK_PWR_MODE, mask_sh), \
+		HWS_SF(, DIO_MEM_PWR_CTRL, I2C_LIGHT_SLEEP_FORCE, mask_sh), \
+		HWS_SF(, DMU_CLK_CNTL, DISPCLK_R_DMU_GATE_DIS, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, DISPCLK_G_RBBMIF_GATE_DIS, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, RBBMIF_FGCG_REP_DIS, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, DPREFCLK_ALLOW_DS_CLKSTOP, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, DISPCLK_ALLOW_DS_CLKSTOP, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, DPPCLK_ALLOW_DS_CLKSTOP, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, DTBCLK_ALLOW_DS_CLKSTOP, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, DCFCLK_ALLOW_DS_CLKSTOP, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, LONO_FGCG_REP_DIS, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, LONO_DISPCLK_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, LONO_SOCCLK_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DMU_CLK_CNTL, LONO_DMCUBCLK_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKA_FE_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKB_FE_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKC_FE_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKD_FE_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKE_FE_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKA_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKB_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKC_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKD_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKE_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYASYMCLK_ROOT_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYBSYMCLK_ROOT_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYCSYMCLK_ROOT_GATE_DISABLE, mask_sh), \
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DTBCLK_P0_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DTBCLK_P1_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DTBCLK_P2_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DTBCLK_P3_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK0_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK1_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK2_GATE_DISABLE, mask_sh),\
+		HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK3_GATE_DISABLE, mask_sh)
+
+static const struct dce_hwseq_shift hwseq_shift = {
+	HWSEQ_DCN42B_MASK_SH_LIST(__SHIFT)};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+	HWSEQ_DCN42B_MASK_SH_LIST(_MASK)};
+
+#define vmid_regs_init(id) \
+	DCN20_VMID_REG_LIST_RI(id)
+
+static struct dcn_vmid_registers vmid_regs[16];
+
+static const struct dcn20_vmid_shift vmid_shifts = {
+	DCN20_VMID_MASK_SH_LIST(__SHIFT)};
+
+static const struct dcn20_vmid_mask vmid_masks = {
+	DCN20_VMID_MASK_SH_LIST(_MASK)};
+
+static const struct resource_caps res_cap_dcn42b = {
+	.num_timing_generator = 3,
+	.num_opp = 3,
+	.num_dpp = 4,
+	.num_video_plane = 4,
+	.num_audio = 4,
+	.num_stream_encoder = 3,
+	.num_dig_link_enc = 3,
+	// .num_usb4_dpia = 6,
+	.num_hpo_dp_stream_encoder = 3,
+	.num_hpo_dp_link_encoder = 2,
+	.num_pll = 3,
+	.num_dwb = 1,
+	.num_ddc = 0,
+	.num_vmid = 16,
+	.num_mpc_3dlut = 2,
+	.num_dsc = 3,
+	.num_rmcm = 2,
+	.num_mpc = 4,
+	.num_aux = 3,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+	.per_pixel_alpha = true,
+
+	.pixel_format_support = {
+		.argb8888 = true,
+		.nv12 = true,
+		.fp16 = true,
+		.p010 = true,
+		.ayuv = false,
+	},
+
+	.max_upscale_factor = {.argb8888 = 16000, .nv12 = 16000, .fp16 = 16000},
+
+	// 6:1 downscaling ratio: 1000/6 = 166.666
+	.max_downscale_factor = {.argb8888 = 167, .nv12 = 167, .fp16 = 167},
+
+	.min_width = 64,
+	.min_height = 64};
+
+static const struct dc_debug_options debug_defaults_drv = {
+	.disable_dmcu = true,
+	.force_abm_enable = false,
+	.clock_trace = true,
+	.disable_pplib_clock_request = false,
+	.disable_dpp_power_gate = true,
+	.disable_hubp_power_gate = true,
+	.disable_optc_power_gate = true,
+	.pipe_split_policy = MPC_SPLIT_AVOID,
+	.force_single_disp_pipe_split = false,
+	.disable_dcc = DCC_ENABLE,
+	.vsr_support = true,
+	.performance_trace = false,
+	.max_downscale_src_width = 4096, /*up to 4K*/
+	.disable_pplib_wm_range = false,
+	.scl_reset_length10 = true,
+	.sanity_checks = false,
+	.underflow_assert_delay_us = 0xFFFFFFFF,
+	.dwb_fi_phase = -1, // -1 = disable,
+	.dmub_command_table = true,
+	.pstate_enabled = true,
+	.enable_mem_low_power = {
+		.bits = {
+			.vga = false,
+			.i2c = true,
+			.dscl = true,
+			.cm = true,
+			.mpc = true,
+			.optc = true,
+			.vpg = true,
+		}},
+	.root_clock_optimization = {
+		.bits = {
+			.dpp = false,
+			.dsc = false,/*dscclk and dsc pg*/
+			.hdmistream = false,
+			.hdmichar = false,
+			.dpstream = false,
+			.symclk32_se = false,
+			.symclk32_le = false,
+			.symclk_fe = false,
+			.physymclk = false,
+			.dpiasymclk = false,
+		}
+	},
+	.seamless_boot_odm_combine = DML_FAIL_SOURCE_PIXEL_FORMAT,
+	.enable_z9_disable_interface = true, /* Allow support for the PMFW interface for disable Z9*/
+	.minimum_z8_residency_time = 1, /* Always allow when other conditions are met */
+	.support_eDP1_5 = true,
+	.use_max_lb = true,
+	.force_disable_subvp = false,
+	.exit_idle_opt_for_cursor_updates = true,
+	.using_dml2 = true,
+	.using_dml21 = true,
+	.enable_single_display_2to1_odm_policy = true,
+
+	// must match enable_single_display_2to1_odm_policy to support dynamic ODM transitions
+	.enable_double_buffered_dsc_pg_support = true,
+	.enable_dp_dig_pixel_rate_div_policy = 1,
+	.allow_sw_cursor_fallback = false,
+	.psp_disabled_wa = true,
+	.alloc_extra_way_for_cursor = true,
+	.min_prefetch_in_strobe_ns = 60000, // 60us
+	.disable_unbounded_requesting = false,
+	.dcc_meta_propagation_delay_us = 10,
+	.disable_timeout = true,
+	.min_disp_clk_khz = 50000,
+	.static_screen_wait_frames = 2,
+	.disable_z10 = false,
+	.ignore_pg = true,
+	.disable_stutter_for_wm_program = true,
+	.min_deep_sleep_dcfclk_khz = 8000,
+	.replay_skip_crtc_disabled = true,
+	.psr_skip_crtc_disable = true,
+};
+
+static const struct dc_check_config config_defaults = {
+	.enable_legacy_fast_update = false,
+};
+
+static struct dce_aux *dcn42b_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT aux_engine_regs
+	aux_engine_regs_init(0),
+		aux_engine_regs_init(1),
+		aux_engine_regs_init(2);
+		/* Not in DCN42B:
+		 * aux_engine_regs_init(3),
+		 * aux_engine_regs_init(4);
+		*/
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+								SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+								&aux_engine_regs[inst],
+								&aux_mask,
+								&aux_shift,
+								ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+
+#define i2c_inst_regs_init(id) \
+	I2C_HW_ENGINE_COMMON_REG_LIST_DCN30_RI(id)
+
+static struct dce_i2c_registers i2c_hw_regs[5];
+
+static const struct dce_i2c_shift i2c_shifts = {
+	I2C_COMMON_MASK_SH_LIST_DCN35(__SHIFT)
+};
+static const struct dce_i2c_mask i2c_masks = {
+	I2C_COMMON_MASK_SH_LIST_DCN35(_MASK)
+};
+
+/* ========================================================== */
+
+/*
+ * DPIA index | Preferred Encoder     |    Host Router
+ *   0        |      C                |       0
+ *   1        |      First Available  |       0
+ *   2        |      D                |       1
+ *   3        |      First Available  |       1
+ *   4        |      E                |       2
+ *   5        |      First Available  |       2
+ */
+/* ========================================================== */
+/* DPIA is not included in DCN42B */
+/*
+static const enum engine_id dpia_to_preferred_enc_id_table[] = {
+		ENGINE_ID_DIGC,
+		ENGINE_ID_DIGC,
+		ENGINE_ID_DIGD,
+		ENGINE_ID_DIGD,
+		ENGINE_ID_DIGE,
+		ENGINE_ID_DIGE
+};
+
+static enum engine_id dcn42b_get_preferred_eng_id_dpia(unsigned int dpia_index)
+{
+	return dpia_to_preferred_enc_id_table[dpia_index];
+}
+*/
+
+static struct dce_i2c_hw *dcn42b_i2c_hw_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT i2c_hw_regs
+	i2c_inst_regs_init(1);
+	/* Not in DCN42B:
+	 * i2c_inst_regs_init(2),
+	 * i2c_inst_regs_init(3),
+	 * i2c_inst_regs_init(4),
+	 * i2c_inst_regs_init(5);
+	 */
+	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+						  &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+
+static struct clock_source *dcn42b_clock_source_create(
+	struct dc_context *ctx,
+	struct dc_bios *bios,
+	enum clock_source_id id,
+	const struct dce110_clk_src_regs *regs,
+	bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dcn401_clk_src_construct(clk_src, ctx, bios, id,
+								 regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static struct hubbub *dcn42b_hubbub_create(struct dc_context *ctx)
+{
+	int i;
+
+	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub),
+					  GFP_KERNEL);
+
+	if (!hubbub3)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT hubbub_reg
+	hubbub_reg_init();
+
+#undef REG_STRUCT
+#define REG_STRUCT vmid_regs
+	vmid_regs_init(0),
+		vmid_regs_init(1),
+		vmid_regs_init(2),
+		vmid_regs_init(3),
+		vmid_regs_init(4),
+		vmid_regs_init(5),
+		vmid_regs_init(6),
+		vmid_regs_init(7),
+		vmid_regs_init(8),
+		vmid_regs_init(9),
+		vmid_regs_init(10),
+		vmid_regs_init(11),
+		vmid_regs_init(12),
+		vmid_regs_init(13),
+		vmid_regs_init(14),
+		vmid_regs_init(15);
+
+	hubbub42_construct(hubbub3, ctx,
+					   &hubbub_reg,
+					   &hubbub_shift,
+					   &hubbub_mask,
+					   DCN42_DEFAULT_DET_SIZE,
+					   8,
+					   DCN42_CRB_SIZE_KB);
+	for (i = 0; i < res_cap_dcn42b.num_vmid; i++) {
+		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
+
+		vmid->ctx = ctx;
+
+		vmid->regs = &vmid_regs[i];
+		vmid->shifts = &vmid_shifts;
+		vmid->masks = &vmid_masks;
+	}
+
+	return &hubbub3->base;
+}
+
+static struct hubp *dcn42b_hubp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn20_hubp *hubp2 =
+		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
+
+	if (!hubp2)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT hubp_regs
+	hubp_regs_init(0),
+		hubp_regs_init(1),
+		hubp_regs_init(2),
+		hubp_regs_init(3);
+
+	if (hubp42_construct(hubp2, ctx, inst,
+						 &hubp_regs[inst], &hubp_shift, &hubp_mask))
+		return &hubp2->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(hubp2);
+	return NULL;
+}
+static const struct dc_panel_config dcn42b_panel_config_defaults = {
+	.psr = {
+		.disable_psr = false,
+		.disallow_psrsu = true,
+		.disallow_replay = false,
+	},
+	.ilr = {
+		.optimize_edp_link_rate = true,
+	},
+};
+
+static void dcn42b_dpp_destroy(struct dpp **dpp)
+{
+	kfree(TO_DCN42_DPP(*dpp));
+	*dpp = NULL;
+}
+
+static struct dpp *dcn42b_dpp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn42_dpp *dpp42b =
+		kzalloc(sizeof(struct dcn42_dpp), GFP_KERNEL);
+
+	if (!dpp42b)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT dpp_regs
+	dpp_regs_init(0),
+		dpp_regs_init(1),
+		dpp_regs_init(2),
+		dpp_regs_init(3);
+
+	if (dpp42_construct(dpp42b, ctx, inst,
+						&dpp_regs[inst], &tf_shift, &tf_mask))
+		return &dpp42b->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(dpp42b);
+	return NULL;
+}
+
+static struct mpc *dcn42b_mpc_create(
+	struct dc_context *ctx,
+	int num_mpcc,
+	int num_rmu)
+{
+	struct dcn42_mpc *mpc42b = kzalloc(sizeof(struct dcn42_mpc),
+										GFP_KERNEL);
+
+	if (!mpc42b)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT mpc_regs
+	dcn_mpc_regs_init();
+
+	dcn42_mpc_construct(mpc42b, ctx,
+						 &mpc_regs,
+						 &mpc_shift,
+						 &mpc_mask,
+						 num_mpcc,
+						 num_rmu);
+
+	return &mpc42b->base;
+}
+
+static struct output_pixel_processor *dcn42b_opp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_opp *opp4 =
+		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
+
+	if (!opp4) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT opp_regs
+	opp_regs_init(0),
+		opp_regs_init(1),
+		opp_regs_init(2);
+		//DCN42B has 3 OPP Pipes
+		//opp_regs_init(3);
+	dcn20_opp_construct(opp4, ctx, inst,
+						&opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp4->base;
+}
+
+static struct timing_generator *dcn42b_timing_generator_create(
+	struct dc_context *ctx,
+	uint32_t instance)
+{
+	struct optc *tgn10 =
+		kzalloc(sizeof(struct optc), GFP_KERNEL);
+
+	if (!tgn10)
+		return NULL;
+#undef REG_STRUCT
+#define REG_STRUCT optc_regs
+	optc_regs_init(0),
+		optc_regs_init(1),
+		optc_regs_init(2);
+		//optc_regs_init(3);
+	tgn10->base.inst = instance;
+	tgn10->base.ctx = ctx;
+
+	tgn10->tg_regs = &optc_regs[instance];
+	tgn10->tg_shift = &optc_shift;
+	tgn10->tg_mask = &optc_mask;
+
+	dcn42_timing_generator_init(tgn10);
+
+	return &tgn10->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+	.max_hdmi_deep_color = COLOR_DEPTH_121212,
+	.max_hdmi_pixel_clock = 600000,
+	.hdmi_ycbcr420_supported = true,
+	.dp_ycbcr420_supported = true,
+	.fec_supported = true,
+	.flags.bits.IS_HBR2_CAPABLE = true,
+	.flags.bits.IS_HBR3_CAPABLE = true,
+	.flags.bits.IS_TPS3_CAPABLE = true,
+	.flags.bits.IS_TPS4_CAPABLE = true};
+
+static struct link_encoder *dcn42b_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dcn20_link_encoder *enc20 =
+		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+
+	if (!enc20 || enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs))
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_aux_regs
+	aux_regs_init(0),
+		aux_regs_init(1),
+		aux_regs_init(2);
+		/*
+		 * Not included in DCN42B:
+		 * aux_regs_init(3),
+		 * aux_regs_init(4);
+		 */
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_hpd_regs
+	hpd_regs_init(0),
+		hpd_regs_init(1),
+		hpd_regs_init(2);
+		/*
+		 * Not included in DCN42B:
+		 * hpd_regs_init(3),
+		 * hpd_regs_init(4);
+		 */
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_regs
+	link_regs_init(0, A),
+		link_regs_init(1, B),
+		link_regs_init(2, C);
+		/*
+		 * Not included in DCN42B:
+		 * link_regs_init(3, D),
+		 * link_regs_init(4, E);
+		 */
+
+	dcn42_link_encoder_construct(enc20,
+								  enc_init_data,
+								  &link_enc_feature,
+								  &link_enc_regs[enc_init_data->transmitter],
+								  &link_enc_aux_regs[enc_init_data->channel - 1],
+								  &link_enc_hpd_regs[enc_init_data->hpd_source],
+								  &le_shift,
+								  &le_mask);
+	return &enc20->enc10.base;
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	generic_reg_get(ctx, regDC_PINSTRAPS + BASE(regDC_PINSTRAPS_BASE_IDX),
+		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+}
+
+static struct audio *dcn42b_create_audio(
+	struct dc_context *ctx, unsigned int inst)
+{
+
+#undef REG_STRUCT
+#define REG_STRUCT audio_regs
+	audio_regs_init(0),
+		audio_regs_init(1),
+		audio_regs_init(2),
+		audio_regs_init(3),
+		audio_regs_init(4);
+
+	return dce_audio_create(ctx, inst,
+							&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static struct vpg *dcn42b_vpg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_vpg *vpg4 = kzalloc(sizeof(struct dcn31_vpg), GFP_KERNEL);
+
+	if (!vpg4)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT vpg_regs
+	vpg_regs_init(0),
+		vpg_regs_init(1),
+		vpg_regs_init(2),
+		vpg_regs_init(3),
+		vpg_regs_init(4),
+		vpg_regs_init(5),
+		vpg_regs_init(6),
+		vpg_regs_init(7),
+		vpg_regs_init(8),
+		vpg_regs_init(9);
+	vpg31_construct(vpg4, ctx, inst,
+					&vpg_regs[inst],
+					&vpg_shift,
+					&vpg_mask);
+
+	return &vpg4->base;
+}
+
+static struct apg *dcn42b_apg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_apg *apg31 = kzalloc(sizeof(struct dcn31_apg), GFP_KERNEL);
+
+	if (!apg31)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT apg_regs
+	apg_regs_init(0),
+	apg_regs_init(1),
+	apg_regs_init(2),
+	apg_regs_init(3),
+	apg_regs_init(4),
+	apg_regs_init(5),
+	apg_regs_init(6),
+	apg_regs_init(7),
+	apg_regs_init(8),
+	apg_regs_init(9);
+
+	apg31_construct(apg31, ctx, inst,
+					&apg_regs[inst],
+					&apg_shift,
+					&apg_mask);
+
+	return &apg31->base;
+}
+
+static struct stream_encoder *dcn42b_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn10_stream_encoder *enc1;
+	struct vpg *vpg;
+	struct apg *apg;
+
+	uint32_t vpg_inst;
+	uint32_t apg_inst;
+
+	/* Mapping of VPG, DME register blocks to DIO block instance */
+	if (eng_id <= ENGINE_ID_DIGF) {
+		vpg_inst = eng_id;
+		apg_inst = eng_id;
+	} else
+		return NULL;
+
+	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
+	vpg = dcn42b_vpg_create(ctx, vpg_inst);
+	apg = dcn42b_apg_create(ctx, apg_inst);
+
+	if (!enc1 || !vpg || !apg) {
+		kfree(enc1);
+		kfree(vpg);
+		kfree(apg);
+		return NULL;
+	}
+#undef REG_STRUCT
+#define REG_STRUCT stream_enc_regs
+	stream_enc_regs_init(0),
+		stream_enc_regs_init(1),
+		stream_enc_regs_init(2);
+		/*
+		 * stream_enc_regs_init(3),
+		 * stream_enc_regs_init(4);
+		 */
+
+	dcn42_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
+									eng_id, vpg, apg,
+									&stream_enc_regs[eng_id],
+									&se_shift, &se_mask);
+	return &enc1->base;
+}
+
+static struct hpo_dp_stream_encoder *dcn42b_hpo_dp_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_stream_encoder *hpo_dp_enc31;
+	struct vpg *vpg;
+	struct apg *apg;
+	uint32_t hpo_dp_inst;
+	uint32_t vpg_inst;
+	uint32_t apg_inst;
+
+	ASSERT((eng_id >= ENGINE_ID_HPO_DP_0) && (eng_id <= ENGINE_ID_HPO_DP_3));
+	hpo_dp_inst = eng_id - ENGINE_ID_HPO_DP_0;
+
+	/* Mapping of VPG register blocks to HPO DP block instance:
+	 * VPG[6] -> HPO_DP[0]
+	 * VPG[7] -> HPO_DP[1]
+	 * VPG[8] -> HPO_DP[2]
+	 * VPG[9] -> HPO_DP[3]
+	 */
+	//specific to DCN42B: regVPG6..regVPG9 are the four HPO DP VPGs
+	vpg_inst = hpo_dp_inst + 6;
+
+	/* Mapping of APG register blocks to HPO DP block instance:
+	 * APG[6] -> HPO_DP[0]
+	 * APG[7] -> HPO_DP[1]
+	 * APG[8] -> HPO_DP[2]
+	 * APG[9] -> HPO_DP[3]
+	 */
+	//specific to DCN42B: APG slots 6..9 correspond to HPO DP encoders 0..3
+	apg_inst = hpo_dp_inst + 6;
+
+	/* allocate HPO stream encoder and create VPG sub-block */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_stream_encoder), GFP_KERNEL);
+	vpg = dcn42b_vpg_create(ctx, vpg_inst);
+	apg = dcn42b_apg_create(ctx, apg_inst);
+
+	if (!hpo_dp_enc31 || !vpg || !apg) {
+		kfree(hpo_dp_enc31);
+		kfree(vpg);
+		kfree(apg);
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT hpo_dp_stream_enc_regs
+	hpo_dp_stream_encoder_reg_init(0),
+		hpo_dp_stream_encoder_reg_init(1),
+		hpo_dp_stream_encoder_reg_init(2);
+		//hpo_dp_stream_encoder_reg_init(3);
+
+	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
+				hpo_dp_inst, eng_id, vpg, apg,
+				&hpo_dp_stream_enc_regs[hpo_dp_inst],
+				&hpo_dp_se_shift, &hpo_dp_se_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+static struct hpo_dp_link_encoder *dcn42b_hpo_dp_link_encoder_create(
+	uint8_t inst,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_link_encoder *hpo_dp_enc31;
+
+	/* allocate HPO link encoder */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
+	if (!hpo_dp_enc31)
+		return NULL; /* out of memory */
+
+/* DCN42B has 2 HPO DP link encoders */
+#undef REG_STRUCT
+#define REG_STRUCT hpo_dp_link_enc_regs
+	hpo_dp_link_encoder_reg_init(0),
+	hpo_dp_link_encoder_reg_init(1);
+
+	hpo_dp_link_encoder42_construct(hpo_dp_enc31, ctx, inst,
+				&hpo_dp_link_enc_regs[inst],
+				&hpo_dp_le_shift, &hpo_dp_le_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+static struct dce_hwseq *dcn42b_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+#undef REG_STRUCT
+#define REG_STRUCT hwseq_reg
+	hwseq_reg_init();
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+
+	return hws;
+}
+
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = dcn42b_create_audio,
+	.create_stream_encoder = dcn42b_stream_encoder_create,
+	.create_hpo_dp_stream_encoder = dcn42b_hpo_dp_stream_encoder_create,
+	.create_hpo_dp_link_encoder = dcn42b_hpo_dp_link_encoder_create,
+	.create_hwseq = dcn42b_hwseq_create,
+};
+
+static void dcn42b_dsc_destroy(struct display_stream_compressor **dsc)
+{
+	kfree(container_of(*dsc, struct dcn401_dsc, base));
+	*dsc = NULL;
+}
+
+static void dcn42b_resource_destruct(struct dcn42b_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL) {
+			if (pool->base.stream_enc[i]->vpg != NULL) {
+				kfree(DCN31_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
+				pool->base.stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.stream_enc[i]->apg != NULL) {
+				kfree(DCN31_APG_FROM_APG(pool->base.stream_enc[i]->apg));
+				pool->base.stream_enc[i]->apg = NULL;
+			}
+			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+			pool->base.stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_stream_enc_count; i++) {
+		if (pool->base.hpo_dp_stream_enc[i] != NULL) {
+			if (pool->base.hpo_dp_stream_enc[i]->vpg != NULL) {
+				kfree(DCN31_VPG_FROM_VPG(pool->base.hpo_dp_stream_enc[i]->vpg));
+				pool->base.hpo_dp_stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.hpo_dp_stream_enc[i]->apg != NULL) {
+				kfree(DCN31_APG_FROM_APG(pool->base.hpo_dp_stream_enc[i]->apg));
+				pool->base.hpo_dp_stream_enc[i]->apg = NULL;
+			}
+			kfree(DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(pool->base.hpo_dp_stream_enc[i]));
+			pool->base.hpo_dp_stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_link_enc_count; i++) {
+		if (pool->base.hpo_dp_link_enc[i] != NULL) {
+			kfree(DCN3_1_HPO_DP_LINK_ENC_FROM_HPO_LINK_ENC(pool->base.hpo_dp_link_enc[i]));
+			pool->base.hpo_dp_link_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
+		if (pool->base.dscs[i] != NULL)
+			dcn42b_dsc_destroy(&pool->base.dscs[i]);
+	}
+
+	if (pool->base.mpc != NULL) {
+		kfree(TO_DCN20_MPC(pool->base.mpc));
+		pool->base.mpc = NULL;
+	}
+	if (pool->base.hubbub != NULL) {
+		kfree(TO_DCN20_HUBBUB(pool->base.hubbub));
+		pool->base.hubbub = NULL;
+	}
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.dpps[i] != NULL)
+			dcn42b_dpp_destroy(&pool->base.dpps[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.hubps[i] != NULL) {
+			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
+			pool->base.hubps[i] = NULL;
+		}
+
+		if (pool->base.irqs != NULL)
+			dal_irq_service_destroy(&pool->base.irqs);
+	}
+
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_aux; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+	}
+
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_opp; i++) {
+		if (pool->base.opps[i] != NULL)
+			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
+	}
+
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.timing_generators[i] != NULL) {
+			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dwb; i++) {
+		if (pool->base.dwbc[i] != NULL) {
+			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
+			pool->base.dwbc[i] = NULL;
+		}
+		if (pool->base.mcif_wb[i] != NULL) {
+			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
+			pool->base.mcif_wb[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.audio_count; i++) {
+		if (pool->base.audios[i])
+			dce_aud_destroy(&pool->base.audios[i]);
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
+			pool->base.clock_sources[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_mpc_3dlut; i++) {
+		if (pool->base.mpc_lut[i] != NULL) {
+			dc_3dlut_func_release(pool->base.mpc_lut[i]);
+			pool->base.mpc_lut[i] = NULL;
+		}
+		if (pool->base.mpc_shaper[i] != NULL) {
+			dc_transfer_func_release(pool->base.mpc_shaper[i]);
+			pool->base.mpc_shaper[i] = NULL;
+		}
+	}
+
+	if (pool->base.dp_clock_source != NULL) {
+		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
+		pool->base.dp_clock_source = NULL;
+	}
+
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.multiple_abms[i] != NULL)
+			dce_abm_destroy(&pool->base.multiple_abms[i]);
+	}
+
+	if (pool->base.psr != NULL)
+		dmub_psr_destroy(&pool->base.psr);
+
+	if (pool->base.pg_cntl != NULL)
+		dcn_pg_cntl_destroy(&pool->base.pg_cntl);
+	if (pool->base.replay != NULL)
+		dmub_replay_destroy(&pool->base.replay);
+	if (pool->base.dccg != NULL)
+		dcn_dccg_destroy(&pool->base.dccg);
+
+	if (pool->base.oem_device != NULL) {
+		struct dc *dc = pool->base.oem_device->ctx->dc;
+
+		dc->link_srv->destroy_ddc_service(&pool->base.oem_device);
+	}
+}
+
+static void dcn42b_build_pipe_pix_clk_params(struct pipe_ctx *pipe_ctx)
+{
+	const struct dc_stream_state *stream = pipe_ctx->stream;
+	struct dc_link *link = stream->link;
+	struct link_encoder *link_enc = pipe_ctx->link_res.dio_link_enc;
+	struct pixel_clk_params *pixel_clk_params = &pipe_ctx->stream_res.pix_clk_params;
+
+	pixel_clk_params->requested_pix_clk_100hz = stream->timing.pix_clk_100hz;
+
+	if (pipe_ctx->dsc_padding_params.dsc_hactive_padding != 0)
+		pixel_clk_params->requested_pix_clk_100hz = pipe_ctx->dsc_padding_params.dsc_pix_clk_100hz;
+
+	if (!pipe_ctx->stream->ctx->dc->config.unify_link_enc_assignment)
+		link_enc = link_enc_cfg_get_link_enc(link);
+	if (link_enc)
+		pixel_clk_params->encoder_object_id = link_enc->id;
+
+	pixel_clk_params->signal_type = pipe_ctx->stream->signal;
+	pixel_clk_params->controller_id = pipe_ctx->stream_res.tg->inst + 1;
+	/* TODO: un-hardcode*/
+
+	/* TODO - DP2.0 HW: calculate requested_sym_clk for UHBR rates */
+
+	pixel_clk_params->requested_sym_clk = LINK_RATE_LOW *
+		LINK_RATE_REF_FREQ_IN_KHZ;
+	pixel_clk_params->flags.ENABLE_SS = 0;
+	pixel_clk_params->color_depth =
+		stream->timing.display_color_depth;
+	pixel_clk_params->flags.DISPLAY_BLANKED = 1;
+	pixel_clk_params->pixel_encoding = stream->timing.pixel_encoding;
+
+	if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR422)
+		pixel_clk_params->color_depth = COLOR_DEPTH_888;
+
+	if (stream->timing.timing_3d_format == TIMING_3D_FORMAT_HW_FRAME_PACKING)
+		pixel_clk_params->requested_pix_clk_100hz *= 2;
+	if (dc_is_tmds_signal(stream->signal) &&
+			stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR420)
+		pixel_clk_params->requested_pix_clk_100hz /= 2;
+
+	pipe_ctx->clock_source->funcs->get_pix_clk_dividers(
+			pipe_ctx->clock_source,
+			&pipe_ctx->stream_res.pix_clk_params,
+			&pipe_ctx->pll_settings);
+
+	pixel_clk_params->dio_se_pix_per_cycle = 1;
+	if (dc_is_tmds_signal(stream->signal) &&
+			stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR420) {
+		pixel_clk_params->dio_se_pix_per_cycle = 2;
+	} else if (dc_is_dp_signal(stream->signal)) {
+		/* round up to nearest power of 2, or max at 8 pixels per cycle */
+		if (pixel_clk_params->requested_pix_clk_100hz > (uint32_t)(4 * stream->ctx->dc->clk_mgr->dprefclk_khz * 10)) {
+			pixel_clk_params->dio_se_pix_per_cycle = 8;
+		} else if (pixel_clk_params->requested_pix_clk_100hz > (uint32_t)(2 * stream->ctx->dc->clk_mgr->dprefclk_khz * 10)) {
+			pixel_clk_params->dio_se_pix_per_cycle = 4;
+		} else if (pixel_clk_params->requested_pix_clk_100hz > (uint32_t)(stream->ctx->dc->clk_mgr->dprefclk_khz * 10)) {
+			pixel_clk_params->dio_se_pix_per_cycle = 2;
+		} else {
+			pixel_clk_params->dio_se_pix_per_cycle = 1;
+		}
+	}
+}
+
+static bool dcn42b_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	unsigned int i;
+	uint32_t dwb_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < dwb_count; i++) {
+		struct dcn30_dwbc *dwbc42 = kzalloc(sizeof(struct dcn30_dwbc),
+											GFP_KERNEL);
+
+		if (!dwbc42) {
+			dm_error("DC: failed to create dwbc42!\n");
+			return false;
+		}
+
+#undef REG_STRUCT
+#define REG_STRUCT dwbc401_regs
+		dwbc_regs_dcn401_init(0);
+
+		dcn30_dwbc_construct(dwbc42, ctx,
+				&dwbc401_regs[i],
+				&dwbc401_shift,
+				&dwbc401_mask,
+				i);
+
+		pool->dwbc[i] = &dwbc42->base;
+	}
+	return true;
+}
+
+static void dcn42b_mmhubbub_init(struct dcn30_mmhubbub *mcif_wb30,
+								struct dc_context *ctx)
+{
+	dcn42_mmhubbub_set_fgcg(
+		mcif_wb30,
+		ctx->dc->debug.enable_fine_grain_clock_gating.bits.mmhubbub);
+}
+
+static bool dcn42b_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	unsigned int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
+												   GFP_KERNEL);
+
+		if (!mcif_wb30) {
+			dm_error("DC: failed to create mcif_wb30!\n");
+			return false;
+		}
+
+#undef REG_STRUCT
+#define REG_STRUCT mcif_wb35_regs
+		mcif_wb_regs_dcn3_init(0);
+
+		dcn35_mmhubbub_construct(mcif_wb30, ctx,
+					&mcif_wb35_regs[i],
+					&mcif_wb35_shift,
+					&mcif_wb35_mask,
+					i);
+
+		dcn42b_mmhubbub_init(mcif_wb30, ctx);
+
+		pool->mcif_wb[i] = &mcif_wb30->base;
+	}
+	return true;
+}
+
+static struct display_stream_compressor *dcn42b_dsc_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn401_dsc *dsc =
+		kzalloc(sizeof(struct dcn401_dsc), GFP_KERNEL);
+
+	if (!dsc) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT dsc_regs
+	dsc_regs_init(0),
+		dsc_regs_init(1),
+		dsc_regs_init(2);
+		//Not in DCN42B: dsc_regs_init(3);
+
+	dsc401_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
+	dsc401_set_fgcg(dsc, ctx->dc->debug.enable_fine_grain_clock_gating.bits.dsc);
+
+	dsc->max_image_width = 5760;
+
+	return &dsc->base;
+}
+
+static void dcn42b_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dcn42b_resource_pool *dcn42b_pool = TO_DCN42B_RES_POOL(*pool);
+
+	dcn42b_resource_destruct(dcn42b_pool);
+	kfree(dcn42b_pool);
+	*pool = NULL;
+}
+
+static struct dc_cap_funcs cap_funcs = {
+	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap};
+
+static void dcn42b_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_params)
+{
+	(void)bw_params;
+	dc_assert_fp_enabled();
+
+	if (dc->current_state && dc->current_state->bw_ctx.dml2)
+		dml2_reinit(dc, &dc->dml2_options, &dc->current_state->bw_ctx.dml2);
+}
+
+static void dcn42b_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
+{
+	DC_FP_START();
+	dcn42b_update_bw_bounding_box_fpu(dc, bw_params);
+	DC_FP_END();
+}
+
+enum dc_status dcn42b_validate_bandwidth(struct dc *dc,
+							  struct dc_state *context,
+							  enum dc_validate_mode validate_mode)
+{
+	bool out = false;
+
+	DC_FP_START();
+
+	out = dml2_validate(dc, context, context->bw_ctx.dml2,
+						validate_mode);
+
+	if (validate_mode == DC_VALIDATE_MODE_AND_PROGRAMMING) {
+		/*not required for mode enumeration*/
+		dcn42_decide_zstate_support(dc, context);
+	}
+
+	DC_FP_END();
+
+	return out ? DC_OK : DC_FAIL_BANDWIDTH_VALIDATE;
+}
+void dcn42b_prepare_mcache_programming(struct dc *dc,
+									  struct dc_state *context)
+{
+	if (dc->debug.using_dml21) {
+		DC_FP_START();
+		dml2_prepare_mcache_programming(dc, context,
+			context->power_source == DC_POWER_SOURCE_DC ?
+			context->bw_ctx.dml2_dc_power_source : context->bw_ctx.dml2);
+		DC_FP_END();
+	}
+}
+/* Create a minimal link encoder object not associated with a particular
+ * physical connector.
+ * resource_funcs.link_enc_create_minimal
+ */
+static struct link_encoder *dcn42b_link_enc_create_minimal(
+		struct dc_context *ctx, enum engine_id eng_id)
+{
+	struct dcn20_link_encoder *enc20;
+
+	if ((unsigned int)(eng_id - ENGINE_ID_DIGA) >= ctx->dc->res_pool->res_cap->num_dig_link_enc)
+		return NULL;
+
+	enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+	if (!enc20)
+		return NULL;
+
+	dcn31_link_encoder_construct_minimal(
+			enc20,
+			ctx,
+			&link_enc_feature,
+			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
+			&le_shift,
+			&le_mask,
+			eng_id);
+
+	return &enc20->enc10.base;
+}
+static void dcn42b_get_panel_config_defaults(struct dc_panel_config *panel_config)
+{
+	*panel_config = dcn42b_panel_config_defaults;
+}
+static unsigned int dcn42b_get_max_hw_cursor_size(const struct dc *dc,
+			struct dc_state *state,
+			const struct dc_stream_state *stream)
+{
+	(void)state;
+	(void)stream;
+
+	return dc->caps.max_cursor_size;
+}
+
+static struct resource_funcs dcn42b_res_pool_funcs = {
+	.destroy = dcn42b_destroy_resource_pool,
+	.link_enc_create = dcn42b_link_encoder_create,
+	.link_enc_create_minimal = dcn42b_link_enc_create_minimal,
+	.link_encs_assign = link_enc_cfg_link_encs_assign,
+	.link_enc_unassign = link_enc_cfg_link_enc_unassign,
+	.panel_cntl_create = dcn32_panel_cntl_create,
+	.validate_bandwidth = dcn42b_validate_bandwidth,
+	.calculate_wm_and_dlg = NULL,
+	.populate_dml_pipes = NULL,
+	.acquire_free_pipe_as_secondary_dpp_pipe = dcn32_acquire_free_pipe_as_secondary_dpp_pipe,
+	.acquire_free_pipe_as_secondary_opp_head = dcn32_acquire_free_pipe_as_secondary_opp_head,
+	.release_pipe = dcn20_release_pipe,
+	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
+	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
+	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
+	.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
+	.set_mcif_arb_params = dcn30_set_mcif_arb_params,
+	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
+	.acquire_post_bldn_3dlut = dcn32_acquire_post_bldn_3dlut,
+	.release_post_bldn_3dlut = dcn32_release_post_bldn_3dlut,
+	.update_bw_bounding_box = dcn42b_update_bw_bounding_box,
+	.patch_unknown_plane_state = dcn35_patch_unknown_plane_state,
+	.get_panel_config_defaults = dcn42b_get_panel_config_defaults,
+	//.get_preferred_eng_id_dpia = dcn42b_get_preferred_eng_id_dpia,
+	.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
+	.add_phantom_pipes = dcn32_add_phantom_pipes,
+	.calculate_mall_ways_from_bytes = dcn32_calculate_mall_ways_from_bytes,
+	.prepare_mcache_programming = dcn42b_prepare_mcache_programming,
+	.build_pipe_pix_clk_params = dcn42b_build_pipe_pix_clk_params,
+	.get_power_profile = dcn401_get_power_profile,
+	.get_vstartup_for_pipe = dcn401_get_vstartup_for_pipe,
+	.get_default_tiling_info = dcn401_get_default_tiling_info,
+	.get_max_hw_cursor_size = dcn42b_get_max_hw_cursor_size,
+};
+
+static bool dcn42b_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dcn42b_resource_pool *pool)
+{
+	int i;
+	struct dc_context *ctx = dc->ctx;
+	struct irq_service_init_data init_data;
+	uint32_t num_pipes = 4;
+
+#undef REG_STRUCT
+#define REG_STRUCT bios_regs
+	bios_regs_init();
+
+#undef REG_STRUCT
+#define REG_STRUCT clk_src_regs
+	clk_src_regs_init(0, A),
+	clk_src_regs_init(1, B),
+	clk_src_regs_init(2, C),
+	clk_src_regs_init(3, D),
+	clk_src_regs_init(4, E);
+
+#undef REG_STRUCT
+#define REG_STRUCT abm_regs
+	abm_regs_init(0),
+		abm_regs_init(1),
+		abm_regs_init(2);
+		//abm_regs_init(3);
+#undef REG_STRUCT
+#define REG_STRUCT dccg_regs
+	dccg_regs_init();
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = &res_cap_dcn42b;
+
+	/* max number of pipes for ASIC before checking for pipe fuses */
+	num_pipes = pool->base.res_cap->num_dpp;
+	pool->base.funcs = &dcn42b_res_pool_funcs;
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = (unsigned int)NO_UNDERLAY_PIPE;
+	pool->base.timing_generator_count = pool->base.res_cap->num_timing_generator;
+	pool->base.pipe_count = num_pipes;
+	pool->base.mpcc_count = num_pipes;
+	dc->caps.ips_v2_support = true;
+	dc->caps.max_downscale_ratio = 600;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a applied by default*/
+	/* TODO: Bring max cursor size back to 256 after subvp cursor corruption is fixed*/
+	dc->caps.max_cursor_size = 64;
+	dc->caps.max_buffered_cursor_size = 64;
+	dc->caps.cursor_not_scaled = true;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dmdata_alloc_size = 2048;
+	dc->caps.mall_size_per_mem_channel = 4;
+	/* total size = mall per channel * num channels * 1024 * 1024 */
+	dc->caps.mall_size_total = dc->caps.mall_size_per_mem_channel *
+		dc->ctx->dc_bios->vram_info.num_chans * 1048576;
+	dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8;
+	dc->caps.cache_line_size = 64;
+	dc->caps.cache_num_ways = 16;
+
+	/* Calculate the available MALL space */
+	dc->caps.max_cab_allocation_bytes =
+		dcn32_calc_num_avail_chans_for_mall(dc, dc->ctx->dc_bios->vram_info.num_chans) *
+				dc->caps.mall_size_per_mem_channel * 1024 * 1024;
+	dc->caps.mall_size_total = dc->caps.max_cab_allocation_bytes;
+
+	dc->caps.subvp_fw_processing_delay_us = 15;
+	dc->caps.subvp_drr_max_vblank_margin_us = 40;
+	dc->caps.subvp_prefetch_end_to_mall_start_us = 15;
+	dc->caps.subvp_swath_height_margin_lines = 16;
+	dc->caps.subvp_pstate_allow_width_us = 20;
+	dc->caps.subvp_vertical_int_margin_us = 30;
+	dc->caps.subvp_drr_vblank_start_margin_us = 100; // 100us margin
+
+	dc->caps.max_slave_planes = 2;
+	dc->caps.max_slave_yuv_planes = 2;
+	dc->caps.max_slave_rgb_planes = 2;
+	dc->caps.post_blend_color_processing = true;
+	dc->caps.force_dp_tps4_for_cp2520 = true;
+	if (dc->config.forceHBR2CP2520)
+		dc->caps.force_dp_tps4_for_cp2520 = false;
+	dc->caps.dp_hdmi21_pcon_support = true;
+	dc->caps.dp_hpo = true;
+	dc->caps.dp_hdmi21_pcon_support = true;
+	dc->caps.edp_dsc_support = true;
+	dc->caps.extended_aux_timeout_support = true;
+	dc->caps.dmcub_support = true;
+	dc->caps.is_apu = true;
+	dc->caps.seamless_odm = true;
+	dc->caps.zstate_support = true;
+	dc->caps.ips_support = true;
+	dc->caps.max_v_total = (1 << 15) - 1;
+	dc->caps.vtotal_limited_by_fp2 = true;
+
+	/* Color pipeline capabilities */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->caps.color.dpp.input_lut_shared = 0;
+	dc->caps.color.dpp.icsc = 1;
+	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
+	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
+	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
+	dc->caps.color.dpp.post_csc = 1;
+	dc->caps.color.dpp.gamma_corr = 1;
+	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
+
+	dc->caps.color.dpp.hw_3d_lut = 0;
+	dc->caps.color.dpp.ogam_ram = 0;
+	// no OGAM ROM on DCN2 and later ASICs
+	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
+	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
+	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.ocsc = 0;
+
+	dc->caps.color.mpc.gamut_remap = 1;
+	//configurable to be before or after BLND in MPCC
+	dc->caps.color.mpc.num_3dluts = (uint16_t)pool->base.res_cap->num_mpc_3dlut;
+	dc->caps.color.mpc.num_rmcm_3dluts = 2;
+	dc->caps.color.mpc.ogam_ram = 1;
+	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
+	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
+	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
+	dc->caps.color.mpc.ocsc = 1;
+	dc->caps.color.mpc.preblend = true;
+	dc->caps.color.mpc.mcm_3d_lut_caps.dma_3d_lut = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.lut_dim_caps.dim_9 = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.lut_dim_caps.dim_17 = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.mem_layout_support.linear_1d = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.mem_layout_support.swizzle_3d_bgr = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.mem_layout_support.swizzle_3d_rgb = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.mem_format_support.unorm_12msb = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.mem_format_support.unorm_12lsb = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.mem_format_support.float_fp1_5_10 = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.mem_pixel_order_support.order_rgba = 1;
+	dc->caps.color.mpc.mcm_3d_lut_caps.mem_pixel_order_support.order_bgra = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.dma_3d_lut = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_17 = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_33 = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.mem_layout_support.linear_1d = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.mem_layout_support.swizzle_3d_bgr = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.mem_layout_support.swizzle_3d_rgb = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.mem_format_support.unorm_12msb = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.mem_format_support.unorm_12lsb = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.mem_format_support.float_fp1_5_10 = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.mem_pixel_order_support.order_rgba = 1;
+	dc->caps.color.mpc.rmcm_3d_lut_caps.mem_pixel_order_support.order_bgra = 1;
+
+	dc->caps.num_of_host_routers = 0;
+	dc->caps.num_of_dpias_per_host_router = 0;
+
+	/* max_disp_clock_khz_at_vmin is slightly lower than the STA value in order
+	 * to provide some margin.
+	 * It's expected for furture ASIC to have equal or higher value, in order to
+	 * have determinstic power improvement from generate to genration.
+	 * (i.e., we should not expect new ASIC generation with lower vmin rate)
+	 */
+	dc->caps.max_disp_clock_khz_at_vmin = 650000;
+	dc->config.use_spl = true;
+	dc->config.prefer_easf = true;
+
+	dc->config.dcn_sharpness_range.sdr_rgb_min = 0;
+	dc->config.dcn_sharpness_range.sdr_rgb_max = 1750;
+	dc->config.dcn_sharpness_range.sdr_rgb_mid = 750;
+	dc->config.dcn_sharpness_range.sdr_yuv_min = 0;
+	dc->config.dcn_sharpness_range.sdr_yuv_max = 3500;
+	dc->config.dcn_sharpness_range.sdr_yuv_mid = 1500;
+	dc->config.dcn_sharpness_range.hdr_rgb_min = 0;
+	dc->config.dcn_sharpness_range.hdr_rgb_max = 2750;
+	dc->config.dcn_sharpness_range.hdr_rgb_mid = 1500;
+
+	dc->config.dcn_override_sharpness_range.sdr_rgb_min = 0;
+	dc->config.dcn_override_sharpness_range.sdr_rgb_max = 3250;
+	dc->config.dcn_override_sharpness_range.sdr_rgb_mid = 1250;
+	dc->config.dcn_override_sharpness_range.sdr_yuv_min = 0;
+	dc->config.dcn_override_sharpness_range.sdr_yuv_max = 3500;
+	dc->config.dcn_override_sharpness_range.sdr_yuv_mid = 1500;
+	dc->config.dcn_override_sharpness_range.hdr_rgb_min = 0;
+	dc->config.dcn_override_sharpness_range.hdr_rgb_max = 2750;
+	dc->config.dcn_override_sharpness_range.hdr_rgb_mid = 1500;
+
+	dc->config.use_pipe_ctx_sync_logic = true;
+	dc->config.dc_mode_clk_limit_support = false;
+	dc->config.enable_windowed_mpo_odm = true;
+	dc->config.set_pipe_unlock_order = true; /* Need to ensure DET gets freed before allocating */
+	/* Use psp mailbox to enable assr */
+	dc->config.use_assr_psp_message = true;
+	/* dcn42 and afterward always support external panel replay */
+	dc->config.frame_update_cmd_version2 = true;
+
+	/* read VBIOS LTTPR caps */
+	{
+		if (ctx->dc_bios->funcs->get_lttpr_caps) {
+			enum bp_result bp_query_result;
+			uint8_t is_vbios_lttpr_enable = 0;
+
+			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
+			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
+		}
+
+		dc->caps.vbios_lttpr_aware = true;
+	}
+	dc->check_config = config_defaults;
+
+	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
+		dc->debug = debug_defaults_drv;
+	/*HW default is to have all the FGCG enabled, SW no need to program them*/
+	dc->debug.enable_fine_grain_clock_gating.u32All = 0xFFFF;
+	// Init the vm_helper
+	if (dc->vm_helper)
+		vm_helper_init(dc->vm_helper, 16);
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	/* Clock Sources for Pixel Clock*/
+	pool->base.clock_sources[DCN401_CLK_SRC_PLL0] =
+		dcn42b_clock_source_create(ctx, ctx->dc_bios,
+								  CLOCK_SOURCE_COMBO_PHY_PLL0,
+								  &clk_src_regs[0], false);
+	pool->base.clock_sources[DCN401_CLK_SRC_PLL1] =
+		dcn42b_clock_source_create(ctx, ctx->dc_bios,
+								  CLOCK_SOURCE_COMBO_PHY_PLL1,
+								  &clk_src_regs[1], false);
+	pool->base.clock_sources[DCN401_CLK_SRC_PLL2] =
+		dcn42b_clock_source_create(ctx, ctx->dc_bios,
+								  CLOCK_SOURCE_COMBO_PHY_PLL2,
+								  &clk_src_regs[2], false);
+
+	pool->base.clk_src_count = DCN401_CLK_SRC_TOTAL;
+
+	/* todo: not reuse phy_pll registers */
+	pool->base.dp_clock_source =
+		dcn42b_clock_source_create(ctx, ctx->dc_bios,
+								  CLOCK_SOURCE_ID_DP_DTO,
+								  &clk_src_regs[0], true);
+
+	for (i = 0; i < (int)pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* DCCG */
+	pool->base.dccg = dccg42_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
+	if (pool->base.dccg == NULL) {
+		dm_error("DC: failed to create dccg!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT pg_cntl_regs
+	pg_cntl_dcn42b_regs_init();
+
+	pool->base.pg_cntl = pg_cntl42_create(ctx, &pg_cntl_regs, &pg_cntl_shift, &pg_cntl_mask);
+	if (pool->base.pg_cntl == NULL) {
+		dm_error("DC: failed to create power gate control!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+	/* IRQ Service */
+	init_data.ctx = dc->ctx;
+	pool->base.irqs = dal_irq_service_dcn42_create(&init_data);
+	if (!pool->base.irqs)
+		goto create_fail;
+
+	/* HUBBUB */
+	pool->base.hubbub = dcn42b_hubbub_create(ctx);
+	if (pool->base.hubbub == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create hubbub!\n");
+		goto create_fail;
+	}
+
+	/* HUBPs, DPPs, OPPs, TGs, ABMs */
+	/* HUBPs */
+	for (i = 0; i < pool->base.res_cap->num_dpp; i++) {
+		pool->base.hubps[i] = dcn42b_hubp_create(ctx, i);
+		if (pool->base.hubps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create hubps!\n");
+			goto create_fail;
+		}
+	}
+
+	/* DPPs */
+	for (i = 0; i < pool->base.res_cap->num_dpp; i++) {
+		pool->base.dpps[i] = dcn42b_dpp_create(ctx, i);
+		if (pool->base.dpps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create dpps!\n");
+			goto create_fail;
+		}
+	}
+
+	/* OPPs */
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		pool->base.opps[i] = dcn42b_opp_create(ctx, i);
+		if (pool->base.opps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create output pixel processor!\n");
+			goto create_fail;
+		}
+	}
+
+	/* TGs */
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		pool->base.timing_generators[i] = dcn42b_timing_generator_create(ctx, i);
+		if (pool->base.timing_generators[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto create_fail;
+		}
+	}
+
+	/* ABMs */
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		pool->base.multiple_abms[i] = dmub_abm_create(ctx,
+													  &abm_regs[i],
+													  &abm_shift,
+													  &abm_mask);
+		if (pool->base.multiple_abms[i] == NULL) {
+			dm_error("DC: failed to create abm for pipe %d!\n", i);
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* PSR */
+	pool->base.psr = dmub_psr_create(ctx);
+	if (pool->base.psr == NULL) {
+		dm_error("DC: failed to create psr obj!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* Replay */
+	pool->base.replay = dmub_replay_create(ctx);
+	if (pool->base.replay == NULL) {
+		dm_error("DC: failed to create replay obj!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* Auto Test */
+
+	/* MPCCs */
+	pool->base.mpc = dcn42b_mpc_create(ctx, pool->base.mpcc_count,
+			pool->base.res_cap->num_mpc_3dlut);
+	if (pool->base.mpc == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mpc!\n");
+		goto create_fail;
+	}
+
+	/* DSCs */
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		pool->base.dscs[i] = dcn42b_dsc_create(ctx, i);
+		if (pool->base.dscs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			goto create_fail;
+		}
+	}
+
+	/* DWB */
+	if (!dcn42b_dwbc_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create dwbc!\n");
+		goto create_fail;
+	}
+
+	/* MMHUBBUB */
+	if (!dcn42b_mmhubbub_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mcif_wb!\n");
+		goto create_fail;
+	}
+
+	/* AUX */
+	for (i = 0; i < pool->base.res_cap->num_aux; i++) {
+		pool->base.engines[i] = dcn42b_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto create_fail;
+		}
+	}
+
+	/* I2C */
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.hw_i2cs[i] = dcn42b_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create hw i2c!!\n");
+			goto create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+	/* DCN4.2B has 0 DPIA */
+	/*pool->base.usb4_dpia_count = dc->caps.num_of_host_routers * dc->caps.num_of_dpias_per_host_router;
+	if (dc->debug.dpia_debug.bits.disable_dpia)*/
+	pool->base.usb4_dpia_count = 0;
+
+	/* Audio, HWSeq, Stream Encoders including HPO and virtual, MPC 3D LUTs */
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+							&res_create_funcs))
+		goto create_fail;
+
+	/* HW Sequencer init functions and Plane caps */
+	dcn42b_hw_sequencer_init_functions(dc);
+
+	dc->caps.max_planes = pool->base.pipe_count;
+	dc->config.dp_connector_no_native_i2c = true;
+
+	for (i = 0; i < (int)dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->caps.max_odm_combine_factor = 4;
+
+	dc->cap_funcs = cap_funcs;
+	dc->dcn_ip->max_num_dpp = pool->base.pipe_count;
+
+	// For now enable SDPIF_REQUEST_RATE_LIMIT on DCN4_01 when vram_info.num_chans provided
+	if (dc->config.sdpif_request_limit_words_per_umc == 0)
+		dc->config.sdpif_request_limit_words_per_umc = 16;
+
+	dc->dml2_options.dcn_pipe_count = pool->base.pipe_count;
+	 /*this will use real soc clock table*/
+	dc->dml2_options.use_native_soc_bb_construction = true;
+	dc->dml2_options.minimize_dispclk_using_odm = false;
+	if (dc->config.EnableMinDispClkODM)
+		dc->dml2_options.minimize_dispclk_using_odm = true;
+	dc->dml2_options.enable_windowed_mpo_odm = dc->config.enable_windowed_mpo_odm;
+	dc->dml2_options.map_dc_pipes_with_callbacks = true;
+	dc->dml2_options.force_tdlut_enable = true;
+
+	resource_init_common_dml2_callbacks(dc, &dc->dml2_options);
+	dc->dml2_options.callbacks.can_support_mclk_switch_using_fw_based_vblank_stretch =
+			&dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch;
+	dc->dml2_options.svp_pstate.callbacks.release_dsc = &dcn20_release_dsc;
+	dc->dml2_options.svp_pstate.callbacks.calculate_mall_ways_from_bytes =
+		pool->base.funcs->calculate_mall_ways_from_bytes;
+
+	dc->dml2_options.svp_pstate.subvp_fw_processing_delay_us = dc->caps.subvp_fw_processing_delay_us;
+	dc->dml2_options.svp_pstate.subvp_prefetch_end_to_mall_start_us = dc->caps.subvp_prefetch_end_to_mall_start_us;
+	dc->dml2_options.svp_pstate.subvp_pstate_allow_width_us = dc->caps.subvp_pstate_allow_width_us;
+	dc->dml2_options.svp_pstate.subvp_swath_height_margin_lines = dc->caps.subvp_swath_height_margin_lines;
+
+	dc->dml2_options.svp_pstate.force_disable_subvp = dc->debug.force_disable_subvp;
+	dc->dml2_options.svp_pstate.force_enable_subvp = dc->debug.force_subvp_mclk_switch;
+
+	dc->dml2_options.mall_cfg.cache_line_size_bytes = dc->caps.cache_line_size;
+	dc->dml2_options.mall_cfg.cache_num_ways = dc->caps.cache_num_ways;
+	dc->dml2_options.mall_cfg.max_cab_allocation_bytes =
+				dc->caps.max_cab_allocation_bytes;
+	dc->dml2_options.mall_cfg.mblk_height_4bpe_pixels = DCN3_2_MBLK_HEIGHT_4BPE;
+	dc->dml2_options.mall_cfg.mblk_height_8bpe_pixels = DCN3_2_MBLK_HEIGHT_8BPE;
+	dc->dml2_options.mall_cfg.mblk_size_bytes = DCN3_2_MALL_MBLK_SIZE_BYTES;
+	dc->dml2_options.mall_cfg.mblk_width_pixels = DCN3_2_MBLK_WIDTH;
+
+	dc->dml2_options.max_segments_per_hubp = 24;
+	dc->dml2_options.det_segment_size = DCN42_CRB_SEGMENT_SIZE_KB;
+	dc->dml2_options.gpuvm_enable = true;
+	dc->dml2_options.hostvm_enable = true;
+
+	/* SPL */
+	dc->caps.scl_caps.sharpener_support = true;
+
+	return true;
+
+create_fail:
+
+	dcn42b_resource_destruct(pool);
+
+	return false;
+}
+struct resource_pool *dcn42b_create_resource_pool(
+	const struct dc_init_data *init_data,
+	struct dc *dc)
+{
+	struct dcn42b_resource_pool *pool =
+		kzalloc(sizeof(struct dcn42b_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dcn42b_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
+		return &pool->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(pool);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.h
index 2da3e3c8304a9..484ba1455468e 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.h
@@ -254,46 +254,6 @@
 		SRI_ARR(STREAM_MAPPER_CONTROL, DIG, id),\
 		SRI_ARR(DIG_FE_AUDIO_CNTL, DIG, id)
 
-/* HPO DP stream encoder */
-/* Not in DCN42B:
- * SRI_ARR(DP_SYM32_ENC_VID_CRC_CONTROL, DP_SYM32_ENC, id),
- * SRI_ARR(DP_SYM32_ENC_HBLANK_CONTROL, DP_SYM32_ENC, id),
- */
-#define DCN42B_HPO_DP_STREAM_ENC_REG_LIST_RI(id)                                             \
-	SR_ARR(DP_STREAM_MAPPER_CONTROL0, id),                                                  \
-		SR_ARR(DP_STREAM_MAPPER_CONTROL1, id),                                              \
-		SR_ARR(DP_STREAM_MAPPER_CONTROL2, id),                                              \
-		SR_ARR(DP_STREAM_MAPPER_CONTROL3, id),                                              \
-		SRI_ARR(DP_STREAM_ENC_CLOCK_CONTROL, DP_STREAM_ENC, id),                            \
-		SRI_ARR(DP_STREAM_ENC_INPUT_MUX_CONTROL, DP_STREAM_ENC, id),                        \
-		SRI_ARR(DP_STREAM_ENC_AUDIO_CONTROL, DP_STREAM_ENC, id),                            \
-		SRI_ARR(DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0, DP_STREAM_ENC, id), \
-		SRI_ARR(DP_SYM32_ENC_CONTROL, DP_SYM32_ENC, id),                                    \
-		SRI_ARR(DP_SYM32_ENC_VID_PIXEL_FORMAT, DP_SYM32_ENC, id),                           \
-		SRI_ARR(DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL, DP_SYM32_ENC, id),     \
-		SRI_ARR(DP_SYM32_ENC_VID_MSA0, DP_SYM32_ENC, id),                                   \
-		SRI_ARR(DP_SYM32_ENC_VID_MSA1, DP_SYM32_ENC, id),                                   \
-		SRI_ARR(DP_SYM32_ENC_VID_MSA2, DP_SYM32_ENC, id),                                   \
-		SRI_ARR(DP_SYM32_ENC_VID_MSA3, DP_SYM32_ENC, id),                                   \
-		SRI_ARR(DP_SYM32_ENC_VID_MSA4, DP_SYM32_ENC, id),                                   \
-		SRI_ARR(DP_SYM32_ENC_VID_MSA5, DP_SYM32_ENC, id),                                   \
-		SRI_ARR(DP_SYM32_ENC_VID_MSA6, DP_SYM32_ENC, id),                                   \
-		SRI_ARR(DP_SYM32_ENC_VID_MSA7, DP_SYM32_ENC, id),                                   \
-		SRI_ARR(DP_SYM32_ENC_VID_MSA8, DP_SYM32_ENC, id),                                   \
-		SRI_ARR(DP_SYM32_ENC_VID_MSA_CONTROL, DP_SYM32_ENC, id),                            \
-		SRI_ARR(DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL, DP_SYM32_ENC, id),              \
-		SRI_ARR(DP_SYM32_ENC_VID_FIFO_CONTROL, DP_SYM32_ENC, id),                           \
-		SRI_ARR(DP_SYM32_ENC_VID_STREAM_CONTROL, DP_SYM32_ENC, id),                         \
-		SRI_ARR(DP_SYM32_ENC_VID_VBID_CONTROL, DP_SYM32_ENC, id),                           \
-		SRI_ARR(DP_SYM32_ENC_SDP_CONTROL, DP_SYM32_ENC, id),                                \
-		SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL0, DP_SYM32_ENC, id),                           \
-		SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL2, DP_SYM32_ENC, id),                           \
-		SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL3, DP_SYM32_ENC, id),                           \
-		SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL5, DP_SYM32_ENC, id),                           \
-		SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL11, DP_SYM32_ENC, id),                          \
-		SRI_ARR(DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL, DP_SYM32_ENC, id),                \
-		SRI_ARR(DP_SYM32_ENC_SDP_AUDIO_CONTROL0, DP_SYM32_ENC, id)
-
 /*HPO DP link encoder regs */
 #define DCN42B_HPO_DP_LINK_ENC_REG_LIST_RI(id)                    \
 	SRI_ARR(DP_LINK_ENC_CLOCK_CONTROL, DP_LINK_ENC, id),         \
@@ -344,7 +304,6 @@
  *  DCCG_SRII(PHASE, DP_DTO, 3),
  *  DCCG_SRII(MODULO, DP_DTO, 3),
  *	SR(DSCCLK3_DTO_PARAM),
- *	SR(HDMISTREAMCLK_CNTL),
  *  SR(SYMCLKD_CLOCK_ENABLE),
  *  SR(SYMCLKE_CLOCK_ENABLE)
  */
@@ -360,6 +319,7 @@
 	SR(PHYBSYMCLK_CLOCK_CNTL), \
 	SR(PHYCSYMCLK_CLOCK_CNTL), \
 	SR(DPSTREAMCLK_CNTL), \
+	SR(HDMISTREAMCLK_CNTL), \
 	SR(SYMCLK32_SE_CNTL), \
 	SR(SYMCLK32_LE_CNTL), \
 	DCCG_SRII(PIXEL_RATE_CNTL, OTG, 0), \
@@ -518,7 +478,8 @@
 		SRII_ARR_2(MODULO, DP_DTO, 2, index),                   \
 		SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 0, index),             \
 		SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 1, index),             \
-		SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 2, index)
+		SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 2, index),				\
+		SR_ARR(OTG_PIXEL_RATE_DIV, index)
 
 /* ABM */
 #define ABM_DCN42B_REG_LIST_RI(id)                               \
@@ -542,120 +503,6 @@
 		SRI_ARR(DC_ABM1_ACE_OFFSET_SLOPE_DATA, ABM, id),        \
 		SRI_ARR(DC_ABM1_ACE_PWL_CNTL, ABM, id)
 
-/* HUBP */
-/* Not in DCN42B: HUBPREQ_DEBUG_DB and HUBPREQ_DEBUG */
-#define HUBP_REG_LIST_DCN42B_RI(id)                                         \
-	SRI_ARR(DCN_DMDATA_VM_CNTL, HUBPREQ, id),                               \
-	SRI_ARR(FLIP_PARAMETERS_3, HUBPREQ, id),                               \
-	SRI_ARR(FLIP_PARAMETERS_4, HUBPREQ, id),                               \
-	SRI_ARR(FLIP_PARAMETERS_5, HUBPREQ, id),                               \
-	SRI_ARR(FLIP_PARAMETERS_6, HUBPREQ, id),                               \
-	SRI_ARR(VBLANK_PARAMETERS_5, HUBPREQ, id),                             \
-	SRI_ARR(VBLANK_PARAMETERS_6, HUBPREQ, id),                             \
-	HUBP_REG_LIST_DCN_VM_RI(id),                                            \
-	SRI_ARR(PREFETCH_SETTINGS, HUBPREQ, id),                               \
-	SRI_ARR(PREFETCH_SETTINGS_C, HUBPREQ, id),                             \
-	SRI_ARR(DCN_VM_SYSTEM_APERTURE_LOW_ADDR, HUBPREQ, id),                 \
-	SRI_ARR(DCN_VM_SYSTEM_APERTURE_HIGH_ADDR, HUBPREQ, id),                \
-	SRI_ARR(CURSOR_SETTINGS, HUBPREQ, id),                                 \
-	SRI_ARR(CURSOR_SURFACE_ADDRESS_HIGH, CURSOR0_, id),                    \
-	SRI_ARR(CURSOR_SURFACE_ADDRESS, CURSOR0_, id),                         \
-	SRI_ARR(CURSOR_SIZE, CURSOR0_, id),                                    \
-	SRI_ARR(CURSOR_CONTROL, CURSOR0_, id),                                 \
-	SRI_ARR(CURSOR_POSITION, CURSOR0_, id),                                \
-	SRI_ARR(CURSOR_HOT_SPOT, CURSOR0_, id),                                \
-	SRI_ARR(CURSOR_DST_OFFSET, CURSOR0_, id),                              \
-	SRI_ARR(DMDATA_ADDRESS_HIGH, CURSOR0_, id),                            \
-	SRI_ARR(DMDATA_ADDRESS_LOW, CURSOR0_, id),                             \
-	SRI_ARR(DMDATA_CNTL, CURSOR0_, id),                                    \
-	SRI_ARR(DMDATA_SW_CNTL, CURSOR0_, id),                                 \
-	SRI_ARR(DMDATA_QOS_CNTL, CURSOR0_, id),                                \
-	SRI_ARR(DMDATA_SW_DATA, CURSOR0_, id),                                 \
-	SRI_ARR(DMDATA_STATUS, CURSOR0_, id),                                  \
-	SRI_ARR(FLIP_PARAMETERS_0, HUBPREQ, id),                               \
-	SRI_ARR(FLIP_PARAMETERS_1, HUBPREQ, id),                               \
-	SRI_ARR(FLIP_PARAMETERS_2, HUBPREQ, id),                               \
-	SRI_ARR(DCN_CUR1_TTU_CNTL0, HUBPREQ, id),                              \
-	SRI_ARR(DCN_CUR1_TTU_CNTL1, HUBPREQ, id),                              \
-	SRI_ARR(DCSURF_FLIP_CONTROL2, HUBPREQ, id),                            \
-	SRI_ARR(VMID_SETTINGS_0, HUBPREQ, id),                                 \
-	SRI_ARR(DCHUBP_CNTL, HUBP, id),                                        \
-	SRI_ARR(DCSURF_ADDR_CONFIG, HUBP, id),                                 \
-	SRI_ARR(DCSURF_TILING_CONFIG, HUBP, id),                               \
-	SRI_ARR(DCSURF_SURFACE_PITCH, HUBPREQ, id),                            \
-	SRI_ARR(DCSURF_SURFACE_PITCH_C, HUBPREQ, id),                          \
-	SRI_ARR(DCSURF_SURFACE_CONFIG, HUBP, id),                              \
-	SRI_ARR(DCSURF_FLIP_CONTROL, HUBPREQ, id),                             \
-	SRI_ARR(DCSURF_PRI_VIEWPORT_DIMENSION, HUBP, id),                      \
-	SRI_ARR(DCSURF_PRI_VIEWPORT_START, HUBP, id),                          \
-	SRI_ARR(DCSURF_SEC_VIEWPORT_DIMENSION, HUBP, id),                      \
-	SRI_ARR(DCSURF_SEC_VIEWPORT_START, HUBP, id),                          \
-	SRI_ARR(DCSURF_PRI_VIEWPORT_DIMENSION_C, HUBP, id),                    \
-	SRI_ARR(DCSURF_PRI_VIEWPORT_START_C, HUBP, id),                        \
-	SRI_ARR(DCSURF_SEC_VIEWPORT_DIMENSION_C, HUBP, id),                    \
-	SRI_ARR(DCSURF_SEC_VIEWPORT_START_C, HUBP, id),                        \
-	SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH, HUBPREQ, id),             \
-	SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS, HUBPREQ, id),                  \
-	SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH, HUBPREQ, id),           \
-	SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS, HUBPREQ, id),                \
-	SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH, HUBPREQ, id),        \
-	SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS, HUBPREQ, id),             \
-	SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH, HUBPREQ, id),      \
-	SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS, HUBPREQ, id),           \
-	SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id),           \
-	SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS_C, HUBPREQ, id),                \
-	SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id),         \
-	SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS_C, HUBPREQ, id),              \
-	SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id),      \
-	SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS_C, HUBPREQ, id),           \
-	SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id),    \
-	SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS_C, HUBPREQ, id),         \
-	SRI_ARR(DCSURF_SURFACE_INUSE, HUBPREQ, id),                            \
-	SRI_ARR(DCSURF_SURFACE_INUSE_HIGH, HUBPREQ, id),                       \
-	SRI_ARR(DCSURF_SURFACE_INUSE_C, HUBPREQ, id),                          \
-	SRI_ARR(DCSURF_SURFACE_INUSE_HIGH_C, HUBPREQ, id),                     \
-	SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE, HUBPREQ, id),                   \
-	SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE_HIGH, HUBPREQ, id),              \
-	SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE_C, HUBPREQ, id),                 \
-	SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C, HUBPREQ, id),            \
-	SRI_ARR(DCSURF_SURFACE_CONTROL, HUBPREQ, id),                          \
-	SRI_ARR(DCSURF_SURFACE_FLIP_INTERRUPT, HUBPREQ, id),                   \
-	SRI_ARR(HUBPRET_CONTROL, HUBPRET, id),                                 \
-	SRI_ARR(HUBPRET_READ_LINE_STATUS, HUBPRET, id),                        \
-	SRI_ARR(DCN_EXPANSION_MODE, HUBPREQ, id),                              \
-	SRI_ARR(DCHUBP_REQ_SIZE_CONFIG, HUBP, id),                             \
-	SRI_ARR(DCHUBP_REQ_SIZE_CONFIG_C, HUBP, id),                           \
-	SRI_ARR(BLANK_OFFSET_0, HUBPREQ, id),                                  \
-	SRI_ARR(BLANK_OFFSET_1, HUBPREQ, id),                                  \
-	SRI_ARR(DST_DIMENSIONS, HUBPREQ, id),                                  \
-	SRI_ARR(DST_AFTER_SCALER, HUBPREQ, id),                                \
-	SRI_ARR(VBLANK_PARAMETERS_0, HUBPREQ, id),                             \
-	SRI_ARR(REF_FREQ_TO_PIX_FREQ, HUBPREQ, id),                            \
-	SRI_ARR(VBLANK_PARAMETERS_1, HUBPREQ, id),                             \
-	SRI_ARR(VBLANK_PARAMETERS_3, HUBPREQ, id),                             \
-	SRI_ARR(NOM_PARAMETERS_4, HUBPREQ, id),                                \
-	SRI_ARR(NOM_PARAMETERS_5, HUBPREQ, id),                                \
-	SRI_ARR(PER_LINE_DELIVERY_PRE, HUBPREQ, id),                           \
-	SRI_ARR(PER_LINE_DELIVERY, HUBPREQ, id),                               \
-	SRI_ARR(VBLANK_PARAMETERS_2, HUBPREQ, id),                             \
-	SRI_ARR(VBLANK_PARAMETERS_4, HUBPREQ, id),                             \
-	SRI_ARR(NOM_PARAMETERS_6, HUBPREQ, id),                                \
-	SRI_ARR(NOM_PARAMETERS_7, HUBPREQ, id),                                \
-	SRI_ARR(DCN_TTU_QOS_WM, HUBPREQ, id),                                  \
-	SRI_ARR(DCN_GLOBAL_TTU_CNTL, HUBPREQ, id),                             \
-	SRI_ARR(DCN_SURF0_TTU_CNTL0, HUBPREQ, id),                             \
-	SRI_ARR(DCN_SURF0_TTU_CNTL1, HUBPREQ, id),                             \
-	SRI_ARR(DCN_SURF1_TTU_CNTL0, HUBPREQ, id),                             \
-	SRI_ARR(DCN_SURF1_TTU_CNTL1, HUBPREQ, id),                             \
-	SRI_ARR(DCN_CUR0_TTU_CNTL0, HUBPREQ, id),                              \
-	SRI_ARR(DCN_CUR0_TTU_CNTL1, HUBPREQ, id),                              \
-	SRI_ARR(HUBP_CLK_CNTL, HUBP, id),                                      \
-	SRI_ARR(HUBPRET_READ_LINE_VALUE, HUBPRET, id),                         \
-	SRI_ARR(DCHUBP_MALL_CONFIG, HUBP, id),                                 \
-	SRI_ARR(DCHUBP_VMPG_CONFIG, HUBP, id),                                 \
-	SRI_ARR(UCLK_PSTATE_FORCE, HUBPREQ, id),                               \
-	SRI_ARR(HUBP_3DLUT_DLG_PARAM, CURSOR0_, id),                           \
-	HUBP_3DLUT_FL_REG_LIST_DCN401(id)
 struct dcn42b_resource_pool {
 	struct resource_pool base;
 };
diff --git a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/Makefile b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/Makefile
index d168fb1eacf76..8a9bb0aef9b77 100644
--- a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/Makefile
@@ -9,13 +9,16 @@ soc_and_ip_translator_rcflags := $(CC_FLAGS_NO_FPU)
 
 CFLAGS_$(AMDDALPATH)/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.o := $(soc_and_ip_translator_ccflags)
 CFLAGS_$(AMDDALPATH)/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.o := $(soc_and_ip_translator_ccflags)
+CFLAGS_$(AMDDALPATH)/dc/soc_and_ip_translator/dcn42b/dcn42b_soc_and_ip_translator.o := $(soc_and_ip_translator_ccflags)
 
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.o := $(soc_and_ip_translator_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.o := $(soc_and_ip_translator_rcflags)
+CFLAGS_REMOVE_$(AMDDALPATH)/dc/soc_and_ip_translator/dcn42b/dcn42b_soc_and_ip_translator.o := $(soc_and_ip_translator_rcflags)
 
 soc_and_ip_translator := soc_and_ip_translator.o
 soc_and_ip_translator += dcn401/dcn401_soc_and_ip_translator.o
 soc_and_ip_translator += dcn42/dcn42_soc_and_ip_translator.o
+soc_and_ip_translator += dcn42b/dcn42b_soc_and_ip_translator.o
 
 AMD_DAL_soc_and_ip_translator := $(addprefix $(AMDDALPATH)/dc/soc_and_ip_translator/, $(soc_and_ip_translator))
 
diff --git a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.c b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.c
index 89f7ccd7f81f0..0c8e652c35321 100644
--- a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.c
+++ b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.c
@@ -269,6 +269,22 @@ void dcn401_update_soc_bb_with_values_from_software_policy(struct dml2_soc_bb *s
 	if (dc->bb_overrides.sr_enter_plus_exit_z8_time_ns)
 		soc_bb->power_management_parameters.z8_stutter_enter_plus_exit_latency_us =
 				dc->bb_overrides.sr_enter_plus_exit_z8_time_ns / 1000.0;
+
+	/* Override per-dpm derates based on a custom derate table.
+	 * Global derate value will be used for derates that aren't populated
+	 * 3 derates for a single DPM level:
+	 *  bits 0-7: dram_derate_percent_pixel
+	 *  bits 8-15: fclk_derate_percent
+	 *  bits 16-23: dcfclk_derate_percent
+	 */
+	for (unsigned int i = 0; i < dc->debug.dml21_custom_derate_num_dpms; i++) {
+		soc_bb->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.dram_derate_percent_pixel[i]
+			= dc->debug.dml21_custom_derate_at_dpm[i] & 0xFF;
+		soc_bb->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.fclk_derate_percent[i]
+			= (dc->debug.dml21_custom_derate_at_dpm[i] >> 8) & 0xFF;
+		soc_bb->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.dcfclk_derate_percent[i]
+			= (dc->debug.dml21_custom_derate_at_dpm[i] >> 16) & 0xFF;
+	}
 }
 
 static void apply_soc_bb_updates(struct dml2_soc_bb *soc_bb, const struct dc *dc, const struct dml2_configuration_options *config)
diff --git a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.c b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.c
index ae2c6a2f3f752..c6c1b19b7370c 100644
--- a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.c
+++ b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.c
@@ -5,22 +5,16 @@
 #include "dcn42_soc_and_ip_translator.h"
 #include "../dcn401/dcn401_soc_and_ip_translator.h"
 #include "bounding_boxes/dcn42_soc_bb.h"
-#include "bounding_boxes/dcn42b_soc_bb.h"
 
 /* soc_and_ip_translator component used to get up-to-date values for bounding box.
  * Bounding box values are stored in several locations and locations can vary with DCN revision.
  * This component provides an interface to get DCN-specific bounding box values.
  */
 
-static void get_default_soc_bb(struct dml2_soc_bb *soc_bb, const struct dc *dc)
+static void get_default_soc_bb(struct dml2_soc_bb *soc_bb)
 {
-	if (dc->ctx->dce_version == DCN_VERSION_4_2B) {
-		memcpy(soc_bb, &dml2_socbb_dcn42b, sizeof(struct dml2_soc_bb));
-		memcpy(&soc_bb->qos_parameters, &dml_dcn42b_variant_a_soc_qos_params, sizeof(struct dml2_soc_qos_parameters));
-	} else {
-		memcpy(soc_bb, &dml2_socbb_dcn42, sizeof(struct dml2_soc_bb));
-		memcpy(&soc_bb->qos_parameters, &dml_dcn42_variant_a_soc_qos_params, sizeof(struct dml2_soc_qos_parameters));
-	}
+	memcpy(soc_bb, &dml2_socbb_dcn42, sizeof(struct dml2_soc_bb));
+	memcpy(&soc_bb->qos_parameters, &dml_dcn42_variant_a_soc_qos_params, sizeof(struct dml2_soc_qos_parameters));
 }
 
 /*
@@ -165,7 +159,7 @@ static void dcn42_update_soc_bb_with_values_from_clk_mgr(struct dml2_soc_bb *soc
 	}
 }
 
-static void apply_soc_bb_updates(struct dml2_soc_bb *soc_bb, const struct dc *dc, const struct dml2_configuration_options *config)
+void dcn42_apply_soc_bb_updates(struct dml2_soc_bb *soc_bb, const struct dc *dc, const struct dml2_configuration_options *config)
 {
 	(void)config;
 	/* Individual modification can be overwritten even if it was obtained by a previous function.
@@ -181,9 +175,9 @@ static void apply_soc_bb_updates(struct dml2_soc_bb *soc_bb, const struct dc *dc
 void dcn42_get_soc_bb(struct dml2_soc_bb *soc_bb, const struct dc *dc, const struct dml2_configuration_options *config)
 {
 	//get default soc_bb with static values
-	get_default_soc_bb(soc_bb, dc);
+	get_default_soc_bb(soc_bb);
 	//update soc_bb values with more accurate values
-	apply_soc_bb_updates(soc_bb, dc, config);
+	dcn42_apply_soc_bb_updates(soc_bb, dc, config);
 }
 
 static void dcn42_get_ip_caps(struct dml2_ip_capabilities *ip_caps)
diff --git a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.h b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.h
index 1dded54261521..8ac90655f2768 100644
--- a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.h
+++ b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.h
@@ -13,5 +13,6 @@
 
 void dcn42_construct_soc_and_ip_translator(struct soc_and_ip_translator *soc_and_ip_translator);
 void dcn42_get_soc_bb(struct dml2_soc_bb *soc_bb, const struct dc *dc, const struct dml2_configuration_options *config);
+void dcn42_apply_soc_bb_updates(struct dml2_soc_bb *soc_bb, const struct dc *dc, const struct dml2_configuration_options *config);
 
 #endif /* _DCN42_SOC_AND_IP_TRANSLATOR_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42b/dcn42b_soc_and_ip_translator.c b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42b/dcn42b_soc_and_ip_translator.c
new file mode 100644
index 0000000000000..50669f458e23c
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42b/dcn42b_soc_and_ip_translator.c
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "../dcn42/dcn42_soc_and_ip_translator.h"
+#include "dcn42b_soc_and_ip_translator.h"
+#include "../dcn401/dcn401_soc_and_ip_translator.h"
+#include "bounding_boxes/dcn42b_soc_bb.h"
+
+/* soc_and_ip_translator component used to get up-to-date values for bounding box.
+ * Bounding box values are stored in several locations and locations can vary with DCN revision.
+ * This component provides an interface to get DCN-specific bounding box values.
+ */
+
+static void get_default_soc_bb(struct dml2_soc_bb *soc_bb)
+{
+	memcpy(soc_bb, &dml2_socbb_dcn42b, sizeof(struct dml2_soc_bb));
+	memcpy(&soc_bb->qos_parameters, &dml_dcn42b_variant_a_soc_qos_params, sizeof(struct dml2_soc_qos_parameters));
+}
+
+void dcn42b_get_soc_bb(struct dml2_soc_bb *soc_bb, const struct dc *dc, const struct dml2_configuration_options *config)
+{
+	//get default soc_bb with static values
+	get_default_soc_bb(soc_bb);
+	//update soc_bb values with more accurate values
+	dcn42_apply_soc_bb_updates(soc_bb, dc, config);
+}
+
+static void dcn42b_get_ip_caps(struct dml2_ip_capabilities *ip_caps)
+{
+	*ip_caps = dml2_dcn42b_max_ip_caps;
+}
+
+static struct soc_and_ip_translator_funcs dcn42b_translator_funcs = {
+	.get_soc_bb = dcn42b_get_soc_bb,
+	.get_ip_caps = dcn42b_get_ip_caps,
+};
+
+void dcn42b_construct_soc_and_ip_translator(struct soc_and_ip_translator *soc_and_ip_translator)
+{
+	soc_and_ip_translator->translator_funcs = &dcn42b_translator_funcs;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42b/dcn42b_soc_and_ip_translator.h b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42b/dcn42b_soc_and_ip_translator.h
new file mode 100644
index 0000000000000..0d4ea613431a6
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42b/dcn42b_soc_and_ip_translator.h
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#ifndef _DCN42B_SOC_AND_IP_TRANSLATOR_H_
+#define _DCN42B_SOC_AND_IP_TRANSLATOR_H_
+
+#include "core_types.h"
+#include "dc.h"
+#include "clk_mgr.h"
+#include "dml_top_soc_parameter_types.h"
+#include "soc_and_ip_translator.h"
+
+void dcn42b_construct_soc_and_ip_translator(struct soc_and_ip_translator *soc_and_ip_translator);
+void dcn42b_get_soc_bb(struct dml2_soc_bb *soc_bb, const struct dc *dc, const struct dml2_configuration_options *config);
+
+#endif /* _DCN42B_SOC_AND_IP_TRANSLATOR_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/soc_and_ip_translator.c b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/soc_and_ip_translator.c
index f99afb22d7da4..1e3ee25732faa 100644
--- a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/soc_and_ip_translator.c
+++ b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/soc_and_ip_translator.c
@@ -5,6 +5,7 @@
 #include "soc_and_ip_translator.h"
 #include "soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.h"
 #include "soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.h"
+#include "soc_and_ip_translator/dcn42b/dcn42b_soc_and_ip_translator.h"
 
 static void dc_construct_soc_and_ip_translator(struct soc_and_ip_translator *soc_and_ip_translator,
 		enum dce_version dc_version)
@@ -14,9 +15,11 @@ static void dc_construct_soc_and_ip_translator(struct soc_and_ip_translator *soc
 		dcn401_construct_soc_and_ip_translator(soc_and_ip_translator);
 		break;
 	case DCN_VERSION_4_2:
-	case DCN_VERSION_4_2B:
 		dcn42_construct_soc_and_ip_translator(soc_and_ip_translator);
 		break;
+	case DCN_VERSION_4_2B:
+		dcn42b_construct_soc_and_ip_translator(soc_and_ip_translator);
+		break;
 	default:
 		break;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/spl_debug.h b/drivers/gpu/drm/amd/display/dc/sspl/spl_debug.h
index a6f6132df2416..a0e9df382582e 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/spl_debug.h
+++ b/drivers/gpu/drm/amd/display/dc/sspl/spl_debug.h
@@ -5,7 +5,7 @@
 #ifndef SPL_DEBUG_H
 #define SPL_DEBUG_H
 
-#if defined(CONFIG_HAVE_KGDB) || defined(CONFIG_KGDB)
+#ifdef CONFIG_KGDB
 #define SPL_ASSERT_CRITICAL(expr) do {	\
 	if (WARN_ON(!(expr))) { \
 		kgdb_breakpoint(); \
@@ -17,7 +17,7 @@
 		; \
 	} \
 } while (0)
-#endif /* CONFIG_HAVE_KGDB || CONFIG_KGDB */
+#endif /* CONFIG_KGDB */
 
 #if defined(CONFIG_DEBUG_KERNEL_DC)
 #define SPL_ASSERT(expr) SPL_ASSERT_CRITICAL(expr)
diff --git a/drivers/gpu/drm/amd/display/dmub/dmub_srv.h b/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
index f4d05dcfef292..94d4b40ee16aa 100644
--- a/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
+++ b/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
@@ -329,7 +329,7 @@ struct dmub_soc_fb_info {
  * @load_inst_const: true if DMUB should load inst const fw
  */
 struct dmub_srv_hw_params {
-	struct dmub_fb *fb[DMUB_WINDOW_TOTAL];
+	struct dmub_srv_fb_info *fb_info;
 	struct dmub_soc_fb_info soc_fb_info;
 	uint32_t psp_version;
 	bool load_inst_const;
@@ -592,9 +592,11 @@ struct dmub_srv {
 	void *user_ctx;
 	uint32_t fw_version;
 	bool is_virtual;
+	bool no_ext_reg_access;
 	struct dmub_fb scratch_mem_fb;
 	struct dmub_fb ib_mem_gart;
 	struct dmub_fb cursor_offload_fb;
+	const struct dmub_srv_fb_info *fb_info;
 	volatile struct dmub_shared_state_feature_block *shared_state;
 	volatile struct dmub_cursor_offload_v1 *cursor_offload_v1;
 	volatile const struct dmub_fw_state *fw_state;
@@ -954,14 +956,15 @@ enum dmub_status dmub_srv_get_gpint_dataout(struct dmub_srv *dmub,
 					     uint32_t *dataout);
 
 /**
- * dmub_flush_buffer_mem() - Read back entire frame buffer region.
+ * dmub_srv_flush_buffer_mem() - Read back entire frame buffer region.
  * This ensures that the write from x86 has been flushed and will not
  * hang the DMCUB.
+ * @dmub: the dmub service
  * @fb: frame buffer to flush
  *
  * Can be called after software initialization.
  */
-void dmub_flush_buffer_mem(const struct dmub_fb *fb);
+void dmub_srv_flush_buffer_mem(struct dmub_srv *dmub, const struct dmub_fb *fb);
 
 /**
  * dmub_srv_get_fw_boot_status() - Returns the DMUB boot status bits.
diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
index 6f6a59a234955..e7879bd81f832 100644
--- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
+++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
@@ -246,14 +246,14 @@
  * OS/FW agnostic memcpy
  */
 #ifndef dmub_memcpy
-#define dmub_memcpy(dest, source, bytes) memcpy((dest), (source), (bytes))
+#define dmub_memcpy(dest, source, bytes) ((void)memcpy((dest), (source), (bytes)))
 #endif
 
 /**
  * OS/FW agnostic memset
  */
 #ifndef dmub_memset
-#define dmub_memset(dest, val, bytes) memset((dest), (val), (bytes))
+#define dmub_memset(dest, val, bytes) ((void)memset((dest), (val), (bytes)))
 #endif
 
 /**
@@ -1702,6 +1702,17 @@ enum dmub_gpint_command {
 	 * ARGS: 1 - Power off
 	 */
 	DMUB_GPINT__PANEL_POWER_OFF_SEQ = 138,
+	/**
+	 * DESC: Gets panel polarity bias.
+	 * ARGS: 0 - Get panel polarity bias
+	 */
+	DMUB_GPINT__PANEL_POLARITY_GET_BIAS = 139,
+	/**
+	 * DESC: Enables panel polarity.
+	 * ARGS: 0 - Disable panel polarity
+	 *       1 - Enable panel polarity
+	 */
+	DMUB_GPINT__PANEL_POLARITY_DEBUG_ENABLE = 140,
 };
 
 /**
@@ -1800,28 +1811,13 @@ enum dmub_inbox0_command {
  *
  * Command IDs should be treated as stable ABI.
  * Do not reuse or modify IDs.
+ * Note that command IDs 1-4 have been deprecated.
  */
 enum dmub_cmd_type {
 	/**
 	 * Invalid command.
 	 */
 	DMUB_CMD__NULL = 0,
-	/**
-	 * Read modify write register sequence offload.
-	 */
-	DMUB_CMD__REG_SEQ_READ_MODIFY_WRITE = 1,
-	/**
-	 * Field update register sequence offload.
-	 */
-	DMUB_CMD__REG_SEQ_FIELD_UPDATE_SEQ = 2,
-	/**
-	 * Burst write sequence offload.
-	 */
-	DMUB_CMD__REG_SEQ_BURST_WRITE = 3,
-	/**
-	 * Reg wait sequence offload.
-	 */
-	DMUB_CMD__REG_REG_WAIT = 4,
 	/**
 	 * Workaround to avoid HUBP underflow during NV12 playback.
 	 */
@@ -1971,6 +1967,11 @@ enum dmub_cmd_type {
 	 */
 	DMUB_CMD__BOOT_TIME_CRC = 96,
 
+	/**
+	 * Command type use for all Panel Polarity commands.
+	 */
+	DMUB_CMD__PANEL_POLARITY = 97,
+
 	/**
 	 * Command type use for VBIOS shared commands.
 	 */
@@ -2041,98 +2042,6 @@ struct dmub_cmd_header {
 	unsigned int reserved1 : 2; /**< reserved bits */
 };
 
-/*
- * struct dmub_cmd_read_modify_write_sequence - Read modify write
- *
- * 60 payload bytes can hold up to 5 sets of read modify writes,
- * each take 3 dwords.
- *
- * number of sequences = header.payload_bytes / sizeof(struct dmub_cmd_read_modify_write_sequence)
- *
- * modify_mask = 0xffff'ffff means all fields are going to be updated.  in this case
- * command parser will skip the read and we can use modify_mask = 0xffff'ffff as reg write
- */
-struct dmub_cmd_read_modify_write_sequence {
-	uint32_t addr; /**< register address */
-	uint32_t modify_mask; /**< modify mask */
-	uint32_t modify_value; /**< modify value */
-};
-
-/**
- * Maximum number of ops in read modify write sequence.
- */
-#define DMUB_READ_MODIFY_WRITE_SEQ__MAX 5
-
-/**
- * struct dmub_cmd_read_modify_write_sequence - Read modify write command.
- */
-struct dmub_rb_cmd_read_modify_write {
-	struct dmub_cmd_header header;  /**< command header */
-	/**
-	 * Read modify write sequence.
-	 */
-	struct dmub_cmd_read_modify_write_sequence seq[DMUB_READ_MODIFY_WRITE_SEQ__MAX];
-};
-
-/*
- * Update a register with specified masks and values sequeunce
- *
- * 60 payload bytes can hold address + up to 7 sets of mask/value combo, each take 2 dword
- *
- * number of field update sequence = (header.payload_bytes - sizeof(addr)) / sizeof(struct read_modify_write_sequence)
- *
- *
- * USE CASE:
- *   1. auto-increment register where additional read would update pointer and produce wrong result
- *   2. toggle a bit without read in the middle
- */
-
-struct dmub_cmd_reg_field_update_sequence {
-	uint32_t modify_mask; /**< 0xffff'ffff to skip initial read */
-	uint32_t modify_value; /**< value to update with */
-};
-
-/**
- * Maximum number of ops in field update sequence.
- */
-#define DMUB_REG_FIELD_UPDATE_SEQ__MAX 7
-
-/**
- * struct dmub_rb_cmd_reg_field_update_sequence - Field update command.
- */
-struct dmub_rb_cmd_reg_field_update_sequence {
-	struct dmub_cmd_header header; /**< command header */
-	uint32_t addr; /**< register address */
-	/**
-	 * Field update sequence.
-	 */
-	struct dmub_cmd_reg_field_update_sequence seq[DMUB_REG_FIELD_UPDATE_SEQ__MAX];
-};
-
-
-/**
- * Maximum number of burst write values.
- */
-#define DMUB_BURST_WRITE_VALUES__MAX  14
-
-/*
- * struct dmub_rb_cmd_burst_write - Burst write
- *
- * support use case such as writing out LUTs.
- *
- * 60 payload bytes can hold up to 14 values to write to given address
- *
- * number of payload = header.payload_bytes / sizeof(struct read_modify_write_sequence)
- */
-struct dmub_rb_cmd_burst_write {
-	struct dmub_cmd_header header; /**< command header */
-	uint32_t addr; /**< register start address */
-	/**
-	 * Burst write register values.
-	 */
-	uint32_t write_values[DMUB_BURST_WRITE_VALUES__MAX];
-};
-
 /**
  * struct dmub_rb_cmd_common - Common command header
  */
@@ -2144,24 +2053,6 @@ struct dmub_rb_cmd_common {
 	uint8_t cmd_buffer[DMUB_RB_CMD_SIZE - sizeof(struct dmub_cmd_header)];
 };
 
-/**
- * struct dmub_cmd_reg_wait_data - Register wait data
- */
-struct dmub_cmd_reg_wait_data {
-	uint32_t addr; /**< Register address */
-	uint32_t mask; /**< Mask for register bits */
-	uint32_t condition_field_value; /**< Value to wait for */
-	uint32_t time_out_us; /**< Time out for reg wait in microseconds */
-};
-
-/**
- * struct dmub_rb_cmd_reg_wait - Register wait command
- */
-struct dmub_rb_cmd_reg_wait {
-	struct dmub_cmd_header header; /**< Command header */
-	struct dmub_cmd_reg_wait_data reg_wait; /**< Register wait data */
-};
-
 /**
  * struct dmub_cmd_PLAT_54186_wa - Underflow workaround
  *
@@ -4490,6 +4381,15 @@ enum dmub_cmd_replay_type {
 	DMUB_CMD__REPLAY_SET_GENERAL_CMD = 16,
 };
 
+/*
+ * Panel Polarity sub-types
+ */
+enum dmub_cmd_panel_polarity_type {
+	DMUB_CMD__PANEL_POLARITY_ENABLE = 0,
+	DMUB_CMD__PANEL_POLARITY_GET_BIAS = 1,
+	DMUB_CMD__PANEL_POLARITY_RESET = 2,
+};
+
 /*
  * Panel Replay sub-types
  */
@@ -6439,10 +6339,43 @@ struct dmub_cmd_cacp_set_backlight_data {
 	 */
 	uint8_t panel_mask;
 
+	/**
+	 * AUX HW Instance.
+	 */
+	uint8_t aux_inst;
+
 	/**
 	 * Explicit padding to 4 byte boundary.
 	 */
-	uint8_t pad[2];
+	uint8_t pad[1];
+
+	/**
+	 * Backlight control type.
+	 * Value 0 is PWM backlight control.
+	 * Value 1 is VAUX backlight control.
+	 * Value 2 is AMD DPCD AUX backlight control.
+	 */
+	enum dmub_backlight_control_type backlight_control_type;
+
+	/**
+	 * Minimum luminance in nits.
+	 */
+	uint32_t min_luminance;
+
+	/**
+	 * Maximum luminance in nits.
+	 */
+	uint32_t max_luminance;
+
+	/**
+	 * Minimum backlight in pwm.
+	 */
+	uint32_t min_backlight_pwm;
+
+	/**
+	 * Maximum backlight in pwm.
+	 */
+	uint32_t max_backlight_pwm;
 };
 
 /**
@@ -7123,6 +7056,80 @@ struct dmub_cmd_pr_enable_data {
 	uint8_t pad[2];
 };
 
+struct dmub_cmd_panel_polarity_enable_data {
+	/**
+	 * Panel Polarity enable or disable.
+	 */
+	uint8_t enable;
+	/**
+	 * OTG instance
+	 */
+	uint8_t otg_inst;
+	/**
+	 * @pad: Align structure to 4 byte boundary.
+	 */
+	uint8_t pad[2];
+};
+
+struct dmub_cmd_panel_polarity_reset_data {
+	/**
+	 * OTG instance
+	 */
+	uint8_t otg_inst;
+	/**
+	 * @pad: Align structure to 4 byte boundary.
+	 */
+	uint8_t pad[3];
+};
+
+struct dmub_cmd_panel_polarity_get_bias_input {
+	/**
+	 * OTG instance
+	 */
+	uint8_t otg_inst;
+	uint8_t pad[3];
+};
+
+struct dmub_cmd_panel_polarity_get_bias_output {
+	/**
+	 * Accumulated Polarity Bias
+	 */
+	int32_t accumulated_bias;
+};
+
+struct dmub_rb_cmd_panel_polarity_enable {
+	/**
+	 * Command header.
+	 */
+	struct dmub_cmd_header header;
+
+	struct dmub_cmd_panel_polarity_enable_data data;
+};
+
+
+struct dmub_rb_cmd_panel_polarity_get_bias {
+	/**
+	 * Command header.
+	 */
+	struct dmub_cmd_header header;
+
+	union dmub_cmd_panel_polarity_get_bias_data {
+		struct dmub_cmd_panel_polarity_get_bias_input input; /**< Input */
+		struct dmub_cmd_panel_polarity_get_bias_output output; /**< Output */
+		uint32_t output_raw; /**< Raw data output */
+	} data;
+};
+
+struct dmub_rb_cmd_panel_polarity_reset {
+	/**
+	 * Command header.
+	 */
+	struct dmub_cmd_header header;
+
+	struct dmub_cmd_panel_polarity_reset_data data;
+};
+
+
 /**
  * Definition of a DMUB_CMD__PR_ENABLE command.
  * Panel Replay enable/disable is controlled using action in data.
@@ -7362,22 +7369,6 @@ union dmub_rb_cmd {
 	 * Elements shared with all commands.
 	 */
 	struct dmub_rb_cmd_common cmd_common;
-	/**
-	 * Definition of a DMUB_CMD__REG_SEQ_READ_MODIFY_WRITE command.
-	 */
-	struct dmub_rb_cmd_read_modify_write read_modify_write;
-	/**
-	 * Definition of a DMUB_CMD__REG_SEQ_FIELD_UPDATE_SEQ command.
-	 */
-	struct dmub_rb_cmd_reg_field_update_sequence reg_field_update_seq;
-	/**
-	 * Definition of a DMUB_CMD__REG_SEQ_BURST_WRITE command.
-	 */
-	struct dmub_rb_cmd_burst_write burst_write;
-	/**
-	 * Definition of a DMUB_CMD__REG_REG_WAIT command.
-	 */
-	struct dmub_rb_cmd_reg_wait reg_wait;
 	/**
 	 * Definition of a DMUB_CMD__VBIOS_DIGX_ENCODER_CONTROL command.
 	 */
@@ -7754,6 +7745,7 @@ union dmub_rb_cmd {
 	struct dmub_rb_cmd_pr_update_state pr_update_state;
 
 	struct dmub_rb_cmd_pr_general_cmd pr_general_cmd;
+
 	/**
 	 * Definition of a DMUB_CMD__IHC command.
 	 */
@@ -7762,6 +7754,13 @@ union dmub_rb_cmd {
 	 * Definition of a DMUB_CMD__BOOT_TIME_CRC_INIT command.
 	 */
 	struct dmub_rb_cmd_boot_time_crc_init boot_time_crc_init;
+
+	/**
+	 * Definition of a DMUB_CMD__PANEL_POLARITY_ENABLE command.
+	 */
+	struct dmub_rb_cmd_panel_polarity_enable panel_polarity_enable;
+	struct dmub_rb_cmd_panel_polarity_get_bias panel_polarity_get_bias;
+	struct dmub_rb_cmd_panel_polarity_reset panel_polarity_reset;
 };
 
 /**
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c
index 7463d2ae50559..5a1a2276c432b 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c
@@ -96,7 +96,7 @@ static inline uint32_t dmub_align(uint32_t val, uint32_t factor)
 	return (val + factor - 1) / factor * factor;
 }
 
-void dmub_flush_buffer_mem(const struct dmub_fb *fb)
+void dmub_srv_flush_buffer_mem(struct dmub_srv *dmub, const struct dmub_fb *fb)
 {
 	const uint8_t *base = (const uint8_t *)fb->cpu_addr;
 	uint8_t buf[64];
@@ -114,6 +114,8 @@ void dmub_flush_buffer_mem(const struct dmub_fb *fb)
 	/* Read anything leftover into the buffer. */
 	if (end < fb->size)
 		dmub_memcpy(buf, base + pos, fb->size - end);
+
+	(void)dmub;
 }
 
 static const struct dmub_fw_meta_info *
@@ -748,31 +750,38 @@ enum dmub_status dmub_srv_is_hw_init(struct dmub_srv *dmub, bool *is_hw_init)
 enum dmub_status dmub_srv_hw_init(struct dmub_srv *dmub,
 				  const struct dmub_srv_hw_params *params)
 {
-	struct dmub_fb *inst_fb = params->fb[DMUB_WINDOW_0_INST_CONST];
-	struct dmub_fb *stack_fb = params->fb[DMUB_WINDOW_1_STACK];
-	struct dmub_fb *data_fb = params->fb[DMUB_WINDOW_2_BSS_DATA];
-	struct dmub_fb *bios_fb = params->fb[DMUB_WINDOW_3_VBIOS];
-	struct dmub_fb *mail_fb = params->fb[DMUB_WINDOW_4_MAILBOX];
-	struct dmub_fb *tracebuff_fb = params->fb[DMUB_WINDOW_5_TRACEBUFF];
-	struct dmub_fb *fw_state_fb = params->fb[DMUB_WINDOW_6_FW_STATE];
-	struct dmub_fb *shared_state_fb = params->fb[DMUB_WINDOW_SHARED_STATE];
+	struct dmub_fb *inst_fb;
+	struct dmub_fb *stack_fb;
+	struct dmub_fb *data_fb;
+	struct dmub_fb *bios_fb;
+	struct dmub_fb *mail_fb;
+	struct dmub_fb *tracebuff_fb;
+	struct dmub_fb *fw_state_fb;
+	struct dmub_fb *shared_state_fb;
 
 	struct dmub_rb_init_params rb_params, outbox0_rb_params;
 	struct dmub_window cw0, cw1, cw2, cw3, cw4, cw5, cw6, region6;
 	struct dmub_region inbox1, outbox1, outbox0;
 
-	uint32_t i;
-
 	if (!dmub->sw_init)
 		return DMUB_STATUS_INVALID;
 
-	for (i = 0; i < DMUB_WINDOW_TOTAL; ++i) {
-		if (!params->fb[i]) {
-			ASSERT(0);
-			return DMUB_STATUS_INVALID;
-		}
+	if (!params->fb_info || params->fb_info->num_fb < DMUB_WINDOW_TOTAL) {
+		ASSERT(0);
+		return DMUB_STATUS_INVALID;
 	}
 
+	inst_fb = &params->fb_info->fb[DMUB_WINDOW_0_INST_CONST];
+	stack_fb = &params->fb_info->fb[DMUB_WINDOW_1_STACK];
+	data_fb = &params->fb_info->fb[DMUB_WINDOW_2_BSS_DATA];
+	bios_fb = &params->fb_info->fb[DMUB_WINDOW_3_VBIOS];
+	mail_fb = &params->fb_info->fb[DMUB_WINDOW_4_MAILBOX];
+	tracebuff_fb = &params->fb_info->fb[DMUB_WINDOW_5_TRACEBUFF];
+	fw_state_fb = &params->fb_info->fb[DMUB_WINDOW_6_FW_STATE];
+	shared_state_fb = &params->fb_info->fb[DMUB_WINDOW_SHARED_STATE];
+
+	dmub->fb_info = params->fb_info;
+
 	memcpy(&dmub->soc_fb_info, &params->soc_fb_info, sizeof(params->soc_fb_info));
 	dmub->psp_version = params->psp_version;
 
@@ -800,7 +809,7 @@ enum dmub_status dmub_srv_hw_init(struct dmub_srv *dmub,
 		 * flushed yet. This only occurs in backdoor loading.
 		 */
 		if (params->mem_access_type == DMUB_MEMORY_ACCESS_CPU)
-			dmub_flush_buffer_mem(inst_fb);
+			dmub_srv_flush_buffer_mem(dmub, inst_fb);
 
 		if (params->fw_in_system_memory && dmub->hw_funcs.backdoor_load_zfb_mode)
 			dmub->hw_funcs.backdoor_load_zfb_mode(dmub, &cw0, &cw1);
@@ -851,10 +860,10 @@ enum dmub_status dmub_srv_hw_init(struct dmub_srv *dmub,
 
 	dmub->shared_state = shared_state_fb->cpu_addr;
 
-	dmub->scratch_mem_fb = *params->fb[DMUB_WINDOW_7_SCRATCH_MEM];
-	dmub->ib_mem_gart = *params->fb[DMUB_WINDOW_IB_MEM];
+	dmub->scratch_mem_fb = params->fb_info->fb[DMUB_WINDOW_7_SCRATCH_MEM];
+	dmub->ib_mem_gart = params->fb_info->fb[DMUB_WINDOW_IB_MEM];
 
-	dmub->cursor_offload_fb = *params->fb[DMUB_WINDOW_CURSOR_OFFLOAD];
+	dmub->cursor_offload_fb = params->fb_info->fb[DMUB_WINDOW_CURSOR_OFFLOAD];
 	dmub->cursor_offload_v1 = (struct dmub_cursor_offload_v1 *)dmub->cursor_offload_fb.cpu_addr;
 
 	if (dmub->hw_funcs.setup_windows)
@@ -1014,13 +1023,14 @@ enum dmub_status dmub_srv_wait_for_hw_pwr_up(struct dmub_srv *dmub,
 enum dmub_status dmub_srv_wait_for_auto_load(struct dmub_srv *dmub,
 					     uint32_t timeout_us)
 {
+	const uint32_t delay_us = 100;
 	uint32_t i;
 	bool hw_on = true;
 
 	if (!dmub->hw_init)
 		return DMUB_STATUS_INVALID;
 
-	for (i = 0; i <= timeout_us; i += 100) {
+	for (i = 0; i <= timeout_us; i += delay_us) {
 		union dmub_fw_boot_status status = dmub->hw_funcs.get_fw_status(dmub);
 
 		if (dmub->hw_funcs.is_hw_powered_up)
@@ -1029,7 +1039,7 @@ enum dmub_status dmub_srv_wait_for_auto_load(struct dmub_srv *dmub,
 		if (status.bits.dal_fw && status.bits.mailbox_rdy && hw_on)
 			return DMUB_STATUS_OK;
 
-		udelay(100);
+		udelay(delay_us);
 	}
 
 	return DMUB_STATUS_TIMEOUT;
@@ -1258,6 +1268,7 @@ bool dmub_srv_get_diagnostic_data(struct dmub_srv *dmub)
 	if (!dmub || !dmub->hw_funcs.get_diagnostic_data)
 		return false;
 	dmub->hw_funcs.get_diagnostic_data(dmub);
+
 	return true;
 }
 
diff --git a/drivers/gpu/drm/amd/display/include/ddc_service_types.h b/drivers/gpu/drm/amd/display/include/ddc_service_types.h
index 53210e3aa0e0d..827e9bd7c5cff 100644
--- a/drivers/gpu/drm/amd/display/include/ddc_service_types.h
+++ b/drivers/gpu/drm/amd/display/include/ddc_service_types.h
@@ -45,6 +45,7 @@
 #define DP_DEVICE_ID_BA4159 0xBA4159
 #define DP_FORCE_PSRSU_CAPABILITY 0x40F
 
+#define DP_SINK_PSR_ACTIVE_VTOTAL_CONTROL_CAP	0x370
 #define DP_SINK_PSR_ACTIVE_VTOTAL		0x373
 #define DP_SINK_PSR_ACTIVE_VTOTAL_CONTROL_MODE	0x375
 #define DP_SOURCE_PSR_ACTIVE_VTOTAL		0x376
diff --git a/drivers/gpu/drm/amd/display/include/gpio_types.h b/drivers/gpu/drm/amd/display/include/gpio_types.h
index 8dd46ed799e5e..afd3fc73a9112 100644
--- a/drivers/gpu/drm/amd/display/include/gpio_types.h
+++ b/drivers/gpu/drm/amd/display/include/gpio_types.h
@@ -277,6 +277,49 @@ enum gpio_config_type {
 	GPIO_CONFIG_TYPE_I2C_AUX_DUAL_MODE
 };
 
+struct gpio_id_offset_entry {
+	uint32_t offset;
+	uint32_t mask;
+
+	bool check_mask;
+
+	enum gpio_id id;
+	uint32_t en;
+};
+
+#define GPIO_ENTRY(_offset, _id, _en) \
+	{ \
+		.offset = REG(_offset), \
+		.check_mask = false, \
+		.id = (_id), \
+		.en = (_en), \
+	}
+
+#define GPIO_MASK_ENTRY(_offset, _mask, _id, _en) \
+	{ \
+		.offset = REG(_offset), \
+		.mask = (_mask), \
+		.check_mask = true, \
+		.id = (_id), \
+		.en = (_en), \
+	}
+
+struct gpio_pin_entry {
+	enum gpio_id id;
+	uint32_t en;
+
+	uint32_t offset;
+	uint32_t mask;
+};
+
+#define GPIO_PIN_ENTRY(_id, _en, _offset, _mask) \
+	{ \
+		.id = (_id), \
+		.en = (_en), \
+		.offset = REG(_offset), \
+		.mask = (_mask), \
+	}
+
 /* DDC configuration */
 
 enum gpio_ddc_config_type {
@@ -293,6 +336,11 @@ struct gpio_ddc_config {
 	bool clock_en_bit_present;
 };
 
+struct gpio_ddc_offset_entry {
+	uint32_t offset;
+	uint32_t en;
+};
+
 /* HPD configuration */
 
 struct gpio_hpd_config {
diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_power.h b/drivers/gpu/drm/amd/display/modules/inc/mod_power.h
index f9814cf7bbdb1..02bee3b1956de 100644
--- a/drivers/gpu/drm/amd/display/modules/inc/mod_power.h
+++ b/drivers/gpu/drm/amd/display/modules/inc/mod_power.h
@@ -328,6 +328,17 @@ bool mod_power_is_abm_active(struct mod_power *mod_power,
 		const struct dc_link *link,
 		unsigned int inst);
 
+bool mod_power_is_abm_supported(struct mod_power *mod_power,
+		unsigned int inst);
+
+bool mod_power_abm_set_event(struct mod_power *mod_power,
+		unsigned int full_screen, unsigned int trans_info,
+		unsigned int hdr_mode, unsigned int scaling_enable,
+		unsigned int scaling_strength_map, unsigned int inst);
+
+bool mod_power_abm_set_strength(struct mod_power *mod_power,
+		unsigned int strength,
+		unsigned int inst);
 
 bool mod_power_set_psr_event(struct mod_power *mod_power,
 		struct dc_stream_state *stream, bool set_event,
diff --git a/drivers/gpu/drm/amd/display/modules/power/Makefile b/drivers/gpu/drm/amd/display/modules/power/Makefile
index 3000f392bdbc3..0746f671eb4d8 100644
--- a/drivers/gpu/drm/amd/display/modules/power/Makefile
+++ b/drivers/gpu/drm/amd/display/modules/power/Makefile
@@ -23,7 +23,7 @@
 # Makefile for the 'power' sub-module of DAL.
 #
 
-MOD_POWER = power_helpers.o power.o power_abm.o power_psr.o power_replay.o
+MOD_POWER = power.o power_abm.o power_psr.o power_replay.o
 
 AMD_DAL_MOD_POWER = $(addprefix $(AMDDALPATH)/modules/power/,$(MOD_POWER))
 #$(info ************  DAL POWER MODULE MAKEFILE ************)
diff --git a/drivers/gpu/drm/amd/display/modules/power/power.c b/drivers/gpu/drm/amd/display/modules/power/power.c
index 5659a38b33663..ee15c14a899ee 100644
--- a/drivers/gpu/drm/amd/display/modules/power/power.c
+++ b/drivers/gpu/drm/amd/display/modules/power/power.c
@@ -270,13 +270,11 @@ struct mod_power *mod_power_create(struct dc *dc,
 fail_bad_brightness_range:
 fail_alloc_backlight_array:
 	for (inst = 0; inst < edp_num; inst++)
-		if (core_power->bl_prop[inst].backlight_lut)
-			kfree(core_power->bl_prop[inst].backlight_lut);
+		kfree(core_power->bl_prop[inst].backlight_lut);
 fail_construct:
-	for (i = 0; i < MOD_POWER_MAX_CONCURRENT_STREAMS; i++) {
-		if (core_power->map[i].psr_context)
-			kfree(core_power->map[i].psr_context);
-	}
+	for (i = 0; i < MOD_POWER_MAX_CONCURRENT_STREAMS; i++)
+		kfree(core_power->map[i].psr_context);
+
 	kfree(core_power->map);
 
 fail_alloc_map:
@@ -295,8 +293,7 @@ void mod_power_destroy(struct mod_power *mod_power)
 				MOD_POWER_TO_CORE(mod_power);
 
 		for (i = 0; i < MOD_POWER_MAX_CONCURRENT_STREAMS; i++)
-			if (core_power->map[i].psr_context)
-				kfree(core_power->map[i].psr_context);
+			kfree(core_power->map[i].psr_context);
 
 		for (i = 0; i < core_power->num_entities; i++)
 			if (core_power->map[i].stream)
@@ -305,8 +302,7 @@ void mod_power_destroy(struct mod_power *mod_power)
 		kfree(core_power->map);
 
 		for (i = 0; i < MAX_NUM_EDP; i++)
-			if (core_power->bl_prop[i].backlight_lut)
-				kfree(core_power->bl_prop[i].backlight_lut);
+			kfree(core_power->bl_prop[i].backlight_lut);
 
 		kfree(core_power);
 	}
@@ -483,12 +479,7 @@ bool mod_power_notify_mode_change(struct mod_power *mod_power,
 	link = dc_stream_get_link(stream);
 
 	if (link != NULL && dc_get_edp_link_panel_inst(dc, link, &panel_inst)) {
-		if (link->ctx->dc->config.dp_connector_no_native_i2c && link->no_ddc_pin) {
-			aux_inst = (uint8_t)link->aux_hw_inst;
-		} else {
-			ASSERT(link->ddc->ddc_pin->hw_info.ddc_channel <= 0xFF);
-			aux_inst = (uint8_t)link->ddc->ddc_pin->hw_info.ddc_channel;
-		}
+		aux_inst = link->dc->link_srv->get_ddc_aux_inst(link);
 
 		mod_power_update_backlight_on_mode_change(core_power, link, panel_inst, aux_inst, is_hdr);
 
@@ -501,3 +492,8 @@ bool mod_power_notify_mode_change(struct mod_power *mod_power,
 
 	return true;
 }
+
+bool mod_power_only_edp(const struct dc_state *context, const struct dc_stream_state *stream)
+{
+	return context && context->stream_count == 1 && dc_is_embedded_signal(stream->signal);
+}
diff --git a/drivers/gpu/drm/amd/display/modules/power/power_abm.c b/drivers/gpu/drm/amd/display/modules/power/power_abm.c
index a1a0563598b5c..b26ceaba940d2 100644
--- a/drivers/gpu/drm/amd/display/modules/power/power_abm.c
+++ b/drivers/gpu/drm/amd/display/modules/power/power_abm.c
@@ -716,8 +716,9 @@ void mod_power_update_backlight_on_mode_change(
 {
     struct set_backlight_level_params backlight_level_params = { 0 };
 
-		if (link->dpcd_sink_ext_caps.bits.hdr_aux_backlight_control == 1 ||
-			link->dpcd_sink_ext_caps.bits.sdr_aux_backlight_control == 1)
+		if ((link->dpcd_sink_ext_caps.bits.hdr_aux_backlight_control == 1 ||
+			link->dpcd_sink_ext_caps.bits.sdr_aux_backlight_control == 1) &&
+			link->backlight_control_type == BACKLIGHT_CONTROL_AMD_AUX)
 			dc_link_set_backlight_level_nits(link, core_power->bl_state[panel_inst].isHDR,
 				core_power->bl_state[panel_inst].backlight_millinit, 0);
 
@@ -746,6 +747,11 @@ static bool set_backlight_millinits_aux(struct core_power *core_power,
 
 	link = dc_stream_get_link(stream);
 
+	// only use internal backlight control if dmub capabilities are not present
+	if (link->backlight_control_type == BACKLIGHT_CONTROL_VESA_AUX &&
+		link->dc->caps.dmub_caps.aux_backlight_support)
+		return true;
+
 	return dc_link_set_backlight_level_nits(link, core_power->bl_state[inst].isHDR,
 			backlight_millinits, transition_time_millisec);
 }
@@ -849,12 +855,7 @@ bool mod_power_set_backlight_nits(struct mod_power *mod_power,
 	core_power = MOD_POWER_TO_CORE(mod_power);
 	link = dc_stream_get_link(stream);
 
-	if (link->ctx->dc->config.dp_connector_no_native_i2c && link->no_ddc_pin) {
-		aux_inst = (uint8_t)link->aux_hw_inst;
-	} else {
-		ASSERT(link->ddc->ddc_pin->hw_info.ddc_channel <= 0xFF);
-		aux_inst = (uint8_t)link->ddc->ddc_pin->hw_info.ddc_channel;
-	}
+	aux_inst = link->dc->link_srv->get_ddc_aux_inst(link);
 
 	if (!dc_get_edp_link_panel_inst(core_power->dc, stream->link, &panel_inst))
 		return false;
@@ -941,12 +942,7 @@ bool mod_power_set_backlight_percent(struct mod_power *mod_power,
 
 	core_power = MOD_POWER_TO_CORE(mod_power);
 	link = dc_stream_get_link(stream);
-	if (link->ctx->dc->config.dp_connector_no_native_i2c && link->no_ddc_pin) {
-		aux_inst = (uint8_t)link->aux_hw_inst;
-	} else {
-		ASSERT(link->ddc->ddc_pin->hw_info.ddc_channel <= 0xFF);
-		aux_inst = (uint8_t)link->ddc->ddc_pin->hw_info.ddc_channel;
-	}
+	aux_inst = link->dc->link_srv->get_ddc_aux_inst(link);
 
 	if (!dc_get_edp_link_panel_inst(core_power->dc, stream->link, &panel_inst))
 		return false;
@@ -1463,6 +1459,76 @@ bool mod_power_is_abm_active(struct mod_power *mod_power,
 	return is_active;
 }
 
+bool mod_power_is_abm_supported(struct mod_power *mod_power,
+		unsigned int inst)
+{
+	struct core_power *core_power = NULL;
+	struct dc *dc = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	dc = core_power->dc;
+
+	// It's only implemented on dmcub.
+	if (dc->ctx->dmub_srv) {
+		if (!dmub_is_abm_supported(dc->res_pool, inst))
+			return false;
+	} else
+		return false;
+
+	return true;
+}
+
+bool mod_power_abm_set_event(struct mod_power *mod_power,
+		unsigned int full_screen, unsigned int trans_info,
+		unsigned int hdr_mode, unsigned int scaling_enable,
+		unsigned int scaling_strength_map, unsigned int inst)
+{
+	struct core_power *core_power = NULL;
+	struct dc *dc = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	dc = core_power->dc;
+
+	// It's only implemented on dmcub.
+	if (dc->ctx->dmub_srv) {
+		if (!dmub_set_abm_event(dc->res_pool, full_screen, trans_info,
+				hdr_mode, scaling_enable, scaling_strength_map, inst))
+			return false;
+	} else
+		return false;
+
+	return true;
+}
+
+bool mod_power_abm_set_strength(struct mod_power *mod_power,
+		unsigned int strength,
+		unsigned int inst)
+{
+	struct core_power *core_power = NULL;
+	struct dc *dc = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	dc = core_power->dc;
+
+	// It's only implemented on dmcub.
+	if (dc->ctx->dmub_srv) {
+		if (!dmub_set_abm_strength(dc->res_pool, strength, inst))
+			return false;
+	} else
+		return false;
+
+	return true;
+}
+
 static void fill_backlight_transform_table(struct dmcu_iram_parameters params,
 		struct iram_table_v_2 *table)
 {
@@ -2000,6 +2066,62 @@ bool dmub_init_abm_config(struct resource_pool *res_pool,
 	return result;
 }
 
+bool dmub_is_abm_supported(struct resource_pool *res_pool, unsigned int inst)
+{
+
+	if (res_pool->abm == NULL && res_pool->multiple_abms[inst] == NULL)
+		return false;
+
+	return true;
+}
+
+bool dmub_set_abm_event(struct resource_pool *res_pool,
+		unsigned int full_screen, unsigned int trans_info,
+		unsigned int hdr_mode, unsigned int scaling_enable, unsigned int scaling_strength_map,
+		unsigned int inst)
+{
+	bool result = false;
+
+	if (res_pool->abm == NULL && res_pool->multiple_abms[inst] == NULL)
+		return false;
+
+	if (res_pool->multiple_abms[inst]) {
+		if (res_pool->multiple_abms[inst]->funcs->set_abm_event)
+			result = res_pool->multiple_abms[inst]->funcs->set_abm_event(
+				res_pool->multiple_abms[inst], full_screen, trans_info,
+					hdr_mode, scaling_enable, scaling_strength_map, inst);
+	} else {
+		if (res_pool->abm->funcs->set_abm_event)
+			result = res_pool->abm->funcs->set_abm_event(
+					res_pool->abm, full_screen, trans_info,
+					hdr_mode, scaling_enable, scaling_strength_map, inst);
+	}
+
+	return result;
+}
+
+bool dmub_set_abm_strength(struct resource_pool *res_pool,
+	unsigned int strength,
+	unsigned int inst)
+{
+	bool result = false;
+
+	if (res_pool->abm == NULL && res_pool->multiple_abms[inst] == NULL)
+		return false;
+
+	if (res_pool->multiple_abms[inst]) {
+		if (res_pool->multiple_abms[inst]->funcs->set_abm_level)
+			result = res_pool->multiple_abms[inst]->funcs->set_abm_level(
+				res_pool->multiple_abms[inst], strength);
+	} else {
+		if (res_pool->abm->funcs->set_abm_level)
+			result = res_pool->abm->funcs->set_abm_level(
+					res_pool->abm, strength);
+	}
+
+	return result;
+}
+
 bool dmcu_load_iram(struct dmcu *dmcu,
 	struct dmcu_iram_parameters params)
 {
diff --git a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c b/drivers/gpu/drm/amd/display/modules/power/power_helpers.c
deleted file mode 100644
index bf0c5901b4ee2..0000000000000
--- a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c
+++ /dev/null
@@ -1,39 +0,0 @@
-/* Copyright 2018 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include "power_helpers.h"
-#include "dc/inc/hw/dmcu.h"
-#include "dc/inc/hw/abm.h"
-#include "dc.h"
-#include "core_types.h"
-#include "dmub_cmd.h"
-
-#define DIV_ROUNDUP(a, b) (((a)+((b)/2))/(b))
-#define bswap16_based_on_endian(big_endian, value) \
-	((big_endian) ? cpu_to_be16(value) : cpu_to_le16(value))
-
-bool mod_power_only_edp(const struct dc_state *context, const struct dc_stream_state *stream)
-{
-	return context && context->stream_count == 1 && dc_is_embedded_signal(stream->signal);
-}
diff --git a/drivers/gpu/drm/amd/display/modules/power/power_helpers.h b/drivers/gpu/drm/amd/display/modules/power/power_helpers.h
index 600da3e331261..548c8ff6ddb46 100644
--- a/drivers/gpu/drm/amd/display/modules/power/power_helpers.h
+++ b/drivers/gpu/drm/amd/display/modules/power/power_helpers.h
@@ -146,6 +146,16 @@ bool dmub_init_abm_config(struct resource_pool *res_pool,
 		struct dmcu_iram_parameters params,
 		unsigned int inst);
 
+bool dmub_is_abm_supported(struct resource_pool *res_pool,
+		unsigned int inst);
+bool dmub_set_abm_event(struct resource_pool *res_pool,
+		unsigned int full_screen, unsigned int trans_info,
+		unsigned int hdr_mode, unsigned int scaling_enable,
+		unsigned int scaling_strength_map, unsigned int inst);
+bool dmub_set_abm_strength(struct resource_pool *res_pool,
+		unsigned int strength,
+		unsigned int inst);
+
 void init_replay_config(struct dc_link *link, struct replay_config *pr_config);
 void set_replay_coasting_vtotal(struct dc_link *link,
 	enum replay_coasting_vtotal_type type,
diff --git a/drivers/gpu/drm/amd/display/modules/power/power_psr.c b/drivers/gpu/drm/amd/display/modules/power/power_psr.c
index 5ecb570c204e1..0ad4c49246965 100644
--- a/drivers/gpu/drm/amd/display/modules/power/power_psr.c
+++ b/drivers/gpu/drm/amd/display/modules/power/power_psr.c
@@ -58,6 +58,13 @@ bool mod_power_psr_notify_mode_change(struct mod_power *mod_power,
 	// stream_index is passed as validated parameter
 	active_psr_events = core_power->map[stream_index].psr_events;
 
+	/* DMSS holds the panel in a forced PSR freeze (e.g. during HDR/SDR toggle).
+	 * Re-running edp_setup_psr would reprogram DPCD 0x170 and disturb the freeze,
+	 * so skip the PSR re-setup until DMSS releases the override.
+	 */
+	if (active_psr_events & psr_event_os_override_hold)
+		return false;
+
 	/* Calculate PSR configurations */
 	mod_power_calc_psr_configs(&psr_config, link, stream);
 
diff --git a/drivers/gpu/drm/amd/display/modules/power/power_replay.c b/drivers/gpu/drm/amd/display/modules/power/power_replay.c
index 983be9759e748..1ad2ee01d5600 100644
--- a/drivers/gpu/drm/amd/display/modules/power/power_replay.c
+++ b/drivers/gpu/drm/amd/display/modules/power/power_replay.c
@@ -175,11 +175,10 @@ static bool mod_power_update_replay_active_status(unsigned int active_replay_eve
 		if (link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS])
 			*coasting_vtotal =
 				link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS];
-		if (link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS]) {
-			ASSERT(link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS] <= 0xFFFF);
-			*frame_skip_number =
-				(uint16_t)link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS];
-		}
+
+		ASSERT(link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS] <= 0xFFFF);
+		*frame_skip_number =
+			(uint16_t)link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS];
 
 		/* During the ultra sleep mode testing, disable the timing sync in short vblank mode */
 		if (active_replay_events & (replay_event_test_harness_enable_replay)) {
@@ -806,6 +805,13 @@ void mod_power_replay_notify_mode_change(struct mod_power *mod_power,
 	core_power = MOD_POWER_TO_CORE(mod_power);
 	active_replay_events = core_power->map[stream_index].replay_events;
 
+	/* DMSS holds the panel in a forced freeze (e.g. during HDR/SDR toggle).
+	 * Re-running dp_setup_replay would reprogram DPCD 0x37B and disturb the
+	 * freeze, so skip the replay re-setup until DMSS releases the override.
+	 */
+	if (active_replay_events & replay_event_os_override_hold)
+		return;
+
 	link->replay_settings.replay_smu_opt_enable =
 		(link->replay_settings.config.replay_smu_opt_supported &&
 		mod_power_only_edp(dc->current_state, stream));
diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
index 3fd38323a88ba..6d0c74ebdb58f 100644
--- a/drivers/gpu/drm/amd/include/amd_shared.h
+++ b/drivers/gpu/drm/amd/include/amd_shared.h
@@ -439,10 +439,7 @@ enum amd_dpm_forced_level;
  * @complete: handles IP specific changes after resume
  * @is_idle: returns current IP block idle status
  * @wait_for_idle: poll for idle
- * @check_soft_reset: check soft reset the IP block
- * @pre_soft_reset: pre soft reset the IP block
  * @soft_reset: soft reset the IP block
- * @post_soft_reset: post soft reset the IP block
  * @set_clockgating_state: enable/disable cg for the IP block
  * @set_powergating_state: enable/disable pg for the IP block
  * @get_clockgating_state: get current clockgating status
@@ -471,10 +468,7 @@ struct amd_ip_funcs {
 	void (*complete)(struct amdgpu_ip_block *ip_block);
 	bool (*is_idle)(struct amdgpu_ip_block *ip_block);
 	int (*wait_for_idle)(struct amdgpu_ip_block *ip_block);
-	bool (*check_soft_reset)(struct amdgpu_ip_block *ip_block);
-	int (*pre_soft_reset)(struct amdgpu_ip_block *ip_block);
 	int (*soft_reset)(struct amdgpu_ip_block *ip_block);
-	int (*post_soft_reset)(struct amdgpu_ip_block *ip_block);
 	int (*set_clockgating_state)(struct amdgpu_ip_block *ip_block,
 				     enum amd_clockgating_state state);
 	int (*set_powergating_state)(struct amdgpu_ip_block *ip_block,
diff --git a/drivers/gpu/drm/amd/include/asic_reg/sdma/sdma_4_4_2_offset.h b/drivers/gpu/drm/amd/include/asic_reg/sdma/sdma_4_4_2_offset.h
index ead81aeffd679..11c32e4274fa5 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/sdma/sdma_4_4_2_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/sdma/sdma_4_4_2_offset.h
@@ -493,6 +493,10 @@
 #define regSDMA_RLC0_MIDCMD_DATA10_BASE_IDX                                                             0
 #define regSDMA_RLC0_MIDCMD_CNTL                                                                        0x017b
 #define regSDMA_RLC0_MIDCMD_CNTL_BASE_IDX                                                               0
+#define regSDMA_RLC0_UTILIZATION_LO                                                                     0x017c
+#define regSDMA_RLC0_UTILIZATION_LO_BASE_IDX                                                            0
+#define regSDMA_RLC0_UTILIZATION_HI                                                                     0x017d
+#define regSDMA_RLC0_UTILIZATION_HI_BASE_IDX                                                            0
 #define regSDMA_RLC1_RB_CNTL                                                                            0x0188
 #define regSDMA_RLC1_RB_CNTL_BASE_IDX                                                                   0
 #define regSDMA_RLC1_RB_BASE                                                                            0x0189
diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
index 44e225e097d0c..965b50c8ca303 100644
--- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
@@ -339,6 +339,9 @@ struct kfd2kgd_calls {
 			     uint32_t *ptl_state,
 			     enum amdgpu_ptl_fmt *fmt1,
 			     enum amdgpu_ptl_fmt *fmt2);
+	int (*hqd_sdma_get_counter)(struct amdgpu_device *adev,
+				    void *mqd, uint32_t num_sdma_queues_per_eng,
+				    uint64_t *val);
 };
 
 #endif	/* KGD_KFD_INTERFACE_H_INCLUDED */
diff --git a/drivers/gpu/drm/amd/include/mes_v11_api_def.h b/drivers/gpu/drm/amd/include/mes_v11_api_def.h
index 7808147ada38c..a8e6ccc7817c7 100644
--- a/drivers/gpu/drm/amd/include/mes_v11_api_def.h
+++ b/drivers/gpu/drm/amd/include/mes_v11_api_def.h
@@ -238,8 +238,9 @@ union MESAPI_SET_HW_RESOURCES {
 				uint32_t enable_mes_sch_stb_log : 1;
 				uint32_t limit_single_process : 1;
 				uint32_t is_strix_tmz_wa_enabled  :1;
-				uint32_t enable_lr_compute_wa : 1;
-				uint32_t reserved : 12;
+				uint32_t enable_lr_compute_wa : 2;
+				uint32_t enable_compute_pipe_reset : 1;
+				uint32_t reserved : 10;
 			};
 			uint32_t	uint32_t_all;
 		};
@@ -328,6 +329,7 @@ union MESAPI__ADD_QUEUE {
 		uint32_t                    pipe_id;
 		uint32_t                    queue_id;
 		uint32_t                    alignment_mode_setting;
+		uint32_t                    full_sh_mem_config_data;
 		uint64_t                    unmap_flag_addr;
 	};
 
@@ -357,6 +359,8 @@ union MESAPI__REMOVE_QUEUE {
 		uint32_t                    tf_data;
 
 		enum MES_QUEUE_TYPE         queue_type;
+		uint64_t                    timestamp;
+		uint32_t                    gang_context_array_index;
 	};
 
 	uint32_t	max_dwords_in_api[API_FRAME_SIZE_IN_DWORDS];
@@ -515,14 +519,50 @@ union MESAPI__SET_LOGGING_BUFFER {
 	uint32_t	max_dwords_in_api[API_FRAME_SIZE_IN_DWORDS];
 };
 
+enum MES_API_QUERY_MES_OPCODE {
+	MES_API_QUERY_MES__GET_CTX_ARRAY_SIZE,
+	MES_API_QUERY_MES__GET_CAPS = MES_API_QUERY_MES__GET_CTX_ARRAY_SIZE,
+	MES_API_QUERY_MES__CHECK_HEALTHY,
+	MES_API_QUERY_MES__MAX,
+};
+
+enum { QUERY_MES_MAX_SIZE_IN_DWORDS = 20 };
+
+/**
+ * Obsolete
+ * to be removed once KMD stopped refering to it.
+ * use MES_API_QUERY_MES__CAPS instead
+*/
+struct MES_API_QUERY_MES__CTX_ARRAY_SIZE {
+    uint64_t    proc_ctx_array_size_addr;
+    uint64_t    gang_ctx_array_size_addr;
+};
+
+struct MES_API_QUERY_MES__HEALTHY_CHECK {
+    uint64_t    healthy_addr;
+};
+
+struct MES_API_QUERY_MES__CAPS {
+    uint64_t    proc_ctx_array_size_addr;
+    uint64_t    gang_ctx_array_size_addr;
+    uint64_t    features_enablement_addr;
+};
+
 union MESAPI__QUERY_MES_STATUS {
 	struct {
-		union MES_API_HEADER	header;
-		bool			mes_healthy; /* 0 - not healthy, 1 - healthy */
-		struct MES_API_STATUS	api_status;
+		union MES_API_HEADER            header;
+		enum MES_API_QUERY_MES_OPCODE   subopcode;
+		struct MES_API_STATUS           api_status;
+		uint64_t                        timestamp;
+		union {
+			struct MES_API_QUERY_MES__CTX_ARRAY_SIZE    ctx_array_size;
+			struct MES_API_QUERY_MES__CAPS              caps;
+			struct MES_API_QUERY_MES__HEALTHY_CHECK     healthy_check;
+			uint32_t data[QUERY_MES_MAX_SIZE_IN_DWORDS];
+		};
 	};
 
-	uint32_t	max_dwords_in_api[API_FRAME_SIZE_IN_DWORDS];
+	uint32_t max_dwords_in_api[API_FRAME_SIZE_IN_DWORDS];
 };
 
 union MESAPI__PROGRAM_GDS {
diff --git a/drivers/gpu/drm/amd/include/mes_v12_api_def.h b/drivers/gpu/drm/amd/include/mes_v12_api_def.h
index e541a43714a10..cb7ebdfffeeb7 100644
--- a/drivers/gpu/drm/amd/include/mes_v12_api_def.h
+++ b/drivers/gpu/drm/amd/include/mes_v12_api_def.h
@@ -294,8 +294,9 @@ union MESAPI_SET_HW_RESOURCES {
 				uint32_t limit_single_process : 1;
 				uint32_t unmapped_doorbell_handling: 2;
 				uint32_t enable_mes_fence_int: 1;
-				uint32_t enable_lr_compute_wa : 1;
-				uint32_t reserved : 9;
+				uint32_t enable_lr_compute_wa : 2;
+				uint32_t enable_compute_pipe_reset : 1;
+				uint32_t reserved : 7;
 			};
 			uint32_t uint32_all;
 		};
diff --git a/drivers/gpu/drm/amd/include/soc15_hw_ip.h b/drivers/gpu/drm/amd/include/soc15_hw_ip.h
index a20e59584dde7..60f588dd01304 100644
--- a/drivers/gpu/drm/amd/include/soc15_hw_ip.h
+++ b/drivers/gpu/drm/amd/include/soc15_hw_ip.h
@@ -44,6 +44,7 @@
 #define SDPMUX_HWID                                       19
 #define NTB_HWID                                          20
 #define VPE_HWID                                          21
+#define UMSCH_HWID                                        22
 #define IOHC_HWID                                         24
 #define L2IMU_HWID                                        28
 #define VCE_HWID                                          32
diff --git a/drivers/gpu/drm/amd/include/v9_structs.h b/drivers/gpu/drm/amd/include/v9_structs.h
index a2f81b9c38afc..e0d387f085765 100644
--- a/drivers/gpu/drm/amd/include/v9_structs.h
+++ b/drivers/gpu/drm/amd/include/v9_structs.h
@@ -69,8 +69,8 @@ struct v9_sdma_mqd {
 	uint32_t sdmax_rlcx_midcmd_cntl;
 	uint32_t reserved_42;
 	uint32_t reserved_43;
-	uint32_t reserved_44;
-	uint32_t reserved_45;
+	uint32_t sdmax_rlcx_utilization_lo;
+	uint32_t sdmax_rlcx_utilization_hi;
 	uint32_t reserved_46;
 	uint32_t reserved_47;
 	uint32_t reserved_48;
diff --git a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
index f76ba67535519..c09c60ad3f384 100644
--- a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
+++ b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
@@ -2110,10 +2110,3 @@ ssize_t amdgpu_dpm_get_xcp_metrics(struct amdgpu_device *adev, int xcp_id,
 
 	return ret;
 }
-
-const struct ras_smu_drv *amdgpu_dpm_get_ras_smu_driver(struct amdgpu_device *adev)
-{
-	void *pp_handle = adev->powerplay.pp_handle;
-
-	return smu_get_ras_smu_driver(pp_handle);
-}
diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
index 97da01aff76c7..f5a5d72b41082 100644
--- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
@@ -100,6 +100,37 @@ const char * const amdgpu_pp_profile_name[] = {
 	"UNCAPPED",
 };
 
+static int amdgpu_pm_parse_long_params(char *str, long *params,
+				       uint32_t max_params,
+				       uint32_t *num_params)
+{
+	const char delimiter[] = { ' ', '\n', '\0' };
+	uint32_t count = 0;
+	char *sub_str;
+	int ret;
+
+	if (!params || !num_params)
+		return -EINVAL;
+
+	while ((sub_str = strsep(&str, delimiter)) != NULL) {
+		if (strlen(sub_str) == 0)
+			continue;
+		if (count >= max_params)
+			return -EINVAL;
+		ret = kstrtol(sub_str, 0, &params[count]);
+		if (ret)
+			return -EINVAL;
+		count++;
+		if (!str)
+			break;
+		while (isspace(*str))
+			str++;
+	}
+	*num_params = count;
+
+	return 0;
+}
+
 /**
  * amdgpu_pm_dev_state_check - Check if device can be accessed.
  * @adev: Target device.
@@ -769,8 +800,6 @@ static ssize_t amdgpu_set_pp_od_clk_voltage(struct device *dev,
 	long parameter[64];
 	char buf_cpy[128];
 	char *tmp_str;
-	char *sub_str;
-	const char delimiter[3] = {' ', '\n', '\0'};
 	uint32_t type;
 
 	if (count > 127 || count == 0)
@@ -805,22 +834,10 @@ static ssize_t amdgpu_set_pp_od_clk_voltage(struct device *dev,
 		tmp_str++;
 	while (isspace(*++tmp_str));
 
-	while ((sub_str = strsep(&tmp_str, delimiter)) != NULL) {
-		if (strlen(sub_str) == 0)
-			continue;
-		if (parameter_size >= ARRAY_SIZE(parameter))
-			return -EINVAL;
-		ret = kstrtol(sub_str, 0, &parameter[parameter_size]);
-		if (ret)
-			return -EINVAL;
-		parameter_size++;
-
-		if (!tmp_str)
-			break;
-
-		while (isspace(*tmp_str))
-			tmp_str++;
-	}
+	ret = amdgpu_pm_parse_long_params(
+		tmp_str, parameter, ARRAY_SIZE(parameter), &parameter_size);
+	if (ret)
+		return ret;
 
 	ret = amdgpu_pm_get_access(adev);
 	if (ret < 0)
@@ -1393,11 +1410,9 @@ static ssize_t amdgpu_set_pp_power_profile_mode(struct device *dev,
 	struct amdgpu_device *adev = drm_to_adev(ddev);
 	uint32_t parameter_size = 0;
 	long parameter[64];
-	char *sub_str, buf_cpy[128];
-	char *tmp_str;
+	char buf_cpy[128];
 	char tmp[2];
 	long int profile_mode = 0;
-	const char delimiter[3] = {' ', '\n', '\0'};
 
 	/* Reject empty/whitespace strings - fuzzing found this is not validated */
 	if (count == 0 || sysfs_streq(buf, ""))
@@ -1415,19 +1430,11 @@ static ssize_t amdgpu_set_pp_power_profile_mode(struct device *dev,
 		while (isspace(*buf))
 			buf++;
 		strscpy(buf_cpy, buf, sizeof(buf_cpy));
-		tmp_str = buf_cpy;
-		while ((sub_str = strsep(&tmp_str, delimiter)) != NULL) {
-			if (strlen(sub_str) == 0)
-				continue;
-			ret = kstrtol(sub_str, 0, &parameter[parameter_size]);
-			if (ret)
-				return -EINVAL;
-			parameter_size++;
-			if (!tmp_str)
-				break;
-			while (isspace(*tmp_str))
-				tmp_str++;
-		}
+		ret = amdgpu_pm_parse_long_params(buf_cpy, parameter,
+						  ARRAY_SIZE(parameter) - 1,
+						  &parameter_size);
+		if (ret)
+			return ret;
 	}
 	parameter[parameter_size] = profile_mode;
 
@@ -3958,18 +3965,14 @@ static int amdgpu_retrieve_od_settings(struct amdgpu_device *adev,
 	return size;
 }
 
-static int parse_input_od_command_lines(const char *buf,
-					size_t count,
-					u32 *type,
-					long *params,
-					size_t params_max,
+static int parse_input_od_command_lines(const char *buf, size_t count,
+					u32 *type, long *params,
+					uint32_t max_params,
 					uint32_t *num_of_params)
 {
-	const char delimiter[3] = {' ', '\n', '\0'};
 	uint32_t parameter_size = 0;
 	char buf_cpy[128] = {0};
-	char *tmp_str, *sub_str;
-	int ret;
+	char *tmp_str;
 
 	if (count > sizeof(buf_cpy) - 1)
 		return -EINVAL;
@@ -3994,28 +3997,8 @@ static int parse_input_od_command_lines(const char *buf,
 		break;
 	}
 
-	while ((sub_str = strsep(&tmp_str, delimiter)) != NULL) {
-		if (strlen(sub_str) == 0)
-			continue;
-
-		if (parameter_size >= params_max)
-			return -EINVAL;
-
-		ret = kstrtol(sub_str, 0, &params[parameter_size]);
-		if (ret)
-			return -EINVAL;
-		parameter_size++;
-
-		if (!tmp_str)
-			break;
-
-		while (isspace(*tmp_str))
-			tmp_str++;
-	}
-
-	*num_of_params = parameter_size;
-
-	return 0;
+	return amdgpu_pm_parse_long_params(tmp_str, params, max_params,
+					   num_of_params);
 }
 
 static int
@@ -4028,10 +4011,7 @@ amdgpu_distribute_custom_od_settings(struct amdgpu_device *adev,
 	long parameter[64];
 	int ret;
 
-	ret = parse_input_od_command_lines(in_buf,
-					   count,
-					   &cmd_type,
-					   parameter,
+	ret = parse_input_od_command_lines(in_buf, count, &cmd_type, parameter,
 					   ARRAY_SIZE(parameter),
 					   &parameter_size);
 	if (ret)
diff --git a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
index aa3f427819a09..c7ea293856827 100644
--- a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
@@ -612,6 +612,5 @@ int amdgpu_dpm_reset_vcn(struct amdgpu_device *adev, uint32_t inst_mask);
 bool amdgpu_dpm_reset_vcn_is_supported(struct amdgpu_device *adev);
 bool amdgpu_dpm_is_temp_metrics_supported(struct amdgpu_device *adev,
 					  enum smu_temp_metric_type type);
-const struct ras_smu_drv *amdgpu_dpm_get_ras_smu_driver(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c
index 8079da7c53350..012227c706001 100644
--- a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c
+++ b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c
@@ -3892,13 +3892,16 @@ static void si_notify_hw_of_powersource(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	/* Check if the platform already manages the AC/DC switch via dedicated GPIO. */
-	if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
-		return;
-
-	/* The SMU automatically notices DC, but needs to be notified when switching to AC. */
-	if (adev->pm.ac_power)
+	/*
+	 * Check if the platform already manages the AC/DC switch via dedicated GPIO.
+	 * Otherwise SMU automatically notices DC, but needs to be notified of AC.
+	 */
+	if (adev->pm.ac_power &&
+	    (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC))
 		amdgpu_si_send_msg_to_smc(adev, PPSMC_MSG_RunningOnAC);
+
+	/* Recompute clocks with updated max_limits. */
+	amdgpu_legacy_dpm_compute_clocks(adev);
 }
 
 static PPSMC_Result si_send_msg_to_smc_with_parameter(struct amdgpu_device *adev,
@@ -7689,7 +7692,7 @@ static int si_dpm_process_interrupt(struct amdgpu_device *adev,
 		break;
 	}
 
-	if (queue_thermal)
+	if (queue_thermal && amdgpu_dpm)
 		schedule_work(&adev->pm.dpm.thermal.work);
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
index 6f5c27bdc1e91..7c70e228a5bae 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
@@ -660,25 +660,20 @@ static int amd_powerplay_reset(void *handle)
 static int pp_dpm_set_pp_table(void *handle, const char *buf, size_t size)
 {
 	struct pp_hwmgr *hwmgr = handle;
+	void *hardcode_pp_table;
 	int ret = -ENOMEM;
 
-	if (!hwmgr || !hwmgr->pm_en)
-		return -EINVAL;
-
-	if (size > hwmgr->soft_pp_table_size)
+	if (!hwmgr || !hwmgr->pm_en || !buf || !size || size > U32_MAX)
 		return -EINVAL;
 
-	if (!hwmgr->hardcode_pp_table) {
-		hwmgr->hardcode_pp_table = kmemdup(hwmgr->soft_pp_table,
-						   hwmgr->soft_pp_table_size,
-						   GFP_KERNEL);
-		if (!hwmgr->hardcode_pp_table)
-			return ret;
-	}
-
-	memcpy(hwmgr->hardcode_pp_table, buf, size);
+	hardcode_pp_table = kmemdup(buf, size, GFP_KERNEL);
+	if (!hardcode_pp_table)
+		return ret;
 
+	kfree(hwmgr->hardcode_pp_table);
+	hwmgr->hardcode_pp_table = hardcode_pp_table;
 	hwmgr->soft_pp_table = hwmgr->hardcode_pp_table;
+	hwmgr->soft_pp_table_size = size;
 
 	ret = amd_powerplay_reset(handle);
 	if (ret)
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c
index 1d6e30269d567..4d553be56396f 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c
@@ -106,11 +106,8 @@ int hwmgr_early_init(struct pp_hwmgr *hwmgr)
 		hwmgr->od_enabled = false;
 		switch (hwmgr->chip_id) {
 		case CHIP_BONAIRE:
-			/* R9 M380 in iMac 2015: SMU hangs when enabling MCLK DPM
-			 * R7 260X cards with old MC ucode: MCLK DPM is unstable
-			 */
-			if (adev->pdev->subsystem_vendor == 0x106B ||
-			    adev->pdev->device == 0x6658) {
+			/* R9 M380 in iMac 2015: SMU hangs when enabling MCLK DPM */
+			if (adev->pdev->subsystem_vendor == 0x106B) {
 				dev_info(adev->dev, "disabling MCLK DPM on quirky ASIC");
 				adev->pm.pp_feature &= ~PP_MCLK_DPM_MASK;
 				hwmgr->feature_mask &= ~PP_MCLK_DPM_MASK;
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c
index ce166a7f8e420..4b796d60b03d4 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c
@@ -46,16 +46,22 @@ union voltage_object_info {
 static int atomctrl_retrieve_ac_timing(
 		uint8_t index,
 		ATOM_INIT_REG_BLOCK *reg_block,
+		u8 *table_end,
 		pp_atomctrl_mc_reg_table *table)
 {
 	uint32_t i, j;
+	u16 stride = le16_to_cpu(reg_block->usRegDataBlkSize);
 	uint8_t tmem_id;
 	ATOM_MEMORY_SETTING_DATA_BLOCK *reg_data = (ATOM_MEMORY_SETTING_DATA_BLOCK *)
 		((uint8_t *)reg_block + (2 * sizeof(uint16_t)) + le16_to_cpu(reg_block->usRegIndexTblSize));
 
 	uint8_t num_ranges = 0;
 
-	while (*(uint32_t *)reg_data != END_OF_REG_DATA_BLOCK &&
+	if (stride < sizeof(uint32_t))
+		return -EINVAL;
+
+	while ((uint8_t *)reg_data + sizeof(uint32_t) <= table_end &&
+	       *(uint32_t *)reg_data != END_OF_REG_DATA_BLOCK &&
 			num_ranges < VBIOS_MAX_AC_TIMING_ENTRIES) {
 		tmem_id = (uint8_t)((*(uint32_t *)reg_data & MEM_ID_MASK) >> MEM_ID_SHIFT);
 
@@ -67,6 +73,10 @@ static int atomctrl_retrieve_ac_timing(
 			for (i = 0, j = 1; i < table->last; i++) {
 				if ((table->mc_reg_address[i].uc_pre_reg_data &
 							LOW_NIBBLE_MASK) == DATA_FROM_TABLE) {
+					if ((uint8_t *)reg_data +
+					    (j + 1) * sizeof(uint32_t) > table_end)
+						return -EINVAL;
+
 					table->mc_reg_table_entry[num_ranges].mc_data[i] =
 						(uint32_t)*((uint32_t *)reg_data + j);
 					j++;
@@ -81,11 +91,13 @@ static int atomctrl_retrieve_ac_timing(
 		}
 
 		reg_data = (ATOM_MEMORY_SETTING_DATA_BLOCK *)
-			((uint8_t *)reg_data + le16_to_cpu(reg_block->usRegDataBlkSize)) ;
+			((uint8_t *)reg_data + stride);
 	}
 
-	PP_ASSERT_WITH_CODE((*(uint32_t *)reg_data == END_OF_REG_DATA_BLOCK),
-			"Invalid VramInfo table.", return -1);
+	if ((uint8_t *)reg_data + sizeof(uint32_t) > table_end ||
+	    *(uint32_t *)reg_data != END_OF_REG_DATA_BLOCK)
+		return -EINVAL;
+
 	table->num_entries = num_ranges;
 
 	return 0;
@@ -136,6 +148,7 @@ int atomctrl_initialize_mc_reg_table(
 {
 	ATOM_VRAM_INFO_HEADER_V2_1 *vram_info;
 	ATOM_INIT_REG_BLOCK *reg_block;
+	u8 *table_end;
 	int result = 0;
 	u8 frev, crev;
 	u16 size;
@@ -157,6 +170,7 @@ int atomctrl_initialize_mc_reg_table(
 	}
 
 	if (0 == result) {
+		table_end = (uint8_t *)vram_info + size;
 		reg_block = (ATOM_INIT_REG_BLOCK *)
 			((uint8_t *)vram_info + le16_to_cpu(vram_info->usMemClkPatchTblOffset));
 		result = atomctrl_set_mc_reg_address_table(reg_block, table);
@@ -164,7 +178,7 @@ int atomctrl_initialize_mc_reg_table(
 
 	if (0 == result) {
 		result = atomctrl_retrieve_ac_timing(module_index,
-					reg_block, table);
+					reg_block, table_end, table);
 	}
 
 	return result;
@@ -177,6 +191,7 @@ int atomctrl_initialize_mc_reg_table_v2_2(
 {
 	ATOM_VRAM_INFO_HEADER_V2_2 *vram_info;
 	ATOM_INIT_REG_BLOCK *reg_block;
+	u8 *table_end;
 	int result = 0;
 	u8 frev, crev;
 	u16 size;
@@ -198,6 +213,7 @@ int atomctrl_initialize_mc_reg_table_v2_2(
 	}
 
 	if (0 == result) {
+		table_end = (uint8_t *)vram_info + size;
 		reg_block = (ATOM_INIT_REG_BLOCK *)
 			((uint8_t *)vram_info + le16_to_cpu(vram_info->usMemClkPatchTblOffset));
 		result = atomctrl_set_mc_reg_address_table(reg_block, table);
@@ -205,7 +221,7 @@ int atomctrl_initialize_mc_reg_table_v2_2(
 
 	if (0 == result) {
 		result = atomctrl_retrieve_ac_timing(module_index,
-					reg_block, table);
+					reg_block, table_end, table);
 	}
 
 	return result;
@@ -268,15 +284,21 @@ static const ATOM_VOLTAGE_OBJECT_V3 *atomctrl_lookup_voltage_type_v3(
 	unsigned int offset = offsetof(ATOM_VOLTAGE_OBJECT_INFO_V3_1, asVoltageObj[0]);
 	uint8_t *start = (uint8_t *)voltage_object_info_table;
 
-	while (offset < size) {
+	while (offset + sizeof(ATOM_VOLTAGE_OBJECT_HEADER_V3) <= size) {
 		const ATOM_VOLTAGE_OBJECT_V3 *voltage_object =
 			(const ATOM_VOLTAGE_OBJECT_V3 *)(start + offset);
+		u16 obj_size;
+
+		obj_size = le16_to_cpu(voltage_object->asGpioVoltageObj.sHeader.usSize);
+		if (obj_size < sizeof(voltage_object->asGpioVoltageObj.sHeader) ||
+		    offset + obj_size > size)
+			break;
 
 		if (voltage_type == voltage_object->asGpioVoltageObj.sHeader.ucVoltageType &&
 			voltage_mode == voltage_object->asGpioVoltageObj.sHeader.ucVoltageMode)
 			return voltage_object;
 
-		offset += le16_to_cpu(voltage_object->asGpioVoltageObj.sHeader.usSize);
+		offset += obj_size;
 	}
 
 	return NULL;
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomfwctrl.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomfwctrl.c
index 6120f14caab08..69aee8661d1e5 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomfwctrl.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomfwctrl.c
@@ -36,16 +36,21 @@ static const union atom_voltage_object_v4 *pp_atomfwctrl_lookup_voltage_type_v4(
 			offsetof(struct atom_voltage_objects_info_v4_1, voltage_object[0]);
 	unsigned long start = (unsigned long)voltage_object_info_table;
 
-	while (offset < size) {
+	while (offset + sizeof(struct atom_voltage_object_header_v4) <= size) {
 		const union atom_voltage_object_v4 *voltage_object =
 			(const union atom_voltage_object_v4 *)(start + offset);
+		u16 obj_size;
+
+		obj_size = le16_to_cpu(voltage_object->gpio_voltage_obj.header.object_size);
+		if (obj_size < sizeof(voltage_object->gpio_voltage_obj.header) ||
+		    offset + obj_size > size)
+			break;
 
 		if (voltage_type == voltage_object->gpio_voltage_obj.header.voltage_type &&
 		    voltage_mode == voltage_object->gpio_voltage_obj.header.voltage_mode)
 			return voltage_object;
 
-		offset += le16_to_cpu(voltage_object->gpio_voltage_obj.header.object_size);
-
+		offset += obj_size;
 	}
 
 	return NULL;
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/process_pptables_v1_0.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/process_pptables_v1_0.c
index 6fcca65bd7d4d..c5673077c8955 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/process_pptables_v1_0.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/process_pptables_v1_0.c
@@ -150,6 +150,368 @@ static const void *get_powerplay_table(struct pp_hwmgr *hwmgr)
 	return table_address;
 }
 
+static bool tonga_pp_table_has_space(struct pp_hwmgr *hwmgr, size_t offset,
+				     size_t size)
+{
+	size_t table_size = hwmgr->soft_pp_table_size;
+
+	return offset <= table_size && size <= table_size - offset;
+}
+
+static int get_tonga_subtable(struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table,
+		u16 table_offset, size_t table_size, const void **table)
+{
+	PP_ASSERT_WITH_CODE((table_offset != 0),
+			    "Invalid PowerPlay Table!", return -1);
+	PP_ASSERT_WITH_CODE((tonga_pp_table_has_space(hwmgr, table_offset,
+						      table_size)),
+			    "Invalid PowerPlay Table!", return -1);
+
+	*table = (const void *)(((unsigned long)powerplay_table) + table_offset);
+
+	return 0;
+}
+
+static int validate_tonga_table_entries(struct pp_hwmgr *hwmgr,
+		u16 table_offset, size_t entries_offset,
+		u8 num_entries, size_t entry_size)
+{
+	size_t table_size;
+
+	PP_ASSERT_WITH_CODE((num_entries != 0),
+			    "Invalid PowerPlay Table!", return -1);
+
+	table_size = entries_offset + num_entries * entry_size;
+	PP_ASSERT_WITH_CODE((tonga_pp_table_has_space(hwmgr, table_offset,
+						      table_size)),
+			    "Invalid PowerPlay Table!", return -1);
+
+	return 0;
+}
+
+static int get_tonga_voltage_lookup_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table,
+		u16 table_offset, uint32_t max_levels,
+		const ATOM_Tonga_Voltage_Lookup_Table **lookup_table)
+{
+	const ATOM_Tonga_Voltage_Lookup_Table *table;
+	size_t table_size;
+	int ret;
+
+	ret = get_tonga_subtable(hwmgr, powerplay_table, table_offset,
+				 sizeof(*table), (const void **)&table);
+	if (ret)
+		return ret;
+
+	PP_ASSERT_WITH_CODE((table->ucNumEntries != 0 &&
+			     table->ucNumEntries <= max_levels),
+			    "Invalid PowerPlay Table!", return -1);
+
+	table_size = offsetof(ATOM_Tonga_Voltage_Lookup_Table, entries) +
+		table->ucNumEntries * sizeof(ATOM_Tonga_Voltage_Lookup_Record);
+	PP_ASSERT_WITH_CODE((tonga_pp_table_has_space(hwmgr, table_offset,
+						      table_size)),
+			    "Invalid PowerPlay Table!", return -1);
+
+	*lookup_table = table;
+
+	return 0;
+}
+
+static int get_tonga_mclk_dependency_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table,
+		const ATOM_Tonga_MCLK_Dependency_Table **mclk_dep_table)
+{
+	const ATOM_Tonga_MCLK_Dependency_Table *table;
+	u16 table_offset;
+	int ret;
+
+	table_offset = le16_to_cpu(powerplay_table->usMclkDependencyTableOffset);
+	ret = get_tonga_subtable(hwmgr, powerplay_table, table_offset,
+				 sizeof(*table), (const void **)&table);
+	if (ret)
+		return ret;
+
+	ret = validate_tonga_table_entries(hwmgr, table_offset,
+					   offsetof(ATOM_Tonga_MCLK_Dependency_Table,
+						    entries),
+					   table->ucNumEntries,
+					   sizeof(ATOM_Tonga_MCLK_Dependency_Record));
+	if (ret)
+		return ret;
+
+	*mclk_dep_table = table;
+
+	return 0;
+}
+
+static int get_tonga_mm_dependency_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table,
+		const ATOM_Tonga_MM_Dependency_Table **mm_dep_table)
+{
+	const ATOM_Tonga_MM_Dependency_Table *table;
+	u16 table_offset;
+	int ret;
+
+	table_offset = le16_to_cpu(powerplay_table->usMMDependencyTableOffset);
+	ret = get_tonga_subtable(hwmgr, powerplay_table, table_offset,
+				 sizeof(*table), (const void **)&table);
+	if (ret)
+		return ret;
+
+	ret = validate_tonga_table_entries(hwmgr, table_offset,
+					   offsetof(ATOM_Tonga_MM_Dependency_Table,
+						    entries),
+					   table->ucNumEntries,
+					   sizeof(ATOM_Tonga_MM_Dependency_Record));
+	if (ret)
+		return ret;
+
+	*mm_dep_table = table;
+
+	return 0;
+}
+
+static int get_tonga_sclk_dependency_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table,
+		const PPTable_Generic_SubTable_Header **sclk_dep_table)
+{
+	const PPTable_Generic_SubTable_Header *header;
+	u16 table_offset;
+	size_t entries_offset;
+	size_t entry_size;
+	u8 num_entries;
+	int ret;
+
+	table_offset = le16_to_cpu(powerplay_table->usSclkDependencyTableOffset);
+	ret = get_tonga_subtable(hwmgr, powerplay_table, table_offset,
+				 sizeof(*header), (const void **)&header);
+	if (ret)
+		return ret;
+
+	if (header->ucRevId < 1) {
+		const ATOM_Tonga_SCLK_Dependency_Table *table =
+			(const ATOM_Tonga_SCLK_Dependency_Table *)header;
+
+		entries_offset = offsetof(ATOM_Tonga_SCLK_Dependency_Table, entries);
+		entry_size = sizeof(ATOM_Tonga_SCLK_Dependency_Record);
+		num_entries = table->ucNumEntries;
+	} else {
+		const ATOM_Polaris_SCLK_Dependency_Table *table =
+			(const ATOM_Polaris_SCLK_Dependency_Table *)header;
+
+		entries_offset = offsetof(ATOM_Polaris_SCLK_Dependency_Table, entries);
+		entry_size = sizeof(ATOM_Polaris_SCLK_Dependency_Record);
+		num_entries = table->ucNumEntries;
+	}
+
+	ret = validate_tonga_table_entries(hwmgr, table_offset, entries_offset,
+					   num_entries, entry_size);
+	if (ret)
+		return ret;
+
+	*sclk_dep_table = header;
+
+	return 0;
+}
+
+static int get_tonga_pcie_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table,
+		const PPTable_Generic_SubTable_Header **pcie_table)
+{
+	const PPTable_Generic_SubTable_Header *header;
+	u16 table_offset;
+	size_t entries_offset;
+	size_t entry_size;
+	u8 num_entries;
+	int ret;
+
+	table_offset = le16_to_cpu(powerplay_table->usPCIETableOffset);
+	ret = get_tonga_subtable(hwmgr, powerplay_table, table_offset,
+				 sizeof(*header), (const void **)&header);
+	if (ret)
+		return ret;
+
+	if (header->ucRevId < 1) {
+		const ATOM_Tonga_PCIE_Table *table =
+			(const ATOM_Tonga_PCIE_Table *)header;
+
+		entries_offset = offsetof(ATOM_Tonga_PCIE_Table, entries);
+		entry_size = sizeof(ATOM_Tonga_PCIE_Record);
+		num_entries = table->ucNumEntries;
+	} else {
+		const ATOM_Polaris10_PCIE_Table *table =
+			(const ATOM_Polaris10_PCIE_Table *)header;
+
+		entries_offset = offsetof(ATOM_Polaris10_PCIE_Table, entries);
+		entry_size = sizeof(ATOM_Polaris10_PCIE_Record);
+		num_entries = table->ucNumEntries;
+	}
+
+	ret = validate_tonga_table_entries(hwmgr, table_offset, entries_offset,
+					   num_entries, entry_size);
+	if (ret)
+		return ret;
+
+	*pcie_table = header;
+
+	return 0;
+}
+
+static int get_tonga_hard_limit_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table,
+		const ATOM_Tonga_Hard_Limit_Table **hard_limit_table)
+{
+	const ATOM_Tonga_Hard_Limit_Table *table;
+	u16 table_offset;
+	int ret;
+
+	table_offset = le16_to_cpu(powerplay_table->usHardLimitTableOffset);
+	ret = get_tonga_subtable(hwmgr, powerplay_table, table_offset,
+				 sizeof(*table), (const void **)&table);
+	if (ret)
+		return ret;
+
+	ret = validate_tonga_table_entries(hwmgr, table_offset,
+					   offsetof(ATOM_Tonga_Hard_Limit_Table,
+						    entries),
+					   table->ucNumEntries,
+					   sizeof(ATOM_Tonga_Hard_Limit_Record));
+	if (ret)
+		return ret;
+
+	*hard_limit_table = table;
+
+	return 0;
+}
+
+static int get_tonga_thermal_controller_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table,
+		const ATOM_Tonga_Thermal_Controller **thermal_controller)
+{
+	u16 table_offset;
+
+	table_offset = le16_to_cpu(powerplay_table->usThermalControllerOffset);
+
+	return get_tonga_subtable(hwmgr, powerplay_table, table_offset,
+				  sizeof(**thermal_controller),
+				  (const void **)thermal_controller);
+}
+
+static int get_tonga_fan_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table,
+		const PPTable_Generic_SubTable_Header **fan_table)
+{
+	const PPTable_Generic_SubTable_Header *header;
+	u16 table_offset;
+	size_t table_size;
+	int ret;
+
+	table_offset = le16_to_cpu(powerplay_table->usFanTableOffset);
+	ret = get_tonga_subtable(hwmgr, powerplay_table, table_offset,
+				 sizeof(*header), (const void **)&header);
+	if (ret)
+		return ret;
+
+	if (header->ucRevId < 8)
+		table_size = sizeof(ATOM_Tonga_Fan_Table);
+	else if (header->ucRevId == 8)
+		table_size = sizeof(ATOM_Fiji_Fan_Table);
+	else
+		table_size = sizeof(ATOM_Polaris_Fan_Table);
+
+	PP_ASSERT_WITH_CODE((tonga_pp_table_has_space(hwmgr, table_offset,
+						      table_size)),
+			    "Invalid PowerPlay Table!", return -1);
+
+	*fan_table = header;
+
+	return 0;
+}
+
+static int get_tonga_power_tune_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table,
+		const PPTable_Generic_SubTable_Header **power_tune_table)
+{
+	const PPTable_Generic_SubTable_Header *header;
+	u16 table_offset;
+	size_t table_size;
+	int ret;
+
+	table_offset = le16_to_cpu(powerplay_table->usPowerTuneTableOffset);
+	ret = get_tonga_subtable(hwmgr, powerplay_table, table_offset,
+				 sizeof(*header), (const void **)&header);
+	if (ret)
+		return ret;
+
+	if (header->ucRevId < 3)
+		table_size = sizeof(ATOM_Tonga_PowerTune_Table);
+	else if (header->ucRevId < 4)
+		table_size = sizeof(ATOM_Fiji_PowerTune_Table);
+	else
+		table_size = sizeof(ATOM_Polaris_PowerTune_Table);
+
+	PP_ASSERT_WITH_CODE((tonga_pp_table_has_space(hwmgr, table_offset,
+						      table_size)),
+			    "Invalid PowerPlay Table!", return -1);
+
+	*power_tune_table = header;
+
+	return 0;
+}
+
+static int get_tonga_ppm_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table,
+		const ATOM_Tonga_PPM_Table **ppm_table)
+{
+	u16 table_offset;
+
+	table_offset = le16_to_cpu(powerplay_table->usPPMTableOffset);
+
+	return get_tonga_subtable(hwmgr, powerplay_table, table_offset,
+				  sizeof(**ppm_table), (const void **)ppm_table);
+}
+
+static int get_tonga_gpio_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table,
+		const ATOM_Tonga_GPIO_Table **gpio_table)
+{
+	u16 table_offset;
+
+	table_offset = le16_to_cpu(powerplay_table->usGPIOTableOffset);
+
+	return get_tonga_subtable(hwmgr, powerplay_table, table_offset,
+				  sizeof(**gpio_table), (const void **)gpio_table);
+}
+
+static int get_tonga_vce_state_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table,
+		const ATOM_Tonga_VCE_State_Table **vce_state_table)
+{
+	const ATOM_Tonga_VCE_State_Table *table;
+	u16 table_offset;
+	int ret;
+
+	table_offset = le16_to_cpu(powerplay_table->usVCEStateTableOffset);
+	ret = get_tonga_subtable(hwmgr, powerplay_table, table_offset,
+				 sizeof(*table), (const void **)&table);
+	if (ret)
+		return ret;
+
+	ret = validate_tonga_table_entries(hwmgr, table_offset,
+					   offsetof(ATOM_Tonga_VCE_State_Table,
+						    entries),
+					   table->ucNumEntries,
+					   sizeof(ATOM_Tonga_VCE_State_Record));
+	if (ret)
+		return ret;
+
+	*vce_state_table = table;
+
+	return 0;
+}
+
 static int get_vddc_lookup_table(
 		struct pp_hwmgr	*hwmgr,
 		phm_ppt_v1_voltage_lookup_table	**lookup_table,
@@ -158,6 +520,7 @@ static int get_vddc_lookup_table(
 		)
 {
 	uint32_t i;
+	uint32_t num_entries;
 	phm_ppt_v1_voltage_lookup_table *table;
 	phm_ppt_v1_voltage_lookup_record *record;
 	ATOM_Tonga_Voltage_Lookup_Record *atom_record;
@@ -165,13 +528,22 @@ static int get_vddc_lookup_table(
 	PP_ASSERT_WITH_CODE((0 != vddc_lookup_pp_tables->ucNumEntries),
 		"Invalid CAC Leakage PowerPlay Table!", return 1);
 
-	table = kzalloc_flex(*table, entries, max_levels);
+	num_entries = min_t(uint32_t, vddc_lookup_pp_tables->ucNumEntries,
+			    min_t(uint32_t, max_levels,
+				  pp_entries_max(hwmgr, vddc_lookup_pp_tables,
+						 sizeof(*vddc_lookup_pp_tables),
+						 sizeof(ATOM_Tonga_Voltage_Lookup_Record))));
+	if (num_entries < vddc_lookup_pp_tables->ucNumEntries)
+		pr_warn("amdgpu: VddcLookup table: clamping ucNumEntries %u -> %u\n",
+			vddc_lookup_pp_tables->ucNumEntries, num_entries);
+
+	table = kzalloc_flex(*table, entries, num_entries);
 	if (!table)
 		return -ENOMEM;
 
-	table->count = vddc_lookup_pp_tables->ucNumEntries;
+	table->count = num_entries;
 
-	for (i = 0; i < vddc_lookup_pp_tables->ucNumEntries; i++) {
+	for (i = 0; i < num_entries; i++) {
 		record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
 					phm_ppt_v1_voltage_lookup_record,
 					entries, table, i);
@@ -198,7 +570,7 @@ static int get_vddc_lookup_table(
  */
 static int get_platform_power_management_table(
 		struct pp_hwmgr *hwmgr,
-		ATOM_Tonga_PPM_Table *atom_ppm_table)
+		const ATOM_Tonga_PPM_Table *atom_ppm_table)
 {
 	struct phm_ppm_table *ptr = kzalloc_obj(*ptr);
 	struct phm_ppt_v1_information *pp_table_information =
@@ -246,7 +618,7 @@ static int init_dpm_2_parameters(
 {
 	int result = 0;
 	struct phm_ppt_v1_information *pp_table_information = (struct phm_ppt_v1_information *)(hwmgr->pptable);
-	ATOM_Tonga_PPM_Table *atom_ppm_table;
+	const ATOM_Tonga_PPM_Table *atom_ppm_table;
 	uint32_t disable_ppm = 0;
 	uint32_t disable_power_control = 0;
 
@@ -275,30 +647,39 @@ static int init_dpm_2_parameters(
 	}
 
 	if (0 != powerplay_table->usVddcLookupTableOffset) {
-		const ATOM_Tonga_Voltage_Lookup_Table *pVddcCACTable =
-			(ATOM_Tonga_Voltage_Lookup_Table *)(((unsigned long)powerplay_table) +
-			le16_to_cpu(powerplay_table->usVddcLookupTableOffset));
-
-		result = get_vddc_lookup_table(hwmgr,
-			&pp_table_information->vddc_lookup_table, pVddcCACTable, 16);
+		const ATOM_Tonga_Voltage_Lookup_Table *pVddcCACTable;
+
+		result = get_tonga_voltage_lookup_table(hwmgr, powerplay_table,
+				le16_to_cpu(powerplay_table->usVddcLookupTableOffset),
+				16, &pVddcCACTable);
+		if (!result)
+			result = get_vddc_lookup_table(hwmgr,
+				&pp_table_information->vddc_lookup_table,
+				pVddcCACTable, 16);
 	}
 
-	if (0 != powerplay_table->usVddgfxLookupTableOffset) {
-		const ATOM_Tonga_Voltage_Lookup_Table *pVddgfxCACTable =
-			(ATOM_Tonga_Voltage_Lookup_Table *)(((unsigned long)powerplay_table) +
-			le16_to_cpu(powerplay_table->usVddgfxLookupTableOffset));
+	if (!result && 0 != powerplay_table->usVddgfxLookupTableOffset) {
+		const ATOM_Tonga_Voltage_Lookup_Table *pVddgfxCACTable;
 
-		result = get_vddc_lookup_table(hwmgr,
-			&pp_table_information->vddgfx_lookup_table, pVddgfxCACTable, 16);
+		result = get_tonga_voltage_lookup_table(hwmgr, powerplay_table,
+				le16_to_cpu(powerplay_table->usVddgfxLookupTableOffset),
+				16, &pVddgfxCACTable);
+		if (!result)
+			result = get_vddc_lookup_table(hwmgr,
+				&pp_table_information->vddgfx_lookup_table,
+				pVddgfxCACTable, 16);
 	}
 
 	disable_ppm = 0;
 	if (0 == disable_ppm) {
-		atom_ppm_table = (ATOM_Tonga_PPM_Table *)
-			(((unsigned long)powerplay_table) + le16_to_cpu(powerplay_table->usPPMTableOffset));
-
 		if (0 != powerplay_table->usPPMTableOffset) {
-			if (get_platform_power_management_table(hwmgr, atom_ppm_table) == 0) {
+			int ret;
+
+			ret = get_tonga_ppm_table(hwmgr, powerplay_table,
+						  &atom_ppm_table);
+			if (!ret &&
+			    get_platform_power_management_table(hwmgr,
+							       atom_ppm_table) == 0) {
 				phm_cap_set(hwmgr->platform_descriptor.platformCaps,
 					PHM_PlatformCaps_EnablePlatformPowerManagement);
 			}
@@ -363,6 +744,7 @@ static int get_mclk_voltage_dependency_table(
 		)
 {
 	uint32_t i;
+	uint32_t num_entries;
 	phm_ppt_v1_clock_voltage_dependency_table *mclk_table;
 	phm_ppt_v1_clock_voltage_dependency_record *mclk_table_record;
 	ATOM_Tonga_MCLK_Dependency_Record *mclk_dep_record;
@@ -370,14 +752,21 @@ static int get_mclk_voltage_dependency_table(
 	PP_ASSERT_WITH_CODE((0 != mclk_dep_table->ucNumEntries),
 		"Invalid PowerPlay Table!", return -1);
 
-	mclk_table = kzalloc_flex(*mclk_table, entries,
-				  mclk_dep_table->ucNumEntries);
+	num_entries = min_t(uint32_t, mclk_dep_table->ucNumEntries,
+			    pp_entries_max(hwmgr, mclk_dep_table,
+					   sizeof(*mclk_dep_table),
+					   sizeof(ATOM_Tonga_MCLK_Dependency_Record)));
+	if (num_entries < mclk_dep_table->ucNumEntries)
+		pr_warn("amdgpu: MCLK dependency table: clamping ucNumEntries %u -> %u\n",
+			mclk_dep_table->ucNumEntries, num_entries);
+
+	mclk_table = kzalloc_flex(*mclk_table, entries, num_entries);
 	if (!mclk_table)
 		return -ENOMEM;
 
-	mclk_table->count = (uint32_t)mclk_dep_table->ucNumEntries;
+	mclk_table->count = num_entries;
 
-	for (i = 0; i < mclk_dep_table->ucNumEntries; i++) {
+	for (i = 0; i < num_entries; i++) {
 		mclk_table_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
 					phm_ppt_v1_clock_voltage_dependency_record,
 						entries, mclk_table, i);
@@ -403,6 +792,7 @@ static int get_sclk_voltage_dependency_table(
 		)
 {
 	uint32_t i;
+	uint32_t num_entries;
 	phm_ppt_v1_clock_voltage_dependency_table *sclk_table;
 	phm_ppt_v1_clock_voltage_dependency_record *sclk_table_record;
 
@@ -414,14 +804,21 @@ static int get_sclk_voltage_dependency_table(
 		PP_ASSERT_WITH_CODE((0 != tonga_table->ucNumEntries),
 			"Invalid PowerPlay Table!", return -1);
 
-		sclk_table = kzalloc_flex(*sclk_table, entries,
-					  tonga_table->ucNumEntries);
+		num_entries = min_t(uint32_t, tonga_table->ucNumEntries,
+				    pp_entries_max(hwmgr, tonga_table,
+						   sizeof(*tonga_table),
+						   sizeof(ATOM_Tonga_SCLK_Dependency_Record)));
+		if (num_entries < tonga_table->ucNumEntries)
+			pr_warn("amdgpu: Tonga SCLK dependency table: clamping ucNumEntries %u -> %u\n",
+				tonga_table->ucNumEntries, num_entries);
+
+		sclk_table = kzalloc_flex(*sclk_table, entries, num_entries);
 		if (!sclk_table)
 			return -ENOMEM;
 
-		sclk_table->count = (uint32_t)tonga_table->ucNumEntries;
+		sclk_table->count = num_entries;
 
-		for (i = 0; i < tonga_table->ucNumEntries; i++) {
+		for (i = 0; i < num_entries; i++) {
 			sclk_dep_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
 						ATOM_Tonga_SCLK_Dependency_Record,
 						entries, tonga_table, i);
@@ -443,14 +840,21 @@ static int get_sclk_voltage_dependency_table(
 		PP_ASSERT_WITH_CODE((0 != polaris_table->ucNumEntries),
 			"Invalid PowerPlay Table!", return -1);
 
-		sclk_table = kzalloc_flex(*sclk_table, entries,
-					  polaris_table->ucNumEntries);
+		num_entries = min_t(uint32_t, polaris_table->ucNumEntries,
+				    pp_entries_max(hwmgr, polaris_table,
+						   sizeof(*polaris_table),
+						   sizeof(ATOM_Polaris_SCLK_Dependency_Record)));
+		if (num_entries < polaris_table->ucNumEntries)
+			pr_warn("amdgpu: Polaris SCLK dependency table: clamping ucNumEntries %u -> %u\n",
+				polaris_table->ucNumEntries, num_entries);
+
+		sclk_table = kzalloc_flex(*sclk_table, entries, num_entries);
 		if (!sclk_table)
 			return -ENOMEM;
 
-		sclk_table->count = (uint32_t)polaris_table->ucNumEntries;
+		sclk_table->count = num_entries;
 
-		for (i = 0; i < polaris_table->ucNumEntries; i++) {
+		for (i = 0; i < num_entries; i++) {
 			sclk_dep_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
 						ATOM_Polaris_SCLK_Dependency_Record,
 						entries, polaris_table, i);
@@ -715,20 +1119,29 @@ static int get_mm_clock_voltage_table(
 		)
 {
 	uint32_t i;
+	uint32_t num_entries;
 	const ATOM_Tonga_MM_Dependency_Record *mm_dependency_record;
 	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table;
 	phm_ppt_v1_mm_clock_voltage_dependency_record *mm_table_record;
 
 	PP_ASSERT_WITH_CODE((0 != mm_dependency_table->ucNumEntries),
 		"Invalid PowerPlay Table!", return -1);
-	mm_table = kzalloc_flex(*mm_table, entries,
-				mm_dependency_table->ucNumEntries);
+
+	num_entries = min_t(uint32_t, mm_dependency_table->ucNumEntries,
+			    pp_entries_max(hwmgr, mm_dependency_table,
+					   sizeof(*mm_dependency_table),
+					   sizeof(ATOM_Tonga_MM_Dependency_Record)));
+	if (num_entries < mm_dependency_table->ucNumEntries)
+		pr_warn("amdgpu: MM dependency table: clamping ucNumEntries %u -> %u\n",
+			mm_dependency_table->ucNumEntries, num_entries);
+
+	mm_table = kzalloc_flex(*mm_table, entries, num_entries);
 	if (!mm_table)
 		return -ENOMEM;
 
-	mm_table->count = mm_dependency_table->ucNumEntries;
+	mm_table->count = num_entries;
 
-	for (i = 0; i < mm_dependency_table->ucNumEntries; i++) {
+	for (i = 0; i < num_entries; i++) {
 		mm_dependency_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
 						ATOM_Tonga_MM_Dependency_Record,
 						entries, mm_dependency_table, i);
@@ -789,28 +1202,13 @@ static int init_clock_voltage_dependency(
 	int result = 0;
 	struct phm_ppt_v1_information *pp_table_information =
 		(struct phm_ppt_v1_information *)(hwmgr->pptable);
-
-	const ATOM_Tonga_MM_Dependency_Table *mm_dependency_table =
-		(const ATOM_Tonga_MM_Dependency_Table *)(((unsigned long) powerplay_table) +
-		le16_to_cpu(powerplay_table->usMMDependencyTableOffset));
-	const PPTable_Generic_SubTable_Header *pPowerTuneTable =
-		(const PPTable_Generic_SubTable_Header *)(((unsigned long) powerplay_table) +
-		le16_to_cpu(powerplay_table->usPowerTuneTableOffset));
-	const ATOM_Tonga_MCLK_Dependency_Table *mclk_dep_table =
-		(const ATOM_Tonga_MCLK_Dependency_Table *)(((unsigned long) powerplay_table) +
-		le16_to_cpu(powerplay_table->usMclkDependencyTableOffset));
-	const PPTable_Generic_SubTable_Header *sclk_dep_table =
-		(const PPTable_Generic_SubTable_Header *)(((unsigned long) powerplay_table) +
-		le16_to_cpu(powerplay_table->usSclkDependencyTableOffset));
-	const ATOM_Tonga_Hard_Limit_Table *pHardLimits =
-		(const ATOM_Tonga_Hard_Limit_Table *)(((unsigned long) powerplay_table) +
-		le16_to_cpu(powerplay_table->usHardLimitTableOffset));
-	const PPTable_Generic_SubTable_Header *pcie_table =
-		(const PPTable_Generic_SubTable_Header *)(((unsigned long) powerplay_table) +
-		le16_to_cpu(powerplay_table->usPCIETableOffset));
-	const ATOM_Tonga_GPIO_Table *gpio_table =
-		(const ATOM_Tonga_GPIO_Table *)(((unsigned long) powerplay_table) +
-		le16_to_cpu(powerplay_table->usGPIOTableOffset));
+	const ATOM_Tonga_MM_Dependency_Table *mm_dependency_table;
+	const PPTable_Generic_SubTable_Header *pPowerTuneTable;
+	const ATOM_Tonga_MCLK_Dependency_Table *mclk_dep_table;
+	const PPTable_Generic_SubTable_Header *sclk_dep_table;
+	const ATOM_Tonga_Hard_Limit_Table *pHardLimits;
+	const PPTable_Generic_SubTable_Header *pcie_table;
+	const ATOM_Tonga_GPIO_Table *gpio_table;
 
 	pp_table_information->vdd_dep_on_sclk = NULL;
 	pp_table_information->vdd_dep_on_mclk = NULL;
@@ -818,29 +1216,58 @@ static int init_clock_voltage_dependency(
 	pp_table_information->pcie_table = NULL;
 	pp_table_information->gpio_table = NULL;
 
-	if (powerplay_table->usMMDependencyTableOffset != 0)
-		result = get_mm_clock_voltage_table(hwmgr,
-		&pp_table_information->mm_dep_table, mm_dependency_table);
+	if (powerplay_table->usMMDependencyTableOffset != 0) {
+		result = get_tonga_mm_dependency_table(hwmgr, powerplay_table,
+						       &mm_dependency_table);
+		if (!result)
+			result = get_mm_clock_voltage_table(hwmgr,
+				&pp_table_information->mm_dep_table,
+				mm_dependency_table);
+	}
 
-	if (result == 0 && powerplay_table->usPowerTuneTableOffset != 0)
-		result = get_cac_tdp_table(hwmgr,
-		&pp_table_information->cac_dtp_table, pPowerTuneTable);
+	if (result == 0 && powerplay_table->usPowerTuneTableOffset != 0) {
+		result = get_tonga_power_tune_table(hwmgr, powerplay_table,
+						    &pPowerTuneTable);
+		if (!result)
+			result = get_cac_tdp_table(hwmgr,
+				&pp_table_information->cac_dtp_table,
+				pPowerTuneTable);
+	}
 
-	if (result == 0 && powerplay_table->usSclkDependencyTableOffset != 0)
-		result = get_sclk_voltage_dependency_table(hwmgr,
-		&pp_table_information->vdd_dep_on_sclk, sclk_dep_table);
+	if (result == 0 && powerplay_table->usSclkDependencyTableOffset != 0) {
+		result = get_tonga_sclk_dependency_table(hwmgr, powerplay_table,
+							 &sclk_dep_table);
+		if (!result)
+			result = get_sclk_voltage_dependency_table(hwmgr,
+				&pp_table_information->vdd_dep_on_sclk,
+				sclk_dep_table);
+	}
 
-	if (result == 0 && powerplay_table->usMclkDependencyTableOffset != 0)
-		result = get_mclk_voltage_dependency_table(hwmgr,
-		&pp_table_information->vdd_dep_on_mclk, mclk_dep_table);
+	if (result == 0 && powerplay_table->usMclkDependencyTableOffset != 0) {
+		result = get_tonga_mclk_dependency_table(hwmgr, powerplay_table,
+							 &mclk_dep_table);
+		if (!result)
+			result = get_mclk_voltage_dependency_table(hwmgr,
+				&pp_table_information->vdd_dep_on_mclk,
+				mclk_dep_table);
+	}
 
-	if (result == 0 && powerplay_table->usPCIETableOffset != 0)
-		result = get_pcie_table(hwmgr,
-		&pp_table_information->pcie_table, pcie_table);
+	if (result == 0 && powerplay_table->usPCIETableOffset != 0) {
+		result = get_tonga_pcie_table(hwmgr, powerplay_table,
+					      &pcie_table);
+		if (!result)
+			result = get_pcie_table(hwmgr,
+				&pp_table_information->pcie_table, pcie_table);
+	}
 
-	if (result == 0 && powerplay_table->usHardLimitTableOffset != 0)
-		result = get_hard_limits(hwmgr,
-		&pp_table_information->max_clock_voltage_on_dc, pHardLimits);
+	if (result == 0 && powerplay_table->usHardLimitTableOffset != 0) {
+		result = get_tonga_hard_limit_table(hwmgr, powerplay_table,
+						    &pHardLimits);
+		if (!result)
+			result = get_hard_limits(hwmgr,
+				&pp_table_information->max_clock_voltage_on_dc,
+				pHardLimits);
+	}
 
 	hwmgr->dyn_state.max_clock_voltage_on_dc.sclk =
 		pp_table_information->max_clock_voltage_on_dc.sclk;
@@ -861,9 +1288,13 @@ static int init_clock_voltage_dependency(
 		result = get_valid_clk(hwmgr, &pp_table_information->valid_sclk_values,
 		pp_table_information->vdd_dep_on_sclk);
 
-	if (!result && gpio_table)
-		result = get_gpio_table(hwmgr, &pp_table_information->gpio_table,
-				gpio_table);
+	if (!result && powerplay_table->usGPIOTableOffset) {
+		result = get_tonga_gpio_table(hwmgr, powerplay_table,
+					      &gpio_table);
+		if (!result)
+			result = get_gpio_table(hwmgr,
+				&pp_table_information->gpio_table, gpio_table);
+	}
 
 	return result;
 }
@@ -908,14 +1339,17 @@ static int init_thermal_controller(
 		)
 {
 	const PPTable_Generic_SubTable_Header *fan_table;
-	ATOM_Tonga_Thermal_Controller *thermal_controller;
+	const ATOM_Tonga_Thermal_Controller *thermal_controller;
+	int ret;
 
-	thermal_controller = (ATOM_Tonga_Thermal_Controller *)
-		(((unsigned long)powerplay_table) +
-		le16_to_cpu(powerplay_table->usThermalControllerOffset));
 	PP_ASSERT_WITH_CODE((0 != powerplay_table->usThermalControllerOffset),
 		"Thermal controller table not set!", return -1);
 
+	ret = get_tonga_thermal_controller_table(hwmgr, powerplay_table,
+						 &thermal_controller);
+	if (ret)
+		return ret;
+
 	hwmgr->thermal_controller.ucType = thermal_controller->ucType;
 	hwmgr->thermal_controller.ucI2cLine = thermal_controller->ucI2cLine;
 	hwmgr->thermal_controller.ucI2cAddress = thermal_controller->ucI2cAddress;
@@ -943,12 +1377,13 @@ static int init_thermal_controller(
 		return 0;
 	}
 
-	fan_table = (const PPTable_Generic_SubTable_Header *)
-		(((unsigned long)powerplay_table) +
-		le16_to_cpu(powerplay_table->usFanTableOffset));
-
 	PP_ASSERT_WITH_CODE((0 != powerplay_table->usFanTableOffset),
 		"Fan table not set!", return -1);
+
+	ret = get_tonga_fan_table(hwmgr, powerplay_table, &fan_table);
+	if (ret)
+		return ret;
+
 	PP_ASSERT_WITH_CODE((0 < fan_table->ucRevId),
 		"Unsupported fan table format!", return -1);
 
@@ -1104,21 +1539,24 @@ static int init_thermal_controller(
 }
 
 /**
- * check_powerplay_tables - Private Function used during initialization.
- * Inspect the PowerPlay table for obvious signs of corruption.
+ * get_tonga_state_array - Get the Tonga state array from the PowerPlay table.
  * @hwmgr: Pointer to the hardware manager.
  * @powerplay_table: Pointer to the PowerPlay Table.
- * Exception:  2 if the powerplay table is incorrect.
+ * @state_array: Pointer to the returned Tonga state array.
+ *
+ * Return: 0 on success, negative error code on failure.
  */
-static int check_powerplay_tables(
-		struct pp_hwmgr *hwmgr,
-		const ATOM_Tonga_POWERPLAYTABLE *powerplay_table
-		)
+static int get_tonga_state_array(struct pp_hwmgr *hwmgr,
+	const ATOM_Tonga_POWERPLAYTABLE *powerplay_table,
+	const ATOM_Tonga_State_Array **state_array)
 {
 	const ATOM_Tonga_State_Array *state_arrays;
+	u16 state_array_offset;
+	size_t state_array_size;
+	size_t table_size = hwmgr->soft_pp_table_size;
 
-	state_arrays = (ATOM_Tonga_State_Array *)(((unsigned long)powerplay_table) +
-		le16_to_cpu(powerplay_table->usStateArrayOffset));
+	PP_ASSERT_WITH_CODE((table_size >= sizeof(*powerplay_table)),
+			    "Invalid PowerPlay Table!", return -1);
 
 	PP_ASSERT_WITH_CODE((ATOM_Tonga_TABLE_REVISION_TONGA <=
 		powerplay_table->sHeader.ucTableFormatRevision),
@@ -1127,12 +1565,34 @@ static int check_powerplay_tables(
 		"State table is not set!", return -1);
 	PP_ASSERT_WITH_CODE((0 < powerplay_table->sHeader.usStructureSize),
 		"Invalid PowerPlay Table!", return -1);
+
+	state_array_offset = le16_to_cpu(powerplay_table->usStateArrayOffset);
+	PP_ASSERT_WITH_CODE((state_array_offset <=
+			     table_size - sizeof(*state_arrays)),
+			    "Invalid PowerPlay Table!", return -1);
+
+	state_arrays = (ATOM_Tonga_State_Array *)(((unsigned long)powerplay_table) +
+		state_array_offset);
 	PP_ASSERT_WITH_CODE((0 < state_arrays->ucNumEntries),
 		"Invalid PowerPlay Table!", return -1);
 
+	state_array_size = struct_size(state_arrays, entries, state_arrays->ucNumEntries);
+	PP_ASSERT_WITH_CODE((state_array_size <= table_size - state_array_offset),
+			    "Invalid PowerPlay Table!", return -1);
+
+	*state_array = state_arrays;
+
 	return 0;
 }
 
+static int check_powerplay_tables(struct pp_hwmgr *hwmgr,
+	const ATOM_Tonga_POWERPLAYTABLE *powerplay_table)
+{
+	const ATOM_Tonga_State_Array *state_arrays;
+
+	return get_tonga_state_array(hwmgr, powerplay_table, &state_arrays);
+}
+
 static int pp_tables_v1_0_initialize(struct pp_hwmgr *hwmgr)
 {
 	int result = 0;
@@ -1236,17 +1696,16 @@ const struct pp_table_func pptable_v1_0_funcs = {
 
 int get_number_of_powerplay_table_entries_v1_0(struct pp_hwmgr *hwmgr)
 {
-	ATOM_Tonga_State_Array const *state_arrays;
+	const ATOM_Tonga_State_Array *state_arrays;
 	const ATOM_Tonga_POWERPLAYTABLE *pp_table = get_powerplay_table(hwmgr);
+	int result;
 
 	PP_ASSERT_WITH_CODE((NULL != pp_table),
 			"Missing PowerPlay Table!", return -1);
-	PP_ASSERT_WITH_CODE((pp_table->sHeader.ucTableFormatRevision >=
-			ATOM_Tonga_TABLE_REVISION_TONGA),
-			"Incorrect PowerPlay table revision!", return -1);
 
-	state_arrays = (ATOM_Tonga_State_Array *)(((unsigned long)pp_table) +
-			le16_to_cpu(pp_table->usStateArrayOffset));
+	result = get_tonga_state_array(hwmgr, pp_table, &state_arrays);
+	PP_ASSERT_WITH_CODE((result == 0),
+			    "Invalid PowerPlay Table State Array.", return result);
 
 	return (uint32_t)(state_arrays->ucNumEntries);
 }
@@ -1287,13 +1746,15 @@ static int ppt_get_num_of_vce_state_table_entries_v1_0(struct pp_hwmgr *hwmgr)
 {
 	const ATOM_Tonga_POWERPLAYTABLE *pp_table = get_powerplay_table(hwmgr);
 	const ATOM_Tonga_VCE_State_Table *vce_state_table;
+	int ret;
 
 
 	if (pp_table == NULL)
 		return 0;
 
-	vce_state_table = (void *)pp_table +
-			le16_to_cpu(pp_table->usVCEStateTableOffset);
+	ret = get_tonga_vce_state_table(hwmgr, pp_table, &vce_state_table);
+	if (ret)
+		return 0;
 
 	return vce_state_table->ucNumEntries;
 }
@@ -1302,18 +1763,39 @@ static int ppt_get_vce_state_table_entry_v1_0(struct pp_hwmgr *hwmgr, uint32_t i
 		struct amd_vce_state *vce_state, void **clock_info, uint32_t *flag)
 {
 	const ATOM_Tonga_VCE_State_Record *vce_state_record;
-	ATOM_Tonga_SCLK_Dependency_Record *sclk_dep_record;
+	ATOM_Tonga_SCLK_Dependency_Record *sclk_dep_record = NULL;
+	ATOM_Polaris_SCLK_Dependency_Record *polaris_sclk_dep_record = NULL;
 	ATOM_Tonga_MCLK_Dependency_Record *mclk_dep_record;
 	ATOM_Tonga_MM_Dependency_Record *mm_dep_record;
 	const ATOM_Tonga_POWERPLAYTABLE *pptable = get_powerplay_table(hwmgr);
-	const ATOM_Tonga_VCE_State_Table *vce_state_table = (ATOM_Tonga_VCE_State_Table *)(((unsigned long)pptable)
-							  + le16_to_cpu(pptable->usVCEStateTableOffset));
-	const ATOM_Tonga_SCLK_Dependency_Table *sclk_dep_table = (ATOM_Tonga_SCLK_Dependency_Table *)(((unsigned long)pptable)
-							  + le16_to_cpu(pptable->usSclkDependencyTableOffset));
-	const ATOM_Tonga_MCLK_Dependency_Table *mclk_dep_table = (ATOM_Tonga_MCLK_Dependency_Table *)(((unsigned long)pptable)
-							  + le16_to_cpu(pptable->usMclkDependencyTableOffset));
-	const ATOM_Tonga_MM_Dependency_Table *mm_dep_table = (ATOM_Tonga_MM_Dependency_Table *)(((unsigned long)pptable)
-							  + le16_to_cpu(pptable->usMMDependencyTableOffset));
+	const ATOM_Tonga_VCE_State_Table *vce_state_table;
+	const PPTable_Generic_SubTable_Header *sclk_dep_table_header;
+	const ATOM_Tonga_SCLK_Dependency_Table *sclk_dep_table;
+	const ATOM_Tonga_MCLK_Dependency_Table *mclk_dep_table;
+	const ATOM_Tonga_MM_Dependency_Table *mm_dep_table;
+	int ret;
+
+	if (!pptable)
+		return -EINVAL;
+
+	ret = get_tonga_vce_state_table(hwmgr, pptable, &vce_state_table);
+	if (ret)
+		return ret;
+
+	ret = get_tonga_sclk_dependency_table(hwmgr, pptable,
+					      &sclk_dep_table_header);
+	if (ret)
+		return ret;
+	sclk_dep_table = (const ATOM_Tonga_SCLK_Dependency_Table *)
+		sclk_dep_table_header;
+
+	ret = get_tonga_mclk_dependency_table(hwmgr, pptable, &mclk_dep_table);
+	if (ret)
+		return ret;
+
+	ret = get_tonga_mm_dependency_table(hwmgr, pptable, &mm_dep_table);
+	if (ret)
+		return ret;
 
 	PP_ASSERT_WITH_CODE((i < vce_state_table->ucNumEntries),
 			 "Requested state entry ID is out of range!",
@@ -1322,10 +1804,27 @@ static int ppt_get_vce_state_table_entry_v1_0(struct pp_hwmgr *hwmgr, uint32_t i
 	vce_state_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
 					ATOM_Tonga_VCE_State_Record,
 					entries, vce_state_table, i);
-	sclk_dep_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
-					ATOM_Tonga_SCLK_Dependency_Record,
-					entries, sclk_dep_table,
-					vce_state_record->ucSCLKIndex);
+	PP_ASSERT_WITH_CODE((vce_state_record->ucSCLKIndex <
+			     sclk_dep_table->ucNumEntries),
+			    "Invalid PowerPlay Table!", return -EINVAL);
+	PP_ASSERT_WITH_CODE((vce_state_record->ucVCEClockIndex <
+			     mm_dep_table->ucNumEntries),
+			    "Invalid PowerPlay Table!", return -EINVAL);
+	PP_ASSERT_WITH_CODE((mclk_dep_table->ucNumEntries != 0),
+			    "Invalid PowerPlay Table!", return -EINVAL);
+
+	if (sclk_dep_table_header->ucRevId < 1)
+		sclk_dep_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+						ATOM_Tonga_SCLK_Dependency_Record,
+						entries, sclk_dep_table,
+						vce_state_record->ucSCLKIndex);
+	else
+		polaris_sclk_dep_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
+						ATOM_Polaris_SCLK_Dependency_Record,
+						entries,
+						(ATOM_Polaris_SCLK_Dependency_Table *)
+						sclk_dep_table_header,
+						vce_state_record->ucSCLKIndex);
 	mm_dep_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
 					ATOM_Tonga_MM_Dependency_Record,
 					entries, mm_dep_table,
@@ -1334,7 +1833,10 @@ static int ppt_get_vce_state_table_entry_v1_0(struct pp_hwmgr *hwmgr, uint32_t i
 
 	vce_state->evclk = le32_to_cpu(mm_dep_record->ulEClk);
 	vce_state->ecclk = le32_to_cpu(mm_dep_record->ulEClk);
-	vce_state->sclk = le32_to_cpu(sclk_dep_record->ulSclk);
+	if (sclk_dep_record)
+		vce_state->sclk = le32_to_cpu(sclk_dep_record->ulSclk);
+	else
+		vce_state->sclk = le32_to_cpu(polaris_sclk_dep_record->ulSclk);
 
 	if (vce_state_record->ucMCLKIndex >= mclk_dep_table->ucNumEntries)
 		mclk_dep_record = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
@@ -1377,15 +1879,11 @@ int get_powerplay_table_entry_v1_0(struct pp_hwmgr *hwmgr,
 
 	if (pp_table->sHeader.ucTableFormatRevision >=
 			ATOM_Tonga_TABLE_REVISION_TONGA) {
-		state_arrays = (ATOM_Tonga_State_Array *)(((unsigned long)pp_table) +
-				le16_to_cpu(pp_table->usStateArrayOffset));
-
-		PP_ASSERT_WITH_CODE((0 < pp_table->usStateArrayOffset),
-				"Invalid PowerPlay Table State Array Offset.", return -1);
-		PP_ASSERT_WITH_CODE((0 < state_arrays->ucNumEntries),
-				"Invalid PowerPlay Table State Array.", return -1);
-		PP_ASSERT_WITH_CODE((entry_index <= state_arrays->ucNumEntries),
-				"Invalid PowerPlay Table State Array Entry.", return -1);
+		result = get_tonga_state_array(hwmgr, pp_table, &state_arrays);
+		PP_ASSERT_WITH_CODE((result == 0),
+				    "Invalid PowerPlay Table State Array.", return result);
+		PP_ASSERT_WITH_CODE((entry_index < state_arrays->ucNumEntries),
+				    "Invalid PowerPlay Table State Array Entry.", return -1);
 
 		state_entry = GET_FLEXIBLE_ARRAY_MEMBER_ADDR(
 						ATOM_Tonga_State, entries,
@@ -1411,4 +1909,3 @@ int get_powerplay_table_entry_v1_0(struct pp_hwmgr *hwmgr,
 
 	return result;
 }
-
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/processpptables.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/processpptables.c
index bfd8fbb0b49dd..56926eec68207 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/processpptables.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/processpptables.c
@@ -47,26 +47,53 @@
 
 #define NUM_BITS_CLOCK_INFO_ARRAY_INDEX 6
 
+static bool pp_table_has_space(struct pp_hwmgr *hwmgr, size_t offset,
+			       size_t size)
+{
+	size_t table_size = hwmgr->soft_pp_table_size;
+
+	return offset <= table_size && size <= table_size - offset;
+}
+
+static const ATOM_PPLIB_EXTENDEDHEADER *
+get_extended_header(struct pp_hwmgr *hwmgr,
+		    const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table,
+		    size_t min_size)
+{
+	const ATOM_PPLIB_POWERPLAYTABLE3 *powerplay_table3;
+	const ATOM_PPLIB_EXTENDEDHEADER *extended_header;
+	u16 offset;
+
+	if (le16_to_cpu(powerplay_table->usTableSize) <
+	    sizeof(ATOM_PPLIB_POWERPLAYTABLE3) ||
+	    !pp_table_has_space(hwmgr, 0, sizeof(ATOM_PPLIB_POWERPLAYTABLE3)))
+		return NULL;
+
+	powerplay_table3 = (const ATOM_PPLIB_POWERPLAYTABLE3 *)powerplay_table;
+	offset = le16_to_cpu(powerplay_table3->usExtendendedHeaderOffset);
+	if (!offset || !pp_table_has_space(hwmgr, offset,
+					   sizeof(extended_header->usSize)))
+		return NULL;
+
+	extended_header = (const ATOM_PPLIB_EXTENDEDHEADER *)
+		(((unsigned long)powerplay_table) + offset);
+	if (le16_to_cpu(extended_header->usSize) < min_size ||
+	    !pp_table_has_space(hwmgr, offset, min_size))
+		return NULL;
+
+	return extended_header;
+}
+
 static uint16_t get_vce_table_offset(struct pp_hwmgr *hwmgr,
 			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
 {
 	uint16_t vce_table_offset = 0;
+	const ATOM_PPLIB_EXTENDEDHEADER *extended_header;
 
-	if (le16_to_cpu(powerplay_table->usTableSize) >=
-	   sizeof(ATOM_PPLIB_POWERPLAYTABLE3)) {
-		const ATOM_PPLIB_POWERPLAYTABLE3 *powerplay_table3 =
-			(const ATOM_PPLIB_POWERPLAYTABLE3 *)powerplay_table;
-
-		if (powerplay_table3->usExtendendedHeaderOffset > 0) {
-			const ATOM_PPLIB_EXTENDEDHEADER  *extended_header =
-						(const ATOM_PPLIB_EXTENDEDHEADER *)
-						(((unsigned long)powerplay_table3) +
-						le16_to_cpu(powerplay_table3->usExtendendedHeaderOffset));
-			if (le16_to_cpu(extended_header->usSize) >=
-			   SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V2)
-				vce_table_offset = le16_to_cpu(extended_header->usVCETableOffset);
-		}
-	}
+	extended_header = get_extended_header(hwmgr, powerplay_table,
+					      SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V2);
+	if (extended_header)
+		vce_table_offset = le16_to_cpu(extended_header->usVCETableOffset);
 
 	return vce_table_offset;
 }
@@ -93,7 +120,14 @@ static uint16_t get_vce_clock_info_array_size(struct pp_hwmgr *hwmgr,
 	if (table_offset > 0) {
 		const VCEClockInfoArray *p = (const VCEClockInfoArray *)
 			(((unsigned long) powerplay_table) + table_offset);
-		table_size = sizeof(uint8_t) + p->ucNumEntries * sizeof(VCEClockInfo);
+		size_t size;
+
+		if (!pp_table_has_space(hwmgr, table_offset, sizeof(p->ucNumEntries)))
+			return 0;
+
+		size = sizeof(uint8_t) + p->ucNumEntries * sizeof(VCEClockInfo);
+		if (pp_table_has_space(hwmgr, table_offset, size))
+			table_size = size;
 	}
 
 	return table_size;
@@ -104,10 +138,13 @@ static uint16_t get_vce_clock_voltage_limit_table_offset(struct pp_hwmgr *hwmgr,
 {
 	uint16_t table_offset = get_vce_clock_info_array_offset(hwmgr,
 							powerplay_table);
+	u16 table_size;
 
-	if (table_offset > 0)
-		return table_offset + get_vce_clock_info_array_size(hwmgr,
-							powerplay_table);
+	if (table_offset > 0) {
+		table_size = get_vce_clock_info_array_size(hwmgr, powerplay_table);
+		if (table_size)
+			return table_offset + table_size;
+	}
 
 	return 0;
 }
@@ -121,8 +158,15 @@ static uint16_t get_vce_clock_voltage_limit_table_size(struct pp_hwmgr *hwmgr,
 	if (table_offset > 0) {
 		const ATOM_PPLIB_VCE_Clock_Voltage_Limit_Table *ptable =
 			(const ATOM_PPLIB_VCE_Clock_Voltage_Limit_Table *)(((unsigned long) powerplay_table) + table_offset);
+		size_t size;
 
-		table_size = sizeof(uint8_t) + ptable->numEntries * sizeof(ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record);
+		if (!pp_table_has_space(hwmgr, table_offset, sizeof(ptable->numEntries)))
+			return 0;
+
+		size = sizeof(uint8_t) +
+			ptable->numEntries * sizeof(ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record);
+		if (pp_table_has_space(hwmgr, table_offset, size))
+			table_size = size;
 	}
 	return table_size;
 }
@@ -130,9 +174,13 @@ static uint16_t get_vce_clock_voltage_limit_table_size(struct pp_hwmgr *hwmgr,
 static uint16_t get_vce_state_table_offset(struct pp_hwmgr *hwmgr, const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
 {
 	uint16_t table_offset = get_vce_clock_voltage_limit_table_offset(hwmgr, powerplay_table);
+	u16 table_size;
 
-	if (table_offset > 0)
-		return table_offset + get_vce_clock_voltage_limit_table_size(hwmgr, powerplay_table);
+	if (table_offset > 0) {
+		table_size = get_vce_clock_voltage_limit_table_size(hwmgr, powerplay_table);
+		if (table_size)
+			return table_offset + table_size;
+	}
 
 	return 0;
 }
@@ -143,8 +191,12 @@ static const ATOM_PPLIB_VCE_State_Table *get_vce_state_table(
 {
 	uint16_t table_offset = get_vce_state_table_offset(hwmgr, powerplay_table);
 
-	if (table_offset > 0)
-		return (const ATOM_PPLIB_VCE_State_Table *)(((unsigned long) powerplay_table) + table_offset);
+	if (table_offset > 0) {
+		if (pp_table_has_space(hwmgr, table_offset,
+				       sizeof(((ATOM_PPLIB_VCE_State_Table *)0)->numEntries)))
+			return (const ATOM_PPLIB_VCE_State_Table *)
+				(((unsigned long)powerplay_table) + table_offset);
+	}
 
 	return NULL;
 }
@@ -153,21 +205,13 @@ static uint16_t get_uvd_table_offset(struct pp_hwmgr *hwmgr,
 			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
 {
 	uint16_t uvd_table_offset = 0;
+	const ATOM_PPLIB_EXTENDEDHEADER *extended_header;
+
+	extended_header = get_extended_header(hwmgr, powerplay_table,
+					      SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V3);
+	if (extended_header)
+		uvd_table_offset = le16_to_cpu(extended_header->usUVDTableOffset);
 
-	if (le16_to_cpu(powerplay_table->usTableSize) >=
-	    sizeof(ATOM_PPLIB_POWERPLAYTABLE3)) {
-		const ATOM_PPLIB_POWERPLAYTABLE3 *powerplay_table3 =
-			(const ATOM_PPLIB_POWERPLAYTABLE3 *)powerplay_table;
-		if (powerplay_table3->usExtendendedHeaderOffset > 0) {
-			const ATOM_PPLIB_EXTENDEDHEADER  *extended_header =
-					(const ATOM_PPLIB_EXTENDEDHEADER *)
-					(((unsigned long)powerplay_table3) +
-				le16_to_cpu(powerplay_table3->usExtendendedHeaderOffset));
-			if (le16_to_cpu(extended_header->usSize) >=
-			    SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V3)
-				uvd_table_offset = le16_to_cpu(extended_header->usUVDTableOffset);
-		}
-	}
 	return uvd_table_offset;
 }
 
@@ -193,8 +237,14 @@ static uint16_t get_uvd_clock_info_array_size(struct pp_hwmgr *hwmgr,
 		const UVDClockInfoArray *p = (const UVDClockInfoArray *)
 					(((unsigned long) powerplay_table)
 					+ table_offset);
-		table_size = sizeof(UCHAR) +
-			     p->ucNumEntries * sizeof(UVDClockInfo);
+		size_t size;
+
+		if (!pp_table_has_space(hwmgr, table_offset, sizeof(p->ucNumEntries)))
+			return 0;
+
+		size = sizeof(UCHAR) + p->ucNumEntries * sizeof(UVDClockInfo);
+		if (pp_table_has_space(hwmgr, table_offset, size))
+			table_size = size;
 	}
 
 	return table_size;
@@ -206,10 +256,13 @@ static uint16_t get_uvd_clock_voltage_limit_table_offset(
 {
 	uint16_t table_offset = get_uvd_clock_info_array_offset(hwmgr,
 						     powerplay_table);
+	u16 table_size;
 
-	if (table_offset > 0)
-		return table_offset +
-			get_uvd_clock_info_array_size(hwmgr, powerplay_table);
+	if (table_offset > 0) {
+		table_size = get_uvd_clock_info_array_size(hwmgr, powerplay_table);
+		if (table_size)
+			return table_offset + table_size;
+	}
 
 	return 0;
 }
@@ -218,21 +271,12 @@ static uint16_t get_samu_table_offset(struct pp_hwmgr *hwmgr,
 			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
 {
 	uint16_t samu_table_offset = 0;
+	const ATOM_PPLIB_EXTENDEDHEADER *extended_header;
 
-	if (le16_to_cpu(powerplay_table->usTableSize) >=
-	    sizeof(ATOM_PPLIB_POWERPLAYTABLE3)) {
-		const ATOM_PPLIB_POWERPLAYTABLE3 *powerplay_table3 =
-			(const ATOM_PPLIB_POWERPLAYTABLE3 *)powerplay_table;
-		if (powerplay_table3->usExtendendedHeaderOffset > 0) {
-			const ATOM_PPLIB_EXTENDEDHEADER  *extended_header =
-				(const ATOM_PPLIB_EXTENDEDHEADER *)
-				(((unsigned long)powerplay_table3) +
-				le16_to_cpu(powerplay_table3->usExtendendedHeaderOffset));
-			if (le16_to_cpu(extended_header->usSize) >=
-			    SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V4)
-				samu_table_offset = le16_to_cpu(extended_header->usSAMUTableOffset);
-		}
-	}
+	extended_header = get_extended_header(hwmgr, powerplay_table,
+					      SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V4);
+	if (extended_header)
+		samu_table_offset = le16_to_cpu(extended_header->usSAMUTableOffset);
 
 	return samu_table_offset;
 }
@@ -254,21 +298,12 @@ static uint16_t get_acp_table_offset(struct pp_hwmgr *hwmgr,
 				const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
 {
 	uint16_t acp_table_offset = 0;
+	const ATOM_PPLIB_EXTENDEDHEADER *extended_header;
 
-	if (le16_to_cpu(powerplay_table->usTableSize) >=
-	    sizeof(ATOM_PPLIB_POWERPLAYTABLE3)) {
-		const ATOM_PPLIB_POWERPLAYTABLE3 *powerplay_table3 =
-			(const ATOM_PPLIB_POWERPLAYTABLE3 *)powerplay_table;
-		if (powerplay_table3->usExtendendedHeaderOffset > 0) {
-			const ATOM_PPLIB_EXTENDEDHEADER  *pExtendedHeader =
-				(const ATOM_PPLIB_EXTENDEDHEADER *)
-				(((unsigned long)powerplay_table3) +
-				le16_to_cpu(powerplay_table3->usExtendendedHeaderOffset));
-			if (le16_to_cpu(pExtendedHeader->usSize) >=
-			    SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V6)
-				acp_table_offset = le16_to_cpu(pExtendedHeader->usACPTableOffset);
-		}
-	}
+	extended_header = get_extended_header(hwmgr, powerplay_table,
+					      SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V6);
+	if (extended_header)
+		acp_table_offset = le16_to_cpu(extended_header->usACPTableOffset);
 
 	return acp_table_offset;
 }
@@ -290,21 +325,12 @@ static uint16_t get_cacp_tdp_table_offset(
 				const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
 {
 	uint16_t cacTdpTableOffset = 0;
+	const ATOM_PPLIB_EXTENDEDHEADER *extended_header;
 
-	if (le16_to_cpu(powerplay_table->usTableSize) >=
-	    sizeof(ATOM_PPLIB_POWERPLAYTABLE3)) {
-		const ATOM_PPLIB_POWERPLAYTABLE3 *powerplay_table3 =
-				(const ATOM_PPLIB_POWERPLAYTABLE3 *)powerplay_table;
-		if (powerplay_table3->usExtendendedHeaderOffset > 0) {
-			const ATOM_PPLIB_EXTENDEDHEADER  *pExtendedHeader =
-					(const ATOM_PPLIB_EXTENDEDHEADER *)
-					(((unsigned long)powerplay_table3) +
-				le16_to_cpu(powerplay_table3->usExtendendedHeaderOffset));
-			if (le16_to_cpu(pExtendedHeader->usSize) >=
-			    SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V7)
-				cacTdpTableOffset = le16_to_cpu(pExtendedHeader->usPowerTuneTableOffset);
-		}
-	}
+	extended_header = get_extended_header(hwmgr, powerplay_table,
+					      SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V7);
+	if (extended_header)
+		cacTdpTableOffset = le16_to_cpu(extended_header->usPowerTuneTableOffset);
 
 	return cacTdpTableOffset;
 }
@@ -341,22 +367,13 @@ static uint16_t get_sclk_vdd_gfx_table_offset(struct pp_hwmgr *hwmgr,
 			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
 {
 	uint16_t sclk_vdd_gfx_table_offset = 0;
+	const ATOM_PPLIB_EXTENDEDHEADER *extended_header;
 
-	if (le16_to_cpu(powerplay_table->usTableSize) >=
-	    sizeof(ATOM_PPLIB_POWERPLAYTABLE3)) {
-		const ATOM_PPLIB_POWERPLAYTABLE3 *powerplay_table3 =
-				(const ATOM_PPLIB_POWERPLAYTABLE3 *)powerplay_table;
-		if (powerplay_table3->usExtendendedHeaderOffset > 0) {
-			const ATOM_PPLIB_EXTENDEDHEADER  *pExtendedHeader =
-				(const ATOM_PPLIB_EXTENDEDHEADER *)
-				(((unsigned long)powerplay_table3) +
-				le16_to_cpu(powerplay_table3->usExtendendedHeaderOffset));
-			if (le16_to_cpu(pExtendedHeader->usSize) >=
-			    SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V8)
-				sclk_vdd_gfx_table_offset =
-					le16_to_cpu(pExtendedHeader->usSclkVddgfxTableOffset);
-		}
-	}
+	extended_header = get_extended_header(hwmgr, powerplay_table,
+					      SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V8);
+	if (extended_header)
+		sclk_vdd_gfx_table_offset =
+			le16_to_cpu(extended_header->usSclkVddgfxTableOffset);
 
 	return sclk_vdd_gfx_table_offset;
 }
@@ -769,20 +786,37 @@ static ULONG size_of_entry_v2(ULONG num_dpm_levels)
 }
 
 static const ATOM_PPLIB_STATE_V2 *get_state_entry_v2(
+					struct pp_hwmgr *hwmgr,
 					const StateArray * pstate_arrays,
+					u16 state_array_offset,
 							 ULONG entry_index)
 {
 	ULONG i;
 	const ATOM_PPLIB_STATE_V2 *pstate;
+	size_t entry_offset;
+	size_t entry_size;
+
+	if (entry_index >= pstate_arrays->ucNumEntries)
+		return NULL;
 
+	entry_offset = state_array_offset + sizeof(pstate_arrays->ucNumEntries);
 	pstate = pstate_arrays->states;
-	if (entry_index <= pstate_arrays->ucNumEntries) {
-		for (i = 0; i < entry_index; i++)
-			pstate = (ATOM_PPLIB_STATE_V2 *)(
-						  (unsigned long)pstate +
-			     size_of_entry_v2(pstate->ucNumDPMLevels));
+	for (i = 0; i <= entry_index; i++) {
+		if (!pp_table_has_space(hwmgr, entry_offset, sizeof(*pstate)))
+			return NULL;
+
+		entry_size = size_of_entry_v2(pstate->ucNumDPMLevels);
+		if (!pp_table_has_space(hwmgr, entry_offset, entry_size))
+			return NULL;
+
+		if (i == entry_index)
+			return pstate;
+
+		entry_offset += entry_size;
+		pstate = (ATOM_PPLIB_STATE_V2 *)((unsigned long)pstate + entry_size);
 	}
-	return pstate;
+
+	return NULL;
 }
 
 static const unsigned char soft_dummy_pp_table[] = {
@@ -850,6 +884,8 @@ int pp_tables_get_response_times(struct pp_hwmgr *hwmgr,
 
 	PP_ASSERT_WITH_CODE(NULL != powerplay_tab,
 			    "Missing PowerPlay Table!", return -EINVAL);
+	PP_ASSERT_WITH_CODE(pp_table_has_space(hwmgr, 0, sizeof(*powerplay_tab)),
+			    "Invalid PowerPlay Table!", return -EINVAL);
 
 	*vol_rep_time = (uint32_t)le16_to_cpu(powerplay_tab->usVoltageTime);
 	*bb_rep_time = (uint32_t)le16_to_cpu(powerplay_tab->usBackbiasTime);
@@ -862,13 +898,21 @@ int pp_tables_get_num_of_entries(struct pp_hwmgr *hwmgr,
 {
 	const StateArray *pstate_arrays;
 	const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table = get_powerplay_table(hwmgr);
+	u16 state_array_offset;
 
 	if (powerplay_table == NULL)
 		return -1;
+	if (!pp_table_has_space(hwmgr, 0, sizeof(*powerplay_table)))
+		return -1;
 
 	if (powerplay_table->sHeader.ucTableFormatRevision >= 6) {
+		state_array_offset = le16_to_cpu(powerplay_table->usStateArrayOffset);
+		if (!pp_table_has_space(hwmgr, state_array_offset,
+					sizeof(pstate_arrays->ucNumEntries)))
+			return -1;
+
 		pstate_arrays = (StateArray *)(((unsigned long)powerplay_table) +
-					le16_to_cpu(powerplay_table->usStateArrayOffset));
+					state_array_offset);
 
 		*num_of_entries = (unsigned long)(pstate_arrays->ucNumEntries);
 	} else
@@ -895,49 +939,128 @@ int pp_tables_get_entry(struct pp_hwmgr *hwmgr,
 	const NonClockInfoArray *pnon_clock_arrays;
 
 	const ATOM_PPLIB_STATE *pstate_entry;
+	u16 state_array_offset;
+	u16 clock_info_array_offset;
+	u16 non_clock_info_array_offset;
+	size_t clock_info_offset;
+	size_t non_clock_info_offset;
+	size_t state_entry_offset;
 
 	if (powerplay_table == NULL)
 		return -1;
+	if (!pp_table_has_space(hwmgr, 0, sizeof(*powerplay_table)))
+		return -1;
 
 	ps->classification.bios_index = entry_index;
 
 	if (powerplay_table->sHeader.ucTableFormatRevision >= 6) {
+		state_array_offset = le16_to_cpu(powerplay_table->usStateArrayOffset);
+		if (!pp_table_has_space(hwmgr, state_array_offset,
+					sizeof(pstate_arrays->ucNumEntries)))
+			return -1;
+
 		pstate_arrays = (StateArray *)(((unsigned long)powerplay_table) +
-					le16_to_cpu(powerplay_table->usStateArrayOffset));
+					state_array_offset);
+
+		if (entry_index >= pstate_arrays->ucNumEntries)
+			return -1;
 
-		if (entry_index > pstate_arrays->ucNumEntries)
+		pstate_entry_v2 = get_state_entry_v2(hwmgr, pstate_arrays,
+						     state_array_offset,
+						     entry_index);
+		if (!pstate_entry_v2)
+			return -1;
+
+		clock_info_array_offset =
+			le16_to_cpu(powerplay_table->usClockInfoArrayOffset);
+		if (!pp_table_has_space(hwmgr, clock_info_array_offset,
+					sizeof(*pclock_arrays)))
 			return -1;
 
-		pstate_entry_v2 = get_state_entry_v2(pstate_arrays, entry_index);
 		pclock_arrays = (ClockInfoArray *)(((unsigned long)powerplay_table) +
-					le16_to_cpu(powerplay_table->usClockInfoArrayOffset));
+					clock_info_array_offset);
+		if (!pclock_arrays->ucEntrySize)
+			return -1;
+
+		non_clock_info_array_offset =
+			le16_to_cpu(powerplay_table->usNonClockInfoArrayOffset);
+		if (!pp_table_has_space(hwmgr, non_clock_info_array_offset,
+					sizeof(*pnon_clock_arrays)))
+			return -1;
 
 		pnon_clock_arrays = (NonClockInfoArray *)(((unsigned long)powerplay_table) +
-						le16_to_cpu(powerplay_table->usNonClockInfoArrayOffset));
+						non_clock_info_array_offset);
+		if (!pnon_clock_arrays->ucEntrySize ||
+		    pnon_clock_arrays->ucEntrySize < ATOM_PPLIB_NONCLOCKINFO_VER1 ||
+		    (pnon_clock_arrays->ucEntrySize > ATOM_PPLIB_NONCLOCKINFO_VER1 &&
+		     pnon_clock_arrays->ucEntrySize < ATOM_PPLIB_NONCLOCKINFO_VER2) ||
+		    pstate_entry_v2->nonClockInfoIndex >= pnon_clock_arrays->ucNumEntries)
+			return -1;
 
+		non_clock_info_offset = non_clock_info_array_offset +
+			offsetof(NonClockInfoArray, nonClockInfo) +
+			pstate_entry_v2->nonClockInfoIndex * pnon_clock_arrays->ucEntrySize;
+		if (!pp_table_has_space(hwmgr, non_clock_info_offset,
+					pnon_clock_arrays->ucEntrySize))
+			return -1;
 		pnon_clock_info = (ATOM_PPLIB_NONCLOCK_INFO *)((unsigned long)(pnon_clock_arrays->nonClockInfo) +
 					(pstate_entry_v2->nonClockInfoIndex * pnon_clock_arrays->ucEntrySize));
 
 		result = init_non_clock_fields(hwmgr, ps, pnon_clock_arrays->ucEntrySize, pnon_clock_info);
 
 		for (i = 0; i < pstate_entry_v2->ucNumDPMLevels; i++) {
-			const void *pclock_info = (const void *)(
-							(unsigned long)(pclock_arrays->clockInfo) +
-							(pstate_entry_v2->clockInfoIndex[i] * pclock_arrays->ucEntrySize));
+			const void *pclock_info;
+
+			if (pstate_entry_v2->clockInfoIndex[i] >=
+			    pclock_arrays->ucNumEntries)
+				return -1;
+
+			clock_info_offset = clock_info_array_offset +
+				offsetof(ClockInfoArray, clockInfo) +
+				pstate_entry_v2->clockInfoIndex[i] * pclock_arrays->ucEntrySize;
+			if (!pp_table_has_space(hwmgr, clock_info_offset,
+						pclock_arrays->ucEntrySize))
+				return -1;
+
+			pclock_info = (const void *)
+					((unsigned long)(pclock_arrays->clockInfo) +
+					(pstate_entry_v2->clockInfoIndex[i] *
+					 pclock_arrays->ucEntrySize));
 			res = func(hwmgr, &ps->hardware, i, pclock_info);
 			if ((0 == result) && (0 != res))
 				result = res;
 		}
 	} else {
-		if (entry_index > powerplay_table->ucNumStates)
+		if (entry_index >= powerplay_table->ucNumStates ||
+		    !powerplay_table->ucStateEntrySize ||
+		    !powerplay_table->ucNonClockSize ||
+		    powerplay_table->ucNonClockSize < ATOM_PPLIB_NONCLOCKINFO_VER1 ||
+		    (powerplay_table->ucNonClockSize > ATOM_PPLIB_NONCLOCKINFO_VER1 &&
+		     powerplay_table->ucNonClockSize < ATOM_PPLIB_NONCLOCKINFO_VER2) ||
+		    !powerplay_table->ucClockInfoSize)
+			return -1;
+
+		state_array_offset = le16_to_cpu(powerplay_table->usStateArrayOffset);
+		state_entry_offset = state_array_offset +
+			entry_index * powerplay_table->ucStateEntrySize;
+		if (!pp_table_has_space(hwmgr, state_entry_offset,
+					powerplay_table->ucStateEntrySize))
 			return -1;
 
 		pstate_entry = (ATOM_PPLIB_STATE *)((unsigned long)powerplay_table +
-						    le16_to_cpu(powerplay_table->usStateArrayOffset) +
+						    state_array_offset +
 						    entry_index * powerplay_table->ucStateEntrySize);
 
+		non_clock_info_array_offset =
+			le16_to_cpu(powerplay_table->usNonClockInfoArrayOffset);
+		non_clock_info_offset = non_clock_info_array_offset +
+			pstate_entry->ucNonClockStateIndex * powerplay_table->ucNonClockSize;
+		if (!pp_table_has_space(hwmgr, non_clock_info_offset,
+					powerplay_table->ucNonClockSize))
+			return -1;
+
 		pnon_clock_info = (ATOM_PPLIB_NONCLOCK_INFO *)((unsigned long)powerplay_table +
-						le16_to_cpu(powerplay_table->usNonClockInfoArrayOffset) +
+						non_clock_info_array_offset +
 						pstate_entry->ucNonClockStateIndex *
 						powerplay_table->ucNonClockSize);
 
@@ -946,12 +1069,23 @@ int pp_tables_get_entry(struct pp_hwmgr *hwmgr,
 							pnon_clock_info);
 
 		for (i = 0; i < powerplay_table->ucStateEntrySize-1; i++) {
-			const void *pclock_info = (const void *)((unsigned long)powerplay_table +
-						le16_to_cpu(powerplay_table->usClockInfoArrayOffset) +
+			const void *pclock_info;
+
+			clock_info_array_offset =
+				le16_to_cpu(powerplay_table->usClockInfoArrayOffset);
+			clock_info_offset = clock_info_array_offset +
+				pstate_entry->ucClockStateIndices[i] *
+				powerplay_table->ucClockInfoSize;
+			if (!pp_table_has_space(hwmgr, clock_info_offset,
+						powerplay_table->ucClockInfoSize))
+				return -1;
+
+			pclock_info = (const void *)((unsigned long)powerplay_table +
+						clock_info_array_offset +
 						pstate_entry->ucClockStateIndices[i] *
 						powerplay_table->ucClockInfoSize);
 
-			int res = func(hwmgr, &ps->hardware, i, pclock_info);
+			res = func(hwmgr, &ps->hardware, i, pclock_info);
 
 			if ((0 == result) && (0 != res))
 					result = res;
@@ -1661,21 +1795,70 @@ static int get_vce_state_table_entry(struct pp_hwmgr *hwmgr,
 							unsigned long *flag)
 {
 	const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table = get_powerplay_table(hwmgr);
+	const ATOM_PPLIB_VCE_State_Table *vce_state_table;
+	const ATOM_PPLIB_VCE_State_Record *record;
+	const VCEClockInfoArray *vce_clock_info_array;
+	const VCEClockInfo *vce_clock_info;
+	const ClockInfoArray *clock_arrays;
+	u16 vce_state_table_offset;
+	u16 vce_clock_info_array_offset;
+	u16 clock_info_array_offset;
+	unsigned long clockInfoIndex;
+	size_t record_offset;
+	size_t vce_clock_info_offset;
+	size_t clock_info_offset;
+
+	if (!powerplay_table || !pp_table_has_space(hwmgr, 0, sizeof(*powerplay_table)))
+		return -1;
 
-	const ATOM_PPLIB_VCE_State_Table *vce_state_table = get_vce_state_table(hwmgr, powerplay_table);
+	vce_state_table_offset = get_vce_state_table_offset(hwmgr, powerplay_table);
+	vce_state_table = get_vce_state_table(hwmgr, powerplay_table);
+	if (!vce_state_table || i >= vce_state_table->numEntries)
+		return -1;
+
+	record_offset = vce_state_table_offset +
+		offsetof(ATOM_PPLIB_VCE_State_Table, entries) +
+		i * sizeof(*record);
+	if (!pp_table_has_space(hwmgr, record_offset, sizeof(*record)))
+		return -1;
+
+	record = &vce_state_table->entries[i];
+
+	vce_clock_info_array_offset =
+		get_vce_clock_info_array_offset(hwmgr, powerplay_table);
+	if (!pp_table_has_space(hwmgr, vce_clock_info_array_offset,
+				sizeof(vce_clock_info_array->ucNumEntries)))
+		return -1;
 
-	unsigned short vce_clock_info_array_offset = get_vce_clock_info_array_offset(hwmgr, powerplay_table);
+	vce_clock_info_array = (const VCEClockInfoArray *)
+		(((unsigned long)powerplay_table) + vce_clock_info_array_offset);
+	if (record->ucVCEClockInfoIndex >= vce_clock_info_array->ucNumEntries)
+		return -1;
 
-	const VCEClockInfoArray *vce_clock_info_array = (const VCEClockInfoArray *)(((unsigned long) powerplay_table) + vce_clock_info_array_offset);
+	vce_clock_info_offset = vce_clock_info_array_offset +
+		offsetof(VCEClockInfoArray, entries) +
+		record->ucVCEClockInfoIndex * sizeof(*vce_clock_info);
+	if (!pp_table_has_space(hwmgr, vce_clock_info_offset, sizeof(*vce_clock_info)))
+		return -1;
 
-	const ClockInfoArray *clock_arrays = (ClockInfoArray *)(((unsigned long)powerplay_table) +
-								le16_to_cpu(powerplay_table->usClockInfoArrayOffset));
+	vce_clock_info = &vce_clock_info_array->entries[record->ucVCEClockInfoIndex];
 
-	const ATOM_PPLIB_VCE_State_Record *record = &vce_state_table->entries[i];
+	clock_info_array_offset = le16_to_cpu(powerplay_table->usClockInfoArrayOffset);
+	if (!pp_table_has_space(hwmgr, clock_info_array_offset,
+				sizeof(*clock_arrays)))
+		return -1;
 
-	const VCEClockInfo *vce_clock_info = &vce_clock_info_array->entries[record->ucVCEClockInfoIndex];
+	clock_arrays = (ClockInfoArray *)(((unsigned long)powerplay_table) +
+					  clock_info_array_offset);
+	clockInfoIndex = record->ucClockInfoIndex & 0x3F;
+	if (!clock_arrays->ucEntrySize || clockInfoIndex >= clock_arrays->ucNumEntries)
+		return -1;
 
-	unsigned long clockInfoIndex = record->ucClockInfoIndex & 0x3F;
+	clock_info_offset = clock_info_array_offset +
+		offsetof(ClockInfoArray, clockInfo) +
+		clockInfoIndex * clock_arrays->ucEntrySize;
+	if (!pp_table_has_space(hwmgr, clock_info_offset, clock_arrays->ucEntrySize))
+		return -1;
 
 	*flag = (record->ucClockInfoIndex >> NUM_BITS_CLOCK_INFO_ARRAY_INDEX);
 
@@ -1699,6 +1882,10 @@ static int pp_tables_initialize(struct pp_hwmgr *hwmgr)
 	hwmgr->need_pp_table_upload = true;
 
 	powerplay_table = get_powerplay_table(hwmgr);
+	PP_ASSERT_WITH_CODE((powerplay_table),
+			    "Missing PowerPlay Table!", return -1);
+	PP_ASSERT_WITH_CODE(pp_table_has_space(hwmgr, 0, sizeof(*powerplay_table)),
+			    "Invalid PowerPlay Table!", return -1);
 
 	result = init_powerplay_tables(hwmgr, powerplay_table);
 
@@ -1801,4 +1988,3 @@ const struct pp_table_func pptable_funcs = {
 	.pptable_get_vce_state_table_entry =
 						get_vce_state_table_entry,
 };
-
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c
index 95bf187f02a5e..fbb59e5a4453a 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c
@@ -2216,12 +2216,24 @@ static int smu7_patch_voltage_dependency_tables_with_lookup_table(
 	if (data->vdd_gfx_control == SMU7_VOLTAGE_CONTROL_BY_SVID2) {
 		for (entry_id = 0; entry_id < sclk_table->count; ++entry_id) {
 			voltage_id = sclk_table->entries[entry_id].vddInd;
+			if (voltage_id >= table_info->vddgfx_lookup_table->count) {
+				pr_err("amdgpu: sclk[%u] vddgfx index %u out of bounds (%u)\n",
+				       entry_id, voltage_id,
+				       table_info->vddgfx_lookup_table->count);
+				return -EINVAL;
+			}
 			sclk_table->entries[entry_id].vddgfx =
 				table_info->vddgfx_lookup_table->entries[voltage_id].us_vdd;
 		}
 	} else {
 		for (entry_id = 0; entry_id < sclk_table->count; ++entry_id) {
 			voltage_id = sclk_table->entries[entry_id].vddInd;
+			if (voltage_id >= table_info->vddc_lookup_table->count) {
+				pr_err("amdgpu: sclk[%u] vddc index %u out of bounds (%u)\n",
+				       entry_id, voltage_id,
+				       table_info->vddc_lookup_table->count);
+				return -EINVAL;
+			}
 			sclk_table->entries[entry_id].vddc =
 				table_info->vddc_lookup_table->entries[voltage_id].us_vdd;
 		}
@@ -2229,12 +2241,24 @@ static int smu7_patch_voltage_dependency_tables_with_lookup_table(
 
 	for (entry_id = 0; entry_id < mclk_table->count; ++entry_id) {
 		voltage_id = mclk_table->entries[entry_id].vddInd;
+		if (voltage_id >= table_info->vddc_lookup_table->count) {
+			pr_err("amdgpu: mclk[%u] vddc index %u out of bounds (%u)\n",
+			       entry_id, voltage_id,
+			       table_info->vddc_lookup_table->count);
+			return -EINVAL;
+		}
 		mclk_table->entries[entry_id].vddc =
 			table_info->vddc_lookup_table->entries[voltage_id].us_vdd;
 	}
 
 	for (entry_id = 0; entry_id < mm_table->count; ++entry_id) {
 		voltage_id = mm_table->entries[entry_id].vddcInd;
+		if (voltage_id >= table_info->vddc_lookup_table->count) {
+			pr_err("amdgpu: mm[%u] vddc index %u out of bounds (%u)\n",
+			       entry_id, voltage_id,
+			       table_info->vddc_lookup_table->count);
+			return -EINVAL;
+		}
 		mm_table->entries[entry_id].vddc =
 			table_info->vddc_lookup_table->entries[voltage_id].us_vdd;
 	}
@@ -5648,23 +5672,29 @@ static int smu7_odn_edit_dpm_table(struct pp_hwmgr *hwmgr,
 	}
 
 	for (i = 0; i < size; i += 3) {
-		if (i + 3 > size || input[i] >= podn_dpm_table_in_backend->num_of_pl) {
-			pr_info("invalid clock voltage input \n");
-			return 0;
+		if (i + 3 > size) {
+			pr_info("truncated clock/voltage input\n");
+			return -EINVAL;
 		}
-		input_level = input[i];
-		input_clk = input[i+1] * 100;
-		input_vol = input[i+2];
-
-		if (smu7_check_clk_voltage_valid(hwmgr, type, input_clk, input_vol)) {
-			podn_dpm_table_in_backend->entries[input_level].clock = input_clk;
-			podn_vdd_dep_in_backend->entries[input_level].clk = input_clk;
-			podn_dpm_table_in_backend->entries[input_level].vddc = input_vol;
-			podn_vdd_dep_in_backend->entries[input_level].vddc = input_vol;
-			podn_vdd_dep_in_backend->entries[input_level].vddgfx = input_vol;
-		} else {
+		if (input[i] < 0 || input[i] >= podn_dpm_table_in_backend->num_of_pl) {
+			pr_info("invalid clock/voltage level\n");
 			return -EINVAL;
 		}
+		input_clk = input[i + 1] * 100;
+		input_vol = input[i + 2];
+		if (!smu7_check_clk_voltage_valid(hwmgr, type, input_clk, input_vol))
+			return -EINVAL;
+	}
+
+	for (i = 0; i < size; i += 3) {
+		input_level = input[i];
+		input_clk = input[i + 1] * 100;
+		input_vol = input[i + 2];
+		podn_dpm_table_in_backend->entries[input_level].clock = input_clk;
+		podn_vdd_dep_in_backend->entries[input_level].clk = input_clk;
+		podn_dpm_table_in_backend->entries[input_level].vddc = input_vol;
+		podn_vdd_dep_in_backend->entries[input_level].vddc = input_vol;
+		podn_vdd_dep_in_backend->entries[input_level].vddgfx = input_vol;
 	}
 
 	return 0;
@@ -5857,15 +5887,19 @@ static int smu7_power_off_asic(struct pp_hwmgr *hwmgr)
 static void smu7_notify_ac_dc(struct pp_hwmgr *hwmgr)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)(hwmgr->adev);
+	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
 
-	/* Check if the platform already manages the AC/DC switch via dedicated GPIO. */
-	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+	/*
+	 * Check if the platform already manages the AC/DC switch via dedicated GPIO.
+	 * Otherwise SMU automatically notices DC, but needs to be notified of AC.
+	 */
+	if (adev->pm.ac_power &&
+	    phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			    PHM_PlatformCaps_AutomaticDCTransition))
-		return;
-
-	/* The SMU automatically notices DC, but needs to be notified when switching to AC. */
-	if (adev->pm.ac_power)
 		smum_send_msg_to_smc(hwmgr, PPSMC_MSG_RunningOnAC, NULL);
+
+	/* Recompute clocks with updated max_limits. */
+	pp_funcs->pm_compute_clocks(adev->powerplay.pp_handle);
 }
 
 static const struct pp_hwmgr_func smu7_hwmgr_funcs = {
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c
index 4b92b52aba2b8..0e237feb1629b 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c
@@ -685,10 +685,18 @@ static int vega10_patch_voltage_dependency_tables_with_lookup_table(
 			case 3: vdt = table_info->vdd_dep_on_pixclk; break;
 			case 4: vdt = table_info->vdd_dep_on_dispclk; break;
 			case 5: vdt = table_info->vdd_dep_on_phyclk; break;
+			default:
+				continue;
 		}
 
 		for (entry_id = 0; entry_id < vdt->count; entry_id++) {
 			voltage_id = vdt->entries[entry_id].vddInd;
+			if (voltage_id >= table_info->vddc_lookup_table->count) {
+				pr_err("amdgpu: clk_dep[%u][%u] vddc index %u out of bounds (%u)\n",
+				       i, entry_id, voltage_id,
+				       table_info->vddc_lookup_table->count);
+				return -EINVAL;
+			}
 			vdt->entries[entry_id].vddc =
 					table_info->vddc_lookup_table->entries[voltage_id].us_vdd;
 		}
@@ -696,23 +704,48 @@ static int vega10_patch_voltage_dependency_tables_with_lookup_table(
 
 	for (entry_id = 0; entry_id < mm_table->count; ++entry_id) {
 		voltage_id = mm_table->entries[entry_id].vddcInd;
+		if (voltage_id >= table_info->vddc_lookup_table->count) {
+			pr_err("amdgpu: mm[%u] vddc index %u out of bounds (%u)\n",
+			       entry_id, voltage_id,
+			       table_info->vddc_lookup_table->count);
+			return -EINVAL;
+		}
 		mm_table->entries[entry_id].vddc =
 			table_info->vddc_lookup_table->entries[voltage_id].us_vdd;
 	}
 
 	for (entry_id = 0; entry_id < mclk_table->count; ++entry_id) {
 		voltage_id = mclk_table->entries[entry_id].vddInd;
+		if (voltage_id >= table_info->vddc_lookup_table->count) {
+			pr_err("amdgpu: mclk[%u] vddc index %u out of bounds (%u)\n",
+			       entry_id, voltage_id,
+			       table_info->vddc_lookup_table->count);
+			return -EINVAL;
+		}
 		mclk_table->entries[entry_id].vddc =
 				table_info->vddc_lookup_table->entries[voltage_id].us_vdd;
+
 		voltage_id = mclk_table->entries[entry_id].vddciInd;
+		if (voltage_id >= table_info->vddci_lookup_table->count) {
+			pr_err("amdgpu: mclk[%u] vddci index %u out of bounds (%u)\n",
+			       entry_id, voltage_id,
+			       table_info->vddci_lookup_table->count);
+			return -EINVAL;
+		}
 		mclk_table->entries[entry_id].vddci =
 				table_info->vddci_lookup_table->entries[voltage_id].us_vdd;
+
 		voltage_id = mclk_table->entries[entry_id].mvddInd;
+		if (voltage_id >= table_info->vddmem_lookup_table->count) {
+			pr_err("amdgpu: mclk[%u] vddmem index %u out of bounds (%u)\n",
+			       entry_id, voltage_id,
+			       table_info->vddmem_lookup_table->count);
+			return -EINVAL;
+		}
 		mclk_table->entries[entry_id].mvdd =
 				table_info->vddmem_lookup_table->entries[voltage_id].us_vdd;
 	}
 
-
 	return 0;
 
 }
@@ -5215,6 +5248,11 @@ static int vega10_set_power_profile_mode(struct pp_hwmgr *hwmgr, long *input, ui
 	uint8_t min_active_level;
 	uint32_t power_profile_mode = input[size];
 
+	if (power_profile_mode > PP_SMC_POWER_PROFILE_CUSTOM) {
+		pr_err("Invalid power profile mode %u\n", power_profile_mode);
+		return -EINVAL;
+	}
+
 	if (power_profile_mode == PP_SMC_POWER_PROFILE_CUSTOM) {
 		if (size != 0 && size != 4)
 			return -EINVAL;
@@ -5230,6 +5268,10 @@ static int vega10_set_power_profile_mode(struct pp_hwmgr *hwmgr, long *input, ui
 				return -EINVAL;
 		}
 
+		if ((input[0] & ~0xFF) || (input[1] & ~0xFF) ||
+		    (input[2] & ~0xFF) || (input[3] & ~0xFF))
+			return -EINVAL;
+
 		data->custom_profile_mode[0] = busy_set_point = input[0];
 		data->custom_profile_mode[1] = FPS = input[1];
 		data->custom_profile_mode[2] = use_rlc_busy = input[2];
@@ -5454,11 +5496,9 @@ static int vega10_odn_edit_dpm_table(struct pp_hwmgr *hwmgr,
 	if (PP_OD_EDIT_SCLK_VDDC_TABLE == type) {
 		dpm_table = &data->dpm_table.gfx_table;
 		podn_vdd_dep_table = &data->odn_dpm_table.vdd_dep_on_sclk;
-		data->need_update_dpm_table |= DPMTABLE_OD_UPDATE_SCLK;
 	} else if (PP_OD_EDIT_MCLK_VDDC_TABLE == type) {
 		dpm_table = &data->dpm_table.mem_table;
 		podn_vdd_dep_table = &data->odn_dpm_table.vdd_dep_on_mclk;
-		data->need_update_dpm_table |= DPMTABLE_OD_UPDATE_MCLK;
 	} else if (PP_OD_RESTORE_DEFAULT_TABLE == type) {
 		memcpy(&(data->dpm_table), &(data->golden_dpm_table), sizeof(struct vega10_dpm_table));
 		vega10_odn_initial_default_setting(hwmgr);
@@ -5476,21 +5516,32 @@ static int vega10_odn_edit_dpm_table(struct pp_hwmgr *hwmgr,
 	}
 
 	for (i = 0; i < size; i += 3) {
-		if (i + 3 > size || input[i] >= podn_vdd_dep_table->count) {
-			pr_info("invalid clock voltage input\n");
-			return 0;
+		if (i + 3 > size) {
+			pr_info("truncated clock/voltage input\n");
+			return -EINVAL;
 		}
-		input_level = input[i];
-		input_clk = input[i+1] * 100;
-		input_vol = input[i+2];
-
-		if (vega10_check_clk_voltage_valid(hwmgr, type, input_clk, input_vol)) {
-			dpm_table->dpm_levels[input_level].value = input_clk;
-			podn_vdd_dep_table->entries[input_level].clk = input_clk;
-			podn_vdd_dep_table->entries[input_level].vddc = input_vol;
-		} else {
+		if (input[i] < 0 || input[i] >= podn_vdd_dep_table->count) {
+			pr_info("invalid clock/voltage level\n");
 			return -EINVAL;
 		}
+		input_clk = input[i + 1] * 100;
+		input_vol = input[i + 2];
+		if (!vega10_check_clk_voltage_valid(hwmgr, type, input_clk, input_vol))
+			return -EINVAL;
+	}
+
+	if (type == PP_OD_EDIT_SCLK_VDDC_TABLE)
+		data->need_update_dpm_table |= DPMTABLE_OD_UPDATE_SCLK;
+	else
+		data->need_update_dpm_table |= DPMTABLE_OD_UPDATE_MCLK;
+
+	for (i = 0; i < size; i += 3) {
+		input_level = input[i];
+		input_clk = input[i + 1] * 100;
+		input_vol = input[i + 2];
+		dpm_table->dpm_levels[input_level].value = input_clk;
+		podn_vdd_dep_table->entries[input_level].clk = input_clk;
+		podn_vdd_dep_table->entries[input_level].vddc = input_vol;
 	}
 	vega10_odn_update_soc_table(hwmgr, type);
 	return 0;
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_processpptables.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_processpptables.c
index 052d139584fd1..62b1e068f90db 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_processpptables.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_processpptables.c
@@ -63,14 +63,57 @@ static const void *get_powerplay_table(struct pp_hwmgr *hwmgr)
 	return table_address;
 }
 
-static int check_powerplay_tables(
-		struct pp_hwmgr *hwmgr,
-		const ATOM_Vega10_POWERPLAYTABLE *powerplay_table)
+static bool vega10_pp_table_has_space(struct pp_hwmgr *hwmgr, size_t offset,
+				      size_t size)
+{
+	size_t table_size = hwmgr->soft_pp_table_size;
+
+	return offset <= table_size && size <= table_size - offset;
+}
+
+static int get_vega10_subtable(struct pp_hwmgr *hwmgr,
+		const ATOM_Vega10_POWERPLAYTABLE *powerplay_table,
+		u16 table_offset, size_t table_size, const void **table)
+{
+	PP_ASSERT_WITH_CODE((table_offset != 0),
+			    "Invalid PowerPlay Table!", return -1);
+	PP_ASSERT_WITH_CODE((vega10_pp_table_has_space(hwmgr, table_offset,
+						       table_size)),
+			    "Invalid PowerPlay Table!", return -1);
+
+	*table = (const void *)(((unsigned long)powerplay_table) + table_offset);
+
+	return 0;
+}
+
+static int validate_vega10_table_entries(struct pp_hwmgr *hwmgr,
+		u16 table_offset, size_t entries_offset,
+		u8 num_entries, size_t entry_size)
+{
+	size_t table_size;
+
+	PP_ASSERT_WITH_CODE((num_entries != 0),
+			    "Invalid PowerPlay Table!", return -1);
+
+	table_size = entries_offset + num_entries * entry_size;
+	PP_ASSERT_WITH_CODE((vega10_pp_table_has_space(hwmgr, table_offset,
+						       table_size)),
+			    "Invalid PowerPlay Table!", return -1);
+
+	return 0;
+}
+
+static int get_vega10_state_array(struct pp_hwmgr *hwmgr,
+	const ATOM_Vega10_POWERPLAYTABLE *powerplay_table,
+	const ATOM_Vega10_State_Array **state_array)
 {
 	const ATOM_Vega10_State_Array *state_arrays;
+	u16 state_array_offset;
+	size_t state_array_size;
+	size_t table_size = hwmgr->soft_pp_table_size;
 
-	state_arrays = (ATOM_Vega10_State_Array *)(((unsigned long)powerplay_table) +
-		le16_to_cpu(powerplay_table->usStateArrayOffset));
+	PP_ASSERT_WITH_CODE((table_size >= sizeof(*powerplay_table)),
+			    "Invalid PowerPlay Table!", return -1);
 
 	PP_ASSERT_WITH_CODE((powerplay_table->sHeader.format_revision >=
 			ATOM_Vega10_TABLE_REVISION_VEGA10),
@@ -79,12 +122,321 @@ static int check_powerplay_tables(
 		"State table is not set!", return -1);
 	PP_ASSERT_WITH_CODE(powerplay_table->sHeader.structuresize > 0,
 		"Invalid PowerPlay Table!", return -1);
+
+	state_array_offset = le16_to_cpu(powerplay_table->usStateArrayOffset);
+	PP_ASSERT_WITH_CODE((state_array_offset <=
+			     table_size - sizeof(*state_arrays)),
+			    "Invalid PowerPlay Table!", return -1);
+
+	state_arrays = (ATOM_Vega10_State_Array *)(((unsigned long)powerplay_table) +
+		state_array_offset);
 	PP_ASSERT_WITH_CODE(state_arrays->ucNumEntries > 0,
 		"Invalid PowerPlay Table!", return -1);
 
+	state_array_size = struct_size(state_arrays, states, state_arrays->ucNumEntries);
+	PP_ASSERT_WITH_CODE((state_array_size <= table_size - state_array_offset),
+			    "Invalid PowerPlay Table!", return -1);
+
+	*state_array = state_arrays;
+
 	return 0;
 }
 
+static int get_vega10_gfxclk_dependency_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Vega10_POWERPLAYTABLE *powerplay_table,
+		const ATOM_Vega10_GFXCLK_Dependency_Table **gfxclk_dep_table)
+{
+	const ATOM_Vega10_GFXCLK_Dependency_Table *table;
+	u16 table_offset;
+	size_t table_size;
+	size_t entry_size;
+
+	table_offset = le16_to_cpu(powerplay_table->usGfxclkDependencyTableOffset);
+	if (!table_offset)
+		return -EINVAL;
+
+	PP_ASSERT_WITH_CODE((vega10_pp_table_has_space(hwmgr, table_offset,
+						       sizeof(*table))),
+			    "Invalid PowerPlay Table!", return -1);
+
+	table = (const ATOM_Vega10_GFXCLK_Dependency_Table *)
+		(((unsigned long)powerplay_table) + table_offset);
+	PP_ASSERT_WITH_CODE((table->ucNumEntries != 0),
+			    "Invalid PowerPlay Table!", return -1);
+
+	if (table->ucRevId == 0)
+		entry_size = sizeof(ATOM_Vega10_GFXCLK_Dependency_Record);
+	else if (table->ucRevId == 1)
+		entry_size = sizeof(ATOM_Vega10_GFXCLK_Dependency_Record_V2);
+	else
+		PP_ASSERT_WITH_CODE(false,
+			"Unsupported GFXClockDependencyTable Revision!",
+			return -EINVAL);
+
+	table_size = offsetof(ATOM_Vega10_GFXCLK_Dependency_Table, entries) +
+		table->ucNumEntries * entry_size;
+	PP_ASSERT_WITH_CODE((vega10_pp_table_has_space(hwmgr, table_offset,
+						       table_size)),
+			    "Invalid PowerPlay Table!", return -1);
+
+	*gfxclk_dep_table = table;
+
+	return 0;
+}
+
+static int get_vega10_clk_dependency_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Vega10_POWERPLAYTABLE *powerplay_table,
+		u16 table_offset,
+		const ATOM_Vega10_SOCCLK_Dependency_Table **clk_dep_table)
+{
+	const ATOM_Vega10_SOCCLK_Dependency_Table *table;
+	int ret;
+
+	ret = get_vega10_subtable(hwmgr, powerplay_table, table_offset,
+				  sizeof(*table), (const void **)&table);
+	if (ret)
+		return ret;
+
+	ret = validate_vega10_table_entries(hwmgr, table_offset,
+					    offsetof(ATOM_Vega10_SOCCLK_Dependency_Table,
+						     entries),
+					    table->ucNumEntries,
+					    sizeof(ATOM_Vega10_CLK_Dependency_Record));
+	if (ret)
+		return ret;
+
+	*clk_dep_table = table;
+
+	return 0;
+}
+
+static int get_vega10_mclk_dependency_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Vega10_POWERPLAYTABLE *powerplay_table,
+		const ATOM_Vega10_MCLK_Dependency_Table **mclk_dep_table)
+{
+	const ATOM_Vega10_MCLK_Dependency_Table *table;
+	u16 table_offset;
+	int ret;
+
+	table_offset = le16_to_cpu(powerplay_table->usMclkDependencyTableOffset);
+	ret = get_vega10_subtable(hwmgr, powerplay_table, table_offset,
+				  sizeof(*table), (const void **)&table);
+	if (ret)
+		return ret;
+
+	ret = validate_vega10_table_entries(hwmgr, table_offset,
+					    offsetof(ATOM_Vega10_MCLK_Dependency_Table,
+						     entries),
+					    table->ucNumEntries,
+					    sizeof(ATOM_Vega10_MCLK_Dependency_Record));
+	if (ret)
+		return ret;
+
+	*mclk_dep_table = table;
+
+	return 0;
+}
+
+static int get_vega10_mm_dependency_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Vega10_POWERPLAYTABLE *powerplay_table,
+		const ATOM_Vega10_MM_Dependency_Table **mm_dep_table)
+{
+	const ATOM_Vega10_MM_Dependency_Table *table;
+	u16 table_offset;
+	int ret;
+
+	table_offset = le16_to_cpu(powerplay_table->usMMDependencyTableOffset);
+	ret = get_vega10_subtable(hwmgr, powerplay_table, table_offset,
+				  sizeof(*table), (const void **)&table);
+	if (ret)
+		return ret;
+
+	ret = validate_vega10_table_entries(hwmgr, table_offset,
+					    offsetof(ATOM_Vega10_MM_Dependency_Table,
+						     entries),
+					    table->ucNumEntries,
+					    sizeof(ATOM_Vega10_MM_Dependency_Record));
+	if (ret)
+		return ret;
+
+	*mm_dep_table = table;
+
+	return 0;
+}
+
+static int get_vega10_pcie_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Vega10_POWERPLAYTABLE *powerplay_table,
+		const Vega10_PPTable_Generic_SubTable_Header **pcie_table)
+{
+	const ATOM_Vega10_PCIE_Table *table;
+	u16 table_offset;
+	int ret;
+
+	table_offset = le16_to_cpu(powerplay_table->usPCIETableOffset);
+	ret = get_vega10_subtable(hwmgr, powerplay_table, table_offset,
+				  sizeof(*table), (const void **)&table);
+	if (ret)
+		return ret;
+
+	if (!table->ucNumEntries) {
+		*pcie_table = (const Vega10_PPTable_Generic_SubTable_Header *)table;
+		return 0;
+	}
+
+	ret = validate_vega10_table_entries(hwmgr, table_offset,
+					    offsetof(ATOM_Vega10_PCIE_Table,
+						     entries),
+					    table->ucNumEntries,
+					    sizeof(ATOM_Vega10_PCIE_Record));
+	if (ret)
+		return ret;
+
+	*pcie_table = (const Vega10_PPTable_Generic_SubTable_Header *)table;
+
+	return 0;
+}
+
+static int get_vega10_hard_limit_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Vega10_POWERPLAYTABLE *powerplay_table,
+		const ATOM_Vega10_Hard_Limit_Table **hard_limit_table)
+{
+	const ATOM_Vega10_Hard_Limit_Table *table;
+	u16 table_offset;
+	int ret;
+
+	table_offset = le16_to_cpu(powerplay_table->usHardLimitTableOffset);
+	ret = get_vega10_subtable(hwmgr, powerplay_table, table_offset,
+				  sizeof(*table), (const void **)&table);
+	if (ret)
+		return ret;
+
+	ret = validate_vega10_table_entries(hwmgr, table_offset,
+					    offsetof(ATOM_Vega10_Hard_Limit_Table,
+						     entries),
+					    table->ucNumEntries,
+					    sizeof(ATOM_Vega10_Hard_Limit_Record));
+	if (ret)
+		return ret;
+
+	*hard_limit_table = table;
+
+	return 0;
+}
+
+static int get_vega10_thermal_controller_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Vega10_POWERPLAYTABLE *powerplay_table,
+		const ATOM_Vega10_Thermal_Controller **thermal_controller)
+{
+	u16 table_offset;
+
+	table_offset = le16_to_cpu(powerplay_table->usThermalControllerOffset);
+
+	return get_vega10_subtable(hwmgr, powerplay_table, table_offset,
+				   sizeof(**thermal_controller),
+				   (const void **)thermal_controller);
+}
+
+static int get_vega10_fan_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Vega10_POWERPLAYTABLE *powerplay_table,
+		const Vega10_PPTable_Generic_SubTable_Header **fan_table)
+{
+	const Vega10_PPTable_Generic_SubTable_Header *header;
+	u16 table_offset;
+	size_t table_size;
+	int ret;
+
+	table_offset = le16_to_cpu(powerplay_table->usFanTableOffset);
+	ret = get_vega10_subtable(hwmgr, powerplay_table, table_offset,
+				  sizeof(*header), (const void **)&header);
+	if (ret)
+		return ret;
+
+	if (header->ucRevId == 10)
+		table_size = sizeof(ATOM_Vega10_Fan_Table);
+	else if (header->ucRevId == 0xb)
+		table_size = sizeof(ATOM_Vega10_Fan_Table_V2);
+	else if (header->ucRevId > 0xb)
+		table_size = sizeof(ATOM_Vega10_Fan_Table_V3);
+	else
+		table_size = sizeof(*header);
+
+	PP_ASSERT_WITH_CODE((vega10_pp_table_has_space(hwmgr, table_offset,
+						       table_size)),
+			    "Invalid PowerPlay Table!", return -1);
+
+	*fan_table = header;
+
+	return 0;
+}
+
+static int get_vega10_power_tune_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Vega10_POWERPLAYTABLE *powerplay_table,
+		const Vega10_PPTable_Generic_SubTable_Header **power_tune_table)
+{
+	const Vega10_PPTable_Generic_SubTable_Header *header;
+	u16 table_offset;
+	size_t table_size;
+	int ret;
+
+	table_offset = le16_to_cpu(powerplay_table->usPowerTuneTableOffset);
+	ret = get_vega10_subtable(hwmgr, powerplay_table, table_offset,
+				  sizeof(*header), (const void **)&header);
+	if (ret)
+		return ret;
+
+	if (header->ucRevId == 5)
+		table_size = sizeof(ATOM_Vega10_PowerTune_Table);
+	else if (header->ucRevId == 6)
+		table_size = sizeof(ATOM_Vega10_PowerTune_Table_V2);
+	else
+		table_size = sizeof(ATOM_Vega10_PowerTune_Table_V3);
+
+	PP_ASSERT_WITH_CODE((vega10_pp_table_has_space(hwmgr, table_offset,
+						       table_size)),
+			    "Invalid PowerPlay Table!", return -1);
+
+	*power_tune_table = header;
+
+	return 0;
+}
+
+static int get_vega10_voltage_lookup_table(struct pp_hwmgr *hwmgr,
+		const ATOM_Vega10_POWERPLAYTABLE *powerplay_table,
+		u16 table_offset, uint32_t max_levels,
+		const ATOM_Vega10_Voltage_Lookup_Table **lookup_table)
+{
+	const ATOM_Vega10_Voltage_Lookup_Table *table;
+	size_t table_size;
+	int ret;
+
+	ret = get_vega10_subtable(hwmgr, powerplay_table, table_offset,
+				  sizeof(*table), (const void **)&table);
+	if (ret)
+		return ret;
+
+	PP_ASSERT_WITH_CODE((table->ucNumEntries != 0 &&
+			     table->ucNumEntries <= max_levels),
+			    "Invalid PowerPlay Table!", return -1);
+
+	table_size = offsetof(ATOM_Vega10_Voltage_Lookup_Table, entries) +
+		table->ucNumEntries * sizeof(ATOM_Vega10_Voltage_Lookup_Record);
+	PP_ASSERT_WITH_CODE((vega10_pp_table_has_space(hwmgr, table_offset,
+						       table_size)),
+			    "Invalid PowerPlay Table!", return -1);
+
+	*lookup_table = table;
+
+	return 0;
+}
+
+static int check_powerplay_tables(struct pp_hwmgr *hwmgr,
+	const ATOM_Vega10_POWERPLAYTABLE *powerplay_table)
+{
+	const ATOM_Vega10_State_Array *state_arrays;
+
+	return get_vega10_state_array(hwmgr, powerplay_table, &state_arrays);
+}
+
 static int set_platform_caps(struct pp_hwmgr *hwmgr, uint32_t powerplay_caps)
 {
 	set_hw_cap(
@@ -124,14 +476,16 @@ static int init_thermal_controller(
 	const ATOM_Vega10_Fan_Table *fan_table_v1;
 	const ATOM_Vega10_Fan_Table_V2 *fan_table_v2;
 	const ATOM_Vega10_Fan_Table_V3 *fan_table_v3;
-
-	thermal_controller = (ATOM_Vega10_Thermal_Controller *)
-			(((unsigned long)powerplay_table) +
-			le16_to_cpu(powerplay_table->usThermalControllerOffset));
+	int ret;
 
 	PP_ASSERT_WITH_CODE((powerplay_table->usThermalControllerOffset != 0),
 			"Thermal controller table not set!", return -EINVAL);
 
+	ret = get_vega10_thermal_controller_table(hwmgr, powerplay_table,
+						  &thermal_controller);
+	if (ret)
+		return ret;
+
 	hwmgr->thermal_controller.ucType = thermal_controller->ucType;
 	hwmgr->thermal_controller.ucI2cLine = thermal_controller->ucI2cLine;
 	hwmgr->thermal_controller.ucI2cAddress = thermal_controller->ucI2cAddress;
@@ -160,9 +514,9 @@ static int init_thermal_controller(
 	if (!powerplay_table->usFanTableOffset)
 		return 0;
 
-	header = (const Vega10_PPTable_Generic_SubTable_Header *)
-			(((unsigned long)powerplay_table) +
-			le16_to_cpu(powerplay_table->usFanTableOffset));
+	ret = get_vega10_fan_table(hwmgr, powerplay_table, &header);
+	if (ret)
+		return ret;
 
 	if (header->ucRevId == 10) {
 		fan_table_v1 = (ATOM_Vega10_Fan_Table *)header;
@@ -307,12 +661,15 @@ static int init_over_drive_limits(
 		struct pp_hwmgr *hwmgr,
 		const ATOM_Vega10_POWERPLAYTABLE *powerplay_table)
 {
-	const ATOM_Vega10_GFXCLK_Dependency_Table *gfxclk_dep_table =
-			(const ATOM_Vega10_GFXCLK_Dependency_Table *)
-			(((unsigned long) powerplay_table) +
-			le16_to_cpu(powerplay_table->usGfxclkDependencyTableOffset));
+	const ATOM_Vega10_GFXCLK_Dependency_Table *gfxclk_dep_table;
 	bool is_acg_enabled = false;
 	ATOM_Vega10_GFXCLK_Dependency_Record_V2 *patom_record_v2;
+	int ret;
+
+	ret = get_vega10_gfxclk_dependency_table(hwmgr, powerplay_table,
+						 &gfxclk_dep_table);
+	if (ret)
+		return ret;
 
 	if (gfxclk_dep_table->ucRevId == 1) {
 		patom_record_v2 =
@@ -344,20 +701,28 @@ static int get_mm_clock_voltage_table(
 		const ATOM_Vega10_MM_Dependency_Table *mm_dependency_table)
 {
 	uint32_t i;
+	uint32_t num_entries;
 	const ATOM_Vega10_MM_Dependency_Record *mm_dependency_record;
 	phm_ppt_v1_mm_clock_voltage_dependency_table *mm_table;
 
 	PP_ASSERT_WITH_CODE((mm_dependency_table->ucNumEntries != 0),
 			"Invalid PowerPlay Table!", return -1);
 
-	mm_table = kzalloc_flex(*mm_table, entries,
-				mm_dependency_table->ucNumEntries);
+	num_entries = min_t(uint32_t, mm_dependency_table->ucNumEntries,
+			    pp_entries_max(hwmgr, mm_dependency_table,
+					   sizeof(*mm_dependency_table),
+					   sizeof(ATOM_Vega10_MM_Dependency_Record)));
+	if (num_entries < mm_dependency_table->ucNumEntries)
+		pr_warn("amdgpu: Vega10 MM dependency table: clamping ucNumEntries %u -> %u\n",
+			mm_dependency_table->ucNumEntries, num_entries);
+
+	mm_table = kzalloc_flex(*mm_table, entries, num_entries);
 	if (!mm_table)
 		return -ENOMEM;
 
-	mm_table->count = mm_dependency_table->ucNumEntries;
+	mm_table->count = num_entries;
 
-	for (i = 0; i < mm_dependency_table->ucNumEntries; i++) {
+	for (i = 0; i < num_entries; i++) {
 		mm_dependency_record = &mm_dependency_table->entries[i];
 		mm_table->entries[i].vddcInd = mm_dependency_record->ucVddcInd;
 		mm_table->entries[i].samclock =
@@ -568,19 +933,27 @@ static int get_socclk_voltage_dependency_table(
 		const ATOM_Vega10_SOCCLK_Dependency_Table *clk_dep_table)
 {
 	uint32_t i;
+	uint32_t num_entries;
 	phm_ppt_v1_clock_voltage_dependency_table *clk_table;
 
 	PP_ASSERT_WITH_CODE(clk_dep_table->ucNumEntries,
 		"Invalid PowerPlay Table!", return -1);
 
-	clk_table = kzalloc_flex(*clk_table, entries,
-				 clk_dep_table->ucNumEntries);
+	num_entries = min_t(uint32_t, clk_dep_table->ucNumEntries,
+			    pp_entries_max(hwmgr, clk_dep_table,
+					   sizeof(*clk_dep_table),
+					   sizeof(ATOM_Vega10_CLK_Dependency_Record)));
+	if (num_entries < clk_dep_table->ucNumEntries)
+		pr_warn("amdgpu: Vega10 SOCCLK dependency table: clamping ucNumEntries %u -> %u\n",
+			clk_dep_table->ucNumEntries, num_entries);
+
+	clk_table = kzalloc_flex(*clk_table, entries, num_entries);
 	if (!clk_table)
 		return -ENOMEM;
 
-	clk_table->count = (uint32_t)clk_dep_table->ucNumEntries;
+	clk_table->count = num_entries;
 
-	for (i = 0; i < clk_dep_table->ucNumEntries; i++) {
+	for (i = 0; i < num_entries; i++) {
 		clk_table->entries[i].vddInd =
 				clk_dep_table->entries[i].ucVddInd;
 		clk_table->entries[i].clk =
@@ -598,19 +971,27 @@ static int get_mclk_voltage_dependency_table(
 		const ATOM_Vega10_MCLK_Dependency_Table *mclk_dep_table)
 {
 	uint32_t i;
+	uint32_t num_entries;
 	phm_ppt_v1_clock_voltage_dependency_table *mclk_table;
 
 	PP_ASSERT_WITH_CODE(mclk_dep_table->ucNumEntries,
 		"Invalid PowerPlay Table!", return -1);
 
-	mclk_table = kzalloc_flex(*mclk_table, entries,
-				  mclk_dep_table->ucNumEntries);
+	num_entries = min_t(uint32_t, mclk_dep_table->ucNumEntries,
+			    pp_entries_max(hwmgr, mclk_dep_table,
+					   sizeof(*mclk_dep_table),
+					   sizeof(ATOM_Vega10_MCLK_Dependency_Record)));
+	if (num_entries < mclk_dep_table->ucNumEntries)
+		pr_warn("amdgpu: Vega10 MCLK dependency table: clamping ucNumEntries %u -> %u\n",
+			mclk_dep_table->ucNumEntries, num_entries);
+
+	mclk_table = kzalloc_flex(*mclk_table, entries, num_entries);
 	if (!mclk_table)
 		return -ENOMEM;
 
-	mclk_table->count = (uint32_t)mclk_dep_table->ucNumEntries;
+	mclk_table->count = num_entries;
 
-	for (i = 0; i < mclk_dep_table->ucNumEntries; i++) {
+	for (i = 0; i < num_entries; i++) {
 		mclk_table->entries[i].vddInd =
 				mclk_dep_table->entries[i].ucVddInd;
 		mclk_table->entries[i].vddciInd =
@@ -633,6 +1014,7 @@ static int get_gfxclk_voltage_dependency_table(
 		const ATOM_Vega10_GFXCLK_Dependency_Table *clk_dep_table)
 {
 	uint32_t i;
+	uint32_t num_entries;
 	struct phm_ppt_v1_clock_voltage_dependency_table
 				*clk_table;
 	ATOM_Vega10_GFXCLK_Dependency_Record_V2 *patom_record_v2;
@@ -640,15 +1022,34 @@ static int get_gfxclk_voltage_dependency_table(
 	PP_ASSERT_WITH_CODE((clk_dep_table->ucNumEntries != 0),
 			"Invalid PowerPlay Table!", return -1);
 
-	clk_table = kzalloc_flex(*clk_table, entries,
-				 clk_dep_table->ucNumEntries);
+	if (clk_dep_table->ucRevId == 0) {
+		num_entries = min_t(uint32_t, clk_dep_table->ucNumEntries,
+				    pp_entries_max(hwmgr, clk_dep_table,
+						   sizeof(*clk_dep_table),
+						   sizeof(ATOM_Vega10_GFXCLK_Dependency_Record)));
+	} else if (clk_dep_table->ucRevId == 1) {
+		num_entries = min_t(uint32_t, clk_dep_table->ucNumEntries,
+				    pp_entries_max(hwmgr, clk_dep_table,
+						   sizeof(*clk_dep_table),
+						   sizeof(ATOM_Vega10_GFXCLK_Dependency_Record_V2)));
+	} else {
+		PP_ASSERT_WITH_CODE(false,
+			"Unsupported GFXClockDependencyTable Revision!",
+			return -EINVAL);
+	}
+
+	if (num_entries < clk_dep_table->ucNumEntries)
+		pr_warn("amdgpu: Vega10 GFXCLK dependency table: clamping ucNumEntries %u -> %u\n",
+			clk_dep_table->ucNumEntries, num_entries);
+
+	clk_table = kzalloc_flex(*clk_table, entries, num_entries);
 	if (!clk_table)
 		return -ENOMEM;
 
-	clk_table->count = clk_dep_table->ucNumEntries;
+	clk_table->count = num_entries;
 
 	if (clk_dep_table->ucRevId == 0) {
-		for (i = 0; i < clk_table->count; i++) {
+		for (i = 0; i < num_entries; i++) {
 			clk_table->entries[i].vddInd =
 				clk_dep_table->entries[i].ucVddInd;
 			clk_table->entries[i].clk =
@@ -661,9 +1062,9 @@ static int get_gfxclk_voltage_dependency_table(
 			clk_table->entries[i].sclk_offset =
 				le16_to_cpu(clk_dep_table->entries[i].usAVFSOffset);
 		}
-	} else if (clk_dep_table->ucRevId == 1) {
+	} else {
 		patom_record_v2 = (ATOM_Vega10_GFXCLK_Dependency_Record_V2 *)clk_dep_table->entries;
-		for (i = 0; i < clk_table->count; i++) {
+		for (i = 0; i < num_entries; i++) {
 			clk_table->entries[i].vddInd =
 					patom_record_v2->ucVddInd;
 			clk_table->entries[i].clk =
@@ -677,11 +1078,6 @@ static int get_gfxclk_voltage_dependency_table(
 					le16_to_cpu(patom_record_v2->usAVFSOffset);
 			patom_record_v2++;
 		}
-	} else {
-		kfree(clk_table);
-		PP_ASSERT_WITH_CODE(false,
-			"Unsupported GFXClockDependencyTable Revision!",
-			return -EINVAL);
 	}
 
 	*pp_vega10_clk_dep_table = clk_table;
@@ -696,20 +1092,28 @@ static int get_pix_clk_voltage_dependency_table(
 		const  ATOM_Vega10_PIXCLK_Dependency_Table *clk_dep_table)
 {
 	uint32_t i;
+	uint32_t num_entries;
 	struct phm_ppt_v1_clock_voltage_dependency_table
 				*clk_table;
 
 	PP_ASSERT_WITH_CODE((clk_dep_table->ucNumEntries != 0),
 			"Invalid PowerPlay Table!", return -1);
 
-	clk_table = kzalloc_flex(*clk_table, entries,
-				 clk_dep_table->ucNumEntries);
+	num_entries = min_t(uint32_t, clk_dep_table->ucNumEntries,
+			    pp_entries_max(hwmgr, clk_dep_table,
+					   sizeof(*clk_dep_table),
+					   sizeof(ATOM_Vega10_CLK_Dependency_Record)));
+	if (num_entries < clk_dep_table->ucNumEntries)
+		pr_warn("amdgpu: Vega10 PIXCLK dependency table: clamping ucNumEntries %u -> %u\n",
+			clk_dep_table->ucNumEntries, num_entries);
+
+	clk_table = kzalloc_flex(*clk_table, entries, num_entries);
 	if (!clk_table)
 		return -ENOMEM;
 
-	clk_table->count = clk_dep_table->ucNumEntries;
+	clk_table->count = num_entries;
 
-	for (i = 0; i < clk_table->count; i++) {
+	for (i = 0; i < num_entries; i++) {
 		clk_table->entries[i].vddInd =
 				clk_dep_table->entries[i].ucVddInd;
 		clk_table->entries[i].clk =
@@ -728,6 +1132,7 @@ static int get_dcefclk_voltage_dependency_table(
 		const ATOM_Vega10_DCEFCLK_Dependency_Table *clk_dep_table)
 {
 	uint32_t i;
+	uint32_t safe_entries;
 	uint8_t num_entries;
 	struct phm_ppt_v1_clock_voltage_dependency_table
 				*clk_table;
@@ -738,6 +1143,14 @@ static int get_dcefclk_voltage_dependency_table(
 	PP_ASSERT_WITH_CODE((clk_dep_table->ucNumEntries != 0),
 			"Invalid PowerPlay Table!", return -1);
 
+	safe_entries = min_t(uint32_t, clk_dep_table->ucNumEntries,
+			     pp_entries_max(hwmgr, clk_dep_table,
+					    sizeof(*clk_dep_table),
+					    sizeof(ATOM_Vega10_CLK_Dependency_Record)));
+	if (safe_entries < clk_dep_table->ucNumEntries)
+		pr_warn("amdgpu: Vega10 DCEFCLK dependency table: clamping ucNumEntries %u -> %u\n",
+			clk_dep_table->ucNumEntries, safe_entries);
+
 /*
  * workaround needed to add another DPM level for pioneer cards
  * as VBIOS is locked down.
@@ -747,12 +1160,12 @@ static int get_dcefclk_voltage_dependency_table(
 	dev_id = adev->pdev->device;
 	rev_id = adev->pdev->revision;
 
-	if (dev_id == 0x6863 && rev_id == 0 &&
-		clk_dep_table->entries[clk_dep_table->ucNumEntries - 1].ulClk < 90000)
-		num_entries = clk_dep_table->ucNumEntries + 1 > NUM_DSPCLK_LEVELS ?
-				NUM_DSPCLK_LEVELS : clk_dep_table->ucNumEntries + 1;
+	if (dev_id == 0x6863 && rev_id == 0 && safe_entries > 0 &&
+		clk_dep_table->entries[safe_entries - 1].ulClk < 90000)
+		num_entries = safe_entries + 1 > NUM_DSPCLK_LEVELS ?
+				NUM_DSPCLK_LEVELS : safe_entries + 1;
 	else
-		num_entries = clk_dep_table->ucNumEntries;
+		num_entries = safe_entries;
 
 
 	clk_table = kzalloc_flex(*clk_table, entries, num_entries);
@@ -761,7 +1174,7 @@ static int get_dcefclk_voltage_dependency_table(
 
 	clk_table->count = (uint32_t)num_entries;
 
-	for (i = 0; i < clk_dep_table->ucNumEntries; i++) {
+	for (i = 0; i < safe_entries; i++) {
 		clk_table->entries[i].vddInd =
 				clk_dep_table->entries[i].ucVddInd;
 		clk_table->entries[i].clk =
@@ -873,51 +1286,13 @@ static int init_powerplay_extended_tables(
 	int result = 0;
 	struct phm_ppt_v2_information *pp_table_info =
 		(struct phm_ppt_v2_information *)(hwmgr->pptable);
-
-	const ATOM_Vega10_MM_Dependency_Table *mm_dependency_table =
-			(const ATOM_Vega10_MM_Dependency_Table *)
-			(((unsigned long) powerplay_table) +
-			le16_to_cpu(powerplay_table->usMMDependencyTableOffset));
-	const Vega10_PPTable_Generic_SubTable_Header *power_tune_table =
-			(const Vega10_PPTable_Generic_SubTable_Header *)
-			(((unsigned long) powerplay_table) +
-			le16_to_cpu(powerplay_table->usPowerTuneTableOffset));
-	const ATOM_Vega10_SOCCLK_Dependency_Table *socclk_dep_table =
-			(const ATOM_Vega10_SOCCLK_Dependency_Table *)
-			(((unsigned long) powerplay_table) +
-			le16_to_cpu(powerplay_table->usSocclkDependencyTableOffset));
-	const ATOM_Vega10_GFXCLK_Dependency_Table *gfxclk_dep_table =
-			(const ATOM_Vega10_GFXCLK_Dependency_Table *)
-			(((unsigned long) powerplay_table) +
-			le16_to_cpu(powerplay_table->usGfxclkDependencyTableOffset));
-	const ATOM_Vega10_DCEFCLK_Dependency_Table *dcefclk_dep_table =
-			(const ATOM_Vega10_DCEFCLK_Dependency_Table *)
-			(((unsigned long) powerplay_table) +
-			le16_to_cpu(powerplay_table->usDcefclkDependencyTableOffset));
-	const ATOM_Vega10_MCLK_Dependency_Table *mclk_dep_table =
-			(const ATOM_Vega10_MCLK_Dependency_Table *)
-			(((unsigned long) powerplay_table) +
-			le16_to_cpu(powerplay_table->usMclkDependencyTableOffset));
-	const ATOM_Vega10_Hard_Limit_Table *hard_limits =
-			(const ATOM_Vega10_Hard_Limit_Table *)
-			(((unsigned long) powerplay_table) +
-			le16_to_cpu(powerplay_table->usHardLimitTableOffset));
-	const Vega10_PPTable_Generic_SubTable_Header *pcie_table =
-			(const Vega10_PPTable_Generic_SubTable_Header *)
-			(((unsigned long) powerplay_table) +
-			le16_to_cpu(powerplay_table->usPCIETableOffset));
-	const ATOM_Vega10_PIXCLK_Dependency_Table *pixclk_dep_table =
-			(const ATOM_Vega10_PIXCLK_Dependency_Table *)
-			(((unsigned long) powerplay_table) +
-			le16_to_cpu(powerplay_table->usPixclkDependencyTableOffset));
-	const ATOM_Vega10_PHYCLK_Dependency_Table *phyclk_dep_table =
-			(const ATOM_Vega10_PHYCLK_Dependency_Table *)
-			(((unsigned long) powerplay_table) +
-			le16_to_cpu(powerplay_table->usPhyClkDependencyTableOffset));
-	const ATOM_Vega10_DISPCLK_Dependency_Table *dispclk_dep_table =
-			(const ATOM_Vega10_DISPCLK_Dependency_Table *)
-			(((unsigned long) powerplay_table) +
-			le16_to_cpu(powerplay_table->usDispClkDependencyTableOffset));
+	const ATOM_Vega10_MM_Dependency_Table *mm_dependency_table;
+	const Vega10_PPTable_Generic_SubTable_Header *power_tune_table;
+	const ATOM_Vega10_GFXCLK_Dependency_Table *gfxclk_dep_table;
+	const ATOM_Vega10_MCLK_Dependency_Table *mclk_dep_table;
+	const ATOM_Vega10_Hard_Limit_Table *hard_limits;
+	const Vega10_PPTable_Generic_SubTable_Header *pcie_table;
+	const ATOM_Vega10_SOCCLK_Dependency_Table *clk_dep_table;
 
 	pp_table_info->vdd_dep_on_socclk = NULL;
 	pp_table_info->vdd_dep_on_sclk = NULL;
@@ -929,63 +1304,114 @@ static int init_powerplay_extended_tables(
 	pp_table_info->vdd_dep_on_phyclk = NULL;
 	pp_table_info->vdd_dep_on_dispclk = NULL;
 
-	if (powerplay_table->usMMDependencyTableOffset)
-		result = get_mm_clock_voltage_table(hwmgr,
+	if (powerplay_table->usMMDependencyTableOffset) {
+		result = get_vega10_mm_dependency_table(hwmgr, powerplay_table,
+							&mm_dependency_table);
+		if (!result)
+			result = get_mm_clock_voltage_table(hwmgr,
 				&pp_table_info->mm_dep_table,
 				mm_dependency_table);
+	}
 
-	if (!result && powerplay_table->usPowerTuneTableOffset)
-		result = get_tdp_table(hwmgr,
+	if (!result && powerplay_table->usPowerTuneTableOffset) {
+		result = get_vega10_power_tune_table(hwmgr, powerplay_table,
+						     &power_tune_table);
+		if (!result)
+			result = get_tdp_table(hwmgr,
 				&pp_table_info->tdp_table,
 				power_tune_table);
+	}
 
-	if (!result && powerplay_table->usSocclkDependencyTableOffset)
-		result = get_socclk_voltage_dependency_table(hwmgr,
+	if (!result && powerplay_table->usSocclkDependencyTableOffset) {
+		result = get_vega10_clk_dependency_table(hwmgr, powerplay_table,
+				le16_to_cpu(powerplay_table->usSocclkDependencyTableOffset),
+				&clk_dep_table);
+		if (!result)
+			result = get_socclk_voltage_dependency_table(hwmgr,
 				&pp_table_info->vdd_dep_on_socclk,
-				socclk_dep_table);
+				(const ATOM_Vega10_SOCCLK_Dependency_Table *)
+				clk_dep_table);
+	}
 
-	if (!result && powerplay_table->usGfxclkDependencyTableOffset)
-		result = get_gfxclk_voltage_dependency_table(hwmgr,
-				&pp_table_info->vdd_dep_on_sclk,
-				gfxclk_dep_table);
+	if (!result && powerplay_table->usGfxclkDependencyTableOffset) {
+		result = get_vega10_gfxclk_dependency_table(hwmgr,
+			powerplay_table, &gfxclk_dep_table);
+		if (!result)
+			result = get_gfxclk_voltage_dependency_table(hwmgr,
+					&pp_table_info->vdd_dep_on_sclk,
+					gfxclk_dep_table);
+	}
 
-	if (!result && powerplay_table->usPixclkDependencyTableOffset)
-		result = get_pix_clk_voltage_dependency_table(hwmgr,
+	if (!result && powerplay_table->usPixclkDependencyTableOffset) {
+		result = get_vega10_clk_dependency_table(hwmgr, powerplay_table,
+				le16_to_cpu(powerplay_table->usPixclkDependencyTableOffset),
+				&clk_dep_table);
+		if (!result)
+			result = get_pix_clk_voltage_dependency_table(hwmgr,
 				&pp_table_info->vdd_dep_on_pixclk,
 				(const ATOM_Vega10_PIXCLK_Dependency_Table *)
-				pixclk_dep_table);
+				clk_dep_table);
+	}
 
-	if (!result && powerplay_table->usPhyClkDependencyTableOffset)
-		result = get_pix_clk_voltage_dependency_table(hwmgr,
+	if (!result && powerplay_table->usPhyClkDependencyTableOffset) {
+		result = get_vega10_clk_dependency_table(hwmgr, powerplay_table,
+				le16_to_cpu(powerplay_table->usPhyClkDependencyTableOffset),
+				&clk_dep_table);
+		if (!result)
+			result = get_pix_clk_voltage_dependency_table(hwmgr,
 				&pp_table_info->vdd_dep_on_phyclk,
 				(const ATOM_Vega10_PIXCLK_Dependency_Table *)
-				phyclk_dep_table);
+				clk_dep_table);
+	}
 
-	if (!result && powerplay_table->usDispClkDependencyTableOffset)
-		result = get_pix_clk_voltage_dependency_table(hwmgr,
+	if (!result && powerplay_table->usDispClkDependencyTableOffset) {
+		result = get_vega10_clk_dependency_table(hwmgr, powerplay_table,
+				le16_to_cpu(powerplay_table->usDispClkDependencyTableOffset),
+				&clk_dep_table);
+		if (!result)
+			result = get_pix_clk_voltage_dependency_table(hwmgr,
 				&pp_table_info->vdd_dep_on_dispclk,
 				(const ATOM_Vega10_PIXCLK_Dependency_Table *)
-				dispclk_dep_table);
+				clk_dep_table);
+	}
 
-	if (!result && powerplay_table->usDcefclkDependencyTableOffset)
-		result = get_dcefclk_voltage_dependency_table(hwmgr,
+	if (!result && powerplay_table->usDcefclkDependencyTableOffset) {
+		result = get_vega10_clk_dependency_table(hwmgr, powerplay_table,
+				le16_to_cpu(powerplay_table->usDcefclkDependencyTableOffset),
+				&clk_dep_table);
+		if (!result)
+			result = get_dcefclk_voltage_dependency_table(hwmgr,
 				&pp_table_info->vdd_dep_on_dcefclk,
-				dcefclk_dep_table);
+				(const ATOM_Vega10_DCEFCLK_Dependency_Table *)
+				clk_dep_table);
+	}
 
-	if (!result && powerplay_table->usMclkDependencyTableOffset)
-		result = get_mclk_voltage_dependency_table(hwmgr,
+	if (!result && powerplay_table->usMclkDependencyTableOffset) {
+		result = get_vega10_mclk_dependency_table(hwmgr, powerplay_table,
+							  &mclk_dep_table);
+		if (!result)
+			result = get_mclk_voltage_dependency_table(hwmgr,
 				&pp_table_info->vdd_dep_on_mclk,
 				mclk_dep_table);
+	}
 
-	if (!result && powerplay_table->usPCIETableOffset)
-		result = get_pcie_table(hwmgr,
+	if (!result && powerplay_table->usPCIETableOffset) {
+		result = get_vega10_pcie_table(hwmgr, powerplay_table,
+					       &pcie_table);
+		if (!result)
+			result = get_pcie_table(hwmgr,
 				&pp_table_info->pcie_table,
 				pcie_table);
+	}
 
-	if (!result && powerplay_table->usHardLimitTableOffset)
-		result = get_hard_limits(hwmgr,
+	if (!result && powerplay_table->usHardLimitTableOffset) {
+		result = get_vega10_hard_limit_table(hwmgr, powerplay_table,
+						     &hard_limits);
+		if (!result)
+			result = get_hard_limits(hwmgr,
 				&pp_table_info->max_clock_voltage_on_dc,
 				hard_limits);
+	}
 
 	hwmgr->dyn_state.max_clock_voltage_on_dc.sclk =
 			pp_table_info->max_clock_voltage_on_dc.sclk;
@@ -1034,18 +1460,28 @@ static int get_vddc_lookup_table(
 		uint32_t max_levels)
 {
 	uint32_t i;
+	uint32_t num_entries;
 	phm_ppt_v1_voltage_lookup_table *table;
 
 	PP_ASSERT_WITH_CODE((vddc_lookup_pp_tables->ucNumEntries != 0),
 			"Invalid SOC_VDDD Lookup Table!", return 1);
 
-	table = kzalloc_flex(*table, entries, max_levels);
+	num_entries = min_t(uint32_t, vddc_lookup_pp_tables->ucNumEntries,
+			    min_t(uint32_t, max_levels,
+				  pp_entries_max(hwmgr, vddc_lookup_pp_tables,
+						 sizeof(*vddc_lookup_pp_tables),
+						 sizeof(ATOM_Vega10_Voltage_Lookup_Record))));
+	if (num_entries < vddc_lookup_pp_tables->ucNumEntries)
+		pr_warn("amdgpu: Vega10 VddcLookup table: clamping ucNumEntries %u -> %u\n",
+			vddc_lookup_pp_tables->ucNumEntries, num_entries);
+
+	table = kzalloc_flex(*table, entries, num_entries);
 	if (!table)
 		return -ENOMEM;
 
-	table->count = vddc_lookup_pp_tables->ucNumEntries;
+	table->count = num_entries;
 
-	for (i = 0; i < vddc_lookup_pp_tables->ucNumEntries; i++)
+	for (i = 0; i < num_entries; i++)
 		table->entries[i].us_vdd =
 				le16_to_cpu(vddc_lookup_pp_tables->entries[i].usVdd);
 
@@ -1113,30 +1549,39 @@ static int init_dpm_2_parameters(
 	}
 
 	if (powerplay_table->usVddcLookupTableOffset) {
-		const ATOM_Vega10_Voltage_Lookup_Table *vddc_table =
-				(ATOM_Vega10_Voltage_Lookup_Table *)
-				(((unsigned long)powerplay_table) +
-				le16_to_cpu(powerplay_table->usVddcLookupTableOffset));
-		result = get_vddc_lookup_table(hwmgr,
-				&pp_table_info->vddc_lookup_table, vddc_table, 8);
+		const ATOM_Vega10_Voltage_Lookup_Table *vddc_table;
+
+		result = get_vega10_voltage_lookup_table(hwmgr, powerplay_table,
+				le16_to_cpu(powerplay_table->usVddcLookupTableOffset),
+				8, &vddc_table);
+		if (!result)
+			result = get_vddc_lookup_table(hwmgr,
+					&pp_table_info->vddc_lookup_table,
+					vddc_table, 8);
 	}
 
-	if (powerplay_table->usVddmemLookupTableOffset) {
-		const ATOM_Vega10_Voltage_Lookup_Table *vdd_mem_table =
-				(ATOM_Vega10_Voltage_Lookup_Table *)
-				(((unsigned long)powerplay_table) +
-				le16_to_cpu(powerplay_table->usVddmemLookupTableOffset));
-		result = get_vddc_lookup_table(hwmgr,
-				&pp_table_info->vddmem_lookup_table, vdd_mem_table, 4);
+	if (!result && powerplay_table->usVddmemLookupTableOffset) {
+		const ATOM_Vega10_Voltage_Lookup_Table *vdd_mem_table;
+
+		result = get_vega10_voltage_lookup_table(hwmgr, powerplay_table,
+				le16_to_cpu(powerplay_table->usVddmemLookupTableOffset),
+				4, &vdd_mem_table);
+		if (!result)
+			result = get_vddc_lookup_table(hwmgr,
+					&pp_table_info->vddmem_lookup_table,
+					vdd_mem_table, 4);
 	}
 
-	if (powerplay_table->usVddciLookupTableOffset) {
-		const ATOM_Vega10_Voltage_Lookup_Table *vddci_table =
-				(ATOM_Vega10_Voltage_Lookup_Table *)
-				(((unsigned long)powerplay_table) +
-				le16_to_cpu(powerplay_table->usVddciLookupTableOffset));
-		result = get_vddc_lookup_table(hwmgr,
-				&pp_table_info->vddci_lookup_table, vddci_table, 4);
+	if (!result && powerplay_table->usVddciLookupTableOffset) {
+		const ATOM_Vega10_Voltage_Lookup_Table *vddci_table;
+
+		result = get_vega10_voltage_lookup_table(hwmgr, powerplay_table,
+				le16_to_cpu(powerplay_table->usVddciLookupTableOffset),
+				4, &vddci_table);
+		if (!result)
+			result = get_vddc_lookup_table(hwmgr,
+					&pp_table_info->vddci_lookup_table,
+					vddci_table, 4);
 	}
 
 	return result;
@@ -1247,15 +1692,14 @@ int vega10_get_number_of_powerplay_table_entries(struct pp_hwmgr *hwmgr)
 {
 	const ATOM_Vega10_State_Array *state_arrays;
 	const ATOM_Vega10_POWERPLAYTABLE *pp_table = get_powerplay_table(hwmgr);
+	int result;
 
 	PP_ASSERT_WITH_CODE((pp_table != NULL),
 			"Missing PowerPlay Table!", return -1);
-	PP_ASSERT_WITH_CODE((pp_table->sHeader.format_revision >=
-			ATOM_Vega10_TABLE_REVISION_VEGA10),
-			"Incorrect PowerPlay table revision!", return -1);
 
-	state_arrays = (ATOM_Vega10_State_Array *)(((unsigned long)pp_table) +
-			le16_to_cpu(pp_table->usStateArrayOffset));
+	result = get_vega10_state_array(hwmgr, pp_table, &state_arrays);
+	PP_ASSERT_WITH_CODE((result == 0),
+			    "Invalid PowerPlay Table State Array.", return result);
 
 	return (uint32_t)(state_arrays->ucNumEntries);
 }
@@ -1306,17 +1750,11 @@ int vega10_get_powerplay_table_entry(struct pp_hwmgr *hwmgr,
 
 	if (pp_table->sHeader.format_revision >=
 			ATOM_Vega10_TABLE_REVISION_VEGA10) {
-		state_arrays = (ATOM_Vega10_State_Array *)
-				(((unsigned long)pp_table) +
-				le16_to_cpu(pp_table->usStateArrayOffset));
-
-		PP_ASSERT_WITH_CODE(pp_table->usStateArrayOffset > 0,
-				"Invalid PowerPlay Table State Array Offset.",
-				return -1);
-		PP_ASSERT_WITH_CODE(state_arrays->ucNumEntries > 0,
+		result = get_vega10_state_array(hwmgr, pp_table, &state_arrays);
+		PP_ASSERT_WITH_CODE((result == 0),
 				"Invalid PowerPlay Table State Array.",
-				return -1);
-		PP_ASSERT_WITH_CODE((entry_index <= state_arrays->ucNumEntries),
+				return result);
+		PP_ASSERT_WITH_CODE((entry_index < state_arrays->ucNumEntries),
 				"Invalid PowerPlay Table State Array Entry.",
 				return -1);
 
@@ -1358,4 +1796,3 @@ int vega10_baco_set_cap(struct pp_hwmgr *hwmgr)
 			PHM_PlatformCaps_BACO);
 	return result;
 }
-
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_processpptables.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_processpptables.c
index 55e13f376039d..dcb9c749eba39 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_processpptables.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_processpptables.c
@@ -64,6 +64,13 @@ static int check_powerplay_tables(
 		struct pp_hwmgr *hwmgr,
 		const ATOM_Vega12_POWERPLAYTABLE *powerplay_table)
 {
+	size_t smc_pptable_size =
+		offsetofend(ATOM_Vega12_POWERPLAYTABLE, smcPPTable);
+	size_t table_size = hwmgr->soft_pp_table_size;
+
+	PP_ASSERT_WITH_CODE((table_size >= smc_pptable_size),
+			    "Invalid PowerPlay Table!", return -1);
+
 	PP_ASSERT_WITH_CODE((powerplay_table->sHeader.format_revision >=
 		ATOM_VEGA12_TABLE_REVISION_VEGA12),
 		"Unsupported PPTable format!", return -1);
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_processpptables.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_processpptables.c
index 36cb7aa80d071..a0c884c2341d3 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_processpptables.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_processpptables.c
@@ -66,6 +66,13 @@ static int check_powerplay_tables(
 		struct pp_hwmgr *hwmgr,
 		const ATOM_Vega20_POWERPLAYTABLE *powerplay_table)
 {
+	size_t smc_pptable_size =
+		offsetofend(ATOM_Vega20_POWERPLAYTABLE, smcPPTable);
+	size_t table_size = hwmgr->soft_pp_table_size;
+
+	PP_ASSERT_WITH_CODE((table_size >= smc_pptable_size),
+			    "Invalid PowerPlay Table!", return -1);
+
 	PP_ASSERT_WITH_CODE((powerplay_table->sHeader.format_revision >=
 		ATOM_VEGA20_TABLE_REVISION_VEGA20),
 		"Unsupported PPTable format!", return -1);
diff --git a/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h b/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h
index ca71efaa16561..e01afbc39d67f 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h
+++ b/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h
@@ -829,4 +829,24 @@ int smu8_init_function_pointers(struct pp_hwmgr *hwmgr);
 int vega12_hwmgr_init(struct pp_hwmgr *hwmgr);
 int vega20_hwmgr_init(struct pp_hwmgr *hwmgr);
 
+static inline uint32_t pp_entries_max(const struct pp_hwmgr *hwmgr,
+				      const void *sub_table,
+				      size_t hdr_size, size_t rec_size)
+{
+	const char *pp_start = hwmgr->soft_pp_table;
+	const char *pp_end   = pp_start + hwmgr->soft_pp_table_size;
+	const char *entries  = (const char *)sub_table + hdr_size;
+
+	if (!hwmgr->hardcode_pp_table) {
+		struct amdgpu_device *adev = hwmgr->adev;
+		const char *bios_end = (const char *)adev->bios + adev->bios_size;
+
+		if (pp_end > bios_end)
+			return 0;
+	}
+	if (!rec_size || entries >= pp_end)
+		return 0;
+	return (uint32_t)((pp_end - entries) / rec_size);
+}
+
 #endif /* _HWMGR_H_ */
diff --git a/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu7_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu7_smumgr.c
index 3e0068a6aeb28..45504ac3a31d4 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu7_smumgr.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/smu7_smumgr.c
@@ -134,14 +134,6 @@ int smu7_send_msg_to_smc(struct pp_hwmgr *hwmgr, uint16_t msg)
 
 	PHM_WAIT_FIELD_UNEQUAL(hwmgr, SMC_RESP_0, SMC_RESP, 0);
 
-	ret = PHM_READ_FIELD(hwmgr->device, SMC_RESP_0, SMC_RESP);
-
-	if (ret == 0xFE)
-		dev_dbg(adev->dev, "last message was not supported\n");
-	else if (ret != 1)
-		dev_info(adev->dev,
-			"\nlast message was failed ret is %d\n", ret);
-
 	cgs_write_register(hwmgr->device, mmSMC_RESP_0, 0);
 	cgs_write_register(hwmgr->device, mmSMC_MESSAGE_0, msg);
 
@@ -149,13 +141,16 @@ int smu7_send_msg_to_smc(struct pp_hwmgr *hwmgr, uint16_t msg)
 
 	ret = PHM_READ_FIELD(hwmgr->device, SMC_RESP_0, SMC_RESP);
 
-	if (ret == 0xFE)
-		dev_dbg(adev->dev, "message %x was not supported\n", msg);
-	else if (ret != 1)
-		dev_dbg(adev->dev,
-			"failed to send message %x ret is %d \n",  msg, ret);
-
-	return 0;
+	switch (ret) {
+	case 1:
+		return 0;
+	case 0xFE:
+		dev_dbg(adev->dev, "SMU message %#x was not supported\n", msg);
+		return -EOPNOTSUPP;
+	default:
+		dev_info(adev->dev, "SMU message %#x failed: response is %d\n", msg, ret);
+		return ret != 0xFFFF ? -EIO : -ENXIO;
+	}
 }
 
 int smu7_send_msg_to_smc_with_parameter(struct pp_hwmgr *hwmgr, uint16_t msg, uint32_t parameter)
diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
index 762ec3cede96e..f4e817885e54c 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
@@ -591,17 +591,13 @@ static int smu_get_power_num_states(void *handle,
 	return 0;
 }
 
-bool is_support_sw_smu(struct amdgpu_device *adev)
+void amdgpu_smu_early_init(struct amdgpu_device *adev)
 {
 	/* vega20 is 11.0.2, but it's supported via the powerplay code */
-	if (adev->asic_type == CHIP_VEGA20)
-		return false;
-
-	if ((amdgpu_ip_version(adev, MP1_HWIP, 0) >= IP_VERSION(11, 0, 0)) &&
-	    amdgpu_device_ip_is_valid(adev, AMD_IP_BLOCK_TYPE_SMC))
-		return true;
-
-	return false;
+	adev->is_sw_smu = adev->asic_type != CHIP_VEGA20 &&
+			  (amdgpu_ip_version(adev, MP1_HWIP, 0) >=
+			   IP_VERSION(11, 0, 0) &&
+			   amdgpu_device_ip_is_valid(adev, AMD_IP_BLOCK_TYPE_SMC));
 }
 
 bool is_support_cclk_dpm(struct amdgpu_device *adev)
@@ -667,25 +663,28 @@ static int smu_sys_set_pp_table(void *handle,
 {
 	struct smu_context *smu = handle;
 	struct smu_table_context *smu_table = &smu->smu_table;
-	ATOM_COMMON_TABLE_HEADER *header = (ATOM_COMMON_TABLE_HEADER *)buf;
+	ATOM_COMMON_TABLE_HEADER *header;
+	void *hardcode_pptable;
 	int ret = 0;
 
 	if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled)
 		return -EOPNOTSUPP;
 
+	if (!buf || size < sizeof(*header))
+		return -EINVAL;
+
+	header = (ATOM_COMMON_TABLE_HEADER *)buf;
 	if (header->usStructureSize != size) {
 		dev_err(smu->adev->dev, "pp table size not matched !\n");
 		return -EIO;
 	}
 
-	if (!smu_table->hardcode_pptable || smu_table->power_play_table_size < size) {
-		kfree(smu_table->hardcode_pptable);
-		smu_table->hardcode_pptable = kzalloc(size, GFP_KERNEL);
-		if (!smu_table->hardcode_pptable)
-			return -ENOMEM;
-	}
+	hardcode_pptable = kmemdup(buf, size, GFP_KERNEL);
+	if (!hardcode_pptable)
+		return -ENOMEM;
 
-	memcpy(smu_table->hardcode_pptable, buf, size);
+	kfree(smu_table->hardcode_pptable);
+	smu_table->hardcode_pptable = hardcode_pptable;
 	smu_table->power_play_table = smu_table->hardcode_pptable;
 	smu_table->power_play_table_size = size;
 
@@ -1367,6 +1366,14 @@ static void smu_feature_cap_init(struct smu_context *smu)
 	bitmap_zero(fea_cap->cap_map, SMU_FEATURE_CAP_ID__COUNT);
 }
 
+static int smu_set_power_dep(struct smu_context *smu, bool enable)
+{
+	if (!smu->ppt_funcs->set_power_dep)
+		return 0;
+
+	return smu->ppt_funcs->set_power_dep(smu, enable);
+}
+
 static int smu_sw_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -1428,6 +1435,8 @@ static int smu_sw_init(struct amdgpu_ip_block *ip_block)
 	if (!smu->ppt_funcs->get_fan_control_mode)
 		smu->adev->pm.no_fan = true;
 
+	smu_set_power_dep(smu, true);
+
 	return 0;
 }
 
@@ -1450,6 +1459,8 @@ static int smu_sw_fini(struct amdgpu_ip_block *ip_block)
 
 	smu_fini_microcode(smu);
 
+	smu_set_power_dep(smu, false);
+
 	return 0;
 }
 
@@ -2773,7 +2784,6 @@ const struct amd_ip_funcs smu_ip_funcs = {
 	.suspend = smu_suspend,
 	.resume = smu_resume,
 	.is_idle = NULL,
-	.check_soft_reset = NULL,
 	.wait_for_idle = NULL,
 	.soft_reset = NULL,
 	.set_clockgating_state = smu_set_clockgating_state,
@@ -2820,17 +2830,6 @@ const struct amdgpu_ip_block_version smu_v15_0_ip_block = {
 	.funcs = &smu_ip_funcs,
 };
 
-const struct ras_smu_drv *smu_get_ras_smu_driver(void *handle)
-{
-	struct smu_context *smu = (struct smu_context *)handle;
-	const struct ras_smu_drv *tmp = NULL;
-	int ret;
-
-	ret = smu_get_ras_smu_drv(smu, &tmp);
-
-	return ret ? NULL : tmp;
-}
-
 static int smu_load_microcode(void *handle)
 {
 	struct smu_context *smu = handle;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
index d76e0b005308f..7ea7c4a5279be 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
@@ -749,6 +749,9 @@ struct smu_context {
 	bool pm_enabled;
 	bool is_apu;
 
+	/* Power dependency link from an integrated xHCI controller to the GPU */
+	struct device_link		*usb_power_link;
+
 	uint32_t smc_driver_if_version;
 	uint32_t smc_fw_if_version;
 	uint32_t smc_fw_version;
@@ -849,8 +852,6 @@ struct pptable_funcs {
 	 */
 	int (*set_default_dpm_table)(struct smu_context *smu);
 
-	int (*set_power_state)(struct smu_context *smu);
-
 	/**
 	 * @populate_umd_state_clk: Populate the UMD power state table with
 	 *                          defaults.
@@ -903,16 +904,6 @@ struct pptable_funcs {
 					      struct
 					      pp_clock_levels_with_latency
 					      *clocks);
-	/**
-	 * @get_clock_by_type_with_voltage: Get the speed and voltage of a clock
-	 *                                  domain.
-	 */
-	int (*get_clock_by_type_with_voltage)(struct smu_context *smu,
-					      enum amd_pp_clock_type type,
-					      struct
-					      pp_clock_levels_with_voltage
-					      *clocks);
-
 	/**
 	 * @get_power_profile_mode: Print all power profile modes to
 	 *                          buffer. Star current mode.
@@ -1354,11 +1345,6 @@ struct pptable_funcs {
 	 */
 	int (*register_irq_handler)(struct smu_context *smu);
 
-	/**
-	 * @set_azalia_d3_pme: Wake the audio decode engine from d3 sleep.
-	 */
-	int (*set_azalia_d3_pme)(struct smu_context *smu);
-
 	/**
 	 * @get_max_sustainable_clocks_by_dc: Get a copy of the max sustainable
 	 *                                    clock speeds table.
@@ -1375,18 +1361,6 @@ struct pptable_funcs {
 	 */
 	int (*get_bamaco_support)(struct smu_context *smu);
 
-	/**
-	 * @baco_get_state: Get the current BACO state.
-	 *
-	 * Return: Current BACO state.
-	 */
-	enum smu_baco_state (*baco_get_state)(struct smu_context *smu);
-
-	/**
-	 * @baco_set_state: Enter/exit BACO.
-	 */
-	int (*baco_set_state)(struct smu_context *smu, enum smu_baco_state state);
-
 	/**
 	 * @baco_enter: Enter BACO.
 	 */
@@ -1648,12 +1622,13 @@ struct pptable_funcs {
 	int (*ras_send_msg)(struct smu_context *smu,
 			    enum smu_message_type msg, uint32_t param, uint32_t *read_arg);
 
-
 	/**
-	 * @get_ras_smu_drv: Get RAS smu driver interface
-	 * Return: ras_smu_drv *
+	 * @set_power_dep: Create or destroy a power dependency link
+	 * from an integrated xHCI controller to the GPU so that the GPU is
+	 * resumed before the USB controller during PM resume. @enable is true
+	 * to create the link and false to tear it down.
 	 */
-	int (*get_ras_smu_drv)(struct smu_context *smu, const struct ras_smu_drv **ras_smu_drv);
+	int (*set_power_dep)(struct smu_context *smu, bool enable);
 };
 
 typedef enum {
@@ -1952,7 +1927,13 @@ int smu_link_reset(struct smu_context *smu);
 
 extern const struct amd_ip_funcs smu_ip_funcs;
 
-bool is_support_sw_smu(struct amdgpu_device *adev);
+void amdgpu_smu_early_init(struct amdgpu_device *adev);
+
+static inline bool is_support_sw_smu(struct amdgpu_device *adev)
+{
+	return adev->is_sw_smu;
+}
+
 bool is_support_cclk_dpm(struct amdgpu_device *adev);
 int smu_write_watermarks_table(struct smu_context *smu);
 
@@ -1995,7 +1976,6 @@ int smu_set_pm_policy(struct smu_context *smu, enum pp_pm_policy p_type,
 		      int level);
 ssize_t smu_get_pm_policy_info(struct smu_context *smu,
 			       enum pp_pm_policy p_type, char *sysbuf);
-const struct ras_smu_drv *smu_get_ras_smu_driver(void *handle);
 
 int amdgpu_smu_ras_send_msg(struct amdgpu_device *adev, enum smu_message_type msg,
 			    uint32_t param, uint32_t *readarg);
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v11_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v11_0.h
index dd94e8a9e218f..c0accee9a9c8c 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v11_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v11_0.h
@@ -199,8 +199,6 @@ int smu_v11_0_gfx_off_control(struct smu_context *smu, bool enable);
 
 int smu_v11_0_register_irq_handler(struct smu_context *smu);
 
-int smu_v11_0_set_azalia_d3_pme(struct smu_context *smu);
-
 int smu_v11_0_get_max_sustainable_clocks_by_dc(struct smu_context *smu,
 		struct pp_smu_nv_clock_table *max_clocks);
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
index 89bbda0670ef1..7f21f867d73c6 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
@@ -180,8 +180,6 @@ int smu_v13_0_gfx_off_control(struct smu_context *smu, bool enable);
 
 int smu_v13_0_register_irq_handler(struct smu_context *smu);
 
-int smu_v13_0_set_azalia_d3_pme(struct smu_context *smu);
-
 int smu_v13_0_get_max_sustainable_clocks_by_dc(struct smu_context *smu,
 					       struct pp_smu_nv_clock_table *max_clocks);
 
@@ -255,10 +253,6 @@ void smu_v13_0_init_msg_ctl(struct smu_context *smu,
 
 int smu_v13_0_mode1_reset(struct smu_context *smu);
 
-int smu_v13_0_get_pptable_from_firmware(struct smu_context *smu,
-					void **table,
-					uint32_t *size,
-					uint32_t pptable_id);
 
 int smu_v13_0_update_pcie_parameters(struct smu_context *smu,
 				     uint8_t pcie_gen_cap,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h
index 4eb40ff8aff20..dc8e13a7c8791 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v14_0.h
@@ -203,10 +203,6 @@ int smu_v14_0_set_gfx_power_up_by_imu(struct smu_context *smu);
 
 int smu_v14_0_set_default_dpm_tables(struct smu_context *smu);
 
-int smu_v14_0_get_pptable_from_firmware(struct smu_context *smu,
-					void **table,
-					uint32_t *size,
-					uint32_t pptable_id);
 
 int smu_v14_0_od_edit_dpm_table(struct smu_context *smu,
 			enum PP_OD_DPM_TABLE_COMMAND type,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h
index e6fd8be2cc4a0..13723d45a7dec 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h
@@ -211,10 +211,6 @@ int smu_v15_0_deep_sleep_control(struct smu_context *smu,
 
 int smu_v15_0_set_gfx_power_up_by_imu(struct smu_context *smu);
 
-int smu_v15_0_get_pptable_from_firmware(struct smu_context *smu,
-					void **table,
-					uint32_t *size,
-					uint32_t pptable_id);
 
 int smu_v15_0_od_edit_dpm_table(struct smu_context *smu,
 				enum PP_OD_DPM_TABLE_COMMAND type,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c
index 54d3dba7d3549..db5db2c9c8e8a 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c
@@ -1466,9 +1466,10 @@ static int arcturus_set_power_profile_mode(struct smu_context *smu,
 				return -ENOMEM;
 		}
 		if (custom_params && custom_params_max_idx) {
-			if (custom_params_max_idx != ARCTURUS_CUSTOM_PARAMS_COUNT)
-				return -EINVAL;
-			if (custom_params[0] >= ARCTURUS_CUSTOM_PARAMS_CLOCK_COUNT)
+			if (!smu_cmn_custom_params_count_valid(custom_params_max_idx,
+							       ARCTURUS_CUSTOM_PARAMS_COUNT) ||
+			    !smu_cmn_custom_params_clock_valid(custom_params[0],
+							       ARCTURUS_CUSTOM_PARAMS_CLOCK_COUNT))
 				return -EINVAL;
 			idx = custom_params[0] * ARCTURUS_CUSTOM_PARAMS_COUNT;
 			smu->custom_profile_params[idx] = 1;
@@ -1932,7 +1933,6 @@ static const struct pptable_funcs arcturus_ppt_funcs = {
 	.set_xgmi_pstate = smu_v11_0_set_xgmi_pstate,
 	.gfx_off_control = smu_v11_0_gfx_off_control,
 	.register_irq_handler = smu_v11_0_register_irq_handler,
-	.set_azalia_d3_pme = smu_v11_0_set_azalia_d3_pme,
 	.get_max_sustainable_clocks_by_dc = smu_v11_0_get_max_sustainable_clocks_by_dc,
 	.get_bamaco_support = smu_v11_0_get_bamaco_support,
 	.baco_enter = smu_v11_0_baco_enter,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
index cd0457e13f54c..8feea44f3ca03 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c
@@ -1843,9 +1843,10 @@ static int navi10_set_power_profile_mode(struct smu_context *smu,
 				return -ENOMEM;
 		}
 		if (custom_params && custom_params_max_idx) {
-			if (custom_params_max_idx != NAVI10_CUSTOM_PARAMS_COUNT)
-				return -EINVAL;
-			if (custom_params[0] >= NAVI10_CUSTOM_PARAMS_CLOCKS_COUNT)
+			if (!smu_cmn_custom_params_count_valid(custom_params_max_idx,
+							       NAVI10_CUSTOM_PARAMS_COUNT) ||
+			    !smu_cmn_custom_params_clock_valid(custom_params[0],
+							       NAVI10_CUSTOM_PARAMS_CLOCKS_COUNT))
 				return -EINVAL;
 			idx = custom_params[0] * NAVI10_CUSTOM_PARAMS_COUNT;
 			smu->custom_profile_params[idx] = 1;
@@ -3337,7 +3338,6 @@ static const struct pptable_funcs navi10_ppt_funcs = {
 	.set_xgmi_pstate = smu_v11_0_set_xgmi_pstate,
 	.gfx_off_control = smu_v11_0_gfx_off_control,
 	.register_irq_handler = smu_v11_0_register_irq_handler,
-	.set_azalia_d3_pme = smu_v11_0_set_azalia_d3_pme,
 	.get_max_sustainable_clocks_by_dc = smu_v11_0_get_max_sustainable_clocks_by_dc,
 	.get_bamaco_support = smu_v11_0_get_bamaco_support,
 	.baco_enter = navi10_baco_enter,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
index f799e489b4817..c0de73b853532 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
@@ -1755,9 +1755,10 @@ static int sienna_cichlid_set_power_profile_mode(struct smu_context *smu,
 				return -ENOMEM;
 		}
 		if (custom_params && custom_params_max_idx) {
-			if (custom_params_max_idx != SIENNA_CICHLID_CUSTOM_PARAMS_COUNT)
-				return -EINVAL;
-			if (custom_params[0] >= SIENNA_CICHLID_CUSTOM_PARAMS_CLOCK_COUNT)
+			if (!smu_cmn_custom_params_count_valid(custom_params_max_idx,
+							       SIENNA_CICHLID_CUSTOM_PARAMS_COUNT) ||
+			    !smu_cmn_custom_params_clock_valid(custom_params[0],
+							       SIENNA_CICHLID_CUSTOM_PARAMS_CLOCK_COUNT))
 				return -EINVAL;
 			idx = custom_params[0] * SIENNA_CICHLID_CUSTOM_PARAMS_COUNT;
 			smu->custom_profile_params[idx] = 1;
@@ -3144,7 +3145,6 @@ static const struct pptable_funcs sienna_cichlid_ppt_funcs = {
 	.set_xgmi_pstate = smu_v11_0_set_xgmi_pstate,
 	.gfx_off_control = smu_v11_0_gfx_off_control,
 	.register_irq_handler = smu_v11_0_register_irq_handler,
-	.set_azalia_d3_pme = smu_v11_0_set_azalia_d3_pme,
 	.get_max_sustainable_clocks_by_dc = smu_v11_0_get_max_sustainable_clocks_by_dc,
 	.get_bamaco_support = smu_v11_0_get_bamaco_support,
 	.baco_enter = sienna_cichlid_baco_enter,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c
index d68ceee16d8f1..a889d846e9c5a 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c
@@ -192,81 +192,22 @@ int smu_v11_0_check_fw_status(struct smu_context *smu)
 	return -EIO;
 }
 
-static int smu_v11_0_set_pptable_v2_0(struct smu_context *smu, void **table, uint32_t *size)
-{
-	struct amdgpu_device *adev = smu->adev;
-	uint32_t ppt_offset_bytes;
-	const struct smc_firmware_header_v2_0 *v2;
-
-	v2 = (const struct smc_firmware_header_v2_0 *) adev->pm.fw->data;
-
-	ppt_offset_bytes = le32_to_cpu(v2->ppt_offset_bytes);
-	*size = le32_to_cpu(v2->ppt_size_bytes);
-	*table = (uint8_t *)v2 + ppt_offset_bytes;
-
-	return 0;
-}
-
-static int smu_v11_0_set_pptable_v2_1(struct smu_context *smu, void **table,
-				      uint32_t *size, uint32_t pptable_id)
-{
-	struct amdgpu_device *adev = smu->adev;
-	const struct smc_firmware_header_v2_1 *v2_1;
-	struct smc_soft_pptable_entry *entries;
-	uint32_t pptable_count = 0;
-	int i = 0;
-
-	v2_1 = (const struct smc_firmware_header_v2_1 *) adev->pm.fw->data;
-	entries = (struct smc_soft_pptable_entry *)
-		((uint8_t *)v2_1 + le32_to_cpu(v2_1->pptable_entry_offset));
-	pptable_count = le32_to_cpu(v2_1->pptable_count);
-	for (i = 0; i < pptable_count; i++) {
-		if (le32_to_cpu(entries[i].id) == pptable_id) {
-			*table = ((uint8_t *)v2_1 + le32_to_cpu(entries[i].ppt_offset_bytes));
-			*size = le32_to_cpu(entries[i].ppt_size_bytes);
-			break;
-		}
-	}
-
-	if (i == pptable_count)
-		return -EINVAL;
-
-	return 0;
-}
-
 int smu_v11_0_setup_pptable(struct smu_context *smu)
 {
 	struct amdgpu_device *adev = smu->adev;
-	const struct smc_firmware_header_v1_0 *hdr;
-	int ret, index;
-	uint32_t size = 0;
 	uint16_t atom_table_size;
 	uint8_t frev, crev;
+	uint32_t size = 0;
+	int ret, index;
 	void *table;
-	uint16_t version_major, version_minor;
-
-	if (!amdgpu_sriov_vf(adev)) {
-		hdr = (const struct smc_firmware_header_v1_0 *) adev->pm.fw->data;
-		version_major = le16_to_cpu(hdr->header.header_version_major);
-		version_minor = le16_to_cpu(hdr->header.header_version_minor);
-		if (version_major == 2 && smu->smu_table.boot_values.pp_table_id > 0) {
-			dev_info(adev->dev, "use driver provided pptable %d\n", smu->smu_table.boot_values.pp_table_id);
-			switch (version_minor) {
-			case 0:
-				ret = smu_v11_0_set_pptable_v2_0(smu, &table, &size);
-				break;
-			case 1:
-				ret = smu_v11_0_set_pptable_v2_1(smu, &table, &size,
-								smu->smu_table.boot_values.pp_table_id);
-				break;
-			default:
-				ret = -EINVAL;
-				break;
-			}
-			if (ret)
-				return ret;
-			goto out;
-		}
+
+	if (!amdgpu_sriov_vf(adev) &&
+	    smu->smu_table.boot_values.pp_table_id > 0) {
+		ret = smu_cmn_get_pptable_from_firmware(smu, &table, &size,
+							smu->smu_table.boot_values.pp_table_id);
+		if (ret)
+			return ret;
+		goto out;
 	}
 
 	dev_info(adev->dev, "use vbios provided pptable\n");
@@ -1476,11 +1417,6 @@ int smu_v11_0_get_max_sustainable_clocks_by_dc(struct smu_context *smu,
 	return 0;
 }
 
-int smu_v11_0_set_azalia_d3_pme(struct smu_context *smu)
-{
-	return smu_cmn_send_smc_msg(smu, SMU_MSG_BacoAudioD3PME, NULL);
-}
-
 int smu_v11_0_baco_set_armd3_sequence(struct smu_context *smu,
 				      enum smu_baco_seq baco_seq)
 {
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c
index 75335da224c7a..2b011610e3c70 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c
@@ -1444,7 +1444,6 @@ static int renoir_get_enabled_mask(struct smu_context *smu,
 }
 
 static const struct pptable_funcs renoir_ppt_funcs = {
-	.set_power_state = NULL,
 	.emit_clk_levels = renoir_emit_clk_levels,
 	.get_current_power_state = renoir_get_current_power_state,
 	.dpm_set_vcn_enable = renoir_dpm_set_vcn_enable,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
index 9d8b1227388fa..cd7bf36673cba 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c
@@ -2004,7 +2004,6 @@ static const struct pptable_funcs aldebaran_ppt_funcs = {
 	.disable_thermal_alert = smu_v13_0_disable_thermal_alert,
 	.set_xgmi_pstate = smu_v13_0_set_xgmi_pstate,
 	.register_irq_handler = smu_v13_0_register_irq_handler,
-	.set_azalia_d3_pme = smu_v13_0_set_azalia_d3_pme,
 	.get_max_sustainable_clocks_by_dc = smu_v13_0_get_max_sustainable_clocks_by_dc,
 	.get_bamaco_support = aldebaran_get_bamaco_support,
 	.get_dpm_ultimate_freq = aldebaran_get_dpm_ultimate_freq,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
index be9a7a32de991..4f10bce367562 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
@@ -218,7 +218,7 @@ int smu_v13_0_init_pptable_microcode(struct smu_context *smu)
 	if (!pptable_id)
 		return 0;
 
-	ret = smu_v13_0_get_pptable_from_firmware(smu, &table, &size, pptable_id);
+	ret = smu_cmn_get_pptable_from_firmware(smu, &table, &size, pptable_id);
 	if (ret)
 		return ret;
 
@@ -258,48 +258,6 @@ int smu_v13_0_check_fw_status(struct smu_context *smu)
 	return -EIO;
 }
 
-static int smu_v13_0_set_pptable_v2_0(struct smu_context *smu, void **table, uint32_t *size)
-{
-	struct amdgpu_device *adev = smu->adev;
-	uint32_t ppt_offset_bytes;
-	const struct smc_firmware_header_v2_0 *v2;
-
-	v2 = (const struct smc_firmware_header_v2_0 *) adev->pm.fw->data;
-
-	ppt_offset_bytes = le32_to_cpu(v2->ppt_offset_bytes);
-	*size = le32_to_cpu(v2->ppt_size_bytes);
-	*table = (uint8_t *)v2 + ppt_offset_bytes;
-
-	return 0;
-}
-
-static int smu_v13_0_set_pptable_v2_1(struct smu_context *smu, void **table,
-				      uint32_t *size, uint32_t pptable_id)
-{
-	struct amdgpu_device *adev = smu->adev;
-	const struct smc_firmware_header_v2_1 *v2_1;
-	struct smc_soft_pptable_entry *entries;
-	uint32_t pptable_count = 0;
-	int i = 0;
-
-	v2_1 = (const struct smc_firmware_header_v2_1 *) adev->pm.fw->data;
-	entries = (struct smc_soft_pptable_entry *)
-		((uint8_t *)v2_1 + le32_to_cpu(v2_1->pptable_entry_offset));
-	pptable_count = le32_to_cpu(v2_1->pptable_count);
-	for (i = 0; i < pptable_count; i++) {
-		if (le32_to_cpu(entries[i].id) == pptable_id) {
-			*table = ((uint8_t *)v2_1 + le32_to_cpu(entries[i].ppt_offset_bytes));
-			*size = le32_to_cpu(entries[i].ppt_size_bytes);
-			break;
-		}
-	}
-
-	if (i == pptable_count)
-		return -EINVAL;
-
-	return 0;
-}
-
 static int smu_v13_0_get_pptable_from_vbios(struct smu_context *smu, void **table, uint32_t *size)
 {
 	struct amdgpu_device *adev = smu->adev;
@@ -322,45 +280,6 @@ static int smu_v13_0_get_pptable_from_vbios(struct smu_context *smu, void **tabl
 	return 0;
 }
 
-int smu_v13_0_get_pptable_from_firmware(struct smu_context *smu,
-					void **table,
-					uint32_t *size,
-					uint32_t pptable_id)
-{
-	const struct smc_firmware_header_v1_0 *hdr;
-	struct amdgpu_device *adev = smu->adev;
-	uint16_t version_major, version_minor;
-	int ret;
-
-	hdr = (const struct smc_firmware_header_v1_0 *) adev->pm.fw->data;
-	if (!hdr)
-		return -EINVAL;
-
-	dev_info(adev->dev, "use driver provided pptable %d\n", pptable_id);
-
-	version_major = le16_to_cpu(hdr->header.header_version_major);
-	version_minor = le16_to_cpu(hdr->header.header_version_minor);
-	if (version_major != 2) {
-		dev_err(adev->dev, "Unsupported smu firmware version %d.%d\n",
-			version_major, version_minor);
-		return -EINVAL;
-	}
-
-	switch (version_minor) {
-	case 0:
-		ret = smu_v13_0_set_pptable_v2_0(smu, table, size);
-		break;
-	case 1:
-		ret = smu_v13_0_set_pptable_v2_1(smu, table, size, pptable_id);
-		break;
-	default:
-		ret = -EINVAL;
-		break;
-	}
-
-	return ret;
-}
-
 int smu_v13_0_setup_pptable(struct smu_context *smu)
 {
 	struct amdgpu_device *adev = smu->adev;
@@ -380,7 +299,7 @@ int smu_v13_0_setup_pptable(struct smu_context *smu)
 	if ((amdgpu_sriov_vf(adev) || !pptable_id) && (amdgpu_emu_mode != 1))
 		ret = smu_v13_0_get_pptable_from_vbios(smu, &table, &size);
 	else
-		ret = smu_v13_0_get_pptable_from_firmware(smu, &table, &size, pptable_id);
+		ret = smu_cmn_get_pptable_from_firmware(smu, &table, &size, pptable_id);
 
 	if (ret)
 		return ret;
@@ -1401,15 +1320,6 @@ int smu_v13_0_get_max_sustainable_clocks_by_dc(struct smu_context *smu,
 	return 0;
 }
 
-int smu_v13_0_set_azalia_d3_pme(struct smu_context *smu)
-{
-	int ret = 0;
-
-	ret = smu_cmn_send_smc_msg(smu, SMU_MSG_BacoAudioD3PME, NULL);
-
-	return ret;
-}
-
 static int smu_v13_0_wait_for_reset_complete(struct smu_context *smu,
 					     uint64_t event_arg)
 {
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
index acbd7046d8a50..4ce1429cf57b0 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
@@ -2619,9 +2619,10 @@ static int smu_v13_0_0_set_power_profile_mode(struct smu_context *smu,
 				return -ENOMEM;
 		}
 		if (custom_params && custom_params_max_idx) {
-			if (custom_params_max_idx != SMU_13_0_0_CUSTOM_PARAMS_COUNT)
-				return -EINVAL;
-			if (custom_params[0] >= SMU_13_0_0_CUSTOM_PARAMS_CLOCK_COUNT)
+			if (!smu_cmn_custom_params_count_valid(custom_params_max_idx,
+							       SMU_13_0_0_CUSTOM_PARAMS_COUNT) ||
+			    !smu_cmn_custom_params_clock_valid(custom_params[0],
+							       SMU_13_0_0_CUSTOM_PARAMS_CLOCK_COUNT))
 				return -EINVAL;
 			idx = custom_params[0] * SMU_13_0_0_CUSTOM_PARAMS_COUNT;
 			smu->custom_profile_params[idx] = 1;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c
index fe929bd89058a..ecdbf9bb8eb72 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c
@@ -1017,139 +1017,3 @@ const struct smu_temp_funcs smu_v13_0_12_temp_funcs = {
 	.temp_metrics_is_supported = smu_v13_0_12_is_temp_metrics_supported,
 	.get_temp_metrics = smu_v13_0_12_get_temp_metrics,
 };
-
-static int smu_v13_0_12_get_ras_table_version(struct amdgpu_device *adev,
-					      uint32_t *table_version)
-{
-	struct smu_context *smu = adev->powerplay.pp_handle;
-
-	return smu_cmn_send_smc_msg_with_param(smu,
-			SMU_MSG_GetRASTableVersion, 0, table_version);
-}
-
-static int smu_v13_0_12_get_badpage_count(struct amdgpu_device *adev, uint32_t *count,
-					  uint32_t timeout)
-{
-	struct smu_context *smu = adev->powerplay.pp_handle;
-	uint64_t end, now;
-	int ret = 0;
-
-	now = (uint64_t)ktime_to_ms(ktime_get());
-	end = now + timeout;
-	do {
-		ret = smu_cmn_send_smc_msg_with_param(smu,
-			SMU_MSG_GetBadPageCount, 0, count);
-		/* eeprom is not ready */
-		if (ret != -EBUSY)
-			return ret;
-		mdelay(10);
-		now = (uint64_t)ktime_to_ms(ktime_get());
-	} while (now < end);
-
-	dev_err(adev->dev,
-			 "smu get bad page count timeout!\n");
-	return ret;
-}
-
-static int smu_v13_0_12_set_timestamp(struct amdgpu_device *adev, uint64_t timestamp)
-{
-	struct smu_context *smu = adev->powerplay.pp_handle;
-
-	return smu_cmn_send_smc_msg_with_param(smu,
-			SMU_MSG_SetTimestamp, (uint32_t)timestamp, 0);
-}
-
-static int smu_v13_0_12_get_timestamp(struct amdgpu_device *adev,
-				      uint16_t index, uint64_t *timestamp)
-{
-	struct smu_context *smu = adev->powerplay.pp_handle;
-	uint32_t temp;
-	int ret;
-
-	ret = smu_cmn_send_smc_msg_with_param(smu,
-			SMU_MSG_GetTimestamp, index, &temp);
-	if (!ret)
-		*timestamp = temp;
-
-	return ret;
-}
-
-static int smu_v13_0_12_get_badpage_ipid(struct amdgpu_device *adev,
-					 uint16_t index, uint64_t *ipid)
-{
-	struct smu_context *smu = adev->powerplay.pp_handle;
-	uint32_t temp_arg, temp_ipid_lo, temp_ipid_high;
-	int ret;
-
-	temp_arg = index | (1 << 16);
-	ret = smu_cmn_send_smc_msg_with_param(smu,
-			SMU_MSG_GetBadPageIpid, temp_arg, &temp_ipid_lo);
-	if (ret)
-		return ret;
-
-	temp_arg = index | (2 << 16);
-	ret = smu_cmn_send_smc_msg_with_param(smu,
-			SMU_MSG_GetBadPageIpid, temp_arg, &temp_ipid_high);
-	if (!ret)
-		*ipid = (uint64_t)temp_ipid_high << 32 | temp_ipid_lo;
-	return ret;
-}
-
-static int smu_v13_0_12_erase_ras_table(struct amdgpu_device *adev,
-					uint32_t *result)
-{
-	struct smu_context *smu = adev->powerplay.pp_handle;
-
-	return smu_cmn_send_smc_msg_with_param(smu,
-			SMU_MSG_EraseRasTable, 0, result);
-}
-
-static int smu_v13_0_12_get_badpage_mca_addr(struct amdgpu_device *adev,
-					     uint16_t index, uint64_t *mca_addr)
-{
-	struct smu_context *smu = adev->powerplay.pp_handle;
-	uint32_t temp_arg, temp_addr_lo, temp_addr_high;
-	int ret;
-
-	temp_arg = index | (1 << 16);
-	ret = smu_cmn_send_smc_msg_with_param(smu,
-			SMU_MSG_GetBadPageMcaAddr, temp_arg, &temp_addr_lo);
-	if (ret)
-		return ret;
-
-	temp_arg = index | (2 << 16);
-	ret = smu_cmn_send_smc_msg_with_param(smu,
-			SMU_MSG_GetBadPageMcaAddr, temp_arg, &temp_addr_high);
-	if (!ret)
-		*mca_addr = (uint64_t)temp_addr_high << 32 | temp_addr_lo;
-	return ret;
-}
-
-static const struct ras_eeprom_smu_funcs smu_v13_0_12_eeprom_smu_funcs = {
-	.get_ras_table_version = smu_v13_0_12_get_ras_table_version,
-	.get_badpage_count = smu_v13_0_12_get_badpage_count,
-	.get_badpage_mca_addr = smu_v13_0_12_get_badpage_mca_addr,
-	.set_timestamp = smu_v13_0_12_set_timestamp,
-	.get_timestamp = smu_v13_0_12_get_timestamp,
-	.get_badpage_ipid = smu_v13_0_12_get_badpage_ipid,
-	.erase_ras_table = smu_v13_0_12_erase_ras_table,
-};
-
-static void smu_v13_0_12_ras_smu_feature_flags(struct amdgpu_device *adev, uint64_t *flags)
-{
-	struct smu_context *smu = adev->powerplay.pp_handle;
-
-	if (!flags)
-		return;
-
-	*flags = 0ULL;
-
-	if (smu_v13_0_6_cap_supported(smu, SMU_CAP(RAS_EEPROM)))
-		*flags |= RAS_SMU_FEATURE_BIT__RAS_EEPROM;
-
-}
-
-const struct ras_smu_drv smu_v13_0_12_ras_smu_drv = {
-	.smu_eeprom_funcs = &smu_v13_0_12_eeprom_smu_funcs,
-	.ras_smu_feature_flags = smu_v13_0_12_ras_smu_feature_flags,
-};
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
index b123881344890..64db62a8d6428 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
@@ -44,8 +44,6 @@
 #include "amdgpu_xgmi.h"
 #include <linux/pci.h>
 #include "amdgpu_ras.h"
-#include "amdgpu_mca.h"
-#include "amdgpu_aca.h"
 #include "smu_cmn.h"
 #include "mp/mp_13_0_6_offset.h"
 #include "mp/mp_13_0_6_sh_mask.h"
@@ -99,25 +97,6 @@ static const struct smu_feature_bits smu_v13_0_6_dpm_features = {
 #define PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK 0xE0
 #define PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT 0x5
 #define LINK_SPEED_MAX 4
-#define MCA_BANK_IPID(_ip, _hwid, _type) \
-	[AMDGPU_MCA_IP_##_ip] = { .hwid = _hwid, .mcatype = _type, }
-
-struct mca_bank_ipid {
-	enum amdgpu_mca_ip ip;
-	uint16_t hwid;
-	uint16_t mcatype;
-};
-
-struct mca_ras_info {
-	enum amdgpu_ras_block blkid;
-	enum amdgpu_mca_ip ip;
-	int *err_code_array;
-	int err_code_count;
-	int (*get_err_count)(const struct mca_ras_info *mca_ras, struct amdgpu_device *adev,
-			     enum amdgpu_mca_error_type type, struct mca_bank_entry *entry, uint32_t *count);
-	bool (*bank_is_valid)(const struct mca_ras_info *mca_ras, struct amdgpu_device *adev,
-			      enum amdgpu_mca_error_type type, struct mca_bank_entry *entry);
-};
 
 #define P2S_TABLE_ID_A 0x50325341
 #define P2S_TABLE_ID_X 0x50325358
@@ -1943,17 +1922,6 @@ static int smu_v13_0_6_notify_unload(struct smu_context *smu)
 	return 0;
 }
 
-static int smu_v13_0_6_mca_set_debug_mode(struct smu_context *smu, bool enable)
-{
-	/* NOTE: this ClearMcaOnRead message is only supported for smu version 85.72.0 or higher */
-	if (!smu_v13_0_6_cap_supported(smu, SMU_CAP(MCA_DEBUG_MODE)))
-		return 0;
-
-	return smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_ClearMcaOnRead,
-					       enable ? 0 : ClearMcaOnRead_UE_FLAG_MASK | ClearMcaOnRead_CE_POLL_MASK,
-					       NULL);
-}
-
 static int smu_v13_0_6_system_features_control(struct smu_context *smu,
 					       bool enable)
 {
@@ -3299,651 +3267,12 @@ static int smu_v13_0_6_post_init(struct smu_context *smu)
 	return 0;
 }
 
-static int mca_smu_set_debug_mode(struct amdgpu_device *adev, bool enable)
-{
-	struct smu_context *smu = adev->powerplay.pp_handle;
-
-	return smu_v13_0_6_mca_set_debug_mode(smu, enable);
-}
-
-static int smu_v13_0_6_get_valid_mca_count(struct smu_context *smu, enum amdgpu_mca_error_type type, uint32_t *count)
-{
-	uint32_t msg;
-	int ret;
-
-	if (!count)
-		return -EINVAL;
-
-	switch (type) {
-	case AMDGPU_MCA_ERROR_TYPE_UE:
-		msg = SMU_MSG_QueryValidMcaCount;
-		break;
-	case AMDGPU_MCA_ERROR_TYPE_CE:
-		msg = SMU_MSG_QueryValidMcaCeCount;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	ret = smu_cmn_send_smc_msg(smu, msg, count);
-	if (ret) {
-		*count = 0;
-		return ret;
-	}
-
-	return 0;
-}
-
-static int __smu_v13_0_6_mca_dump_bank(struct smu_context *smu, enum amdgpu_mca_error_type type,
-				       int idx, int offset, uint32_t *val)
-{
-	uint32_t msg, param;
-
-	switch (type) {
-	case AMDGPU_MCA_ERROR_TYPE_UE:
-		msg = SMU_MSG_McaBankDumpDW;
-		break;
-	case AMDGPU_MCA_ERROR_TYPE_CE:
-		msg = SMU_MSG_McaBankCeDumpDW;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	param = ((idx & 0xffff) << 16) | (offset & 0xfffc);
-
-	return smu_cmn_send_smc_msg_with_param(smu, msg, param, val);
-}
-
-static int smu_v13_0_6_mca_dump_bank(struct smu_context *smu, enum amdgpu_mca_error_type type,
-				     int idx, int offset, uint32_t *val, int count)
-{
-	int ret, i;
-
-	if (!val)
-		return -EINVAL;
-
-	for (i = 0; i < count; i++) {
-		ret = __smu_v13_0_6_mca_dump_bank(smu, type, idx, offset + (i << 2), &val[i]);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static const struct mca_bank_ipid smu_v13_0_6_mca_ipid_table[AMDGPU_MCA_IP_COUNT] = {
-	MCA_BANK_IPID(UMC, 0x96, 0x0),
-	MCA_BANK_IPID(SMU, 0x01, 0x1),
-	MCA_BANK_IPID(MP5, 0x01, 0x2),
-	MCA_BANK_IPID(PCS_XGMI, 0x50, 0x0),
-};
-
-static void mca_bank_entry_info_decode(struct mca_bank_entry *entry, struct mca_bank_info *info)
-{
-	u64 ipid = entry->regs[MCA_REG_IDX_IPID];
-	u32 instidhi, instid;
-
-	/* NOTE: All MCA IPID register share the same format,
-	 * so the driver can share the MCMP1 register header file.
-	 * */
-
-	info->hwid = REG_GET_FIELD(ipid, MCMP1_IPIDT0, HardwareID);
-	info->mcatype = REG_GET_FIELD(ipid, MCMP1_IPIDT0, McaType);
-
-	/*
-	 * Unfied DieID Format: SAASS. A:AID, S:Socket.
-	 * Unfied DieID[4] = InstanceId[0]
-	 * Unfied DieID[0:3] = InstanceIdHi[0:3]
-	 */
-	instidhi = REG_GET_FIELD(ipid, MCMP1_IPIDT0, InstanceIdHi);
-	instid = REG_GET_FIELD(ipid, MCMP1_IPIDT0, InstanceIdLo);
-	info->aid = ((instidhi >> 2) & 0x03);
-	info->socket_id = ((instid & 0x1) << 2) | (instidhi & 0x03);
-}
-
-static int mca_bank_read_reg(struct amdgpu_device *adev, enum amdgpu_mca_error_type type,
-			     int idx, int reg_idx, uint64_t *val)
-{
-	struct smu_context *smu = adev->powerplay.pp_handle;
-	uint32_t data[2] = {0, 0};
-	int ret;
-
-	if (!val || reg_idx >= MCA_REG_IDX_COUNT)
-		return -EINVAL;
-
-	ret = smu_v13_0_6_mca_dump_bank(smu, type, idx, reg_idx * 8, data, ARRAY_SIZE(data));
-	if (ret)
-		return ret;
-
-	*val = (uint64_t)data[1] << 32 | data[0];
-
-	dev_dbg(adev->dev, "mca read bank reg: type:%s, index: %d, reg_idx: %d, val: 0x%016llx\n",
-		type == AMDGPU_MCA_ERROR_TYPE_UE ? "UE" : "CE", idx, reg_idx, *val);
-
-	return 0;
-}
-
-static int mca_get_mca_entry(struct amdgpu_device *adev, enum amdgpu_mca_error_type type,
-			     int idx, struct mca_bank_entry *entry)
-{
-	int i, ret;
-
-	/* NOTE: populated all mca register by default */
-	for (i = 0; i < ARRAY_SIZE(entry->regs); i++) {
-		ret = mca_bank_read_reg(adev, type, idx, i, &entry->regs[i]);
-		if (ret)
-			return ret;
-	}
-
-	entry->idx = idx;
-	entry->type = type;
-
-	mca_bank_entry_info_decode(entry, &entry->info);
-
-	return 0;
-}
-
-static int mca_decode_ipid_to_hwip(uint64_t val)
-{
-	const struct mca_bank_ipid *ipid;
-	uint16_t hwid, mcatype;
-	int i;
-
-	hwid = REG_GET_FIELD(val, MCMP1_IPIDT0, HardwareID);
-	mcatype = REG_GET_FIELD(val, MCMP1_IPIDT0, McaType);
-
-	for (i = 0; i < ARRAY_SIZE(smu_v13_0_6_mca_ipid_table); i++) {
-		ipid = &smu_v13_0_6_mca_ipid_table[i];
-
-		if (!ipid->hwid)
-			continue;
-
-		if (ipid->hwid == hwid && ipid->mcatype == mcatype)
-			return i;
-	}
-
-	return AMDGPU_MCA_IP_UNKNOW;
-}
-
-static int mca_umc_mca_get_err_count(const struct mca_ras_info *mca_ras, struct amdgpu_device *adev,
-				     enum amdgpu_mca_error_type type, struct mca_bank_entry *entry, uint32_t *count)
-{
-	uint64_t status0;
-	uint32_t ext_error_code;
-	uint32_t odecc_err_cnt;
-
-	status0 = entry->regs[MCA_REG_IDX_STATUS];
-	ext_error_code = MCA_REG__STATUS__ERRORCODEEXT(status0);
-	odecc_err_cnt = MCA_REG__MISC0__ERRCNT(entry->regs[MCA_REG_IDX_MISC0]);
-
-	if (!REG_GET_FIELD(status0, MCMP1_STATUST0, Val)) {
-		*count = 0;
-		return 0;
-	}
-
-	if (umc_v12_0_is_deferred_error(adev, status0) ||
-	    umc_v12_0_is_uncorrectable_error(adev, status0) ||
-	    umc_v12_0_is_correctable_error(adev, status0))
-		*count = (ext_error_code == 0) ? odecc_err_cnt : 1;
-
-	amdgpu_umc_update_ecc_status(adev,
-			entry->regs[MCA_REG_IDX_STATUS],
-			entry->regs[MCA_REG_IDX_IPID],
-			entry->regs[MCA_REG_IDX_ADDR]);
-
-	return 0;
-}
-
-static int mca_pcs_xgmi_mca_get_err_count(const struct mca_ras_info *mca_ras, struct amdgpu_device *adev,
-					  enum amdgpu_mca_error_type type, struct mca_bank_entry *entry,
-					  uint32_t *count)
-{
-	u32 ext_error_code;
-	u32 err_cnt;
-
-	ext_error_code = MCA_REG__STATUS__ERRORCODEEXT(entry->regs[MCA_REG_IDX_STATUS]);
-	err_cnt = MCA_REG__MISC0__ERRCNT(entry->regs[MCA_REG_IDX_MISC0]);
-
-	if (type == AMDGPU_MCA_ERROR_TYPE_UE &&
-	    (ext_error_code == 0 || ext_error_code == 9))
-		*count = err_cnt;
-	else if (type == AMDGPU_MCA_ERROR_TYPE_CE && ext_error_code == 6)
-		*count = err_cnt;
-
-	return 0;
-}
-
-static bool mca_smu_check_error_code(struct amdgpu_device *adev, const struct mca_ras_info *mca_ras,
-				     uint32_t errcode)
-{
-	int i;
-
-	if (!mca_ras->err_code_count || !mca_ras->err_code_array)
-		return true;
-
-	for (i = 0; i < mca_ras->err_code_count; i++) {
-		if (errcode == mca_ras->err_code_array[i])
-			return true;
-	}
-
-	return false;
-}
-
-static int mca_gfx_mca_get_err_count(const struct mca_ras_info *mca_ras, struct amdgpu_device *adev,
-				     enum amdgpu_mca_error_type type, struct mca_bank_entry *entry, uint32_t *count)
-{
-	uint64_t status0, misc0;
-
-	status0 = entry->regs[MCA_REG_IDX_STATUS];
-	if (!REG_GET_FIELD(status0, MCMP1_STATUST0, Val)) {
-		*count = 0;
-		return 0;
-	}
-
-	if (type == AMDGPU_MCA_ERROR_TYPE_UE &&
-	    REG_GET_FIELD(status0, MCMP1_STATUST0, UC) == 1 &&
-	    REG_GET_FIELD(status0, MCMP1_STATUST0, PCC) == 1) {
-		*count = 1;
-		return 0;
-	} else {
-		misc0 = entry->regs[MCA_REG_IDX_MISC0];
-		*count = REG_GET_FIELD(misc0, MCMP1_MISC0T0, ErrCnt);
-	}
-
-	return 0;
-}
-
-static int mca_smu_mca_get_err_count(const struct mca_ras_info *mca_ras, struct amdgpu_device *adev,
-				     enum amdgpu_mca_error_type type, struct mca_bank_entry *entry, uint32_t *count)
-{
-	uint64_t status0, misc0;
-
-	status0 = entry->regs[MCA_REG_IDX_STATUS];
-	if (!REG_GET_FIELD(status0, MCMP1_STATUST0, Val)) {
-		*count = 0;
-		return 0;
-	}
-
-	if (type == AMDGPU_MCA_ERROR_TYPE_UE &&
-	    REG_GET_FIELD(status0, MCMP1_STATUST0, UC) == 1 &&
-	    REG_GET_FIELD(status0, MCMP1_STATUST0, PCC) == 1) {
-		if (count)
-			*count = 1;
-		return 0;
-	}
-
-	misc0 = entry->regs[MCA_REG_IDX_MISC0];
-	*count = REG_GET_FIELD(misc0, MCMP1_MISC0T0, ErrCnt);
-
-	return 0;
-}
-
-static bool mca_gfx_smu_bank_is_valid(const struct mca_ras_info *mca_ras, struct amdgpu_device *adev,
-				      enum amdgpu_mca_error_type type, struct mca_bank_entry *entry)
-{
-	uint32_t instlo;
-
-	instlo = REG_GET_FIELD(entry->regs[MCA_REG_IDX_IPID], MCMP1_IPIDT0, InstanceIdLo);
-	instlo &= GENMASK(31, 1);
-	switch (instlo) {
-	case 0x36430400: /* SMNAID XCD 0 */
-	case 0x38430400: /* SMNAID XCD 1 */
-	case 0x40430400: /* SMNXCD XCD 0, NOTE: FIXME: fix this error later */
-		return true;
-	default:
-		return false;
-	}
-
-	return false;
-};
-
-static bool mca_smu_bank_is_valid(const struct mca_ras_info *mca_ras, struct amdgpu_device *adev,
-				  enum amdgpu_mca_error_type type, struct mca_bank_entry *entry)
-{
-	struct smu_context *smu = adev->powerplay.pp_handle;
-	uint32_t errcode, instlo;
-
-	instlo = REG_GET_FIELD(entry->regs[MCA_REG_IDX_IPID], MCMP1_IPIDT0, InstanceIdLo);
-	instlo &= GENMASK(31, 1);
-	if (instlo != 0x03b30400)
-		return false;
-
-	if (smu_v13_0_6_cap_supported(smu, SMU_CAP(ACA_SYND))) {
-		errcode = MCA_REG__SYND__ERRORINFORMATION(entry->regs[MCA_REG_IDX_SYND]);
-		errcode &= 0xff;
-	} else {
-		errcode = REG_GET_FIELD(entry->regs[MCA_REG_IDX_STATUS], MCMP1_STATUST0, ErrorCode);
-	}
-
-	return mca_smu_check_error_code(adev, mca_ras, errcode);
-}
-
-static int sdma_err_codes[] = { CODE_SDMA0, CODE_SDMA1, CODE_SDMA2, CODE_SDMA3 };
-static int mmhub_err_codes[] = {
-	CODE_DAGB0, CODE_DAGB0 + 1, CODE_DAGB0 + 2, CODE_DAGB0 + 3, CODE_DAGB0 + 4, /* DAGB0-4 */
-	CODE_EA0, CODE_EA0 + 1, CODE_EA0 + 2, CODE_EA0 + 3, CODE_EA0 + 4,	/* MMEA0-4*/
-	CODE_VML2, CODE_VML2_WALKER, CODE_MMCANE,
-};
-
-static int vcn_err_codes[] = {
-	CODE_VIDD, CODE_VIDV,
-};
-static int jpeg_err_codes[] = {
-	CODE_JPEG0S, CODE_JPEG0D, CODE_JPEG1S, CODE_JPEG1D,
-	CODE_JPEG2S, CODE_JPEG2D, CODE_JPEG3S, CODE_JPEG3D,
-	CODE_JPEG4S, CODE_JPEG4D, CODE_JPEG5S, CODE_JPEG5D,
-	CODE_JPEG6S, CODE_JPEG6D, CODE_JPEG7S, CODE_JPEG7D,
-};
-
-static const struct mca_ras_info mca_ras_table[] = {
-	{
-		.blkid = AMDGPU_RAS_BLOCK__UMC,
-		.ip = AMDGPU_MCA_IP_UMC,
-		.get_err_count = mca_umc_mca_get_err_count,
-	}, {
-		.blkid = AMDGPU_RAS_BLOCK__GFX,
-		.ip = AMDGPU_MCA_IP_SMU,
-		.get_err_count = mca_gfx_mca_get_err_count,
-		.bank_is_valid = mca_gfx_smu_bank_is_valid,
-	}, {
-		.blkid = AMDGPU_RAS_BLOCK__SDMA,
-		.ip = AMDGPU_MCA_IP_SMU,
-		.err_code_array = sdma_err_codes,
-		.err_code_count = ARRAY_SIZE(sdma_err_codes),
-		.get_err_count = mca_smu_mca_get_err_count,
-		.bank_is_valid = mca_smu_bank_is_valid,
-	}, {
-		.blkid = AMDGPU_RAS_BLOCK__MMHUB,
-		.ip = AMDGPU_MCA_IP_SMU,
-		.err_code_array = mmhub_err_codes,
-		.err_code_count = ARRAY_SIZE(mmhub_err_codes),
-		.get_err_count = mca_smu_mca_get_err_count,
-		.bank_is_valid = mca_smu_bank_is_valid,
-	}, {
-		.blkid = AMDGPU_RAS_BLOCK__XGMI_WAFL,
-		.ip = AMDGPU_MCA_IP_PCS_XGMI,
-		.get_err_count = mca_pcs_xgmi_mca_get_err_count,
-	}, {
-		.blkid = AMDGPU_RAS_BLOCK__VCN,
-		.ip = AMDGPU_MCA_IP_SMU,
-		.err_code_array = vcn_err_codes,
-		.err_code_count = ARRAY_SIZE(vcn_err_codes),
-		.get_err_count = mca_smu_mca_get_err_count,
-		.bank_is_valid = mca_smu_bank_is_valid,
-	}, {
-		.blkid = AMDGPU_RAS_BLOCK__JPEG,
-		.ip = AMDGPU_MCA_IP_SMU,
-		.err_code_array = jpeg_err_codes,
-		.err_code_count = ARRAY_SIZE(jpeg_err_codes),
-		.get_err_count = mca_smu_mca_get_err_count,
-		.bank_is_valid = mca_smu_bank_is_valid,
-	},
-};
-
-static const struct mca_ras_info *mca_get_mca_ras_info(struct amdgpu_device *adev, enum amdgpu_ras_block blkid)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(mca_ras_table); i++) {
-		if (mca_ras_table[i].blkid == blkid)
-			return &mca_ras_table[i];
-	}
-
-	return NULL;
-}
-
-static int mca_get_valid_mca_count(struct amdgpu_device *adev, enum amdgpu_mca_error_type type, uint32_t *count)
-{
-	struct smu_context *smu = adev->powerplay.pp_handle;
-	int ret;
-
-	switch (type) {
-	case AMDGPU_MCA_ERROR_TYPE_UE:
-	case AMDGPU_MCA_ERROR_TYPE_CE:
-		ret = smu_v13_0_6_get_valid_mca_count(smu, type, count);
-		break;
-	default:
-		ret = -EINVAL;
-		break;
-	}
-
-	return ret;
-}
-
-static bool mca_bank_is_valid(struct amdgpu_device *adev, const struct mca_ras_info *mca_ras,
-			      enum amdgpu_mca_error_type type, struct mca_bank_entry *entry)
-{
-	if (mca_decode_ipid_to_hwip(entry->regs[MCA_REG_IDX_IPID]) != mca_ras->ip)
-		return false;
-
-	if (mca_ras->bank_is_valid)
-		return mca_ras->bank_is_valid(mca_ras, adev, type, entry);
-
-	return true;
-}
-
-static int mca_smu_parse_mca_error_count(struct amdgpu_device *adev, enum amdgpu_ras_block blk, enum amdgpu_mca_error_type type,
-					 struct mca_bank_entry *entry, uint32_t *count)
-{
-	const struct mca_ras_info *mca_ras;
-
-	if (!entry || !count)
-		return -EINVAL;
-
-	mca_ras = mca_get_mca_ras_info(adev, blk);
-	if (!mca_ras)
-		return -EOPNOTSUPP;
-
-	if (!mca_bank_is_valid(adev, mca_ras, type, entry)) {
-		*count = 0;
-		return 0;
-	}
-
-	return mca_ras->get_err_count(mca_ras, adev, type, entry, count);
-}
-
-static int mca_smu_get_mca_entry(struct amdgpu_device *adev,
-				 enum amdgpu_mca_error_type type, int idx, struct mca_bank_entry *entry)
-{
-	return mca_get_mca_entry(adev, type, idx, entry);
-}
-
-static int mca_smu_get_valid_mca_count(struct amdgpu_device *adev,
-				       enum amdgpu_mca_error_type type, uint32_t *count)
-{
-	return mca_get_valid_mca_count(adev, type, count);
-}
-
-static const struct amdgpu_mca_smu_funcs smu_v13_0_6_mca_smu_funcs = {
-	.max_ue_count = 12,
-	.max_ce_count = 12,
-	.mca_set_debug_mode = mca_smu_set_debug_mode,
-	.mca_parse_mca_error_count = mca_smu_parse_mca_error_count,
-	.mca_get_mca_entry = mca_smu_get_mca_entry,
-	.mca_get_valid_mca_count = mca_smu_get_valid_mca_count,
-};
-
-static int aca_smu_set_debug_mode(struct amdgpu_device *adev, bool enable)
-{
-	struct smu_context *smu = adev->powerplay.pp_handle;
-
-	return smu_v13_0_6_mca_set_debug_mode(smu, enable);
-}
-
-static int smu_v13_0_6_get_valid_aca_count(struct smu_context *smu, enum aca_smu_type type, u32 *count)
-{
-	uint32_t msg;
-	int ret;
-
-	if (!count)
-		return -EINVAL;
-
-	switch (type) {
-	case ACA_SMU_TYPE_UE:
-		msg = SMU_MSG_QueryValidMcaCount;
-		break;
-	case ACA_SMU_TYPE_CE:
-		msg = SMU_MSG_QueryValidMcaCeCount;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	ret = smu_cmn_send_smc_msg(smu, msg, count);
-	if (ret) {
-		*count = 0;
-		return ret;
-	}
-
-	return 0;
-}
-
-static int aca_smu_get_valid_aca_count(struct amdgpu_device *adev,
-				       enum aca_smu_type type, u32 *count)
-{
-	struct smu_context *smu = adev->powerplay.pp_handle;
-	int ret;
-
-	switch (type) {
-	case ACA_SMU_TYPE_UE:
-	case ACA_SMU_TYPE_CE:
-		ret = smu_v13_0_6_get_valid_aca_count(smu, type, count);
-		break;
-	default:
-		ret = -EINVAL;
-		break;
-	}
-
-	return ret;
-}
-
-static int __smu_v13_0_6_aca_bank_dump(struct smu_context *smu, enum aca_smu_type type,
-				       int idx, int offset, u32 *val)
-{
-	uint32_t msg, param;
-
-	switch (type) {
-	case ACA_SMU_TYPE_UE:
-		msg = SMU_MSG_McaBankDumpDW;
-		break;
-	case ACA_SMU_TYPE_CE:
-		msg = SMU_MSG_McaBankCeDumpDW;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	param = ((idx & 0xffff) << 16) | (offset & 0xfffc);
-
-	return smu_cmn_send_smc_msg_with_param(smu, msg, param, (uint32_t *)val);
-}
-
-static int smu_v13_0_6_aca_bank_dump(struct smu_context *smu, enum aca_smu_type type,
-				     int idx, int offset, u32 *val, int count)
-{
-	int ret, i;
-
-	if (!val)
-		return -EINVAL;
-
-	for (i = 0; i < count; i++) {
-		ret = __smu_v13_0_6_aca_bank_dump(smu, type, idx, offset + (i << 2), &val[i]);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static int aca_bank_read_reg(struct amdgpu_device *adev, enum aca_smu_type type,
-			     int idx, int reg_idx, u64 *val)
-{
-	struct smu_context *smu = adev->powerplay.pp_handle;
-	u32 data[2] = {0, 0};
-	int ret;
-
-	if (!val || reg_idx >= ACA_REG_IDX_COUNT)
-		return -EINVAL;
-
-	ret = smu_v13_0_6_aca_bank_dump(smu, type, idx, reg_idx * 8, data, ARRAY_SIZE(data));
-	if (ret)
-		return ret;
-
-	*val = (u64)data[1] << 32 | data[0];
-
-	dev_dbg(adev->dev, "mca read bank reg: type:%s, index: %d, reg_idx: %d, val: 0x%016llx\n",
-		type == ACA_SMU_TYPE_UE ? "UE" : "CE", idx, reg_idx, *val);
-
-	return 0;
-}
-
-static int aca_smu_get_valid_aca_bank(struct amdgpu_device *adev,
-				      enum aca_smu_type type, int idx, struct aca_bank *bank)
-{
-	int i, ret, count;
-
-	count = min_t(int, 16, ARRAY_SIZE(bank->regs));
-	for (i = 0; i < count; i++) {
-		ret = aca_bank_read_reg(adev, type, idx, i, &bank->regs[i]);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static int aca_smu_parse_error_code(struct amdgpu_device *adev, struct aca_bank *bank)
-{
-	struct smu_context *smu = adev->powerplay.pp_handle;
-	int error_code;
-
-	if (smu_v13_0_6_cap_supported(smu, SMU_CAP(ACA_SYND)))
-		error_code = ACA_REG__SYND__ERRORINFORMATION(bank->regs[ACA_REG_IDX_SYND]);
-	else
-		error_code = ACA_REG__STATUS__ERRORCODE(bank->regs[ACA_REG_IDX_STATUS]);
-
-	return error_code & 0xff;
-}
-
-static const struct aca_smu_funcs smu_v13_0_6_aca_smu_funcs = {
-	.max_ue_bank_count = 12,
-	.max_ce_bank_count = 12,
-	.set_debug_mode = aca_smu_set_debug_mode,
-	.get_valid_aca_count = aca_smu_get_valid_aca_count,
-	.get_valid_aca_bank = aca_smu_get_valid_aca_bank,
-	.parse_error_code = aca_smu_parse_error_code,
-};
-
 static void smu_v13_0_6_set_temp_funcs(struct smu_context *smu)
 {
 	smu->smu_temp.temp_funcs = (amdgpu_ip_version(smu->adev, MP1_HWIP, 0)
 			== IP_VERSION(13, 0, 12)) ? &smu_v13_0_12_temp_funcs : NULL;
 }
 
-static int smu_v13_0_6_get_ras_smu_drv(struct smu_context *smu, const struct ras_smu_drv **ras_smu_drv)
-{
-	if (!ras_smu_drv)
-		return -EINVAL;
-
-	if (amdgpu_sriov_vf(smu->adev))
-		return -EOPNOTSUPP;
-
-	if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_HROM_EN_BIT))
-		smu_v13_0_6_cap_set(smu, SMU_CAP(RAS_EEPROM));
-
-	switch (amdgpu_ip_version(smu->adev, MP1_HWIP, 0)) {
-	case IP_VERSION(13, 0, 12):
-		*ras_smu_drv = &smu_v13_0_12_ras_smu_drv;
-		break;
-	default:
-		*ras_smu_drv = NULL;
-		break;
-	}
-
-	return 0;
-}
-
 static const struct pptable_funcs smu_v13_0_6_ppt_funcs = {
 	/* init dpm */
 	.init_allowed_features = smu_v13_0_6_init_allowed_features,
@@ -4002,7 +3331,6 @@ static const struct pptable_funcs smu_v13_0_6_ppt_funcs = {
 	.dpm_reset_vcn = smu_v13_0_6_reset_vcn,
 	.post_init = smu_v13_0_6_post_init,
 	.ras_send_msg = smu_v13_0_6_ras_send_msg,
-	.get_ras_smu_drv = smu_v13_0_6_get_ras_smu_drv,
 };
 
 void smu_v13_0_6_set_ppt_funcs(struct smu_context *smu)
@@ -4020,7 +3348,5 @@ void smu_v13_0_6_set_ppt_funcs(struct smu_context *smu)
 	smu->smc_fw_caps |= SMU_FW_CAP_RAS_PRI;
 	smu_v13_0_init_msg_ctl(smu, message_map);
 	smu_v13_0_6_set_temp_funcs(smu);
-	amdgpu_mca_smu_init_funcs(smu->adev, &smu_v13_0_6_mca_smu_funcs);
-	amdgpu_aca_set_smu_funcs(smu->adev, &smu_v13_0_6_aca_smu_funcs);
 }
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h
index a150fc88902c4..6554780afff77 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h
@@ -117,7 +117,6 @@ int smu_v13_0_12_get_system_power(struct smu_context *smu,
 extern const struct cmn2asic_mapping smu_v13_0_12_feature_mask_map[];
 extern const struct cmn2asic_msg_mapping smu_v13_0_12_message_map[];
 extern const struct smu_temp_funcs smu_v13_0_12_temp_funcs;
-extern const struct ras_smu_drv smu_v13_0_12_ras_smu_drv;
 
 #if defined(SWSMU_CODE_LAYER_L2)
 #include "smu_cmn.h"
@@ -145,15 +144,15 @@ extern const struct ras_smu_drv smu_v13_0_12_ras_smu_drv;
 	SMU_SCALAR(SMU_MATTR(ACCUMULATION_COUNTER), SMU_MUNIT(NONE),           \
 		   SMU_MTYPE(U64), accumulation_counter);                      \
 	SMU_SCALAR(SMU_MATTR(PROCHOT_RESIDENCY_ACC), SMU_MUNIT(NONE),          \
-		   SMU_MTYPE(U32), prochot_residency_acc);                     \
+		   SMU_MTYPE(U64), prochot_residency_acc);                     \
 	SMU_SCALAR(SMU_MATTR(PPT_RESIDENCY_ACC), SMU_MUNIT(NONE),              \
-		   SMU_MTYPE(U32), ppt_residency_acc);                         \
+		   SMU_MTYPE(U64), ppt_residency_acc);                         \
 	SMU_SCALAR(SMU_MATTR(SOCKET_THM_RESIDENCY_ACC), SMU_MUNIT(NONE),       \
-		   SMU_MTYPE(U32), socket_thm_residency_acc);                  \
+		   SMU_MTYPE(U64), socket_thm_residency_acc);                  \
 	SMU_SCALAR(SMU_MATTR(VR_THM_RESIDENCY_ACC), SMU_MUNIT(NONE),           \
-		   SMU_MTYPE(U32), vr_thm_residency_acc);                      \
+		   SMU_MTYPE(U64), vr_thm_residency_acc);                      \
 	SMU_SCALAR(SMU_MATTR(HBM_THM_RESIDENCY_ACC), SMU_MUNIT(NONE),          \
-		   SMU_MTYPE(U32), hbm_thm_residency_acc);                     \
+		   SMU_MTYPE(U64), hbm_thm_residency_acc);                     \
 	SMU_SCALAR(SMU_MATTR(GFXCLK_LOCK_STATUS), SMU_MUNIT(NONE),             \
 		   SMU_MTYPE(U32), gfxclk_lock_status);                        \
 	SMU_SCALAR(SMU_MATTR(PCIE_LINK_WIDTH), SMU_MUNIT(NONE),                \
@@ -164,11 +163,11 @@ extern const struct ras_smu_drv smu_v13_0_12_ras_smu_drv;
 		   SMU_MTYPE(U16), xgmi_link_width);                           \
 	SMU_SCALAR(SMU_MATTR(XGMI_LINK_SPEED), SMU_MUNIT(SPEED_1),             \
 		   SMU_MTYPE(U16), xgmi_link_speed);                           \
-	SMU_SCALAR(SMU_MATTR(GFX_ACTIVITY_ACC), SMU_MUNIT(PERCENT),            \
-		   SMU_MTYPE(U32), gfx_activity_acc);                          \
-	SMU_SCALAR(SMU_MATTR(MEM_ACTIVITY_ACC), SMU_MUNIT(PERCENT),            \
-		   SMU_MTYPE(U32), mem_activity_acc);                          \
-	SMU_SCALAR(SMU_MATTR(PCIE_BANDWIDTH_ACC), SMU_MUNIT(PERCENT),          \
+	SMU_SCALAR(SMU_MATTR(GFX_ACTIVITY_ACC), SMU_MUNIT(NONE),            \
+		   SMU_MTYPE(U64), gfx_activity_acc);                          \
+	SMU_SCALAR(SMU_MATTR(MEM_ACTIVITY_ACC), SMU_MUNIT(NONE),            \
+		   SMU_MTYPE(U64), mem_activity_acc);                          \
+	SMU_SCALAR(SMU_MATTR(PCIE_BANDWIDTH_ACC), SMU_MUNIT(NONE),          \
 		   SMU_MTYPE(U64), pcie_bandwidth_acc);                        \
 	SMU_SCALAR(SMU_MATTR(PCIE_BANDWIDTH_INST), SMU_MUNIT(BW_1),            \
 		   SMU_MTYPE(U64), pcie_bandwidth_inst);                       \
@@ -179,9 +178,9 @@ extern const struct ras_smu_drv smu_v13_0_12_ras_smu_drv;
 	SMU_SCALAR(SMU_MATTR(PCIE_REPLAY_ROVER_COUNT_ACC), SMU_MUNIT(NONE),    \
 		   SMU_MTYPE(U64), pcie_replay_rover_count_acc);               \
 	SMU_SCALAR(SMU_MATTR(PCIE_NAK_SENT_COUNT_ACC), SMU_MUNIT(NONE),        \
-		   SMU_MTYPE(U32), pcie_nak_sent_count_acc);                   \
+		   SMU_MTYPE(U64), pcie_nak_sent_count_acc);                   \
 	SMU_SCALAR(SMU_MATTR(PCIE_NAK_RCVD_COUNT_ACC), SMU_MUNIT(NONE),        \
-		   SMU_MTYPE(U32), pcie_nak_rcvd_count_acc);                   \
+		   SMU_MTYPE(U64), pcie_nak_rcvd_count_acc);                   \
 	SMU_ARRAY(SMU_MATTR(XGMI_READ_DATA_ACC), SMU_MUNIT(DATA_1),            \
 		  SMU_MTYPE(U64), xgmi_read_data_acc,                          \
 		  SMU_13_0_6_NUM_XGMI_LINKS);                                  \
@@ -204,7 +203,7 @@ extern const struct ras_smu_drv smu_v13_0_12_ras_smu_drv;
 	SMU_SCALAR(SMU_MATTR(CURRENT_UCLK), SMU_MUNIT(CLOCK_1),                \
 		   SMU_MTYPE(U16), current_uclk);                              \
 	SMU_SCALAR(SMU_MATTR(PCIE_LC_PERF_OTHER_END_RECOVERY),                 \
-		   SMU_MUNIT(NONE), SMU_MTYPE(U32),                            \
+		   SMU_MUNIT(NONE), SMU_MTYPE(U64),                            \
 		   pcie_lc_perf_other_end_recovery);                           \
 	SMU_ARRAY(SMU_MATTR(GFX_BUSY_INST), SMU_MUNIT(PERCENT),                \
 		  SMU_MTYPE(U32), gfx_busy_inst, SMU_13_0_6_MAX_XCC);          \
@@ -212,7 +211,7 @@ extern const struct ras_smu_drv smu_v13_0_12_ras_smu_drv;
 		  jpeg_busy, SMU_13_0_6_MAX_JPEG);                             \
 	SMU_ARRAY(SMU_MATTR(VCN_BUSY), SMU_MUNIT(PERCENT), SMU_MTYPE(U16),     \
 		  vcn_busy, SMU_13_0_6_MAX_VCN);                               \
-	SMU_ARRAY(SMU_MATTR(GFX_BUSY_ACC), SMU_MUNIT(PERCENT), SMU_MTYPE(U64), \
+	SMU_ARRAY(SMU_MATTR(GFX_BUSY_ACC), SMU_MUNIT(NONE), SMU_MTYPE(U64), \
 		  gfx_busy_acc, SMU_13_0_6_MAX_XCC);                           \
 	SMU_ARRAY(SMU_MATTR(GFX_BELOW_HOST_LIMIT_PPT_ACC), SMU_MUNIT(NONE),    \
 		  SMU_MTYPE(U64), gfx_below_host_limit_ppt_acc,                \
@@ -257,7 +256,7 @@ void smu_v13_0_12_get_gpu_metrics(struct smu_context *smu, void **table,
 		  jpeg_busy, SMU_13_0_6_MAX_JPEG);                             \
 	SMU_ARRAY(SMU_MATTR(VCN_BUSY), SMU_MUNIT(PERCENT), SMU_MTYPE(U16),     \
 		  vcn_busy, SMU_13_0_6_MAX_VCN);                               \
-	SMU_ARRAY(SMU_MATTR(GFX_BUSY_ACC), SMU_MUNIT(PERCENT), SMU_MTYPE(U64), \
+	SMU_ARRAY(SMU_MATTR(GFX_BUSY_ACC), SMU_MUNIT(NONE), SMU_MTYPE(U64), \
 		  gfx_busy_acc, SMU_13_0_6_MAX_XCC);                           \
 	SMU_ARRAY(SMU_MATTR(GFX_BELOW_HOST_LIMIT_PPT_ACC), SMU_MUNIT(NONE),    \
 		  SMU_MTYPE(U64), gfx_below_host_limit_ppt_acc,                \
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
index 42c9ceeb4f7db..5f23f2e7f401c 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
@@ -2574,9 +2574,10 @@ static int smu_v13_0_7_set_power_profile_mode(struct smu_context *smu,
 				return -ENOMEM;
 		}
 		if (custom_params && custom_params_max_idx) {
-			if (custom_params_max_idx != SMU_13_0_7_CUSTOM_PARAMS_COUNT)
-				return -EINVAL;
-			if (custom_params[0] >= SMU_13_0_7_CUSTOM_PARAMS_CLOCK_COUNT)
+			if (!smu_cmn_custom_params_count_valid(custom_params_max_idx,
+							       SMU_13_0_7_CUSTOM_PARAMS_COUNT) ||
+			    !smu_cmn_custom_params_clock_valid(custom_params[0],
+							       SMU_13_0_7_CUSTOM_PARAMS_CLOCK_COUNT))
 				return -EINVAL;
 			idx = custom_params[0] * SMU_13_0_7_CUSTOM_PARAMS_COUNT;
 			smu->custom_profile_params[idx] = 1;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c
index d0a8df1aa6b64..2a0c7cde938d3 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c
@@ -194,7 +194,7 @@ int smu_v14_0_init_pptable_microcode(struct smu_context *smu)
 	if (!pptable_id)
 		return 0;
 
-	ret = smu_v14_0_get_pptable_from_firmware(smu, &table, &size, pptable_id);
+	ret = smu_cmn_get_pptable_from_firmware(smu, &table, &size, pptable_id);
 	if (ret)
 		return ret;
 
@@ -229,48 +229,6 @@ int smu_v14_0_check_fw_status(struct smu_context *smu)
 	return -EIO;
 }
 
-static int smu_v14_0_set_pptable_v2_0(struct smu_context *smu, void **table, uint32_t *size)
-{
-	struct amdgpu_device *adev = smu->adev;
-	uint32_t ppt_offset_bytes;
-	const struct smc_firmware_header_v2_0 *v2;
-
-	v2 = (const struct smc_firmware_header_v2_0 *) adev->pm.fw->data;
-
-	ppt_offset_bytes = le32_to_cpu(v2->ppt_offset_bytes);
-	*size = le32_to_cpu(v2->ppt_size_bytes);
-	*table = (uint8_t *)v2 + ppt_offset_bytes;
-
-	return 0;
-}
-
-static int smu_v14_0_set_pptable_v2_1(struct smu_context *smu, void **table,
-				      uint32_t *size, uint32_t pptable_id)
-{
-	struct amdgpu_device *adev = smu->adev;
-	const struct smc_firmware_header_v2_1 *v2_1;
-	struct smc_soft_pptable_entry *entries;
-	uint32_t pptable_count = 0;
-	int i = 0;
-
-	v2_1 = (const struct smc_firmware_header_v2_1 *) adev->pm.fw->data;
-	entries = (struct smc_soft_pptable_entry *)
-		((uint8_t *)v2_1 + le32_to_cpu(v2_1->pptable_entry_offset));
-	pptable_count = le32_to_cpu(v2_1->pptable_count);
-	for (i = 0; i < pptable_count; i++) {
-		if (le32_to_cpu(entries[i].id) == pptable_id) {
-			*table = ((uint8_t *)v2_1 + le32_to_cpu(entries[i].ppt_offset_bytes));
-			*size = le32_to_cpu(entries[i].ppt_size_bytes);
-			break;
-		}
-	}
-
-	if (i == pptable_count)
-		return -EINVAL;
-
-	return 0;
-}
-
 static int smu_v14_0_get_pptable_from_vbios(struct smu_context *smu, void **table, uint32_t *size)
 {
 	struct amdgpu_device *adev = smu->adev;
@@ -293,45 +251,6 @@ static int smu_v14_0_get_pptable_from_vbios(struct smu_context *smu, void **tabl
 	return 0;
 }
 
-int smu_v14_0_get_pptable_from_firmware(struct smu_context *smu,
-					void **table,
-					uint32_t *size,
-					uint32_t pptable_id)
-{
-	const struct smc_firmware_header_v1_0 *hdr;
-	struct amdgpu_device *adev = smu->adev;
-	uint16_t version_major, version_minor;
-	int ret;
-
-	hdr = (const struct smc_firmware_header_v1_0 *) adev->pm.fw->data;
-	if (!hdr)
-		return -EINVAL;
-
-	dev_info(adev->dev, "use driver provided pptable %d\n", pptable_id);
-
-	version_major = le16_to_cpu(hdr->header.header_version_major);
-	version_minor = le16_to_cpu(hdr->header.header_version_minor);
-	if (version_major != 2) {
-		dev_err(adev->dev, "Unsupported smu firmware version %d.%d\n",
-			version_major, version_minor);
-		return -EINVAL;
-	}
-
-	switch (version_minor) {
-	case 0:
-		ret = smu_v14_0_set_pptable_v2_0(smu, table, size);
-		break;
-	case 1:
-		ret = smu_v14_0_set_pptable_v2_1(smu, table, size, pptable_id);
-		break;
-	default:
-		ret = -EINVAL;
-		break;
-	}
-
-	return ret;
-}
-
 int smu_v14_0_setup_pptable(struct smu_context *smu)
 {
 	struct amdgpu_device *adev = smu->adev;
@@ -351,7 +270,7 @@ int smu_v14_0_setup_pptable(struct smu_context *smu)
 	if ((amdgpu_sriov_vf(adev) || !pptable_id) && (amdgpu_emu_mode != 1))
 		ret = smu_v14_0_get_pptable_from_vbios(smu, &table, &size);
 	else
-		ret = smu_v14_0_get_pptable_from_firmware(smu, &table, &size, pptable_id);
+		ret = smu_cmn_get_pptable_from_firmware(smu, &table, &size, pptable_id);
 
 	if (ret)
 		return ret;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c
index 75719c47a41e2..3d73f2050bbef 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_0_ppt.c
@@ -1701,6 +1701,50 @@ static int smu_v14_0_0_restore_user_od_settings(struct smu_context *smu)
 	return 0;
 }
 
+/*
+ * Link any xHCI controller sharing the GPU's PCIe root port as a consumer
+ * of the GPU so the GPU resumes first, avoiding an xHCI resume race.
+ */
+static int smu_v14_0_0_set_power_dep(struct smu_context *smu, bool enable)
+{
+	struct amdgpu_device *adev = smu->adev;
+	struct pci_dev *gpu_pdev = adev->pdev;
+	struct pci_dev *root_port, *usb_pdev = NULL;
+	struct device_link *link;
+
+	if (!enable) {
+		if (smu->usb_power_link) {
+			device_link_del(smu->usb_power_link);
+			smu->usb_power_link = NULL;
+		}
+		return 0;
+	}
+
+	root_port = pcie_find_root_port(gpu_pdev);
+	while ((usb_pdev = pci_get_class(PCI_CLASS_SERIAL_USB_XHCI, usb_pdev))) {
+		struct pci_dev *usb_root;
+
+		usb_root = pcie_find_root_port(usb_pdev);
+		if (usb_root != root_port)
+			continue;
+
+		/* Create device link: USB (consumer) depends on GPU (supplier) */
+		link = device_link_add(&usb_pdev->dev, &gpu_pdev->dev,
+				       DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME);
+		if (link) {
+			smu->usb_power_link = link;
+			drm_info(adev_to_drm(adev), "USB controller %s D0 power state depends on %s\n",
+				 pci_name(usb_pdev), pci_name(gpu_pdev));
+			/* Only create one link for the first USB controller found */
+			break;
+		}
+	}
+
+	pci_dev_put(usb_pdev);
+
+	return 0;
+}
+
 static const struct pptable_funcs smu_v14_0_0_ppt_funcs = {
 	.check_fw_status = smu_v14_0_check_fw_status,
 	.check_fw_version = smu_cmn_check_fw_version,
@@ -1734,6 +1778,7 @@ static const struct pptable_funcs smu_v14_0_0_ppt_funcs = {
 	.dpm_set_umsch_mm_enable = smu_v14_0_0_set_umsch_mm_enable,
 	.get_dpm_clock_table = smu_v14_0_common_get_dpm_table,
 	.set_mall_enable = smu_v14_0_common_set_mall_enable,
+	.set_power_dep = smu_v14_0_0_set_power_dep,
 };
 
 static void smu_v14_0_0_init_msg_ctl(struct smu_context *smu)
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c
index a6a88e7b26685..0fbcaf884419d 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c
@@ -1832,9 +1832,10 @@ static int smu_v14_0_2_set_power_profile_mode(struct smu_context *smu,
 				return -ENOMEM;
 		}
 		if (custom_params && custom_params_max_idx) {
-			if (custom_params_max_idx != SMU_14_0_2_CUSTOM_PARAMS_COUNT)
-				return -EINVAL;
-			if (custom_params[0] >= SMU_14_0_2_CUSTOM_PARAMS_CLOCK_COUNT)
+			if (!smu_cmn_custom_params_count_valid(custom_params_max_idx,
+							       SMU_14_0_2_CUSTOM_PARAMS_COUNT) ||
+			    !smu_cmn_custom_params_clock_valid(custom_params[0],
+							       SMU_14_0_2_CUSTOM_PARAMS_CLOCK_COUNT))
 				return -EINVAL;
 			idx = custom_params[0] * SMU_14_0_2_CUSTOM_PARAMS_COUNT;
 			smu->custom_profile_params[idx] = 1;
@@ -2893,8 +2894,6 @@ static const struct pptable_funcs smu_v14_0_2_ppt_funcs = {
 	.deep_sleep_control = smu_v14_0_deep_sleep_control,
 	.gfx_ulv_control = smu_v14_0_gfx_ulv_control,
 	.get_bamaco_support = smu_v14_0_get_bamaco_support,
-	.baco_get_state = smu_v14_0_baco_get_state,
-	.baco_set_state = smu_v14_0_baco_set_state,
 	.baco_enter = smu_v14_0_2_baco_enter,
 	.baco_exit = smu_v14_0_2_baco_exit,
 	.mode1_reset_is_support = smu_v14_0_2_is_mode1_reset_supported,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c
index 8fc99e93ac536..046069e938540 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c
@@ -174,7 +174,7 @@ int smu_v15_0_init_pptable_microcode(struct smu_context *smu)
 	if (!pptable_id)
 		return 0;
 
-	ret = smu_v15_0_get_pptable_from_firmware(smu, &table, &size, pptable_id);
+	ret = smu_cmn_get_pptable_from_firmware(smu, &table, &size, pptable_id);
 	if (ret)
 		return ret;
 
@@ -207,48 +207,6 @@ int smu_v15_0_check_fw_status(struct smu_context *smu)
 	return -EIO;
 }
 
-static int smu_v15_0_set_pptable_v2_0(struct smu_context *smu, void **table, uint32_t *size)
-{
-	struct amdgpu_device *adev = smu->adev;
-	uint32_t ppt_offset_bytes;
-	const struct smc_firmware_header_v2_0 *v2;
-
-	v2 = (const struct smc_firmware_header_v2_0 *) adev->pm.fw->data;
-
-	ppt_offset_bytes = le32_to_cpu(v2->ppt_offset_bytes);
-	*size = le32_to_cpu(v2->ppt_size_bytes);
-	*table = (uint8_t *)v2 + ppt_offset_bytes;
-
-	return 0;
-}
-
-static int smu_v15_0_set_pptable_v2_1(struct smu_context *smu, void **table,
-				      uint32_t *size, uint32_t pptable_id)
-{
-	struct amdgpu_device *adev = smu->adev;
-	const struct smc_firmware_header_v2_1 *v2_1;
-	struct smc_soft_pptable_entry *entries;
-	uint32_t pptable_count = 0;
-	int i = 0;
-
-	v2_1 = (const struct smc_firmware_header_v2_1 *) adev->pm.fw->data;
-	entries = (struct smc_soft_pptable_entry *)
-		((uint8_t *)v2_1 + le32_to_cpu(v2_1->pptable_entry_offset));
-	pptable_count = le32_to_cpu(v2_1->pptable_count);
-	for (i = 0; i < pptable_count; i++) {
-		if (le32_to_cpu(entries[i].id) == pptable_id) {
-			*table = ((uint8_t *)v2_1 + le32_to_cpu(entries[i].ppt_offset_bytes));
-			*size = le32_to_cpu(entries[i].ppt_size_bytes);
-			break;
-		}
-	}
-
-	if (i == pptable_count)
-		return -EINVAL;
-
-	return 0;
-}
-
 static int smu_v15_0_get_pptable_from_vbios(struct smu_context *smu, void **table, uint32_t *size)
 {
 	struct amdgpu_device *adev = smu->adev;
@@ -271,45 +229,6 @@ static int smu_v15_0_get_pptable_from_vbios(struct smu_context *smu, void **tabl
 	return 0;
 }
 
-int smu_v15_0_get_pptable_from_firmware(struct smu_context *smu,
-					void **table,
-					uint32_t *size,
-					uint32_t pptable_id)
-{
-	const struct smc_firmware_header_v1_0 *hdr;
-	struct amdgpu_device *adev = smu->adev;
-	uint16_t version_major, version_minor;
-	int ret;
-
-	hdr = (const struct smc_firmware_header_v1_0 *) adev->pm.fw->data;
-	if (!hdr)
-		return -EINVAL;
-
-	dev_info(adev->dev, "use driver provided pptable %d\n", pptable_id);
-
-	version_major = le16_to_cpu(hdr->header.header_version_major);
-	version_minor = le16_to_cpu(hdr->header.header_version_minor);
-	if (version_major != 2) {
-		dev_err(adev->dev, "Unsupported smu firmware version %d.%d\n",
-			version_major, version_minor);
-		return -EINVAL;
-	}
-
-	switch (version_minor) {
-	case 0:
-		ret = smu_v15_0_set_pptable_v2_0(smu, table, size);
-		break;
-	case 1:
-		ret = smu_v15_0_set_pptable_v2_1(smu, table, size, pptable_id);
-		break;
-	default:
-		ret = -EINVAL;
-		break;
-	}
-
-	return ret;
-}
-
 int smu_v15_0_setup_pptable(struct smu_context *smu)
 {
 	struct amdgpu_device *adev = smu->adev;
@@ -329,7 +248,7 @@ int smu_v15_0_setup_pptable(struct smu_context *smu)
 	if ((amdgpu_sriov_vf(adev) || !pptable_id) && (amdgpu_emu_mode != 1))
 		ret = smu_v15_0_get_pptable_from_vbios(smu, &table, &size);
 	else
-		ret = smu_v15_0_get_pptable_from_firmware(smu, &table, &size, pptable_id);
+		ret = smu_cmn_get_pptable_from_firmware(smu, &table, &size, pptable_id);
 
 	if (ret)
 		return ret;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c
index 2105a1d7bb34b..227cd86e13e88 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c
@@ -1766,12 +1766,17 @@ static ssize_t smu_v15_0_8_get_gpu_metrics(struct smu_context *smu, void **table
 		idx++;
 	}
 
-	/* Per-VCN clocks */
+	/* Per-VCN clocks and busy */
 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 		inst = GET_INST(VCN, i);
 		if (inst >= 0) {
 			gpu_metrics->current_vclk0[i] = SMUQ10_ROUND(metrics->VclkFrequency[inst]);
 			gpu_metrics->current_dclk0[i] = SMUQ10_ROUND(metrics->DclkFrequency[inst]);
+			gpu_metrics->vcn_busy[i] = SMUQ10_ROUND(metrics->VcnBusy[inst]);
+			for (j = 0; j < NUM_JPEG_RINGS_FW; ++j)
+				gpu_metrics->jpeg_busy[(i * NUM_JPEG_RINGS_FW) + j] =
+					SMUQ10_ROUND(metrics->JpegBusy[(inst * NUM_JPEG_RINGS_FW)
+								       + j]);
 		}
 	}
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h
index 4dfc40aaffcb4..46fce0a79c21e 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h
@@ -125,7 +125,7 @@ typedef struct {
 	SMU_ARRAY(SMU_MATTR(PCIE_BANDWIDTH_ACC), SMU_MUNIT(NONE),              \
 		  SMU_MTYPE(U64), pcie_bandwidth_acc, SMU_15_0_8_MAX_MID);     \
 	SMU_ARRAY(SMU_MATTR(PCIE_BANDWIDTH_INST), SMU_MUNIT(BW_1),             \
-		  SMU_MTYPE(U32), pcie_bandwidth_inst, SMU_15_0_8_MAX_MID);    \
+		  SMU_MTYPE(U64), pcie_bandwidth_inst, SMU_15_0_8_MAX_MID);    \
 	SMU_SCALAR(SMU_MATTR(PCIE_L0_TO_RECOV_COUNT_ACC), SMU_MUNIT(NONE),     \
 		   SMU_MTYPE(U64), pcie_l0_to_recov_count_acc);                \
 	SMU_SCALAR(SMU_MATTR(PCIE_REPLAY_COUNT_ACC), SMU_MUNIT(NONE),          \
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
index d365f06ac1ac1..2bd3ea17e7897 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
@@ -1544,3 +1544,143 @@ int smu_cmn_dpm_pcie_width_idx(int width)
 
 	return ret;
 }
+
+static int smu_cmn_get_pptable_v2_0(struct smu_context *smu, void **table, uint32_t *size)
+{
+	const struct smc_firmware_header_v2_0 *v2;
+	struct amdgpu_device *adev = smu->adev;
+	size_t fw_size = adev->pm.fw->size;
+	uint32_t ppt_offset_bytes;
+	uint32_t ppt_size_bytes;
+
+	if (fw_size < sizeof(*v2)) {
+		dev_err(adev->dev,
+			"SMC firmware too small for v2.0 header: %zu < %zu\n",
+			fw_size, sizeof(*v2));
+		return -EINVAL;
+	}
+
+	v2 = (const struct smc_firmware_header_v2_0 *)adev->pm.fw->data;
+
+	ppt_offset_bytes = le32_to_cpu(v2->ppt_offset_bytes);
+	ppt_size_bytes   = le32_to_cpu(v2->ppt_size_bytes);
+
+	if (ppt_offset_bytes > fw_size ||
+	    ppt_size_bytes > fw_size - ppt_offset_bytes) {
+		dev_err(adev->dev,
+			"pptable v2.0 exceeds firmware binary: offset %u + size %u > %zu\n",
+			ppt_offset_bytes, ppt_size_bytes, fw_size);
+		return -EINVAL;
+	}
+
+	*size  = ppt_size_bytes;
+	*table = (uint8_t *)v2 + ppt_offset_bytes;
+
+	return 0;
+}
+
+static int smu_cmn_get_pptable_v2_1(struct smu_context *smu, void **table,
+				     uint32_t *size, uint32_t pptable_id)
+{
+	const struct smc_firmware_header_v2_1 *v2_1;
+	struct amdgpu_device *adev = smu->adev;
+	struct smc_soft_pptable_entry *entries;
+	size_t fw_size = adev->pm.fw->size;
+	uint32_t pptable_entry_offset;
+	uint32_t ppt_offset_bytes;
+	uint32_t ppt_size_bytes;
+	uint32_t pptable_count;
+	int i;
+
+	if (fw_size < sizeof(*v2_1)) {
+		dev_err(adev->dev,
+			"SMC firmware too small for v2.1 header: %zu < %zu\n",
+			fw_size, sizeof(*v2_1));
+		return -EINVAL;
+	}
+
+	v2_1 = (const struct smc_firmware_header_v2_1 *)adev->pm.fw->data;
+
+	pptable_entry_offset = le32_to_cpu(v2_1->pptable_entry_offset);
+	pptable_count        = le32_to_cpu(v2_1->pptable_count);
+
+	if (pptable_entry_offset > fw_size ||
+	    pptable_count > (fw_size - pptable_entry_offset) / sizeof(*entries)) {
+		dev_err(adev->dev,
+			"pptable v2.1 entry array exceeds firmware binary: offset %u, count %u\n",
+			pptable_entry_offset, pptable_count);
+		return -EINVAL;
+	}
+
+	entries = (struct smc_soft_pptable_entry *)
+		((uint8_t *)v2_1 + pptable_entry_offset);
+
+	for (i = 0; i < pptable_count; i++) {
+		if (le32_to_cpu(entries[i].id) != pptable_id)
+			continue;
+
+		ppt_offset_bytes = le32_to_cpu(entries[i].ppt_offset_bytes);
+		ppt_size_bytes   = le32_to_cpu(entries[i].ppt_size_bytes);
+
+		if (ppt_offset_bytes > fw_size ||
+		    ppt_size_bytes > fw_size - ppt_offset_bytes) {
+			dev_err(adev->dev,
+				"pptable entry %d exceeds firmware binary: offset %u + size %u > %zu\n",
+				i, ppt_offset_bytes, ppt_size_bytes, fw_size);
+			return -EINVAL;
+		}
+
+		*table = (uint8_t *)v2_1 + ppt_offset_bytes;
+		*size  = ppt_size_bytes;
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+/**
+ * smu_cmn_get_pptable_from_firmware - locate the soft pptable embedded in the
+ *                                     SMC firmware binary.
+ * @smu:        SMU context
+ * @table:      on success, set to the start of the pptable within the firmware
+ *              blob
+ * @size:       on success, set to the pptable size in bytes
+ * @pptable_id: the entry ID to search for (used only for v2.1 binaries)
+ *
+ * Reads the firmware header version and dispatches to the appropriate v2.x
+ * parser.  Only major version 2 is supported; minor version selects between
+ * the single-entry (v2.0) and multi-entry directory (v2.1) layouts.
+ *
+ * Return: 0 on success, -EINVAL for an unsupported version or if the
+ *         requested pptable cannot be found or exceeds the binary bounds.
+ */
+int smu_cmn_get_pptable_from_firmware(struct smu_context *smu, void **table,
+				      uint32_t *size, uint32_t pptable_id)
+{
+	const struct smc_firmware_header_v1_0 *hdr;
+	struct amdgpu_device *adev = smu->adev;
+	uint16_t version_major, version_minor;
+
+	hdr = (const struct smc_firmware_header_v1_0 *)adev->pm.fw->data;
+	if (!hdr)
+		return -EINVAL;
+
+	dev_info(adev->dev, "use driver provided pptable %d\n", pptable_id);
+
+	version_major = le16_to_cpu(hdr->header.header_version_major);
+	version_minor = le16_to_cpu(hdr->header.header_version_minor);
+	if (version_major != 2) {
+		dev_err(adev->dev, "Unsupported smu firmware version %d.%d\n",
+			version_major, version_minor);
+		return -EINVAL;
+	}
+
+	switch (version_minor) {
+	case 0:
+		return smu_cmn_get_pptable_v2_0(smu, table, size);
+	case 1:
+		return smu_cmn_get_pptable_v2_1(smu, table, size, pptable_id);
+	default:
+		return -EINVAL;
+	}
+}
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h
index 0e119965ce131..ae6742f5298f5 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h
@@ -113,6 +113,16 @@ static inline int pcie_gen_to_speed(uint32_t gen)
 	return ((gen == 0) ? link_speed[0] : link_speed[gen - 1]);
 }
 
+static inline bool smu_cmn_custom_params_count_valid(u32 max_idx, u32 params_count)
+{
+	return max_idx == params_count;
+}
+
+static inline bool smu_cmn_custom_params_clock_valid(long clock_idx, long clock_count)
+{
+	return clock_idx >= 0 && clock_idx < clock_count;
+}
+
 int smu_cmn_send_smc_msg_with_param(struct smu_context *smu,
 				    enum smu_message_type msg,
 				    uint32_t param,
@@ -239,6 +249,9 @@ int smu_cmn_dpm_pcie_gen_idx(int gen);
 int smu_cmn_dpm_pcie_width_idx(int width);
 int smu_cmn_check_fw_version(struct smu_context *smu);
 
+int smu_cmn_get_pptable_from_firmware(struct smu_context *smu, void **table,
+				      uint32_t *size, uint32_t pptable_id);
+
 /*SMU gpu metrics */
 
 /* Attribute ID mapping */
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_internal.h b/drivers/gpu/drm/amd/pm/swsmu/smu_internal.h
index 24848da902343..45e3758e9ed8b 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu_internal.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu_internal.h
@@ -98,7 +98,6 @@
 #define smu_is_asic_wbrf_supported(smu)			smu_ppt_funcs(is_asic_wbrf_supported, false, smu)
 #define smu_enable_uclk_shadow(smu, enable)		smu_ppt_funcs(enable_uclk_shadow, 0, smu, enable)
 #define smu_set_wbrf_exclusion_ranges(smu, freq_band_range)		smu_ppt_funcs(set_wbrf_exclusion_ranges, -EOPNOTSUPP, smu, freq_band_range)
-#define smu_get_ras_smu_drv(smu, ras_smu_drv)			smu_ppt_funcs(get_ras_smu_drv, -EOPNOTSUPP, smu, ras_smu_drv)
 
 #endif
 #endif
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c
index 658bf3fdb66b1..c2285fde8b3cc 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c
@@ -30,9 +30,6 @@
 #include "amdgpu_ras_mgr.h"
 #include "amdgpu_virt_ras_cmd.h"
 
-/* inject address is 52 bits */
-#define	RAS_UMC_INJECT_ADDR_LIMIT	(0x1ULL << 52)
-
 #define AMDGPU_RAS_TYPE_RASCORE  0x1
 #define AMDGPU_RAS_TYPE_AMDGPU   0x2
 #define AMDGPU_RAS_TYPE_VF       0x3
@@ -85,6 +82,43 @@ static uint64_t local_addr_to_xgmi_global_addr(struct ras_core_context *ras_core
 	return (addr + xgmi->physical_node_id * xgmi->node_segment_size);
 }
 
+/*
+ * UMC error injection is dispatched by the RAS TA using the injection method
+ * carried in struct ras_cmd_inject_error_req. Only the "coherent" methods
+ * program an explicit injection address and are therefore address-based; the
+ * single-shot, persistent and ac-parity methods ignore the address.
+ *
+ * Keep these values in sync with the RAS TA.
+ */
+enum umc_inject_method {
+	UMC_METHOD_COHERENT		= 0,
+	UMC_METHOD_SINGLE_SHOT		= 1,
+	UMC_METHOD_PERSISTENT		= 2,
+	UMC_METHOD_PERSISTENT_DISABLE	= 3,
+	UMC_METHOD_COHERENT_NO_DETECTION	= 4,
+	UMC_METHOD_COHERENT_WR		= 5,
+	UMC_METHOD_SINGLE_SHOT_WR		= 6,
+	UMC_METHOD_PERSISTENT_WR		= 7,
+	UMC_METHOD_SINGLE_SHOT_CLEAN	= 8,
+};
+
+/*
+ * Return true when @method does not program an explicit injection address.
+ * Only the coherent methods are address-based; every other method ignores the
+ * address, so userspace signals them by setting the address to U64_MAX.
+ */
+static bool amdgpu_ras_mgr_is_non_address_injection(u64 method)
+{
+	switch (method) {
+	case UMC_METHOD_COHERENT:
+	case UMC_METHOD_COHERENT_NO_DETECTION:
+	case UMC_METHOD_COHERENT_WR:
+		return false;
+	default:
+		return true;
+	}
+}
+
 static int amdgpu_ras_inject_error(struct ras_core_context *ras_core,
 			struct ras_cmd_ctx *cmd, void *data)
 {
@@ -94,25 +128,35 @@ static int amdgpu_ras_inject_error(struct ras_core_context *ras_core,
 	int ret = RAS_CMD__ERROR_GENERIC;
 
 	if (req->block_id == RAS_BLOCK_ID__UMC) {
-		if (amdgpu_ras_mgr_check_retired_addr(adev, req->address)) {
-			RAS_DEV_WARN(ras_core->dev,
-				"RAS WARN: inject: 0x%llx has already been marked as bad!\n",
-				req->address);
-			return RAS_CMD__ERROR_ACCESS_DENIED;
-		}
-
-		if ((req->address >= adev->gmc.mc_vram_size &&
-			adev->gmc.mc_vram_size) ||
-			(req->address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
-			RAS_DEV_WARN(adev, "RAS WARN: input address 0x%llx is invalid.",
+		/*
+		 * Only address-based UMC injections carry an explicit injection
+		 * address that has to be validated. A non address-based method
+		 * ignores the address, and userspace flags such an injection by
+		 * setting the address to U64_MAX. When both the sentinel and the
+		 * method agree, clear the address so the RAS TA ignores it and
+		 * skip the validation; otherwise validate the injection address.
+		 */
+		if (req->address == U64_MAX && amdgpu_ras_mgr_is_non_address_injection(req->method)) {
+			req->address = 0x0;
+		} else {
+			if (amdgpu_ras_mgr_check_retired_addr(adev, req->address)) {
+				RAS_DEV_WARN(ras_core->dev,
+					"RAS WARN: inject: 0x%llx has already been marked as bad!\n",
 					req->address);
-			return RAS_CMD__ERROR_INVALID_INPUT_DATA;
-		}
-
-		/* Calculate XGMI relative offset */
-		if (adev->gmc.xgmi.num_physical_nodes > 1 &&
-			req->block_id != RAS_BLOCK_ID__GFX) {
-			req->address = local_addr_to_xgmi_global_addr(ras_core, req->address);
+				return RAS_CMD__ERROR_ACCESS_DENIED;
+			}
+
+			if ((req->address >= adev->gmc.mc_vram_size &&
+				adev->gmc.mc_vram_size) ||
+				(req->address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
+				RAS_DEV_WARN(adev, "RAS WARN: input address 0x%llx is invalid.",
+						req->address);
+				return RAS_CMD__ERROR_INVALID_INPUT_DATA;
+			}
+
+			/* Calculate XGMI relative offset */
+			if (adev->gmc.xgmi.num_physical_nodes > 1)
+				req->address = local_addr_to_xgmi_global_addr(ras_core, req->address);
 		}
 	}
 
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_eeprom_i2c.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_eeprom_i2c.c
index 3ed3ff42b7e14..9c6d0024210d7 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_eeprom_i2c.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_eeprom_i2c.c
@@ -67,7 +67,7 @@ static int ras_eeprom_i2c_config(struct ras_core_context *ras_core)
 	struct ras_eeprom_control *control = &ras_core->ras_eeprom;
 	u8 i2c_addr;
 
-	if (amdgpu_atomfirmware_ras_rom_addr(adev, &i2c_addr)) {
+	if (adev->bios && amdgpu_atomfirmware_ras_rom_addr(adev, &i2c_addr)) {
 		/* The address given by VBIOS is an 8-bit, wire-format
 		 * address, i.e. the most significant byte.
 		 *
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
index a22d1aebbeb94..3b5da8515a20b 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
@@ -95,11 +95,35 @@ static int amdgpu_ras_mgr_init_aca_config(struct amdgpu_device *adev,
 	return 0;
 }
 
+static uint64_t amdgpu_ras_mgr_reserved_vram_size(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	uint64_t reserved_pages_in_bytes = 0;
+
+	if (!con || (adev->flags & AMD_IS_APU))
+		return 0;
+
+	switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
+	case IP_VERSION(13, 0, 6):
+	case IP_VERSION(13, 0, 12):
+		reserved_pages_in_bytes = RAS_RESERVED_VRAM_SIZE_DEFAULT;
+		break;
+	case IP_VERSION(13, 0, 14):
+		reserved_pages_in_bytes = (RAS_RESERVED_VRAM_SIZE_DEFAULT << 1);
+		break;
+	default:
+		break;
+	}
+	return reserved_pages_in_bytes;
+}
+
 static int amdgpu_ras_mgr_init_eeprom_config(struct amdgpu_device *adev,
 		struct ras_core_config *config)
 {
 	struct ras_eeprom_config *eeprom_cfg = &config->eeprom_cfg;
+	uint64_t ras_reserved_vram_size;
 
+	ras_reserved_vram_size = amdgpu_ras_mgr_reserved_vram_size(adev);
 	eeprom_cfg->eeprom_sys_fn = &amdgpu_ras_eeprom_i2c_sys_func;
 	eeprom_cfg->eeprom_i2c_adapter = adev->pm.ras_eeprom_i2c_bus;
 	if (eeprom_cfg->eeprom_i2c_adapter) {
@@ -133,7 +157,7 @@ static int amdgpu_ras_mgr_init_eeprom_config(struct amdgpu_device *adev,
 			div64_u64(adev->gmc.mc_vram_size, TYPICAL_ECC_BAD_PAGE_RATE);
 	else if (amdgpu_bad_page_threshold == WARN_NONSTOP_OVER_THRESHOLD)
 		eeprom_cfg->eeprom_record_threshold_count =
-				COUNT_BAD_PAGE_THRESHOLD(RAS_RESERVED_VRAM_SIZE_DEFAULT);
+				COUNT_BAD_PAGE_THRESHOLD(ras_reserved_vram_size);
 	else
 		eeprom_cfg->eeprom_record_threshold_count = amdgpu_bad_page_threshold;
 
@@ -142,6 +166,21 @@ static int amdgpu_ras_mgr_init_eeprom_config(struct amdgpu_device *adev,
 	return 0;
 }
 
+static bool amdgpu_ras_mgr_eeprom_is_supported(struct amdgpu_device *adev)
+{
+	if (amdgpu_sriov_vf(adev))
+		return false;
+
+	switch (amdgpu_ip_version(adev, MP1_HWIP, 0)) {
+	case IP_VERSION(13, 0, 6):
+	case IP_VERSION(13, 0, 12):
+	case IP_VERSION(13, 0, 14):
+		return (adev->gmc.is_app_apu) ? false : true;
+	default:
+		return false;
+	}
+}
+
 static int amdgpu_ras_mgr_init_mp1_config(struct amdgpu_device *adev,
 		struct ras_core_config *config)
 {
@@ -266,7 +305,8 @@ static struct ras_core_context *amdgpu_ras_mgr_create_ras_core(struct amdgpu_dev
 		init_config.aca_ip_version = IP_VERSION(1, 0, 0);
 
 	init_config.sys_fn = &amdgpu_ras_sys_fn;
-	init_config.ras_eeprom_supported = true;
+	init_config.ras_eeprom_supported =
+		amdgpu_ras_mgr_eeprom_is_supported(adev);
 	init_config.poison_supported =
 		amdgpu_ras_is_poison_mode_supported(adev);
 
@@ -292,8 +332,7 @@ static int amdgpu_ras_mgr_sw_init(struct amdgpu_ip_block *ip_block)
 
 	if (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14) ||
 	    amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) ||
-	    amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) ||
-	    adev->debug_enable_ras_aca)
+	    amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6))
 		con->uniras_enabled = true;
 	else
 		return 0;
@@ -425,6 +464,28 @@ static int amdgpu_ras_mgr_hw_fini(struct amdgpu_ip_block *ip_block)
 	return 0;
 }
 
+int amdgpu_ras_mgr_resume_after_reset(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	struct amdgpu_ip_block *ip_block;
+
+	if (!con || !con->uniras_enabled)
+		return 0;
+
+	if (!ras_mgr || !ras_mgr->ras_core)
+		return -EINVAL;
+
+	if (ras_mgr->ras_is_ready)
+		return 0;
+
+	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_RAS);
+	if (!ip_block)
+		return -EINVAL;
+
+	return amdgpu_ras_mgr_hw_init(ip_block);
+}
+
 struct amdgpu_ras_mgr *amdgpu_ras_mgr_get_context(struct amdgpu_device *adev)
 {
 	if (!adev || !adev->psp.ras_context.ras)
@@ -733,3 +794,13 @@ int amdgpu_ras_mgr_lookup_bad_pages_in_a_row(struct amdgpu_device *adev,
 	return ras_core_convert_soc_pa_to_cur_nps_pages(ras_mgr->ras_core,
 			addr, nps_page_addr, max_page_count);
 }
+
+int amdgpu_ras_mgr_set_debug_mode(struct amdgpu_device *adev, bool enable)
+{
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+
+	if (!ras_mgr || !ras_mgr->ras_core || !ras_mgr->ras_is_ready)
+		return false;
+
+	return ras_core_set_debug_mode(ras_mgr->ras_core, enable);
+}
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h
index 4f44a917d48be..a20bb8fdce871 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h
@@ -82,6 +82,8 @@ int amdgpu_ras_mgr_handle_ras_cmd(struct amdgpu_device *adev,
 		void *output, uint32_t out_size);
 int amdgpu_ras_mgr_pre_reset(struct amdgpu_device *adev);
 int amdgpu_ras_mgr_post_reset(struct amdgpu_device *adev);
+int amdgpu_ras_mgr_resume_after_reset(struct amdgpu_device *adev);
 int amdgpu_ras_mgr_lookup_bad_pages_in_a_row(struct amdgpu_device *adev,
 		uint64_t addr, uint64_t *nps_page_addr, uint32_t max_page_count);
+int amdgpu_ras_mgr_set_debug_mode(struct amdgpu_device *adev, bool enable);
 #endif
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.c
index 2098f24d4940c..3c4575a5d902d 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.c
@@ -24,6 +24,7 @@
 #include "amdgpu_smu.h"
 #include "amdgpu_reset.h"
 #include "amdgpu_ras_mp1_v13_0.h"
+#include "smu13_driver_if_v13_0_6.h"
 
 #define RAS_MP1_MSG_QueryValidMcaCeCount  0x3A
 #define RAS_MP1_MSG_McaBankCeDumpDW       0x3B
@@ -131,10 +132,23 @@ static int mp1_v13_0_get_ras_enabled_mask(struct ras_core_context *ras_core,
 	return ret;
 }
 
+static int mp1_v13_0_set_debug_mode(struct ras_core_context *ras_core, bool enable)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
+	int ret;
+	u32 smu_msg = SMU_MSG_ClearMcaOnRead;
+
+	ret = amdgpu_smu_ras_send_msg(adev, smu_msg,
+			enable ? 0 : ClearMcaOnRead_UE_FLAG_MASK |
+			ClearMcaOnRead_CE_POLL_MASK, NULL);
+	return ret;
+}
+
 const struct ras_mp1_sys_func amdgpu_ras_mp1_sys_func_v13_0 = {
 	.mp1_get_valid_bank_count = mp1_v13_0_get_valid_bank_count,
 	.mp1_dump_valid_bank = mp1_v13_0_dump_valid_bank,
 	.mp1_send_eeprom_msg = mp1_v13_0_eeprom_send_msg,
 	.mp1_get_ras_enabled_mask = mp1_v13_0_get_ras_enabled_mask,
+	.mp1_set_debug_mode = mp1_v13_0_set_debug_mode,
 };
 
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_sys.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_sys.c
index 7d728e5236042..e4444798bc732 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_sys.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_sys.c
@@ -267,6 +267,24 @@ static int amdgpu_ras_sys_put_gpu_mem(struct ras_core_context *ras_core,
 	return 0;
 }
 
+static int amdgpu_ras_sys_check_address_sanity(struct ras_core_context *ras_core,
+						uint64_t addr)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
+
+	if ((addr >= adev->gmc.mc_vram_size &&
+	    adev->gmc.mc_vram_size) ||
+	    (addr >= RAS_UMC_INJECT_ADDR_LIMIT))
+		return -EINVAL;
+
+	if (addr >= adev->gmc.real_vram_size) {
+		RAS_DEV_WARN(ras_core->dev, "Recorded address out of range: 0x%llx!\n", addr);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 const struct ras_sys_func amdgpu_ras_sys_fn = {
 	.ras_notifier = amdgpu_ras_sys_event_notifier,
 	.get_utc_second_timestamp = amdgpu_ras_sys_get_utc_second_timestamp,
@@ -277,4 +295,5 @@ const struct ras_sys_func amdgpu_ras_sys_fn = {
 	.detect_ras_interrupt = amdgpu_ras_sys_detect_ras_interrupt,
 	.get_gpu_mem = amdgpu_ras_sys_get_gpu_mem,
 	.put_gpu_mem = amdgpu_ras_sys_put_gpu_mem,
+	.check_address_sanity = amdgpu_ras_sys_check_address_sanity,
 };
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/ras_sys.h b/drivers/gpu/drm/amd/ras/ras_mgr/ras_sys.h
index f34dda7ce87b1..2775c7bf41b7e 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/ras_sys.h
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/ras_sys.h
@@ -30,6 +30,9 @@
 #include <linux/mempool.h>
 #include "amdgpu.h"
 
+/* inject address is 52 bits */
+#define RAS_UMC_INJECT_ADDR_LIMIT       (0x1ULL << 52)
+
 #define RAS_DEV_ERR(device, fmt, ...)                                               \
 	do {                                                                      \
 		if (device)                                                             \
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras.h b/drivers/gpu/drm/amd/ras/rascore/ras.h
index c059fcebaf005..878dfdfcb18ac 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras.h
+++ b/drivers/gpu/drm/amd/ras/rascore/ras.h
@@ -167,6 +167,7 @@ struct ras_mp1_sys_func {
 			enum ras_fw_eeprom_cmd index, uint32_t param, uint32_t *read_arg);
 	int (*mp1_get_ras_enabled_mask)(struct ras_core_context *ras_core,
 			uint64_t *enabled_mask);
+	int (*mp1_set_debug_mode)(struct ras_core_context *ras_core, bool enable);
 };
 
 struct ras_eeprom_sys_func {
@@ -231,6 +232,7 @@ struct ras_sys_func {
 		enum gpu_mem_type mem_type, struct gpu_mem_block *gpu_mem);
 	int (*put_gpu_mem)(struct ras_core_context *ras_core,
 		enum gpu_mem_type mem_type, struct gpu_mem_block *gpu_mem);
+	int (*check_address_sanity)(struct ras_core_context *ras_core, uint64_t addr);
 };
 
 struct ras_ecc_count {
@@ -398,4 +400,7 @@ int ras_core_get_device_system_info(struct ras_core_context *ras_core,
 		struct device_system_info *dev_info);
 int ras_core_convert_soc_pa_to_cur_nps_pages(struct ras_core_context *ras_core,
 		uint64_t soc_pa, uint64_t *page_pfn, uint32_t max_pages);
+int ras_core_check_address_sanity(struct ras_core_context *ras_core, uint64_t addr);
+
+int ras_core_set_debug_mode(struct ras_core_context *ras_core, bool enable);
 #endif
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_aca_v1_0.c b/drivers/gpu/drm/amd/ras/rascore/ras_aca_v1_0.c
index 210fbd8851a6d..840610538c1f3 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_aca_v1_0.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_aca_v1_0.c
@@ -213,7 +213,7 @@ static int aca_parse_umc_bank(struct ras_core_context *ras_core,
 	struct aca_bank_reg *bank = (struct aca_bank_reg *)data;
 	struct aca_bank_ecc *ecc = (struct aca_bank_ecc *)buf;
 	struct aca_ecc_info bank_info;
-	uint32_t ext_error_code;
+	uint32_t ext_error_code, misc0_errcnt;
 	uint64_t status0;
 
 	status0 = bank->regs[ACA_REG_IDX__STATUS];
@@ -228,15 +228,14 @@ static int aca_parse_umc_bank(struct ras_core_context *ras_core,
 	ecc->bank_info.addr = bank->regs[ACA_REG_IDX__ADDR];
 
 	ext_error_code = ACA_REG_STATUS_ERRORCODEEXT(status0);
+	misc0_errcnt = ACA_REG_MISC0_ERRCNT(bank->regs[ACA_REG_IDX__MISC0]);
 
 	if (aca_check_umc_de(ras_core, status0))
-		ecc->de_count = 1;
+		ecc->de_count = misc0_errcnt ? misc0_errcnt : 1;
 	else if (aca_check_umc_ue(ras_core, status0))
-		ecc->ue_count = ext_error_code ?
-			1 : ACA_REG_MISC0_ERRCNT(bank->regs[ACA_REG_IDX__MISC0]);
+		ecc->ue_count = ext_error_code ? 1 : misc0_errcnt;
 	else if (aca_check_umc_ce(ras_core, status0))
-		ecc->ce_count = ext_error_code ?
-			1 : ACA_REG_MISC0_ERRCNT(bank->regs[ACA_REG_IDX__MISC0]);
+		ecc->ce_count = ext_error_code ? 1 : misc0_errcnt;
 
 	return 0;
 }
@@ -266,7 +265,7 @@ static int aca_parse_bank_default(struct ras_core_context *ras_core,
 	ecc->bank_info.addr = bank->regs[ACA_REG_IDX__ADDR];
 
 	if (aca_check_bank_is_de(ras_core, status)) {
-		ecc->de_count = 1;
+		ecc->de_count = 0;
 	} else {
 		if (bank->ecc_type == RAS_ERR_TYPE__UE)
 			ecc->ue_count = 1;
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_core.c b/drivers/gpu/drm/amd/ras/rascore/ras_core.c
index 62d124a3eeac2..c63a358b7e573 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_core.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_core.c
@@ -151,6 +151,11 @@ bool ras_core_gpu_is_rma(struct ras_core_context *ras_core)
 	return ras_core->is_rma;
 }
 
+int ras_core_set_debug_mode(struct ras_core_context *ras_core, bool enable)
+{
+	return ras_mp1_set_debug_mode(ras_core, enable);
+}
+
 static int ras_core_seqno_fifo_write(struct ras_core_context *ras_core,
 		enum ras_seqno_fifo fifo_type, uint64_t seqno)
 {
@@ -676,3 +681,13 @@ int ras_core_convert_soc_pa_to_cur_nps_pages(struct ras_core_context *ras_core,
 
 	return count;
 }
+
+int ras_core_check_address_sanity(struct ras_core_context *ras_core,
+		uint64_t addr)
+{
+	if (ras_core && ras_core->sys_fn &&
+		ras_core->sys_fn->check_address_sanity)
+		return ras_core->sys_fn->check_address_sanity(ras_core, addr);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c
index 3a0ea036c9be9..62d1a319c08cd 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c
@@ -746,6 +746,9 @@ static int ras_eeprom_update_header(struct ras_eeprom_control *control)
 	int res;
 
 	bad_page_count = ras_umc_get_badpage_count(ras_core);
+	ras_core_event_notify(ras_core, RAS_EVENT_ID__UPDATE_BAD_PAGE_NUM,
+			      &bad_page_count);
+
 	/* Modify the header if it exceeds.
 	 */
 	if (threshold_config != 0 &&
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c
index f5fa80db91fb1..59e195652e427 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c
@@ -72,7 +72,7 @@ int ras_fw_get_badpage_count(struct ras_core_context *ras_core,
 		if (ret != -EBUSY)
 			return ret;
 
-		mdelay(10);
+		usleep_range(10000, 15000);
 		now = (uint64_t)ktime_to_ms(ktime_get());
 	} while (now < end);
 
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_mp1.c b/drivers/gpu/drm/amd/ras/rascore/ras_mp1.c
index f3321df850212..26af09f3574a8 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_mp1.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_mp1.c
@@ -59,9 +59,20 @@ int ras_mp1_dump_bank(struct ras_core_context *ras_core,
 	return mp1->ip_func->dump_valid_bank(ras_core, type, idx, reg_idx, val);
 }
 
+int ras_mp1_set_debug_mode(struct ras_core_context *ras_core, bool enable)
+{
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+
+	if (!mp1->ip_func || !mp1->ip_func->set_debug_mode)
+		return -EOPNOTSUPP;
+
+	return mp1->ip_func->set_debug_mode(ras_core, enable);
+}
+
 int ras_mp1_hw_init(struct ras_core_context *ras_core)
 {
 	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	int ret = 0;
 
 	mp1->mp1_ip_version = ras_core->config->mp1_ip_version;
 	mp1->sys_func = ras_core->config->mp1_cfg.mp1_sys_fn;
@@ -71,8 +82,14 @@ int ras_mp1_hw_init(struct ras_core_context *ras_core)
 	}
 
 	mp1->ip_func = ras_mp1_get_ip_funcs(ras_core, mp1->mp1_ip_version);
+	if (!mp1->ip_func)
+		return -EINVAL;
+
+	ret = ras_mp1_set_debug_mode(ras_core, false);
+	if (ret)
+		return -EINVAL;
 
-	return mp1->ip_func ? RAS_CORE_OK : -EINVAL;
+	return ret;
 }
 
 int ras_mp1_hw_fini(struct ras_core_context *ras_core)
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_mp1.h b/drivers/gpu/drm/amd/ras/rascore/ras_mp1.h
index de1d08286f418..5bc7c1b7fdab6 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_mp1.h
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_mp1.h
@@ -31,6 +31,7 @@ struct ras_mp1_ip_func {
 			enum ras_err_type type, u32 *count);
 	int (*dump_valid_bank)(struct ras_core_context *ras_core,
 		enum ras_err_type type, u32 idx, u32 reg_idx, u64 *val);
+	int (*set_debug_mode)(struct ras_core_context *ras_core, bool enable);
 };
 
 struct ras_mp1 {
@@ -47,4 +48,6 @@ int ras_mp1_get_bank_count(struct ras_core_context *ras_core,
 
 int ras_mp1_dump_bank(struct ras_core_context *ras_core,
 		u32 ecc_type, u32 idx, u32 reg_idx, u64 *val);
+
+int ras_mp1_set_debug_mode(struct ras_core_context *ras_core, bool enable);
 #endif
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_mp1_v13_0.c b/drivers/gpu/drm/amd/ras/rascore/ras_mp1_v13_0.c
index 310d39fc816bf..1fcfc1995ad3b 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_mp1_v13_0.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_mp1_v13_0.c
@@ -99,7 +99,20 @@ static int mp1_v13_0_dump_bank(struct ras_core_context *ras_core,
 	return sys_func->mp1_dump_valid_bank(ras_core, msg, idx, reg_idx, val);
 }
 
+static int mp1_v13_0_set_debug_mode(struct ras_core_context *ras_core, bool enable)
+{
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
+
+	if (!sys_func || !sys_func->mp1_set_debug_mode)
+		return -RAS_CORE_NOT_SUPPORTED;
+
+	return sys_func->mp1_set_debug_mode(ras_core, enable);
+}
+
+
 const struct ras_mp1_ip_func mp1_ras_func_v13_0 = {
 	.get_valid_bank_count = mp1_v13_0_get_bank_count,
 	.dump_valid_bank = mp1_v13_0_dump_bank,
+	.set_debug_mode = mp1_v13_0_set_debug_mode,
 };
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_process.c b/drivers/gpu/drm/amd/ras/rascore/ras_process.c
index 3267dcdb169cd..c001074c8c568 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_process.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_process.c
@@ -248,9 +248,10 @@ int ras_process_init(struct ras_core_context *ras_core)
 
 	ras_proc->ras_process_thread = kthread_run(ras_process_thread,
 							(void *)ras_core, "ras_process_thread");
-	if (!ras_proc->ras_process_thread) {
+	if (IS_ERR(ras_proc->ras_process_thread)) {
 		RAS_DEV_ERR(ras_core->dev, "Failed to create ras_process_thread.\n");
-		ret =  -ENOMEM;
+		ret = PTR_ERR(ras_proc->ras_process_thread);
+		ras_proc->ras_process_thread = NULL;
 		goto err;
 	}
 
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_umc.c b/drivers/gpu/drm/amd/ras/rascore/ras_umc.c
index f32ee2fecf536..e366fb97293e8 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_umc.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_umc.c
@@ -406,7 +406,7 @@ static int ras_umc_update_eeprom_ram_data(struct ras_core_context *ras_core,
 	struct ras_umc *ras_umc = &ras_core->ras_umc;
 	struct eeprom_store_record *data = &ras_umc->umc_err_data.ram_data;
 	uint64_t page_pfn[16];
-	int count = 0, j;
+	int count = 0, i, j;
 
 	if (!data->space_left &&
 		ras_umc_realloc_err_data_space(ras_core, data, 256)) {
@@ -418,10 +418,23 @@ static int ras_umc_update_eeprom_ram_data(struct ras_core_context *ras_core,
 					bps, bps->cur_nps, page_pfn, ARRAY_SIZE(page_pfn));
 	if (count > 0) {
 		for (j = 0; j < count; j++) {
+			if (ras_core_check_address_sanity(ras_core,
+				page_pfn[j] << AMDGPU_GPU_PAGE_SHIFT)) {
+
+				for (i = 0; i < data->count; i++)
+					if (page_pfn[j] == data->bps[i].cur_nps_retired_row_pfn)
+						break;
+				data->bps[data->count].cur_nps_retired_row_pfn = U64_MAX;
+				data->count++;
+				data->space_left--;
+				continue;
+			}
+
 			bps->cur_nps_retired_row_pfn = page_pfn[j];
 			memcpy(&data->bps[data->count], bps, sizeof(*data->bps));
 			data->count++;
 			data->space_left--;
+			data->bad_page_num++;
 		}
 	} else {
 		RAS_DEV_ERR(ras_core->dev, "Failed to convert record to nps pages!");
@@ -431,6 +444,14 @@ static int ras_umc_update_eeprom_ram_data(struct ras_core_context *ras_core,
 	return 0;
 }
 
+static void ras_umc_update_bad_pages(struct ras_core_context *ras_core)
+{
+	struct ras_umc *ras_umc = &ras_core->ras_umc;
+	struct eeprom_store_record *data = &ras_umc->umc_err_data.ram_data;
+
+	data->bad_page_num_old = data->bad_page_num;
+}
+
 /* it deal with vram only. */
 static int ras_umc_add_bad_pages(struct ras_core_context *ras_core,
 				 struct eeprom_umc_record *bps,
@@ -506,6 +527,7 @@ int ras_umc_load_bad_pages(struct ras_core_context *ras_core)
 	} else {
 		ras_core->ras_umc.umc_err_data.last_retired_pfn = UMC_INV_MEM_PFN;
 		ret = ras_umc_add_bad_pages(ras_core, bps, ras_num_recs, true);
+		ras_umc_update_bad_pages(ras_core);
 	}
 
 	kfree(bps);
@@ -521,7 +543,8 @@ static int ras_umc_save_bad_pages(struct ras_core_context *ras_core)
 {
 	struct ras_umc *ras_umc = &ras_core->ras_umc;
 	struct eeprom_store_record *data = &ras_umc->umc_err_data.rom_data;
-	uint32_t eeprom_record_num;
+	struct eeprom_store_record *ram_data = &ras_umc->umc_err_data.ram_data;
+	uint32_t eeprom_record_num, logical_count = 0;
 	int save_count;
 	int ret = 0;
 
@@ -534,6 +557,7 @@ static int ras_umc_save_bad_pages(struct ras_core_context *ras_core)
 		eeprom_record_num = ras_eeprom_get_record_count(ras_core);
 	mutex_lock(&ras_umc->umc_lock);
 	save_count = data->count - eeprom_record_num;
+	logical_count = ram_data->bad_page_num - ram_data->bad_page_num_old;
 	/* only new entries are saved */
 	if (save_count > 0) {
 		if (ras_fw_eeprom_supported(ras_core))
@@ -547,8 +571,8 @@ static int ras_umc_save_bad_pages(struct ras_core_context *ras_core)
 			ret = -EIO;
 			goto exit;
 		}
-
-		RAS_DEV_INFO(ras_core->dev, "Saved %d pages to EEPROM table.\n", save_count);
+		ras_umc_update_bad_pages(ras_core);
+		RAS_DEV_INFO(ras_core->dev, "Saved %d pages to EEPROM table.\n", logical_count);
 	}
 
 exit:
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_umc.h b/drivers/gpu/drm/amd/ras/rascore/ras_umc.h
index 237525b46b9bb..ee7100f25f51a 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_umc.h
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_umc.h
@@ -119,6 +119,12 @@ struct eeprom_store_record {
 	int count;
 	/* the space can place new entries */
 	int space_left;
+	/* logical bad page number */
+	int bad_page_num;
+	/* the bad page number is ras_num_recs or
+	 * ras_num_recs * retire_unit
+	 */
+	int bad_page_num_old;
 };
 
 struct ras_umc_err_data {
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_umc_v12_0.h b/drivers/gpu/drm/amd/ras/rascore/ras_umc_v12_0.h
index 8a35ad856165c..650b5f1f22f71 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_umc_v12_0.h
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_umc_v12_0.h
@@ -290,6 +290,8 @@
 /* R13 bit shift should be considered, double the number */
 #define UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL (UMC_V12_0_NA_MAP_PA_NUM * 2)
 
+/* UMC register per channel offset */
+#define UMC_V12_0_PER_CHANNEL_OFFSET    0x400
 
 /* C2, C3, C4, R13, four MCA bits are looped in page retirement */
 #define UMC_V12_0_RETIRE_LOOP_BITS 4
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
index 02f8f7e78a163..db99e252aeebc 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
@@ -1004,7 +1004,7 @@ static const struct drm_bridge_funcs adv7511_bridge_funcs = {
 	.atomic_disable = adv7511_bridge_atomic_disable,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 
 	.hdmi_tmds_char_rate_valid = adv7511_bridge_hdmi_tmds_char_rate_valid,
 	.hdmi_clear_audio_infoframe = adv7511_bridge_hdmi_clear_audio_infoframe,
diff --git a/drivers/gpu/drm/bridge/analogix/analogix-anx6345.c b/drivers/gpu/drm/bridge/analogix/analogix-anx6345.c
index f3fe47b12edca..eba5c6dcb5ada 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix-anx6345.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix-anx6345.c
@@ -566,7 +566,8 @@ anx6345_bridge_mode_valid(struct drm_bridge *bridge,
 	return MODE_OK;
 }
 
-static void anx6345_bridge_disable(struct drm_bridge *bridge)
+static void anx6345_bridge_disable(struct drm_bridge *bridge,
+				   struct drm_atomic_commit *commit)
 {
 	struct anx6345 *anx6345 = bridge_to_anx6345(bridge);
 
@@ -580,7 +581,8 @@ static void anx6345_bridge_disable(struct drm_bridge *bridge)
 		anx6345_poweroff(anx6345);
 }
 
-static void anx6345_bridge_enable(struct drm_bridge *bridge)
+static void anx6345_bridge_enable(struct drm_bridge *bridge,
+				  struct drm_atomic_commit *commit)
 {
 	struct anx6345 *anx6345 = bridge_to_anx6345(bridge);
 	int err;
@@ -600,11 +602,14 @@ static void anx6345_bridge_enable(struct drm_bridge *bridge)
 }
 
 static const struct drm_bridge_funcs anx6345_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach = anx6345_bridge_attach,
 	.detach = anx6345_bridge_detach,
 	.mode_valid = anx6345_bridge_mode_valid,
-	.disable = anx6345_bridge_disable,
-	.enable = anx6345_bridge_enable,
+	.atomic_disable = anx6345_bridge_disable,
+	.atomic_enable = anx6345_bridge_enable,
 };
 
 static void unregister_i2c_dummy_clients(struct anx6345 *anx6345)
diff --git a/drivers/gpu/drm/bridge/analogix/analogix-anx78xx.c b/drivers/gpu/drm/bridge/analogix/analogix-anx78xx.c
index ba0fc149a9e72..32186efa6f501 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix-anx78xx.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix-anx78xx.c
@@ -937,7 +937,8 @@ anx78xx_bridge_mode_valid(struct drm_bridge *bridge,
 	return MODE_OK;
 }
 
-static void anx78xx_bridge_disable(struct drm_bridge *bridge)
+static void anx78xx_bridge_disable(struct drm_bridge *bridge,
+				   struct drm_atomic_commit *commit)
 {
 	struct anx78xx *anx78xx = bridge_to_anx78xx(bridge);
 
@@ -975,7 +976,8 @@ static void anx78xx_bridge_mode_set(struct drm_bridge *bridge,
 	mutex_unlock(&anx78xx->lock);
 }
 
-static void anx78xx_bridge_enable(struct drm_bridge *bridge)
+static void anx78xx_bridge_enable(struct drm_bridge *bridge,
+				  struct drm_atomic_commit *commit)
 {
 	struct anx78xx *anx78xx = bridge_to_anx78xx(bridge);
 	int err;
@@ -992,12 +994,15 @@ static void anx78xx_bridge_enable(struct drm_bridge *bridge)
 }
 
 static const struct drm_bridge_funcs anx78xx_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach = anx78xx_bridge_attach,
 	.detach = anx78xx_bridge_detach,
 	.mode_valid = anx78xx_bridge_mode_valid,
-	.disable = anx78xx_bridge_disable,
+	.atomic_disable = anx78xx_bridge_disable,
 	.mode_set = anx78xx_bridge_mode_set,
-	.enable = anx78xx_bridge_enable,
+	.atomic_enable = anx78xx_bridge_enable,
 };
 
 static irqreturn_t anx78xx_hpd_threaded_handler(int irq, void *data)
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
index 5006ac181b2d7..30c0c0b41d5aa 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
@@ -330,7 +330,7 @@ static int analogix_dp_process_clock_recovery(struct analogix_dp_device *dp)
 	u8 voltage_swing, pre_emphasis, training_lane;
 	u8 link_status[DP_LINK_STATUS_SIZE];
 
-	usleep_range(100, 101);
+	drm_dp_link_train_clock_recovery_delay(&dp->aux, dp->dpcd);
 
 	lane_count = dp->link_train.lane_count;
 
@@ -393,7 +393,7 @@ static int analogix_dp_process_equalizer_training(struct analogix_dp_device *dp)
 	u32 reg;
 	u8 link_status[DP_LINK_STATUS_SIZE];
 
-	usleep_range(400, 401);
+	drm_dp_link_train_channel_eq_delay(&dp->aux, dp->dpcd);
 
 	lane_count = dp->link_train.lane_count;
 
@@ -623,7 +623,7 @@ static int analogix_dp_config_video(struct analogix_dp_device *dp)
 
 	for (;;) {
 		timeout_loop++;
-		if (analogix_dp_is_slave_video_stream_clock_on(dp) == 0)
+		if (analogix_dp_is_slave_video_stream_clock_on(dp))
 			break;
 		if (timeout_loop > DP_TIMEOUT_LOOP_COUNT) {
 			dev_err(dp->dev, "Timeout of slave video streamclk ok\n");
@@ -651,7 +651,7 @@ static int analogix_dp_config_video(struct analogix_dp_device *dp)
 
 	for (;;) {
 		timeout_loop++;
-		if (analogix_dp_is_video_stream_on(dp) == 0) {
+		if (analogix_dp_is_video_stream_on(dp)) {
 			done_count++;
 			if (done_count > 10)
 				break;
@@ -754,6 +754,12 @@ static int analogix_dp_commit(struct analogix_dp_device *dp)
 {
 	int ret;
 
+	ret = drm_dp_read_dpcd_caps(&dp->aux, dp->dpcd);
+	if (ret < 0) {
+		dev_err(dp->dev, "failed to read dpcd caps: %d\n", ret);
+		return ret;
+	}
+
 	ret = analogix_dp_train_link(dp);
 	if (ret) {
 		dev_err(dp->dev, "unable to do link train, ret=%d\n", ret);
@@ -874,7 +880,7 @@ static int analogix_dp_bridge_atomic_check(struct drm_bridge *bridge,
 	struct drm_display_info *di = &conn_state->connector->display_info;
 	u32 mask = BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444) | BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422);
 
-	if (is_rockchip(dp->plat_data->dev_type)) {
+	if (analogix_dp_is_rockchip(dp->plat_data->dev_type)) {
 		if ((di->color_formats & mask)) {
 			DRM_DEBUG_KMS("Swapping display color format from YUV to RGB\n");
 			di->color_formats &= ~mask;
@@ -1227,7 +1233,7 @@ static void analogix_dp_bridge_atomic_post_disable(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs analogix_dp_bridge_funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_pre_enable = analogix_dp_bridge_atomic_pre_enable,
 	.atomic_enable = analogix_dp_bridge_atomic_enable,
 	.atomic_disable = analogix_dp_bridge_atomic_disable,
@@ -1253,6 +1259,7 @@ static int analogix_dp_dt_parse_pdata(struct analogix_dp_device *dp)
 		video_info->max_link_rate = 0x0A;
 		video_info->max_lane_count = 0x04;
 		break;
+	case RK3576_EDP:
 	case RK3588_EDP:
 		video_info->max_link_rate = 0x14;
 		video_info->max_lane_count = 0x04;
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
index 17347448c6b0b..c7997677a2862 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
@@ -169,6 +169,8 @@ struct analogix_dp_device {
 	bool			fast_train_enable;
 	bool			psr_supported;
 
+	u8 dpcd[DP_RECEIVER_CAP_SIZE];
+
 	struct analogix_dp_plat_data *plat_data;
 };
 
@@ -211,7 +213,7 @@ void analogix_dp_reset_macro(struct analogix_dp_device *dp);
 void analogix_dp_init_video(struct analogix_dp_device *dp);
 
 void analogix_dp_set_video_color_format(struct analogix_dp_device *dp);
-int analogix_dp_is_slave_video_stream_clock_on(struct analogix_dp_device *dp);
+bool analogix_dp_is_slave_video_stream_clock_on(struct analogix_dp_device *dp);
 void analogix_dp_set_video_cr_mn(struct analogix_dp_device *dp,
 				 enum clock_recovery_m_value_type type,
 				 u32 m_value,
@@ -220,7 +222,7 @@ void analogix_dp_set_video_timing_mode(struct analogix_dp_device *dp, u32 type);
 void analogix_dp_enable_video_master(struct analogix_dp_device *dp,
 				     bool enable);
 void analogix_dp_start_video(struct analogix_dp_device *dp);
-int analogix_dp_is_video_stream_on(struct analogix_dp_device *dp);
+bool analogix_dp_is_video_stream_on(struct analogix_dp_device *dp);
 void analogix_dp_config_video_slave_mode(struct analogix_dp_device *dp);
 void analogix_dp_enable_scrambling(struct analogix_dp_device *dp);
 void analogix_dp_disable_scrambling(struct analogix_dp_device *dp);
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
index 38fd8d5014d29..ea8401293a23a 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
@@ -72,7 +72,7 @@ void analogix_dp_init_analog_param(struct analogix_dp_device *dp)
 	reg = SEL_24M | TX_DVDD_BIT_1_0625V;
 	writel(reg, dp->reg_base + ANALOGIX_DP_ANALOG_CTL_2);
 
-	if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) {
+	if (dp->plat_data && analogix_dp_is_rockchip(dp->plat_data->dev_type)) {
 		reg = REF_CLK_24M;
 		if (dp->plat_data->dev_type == RK3288_DP)
 			reg ^= REF_CLK_MASK;
@@ -123,7 +123,7 @@ void analogix_dp_reset(struct analogix_dp_device *dp)
 	analogix_dp_stop_video(dp);
 	analogix_dp_enable_video_mute(dp, 0);
 
-	if (dp->plat_data && is_rockchip(dp->plat_data->dev_type))
+	if (dp->plat_data && analogix_dp_is_rockchip(dp->plat_data->dev_type))
 		reg = RK_VID_CAP_FUNC_EN_N | RK_VID_FIFO_FUNC_EN_N |
 			SW_FUNC_EN_N;
 	else
@@ -233,7 +233,7 @@ void analogix_dp_set_pll_power_down(struct analogix_dp_device *dp, bool enable)
 	u32 mask = DP_PLL_PD;
 	u32 pd_addr = ANALOGIX_DP_PLL_CTL;
 
-	if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) {
+	if (dp->plat_data && analogix_dp_is_rockchip(dp->plat_data->dev_type)) {
 		pd_addr = ANALOGIX_DP_PD;
 		mask = RK_PLL_PD;
 	}
@@ -254,12 +254,12 @@ void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp,
 	u32 phy_pd_addr = ANALOGIX_DP_PHY_PD;
 	u32 mask;
 
-	if (dp->plat_data && is_rockchip(dp->plat_data->dev_type))
+	if (dp->plat_data && analogix_dp_is_rockchip(dp->plat_data->dev_type))
 		phy_pd_addr = ANALOGIX_DP_PD;
 
 	switch (block) {
 	case AUX_BLOCK:
-		if (dp->plat_data && is_rockchip(dp->plat_data->dev_type))
+		if (dp->plat_data && analogix_dp_is_rockchip(dp->plat_data->dev_type))
 			mask = RK_AUX_PD;
 		else
 			mask = AUX_PD;
@@ -317,7 +317,7 @@ void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp,
 		 * to power off everything instead of DP_PHY_PD in
 		 * Rockchip
 		 */
-		if (dp->plat_data && is_rockchip(dp->plat_data->dev_type))
+		if (dp->plat_data && analogix_dp_is_rockchip(dp->plat_data->dev_type))
 			mask = DP_INC_BG;
 		else
 			mask = DP_PHY_PD;
@@ -329,7 +329,7 @@ void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp,
 			reg &= ~mask;
 
 		writel(reg, dp->reg_base + phy_pd_addr);
-		if (dp->plat_data && is_rockchip(dp->plat_data->dev_type))
+		if (dp->plat_data && analogix_dp_is_rockchip(dp->plat_data->dev_type))
 			usleep_range(10, 15);
 		break;
 	case POWER_ALL:
@@ -465,7 +465,7 @@ void analogix_dp_init_aux(struct analogix_dp_device *dp)
 	analogix_dp_reset_aux(dp);
 
 	/* AUX_BIT_PERIOD_EXPECTED_DELAY doesn't apply to Rockchip IP */
-	if (dp->plat_data && is_rockchip(dp->plat_data->dev_type))
+	if (dp->plat_data && analogix_dp_is_rockchip(dp->plat_data->dev_type))
 		reg = 0;
 	else
 		reg = AUX_BIT_PERIOD_EXPECTED_DELAY(3);
@@ -713,7 +713,7 @@ void analogix_dp_set_video_color_format(struct analogix_dp_device *dp)
 	writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_3);
 }
 
-int analogix_dp_is_slave_video_stream_clock_on(struct analogix_dp_device *dp)
+bool analogix_dp_is_slave_video_stream_clock_on(struct analogix_dp_device *dp)
 {
 	u32 reg;
 
@@ -724,7 +724,7 @@ int analogix_dp_is_slave_video_stream_clock_on(struct analogix_dp_device *dp)
 
 	if (!(reg & DET_STA)) {
 		dev_dbg(dp->dev, "Input stream clock not detected.\n");
-		return -EINVAL;
+		return false;
 	}
 
 	reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_2);
@@ -735,10 +735,10 @@ int analogix_dp_is_slave_video_stream_clock_on(struct analogix_dp_device *dp)
 
 	if (reg & CHA_STA) {
 		dev_dbg(dp->dev, "Input stream clk is changing\n");
-		return -EINVAL;
+		return false;
 	}
 
-	return 0;
+	return true;
 }
 
 void analogix_dp_set_video_cr_mn(struct analogix_dp_device *dp,
@@ -816,7 +816,7 @@ void analogix_dp_start_video(struct analogix_dp_device *dp)
 	writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1);
 }
 
-int analogix_dp_is_video_stream_on(struct analogix_dp_device *dp)
+bool analogix_dp_is_video_stream_on(struct analogix_dp_device *dp)
 {
 	u32 reg;
 
@@ -826,10 +826,10 @@ int analogix_dp_is_video_stream_on(struct analogix_dp_device *dp)
 	reg = readl(dp->reg_base + ANALOGIX_DP_SYS_CTL_3);
 	if (!(reg & STRM_VALID)) {
 		dev_dbg(dp->dev, "Input video stream is not detected.\n");
-		return -EINVAL;
+		return false;
 	}
 
-	return 0;
+	return true;
 }
 
 void analogix_dp_config_video_slave_mode(struct analogix_dp_device *dp)
@@ -837,7 +837,7 @@ void analogix_dp_config_video_slave_mode(struct analogix_dp_device *dp)
 	u32 reg;
 
 	reg = readl(dp->reg_base + ANALOGIX_DP_FUNC_EN_1);
-	if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) {
+	if (dp->plat_data && analogix_dp_is_rockchip(dp->plat_data->dev_type)) {
 		reg &= ~(RK_VID_CAP_FUNC_EN_N | RK_VID_FIFO_FUNC_EN_N);
 	} else {
 		reg &= ~(MASTER_VID_FUNC_EN_N | SLAVE_VID_FUNC_EN_N);
diff --git a/drivers/gpu/drm/bridge/analogix/anx7625.c b/drivers/gpu/drm/bridge/analogix/anx7625.c
index fffcd6154c710..ebd34657908ea 100644
--- a/drivers/gpu/drm/bridge/analogix/anx7625.c
+++ b/drivers/gpu/drm/bridge/analogix/anx7625.c
@@ -2679,7 +2679,7 @@ static const struct drm_bridge_funcs anx7625_bridge_funcs = {
 	.atomic_disable = anx7625_bridge_atomic_disable,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.detect = anx7625_bridge_detect,
 	.edid_read = anx7625_bridge_edid_read,
 	.hpd_enable = anx7625_bridge_hpd_enable,
diff --git a/drivers/gpu/drm/bridge/aux-bridge.c b/drivers/gpu/drm/bridge/aux-bridge.c
index 1ed21a8713bfd..169939d233102 100644
--- a/drivers/gpu/drm/bridge/aux-bridge.c
+++ b/drivers/gpu/drm/bridge/aux-bridge.c
@@ -9,6 +9,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/bridge/aux-bridge.h>
 
@@ -104,6 +105,9 @@ static int drm_aux_bridge_attach(struct drm_bridge *bridge,
 }
 
 static const struct drm_bridge_funcs drm_aux_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach	= drm_aux_bridge_attach,
 };
 
diff --git a/drivers/gpu/drm/bridge/aux-hpd-bridge.c b/drivers/gpu/drm/bridge/aux-hpd-bridge.c
index f02a38a2638ad..a2e525aa57887 100644
--- a/drivers/gpu/drm/bridge/aux-hpd-bridge.c
+++ b/drivers/gpu/drm/bridge/aux-hpd-bridge.c
@@ -9,6 +9,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/bridge/aux-bridge.h>
 
@@ -165,6 +166,9 @@ static int drm_aux_hpd_bridge_attach(struct drm_bridge *bridge,
 }
 
 static const struct drm_bridge_funcs drm_aux_hpd_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach	= drm_aux_hpd_bridge_attach,
 };
 
diff --git a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c
index cf90d4468b5c1..c3e466ded84ab 100644
--- a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c
+++ b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c
@@ -1009,16 +1009,15 @@ cdns_dsi_bridge_atomic_destroy_state(struct drm_bridge *bridge,
 }
 
 static struct drm_bridge_state *
-cdns_dsi_bridge_atomic_reset(struct drm_bridge *bridge)
+cdns_dsi_bridge_atomic_create_state(struct drm_bridge *bridge)
 {
 	struct cdns_dsi_bridge_state *dsi_state;
 
 	dsi_state = kzalloc_obj(*dsi_state);
 	if (!dsi_state)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
-	memset(dsi_state, 0, sizeof(*dsi_state));
-	dsi_state->base.bridge = bridge;
+	__drm_atomic_helper_bridge_state_init(&dsi_state->base, bridge);
 
 	return &dsi_state->base;
 }
@@ -1029,7 +1028,7 @@ static const struct drm_bridge_funcs cdns_dsi_bridge_funcs = {
 	.atomic_pre_enable = cdns_dsi_bridge_atomic_pre_enable,
 	.atomic_post_disable = cdns_dsi_bridge_atomic_post_disable,
 	.atomic_check = cdns_dsi_bridge_atomic_check,
-	.atomic_reset = cdns_dsi_bridge_atomic_reset,
+	.atomic_create_state = cdns_dsi_bridge_atomic_create_state,
 	.atomic_duplicate_state = cdns_dsi_bridge_atomic_duplicate_state,
 	.atomic_destroy_state = cdns_dsi_bridge_atomic_destroy_state,
 	.atomic_get_input_bus_fmts = cdns_dsi_bridge_get_input_bus_fmts,
diff --git a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
index 36c07b71fe04b..504a3186ebb3a 100644
--- a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
+++ b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
@@ -1921,15 +1921,15 @@ cdns_mhdp_bridge_atomic_destroy_state(struct drm_bridge *bridge,
 }
 
 static struct drm_bridge_state *
-cdns_mhdp_bridge_atomic_reset(struct drm_bridge *bridge)
+cdns_mhdp_bridge_atomic_create_state(struct drm_bridge *bridge)
 {
 	struct cdns_mhdp_bridge_state *cdns_mhdp_state;
 
 	cdns_mhdp_state = kzalloc_obj(*cdns_mhdp_state);
 	if (!cdns_mhdp_state)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
-	__drm_atomic_helper_bridge_reset(bridge, &cdns_mhdp_state->base);
+	__drm_atomic_helper_bridge_state_init(&cdns_mhdp_state->base, bridge);
 
 	return &cdns_mhdp_state->base;
 }
@@ -2051,7 +2051,7 @@ static const struct drm_bridge_funcs cdns_mhdp_bridge_funcs = {
 	.detach = cdns_mhdp_detach,
 	.atomic_duplicate_state = cdns_mhdp_bridge_atomic_duplicate_state,
 	.atomic_destroy_state = cdns_mhdp_bridge_atomic_destroy_state,
-	.atomic_reset = cdns_mhdp_bridge_atomic_reset,
+	.atomic_create_state = cdns_mhdp_bridge_atomic_create_state,
 	.atomic_get_input_bus_fmts = cdns_mhdp_get_input_bus_fmts,
 	.detect = cdns_mhdp_bridge_detect,
 	.edid_read = cdns_mhdp_bridge_edid_read,
diff --git a/drivers/gpu/drm/bridge/chipone-icn6211.c b/drivers/gpu/drm/bridge/chipone-icn6211.c
index d8344e011818c..eb628a0ce6ecf 100644
--- a/drivers/gpu/drm/bridge/chipone-icn6211.c
+++ b/drivers/gpu/drm/bridge/chipone-icn6211.c
@@ -618,7 +618,7 @@ chipone_atomic_get_input_bus_fmts(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs chipone_bridge_funcs = {
 	.atomic_duplicate_state	= drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset		= drm_atomic_helper_bridge_reset,
+	.atomic_create_state		= drm_atomic_helper_bridge_create_state,
 	.atomic_pre_enable	= chipone_atomic_pre_enable,
 	.atomic_enable		= chipone_atomic_enable,
 	.atomic_post_disable	= chipone_atomic_post_disable,
diff --git a/drivers/gpu/drm/bridge/chrontel-ch7033.c b/drivers/gpu/drm/bridge/chrontel-ch7033.c
index a237c65ebd690..92d464727e414 100644
--- a/drivers/gpu/drm/bridge/chrontel-ch7033.c
+++ b/drivers/gpu/drm/bridge/chrontel-ch7033.c
@@ -330,7 +330,8 @@ static enum drm_mode_status ch7033_bridge_mode_valid(struct drm_bridge *bridge,
 	return MODE_OK;
 }
 
-static void ch7033_bridge_disable(struct drm_bridge *bridge)
+static void ch7033_bridge_disable(struct drm_bridge *bridge,
+				  struct drm_atomic_commit *commit)
 {
 	struct ch7033_priv *priv = bridge_to_ch7033_priv(bridge);
 
@@ -338,7 +339,8 @@ static void ch7033_bridge_disable(struct drm_bridge *bridge)
 	regmap_update_bits(priv->regmap, 0x52, RESETDB, 0x00);
 }
 
-static void ch7033_bridge_enable(struct drm_bridge *bridge)
+static void ch7033_bridge_enable(struct drm_bridge *bridge,
+				 struct drm_atomic_commit *commit)
 {
 	struct ch7033_priv *priv = bridge_to_ch7033_priv(bridge);
 
@@ -514,11 +516,14 @@ static void ch7033_bridge_mode_set(struct drm_bridge *bridge,
 }
 
 static const struct drm_bridge_funcs ch7033_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach = ch7033_bridge_attach,
 	.detach = ch7033_bridge_detach,
 	.mode_valid = ch7033_bridge_mode_valid,
-	.disable = ch7033_bridge_disable,
-	.enable = ch7033_bridge_enable,
+	.atomic_disable = ch7033_bridge_disable,
+	.atomic_enable = ch7033_bridge_enable,
 	.mode_set = ch7033_bridge_mode_set,
 };
 
diff --git a/drivers/gpu/drm/bridge/cros-ec-anx7688.c b/drivers/gpu/drm/bridge/cros-ec-anx7688.c
index a35dae9b56e24..da16c793e3bab 100644
--- a/drivers/gpu/drm/bridge/cros-ec-anx7688.c
+++ b/drivers/gpu/drm/bridge/cros-ec-anx7688.c
@@ -5,6 +5,7 @@
  * Copyright 2020 Google LLC
  */
 
+#include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_print.h>
 #include <linux/i2c.h>
@@ -92,6 +93,9 @@ static bool cros_ec_anx7688_bridge_mode_fixup(struct drm_bridge *bridge,
 }
 
 static const struct drm_bridge_funcs cros_ec_anx7688_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.mode_fixup = cros_ec_anx7688_bridge_mode_fixup,
 };
 
diff --git a/drivers/gpu/drm/bridge/display-connector.c b/drivers/gpu/drm/bridge/display-connector.c
index 16c0631adeb18..d1e74dd2e0518 100644
--- a/drivers/gpu/drm/bridge/display-connector.c
+++ b/drivers/gpu/drm/bridge/display-connector.c
@@ -12,6 +12,7 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/regulator/consumer.h>
+#include <linux/workqueue.h>
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
@@ -25,6 +26,8 @@ struct display_connector {
 
 	struct regulator	*supply;
 	struct gpio_desc	*ddc_en;
+
+	struct work_struct	hpd_work;
 };
 
 static inline struct display_connector *
@@ -87,6 +90,34 @@ display_connector_bridge_detect(struct drm_bridge *bridge, struct drm_connector
 	return display_connector_detect(bridge);
 }
 
+static void display_connector_hpd_enable(struct drm_bridge *bridge)
+{
+	struct display_connector *conn = to_display_connector(bridge);
+
+	enable_irq(conn->hpd_irq);
+
+	if (conn->bridge.type == DRM_MODE_CONNECTOR_DisplayPort)
+		schedule_work(&conn->hpd_work);
+}
+
+static void display_connector_hpd_disable(struct drm_bridge *bridge)
+{
+	struct display_connector *conn = to_display_connector(bridge);
+
+	if (conn->bridge.type == DRM_MODE_CONNECTOR_DisplayPort)
+		cancel_work_sync(&conn->hpd_work);
+
+	disable_irq(conn->hpd_irq);
+}
+
+static void display_connector_hpd_work(struct work_struct *work)
+{
+	struct display_connector *conn = container_of(work, struct display_connector, hpd_work);
+	struct drm_bridge *bridge = &conn->bridge;
+
+	drm_bridge_hpd_notify(bridge, display_connector_detect(bridge));
+}
+
 static const struct drm_edid *display_connector_edid_read(struct drm_bridge *bridge,
 							  struct drm_connector *connector)
 {
@@ -178,12 +209,14 @@ static u32 *display_connector_get_input_bus_fmts(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs display_connector_bridge_funcs = {
 	.attach = display_connector_attach,
 	.detect = display_connector_bridge_detect,
+	.hpd_enable = display_connector_hpd_enable,
+	.hpd_disable = display_connector_hpd_disable,
 	.edid_read = display_connector_edid_read,
 	.atomic_get_output_bus_fmts = display_connector_get_output_bus_fmts,
 	.atomic_get_input_bus_fmts = display_connector_get_input_bus_fmts,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 };
 
 static irqreturn_t display_connector_hpd_irq(int irq, void *arg)
@@ -307,6 +340,7 @@ static int display_connector_probe(struct platform_device *pdev)
 						NULL, display_connector_hpd_irq,
 						IRQF_TRIGGER_RISING |
 						IRQF_TRIGGER_FALLING |
+						IRQF_NO_AUTOEN |
 						IRQF_ONESHOT,
 						"HPD", conn);
 		if (ret) {
@@ -378,6 +412,8 @@ static int display_connector_probe(struct platform_device *pdev)
 		conn->bridge.ops |= DRM_BRIDGE_OP_DETECT;
 	if (conn->hpd_irq >= 0)
 		conn->bridge.ops |= DRM_BRIDGE_OP_HPD;
+	if (conn->hpd_irq >= 0 && type == DRM_MODE_CONNECTOR_DisplayPort)
+		INIT_WORK(&conn->hpd_work, display_connector_hpd_work);
 
 	dev_dbg(&pdev->dev,
 		"Found %s display connector '%s' %s DDC bus and %s HPD GPIO (ops 0x%x)\n",
diff --git a/drivers/gpu/drm/bridge/fsl-ldb.c b/drivers/gpu/drm/bridge/fsl-ldb.c
index 9bfaa3f933709..cc1f88e7873ef 100644
--- a/drivers/gpu/drm/bridge/fsl-ldb.c
+++ b/drivers/gpu/drm/bridge/fsl-ldb.c
@@ -289,7 +289,7 @@ static const struct drm_bridge_funcs funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
 	.atomic_get_input_bus_fmts = fsl_ldb_atomic_get_input_bus_fmts,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.mode_valid = fsl_ldb_mode_valid,
 };
 
diff --git a/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-pvi.c b/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-pvi.c
index 7d5fda7173e5f..a31d4fc76f35b 100644
--- a/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-pvi.c
+++ b/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-pvi.c
@@ -131,7 +131,7 @@ static const struct drm_bridge_funcs imx_hdmi_pvi_bridge_funcs = {
 	.atomic_get_input_bus_fmts = imx8mp_hdmi_pvi_bridge_get_input_bus_fmts,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 };
 
 static int imx8mp_hdmi_pvi_probe(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/bridge/imx/imx8qm-ldb.c b/drivers/gpu/drm/bridge/imx/imx8qm-ldb.c
index a6ca4f5c6cc60..5884af9915477 100644
--- a/drivers/gpu/drm/bridge/imx/imx8qm-ldb.c
+++ b/drivers/gpu/drm/bridge/imx/imx8qm-ldb.c
@@ -389,7 +389,7 @@ imx8qm_ldb_bridge_mode_valid(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs imx8qm_ldb_bridge_funcs = {
 	.atomic_duplicate_state	= drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset		= drm_atomic_helper_bridge_reset,
+	.atomic_create_state	= drm_atomic_helper_bridge_create_state,
 	.mode_valid		= imx8qm_ldb_bridge_mode_valid,
 	.attach			= ldb_bridge_attach_helper,
 	.atomic_check		= imx8qm_ldb_bridge_atomic_check,
diff --git a/drivers/gpu/drm/bridge/imx/imx8qxp-ldb.c b/drivers/gpu/drm/bridge/imx/imx8qxp-ldb.c
index a7906314ade14..b95c640564e4a 100644
--- a/drivers/gpu/drm/bridge/imx/imx8qxp-ldb.c
+++ b/drivers/gpu/drm/bridge/imx/imx8qxp-ldb.c
@@ -404,7 +404,7 @@ static const struct drm_bridge_funcs imx8qxp_ldb_bridge_funcs = {
 	.destroy		= imx8qxp_ldb_bridge_destroy,
 	.atomic_duplicate_state	= drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset		= drm_atomic_helper_bridge_reset,
+	.atomic_create_state	= drm_atomic_helper_bridge_create_state,
 	.mode_valid		= imx8qxp_ldb_bridge_mode_valid,
 	.attach			= ldb_bridge_attach_helper,
 	.atomic_check		= imx8qxp_ldb_bridge_atomic_check,
diff --git a/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-combiner.c b/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-combiner.c
index e0ee51a9ca7f8..a5a1bf42325b6 100644
--- a/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-combiner.c
+++ b/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-combiner.c
@@ -251,7 +251,7 @@ imx8qxp_pc_bridge_atomic_get_output_bus_fmts(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs imx8qxp_pc_bridge_funcs = {
 	.atomic_duplicate_state	= drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset		= drm_atomic_helper_bridge_reset,
+	.atomic_create_state	= drm_atomic_helper_bridge_create_state,
 	.mode_valid		= imx8qxp_pc_bridge_mode_valid,
 	.attach			= imx8qxp_pc_bridge_attach,
 	.mode_set		= imx8qxp_pc_bridge_mode_set,
diff --git a/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c b/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c
index ee6b6dbbe9527..56ce6aa761818 100644
--- a/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c
+++ b/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c
@@ -229,7 +229,7 @@ imx8qxp_pixel_link_bridge_atomic_get_output_bus_fmts(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs imx8qxp_pixel_link_bridge_funcs = {
 	.atomic_duplicate_state	= drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset		= drm_atomic_helper_bridge_reset,
+	.atomic_create_state	= drm_atomic_helper_bridge_create_state,
 	.attach			= imx8qxp_pixel_link_bridge_attach,
 	.mode_set		= imx8qxp_pixel_link_bridge_mode_set,
 	.atomic_enable		= imx8qxp_pixel_link_bridge_atomic_enable,
diff --git a/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c b/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c
index 87305d3e0c398..dce1a5b360240 100644
--- a/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c
+++ b/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c
@@ -210,7 +210,7 @@ imx8qxp_pxl2dpi_bridge_atomic_get_output_bus_fmts(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs imx8qxp_pxl2dpi_bridge_funcs = {
 	.atomic_duplicate_state	= drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset		= drm_atomic_helper_bridge_reset,
+	.atomic_create_state	= drm_atomic_helper_bridge_create_state,
 	.attach			= imx8qxp_pxl2dpi_bridge_attach,
 	.destroy		= imx8qxp_pxl2dpi_bridge_destroy,
 	.atomic_check		= imx8qxp_pxl2dpi_bridge_atomic_check,
diff --git a/drivers/gpu/drm/bridge/inno-hdmi.c b/drivers/gpu/drm/bridge/inno-hdmi.c
index 9a62bf59a4033..dd35f5a875d3d 100644
--- a/drivers/gpu/drm/bridge/inno-hdmi.c
+++ b/drivers/gpu/drm/bridge/inno-hdmi.c
@@ -878,7 +878,7 @@ static void inno_hdmi_bridge_atomic_disable(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs inno_hdmi_bridge_funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_enable = inno_hdmi_bridge_atomic_enable,
 	.atomic_disable = inno_hdmi_bridge_atomic_disable,
 	.detect = inno_hdmi_bridge_detect,
diff --git a/drivers/gpu/drm/bridge/ite-it6263.c b/drivers/gpu/drm/bridge/ite-it6263.c
index c7d588be12cbf..4c0b670806cc5 100644
--- a/drivers/gpu/drm/bridge/ite-it6263.c
+++ b/drivers/gpu/drm/bridge/ite-it6263.c
@@ -829,7 +829,7 @@ static int it6263_hdmi_write_hdmi_infoframe(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs it6263_bridge_funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.attach = it6263_bridge_attach,
 	.mode_valid = it6263_bridge_mode_valid,
 	.atomic_disable = it6263_bridge_atomic_disable,
diff --git a/drivers/gpu/drm/bridge/ite-it6505.c b/drivers/gpu/drm/bridge/ite-it6505.c
index 6b7cb132a4371..8ecb43611dba8 100644
--- a/drivers/gpu/drm/bridge/ite-it6505.c
+++ b/drivers/gpu/drm/bridge/ite-it6505.c
@@ -3239,7 +3239,7 @@ static const struct drm_edid *it6505_bridge_edid_read(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs it6505_bridge_funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.attach = it6505_bridge_attach,
 	.detach = it6505_bridge_detach,
 	.mode_valid = it6505_bridge_mode_valid,
diff --git a/drivers/gpu/drm/bridge/ite-it66121.c b/drivers/gpu/drm/bridge/ite-it66121.c
index a2f48db369e30..51fc68ffdf8f4 100644
--- a/drivers/gpu/drm/bridge/ite-it66121.c
+++ b/drivers/gpu/drm/bridge/ite-it66121.c
@@ -1543,7 +1543,7 @@ static int it66121_hdmi_audio_mute_stream(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs it66121_bridge_funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.attach = it66121_bridge_attach,
 	.atomic_get_output_bus_fmts = it66121_bridge_atomic_get_output_bus_fmts,
 	.atomic_get_input_bus_fmts = it66121_bridge_atomic_get_input_bus_fmts,
diff --git a/drivers/gpu/drm/bridge/lontium-lt8713sx.c b/drivers/gpu/drm/bridge/lontium-lt8713sx.c
index cee485adf5e56..4a6420b876da6 100644
--- a/drivers/gpu/drm/bridge/lontium-lt8713sx.c
+++ b/drivers/gpu/drm/bridge/lontium-lt8713sx.c
@@ -18,6 +18,7 @@
 #include <linux/wait.h>
 #include <linux/workqueue.h>
 
+#include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_of.h>
 
@@ -508,6 +509,9 @@ static const struct attribute_group *lt8713sx_attr_groups[] = {
 };
 
 static const struct drm_bridge_funcs lt8713sx_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach = lt8713sx_bridge_attach,
 };
 
diff --git a/drivers/gpu/drm/bridge/lontium-lt8912b.c b/drivers/gpu/drm/bridge/lontium-lt8912b.c
index 729b12b674706..df5d07e7644c2 100644
--- a/drivers/gpu/drm/bridge/lontium-lt8912b.c
+++ b/drivers/gpu/drm/bridge/lontium-lt8912b.c
@@ -457,7 +457,8 @@ static void lt8912_bridge_mode_set(struct drm_bridge *bridge,
 	drm_display_mode_to_videomode(adj, &lt->mode);
 }
 
-static void lt8912_bridge_enable(struct drm_bridge *bridge)
+static void lt8912_bridge_enable(struct drm_bridge *bridge,
+				 struct drm_atomic_commit *commit)
 {
 	struct lt8912 *lt = bridge_to_lt8912(bridge);
 
@@ -634,11 +635,14 @@ static const struct drm_edid *lt8912_bridge_edid_read(struct drm_bridge *bridge,
 }
 
 static const struct drm_bridge_funcs lt8912_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach = lt8912_bridge_attach,
 	.detach = lt8912_bridge_detach,
 	.mode_valid = lt8912_bridge_mode_valid,
 	.mode_set = lt8912_bridge_mode_set,
-	.enable = lt8912_bridge_enable,
+	.atomic_enable = lt8912_bridge_enable,
 	.detect = lt8912_bridge_detect,
 	.edid_read = lt8912_bridge_edid_read,
 };
diff --git a/drivers/gpu/drm/bridge/lontium-lt9211.c b/drivers/gpu/drm/bridge/lontium-lt9211.c
index 870175a848390..f39d83a5ae377 100644
--- a/drivers/gpu/drm/bridge/lontium-lt9211.c
+++ b/drivers/gpu/drm/bridge/lontium-lt9211.c
@@ -619,7 +619,7 @@ static const struct drm_bridge_funcs lt9211_funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_bridge_destroy_state,
 	.atomic_get_input_bus_fmts = lt9211_atomic_get_input_bus_fmts,
-	.atomic_reset		= drm_atomic_helper_bridge_reset,
+	.atomic_create_state		= drm_atomic_helper_bridge_create_state,
 };
 
 static int lt9211_parse_dt(struct lt9211 *ctx)
diff --git a/drivers/gpu/drm/bridge/lontium-lt9611.c b/drivers/gpu/drm/bridge/lontium-lt9611.c
index 21305296e1117..b6a368f712853 100644
--- a/drivers/gpu/drm/bridge/lontium-lt9611.c
+++ b/drivers/gpu/drm/bridge/lontium-lt9611.c
@@ -1026,7 +1026,7 @@ static const struct drm_bridge_funcs lt9611_bridge_funcs = {
 	.atomic_post_disable = lt9611_bridge_atomic_post_disable,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_get_input_bus_fmts = lt9611_atomic_get_input_bus_fmts,
 
 	.hdmi_tmds_char_rate_valid = lt9611_hdmi_tmds_char_rate_valid,
diff --git a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
index 9427cc2358ae1..6625ba23761c5 100644
--- a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
+++ b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
@@ -465,6 +465,9 @@ static void lt9611uxc_hdmi_audio_shutdown(struct drm_bridge *bridge,
 }
 
 static const struct drm_bridge_funcs lt9611uxc_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach = lt9611uxc_bridge_attach,
 	.mode_valid = lt9611uxc_bridge_mode_valid,
 	.mode_set = lt9611uxc_bridge_mode_set,
diff --git a/drivers/gpu/drm/bridge/lvds-codec.c b/drivers/gpu/drm/bridge/lvds-codec.c
index e6a7147e141b6..a82ea0c944ebf 100644
--- a/drivers/gpu/drm/bridge/lvds-codec.c
+++ b/drivers/gpu/drm/bridge/lvds-codec.c
@@ -43,7 +43,8 @@ static int lvds_codec_attach(struct drm_bridge *bridge,
 				 bridge, flags);
 }
 
-static void lvds_codec_enable(struct drm_bridge *bridge)
+static void lvds_codec_enable(struct drm_bridge *bridge,
+			      struct drm_atomic_commit *commit)
 {
 	struct lvds_codec *lvds_codec = to_lvds_codec(bridge);
 	int ret;
@@ -59,7 +60,8 @@ static void lvds_codec_enable(struct drm_bridge *bridge)
 		gpiod_set_value_cansleep(lvds_codec->powerdown_gpio, 0);
 }
 
-static void lvds_codec_disable(struct drm_bridge *bridge)
+static void lvds_codec_disable(struct drm_bridge *bridge,
+			       struct drm_atomic_commit *commit)
 {
 	struct lvds_codec *lvds_codec = to_lvds_codec(bridge);
 	int ret;
@@ -100,11 +102,11 @@ lvds_codec_atomic_get_input_bus_fmts(struct drm_bridge *bridge,
 
 static const struct drm_bridge_funcs funcs = {
 	.attach = lvds_codec_attach,
-	.enable = lvds_codec_enable,
-	.disable = lvds_codec_disable,
+	.atomic_enable = lvds_codec_enable,
+	.atomic_disable = lvds_codec_disable,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_get_input_bus_fmts = lvds_codec_atomic_get_input_bus_fmts,
 };
 
diff --git a/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c b/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
index 2d02cc69f2374..25ff0a0acc934 100644
--- a/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
+++ b/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
@@ -214,6 +214,9 @@ static int ge_b850v3_lvds_attach(struct drm_bridge *bridge,
 }
 
 static const struct drm_bridge_funcs ge_b850v3_lvds_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach = ge_b850v3_lvds_attach,
 	.detect = ge_b850v3_lvds_bridge_detect,
 	.edid_read = ge_b850v3_lvds_edid_read,
diff --git a/drivers/gpu/drm/bridge/microchip-lvds.c b/drivers/gpu/drm/bridge/microchip-lvds.c
index 5fb8633f43c5c..dd4ffc9f4df3f 100644
--- a/drivers/gpu/drm/bridge/microchip-lvds.c
+++ b/drivers/gpu/drm/bridge/microchip-lvds.c
@@ -167,6 +167,9 @@ static void mchp_lvds_atomic_disable(struct drm_bridge *bridge,
 }
 
 static const struct drm_bridge_funcs mchp_lvds_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach = mchp_lvds_attach,
 	.atomic_enable = mchp_lvds_atomic_enable,
 	.atomic_disable = mchp_lvds_atomic_disable,
diff --git a/drivers/gpu/drm/bridge/nwl-dsi.c b/drivers/gpu/drm/bridge/nwl-dsi.c
index 3c9ae93c4f671..09992529f3d85 100644
--- a/drivers/gpu/drm/bridge/nwl-dsi.c
+++ b/drivers/gpu/drm/bridge/nwl-dsi.c
@@ -960,7 +960,7 @@ static u32 *nwl_bridge_atomic_get_input_bus_fmts(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs nwl_dsi_bridge_funcs = {
 	.atomic_duplicate_state	= drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset		= drm_atomic_helper_bridge_reset,
+	.atomic_create_state		= drm_atomic_helper_bridge_create_state,
 	.atomic_check		= nwl_dsi_bridge_atomic_check,
 	.atomic_enable		= nwl_dsi_bridge_atomic_enable,
 	.atomic_disable		= nwl_dsi_bridge_atomic_disable,
diff --git a/drivers/gpu/drm/bridge/nxp-ptn3460.c b/drivers/gpu/drm/bridge/nxp-ptn3460.c
index 7acb11f16dc19..a9b4ac2defc99 100644
--- a/drivers/gpu/drm/bridge/nxp-ptn3460.c
+++ b/drivers/gpu/drm/bridge/nxp-ptn3460.c
@@ -111,7 +111,8 @@ static int ptn3460_select_edid(struct ptn3460_bridge *ptn_bridge)
 	return 0;
 }
 
-static void ptn3460_pre_enable(struct drm_bridge *bridge)
+static void ptn3460_pre_enable(struct drm_bridge *bridge,
+			       struct drm_atomic_commit *commit)
 {
 	struct ptn3460_bridge *ptn_bridge = bridge_to_ptn3460(bridge);
 	int ret;
@@ -139,7 +140,8 @@ static void ptn3460_pre_enable(struct drm_bridge *bridge)
 	ptn_bridge->enabled = true;
 }
 
-static void ptn3460_disable(struct drm_bridge *bridge)
+static void ptn3460_disable(struct drm_bridge *bridge,
+			    struct drm_atomic_commit *commit)
 {
 	struct ptn3460_bridge *ptn_bridge = bridge_to_ptn3460(bridge);
 
@@ -163,7 +165,7 @@ static const struct drm_edid *ptn3460_edid_read(struct drm_bridge *bridge,
 	int ret;
 
 	power_off = !ptn_bridge->enabled;
-	ptn3460_pre_enable(&ptn_bridge->bridge);
+	ptn3460_pre_enable(&ptn_bridge->bridge, NULL);
 
 	edid = kmalloc(EDID_LENGTH, GFP_KERNEL);
 	if (!edid) {
@@ -182,7 +184,7 @@ static const struct drm_edid *ptn3460_edid_read(struct drm_bridge *bridge,
 
 out:
 	if (power_off)
-		ptn3460_disable(&ptn_bridge->bridge);
+		ptn3460_disable(&ptn_bridge->bridge, NULL);
 
 	return drm_edid;
 }
@@ -248,8 +250,11 @@ static int ptn3460_bridge_attach(struct drm_bridge *bridge,
 }
 
 static const struct drm_bridge_funcs ptn3460_bridge_funcs = {
-	.pre_enable = ptn3460_pre_enable,
-	.disable = ptn3460_disable,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+	.atomic_pre_enable = ptn3460_pre_enable,
+	.atomic_disable = ptn3460_disable,
 	.attach = ptn3460_bridge_attach,
 	.edid_read = ptn3460_edid_read,
 };
diff --git a/drivers/gpu/drm/bridge/of-display-mode-bridge.c b/drivers/gpu/drm/bridge/of-display-mode-bridge.c
index cb15713f3a796..af3e727e91c90 100644
--- a/drivers/gpu/drm/bridge/of-display-mode-bridge.c
+++ b/drivers/gpu/drm/bridge/of-display-mode-bridge.c
@@ -9,6 +9,7 @@
 
 #include <linux/export.h>
 
+#include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_modes.h>
 #include <drm/drm_probe_helper.h>
@@ -51,7 +52,10 @@ static int of_display_mode_bridge_get_modes(struct drm_bridge *bridge,
 	return 0;
 }
 
-struct drm_bridge_funcs of_display_mode_bridge_funcs = {
+static const struct drm_bridge_funcs of_display_mode_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach = of_display_mode_bridge_attach,
 	.get_modes = of_display_mode_bridge_get_modes,
 };
diff --git a/drivers/gpu/drm/bridge/panel.c b/drivers/gpu/drm/bridge/panel.c
index 4978ec98a0828..62af499f1f5cc 100644
--- a/drivers/gpu/drm/bridge/panel.c
+++ b/drivers/gpu/drm/bridge/panel.c
@@ -213,7 +213,7 @@ static const struct drm_bridge_funcs panel_bridge_bridge_funcs = {
 	.atomic_disable = panel_bridge_atomic_disable,
 	.atomic_post_disable = panel_bridge_atomic_post_disable,
 	.get_modes = panel_bridge_get_modes,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
 	.atomic_get_input_bus_fmts = drm_atomic_helper_bridge_propagate_bus_fmt,
diff --git a/drivers/gpu/drm/bridge/parade-ps8622.c b/drivers/gpu/drm/bridge/parade-ps8622.c
index f879a1df077d2..33844db753aa5 100644
--- a/drivers/gpu/drm/bridge/parade-ps8622.c
+++ b/drivers/gpu/drm/bridge/parade-ps8622.c
@@ -336,7 +336,8 @@ static const struct backlight_ops ps8622_backlight_ops = {
 	.update_status	= ps8622_backlight_update,
 };
 
-static void ps8622_pre_enable(struct drm_bridge *bridge)
+static void ps8622_pre_enable(struct drm_bridge *bridge,
+			      struct drm_atomic_commit *commit)
 {
 	struct ps8622_bridge *ps8622 = bridge_to_ps8622(bridge);
 	int ret;
@@ -381,13 +382,15 @@ static void ps8622_pre_enable(struct drm_bridge *bridge)
 	ps8622->enabled = true;
 }
 
-static void ps8622_disable(struct drm_bridge *bridge)
+static void ps8622_disable(struct drm_bridge *bridge,
+			   struct drm_atomic_commit *commit)
 {
 	/* Delay after panel is disabled */
 	msleep(PS8622_PWMO_END_T12_MS);
 }
 
-static void ps8622_post_disable(struct drm_bridge *bridge)
+static void ps8622_post_disable(struct drm_bridge *bridge,
+				struct drm_atomic_commit *commit)
 {
 	struct ps8622_bridge *ps8622 = bridge_to_ps8622(bridge);
 
@@ -428,9 +431,12 @@ static int ps8622_attach(struct drm_bridge *bridge,
 }
 
 static const struct drm_bridge_funcs ps8622_bridge_funcs = {
-	.pre_enable = ps8622_pre_enable,
-	.disable = ps8622_disable,
-	.post_disable = ps8622_post_disable,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+	.atomic_pre_enable = ps8622_pre_enable,
+	.atomic_disable = ps8622_disable,
+	.atomic_post_disable = ps8622_post_disable,
 	.attach = ps8622_attach,
 };
 
diff --git a/drivers/gpu/drm/bridge/parade-ps8640.c b/drivers/gpu/drm/bridge/parade-ps8640.c
index b93514023baa8..96332721cb69b 100644
--- a/drivers/gpu/drm/bridge/parade-ps8640.c
+++ b/drivers/gpu/drm/bridge/parade-ps8640.c
@@ -556,7 +556,7 @@ static const struct drm_bridge_funcs ps8640_bridge_funcs = {
 	.atomic_pre_enable = ps8640_atomic_pre_enable,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 };
 
 static int ps8640_bridge_get_dsi_resources(struct device *dev, struct ps8640 *ps_bridge)
diff --git a/drivers/gpu/drm/bridge/samsung-dsim.c b/drivers/gpu/drm/bridge/samsung-dsim.c
index 9ee0515074c78..25ab475309e87 100644
--- a/drivers/gpu/drm/bridge/samsung-dsim.c
+++ b/drivers/gpu/drm/bridge/samsung-dsim.c
@@ -1835,7 +1835,7 @@ static int samsung_dsim_attach(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs samsung_dsim_bridge_funcs = {
 	.atomic_duplicate_state		= drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state		= drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset			= drm_atomic_helper_bridge_reset,
+	.atomic_create_state			= drm_atomic_helper_bridge_create_state,
 	.atomic_get_input_bus_fmts	= samsung_dsim_atomic_get_input_bus_fmts,
 	.atomic_check			= samsung_dsim_atomic_check,
 	.atomic_pre_enable		= samsung_dsim_atomic_pre_enable,
@@ -1919,7 +1919,7 @@ static int samsung_dsim_host_attach(struct mipi_dsi_host *host,
 	/*
 	 * of_graph_get_remote_node() produces a noisy error message if port
 	 * node isn't found and the absence of the port is a legit case here,
-	 * so at first we silently check whether graph presents in the
+	 * so at first we silently check whether a graph is present in the
 	 * device-tree node.
 	 */
 	if (!of_graph_is_present(np))
diff --git a/drivers/gpu/drm/bridge/sii902x.c b/drivers/gpu/drm/bridge/sii902x.c
index ea543be4ae3e2..f575b2dbe32be 100644
--- a/drivers/gpu/drm/bridge/sii902x.c
+++ b/drivers/gpu/drm/bridge/sii902x.c
@@ -544,7 +544,7 @@ static const struct drm_bridge_funcs sii902x_bridge_funcs = {
 	.atomic_enable = sii902x_bridge_atomic_enable,
 	.detect = sii902x_bridge_detect,
 	.edid_read = sii902x_bridge_edid_read,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
 	.atomic_get_input_bus_fmts = sii902x_bridge_atomic_get_input_bus_fmts,
@@ -1158,9 +1158,7 @@ static int sii902x_probe(struct i2c_client *client)
 	static const char * const supplies[] = {"iovcc", "cvcc12"};
 	int ret;
 
-	ret = i2c_check_functionality(client->adapter,
-				      I2C_FUNC_SMBUS_BYTE_DATA);
-	if (!ret) {
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_BYTE_DATA)) {
 		dev_err(dev, "I2C adapter not suitable\n");
 		return -EIO;
 	}
diff --git a/drivers/gpu/drm/bridge/sii9234.c b/drivers/gpu/drm/bridge/sii9234.c
index bb1bed03eb5b7..be67642ab7d31 100644
--- a/drivers/gpu/drm/bridge/sii9234.c
+++ b/drivers/gpu/drm/bridge/sii9234.c
@@ -13,6 +13,7 @@
  *    Dharam Kumar <dharam.kr@samsung.com>
  */
 #include <drm/bridge/mhl.h>
+#include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_edid.h>
@@ -339,7 +340,7 @@ static int sii9234_cbus_reset(struct sii9234 *ctx)
 	return sii9234_clear_error(ctx);
 }
 
-/* Require to chek mhl imformation of samsung in cbus_init_register */
+/* Require to check mhl information of samsung in cbus_init_register */
 static int sii9234_cbus_init(struct sii9234 *ctx)
 {
 	cbus_writeb(ctx, 0x07, 0xF2);
@@ -614,7 +615,7 @@ static void sii9234_cable_out(struct sii9234 *ctx)
 
 	disable_irq(to_i2c_client(ctx->dev)->irq);
 	tpi_writeb(ctx, TPI_DPD_REG, 0);
-	/* Turn on&off hpd festure for only QCT HDMI */
+	/* Turn on&off hpd feature for only QCT HDMI */
 	sii9234_hw_off(ctx);
 
 	ctx->state = ST_OFF;
@@ -708,7 +709,7 @@ static enum sii9234_state sii9234_rsen_change(struct sii9234 *ctx)
 {
 	int value;
 
-	/* Work_around code to handle wrong interrupt */
+	/* Workaround code to handle wrong interrupt */
 	if (ctx->state != ST_RGND_1K) {
 		dev_err(ctx->dev, "RSEN_HIGH without RGND_1K\n");
 		return ST_FAILURE;
@@ -723,9 +724,9 @@ static enum sii9234_state sii9234_rsen_change(struct sii9234 *ctx)
 	}
 	dev_dbg(ctx->dev, "RSEN lost\n");
 	/*
-	 * Once RSEN loss is confirmed,we need to check
-	 * based on cable status and chip power status,whether
-	 * it is SINK Loss(HDMI cable not connected, TV Off)
+	 * Once RSEN loss is confirmed, we need to check
+	 * based on cable status and chip power status, whether
+	 * it is SINK Loss (HDMI cable not connected, TV Off)
 	 * or MHL cable disconnection
 	 * TODO: Define the below mhl_disconnection()
 	 */
@@ -820,26 +821,22 @@ static int sii9234_init_resources(struct sii9234 *ctx,
 	int ret;
 
 	if (!ctx->dev->of_node) {
-		dev_err(ctx->dev, "not DT device\n");
+		dev_err(ctx->dev, "no DT device\n");
 		return -ENODEV;
 	}
 
 	ctx->gpio_reset = devm_gpiod_get(ctx->dev, "reset", GPIOD_OUT_LOW);
-	if (IS_ERR(ctx->gpio_reset)) {
-		dev_err(ctx->dev, "failed to get reset gpio from DT\n");
-		return PTR_ERR(ctx->gpio_reset);
-	}
+	if (IS_ERR(ctx->gpio_reset))
+		return dev_err_probe(ctx->dev, PTR_ERR(ctx->gpio_reset),
+				     "failed to get reset gpio from DT\n");
 
 	ctx->supplies[0].supply = "avcc12";
 	ctx->supplies[1].supply = "avcc33";
 	ctx->supplies[2].supply = "iovcc18";
 	ctx->supplies[3].supply = "cvcc12";
 	ret = devm_regulator_bulk_get(ctx->dev, 4, ctx->supplies);
-	if (ret) {
-		if (ret != -EPROBE_DEFER)
-			dev_err(ctx->dev, "regulator_bulk failed\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(ctx->dev, ret, "regulator_bulk failed\n");
 
 	ctx->client[I2C_MHL] = client;
 
@@ -878,6 +875,9 @@ static enum drm_mode_status sii9234_mode_valid(struct drm_bridge *bridge,
 }
 
 static const struct drm_bridge_funcs sii9234_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.mode_valid = sii9234_mode_valid,
 };
 
@@ -911,10 +911,9 @@ static int sii9234_probe(struct i2c_client *client)
 					sii9234_irq_thread,
 					IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
 					"sii9234", ctx);
-	if (ret < 0) {
-		dev_err(dev, "failed to install IRQ handler\n");
-		return ret;
-	}
+	if (ret < 0)
+		return dev_err_probe(dev, ret,
+				     "failed to install IRQ handler\n");
 
 	ret = sii9234_init_resources(ctx, client);
 	if (ret < 0)
diff --git a/drivers/gpu/drm/bridge/sil-sii8620.c b/drivers/gpu/drm/bridge/sil-sii8620.c
index 982306eb4f0a7..22c81b38b41e8 100644
--- a/drivers/gpu/drm/bridge/sil-sii8620.c
+++ b/drivers/gpu/drm/bridge/sil-sii8620.c
@@ -9,6 +9,7 @@
 #include <linux/unaligned.h>
 
 #include <drm/bridge/mhl.h>
+#include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_edid.h>
@@ -2280,6 +2281,9 @@ static bool sii8620_mode_fixup(struct drm_bridge *bridge,
 }
 
 static const struct drm_bridge_funcs sii8620_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach = sii8620_attach,
 	.detach = sii8620_detach,
 	.mode_fixup = sii8620_mode_fixup,
diff --git a/drivers/gpu/drm/bridge/simple-bridge.c b/drivers/gpu/drm/bridge/simple-bridge.c
index cc13c98f9be6b..3fba7002322e9 100644
--- a/drivers/gpu/drm/bridge/simple-bridge.c
+++ b/drivers/gpu/drm/bridge/simple-bridge.c
@@ -132,7 +132,8 @@ static int simple_bridge_attach(struct drm_bridge *bridge,
 	return 0;
 }
 
-static void simple_bridge_enable(struct drm_bridge *bridge)
+static void simple_bridge_enable(struct drm_bridge *bridge,
+				 struct drm_atomic_commit *commit)
 {
 	struct simple_bridge *sbridge = drm_bridge_to_simple_bridge(bridge);
 	int ret;
@@ -146,7 +147,8 @@ static void simple_bridge_enable(struct drm_bridge *bridge)
 	gpiod_set_value_cansleep(sbridge->enable, 1);
 }
 
-static void simple_bridge_disable(struct drm_bridge *bridge)
+static void simple_bridge_disable(struct drm_bridge *bridge,
+				  struct drm_atomic_commit *commit)
 {
 	struct simple_bridge *sbridge = drm_bridge_to_simple_bridge(bridge);
 
@@ -157,9 +159,12 @@ static void simple_bridge_disable(struct drm_bridge *bridge)
 }
 
 static const struct drm_bridge_funcs simple_bridge_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach		= simple_bridge_attach,
-	.enable		= simple_bridge_enable,
-	.disable	= simple_bridge_disable,
+	.atomic_enable = simple_bridge_enable,
+	.atomic_disable = simple_bridge_disable,
 };
 
 static int simple_bridge_probe(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/bridge/ssd2825.c b/drivers/gpu/drm/bridge/ssd2825.c
index 54ca6bd6883ae..00a4ed4a07005 100644
--- a/drivers/gpu/drm/bridge/ssd2825.c
+++ b/drivers/gpu/drm/bridge/ssd2825.c
@@ -679,7 +679,7 @@ static const struct drm_bridge_funcs ssd2825_bridge_funcs = {
 	.atomic_enable = ssd2825_bridge_atomic_enable,
 	.atomic_disable = ssd2825_bridge_atomic_disable,
 
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
 };
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-dp.c b/drivers/gpu/drm/bridge/synopsys/dw-dp.c
index 21541be094c47..3445c82e6f50e 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-dp.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-dp.c
@@ -1816,7 +1816,7 @@ static struct drm_bridge_state *dw_dp_bridge_atomic_duplicate_state(struct drm_b
 static const struct drm_bridge_funcs dw_dp_bridge_funcs = {
 	.atomic_duplicate_state = dw_dp_bridge_atomic_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_get_input_bus_fmts = drm_atomic_helper_bridge_propagate_bus_fmt,
 	.atomic_get_output_bus_fmts = dw_dp_bridge_atomic_get_output_bus_fmts,
 	.atomic_check = dw_dp_bridge_atomic_check,
@@ -2093,6 +2093,12 @@ struct dw_dp *dw_dp_bind(struct device *dev, struct drm_encoder *encoder,
 }
 EXPORT_SYMBOL_GPL(dw_dp_bind);
 
+void dw_dp_unbind(struct dw_dp *dp)
+{
+	drm_dp_aux_unregister(&dp->aux);
+}
+EXPORT_SYMBOL_GPL(dw_dp_unbind);
+
 MODULE_AUTHOR("Andy Yan <andyshrk@163.com>");
 MODULE_DESCRIPTION("DW DP Core Library");
 MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c
index 1c214a8e6dc2d..cfd1b18c06e62 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c
@@ -1190,9 +1190,10 @@ static int dw_hdmi_qp_cec_transmit(struct drm_bridge *bridge, u8 attempts,
 #endif /* CONFIG_DRM_DW_HDMI_QP_CEC */
 
 static const struct drm_bridge_funcs dw_hdmi_qp_bridge_funcs = {
+	.atomic_get_output_bus_fmts = drm_atomic_helper_bridge_get_hdmi_output_bus_fmts,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_enable = dw_hdmi_qp_bridge_atomic_enable,
 	.atomic_disable = dw_hdmi_qp_bridge_atomic_disable,
 	.detect = dw_hdmi_qp_bridge_detect,
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
index 17d5caedb32e0..a408dbd414bfa 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
@@ -3027,7 +3027,7 @@ static const struct drm_edid *dw_hdmi_bridge_edid_read(struct drm_bridge *bridge
 static const struct drm_bridge_funcs dw_hdmi_bridge_funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.attach = dw_hdmi_bridge_attach,
 	.detach = dw_hdmi_bridge_detach,
 	.atomic_check = dw_hdmi_bridge_atomic_check,
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
index af176e136f85e..5dd475fcaacf8 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
@@ -1095,7 +1095,7 @@ static const struct drm_bridge_funcs dw_mipi_dsi_bridge_funcs = {
 	.atomic_destroy_state	= drm_atomic_helper_bridge_destroy_state,
 	.atomic_get_input_bus_fmts = dw_mipi_dsi_bridge_atomic_get_input_bus_fmts,
 	.atomic_check		= dw_mipi_dsi_bridge_atomic_check,
-	.atomic_reset		= drm_atomic_helper_bridge_reset,
+	.atomic_create_state		= drm_atomic_helper_bridge_create_state,
 	.atomic_pre_enable	= dw_mipi_dsi_bridge_atomic_pre_enable,
 	.atomic_enable		= dw_mipi_dsi_bridge_atomic_enable,
 	.atomic_post_disable	= dw_mipi_dsi_bridge_post_atomic_disable,
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi2.c b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi2.c
index b2a022d266d90..5f06cb03b4c04 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi2.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi2.c
@@ -893,7 +893,7 @@ static const struct drm_bridge_funcs dw_mipi_dsi2_bridge_funcs = {
 	.atomic_destroy_state	= drm_atomic_helper_bridge_destroy_state,
 	.atomic_get_input_bus_fmts = dw_mipi_dsi2_bridge_atomic_get_input_bus_fmts,
 	.atomic_check		= dw_mipi_dsi2_bridge_atomic_check,
-	.atomic_reset		= drm_atomic_helper_bridge_reset,
+	.atomic_create_state		= drm_atomic_helper_bridge_create_state,
 	.atomic_pre_enable	= dw_mipi_dsi2_bridge_atomic_pre_enable,
 	.atomic_enable		= dw_mipi_dsi2_bridge_atomic_enable,
 	.atomic_post_disable	= dw_mipi_dsi2_bridge_post_atomic_disable,
diff --git a/drivers/gpu/drm/bridge/tc358762.c b/drivers/gpu/drm/bridge/tc358762.c
index 6aa93b3274dd8..6124d3123f577 100644
--- a/drivers/gpu/drm/bridge/tc358762.c
+++ b/drivers/gpu/drm/bridge/tc358762.c
@@ -225,7 +225,7 @@ static const struct drm_bridge_funcs tc358762_bridge_funcs = {
 	.atomic_enable = tc358762_enable,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.attach = tc358762_attach,
 	.mode_set = tc358762_bridge_mode_set,
 };
diff --git a/drivers/gpu/drm/bridge/tc358764.c b/drivers/gpu/drm/bridge/tc358764.c
index 12b43245bb8f6..f55a56cbd88fd 100644
--- a/drivers/gpu/drm/bridge/tc358764.c
+++ b/drivers/gpu/drm/bridge/tc358764.c
@@ -266,7 +266,8 @@ static void tc358764_reset(struct tc358764 *ctx)
 	usleep_range(1000, 2000);
 }
 
-static void tc358764_post_disable(struct drm_bridge *bridge)
+static void tc358764_post_disable(struct drm_bridge *bridge,
+				  struct drm_atomic_commit *commit)
 {
 	struct tc358764 *ctx = bridge_to_tc358764(bridge);
 	int ret;
@@ -278,7 +279,8 @@ static void tc358764_post_disable(struct drm_bridge *bridge)
 		dev_err(ctx->dev, "error disabling regulators (%d)\n", ret);
 }
 
-static void tc358764_pre_enable(struct drm_bridge *bridge)
+static void tc358764_pre_enable(struct drm_bridge *bridge,
+				struct drm_atomic_commit *commit)
 {
 	struct tc358764 *ctx = bridge_to_tc358764(bridge);
 	int ret;
@@ -303,8 +305,11 @@ static int tc358764_attach(struct drm_bridge *bridge,
 }
 
 static const struct drm_bridge_funcs tc358764_bridge_funcs = {
-	.post_disable = tc358764_post_disable,
-	.pre_enable = tc358764_pre_enable,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+	.atomic_post_disable = tc358764_post_disable,
+	.atomic_pre_enable = tc358764_pre_enable,
 	.attach = tc358764_attach,
 };
 
diff --git a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c
index 7188935fdb826..ce1b73214f26c 100644
--- a/drivers/gpu/drm/bridge/tc358767.c
+++ b/drivers/gpu/drm/bridge/tc358767.c
@@ -527,7 +527,7 @@ static ssize_t tc_aux_transfer(struct drm_dp_aux *aux,
 	 * address-only transfer
 	 */
 	if (size)
-		size = FIELD_GET(AUX_BYTES, auxstatus);
+		size = min_t(size_t, size, FIELD_GET(AUX_BYTES, auxstatus));
 	msg->reply = FIELD_GET(AUX_STATUS, auxstatus);
 
 	switch (request) {
@@ -1933,7 +1933,7 @@ static const struct drm_bridge_funcs tc_dpi_bridge_funcs = {
 	.atomic_disable = tc_dpi_bridge_atomic_disable,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_get_input_bus_fmts = tc_dpi_atomic_get_input_bus_fmts,
 };
 
@@ -1949,7 +1949,7 @@ static const struct drm_bridge_funcs tc_edp_bridge_funcs = {
 	.edid_read = tc_edid_read,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_get_input_bus_fmts = drm_atomic_helper_bridge_propagate_bus_fmt,
 	.atomic_get_output_bus_fmts = tc_edp_atomic_get_output_bus_fmts,
 };
diff --git a/drivers/gpu/drm/bridge/tc358768.c b/drivers/gpu/drm/bridge/tc358768.c
index 0f2820b373029..0d85120fcc7a3 100644
--- a/drivers/gpu/drm/bridge/tc358768.c
+++ b/drivers/gpu/drm/bridge/tc358768.c
@@ -1290,7 +1290,7 @@ static const struct drm_bridge_funcs tc358768_bridge_funcs = {
 
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_get_input_bus_fmts = tc358768_atomic_get_input_bus_fmts,
 };
 
diff --git a/drivers/gpu/drm/bridge/tc358775.c b/drivers/gpu/drm/bridge/tc358775.c
index e067b671cecee..728938f9dcb45 100644
--- a/drivers/gpu/drm/bridge/tc358775.c
+++ b/drivers/gpu/drm/bridge/tc358775.c
@@ -598,7 +598,7 @@ static const struct drm_bridge_funcs tc_bridge_funcs = {
 	.atomic_enable = tc_bridge_atomic_enable,
 	.mode_valid = tc_mode_valid,
 	.atomic_post_disable = tc_bridge_atomic_post_disable,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
 };
diff --git a/drivers/gpu/drm/bridge/tda998x_drv.c b/drivers/gpu/drm/bridge/tda998x_drv.c
index 6c427bc75896b..16c7455911758 100644
--- a/drivers/gpu/drm/bridge/tda998x_drv.c
+++ b/drivers/gpu/drm/bridge/tda998x_drv.c
@@ -1410,7 +1410,8 @@ static enum drm_mode_status tda998x_bridge_mode_valid(struct drm_bridge *bridge,
 	return MODE_OK;
 }
 
-static void tda998x_bridge_enable(struct drm_bridge *bridge)
+static void tda998x_bridge_enable(struct drm_bridge *bridge,
+				  struct drm_atomic_commit *commit)
 {
 	struct tda998x_priv *priv = bridge_to_tda998x_priv(bridge);
 
@@ -1428,7 +1429,8 @@ static void tda998x_bridge_enable(struct drm_bridge *bridge)
 	}
 }
 
-static void tda998x_bridge_disable(struct drm_bridge *bridge)
+static void tda998x_bridge_disable(struct drm_bridge *bridge,
+				   struct drm_atomic_commit *commit)
 {
 	struct tda998x_priv *priv = bridge_to_tda998x_priv(bridge);
 
@@ -1745,12 +1747,15 @@ static void tda998x_bridge_hpd_disable(struct drm_bridge *bridge)
 }
 
 static const struct drm_bridge_funcs tda998x_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach = tda998x_bridge_attach,
 	.detach = tda998x_bridge_detach,
 	.mode_valid = tda998x_bridge_mode_valid,
-	.disable = tda998x_bridge_disable,
+	.atomic_disable = tda998x_bridge_disable,
 	.mode_set = tda998x_bridge_mode_set,
-	.enable = tda998x_bridge_enable,
+	.atomic_enable = tda998x_bridge_enable,
 	.edid_read = tda998x_bridge_edid_read,
 	.detect = tda998x_bridge_detect,
 	.hpd_enable = tda998x_bridge_hpd_enable,
diff --git a/drivers/gpu/drm/bridge/thc63lvd1024.c b/drivers/gpu/drm/bridge/thc63lvd1024.c
index c804222846c38..679b424ba0c36 100644
--- a/drivers/gpu/drm/bridge/thc63lvd1024.c
+++ b/drivers/gpu/drm/bridge/thc63lvd1024.c
@@ -13,6 +13,7 @@
 #include <linux/regulator/consumer.h>
 #include <linux/slab.h>
 
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_panel.h>
 
@@ -81,7 +82,8 @@ static enum drm_mode_status thc63_mode_valid(struct drm_bridge *bridge,
 	return MODE_OK;
 }
 
-static void thc63_enable(struct drm_bridge *bridge)
+static void thc63_enable(struct drm_bridge *bridge,
+			 struct drm_atomic_commit *commit)
 {
 	struct thc63_dev *thc63 = to_thc63(bridge);
 	int ret;
@@ -97,7 +99,8 @@ static void thc63_enable(struct drm_bridge *bridge)
 	gpiod_set_value(thc63->oe, 1);
 }
 
-static void thc63_disable(struct drm_bridge *bridge)
+static void thc63_disable(struct drm_bridge *bridge,
+			  struct drm_atomic_commit *commit)
 {
 	struct thc63_dev *thc63 = to_thc63(bridge);
 	int ret;
@@ -112,10 +115,13 @@ static void thc63_disable(struct drm_bridge *bridge)
 }
 
 static const struct drm_bridge_funcs thc63_bridge_func = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach	= thc63_attach,
 	.mode_valid = thc63_mode_valid,
-	.enable = thc63_enable,
-	.disable = thc63_disable,
+	.atomic_enable = thc63_enable,
+	.atomic_disable = thc63_disable,
 };
 
 static int thc63_parse_dt(struct thc63_dev *thc63)
diff --git a/drivers/gpu/drm/bridge/ti-dlpc3433.c b/drivers/gpu/drm/bridge/ti-dlpc3433.c
index 9c35eade0a78e..c16eb3a05cfac 100644
--- a/drivers/gpu/drm/bridge/ti-dlpc3433.c
+++ b/drivers/gpu/drm/bridge/ti-dlpc3433.c
@@ -254,7 +254,7 @@ static const struct drm_bridge_funcs dlpc_bridge_funcs = {
 	.atomic_duplicate_state		= drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state		= drm_atomic_helper_bridge_destroy_state,
 	.atomic_get_input_bus_fmts	= dlpc_atomic_get_input_bus_fmts,
-	.atomic_reset			= drm_atomic_helper_bridge_reset,
+	.atomic_create_state			= drm_atomic_helper_bridge_create_state,
 	.atomic_pre_enable		= dlpc_atomic_pre_enable,
 	.atomic_enable			= dlpc_atomic_enable,
 	.atomic_post_disable		= dlpc_atomic_post_disable,
diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi83.c b/drivers/gpu/drm/bridge/ti-sn65dsi83.c
index 42b451432bbb6..e6dbe51d0dba9 100644
--- a/drivers/gpu/drm/bridge/ti-sn65dsi83.c
+++ b/drivers/gpu/drm/bridge/ti-sn65dsi83.c
@@ -801,7 +801,7 @@ static const struct drm_bridge_funcs sn65dsi83_funcs = {
 
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_get_input_bus_fmts = sn65dsi83_atomic_get_input_bus_fmts,
 };
 
@@ -977,7 +977,6 @@ static int sn65dsi83_host_attach(struct sn65dsi83 *ctx)
 	dsi->lanes = dsi_lanes;
 	dsi->format = MIPI_DSI_FMT_RGB888;
 	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
-			  MIPI_DSI_MODE_VIDEO_NO_HFP | MIPI_DSI_MODE_VIDEO_NO_HBP |
 			  MIPI_DSI_MODE_VIDEO_NO_HSA | MIPI_DSI_MODE_NO_EOT_PACKET;
 
 	ret = devm_mipi_dsi_attach(dev, dsi);
diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi86.c b/drivers/gpu/drm/bridge/ti-sn65dsi86.c
index 0884ec459cc54..1da242f174137 100644
--- a/drivers/gpu/drm/bridge/ti-sn65dsi86.c
+++ b/drivers/gpu/drm/bridge/ti-sn65dsi86.c
@@ -1302,7 +1302,7 @@ static const struct drm_bridge_funcs ti_sn_bridge_funcs = {
 	.atomic_enable = ti_sn_bridge_atomic_enable,
 	.atomic_disable = ti_sn_bridge_atomic_disable,
 	.atomic_post_disable = ti_sn_bridge_atomic_post_disable,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
 	.debugfs_init = ti_sn65dsi86_debugfs_init,
diff --git a/drivers/gpu/drm/bridge/ti-tdp158.c b/drivers/gpu/drm/bridge/ti-tdp158.c
index 3a19967f213fb..01fc643bd78f1 100644
--- a/drivers/gpu/drm/bridge/ti-tdp158.c
+++ b/drivers/gpu/drm/bridge/ti-tdp158.c
@@ -60,7 +60,7 @@ static const struct drm_bridge_funcs tdp158_bridge_funcs = {
 	.atomic_disable = tdp158_disable,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 };
 
 static int tdp158_probe(struct i2c_client *client)
diff --git a/drivers/gpu/drm/bridge/ti-tfp410.c b/drivers/gpu/drm/bridge/ti-tfp410.c
index bf4ab4eaf2697..f0a14f52aaace 100644
--- a/drivers/gpu/drm/bridge/ti-tfp410.c
+++ b/drivers/gpu/drm/bridge/ti-tfp410.c
@@ -174,14 +174,16 @@ static void tfp410_detach(struct drm_bridge *bridge)
 	}
 }
 
-static void tfp410_enable(struct drm_bridge *bridge)
+static void tfp410_enable(struct drm_bridge *bridge,
+			  struct drm_atomic_commit *commit)
 {
 	struct tfp410 *dvi = drm_bridge_to_tfp410(bridge);
 
 	gpiod_set_value_cansleep(dvi->powerdown, 0);
 }
 
-static void tfp410_disable(struct drm_bridge *bridge)
+static void tfp410_disable(struct drm_bridge *bridge,
+			   struct drm_atomic_commit *commit)
 {
 	struct tfp410 *dvi = drm_bridge_to_tfp410(bridge);
 
@@ -242,10 +244,10 @@ static int tfp410_atomic_check(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs tfp410_bridge_funcs = {
 	.attach		= tfp410_attach,
 	.detach		= tfp410_detach,
-	.enable		= tfp410_enable,
-	.disable	= tfp410_disable,
+	.atomic_enable = tfp410_enable,
+	.atomic_disable = tfp410_disable,
 	.mode_valid	= tfp410_mode_valid,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
 	.atomic_get_input_bus_fmts = tfp410_get_input_bus_fmts,
diff --git a/drivers/gpu/drm/bridge/ti-tpd12s015.c b/drivers/gpu/drm/bridge/ti-tpd12s015.c
index 136e47ad1a10d..e7b172ab7f552 100644
--- a/drivers/gpu/drm/bridge/ti-tpd12s015.c
+++ b/drivers/gpu/drm/bridge/ti-tpd12s015.c
@@ -19,6 +19,7 @@
 #include <linux/of_graph.h>
 #include <linux/platform_device.h>
 
+#include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
 
 struct tpd12s015_device {
@@ -96,6 +97,9 @@ static void tpd12s015_hpd_disable(struct drm_bridge *bridge)
 }
 
 static const struct drm_bridge_funcs tpd12s015_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach			= tpd12s015_attach,
 	.detach			= tpd12s015_detach,
 	.detect			= tpd12s015_bridge_detect,
diff --git a/drivers/gpu/drm/bridge/waveshare-dsi.c b/drivers/gpu/drm/bridge/waveshare-dsi.c
index ded57f298d640..9c9825c9b7c59 100644
--- a/drivers/gpu/drm/bridge/waveshare-dsi.c
+++ b/drivers/gpu/drm/bridge/waveshare-dsi.c
@@ -12,6 +12,7 @@
 #include <linux/of_graph.h>
 #include <linux/regmap.h>
 
+#include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_of.h>
@@ -92,7 +93,8 @@ static int ws_bridge_bridge_attach(struct drm_bridge *bridge,
 				 &ws->bridge, flags);
 }
 
-static void ws_bridge_bridge_enable(struct drm_bridge *bridge)
+static void ws_bridge_bridge_enable(struct drm_bridge *bridge,
+				    struct drm_atomic_commit *commit)
 {
 	struct ws_bridge *ws = bridge_to_ws_bridge(bridge);
 
@@ -100,7 +102,8 @@ static void ws_bridge_bridge_enable(struct drm_bridge *bridge)
 	backlight_enable(ws->backlight);
 }
 
-static void ws_bridge_bridge_disable(struct drm_bridge *bridge)
+static void ws_bridge_bridge_disable(struct drm_bridge *bridge,
+				     struct drm_atomic_commit *commit)
 {
 	struct ws_bridge *ws = bridge_to_ws_bridge(bridge);
 
@@ -109,8 +112,11 @@ static void ws_bridge_bridge_disable(struct drm_bridge *bridge)
 }
 
 static const struct drm_bridge_funcs ws_bridge_bridge_funcs = {
-	.enable = ws_bridge_bridge_enable,
-	.disable = ws_bridge_bridge_disable,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+	.atomic_enable = ws_bridge_bridge_enable,
+	.atomic_disable = ws_bridge_bridge_disable,
 	.attach = ws_bridge_bridge_attach,
 };
 
diff --git a/drivers/gpu/drm/ci/arm64.config b/drivers/gpu/drm/ci/arm64.config
index 563a69669a7ba..c46125c1f80fe 100644
--- a/drivers/gpu/drm/ci/arm64.config
+++ b/drivers/gpu/drm/ci/arm64.config
@@ -78,7 +78,6 @@ CONFIG_INTERCONNECT_QCOM_MSM8996=y
 CONFIG_INTERCONNECT_QCOM_OSM_L3=y
 CONFIG_INTERCONNECT_QCOM_SC7180=y
 CONFIG_INTERCONNECT_QCOM_SM8350=y
-CONFIG_CRYPTO_DEV_QCOM_RNG=y
 CONFIG_SC_DISPCC_7180=y
 CONFIG_SC_GPUCC_7180=y
 CONFIG_SM_GPUCC_8350=y
@@ -189,6 +188,7 @@ CONFIG_GNSS=y
 CONFIG_GNSS_MTK_SERIAL=y
 CONFIG_HW_RANDOM=y
 CONFIG_HW_RANDOM_MTK=y
+CONFIG_HW_RANDOM_QCOM=y
 CONFIG_MTK_DEVAPC=y
 CONFIG_PWM_MTK_DISP=y
 CONFIG_MTK_CMDQ=y
diff --git a/drivers/gpu/drm/clients/drm_log.c b/drivers/gpu/drm/clients/drm_log.c
index 8d21b785bead6..e3e02c84a4cfe 100644
--- a/drivers/gpu/drm/clients/drm_log.c
+++ b/drivers/gpu/drm/clients/drm_log.c
@@ -122,10 +122,12 @@ static void drm_log_draw_line(struct drm_log_scanout *scanout, const char *s,
 	iosys_map_incr(&map, r.y1 * fb->pitches[0]);
 	for (i = 0; i < len && i < scanout->columns; i++) {
 		u32 color = (i < prefix_len) ? scanout->prefix_color : scanout->front_color;
-		src = drm_draw_get_char_bitmap(font, s[i], font_pitch);
-		drm_log_blit(&map, fb->pitches[0], src, font_pitch,
-			     scanout->scaled_font_h, scanout->scaled_font_w,
-			     px_width, color);
+		src = font_data_glyph_buf(font->data, font->width, font->height,
+					  (unsigned char)s[i]);
+		if (src)
+			drm_log_blit(&map, fb->pitches[0], src, font_pitch,
+				     scanout->scaled_font_h, scanout->scaled_font_w,
+				     px_width, color);
 		iosys_map_incr(&map, scanout->scaled_font_w * px_width);
 	}
 
diff --git a/drivers/gpu/drm/display/drm_bridge_connector.c b/drivers/gpu/drm/display/drm_bridge_connector.c
index 649969fca1413..8b54069fa53a4 100644
--- a/drivers/gpu/drm/display/drm_bridge_connector.c
+++ b/drivers/gpu/drm/display/drm_bridge_connector.c
@@ -147,7 +147,7 @@ static void drm_bridge_connector_hpd_notify(struct drm_connector *connector,
 		to_drm_bridge_connector(connector);
 
 	/* Notify all bridges in the pipeline of hotplug events. */
-	drm_for_each_bridge_in_chain_scoped(bridge_connector->encoder, bridge) {
+	drm_for_each_bridge_in_chain(bridge_connector->encoder, bridge) {
 		if (bridge->funcs->hpd_notify)
 			bridge->funcs->hpd_notify(bridge, connector, status);
 	}
@@ -265,26 +265,57 @@ static void drm_bridge_connector_debugfs_init(struct drm_connector *connector,
 	}
 }
 
-static void drm_bridge_connector_reset(struct drm_connector *connector)
+static struct drm_connector_state *
+drm_bridge_connector_create_state(struct drm_connector *connector)
 {
 	struct drm_bridge_connector *bridge_connector =
 		to_drm_bridge_connector(connector);
+	struct drm_connector_state *conn_state;
+
+	conn_state = drm_atomic_helper_connector_create_state(connector);
+	if (IS_ERR(conn_state))
+		return conn_state;
 
-	drm_atomic_helper_connector_reset(connector);
 	if (bridge_connector->bridge_hdmi)
-		__drm_atomic_helper_connector_hdmi_reset(connector,
-							 connector->state);
+		__drm_atomic_helper_connector_hdmi_state_init(connector,
+							      conn_state);
+
+	return conn_state;
+}
+
+static enum drm_connector_color_format
+drm_bridge_connector_color_format(const struct drm_connector_state *conn_state)
+{
+	struct drm_bridge_connector *bridge_connector =
+		to_drm_bridge_connector(conn_state->connector);
+
+	if (bridge_connector->bridge_hdmi) {
+		switch (conn_state->hdmi.output_format) {
+		default:
+		case DRM_OUTPUT_COLOR_FORMAT_RGB444:
+			return DRM_CONNECTOR_COLOR_FORMAT_RGB444;
+		case DRM_OUTPUT_COLOR_FORMAT_YCBCR444:
+			return DRM_CONNECTOR_COLOR_FORMAT_YCBCR444;
+		case DRM_OUTPUT_COLOR_FORMAT_YCBCR422:
+			return DRM_CONNECTOR_COLOR_FORMAT_YCBCR422;
+		case DRM_OUTPUT_COLOR_FORMAT_YCBCR420:
+			return DRM_CONNECTOR_COLOR_FORMAT_YCBCR420;
+		}
+	}
+
+	return conn_state->color_format;
 }
 
 static const struct drm_connector_funcs drm_bridge_connector_funcs = {
-	.reset = drm_bridge_connector_reset,
 	.detect = drm_bridge_connector_detect,
 	.force = drm_bridge_connector_force,
 	.fill_modes = drm_helper_probe_single_connector_modes,
+	.atomic_create_state = drm_bridge_connector_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 	.debugfs_init = drm_bridge_connector_debugfs_init,
 	.oob_hotplug_event = drm_bridge_connector_oob_hotplug_event,
+	.color_format = drm_bridge_connector_color_format,
 };
 
 /* -----------------------------------------------------------------------------
@@ -823,7 +854,7 @@ struct drm_connector *drm_bridge_connector_init(struct drm_device *drm,
 	 * detection are available, we don't support hotplug detection at all.
 	 */
 	connector_type = DRM_MODE_CONNECTOR_Unknown;
-	drm_for_each_bridge_in_chain_scoped(encoder, bridge) {
+	drm_for_each_bridge_in_chain(encoder, bridge) {
 		if (!bridge->interlace_allowed)
 			connector->interlace_allowed = false;
 		if (!bridge->ycbcr_420_allowed)
diff --git a/drivers/gpu/drm/display/drm_hdmi_state_helper.c b/drivers/gpu/drm/display/drm_hdmi_state_helper.c
index 4867edbf2622b..ce17eeefc2da4 100644
--- a/drivers/gpu/drm/display/drm_hdmi_state_helper.c
+++ b/drivers/gpu/drm/display/drm_hdmi_state_helper.c
@@ -306,17 +306,18 @@
  */
 
 /**
- * __drm_atomic_helper_connector_hdmi_reset() - Initializes all HDMI @drm_connector_state resources
+ * __drm_atomic_helper_connector_hdmi_state_init() - Initialize all HDMI @drm_connector_state resources
  * @connector: DRM connector
- * @new_conn_state: connector state to reset
+ * @new_conn_state: connector state to initialize
  *
  * Initializes all HDMI resources from a @drm_connector_state without
  * actually allocating it. This is useful for HDMI drivers, in
- * combination with __drm_atomic_helper_connector_reset() or
- * drm_atomic_helper_connector_reset().
+ * combination with __drm_atomic_helper_connector_state_init(),
+ * drm_atomic_helper_connector_reset(), or
+ * drm_atomic_helper_connector_create_state().
  */
-void __drm_atomic_helper_connector_hdmi_reset(struct drm_connector *connector,
-					      struct drm_connector_state *new_conn_state)
+void __drm_atomic_helper_connector_hdmi_state_init(struct drm_connector *connector,
+						   struct drm_connector_state *new_conn_state)
 {
 	unsigned int max_bpc = connector->max_bpc;
 
@@ -324,7 +325,7 @@ void __drm_atomic_helper_connector_hdmi_reset(struct drm_connector *connector,
 	new_conn_state->max_requested_bpc = max_bpc;
 	new_conn_state->hdmi.broadcast_rgb = DRM_HDMI_BROADCAST_RGB_AUTO;
 }
-EXPORT_SYMBOL(__drm_atomic_helper_connector_hdmi_reset);
+EXPORT_SYMBOL(__drm_atomic_helper_connector_hdmi_state_init);
 
 static enum hdmi_colorspace
 output_color_format_to_hdmi_colorspace(const struct drm_connector *connector,
@@ -541,10 +542,11 @@ sink_supports_format_bpc(const struct drm_connector *connector,
 		drm_dbg_kms(dev, "YUV444 format supported in that configuration.\n");
 
 		return true;
-	}
 
-	drm_dbg_kms(dev, "Unsupported pixel format.\n");
-	return false;
+	default:
+		drm_dbg_kms(dev, "Unsupported pixel format.\n");
+		return false;
+	}
 }
 
 static enum drm_mode_status
@@ -669,8 +671,39 @@ hdmi_compute_config(const struct drm_connector *connector,
 	unsigned int max_bpc = clamp_t(unsigned int,
 				       conn_state->max_bpc,
 				       8, connector->max_bpc);
+	enum drm_output_color_format fmt;
 	int ret;
 
+	if (conn_state->color_format != DRM_CONNECTOR_COLOR_FORMAT_AUTO) {
+		switch (conn_state->color_format) {
+		case DRM_CONNECTOR_COLOR_FORMAT_AUTO:
+			drm_warn(connector->dev, "AUTO format in non-AUTO path.\n");
+			fallthrough;
+		case DRM_CONNECTOR_COLOR_FORMAT_RGB444:
+			fmt = DRM_OUTPUT_COLOR_FORMAT_RGB444;
+			break;
+		case DRM_CONNECTOR_COLOR_FORMAT_YCBCR444:
+			fmt = DRM_OUTPUT_COLOR_FORMAT_YCBCR444;
+			break;
+		case DRM_CONNECTOR_COLOR_FORMAT_YCBCR422:
+			fmt = DRM_OUTPUT_COLOR_FORMAT_YCBCR422;
+			break;
+		case DRM_CONNECTOR_COLOR_FORMAT_YCBCR420:
+			fmt = DRM_OUTPUT_COLOR_FORMAT_YCBCR420;
+			break;
+		default:
+			drm_dbg_kms(connector->dev, "HDMI does not support color format '%d'.\n",
+				conn_state->color_format);
+			return -EINVAL;
+		}
+
+		return hdmi_compute_format_bpc(connector, conn_state, mode, max_bpc, fmt);
+	}
+
+	/*
+	 * For %DRM_CONNECTOR_COLOR_FORMAT_AUTO, try RGB first, and fall back
+	 * to the less bandwidth-intensive YCBCR420 if RGB fails.
+	 */
 	ret = hdmi_compute_format_bpc(connector, conn_state, mode, max_bpc,
 				      DRM_OUTPUT_COLOR_FORMAT_RGB444);
 	if (ret) {
@@ -910,8 +943,21 @@ drm_hdmi_connector_mode_valid(struct drm_connector *connector,
 			      const struct drm_display_mode *mode)
 {
 	unsigned long long clock;
+	enum drm_output_color_format fmt;
+
+	if (drm_mode_is_420_only(&connector->display_info, mode)) {
+		if (connector->ycbcr_420_allowed)
+			fmt = DRM_OUTPUT_COLOR_FORMAT_YCBCR420;
+		else
+			return MODE_NO_420;
+	} else if (drm_mode_is_420_also(&connector->display_info, mode) &&
+		   connector->ycbcr_420_allowed) {
+		fmt = DRM_OUTPUT_COLOR_FORMAT_YCBCR420;
+	} else {
+		fmt = DRM_OUTPUT_COLOR_FORMAT_RGB444;
+	}
 
-	clock = drm_hdmi_compute_mode_clock(mode, 8, DRM_OUTPUT_COLOR_FORMAT_RGB444);
+	clock = drm_hdmi_compute_mode_clock(mode, 8, fmt);
 	if (!clock)
 		return MODE_ERROR;
 
diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c
index 080aec5a97746..e5c8ef06caedd 100644
--- a/drivers/gpu/drm/drm_atomic.c
+++ b/drivers/gpu/drm/drm_atomic.c
@@ -47,6 +47,86 @@
 #include "drm_crtc_internal.h"
 #include "drm_internal.h"
 
+/**
+ * DOC: state lifetime
+ *
+ * &drm_atomic_commit represents an update to modeset pipeline state.
+ * It's a transient object that holds a state update as a collection of
+ * pointers to individual objects' states. &drm_atomic_commit has a much
+ * shorter lifetime than the objects' states, since it's only allocated
+ * while preparing, checking or committing the update, while object
+ * states are allocated when preparing the update and kept alive as long
+ * as they are active in the device.
+ *
+ * Their respective lifetimes are:
+ *
+ * - at driver initialization time, the driver calls
+ *   drm_mode_config_create_initial_state() to allocate an initial,
+ *   pristine, state for each object and stores it in the objects state
+ *   pointer. Historically, this was one of drm_mode_config_reset() job,
+ *   so one might still encounter it in a driver.
+ *
+ * - When resuming from suspend, drm_mode_config_reset() resets the
+ *   software and hardware state to a known default and stores it in the
+ *   object's state pointer. Not all objects are affected by
+ *   drm_mode_config_reset() though.
+ *
+ * - whenever a new update is needed:
+ *
+ *   + drm_atomic_commit_alloc() allocates a new &drm_atomic_commit
+ *     instance.
+ *
+ *   + The code triggering the commit (ioctl, client modeset,
+ *     drm_atomic_helper_reset_crtc(), etc.) copies the current active
+ *     state of all entities affected by the update into this new
+ *     &drm_atomic_commit using drm_atomic_get_plane_state(),
+ *     drm_atomic_get_crtc_state(), drm_atomic_get_connector_state(), or
+ *     drm_atomic_get_private_obj_state(). This new state can then be
+ *     modified.
+ *
+ *     At that point, &drm_atomic_commit stores three state pointers for
+ *     any affected entity: the "old" and "new" states, and
+ *     state_to_destroy. The old state is the state currently active in
+ *     the hardware, which is either the one initialized by reset() or a
+ *     newer one if a commit has been made. The new state is the state
+ *     we just allocated and we might eventually commit to the hardware.
+ *     The state_to_destroy points to the state we'll eventually have to
+ *     free when the drm_atomic_commit will be destroyed, and points to
+ *     the new state for now since the old state is still the active
+ *     state.
+ *
+ *   + After the calling code populated the commit with the entities
+ *     states, it updates the new states with the new values we need to
+ *     commit. The new commit instance is now ready.
+ *
+ *   + Then we have two branches depending on the calling code intent:
+ *
+ *     - If the calling code only wants to check that the commit would
+ *       work (for example because of the DRM_MODE_ATOMIC_TEST_ONLY
+ *       flag). It calls drm_atomic_check_only(), which in turn checks
+ *       all these states by invoking atomic_check on all affected
+ *       pipeline stages.
+ *
+ *     - If the calling code actually wants to trigger a commit, it
+ *       calls drm_atomic_commit(). The first stage is the check
+ *       mentioned above, and if the check is successful, it performs
+ *       the commit. Part of the commit is a call to
+ *       drm_atomic_helper_swap_state() which turns the new states into
+ *       the active states. After swapping states, each object's state
+ *       pointer now refers to the formerly new state. The
+ *       state_to_destroy now refers to the formerly old state.
+ *
+ *   + Once done, and when the last reference to our &drm_atomic_commit
+ *     is given up through drm_atomic_commit_put(), it calls
+ *     __drm_atomic_commit_free(). In turn, __drm_atomic_commit_free()
+ *     calls drm_atomic_commit_clear() that will free all
+ *     state_to_destroy (ie. old states), and it finally frees
+ *     &drm_atomic_commit instance.
+ *
+ *   + Now, we don't have any active &drm_atomic_commit anymore, and
+ *     only the entity active states remain allocated.
+ */
+
 void __drm_crtc_commit_free(struct kref *kref)
 {
 	struct drm_crtc_commit *commit =
@@ -1474,7 +1554,7 @@ drm_atomic_add_encoder_bridges(struct drm_atomic_commit *state,
 		       "Adding all bridges for [encoder:%d:%s] to %p\n",
 		       encoder->base.id, encoder->name, state);
 
-	drm_for_each_bridge_in_chain_scoped(encoder, bridge) {
+	drm_for_each_bridge_in_chain(encoder, bridge) {
 		/* Skip bridges that don't implement the atomic state hooks. */
 		if (!bridge->funcs->atomic_duplicate_state)
 			continue;
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index 51f39edc31edc..285aac3554dfd 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -28,6 +28,7 @@
 #include <linux/export.h>
 #include <linux/dma-fence.h>
 #include <linux/ktime.h>
+#include <linux/media-bus-format.h>
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
@@ -737,6 +738,11 @@ drm_atomic_helper_check_modeset(struct drm_device *dev,
 			if (old_connector_state->max_requested_bpc !=
 			    new_connector_state->max_requested_bpc)
 				new_crtc_state->connectors_changed = true;
+
+			if (old_connector_state->color_format !=
+			    new_connector_state->color_format)
+				new_crtc_state->connectors_changed = true;
+
 		}
 
 		if (funcs->atomic_check)
@@ -4102,3 +4108,83 @@ drm_atomic_helper_bridge_propagate_bus_fmt(struct drm_bridge *bridge,
 	return input_fmts;
 }
 EXPORT_SYMBOL(drm_atomic_helper_bridge_propagate_bus_fmt);
+
+/**
+ * drm_atomic_helper_bridge_get_hdmi_output_bus_fmts - helper implementing
+ *                                           atomic_get_output_bus_fmts for HDMI
+ * @bridge: pointer to &struct drm_bridge
+ * @bridge_state: pointer to the current bridge state
+ * @crtc_state: pointer to the current CRTC state
+ * @conn_state: pointer to the current connector state
+ * @num_output_fmts: pointer to where the number of entries in the returned array
+ *                   will be stored. Set to 0 if unsuccessful.
+ *
+ * Common implementation for the &drm_bridge_funcs.atomic_get_output_bus_fmts
+ * operation that's applicable to HDMI connectors.
+ *
+ * Returns: a newly allocated array of u32 values of length \*@num_output_fmts,
+ * representing all the MEDIA_BUS_FMTS\_ for the current connector state's
+ * chosen HDMI output bits per compoennt, or %NULL if it fails to allocate one.
+ */
+u32 *
+drm_atomic_helper_bridge_get_hdmi_output_bus_fmts(struct drm_bridge *bridge,
+					struct drm_bridge_state *bridge_state,
+					struct drm_crtc_state *crtc_state,
+					struct drm_connector_state *conn_state,
+					unsigned int *num_output_fmts)
+{
+	unsigned int num_fmts = 0;
+	u32 *out_fmts;
+
+	/*
+	 * bridge->supported_formats is a bit field of BIT(enum drm_output_color_format)
+	 * values. The smallest hweight that is smaller than or equal to
+	 * %DRM_OUTPUT_COLOR_FORMAT_COUNT will do for counting set bits here.
+	 */
+	BUILD_BUG_ON(const_true(DRM_OUTPUT_COLOR_FORMAT_COUNT > 8));
+	out_fmts = kmalloc_array(hweight8(bridge->supported_formats),
+				 sizeof(u32), GFP_KERNEL);
+	if (!out_fmts) {
+		*num_output_fmts = 0;
+		return NULL;
+	}
+
+	switch (conn_state->hdmi.output_bpc) {
+	case 12:
+		if (bridge->supported_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444))
+			out_fmts[num_fmts++] = MEDIA_BUS_FMT_RGB121212_1X36;
+		if (bridge->supported_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444))
+			out_fmts[num_fmts++] = MEDIA_BUS_FMT_YUV12_1X36;
+		if (bridge->supported_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422))
+			out_fmts[num_fmts++] = MEDIA_BUS_FMT_UYVY12_1X24;
+		if (bridge->supported_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420))
+			out_fmts[num_fmts++] = MEDIA_BUS_FMT_UYYVYY12_0_5X36;
+		break;
+	case 10:
+		if (bridge->supported_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444))
+			out_fmts[num_fmts++] = MEDIA_BUS_FMT_RGB101010_1X30;
+		if (bridge->supported_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444))
+			out_fmts[num_fmts++] = MEDIA_BUS_FMT_YUV10_1X30;
+		if (bridge->supported_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422))
+			out_fmts[num_fmts++] = MEDIA_BUS_FMT_UYVY10_1X20;
+		if (bridge->supported_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420))
+			out_fmts[num_fmts++] = MEDIA_BUS_FMT_UYYVYY10_0_5X30;
+		break;
+	default:
+		if (bridge->supported_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444))
+			out_fmts[num_fmts++] = MEDIA_BUS_FMT_RGB888_1X24;
+		if (bridge->supported_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444))
+			out_fmts[num_fmts++] = MEDIA_BUS_FMT_YUV8_1X24;
+		if (bridge->supported_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422))
+			out_fmts[num_fmts++] = MEDIA_BUS_FMT_UYVY8_1X16;
+		if (bridge->supported_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420))
+			out_fmts[num_fmts++] = MEDIA_BUS_FMT_UYYVYY8_0_5X24;
+		break;
+	}
+
+	*num_output_fmts = num_fmts;
+
+	return out_fmts;
+}
+EXPORT_SYMBOL(drm_atomic_helper_bridge_get_hdmi_output_bus_fmts);
+
diff --git a/drivers/gpu/drm/drm_atomic_state_helper.c b/drivers/gpu/drm/drm_atomic_state_helper.c
index cc70508d4fdba..d90d1d7c9cf9f 100644
--- a/drivers/gpu/drm/drm_atomic_state_helper.c
+++ b/drivers/gpu/drm/drm_atomic_state_helper.c
@@ -63,7 +63,7 @@
  */
 
 /**
- * __drm_atomic_helper_crtc_state_reset - reset the CRTC state
+ * __drm_atomic_helper_crtc_state_init - Initialize the CRTC state
  * @crtc_state: atomic CRTC state, must not be NULL
  * @crtc: CRTC object, must not be NULL
  *
@@ -71,13 +71,13 @@
  * values. This is useful for drivers that subclass the CRTC state.
  */
 void
-__drm_atomic_helper_crtc_state_reset(struct drm_crtc_state *crtc_state,
-				     struct drm_crtc *crtc)
+__drm_atomic_helper_crtc_state_init(struct drm_crtc_state *crtc_state,
+				    struct drm_crtc *crtc)
 {
 	crtc_state->crtc = crtc;
 	crtc_state->background_color = DRM_ARGB64_PREP(0xffff, 0, 0, 0);
 }
-EXPORT_SYMBOL(__drm_atomic_helper_crtc_state_reset);
+EXPORT_SYMBOL(__drm_atomic_helper_crtc_state_init);
 
 /**
  * __drm_atomic_helper_crtc_reset - reset state on CRTC
@@ -96,7 +96,7 @@ __drm_atomic_helper_crtc_reset(struct drm_crtc *crtc,
 			       struct drm_crtc_state *crtc_state)
 {
 	if (crtc_state)
-		__drm_atomic_helper_crtc_state_reset(crtc_state, crtc);
+		__drm_atomic_helper_crtc_state_init(crtc_state, crtc);
 
 	if (drm_dev_has_vblank(crtc->dev))
 		drm_crtc_vblank_reset(crtc);
@@ -124,6 +124,31 @@ void drm_atomic_helper_crtc_reset(struct drm_crtc *crtc)
 }
 EXPORT_SYMBOL(drm_atomic_helper_crtc_reset);
 
+/**
+ * drm_atomic_helper_crtc_create_state - default &drm_crtc_funcs.atomic_create_state hook for crtcs
+ * @crtc: crtc object
+ *
+ * Allocates and  initializes pristine @drm_crtc_state.
+ *
+ * This is useful for drivers that don't subclass @drm_crtc_state.
+ *
+ * RETURNS:
+ * Pointer to new crtc state, or ERR_PTR on failure.
+ */
+struct drm_crtc_state *drm_atomic_helper_crtc_create_state(struct drm_crtc *crtc)
+{
+	struct drm_crtc_state *state;
+
+	state = kzalloc_obj(*state);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_crtc_state_init(state, crtc);
+
+	return state;
+}
+EXPORT_SYMBOL(drm_atomic_helper_crtc_create_state);
+
 /**
  * __drm_atomic_helper_crtc_duplicate_state - copy atomic CRTC state
  * @crtc: CRTC object
@@ -237,15 +262,15 @@ void drm_atomic_helper_crtc_destroy_state(struct drm_crtc *crtc,
 EXPORT_SYMBOL(drm_atomic_helper_crtc_destroy_state);
 
 /**
- * __drm_atomic_helper_plane_state_reset - resets plane state to default values
+ * __drm_atomic_helper_plane_state_init - Initialize the plane state
  * @plane_state: atomic plane state, must not be NULL
  * @plane: plane object, must not be NULL
  *
  * Initializes the newly allocated @plane_state with default
- * values. This is useful for drivers that subclass the CRTC state.
+ * values. This is useful for drivers that subclass the plane state.
  */
-void __drm_atomic_helper_plane_state_reset(struct drm_plane_state *plane_state,
-					   struct drm_plane *plane)
+void __drm_atomic_helper_plane_state_init(struct drm_plane_state *plane_state,
+					  struct drm_plane *plane)
 {
 	u64 val;
 
@@ -297,7 +322,7 @@ void __drm_atomic_helper_plane_state_reset(struct drm_plane_state *plane_state,
 			plane_state->hotspot_y = val;
 	}
 }
-EXPORT_SYMBOL(__drm_atomic_helper_plane_state_reset);
+EXPORT_SYMBOL(__drm_atomic_helper_plane_state_init);
 
 /**
  * __drm_atomic_helper_plane_reset - reset state on plane
@@ -305,7 +330,7 @@ EXPORT_SYMBOL(__drm_atomic_helper_plane_state_reset);
  * @plane_state: plane state to assign
  *
  * Initializes the newly allocated @plane_state and assigns it to
- * the &drm_crtc->state pointer of @plane, usually required when
+ * the &drm_plane->state pointer of @plane, usually required when
  * initializing the drivers or when called from the &drm_plane_funcs.reset
  * hook.
  *
@@ -315,7 +340,7 @@ void __drm_atomic_helper_plane_reset(struct drm_plane *plane,
 				     struct drm_plane_state *plane_state)
 {
 	if (plane_state)
-		__drm_atomic_helper_plane_state_reset(plane_state, plane);
+		__drm_atomic_helper_plane_state_init(plane_state, plane);
 
 	plane->state = plane_state;
 }
@@ -340,6 +365,31 @@ void drm_atomic_helper_plane_reset(struct drm_plane *plane)
 }
 EXPORT_SYMBOL(drm_atomic_helper_plane_reset);
 
+/**
+ * drm_atomic_helper_plane_create_state - default &drm_plane_funcs.atomic_create_state hook for planes
+ * @plane: plane object
+ *
+ * Allocates and initializes pristine @drm_plane_state.
+ *
+ * This is useful for drivers that don't subclass @drm_plane_state.
+ *
+ * RETURNS:
+ * Pointer to new plane state, or ERR_PTR on failure.
+ */
+struct drm_plane_state *drm_atomic_helper_plane_create_state(struct drm_plane *plane)
+{
+	struct drm_plane_state *state;
+
+	state = kzalloc_obj(*state);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_plane_state_init(state, plane);
+
+	return state;
+}
+EXPORT_SYMBOL(drm_atomic_helper_plane_create_state);
+
 /**
  * __drm_atomic_helper_plane_duplicate_state - copy atomic plane state
  * @plane: plane object
@@ -426,20 +476,20 @@ void drm_atomic_helper_plane_destroy_state(struct drm_plane *plane,
 EXPORT_SYMBOL(drm_atomic_helper_plane_destroy_state);
 
 /**
- * __drm_atomic_helper_connector_state_reset - reset the connector state
+ * __drm_atomic_helper_connector_state_init - Initialize the connector state
  * @conn_state: atomic connector state, must not be NULL
- * @connector: connectotr object, must not be NULL
+ * @connector: connector object, must not be NULL
  *
  * Initializes the newly allocated @conn_state with default
  * values. This is useful for drivers that subclass the connector state.
  */
 void
-__drm_atomic_helper_connector_state_reset(struct drm_connector_state *conn_state,
-					  struct drm_connector *connector)
+__drm_atomic_helper_connector_state_init(struct drm_connector_state *conn_state,
+					 struct drm_connector *connector)
 {
 	conn_state->connector = connector;
 }
-EXPORT_SYMBOL(__drm_atomic_helper_connector_state_reset);
+EXPORT_SYMBOL(__drm_atomic_helper_connector_state_init);
 
 /**
  * __drm_atomic_helper_connector_reset - reset state on connector
@@ -458,7 +508,7 @@ __drm_atomic_helper_connector_reset(struct drm_connector *connector,
 				    struct drm_connector_state *conn_state)
 {
 	if (conn_state)
-		__drm_atomic_helper_connector_state_reset(conn_state, connector);
+		__drm_atomic_helper_connector_state_init(conn_state, connector);
 
 	connector->state = conn_state;
 }
@@ -484,6 +534,32 @@ void drm_atomic_helper_connector_reset(struct drm_connector *connector)
 }
 EXPORT_SYMBOL(drm_atomic_helper_connector_reset);
 
+/**
+ * drm_atomic_helper_connector_create_state - default &drm_connector_funcs.atomic_create_state hook for connectors
+ * @connector: connector object
+ *
+ * Allocates and  initializes pristine @drm_connector_state.
+ *
+ * This is useful for drivers that don't subclass @drm_connector_state.
+ *
+ * RETURNS:
+ * Pointer to new connector state, or ERR_PTR on failure.
+ */
+struct drm_connector_state *
+drm_atomic_helper_connector_create_state(struct drm_connector *connector)
+{
+	struct drm_connector_state *state;
+
+	state = kzalloc_obj(*state);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_connector_state_init(state, connector);
+
+	return state;
+}
+EXPORT_SYMBOL(drm_atomic_helper_connector_create_state);
+
 /**
  * drm_atomic_helper_connector_tv_margins_reset - Resets TV connector properties
  * @connector: DRM connector
@@ -731,8 +807,6 @@ void __drm_atomic_helper_private_obj_create_state(struct drm_private_obj *obj,
 {
 	if (state)
 		state->obj = obj;
-
-	obj->state = state;
 }
 EXPORT_SYMBOL(__drm_atomic_helper_private_obj_create_state);
 
@@ -799,7 +873,7 @@ EXPORT_SYMBOL(drm_atomic_helper_bridge_duplicate_state);
  * @state: bridge state to destroy
  *
  * Destroys a bridge state previously created by
- * &drm_atomic_helper_bridge_reset() or
+ * &drm_atomic_helper_bridge_create_state() or
  * &drm_atomic_helper_bridge_duplicate_state(). This helper is meant to be
  * used as a bridge &drm_bridge_funcs.atomic_destroy_state hook for bridges
  * that don't subclass the bridge state.
@@ -812,35 +886,39 @@ void drm_atomic_helper_bridge_destroy_state(struct drm_bridge *bridge,
 EXPORT_SYMBOL(drm_atomic_helper_bridge_destroy_state);
 
 /**
- * __drm_atomic_helper_bridge_reset() - Initialize a bridge state to its
+ * __drm_atomic_helper_bridge_state_init() - Initialize a bridge state to its
  *					default
- * @bridge: the bridge this state refers to
  * @state: bridge state to initialize
+ * @bridge: the bridge this state refers to
+ *
+ * @state is assumed to be zeroed.
  *
  * Initializes the bridge state to default values. This is meant to be called
- * by the bridge &drm_bridge_funcs.atomic_reset hook for bridges that subclass
- * the bridge state.
+ * by the bridge &drm_bridge_funcs.atomic_create_state hook for bridges that
+ * subclass the bridge state.
  */
-void __drm_atomic_helper_bridge_reset(struct drm_bridge *bridge,
-				      struct drm_bridge_state *state)
+void __drm_atomic_helper_bridge_state_init(struct drm_bridge_state *state,
+					   struct drm_bridge *bridge)
 {
-	memset(state, 0, sizeof(*state));
 	__drm_atomic_helper_private_obj_create_state(&bridge->base, &state->base);
 	state->bridge = bridge;
 }
-EXPORT_SYMBOL(__drm_atomic_helper_bridge_reset);
+EXPORT_SYMBOL(__drm_atomic_helper_bridge_state_init);
 
 /**
- * drm_atomic_helper_bridge_reset() - Allocate and initialize a bridge state
- *				      to its default
- * @bridge: the bridge this state refers to
+ * drm_atomic_helper_bridge_create_state - default
+ *              &drm_bridge_funcs.atomic_create_state hook for bridges
+ * @bridge: bridge object
+ *
+ * Allocates and initializes pristine @drm_bridge_state.
+ *
+ * This is useful for drivers that don't subclass @drm_bridge_state.
  *
- * Allocates the bridge state and initializes it to default values. This helper
- * is meant to be used as a bridge &drm_bridge_funcs.atomic_reset hook for
- * bridges that don't subclass the bridge state.
+ * RETURNS:
+ * Pointer to new bridge state, or ERR_PTR on failure.
  */
 struct drm_bridge_state *
-drm_atomic_helper_bridge_reset(struct drm_bridge *bridge)
+drm_atomic_helper_bridge_create_state(struct drm_bridge *bridge)
 {
 	struct drm_bridge_state *bridge_state;
 
@@ -848,7 +926,7 @@ drm_atomic_helper_bridge_reset(struct drm_bridge *bridge)
 	if (!bridge_state)
 		return ERR_PTR(-ENOMEM);
 
-	__drm_atomic_helper_bridge_reset(bridge, bridge_state);
+	__drm_atomic_helper_bridge_state_init(bridge_state, bridge);
 	return bridge_state;
 }
-EXPORT_SYMBOL(drm_atomic_helper_bridge_reset);
+EXPORT_SYMBOL(drm_atomic_helper_bridge_create_state);
diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c
index e997917819e8e..1050dddadb17e 100644
--- a/drivers/gpu/drm/drm_atomic_uapi.c
+++ b/drivers/gpu/drm/drm_atomic_uapi.c
@@ -960,6 +960,8 @@ static int drm_atomic_connector_set_property(struct drm_connector *connector,
 		state->privacy_screen_sw_state = val;
 	} else if (property == connector->broadcast_rgb_property) {
 		state->hdmi.broadcast_rgb = val;
+	} else if (property == connector->color_format_property) {
+		state->color_format = val;
 	} else if (connector->funcs->atomic_set_property) {
 		return connector->funcs->atomic_set_property(connector,
 				state, property, val);
@@ -1045,6 +1047,8 @@ drm_atomic_connector_get_property(struct drm_connector *connector,
 		*val = state->privacy_screen_sw_state;
 	} else if (property == connector->broadcast_rgb_property) {
 		*val = state->hdmi.broadcast_rgb;
+	} else if (property == connector->color_format_property) {
+		*val = state->color_format;
 	} else if (connector->funcs->atomic_get_property) {
 		return connector->funcs->atomic_get_property(connector,
 				state, property, val);
diff --git a/drivers/gpu/drm/drm_blend.c b/drivers/gpu/drm/drm_blend.c
index 2f0d1ba285be8..d17ae964fb21f 100644
--- a/drivers/gpu/drm/drm_blend.c
+++ b/drivers/gpu/drm/drm_blend.c
@@ -563,10 +563,10 @@ EXPORT_SYMBOL(drm_atomic_normalize_zpos);
 /**
  * drm_plane_create_blend_mode_property - create a new blend mode property
  * @plane: drm plane
- * @supported_modes: bitmask of supported modes, must include
- *		     BIT(DRM_MODE_BLEND_PREMULTI). Current DRM assumption is
- *		     that alpha is premultiplied, and old userspace can break if
- *		     the property defaults to anything else.
+ * @supported_modes: bitmask of supported modes. When
+ *		     BIT(DRM_MODE_BLEND_PREMULTI) is included, it will be used
+ *		     as the default. Otherwise, the default will fallback to one
+ *		     of the supported modes.
  *
  * This creates a new property describing the blend mode.
  *
@@ -599,13 +599,14 @@ int drm_plane_create_blend_mode_property(struct drm_plane *plane,
 		{ DRM_MODE_BLEND_PREMULTI, "Pre-multiplied" },
 		{ DRM_MODE_BLEND_COVERAGE, "Coverage" },
 	};
+	unsigned int default_mode;
 	unsigned int valid_mode_mask = BIT(DRM_MODE_BLEND_PIXEL_NONE) |
 				       BIT(DRM_MODE_BLEND_PREMULTI)   |
 				       BIT(DRM_MODE_BLEND_COVERAGE);
 	int i;
 
 	if (WARN_ON((supported_modes & ~valid_mode_mask) ||
-		    ((supported_modes & BIT(DRM_MODE_BLEND_PREMULTI)) == 0)))
+		    (supported_modes == 0)))
 		return -EINVAL;
 
 	prop = drm_property_create(dev, DRM_MODE_PROP_ENUM,
@@ -630,7 +631,14 @@ int drm_plane_create_blend_mode_property(struct drm_plane *plane,
 		}
 	}
 
-	drm_object_attach_property(&plane->base, prop, DRM_MODE_BLEND_PREMULTI);
+	if (supported_modes & BIT(DRM_MODE_BLEND_PREMULTI))
+		default_mode = DRM_MODE_BLEND_PREMULTI;
+	else if (supported_modes & BIT(DRM_MODE_BLEND_COVERAGE))
+		default_mode = DRM_MODE_BLEND_COVERAGE;
+	else
+		default_mode = DRM_MODE_BLEND_PIXEL_NONE;
+
+	drm_object_attach_property(&plane->base, prop, default_mode);
 	plane->blend_mode_property = prop;
 
 	return 0;
diff --git a/drivers/gpu/drm/drm_bridge.c b/drivers/gpu/drm/drm_bridge.c
index 687b36eea0c70..83f1809a5d376 100644
--- a/drivers/gpu/drm/drm_bridge.c
+++ b/drivers/gpu/drm/drm_bridge.c
@@ -198,6 +198,46 @@
  * driver.
  */
 
+/**
+ * DOC: bridge chain format selection
+ *
+ * A bridge chain, from display output processor to connector, may contain
+ * bridges capable of converting between bus formats on their inputs, and
+ * output formats on their outputs. For example, a bridge may be able to convert
+ * from RGB to YCbCr 4:4:4, and pass through YCbCr 4:2:0 as-is, but not convert
+ * from RGB to YCbCr 4:2:0. This means not all input formats map to all output
+ * formats.
+ *
+ * Further adding to this, a desired output color format, as specified with the
+ * "color format" DRM property, might not correspond 1:1 to what the display
+ * driver should set at its output. The bridge chain it feeds into may only be
+ * able to reach the desired output format, if a conversion from a different
+ * starting format is performed.
+ *
+ * To deal with this complexity, the recursive bridge chain bus format selection
+ * logic starts with the last bridge in the chain, usually the connector, and
+ * then recursively walks the chain of bridges backwards to the first bridge,
+ * trying to find a path.
+ *
+ * For a display driver to work in such a scenario, it should read the first
+ * bridge's bridge state to figure out which bus format the chain resolved to.
+ * If the first bridge's input format resolved to %MEDIA_BUS_FMT_FIXED, then its
+ * output format should be used.
+ *
+ * Special handling is done for HDMI as it relates to format selection. Instead
+ * of directly using the "color format" DRM property for bridge chains that end
+ * in HDMI bridges, the bridge chain format selection logic will trust the logic
+ * that set the HDMI output format. For the common HDMI state helper
+ * functionality, this means that %DRM_CONNECTOR_COLOR_FORMAT_AUTO will allow
+ * fallbacks to YCBCr 4:2:0 if the bandwidth requirements would otherwise be too
+ * high but the mode and connector allow it.
+ *
+ * For bridge chains that do not end in an HDMI bridge,
+ * %DRM_CONNECTOR_COLOR_FORMAT_AUTO will be satisfied with the first output
+ * format on the last bridge for which it can find a path back to the first
+ * bridge.
+ */
+
 /* Protect bridge_list and bridge_lingering_list */
 static DEFINE_MUTEX(bridge_lock);
 static LIST_HEAD(bridge_list);
@@ -417,6 +457,7 @@ void drm_bridge_add(struct drm_bridge *bridge)
 	if (!list_empty(&bridge->list))
 		list_del_init(&bridge->list);
 
+	mutex_init(&bridge->hpd_state_mutex);
 	mutex_init(&bridge->hpd_mutex);
 
 	if (bridge->ops & DRM_BRIDGE_OP_HDMI)
@@ -469,6 +510,7 @@ void drm_bridge_remove(struct drm_bridge *bridge)
 	mutex_unlock(&bridge_lock);
 
 	mutex_destroy(&bridge->hpd_mutex);
+	mutex_destroy(&bridge->hpd_state_mutex);
 
 	drm_bridge_put(bridge);
 }
@@ -500,7 +542,7 @@ drm_bridge_atomic_create_priv_state(struct drm_private_obj *obj)
 	struct drm_bridge *bridge = drm_priv_to_bridge(obj);
 	struct drm_bridge_state *state;
 
-	state = bridge->funcs->atomic_reset(bridge);
+	state = bridge->funcs->atomic_create_state(bridge);
 	if (IS_ERR(state))
 		return ERR_CAST(state);
 
@@ -513,11 +555,6 @@ static const struct drm_private_state_funcs drm_bridge_priv_state_funcs = {
 	.atomic_destroy_state = drm_bridge_atomic_destroy_priv_state,
 };
 
-static bool drm_bridge_is_atomic(struct drm_bridge *bridge)
-{
-	return bridge->funcs->atomic_reset != NULL;
-}
-
 /**
  * drm_bridge_attach - attach the bridge to an encoder's chain
  *
@@ -587,9 +624,8 @@ int drm_bridge_attach(struct drm_encoder *encoder, struct drm_bridge *bridge,
 			goto err_reset_bridge;
 	}
 
-	if (drm_bridge_is_atomic(bridge))
-		drm_atomic_private_obj_init(bridge->dev, &bridge->base,
-					    &drm_bridge_priv_state_funcs);
+	drm_atomic_private_obj_init(bridge->dev, &bridge->base,
+				    &drm_bridge_priv_state_funcs);
 
 	return 0;
 
@@ -622,8 +658,7 @@ void drm_bridge_detach(struct drm_bridge *bridge)
 	if (WARN_ON(!bridge->dev))
 		return;
 
-	if (drm_bridge_is_atomic(bridge))
-		drm_atomic_private_obj_fini(&bridge->base);
+	drm_atomic_private_obj_fini(&bridge->base);
 
 	if (bridge->funcs->detach)
 		bridge->funcs->detach(bridge);
@@ -648,9 +683,9 @@ void drm_bridge_detach(struct drm_bridge *bridge)
  *   disable the bridge automatically.
  *
  *   The enable and disable operations are split in
- *   &drm_bridge_funcs.pre_enable, &drm_bridge_funcs.enable,
- *   &drm_bridge_funcs.disable and &drm_bridge_funcs.post_disable to provide
- *   finer-grained control.
+ *   &drm_bridge_funcs.atomic_pre_enable, &drm_bridge_funcs.atomic_enable,
+ *   &drm_bridge_funcs.atomic_disable and &drm_bridge_funcs.atomic_post_disable
+ *   to provide finer-grained control.
  *
  *   Bridge drivers may implement the legacy version of those operations, or
  *   the atomic version (prefixed with atomic\_), in which case they shall also
@@ -791,10 +826,9 @@ EXPORT_SYMBOL(drm_bridge_chain_mode_set);
  * @bridge: bridge control structure
  * @state: atomic state being committed
  *
- * Calls &drm_bridge_funcs.atomic_disable (falls back on
- * &drm_bridge_funcs.disable) op for all the bridges in the encoder chain,
- * starting from the last bridge to the first. These are called before calling
- * &drm_encoder_helper_funcs.atomic_disable
+ * Calls &drm_bridge_funcs.atomic_disable op for all the bridges in the encoder
+ * chain, starting from the last bridge to the first. These are called before
+ * calling &drm_encoder_helper_funcs.atomic_disable
  *
  * Note: the bridge passed should be the one closest to the encoder
  */
@@ -810,11 +844,8 @@ void drm_atomic_bridge_chain_disable(struct drm_bridge *bridge,
 	encoder = bridge->encoder;
 	mutex_lock(&encoder->bridge_chain_mutex);
 	list_for_each_entry_reverse(iter, &encoder->bridge_chain, chain_node) {
-		if (iter->funcs->atomic_disable) {
+		if (iter->funcs->atomic_disable)
 			iter->funcs->atomic_disable(iter, state);
-		} else if (iter->funcs->disable) {
-			iter->funcs->disable(iter);
-		}
 
 		if (iter == bridge)
 			break;
@@ -828,8 +859,6 @@ static void drm_atomic_bridge_call_post_disable(struct drm_bridge *bridge,
 {
 	if (state && bridge->funcs->atomic_post_disable)
 		bridge->funcs->atomic_post_disable(bridge, state);
-	else if (bridge->funcs->post_disable)
-		bridge->funcs->post_disable(bridge);
 }
 
 /**
@@ -838,10 +867,9 @@ static void drm_atomic_bridge_call_post_disable(struct drm_bridge *bridge,
  * @bridge: bridge control structure
  * @state: atomic state being committed
  *
- * Calls &drm_bridge_funcs.atomic_post_disable (falls back on
- * &drm_bridge_funcs.post_disable) op for all the bridges in the encoder chain,
- * starting from the first bridge to the last. These are called after completing
- * &drm_encoder_helper_funcs.atomic_disable
+ * Calls &drm_bridge_funcs.atomic_post_disable op for all the bridges in the
+ * encoder chain, starting from the first bridge to the last. These are called
+ * after completing &drm_encoder_helper_funcs.atomic_disable
  *
  * If a bridge sets @pre_enable_prev_first, then the @post_disable for that
  * bridge will be called before the previous one to reverse the @pre_enable
@@ -925,8 +953,6 @@ static void drm_atomic_bridge_call_pre_enable(struct drm_bridge *bridge,
 {
 	if (state && bridge->funcs->atomic_pre_enable)
 		bridge->funcs->atomic_pre_enable(bridge, state);
-	else if (bridge->funcs->pre_enable)
-		bridge->funcs->pre_enable(bridge);
 }
 
 /**
@@ -935,10 +961,9 @@ static void drm_atomic_bridge_call_pre_enable(struct drm_bridge *bridge,
  * @bridge: bridge control structure
  * @state: atomic state being committed
  *
- * Calls &drm_bridge_funcs.atomic_pre_enable (falls back on
- * &drm_bridge_funcs.pre_enable) op for all the bridges in the encoder chain,
- * starting from the last bridge to the first. These are called before calling
- * &drm_encoder_helper_funcs.atomic_enable
+ * Calls &drm_bridge_funcs.atomic_pre_enable op for all the bridges in the
+ * encoder chain, starting from the last bridge to the first. These are called
+ * before calling &drm_encoder_helper_funcs.atomic_enable
  *
  * If a bridge sets @pre_enable_prev_first, then the pre_enable for the
  * prev bridge will be called before pre_enable of this bridge.
@@ -1015,10 +1040,9 @@ EXPORT_SYMBOL(drm_atomic_bridge_chain_pre_enable);
  * @first_bridge: bridge control structure
  * @state: atomic state being committed
  *
- * Calls &drm_bridge_funcs.atomic_enable (falls back on
- * &drm_bridge_funcs.enable) op for all the bridges in the encoder chain,
- * starting from the first bridge to the last. These are called after completing
- * &drm_encoder_helper_funcs.atomic_enable
+ * Calls &drm_bridge_funcs.atomic_enable op for all the bridges in the encoder
+ * chain, starting from the first bridge to the last. These are called after
+ * completing &drm_encoder_helper_funcs.atomic_enable
  *
  * Note: the bridge passed should be the one closest to the encoder
  */
@@ -1029,11 +1053,8 @@ void drm_atomic_bridge_chain_enable(struct drm_bridge *first_bridge,
 		return;
 
 	drm_for_each_bridge_in_chain_from(first_bridge, bridge)
-		if (bridge->funcs->atomic_enable) {
+		if (bridge->funcs->atomic_enable)
 			bridge->funcs->atomic_enable(bridge, state);
-		} else if (bridge->funcs->enable) {
-			bridge->funcs->enable(bridge);
-		}
 }
 EXPORT_SYMBOL(drm_atomic_bridge_chain_enable);
 
@@ -1149,6 +1170,47 @@ static int select_bus_fmt_recursive(struct drm_bridge *first_bridge,
 	return ret;
 }
 
+static bool __pure bus_format_is_color_fmt(u32 bus_fmt, enum drm_connector_color_format fmt)
+{
+	if (fmt == DRM_CONNECTOR_COLOR_FORMAT_AUTO)
+		return true;
+
+	switch (bus_fmt) {
+	case MEDIA_BUS_FMT_FIXED:
+		return true;
+	case MEDIA_BUS_FMT_RGB888_1X24:
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+	case MEDIA_BUS_FMT_RGB121212_1X36:
+	case MEDIA_BUS_FMT_RGB161616_1X48:
+		return fmt == DRM_CONNECTOR_COLOR_FORMAT_RGB444;
+	case MEDIA_BUS_FMT_YUV8_1X24:
+	case MEDIA_BUS_FMT_YUV10_1X30:
+	case MEDIA_BUS_FMT_YUV12_1X36:
+	case MEDIA_BUS_FMT_YUV16_1X48:
+		return fmt == DRM_CONNECTOR_COLOR_FORMAT_YCBCR444;
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+	case MEDIA_BUS_FMT_VYUY8_1X16:
+	case MEDIA_BUS_FMT_YUYV8_1X16:
+	case MEDIA_BUS_FMT_YVYU8_1X16:
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+	case MEDIA_BUS_FMT_YUYV10_1X20:
+	case MEDIA_BUS_FMT_VYUY10_1X20:
+	case MEDIA_BUS_FMT_YVYU10_1X20:
+	case MEDIA_BUS_FMT_UYVY12_1X24:
+	case MEDIA_BUS_FMT_VYUY12_1X24:
+	case MEDIA_BUS_FMT_YUYV12_1X24:
+	case MEDIA_BUS_FMT_YVYU12_1X24:
+		return fmt == DRM_CONNECTOR_COLOR_FORMAT_YCBCR422;
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+	case MEDIA_BUS_FMT_UYYVYY12_0_5X36:
+	case MEDIA_BUS_FMT_UYYVYY16_0_5X48:
+		return fmt == DRM_CONNECTOR_COLOR_FORMAT_YCBCR420;
+	default:
+		return false;
+	}
+}
+
 /*
  * This function is called by &drm_atomic_bridge_chain_check() just before
  * calling &drm_bridge_funcs.atomic_check() on all elements of the chain.
@@ -1192,6 +1254,7 @@ drm_atomic_bridge_chain_select_bus_fmts(struct drm_bridge *bridge,
 	struct drm_encoder *encoder = bridge->encoder;
 	struct drm_bridge_state *last_bridge_state;
 	unsigned int i, num_out_bus_fmts = 0;
+	enum drm_connector_color_format fmt;
 	u32 *out_bus_fmts;
 	int ret = 0;
 
@@ -1233,11 +1296,31 @@ drm_atomic_bridge_chain_select_bus_fmts(struct drm_bridge *bridge,
 			out_bus_fmts[0] = MEDIA_BUS_FMT_FIXED;
 	}
 
+	/*
+	 * Instead of directly accessing conn_state.color_format, call into a
+	 * connector function that allows connector implementations (e.g. for
+	 * bridge connectors including HDMI bridges, where the HDMI helpers will
+	 * have already chosen an appropriate output format) to override the
+	 * selected format.
+	 */
+	fmt = drm_connector_get_color_format(conn_state);
+
 	for (i = 0; i < num_out_bus_fmts; i++) {
+		if (!bus_format_is_color_fmt(out_bus_fmts[i], fmt)) {
+			drm_dbg_kms(last_bridge->dev,
+				    "Skipping bus format 0x%04x as it doesn't match format %d\n",
+				    out_bus_fmts[i], fmt);
+			ret = -ENOTSUPP;
+			continue;
+		}
 		ret = select_bus_fmt_recursive(bridge, last_bridge, crtc_state,
 					       conn_state, out_bus_fmts[i]);
-		if (ret != -ENOTSUPP)
+		if (ret != -ENOTSUPP) {
+			drm_dbg_kms(last_bridge->dev,
+				    "Found bridge chain ending with bus format 0x%04x\n",
+				    out_bus_fmts[i]);
 			break;
+		}
 	}
 
 	kfree(out_bus_fmts);
@@ -1451,19 +1534,25 @@ void drm_bridge_hpd_enable(struct drm_bridge *bridge,
 	if (!(bridge->ops & DRM_BRIDGE_OP_HPD))
 		return;
 
+	mutex_lock(&bridge->hpd_state_mutex);
+
 	mutex_lock(&bridge->hpd_mutex);
 
-	if (WARN(bridge->hpd_cb, "Hot plug detection already enabled\n"))
+	if (WARN(bridge->hpd_cb, "Hot plug detection already enabled\n")) {
+		mutex_unlock(&bridge->hpd_mutex);
 		goto unlock;
+	}
 
 	bridge->hpd_cb = cb;
 	bridge->hpd_data = data;
 
+	mutex_unlock(&bridge->hpd_mutex);
+
 	if (bridge->funcs->hpd_enable)
 		bridge->funcs->hpd_enable(bridge);
 
 unlock:
-	mutex_unlock(&bridge->hpd_mutex);
+	mutex_unlock(&bridge->hpd_state_mutex);
 }
 EXPORT_SYMBOL_GPL(drm_bridge_hpd_enable);
 
@@ -1484,13 +1573,15 @@ void drm_bridge_hpd_disable(struct drm_bridge *bridge)
 	if (!(bridge->ops & DRM_BRIDGE_OP_HPD))
 		return;
 
-	mutex_lock(&bridge->hpd_mutex);
+	mutex_lock(&bridge->hpd_state_mutex);
 	if (bridge->funcs->hpd_disable)
 		bridge->funcs->hpd_disable(bridge);
 
+	mutex_lock(&bridge->hpd_mutex);
 	bridge->hpd_cb = NULL;
 	bridge->hpd_data = NULL;
 	mutex_unlock(&bridge->hpd_mutex);
+	mutex_unlock(&bridge->hpd_state_mutex);
 }
 EXPORT_SYMBOL_GPL(drm_bridge_hpd_disable);
 
@@ -1605,7 +1696,7 @@ struct drm_bridge *of_drm_get_bridge_by_endpoint(const struct device_node *np,
 	/*
 	 * of_graph_get_remote_node() produces a noisy error message if port
 	 * node isn't found and the absence of the port is a legit case here,
-	 * so at first we silently check whether graph is present in the
+	 * so at first we silently check whether a graph is present in the
 	 * device-tree node.
 	 */
 	if (!of_graph_is_present(np))
@@ -1705,7 +1796,7 @@ static int encoder_bridges_show(struct seq_file *m, void *data)
 	struct drm_printer p = drm_seq_file_printer(m);
 	unsigned int idx = 0;
 
-	drm_for_each_bridge_in_chain_scoped(encoder, bridge)
+	drm_for_each_bridge_in_chain(encoder, bridge)
 		drm_bridge_debugfs_show_bridge(&p, bridge, idx++, false, true);
 
 	return 0;
diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c
index faa025498de45..1536e59c6fe71 100644
--- a/drivers/gpu/drm/drm_buddy.c
+++ b/drivers/gpu/drm/drm_buddy.c
@@ -47,29 +47,23 @@ void drm_buddy_print(struct gpu_buddy *mm, struct drm_printer *p)
 		   mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20, mm->clear_avail >> 20);
 
 	for (order = mm->max_order; order >= 0; order--) {
-		struct gpu_buddy_block *block, *tmp;
-		struct rb_root *root;
-		u64 count = 0, free;
-		unsigned int tree;
-
-		for_each_free_tree(tree) {
-			root = &mm->free_trees[tree][order];
-
-			rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) {
-				BUG_ON(!gpu_buddy_block_is_free(block));
-				count++;
-			}
-		}
+		u64 free_count = mm->free_scoreboard[order];
+		u64 used_count = mm->used_scoreboard[order];
+		u64 block_size = mm->chunk_size << order;
+		u64 free = free_count * block_size;
+		u64 used = used_count * block_size;
 
 		drm_printf(p, "order-%2d ", order);
 
-		free = count * (mm->chunk_size << order);
-		if (free < SZ_1M)
-			drm_printf(p, "free: %8llu KiB", free >> 10);
+		if (block_size < SZ_1M)
+			drm_printf(p, "free: %8llu KiB, used: %8llu KiB",
+				   free >> 10, used >> 10);
 		else
-			drm_printf(p, "free: %8llu MiB", free >> 20);
+			drm_printf(p, "free: %8llu MiB, used: %8llu MiB",
+				   free >> 20, used >> 20);
 
-		drm_printf(p, ", blocks: %llu\n", count);
+		drm_printf(p, ", free_blocks: %llu, used_blocks: %llu\n",
+			   free_count, used_count);
 	}
 }
 EXPORT_SYMBOL(drm_buddy_print);
diff --git a/drivers/gpu/drm/drm_cache.c b/drivers/gpu/drm/drm_cache.c
index ea1d2d5d2c66c..2eed55b46e298 100644
--- a/drivers/gpu/drm/drm_cache.c
+++ b/drivers/gpu/drm/drm_cache.c
@@ -88,7 +88,7 @@ drm_clflush_pages(struct page *pages[], unsigned long num_pages)
 {
 
 #if defined(CONFIG_X86)
-	if (static_cpu_has(X86_FEATURE_CLFLUSH)) {
+	if (cpu_feature_enabled(X86_FEATURE_CLFLUSH)) {
 		drm_cache_flush_clflush(pages, num_pages);
 		return;
 	}
@@ -127,7 +127,7 @@ void
 drm_clflush_sg(struct sg_table *st)
 {
 #if defined(CONFIG_X86)
-	if (static_cpu_has(X86_FEATURE_CLFLUSH)) {
+	if (cpu_feature_enabled(X86_FEATURE_CLFLUSH)) {
 		struct sg_page_iter sg_iter;
 
 		mb(); /*CLFLUSH is ordered only by using memory barriers*/
@@ -157,7 +157,7 @@ void
 drm_clflush_virt_range(void *addr, unsigned long length)
 {
 #if defined(CONFIG_X86)
-	if (static_cpu_has(X86_FEATURE_CLFLUSH)) {
+	if (cpu_feature_enabled(X86_FEATURE_CLFLUSH)) {
 		const int size = boot_cpu_data.x86_clflush_size;
 		void *end = addr + length;
 
@@ -332,7 +332,7 @@ void drm_memcpy_init_early(void)
 	 * Some hypervisors (e.g. KVM) don't support VEX-prefix instructions
 	 * emulation. So don't enable movntdqa in hypervisor guest.
 	 */
-	if (static_cpu_has(X86_FEATURE_XMM4_1) &&
+	if (cpu_feature_enabled(X86_FEATURE_XMM4_1) &&
 	    !boot_cpu_has(X86_FEATURE_HYPERVISOR))
 		static_branch_enable(&has_movntdqa);
 }
diff --git a/drivers/gpu/drm/drm_colorop.c b/drivers/gpu/drm/drm_colorop.c
index 65fec53f70fa6..682fcc6515259 100644
--- a/drivers/gpu/drm/drm_colorop.c
+++ b/drivers/gpu/drm/drm_colorop.c
@@ -498,15 +498,15 @@ void drm_colorop_atomic_destroy_state(struct drm_colorop *colorop,
 }
 
 /**
- * __drm_colorop_state_reset - resets colorop state to default values
+ * __drm_colorop_state_init - Initializes colorop state to default values
  * @colorop_state: atomic colorop state, must not be NULL
  * @colorop: colorop object, must not be NULL
  *
  * Initializes the newly allocated @colorop_state with default
- * values. This is useful for drivers that subclass the CRTC state.
+ * values. This is useful for drivers that subclass the colorop state.
  */
-static void __drm_colorop_state_reset(struct drm_colorop_state *colorop_state,
-				      struct drm_colorop *colorop)
+static void __drm_colorop_state_init(struct drm_colorop_state *colorop_state,
+				     struct drm_colorop *colorop)
 {
 	u64 val;
 
@@ -535,15 +535,38 @@ static void __drm_colorop_state_reset(struct drm_colorop_state *colorop_state,
 	}
 }
 
+/**
+ * drm_atomic_helper_colorop_create_state - Allocates and initializes colorop atomic state
+ * @colorop: drm colorop
+ *
+ * Initializes a pristine @drm_colorop_state.
+ *
+ * RETURNS:
+ * Pointer to new colorop state, or ERR_PTR on failure.
+ */
+struct drm_colorop_state *
+drm_atomic_helper_colorop_create_state(struct drm_colorop *colorop)
+{
+	struct drm_colorop_state *state;
+
+	state = kzalloc_obj(*state);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_colorop_state_init(state, colorop);
+
+	return state;
+}
+
 /**
  * __drm_colorop_reset - reset state on colorop
  * @colorop: drm colorop
  * @colorop_state: colorop state to assign
  *
- * Initializes the newly allocated @colorop_state and assigns it to
- * the &drm_crtc->state pointer of @colorop, usually required when
- * initializing the drivers or when called from the &drm_colorop_funcs.reset
- * hook.
+ * Initializes the newly allocated @colorop_state and assigns it to the
+ * &drm_colorop->state pointer of @colorop, usually required when
+ * initializing the drivers or when called from the
+ * &drm_colorop_funcs.reset hook.
  *
  * This is useful for drivers that subclass the colorop state.
  */
@@ -551,7 +574,7 @@ static void __drm_colorop_reset(struct drm_colorop *colorop,
 				struct drm_colorop_state *colorop_state)
 {
 	if (colorop_state)
-		__drm_colorop_state_reset(colorop_state, colorop);
+		__drm_colorop_state_init(colorop_state, colorop);
 
 	colorop->state = colorop_state;
 }
diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
index 11646453aaac9..8b4baed060f3a 100644
--- a/drivers/gpu/drm/drm_connector.c
+++ b/drivers/gpu/drm/drm_connector.c
@@ -618,8 +618,17 @@ int drmm_connector_hdmi_init(struct drm_device *dev,
 	 * drm_connector_attach_max_bpc_property() requires the
 	 * connector to have a state.
 	 */
-	if (connector->funcs->reset)
+	if (connector->funcs->atomic_create_state) {
+		struct drm_connector_state *state;
+
+		state = connector->funcs->atomic_create_state(connector);
+		if (IS_ERR(state))
+			return PTR_ERR(state);
+
+		connector->state = state;
+	} else if (connector->funcs->reset) {
 		connector->funcs->reset(connector);
+	}
 
 	drm_connector_attach_max_bpc_property(connector, 8, max_bpc);
 	connector->max_bpc = max_bpc;
@@ -627,6 +636,10 @@ int drmm_connector_hdmi_init(struct drm_device *dev,
 	if (max_bpc > 8)
 		drm_connector_attach_hdr_output_metadata_property(connector);
 
+	ret = drm_connector_attach_color_format_property(connector, supported_formats);
+	if (ret)
+		return ret;
+
 	connector->hdmi.funcs = hdmi_funcs;
 
 	return 0;
@@ -1176,6 +1189,7 @@ static const struct drm_prop_enum_list drm_link_status_enum_list[] = {
 static const struct drm_prop_enum_list drm_panel_type_enum_list[] = {
 	{ DRM_MODE_PANEL_TYPE_UNKNOWN, "unknown" },
 	{ DRM_MODE_PANEL_TYPE_OLED, "OLED" },
+	{ DRM_MODE_PANEL_TYPE_LCD, "LCD" },
 };
 
 /**
@@ -1388,6 +1402,18 @@ static const u32 hdmi_colorspaces =
 	BIT(DRM_MODE_COLORIMETRY_DCI_P3_RGB_D65) |
 	BIT(DRM_MODE_COLORIMETRY_DCI_P3_RGB_THEATER);
 
+static const u32 hdmi_colorformats =
+	BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444) |
+	BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444) |
+	BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422) |
+	BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420);
+
+static const u32 dp_colorformats =
+	BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444) |
+	BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444) |
+	BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422) |
+	BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420);
+
 /*
  * As per DP 1.4a spec, 2.2.5.7.5 VSC SDP Payload for Pixel Encoding/Colorimetry
  * Format Table 2-120
@@ -1508,7 +1534,7 @@ EXPORT_SYMBOL(drm_hdmi_connector_get_output_format_name);
  * 	never read back the value of "DPMS" because it can be incorrect.
  * panel_type:
  * 	Immutable enum property to indicate the type of connected panel.
- * 	Possible values are "unknown" (default) and "OLED".
+ * 	Possible values are "unknown" (default), "OLED", and "LCD".
  * PATH:
  * 	Connector path property to identify how this sink is physically
  * 	connected. Used by DP MST. This should be set by calling
@@ -2557,7 +2583,8 @@ EXPORT_SYMBOL(drm_mode_create_aspect_ratio_property);
  *		conversion matrix and convert to the appropriate quantization
  *		range.
  *		The variants BT2020_RGB and BT2020_YCC are equivalent and the
- *		driver chooses between RGB and YCbCr on its own.
+ *		driver chooses between RGB and YCbCr based on the color format
+ *		property.
  *
  *	SMPTE_170M_YCC:
  *	BT709_YCC:
@@ -2935,6 +2962,149 @@ int drm_connector_attach_colorspace_property(struct drm_connector *connector)
 }
 EXPORT_SYMBOL(drm_connector_attach_colorspace_property);
 
+/**
+ * DOC: Color format
+ *
+ * The connector "color format" property allows userspace to request a specific
+ * color model on the output of the connector. Not all values listed by the
+ * property are guaranteed to work for every sink; rather, it is an optimistic
+ * listing of color formats that the source could output depending on
+ * circumstances.
+ *
+ * Whether it actually can output a certain color format is determined during
+ * the atomic check phase. Consequently, a userspace application that sets the
+ * color format to a value other than "AUTO" should check whether its atomic
+ * commit succeeded.
+ *
+ * Possible values for "color format":
+ *
+ * "AUTO":
+ *	The driver or display protocol helpers should pick a suitable color
+ *	format. All implementations of a specific display protocol will behave
+ *	the same way with "AUTO", but different display protocols do not
+ *	necessarily have the same "AUTO" semantics.
+ *
+ *	For HDMI connectors, "AUTO" picks RGB, but falls back to YUV 4:2:0 if
+ *	the bandwidth required for full-scale RGB is not available, or the mode
+ *	is YUV 4:2:0-only, as long as the mode, source, and sink all support
+ *	YUV 4:2:0.
+ * "RGB":
+ *	RGB output format. The quantization range (limited/full) depends on the
+ *	value of the "Broadcast RGB" property if it is present on the connector.
+ * "YUV 4:4:4":
+ *	YUV 4:4:4 (a.k.a. YCbCr 4:4:4) output format. Chroma is not subsampled.
+ *	The quantization range defaults to limited.
+ * "YUV 4:2:2":
+ *	YUV 4:2:2 (a.k.a. YCbCr 4:2:2) output format. Chroma has half the
+ *	horizontal resolution of Luma. The quantization range defaults to
+ *	limited.
+ * "YUV 4:2:0":
+ *	YUV 4:2:0 (a.k.a. YCbCr 4:2:0) output format. Chroma has half the
+ *	horizontal and vertical resolution of Luma. The quantization range
+ *	defaults to limited.
+ *
+ * A sink may only support some color formats in specific modes and at specific
+ * bit depths. The atomic modesetting API should be used to set a working
+ * configuration in one go, as an unsupported combination of parameters is
+ * rejected.
+ */
+
+/**
+ * drm_connector_attach_color_format_property - create and attach color format property
+ * @connector: connector to create the color format property on
+ * @supported_color_formats: bitmask of bit-shifted &enum drm_output_color_format
+ *                           values the connector supports
+ *
+ * Called by a driver to create a color format property. The property is
+ * attached to the connector automatically on success.
+ *
+ * @supported_color_formats should only include color formats the connector
+ * type can actually support.
+ *
+ * Returns:
+ * 0 on success, negative errno on error
+ */
+int drm_connector_attach_color_format_property(struct drm_connector *connector,
+					       unsigned long supported_color_formats)
+{
+	struct drm_device *dev = connector->dev;
+	struct drm_prop_enum_list enum_list[DRM_CONNECTOR_COLOR_FORMAT_COUNT];
+	unsigned int i = 0;
+	unsigned long fmt;
+
+	if (connector->color_format_property)
+		return 0;
+
+	if (!supported_color_formats) {
+		drm_err(dev, "No supported color formats provided on [CONNECTOR:%d:%s]\n",
+			connector->base.id, connector->name);
+		return -EINVAL;
+	}
+
+	if (supported_color_formats & ~GENMASK(DRM_OUTPUT_COLOR_FORMAT_COUNT - 1, 0)) {
+		drm_err(dev, "Unknown color formats provided on [CONNECTOR:%d:%s]\n",
+			connector->base.id, connector->name);
+		return -EINVAL;
+	}
+
+	switch (connector->connector_type) {
+	case DRM_MODE_CONNECTOR_HDMIA:
+	case DRM_MODE_CONNECTOR_HDMIB:
+		if (supported_color_formats & ~hdmi_colorformats) {
+			drm_err(dev, "Color formats not allowed for HDMI on [CONNECTOR:%d:%s]\n",
+				connector->base.id, connector->name);
+			return -EINVAL;
+		}
+		break;
+	case DRM_MODE_CONNECTOR_DisplayPort:
+	case DRM_MODE_CONNECTOR_eDP:
+		if (supported_color_formats & ~dp_colorformats) {
+			drm_err(dev, "Color formats not allowed for DP on [CONNECTOR:%d:%s]\n",
+				connector->base.id, connector->name);
+			return -EINVAL;
+		}
+		break;
+	}
+
+	enum_list[0].name = "AUTO";
+	enum_list[0].type = DRM_CONNECTOR_COLOR_FORMAT_AUTO;
+
+	for_each_set_bit(fmt, &supported_color_formats, DRM_OUTPUT_COLOR_FORMAT_COUNT) {
+		switch (fmt) {
+		case DRM_OUTPUT_COLOR_FORMAT_RGB444:
+			enum_list[++i].type = DRM_CONNECTOR_COLOR_FORMAT_RGB444;
+			break;
+		case DRM_OUTPUT_COLOR_FORMAT_YCBCR444:
+			enum_list[++i].type = DRM_CONNECTOR_COLOR_FORMAT_YCBCR444;
+			break;
+		case DRM_OUTPUT_COLOR_FORMAT_YCBCR422:
+			enum_list[++i].type = DRM_CONNECTOR_COLOR_FORMAT_YCBCR422;
+			break;
+		case DRM_OUTPUT_COLOR_FORMAT_YCBCR420:
+			enum_list[++i].type = DRM_CONNECTOR_COLOR_FORMAT_YCBCR420;
+			break;
+		default:
+			drm_warn(dev, "Unknown supported format %ld on [CONNECTOR:%d:%s]\n",
+				 fmt, connector->base.id, connector->name);
+			continue;
+		}
+		enum_list[i].name = drm_hdmi_connector_get_output_format_name(fmt);
+	}
+
+	connector->color_format_property =
+		drm_property_create_enum(dev, DRM_MODE_PROP_ENUM, "color format",
+					 enum_list, i + 1);
+
+	if (!connector->color_format_property)
+		return -ENOMEM;
+
+	drm_object_attach_property(&connector->base, connector->color_format_property,
+				   DRM_CONNECTOR_COLOR_FORMAT_AUTO);
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_connector_attach_color_format_property);
+
 /**
  * drm_connector_atomic_hdr_metadata_equal - checks if the hdr metadata changed
  * @old_state: old connector state to compare
@@ -3530,6 +3700,22 @@ void drm_connector_oob_hotplug_event(struct fwnode_handle *connector_fwnode,
 }
 EXPORT_SYMBOL(drm_connector_oob_hotplug_event);
 
+/**
+ * drm_connector_get_color_format - Return connector color format of @conn_state
+ * @conn_state: pointer to the &struct drm_connector_state to go check
+ *
+ */
+enum drm_connector_color_format
+drm_connector_get_color_format(const struct drm_connector_state *conn_state)
+{
+	struct drm_connector *connector = conn_state->connector;
+
+	if (connector->funcs->color_format)
+		return connector->funcs->color_format(conn_state);
+
+	return conn_state->color_format;
+}
+EXPORT_SYMBOL(drm_connector_get_color_format);
 
 /**
  * DOC: Tile group
diff --git a/drivers/gpu/drm/drm_displayid_internal.h b/drivers/gpu/drm/drm_displayid_internal.h
index 4590d6a3d8215..ef7e6c9bd4dbf 100644
--- a/drivers/gpu/drm/drm_displayid_internal.h
+++ b/drivers/gpu/drm/drm_displayid_internal.h
@@ -142,6 +142,30 @@ struct displayid_formula_timing_block {
 	struct displayid_formula_timings_9 timings[];
 } __packed;
 
+#define DISPLAYID_DEVICE_TECH_UNSPECIFIED	0
+#define DISPLAYID_DEVICE_TECH_LCD		1
+#define DISPLAYID_DEVICE_TECH_OLED		2
+
+#define DISPLAYID_DISPLAY_PARAMS_DEVICE_TECH	GENMASK(6, 4)
+
+struct displayid_display_params_block {
+	struct displayid_block base;
+	__le16 horiz_image_size;
+	__le16 vert_image_size;
+	__le16 horiz_pixel_count;
+	__le16 vert_pixel_count;
+	u8 features;
+	u8 primary_color1[3];
+	u8 primary_color2[3];
+	u8 primary_color3[3];
+	u8 white_point[3];
+	__le16 max_luminance_full;
+	__le16 max_luminance_10;
+	__le16 min_luminance;
+	u8 color_depth_and_tech;	/* [2:0] depth, [6:4] device tech, [7] theme */
+	u8 gamma_eotf;
+} __packed;
+
 #define DISPLAYID_VESA_MSO_OVERLAP	GENMASK(3, 0)
 #define DISPLAYID_VESA_MSO_MODE		GENMASK(6, 5)
 
diff --git a/drivers/gpu/drm/drm_draw_internal.h b/drivers/gpu/drm/drm_draw_internal.h
index 2619671456359..44ddcee4744ca 100644
--- a/drivers/gpu/drm/drm_draw_internal.h
+++ b/drivers/gpu/drm/drm_draw_internal.h
@@ -7,7 +7,6 @@
 #ifndef __DRM_DRAW_INTERNAL_H__
 #define __DRM_DRAW_INTERNAL_H__
 
-#include <linux/font.h>
 #include <linux/types.h>
 
 struct iosys_map;
@@ -18,12 +17,6 @@ static inline bool drm_draw_is_pixel_fg(const u8 *sbuf8, unsigned int spitch, in
 	return (sbuf8[(y * spitch) + x / 8] & (0x80 >> (x % 8))) != 0;
 }
 
-static inline const u8 *drm_draw_get_char_bitmap(const struct font_desc *font,
-						 char c, size_t font_pitch)
-{
-	return font->data + (c * font->height) * font_pitch;
-}
-
 bool drm_draw_can_convert_from_xrgb8888(u32 format);
 
 u32 drm_draw_color_from_xrgb8888(u32 color, u32 format);
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 675675480da49..e51ed959da89f 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -342,7 +342,9 @@ void drm_minor_release(struct drm_minor *minor)
  *
  *		platform_set_drvdata(pdev, drm);
  *
- *		drm_mode_config_reset(drm);
+ *		ret = drm_mode_config_create_initial_state(drm);
+ *		if (ret)
+ *			return ret;
  *
  *		ret = drm_dev_register(drm);
  *		if (ret)
@@ -473,6 +475,22 @@ void drm_dev_exit(int idx)
 }
 EXPORT_SYMBOL(drm_dev_exit);
 
+/*
+ * Mark the device as unplugged and wait for any in-flight drm_dev_enter()
+ * critical sections to complete.
+ */
+static void drm_dev_synchronize_unplug(struct drm_device *dev)
+{
+	/*
+	 * After synchronizing any critical read section is guaranteed to see
+	 * the new value of ->unplugged, and any critical section which might
+	 * still have seen the old value of ->unplugged is guaranteed to have
+	 * finished.
+	 */
+	dev->unplugged = true;
+	synchronize_srcu(&drm_unplug_srcu);
+}
+
 /**
  * drm_dev_unplug - unplug a DRM device
  * @dev: DRM device
@@ -485,15 +503,7 @@ EXPORT_SYMBOL(drm_dev_exit);
  */
 void drm_dev_unplug(struct drm_device *dev)
 {
-	/*
-	 * After synchronizing any critical read section is guaranteed to see
-	 * the new value of ->unplugged, and any critical section which might
-	 * still have seen the old value of ->unplugged is guaranteed to have
-	 * finished.
-	 */
-	dev->unplugged = true;
-	synchronize_srcu(&drm_unplug_srcu);
-
+	drm_dev_synchronize_unplug(dev);
 	drm_dev_unregister(dev);
 
 	/* Clear all CPU mappings pointing to this device */
@@ -1091,6 +1101,7 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags)
 		goto err_minors;
 
 	dev->registered = true;
+	dev->unplugged = false;
 
 	if (driver->load) {
 		ret = driver->load(dev, flags);
@@ -1118,6 +1129,13 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags)
 	if (dev->driver->unload)
 		dev->driver->unload(dev);
 err_minors:
+	/*
+	 * If a minor was registered before the failure, userspace could have
+	 * opened it and entered a drm_dev_enter() critical section. Ensure all
+	 * such sections complete before we clean up.
+	 */
+	drm_dev_synchronize_unplug(dev);
+
 	remove_compat_control_link(dev);
 	drm_minor_unregister(dev, DRM_MINOR_ACCEL);
 	drm_minor_unregister(dev, DRM_MINOR_PRIMARY);
diff --git a/drivers/gpu/drm/drm_dumb_buffers.c b/drivers/gpu/drm/drm_dumb_buffers.c
index 2156dbe601c9c..8e9ff17538e73 100644
--- a/drivers/gpu/drm/drm_dumb_buffers.c
+++ b/drivers/gpu/drm/drm_dumb_buffers.c
@@ -32,7 +32,6 @@
 #include <drm/drm_print.h>
 
 #include "drm_crtc_internal.h"
-#include "drm_internal.h"
 
 /**
  * DOC: overview
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index df3c25bac761d..07970e5b5f65e 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -6713,32 +6713,82 @@ static void drm_reset_display_info(struct drm_connector *connector)
 
 	info->source_physical_address = CEC_PHYS_ADDR_INVALID;
 	memset(&info->amd_vsdb, 0, sizeof(info->amd_vsdb));
+
+	info->panel_type = DRM_MODE_PANEL_TYPE_UNKNOWN;
+}
+
+static void drm_displayid_process_base_section_header(struct drm_connector *connector,
+						      const struct displayid_iter *iter)
+{
+	struct drm_display_info *info = &connector->display_info;
+
+	drm_dbg_kms(connector->dev,
+		    "[CONNECTOR:%d:%s] DisplayID extension version 0x%02x, primary use 0x%02x\n",
+		    connector->base.id, connector->name,
+		    displayid_version(iter),
+		    displayid_primary_use(iter));
+	if (displayid_version(iter) == DISPLAY_ID_STRUCTURE_VER_20 &&
+	    (displayid_primary_use(iter) == PRIMARY_USE_HEAD_MOUNTED_VR ||
+	     displayid_primary_use(iter) == PRIMARY_USE_HEAD_MOUNTED_AR))
+		info->non_desktop = true;
+}
+
+static void
+drm_displayid_parse_display_params(struct drm_connector *connector,
+				   const struct displayid_block *block)
+{
+	struct drm_display_info *info = &connector->display_info;
+	const struct displayid_display_params_block *params =
+		(const struct displayid_display_params_block *)block;
+	u8 tech;
+
+	if (block->num_bytes < sizeof(*params) - sizeof(params->base)) {
+		drm_dbg_kms(connector->dev,
+			    "[CONNECTOR:%d:%s] DisplayID Display Parameters block too short (%u < %zu)\n",
+			    connector->base.id, connector->name,
+			    block->num_bytes,
+			    sizeof(*params) - sizeof(params->base));
+		return;
+	}
+
+	tech = FIELD_GET(DISPLAYID_DISPLAY_PARAMS_DEVICE_TECH,
+			 params->color_depth_and_tech);
+
+	drm_dbg_kms(connector->dev,
+		    "[CONNECTOR:%d:%s] DisplayID Display Parameters: device technology %s\n",
+		    connector->base.id, connector->name,
+		    tech == DISPLAYID_DEVICE_TECH_LCD ? "LCD" :
+		    tech == DISPLAYID_DEVICE_TECH_OLED ? "OLED" : "unspecified");
+
+	switch (tech) {
+	case DISPLAYID_DEVICE_TECH_LCD:
+		info->panel_type = DRM_MODE_PANEL_TYPE_LCD;
+		break;
+	case DISPLAYID_DEVICE_TECH_OLED:
+		info->panel_type = DRM_MODE_PANEL_TYPE_OLED;
+		break;
+	default:
+		break;
+	}
 }
 
 static void update_displayid_info(struct drm_connector *connector,
 				  const struct drm_edid *drm_edid)
 {
-	struct drm_display_info *info = &connector->display_info;
 	const struct displayid_block *block;
 	struct displayid_iter iter;
+	bool base_section_header_processed = false;
 
 	displayid_iter_edid_begin(drm_edid, &iter);
 	displayid_iter_for_each(block, &iter) {
-		drm_dbg_kms(connector->dev,
-			    "[CONNECTOR:%d:%s] DisplayID extension version 0x%02x, primary use 0x%02x\n",
-			    connector->base.id, connector->name,
-			    displayid_version(&iter),
-			    displayid_primary_use(&iter));
-		if (displayid_version(&iter) == DISPLAY_ID_STRUCTURE_VER_20 &&
-		    (displayid_primary_use(&iter) == PRIMARY_USE_HEAD_MOUNTED_VR ||
-		     displayid_primary_use(&iter) == PRIMARY_USE_HEAD_MOUNTED_AR))
-			info->non_desktop = true;
+		if (!base_section_header_processed) {
+			drm_displayid_process_base_section_header(connector, &iter);
+			base_section_header_processed = true;
+		}
 
-		/*
-		 * We're only interested in the base section here, no need to
-		 * iterate further.
-		 */
-		break;
+		if (displayid_version(&iter) == DISPLAY_ID_STRUCTURE_VER_20 &&
+		    block->tag == DATA_BLOCK_2_DISPLAY_PARAMETERS)
+			drm_displayid_parse_display_params(connector, block);
 	}
 	displayid_iter_end(&iter);
 }
diff --git a/drivers/gpu/drm/drm_exec.c b/drivers/gpu/drm/drm_exec.c
index 7988f5e7d56a3..fa923852fae45 100644
--- a/drivers/gpu/drm/drm_exec.c
+++ b/drivers/gpu/drm/drm_exec.c
@@ -79,7 +79,7 @@ void drm_exec_init(struct drm_exec *exec, u32 flags, unsigned nr)
 		nr = PAGE_SIZE / sizeof(void *);
 
 	exec->flags = flags;
-	exec->objects = kvmalloc_array(nr, sizeof(void *), GFP_KERNEL);
+	exec->objects = kvmalloc_objs(*exec->objects, nr, GFP_KERNEL);
 
 	/* If allocation here fails, just delay that till the first use */
 	exec->max_objects = exec->objects ? nr : 0;
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index 80ca785bdb262..d4664ed468b26 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -1834,21 +1834,17 @@ EXPORT_SYMBOL(drm_fb_helper_initial_config);
  */
 int drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper)
 {
-	int err = 0;
-
 	if (!drm_fbdev_emulation || !fb_helper)
 		return 0;
 
 	mutex_lock(&fb_helper->lock);
-	if (fb_helper->deferred_setup) {
-		err = __drm_fb_helper_initial_config_and_unlock(fb_helper);
-		return err;
-	}
+	if (fb_helper->deferred_setup)
+		return __drm_fb_helper_initial_config_and_unlock(fb_helper);
 
 	if (!fb_helper->fb || !drm_master_internal_acquire(fb_helper->dev)) {
 		fb_helper->delayed_hotplug = true;
 		mutex_unlock(&fb_helper->lock);
-		return err;
+		return 0;
 	}
 
 	drm_master_internal_release(fb_helper->dev);
@@ -1892,4 +1888,3 @@ bool drm_fb_helper_gem_is_fb(const struct drm_fb_helper *fb_helper,
 	return gem == obj;
 }
 EXPORT_SYMBOL_GPL(drm_fb_helper_gem_is_fb);
-
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index e3ed684ddcf29..b0e0e4118573d 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -232,8 +232,7 @@ void drm_gem_private_object_init(struct drm_device *dev,
 	if (!obj->resv)
 		obj->resv = &obj->_resv;
 
-	if (drm_core_check_feature(dev, DRIVER_GEM_GPUVA))
-		drm_gem_gpuva_init(obj);
+	drm_gem_gpuva_init(obj);
 
 	drm_vma_node_reset(&obj->vma_node);
 	INIT_LIST_HEAD(&obj->lru_node);
@@ -1252,7 +1251,7 @@ int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size,
 		}
 
 		vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
-		vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+		vma->vm_page_prot = pgprot_writecombine(vma_get_page_prot(vma));
 		vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
 	}
 
diff --git a/drivers/gpu/drm/drm_gem_dma_helper.c b/drivers/gpu/drm/drm_gem_dma_helper.c
index 1c00a71ab3c9d..7d9612075d310 100644
--- a/drivers/gpu/drm/drm_gem_dma_helper.c
+++ b/drivers/gpu/drm/drm_gem_dma_helper.c
@@ -540,7 +540,7 @@ int drm_gem_dma_mmap(struct drm_gem_dma_object *dma_obj, struct vm_area_struct *
 	vm_flags_mod(vma, VM_DONTDUMP | VM_DONTEXPAND, VM_PFNMAP);
 
 	if (dma_obj->map_noncoherent) {
-		vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+		vma->vm_page_prot = vma_get_page_prot(vma);
 
 		ret = dma_mmap_pages(drm_dev_dma_dev(dma_obj->base.dev),
 				     vma, vma->vm_end - vma->vm_start,
diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
index c989459eb2159..68bfb9fde2119 100644
--- a/drivers/gpu/drm/drm_gem_shmem_helper.c
+++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
@@ -473,10 +473,23 @@ void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem,
 }
 EXPORT_SYMBOL_GPL(drm_gem_shmem_vunmap_locked);
 
-static int
-drm_gem_shmem_create_with_handle(struct drm_file *file_priv,
-				 struct drm_device *dev, size_t size,
-				 uint32_t *handle)
+/**
+ * drm_gem_shmem_create_with_handle - Allocate an object with the given size and
+ *	returns a GEM handle
+ * @file_priv: DRM file structure to create the dumb buffer for
+ * @dev: DRM device
+ * @size: Size of the object to allocate
+ * @handle: Returns the GEM handle on success
+ *
+ * Allocates an shmem GEM buffer using drm_gem_shmem_create() and returns
+ * a GEM handle to it.
+ *
+ * Returns:
+ * Zero on success, or an error code otherwise.
+ */
+int drm_gem_shmem_create_with_handle(struct drm_file *file_priv,
+				     struct drm_device *dev, size_t size,
+				     uint32_t *handle)
 {
 	struct drm_gem_shmem_object *shmem;
 	int ret;
@@ -495,6 +508,7 @@ drm_gem_shmem_create_with_handle(struct drm_file *file_priv,
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(drm_gem_shmem_create_with_handle);
 
 /* Update madvise status, returns true if not purged, else
  * false or -errno.
@@ -764,7 +778,7 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct
 		return ret;
 
 	vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
-	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+	vma->vm_page_prot = vma_get_page_prot(vma);
 	if (shmem->map_wc)
 		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 
diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c
index 958cb605aedd5..4fe2ef8c0991c 100644
--- a/drivers/gpu/drm/drm_gpusvm.c
+++ b/drivers/gpu/drm/drm_gpusvm.c
@@ -48,6 +48,47 @@
  *	event. As mentioned above, ranges are tracked in a notifier's Red-Black
  *	tree.
  *
+ * - Pages:
+ *	struct drm_gpusvm_pages holds the DMA mapping state for a range of
+ *	CPU virtual addresses: the DMA mapped device addresses,
+ *	the device private pagemap, the IOVA state, the per mapping
+ *	notifier sequence number, and the drm_device that owns the DMA
+ *	mappings.
+ *	A driver embeds one or more struct drm_gpusvm_pages alongside its
+ *	struct drm_gpusvm_range, choosing one of two layouts:
+ *
+ *	1:1 - one drm_gpusvm_pages per range (one drm_device). Simplest
+ *	layout; to mirror a VA range on several devices a driver uses a
+ *	separate range (and notifier) per device, so the HMM fault is taken
+ *	once per device.
+ *
+ *	N:1 - one drm_gpusvm_pages per drm_device, all sharing one range and
+ *	notifier; only the per-device DMA mapping differs. The instances must
+ *	sit in contiguous memory so a single drm_gpusvm_range_set_unmapped()
+ *	can mark them all. A driver can keep one instance inline for the single
+ *	device case and switch to a heap array only when more devices join,
+ *	e.g.:
+ *
+ *	.. code-block:: c
+ *
+ *		struct driver_range {
+ *			struct drm_gpusvm_range base;
+ *			unsigned int num_pages;	// 1: inline_pages, >1: pages[]
+ *			union {
+ *				struct drm_gpusvm_pages inline_pages;
+ *				struct drm_gpusvm_pages *pages;
+ *			};
+ *		};
+ *
+ *	In the N:1 case the driver allocates the pages array with a zeroing
+ *	allocator (e.g. kcalloc(num_pages, ...)), initialises each entry with
+ *	drm_gpusvm_init_pages(), and frees each entry with
+ *	drm_gpusvm_free_pages() plus the array itself, from its range free
+ *	callback. Each drm_gpusvm_pages is mapped independently by their own
+ *	drm_device.
+ *	Each drm_gpusvm_pages must be zero-initialised and initialised with
+ *	drm_gpusvm_init_pages(), called once per entry.
+ *
  * - Operations:
  *	Define the interface for driver-specific GPU SVM operations such as
  *	range allocation, notifier allocation, and invalidations.
@@ -92,7 +133,7 @@
  * range RB tree and list, as well as the range's DMA mappings and sequence
  * number. GPU SVM manages all necessary locking and unlocking operations,
  * except for the recheck range's pages being valid
- * (drm_gpusvm_range_pages_valid) when the driver is committing GPU bindings.
+ * (drm_gpusvm_pages_valid) when the driver is committing GPU bindings.
  * This lock corresponds to the ``driver->update`` lock mentioned in
  * Documentation/mm/hmm.rst. Future revisions may transition from a GPU SVM
  * global lock to a per-notifier lock if finer-grained locking is deemed
@@ -140,15 +181,20 @@
  *
  * .. code-block:: c
  *
- *	int driver_bind_range(struct drm_gpusvm *gpusvm, struct drm_gpusvm_range *range)
+ *	struct driver_range {
+ *		struct drm_gpusvm_range base;
+ *		struct drm_gpusvm_pages pages;
+ *	};
+ *
+ *	int driver_bind_range(struct drm_gpusvm *gpusvm, struct driver_range *drange)
  *	{
  *		int err = 0;
  *
- *		driver_alloc_and_setup_memory_for_bind(gpusvm, range);
+ *		driver_alloc_and_setup_memory_for_bind(gpusvm, drange);
  *
  *		drm_gpusvm_notifier_lock(gpusvm);
- *		if (drm_gpusvm_range_pages_valid(range))
- *			driver_commit_bind(gpusvm, range);
+ *		if (drm_gpusvm_pages_valid(gpusvm, &drange->pages))
+ *			driver_commit_bind(gpusvm, drange);
  *		else
  *			err = -EAGAIN;
  *		drm_gpusvm_notifier_unlock(gpusvm);
@@ -160,6 +206,8 @@
  *			     unsigned long gpuva_start, unsigned long gpuva_end)
  *	{
  *		struct drm_gpusvm_ctx ctx = {};
+ *		struct driver_range *drange;
+ *		struct drm_gpusvm_range *range;
  *		int err;
  *
  *		driver_svm_lock();
@@ -174,6 +222,7 @@
  *			err = PTR_ERR(range);
  *			goto unlock;
  *		}
+ *		drange = container_of(range, struct driver_range, base);
  *
  *		if (driver_migration_policy(range)) {
  *			err = drm_pagemap_populate_mm(driver_choose_drm_pagemap(),
@@ -183,7 +232,10 @@
  *				goto retry;
  *		}
  *
- *		err = drm_gpusvm_range_get_pages(gpusvm, range, &ctx);
+ *		err = drm_gpusvm_get_pages(gpusvm, &drange->pages,
+ *					   gpusvm->mm, &range->notifier->notifier,
+ *					   drm_gpusvm_range_start(range),
+ *					   drm_gpusvm_range_end(range), &ctx);
  *		if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM) {	// CPU mappings changed
  *			if (err == -EOPNOTSUPP)
  *				drm_gpusvm_range_evict(gpusvm, range);
@@ -192,7 +244,7 @@
  *			goto unlock;
  *		}
  *
- *		err = driver_bind_range(gpusvm, range);
+ *		err = driver_bind_range(gpusvm, drange);
  *		if (err == -EAGAIN)	// CPU mappings changed
  *			goto retry
  *
@@ -205,6 +257,21 @@
  *
  * .. code-block:: c
  *
+ *	// The driver owns the drm_gpusvm_pages lifecycle. ops->range_free is
+ *	// the final fallback: drm_gpusvm_free_pages() unmaps any
+ *	// lingering DMA mapping and a no-op if already unmapped and frees the
+ *	// dma_addr array. The normal flow is to DMA unmap before
+ *	// drm_gpusvm_range_remove() (before the range leaves the tree).
+ *	void driver_range_free(struct drm_gpusvm_range *range)
+ *	{
+ *		struct driver_range *drange =
+ *			container_of(range, struct driver_range, base);
+ *
+ *		drm_gpusvm_free_pages(range->gpusvm, &drange->pages,
+ *				      drm_gpusvm_range_size(range) >> PAGE_SHIFT);
+ *		kfree(drange);
+ *	}
+ *
  *	void __driver_garbage_collector(struct drm_gpusvm *gpusvm,
  *					struct drm_gpusvm_range *range)
  *	{
@@ -215,6 +282,14 @@
  *			drm_gpusvm_range_evict(gpusvm, range);
  *
  *		driver_unbind_range(range);
+ *		// The pages must be DMA unmapped before drm_gpusvm_range_remove()
+ *		// , so a range is never off the MMU interval tree while still DMA
+ *		// mapped as the original drmsvm design flow. Otherwise a concurrent CPU
+ *		// munmap's notifier could miss this range and free pages still mapped
+ *		// for device DMA. This is the normal unmap point.
+ *		drm_gpusvm_unmap_pages(gpusvm, &drange->pages,
+ *				       drm_gpusvm_range_size(range) >> PAGE_SHIFT,
+ *				       &(struct drm_gpusvm_ctx){ .in_notifier = false });
  *		drm_gpusvm_range_remove(gpusvm, range);
  *	}
  *
@@ -236,17 +311,22 @@
  *	{
  *		struct drm_gpusvm_ctx ctx = { .in_notifier = true, };
  *		struct drm_gpusvm_range *range = NULL;
+ *		struct driver_range *drange;
  *
  *		driver_invalidate_device_pages(gpusvm, mmu_range->start, mmu_range->end);
  *
  *		drm_gpusvm_for_each_range(range, notifier, mmu_range->start,
  *					  mmu_range->end) {
- *			drm_gpusvm_range_unmap_pages(gpusvm, range, &ctx);
+ *			drange = container_of(range, struct driver_range, base);
+ *
+ *			drm_gpusvm_unmap_pages(gpusvm, &drange->pages,
+ *					       drm_gpusvm_range_size(range) >> PAGE_SHIFT,
+ *					       &ctx);
  *
  *			if (mmu_range->event != MMU_NOTIFY_UNMAP)
  *				continue;
  *
- *			drm_gpusvm_range_set_unmapped(range, mmu_range);
+ *			drm_gpusvm_range_set_unmapped(range, &drange->pages, 1, mmu_range);
  *			driver_garbage_collector_add(gpusvm, range);
  *		}
  *	}
@@ -359,7 +439,6 @@ static const struct mmu_interval_notifier_ops drm_gpusvm_notifier_ops = {
  * drm_gpusvm_init() - Initialize the GPU SVM.
  * @gpusvm: Pointer to the GPU SVM structure.
  * @name: Name of the GPU SVM.
- * @drm: Pointer to the DRM device structure.
  * @mm: Pointer to the mm_struct for the address space.
  * @mm_start: Start address of GPU SVM.
  * @mm_range: Range of the GPU SVM.
@@ -373,7 +452,9 @@ static const struct mmu_interval_notifier_ops drm_gpusvm_notifier_ops = {
  * This function initializes the GPU SVM.
  *
  * Note: If only using the simple drm_gpusvm_pages API (get/unmap/free),
- * then only @gpusvm, @name, and @drm are expected. However, the same base
+ * then only @gpusvm and @name are expected. The @drm drm_device for dma
+ * mappings is bound per-pages via drm_gpusvm_init_pages() before the first
+ * drm_gpusvm_get_pages() call. However, the same base
  * @gpusvm can also be used with both modes together in which case the full
  * setup is needed, where the core drm_gpusvm_pages API will simply never use
  * the other fields.
@@ -381,7 +462,7 @@ static const struct mmu_interval_notifier_ops drm_gpusvm_notifier_ops = {
  * Return: 0 on success, a negative error code on failure.
  */
 int drm_gpusvm_init(struct drm_gpusvm *gpusvm,
-		    const char *name, struct drm_device *drm,
+		    const char *name,
 		    struct mm_struct *mm,
 		    unsigned long mm_start, unsigned long mm_range,
 		    unsigned long notifier_size,
@@ -399,7 +480,6 @@ int drm_gpusvm_init(struct drm_gpusvm *gpusvm,
 	}
 
 	gpusvm->name = name;
-	gpusvm->drm = drm;
 	gpusvm->mm = mm;
 	gpusvm->mm_start = mm_start;
 	gpusvm->mm_range = mm_range;
@@ -640,8 +720,7 @@ drm_gpusvm_range_alloc(struct drm_gpusvm *gpusvm,
 	range->itree.start = ALIGN_DOWN(fault_addr, chunk_size);
 	range->itree.last = ALIGN(fault_addr + 1, chunk_size) - 1;
 	INIT_LIST_HEAD(&range->entry);
-	range->pages.notifier_seq = LONG_MAX;
-	range->pages.flags.migrate_devmem = migrate_devmem ? 1 : 0;
+	range->flags.migrate_devmem = migrate_devmem ? 1 : 0;
 
 	return range;
 }
@@ -773,8 +852,7 @@ enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range,
 		.end = end,
 		.dev_private_owner = dev_private_owner,
 	};
-	unsigned long timeout =
-		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
+	unsigned long timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
 	enum drm_gpusvm_scan_result state = DRM_GPUSVM_SCAN_UNPOPULATED, new_state;
 	unsigned long *pfns;
 	unsigned long npages = npages_in_range(start, end);
@@ -788,22 +866,7 @@ enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range,
 	hmm_range.hmm_pfns = pfns;
 
 retry:
-	hmm_range.notifier_seq = mmu_interval_read_begin(notifier);
-	mmap_read_lock(range->gpusvm->mm);
-
-	while (true) {
-		err = hmm_range_fault(&hmm_range);
-		if (err == -EBUSY) {
-			if (time_after(jiffies, timeout))
-				break;
-
-			hmm_range.notifier_seq =
-				mmu_interval_read_begin(notifier);
-			continue;
-		}
-		break;
-	}
-	mmap_read_unlock(range->gpusvm->mm);
+	err = hmm_range_fault_unlocked_timeout(&hmm_range, timeout);
 	if (err)
 		goto err_free;
 
@@ -929,7 +992,7 @@ drm_gpusvm_range_chunk_size(struct drm_gpusvm *gpusvm,
 		 * mallocs 16k but the CPU VMA is ~128k which results in 64k SVM
 		 * ranges. When migrating the SVM ranges, some processes fail in
 		 * drm_pagemap_migrate_to_devmem with 'migrate.cpages != npages'
-		 * and then upon drm_gpusvm_range_get_pages device pages from
+		 * and then upon drm_gpusvm_get_pages device pages from
 		 * other processes are collected + faulted in which creates all
 		 * sorts of problems. Unsure exactly how this happening, also
 		 * problem goes away if 'xe_exec_system_allocator --r
@@ -1135,31 +1198,51 @@ static void __drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
 				     unsigned long npages)
 {
 	struct drm_pagemap *dpagemap = svm_pages->dpagemap;
-	struct device *dev = gpusvm->drm->dev;
+	struct device *dev;
 	unsigned long i, j;
 
 	lockdep_assert_held(&gpusvm->notifier_lock);
 
+	if (!svm_pages->drm)
+		return;
+
+	dev = svm_pages->drm->dev;
+
 	if (svm_pages->flags.has_dma_mapping) {
 		struct drm_gpusvm_pages_flags flags = {
 			.__flags = svm_pages->flags.__flags,
 		};
 		bool use_iova = dma_use_iova(&svm_pages->state);
 
-		if (use_iova)
-			dma_iova_destroy(dev, &svm_pages->state,
-					 svm_pages->state_offset,
-					 svm_pages->dma_addr[0].dir, 0);
+		/*
+		 * IOVA is reserved for the whole range but only the linked
+		 * system pages (state_offset bytes) need unlinking; free the
+		 * entire reservation to avoid leaking the device-page part.
+		 * On the error path state_offset is 0, so just free it.
+		 */
+		if (use_iova) {
+			if (svm_pages->state_offset)
+				dma_iova_unlink(dev, &svm_pages->state, 0,
+						svm_pages->state_offset,
+						svm_pages->dma_addr[0].dir, 0);
+			dma_iova_free(dev, &svm_pages->state);
+		}
 
 		for (i = 0, j = 0; i < npages; j++) {
 			struct drm_pagemap_addr *addr = &svm_pages->dma_addr[j];
 
-			if (!use_iova && addr->proto == DRM_INTERCONNECT_SYSTEM)
-				dma_unmap_page(dev,
-					       addr->addr,
-					       PAGE_SIZE << addr->order,
-					       addr->dir);
-			else if (dpagemap && dpagemap->ops->device_unmap)
+			if (addr->proto == DRM_INTERCONNECT_SYSTEM) {
+				/*
+				 * Linked IOVA pages were already torn down by
+				 * the dma_iova_unlink()/dma_iova_free() above;
+				 * only the non-IOVA mappings need unmap here.
+				 */
+				if (!use_iova)
+					dma_unmap_page(dev,
+						       addr->addr,
+						       PAGE_SIZE << addr->order,
+						       addr->dir);
+			} else if (dpagemap && dpagemap->ops->device_unmap)
 				dpagemap->ops->device_unmap(dpagemap,
 							    dev, addr);
 			i += 1 << addr->order;
@@ -1222,12 +1305,17 @@ EXPORT_SYMBOL_GPL(drm_gpusvm_free_pages);
  * This function removes the specified GPU SVM range and also removes the parent
  * GPU SVM notifier if no more ranges remain in the notifier. The caller must
  * hold a lock to protect range and notifier removal.
+ *
+ * This function does not unmap or free the drm_gpusvm_pages, the driver owns
+ * that lifecycle. The caller must DMA unmap the range's pages before calling
+ * this function, so a range is never removed from the MMU interval tree while
+ * still DMA mapped. Typically the driver calls drm_gpusvm_unmap_pages() first.
+ * And the range_free callback's drm_gpusvm_free_pages() is a final fallback safe
+ * net.
  */
 void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm,
 			     struct drm_gpusvm_range *range)
 {
-	unsigned long npages = npages_in_range(drm_gpusvm_range_start(range),
-					       drm_gpusvm_range_end(range));
 	struct drm_gpusvm_notifier *notifier;
 
 	drm_gpusvm_driver_lock_held(gpusvm);
@@ -1239,8 +1327,6 @@ void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm,
 		return;
 
 	drm_gpusvm_notifier_lock(gpusvm);
-	__drm_gpusvm_unmap_pages(gpusvm, &range->pages, npages);
-	__drm_gpusvm_free_pages(gpusvm, &range->pages);
 	__drm_gpusvm_range_remove(notifier, range);
 	drm_gpusvm_notifier_unlock(gpusvm);
 
@@ -1319,34 +1405,14 @@ EXPORT_SYMBOL_GPL(drm_gpusvm_range_put);
  *
  * Return: True if GPU SVM range has valid pages, False otherwise
  */
-static bool drm_gpusvm_pages_valid(struct drm_gpusvm *gpusvm,
-				   struct drm_gpusvm_pages *svm_pages)
+bool drm_gpusvm_pages_valid(struct drm_gpusvm *gpusvm,
+			    struct drm_gpusvm_pages *svm_pages)
 {
 	lockdep_assert_held(&gpusvm->notifier_lock);
 
 	return svm_pages->flags.has_devmem_pages || svm_pages->flags.has_dma_mapping;
 }
-
-/**
- * drm_gpusvm_range_pages_valid() - GPU SVM range pages valid
- * @gpusvm: Pointer to the GPU SVM structure
- * @range: Pointer to the GPU SVM range structure
- *
- * This function determines if a GPU SVM range pages are valid. Expected be
- * called holding gpusvm->notifier_lock and as the last step before committing a
- * GPU binding. This is akin to a notifier seqno check in the HMM documentation
- * but due to wider notifiers (i.e., notifiers which span multiple ranges) this
- * function is required for finer grained checking (i.e., per range) if pages
- * are valid.
- *
- * Return: True if GPU SVM range has valid pages, False otherwise
- */
-bool drm_gpusvm_range_pages_valid(struct drm_gpusvm *gpusvm,
-				  struct drm_gpusvm_range *range)
-{
-	return drm_gpusvm_pages_valid(gpusvm, &range->pages);
-}
-EXPORT_SYMBOL_GPL(drm_gpusvm_range_pages_valid);
+EXPORT_SYMBOL_GPL(drm_gpusvm_pages_valid);
 
 /**
  * drm_gpusvm_pages_valid_unlocked() - GPU SVM pages valid unlocked
@@ -1406,8 +1472,7 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 		.dev_private_owner = ctx->device_private_page_owner,
 	};
 	void *zdd;
-	unsigned long timeout =
-		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
+	unsigned long timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
 	unsigned long i, j;
 	unsigned long npages = npages_in_range(pages_start, pages_end);
 	unsigned long num_dma_mapped;
@@ -1421,10 +1486,10 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 							   DMA_BIDIRECTIONAL;
 	struct dma_iova_state *state = &svm_pages->state;
 
-retry:
-	if (time_after(jiffies, timeout))
-		return -EBUSY;
+	if (!svm_pages->drm)
+		return -EINVAL;
 
+retry:
 	hmm_range.notifier_seq = mmu_interval_read_begin(notifier);
 	if (drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages))
 		goto set_seqno;
@@ -1439,21 +1504,7 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 	}
 
 	hmm_range.hmm_pfns = pfns;
-	while (true) {
-		mmap_read_lock(mm);
-		err = hmm_range_fault(&hmm_range);
-		mmap_read_unlock(mm);
-
-		if (err == -EBUSY) {
-			if (time_after(jiffies, timeout))
-				break;
-
-			hmm_range.notifier_seq =
-				mmu_interval_read_begin(notifier);
-			continue;
-		}
-		break;
-	}
+	err = hmm_range_fault_unlocked_timeout(&hmm_range, timeout);
 	mmput(mm);
 	if (err)
 		goto err_free;
@@ -1486,7 +1537,7 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 		/* Unlock and restart mapping to allocate memory. */
 		drm_gpusvm_notifier_unlock(gpusvm);
 		svm_pages->dma_addr =
-			kvmalloc_objs(*svm_pages->dma_addr, npages);
+			kvzalloc_objs(*svm_pages->dma_addr, npages);
 		if (!svm_pages->dma_addr) {
 			err = -ENOMEM;
 			goto err_free;
@@ -1520,7 +1571,7 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 
 				pagemap = page_pgmap(page);
 				dpagemap = drm_pagemap_page_to_dpagemap(page);
-				if (drm_WARN_ON(gpusvm->drm, !dpagemap)) {
+				if (drm_WARN_ON(svm_pages->drm, !dpagemap)) {
 					/*
 					 * Raced. This is not supposed to happen
 					 * since hmm_range_fault() should've migrated
@@ -1529,13 +1580,23 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 					err = -EAGAIN;
 					goto err_unmap;
 				}
+
+				/*
+				 * Set the dpagemap as soon as the first
+				 * device page is mapped so the err_unmap path
+				 * can device_unmap() the device mappings that
+				 * have already been created.
+				 */
+				drm_pagemap_get(dpagemap);
+				drm_pagemap_put(svm_pages->dpagemap);
+				svm_pages->dpagemap = dpagemap;
 			}
 			svm_pages->dma_addr[j] =
 				dpagemap->ops->device_map(dpagemap,
-							  gpusvm->drm->dev,
+							  svm_pages->drm->dev,
 							  page, order,
 							  dma_dir);
-			if (dma_mapping_error(gpusvm->drm->dev,
+			if (dma_mapping_error(svm_pages->drm->dev,
 					      svm_pages->dma_addr[j].addr)) {
 				err = -EFAULT;
 				goto err_unmap;
@@ -1555,11 +1616,11 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 			}
 
 			if (!i)
-				dma_iova_try_alloc(gpusvm->drm->dev, state,
+				dma_iova_try_alloc(svm_pages->drm->dev, state,
 						   0, npages * PAGE_SIZE);
 
 			if (dma_use_iova(state)) {
-				err = dma_iova_link(gpusvm->drm->dev, state,
+				err = dma_iova_link(svm_pages->drm->dev, state,
 						    hmm_pfn_to_phys(pfns[i]),
 						    svm_pages->state_offset,
 						    PAGE_SIZE << order,
@@ -1570,11 +1631,11 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 				addr = state->addr + svm_pages->state_offset;
 				svm_pages->state_offset += PAGE_SIZE << order;
 			} else {
-				addr = dma_map_page(gpusvm->drm->dev,
+				addr = dma_map_page(svm_pages->drm->dev,
 						    page, 0,
 						    PAGE_SIZE << order,
 						    dma_dir);
-				if (dma_mapping_error(gpusvm->drm->dev, addr)) {
+				if (dma_mapping_error(svm_pages->drm->dev, addr)) {
 					err = -EFAULT;
 					goto err_unmap;
 				}
@@ -1590,18 +1651,14 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 	}
 
 	if (dma_use_iova(state)) {
-		err = dma_iova_sync(gpusvm->drm->dev, state, 0,
+		err = dma_iova_sync(svm_pages->drm->dev, state, 0,
 				    svm_pages->state_offset);
 		if (err)
 			goto err_unmap;
 	}
 
-	if (pagemap) {
+	if (pagemap)
 		flags.has_devmem_pages = true;
-		drm_pagemap_get(dpagemap);
-		drm_pagemap_put(svm_pages->dpagemap);
-		svm_pages->dpagemap = dpagemap;
-	}
 
 	/* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
 	WRITE_ONCE(svm_pages->flags.__flags, flags.__flags);
@@ -1625,28 +1682,6 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 }
 EXPORT_SYMBOL_GPL(drm_gpusvm_get_pages);
 
-/**
- * drm_gpusvm_range_get_pages() - Get pages for a GPU SVM range
- * @gpusvm: Pointer to the GPU SVM structure
- * @range: Pointer to the GPU SVM range structure
- * @ctx: GPU SVM context
- *
- * This function gets pages for a GPU SVM range and ensures they are mapped for
- * DMA access.
- *
- * Return: 0 on success, negative error code on failure.
- */
-int drm_gpusvm_range_get_pages(struct drm_gpusvm *gpusvm,
-			       struct drm_gpusvm_range *range,
-			       const struct drm_gpusvm_ctx *ctx)
-{
-	return drm_gpusvm_get_pages(gpusvm, &range->pages, gpusvm->mm,
-				    &range->notifier->notifier,
-				    drm_gpusvm_range_start(range),
-				    drm_gpusvm_range_end(range), ctx);
-}
-EXPORT_SYMBOL_GPL(drm_gpusvm_range_get_pages);
-
 /**
  * drm_gpusvm_unmap_pages() - Unmap GPU svm pages
  * @gpusvm: Pointer to the GPU SVM structure
@@ -1677,29 +1712,6 @@ void drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
 }
 EXPORT_SYMBOL_GPL(drm_gpusvm_unmap_pages);
 
-/**
- * drm_gpusvm_range_unmap_pages() - Unmap pages associated with a GPU SVM range
- * @gpusvm: Pointer to the GPU SVM structure
- * @range: Pointer to the GPU SVM range structure
- * @ctx: GPU SVM context
- *
- * This function unmaps pages associated with a GPU SVM range. If @in_notifier
- * is set, it is assumed that gpusvm->notifier_lock is held in write mode; if it
- * is clear, it acquires gpusvm->notifier_lock in read mode. Must be called on
- * each GPU SVM range attached to notifier in gpusvm->ops->invalidate for IOMMU
- * security model.
- */
-void drm_gpusvm_range_unmap_pages(struct drm_gpusvm *gpusvm,
-				  struct drm_gpusvm_range *range,
-				  const struct drm_gpusvm_ctx *ctx)
-{
-	unsigned long npages = npages_in_range(drm_gpusvm_range_start(range),
-					       drm_gpusvm_range_end(range));
-
-	return drm_gpusvm_unmap_pages(gpusvm, &range->pages, npages, ctx);
-}
-EXPORT_SYMBOL_GPL(drm_gpusvm_range_unmap_pages);
-
 /**
  * drm_gpusvm_range_evict() - Evict GPU SVM range
  * @gpusvm: Pointer to the GPU SVM structure
@@ -1720,8 +1732,7 @@ int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm,
 		.end = drm_gpusvm_range_end(range),
 		.dev_private_owner = NULL,
 	};
-	unsigned long timeout =
-		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
+	unsigned long timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
 	unsigned long *pfns;
 	unsigned long npages = npages_in_range(drm_gpusvm_range_start(range),
 					       drm_gpusvm_range_end(range));
@@ -1736,24 +1747,12 @@ int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm,
 		return -ENOMEM;
 
 	hmm_range.hmm_pfns = pfns;
-	while (!time_after(jiffies, timeout)) {
-		hmm_range.notifier_seq = mmu_interval_read_begin(notifier);
-		if (time_after(jiffies, timeout)) {
-			err = -ETIME;
-			break;
-		}
-
-		mmap_read_lock(mm);
-		err = hmm_range_fault(&hmm_range);
-		mmap_read_unlock(mm);
-		if (err != -EBUSY)
-			break;
-	}
+	err = hmm_range_fault_unlocked_timeout(&hmm_range, timeout);
 
 	kvfree(pfns);
 	mmput(mm);
 
-	return err;
+	return err == -EBUSY ? -ETIME : err;
 }
 EXPORT_SYMBOL_GPL(drm_gpusvm_range_evict);
 
@@ -1784,20 +1783,40 @@ EXPORT_SYMBOL_GPL(drm_gpusvm_has_mapping);
 /**
  * drm_gpusvm_range_set_unmapped() - Mark a GPU SVM range as unmapped
  * @range: Pointer to the GPU SVM range structure.
+ * @pages: Pointer to the GPU SVM pages structure(s).
+ * @pages_count: Number of GPU SVM pages structure(s) passed in.
  * @mmu_range: Pointer to the MMU notifier range structure.
  *
  * This function marks a GPU SVM range as unmapped and sets the partial_unmap flag
  * if the range partially falls within the provided MMU notifier range.
  */
 void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range,
+				   struct drm_gpusvm_pages *pages,
+				   unsigned int pages_count,
 				   const struct mmu_notifier_range *mmu_range)
 {
+	struct drm_gpusvm_range_flags range_flags = {
+		.__flags = range->flags.__flags,
+	};
+	unsigned int i;
+
 	lockdep_assert_held_write(&range->gpusvm->notifier_lock);
 
-	range->pages.flags.unmapped = true;
+	range_flags.unmapped = true;
+	for (i = 0; i < pages_count; ++i) {
+		struct drm_gpusvm_pages_flags flags = {
+			.__flags = pages[i].flags.__flags,
+		};
+
+		flags.unmapped = true;
+		/* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
+		WRITE_ONCE(pages[i].flags.__flags, flags.__flags);
+	}
 	if (drm_gpusvm_range_start(range) < mmu_range->start ||
 	    drm_gpusvm_range_end(range) > mmu_range->end)
-		range->pages.flags.partial_unmap = true;
+		range_flags.partial_unmap = true;
+	/* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
+	WRITE_ONCE(range->flags.__flags, range_flags.__flags);
 }
 EXPORT_SYMBOL_GPL(drm_gpusvm_range_set_unmapped);
 
diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c
index c422c5af1f4bf..d1c80ad3deadb 100644
--- a/drivers/gpu/drm/drm_gpuvm.c
+++ b/drivers/gpu/drm/drm_gpuvm.c
@@ -2229,50 +2229,6 @@ drm_gpuva_find(struct drm_gpuvm *gpuvm,
 }
 EXPORT_SYMBOL_GPL(drm_gpuva_find);
 
-/**
- * drm_gpuva_find_prev() - find the &drm_gpuva before the given address
- * @gpuvm: the &drm_gpuvm to search in
- * @start: the given GPU VA's start address
- *
- * Find the adjacent &drm_gpuva before the GPU VA with given &start address.
- *
- * Note that if there is any free space between the GPU VA mappings no mapping
- * is returned.
- *
- * Returns: a pointer to the found &drm_gpuva or NULL if none was found
- */
-struct drm_gpuva *
-drm_gpuva_find_prev(struct drm_gpuvm *gpuvm, u64 start)
-{
-	if (!drm_gpuvm_range_valid(gpuvm, start - 1, 1))
-		return NULL;
-
-	return drm_gpuva_it_iter_first(&gpuvm->rb.tree, start - 1, start);
-}
-EXPORT_SYMBOL_GPL(drm_gpuva_find_prev);
-
-/**
- * drm_gpuva_find_next() - find the &drm_gpuva after the given address
- * @gpuvm: the &drm_gpuvm to search in
- * @end: the given GPU VA's end address
- *
- * Find the adjacent &drm_gpuva after the GPU VA with given &end address.
- *
- * Note that if there is any free space between the GPU VA mappings no mapping
- * is returned.
- *
- * Returns: a pointer to the found &drm_gpuva or NULL if none was found
- */
-struct drm_gpuva *
-drm_gpuva_find_next(struct drm_gpuvm *gpuvm, u64 end)
-{
-	if (!drm_gpuvm_range_valid(gpuvm, end, 1))
-		return NULL;
-
-	return drm_gpuva_it_iter_first(&gpuvm->rb.tree, end, end + 1);
-}
-EXPORT_SYMBOL_GPL(drm_gpuva_find_next);
-
 /**
  * drm_gpuvm_interval_empty() - indicate whether a given interval of the VA space
  * is empty
diff --git a/drivers/gpu/drm/drm_mode_config.c b/drivers/gpu/drm/drm_mode_config.c
index 66f7dc37b5970..3bcc7bf0900c4 100644
--- a/drivers/gpu/drm/drm_mode_config.c
+++ b/drivers/gpu/drm/drm_mode_config.c
@@ -23,6 +23,7 @@
 #include <linux/export.h>
 #include <linux/uaccess.h>
 
+#include <drm/drm_atomic.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_encoder.h>
 #include <drm/drm_file.h>
@@ -182,6 +183,87 @@ int drm_mode_getresources(struct drm_device *dev, void *data,
 	return ret;
 }
 
+static int drm_mode_config_plane_create_state(struct drm_plane *plane)
+{
+	struct drm_plane_state *plane_state;
+
+	if (!plane->funcs->atomic_create_state)
+		return 0;
+
+	plane_state = plane->funcs->atomic_create_state(plane);
+	if (IS_ERR(plane_state))
+		return PTR_ERR(plane_state);
+
+	plane->state = plane_state;
+
+	return 0;
+}
+
+static int drm_mode_config_plane_reset_with_create_state(struct drm_plane *plane)
+{
+	if (plane->state) {
+		plane->funcs->atomic_destroy_state(plane, plane->state);
+		plane->state = NULL;
+	}
+
+	return drm_mode_config_plane_create_state(plane);
+}
+
+static int drm_mode_config_crtc_create_state(struct drm_crtc *crtc)
+{
+	struct drm_crtc_state *crtc_state;
+
+	if (!crtc->funcs->atomic_create_state)
+		return 0;
+
+	crtc_state = crtc->funcs->atomic_create_state(crtc);
+	if (IS_ERR(crtc_state))
+		return PTR_ERR(crtc_state);
+
+	if (drm_dev_has_vblank(crtc->dev))
+		drm_crtc_vblank_reset(crtc);
+
+	crtc->state = crtc_state;
+
+	return 0;
+}
+
+static int drm_mode_config_crtc_reset_with_create_state(struct drm_crtc *crtc)
+{
+	if (crtc->state) {
+		crtc->funcs->atomic_destroy_state(crtc, crtc->state);
+		crtc->state = NULL;
+	}
+
+	return drm_mode_config_crtc_create_state(crtc);
+}
+
+static int drm_mode_config_connector_create_state(struct drm_connector *connector)
+{
+	struct drm_connector_state *conn_state;
+
+	if (!connector->funcs->atomic_create_state)
+		return 0;
+
+	conn_state = connector->funcs->atomic_create_state(connector);
+	if (IS_ERR(conn_state))
+		return PTR_ERR(conn_state);
+
+	connector->state = conn_state;
+
+	return 0;
+}
+
+static int drm_mode_config_connector_reset_with_create_state(struct drm_connector *connector)
+{
+	if (connector->state) {
+		connector->funcs->atomic_destroy_state(connector, connector->state);
+		connector->state = NULL;
+	}
+
+	return drm_mode_config_connector_create_state(connector);
+}
+
 /**
  * drm_mode_config_reset - call ->reset callbacks
  * @dev: drm device
@@ -189,6 +271,10 @@ int drm_mode_getresources(struct drm_device *dev, void *data,
  * This functions calls all the crtc's, encoder's and connector's ->reset
  * callback. Drivers can use this in e.g. their driver load or resume code to
  * reset hardware and software state.
+ *
+ * Note that &drm_private_obj structures are expected to be stable across
+ * suspend/resume cycles, and drm_mode_config_reset() does not affect these
+ * structures.
  */
 void drm_mode_config_reset(struct drm_device *dev)
 {
@@ -202,26 +288,123 @@ void drm_mode_config_reset(struct drm_device *dev)
 	drm_for_each_colorop(colorop, dev)
 		drm_colorop_reset(colorop);
 
-	drm_for_each_plane(plane, dev)
+	drm_for_each_plane(plane, dev) {
 		if (plane->funcs->reset)
 			plane->funcs->reset(plane);
+		else if (plane->funcs->atomic_create_state)
+			drm_mode_config_plane_reset_with_create_state(plane);
+	}
 
-	drm_for_each_crtc(crtc, dev)
+	drm_for_each_crtc(crtc, dev) {
 		if (crtc->funcs->reset)
 			crtc->funcs->reset(crtc);
+		else if (crtc->funcs->atomic_create_state)
+			drm_mode_config_crtc_reset_with_create_state(crtc);
+	}
 
 	drm_for_each_encoder(encoder, dev)
 		if (encoder->funcs && encoder->funcs->reset)
 			encoder->funcs->reset(encoder);
 
 	drm_connector_list_iter_begin(dev, &conn_iter);
-	drm_for_each_connector_iter(connector, &conn_iter)
+	drm_for_each_connector_iter(connector, &conn_iter) {
 		if (connector->funcs->reset)
 			connector->funcs->reset(connector);
+		else if (connector->funcs->atomic_create_state)
+			drm_mode_config_connector_reset_with_create_state(connector);
+	}
 	drm_connector_list_iter_end(&conn_iter);
 }
 EXPORT_SYMBOL(drm_mode_config_reset);
 
+/**
+ * drm_mode_config_create_initial_state - Allocates the initial state
+ * @dev: drm device
+ *
+ * This functions creates the initial state for all the objects. Drivers
+ * can use this in e.g. probe to initialize their software state.
+ *
+ * It has two main differences with drm_mode_config_reset(): the reset()
+ * hooks aren't called and thus the hardware will be left untouched, but
+ * also the &drm_private_obj structures will be initialized as opposed
+ * to drm_mode_config_reset() that skips them.
+ *
+ * Returns: 0 on success, negative error value on failure.
+ */
+int drm_mode_config_create_initial_state(struct drm_device *dev)
+{
+	struct drm_crtc *crtc;
+	struct drm_colorop *colorop;
+	struct drm_plane *plane;
+	struct drm_connector *connector;
+	struct drm_connector_list_iter conn_iter;
+	struct drm_private_obj *privobj;
+	int ret;
+
+	drm_for_each_privobj(privobj, dev) {
+		struct drm_private_state *privobj_state;
+
+		if (privobj->state)
+			continue;
+
+		if (!privobj->funcs->atomic_create_state)
+			continue;
+
+		privobj_state = privobj->funcs->atomic_create_state(privobj);
+		if (IS_ERR(privobj_state))
+			return PTR_ERR(privobj_state);
+
+		privobj->state = privobj_state;
+	}
+
+	drm_for_each_colorop(colorop, dev) {
+		struct drm_colorop_state *colorop_state;
+
+		if (colorop->state)
+			continue;
+
+		colorop_state = drm_atomic_helper_colorop_create_state(colorop);
+		if (IS_ERR(colorop_state))
+			return PTR_ERR(colorop_state);
+
+		colorop->state = colorop_state;
+	}
+
+	drm_for_each_plane(plane, dev) {
+		if (plane->state)
+			continue;
+
+		ret = drm_mode_config_plane_create_state(plane);
+		if (ret)
+			return ret;
+	}
+
+	drm_for_each_crtc(crtc, dev) {
+		if (crtc->state)
+			continue;
+
+		ret = drm_mode_config_crtc_create_state(crtc);
+		if (ret)
+			return ret;
+	}
+
+	drm_connector_list_iter_begin(dev, &conn_iter);
+	drm_for_each_connector_iter(connector, &conn_iter) {
+		if (connector->state)
+			continue;
+
+		ret = drm_mode_config_connector_create_state(connector);
+		if (ret) {
+			drm_connector_list_iter_end(&conn_iter);
+			return ret;
+		}
+	}
+	drm_connector_list_iter_end(&conn_iter);
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_mode_config_create_initial_state);
+
 /*
  * Global properties
  */
@@ -674,6 +857,25 @@ static void validate_encoder_possible_crtcs(struct drm_encoder *encoder)
 	     encoder->possible_crtcs, crtc_mask);
 }
 
+static void validate_blend_mode_for_alpha_formats(struct drm_plane *plane)
+{
+	const struct drm_format_info *fmt;
+	u32 i;
+
+	/* blend mode property supported, no need to check anything */
+	if (plane->blend_mode_property)
+		return;
+
+	for (i = 0; i < plane->format_count; i++) {
+		fmt = drm_format_info(plane->format_types[i]);
+		if (fmt->has_alpha) {
+			WARN(1, "[PLANE:%d:%s] pixel format with alpha exposed but blend mode not setup",
+			     plane->base.id, plane->name);
+			break;
+		}
+	}
+}
+
 void drm_mode_config_validate(struct drm_device *dev)
 {
 	struct drm_encoder *encoder;
@@ -732,6 +934,8 @@ void drm_mode_config_validate(struct drm_device *dev)
 	drm_for_each_plane(plane, dev) {
 		if (plane->type == DRM_PLANE_TYPE_PRIMARY)
 			num_primary++;
+
+		validate_blend_mode_for_alpha_formats(plane);
 	}
 
 	WARN(num_primary != dev->mode_config.num_crtc,
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index 3f8e025fd6d99..e1eed13a8e94f 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -1469,6 +1469,25 @@ struct drm_display_mode *drm_mode_duplicate(struct drm_device *dev,
 }
 EXPORT_SYMBOL(drm_mode_duplicate);
 
+static bool drm_mode_match_timings_vrr(const struct drm_display_mode *mode1,
+				       const struct drm_display_mode *mode2)
+{
+	int mode1_vsync_start_offset = mode1->vtotal - mode1->vsync_start;
+	int mode1_vsync_end_offset = mode1->vtotal - mode1->vsync_end;
+	int mode2_vsync_start_offset = mode2->vtotal - mode2->vsync_start;
+	int mode2_vsync_end_offset = mode2->vtotal - mode2->vsync_end;
+
+	return mode1->hdisplay == mode2->hdisplay &&
+		mode1->hsync_start == mode2->hsync_start &&
+		mode1->hsync_end == mode2->hsync_end &&
+		mode1->htotal == mode2->htotal &&
+		mode1->hskew == mode2->hskew &&
+		mode1->vdisplay == mode2->vdisplay &&
+		mode1_vsync_start_offset == mode2_vsync_start_offset &&
+		mode1_vsync_end_offset == mode2_vsync_end_offset &&
+		mode1->vscan == mode2->vscan;
+}
+
 static bool drm_mode_match_timings(const struct drm_display_mode *mode1,
 				   const struct drm_display_mode *mode2)
 {
@@ -1538,6 +1557,10 @@ bool drm_mode_match(const struct drm_display_mode *mode1,
 	if (!mode1 || !mode2)
 		return false;
 
+	if (match_flags & DRM_MODE_MATCH_TIMINGS_VRR &&
+	    !drm_mode_match_timings_vrr(mode1, mode2))
+		return false;
+
 	if (match_flags & DRM_MODE_MATCH_TIMINGS &&
 	    !drm_mode_match_timings(mode1, mode2))
 		return false;
diff --git a/drivers/gpu/drm/drm_of.c b/drivers/gpu/drm/drm_of.c
index d03ada82eac96..1f4cff6bb4a7a 100644
--- a/drivers/gpu/drm/drm_of.c
+++ b/drivers/gpu/drm/drm_of.c
@@ -7,6 +7,7 @@
 #include <linux/of_graph.h>
 
 #include <drm/drm_bridge.h>
+#include <drm/drm_connector.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_device.h>
 #include <drm/drm_encoder.h>
@@ -220,6 +221,49 @@ int drm_of_encoder_active_endpoint(struct device_node *node,
 }
 EXPORT_SYMBOL_GPL(drm_of_encoder_active_endpoint);
 
+/**
+ * drm_of_get_panel_orientation - look up the orientation of the panel through
+ * the "rotation" binding from a device tree node
+ * @np: device tree node of the panel
+ * @orientation: orientation enum to be filled in
+ *
+ * Looks up the rotation of a panel in the device tree. The orientation of the
+ * panel is expressed as a property name "rotation" in the device tree. The
+ * rotation in the device tree is counter clockwise.
+ *
+ * Return: 0 when a valid rotation value (0, 90, 180, or 270) is read or the
+ * rotation property doesn't exist. Return a negative error code on failure.
+ */
+int drm_of_get_panel_orientation(const struct device_node *np,
+				 enum drm_panel_orientation *orientation)
+{
+	int rotation, ret;
+
+	ret = of_property_read_u32(np, "rotation", &rotation);
+	if (ret == -EINVAL) {
+		/* Don't return an error if there's no rotation property. */
+		*orientation = DRM_MODE_PANEL_ORIENTATION_UNKNOWN;
+		return 0;
+	}
+
+	if (ret < 0)
+		return ret;
+
+	if (rotation == 0)
+		*orientation = DRM_MODE_PANEL_ORIENTATION_NORMAL;
+	else if (rotation == 90)
+		*orientation = DRM_MODE_PANEL_ORIENTATION_RIGHT_UP;
+	else if (rotation == 180)
+		*orientation = DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP;
+	else if (rotation == 270)
+		*orientation = DRM_MODE_PANEL_ORIENTATION_LEFT_UP;
+	else
+		return -EINVAL;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(drm_of_get_panel_orientation);
+
 /**
  * drm_of_find_panel_or_bridge - return connected panel or bridge device
  * @np: device tree node containing encoder output ports
@@ -253,7 +297,7 @@ int drm_of_find_panel_or_bridge(const struct device_node *np,
 	/*
 	 * of_graph_get_remote_node() produces a noisy error message if port
 	 * node isn't found and the absence of the port is a legit case here,
-	 * so at first we silently check whether graph presents in the
+	 * so at first we silently check whether a graph is present in the
 	 * device-tree node.
 	 */
 	if (!of_graph_is_present(np))
diff --git a/drivers/gpu/drm/drm_panel.c b/drivers/gpu/drm/drm_panel.c
index 2c5649e433dfb..d7c6f4824b2de 100644
--- a/drivers/gpu/drm/drm_panel.c
+++ b/drivers/gpu/drm/drm_panel.c
@@ -28,6 +28,7 @@
 #include <linux/of.h>
 
 #include <drm/drm_crtc.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 
@@ -491,49 +492,6 @@ struct drm_panel *of_drm_find_panel(const struct device_node *np)
 	return ERR_PTR(-EPROBE_DEFER);
 }
 EXPORT_SYMBOL(of_drm_find_panel);
-
-/**
- * of_drm_get_panel_orientation - look up the orientation of the panel through
- * the "rotation" binding from a device tree node
- * @np: device tree node of the panel
- * @orientation: orientation enum to be filled in
- *
- * Looks up the rotation of a panel in the device tree. The orientation of the
- * panel is expressed as a property name "rotation" in the device tree. The
- * rotation in the device tree is counter clockwise.
- *
- * Return: 0 when a valid rotation value (0, 90, 180, or 270) is read or the
- * rotation property doesn't exist. Return a negative error code on failure.
- */
-int of_drm_get_panel_orientation(const struct device_node *np,
-				 enum drm_panel_orientation *orientation)
-{
-	int rotation, ret;
-
-	ret = of_property_read_u32(np, "rotation", &rotation);
-	if (ret == -EINVAL) {
-		/* Don't return an error if there's no rotation property. */
-		*orientation = DRM_MODE_PANEL_ORIENTATION_UNKNOWN;
-		return 0;
-	}
-
-	if (ret < 0)
-		return ret;
-
-	if (rotation == 0)
-		*orientation = DRM_MODE_PANEL_ORIENTATION_NORMAL;
-	else if (rotation == 90)
-		*orientation = DRM_MODE_PANEL_ORIENTATION_RIGHT_UP;
-	else if (rotation == 180)
-		*orientation = DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP;
-	else if (rotation == 270)
-		*orientation = DRM_MODE_PANEL_ORIENTATION_LEFT_UP;
-	else
-		return -EINVAL;
-
-	return 0;
-}
-EXPORT_SYMBOL(of_drm_get_panel_orientation);
 #endif
 
 /* Find panel by fwnode. This should be identical to of_drm_find_panel(). */
diff --git a/drivers/gpu/drm/drm_panel_backlight_quirks.c b/drivers/gpu/drm/drm_panel_backlight_quirks.c
index f85cb293a3dbc..e417c7533053d 100644
--- a/drivers/gpu/drm/drm_panel_backlight_quirks.c
+++ b/drivers/gpu/drm/drm_panel_backlight_quirks.c
@@ -20,6 +20,15 @@ struct drm_get_panel_backlight_quirk {
 };
 
 static const struct drm_get_panel_backlight_quirk drm_panel_min_backlight_quirks[] = {
+	/* Lenovo Legion 5 15ARH05, AUX backlight non-functional, force PWM */
+	{
+		.dmi_match.field = DMI_SYS_VENDOR,
+		.dmi_match.value = "LENOVO",
+		.dmi_match_other.field = DMI_PRODUCT_VERSION,
+		.dmi_match_other.value = "Lenovo Legion 5 15ARH05",
+		.ident.panel_id = drm_edid_encode_panel_id('B', 'O', 'E', 0x08df),
+		.quirk = { .force_pwm = true, },
+	},
 	/* 13 inch matte panel */
 	{
 		.dmi_match.field = DMI_BOARD_VENDOR,
diff --git a/drivers/gpu/drm/drm_panic.c b/drivers/gpu/drm/drm_panic.c
index d6d3b8d85dea2..e576c4791861d 100644
--- a/drivers/gpu/drm/drm_panic.c
+++ b/drivers/gpu/drm/drm_panic.c
@@ -443,9 +443,11 @@ static void draw_txt_rectangle(struct drm_scanout_buffer *sb,
 			rec.x1 += (drm_rect_width(clip) - (line_len * font->width)) / 2;
 
 		for (j = 0; j < line_len; j++) {
-			src = drm_draw_get_char_bitmap(font, msg[i].txt[j], font_pitch);
+			src = font_data_glyph_buf(font->data, font->width, font->height,
+						  (unsigned char)msg[i].txt[j]);
 			rec.x2 = rec.x1 + font->width;
-			drm_panic_blit(sb, &rec, src, font_pitch, 1, color);
+			if (src)
+				drm_panic_blit(sb, &rec, src, font_pitch, 1, color);
 			rec.x1 += font->width;
 		}
 	}
diff --git a/drivers/gpu/drm/drm_print.c b/drivers/gpu/drm/drm_print.c
index ded9461df5f25..86cef1a37678e 100644
--- a/drivers/gpu/drm/drm_print.c
+++ b/drivers/gpu/drm/drm_print.c
@@ -50,8 +50,10 @@ MODULE_PARM_DESC(debug, "Enable debug output, where each bit enables a debug cat
 "\t\tBit 3 (0x08)  will enable PRIME messages (prime code)\n"
 "\t\tBit 4 (0x10)  will enable ATOMIC messages (atomic code)\n"
 "\t\tBit 5 (0x20)  will enable VBL messages (vblank code)\n"
+"\t\tBit 6 (0x40)  will enable STATE messages (atomic state code)\n"
 "\t\tBit 7 (0x80)  will enable LEASE messages (leasing code)\n"
-"\t\tBit 8 (0x100) will enable DP messages (displayport code)");
+"\t\tBit 8 (0x100) will enable DP messages (displayport code)\n"
+"\t\tBit 9 (0x200) will enable DRMRES messages (managed resources code)");
 
 #if !defined(CONFIG_DRM_USE_DYNAMIC_DEBUG)
 module_param_named(debug, __drm_debug, ulong, 0600);
diff --git a/drivers/gpu/drm/drm_syncobj.c b/drivers/gpu/drm/drm_syncobj.c
index c9dbf64c0c9f2..2fa170a29a626 100644
--- a/drivers/gpu/drm/drm_syncobj.c
+++ b/drivers/gpu/drm/drm_syncobj.c
@@ -1619,7 +1619,7 @@ drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
 		goto err_points;
 	}
 
-	chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL);
+	chains = kmalloc_objs(*chains, args->count_handles, GFP_KERNEL);
 	if (!chains) {
 		ret = -ENOMEM;
 		goto err_points;
diff --git a/drivers/gpu/drm/drm_writeback.c b/drivers/gpu/drm/drm_writeback.c
index 4da5d60947216..b4a002e6d043d 100644
--- a/drivers/gpu/drm/drm_writeback.c
+++ b/drivers/gpu/drm/drm_writeback.c
@@ -58,6 +58,12 @@
  *	framebuffer to be written by the writeback connector. This property is
  *	similar to the FB_ID property on planes, but will always read as zero
  *	and is not preserved across commits.
+ *	If the width and height of the framebuffer do not match those of the
+ *	attached CRTC, the driver must either fail or scale (not crop) the
+ *	content to exactly fit the framebuffer.
+ *	If the driver is unable to exactly fill the framebuffer for any reason,
+ *	such as hardware scaler constraints or an odd width for a sub-sampled
+ *	format, the writeback must fail instead of partially filling the buffer.
  *	Userspace must set this property to an output buffer every time it
  *	wishes the buffer to get filled.
  *
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
index b0436a1e103fd..f9c8b7b2bfc7a 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
@@ -6,6 +6,7 @@
 #include <drm/drm_prime.h>
 #include <drm/drm_print.h>
 #include <linux/dma-mapping.h>
+#include <linux/pagemap.h>
 #include <linux/shmem_fs.h>
 #include <linux/spinlock.h>
 #include <linux/vmalloc.h>
@@ -132,7 +133,7 @@ static int etnaviv_gem_mmap_obj(struct etnaviv_gem_object *etnaviv_obj,
 
 	vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
 
-	vm_page_prot = vm_get_page_prot(vma->vm_flags);
+	vm_page_prot = vma_get_page_prot(vma);
 
 	if (etnaviv_obj->flags & ETNA_BO_WC) {
 		vma->vm_page_prot = pgprot_writecombine(vm_page_prot);
@@ -188,7 +189,7 @@ static vm_fault_t etnaviv_gem_fault(struct vm_fault *vmf)
 	}
 
 	/* We don't use vmf->pgoff since that has the fake offset: */
-	pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
+	pgoff = linear_page_delta(vma, vmf->address);
 
 	pfn = page_to_pfn(pages[pgoff]);
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index c314b3cb5e709..4dbc8a7ea6af0 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -584,9 +584,9 @@ static int etnaviv_hw_reset(struct etnaviv_gpu *gpu)
 		/* read idle register. */
 		idle = gpu_read(gpu, VIVS_HI_IDLE_STATE);
 
-		/* try resetting again if FE is not idle */
-		if ((idle & VIVS_HI_IDLE_STATE_FE) == 0) {
-			dev_dbg(gpu->dev, "FE is not idle\n");
+		/* try resetting again if any module is not idle */
+		if ((idle & gpu->idle_mask) != gpu->idle_mask) {
+			dev_dbg(gpu->dev, "GPU modules not idle\n");
 			continue;
 		}
 
@@ -600,6 +600,23 @@ static int etnaviv_hw_reset(struct etnaviv_gpu *gpu)
 			continue;
 		}
 
+		/* try resetting again if MMUv2 is not disabled */
+		if (gpu->identity.minor_features1 & chipMinorFeatures1_MMU_VERSION) {
+			if (gpu->sec_mode == ETNA_SEC_KERNEL) {
+				if (gpu_read(gpu, VIVS_MMUv2_SEC_CONTROL) &
+				    VIVS_MMUv2_SEC_CONTROL_ENABLE) {
+					dev_dbg(gpu->dev, "MMU is not disabled\n");
+					continue;
+				}
+			} else {
+				if (gpu_read(gpu, VIVS_MMUv2_CONTROL) &
+				    VIVS_MMUv2_CONTROL_ENABLE) {
+					dev_dbg(gpu->dev, "MMU is not disabled\n");
+					continue;
+				}
+			}
+		}
+
 		/* enable debug register access */
 		control &= ~VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS;
 		gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.c b/drivers/gpu/drm/exynos/exynos_drm_gem.c
index 9a6270f3dca6b..0208c92595725 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gem.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_gem.c
@@ -377,13 +377,13 @@ static int exynos_drm_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct
 
 	/* non-cachable as default. */
 	if (exynos_gem->flags & EXYNOS_BO_CACHABLE)
-		vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+		vma->vm_page_prot = vma_get_page_prot(vma);
 	else if (exynos_gem->flags & EXYNOS_BO_WC)
 		vma->vm_page_prot =
-			pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+			pgprot_writecombine(vma_get_page_prot(vma));
 	else
 		vma->vm_page_prot =
-			pgprot_noncached(vm_get_page_prot(vma->vm_flags));
+			pgprot_noncached(vma_get_page_prot(vma));
 
 	ret = exynos_drm_gem_mmap_buffer(exynos_gem, vma);
 	if (ret)
diff --git a/drivers/gpu/drm/exynos/exynos_drm_mic.c b/drivers/gpu/drm/exynos/exynos_drm_mic.c
index e68c954ec3e61..3069f958137f5 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_mic.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_mic.c
@@ -21,6 +21,7 @@
 #include <video/of_videomode.h>
 #include <video/videomode.h>
 
+#include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_encoder.h>
 #include <drm/drm_print.h>
@@ -228,7 +229,8 @@ static void mic_set_reg_on(struct exynos_mic *mic, bool enable)
 	writel(reg, mic->reg + MIC_OP);
 }
 
-static void mic_post_disable(struct drm_bridge *bridge)
+static void mic_post_disable(struct drm_bridge *bridge,
+			     struct drm_atomic_commit *commit)
 {
 	struct exynos_mic *mic = bridge->driver_private;
 
@@ -257,7 +259,8 @@ static void mic_mode_set(struct drm_bridge *bridge,
 	mutex_unlock(&mic_mutex);
 }
 
-static void mic_pre_enable(struct drm_bridge *bridge)
+static void mic_pre_enable(struct drm_bridge *bridge,
+			   struct drm_atomic_commit *commit)
 {
 	struct exynos_mic *mic = bridge->driver_private;
 	int ret;
@@ -295,9 +298,12 @@ static void mic_pre_enable(struct drm_bridge *bridge)
 }
 
 static const struct drm_bridge_funcs mic_bridge_funcs = {
-	.post_disable = mic_post_disable,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+	.atomic_post_disable = mic_post_disable,
 	.mode_set = mic_mode_set,
-	.pre_enable = mic_pre_enable,
+	.atomic_pre_enable = mic_pre_enable,
 };
 
 static int exynos_mic_bind(struct device *dev, struct device *master,
diff --git a/drivers/gpu/drm/gma500/cdv_intel_crt.c b/drivers/gpu/drm/gma500/cdv_intel_crt.c
index 342a57c828463..e51a965ddeb86 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_crt.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_crt.c
@@ -29,8 +29,8 @@
 #include <linux/pm_runtime.h>
 
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_modeset_helper_vtables.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "cdv_device.h"
 #include "intel_bios.h"
@@ -217,6 +217,10 @@ static int cdv_intel_crt_set_property(struct drm_connector *connector,
  * Routines for controlling stuff on the analog port
  */
 
+static const struct drm_encoder_funcs cdv_intel_crt_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs cdv_intel_crt_helper_funcs = {
 	.dpms = cdv_intel_crt_dpms,
 	.prepare = gma_encoder_prepare,
@@ -275,7 +279,8 @@ void cdv_intel_crt_init(struct drm_device *dev,
 		goto err_ddc_destroy;
 
 	encoder = &gma_encoder->base;
-	ret = drm_simple_encoder_init(dev, encoder, DRM_MODE_ENCODER_DAC);
+	ret = drm_encoder_init(dev, encoder, &cdv_intel_crt_funcs,
+			       DRM_MODE_ENCODER_DAC, NULL);
 	if (ret)
 		goto err_connector_cleanup;
 
diff --git a/drivers/gpu/drm/gma500/cdv_intel_dp.c b/drivers/gpu/drm/gma500/cdv_intel_dp.c
index fbed35cf76037..8d0626ddf1c3a 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_dp.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_dp.c
@@ -33,9 +33,9 @@
 #include <drm/drm_crtc.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_print.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "gma_display.h"
 #include "psb_drv.h"
@@ -232,16 +232,17 @@ i2c_dp_aux_add_bus(struct i2c_adapter *adapter)
 }
 
 #define _wait_for(COND, MS, W) ({ \
-        unsigned long timeout__ = jiffies + msecs_to_jiffies(MS);       \
-        int ret__ = 0;                                                  \
-        while (! (COND)) {                                              \
-                if (time_after(jiffies, timeout__)) {                   \
-                        ret__ = -ETIMEDOUT;                             \
-                        break;                                          \
-                }                                                       \
-                if (W && !in_dbg_master()) msleep(W);                   \
-        }                                                               \
-        ret__;                                                          \
+	unsigned long timeout__ = jiffies + msecs_to_jiffies(MS);       \
+	int ret__ = 0;                                                  \
+	while (!(COND)) {                                               \
+		if (time_after(jiffies, timeout__)) {                   \
+			ret__ = -ETIMEDOUT;                             \
+			break;                                          \
+		}                                                       \
+		if (W && !in_dbg_master())                              \
+			msleep(W);                                      \
+	}                                                               \
+	ret__;                                                          \
 })
 
 #define wait_for(COND, MS) _wait_for(COND, MS, 1)
@@ -296,10 +297,10 @@ static struct ddi_regoff ddi_DP_train_table[] = {
 };
 
 static uint32_t dp_vswing_premph_table[] = {
-        0x55338954,	0x4000,
-        0x554d8954,	0x2000,
-        0x55668954,	0,
-        0x559ac0d4,	0x6000,
+	0x55338954,	0x4000,
+	0x554d8954,	0x2000,
+	0x55668954,	0,
+	0x559ac0d4,	0x6000,
 };
 /**
  * is_edp - is the given port attached to an eDP panel (either CPU or PCH)
@@ -1141,7 +1142,7 @@ static void cdv_intel_dp_prepare(struct drm_encoder *encoder)
 		cdv_intel_edp_backlight_off(intel_encoder);
 		cdv_intel_edp_panel_off(intel_encoder);
 		cdv_intel_edp_panel_vdd_on(intel_encoder);
-        }
+	}
 	/* Wake up the sink first */
 	cdv_intel_dp_sink_dpms(intel_encoder, DRM_MODE_DPMS_ON);
 	cdv_intel_dp_link_down(intel_encoder);
@@ -1183,7 +1184,7 @@ cdv_intel_dp_dpms(struct drm_encoder *encoder, int mode)
 			cdv_intel_edp_panel_off(intel_encoder);
 		}
 	} else {
-        	if (edp)
+		if (edp)
 			cdv_intel_edp_panel_on(intel_encoder);
 		cdv_intel_dp_sink_dpms(intel_encoder, mode);
 		if (!(dp_reg & DP_PORT_EN)) {
@@ -1191,7 +1192,7 @@ cdv_intel_dp_dpms(struct drm_encoder *encoder, int mode)
 			cdv_intel_dp_complete_link_train(intel_encoder);
 		}
 		if (edp)
-        		cdv_intel_edp_backlight_on(intel_encoder);
+			cdv_intel_edp_backlight_on(intel_encoder);
 	}
 }
 
@@ -1419,8 +1420,8 @@ cdv_intel_dp_set_vswing_premph(struct gma_encoder *encoder, uint8_t signal_level
 	DRM_DEBUG_KMS("Test2\n");
 	//return ;
 	cdv_sb_reset(dev);
-	/* ;Swing voltage programming
-        ;gfx_dpio_set_reg(0xc058, 0x0505313A) */
+	/* ;Swing voltage programming */
+	/* ;gfx_dpio_set_reg(0xc058, 0x0505313A) */
 	cdv_sb_write(dev, ddi_reg->VSwing5, 0x0505313A);
 
 	/* ;gfx_dpio_set_reg(0x8154, 0x43406055) */
@@ -1575,7 +1576,7 @@ cdv_intel_dp_complete_link_train(struct gma_encoder *encoder)
 				intel_dp->train_set[0],
 				intel_dp->link_configuration[0],
 				intel_dp->link_configuration[1]);
-        	/* channel eq pattern */
+		/* channel eq pattern */
 
 		if (!cdv_intel_dp_set_link_train(encoder, reg,
 					     DP_TRAINING_PATTERN_2)) {
@@ -1704,7 +1705,7 @@ cdv_intel_dp_detect(struct drm_connector *connector, bool force)
 		if (edp)
 			cdv_intel_edp_panel_vdd_off(encoder);
 		return status;
-        }
+	}
 
 	if (intel_dp->force_audio) {
 		intel_dp->has_audio = intel_dp->force_audio > 0;
@@ -1872,6 +1873,10 @@ cdv_intel_dp_destroy(struct drm_connector *connector)
 	kfree(gma_connector);
 }
 
+static const struct drm_encoder_funcs cdv_intel_dp_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs cdv_intel_dp_helper_funcs = {
 	.dpms = cdv_intel_dp_dpms,
 	.mode_fixup = cdv_intel_dp_mode_fixup,
@@ -1956,12 +1961,12 @@ cdv_intel_dp_init(struct drm_device *dev, struct psb_intel_mode_device *mode_dev
 	gma_encoder = kzalloc_obj(struct gma_encoder);
 	if (!gma_encoder)
 		return;
-        gma_connector = kzalloc_obj(struct gma_connector);
-        if (!gma_connector)
-                goto err_connector;
+	gma_connector = kzalloc_obj(struct gma_connector);
+	if (!gma_connector)
+		goto err_connector;
 	intel_dp = kzalloc_obj(struct cdv_intel_dp);
 	if (!intel_dp)
-	        goto err_priv;
+		goto err_priv;
 
 	if ((output_reg == DP_C) && cdv_intel_dpc_is_edp(dev))
 		type = DRM_MODE_CONNECTOR_eDP;
@@ -1970,13 +1975,14 @@ cdv_intel_dp_init(struct drm_device *dev, struct psb_intel_mode_device *mode_dev
 	encoder = &gma_encoder->base;
 
 	drm_connector_init(dev, connector, &cdv_intel_dp_connector_funcs, type);
-	drm_simple_encoder_init(dev, encoder, DRM_MODE_ENCODER_TMDS);
+	drm_encoder_init(dev, encoder, &cdv_intel_dp_funcs,
+			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	gma_connector_attach_encoder(gma_connector, gma_encoder);
 
 	if (type == DRM_MODE_CONNECTOR_DisplayPort)
 		gma_encoder->type = INTEL_OUTPUT_DISPLAYPORT;
-        else
+	else
 		gma_encoder->type = INTEL_OUTPUT_EDP;
 
 
@@ -2006,18 +2012,18 @@ cdv_intel_dp_init(struct drm_device *dev, struct psb_intel_mode_device *mode_dev
 	cdv_disable_intel_clock_gating(dev);
 
 	cdv_intel_dp_i2c_init(gma_connector, gma_encoder, name);
-        /* FIXME:fail check */
+	/* FIXME:fail check */
 	cdv_intel_dp_add_properties(connector);
 
 	if (is_edp(gma_encoder)) {
 		int ret;
 		struct edp_power_seq cur;
-                u32 pp_on, pp_off, pp_div;
+		u32 pp_on, pp_off, pp_div;
 		u32 pwm_ctrl;
 
 		pp_on = REG_READ(PP_CONTROL);
 		pp_on &= ~PANEL_UNLOCK_MASK;
-	        pp_on |= PANEL_UNLOCK_REGS;
+		pp_on |= PANEL_UNLOCK_REGS;
 
 		REG_WRITE(PP_CONTROL, pp_on);
 
@@ -2025,42 +2031,42 @@ cdv_intel_dp_init(struct drm_device *dev, struct psb_intel_mode_device *mode_dev
 		pwm_ctrl |= PWM_PIPE_B;
 		REG_WRITE(BLC_PWM_CTL2, pwm_ctrl);
 
-                pp_on = REG_READ(PP_ON_DELAYS);
-                pp_off = REG_READ(PP_OFF_DELAYS);
-                pp_div = REG_READ(PP_DIVISOR);
+		pp_on = REG_READ(PP_ON_DELAYS);
+		pp_off = REG_READ(PP_OFF_DELAYS);
+		pp_div = REG_READ(PP_DIVISOR);
 
 		/* Pull timing values out of registers */
-                cur.t1_t3 = (pp_on & PANEL_POWER_UP_DELAY_MASK) >>
-                        PANEL_POWER_UP_DELAY_SHIFT;
+		cur.t1_t3 = (pp_on & PANEL_POWER_UP_DELAY_MASK) >>
+			    PANEL_POWER_UP_DELAY_SHIFT;
 
-                cur.t8 = (pp_on & PANEL_LIGHT_ON_DELAY_MASK) >>
-                        PANEL_LIGHT_ON_DELAY_SHIFT;
+		cur.t8 = (pp_on & PANEL_LIGHT_ON_DELAY_MASK) >>
+			 PANEL_LIGHT_ON_DELAY_SHIFT;
 
-                cur.t9 = (pp_off & PANEL_LIGHT_OFF_DELAY_MASK) >>
-                        PANEL_LIGHT_OFF_DELAY_SHIFT;
+		cur.t9 = (pp_off & PANEL_LIGHT_OFF_DELAY_MASK) >>
+			 PANEL_LIGHT_OFF_DELAY_SHIFT;
 
-                cur.t10 = (pp_off & PANEL_POWER_DOWN_DELAY_MASK) >>
-                        PANEL_POWER_DOWN_DELAY_SHIFT;
+		cur.t10 = (pp_off & PANEL_POWER_DOWN_DELAY_MASK) >>
+			  PANEL_POWER_DOWN_DELAY_SHIFT;
 
-                cur.t11_t12 = ((pp_div & PANEL_POWER_CYCLE_DELAY_MASK) >>
-                               PANEL_POWER_CYCLE_DELAY_SHIFT);
+		cur.t11_t12 = ((pp_div & PANEL_POWER_CYCLE_DELAY_MASK) >>
+			      PANEL_POWER_CYCLE_DELAY_SHIFT);
 
-                DRM_DEBUG_KMS("cur t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n",
-                              cur.t1_t3, cur.t8, cur.t9, cur.t10, cur.t11_t12);
+		DRM_DEBUG_KMS("cur t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n",
+			      cur.t1_t3, cur.t8, cur.t9, cur.t10, cur.t11_t12);
 
 
 		intel_dp->panel_power_up_delay = cur.t1_t3 / 10;
-                intel_dp->backlight_on_delay = cur.t8 / 10;
-                intel_dp->backlight_off_delay = cur.t9 / 10;
-                intel_dp->panel_power_down_delay = cur.t10 / 10;
-                intel_dp->panel_power_cycle_delay = (cur.t11_t12 - 1) * 100;
+		intel_dp->backlight_on_delay = cur.t8 / 10;
+		intel_dp->backlight_off_delay = cur.t9 / 10;
+		intel_dp->panel_power_down_delay = cur.t10 / 10;
+		intel_dp->panel_power_cycle_delay = (cur.t11_t12 - 1) * 100;
 
-                DRM_DEBUG_KMS("panel power up delay %d, power down delay %d, power cycle delay %d\n",
-                              intel_dp->panel_power_up_delay, intel_dp->panel_power_down_delay,
-                              intel_dp->panel_power_cycle_delay);
+		DRM_DEBUG_KMS("panel power up delay %d, power down delay %d, power cycle delay %d\n",
+			      intel_dp->panel_power_up_delay, intel_dp->panel_power_down_delay,
+			      intel_dp->panel_power_cycle_delay);
 
-                DRM_DEBUG_KMS("backlight on delay %d, off delay %d\n",
-                              intel_dp->backlight_on_delay, intel_dp->backlight_off_delay);
+		DRM_DEBUG_KMS("backlight on delay %d, off delay %d\n",
+			      intel_dp->backlight_on_delay, intel_dp->backlight_off_delay);
 
 
 		cdv_intel_edp_panel_vdd_on(gma_encoder);
@@ -2075,7 +2081,7 @@ cdv_intel_dp_init(struct drm_device *dev, struct psb_intel_mode_device *mode_dev
 			cdv_intel_dp_destroy(connector);
 			goto err_connector;
 		} else {
-        		DRM_DEBUG_KMS("DPCD: Rev=%x LN_Rate=%x LN_CNT=%x LN_DOWNSP=%x\n",
+			DRM_DEBUG_KMS("DPCD: Rev=%x LN_Rate=%x LN_CNT=%x LN_DOWNSP=%x\n",
 				intel_dp->dpcd[0], intel_dp->dpcd[1],
 				intel_dp->dpcd[2], intel_dp->dpcd[3]);
 
@@ -2083,7 +2089,7 @@ cdv_intel_dp_init(struct drm_device *dev, struct psb_intel_mode_device *mode_dev
 		/* The CDV reference driver moves pnale backlight setup into the displays that
 		   have a backlight: this is a good idea and one we should probably adopt, however
 		   we need to migrate all the drivers before we can do that */
-                /*cdv_intel_panel_setup_backlight(dev); */
+		/*cdv_intel_panel_setup_backlight(dev); */
 	}
 	return;
 
diff --git a/drivers/gpu/drm/gma500/cdv_intel_hdmi.c b/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
index ce78506477787..757ff9408250a 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
@@ -30,9 +30,9 @@
 #include <drm/drm_crtc.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_print.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "cdv_device.h"
 #include "psb_drv.h"
@@ -251,6 +251,10 @@ static void cdv_hdmi_destroy(struct drm_connector *connector)
 	kfree(gma_connector);
 }
 
+static const struct drm_encoder_funcs cdv_hdmi_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs cdv_hdmi_helper_funcs = {
 	.dpms = cdv_hdmi_dpms,
 	.prepare = gma_encoder_prepare,
@@ -329,8 +333,8 @@ void cdv_hdmi_init(struct drm_device *dev,
 	if (ret)
 		goto err_ddc_destroy;
 
-	ret = drm_simple_encoder_init(dev, &gma_encoder->base,
-				      DRM_MODE_ENCODER_TMDS);
+	ret = drm_encoder_init(dev, &gma_encoder->base, &cdv_hdmi_funcs,
+			       DRM_MODE_ENCODER_TMDS, NULL);
 	if (ret)
 		goto err_connector_cleanup;
 
diff --git a/drivers/gpu/drm/gma500/cdv_intel_lvds.c b/drivers/gpu/drm/gma500/cdv_intel_lvds.c
index d7fd9a783cde4..336ab411d6995 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_lvds.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_lvds.c
@@ -13,9 +13,9 @@
 #include <linux/pm_runtime.h>
 
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_print.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "cdv_device.h"
 #include "intel_bios.h"
@@ -394,6 +394,10 @@ static int cdv_intel_lvds_set_property(struct drm_connector *connector,
 	return 0;
 }
 
+static const struct drm_encoder_funcs cdv_intel_lvds_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs
 					cdv_intel_lvds_helper_funcs = {
 	.dpms = cdv_intel_lvds_encoder_dpms,
@@ -535,7 +539,8 @@ void cdv_intel_lvds_init(struct drm_device *dev,
 	if (ret)
 		goto err_destroy_ddc;
 
-	ret = drm_simple_encoder_init(dev, encoder, DRM_MODE_ENCODER_LVDS);
+	ret = drm_encoder_init(dev, encoder, &cdv_intel_lvds_funcs,
+			       DRM_MODE_ENCODER_LVDS, NULL);
 	if (ret)
 		goto err_connector_cleanup;
 
diff --git a/drivers/gpu/drm/gma500/gem.c b/drivers/gpu/drm/gma500/gem.c
index 88f1e86c8903b..2708e8c68f4c5 100644
--- a/drivers/gpu/drm/gma500/gem.c
+++ b/drivers/gpu/drm/gma500/gem.c
@@ -288,7 +288,7 @@ static vm_fault_t psb_gem_fault(struct vm_fault *vmf)
 
 	/* Page relative to the VMA start - we must calculate this ourselves
 	   because vmf->pgoff is the fake GEM offset */
-	page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
+	page_offset = linear_page_delta(vma, vmf->address);
 
 	/* CPU view of the page, don't go via the GART for CPU writes */
 	if (pobj->stolen)
diff --git a/drivers/gpu/drm/gma500/oaktrail_hdmi.c b/drivers/gpu/drm/gma500/oaktrail_hdmi.c
index 403d21cbb3a23..5a61321760861 100644
--- a/drivers/gpu/drm/gma500/oaktrail_hdmi.c
+++ b/drivers/gpu/drm/gma500/oaktrail_hdmi.c
@@ -29,9 +29,9 @@
 #include <drm/drm.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_print.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "psb_drv.h"
 #include "psb_intel_drv.h"
@@ -605,6 +605,10 @@ static void oaktrail_hdmi_destroy(struct drm_connector *connector)
 	return;
 }
 
+static const struct drm_encoder_funcs oaktrail_hdmi_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs oaktrail_hdmi_helper_funcs = {
 	.dpms = oaktrail_hdmi_dpms,
 	.prepare = gma_encoder_prepare,
@@ -648,7 +652,8 @@ void oaktrail_hdmi_init(struct drm_device *dev,
 			   &oaktrail_hdmi_connector_funcs,
 			   DRM_MODE_CONNECTOR_DVID);
 
-	drm_simple_encoder_init(dev, encoder, DRM_MODE_ENCODER_TMDS);
+	drm_encoder_init(dev, encoder, &oaktrail_hdmi_funcs,
+			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	gma_connector_attach_encoder(gma_connector, gma_encoder);
 
diff --git a/drivers/gpu/drm/gma500/oaktrail_lvds.c b/drivers/gpu/drm/gma500/oaktrail_lvds.c
index e194d0cce0671..425796d8d6c3d 100644
--- a/drivers/gpu/drm/gma500/oaktrail_lvds.c
+++ b/drivers/gpu/drm/gma500/oaktrail_lvds.c
@@ -12,9 +12,9 @@
 #include <linux/pm_runtime.h>
 
 #include <drm/drm_edid.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_print.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "intel_bios.h"
 #include "power.h"
@@ -202,6 +202,10 @@ static void oaktrail_lvds_commit(struct drm_encoder *encoder)
 	oaktrail_lvds_set_power(dev, gma_encoder, true);
 }
 
+static const struct drm_encoder_funcs oaktrail_lvds_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs oaktrail_lvds_helper_funcs = {
 	.dpms = oaktrail_lvds_dpms,
 	.mode_fixup = psb_intel_lvds_mode_fixup,
@@ -319,7 +323,8 @@ void oaktrail_lvds_init(struct drm_device *dev,
 	if (ret)
 		goto err_free_connector;
 
-	ret = drm_simple_encoder_init(dev, encoder, DRM_MODE_ENCODER_LVDS);
+	ret = drm_encoder_init(dev, encoder, &oaktrail_lvds_funcs,
+			       DRM_MODE_ENCODER_LVDS, NULL);
 	if (ret)
 		goto err_connector_cleanup;
 
diff --git a/drivers/gpu/drm/gma500/psb_drv.c b/drivers/gpu/drm/gma500/psb_drv.c
index 947f7c4ff4e9a..d17cb5b4a4bf2 100644
--- a/drivers/gpu/drm/gma500/psb_drv.c
+++ b/drivers/gpu/drm/gma500/psb_drv.c
@@ -55,36 +55,36 @@ static int psb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
  */
 static const struct pci_device_id pciidlist[] = {
 	/* Poulsbo */
-	{ 0x8086, 0x8108, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &psb_chip_ops },
-	{ 0x8086, 0x8109, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &psb_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x8108), .driver_data = (long)&psb_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x8109), .driver_data = (long)&psb_chip_ops },
 	/* Oak Trail */
-	{ 0x8086, 0x4100, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
-	{ 0x8086, 0x4101, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
-	{ 0x8086, 0x4102, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
-	{ 0x8086, 0x4103, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
-	{ 0x8086, 0x4104, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
-	{ 0x8086, 0x4105, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
-	{ 0x8086, 0x4106, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
-	{ 0x8086, 0x4107, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
-	{ 0x8086, 0x4108, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x4100), .driver_data = (long)&oaktrail_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x4101), .driver_data = (long)&oaktrail_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x4102), .driver_data = (long)&oaktrail_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x4103), .driver_data = (long)&oaktrail_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x4104), .driver_data = (long)&oaktrail_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x4105), .driver_data = (long)&oaktrail_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x4106), .driver_data = (long)&oaktrail_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x4107), .driver_data = (long)&oaktrail_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x4108), .driver_data = (long)&oaktrail_chip_ops },
 	/* Cedar Trail */
-	{ 0x8086, 0x0be0, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
-	{ 0x8086, 0x0be1, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
-	{ 0x8086, 0x0be2, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
-	{ 0x8086, 0x0be3, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
-	{ 0x8086, 0x0be4, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
-	{ 0x8086, 0x0be5, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
-	{ 0x8086, 0x0be6, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
-	{ 0x8086, 0x0be7, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
-	{ 0x8086, 0x0be8, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
-	{ 0x8086, 0x0be9, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
-	{ 0x8086, 0x0bea, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
-	{ 0x8086, 0x0beb, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
-	{ 0x8086, 0x0bec, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
-	{ 0x8086, 0x0bed, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
-	{ 0x8086, 0x0bee, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
-	{ 0x8086, 0x0bef, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
-	{ 0, }
+	{ PCI_DEVICE(0x8086, 0x0be0), .driver_data = (long)&cdv_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x0be1), .driver_data = (long)&cdv_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x0be2), .driver_data = (long)&cdv_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x0be3), .driver_data = (long)&cdv_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x0be4), .driver_data = (long)&cdv_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x0be5), .driver_data = (long)&cdv_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x0be6), .driver_data = (long)&cdv_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x0be7), .driver_data = (long)&cdv_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x0be8), .driver_data = (long)&cdv_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x0be9), .driver_data = (long)&cdv_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x0bea), .driver_data = (long)&cdv_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x0beb), .driver_data = (long)&cdv_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x0bec), .driver_data = (long)&cdv_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x0bed), .driver_data = (long)&cdv_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x0bee), .driver_data = (long)&cdv_chip_ops },
+	{ PCI_DEVICE(0x8086, 0x0bef), .driver_data = (long)&cdv_chip_ops },
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, pciidlist);
 
diff --git a/drivers/gpu/drm/gma500/psb_intel_lvds.c b/drivers/gpu/drm/gma500/psb_intel_lvds.c
index 2ca164b212933..cd882786d9461 100644
--- a/drivers/gpu/drm/gma500/psb_intel_lvds.c
+++ b/drivers/gpu/drm/gma500/psb_intel_lvds.c
@@ -12,9 +12,9 @@
 #include <linux/pm_runtime.h>
 
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_print.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "intel_bios.h"
 #include "power.h"
@@ -593,6 +593,10 @@ int psb_intel_lvds_set_property(struct drm_connector *connector,
 	return -1;
 }
 
+static const struct drm_encoder_funcs psb_intel_lvds_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs psb_intel_lvds_helper_funcs = {
 	.dpms = psb_intel_lvds_encoder_dpms,
 	.mode_fixup = psb_intel_lvds_mode_fixup,
@@ -679,7 +683,8 @@ void psb_intel_lvds_init(struct drm_device *dev,
 	if (ret)
 		goto err_ddc_destroy;
 
-	ret = drm_simple_encoder_init(dev, encoder, DRM_MODE_ENCODER_LVDS);
+	ret = drm_encoder_init(dev, encoder, &psb_intel_lvds_funcs,
+			       DRM_MODE_ENCODER_LVDS, NULL);
 	if (ret)
 		goto err_connector_cleanup;
 
diff --git a/drivers/gpu/drm/hisilicon/hibmc/Kconfig b/drivers/gpu/drm/hisilicon/hibmc/Kconfig
index d1f3f5793f340..adf4516bf8f64 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/Kconfig
+++ b/drivers/gpu/drm/hisilicon/hibmc/Kconfig
@@ -5,10 +5,8 @@ config DRM_HISI_HIBMC
 	select DRM_CLIENT_SELECTION
 	select DRM_DISPLAY_HELPER
 	select DRM_DISPLAY_DP_HELPER
+	select DRM_GEM_SHMEM_HELPER
 	select DRM_KMS_HELPER
-	select DRM_VRAM_HELPER
-	select DRM_TTM
-	select DRM_TTM_HELPER
 	select I2C
 	select I2C_ALGOBIT
 	help
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c
index 3066dc9ebc647..b4ab53db1c087 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c
@@ -15,8 +15,10 @@
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_damage_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_gem_vram_helper.h>
+#include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_vblank.h>
 
 #include "hibmc_drm_drv.h"
@@ -72,74 +74,69 @@ static int hibmc_get_best_clock_idx(const struct drm_display_mode *mode)
 static int hibmc_plane_atomic_check(struct drm_plane *plane,
 				    struct drm_atomic_commit *state)
 {
-	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
-										 plane);
-	struct drm_framebuffer *fb = new_plane_state->fb;
-	struct drm_crtc *crtc = new_plane_state->crtc;
-	struct drm_crtc_state *crtc_state;
-	u32 src_w = new_plane_state->src_w >> 16;
-	u32 src_h = new_plane_state->src_h >> 16;
-
-	if (!crtc || !fb)
-		return 0;
-
-	crtc_state = drm_atomic_get_crtc_state(state, crtc);
-	if (IS_ERR(crtc_state))
-		return PTR_ERR(crtc_state);
-
-	if (src_w != new_plane_state->crtc_w || src_h != new_plane_state->crtc_h) {
-		drm_dbg_atomic(plane->dev, "scale not support\n");
-		return -EINVAL;
-	}
+	struct drm_plane_state *new_plane_state =
+		drm_atomic_get_new_plane_state(state, plane);
+	struct drm_crtc_state *new_crtc_state = NULL;
+	int ret;
 
-	if (new_plane_state->crtc_x < 0 || new_plane_state->crtc_y < 0) {
-		drm_dbg_atomic(plane->dev, "crtc_x/y of drm_plane state is invalid\n");
-		return -EINVAL;
-	}
+	if (new_plane_state->crtc)
+		new_crtc_state = drm_atomic_get_new_crtc_state(state, new_plane_state->crtc);
 
-	if (!crtc_state->enable)
+	ret = drm_atomic_helper_check_plane_state(new_plane_state, new_crtc_state,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  false, true);
+	if (ret)
+		return ret;
+	else if (!new_plane_state->visible)
 		return 0;
 
-	if (new_plane_state->crtc_x + new_plane_state->crtc_w >
-	    crtc_state->adjusted_mode.hdisplay ||
-	    new_plane_state->crtc_y + new_plane_state->crtc_h >
-	    crtc_state->adjusted_mode.vdisplay) {
-		drm_dbg_atomic(plane->dev, "visible portion of plane is invalid\n");
-		return -EINVAL;
-	}
-
 	if (new_plane_state->fb->pitches[0] % 128 != 0) {
 		drm_dbg_atomic(plane->dev, "wrong stride with 128-byte aligned\n");
 		return -EINVAL;
 	}
+
 	return 0;
 }
 
 static void hibmc_plane_atomic_update(struct drm_plane *plane,
 				      struct drm_atomic_commit *state)
 {
-	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
-									   plane);
+	struct hibmc_drm_private *priv = to_hibmc_drm_private(plane->dev);
+	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state, plane);
+	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(new_state);
+	struct drm_framebuffer *fb = new_state->fb;
+	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state, plane);
+	u32 gpu_addr = 0;
 	u32 reg;
-	s64 gpu_addr = 0;
 	u32 line_l;
-	struct hibmc_drm_private *priv = to_hibmc_drm_private(plane->dev);
-	struct drm_gem_vram_object *gbo;
 
-	if (!new_state->fb)
+	if (!fb)
 		return;
 
-	gbo = drm_gem_vram_of_gem(new_state->fb->obj[0]);
+	if (drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE) == 0) {
+		struct drm_rect damage;
+		struct drm_atomic_helper_damage_iter iter;
 
-	gpu_addr = drm_gem_vram_offset(gbo);
-	if (WARN_ON_ONCE(gpu_addr < 0))
-		return; /* Bug: we didn't pin the BO to VRAM in prepare_fb. */
+		drm_atomic_helper_damage_iter_init(&iter, old_state, new_state);
+		drm_atomic_for_each_plane_damage(&iter, &damage) {
+			struct iosys_map dst[DRM_FORMAT_MAX_PLANES] = {
+				IOSYS_MAP_INIT_VADDR_IOMEM(priv->vram + gpu_addr),
+			};
+
+			iosys_map_incr(&dst[0],
+				       drm_fb_clip_offset(fb->pitches[0], fb->format, &damage));
+			drm_fb_memcpy(dst, fb->pitches, shadow_plane_state->data, fb, &damage);
+		}
+
+		drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE);
+	}
 
 	writel(gpu_addr, priv->mmio + HIBMC_CRT_FB_ADDRESS);
 
-	reg = new_state->fb->width * (new_state->fb->format->cpp[0]);
+	reg = drm_format_info_min_pitch(fb->format, 0, fb->width);
 
-	line_l = new_state->fb->pitches[0];
+	line_l = fb->pitches[0];
 	writel(HIBMC_FIELD(HIBMC_CRT_FB_WIDTH_WIDTH, reg) |
 	       HIBMC_FIELD(HIBMC_CRT_FB_WIDTH_OFFS, line_l),
 	       priv->mmio + HIBMC_CRT_FB_WIDTH);
@@ -147,29 +144,31 @@ static void hibmc_plane_atomic_update(struct drm_plane *plane,
 	/* SET PIXEL FORMAT */
 	reg = readl(priv->mmio + HIBMC_CRT_DISP_CTL);
 	reg &= ~HIBMC_CRT_DISP_CTL_FORMAT_MASK;
-	reg |= HIBMC_FIELD(HIBMC_CRT_DISP_CTL_FORMAT,
-			   new_state->fb->format->cpp[0] * 8 / 16);
+	switch (fb->format->format) {
+	case DRM_FORMAT_XRGB8888:
+		reg |= HIBMC_FIELD(HIBMC_CRT_DISP_CTL_FORMAT, 2);
+		break;
+	case DRM_FORMAT_RGB565:
+		reg |= HIBMC_FIELD(HIBMC_CRT_DISP_CTL_FORMAT, 1);
+		break;
+	}
 	writel(reg, priv->mmio + HIBMC_CRT_DISP_CTL);
 }
 
 static const u32 channel_formats1[] = {
-	DRM_FORMAT_RGB565, DRM_FORMAT_BGR565, DRM_FORMAT_RGB888,
-	DRM_FORMAT_BGR888, DRM_FORMAT_XRGB8888, DRM_FORMAT_XBGR8888,
-	DRM_FORMAT_RGBA8888, DRM_FORMAT_BGRA8888, DRM_FORMAT_ARGB8888,
-	DRM_FORMAT_ABGR8888
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_RGB565,
 };
 
 static const struct drm_plane_funcs hibmc_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy = drm_plane_cleanup,
-	.reset = drm_atomic_helper_plane_reset,
-	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
+	DRM_GEM_SHADOW_PLANE_FUNCS,
 };
 
 static const struct drm_plane_helper_funcs hibmc_plane_helper_funcs = {
-	DRM_GEM_VRAM_PLANE_HELPER_FUNCS,
+	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS,
 	.atomic_check = hibmc_plane_atomic_check,
 	.atomic_update = hibmc_plane_atomic_update,
 };
@@ -529,6 +528,7 @@ int hibmc_de_init(struct hibmc_drm_private *priv)
 	}
 
 	drm_plane_helper_add(plane, &hibmc_plane_helper_funcs);
+	drm_plane_enable_fb_damage_clips(plane);
 
 	ret = drm_crtc_init_with_planes(dev, crtc, plane,
 					NULL, &hibmc_crtc_funcs, NULL);
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c
index 99b36de1fe137..4d85c89f3f882 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c
@@ -14,13 +14,15 @@
 #include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/pci.h>
+#include <linux/sizes.h>
 
 #include <drm/clients/drm_client_setup.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fbdev_ttm.h>
+#include <drm/drm_dumb_buffers.h>
+#include <drm/drm_fbdev_shmem.h>
 #include <drm/drm_gem_framebuffer_helper.h>
-#include <drm/drm_gem_vram_helper.h>
+#include <drm/drm_gem_shmem_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_module.h>
 #include <drm/drm_vblank.h>
@@ -71,7 +73,13 @@ static irqreturn_t hibmc_dp_interrupt(int irq, void *arg)
 static int hibmc_dumb_create(struct drm_file *file, struct drm_device *dev,
 			     struct drm_mode_create_dumb *args)
 {
-	return drm_gem_vram_fill_create_dumb(file, dev, 0, 128, args);
+	int ret;
+
+	ret = drm_mode_size_dumb(dev, args, SZ_128, 0);
+	if (ret)
+		return ret;
+
+	return drm_gem_shmem_create_with_handle(file, dev, args->size, &args->handle);
 }
 
 static const struct drm_driver hibmc_driver = {
@@ -81,10 +89,9 @@ static const struct drm_driver hibmc_driver = {
 	.desc			= "hibmc drm driver",
 	.major			= 1,
 	.minor			= 0,
-	.debugfs_init		= drm_vram_mm_debugfs_init,
-	.dumb_create            = hibmc_dumb_create,
-	.dumb_map_offset        = drm_gem_ttm_dumb_map_offset,
-	DRM_FBDEV_TTM_DRIVER_OPS,
+	.gem_prime_import	= drm_gem_shmem_prime_import_no_map,
+	.dumb_create		= hibmc_dumb_create,
+	DRM_FBDEV_SHMEM_DRIVER_OPS,
 };
 
 static int __maybe_unused hibmc_pm_suspend(struct device *dev)
@@ -106,11 +113,47 @@ static const struct dev_pm_ops hibmc_pm_ops = {
 				hibmc_pm_resume)
 };
 
+static enum drm_mode_status hibmc_mode_config_mode_valid(struct drm_device *dev,
+							 const struct drm_display_mode *mode)
+{
+	const struct drm_format_info *info =
+		drm_get_format_info(dev, DRM_FORMAT_XRGB8888, DRM_FORMAT_MOD_LINEAR);
+	struct hibmc_drm_private *priv = to_hibmc_drm_private(dev);
+	unsigned long max_fb_size = priv->vram_size;
+	u64 pitch;
+
+	if (drm_WARN_ON_ONCE(dev, !info))
+		return MODE_ERROR; /* driver bug */
+
+	pitch = drm_format_info_min_pitch(info, 0, mode->hdisplay);
+	if (!pitch)
+		return MODE_BAD_WIDTH;
+	else if (pitch > max_fb_size / mode->vdisplay)
+		return MODE_MEM;
+
+	return MODE_OK;
+}
+
+static struct drm_framebuffer *hibmc_mode_config_fb_create(struct drm_device *dev,
+							   struct drm_file *file_priv,
+							   const struct drm_format_info *info,
+							   const struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	int i;
+
+	for (i = 0; i < info->num_planes; ++i) {
+		if (mode_cmd->pitches[i] % SZ_128)
+			return ERR_PTR(-EINVAL);
+	}
+
+	return drm_gem_fb_create_with_dirty(dev, file_priv, info, mode_cmd);
+}
+
 static const struct drm_mode_config_funcs hibmc_mode_funcs = {
-	.mode_valid = drm_vram_helper_mode_valid,
+	.mode_valid = hibmc_mode_config_mode_valid,
 	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = drm_atomic_helper_commit,
-	.fb_create = drm_gem_fb_create,
+	.fb_create = hibmc_mode_config_fb_create,
 };
 
 static int hibmc_kms_init(struct hibmc_drm_private *priv)
@@ -130,7 +173,6 @@ static int hibmc_kms_init(struct hibmc_drm_private *priv)
 	dev->mode_config.max_height = 1200;
 
 	dev->mode_config.preferred_depth = 24;
-	dev->mode_config.prefer_shadow = 1;
 
 	dev->mode_config.funcs = (void *)&hibmc_mode_funcs;
 
@@ -335,18 +377,22 @@ static int hibmc_load(struct drm_device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev->dev);
 	struct hibmc_drm_private *priv = to_hibmc_drm_private(dev);
+	resource_size_t vram_base, vram_size;
 	int ret;
 
 	ret = hibmc_hw_init(priv);
 	if (ret)
 		return ret;
 
-	ret = drmm_vram_helper_init(dev, pci_resource_start(pdev, 0),
-				    pci_resource_len(pdev, 0));
-	if (ret) {
-		drm_err(dev, "Error initializing VRAM MM; %d\n", ret);
-		return ret;
-	}
+	vram_base = pci_resource_start(pdev, 0);
+	vram_size = pci_resource_len(pdev, 0);
+
+	priv->vram = devm_ioremap_wc(dev->dev, vram_base, vram_size);
+	if (!priv->vram)
+		return -ENOMEM;
+
+	priv->vram_base = vram_base;
+	priv->vram_size = vram_size;
 
 	ret = hibmc_kms_init(priv);
 	if (ret)
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h
index cd3a3fca1fe60..dce8572bf63e8 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h
@@ -38,6 +38,11 @@ struct hibmc_drm_private {
 	/* hw */
 	void __iomem   *mmio;
 
+	/* vram */
+	void __iomem *vram;
+	resource_size_t	vram_base;
+	resource_size_t	vram_size;
+
 	/* drm */
 	struct drm_device dev;
 	struct drm_plane primary_plane;
diff --git a/drivers/gpu/drm/hyperv/Kconfig b/drivers/gpu/drm/hyperv/Kconfig
index 86234f6a73f2a..e48e35fb7f8b2 100644
--- a/drivers/gpu/drm/hyperv/Kconfig
+++ b/drivers/gpu/drm/hyperv/Kconfig
@@ -8,7 +8,6 @@ config DRM_HYPERV
 	help
 	 This is a KMS driver for Hyper-V synthetic video device. Choose this
 	 option if you would like to enable drm driver for Hyper-V virtual
-	 machine. Unselect Hyper-V framebuffer driver (CONFIG_FB_HYPERV) so
-	 that DRM driver is used by default.
+	 machine.
 
 	 If M is selected the module will be called hyperv_drm.
diff --git a/drivers/gpu/drm/hyperv/hyperv_drm.h b/drivers/gpu/drm/hyperv/hyperv_drm.h
index 9e776112c03ec..78136ec2c2f4e 100644
--- a/drivers/gpu/drm/hyperv/hyperv_drm.h
+++ b/drivers/gpu/drm/hyperv/hyperv_drm.h
@@ -8,7 +8,7 @@
 
 #define VMBUS_MAX_PACKET_SIZE 0x4000
 
-struct hyperv_drm_device {
+struct hv_drm_device {
 	/* drm */
 	struct drm_device dev;
 	struct drm_plane plane;
@@ -39,17 +39,17 @@ struct hyperv_drm_device {
 	struct hv_device *hdev;
 };
 
-#define to_hv(_dev) container_of(_dev, struct hyperv_drm_device, dev)
+#define to_hv_drm(_dev) container_of(_dev, struct hv_drm_device, dev)
 
 /* hyperv_drm_modeset */
-int hyperv_mode_config_init(struct hyperv_drm_device *hv);
+int hv_drm_mode_config_init(struct hv_drm_device *hv);
 
 /* hyperv_drm_proto */
-int hyperv_update_vram_location(struct hv_device *hdev, phys_addr_t vram_pp);
-int hyperv_update_situation(struct hv_device *hdev, u8 active, u32 bpp,
+int hv_drm_update_vram_location(struct hv_device *hdev, phys_addr_t vram_pp);
+int hv_drm_update_situation(struct hv_device *hdev, u8 active, u32 bpp,
 			    u32 w, u32 h, u32 pitch);
-int hyperv_hide_hw_ptr(struct hv_device *hdev);
-int hyperv_update_dirt(struct hv_device *hdev, struct drm_rect *rect);
-int hyperv_connect_vsp(struct hv_device *hdev);
+int hv_drm_hide_hw_ptr(struct hv_device *hdev);
+int hv_drm_update_dirt(struct hv_device *hdev, struct drm_rect *rect);
+int hv_drm_connect_vsp(struct hv_device *hdev);
 
 #endif
diff --git a/drivers/gpu/drm/hyperv/hyperv_drm_drv.c b/drivers/gpu/drm/hyperv/hyperv_drm_drv.c
index b6bf6412ae34a..20f35c48c0b82 100644
--- a/drivers/gpu/drm/hyperv/hyperv_drm_drv.c
+++ b/drivers/gpu/drm/hyperv/hyperv_drm_drv.c
@@ -24,9 +24,9 @@
 #define DRIVER_MAJOR 1
 #define DRIVER_MINOR 0
 
-DEFINE_DRM_GEM_FOPS(hv_fops);
+DEFINE_DRM_GEM_FOPS(hv_drm_fops);
 
-static struct drm_driver hyperv_driver = {
+static struct drm_driver hv_drm_driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC,
 
 	.name		 = DRIVER_NAME,
@@ -34,22 +34,22 @@ static struct drm_driver hyperv_driver = {
 	.major		 = DRIVER_MAJOR,
 	.minor		 = DRIVER_MINOR,
 
-	.fops		 = &hv_fops,
+	.fops		 = &hv_drm_fops,
 	DRM_GEM_SHMEM_DRIVER_OPS,
 	DRM_FBDEV_SHMEM_DRIVER_OPS,
 };
 
-static int hyperv_pci_probe(struct pci_dev *pdev,
+static int hv_drm_pci_probe(struct pci_dev *pdev,
 			    const struct pci_device_id *ent)
 {
 	return 0;
 }
 
-static void hyperv_pci_remove(struct pci_dev *pdev)
+static void hv_drm_pci_remove(struct pci_dev *pdev)
 {
 }
 
-static const struct pci_device_id hyperv_pci_tbl[] = {
+static const struct pci_device_id hv_drm_pci_tbl[] = {
 	{
 		.vendor = PCI_VENDOR_ID_MICROSOFT,
 		.device = PCI_DEVICE_ID_HYPERV_VIDEO,
@@ -60,14 +60,14 @@ static const struct pci_device_id hyperv_pci_tbl[] = {
 /*
  * PCI stub to support gen1 VM.
  */
-static struct pci_driver hyperv_pci_driver = {
+static struct pci_driver hv_drm_pci_driver = {
 	.name =		KBUILD_MODNAME,
-	.id_table =	hyperv_pci_tbl,
-	.probe =	hyperv_pci_probe,
-	.remove =	hyperv_pci_remove,
+	.id_table =	hv_drm_pci_tbl,
+	.probe =	hv_drm_pci_probe,
+	.remove =	hv_drm_pci_remove,
 };
 
-static int hyperv_setup_vram(struct hyperv_drm_device *hv,
+static int hv_drm_setup_vram(struct hv_drm_device *hv,
 			     struct hv_device *hdev)
 {
 	struct drm_device *dev = &hv->dev;
@@ -102,15 +102,15 @@ static int hyperv_setup_vram(struct hyperv_drm_device *hv,
 	return ret;
 }
 
-static int hyperv_vmbus_probe(struct hv_device *hdev,
+static int hv_drm_vmbus_probe(struct hv_device *hdev,
 			      const struct hv_vmbus_device_id *dev_id)
 {
-	struct hyperv_drm_device *hv;
+	struct hv_drm_device *hv;
 	struct drm_device *dev;
 	int ret;
 
-	hv = devm_drm_dev_alloc(&hdev->device, &hyperv_driver,
-				struct hyperv_drm_device, dev);
+	hv = devm_drm_dev_alloc(&hdev->device, &hv_drm_driver,
+				struct hv_drm_device, dev);
 	if (IS_ERR(hv))
 		return PTR_ERR(hv);
 
@@ -119,15 +119,15 @@ static int hyperv_vmbus_probe(struct hv_device *hdev,
 	hv_set_drvdata(hdev, hv);
 	hv->hdev = hdev;
 
-	ret = hyperv_connect_vsp(hdev);
+	ret = hv_drm_connect_vsp(hdev);
 	if (ret) {
 		drm_err(dev, "Failed to connect to vmbus.\n");
 		goto err_hv_set_drv_data;
 	}
 
-	aperture_remove_all_conflicting_devices(hyperv_driver.name);
+	aperture_remove_all_conflicting_devices(hv_drm_driver.name);
 
-	ret = hyperv_setup_vram(hv, hdev);
+	ret = hv_drm_setup_vram(hv, hdev);
 	if (ret)
 		goto err_vmbus_close;
 
@@ -136,11 +136,11 @@ static int hyperv_vmbus_probe(struct hv_device *hdev,
 	 * vram location is not fatal. Device will update dirty area till
 	 * preferred resolution only.
 	 */
-	ret = hyperv_update_vram_location(hdev, hv->fb_base);
+	ret = hv_drm_update_vram_location(hdev, hv->fb_base);
 	if (ret)
 		drm_warn(dev, "Failed to update vram location.\n");
 
-	ret = hyperv_mode_config_init(hv);
+	ret = hv_drm_mode_config_init(hv);
 	if (ret)
 		goto err_free_mmio;
 
@@ -168,10 +168,10 @@ static int hyperv_vmbus_probe(struct hv_device *hdev,
 	return ret;
 }
 
-static void hyperv_vmbus_remove(struct hv_device *hdev)
+static void hv_drm_vmbus_remove(struct hv_device *hdev)
 {
 	struct drm_device *dev = hv_get_drvdata(hdev);
-	struct hyperv_drm_device *hv = to_hv(dev);
+	struct hv_drm_device *hv = to_hv_drm(dev);
 
 	vmbus_set_skip_unload(false);
 	drm_dev_unplug(dev);
@@ -183,12 +183,12 @@ static void hyperv_vmbus_remove(struct hv_device *hdev)
 	vmbus_free_mmio(hv->mem->start, hv->fb_size);
 }
 
-static void hyperv_vmbus_shutdown(struct hv_device *hdev)
+static void hv_drm_vmbus_shutdown(struct hv_device *hdev)
 {
 	drm_atomic_helper_shutdown(hv_get_drvdata(hdev));
 }
 
-static int hyperv_vmbus_suspend(struct hv_device *hdev)
+static int hv_drm_vmbus_suspend(struct hv_device *hdev)
 {
 	struct drm_device *dev = hv_get_drvdata(hdev);
 	int ret;
@@ -202,67 +202,67 @@ static int hyperv_vmbus_suspend(struct hv_device *hdev)
 	return 0;
 }
 
-static int hyperv_vmbus_resume(struct hv_device *hdev)
+static int hv_drm_vmbus_resume(struct hv_device *hdev)
 {
 	struct drm_device *dev = hv_get_drvdata(hdev);
-	struct hyperv_drm_device *hv = to_hv(dev);
+	struct hv_drm_device *hv = to_hv_drm(dev);
 	int ret;
 
-	ret = hyperv_connect_vsp(hdev);
+	ret = hv_drm_connect_vsp(hdev);
 	if (ret)
 		return ret;
 
-	ret = hyperv_update_vram_location(hdev, hv->fb_base);
+	ret = hv_drm_update_vram_location(hdev, hv->fb_base);
 	if (ret)
 		return ret;
 
 	return drm_mode_config_helper_resume(dev);
 }
 
-static const struct hv_vmbus_device_id hyperv_vmbus_tbl[] = {
+static const struct hv_vmbus_device_id hv_drm_vmbus_tbl[] = {
 	/* Synthetic Video Device GUID */
 	{HV_SYNTHVID_GUID},
 	{}
 };
 
-static struct hv_driver hyperv_hv_driver = {
+static struct hv_driver hv_drm_hv_driver = {
 	.name = KBUILD_MODNAME,
-	.id_table = hyperv_vmbus_tbl,
-	.probe = hyperv_vmbus_probe,
-	.remove = hyperv_vmbus_remove,
-	.shutdown = hyperv_vmbus_shutdown,
-	.suspend = hyperv_vmbus_suspend,
-	.resume = hyperv_vmbus_resume,
+	.id_table = hv_drm_vmbus_tbl,
+	.probe = hv_drm_vmbus_probe,
+	.remove = hv_drm_vmbus_remove,
+	.shutdown = hv_drm_vmbus_shutdown,
+	.suspend = hv_drm_vmbus_suspend,
+	.resume = hv_drm_vmbus_resume,
 	.driver = {
 		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
-static int __init hyperv_init(void)
+static int __init hv_drm_init(void)
 {
 	int ret;
 
 	if (drm_firmware_drivers_only())
 		return -ENODEV;
 
-	ret = pci_register_driver(&hyperv_pci_driver);
+	ret = pci_register_driver(&hv_drm_pci_driver);
 	if (ret != 0)
 		return ret;
 
-	return vmbus_driver_register(&hyperv_hv_driver);
+	return vmbus_driver_register(&hv_drm_hv_driver);
 }
 
-static void __exit hyperv_exit(void)
+static void __exit hv_drm_exit(void)
 {
-	vmbus_driver_unregister(&hyperv_hv_driver);
-	pci_unregister_driver(&hyperv_pci_driver);
+	vmbus_driver_unregister(&hv_drm_hv_driver);
+	pci_unregister_driver(&hv_drm_pci_driver);
 }
 
-module_init(hyperv_init);
-module_exit(hyperv_exit);
+module_init(hv_drm_init);
+module_exit(hv_drm_exit);
 
-MODULE_DEVICE_TABLE(pci, hyperv_pci_tbl);
-MODULE_DEVICE_TABLE(vmbus, hyperv_vmbus_tbl);
+MODULE_DEVICE_TABLE(pci, hv_drm_pci_tbl);
+MODULE_DEVICE_TABLE(vmbus, hv_drm_vmbus_tbl);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Deepak Rawat <drawat.floss@gmail.com>");
 MODULE_DESCRIPTION("DRM driver for Hyper-V synthetic video device");
diff --git a/drivers/gpu/drm/hyperv/hyperv_drm_modeset.c b/drivers/gpu/drm/hyperv/hyperv_drm_modeset.c
index 793dbbf618931..1855749c1e41f 100644
--- a/drivers/gpu/drm/hyperv/hyperv_drm_modeset.c
+++ b/drivers/gpu/drm/hyperv/hyperv_drm_modeset.c
@@ -25,11 +25,11 @@
 
 #include "hyperv_drm.h"
 
-static int hyperv_blit_to_vram_rect(struct drm_framebuffer *fb,
+static int hv_drm_blit_to_vram_rect(struct drm_framebuffer *fb,
 				    const struct iosys_map *vmap,
 				    struct drm_rect *rect)
 {
-	struct hyperv_drm_device *hv = to_hv(fb->dev);
+	struct hv_drm_device *hv = to_hv_drm(fb->dev);
 	struct iosys_map dst = IOSYS_MAP_INIT_VADDR_IOMEM(hv->vram);
 	int idx;
 
@@ -44,9 +44,9 @@ static int hyperv_blit_to_vram_rect(struct drm_framebuffer *fb,
 	return 0;
 }
 
-static int hyperv_connector_get_modes(struct drm_connector *connector)
+static int hv_drm_connector_get_modes(struct drm_connector *connector)
 {
-	struct hyperv_drm_device *hv = to_hv(connector->dev);
+	struct hv_drm_device *hv = to_hv_drm(connector->dev);
 	int count;
 
 	count = drm_add_modes_noedid(connector,
@@ -58,11 +58,11 @@ static int hyperv_connector_get_modes(struct drm_connector *connector)
 	return count;
 }
 
-static const struct drm_connector_helper_funcs hyperv_connector_helper_funcs = {
-	.get_modes = hyperv_connector_get_modes,
+static const struct drm_connector_helper_funcs hv_drm_connector_helper_funcs = {
+	.get_modes = hv_drm_connector_get_modes,
 };
 
-static const struct drm_connector_funcs hyperv_connector_funcs = {
+static const struct drm_connector_funcs hv_drm_connector_funcs = {
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
@@ -70,15 +70,15 @@ static const struct drm_connector_funcs hyperv_connector_funcs = {
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 };
 
-static inline int hyperv_conn_init(struct hyperv_drm_device *hv)
+static inline int hv_drm_conn_init(struct hv_drm_device *hv)
 {
-	drm_connector_helper_add(&hv->connector, &hyperv_connector_helper_funcs);
+	drm_connector_helper_add(&hv->connector, &hv_drm_connector_helper_funcs);
 	return drm_connector_init(&hv->dev, &hv->connector,
-				  &hyperv_connector_funcs,
+				  &hv_drm_connector_funcs,
 				  DRM_MODE_CONNECTOR_VIRTUAL);
 }
 
-static int hyperv_check_size(struct hyperv_drm_device *hv, int w, int h,
+static int hv_drm_check_size(struct hv_drm_device *hv, int w, int h,
 			     struct drm_framebuffer *fb)
 {
 	u32 pitch = w * (hv->screen_depth / 8);
@@ -92,25 +92,25 @@ static int hyperv_check_size(struct hyperv_drm_device *hv, int w, int h,
 	return 0;
 }
 
-static const uint32_t hyperv_formats[] = {
+static const uint32_t hv_drm_formats[] = {
 	DRM_FORMAT_XRGB8888,
 };
 
-static const uint64_t hyperv_modifiers[] = {
+static const uint64_t hv_drm_modifiers[] = {
 	DRM_FORMAT_MOD_LINEAR,
 	DRM_FORMAT_MOD_INVALID
 };
 
-static void hyperv_crtc_helper_atomic_enable(struct drm_crtc *crtc,
+static void hv_drm_crtc_helper_atomic_enable(struct drm_crtc *crtc,
 					     struct drm_atomic_commit *state)
 {
-	struct hyperv_drm_device *hv = to_hv(crtc->dev);
+	struct hv_drm_device *hv = to_hv_drm(crtc->dev);
 	struct drm_plane *plane = &hv->plane;
 	struct drm_plane_state *plane_state = plane->state;
 	struct drm_crtc_state *crtc_state = crtc->state;
 
-	hyperv_hide_hw_ptr(hv->hdev);
-	hyperv_update_situation(hv->hdev, 1,  hv->screen_depth,
+	hv_drm_hide_hw_ptr(hv->hdev);
+	hv_drm_update_situation(hv->hdev, 1,  hv->screen_depth,
 				crtc_state->mode.hdisplay,
 				crtc_state->mode.vdisplay,
 				plane_state->fb->pitches[0]);
@@ -118,14 +118,14 @@ static void hyperv_crtc_helper_atomic_enable(struct drm_crtc *crtc,
 	drm_crtc_vblank_on(crtc);
 }
 
-static const struct drm_crtc_helper_funcs hyperv_crtc_helper_funcs = {
+static const struct drm_crtc_helper_funcs hv_drm_crtc_helper_funcs = {
 	.atomic_check = drm_crtc_helper_atomic_check,
 	.atomic_flush = drm_crtc_vblank_atomic_flush,
-	.atomic_enable = hyperv_crtc_helper_atomic_enable,
+	.atomic_enable = hv_drm_crtc_helper_atomic_enable,
 	.atomic_disable = drm_crtc_vblank_atomic_disable,
 };
 
-static const struct drm_crtc_funcs hyperv_crtc_funcs = {
+static const struct drm_crtc_funcs hv_drm_crtc_funcs = {
 	.reset = drm_atomic_helper_crtc_reset,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
@@ -135,11 +135,11 @@ static const struct drm_crtc_funcs hyperv_crtc_funcs = {
 	DRM_CRTC_VBLANK_TIMER_FUNCS,
 };
 
-static int hyperv_plane_atomic_check(struct drm_plane *plane,
+static int hv_drm_plane_atomic_check(struct drm_plane *plane,
 				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
-	struct hyperv_drm_device *hv = to_hv(plane->dev);
+	struct hv_drm_device *hv = to_hv_drm(plane->dev);
 	struct drm_framebuffer *fb = plane_state->fb;
 	struct drm_crtc *crtc = plane_state->crtc;
 	struct drm_crtc_state *crtc_state = NULL;
@@ -167,10 +167,10 @@ static int hyperv_plane_atomic_check(struct drm_plane *plane,
 	return 0;
 }
 
-static void hyperv_plane_atomic_update(struct drm_plane *plane,
+static void hv_drm_plane_atomic_update(struct drm_plane *plane,
 				       struct drm_atomic_commit *state)
 {
-	struct hyperv_drm_device *hv = to_hv(plane->dev);
+	struct hv_drm_device *hv = to_hv_drm(plane->dev);
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state, plane);
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state, plane);
 	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(new_state);
@@ -185,15 +185,15 @@ static void hyperv_plane_atomic_update(struct drm_plane *plane,
 		if (!drm_rect_intersect(&dst_clip, &damage))
 			continue;
 
-		hyperv_blit_to_vram_rect(new_state->fb, &shadow_plane_state->data[0], &damage);
-		hyperv_update_dirt(hv->hdev, &damage);
+		hv_drm_blit_to_vram_rect(new_state->fb, &shadow_plane_state->data[0], &damage);
+		hv_drm_update_dirt(hv->hdev, &damage);
 	}
 }
 
-static int hyperv_plane_get_scanout_buffer(struct drm_plane *plane,
+static int hv_drm_plane_get_scanout_buffer(struct drm_plane *plane,
 					   struct drm_scanout_buffer *sb)
 {
-	struct hyperv_drm_device *hv = to_hv(plane->dev);
+	struct hv_drm_device *hv = to_hv_drm(plane->dev);
 	struct iosys_map map = IOSYS_MAP_INIT_VADDR_IOMEM(hv->vram);
 
 	if (plane->state && plane->state->fb) {
@@ -207,9 +207,9 @@ static int hyperv_plane_get_scanout_buffer(struct drm_plane *plane,
 	return -ENODEV;
 }
 
-static void hyperv_plane_panic_flush(struct drm_plane *plane)
+static void hv_drm_plane_panic_flush(struct drm_plane *plane)
 {
-	struct hyperv_drm_device *hv = to_hv(plane->dev);
+	struct hv_drm_device *hv = to_hv_drm(plane->dev);
 	struct drm_rect rect;
 
 	if (plane->state && plane->state->fb) {
@@ -218,32 +218,32 @@ static void hyperv_plane_panic_flush(struct drm_plane *plane)
 		rect.x2 = plane->state->fb->width;
 		rect.y2 = plane->state->fb->height;
 
-		hyperv_update_dirt(hv->hdev, &rect);
+		hv_drm_update_dirt(hv->hdev, &rect);
 	}
 
 	vmbus_initiate_unload(true);
 }
 
-static const struct drm_plane_helper_funcs hyperv_plane_helper_funcs = {
+static const struct drm_plane_helper_funcs hv_drm_plane_helper_funcs = {
 	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS,
-	.atomic_check = hyperv_plane_atomic_check,
-	.atomic_update = hyperv_plane_atomic_update,
-	.get_scanout_buffer = hyperv_plane_get_scanout_buffer,
-	.panic_flush = hyperv_plane_panic_flush,
+	.atomic_check = hv_drm_plane_atomic_check,
+	.atomic_update = hv_drm_plane_atomic_update,
+	.get_scanout_buffer = hv_drm_plane_get_scanout_buffer,
+	.panic_flush = hv_drm_plane_panic_flush,
 };
 
-static const struct drm_plane_funcs hyperv_plane_funcs = {
+static const struct drm_plane_funcs hv_drm_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
 	.destroy		= drm_plane_cleanup,
 	DRM_GEM_SHADOW_PLANE_FUNCS,
 };
 
-static const struct drm_encoder_funcs hyperv_drm_simple_encoder_funcs_cleanup = {
+static const struct drm_encoder_funcs hv_drm_simple_encoder_funcs_cleanup = {
 	.destroy = drm_encoder_cleanup,
 };
 
-static inline int hyperv_pipe_init(struct hyperv_drm_device *hv)
+static inline int hv_drm_pipe_init(struct hv_drm_device *hv)
 {
 	struct drm_device *dev = &hv->dev;
 	struct drm_encoder *encoder = &hv->encoder;
@@ -253,29 +253,29 @@ static inline int hyperv_pipe_init(struct hyperv_drm_device *hv)
 	int ret;
 
 	ret = drm_universal_plane_init(dev, plane, 0,
-				       &hyperv_plane_funcs,
-				       hyperv_formats, ARRAY_SIZE(hyperv_formats),
-				       hyperv_modifiers,
+				       &hv_drm_plane_funcs,
+				       hv_drm_formats, ARRAY_SIZE(hv_drm_formats),
+				       hv_drm_modifiers,
 				       DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (ret)
 		return ret;
-	drm_plane_helper_add(plane, &hyperv_plane_helper_funcs);
+	drm_plane_helper_add(plane, &hv_drm_plane_helper_funcs);
 	drm_plane_enable_fb_damage_clips(plane);
 
 	ret = drm_crtc_init_with_planes(dev, crtc, plane, NULL,
-					&hyperv_crtc_funcs, NULL);
+					&hv_drm_crtc_funcs, NULL);
 	if (ret)
 		return ret;
-	drm_crtc_helper_add(crtc, &hyperv_crtc_helper_funcs);
+	drm_crtc_helper_add(crtc, &hv_drm_crtc_helper_funcs);
 
 	encoder->possible_crtcs = drm_crtc_mask(crtc);
 	ret = drm_encoder_init(dev, encoder,
-			       &hyperv_drm_simple_encoder_funcs_cleanup,
+			       &hv_drm_simple_encoder_funcs_cleanup,
 			       DRM_MODE_ENCODER_NONE, NULL);
 	if (ret)
 		return ret;
 
-	ret = hyperv_conn_init(hv);
+	ret = hv_drm_conn_init(hv);
 	if (ret) {
 		drm_err(dev, "Failed to initialized connector.\n");
 		return ret;
@@ -285,25 +285,25 @@ static inline int hyperv_pipe_init(struct hyperv_drm_device *hv)
 }
 
 static enum drm_mode_status
-hyperv_mode_valid(struct drm_device *dev,
+hv_drm_mode_valid(struct drm_device *dev,
 		  const struct drm_display_mode *mode)
 {
-	struct hyperv_drm_device *hv = to_hv(dev);
+	struct hv_drm_device *hv = to_hv_drm(dev);
 
-	if (hyperv_check_size(hv, mode->hdisplay, mode->vdisplay, NULL))
+	if (hv_drm_check_size(hv, mode->hdisplay, mode->vdisplay, NULL))
 		return MODE_BAD;
 
 	return MODE_OK;
 }
 
-static const struct drm_mode_config_funcs hyperv_mode_config_funcs = {
+static const struct drm_mode_config_funcs hv_drm_mode_config_funcs = {
 	.fb_create = drm_gem_fb_create_with_dirty,
-	.mode_valid = hyperv_mode_valid,
+	.mode_valid = hv_drm_mode_valid,
 	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
 
-int hyperv_mode_config_init(struct hyperv_drm_device *hv)
+int hv_drm_mode_config_init(struct hv_drm_device *hv)
 {
 	struct drm_device *dev = &hv->dev;
 	int ret;
@@ -322,9 +322,9 @@ int hyperv_mode_config_init(struct hyperv_drm_device *hv)
 	dev->mode_config.preferred_depth = hv->screen_depth;
 	dev->mode_config.prefer_shadow = 0;
 
-	dev->mode_config.funcs = &hyperv_mode_config_funcs;
+	dev->mode_config.funcs = &hv_drm_mode_config_funcs;
 
-	ret = hyperv_pipe_init(hv);
+	ret = hv_drm_pipe_init(hv);
 	if (ret) {
 		drm_err(dev, "Failed to initialized pipe.\n");
 		return ret;
diff --git a/drivers/gpu/drm/hyperv/hyperv_drm_proto.c b/drivers/gpu/drm/hyperv/hyperv_drm_proto.c
index 6e09b0218df4e..f0ef627b4898a 100644
--- a/drivers/gpu/drm/hyperv/hyperv_drm_proto.c
+++ b/drivers/gpu/drm/hyperv/hyperv_drm_proto.c
@@ -181,7 +181,7 @@ struct synthvid_msg {
 	};
 } __packed;
 
-static inline bool hyperv_version_ge(u32 ver1, u32 ver2)
+static inline bool hv_drm_version_ge(u32 ver1, u32 ver2)
 {
 	if (SYNTHVID_VER_GET_MAJOR(ver1) > SYNTHVID_VER_GET_MAJOR(ver2) ||
 	    (SYNTHVID_VER_GET_MAJOR(ver1) == SYNTHVID_VER_GET_MAJOR(ver2) &&
@@ -191,10 +191,10 @@ static inline bool hyperv_version_ge(u32 ver1, u32 ver2)
 	return false;
 }
 
-static inline int hyperv_sendpacket(struct hv_device *hdev, struct synthvid_msg *msg)
+static inline int hv_drm_sendpacket(struct hv_device *hdev, struct synthvid_msg *msg)
 {
 	static atomic64_t request_id = ATOMIC64_INIT(0);
-	struct hyperv_drm_device *hv = hv_get_drvdata(hdev);
+	struct hv_drm_device *hv = hv_get_drvdata(hdev);
 	int ret;
 
 	msg->pipe_hdr.type = PIPE_MSG_DATA;
@@ -211,9 +211,9 @@ static inline int hyperv_sendpacket(struct hv_device *hdev, struct synthvid_msg
 	return ret;
 }
 
-static int hyperv_negotiate_version(struct hv_device *hdev, u32 ver)
+static int hv_drm_negotiate_version(struct hv_device *hdev, u32 ver)
 {
-	struct hyperv_drm_device *hv = hv_get_drvdata(hdev);
+	struct hv_drm_device *hv = hv_get_drvdata(hdev);
 	struct synthvid_msg *msg = (struct synthvid_msg *)hv->init_buf;
 	struct drm_device *dev = &hv->dev;
 	unsigned long t;
@@ -223,7 +223,7 @@ static int hyperv_negotiate_version(struct hv_device *hdev, u32 ver)
 	msg->vid_hdr.size = sizeof(struct synthvid_msg_hdr) +
 		sizeof(struct synthvid_version_req);
 	msg->ver_req.version = ver;
-	hyperv_sendpacket(hdev, msg);
+	hv_drm_sendpacket(hdev, msg);
 
 	t = wait_for_completion_timeout(&hv->wait, VMBUS_VSP_TIMEOUT);
 	if (!t) {
@@ -243,9 +243,9 @@ static int hyperv_negotiate_version(struct hv_device *hdev, u32 ver)
 	return 0;
 }
 
-int hyperv_update_vram_location(struct hv_device *hdev, phys_addr_t vram_pp)
+int hv_drm_update_vram_location(struct hv_device *hdev, phys_addr_t vram_pp)
 {
-	struct hyperv_drm_device *hv = hv_get_drvdata(hdev);
+	struct hv_drm_device *hv = hv_get_drvdata(hdev);
 	struct synthvid_msg *msg = (struct synthvid_msg *)hv->init_buf;
 	struct drm_device *dev = &hv->dev;
 	unsigned long t;
@@ -257,7 +257,7 @@ int hyperv_update_vram_location(struct hv_device *hdev, phys_addr_t vram_pp)
 	msg->vram.user_ctx = vram_pp;
 	msg->vram.vram_gpa = vram_pp;
 	msg->vram.is_vram_gpa_specified = 1;
-	hyperv_sendpacket(hdev, msg);
+	hv_drm_sendpacket(hdev, msg);
 
 	t = wait_for_completion_timeout(&hv->wait, VMBUS_VSP_TIMEOUT);
 	if (!t) {
@@ -272,7 +272,7 @@ int hyperv_update_vram_location(struct hv_device *hdev, phys_addr_t vram_pp)
 	return 0;
 }
 
-int hyperv_update_situation(struct hv_device *hdev, u8 active, u32 bpp,
+int hv_drm_update_situation(struct hv_device *hdev, u8 active, u32 bpp,
 			    u32 w, u32 h, u32 pitch)
 {
 	struct synthvid_msg msg;
@@ -292,7 +292,7 @@ int hyperv_update_situation(struct hv_device *hdev, u8 active, u32 bpp,
 	msg.situ.video_output[0].height_pixels = h;
 	msg.situ.video_output[0].pitch_bytes = pitch;
 
-	hyperv_sendpacket(hdev, &msg);
+	hv_drm_sendpacket(hdev, &msg);
 
 	return 0;
 }
@@ -306,11 +306,11 @@ int hyperv_update_situation(struct hv_device *hdev, u8 active, u32 bpp,
  * the msg.ptr_shape.data. Note: setting msg.ptr_pos.is_visible to 0 doesn't
  * work in tests.
  *
- * The hyperv_hide_hw_ptr() is also called in the handler of the
+ * The hv_drm_hide_hw_ptr() is also called in the handler of the
  * SYNTHVID_FEATURE_CHANGE event, otherwise the host still draws an extra
  * unwanted mouse pointer after the VM Connection window is closed and reopened.
  */
-int hyperv_hide_hw_ptr(struct hv_device *hdev)
+int hv_drm_hide_hw_ptr(struct hv_device *hdev)
 {
 	struct synthvid_msg msg;
 
@@ -322,7 +322,7 @@ int hyperv_hide_hw_ptr(struct hv_device *hdev)
 	msg.ptr_pos.video_output = 0;
 	msg.ptr_pos.image_x = 0;
 	msg.ptr_pos.image_y = 0;
-	hyperv_sendpacket(hdev, &msg);
+	hv_drm_sendpacket(hdev, &msg);
 
 	memset(&msg, 0, sizeof(struct synthvid_msg));
 	msg.vid_hdr.type = SYNTHVID_POINTER_SHAPE;
@@ -338,14 +338,14 @@ int hyperv_hide_hw_ptr(struct hv_device *hdev)
 	msg.ptr_shape.data[1] = 1;
 	msg.ptr_shape.data[2] = 1;
 	msg.ptr_shape.data[3] = 1;
-	hyperv_sendpacket(hdev, &msg);
+	hv_drm_sendpacket(hdev, &msg);
 
 	return 0;
 }
 
-int hyperv_update_dirt(struct hv_device *hdev, struct drm_rect *rect)
+int hv_drm_update_dirt(struct hv_device *hdev, struct drm_rect *rect)
 {
-	struct hyperv_drm_device *hv = hv_get_drvdata(hdev);
+	struct hv_drm_device *hv = hv_get_drvdata(hdev);
 	struct synthvid_msg msg;
 
 	if (!hv->dirt_needed)
@@ -363,14 +363,14 @@ int hyperv_update_dirt(struct hv_device *hdev, struct drm_rect *rect)
 	msg.dirt.rect[0].x2 = rect->x2;
 	msg.dirt.rect[0].y2 = rect->y2;
 
-	hyperv_sendpacket(hdev, &msg);
+	hv_drm_sendpacket(hdev, &msg);
 
 	return 0;
 }
 
-static int hyperv_get_supported_resolution(struct hv_device *hdev)
+static int hv_drm_get_supported_resolution(struct hv_device *hdev)
 {
-	struct hyperv_drm_device *hv = hv_get_drvdata(hdev);
+	struct hv_drm_device *hv = hv_get_drvdata(hdev);
 	struct synthvid_msg *msg = (struct synthvid_msg *)hv->init_buf;
 	struct drm_device *dev = &hv->dev;
 	unsigned long t;
@@ -383,7 +383,7 @@ static int hyperv_get_supported_resolution(struct hv_device *hdev)
 		sizeof(struct synthvid_supported_resolution_req);
 	msg->resolution_req.maximum_resolution_count =
 		SYNTHVID_MAX_RESOLUTION_COUNT;
-	hyperv_sendpacket(hdev, msg);
+	hv_drm_sendpacket(hdev, msg);
 
 	t = wait_for_completion_timeout(&hv->wait, VMBUS_VSP_TIMEOUT);
 	if (!t) {
@@ -420,9 +420,9 @@ static int hyperv_get_supported_resolution(struct hv_device *hdev)
 	return 0;
 }
 
-static void hyperv_receive_sub(struct hv_device *hdev, u32 bytes_recvd)
+static void hv_drm_receive_sub(struct hv_device *hdev, u32 bytes_recvd)
 {
-	struct hyperv_drm_device *hv = hv_get_drvdata(hdev);
+	struct hv_drm_device *hv = hv_get_drvdata(hdev);
 	struct synthvid_msg *msg;
 	size_t hdr_size;
 	size_t need;
@@ -486,7 +486,7 @@ static void hyperv_receive_sub(struct hv_device *hdev, u32 bytes_recvd)
 		}
 		hv->dirt_needed = msg->feature_chg.is_dirt_needed;
 		if (hv->dirt_needed)
-			hyperv_hide_hw_ptr(hv->hdev);
+			hv_drm_hide_hw_ptr(hv->hdev);
 		return;
 	default:
 		return;
@@ -508,10 +508,10 @@ static void hyperv_receive_sub(struct hv_device *hdev, u32 bytes_recvd)
 	complete(&hv->wait);
 }
 
-static void hyperv_receive(void *ctx)
+static void hv_drm_receive(void *ctx)
 {
 	struct hv_device *hdev = ctx;
-	struct hyperv_drm_device *hv = hv_get_drvdata(hdev);
+	struct hv_drm_device *hv = hv_get_drvdata(hdev);
 	struct synthvid_msg *recv_buf;
 	u32 bytes_recvd;
 	u64 req_id;
@@ -539,19 +539,19 @@ static void hyperv_receive(void *ctx)
 					    ret, bytes_recvd);
 		} else if (bytes_recvd > 0 &&
 			   recv_buf->pipe_hdr.type == PIPE_MSG_DATA) {
-			hyperv_receive_sub(hdev, bytes_recvd);
+			hv_drm_receive_sub(hdev, bytes_recvd);
 		}
 	} while (bytes_recvd > 0 && ret == 0);
 }
 
-int hyperv_connect_vsp(struct hv_device *hdev)
+int hv_drm_connect_vsp(struct hv_device *hdev)
 {
-	struct hyperv_drm_device *hv = hv_get_drvdata(hdev);
+	struct hv_drm_device *hv = hv_get_drvdata(hdev);
 	struct drm_device *dev = &hv->dev;
 	int ret;
 
 	ret = vmbus_open(hdev->channel, VMBUS_RING_BUFSIZE, VMBUS_RING_BUFSIZE,
-			 NULL, 0, hyperv_receive, hdev);
+			 NULL, 0, hv_drm_receive, hdev);
 	if (ret) {
 		drm_err(dev, "Unable to open vmbus channel\n");
 		return ret;
@@ -561,16 +561,16 @@ int hyperv_connect_vsp(struct hv_device *hdev)
 	switch (vmbus_proto_version) {
 	case VERSION_WIN10:
 	case VERSION_WIN10_V5:
-		ret = hyperv_negotiate_version(hdev, SYNTHVID_VERSION_WIN10);
+		ret = hv_drm_negotiate_version(hdev, SYNTHVID_VERSION_WIN10);
 		if (!ret)
 			break;
 		fallthrough;
 	case VERSION_WIN8:
 	case VERSION_WIN8_1:
-		ret = hyperv_negotiate_version(hdev, SYNTHVID_VERSION_WIN8);
+		ret = hv_drm_negotiate_version(hdev, SYNTHVID_VERSION_WIN8);
 		break;
 	default:
-		ret = hyperv_negotiate_version(hdev, SYNTHVID_VERSION_WIN10);
+		ret = hv_drm_negotiate_version(hdev, SYNTHVID_VERSION_WIN10);
 		break;
 	}
 
@@ -581,8 +581,8 @@ int hyperv_connect_vsp(struct hv_device *hdev)
 
 	hv->screen_depth = SYNTHVID_DEPTH_WIN8;
 
-	if (hyperv_version_ge(hv->synthvid_version, SYNTHVID_VERSION_WIN10)) {
-		ret = hyperv_get_supported_resolution(hdev);
+	if (hv_drm_version_ge(hv->synthvid_version, SYNTHVID_VERSION_WIN10)) {
+		ret = hv_drm_get_supported_resolution(hdev);
 		if (ret)
 			drm_err(dev, "Failed to get supported resolution from host, use default\n");
 	}
diff --git a/drivers/gpu/drm/i915/.kunitconfig b/drivers/gpu/drm/i915/.kunitconfig
new file mode 100644
index 0000000000000..1b47fe62b2854
--- /dev/null
+++ b/drivers/gpu/drm/i915/.kunitconfig
@@ -0,0 +1,9 @@
+CONFIG_EXPERT=y
+CONFIG_MODULES=y
+CONFIG_KUNIT=y
+CONFIG_PCI=y
+CONFIG_DEBUG_FS=y
+CONFIG_MAGIC_SYSRQ=y
+CONFIG_DRM=y
+CONFIG_DRM_I915=y
+CONFIG_DRM_I915_KUNIT_TEST=y
diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug
index 52a3a59b4ba2c..dc43dcfbadb6e 100644
--- a/drivers/gpu/drm/i915/Kconfig.debug
+++ b/drivers/gpu/drm/i915/Kconfig.debug
@@ -220,6 +220,18 @@ config DRM_I915_SELFTEST_BROKEN
 
 	  If in doubt, say "N".
 
+config DRM_I915_KUNIT_TEST
+	tristate "KUnit tests for the drm i915 driver" if !KUNIT_ALL_TESTS
+	depends on DRM_I915 && KUNIT && DEBUG_FS
+	default KUNIT_ALL_TESTS
+	help
+	  Choose this option to allow the driver to perform selftests under
+	  the kunit framework
+
+	  Recommended for driver developers only.
+
+	  If in doubt, say "N".
+
 config DRM_I915_LOW_LEVEL_TRACEPOINTS
 	bool "Enable low level request tracing events"
 	depends on DRM_I915
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 07802a7f4ce50..a83fa8be0aba1 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -83,8 +83,7 @@ i915-y += \
 	i915_fb_pin.o \
 	i915_hdcp_gsc.o \
 	i915_initial_plane.o \
-	i915_overlay.o \
-	i915_panic.o
+	i915_overlay.o
 
 # "Graphics Technology" (aka we talk to the gpu)
 gt-y += \
@@ -167,6 +166,7 @@ gem-y += \
 	gem/i915_gem_object.o \
 	gem/i915_gem_object_frontbuffer.o \
 	gem/i915_gem_pages.o \
+	gem/i915_gem_panic.o \
 	gem/i915_gem_phys.o \
 	gem/i915_gem_pm.o \
 	gem/i915_gem_region.o \
@@ -356,6 +356,7 @@ i915-y += \
 	display/intel_dp_aux.o \
 	display/intel_dp_aux_backlight.o \
 	display/intel_dp_hdcp.o \
+	display/intel_dp_link_caps.o \
 	display/intel_dp_link_training.o \
 	display/intel_dp_mst.o \
 	display/intel_dp_test.o \
@@ -385,6 +386,8 @@ i915-y += \
 i915-$(CONFIG_DRM_I915_DP_TUNNEL) += \
 	display/intel_dp_tunnel.o
 
+obj-$(CONFIG_DRM_I915_KUNIT_TEST) += display/tests/
+
 i915-$(CONFIG_DRM_I915_GVT) += \
 	display/intel_gvt_api.o
 
diff --git a/drivers/gpu/drm/i915/display/g4x_dp.c b/drivers/gpu/drm/i915/display/g4x_dp.c
index 5ff1cdf4581a5..b867443ff2272 100644
--- a/drivers/gpu/drm/i915/display/g4x_dp.c
+++ b/drivers/gpu/drm/i915/display/g4x_dp.c
@@ -1222,7 +1222,8 @@ static bool ilk_digital_port_connected(struct intel_encoder *encoder)
 	return intel_de_read(display, DEISR) & bit;
 }
 
-static int g4x_dp_compute_config(struct intel_encoder *encoder,
+static int g4x_dp_compute_config(struct intel_atomic_state *state,
+				 struct intel_encoder *encoder,
 				 struct intel_crtc_state *crtc_state,
 				 struct drm_connector_state *conn_state)
 {
@@ -1232,7 +1233,7 @@ static int g4x_dp_compute_config(struct intel_encoder *encoder,
 	if (HAS_PCH_SPLIT(display) && encoder->port != PORT_A)
 		crtc_state->has_pch_encoder = true;
 
-	ret = intel_dp_compute_config(encoder, crtc_state, conn_state);
+	ret = intel_dp_compute_config(state, encoder, crtc_state, conn_state);
 	if (ret)
 		return ret;
 
@@ -1252,10 +1253,13 @@ static void g4x_dp_suspend_complete(struct intel_encoder *encoder)
 
 static void intel_dp_encoder_destroy(struct drm_encoder *encoder)
 {
+	struct intel_digital_port *dig_port = enc_to_dig_port(to_intel_encoder(encoder));
+
 	intel_dp_encoder_flush_work(encoder);
 
 	drm_encoder_cleanup(encoder);
-	kfree(enc_to_dig_port(to_intel_encoder(encoder)));
+	intel_dp_link_cleanup(&dig_port->dp);
+	kfree(dig_port);
 }
 
 static void intel_dp_encoder_reset(struct drm_encoder *encoder)
@@ -1350,6 +1354,9 @@ bool g4x_dp_init(struct intel_display *display,
 	intel_encoder->audio_enable = g4x_dp_audio_enable;
 	intel_encoder->audio_disable = g4x_dp_audio_disable;
 
+	if (intel_dp_link_init(&dig_port->dp) != 0)
+		goto err_dp_init;
+
 	if ((display->platform.ivybridge && port == PORT_A) ||
 	    (HAS_PCH_CPT(display) && port != PORT_A)) {
 		dig_port->dp.set_link_train = cpt_set_link_train;
@@ -1419,6 +1426,8 @@ bool g4x_dp_init(struct intel_display *display,
 	return true;
 
 err_init_connector:
+	intel_dp_link_cleanup(&dig_port->dp);
+err_dp_init:
 	drm_encoder_cleanup(encoder);
 err_encoder_init:
 	kfree(intel_connector);
diff --git a/drivers/gpu/drm/i915/display/g4x_hdmi.c b/drivers/gpu/drm/i915/display/g4x_hdmi.c
index acb36cab999c5..4c33aa1d1d32e 100644
--- a/drivers/gpu/drm/i915/display/g4x_hdmi.c
+++ b/drivers/gpu/drm/i915/display/g4x_hdmi.c
@@ -126,12 +126,12 @@ static bool g4x_compute_has_hdmi_sink(struct intel_atomic_state *state,
 	return false;
 }
 
-static int g4x_hdmi_compute_config(struct intel_encoder *encoder,
+static int g4x_hdmi_compute_config(struct intel_atomic_state *state,
+				   struct intel_encoder *encoder,
 				   struct intel_crtc_state *crtc_state,
 				   struct drm_connector_state *conn_state)
 {
 	struct intel_display *display = to_intel_display(encoder);
-	struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 
 	if (HAS_PCH_SPLIT(display))
diff --git a/drivers/gpu/drm/i915/display/i9xx_plane.c b/drivers/gpu/drm/i915/display/i9xx_plane.c
index 70734d32a4090..490908f59141a 100644
--- a/drivers/gpu/drm/i915/display/i9xx_plane.c
+++ b/drivers/gpu/drm/i915/display/i9xx_plane.c
@@ -1108,6 +1108,10 @@ intel_primary_plane_create(struct intel_display *display, enum pipe pipe)
 						   DRM_MODE_ROTATE_0,
 						   supported_rotations);
 
+	if (display->platform.valleyview || display->platform.cherryview)
+		drm_plane_create_blend_mode_property(&plane->base,
+						     BIT(DRM_MODE_BLEND_PREMULTI));
+
 	zpos = 0;
 	drm_plane_create_zpos_immutable_property(&plane->base, zpos);
 
diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c
index a549f1fac8103..ea0cdb7822f39 100644
--- a/drivers/gpu/drm/i915/display/icl_dsi.c
+++ b/drivers/gpu/drm/i915/display/icl_dsi.c
@@ -1657,7 +1657,8 @@ static int gen11_dsi_dsc_compute_config(struct intel_encoder *encoder,
 	return 0;
 }
 
-static int gen11_dsi_compute_config(struct intel_encoder *encoder,
+static int gen11_dsi_compute_config(struct intel_atomic_state *state,
+				    struct intel_encoder *encoder,
 				    struct intel_crtc_state *pipe_config,
 				    struct drm_connector_state *conn_state)
 {
@@ -1671,7 +1672,7 @@ static int gen11_dsi_compute_config(struct intel_encoder *encoder,
 	pipe_config->sink_format = INTEL_OUTPUT_FORMAT_RGB;
 	pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
 
-	ret = intel_panel_compute_config(intel_connector, adjusted_mode);
+	ret = intel_panel_compute_config(state, pipe_config, intel_connector);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/i915/display/intel_alpm.c b/drivers/gpu/drm/i915/display/intel_alpm.c
index c6963ea420cc9..f1383764b7028 100644
--- a/drivers/gpu/drm/i915/display/intel_alpm.c
+++ b/drivers/gpu/drm/i915/display/intel_alpm.c
@@ -291,7 +291,9 @@ void intel_alpm_lobf_compute_config_late(struct intel_dp *intel_dp,
 	if (!crtc_state->has_lobf)
 		return;
 
-	if (!intel_alpm_lobf_is_window1_sufficient(crtc_state)) {
+	if (crtc_state->has_psr ||
+	    !intel_vrr_is_fixed_rr(crtc_state) ||
+	    !intel_alpm_lobf_is_window1_sufficient(crtc_state)) {
 		crtc_state->has_lobf = false;
 		return;
 	}
@@ -343,11 +345,7 @@ void intel_alpm_lobf_compute_config(struct intel_dp *intel_dp,
 	if (!intel_dp->as_sdp_supported)
 		return;
 
-	if (crtc_state->has_psr)
-		return;
-
-	if (!intel_vrr_always_use_vrr_tg(display) ||
-	    !intel_vrr_is_fixed_rr(crtc_state))
+	if (!intel_vrr_always_use_vrr_tg(display))
 		return;
 
 	if (!(intel_alpm_aux_wake_supported(intel_dp) ||
@@ -407,6 +405,11 @@ static void lnl_alpm_configure(struct intel_dp *intel_dp,
 			if (crtc_state->disable_as_sdp_when_pr_active)
 				pr_alpm_ctl |= PR_ALPM_CTL_AS_SDP_TRANSMISSION_IN_ACTIVE_DISABLE;
 
+			if (intel_display_power_dc3co_allowed(display))
+				pr_alpm_ctl |= PR_ALPM_CTL_USE_DC3CO_IDLE_PROTOCOL;
+			else
+				pr_alpm_ctl &= ~PR_ALPM_CTL_USE_DC3CO_IDLE_PROTOCOL;
+
 			intel_de_write(display, PR_ALPM_CTL(display, cpu_transcoder),
 				       pr_alpm_ctl);
 		}
diff --git a/drivers/gpu/drm/i915/display/intel_audio.c b/drivers/gpu/drm/i915/display/intel_audio.c
index 9729f1837d2c1..eae76e9611055 100644
--- a/drivers/gpu/drm/i915/display/intel_audio.c
+++ b/drivers/gpu/drm/i915/display/intel_audio.c
@@ -38,6 +38,7 @@
 #include "intel_de.h"
 #include "intel_display_types.h"
 #include "intel_display_wa.h"
+#include "intel_dp.h"
 #include "intel_lpe_audio.h"
 
 /**
@@ -696,6 +697,13 @@ static void ibx_audio_codec_enable(struct intel_encoder *encoder,
 	mutex_unlock(&display->audio.mutex);
 }
 
+static
+bool intel_audio_needs_cpu_transcoder_id(const struct intel_crtc_state *crtc_state)
+{
+	return intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST) ||
+	       intel_dp_is_uhbr(crtc_state);
+}
+
 bool intel_audio_compute_config(struct intel_encoder *encoder,
 				struct intel_crtc_state *crtc_state,
 				struct drm_connector_state *conn_state)
@@ -762,6 +770,8 @@ void intel_audio_codec_enable(struct intel_encoder *encoder,
 	audio_state = &display->audio.state[cpu_transcoder];
 
 	audio_state->encoder = encoder;
+	audio_state->needs_cpu_transcoder_id =
+			intel_audio_needs_cpu_transcoder_id(crtc_state);
 	BUILD_BUG_ON(sizeof(audio_state->eld) != sizeof(crtc_state->eld));
 	memcpy(audio_state->eld, crtc_state->eld, sizeof(audio_state->eld));
 
@@ -769,8 +779,12 @@ void intel_audio_codec_enable(struct intel_encoder *encoder,
 
 	if (acomp && acomp->base.audio_ops &&
 	    acomp->base.audio_ops->pin_eld_notify) {
-		/* audio drivers expect cpu_transcoder = -1 to indicate Non-MST cases */
-		if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST))
+		/*
+		 * Audio drivers expect cpu_transcoder = -1 to indicate
+		 * Non-MST/HBR cases. MST and UHBR SST are addressed by
+		 * a real cpu_transcoder.
+		 */
+		if (!intel_audio_needs_cpu_transcoder_id(crtc_state))
 			cpu_transcoder = -1;
 		acomp->base.audio_ops->pin_eld_notify(acomp->base.audio_ops->audio_ptr,
 						      (int)port, (int)cpu_transcoder);
@@ -819,14 +833,19 @@ void intel_audio_codec_disable(struct intel_encoder *encoder,
 	audio_state = &display->audio.state[cpu_transcoder];
 
 	audio_state->encoder = NULL;
+	audio_state->needs_cpu_transcoder_id = false;
 	memset(audio_state->eld, 0, sizeof(audio_state->eld));
 
 	mutex_unlock(&display->audio.mutex);
 
 	if (acomp && acomp->base.audio_ops &&
 	    acomp->base.audio_ops->pin_eld_notify) {
-		/* audio drivers expect cpu_transcoder = -1 to indicate Non-MST cases */
-		if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST))
+		/*
+		 * Audio drivers expect cpu_transcoder = -1 to indicate
+		 * Non-MST/HBR cases. MST and UHBR SST are addressed by
+		 * a real cpu_transcoder.
+		 */
+		if (!intel_audio_needs_cpu_transcoder_id(old_crtc_state))
 			cpu_transcoder = -1;
 		acomp->base.audio_ops->pin_eld_notify(acomp->base.audio_ops->audio_ptr,
 						      (int)port, (int)cpu_transcoder);
@@ -1118,18 +1137,24 @@ static int intel_audio_component_get_cdclk_freq(struct device *kdev)
 }
 
 /*
- * get the intel audio state according to the parameter port and cpu_transcoder
- * MST & (cpu_transcoder >= 0): return the audio.state[cpu_transcoder].encoder],
- *   when port is matched
- * MST & (cpu_transcoder < 0): this is invalid
- * Non-MST & (cpu_transcoder >= 0): only cpu_transcoder = 0 (the first device entry)
- *   will get the right intel_encoder with port matched
- * Non-MST & (cpu_transcoder < 0): get the right intel_encoder with port matched
+ * Get the intel audio state for a given (port, cpu_transcoder).
+ *
+ * Streams are addressed either by a real cpu_transcoder (DP MST and UHBR SST,
+ * i.e. entries whose stored needs_cpu_transcoder_id is true) or by port alone
+ * (legacy SST). Both the signalling side (pin_eld_notify()) and the lookup
+ * side use the same predicate, so the two are symmetric.
+ *
+ * cpu_transcoder >= 0 & needs_cpu_transcoder_id: return audio.state[cpu_transcoder]
+ *   when the port matches.
+ * cpu_transcoder <  0 & !needs_cpu_transcoder_id: return the first port-matching
+ *   entry.
+ * cpu_transcoder =  0 & !needs_cpu_transcoder_id: falls to the port-only
+ *   loop so the first device entry of a legacy SST port is still found.
  */
 static struct intel_audio_state *find_audio_state(struct intel_display *display,
 						  int port, int cpu_transcoder)
 {
-	/* MST */
+	/* MST, or UHBR SST. */
 	if (cpu_transcoder >= 0) {
 		struct intel_audio_state *audio_state;
 		struct intel_encoder *encoder;
@@ -1142,11 +1167,11 @@ static struct intel_audio_state *find_audio_state(struct intel_display *display,
 		encoder = audio_state->encoder;
 
 		if (encoder && encoder->port == port &&
-		    encoder->type == INTEL_OUTPUT_DP_MST)
+		    audio_state->needs_cpu_transcoder_id)
 			return audio_state;
 	}
 
-	/* Non-MST */
+	/* Legacy SST. */
 	if (cpu_transcoder > 0)
 		return NULL;
 
@@ -1158,7 +1183,7 @@ static struct intel_audio_state *find_audio_state(struct intel_display *display,
 		encoder = audio_state->encoder;
 
 		if (encoder && encoder->port == port &&
-		    encoder->type != INTEL_OUTPUT_DP_MST)
+		    !audio_state->needs_cpu_transcoder_id)
 			return audio_state;
 	}
 
diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c
index ded2ee497bbf2..97cbae2e547e2 100644
--- a/drivers/gpu/drm/i915/display/intel_bios.c
+++ b/drivers/gpu/drm/i915/display/intel_bios.c
@@ -2217,52 +2217,52 @@ static u8 translate_iboost(struct intel_display *display, u8 val)
 
 static const u8 cnp_ddc_pin_map[] = {
 	[0] = 0, /* N/A */
-	[GMBUS_PIN_1_BXT] = DDC_BUS_DDI_B,
-	[GMBUS_PIN_2_BXT] = DDC_BUS_DDI_C,
-	[GMBUS_PIN_4_CNP] = DDC_BUS_DDI_D, /* sic */
-	[GMBUS_PIN_3_BXT] = DDC_BUS_DDI_F, /* sic */
+	[GMBUS_PIN_1] = DDC_BUS_DDI_B,
+	[GMBUS_PIN_2] = DDC_BUS_DDI_C,
+	[GMBUS_PIN_4] = DDC_BUS_DDI_D, /* sic */
+	[GMBUS_PIN_3] = DDC_BUS_DDI_F, /* sic */
 };
 
 static const u8 icp_ddc_pin_map[] = {
-	[GMBUS_PIN_1_BXT] = ICL_DDC_BUS_DDI_A,
-	[GMBUS_PIN_2_BXT] = ICL_DDC_BUS_DDI_B,
-	[GMBUS_PIN_3_BXT] = TGL_DDC_BUS_DDI_C,
-	[GMBUS_PIN_9_TC1_ICP] = ICL_DDC_BUS_PORT_1,
-	[GMBUS_PIN_10_TC2_ICP] = ICL_DDC_BUS_PORT_2,
-	[GMBUS_PIN_11_TC3_ICP] = ICL_DDC_BUS_PORT_3,
-	[GMBUS_PIN_12_TC4_ICP] = ICL_DDC_BUS_PORT_4,
-	[GMBUS_PIN_13_TC5_TGP] = TGL_DDC_BUS_PORT_5,
-	[GMBUS_PIN_14_TC6_TGP] = TGL_DDC_BUS_PORT_6,
+	[GMBUS_PIN_1] = ICL_DDC_BUS_DDI_A,
+	[GMBUS_PIN_2] = ICL_DDC_BUS_DDI_B,
+	[GMBUS_PIN_3] = TGL_DDC_BUS_DDI_C,
+	[GMBUS_PIN_9_TC1] = ICL_DDC_BUS_PORT_1,
+	[GMBUS_PIN_10_TC2] = ICL_DDC_BUS_PORT_2,
+	[GMBUS_PIN_11_TC3] = ICL_DDC_BUS_PORT_3,
+	[GMBUS_PIN_12_TC4] = ICL_DDC_BUS_PORT_4,
+	[GMBUS_PIN_13_TC5] = TGL_DDC_BUS_PORT_5,
+	[GMBUS_PIN_14_TC6] = TGL_DDC_BUS_PORT_6,
 };
 
 static const u8 rkl_pch_tgp_ddc_pin_map[] = {
-	[GMBUS_PIN_1_BXT] = ICL_DDC_BUS_DDI_A,
-	[GMBUS_PIN_2_BXT] = ICL_DDC_BUS_DDI_B,
-	[GMBUS_PIN_9_TC1_ICP] = RKL_DDC_BUS_DDI_D,
-	[GMBUS_PIN_10_TC2_ICP] = RKL_DDC_BUS_DDI_E,
+	[GMBUS_PIN_1] = ICL_DDC_BUS_DDI_A,
+	[GMBUS_PIN_2] = ICL_DDC_BUS_DDI_B,
+	[GMBUS_PIN_9_TC1] = RKL_DDC_BUS_DDI_D,
+	[GMBUS_PIN_10_TC2] = RKL_DDC_BUS_DDI_E,
 };
 
 static const u8 adls_ddc_pin_map[] = {
-	[GMBUS_PIN_1_BXT] = ICL_DDC_BUS_DDI_A,
-	[GMBUS_PIN_9_TC1_ICP] = ADLS_DDC_BUS_PORT_TC1,
-	[GMBUS_PIN_10_TC2_ICP] = ADLS_DDC_BUS_PORT_TC2,
-	[GMBUS_PIN_11_TC3_ICP] = ADLS_DDC_BUS_PORT_TC3,
-	[GMBUS_PIN_12_TC4_ICP] = ADLS_DDC_BUS_PORT_TC4,
+	[GMBUS_PIN_1] = ICL_DDC_BUS_DDI_A,
+	[GMBUS_PIN_9_TC1] = ADLS_DDC_BUS_PORT_TC1,
+	[GMBUS_PIN_10_TC2] = ADLS_DDC_BUS_PORT_TC2,
+	[GMBUS_PIN_11_TC3] = ADLS_DDC_BUS_PORT_TC3,
+	[GMBUS_PIN_12_TC4] = ADLS_DDC_BUS_PORT_TC4,
 };
 
 static const u8 gen9bc_tgp_ddc_pin_map[] = {
-	[GMBUS_PIN_2_BXT] = DDC_BUS_DDI_B,
-	[GMBUS_PIN_9_TC1_ICP] = DDC_BUS_DDI_C,
-	[GMBUS_PIN_10_TC2_ICP] = DDC_BUS_DDI_D,
+	[GMBUS_PIN_2] = DDC_BUS_DDI_B,
+	[GMBUS_PIN_9_TC1] = DDC_BUS_DDI_C,
+	[GMBUS_PIN_10_TC2] = DDC_BUS_DDI_D,
 };
 
 static const u8 adlp_ddc_pin_map[] = {
-	[GMBUS_PIN_1_BXT] = ICL_DDC_BUS_DDI_A,
-	[GMBUS_PIN_2_BXT] = ICL_DDC_BUS_DDI_B,
-	[GMBUS_PIN_9_TC1_ICP] = ADLP_DDC_BUS_PORT_TC1,
-	[GMBUS_PIN_10_TC2_ICP] = ADLP_DDC_BUS_PORT_TC2,
-	[GMBUS_PIN_11_TC3_ICP] = ADLP_DDC_BUS_PORT_TC3,
-	[GMBUS_PIN_12_TC4_ICP] = ADLP_DDC_BUS_PORT_TC4,
+	[GMBUS_PIN_1] = ICL_DDC_BUS_DDI_A,
+	[GMBUS_PIN_2] = ICL_DDC_BUS_DDI_B,
+	[GMBUS_PIN_9_TC1] = ADLP_DDC_BUS_PORT_TC1,
+	[GMBUS_PIN_10_TC2] = ADLP_DDC_BUS_PORT_TC2,
+	[GMBUS_PIN_11_TC3] = ADLP_DDC_BUS_PORT_TC3,
+	[GMBUS_PIN_12_TC4] = ADLP_DDC_BUS_PORT_TC4,
 };
 
 static u8 map_ddc_pin(struct intel_display *display, u8 vbt_pin)
diff --git a/drivers/gpu/drm/i915/display/intel_bw.c b/drivers/gpu/drm/i915/display/intel_bw.c
index dc5a5b639d872..546d5ccbe052a 100644
--- a/drivers/gpu/drm/i915/display/intel_bw.c
+++ b/drivers/gpu/drm/i915/display/intel_bw.c
@@ -52,6 +52,8 @@ struct intel_qgv_point {
 
 #define DEPROGBWPCLIMIT		60
 
+#define PEAK_BW_THRESHOLD	20000
+
 struct intel_psf_gv_point {
 	u8 clk; /* clock in multiples of 16.6666 MHz */
 };
@@ -59,7 +61,7 @@ struct intel_psf_gv_point {
 struct intel_qgv_info {
 	struct intel_qgv_point points[I915_NUM_QGV_POINTS];
 	struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS];
-	u8 num_points;
+	u8 num_qgv_points;
 	u8 num_psf_points;
 	u8 t_bl;
 	u8 max_numchannels;
@@ -155,8 +157,8 @@ static int adls_pcode_read_psf_gv_point_info(struct intel_display *display,
 
 static u16 icl_qgv_points_mask(struct intel_display *display)
 {
-	unsigned int num_psf_gv_points = display->bw.max[0].num_psf_gv_points;
-	unsigned int num_qgv_points = display->bw.max[0].num_qgv_points;
+	unsigned int num_psf_gv_points = display->bw.num_psf_gv_points;
+	unsigned int num_qgv_points = display->bw.num_qgv_points;
 	u16 qgv_points = 0, psf_points = 0;
 
 	/*
@@ -184,7 +186,7 @@ static int icl_pcode_restrict_qgv_points(struct intel_display *display,
 {
 	int ret;
 
-	if (DISPLAY_VER(display) >= 14)
+	if (HAS_PMDEMAND(display))
 		return 0;
 
 	/* bspec says to keep retrying for at least 1 ms */
@@ -246,13 +248,11 @@ static bool is_y_tile(struct intel_display *display)
 	return !HAS_4TILE(display);
 }
 
-static int icl_get_qgv_points(struct intel_display *display,
-			      const struct dram_info *dram_info,
-			      struct intel_qgv_info *qi)
+static int icl_init_qgv_info(struct intel_display *display,
+			     const struct dram_info *dram_info,
+			     struct intel_qgv_info *qi)
 {
-	int i, ret;
-
-	qi->num_points = dram_info->num_qgv_points;
+	qi->num_qgv_points = dram_info->num_qgv_points;
 	qi->num_psf_points = dram_info->num_psf_gv_points;
 
 	if (DISPLAY_VER(display) >= 14) {
@@ -323,11 +323,23 @@ static int icl_get_qgv_points(struct intel_display *display,
 		qi->max_numchannels = 1;
 	}
 
+	return 0;
+}
+
+static int icl_get_qgv_points(struct intel_display *display,
+			      const struct dram_info *dram_info,
+			      struct intel_qgv_info *qi)
+{
+	int i, ret;
+
+	if (icl_init_qgv_info(display, dram_info, qi))
+		return -EINVAL;
+
 	if (drm_WARN_ON(display->drm,
-			qi->num_points > ARRAY_SIZE(qi->points)))
-		qi->num_points = ARRAY_SIZE(qi->points);
+			qi->num_qgv_points > ARRAY_SIZE(qi->points)))
+		qi->num_qgv_points = ARRAY_SIZE(qi->points);
 
-	for (i = 0; i < qi->num_points; i++) {
+	for (i = 0; i < qi->num_qgv_points; i++) {
 		struct intel_qgv_point *sp = &qi->points[i];
 
 		ret = intel_read_qgv_point_info(display, sp, i);
@@ -373,7 +385,7 @@ static int icl_sagv_max_dclk(const struct intel_qgv_info *qi)
 	u16 dclk = 0;
 	int i;
 
-	for (i = 0; i < qi->num_points; i++)
+	for (i = 0; i < qi->num_qgv_points; i++)
 		dclk = max(dclk, qi->points[i].dclk);
 
 	return dclk;
@@ -511,6 +523,19 @@ static const struct intel_display_bw_params *get_display_bw_params(struct intel_
 	return NULL;
 }
 
+static void update_sagv_status(struct intel_display *display, int qgv_points)
+{
+	/*
+	 * In case if SAGV is disabled in BIOS, we always get 1
+	 * SAGV point, but we can't send PCode commands to restrict it
+	 * as it will fail and pointless anyway.
+	 */
+	if (qgv_points == 1)
+		display->sagv.status = I915_SAGV_NOT_CONTROLLED;
+	else
+		display->sagv.status = I915_SAGV_ENABLED;
+}
+
 static int icl_get_bw_info(struct intel_display *display,
 			   const struct dram_info *dram_info,
 			   const struct intel_soc_bw_params *soc_bw_params,
@@ -536,6 +561,9 @@ static int icl_get_bw_info(struct intel_display *display,
 	ipqdepth = min(ipqdepthpch, display_bw_params->displayrtids / num_channels);
 	qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile(display) ? 4 : 2);
 
+	display->bw.num_qgv_points = qi.num_qgv_points;
+	display->bw.num_psf_gv_points = qi.num_psf_points;
+
 	for (i = 0; i < num_groups; i++) {
 		struct intel_bw_info *bi = &display->bw.max[i];
 		int clpchgroup;
@@ -544,10 +572,7 @@ static int icl_get_bw_info(struct intel_display *display,
 		clpchgroup = (display_bw_params->deburst * qi.deinterleave / num_channels) << i;
 		bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1;
 
-		bi->num_qgv_points = qi.num_points;
-		bi->num_psf_gv_points = qi.num_psf_points;
-
-		for (j = 0; j < qi.num_points; j++) {
+		for (j = 0; j < qi.num_qgv_points; j++) {
 			const struct intel_qgv_point *sp = &qi.points[j];
 			int ct, bw;
 
@@ -569,15 +594,6 @@ static int icl_get_bw_info(struct intel_display *display,
 				    i, j, bi->num_planes, bi->deratedbw[j]);
 		}
 	}
-	/*
-	 * In case if SAGV is disabled in BIOS, we always get 1
-	 * SAGV point, but we can't send PCode commands to restrict it
-	 * as it will fail and pointless anyway.
-	 */
-	if (qi.num_points == 1)
-		display->sagv.status = I915_SAGV_NOT_CONTROLLED;
-	else
-		display->sagv.status = I915_SAGV_ENABLED;
 
 	return 0;
 }
@@ -587,6 +603,32 @@ static int tgl_peakbw(int num_channels, int channel_width, int dclk)
 	return num_channels * (channel_width / 8) * dclk;
 }
 
+static void xe3_add_peakbw_threshold(struct intel_display *display)
+{
+	u8 qgv_points = display->bw.num_qgv_points;
+
+	if (!HAS_PEAK_BW_THRESHOLD(display))
+		return;
+
+	if (qgv_points >= I915_NUM_QGV_POINTS) {
+		drm_dbg_kms(display->drm, "QGV points maxed out; skipping peak bandwidth threshold.\n");
+		return;
+	}
+
+	if (qgv_points <= 1)
+		return;
+
+	display->bw.num_qgv_points++;
+
+	display->bw.peakbw[qgv_points] = PEAK_BW_THRESHOLD;
+
+	for (int i = 0; i < ARRAY_SIZE(display->bw.max); i++)
+		display->bw.max[i].deratedbw[qgv_points] = PEAK_BW_THRESHOLD;
+
+	drm_dbg_kms(display->drm, "An extra QGV point %d added for Peak bw threshod of %d\n",
+		    qgv_points, PEAK_BW_THRESHOLD);
+}
+
 static int tgl_get_bw_info(struct intel_display *display,
 			   const struct dram_info *dram_info,
 			   const struct intel_soc_bw_params *soc_bw_params,
@@ -629,16 +671,20 @@ static int tgl_get_bw_info(struct intel_display *display,
 	 */
 	clperchgroup = 4 * (8 / num_channels) * qi.deinterleave;
 
+	display->bw.num_qgv_points = qi.num_qgv_points;
+	display->bw.num_psf_gv_points = qi.num_psf_points;
+
+	display->bw.max[0].num_planes = U8_MAX;
+
 	for (i = 0; i < num_groups; i++) {
 		struct intel_bw_info *bi = &display->bw.max[i];
-		struct intel_bw_info *bi_next;
 		int clpchgroup;
 		int j;
 
 		clpchgroup = (display_bw_params->deburst * qi.deinterleave / num_channels) << i;
 
 		if (i < num_groups - 1) {
-			bi_next = &display->bw.max[i + 1];
+			struct intel_bw_info *bi_next = &display->bw.max[i + 1];
 
 			if (clpchgroup < clperchgroup)
 				bi_next->num_planes = (ipqdepth - clpchgroup) / clpchgroup;
@@ -646,10 +692,7 @@ static int tgl_get_bw_info(struct intel_display *display,
 				bi_next->num_planes = 0;
 		}
 
-		bi->num_qgv_points = qi.num_points;
-		bi->num_psf_gv_points = qi.num_psf_points;
-
-		for (j = 0; j < qi.num_points; j++) {
+		for (j = 0; j < qi.num_qgv_points; j++) {
 			const struct intel_qgv_point *sp = &qi.points[j];
 			int ct, bw;
 
@@ -665,61 +708,51 @@ static int tgl_get_bw_info(struct intel_display *display,
 
 			bi->deratedbw[j] = min(maxdebw,
 					       bw * (100 - soc_bw_params->derating) / 100);
-			bi->peakbw[j] = tgl_peakbw(num_channels, qi.channel_width, sp->dclk);
 
 			drm_dbg_kms(display->drm,
-				    "BW%d / QGV %d: num_planes=%d deratedbw=%u peakbw: %u\n",
-				    i, j, bi->num_planes, bi->deratedbw[j],
-				    bi->peakbw[j]);
+				    "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
+				    i, j, bi->num_planes, bi->deratedbw[j]);
 		}
+	}
 
-		for (j = 0; j < qi.num_psf_points; j++) {
-			const struct intel_psf_gv_point *sp = &qi.psf_points[j];
+	for (i = 0; i < qi.num_qgv_points; i++) {
+		const struct intel_qgv_point *sp = &qi.points[i];
 
-			bi->psf_bw[j] = adl_calc_psf_bw(sp->clk);
+		display->bw.peakbw[i] = tgl_peakbw(num_channels, qi.channel_width, sp->dclk);
 
-			drm_dbg_kms(display->drm,
-				    "BW%d / PSF GV %d: num_planes=%d bw=%u\n",
-				    i, j, bi->num_planes, bi->psf_bw[j]);
-		}
+		drm_dbg_kms(display->drm, "QGV %d: peakbw=%u\n", i, display->bw.peakbw[i]);
 	}
 
-	/*
-	 * In case if SAGV is disabled in BIOS, we always get 1
-	 * SAGV point, but we can't send PCode commands to restrict it
-	 * as it will fail and pointless anyway.
-	 */
-	if (qi.num_points == 1)
-		display->sagv.status = I915_SAGV_NOT_CONTROLLED;
-	else
-		display->sagv.status = I915_SAGV_ENABLED;
+	/* For xe3 cases add an extra qgv point for Peak bw threshold */
+	xe3_add_peakbw_threshold(display);
+
+	for (i = 0; i < qi.num_psf_points; i++) {
+		const struct intel_psf_gv_point *sp = &qi.psf_points[i];
+
+		display->bw.psf_bw[i] = adl_calc_psf_bw(sp->clk);
+
+		drm_dbg_kms(display->drm, "PSF GV %d: bw=%u\n", i, display->bw.psf_bw[i]);
+	}
 
 	return 0;
 }
 
 static void dg2_get_bw_info(struct intel_display *display)
 {
-	unsigned int deratedbw = display->platform.dg2_g11 ? 38000 : 50000;
-	int num_groups = ARRAY_SIZE(display->bw.max);
 	int i;
 
-	/*
-	 * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth
-	 * that doesn't depend on the number of planes enabled. So fill all the
-	 * plane group with constant bw information for uniformity with other
-	 * platforms. DG2-G10 platforms have a constant 50 GB/s bandwidth,
-	 * whereas DG2-G11 platforms have 38 GB/s.
-	 */
-	for (i = 0; i < num_groups; i++) {
-		struct intel_bw_info *bi = &display->bw.max[i];
+	display->bw.num_qgv_points = 1;
 
-		bi->num_planes = 1;
-		/* Need only one dummy QGV point per group */
-		bi->num_qgv_points = 1;
-		bi->deratedbw[0] = deratedbw;
-	}
+	display->bw.max[0].num_planes = U8_MAX;
+	display->bw.max[0].deratedbw[0] = display->platform.dg2_g11 ? 38000 : 50000;
 
-	display->sagv.status = I915_SAGV_NOT_CONTROLLED;
+	drm_dbg_kms(display->drm,
+		    "QGV 0: deratedbw=%u\n",
+		    display->bw.max[0].deratedbw[0]);
+
+	/* Bandwidth does not depend on # of planes; set all groups the same */
+	for (i = 1; i < ARRAY_SIZE(display->bw.max); i++)
+		display->bw.max[i] = display->bw.max[0];
 }
 
 static int xe2_hpd_get_bw_info(struct intel_display *display,
@@ -741,32 +774,32 @@ static int xe2_hpd_get_bw_info(struct intel_display *display,
 	peakbw = tgl_peakbw(num_channels, qi.channel_width, icl_sagv_max_dclk(&qi));
 	maxdebw = min(soc_bw_params->deprogbwlimit * 1000, peakbw * DEPROGBWPCLIMIT / 100);
 
-	for (i = 0; i < qi.num_points; i++) {
+	display->bw.num_qgv_points = qi.num_qgv_points;
+
+	display->bw.max[0].num_planes = U8_MAX;
+
+	for (i = 0; i < qi.num_qgv_points; i++) {
 		const struct intel_qgv_point *sp = &qi.points[i];
 		int bw = tgl_peakbw(num_channels, qi.channel_width, sp->dclk);
 
 		display->bw.max[0].deratedbw[i] =
 			min(maxdebw, (100 - soc_bw_params->derating) * bw / 100);
-		display->bw.max[0].peakbw[i] = bw;
 
-		drm_dbg_kms(display->drm, "QGV %d: deratedbw=%u peakbw: %u\n",
-			    i, display->bw.max[0].deratedbw[i],
-			    display->bw.max[0].peakbw[i]);
+		display->bw.peakbw[i] = bw;
+
+		drm_dbg_kms(display->drm, "QGV %d: deratedbw=%u peakbw=%u\n",
+			    i, display->bw.max[0].deratedbw[i], display->bw.peakbw[i]);
 	}
 
 	/* Bandwidth does not depend on # of planes; set all groups the same */
-	display->bw.max[0].num_planes = 1;
-	display->bw.max[0].num_qgv_points = qi.num_points;
 	for (i = 1; i < ARRAY_SIZE(display->bw.max); i++)
-		memcpy(&display->bw.max[i], &display->bw.max[0],
-		       sizeof(display->bw.max[0]));
+		display->bw.max[i] = display->bw.max[0];
 
 	/*
 	 * Xe2_HPD should always have exactly two QGV points representing
 	 * battery and plugged-in operation.
 	 */
-	drm_WARN_ON(display->drm, qi.num_points != 2);
-	display->sagv.status = I915_SAGV_ENABLED;
+	drm_WARN_ON(display->drm, qi.num_qgv_points != 2);
 
 	return 0;
 }
@@ -776,6 +809,9 @@ static unsigned int icl_max_bw_index(struct intel_display *display,
 {
 	int i;
 
+	if (qgv_point >= display->bw.num_qgv_points)
+		return UINT_MAX;
+
 	/*
 	 * Let's return max bw for 0 planes
 	 */
@@ -785,13 +821,6 @@ static unsigned int icl_max_bw_index(struct intel_display *display,
 		const struct intel_bw_info *bi =
 			&display->bw.max[i];
 
-		/*
-		 * Pcode will not expose all QGV points when
-		 * SAGV is forced to off/min/med/max.
-		 */
-		if (qgv_point >= bi->num_qgv_points)
-			return UINT_MAX;
-
 		if (num_planes >= bi->num_planes)
 			return i;
 	}
@@ -804,31 +833,24 @@ static unsigned int tgl_max_bw_index(struct intel_display *display,
 {
 	int i;
 
+	if (qgv_point >= display->bw.num_qgv_points)
+		return UINT_MAX;
+
 	for (i = ARRAY_SIZE(display->bw.max) - 1; i >= 0; i--) {
 		const struct intel_bw_info *bi =
 			&display->bw.max[i];
 
-		/*
-		 * Pcode will not expose all QGV points when
-		 * SAGV is forced to off/min/med/max.
-		 */
-		if (qgv_point >= bi->num_qgv_points)
-			return UINT_MAX;
-
 		if (num_planes <= bi->num_planes)
 			return i;
 	}
 
-	return 0;
+	return UINT_MAX;
 }
 
 static unsigned int adl_psf_bw(struct intel_display *display,
 			       int psf_gv_point)
 {
-	const struct intel_bw_info *bi =
-			&display->bw.max[0];
-
-	return bi->psf_bw[psf_gv_point];
+	return display->bw.psf_bw[psf_gv_point];
 }
 
 static unsigned int icl_qgv_bw(struct intel_display *display,
@@ -878,6 +900,8 @@ void intel_bw_init_hw(struct intel_display *display)
 	} else if (DISPLAY_VER(display) == 11) {
 		icl_get_bw_info(display, dram_info, soc_bw_params, display_bw_params);
 	}
+
+	update_sagv_status(display, display->bw.num_qgv_points);
 }
 
 static unsigned int intel_bw_num_active_planes(struct intel_display *display,
@@ -950,7 +974,7 @@ intel_atomic_get_bw_state(struct intel_atomic_state *state)
 static unsigned int icl_max_bw_qgv_point_mask(struct intel_display *display,
 					      int num_active_planes)
 {
-	unsigned int num_qgv_points = display->bw.max[0].num_qgv_points;
+	unsigned int num_qgv_points = display->bw.num_qgv_points;
 	unsigned int max_bw_point = 0;
 	unsigned int max_bw = 0;
 	int i;
@@ -986,7 +1010,7 @@ static u16 icl_prepare_qgv_points_mask(struct intel_display *display,
 
 static unsigned int icl_max_bw_psf_gv_point_mask(struct intel_display *display)
 {
-	unsigned int num_psf_gv_points = display->bw.max[0].num_psf_gv_points;
+	unsigned int num_psf_gv_points = display->bw.num_psf_gv_points;
 	unsigned int max_bw_point_mask = 0;
 	unsigned int max_bw = 0;
 	int i;
@@ -1091,7 +1115,7 @@ static int mtl_find_qgv_points(struct intel_display *display,
 			       struct intel_bw_state *new_bw_state)
 {
 	unsigned int best_rate = UINT_MAX;
-	unsigned int num_qgv_points = display->bw.max[0].num_qgv_points;
+	unsigned int num_qgv_points = display->bw.num_qgv_points;
 	unsigned int qgv_peak_bw  = 0;
 	int i;
 	int ret;
@@ -1116,21 +1140,15 @@ static int mtl_find_qgv_points(struct intel_display *display,
 	 * offered per plane group
 	 */
 	for (i = 0; i < num_qgv_points; i++) {
-		unsigned int bw_index =
-			tgl_max_bw_index(display, num_active_planes, i);
-		unsigned int max_data_rate;
-
-		if (bw_index >= ARRAY_SIZE(display->bw.max))
-			continue;
-
-		max_data_rate = display->bw.max[bw_index].deratedbw[i];
+		unsigned int max_data_rate =
+			icl_qgv_bw(display, num_active_planes, i);
 
 		if (max_data_rate < data_rate)
 			continue;
 
-		if (max_data_rate - data_rate < best_rate) {
-			best_rate = max_data_rate - data_rate;
-			qgv_peak_bw = display->bw.max[bw_index].peakbw[i];
+		if (max_data_rate < best_rate) {
+			best_rate = max_data_rate;
+			qgv_peak_bw = display->bw.peakbw[i];
 		}
 
 		drm_dbg_kms(display->drm, "QGV point %d: max bw %d required %d qgv_peak_bw: %d\n",
@@ -1162,8 +1180,8 @@ static int icl_find_qgv_points(struct intel_display *display,
 			       const struct intel_bw_state *old_bw_state,
 			       struct intel_bw_state *new_bw_state)
 {
-	unsigned int num_psf_gv_points = display->bw.max[0].num_psf_gv_points;
-	unsigned int num_qgv_points = display->bw.max[0].num_qgv_points;
+	unsigned int num_psf_gv_points = display->bw.num_psf_gv_points;
+	unsigned int num_qgv_points = display->bw.num_qgv_points;
 	u16 psf_points = 0;
 	u16 qgv_points = 0;
 	int i;
@@ -1254,7 +1272,7 @@ static int intel_bw_check_qgv_points(struct intel_display *display,
 
 	data_rate = DIV_ROUND_UP(data_rate, 1000);
 
-	if (DISPLAY_VER(display) >= 14)
+	if (HAS_PMDEMAND(display))
 		return mtl_find_qgv_points(display, data_rate, num_active_planes,
 					   new_bw_state);
 	else
diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c
index 7bc9b956554ba..d3c5e3438d192 100644
--- a/drivers/gpu/drm/i915/display/intel_cdclk.c
+++ b/drivers/gpu/drm/i915/display/intel_cdclk.c
@@ -1229,23 +1229,28 @@ static void skl_set_cdclk(struct intel_display *display,
 
 static void skl_sanitize_cdclk(struct intel_display *display)
 {
-	u32 cdctl, expected;
+	u32 cdctl, expected, swf18;
 
 	/*
 	 * check if the pre-os initialized the display
 	 * There is SWF18 scratchpad register defined which is set by the
 	 * pre-os which can be used by the OS drivers to check the status
 	 */
-	if ((intel_de_read(display, SWF_ILK(0x18)) & 0x00FFFFFF) == 0)
+	swf18 = intel_de_read(display, SWF_ILK(0x18));
+	if ((swf18 & 0x00FFFFFF) == 0) {
+		drm_dbg_kms(display->drm, "Sanitizing CDCLK due to SWF18 0x%x\n", swf18);
 		goto sanitize;
+	}
 
 	intel_update_cdclk(display);
 	intel_cdclk_dump_config(display, &display->cdclk.hw, "Current CDCLK");
 
 	/* Is PLL enabled and locked ? */
 	if (display->cdclk.hw.vco == 0 ||
-	    display->cdclk.hw.cdclk == display->cdclk.hw.bypass)
+	    display->cdclk.hw.cdclk == display->cdclk.hw.bypass) {
+		drm_dbg_kms(display->drm, "Sanitizing CDCLK due to PLL not enabled/locked\n");
 		goto sanitize;
+	}
 
 	/* DPLL okay; verify the cdclock
 	 *
@@ -1261,8 +1266,11 @@ static void skl_sanitize_cdclk(struct intel_display *display)
 		cdctl &= ~CDCLK_FREQ_DECIMAL_MASK;
 		cdctl |= expected & CDCLK_FREQ_DECIMAL_MASK;
 
-		if (cdctl != expected)
+		if (cdctl != expected) {
+			drm_dbg_kms(display->drm, "Sanitizing CDCLK due to CDCLK_CTL 0x%x, expected 0x%x)\n",
+				    intel_de_read(display, CDCLK_CTL), expected);
 			goto sanitize;
+		}
 
 		drm_dbg_kms(display->drm, "Sanitizing CDCLK decimal divider (CDCLK_CTL 0x%x, expected 0x%x)\n",
 			    intel_de_read(display, CDCLK_CTL), expected);
@@ -1274,8 +1282,6 @@ static void skl_sanitize_cdclk(struct intel_display *display)
 	return;
 
 sanitize:
-	drm_dbg_kms(display->drm, "Sanitizing cdclk programmed by pre-os\n");
-
 	/* force cdclk programming */
 	display->cdclk.hw.cdclk = 0;
 	/* force full PLL disable + enable */
@@ -1938,14 +1944,17 @@ static void adlp_cdclk_pll_crawl(struct intel_display *display, int vco)
 	display->cdclk.hw.vco = vco;
 }
 
+static u32 bxt_cdclk_cd2x_pipe_mask(struct intel_display *display)
+{
+	if (DISPLAY_VER(display) >= 11)
+		return ICL_CDCLK_CD2X_PIPE_MASK;
+	else
+		return BXT_CDCLK_CD2X_PIPE_MASK;
+}
+
 static u32 bxt_cdclk_cd2x_pipe(struct intel_display *display, enum pipe pipe)
 {
-	if (DISPLAY_VER(display) >= 12) {
-		if (pipe == INVALID_PIPE)
-			return TGL_CDCLK_CD2X_PIPE_NONE;
-		else
-			return TGL_CDCLK_CD2X_PIPE(pipe);
-	} else if (DISPLAY_VER(display) >= 11) {
+	if (DISPLAY_VER(display) >= 11) {
 		if (pipe == INVALID_PIPE)
 			return ICL_CDCLK_CD2X_PIPE_NONE;
 		else
@@ -2340,18 +2349,24 @@ static void bxt_sanitize_cdclk(struct intel_display *display)
 	intel_cdclk_dump_config(display, &display->cdclk.hw, "Current CDCLK");
 
 	if (display->cdclk.hw.vco == 0 ||
-	    display->cdclk.hw.cdclk == display->cdclk.hw.bypass)
+	    display->cdclk.hw.cdclk == display->cdclk.hw.bypass) {
+		drm_dbg_kms(display->drm, "Sanitizing CDCLK due to PLL not enabled/locked\n");
 		goto sanitize;
+	}
 
 	/* Make sure this is a legal cdclk value for the platform */
 	cdclk = bxt_calc_cdclk(display, display->cdclk.hw.cdclk);
-	if (cdclk != display->cdclk.hw.cdclk)
+	if (cdclk != display->cdclk.hw.cdclk) {
+		drm_dbg_kms(display->drm, "Sanitizing CDCLK due to bad CDCLK frequency\n");
 		goto sanitize;
+	}
 
 	/* Make sure the VCO is correct for the cdclk */
 	vco = bxt_calc_cdclk_pll_vco(display, cdclk);
-	if (vco != display->cdclk.hw.vco)
+	if (vco != display->cdclk.hw.vco) {
+		drm_dbg_kms(display->drm, "Sanitizing CDCLK due to bad VCO frequency\n");
 		goto sanitize;
+	}
 
 	/*
 	 * Some BIOS versions leave an incorrect decimal frequency value and
@@ -2366,7 +2381,7 @@ static void bxt_sanitize_cdclk(struct intel_display *display)
 	 * dividers both syncing to an active pipe, or asynchronously
 	 * (PIPE_NONE).
 	 */
-	cdctl &= ~bxt_cdclk_cd2x_pipe(display, INVALID_PIPE);
+	cdctl &= ~bxt_cdclk_cd2x_pipe_mask(display);
 	cdctl |= bxt_cdclk_cd2x_pipe(display, INVALID_PIPE);
 
 	if (cdctl != expected) {
@@ -2375,8 +2390,11 @@ static void bxt_sanitize_cdclk(struct intel_display *display)
 			cdctl |= expected & CDCLK_FREQ_DECIMAL_MASK;
 		}
 
-		if (cdctl != expected)
+		if (cdctl != expected) {
+			drm_dbg_kms(display->drm, "Sanitizing CDCLK due to CDCLK_CTL 0x%x, expected 0x%x\n",
+				    intel_de_read(display, CDCLK_CTL), expected);
 			goto sanitize;
+		}
 
 		drm_dbg_kms(display->drm, "Sanitizing CDCLK decimal divider (CDCLK_CTL 0x%x, expected 0x%x)\n",
 			    intel_de_read(display, CDCLK_CTL), expected);
@@ -2388,8 +2406,6 @@ static void bxt_sanitize_cdclk(struct intel_display *display)
 	return;
 
 sanitize:
-	drm_dbg_kms(display->drm, "Sanitizing cdclk programmed by pre-os\n");
-
 	/* force cdclk programming */
 	display->cdclk.hw.cdclk = 0;
 
diff --git a/drivers/gpu/drm/i915/display/intel_cmtg.c b/drivers/gpu/drm/i915/display/intel_cmtg.c
index e1fdc6fe9762a..c8e0f90af9108 100644
--- a/drivers/gpu/drm/i915/display/intel_cmtg.c
+++ b/drivers/gpu/drm/i915/display/intel_cmtg.c
@@ -4,7 +4,6 @@
  */
 
 #include <linux/string_choices.h>
-#include <linux/types.h>
 
 #include <drm/drm_device.h>
 #include <drm/drm_print.h>
@@ -13,9 +12,14 @@
 #include "intel_cmtg_regs.h"
 #include "intel_crtc.h"
 #include "intel_de.h"
+#include "intel_display.h"
 #include "intel_display_device.h"
+#include "intel_display_irq.h"
 #include "intel_display_power.h"
 #include "intel_display_regs.h"
+#include "intel_display_types.h"
+#include "intel_vrr.h"
+#include "intel_vrr_regs.h"
 
 /**
  * DOC: Common Primary Timing Generator (CMTG)
@@ -81,6 +85,18 @@ static void intel_cmtg_dump_config(struct intel_display *display,
 		    str_yes_no(cmtg_config->trans_b_secondary));
 }
 
+static inline enum transcoder to_cmtg_transcoder(enum transcoder cpu_transcoder)
+{
+	switch (cpu_transcoder) {
+	case TRANSCODER_A:
+		return TRANSCODER_CMTG0;
+	case TRANSCODER_B:
+		return TRANSCODER_CMTG1;
+	default:
+		return INVALID_TRANSCODER;
+	}
+}
+
 static bool intel_cmtg_transcoder_is_secondary(struct intel_display *display,
 					       enum transcoder trans)
 {
@@ -103,11 +119,11 @@ static void intel_cmtg_get_config(struct intel_display *display,
 {
 	u32 val;
 
-	val = intel_de_read(display, TRANS_CMTG_CTL_A);
+	val = intel_de_read(display, TRANS_CMTG_CTL(TRANSCODER_A));
 	cmtg_config->cmtg_a_enable = val & CMTG_ENABLE;
 
 	if (intel_cmtg_has_cmtg_b(display)) {
-		val = intel_de_read(display, TRANS_CMTG_CTL_B);
+		val = intel_de_read(display, TRANS_CMTG_CTL(TRANSCODER_B));
 		cmtg_config->cmtg_b_enable = val & CMTG_ENABLE;
 	}
 
@@ -124,8 +140,8 @@ static bool intel_cmtg_disable_requires_modeset(struct intel_display *display,
 	return cmtg_config->trans_a_secondary || cmtg_config->trans_b_secondary;
 }
 
-static void intel_cmtg_disable(struct intel_display *display,
-			       struct intel_cmtg_config *cmtg_config)
+static void intel_cmtg_disable_all(struct intel_display *display,
+				   struct intel_cmtg_config *cmtg_config)
 {
 	u32 clk_sel_clr = 0;
 	u32 clk_sel_set = 0;
@@ -140,14 +156,14 @@ static void intel_cmtg_disable(struct intel_display *display,
 
 	if (cmtg_config->cmtg_a_enable) {
 		drm_dbg_kms(display->drm, "Disabling CMTG A\n");
-		intel_de_rmw(display, TRANS_CMTG_CTL_A, CMTG_ENABLE, 0);
+		intel_de_rmw(display, TRANS_CMTG_CTL(TRANSCODER_A), CMTG_ENABLE, 0);
 		clk_sel_clr |= CMTG_CLK_SEL_A_MASK;
 		clk_sel_set |= CMTG_CLK_SEL_A_DISABLED;
 	}
 
 	if (cmtg_config->cmtg_b_enable) {
 		drm_dbg_kms(display->drm, "Disabling CMTG B\n");
-		intel_de_rmw(display, TRANS_CMTG_CTL_B, CMTG_ENABLE, 0);
+		intel_de_rmw(display, TRANS_CMTG_CTL(TRANSCODER_B), CMTG_ENABLE, 0);
 		clk_sel_clr |= CMTG_CLK_SEL_B_MASK;
 		clk_sel_set |= CMTG_CLK_SEL_B_DISABLED;
 	}
@@ -156,6 +172,57 @@ static void intel_cmtg_disable(struct intel_display *display,
 		intel_de_rmw(display, CMTG_CLK_SEL, clk_sel_clr, clk_sel_set);
 }
 
+void intel_cmtg_disable(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
+	enum transcoder cmtg_transcoder = to_cmtg_transcoder(crtc_state->cpu_transcoder);
+	u32 clk_sel_clr = 0, interrupt_mask = 0;
+
+	if (!crtc->cmtg.enabled)
+		return;
+
+	if (drm_WARN_ON(display->drm, cmtg_transcoder == INVALID_TRANSCODER))
+		return;
+
+	crtc->cmtg.enabled = false;
+	intel_de_rmw(display, TRANS_VRR_CTL(display, cmtg_transcoder),
+		     VRR_CTL_VRR_ENABLE | VRR_CTL_FLIP_LINE_EN, 0);
+
+	/*
+	 * Use cpu_transcoder for:
+	 * 1. Exclusive CMTG registers that do not use the standard transcoder offset
+	 *    (e.g., TRANS_CMTG_CTL, CMTG_CLK_SEL).
+	 * 2. Registers shared between the eDP and CMTG transcoders.
+	 *    (e.g., TRANS_DDI_FUNC_CTL2).
+	 */
+
+	intel_de_rmw(display, TRANS_DDI_FUNC_CTL2(display, cpu_transcoder),
+		     CMTG_SECONDARY_MODE, 0);
+	intel_de_rmw(display, CMTG_SCANLINE_GB1(cpu_transcoder), CMTG_HW_GB_ENABLE, 0);
+
+	intel_de_rmw(display, TRANS_CMTG_CTL(cpu_transcoder), CMTG_ENABLE, 0);
+
+	if (intel_de_wait_for_clear_ms(display, TRANS_CMTG_CTL(cpu_transcoder), CMTG_STATE, 50)) {
+		drm_WARN(display->drm, 1, "CMTG: %s disable timeout\n",
+			 transcoder_name(cpu_transcoder));
+		return;
+	}
+
+	clk_sel_clr = cpu_transcoder == TRANSCODER_A ? CMTG_CLK_SEL_A_MASK : CMTG_CLK_SEL_B_MASK;
+	intel_de_rmw(display, CMTG_CLK_SEL, clk_sel_clr, 0);
+
+	drm_dbg_kms(display->drm, "CMTG: %s disabled\n", transcoder_name(cpu_transcoder));
+
+	if (cpu_transcoder == TRANSCODER_A)
+		interrupt_mask = CMTG_VBLANK_A;
+	else if (cpu_transcoder == TRANSCODER_B)
+		interrupt_mask = CMTG_VBLANK_B;
+
+	intel_display_irq_port_interrupt_mask(display, interrupt_mask, true);
+}
+
 /*
  * Read out CMTG configuration and, on platforms that allow disabling it without
  * a modeset, do it.
@@ -183,5 +250,202 @@ void intel_cmtg_sanitize(struct intel_display *display)
 	if (intel_cmtg_disable_requires_modeset(display, &cmtg_config))
 		return;
 
-	intel_cmtg_disable(display, &cmtg_config);
+	intel_cmtg_disable_all(display, &cmtg_config);
+}
+
+bool intel_cmtg_is_allowed(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
+
+	if ((cpu_transcoder == TRANSCODER_A || cpu_transcoder == TRANSCODER_B) &&
+	    DISPLAY_VER(display) == 35 && intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
+		return true;
+
+	return false;
+}
+
+void intel_cmtg_set_clk_select(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
+	u32 clk_sel_clr = 0;
+	u32 clk_sel_set = 0;
+
+	if (!intel_cmtg_is_allowed(crtc_state))
+		return;
+
+	if (cpu_transcoder == TRANSCODER_A) {
+		clk_sel_clr = CMTG_CLK_SEL_A_MASK;
+		clk_sel_set = CMTG_CLK_SELECT_PHYA_ENABLE;
+	} else if (cpu_transcoder == TRANSCODER_B) {
+		clk_sel_clr = CMTG_CLK_SEL_B_MASK;
+		clk_sel_set = CMTG_CLK_SELECT_PHYB_ENABLE;
+	}
+
+	if (clk_sel_set)
+		intel_de_rmw(display, CMTG_CLK_SEL, clk_sel_clr, clk_sel_set);
+}
+
+void intel_cmtg_set_timings(const struct intel_crtc_state *crtc_state, enum set_timing_type type)
+{
+	enum transcoder cmtg_transcoder = to_cmtg_transcoder(crtc_state->cpu_transcoder);
+
+	if (cmtg_transcoder == INVALID_TRANSCODER)
+		return;
+
+	if (!intel_cmtg_is_allowed(crtc_state))
+		return;
+
+	if (type == LRR)
+		intel_set_transcoder_timings_lrr(crtc_state, cmtg_transcoder);
+	else
+		intel_set_transcoder_timings(crtc_state, cmtg_transcoder);
+}
+
+void intel_cmtg_set_vrr_timings(const struct intel_crtc_state *crtc_state)
+{
+	enum transcoder cmtg_transcoder = to_cmtg_transcoder(crtc_state->cpu_transcoder);
+
+	if (!intel_cmtg_is_allowed(crtc_state))
+		return;
+
+	intel_vrr_set_fixed_rr_timings(crtc_state, cmtg_transcoder);
+}
+
+void intel_cmtg_set_vrr_ctl(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	enum transcoder cmtg_transcoder = to_cmtg_transcoder(crtc_state->cpu_transcoder);
+	u32 vrr_ctl;
+
+	if (!intel_cmtg_is_allowed(crtc_state))
+		return;
+
+	vrr_ctl = VRR_CTL_VRR_ENABLE | VRR_CTL_FLIP_LINE_EN |
+		  XELPD_VRR_CTL_VRR_GUARDBAND(crtc_state->vrr.guardband);
+
+	/* TODO: The code below may need to be revisited once CMRR is enabled */
+	if (crtc_state->cmrr.enable)
+		vrr_ctl |= VRR_CTL_CMRR_ENABLE;
+
+	intel_de_write(display, TRANS_VRR_CTL(display, cmtg_transcoder), vrr_ctl);
+}
+
+void intel_cmtg_set_m_n(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	enum transcoder cmtg_transcoder = to_cmtg_transcoder(crtc_state->cpu_transcoder);
+	const struct intel_link_m_n *m_n = &crtc_state->dp_m_n;
+
+	if (!intel_cmtg_is_allowed(crtc_state))
+		return;
+
+	intel_de_write(display, PIPE_LINK_M1(display, cmtg_transcoder), m_n->link_m);
+	intel_de_write(display, PIPE_LINK_N1(display, cmtg_transcoder), m_n->link_n);
+}
+
+static void intel_cmtg_enable_sync(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
+	u32 cmtg_ctl;
+
+	cmtg_ctl = CMTG_SYNC_TO_PORT | CMTG_ENABLE;
+
+	intel_de_rmw(display, TRANS_CMTG_CTL(cpu_transcoder), 0, cmtg_ctl);
+	if (intel_de_wait_for_clear_ms(display, TRANS_CMTG_CTL(cpu_transcoder),
+				       CMTG_SYNC_TO_PORT, 50)) {
+		drm_WARN(display->drm, 1, "CMTG: %s enable timeout\n",
+			 transcoder_name(cpu_transcoder));
+	}
+}
+
+static void intel_cmtg_enable_ddi(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
+	u32 interrupt_mask = 0;
+
+	intel_de_rmw(display, TRANS_DDI_FUNC_CTL2(display, cpu_transcoder), 0, CMTG_SECONDARY_MODE);
+	intel_de_rmw(display, CMTG_SCANLINE_GB1(cpu_transcoder), 0, CMTG_HW_GB_ENABLE);
+	crtc->cmtg.enabled = true;
+	drm_dbg_kms(display->drm, "CMTG: %s enabled\n", transcoder_name(cpu_transcoder));
+
+	/*
+	 * TODO: Currently cmtg is enabled along with eDP transcoder so cmtg
+	 * interrupt is not enabled through IER, need to do some fine
+	 * tuning in future.
+	 */
+
+	if (cpu_transcoder == TRANSCODER_A)
+		interrupt_mask = CMTG_VBLANK_A;
+	else if (cpu_transcoder == TRANSCODER_B)
+		interrupt_mask = CMTG_VBLANK_B;
+
+	intel_display_irq_port_interrupt_mask(display, interrupt_mask, false);
+}
+
+/* Bspec: 75253 */
+#define DC3CO_ENTRY_LATENCY_US	55
+#define DC3CO_EXIT_LATENCY_US	40
+
+static void intel_cmtg_set_hwgb(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
+	u32 breakeven_gb;
+	u32 dc5_exit_latency;
+	u32 line_time_us = 75;	/* Max default initialization value */
+	u32 val;
+
+	if (crtc_state->linetime)
+		line_time_us = DIV_ROUND_UP(crtc_state->linetime, 8);
+
+	/* Break Even Guardband - DC3co Entry Latency / linetime */
+	breakeven_gb = DIV_ROUND_UP(DC3CO_ENTRY_LATENCY_US, line_time_us);
+
+	/* DC5 Exit Latency - DC3co Exit Latency / linetime */
+	dc5_exit_latency = DIV_ROUND_UP(DC3CO_EXIT_LATENCY_US, line_time_us);
+
+	val = REG_FIELD_PREP(CMTG_HW_GB_BREAKEVEN_MASK, breakeven_gb) |
+	      REG_FIELD_PREP(CMTG_HW_GB_DC5_EXIT_LATENCY_MASK, dc5_exit_latency) |
+	      REG_FIELD_PREP(CMTG_HW_GB_UP_LW_BG_DIFF_MASK, 1);
+
+	intel_de_write(display, CMTG_HW_GB(cpu_transcoder), val);
+}
+
+static void intel_cmtg_restore(const struct intel_crtc_state *crtc_state)
+{
+	intel_cmtg_set_clk_select(crtc_state);
+	intel_cmtg_set_timings(crtc_state, MODESET);
+	intel_cmtg_set_vrr_timings(crtc_state);
+	intel_cmtg_set_vrr_ctl(crtc_state);
+	intel_cmtg_set_m_n(crtc_state);
+}
+
+void intel_cmtg_program(struct intel_atomic_state *state)
+{
+	struct intel_display *display = to_intel_display(state);
+	struct intel_crtc *crtc;
+	struct intel_crtc_state *new_crtc_state;
+	bool dc3co_to_dc6 = intel_display_power_get_and_reset_dc3co_to_dc6(display);
+
+	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state) {
+		bool modeset = intel_crtc_needs_modeset(new_crtc_state);
+
+		if (!intel_cmtg_is_allowed(new_crtc_state))
+			continue;
+
+		if ((modeset || dc3co_to_dc6) &&
+		    new_crtc_state->hw.active && !crtc->cmtg.enabled) {
+			if (dc3co_to_dc6)
+				intel_cmtg_restore(new_crtc_state);
+
+			intel_cmtg_enable_sync(new_crtc_state);
+			intel_cmtg_set_hwgb(new_crtc_state);
+			intel_cmtg_enable_ddi(new_crtc_state);
+		}
+	}
 }
diff --git a/drivers/gpu/drm/i915/display/intel_cmtg.h b/drivers/gpu/drm/i915/display/intel_cmtg.h
index ba62199adaa21..a08cb2dcee670 100644
--- a/drivers/gpu/drm/i915/display/intel_cmtg.h
+++ b/drivers/gpu/drm/i915/display/intel_cmtg.h
@@ -6,8 +6,25 @@
 #ifndef __INTEL_CMTG_H__
 #define __INTEL_CMTG_H__
 
+#include <linux/types.h>
+
+struct intel_atomic_state;
 struct intel_display;
+struct intel_crtc_state;
+
+enum set_timing_type {
+	MODESET = 0,
+	LRR
+};
 
+void intel_cmtg_disable(const struct intel_crtc_state *crtc_state);
+void intel_cmtg_set_m_n(const struct intel_crtc_state *crtc_state);
+void intel_cmtg_set_vrr_timings(const struct intel_crtc_state *crtc_state);
+void intel_cmtg_set_vrr_ctl(const struct intel_crtc_state *crtc_state);
+void intel_cmtg_set_timings(const struct intel_crtc_state *crtc_state, enum set_timing_type type);
+void intel_cmtg_set_clk_select(const struct intel_crtc_state *crtc_state);
 void intel_cmtg_sanitize(struct intel_display *display);
+bool intel_cmtg_is_allowed(const struct intel_crtc_state *crtc_state);
+void intel_cmtg_program(struct intel_atomic_state *state);
 
 #endif /* __INTEL_CMTG_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_cmtg_regs.h b/drivers/gpu/drm/i915/display/intel_cmtg_regs.h
index 945a355782841..18dcb665df049 100644
--- a/drivers/gpu/drm/i915/display/intel_cmtg_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_cmtg_regs.h
@@ -10,12 +10,32 @@
 
 #define CMTG_CLK_SEL			_MMIO(0x46160)
 #define CMTG_CLK_SEL_A_MASK		REG_GENMASK(31, 29)
+#define CMTG_CLK_SELECT_PHYA_ENABLE	REG_FIELD_PREP(CMTG_CLK_SEL_A_MASK, 0x4)
 #define CMTG_CLK_SEL_A_DISABLED		REG_FIELD_PREP(CMTG_CLK_SEL_A_MASK, 0)
 #define CMTG_CLK_SEL_B_MASK		REG_GENMASK(15, 13)
+#define CMTG_CLK_SELECT_PHYB_ENABLE	REG_FIELD_PREP(CMTG_CLK_SEL_B_MASK, 0x6)
 #define CMTG_CLK_SEL_B_DISABLED		REG_FIELD_PREP(CMTG_CLK_SEL_B_MASK, 0)
 
-#define TRANS_CMTG_CTL_A		_MMIO(0x6fa88)
-#define TRANS_CMTG_CTL_B		_MMIO(0x6fb88)
+#define _TRANS_CMTG_CTL_A		0x6fa88
+#define _TRANS_CMTG_CTL_B		0x6fb88
+#define TRANS_CMTG_CTL(trans)		_MMIO_TRANS((trans), \
+						    _TRANS_CMTG_CTL_A, _TRANS_CMTG_CTL_B)
 #define  CMTG_ENABLE			REG_BIT(31)
+#define  CMTG_SYNC_TO_PORT		REG_BIT(29)
+#define  CMTG_STATE			REG_BIT(23)
+
+#define _CMTG_HW_GB_A				0x6fa8c
+#define _CMTG_HW_GB_B				0x6fb8c
+#define CMTG_HW_GB(trans)			_MMIO_TRANS((trans), \
+							    _CMTG_HW_GB_A, _CMTG_HW_GB_B)
+#define CMTG_HW_GB_BREAKEVEN_MASK		REG_GENMASK(11, 0)
+#define CMTG_HW_GB_DC5_EXIT_LATENCY_MASK	REG_GENMASK(27, 16)
+#define CMTG_HW_GB_UP_LW_BG_DIFF_MASK		REG_GENMASK(31, 28)
+
+#define _CMTG_SCANLINE_GB1_A		0x456A0
+#define _CMTG_SCANLINE_GB1_B		0x456C0
+#define CMTG_SCANLINE_GB1(trans)	_MMIO_TRANS((trans), \
+						    _CMTG_SCANLINE_GB1_A, _CMTG_SCANLINE_GB1_B)
+#define  CMTG_HW_GB_ENABLE		REG_BIT(31)
 
 #endif /* __INTEL_CMTG_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c
index 7ef870cd9a169..87ced9f6ff408 100644
--- a/drivers/gpu/drm/i915/display/intel_color.c
+++ b/drivers/gpu/drm/i915/display/intel_color.c
@@ -3967,48 +3967,34 @@ xelpd_program_plane_pre_csc_lut(struct intel_dsb *dsb,
 	enum pipe pipe = to_intel_plane(state->plane)->pipe;
 	enum plane_id plane = to_intel_plane(state->plane)->id;
 	const struct drm_color_lut32 *pre_csc_lut = plane_state->hw.degamma_lut->data;
-	u32 i, lut_size;
+	int i, lut_size = 128;
+	u32 lut_val;
 
-	if (icl_is_hdr_plane(display, plane)) {
-		lut_size = 128;
+	if (!icl_is_hdr_plane(display, plane))
+		return;
 
-		intel_de_write_dsb(display, dsb,
-				   PLANE_PRE_CSC_GAMC_INDEX_ENH(pipe, plane, 0),
-				   PLANE_PAL_PREC_AUTO_INCREMENT);
+	intel_de_write_dsb(display, dsb,
+			   PLANE_PRE_CSC_GAMC_INDEX_ENH(pipe, plane, 0),
+			   PLANE_PAL_PREC_AUTO_INCREMENT);
 
+	for (i = 0; i < lut_size + 3; i++) {
 		if (pre_csc_lut) {
-			for (i = 0; i < lut_size; i++) {
-				u32 lut_val = drm_color_lut32_extract(pre_csc_lut[i].green, 24);
-
-				intel_de_write_dsb(display, dsb,
-						   PLANE_PRE_CSC_GAMC_DATA_ENH(pipe, plane, 0),
-						   lut_val);
-			}
-
-			/* Program the max register to clamp values > 1.0. */
-			/* TODO: Restrict to 0x7ffffff */
-			do {
-				intel_de_write_dsb(display, dsb,
-						   PLANE_PRE_CSC_GAMC_DATA_ENH(pipe, plane, 0),
-						   (1 << 24));
-			} while (i++ < 130);
+			if (i < lut_size)
+				lut_val = drm_color_lut32_extract(pre_csc_lut[i].green, 24);
+			/* else duplicate last lut_val */
 		} else {
-			for (i = 0; i < lut_size; i++) {
-				u32 v = (i * ((1 << 24) - 1)) / (lut_size - 1);
-
-				intel_de_write_dsb(display, dsb,
-						   PLANE_PRE_CSC_GAMC_DATA_ENH(pipe, plane, 0), v);
-			}
-
-			do {
-				intel_de_write_dsb(display, dsb,
-						   PLANE_PRE_CSC_GAMC_DATA_ENH(pipe, plane, 0),
-						   1 << 24);
-			} while (i++ < 130);
+			if (i < lut_size)
+				lut_val = (i * ((1 << 24) - 1)) / (lut_size - 1);
+			else
+				lut_val = 1 << 24;
 		}
 
-		intel_de_write_dsb(display, dsb, PLANE_PRE_CSC_GAMC_INDEX_ENH(pipe, plane, 0), 0);
+		intel_de_write_dsb(display, dsb,
+				   PLANE_PRE_CSC_GAMC_DATA_ENH(pipe, plane, 0),
+				   lut_val);
 	}
+
+	intel_de_write_dsb(display, dsb, PLANE_PRE_CSC_GAMC_INDEX_ENH(pipe, plane, 0), 0);
 }
 
 static void
@@ -4020,51 +4006,38 @@ xelpd_program_plane_post_csc_lut(struct intel_dsb *dsb,
 	enum pipe pipe = to_intel_plane(state->plane)->pipe;
 	enum plane_id plane = to_intel_plane(state->plane)->id;
 	const struct drm_color_lut32 *post_csc_lut = plane_state->hw.gamma_lut->data;
-	u32 i, lut_size, lut_val;
-
-	if (icl_is_hdr_plane(display, plane)) {
-		intel_de_write_dsb(display, dsb, PLANE_POST_CSC_GAMC_INDEX_ENH(pipe, plane, 0),
-				   PLANE_PAL_PREC_AUTO_INCREMENT);
-		/* TODO: Add macro */
-		intel_de_write_dsb(display, dsb, PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH(pipe, plane, 0),
-				   PLANE_PAL_PREC_AUTO_INCREMENT);
+	int i, lut_size = 32;
+	u32 lut_val;
+
+	if (!icl_is_hdr_plane(display, plane))
+		return;
+
+	intel_de_write_dsb(display, dsb, PLANE_POST_CSC_GAMC_INDEX_ENH(pipe, plane, 0),
+			   PLANE_PAL_PREC_AUTO_INCREMENT);
+	/* TODO: Add macro */
+	intel_de_write_dsb(display, dsb, PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH(pipe, plane, 0),
+			   PLANE_PAL_PREC_AUTO_INCREMENT);
+
+	for (i = 0; i < lut_size + 3; i++) {
 		if (post_csc_lut) {
-			lut_size = 32;
-			for (i = 0; i < lut_size; i++) {
+			if (i < lut_size)
 				lut_val = drm_color_lut32_extract(post_csc_lut[i].green, 24);
-
-				intel_de_write_dsb(display, dsb,
-						   PLANE_POST_CSC_GAMC_DATA_ENH(pipe, plane, 0),
-						   lut_val);
-			}
-
-			/* Segment 2 */
-			do {
-				intel_de_write_dsb(display, dsb,
-						   PLANE_POST_CSC_GAMC_DATA_ENH(pipe, plane, 0),
-						   (1 << 24));
-			} while (i++ < 34);
+			/* else clamp to the last LUT value to prevent step discontinuity */
 		} else {
-			/*TODO: Add for segment 0 */
-			lut_size = 32;
-			for (i = 0; i < lut_size; i++) {
-				u32 v = (i * ((1 << 24) - 1)) / (lut_size - 1);
-
-				intel_de_write_dsb(display, dsb,
-						   PLANE_POST_CSC_GAMC_DATA_ENH(pipe, plane, 0), v);
-			}
-
-			do {
-				intel_de_write_dsb(display, dsb,
-						   PLANE_POST_CSC_GAMC_DATA_ENH(pipe, plane, 0),
-						   1 << 24);
-			} while (i++ < 34);
+			if (i < lut_size)
+				lut_val = (i * ((1 << 24) - 1)) / (lut_size - 1);
+			else
+				lut_val = 1 << 24;
 		}
 
-		intel_de_write_dsb(display, dsb, PLANE_POST_CSC_GAMC_INDEX_ENH(pipe, plane, 0), 0);
 		intel_de_write_dsb(display, dsb,
-				   PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH(pipe, plane, 0), 0);
+				   PLANE_POST_CSC_GAMC_DATA_ENH(pipe, plane, 0),
+				   lut_val);
 	}
+
+	intel_de_write_dsb(display, dsb, PLANE_POST_CSC_GAMC_INDEX_ENH(pipe, plane, 0), 0);
+	intel_de_write_dsb(display, dsb,
+			   PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH(pipe, plane, 0), 0);
 }
 
 static void
diff --git a/drivers/gpu/drm/i915/display/intel_crt.c b/drivers/gpu/drm/i915/display/intel_crt.c
index 243e332bef57a..5b8968197fbcf 100644
--- a/drivers/gpu/drm/i915/display/intel_crt.c
+++ b/drivers/gpu/drm/i915/display/intel_crt.c
@@ -397,7 +397,8 @@ intel_crt_mode_valid(struct drm_connector *connector,
 	return MODE_OK;
 }
 
-static int intel_crt_compute_config(struct intel_encoder *encoder,
+static int intel_crt_compute_config(struct intel_atomic_state *state,
+				    struct intel_encoder *encoder,
 				    struct intel_crtc_state *crtc_state,
 				    struct drm_connector_state *conn_state)
 {
@@ -413,7 +414,8 @@ static int intel_crt_compute_config(struct intel_encoder *encoder,
 	return 0;
 }
 
-static int pch_crt_compute_config(struct intel_encoder *encoder,
+static int pch_crt_compute_config(struct intel_atomic_state *state,
+				  struct intel_encoder *encoder,
 				  struct intel_crtc_state *crtc_state,
 				  struct drm_connector_state *conn_state)
 {
@@ -432,7 +434,8 @@ static int pch_crt_compute_config(struct intel_encoder *encoder,
 	return 0;
 }
 
-static int hsw_crt_compute_config(struct intel_encoder *encoder,
+static int hsw_crt_compute_config(struct intel_atomic_state *state,
+				  struct intel_encoder *encoder,
 				  struct intel_crtc_state *crtc_state,
 				  struct drm_connector_state *conn_state)
 {
diff --git a/drivers/gpu/drm/i915/display/intel_crtc.c b/drivers/gpu/drm/i915/display/intel_crtc.c
index 7ce2b52297d17..10ed9bdfee763 100644
--- a/drivers/gpu/drm/i915/display/intel_crtc.c
+++ b/drivers/gpu/drm/i915/display/intel_crtc.c
@@ -182,7 +182,7 @@ void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
 {
 	memset(crtc_state, 0, sizeof(*crtc_state));
 
-	__drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
+	__drm_atomic_helper_crtc_state_init(&crtc_state->uapi, &crtc->base);
 
 	crtc_state->cpu_transcoder = INVALID_TRANSCODER;
 	crtc_state->master_transcoder = INVALID_TRANSCODER;
diff --git a/drivers/gpu/drm/i915/display/intel_cursor.c b/drivers/gpu/drm/i915/display/intel_cursor.c
index 52347668f27d6..0673f16f6fd0d 100644
--- a/drivers/gpu/drm/i915/display/intel_cursor.c
+++ b/drivers/gpu/drm/i915/display/intel_cursor.c
@@ -876,7 +876,7 @@ intel_legacy_cursor_update(struct drm_plane *_plane,
 	new_plane_state->uapi.crtc_w = crtc_w;
 	new_plane_state->uapi.crtc_h = crtc_h;
 
-	intel_plane_copy_uapi_to_hw_state(new_plane_state, new_plane_state, crtc);
+	intel_plane_copy_uapi_to_hw_state(NULL, new_plane_state, new_plane_state, crtc);
 
 	ret = intel_plane_atomic_check_with_state(crtc_state, new_crtc_state,
 						  old_plane_state, new_plane_state);
@@ -1078,6 +1078,9 @@ intel_cursor_plane_create(struct intel_display *display,
 
 	intel_cursor_add_size_hints_property(cursor);
 
+	drm_plane_create_blend_mode_property(&cursor->base,
+					     BIT(DRM_MODE_BLEND_PREMULTI));
+
 	zpos = DISPLAY_RUNTIME_INFO(display)->num_sprites[pipe] + 1;
 	drm_plane_create_zpos_immutable_property(&cursor->base, zpos);
 
diff --git a/drivers/gpu/drm/i915/display/intel_cx0_phy.c b/drivers/gpu/drm/i915/display/intel_cx0_phy.c
index 24a51ab21b557..452062417ce94 100644
--- a/drivers/gpu/drm/i915/display/intel_cx0_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_cx0_phy.c
@@ -9,6 +9,7 @@
 #include <drm/drm_print.h>
 
 #include "intel_alpm.h"
+#include "intel_cmtg.h"
 #include "intel_cx0_phy.h"
 #include "intel_cx0_phy_regs.h"
 #include "intel_display_regs.h"
@@ -3418,10 +3419,20 @@ void intel_mtl_pll_enable(struct intel_encoder *encoder,
 void intel_mtl_pll_enable_clock(struct intel_encoder *encoder,
 				const struct intel_crtc_state *crtc_state)
 {
+	struct intel_display *display = to_intel_display(encoder);
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 
 	if (intel_tc_port_in_tbt_alt_mode(dig_port))
 		intel_mtl_tbt_pll_enable_clock(encoder, crtc_state->port_clock);
+
+	/*
+	 * CMTG can be enabled only when the transcoder and port are compatible
+	 * (transcoder A with port A, transcoder B with port B).
+	 */
+	if (HAS_LT_PHY(display) &&
+	    ((crtc_state->cpu_transcoder == TRANSCODER_A && encoder->port == PORT_A) ||
+	     (crtc_state->cpu_transcoder == TRANSCODER_B && encoder->port == PORT_B)))
+		intel_cmtg_set_clk_select(crtc_state);
 }
 
 /*
diff --git a/drivers/gpu/drm/i915/display/intel_cx0_phy.h b/drivers/gpu/drm/i915/display/intel_cx0_phy.h
index 1428e7a5a318b..95c20eb4b4b7f 100644
--- a/drivers/gpu/drm/i915/display/intel_cx0_phy.h
+++ b/drivers/gpu/drm/i915/display/intel_cx0_phy.h
@@ -60,7 +60,6 @@ void intel_cx0_phy_set_signal_levels(struct intel_encoder *encoder,
 				     const struct intel_crtc_state *crtc_state);
 void intel_cx0_powerdown_change_sequence(struct intel_encoder *encoder,
 					 u8 lane_mask, u8 state);
-int intel_cx0_phy_check_hdmi_link_rate(struct intel_hdmi *hdmi, int clock);
 void intel_cx0_setup_powerdown(struct intel_encoder *encoder);
 bool intel_cx0_is_hdmi_frl(u32 clock);
 u8 intel_cx0_read(struct intel_encoder *encoder, u8 lane_mask, u16 addr);
diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c
index 6296635c4e792..617106c68cb46 100644
--- a/drivers/gpu/drm/i915/display/intel_ddi.c
+++ b/drivers/gpu/drm/i915/display/intel_ddi.c
@@ -3501,6 +3501,8 @@ static void intel_ddi_enable_hdmi(struct intel_atomic_state *state,
 	}
 
 	intel_ddi_buf_enable(encoder, buf_ctl);
+
+	intel_hdmi_poll_for_scrambling_enable(crtc_state, connector);
 }
 
 static void intel_ddi_enable(struct intel_atomic_state *state,
@@ -4483,7 +4485,8 @@ intel_ddi_compute_output_type(struct intel_encoder *encoder,
 	}
 }
 
-static int intel_ddi_compute_config(struct intel_encoder *encoder,
+static int intel_ddi_compute_config(struct intel_atomic_state *state,
+				    struct intel_encoder *encoder,
 				    struct intel_crtc_state *pipe_config,
 				    struct drm_connector_state *conn_state)
 {
@@ -4501,7 +4504,7 @@ static int intel_ddi_compute_config(struct intel_encoder *encoder,
 
 		ret = intel_hdmi_compute_config(encoder, pipe_config, conn_state);
 	} else {
-		ret = intel_dp_compute_config(encoder, pipe_config, conn_state);
+		ret = intel_dp_compute_config(state, encoder, pipe_config, conn_state);
 	}
 
 	if (ret)
@@ -4560,7 +4563,7 @@ static bool crtcs_port_sync_compatible(const struct intel_crtc_state *crtc_state
 		m_n_equal(&crtc_state1->dp_m_n, &crtc_state2->dp_m_n);
 }
 
-static u8
+static u16
 intel_ddi_port_sync_transcoders(const struct intel_crtc_state *ref_crtc_state,
 				int tile_group_id)
 {
@@ -4569,7 +4572,7 @@ intel_ddi_port_sync_transcoders(const struct intel_crtc_state *ref_crtc_state,
 	const struct drm_connector_state *conn_state;
 	struct intel_atomic_state *state =
 		to_intel_atomic_state(ref_crtc_state->uapi.state);
-	u8 transcoders = 0;
+	u16 transcoders = 0;
 	int i;
 
 	/*
@@ -4606,13 +4609,14 @@ intel_ddi_port_sync_transcoders(const struct intel_crtc_state *ref_crtc_state,
 	return transcoders;
 }
 
-static int intel_ddi_compute_config_late(struct intel_encoder *encoder,
+static int intel_ddi_compute_config_late(struct intel_atomic_state *state,
+					 struct intel_encoder *encoder,
 					 struct intel_crtc_state *crtc_state,
 					 struct drm_connector_state *conn_state)
 {
 	struct intel_display *display = to_intel_display(encoder);
 	struct drm_connector *connector = conn_state->connector;
-	u8 port_sync_transcoders = 0;
+	u16 port_sync_transcoders = 0;
 	int ret = 0;
 
 	if (intel_crtc_has_dp_encoder(crtc_state))
@@ -4659,6 +4663,7 @@ static void intel_ddi_encoder_destroy(struct drm_encoder *encoder)
 
 	drm_encoder_cleanup(encoder);
 	kfree(dig_port->hdcp.port_data.streams);
+	intel_dp_link_cleanup(&dig_port->dp);
 	kfree(dig_port);
 }
 
@@ -4696,11 +4701,16 @@ static int intel_ddi_init_dp_connector(struct intel_digital_port *dig_port)
 	struct intel_display *display = to_intel_display(dig_port);
 	struct intel_connector *connector;
 	enum port port = dig_port->base.port;
+	int err;
 
 	connector = intel_connector_alloc();
 	if (!connector)
 		return -ENOMEM;
 
+	err = intel_dp_link_init(&dig_port->dp);
+	if (err)
+		goto err_dp_init;
+
 	dig_port->dp.output_reg = DDI_BUF_CTL(port);
 	if (DISPLAY_VER(display) >= 14)
 		dig_port->dp.prepare_link_retrain = mtl_ddi_prepare_link_retrain;
@@ -4713,8 +4723,9 @@ static int intel_ddi_init_dp_connector(struct intel_digital_port *dig_port)
 	dig_port->dp.preemph_max = intel_ddi_dp_preemph_max;
 
 	if (!intel_dp_init_connector(dig_port, connector)) {
-		kfree(connector);
-		return -EINVAL;
+		err = -EINVAL;
+
+		goto err_init_connector;
 	}
 
 	if (dig_port->base.type == INTEL_OUTPUT_EDP) {
@@ -4730,6 +4741,13 @@ static int intel_ddi_init_dp_connector(struct intel_digital_port *dig_port)
 	}
 
 	return 0;
+
+err_init_connector:
+	intel_dp_link_cleanup(&dig_port->dp);
+err_dp_init:
+	kfree(connector);
+
+	return err;
 }
 
 static void intel_ddi_cleanup_dp_connector(struct intel_digital_port *dig_port)
@@ -4738,6 +4756,7 @@ static void intel_ddi_cleanup_dp_connector(struct intel_digital_port *dig_port)
 	struct intel_connector *connector = intel_dp->attached_connector;
 
 	intel_dp_cleanup_connector(dig_port, connector);
+	intel_dp_link_cleanup(intel_dp);
 	kfree(connector);
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_de.c b/drivers/gpu/drm/i915/display/intel_de.c
index 6daee9e82503b..a7417905192d1 100644
--- a/drivers/gpu/drm/i915/display/intel_de.c
+++ b/drivers/gpu/drm/i915/display/intel_de.c
@@ -9,11 +9,11 @@
 
 #include "intel_de.h"
 
-static int __intel_de_wait_for_register(struct intel_display *display,
-					intel_reg_t reg, u32 mask, u32 value,
-					unsigned int timeout_us,
-					u32 (*read)(struct intel_display *display, intel_reg_t reg),
-					u32 *out_val, bool is_atomic)
+static int intel_de_wait_for_register(struct intel_display *display,
+				      intel_reg_t reg, u32 mask, u32 value,
+				      unsigned int timeout_us,
+				      u32 (*read)(struct intel_display *display, intel_reg_t reg),
+				      u32 *out_val, bool is_atomic)
 {
 	const ktime_t end = ktime_add_us(ktime_get_raw(), timeout_us);
 	int wait_max = 1000;
@@ -60,28 +60,6 @@ static int __intel_de_wait_for_register(struct intel_display *display,
 	return ret;
 }
 
-static int intel_de_wait_for_register(struct intel_display *display,
-				      intel_reg_t reg, u32 mask, u32 value,
-				      unsigned int fast_timeout_us,
-				      unsigned int slow_timeout_us,
-				      u32 (*read)(struct intel_display *display, intel_reg_t reg),
-				      u32 *out_value, bool is_atomic)
-{
-	int ret = -EINVAL;
-
-	if (fast_timeout_us)
-		ret = __intel_de_wait_for_register(display, reg, mask, value,
-						   fast_timeout_us, read,
-						   out_value, is_atomic);
-
-	if (ret && slow_timeout_us)
-		ret = __intel_de_wait_for_register(display, reg, mask, value,
-						   slow_timeout_us, read,
-						   out_value, is_atomic);
-
-	return ret;
-}
-
 int intel_de_wait_us(struct intel_display *display, intel_reg_t reg,
 		     u32 mask, u32 value, unsigned int timeout_us,
 		     u32 *out_value)
@@ -91,7 +69,7 @@ int intel_de_wait_us(struct intel_display *display, intel_reg_t reg,
 	intel_dmc_wl_get(display, reg);
 
 	ret = intel_de_wait_for_register(display, reg, mask, value,
-					 timeout_us, 0,
+					 timeout_us,
 					 intel_de_read,
 					 out_value, false);
 
@@ -109,7 +87,7 @@ int intel_de_wait_ms(struct intel_display *display, intel_reg_t reg,
 	intel_dmc_wl_get(display, reg);
 
 	ret = intel_de_wait_for_register(display, reg, mask, value,
-					 2, timeout_ms * 1000,
+					 timeout_ms * 1000,
 					 intel_de_read,
 					 out_value, false);
 
@@ -123,7 +101,7 @@ int intel_de_wait_fw_ms(struct intel_display *display, intel_reg_t reg,
 			u32 *out_value)
 {
 	return intel_de_wait_for_register(display, reg, mask, value,
-					  2, timeout_ms * 1000,
+					  timeout_ms * 1000,
 					  intel_de_read_fw,
 					  out_value, false);
 }
@@ -133,7 +111,7 @@ int intel_de_wait_fw_us_atomic(struct intel_display *display, intel_reg_t reg,
 			       u32 *out_value)
 {
 	return intel_de_wait_for_register(display, reg, mask, value,
-					  timeout_us, 0,
+					  timeout_us,
 					  intel_de_read_fw,
 					  out_value, true);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c
index 8e269b71f18e4..38763a6802c51 100644
--- a/drivers/gpu/drm/i915/display/intel_display.c
+++ b/drivers/gpu/drm/i915/display/intel_display.c
@@ -45,6 +45,7 @@
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_rect.h>
 #include <drm/drm_vblank.h>
+#include <drm/intel/step.h>
 
 #include "g4x_dp.h"
 #include "g4x_hdmi.h"
@@ -60,6 +61,7 @@
 #include "intel_bw.h"
 #include "intel_cdclk.h"
 #include "intel_clock_gating.h"
+#include "intel_cmtg.h"
 #include "intel_color.h"
 #include "intel_crt.h"
 #include "intel_crtc.h"
@@ -132,7 +134,6 @@
 #include "vlv_dsi_pll.h"
 #include "vlv_dsi_regs.h"
 
-static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_state);
 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
 static void hsw_set_transconf(const struct intel_crtc_state *crtc_state);
 static void bdw_set_pipe_misc(struct intel_dsb *dsb,
@@ -1504,7 +1505,7 @@ static void ilk_configure_cpu_transcoder(const struct intel_crtc_state *crtc_sta
 					       &crtc_state->dp_m2_n2);
 	}
 
-	intel_set_transcoder_timings(crtc_state);
+	intel_set_transcoder_timings(crtc_state, crtc_state->cpu_transcoder);
 
 	ilk_set_pipeconf(crtc_state);
 }
@@ -1635,7 +1636,9 @@ static void hsw_configure_cpu_transcoder(const struct intel_crtc_state *crtc_sta
 					       &crtc_state->dp_m2_n2);
 	}
 
-	intel_set_transcoder_timings(crtc_state);
+	intel_cmtg_set_m_n(crtc_state);
+	intel_set_transcoder_timings(crtc_state, crtc_state->cpu_transcoder);
+	intel_cmtg_set_timings(crtc_state, MODESET);
 
 	if (cpu_transcoder != TRANSCODER_EDP)
 		intel_de_write(display, TRANS_MULT(display, cpu_transcoder),
@@ -1788,6 +1791,10 @@ static void hsw_crtc_disable(struct intel_atomic_state *state,
 		intel_atomic_get_old_crtc_state(state, crtc);
 	struct intel_crtc *pipe_crtc;
 
+	if (crtc->cmtg.enabled && intel_cmtg_is_allowed(old_crtc_state)) {
+		intel_cmtg_set_clk_select(old_crtc_state);
+		intel_cmtg_disable(old_crtc_state);
+	}
 	/*
 	 * FIXME collapse everything to one hook.
 	 * Need care with mst->ddi interactions.
@@ -2048,7 +2055,7 @@ static void i9xx_configure_cpu_transcoder(const struct intel_crtc_state *crtc_st
 					       &crtc_state->dp_m2_n2);
 	}
 
-	intel_set_transcoder_timings(crtc_state);
+	intel_set_transcoder_timings(crtc_state, crtc_state->cpu_transcoder);
 
 	i9xx_set_pipeconf(crtc_state);
 }
@@ -2664,17 +2671,17 @@ transcoder_has_vrr(const struct intel_crtc_state *crtc_state)
 	return HAS_VRR(display) && !transcoder_is_dsi(cpu_transcoder);
 }
 
-static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_state)
+void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_state,
+				  enum transcoder transcoder)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	enum pipe pipe = crtc->pipe;
-	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
 	u32 crtc_vdisplay, crtc_vtotal, crtc_vblank_start, crtc_vblank_end;
 	int vsyncshift = 0;
 
-	drm_WARN_ON(display->drm, transcoder_is_dsi(cpu_transcoder));
+	drm_WARN_ON(display->drm, transcoder_is_dsi(transcoder));
 
 	/* We need to be careful not to changed the adjusted mode, for otherwise
 	 * the hw state checker will get angry at the mismatch. */
@@ -2703,7 +2710,7 @@ static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_sta
 	 */
 	if (DISPLAY_VER(display) >= 13) {
 		intel_de_write(display,
-			       TRANS_SET_CONTEXT_LATENCY(display, cpu_transcoder),
+			       TRANS_SET_CONTEXT_LATENCY(display, transcoder),
 			       crtc_state->set_context_latency);
 
 		/*
@@ -2718,35 +2725,30 @@ static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_sta
 
 	if (DISPLAY_VER(display) >= 4 && DISPLAY_VER(display) < 35)
 		intel_de_write(display,
-			       TRANS_VSYNCSHIFT(display, cpu_transcoder),
+			       TRANS_VSYNCSHIFT(display, transcoder),
 			       vsyncshift);
 
-	intel_de_write(display, TRANS_HTOTAL(display, cpu_transcoder),
+	intel_de_write(display, TRANS_HTOTAL(display, transcoder),
 		       HACTIVE(adjusted_mode->crtc_hdisplay - 1) |
 		       HTOTAL(adjusted_mode->crtc_htotal - 1));
-	intel_de_write(display, TRANS_HBLANK(display, cpu_transcoder),
+	intel_de_write(display, TRANS_HBLANK(display, transcoder),
 		       HBLANK_START(adjusted_mode->crtc_hblank_start - 1) |
 		       HBLANK_END(adjusted_mode->crtc_hblank_end - 1));
-	intel_de_write(display, TRANS_HSYNC(display, cpu_transcoder),
+	intel_de_write(display, TRANS_HSYNC(display, transcoder),
 		       HSYNC_START(adjusted_mode->crtc_hsync_start - 1) |
 		       HSYNC_END(adjusted_mode->crtc_hsync_end - 1));
 
-	/*
-	 * For platforms that always use VRR Timing Generator, the VTOTAL.Vtotal
-	 * bits are not required. Since the support for these bits is going to
-	 * be deprecated in upcoming platforms, avoid writing these bits for the
-	 * platforms that do not use legacy Timing Generator.
-	 */
-	if (intel_vrr_always_use_vrr_tg(display))
+	if (display->platform.novalake &&
+	    IS_DISPLAY_STEP(display, STEP_A0, STEP_C0))
 		crtc_vtotal = 1;
 
-	intel_de_write(display, TRANS_VTOTAL(display, cpu_transcoder),
+	intel_de_write(display, TRANS_VTOTAL(display, transcoder),
 		       VACTIVE(crtc_vdisplay - 1) |
 		       VTOTAL(crtc_vtotal - 1));
-	intel_de_write(display, TRANS_VBLANK(display, cpu_transcoder),
+	intel_de_write(display, TRANS_VBLANK(display, transcoder),
 		       VBLANK_START(crtc_vblank_start - 1) |
 		       VBLANK_END(crtc_vblank_end - 1));
-	intel_de_write(display, TRANS_VSYNC(display, cpu_transcoder),
+	intel_de_write(display, TRANS_VSYNC(display, transcoder),
 		       VSYNC_START(adjusted_mode->crtc_vsync_start - 1) |
 		       VSYNC_END(adjusted_mode->crtc_vsync_end - 1));
 
@@ -2754,13 +2756,15 @@ static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_sta
 	 * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
 	 * documented on the DDI_FUNC_CTL register description, EDP Input Select
 	 * bits. */
-	if (display->platform.haswell && cpu_transcoder == TRANSCODER_EDP &&
+	if (display->platform.haswell && transcoder == TRANSCODER_EDP &&
 	    (pipe == PIPE_B || pipe == PIPE_C))
 		intel_de_write(display, TRANS_VTOTAL(display, pipe),
 			       VACTIVE(crtc_vdisplay - 1) |
 			       VTOTAL(crtc_vtotal - 1));
 
-	if (DISPLAY_VER(display) >= 30) {
+	if (DISPLAY_VER(display) >= 30 &&
+	    transcoder != TRANSCODER_CMTG0 &&
+	    transcoder != TRANSCODER_CMTG1) {
 		/*
 		 * Address issues for resolutions with high refresh rate that
 		 * have small Hblank, specifically where Hblank is smaller than
@@ -2769,19 +2773,19 @@ static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_sta
 		 * followed by BE which DPRX devices are unable to handle.
 		 * https://groups.vesa.org/wg/DP/document/20494
 		 */
-		intel_de_write(display, DP_MIN_HBLANK_CTL(cpu_transcoder),
+		intel_de_write(display, DP_MIN_HBLANK_CTL(transcoder),
 			       crtc_state->min_hblank);
 	}
 }
 
-static void intel_set_transcoder_timings_lrr(const struct intel_crtc_state *crtc_state)
+void intel_set_transcoder_timings_lrr(const struct intel_crtc_state *crtc_state,
+				      enum transcoder transcoder)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
-	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
 	u32 crtc_vdisplay, crtc_vtotal, crtc_vblank_start, crtc_vblank_end;
 
-	drm_WARN_ON(display->drm, transcoder_is_dsi(cpu_transcoder));
+	drm_WARN_ON(display->drm, transcoder_is_dsi(transcoder));
 
 	crtc_vdisplay = adjusted_mode->crtc_vdisplay;
 	crtc_vtotal = adjusted_mode->crtc_vtotal;
@@ -2796,7 +2800,7 @@ static void intel_set_transcoder_timings_lrr(const struct intel_crtc_state *crtc
 
 	if (DISPLAY_VER(display) >= 13) {
 		intel_de_write(display,
-			       TRANS_SET_CONTEXT_LATENCY(display, cpu_transcoder),
+			       TRANS_SET_CONTEXT_LATENCY(display, transcoder),
 			       crtc_state->set_context_latency);
 
 		/*
@@ -2813,28 +2817,31 @@ static void intel_set_transcoder_timings_lrr(const struct intel_crtc_state *crtc
 	 * The hardware actually ignores TRANS_VBLANK.VBLANK_END in DP mode.
 	 * But let's write it anyway to keep the state checker happy.
 	 */
-	intel_de_write(display, TRANS_VBLANK(display, cpu_transcoder),
+	intel_de_write(display, TRANS_VBLANK(display, transcoder),
 		       VBLANK_START(crtc_vblank_start - 1) |
 		       VBLANK_END(crtc_vblank_end - 1));
+
 	/*
-	 * For platforms that always use VRR Timing Generator, the VTOTAL.Vtotal
-	 * bits are not required. Since the support for these bits is going to
-	 * be deprecated in upcoming platforms, avoid writing these bits for the
-	 * platforms that do not use legacy Timing Generator.
+	 * DP doesn't have vertical sync, so TRANS_VSYNC only affects
+	 * the position of the vsync interrupt (and does so even when
+	 * using the VRR timing generator!). Thus updating TRANS_VSYNC
+	 * here seems fine even if it isn't double buffered.
 	 */
-	if (intel_vrr_always_use_vrr_tg(display))
-		crtc_vtotal = 1;
+	intel_de_write(display, TRANS_VSYNC(display, transcoder),
+		       VSYNC_START(adjusted_mode->crtc_vsync_start - 1) |
+		       VSYNC_END(adjusted_mode->crtc_vsync_end - 1));
 
 	/*
 	 * The double buffer latch point for TRANS_VTOTAL
 	 * is the transcoder's undelayed vblank.
 	 */
-	intel_de_write(display, TRANS_VTOTAL(display, cpu_transcoder),
+	if (display->platform.novalake &&
+	    IS_DISPLAY_STEP(display, STEP_A0, STEP_C0))
+		crtc_vtotal = 1;
+
+	intel_de_write(display, TRANS_VTOTAL(display, transcoder),
 		       VACTIVE(crtc_vdisplay - 1) |
 		       VTOTAL(crtc_vtotal - 1));
-
-	intel_vrr_set_fixed_rr_timings(crtc_state);
-	intel_vrr_transcoder_enable(crtc_state);
 }
 
 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state)
@@ -3760,9 +3767,9 @@ static void enabled_joiner_pipes(struct intel_display *display,
 	}
 }
 
-static u8 hsw_panel_transcoders(struct intel_display *display)
+static u16 hsw_panel_transcoders(struct intel_display *display)
 {
-	u8 panel_transcoder_mask = BIT(TRANSCODER_EDP);
+	u16 panel_transcoder_mask = BIT(TRANSCODER_EDP);
 
 	if (DISPLAY_VER(display) >= 11)
 		panel_transcoder_mask |= BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1);
@@ -3770,13 +3777,13 @@ static u8 hsw_panel_transcoders(struct intel_display *display)
 	return panel_transcoder_mask;
 }
 
-static u8 hsw_enabled_transcoders(struct intel_crtc *crtc)
+static u16 hsw_enabled_transcoders(struct intel_crtc *crtc)
 {
 	struct intel_display *display = to_intel_display(crtc);
-	u8 panel_transcoder_mask = hsw_panel_transcoders(display);
+	u16 panel_transcoder_mask = hsw_panel_transcoders(display);
 	enum transcoder cpu_transcoder;
 	u8 primary_pipe, secondary_pipes;
-	u8 enabled_transcoders = 0;
+	u16 enabled_transcoders = 0;
 
 	/*
 	 * XXX: Do intel_display_power_get_if_enabled before reading this (for
@@ -3837,18 +3844,18 @@ static u8 hsw_enabled_transcoders(struct intel_crtc *crtc)
 	return enabled_transcoders;
 }
 
-static bool has_edp_transcoders(u8 enabled_transcoders)
+static bool has_edp_transcoders(u16 enabled_transcoders)
 {
 	return enabled_transcoders & BIT(TRANSCODER_EDP);
 }
 
-static bool has_dsi_transcoders(u8 enabled_transcoders)
+static bool has_dsi_transcoders(u16 enabled_transcoders)
 {
 	return enabled_transcoders & (BIT(TRANSCODER_DSI_0) |
 				      BIT(TRANSCODER_DSI_1));
 }
 
-static bool has_pipe_transcoders(u8 enabled_transcoders)
+static bool has_pipe_transcoders(u16 enabled_transcoders)
 {
 	return enabled_transcoders & ~(BIT(TRANSCODER_EDP) |
 				       BIT(TRANSCODER_DSI_0) |
@@ -3856,7 +3863,7 @@ static bool has_pipe_transcoders(u8 enabled_transcoders)
 }
 
 static void assert_enabled_transcoders(struct intel_display *display,
-				       u8 enabled_transcoders)
+				       u16 enabled_transcoders)
 {
 	/* Only one type of transcoder please */
 	drm_WARN_ON(display->drm,
@@ -4783,7 +4790,7 @@ intel_modeset_pipe_config(struct intel_atomic_state *state,
 		if (connector_state->crtc != &crtc->base)
 			continue;
 
-		ret = encoder->compute_config(encoder, crtc_state,
+		ret = encoder->compute_config(state, encoder, crtc_state,
 					      connector_state);
 		if (ret == -EDEADLK)
 			return ret;
@@ -4843,7 +4850,7 @@ intel_modeset_pipe_config_late(struct intel_atomic_state *state,
 		    !encoder->compute_config_late)
 			continue;
 
-		ret = encoder->compute_config_late(encoder, crtc_state,
+		ret = encoder->compute_config_late(state, encoder, crtc_state,
 						   conn_state);
 		if (ret)
 			return ret;
@@ -5189,9 +5196,9 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 	PIPE_CONF_CHECK_I(name.crtc_vdisplay); \
 	if (!fastset || !allow_vblank_delay_fastset(current_config)) \
 		PIPE_CONF_CHECK_I(name.crtc_vblank_start); \
-	PIPE_CONF_CHECK_I(name.crtc_vsync_start); \
-	PIPE_CONF_CHECK_I(name.crtc_vsync_end); \
 	if (!fastset || !pipe_config->update_lrr) { \
+		PIPE_CONF_CHECK_I(name.crtc_vsync_start); \
+		PIPE_CONF_CHECK_I(name.crtc_vsync_end); \
 		PIPE_CONF_CHECK_I(name.crtc_vtotal); \
 		PIPE_CONF_CHECK_I(name.crtc_vblank_end); \
 	} \
@@ -5658,35 +5665,43 @@ int intel_modeset_all_pipes_late(struct intel_atomic_state *state,
 	return 0;
 }
 
+int intel_modeset_commit_pipes_for_atomic_state(struct intel_atomic_state *state,
+						u8 pipe_mask,
+						struct drm_modeset_acquire_ctx *ctx)
+{
+	struct intel_display *display = to_intel_display(state);
+	struct intel_crtc *crtc;
+
+	state->base.acquire_ctx = ctx;
+	state->internal = true;
+
+	for_each_intel_crtc_in_pipe_mask(display, crtc, pipe_mask) {
+		struct intel_crtc_state *crtc_state =
+			intel_atomic_get_crtc_state(&state->base, crtc);
+
+		if (IS_ERR(crtc_state))
+			return PTR_ERR(crtc_state);
+
+		crtc_state->uapi.connectors_changed = true;
+	}
+
+	return drm_atomic_commit(&state->base);
+}
+
 int intel_modeset_commit_pipes(struct intel_display *display,
 			       u8 pipe_mask,
 			       struct drm_modeset_acquire_ctx *ctx)
 {
 	struct drm_atomic_commit *state;
-	struct intel_crtc *crtc;
 	int ret;
 
 	state = drm_atomic_commit_alloc(display->drm);
 	if (!state)
 		return -ENOMEM;
 
-	state->acquire_ctx = ctx;
-	to_intel_atomic_state(state)->internal = true;
-
-	for_each_intel_crtc_in_pipe_mask(display, crtc, pipe_mask) {
-		struct intel_crtc_state *crtc_state =
-			intel_atomic_get_crtc_state(state, crtc);
-
-		if (IS_ERR(crtc_state)) {
-			ret = PTR_ERR(crtc_state);
-			goto out;
-		}
-
-		crtc_state->uapi.connectors_changed = true;
-	}
+	ret = intel_modeset_commit_pipes_for_atomic_state(to_intel_atomic_state(state),
+							  pipe_mask, ctx);
 
-	ret = drm_atomic_commit(state);
-out:
 	drm_atomic_commit_put(state);
 
 	return ret;
@@ -5805,6 +5820,8 @@ static bool lrr_params_changed(const struct intel_crtc_state *old_crtc_state,
 
 	return old_adjusted_mode->crtc_vblank_start != new_adjusted_mode->crtc_vblank_start ||
 		old_adjusted_mode->crtc_vblank_end != new_adjusted_mode->crtc_vblank_end ||
+		old_adjusted_mode->crtc_vsync_start != new_adjusted_mode->crtc_vsync_start ||
+		old_adjusted_mode->crtc_vsync_end != new_adjusted_mode->crtc_vsync_end ||
 		old_adjusted_mode->crtc_vtotal != new_adjusted_mode->crtc_vtotal ||
 		old_crtc_state->set_context_latency != new_crtc_state->set_context_latency;
 }
@@ -5863,7 +5880,7 @@ static int intel_atomic_check_crtcs(struct intel_atomic_state *state)
 }
 
 static bool intel_cpu_transcoders_need_modeset(struct intel_atomic_state *state,
-					       u8 transcoders)
+					       u16 transcoders)
 {
 	const struct intel_crtc_state *new_crtc_state;
 	struct intel_crtc *crtc;
@@ -6499,7 +6516,7 @@ int intel_atomic_check(struct drm_device *dev,
 		}
 
 		if (is_trans_port_sync_mode(new_crtc_state)) {
-			u8 trans = new_crtc_state->sync_mode_slaves_mask;
+			u16 trans = new_crtc_state->sync_mode_slaves_mask;
 
 			if (new_crtc_state->master_transcoder != INVALID_TRANSCODER)
 				trans |= BIT(new_crtc_state->master_transcoder);
@@ -6667,12 +6684,19 @@ static void intel_pipe_fastset(const struct intel_crtc_state *old_crtc_state,
 	    display->platform.broadwell || display->platform.haswell)
 		hsw_set_linetime_wm(new_crtc_state);
 
-	if (new_crtc_state->update_m_n)
+	if (new_crtc_state->update_m_n) {
 		intel_cpu_transcoder_set_m1_n1(crtc, new_crtc_state->cpu_transcoder,
 					       &new_crtc_state->dp_m_n);
+		intel_cmtg_set_m_n(new_crtc_state);
+	}
 
-	if (new_crtc_state->update_lrr)
-		intel_set_transcoder_timings_lrr(new_crtc_state);
+	if (new_crtc_state->update_lrr) {
+		intel_set_transcoder_timings_lrr(new_crtc_state, new_crtc_state->cpu_transcoder);
+		intel_cmtg_set_timings(new_crtc_state, LRR);
+		intel_vrr_set_fixed_rr_timings(new_crtc_state, new_crtc_state->cpu_transcoder);
+		intel_cmtg_set_vrr_timings(new_crtc_state);
+		intel_vrr_transcoder_enable(new_crtc_state);
+	}
 }
 
 static void commit_pipe_pre_planes(struct intel_atomic_state *state,
@@ -6870,6 +6894,12 @@ static void intel_update_crtc(struct intel_atomic_state *state,
 	if (intel_crtc_needs_fastset(new_crtc_state) &&
 	    old_crtc_state->inherited)
 		intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
+
+	if (crtc->cmtg.enabled && intel_crtc_vrr_enabling(state, crtc) &&
+	    intel_cmtg_is_allowed(new_crtc_state)) {
+		intel_cmtg_set_clk_select(new_crtc_state);
+		intel_cmtg_disable(new_crtc_state);
+	}
 }
 
 static void intel_old_crtc_state_disables(struct intel_atomic_state *state,
@@ -7428,6 +7458,7 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state)
 	struct intel_crtc *crtc;
 	struct intel_power_domain_mask put_domains[I915_MAX_PIPES] = {};
 	struct ref_tracker *wakeref = NULL;
+	int power_async_delay;
 
 	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state)
 		intel_atomic_dsb_prepare(state, crtc);
@@ -7536,9 +7567,14 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state)
 	/* Now enable the clocks, plane, pipe, and connectors that we set up. */
 	display->modeset.funcs->commit_modeset_enables(state);
 
+	intel_display_power_dc3co_compute(state);
+
 	/* FIXME probably need to sequence this properly */
 	intel_program_dpkgc_latency(state);
 
+	if (intel_display_power_dc3co_allowed(display))
+		intel_cmtg_program(state);
+
 	intel_wait_for_vblank_workers(state);
 
 	/* FIXME: We should call drm_atomic_helper_commit_hw_done() here
@@ -7632,11 +7668,12 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state)
 		 */
 		intel_uncore_arm_unclaimed_mmio_detection(uncore);
 	}
-	/*
-	 * Delay re-enabling DC states by 17 ms to avoid the off->on->off
-	 * toggling overhead at and above 60 FPS.
-	 */
-	intel_display_power_put_async_delay(display, POWER_DOMAIN_DC_OFF, wakeref, 17);
+
+	power_async_delay = intel_display_power_select_target_dc_state(state);
+
+	intel_display_power_put_async_delay(display,
+					    POWER_DOMAIN_DC_OFF, wakeref, power_async_delay);
+
 	intel_display_rpm_put(display, state->wakeref);
 
 	/*
diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h
index 1963dbc802217..57ea4f2edf2a5 100644
--- a/drivers/gpu/drm/i915/display/intel_display.h
+++ b/drivers/gpu/drm/i915/display/intel_display.h
@@ -424,6 +424,10 @@ void intel_set_m_n(struct intel_display *display,
 		   const struct intel_link_m_n *m_n,
 		   intel_reg_t data_m_reg, intel_reg_t data_n_reg,
 		   intel_reg_t link_m_reg, intel_reg_t link_n_reg);
+void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_state,
+				  enum transcoder transcoder);
+void intel_set_transcoder_timings_lrr(const struct intel_crtc_state *crtc_state,
+				      enum transcoder transcoder);
 void intel_get_m_n(struct intel_display *display,
 		   struct intel_link_m_n *m_n,
 		   intel_reg_t data_m_reg, intel_reg_t data_n_reg,
@@ -474,6 +478,9 @@ int intel_modeset_pipes_in_mask_early(struct intel_atomic_state *state,
 				      const char *reason, u8 pipe_mask);
 int intel_modeset_all_pipes_late(struct intel_atomic_state *state,
 				 const char *reason);
+int intel_modeset_commit_pipes_for_atomic_state(struct intel_atomic_state *state,
+						u8 pipe_mask,
+						struct drm_modeset_acquire_ctx *ctx);
 int intel_modeset_commit_pipes(struct intel_display *display,
 			       u8 pipe_mask,
 			       struct drm_modeset_acquire_ctx *ctx);
diff --git a/drivers/gpu/drm/i915/display/intel_display_core.h b/drivers/gpu/drm/i915/display/intel_display_core.h
index 09ce25a6d4b11..7e988b7b1fe7a 100644
--- a/drivers/gpu/drm/i915/display/intel_display_core.h
+++ b/drivers/gpu/drm/i915/display/intel_display_core.h
@@ -97,6 +97,8 @@ struct intel_wm_funcs {
 struct intel_audio_state {
 	struct intel_encoder *encoder;
 	u8 eld[MAX_ELD_BYTES];
+	/* MST, or SST on UHBR link */
+	bool needs_cpu_transcoder_id;
 };
 
 struct intel_audio {
@@ -325,14 +327,14 @@ struct intel_display {
 		struct intel_bw_info {
 			/* for each QGV point */
 			unsigned int deratedbw[I915_NUM_QGV_POINTS];
-			/* for each PSF GV point */
-			unsigned int psf_bw[I915_NUM_PSF_GV_POINTS];
-			/* Peak BW for each QGV point */
-			unsigned int peakbw[I915_NUM_QGV_POINTS];
-			u8 num_qgv_points;
-			u8 num_psf_gv_points;
 			u8 num_planes;
 		} max[6];
+		/* for each PSF GV point */
+		unsigned int psf_bw[I915_NUM_PSF_GV_POINTS];
+		/* Peak BW for each QGV point */
+		unsigned int peakbw[I915_NUM_QGV_POINTS];
+		u8 num_qgv_points;
+		u8 num_psf_gv_points;
 	} bw;
 
 	struct {
@@ -538,6 +540,8 @@ struct intel_display {
 
 	struct {
 		struct i915_power_domains domains;
+		/* DC3CO state */
+		struct intel_dc3co_state dc3co;
 
 		/* Shadow for DISPLAY_PHY_CONTROL which can't be safely read */
 		u32 chv_phy_control;
diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
index 08004c1ba03f4..3f02868ef105b 100644
--- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c
+++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
@@ -32,6 +32,7 @@
 #include "intel_display_types.h"
 #include "intel_dmc.h"
 #include "intel_dp.h"
+#include "intel_dp_link_caps.h"
 #include "intel_dp_link_training.h"
 #include "intel_dp_mst.h"
 #include "intel_dp_test.h"
@@ -1342,6 +1343,7 @@ void intel_connector_debugfs_add(struct intel_connector *connector)
 	intel_psr_connector_debugfs_add(connector);
 	intel_alpm_lobf_debugfs_add(connector);
 	intel_dp_link_training_debugfs_add(connector);
+	intel_dp_link_caps_debugfs_add(connector);
 	intel_link_bw_connector_debugfs_add(connector);
 
 	if (DISPLAY_VER(display) >= 11 &&
diff --git a/drivers/gpu/drm/i915/display/intel_display_device.c b/drivers/gpu/drm/i915/display/intel_display_device.c
index 69a9f782935cf..f17fc2c684724 100644
--- a/drivers/gpu/drm/i915/display/intel_display_device.c
+++ b/drivers/gpu/drm/i915/display/intel_display_device.c
@@ -101,6 +101,8 @@ static const struct intel_display_device_info no_display = {};
 #define TRANSCODER_EDP_OFFSET 0x6f000
 #define TRANSCODER_DSI0_OFFSET	0x6b000
 #define TRANSCODER_DSI1_OFFSET	0x6b800
+#define TRANSCODER_CMTG0_OFFSET 0x6F000
+#define TRANSCODER_CMTG1_OFFSET 0x6F100
 
 #define CURSOR_A_OFFSET 0x70080
 #define CURSOR_B_OFFSET 0x700c0
@@ -1352,6 +1354,18 @@ static const struct intel_display_device_info xe2_lpd_display = {
 		BIT(INTEL_FBC_A) | BIT(INTEL_FBC_B) |
 		BIT(INTEL_FBC_C) | BIT(INTEL_FBC_D),
 	.__runtime_defaults.has_dbuf_overlap_detection = true,
+	.trans_offsets = {
+		[TRANSCODER_A] = TRANSCODER_A_OFFSET,
+		[TRANSCODER_B] = TRANSCODER_B_OFFSET,
+		[TRANSCODER_C] = TRANSCODER_C_OFFSET,
+		[TRANSCODER_D] = TRANSCODER_D_OFFSET,
+		[TRANSCODER_CMTG0] = TRANSCODER_CMTG0_OFFSET,
+		[TRANSCODER_CMTG1] = TRANSCODER_CMTG1_OFFSET,
+	},
+	.__runtime_defaults.cpu_transcoder_mask =
+		BIT(TRANSCODER_A) | BIT(TRANSCODER_B) |
+		BIT(TRANSCODER_C) | BIT(TRANSCODER_D) |
+		BIT(TRANSCODER_CMTG0) | BIT(TRANSCODER_CMTG1),
 };
 
 static const struct intel_display_device_info wcl_display = {
diff --git a/drivers/gpu/drm/i915/display/intel_display_device.h b/drivers/gpu/drm/i915/display/intel_display_device.h
index 12e5a522a2994..9aeba329b9bde 100644
--- a/drivers/gpu/drm/i915/display/intel_display_device.h
+++ b/drivers/gpu/drm/i915/display/intel_display_device.h
@@ -159,6 +159,7 @@ struct intel_display_platforms {
 #define HAS_CUR_FBC(__display)		(!HAS_GMCH(__display) && IS_DISPLAY_VER(__display, 7, 13))
 #define HAS_D12_PLANE_MINIMIZATION(__display)	((__display)->platform.rocketlake || (__display)->platform.alderlake_s)
 #define HAS_DBUF_OVERLAP_DETECTION(__display)	(DISPLAY_RUNTIME_INFO(__display)->has_dbuf_overlap_detection)
+#define HAS_DC3CO(__display)		(DISPLAY_VER(__display) >= 35)
 #define HAS_DDI(__display)		(DISPLAY_INFO(__display)->has_ddi)
 #define HAS_DISPLAY(__display)		(DISPLAY_RUNTIME_INFO(__display)->pipe_mask != 0)
 #define HAS_DMC(__display)		(DISPLAY_RUNTIME_INFO(__display)->has_dmc)
@@ -191,8 +192,10 @@ struct intel_display_platforms {
 #define HAS_MBUS_JOINING(__display)	((__display)->platform.alderlake_p || DISPLAY_VER(__display) >= 14)
 #define HAS_MSO(__display)		(DISPLAY_VER(__display) >= 12)
 #define HAS_OVERLAY(__display)		(DISPLAY_INFO(__display)->has_overlay)
+#define HAS_PEAK_BW_THRESHOLD(__display)	(DISPLAY_VER(__display) >= 30)
 #define HAS_PIPEDMC(__display)		(DISPLAY_VER(__display) >= 12)
 #define HAS_PIXEL_NORMALIZER(__display)	(DISPLAY_VER(__display) >= 35)
+#define HAS_PMDEMAND(__display)		(DISPLAY_VER(__display) >= 14)
 #define HAS_PSR(__display)		(DISPLAY_INFO(__display)->has_psr)
 #define HAS_PSR_HW_TRACKING(__display)	(DISPLAY_INFO(__display)->has_psr_hw_tracking)
 #define HAS_PSR2_SEL_FETCH(__display)	(DISPLAY_VER(__display) >= 12)
@@ -292,7 +295,7 @@ struct intel_display_runtime_info {
 	u32 rawclk_freq;
 
 	u8 pipe_mask;
-	u8 cpu_transcoder_mask;
+	u16 cpu_transcoder_mask;
 	u16 port_mask;
 
 	u8 num_sprites[I915_MAX_PIPES];
diff --git a/drivers/gpu/drm/i915/display/intel_display_driver.c b/drivers/gpu/drm/i915/display/intel_display_driver.c
index d0729936f6816..70d112fd4287a 100644
--- a/drivers/gpu/drm/i915/display/intel_display_driver.c
+++ b/drivers/gpu/drm/i915/display/intel_display_driver.c
@@ -43,6 +43,8 @@
 #include "intel_dp_tunnel.h"
 #include "intel_dpll.h"
 #include "intel_dpll_mgr.h"
+#include "intel_dram.h"
+#include "intel_encoder.h"
 #include "intel_fb.h"
 #include "intel_fbc.h"
 #include "intel_fbdev.h"
@@ -65,6 +67,8 @@
 #include "intel_wm.h"
 #include "skl_watermark.h"
 
+static int __intel_display_driver_pm_suspend(struct intel_display *display, bool shutdown);
+
 bool intel_display_driver_probe_defer(struct pci_dev *pdev)
 {
 	struct drm_privacy_screen *privacy_screen;
@@ -200,11 +204,23 @@ int intel_display_driver_probe_noirq(struct intel_display *display)
 {
 	int ret;
 
+	intel_opregion_setup(display);
+
+	/*
+	 * Fill the dram structure to get the system dram info. This will be
+	 * used for memory latency calculation.
+	 */
+	ret = intel_dram_detect(display);
+	if (ret)
+		goto cleanup_opregion;
+
+	intel_bw_init_hw(display);
+
 	if (HAS_DISPLAY(display)) {
 		ret = drm_vblank_init(display->drm,
 				      INTEL_NUM_PIPES(display));
 		if (ret)
-			return ret;
+			goto cleanup_opregion;
 	}
 
 	intel_bios_init(display);
@@ -303,6 +319,8 @@ int intel_display_driver_probe_noirq(struct intel_display *display)
 	intel_display_power_driver_remove(display);
 cleanup_bios:
 	intel_bios_driver_remove(display);
+cleanup_opregion:
+	intel_opregion_cleanup(display);
 
 	return ret;
 }
@@ -604,6 +622,8 @@ void intel_display_driver_remove_noirq(struct intel_display *display)
 	if (!HAS_DISPLAY(display))
 		return;
 
+	intel_hpd_cancel_work(display);
+
 	intel_display_driver_suspend_access(display);
 
 	/*
@@ -644,6 +664,8 @@ void intel_display_driver_remove_nogem(struct intel_display *display)
 	intel_display_power_driver_remove(display);
 
 	intel_bios_driver_remove(display);
+
+	intel_opregion_cleanup(display);
 }
 
 void intel_display_driver_unregister(struct intel_display *display)
@@ -674,34 +696,102 @@ void intel_display_driver_unregister(struct intel_display *display)
 	intel_vga_unregister(display);
 }
 
+void intel_display_driver_shutdown(struct intel_display *display)
+{
+	if (!HAS_DISPLAY(display))
+		return;
+
+	__intel_display_driver_pm_suspend(display, true);
+
+	intel_encoder_shutdown_all(display);
+}
+
+void intel_display_driver_shutdown_late(struct intel_display *display)
+{
+	if (!HAS_DISPLAY(display))
+		return;
+
+	/*
+	 * The only requirement is to reboot with display DC states disabled,
+	 * for now leaving all display power wells in the INIT power domain
+	 * enabled.
+	 */
+	intel_display_power_driver_remove(display);
+}
+
 /*
  * turn all crtc's off, but do not adjust state
  * This has to be paired with a call to intel_modeset_setup_hw_state.
  */
-int intel_display_driver_suspend(struct intel_display *display)
+static int __intel_display_driver_pm_suspend(struct intel_display *display, bool shutdown)
 {
-	struct drm_atomic_commit *state;
-	int ret;
+	int ret = 0;
 
 	if (!HAS_DISPLAY(display))
 		return 0;
 
-	state = drm_atomic_helper_suspend(display->drm);
-	ret = PTR_ERR_OR_ZERO(state);
-	if (ret)
-		drm_err(display->drm, "Suspending crtc's failed with %i\n",
-			ret);
-	else
-		display->restore.modeset_state = state;
+	/*
+	 * We do a lot of poking in a lot of registers, make sure they work
+	 * properly.
+	 */
+	intel_display_power_disable(display);
+
+	drm_client_dev_suspend(display->drm);
+
+	drm_kms_helper_poll_disable(display->drm);
+	intel_display_driver_disable_user_access(display);
+
+	if (shutdown) {
+		drm_atomic_helper_shutdown(display->drm);
+	} else {
+		struct drm_atomic_commit *state;
+
+		state = drm_atomic_helper_suspend(display->drm);
+		ret = PTR_ERR_OR_ZERO(state);
+		if (ret)
+			drm_err(display->drm, "Suspending crtc's failed with %i\n",
+				ret);
+		else
+			display->restore.modeset_state = state;
+	}
 
 	/* ensure all DPT VMAs have been unpinned for intel_dpt_suspend() */
 	flush_workqueue(display->wq.cleanup);
 
 	intel_dp_mst_suspend(display);
 
+	intel_encoder_block_all_hpds(display);
+
+	intel_hpd_cancel_work(display);
+
+	intel_display_driver_suspend_access(display);
+
+	intel_encoder_suspend_all(display);
+
 	return ret;
 }
 
+int intel_display_driver_pm_suspend(struct intel_display *display)
+{
+	return __intel_display_driver_pm_suspend(display, false);
+}
+
+void intel_display_driver_pm_suspend_late(struct intel_display *display, bool s2idle)
+{
+	if (!HAS_DISPLAY(display))
+		return;
+
+	intel_display_power_suspend_late(display, s2idle);
+}
+
+void intel_display_driver_pm_resume_early(struct intel_display *display)
+{
+	if (!HAS_DISPLAY(display))
+		return;
+
+	intel_display_power_resume_early(display);
+}
+
 int
 __intel_display_driver_resume(struct intel_display *display,
 			      struct drm_atomic_commit *state,
@@ -741,7 +831,7 @@ __intel_display_driver_resume(struct intel_display *display,
 	return ret;
 }
 
-void intel_display_driver_resume(struct intel_display *display)
+void intel_display_driver_pm_resume(struct intel_display *display)
 {
 	struct drm_atomic_commit *state = display->restore.modeset_state;
 	struct drm_modeset_acquire_ctx ctx;
@@ -750,6 +840,12 @@ void intel_display_driver_resume(struct intel_display *display)
 	if (!HAS_DISPLAY(display))
 		return;
 
+	intel_display_driver_resume_access(display);
+
+	intel_hpd_init(display);
+
+	intel_encoder_unblock_all_hpds(display);
+
 	/* MST sideband requires HPD interrupts enabled */
 	intel_dp_mst_resume(display);
 
@@ -779,4 +875,86 @@ void intel_display_driver_resume(struct intel_display *display)
 			"Restoring old state failed with %i\n", ret);
 	if (state)
 		drm_atomic_commit_put(state);
+
+	intel_display_driver_enable_user_access(display);
+	drm_kms_helper_poll_enable(display->drm);
+
+	intel_hpd_poll_disable(display);
+
+	intel_opregion_resume(display);
+
+	drm_client_dev_resume(display->drm);
+
+	intel_display_power_enable(display);
+}
+
+void intel_display_driver_runtime_pm_enable(struct intel_display *display)
+{
+	intel_display_power_enable(display);
+}
+
+void intel_display_driver_runtime_pm_disable(struct intel_display *display)
+{
+	intel_display_power_disable(display);
+}
+
+/* before irq suspend */
+void intel_display_driver_pm_runtime_suspend(struct intel_display *display)
+{
+}
+
+/* after irq suspend */
+void intel_display_driver_pm_runtime_suspend_late(struct intel_display *display)
+{
+	intel_display_power_runtime_suspend(display);
+
+	/*
+	 * FIXME: We really should find a document that references the arguments
+	 * used below!
+	 */
+	if (display->platform.broadwell) {
+		/*
+		 * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop
+		 * being detected, and the call we do at i915_pm_runtime_resume()
+		 * won't be able to restore them. Since PCI_D3hot matches the
+		 * actual specification and appears to be working, use it.
+		 */
+		intel_opregion_notify_adapter(display, PCI_D3hot);
+	} else {
+		/*
+		 * current versions of firmware which depend on this opregion
+		 * notification have repurposed the D1 definition to mean
+		 * "runtime suspended" vs. what you would normally expect (D3)
+		 * to distinguish it from notifications that might be sent via
+		 * the suspend path.
+		 */
+		intel_opregion_notify_adapter(display, PCI_D1);
+	}
+
+	if (!display->platform.valleyview && !display->platform.cherryview)
+		intel_hpd_poll_enable(display);
+}
+
+/* before irq resume */
+void intel_display_driver_pm_runtime_resume_early(struct intel_display *display)
+{
+	intel_opregion_notify_adapter(display, PCI_D0);
+
+	intel_display_power_runtime_resume(display);
+}
+
+/* after irq resume */
+void intel_display_driver_pm_runtime_resume(struct intel_display *display)
+{
+	/*
+	 * On VLV/CHV display interrupts are part of the display
+	 * power well, so hpd is reinitialized from there. For
+	 * everyone else do it here.
+	 */
+	if (!display->platform.valleyview && !display->platform.cherryview) {
+		intel_hpd_init(display);
+		intel_hpd_poll_disable(display);
+	}
+
+	skl_watermark_ipc_update(display);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_display_driver.h b/drivers/gpu/drm/i915/display/intel_display_driver.h
index 5270c26a32e0a..1ae2ad7e95f6c 100644
--- a/drivers/gpu/drm/i915/display/intel_display_driver.h
+++ b/drivers/gpu/drm/i915/display/intel_display_driver.h
@@ -24,8 +24,13 @@ void intel_display_driver_remove(struct intel_display *display);
 void intel_display_driver_remove_noirq(struct intel_display *display);
 void intel_display_driver_remove_nogem(struct intel_display *display);
 void intel_display_driver_unregister(struct intel_display *display);
-int intel_display_driver_suspend(struct intel_display *display);
-void intel_display_driver_resume(struct intel_display *display);
+void intel_display_driver_shutdown(struct intel_display *display);
+void intel_display_driver_shutdown_late(struct intel_display *display);
+
+int intel_display_driver_pm_suspend(struct intel_display *display);
+void intel_display_driver_pm_suspend_late(struct intel_display *display, bool s2idle);
+void intel_display_driver_pm_resume_early(struct intel_display *display);
+void intel_display_driver_pm_resume(struct intel_display *display);
 
 /* interface for intel_display_reset.c */
 int __intel_display_driver_resume(struct intel_display *display,
@@ -38,5 +43,13 @@ void intel_display_driver_suspend_access(struct intel_display *display);
 void intel_display_driver_resume_access(struct intel_display *display);
 bool intel_display_driver_check_access(struct intel_display *display);
 
+void intel_display_driver_runtime_pm_enable(struct intel_display *display);
+void intel_display_driver_runtime_pm_disable(struct intel_display *display);
+
+void intel_display_driver_pm_runtime_suspend(struct intel_display *display);
+void intel_display_driver_pm_runtime_suspend_late(struct intel_display *display);
+void intel_display_driver_pm_runtime_resume_early(struct intel_display *display);
+void intel_display_driver_pm_runtime_resume(struct intel_display *display);
+
 #endif /* __INTEL_DISPLAY_DRIVER_H__ */
 
diff --git a/drivers/gpu/drm/i915/display/intel_display_irq.c b/drivers/gpu/drm/i915/display/intel_display_irq.c
index 4a821b0674fda..a59b75830bd13 100644
--- a/drivers/gpu/drm/i915/display/intel_display_irq.c
+++ b/drivers/gpu/drm/i915/display/intel_display_irq.c
@@ -1262,7 +1262,7 @@ gen8_de_misc_irq_handler(struct intel_display *display, u32 iir)
 		}
 	}
 
-	if (DISPLAY_VER(display) >= 14) {
+	if (HAS_PMDEMAND(display)) {
 		if (iir & (XELPDP_PMDEMAND_RSP |
 			   XELPDP_PMDEMAND_RSPTOUT_ERR)) {
 			if (iir & XELPDP_PMDEMAND_RSPTOUT_ERR)
@@ -1469,6 +1469,18 @@ static void gen8_de_irq_handler(struct intel_display *display, u32 master_ctl)
 				found = true;
 			}
 
+			if (DISPLAY_VER(display) == 35) {
+				if (iir & CMTG_VBLANK_A) {
+					intel_handle_vblank(display, PIPE_A);
+					found = true;
+				}
+
+				if (iir & CMTG_VBLANK_B) {
+					intel_handle_vblank(display, PIPE_B);
+					found = true;
+				}
+			}
+
 			if (DISPLAY_VER(display) >= 11) {
 				u32 te_trigger = iir & (DSI0_TE | DSI1_TE);
 
@@ -2666,3 +2678,10 @@ void intel_display_irq_snapshot_print(const struct intel_display_irq_snapshot *s
 	drm_printf(p, "DERRMR: 0x%08x\n", snapshot->derrmr);
 	drm_printf(p, "ERR_INT: 0x%08x\n", snapshot->err_int);
 }
+
+void intel_display_irq_port_interrupt_mask(struct intel_display *display, u32 bits, bool mask)
+{
+	spin_lock_irq(&display->irq.lock);
+	bdw_update_port_irq(display, bits, mask ? 0 : bits);
+	spin_unlock_irq(&display->irq.lock);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_display_irq.h b/drivers/gpu/drm/i915/display/intel_display_irq.h
index a1227cee885ae..84446bf53401b 100644
--- a/drivers/gpu/drm/i915/display/intel_display_irq.h
+++ b/drivers/gpu/drm/i915/display/intel_display_irq.h
@@ -82,4 +82,6 @@ void i915gm_irq_cstate_wa(struct intel_display *display, bool enable);
 struct intel_display_irq_snapshot *intel_display_irq_snapshot_capture(struct intel_display *display);
 void intel_display_irq_snapshot_print(const struct intel_display_irq_snapshot *snapshot, struct drm_printer *p);
 
+void intel_display_irq_port_interrupt_mask(struct intel_display *display, u32 bits, bool mask);
+
 #endif /* __INTEL_DISPLAY_IRQ_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_display_limits.h b/drivers/gpu/drm/i915/display/intel_display_limits.h
index 453f7b7208157..ea89473c177f4 100644
--- a/drivers/gpu/drm/i915/display/intel_display_limits.h
+++ b/drivers/gpu/drm/i915/display/intel_display_limits.h
@@ -45,6 +45,8 @@ enum transcoder {
 	TRANSCODER_DSI_1,
 	TRANSCODER_DSI_A = TRANSCODER_DSI_0,	/* legacy DSI */
 	TRANSCODER_DSI_C = TRANSCODER_DSI_1,	/* legacy DSI */
+	TRANSCODER_CMTG0,
+	TRANSCODER_CMTG1,
 
 	I915_MAX_TRANSCODERS
 };
diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c
index 2e51dfcd5dce2..0ebec6e0c2400 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power.c
@@ -14,7 +14,9 @@
 #include "intel_cdclk.h"
 #include "intel_clock_gating.h"
 #include "intel_combo_phy.h"
+#include "intel_crtc.h"
 #include "intel_de.h"
+#include "intel_display.h"
 #include "intel_display_power.h"
 #include "intel_display_power_map.h"
 #include "intel_display_power_well.h"
@@ -30,6 +32,8 @@
 #include "intel_pch_refclk.h"
 #include "intel_pmdemand.h"
 #include "intel_pps_regs.h"
+#include "intel_psr.h"
+#include "intel_psr_regs.h"
 #include "intel_snps_phy.h"
 #include "skl_watermark.h"
 #include "skl_watermark_regs.h"
@@ -267,7 +271,7 @@ sanitize_target_dc_state(struct intel_display *display,
 	static const u32 states[] = {
 		DC_STATE_EN_UPTO_DC6,
 		DC_STATE_EN_UPTO_DC5,
-		DC_STATE_EN_DC3CO,
+		DC_STATE_EN_UPTO_DC3CO,
 		DC_STATE_DISABLE,
 	};
 	int i;
@@ -285,6 +289,19 @@ sanitize_target_dc_state(struct intel_display *display,
 	return target_dc_state;
 }
 
+bool intel_display_power_get_and_reset_dc3co_to_dc6(struct intel_display *display)
+{
+	struct i915_power_domains *power_domains = &display->power.domains;
+	bool ret;
+
+	mutex_lock(&power_domains->lock);
+	ret = power_domains->dc3co_to_dc6;
+	power_domains->dc3co_to_dc6 = false;
+	mutex_unlock(&power_domains->lock);
+
+	return ret;
+}
+
 /**
  * intel_display_power_set_target_dc_state - Set target dc state.
  * @display: display device
@@ -300,6 +317,7 @@ void intel_display_power_set_target_dc_state(struct intel_display *display,
 	struct i915_power_well *power_well;
 	bool dc_off_enabled;
 	struct i915_power_domains *power_domains = &display->power.domains;
+	u32 old_target_dc_state;
 
 	mutex_lock(&power_domains->lock);
 	power_well = lookup_power_well(display, SKL_DISP_DC_OFF);
@@ -320,8 +338,17 @@ void intel_display_power_set_target_dc_state(struct intel_display *display,
 	if (!dc_off_enabled)
 		intel_power_well_enable(display, power_well);
 
+	old_target_dc_state =  power_domains->target_dc_state;
 	power_domains->target_dc_state = state;
 
+	/*
+	 * CMTG must be restored explicitly after DC6 exit. The dc3co_to_dc6
+	 * flag helps CMTG determine whether restoration is required.
+	 */
+	if (old_target_dc_state == DC_STATE_EN_UPTO_DC3CO &&
+	    power_domains->target_dc_state == DC_STATE_EN_UPTO_DC6)
+		power_domains->dc3co_to_dc6 = true;
+
 	if (!dc_off_enabled)
 		intel_power_well_disable(display, power_well);
 
@@ -358,6 +385,144 @@ u32 intel_display_power_get_current_dc_state(struct intel_display *display)
 	return current_dc_state;
 }
 
+bool intel_display_power_dc3co_supported(struct intel_display *display)
+{
+	struct i915_power_domains *power_domains = &display->power.domains;
+
+	if (!HAS_DC3CO(display))
+		return false;
+
+	return (power_domains->allowed_dc_mask & DC_STATE_EN_UPTO_DC3CO) == DC_STATE_EN_UPTO_DC3CO;
+}
+
+bool intel_display_power_dc3co_allowed(struct intel_display *display)
+{
+	struct intel_dc3co_state *dc3co = &display->power.dc3co;
+	bool allowed;
+
+	if (!intel_display_power_dc3co_supported(display))
+		return false;
+
+	mutex_lock(&dc3co->lock);
+	allowed = dc3co->allowed;
+	mutex_unlock(&dc3co->lock);
+
+	return allowed;
+}
+
+void intel_display_power_dc3co_update(struct intel_display *display, u32 trigger)
+{
+	struct intel_dc3co_state *dc3co = &display->power.dc3co;
+
+	if (!intel_display_power_dc3co_supported(display))
+		return;
+
+	mutex_lock(&dc3co->lock);
+	dc3co->trigger = trigger;
+	dc3co->allowed = !!trigger;
+	mutex_unlock(&dc3co->lock);
+}
+
+static bool intel_dc3co_port_pipe_compatible(struct intel_dp *intel_dp,
+					     const struct intel_crtc_state *crtc_state)
+{
+	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+	enum pipe pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe;
+	enum port port = dig_port->base.port;
+	int num_pipes = intel_crtc_num_joined_pipes(crtc_state);
+
+	/* Need to follow 1:1 mapping because of CMTG restriction */
+	if (DISPLAY_VER(to_intel_display(crtc_state)) == 35)
+		return num_pipes == 1 &&
+		       ((pipe == PIPE_A && port == PORT_A) ||
+			(pipe == PIPE_B && port == PORT_B));
+	else
+		return num_pipes == 1 && pipe <= PIPE_B && port <= PORT_B;
+}
+
+void intel_display_power_dc3co_compute(struct intel_atomic_state *state)
+{
+	struct intel_display *display = to_intel_display(state);
+	struct intel_crtc *crtc;
+	struct intel_crtc_state *crtc_state;
+	struct intel_encoder *encoder;
+	struct intel_dp *intel_dp;
+	u8 active_pipes = 0;
+	enum pipe pipe;
+	u32 trigger = DC3CO_TRIGGER_NONE;
+
+	if (!intel_display_power_dc3co_supported(display))
+		return;
+
+	for_each_intel_crtc(display, crtc)
+		active_pipes |= crtc->active ? BIT(crtc->pipe) : 0;
+
+	active_pipes = intel_calc_active_pipes(state, active_pipes);
+
+	if (hweight8(active_pipes) != 1)
+		goto done;
+
+	pipe = ffs(active_pipes) - 1;
+	crtc = intel_crtc_for_pipe(display, pipe);
+
+	crtc_state = to_intel_crtc_state(crtc->base.state);
+
+	for_each_intel_encoder_mask(display->drm, encoder,
+				    crtc_state->uapi.encoder_mask) {
+		if (encoder->type != INTEL_OUTPUT_EDP)
+			goto done;
+
+		intel_dp = enc_to_intel_dp(encoder);
+
+		if (!intel_dc3co_port_pipe_compatible(intel_dp, crtc_state))
+			goto done;
+
+		if (intel_psr2_in_deep_sleep(intel_dp))
+			goto done;
+	}
+
+	if (crtc_state->has_lobf)
+		trigger |= DC3CO_TRIGGER_LOBF;
+	if (crtc_state->has_panel_replay && intel_dp->as_sdp_supported)
+		trigger |= DC3CO_TRIGGER_PANEL_REPLAY;
+	if (crtc_state->has_sel_update)
+		trigger |= DC3CO_TRIGGER_PSR2;
+
+done:
+	intel_display_power_dc3co_update(display, trigger);
+}
+
+/*
+ * Select the target DC state for this commit and return the async-put delay
+ * to use when releasing the DC_OFF reference.
+ *
+ * Picks DC_STATE_EN_UPTO_DC3CO when DC3CO can be enabled
+ * otherwise falls back to default DC state of DC_STATE_EN_UPTO_DC6.
+ * The chosen target is programmed via intel_display_power_set_target_dc_state().
+ *
+ * Returns the async-put delay (in ms) to use when releasing the DC_OFF
+ * reference: DC3CO_PUT_ASYNC_DELAY_MS when DC3CO was selected, otherwise
+ * DC6_PUT_ASYNC_DELAY_MS.
+ */
+int intel_display_power_select_target_dc_state(struct intel_atomic_state *state)
+{
+	struct intel_display *display = to_intel_display(state);
+	u32 target_dc_state;
+
+	if (!intel_display_power_dc3co_supported(display))
+		return DC6_PUT_ASYNC_DELAY_MS;
+
+	if (intel_display_power_dc3co_allowed(display))
+		target_dc_state = DC_STATE_EN_UPTO_DC3CO;
+	else
+		target_dc_state = DC_STATE_EN_UPTO_DC6;
+
+	intel_display_power_set_target_dc_state(display, target_dc_state);
+
+	return target_dc_state == DC_STATE_EN_UPTO_DC3CO ?
+		DC3CO_PUT_ASYNC_DELAY_MS : DC6_PUT_ASYNC_DELAY_MS;
+}
+
 static void __async_put_domains_mask(struct i915_power_domains *power_domains,
 				     struct intel_power_domain_mask *mask)
 {
@@ -956,7 +1121,9 @@ static u32 get_allowed_dc_mask(struct intel_display *display, int enable_dc)
 	if (!HAS_DISPLAY(display))
 		return 0;
 
-	if (DISPLAY_VER(display) >= 20)
+	if (DISPLAY_VER(display) >= 35)
+		max_dc = 4;
+	else if (DISPLAY_VER(display) >= 20)
 		max_dc = 2;
 	else if (display->platform.dg2)
 		max_dc = 1;
@@ -999,10 +1166,10 @@ static u32 get_allowed_dc_mask(struct intel_display *display, int enable_dc)
 
 	switch (requested_dc) {
 	case 4:
-		mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC6;
+		mask |= DC_STATE_EN_UPTO_DC3CO | DC_STATE_EN_UPTO_DC6;
 		break;
 	case 3:
-		mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC5;
+		mask |= DC_STATE_EN_UPTO_DC3CO | DC_STATE_EN_UPTO_DC5;
 		break;
 	case 2:
 		mask |= DC_STATE_EN_UPTO_DC6;
@@ -1037,6 +1204,7 @@ int intel_display_power_init(struct intel_display *display)
 		sanitize_target_dc_state(display, DC_STATE_EN_UPTO_DC6);
 
 	mutex_init(&power_domains->lock);
+	mutex_init(&display->power.dc3co.lock);
 
 	INIT_DELAYED_WORK(&power_domains->async_put_work,
 			  intel_display_power_put_async_work);
@@ -1122,7 +1290,7 @@ static void gen9_dbuf_enable(struct intel_display *display)
 
 	slices_mask = BIT(DBUF_S1) | display->dbuf.enabled_slices;
 
-	if (DISPLAY_VER(display) >= 14)
+	if (HAS_PMDEMAND(display))
 		intel_pmdemand_program_dbuf(display, slices_mask);
 
 	/*
@@ -1136,7 +1304,7 @@ static void gen9_dbuf_disable(struct intel_display *display)
 {
 	gen9_dbuf_slices_update(display, 0);
 
-	if (DISPLAY_VER(display) >= 14)
+	if (HAS_PMDEMAND(display))
 		intel_pmdemand_program_dbuf(display, 0);
 }
 
@@ -1634,9 +1802,12 @@ static void tgl_bw_buddy_init(struct intel_display *display)
 	if (table[config].page_mask == 0) {
 		drm_dbg_kms(display->drm,
 			    "Unknown memory configuration; disabling address buddy logic.\n");
-		for_each_set_bit(i, &abox_mask, BITS_PER_TYPE(abox_mask))
-			intel_de_write(display, BW_BUDDY_CTL(i),
-				       BW_BUDDY_DISABLE);
+
+		if (DISPLAY_VER(display) < 20) {
+			for_each_set_bit(i, &abox_mask, BITS_PER_TYPE(abox_mask))
+				intel_de_write(display, BW_BUDDY_CTL(i),
+					       BW_BUDDY_DISABLE);
+		}
 	} else {
 		for_each_set_bit(i, &abox_mask, BITS_PER_TYPE(abox_mask)) {
 			intel_de_write(display, BW_BUDDY_PAGE_MASK(i),
diff --git a/drivers/gpu/drm/i915/display/intel_display_power.h b/drivers/gpu/drm/i915/display/intel_display_power.h
index 56dc89eed3f89..546af67b680b8 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power.h
+++ b/drivers/gpu/drm/i915/display/intel_display_power.h
@@ -9,9 +9,12 @@
 #include <linux/mutex.h>
 #include <linux/workqueue.h>
 
+#include "intel_display_limits.h"
+
 enum aux_ch;
 enum port;
 struct i915_power_well;
+struct intel_atomic_state;
 struct intel_display;
 struct intel_encoder;
 struct ref_tracker;
@@ -131,6 +134,36 @@ struct intel_power_domain_mask {
 	DECLARE_BITMAP(bits, POWER_DOMAIN_NUM);
 };
 
+/*
+ * DC3CO enabling triggers (bitmask).
+ * DC3CO may be enabled when at least one of these triggers is active.
+ * Additional constraints may still apply.
+ */
+#define DC3CO_TRIGGER_NONE		(0)
+#define DC3CO_TRIGGER_PSR2		BIT(0)
+#define DC3CO_TRIGGER_LOBF		BIT(1)
+#define DC3CO_TRIGGER_PANEL_REPLAY	BIT(2)
+#define DC3CO_TRIGGER_ALL		(DC3CO_TRIGGER_PSR2 | \
+					 DC3CO_TRIGGER_LOBF | \
+					 DC3CO_TRIGGER_PANEL_REPLAY)
+
+/*
+ * Delay to re-enable DC5/DC6 states by 17 ms to avoid the off->on->off
+ * toggling overhead at and above 60 FPS.
+ */
+#define DC6_PUT_ASYNC_DELAY_MS		17
+/*
+ * Use minimal re-enable delay to allow DC3CO entry on
+ * the next idle frame.
+ */
+#define DC3CO_PUT_ASYNC_DELAY_MS	1
+
+struct intel_dc3co_state {
+	struct mutex lock; /* protects allowed and trigger fields */
+	bool allowed; /* DC3CO compute result */
+	u32 trigger; /* Bitmask of active DC3CO triggers */
+};
+
 struct i915_power_domains {
 	/*
 	 * Power wells needed for initialization at driver init and suspend
@@ -138,6 +171,7 @@ struct i915_power_domains {
 	 */
 	bool initializing;
 	bool display_core_suspended;
+	bool dc3co_to_dc6;
 	int power_well_count;
 
 	u32 dc_state;
@@ -179,9 +213,15 @@ void intel_display_power_sanitize_state(struct intel_display *display);
 
 void intel_display_power_suspend_late(struct intel_display *display, bool s2idle);
 void intel_display_power_resume_early(struct intel_display *display);
+bool intel_display_power_get_and_reset_dc3co_to_dc6(struct intel_display *display);
 void intel_display_power_set_target_dc_state(struct intel_display *display,
 					     u32 state);
 u32 intel_display_power_get_current_dc_state(struct intel_display *display);
+bool intel_display_power_dc3co_supported(struct intel_display *display);
+void intel_display_power_dc3co_update(struct intel_display *display, u32 trigger);
+bool intel_display_power_dc3co_allowed(struct intel_display *display);
+void intel_display_power_dc3co_compute(struct intel_atomic_state *state);
+int intel_display_power_select_target_dc_state(struct intel_atomic_state *state);
 
 void intel_display_power_runtime_suspend(struct intel_display *display);
 void intel_display_power_runtime_resume(struct intel_display *display);
diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.c b/drivers/gpu/drm/i915/display/intel_display_power_well.c
index 04bd0dde5beda..02cb4d800e238 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power_well.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power_well.c
@@ -726,12 +726,28 @@ static void assert_can_disable_dc9(struct intel_display *display)
 	  */
 }
 
+static u32 dc_state_ro_mask(struct intel_display *display)
+{
+	if (DISPLAY_VER(display) >= 20)
+		return DC_STATE_EN_CSR_MASK_CMTG_1 | DC_STATE_EN_CSR_MASK_CMTG_0;
+	else if (DISPLAY_VER(display) >= 13 && !display->platform.dg2)
+		return DC_STATE_EN_CSR_MASK_CMTG_0;
+
+	return 0;
+}
+
 static void gen9_write_dc_state(struct intel_display *display,
 				u32 state)
 {
 	int rewrites = 0;
 	int rereads = 0;
 	u32 v;
+	/*
+	 * Mask out RO status bits from read-back comparison.
+	 * HW may set these bits independently, so exclude them
+	 * to prevent the verify loop from retrying due to RO bits mismatch.
+	 */
+	u32 ro_mask = dc_state_ro_mask(display);
 
 	intel_de_write(display, DC_STATE_EN, state);
 
@@ -743,7 +759,7 @@ static void gen9_write_dc_state(struct intel_display *display,
 	do  {
 		v = intel_de_read(display, DC_STATE_EN);
 
-		if (v != state) {
+		if ((v & ~ro_mask) != (state & ~ro_mask)) {
 			intel_de_write(display, DC_STATE_EN, state);
 			rewrites++;
 			rereads = 0;
@@ -753,10 +769,10 @@ static void gen9_write_dc_state(struct intel_display *display,
 
 	} while (rewrites < 100);
 
-	if (v != state)
+	if ((v & ~ro_mask) != (state & ~ro_mask))
 		drm_err(display->drm,
-			"Writing dc state to 0x%x failed, now 0x%x\n",
-			state, v);
+			"Writing dc state to 0x%x failed, now 0x%x (ro_mask=0x%x)\n",
+			state, v, ro_mask);
 
 	/* Most of the times we need one retry, avoid spam */
 	if (rewrites > 1)
@@ -772,7 +788,7 @@ static u32 gen9_dc_mask(struct intel_display *display)
 	mask = DC_STATE_EN_UPTO_DC5;
 
 	if (DISPLAY_VER(display) >= 12)
-		mask |= DC_STATE_EN_DC3CO | DC_STATE_EN_UPTO_DC6
+		mask |= DC_STATE_EN_UPTO_DC3CO | DC_STATE_EN_UPTO_DC6
 					  | DC_STATE_EN_DC9;
 	else if (DISPLAY_VER(display) == 11)
 		mask |= DC_STATE_EN_UPTO_DC6 | DC_STATE_EN_DC9;
@@ -866,21 +882,30 @@ void gen9_set_dc_state(struct intel_display *display, u32 state)
 	power_domains->dc_state = val & mask;
 }
 
-static void tgl_enable_dc3co(struct intel_display *display)
+void xe3lpd_enable_dc_count(struct intel_display *display)
 {
-	drm_dbg_kms(display->drm, "Enabling DC3CO\n");
-	gen9_set_dc_state(display, DC_STATE_EN_DC3CO);
+	if (DISPLAY_VER(display) < 35)
+		return;
+
+	intel_de_write(display, DC_COUNT_EN, DC_COUNT_EN_COUNTER_ENABLE);
 }
 
-static void tgl_disable_dc3co(struct intel_display *display)
+static void assert_can_enable_dc3co(struct intel_display *display)
 {
-	drm_dbg_kms(display->drm, "Disabling DC3CO\n");
-	intel_de_rmw(display, DC_STATE_EN, DC_STATE_DC3CO_STATUS, 0);
-	gen9_set_dc_state(display, DC_STATE_DISABLE);
-	/*
-	 * Delay of 200us DC3CO Exit time B.Spec 49196
-	 */
-	usleep_range(200, 210);
+	drm_WARN_ONCE(display->drm,
+		      (intel_de_read(display, DC_STATE_EN) &
+		       DC_STATE_EN_UPTO_DC3CO),
+		      "DC3CO already programmed to be enabled.\n");
+
+	assert_main_dmc_loaded(display);
+}
+
+static void xe3lpd_enable_dc3co(struct intel_display *display)
+{
+	assert_can_enable_dc3co(display);
+	drm_dbg_kms(display->drm, "Enabling DC3CO\n");
+	intel_dmc_wl_enable(display, DC_STATE_EN_UPTO_DC3CO);
+	gen9_set_dc_state(display, DC_STATE_EN_UPTO_DC3CO);
 }
 
 static void assert_can_enable_dc5(struct intel_display *display)
@@ -1039,8 +1064,8 @@ static void bxt_verify_dpio_phy_power_wells(struct intel_display *display)
 static bool gen9_dc_off_power_well_enabled(struct intel_display *display,
 					   struct i915_power_well *power_well)
 {
-	return ((intel_de_read(display, DC_STATE_EN) & DC_STATE_EN_DC3CO) == 0 &&
-		(intel_de_read(display, DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0);
+	return ((intel_de_read(display, DC_STATE_EN) & DC_STATE_EN_UPTO_DC3CO) == 0 &&
+		(intel_de_read(display, DC_STATE_EN) & DC_STATE_EN_UPTO_DC3CO_DC5_DC6_MASK) == 0);
 }
 
 static void gen9_assert_dbuf_enabled(struct intel_display *display)
@@ -1061,11 +1086,6 @@ void gen9_disable_dc_states(struct intel_display *display)
 	struct intel_cdclk_config cdclk_config = {};
 	u32 old_state = power_domains->dc_state;
 
-	if (power_domains->target_dc_state == DC_STATE_EN_DC3CO) {
-		tgl_disable_dc3co(display);
-		return;
-	}
-
 	if (HAS_DISPLAY(display)) {
 		intel_dmc_wl_get_noreg(display);
 		gen9_set_dc_state(display, DC_STATE_DISABLE);
@@ -1076,9 +1096,13 @@ void gen9_disable_dc_states(struct intel_display *display)
 	}
 
 	if (old_state == DC_STATE_EN_UPTO_DC5 ||
-	    old_state == DC_STATE_EN_UPTO_DC6)
+	    old_state == DC_STATE_EN_UPTO_DC6 ||
+	    old_state == DC_STATE_EN_UPTO_DC3CO)
 		intel_dmc_wl_disable(display);
 
+	if (old_state == DC_STATE_EN_UPTO_DC3CO)
+		return;
+
 	intel_cdclk_get_cdclk(display, &cdclk_config);
 	/* Can't read out voltage_level so can't use intel_cdclk_changed() */
 	drm_WARN_ON(display->drm,
@@ -1114,8 +1138,8 @@ static void gen9_dc_off_power_well_disable(struct intel_display *display,
 		return;
 
 	switch (power_domains->target_dc_state) {
-	case DC_STATE_EN_DC3CO:
-		tgl_enable_dc3co(display);
+	case DC_STATE_EN_UPTO_DC3CO:
+		xe3lpd_enable_dc3co(display);
 		break;
 	case DC_STATE_EN_UPTO_DC6:
 		skl_enable_dc6(display);
diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.h b/drivers/gpu/drm/i915/display/intel_display_power_well.h
index 8f5524da2d061..0ce64b894436b 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power_well.h
+++ b/drivers/gpu/drm/i915/display/intel_display_power_well.h
@@ -159,6 +159,7 @@ void gen9_set_dc_state(struct intel_display *display, u32 state);
 void gen9_disable_dc_states(struct intel_display *display);
 void bxt_enable_dc9(struct intel_display *display);
 void bxt_disable_dc9(struct intel_display *display);
+void xe3lpd_enable_dc_count(struct intel_display *display);
 
 extern const struct i915_power_well_ops i9xx_always_on_power_well_ops;
 extern const struct i915_power_well_ops chv_pipe_power_well_ops;
diff --git a/drivers/gpu/drm/i915/display/intel_display_regs.h b/drivers/gpu/drm/i915/display/intel_display_regs.h
index 4321f8b529dae..0f5018482497b 100644
--- a/drivers/gpu/drm/i915/display/intel_display_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_display_regs.h
@@ -1458,6 +1458,8 @@
 #define  GEN9_AUX_CHANNEL_B		(1 << 25)
 #define  DSI1_TE			(1 << 24)
 #define  DSI0_TE			(1 << 23)
+#define  CMTG_VBLANK_B			(1 << 17)
+#define  CMTG_VBLANK_A			(1 << 14)
 #define  GEN8_DE_PORT_HOTPLUG(hpd_pin)	REG_BIT(3 + _HPD_PIN_DDI(hpd_pin))
 #define  BXT_DE_PORT_HOTPLUG_MASK	(GEN8_DE_PORT_HOTPLUG(HPD_PORT_A) | \
 					 GEN8_DE_PORT_HOTPLUG(HPD_PORT_B) | \
@@ -1745,6 +1747,9 @@
 #define XELPD_CHICKEN_DCPR_3			_MMIO(0x46438)
 #define   DMD_RSP_TIMEOUT_DISABLE		REG_BIT(19)
 
+#define XE3P_CHICKEN_DCPR_4			_MMIO(0x454a0)
+#define   DCPR4_BLOCK_DC3CO_ACTIVE_FRAME	REG_BIT(24)
+
 #define SKL_DFSM			_MMIO(0x51000)
 #define   SKL_DFSM_DISPLAY_PM_DISABLE	(1 << 27)
 #define   SKL_DFSM_DISPLAY_HDCP_DISABLE	(1 << 25)
@@ -2767,28 +2772,28 @@ enum skl_power_gate {
  */
 /* CDCLK_CTL */
 #define CDCLK_CTL			_MMIO(0x46000)
-#define  CDCLK_FREQ_SEL_MASK		REG_GENMASK(27, 26)
+#define  CDCLK_FREQ_SEL_MASK		REG_GENMASK(27, 26)  /* skl */
 #define  CDCLK_FREQ_450_432		REG_FIELD_PREP(CDCLK_FREQ_SEL_MASK, 0)
-#define  CDCLK_FREQ_540		REG_FIELD_PREP(CDCLK_FREQ_SEL_MASK, 1)
+#define  CDCLK_FREQ_540			REG_FIELD_PREP(CDCLK_FREQ_SEL_MASK, 1)
 #define  CDCLK_FREQ_337_308		REG_FIELD_PREP(CDCLK_FREQ_SEL_MASK, 2)
 #define  CDCLK_FREQ_675_617		REG_FIELD_PREP(CDCLK_FREQ_SEL_MASK, 3)
-#define  MDCLK_SOURCE_SEL_MASK		REG_GENMASK(25, 25)
+#define  MDCLK_SOURCE_SEL_MASK		REG_GENMASK(25, 25) /* lnl+ */
 #define  MDCLK_SOURCE_SEL_CD2XCLK	REG_FIELD_PREP(MDCLK_SOURCE_SEL_MASK, 0)
 #define  MDCLK_SOURCE_SEL_CDCLK_PLL	REG_FIELD_PREP(MDCLK_SOURCE_SEL_MASK, 1)
-#define  BXT_CDCLK_CD2X_DIV_SEL_MASK	REG_GENMASK(23, 22)
+#define  BXT_CDCLK_CD2X_DIV_SEL_MASK	REG_GENMASK(23, 22) /* bxt+ */
 #define  BXT_CDCLK_CD2X_DIV_SEL_1	REG_FIELD_PREP(BXT_CDCLK_CD2X_DIV_SEL_MASK, 0)
 #define  BXT_CDCLK_CD2X_DIV_SEL_1_5	REG_FIELD_PREP(BXT_CDCLK_CD2X_DIV_SEL_MASK, 1)
 #define  BXT_CDCLK_CD2X_DIV_SEL_2	REG_FIELD_PREP(BXT_CDCLK_CD2X_DIV_SEL_MASK, 2)
 #define  BXT_CDCLK_CD2X_DIV_SEL_4	REG_FIELD_PREP(BXT_CDCLK_CD2X_DIV_SEL_MASK, 3)
-#define  BXT_CDCLK_CD2X_PIPE(pipe)	((pipe) << 20)
-#define  CDCLK_DIVMUX_CD_OVERRIDE	(1 << 19)
-#define  BXT_CDCLK_CD2X_PIPE_NONE	BXT_CDCLK_CD2X_PIPE(3)
-#define  ICL_CDCLK_CD2X_PIPE(pipe)	(_PICK(pipe, 0, 2, 6) << 19)
-#define  ICL_CDCLK_CD2X_PIPE_NONE	(7 << 19)
-#define  TGL_CDCLK_CD2X_PIPE(pipe)	BXT_CDCLK_CD2X_PIPE(pipe)
-#define  TGL_CDCLK_CD2X_PIPE_NONE	ICL_CDCLK_CD2X_PIPE_NONE
-#define  BXT_CDCLK_SSA_PRECHARGE_ENABLE	(1 << 16)
-#define  CDCLK_FREQ_DECIMAL_MASK	(0x7ff)
+#define  BXT_CDCLK_CD2X_PIPE_MASK	REG_GENMASK(21, 20) /* bxt/glk */
+#define  BXT_CDCLK_CD2X_PIPE(pipe)	REG_FIELD_PREP(BXT_CDCLK_CD2X_PIPE_MASK, (pipe))
+#define  BXT_CDCLK_CD2X_PIPE_NONE	REG_FIELD_PREP(BXT_CDCLK_CD2X_PIPE_MASK, 3)
+#define  ICL_CDCLK_CD2X_PIPE_MASK	REG_GENMASK(21, 19) /* icl+ */
+#define  ICL_CDCLK_CD2X_PIPE(pipe)	REG_FIELD_PREP(ICL_CDCLK_CD2X_PIPE_MASK, (pipe) << 1)
+#define  ICL_CDCLK_CD2X_PIPE_NONE	REG_FIELD_PREP(ICL_CDCLK_CD2X_PIPE_MASK, 7)
+#define  CDCLK_DIVMUX_CD_OVERRIDE	REG_BIT(19) /* pre-icl */
+#define  BXT_CDCLK_SSA_PRECHARGE_ENABLE	REG_BIT(16) /* bxt/glk */
+#define  CDCLK_FREQ_DECIMAL_MASK	REG_GENMASK(10, 0) /* pre-lnl */
 
 /* CDCLK_SQUASH_CTL */
 #define CDCLK_SQUASH_CTL		_MMIO(0x46008)
@@ -3070,19 +3075,29 @@ enum skl_power_gate {
 /* GEN9 DC */
 #define DC_STATE_EN			_MMIO(0x45504)
 #define  DC_STATE_DISABLE		0
-#define  DC_STATE_EN_DC3CO		REG_BIT(30)
 #define  DC_STATE_DC3CO_STATUS		REG_BIT(29)
 #define  HOLD_PHY_CLKREQ_PG1_LATCH	REG_BIT(21)
 #define  HOLD_PHY_PG1_LATCH		REG_BIT(20)
-#define  DC_STATE_EN_UPTO_DC5		(1 << 0)
 #define  DC_STATE_EN_DC9		(1 << 3)
-#define  DC_STATE_EN_UPTO_DC6		(2 << 0)
-#define  DC_STATE_EN_UPTO_DC5_DC6_MASK   0x3
+#define  DC_STATE_EN_UPTO_DC3CO_DC5_DC6_MASK	REG_GENMASK(1, 0)
+#define  DC_STATE_EN_DISABLE		REG_FIELD_PREP(DC_STATE_EN_UPTO_DC3CO_DC5_DC6_MASK, 0)
+#define  DC_STATE_EN_UPTO_DC5		REG_FIELD_PREP(DC_STATE_EN_UPTO_DC3CO_DC5_DC6_MASK, 1)
+#define  DC_STATE_EN_UPTO_DC6		REG_FIELD_PREP(DC_STATE_EN_UPTO_DC3CO_DC5_DC6_MASK, 2)
+#define  DC_STATE_EN_UPTO_DC3CO	REG_FIELD_PREP(DC_STATE_EN_UPTO_DC3CO_DC5_DC6_MASK, 3)
+/* display version 20+ */
+#define  DC_STATE_EN_CSR_MASK_CMTG_1	REG_BIT(11)
+/* display version 13+, except dg2 */
+#define  DC_STATE_EN_CSR_MASK_CMTG_0	REG_BIT(10)
 
 #define  DC_STATE_DEBUG                  _MMIO(0x45520)
 #define  DC_STATE_DEBUG_MASK_CORES	(1 << 0)
 #define  DC_STATE_DEBUG_MASK_MEMORY_UP	(1 << 1)
 
+#define DC_COUNT_EN			_MMIO(0x457B4)
+#define DC_COUNT_EN_COUNTER_ENABLE	REG_BIT(31)
+
+#define DC_STATE_DC3CO_RESIDENCY	_MMIO(0x457B8)
+
 #define D_COMP_BDW			_MMIO(0x138144)
 
 /* Pipe WM_LINETIME - watermark line time */
diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h
index c21e0c0ef0b12..8bd213db5e7a6 100644
--- a/drivers/gpu/drm/i915/display/intel_display_types.h
+++ b/drivers/gpu/drm/i915/display/intel_display_types.h
@@ -58,6 +58,8 @@ struct cec_notifier;
 struct drm_printer;
 struct intel_connector;
 struct intel_ddi_buf_trans;
+struct intel_dp_link_caps;
+struct intel_dp_link_training;
 struct intel_fbc;
 struct intel_global_objs_state;
 struct intel_hdcp_shim;
@@ -150,7 +152,6 @@ struct intel_framebuffer {
 	unsigned int min_alignment;
 	unsigned int vtd_guard;
 
-	unsigned int (*panic_tiling)(unsigned int x, unsigned int y, unsigned int width);
 	struct intel_panic *panic;
 };
 
@@ -177,10 +178,12 @@ struct intel_encoder {
 	enum intel_output_type (*compute_output_type)(struct intel_encoder *,
 						      struct intel_crtc_state *,
 						      struct drm_connector_state *);
-	int (*compute_config)(struct intel_encoder *,
+	int (*compute_config)(struct intel_atomic_state *,
+			      struct intel_encoder *,
 			      struct intel_crtc_state *,
 			      struct drm_connector_state *);
-	int (*compute_config_late)(struct intel_encoder *,
+	int (*compute_config_late)(struct intel_atomic_state *,
+				   struct intel_encoder *,
 				   struct intel_crtc_state *,
 				   struct drm_connector_state *);
 	void (*pre_pll_enable)(struct intel_atomic_state *,
@@ -1185,7 +1188,6 @@ struct intel_crtc_state {
 	bool pkg_c_latency_used;
 	/* Only used for state verification. */
 	enum intel_panel_replay_dsc_support panel_replay_dsc_support;
-	u32 dc3co_exitline;
 	u16 su_y_granularity;
 	u8 active_non_psr_pipes;
 	u8 entry_setup_frames;
@@ -1370,7 +1372,7 @@ struct intel_crtc_state {
 	enum transcoder master_transcoder;
 
 	/* Bitmask to indicate slaves attached */
-	u8 sync_mode_slaves_mask;
+	u16 sync_mode_slaves_mask;
 
 	/* Only valid on TGL+ */
 	enum transcoder mst_master_transcoder;
@@ -1573,6 +1575,10 @@ struct intel_crtc {
 #endif
 
 	bool vblank_psr_notify;
+
+	struct {
+		bool enabled;
+	} cmtg;
 };
 
 struct intel_plane_error {
@@ -1772,14 +1778,16 @@ struct intel_psr {
 	ktime_t last_exit;
 	bool sink_not_reliable;
 	bool irq_aux_error;
+	/* DC3CO allowed used to control PSR configuration */
+	bool dc3co_allowed;
+	/* DC3CO disable work */
+	struct delayed_work dc3co_work;
 	u16 su_w_granularity;
 	u16 su_y_granularity;
 	bool source_panel_replay_support;
 	bool sink_panel_replay_support;
 	bool panel_replay_enabled;
-	u32 dc3co_exitline;
 	u32 dc3co_exit_delay;
-	struct delayed_work dc3co_work;
 	u8 entry_setup_frames;
 
 	u8 io_wake_lines;
@@ -1795,6 +1803,14 @@ struct intel_psr {
 	struct ref_tracker *vblank_wakeref;
 };
 
+struct intel_dp_link_config {
+	int rate;
+	int lane_count;
+};
+
+#define INTEL_DP_LINK_CONFIG_NULL \
+	((struct intel_dp_link_config){})
+
 struct intel_dp {
 	intel_reg_t output_reg;
 	u32 DP;
@@ -1820,29 +1836,9 @@ struct intel_dp {
 	bool use_rate_select;
 	/* Max sink lane count as reported by DP_MAX_LANE_COUNT */
 	int max_sink_lane_count;
-	/* intersection of source and sink rates */
-	int num_common_rates;
-	int common_rates[DP_MAX_SUPPORTED_RATES];
-	int max_common_lane_count;
 	struct {
 		/* TODO: move the rest of link specific fields to here */
 		bool active;
-		/* common rate,lane_count configs in bw order */
-		int num_configs;
-#define INTEL_DP_MAX_LANE_COUNT			4
-#define INTEL_DP_MAX_SUPPORTED_LANE_CONFIGS	(ilog2(INTEL_DP_MAX_LANE_COUNT) + 1)
-#define INTEL_DP_LANE_COUNT_EXP_BITS		order_base_2(INTEL_DP_MAX_SUPPORTED_LANE_CONFIGS)
-#define INTEL_DP_LINK_RATE_IDX_BITS		(BITS_PER_TYPE(u8) - INTEL_DP_LANE_COUNT_EXP_BITS)
-#define INTEL_DP_MAX_LINK_CONFIGS		(DP_MAX_SUPPORTED_RATES * \
-						 INTEL_DP_MAX_SUPPORTED_LANE_CONFIGS)
-		struct intel_dp_link_config {
-			u8 link_rate_idx:INTEL_DP_LINK_RATE_IDX_BITS;
-			u8 lane_count_exp:INTEL_DP_LANE_COUNT_EXP_BITS;
-		} configs[INTEL_DP_MAX_LINK_CONFIGS];
-		/* Max lane count for the current link */
-		int max_lane_count;
-		/* Max rate for the current link */
-		int max_rate;
 		/*
 		 * Link parameters for which the MST topology was probed.
 		 * Tracking these ensures that the MST path resources are
@@ -1851,13 +1847,8 @@ struct intel_dp {
 		 */
 		int mst_probed_lane_count;
 		int mst_probed_rate;
-		int force_lane_count;
-		int force_rate;
-		bool retrain_disabled;
-		/* Sequential link training failures after a passing LT */
-		int seq_train_failures;
-		int force_train_failure;
-		bool force_retrain;
+		struct intel_dp_link_training *training;
+		struct intel_dp_link_caps *caps;
 	} link;
 	bool reset_link_params;
 	int mso_link_count;
diff --git a/drivers/gpu/drm/i915/display/intel_display_wa.c b/drivers/gpu/drm/i915/display/intel_display_wa.c
index 2094eda09c912..b4c49816f7eb4 100644
--- a/drivers/gpu/drm/i915/display/intel_display_wa.c
+++ b/drivers/gpu/drm/i915/display/intel_display_wa.c
@@ -112,6 +112,8 @@ bool __intel_display_wa(struct intel_display *display, enum intel_display_wa wa,
 			DISPLAY_VERx100(display) == 1401;
 	case INTEL_DISPLAY_WA_14025769978:
 		return DISPLAY_VER(display) == 35;
+	case INTEL_DISPLAY_WA_14026643300:
+		return DISPLAY_VER(display) == 35;
 	case INTEL_DISPLAY_WA_15013987218:
 		return DISPLAY_VER(display) == 20;
 	case INTEL_DISPLAY_WA_15018326506:
diff --git a/drivers/gpu/drm/i915/display/intel_display_wa.h b/drivers/gpu/drm/i915/display/intel_display_wa.h
index 9cdd148ea4fad..92b3980bea841 100644
--- a/drivers/gpu/drm/i915/display/intel_display_wa.h
+++ b/drivers/gpu/drm/i915/display/intel_display_wa.h
@@ -43,6 +43,7 @@ enum intel_display_wa {
 	INTEL_DISPLAY_WA_14016740474,
 	INTEL_DISPLAY_WA_14020863754,
 	INTEL_DISPLAY_WA_14025769978,
+	INTEL_DISPLAY_WA_14026643300,
 	INTEL_DISPLAY_WA_15013987218,
 	INTEL_DISPLAY_WA_15018326506,
 	INTEL_DISPLAY_WA_16011181250,
diff --git a/drivers/gpu/drm/i915/display/intel_dmc.c b/drivers/gpu/drm/i915/display/intel_dmc.c
index 481fb65b71102..a191eee240d9f 100644
--- a/drivers/gpu/drm/i915/display/intel_dmc.c
+++ b/drivers/gpu/drm/i915/display/intel_dmc.c
@@ -505,14 +505,16 @@ static void pipedmc_clock_gating_wa(struct intel_display *display, bool enable)
 
 static u32 pipedmc_interrupt_mask(struct intel_display *display)
 {
-	/*
-	 * TODO: Check if PIPEDMC_ERROR bit enabling causes errors
-	 * on PTL, enable it if validation passes
-	 */
 	if (DISPLAY_VER(display) >= 35)
 		return PIPEDMC_FLIPQ_PROG_DONE |
 			PIPEDMC_ERROR;
 
+	if (DISPLAY_VER(display) >= 30)
+		return PIPEDMC_FLIPQ_PROG_DONE |
+			PIPEDMC_GTT_FAULT |
+			PIPEDMC_ATS_FAULT |
+			PIPEDMC_ERROR;
+
 	/*
 	 * FIXME PIPEDMC_ERROR not enabled for now due to LNL pipe B
 	 * triggering it during the first DC state transition. Figure
@@ -941,6 +943,8 @@ void intel_dmc_load_program(struct intel_display *display)
 
 	gen9_set_dc_state_debugmask(display);
 
+	xe3lpd_enable_dc_count(display);
+
 	pipedmc_clock_gating_wa(display, false);
 }
 
@@ -1647,19 +1651,20 @@ static int intel_dmc_debugfs_status_show(struct seq_file *m, void *unused)
 		   DMC_VERSION_MINOR(dmc->version));
 
 	if (DISPLAY_VER(display) >= 12) {
-		intel_reg_t dc3co_reg;
+		if (DISPLAY_VER(display) >= 35) {
+			dc5_reg = DG1_DMC_DEBUG_DC5_COUNT;
+			seq_printf(m, "DC3CO count: %d\n",
+				   intel_de_read(display, XE3P_DMC_DC3CO_COUNT));
 
-		if (display->platform.dgfx || DISPLAY_VER(display) >= 14) {
-			dc3co_reg = DG1_DMC_DEBUG3;
+			seq_printf(m, "DC3CO residency: %d\n",
+				   intel_de_read(display, DC_STATE_DC3CO_RESIDENCY));
+		} else if (display->platform.dgfx || DISPLAY_VER(display) >= 14) {
 			dc5_reg = DG1_DMC_DEBUG_DC5_COUNT;
 		} else {
-			dc3co_reg = TGL_DMC_DEBUG3;
 			dc5_reg = TGL_DMC_DEBUG_DC5_COUNT;
 			dc6_reg = TGL_DMC_DEBUG_DC6_COUNT;
 		}
 
-		seq_printf(m, "DC3CO count: %d\n",
-			   intel_de_read(display, dc3co_reg));
 	} else {
 		dc5_reg = display->platform.broxton ? BXT_DMC_DC3_DC5_COUNT :
 			SKL_DMC_DC3_DC5_COUNT;
diff --git a/drivers/gpu/drm/i915/display/intel_dmc_regs.h b/drivers/gpu/drm/i915/display/intel_dmc_regs.h
index 38e342b45af06..6b7978fb8986f 100644
--- a/drivers/gpu/drm/i915/display/intel_dmc_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_dmc_regs.h
@@ -531,6 +531,8 @@ enum pipedmc_event_id {
 #define TGL_DMC_DEBUG3		_MMIO(0x101090)
 #define DG1_DMC_DEBUG3		_MMIO(0x13415c)
 
+#define XE3P_DMC_DC3CO_COUNT	_MMIO(0x8f05c)
+
 #define DMC_WAKELOCK_CFG	_MMIO(0x8F1B0)
 #define  DMC_WAKELOCK_CFG_ENABLE REG_BIT(31)
 #define DMC_WAKELOCK1_CTL	_MMIO(0x8F140)
diff --git a/drivers/gpu/drm/i915/display/intel_dmc_wl.c b/drivers/gpu/drm/i915/display/intel_dmc_wl.c
index b007343721e12..ab4e0e9573dfb 100644
--- a/drivers/gpu/drm/i915/display/intel_dmc_wl.c
+++ b/drivers/gpu/drm/i915/display/intel_dmc_wl.c
@@ -267,7 +267,7 @@ static bool intel_dmc_wl_check_range(struct intel_display *display,
 	 * the DMC and requires a DC exit for proper access.
 	 */
 	switch (dc_state) {
-	case DC_STATE_EN_DC3CO:
+	case DC_STATE_EN_UPTO_DC3CO:
 		ranges = xe3lpd_dc3co_dmc_ranges;
 		break;
 	case DC_STATE_EN_UPTO_DC5:
diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c
index 7ff5712f8b199..b2464a3dac056 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.c
+++ b/drivers/gpu/drm/i915/display/intel_dp.c
@@ -33,7 +33,6 @@
 #include <linux/notifier.h>
 #include <linux/seq_buf.h>
 #include <linux/slab.h>
-#include <linux/sort.h>
 #include <linux/string_helpers.h>
 #include <linux/timekeeping.h>
 #include <linux/types.h>
@@ -71,6 +70,7 @@
 #include "intel_dp.h"
 #include "intel_dp_aux.h"
 #include "intel_dp_hdcp.h"
+#include "intel_dp_link_caps.h"
 #include "intel_dp_link_training.h"
 #include "intel_dp_mst.h"
 #include "intel_dp_test.h"
@@ -313,7 +313,7 @@ static void intel_dp_set_max_sink_lane_count(struct intel_dp *intel_dp)
 }
 
 /* Get length of rates array potentially limited by max_rate. */
-static int intel_dp_rate_limit_len(const int *rates, int len, int max_rate)
+int intel_dp_rate_limit_len(const int *rates, int len, int max_rate)
 {
 	int i;
 
@@ -326,31 +326,6 @@ static int intel_dp_rate_limit_len(const int *rates, int len, int max_rate)
 	return 0;
 }
 
-/* Get length of common rates array potentially limited by max_rate. */
-static int intel_dp_common_len_rate_limit(const struct intel_dp *intel_dp,
-					  int max_rate)
-{
-	return intel_dp_rate_limit_len(intel_dp->common_rates,
-				       intel_dp->num_common_rates, max_rate);
-}
-
-int intel_dp_common_rate(struct intel_dp *intel_dp, int index)
-{
-	struct intel_display *display = to_intel_display(intel_dp);
-
-	if (drm_WARN_ON(display->drm,
-			index < 0 || index >= intel_dp->num_common_rates))
-		return 162000;
-
-	return intel_dp->common_rates[index];
-}
-
-/* Theoretical max between source and sink */
-int intel_dp_max_common_rate(struct intel_dp *intel_dp)
-{
-	return intel_dp_common_rate(intel_dp, intel_dp->num_common_rates - 1);
-}
-
 int intel_dp_max_source_lane_count(struct intel_digital_port *dig_port)
 {
 	int vbt_max_lanes = intel_bios_dp_max_lane_count(dig_port->base.devdata);
@@ -364,61 +339,19 @@ int intel_dp_max_source_lane_count(struct intel_digital_port *dig_port)
 
 /*
  * Theoretical max between source and sink.
- * Return %true if the max common lane count changed.
  */
-static bool intel_dp_set_max_common_lane_count(struct intel_dp *intel_dp)
+static int intel_dp_get_max_common_lane_count(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	int source_max = intel_dp_max_source_lane_count(dig_port);
 	int sink_max = intel_dp->max_sink_lane_count;
 	int lane_max = intel_tc_port_max_lane_count(dig_port);
 	int lttpr_max = drm_dp_lttpr_max_lane_count(intel_dp->lttpr_common_caps);
-	int old_max_common_lane_count = intel_dp->max_common_lane_count;
 
 	if (lttpr_max)
 		sink_max = min(sink_max, lttpr_max);
 
-	intel_dp->max_common_lane_count = min3(source_max, sink_max, lane_max);
-
-	return intel_dp->max_common_lane_count != old_max_common_lane_count;
-}
-
-int intel_dp_max_common_lane_count(struct intel_dp *intel_dp)
-{
-	return intel_dp->max_common_lane_count;
-}
-
-static int forced_lane_count(struct intel_dp *intel_dp)
-{
-	return clamp(intel_dp->link.force_lane_count, 1, intel_dp_max_common_lane_count(intel_dp));
-}
-
-int intel_dp_max_lane_count(struct intel_dp *intel_dp)
-{
-	int lane_count;
-
-	if (intel_dp->link.force_lane_count)
-		lane_count = forced_lane_count(intel_dp);
-	else
-		lane_count = intel_dp->link.max_lane_count;
-
-	switch (lane_count) {
-	case 1:
-	case 2:
-	case 4:
-		return lane_count;
-	default:
-		MISSING_CASE(lane_count);
-		return 1;
-	}
-}
-
-static int intel_dp_min_lane_count(struct intel_dp *intel_dp)
-{
-	if (intel_dp->link.force_lane_count)
-		return forced_lane_count(intel_dp);
-
-	return 1;
+	return min3(source_max, sink_max, lane_max);
 }
 
 int intel_dp_link_bw_overhead(int link_clock, int lane_count, int hdisplay,
@@ -698,174 +631,45 @@ int intel_dp_rate_index(const int *rates, int len, int rate)
 	return -1;
 }
 
-static int intel_dp_link_config_rate(struct intel_dp *intel_dp,
-				     const struct intel_dp_link_config *lc)
-{
-	return intel_dp_common_rate(intel_dp, lc->link_rate_idx);
-}
-
-static int intel_dp_link_config_lane_count(const struct intel_dp_link_config *lc)
-{
-	return 1 << lc->lane_count_exp;
-}
-
-static int intel_dp_link_config_bw(struct intel_dp *intel_dp,
-				   const struct intel_dp_link_config *lc)
-{
-	return drm_dp_max_dprx_data_rate(intel_dp_link_config_rate(intel_dp, lc),
-					 intel_dp_link_config_lane_count(lc));
-}
-
-static int link_config_cmp_by_bw(const void *a, const void *b, const void *p)
-{
-	struct intel_dp *intel_dp = (struct intel_dp *)p;	/* remove const */
-	const struct intel_dp_link_config *lc_a = a;
-	const struct intel_dp_link_config *lc_b = b;
-	int bw_a = intel_dp_link_config_bw(intel_dp, lc_a);
-	int bw_b = intel_dp_link_config_bw(intel_dp, lc_b);
-
-	if (bw_a != bw_b)
-		return bw_a - bw_b;
-
-	return intel_dp_link_config_rate(intel_dp, lc_a) -
-	       intel_dp_link_config_rate(intel_dp, lc_b);
-}
-
-static void intel_dp_link_config_init(struct intel_dp *intel_dp)
-{
-	struct intel_display *display = to_intel_display(intel_dp);
-	struct intel_dp_link_config *lc;
-	int num_common_lane_configs;
-	int i;
-	int j;
-
-	if (drm_WARN_ON(display->drm, !is_power_of_2(intel_dp_max_common_lane_count(intel_dp))))
-		return;
-
-	num_common_lane_configs = ilog2(intel_dp_max_common_lane_count(intel_dp)) + 1;
-
-	if (drm_WARN_ON(display->drm, intel_dp->num_common_rates * num_common_lane_configs >
-				    ARRAY_SIZE(intel_dp->link.configs)))
-		return;
-
-	intel_dp->link.num_configs = intel_dp->num_common_rates * num_common_lane_configs;
-
-	lc = &intel_dp->link.configs[0];
-	for (i = 0; i < intel_dp->num_common_rates; i++) {
-		for (j = 0; j < num_common_lane_configs; j++) {
-			lc->lane_count_exp = j;
-			lc->link_rate_idx = i;
-
-			lc++;
-		}
-	}
-
-	sort_r(intel_dp->link.configs, intel_dp->link.num_configs,
-	       sizeof(intel_dp->link.configs[0]),
-	       link_config_cmp_by_bw, NULL,
-	       intel_dp);
-}
-
-void intel_dp_link_config_get(struct intel_dp *intel_dp, int idx, int *link_rate, int *lane_count)
-{
-	struct intel_display *display = to_intel_display(intel_dp);
-	const struct intel_dp_link_config *lc;
-
-	if (drm_WARN_ON(display->drm, idx < 0 || idx >= intel_dp->link.num_configs))
-		idx = 0;
-
-	lc = &intel_dp->link.configs[idx];
-
-	*link_rate = intel_dp_link_config_rate(intel_dp, lc);
-	*lane_count = intel_dp_link_config_lane_count(lc);
-}
-
-int intel_dp_link_config_index(struct intel_dp *intel_dp, int link_rate, int lane_count)
-{
-	int link_rate_idx = intel_dp_rate_index(intel_dp->common_rates, intel_dp->num_common_rates,
-						link_rate);
-	int lane_count_exp = ilog2(lane_count);
-	int i;
-
-	for (i = 0; i < intel_dp->link.num_configs; i++) {
-		const struct intel_dp_link_config *lc = &intel_dp->link.configs[i];
-
-		if (lc->lane_count_exp == lane_count_exp &&
-		    lc->link_rate_idx == link_rate_idx)
-			return i;
-	}
-
-	return -1;
-}
-
-/* Return %true if the common rates changed. */
-static bool intel_dp_set_common_rates(struct intel_dp *intel_dp)
+static void intel_dp_get_common_rates(struct intel_dp *intel_dp,
+				      int common_rates[DP_MAX_SUPPORTED_RATES],
+				      int *num_common_rates)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
-	int num_old_common_rates = intel_dp->num_common_rates;
-	int old_common_rates[DP_MAX_SUPPORTED_RATES];
 
 	drm_WARN_ON(display->drm,
 		    !intel_dp->num_source_rates || !intel_dp->num_sink_rates);
 
-	/* TODO: Add a struct containing both rates and number of rates. */
-	static_assert(__same_type(old_common_rates[0], intel_dp->common_rates[0]) &&
-		      sizeof(old_common_rates) == sizeof(intel_dp->common_rates));
-	memcpy(old_common_rates, intel_dp->common_rates,
-	       num_old_common_rates * sizeof(old_common_rates[0]));
-
-	intel_dp->num_common_rates = intersect_rates(intel_dp->source_rates,
-						     intel_dp->num_source_rates,
-						     intel_dp->sink_rates,
-						     intel_dp->num_sink_rates,
-						     intel_dp->common_rates);
+	*num_common_rates = intersect_rates(intel_dp->source_rates,
+					    intel_dp->num_source_rates,
+					    intel_dp->sink_rates,
+					    intel_dp->num_sink_rates,
+					    common_rates);
 
 	/* Paranoia, there should always be something in common. */
-	if (drm_WARN_ON(display->drm, intel_dp->num_common_rates == 0)) {
-		intel_dp->common_rates[0] = 162000;
-		intel_dp->num_common_rates = 1;
+	if (drm_WARN_ON(display->drm, *num_common_rates == 0)) {
+		common_rates[0] = 162000;
+		*num_common_rates = 1;
 	}
-
-	return num_old_common_rates != intel_dp->num_common_rates ||
-	       memcmp(old_common_rates, intel_dp->common_rates,
-		      num_old_common_rates * sizeof(old_common_rates[0]));
 }
 
 /* Return %true if any common link param changed. */
 static bool intel_dp_set_common_link_params(struct intel_dp *intel_dp)
 {
+	int num_common_rates;
+	int common_rates[DP_MAX_SUPPORTED_RATES];
 	bool params_changed = false;
 
-	if (intel_dp_set_common_rates(intel_dp))
-		params_changed = true;
-
-	if (intel_dp_set_max_common_lane_count(intel_dp))
+	intel_dp_get_common_rates(intel_dp, common_rates, &num_common_rates);
+	if (intel_dp_link_caps_update(intel_dp->link.caps,
+				      common_rates, num_common_rates,
+				      intel_dp_get_max_common_lane_count(intel_dp),
+				      intel_dp->reset_link_params))
 		params_changed = true;
 
-	intel_dp_link_config_init(intel_dp);
-
 	return params_changed;
 }
 
-bool intel_dp_link_params_valid(struct intel_dp *intel_dp, int link_rate,
-				u8 lane_count)
-{
-	/*
-	 * FIXME: we need to synchronize the current link parameters with
-	 * hardware readout. Currently fast link training doesn't work on
-	 * boot-up.
-	 */
-	if (link_rate == 0 ||
-	    link_rate > intel_dp->link.max_rate)
-		return false;
-
-	if (lane_count == 0 ||
-	    lane_count > intel_dp_max_lane_count(intel_dp))
-		return false;
-
-	return true;
-}
-
 u32 intel_dp_mode_to_fec_clock(u32 mode_clock)
 {
 	return div_u64(mul_u32_u32(mode_clock, DP_DSC_FEC_OVERHEAD_FACTOR),
@@ -1489,6 +1293,7 @@ intel_dp_mode_valid_format(struct intel_connector *connector,
 	struct intel_dp *intel_dp = intel_attached_dp(connector);
 	enum intel_output_format output_format;
 	int max_rate, mode_rate, max_lanes, max_link_clock;
+	struct intel_dp_link_config max_bw_config;
 	u16 dsc_max_compressed_bpp = 0;
 	enum drm_mode_status status;
 	bool dsc = false;
@@ -1501,8 +1306,9 @@ intel_dp_mode_valid_format(struct intel_connector *connector,
 
 	output_format = intel_dp_output_format(connector, sink_format);
 
-	max_link_clock = intel_dp_max_link_rate(intel_dp);
-	max_lanes = intel_dp_max_lane_count(intel_dp);
+	intel_dp_link_caps_get_max_bw_config(intel_dp->link.caps, &max_bw_config);
+	max_link_clock = max_bw_config.rate;
+	max_lanes = max_bw_config.lane_count;
 
 	max_rate = intel_dp_max_link_data_rate(intel_dp, max_link_clock, max_lanes);
 
@@ -1693,41 +1499,7 @@ static void intel_dp_print_rates(struct intel_dp *intel_dp)
 	seq_buf_print_array(&s, intel_dp->sink_rates, intel_dp->num_sink_rates);
 	drm_dbg_kms(display->drm, "sink rates: %s\n", seq_buf_str(&s));
 
-	seq_buf_clear(&s);
-	seq_buf_print_array(&s, intel_dp->common_rates, intel_dp->num_common_rates);
-	drm_dbg_kms(display->drm, "common rates: %s\n", seq_buf_str(&s));
-}
-
-static int forced_link_rate(struct intel_dp *intel_dp)
-{
-	int len = intel_dp_common_len_rate_limit(intel_dp, intel_dp->link.force_rate);
-
-	if (len == 0)
-		return intel_dp_common_rate(intel_dp, 0);
-
-	return intel_dp_common_rate(intel_dp, len - 1);
-}
-
-int
-intel_dp_max_link_rate(struct intel_dp *intel_dp)
-{
-	int len;
-
-	if (intel_dp->link.force_rate)
-		return forced_link_rate(intel_dp);
-
-	len = intel_dp_common_len_rate_limit(intel_dp, intel_dp->link.max_rate);
-
-	return intel_dp_common_rate(intel_dp, len - 1);
-}
-
-static int
-intel_dp_min_link_rate(struct intel_dp *intel_dp)
-{
-	if (intel_dp->link.force_rate)
-		return forced_link_rate(intel_dp);
-
-	return intel_dp_common_rate(intel_dp, 0);
+	intel_dp_link_caps_print_common_rates(intel_dp->link.caps);
 }
 
 int intel_dp_rate_select(struct intel_dp *intel_dp, int rate)
@@ -1911,42 +1683,42 @@ intel_dp_compute_link_config_wide(struct intel_dp *intel_dp,
 				  const struct drm_connector_state *conn_state,
 				  const struct link_config_limits *limits)
 {
-	int bpp, i, lane_count, clock = intel_dp_mode_clock(pipe_config, conn_state);
-	int link_rate, link_avail;
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
+	int bpp, clock = intel_dp_mode_clock(pipe_config, conn_state);
+	struct intel_dp_link_caps *link_caps = intel_dp->link.caps;
+	struct intel_dp_link_caps_order order =
+		intel_dp_link_caps_connector_compute_order(connector);
+	int link_avail;
 
 	for (bpp = fxp_q4_to_int(limits->link.max_bpp_x16);
 	     bpp >= fxp_q4_to_int(limits->link.min_bpp_x16);
 	     bpp -= 2 * 3) {
 		int link_bpp_x16 =
 			intel_dp_output_format_link_bpp_x16(pipe_config->output_format, bpp);
+		struct intel_dp_link_config link_config;
+		struct intel_dp_link_caps_iter iter;
 
-		for (i = 0; i < intel_dp->num_common_rates; i++) {
-			link_rate = intel_dp_common_rate(intel_dp, i);
-			if (link_rate < limits->min_rate ||
-			    link_rate > limits->max_rate)
-				continue;
-
-			for (lane_count = limits->min_lane_count;
-			     lane_count <= limits->max_lane_count;
-			     lane_count <<= 1) {
-				const struct drm_display_mode *adjusted_mode =
+		intel_dp_link_caps_iter_start(&iter, link_caps, order, limits->link_config_filter);
+		for_each_dp_link_config(&iter, &link_config) {
+			const struct drm_display_mode *adjusted_mode =
 					&pipe_config->hw.adjusted_mode;
-				int mode_rate =
-					intel_dp_link_required(link_rate, lane_count,
-							       clock, adjusted_mode->hdisplay,
-							       link_bpp_x16, 0);
-
-				link_avail = intel_dp_max_link_data_rate(intel_dp,
-									 link_rate,
-									 lane_count);
-
-				if (mode_rate <= link_avail) {
-					pipe_config->lane_count = lane_count;
-					pipe_config->pipe_bpp = bpp;
-					pipe_config->port_clock = link_rate;
-
-					return 0;
-				}
+			int mode_rate;
+
+			mode_rate = intel_dp_link_required(link_config.rate,
+							   link_config.lane_count,
+							   clock, adjusted_mode->hdisplay,
+							   link_bpp_x16, 0);
+
+			link_avail = intel_dp_max_link_data_rate(intel_dp,
+								 link_config.rate,
+								 link_config.lane_count);
+
+			if (mode_rate <= link_avail) {
+				pipe_config->lane_count = link_config.lane_count;
+				pipe_config->pipe_bpp = bpp;
+				pipe_config->port_clock = link_config.rate;
+
+				return 0;
 			}
 		}
 	}
@@ -2146,74 +1918,110 @@ static int dsc_compute_link_config(struct intel_dp *intel_dp,
 				   int dsc_bpp_x16)
 {
 	const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
-	int link_rate, lane_count;
-	int i;
-
-	for (i = 0; i < intel_dp->num_common_rates; i++) {
-		link_rate = intel_dp_common_rate(intel_dp, i);
-		if (link_rate < limits->min_rate || link_rate > limits->max_rate)
-			continue;
-
-		for (lane_count = limits->min_lane_count;
-		     lane_count <= limits->max_lane_count;
-		     lane_count <<= 1) {
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
+	struct intel_dp_link_caps *link_caps = intel_dp->link.caps;
+	struct intel_dp_link_caps_order order =
+		intel_dp_link_caps_connector_compute_order(connector);
+	struct intel_dp_link_config link_config;
+	struct intel_dp_link_caps_iter iter;
+
+	intel_dp_link_caps_iter_start(&iter, link_caps, order, limits->link_config_filter);
+	for_each_dp_link_config(&iter, &link_config) {
+		/*
+		 * FIXME: intel_dp_mtp_tu_compute_config() requires
+		 * ->lane_count and ->port_clock set before we know
+		 * they'll work. If we end up failing altogether,
+		 * they'll remain in crtc state. This shouldn't matter,
+		 * as we'd then bail out from compute config, but it's
+		 * just ugly.
+		 */
+		pipe_config->lane_count = link_config.lane_count;
+		pipe_config->port_clock = link_config.rate;
 
-			/*
-			 * FIXME: intel_dp_mtp_tu_compute_config() requires
-			 * ->lane_count and ->port_clock set before we know
-			 * they'll work. If we end up failing altogether,
-			 * they'll remain in crtc state. This shouldn't matter,
-			 * as we'd then bail out from compute config, but it's
-			 * just ugly.
-			 */
-			pipe_config->lane_count = lane_count;
-			pipe_config->port_clock = link_rate;
-
-			if (drm_dp_is_uhbr_rate(link_rate)) {
-				int ret;
-
-				ret = intel_dp_mtp_tu_compute_config(intel_dp,
-								     pipe_config,
-								     conn_state,
-								     dsc_bpp_x16,
-								     dsc_bpp_x16,
-								     0, true);
-				if (ret)
-					continue;
-			} else {
-				unsigned long bw_overhead_flags =
-					pipe_config->fec_enable ? DRM_DP_BW_OVERHEAD_FEC : 0;
-				int line_slice_count =
-					intel_dsc_line_slice_count(&pipe_config->dsc.slice_config);
-
-				if (!is_bw_sufficient_for_dsc_config(intel_dp,
-								     link_rate, lane_count,
-								     adjusted_mode->crtc_clock,
-								     adjusted_mode->hdisplay,
-								     line_slice_count,
-								     dsc_bpp_x16,
-								     bw_overhead_flags))
-					continue;
-			}
+		if (drm_dp_is_uhbr_rate(link_config.rate)) {
+			int ret;
 
-			return 0;
+			ret = intel_dp_mtp_tu_compute_config(intel_dp,
+							     pipe_config,
+							     conn_state,
+							     dsc_bpp_x16,
+							     dsc_bpp_x16,
+							     0, true);
+			if (ret)
+				continue;
+		} else {
+			unsigned long bw_overhead_flags =
+				pipe_config->fec_enable ? DRM_DP_BW_OVERHEAD_FEC : 0;
+			int line_slice_count =
+				intel_dsc_line_slice_count(&pipe_config->dsc.slice_config);
+
+			if (!is_bw_sufficient_for_dsc_config(intel_dp,
+							     link_config.rate,
+							     link_config.lane_count,
+							     adjusted_mode->crtc_clock,
+							     adjusted_mode->hdisplay,
+							     line_slice_count,
+							     dsc_bpp_x16,
+							     bw_overhead_flags))
+				continue;
 		}
+
+		return 0;
 	}
 
 	return -EINVAL;
 }
 
+static u16 intel_dp_dsc_max_delta_bppx16(const struct intel_connector *connector,
+					 enum intel_output_format output_format)
+{
+	const u8 *dsc_dpcd = connector->dp.dsc_dpcd;
+	u8 max_bpp_delta_v1 = dsc_dpcd[DP_DSC_MAX_BPP_DELTA_VERSION_1 - DP_DSC_SUPPORT];
+	int max_bpp;
+
+	if (!(dsc_dpcd[DP_DSC_MAX_BITS_PER_PIXEL_HI - DP_DSC_SUPPORT] &
+	    DP_DSC_MAX_BPP_DELTA_AVAILABILITY))
+		return 0;
+
+	switch (output_format) {
+	case INTEL_OUTPUT_FORMAT_RGB:
+	case INTEL_OUTPUT_FORMAT_YCBCR444:
+		max_bpp =  max_bpp_delta_v1 & DP_DSC_RGB_YCbCr444_MAX_BPP_DELTA_MASK;
+		if (max_bpp >= 1 && max_bpp <= 21)
+			max_bpp =  max_bpp + DP_DSC_BPP_DELTA_444 - 1;
+		else
+			max_bpp = 0;
+		break;
+	case INTEL_OUTPUT_FORMAT_YCBCR420:
+		max_bpp = (max_bpp_delta_v1 & DP_DSC_NATIVE_YCbCr420_MAX_BPP_DELTA_MASK) >>
+			  DP_DSC_BPP_DELTA_SHIFT_420;
+		if (max_bpp >= 1 && max_bpp <= 7)
+			max_bpp = max_bpp + DP_DSC_BPP_DELTA_420 - 1;
+		break;
+	default:
+		MISSING_CASE(output_format);
+		return 0;
+	}
+
+	return max_bpp << 4;
+}
+
 static
 u16 intel_dp_dsc_max_sink_compressed_bppx16(const struct intel_connector *connector,
 					    enum intel_output_format output_format,
 					    int bpc)
 {
-	u16 max_bppx16 = drm_edp_dsc_sink_output_bpp(connector->dp.dsc_dpcd);
+	u16 max_bppx16 = intel_dp_dsc_max_delta_bppx16(connector, output_format);
+
+	if (max_bppx16)
+		return max_bppx16;
+
+	max_bppx16 = drm_edp_dsc_sink_output_bpp(connector->dp.dsc_dpcd);
 
 	if (max_bppx16)
 		return max_bppx16;
 	/*
-	 * If support not given in DPCD 67h, 68h use the Maximum Allowed bit rate
+	 * If support not given in DPCD 67h, 68h, 6Eh, 6Fh use the Maximum Allowed bit rate
 	 * values as given in spec Table 2-157 DP v2.0
 	 */
 	switch (output_format) {
@@ -2369,7 +2177,7 @@ static int dsc_compute_compressed_bpp(struct intel_dp *intel_dp,
 				      int pipe_bpp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
-	const struct intel_connector *connector = to_intel_connector(conn_state->connector);
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	int min_bpp_x16, max_bpp_x16, bpp_step_x16;
 	int bpp_x16;
 	int ret;
@@ -2381,8 +2189,19 @@ static int dsc_compute_compressed_bpp(struct intel_dp *intel_dp,
 	max_bpp_x16 = align_max_compressed_bpp_x16(connector, pipe_config->output_format,
 						   pipe_bpp, max_bpp_x16);
 	if (intel_dp_is_edp(intel_dp)) {
-		pipe_config->port_clock = limits->max_rate;
-		pipe_config->lane_count = limits->max_lane_count;
+		struct intel_dp_link_config max_link_config;
+
+		/*
+		 * FIXME: Clarify why eDP does not use the regular SST BW
+		 * check and instead always uses the maximum link config,
+		 * regardless of intel_dp::use_max_params. Then unify this eDP
+		 * path with the regular DP path.
+		 */
+		if (!intel_dp_get_connector_max_link_config(connector, limits, &max_link_config))
+			return -EINVAL;
+
+		pipe_config->port_clock = max_link_config.rate;
+		pipe_config->lane_count = max_link_config.lane_count;
 
 		pipe_config->dsc.compressed_bpp_x16 = max_bpp_x16;
 
@@ -2695,6 +2514,20 @@ bool intel_dp_mode_valid_with_dsc(struct intel_connector *connector,
 					       bw_overhead_flags);
 }
 
+bool
+intel_dp_get_connector_max_link_config(struct intel_connector *connector,
+				       const struct link_config_limits *limits,
+				       struct intel_dp_link_config *max_link_config)
+{
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	struct intel_dp_link_caps *link_caps = intel_dp->link.caps;
+	struct intel_dp_link_caps_order order =
+		intel_dp_link_caps_connector_compute_order(connector);
+
+	return intel_dp_link_caps_get_max_config(link_caps, order.key, limits->link_config_filter,
+						 max_link_config);
+}
+
 /*
  * Calculate the output link min, max bpp values in limits based on the pipe bpp
  * range, crtc_state and dsc mode. Return true on success.
@@ -2711,6 +2544,7 @@ intel_dp_compute_config_link_bpp_limits(struct intel_connector *connector,
 		&crtc_state->hw.adjusted_mode;
 	const struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	const struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
+	struct intel_dp_link_config max_link_config;
 	int max_link_bpp_x16;
 
 	max_link_bpp_x16 = min(crtc_state->max_link_bpp_x16,
@@ -2740,14 +2574,17 @@ intel_dp_compute_config_link_bpp_limits(struct intel_connector *connector,
 
 	limits->link.max_bpp_x16 = max_link_bpp_x16;
 
+	if (!intel_dp_get_connector_max_link_config(connector, limits, &max_link_config))
+		return false;
+
 	drm_dbg_kms(display->drm,
-		    "[ENCODER:%d:%s][CRTC:%d:%s] DP link limits: pixel clock %d kHz DSC %s max lanes %d max rate %d max pipe_bpp %d min link_bpp " FXP_Q4_FMT " max link_bpp " FXP_Q4_FMT "\n",
+		    "[ENCODER:%d:%s][CRTC:%d:%s] DP link limits: pixel clock %d kHz DSC %s max link %dx%d max pipe_bpp %d min link_bpp " FXP_Q4_FMT " max link_bpp " FXP_Q4_FMT "\n",
 		    encoder->base.base.id, encoder->base.name,
 		    crtc->base.base.id, crtc->base.name,
 		    adjusted_mode->crtc_clock,
 		    str_on_off(dsc),
-		    limits->max_lane_count,
-		    limits->max_rate,
+		    max_link_config.lane_count,
+		    max_link_config.rate,
 		    limits->pipe.max_bpp,
 		    FXP_Q4_ARGS(limits->link.min_bpp_x16),
 		    FXP_Q4_ARGS(limits->link.max_bpp_x16));
@@ -2798,17 +2635,12 @@ intel_dp_compute_config_limits(struct intel_dp *intel_dp,
 			       struct link_config_limits *limits)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
+	struct intel_dp_link_caps *link_caps = intel_dp->link.caps;
 	bool is_mst = intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST);
 	struct intel_connector *connector =
 		to_intel_connector(conn_state->connector);
 
-	limits->min_rate = intel_dp_min_link_rate(intel_dp);
-	limits->max_rate = intel_dp_max_link_rate(intel_dp);
-
-	limits->min_rate = min(limits->min_rate, limits->max_rate);
-
-	limits->min_lane_count = intel_dp_min_lane_count(intel_dp);
-	limits->max_lane_count = intel_dp_max_lane_count(intel_dp);
+	limits->link_config_filter = INTEL_DP_LINK_CAPS_FILTER_ALL;
 
 	limits->pipe.min_bpp = intel_dp_min_bpp(crtc_state->output_format);
 	if (is_mst) {
@@ -2873,6 +2705,9 @@ intel_dp_compute_config_limits(struct intel_dp *intel_dp,
 			    crtc_state->pipe_bpp, limits->pipe.max_bpp);
 
 	if (is_mst || intel_dp->use_max_params) {
+		struct intel_dp_link_caps_filter new_filter = INTEL_DP_LINK_CAPS_FILTER_NONE;
+		struct intel_dp_link_config max_config;
+
 		/*
 		 * For MST we always configure max link bw - the spec doesn't
 		 * seem to suggest we should do otherwise.
@@ -2884,11 +2719,17 @@ intel_dp_compute_config_limits(struct intel_dp *intel_dp,
 		 * configuration, and typically on older panels these
 		 * values correspond to the native resolution of the panel.
 		 */
-		limits->min_lane_count = limits->max_lane_count;
-		limits->min_rate = limits->max_rate;
+		if (!intel_dp_get_connector_max_link_config(connector, limits, &max_config))
+			return false;
+
+		if (!intel_dp_link_caps_filter_add(link_caps, &new_filter, &max_config))
+			return false;
+
+		limits->link_config_filter = new_filter;
 	}
 
-	intel_dp_test_compute_config(intel_dp, crtc_state, limits);
+	if (!intel_dp_test_compute_config(connector, crtc_state, limits))
+		return false;
 
 	return intel_dp_compute_config_link_bpp_limits(connector,
 						       crtc_state,
@@ -3646,19 +3487,19 @@ int intel_dp_compute_min_hblank(struct intel_crtc_state *crtc_state,
 }
 
 int
-intel_dp_compute_config(struct intel_encoder *encoder,
+intel_dp_compute_config(struct intel_atomic_state *state,
+			struct intel_encoder *encoder,
 			struct intel_crtc_state *pipe_config,
 			struct drm_connector_state *conn_state)
 {
 	struct intel_display *display = to_intel_display(encoder);
-	struct intel_atomic_state *state = to_intel_atomic_state(conn_state->state);
 	struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 	struct intel_connector *connector = intel_dp->attached_connector;
 	int ret = 0, link_bpp_x16;
 
 	if (intel_dp_is_edp(intel_dp)) {
-		ret = intel_panel_compute_config(connector, adjusted_mode);
+		ret = intel_panel_compute_config(state, pipe_config, connector);
 		if (ret)
 			return ret;
 	}
@@ -3771,12 +3612,15 @@ void intel_dp_set_link_params(struct intel_dp *intel_dp,
 
 void intel_dp_reset_link_params(struct intel_dp *intel_dp)
 {
-	intel_dp->link.max_lane_count = intel_dp_max_common_lane_count(intel_dp);
-	intel_dp->link.max_rate = intel_dp_max_common_rate(intel_dp);
+	/*
+	 * TODO: Remove the following reset of link capabilities, as
+	 * this isn't needed after intel_dp_link_caps_update(reset=true)
+	 * was called.
+	 */
+	intel_dp_link_caps_reset(intel_dp->link.caps);
 	intel_dp->link.mst_probed_lane_count = 0;
 	intel_dp->link.mst_probed_rate = 0;
-	intel_dp->link.retrain_disabled = false;
-	intel_dp->link.seq_train_failures = 0;
+	intel_dp_link_training_reset(intel_dp->link.training);
 }
 
 /* Enable backlight PWM and backlight PP control. */
@@ -5676,32 +5520,6 @@ void intel_read_dp_sdp(struct intel_encoder *encoder,
 	}
 }
 
-static bool intel_dp_link_ok(struct intel_dp *intel_dp,
-			     u8 link_status[DP_LINK_STATUS_SIZE])
-{
-	struct intel_display *display = to_intel_display(intel_dp);
-	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
-	bool uhbr = intel_dp->link_rate >= 1000000;
-	bool ok;
-
-	if (uhbr)
-		ok = drm_dp_128b132b_lane_channel_eq_done(link_status,
-							  intel_dp->lane_count);
-	else
-		ok = drm_dp_channel_eq_ok(link_status, intel_dp->lane_count);
-
-	if (ok)
-		return true;
-
-	intel_dp_dump_link_status(intel_dp, DP_PHY_DPRX, link_status);
-	drm_dbg_kms(display->drm,
-		    "[ENCODER:%d:%s] %s link not ok, retraining\n",
-		    encoder->base.base.id, encoder->base.name,
-		    uhbr ? "128b/132b" : "8b/10b");
-
-	return false;
-}
-
 static void
 intel_dp_mst_hpd_irq(struct intel_dp *intel_dp, u8 *esi, u8 *ack)
 {
@@ -5735,7 +5553,7 @@ static bool
 intel_dp_check_mst_status(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
-	bool force_retrain = intel_dp->link.force_retrain;
+	bool force_retrain = intel_dp_link_training_get_force_retrain(intel_dp->link.training);
 	bool reprobe_needed = false;
 	int tries = 33;
 
@@ -5814,78 +5632,6 @@ intel_dp_handle_hdmi_link_status_change(struct intel_dp *intel_dp)
 	}
 }
 
-static int
-intel_dp_read_link_status(struct intel_dp *intel_dp, u8 link_status[DP_LINK_STATUS_SIZE])
-{
-	int err;
-
-	memset(link_status, 0, DP_LINK_STATUS_SIZE);
-
-	if (intel_dp_mst_active_streams(intel_dp) > 0)
-		err = drm_dp_dpcd_read_data(&intel_dp->aux, DP_LANE0_1_STATUS_ESI,
-					    link_status, DP_LINK_STATUS_SIZE - 2);
-	else
-		err = drm_dp_dpcd_read_phy_link_status(&intel_dp->aux, DP_PHY_DPRX,
-						       link_status);
-
-	if (err)
-		return err;
-
-	if (link_status[DP_LANE_ALIGN_STATUS_UPDATED - DP_LANE0_1_STATUS] &
-	    DP_DOWNSTREAM_PORT_STATUS_CHANGED)
-		WRITE_ONCE(intel_dp->downstream_port_changed, true);
-
-	return 0;
-}
-
-static bool
-intel_dp_needs_link_retrain(struct intel_dp *intel_dp)
-{
-	u8 link_status[DP_LINK_STATUS_SIZE];
-
-	if (!intel_dp->link.active)
-		return false;
-
-	/*
-	 * While PSR source HW is enabled, it will control main-link sending
-	 * frames, enabling and disabling it so trying to do a retrain will fail
-	 * as the link would or not be on or it could mix training patterns
-	 * and frame data at the same time causing retrain to fail.
-	 * Also when exiting PSR, HW will retrain the link anyways fixing
-	 * any link status error.
-	 */
-	if (intel_psr_enabled(intel_dp))
-		return false;
-
-	if (intel_dp->link.force_retrain)
-		return true;
-
-	if (intel_dp_read_link_status(intel_dp, link_status) < 0)
-		return false;
-
-	/*
-	 * Validate the cached values of intel_dp->link_rate and
-	 * intel_dp->lane_count before attempting to retrain.
-	 *
-	 * FIXME would be nice to user the crtc state here, but since
-	 * we need to call this from the short HPD handler that seems
-	 * a bit hard.
-	 */
-	if (!intel_dp_link_params_valid(intel_dp, intel_dp->link_rate,
-					intel_dp->lane_count))
-		return false;
-
-	if (intel_dp->link.retrain_disabled)
-		return false;
-
-	if (intel_dp->link.seq_train_failures)
-		return true;
-
-	/* Retrain if link not ok */
-	return !intel_dp_link_ok(intel_dp, link_status) &&
-		!intel_psr_link_ok(intel_dp);
-}
-
 bool intel_dp_has_connector(struct intel_dp *intel_dp,
 			    const struct drm_connector_state *conn_state)
 {
@@ -5977,86 +5723,6 @@ void intel_dp_flush_connector_commits(struct intel_connector *connector)
 	wait_for_connector_hw_done(connector->base.state);
 }
 
-static bool intel_dp_is_connected(struct intel_dp *intel_dp)
-{
-	struct intel_connector *connector = intel_dp->attached_connector;
-
-	return connector->base.status == connector_status_connected ||
-		intel_dp->is_mst;
-}
-
-static int intel_dp_retrain_link(struct intel_encoder *encoder,
-				 struct drm_modeset_acquire_ctx *ctx)
-{
-	struct intel_display *display = to_intel_display(encoder);
-	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-	u8 pipe_mask;
-	int ret;
-
-	if (!intel_dp_is_connected(intel_dp))
-		return 0;
-
-	ret = drm_modeset_lock(&display->drm->mode_config.connection_mutex,
-			       ctx);
-	if (ret)
-		return ret;
-
-	if (!intel_dp_needs_link_retrain(intel_dp))
-		return 0;
-
-	ret = intel_dp_get_active_pipes(intel_dp, ctx, &pipe_mask);
-	if (ret)
-		return ret;
-
-	if (pipe_mask == 0)
-		return 0;
-
-	if (!intel_dp_needs_link_retrain(intel_dp))
-		return 0;
-
-	drm_dbg_kms(display->drm,
-		    "[ENCODER:%d:%s] retraining link (forced %s)\n",
-		    encoder->base.base.id, encoder->base.name,
-		    str_yes_no(intel_dp->link.force_retrain));
-
-	ret = intel_modeset_commit_pipes(display, pipe_mask, ctx);
-	if (ret == -EDEADLK)
-		return ret;
-
-	intel_dp->link.force_retrain = false;
-
-	if (ret)
-		drm_dbg_kms(display->drm,
-			    "[ENCODER:%d:%s] link retraining failed: %pe\n",
-			    encoder->base.base.id, encoder->base.name,
-			    ERR_PTR(ret));
-
-	return ret;
-}
-
-void intel_dp_link_check(struct intel_encoder *encoder)
-{
-	struct drm_modeset_acquire_ctx ctx;
-	int ret;
-
-	intel_modeset_lock_ctx_retry(&ctx, NULL, 0, ret)
-		ret = intel_dp_retrain_link(encoder, &ctx);
-}
-
-void intel_dp_check_link_state(struct intel_dp *intel_dp)
-{
-	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct intel_encoder *encoder = &dig_port->base;
-
-	if (!intel_dp_is_connected(intel_dp))
-		return;
-
-	if (!intel_dp_needs_link_retrain(intel_dp))
-		return;
-
-	intel_encoder_link_check_queue_work(encoder, 0);
-}
-
 static void intel_dp_handle_device_service_irq(struct intel_dp *intel_dp, u8 irq_mask)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
@@ -6152,7 +5818,7 @@ intel_dp_short_pulse(struct intel_dp *intel_dp)
 	/*
 	 * Force checking the link status for DPCD_REV < 1.2
 	 * TODO: let the link status check depend on LINK_STATUS_CHANGED
-	 * or intel_dp->link.force_retrain for DPCD_REV >= 1.2
+	 * or intel_dp->link.training.force_retrain for DPCD_REV >= 1.2
 	 */
 	esi[3] |= LINK_STATUS_CHANGED;
 	if (intel_dp_handle_link_service_irq(intel_dp, esi[3]))
@@ -6882,7 +6548,7 @@ static int intel_modeset_tile_group(struct intel_atomic_state *state,
 	return ret;
 }
 
-static int intel_modeset_affected_transcoders(struct intel_atomic_state *state, u8 transcoders)
+static int intel_modeset_affected_transcoders(struct intel_atomic_state *state, u16 transcoders)
 {
 	struct intel_display *display = to_intel_display(state);
 	struct intel_crtc *crtc;
@@ -6930,7 +6596,7 @@ static int intel_modeset_synced_crtcs(struct intel_atomic_state *state,
 		drm_atomic_get_old_connector_state(&state->base, &connector->base);
 	const struct intel_crtc_state *old_crtc_state;
 	struct intel_crtc *crtc;
-	u8 transcoders;
+	u16 transcoders;
 
 	crtc = to_intel_crtc(old_conn_state->crtc);
 	if (!crtc)
@@ -7655,3 +7321,25 @@ u8 intel_dp_as_sdp_transmission_time(void)
 
 	return DP_PR_AS_SDP_SETUP_TIME_T1;
 }
+
+int intel_dp_link_init(struct intel_dp *intel_dp)
+{
+	intel_dp->link.training = intel_dp_link_training_init(intel_dp);
+	if (!intel_dp->link.training)
+		return -ENOMEM;
+
+	intel_dp->link.caps = intel_dp_link_caps_init(intel_dp);
+	if (!intel_dp->link.caps) {
+		intel_dp_link_training_cleanup(intel_dp->link.training);
+
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+void intel_dp_link_cleanup(struct intel_dp *intel_dp)
+{
+	intel_dp_link_caps_cleanup(intel_dp->link.caps);
+	intel_dp_link_training_cleanup(intel_dp->link.training);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_dp.h b/drivers/gpu/drm/i915/display/intel_dp.h
index 46a7f5c70981f..0ec519fa12368 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.h
+++ b/drivers/gpu/drm/i915/display/intel_dp.h
@@ -8,6 +8,8 @@
 
 #include <linux/types.h>
 
+#include "intel_dp_link_caps.h"
+
 enum intel_output_format;
 enum pipe;
 enum port;
@@ -22,11 +24,11 @@ struct intel_crtc_state;
 struct intel_digital_port;
 struct intel_display;
 struct intel_dp;
+struct intel_dp_link_config;
 struct intel_encoder;
 
 struct link_config_limits {
-	int min_rate, max_rate;
-	int min_lane_count, max_lane_count;
+	struct intel_dp_link_caps_filter link_config_filter;
 	struct {
 		/* Uncompressed DSC input or link output bpp in 1 bpp units */
 		int min_bpp, max_bpp;
@@ -59,8 +61,6 @@ int intel_dp_get_active_pipes(struct intel_dp *intel_dp,
 			      struct drm_modeset_acquire_ctx *ctx,
 			      u8 *pipe_mask);
 void intel_dp_flush_connector_commits(struct intel_connector *connector);
-void intel_dp_link_check(struct intel_encoder *encoder);
-void intel_dp_check_link_state(struct intel_dp *intel_dp);
 void intel_dp_set_power(struct intel_dp *intel_dp, u8 mode);
 void intel_dp_configure_protocol_converter(struct intel_dp *intel_dp,
 					   const struct intel_crtc_state *crtc_state);
@@ -73,7 +73,8 @@ void intel_dp_sink_disable_decompression(struct intel_atomic_state *state,
 void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder);
 void intel_dp_encoder_shutdown(struct intel_encoder *intel_encoder);
 void intel_dp_encoder_flush_work(struct drm_encoder *encoder);
-int intel_dp_compute_config(struct intel_encoder *encoder,
+int intel_dp_compute_config(struct intel_atomic_state *state,
+			    struct intel_encoder *encoder,
 			    struct intel_crtc_state *pipe_config,
 			    struct drm_connector_state *conn_state);
 bool intel_dp_needs_8b10b_fec(const struct intel_crtc_state *crtc_state,
@@ -102,17 +103,11 @@ void intel_edp_backlight_off(const struct drm_connector_state *conn_state);
 void intel_edp_fixup_vbt_bpp(struct intel_encoder *encoder, int pipe_bpp);
 void intel_dp_mst_suspend(struct intel_display *display);
 void intel_dp_mst_resume(struct intel_display *display);
+int intel_dp_rate_limit_len(const int *rates, int len, int max_rate);
 int intel_dp_max_source_lane_count(struct intel_digital_port *dig_port);
-int intel_dp_max_link_rate(struct intel_dp *intel_dp);
-int intel_dp_max_lane_count(struct intel_dp *intel_dp);
 int intel_dp_config_required_rate(const struct intel_crtc_state *crtc_state);
 int intel_dp_rate_select(struct intel_dp *intel_dp, int rate);
-int intel_dp_max_common_rate(struct intel_dp *intel_dp);
-int intel_dp_max_common_lane_count(struct intel_dp *intel_dp);
-int intel_dp_common_rate(struct intel_dp *intel_dp, int index);
 int intel_dp_rate_index(const int *rates, int len, int rate);
-int intel_dp_link_config_index(struct intel_dp *intel_dp, int link_rate, int lane_count);
-void intel_dp_link_config_get(struct intel_dp *intel_dp, int idx, int *link_rate, int *lane_count);
 void intel_dp_update_sink_caps(struct intel_dp *intel_dp);
 void intel_dp_reset_link_params(struct intel_dp *intel_dp);
 
@@ -149,6 +144,9 @@ int intel_dp_dsc_compute_max_bpp(const struct intel_connector *connector,
 				 u8 dsc_max_bpc);
 int intel_dp_compute_min_compressed_bpp_x16(struct intel_connector *connector,
 					    enum intel_output_format output_format);
+bool intel_dp_get_connector_max_link_config(struct intel_connector *connector,
+					    const struct link_config_limits *limits,
+					    struct intel_dp_link_config *max_link_config);
 bool intel_dp_mode_valid_with_dsc(struct intel_connector *connector,
 				  int link_clock, int lane_count,
 				  int mode_clock, int mode_hdisplay,
@@ -209,8 +207,6 @@ void intel_dp_get_dsc_sink_cap(u8 dpcd_rev,
 			       struct intel_connector *connector);
 bool intel_dp_has_gamut_metadata_dip(struct intel_encoder *encoder);
 
-bool intel_dp_link_params_valid(struct intel_dp *intel_dp, int link_rate,
-				u8 lane_count);
 bool intel_dp_has_connector(struct intel_dp *intel_dp,
 			    const struct drm_connector_state *conn_state);
 int intel_dp_dsc_max_src_input_bpc(struct intel_display *display);
@@ -243,4 +239,7 @@ bool intel_dp_joiner_candidate_valid(struct intel_connector *connector,
 
 u8 intel_dp_as_sdp_transmission_time(void);
 
+int intel_dp_link_init(struct intel_dp *intel_dp);
+void intel_dp_link_cleanup(struct intel_dp *intel_dp);
+
 #endif /* __INTEL_DP_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_caps.c b/drivers/gpu/drm/i915/display/intel_dp_link_caps.c
new file mode 100644
index 0000000000000..7b6cc6055da82
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_dp_link_caps.c
@@ -0,0 +1,1312 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <linux/bitops.h>
+#include <linux/debugfs.h>
+#include <linux/log2.h>
+#include <linux/seq_buf.h>
+#include <linux/slab.h>
+#include <linux/sort.h>
+#include <linux/string.h>
+#include <linux/types.h>
+
+#include <drm/drm_print.h>
+
+#include "intel_display_core.h"
+#include "intel_display_types.h"
+#include "intel_display_utils.h"
+#include "intel_dp.h"
+#include "intel_dp_link_caps.h"
+
+/**
+ * DOC: DisplayPort link capabilities
+ *
+ * The Intel DP link caps API tracks the supported and allowed
+ * DisplayPort link configurations for a DP encoder and its attached
+ * connectors, and provides helpers to iterate over the allowed
+ * configurations and constrain them by filtering, disabling, or
+ * limiting them to maximum link parameters.
+ *
+ * Locking
+ * -------
+ *
+ * All accesses to this API must be serialized. The only exception
+ * is intel_dp_link_caps_get_max_limits(), which allow lockless
+ * lookup. Such lookups may observe an out-of-sync &struct
+ * intel_dp_link_config tuple, i.e. a rate from one state and a lane
+ * count from another.
+ *
+ * The Intel i915/xe drivers ensure the above serialization by holding
+ * &drm_mode_config.connection_mutex and, while holding the lock,
+ * waiting for any pending asynchronous atomic commits. This also allows
+ * use of the API from the tails of asynchronous atomic commits, which
+ * cannot hold the lock.
+ *
+ * Iterating and restricting link configurations
+ * ---------------------------------------------
+ *
+ * The link configuration iterators can iterate the ``allowed
+ * configurations`` during modeset configuration selection or link
+ * training fallback handling in a configurable order.
+ *
+ * The iteration order can depend on connector type (eDP, DP SST,
+ * DP MST) and modeset-specific conditions or driver policies, such
+ * as DSC vs. non-DSC modes, power saving vs. better user experience,
+ * or policy changes after a link training failure.
+ *
+ * The configurations exposed via the iterators can be additionally
+ * constrained in the following ways:
+ *
+ * - Filtered for a given modeset based on modeset-specific conditions.
+ *   Examples for such conditions include driver policies preferring
+ *   power saving or better user experience, post-link training failure
+ *   preference changes, or sink automated test requests limiting the
+ *   usable configurations.
+ *
+ * - Disabled permanently for the connected sink. Examples of reasons
+ *   to disable a configuration include a link training failure for a
+ *   given configuration or a driver workaround preventing the use of
+ *   a particular configuration.
+ *
+ * - Limited via a maximum link rate and lane count. For example, after
+ *   a link training failure, subsequent modesets may be limited to
+ *   configurations at or below the failed parameters.
+ *
+ *   This mechanism exists for backward compatibility only. Eventually,
+ *   it will be removed in favor of relying solely on individually
+ *   disabled configurations, as described above.
+ *
+ * Terminology
+ * -----------
+ *
+ * ``Common link capabilities`` (or ``common caps``) refer to the link
+ * rates and maximum lane count supported by both the source and the
+ * sink, i.e. the intersection of their respective capabilities.
+ *
+ * ``Supported configurations`` are all configurations defined by the
+ * ``Common link capabilities``' link rates and maximum lane count.
+ *
+ * ``Disabled configurations`` are ``Supported configurations`` disabled
+ * via this API.
+ *
+ * ``Enabled configurations`` are ``Supported configurations`` that are
+ * not disabled.
+ *
+ * ``Forced configurations`` are ``Enabled configurations`` forced via
+ * forced link parameter debugfs entries.
+ *
+ * ``Allowed configurations`` are the ``Enabled configurations``, or if
+ * forcing is in effect the ``Forced configurations``, constrained by a
+ * maximum rate and lane count set via the API.
+ */
+struct intel_dp_link_caps {
+	struct intel_dp *dp;
+
+	/* Rate, lane count caps common to source and sink. */
+	int num_rates;
+	int rates[DP_MAX_SUPPORTED_RATES];
+	int max_lane_count;
+
+	/* common rate,lane_count configs in bw order */
+	int num_configs;
+#define INTEL_DP_MAX_LANE_COUNT			4
+#define INTEL_DP_MAX_SUPPORTED_LANE_CONFIGS	(ilog2(INTEL_DP_MAX_LANE_COUNT) + 1)
+#define INTEL_DP_LANE_COUNT_EXP_BITS		order_base_2(INTEL_DP_MAX_SUPPORTED_LANE_CONFIGS)
+#define INTEL_DP_LINK_RATE_IDX_BITS		(BITS_PER_TYPE(u8) - INTEL_DP_LANE_COUNT_EXP_BITS)
+#define INTEL_DP_MAX_LINK_CONFIGS		(DP_MAX_SUPPORTED_RATES * \
+						 INTEL_DP_MAX_SUPPORTED_LANE_CONFIGS)
+	struct intel_dp_link_config_entry {
+		/* index into rates[] */
+		u8 link_rate_idx:INTEL_DP_LINK_RATE_IDX_BITS;
+		u8 lane_count_exp:INTEL_DP_LANE_COUNT_EXP_BITS;
+	} configs[INTEL_DP_MAX_LINK_CONFIGS];
+
+	/*
+	 * Indices to intel_dp_link_caps::configs[] in rate/lane count,
+	 * lane_count/rate order.
+	 */
+	u8 rate_lane_map[INTEL_DP_MAX_LINK_CONFIGS];
+	u8 lane_rate_map[INTEL_DP_MAX_LINK_CONFIGS];
+
+	/*
+	 * Filter of configurations enabled for the current sink
+	 * connection.
+	 *
+	 * Each bit in the filter's configuration mask corresponds to a
+	 * configuration index in the intel_dp_link_caps::configs[] array.
+	 *
+	 * All configurations start out enabled in the filter after a
+	 * new sink is connected. Users disable configurations afterwards
+	 * via the link caps API. All configurations get re-enabled
+	 * internally in the following cases:
+	 * - when forcing a link rate or lane count
+	 * - when intel_dp_link_caps_update(reset=true) is called after
+	 *   a new sink is connected
+	 * - when intel_dp_link_caps_update(reset=false) with changed
+	 *   link capabilities is called
+	 * - when intel_dp_link_caps_reset() is called after a new sink
+	 *   is connected
+	 */
+	struct intel_dp_link_caps_filter enabled_configs;
+
+	/*
+	 * Allowed configurations are the supported configurations defined by
+	 * config_table.rates and config_table.max_lane_count, constrained by
+	 * config_table.enabled_configs and the forced_params and
+	 * max_limits values below.
+	 *
+	 * See get_allowed_config_filter() for the filter of these
+	 * configurations.
+	 */
+
+	/*
+	 * Forced parameters requested via debugfs. Remains set across sink
+	 * disconnects.
+	 */
+	struct intel_dp_link_config forced_params;
+
+	/*
+	 * User set maximum limits. These limits constrain the currently
+	 * allowed set of configurations and are not adjusted when sink
+	 * capabilities change.
+	 *
+	 * max_limits.rate/lane_count may come from different allowed
+	 * configurations, i.e. the (max_limits.rate, max_limits.lane_count)
+	 * tuple itself may not be an allowed configuration.
+	 */
+	struct intel_dp_link_config max_limits;
+};
+static_assert(BITS_PER_TYPE(((struct intel_dp_link_caps_filter *)NULL)->config_mask) >=
+	      ARRAY_SIZE(((struct intel_dp_link_caps *)NULL)->configs));
+
+static struct intel_dp_link_caps_order bw_desc_config_order(void)
+{
+	struct intel_dp_link_caps_order order = {
+		.key = INTEL_DP_LINK_CAPS_ORDER_KEY_BW,
+		.dir = INTEL_DP_LINK_CAPS_ORDER_DIR_DESC,
+	};
+
+	return order;
+}
+
+static enum intel_dp_link_caps_order_key
+connector_compute_order_key(bool is_mst)
+{
+	if (is_mst)
+		return INTEL_DP_LINK_CAPS_ORDER_KEY_BW;
+	else
+		return INTEL_DP_LINK_CAPS_ORDER_KEY_RATE_LANE;
+}
+
+static enum intel_dp_link_caps_order_key
+connector_fallback_order_key(bool is_mst)
+{
+	if (is_mst)
+		return INTEL_DP_LINK_CAPS_ORDER_KEY_BW;
+	else
+		return INTEL_DP_LINK_CAPS_ORDER_KEY_LANE_RATE;
+}
+
+static enum intel_dp_link_caps_order_direction
+connector_compute_order_dir(bool is_mst, bool use_max_params)
+{
+	if (is_mst || use_max_params)
+		return INTEL_DP_LINK_CAPS_ORDER_DIR_DESC;
+	else
+		return INTEL_DP_LINK_CAPS_ORDER_DIR_ASC;
+}
+
+struct intel_dp_link_caps_order
+intel_dp_link_caps_connector_compute_order(struct intel_connector *connector)
+{
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	struct intel_dp_link_caps_order order = {
+		.key = connector_compute_order_key(connector->mst.dp),
+		.dir = connector_compute_order_dir(connector->mst.dp, intel_dp->use_max_params)
+	};
+
+	return order;
+}
+
+struct intel_dp_link_caps_order
+intel_dp_link_caps_connector_fallback_order(bool is_mst)
+{
+	struct intel_dp_link_caps_order order = {
+		.key = connector_fallback_order_key(is_mst),
+		.dir = INTEL_DP_LINK_CAPS_ORDER_DIR_DESC,
+	};
+
+	return order;
+}
+
+/* Get length of common rates array potentially limited by max_rate. */
+static int intel_dp_common_len_rate_limit(struct intel_dp_link_caps *link_caps,
+					  int max_rate)
+{
+	return intel_dp_rate_limit_len(link_caps->rates,
+				       link_caps->num_rates, max_rate);
+}
+
+static int intel_dp_common_rate(struct intel_dp_link_caps *link_caps, int index)
+{
+	struct intel_display *display = to_intel_display(link_caps->dp);
+
+	if (drm_WARN_ON(display->drm,
+			index < 0 || index >= link_caps->num_rates))
+		return 162000;
+
+	return link_caps->rates[index];
+}
+
+/* Theoretical max between source and sink */
+static int intel_dp_max_common_rate(struct intel_dp_link_caps *link_caps)
+{
+	return intel_dp_common_rate(link_caps, link_caps->num_rates - 1);
+}
+
+void intel_dp_link_caps_print_common_rates(struct intel_dp_link_caps *link_caps)
+{
+	struct intel_display *display = to_intel_display(link_caps->dp);
+	DECLARE_SEQ_BUF(s, 128);
+	int i;
+
+	for (i = 0; i < link_caps->num_rates; i++)
+		seq_buf_printf(&s, "%s%d", i ? ", " : "", link_caps->rates[i]);
+
+	drm_dbg_kms(display->drm, "common rates: %s\n", seq_buf_str(&s));
+}
+
+static int intel_dp_link_caps_max_common_lane_count(struct intel_dp_link_caps *link_caps)
+{
+	return link_caps->max_lane_count;
+}
+
+static int forced_lane_count(struct intel_dp_link_caps *link_caps)
+{
+	if (!link_caps->forced_params.lane_count)
+		return 0;
+
+	return clamp(link_caps->forced_params.lane_count,
+		     1, intel_dp_link_caps_max_common_lane_count(link_caps));
+}
+
+static int forced_link_rate(struct intel_dp_link_caps *link_caps)
+{
+	int len;
+
+	if (!link_caps->forced_params.rate)
+		return 0;
+
+	len = intel_dp_common_len_rate_limit(link_caps, link_caps->forced_params.rate);
+	if (len == 0)
+		return intel_dp_common_rate(link_caps, 0);
+
+	return intel_dp_common_rate(link_caps, len - 1);
+}
+
+void intel_dp_link_caps_get_forced_params(struct intel_dp_link_caps *link_caps,
+					  struct intel_dp_link_config *forced_params)
+{
+	forced_params->rate = forced_link_rate(link_caps);
+	forced_params->lane_count = forced_lane_count(link_caps);
+}
+
+static int intel_dp_link_config_rate(struct intel_dp_link_caps *link_caps,
+				     const struct intel_dp_link_config_entry *lce)
+{
+	return intel_dp_common_rate(link_caps, lce->link_rate_idx);
+}
+
+static int intel_dp_link_config_lane_count(const struct intel_dp_link_config_entry *lce)
+{
+	return 1 << lce->lane_count_exp;
+}
+
+static void
+to_intel_dp_link_config(struct intel_dp_link_caps *link_caps,
+			int config_idx, struct intel_dp_link_config *config)
+{
+	const struct intel_dp_link_config_entry *lce = &link_caps->configs[config_idx];
+
+	config->rate = intel_dp_link_config_rate(link_caps, lce);
+	config->lane_count = intel_dp_link_config_lane_count(lce);
+}
+
+static int
+iter_pos_to_idx(struct intel_dp_link_caps *link_caps,
+		struct intel_dp_link_caps_order config_order,
+		int iter_pos)
+{
+	int config_idx;
+
+	if (!in_range(iter_pos, 0, link_caps->num_configs))
+		return -1;
+
+	switch (config_order.dir) {
+	case INTEL_DP_LINK_CAPS_ORDER_DIR_ASC:
+		break;
+	case INTEL_DP_LINK_CAPS_ORDER_DIR_DESC:
+		iter_pos = link_caps->num_configs - 1 - iter_pos;
+
+		break;
+	default:
+		MISSING_CASE(config_order.dir);
+
+		return -1;
+	}
+
+	switch (config_order.key) {
+	case INTEL_DP_LINK_CAPS_ORDER_KEY_BW:
+		config_idx = iter_pos;
+
+		break;
+	case INTEL_DP_LINK_CAPS_ORDER_KEY_RATE_LANE:
+		config_idx = link_caps->rate_lane_map[iter_pos];
+
+		break;
+	case INTEL_DP_LINK_CAPS_ORDER_KEY_LANE_RATE:
+		config_idx = link_caps->lane_rate_map[iter_pos];
+
+		break;
+	default:
+		MISSING_CASE(config_order.key);
+
+		return -1;
+	}
+
+	return config_idx;
+}
+
+static bool iter_get_next_config(struct intel_dp_link_caps_iter *iter,
+				 struct intel_dp_link_config *config)
+{
+	while (true) {
+		int config_idx;
+
+		iter->pos++;
+
+		config_idx = iter_pos_to_idx(iter->link_caps, iter->order, iter->pos);
+		if (config_idx < 0) {
+			iter->pos = -1;
+			*config = INTEL_DP_LINK_CONFIG_NULL;
+
+			break;
+		}
+
+		if (!(BIT(config_idx) & iter->filter.config_mask))
+			continue;
+
+		to_intel_dp_link_config(iter->link_caps, config_idx, config);
+
+		break;
+	}
+
+	return iter->pos >= 0;
+}
+
+static void iter_start(struct intel_dp_link_caps_iter *iter,
+		       struct intel_dp_link_caps *link_caps,
+		       struct intel_dp_link_caps_order order,
+		       struct intel_dp_link_caps_filter filter)
+{
+	iter->link_caps = link_caps;
+	iter->pos = -1;
+	iter->order = order;
+	iter->filter = filter;
+
+	iter->get_next_config = iter_get_next_config;
+}
+
+static struct intel_dp_link_caps_filter
+calc_allowed_config_filter(struct intel_dp_link_caps *link_caps,
+			   struct intel_dp_link_caps_filter enabled_configs,
+			   const struct intel_dp_link_config *max_limits,
+			   const struct intel_dp_link_config *forced_params)
+{
+	struct intel_dp_link_caps_filter allowed_configs = INTEL_DP_LINK_CAPS_FILTER_NONE;
+	struct intel_dp_link_caps_order order = bw_desc_config_order();
+	struct intel_dp_link_caps_iter iter;
+	struct intel_dp_link_config config;
+
+	iter_start(&iter, link_caps, order, enabled_configs);
+	for_each_dp_link_config(&iter, &config) {
+		if (forced_params->rate &&
+		    forced_params->rate != config.rate)
+			continue;
+
+		if (forced_params->lane_count &&
+		    forced_params->lane_count != config.lane_count)
+			continue;
+
+		if (config.rate > max_limits->rate)
+			continue;
+
+		if (config.lane_count > max_limits->lane_count)
+			continue;
+
+		allowed_configs.config_mask |= BIT(iter_pos_to_idx(link_caps, order, iter.pos));
+	}
+	intel_dp_link_caps_iter_end(&iter);
+
+	return allowed_configs;
+}
+
+/*
+ * get_allowed_config_filter - get filter for the currently allowed configs
+ * @link_caps: link capabilities state
+ *
+ * Return:
+ * Filter of link configurations allowed after applying the current
+ * maximum link limits, and further narrowing them by removing any disabled
+ * configuration and limiting to forced link parameters.
+ *
+ * See also:
+ * - intel_dp_link_caps_get_max_limits()
+ * - intel_dp_link_caps_get_forced_params()
+ */
+static struct intel_dp_link_caps_filter
+get_allowed_config_filter(struct intel_dp_link_caps *link_caps)
+{
+	struct intel_dp_link_config forced_params;
+
+	intel_dp_link_caps_get_forced_params(link_caps, &forced_params);
+
+	return calc_allowed_config_filter(link_caps, link_caps->enabled_configs,
+					  &link_caps->max_limits, &forced_params);
+}
+
+void intel_dp_link_caps_iter_start(struct intel_dp_link_caps_iter *iter,
+				   struct intel_dp_link_caps *link_caps,
+				   struct intel_dp_link_caps_order order,
+				   struct intel_dp_link_caps_filter filter)
+{
+	filter.config_mask &= get_allowed_config_filter(link_caps).config_mask;
+
+	iter_start(iter, link_caps, order, filter);
+}
+
+void intel_dp_link_caps_iter_end(struct intel_dp_link_caps_iter *iter)
+{
+	memset(iter, 0, sizeof(*iter));
+}
+
+/**
+ * intel_dp_link_caps_get_max_config - get the maximum config in a given order
+ * @link_caps: link capabilities state
+ * @order_key: ordering key used to rank candidate configurations
+ * @filter: filter for candidate configurations
+ * @max_config: returned maximum link configuration
+ *
+ * Find the last configuration among the currently allowed
+ * configurations filtered by @filter in the iteration order
+ * selected by @order_key, and store it in @max_config.
+ *
+ * See also:
+ * - &enum intel_dp_link_caps_order_key
+ *
+ * Returns:
+ * %true if a maximum config is returned
+ * %false otherwise.
+ */
+bool intel_dp_link_caps_get_max_config(struct intel_dp_link_caps *link_caps,
+				       enum intel_dp_link_caps_order_key order_key,
+				       struct intel_dp_link_caps_filter filter,
+				       struct intel_dp_link_config *max_config)
+{
+	struct intel_dp_link_caps_order order = {
+		.key = order_key,
+		.dir = INTEL_DP_LINK_CAPS_ORDER_DIR_DESC
+	};
+	struct intel_dp_link_config iter_config;
+	struct intel_dp_link_caps_iter iter;
+	bool found = false;
+
+	intel_dp_link_caps_iter_start(&iter, link_caps, order, filter);
+	for_each_dp_link_config(&iter, &iter_config) {
+		found = true;
+		break;
+	}
+	intel_dp_link_caps_iter_end(&iter);
+
+	if (!found)
+		return false;
+
+	*max_config = iter_config;
+
+	return true;
+}
+
+/**
+ * intel_dp_link_caps_get_max_bw_config - get maximum BW link configuration
+ * @link_caps: link capabilities state
+ * @max_config: returned maximum link configuration
+ *
+ * Return the maximum BW link configuration among the currently
+ * allowed configurations.
+ */
+void intel_dp_link_caps_get_max_bw_config(struct intel_dp_link_caps *link_caps,
+					  struct intel_dp_link_config *max_config)
+{
+	if (!intel_dp_link_caps_get_max_config(link_caps,
+					       bw_desc_config_order().key, INTEL_DP_LINK_CAPS_FILTER_ALL,
+					       max_config))
+		*max_config = INTEL_DP_LINK_CONFIG_NULL;
+}
+
+static int find_config_idx(struct intel_dp_link_caps *link_caps,
+			   struct intel_dp_link_caps_filter filter,
+			   const struct intel_dp_link_config *link_config)
+{
+	struct intel_dp_link_caps_order order = bw_desc_config_order();
+	struct intel_dp_link_config iter_config;
+	struct intel_dp_link_caps_iter iter;
+	int pos = -1;
+
+	intel_dp_link_caps_iter_start(&iter, link_caps, order, filter);
+	for_each_dp_link_config(&iter, &iter_config) {
+		if (iter_config.rate == link_config->rate &&
+		    iter_config.lane_count == link_config->lane_count) {
+			pos = iter.pos;
+
+			break;
+		}
+	}
+	intel_dp_link_caps_iter_end(&iter);
+
+	if (pos < 0)
+		return pos;
+
+	return iter_pos_to_idx(link_caps, order, pos);
+}
+
+bool intel_dp_link_caps_filter_add(struct intel_dp_link_caps *link_caps,
+				   struct intel_dp_link_caps_filter *filter,
+				   const struct intel_dp_link_config *config)
+{
+	int idx;
+
+	idx = find_config_idx(link_caps, get_allowed_config_filter(link_caps), config);
+	if (idx < 0)
+		return false;
+
+	filter->config_mask |= BIT(idx);
+
+	return true;
+}
+
+static bool intel_dp_link_caps_filter_remove(struct intel_dp_link_caps *link_caps,
+					     struct intel_dp_link_caps_filter *filter,
+					     const struct intel_dp_link_config *config)
+{
+	int idx;
+
+	idx = find_config_idx(link_caps, get_allowed_config_filter(link_caps), config);
+	if (idx < 0)
+		return false;
+
+	filter->config_mask &= ~BIT(idx);
+
+	return true;
+}
+
+static void set_max_link_limits(struct intel_dp_link_caps *link_caps,
+				const struct intel_dp_link_config *max_link_limits)
+{
+	link_caps->max_limits = *max_link_limits;
+}
+
+static void reset_max_link_limits(struct intel_dp_link_caps *link_caps)
+{
+	struct intel_dp_link_config max_link_limits = {
+		.rate = intel_dp_max_common_rate(link_caps),
+		.lane_count = intel_dp_link_caps_max_common_lane_count(link_caps),
+	};
+
+	set_max_link_limits(link_caps, &max_link_limits);
+}
+
+static void reset_max_link_limits_reenable_all(struct intel_dp_link_caps *link_caps)
+{
+	link_caps->enabled_configs = INTEL_DP_LINK_CAPS_FILTER_ALL;
+	reset_max_link_limits(link_caps);
+}
+
+/**
+ * intel_dp_link_caps_disable_config - disable a configuration
+ * @link_caps: link capabilities state
+ * @config: configuration to disable
+ *
+ * Disable the configuration identified by @config. This removes the
+ * configuration from the set of allowed configurations. The disabling
+ * shouldn't leave the remaining configuration set empty.
+ *
+ * The configuration remains disallowed until intel_dp_link_caps() with
+ * reset=%true or changed sink capabilities is called, or
+ * intel_dp_link_caps_reset() is called. Each of these happens after a
+ * new sink is connected or the currently connected sink changes its
+ * capabilities.
+ *
+ * Return:
+ * - %true  if @config was valid and the derived state was updated.
+ * - %false if @config was invalid or the remaining configuration set
+ *   would remain empty.
+ */
+bool intel_dp_link_caps_disable_config(struct intel_dp_link_caps *link_caps,
+				       const struct intel_dp_link_config *config)
+{
+	struct intel_dp_link_caps_filter enabled_configs = link_caps->enabled_configs;
+	struct intel_dp_link_config forced_params;
+
+	if (!intel_dp_link_caps_filter_remove(link_caps, &enabled_configs, config))
+		return false;
+
+	intel_dp_link_caps_get_forced_params(link_caps, &forced_params);
+
+	if (!calc_allowed_config_filter(link_caps, enabled_configs,
+					&link_caps->max_limits, &forced_params).config_mask)
+		return false;
+
+	link_caps->enabled_configs = enabled_configs;
+
+	return true;
+}
+
+/**
+ * intel_dp_link_caps_get_max_limits - get the current maximum link limits
+ * @link_caps: link capabilities state
+ * @max_link_limits: returned maximum link limits
+ *
+ * Return the current maximum rate and lane count limits in
+ * @max_link_limits.
+ *
+ * These limits constrain the set of allowed configurations.
+ *
+ * The limits are set to the maximum common supported values after
+ * intel_dp_link_caps_reset() is called, and can later be modified by
+ * intel_dp_link_caps_set_max_limits(). The max rate and lane count
+ * parameters are independent limits, so the pair does not necessarily
+ * define a valid configuration.
+ *
+ * This function may be called without serializing against updates to
+ * @link_caps. However, without such serialization the returned value may be
+ * an out-of-sync (link rate, lane count) tuple, i.e. the parameters may
+ * belong to different update snapshots in time.
+ */
+void intel_dp_link_caps_get_max_limits(struct intel_dp_link_caps *link_caps,
+				       struct intel_dp_link_config *max_link_limits)
+{
+	*max_link_limits = link_caps->max_limits;
+}
+
+static bool max_link_limits_valid(struct intel_dp_link_caps *link_caps,
+				  const struct intel_dp_link_config *max_link_limits)
+{
+	struct intel_dp_link_caps_filter allowed_configs;
+	struct intel_dp_link_config forced_params;
+
+	if (max_link_limits->lane_count > INTEL_DP_MAX_LANE_COUNT ||
+	    !is_power_of_2(max_link_limits->lane_count))
+		return false;
+
+	/* TODO: Validate max_link_limits->rate against the source supported rates. */
+
+	intel_dp_link_caps_get_forced_params(link_caps, &forced_params);
+	allowed_configs = calc_allowed_config_filter(link_caps, link_caps->enabled_configs,
+						     max_link_limits, &forced_params);
+
+	return allowed_configs.config_mask != 0;
+}
+
+/**
+ * intel_dp_link_caps_set_max_limits - set the current maximum link limits
+ * @link_caps: link capabilities state
+ * @max_link_limits: new maximum link limits
+ *
+ * Set the current maximum rate and lane count limits to @max_link_limits,
+ * constraining the set of allowed configurations.
+ *
+ * The new limits must leave at least one configuration allowed: the limits
+ * must not be below the currently active forced parameters or below all the
+ * configurations that remain after disabled configurations are excluded.
+ *
+ * Unlike intel_dp_link_caps_get_max_limits(), the caller must serialize
+ * this call against concurrent queries and updates to @link_caps, in line
+ * with the rest of the API.
+ *
+ * Return:
+ * - %true  if the @link_caps cached max limits value got updated with
+ *          @max_link_limits.
+ * - %false if @max_link_limits is invalid.
+ */
+bool intel_dp_link_caps_set_max_limits(struct intel_dp_link_caps *link_caps,
+				       const struct intel_dp_link_config *max_link_limits)
+{
+	if (!max_link_limits_valid(link_caps, max_link_limits))
+		return false;
+
+	set_max_link_limits(link_caps, max_link_limits);
+
+	return true;
+}
+
+/**
+ * intel_dp_link_caps_reset_max_limits - reset the current maximum link limits
+ * @link_caps: link capabilities state
+ *
+ * Reset the current maximum link limits to the maximum supported common link
+ * rate and lane count.
+ */
+void intel_dp_link_caps_reset_max_limits(struct intel_dp_link_caps *link_caps)
+{
+	reset_max_link_limits(link_caps);
+}
+
+static int intel_dp_link_config_bw(struct intel_dp_link_caps *link_caps,
+				   const struct intel_dp_link_config_entry *lce)
+{
+	return drm_dp_max_dprx_data_rate(intel_dp_link_config_rate(link_caps, lce),
+					 intel_dp_link_config_lane_count(lce));
+}
+
+static int link_config_cmp_by_bw(const void *a, const void *b, const void *p)
+{
+	struct intel_dp *intel_dp = (struct intel_dp *)p;	/* remove const */
+	struct intel_dp_link_caps *link_caps = intel_dp->link.caps;
+
+	const struct intel_dp_link_config_entry *lce_a = a;
+	const struct intel_dp_link_config_entry *lce_b = b;
+	int bw_a = intel_dp_link_config_bw(link_caps, lce_a);
+	int bw_b = intel_dp_link_config_bw(link_caps, lce_b);
+
+	if (bw_a != bw_b)
+		return bw_a - bw_b;
+
+	return intel_dp_link_config_rate(link_caps, lce_a) -
+	       intel_dp_link_config_rate(link_caps, lce_b);
+}
+
+static int link_config_cmp_by_rate_lane(const void *a, const void *b, const void *p)
+{
+	const struct intel_dp_link_caps *link_caps = p;
+	u8 *lce_a_idx = (u8 *)a;
+	u8 *lce_b_idx = (u8 *)b;
+	const struct intel_dp_link_config_entry *lce_a = &link_caps->configs[*lce_a_idx];
+	const struct intel_dp_link_config_entry *lce_b = &link_caps->configs[*lce_b_idx];
+
+	if (lce_a->link_rate_idx != lce_b->link_rate_idx)
+		return lce_a->link_rate_idx - lce_b->link_rate_idx;
+
+	return lce_a->lane_count_exp - lce_b->lane_count_exp;
+}
+
+static int link_config_cmp_by_lane_rate(const void *a, const void *b, const void *p)
+{
+	const struct intel_dp_link_caps *link_caps = p;
+	u8 *lce_a_idx = (u8 *)a;
+	u8 *lce_b_idx = (u8 *)b;
+	const struct intel_dp_link_config_entry *lce_a = &link_caps->configs[*lce_a_idx];
+	const struct intel_dp_link_config_entry *lce_b = &link_caps->configs[*lce_b_idx];
+
+	if (lce_a->lane_count_exp != lce_b->lane_count_exp)
+		return lce_a->lane_count_exp - lce_b->lane_count_exp;
+
+	return lce_a->link_rate_idx - lce_b->link_rate_idx;
+}
+
+/**
+ * intel_dp_link_caps_update - rebuild the supported link configuration state
+ * @link_caps: link capabilities state
+ * @rates: supported common link rates
+ * @num_rates: number of entries in @rates
+ * @max_lane_count: supported maximum lane count
+ * @reset: reset limits and disabled configs
+ *
+ * Rebuild the supported link configuration state from @rates and
+ * @max_lane_count.
+ *
+ * If @reset is %true, reset the maximum link limits to the maximum
+ * supported rate and lane count, and re-enable all configurations.
+ *
+ * This function is called regularly, at least after a sink is connected,
+ * but it may also be called later whenever the sink capabilities may have
+ * changed, for example in response to HPD IRQ / RX_CAP_CHANGED signaling.
+ *
+ * In the Intel driver this function is currently called whenever the
+ * connector detect handler runs, after reading the sink capabilities. This
+ * may change if those capabilities are cached until the sink is
+ * disconnected, or until RX_CAP_CHANGED is signaled. In any case, this
+ * function should be called whenever the sink capabilities were read out
+ * and may have changed.
+ *
+ * Returns:
+ * - %true if the link capabilities have changed, %false otherwise.
+ */
+bool intel_dp_link_caps_update(struct intel_dp_link_caps *link_caps,
+			       const int *rates, int num_rates, int max_lane_count,
+			       bool reset)
+{
+	struct intel_dp *intel_dp = link_caps->dp;
+	struct intel_display *display = to_intel_display(intel_dp);
+	struct intel_dp_link_config_entry *lce;
+	bool link_params_changed = reset;
+	int num_common_lane_configs;
+	int i;
+	int j;
+
+	if (drm_WARN_ON(display->drm, !is_power_of_2(max_lane_count)))
+		return false;
+
+	if (drm_WARN_ON(display->drm, num_rates > ARRAY_SIZE(link_caps->rates)))
+		return false;
+
+	num_common_lane_configs = ilog2(max_lane_count) + 1;
+
+	if (drm_WARN_ON(display->drm, num_rates * num_common_lane_configs >
+				    ARRAY_SIZE(link_caps->configs)))
+		return false;
+
+	/* TODO: Add a struct containing both rates and number of rates. */
+	static_assert(__same_type(rates[0], link_caps->rates[0]));
+	if (num_rates != link_caps->num_rates ||
+	    memcmp(rates, link_caps->rates, num_rates * sizeof(rates[0])))
+		link_params_changed = true;
+
+	if (max_lane_count != link_caps->max_lane_count)
+		link_params_changed = true;
+
+	memcpy(link_caps->rates, rates, num_rates * sizeof(rates[0]));
+	link_caps->num_rates = num_rates;
+	link_caps->max_lane_count = max_lane_count;
+
+	link_caps->num_configs = num_rates * num_common_lane_configs;
+
+	lce = &link_caps->configs[0];
+	for (i = 0; i < link_caps->num_rates; i++) {
+		for (j = 0; j < num_common_lane_configs; j++) {
+			lce->lane_count_exp = j;
+			lce->link_rate_idx = i;
+
+			lce++;
+		}
+	}
+
+	sort_r(link_caps->configs, link_caps->num_configs,
+	       sizeof(link_caps->configs[0]),
+	       link_config_cmp_by_bw, NULL,
+	       intel_dp);
+
+	for (i = 0; i < link_caps->num_configs; i++) {
+		link_caps->rate_lane_map[i] = i;
+		link_caps->lane_rate_map[i] = i;
+	}
+
+	sort_r(link_caps->rate_lane_map, link_caps->num_configs,
+	       sizeof(link_caps->rate_lane_map[0]),
+	       link_config_cmp_by_rate_lane, NULL,
+	       link_caps);
+
+	sort_r(link_caps->lane_rate_map, link_caps->num_configs,
+	       sizeof(link_caps->lane_rate_map[0]),
+	       link_config_cmp_by_lane_rate, NULL,
+	       link_caps);
+
+	if (link_params_changed)
+		reset_max_link_limits_reenable_all(link_caps);
+
+	return link_params_changed;
+}
+
+/**
+ * intel_dp_link_caps_reset - reset link capability restrictions
+ * @link_caps: link capabilities state
+ *
+ * Reset all current restrictions except for the user requested forced
+ * parameters, thus updating the set of allowed configurations and the
+ * derived maximum link information accordingly.
+ *
+ * This function is regularly called after a sink is connected, either
+ * for the first time to the connector or after a previous sink was
+ * disconnected from it, and intel_dp_link_caps_update() was called.
+ */
+void intel_dp_link_caps_reset(struct intel_dp_link_caps *link_caps)
+{
+	reset_max_link_limits_reenable_all(link_caps);
+}
+
+static int i915_dp_force_link_rate_show(struct seq_file *m, void *data)
+{
+	struct intel_connector *connector = to_intel_connector(m->private);
+	struct intel_display *display = to_intel_display(connector);
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	struct intel_dp_link_caps *link_caps = intel_dp->link.caps;
+	int current_rate = -1;
+	int force_rate;
+	int err;
+	int i;
+
+	err = drm_modeset_lock_single_interruptible(&display->drm->mode_config.connection_mutex);
+	if (err)
+		return err;
+
+	intel_dp_flush_connector_commits(connector);
+
+	if (intel_dp->link.active)
+		current_rate = intel_dp->link_rate;
+
+	force_rate = link_caps->forced_params.rate;
+
+	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
+
+	seq_printf(m, "%sauto%s",
+		   force_rate == 0 ? "[" : "",
+		   force_rate == 0 ? "]" : "");
+
+	for (i = 0; i < intel_dp->num_source_rates; i++)
+		seq_printf(m, " %s%d%s%s",
+			   intel_dp->source_rates[i] == force_rate ? "[" : "",
+			   intel_dp->source_rates[i],
+			   intel_dp->source_rates[i] == current_rate ? "*" : "",
+			   intel_dp->source_rates[i] == force_rate ? "]" : "");
+
+	seq_putc(m, '\n');
+
+	return 0;
+}
+
+static int parse_link_rate(struct intel_dp_link_caps *link_caps, const char __user *ubuf, size_t len)
+{
+	struct intel_dp *intel_dp = link_caps->dp;
+	char *kbuf;
+	const char *p;
+	int rate;
+	int ret = 0;
+
+	kbuf = memdup_user_nul(ubuf, len);
+	if (IS_ERR(kbuf))
+		return PTR_ERR(kbuf);
+
+	p = strim(kbuf);
+
+	if (!strcmp(p, "auto")) {
+		rate = 0;
+	} else {
+		ret = kstrtoint(p, 0, &rate);
+		if (ret < 0)
+			goto out_free;
+
+		if (intel_dp_rate_index(intel_dp->source_rates,
+					intel_dp->num_source_rates,
+					rate) < 0)
+			ret = -EINVAL;
+	}
+
+out_free:
+	kfree(kbuf);
+
+	return ret < 0 ? ret : rate;
+}
+
+static ssize_t i915_dp_force_link_rate_write(struct file *file,
+					     const char __user *ubuf,
+					     size_t len, loff_t *offp)
+{
+	struct seq_file *m = file->private_data;
+	struct intel_connector *connector = to_intel_connector(m->private);
+	struct intel_display *display = to_intel_display(connector);
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	struct intel_dp_link_caps *link_caps = intel_dp->link.caps;
+	int rate;
+	int err;
+
+	rate = parse_link_rate(link_caps, ubuf, len);
+	if (rate < 0)
+		return rate;
+
+	err = drm_modeset_lock_single_interruptible(&display->drm->mode_config.connection_mutex);
+	if (err)
+		return err;
+
+	intel_dp_flush_connector_commits(connector);
+
+	intel_dp_reset_link_params(intel_dp);
+	link_caps->forced_params.rate = rate;
+
+	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
+
+	*offp += len;
+
+	return len;
+}
+DEFINE_SHOW_STORE_ATTRIBUTE(i915_dp_force_link_rate);
+
+static int i915_dp_force_lane_count_show(struct seq_file *m, void *data)
+{
+	struct intel_connector *connector = to_intel_connector(m->private);
+	struct intel_display *display = to_intel_display(connector);
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	struct intel_dp_link_caps *link_caps = intel_dp->link.caps;
+	int current_lane_count = -1;
+	int force_lane_count;
+	int err;
+	int i;
+
+	err = drm_modeset_lock_single_interruptible(&display->drm->mode_config.connection_mutex);
+	if (err)
+		return err;
+
+	intel_dp_flush_connector_commits(connector);
+
+	if (intel_dp->link.active)
+		current_lane_count = intel_dp->lane_count;
+	force_lane_count = link_caps->forced_params.lane_count;
+
+	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
+
+	seq_printf(m, "%sauto%s",
+		   force_lane_count == 0 ? "[" : "",
+		   force_lane_count == 0 ? "]" : "");
+
+	for (i = 1; i <= 4; i <<= 1)
+		seq_printf(m, " %s%d%s%s",
+			   i == force_lane_count ? "[" : "",
+			   i,
+			   i == current_lane_count ? "*" : "",
+			   i == force_lane_count ? "]" : "");
+
+	seq_putc(m, '\n');
+
+	return 0;
+}
+
+static int parse_lane_count(const char __user *ubuf, size_t len)
+{
+	char *kbuf;
+	const char *p;
+	int lane_count;
+	int ret = 0;
+
+	kbuf = memdup_user_nul(ubuf, len);
+	if (IS_ERR(kbuf))
+		return PTR_ERR(kbuf);
+
+	p = strim(kbuf);
+
+	if (!strcmp(p, "auto")) {
+		lane_count = 0;
+	} else {
+		ret = kstrtoint(p, 0, &lane_count);
+		if (ret < 0)
+			goto out_free;
+
+		switch (lane_count) {
+		case 1:
+		case 2:
+		case 4:
+			break;
+		default:
+			ret = -EINVAL;
+		}
+	}
+
+out_free:
+	kfree(kbuf);
+
+	return ret < 0 ? ret : lane_count;
+}
+
+static ssize_t i915_dp_force_lane_count_write(struct file *file,
+					      const char __user *ubuf,
+					      size_t len, loff_t *offp)
+{
+	struct seq_file *m = file->private_data;
+	struct intel_connector *connector = to_intel_connector(m->private);
+	struct intel_display *display = to_intel_display(connector);
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	struct intel_dp_link_caps *link_caps = intel_dp->link.caps;
+	int lane_count;
+	int err;
+
+	lane_count = parse_lane_count(ubuf, len);
+	if (lane_count < 0)
+		return lane_count;
+
+	err = drm_modeset_lock_single_interruptible(&display->drm->mode_config.connection_mutex);
+	if (err)
+		return err;
+
+	intel_dp_flush_connector_commits(connector);
+
+	intel_dp_reset_link_params(intel_dp);
+	link_caps->forced_params.lane_count = lane_count;
+
+	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
+
+	*offp += len;
+
+	return len;
+}
+DEFINE_SHOW_STORE_ATTRIBUTE(i915_dp_force_lane_count);
+
+static int i915_dp_max_link_rate_show(void *data, u64 *val)
+{
+	struct intel_connector *connector = to_intel_connector(data);
+	struct intel_display *display = to_intel_display(connector);
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	struct intel_dp_link_config max_link_limits;
+	int err;
+
+	err = drm_modeset_lock_single_interruptible(&display->drm->mode_config.connection_mutex);
+	if (err)
+		return err;
+
+	intel_dp_flush_connector_commits(connector);
+
+	intel_dp_link_caps_get_max_limits(intel_dp->link.caps, &max_link_limits);
+	*val = max_link_limits.rate;
+
+	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
+
+	return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(i915_dp_max_link_rate_fops, i915_dp_max_link_rate_show, NULL, "%llu\n");
+
+static int i915_dp_max_lane_count_show(void *data, u64 *val)
+{
+	struct intel_connector *connector = to_intel_connector(data);
+	struct intel_display *display = to_intel_display(connector);
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	struct intel_dp_link_config max_link_limits;
+	int err;
+
+	err = drm_modeset_lock_single_interruptible(&display->drm->mode_config.connection_mutex);
+	if (err)
+		return err;
+
+	intel_dp_flush_connector_commits(connector);
+
+	intel_dp_link_caps_get_max_limits(intel_dp->link.caps, &max_link_limits);
+	*val = max_link_limits.lane_count;
+
+	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
+
+	return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(i915_dp_max_lane_count_fops, i915_dp_max_lane_count_show, NULL, "%llu\n");
+
+static int intel_dp_allowed_link_configs_show(struct seq_file *m, void *data)
+{
+	struct intel_connector *connector = to_intel_connector(m->private);
+	struct intel_display *display = to_intel_display(connector);
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	struct intel_dp_link_caps *link_caps = intel_dp->link.caps;
+	struct intel_dp_link_config link_config;
+	struct intel_dp_link_caps_iter iter;
+	int err;
+	int i;
+
+	err = drm_modeset_lock_single_interruptible(&display->drm->mode_config.connection_mutex);
+	if (err)
+		return err;
+
+	intel_dp_flush_connector_commits(connector);
+
+	i = 0;
+	intel_dp_link_caps_iter_start(&iter,
+				      link_caps,
+				      intel_dp_link_caps_connector_compute_order(connector),
+				      INTEL_DP_LINK_CAPS_FILTER_ALL);
+	for_each_dp_link_config(&iter, &link_config) {
+		seq_printf(m, "%s%dx%d",
+			   i ? " " : "",
+			   link_config.lane_count, link_config.rate);
+		i++;
+	}
+	intel_dp_link_caps_iter_end(&iter);
+
+	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
+
+	seq_putc(m, '\n');
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(intel_dp_allowed_link_configs);
+
+/**
+ * intel_dp_link_caps_debugfs_add - add link caps debugfs files for a connector
+ * @connector: connector to add the debugfs files for
+ *
+ * Add the link-capability debugfs files for a DP @connector.
+ */
+void intel_dp_link_caps_debugfs_add(struct intel_connector *connector)
+{
+	struct dentry *root = connector->base.debugfs_entry;
+
+	if (connector->base.connector_type != DRM_MODE_CONNECTOR_DisplayPort &&
+	    connector->base.connector_type != DRM_MODE_CONNECTOR_eDP)
+		return;
+
+	debugfs_create_file("i915_dp_force_link_rate", 0644, root,
+			    connector, &i915_dp_force_link_rate_fops);
+
+	debugfs_create_file("i915_dp_force_lane_count", 0644, root,
+			    connector, &i915_dp_force_lane_count_fops);
+
+	debugfs_create_file("i915_dp_max_link_rate", 0444, root,
+			    connector, &i915_dp_max_link_rate_fops);
+
+	debugfs_create_file("i915_dp_max_lane_count", 0444, root,
+			    connector, &i915_dp_max_lane_count_fops);
+
+	debugfs_create_file("intel_dp_allowed_link_configs", 0444, root,
+			    connector, &intel_dp_allowed_link_configs_fops);
+}
+
+struct intel_dp_link_caps *intel_dp_link_caps_init(struct intel_dp *intel_dp)
+{
+	struct intel_dp_link_caps *link_caps;
+
+	link_caps = kzalloc_obj(*link_caps);
+	if (!link_caps)
+		return NULL;
+
+	link_caps->dp = intel_dp;
+	link_caps->enabled_configs = INTEL_DP_LINK_CAPS_FILTER_ALL;
+
+	return link_caps;
+}
+
+void intel_dp_link_caps_cleanup(struct intel_dp_link_caps *link_caps)
+{
+	kfree(link_caps);
+}
+
+#if IS_ENABLED(CONFIG_KUNIT)
+
+#define __INIT_MEMBER(__name, __fn) \
+	.__name = __fn,
+
+#define INTEL_DP_LINK_CAPS_TEST_OPS_INIT \
+	INTEL_DP_LINK_CAPS_TEST_OPS_MEMBERS(__INIT_MEMBER)
+
+#ifdef I915
+
+const struct intel_dp_link_caps_test_ops i915_display_dp_link_caps_test_ops = {
+	INTEL_DP_LINK_CAPS_TEST_OPS_INIT
+};
+EXPORT_SYMBOL(i915_display_dp_link_caps_test_ops);
+
+#else
+
+const struct intel_dp_link_caps_test_ops intel_display_dp_link_caps_test_ops = {
+	INTEL_DP_LINK_CAPS_TEST_OPS_INIT
+};
+EXPORT_SYMBOL(intel_display_dp_link_caps_test_ops);
+
+#endif	/* I915 */
+
+#undef INTEL_DP_LINK_CAPS_TEST_OPS_INIT
+#undef __INIT_MEMBER
+
+#endif	/* CONFIG_KUNIT */
diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_caps.h b/drivers/gpu/drm/i915/display/intel_dp_link_caps.h
new file mode 100644
index 0000000000000..a0a88efb95463
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_dp_link_caps.h
@@ -0,0 +1,195 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef __INTEL_DP_LINK_CAPS_H__
+#define __INTEL_DP_LINK_CAPS_H__
+
+#include <linux/bitops.h>
+#include <linux/types.h>
+
+struct intel_connector;
+struct intel_dp;
+struct intel_dp_link_caps;
+struct intel_dp_link_config;
+
+/**
+ * enum intel_dp_link_caps_order_key - key used to order configurations
+ * @INTEL_DP_LINK_CAPS_ORDER_KEY_BW:
+ *   Order configurations by bandwidth, then by link rate.
+ * @INTEL_DP_LINK_CAPS_ORDER_KEY_RATE_LANE:
+ *   Order configurations by link rate, then by lane count.
+ * @INTEL_DP_LINK_CAPS_ORDER_KEY_LANE_RATE:
+ *   Order configurations by lane count, then by link rate.
+ * @INTEL_DP_LINK_CAPS_ORDER_KEY_NUM:
+ *   Number of ordering keys.
+ *
+ * Selects how a caller wants the configuration table to be ordered,
+ * together with an &enum intel_dp_link_caps_order_direction, for
+ * iteration queries.
+ *
+ * See also:
+ *  - &struct intel_dp_link_caps_order
+ *  - intel_dp_link_caps_get_max_config()
+ */
+enum intel_dp_link_caps_order_key {
+	INTEL_DP_LINK_CAPS_ORDER_KEY_BW,
+	INTEL_DP_LINK_CAPS_ORDER_KEY_RATE_LANE,
+	INTEL_DP_LINK_CAPS_ORDER_KEY_LANE_RATE,
+
+	INTEL_DP_LINK_CAPS_ORDER_KEY_NUM
+};
+
+/**
+ * enum intel_dp_link_caps_order_direction - iteration direction
+ * @INTEL_DP_LINK_CAPS_ORDER_DIR_ASC:
+ *   Iterate in ascending order according to the selected ordering key.
+ * @INTEL_DP_LINK_CAPS_ORDER_DIR_DESC:
+ *   Iterate in descending order according to the selected ordering key.
+ * @INTEL_DP_LINK_CAPS_ORDER_DIR_NUM:
+ *   Number of ordering directions.
+ *
+ * Selects the direction associated with an
+ * &enum intel_dp_link_caps_order_key for iteration queries.
+ *
+ * See also:
+ *  - &struct intel_dp_link_caps_order
+ */
+enum intel_dp_link_caps_order_direction {
+	INTEL_DP_LINK_CAPS_ORDER_DIR_ASC,
+	INTEL_DP_LINK_CAPS_ORDER_DIR_DESC,
+
+	INTEL_DP_LINK_CAPS_ORDER_DIR_NUM
+};
+
+/**
+ * struct intel_dp_link_caps_order - configuration ordering
+ * @key:
+ *   Key used to order configurations.
+ * @dir:
+ *   Direction of the selected ordering.
+ *
+ * Describes an iteration order for link configurations.
+ *
+ * See also:
+ *  - for_each_dp_link_config()
+ */
+struct intel_dp_link_caps_order {
+	enum intel_dp_link_caps_order_key key;
+	enum intel_dp_link_caps_order_direction dir;
+};
+
+struct intel_dp_link_caps_filter {
+	u32 config_mask;
+};
+
+#define INTEL_DP_LINK_CAPS_FILTER_NONE	\
+	((struct intel_dp_link_caps_filter){ .config_mask = 0 })
+#define INTEL_DP_LINK_CAPS_FILTER_ALL	\
+	((struct intel_dp_link_caps_filter){ .config_mask = (u32)-1 })
+
+struct intel_dp_link_caps_iter {
+	struct intel_dp_link_caps *link_caps;
+	int pos;
+	struct intel_dp_link_caps_order order;
+	struct intel_dp_link_caps_filter filter;
+
+	bool (*get_next_config)(struct intel_dp_link_caps_iter *iter,
+				struct intel_dp_link_config *config);
+};
+
+/**
+ * for_each_dp_link_config - iterate allowed link configurations
+ * @__iter:
+ *   &struct intel_dp_link_caps_iter being iterated
+ * @__config:
+ *   pointer to &struct intel_dp_link_config filled for each match
+ */
+#define for_each_dp_link_config(__iter, __config) \
+	while ((__iter)->get_next_config((__iter), (__config)))
+
+void intel_dp_link_caps_iter_start(struct intel_dp_link_caps_iter *iter,
+				   struct intel_dp_link_caps *link_caps,
+				   struct intel_dp_link_caps_order order,
+				   struct intel_dp_link_caps_filter filter);
+
+void intel_dp_link_caps_iter_end(struct intel_dp_link_caps_iter *iter);
+
+struct intel_dp_link_caps_order
+intel_dp_link_caps_connector_compute_order(struct intel_connector *connector);
+struct intel_dp_link_caps_order
+intel_dp_link_caps_connector_fallback_order(bool is_mst);
+
+void intel_dp_link_caps_print_common_rates(struct intel_dp_link_caps *link_caps);
+
+void intel_dp_link_caps_get_forced_params(struct intel_dp_link_caps *link_caps,
+					  struct intel_dp_link_config *forced_params);
+
+bool intel_dp_link_caps_filter_add(struct intel_dp_link_caps *link_caps,
+				   struct intel_dp_link_caps_filter *filter,
+				   const struct intel_dp_link_config *config);
+
+bool intel_dp_link_caps_get_max_config(struct intel_dp_link_caps *link_caps,
+				       enum intel_dp_link_caps_order_key order_key,
+				       struct intel_dp_link_caps_filter filter,
+				       struct intel_dp_link_config *max_config);
+
+void intel_dp_link_caps_get_max_bw_config(struct intel_dp_link_caps *link_caps,
+					  struct intel_dp_link_config *max_config);
+
+bool intel_dp_link_caps_disable_config(struct intel_dp_link_caps *link_caps,
+				       const struct intel_dp_link_config *config);
+
+void intel_dp_link_caps_get_max_limits(struct intel_dp_link_caps *link_caps,
+				       struct intel_dp_link_config *max_link_limits);
+bool intel_dp_link_caps_set_max_limits(struct intel_dp_link_caps *link_caps,
+				       const struct intel_dp_link_config *max_link_limits);
+void intel_dp_link_caps_reset_max_limits(struct intel_dp_link_caps *link_caps);
+
+bool intel_dp_link_caps_update(struct intel_dp_link_caps *link_caps,
+			       const int *rates, int num_rates, int max_lane_count,
+			       bool reset);
+void intel_dp_link_caps_reset(struct intel_dp_link_caps *link_caps);
+
+void intel_dp_link_caps_debugfs_add(struct intel_connector *connector);
+
+struct intel_dp_link_caps *intel_dp_link_caps_init(struct intel_dp *intel_dp);
+void intel_dp_link_caps_cleanup(struct intel_dp_link_caps *link_caps);
+
+#if IS_ENABLED(CONFIG_KUNIT)
+
+#define INTEL_DP_LINK_CAPS_TEST_OPS_MEMBERS(__X) \
+	__X(connector_compute_order,	intel_dp_link_caps_connector_compute_order) \
+	__X(connector_fallback_order,	intel_dp_link_caps_connector_fallback_order) \
+	__X(iter_start,			intel_dp_link_caps_iter_start) \
+	__X(iter_end,			intel_dp_link_caps_iter_end) \
+	__X(set_max_limits,		intel_dp_link_caps_set_max_limits) \
+	__X(get_max_limits,		intel_dp_link_caps_get_max_limits) \
+	__X(get_max_bw_config,		intel_dp_link_caps_get_max_bw_config) \
+	__X(reset_max_limits,		intel_dp_link_caps_reset_max_limits) \
+	__X(disable_config,		intel_dp_link_caps_disable_config) \
+	__X(update,			intel_dp_link_caps_update) \
+	__X(init,			intel_dp_link_caps_init) \
+	__X(cleanup,			intel_dp_link_caps_cleanup)
+
+#define __DECLARE_MEMBER(__name, __fn) \
+	typeof(__fn) *__name;
+
+#define INTEL_DP_LINK_CAPS_TEST_OPS_DECLARE \
+	INTEL_DP_LINK_CAPS_TEST_OPS_MEMBERS(__DECLARE_MEMBER)
+
+struct intel_dp_link_caps_test_ops {
+	INTEL_DP_LINK_CAPS_TEST_OPS_DECLARE
+};
+
+#undef INTEL_DP_LINK_CAPS_TEST_OPS_DECLARE
+#undef __DECLARE_MEMBER
+
+#ifdef I915
+extern const struct intel_dp_link_caps_test_ops i915_display_dp_link_caps_test_ops;
+#else
+extern const struct intel_dp_link_caps_test_ops intel_display_dp_link_caps_test_ops;
+#endif	/* I915 */
+
+#endif	/* CONFIG_KUNIT */
+
+#endif /* __INTEL_DP_LINK_CAPS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.c b/drivers/gpu/drm/i915/display/intel_dp_link_training.c
index e566f2b49594a..fa55664c9d98e 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_link_training.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.c
@@ -21,24 +21,352 @@
  * IN THE SOFTWARE.
  */
 
+#include <kunit/visibility.h>
+
 #include <linux/debugfs.h>
 #include <linux/iopoll.h>
 
 #include <drm/display/drm_dp_helper.h>
 #include <drm/drm_print.h>
 
+#include "intel_display.h"
 #include "intel_display_core.h"
 #include "intel_display_jiffies.h"
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
 #include "intel_dp.h"
+#include "intel_dp_link_caps.h"
 #include "intel_dp_link_training.h"
+#include "intel_dp_mst.h"
 #include "intel_encoder.h"
 #include "intel_hdmi.h"
 #include "intel_hotplug.h"
+#include "intel_modeset_lock.h"
 #include "intel_panel.h"
 #include "intel_psr.h"
 
+/**
+ * DOC: DisplayPort link training
+ *
+ * This documents the Intel DisplayPort link training implementation and
+ * its internal interfaces, with a current focus on link recovery.
+ *
+ * Documentation of the full link training procedure is not yet included.
+ *
+ * The Intel DP link recovery logic governs how the driver reacts to
+ * link training failures and to links that degrade asynchronously
+ * after a previously successful training. Recovery is first attempted
+ * via automatic retraining (``autoretrain``) and, when that is no
+ * longer possible, by selecting fallback link configurations and
+ * notifying userspace to recover the link via a modeset.
+ *
+ * Recovery sequence and userspace notification
+ * --------------------------------------------
+ *
+ * After the first link training failure following initialization or a
+ * previously successful training, recovery is first attempted by the
+ * driver via automatic retraining, without userspace involvement.
+ * During this phase, a given link configuration is attempted twice
+ * before being abandoned: after the initial link training failure, an
+ * automatic retraining modeset is performed with the same link
+ * parameters, constituting the second attempt.
+ *
+ * Once automatic retraining is no longer possible, recovery is delegated
+ * to userspace, which must select a new modeset configuration, as the
+ * kernel must not do so. From this point onwards, each link configuration
+ * may be attempted only once as userspace iterates through alternative
+ * configurations. A successful link training restores the automatic
+ * retraining model for subsequent failures.
+ *
+ * The failure of the last automatic retraining attempt is reported to
+ * userspace, and from that point onward the driver notifies userspace of
+ * each subsequent failure. This allows userspace to both initiate
+ * recovery via modesets and observe the outcome of those recovery
+ * attempts, even when no further fallback configurations remain.
+ *
+ * Link training failures are always reported to userspace, even when they
+ * result from a kernel-internal modeset. Such modesets only re-apply the
+ * existing userspace-provided state and must not modify it. A failure
+ * triggered by such a modeset is therefore treated the same as a link
+ * degradation after a previously successful training, and recovery is
+ * handled by userspace in place of the kernel caller.
+ *
+ * Contexts
+ * --------
+ *
+ * The following execution contexts (A/B/C) describe how the different
+ * recovery states are reached but are not themselves implementation
+ * states. The actual state machine is defined by &enum
+ * intel_dp_link_training_recovery_state.
+ *
+ * A. Modeset context:
+ *
+ *   Triggered by:
+ *     - link training during a modeset, or
+ *     - via the "i915_dp_force_link_training_failure" debugfs entry,
+ *       forcing this path by emulating a link training failure.
+ *
+ *   Transitions:
+ *     - A1 Link training succeeds.
+ *
+ *       A link check work to recover any degraded link is scheduled
+ *       (and handled if needed in context B).
+ *
+ *       State -> %INTEL_DP_LINK_RECOVERY_IDLE.
+ *
+ *     - A2 First link training fails after initialization or a previously
+ *       successful link training.
+ *
+ *       An automatic retraining work is scheduled (and handled in
+ *       context B) with the same link parameters with which the link
+ *       training failed.
+ *
+ *       State -> %INTEL_DP_LINK_RECOVERY_AUTORETRAIN_PENDING.
+ *
+ *     - A3 Link training fails again after A2 or A3.
+ *
+ *       Through fallback selection, the driver attempts to restrict the
+ *       allowed link configurations for subsequent modesets. This may
+ *       be done either by lowering global limits (rate/lane caps), or by
+ *       disabling only the currently failing configuration while leaving
+ *       all other configurations allowed, even if they use higher rate or
+ *       lane count.
+ *
+ *       (The current implementation may still apply parameter capping as
+ *       a coarse fallback selection mechanism. This is transitional and is
+ *       expected to be replaced by a scheme that disables only the failing
+ *       configuration, rather than removing configurations that have not
+ *       been observed to fail and may still train successfully.)
+ *
+ *       This case may repeat in a loop:
+ *           %INTEL_DP_LINK_RECOVERY_AUTORETRAIN_DISABLED ->
+ *           %INTEL_DP_LINK_RECOVERY_AUTORETRAIN_DISABLED
+ *
+ *       via repeated A3a -> A3a transitions until the configuration fallback
+ *       space is exhausted, reaching the A3b terminal case.
+ *
+ *       - A3a Fallback selection succeeds.
+ *
+ *         Userspace is notified to retry the modeset.
+ *
+ *         State -> %INTEL_DP_LINK_RECOVERY_AUTORETRAIN_DISABLED.
+ *
+ *       - A3b Fallback selection fails.
+ *
+ *         Userspace is notified of the failure and may continue recovery
+ *         by retrying the modeset with the remaining allowed link
+ *         configuration.
+ *
+ *         State -> %INTEL_DP_LINK_RECOVERY_NO_FALLBACK.
+ *
+ * B. Automatic retraining context:
+ *
+ *   Triggered by:
+ *     - after a successful link training in context A1 followed by
+ *       asynchronous link degradation, or
+ *     - after the first failed link training attempt in context A2, or
+ *     - via the "i915_dp_force_link_retrain" debugfs entry, which may
+ *       bypass normal gating and force this path.
+ *
+ *   Transitions:
+ *     - B1 ``Autoretrain`` modeset check and link training succeeds.
+ *
+ *       The case is handled as in A1, scheduling a link check work to
+ *       recover any degraded link.
+ *
+ *       State -> %INTEL_DP_LINK_RECOVERY_IDLE.
+ *
+ *     - B2 ``Autoretrain`` modeset check succeeds but link training fails.
+ *
+ *       - B2a Previously the link degraded asynchronously (current state
+ *         is %INTEL_DP_LINK_RECOVERY_IDLE).
+ *
+ *         This corresponds to a first failure in a new failure
+ *         sequence and is handled as in A2: an automatic retraining
+ *         attempt is scheduled with the same link parameters.
+ *
+ *         State -> %INTEL_DP_LINK_RECOVERY_AUTORETRAIN_PENDING.
+ *
+ *       - B2b Previously a link training failed (current state is
+ *         %INTEL_DP_LINK_RECOVERY_AUTORETRAIN_PENDING).
+ *
+ *         In non-regular (debug-forced) scenarios this may also be
+ *         reached from
+ *         %INTEL_DP_LINK_RECOVERY_AUTORETRAIN_DISABLED or
+ *         %INTEL_DP_LINK_RECOVERY_NO_FALLBACK, effectively behaving
+ *         like a userspace-driven recovery attempt.
+ *
+ *         The failure is handled as in A3, performing a fallback selection:
+ *
+ *         State -> %INTEL_DP_LINK_RECOVERY_AUTORETRAIN_DISABLED (via A3a).
+ *
+ *         or
+ *
+ *         State -> %INTEL_DP_LINK_RECOVERY_NO_FALLBACK (via A3b).
+ *
+ *     - B3 ``Autoretrain`` modeset check fails (and hence the link training
+ *       cannot be started).
+ *
+ *       The modeset check may fail, for example, due to external conditions
+ *       such as changed shared link bandwidth, which can make previously
+ *       valid modeset parameters no longer acceptable.
+ *
+ *       In this case, automatic retraining is disabled without selecting
+ *       a fallback configuration. The driver hands recovery over to
+ *       userspace without modifying the allowed configuration set, so a
+ *       subsequent userspace modeset will retry with the current link
+ *       configuration. Userspace is in a better position to select new
+ *       modeset parameters (e.g. video mode or enabled outputs) that
+ *       satisfy the updated constraints, as the driver is only allowed
+ *       to retry the modeset with the existing userspace-provided modeset
+ *       configuration.
+ *
+ *       This policy preserves the normal retry model, where a given link
+ *       configuration is attempted twice in the automatic retraining
+ *       flow before being abandoned: after a first link training failure,
+ *       an automatic retraining modeset is performed with the same link
+ *       parameters, and if its atomic check passes, the link training
+ *       itself may either succeed or fail, constituting the second
+ *       attempt. In this case, however, the retry modeset's atomic check
+ *       failed, so no second link training attempt with those parameters
+ *       was performed, and selecting a fallback would cause that
+ *       configuration to be tried only once rather than twice.
+ *
+ *       The userspace-driven link recovery continues with subsequent
+ *       userspace modesets handled in A3.
+ *
+ *       State -> %INTEL_DP_LINK_RECOVERY_AUTORETRAIN_DISABLED.
+ *
+ * C. State reset context:
+ *
+ *   Triggered by:
+ *     - sink capability changes, or
+ *     - sink disconnect/reconnect, or
+ *     - system suspend/resume or power transitions where HPD
+ *       handling may have been suppressed, or
+ *     - successful link training.
+ *
+ *   Transitions:
+ *     - The recovery state is reset from any of the recovery states
+ *
+ *       State -> %INTEL_DP_LINK_RECOVERY_IDLE.
+ *
+ *       After reset, the driver may re-check link status and schedule
+ *       retraining if the link is found to remain degraded.
+ *
+ * State transition summary
+ * ------------------------
+ *
+ * - From %INTEL_DP_LINK_RECOVERY_IDLE
+ *
+ *   - To %INTEL_DP_LINK_RECOVERY_IDLE
+ *
+ *     - | In context: B1
+ *       | Action: no action
+ *
+ *   - To %INTEL_DP_LINK_RECOVERY_AUTORETRAIN_PENDING
+ *
+ *     - | In contexts: A2, B2a
+ *       | Action: queue ``autoretrain`` work
+ *
+ *   - To %INTEL_DP_LINK_RECOVERY_AUTORETRAIN_DISABLED
+ *
+ *     - | In context: B3
+ *       | Action: notify userspace
+ *
+ * - From %INTEL_DP_LINK_RECOVERY_AUTORETRAIN_PENDING
+ *
+ *   - To %INTEL_DP_LINK_RECOVERY_AUTORETRAIN_DISABLED
+ *
+ *     - | In contexts: A3a, B2b
+ *       | Action: select fallback configurations, notify userspace
+ *
+ *     - | In context: B3
+ *       | Action: notify userspace
+ *
+ *   - To %INTEL_DP_LINK_RECOVERY_NO_FALLBACK
+ *
+ *     - | In contexts: A3b, B2b
+ *       | Action: notify userspace
+ *
+ * - From %INTEL_DP_LINK_RECOVERY_AUTORETRAIN_DISABLED
+ *
+ *   - To %INTEL_DP_LINK_RECOVERY_AUTORETRAIN_DISABLED
+ *
+ *     - | In contexts: A3a, B2b
+ *       | Action: select fallback configurations, notify userspace
+ *
+ *   - To %INTEL_DP_LINK_RECOVERY_NO_FALLBACK
+ *
+ *     - | In contexts: A3b, B2b
+ *       | Action: notify userspace
+ *
+ * - From %INTEL_DP_LINK_RECOVERY_NO_FALLBACK
+ *
+ *   - To %INTEL_DP_LINK_RECOVERY_NO_FALLBACK
+ *
+ *     - | In contexts: A3b
+ *       | Action: notify userspace
+ *
+ * - From any state
+ *
+ *   - To %INTEL_DP_LINK_RECOVERY_IDLE
+ *
+ *     - | In contexts: C
+ *       | Action: no action
+ *
+ * Recovery flows
+ * --------------
+ *
+ * Userspace modeset link recovery::
+ *
+ *                       [IDLE]
+ *                          |
+ *                          | userspace modeset link training fails
+ *                          | (autoretrain link recovery work scheduled)
+ *                          v
+ *               [AUTORETRAIN_PENDING]-- autoretrain link recovery succeeds -> [IDLE]
+ *                          |
+ *                          | autoretrain link recovery modeset check or link training fails
+ *                          |
+ *                       +--o--+
+ *  modeset check fails  |     | link training fails
+ *  (userspace notified) |     |
+ *                       |     o-------- no fallback (userspace notified) ---> [NO_FALLBACK]
+ *                       |     |
+ *   +-------------+     |     | fallback selected (userspace notified)
+ *   |             |     |     |
+ *   |             v     v     v
+ *   |        [AUTORETRAIN_DISABLED]--- userspace link recovery succeeds ----> [IDLE]
+ *   |                   |
+ *   |                   | userspace link recovery fails
+ *   |                   |
+ *   +-------------------o------------- no fallback (userspace notified) ----> [NO_FALLBACK]
+ *   fallback selected
+ *   (userspace notified)
+ *
+ * Asynchronous link degradation recovery::
+ *
+ *                       [IDLE]
+ *                          |
+ *                          | link degrades
+ *                          | (autoretrain link recovery performed)
+ *                          |
+ *                          o--- autoretrain link recovery succeeds ---> [IDLE]
+ *                          |
+ *                          | autoretrain link recovery modeset check or link training fails
+ *                          |
+ *                       +--o--+
+ *  modeset check fails  |     | link training fails
+ *  (userspace notified) |     | (autoretrain work scheduled)
+ *                       v     v
+ *  [AUTORETRAIN_DISABLED*]   [AUTORETRAIN_PENDING*]
+ *
+ * ``*`` marks states where the sequence continues from the corresponding state
+ * in the Userspace modeset link recovery flow above.
+ *
+ */
+
 #define LT_MSG_PREFIX			"[CONNECTOR:%d:%s][ENCODER:%d:%s][%s] "
 #define LT_MSG_ARGS(_intel_dp, _dp_phy)	(_intel_dp)->attached_connector->base.base.id, \
 					(_intel_dp)->attached_connector->base.name, \
@@ -60,7 +388,72 @@
 		lt_dbg(_intel_dp, _dp_phy, "Sink disconnected: " _format, ## __VA_ARGS__); \
 } while (0)
 
-#define MAX_SEQ_TRAIN_FAILURES 2
+/*
+ * enum intel_dp_link_recovery_state - LT recovery state
+ * @INTEL_DP_LINK_RECOVERY_IDLE:
+ *   No link training failure is currently tracked and no recovery is
+ *   in progress. This is the initial state after driver initialization,
+ *   power state transitions, sink (re-)connection, or after a successful
+ *   link training.
+ *
+ * @INTEL_DP_LINK_RECOVERY_AUTORETRAIN_PENDING:
+ *   A first link training failure has been observed and an automatic
+ *   retraining attempt with the same link parameters is pending. Exactly
+ *   one such attempt is allowed before switching to userspace-driven
+ *   recovery.
+ *
+ * @INTEL_DP_LINK_RECOVERY_AUTORETRAIN_DISABLED:
+ *   Automatic retraining is no longer possible. At this point, a
+ *   fallback selection is made and userspace is notified to take over
+ *   recovery, performing modesets with parameters it determines are
+ *   required. The driver then selects a link configuration from the
+ *   remaining fallback configuration set. Subsequent link training
+ *   failures trigger further fallback selections and userspace
+ *   notifications.
+ *
+ * @INTEL_DP_LINK_RECOVERY_NO_FALLBACK:
+ *   Fallback selection is no longer possible, as no usable fallback link
+ *   configurations remain. Recovery must proceed via userspace modesets
+ *   using the remaining allowed link configuration. Userspace continues
+ *   to be notified of subsequent link training failures.
+ *
+ * Describes the link recovery state used by the Intel DP link recovery
+ * logic.
+ *
+ * See also:
+ *   - DOC: DisplayPort link training
+ *   - link_recovery_autoretrain_pending()
+ *   - link_recovery_autoretrain_allowed()
+ *   - link_recovery_has_no_fallback()
+ *   - link_recovery_mark_train_failure()
+ *   - link_recovery_mark_autoretrain_modeset_failure()
+ *   - link_recovery_mark_no_fallback()
+ *   - link_recovery_reset()
+ */
+enum intel_dp_link_recovery_state {
+	/*
+	 * Keep the enum values ordered from least to most severe
+	 * recovery state; helper logic relies on that ordering.
+	 */
+	INTEL_DP_LINK_RECOVERY_IDLE,
+	INTEL_DP_LINK_RECOVERY_AUTORETRAIN_PENDING,
+	INTEL_DP_LINK_RECOVERY_AUTORETRAIN_DISABLED,
+	INTEL_DP_LINK_RECOVERY_NO_FALLBACK,
+};
+
+struct intel_dp_link_training {
+	struct intel_dp *dp;
+
+	enum intel_dp_link_recovery_state recovery_state;
+
+	int force_train_failure;
+	bool force_retrain;
+};
+
+static struct intel_dp_link_training *connector_to_link_training(struct intel_connector *connector)
+{
+	return intel_attached_dp(connector)->link.training;
+}
 
 static void intel_dp_reset_lttpr_common_caps(struct intel_dp *intel_dp)
 {
@@ -245,12 +638,12 @@ int intel_dp_read_dprx_caps(struct intel_dp *intel_dp, u8 dpcd[DP_RECEIVER_CAP_S
  * transparent mode link training mode.
  *
  * Returns:
- *   >0  if LTTPRs were detected and the non-transparent LT mode was set. The
+ * - >0  if LTTPRs were detected and the non-transparent LT mode was
+ *       set. The DPRX capabilities are read out.
+ * -  0  if no LTTPRs or more than 8 LTTPRs were detected or in case of
+ *       a detection failure and the transparent LT mode was set. The
  *       DPRX capabilities are read out.
- *    0  if no LTTPRs or more than 8 LTTPRs were detected or in case of a
- *       detection failure and the transparent LT mode was set. The DPRX
- *       capabilities are read out.
- *   <0  Reading out the DPRX capabilities failed.
+ * - <0  Reading out the DPRX capabilities failed.
  */
 int intel_dp_init_lttpr_and_dprx_caps(struct intel_dp *intel_dp)
 {
@@ -1247,6 +1640,124 @@ intel_dp_128b132b_intra_hop(struct intel_dp *intel_dp,
 	return sink_status & DP_INTRA_HOP_AUX_REPLY_INDICATION ? 1 : 0;
 }
 
+static bool
+link_recovery_autoretrain_pending(struct intel_dp_link_training *link_training)
+{
+	return link_training->recovery_state == INTEL_DP_LINK_RECOVERY_AUTORETRAIN_PENDING;
+}
+
+/*
+ * Automatic retraining is a driver-driven link recovery mechanism that
+ * retrains the link with the current userspace provided modeset
+ * configuration and link parameters.
+ *
+ * Autoretrain is allowed while the link configurations available for
+ * retraining, i.e. those not disabled yet via fallback selection, still
+ * make it possible to retrain the link for the current userspace provided
+ * modeset configuration.
+ *
+ * Once automatic retraining is no longer allowed, userspace driven link
+ * recovery via userspace notifications and userspace modesets takes over.
+ *
+ * See also:
+ *   - DOC: DisplayPort link training
+ */
+static bool
+link_recovery_autoretrain_allowed(struct intel_dp_link_training *link_training)
+{
+	switch (link_training->recovery_state) {
+	case INTEL_DP_LINK_RECOVERY_IDLE:
+	case INTEL_DP_LINK_RECOVERY_AUTORETRAIN_PENDING:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool
+link_recovery_has_no_fallback(struct intel_dp_link_training *link_training)
+{
+	return link_training->recovery_state == INTEL_DP_LINK_RECOVERY_NO_FALLBACK;
+}
+
+/*
+ * Record a link training failure and advance the recovery state to
+ * indicate the next required recovery step.
+ *
+ * The caller must proceed with recovery as instructed by the return
+ * value, either via automatic retraining or, once automatic retraining
+ * is no longer possible, via userspace modesets after fallback
+ * selection.
+ *
+ * Note that the error reported via this function is the error seen by
+ * the link training failure handler proper after an actual link
+ * training failure indicated by the sink device, and so the error and
+ * corresponding actions required are distinct from an autoretrain
+ * modeset failure. See link_recovery_mark_autoretrain_modeset_failure() to
+ * report a modeset failure.
+ *
+ * See also:
+ *   - DOC: DisplayPort link training
+ */
+static bool
+link_recovery_mark_train_failure(struct intel_dp_link_training *link_training)
+{
+	switch (link_training->recovery_state) {
+	case INTEL_DP_LINK_RECOVERY_IDLE:
+		link_training->recovery_state = INTEL_DP_LINK_RECOVERY_AUTORETRAIN_PENDING;
+		break;
+	case INTEL_DP_LINK_RECOVERY_AUTORETRAIN_PENDING:
+		link_training->recovery_state = INTEL_DP_LINK_RECOVERY_AUTORETRAIN_DISABLED;
+		break;
+	default:
+		break;
+	}
+
+	return link_recovery_autoretrain_allowed(link_training);
+}
+
+/*
+ * Record a failure of the autoretrain modeset before link training
+ * itself could run.
+ *
+ * Note that the error reported via this function and the corresponding
+ * expected actions are distinct from an actual link training failure:
+ * the modeset failed before a link training attempt could be performed.
+ * See link_recovery_mark_train_failure() to report an actual link
+ * training failure.
+ *
+ * Update the state to indicate that further recovery is to be delegated to
+ * userspace via a regular modeset.
+ *
+ * See also:
+ *   - DOC: DisplayPort link training
+ */
+static void
+link_recovery_mark_autoretrain_modeset_failure(struct intel_dp_link_training *link_training)
+{
+	if (link_recovery_autoretrain_allowed(link_training))
+		link_training->recovery_state = INTEL_DP_LINK_RECOVERY_AUTORETRAIN_DISABLED;
+}
+
+/* Record that no more link fallback configuration is available. */
+static void
+link_recovery_mark_no_fallback(struct intel_dp_link_training *link_training)
+{
+	link_training->recovery_state = INTEL_DP_LINK_RECOVERY_NO_FALLBACK;
+}
+
+/**
+ * link_recovery_reset - reset the link recovery state
+ * @link_training: link training state
+ *
+ * Reset the link recovery state to indicate that no link recovery is
+ * required.
+ */
+static void link_recovery_reset(struct intel_dp_link_training *link_training)
+{
+	link_training->recovery_state = INTEL_DP_LINK_RECOVERY_IDLE;
+}
+
 /**
  * intel_dp_stop_link_train - stop link training
  * @intel_dp: DP struct
@@ -1266,6 +1777,7 @@ intel_dp_128b132b_intra_hop(struct intel_dp *intel_dp,
 void intel_dp_stop_link_train(struct intel_dp *intel_dp,
 			      const struct intel_crtc_state *crtc_state)
 {
+	struct intel_dp_link_training *link_training = intel_dp->link.training;
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
 	int ret;
@@ -1286,8 +1798,8 @@ void intel_dp_stop_link_train(struct intel_dp *intel_dp,
 	intel_hpd_unblock(encoder);
 
 	if (!display->hotplug.ignore_long_hpd &&
-	    intel_dp->link.seq_train_failures < MAX_SEQ_TRAIN_FAILURES) {
-		int delay_ms = intel_dp->link.seq_train_failures ? 0 : 2000;
+	    link_recovery_autoretrain_allowed(link_training)) {
+		int delay_ms = link_recovery_autoretrain_pending(link_training) ? 0 : 2000;
 
 		intel_encoder_link_check_queue_work(encoder, delay_ms);
 	}
@@ -1336,111 +1848,62 @@ static bool intel_dp_can_link_train_fallback_for_edp(struct intel_dp *intel_dp,
 	return true;
 }
 
-static bool reduce_link_params_in_bw_order(struct intel_dp *intel_dp,
-					   const struct intel_crtc_state *crtc_state,
-					   int *new_link_rate, int *new_lane_count)
+static bool reduce_link_params(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state,
+			       int *new_link_rate, int *new_lane_count)
 {
-	int link_rate;
-	int lane_count;
-	int i;
-
-	i = intel_dp_link_config_index(intel_dp, crtc_state->port_clock, crtc_state->lane_count);
-	for (i--; i >= 0; i--) {
-		intel_dp_link_config_get(intel_dp, i, &link_rate, &lane_count);
+	struct intel_dp_link_caps *link_caps = intel_dp->link.caps;
+	bool is_mst = intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST);
+	struct intel_dp_link_caps_order order =
+		intel_dp_link_caps_connector_fallback_order(is_mst);
+	struct intel_dp_link_config old_config = {
+		.rate = crtc_state->port_clock,
+		.lane_count = crtc_state->lane_count,
+	};
+	struct intel_dp_link_caps_iter iter;
+	struct intel_dp_link_config config;
+	bool old_found = false;
+	bool new_found = false;
+
+	intel_dp_link_caps_iter_start(&iter, link_caps, order, INTEL_DP_LINK_CAPS_FILTER_ALL);
+	for_each_dp_link_config(&iter, &config) {
+		if (!old_found) {
+			if (config.rate == old_config.rate &&
+			    config.lane_count == old_config.lane_count)
+				old_found = true;
 
-		if ((intel_dp->link.force_rate &&
-		     intel_dp->link.force_rate != link_rate) ||
-		    (intel_dp->link.force_lane_count &&
-		     intel_dp->link.force_lane_count != lane_count))
 			continue;
+		}
 
-		break;
-	}
-
-	if (i < 0)
-		return false;
-
-	*new_link_rate = link_rate;
-	*new_lane_count = lane_count;
-
-	return true;
-}
-
-static int reduce_link_rate(struct intel_dp *intel_dp, int current_rate)
-{
-	int rate_index;
-	int new_rate;
-
-	if (intel_dp->link.force_rate)
-		return -1;
-
-	rate_index = intel_dp_rate_index(intel_dp->common_rates,
-					 intel_dp->num_common_rates,
-					 current_rate);
-
-	if (rate_index <= 0)
-		return -1;
-
-	new_rate = intel_dp_common_rate(intel_dp, rate_index - 1);
-
-	/* TODO: Make switching from UHBR to non-UHBR rates work. */
-	if (drm_dp_is_uhbr_rate(current_rate) != drm_dp_is_uhbr_rate(new_rate))
-		return -1;
-
-	return new_rate;
-}
-
-static int reduce_lane_count(struct intel_dp *intel_dp, int current_lane_count)
-{
-	if (intel_dp->link.force_lane_count)
-		return -1;
-
-	if (current_lane_count == 1)
-		return -1;
-
-	return current_lane_count >> 1;
-}
+		if (!is_mst &&
+		    drm_dp_is_uhbr_rate(config.rate) !=
+		    drm_dp_is_uhbr_rate(old_config.rate))
+			continue;
 
-static bool reduce_link_params_in_rate_lane_order(struct intel_dp *intel_dp,
-						  const struct intel_crtc_state *crtc_state,
-						  int *new_link_rate, int *new_lane_count)
-{
-	int link_rate;
-	int lane_count;
+		*new_link_rate = config.rate;
+		*new_lane_count = config.lane_count;
+		new_found = true;
 
-	lane_count = crtc_state->lane_count;
-	link_rate = reduce_link_rate(intel_dp, crtc_state->port_clock);
-	if (link_rate < 0) {
-		lane_count = reduce_lane_count(intel_dp, crtc_state->lane_count);
-		link_rate = intel_dp_max_common_rate(intel_dp);
+		break;
 	}
+	intel_dp_link_caps_iter_end(&iter);
 
-	if (lane_count < 0)
-		return false;
-
-	*new_link_rate = link_rate;
-	*new_lane_count = lane_count;
-
-	return true;
+	return new_found;
 }
 
-static bool reduce_link_params(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state,
-			       int *new_link_rate, int *new_lane_count)
-{
-	/* TODO: Use the same fallback logic on SST as on MST. */
-	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST))
-		return reduce_link_params_in_bw_order(intel_dp, crtc_state,
-						      new_link_rate, new_lane_count);
-	else
-		return reduce_link_params_in_rate_lane_order(intel_dp, crtc_state,
-							     new_link_rate, new_lane_count);
-}
-
-static int intel_dp_get_link_train_fallback_values(struct intel_dp *intel_dp,
-						   const struct intel_crtc_state *crtc_state)
+VISIBLE_IF_KUNIT
+int intel_dp_get_link_train_fallback_values(struct intel_dp *intel_dp,
+					    const struct intel_crtc_state *crtc_state)
 {
+	struct intel_display *display = to_intel_display(intel_dp);
+	struct intel_dp_link_caps *link_caps = intel_dp->link.caps;
+	struct intel_dp_link_config max_link_limits;
+	struct intel_dp_link_config current_config = {
+		.rate = crtc_state->port_clock,
+		.lane_count = crtc_state->lane_count,
+	};
 	int new_link_rate;
 	int new_lane_count;
+	int err = -1;
 
 	if (intel_dp_is_edp(intel_dp) && !intel_dp->use_max_params) {
 		lt_dbg(intel_dp, DP_PHY_DPRX,
@@ -1449,25 +1912,70 @@ static int intel_dp_get_link_train_fallback_values(struct intel_dp *intel_dp,
 		return 0;
 	}
 
+	/*
+	 * Temporarily reset the max link limit before selecting the fallback
+	 * config.
+	 *
+	 * After fallback, the current logic narrows the allowed configurations
+	 * to the selected config's rate and lane count. That can make a later
+	 * fallback candidate fall outside the current max_limit, so reset it
+	 * before searching.
+	 *
+	 * TODO: Constrain the allowed configurations by only disabling individual
+	 * configurations and remove setting maximum link parameters.
+	 */
+	intel_dp_link_caps_get_max_limits(link_caps, &max_link_limits);
+	intel_dp_link_caps_reset_max_limits(link_caps);
+
+	/*
+	 * TODO: Make fallback depend only on disabling the current config,
+	 * once max_limit no longer constrains the allowed config set. Then
+	 * disabling the current config will define the allowed configs for
+	 * the subsequent modeset, so there will be no need to select a
+	 * reduced config separately here.
+	 */
 	if (!reduce_link_params(intel_dp, crtc_state, &new_link_rate, &new_lane_count))
-		return -1;
+		goto out_restore_max_limits;
 
 	if (intel_dp_is_edp(intel_dp) &&
 	    !intel_dp_can_link_train_fallback_for_edp(intel_dp, new_link_rate, new_lane_count)) {
 		lt_dbg(intel_dp, DP_PHY_DPRX,
 		       "Retrying Link training for eDP with same parameters\n");
-		return 0;
+
+		err = 0;
+
+		goto out_restore_max_limits;
 	}
 
+	/*
+	 * Shouldn't fail: the current config was enabled, and reducing the
+	 * link parameters should still leave the fallback config allowed.
+	 */
+	if (drm_WARN_ON(display->drm,
+			!intel_dp_link_caps_disable_config(link_caps, &current_config)))
+		return -1;
+
 	lt_dbg(intel_dp, DP_PHY_DPRX,
 	       "Reducing link parameters from %dx%d to %dx%d\n",
 	       crtc_state->lane_count, crtc_state->port_clock,
 	       new_lane_count, new_link_rate);
 
-	intel_dp->link.max_rate = new_link_rate;
-	intel_dp->link.max_lane_count = new_lane_count;
+	max_link_limits.rate = new_link_rate;
+	max_link_limits.lane_count = new_lane_count;
 
-	return 0;
+	err = 0;
+
+out_restore_max_limits:
+	/*
+	 * Shouldn't fail: setting max_limits can only fail if they drop below
+	 * the optionally forced rate/lane-count parameters, but the reduced
+	 * config was chosen to satisfy those constraints.
+	 */
+	if (drm_WARN_ON(display->drm,
+			!intel_dp_link_caps_set_max_limits(link_caps, &max_link_limits)))
+		err = -1;
+
+	return err;
 }
 
 static bool intel_dp_schedule_fallback_link_training(struct intel_atomic_state *state,
@@ -1780,6 +2288,9 @@ void intel_dp_start_link_train(struct intel_atomic_state *state,
 	struct intel_display *display = to_intel_display(state);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	struct intel_encoder *encoder = &dig_port->base;
+	struct intel_dp_link_training *link_training =
+		intel_dp->link.training;
+	bool autoretrain_allowed;
 	bool passed;
 	/*
 	 * Reinit the LTTPRs here to ensure that they are switched to
@@ -1803,15 +2314,15 @@ void intel_dp_start_link_train(struct intel_atomic_state *state,
 	else
 		passed = intel_dp_link_train_all_phys(intel_dp, crtc_state, lttpr_count);
 
-	if (intel_dp->link.force_train_failure) {
-		intel_dp->link.force_train_failure--;
+	if (link_training->force_train_failure) {
+		link_training->force_train_failure--;
 		lt_dbg(intel_dp, DP_PHY_DPRX, "Forcing link training failure\n");
 	} else if (passed) {
-		intel_dp->link.seq_train_failures = 0;
+		link_recovery_reset(link_training);
 		return;
 	}
 
-	intel_dp->link.seq_train_failures++;
+	autoretrain_allowed = link_recovery_mark_train_failure(link_training);
 
 	/*
 	 * Ignore the link failure in CI
@@ -1830,13 +2341,13 @@ void intel_dp_start_link_train(struct intel_atomic_state *state,
 		return;
 	}
 
-	if (intel_dp->link.seq_train_failures < MAX_SEQ_TRAIN_FAILURES)
+	if (autoretrain_allowed)
 		return;
 
 	if (intel_dp_schedule_fallback_link_training(state, intel_dp, crtc_state))
 		return;
 
-	intel_dp->link.retrain_disabled = true;
+	link_recovery_mark_no_fallback(link_training);
 
 	if (!passed)
 		lt_err(intel_dp, DP_PHY_DPRX, "Can't reduce link training parameters after failure\n");
@@ -1864,256 +2375,307 @@ void intel_dp_128b132b_sdp_crc16(struct intel_dp *intel_dp,
 	lt_dbg(intel_dp, DP_PHY_DPRX, "DP2.0 SDP CRC16 for 128b/132b enabled\n");
 }
 
-static int i915_dp_force_link_rate_show(struct seq_file *m, void *data)
+bool intel_dp_link_params_valid(struct intel_dp *intel_dp, int link_rate,
+				u8 lane_count)
 {
-	struct intel_connector *connector = to_intel_connector(m->private);
-	struct intel_display *display = to_intel_display(connector);
-	struct intel_dp *intel_dp = intel_attached_dp(connector);
-	int current_rate = -1;
-	int force_rate;
-	int err;
-	int i;
-
-	err = drm_modeset_lock_single_interruptible(&display->drm->mode_config.connection_mutex);
-	if (err)
-		return err;
-
-	if (intel_dp->link.active)
-		current_rate = intel_dp->link_rate;
-	force_rate = intel_dp->link.force_rate;
+	struct intel_dp_link_config max_link_limits;
 
-	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
-
-	seq_printf(m, "%sauto%s",
-		   force_rate == 0 ? "[" : "",
-		   force_rate == 0 ? "]" : "");
+	/*
+	 * FIXME: we need to synchronize the current link parameters with
+	 * hardware readout. Currently fast link training doesn't work on
+	 * boot-up.
+	 *
+	 * NOTE:
+	 * This may be called from both serialized (locked and synced against
+	 * async commit tails) and unserialized (e.g. HPD IRQ) contexts. It
+	 * uses the current max link limits as upper bounds to reject
+	 * obviously bogus values, even if those bounds may be observed in a
+	 * transient or slightly stale state.
+	 *
+	 * This is not a full validation of the link configuration. Even in
+	 * serialized contexts, additional constraints (e.g. source limitations,
+	 * bandwidth checks, and other atomic state dependencies) are only
+	 * verified during the atomic check of the subsequent commit.
+	 *
+	 * max_link_limits only provides independent upper bounds for rate and
+	 * lane count. Callers must not assume it is itself an allowed link
+	 * configuration. Although that happens to be true for now, it will
+	 * stop being guaranteed once fallback depends only on disabled configs.
+	 */
+	intel_dp_link_caps_get_max_limits(intel_dp->link.caps, &max_link_limits);
 
-	for (i = 0; i < intel_dp->num_source_rates; i++)
-		seq_printf(m, " %s%d%s%s",
-			   intel_dp->source_rates[i] == force_rate ? "[" : "",
-			   intel_dp->source_rates[i],
-			   intel_dp->source_rates[i] == current_rate ? "*" : "",
-			   intel_dp->source_rates[i] == force_rate ? "]" : "");
+	if (link_rate == 0 ||
+	    link_rate > max_link_limits.rate)
+		return false;
 
-	seq_putc(m, '\n');
+	if (lane_count == 0 ||
+	    lane_count > max_link_limits.lane_count)
+		return false;
 
-	return 0;
+	return true;
 }
 
-static int parse_link_rate(struct intel_dp *intel_dp, const char __user *ubuf, size_t len)
+static bool intel_dp_link_ok(struct intel_dp *intel_dp,
+			     u8 link_status[DP_LINK_STATUS_SIZE])
 {
-	char *kbuf;
-	const char *p;
-	int rate;
-	int ret = 0;
-
-	kbuf = memdup_user_nul(ubuf, len);
-	if (IS_ERR(kbuf))
-		return PTR_ERR(kbuf);
+	struct intel_display *display = to_intel_display(intel_dp);
+	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
+	bool uhbr = intel_dp->link_rate >= 1000000;
+	bool ok;
 
-	p = strim(kbuf);
+	if (uhbr)
+		ok = drm_dp_128b132b_lane_channel_eq_done(link_status,
+							  intel_dp->lane_count);
+	else
+		ok = drm_dp_channel_eq_ok(link_status, intel_dp->lane_count);
 
-	if (!strcmp(p, "auto")) {
-		rate = 0;
-	} else {
-		ret = kstrtoint(p, 0, &rate);
-		if (ret < 0)
-			goto out_free;
-
-		if (intel_dp_rate_index(intel_dp->source_rates,
-					intel_dp->num_source_rates,
-					rate) < 0)
-			ret = -EINVAL;
-	}
+	if (ok)
+		return true;
 
-out_free:
-	kfree(kbuf);
+	intel_dp_dump_link_status(intel_dp, DP_PHY_DPRX, link_status);
+	drm_dbg_kms(display->drm,
+		    "[ENCODER:%d:%s] %s link not ok, retraining\n",
+		    encoder->base.base.id, encoder->base.name,
+		    uhbr ? "128b/132b" : "8b/10b");
 
-	return ret < 0 ? ret : rate;
+	return false;
 }
 
-static ssize_t i915_dp_force_link_rate_write(struct file *file,
-					     const char __user *ubuf,
-					     size_t len, loff_t *offp)
+static int
+intel_dp_read_link_status(struct intel_dp *intel_dp, u8 link_status[DP_LINK_STATUS_SIZE])
 {
-	struct seq_file *m = file->private_data;
-	struct intel_connector *connector = to_intel_connector(m->private);
-	struct intel_display *display = to_intel_display(connector);
-	struct intel_dp *intel_dp = intel_attached_dp(connector);
-	int rate;
 	int err;
 
-	rate = parse_link_rate(intel_dp, ubuf, len);
-	if (rate < 0)
-		return rate;
+	memset(link_status, 0, DP_LINK_STATUS_SIZE);
+
+	if (intel_dp_mst_active_streams(intel_dp) > 0)
+		err = drm_dp_dpcd_read_data(&intel_dp->aux, DP_LANE0_1_STATUS_ESI,
+					    link_status, DP_LINK_STATUS_SIZE - 2);
+	else
+		err = drm_dp_dpcd_read_phy_link_status(&intel_dp->aux, DP_PHY_DPRX,
+						       link_status);
 
-	err = drm_modeset_lock_single_interruptible(&display->drm->mode_config.connection_mutex);
 	if (err)
 		return err;
 
-	intel_dp_reset_link_params(intel_dp);
-	intel_dp->link.force_rate = rate;
+	if (link_status[DP_LANE_ALIGN_STATUS_UPDATED - DP_LANE0_1_STATUS] &
+	    DP_DOWNSTREAM_PORT_STATUS_CHANGED)
+		WRITE_ONCE(intel_dp->downstream_port_changed, true);
 
-	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
+	return 0;
+}
 
-	*offp += len;
+bool intel_dp_link_training_get_force_retrain(struct intel_dp_link_training *link_training)
+{
+	return link_training->force_retrain;
+}
 
-	return len;
+static void intel_dp_link_training_set_force_retrain(struct intel_dp_link_training *link_training,
+						     bool forced)
+{
+	link_training->force_retrain = forced;
 }
-DEFINE_SHOW_STORE_ATTRIBUTE(i915_dp_force_link_rate);
 
-static int i915_dp_force_lane_count_show(struct seq_file *m, void *data)
+static bool
+intel_dp_needs_link_retrain(struct intel_dp *intel_dp)
 {
-	struct intel_connector *connector = to_intel_connector(m->private);
-	struct intel_display *display = to_intel_display(connector);
-	struct intel_dp *intel_dp = intel_attached_dp(connector);
-	int current_lane_count = -1;
-	int force_lane_count;
-	int err;
-	int i;
+	struct intel_dp_link_training *link_training = intel_dp->link.training;
+	u8 link_status[DP_LINK_STATUS_SIZE];
 
-	err = drm_modeset_lock_single_interruptible(&display->drm->mode_config.connection_mutex);
-	if (err)
-		return err;
+	if (!intel_dp->link.active)
+		return false;
 
-	if (intel_dp->link.active)
-		current_lane_count = intel_dp->lane_count;
-	force_lane_count = intel_dp->link.force_lane_count;
+	/*
+	 * While PSR source HW is enabled, it will control main-link sending
+	 * frames, enabling and disabling it so trying to do a retrain will fail
+	 * as the link would or not be on or it could mix training patterns
+	 * and frame data at the same time causing retrain to fail.
+	 * Also when exiting PSR, HW will retrain the link anyways fixing
+	 * any link status error.
+	 */
+	if (intel_psr_enabled(intel_dp))
+		return false;
 
-	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
+	if (intel_dp_link_training_get_force_retrain(link_training))
+		return true;
 
-	seq_printf(m, "%sauto%s",
-		   force_lane_count == 0 ? "[" : "",
-		   force_lane_count == 0 ? "]" : "");
+	if (intel_dp_read_link_status(intel_dp, link_status) < 0)
+		return false;
 
-	for (i = 1; i <= 4; i <<= 1)
-		seq_printf(m, " %s%d%s%s",
-			   i == force_lane_count ? "[" : "",
-			   i,
-			   i == current_lane_count ? "*" : "",
-			   i == force_lane_count ? "]" : "");
+	/*
+	 * Validate the cached values of intel_dp->link_rate and
+	 * intel_dp->lane_count before attempting to retrain.
+	 *
+	 * FIXME would be nice to user the crtc state here, but since
+	 * we need to call this from the short HPD handler that seems
+	 * a bit hard.
+	 */
+	if (!intel_dp_link_params_valid(intel_dp, intel_dp->link_rate,
+					intel_dp->lane_count))
+		return false;
 
-	seq_putc(m, '\n');
+	if (!link_recovery_autoretrain_allowed(link_training))
+		return false;
 
-	return 0;
+	if (link_recovery_autoretrain_pending(link_training))
+		return true;
+
+	/* Retrain if link not ok */
+	return !intel_dp_link_ok(intel_dp, link_status) &&
+		!intel_psr_link_ok(intel_dp);
 }
 
-static int parse_lane_count(const char __user *ubuf, size_t len)
+static bool intel_dp_is_connected(struct intel_dp *intel_dp)
 {
-	char *kbuf;
-	const char *p;
-	int lane_count;
-	int ret = 0;
-
-	kbuf = memdup_user_nul(ubuf, len);
-	if (IS_ERR(kbuf))
-		return PTR_ERR(kbuf);
+	struct intel_connector *connector = intel_dp->attached_connector;
 
-	p = strim(kbuf);
+	return connector->base.status == connector_status_connected ||
+		intel_dp->is_mst;
+}
 
-	if (!strcmp(p, "auto")) {
-		lane_count = 0;
-	} else {
-		ret = kstrtoint(p, 0, &lane_count);
-		if (ret < 0)
-			goto out_free;
-
-		switch (lane_count) {
-		case 1:
-		case 2:
-		case 4:
-			break;
-		default:
-			ret = -EINVAL;
-		}
-	}
+static void queue_modeset_retry_for_links_in_state(struct intel_atomic_state *state,
+						   struct intel_encoder *encoder,
+						   u8 pipe_mask)
+{
+	const struct intel_crtc_state *crtc_state;
+	struct intel_crtc *crtc;
 
-out_free:
-	kfree(kbuf);
+	for_each_new_intel_crtc_in_state(state, crtc, crtc_state) {
+		if (!(BIT(crtc->pipe) & pipe_mask))
+			continue;
 
-	return ret < 0 ? ret : lane_count;
+		intel_dp_queue_modeset_retry_for_link(state, encoder, crtc_state);
+	}
 }
 
-static ssize_t i915_dp_force_lane_count_write(struct file *file,
-					      const char __user *ubuf,
-					      size_t len, loff_t *offp)
+static int intel_dp_retrain_link(struct intel_encoder *encoder,
+				 struct drm_modeset_acquire_ctx *ctx)
 {
-	struct seq_file *m = file->private_data;
-	struct intel_connector *connector = to_intel_connector(m->private);
-	struct intel_display *display = to_intel_display(connector);
-	struct intel_dp *intel_dp = intel_attached_dp(connector);
-	int lane_count;
-	int err;
+	struct intel_display *display = to_intel_display(encoder);
+	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+	struct intel_dp_link_training *link_training =
+		intel_dp->link.training;
+	struct intel_atomic_state *state;
+	struct drm_atomic_commit *_state;
+	u8 pipe_mask;
+	int ret;
 
-	lane_count = parse_lane_count(ubuf, len);
-	if (lane_count < 0)
-		return lane_count;
+	if (!intel_dp_is_connected(intel_dp))
+		return 0;
 
-	err = drm_modeset_lock_single_interruptible(&display->drm->mode_config.connection_mutex);
-	if (err)
-		return err;
+	ret = drm_modeset_lock(&display->drm->mode_config.connection_mutex,
+			       ctx);
+	if (ret)
+		return ret;
 
-	intel_dp_reset_link_params(intel_dp);
-	intel_dp->link.force_lane_count = lane_count;
+	if (!intel_dp_needs_link_retrain(intel_dp))
+		return 0;
 
-	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
+	ret = intel_dp_get_active_pipes(intel_dp, ctx, &pipe_mask);
+	if (ret)
+		return ret;
 
-	*offp += len;
+	if (pipe_mask == 0)
+		return 0;
 
-	return len;
-}
-DEFINE_SHOW_STORE_ATTRIBUTE(i915_dp_force_lane_count);
+	if (!intel_dp_needs_link_retrain(intel_dp))
+		return 0;
 
-static int i915_dp_max_link_rate_show(void *data, u64 *val)
-{
-	struct intel_connector *connector = to_intel_connector(data);
-	struct intel_display *display = to_intel_display(connector);
-	struct intel_dp *intel_dp = intel_attached_dp(connector);
-	int err;
+	drm_dbg_kms(display->drm,
+		    "[ENCODER:%d:%s] retraining link (forced %s)\n",
+		    encoder->base.base.id, encoder->base.name,
+		    str_yes_no(intel_dp_link_training_get_force_retrain(link_training)));
 
-	err = drm_modeset_lock_single_interruptible(&display->drm->mode_config.connection_mutex);
-	if (err)
-		return err;
+	_state = drm_atomic_commit_alloc(display->drm);
+	if (!_state)
+		return -ENOMEM;
 
-	*val = intel_dp->link.max_rate;
+	state = to_intel_atomic_state(_state);
 
-	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
+	ret = intel_modeset_commit_pipes_for_atomic_state(state, pipe_mask, ctx);
+	if (ret == -EDEADLK)
+		goto out;
 
-	return 0;
+	intel_dp_link_training_set_force_retrain(link_training, false);
+
+	if (ret) {
+		drm_dbg_kms(display->drm,
+			    "[ENCODER:%d:%s] link retraining failed: %pe\n",
+			    encoder->base.base.id, encoder->base.name,
+			    ERR_PTR(ret));
+		/*
+		 * intel_dp_needs_link_retrain() only performs a coarse check of
+		 * retrainability, so the modeset commit may still fail. Disable
+		 * further auto-retrain attempts in that case.
+		 *
+		 * A sink capability change may restore the retrainable state (see
+		 * intel_dp_update_sink_caps(), intel_dp_reset_link_params()),
+		 * allowing retraining to be attempted again.
+		 */
+		link_recovery_mark_autoretrain_modeset_failure(link_training);
+		queue_modeset_retry_for_links_in_state(state, encoder, pipe_mask);
+	}
+out:
+	drm_atomic_commit_put(&state->base);
+
+	return ret;
 }
-DEFINE_DEBUGFS_ATTRIBUTE(i915_dp_max_link_rate_fops, i915_dp_max_link_rate_show, NULL, "%llu\n");
 
-static int i915_dp_max_lane_count_show(void *data, u64 *val)
+void intel_dp_link_check(struct intel_encoder *encoder)
 {
-	struct intel_connector *connector = to_intel_connector(data);
-	struct intel_display *display = to_intel_display(connector);
-	struct intel_dp *intel_dp = intel_attached_dp(connector);
-	int err;
+	struct drm_modeset_acquire_ctx ctx;
+	int ret;
 
-	err = drm_modeset_lock_single_interruptible(&display->drm->mode_config.connection_mutex);
-	if (err)
-		return err;
+	intel_modeset_lock_ctx_retry(&ctx, NULL, 0, ret)
+		ret = intel_dp_retrain_link(encoder, &ctx);
+}
 
-	*val = intel_dp->link.max_lane_count;
+void intel_dp_check_link_state(struct intel_dp *intel_dp)
+{
+	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+	struct intel_encoder *encoder = &dig_port->base;
 
-	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
+	if (!intel_dp_is_connected(intel_dp))
+		return;
 
-	return 0;
+	/*
+	 * NOTE:
+	 * This may race with an ongoing modeset updating the max link limits
+	 * and, with that, the link's retrainability, so
+	 * intel_dp_needs_link_retrain() may observe stale state.
+	 *
+	 * This is harmless: stale params captured as valid may spuriously
+	 * allow retraining here, but the decision is rechecked later in a
+	 * properly serialized context.
+	 *
+	 * Conversely, stale params captured as invalid may skip retraining,
+	 * but that can only happen before the modeset has completed its own
+	 * link training for the new, valid configuration, after which the
+	 * link state is rechecked.
+	 *
+	 * See intel_dp_link_params_valid() for capturing and validating the
+	 * params.
+	 */
+	if (!intel_dp_needs_link_retrain(intel_dp))
+		return;
+
+	intel_encoder_link_check_queue_work(encoder, 0);
 }
-DEFINE_DEBUGFS_ATTRIBUTE(i915_dp_max_lane_count_fops, i915_dp_max_lane_count_show, NULL, "%llu\n");
 
 static int i915_dp_force_link_training_failure_show(void *data, u64 *val)
 {
 	struct intel_connector *connector = to_intel_connector(data);
 	struct intel_display *display = to_intel_display(connector);
-	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	struct intel_dp_link_training *link_training = connector_to_link_training(connector);
 	int err;
 
 	err = drm_modeset_lock_single_interruptible(&display->drm->mode_config.connection_mutex);
 	if (err)
 		return err;
 
-	*val = intel_dp->link.force_train_failure;
+	intel_dp_flush_connector_commits(connector);
+
+	*val = link_training->force_train_failure;
 
 	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
 
@@ -2124,7 +2686,7 @@ static int i915_dp_force_link_training_failure_write(void *data, u64 val)
 {
 	struct intel_connector *connector = to_intel_connector(data);
 	struct intel_display *display = to_intel_display(connector);
-	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	struct intel_dp_link_training *link_training = connector_to_link_training(connector);
 	int err;
 
 	if (val > 2)
@@ -2134,7 +2696,9 @@ static int i915_dp_force_link_training_failure_write(void *data, u64 val)
 	if (err)
 		return err;
 
-	intel_dp->link.force_train_failure = val;
+	intel_dp_flush_connector_commits(connector);
+
+	link_training->force_train_failure = val;
 
 	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
 
@@ -2148,14 +2712,16 @@ static int i915_dp_force_link_retrain_show(void *data, u64 *val)
 {
 	struct intel_connector *connector = to_intel_connector(data);
 	struct intel_display *display = to_intel_display(connector);
-	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	struct intel_dp_link_training *link_training = connector_to_link_training(connector);
 	int err;
 
 	err = drm_modeset_lock_single_interruptible(&display->drm->mode_config.connection_mutex);
 	if (err)
 		return err;
 
-	*val = intel_dp->link.force_retrain;
+	intel_dp_flush_connector_commits(connector);
+
+	*val = intel_dp_link_training_get_force_retrain(link_training);
 
 	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
 
@@ -2166,14 +2732,17 @@ static int i915_dp_force_link_retrain_write(void *data, u64 val)
 {
 	struct intel_connector *connector = to_intel_connector(data);
 	struct intel_display *display = to_intel_display(connector);
-	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	struct intel_dp_link_training *link_training = connector_to_link_training(connector);
+	struct intel_dp *intel_dp = link_training->dp;
 	int err;
 
 	err = drm_modeset_lock_single_interruptible(&display->drm->mode_config.connection_mutex);
 	if (err)
 		return err;
 
-	intel_dp->link.force_retrain = val;
+	intel_dp_flush_connector_commits(connector);
+
+	intel_dp_link_training_set_force_retrain(link_training, val);
 
 	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
 
@@ -2189,14 +2758,17 @@ static int i915_dp_link_retrain_disabled_show(struct seq_file *m, void *data)
 {
 	struct intel_connector *connector = to_intel_connector(m->private);
 	struct intel_display *display = to_intel_display(connector);
-	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	struct intel_dp_link_training *link_training = connector_to_link_training(connector);
 	int err;
 
 	err = drm_modeset_lock_single_interruptible(&display->drm->mode_config.connection_mutex);
 	if (err)
 		return err;
 
-	seq_printf(m, "%s\n", str_yes_no(intel_dp->link.retrain_disabled));
+	intel_dp_flush_connector_commits(connector);
+
+	/* TODO: Expose this via a debugfs entry reflecting what the state represents. */
+	seq_printf(m, "%s\n", str_yes_no(link_recovery_has_no_fallback(link_training)));
 
 	drm_modeset_unlock(&display->drm->mode_config.connection_mutex);
 
@@ -2212,18 +2784,6 @@ void intel_dp_link_training_debugfs_add(struct intel_connector *connector)
 	    connector->base.connector_type != DRM_MODE_CONNECTOR_eDP)
 		return;
 
-	debugfs_create_file("i915_dp_force_link_rate", 0644, root,
-			    connector, &i915_dp_force_link_rate_fops);
-
-	debugfs_create_file("i915_dp_force_lane_count", 0644, root,
-			    connector, &i915_dp_force_lane_count_fops);
-
-	debugfs_create_file("i915_dp_max_link_rate", 0444, root,
-			    connector, &i915_dp_max_link_rate_fops);
-
-	debugfs_create_file("i915_dp_max_lane_count", 0444, root,
-			    connector, &i915_dp_max_lane_count_fops);
-
 	debugfs_create_file("i915_dp_force_link_training_failure", 0644, root,
 			    connector, &i915_dp_force_link_training_failure_fops);
 
@@ -2233,3 +2793,55 @@ void intel_dp_link_training_debugfs_add(struct intel_connector *connector)
 	debugfs_create_file("i915_dp_link_retrain_disabled", 0444, root,
 			    connector, &i915_dp_link_retrain_disabled_fops);
 }
+
+void intel_dp_link_training_reset(struct intel_dp_link_training *link_training)
+{
+	link_recovery_reset(link_training);
+}
+
+struct intel_dp_link_training *intel_dp_link_training_init(struct intel_dp *intel_dp)
+{
+	struct intel_dp_link_training *link_training;
+
+	link_training = kzalloc_obj(*link_training);
+	if (!link_training)
+		return NULL;
+
+	link_training->dp = intel_dp;
+
+	return link_training;
+}
+
+void intel_dp_link_training_cleanup(struct intel_dp_link_training *link_training)
+{
+	kfree(link_training);
+}
+
+#if IS_ENABLED(CONFIG_KUNIT)
+
+#define __INIT_MEMBER(__name, __fn) \
+	.__name = __fn,
+
+#define INTEL_DP_LINK_TRAINING_TEST_OPS_INIT \
+	INTEL_DP_LINK_TRAINING_TEST_OPS_MEMBERS(__INIT_MEMBER)
+
+#ifdef I915
+
+const struct intel_dp_link_training_test_ops i915_display_dp_link_training_test_ops = {
+	INTEL_DP_LINK_TRAINING_TEST_OPS_INIT
+};
+EXPORT_SYMBOL(i915_display_dp_link_training_test_ops);
+
+#else
+
+const struct intel_dp_link_training_test_ops intel_display_dp_link_training_test_ops = {
+	INTEL_DP_LINK_TRAINING_TEST_OPS_INIT
+};
+EXPORT_SYMBOL(intel_display_dp_link_training_test_ops);
+
+#endif	/* I915 */
+
+#undef INTEL_DP_LINK_TRAINING_TEST_OPS_INIT
+#undef __INIT_MEMBER
+
+#endif	/* CONFIG_KUNIT */
diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.h b/drivers/gpu/drm/i915/display/intel_dp_link_training.h
index 18c34c1a472fa..581f2361fdfd5 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_link_training.h
+++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.h
@@ -8,10 +8,14 @@
 
 #include <drm/display/drm_dp_helper.h>
 
+#include "intel_dp_link_caps.h"
+
 struct intel_atomic_state;
 struct intel_connector;
 struct intel_crtc_state;
 struct intel_dp;
+struct intel_dp_link_training;
+struct intel_encoder;
 
 int intel_dp_read_dprx_caps(struct intel_dp *intel_dp, u8 dpcd[DP_RECEIVER_CAP_SIZE]);
 int intel_dp_init_lttpr_and_dprx_caps(struct intel_dp *intel_dp);
@@ -54,6 +58,48 @@ static inline u8 intel_dp_training_pattern_symbol(u8 pattern)
 void intel_dp_128b132b_sdp_crc16(struct intel_dp *intel_dp,
 				 const struct intel_crtc_state *crtc_state);
 
+bool intel_dp_link_params_valid(struct intel_dp *intel_dp, int link_rate,
+				u8 lane_count);
+
+bool intel_dp_link_training_get_force_retrain(struct intel_dp_link_training *link_training);
+
+void intel_dp_link_check(struct intel_encoder *encoder);
+void intel_dp_check_link_state(struct intel_dp *intel_dp);
+
 void intel_dp_link_training_debugfs_add(struct intel_connector *connector);
 
+void intel_dp_link_training_reset(struct intel_dp_link_training *link_training);
+
+struct intel_dp_link_training *intel_dp_link_training_init(struct intel_dp *intel_dp);
+void intel_dp_link_training_cleanup(struct intel_dp_link_training *link_training);
+
+#if IS_ENABLED(CONFIG_KUNIT)
+
+int intel_dp_get_link_train_fallback_values(struct intel_dp *intel_dp,
+					    const struct intel_crtc_state *crtc_state);
+
+#define INTEL_DP_LINK_TRAINING_TEST_OPS_MEMBERS(__X) \
+	__X(get_fallback_values,	intel_dp_get_link_train_fallback_values)
+
+#define __DECLARE_MEMBER(__name, __fn) \
+	typeof(__fn) *__name;
+
+#define INTEL_DP_LINK_TRAINING_TEST_OPS_DECLARE \
+	INTEL_DP_LINK_TRAINING_TEST_OPS_MEMBERS(__DECLARE_MEMBER)
+
+struct intel_dp_link_training_test_ops {
+	INTEL_DP_LINK_TRAINING_TEST_OPS_DECLARE
+};
+
+#undef INTEL_DP_LINK_TRAINING_TEST_OPS_DECLARE
+#undef __DECLARE_MEMBER
+
+#ifdef I915
+extern const struct intel_dp_link_training_test_ops i915_display_dp_link_training_test_ops;
+#else
+extern const struct intel_dp_link_training_test_ops intel_display_dp_link_training_test_ops;
+#endif	/* I915 */
+
+#endif	/* CONFIG_KUNIT */
+
 #endif /* __INTEL_DP_LINK_TRAINING_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c
index 0aa3e6b4c781d..b5af011599496 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c
@@ -47,6 +47,7 @@
 #include "intel_display_wa.h"
 #include "intel_dp.h"
 #include "intel_dp_hdcp.h"
+#include "intel_dp_link_caps.h"
 #include "intel_dp_link_training.h"
 #include "intel_dp_mst.h"
 #include "intel_dp_test.h"
@@ -444,8 +445,20 @@ static int mst_stream_compute_link_config(struct intel_dp *intel_dp,
 					  struct drm_connector_state *conn_state,
 					  const struct link_config_limits *limits)
 {
-	crtc_state->lane_count = limits->max_lane_count;
-	crtc_state->port_clock = limits->max_rate;
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
+	struct intel_dp_link_config max_link_config;
+
+	/*
+	 * FIXME: Use a proper iteration over the link configurations, instead
+	 * of using only the max BW config. For instance UHBR rate configs may
+	 * have additional limitations over non-UHBR ones, due to the DSC DPT
+	 * bpp maximum limit.
+	 */
+	if (!intel_dp_get_connector_max_link_config(connector, limits, &max_link_config))
+		return -EINVAL;
+
+	crtc_state->port_clock = max_link_config.rate;
+	crtc_state->lane_count = max_link_config.lane_count;
 
 	/*
 	 * FIXME: allocate the BW according to link_bpp, which in the case of
@@ -464,6 +477,7 @@ static int mst_stream_dsc_compute_link_config(struct intel_dp *intel_dp,
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_connector *connector = to_intel_connector(conn_state->connector);
+	struct intel_dp_link_config max_link_config;
 
 	crtc_state->pipe_bpp = limits->pipe.max_bpp;
 
@@ -471,8 +485,17 @@ static int mst_stream_dsc_compute_link_config(struct intel_dp *intel_dp,
 		    "DSC Sink supported compressed min bpp " FXP_Q4_FMT " compressed max bpp " FXP_Q4_FMT "\n",
 		    FXP_Q4_ARGS(limits->link.min_bpp_x16), FXP_Q4_ARGS(limits->link.max_bpp_x16));
 
-	crtc_state->lane_count = limits->max_lane_count;
-	crtc_state->port_clock = limits->max_rate;
+	/*
+	 * FIXME: Use a proper iteration over the link configurations, instead
+	 * of using only the max BW config. For instance UHBR rate configs may
+	 * have additional limitations over non-UHBR ones, due to the DSC DPT
+	 * bpp maximum limit.
+	 */
+	if (!intel_dp_get_connector_max_link_config(connector, limits, &max_link_config))
+		return -EINVAL;
+
+	crtc_state->port_clock = max_link_config.rate;
+	crtc_state->lane_count = max_link_config.lane_count;
 
 	return intel_dp_mtp_tu_compute_config(intel_dp, crtc_state, conn_state,
 					      limits->link.min_bpp_x16,
@@ -488,6 +511,20 @@ static int mode_hblank_period_ns(const struct drm_display_mode *mode)
 				     mode->crtc_clock);
 }
 
+static int get_connector_max_rate(const struct intel_connector *connector,
+				  const struct link_config_limits *limits)
+{
+	struct intel_dp *intel_dp = intel_attached_dp((struct intel_connector *)connector);
+	struct intel_dp_link_caps *link_caps = intel_dp->link.caps;
+	struct intel_dp_link_config max_link_config;
+
+	intel_dp_link_caps_get_max_config(link_caps,
+					  INTEL_DP_LINK_CAPS_ORDER_KEY_RATE_LANE,
+					  limits->link_config_filter, &max_link_config);
+
+	return max_link_config.rate;
+}
+
 static bool
 hblank_expansion_quirk_needs_dsc(const struct intel_connector *connector,
 				 const struct intel_crtc_state *crtc_state,
@@ -498,11 +535,13 @@ hblank_expansion_quirk_needs_dsc(const struct intel_connector *connector,
 	bool is_uhbr_sink = connector->mst.dp &&
 			    drm_dp_128b132b_supported(connector->mst.dp->dpcd);
 	int hblank_limit = is_uhbr_sink ? 500 : 300;
+	int max_rate;
 
 	if (!connector->dp.dsc_hblank_expansion_quirk)
 		return false;
 
-	if (is_uhbr_sink && !drm_dp_is_uhbr_rate(limits->max_rate))
+	max_rate = get_connector_max_rate(connector, limits);
+	if (is_uhbr_sink && !drm_dp_is_uhbr_rate(max_rate))
 		return false;
 
 	if (mode_hblank_period_ns(adjusted_mode) > hblank_limit)
@@ -524,6 +563,7 @@ adjust_limits_for_dsc_hblank_expansion_quirk(struct intel_dp *intel_dp,
 	struct intel_display *display = to_intel_display(connector);
 	const struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	int min_bpp_x16 = limits->link.min_bpp_x16;
+	int max_rate;
 
 	if (!hblank_expansion_quirk_needs_dsc(connector, crtc_state, limits))
 		return true;
@@ -550,11 +590,10 @@ adjust_limits_for_dsc_hblank_expansion_quirk(struct intel_dp *intel_dp,
 		return true;
 	}
 
-	drm_WARN_ON(display->drm, limits->min_rate != limits->max_rate);
-
-	if (limits->max_rate < 540000)
+	max_rate = get_connector_max_rate(connector, limits);
+	if (max_rate < 540000)
 		min_bpp_x16 = fxp_q4_from_int(13);
-	else if (limits->max_rate < 810000)
+	else if (max_rate < 810000)
 		min_bpp_x16 = fxp_q4_from_int(10);
 
 	if (limits->link.min_bpp_x16 >= min_bpp_x16)
@@ -697,12 +736,12 @@ static int mst_stream_compute_link_for_joined_pipes(struct intel_encoder *encode
 	return 0;
 }
 
-static int mst_stream_compute_config(struct intel_encoder *encoder,
+static int mst_stream_compute_config(struct intel_atomic_state *state,
+				     struct intel_encoder *encoder,
 				     struct intel_crtc_state *pipe_config,
 				     struct drm_connector_state *conn_state)
 {
 	struct intel_display *display = to_intel_display(encoder);
-	struct intel_atomic_state *state = to_intel_atomic_state(conn_state->state);
 	struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
 	struct intel_dp *intel_dp = to_primary_dp(encoder);
 	struct intel_connector *connector =
@@ -776,7 +815,7 @@ intel_dp_mst_transcoder_mask(struct intel_atomic_state *state,
 	struct intel_display *display = to_intel_display(state);
 	const struct intel_digital_connector_state *conn_state;
 	struct intel_connector *connector;
-	u8 transcoders = 0;
+	u16 transcoders = 0;
 	int i;
 
 	if (DISPLAY_VER(display) < 12)
@@ -925,11 +964,11 @@ int intel_dp_mst_atomic_check_link(struct intel_atomic_state *state,
 	return 0;
 }
 
-static int mst_stream_compute_config_late(struct intel_encoder *encoder,
+static int mst_stream_compute_config_late(struct intel_atomic_state *state,
+					  struct intel_encoder *encoder,
 					  struct intel_crtc_state *crtc_state,
 					  struct drm_connector_state *conn_state)
 {
-	struct intel_atomic_state *state = to_intel_atomic_state(conn_state->state);
 	struct intel_dp *intel_dp = to_primary_dp(encoder);
 
 	/* lowest numbered transcoder will be designated master */
@@ -1476,6 +1515,7 @@ mst_connector_mode_valid_ctx(struct drm_connector *_connector,
 	unsigned long bw_overhead_flags =
 		DRM_DP_BW_OVERHEAD_MST | DRM_DP_BW_OVERHEAD_SSC_REF_CLK;
 	int min_link_bpp_x16 = fxp_q4_from_int(18);
+	struct intel_dp_link_config max_bw_config;
 	static bool supports_dsc;
 	int ret;
 	bool dsc = false;
@@ -1508,8 +1548,9 @@ mst_connector_mode_valid_ctx(struct drm_connector *_connector,
 		min_link_bpp_x16 = intel_dp_compute_min_compressed_bpp_x16(connector,
 									   INTEL_OUTPUT_FORMAT_RGB);
 
-	max_link_clock = intel_dp_max_link_rate(intel_dp);
-	max_lanes = intel_dp_max_lane_count(intel_dp);
+	intel_dp_link_caps_get_max_bw_config(intel_dp->link.caps, &max_bw_config);
+	max_link_clock = max_bw_config.rate;
+	max_lanes = max_bw_config.lane_count;
 
 	max_rate = intel_dp_max_link_data_rate(intel_dp,
 					       max_link_clock, max_lanes);
@@ -2135,14 +2176,19 @@ bool intel_dp_mst_crtc_needs_modeset(struct intel_atomic_state *state,
  */
 void intel_dp_mst_prepare_probe(struct intel_dp *intel_dp)
 {
-	int link_rate = intel_dp_max_link_rate(intel_dp);
-	int lane_count = intel_dp_max_lane_count(intel_dp);
+	struct intel_dp_link_config max_bw_config;
+	int link_rate;
+	int lane_count;
 	u8 rate_select;
 	u8 link_bw;
 
 	if (intel_dp->link.active)
 		return;
 
+	intel_dp_link_caps_get_max_bw_config(intel_dp->link.caps, &max_bw_config);
+	link_rate = max_bw_config.rate;
+	lane_count = max_bw_config.lane_count;
+
 	if (intel_mst_probed_link_params_valid(intel_dp, link_rate, lane_count))
 		return;
 
diff --git a/drivers/gpu/drm/i915/display/intel_dp_test.c b/drivers/gpu/drm/i915/display/intel_dp_test.c
index ba44769c9cfbe..0551a1ce60d39 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_test.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_test.c
@@ -14,6 +14,7 @@
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
 #include "intel_dp.h"
+#include "intel_dp_link_caps.h"
 #include "intel_dp_link_training.h"
 #include "intel_dp_mst.h"
 #include "intel_dp_test.h"
@@ -27,11 +28,92 @@ void intel_dp_test_reset(struct intel_dp *intel_dp)
 	memset(&intel_dp->compliance, 0, sizeof(intel_dp->compliance));
 }
 
+static bool set_filter_for_lane_count(struct intel_connector *connector,
+				      struct intel_dp_link_caps *link_caps,
+				      int lane_count,
+				      struct link_config_limits *limits)
+{
+	struct intel_dp_link_config link_config;
+	struct intel_dp_link_caps_order order =
+		intel_dp_link_caps_connector_compute_order(connector);
+	struct intel_dp_link_caps_filter new_filter = INTEL_DP_LINK_CAPS_FILTER_NONE;
+	struct intel_dp_link_caps_iter iter;
+	bool found = false;
+
+	intel_dp_link_caps_iter_start(&iter, link_caps, order, limits->link_config_filter);
+	for_each_dp_link_config(&iter, &link_config) {
+		if (link_config.lane_count != lane_count)
+			continue;
+
+		intel_dp_link_caps_filter_add(link_caps, &new_filter, &link_config);
+		found = true;
+	}
+	intel_dp_link_caps_iter_end(&iter);
+
+	if (!found)
+		return false;
+
+	limits->link_config_filter = new_filter;
+
+	return true;
+}
+
+static bool set_filter_for_link_config(struct intel_connector *connector,
+				       struct intel_dp_link_caps *link_caps,
+				       const struct intel_dp_link_config *link_params,
+				       struct link_config_limits *limits)
+{
+	struct intel_dp_link_caps_filter new_filter = INTEL_DP_LINK_CAPS_FILTER_NONE;
+
+	if (!intel_dp_link_caps_filter_add(link_caps, &new_filter, link_params))
+		return false;
+
+	limits->link_config_filter = new_filter;
+
+	return true;
+}
+
+static bool set_filter_for_link_params(struct intel_connector *connector,
+				       int link_rate, int lane_count,
+				       struct link_config_limits *limits)
+{
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
+	struct intel_dp_link_caps *link_caps = intel_dp->link.caps;
+	struct intel_display *display = to_intel_display(intel_dp);
+	struct intel_dp_link_config requested_config;
+
+	requested_config.rate = link_rate;
+	requested_config.lane_count = lane_count;
+
+	if (set_filter_for_link_config(connector, link_caps, &requested_config, limits))
+		return true;
+
+	/*
+	 * Preserve the legacy behavior: if the requested (rate, lane_count)
+	 * combination is not an allowed config, fall back to all configs
+	 * matching the requested lane count.
+	 *
+	 * TODO: Recheck whether this behavior is actually correct.
+	 */
+	if (set_filter_for_lane_count(connector, link_caps, lane_count, limits))
+		return true;
+
+	drm_dbg_kms(display->drm,
+		    "[ENCODER:%d:%s] Invalid autotest link config parameters: %dx%d\n",
+		    encoder->base.base.id, encoder->base.name,
+		    requested_config.lane_count,
+		    requested_config.rate);
+
+	return false;
+}
+
 /* Adjust link config limits based on compliance test requests. */
-void intel_dp_test_compute_config(struct intel_dp *intel_dp,
+bool intel_dp_test_compute_config(struct intel_connector *connector,
 				  struct intel_crtc_state *pipe_config,
 				  struct link_config_limits *limits)
 {
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
 	struct intel_display *display = to_intel_display(intel_dp);
 
 	/* For DP Compliance we override the computed bpp for the pipe */
@@ -47,24 +129,14 @@ void intel_dp_test_compute_config(struct intel_dp *intel_dp,
 
 	/* Use values requested by Compliance Test Request */
 	if (intel_dp->compliance.test_type == DP_TEST_LINK_TRAINING) {
-		int index;
-
-		/* Validate the compliance test data since max values
-		 * might have changed due to link train fallback.
-		 */
-		if (intel_dp_link_params_valid(intel_dp, intel_dp->compliance.test_link_rate,
-					       intel_dp->compliance.test_lane_count)) {
-			index = intel_dp_rate_index(intel_dp->common_rates,
-						    intel_dp->num_common_rates,
-						    intel_dp->compliance.test_link_rate);
-			if (index >= 0) {
-				limits->min_rate = intel_dp->compliance.test_link_rate;
-				limits->max_rate = intel_dp->compliance.test_link_rate;
-			}
-			limits->min_lane_count = intel_dp->compliance.test_lane_count;
-			limits->max_lane_count = intel_dp->compliance.test_lane_count;
-		}
+		if (!set_filter_for_link_params(connector,
+						intel_dp->compliance.test_link_rate,
+						intel_dp->compliance.test_lane_count,
+						limits))
+			return false;
 	}
+
+	return true;
 }
 
 /* Compliance test status bits  */
diff --git a/drivers/gpu/drm/i915/display/intel_dp_test.h b/drivers/gpu/drm/i915/display/intel_dp_test.h
index dcc167e4c7f65..a08f37a63dc9a 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_test.h
+++ b/drivers/gpu/drm/i915/display/intel_dp_test.h
@@ -6,6 +6,7 @@
 
 #include <linux/types.h>
 
+struct intel_connector;
 struct intel_crtc_state;
 struct intel_display;
 struct intel_dp;
@@ -13,7 +14,7 @@ struct link_config_limits;
 
 void intel_dp_test_reset(struct intel_dp *intel_dp);
 void intel_dp_test_request(struct intel_dp *intel_dp);
-void intel_dp_test_compute_config(struct intel_dp *intel_dp,
+bool intel_dp_test_compute_config(struct intel_connector *connector,
 				  struct intel_crtc_state *pipe_config,
 				  struct link_config_limits *limits);
 bool intel_dp_test_phy(struct intel_dp *intel_dp);
diff --git a/drivers/gpu/drm/i915/display/intel_dp_tunnel.c b/drivers/gpu/drm/i915/display/intel_dp_tunnel.c
index d6bd1f7e01e18..49fa4c9699b61 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_tunnel.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_tunnel.c
@@ -11,6 +11,7 @@
 #include "intel_display_limits.h"
 #include "intel_display_types.h"
 #include "intel_dp.h"
+#include "intel_dp_link_caps.h"
 #include "intel_dp_link_training.h"
 #include "intel_dp_mst.h"
 #include "intel_dp_tunnel.h"
@@ -56,10 +57,13 @@ static int kbytes_to_mbits(int kbytes)
 
 static int get_current_link_bw(struct intel_dp *intel_dp)
 {
-	int rate = intel_dp_max_common_rate(intel_dp);
-	int lane_count = intel_dp_max_common_lane_count(intel_dp);
+	struct intel_dp_link_caps *link_caps = intel_dp->link.caps;
+	struct intel_dp_link_config max_bw_config;
 
-	return intel_dp_max_link_data_rate(intel_dp, rate, lane_count);
+	intel_dp_link_caps_get_max_bw_config(link_caps, &max_bw_config);
+
+	return intel_dp_max_link_data_rate(intel_dp, max_bw_config.rate,
+						     max_bw_config.lane_count);
 }
 
 static int __update_tunnel_state(struct intel_dp *intel_dp, bool force_sink_update)
diff --git a/drivers/gpu/drm/i915/display/intel_dsb.c b/drivers/gpu/drm/i915/display/intel_dsb.c
index fec8a56e21eaf..d9a270362a82c 100644
--- a/drivers/gpu/drm/i915/display/intel_dsb.c
+++ b/drivers/gpu/drm/i915/display/intel_dsb.c
@@ -210,6 +210,18 @@ static int dsb_scanline_to_hw(struct intel_atomic_state *state,
 	return (scanline + vtotal - intel_crtc_scanline_offset(crtc_state)) % vtotal;
 }
 
+static
+bool pre_commit_use_safe_window(struct intel_atomic_state *state,
+				struct intel_crtc *crtc)
+{
+	struct intel_display *display = to_intel_display(crtc->base.dev);
+
+	if (intel_vrr_always_use_vrr_tg(display))
+		return true;
+
+	return pre_commit_is_vrr_active(state, crtc);
+}
+
 /*
  * Bspec suggests that we should always set DSB_SKIP_WAITS_EN. We have approach
  * different from what is explained in Bspec on how flip is considered being
@@ -229,7 +241,7 @@ static u32 dsb_chicken(struct intel_atomic_state *state,
 	u32 chicken = intel_psr_use_trans_push(new_crtc_state) ?
 		DSB_SKIP_WAITS_EN : 0;
 
-	if (pre_commit_is_vrr_active(state, crtc))
+	if (pre_commit_use_safe_window(state, crtc))
 		chicken |= DSB_CTRL_WAIT_SAFE_WINDOW |
 			DSB_CTRL_NO_WAIT_VBLANK |
 			DSB_INST_WAIT_SAFE_WINDOW |
@@ -798,6 +810,12 @@ void intel_dsb_vblank_evade(struct intel_atomic_state *state,
 		end = intel_vrr_vmax_vblank_start(crtc_state);
 		start = end - vblank_delay - latency;
 		intel_dsb_wait_scanline_out(state, dsb, start, end);
+	} else if (pre_commit_use_safe_window(state, crtc)) {
+		int vblank_delay = crtc_state->set_context_latency;
+
+		end = intel_mode_vblank_start(&crtc_state->hw.adjusted_mode);
+		start = end - vblank_delay - latency;
+		intel_dsb_wait_scanline_out(state, dsb, start, end);
 	} else {
 		int vblank_delay = intel_mode_vblank_delay(&crtc_state->hw.adjusted_mode);
 
@@ -891,7 +909,7 @@ void intel_dsb_wait_for_delayed_vblank(struct intel_atomic_state *state,
 		&crtc_state->hw.adjusted_mode;
 	int wait_scanlines;
 
-	if (pre_commit_is_vrr_active(state, crtc)) {
+	if (pre_commit_use_safe_window(state, crtc)) {
 		/*
 		 * If the push happened before the vmin decision boundary
 		 * we don't know how far we are from the undelayed vblank.
@@ -902,9 +920,17 @@ void intel_dsb_wait_for_delayed_vblank(struct intel_atomic_state *state,
 		 * the hardware itself guarantees that we're SCL lines
 		 * away from the delayed vblank, and we won't be inside
 		 * the vmin safe window so this extra wait does nothing.
+		 *
+		 * Experimentally, DSB may observe a slightly stale
+		 * PIPEDSL value. When the actual scanline has just reached
+		 * safe_window_start, WAIT_DSL_OUT may complete immediately
+		 * due to the stale value.
+		 *
+		 * Shift the start back by one scanline to ensure the wait
+		 * window is entered reliably.
 		 */
 		intel_dsb_wait_scanline_out(state, dsb,
-					    intel_vrr_safe_window_start(crtc_state),
+					    intel_vrr_safe_window_start(crtc_state) - 1,
 					    intel_vrr_vmin_safe_window_end(crtc_state));
 		/*
 		 * When the push is sent during vblank it will trigger
diff --git a/drivers/gpu/drm/i915/display/intel_dvo.c b/drivers/gpu/drm/i915/display/intel_dvo.c
index dd1a995c29799..f157699a7c4c4 100644
--- a/drivers/gpu/drm/i915/display/intel_dvo.c
+++ b/drivers/gpu/drm/i915/display/intel_dvo.c
@@ -242,7 +242,8 @@ intel_dvo_mode_valid(struct drm_connector *_connector,
 	return intel_dvo->dev.dev_ops->mode_valid(&intel_dvo->dev, mode);
 }
 
-static int intel_dvo_compute_config(struct intel_encoder *encoder,
+static int intel_dvo_compute_config(struct intel_atomic_state *state,
+				    struct intel_encoder *encoder,
 				    struct intel_crtc_state *pipe_config,
 				    struct drm_connector_state *conn_state)
 {
@@ -256,7 +257,7 @@ static int intel_dvo_compute_config(struct intel_encoder *encoder,
 	 * with the panel scaling set up to source from the H/VDisplay
 	 * of the original mode.
 	 */
-	ret = intel_panel_compute_config(connector, adjusted_mode);
+	ret = intel_panel_compute_config(state, pipe_config, connector);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c
index c8d4e3a5ce6b1..db0e36dd8722f 100644
--- a/drivers/gpu/drm/i915/display/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/display/intel_fbdev.c
@@ -262,6 +262,32 @@ __intel_fbdev_fb_alloc(struct intel_display *display,
 
 }
 
+static bool bios_fb_ok(const struct intel_framebuffer *fb,
+		       const struct drm_fb_helper_surface_size *sizes)
+{
+	struct intel_display *display = to_intel_display(fb->base.dev);
+	int width = fb->base.width;
+	int height = fb->base.height;
+	int depth = fb->base.format->depth;
+	int bpp = fb->base.format->cpp[0] * 8;
+
+	if (sizes->fb_width > width || sizes->fb_height > height) {
+		drm_dbg_kms(display->drm,
+			    "BIOS fb too small (%dx%d), we require (%dx%d), releasing it\n",
+			    width, height, sizes->fb_width, sizes->fb_height);
+		return false;
+	}
+
+	if (sizes->surface_depth != depth || sizes->surface_bpp != bpp) {
+		drm_dbg_kms(display->drm,
+			    "BIOS fb using wrong depth/bpp (%d/%d), we require (%d/%d), releasing it\n",
+			    depth, bpp, sizes->surface_depth, sizes->surface_bpp);
+		return false;
+	}
+
+	return true;
+}
+
 int intel_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper,
 				   struct drm_fb_helper_surface_size *sizes)
 {
@@ -279,14 +305,7 @@ int intel_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper,
 
 	ifbdev->fb = NULL;
 
-	if (fb &&
-	    (sizes->fb_width > fb->base.width ||
-	     sizes->fb_height > fb->base.height)) {
-		drm_dbg_kms(display->drm,
-			    "BIOS fb too small (%dx%d), we require (%dx%d),"
-			    " releasing it\n",
-			    fb->base.width, fb->base.height,
-			    sizes->fb_width, sizes->fb_height);
+	if (fb && !bios_fb_ok(fb, sizes)) {
 		drm_framebuffer_put(&fb->base);
 		fb = NULL;
 	}
diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.c b/drivers/gpu/drm/i915/display/intel_gmbus.c
index 049157c41fe25..60a70dea5d856 100644
--- a/drivers/gpu/drm/i915/display/intel_gmbus.c
+++ b/drivers/gpu/drm/i915/display/intel_gmbus.c
@@ -54,21 +54,21 @@ struct intel_gmbus {
 };
 
 enum gmbus_gpio {
-	GPIOA,
-	GPIOB,
-	GPIOC,
-	GPIOD,
-	GPIOE,
-	GPIOF,
-	GPIOG,
-	GPIOH,
-	__GPIOI_UNUSED,
-	GPIOJ,
-	GPIOK,
-	GPIOL,
-	GPIOM,
-	GPION,
-	GPIOO,
+	GPIO_0,
+	GPIO_1,
+	GPIO_2,
+	GPIO_3,
+	GPIO_4,
+	GPIO_5,
+	GPIO_6,
+	GPIO_7,
+	GPIO_8,
+	GPIO_9,
+	GPIO_10,
+	GPIO_11,
+	GPIO_12,
+	GPIO_13,
+	GPIO_14,
 };
 
 struct gmbus_pin {
@@ -78,77 +78,82 @@ struct gmbus_pin {
 
 /* Map gmbus pin pairs to names and registers. */
 static const struct gmbus_pin gmbus_pins[] = {
-	[GMBUS_PIN_SSC] = { "ssc", GPIOB },
-	[GMBUS_PIN_VGADDC] = { "vga", GPIOA },
-	[GMBUS_PIN_PANEL] = { "panel", GPIOC },
-	[GMBUS_PIN_DPC] = { "dpc", GPIOD },
-	[GMBUS_PIN_DPB] = { "dpb", GPIOE },
-	[GMBUS_PIN_DPD] = { "dpd", GPIOF },
+	[GMBUS_PIN_SSC] = { "ssc", GPIO_1 },
+	[GMBUS_PIN_VGADDC] = { "vga", GPIO_0 },
+	[GMBUS_PIN_PANEL] = { "panel", GPIO_2 },
+	[GMBUS_PIN_DPC] = { "dpc", GPIO_3 },
+	[GMBUS_PIN_DPB] = { "dpb", GPIO_4 },
+	[GMBUS_PIN_DPD] = { "dpd", GPIO_5 },
 };
 
-static const struct gmbus_pin gmbus_pins_bdw[] = {
-	[GMBUS_PIN_VGADDC] = { "vga", GPIOA },
-	[GMBUS_PIN_DPC] = { "dpc", GPIOD },
-	[GMBUS_PIN_DPB] = { "dpb", GPIOE },
-	[GMBUS_PIN_DPD] = { "dpd", GPIOF },
+static const struct gmbus_pin gmbus_pins_lpt_h[] = {
+	[GMBUS_PIN_VGADDC] = { "vga", GPIO_0 },
+	[GMBUS_PIN_DPC] = { "dpc", GPIO_3 },
+	[GMBUS_PIN_DPB] = { "dpb", GPIO_4 },
+	[GMBUS_PIN_DPD] = { "dpd", GPIO_5 },
 };
 
-static const struct gmbus_pin gmbus_pins_skl[] = {
-	[GMBUS_PIN_DPC] = { "dpc", GPIOD },
-	[GMBUS_PIN_DPB] = { "dpb", GPIOE },
-	[GMBUS_PIN_DPD] = { "dpd", GPIOF },
+static const struct gmbus_pin gmbus_pins_lpt_lp[] = {
+	[GMBUS_PIN_DPC] = { "dpc", GPIO_3 },
+	[GMBUS_PIN_DPB] = { "dpb", GPIO_4 },
+};
+
+static const struct gmbus_pin gmbus_pins_spt[] = {
+	[GMBUS_PIN_DPC] = { "dpc", GPIO_3 },
+	[GMBUS_PIN_DPB] = { "dpb", GPIO_4 },
+	[GMBUS_PIN_DPD] = { "dpd", GPIO_5 },
 };
 
 static const struct gmbus_pin gmbus_pins_bxt[] = {
-	[GMBUS_PIN_1_BXT] = { "dpb", GPIOB },
-	[GMBUS_PIN_2_BXT] = { "dpc", GPIOC },
-	[GMBUS_PIN_3_BXT] = { "misc", GPIOD },
+	[GMBUS_PIN_1] = { "dpb", GPIO_1 },
+	[GMBUS_PIN_2] = { "dpc", GPIO_2 },
+	[GMBUS_PIN_3] = { "misc", GPIO_3 },
 };
 
 static const struct gmbus_pin gmbus_pins_cnp[] = {
-	[GMBUS_PIN_1_BXT] = { "dpb", GPIOB },
-	[GMBUS_PIN_2_BXT] = { "dpc", GPIOC },
-	[GMBUS_PIN_3_BXT] = { "misc", GPIOD },
-	[GMBUS_PIN_4_CNP] = { "dpd", GPIOE },
+	[GMBUS_PIN_1] = { "dpb", GPIO_1 },
+	[GMBUS_PIN_2] = { "dpc", GPIO_2 },
+	[GMBUS_PIN_3] = { "misc", GPIO_3 },
+	[GMBUS_PIN_4] = { "dpd", GPIO_4 },
 };
 
 static const struct gmbus_pin gmbus_pins_icp[] = {
-	[GMBUS_PIN_1_BXT] = { "dpa", GPIOB },
-	[GMBUS_PIN_2_BXT] = { "dpb", GPIOC },
-	[GMBUS_PIN_3_BXT] = { "dpc", GPIOD },
-	[GMBUS_PIN_9_TC1_ICP] = { "tc1", GPIOJ },
-	[GMBUS_PIN_10_TC2_ICP] = { "tc2", GPIOK },
-	[GMBUS_PIN_11_TC3_ICP] = { "tc3", GPIOL },
-	[GMBUS_PIN_12_TC4_ICP] = { "tc4", GPIOM },
-	[GMBUS_PIN_13_TC5_TGP] = { "tc5", GPION },
-	[GMBUS_PIN_14_TC6_TGP] = { "tc6", GPIOO },
+	[GMBUS_PIN_1] = { "dpa", GPIO_1 },
+	[GMBUS_PIN_2] = { "dpb", GPIO_2 },
+	[GMBUS_PIN_3] = { "dpc", GPIO_3 },
+	[GMBUS_PIN_9_TC1] = { "tc1", GPIO_9 },
+	[GMBUS_PIN_10_TC2] = { "tc2", GPIO_10 },
+	[GMBUS_PIN_11_TC3] = { "tc3", GPIO_11 },
+	[GMBUS_PIN_12_TC4] = { "tc4", GPIO_12 },
+	[GMBUS_PIN_13_TC5] = { "tc5", GPIO_13 },
+	[GMBUS_PIN_14_TC6] = { "tc6", GPIO_14 },
 };
 
 static const struct gmbus_pin gmbus_pins_dg1[] = {
-	[GMBUS_PIN_1_BXT] = { "dpa", GPIOB },
-	[GMBUS_PIN_2_BXT] = { "dpb", GPIOC },
-	[GMBUS_PIN_3_BXT] = { "dpc", GPIOD },
-	[GMBUS_PIN_4_CNP] = { "dpd", GPIOE },
+	[GMBUS_PIN_1] = { "dpa", GPIO_1 },
+	[GMBUS_PIN_2] = { "dpb", GPIO_2 },
+	[GMBUS_PIN_3] = { "dpc", GPIO_3 },
+	[GMBUS_PIN_4] = { "dpd", GPIO_4 },
 };
 
 static const struct gmbus_pin gmbus_pins_dg2[] = {
-	[GMBUS_PIN_1_BXT] = { "dpa", GPIOB },
-	[GMBUS_PIN_2_BXT] = { "dpb", GPIOC },
-	[GMBUS_PIN_3_BXT] = { "dpc", GPIOD },
-	[GMBUS_PIN_4_CNP] = { "dpd", GPIOE },
-	[GMBUS_PIN_9_TC1_ICP] = { "tc1", GPIOJ },
+	[GMBUS_PIN_1] = { "dpa", GPIO_1 },
+	[GMBUS_PIN_2] = { "dpb", GPIO_2 },
+	[GMBUS_PIN_3] = { "dpc", GPIO_3 },
+	[GMBUS_PIN_4] = { "dpd", GPIO_4 },
+	[GMBUS_PIN_9_TC1] = { "tc1", GPIO_9 },
 };
 
 static const struct gmbus_pin gmbus_pins_mtp[] = {
-	[GMBUS_PIN_1_BXT] = { "dpa", GPIOB },
-	[GMBUS_PIN_2_BXT] = { "dpb", GPIOC },
-	[GMBUS_PIN_3_BXT] = { "dpc", GPIOD },
-	[GMBUS_PIN_4_CNP] = { "dpd", GPIOE },
-	[GMBUS_PIN_5_MTP] = { "dpe", GPIOF },
-	[GMBUS_PIN_9_TC1_ICP] = { "tc1", GPIOJ },
-	[GMBUS_PIN_10_TC2_ICP] = { "tc2", GPIOK },
-	[GMBUS_PIN_11_TC3_ICP] = { "tc3", GPIOL },
-	[GMBUS_PIN_12_TC4_ICP] = { "tc4", GPIOM },
+	[GMBUS_PIN_1] = { "dpa", GPIO_1 },
+	[GMBUS_PIN_2] = { "dpb", GPIO_2 },
+	[GMBUS_PIN_3] = { "dpc", GPIO_3 },
+	[GMBUS_PIN_4] = { "dpd", GPIO_4 },
+	[GMBUS_PIN_5] = { "dpe", GPIO_5 },
+	[GMBUS_PIN_9_TC1] = { "tc1", GPIO_9 },
+	[GMBUS_PIN_10_TC2] = { "tc2", GPIO_10 },
+	[GMBUS_PIN_11_TC3] = { "tc3", GPIO_11 },
+	[GMBUS_PIN_12_TC4] = { "tc4", GPIO_12 },
 };
 
 static const struct gmbus_pin *get_gmbus_pin(struct intel_display *display,
@@ -175,12 +180,15 @@ static const struct gmbus_pin *get_gmbus_pin(struct intel_display *display,
 	} else if (display->platform.geminilake || display->platform.broxton) {
 		pins = gmbus_pins_bxt;
 		size = ARRAY_SIZE(gmbus_pins_bxt);
-	} else if (DISPLAY_VER(display) == 9) {
-		pins = gmbus_pins_skl;
-		size = ARRAY_SIZE(gmbus_pins_skl);
-	} else if (display->platform.broadwell) {
-		pins = gmbus_pins_bdw;
-		size = ARRAY_SIZE(gmbus_pins_bdw);
+	} else if (HAS_PCH_SPT(display)) {
+		pins = gmbus_pins_spt;
+		size = ARRAY_SIZE(gmbus_pins_spt);
+	} else if (HAS_PCH_LPT_LP(display)) {
+		pins = gmbus_pins_lpt_lp;
+		size = ARRAY_SIZE(gmbus_pins_lpt_lp);
+	} else if (HAS_PCH_LPT(display)) {
+		pins = gmbus_pins_lpt_h;
+		size = ARRAY_SIZE(gmbus_pins_lpt_h);
 	} else {
 		pins = gmbus_pins;
 		size = ARRAY_SIZE(gmbus_pins);
diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.h b/drivers/gpu/drm/i915/display/intel_gmbus.h
index 35a200a9efc00..5fdeab1aa794f 100644
--- a/drivers/gpu/drm/i915/display/intel_gmbus.h
+++ b/drivers/gpu/drm/i915/display/intel_gmbus.h
@@ -20,17 +20,17 @@ struct intel_display;
 #define GMBUS_PIN_DPB		5 /* SDVO, HDMIB */
 #define GMBUS_PIN_DPD		6 /* HDMID */
 #define GMBUS_PIN_RESERVED	7 /* 7 reserved */
-#define GMBUS_PIN_1_BXT		1 /* BXT+ (atom) and CNP+ (big core) */
-#define GMBUS_PIN_2_BXT		2
-#define GMBUS_PIN_3_BXT		3
-#define GMBUS_PIN_4_CNP		4
-#define GMBUS_PIN_5_MTP		5
-#define GMBUS_PIN_9_TC1_ICP	9
-#define GMBUS_PIN_10_TC2_ICP	10
-#define GMBUS_PIN_11_TC3_ICP	11
-#define GMBUS_PIN_12_TC4_ICP	12
-#define GMBUS_PIN_13_TC5_TGP	13
-#define GMBUS_PIN_14_TC6_TGP	14
+#define GMBUS_PIN_1		1 /* BXT+ (atom) and CNP+ (big core) */
+#define GMBUS_PIN_2		2
+#define GMBUS_PIN_3		3
+#define GMBUS_PIN_4		4
+#define GMBUS_PIN_5		5
+#define GMBUS_PIN_9_TC1		9 /* ICP+ */
+#define GMBUS_PIN_10_TC2	10
+#define GMBUS_PIN_11_TC3	11
+#define GMBUS_PIN_12_TC4	12
+#define GMBUS_PIN_13_TC5	13
+#define GMBUS_PIN_14_TC6	14
 
 #define GMBUS_NUM_PINS	15 /* including 0 */
 
diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c
index 9076c2b176ec7..8a019d3574df2 100644
--- a/drivers/gpu/drm/i915/display/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/display/intel_hdmi.c
@@ -71,7 +71,7 @@
 
 bool intel_hdmi_is_frl(u32 clock)
 {
-	u32 rates[] = { 300000, 600000, 800000, 1000000, 1200000 };
+	static const u32 rates[] = { 300000, 600000, 800000, 1000000, 1200000 };
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(rates); i++)
@@ -2728,6 +2728,32 @@ intel_hdmi_add_properties(struct intel_hdmi *intel_hdmi, struct drm_connector *_
 		drm_connector_attach_max_bpc_property(&connector->base, 8, 12);
 }
 
+/*
+ * HDMI 2.0 spec, section 6.1.3.1 (Scrambling Control): after
+ * enabling Scrambling_Enable and starting scrambled video
+ * transmission, poll Scrambling_Status for up to 200 ms.
+ */
+void
+intel_hdmi_poll_for_scrambling_enable(const struct intel_crtc_state *crtc_state,
+				      struct drm_connector *_connector)
+{
+	struct intel_connector *connector = to_intel_connector(_connector);
+	struct intel_display *display = to_intel_display(crtc_state);
+	bool scrambling_enabled = false;
+	int ret;
+
+	if (!crtc_state->hdmi_scrambling)
+		return;
+
+	/* Poll for a max of 200 msec as per HDMI spec */
+	ret = poll_timeout_us(scrambling_enabled = drm_scdc_get_scrambling_status(&connector->base),
+			      scrambling_enabled, 1000, 200 * 1000, false);
+	if (ret)
+		drm_dbg_kms(display->drm,
+			    "[CONNECTOR:%d:%s] Timed out waiting for scrambling enable\n",
+			    connector->base.base.id, connector->base.name);
+}
+
 /*
  * intel_hdmi_handle_sink_scrambling: handle sink scrambling/clock ratio setup
  * @encoder: intel_encoder
@@ -2799,14 +2825,14 @@ static u8 bxt_encoder_to_ddc_pin(struct intel_encoder *encoder)
 
 	switch (port) {
 	case PORT_B:
-		ddc_pin = GMBUS_PIN_1_BXT;
+		ddc_pin = GMBUS_PIN_1;
 		break;
 	case PORT_C:
-		ddc_pin = GMBUS_PIN_2_BXT;
+		ddc_pin = GMBUS_PIN_2;
 		break;
 	default:
 		MISSING_CASE(port);
-		ddc_pin = GMBUS_PIN_1_BXT;
+		ddc_pin = GMBUS_PIN_1;
 		break;
 	}
 	return ddc_pin;
@@ -2819,20 +2845,17 @@ static u8 cnp_encoder_to_ddc_pin(struct intel_encoder *encoder)
 
 	switch (port) {
 	case PORT_B:
-		ddc_pin = GMBUS_PIN_1_BXT;
+		ddc_pin = GMBUS_PIN_1;
 		break;
 	case PORT_C:
-		ddc_pin = GMBUS_PIN_2_BXT;
+		ddc_pin = GMBUS_PIN_2;
 		break;
 	case PORT_D:
-		ddc_pin = GMBUS_PIN_4_CNP;
-		break;
-	case PORT_F:
-		ddc_pin = GMBUS_PIN_3_BXT;
+		ddc_pin = GMBUS_PIN_4;
 		break;
 	default:
 		MISSING_CASE(port);
-		ddc_pin = GMBUS_PIN_1_BXT;
+		ddc_pin = GMBUS_PIN_1;
 		break;
 	}
 	return ddc_pin;
@@ -2844,12 +2867,12 @@ static u8 icl_encoder_to_ddc_pin(struct intel_encoder *encoder)
 	enum port port = encoder->port;
 
 	if (intel_encoder_is_combo(encoder))
-		return GMBUS_PIN_1_BXT + port;
+		return GMBUS_PIN_1 + port;
 	else if (intel_encoder_is_tc(encoder))
-		return GMBUS_PIN_9_TC1_ICP + intel_encoder_to_tc(encoder);
+		return GMBUS_PIN_9_TC1 + intel_encoder_to_tc(encoder);
 
 	drm_WARN(display->drm, 1, "Unknown port:%c\n", port_name(port));
-	return GMBUS_PIN_2_BXT;
+	return GMBUS_PIN_2;
 }
 
 static u8 mcc_encoder_to_ddc_pin(struct intel_encoder *encoder)
@@ -2859,17 +2882,17 @@ static u8 mcc_encoder_to_ddc_pin(struct intel_encoder *encoder)
 
 	switch (phy) {
 	case PHY_A:
-		ddc_pin = GMBUS_PIN_1_BXT;
+		ddc_pin = GMBUS_PIN_1;
 		break;
 	case PHY_B:
-		ddc_pin = GMBUS_PIN_2_BXT;
+		ddc_pin = GMBUS_PIN_2;
 		break;
 	case PHY_C:
-		ddc_pin = GMBUS_PIN_9_TC1_ICP;
+		ddc_pin = GMBUS_PIN_9_TC1;
 		break;
 	default:
 		MISSING_CASE(phy);
-		ddc_pin = GMBUS_PIN_1_BXT;
+		ddc_pin = GMBUS_PIN_1;
 		break;
 	}
 	return ddc_pin;
@@ -2889,9 +2912,9 @@ static u8 rkl_encoder_to_ddc_pin(struct intel_encoder *encoder)
 	 * all outputs.
 	 */
 	if (INTEL_PCH_TYPE(display) >= PCH_TGP && phy >= PHY_C)
-		return GMBUS_PIN_9_TC1_ICP + phy - PHY_C;
+		return GMBUS_PIN_9_TC1 + phy - PHY_C;
 
-	return GMBUS_PIN_1_BXT + phy;
+	return GMBUS_PIN_1 + phy;
 }
 
 static u8 gen9bc_tgp_encoder_to_ddc_pin(struct intel_encoder *encoder)
@@ -2908,9 +2931,9 @@ static u8 gen9bc_tgp_encoder_to_ddc_pin(struct intel_encoder *encoder)
 	 * all outputs.
 	 */
 	if (INTEL_PCH_TYPE(display) >= PCH_TGP && phy >= PHY_C)
-		return GMBUS_PIN_9_TC1_ICP + phy - PHY_C;
+		return GMBUS_PIN_9_TC1 + phy - PHY_C;
 
-	return GMBUS_PIN_1_BXT + phy;
+	return GMBUS_PIN_1 + phy;
 }
 
 static u8 dg1_encoder_to_ddc_pin(struct intel_encoder *encoder)
@@ -2929,9 +2952,9 @@ static u8 adls_encoder_to_ddc_pin(struct intel_encoder *encoder)
 	 * except first combo output.
 	 */
 	if (phy == PHY_A)
-		return GMBUS_PIN_1_BXT;
+		return GMBUS_PIN_1;
 
-	return GMBUS_PIN_9_TC1_ICP + phy - PHY_B;
+	return GMBUS_PIN_9_TC1 + phy - PHY_B;
 }
 
 static u8 g4x_encoder_to_ddc_pin(struct intel_encoder *encoder)
diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.h b/drivers/gpu/drm/i915/display/intel_hdmi.h
index be2fad57e4ad0..0fa3661568e86 100644
--- a/drivers/gpu/drm/i915/display/intel_hdmi.h
+++ b/drivers/gpu/drm/i915/display/intel_hdmi.h
@@ -70,5 +70,7 @@ void hsw_read_infoframe(struct intel_encoder *encoder,
 			const struct intel_crtc_state *crtc_state,
 			unsigned int type,
 			void *frame, ssize_t len);
+void intel_hdmi_poll_for_scrambling_enable(const struct intel_crtc_state *crtc_state,
+					   struct drm_connector *_connector);
 
 #endif /* __INTEL_HDMI_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_initial_plane.c b/drivers/gpu/drm/i915/display/intel_initial_plane.c
index 6aa253678ec94..7427f41e4d7e1 100644
--- a/drivers/gpu/drm/i915/display/intel_initial_plane.c
+++ b/drivers/gpu/drm/i915/display/intel_initial_plane.c
@@ -128,11 +128,10 @@ intel_find_initial_plane_obj(struct intel_crtc *crtc,
 
 	/*
 	 * TODO:
-	 *   Disable planes if get_initial_plane_config() failed.
 	 *   Make sure things work if the surface base is not page aligned.
 	 */
 	if (!plane_config->fb)
-		return;
+		goto nofb;
 
 	if (intel_alloc_initial_plane_obj(display, plane_config)) {
 		fb = plane_config->fb;
@@ -170,7 +169,7 @@ intel_find_initial_plane_obj(struct intel_crtc *crtc,
 	drm_framebuffer_get(fb);
 
 	plane_state->uapi.crtc = &crtc->base;
-	intel_plane_copy_uapi_to_hw_state(plane_state, plane_state, crtc);
+	intel_plane_copy_uapi_to_hw_state(NULL, plane_state, plane_state, crtc);
 
 	atomic_or(plane->frontbuffer_bit, &to_intel_frontbuffer(fb)->bits);
 
@@ -184,7 +183,8 @@ intel_find_initial_plane_obj(struct intel_crtc *crtc,
 	 * simplest solution is to just disable the primary plane now and
 	 * pretend the BIOS never had it enabled.
 	 */
-	intel_plane_disable_noatomic(crtc, plane);
+	if (plane_state->uapi.visible)
+		intel_plane_disable_noatomic(crtc, plane);
 }
 
 static void plane_config_fini(struct intel_display *display,
diff --git a/drivers/gpu/drm/i915/display/intel_lpe_audio.c b/drivers/gpu/drm/i915/display/intel_lpe_audio.c
index 775493306a833..ff2cf479d8e18 100644
--- a/drivers/gpu/drm/i915/display/intel_lpe_audio.c
+++ b/drivers/gpu/drm/i915/display/intel_lpe_audio.c
@@ -262,10 +262,10 @@ void intel_lpe_audio_irq_handler(struct intel_display *display)
 	if (!HAS_LPE_AUDIO(display))
 		return;
 
-	ret = generic_handle_irq(display->audio.lpe.irq);
+	ret = generic_handle_irq_safe(display->audio.lpe.irq);
 	if (ret)
 		drm_err_ratelimited(display->drm,
-				    "error handling LPE audio irq: %d\n", ret);
+				    "LPE audio: irq handling failed (%pe)\n", ERR_PTR(ret));
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/display/intel_lt_phy.c b/drivers/gpu/drm/i915/display/intel_lt_phy.c
index 34dbe450cc5b5..8fc6d230493f7 100644
--- a/drivers/gpu/drm/i915/display/intel_lt_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_lt_phy.c
@@ -2174,8 +2174,9 @@ void intel_lt_phy_dump_hw_state(struct drm_printer *p,
 {
 	int i, j;
 
-	drm_printf(p, "lt_phy_pll_hw_state: lane count: %d, ssc enabled: %d, tbt mode: %d\n",
-		   hw_state->lane_count, hw_state->ssc_enabled, hw_state->tbt_mode);
+	drm_printf(p, "lt_phy_pll_hw_state: lane count: %d, ssc enabled: %s, tbt mode: %s\n",
+		   hw_state->lane_count, str_yes_no(hw_state->ssc_enabled),
+		   str_yes_no(hw_state->tbt_mode));
 
 	for (i = 0; i < 3; i++) {
 		drm_printf(p, "config[%d] = 0x%.4x,\n",
@@ -2217,6 +2218,14 @@ static bool intel_lt_phy_pll_is_enabled(struct intel_encoder *encoder)
 			     XELPDP_LANE_PCLK_PLL_ACK(0);
 }
 
+static bool readout_ssc_state(struct intel_encoder *encoder)
+{
+	struct intel_display *display = to_intel_display(encoder);
+
+	return intel_de_read(display, XELPDP_PORT_CLOCK_CTL(display, encoder->port)) &
+		XELPDP_SSC_ENABLE_PLLA;
+}
+
 bool intel_lt_phy_tbt_pll_readout_hw_state(struct intel_display *display,
 					   struct intel_dpll *pll,
 					   struct intel_dpll_hw_state *hw_state)
@@ -2246,6 +2255,7 @@ bool intel_lt_phy_pll_readout_hw_state(struct intel_encoder *encoder,
 	owned_lane_mask = intel_lt_phy_get_owned_lane_mask(encoder);
 	lane = owned_lane_mask & INTEL_LT_PHY_LANE0 ? : INTEL_LT_PHY_LANE1;
 	wakeref = intel_lt_phy_transaction_begin(encoder);
+	pll_state->ssc_enabled = readout_ssc_state(encoder);
 
 	pll_state->lane_count = intel_readout_lane_count(encoder, INTEL_LT_PHY_LANE0,
 							 INTEL_LT_PHY_LANE1);
diff --git a/drivers/gpu/drm/i915/display/intel_lvds.c b/drivers/gpu/drm/i915/display/intel_lvds.c
index c8098104d853a..872753478cf21 100644
--- a/drivers/gpu/drm/i915/display/intel_lvds.c
+++ b/drivers/gpu/drm/i915/display/intel_lvds.c
@@ -413,7 +413,8 @@ intel_lvds_mode_valid(struct drm_connector *_connector,
 	return MODE_OK;
 }
 
-static int intel_lvds_compute_config(struct intel_encoder *encoder,
+static int intel_lvds_compute_config(struct intel_atomic_state *state,
+				     struct intel_encoder *encoder,
 				     struct intel_crtc_state *crtc_state,
 				     struct drm_connector_state *conn_state)
 {
@@ -459,7 +460,7 @@ static int intel_lvds_compute_config(struct intel_encoder *encoder,
 	 * with the panel scaling set up to source from the H/VDisplay
 	 * of the original mode.
 	 */
-	ret = intel_panel_compute_config(connector, adjusted_mode);
+	ret = intel_panel_compute_config(state, crtc_state, connector);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/i915/display/intel_modeset_setup.c b/drivers/gpu/drm/i915/display/intel_modeset_setup.c
index e8730b5baf2a4..e27a531e1aa2f 100644
--- a/drivers/gpu/drm/i915/display/intel_modeset_setup.c
+++ b/drivers/gpu/drm/i915/display/intel_modeset_setup.c
@@ -186,11 +186,11 @@ static void intel_crtc_disable_noatomic_complete(struct intel_crtc *crtc)
  * Return all the pipes using a transcoder in @transcoder_mask.
  * For joiner configs return only the joiner primary.
  */
-static u8 get_transcoder_pipes(struct intel_display *display,
-			       u8 transcoder_mask)
+static u16 get_transcoder_pipes(struct intel_display *display,
+				u16 transcoder_mask)
 {
 	struct intel_crtc *temp_crtc;
-	u8 pipes = 0;
+	u16 pipes = 0;
 
 	for_each_intel_crtc(display, temp_crtc) {
 		struct intel_crtc_state *temp_crtc_state =
@@ -214,7 +214,7 @@ static u8 get_transcoder_pipes(struct intel_display *display,
  * For joiner configs return only the joiner primary pipes.
  */
 static void get_portsync_pipes(struct intel_crtc *crtc,
-			       u8 *master_pipe_mask, u8 *slave_pipes_mask)
+			       u16 *master_pipe_mask, u16 *slave_pipes_mask)
 {
 	struct intel_display *display = to_intel_display(crtc);
 	struct intel_crtc_state *crtc_state =
@@ -243,10 +243,10 @@ static void get_portsync_pipes(struct intel_crtc *crtc,
 	*slave_pipes_mask = get_transcoder_pipes(display, master_crtc_state->sync_mode_slaves_mask);
 }
 
-static u8 get_joiner_secondary_pipes(struct intel_display *display, u8 primary_pipes_mask)
+static u16 get_joiner_secondary_pipes(struct intel_display *display, u16 primary_pipes_mask)
 {
 	struct intel_crtc *primary_crtc;
-	u8 pipes = 0;
+	u16 pipes = 0;
 
 	for_each_intel_crtc_in_pipe_mask(display, primary_crtc, primary_pipes_mask) {
 		struct intel_crtc_state *primary_crtc_state =
@@ -263,9 +263,9 @@ static void intel_crtc_disable_noatomic(struct intel_crtc *crtc,
 {
 	struct intel_display *display = to_intel_display(crtc);
 	struct intel_crtc *temp_crtc;
-	u8 portsync_master_mask;
-	u8 portsync_slaves_mask;
-	u8 joiner_secondaries_mask;
+	u16 portsync_master_mask;
+	u16 portsync_slaves_mask;
+	u16 joiner_secondaries_mask;
 
 	/* TODO: Add support for MST */
 	get_portsync_pipes(crtc, &portsync_master_mask, &portsync_slaves_mask);
diff --git a/drivers/gpu/drm/i915/display/intel_panel.c b/drivers/gpu/drm/i915/display/intel_panel.c
index 20c548eea6da3..81e638d0c7b33 100644
--- a/drivers/gpu/drm/i915/display/intel_panel.c
+++ b/drivers/gpu/drm/i915/display/intel_panel.c
@@ -67,21 +67,62 @@ static bool is_best_fixed_mode(struct intel_connector *connector,
 	if (!best_mode)
 		return true;
 
-	/*
-	 * With VRR always pick a mode with equal/higher than requested
-	 * vrefresh, which we can then reduce to match the requested
-	 * vrefresh by extending the vblank length.
-	 */
-	if (intel_vrr_is_in_range(connector, vrefresh) &&
-	    intel_vrr_is_in_range(connector, fixed_mode_vrefresh) &&
-	    fixed_mode_vrefresh < vrefresh)
-		return false;
-
 	/* pick the fixed_mode that is closest in terms of vrefresh */
 	return abs(fixed_mode_vrefresh - vrefresh) <
 		abs(drm_mode_vrefresh(best_mode) - vrefresh);
 }
 
+static bool is_vrr_compatible(const struct drm_display_mode *mode1,
+			      const struct drm_display_mode *mode2)
+{
+	return drm_mode_match(mode1, mode2,
+			      DRM_MODE_MATCH_CLOCK |
+			      DRM_MODE_MATCH_TIMINGS_VRR |
+			      DRM_MODE_MATCH_FLAGS |
+			      DRM_MODE_MATCH_3D_FLAGS);
+}
+
+static const struct drm_display_mode *
+intel_panel_fixed_mode_vrr(struct intel_connector *connector,
+			   const struct drm_display_mode *mode,
+			   const struct drm_display_mode *vrr_ref_mode)
+{
+	const struct drm_display_mode *fixed_mode, *best_mode = NULL;
+	int vrefresh = drm_mode_vrefresh(mode);
+
+	if (!intel_vrr_is_in_range(connector, vrefresh))
+		return NULL;
+
+	if (vrr_ref_mode &&
+	    !intel_vrr_is_in_range(connector, drm_mode_vrefresh(vrr_ref_mode)))
+		return NULL;
+
+	list_for_each_entry(fixed_mode, &connector->panel.fixed_modes, head) {
+		int fixed_mode_vrefresh = drm_mode_vrefresh(fixed_mode);
+
+		if (!intel_vrr_is_in_range(connector, fixed_mode_vrefresh))
+			continue;
+
+		/*
+		 * With VRR always pick a mode with equal/higher than requested
+		 * vrefresh, which we can then reduce to match the requested
+		 * vrefresh by extending the vblank length.
+		 */
+		if (fixed_mode_vrefresh < vrefresh)
+			continue;
+
+		if (vrr_ref_mode &&
+		    !is_vrr_compatible(fixed_mode, vrr_ref_mode))
+			continue;
+
+		if (is_best_fixed_mode(connector, vrefresh,
+				       fixed_mode_vrefresh, best_mode))
+			best_mode = fixed_mode;
+	}
+
+	return best_mode;
+}
+
 const struct drm_display_mode *
 intel_panel_fixed_mode(struct intel_connector *connector,
 		       const struct drm_display_mode *mode)
@@ -197,14 +238,66 @@ enum drrs_type intel_panel_drrs_type(struct intel_connector *connector)
 	return connector->panel.vbt.drrs_type;
 }
 
-int intel_panel_compute_config(struct intel_connector *connector,
-			       struct drm_display_mode *adjusted_mode)
+static int intel_panel_compute_config_vrr(struct intel_atomic_state *state,
+					  struct intel_crtc_state *crtc_state,
+					  struct intel_connector *connector)
 {
-	const struct drm_display_mode *fixed_mode =
-		intel_panel_fixed_mode(connector, adjusted_mode);
+	struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
+	const struct drm_display_mode *fixed_mode = NULL;
 	int vrefresh, fixed_mode_vrefresh;
-	bool is_vrr;
+
+	/*
+	 * Attempt a VRR based refresh rate change if possible
+	 * when userspace has forbidden a full modeset.
+	 */
+	if (!state->base.allow_modeset) {
+		struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+		const struct intel_crtc_state *old_crtc_state =
+			intel_atomic_get_old_crtc_state(state, crtc);
+
+		if (old_crtc_state->hw.enable &&
+		    old_crtc_state->uapi.encoder_mask == crtc_state->uapi.encoder_mask)
+			fixed_mode = intel_panel_fixed_mode_vrr(connector, adjusted_mode,
+								&old_crtc_state->hw.adjusted_mode);
+	}
 
+	if (!fixed_mode)
+		fixed_mode = intel_panel_fixed_mode_vrr(connector, adjusted_mode, NULL);
+
+	if (!fixed_mode)
+		return -EINVAL;
+
+	vrefresh = drm_mode_vrefresh(adjusted_mode);
+	fixed_mode_vrefresh = drm_mode_vrefresh(fixed_mode);
+
+	drm_mode_copy(adjusted_mode, fixed_mode);
+
+	if (fixed_mode_vrefresh != vrefresh) {
+		int vsync_start_offset = adjusted_mode->vtotal - adjusted_mode->vsync_start;
+		int vsync_end_offset = adjusted_mode->vtotal - adjusted_mode->vsync_end;
+
+		adjusted_mode->vtotal =
+			DIV_ROUND_CLOSEST(adjusted_mode->clock * 1000,
+					  adjusted_mode->htotal * vrefresh);
+
+		adjusted_mode->vsync_start = adjusted_mode->vtotal - vsync_start_offset;
+		adjusted_mode->vsync_end = adjusted_mode->vtotal - vsync_end_offset;
+	}
+
+	drm_mode_set_crtcinfo(adjusted_mode, 0);
+
+	return 0;
+}
+
+static int intel_panel_compute_config_fixed_rr(struct intel_atomic_state *state,
+					       struct intel_crtc_state *crtc_state,
+					       struct intel_connector *connector)
+{
+	struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
+	const struct drm_display_mode *fixed_mode;
+	int vrefresh, fixed_mode_vrefresh;
+
+	fixed_mode = intel_panel_fixed_mode(connector, adjusted_mode);
 	if (!fixed_mode)
 		return 0;
 
@@ -212,41 +305,40 @@ int intel_panel_compute_config(struct intel_connector *connector,
 	fixed_mode_vrefresh = drm_mode_vrefresh(fixed_mode);
 
 	/*
-	 * Assume that we shouldn't muck about with the
-	 * timings if they don't land in the VRR range.
+	 * We don't want to lie too much to the user about the refresh
+	 * rate they're going to get. But we have to allow a bit of latitude
+	 * for Xorg since it likes to automagically cook up modes with slightly
+	 * off refresh rates.
 	 */
-	is_vrr = intel_vrr_is_in_range(connector, vrefresh) &&
-		intel_vrr_is_in_range(connector, fixed_mode_vrefresh);
-
-	if (!is_vrr) {
-		/*
-		 * We don't want to lie too much to the user about the refresh
-		 * rate they're going to get. But we have to allow a bit of latitude
-		 * for Xorg since it likes to automagically cook up modes with slightly
-		 * off refresh rates.
-		 */
-		if (abs(vrefresh - fixed_mode_vrefresh) > 1) {
-			drm_dbg_kms(connector->base.dev,
-				    "[CONNECTOR:%d:%s] Requested mode vrefresh (%d Hz) does not match fixed mode vrefresh (%d Hz)\n",
-				    connector->base.base.id, connector->base.name,
-				    vrefresh, fixed_mode_vrefresh);
+	if (abs(vrefresh - fixed_mode_vrefresh) > 1) {
+		drm_dbg_kms(connector->base.dev,
+			    "[CONNECTOR:%d:%s] Requested mode vrefresh (%d Hz) does not match fixed mode vrefresh (%d Hz)\n",
+			    connector->base.base.id, connector->base.name,
+			    vrefresh, fixed_mode_vrefresh);
 
-			return -EINVAL;
-		}
+		return -EINVAL;
 	}
 
 	drm_mode_copy(adjusted_mode, fixed_mode);
 
-	if (is_vrr && fixed_mode_vrefresh != vrefresh)
-		adjusted_mode->vtotal =
-			DIV_ROUND_CLOSEST(adjusted_mode->clock * 1000,
-					  adjusted_mode->htotal * vrefresh);
-
 	drm_mode_set_crtcinfo(adjusted_mode, 0);
 
 	return 0;
 }
 
+int intel_panel_compute_config(struct intel_atomic_state *state,
+			       struct intel_crtc_state *crtc_state,
+			       struct intel_connector *connector)
+{
+	int ret;
+
+	ret = intel_panel_compute_config_vrr(state, crtc_state, connector);
+	if (ret)
+		ret = intel_panel_compute_config_fixed_rr(state, crtc_state, connector);
+
+	return ret;
+}
+
 static void intel_panel_add_edid_alt_fixed_modes(struct intel_connector *connector)
 {
 	struct intel_display *display = to_intel_display(connector);
diff --git a/drivers/gpu/drm/i915/display/intel_panel.h b/drivers/gpu/drm/i915/display/intel_panel.h
index 23bd227826c9e..30c6078ecb1b9 100644
--- a/drivers/gpu/drm/i915/display/intel_panel.h
+++ b/drivers/gpu/drm/i915/display/intel_panel.h
@@ -14,6 +14,7 @@ struct drm_connector;
 struct drm_connector_state;
 struct drm_display_mode;
 struct drm_edid;
+struct intel_atomic_state;
 struct intel_connector;
 struct intel_crtc_state;
 struct intel_display;
@@ -45,8 +46,9 @@ enum drm_mode_status
 intel_panel_mode_valid(struct intel_connector *connector,
 		       const struct drm_display_mode *mode,
 		       int *target_clock);
-int intel_panel_compute_config(struct intel_connector *connector,
-			       struct drm_display_mode *adjusted_mode);
+int intel_panel_compute_config(struct intel_atomic_state *state,
+			       struct intel_crtc_state *crtc_state,
+			       struct intel_connector *connector);
 void intel_panel_add_edid_fixed_modes(struct intel_connector *connector,
 				      bool use_alt_fixed_modes);
 void intel_panel_add_vbt_lfp_fixed_mode(struct intel_connector *connector);
diff --git a/drivers/gpu/drm/i915/display/intel_parent.c b/drivers/gpu/drm/i915/display/intel_parent.c
index a5816561be408..a5e41ea669213 100644
--- a/drivers/gpu/drm/i915/display/intel_parent.c
+++ b/drivers/gpu/drm/i915/display/intel_parent.c
@@ -251,9 +251,11 @@ struct intel_panic *intel_parent_panic_alloc(struct intel_display *display)
 	return display->parent->panic->alloc();
 }
 
-int intel_parent_panic_setup(struct intel_display *display, struct intel_panic *panic, struct drm_scanout_buffer *sb)
+int intel_parent_panic_setup(struct intel_display *display, struct intel_panic *panic,
+			     struct drm_scanout_buffer *sb, struct drm_gem_object *obj,
+			     unsigned int (*tiling)(unsigned int x, unsigned int y, unsigned int width))
 {
-	return display->parent->panic->setup(panic, sb);
+	return display->parent->panic->setup(panic, sb, obj, tiling);
 }
 
 void intel_parent_panic_finish(struct intel_display *display, struct intel_panic *panic)
diff --git a/drivers/gpu/drm/i915/display/intel_parent.h b/drivers/gpu/drm/i915/display/intel_parent.h
index 27e35f891a6bb..595d4148b8eb4 100644
--- a/drivers/gpu/drm/i915/display/intel_parent.h
+++ b/drivers/gpu/drm/i915/display/intel_parent.h
@@ -105,7 +105,9 @@ void intel_parent_overlay_cleanup(struct intel_display *display);
 
 /* panic */
 struct intel_panic *intel_parent_panic_alloc(struct intel_display *display);
-int intel_parent_panic_setup(struct intel_display *display, struct intel_panic *panic, struct drm_scanout_buffer *sb);
+int intel_parent_panic_setup(struct intel_display *display, struct intel_panic *panic,
+			     struct drm_scanout_buffer *sb, struct drm_gem_object *obj,
+			     unsigned int (*tiling)(unsigned int x, unsigned int y, unsigned int width));
 void intel_parent_panic_finish(struct intel_display *display, struct intel_panic *panic);
 
 /* pc8 */
diff --git a/drivers/gpu/drm/i915/display/intel_plane.c b/drivers/gpu/drm/i915/display/intel_plane.c
index e087554488b44..a440f92ff00cd 100644
--- a/drivers/gpu/drm/i915/display/intel_plane.c
+++ b/drivers/gpu/drm/i915/display/intel_plane.c
@@ -67,7 +67,7 @@ static void intel_plane_state_reset(struct intel_plane_state *plane_state,
 {
 	memset(plane_state, 0, sizeof(*plane_state));
 
-	__drm_atomic_helper_plane_state_reset(&plane_state->uapi, &plane->base);
+	__drm_atomic_helper_plane_state_init(&plane_state->uapi, &plane->base);
 
 	plane_state->scaler_id = -1;
 	plane_state->fence_id = -1;
@@ -408,25 +408,27 @@ intel_plane_colorop_replace_blob(struct intel_plane_state *plane_state,
 }
 
 static void
-intel_plane_color_copy_uapi_to_hw_state(struct intel_plane_state *plane_state,
+intel_plane_color_copy_uapi_to_hw_state(struct intel_atomic_state *state,
+					struct intel_plane_state *plane_state,
 					const struct intel_plane_state *from_plane_state,
 					struct intel_crtc *crtc)
 {
 	struct drm_colorop *iter_colorop, *colorop;
 	struct drm_colorop_state *new_colorop_state;
-	struct drm_atomic_commit *state = plane_state->uapi.state;
 	struct intel_colorop *intel_colorop;
 	struct drm_property_blob *blob;
-	struct intel_atomic_state *intel_atomic_state = to_intel_atomic_state(state);
-	struct intel_crtc_state *new_crtc_state = intel_atomic_state ?
-		intel_atomic_get_new_crtc_state(intel_atomic_state, crtc) : NULL;
+	struct intel_crtc_state *new_crtc_state = state ?
+		intel_atomic_get_new_crtc_state(state, crtc) : NULL;
 	bool changed = false;
 	int i = 0;
 
+	if (!state)
+		return;
+
 	iter_colorop = from_plane_state->uapi.color_pipeline;
 
 	while (iter_colorop) {
-		for_each_new_colorop_in_state(state, colorop, new_colorop_state, i) {
+		for_each_new_colorop_in_state(&state->base, colorop, new_colorop_state, i) {
 			if (new_colorop_state->colorop == iter_colorop) {
 				blob = new_colorop_state->bypass ? NULL : new_colorop_state->data;
 				intel_colorop = to_intel_colorop(colorop);
@@ -442,7 +444,8 @@ intel_plane_color_copy_uapi_to_hw_state(struct intel_plane_state *plane_state,
 		new_crtc_state->plane_color_changed = true;
 }
 
-void intel_plane_copy_uapi_to_hw_state(struct intel_plane_state *plane_state,
+void intel_plane_copy_uapi_to_hw_state(struct intel_atomic_state *state,
+				       struct intel_plane_state *plane_state,
 				       const struct intel_plane_state *from_plane_state,
 				       struct intel_crtc *crtc)
 {
@@ -471,19 +474,25 @@ void intel_plane_copy_uapi_to_hw_state(struct intel_plane_state *plane_state,
 	plane_state->uapi.src = drm_plane_state_src(&from_plane_state->uapi);
 	plane_state->uapi.dst = drm_plane_state_dest(&from_plane_state->uapi);
 
-	intel_plane_color_copy_uapi_to_hw_state(plane_state, from_plane_state, crtc);
+	intel_plane_color_copy_uapi_to_hw_state(state, plane_state, from_plane_state, crtc);
 }
 
-void intel_plane_copy_hw_state(struct intel_plane_state *plane_state,
-			       const struct intel_plane_state *from_plane_state)
+static void intel_plane_y_copy_hw_state(struct intel_plane_state *y_plane_state,
+					const struct intel_plane_state *uv_plane_state)
 {
-	intel_plane_clear_hw_state(plane_state);
+	intel_plane_clear_hw_state(y_plane_state);
 
-	memcpy(&plane_state->hw, &from_plane_state->hw,
-	       sizeof(plane_state->hw));
+	y_plane_state->hw.crtc = uv_plane_state->hw.crtc;
+	y_plane_state->hw.fb = uv_plane_state->hw.fb;
+	if (y_plane_state->hw.fb)
+		drm_framebuffer_get(y_plane_state->hw.fb);
 
-	if (plane_state->hw.fb)
-		drm_framebuffer_get(plane_state->hw.fb);
+	y_plane_state->hw.alpha	= uv_plane_state->hw.alpha;
+	y_plane_state->hw.pixel_blend_mode = uv_plane_state->hw.pixel_blend_mode;
+	y_plane_state->hw.rotation = uv_plane_state->hw.rotation;
+	y_plane_state->hw.color_encoding = uv_plane_state->hw.color_encoding;
+	y_plane_state->hw.color_range = uv_plane_state->hw.color_range;
+	y_plane_state->hw.scaling_filter = uv_plane_state->hw.scaling_filter;
 }
 
 static void unlink_nv12_plane(struct intel_crtc_state *crtc_state,
@@ -868,7 +877,8 @@ static int plane_atomic_check(struct intel_atomic_state *state,
 					   old_primary_crtc_plane_state,
 					   new_primary_crtc_plane_state);
 
-	intel_plane_copy_uapi_to_hw_state(new_plane_state,
+	intel_plane_copy_uapi_to_hw_state(state,
+					  new_plane_state,
 					  new_primary_crtc_plane_state,
 					  crtc);
 
@@ -1607,17 +1617,17 @@ static int intel_get_scanout_buffer(struct drm_plane *plane,
 	if (fb == intel_fbdev_framebuffer(display->fbdev.fbdev)) {
 		intel_fbdev_get_map(display, &sb->map[0]);
 	} else {
+		unsigned int (*tiling)(unsigned int x, unsigned int y, unsigned int width) = NULL;
 		int ret;
 		/* Can't disable tiling if DPT is in use */
 		if (intel_fb_uses_dpt(&fb->base)) {
 			if (fb->base.format->cpp[0] != 4)
 				return -EOPNOTSUPP;
-			fb->panic_tiling = intel_get_tiling_func(fb->base.modifier);
-			if (!fb->panic_tiling)
+			tiling = intel_get_tiling_func(fb->base.modifier);
+			if (!tiling)
 				return -EOPNOTSUPP;
 		}
-		sb->private = fb;
-		ret = intel_parent_panic_setup(display, fb->panic, sb);
+		ret = intel_parent_panic_setup(display, fb->panic, sb, obj, tiling);
 		if (ret)
 			return ret;
 	}
@@ -1688,7 +1698,7 @@ static void link_nv12_planes(struct intel_crtc_state *crtc_state,
 	crtc_state->rel_data_rate[y_plane->id] = crtc_state->rel_data_rate_y[uv_plane->id];
 
 	/* Copy parameters to Y plane */
-	intel_plane_copy_hw_state(y_plane_state, uv_plane_state);
+	intel_plane_y_copy_hw_state(y_plane_state, uv_plane_state);
 	y_plane_state->uapi.src = uv_plane_state->uapi.src;
 	y_plane_state->uapi.dst = uv_plane_state->uapi.dst;
 
diff --git a/drivers/gpu/drm/i915/display/intel_plane.h b/drivers/gpu/drm/i915/display/intel_plane.h
index a6338bba72d97..31a6229aea73b 100644
--- a/drivers/gpu/drm/i915/display/intel_plane.h
+++ b/drivers/gpu/drm/i915/display/intel_plane.h
@@ -35,11 +35,10 @@ unsigned int intel_plane_pixel_rate(const struct intel_crtc_state *crtc_state,
 unsigned int intel_plane_data_rate(const struct intel_crtc_state *crtc_state,
 				   const struct intel_plane_state *plane_state,
 				   int color_plane);
-void intel_plane_copy_uapi_to_hw_state(struct intel_plane_state *plane_state,
+void intel_plane_copy_uapi_to_hw_state(struct intel_atomic_state *state,
+				       struct intel_plane_state *plane_state,
 				       const struct intel_plane_state *from_plane_state,
 				       struct intel_crtc *crtc);
-void intel_plane_copy_hw_state(struct intel_plane_state *plane_state,
-			       const struct intel_plane_state *from_plane_state);
 void intel_plane_async_flip(struct intel_dsb *dsb,
 			    struct intel_plane *plane,
 			    const struct intel_crtc_state *crtc_state,
diff --git a/drivers/gpu/drm/i915/display/intel_pmdemand.c b/drivers/gpu/drm/i915/display/intel_pmdemand.c
index 6d32c52269a66..f96545a5b8819 100644
--- a/drivers/gpu/drm/i915/display/intel_pmdemand.c
+++ b/drivers/gpu/drm/i915/display/intel_pmdemand.c
@@ -152,7 +152,7 @@ intel_pmdemand_update_phys_mask(struct intel_display *display,
 {
 	enum phy phy;
 
-	if (DISPLAY_VER(display) < 14)
+	if (!HAS_PMDEMAND(display))
 		return;
 
 	if (!encoder)
@@ -174,7 +174,7 @@ intel_pmdemand_update_port_clock(struct intel_display *display,
 				 struct intel_pmdemand_state *pmdemand_state,
 				 enum pipe pipe, int port_clock)
 {
-	if (DISPLAY_VER(display) < 14)
+	if (!HAS_PMDEMAND(display))
 		return;
 
 	pmdemand_state->ddi_clocks[pipe] = port_clock;
@@ -324,7 +324,7 @@ int intel_pmdemand_atomic_check(struct intel_atomic_state *state)
 	const struct intel_dbuf_state *new_dbuf_state;
 	struct intel_pmdemand_state *new_pmdemand_state;
 
-	if (DISPLAY_VER(display) < 14)
+	if (!HAS_PMDEMAND(display))
 		return 0;
 
 	if (!intel_pmdemand_needs_update(state))
@@ -404,7 +404,7 @@ intel_pmdemand_init_pmdemand_params(struct intel_display *display,
 {
 	u32 reg1, reg2;
 
-	if (DISPLAY_VER(display) < 14)
+	if (!HAS_PMDEMAND(display))
 		return;
 
 	mutex_lock(&display->pmdemand.lock);
@@ -637,7 +637,7 @@ void intel_pmdemand_pre_plane_update(struct intel_atomic_state *state)
 	const struct intel_pmdemand_state *old_pmdemand_state =
 		intel_atomic_get_old_pmdemand_state(state);
 
-	if (DISPLAY_VER(display) < 14)
+	if (!HAS_PMDEMAND(display))
 		return;
 
 	if (!new_pmdemand_state ||
@@ -660,7 +660,7 @@ void intel_pmdemand_post_plane_update(struct intel_atomic_state *state)
 	const struct intel_pmdemand_state *old_pmdemand_state =
 		intel_atomic_get_old_pmdemand_state(state);
 
-	if (DISPLAY_VER(display) < 14)
+	if (!HAS_PMDEMAND(display))
 		return;
 
 	if (!new_pmdemand_state ||
diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c
index beaa1d62613db..40e3d70959965 100644
--- a/drivers/gpu/drm/i915/display/intel_psr.c
+++ b/drivers/gpu/drm/i915/display/intel_psr.c
@@ -88,22 +88,6 @@
  * issues the self-refresh re-enable code is done from a work queue, which
  * must be correctly synchronized/cancelled when shutting down the pipe."
  *
- * DC3CO (DC3 clock off)
- *
- * On top of PSR2, GEN12 adds a intermediate power savings state that turns
- * clock off automatically during PSR2 idle state.
- * The smaller overhead of DC3co entry/exit vs. the overhead of PSR2 deep sleep
- * entry/exit allows the HW to enter a low-power state even when page flipping
- * periodically (for instance a 30fps video playback scenario).
- *
- * Every time a flips occurs PSR2 will get out of deep sleep state(if it was),
- * so DC3CO is enabled and tgl_dc3co_disable_work is schedule to run after 6
- * frames, if no other flip occurs and the function above is executed, DC3CO is
- * disabled and PSR2 is configured to enter deep sleep, resetting again in case
- * of another flip.
- * Front buffer modifications do not trigger DC3CO activation on purpose as it
- * would bring a lot of complexity and most of the moderns systems will only
- * use page flips.
  */
 
 /*
@@ -1098,10 +1082,11 @@ static void hsw_activate_psr2(struct intel_dp *intel_dp)
 	u32 psr_val = 0;
 	u8 idle_frames;
 
-	/* Wa_16025596647 */
-	if ((DISPLAY_VER(display) == 20 ||
-	     IS_DISPLAY_VERx100_STEP(display, 3000, STEP_A0, STEP_B0)) &&
-	    is_dc5_dc6_blocked(intel_dp) && intel_dp->psr.pkg_c_latency_used)
+	/* DC3CO / Wa_16025596647 */
+	if (intel_dp->psr.dc3co_allowed ||
+	    ((DISPLAY_VER(display) == 20 ||
+	      IS_DISPLAY_VERx100_STEP(display, 3000, STEP_A0, STEP_B0)) &&
+	     is_dc5_dc6_blocked(intel_dp) && intel_dp->psr.pkg_c_latency_used))
 		idle_frames = 0;
 	else
 		idle_frames = psr_compute_idle_frames(intel_dp);
@@ -1220,108 +1205,6 @@ static void psr2_program_idle_frames(struct intel_dp *intel_dp,
 		     EDP_PSR2_IDLE_FRAMES(idle_frames));
 }
 
-static void tgl_psr2_enable_dc3co(struct intel_dp *intel_dp)
-{
-	struct intel_display *display = to_intel_display(intel_dp);
-
-	psr2_program_idle_frames(intel_dp, 0);
-	intel_display_power_set_target_dc_state(display, DC_STATE_EN_DC3CO);
-}
-
-static void tgl_psr2_disable_dc3co(struct intel_dp *intel_dp)
-{
-	struct intel_display *display = to_intel_display(intel_dp);
-
-	intel_display_power_set_target_dc_state(display, DC_STATE_EN_UPTO_DC6);
-	psr2_program_idle_frames(intel_dp, psr_compute_idle_frames(intel_dp));
-}
-
-static void tgl_dc3co_disable_work(struct work_struct *work)
-{
-	struct intel_dp *intel_dp =
-		container_of(work, typeof(*intel_dp), psr.dc3co_work.work);
-
-	mutex_lock(&intel_dp->psr.lock);
-	/* If delayed work is pending, it is not idle */
-	if (delayed_work_pending(&intel_dp->psr.dc3co_work))
-		goto unlock;
-
-	tgl_psr2_disable_dc3co(intel_dp);
-unlock:
-	mutex_unlock(&intel_dp->psr.lock);
-}
-
-static void tgl_disallow_dc3co_on_psr2_exit(struct intel_dp *intel_dp)
-{
-	if (!intel_dp->psr.dc3co_exitline)
-		return;
-
-	cancel_delayed_work(&intel_dp->psr.dc3co_work);
-	/* Before PSR2 exit disallow dc3co*/
-	tgl_psr2_disable_dc3co(intel_dp);
-}
-
-static bool
-dc3co_is_pipe_port_compatible(struct intel_dp *intel_dp,
-			      struct intel_crtc_state *crtc_state)
-{
-	struct intel_display *display = to_intel_display(intel_dp);
-	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	enum pipe pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe;
-	enum port port = dig_port->base.port;
-
-	if (display->platform.alderlake_p || DISPLAY_VER(display) >= 14)
-		return pipe <= PIPE_B && port <= PORT_B;
-	else
-		return pipe == PIPE_A && port == PORT_A;
-}
-
-static void
-tgl_dc3co_exitline_compute_config(struct intel_dp *intel_dp,
-				  struct intel_crtc_state *crtc_state)
-{
-	struct intel_display *display = to_intel_display(intel_dp);
-	const u32 crtc_vdisplay = crtc_state->uapi.adjusted_mode.crtc_vdisplay;
-	struct i915_power_domains *power_domains = &display->power.domains;
-	u32 exit_scanlines;
-
-	/*
-	 * FIXME: Due to the changed sequence of activating/deactivating DC3CO,
-	 * disable DC3CO until the changed dc3co activating/deactivating sequence
-	 * is applied. B.Specs:49196
-	 */
-	return;
-
-	/*
-	 * DMC's DC3CO exit mechanism has an issue with Selective Fecth
-	 * TODO: when the issue is addressed, this restriction should be removed.
-	 */
-	if (crtc_state->enable_psr2_sel_fetch)
-		return;
-
-	if (!(power_domains->allowed_dc_mask & DC_STATE_EN_DC3CO))
-		return;
-
-	if (!dc3co_is_pipe_port_compatible(intel_dp, crtc_state))
-		return;
-
-	/* Wa_16011303918:adl-p */
-	if (intel_display_wa(display, INTEL_DISPLAY_WA_16011303918))
-		return;
-
-	/*
-	 * DC3CO Exit time 200us B.Spec 49196
-	 * PSR2 transcoder Early Exit scanlines = ROUNDUP(200 / line time) + 1
-	 */
-	exit_scanlines =
-		intel_usecs_to_scanlines(&crtc_state->uapi.adjusted_mode, 200) + 1;
-
-	if (drm_WARN_ON(display->drm, exit_scanlines > crtc_vdisplay))
-		return;
-
-	crtc_state->dc3co_exitline = crtc_vdisplay - exit_scanlines;
-}
-
 static bool intel_psr2_sel_fetch_config_valid(struct intel_dp *intel_dp,
 					      struct intel_crtc_state *crtc_state)
 {
@@ -1694,8 +1577,6 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
 		return false;
 	}
 
-	tgl_dc3co_exitline_compute_config(intel_dp, crtc_state);
-
 	return true;
 }
 
@@ -1890,6 +1771,50 @@ static bool intel_psr_needs_wa_18037818876(struct intel_dp *intel_dp,
 		!crtc_state->has_sel_update);
 }
 
+static void psr2_dc3co_disable_locked(struct intel_dp *intel_dp)
+{
+	struct intel_display *display = to_intel_display(intel_dp);
+
+	if (intel_dp->psr.dc3co_allowed) {
+		intel_dp->psr.dc3co_allowed = false;
+		intel_display_power_set_target_dc_state(display, DC_STATE_EN_UPTO_DC6);
+		psr2_program_idle_frames(intel_dp, psr_compute_idle_frames(intel_dp));
+	}
+}
+
+static void psr2_dc3co_disable_work(struct work_struct *work)
+{
+	struct intel_dp *intel_dp =
+		container_of(work, typeof(*intel_dp), psr.dc3co_work.work);
+
+	mutex_lock(&intel_dp->psr.lock);
+	psr2_dc3co_disable_locked(intel_dp);
+	mutex_unlock(&intel_dp->psr.lock);
+}
+
+static void
+psr2_dc3co_flush_locked(struct intel_dp *intel_dp, unsigned int frontbuffer_bits)
+{
+	struct intel_display *display = to_intel_display(intel_dp);
+
+	if (!intel_dp->psr.dc3co_allowed)
+		return;
+
+	if (!intel_dp->psr.sel_update_enabled ||
+	    !intel_dp->psr.active)
+		return;
+	/*
+	 * At every frontbuffer flush flip event modified delay of delayed work,
+	 * when delayed work schedules that means display has been idle.
+	 */
+	if (!(frontbuffer_bits &
+	    INTEL_FRONTBUFFER_ALL_MASK(intel_dp->psr.pipe)))
+		return;
+
+	mod_delayed_work(display->wq.unordered, &intel_dp->psr.dc3co_work,
+			 intel_dp->psr.dc3co_exit_delay);
+}
+
 static
 void intel_psr_set_non_psr_pipes(struct intel_dp *intel_dp,
 				 struct intel_crtc_state *crtc_state)
@@ -2010,12 +1935,6 @@ void intel_psr_get_config(struct intel_encoder *encoder,
 	}
 
 	pipe_config->enable_psr2_su_region_et = intel_dp->psr.su_region_et_enabled;
-
-	if (DISPLAY_VER(display) >= 12) {
-		val = intel_de_read(display,
-				    TRANS_EXITLINE(display, cpu_transcoder));
-		pipe_config->dc3co_exitline = REG_FIELD_GET(EXITLINE_MASK, val);
-	}
 unlock:
 	mutex_unlock(&intel_dp->psr.lock);
 }
@@ -2143,16 +2062,6 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp,
 
 	psr_irq_control(intel_dp);
 
-	/*
-	 * TODO: if future platforms supports DC3CO in more than one
-	 * transcoder, EXITLINE will need to be unset when disabling PSR
-	 */
-	if (intel_dp->psr.dc3co_exitline)
-		intel_de_rmw(display,
-			     TRANS_EXITLINE(display, cpu_transcoder),
-			     EXITLINE_MASK,
-			     intel_dp->psr.dc3co_exitline << EXITLINE_SHIFT | EXITLINE_ENABLE);
-
 	if (HAS_PSR_HW_TRACKING(display) && HAS_PSR2_SEL_FETCH(display))
 		intel_de_rmw(display, CHICKEN_PAR1_1, IGNORE_PSR2_HW_TRACKING,
 			     intel_dp->psr.psr2_sel_fetch_enabled ?
@@ -2191,6 +2100,18 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp,
 		else if (display->platform.alderlake_p)
 			intel_de_rmw(display, CLKGATE_DIS_MISC, 0,
 				     CLKGATE_DIS_MISC_DMASC_GATING_DIS);
+
+		/*
+		 * Wa_14026643300
+		 * On Xe3P, restrict DC3CO entry during active frame when PSR2 is
+		 * enabled without panel Early Transport; required to avoid pipe bad state.
+		 * DMC honours CHICKEN_DCPR_4 bit 24 to block DC3CO entry during active frame.
+		 */
+		if (intel_display_wa(display, INTEL_DISPLAY_WA_14026643300) &&
+		    !intel_dp->psr.panel_replay_enabled &&
+		    !intel_dp->psr.su_region_et_enabled)
+			intel_de_rmw(display, XE3P_CHICKEN_DCPR_4,
+				     0, DCPR4_BLOCK_DC3CO_ACTIVE_FRAME);
 	}
 
 	/* Wa_16025596647 */
@@ -2252,7 +2173,6 @@ static void intel_psr_enable_locked(struct intel_dp *intel_dp,
 	/* DC5/DC6 requires at least 6 idle frames */
 	val = usecs_to_jiffies(intel_get_frame_time_us(crtc_state) * 6);
 	intel_dp->psr.dc3co_exit_delay = val;
-	intel_dp->psr.dc3co_exitline = crtc_state->dc3co_exitline;
 	intel_dp->psr.psr2_sel_fetch_enabled = crtc_state->enable_psr2_sel_fetch;
 	intel_dp->psr.su_region_et_enabled = crtc_state->enable_psr2_su_region_et;
 	intel_dp->psr.psr2_sel_fetch_cff_enabled = false;
@@ -2331,8 +2251,6 @@ static void intel_psr_exit(struct intel_dp *intel_dp)
 		intel_de_rmw(display, TRANS_DP2_CTL(intel_dp->psr.transcoder),
 			     TRANS_DP2_PANEL_REPLAY_ENABLE, 0);
 	} else if (intel_dp->psr.sel_update_enabled) {
-		tgl_disallow_dc3co_on_psr2_exit(intel_dp);
-
 		val = intel_de_rmw(display,
 				   EDP_PSR2_CTL(display, cpu_transcoder),
 				   EDP_PSR2_ENABLE, 0);
@@ -2355,6 +2273,27 @@ static void intel_psr_exit(struct intel_dp *intel_dp)
 	intel_dp->psr.active = false;
 }
 
+bool intel_psr2_in_deep_sleep(struct intel_dp *intel_dp)
+{
+	struct intel_display *display = to_intel_display(intel_dp);
+	enum transcoder cpu_transcoder;
+	bool in_deep_sleep = false;
+	u32 val;
+
+	mutex_lock(&intel_dp->psr.lock);
+
+	if (!intel_dp->psr.enabled || !intel_dp->psr.sel_update_enabled)
+		goto out;
+
+	cpu_transcoder = intel_dp->psr.transcoder;
+	val = intel_de_read(display, EDP_PSR2_STATUS(display, cpu_transcoder));
+	in_deep_sleep = (val & EDP_PSR2_STATUS_STATE_MASK) ==
+		EDP_PSR2_STATUS_STATE_DEEP_SLEEP;
+out:
+	mutex_unlock(&intel_dp->psr.lock);
+	return in_deep_sleep;
+}
+
 static void intel_psr_wait_exit_locked(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
@@ -2414,6 +2353,10 @@ static void intel_psr_disable_locked(struct intel_dp *intel_dp)
 		else if (display->platform.alderlake_p)
 			intel_de_rmw(display, CLKGATE_DIS_MISC,
 				     CLKGATE_DIS_MISC_DMASC_GATING_DIS, 0);
+
+		if (intel_display_wa(display, INTEL_DISPLAY_WA_14026643300))
+			intel_de_rmw(display, XE3P_CHICKEN_DCPR_4,
+				     DCPR4_BLOCK_DC3CO_ACTIVE_FRAME, 0);
 	}
 
 	if (intel_dp_is_edp(intel_dp))
@@ -2445,6 +2388,8 @@ static void intel_psr_disable_locked(struct intel_dp *intel_dp)
 	intel_dp->psr.psr2_sel_fetch_cff_enabled = false;
 	intel_dp->psr.active_non_psr_pipes = 0;
 	intel_dp->psr.pkg_c_latency_used = 0;
+	cancel_delayed_work(&intel_dp->psr.dc3co_work);
+	intel_dp->psr.dc3co_allowed = false;
 }
 
 /**
@@ -2532,8 +2477,13 @@ void intel_psr_resume(struct intel_dp *intel_dp)
 		goto out;
 	}
 
-	if (--intel_dp->psr.pause_counter == 0)
+	if (--intel_dp->psr.pause_counter == 0) {
 		intel_psr_activate(intel_dp);
+		/* re-arm cancelled dc3co work from pause */
+		if (intel_dp->psr.dc3co_allowed)
+			mod_delayed_work(display->wq.unordered, &intel_dp->psr.dc3co_work,
+					 intel_dp->psr.dc3co_exit_delay);
+	}
 
 out:
 	mutex_unlock(&psr->lock);
@@ -3237,10 +3187,13 @@ void intel_psr_post_plane_update(struct intel_atomic_state *state,
 	const struct intel_crtc_state *crtc_state =
 		intel_atomic_get_new_crtc_state(state, crtc);
 	struct intel_encoder *encoder;
+	bool dc3co_allowed;
 
 	if (!crtc_state->has_psr)
 		return;
 
+	dc3co_allowed = intel_display_power_dc3co_allowed(display);
+
 	verify_panel_replay_dsc_state(crtc_state);
 
 	for_each_intel_encoder_mask_with_psr(state->base.dev, encoder,
@@ -3268,6 +3221,8 @@ void intel_psr_post_plane_update(struct intel_atomic_state *state,
 			keep_disabled = true;
 		}
 
+		intel_dp->psr.dc3co_allowed = dc3co_allowed;
+
 		if (!psr->enabled && !keep_disabled)
 			intel_psr_enable_locked(intel_dp, crtc_state);
 		else if (psr->enabled && !crtc_state->wm_level_disabled)
@@ -3284,6 +3239,11 @@ void intel_psr_post_plane_update(struct intel_atomic_state *state,
 		 */
 		intel_dp->psr.busy_frontbuffer_bits = 0;
 
+		if (intel_dp->psr.dc3co_allowed) {
+			mod_delayed_work(display->wq.unordered, &intel_dp->psr.dc3co_work,
+					 intel_dp->psr.dc3co_exit_delay);
+		}
+
 		mutex_unlock(&psr->lock);
 	}
 }
@@ -3656,34 +3616,6 @@ void intel_psr_invalidate(struct intel_display *display,
 		mutex_unlock(&intel_dp->psr.lock);
 	}
 }
-/*
- * When we will be completely rely on PSR2 S/W tracking in future,
- * intel_psr_flush() will invalidate and flush the PSR for ORIGIN_FLIP
- * event also therefore tgl_dc3co_flush_locked() require to be changed
- * accordingly in future.
- */
-static void
-tgl_dc3co_flush_locked(struct intel_dp *intel_dp, unsigned int frontbuffer_bits,
-		       enum fb_op_origin origin)
-{
-	struct intel_display *display = to_intel_display(intel_dp);
-
-	if (!intel_dp->psr.dc3co_exitline || !intel_dp->psr.sel_update_enabled ||
-	    !intel_dp->psr.active)
-		return;
-
-	/*
-	 * At every frontbuffer flush flip event modified delay of delayed work,
-	 * when delayed work schedules that means display has been idle.
-	 */
-	if (!(frontbuffer_bits &
-	    INTEL_FRONTBUFFER_ALL_MASK(intel_dp->psr.pipe)))
-		return;
-
-	tgl_psr2_enable_dc3co(intel_dp);
-	mod_delayed_work(display->wq.unordered, &intel_dp->psr.dc3co_work,
-			 intel_dp->psr.dc3co_exit_delay);
-}
 
 static void _psr_flush_handle(struct intel_dp *intel_dp)
 {
@@ -3770,7 +3702,7 @@ void intel_psr_flush(struct intel_display *display,
 		if (origin == ORIGIN_FLIP ||
 		    (origin == ORIGIN_CURSOR_UPDATE &&
 		     !intel_dp->psr.psr2_sel_fetch_enabled)) {
-			tgl_dc3co_flush_locked(intel_dp, frontbuffer_bits, origin);
+			psr2_dc3co_flush_locked(intel_dp, frontbuffer_bits);
 			goto unlock;
 		}
 
@@ -3829,7 +3761,7 @@ void intel_psr_init(struct intel_dp *intel_dp)
 		intel_dp->psr.link_standby = connector->panel.vbt.psr.full_link;
 
 	INIT_WORK(&intel_dp->psr.work, intel_psr_work);
-	INIT_DELAYED_WORK(&intel_dp->psr.dc3co_work, tgl_dc3co_disable_work);
+	INIT_DELAYED_WORK(&intel_dp->psr.dc3co_work, psr2_dc3co_disable_work);
 	mutex_init(&intel_dp->psr.lock);
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_psr.h b/drivers/gpu/drm/i915/display/intel_psr.h
index 29723e63888f8..d545fdaa0de7f 100644
--- a/drivers/gpu/drm/i915/display/intel_psr.h
+++ b/drivers/gpu/drm/i915/display/intel_psr.h
@@ -87,5 +87,6 @@ void intel_psr_compute_config_late(struct intel_dp *intel_dp,
 int intel_psr_min_guardband(struct intel_crtc_state *crtc_state);
 bool intel_psr_use_trans_push(const struct intel_crtc_state *crtc_state);
 bool intel_psr_pr_async_video_timing_supported(struct intel_dp *intel_dp);
+bool intel_psr2_in_deep_sleep(struct intel_dp *intel_dp);
 
 #endif /* __INTEL_PSR_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_psr_regs.h b/drivers/gpu/drm/i915/display/intel_psr_regs.h
index 8afbf5a383353..16a9e3af198d0 100644
--- a/drivers/gpu/drm/i915/display/intel_psr_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_psr_regs.h
@@ -268,6 +268,7 @@
 
 #define _PR_ALPM_CTL_A	0x60948
 #define PR_ALPM_CTL(dev_priv, tran)	_MMIO_TRANS2(dev_priv, tran, _PR_ALPM_CTL_A)
+#define  PR_ALPM_CTL_USE_DC3CO_IDLE_PROTOCOL			BIT(7)
 #define  PR_ALPM_CTL_ALLOW_LINK_OFF_BETWEEN_AS_SDP_AND_SU	BIT(6)
 #define  PR_ALPM_CTL_RFB_UPDATE_CONTROL				BIT(5)
 #define  PR_ALPM_CTL_AS_SDP_TRANSMISSION_IN_ACTIVE_DISABLE	BIT(4)
diff --git a/drivers/gpu/drm/i915/display/intel_sdvo.c b/drivers/gpu/drm/i915/display/intel_sdvo.c
index d83d350959d88..3075ef04df566 100644
--- a/drivers/gpu/drm/i915/display/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/display/intel_sdvo.c
@@ -1354,7 +1354,8 @@ static bool intel_sdvo_has_audio(struct intel_encoder *encoder,
 		return intel_conn_state->force_audio == HDMI_AUDIO_ON;
 }
 
-static int intel_sdvo_compute_config(struct intel_encoder *encoder,
+static int intel_sdvo_compute_config(struct intel_atomic_state *state,
+				     struct intel_encoder *encoder,
 				     struct intel_crtc_state *pipe_config,
 				     struct drm_connector_state *conn_state)
 {
@@ -1398,8 +1399,8 @@ static int intel_sdvo_compute_config(struct intel_encoder *encoder,
 		const struct drm_display_mode *fixed_mode;
 		int ret;
 
-		ret = intel_panel_compute_config(&intel_sdvo_connector->base,
-						 adjusted_mode);
+		ret = intel_panel_compute_config(state, pipe_config,
+						 &intel_sdvo_connector->base);
 		if (ret)
 			return ret;
 
diff --git a/drivers/gpu/drm/i915/display/intel_sprite.c b/drivers/gpu/drm/i915/display/intel_sprite.c
index 6a65f92e8a031..3e38960bbbd8f 100644
--- a/drivers/gpu/drm/i915/display/intel_sprite.c
+++ b/drivers/gpu/drm/i915/display/intel_sprite.c
@@ -1722,6 +1722,10 @@ intel_sprite_plane_create(struct intel_display *display,
 					  DRM_COLOR_YCBCR_BT709,
 					  DRM_COLOR_YCBCR_LIMITED_RANGE);
 
+	if (display->platform.valleyview || display->platform.cherryview)
+		drm_plane_create_blend_mode_property(&plane->base,
+						     BIT(DRM_MODE_BLEND_PREMULTI));
+
 	zpos = sprite + 1;
 	drm_plane_create_zpos_immutable_property(&plane->base, zpos);
 
diff --git a/drivers/gpu/drm/i915/display/intel_tv.c b/drivers/gpu/drm/i915/display/intel_tv.c
index 0a926c6f25f46..840e1dcdc2d0f 100644
--- a/drivers/gpu/drm/i915/display/intel_tv.c
+++ b/drivers/gpu/drm/i915/display/intel_tv.c
@@ -1187,13 +1187,12 @@ static bool intel_tv_vert_scaling(const struct drm_display_mode *tv_mode,
 }
 
 static int
-intel_tv_compute_config(struct intel_encoder *encoder,
+intel_tv_compute_config(struct intel_atomic_state *state,
+			struct intel_encoder *encoder,
 			struct intel_crtc_state *pipe_config,
 			struct drm_connector_state *conn_state)
 {
 	struct intel_display *display = to_intel_display(encoder);
-	struct intel_atomic_state *state =
-		to_intel_atomic_state(pipe_config->uapi.state);
 	struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
 	struct intel_tv_connector_state *tv_conn_state =
 		to_intel_tv_connector_state(conn_state);
diff --git a/drivers/gpu/drm/i915/display/intel_vrr.c b/drivers/gpu/drm/i915/display/intel_vrr.c
index aa587be908f15..51e4f3309b8b1 100644
--- a/drivers/gpu/drm/i915/display/intel_vrr.c
+++ b/drivers/gpu/drm/i915/display/intel_vrr.c
@@ -5,10 +5,13 @@
  */
 
 #include <drm/drm_print.h>
+#include <drm/intel/step.h>
 
 #include "intel_alpm.h"
+#include "intel_cmtg.h"
 #include "intel_crtc.h"
 #include "intel_de.h"
+#include "intel_display_limits.h"
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
 #include "intel_dmc.h"
@@ -322,19 +325,19 @@ int intel_vrr_fixed_rr_hw_flipline(const struct intel_crtc_state *crtc_state)
 	return intel_vrr_fixed_rr_hw_vtotal(crtc_state);
 }
 
-void intel_vrr_set_fixed_rr_timings(const struct intel_crtc_state *crtc_state)
+void intel_vrr_set_fixed_rr_timings(const struct intel_crtc_state *crtc_state,
+				    enum transcoder transcoder)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
-	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 
 	if (!intel_vrr_possible(crtc_state))
 		return;
 
-	intel_de_write(display, TRANS_VRR_VMIN(display, cpu_transcoder),
+	intel_de_write(display, TRANS_VRR_VMIN(display, transcoder),
 		       intel_vrr_fixed_rr_hw_vmin(crtc_state) - 1);
-	intel_de_write(display, TRANS_VRR_VMAX(display, cpu_transcoder),
+	intel_de_write(display, TRANS_VRR_VMAX(display, transcoder),
 		       intel_vrr_fixed_rr_hw_vmax(crtc_state) - 1);
-	intel_de_write(display, TRANS_VRR_FLIPLINE(display, cpu_transcoder),
+	intel_de_write(display, TRANS_VRR_FLIPLINE(display, transcoder),
 		       intel_vrr_fixed_rr_hw_flipline(crtc_state) - 1);
 }
 
@@ -649,7 +652,8 @@ void intel_vrr_set_transcoder_timings(const struct intel_crtc_state *crtc_state)
 			       lower_32_bits(crtc_state->cmrr.cmrr_n));
 	}
 
-	intel_vrr_set_fixed_rr_timings(crtc_state);
+	intel_vrr_set_fixed_rr_timings(crtc_state, cpu_transcoder);
+	intel_cmtg_set_vrr_timings(crtc_state);
 
 	if (!intel_vrr_always_use_vrr_tg(display))
 		intel_de_write(display, TRANS_VRR_CTL(display, cpu_transcoder),
@@ -934,6 +938,8 @@ static void intel_vrr_tg_enable(const struct intel_crtc_state *crtc_state,
 		vrr_ctl |= VRR_CTL_CMRR_ENABLE;
 
 	intel_de_write(display, TRANS_VRR_CTL(display, cpu_transcoder), vrr_ctl);
+
+	intel_cmtg_set_vrr_ctl(crtc_state);
 }
 
 static void intel_vrr_tg_disable(const struct intel_crtc_state *old_crtc_state)
@@ -978,7 +984,7 @@ void intel_vrr_disable(const struct intel_crtc_state *old_crtc_state)
 		intel_vrr_tg_disable(old_crtc_state);
 
 	intel_vrr_disable_dc_balancing(old_crtc_state);
-	intel_vrr_set_fixed_rr_timings(old_crtc_state);
+	intel_vrr_set_fixed_rr_timings(old_crtc_state, old_crtc_state->cpu_transcoder);
 }
 
 void intel_vrr_transcoder_enable(const struct intel_crtc_state *crtc_state)
@@ -1101,13 +1107,8 @@ void intel_vrr_get_config(struct intel_crtc_state *crtc_state)
 			crtc_state->vrr.vmin += intel_vrr_vmin_flipline_offset(display);
 		}
 
-		/*
-		 * For platforms that always use VRR Timing Generator, the VTOTAL.Vtotal
-		 * bits are not filled. Since for these platforms TRAN_VMIN is always
-		 * filled with crtc_vtotal, use TRAN_VRR_VMIN to get the vtotal for
-		 * adjusted_mode.
-		 */
-		if (intel_vrr_always_use_vrr_tg(display))
+		if (display->platform.novalake &&
+		    IS_DISPLAY_STEP(display, STEP_A0, STEP_C0))
 			crtc_state->hw.adjusted_mode.crtc_vtotal =
 				intel_vrr_vmin_vtotal(crtc_state);
 
diff --git a/drivers/gpu/drm/i915/display/intel_vrr.h b/drivers/gpu/drm/i915/display/intel_vrr.h
index 4f16ca4af91fc..55e9c429f5793 100644
--- a/drivers/gpu/drm/i915/display/intel_vrr.h
+++ b/drivers/gpu/drm/i915/display/intel_vrr.h
@@ -15,6 +15,7 @@ struct intel_crtc;
 struct intel_crtc_state;
 struct intel_dsb;
 struct intel_display;
+enum transcoder;
 
 bool intel_vrr_is_capable(struct intel_connector *connector);
 bool intel_vrr_is_in_range(struct intel_connector *connector, int vrefresh);
@@ -42,7 +43,8 @@ int intel_vrr_vmin_vblank_start(const struct intel_crtc_state *crtc_state);
 bool intel_vrr_is_fixed_rr(const struct intel_crtc_state *crtc_state);
 void intel_vrr_transcoder_enable(const struct intel_crtc_state *crtc_state);
 void intel_vrr_transcoder_disable(const struct intel_crtc_state *crtc_state);
-void intel_vrr_set_fixed_rr_timings(const struct intel_crtc_state *crtc_state);
+void intel_vrr_set_fixed_rr_timings(const struct intel_crtc_state *crtc_state,
+				    enum transcoder transcoder);
 void intel_vrr_dcb_reset(const struct intel_crtc_state *old_crtc_state,
 			 struct intel_crtc *crtc);
 bool intel_vrr_always_use_vrr_tg(struct intel_display *display);
diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c
index ad4bfff6903da..164b7d61c9a31 100644
--- a/drivers/gpu/drm/i915/display/skl_universal_plane.c
+++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c
@@ -2126,19 +2126,6 @@ static int skl_check_main_surface(struct intel_plane_state *plane_state)
 	return 0;
 }
 
-
-/* Divide a U16.16 fixed-point value by 2, staying in fixed-point domain */
-static inline u32 fp_16_16_div2(u32 fp)
-{
-	return fp >> 1;
-}
-
-/* Convert a U16.16 fixed-point value to integer, rounding up */
-static inline int fp_16_16_to_int_ceil(u32 fp)
-{
-	return DIV_ROUND_UP(fp, 1 << 16);
-}
-
 static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
 {
 	struct intel_display *display = to_intel_display(plane_state);
@@ -2154,14 +2141,20 @@ static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
 	int max_height = intel_plane_max_height(plane, fb, uv_plane, rotation);
 
 	/*
-	 * LNL+ UV surface start/size =
-	 * ceiling(half of Y plane start/size). Use ceiling division
-	 * unconditionally; it is a no-op for even values.
+	 * UV (chroma) start/size = ceiling(half of the *integer* Y plane
+	 * start/size), i.e. the value the luma surface programs (src >> 16),
+	 * not the raw U16.16. A bigjoiner seam mapped through the scaler can
+	 * give a fractional luma src; ceiling that directly would round the
+	 * chroma one column too far and read past the chroma surface.
 	 */
-	int x = fp_16_16_to_int_ceil(fp_16_16_div2(plane_state->uapi.src.x1));
-	int y = fp_16_16_to_int_ceil(fp_16_16_div2(plane_state->uapi.src.y1));
-	int w = fp_16_16_to_int_ceil(fp_16_16_div2(drm_rect_width(&plane_state->uapi.src)));
-	int h = fp_16_16_to_int_ceil(fp_16_16_div2(drm_rect_height(&plane_state->uapi.src)));
+	int luma_x = plane_state->uapi.src.x1 >> 16;
+	int luma_y = plane_state->uapi.src.y1 >> 16;
+	int luma_w = drm_rect_width(&plane_state->uapi.src) >> 16;
+	int luma_h = drm_rect_height(&plane_state->uapi.src) >> 16;
+	int x = DIV_ROUND_UP(luma_x, 2);
+	int y = DIV_ROUND_UP(luma_y, 2);
+	int w = DIV_ROUND_UP(luma_x + luma_w, 2) - x;
+	int h = DIV_ROUND_UP(luma_y + luma_h, 2) - y;
 	u32 offset;
 
 	/* FIXME not quite sure how/if these apply to the chroma plane */
diff --git a/drivers/gpu/drm/i915/display/skl_watermark.c b/drivers/gpu/drm/i915/display/skl_watermark.c
index 5a3677ea25b07..ec1bfe46edefa 100644
--- a/drivers/gpu/drm/i915/display/skl_watermark.c
+++ b/drivers/gpu/drm/i915/display/skl_watermark.c
@@ -275,6 +275,9 @@ void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
 	if (!intel_has_sagv(display))
 		return;
 
+	if (HAS_PMDEMAND(display))
+		return;
+
 	if (DISPLAY_VER(display) >= 11)
 		icl_sagv_pre_plane_update(state);
 	else
@@ -295,6 +298,9 @@ void intel_sagv_post_plane_update(struct intel_atomic_state *state)
 	if (!intel_has_sagv(display))
 		return;
 
+	if (HAS_PMDEMAND(display))
+		return;
+
 	if (DISPLAY_VER(display) >= 11)
 		icl_sagv_post_plane_update(state);
 	else
@@ -3856,7 +3862,7 @@ void skl_wm_plane_disable_noatomic(struct intel_crtc *crtc,
 		return;
 
 	skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[plane->id], 0, 0);
-	skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[plane->id], 0, 0);
+	skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb_y[plane->id], 0, 0);
 
 	crtc_state->wm.skl.plane_min_ddb[plane->id] = 0;
 	crtc_state->wm.skl.plane_interim_ddb[plane->id] = 0;
diff --git a/drivers/gpu/drm/i915/display/tests/Makefile b/drivers/gpu/drm/i915/display/tests/Makefile
new file mode 100644
index 0000000000000..ad250974160f6
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/tests/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0
+
+subdir-ccflags-y += -I$(srctree)/drivers/gpu/drm/i915/display/
+
+obj-$(CONFIG_DRM_I915_KUNIT_TEST) += i915_display_test.o
+i915_display_test-y = \
+	intel_dp_link_test.o
diff --git a/drivers/gpu/drm/i915/display/tests/intel_dp_link_test.c b/drivers/gpu/drm/i915/display/tests/intel_dp_link_test.c
new file mode 100644
index 0000000000000..e1c356ba11b59
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/tests/intel_dp_link_test.c
@@ -0,0 +1,1440 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <kunit/test.h>
+
+#include <linux/compiler.h>
+#include <linux/device.h>
+#include <linux/log2.h>
+#include <linux/prandom.h>
+#include <linux/random.h>
+
+#include <drm/display/drm_dp_helper.h>
+
+#include <drm/intel/display_member.h>
+
+#include "intel_connector.h"
+#include "intel_display_core.h"
+#include "intel_display_types.h"
+#include "intel_dp_link_caps.h"
+#include "intel_dp_link_training.h"
+
+#define LINK_TEST_NUM_LANE_CONFIGS(__max_lane_count) \
+	(ilog2(__max_lane_count) + 1)
+
+#define LINK_TEST_NUM_CONFIGS(__num_rates, __max_lane_count) \
+	((__num_rates) * LINK_TEST_NUM_LANE_CONFIGS(__max_lane_count))
+
+#define LINK_TEST_MAX_LANE_COUNT		((u32)4)
+#define LINK_TEST_MAX_CONFIGS			LINK_TEST_NUM_CONFIGS(DP_MAX_SUPPORTED_RATES, \
+								      LINK_TEST_MAX_LANE_COUNT)
+
+#define LINK_TEST_NUM_RANDOM_ITERATIONS		50
+
+struct test_ctx {
+	struct {
+		struct intel_display display;
+		struct device device;
+		struct __intel_generic_device generic_device;
+
+		struct intel_connector connector;
+		struct intel_digital_port dig_port;
+
+		struct intel_crtc_state crtc_state;
+	} dev;
+
+	const struct intel_dp_link_caps_test_ops *link_caps_ops;
+	const struct intel_dp_link_training_test_ops *link_training_ops;
+
+	struct rnd_state rnd;
+};
+
+struct link_rate_set {
+	const int *entries;
+	int size;
+};
+
+struct link_config_set {
+	struct intel_dp_link_config entries[LINK_TEST_MAX_CONFIGS];
+	int size;
+};
+
+struct test_config_table {
+	struct kunit *test;
+
+	struct link_rate_set rates;
+	int max_lane_count;
+	struct link_config_set disabled_configs;
+};
+
+static const int standard_dp_link_rates[] = {
+	162000, 270000, 540000, 810000, 1000000, 1350000, 2000000
+};
+
+#define LINK_TEST_NUM_STANDARD_RATES (ARRAY_SIZE(standard_dp_link_rates))
+
+#define INIT_STANDARD_TABLE(__test, __num_rates, __max_lane_count) { \
+	.test = (__test), \
+	.rates = { \
+		.entries = standard_dp_link_rates, \
+		.size = (__num_rates), \
+	}, \
+	.max_lane_count = (__max_lane_count), \
+}
+
+static const struct link_config_set standard_dp_link_configs[] = {
+	[INTEL_DP_LINK_CAPS_ORDER_KEY_BW] = {                        /* MBps    PBN    */
+		.entries = {
+			{ .rate =  162000, .lane_count = 1 }, /*  162.0    3.00 */
+			{ .rate =  270000, .lane_count = 1 }, /*  270.0    5.00 */
+			{ .rate =  162000, .lane_count = 2 }, /*  324.0    6.00 */
+			{ .rate =  270000, .lane_count = 2 }, /*  540.0   10.00 */
+			{ .rate =  540000, .lane_count = 1 }, /*  540.0   10.00 */
+			{ .rate =  162000, .lane_count = 4 }, /*  648.0   12.00 */
+			{ .rate =  810000, .lane_count = 1 }, /*  810.0   15.00 */
+			{ .rate =  270000, .lane_count = 4 }, /* 1080.0   20.00 */
+			{ .rate =  540000, .lane_count = 2 }, /* 1080.0   20.00 */
+			{ .rate = 1000000, .lane_count = 1 }, /* 1208.9   22.39 */
+			{ .rate =  810000, .lane_count = 2 }, /* 1620.0   30.00 */
+			{ .rate = 1350000, .lane_count = 1 }, /* 1632.0   30.22 */
+			{ .rate =  540000, .lane_count = 4 }, /* 2160.0   40.00 */
+			{ .rate = 1000000, .lane_count = 2 }, /* 2417.8   44.77 */
+			{ .rate = 2000000, .lane_count = 1 }, /* 2417.8   44.77 */
+			{ .rate =  810000, .lane_count = 4 }, /* 3240.0   60.00 */
+			{ .rate = 1350000, .lane_count = 2 }, /* 3264.0   60.44 */
+			{ .rate = 1000000, .lane_count = 4 }, /* 4835.6   89.55 */
+			{ .rate = 2000000, .lane_count = 2 }, /* 4835.6   89.55 */
+			{ .rate = 1350000, .lane_count = 4 }, /* 6527.9  120.89 */
+			{ .rate = 2000000, .lane_count = 4 }, /* 9671.1  179.09 */
+		},
+		.size = LINK_TEST_NUM_CONFIGS(ARRAY_SIZE(standard_dp_link_rates),
+					      LINK_TEST_MAX_LANE_COUNT),
+	},
+	[INTEL_DP_LINK_CAPS_ORDER_KEY_RATE_LANE] = {
+		.entries = {
+			{ .rate = 162000,  .lane_count = 1 },
+			{ .rate = 162000,  .lane_count = 2 },
+			{ .rate = 162000,  .lane_count = 4 },
+
+			{ .rate = 270000,  .lane_count = 1 },
+			{ .rate = 270000,  .lane_count = 2 },
+			{ .rate = 270000,  .lane_count = 4 },
+
+			{ .rate = 540000,  .lane_count = 1 },
+			{ .rate = 540000,  .lane_count = 2 },
+			{ .rate = 540000,  .lane_count = 4 },
+
+			{ .rate = 810000,  .lane_count = 1 },
+			{ .rate = 810000,  .lane_count = 2 },
+			{ .rate = 810000,  .lane_count = 4 },
+
+			{ .rate = 1000000, .lane_count = 1 },
+			{ .rate = 1000000, .lane_count = 2 },
+			{ .rate = 1000000, .lane_count = 4 },
+
+			{ .rate = 1350000, .lane_count = 1 },
+			{ .rate = 1350000, .lane_count = 2 },
+			{ .rate = 1350000, .lane_count = 4 },
+
+			{ .rate = 2000000, .lane_count = 1 },
+			{ .rate = 2000000, .lane_count = 2 },
+			{ .rate = 2000000, .lane_count = 4 },
+		},
+		.size = LINK_TEST_NUM_CONFIGS(ARRAY_SIZE(standard_dp_link_rates),
+					      LINK_TEST_MAX_LANE_COUNT),
+	},
+	[INTEL_DP_LINK_CAPS_ORDER_KEY_LANE_RATE] = {
+		.entries = {
+			{ .rate = 162000,  .lane_count = 1 },
+			{ .rate = 270000,  .lane_count = 1 },
+			{ .rate = 540000,  .lane_count = 1 },
+			{ .rate = 810000,  .lane_count = 1 },
+			{ .rate = 1000000, .lane_count = 1 },
+			{ .rate = 1350000, .lane_count = 1 },
+			{ .rate = 2000000, .lane_count = 1 },
+
+			{ .rate = 162000,  .lane_count = 2 },
+			{ .rate = 270000,  .lane_count = 2 },
+			{ .rate = 540000,  .lane_count = 2 },
+			{ .rate = 810000,  .lane_count = 2 },
+			{ .rate = 1000000, .lane_count = 2 },
+			{ .rate = 1350000, .lane_count = 2 },
+			{ .rate = 2000000, .lane_count = 2 },
+
+			{ .rate = 162000,  .lane_count = 4 },
+			{ .rate = 270000,  .lane_count = 4 },
+			{ .rate = 540000,  .lane_count = 4 },
+			{ .rate = 810000,  .lane_count = 4 },
+			{ .rate = 1000000, .lane_count = 4 },
+			{ .rate = 1350000, .lane_count = 4 },
+			{ .rate = 2000000, .lane_count = 4 },
+		},
+		.size = LINK_TEST_NUM_CONFIGS(ARRAY_SIZE(standard_dp_link_rates),
+					      LINK_TEST_MAX_LANE_COUNT),
+	},
+};
+
+static int lookup_rate(const struct link_rate_set *rate_set, int rate)
+{
+	int i;
+
+	for (i = 0; i < rate_set->size; i++)
+		if (rate_set->entries[i] == rate)
+			return i;
+
+	return -1;
+}
+
+static bool has_rate(const struct link_rate_set *rate_set, int rate)
+{
+	return lookup_rate(rate_set, rate) >= 0;
+}
+
+static bool link_configs_match(const struct intel_dp_link_config *a,
+			       const struct intel_dp_link_config *b)
+{
+	return a->rate == b->rate && a->lane_count == b->lane_count;
+}
+
+static int lookup_config(const struct link_config_set *config_set,
+			 const struct intel_dp_link_config *config)
+{
+	int i;
+
+	for (i = 0; i < config_set->size; i++)
+		if (link_configs_match(&config_set->entries[i], config))
+			return i;
+
+	return -1;
+}
+
+static bool has_config(const struct link_config_set *config_set,
+		       const struct intel_dp_link_config *config)
+{
+	return lookup_config(config_set, config) >= 0;
+}
+
+static void add_config(struct kunit *test,
+		       struct link_config_set *config_set,
+		       const struct intel_dp_link_config *config)
+{
+	KUNIT_ASSERT_LT(test, config_set->size, ARRAY_SIZE(config_set->entries));
+
+	config_set->entries[config_set->size] = *config;
+	config_set->size++;
+}
+
+static const struct intel_dp_link_caps_order config_orders[] = {
+	{
+		.key = INTEL_DP_LINK_CAPS_ORDER_KEY_BW,
+		.dir = INTEL_DP_LINK_CAPS_ORDER_DIR_ASC,
+	}, {
+		.key = INTEL_DP_LINK_CAPS_ORDER_KEY_BW,
+		.dir = INTEL_DP_LINK_CAPS_ORDER_DIR_DESC,
+	}, {
+		.key = INTEL_DP_LINK_CAPS_ORDER_KEY_RATE_LANE,
+		.dir = INTEL_DP_LINK_CAPS_ORDER_DIR_ASC,
+	}, {
+		.key = INTEL_DP_LINK_CAPS_ORDER_KEY_RATE_LANE,
+		.dir = INTEL_DP_LINK_CAPS_ORDER_DIR_DESC,
+	}, {
+		.key = INTEL_DP_LINK_CAPS_ORDER_KEY_LANE_RATE,
+		.dir = INTEL_DP_LINK_CAPS_ORDER_DIR_ASC,
+	}, {
+		.key = INTEL_DP_LINK_CAPS_ORDER_KEY_LANE_RATE,
+		.dir = INTEL_DP_LINK_CAPS_ORDER_DIR_DESC,
+	}
+};
+
+static const struct link_config_set *
+link_caps_config_order_key_to_set(struct kunit *test, enum intel_dp_link_caps_order_key key)
+{
+	return &standard_dp_link_configs[key];
+}
+
+/*
+ * TEST: Baseline with fixed reference table
+ * -----------------------------------------
+ * Verify the link_caps config iterator using fixed standard DP config tables.
+ */
+static void baseline_test_for_order(struct kunit *test,
+				    struct intel_dp_link_caps *link_caps,
+				    struct intel_dp_link_caps_order config_order)
+{
+	struct test_ctx *ctx = test->priv;
+	const struct link_config_set *config_set =
+		link_caps_config_order_key_to_set(test, config_order.key);
+	const struct intel_dp_link_caps_test_ops *ops = ctx->link_caps_ops;
+	struct intel_dp_link_config iter_config;
+	struct intel_dp_link_caps_iter iter;
+	int pos = 0;
+
+	ops->iter_start(&iter, link_caps, config_order, INTEL_DP_LINK_CAPS_FILTER_ALL);
+	for_each_dp_link_config(&iter, &iter_config) {
+		int idx = pos;
+
+		if (config_order.dir == INTEL_DP_LINK_CAPS_ORDER_DIR_DESC)
+			idx = config_set->size - idx - 1;
+
+		KUNIT_EXPECT_TRUE(test, link_configs_match(&iter_config,
+							   &config_set->entries[idx]));
+
+		pos++;
+	}
+	ops->iter_end(&iter);
+}
+
+static void intel_dp_link_caps_test_baseline(struct kunit *test)
+{
+	struct test_ctx *ctx = test->priv;
+	struct intel_dp_link_caps *link_caps = ctx->dev.dig_port.dp.link.caps;
+	const struct intel_dp_link_caps_test_ops *ops =
+		ctx->link_caps_ops;
+	int i;
+
+	ops->update(link_caps,
+		    standard_dp_link_rates, LINK_TEST_NUM_STANDARD_RATES,
+		    LINK_TEST_MAX_LANE_COUNT,
+		    true);
+
+	for (i = 0; i < ARRAY_SIZE(config_orders); i++)
+		baseline_test_for_order(test, link_caps, config_orders[i]);
+}
+
+static int get_num_configs(int num_rates, int max_lane_count)
+{
+	return num_rates * LINK_TEST_NUM_LANE_CONFIGS(max_lane_count);
+}
+
+static int rand_in_range(struct test_ctx *ctx, int min, int max)
+{
+	return min + (prandom_u32_state(&ctx->rnd) % (max - min + 1));
+}
+
+/*
+ * TEST: Update reset
+ * ------------------
+ * Verify that resetting link_caps with the DP standard rates/lane
+ * counts updates the configuration table accordingly for all
+ * combinations.
+ */
+static void verify_bw_asc_config_order(struct kunit *test,
+				       const struct intel_dp_link_config *last_config,
+				       const struct intel_dp_link_config *config)
+{
+	int config_bw = drm_dp_max_dprx_data_rate(config->rate,
+						  config->lane_count);
+	int last_config_bw = drm_dp_max_dprx_data_rate(last_config->rate,
+						       last_config->lane_count);
+
+	KUNIT_EXPECT_GE(test, config_bw, last_config_bw);
+	if (config_bw == last_config_bw)
+		KUNIT_EXPECT_GT(test, config->rate, last_config->rate);
+}
+
+static void verify_bw_desc_config_order(struct kunit *test,
+					const struct intel_dp_link_config *last_config,
+					const struct intel_dp_link_config *config)
+{
+	int config_bw = drm_dp_max_dprx_data_rate(config->rate,
+						  config->lane_count);
+	int last_config_bw = drm_dp_max_dprx_data_rate(last_config->rate,
+						       last_config->lane_count);
+
+	KUNIT_EXPECT_LE(test, config_bw, last_config_bw);
+	if (config_bw == last_config_bw)
+		KUNIT_EXPECT_LT(test, config->rate, last_config->rate);
+}
+
+static void verify_rate_lane_asc_config_order(struct kunit *test,
+					      const struct intel_dp_link_config *last_config,
+					      const struct intel_dp_link_config *config)
+{
+	KUNIT_EXPECT_GE(test, config->rate, last_config->rate);
+	if (config->rate == last_config->rate)
+		KUNIT_EXPECT_GT(test, config->lane_count, last_config->lane_count);
+}
+
+static void verify_rate_lane_desc_config_order(struct kunit *test,
+					       const struct intel_dp_link_config *last_config,
+					       const struct intel_dp_link_config *config)
+{
+	KUNIT_EXPECT_LE(test, config->rate, last_config->rate);
+	if (config->rate == last_config->rate)
+		KUNIT_EXPECT_LT(test, config->lane_count, last_config->lane_count);
+}
+
+static void verify_lane_rate_asc_config_order(struct kunit *test,
+					      const struct intel_dp_link_config *last_config,
+					      const struct intel_dp_link_config *config)
+{
+	KUNIT_EXPECT_GE(test, config->lane_count, last_config->lane_count);
+	if (config->lane_count == last_config->lane_count)
+		KUNIT_EXPECT_GT(test, config->rate, last_config->rate);
+}
+
+static void verify_lane_rate_desc_config_order(struct kunit *test,
+					       const struct intel_dp_link_config *last_config,
+					       const struct intel_dp_link_config *config)
+{
+	KUNIT_EXPECT_LE(test, config->lane_count, last_config->lane_count);
+	if (config->lane_count == last_config->lane_count)
+		KUNIT_EXPECT_LT(test, config->rate, last_config->rate);
+}
+
+static void verify_config_order(struct kunit *test,
+				struct intel_dp_link_caps_order config_order,
+				const struct intel_dp_link_config *last_config,
+				const struct intel_dp_link_config *config)
+{
+	switch (config_order.key) {
+	case INTEL_DP_LINK_CAPS_ORDER_KEY_BW:
+		if (config_order.dir == INTEL_DP_LINK_CAPS_ORDER_DIR_ASC)
+			verify_bw_asc_config_order(test, last_config, config);
+		else
+			verify_bw_desc_config_order(test, last_config, config);
+		break;
+	case INTEL_DP_LINK_CAPS_ORDER_KEY_RATE_LANE:
+		if (config_order.dir == INTEL_DP_LINK_CAPS_ORDER_DIR_ASC)
+			verify_rate_lane_asc_config_order(test, last_config, config);
+		else
+			verify_rate_lane_desc_config_order(test, last_config, config);
+		break;
+	case INTEL_DP_LINK_CAPS_ORDER_KEY_LANE_RATE:
+		if (config_order.dir == INTEL_DP_LINK_CAPS_ORDER_DIR_ASC)
+			verify_lane_rate_asc_config_order(test, last_config, config);
+		else
+			verify_lane_rate_desc_config_order(test, last_config, config);
+		break;
+	default:
+		KUNIT_FAIL_AND_ABORT(test, "Missing order key: %d", config_order.key);
+	}
+}
+
+static int expected_num_configs(const struct test_config_table *expected_table,
+				const struct intel_dp_link_config *max_limits)
+{
+	int num_configs = 0;
+	int lane_count;
+	int rate_idx;
+
+	for (rate_idx = 0; rate_idx < expected_table->rates.size; rate_idx++) {
+		for (lane_count = 1; lane_count <= expected_table->max_lane_count; lane_count <<= 1) {
+			struct intel_dp_link_config config = {
+				.rate = expected_table->rates.entries[rate_idx],
+				.lane_count = lane_count,
+			};
+
+			if (config.rate > max_limits->rate ||
+			    config.lane_count > max_limits->lane_count)
+				continue;
+
+			if (has_config(&expected_table->disabled_configs, &config))
+				continue;
+
+			num_configs++;
+		}
+	}
+
+	return num_configs;
+}
+
+static void
+verify_link_caps_for_order(const struct test_config_table *expected_table,
+			   struct intel_dp_link_caps *link_caps,
+			   struct intel_dp_link_caps_order config_order,
+			   const struct intel_dp_link_config *max_limits)
+{
+	struct kunit *test = expected_table->test;
+	struct test_ctx *ctx = test->priv;
+	const struct intel_dp_link_caps_test_ops *ops =
+		ctx->link_caps_ops;
+	struct intel_dp_link_config expected_max_bw_config = {};
+	struct intel_dp_link_config actual_max_bw_config;
+	struct intel_dp_link_config last_config = {};
+	struct intel_dp_link_config old_max_limits;
+	struct intel_dp_link_config iter_config;
+	struct intel_dp_link_caps_iter iter;
+	int num_actual_configs = 0;
+	int max_bw = 0;
+
+	ops->get_max_limits(link_caps, &old_max_limits);
+	ops->set_max_limits(link_caps, max_limits);
+
+	ops->iter_start(&iter, link_caps, config_order, INTEL_DP_LINK_CAPS_FILTER_ALL);
+	for_each_dp_link_config(&iter, &iter_config) {
+		int bw;
+
+		KUNIT_EXPECT_LE(test, iter_config.rate, max_limits->rate);
+		KUNIT_EXPECT_LE(test, iter_config.lane_count, max_limits->lane_count);
+
+		num_actual_configs++;
+
+		/*
+		 * Verify the config's rate/lane-count values and its ordering relative
+		 * to the previous config.
+		 */
+		if (last_config.rate)
+			verify_config_order(test, config_order, &last_config, &iter_config);
+		last_config = iter_config;
+
+		KUNIT_EXPECT_TRUE(test, has_rate(&expected_table->rates,
+						 iter_config.rate));
+		KUNIT_EXPECT_LE(test, iter_config.lane_count,
+				      expected_table->max_lane_count);
+		KUNIT_EXPECT_TRUE(test, is_power_of_2(iter_config.lane_count));
+
+		/* Verify the config's disabled state */
+		KUNIT_EXPECT_FALSE(test, has_config(&expected_table->disabled_configs,
+						    &iter_config));
+
+		/*
+		 * Update the max limits for allowed configs, verified at the
+		 * end for the whole config table.
+		 */
+
+		bw = drm_dp_max_dprx_data_rate(iter_config.rate, iter_config.lane_count);
+		if (bw > max_bw ||
+		    (bw == max_bw && iter_config.rate > expected_max_bw_config.rate)) {
+			max_bw = bw;
+			expected_max_bw_config = iter_config;
+		}
+	}
+	ops->iter_end(&iter);
+
+	KUNIT_EXPECT_EQ(test, num_actual_configs, expected_num_configs(expected_table, max_limits));
+
+	ops->get_max_bw_config(link_caps, &actual_max_bw_config);
+	KUNIT_EXPECT_TRUE(test, link_configs_match(&expected_max_bw_config,
+						   &actual_max_bw_config));
+
+	KUNIT_ASSERT_TRUE(test, ops->set_max_limits(link_caps, &old_max_limits));
+}
+
+static bool max_limits_valid(const struct test_config_table *expected_table,
+			     const struct intel_dp_link_config *max_limits)
+{
+	int lane_count;
+	int rate_idx;
+
+	for (rate_idx = 0; rate_idx < expected_table->rates.size; rate_idx++) {
+		for (lane_count = 1; lane_count <= expected_table->max_lane_count; lane_count <<= 1) {
+			struct intel_dp_link_config config = {
+				.rate = expected_table->rates.entries[rate_idx],
+				.lane_count = lane_count,
+			};
+
+			if (has_config(&expected_table->disabled_configs, &config))
+				continue;
+
+			if (config.rate <= max_limits->rate &&
+			    config.lane_count <= max_limits->lane_count)
+				return true;
+		}
+	}
+
+	return false;
+}
+
+static void get_max_limits(const struct test_config_table *expected_table,
+			   struct intel_dp_link_config *max_limits)
+{
+	int lane_count;
+	int rate_idx;
+
+	max_limits->rate = 0;
+	max_limits->lane_count = 0;
+
+	for (rate_idx = 0; rate_idx < expected_table->rates.size; rate_idx++) {
+		for (lane_count = 1; lane_count <= expected_table->max_lane_count; lane_count <<= 1) {
+			struct intel_dp_link_config config = {
+				.rate = expected_table->rates.entries[rate_idx],
+				.lane_count = lane_count,
+			};
+
+			if (has_config(&expected_table->disabled_configs, &config))
+				continue;
+
+			max_limits->rate = max(max_limits->rate, config.rate);
+			max_limits->lane_count = max(max_limits->lane_count, config.lane_count);
+		}
+	}
+}
+
+static void verify_link_caps(const struct test_config_table *expected_table,
+			     struct intel_dp_link_caps *link_caps)
+{
+	struct kunit *test = expected_table->test;
+	struct test_ctx *ctx = test->priv;
+	const struct intel_dp_link_caps_test_ops *ops = ctx->link_caps_ops;
+	struct intel_dp_link_config max_limits;
+	int i;
+
+	get_max_limits(expected_table, &max_limits);
+
+	for (i = 0; i < ARRAY_SIZE(config_orders); i++) {
+		int lane_count;
+		int rate_idx;
+
+		verify_link_caps_for_order(expected_table, link_caps, config_orders[i], &max_limits);
+		/*
+		 * Verify iteration after setting the max limits to each
+		 * configurations.
+		 */
+		for (rate_idx = 0; rate_idx < expected_table->rates.size; rate_idx++) {
+			for (lane_count = 1; lane_count <= expected_table->max_lane_count; lane_count <<= 1) {
+				struct intel_dp_link_config config = {
+					.rate = expected_table->rates.entries[rate_idx],
+					.lane_count = lane_count,
+				};
+
+				if (!max_limits_valid(expected_table, &config)) {
+					/* Verify that invalid max limits are rejected. */
+					KUNIT_EXPECT_FALSE(test, ops->set_max_limits(link_caps, &config));
+
+					continue;
+				}
+
+				verify_link_caps_for_order(expected_table, link_caps, config_orders[i],
+							   &config);
+			}
+		}
+	}
+}
+
+static void update_link_caps_and_verify(struct test_config_table *expected_table,
+					struct intel_dp_link_caps *link_caps,
+					bool reset)
+{
+	struct kunit *test = expected_table->test;
+	struct test_ctx *ctx = test->priv;
+	const struct intel_dp_link_caps_test_ops *ops =
+		ctx->link_caps_ops;
+	bool link_params_changed;
+
+	link_params_changed = ops->update(link_caps,
+					  expected_table->rates.entries,
+					  expected_table->rates.size,
+					  expected_table->max_lane_count,
+					  reset);
+	KUNIT_EXPECT_TRUE(test, !reset || link_params_changed);
+
+	/*
+	 * ops->update() re-enables all configurations when called with
+	 * reset=true, or changed link parameters.
+	 */
+	if (link_params_changed)
+		expected_table->disabled_configs.size = 0;
+
+	verify_link_caps(expected_table, link_caps);
+}
+
+static void intel_dp_link_caps_test_update_reset(struct kunit *test)
+{
+	struct test_ctx *ctx = test->priv;
+	struct intel_dp_link_caps *link_caps = ctx->dev.dig_port.dp.link.caps;
+	int max_lane_count;
+	int num_rates;
+
+	for (max_lane_count = 1;
+	     max_lane_count <= LINK_TEST_MAX_LANE_COUNT;
+	     max_lane_count <<= 1) {
+		for (num_rates = 1;
+		     num_rates <= LINK_TEST_NUM_STANDARD_RATES;
+		     num_rates++) {
+			struct test_config_table expected_table =
+				INIT_STANDARD_TABLE(test, num_rates,
+						    max_lane_count);
+
+			update_link_caps_and_verify(&expected_table, link_caps, true);
+		}
+	}
+}
+
+/*
+ * TEST: Update shrink and expand
+ * ------------------------------
+ * Verify that removing or adding supported rates/lane counts updates
+ * the configuration table accordingly.
+ */
+static void disable_configs_and_verify(struct kunit *test,
+				       struct intel_dp_link_caps *link_caps,
+				       struct test_config_table *expected_table,
+				       const struct link_config_set *config_set)
+{
+	struct test_ctx *ctx = test->priv;
+	const struct intel_dp_link_caps_test_ops *ops =
+		ctx->link_caps_ops;
+	int i;
+
+	for (i = 0; i < config_set->size; i++) {
+		KUNIT_ASSERT_FALSE(test, has_config(&expected_table->disabled_configs,
+						    &config_set->entries[i]));
+		add_config(test, &expected_table->disabled_configs, &config_set->entries[i]);
+
+		KUNIT_ASSERT_TRUE(test, ops->disable_config(link_caps, &config_set->entries[i]));
+
+		verify_link_caps(expected_table, link_caps);
+	}
+}
+
+static void disable_configs_for_shrink_and_verify(struct test_config_table *expected_table,
+						  struct intel_dp_link_caps *link_caps)
+{
+	struct kunit *test = expected_table->test;
+	struct link_config_set config_set = {};
+	struct intel_dp_link_config max_config;
+
+	/*
+	 * When configs shrink disable the config with the
+	 * second-highest rate, lane params, so the disabled config
+	 * stays around after the configs got shrunk.
+	 */
+	KUNIT_ASSERT_GE(test, expected_table->rates.size, 2);
+	KUNIT_ASSERT_GE(test, expected_table->max_lane_count, 2);
+
+	max_config.rate = expected_table->rates.entries[expected_table->rates.size - 2];
+	max_config.lane_count = expected_table->max_lane_count >> 1;
+
+	add_config(test, &config_set, &max_config);
+	disable_configs_and_verify(test, link_caps, expected_table,
+				   &config_set);
+}
+
+static void disable_configs_for_expand_and_verify(struct test_config_table *expected_table,
+						  struct intel_dp_link_caps *link_caps)
+{
+	struct kunit *test = expected_table->test;
+	struct link_config_set config_set = {};
+	struct intel_dp_link_config max_config;
+
+	KUNIT_ASSERT_GE(test, expected_table->rates.size, 1);
+
+	max_config.rate = expected_table->rates.entries[expected_table->rates.size - 1];
+	max_config.lane_count = expected_table->max_lane_count;
+
+	add_config(test, &config_set, &max_config);
+	disable_configs_and_verify(test, link_caps, expected_table,
+				   &config_set);
+}
+
+static void get_nth_rate_lane_config(const struct test_config_table *expected_table, int n,
+				     struct intel_dp_link_config *config)
+{
+	int num_lane_configs = LINK_TEST_NUM_LANE_CONFIGS(expected_table->max_lane_count);
+	int rate_idx = n / num_lane_configs;
+	int lane_count_exp = n % num_lane_configs;
+
+	config->rate = expected_table->rates.entries[rate_idx];
+	config->lane_count = 1 << lane_count_exp;
+}
+
+static void test_update_rates_shrink(struct kunit *test, bool disable_configs)
+{
+	struct test_ctx *ctx = test->priv;
+	struct intel_dp_link_caps *link_caps = ctx->dev.dig_port.dp.link.caps;
+	struct test_config_table expected_table =
+		INIT_STANDARD_TABLE(test, LINK_TEST_NUM_STANDARD_RATES,
+					  LINK_TEST_MAX_LANE_COUNT);
+
+	update_link_caps_and_verify(&expected_table, link_caps, true);
+
+	while (expected_table.rates.size > 1) {
+		if (disable_configs)
+			disable_configs_for_shrink_and_verify(&expected_table, link_caps);
+
+		expected_table.rates.size--;
+
+		update_link_caps_and_verify(&expected_table, link_caps, false);
+	}
+}
+
+static void intel_dp_link_caps_test_update_rates_shrink(struct kunit *test)
+{
+	test_update_rates_shrink(test, false);
+}
+
+static void intel_dp_link_caps_test_update_rates_shrink_disable(struct kunit *test)
+{
+	test_update_rates_shrink(test, true);
+}
+
+static void test_update_rates_expand(struct kunit *test, bool disable_configs)
+{
+	struct test_ctx *ctx = test->priv;
+	struct intel_dp_link_caps *link_caps = ctx->dev.dig_port.dp.link.caps;
+	struct test_config_table expected_table =
+		INIT_STANDARD_TABLE(test, 1, LINK_TEST_MAX_LANE_COUNT);
+
+	update_link_caps_and_verify(&expected_table, link_caps, true);
+
+	while (expected_table.rates.size < LINK_TEST_NUM_STANDARD_RATES) {
+		if (disable_configs)
+			disable_configs_for_expand_and_verify(&expected_table, link_caps);
+
+		expected_table.rates.size++;
+
+		update_link_caps_and_verify(&expected_table, link_caps, false);
+	}
+}
+
+static void intel_dp_link_caps_test_update_rates_expand(struct kunit *test)
+{
+	test_update_rates_expand(test, false);
+}
+
+static void intel_dp_link_caps_test_update_rates_expand_disable(struct kunit *test)
+{
+	test_update_rates_expand(test, true);
+}
+
+static void test_update_lanes_shrink(struct kunit *test, bool disable_configs)
+{
+	struct test_ctx *ctx = test->priv;
+	struct intel_dp_link_caps *link_caps = ctx->dev.dig_port.dp.link.caps;
+	struct test_config_table expected_table =
+		INIT_STANDARD_TABLE(test, LINK_TEST_NUM_STANDARD_RATES,
+					  LINK_TEST_MAX_LANE_COUNT);
+
+	update_link_caps_and_verify(&expected_table, link_caps, true);
+
+	while (expected_table.max_lane_count > 1) {
+		if (disable_configs)
+			disable_configs_for_shrink_and_verify(&expected_table, link_caps);
+
+		expected_table.max_lane_count >>= 1;
+
+		update_link_caps_and_verify(&expected_table, link_caps, false);
+	}
+}
+
+static void intel_dp_link_caps_test_update_lanes_shrink(struct kunit *test)
+{
+	test_update_lanes_shrink(test, false);
+}
+
+static void intel_dp_link_caps_test_update_lanes_shrink_disable(struct kunit *test)
+{
+	test_update_lanes_shrink(test, true);
+}
+
+static void test_update_lanes_expand(struct kunit *test, bool disable_configs)
+{
+	struct test_ctx *ctx = test->priv;
+	struct intel_dp_link_caps *link_caps = ctx->dev.dig_port.dp.link.caps;
+	struct test_config_table expected_table =
+		INIT_STANDARD_TABLE(test, LINK_TEST_NUM_STANDARD_RATES, 1);
+
+	update_link_caps_and_verify(&expected_table, link_caps, true);
+
+	while (expected_table.max_lane_count < LINK_TEST_MAX_LANE_COUNT) {
+		if (disable_configs)
+			disable_configs_for_expand_and_verify(&expected_table, link_caps);
+
+		expected_table.max_lane_count <<= 1;
+
+		update_link_caps_and_verify(&expected_table, link_caps, false);
+	}
+}
+
+static void intel_dp_link_caps_test_update_lanes_expand(struct kunit *test)
+{
+	test_update_lanes_expand(test, false);
+}
+
+static void intel_dp_link_caps_test_update_lanes_expand_disable(struct kunit *test)
+{
+	test_update_lanes_expand(test, true);
+}
+
+static void disable_random_configs_and_verify(struct test_config_table *expected_table,
+					      struct intel_dp_link_caps *link_caps)
+{
+	struct kunit *test = expected_table->test;
+	struct test_ctx *ctx = test->priv;
+	struct link_config_set config_set = {};
+	u32 disabled_config_mask;
+	int num_configs;
+	int i;
+
+	num_configs = get_num_configs(expected_table->rates.size,
+				      expected_table->max_lane_count);
+	disabled_config_mask = prandom_u32_state(&ctx->rnd) &
+			       GENMASK_U32(num_configs - 1, 0);
+
+	for (i = 0; i < num_configs; i++) {
+		struct intel_dp_link_config config;
+
+		/* At least one config must remain enabled. */
+		if (expected_table->disabled_configs.size +
+		    config_set.size + 1 >= num_configs)
+			break;
+
+		if (!(BIT(i) & disabled_config_mask))
+			continue;
+
+		get_nth_rate_lane_config(expected_table, i, &config);
+		/* Don't disable a config twice. */
+		if (has_config(&expected_table->disabled_configs, &config))
+			continue;
+
+		add_config(test, &config_set, &config);
+	}
+
+	disable_configs_and_verify(test, link_caps, expected_table,
+				   &config_set);
+}
+
+static void get_params_shrink_step(struct test_ctx *ctx,
+				   int num_rates, int max_lane_count,
+				   int *rates_step, int *lanes_step)
+{
+	int shrink_mask;
+
+	*rates_step = 0;
+	*lanes_step = 0;
+
+	if (num_rates == 1)
+		shrink_mask = BIT(0);				/* shrink only lanes */
+	else if (max_lane_count == 1)
+		shrink_mask = BIT(1);				/* shrink only rates */
+	else
+		shrink_mask = rand_in_range(ctx,
+					    BIT(0),
+					    BIT(0) | BIT(1));	/* shrink one or both params */
+
+	if (shrink_mask & BIT(1))
+		*rates_step = rand_in_range(ctx, 1, num_rates - 1);
+
+	if (shrink_mask & BIT(0))
+		*lanes_step = rand_in_range(ctx, 1, ilog2(max_lane_count));
+}
+
+static void get_params_expand_step(struct test_ctx *ctx,
+				   int max_num_rates, int num_rates,
+				   int max_supported_lane_count, int max_lane_count,
+				   int *rates_step, int *lanes_step)
+{
+	int expand_mask;
+
+	*rates_step = 0;
+	*lanes_step = 0;
+
+	if (num_rates == max_num_rates)
+		expand_mask = BIT(0);				/* expand only lanes */
+	else if (max_lane_count == max_supported_lane_count)
+		expand_mask = BIT(1);				/* expand only rates */
+	else
+		expand_mask = rand_in_range(ctx,
+					    BIT(0),
+					    BIT(0) | BIT(1));	/* expand one or both params */
+
+	if (expand_mask & BIT(1))
+		*rates_step = rand_in_range(ctx, 1, max_num_rates - num_rates);
+
+	if (expand_mask & BIT(0))
+		*lanes_step = rand_in_range(ctx, 1, ilog2(max_supported_lane_count /
+							  max_lane_count));
+}
+
+static void test_update_params_shrink_random(struct kunit *test, bool disable_configs)
+{
+	struct test_ctx *ctx = test->priv;
+	struct intel_dp_link_caps *link_caps = ctx->dev.dig_port.dp.link.caps;
+	struct test_config_table expected_table =
+		INIT_STANDARD_TABLE(test, LINK_TEST_NUM_STANDARD_RATES,
+					  LINK_TEST_MAX_LANE_COUNT);
+
+	update_link_caps_and_verify(&expected_table, link_caps, true);
+
+	while (expected_table.rates.size > 1 || expected_table.max_lane_count > 1) {
+		int rates_step;
+		int lanes_step;
+
+		if (disable_configs)
+			disable_random_configs_and_verify(&expected_table, link_caps);
+
+		get_params_shrink_step(ctx,
+				       expected_table.rates.size,
+				       expected_table.max_lane_count,
+				       &rates_step, &lanes_step);
+
+		expected_table.rates.size -= rates_step;
+		expected_table.max_lane_count >>= lanes_step;
+
+		update_link_caps_and_verify(&expected_table, link_caps, false);
+	}
+}
+
+static void intel_dp_link_caps_test_update_params_shrink_random(struct kunit *test)
+{
+	int i;
+
+	for (i = 0; i < LINK_TEST_NUM_RANDOM_ITERATIONS; i++)
+		test_update_params_shrink_random(test, false);
+}
+
+static void intel_dp_link_caps_test_update_params_shrink_disable_random(struct kunit *test)
+{
+	int i;
+
+	for (i = 0; i < LINK_TEST_NUM_RANDOM_ITERATIONS; i++)
+		test_update_params_shrink_random(test, true);
+}
+
+static void test_update_params_expand_random(struct kunit *test, bool disable_configs)
+{
+	struct test_ctx *ctx = test->priv;
+	struct intel_dp_link_caps *link_caps = ctx->dev.dig_port.dp.link.caps;
+	struct test_config_table expected_table =
+		INIT_STANDARD_TABLE(test, 1, 1);
+
+	update_link_caps_and_verify(&expected_table, link_caps, true);
+
+	while (expected_table.rates.size < LINK_TEST_NUM_STANDARD_RATES ||
+	       expected_table.max_lane_count < LINK_TEST_MAX_LANE_COUNT) {
+		int rates_step;
+		int lanes_step;
+
+		if (disable_configs)
+			disable_random_configs_and_verify(&expected_table, link_caps);
+
+		get_params_expand_step(ctx,
+				       LINK_TEST_NUM_STANDARD_RATES,
+				       expected_table.rates.size,
+				       LINK_TEST_MAX_LANE_COUNT,
+				       expected_table.max_lane_count,
+				       &rates_step, &lanes_step);
+
+		expected_table.rates.size += rates_step;
+		expected_table.max_lane_count <<= lanes_step;
+
+		update_link_caps_and_verify(&expected_table, link_caps, false);
+	}
+}
+
+static void intel_dp_link_caps_test_update_params_expand_random(struct kunit *test)
+{
+	int i;
+
+	for (i = 0; i < LINK_TEST_NUM_RANDOM_ITERATIONS; i++)
+		test_update_params_expand_random(test, false);
+}
+
+static void intel_dp_link_caps_test_update_params_expand_disable_random(struct kunit *test)
+{
+	int i;
+
+	for (i = 0; i < LINK_TEST_NUM_RANDOM_ITERATIONS; i++)
+		test_update_params_expand_random(test, true);
+}
+
+/*
+ * TEST: Fallback sequence
+ * -----------------------
+ * Verify the eDP fallback logic to set the maximum supported configuration
+ * as a preference.
+ *
+ * For DP SST and MST verify fallback selection from the connector's
+ * maximum configuration and iteration of the resulting allowed
+ * configurations.
+ */
+static void intel_dp_link_test_fallback_for_edp(struct kunit *test)
+{
+	struct test_ctx *ctx = test->priv;
+	struct intel_dp_link_caps *link_caps = ctx->dev.dig_port.dp.link.caps;
+	struct test_config_table expected_table =
+		INIT_STANDARD_TABLE(test, LINK_TEST_NUM_STANDARD_RATES,
+					  LINK_TEST_MAX_LANE_COUNT);
+	struct intel_digital_port *dig_port = &ctx->dev.dig_port;
+	const struct intel_dp_link_training_test_ops *lt_ops =
+		ctx->link_training_ops;
+	const struct intel_dp_link_caps_test_ops *lc_ops =
+		ctx->link_caps_ops;
+	struct intel_dp_link_config min_config = {
+		.rate = expected_table.rates.entries[0],
+		.lane_count = 1,
+	};
+	struct intel_dp_link_config max_config = {
+		.rate = expected_table.rates.entries[expected_table.rates.size - 1],
+		.lane_count = LINK_TEST_MAX_LANE_COUNT,
+	};
+	struct intel_dp_link_caps_order order;
+	struct intel_dp_link_config iter_config;
+	struct intel_dp_link_caps_iter iter;
+	int fallback_err;
+
+	dig_port->base.type = INTEL_OUTPUT_EDP;
+	ctx->dev.dig_port.dp.use_max_params = false;
+
+	update_link_caps_and_verify(&expected_table, link_caps, true);
+
+	order = lc_ops->connector_compute_order(&ctx->dev.connector);
+
+	lc_ops->iter_start(&iter, link_caps, order, INTEL_DP_LINK_CAPS_FILTER_ALL);
+	for_each_dp_link_config(&iter, &iter_config)
+		break;
+	lc_ops->iter_end(&iter);
+
+	KUNIT_EXPECT_FALSE(test, ctx->dev.dig_port.dp.use_max_params);
+	KUNIT_EXPECT_TRUE(test, link_configs_match(&iter_config, &min_config));
+
+	ctx->dev.crtc_state.output_types = BIT(dig_port->base.type);
+	ctx->dev.crtc_state.port_clock = min_config.rate;
+	ctx->dev.crtc_state.lane_count = min_config.lane_count;
+
+	fallback_err = lt_ops->get_fallback_values(&ctx->dev.dig_port.dp, &ctx->dev.crtc_state);
+	KUNIT_EXPECT_EQ(test, fallback_err, 0);
+
+	/* The fallback should've changed the order. */
+	order = lc_ops->connector_compute_order(&ctx->dev.connector);
+
+	lc_ops->iter_start(&iter, link_caps, order, INTEL_DP_LINK_CAPS_FILTER_ALL);
+	for_each_dp_link_config(&iter, &iter_config)
+		break;
+	lc_ops->iter_end(&iter);
+
+	KUNIT_EXPECT_TRUE(test, ctx->dev.dig_port.dp.use_max_params);
+	KUNIT_EXPECT_TRUE(test, link_configs_match(&iter_config, &max_config));
+}
+
+static bool test_fallback_from_target(struct test_config_table *expected_table,
+				      enum intel_output_type output_type, int max_rate,
+				      const struct intel_dp_link_config *expected_target_config,
+				      const struct intel_dp_link_config *expected_fallback_config)
+{
+	struct kunit *test = expected_table->test;
+	struct test_ctx *ctx = test->priv;
+	struct intel_dp_link_caps *link_caps = ctx->dev.dig_port.dp.link.caps;
+	struct intel_dp_link_config iter_config;
+	const struct intel_dp_link_training_test_ops *lt_ops =
+		ctx->link_training_ops;
+	const struct intel_dp_link_caps_test_ops *lc_ops =
+		ctx->link_caps_ops;
+	/* Modify default order direction for max config lookup. */
+	struct intel_dp_link_caps_order fallback_order =
+		lc_ops->connector_fallback_order(ctx->dev.connector.mst.dp);
+	struct intel_dp_link_caps_iter iter;
+	int expected_fallback_err = 0;
+	int fallback_err;
+
+	/* Get the max connector config, optionally filtered to the max_rate limit. */
+	lc_ops->iter_start(&iter, link_caps, fallback_order, INTEL_DP_LINK_CAPS_FILTER_ALL);
+	for_each_dp_link_config(&iter, &iter_config)
+		if (max_rate == 0 || iter_config.rate <= max_rate)
+			break;
+	lc_ops->iter_end(&iter);
+
+	KUNIT_EXPECT_TRUE(test, link_configs_match(&iter_config,
+						   expected_target_config));
+	KUNIT_EXPECT_FALSE(test, link_configs_match(&iter_config,
+						    &INTEL_DP_LINK_CONFIG_NULL));
+
+	ctx->dev.crtc_state.output_types = BIT(output_type);
+	ctx->dev.crtc_state.port_clock = expected_target_config->rate;
+	ctx->dev.crtc_state.lane_count = expected_target_config->lane_count;
+
+	if (link_configs_match(expected_fallback_config, &INTEL_DP_LINK_CONFIG_NULL))
+		expected_fallback_err = -1;
+
+	fallback_err = lt_ops->get_fallback_values(&ctx->dev.dig_port.dp, &ctx->dev.crtc_state);
+	KUNIT_EXPECT_EQ(test, fallback_err, expected_fallback_err);
+
+	if (!fallback_err) {
+		/*
+		 * NOTE: This test does not verify any implied fallback
+		 * target selection.
+		 *
+		 * The current driver behavior may still select a fallback
+		 * configuration indirectly via max_limits, but that is an
+		 * implementation artifact rather than part of the intended
+		 * fallback API behavior, and is therefore not verified here.
+		 *
+		 * Instead, the effect of the fallback logic is verified by
+		 * checking that the failed target configuration is disabled.
+		 * Selecting the next target configuration from the remaining
+		 * allowed configurations belongs to the modeset link target
+		 * selection logic.
+		 */
+		add_config(test, &expected_table->disabled_configs,
+			   expected_target_config);
+	}
+
+	verify_link_caps(expected_table, link_caps);
+
+	return !fallback_err;
+}
+
+static const struct link_config_set *
+get_target_configs_for_output_type(struct kunit *test,
+				   enum intel_output_type output_type)
+{
+	switch (output_type) {
+	case INTEL_OUTPUT_DDI:
+	case INTEL_OUTPUT_DP:
+	case INTEL_OUTPUT_EDP:
+		return &standard_dp_link_configs[INTEL_DP_LINK_CAPS_ORDER_KEY_RATE_LANE];
+	case INTEL_OUTPUT_DP_MST:
+		return &standard_dp_link_configs[INTEL_DP_LINK_CAPS_ORDER_KEY_BW];
+	default:
+		KUNIT_FAIL_AND_ABORT(test, "Missing output type: %d", output_type);
+	}
+}
+
+static const struct link_config_set *
+get_fallback_configs_for_output_type(struct kunit *test,
+				     enum intel_output_type output_type)
+{
+	switch (output_type) {
+	case INTEL_OUTPUT_DDI:
+	case INTEL_OUTPUT_DP:
+	case INTEL_OUTPUT_EDP:
+		return &standard_dp_link_configs[INTEL_DP_LINK_CAPS_ORDER_KEY_LANE_RATE];
+	case INTEL_OUTPUT_DP_MST:
+		return &standard_dp_link_configs[INTEL_DP_LINK_CAPS_ORDER_KEY_BW];
+	default:
+		KUNIT_FAIL_AND_ABORT(test, "Missing output type: %d", output_type);
+	}
+}
+
+static bool output_type_allows_uhbr_fallback(enum intel_output_type output_type)
+{
+	return output_type == INTEL_OUTPUT_DP_MST;
+}
+
+static void assert_config_is_supported(const struct test_config_table *expected_table,
+				       const struct intel_dp_link_config *config)
+{
+	struct kunit *test = expected_table->test;
+
+	KUNIT_ASSERT_TRUE(test, has_rate(&expected_table->rates, config->rate));
+	KUNIT_ASSERT_LE(test, config->lane_count, expected_table->max_lane_count);
+}
+
+static bool get_fallback_config(const struct test_config_table *expected_table,
+				enum intel_output_type output_type,
+				const struct intel_dp_link_config *target_config,
+				struct intel_dp_link_config *fallback_config)
+{
+	struct kunit *test = expected_table->test;
+	const struct link_config_set *config_set =
+		get_fallback_configs_for_output_type(test, output_type);
+	int i;
+
+	i = lookup_config(config_set, target_config);
+	KUNIT_ASSERT_GE(test, i, 0);
+
+	for (i--; i >= 0; i--) {
+		const struct intel_dp_link_config *config =
+			&config_set->entries[i];
+
+		if (output_type_allows_uhbr_fallback(output_type) ||
+		    (drm_dp_is_uhbr_rate(target_config->rate) ==
+		     drm_dp_is_uhbr_rate(config->rate))) {
+			assert_config_is_supported(expected_table, config);
+			*fallback_config = *config;
+
+			return true;
+		}
+	}
+
+	return false;
+}
+
+static bool get_target_config(const struct test_config_table *expected_table,
+			      enum intel_output_type output_type,
+			      int max_rate,
+			      struct intel_dp_link_config *target)
+{
+	struct kunit *test = expected_table->test;
+	const struct link_config_set *config_set =
+		get_target_configs_for_output_type(test, output_type);
+	int i;
+
+	for (i = config_set->size - 1; i >= 0; i--) {
+		const struct intel_dp_link_config *config =
+			&config_set->entries[i];
+
+		if (config->rate <= max_rate) {
+			assert_config_is_supported(expected_table, config);
+			*target = *config;
+
+			return true;
+		}
+	}
+
+	return false;
+}
+
+static void test_fallback_seq(struct kunit *test,
+			      enum intel_output_type output_type,
+			      bool uhbr)
+{
+	struct test_ctx *ctx = test->priv;
+	struct intel_dp_link_caps *link_caps = ctx->dev.dig_port.dp.link.caps;
+	struct test_config_table expected_table =
+		INIT_STANDARD_TABLE(test, LINK_TEST_NUM_STANDARD_RATES,
+					  LINK_TEST_MAX_LANE_COUNT);
+	struct intel_digital_port *dig_port = &ctx->dev.dig_port;
+	struct intel_dp_link_config fallback_config = {};
+	struct intel_dp_link_config target_config;
+	int fallback_count = 0;
+	bool target_found;
+	int max_rate;
+
+	if (uhbr)
+		max_rate = expected_table.rates.entries[expected_table.rates.size - 1];
+	else
+		max_rate = 810000;
+
+	dig_port->base.type = output_type;
+	ctx->dev.dig_port.dp.use_max_params = false;
+
+	update_link_caps_and_verify(&expected_table, link_caps, true);
+
+	/* Get the initial target config. */
+	target_found = get_target_config(&expected_table, output_type,
+					 max_rate, &target_config);
+	KUNIT_ASSERT_TRUE(test, target_found);
+
+	for (;;) {
+		/* Also test the case where no fallback is available. */
+		if (!get_fallback_config(&expected_table, output_type,
+					 &target_config, &fallback_config))
+			fallback_config = INTEL_DP_LINK_CONFIG_NULL;
+
+		if (!test_fallback_from_target(&expected_table, output_type, max_rate,
+					       &target_config, &fallback_config))
+			break;
+
+		/*
+		 * The fallback changed the max rate allowed for the next
+		 * target.
+		 */
+		max_rate = fallback_config.rate;
+
+		/* Simply select the fallback config as the next target. */
+		target_config = fallback_config;
+
+		fallback_count++;
+		KUNIT_ASSERT_LT(test, fallback_count, LINK_TEST_MAX_CONFIGS);
+	}
+}
+
+static void intel_dp_link_test_fallback_for_sst_max_non_uhbr(struct kunit *test)
+{
+	test_fallback_seq(test, INTEL_OUTPUT_DP, false);
+}
+
+static void intel_dp_link_test_fallback_for_sst_max_uhbr(struct kunit *test)
+{
+	test_fallback_seq(test, INTEL_OUTPUT_DP, true);
+}
+
+static void intel_dp_link_test_fallback_for_mst(struct kunit *test)
+{
+	struct test_ctx *ctx = test->priv;
+
+	ctx->dev.connector.mst.dp = &ctx->dev.dig_port.dp;
+
+	test_fallback_seq(test, INTEL_OUTPUT_DP_MST, true);
+}
+
+static struct kunit_case intel_dp_link_test_cases[] = {
+	KUNIT_CASE(intel_dp_link_caps_test_baseline),
+
+	KUNIT_CASE(intel_dp_link_caps_test_update_reset),
+
+	KUNIT_CASE(intel_dp_link_caps_test_update_rates_shrink),
+	KUNIT_CASE(intel_dp_link_caps_test_update_rates_shrink_disable),
+	KUNIT_CASE(intel_dp_link_caps_test_update_rates_expand),
+	KUNIT_CASE(intel_dp_link_caps_test_update_rates_expand_disable),
+	KUNIT_CASE(intel_dp_link_caps_test_update_lanes_shrink),
+	KUNIT_CASE(intel_dp_link_caps_test_update_lanes_shrink_disable),
+	KUNIT_CASE(intel_dp_link_caps_test_update_lanes_expand),
+	KUNIT_CASE(intel_dp_link_caps_test_update_lanes_expand_disable),
+	KUNIT_CASE(intel_dp_link_caps_test_update_params_shrink_random),
+	KUNIT_CASE(intel_dp_link_caps_test_update_params_shrink_disable_random),
+	KUNIT_CASE(intel_dp_link_caps_test_update_params_expand_random),
+	KUNIT_CASE(intel_dp_link_caps_test_update_params_expand_disable_random),
+
+	KUNIT_CASE(intel_dp_link_test_fallback_for_edp),
+	KUNIT_CASE(intel_dp_link_test_fallback_for_sst_max_non_uhbr),
+	KUNIT_CASE(intel_dp_link_test_fallback_for_sst_max_uhbr),
+	KUNIT_CASE(intel_dp_link_test_fallback_for_mst),
+
+	{}
+};
+
+static struct test_ctx test_ctx;
+
+static int intel_dp_link_test_init(struct kunit *test)
+{
+	struct intel_digital_port *dig_port;
+	struct intel_encoder *encoder;
+	struct intel_dp *intel_dp;
+
+	/* Reset the dev state for each test. */
+	memset(&test_ctx.dev, 0, sizeof(test_ctx.dev));
+
+	test_ctx.dev.generic_device.drm.dev = &test_ctx.dev.device;
+
+	test_ctx.dev.display.drm = &test_ctx.dev.generic_device.drm;
+	test_ctx.dev.generic_device.display = &test_ctx.dev.display;
+
+	encoder = &test_ctx.dev.dig_port.base;
+	encoder->base.dev = &test_ctx.dev.generic_device.drm;
+
+	dig_port = &test_ctx.dev.dig_port;
+	dig_port->base.type = INTEL_OUTPUT_DP;
+
+	test_ctx.dev.connector.encoder = encoder;
+
+	intel_dp = &dig_port->dp;
+	intel_dp->attached_connector = &test_ctx.dev.connector;
+
+	intel_dp->link.caps = test_ctx.link_caps_ops->init(intel_dp);
+
+	test->priv = &test_ctx;
+
+	return 0;
+}
+
+static void intel_dp_link_test_exit(struct kunit *test)
+{
+	struct test_ctx *ctx = test->priv;
+
+	ctx->link_caps_ops->cleanup(ctx->dev.dig_port.dp.link.caps);
+}
+
+static int intel_dp_link_test_suite_init(struct kunit_suite *test_suite)
+{
+#ifdef I915
+	test_ctx.link_caps_ops = &i915_display_dp_link_caps_test_ops;
+	test_ctx.link_training_ops = &i915_display_dp_link_training_test_ops;
+#else
+	test_ctx.link_caps_ops = &intel_display_dp_link_caps_test_ops;
+	test_ctx.link_training_ops = &intel_display_dp_link_training_test_ops;
+#endif
+	prandom_seed_state(&test_ctx.rnd, 0);
+
+	return 0;
+}
+
+static struct kunit_suite intel_dp_link_test_suite = {
+	.name = "intel_dp_link",
+	.suite_init = intel_dp_link_test_suite_init,
+	.init = intel_dp_link_test_init,
+	.exit = intel_dp_link_test_exit,
+	.test_cases = intel_dp_link_test_cases,
+};
+
+kunit_test_suites(&intel_dp_link_test_suite);
+
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
+
+MODULE_AUTHOR("Intel Corporation");
+MODULE_LICENSE("GPL and additional rights");
+MODULE_DESCRIPTION("Intel DP link KUnit tests");
diff --git a/drivers/gpu/drm/i915/display/vlv_dsi.c b/drivers/gpu/drm/i915/display/vlv_dsi.c
index 877eab75f19af..8829f365592ef 100644
--- a/drivers/gpu/drm/i915/display/vlv_dsi.c
+++ b/drivers/gpu/drm/i915/display/vlv_dsi.c
@@ -266,7 +266,8 @@ static void band_gap_reset(struct intel_display *display)
 	vlv_flisdsi_put(display);
 }
 
-static int intel_dsi_compute_config(struct intel_encoder *encoder,
+static int intel_dsi_compute_config(struct intel_atomic_state *state,
+				    struct intel_encoder *encoder,
 				    struct intel_crtc_state *pipe_config,
 				    struct drm_connector_state *conn_state)
 {
@@ -280,7 +281,7 @@ static int intel_dsi_compute_config(struct intel_encoder *encoder,
 	pipe_config->sink_format = INTEL_OUTPUT_FORMAT_RGB;
 	pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
 
-	ret = intel_panel_compute_config(intel_connector, adjusted_mode);
+	ret = intel_panel_compute_config(state, pipe_config, intel_connector);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_domain.c b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
index df7502391b504..b7297c0a3a48e 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_domain.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
@@ -681,7 +681,7 @@ int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj,
 		return ret;
 
 	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||
-	    !static_cpu_has(X86_FEATURE_CLFLUSH)) {
+	    !cpu_feature_enabled(X86_FEATURE_CLFLUSH)) {
 		ret = i915_gem_object_set_to_cpu_domain(obj, false);
 		if (ret)
 			goto err_unpin;
@@ -732,7 +732,7 @@ int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj,
 		return ret;
 
 	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||
-	    !static_cpu_has(X86_FEATURE_CLFLUSH)) {
+	    !cpu_feature_enabled(X86_FEATURE_CLFLUSH)) {
 		ret = i915_gem_object_set_to_cpu_domain(obj, true);
 		if (ret)
 			goto err_unpin;
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
index 0644f85c6c8e5..9ca90c1bb5b42 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
@@ -112,7 +112,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 		vma = find_vma(mm, addr);
 		if (vma && __vma_matches(vma, obj->base.filp, addr, args->size))
 			vma->vm_page_prot =
-				pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+				pgprot_writecombine(vma_get_page_prot(vma));
 		else
 			addr = -ENOMEM;
 		mmap_write_unlock(mm);
@@ -1024,7 +1024,7 @@ i915_gem_object_mmap(struct drm_i915_gem_object *obj,
 	fput(anon);
 
 	if (obj->ops->mmap_ops) {
-		vma->vm_page_prot = pgprot_decrypted(vm_get_page_prot(vma->vm_flags));
+		vma->vm_page_prot = pgprot_decrypted(vma_get_page_prot(vma));
 		vma->vm_ops = obj->ops->mmap_ops;
 		vma->vm_private_data = obj->base.vma_node.driver_private;
 		return 0;
@@ -1035,7 +1035,7 @@ i915_gem_object_mmap(struct drm_i915_gem_object *obj,
 	switch (mmo->mmap_type) {
 	case I915_MMAP_TYPE_WC:
 		vma->vm_page_prot =
-			pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+			pgprot_writecombine(vma_get_page_prot(vma));
 		vma->vm_ops = &vm_ops_cpu;
 		break;
 
@@ -1043,19 +1043,19 @@ i915_gem_object_mmap(struct drm_i915_gem_object *obj,
 		GEM_WARN_ON(1);
 		fallthrough;
 	case I915_MMAP_TYPE_WB:
-		vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+		vma->vm_page_prot = vma_get_page_prot(vma);
 		vma->vm_ops = &vm_ops_cpu;
 		break;
 
 	case I915_MMAP_TYPE_UC:
 		vma->vm_page_prot =
-			pgprot_noncached(vm_get_page_prot(vma->vm_flags));
+			pgprot_noncached(vma_get_page_prot(vma));
 		vma->vm_ops = &vm_ops_cpu;
 		break;
 
 	case I915_MMAP_TYPE_GTT:
 		vma->vm_page_prot =
-			pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+			pgprot_writecombine(vma_get_page_prot(vma));
 		vma->vm_ops = &vm_ops_gtt;
 		break;
 	}
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h
index 8878539c10edd..2c5d20e4dbafc 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h
@@ -17,8 +17,6 @@
 #include "i915_vma_types.h"
 
 enum intel_region_id;
-struct drm_scanout_buffer;
-struct intel_panic;
 
 #define obj_to_i915(obj__) to_i915((obj__)->base.dev)
 
@@ -693,11 +691,6 @@ i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
 int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
 int i915_gem_object_truncate(struct drm_i915_gem_object *obj);
 
-struct intel_panic *i915_gem_object_alloc_panic(void);
-int i915_gem_object_panic_setup(struct intel_panic *panic, struct drm_scanout_buffer *sb,
-				struct drm_gem_object *_obj, bool panic_tiling);
-void i915_gem_object_panic_finish(struct intel_panic *panic);
-
 /**
  * i915_gem_object_pin_map - return a contiguous mapping of the entire object
  * @obj: the object to map into kernel address space
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c
index df35bdb755e46..59e3d4de7d3c8 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c
@@ -6,11 +6,8 @@
 #include <linux/vmalloc.h>
 
 #include <drm/drm_cache.h>
-#include <drm/drm_panic.h>
 #include <drm/drm_print.h>
 
-#include "display/intel_fb.h"
-#include "display/intel_display_types.h"
 #include "gt/intel_gt.h"
 #include "gt/intel_tlb.h"
 
@@ -359,131 +356,6 @@ static void *i915_gem_object_map_pfn(struct drm_i915_gem_object *obj,
 	return vaddr ?: ERR_PTR(-ENOMEM);
 }
 
-struct intel_panic {
-	struct page **pages;
-	int page;
-	void *vaddr;
-};
-
-static void i915_panic_kunmap(struct intel_panic *panic)
-{
-	if (panic->vaddr) {
-		drm_clflush_virt_range(panic->vaddr, PAGE_SIZE);
-		kunmap_local(panic->vaddr);
-		panic->vaddr = NULL;
-	}
-}
-
-static struct page **i915_gem_object_panic_pages(struct drm_i915_gem_object *obj)
-{
-	unsigned long n_pages = obj->base.size >> PAGE_SHIFT, i;
-	struct page *page;
-	struct page **pages;
-	struct sgt_iter iter;
-
-	/* For a 3840x2160 32 bits Framebuffer, this should require ~64K */
-	pages = kmalloc_objs(*pages, n_pages, GFP_ATOMIC);
-	if (!pages)
-		return NULL;
-
-	i = 0;
-	for_each_sgt_page(page, iter, obj->mm.pages)
-		pages[i++] = page;
-	return pages;
-}
-
-static void i915_gem_object_panic_map_set_pixel(struct drm_scanout_buffer *sb, unsigned int x,
-						unsigned int y, u32 color)
-{
-	struct intel_framebuffer *fb = (struct intel_framebuffer *)sb->private;
-	unsigned int offset = fb->panic_tiling(sb->width, x, y);
-
-	iosys_map_wr(&sb->map[0], offset, u32, color);
-}
-
-/*
- * The scanout buffer pages are not mapped, so for each pixel,
- * use kmap_local_page_try_from_panic() to map the page, and write the pixel.
- * Try to keep the map from the previous pixel, to avoid too much map/unmap.
- */
-static void i915_gem_object_panic_page_set_pixel(struct drm_scanout_buffer *sb, unsigned int x,
-						 unsigned int y, u32 color)
-{
-	unsigned int new_page;
-	unsigned int offset;
-	struct intel_framebuffer *fb = (struct intel_framebuffer *)sb->private;
-	struct intel_panic *panic = fb->panic;
-
-	if (fb->panic_tiling)
-		offset = fb->panic_tiling(sb->width, x, y);
-	else
-		offset = y * sb->pitch[0] + x * sb->format->cpp[0];
-
-	new_page = offset >> PAGE_SHIFT;
-	offset = offset % PAGE_SIZE;
-	if (new_page != panic->page) {
-		i915_panic_kunmap(panic);
-		panic->page = new_page;
-		panic->vaddr =
-			kmap_local_page_try_from_panic(panic->pages[panic->page]);
-	}
-	if (panic->vaddr) {
-		u32 *pix = panic->vaddr + offset;
-		*pix = color;
-	}
-}
-
-struct intel_panic *i915_gem_object_alloc_panic(void)
-{
-	struct intel_panic *panic;
-
-	panic = kzalloc_obj(*panic);
-
-	return panic;
-}
-
-/*
- * Setup the gem framebuffer for drm_panic access.
- * Use current vaddr if it exists, or setup a list of pages.
- * pfn is not supported yet.
- */
-int i915_gem_object_panic_setup(struct intel_panic *panic, struct drm_scanout_buffer *sb,
-				struct drm_gem_object *_obj, bool panic_tiling)
-{
-	enum i915_map_type has_type;
-	struct drm_i915_gem_object *obj = to_intel_bo(_obj);
-	void *ptr;
-
-	ptr = page_unpack_bits(obj->mm.mapping, &has_type);
-	if (ptr) {
-		if (i915_gem_object_has_iomem(obj))
-			iosys_map_set_vaddr_iomem(&sb->map[0], (void __iomem *)ptr);
-		else
-			iosys_map_set_vaddr(&sb->map[0], ptr);
-
-		if (panic_tiling)
-			sb->set_pixel = i915_gem_object_panic_map_set_pixel;
-		return 0;
-	}
-	if (i915_gem_object_has_struct_page(obj)) {
-		panic->pages = i915_gem_object_panic_pages(obj);
-		if (!panic->pages)
-			return -ENOMEM;
-		panic->page = -1;
-		sb->set_pixel = i915_gem_object_panic_page_set_pixel;
-		return 0;
-	}
-	return -EOPNOTSUPP;
-}
-
-void i915_gem_object_panic_finish(struct intel_panic *panic)
-{
-	i915_panic_kunmap(panic);
-	panic->page = -1;
-	kfree(panic->pages);
-	panic->pages = NULL;
-}
-
 /* get, pin, and map the pages of the object into kernel space */
 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
 			      enum i915_map_type type)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_panic.c b/drivers/gpu/drm/i915/gem/i915_gem_panic.c
new file mode 100644
index 0000000000000..91389d36f1019
--- /dev/null
+++ b/drivers/gpu/drm/i915/gem/i915_gem_panic.c
@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: MIT
+/* Copyright © 2026 Intel Corporation */
+
+#include <drm/drm_cache.h>
+#include <drm/drm_panic.h>
+#include <drm/intel/display_parent_interface.h>
+
+#include "i915_gem_object.h"
+#include "i915_gem_panic.h"
+
+struct intel_panic {
+	struct page **pages;
+	int page;
+	void *vaddr;
+
+	unsigned int (*tiling)(unsigned int x, unsigned int y, unsigned int width);
+};
+
+static void i915_panic_kunmap(struct intel_panic *panic)
+{
+	if (panic->vaddr) {
+		drm_clflush_virt_range(panic->vaddr, PAGE_SIZE);
+		kunmap_local(panic->vaddr);
+		panic->vaddr = NULL;
+	}
+}
+
+static struct page **i915_gem_object_panic_pages(struct drm_i915_gem_object *obj)
+{
+	unsigned long n_pages = obj->base.size >> PAGE_SHIFT, i;
+	struct page *page;
+	struct page **pages;
+	struct sgt_iter iter;
+
+	/* For a 3840x2160 32 bits Framebuffer, this should require ~64K */
+	pages = kmalloc_objs(*pages, n_pages, GFP_ATOMIC);
+	if (!pages)
+		return NULL;
+
+	i = 0;
+	for_each_sgt_page(page, iter, obj->mm.pages)
+		pages[i++] = page;
+	return pages;
+}
+
+static void i915_gem_object_panic_map_set_pixel(struct drm_scanout_buffer *sb, unsigned int x,
+						unsigned int y, u32 color)
+{
+	struct intel_panic *panic = sb->private;
+	unsigned int offset = panic->tiling(sb->width, x, y);
+
+	iosys_map_wr(&sb->map[0], offset, u32, color);
+}
+
+/*
+ * The scanout buffer pages are not mapped, so for each pixel,
+ * use kmap_local_page_try_from_panic() to map the page, and write the pixel.
+ * Try to keep the map from the previous pixel, to avoid too much map/unmap.
+ */
+static void i915_gem_object_panic_page_set_pixel(struct drm_scanout_buffer *sb, unsigned int x,
+						 unsigned int y, u32 color)
+{
+	struct intel_panic *panic = sb->private;
+	unsigned int new_page;
+	unsigned int offset;
+
+	if (panic->tiling)
+		offset = panic->tiling(sb->width, x, y);
+	else
+		offset = y * sb->pitch[0] + x * sb->format->cpp[0];
+
+	new_page = offset >> PAGE_SHIFT;
+	offset = offset % PAGE_SIZE;
+	if (new_page != panic->page) {
+		i915_panic_kunmap(panic);
+		panic->page = new_page;
+		panic->vaddr =
+			kmap_local_page_try_from_panic(panic->pages[panic->page]);
+	}
+	if (panic->vaddr) {
+		u32 *pix = panic->vaddr + offset;
+		*pix = color;
+	}
+}
+
+static struct intel_panic *i915_gem_object_alloc_panic(void)
+{
+	struct intel_panic *panic;
+
+	panic = kzalloc_obj(*panic);
+
+	return panic;
+}
+
+/*
+ * Setup the gem framebuffer for drm_panic access.
+ * Use current vaddr if it exists, or setup a list of pages.
+ * pfn is not supported yet.
+ */
+static int i915_gem_object_panic_setup(struct intel_panic *panic, struct drm_scanout_buffer *sb,
+				       struct drm_gem_object *_obj,
+				       unsigned int (*tiling)(unsigned int x, unsigned int y, unsigned int width))
+{
+	enum i915_map_type has_type;
+	struct drm_i915_gem_object *obj = to_intel_bo(_obj);
+	void *ptr;
+
+	sb->private = panic;
+
+	ptr = page_unpack_bits(obj->mm.mapping, &has_type);
+	if (ptr) {
+		if (i915_gem_object_has_iomem(obj))
+			iosys_map_set_vaddr_iomem(&sb->map[0], (void __iomem *)ptr);
+		else
+			iosys_map_set_vaddr(&sb->map[0], ptr);
+
+		if (tiling) {
+			panic->tiling = tiling;
+			sb->set_pixel = i915_gem_object_panic_map_set_pixel;
+		}
+		return 0;
+	}
+	if (i915_gem_object_has_struct_page(obj)) {
+		panic->pages = i915_gem_object_panic_pages(obj);
+		if (!panic->pages)
+			return -ENOMEM;
+		panic->page = -1;
+		panic->tiling = tiling;
+		sb->set_pixel = i915_gem_object_panic_page_set_pixel;
+		return 0;
+	}
+	return -EOPNOTSUPP;
+}
+
+static void i915_gem_object_panic_finish(struct intel_panic *panic)
+{
+	i915_panic_kunmap(panic);
+	panic->page = -1;
+	kfree(panic->pages);
+	panic->pages = NULL;
+}
+
+const struct intel_display_panic_interface i915_display_panic_interface = {
+	.alloc = i915_gem_object_alloc_panic,
+	.setup = i915_gem_object_panic_setup,
+	.finish = i915_gem_object_panic_finish,
+};
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_panic.h b/drivers/gpu/drm/i915/gem/i915_gem_panic.h
new file mode 100644
index 0000000000000..82c3aca6f1f30
--- /dev/null
+++ b/drivers/gpu/drm/i915/gem/i915_gem_panic.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef __I915_GEM_PANIC_H__
+#define __I915_GEM_PANIC_H__
+
+#include <linux/types.h>
+
+extern const struct intel_display_panic_interface i915_display_panic_interface;
+
+#endif /* __I915_GEM_PANIC_H__ */
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
index 06543ae60706e..ef94401662955 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
@@ -325,7 +325,7 @@ void __shmem_writeback(size_t size, struct address_space *mapping)
 		if (folio_mapped(folio))
 			folio_redirty_for_writepage(&wbc, folio);
 		else
-			error = shmem_writeout(folio, NULL, NULL);
+			error = shmem_write_folio(folio);
 	}
 }
 
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_user.c b/drivers/gpu/drm/i915/gt/intel_engine_user.c
index be4bbff1a57c9..d5190e11b2706 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_user.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_user.c
@@ -259,7 +259,7 @@ void intel_engines_driver_register(struct drm_i915_private *i915)
 		p = &prev->rb_right;
 	}
 
-	if (IS_ENABLED(CONFIG_DRM_I915_SELFTESTS) &&
+	if (IS_ENABLED(CONFIG_DRM_I915_SELFTEST) &&
 	    IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) {
 		struct intel_engine_cs *engine;
 		unsigned int isolation;
diff --git a/drivers/gpu/drm/i915/gt/intel_gsc.c b/drivers/gpu/drm/i915/gt/intel_gsc.c
index 050d909fb4f87..1c06bf76568a6 100644
--- a/drivers/gpu/drm/i915/gt/intel_gsc.c
+++ b/drivers/gpu/drm/i915/gt/intel_gsc.c
@@ -286,7 +286,7 @@ static void gsc_irq_handler(struct intel_gt *gt, unsigned int intf_id)
 
 	ret = generic_handle_irq_safe(gt->gsc.intf[intf_id].irq);
 	if (ret)
-		gt_err_ratelimited(gt, "error handling GSC irq: %d\n", ret);
+		gt_err_ratelimited(gt, "GSC: irq handling failed (%pe)\n", ERR_PTR(ret));
 }
 
 void intel_gsc_irq_handler(struct intel_gt *gt, u32 iir)
diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c
index b2cf672564dd9..6053f3e8aa27c 100644
--- a/drivers/gpu/drm/i915/gt/intel_reset.c
+++ b/drivers/gpu/drm/i915/gt/intel_reset.c
@@ -797,7 +797,7 @@ static int __intel_gt_reset(struct intel_gt *gt, intel_engine_mask_t engine_mask
 bool intel_has_gpu_reset(const struct intel_gt *gt)
 {
 	if (!gt->i915->params.reset)
-		return NULL;
+		return false;
 
 	return intel_get_gpu_reset(gt);
 }
diff --git a/drivers/gpu/drm/i915/gt/selftest_gt_pm.c b/drivers/gpu/drm/i915/gt/selftest_gt_pm.c
index 33351deeea4f0..07eaf71955c44 100644
--- a/drivers/gpu/drm/i915/gt/selftest_gt_pm.c
+++ b/drivers/gpu/drm/i915/gt/selftest_gt_pm.c
@@ -16,9 +16,9 @@ static int cmp_u64(const void *A, const void *B)
 {
 	const u64 *a = A, *b = B;
 
-	if (a < b)
+	if (*a < *b)
 		return -1;
-	else if (a > b)
+	else if (*a > *b)
 		return 1;
 	else
 		return 0;
@@ -28,9 +28,9 @@ static int cmp_u32(const void *A, const void *B)
 {
 	const u32 *a = A, *b = B;
 
-	if (a < b)
+	if (*a < *b)
 		return -1;
-	else if (a > b)
+	else if (*a > *b)
 		return 1;
 	else
 		return 0;
diff --git a/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c b/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c
index 28e8a092f4e7d..18077801b2e60 100644
--- a/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c
+++ b/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c
@@ -44,7 +44,7 @@ multi_lrc_create_parent(struct intel_gt *gt, u8 class,
 	}
 
 	if (i <= 1)
-		return ERR_PTR(0);
+		return NULL;
 
 	logical_sort(siblings, i);
 
diff --git a/drivers/gpu/drm/i915/gvt/edid.c b/drivers/gpu/drm/i915/gvt/edid.c
index ca5b54466a65c..dc9ef98ff51b4 100644
--- a/drivers/gpu/drm/i915/gvt/edid.c
+++ b/drivers/gpu/drm/i915/gvt/edid.c
@@ -90,13 +90,13 @@ static inline int cnp_get_port_from_gmbus0(u32 gmbus0)
 	int port_select = gmbus0 & _GMBUS_PIN_SEL_MASK;
 	int port = -EINVAL;
 
-	if (port_select == GMBUS_PIN_1_BXT)
+	if (port_select == GMBUS_PIN_1)
 		port = PORT_B;
-	else if (port_select == GMBUS_PIN_2_BXT)
+	else if (port_select == GMBUS_PIN_2)
 		port = PORT_C;
-	else if (port_select == GMBUS_PIN_3_BXT)
+	else if (port_select == GMBUS_PIN_3)
 		port = PORT_D;
-	else if (port_select == GMBUS_PIN_4_CNP)
+	else if (port_select == GMBUS_PIN_4)
 		port = PORT_E;
 	return port;
 }
@@ -106,11 +106,11 @@ static inline int bxt_get_port_from_gmbus0(u32 gmbus0)
 	int port_select = gmbus0 & _GMBUS_PIN_SEL_MASK;
 	int port = -EINVAL;
 
-	if (port_select == GMBUS_PIN_1_BXT)
+	if (port_select == GMBUS_PIN_1)
 		port = PORT_B;
-	else if (port_select == GMBUS_PIN_2_BXT)
+	else if (port_select == GMBUS_PIN_2)
 		port = PORT_C;
-	else if (port_select == GMBUS_PIN_3_BXT)
+	else if (port_select == GMBUS_PIN_3)
 		port = PORT_D;
 	return port;
 }
diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c
index aa77def0bc0d2..b2442f05825ee 100644
--- a/drivers/gpu/drm/i915/i915_active.c
+++ b/drivers/gpu/drm/i915/i915_active.c
@@ -548,7 +548,7 @@ static void enable_signaling(struct i915_active_fence *active)
 	if (!fence)
 		return;
 
-	dma_fence_enable_sw_signaling(fence);
+	dma_fence_enable_signaling(fence);
 	dma_fence_put(fence);
 }
 
diff --git a/drivers/gpu/drm/i915/i915_driver.c b/drivers/gpu/drm/i915/i915_driver.c
index 58081b52461ac..ce6d20958320c 100644
--- a/drivers/gpu/drm/i915/i915_driver.c
+++ b/drivers/gpu/drm/i915/i915_driver.c
@@ -51,7 +51,6 @@
 #include <drm/intel/intel_pcode_regs.h>
 
 #include "display/i9xx_display_sr.h"
-#include "display/intel_bw.h"
 #include "display/intel_cdclk.h"
 #include "display/intel_crtc.h"
 #include "display/intel_display_device.h"
@@ -60,8 +59,6 @@
 #include "display/intel_dmc.h"
 #include "display/intel_dp.h"
 #include "display/intel_dpt.h"
-#include "display/intel_dram.h"
-#include "display/intel_encoder.h"
 #include "display/intel_fbdev.h"
 #include "display/intel_gmbus.h"
 #include "display/intel_hotplug.h"
@@ -79,6 +76,7 @@
 #include "gem/i915_gem_ioctls.h"
 #include "gem/i915_gem_mman.h"
 #include "gem/i915_gem_object_frontbuffer.h"
+#include "gem/i915_gem_panic.h"
 #include "gem/i915_gem_pm.h"
 #include "gt/intel_gt.h"
 #include "gt/intel_gt_pm.h"
@@ -111,7 +109,6 @@
 #include "i915_irq.h"
 #include "i915_memcpy.h"
 #include "i915_overlay.h"
-#include "i915_panic.h"
 #include "i915_perf.h"
 #include "i915_query.h"
 #include "i915_reg.h"
@@ -470,7 +467,6 @@ static int i915_pcode_init(struct drm_i915_private *i915)
  */
 static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
 {
-	struct intel_display *display = dev_priv->display;
 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 	int ret;
 
@@ -564,28 +560,14 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
 			drm_dbg(&dev_priv->drm, "can't enable MSI");
 	}
 
-	intel_opregion_setup(display);
-
 	ret = i915_pcode_init(dev_priv);
 	if (ret)
-		goto err_opregion;
-
-	/*
-	 * Fill the dram structure to get the system dram info. This will be
-	 * used for memory latency calculation.
-	 */
-	ret = intel_dram_detect(display);
-	if (ret)
-		goto err_opregion;
-
-	intel_bw_init_hw(display);
+		goto err_msi;
 
 	return 0;
 
-err_opregion:
-	intel_opregion_cleanup(display);
-	if (pdev->msi_enabled)
-		pci_disable_msi(pdev);
+err_msi:
+	pci_disable_msi(pdev);
 err_mem_regions:
 	intel_memory_regions_driver_release(dev_priv);
 err_ggtt:
@@ -604,15 +586,11 @@ ALLOW_ERROR_INJECTION(i915_driver_hw_probe, ERRNO);
  */
 static void i915_driver_hw_remove(struct drm_i915_private *dev_priv)
 {
-	struct intel_display *display = dev_priv->display;
 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 
 	i915_perf_fini(dev_priv);
 
-	intel_opregion_cleanup(display);
-
-	if (pdev->msi_enabled)
-		pci_disable_msi(pdev);
+	pci_disable_msi(pdev);
 }
 
 /**
@@ -660,7 +638,7 @@ static int i915_driver_register(struct drm_i915_private *dev_priv)
 
 	intel_display_driver_register(display);
 
-	intel_display_power_enable(display);
+	intel_display_driver_runtime_pm_enable(display);
 	intel_runtime_pm_enable(&dev_priv->runtime_pm);
 
 	if (i915_switcheroo_register(dev_priv))
@@ -682,7 +660,7 @@ static void i915_driver_unregister(struct drm_i915_private *dev_priv)
 	i915_switcheroo_unregister(dev_priv);
 
 	intel_runtime_pm_disable(&dev_priv->runtime_pm);
-	intel_display_power_disable(display);
+	intel_display_driver_runtime_pm_disable(display);
 
 	intel_display_driver_unregister(display);
 
@@ -803,7 +781,6 @@ i915_driver_create(struct pci_dev *pdev, const struct pci_device_id *ent)
 	const struct intel_device_info *match_info =
 		(struct intel_device_info *)ent->driver_data;
 	struct drm_i915_private *i915;
-	struct intel_display *display;
 
 	i915 = devm_drm_dev_alloc(&pdev->dev, &i915_drm_driver,
 				  struct drm_i915_private, drm);
@@ -818,12 +795,6 @@ i915_driver_create(struct pci_dev *pdev, const struct pci_device_id *ent)
 	/* Set up device info and initial runtime info. */
 	intel_device_info_driver_create(i915, pdev->device, match_info);
 
-	display = intel_display_device_probe(pdev, &parent);
-	if (IS_ERR(display))
-		return ERR_CAST(display);
-
-	i915->display = display;
-
 	return i915;
 }
 
@@ -856,7 +827,13 @@ int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		return PTR_ERR(i915);
 	}
 
-	display = i915->display;
+	display = intel_display_device_probe(pdev, &parent);
+	if (IS_ERR(display)) {
+		ret = PTR_ERR(display);
+		goto out_pci_disable;
+	}
+
+	i915->display = display;
 
 	ret = i915_driver_early_probe(i915);
 	if (ret < 0)
@@ -971,7 +948,6 @@ void i915_driver_remove(struct drm_i915_private *i915)
 	intel_display_driver_remove(display);
 
 	intel_irq_uninstall(i915);
-	intel_hpd_cancel_work(display);
 
 	intel_display_driver_remove_noirq(display);
 
@@ -1043,43 +1019,24 @@ void i915_driver_shutdown(struct drm_i915_private *i915)
 
 	disable_rpm_wakeref_asserts(&i915->runtime_pm);
 	intel_runtime_pm_disable(&i915->runtime_pm);
-	intel_display_power_disable(display);
 
-	drm_client_dev_suspend(&i915->drm);
-	if (intel_display_device_present(display)) {
-		drm_kms_helper_poll_disable(&i915->drm);
-		intel_display_driver_disable_user_access(display);
-
-		drm_atomic_helper_shutdown(&i915->drm);
-	}
-
-	intel_dp_mst_suspend(display);
+	intel_display_driver_shutdown(display);
 
 	intel_irq_suspend(i915);
-	intel_hpd_cancel_work(display);
-
-	if (intel_display_device_present(display))
-		intel_display_driver_suspend_access(display);
-
-	intel_encoder_suspend_all(display);
-	intel_encoder_shutdown_all(display);
 
 	intel_dmc_suspend(display);
 
 	i915_gem_suspend(i915);
 
 	/*
-	 * The only requirement is to reboot with display DC states disabled,
-	 * for now leaving all display power wells in the INIT power domain
-	 * enabled.
-	 *
 	 * TODO:
 	 * - unify the pci_driver::shutdown sequence here with the
 	 *   pci_driver.driver.pm.poweroff,poweroff_late sequence.
 	 * - unify the driver remove and system/runtime suspend sequences with
 	 *   the above unified shutdown/poweroff sequence.
 	 */
-	intel_display_power_driver_remove(display);
+
+	intel_display_driver_shutdown_late(display);
 	enable_rpm_wakeref_asserts(&i915->runtime_pm);
 
 	intel_runtime_pm_driver_last_release(&i915->runtime_pm);
@@ -1108,10 +1065,10 @@ static int i915_drm_prepare(struct drm_device *dev)
 	intel_pxp_suspend_prepare(i915->pxp);
 
 	/*
-	 * NB intel_display_driver_suspend() may issue new requests after we've
-	 * ostensibly marked the GPU as ready-to-sleep here. We need to
-	 * split out that work and pull it forward so that after point,
-	 * the GPU is not woken again.
+	 * NB intel_display_driver_pm_suspend() may issue new requests after
+	 * we've ostensibly marked the GPU as ready-to-sleep here. We need to
+	 * split out that work and pull it forward so that after point, the GPU
+	 * is not woken again.
 	 */
 	return i915_gem_backup_suspend(i915);
 }
@@ -1124,24 +1081,9 @@ static int i915_drm_suspend(struct drm_device *dev)
 
 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
 
-	/* We do a lot of poking in a lot of registers, make sure they work
-	 * properly. */
-	intel_display_power_disable(display);
-	drm_client_dev_suspend(dev);
-	if (intel_display_device_present(display)) {
-		drm_kms_helper_poll_disable(dev);
-		intel_display_driver_disable_user_access(display);
-	}
-
-	intel_display_driver_suspend(display);
+	intel_display_driver_pm_suspend(display);
 
 	intel_irq_suspend(dev_priv);
-	intel_hpd_cancel_work(display);
-
-	if (intel_display_device_present(display))
-		intel_display_driver_suspend_access(display);
-
-	intel_encoder_suspend_all(display);
 
 	/* Must be called before GGTT is suspended. */
 	intel_dpt_suspend(display);
@@ -1181,12 +1123,12 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
 	for_each_gt(gt, dev_priv, i)
 		intel_uncore_suspend(gt->uncore);
 
-	intel_display_power_suspend_late(display, s2idle);
+	intel_display_driver_pm_suspend_late(display, s2idle);
 
 	ret = vlv_suspend_complete(dev_priv);
 	if (ret) {
 		drm_err(&dev_priv->drm, "Suspend complete failed: %d\n", ret);
-		intel_display_power_resume_early(display);
+		intel_display_driver_pm_resume_early(display);
 	}
 
 	enable_rpm_wakeref_asserts(rpm);
@@ -1309,24 +1251,7 @@ static int i915_drm_resume(struct drm_device *dev)
 
 	intel_clock_gating_init(&dev_priv->drm);
 
-	if (intel_display_device_present(display))
-		intel_display_driver_resume_access(display);
-
-	intel_hpd_init(display);
-
-	intel_display_driver_resume(display);
-
-	if (intel_display_device_present(display)) {
-		intel_display_driver_enable_user_access(display);
-		drm_kms_helper_poll_enable(dev);
-	}
-	intel_hpd_poll_disable(display);
-
-	intel_opregion_resume(display);
-
-	drm_client_dev_resume(dev);
-
-	intel_display_power_enable(display);
+	intel_display_driver_pm_resume(display);
 
 	intel_gvt_resume(dev_priv);
 
@@ -1362,7 +1287,7 @@ static int i915_drm_resume_early(struct drm_device *dev)
 	for_each_gt(gt, dev_priv, i)
 		intel_gt_resume_early(gt);
 
-	intel_display_power_resume_early(display);
+	intel_display_driver_pm_resume_early(display);
 
 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
 
@@ -1586,24 +1511,32 @@ static int i915_pm_runtime_suspend(struct device *kdev)
 	for_each_gt(gt, dev_priv, i)
 		intel_gt_runtime_suspend(gt);
 
+	intel_display_driver_pm_runtime_suspend(display);
+
 	intel_irq_suspend(dev_priv);
 
 	for_each_gt(gt, dev_priv, i)
 		intel_uncore_suspend(gt->uncore);
 
-	intel_display_power_runtime_suspend(display);
+	intel_display_driver_pm_runtime_suspend_late(display);
 
 	ret = vlv_suspend_complete(dev_priv);
 	if (ret) {
 		drm_err(&dev_priv->drm,
 			"Runtime suspend failed, disabling it (%d)\n", ret);
-		intel_uncore_runtime_resume(&dev_priv->uncore);
+
+		intel_display_driver_pm_runtime_resume_early(display);
+
+		for_each_gt(gt, dev_priv, i)
+			intel_uncore_runtime_resume(gt->uncore);
 
 		intel_irq_resume(dev_priv);
 
 		for_each_gt(gt, dev_priv, i)
 			intel_gt_runtime_resume(gt);
 
+		intel_display_driver_pm_runtime_resume(display);
+
 		enable_rpm_wakeref_asserts(rpm);
 
 		return ret;
@@ -1625,34 +1558,8 @@ static int i915_pm_runtime_suspend(struct device *kdev)
 	if (root_pdev)
 		pci_d3cold_disable(root_pdev);
 
-	/*
-	 * FIXME: We really should find a document that references the arguments
-	 * used below!
-	 */
-	if (IS_BROADWELL(dev_priv)) {
-		/*
-		 * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop
-		 * being detected, and the call we do at i915_pm_runtime_resume()
-		 * won't be able to restore them. Since PCI_D3hot matches the
-		 * actual specification and appears to be working, use it.
-		 */
-		intel_opregion_notify_adapter(display, PCI_D3hot);
-	} else {
-		/*
-		 * current versions of firmware which depend on this opregion
-		 * notification have repurposed the D1 definition to mean
-		 * "runtime suspended" vs. what you would normally expect (D3)
-		 * to distinguish it from notifications that might be sent via
-		 * the suspend path.
-		 */
-		intel_opregion_notify_adapter(display, PCI_D1);
-	}
-
 	assert_forcewakes_inactive(&dev_priv->uncore);
 
-	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
-		intel_hpd_poll_enable(display);
-
 	drm_dbg(&dev_priv->drm, "Device suspended\n");
 	return 0;
 }
@@ -1675,8 +1582,6 @@ static int i915_pm_runtime_resume(struct device *kdev)
 	drm_WARN_ON_ONCE(&dev_priv->drm, atomic_read(&rpm->wakeref_count));
 	disable_rpm_wakeref_asserts(rpm);
 
-	intel_opregion_notify_adapter(display, PCI_D0);
-
 	root_pdev = pcie_find_root_port(pdev);
 	if (root_pdev)
 		pci_d3cold_enable(root_pdev);
@@ -1685,7 +1590,7 @@ static int i915_pm_runtime_resume(struct device *kdev)
 		drm_dbg(&dev_priv->drm,
 			"Unclaimed access during suspend, bios?\n");
 
-	intel_display_power_runtime_resume(display);
+	intel_display_driver_pm_runtime_resume_early(display);
 
 	ret = vlv_resume_prepare(dev_priv, true);
 
@@ -1703,17 +1608,7 @@ static int i915_pm_runtime_resume(struct device *kdev)
 
 	intel_pxp_runtime_resume(dev_priv->pxp);
 
-	/*
-	 * On VLV/CHV display interrupts are part of the display
-	 * power well, so hpd is reinitialized from there. For
-	 * everyone else do it here.
-	 */
-	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
-		intel_hpd_init(display);
-		intel_hpd_poll_disable(display);
-	}
-
-	skl_watermark_ipc_update(display);
+	intel_display_driver_pm_runtime_resume(display);
 
 	enable_rpm_wakeref_asserts(rpm);
 
diff --git a/drivers/gpu/drm/i915/i915_initial_plane.c b/drivers/gpu/drm/i915/i915_initial_plane.c
index 4902c07486640..98858a78e46a3 100644
--- a/drivers/gpu/drm/i915/i915_initial_plane.c
+++ b/drivers/gpu/drm/i915/i915_initial_plane.c
@@ -58,23 +58,23 @@ initial_plane_phys(struct drm_i915_private *i915,
 
 	if (intel_memory_type_is_local(mem->type) != is_local) {
 		drm_err(&i915->drm, "Initial plane FB PTE unsuitable for %s\n",
-			mem->region.name);
+			mem->name);
 		return false;
 	}
 
 	if (dma_addr < mem->region.start || dma_addr > mem->region.end) {
 		drm_err(&i915->drm,
 			"Initial plane programming using invalid range, dma_addr=%pa (%s [%pa-%pa])\n",
-			&dma_addr, mem->region.name, &mem->region.start, &mem->region.end);
+			&dma_addr, mem->name, &mem->region.start, &mem->region.end);
 		return false;
 	}
 
-	drm_dbg(&i915->drm, "Using dma_addr=%pa, based on initial plane programming\n",
-		&dma_addr);
-
 	*out_phys_base = dma_addr - mem->region.start;
 	*out_mem = mem;
 
+	drm_dbg_kms(&i915->drm, "Initial plane dma_addr=%pa phys_base=%pa mem=%s\n",
+		    &dma_addr, out_phys_base, mem->name);
+
 	return true;
 }
 
@@ -120,7 +120,7 @@ initial_plane_vma(struct drm_i915_private *i915,
 					       I915_BO_PREALLOC);
 	if (IS_ERR(obj)) {
 		drm_dbg_kms(&i915->drm, "Failed to preallocate initial FB in %s\n",
-			    mem->region.name);
+			    mem->name);
 		return NULL;
 	}
 
diff --git a/drivers/gpu/drm/i915/i915_memcpy.c b/drivers/gpu/drm/i915/i915_memcpy.c
index cc41974cee746..20433621e09ad 100644
--- a/drivers/gpu/drm/i915/i915_memcpy.c
+++ b/drivers/gpu/drm/i915/i915_memcpy.c
@@ -166,7 +166,7 @@ void i915_memcpy_init_early(struct drm_i915_private *dev_priv)
 	 * Some hypervisors (e.g. KVM) don't support VEX-prefix instructions
 	 * emulation. So don't enable movntdqa in hypervisor guest.
 	 */
-	if (static_cpu_has(X86_FEATURE_XMM4_1) &&
+	if (cpu_feature_enabled(X86_FEATURE_XMM4_1) &&
 	    !boot_cpu_has(X86_FEATURE_HYPERVISOR))
 		static_branch_enable(&has_movntdqa);
 }
diff --git a/drivers/gpu/drm/i915/i915_panic.c b/drivers/gpu/drm/i915/i915_panic.c
deleted file mode 100644
index 728be077e8e8c..0000000000000
--- a/drivers/gpu/drm/i915/i915_panic.c
+++ /dev/null
@@ -1,35 +0,0 @@
-// SPDX-License-Identifier: MIT
-/* Copyright © 2025 Intel Corporation */
-
-#include <drm/drm_panic.h>
-#include <drm/intel/display_parent_interface.h>
-
-#include "display/intel_display_types.h"
-#include "display/intel_fb.h"
-#include "gem/i915_gem_object.h"
-
-#include "i915_panic.h"
-
-static struct intel_panic *intel_panic_alloc(void)
-{
-	return i915_gem_object_alloc_panic();
-}
-
-static int intel_panic_setup(struct intel_panic *panic, struct drm_scanout_buffer *sb)
-{
-	struct intel_framebuffer *fb = sb->private;
-	struct drm_gem_object *obj = intel_fb_bo(&fb->base);
-
-	return i915_gem_object_panic_setup(panic, sb, obj, fb->panic_tiling);
-}
-
-static void intel_panic_finish(struct intel_panic *panic)
-{
-	return i915_gem_object_panic_finish(panic);
-}
-
-const struct intel_display_panic_interface i915_display_panic_interface = {
-	.alloc = intel_panic_alloc,
-	.setup = intel_panic_setup,
-	.finish = intel_panic_finish,
-};
diff --git a/drivers/gpu/drm/i915/i915_panic.h b/drivers/gpu/drm/i915/i915_panic.h
deleted file mode 100644
index 743d8c861c429..0000000000000
--- a/drivers/gpu/drm/i915/i915_panic.h
+++ /dev/null
@@ -1,9 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/* Copyright © 2025 Intel Corporation */
-
-#ifndef __I915_PANIC_H__
-#define __I915_PANIC_H__
-
-extern const struct intel_display_panic_interface i915_display_panic_interface;
-
-#endif /* __I915_PANIC_H__ */
diff --git a/drivers/gpu/drm/imagination/pvr_drv.c b/drivers/gpu/drm/imagination/pvr_drv.c
index 58087e8517d60..8897b3bdeb4a4 100644
--- a/drivers/gpu/drm/imagination/pvr_drv.c
+++ b/drivers/gpu/drm/imagination/pvr_drv.c
@@ -1378,7 +1378,7 @@ pvr_drm_driver_postclose(__always_unused struct drm_device *drm_dev,
 DEFINE_DRM_GEM_FOPS(pvr_drm_driver_fops);
 
 static struct drm_driver pvr_drm_driver = {
-	.driver_features = DRIVER_GEM | DRIVER_GEM_GPUVA | DRIVER_RENDER |
+	.driver_features = DRIVER_GEM | DRIVER_RENDER |
 			   DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE,
 	.open = pvr_drm_driver_open,
 	.postclose = pvr_drm_driver_postclose,
diff --git a/drivers/gpu/drm/imx/dc/dc-ic.c b/drivers/gpu/drm/imx/dc/dc-ic.c
index a270ae4030cdc..fa87bf1f45087 100644
--- a/drivers/gpu/drm/imx/dc/dc-ic.c
+++ b/drivers/gpu/drm/imx/dc/dc-ic.c
@@ -14,6 +14,8 @@
 #include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 
+#include "dc-drv.h"
+
 #define USERINTERRUPTMASK(n)	(0x8 + 4 * (n))
 #define INTERRUPTENABLE(n)	(0x10 + 4 * (n))
 #define INTERRUPTPRESET(n)	(0x18 + 4 * (n))
@@ -174,8 +176,8 @@ static int dc_ic_probe(struct platform_device *pdev)
 		regmap_write(data->regs, USERINTERRUPTMASK(i), 0xffffffff);
 	}
 
-	data->domain = irq_domain_add_linear(dev->of_node, IRQ_COUNT,
-					     &irq_generic_chip_ops, data);
+	data->domain = irq_domain_create_linear(of_fwnode_handle(dev->of_node), IRQ_COUNT,
+						&irq_generic_chip_ops, data);
 	if (!data->domain) {
 		dev_err(dev, "failed to create IRQ domain\n");
 		pm_runtime_put(dev);
diff --git a/drivers/gpu/drm/imx/ipuv3/parallel-display.c b/drivers/gpu/drm/imx/ipuv3/parallel-display.c
index 0f06db95f00f1..eb75827394f84 100644
--- a/drivers/gpu/drm/imx/ipuv3/parallel-display.c
+++ b/drivers/gpu/drm/imx/ipuv3/parallel-display.c
@@ -180,7 +180,7 @@ static int imx_pd_bridge_attach(struct drm_bridge *bridge,
 
 static const struct drm_bridge_funcs imx_pd_bridge_funcs = {
 	.attach = imx_pd_bridge_attach,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
 	.atomic_check = imx_pd_bridge_atomic_check,
diff --git a/drivers/gpu/drm/ingenic/ingenic-drm-drv.c b/drivers/gpu/drm/ingenic/ingenic-drm-drv.c
index 42c86f195c66b..7e569af223916 100644
--- a/drivers/gpu/drm/ingenic/ingenic-drm-drv.c
+++ b/drivers/gpu/drm/ingenic/ingenic-drm-drv.c
@@ -1031,7 +1031,7 @@ static const struct drm_bridge_funcs ingenic_drm_bridge_funcs = {
 	.atomic_enable		= ingenic_drm_bridge_atomic_enable,
 	.atomic_disable		= ingenic_drm_bridge_atomic_disable,
 	.atomic_check		= ingenic_drm_bridge_atomic_check,
-	.atomic_reset		= drm_atomic_helper_bridge_reset,
+	.atomic_create_state		= drm_atomic_helper_bridge_create_state,
 	.atomic_duplicate_state	= drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_bridge_destroy_state,
 	.atomic_get_input_bus_fmts = ingenic_drm_bridge_atomic_get_input_bus_fmts,
diff --git a/drivers/gpu/drm/lima/lima_device.c b/drivers/gpu/drm/lima/lima_device.c
index 0bf7105c8748b..7c873e62c16da 100644
--- a/drivers/gpu/drm/lima/lima_device.c
+++ b/drivers/gpu/drm/lima/lima_device.c
@@ -368,12 +368,6 @@ int lima_device_init(struct lima_device *ldev)
 	if (err)
 		goto err_out0;
 
-	ldev->empty_vm = lima_vm_create(ldev);
-	if (!ldev->empty_vm) {
-		err = -ENOMEM;
-		goto err_out1;
-	}
-
 	ldev->va_start = 0;
 	if (ldev->id == lima_gpu_mali450) {
 		ldev->va_end = LIMA_VA_RESERVE_START;
@@ -387,6 +381,12 @@ int lima_device_init(struct lima_device *ldev)
 	} else
 		ldev->va_end = LIMA_VA_RESERVE_END;
 
+	ldev->empty_vm = lima_vm_create(ldev);
+	if (!ldev->empty_vm) {
+		err = -ENOMEM;
+		goto err_out1;
+	}
+
 	ldev->iomem = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(ldev->iomem)) {
 		dev_err(ldev->dev, "fail to ioremap iomem\n");
diff --git a/drivers/gpu/drm/loongson/lsdc_drv.c b/drivers/gpu/drm/loongson/lsdc_drv.c
index 34405073c4d48..3fd828d23271c 100644
--- a/drivers/gpu/drm/loongson/lsdc_drv.c
+++ b/drivers/gpu/drm/loongson/lsdc_drv.c
@@ -444,8 +444,8 @@ static const struct dev_pm_ops lsdc_pm_ops = {
 };
 
 static const struct pci_device_id lsdc_pciid_list[] = {
-	{PCI_VDEVICE(LOONGSON, 0x7a06), CHIP_LS7A1000},
-	{PCI_VDEVICE(LOONGSON, 0x7a36), CHIP_LS7A2000},
+	{ PCI_VDEVICE(LOONGSON, 0x7a06), .driver_data = CHIP_LS7A1000 },
+	{ PCI_VDEVICE(LOONGSON, 0x7a36), .driver_data = CHIP_LS7A2000 },
 	{ }
 };
 
diff --git a/drivers/gpu/drm/mcde/mcde_dsi.c b/drivers/gpu/drm/mcde/mcde_dsi.c
index 47d45897ed069..5cf44ccb02cf9 100644
--- a/drivers/gpu/drm/mcde/mcde_dsi.c
+++ b/drivers/gpu/drm/mcde/mcde_dsi.c
@@ -1063,6 +1063,9 @@ static int mcde_dsi_bridge_attach(struct drm_bridge *bridge,
 }
 
 static const struct drm_bridge_funcs mcde_dsi_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach = mcde_dsi_bridge_attach,
 	.mode_set = mcde_dsi_bridge_mode_set,
 };
diff --git a/drivers/gpu/drm/mediatek/mtk_dp.c b/drivers/gpu/drm/mediatek/mtk_dp.c
index eefbc7e0f9c8d..f656b85b84212 100644
--- a/drivers/gpu/drm/mediatek/mtk_dp.c
+++ b/drivers/gpu/drm/mediatek/mtk_dp.c
@@ -2578,7 +2578,7 @@ static const struct drm_bridge_funcs mtk_dp_bridge_funcs = {
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
 	.atomic_get_output_bus_fmts = mtk_dp_bridge_atomic_get_output_bus_fmts,
 	.atomic_get_input_bus_fmts = mtk_dp_bridge_atomic_get_input_bus_fmts,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.attach = mtk_dp_bridge_attach,
 	.detach = mtk_dp_bridge_detach,
 	.atomic_enable = mtk_dp_bridge_atomic_enable,
diff --git a/drivers/gpu/drm/mediatek/mtk_dpi.c b/drivers/gpu/drm/mediatek/mtk_dpi.c
index bb4b7e6f1e298..0e4f430de9832 100644
--- a/drivers/gpu/drm/mediatek/mtk_dpi.c
+++ b/drivers/gpu/drm/mediatek/mtk_dpi.c
@@ -850,7 +850,8 @@ static void mtk_dpi_bridge_mode_set(struct drm_bridge *bridge,
 	drm_mode_copy(&dpi->mode, adjusted_mode);
 }
 
-static void mtk_dpi_bridge_disable(struct drm_bridge *bridge)
+static void mtk_dpi_bridge_disable(struct drm_bridge *bridge,
+				   struct drm_atomic_commit *commit)
 {
 	struct mtk_dpi *dpi = bridge_to_dpi(bridge);
 
@@ -860,7 +861,8 @@ static void mtk_dpi_bridge_disable(struct drm_bridge *bridge)
 		pinctrl_select_state(dpi->pinctrl, dpi->pins_gpio);
 }
 
-static void mtk_dpi_bridge_enable(struct drm_bridge *bridge)
+static void mtk_dpi_bridge_enable(struct drm_bridge *bridge,
+				  struct drm_atomic_commit *commit)
 {
 	struct mtk_dpi *dpi = bridge_to_dpi(bridge);
 
@@ -982,14 +984,14 @@ static const struct drm_bridge_funcs mtk_dpi_bridge_funcs = {
 	.attach = mtk_dpi_bridge_attach,
 	.mode_set = mtk_dpi_bridge_mode_set,
 	.mode_valid = mtk_dpi_bridge_mode_valid,
-	.disable = mtk_dpi_bridge_disable,
-	.enable = mtk_dpi_bridge_enable,
+	.atomic_disable = mtk_dpi_bridge_disable,
+	.atomic_enable = mtk_dpi_bridge_enable,
 	.atomic_check = mtk_dpi_bridge_atomic_check,
 	.atomic_get_output_bus_fmts = mtk_dpi_bridge_atomic_get_output_bus_fmts,
 	.atomic_get_input_bus_fmts = mtk_dpi_bridge_atomic_get_input_bus_fmts,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.debugfs_init = mtk_dpi_debugfs_init,
 };
 
diff --git a/drivers/gpu/drm/mediatek/mtk_dsi.c b/drivers/gpu/drm/mediatek/mtk_dsi.c
index eb325e68aa592..3f3f56eed3f92 100644
--- a/drivers/gpu/drm/mediatek/mtk_dsi.c
+++ b/drivers/gpu/drm/mediatek/mtk_dsi.c
@@ -894,7 +894,7 @@ static const struct drm_bridge_funcs mtk_dsi_bridge_funcs = {
 	.atomic_enable = mtk_dsi_bridge_atomic_enable,
 	.atomic_pre_enable = mtk_dsi_bridge_atomic_pre_enable,
 	.atomic_post_disable = mtk_dsi_bridge_atomic_post_disable,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.mode_valid = mtk_dsi_bridge_mode_valid,
 	.mode_set = mtk_dsi_bridge_mode_set,
 };
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi.c b/drivers/gpu/drm/mediatek/mtk_hdmi.c
index 38a7c7953874a..738090a47176a 100644
--- a/drivers/gpu/drm/mediatek/mtk_hdmi.c
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi.c
@@ -1072,7 +1072,7 @@ static const struct drm_bridge_funcs mtk_hdmi_bridge_funcs = {
 	.mode_valid = mtk_hdmi_bridge_mode_valid,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.attach = mtk_hdmi_bridge_attach,
 	.mode_fixup = mtk_hdmi_bridge_mode_fixup,
 	.atomic_disable = mtk_hdmi_bridge_atomic_disable,
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi_v2.c b/drivers/gpu/drm/mediatek/mtk_hdmi_v2.c
index 7bbf463056c95..d9316d30fd5f7 100644
--- a/drivers/gpu/drm/mediatek/mtk_hdmi_v2.c
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi_v2.c
@@ -1326,7 +1326,7 @@ static const struct drm_bridge_funcs mtk_v2_hdmi_bridge_funcs = {
 	.atomic_post_disable = mtk_hdmi_v2_bridge_post_disable,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.detect = mtk_hdmi_v2_bridge_detect,
 	.edid_read = mtk_hdmi_v2_bridge_edid_read,
 	.hpd_enable = mtk_hdmi_v2_hpd_enable,
diff --git a/drivers/gpu/drm/meson/meson_encoder_cvbs.c b/drivers/gpu/drm/meson/meson_encoder_cvbs.c
index 8b26a0031cde0..22cacb1660c4f 100644
--- a/drivers/gpu/drm/meson/meson_encoder_cvbs.c
+++ b/drivers/gpu/drm/meson/meson_encoder_cvbs.c
@@ -215,7 +215,7 @@ static const struct drm_bridge_funcs meson_encoder_cvbs_bridge_funcs = {
 	.atomic_check = meson_encoder_cvbs_atomic_check,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 };
 
 int meson_encoder_cvbs_probe(struct meson_drm *priv)
diff --git a/drivers/gpu/drm/meson/meson_encoder_dsi.c b/drivers/gpu/drm/meson/meson_encoder_dsi.c
index c1f4685073bbc..3e422b612f74b 100644
--- a/drivers/gpu/drm/meson/meson_encoder_dsi.c
+++ b/drivers/gpu/drm/meson/meson_encoder_dsi.c
@@ -96,7 +96,7 @@ static const struct drm_bridge_funcs meson_encoder_dsi_bridge_funcs = {
 	.atomic_disable	= meson_encoder_dsi_atomic_disable,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 };
 
 int meson_encoder_dsi_probe(struct meson_drm *priv)
diff --git a/drivers/gpu/drm/meson/meson_encoder_hdmi.c b/drivers/gpu/drm/meson/meson_encoder_hdmi.c
index 55c0601df3c62..0c7a72cb514ae 100644
--- a/drivers/gpu/drm/meson/meson_encoder_hdmi.c
+++ b/drivers/gpu/drm/meson/meson_encoder_hdmi.c
@@ -366,7 +366,7 @@ static const struct drm_bridge_funcs meson_encoder_hdmi_bridge_funcs = {
 	.atomic_check = meson_encoder_hdmi_atomic_check,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 };
 
 int meson_encoder_hdmi_probe(struct meson_drm *priv)
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.c b/drivers/gpu/drm/mgag200/mgag200_drv.c
index d7b5dfdb28aa0..dfae5812284dd 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.c
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.c
@@ -204,18 +204,18 @@ int mgag200_device_init(struct mga_device *mdev,
  */
 
 static const struct pci_device_id mgag200_pciidlist[] = {
-	{ PCI_VENDOR_ID_MATROX, 0x520, PCI_ANY_ID, PCI_ANY_ID, 0, 0, G200_PCI },
-	{ PCI_VENDOR_ID_MATROX, 0x521, PCI_ANY_ID, PCI_ANY_ID, 0, 0, G200_AGP },
-	{ PCI_VENDOR_ID_MATROX, 0x522, PCI_ANY_ID, PCI_ANY_ID, 0, 0, G200_SE_A },
-	{ PCI_VENDOR_ID_MATROX, 0x524, PCI_ANY_ID, PCI_ANY_ID, 0, 0, G200_SE_B },
-	{ PCI_VENDOR_ID_MATROX, 0x530, PCI_ANY_ID, PCI_ANY_ID, 0, 0, G200_EV },
-	{ PCI_VENDOR_ID_MATROX, 0x532, PCI_ANY_ID, PCI_ANY_ID, 0, 0, G200_WB },
-	{ PCI_VENDOR_ID_MATROX, 0x533, PCI_ANY_ID, PCI_ANY_ID, 0, 0, G200_EH },
-	{ PCI_VENDOR_ID_MATROX, 0x534, PCI_ANY_ID, PCI_ANY_ID, 0, 0, G200_ER },
-	{ PCI_VENDOR_ID_MATROX, 0x536, PCI_ANY_ID, PCI_ANY_ID, 0, 0, G200_EW3 },
-	{ PCI_VENDOR_ID_MATROX, 0x538, PCI_ANY_ID, PCI_ANY_ID, 0, 0, G200_EH3 },
-	{ PCI_VENDOR_ID_MATROX, 0x53a, PCI_ANY_ID, PCI_ANY_ID, 0, 0, G200_EH5 },
-	{0,}
+	{ PCI_VDEVICE(MATROX, 0x0520), .driver_data = G200_PCI },
+	{ PCI_VDEVICE(MATROX, 0x0521), .driver_data = G200_AGP },
+	{ PCI_VDEVICE(MATROX, 0x0522), .driver_data = G200_SE_A },
+	{ PCI_VDEVICE(MATROX, 0x0524), .driver_data = G200_SE_B },
+	{ PCI_VDEVICE(MATROX, 0x0530), .driver_data = G200_EV },
+	{ PCI_VDEVICE(MATROX, 0x0532), .driver_data = G200_WB },
+	{ PCI_VDEVICE(MATROX, 0x0533), .driver_data = G200_EH },
+	{ PCI_VDEVICE(MATROX, 0x0534), .driver_data = G200_ER },
+	{ PCI_VDEVICE(MATROX, 0x0536), .driver_data = G200_EW3 },
+	{ PCI_VDEVICE(MATROX, 0x0538), .driver_data = G200_EH3 },
+	{ PCI_VDEVICE(MATROX, 0x053a), .driver_data = G200_EH5 },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, mgag200_pciidlist);
diff --git a/drivers/gpu/drm/msm/Kconfig b/drivers/gpu/drm/msm/Kconfig
index 250246f81ea94..09469d56513b0 100644
--- a/drivers/gpu/drm/msm/Kconfig
+++ b/drivers/gpu/drm/msm/Kconfig
@@ -21,6 +21,7 @@ config DRM_MSM
 	select SHMEM
 	select TMPFS
 	select QCOM_SCM
+	select QCOM_PAS
 	select QCOM_UBWC_CONFIG
 	select WANT_DEV_COREDUMP
 	select SND_SOC_HDMI_CODEC if SND_SOC
diff --git a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
index c17e9777beae1..0259e60b80115 100644
--- a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
@@ -25,6 +25,8 @@
 
 extern bool hang_debug;
 
+#define A3XX_VBIF_XIN_HALT_CTRL0_MASK	GENMASK(5, 0)
+
 static void a3xx_dump(struct msm_gpu *gpu);
 static bool a3xx_idle(struct msm_gpu *gpu);
 
@@ -496,6 +498,38 @@ static u64 a3xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
 	return busy_cycles;
 }
 
+static int a3xx_vbif_halt(struct msm_gpu *gpu)
+{
+	u32 ack;
+	int ret;
+
+	gpu_write(gpu, REG_A3XX_VBIF_XIN_HALT_CTRL0,
+		  A3XX_VBIF_XIN_HALT_CTRL0_MASK);
+	ret = spin_until(((ack = gpu_read(gpu, REG_A3XX_VBIF_XIN_HALT_CTRL1)) &
+			  A3XX_VBIF_XIN_HALT_CTRL0_MASK) ==
+			 A3XX_VBIF_XIN_HALT_CTRL0_MASK);
+	gpu_write(gpu, REG_A3XX_VBIF_XIN_HALT_CTRL0, 0);
+
+	if (ret)
+		return -EBUSY;
+
+	return 0;
+}
+
+static int a3xx_pm_suspend(struct msm_gpu *gpu)
+{
+	int ret;
+
+	if (!a3xx_idle(gpu))
+		return -EBUSY;
+
+	ret = a3xx_vbif_halt(gpu);
+	if (ret)
+		return ret;
+
+	return msm_gpu_pm_suspend(gpu);
+}
+
 static u32 a3xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
 {
 	ring->memptrs->rptr = gpu_read(gpu, REG_AXXX_CP_RB_RPTR);
@@ -581,7 +615,7 @@ const struct adreno_gpu_funcs a3xx_gpu_funcs = {
 		.get_param = adreno_get_param,
 		.set_param = adreno_set_param,
 		.hw_init = a3xx_hw_init,
-		.pm_suspend = msm_gpu_pm_suspend,
+		.pm_suspend = a3xx_pm_suspend,
 		.pm_resume = msm_gpu_pm_resume,
 		.recover = a3xx_recover,
 		.submit = a3xx_submit,
diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
index 2c0bbac43c52b..5d96b8f631564 100644
--- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
@@ -5,7 +5,7 @@
 #include <linux/kernel.h>
 #include <linux/types.h>
 #include <linux/cpumask.h>
-#include <linux/firmware/qcom/qcom_scm.h>
+#include <linux/firmware/qcom/qcom_pas.h>
 #include <linux/pm_opp.h>
 #include <linux/nvmem-consumer.h>
 #include <linux/slab.h>
@@ -653,7 +653,7 @@ static int a5xx_zap_shader_resume(struct msm_gpu *gpu)
 	if (adreno_is_a506(adreno_gpu))
 		return 0;
 
-	ret = qcom_scm_set_remote_state(SCM_GPU_ZAP_SHADER_RESUME, GPU_PAS_ID);
+	ret = qcom_pas_set_remote_state(SCM_GPU_ZAP_SHADER_RESUME, GPU_PAS_ID);
 	if (ret)
 		DRM_ERROR("%s: zap-shader resume failed: %d\n",
 			gpu->name, ret);
@@ -1275,6 +1275,11 @@ static irqreturn_t a5xx_irq(struct msm_gpu *gpu)
 		status & ~A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR);
 
 	if (priv->disable_err_irq) {
+		/* Turn off interrupts to avoid interrupt storm */
+		gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK,
+			       A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS |
+			       A5XX_RBBM_INT_0_MASK_CP_SW);
+
 		status &= A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS |
 			  A5XX_RBBM_INT_0_MASK_CP_SW;
 	}
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
index 2e5d7b53a0c38..a2f6918c4f7f2 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
@@ -642,7 +642,7 @@ static void a6xx_rpmh_stop(struct a6xx_gmu *gmu)
 	int ret;
 	u32 val;
 
-	if (test_and_clear_bit(GMU_STATUS_FW_START, &gmu->status))
+	if (!test_and_clear_bit(GMU_STATUS_FW_START, &gmu->status))
 		return;
 
 	if (adreno_is_a840(adreno_gpu))
@@ -1465,6 +1465,9 @@ static void a6xx_gmu_shutdown(struct a6xx_gmu *gmu)
 	/* Stop the interrupts and mask the hardware */
 	a6xx_gmu_irq_disable(gmu);
 
+	/* Halt the gmu cm3 core */
+	gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1);
+
 	/* Tell RPMh to power off the GPU */
 	a6xx_rpmh_stop(gmu);
 
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
index 8b3bb2fd433ba..e293b4ca808a4 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
@@ -1911,8 +1911,11 @@ static irqreturn_t a6xx_irq(struct msm_gpu *gpu)
 
 	gpu_write(gpu, REG_A6XX_RBBM_INT_CLEAR_CMD, status);
 
-	if (priv->disable_err_irq)
+	if (priv->disable_err_irq) {
+		/* Turn off interrupts to avoid interrupt storm */
+		gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK, A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS);
 		status &= A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS;
+	}
 
 	if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT)
 		a6xx_fault_detect_irq(gpu);
@@ -2770,8 +2773,9 @@ static struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
 
 	adreno_gpu->ubwc_config = qcom_ubwc_config_get_data();
 	if (IS_ERR(adreno_gpu->ubwc_config)) {
+		ret = PTR_ERR(adreno_gpu->ubwc_config);
 		a6xx_destroy(&(a6xx_gpu->base.base));
-		return ERR_CAST(adreno_gpu->ubwc_config);
+		return ERR_PTR(ret);
 	}
 
 	/* Set up the preemption specific bits and pieces for each ringbuffer */
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c
index 166365359fa6d..3ea8ff8c74044 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c
@@ -1244,9 +1244,12 @@ static void a6xx_get_gmu_registers(struct msm_gpu *gpu,
 	_a6xx_get_gmu_registers(gpu, a6xx_state, &a6xx_gmu_reglist[1],
 		&a6xx_state->gmu_registers[1], true);
 
-	if (adreno_is_a621(adreno_gpu) || adreno_is_a623(adreno_gpu))
+	if (adreno_is_a621(adreno_gpu))
 		_a6xx_get_gmu_registers(gpu, a6xx_state, &a621_gpucc_reg,
 			&a6xx_state->gmu_registers[2], false);
+	else if (adreno_is_a623(adreno_gpu) || adreno_is_a663(adreno_gpu))
+		_a6xx_get_gmu_registers(gpu, a6xx_state, &a623_gpucc_reg,
+			&a6xx_state->gmu_registers[2], false);
 	else
 		_a6xx_get_gmu_registers(gpu, a6xx_state, &a6xx_gpucc_reg,
 			&a6xx_state->gmu_registers[2], false);
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h
index b49d8427b59e6..0a13a65f89ac8 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h
@@ -377,6 +377,17 @@ static const u32 a6xx_gmu_gpucc_registers[] = {
 };
 
 static const u32 a621_gmu_gpucc_registers[] = {
+	/* GPU CC */
+	0x24000, 0x2400e, 0x24400, 0x2440e, 0x24800, 0x24805, 0x24c00, 0x24cff,
+	0x25800, 0x25804, 0x25c00, 0x25c04, 0x26000, 0x26004, 0x26400, 0x26405,
+	0x26414, 0x2641d, 0x2642a, 0x26430, 0x26432, 0x26432, 0x26441, 0x26455,
+	0x26466, 0x26468, 0x26478, 0x2647a, 0x26489, 0x2648a, 0x2649c, 0x2649e,
+	0x264a0, 0x264a3, 0x264b3, 0x264b5, 0x264c5, 0x264c7, 0x264d6, 0x264d8,
+	0x264e8, 0x264e9, 0x264f9, 0x264fc, 0x2650b, 0x2650c, 0x2651c, 0x2651e,
+	0x26540, 0x26570, 0x26600, 0x26616, 0x26620, 0x2662d,
+};
+
+static const u32 a623_gmu_gpucc_registers[] = {
 	/* GPU CC */
 	0x24000, 0x2400e, 0x24400, 0x2440e, 0x25800, 0x25804, 0x25c00, 0x25c04,
 	0x26000, 0x26004, 0x26400, 0x26405, 0x26414, 0x2641d, 0x2642a, 0x26430,
@@ -402,6 +413,7 @@ static const struct a6xx_registers a6xx_gmu_reglist[] = {
 
 static const struct a6xx_registers a6xx_gpucc_reg = REGS(a6xx_gmu_gpucc_registers, 0, 0);
 static const struct a6xx_registers a621_gpucc_reg = REGS(a621_gmu_gpucc_registers, 0, 0);
+static const struct a6xx_registers a623_gpucc_reg = REGS(a623_gmu_gpucc_registers, 0, 0);
 
 static u32 a6xx_get_cp_roq_size(struct msm_gpu *gpu);
 static u32 a7xx_get_cp_roq_size(struct msm_gpu *gpu);
diff --git a/drivers/gpu/drm/msm/adreno/a8xx_gpu.c b/drivers/gpu/drm/msm/adreno/a8xx_gpu.c
index 9e44fd1ae6346..0f6fd35bd5878 100644
--- a/drivers/gpu/drm/msm/adreno/a8xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a8xx_gpu.c
@@ -1211,8 +1211,11 @@ irqreturn_t a8xx_irq(struct msm_gpu *gpu)
 
 	gpu_write(gpu, REG_A8XX_RBBM_INT_CLEAR_CMD, status);
 
-	if (priv->disable_err_irq)
+	if (priv->disable_err_irq) {
+		/* Turn off interrupts to avoid interrupt storm */
+		gpu_write(gpu, REG_A8XX_RBBM_INT_0_MASK, A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS);
 		status &= A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS;
+	}
 
 	if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT)
 		a8xx_fault_detect_irq(gpu);
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
index c62c45bb0ddbb..489462065ea95 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
@@ -8,6 +8,7 @@
 
 #include <linux/ascii85.h>
 #include <linux/interconnect.h>
+#include <linux/firmware/qcom/qcom_pas.h>
 #include <linux/firmware/qcom/qcom_scm.h>
 #include <linux/kernel.h>
 #include <linux/of_reserved_mem.h>
@@ -146,10 +147,10 @@ static int zap_shader_load_mdt(struct msm_gpu *gpu, const char *fwname,
 		goto out;
 
 	/* Send the image to the secure world */
-	ret = qcom_scm_pas_auth_and_reset(pasid);
+	ret = qcom_pas_auth_and_reset(pasid);
 
 	/*
-	 * If the scm call returns -EOPNOTSUPP we assume that this target
+	 * If the pas call returns -EOPNOTSUPP we assume that this target
 	 * doesn't need/support the zap shader so quietly fail
 	 */
 	if (ret == -EOPNOTSUPP)
@@ -175,9 +176,9 @@ int adreno_zap_shader_load(struct msm_gpu *gpu, u32 pasid)
 	if (!zap_available)
 		return -ENODEV;
 
-	/* We need SCM to be able to load the firmware */
-	if (!qcom_scm_is_available()) {
-		DRM_DEV_ERROR(&pdev->dev, "SCM is not available\n");
+	/* We need PAS to be able to load the firmware */
+	if (!qcom_pas_is_available()) {
+		DRM_DEV_ERROR(&pdev->dev, "PAS is not available\n");
 		return -EPROBE_DEFER;
 	}
 
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c
index 13cc658065c56..2ff255d7795e5 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c
@@ -34,7 +34,7 @@ enum dpu_perf_mode {
 /**
  * dpu_core_perf_adjusted_mode_clk - Adjust given mode clock rate according to
  *   the perf clock factor.
- * @crtc_clk_rate - Unadjusted mode clock rate
+ * @mode_clk_rate: unadjusted mode clock rate
  * @perf_cfg: performance configuration
  */
 u64 dpu_core_perf_adjusted_mode_clk(u64 mode_clk_rate,
diff --git a/drivers/gpu/drm/msm/dp/dp_drm.c b/drivers/gpu/drm/msm/dp/dp_drm.c
index b659d22f5f284..06881bfd61812 100644
--- a/drivers/gpu/drm/msm/dp/dp_drm.c
+++ b/drivers/gpu/drm/msm/dp/dp_drm.c
@@ -52,7 +52,7 @@ static void msm_dp_bridge_debugfs_init(struct drm_bridge *bridge, struct dentry
 static const struct drm_bridge_funcs msm_dp_bridge_ops = {
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state   = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset           = drm_atomic_helper_bridge_reset,
+	.atomic_create_state           = drm_atomic_helper_bridge_create_state,
 	.atomic_enable          = msm_dp_bridge_atomic_enable,
 	.atomic_disable         = msm_dp_bridge_atomic_disable,
 	.atomic_post_disable    = msm_dp_bridge_atomic_post_disable,
@@ -235,7 +235,7 @@ static const struct drm_bridge_funcs msm_edp_bridge_ops = {
 	.atomic_post_disable = msm_edp_bridge_atomic_post_disable,
 	.mode_set = msm_dp_bridge_mode_set,
 	.mode_valid = msm_edp_bridge_mode_valid,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
 	.atomic_check = msm_edp_bridge_atomic_check,
diff --git a/drivers/gpu/drm/msm/dp/dp_panel.c b/drivers/gpu/drm/msm/dp/dp_panel.c
index 6bb021820d7c5..5b4954e7cb748 100644
--- a/drivers/gpu/drm/msm/dp/dp_panel.c
+++ b/drivers/gpu/drm/msm/dp/dp_panel.c
@@ -332,8 +332,10 @@ int msm_dp_panel_get_modes(struct msm_dp_panel *msm_dp_panel,
 		return -EINVAL;
 	}
 
-	if (msm_dp_panel->drm_edid)
+	if (msm_dp_panel->drm_edid) {
+		drm_edid_connector_update(connector, msm_dp_panel->drm_edid);
 		return drm_edid_connector_add_modes(connector);
+	}
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c
index 982abaaac00d5..59125d342fcb6 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_host.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_host.c
@@ -166,6 +166,7 @@ struct msm_dsi_host {
 
 	struct drm_display_mode *mode;
 	struct drm_dsc_config *dsc;
+	unsigned int dsc_slice_per_pkt;
 
 	/* connected device info */
 	unsigned int channel;
@@ -670,12 +671,24 @@ static void dsi_calc_pclk(struct msm_dsi_host *msm_host, bool is_bonded_dsi)
 
 int dsi_calc_clk_rate_6g(struct msm_dsi_host *msm_host, bool is_bonded_dsi)
 {
+	long rounded_byte_clk_rate;
+
 	if (!msm_host->mode) {
 		pr_err("%s: mode not set\n", __func__);
 		return -EINVAL;
 	}
 
 	dsi_calc_pclk(msm_host, is_bonded_dsi);
+
+	rounded_byte_clk_rate = clk_round_rate(msm_host->byte_clk,
+					       msm_host->byte_clk_rate);
+	if (rounded_byte_clk_rate < 0) {
+		pr_err("%s: failed to round byte clock rate, %ld\n",
+		       __func__, rounded_byte_clk_rate);
+		return rounded_byte_clk_rate;
+	}
+
+	msm_host->byte_clk_rate = rounded_byte_clk_rate;
 	msm_host->esc_clk_rate = clk_get_rate(msm_host->esc_clk);
 	return 0;
 }
@@ -938,17 +951,10 @@ static void dsi_update_dsc_timing(struct msm_dsi_host *msm_host, bool is_cmd_mod
 	slice_per_intf = dsc->slice_count;
 
 	total_bytes_per_intf = dsc->slice_chunk_size * slice_per_intf;
-	bytes_per_pkt = dsc->slice_chunk_size; /* * slice_per_pkt; */
+	bytes_per_pkt = dsc->slice_chunk_size * msm_host->dsc_slice_per_pkt;
 
 	eol_byte_num = total_bytes_per_intf % 3;
-
-	/*
-	 * Typically, pkt_per_line = slice_per_intf * slice_per_pkt.
-	 *
-	 * Since the current driver only supports slice_per_pkt = 1,
-	 * pkt_per_line will be equal to slice per intf for now.
-	 */
-	pkt_per_line = slice_per_intf;
+	pkt_per_line = slice_per_intf / msm_host->dsc_slice_per_pkt;
 
 	if (is_cmd_mode) /* packet data type */
 		reg = DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_DATATYPE(MIPI_DSI_DCS_LONG_WRITE);
@@ -1104,12 +1110,8 @@ static void dsi_timing_setup(struct msm_dsi_host *msm_host, bool is_bonded_dsi)
 		else
 			/*
 			 * When DSC is enabled, WC = slice_chunk_size * slice_per_pkt + 1.
-			 * Currently, the driver only supports default value of slice_per_pkt = 1
-			 *
-			 * TODO: Expand mipi_dsi_device struct to hold slice_per_pkt info
-			 *       and adjust DSC math to account for slice_per_pkt.
 			 */
-			wc = msm_host->dsc->slice_chunk_size + 1;
+			wc = msm_host->dsc->slice_chunk_size * msm_host->dsc_slice_per_pkt + 1;
 
 		dsi_write(msm_host, REG_DSI_CMD_MDP_STREAM0_CTRL,
 			DSI_CMD_MDP_STREAM0_CTRL_WORD_COUNT(wc) |
@@ -1718,8 +1720,13 @@ static int dsi_host_attach(struct mipi_dsi_host *host,
 	msm_host->lanes = dsi->lanes;
 	msm_host->format = dsi->format;
 	msm_host->mode_flags = dsi->mode_flags;
-	if (dsi->dsc)
+	if (dsi->dsc) {
 		msm_host->dsc = dsi->dsc;
+		if (dsi->mode_flags & MIPI_DSI_MODE_DSC_ALL_SLICES_IN_PKT)
+			msm_host->dsc_slice_per_pkt = dsi->dsc->slice_count;
+		else
+			msm_host->dsc_slice_per_pkt = 1;
+	}
 
 	if (msm_host->format == MIPI_DSI_FMT_RGB101010) {
 		if (!msm_dsi_host_version_geq(msm_host, MSM_DSI_VER_MAJOR_6G,
diff --git a/drivers/gpu/drm/msm/dsi/dsi_manager.c b/drivers/gpu/drm/msm/dsi/dsi_manager.c
index 46faeaae9d94a..959daa88b1d02 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_manager.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_manager.c
@@ -275,7 +275,8 @@ static void dsi_mgr_bridge_power_off(struct drm_bridge *bridge)
 	dsi_mgr_phy_disable(id);
 }
 
-static void dsi_mgr_bridge_pre_enable(struct drm_bridge *bridge)
+static void dsi_mgr_bridge_pre_enable(struct drm_bridge *bridge,
+				      struct drm_atomic_commit *commit)
 {
 	int id = dsi_mgr_bridge_get_id(bridge);
 	struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id);
@@ -331,7 +332,8 @@ void msm_dsi_manager_tpg_enable(void)
 	}
 }
 
-static void dsi_mgr_bridge_post_disable(struct drm_bridge *bridge)
+static void dsi_mgr_bridge_post_disable(struct drm_bridge *bridge,
+					struct drm_atomic_commit *commit)
 {
 	int id = dsi_mgr_bridge_get_id(bridge);
 	struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id);
@@ -445,9 +447,12 @@ static int dsi_mgr_bridge_attach(struct drm_bridge *bridge,
 }
 
 static const struct drm_bridge_funcs dsi_mgr_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach = dsi_mgr_bridge_attach,
-	.pre_enable = dsi_mgr_bridge_pre_enable,
-	.post_disable = dsi_mgr_bridge_post_disable,
+	.atomic_pre_enable = dsi_mgr_bridge_pre_enable,
+	.atomic_post_disable = dsi_mgr_bridge_post_disable,
 	.mode_set = dsi_mgr_bridge_mode_set,
 	.mode_valid = dsi_mgr_bridge_mode_valid,
 };
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
index 21a59d66e8dc5..f5d3e806f8fd5 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
@@ -111,6 +111,7 @@ struct msm_dsi_phy {
 	struct msm_dsi_dphy_timing timing;
 	const struct msm_dsi_phy_cfg *cfg;
 	void *tuning_cfg;
+	void *pll_data;
 
 	enum msm_dsi_phy_usecase usecase;
 	bool regulator_ldo_mode;
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c
index 984a66085dfbf..5d805a797abdb 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c
@@ -426,8 +426,11 @@ static void dsi_pll_enable_pll_bias(struct dsi_pll_7nm *pll)
 	u32 data;
 
 	spin_lock_irqsave(&pll->pll_enable_lock, flags);
-	pll->pll_enable_cnt++;
-	WARN_ON(pll->pll_enable_cnt == INT_MAX);
+	if (pll->pll_enable_cnt++) {
+		spin_unlock_irqrestore(&pll->pll_enable_lock, flags);
+		WARN_ON(pll->pll_enable_cnt == INT_MAX);
+		return;
+	}
 
 	data = readl(pll->phy->base + REG_DSI_7nm_PHY_CMN_CTRL_0);
 	data |= DSI_7nm_PHY_CMN_CTRL_0_PLL_SHUTDOWNB;
@@ -873,6 +876,7 @@ static int dsi_pll_7nm_init(struct msm_dsi_phy *phy)
 	spin_lock_init(&pll_7nm->pll_enable_lock);
 
 	pll_7nm->phy = phy;
+	phy->pll_data = pll_7nm;
 
 	ret = pll_7nm_register(pll_7nm, phy->provided_clocks->hws);
 	if (ret) {
@@ -961,8 +965,10 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy,
 	u32 const delay_us = 5;
 	u32 const timeout_us = 1000;
 	struct msm_dsi_dphy_timing *timing = &phy->timing;
+	struct dsi_pll_7nm *pll = phy->pll_data;
 	void __iomem *base = phy->base;
 	bool less_than_1500_mhz;
+	unsigned long flags;
 	u32 vreg_ctrl_0, vreg_ctrl_1, lane_ctrl0;
 	u32 glbl_pemph_ctrl_0;
 	u32 glbl_str_swi_cal_sel_ctrl, glbl_hstx_str_ctrl_0;
@@ -1084,10 +1090,13 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy,
 		glbl_rescode_bot_ctrl = 0x3c;
 	}
 
+	spin_lock_irqsave(&pll->pll_enable_lock, flags);
+	pll->pll_enable_cnt = 1;
 	/* de-assert digital and pll power down */
 	data = DSI_7nm_PHY_CMN_CTRL_0_DIGTOP_PWRDN_B |
 	       DSI_7nm_PHY_CMN_CTRL_0_PLL_SHUTDOWNB;
 	writel(data, base + REG_DSI_7nm_PHY_CMN_CTRL_0);
+	spin_unlock_irqrestore(&pll->pll_enable_lock, flags);
 
 	/* Assert PLL core reset */
 	writel(0x00, base + REG_DSI_7nm_PHY_CMN_PLL_CNTRL);
@@ -1200,7 +1209,9 @@ static bool dsi_7nm_set_continuous_clock(struct msm_dsi_phy *phy, bool enable)
 
 static void dsi_7nm_phy_disable(struct msm_dsi_phy *phy)
 {
+	struct dsi_pll_7nm *pll = phy->pll_data;
 	void __iomem *base = phy->base;
+	unsigned long flags;
 	u32 data;
 
 	DBG("");
@@ -1227,8 +1238,11 @@ static void dsi_7nm_phy_disable(struct msm_dsi_phy *phy)
 	writel(data, base + REG_DSI_7nm_PHY_CMN_CTRL_0);
 	writel(0, base + REG_DSI_7nm_PHY_CMN_LANE_CTRL0);
 
+	spin_lock_irqsave(&pll->pll_enable_lock, flags);
+	pll->pll_enable_cnt = 0;
 	/* Turn off all PHY blocks */
 	writel(0x00, base + REG_DSI_7nm_PHY_CMN_CTRL_0);
+	spin_unlock_irqrestore(&pll->pll_enable_lock, flags);
 
 	/* make sure phy is turned off */
 	wmb();
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c b/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c
index c4d5e0faf3b32..7abb9243dba57 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c
@@ -455,7 +455,7 @@ static enum drm_mode_status msm_hdmi_bridge_tmds_char_rate_valid(const struct dr
 static const struct drm_bridge_funcs msm_hdmi_bridge_funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_pre_enable = msm_hdmi_bridge_atomic_pre_enable,
 	.atomic_post_disable = msm_hdmi_bridge_atomic_post_disable,
 	.edid_read = msm_hdmi_bridge_edid_read,
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index 32d5ebea2596f..ac86b427c0e53 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -824,7 +824,6 @@ static const struct file_operations fops = {
 
 #define DRIVER_FEATURES_GPU ( \
 		DRIVER_GEM | \
-		DRIVER_GEM_GPUVA | \
 		DRIVER_RENDER | \
 		DRIVER_SYNCOBJ | \
 		DRIVER_SYNCOBJ_TIMELINE | \
@@ -832,7 +831,6 @@ static const struct file_operations fops = {
 
 #define DRIVER_FEATURES_KMS ( \
 		DRIVER_GEM | \
-		DRIVER_GEM_GPUVA | \
 		DRIVER_ATOMIC | \
 		DRIVER_MODESET | \
 		0 )
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index efd3d3c9a4490..6a78e242de7c6 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -9,6 +9,7 @@
 #include <linux/spinlock.h>
 #include <linux/shmem_fs.h>
 #include <linux/dma-buf.h>
+#include <linux/pagemap.h>
 
 #include <drm/drm_dumb_buffers.h>
 #include <drm/drm_prime.h>
@@ -360,7 +361,7 @@ static vm_fault_t msm_gem_fault(struct vm_fault *vmf)
 	}
 
 	/* We don't use vmf->pgoff since that has the fake offset: */
-	pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
+	pgoff = linear_page_delta(vma, vmf->address);
 
 	pfn = page_to_pfn(pages[pgoff]);
 
@@ -1124,7 +1125,7 @@ static int msm_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 
 	vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
-	vma->vm_page_prot = msm_gem_pgprot(msm_obj, vm_get_page_prot(vma->vm_flags));
+	vma->vm_page_prot = msm_gem_pgprot(msm_obj, vma_get_page_prot(vma));
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index 18ed00e5f143b..03c0578560658 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -505,6 +505,8 @@ static void recover_worker(struct kthread_work *work)
 		 */
 		if (!vm->managed)
 			msm_gem_vm_unusable(submit->vm);
+
+		put_task_struct(task);
 	}
 
 	noreclaim_flag = memalloc_noreclaim_save();
@@ -552,11 +554,13 @@ static void recover_worker(struct kthread_work *work)
 		msm_update_fence(ring->fctx, fence);
 	}
 
-	/* retire completed submits, plus the one that hung: */
-	retire_submits(gpu);
+	priv->disable_err_irq = false;
 
 	gpu->funcs->recover(gpu);
 
+	/* retire completed submits, plus the one that hung: */
+	retire_submits(gpu);
+
 	/*
 	 * Replay all remaining submits starting with highest priority
 	 * ring
diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c
index 0d14c31bd4e47..af2b0edc4d586 100644
--- a/drivers/gpu/drm/msm/msm_ringbuffer.c
+++ b/drivers/gpu/drm/msm/msm_ringbuffer.c
@@ -108,9 +108,9 @@ struct msm_ringbuffer *msm_ringbuffer_new(struct msm_gpu *gpu, int id,
 	ring->memptrs_iova = memptrs_iova;
 
 	ret = drm_sched_init(&ring->sched, &args);
-	if (ret) {
+	if (ret)
 		goto fail;
-	}
+	ring->sched_initialized = true;
 
 	INIT_LIST_HEAD(&ring->submits);
 	spin_lock_init(&ring->submit_lock);
@@ -132,7 +132,8 @@ void msm_ringbuffer_destroy(struct msm_ringbuffer *ring)
 	if (IS_ERR_OR_NULL(ring))
 		return;
 
-	drm_sched_fini(&ring->sched);
+	if (ring->sched_initialized)
+		drm_sched_fini(&ring->sched);
 
 	msm_fence_context_free(ring->fctx);
 
diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.h b/drivers/gpu/drm/msm/msm_ringbuffer.h
index 28ca8c9f7463d..3631ec283c6e5 100644
--- a/drivers/gpu/drm/msm/msm_ringbuffer.h
+++ b/drivers/gpu/drm/msm/msm_ringbuffer.h
@@ -56,6 +56,7 @@ struct msm_ringbuffer {
 	 * The job scheduler for this ring.
 	 */
 	struct drm_gpu_scheduler sched;
+	bool sched_initialized;
 
 	/*
 	 * List of in-flight submits on this ring.  Protected by submit_lock.
diff --git a/drivers/gpu/drm/msm/registers/adreno/a3xx.xml b/drivers/gpu/drm/msm/registers/adreno/a3xx.xml
index 09c9606fc3e18..21b00a17ef3e9 100644
--- a/drivers/gpu/drm/msm/registers/adreno/a3xx.xml
+++ b/drivers/gpu/drm/msm/registers/adreno/a3xx.xml
@@ -1495,6 +1495,8 @@ xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd">
 	<reg32 offset="0x3058" name="VBIF_OUT_AXI_AMEMTYPE_CONF0"/>
 	<reg32 offset="0x305e" name="VBIF_OUT_AXI_AOOO_EN"/>
 	<reg32 offset="0x305f" name="VBIF_OUT_AXI_AOOO"/>
+	<reg32 offset="0x3080" name="VBIF_XIN_HALT_CTRL0"/>
+	<reg32 offset="0x3081" name="VBIF_XIN_HALT_CTRL1"/>
 
 	<bitset name="a3xx_vbif_perf_cnt" inline="yes">
 		<bitfield name="CNT0" pos="0" type="boolean"/>
diff --git a/drivers/gpu/drm/mxsfb/lcdif_drv.c b/drivers/gpu/drm/mxsfb/lcdif_drv.c
index f5bb59cd50285..e2173c4d6fc28 100644
--- a/drivers/gpu/drm/mxsfb/lcdif_drv.c
+++ b/drivers/gpu/drm/mxsfb/lcdif_drv.c
@@ -186,7 +186,7 @@ static int lcdif_load(struct drm_device *drm)
 
 	ret = lcdif_attach_bridge(lcdif);
 	if (ret)
-		return dev_err_probe(drm->dev, ret, "Cannot connect bridge\n");
+		return ret;
 
 	drm->mode_config.min_width	= LCDIF_MIN_XRES;
 	drm->mode_config.min_height	= LCDIF_MIN_YRES;
diff --git a/drivers/gpu/drm/mxsfb/lcdif_kms.c b/drivers/gpu/drm/mxsfb/lcdif_kms.c
index ade76c3f4e4e9..79e0f483ca20d 100644
--- a/drivers/gpu/drm/mxsfb/lcdif_kms.c
+++ b/drivers/gpu/drm/mxsfb/lcdif_kms.c
@@ -338,7 +338,8 @@ static void lcdif_set_mode(struct lcdif_drm_private *lcdif, u32 bus_flags)
 	 * Downstream set it to 256B burst size to improve the memory
 	 * efficiency so set it here too.
 	 */
-	ctrl = CTRLDESCL0_3_P_SIZE(2) | CTRLDESCL0_3_T_SIZE(2) |
+	ctrl = CTRLDESCL0_3_STATE_CLEAR_VSYNC |
+	       CTRLDESCL0_3_P_SIZE(2) | CTRLDESCL0_3_T_SIZE(2) |
 	       CTRLDESCL0_3_PITCH(lcdif->crtc.primary->state->fb->pitches[0]);
 	writel(ctrl, lcdif->base + LCDC_V8_CTRLDESCL0_3);
 }
@@ -374,14 +375,23 @@ static void lcdif_disable_controller(struct lcdif_drm_private *lcdif)
 	int ret;
 
 	reg = readl(lcdif->base + LCDC_V8_CTRLDESCL0_5);
+	/* Disable the layer for DMA. */
 	reg &= ~CTRLDESCL0_5_EN;
+	/*
+	 * It is necessary to wait for the full frame to finish streaming
+	 * through the DMA engine before we can safely disable it by removing
+	 * the DISP_PARA_DISP_ON bit. Disabling it in-flight can leave the
+	 * hardware confused and unable to resume streaming for the next frame.
+	 */
+	reg |= CTRLDESCL0_5_SHADOW_LOAD_EN;
 	writel(reg, lcdif->base + LCDC_V8_CTRLDESCL0_5);
 
+	/* Wait for the frame to finish or timeout after 50 ms. */
 	ret = readl_poll_timeout(lcdif->base + LCDC_V8_CTRLDESCL0_5,
-				 reg, !(reg & CTRLDESCL0_5_EN),
-				 0, 36000);	/* Wait ~2 frame times max */
+				 reg, !(reg & CTRLDESCL0_5_SHADOW_LOAD_EN),
+				 200, 50000);
 	if (ret)
-		drm_err(lcdif->drm, "Failed to disable controller!\n");
+		drm_err(lcdif->drm, "Timed out waiting for final vblank!\n");
 
 	reg = readl(lcdif->base + LCDC_V8_DISP_PARA);
 	reg &= ~DISP_PARA_DISP_ON;
diff --git a/drivers/gpu/drm/mxsfb/lcdif_regs.h b/drivers/gpu/drm/mxsfb/lcdif_regs.h
index c55dfb236c1d3..17882c593d274 100644
--- a/drivers/gpu/drm/mxsfb/lcdif_regs.h
+++ b/drivers/gpu/drm/mxsfb/lcdif_regs.h
@@ -190,6 +190,7 @@
 #define CTRLDESCL0_1_WIDTH(n)		((n) & 0xffff)
 #define CTRLDESCL0_1_WIDTH_MASK		GENMASK(15, 0)
 
+#define CTRLDESCL0_3_STATE_CLEAR_VSYNC	BIT(23)
 #define CTRLDESCL0_3_P_SIZE(n)		(((n) << 20) & CTRLDESCL0_3_P_SIZE_MASK)
 #define CTRLDESCL0_3_P_SIZE_MASK	GENMASK(22, 20)
 #define CTRLDESCL0_3_T_SIZE(n)		(((n) << 16) & CTRLDESCL0_3_T_SIZE_MASK)
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index e16f59b00f6fa..7ea95ab960a0e 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -153,22 +153,6 @@ nouveau_name(struct drm_device *dev)
 		return nouveau_platform_name(to_platform_device(dev->dev));
 }
 
-static inline bool
-nouveau_cli_work_ready(struct dma_fence *fence)
-{
-	unsigned long flags;
-	bool ret = true;
-
-	dma_fence_lock_irqsave(fence, flags);
-	if (!dma_fence_is_signaled_locked(fence))
-		ret = false;
-	dma_fence_unlock_irqrestore(fence, flags);
-
-	if (ret == true)
-		dma_fence_put(fence);
-	return ret;
-}
-
 static void
 nouveau_cli_work(struct work_struct *w)
 {
@@ -176,9 +160,25 @@ nouveau_cli_work(struct work_struct *w)
 	struct nouveau_cli_work *work, *wtmp;
 	mutex_lock(&cli->lock);
 	list_for_each_entry_safe(work, wtmp, &cli->worker, head) {
-		if (!work->fence || nouveau_cli_work_ready(work->fence)) {
+		struct dma_fence *fence = work->fence;
+
+		if (!fence || dma_fence_is_signaled(fence)) {
+			if (fence) {
+				unsigned long flags;
+
+				/*
+				 * Because fence can still be in the process of
+				 * processing the callback list, and the
+				 * callback references the work we are about to
+				 * free, we need to sync with the callback
+				 * processing before freeing the work.
+				 */
+				dma_fence_lock_irqsave(fence, flags);
+				dma_fence_unlock_irqrestore(fence, flags);
+			}
 			list_del(&work->head);
 			work->func(work);
+			dma_fence_put(fence);
 		}
 	}
 	mutex_unlock(&cli->lock);
@@ -1364,7 +1364,6 @@ static struct drm_driver
 driver_stub = {
 	.driver_features = DRIVER_GEM |
 			   DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE |
-			   DRIVER_GEM_GPUVA |
 			   DRIVER_MODESET |
 			   DRIVER_RENDER,
 	.open = nouveau_drm_open,
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index 20dba02d6175b..0b7123b163e49 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -55,7 +55,7 @@ static vm_fault_t nouveau_ttm_fault(struct vm_fault *vmf)
 		goto error_unlock;
 
 	nouveau_bo_del_io_reserve_lru(bo);
-	prot = vm_get_page_prot(vma->vm_flags);
+	prot = vma_get_page_prot(vma);
 	ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT);
 	nouveau_bo_add_io_reserve_lru(bo);
 	if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
@@ -313,11 +313,20 @@ nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem,
 	rep->offset = nvbo->offset;
 	if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50 &&
 	    !nouveau_cli_uvmm(cli)) {
+		int ret;
+
+		ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
+		if (ret)
+			return ret;
+
 		vma = nouveau_vma_find(nvbo, vmm);
-		if (!vma)
+		if (!vma) {
+			ttm_bo_unreserve(&nvbo->bo);
 			return -EINVAL;
+		}
 
 		rep->offset = vma->addr;
+		ttm_bo_unreserve(&nvbo->bo);
 	} else
 		rep->offset = 0;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_svm.c b/drivers/gpu/drm/nouveau/nouveau_svm.c
index dcc92131488ea..58735446d7832 100644
--- a/drivers/gpu/drm/nouveau/nouveau_svm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_svm.c
@@ -678,20 +678,22 @@ static int nouveau_range_fault(struct nouveau_svmm *svmm,
 	range.end = notifier->notifier.interval_tree.last + 1;
 
 	while (true) {
-		if (time_after(jiffies, timeout)) {
+		long remaining = timeout - jiffies;
+
+		/*
+		 * The HMM timeout only bounds retries while HMM is walking and
+		 * faulting the range. This fault is handled by a kernel worker,
+		 * so fatal signals from the faulting process cannot stop an
+		 * endless stream of invalidations here.
+		 */
+		if (time_after_eq(jiffies, timeout)) {
 			ret = -EBUSY;
 			goto out;
 		}
 
-		range.notifier_seq = mmu_interval_read_begin(range.notifier);
-		mmap_read_lock(mm);
-		ret = hmm_range_fault(&range);
-		mmap_read_unlock(mm);
-		if (ret) {
-			if (ret == -EBUSY)
-				continue;
+		ret = hmm_range_fault_unlocked_timeout(&range, remaining);
+		if (ret)
 			goto out;
-		}
 
 		mutex_lock(&svmm->mutex);
 		if (mmu_interval_read_retry(range.notifier,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c
index dbd984da75014..0608266188d3e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c
@@ -253,7 +253,7 @@ gv100_head_vblank_get(struct nvkm_head *head)
 	nvkm_mask(device, 0x611d80 + (head->id * 4), 0x00000004, 0x00000004);
 }
 
-static void
+void
 gv100_head_rgpos(struct nvkm_head *head, u16 *hline, u16 *vline)
 {
 	struct nvkm_device *device = head->disp->engine.subdev.device;
@@ -263,7 +263,7 @@ gv100_head_rgpos(struct nvkm_head *head, u16 *hline, u16 *vline)
 	*hline = nvkm_rd32(device, 0x616334 + hoff) & 0x0000ffff;
 }
 
-static void
+void
 gv100_head_state(struct nvkm_head *head, struct nvkm_head_state *state)
 {
 	struct nvkm_device *device = head->disp->engine.subdev.device;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h
index 856252bf559a4..b642729c254fe 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h
@@ -53,6 +53,8 @@ void gf119_head_rgclk(struct nvkm_head *, int);
 
 int gv100_head_cnt(struct nvkm_disp *, unsigned long *);
 int gv100_head_new(struct nvkm_disp *, int id);
+void gv100_head_state(struct nvkm_head *head, struct nvkm_head_state *state);
+void gv100_head_rgpos(struct nvkm_head *head, u16 *hline, u16 *vline);
 
 #define HEAD_MSG(h,l,f,a...) do {                                              \
 	struct nvkm_head *_h = (h);                                            \
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c
index 1155f079b0c3d..e77733a5d9c3c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c
@@ -625,14 +625,10 @@ r535_head_vblank_get(struct nvkm_head *head)
 	nvkm_mask(device, 0x611d80 + (head->id * 4), 0x00000002, 0x00000002);
 }
 
-static void
-r535_head_state(struct nvkm_head *head, struct nvkm_head_state *state)
-{
-}
-
 static const struct nvkm_head_func
 r535_head = {
-	.state = r535_head_state,
+	.state = gv100_head_state,
+	.rgpos = gv100_head_rgpos,
 	.vblank_get = r535_head_vblank_get,
 	.vblank_put = r535_head_vblank_put,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/fbsr.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/fbsr.c
index 700cea5def357..f128330f30d7b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/fbsr.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/fbsr.c
@@ -297,6 +297,7 @@ r535_fbsr = {
 static void *
 r535_instmem_dtor(struct nvkm_instmem *imem)
 {
+	nv50_instmem_dtor(imem);
 	kfree(imem->func);
 	return imem;
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c
index 6fbaa1e5876dc..f4489efc94a79 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c
@@ -30,10 +30,14 @@
 #include <subdev/gsp.h>
 #include <subdev/mmu.h>
 
+#include <linux/io-mapping.h>
+
 struct nv50_instmem {
 	struct nvkm_instmem base;
 	u64 addr;
 
+	struct io_mapping iomap;
+
 	/* Mappings that can be evicted when BAR2 space has been exhausted. */
 	struct list_head lru;
 };
@@ -124,7 +128,6 @@ nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm)
 	struct nv50_instobj *eobj;
 	struct nvkm_memory *memory = &iobj->base.memory;
 	struct nvkm_subdev *subdev = &imem->base.subdev;
-	struct nvkm_device *device = subdev->device;
 	struct nvkm_vma *bar = NULL, *ebar;
 	u64 size = nvkm_memory_size(memory);
 	void *emap;
@@ -155,7 +158,7 @@ nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm)
 		mutex_unlock(&imem->base.mutex);
 		if (!eobj)
 			break;
-		iounmap(emap);
+		io_mapping_unmap(emap);
 		nvkm_vmm_put(vmm, &ebar);
 	}
 
@@ -172,8 +175,7 @@ nv50_instobj_kmap(struct nv50_instobj *iobj, struct nvkm_vmm *vmm)
 
 	/* Make the mapping visible to the host. */
 	iobj->bar = bar;
-	iobj->map = ioremap_wc(device->func->resource_addr(device, NVKM_BAR2_INST) +
-			       (u32)iobj->bar->addr, size);
+	iobj->map = io_mapping_map_wc(&imem->iomap, (u32)iobj->bar->addr, size);
 	if (!iobj->map) {
 		nvkm_warn(subdev, "PRAMIN ioremap failed\n");
 		nvkm_vmm_put(vmm, &iobj->bar);
@@ -188,6 +190,16 @@ nv50_instobj_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm,
 	return nvkm_memory_map(memory, offset, vmm, vma, argv, argc);
 }
 
+static bool
+check_io_mapping(struct nv50_instmem *imem)
+{
+	struct nvkm_device *device = imem->base.subdev.device;
+
+	return io_mapping_init_wc(&imem->iomap,
+				  device->func->resource_addr(device, NVKM_BAR2_INST),
+				  device->func->resource_size(device, NVKM_BAR2_INST)) != NULL;
+}
+
 static void
 nv50_instobj_release(struct nvkm_memory *memory)
 {
@@ -239,7 +251,7 @@ nv50_instobj_acquire(struct nvkm_memory *memory)
 
 	/* Attempt to get a direct CPU mapping of the object. */
 	if ((vmm = nvkm_bar_bar2_vmm(imem->subdev.device))) {
-		if (!iobj->map)
+		if (!iobj->map && iobj->imem->iomap.size)
 			nv50_instobj_kmap(iobj, vmm);
 		map = iobj->map;
 	}
@@ -277,7 +289,12 @@ nv50_instobj_boot(struct nvkm_memory *memory, struct nvkm_vmm *vmm)
 		iobj->lru.next = NULL;
 	}
 
-	nv50_instobj_kmap(iobj, vmm);
+	/*
+	 * boot is only called on BAR2, if we can't remap the complete
+	 * BAR it's unlikely things are functioning well.
+	 */
+	if (check_io_mapping(iobj->imem))
+		nv50_instobj_kmap(iobj, vmm);
 	nvkm_instmem_boot(imem);
 	mutex_unlock(&imem->mutex);
 }
@@ -330,7 +347,7 @@ nv50_instobj_dtor(struct nvkm_memory *memory)
 
 	if (map) {
 		struct nvkm_vmm *vmm = nvkm_bar_bar2_vmm(imem->subdev.device);
-		iounmap(map);
+		io_mapping_unmap(map);
 		if (likely(vmm)) /* Can be NULL during BAR destructor. */
 			nvkm_vmm_put(vmm, &bar);
 	}
@@ -406,10 +423,14 @@ nv50_instmem_fini(struct nvkm_instmem *base)
 	nv50_instmem(base)->addr = ~0ULL;
 }
 
-static void *
+void *
 nv50_instmem_dtor(struct nvkm_instmem *base)
 {
-	return nv50_instmem(base);
+	struct nv50_instmem *imem = nv50_instmem(base);
+
+	if (imem->iomap.size)
+		io_mapping_fini(&imem->iomap);
+	return imem;
 }
 
 static const struct nvkm_instmem_func
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h
index 87bbdd786eaaf..b423f01f5fcd1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h
@@ -26,6 +26,7 @@ int nv50_instobj_new(struct nvkm_instmem *, u32 size, u32 align, bool zero,
 		     struct nvkm_memory **);
 int nv50_instobj_wrap(struct nvkm_instmem *, struct nvkm_memory *vram,
 		      struct nvkm_memory **bar2);
+void *nv50_instmem_dtor(struct nvkm_instmem *base);
 
 void nvkm_instmem_ctor(const struct nvkm_instmem_func *, struct nvkm_device *,
 		       enum nvkm_subdev_type, int, struct nvkm_instmem *);
diff --git a/drivers/gpu/drm/nova/Makefile b/drivers/gpu/drm/nova/Makefile
index f8527b2b7b4ab..6355f7502c485 100644
--- a/drivers/gpu/drm/nova/Makefile
+++ b/drivers/gpu/drm/nova/Makefile
@@ -1,4 +1,3 @@
 # SPDX-License-Identifier: GPL-2.0
-
-obj-$(CONFIG_DRM_NOVA) += nova-drm.o
-nova-drm-y := nova.o
+# nova-drm is built from drivers/gpu/Makefile.
+# nova.o (rust-analyzer marker - DO NOT REMOVE).
diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs
index 48933d86ddda9..739690bc2db51 100644
--- a/drivers/gpu/drm/nova/driver.rs
+++ b/drivers/gpu/drm/nova/driver.rs
@@ -2,7 +2,10 @@
 
 use kernel::{
     auxiliary,
-    device::Core,
+    device::{
+        Core,
+        DeviceContext, //
+    },
     drm::{
         self,
         gem,
@@ -17,18 +20,14 @@
 
 pub(crate) struct NovaDriver;
 
-pub(crate) struct Nova {
+pub(crate) struct Nova<'bound> {
     #[expect(unused)]
     drm: ARef<drm::Device<NovaDriver>>,
+    _reg: drm::Registration<'bound, NovaDriver>,
 }
 
 /// Convienence type alias for the DRM device type for this driver
-pub(crate) type NovaDevice<Ctx = drm::Registered> = drm::Device<NovaDriver, Ctx>;
-
-#[pin_data]
-pub(crate) struct NovaData {
-    pub(crate) adev: ARef<auxiliary::Device>,
-}
+pub(crate) type NovaDevice<Ctx = drm::Normal> = drm::Device<NovaDriver, Ctx>;
 
 const INFO: drm::DriverInfo = drm::DriverInfo {
     major: 0,
@@ -53,27 +52,32 @@ pub(crate) struct NovaData {
 
 impl auxiliary::Driver for NovaDriver {
     type IdInfo = ();
-    type Data<'bound> = Nova;
+    type Data<'bound> = Nova<'bound>;
     const ID_TABLE: auxiliary::IdTable<Self::IdInfo> = &AUX_TABLE;
 
     fn probe<'bound>(
         adev: &'bound auxiliary::Device<Core<'_>>,
         _info: &'bound Self::IdInfo,
     ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound {
-        let data = try_pin_init!(NovaData { adev: adev.into() });
-
-        let drm = drm::UnregisteredDevice::<Self>::new(adev.as_ref(), data)?;
-        let drm = drm::Registration::new_foreign_owned(drm, adev.as_ref(), 0)?;
-
-        Ok(Nova { drm: drm.into() })
+        let drm = drm::UnregisteredDevice::<Self>::new(adev, Ok(()))?;
+        // SAFETY: `reg` is stored in `Nova` and dropped when the driver is unbound; it is
+        // never forgotten.
+        let reg = unsafe { drm::Registration::new(adev.as_ref(), drm, (), 0)? };
+
+        Ok(Nova {
+            drm: reg.device().into(),
+            _reg: reg,
+        })
     }
 }
 
 #[vtable]
 impl drm::Driver for NovaDriver {
-    type Data = NovaData;
+    type Data = ();
+    type RegistrationData<'a> = ();
     type File = File;
-    type Object<Ctx: drm::DeviceContext> = gem::Object<NovaObject, Ctx>;
+    type Object = gem::Object<NovaObject>;
+    type ParentDevice<Ctx: DeviceContext> = auxiliary::Device<Ctx>;
 
     const INFO: drm::DriverInfo = INFO;
 
diff --git a/drivers/gpu/drm/nova/file.rs b/drivers/gpu/drm/nova/file.rs
index a3b7bd36792c1..298c02bacb4bc 100644
--- a/drivers/gpu/drm/nova/file.rs
+++ b/drivers/gpu/drm/nova/file.rs
@@ -4,7 +4,13 @@
 use crate::gem::NovaObject;
 use kernel::{
     alloc::flags::*,
-    drm::{self, gem::BaseObject},
+    auxiliary,
+    device::Bound,
+    drm::{
+        self,
+        gem::BaseObject,
+        Registered, //
+    },
     pci,
     prelude::*,
     uapi,
@@ -23,13 +29,13 @@ fn open(_dev: &NovaDevice) -> Result<Pin<KBox<Self>>> {
 impl File {
     /// IOCTL: get_param: Query GPU / driver metadata.
     pub(crate) fn get_param(
-        dev: &NovaDevice,
+        dev: &NovaDevice<Registered>,
+        _reg_data: &(),
         getparam: &mut uapi::drm_nova_getparam,
         _file: &drm::File<File>,
     ) -> Result<u32> {
-        let adev = &dev.adev;
-        let parent = adev.parent();
-        let pdev: &pci::Device = parent.try_into()?;
+        let adev: &auxiliary::Device<Bound> = dev.as_ref();
+        let pdev: &pci::Device<Bound> = adev.parent().try_into()?;
 
         let value = match getparam.param as u32 {
             uapi::NOVA_GETPARAM_VRAM_BAR_SIZE => pdev.resource_len(1)?,
@@ -43,7 +49,8 @@ pub(crate) fn get_param(
 
     /// IOCTL: gem_create: Create a new DRM GEM object.
     pub(crate) fn gem_create(
-        dev: &NovaDevice,
+        dev: &NovaDevice<Registered>,
+        _reg_data: &(),
         req: &mut uapi::drm_nova_gem_create,
         file: &drm::File<File>,
     ) -> Result<u32> {
@@ -56,7 +63,8 @@ pub(crate) fn gem_create(
 
     /// IOCTL: gem_info: Query GEM metadata.
     pub(crate) fn gem_info(
-        _dev: &NovaDevice,
+        _dev: &NovaDevice<Registered>,
+        _reg_data: &(),
         req: &mut uapi::drm_nova_gem_info,
         file: &drm::File<File>,
     ) -> Result<u32> {
diff --git a/drivers/gpu/drm/nova/gem.rs b/drivers/gpu/drm/nova/gem.rs
index 9d8ff7de2c0f7..2b6fe9dc0bfa1 100644
--- a/drivers/gpu/drm/nova/gem.rs
+++ b/drivers/gpu/drm/nova/gem.rs
@@ -2,7 +2,10 @@
 
 use kernel::{
     drm,
-    drm::{gem, gem::BaseObject, DeviceContext},
+    drm::{
+        gem,
+        gem::BaseObject, //
+    },
     page,
     prelude::*,
     sync::aref::ARef,
@@ -21,27 +24,20 @@ impl gem::DriverObject for NovaObject {
     type Driver = NovaDriver;
     type Args = ();
 
-    fn new<Ctx: DeviceContext>(
-        _dev: &NovaDevice<Ctx>,
-        _size: usize,
-        _args: Self::Args,
-    ) -> impl PinInit<Self, Error> {
+    fn new(_dev: &NovaDevice, _size: usize, _args: Self::Args) -> impl PinInit<Self, Error> {
         try_pin_init!(NovaObject {})
     }
 }
 
 impl NovaObject {
     /// Create a new DRM GEM object.
-    pub(crate) fn new<Ctx: DeviceContext>(
-        dev: &NovaDevice<Ctx>,
-        size: usize,
-    ) -> Result<ARef<gem::Object<Self, Ctx>>> {
+    pub(crate) fn new(dev: &NovaDevice, size: usize) -> Result<ARef<gem::Object<Self>>> {
         if size == 0 {
             return Err(EINVAL);
         }
         let aligned_size = page::page_align(size).ok_or(EINVAL)?;
 
-        gem::Object::<Self, Ctx>::new(dev, aligned_size, ())
+        gem::Object::<Self>::new(dev, aligned_size, ())
     }
 
     /// Look up a GEM object handle for a `File` and return an `ObjectRef` for it.
diff --git a/drivers/gpu/drm/omapdrm/dss/dpi.c b/drivers/gpu/drm/omapdrm/dss/dpi.c
index 9f86db774c395..419915e338163 100644
--- a/drivers/gpu/drm/omapdrm/dss/dpi.c
+++ b/drivers/gpu/drm/omapdrm/dss/dpi.c
@@ -22,6 +22,7 @@
 #include <linux/string.h>
 #include <linux/sys_soc.h>
 
+#include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
 
 #include "dss.h"
@@ -482,7 +483,8 @@ static void dpi_bridge_mode_set(struct drm_bridge *bridge,
 	dpi->pixelclock = adjusted_mode->clock * 1000;
 }
 
-static void dpi_bridge_enable(struct drm_bridge *bridge)
+static void dpi_bridge_enable(struct drm_bridge *bridge,
+			      struct drm_atomic_commit *commit)
 {
 	struct dpi_data *dpi = drm_bridge_to_dpi(bridge);
 	int r;
@@ -533,7 +535,8 @@ static void dpi_bridge_enable(struct drm_bridge *bridge)
 		regulator_disable(dpi->vdds_dsi_reg);
 }
 
-static void dpi_bridge_disable(struct drm_bridge *bridge)
+static void dpi_bridge_disable(struct drm_bridge *bridge,
+			       struct drm_atomic_commit *commit)
 {
 	struct dpi_data *dpi = drm_bridge_to_dpi(bridge);
 
@@ -552,12 +555,15 @@ static void dpi_bridge_disable(struct drm_bridge *bridge)
 }
 
 static const struct drm_bridge_funcs dpi_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach = dpi_bridge_attach,
 	.mode_valid = dpi_bridge_mode_valid,
 	.mode_fixup = dpi_bridge_mode_fixup,
 	.mode_set = dpi_bridge_mode_set,
-	.enable = dpi_bridge_enable,
-	.disable = dpi_bridge_disable,
+	.atomic_enable = dpi_bridge_enable,
+	.atomic_disable = dpi_bridge_disable,
 };
 
 static void dpi_bridge_init(struct dpi_data *dpi)
diff --git a/drivers/gpu/drm/omapdrm/dss/dsi.c b/drivers/gpu/drm/omapdrm/dss/dsi.c
index 27fe7bca9e2cf..7aaee94f86e41 100644
--- a/drivers/gpu/drm/omapdrm/dss/dsi.c
+++ b/drivers/gpu/drm/omapdrm/dss/dsi.c
@@ -35,6 +35,7 @@
 #include <linux/component.h>
 #include <linux/sys_soc.h>
 
+#include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_panel.h>
@@ -4654,7 +4655,8 @@ static void dsi_bridge_mode_set(struct drm_bridge *bridge,
 	dsi_set_config(&dsi->output, adjusted_mode);
 }
 
-static void dsi_bridge_enable(struct drm_bridge *bridge)
+static void dsi_bridge_enable(struct drm_bridge *bridge,
+			      struct drm_atomic_commit *commit)
 {
 	struct dsi_data *dsi = drm_bridge_to_dsi(bridge);
 	struct omap_dss_device *dssdev = &dsi->output;
@@ -4673,7 +4675,8 @@ static void dsi_bridge_enable(struct drm_bridge *bridge)
 	dsi_bus_unlock(dsi);
 }
 
-static void dsi_bridge_disable(struct drm_bridge *bridge)
+static void dsi_bridge_disable(struct drm_bridge *bridge,
+			       struct drm_atomic_commit *commit)
 {
 	struct dsi_data *dsi = drm_bridge_to_dsi(bridge);
 	struct omap_dss_device *dssdev = &dsi->output;
@@ -4692,11 +4695,14 @@ static void dsi_bridge_disable(struct drm_bridge *bridge)
 }
 
 static const struct drm_bridge_funcs dsi_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach = dsi_bridge_attach,
 	.mode_valid = dsi_bridge_mode_valid,
 	.mode_set = dsi_bridge_mode_set,
-	.enable = dsi_bridge_enable,
-	.disable = dsi_bridge_disable,
+	.atomic_enable = dsi_bridge_enable,
+	.atomic_disable = dsi_bridge_disable,
 };
 
 static void dsi_bridge_init(struct dsi_data *dsi)
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi4.c b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
index e306247ed8a05..61cfd003569c6 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi4.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
@@ -497,7 +497,7 @@ static const struct drm_bridge_funcs hdmi4_bridge_funcs = {
 	.mode_set = hdmi4_bridge_mode_set,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_enable = hdmi4_bridge_enable,
 	.atomic_disable = hdmi4_bridge_disable,
 	.hpd_notify = hdmi4_bridge_hpd_notify,
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi5.c b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
index ab2d4eb6787f2..1ee3bbe205831 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi5.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
@@ -472,7 +472,7 @@ static const struct drm_bridge_funcs hdmi5_bridge_funcs = {
 	.mode_set = hdmi5_bridge_mode_set,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_enable = hdmi5_bridge_enable,
 	.atomic_disable = hdmi5_bridge_disable,
 	.edid_read = hdmi5_bridge_edid_read,
diff --git a/drivers/gpu/drm/omapdrm/dss/sdi.c b/drivers/gpu/drm/omapdrm/dss/sdi.c
index df4cbc683e2ca..d4a3c0ecaaf25 100644
--- a/drivers/gpu/drm/omapdrm/dss/sdi.c
+++ b/drivers/gpu/drm/omapdrm/dss/sdi.c
@@ -16,6 +16,7 @@
 #include <linux/regulator/consumer.h>
 #include <linux/string.h>
 
+#include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
 
 #include "dss.h"
@@ -197,7 +198,8 @@ static void sdi_bridge_mode_set(struct drm_bridge *bridge,
 	sdi->pixelclock = adjusted_mode->clock * 1000;
 }
 
-static void sdi_bridge_enable(struct drm_bridge *bridge)
+static void sdi_bridge_enable(struct drm_bridge *bridge,
+			      struct drm_atomic_commit *commit)
 {
 	struct sdi_device *sdi = drm_bridge_to_sdi(bridge);
 	struct dispc_clock_info dispc_cinfo;
@@ -260,7 +262,8 @@ static void sdi_bridge_enable(struct drm_bridge *bridge)
 	regulator_disable(sdi->vdds_sdi_reg);
 }
 
-static void sdi_bridge_disable(struct drm_bridge *bridge)
+static void sdi_bridge_disable(struct drm_bridge *bridge,
+			       struct drm_atomic_commit *commit)
 {
 	struct sdi_device *sdi = drm_bridge_to_sdi(bridge);
 
@@ -274,12 +277,15 @@ static void sdi_bridge_disable(struct drm_bridge *bridge)
 }
 
 static const struct drm_bridge_funcs sdi_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach = sdi_bridge_attach,
 	.mode_valid = sdi_bridge_mode_valid,
 	.mode_fixup = sdi_bridge_mode_fixup,
 	.mode_set = sdi_bridge_mode_set,
-	.enable = sdi_bridge_enable,
-	.disable = sdi_bridge_disable,
+	.atomic_enable = sdi_bridge_enable,
+	.atomic_disable = sdi_bridge_disable,
 };
 
 static void sdi_bridge_init(struct sdi_device *sdi)
diff --git a/drivers/gpu/drm/omapdrm/dss/venc.c b/drivers/gpu/drm/omapdrm/dss/venc.c
index 9b5d53dc361e6..5416c668a2055 100644
--- a/drivers/gpu/drm/omapdrm/dss/venc.c
+++ b/drivers/gpu/drm/omapdrm/dss/venc.c
@@ -25,6 +25,7 @@
 #include <linux/component.h>
 #include <linux/sys_soc.h>
 
+#include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
 
 #include "omapdss.h"
@@ -614,14 +615,16 @@ static void venc_bridge_mode_set(struct drm_bridge *bridge,
 	dispc_set_tv_pclk(venc->dss->dispc, 13500000);
 }
 
-static void venc_bridge_enable(struct drm_bridge *bridge)
+static void venc_bridge_enable(struct drm_bridge *bridge,
+			       struct drm_atomic_commit *commit)
 {
 	struct venc_device *venc = drm_bridge_to_venc(bridge);
 
 	venc_power_on(venc);
 }
 
-static void venc_bridge_disable(struct drm_bridge *bridge)
+static void venc_bridge_disable(struct drm_bridge *bridge,
+				struct drm_atomic_commit *commit)
 {
 	struct venc_device *venc = drm_bridge_to_venc(bridge);
 
@@ -653,12 +656,15 @@ static int venc_bridge_get_modes(struct drm_bridge *bridge,
 }
 
 static const struct drm_bridge_funcs venc_bridge_funcs = {
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.attach = venc_bridge_attach,
 	.mode_valid = venc_bridge_mode_valid,
 	.mode_fixup = venc_bridge_mode_fixup,
 	.mode_set = venc_bridge_mode_set,
-	.enable = venc_bridge_enable,
-	.disable = venc_bridge_disable,
+	.atomic_enable = venc_bridge_enable,
+	.atomic_disable = venc_bridge_disable,
 	.get_modes = venc_bridge_get_modes,
 };
 
diff --git a/drivers/gpu/drm/omapdrm/omap_fbdev.c b/drivers/gpu/drm/omapdrm/omap_fbdev.c
index ca3fb186bf195..4881777642d29 100644
--- a/drivers/gpu/drm/omapdrm/omap_fbdev.c
+++ b/drivers/gpu/drm/omapdrm/omap_fbdev.c
@@ -84,7 +84,7 @@ static int omap_fbdev_pan_display(struct fb_var_screeninfo *var, struct fb_info
 
 static int omap_fbdev_fb_mmap(struct fb_info *info, struct vm_area_struct *vma)
 {
-	vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+	vma->vm_page_prot = pgprot_writecombine(vma_get_page_prot(vma));
 
 	return fb_deferred_io_mmap(info, vma);
 }
diff --git a/drivers/gpu/drm/omapdrm/omap_gem.c b/drivers/gpu/drm/omapdrm/omap_gem.c
index 8e013e4f2c6bf..fb0e6f556b314 100644
--- a/drivers/gpu/drm/omapdrm/omap_gem.c
+++ b/drivers/gpu/drm/omapdrm/omap_gem.c
@@ -5,6 +5,7 @@
  */
 
 #include <linux/dma-mapping.h>
+#include <linux/pagemap.h>
 #include <linux/seq_file.h>
 #include <linux/shmem_fs.h>
 #include <linux/spinlock.h>
@@ -359,7 +360,7 @@ static vm_fault_t omap_gem_fault_1d(struct drm_gem_object *obj,
 	pgoff_t pgoff;
 
 	/* We don't use vmf->pgoff since that has the fake offset: */
-	pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
+	pgoff = linear_page_delta(vma, vmf->address);
 
 	if (omap_obj->pages) {
 		omap_gem_cpu_sync_page(obj, pgoff);
@@ -407,7 +408,7 @@ static vm_fault_t omap_gem_fault_2d(struct drm_gem_object *obj,
 	const int m = DIV_ROUND_UP(omap_obj->width << fmt, PAGE_SIZE);
 
 	/* We don't use vmf->pgoff since that has the fake offset: */
-	pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
+	pgoff = linear_page_delta(vma, vmf->address);
 
 	/*
 	 * Actual address we start mapping at is rounded down to previous slot
@@ -537,9 +538,9 @@ static int omap_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struc
 	vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP | VM_IO | VM_MIXEDMAP);
 
 	if (omap_obj->flags & OMAP_BO_WC) {
-		vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+		vma->vm_page_prot = pgprot_writecombine(vma_get_page_prot(vma));
 	} else if (omap_obj->flags & OMAP_BO_UNCACHED) {
-		vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags));
+		vma->vm_page_prot = pgprot_noncached(vma_get_page_prot(vma));
 	} else {
 		/*
 		 * We do have some private objects, at least for scanout buffers
@@ -557,7 +558,7 @@ static int omap_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struc
 		vma->vm_pgoff -= drm_vma_node_start(&obj->vma_node);
 		vma_set_file(vma, obj->filp);
 
-		vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+		vma->vm_page_prot = vma_get_page_prot(vma);
 	}
 
 	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig
index 7450b27622a23..cfbfb371bc67e 100644
--- a/drivers/gpu/drm/panel/Kconfig
+++ b/drivers/gpu/drm/panel/Kconfig
@@ -17,6 +17,16 @@ config DRM_PANEL_ABT_Y030XX067A
 	  Y030XX067A 320x480 3.0" panel as found in the YLM RG-280M, RG-300
 	  and RG-99 handheld gaming consoles.
 
+config DRM_PANEL_ANBERNIC_TD4310
+	tristate "Anbernic TD4310 LCD panel"
+	depends on GPIOLIB && OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Say Y here to enable support for Anbernic designed panels with the
+	  TD4310 panel controller such as the ones used on the Anbernic RG
+	  Vita Pro.
+
 config DRM_PANEL_ARM_VERSATILE
 	tristate "ARM Versatile panel driver"
 	depends on OF
@@ -105,6 +115,17 @@ config DRM_PANEL_BOE_TV101WUM_LL2
 	  Say Y here if you want to support for BOE TV101WUM-LL2
 	  WUXGA PANEL DSI Video Mode panel
 
+config DRM_PANEL_CHIPONE_ICNA35XX
+	tristate "Chipone ICNA35XX panel driver"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	select DRM_DISPLAY_HELPER
+	help
+	  Say Y here if you want to enable support for the panels built
+	  around the Chipone ICNA3512 and ICNA3520 display controllers,
+	  such as some Tianma panels used in AYN Odin2 Portal and Thor.
+
 config DRM_PANEL_CHIPWEALTH_CH13726A
 	tristate "CHIPWEALTH CH13726A-based DSI panel"
 	depends on OF
@@ -264,6 +285,18 @@ config DRM_PANEL_HYDIS_HV101HD1
 
 	  If M is selected the module will be called panel-hydis-hv101hd1
 
+config DRM_PANEL_ILITEK_ILI7807S
+	tristate "Ilitek ILI7807S-based panels"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Say Y if you want to enable support for panels based on the
+	  Ilitek ILI7807S display controller, such as the DLC DLC0697
+	  1080x1920 MIPI DSI panel.
+
+	  If M is selected the module will be called panel-ilitek-ili7807s.
+
 config DRM_PANEL_ILITEK_IL9322
 	tristate "Ilitek ILI9322 320x240 QVGA panels"
 	depends on OF && SPI
@@ -284,6 +317,15 @@ config DRM_PANEL_ILITEK_ILI9341
 	  QVGA (240x320) RGB panels. support serial & parallel rgb
 	  interface.
 
+config DRM_PANEL_ILITEK_ILI9488
+	tristate "Ilitek ILI9488-based panels"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Say Y if you want to enable support for panels based on the
+	  Ilitek ILI9488 controller.
+
 config DRM_PANEL_ILITEK_ILI9805
 	tristate "Ilitek ILI9805-based panels"
 	depends on OF
@@ -796,6 +838,18 @@ config DRM_PANEL_RENESAS_R61307
 	  This panel controller can be found in LG Optimus Vu P895 smartphone
 	  in combination with LCD panel.
 
+config DRM_PANEL_RENESAS_R63419
+	tristate "Renesas R63419 dual-DSI video mode panels"
+	depends on OF && GPIOLIB
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Say Y here if you want to enable support for Ayaneo WT0600 and WT0630
+	  1440x2560 60Hz dual-DSI video mode display panels with Renesas
+	  R63419 IC.
+
+	  These panels are used in Ayaneo handheld gaming devices.
+
 config DRM_PANEL_RENESAS_R69328
 	tristate "Renesas R69328 720x1280 DSI video mode panel"
 	depends on OF
@@ -1322,6 +1376,8 @@ config DRM_PANEL_VISIONOX_VTDR6130
 	depends on OF
 	depends on DRM_MIPI_DSI
 	depends on BACKLIGHT_CLASS_DEVICE
+	select DRM_DISPLAY_DSC_HELPER
+	select DRM_DISPLAY_HELPER
 	help
 	  Say Y here if you want to enable support for Visionox
 	  VTDR6130 1080x2400 AMOLED DSI panel.
diff --git a/drivers/gpu/drm/panel/Makefile b/drivers/gpu/drm/panel/Makefile
index c2c5cf8171163..0f29f22f589ec 100644
--- a/drivers/gpu/drm/panel/Makefile
+++ b/drivers/gpu/drm/panel/Makefile
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_DRM_PANEL_ABT_Y030XX067A) += panel-abt-y030xx067a.o
+obj-$(CONFIG_DRM_PANEL_ANBERNIC_TD4310) += panel-anbernic-td4310.o
 obj-$(CONFIG_DRM_PANEL_ARM_VERSATILE) += panel-arm-versatile.o
 obj-$(CONFIG_DRM_PANEL_ASUS_Z00T_TM5P5_NT35596) += panel-asus-z00t-tm5p5-n35596.o
 obj-$(CONFIG_DRM_PANEL_AUO_A030JTN01) += panel-auo-a030jtn01.o
@@ -9,6 +10,7 @@ obj-$(CONFIG_DRM_PANEL_BOE_TD4320) += panel-boe-td4320.o
 obj-$(CONFIG_DRM_PANEL_BOE_TH101MB31UIG002_28A) += panel-boe-th101mb31ig002-28a.o
 obj-$(CONFIG_DRM_PANEL_BOE_TV101WUM_LL2) += panel-boe-tv101wum-ll2.o
 obj-$(CONFIG_DRM_PANEL_BOE_TV101WUM_NL6) += panel-boe-tv101wum-nl6.o
+obj-$(CONFIG_DRM_PANEL_CHIPONE_ICNA35XX) += panel-chipone-icna35xx.o
 obj-$(CONFIG_DRM_PANEL_CHIPWEALTH_CH13726A) += panel-chipwealth-ch13726a.o
 obj-$(CONFIG_DRM_PANEL_DSI_CM) += panel-dsi-cm.o
 obj-$(CONFIG_DRM_PANEL_LVDS) += panel-lvds.o
@@ -26,8 +28,10 @@ obj-$(CONFIG_DRM_PANEL_HIMAX_HX83112B) += panel-himax-hx83112b.o
 obj-$(CONFIG_DRM_PANEL_HIMAX_HX83121A) += panel-himax-hx83121a.o
 obj-$(CONFIG_DRM_PANEL_HIMAX_HX8394) += panel-himax-hx8394.o
 obj-$(CONFIG_DRM_PANEL_HYDIS_HV101HD1) += panel-hydis-hv101hd1.o
+obj-$(CONFIG_DRM_PANEL_ILITEK_ILI7807S) += panel-ilitek-ili7807s.o
 obj-$(CONFIG_DRM_PANEL_ILITEK_IL9322) += panel-ilitek-ili9322.o
 obj-$(CONFIG_DRM_PANEL_ILITEK_ILI9341) += panel-ilitek-ili9341.o
+obj-$(CONFIG_DRM_PANEL_ILITEK_ILI9488) += panel-ilitek-ili9488.o
 obj-$(CONFIG_DRM_PANEL_ILITEK_ILI9805) += panel-ilitek-ili9805.o
 obj-$(CONFIG_DRM_PANEL_ILITEK_ILI9806E_CORE) += panel-ilitek-ili9806e-core.o
 obj-$(CONFIG_DRM_PANEL_ILITEK_ILI9806E_DSI) += panel-ilitek-ili9806e-dsi.o
@@ -78,6 +82,7 @@ obj-$(CONFIG_DRM_PANEL_RAYDIUM_RM68200) += panel-raydium-rm68200.o
 obj-$(CONFIG_DRM_PANEL_RAYDIUM_RM692E5) += panel-raydium-rm692e5.o
 obj-$(CONFIG_DRM_PANEL_RAYDIUM_RM69380) += panel-raydium-rm69380.o
 obj-$(CONFIG_DRM_PANEL_RENESAS_R61307) += panel-renesas-r61307.o
+obj-$(CONFIG_DRM_PANEL_RENESAS_R63419) += panel-renesas-r63419.o
 obj-$(CONFIG_DRM_PANEL_RENESAS_R69328) += panel-renesas-r69328.o
 obj-$(CONFIG_DRM_PANEL_RONBO_RB070D30) += panel-ronbo-rb070d30.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_AMS581VF01) += panel-samsung-ams581vf01.o
diff --git a/drivers/gpu/drm/panel/panel-anbernic-td4310.c b/drivers/gpu/drm/panel/panel-anbernic-td4310.c
new file mode 100644
index 0000000000000..3b6de1b2fbc64
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-anbernic-td4310.c
@@ -0,0 +1,258 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for Anbernic panels with TD4310 panel controller.
+ *
+ * Copyright (C) 2026 Chris Morgan <macromorgan@hotmail.com>
+ *
+ */
+
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/regulator/consumer.h>
+
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_of.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_probe_helper.h>
+
+#include <video/mipi_display.h>
+
+struct anbernic_panel_td4310_info {
+	const struct drm_display_mode *display_mode;
+	u16 width_mm;
+	u16 height_mm;
+	u32 bus_flags;
+	unsigned long mode_flags;
+	u32 format;
+	u32 lanes;
+	u16 prepare_delay;
+	u16 reset_delay;
+	u16 init_delay;
+	u16 enable_delay;
+	u16 disable_delay;
+	u16 unprepare_delay;
+};
+
+struct anbernic_panel_td4310 {
+	struct device *dev;
+	struct mipi_dsi_device *dsi;
+	struct drm_panel panel;
+	const struct anbernic_panel_td4310_info *panel_info;
+	struct gpio_desc *reset_gpio;
+	struct gpio_desc *enable_gpio;
+	struct regulator *vdd;
+	enum drm_panel_orientation orientation;
+};
+
+static inline struct anbernic_panel_td4310 *panel_to_anbernic_panel_td4310(struct drm_panel *panel)
+{
+	return container_of(panel, struct anbernic_panel_td4310, panel);
+}
+
+static int panel_anbernic_td4310_prepare(struct drm_panel *panel)
+{
+	struct anbernic_panel_td4310 *ctx = panel_to_anbernic_panel_td4310(panel);
+	struct mipi_dsi_device *dsi = ctx->dsi;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
+	int ret;
+
+	ret = regulator_enable(ctx->vdd);
+	if (ret)
+		return ret;
+
+	ret = gpiod_set_value_cansleep(ctx->enable_gpio, 1);
+	if (ret)
+		goto err_enable;
+
+	if (ctx->panel_info->enable_delay)
+		mipi_dsi_msleep(&dsi_ctx, ctx->panel_info->enable_delay);
+
+	ret = gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+	if (ret)
+		goto err_reset;
+
+	mipi_dsi_msleep(&dsi_ctx, 10);
+
+	ret = gpiod_set_value_cansleep(ctx->reset_gpio, 0);
+	if (ret)
+		goto err_reset;
+
+	if (ctx->panel_info->reset_delay)
+		mipi_dsi_msleep(&dsi_ctx, ctx->panel_info->reset_delay);
+
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, ctx->panel_info->prepare_delay);
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, ctx->panel_info->prepare_delay);
+
+	if (dsi_ctx.accum_err) {
+		ret = dsi_ctx.accum_err;
+		goto err_reset;
+	}
+
+	return 0;
+
+err_reset:
+	gpiod_set_value_cansleep(ctx->enable_gpio, 0);
+err_enable:
+	regulator_disable(ctx->vdd);
+	return ret;
+}
+
+static int panel_anbernic_td4310_unprepare(struct drm_panel *panel)
+{
+	struct anbernic_panel_td4310 *ctx = panel_to_anbernic_panel_td4310(panel);
+	struct mipi_dsi_device *dsi = ctx->dsi;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
+
+	mipi_dsi_dcs_set_display_off_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, ctx->panel_info->unprepare_delay);
+	mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, ctx->panel_info->disable_delay);
+
+	gpiod_set_value_cansleep(ctx->enable_gpio, 0);
+
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+
+	regulator_disable(ctx->vdd);
+
+	return 0;
+}
+
+static int panel_anbernic_td4310_get_mode(struct drm_panel *panel,
+				    struct drm_connector *connector)
+{
+	struct anbernic_panel_td4310 *ctx = panel_to_anbernic_panel_td4310(panel);
+	const struct anbernic_panel_td4310_info *panel_info = ctx->panel_info;
+
+	connector->display_info.bpc = 8;
+	connector->display_info.width_mm = panel_info->width_mm;
+	connector->display_info.height_mm = panel_info->height_mm;
+	connector->display_info.bus_flags = panel_info->bus_flags;
+
+	return drm_connector_helper_get_modes_fixed(connector, panel_info->display_mode);
+}
+
+static enum drm_panel_orientation panel_anbernic_td4310_get_orientation(struct drm_panel *panel)
+{
+	struct anbernic_panel_td4310 *ctx = panel_to_anbernic_panel_td4310(panel);
+
+	return ctx->orientation;
+}
+
+static const struct drm_panel_funcs panel_anbernic_td4310_funcs = {
+	.prepare = panel_anbernic_td4310_prepare,
+	.unprepare = panel_anbernic_td4310_unprepare,
+	.get_modes = panel_anbernic_td4310_get_mode,
+	.get_orientation = panel_anbernic_td4310_get_orientation,
+};
+
+static int panel_anbernic_td4310_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	struct anbernic_panel_td4310 *ctx;
+	int ret;
+
+	ctx = devm_drm_panel_alloc(dev, struct anbernic_panel_td4310, panel,
+				   &panel_anbernic_td4310_funcs,
+				   DRM_MODE_CONNECTOR_DSI);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+
+	ctx->dev = dev;
+
+	ctx->panel_info = of_device_get_match_data(dev);
+	if (!ctx->panel_info)
+		return -EINVAL;
+
+	ret = drm_of_get_panel_orientation(dev->of_node, &ctx->orientation);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to get panel orientation\n");
+
+	ctx->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(ctx->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(ctx->reset_gpio),
+				     "Cannot get reset gpio\n");
+
+	ctx->enable_gpio = devm_gpiod_get_optional(dev, "enable", GPIOD_OUT_LOW);
+	if (IS_ERR(ctx->enable_gpio))
+		return dev_err_probe(dev, PTR_ERR(ctx->enable_gpio),
+				     "Cannot get enable gpio\n");
+
+	ctx->vdd = devm_regulator_get(dev, "vdd");
+	if (IS_ERR(ctx->vdd))
+		return dev_err_probe(dev, PTR_ERR(ctx->vdd),
+				     "Failed to request vdd regulator\n");
+
+	ctx->dsi = dsi;
+	mipi_dsi_set_drvdata(dsi, ctx);
+
+	dsi->lanes = ctx->panel_info->lanes;
+	dsi->format = ctx->panel_info->format;
+	dsi->mode_flags = ctx->panel_info->mode_flags;
+
+	ret = drm_panel_of_backlight(&ctx->panel);
+	if (ret)
+		return ret;
+
+	devm_drm_panel_add(dev, &ctx->panel);
+
+	ret = devm_mipi_dsi_attach(dev, dsi);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to attach to DSI host\n");
+
+	return 0;
+}
+
+static const struct drm_display_mode anbernic_vitapro_mode = {
+	.clock = 140020,
+	.hdisplay = 1080,
+	.hsync_start = 1080 + 50,
+	.hsync_end = 1080 + 50 + 4,
+	.htotal = 1080 + 50 + 4 + 50,
+	.vdisplay = 1920,
+	.vsync_start = 1920 + 15,
+	.vsync_end = 1920 + 15 + 4,
+	.vtotal = 1920 + 15 + 4 + 32,
+	.flags = DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
+};
+
+static const struct anbernic_panel_td4310_info anbernic_vitapro_info = {
+	.display_mode = &anbernic_vitapro_mode,
+	.width_mm = 69,
+	.height_mm = 121,
+	.bus_flags = DRM_BUS_FLAG_DE_LOW | DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE,
+	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
+		      MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_NO_EOT_PACKET |
+		      MIPI_DSI_CLOCK_NON_CONTINUOUS,
+	.format = MIPI_DSI_FMT_RGB888,
+	.lanes = 4,
+	.prepare_delay = 50,
+	.reset_delay = 220,
+	.enable_delay = 120,
+	.disable_delay = 50,
+	.unprepare_delay = 20,
+};
+
+static const struct of_device_id panel_anbernic_td4310_of_match[] = {
+	{
+		.compatible = "anbernic,panel-vita-pro",
+		.data = &anbernic_vitapro_info,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, panel_anbernic_td4310_of_match);
+
+static struct mipi_dsi_driver anbernic_panel_td4310_driver = {
+	.driver = {
+		.name = "panel-anbernic-td4310",
+		.of_match_table = panel_anbernic_td4310_of_match,
+	},
+	.probe	= panel_anbernic_td4310_probe,
+};
+module_mipi_dsi_driver(anbernic_panel_td4310_driver);
+
+MODULE_AUTHOR("Chris Morgan <macromorgan@hotmail.com>");
+MODULE_DESCRIPTION("DRM driver for Anbernic TD4310 MIPI DSI panels");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-boe-th101mb31ig002-28a.c b/drivers/gpu/drm/panel/panel-boe-th101mb31ig002-28a.c
index 01b4458e55ad9..a70a2e58f88c1 100644
--- a/drivers/gpu/drm/panel/panel-boe-th101mb31ig002-28a.c
+++ b/drivers/gpu/drm/panel/panel-boe-th101mb31ig002-28a.c
@@ -15,6 +15,7 @@
 #include <drm/drm_connector.h>
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_modes.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_probe_helper.h>
 
@@ -380,7 +381,7 @@ static int boe_th101mb31ig002_dsi_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(&dsi->dev, PTR_ERR(ctx->reset),
 				     "Failed to get reset GPIO\n");
 
-	ret = of_drm_get_panel_orientation(dsi->dev.of_node,
+	ret = drm_of_get_panel_orientation(dsi->dev.of_node,
 					   &ctx->orientation);
 	if (ret)
 		return dev_err_probe(&dsi->dev, ret,
diff --git a/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c b/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c
index 658ce64c71eb2..150dff3ab6c3e 100644
--- a/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c
+++ b/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c
@@ -13,6 +13,7 @@
 #include <drm/drm_connector.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_mipi_dsi.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 
 #include <video/mipi_display.h>
@@ -1722,7 +1723,7 @@ static int boe_panel_add(struct boe_panel *boe)
 
 	boe->base.prepare_prev_first = true;
 
-	err = of_drm_get_panel_orientation(dev->of_node, &boe->orientation);
+	err = drm_of_get_panel_orientation(dev->of_node, &boe->orientation);
 	if (err < 0) {
 		dev_err(dev, "%pOF: failed to get orientation %d\n", dev->of_node, err);
 		return err;
diff --git a/drivers/gpu/drm/panel/panel-chipone-icna35xx.c b/drivers/gpu/drm/panel/panel-chipone-icna35xx.c
new file mode 100644
index 0000000000000..2cc16494a4cef
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-chipone-icna35xx.c
@@ -0,0 +1,423 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Chipone ICNA35XX Driver IC panels driver
+ *
+ * Copyright (c) 2025 Teguh Sobirin <teguh@sobir.in>
+ */
+
+#include <linux/backlight.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+#include <linux/regulator/consumer.h>
+
+#include <video/mipi_display.h>
+
+#include <drm/display/drm_dsc.h>
+#include <drm/display/drm_dsc_helper.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_of.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_probe_helper.h>
+
+struct panel_info {
+	struct drm_panel panel;
+	struct drm_connector *connector;
+	struct mipi_dsi_device *dsi;
+	struct panel_desc *desc;
+	enum drm_panel_orientation orientation;
+
+	struct gpio_desc *reset_gpio;
+	struct regulator_bulk_data *supplies;
+};
+
+struct panel_desc {
+	unsigned int width_mm;
+	unsigned int height_mm;
+
+	unsigned int bpc;
+	unsigned int lanes;
+	unsigned long mode_flags;
+	enum mipi_dsi_pixel_format format;
+
+	const struct drm_display_mode *modes;
+	unsigned int num_modes;
+	int (*init_sequence)(struct panel_info *pinfo);
+
+	struct drm_dsc_config dsc;
+};
+
+static const struct regulator_bulk_data panel_supplies[] = {
+	{ .supply = "vdd" },
+	{ .supply = "vddio" },
+	{ .supply = "vci" },
+	{ .supply = "disp" },
+	{ .supply = "blvdd" },
+};
+
+static inline struct panel_info *to_panel_info(struct drm_panel *panel)
+{
+	return container_of(panel, struct panel_info, panel);
+}
+
+static int icna3512_init_sequence(struct panel_info *pinfo)
+{
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = pinfo->dsi };
+	struct drm_dsc_picture_parameter_set pps;
+
+	pinfo->dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9C, 0xA5, 0xA5);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xFD, 0x5A, 0x5A);
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x53, 0xE0);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x35, 0x00);
+
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+
+	mipi_dsi_msleep(&dsi_ctx, 120);
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9F, 0x0F);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xCE, 0x22);
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9F, 0x01);
+
+	/* 165 hz */
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x48, 0x20);
+
+	drm_dsc_pps_payload_pack(&pps, &pinfo->desc->dsc);
+	mipi_dsi_picture_parameter_set_multi(&dsi_ctx, &pps);
+
+	mipi_dsi_msleep(&dsi_ctx, 20);
+
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
+
+	return dsi_ctx.accum_err;
+}
+
+static int icna3520_init_sequence(struct panel_info *pinfo)
+{
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = pinfo->dsi };
+	struct drm_dsc_picture_parameter_set pps;
+
+	pinfo->dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9C, 0xA5, 0xA5);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xFD, 0x5A, 0x5A);
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x53, 0xE0);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x35, 0x00);
+
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+
+	mipi_dsi_msleep(&dsi_ctx, 120);
+
+	/* 120 hz */
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x48, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9F, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xB3,
+		0x00, 0xD8, 0x00, 0x1C, 0x00, 0x4C);
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9F, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xB2, 0x00);
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9F, 0x0D);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xB2, 0x27);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xB6, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xBB, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xB2, 0x24);
+
+	drm_dsc_pps_payload_pack(&pps, &pinfo->desc->dsc);
+	mipi_dsi_picture_parameter_set_multi(&dsi_ctx, &pps);
+
+	mipi_dsi_msleep(&dsi_ctx, 20);
+
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
+
+	return dsi_ctx.accum_err;
+}
+
+static const struct drm_display_mode odin2portal_modes[] = {
+	{
+		/* 165Hz */
+		.clock = (1080 + 98 + 1 + 23) * (1920 + 20 + 1 + 15) * 165 / 1000,
+		.hdisplay = 1080,
+		.hsync_start = 1080 + 98,
+		.hsync_end = 1080 + 98 + 1,
+		.htotal = 1080 + 98 + 1 + 23,
+		.vdisplay = 1920,
+		.vsync_start = 1920 + 20,
+		.vsync_end = 1920 + 20 + 1,
+		.vtotal = 1920 + 20 + 1 + 15,
+	}
+};
+
+static const struct drm_display_mode thor_top_modes[] = {
+	{
+		/* 120Hz */
+		.clock = (1080 + 24 + 1 + 24) * (1920 + 28 + 1 + 28) * 120 / 1000,
+		.hdisplay = 1080,
+		.hsync_start = 1080 + 24,
+		.hsync_end = 1080 + 24 + 1,
+		.htotal = 1080 + 24 + 1 + 24,
+		.vdisplay = 1920,
+		.vsync_start = 1920 + 28,
+		.vsync_end = 1920 + 28 + 1,
+		.vtotal = 1920 + 28 + 1 + 28,
+	}
+};
+
+static struct panel_desc odin2portal_desc = {
+	.modes = odin2portal_modes,
+	.num_modes = ARRAY_SIZE(odin2portal_modes),
+	.width_mm = 160,
+	.height_mm = 89,
+	.bpc = 8,
+	.lanes = 4,
+	.format = MIPI_DSI_FMT_RGB888,
+	.mode_flags = MIPI_DSI_MODE_NO_EOT_PACKET | MIPI_DSI_CLOCK_NON_CONTINUOUS |
+			MIPI_DSI_MODE_LPM,
+	.init_sequence = icna3512_init_sequence,
+	.dsc = {
+		.dsc_version_major = 0x1,
+		.dsc_version_minor = 0x1,
+		.slice_height = 20,
+		.slice_width = 540,
+		.slice_count = 2,
+		.bits_per_component = 8,
+		.bits_per_pixel = 8 << 4,
+		.block_pred_enable = true,
+	},
+};
+
+static struct panel_desc thor_top_desc = {
+	.modes = thor_top_modes,
+	.num_modes = ARRAY_SIZE(thor_top_modes),
+	.width_mm = 136,
+	.height_mm = 68,
+	.bpc = 8,
+	.lanes = 4,
+	.format = MIPI_DSI_FMT_RGB888,
+	.mode_flags =  MIPI_DSI_MODE_NO_EOT_PACKET | MIPI_DSI_CLOCK_NON_CONTINUOUS |
+			MIPI_DSI_MODE_LPM,
+	.init_sequence = icna3520_init_sequence,
+	.dsc = {
+		.dsc_version_major = 0x1,
+		.dsc_version_minor = 0x1,
+		.slice_height = 12,
+		.slice_width = 540,
+		.slice_count = 2,
+		.bits_per_component = 8,
+		.bits_per_pixel = 8 << 4,
+		.block_pred_enable = true,
+	},
+};
+
+static void icna35xx_reset(struct panel_info *pinfo)
+{
+	gpiod_set_value_cansleep(pinfo->reset_gpio, 0);
+	usleep_range(20000, 21000);
+	gpiod_set_value_cansleep(pinfo->reset_gpio, 1);
+	usleep_range(20000, 21000);
+	gpiod_set_value_cansleep(pinfo->reset_gpio, 0);
+	usleep_range(20000, 21000);
+}
+
+static int icna35xx_prepare(struct drm_panel *panel)
+{
+	struct panel_info *pinfo = to_panel_info(panel);
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(panel_supplies), pinfo->supplies);
+	if (ret < 0) {
+		dev_err(panel->dev, "failed to enable regulators: %d\n", ret);
+		return ret;
+	}
+
+	icna35xx_reset(pinfo);
+
+	ret = pinfo->desc->init_sequence(pinfo);
+	if (ret < 0) {
+		regulator_bulk_disable(ARRAY_SIZE(panel_supplies), pinfo->supplies);
+		dev_err(panel->dev, "failed to initialize panel: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int icna35xx_disable(struct drm_panel *panel)
+{
+	struct panel_info *pinfo = to_panel_info(panel);
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = pinfo->dsi };
+
+	pinfo->dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	mipi_dsi_dcs_set_display_off_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 50);
+	mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 120);
+
+	return dsi_ctx.accum_err;
+}
+
+static int icna35xx_unprepare(struct drm_panel *panel)
+{
+	struct panel_info *pinfo = to_panel_info(panel);
+
+	gpiod_set_value_cansleep(pinfo->reset_gpio, 1);
+	regulator_bulk_disable(ARRAY_SIZE(panel_supplies), pinfo->supplies);
+
+	return 0;
+}
+
+static int icna35xx_get_modes(struct drm_panel *panel,
+			       struct drm_connector *connector)
+{
+	struct panel_info *pinfo = to_panel_info(panel);
+
+	return drm_connector_helper_get_modes_fixed(connector, pinfo->desc->modes);
+}
+
+static enum drm_panel_orientation icna35xx_get_orientation(struct drm_panel *panel)
+{
+	struct panel_info *pinfo = to_panel_info(panel);
+
+	return pinfo->orientation;
+}
+
+static const struct drm_panel_funcs icna35xx_panel_funcs = {
+	.disable = icna35xx_disable,
+	.prepare = icna35xx_prepare,
+	.unprepare = icna35xx_unprepare,
+	.get_modes = icna35xx_get_modes,
+	.get_orientation = icna35xx_get_orientation,
+};
+
+static int icna35xx_bl_update_status(struct backlight_device *bl)
+{
+	struct mipi_dsi_device *dsi = bl_get_data(bl);
+	u16 brightness = backlight_get_brightness(bl);
+	int ret;
+
+	dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_set_display_brightness_large(dsi, brightness);
+
+	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	return ret;
+}
+
+static int icna35xx_bl_get_brightness(struct backlight_device *bl)
+{
+	struct mipi_dsi_device *dsi = bl_get_data(bl);
+	u16 brightness;
+	int ret;
+
+	dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_get_display_brightness_large(dsi, &brightness);
+
+	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	return ret < 0 ? ret : brightness;
+}
+
+static const struct backlight_ops icna35xx_bl_ops = {
+	.update_status = icna35xx_bl_update_status,
+	.get_brightness = icna35xx_bl_get_brightness,
+};
+
+static struct backlight_device *icna35xx_create_backlight(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	const struct backlight_properties props = {
+		.type = BACKLIGHT_RAW,
+		.brightness = 4096,
+		.max_brightness = 4096,
+	};
+
+	return devm_backlight_device_register(dev, dev_name(dev), dev, dsi,
+					      &icna35xx_bl_ops, &props);
+}
+
+static int icna35xx_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	struct panel_info *pinfo;
+	int ret;
+
+	pinfo = devm_drm_panel_alloc(dev, __typeof(*pinfo), panel,
+				     &icna35xx_panel_funcs,
+				     DRM_MODE_CONNECTOR_DSI);
+	if (IS_ERR(pinfo))
+		return PTR_ERR(pinfo);
+
+	ret = devm_regulator_bulk_get_const(dev, ARRAY_SIZE(panel_supplies),
+					    panel_supplies, &pinfo->supplies);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to get regulators\n");
+
+	pinfo->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(pinfo->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(pinfo->reset_gpio), "failed to get reset gpio\n");
+
+	pinfo->desc = (struct panel_desc *)of_device_get_match_data(dev);
+	if (!pinfo->desc)
+		return -ENODEV;
+
+	pinfo->dsi = dsi;
+	mipi_dsi_set_drvdata(dsi, pinfo);
+
+	ret = drm_of_get_panel_orientation(dev->of_node, &pinfo->orientation);
+	if (ret < 0) {
+		dev_err(dev, "%pOF: failed to get orientation %d\n", dev->of_node, ret);
+		return ret;
+	}
+
+	pinfo->panel.prepare_prev_first = true;
+
+	pinfo->panel.backlight = icna35xx_create_backlight(dsi);
+	if (IS_ERR(pinfo->panel.backlight))
+		return dev_err_probe(dev, PTR_ERR(pinfo->panel.backlight),
+				     "Failed to create backlight\n");
+
+	ret = devm_drm_panel_add(dev, &pinfo->panel);
+	if (ret)
+		return ret;
+
+	pinfo->dsi->lanes = pinfo->desc->lanes;
+	pinfo->dsi->format = pinfo->desc->format;
+	pinfo->dsi->mode_flags = pinfo->desc->mode_flags;
+	pinfo->dsi->dsc = &pinfo->desc->dsc;
+
+	return devm_mipi_dsi_attach(dev, dsi);
+}
+
+static const struct of_device_id icna35xx_of_match[] = {
+	{ .compatible = "ayaneo,pocketds-panel-top", .data = &odin2portal_desc },
+	{ .compatible = "ayntec,odin2portal-panel", .data = &odin2portal_desc },
+	{ .compatible = "ayntec,odin3-panel", .data = &thor_top_desc },
+	{ .compatible = "ayntec,thor-panel-top", .data = &thor_top_desc },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, icna35xx_of_match);
+
+static struct mipi_dsi_driver icna35xx_driver = {
+	.probe = icna35xx_probe,
+	.driver = {
+		.name = "panel-chipone-icna35xx",
+		.of_match_table = icna35xx_of_match,
+	},
+};
+module_mipi_dsi_driver(icna35xx_driver);
+
+MODULE_AUTHOR("Teguh Sobirin <teguh@sobir.in>");
+MODULE_DESCRIPTION("DRM driver for Chipone ICNA35XX based MIPI DSI panels");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-chipwealth-ch13726a.c b/drivers/gpu/drm/panel/panel-chipwealth-ch13726a.c
index be76bc825c3fd..562dc573528dd 100644
--- a/drivers/gpu/drm/panel/panel-chipwealth-ch13726a.c
+++ b/drivers/gpu/drm/panel/panel-chipwealth-ch13726a.c
@@ -13,6 +13,7 @@
 
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_modes.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 
 #include <video/mipi_display.h>
@@ -268,7 +269,7 @@ static int ch13726a_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(dev, PTR_ERR(ctx->reset_gpio),
 				     "Failed to get reset-gpios\n");
 
-	ret = of_drm_get_panel_orientation(dev->of_node, &ctx->orientation);
+	ret = drm_of_get_panel_orientation(dev->of_node, &ctx->orientation);
 	if (ret < 0) {
 		dev_err(dev, "%pOF: failed to get orientation %d\n", dev->of_node, ret);
 		return ret;
diff --git a/drivers/gpu/drm/panel/panel-edp.c b/drivers/gpu/drm/panel/panel-edp.c
index aa27d6cd932e8..f66474d3ef65e 100644
--- a/drivers/gpu/drm/panel/panel-edp.c
+++ b/drivers/gpu/drm/panel/panel-edp.c
@@ -40,6 +40,7 @@
 #include <drm/drm_crtc.h>
 #include <drm/drm_device.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 
 /**
@@ -865,7 +866,7 @@ static int panel_edp_probe(struct device *dev, const struct panel_desc *desc,
 		return dev_err_probe(dev, PTR_ERR(panel->enable_gpio),
 				     "failed to request GPIO\n");
 
-	err = of_drm_get_panel_orientation(dev->of_node, &panel->orientation);
+	err = drm_of_get_panel_orientation(dev->of_node, &panel->orientation);
 	if (err) {
 		dev_err(dev, "%pOF: failed to get orientation %d\n", dev->of_node, err);
 		return err;
@@ -1855,6 +1856,14 @@ static const struct panel_delay delay_200_500_e50_po2e200 = {
 	.powered_on_to_enable = 200,
 };
 
+static const struct panel_delay delay_200_500_e50_po2e200_d100 = {
+	.hpd_absent = 200,
+	.unprepare = 500,
+	.enable = 50,
+	.powered_on_to_enable = 200,
+	.disable = 100,
+};
+
 static const struct panel_delay delay_200_150_e50 = {
 	.hpd_absent = 200,
 	.unprepare = 150,
@@ -1934,6 +1943,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x235c, &delay_200_500_e50, "B116XTN02.3"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x239b, &delay_200_500_e50, "B116XAN06.1"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x255c, &delay_200_500_e50, "B116XTN02.5"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x29c0, &delay_200_500_e50, "B116XAT04.3"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x30ed, &delay_200_500_e50, "G156HAN03.0"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x3c9f, &delay_200_500_e50, "B140HAK03.5"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x403d, &delay_200_500_e50, "B140HAN04.0"),
@@ -2004,6 +2014,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0964, &delay_200_500_e50, "NV133WUM-N61"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x096e, &delay_200_500_e50_po2e200, "NV116WHM-T07 V8.0"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0979, &delay_200_500_e50, "NV116WHM-N49 V8.0"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0985, &delay_200_500_e50, "NE160QDM-NY1"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x098d, &boe_nv110wtm_n61.delay, "NV110WTM-N61"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0993, &delay_200_500_e80, "NV116WHM-T14 V8.0"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x09ad, &delay_200_500_e80, "NV116WHM-N47"),
@@ -2036,8 +2047,13 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cfa, &delay_200_500_e50, "NV116WHM-A4D"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0d45, &delay_200_500_e80, "NV116WHM-N4B"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0d73, &delay_200_500_e80, "NE140WUM-N6S"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0d98, &delay_200_500_e50_po2e200_d100, "NV140FHM-N5B"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0db3, &delay_200_500_e80, "NV153WUM-N42"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0ddf, &delay_200_500_e80, "NV116WHM-T01"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x386e, &delay_200_500_e80, "NV116WH2-M30"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x3879, &delay_200_500_e80, "NT116WHM-N21"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x388b, &delay_200_500_e80, "NV116FH1-M31"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x388c, &delay_200_500_e80, "NV116FH1-M30"),
 
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1130, &delay_200_500_e50, "N116BGE-EB2"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1132, &delay_200_500_e80_d50, "N116BGE-EA2"),
@@ -2060,6 +2076,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1161, &delay_200_500_e80, "N116BCP-EA2"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1163, &delay_200_500_e80_d50, "N116BCJ-EAK"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1169, &delay_200_500_e80_d50, "N116BCN-EA1"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x116d, &delay_200_500_e80_d50, "N116BCP-EA2"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x117a, &delay_200_500_e80_d50, "N116BCL-EAK"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1247, &delay_200_500_e80_d50, "N120ACA-EA1"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x124c, &delay_200_500_e80_d50, "N122JCA-ENK"),
@@ -2086,6 +2103,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1103, &delay_200_500_e80_d50, "MNB601LS1-3"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1104, &delay_200_500_e50_d100, "MNB601LS1-4"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x110a, &delay_200_500_e50, "PNB601LS1-2"),
+	EDP_PANEL_ENTRY('C', 'S', 'W', 0x110d, &delay_200_500_e50, "MNB601LS1-8"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x143f, &delay_200_500_e50, "MNE007QS3-6"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1448, &delay_200_500_e50, "MNE007QS3-7"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x144b, &delay_200_500_e80, "MNE001BS1-4"),
@@ -2094,11 +2112,14 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1468, &delay_200_500_e50, "MNE007QB2-2"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x146e, &delay_80_500_e50_d50, "MNE007QB3-1"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x147c, &delay_200_500_e50_d100, "MNE007QB3-1"),
+	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1486, &delay_200_500_e50_d100, "MNE001BS6-2"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1519, &delay_200_500_e80_d50, "MNF601BS1-3"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1529, &delay_200_500_e80_d50, "MNF307QS3-2"),
+	EDP_PANEL_ENTRY('C', 'S', 'W', 0x153f, &delay_200_500_e200_d50, "MNF601BS4-1"),
 
 	EDP_PANEL_ENTRY('E', 'T', 'C', 0x0000, &delay_50_500_e200_d200_po2e335, "LP079QX1-SP0V"),
 
+	EDP_PANEL_ENTRY('H', 'K', 'C', 0x1203, &delay_200_500_e80, "MB116AS01'3"),
 	EDP_PANEL_ENTRY('H', 'K', 'C', 0x2d51, &delay_200_500_e200, "Unknown"),
 	EDP_PANEL_ENTRY('H', 'K', 'C', 0x2d5b, &delay_200_500_e200, "MB116AN01"),
 	EDP_PANEL_ENTRY('H', 'K', 'C', 0x2d5c, &delay_200_500_e200, "MB116AN01-2"),
@@ -2151,6 +2172,7 @@ static const struct edp_panel_entry edp_panels[] = {
 
 	EDP_PANEL_ENTRY('T', 'M', 'A', 0x0811, &delay_200_500_e80_d50, "TM140VDXP01-04"),
 	EDP_PANEL_ENTRY('T', 'M', 'A', 0x2094, &delay_200_500_e50_d100, "TL140VDMS03-01"),
+	EDP_PANEL_ENTRY('T', 'M', 'A', 0x2139, &delay_200_500_e50_d100, "TM156VDXP25"),
 
 	{ /* sentinal */ }
 };
diff --git a/drivers/gpu/drm/panel/panel-elida-kd35t133.c b/drivers/gpu/drm/panel/panel-elida-kd35t133.c
index 1f177834d6298..d23002b5a2d78 100644
--- a/drivers/gpu/drm/panel/panel-elida-kd35t133.c
+++ b/drivers/gpu/drm/panel/panel-elida-kd35t133.c
@@ -21,6 +21,7 @@
 
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_modes.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 
 /* Manufacturer specific Commands send via DSI */
@@ -233,7 +234,7 @@ static int kd35t133_probe(struct mipi_dsi_device *dsi)
 		return ret;
 	}
 
-	ret = of_drm_get_panel_orientation(dev->of_node, &ctx->orientation);
+	ret = drm_of_get_panel_orientation(dev->of_node, &ctx->orientation);
 	if (ret < 0) {
 		dev_err(dev, "%pOF: failed to get orientation %d\n", dev->of_node, ret);
 		return ret;
diff --git a/drivers/gpu/drm/panel/panel-focaltech-ota7290b.c b/drivers/gpu/drm/panel/panel-focaltech-ota7290b.c
index ed02a8daf96f9..bbc870b8fda8e 100644
--- a/drivers/gpu/drm/panel/panel-focaltech-ota7290b.c
+++ b/drivers/gpu/drm/panel/panel-focaltech-ota7290b.c
@@ -17,6 +17,7 @@
 
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_modes.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_probe_helper.h>
 
@@ -181,7 +182,7 @@ static int ota7290b_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(&dsi->dev, PTR_ERR(ctx->vdd),
 					"Couldn't get our VDD supply\n");
 
-	ret = of_drm_get_panel_orientation(dsi->dev.of_node, &ctx->orientation);
+	ret = drm_of_get_panel_orientation(dsi->dev.of_node, &ctx->orientation);
 	if (ret) {
 		dev_err(&dsi->dev, "%pOF: failed to get orientation: %d\n",
 			dsi->dev.of_node, ret);
diff --git a/drivers/gpu/drm/panel/panel-himax-hx83102.c b/drivers/gpu/drm/panel/panel-himax-hx83102.c
index d7e5664a5838a..6a0851ccf9bb0 100644
--- a/drivers/gpu/drm/panel/panel-himax-hx83102.c
+++ b/drivers/gpu/drm/panel/panel-himax-hx83102.c
@@ -17,6 +17,7 @@
 #include <drm/drm_connector.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_mipi_dsi.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 
 #include <video/mipi_display.h>
@@ -1264,7 +1265,7 @@ static int hx83102_panel_add(struct hx83102 *ctx)
 
 	ctx->base.prepare_prev_first = true;
 
-	err = of_drm_get_panel_orientation(dev->of_node, &ctx->orientation);
+	err = drm_of_get_panel_orientation(dev->of_node, &ctx->orientation);
 	if (err < 0)
 		return dev_err_probe(dev, err, "failed to get orientation\n");
 
diff --git a/drivers/gpu/drm/panel/panel-himax-hx83121a.c b/drivers/gpu/drm/panel/panel-himax-hx83121a.c
index 8b11fce4c7c5a..3b6a562c4d115 100644
--- a/drivers/gpu/drm/panel/panel-himax-hx83121a.c
+++ b/drivers/gpu/drm/panel/panel-himax-hx83121a.c
@@ -33,7 +33,9 @@ struct himax {
 	struct drm_dsc_config dsc;
 	struct gpio_desc *reset_gpio;
 	struct regulator_bulk_data *supplies;
+	struct regulator *bl_supply;
 	struct backlight_device *backlight;
+	bool backlight_enabled;
 };
 
 struct panel_desc {
@@ -192,10 +194,31 @@ static const struct drm_panel_funcs himax_panel_funcs = {
 
 static int himax_bl_update_status(struct backlight_device *bl)
 {
-	struct mipi_dsi_device *dsi = bl_get_data(bl);
+	struct himax *ctx = bl_get_data(bl);
 	u16 brightness = backlight_get_brightness(bl);
+	int ret = 0;
+
+	if (!brightness) {
+		if (ctx->backlight_enabled)
+			ret = regulator_disable(ctx->bl_supply);
+		if (ret)
+			return ret;
+
+		ctx->backlight_enabled = false;
+
+		return 0;
+	}
+
 	/* TODO: brightness to raw map table */
-	return mipi_dsi_dcs_set_display_brightness_large(dsi, brightness);
+	if (!ctx->backlight_enabled)
+		ret = regulator_enable(ctx->bl_supply);
+	if (ret)
+		return ret;
+
+	ctx->backlight_enabled = true;
+
+	return mipi_dsi_dcs_set_display_brightness_large(to_primary_dsi(ctx),
+							 brightness);
 }
 
 static const struct backlight_ops himax_bl_ops = {
@@ -204,9 +227,9 @@ static const struct backlight_ops himax_bl_ops = {
 };
 
 static struct backlight_device *
-himax_create_backlight(struct mipi_dsi_device *dsi)
+himax_create_backlight(struct himax *ctx)
 {
-	struct device *dev = &dsi->dev;
+	struct device *dev = &to_primary_dsi(ctx)->dev;
 	const struct backlight_properties props = {
 		.type = BACKLIGHT_RAW,
 		.brightness = 512,
@@ -214,7 +237,7 @@ himax_create_backlight(struct mipi_dsi_device *dsi)
 		.scale = BACKLIGHT_SCALE_NON_LINEAR,
 	};
 
-	return devm_backlight_device_register(dev, dev_name(dev), dev, dsi,
+	return devm_backlight_device_register(dev, dev_name(dev), dev, ctx,
 					      &himax_bl_ops, &props);
 }
 
@@ -645,7 +668,11 @@ static int himax_probe(struct mipi_dsi_device *dsi)
 	ctx->panel.prepare_prev_first = true;
 
 	if (desc->has_dcs_backlight) {
-		ctx->backlight = himax_create_backlight(to_primary_dsi(ctx));
+		ctx->bl_supply = devm_regulator_get_optional(dev, "bl");
+		if (IS_ERR(ctx->bl_supply))
+			return dev_err_probe(dev, PTR_ERR(ctx->bl_supply),
+					     "Failed to get backlight supply\n");
+		ctx->backlight = himax_create_backlight(ctx);
 		if (IS_ERR(ctx->backlight))
 			return dev_err_probe(dev, PTR_ERR(ctx->backlight),
 					     "Failed to create backlight\n");
diff --git a/drivers/gpu/drm/panel/panel-himax-hx8394.c b/drivers/gpu/drm/panel/panel-himax-hx8394.c
index 416203da2f45f..09124610fdc86 100644
--- a/drivers/gpu/drm/panel/panel-himax-hx8394.c
+++ b/drivers/gpu/drm/panel/panel-himax-hx8394.c
@@ -21,6 +21,7 @@
 
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_modes.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 
 #define DRV_NAME "panel-himax-hx8394"
@@ -991,7 +992,7 @@ static int hx8394_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(dev, PTR_ERR(ctx->reset_gpio),
 				     "Failed to get reset gpio\n");
 
-	ret = of_drm_get_panel_orientation(dev->of_node, &ctx->orientation);
+	ret = drm_of_get_panel_orientation(dev->of_node, &ctx->orientation);
 	if (ret < 0) {
 		dev_err(dev, "%pOF: failed to get orientation %d\n", dev->of_node, ret);
 		return ret;
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili7807s.c b/drivers/gpu/drm/panel/panel-ilitek-ili7807s.c
new file mode 100644
index 0000000000000..12b491b0bca4c
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili7807s.c
@@ -0,0 +1,285 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/backlight.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/regulator/consumer.h>
+
+#include <video/mipi_display.h>
+
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_probe_helper.h>
+
+struct panel_desc {
+	const struct drm_display_mode *mode;
+	unsigned int lanes;
+	enum mipi_dsi_pixel_format format;
+	unsigned long mode_flags;
+	void (*init)(struct mipi_dsi_multi_context *dsi_ctx);
+};
+
+struct ili7807s {
+	struct drm_panel panel;
+	struct mipi_dsi_device *dsi;
+	const struct panel_desc *desc;
+
+	struct regulator_bulk_data *supplies;
+	struct gpio_desc *reset_gpio;
+};
+
+static const struct regulator_bulk_data ili7807s_supplies[] = {
+	{ .supply = "vddi" },
+	{ .supply = "avdd" },
+	{ .supply = "avee" },
+};
+
+static inline struct ili7807s *to_ili7807s(struct drm_panel *panel)
+{
+	return container_of(panel, struct ili7807s, panel);
+}
+
+static void ili7807s_reset(struct ili7807s *ctx)
+{
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+	usleep_range(10000, 11000);
+	gpiod_set_value_cansleep(ctx->reset_gpio, 0);
+	usleep_range(10000, 11000);
+}
+
+static void dlc0697_init_sequence(struct mipi_dsi_multi_context *dsi_ctx)
+{
+	mipi_dsi_dcs_soft_reset_multi(dsi_ctx);
+	mipi_dsi_msleep(dsi_ctx, 120);
+
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xff, 0x78, 0x07, 0x00);
+	mipi_dsi_dcs_set_tear_on_multi(dsi_ctx, MIPI_DSI_DCS_TEAR_MODE_VBLANK);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, MIPI_DCS_SET_CABC_MIN_BRIGHTNESS, 0x09, 0x99);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, MIPI_DCS_WRITE_CONTROL_DISPLAY, 0x24);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, MIPI_DCS_WRITE_POWER_SAVE, 0x01);
+	mipi_dsi_dcs_set_display_brightness_multi(dsi_ctx, 0x1fff);
+
+	mipi_dsi_dcs_exit_sleep_mode_multi(dsi_ctx);
+	mipi_dsi_msleep(dsi_ctx, 120);
+
+	mipi_dsi_dcs_set_display_on_multi(dsi_ctx);
+	mipi_dsi_msleep(dsi_ctx, 20);
+}
+
+static int ili7807s_on(struct ili7807s *ctx)
+{
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
+
+	ctx->desc->init(&dsi_ctx);
+
+	return dsi_ctx.accum_err;
+}
+
+static int ili7807s_off(struct ili7807s *ctx)
+{
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
+
+	mipi_dsi_dcs_set_display_off_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 20);
+
+	mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 120);
+
+	return dsi_ctx.accum_err;
+}
+
+static int ili7807s_prepare(struct drm_panel *panel)
+{
+	struct ili7807s *ctx = to_ili7807s(panel);
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(ili7807s_supplies), ctx->supplies);
+	if (ret < 0) {
+		dev_err(ctx->panel.dev, "failed to enable regulators: %d\n", ret);
+		return ret;
+	}
+
+	msleep(20);
+
+	ili7807s_reset(ctx);
+
+	ret = ili7807s_on(ctx);
+	if (ret < 0) {
+		dev_err(ctx->panel.dev, "failed to initialise panel: %d\n", ret);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+
+	regulator_bulk_disable(ARRAY_SIZE(ili7807s_supplies), ctx->supplies);
+	return ret;
+}
+
+static int ili7807s_unprepare(struct drm_panel *panel)
+{
+	struct ili7807s *ctx = to_ili7807s(panel);
+	int ret;
+
+	ret = ili7807s_off(ctx);
+	if (ret < 0)
+		dev_err(ctx->panel.dev, "failed to disable panel: %d\n", ret);
+
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+
+	regulator_bulk_disable(ARRAY_SIZE(ili7807s_supplies), ctx->supplies);
+
+	return 0;
+}
+
+static int ili7807s_get_modes(struct drm_panel *panel,
+			      struct drm_connector *connector)
+{
+	struct ili7807s *ctx = to_ili7807s(panel);
+
+	return drm_connector_helper_get_modes_fixed(connector, ctx->desc->mode);
+}
+
+static const struct drm_panel_funcs ili7807s_panel_funcs = {
+	.prepare = ili7807s_prepare,
+	.unprepare = ili7807s_unprepare,
+	.get_modes = ili7807s_get_modes,
+};
+
+static int ili7807s_bl_update_status(struct backlight_device *bl)
+{
+	struct mipi_dsi_device *dsi = bl_get_data(bl);
+	u16 brightness = backlight_get_brightness(bl);
+	int ret;
+
+	dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_set_display_brightness_large(dsi, brightness);
+
+	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	return ret;
+}
+
+static const struct backlight_ops ili7807s_bl_ops = {
+	.update_status = ili7807s_bl_update_status,
+};
+
+static struct backlight_device *ili7807s_create_backlight(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	const struct backlight_properties props = {
+		.type           = BACKLIGHT_RAW,
+		.brightness     = 0x3fff,
+		.max_brightness = 0x3fff,
+	};
+
+	return devm_backlight_device_register(dev, dev_name(dev), dev, dsi,
+					      &ili7807s_bl_ops, &props);
+}
+
+static const struct drm_display_mode dlc0697_mode = {
+	.clock = (1080 + 18 + 2 + 16) * (1920 + 26 + 4 + 20) * 60 / 1000,
+
+	.hdisplay    = 1080,
+	.hsync_start = 1080 + 18,
+	.hsync_end   = 1080 + 18 + 2,
+	.htotal      = 1080 + 18 + 2 + 16,
+
+	.vdisplay    = 1920,
+	.vsync_start = 1920 + 26,
+	.vsync_end   = 1920 + 26 + 4,
+	.vtotal      = 1920 + 26 + 4 + 20,
+
+	.width_mm  = 87,
+	.height_mm = 154,
+	.type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+};
+
+static const struct panel_desc dlc0697_desc = {
+	.mode       = &dlc0697_mode,
+	.lanes      = 4,
+	.format     = MIPI_DSI_FMT_RGB888,
+	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST,
+	.init       = dlc0697_init_sequence,
+};
+
+static int ili7807s_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	const struct panel_desc *desc;
+	struct ili7807s *ctx;
+	int ret;
+
+	ctx = devm_drm_panel_alloc(dev, struct ili7807s, panel,
+				   &ili7807s_panel_funcs,
+				   DRM_MODE_CONNECTOR_DSI);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+
+	desc = of_device_get_match_data(dev);
+	ctx->desc = desc;
+
+	ret = devm_regulator_bulk_get_const(dev, ARRAY_SIZE(ili7807s_supplies),
+					    ili7807s_supplies, &ctx->supplies);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to get regulators\n");
+
+	ctx->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(ctx->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(ctx->reset_gpio),
+				     "failed to get reset gpio\n");
+
+	ctx->dsi = dsi;
+	mipi_dsi_set_drvdata(dsi, ctx);
+
+	dsi->lanes      = desc->lanes;
+	dsi->format     = desc->format;
+	dsi->mode_flags = desc->mode_flags | MIPI_DSI_MODE_LPM;
+
+	ctx->panel.prepare_prev_first = true;
+
+	ret = drm_panel_of_backlight(&ctx->panel);
+	if (ret)
+		return ret;
+
+	if (!ctx->panel.backlight) {
+		ctx->panel.backlight = ili7807s_create_backlight(dsi);
+		if (IS_ERR(ctx->panel.backlight))
+			return dev_err_probe(dev, PTR_ERR(ctx->panel.backlight),
+					     "failed to create backlight\n");
+	}
+
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to add panel\n");
+
+	return devm_mipi_dsi_attach(dev, dsi);
+}
+
+static const struct of_device_id ili7807s_of_match[] = {
+	{ .compatible = "dlc,dlc0697", .data = &dlc0697_desc },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ili7807s_of_match);
+
+static struct mipi_dsi_driver ili7807s_dsi_driver = {
+	.probe  = ili7807s_probe,
+	.driver = {
+		.name           = "panel-ilitek-ili7807s",
+		.of_match_table = ili7807s_of_match,
+	},
+};
+module_mipi_dsi_driver(ili7807s_dsi_driver);
+
+MODULE_AUTHOR("Arpit Saini <arpit.saini@oss.qualcomm.com>");
+MODULE_DESCRIPTION("Panel driver for Ilitek ILI7807S LCD DSI panel");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9488.c b/drivers/gpu/drm/panel/panel-ilitek-ili9488.c
new file mode 100644
index 0000000000000..6ab8ae72abbfa
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9488.c
@@ -0,0 +1,290 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+
+#include <linux/gpio/consumer.h>
+#include <linux/regulator/consumer.h>
+
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_of.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_probe_helper.h>
+
+#include <video/mipi_display.h>
+
+struct ili9488_desc {
+	const struct drm_display_mode *display_mode;
+	unsigned long mode_flags;
+	enum mipi_dsi_pixel_format format;
+	unsigned int lanes;
+	unsigned int bpc;
+	void (*init_sequence)(struct mipi_dsi_multi_context *ctx);
+};
+
+struct ili9488 {
+	struct drm_panel panel;
+	struct mipi_dsi_device *dsi;
+	struct gpio_desc *reset;
+	struct regulator_bulk_data supplies[2];
+	const struct ili9488_desc *desc;
+	enum drm_panel_orientation orientation;
+};
+
+static const char * const regulator_names[] = {
+	"vci",
+	"iovcc",
+};
+
+static void e35gh_i_mw800cb_init(struct mipi_dsi_multi_context *ctx)
+{
+	/* Gamma control 1,2 */
+	mipi_dsi_dcs_write_seq_multi(ctx, 0xE0, 0x00, 0x10, 0x14, 0x01, 0x0E, 0x04, 0x33,
+				     0x56, 0x48, 0x03, 0x0C, 0x0B, 0x2B, 0x34, 0x0F);
+	mipi_dsi_dcs_write_seq_multi(ctx, 0xE1, 0x00, 0x12, 0x18, 0x05, 0x12, 0x06, 0x40,
+				     0x34, 0x57, 0x06, 0x10, 0x0C, 0x3B, 0x3F, 0x0F);
+	/* Power control 1,2 */
+	mipi_dsi_dcs_write_seq_multi(ctx, 0xC0, 0x0F, 0x0C);
+	mipi_dsi_dcs_write_seq_multi(ctx, 0xC1, 0x41);
+	/* VCOM Control */
+	mipi_dsi_dcs_write_seq_multi(ctx, 0xC5, 0x00, 0x25, 0x80);
+	mipi_dsi_dcs_write_seq_multi(ctx, 0x36, 0x48);
+	/* Interface pixel format 18bpp */
+	mipi_dsi_dcs_write_seq_multi(ctx, 0x3A, 0x66);
+	mipi_dsi_dcs_write_seq_multi(ctx, 0xB0, 0x00);
+	mipi_dsi_dcs_write_seq_multi(ctx, 0xB1, 0xA0);
+	mipi_dsi_dcs_write_seq_multi(ctx, 0xB4, 0x02);
+	mipi_dsi_dcs_write_seq_multi(ctx, 0xB6, 0x02, 0x02, 0x3B);
+	mipi_dsi_dcs_write_seq_multi(ctx, 0xE9, 0x00);
+	mipi_dsi_dcs_write_seq_multi(ctx, 0xF7, 0xA9, 0x51, 0x2C, 0x82);
+	mipi_dsi_dcs_write_seq_multi(ctx, 0x21);
+}
+
+static const struct drm_display_mode e35gh_i_mw800cb_display_mode = {
+	.clock = 14400,
+
+	.hdisplay = 320,
+	.hsync_start = 320 + 60,
+	.hsync_end = 320 + 60 + 20,
+	.htotal = 320 + 60 + 20 + 42,
+
+	.vdisplay = 480,
+	.vsync_start = 480 + 20,
+	.vsync_end = 480 + 20 + 10,
+	.vtotal = 480 + 20 + 10 + 33,
+
+	.width_mm = 48,
+	.height_mm = 73,
+
+	.flags = DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
+	.type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+};
+
+static inline struct ili9488 *panel_to_ili9488(struct drm_panel *panel)
+{
+	return container_of(panel, struct ili9488, panel);
+}
+
+static int ili9488_power_on(struct ili9488 *ili)
+{
+	struct mipi_dsi_device *dsi = ili->dsi;
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(ili->supplies), ili->supplies);
+	if (ret < 0) {
+		dev_err(&dsi->dev, "regulator bulk enable failed: %d\n", ret);
+		return ret;
+	}
+
+	gpiod_set_value_cansleep(ili->reset, 0);
+	usleep_range(1000, 5000);
+	gpiod_set_value_cansleep(ili->reset, 1);
+	usleep_range(1000, 5000);
+	gpiod_set_value_cansleep(ili->reset, 0);
+	usleep_range(5000, 10000);
+
+	return 0;
+}
+
+static int ili9488_power_off(struct ili9488 *ili)
+{
+	struct mipi_dsi_device *dsi = ili->dsi;
+	int ret;
+
+	gpiod_set_value_cansleep(ili->reset, 1);
+
+	ret = regulator_bulk_disable(ARRAY_SIZE(ili->supplies), ili->supplies);
+	if (ret)
+		dev_err(&dsi->dev, "regulator bulk disable failed: %d\n", ret);
+
+	return ret;
+}
+
+static int ili9488_activate(struct ili9488 *ili)
+{
+	struct mipi_dsi_multi_context ctx = { .dsi = ili->dsi };
+
+	if (ili->desc->init_sequence)
+		ili->desc->init_sequence(&ctx);
+
+	mipi_dsi_dcs_exit_sleep_mode_multi(&ctx);
+	mipi_dsi_msleep(&ctx, 120);
+	mipi_dsi_dcs_set_display_on_multi(&ctx);
+
+	return ctx.accum_err;
+}
+
+static int ili9488_prepare(struct drm_panel *panel)
+{
+	struct ili9488 *ili = panel_to_ili9488(panel);
+	int ret;
+
+	ret = ili9488_power_on(ili);
+	if (ret)
+		return ret;
+
+	ret = ili9488_activate(ili);
+	if (ret) {
+		ili9488_power_off(ili);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int ili9488_deactivate(struct ili9488 *ili)
+{
+	struct mipi_dsi_multi_context ctx = { .dsi = ili->dsi };
+
+	mipi_dsi_dcs_set_display_off_multi(&ctx);
+	mipi_dsi_dcs_enter_sleep_mode_multi(&ctx);
+	mipi_dsi_msleep(&ctx, 120);
+
+	return ctx.accum_err;
+}
+
+static int ili9488_unprepare(struct drm_panel *panel)
+{
+	struct ili9488 *ili = panel_to_ili9488(panel);
+	struct mipi_dsi_device *dsi = ili->dsi;
+	int ret;
+
+	ili9488_deactivate(ili);
+	ret = ili9488_power_off(ili);
+	if (ret < 0)
+		dev_err(&dsi->dev, "power off failed: %d\n", ret);
+
+	return ret;
+}
+
+static int ili9488_get_modes(struct drm_panel *panel, struct drm_connector *connector)
+{
+	struct ili9488 *ili = panel_to_ili9488(panel);
+	const struct drm_display_mode *mode = ili->desc->display_mode;
+
+	connector->display_info.bpc = ili->desc->bpc;
+
+	return drm_connector_helper_get_modes_fixed(connector, mode);
+}
+
+static enum drm_panel_orientation ili9488_get_orientation(struct drm_panel *panel)
+{
+	struct ili9488 *ili = panel_to_ili9488(panel);
+
+	return ili->orientation;
+}
+
+static const struct drm_panel_funcs ili9488_funcs = {
+	.prepare	= ili9488_prepare,
+	.unprepare	= ili9488_unprepare,
+	.get_modes	= ili9488_get_modes,
+	.get_orientation = ili9488_get_orientation,
+};
+
+static int ili9488_dsi_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	struct ili9488 *ili;
+	int i, ret;
+
+	ili = devm_drm_panel_alloc(dev, struct ili9488, panel, &ili9488_funcs,
+				   DRM_MODE_CONNECTOR_DSI);
+	if (IS_ERR(ili))
+		return PTR_ERR(ili);
+
+	ili->desc = device_get_match_data(dev);
+	ili->dsi = dsi;
+
+	dsi->mode_flags = ili->desc->mode_flags;
+	dsi->format = ili->desc->format;
+	dsi->lanes = ili->desc->lanes;
+
+	ili->reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(ili->reset))
+		return dev_err_probe(dev, PTR_ERR(ili->reset),
+				     "failed to get reset-gpios\n");
+
+	for (i = 0; i < ARRAY_SIZE(ili->supplies); i++)
+		ili->supplies[i].supply = regulator_names[i];
+
+	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(ili->supplies),
+				      ili->supplies);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to get regulators\n");
+
+	ret = drm_of_get_panel_orientation(dev->of_node, &ili->orientation);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to get orientation\n");
+
+	ret = drm_panel_of_backlight(&ili->panel);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to get backlight\n");
+
+	ili->panel.prepare_prev_first = true;
+
+	ret = devm_drm_panel_add(dev, &ili->panel);
+	if (ret)
+		return ret;
+
+	ret = devm_mipi_dsi_attach(dev, dsi);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to attach to DSI host\n");
+
+	return 0;
+}
+
+static const struct ili9488_desc e35gh_i_mw800cb_desc = {
+	.init_sequence = e35gh_i_mw800cb_init,
+	.display_mode = &e35gh_i_mw800cb_display_mode,
+	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_SYNC_PULSE |
+		      MIPI_DSI_MODE_LPM | MIPI_DSI_CLOCK_NON_CONTINUOUS,
+	.format = MIPI_DSI_FMT_RGB666_PACKED,
+	.lanes = 1,
+	.bpc = 6,
+};
+
+static const struct of_device_id ili9488_of_match[] = {
+	{ .compatible = "focuslcds,e35gh-i-mw800cb", .data = &e35gh_i_mw800cb_desc },
+	{ }
+};
+
+MODULE_DEVICE_TABLE(of, ili9488_of_match);
+
+static struct mipi_dsi_driver ili9488_dsi_driver = {
+	.probe	= ili9488_dsi_probe,
+	.driver = {
+		.name		= "ili9488-dsi",
+		.of_match_table	= ili9488_of_match,
+	},
+};
+module_mipi_dsi_driver(ili9488_dsi_driver);
+
+MODULE_AUTHOR("Igor Reznichenko <igor@reznichenko.net>");
+MODULE_DESCRIPTION("Ilitek ILI9488 Controller Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9806e-dsi.c b/drivers/gpu/drm/panel/panel-ilitek-ili9806e-dsi.c
index ad33414719fc1..0fb06c4255451 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9806e-dsi.c
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9806e-dsi.c
@@ -9,6 +9,7 @@
 
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_modes.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_probe_helper.h>
 
@@ -131,7 +132,7 @@ static int ili9806e_dsi_probe(struct mipi_dsi_device *dsi)
 	dsi->format = ctx->desc->format;
 	dsi->lanes = ctx->desc->lanes;
 
-	ret = of_drm_get_panel_orientation(dev->of_node, &ctx->orientation);
+	ret = drm_of_get_panel_orientation(dev->of_node, &ctx->orientation);
 	if (ret)
 		return dev_err_probe(dev, ret, "Failed to get orientation\n");
 
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c b/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c
index 0652cdb57d112..3abdd0870e831 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c
@@ -16,6 +16,7 @@
 
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_modes.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 
 #include <video/mipi_display.h>
@@ -2571,7 +2572,7 @@ static int ili9881c_dsi_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(&dsi->dev, PTR_ERR(ctx->reset),
 				     "Couldn't get our reset GPIO\n");
 
-	ret = of_drm_get_panel_orientation(dsi->dev.of_node, &ctx->orientation);
+	ret = drm_of_get_panel_orientation(dsi->dev.of_node, &ctx->orientation);
 	if (ret) {
 		dev_err(&dsi->dev, "%pOF: failed to get orientation: %d\n",
 			dsi->dev.of_node, ret);
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9882t.c b/drivers/gpu/drm/panel/panel-ilitek-ili9882t.c
index 5f4e0d82ee673..6d07fe901357e 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9882t.c
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9882t.c
@@ -13,6 +13,7 @@
 #include <drm/drm_connector.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_mipi_dsi.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 
 #include <video/mipi_display.h>
@@ -795,7 +796,7 @@ static int ili9882t_add(struct ili9882t *ili)
 
 	gpiod_set_value_cansleep(ili->enable_gpio, 0);
 
-	err = of_drm_get_panel_orientation(dev->of_node, &ili->orientation);
+	err = drm_of_get_panel_orientation(dev->of_node, &ili->orientation);
 	if (err < 0) {
 		dev_err(dev, "%pOF: failed to get orientation %d\n", dev->of_node, err);
 		return err;
diff --git a/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c b/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c
index f6b04de1182e3..67ef9e0fea03c 100644
--- a/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c
+++ b/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c
@@ -2992,7 +2992,7 @@ static int jadard_dsi_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(&dsi->dev, PTR_ERR(jadard->vccio),
 				"failed to get vccio regulator\n");
 
-	ret = of_drm_get_panel_orientation(dev->of_node, &jadard->orientation);
+	ret = drm_of_get_panel_orientation(dev->of_node, &jadard->orientation);
 	if (ret < 0)
 		return dev_err_probe(dev, ret, "failed to get orientation\n");
 
diff --git a/drivers/gpu/drm/panel/panel-lvds.c b/drivers/gpu/drm/panel/panel-lvds.c
index 46b07f38559f3..37f7498449ec2 100644
--- a/drivers/gpu/drm/panel/panel-lvds.c
+++ b/drivers/gpu/drm/panel/panel-lvds.c
@@ -126,7 +126,7 @@ static int panel_lvds_parse_dt(struct panel_lvds *lvds)
 	struct device_node *np = lvds->dev->of_node;
 	int ret;
 
-	ret = of_drm_get_panel_orientation(np, &lvds->orientation);
+	ret = drm_of_get_panel_orientation(np, &lvds->orientation);
 	if (ret < 0) {
 		dev_err(lvds->dev, "%pOF: failed to get orientation %d\n", np, ret);
 		return ret;
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt36523.c b/drivers/gpu/drm/panel/panel-novatek-nt36523.c
index 226d91daf8c79..34b9123d3fd75 100644
--- a/drivers/gpu/drm/panel/panel-novatek-nt36523.c
+++ b/drivers/gpu/drm/panel/panel-novatek-nt36523.c
@@ -19,6 +19,7 @@
 #include <drm/drm_crtc.h>
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_modes.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 
 #define DSI_NUM_MIN 1
@@ -1206,7 +1207,7 @@ static int nt36523_probe(struct mipi_dsi_device *dsi)
 	pinfo->dsi[0] = dsi;
 	mipi_dsi_set_drvdata(dsi, pinfo);
 
-	ret = of_drm_get_panel_orientation(dev->of_node, &pinfo->orientation);
+	ret = drm_of_get_panel_orientation(dev->of_node, &pinfo->orientation);
 	if (ret < 0) {
 		dev_err(dev, "%pOF: failed to get orientation %d\n", dev->of_node, ret);
 		return ret;
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt36672a.c b/drivers/gpu/drm/panel/panel-novatek-nt36672a.c
index 7e8b5e0595759..464d9ce47d87d 100644
--- a/drivers/gpu/drm/panel/panel-novatek-nt36672a.c
+++ b/drivers/gpu/drm/panel/panel-novatek-nt36672a.c
@@ -28,10 +28,6 @@
 
 #include <video/mipi_display.h>
 
-struct nt36672a_panel_cmd {
-	const char data[2];
-};
-
 static const char * const nt36672a_regulator_names[] = {
 	"vddio",
 	"vddpos",
@@ -55,13 +51,9 @@ struct nt36672a_panel_desc {
 	enum mipi_dsi_pixel_format format;
 	unsigned int lanes;
 
-	unsigned int num_on_cmds_1;
-	const struct nt36672a_panel_cmd *on_cmds_1;
-	unsigned int num_on_cmds_2;
-	const struct nt36672a_panel_cmd *on_cmds_2;
-
-	unsigned int num_off_cmds;
-	const struct nt36672a_panel_cmd *off_cmds;
+	void (*send_init_cmds_1)(struct mipi_dsi_multi_context *dsi_ctx);
+	void (*send_init_cmds_2)(struct mipi_dsi_multi_context *dsi_ctx);
+	void (*send_deinit_cmds)(struct mipi_dsi_multi_context *dsi_ctx);
 };
 
 struct nt36672a_panel {
@@ -79,19 +71,6 @@ static inline struct nt36672a_panel *to_nt36672a_panel(struct drm_panel *panel)
 	return container_of(panel, struct nt36672a_panel, base);
 }
 
-static void nt36672a_send_cmds(struct mipi_dsi_multi_context *dsi_ctx,
-			       const struct nt36672a_panel_cmd *cmds, int num)
-{
-	unsigned int i;
-
-	for (i = 0; i < num; i++) {
-		const struct nt36672a_panel_cmd *cmd = &cmds[i];
-
-		/* cmd->data[0] is the DCS command, cmd->data[1] is the parameter */
-		mipi_dsi_dcs_write_buffer_multi(dsi_ctx, cmd->data, sizeof(cmd->data));
-	}
-}
-
 static void nt36672a_panel_power_off(struct drm_panel *panel)
 {
 	struct nt36672a_panel *pinfo = to_nt36672a_panel(panel);
@@ -110,8 +89,8 @@ static int nt36672a_panel_unprepare(struct drm_panel *panel)
 	struct mipi_dsi_multi_context dsi_ctx = { .dsi = pinfo->link };
 
 	/* send off cmds */
-	nt36672a_send_cmds(&dsi_ctx, pinfo->desc->off_cmds,
-			   pinfo->desc->num_off_cmds);
+	if (pinfo->desc->send_deinit_cmds)
+		pinfo->desc->send_deinit_cmds(&dsi_ctx);
 
 	/* Reset error to continue with display off even if send_cmds failed */
 	dsi_ctx.accum_err = 0;
@@ -162,8 +141,8 @@ static int nt36672a_panel_prepare(struct drm_panel *panel)
 	dsi_ctx.accum_err = nt36672a_panel_power_on(pinfo);
 
 	/* send first part of init cmds */
-	nt36672a_send_cmds(&dsi_ctx, pinfo->desc->on_cmds_1,
-			   pinfo->desc->num_on_cmds_1);
+	if (pinfo->desc->send_init_cmds_1)
+		pinfo->desc->send_init_cmds_1(&dsi_ctx);
 
 	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
 
@@ -173,8 +152,8 @@ static int nt36672a_panel_prepare(struct drm_panel *panel)
 	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
 
 	/* Send rest of the init cmds */
-	nt36672a_send_cmds(&dsi_ctx, pinfo->desc->on_cmds_2,
-			   pinfo->desc->num_on_cmds_2);
+	if (pinfo->desc->send_init_cmds_2)
+		pinfo->desc->send_init_cmds_2(&dsi_ctx);
 
 	mipi_dsi_msleep(&dsi_ctx, 120);
 
@@ -213,310 +192,184 @@ static const struct drm_panel_funcs panel_funcs = {
 	.get_modes = nt36672a_panel_get_modes,
 };
 
-static const struct nt36672a_panel_cmd tianma_fhd_video_on_cmds_1[] = {
+static void tianma_fhd_video_send_init_cmds_1(struct mipi_dsi_multi_context *dsi_ctx)
+{
+	u8 reg;
+
 	/* skin enhancement mode */
-	{ .data = {0xFF, 0x22} },
-	{ .data = {0x00, 0x40} },
-	{ .data = {0x01, 0xC0} },
-	{ .data = {0x02, 0x40} },
-	{ .data = {0x03, 0x40} },
-	{ .data = {0x04, 0x40} },
-	{ .data = {0x05, 0x40} },
-	{ .data = {0x06, 0x40} },
-	{ .data = {0x07, 0x40} },
-	{ .data = {0x08, 0x40} },
-	{ .data = {0x09, 0x40} },
-	{ .data = {0x0A, 0x40} },
-	{ .data = {0x0B, 0x40} },
-	{ .data = {0x0C, 0x40} },
-	{ .data = {0x0D, 0x40} },
-	{ .data = {0x0E, 0x40} },
-	{ .data = {0x0F, 0x40} },
-	{ .data = {0x10, 0x40} },
-	{ .data = {0x11, 0x50} },
-	{ .data = {0x12, 0x60} },
-	{ .data = {0x13, 0x70} },
-	{ .data = {0x14, 0x58} },
-	{ .data = {0x15, 0x68} },
-	{ .data = {0x16, 0x78} },
-	{ .data = {0x17, 0x77} },
-	{ .data = {0x18, 0x39} },
-	{ .data = {0x19, 0x2D} },
-	{ .data = {0x1A, 0x2E} },
-	{ .data = {0x1B, 0x32} },
-	{ .data = {0x1C, 0x37} },
-	{ .data = {0x1D, 0x3A} },
-	{ .data = {0x1E, 0x40} },
-	{ .data = {0x1F, 0x40} },
-	{ .data = {0x20, 0x40} },
-	{ .data = {0x21, 0x40} },
-	{ .data = {0x22, 0x40} },
-	{ .data = {0x23, 0x40} },
-	{ .data = {0x24, 0x40} },
-	{ .data = {0x25, 0x40} },
-	{ .data = {0x26, 0x40} },
-	{ .data = {0x27, 0x40} },
-	{ .data = {0x28, 0x40} },
-	{ .data = {0x2D, 0x00} },
-	{ .data = {0x2F, 0x40} },
-	{ .data = {0x30, 0x40} },
-	{ .data = {0x31, 0x40} },
-	{ .data = {0x32, 0x40} },
-	{ .data = {0x33, 0x40} },
-	{ .data = {0x34, 0x40} },
-	{ .data = {0x35, 0x40} },
-	{ .data = {0x36, 0x40} },
-	{ .data = {0x37, 0x40} },
-	{ .data = {0x38, 0x40} },
-	{ .data = {0x39, 0x40} },
-	{ .data = {0x3A, 0x40} },
-	{ .data = {0x3B, 0x40} },
-	{ .data = {0x3D, 0x40} },
-	{ .data = {0x3F, 0x40} },
-	{ .data = {0x40, 0x40} },
-	{ .data = {0x41, 0x40} },
-	{ .data = {0x42, 0x40} },
-	{ .data = {0x43, 0x40} },
-	{ .data = {0x44, 0x40} },
-	{ .data = {0x45, 0x40} },
-	{ .data = {0x46, 0x40} },
-	{ .data = {0x47, 0x40} },
-	{ .data = {0x48, 0x40} },
-	{ .data = {0x49, 0x40} },
-	{ .data = {0x4A, 0x40} },
-	{ .data = {0x4B, 0x40} },
-	{ .data = {0x4C, 0x40} },
-	{ .data = {0x4D, 0x40} },
-	{ .data = {0x4E, 0x40} },
-	{ .data = {0x4F, 0x40} },
-	{ .data = {0x50, 0x40} },
-	{ .data = {0x51, 0x40} },
-	{ .data = {0x52, 0x40} },
-	{ .data = {0x53, 0x01} },
-	{ .data = {0x54, 0x01} },
-	{ .data = {0x55, 0xFE} },
-	{ .data = {0x56, 0x77} },
-	{ .data = {0x58, 0xCD} },
-	{ .data = {0x59, 0xD0} },
-	{ .data = {0x5A, 0xD0} },
-	{ .data = {0x5B, 0x50} },
-	{ .data = {0x5C, 0x50} },
-	{ .data = {0x5D, 0x50} },
-	{ .data = {0x5E, 0x50} },
-	{ .data = {0x5F, 0x50} },
-	{ .data = {0x60, 0x50} },
-	{ .data = {0x61, 0x50} },
-	{ .data = {0x62, 0x50} },
-	{ .data = {0x63, 0x50} },
-	{ .data = {0x64, 0x50} },
-	{ .data = {0x65, 0x50} },
-	{ .data = {0x66, 0x50} },
-	{ .data = {0x67, 0x50} },
-	{ .data = {0x68, 0x50} },
-	{ .data = {0x69, 0x50} },
-	{ .data = {0x6A, 0x50} },
-	{ .data = {0x6B, 0x50} },
-	{ .data = {0x6C, 0x50} },
-	{ .data = {0x6D, 0x50} },
-	{ .data = {0x6E, 0x50} },
-	{ .data = {0x6F, 0x50} },
-	{ .data = {0x70, 0x07} },
-	{ .data = {0x71, 0x00} },
-	{ .data = {0x72, 0x00} },
-	{ .data = {0x73, 0x00} },
-	{ .data = {0x74, 0x06} },
-	{ .data = {0x75, 0x0C} },
-	{ .data = {0x76, 0x03} },
-	{ .data = {0x77, 0x09} },
-	{ .data = {0x78, 0x0F} },
-	{ .data = {0x79, 0x68} },
-	{ .data = {0x7A, 0x88} },
-	{ .data = {0x7C, 0x80} },
-	{ .data = {0x7D, 0x80} },
-	{ .data = {0x7E, 0x80} },
-	{ .data = {0x7F, 0x00} },
-	{ .data = {0x80, 0x00} },
-	{ .data = {0x81, 0x00} },
-	{ .data = {0x83, 0x01} },
-	{ .data = {0x84, 0x00} },
-	{ .data = {0x85, 0x80} },
-	{ .data = {0x86, 0x80} },
-	{ .data = {0x87, 0x80} },
-	{ .data = {0x88, 0x40} },
-	{ .data = {0x89, 0x91} },
-	{ .data = {0x8A, 0x98} },
-	{ .data = {0x8B, 0x80} },
-	{ .data = {0x8C, 0x80} },
-	{ .data = {0x8D, 0x80} },
-	{ .data = {0x8E, 0x80} },
-	{ .data = {0x8F, 0x80} },
-	{ .data = {0x90, 0x80} },
-	{ .data = {0x91, 0x80} },
-	{ .data = {0x92, 0x80} },
-	{ .data = {0x93, 0x80} },
-	{ .data = {0x94, 0x80} },
-	{ .data = {0x95, 0x80} },
-	{ .data = {0x96, 0x80} },
-	{ .data = {0x97, 0x80} },
-	{ .data = {0x98, 0x80} },
-	{ .data = {0x99, 0x80} },
-	{ .data = {0x9A, 0x80} },
-	{ .data = {0x9B, 0x80} },
-	{ .data = {0x9C, 0x80} },
-	{ .data = {0x9D, 0x80} },
-	{ .data = {0x9E, 0x80} },
-	{ .data = {0x9F, 0x80} },
-	{ .data = {0xA0, 0x8A} },
-	{ .data = {0xA2, 0x80} },
-	{ .data = {0xA6, 0x80} },
-	{ .data = {0xA7, 0x80} },
-	{ .data = {0xA9, 0x80} },
-	{ .data = {0xAA, 0x80} },
-	{ .data = {0xAB, 0x80} },
-	{ .data = {0xAC, 0x80} },
-	{ .data = {0xAD, 0x80} },
-	{ .data = {0xAE, 0x80} },
-	{ .data = {0xAF, 0x80} },
-	{ .data = {0xB7, 0x76} },
-	{ .data = {0xB8, 0x76} },
-	{ .data = {0xB9, 0x05} },
-	{ .data = {0xBA, 0x0D} },
-	{ .data = {0xBB, 0x14} },
-	{ .data = {0xBC, 0x0F} },
-	{ .data = {0xBD, 0x18} },
-	{ .data = {0xBE, 0x1F} },
-	{ .data = {0xBF, 0x05} },
-	{ .data = {0xC0, 0x0D} },
-	{ .data = {0xC1, 0x14} },
-	{ .data = {0xC2, 0x03} },
-	{ .data = {0xC3, 0x07} },
-	{ .data = {0xC4, 0x0A} },
-	{ .data = {0xC5, 0xA0} },
-	{ .data = {0xC6, 0x55} },
-	{ .data = {0xC7, 0xFF} },
-	{ .data = {0xC8, 0x39} },
-	{ .data = {0xC9, 0x44} },
-	{ .data = {0xCA, 0x12} },
-	{ .data = {0xCD, 0x80} },
-	{ .data = {0xDB, 0x80} },
-	{ .data = {0xDC, 0x80} },
-	{ .data = {0xDD, 0x80} },
-	{ .data = {0xE0, 0x80} },
-	{ .data = {0xE1, 0x80} },
-	{ .data = {0xE2, 0x80} },
-	{ .data = {0xE3, 0x80} },
-	{ .data = {0xE4, 0x80} },
-	{ .data = {0xE5, 0x40} },
-	{ .data = {0xE6, 0x40} },
-	{ .data = {0xE7, 0x40} },
-	{ .data = {0xE8, 0x40} },
-	{ .data = {0xE9, 0x40} },
-	{ .data = {0xEA, 0x40} },
-	{ .data = {0xEB, 0x40} },
-	{ .data = {0xEC, 0x40} },
-	{ .data = {0xED, 0x40} },
-	{ .data = {0xEE, 0x40} },
-	{ .data = {0xEF, 0x40} },
-	{ .data = {0xF0, 0x40} },
-	{ .data = {0xF1, 0x40} },
-	{ .data = {0xF2, 0x40} },
-	{ .data = {0xF3, 0x40} },
-	{ .data = {0xF4, 0x40} },
-	{ .data = {0xF5, 0x40} },
-	{ .data = {0xF6, 0x40} },
-	{ .data = {0xFB, 0x1} },
-	{ .data = {0xFF, 0x23} },
-	{ .data = {0xFB, 0x01} },
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xff, 0x22);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x00, 0x40);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x01, 0xc0);
+	for (reg = 0x02; reg <= 0x10; reg++)
+		mipi_dsi_dcs_write_var_seq_multi(dsi_ctx, reg, 0x40);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x11, 0x50);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x12, 0x60);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x13, 0x70);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x14, 0x58);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x15, 0x68);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x16, 0x78);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x17, 0x77);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x18, 0x39);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x19, 0x2d);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x1a, 0x2e);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x1b, 0x32);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x1c, 0x37);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x1d, 0x3a);
+	for (reg = 0x1e; reg <= 0x28; reg++)
+		mipi_dsi_dcs_write_var_seq_multi(dsi_ctx, reg, 0x40);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x2d, 0x00);
+	for (reg = 0x2f; reg <= 0x3b; reg++)
+		mipi_dsi_dcs_write_var_seq_multi(dsi_ctx, reg, 0x40);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x3d, 0x40);
+	for (reg = 0x3f; reg <= 0x52; reg++)
+		mipi_dsi_dcs_write_var_seq_multi(dsi_ctx, reg, 0x40);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x53, 0x01);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x54, 0x01);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x55, 0xfe);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x56, 0x77);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x58, 0xcd);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x59, 0xd0);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x5a, 0xd0);
+	for (reg = 0x5b; reg <= 0x6f; reg++)
+		mipi_dsi_dcs_write_var_seq_multi(dsi_ctx, reg, 0x50);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x70, 0x07);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x71, 0x00);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x72, 0x00);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x73, 0x00);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x74, 0x06);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x75, 0x0c);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x76, 0x03);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x77, 0x09);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x78, 0x0f);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x79, 0x68);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x7a, 0x88);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x7c, 0x80);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x7d, 0x80);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x7e, 0x80);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x7f, 0x00);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x80, 0x00);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x81, 0x00);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x83, 0x01);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x84, 0x00);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x85, 0x80);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x86, 0x80);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x87, 0x80);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x88, 0x40);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x89, 0x91);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x8a, 0x98);
+	for (reg = 0x8b; reg <= 0x9f; reg++)
+		mipi_dsi_dcs_write_var_seq_multi(dsi_ctx, reg, 0x80);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xa0, 0x8a);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xa2, 0x80);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xa6, 0x80);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xa7, 0x80);
+	for (reg = 0xa9; reg <= 0xaf; reg++)
+		mipi_dsi_dcs_write_var_seq_multi(dsi_ctx, reg, 0x80);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xb7, 0x76);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xb8, 0x76);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xb9, 0x05);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xba, 0x0d);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xbb, 0x14);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xbc, 0x0f);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xbd, 0x18);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xbe, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xbf, 0x05);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xc0, 0x0d);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xc1, 0x14);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xc2, 0x03);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xc3, 0x07);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xc4, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xc5, 0xa0);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xc6, 0x55);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xc7, 0xff);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xc8, 0x39);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xc9, 0x44);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xca, 0x12);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xcd, 0x80);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xdb, 0x80);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xdc, 0x80);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xdd, 0x80);
+	for (reg = 0xe0; reg <= 0xe4; reg++)
+		mipi_dsi_dcs_write_var_seq_multi(dsi_ctx, reg, 0x80);
+	for (reg = 0xe5; reg <= 0xf6; reg++)
+		mipi_dsi_dcs_write_var_seq_multi(dsi_ctx, reg, 0x40);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xff, 0x23);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xfb, 0x01);
 	/* dimming enable */
-	{ .data = {0x01, 0x84} },
-	{ .data = {0x05, 0x2D} },
-	{ .data = {0x06, 0x00} },
-	 /* resolution 1080*2246 */
-	{ .data = {0x11, 0x01} },
-	{ .data = {0x12, 0x7B} },
-	{ .data = {0x15, 0x6F} },
-	{ .data = {0x16, 0x0B} },
-	 /* UI mode */
-	{ .data = {0x29, 0x0A} },
-	{ .data = {0x30, 0xFF} },
-	{ .data = {0x31, 0xFF} },
-	{ .data = {0x32, 0xFF} },
-	{ .data = {0x33, 0xFF} },
-	{ .data = {0x34, 0xFF} },
-	{ .data = {0x35, 0xFF} },
-	{ .data = {0x36, 0xFF} },
-	{ .data = {0x37, 0xFF} },
-	{ .data = {0x38, 0xFC} },
-	{ .data = {0x39, 0xF8} },
-	{ .data = {0x3A, 0xF4} },
-	{ .data = {0x3B, 0xF1} },
-	{ .data = {0x3D, 0xEE} },
-	{ .data = {0x3F, 0xEB} },
-	{ .data = {0x40, 0xE8} },
-	{ .data = {0x41, 0xE5} },
-	 /* STILL mode */
-	{ .data = {0x2A, 0x13} },
-	{ .data = {0x45, 0xFF} },
-	{ .data = {0x46, 0xFF} },
-	{ .data = {0x47, 0xFF} },
-	{ .data = {0x48, 0xFF} },
-	{ .data = {0x49, 0xFF} },
-	{ .data = {0x4A, 0xFF} },
-	{ .data = {0x4B, 0xFF} },
-	{ .data = {0x4C, 0xFF} },
-	{ .data = {0x4D, 0xED} },
-	{ .data = {0x4E, 0xD5} },
-	{ .data = {0x4F, 0xBF} },
-	{ .data = {0x50, 0xA6} },
-	{ .data = {0x51, 0x96} },
-	{ .data = {0x52, 0x86} },
-	{ .data = {0x53, 0x76} },
-	{ .data = {0x54, 0x66} },
-	 /* MOVING mode */
-	{ .data = {0x2B, 0x0E} },
-	{ .data = {0x58, 0xFF} },
-	{ .data = {0x59, 0xFF} },
-	{ .data = {0x5A, 0xFF} },
-	{ .data = {0x5B, 0xFF} },
-	{ .data = {0x5C, 0xFF} },
-	{ .data = {0x5D, 0xFF} },
-	{ .data = {0x5E, 0xFF} },
-	{ .data = {0x5F, 0xFF} },
-	{ .data = {0x60, 0xF6} },
-	{ .data = {0x61, 0xEA} },
-	{ .data = {0x62, 0xE1} },
-	{ .data = {0x63, 0xD8} },
-	{ .data = {0x64, 0xCE} },
-	{ .data = {0x65, 0xC3} },
-	{ .data = {0x66, 0xBA} },
-	{ .data = {0x67, 0xB3} },
-	{ .data = {0xFF, 0x25} },
-	{ .data = {0xFB, 0x01} },
-	{ .data = {0x05, 0x04} },
-	{ .data = {0xFF, 0x26} },
-	{ .data = {0xFB, 0x01} },
-	{ .data = {0x1C, 0xAF} },
-	{ .data = {0xFF, 0x10} },
-	{ .data = {0xFB, 0x01} },
-	{ .data = {0x51, 0xFF} },
-	{ .data = {0x53, 0x24} },
-	{ .data = {0x55, 0x00} },
-};
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x01, 0x84);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x05, 0x2d);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x06, 0x00);
+	/* resolution 1080*2246 */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x11, 0x01);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x12, 0x7b);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x15, 0x6f);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x16, 0x0b);
+	/* UI mode */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x29, 0x0a);
+	for (reg = 0x30; reg <= 0x37; reg++)
+		mipi_dsi_dcs_write_var_seq_multi(dsi_ctx, reg, 0xff);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x38, 0xfc);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x39, 0xf8);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x3a, 0xf4);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x3b, 0xf1);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x3d, 0xee);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x3f, 0xeb);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x40, 0xe8);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x41, 0xe5);
+	/* STILL mode */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x2a, 0x13);
+	for (reg = 0x45; reg <= 0x4c; reg++)
+		mipi_dsi_dcs_write_var_seq_multi(dsi_ctx, reg, 0xff);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x4d, 0xed);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x4e, 0xd5);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x4f, 0xbf);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x50, 0xa6);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x51, 0x96);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x52, 0x86);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x53, 0x76);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x54, 0x66);
+	/* MOVING mode */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x2b, 0x0e);
+	for (reg = 0x58; reg <= 0x5f; reg++)
+		mipi_dsi_dcs_write_var_seq_multi(dsi_ctx, reg, 0xff);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x60, 0xf6);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x61, 0xea);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x62, 0xe1);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x63, 0xd8);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x64, 0xce);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x65, 0xc3);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x66, 0xba);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x67, 0xb3);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xff, 0x25);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x05, 0x04);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xff, 0x26);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x1c, 0xaf);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xff, 0x10);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x51, 0xff);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x53, 0x24);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x55, 0x00);
+}
 
-static const struct nt36672a_panel_cmd tianma_fhd_video_on_cmds_2[] = {
-	{ .data = {0xFF, 0x24} },
-	{ .data = {0xFB, 0x01} },
-	{ .data = {0xC3, 0x01} },
-	{ .data = {0xC4, 0x54} },
-	{ .data = {0xFF, 0x10} },
-};
+static void tianma_fhd_video_send_init_cmds_2(struct mipi_dsi_multi_context *dsi_ctx)
+{
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xff, 0x24);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xc3, 0x01);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xc4, 0x54);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xff, 0x10);
+}
 
-static const struct nt36672a_panel_cmd tianma_fhd_video_off_cmds[] = {
-	{ .data = {0xFF, 0x24} },
-	{ .data = {0xFB, 0x01} },
-	{ .data = {0xC3, 0x01} },
-	{ .data = {0xFF, 0x10} },
-};
+static void tianma_fhd_video_send_deinit_cmds(struct mipi_dsi_multi_context *dsi_ctx)
+{
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xff, 0x24);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xc3, 0x01);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xff, 0x10);
+}
 
 static const struct drm_display_mode tianma_fhd_video_panel_default_mode = {
 	.clock		= 161331,
@@ -546,12 +399,9 @@ static const struct nt36672a_panel_desc tianma_fhd_video_panel_desc = {
 			| MIPI_DSI_MODE_VIDEO_BURST,
 	.format = MIPI_DSI_FMT_RGB888,
 	.lanes = 4,
-	.on_cmds_1 = tianma_fhd_video_on_cmds_1,
-	.num_on_cmds_1 = ARRAY_SIZE(tianma_fhd_video_on_cmds_1),
-	.on_cmds_2 = tianma_fhd_video_on_cmds_2,
-	.num_on_cmds_2 = ARRAY_SIZE(tianma_fhd_video_on_cmds_2),
-	.off_cmds = tianma_fhd_video_off_cmds,
-	.num_off_cmds = ARRAY_SIZE(tianma_fhd_video_off_cmds),
+	.send_init_cmds_1 = tianma_fhd_video_send_init_cmds_1,
+	.send_init_cmds_2 = tianma_fhd_video_send_init_cmds_2,
+	.send_deinit_cmds = tianma_fhd_video_send_deinit_cmds,
 };
 
 static int nt36672a_panel_add(struct nt36672a_panel *pinfo)
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt37801.c b/drivers/gpu/drm/panel/panel-novatek-nt37801.c
index 861e999250f97..5e46c4c6d7716 100644
--- a/drivers/gpu/drm/panel/panel-novatek-nt37801.c
+++ b/drivers/gpu/drm/panel/panel-novatek-nt37801.c
@@ -51,61 +51,56 @@ static void novatek_nt37801_reset(struct novatek_nt37801 *ctx)
 	mipi_dsi_dcs_write_seq_multi((dsi_ctx), NT37801_DCS_SWITCH_PAGE, \
 				     0x55, 0xaa, 0x52, 0x08, (page))
 
-static int novatek_nt37801_on(struct novatek_nt37801 *ctx)
+static void novatek_nt37801_on(struct mipi_dsi_multi_context *dsi_ctx)
 {
-	struct mipi_dsi_device *dsi = ctx->dsi;
-	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
-
-	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
-
-	novatek_nt37801_switch_page(&dsi_ctx, 0x01);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x01);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xc5, 0x0b, 0x0b, 0x0b);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0xaa, 0x55, 0xa5, 0x80);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x02);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xf5, 0x10);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x1b);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xf4, 0x55);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x18);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xf8, 0x19);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x0f);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xfc, 0x00);
-	mipi_dsi_dcs_set_column_address_multi(&dsi_ctx, 0x0000, 0x059f);
-	mipi_dsi_dcs_set_page_address_multi(&dsi_ctx, 0x0000, 0x0c7f);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x90, 0x03, 0x03);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x91,
+	dsi_ctx->dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	novatek_nt37801_switch_page(dsi_ctx, 0x01);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x6f, 0x01);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xc5, 0x0b, 0x0b, 0x0b);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xff, 0xaa, 0x55, 0xa5, 0x80);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x6f, 0x02);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xf5, 0x10);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x6f, 0x1b);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xf4, 0x55);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x6f, 0x18);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xf8, 0x19);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x6f, 0x0f);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xfc, 0x00);
+	mipi_dsi_dcs_set_column_address_multi(dsi_ctx, 0x0000, 0x059f);
+	mipi_dsi_dcs_set_page_address_multi(dsi_ctx, 0x0000, 0x0c7f);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x90, 0x03, 0x03);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x91,
 				     0x89, 0x28, 0x00, 0x28, 0xc2, 0x00, 0x02,
 				     0x68, 0x04, 0x6c, 0x00, 0x0a, 0x02, 0x77,
 				     0x01, 0xe9, 0x10, 0xf0);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0xaa, 0x55, 0xa5, 0x81);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x23);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xfb,
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xff, 0xaa, 0x55, 0xa5, 0x81);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x6f, 0x23);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xfb,
 				     0x00, 0x01, 0x00, 0x11, 0x33, 0x33, 0x33,
 				     0x55, 0x57, 0xd0, 0x00, 0x00, 0x44, 0x56,
 				     0x77, 0x78, 0x9a, 0xbc, 0xdd, 0xf0);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x06);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xf3, 0xdc);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_SET_GAMMA_CURVE, 0x00);
-	mipi_dsi_dcs_set_tear_on_multi(&dsi_ctx, MIPI_DSI_DCS_TEAR_MODE_VBLANK);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3b, 0x00, 0x18, 0x00, 0x10);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_WRITE_CONTROL_DISPLAY,
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x6f, 0x06);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xf3, 0xdc);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, MIPI_DCS_SET_GAMMA_CURVE, 0x00);
+	mipi_dsi_dcs_set_tear_on_multi(dsi_ctx, MIPI_DSI_DCS_TEAR_MODE_VBLANK);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x3b, 0x00, 0x18, 0x00, 0x10);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, MIPI_DCS_WRITE_CONTROL_DISPLAY,
 				     0x20);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x51,
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x51,
 				     0x07, 0xff, 0x07, 0xff, 0x0f, 0xff);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5a, 0x01);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5f, 0x00);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x9c, 0x01);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_WRITE_MEMORY_START);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2f, 0x00);
-
-	novatek_nt37801_switch_page(&dsi_ctx, 0x01);
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb2, 0x55, 0x01, 0xff, 0x03);
-	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
-	mipi_dsi_msleep(&dsi_ctx, 120);
-	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
-	mipi_dsi_msleep(&dsi_ctx, 20);
-
-	return dsi_ctx.accum_err;
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x5a, 0x01);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x5f, 0x00);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x9c, 0x01);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, MIPI_DCS_WRITE_MEMORY_START);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x2f, 0x00);
+
+	novatek_nt37801_switch_page(dsi_ctx, 0x01);
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xb2, 0x55, 0x01, 0xff, 0x03);
+	mipi_dsi_dcs_exit_sleep_mode_multi(dsi_ctx);
+	mipi_dsi_msleep(dsi_ctx, 120);
+	mipi_dsi_dcs_set_display_on_multi(dsi_ctx);
+	mipi_dsi_msleep(dsi_ctx, 20);
 }
 
 static int novatek_nt37801_off(struct novatek_nt37801 *ctx)
@@ -127,7 +122,8 @@ static int novatek_nt37801_off(struct novatek_nt37801 *ctx)
 static int novatek_nt37801_prepare(struct drm_panel *panel)
 {
 	struct novatek_nt37801 *ctx = to_novatek_nt37801(panel);
-	struct device *dev = &ctx->dsi->dev;
+	struct mipi_dsi_device *dsi = ctx->dsi;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
 	struct drm_dsc_picture_parameter_set pps;
 	int ret;
 
@@ -137,22 +133,12 @@ static int novatek_nt37801_prepare(struct drm_panel *panel)
 		return ret;
 
 	novatek_nt37801_reset(ctx);
-
-	ret = novatek_nt37801_on(ctx);
-	if (ret < 0)
-		goto err;
-
+	novatek_nt37801_on(&dsi_ctx);
 	drm_dsc_pps_payload_pack(&pps, &ctx->dsc);
-
-	ret = mipi_dsi_picture_parameter_set(ctx->dsi, &pps);
-	if (ret < 0) {
-		dev_err(panel->dev, "failed to transmit PPS: %d\n", ret);
-		goto err;
-	}
-
-	ret = mipi_dsi_compression_mode(ctx->dsi, true);
-	if (ret < 0) {
-		dev_err(dev, "failed to enable compression mode: %d\n", ret);
+	mipi_dsi_picture_parameter_set_multi(&dsi_ctx, &pps);
+	mipi_dsi_compression_mode_multi(&dsi_ctx, true);
+	if (dsi_ctx.accum_err) {
+		ret = dsi_ctx.accum_err;
 		goto err;
 	}
 
diff --git a/drivers/gpu/drm/panel/panel-renesas-r63419.c b/drivers/gpu/drm/panel/panel-renesas-r63419.c
new file mode 100644
index 0000000000000..ff3846d574e58
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-renesas-r63419.c
@@ -0,0 +1,351 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * DRM driver for Renesas R63419 based dual-DSI video mode panels
+ *
+ * Copyright (c) 2025, Kancy Joe <kancy2333@outlook.com>
+ * Copyright (C) 2026 Linaro Limited
+ * Author: Neil Armstrong <neil.armstrong@linaro.org>
+ */
+
+#include <linux/backlight.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+#include <linux/regulator/consumer.h>
+
+#include <video/mipi_display.h>
+
+#include <drm/drm_connector.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_of.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_probe_helper.h>
+
+struct renesas_r63419_panel {
+	struct drm_panel panel;
+	struct mipi_dsi_device *dsi[2];
+	const struct panel_desc *desc;
+
+	struct gpio_desc *reset_gpio;
+	struct regulator_bulk_data *vdd_supplies;
+	struct regulator_bulk_data *vcc_supplies;
+	enum drm_panel_orientation orientation;
+};
+
+/* VDDIO/VDD Supplies */
+static const struct regulator_bulk_data renesas_r63419_vdd_supplies[] = {
+	{ .supply = "vddio" },
+	{ .supply = "vdd" },
+};
+
+/* VSP/VSN/VCI Supplies */
+static const struct regulator_bulk_data renesas_r63419_vcc_supplies[] = {
+	{ .supply = "vsp" },
+	{ .supply = "vsn" },
+	{ .supply = "vci" },
+};
+
+struct panel_desc {
+	const struct drm_display_mode *mode;
+	unsigned int lanes;
+	unsigned long mode_flags;
+	enum mipi_dsi_pixel_format format;
+	const struct mipi_dsi_device_info dsi_info;
+};
+
+static const struct drm_display_mode wt0600_mode = {
+	/* Dual dsi */
+	.clock = 2 * (720 + 100 + 8 + 40) * (2560 + 15 + 2 + 8) * 60 / 1000,
+	.hdisplay = 2 * 720,
+	.hsync_start = 2 * (720 + 100),
+	.hsync_end = 2 * (720 + 100 + 8),
+	.htotal = 2 * (720 + 100 + 8 + 40),
+	.vdisplay = 2560,
+	.vsync_start = 2560 + 15,
+	.vsync_end = 2560 + 15 + 2,
+	.vtotal = 2560 + 15 + 2 + 8,
+	.type = DRM_MODE_TYPE_DRIVER,
+	.width_mm = 74,
+	.height_mm = 131,
+};
+
+static const struct drm_display_mode wt0630_mode = {
+	/* Dual dsi */
+	.clock = 2 * (720 + 100 + 8 + 40) * (2560 + 15 + 2 + 8) * 60 / 1000,
+	.hdisplay = 2 * 720,
+	.hsync_start = 2 * (720 + 100),
+	.hsync_end = 2 * (720 + 100 + 8),
+	.htotal = 2 * (720 + 100 + 8 + 40),
+	.vdisplay = 2560,
+	.vsync_start = 2560 + 15,
+	.vsync_end = 2560 + 15 + 2,
+	.vtotal = 2560 + 15 + 2 + 8,
+	.type = DRM_MODE_TYPE_DRIVER,
+	.width_mm = 78,
+	.height_mm = 140,
+};
+
+static struct panel_desc wt0600_desc = {
+	.lanes = 4,
+	.mode = &wt0600_mode,
+	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
+		      MIPI_DSI_CLOCK_NON_CONTINUOUS | MIPI_DSI_MODE_LPM,
+	.format = MIPI_DSI_FMT_RGB888,
+};
+
+static struct panel_desc wt0630_desc = {
+	.lanes = 4,
+	.mode = &wt0630_mode,  /* wt0600 only has different screen size */
+	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
+		      MIPI_DSI_CLOCK_NON_CONTINUOUS | MIPI_DSI_MODE_LPM,
+	.format = MIPI_DSI_FMT_RGB888,
+};
+
+static inline struct renesas_r63419_panel *
+to_renesas_r63419_panel(struct drm_panel *panel)
+{
+	return container_of(panel, struct renesas_r63419_panel, panel);
+}
+
+static int renesas_r63419_on(struct renesas_r63419_panel *ctx)
+{
+	struct mipi_dsi_multi_context dsi_ctx = { 0 };
+
+	/*
+	 * Panel registers are loaded from DDIC Non Volatile Memory
+	 *
+	 * The DDIC expects this sequence to get out of sleep and enable display
+	 */
+
+	mipi_dsi_dual(mipi_dsi_dcs_set_display_on_multi,
+		      &dsi_ctx, ctx->dsi[0], ctx->dsi[1]);
+	mipi_dsi_msleep(&dsi_ctx, 50);
+
+	mipi_dsi_dual(mipi_dsi_dcs_exit_sleep_mode_multi,
+		      &dsi_ctx, ctx->dsi[0], ctx->dsi[1]);
+	mipi_dsi_msleep(&dsi_ctx, 150);
+
+	return dsi_ctx.accum_err;
+}
+
+static int renesas_r63419_disable(struct drm_panel *panel)
+{
+	struct renesas_r63419_panel *ctx = to_renesas_r63419_panel(panel);
+	struct mipi_dsi_multi_context dsi_ctx = { 0 };
+
+	mipi_dsi_dual(mipi_dsi_dcs_set_display_off_multi,
+		      &dsi_ctx, ctx->dsi[0], ctx->dsi[1]);
+	mipi_dsi_msleep(&dsi_ctx, 50);
+
+	mipi_dsi_dual(mipi_dsi_dcs_enter_sleep_mode_multi,
+		      &dsi_ctx, ctx->dsi[0], ctx->dsi[1]);
+	mipi_dsi_msleep(&dsi_ctx, 120);
+
+	return 0;
+}
+
+static int renesas_r63419_prepare(struct drm_panel *panel)
+{
+	struct renesas_r63419_panel *ctx = to_renesas_r63419_panel(panel);
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(renesas_r63419_vdd_supplies),
+				    ctx->vdd_supplies);
+	if (ret < 0)
+		return ret;
+
+	usleep_range(1000, 2000);
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(renesas_r63419_vcc_supplies),
+				    ctx->vcc_supplies);
+	if (ret < 0) {
+		regulator_bulk_disable(ARRAY_SIZE(renesas_r63419_vdd_supplies),
+				       ctx->vdd_supplies);
+		return ret;
+	}
+
+	usleep_range(1000, 2000);
+
+	gpiod_set_value_cansleep(ctx->reset_gpio, 0);
+
+	usleep_range(3000, 4000);
+
+	ret = renesas_r63419_on(ctx);
+	if (ret < 0) {
+		dev_err(panel->dev, "Failed to initialize panel: %d\n", ret);
+
+		/* Power off sequence from the r63419 datasheet */
+		regulator_bulk_disable(ARRAY_SIZE(renesas_r63419_vdd_supplies),
+				       ctx->vdd_supplies);
+
+		gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+
+		regulator_bulk_disable(ARRAY_SIZE(renesas_r63419_vcc_supplies),
+				       ctx->vcc_supplies);
+
+		return ret;
+	}
+
+	return 0;
+}
+
+static int renesas_r63419_unprepare(struct drm_panel *panel)
+{
+	struct renesas_r63419_panel *ctx = to_renesas_r63419_panel(panel);
+
+	/* Power off sequence from the r63419 datasheet */
+	regulator_bulk_disable(ARRAY_SIZE(renesas_r63419_vdd_supplies), ctx->vdd_supplies);
+
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+
+	regulator_bulk_disable(ARRAY_SIZE(renesas_r63419_vcc_supplies), ctx->vcc_supplies);
+
+	return 0;
+}
+
+static int renesas_r63419_get_modes(struct drm_panel *panel,
+				    struct drm_connector *connector)
+{
+	struct renesas_r63419_panel *ctx = to_renesas_r63419_panel(panel);
+	const struct drm_display_mode *mode = ctx->desc->mode;
+
+	return drm_connector_helper_get_modes_fixed(connector, mode);
+}
+
+static enum drm_panel_orientation
+renesas_r63419_get_orientation(struct drm_panel *panel)
+{
+	struct renesas_r63419_panel *ctx = to_renesas_r63419_panel(panel);
+
+	return ctx->orientation;
+}
+
+static const struct drm_panel_funcs renesas_r63419_panel_funcs = {
+	.disable = renesas_r63419_disable,
+	.prepare = renesas_r63419_prepare,
+	.unprepare = renesas_r63419_unprepare,
+	.get_modes = renesas_r63419_get_modes,
+	.get_orientation = renesas_r63419_get_orientation,
+};
+
+static int renesas_r63419_probe(struct mipi_dsi_device *dsi)
+{
+	struct mipi_dsi_device_info info = { 0 };
+	struct device *dev = &dsi->dev;
+	struct renesas_r63419_panel *ctx;
+	struct device_node *dsi1_node;
+	struct mipi_dsi_host *dsi1_host;
+	int ret, i;
+
+	ctx = devm_drm_panel_alloc(dev, struct renesas_r63419_panel, panel,
+				   &renesas_r63419_panel_funcs, DRM_MODE_CONNECTOR_DSI);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+
+	ctx->desc = of_device_get_match_data(dev);
+	if (!ctx->desc)
+		return dev_err_probe(dev, -ENODEV,
+				     "Failed to get panel description\n");
+
+	ret = devm_regulator_bulk_get_const(&dsi->dev,
+					    ARRAY_SIZE(renesas_r63419_vdd_supplies),
+					    renesas_r63419_vdd_supplies, &ctx->vdd_supplies);
+	if (ret < 0)
+		return ret;
+
+	ret = devm_regulator_bulk_get_const(&dsi->dev,
+					    ARRAY_SIZE(renesas_r63419_vcc_supplies),
+					    renesas_r63419_vcc_supplies, &ctx->vcc_supplies);
+	if (ret < 0)
+		return ret;
+
+	ctx->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(ctx->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(ctx->reset_gpio),
+				     "Failed to get reset gpio\n");
+
+	/* Get second DSI host */
+	dsi1_node = of_graph_get_remote_node(dsi->dev.of_node, 1, -1);
+	if (!dsi1_node)
+		return dev_err_probe(dev, -ENODEV,
+				     "Failed to get remote node for second DSI\n");
+
+	dsi1_host = of_find_mipi_dsi_host_by_node(dsi1_node);
+	of_node_put(dsi1_node);
+	if (!dsi1_host)
+		return dev_err_probe(dev, -EPROBE_DEFER,
+				     "Failed to find second DSI host\n");
+
+	/* Copy current DSI info, do not provide OF node since no driver needs to be attached */
+	strscpy(info.type, dsi->name);
+	info.channel = dsi->channel;
+
+	/* Register the second DSI device */
+	ctx->dsi[1] = devm_mipi_dsi_device_register_full(dev, dsi1_host, &info);
+	if (IS_ERR(ctx->dsi[1]))
+		return dev_err_probe(dev, PTR_ERR(ctx->dsi[1]),
+				     "Failed to register second DSI device\n");
+
+	ctx->dsi[0] = dsi;
+	mipi_dsi_set_drvdata(dsi, ctx);
+
+	/* Get panel orientation */
+	ret = drm_of_get_panel_orientation(dev->of_node, &ctx->orientation);
+	if (ret < 0 && ret != -ENODEV)
+		return dev_err_probe(dev, ret,
+				     "Failed to get panel orientation\n");
+
+	ctx->panel.prepare_prev_first = true;
+
+	ret = drm_panel_of_backlight(&ctx->panel);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to get backlight\n");
+
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to add panel\n");
+
+	/* Configure and attach both DSI devices */
+	for (i = 0; i < ARRAY_SIZE(ctx->dsi); i++) {
+		ctx->dsi[i]->lanes = ctx->desc->lanes;
+		ctx->dsi[i]->format = ctx->desc->format;
+		ctx->dsi[i]->mode_flags = ctx->desc->mode_flags;
+
+		ret = devm_mipi_dsi_attach(dev, ctx->dsi[i]);
+		if (ret < 0)
+			return dev_err_probe(dev, ret,
+					     "Failed to attach DSI device %d\n", i);
+	}
+
+	return 0;
+}
+
+static const struct of_device_id renesas_r63419_of_match[] = {
+	{
+		.compatible = "ayaneo,wt0600-2k",
+		.data = &wt0600_desc,
+	},
+	{
+		.compatible = "ayaneo,wt0630-2k",
+		.data = &wt0630_desc,
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, renesas_r63419_of_match);
+
+static struct mipi_dsi_driver renesas_r63419_driver = {
+	.probe = renesas_r63419_probe,
+	.driver = {
+		.name = "panel-renesas-r63419",
+		.of_match_table = renesas_r63419_of_match,
+	},
+};
+module_mipi_dsi_driver(renesas_r63419_driver);
+
+MODULE_AUTHOR("Kancy Joe <kancy2333@outlook.com>");
+MODULE_AUTHOR("Neil Armstrong <neil.armstrong@linaro.org>");
+MODULE_DESCRIPTION("DRM driver for Renesas R63419 based dual-DSI video mode panels");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c b/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c
index 1b14aa4efe359..54a65abf7e890 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c
@@ -88,16 +88,22 @@ static int s6d16d0_prepare(struct drm_panel *panel)
 				       MIPI_DSI_DCS_TEAR_MODE_VBLANK);
 	if (ret) {
 		dev_err(s6->dev, "failed to enable vblank TE (%d)\n", ret);
-		return ret;
+		goto err_power_off;
 	}
 	/* Exit sleep mode and power on */
 	ret = mipi_dsi_dcs_exit_sleep_mode(dsi);
 	if (ret) {
 		dev_err(s6->dev, "failed to exit sleep mode (%d)\n", ret);
-		return ret;
+		goto err_power_off;
 	}
 
 	return 0;
+
+err_power_off:
+	gpiod_set_value_cansleep(s6->reset_gpio, 1);
+	regulator_disable(s6->supply);
+
+	return ret;
 }
 
 static int s6d16d0_enable(struct drm_panel *panel)
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index c09bf3db5e78a..5c169fdf265c1 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -693,7 +693,7 @@ static struct panel_simple *panel_simple_probe(struct device *dev)
 		return dev_err_cast_probe(dev, panel->enable_gpio,
 					  "failed to request GPIO\n");
 
-	err = of_drm_get_panel_orientation(dev->of_node, &panel->orientation);
+	err = drm_of_get_panel_orientation(dev->of_node, &panel->orientation);
 	if (err) {
 		dev_err(dev, "%pOF: failed to get orientation %d\n", dev->of_node, err);
 		return ERR_PTR(err);
diff --git a/drivers/gpu/drm/panel/panel-sitronix-st7701.c b/drivers/gpu/drm/panel/panel-sitronix-st7701.c
index 2f79ec4a20632..f16e0de1ea606 100644
--- a/drivers/gpu/drm/panel/panel-sitronix-st7701.c
+++ b/drivers/gpu/drm/panel/panel-sitronix-st7701.c
@@ -7,6 +7,7 @@
 #include <drm/drm_mipi_dbi.h>
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_modes.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 
 #include <linux/bitfield.h>
@@ -1296,7 +1297,7 @@ static int st7701_probe(struct device *dev, int connector_type)
 		return PTR_ERR(st7701->reset);
 	}
 
-	ret = of_drm_get_panel_orientation(dev->of_node, &st7701->orientation);
+	ret = drm_of_get_panel_orientation(dev->of_node, &st7701->orientation);
 	if (ret < 0)
 		return dev_err_probe(dev, ret, "Failed to get orientation\n");
 
diff --git a/drivers/gpu/drm/panel/panel-sitronix-st7703.c b/drivers/gpu/drm/panel/panel-sitronix-st7703.c
index f1641c9c7d132..fcbb0b7d50e4b 100644
--- a/drivers/gpu/drm/panel/panel-sitronix-st7703.c
+++ b/drivers/gpu/drm/panel/panel-sitronix-st7703.c
@@ -20,6 +20,7 @@
 
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_modes.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 
 #define DRV_NAME "panel-sitronix-st7703"
@@ -873,7 +874,7 @@ static int st7703_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(dev, PTR_ERR(ctx->iovcc),
 				     "Failed to request iovcc regulator\n");
 
-	ret = of_drm_get_panel_orientation(dsi->dev.of_node, &ctx->orientation);
+	ret = drm_of_get_panel_orientation(dsi->dev.of_node, &ctx->orientation);
 	if (ret < 0)
 		return dev_err_probe(&dsi->dev, ret, "Failed to get orientation\n");
 
diff --git a/drivers/gpu/drm/panel/panel-sitronix-st7789v.c b/drivers/gpu/drm/panel/panel-sitronix-st7789v.c
index d5f821d6b23cb..da6ed31d151e0 100644
--- a/drivers/gpu/drm/panel/panel-sitronix-st7789v.c
+++ b/drivers/gpu/drm/panel/panel-sitronix-st7789v.c
@@ -14,6 +14,7 @@
 
 #include <drm/drm_device.h>
 #include <drm/drm_modes.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 
 #define ST7789V_RAMCTRL_CMD		0xb0
@@ -646,7 +647,7 @@ static int st7789v_probe(struct spi_device *spi)
 	if (ret)
 		return dev_err_probe(dev, ret, "Failed to get backlight\n");
 
-	ret = of_drm_get_panel_orientation(spi->dev.of_node, &ctx->orientation);
+	ret = drm_of_get_panel_orientation(spi->dev.of_node, &ctx->orientation);
 	if (ret)
 		return dev_err_probe(&spi->dev, ret, "Failed to get orientation\n");
 
diff --git a/drivers/gpu/drm/panel/panel-visionox-vtdr6130.c b/drivers/gpu/drm/panel/panel-visionox-vtdr6130.c
index 97a79411e1ecd..6733e0e1d0612 100644
--- a/drivers/gpu/drm/panel/panel-visionox-vtdr6130.c
+++ b/drivers/gpu/drm/panel/panel-visionox-vtdr6130.c
@@ -9,6 +9,7 @@
 #include <linux/of.h>
 
 #include <drm/display/drm_dsc.h>
+#include <drm/display/drm_dsc_helper.h>
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_modes.h>
 #include <drm/drm_panel.h>
@@ -17,6 +18,7 @@
 
 struct visionox_vtdr6130 {
 	struct drm_panel panel;
+	struct drm_dsc_config dsc;
 	struct mipi_dsi_device *dsi;
 	struct gpio_desc *reset_gpio;
 	struct regulator_bulk_data *supplies;
@@ -47,9 +49,12 @@ static int visionox_vtdr6130_on(struct visionox_vtdr6130 *ctx)
 {
 	struct mipi_dsi_device *dsi = ctx->dsi;
 	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
+	struct drm_dsc_picture_parameter_set pps;
 
 	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
 
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x03, 0x01);
+
 	mipi_dsi_dcs_set_tear_on_multi(&dsi_ctx, MIPI_DSI_DCS_TEAR_MODE_VBLANK);
 
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx,
@@ -123,6 +128,9 @@ static int visionox_vtdr6130_on(struct visionox_vtdr6130 *ctx)
 	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
 	mipi_dsi_msleep(&dsi_ctx, 20);
 
+	drm_dsc_pps_payload_pack(&pps, dsi->dsc);
+	mipi_dsi_picture_parameter_set_multi(&dsi_ctx, &pps);
+
 	return dsi_ctx.accum_err;
 }
 
@@ -269,6 +277,16 @@ static int visionox_vtdr6130_probe(struct mipi_dsi_device *dsi)
 	ctx->dsi = dsi;
 	mipi_dsi_set_drvdata(dsi, ctx);
 
+	ctx->dsc.dsc_version_major = 0x1;
+	ctx->dsc.dsc_version_minor = 0x2;
+	ctx->dsc.slice_height = 40;
+	ctx->dsc.slice_width = 540;
+	ctx->dsc.slice_count = 2;
+	ctx->dsc.bits_per_component = 8;
+	ctx->dsc.bits_per_pixel = 8 << 4;
+	ctx->dsc.block_pred_enable = true;
+
+	dsi->dsc = &ctx->dsc;
 	dsi->lanes = 4;
 	dsi->format = MIPI_DSI_FMT_RGB888;
 	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_NO_EOT_PACKET |
@@ -280,28 +298,11 @@ static int visionox_vtdr6130_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(dev, PTR_ERR(ctx->panel.backlight),
 				     "Failed to create backlight\n");
 
-	drm_panel_add(&ctx->panel);
-
-	ret = mipi_dsi_attach(dsi);
-	if (ret < 0) {
-		dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
-		drm_panel_remove(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
 		return ret;
-	}
-
-	return 0;
-}
-
-static void visionox_vtdr6130_remove(struct mipi_dsi_device *dsi)
-{
-	struct visionox_vtdr6130 *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
 
-	drm_panel_remove(&ctx->panel);
+	return devm_mipi_dsi_attach(dev, dsi);
 }
 
 static const struct of_device_id visionox_vtdr6130_of_match[] = {
@@ -312,7 +313,6 @@ MODULE_DEVICE_TABLE(of, visionox_vtdr6130_of_match);
 
 static struct mipi_dsi_driver visionox_vtdr6130_driver = {
 	.probe = visionox_vtdr6130_probe,
-	.remove = visionox_vtdr6130_remove,
 	.driver = {
 		.name = "panel-visionox-vtdr6130",
 		.of_match_table = visionox_vtdr6130_of_match,
diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c
index 784e36d72c2ba..36cc2e67a3080 100644
--- a/drivers/gpu/drm/panfrost/panfrost_drv.c
+++ b/drivers/gpu/drm/panfrost/panfrost_drv.c
@@ -21,6 +21,7 @@
 #include <drm/drm_utils.h>
 
 #include "panfrost_device.h"
+#include "panfrost_drv.h"
 #include "panfrost_gem.h"
 #include "panfrost_mmu.h"
 #include "panfrost_job.h"
diff --git a/drivers/gpu/drm/panthor/panthor_device.c b/drivers/gpu/drm/panthor/panthor_device.c
index 0b25abebb803c..9687c59de3505 100644
--- a/drivers/gpu/drm/panthor/panthor_device.c
+++ b/drivers/gpu/drm/panthor/panthor_device.c
@@ -182,7 +182,10 @@ int panthor_device_init(struct panthor_device *ptdev)
 		return ret;
 
 #ifdef CONFIG_DEBUG_FS
-	drmm_mutex_init(&ptdev->base, &ptdev->gems.lock);
+	ret = drmm_mutex_init(&ptdev->base, &ptdev->gems.lock);
+	if (ret)
+		return ret;
+
 	INIT_LIST_HEAD(&ptdev->gems.node);
 #endif
 
diff --git a/drivers/gpu/drm/panthor/panthor_device.h b/drivers/gpu/drm/panthor/panthor_device.h
index 98828e81db0b4..0fda64fbe5f22 100644
--- a/drivers/gpu/drm/panthor/panthor_device.h
+++ b/drivers/gpu/drm/panthor/panthor_device.h
@@ -161,6 +161,9 @@ struct panthor_device {
 	/** @csif_info: Command stream interface information. */
 	struct drm_panthor_csif_info csif_info;
 
+	/** @mmu_info: MMU info */
+	struct drm_panthor_mmu_info mmu_info;
+
 	/** @hw: GPU-specific data. */
 	struct panthor_hw *hw;
 
diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c
index e8dc4096c1d25..106da676fd2ee 100644
--- a/drivers/gpu/drm/panthor/panthor_drv.c
+++ b/drivers/gpu/drm/panthor/panthor_drv.c
@@ -175,6 +175,7 @@ panthor_get_uobj_array(const struct drm_panthor_obj_array *in, u32 min_stride,
 	_Generic(_obj_name, \
 		 PANTHOR_UOBJ_DECL(struct drm_panthor_gpu_info, tiler_present), \
 		 PANTHOR_UOBJ_DECL(struct drm_panthor_csif_info, pad), \
+		 PANTHOR_UOBJ_DECL(struct drm_panthor_mmu_info, page_size_bitmap), \
 		 PANTHOR_UOBJ_DECL(struct drm_panthor_timestamp_info, current_timestamp), \
 		 PANTHOR_UOBJ_DECL(struct drm_panthor_group_priorities_info, pad), \
 		 PANTHOR_UOBJ_DECL(struct drm_panthor_sync_op, timeline_value), \
@@ -954,6 +955,10 @@ static int panthor_ioctl_dev_query(struct drm_device *ddev, void *data, struct d
 			args->size = sizeof(priorities_info);
 			return 0;
 
+		case DRM_PANTHOR_DEV_QUERY_MMU_INFO:
+			args->size = sizeof(ptdev->mmu_info);
+			return 0;
+
 		default:
 			return -EINVAL;
 		}
@@ -984,6 +989,9 @@ static int panthor_ioctl_dev_query(struct drm_device *ddev, void *data, struct d
 		panthor_query_group_priorities_info(file, &priorities_info);
 		return PANTHOR_UOBJ_SET(args->pointer, args->size, priorities_info);
 
+	case DRM_PANTHOR_DEV_QUERY_MMU_INFO:
+		return PANTHOR_UOBJ_SET(args->pointer, args->size, ptdev->mmu_info);
+
 	default:
 		return -EINVAL;
 	}
@@ -1779,10 +1787,12 @@ static void panthor_debugfs_init(struct drm_minor *minor)
  *       - adds DRM_IOCTL_PANTHOR_BO_QUERY_INFO ioctl
  *       - adds drm_panthor_gpu_info::selected_coherency
  * - 1.8 - extends DEV_QUERY_TIMESTAMP_INFO with flags
+ * - 1.9 - adds DRM_PANTHOR_DEV_QUERY_MMU_INFO query
+ *       - adds DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE flag
  */
 static const struct drm_driver panthor_drm_driver = {
 	.driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ |
-			   DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA,
+			   DRIVER_SYNCOBJ_TIMELINE,
 	.open = panthor_open,
 	.postclose = panthor_postclose,
 	.show_fdinfo = panthor_show_fdinfo,
@@ -1792,7 +1802,7 @@ static const struct drm_driver panthor_drm_driver = {
 	.name = "panthor",
 	.desc = "Panthor DRM driver",
 	.major = 1,
-	.minor = 8,
+	.minor = 9,
 
 	.gem_prime_import_sg_table = panthor_gem_prime_import_sg_table,
 	.gem_prime_import = panthor_gem_prime_import,
diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c
index de8e6689a869a..90f59d782a809 100644
--- a/drivers/gpu/drm/panthor/panthor_fw.c
+++ b/drivers/gpu/drm/panthor/panthor_fw.c
@@ -829,6 +829,7 @@ static int panthor_fw_load(struct panthor_device *ptdev)
 	}
 
 	if (hdr.size > iter.size) {
+		ret = -EINVAL;
 		drm_err(&ptdev->base, "Firmware image is truncated\n");
 		goto out;
 	}
diff --git a/drivers/gpu/drm/panthor/panthor_gem.c b/drivers/gpu/drm/panthor/panthor_gem.c
index a1e2eb1ca7bbe..aa55c9e33725f 100644
--- a/drivers/gpu/drm/panthor/panthor_gem.c
+++ b/drivers/gpu/drm/panthor/panthor_gem.c
@@ -776,7 +776,7 @@ static int panthor_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *v
 	}
 
 	vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
-	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+	vma->vm_page_prot = vma_get_page_prot(vma);
 	if (should_map_wc(bo))
 		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 
@@ -1347,6 +1347,24 @@ panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm,
 	return ERR_PTR(ret);
 }
 
+/**
+ * panthor_dummy_bo_create() - Create a Panthor BO meant to back sparse bindings.
+ * @ptdev: Device.
+ *
+ * Return: A valid pointer in case of success, an ERR_PTR() otherwise.
+ */
+struct panthor_gem_object *
+panthor_dummy_bo_create(struct panthor_device *ptdev)
+{
+	/* Since even when the DRM device's mount point has enabled THP we have no guarantee
+	 * that drm_gem_get_pages() will return a single 2MiB PMD, and also we cannot be sure
+	 * that the 2MiB won't be reclaimed and re-allocated later on as 4KiB chunks, it doesn't
+	 * make sense to pre-populate this object's page array, nor to fall back on a BO size
+	 * of 4KiB. Sticking to a dummy object size of 2MiB lets us keep things simple for now.
+	 */
+	return panthor_gem_create(&ptdev->base, SZ_2M, DRM_PANTHOR_BO_NO_MMAP, NULL, 0);
+}
+
 static bool can_swap(void)
 {
 	return get_nr_swap_pages() > 0;
diff --git a/drivers/gpu/drm/panthor/panthor_gem.h b/drivers/gpu/drm/panthor/panthor_gem.h
index 56d63137b4ebb..5ae37d0d3646f 100644
--- a/drivers/gpu/drm/panthor/panthor_gem.h
+++ b/drivers/gpu/drm/panthor/panthor_gem.h
@@ -325,6 +325,8 @@ panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm,
 
 void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo);
 
+struct panthor_gem_object *panthor_dummy_bo_create(struct panthor_device *ptdev);
+
 #ifdef CONFIG_DEBUG_FS
 void panthor_gem_debugfs_init(struct drm_minor *minor);
 #endif
diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c
index e592a8ebb4780..f45ef5824ff2f 100644
--- a/drivers/gpu/drm/panthor/panthor_mmu.c
+++ b/drivers/gpu/drm/panthor/panthor_mmu.c
@@ -116,6 +116,17 @@ struct panthor_mmu {
 struct panthor_vm_pool {
 	/** @xa: Array used for VM handle tracking. */
 	struct xarray xa;
+
+	/**
+	 * @dummy: Dummy object used for sparse mappings
+	 *
+	 * Sparse bindings map virtual address ranges onto a dummy
+	 * BO in a modulo fashion. Even though sparse writes are meant
+	 * to be discarded and reads undefined, writes are still reflected
+	 * in the dummy buffer. That means we must keep a dummy object per
+	 * file context, to avoid data leaks between them.
+	 */
+	struct panthor_gem_object *dummy;
 };
 
 /**
@@ -199,14 +210,6 @@ struct panthor_vm_op_ctx {
 		/** @map.bo_offset: Offset in the buffer object. */
 		u64 bo_offset;
 
-		/**
-		 * @map.sgt: sg-table pointing to pages backing the GEM object.
-		 *
-		 * This is gathered at job creation time, such that we don't have
-		 * to allocate in ::run_job().
-		 */
-		struct sg_table *sgt;
-
 		/** @map.bo: the BO being mapped. */
 		struct panthor_gem_object *bo;
 	} map;
@@ -403,6 +406,15 @@ struct panthor_vm {
 		 */
 		struct list_head lru_node;
 	} reclaim;
+
+	/**
+	 * @dummy: Dummy object used for sparse mappings.
+	 *
+	 * VM's must keep a reference to the file context-wide dummy BO because
+	 * they can outlive the file context, which includes the VM pool holding
+	 * the original dummy BO reference.
+	 */
+	struct panthor_gem_object *dummy;
 };
 
 /**
@@ -1035,6 +1047,30 @@ panthor_vm_map_pages(struct panthor_vm *vm, u64 iova, int prot,
 	return 0;
 }
 
+static int
+panthor_vm_map_sparse(struct panthor_vm *vm, u64 iova, int prot,
+		      struct sg_table *sgt, u64 size)
+{
+	u64 mapped = 0;
+	int ret;
+
+	while (mapped < size) {
+		u64 addr = iova + mapped;
+		u32 chunk_size = min(size - mapped, SZ_2M - (addr & (SZ_2M - 1)));
+
+		ret = panthor_vm_map_pages(vm, addr, prot, sgt,
+					   addr % SZ_2M, chunk_size);
+		if (ret) {
+			panthor_vm_unmap_pages(vm, iova, mapped);
+			return ret;
+		}
+
+		mapped += chunk_size;
+	}
+
+	return 0;
+}
+
 static int flags_to_prot(u32 flags)
 {
 	int prot = 0;
@@ -1277,15 +1313,15 @@ static int panthor_vm_op_ctx_prealloc_pts(struct panthor_vm_op_ctx *op_ctx)
 	(DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \
 	 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \
 	 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED | \
+	 DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE | \
 	 DRM_PANTHOR_VM_BIND_OP_TYPE_MASK)
 
 static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
 					 struct panthor_vm *vm,
 					 struct panthor_gem_object *bo,
-					 u64 offset,
-					 u64 size, u64 va,
-					 u32 flags)
+					 const struct drm_panthor_vm_bind_op *op)
 {
+	bool is_sparse = op->flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE;
 	struct drm_gpuvm_bo *preallocated_vm_bo;
 	struct sg_table *sgt = NULL;
 	int ret;
@@ -1293,12 +1329,25 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
 	if (!bo)
 		return -EINVAL;
 
-	if ((flags & ~PANTHOR_VM_BIND_OP_MAP_FLAGS) ||
-	    (flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) != DRM_PANTHOR_VM_BIND_OP_TYPE_MAP)
+	if ((op->flags & ~PANTHOR_VM_BIND_OP_MAP_FLAGS) ||
+	    (op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) != DRM_PANTHOR_VM_BIND_OP_TYPE_MAP)
 		return -EINVAL;
 
-	/* Make sure the VA and size are in-bounds. */
-	if (size > bo->base.size || offset > bo->base.size - size)
+	/* uAPI mandates sparsely bound regions must not be executable. */
+	if (is_sparse && !(op->flags & DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC))
+		return -EINVAL;
+
+	/* For non-sparse, make sure the VA and size are in-bounds.
+	 * For sparse, this is not applicable, because the dummy BO is
+	 * repeatedly mapped over a potentially wider VA range.
+	 */
+	if (!is_sparse && (op->size > bo->base.size || op->bo_offset > bo->base.size - op->size))
+		return -EINVAL;
+
+	/* For sparse, we don't expect any user BO, the BO we get passed
+	 * is the dummy BO attached to the VM pool.
+	 */
+	if (is_sparse && (op->bo_handle || op->bo_offset))
 		return -EINVAL;
 
 	/* If the BO has an exclusive VM attached, it can't be mapped to other VMs. */
@@ -1306,7 +1355,7 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
 	    bo->exclusive_vm_root_gem != panthor_vm_root_gem(vm))
 		return -EINVAL;
 
-	panthor_vm_init_op_ctx(op_ctx, size, va, flags);
+	panthor_vm_init_op_ctx(op_ctx, op->size, op->va, op->flags);
 
 	ret = panthor_vm_op_ctx_prealloc_vmas(op_ctx);
 	if (ret)
@@ -1335,7 +1384,7 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
 	}
 
 	op_ctx->map.vm_bo = drm_gpuvm_bo_obtain_prealloc(preallocated_vm_bo);
-	op_ctx->map.bo_offset = offset;
+	op_ctx->map.bo_offset = op->bo_offset;
 
 	ret = panthor_vm_op_ctx_prealloc_pts(op_ctx);
 	if (ret)
@@ -1446,7 +1495,9 @@ panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset)
 	if (vma && vma->base.gem.obj) {
 		drm_gem_object_get(vma->base.gem.obj);
 		bo = to_panthor_bo(vma->base.gem.obj);
-		*bo_offset = vma->base.gem.offset + (va - vma->base.va.addr);
+		*bo_offset = !(vma->flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE) ?
+			vma->base.gem.offset + (va - vma->base.va.addr) :
+			va & (SZ_2M - 1);
 	}
 	mutex_unlock(&vm->op_lock);
 
@@ -1551,6 +1602,9 @@ int panthor_vm_pool_create_vm(struct panthor_device *ptdev,
 	if (IS_ERR(vm))
 		return PTR_ERR(vm);
 
+	drm_gem_object_get(&pool->dummy->base);
+	vm->dummy = pool->dummy;
+
 	ret = xa_alloc(&pool->xa, &id, vm,
 		       XA_LIMIT(1, PANTHOR_MAX_VMS_PER_FILE), GFP_KERNEL);
 
@@ -1650,6 +1704,8 @@ void panthor_vm_pool_destroy(struct panthor_file *pfile)
 	xa_for_each(&pfile->vms->xa, i, vm)
 		panthor_vm_destroy(vm);
 
+	if (pfile->vms->dummy)
+		drm_gem_object_put(&pfile->vms->dummy->base);
 	xa_destroy(&pfile->vms->xa);
 	kfree(pfile->vms);
 }
@@ -1662,12 +1718,28 @@ void panthor_vm_pool_destroy(struct panthor_file *pfile)
  */
 int panthor_vm_pool_create(struct panthor_file *pfile)
 {
+	struct panthor_gem_object *dummy;
+	int ret;
+
 	pfile->vms = kzalloc_obj(*pfile->vms);
 	if (!pfile->vms)
 		return -ENOMEM;
 
 	xa_init_flags(&pfile->vms->xa, XA_FLAGS_ALLOC1);
+
+	dummy = panthor_dummy_bo_create(pfile->ptdev);
+	if (IS_ERR(dummy)) {
+		ret = PTR_ERR(dummy);
+		goto err_destroy_vm_pool;
+	}
+
+	pfile->vms->dummy = dummy;
+
 	return 0;
+
+err_destroy_vm_pool:
+	panthor_vm_pool_destroy(pfile);
+	return ret;
 }
 
 /* dummy TLB ops, the real TLB flush happens in panthor_vm_flush_range() */
@@ -2004,6 +2076,9 @@ static void panthor_vm_free(struct drm_gpuvm *gpuvm)
 
 	free_io_pgtable_ops(vm->pgtbl_ops);
 
+	if (vm->dummy)
+		drm_gem_object_put(&vm->dummy->base);
+
 	drm_mm_takedown(&vm->mm);
 	kfree(vm);
 }
@@ -2163,7 +2238,30 @@ static void panthor_vma_init(struct panthor_vma *vma, u32 flags)
 #define PANTHOR_VM_MAP_FLAGS \
 	(DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \
 	 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \
-	 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED)
+	 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED | \
+	 DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE)
+
+static void
+panthor_fix_sparse_map_offset(struct drm_gpuva_op_map *op, u32 flags)
+{
+	if (op && (flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE))
+		op->gem.offset = op->va.addr & (SZ_2M - 1);
+}
+
+static int
+panthor_vm_exec_map_op(struct panthor_vm *vm, u32 flags,
+		       const struct drm_gpuva_op_map *op)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(op->gem.obj);
+	int prot = flags_to_prot(flags);
+
+	if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE)
+		return panthor_vm_map_sparse(vm, op->va.addr, prot,
+					     bo->dmap.sgt, op->va.range);
+
+	return panthor_vm_map_pages(vm, op->va.addr, prot, bo->dmap.sgt,
+				    op->gem.offset, op->va.range);
+}
 
 static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv)
 {
@@ -2176,10 +2274,9 @@ static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv)
 		return -EINVAL;
 
 	panthor_vma_init(vma, op_ctx->flags & PANTHOR_VM_MAP_FLAGS);
+	panthor_fix_sparse_map_offset(&op->map, vma->flags);
 
-	ret = panthor_vm_map_pages(vm, op->map.va.addr, flags_to_prot(vma->flags),
-				   op_ctx->map.bo->dmap.sgt, op->map.gem.offset,
-				   op->map.va.range);
+	ret = panthor_vm_exec_map_op(vm, vma->flags, &op->map);
 	if (ret) {
 		panthor_vm_op_ctx_return_vma(op_ctx, vma);
 		return ret;
@@ -2211,6 +2308,8 @@ static void
 unmap_hugepage_align(const struct drm_gpuva_op_remap *op,
 		     u64 *unmap_start, u64 *unmap_range)
 {
+	struct panthor_vma *unmap_vma = container_of(op->unmap->va, struct panthor_vma, base);
+	bool is_sparse = unmap_vma->flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE;
 	u64 aligned_unmap_start, aligned_unmap_end, unmap_end;
 
 	unmap_end = *unmap_start + *unmap_range;
@@ -2218,11 +2317,15 @@ unmap_hugepage_align(const struct drm_gpuva_op_remap *op,
 	aligned_unmap_end = ALIGN(unmap_end, SZ_2M);
 
 	/* If we're dealing with a huge page, make sure the unmap region is
-	 * aligned on the start of the page.
+	 * aligned on the start of the page. If the unmapped VMA stands for
+	 * a sparse mapping, always assume the backing storage is a THP, since
+	 * the overhead of unmapping 2MiB worth of 4KiB pages and remapping
+	 * some of them is offset by the logic of working out whether it's
+	 * the opposite case right below. This also holds true for op->next.
 	 */
 	if (op->prev && aligned_unmap_start < *unmap_start &&
 	    op->prev->va.addr <= aligned_unmap_start &&
-	    iova_mapped_as_huge_page(op->prev, *unmap_start)) {
+	    (is_sparse || iova_mapped_as_huge_page(op->prev, *unmap_start))) {
 		*unmap_range += *unmap_start - aligned_unmap_start;
 		*unmap_start = aligned_unmap_start;
 	}
@@ -2232,7 +2335,7 @@ unmap_hugepage_align(const struct drm_gpuva_op_remap *op,
 	 */
 	if (op->next && aligned_unmap_end > unmap_end &&
 	    op->next->va.addr + op->next->va.range >= aligned_unmap_end &&
-	    iova_mapped_as_huge_page(op->next, unmap_end - 1)) {
+	    (is_sparse || iova_mapped_as_huge_page(op->next, unmap_end - 1))) {
 		*unmap_range += aligned_unmap_end - unmap_end;
 	}
 }
@@ -2249,33 +2352,43 @@ static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op,
 
 	drm_gpuva_op_remap_to_unmap_range(&op->remap, &unmap_start, &unmap_range);
 
-	/*
-	 * ARM IOMMU page table management code disallows partial unmaps of huge pages,
-	 * so when a partial unmap is requested, we must first unmap the entire huge
-	 * page and then remap the difference between the huge page minus the requested
-	 * unmap region. Calculating the right start address and range for the expanded
-	 * unmap operation is the responsibility of the following function.
+	/* op->remap.prev's BO offset is always the same as the unmap va's, but
+	 * that of op->remap.next must be adjusted so as to remain < SZ_2M
 	 */
-	unmap_hugepage_align(&op->remap, &unmap_start, &unmap_range);
+	panthor_fix_sparse_map_offset(op->remap.next, unmap_vma->flags);
 
-	/* If the range changed, we might have to lock a wider region to guarantee
-	 * atomicity. panthor_vm_lock_region() bails out early if the new region
-	 * is already part of the locked region, so no need to do this check here.
-	 */
 	if (!unmap_vma->evicted) {
+		/*
+		 * ARM IOMMU page table management code disallows partial unmaps of huge pages,
+		 * so when a partial unmap is requested, we must first unmap the entire huge
+		 * page and then remap the difference between the huge page minus the requested
+		 * unmap region. Calculating the right start address and range for the expanded
+		 * unmap operation is the responsibility of the following function.
+		 */
+		unmap_hugepage_align(&op->remap, &unmap_start, &unmap_range);
+
+		/* If the range changed, we might have to lock a wider region to guarantee
+		 * atomicity. panthor_vm_lock_region() bails out early if the new region
+		 * is already part of the locked region, so no need to do this check here.
+		 */
 		panthor_vm_lock_region(vm, unmap_start, unmap_range);
 		panthor_vm_unmap_pages(vm, unmap_start, unmap_range);
 	}
 
 	if (op->remap.prev) {
-		struct panthor_gem_object *bo = to_panthor_bo(op->remap.prev->gem.obj);
 		u64 offset = op->remap.prev->gem.offset + unmap_start - op->remap.prev->va.addr;
 		u64 size = op->remap.prev->va.addr + op->remap.prev->va.range - unmap_start;
 
-		if (!unmap_vma->evicted) {
-			ret = panthor_vm_map_pages(vm, unmap_start,
-						   flags_to_prot(unmap_vma->flags),
-						   bo->dmap.sgt, offset, size);
+		if (!unmap_vma->evicted && size > 0) {
+			struct drm_gpuva_op_map map_op = {
+				.va.addr = unmap_start,
+				.va.range = size,
+				.gem.obj = op->remap.prev->gem.obj,
+				.gem.offset = offset,
+			};
+			panthor_fix_sparse_map_offset(&map_op, unmap_vma->flags);
+
+			ret = panthor_vm_exec_map_op(vm, unmap_vma->flags, &map_op);
 			if (ret)
 				return ret;
 		}
@@ -2286,14 +2399,19 @@ static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op,
 	}
 
 	if (op->remap.next) {
-		struct panthor_gem_object *bo = to_panthor_bo(op->remap.next->gem.obj);
 		u64 addr = op->remap.next->va.addr;
 		u64 size = unmap_start + unmap_range - op->remap.next->va.addr;
 
-		if (!unmap_vma->evicted) {
-			ret = panthor_vm_map_pages(vm, addr, flags_to_prot(unmap_vma->flags),
-						   bo->dmap.sgt, op->remap.next->gem.offset,
-						   size);
+		if (!unmap_vma->evicted && size > 0) {
+			struct drm_gpuva_op_map map_op = {
+				.va.addr = addr,
+				.va.range = size,
+				.gem.obj = op->remap.next->gem.obj,
+				.gem.offset = op->remap.next->gem.offset,
+			};
+			panthor_fix_sparse_map_offset(&map_op, unmap_vma->flags);
+
+			ret = panthor_vm_exec_map_op(vm, unmap_vma->flags, &map_op);
 			if (ret)
 				return ret;
 		}
@@ -2490,11 +2608,17 @@ static int remap_evicted_vma(struct drm_gpuvm_bo *vm_bo,
 		ret = panthor_vm_lock_region(vm, evicted_vma->base.va.addr,
 					     evicted_vma->base.va.range);
 		if (!ret) {
-			ret = panthor_vm_map_pages(vm, evicted_vma->base.va.addr,
-						   flags_to_prot(evicted_vma->flags),
-						   bo->dmap.sgt,
-						   evicted_vma->base.gem.offset,
-						   evicted_vma->base.va.range);
+			struct drm_gpuva_op_map map_op = {
+				.va.addr = evicted_vma->base.va.addr,
+				.va.range = evicted_vma->base.va.range,
+				.gem.obj = &bo->base,
+				.gem.offset = evicted_vma->base.gem.offset,
+			};
+			if (evicted_vma->flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE)
+				drm_WARN_ON_ONCE(&vm->ptdev->base, map_op.gem.offset !=
+						 (map_op.va.addr & (SZ_2M - 1)));
+
+			ret = panthor_vm_exec_map_op(vm, evicted_vma->flags, &map_op);
 			if (!ret)
 				evicted_vma->evicted = false;
 
@@ -2781,7 +2905,7 @@ panthor_vm_create(struct panthor_device *ptdev, bool for_mcu,
 	refcount_set(&vm->as.active_cnt, 0);
 
 	pgtbl_cfg = (struct io_pgtable_cfg) {
-		.pgsize_bitmap	= SZ_4K | SZ_2M,
+		.pgsize_bitmap	= ptdev->mmu_info.page_size_bitmap,
 		.ias		= va_bits,
 		.oas		= pa_bits,
 		.coherent_walk	= ptdev->coherent,
@@ -2858,13 +2982,16 @@ panthor_vm_bind_prepare_op_ctx(struct drm_file *file,
 
 	switch (op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) {
 	case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:
-		gem = drm_gem_object_lookup(file, op->bo_handle);
+		if (!(op->flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE)) {
+			gem = drm_gem_object_lookup(file, op->bo_handle);
+		} else {
+			gem = &vm->dummy->base;
+			drm_gem_object_get(&vm->dummy->base);
+		}
+
 		ret = panthor_vm_prepare_map_op_ctx(op_ctx, vm,
 						    gem ? to_panthor_bo(gem) : NULL,
-						    op->bo_offset,
-						    op->size,
-						    op->va,
-						    op->flags);
+						    op);
 		drm_gem_object_put(gem);
 		return ret;
 
@@ -3060,10 +3187,19 @@ int panthor_vm_bind_exec_sync_op(struct drm_file *file,
 int panthor_vm_map_bo_range(struct panthor_vm *vm, struct panthor_gem_object *bo,
 			    u64 offset, u64 size, u64 va, u32 flags)
 {
+	struct drm_panthor_vm_bind_op op = {
+		.bo_offset = offset,
+		.size = size,
+		.va = va,
+		.flags = flags,
+	};
 	struct panthor_vm_op_ctx op_ctx;
 	int ret;
 
-	ret = panthor_vm_prepare_map_op_ctx(&op_ctx, vm, bo, offset, size, va, flags);
+	if (drm_WARN_ON(&vm->ptdev->base, flags & DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE))
+		return -EINVAL;
+
+	ret = panthor_vm_prepare_map_op_ctx(&op_ctx, vm, bo, &op);
 	if (ret)
 		return ret;
 
@@ -3227,6 +3363,11 @@ static void panthor_mmu_release_wq(struct drm_device *ddev, void *res)
 	destroy_workqueue(res);
 }
 
+static void panthor_mmu_info_init(struct panthor_device *ptdev)
+{
+	ptdev->mmu_info.page_size_bitmap = SZ_4K | SZ_2M;
+}
+
 /**
  * panthor_mmu_init() - Initialize the MMU logic.
  * @ptdev: Device.
@@ -3239,6 +3380,8 @@ int panthor_mmu_init(struct panthor_device *ptdev)
 	struct panthor_mmu *mmu;
 	int ret, irq;
 
+	panthor_mmu_info_init(ptdev);
+
 	mmu = drmm_kzalloc(&ptdev->base, sizeof(*mmu), GFP_KERNEL);
 	if (!mmu)
 		return -ENOMEM;
diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c
index 298b046c95edb..5832dccfc0932 100644
--- a/drivers/gpu/drm/panthor/panthor_sched.c
+++ b/drivers/gpu/drm/panthor/panthor_sched.c
@@ -3446,7 +3446,6 @@ queue_timedout_job(struct drm_sched_job *sched_job)
 
 static void queue_free_job(struct drm_sched_job *sched_job)
 {
-	drm_sched_job_cleanup(sched_job);
 	panthor_job_put(sched_job);
 }
 
diff --git a/drivers/gpu/drm/qxl/qxl_display.c b/drivers/gpu/drm/qxl/qxl_display.c
index a026bd35ef485..7f4178800afd7 100644
--- a/drivers/gpu/drm/qxl/qxl_display.c
+++ b/drivers/gpu/drm/qxl/qxl_display.c
@@ -31,12 +31,12 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_plane_helper.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_vblank.h>
 #include <drm/drm_vblank_helper.h>
@@ -1095,6 +1095,10 @@ static const struct drm_connector_helper_funcs qxl_connector_helper_funcs = {
 	.best_encoder = qxl_best_encoder,
 };
 
+static const struct drm_encoder_funcs qxl_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static enum drm_connector_status qxl_conn_detect(
 			struct drm_connector *connector,
 			bool force)
@@ -1169,10 +1173,10 @@ static int qdev_output_init(struct drm_device *dev, int num_output)
 	drm_connector_init(dev, &qxl_output->base,
 			   &qxl_connector_funcs, DRM_MODE_CONNECTOR_VIRTUAL);
 
-	ret = drm_simple_encoder_init(dev, &qxl_output->enc,
-				      DRM_MODE_ENCODER_VIRTUAL);
+	ret = drm_encoder_init(dev, &qxl_output->enc, &qxl_encoder_funcs,
+			       DRM_MODE_ENCODER_VIRTUAL, NULL);
 	if (ret) {
-		drm_err(dev, "drm_simple_encoder_init() failed, error %d\n",
+		drm_err(dev, "drm_encoder_init() failed, error %d\n",
 			ret);
 		goto err_drm_connector_cleanup;
 	}
diff --git a/drivers/gpu/drm/qxl/qxl_drv.c b/drivers/gpu/drm/qxl/qxl_drv.c
index 1e6a2392d7c6c..97df9d3d69573 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.c
+++ b/drivers/gpu/drm/qxl/qxl_drv.c
@@ -50,11 +50,16 @@
 #include "qxl_object.h"
 
 static const struct pci_device_id pciidlist[] = {
-	{ 0x1b36, 0x100, PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_DISPLAY_VGA << 8,
-	  0xffff00, 0 },
-	{ 0x1b36, 0x100, PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_DISPLAY_OTHER << 8,
-	  0xffff00, 0 },
-	{ 0, 0, 0 },
+	{
+		PCI_DEVICE(0x1b36, 0x0100),
+		.class = PCI_CLASS_DISPLAY_VGA << 8,
+		.class_mask = 0xffff00
+	}, {
+		PCI_DEVICE(0x1b36, 0x0100),
+		.class = PCI_CLASS_DISPLAY_OTHER << 8,
+		.class_mask = 0xffff00
+	},
+	{ },
 };
 MODULE_DEVICE_TABLE(pci, pciidlist);
 
diff --git a/drivers/gpu/drm/radeon/r600_dpm.c b/drivers/gpu/drm/radeon/r600_dpm.c
index 83f1ae31cbdbc..9755e717ca8bb 100644
--- a/drivers/gpu/drm/radeon/r600_dpm.c
+++ b/drivers/gpu/drm/radeon/r600_dpm.c
@@ -932,7 +932,7 @@ int r600_parse_extended_power_table(struct radeon_device *rdev)
 			ret = r600_parse_clk_voltage_dep_table(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
 							       dep_table);
 			if (ret) {
-				kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries);
+				r600_free_extended_power_table(rdev);
 				return ret;
 			}
 		}
@@ -943,8 +943,7 @@ int r600_parse_extended_power_table(struct radeon_device *rdev)
 			ret = r600_parse_clk_voltage_dep_table(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
 							       dep_table);
 			if (ret) {
-				kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries);
-				kfree(rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk.entries);
+				r600_free_extended_power_table(rdev);
 				return ret;
 			}
 		}
@@ -955,9 +954,7 @@ int r600_parse_extended_power_table(struct radeon_device *rdev)
 			ret = r600_parse_clk_voltage_dep_table(&rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk,
 							       dep_table);
 			if (ret) {
-				kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries);
-				kfree(rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk.entries);
-				kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk.entries);
+				r600_free_extended_power_table(rdev);
 				return ret;
 			}
 		}
@@ -1296,17 +1293,29 @@ void r600_free_extended_power_table(struct radeon_device *rdev)
 	struct radeon_dpm_dynamic_state *dyn_state = &rdev->pm.dpm.dyn_state;
 
 	kfree(dyn_state->vddc_dependency_on_sclk.entries);
+	dyn_state->vddc_dependency_on_sclk.entries = NULL;
 	kfree(dyn_state->vddci_dependency_on_mclk.entries);
+	dyn_state->vddci_dependency_on_mclk.entries = NULL;
 	kfree(dyn_state->vddc_dependency_on_mclk.entries);
+	dyn_state->vddc_dependency_on_mclk.entries = NULL;
 	kfree(dyn_state->mvdd_dependency_on_mclk.entries);
+	dyn_state->mvdd_dependency_on_mclk.entries = NULL;
 	kfree(dyn_state->cac_leakage_table.entries);
+	dyn_state->cac_leakage_table.entries = NULL;
 	kfree(dyn_state->phase_shedding_limits_table.entries);
+	dyn_state->phase_shedding_limits_table.entries = NULL;
 	kfree(dyn_state->ppm_table);
+	dyn_state->ppm_table = NULL;
 	kfree(dyn_state->cac_tdp_table);
+	dyn_state->cac_tdp_table = NULL;
 	kfree(dyn_state->vce_clock_voltage_dependency_table.entries);
+	dyn_state->vce_clock_voltage_dependency_table.entries = NULL;
 	kfree(dyn_state->uvd_clock_voltage_dependency_table.entries);
+	dyn_state->uvd_clock_voltage_dependency_table.entries = NULL;
 	kfree(dyn_state->samu_clock_voltage_dependency_table.entries);
+	dyn_state->samu_clock_voltage_dependency_table.entries = NULL;
 	kfree(dyn_state->acp_clock_voltage_dependency_table.entries);
+	dyn_state->acp_clock_voltage_dependency_table.entries = NULL;
 }
 
 enum radeon_pcie_gen r600_get_pcie_gen_support(struct radeon_device *rdev,
diff --git a/drivers/gpu/drm/radeon/radeon_bios.c b/drivers/gpu/drm/radeon/radeon_bios.c
index 3a8c5199a0fe7..8595f4c6e2e5c 100644
--- a/drivers/gpu/drm/radeon/radeon_bios.c
+++ b/drivers/gpu/drm/radeon/radeon_bios.c
@@ -597,19 +597,58 @@ static bool radeon_read_disabled_bios(struct radeon_device *rdev)
 }
 
 #ifdef CONFIG_ACPI
+/**
+ * radeon_acpi_vfct_match() - Check if a VFCT entry matches the device
+ * @rdev: Radeon device
+ * @vhdr: VFCT image header to check
+ *
+ * VFCT entries contain the PCI bus number as recorded during BIOS POST.
+ * On systems where the kernel renumbers PCI buses (e.g. pci=realloc or
+ * resource conflicts), the runtime bus number may differ from the POST
+ * value.  Match by device identity (vendor + device + function) and use
+ * the bus number as a preference: exact bus match is preferred, but when
+ * the bus numbers disagree we accept the entry if the device identity
+ * matches.
+ *
+ * Returns: 0 on match, -ENODEV on no match
+ */
+static int radeon_acpi_vfct_match(struct radeon_device *rdev,
+				  VFCT_IMAGE_HEADER *vhdr)
+{
+	/* Vendor and device IDs must always match */
+	if (vhdr->VendorID != rdev->pdev->vendor ||
+	    vhdr->DeviceID != rdev->pdev->device)
+		return -ENODEV;
+
+	if (vhdr->PCIDevice != PCI_SLOT(rdev->pdev->devfn) ||
+	    vhdr->PCIFunction != PCI_FUNC(rdev->pdev->devfn))
+		return -ENODEV;
+
+	/* Exact bus number match - preferred */
+	if (vhdr->PCIBus == rdev->pdev->bus->number)
+		return 0;
+
+	/* Bus mismatch but device identity matches (PCI renumbering case) */
+	dev_notice(&rdev->pdev->dev,
+		   "VFCT bus number mismatch: table %u != runtime %u, matching by device identity (vendor 0x%04x device 0x%04x)\n",
+		   vhdr->PCIBus, rdev->pdev->bus->number,
+		   rdev->pdev->vendor, rdev->pdev->device);
+	return 0;
+}
+
 static bool radeon_acpi_vfct_bios(struct radeon_device *rdev)
 {
 	struct acpi_table_header *hdr;
 	acpi_size tbl_size;
 	UEFI_ACPI_VFCT *vfct;
-	unsigned offset;
+	unsigned int offset;
 	bool r = false;
 
 	if (!ACPI_SUCCESS(acpi_get_table("VFCT", 1, &hdr)))
 		return false;
 	tbl_size = hdr->length;
 	if (tbl_size < sizeof(UEFI_ACPI_VFCT)) {
-		DRM_ERROR("ACPI VFCT table present but broken (too short #1)\n");
+		dev_warn(&rdev->pdev->dev, "ACPI VFCT table present but broken (too short #1),skipping\n");
 		goto out;
 	}
 
@@ -622,33 +661,34 @@ static bool radeon_acpi_vfct_bios(struct radeon_device *rdev)
 
 		offset += sizeof(VFCT_IMAGE_HEADER);
 		if (offset > tbl_size) {
-			DRM_ERROR("ACPI VFCT image header truncated\n");
+			dev_warn(&rdev->pdev->dev, "ACPI VFCT image header truncated,skipping\n");
 			goto out;
 		}
 
 		offset += vhdr->ImageLength;
 		if (offset > tbl_size) {
-			DRM_ERROR("ACPI VFCT image truncated\n");
+			dev_warn(&rdev->pdev->dev, "ACPI VFCT image truncated,skipping\n");
 			goto out;
 		}
 
 		if (vhdr->ImageLength &&
-		    vhdr->PCIBus == rdev->pdev->bus->number &&
-		    vhdr->PCIDevice == PCI_SLOT(rdev->pdev->devfn) &&
-		    vhdr->PCIFunction == PCI_FUNC(rdev->pdev->devfn) &&
-		    vhdr->VendorID == rdev->pdev->vendor &&
-		    vhdr->DeviceID == rdev->pdev->device) {
+		    !radeon_acpi_vfct_match(rdev, vhdr)) {
 			rdev->bios = kmemdup(&vbios->VbiosContent,
 					     vhdr->ImageLength,
 					     GFP_KERNEL);
-			if (rdev->bios)
-				r = true;
 
+			if (!rdev->bios ||
+			    rdev->bios[0] != 0x55 || rdev->bios[1] != 0xaa) {
+				kfree(rdev->bios);
+				rdev->bios = NULL;
+				goto out;
+			}
+			r = true;
 			goto out;
 		}
 	}
 
-	DRM_ERROR("ACPI VFCT table present but broken (too short #2)\n");
+	dev_warn(&rdev->pdev->dev, "ACPI VFCT table present but broken (too short #2),skipping\n");
 
 out:
 	acpi_put_table(hdr);
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_du_drv.c b/drivers/gpu/drm/renesas/rcar-du/rcar_du_drv.c
index 031d07f4508e9..d9926c24984eb 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_du_drv.c
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_du_drv.c
@@ -137,7 +137,7 @@ static const struct rcar_du_device_info rcar_du_r8a774a1_info = {
 	},
 	.num_lvds = 1,
 	.num_rpf = 5,
-	.dpll_mask =  BIT(1),
+	.dpll_mask = BIT(1),
 };
 
 static const struct rcar_du_device_info rcar_du_r8a774b1_info = {
@@ -168,7 +168,7 @@ static const struct rcar_du_device_info rcar_du_r8a774b1_info = {
 	},
 	.num_lvds = 1,
 	.num_rpf = 5,
-	.dpll_mask =  BIT(1),
+	.dpll_mask = BIT(1),
 };
 
 static const struct rcar_du_device_info rcar_du_r8a774c0_info = {
@@ -196,7 +196,7 @@ static const struct rcar_du_device_info rcar_du_r8a774c0_info = {
 	},
 	.num_lvds = 2,
 	.num_rpf = 4,
-	.lvds_clk_mask =  BIT(1) | BIT(0),
+	.lvds_clk_mask = BIT(1) | BIT(0),
 };
 
 static const struct rcar_du_device_info rcar_du_r8a774e1_info = {
@@ -227,7 +227,7 @@ static const struct rcar_du_device_info rcar_du_r8a774e1_info = {
 	},
 	.num_lvds = 1,
 	.num_rpf = 5,
-	.dpll_mask =  BIT(1),
+	.dpll_mask = BIT(1),
 };
 
 static const struct rcar_du_device_info rcar_du_r8a7779_info = {
@@ -385,7 +385,7 @@ static const struct rcar_du_device_info rcar_du_r8a7795_info = {
 	},
 	.num_lvds = 1,
 	.num_rpf = 5,
-	.dpll_mask =  BIT(2) | BIT(1),
+	.dpll_mask = BIT(2) | BIT(1),
 };
 
 static const struct rcar_du_device_info rcar_du_r8a7796_info = {
@@ -416,7 +416,7 @@ static const struct rcar_du_device_info rcar_du_r8a7796_info = {
 	},
 	.num_lvds = 1,
 	.num_rpf = 5,
-	.dpll_mask =  BIT(1),
+	.dpll_mask = BIT(1),
 };
 
 static const struct rcar_du_device_info rcar_du_r8a77965_info = {
@@ -447,7 +447,7 @@ static const struct rcar_du_device_info rcar_du_r8a77965_info = {
 	},
 	.num_lvds = 1,
 	.num_rpf = 5,
-	.dpll_mask =  BIT(1),
+	.dpll_mask = BIT(1),
 };
 
 static const struct rcar_du_device_info rcar_du_r8a77970_info = {
@@ -502,7 +502,7 @@ static const struct rcar_du_device_info rcar_du_r8a7799x_info = {
 	},
 	.num_lvds = 2,
 	.num_rpf = 5,
-	.lvds_clk_mask =  BIT(1) | BIT(0),
+	.lvds_clk_mask = BIT(1) | BIT(0),
 };
 
 static const struct rcar_du_device_info rcar_du_r8a779a0_info = {
@@ -523,7 +523,7 @@ static const struct rcar_du_device_info rcar_du_r8a779a0_info = {
 		},
 	},
 	.num_rpf = 5,
-	.dsi_clk_mask =  BIT(1) | BIT(0),
+	.dsi_clk_mask = BIT(1) | BIT(0),
 };
 
 static const struct rcar_du_device_info rcar_du_r8a779g0_info = {
@@ -544,7 +544,7 @@ static const struct rcar_du_device_info rcar_du_r8a779g0_info = {
 		},
 	},
 	.num_rpf = 5,
-	.dsi_clk_mask =  BIT(1) | BIT(0),
+	.dsi_clk_mask = BIT(1) | BIT(0),
 };
 
 static const struct rcar_du_device_info rcar_du_r8a779h0_info = {
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_lvds.c b/drivers/gpu/drm/renesas/rcar-du/rcar_lvds.c
index 154410745a74b..e433ce61d4314 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_lvds.c
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_lvds.c
@@ -658,7 +658,7 @@ static const struct drm_bridge_funcs rcar_lvds_bridge_ops = {
 	.destroy = rcar_lvds_destroy,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_enable = rcar_lvds_atomic_enable,
 	.atomic_disable = rcar_lvds_atomic_disable,
 	.mode_fixup = rcar_lvds_mode_fixup,
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c b/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c
index aaafee1c060bc..6e46d6d99f3c0 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c
@@ -924,7 +924,7 @@ static const struct drm_bridge_funcs rcar_mipi_dsi_bridge_ops = {
 	.attach = rcar_mipi_dsi_attach,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_enable = rcar_mipi_dsi_atomic_enable,
 	.atomic_disable = rcar_mipi_dsi_atomic_disable,
 	.mode_valid = rcar_mipi_dsi_bridge_mode_valid,
diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c
index a2c8648600756..0590ade96b91b 100644
--- a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c
+++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c
@@ -1109,7 +1109,7 @@ static const struct drm_bridge_funcs rzg2l_mipi_dsi_bridge_ops = {
 	.attach = rzg2l_mipi_dsi_attach,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_pre_enable = rzg2l_mipi_dsi_atomic_pre_enable,
 	.atomic_enable = rzg2l_mipi_dsi_atomic_enable,
 	.atomic_disable = rzg2l_mipi_dsi_atomic_disable,
diff --git a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
index 06072efd7fca3..46c245e35d215 100644
--- a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
+++ b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
@@ -8,6 +8,7 @@
  *         Jeff Chen <jeff.chen@rock-chips.com>
  */
 
+#include <linux/cleanup.h>
 #include <linux/component.h>
 #include <linux/mfd/syscon.h>
 #include <linux/of.h>
@@ -29,7 +30,6 @@
 #include <drm/drm_of.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "rockchip_drm_drv.h"
 
@@ -206,7 +206,6 @@ static void rockchip_dp_drm_encoder_enable(struct drm_encoder *encoder,
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *old_crtc_state;
 	struct of_endpoint endpoint;
-	struct device_node *remote_port, *remote_port_parent;
 	char name[32];
 	u32 port_id;
 	int ret;
@@ -230,18 +229,22 @@ static void rockchip_dp_drm_encoder_enable(struct drm_encoder *encoder,
 	if (ret < 0)
 		return;
 
-	remote_port_parent = of_graph_get_remote_port_parent(endpoint.local_node);
+	struct device_node *remote_port_parent __free(device_node) =
+		of_graph_get_remote_port_parent(endpoint.local_node);
 	if (remote_port_parent) {
-		if (of_get_child_by_name(remote_port_parent, "ports")) {
-			remote_port = of_graph_get_remote_port(endpoint.local_node);
+		struct device_node *ports __free(device_node) =
+			of_get_child_by_name(remote_port_parent, "ports");
+
+		if (ports) {
+			struct device_node *remote_port __free(device_node) =
+				of_graph_get_remote_port(endpoint.local_node);
+
 			of_property_read_u32(remote_port, "reg", &port_id);
-			of_node_put(remote_port);
 			sprintf(name, "%s vp%d", remote_port_parent->full_name, port_id);
 		} else {
 			sprintf(name, "%s %s",
 				remote_port_parent->full_name, endpoint.id ? "vopl" : "vopb");
 		}
-		of_node_put(remote_port_parent);
 
 		DRM_DEV_DEBUG(dp->dev, "vop %s output to dp\n", (ret) ? "LIT" : "BIG");
 	}
@@ -299,6 +302,10 @@ rockchip_dp_drm_encoder_atomic_check(struct drm_encoder *encoder,
 	return 0;
 }
 
+static const struct drm_encoder_funcs rockchip_dp_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs rockchip_dp_encoder_helper_funcs = {
 	.mode_fixup = rockchip_dp_drm_encoder_mode_fixup,
 	.mode_set = rockchip_dp_drm_encoder_mode_set,
@@ -311,6 +318,7 @@ static int rockchip_dp_of_probe(struct rockchip_dp_device *dp)
 {
 	struct device *dev = dp->dev;
 	struct device_node *np = dev->of_node;
+	struct clk *clk;
 
 	dp->grf = syscon_regmap_lookup_by_phandle(np, "rockchip,grf");
 	if (IS_ERR(dp->grf))
@@ -327,6 +335,11 @@ static int rockchip_dp_of_probe(struct rockchip_dp_device *dp)
 		return dev_err_probe(dev, PTR_ERR(dp->pclk),
 				     "failed to get pclk property\n");
 
+	clk = devm_clk_get_optional_enabled(dev, "hclk");
+	if (IS_ERR(clk))
+		return dev_err_probe(dev, PTR_ERR(clk),
+				     "failed to get hclk property\n");
+
 	dp->rst = devm_reset_control_get(dev, "dp");
 	if (IS_ERR(dp->rst))
 		return dev_err_probe(dev, PTR_ERR(dp->rst),
@@ -351,8 +364,8 @@ static int rockchip_dp_drm_create_encoder(struct rockchip_dp_device *dp)
 							     dev->of_node);
 	DRM_DEBUG_KMS("possible_crtcs = 0x%x\n", encoder->possible_crtcs);
 
-	ret = drm_simple_encoder_init(drm_dev, encoder,
-				      DRM_MODE_ENCODER_TMDS);
+	ret = drm_encoder_init(drm_dev, encoder, &rockchip_dp_encoder_funcs,
+			       DRM_MODE_ENCODER_TMDS, NULL);
 	if (ret) {
 		DRM_ERROR("failed to initialize encoder with drm\n");
 		return ret;
@@ -516,6 +529,14 @@ static const struct rockchip_dp_chip_data rk3288_dp[] = {
 	{ /* sentinel */ }
 };
 
+static const struct rockchip_dp_chip_data rk3576_edp[] = {
+	{
+		.chip_type = RK3576_EDP,
+		.reg = 0x27dc0000,
+	},
+	{ /* sentinel */ }
+};
+
 static const struct rockchip_dp_chip_data rk3588_edp[] = {
 	{
 		.edp_mode = GRF_REG_FIELD(0x0000, 0, 0),
@@ -533,6 +554,7 @@ static const struct rockchip_dp_chip_data rk3588_edp[] = {
 static const struct of_device_id rockchip_dp_dt_ids[] = {
 	{.compatible = "rockchip,rk3288-dp", .data = &rk3288_dp },
 	{.compatible = "rockchip,rk3399-edp", .data = &rk3399_edp },
+	{.compatible = "rockchip,rk3576-edp", .data = &rk3576_edp },
 	{.compatible = "rockchip,rk3588-edp", .data = &rk3588_edp },
 	{}
 };
diff --git a/drivers/gpu/drm/rockchip/cdn-dp-core.c b/drivers/gpu/drm/rockchip/cdn-dp-core.c
index 190cce9530c82..68aa66915e8ac 100644
--- a/drivers/gpu/drm/rockchip/cdn-dp-core.c
+++ b/drivers/gpu/drm/rockchip/cdn-dp-core.c
@@ -23,7 +23,6 @@
 #include <drm/drm_of.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "cdn-dp-core.h"
 #include "cdn-dp-reg.h"
@@ -671,6 +670,10 @@ static int cdn_dp_encoder_atomic_check(struct drm_encoder *encoder,
 	return 0;
 }
 
+static const struct drm_encoder_funcs cdn_dp_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs cdn_dp_encoder_helper_funcs = {
 	.atomic_check = cdn_dp_encoder_atomic_check,
 };
@@ -825,7 +828,7 @@ static int cdn_dp_audio_mute_stream(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs cdn_dp_bridge_funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.detect = cdn_dp_bridge_detect,
 	.edid_read = cdn_dp_bridge_edid_read,
 	.atomic_enable = cdn_dp_bridge_atomic_enable,
@@ -988,8 +991,8 @@ static int cdn_dp_bind(struct device *dev, struct device *master, void *data)
 							     dev->of_node);
 	DRM_DEBUG_KMS("possible_crtcs = 0x%x\n", encoder->possible_crtcs);
 
-	ret = drm_simple_encoder_init(drm_dev, encoder,
-				      DRM_MODE_ENCODER_TMDS);
+	ret = drm_encoder_init(drm_dev, encoder, &cdn_dp_encoder_funcs,
+			       DRM_MODE_ENCODER_TMDS, NULL);
 	if (ret) {
 		DRM_ERROR("failed to initialize encoder with drm\n");
 		return ret;
diff --git a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c
index 3547d91b25d31..268d4993a04fd 100644
--- a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c
+++ b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c
@@ -25,7 +25,6 @@
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_of.h>
 #include <drm/drm_print.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "rockchip_drm_drv.h"
 
@@ -825,6 +824,10 @@ static void dw_mipi_dsi_encoder_enable(struct drm_encoder *encoder)
 	clk_disable_unprepare(dsi->grf_clk);
 }
 
+static const struct drm_encoder_funcs dw_mipi_dsi_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs
 dw_mipi_dsi_encoder_helper_funcs = {
 	.atomic_check = dw_mipi_dsi_encoder_atomic_check,
@@ -840,7 +843,9 @@ static int rockchip_dsi_drm_create_encoder(struct dw_mipi_dsi_rockchip *dsi,
 	encoder->possible_crtcs = drm_of_find_possible_crtcs(drm_dev,
 							     dsi->dev->of_node);
 
-	ret = drm_simple_encoder_init(drm_dev, encoder, DRM_MODE_ENCODER_DSI);
+	ret = drm_encoder_init(drm_dev, encoder,
+			       &dw_mipi_dsi_encoder_funcs,
+			       DRM_MODE_ENCODER_DSI, NULL);
 	if (ret) {
 		DRM_ERROR("Failed to initialize encoder with drm\n");
 		return ret;
diff --git a/drivers/gpu/drm/rockchip/dw-mipi-dsi2-rockchip.c b/drivers/gpu/drm/rockchip/dw-mipi-dsi2-rockchip.c
index a2810e16765cf..6db88665060d6 100644
--- a/drivers/gpu/drm/rockchip/dw-mipi-dsi2-rockchip.c
+++ b/drivers/gpu/drm/rockchip/dw-mipi-dsi2-rockchip.c
@@ -22,7 +22,6 @@
 #include <drm/bridge/dw_mipi_dsi2.h>
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_of.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include <uapi/linux/videodev2.h>
 
@@ -274,6 +273,10 @@ dw_mipi_dsi2_encoder_atomic_check(struct drm_encoder *encoder,
 	return 0;
 }
 
+static const struct drm_encoder_funcs dw_mipi_dsi2_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs
 dw_mipi_dsi2_encoder_helper_funcs = {
 	.atomic_enable = dw_mipi_dsi2_encoder_atomic_enable,
@@ -289,7 +292,8 @@ static int rockchip_dsi2_drm_create_encoder(struct dw_mipi_dsi2_rockchip *dsi2,
 	encoder->possible_crtcs = drm_of_find_possible_crtcs(drm_dev,
 							     dsi2->dev->of_node);
 
-	ret = drm_simple_encoder_init(drm_dev, encoder, DRM_MODE_ENCODER_DSI);
+	ret = drm_encoder_init(drm_dev, encoder, &dw_mipi_dsi2_encoder_funcs,
+			       DRM_MODE_ENCODER_DSI, NULL);
 	if (ret) {
 		dev_err(dsi2->dev, "Failed to initialize encoder with drm\n");
 		return ret;
diff --git a/drivers/gpu/drm/rockchip/dw_dp-rockchip.c b/drivers/gpu/drm/rockchip/dw_dp-rockchip.c
index 32bc73a1d5e45..b23efb153c9e6 100644
--- a/drivers/gpu/drm/rockchip/dw_dp-rockchip.c
+++ b/drivers/gpu/drm/rockchip/dw_dp-rockchip.c
@@ -20,7 +20,6 @@
 #include <drm/drm_of.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "rockchip_drm_drv.h"
 
@@ -107,15 +106,26 @@ static int dw_dp_rockchip_bind(struct device *dev, struct device *master, void *
 		return PTR_ERR(dp->base);
 
 	connector = drm_bridge_connector_init(drm_dev, encoder);
-	if (IS_ERR(connector))
+	if (IS_ERR(connector)) {
+		dw_dp_unbind(dp->base);
 		return dev_err_probe(dev, PTR_ERR(connector),
-				     "Failed to init bridge connector");
+				     "Failed to init bridge connector\n");
+	}
 
 	return 0;
 }
 
+static void dw_dp_rockchip_unbind(struct device *dev, struct device *master,
+				  void *data)
+{
+	struct rockchip_dw_dp *dp = dev_get_drvdata(dev);
+
+	dw_dp_unbind(dp->base);
+}
+
 static const struct component_ops dw_dp_rockchip_component_ops = {
 	.bind = dw_dp_rockchip_bind,
+	.unbind = dw_dp_rockchip_unbind,
 };
 
 static int dw_dp_probe(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c b/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
index 7abb42e486c02..b6e154c35e7c2 100644
--- a/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
+++ b/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
@@ -5,6 +5,7 @@
 
 #include <linux/clk.h>
 #include <linux/hw_bitfield.h>
+#include <linux/media-bus-format.h>
 #include <linux/mfd/syscon.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
@@ -14,9 +15,9 @@
 
 #include <drm/bridge/dw_hdmi.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_managed.h>
 #include <drm/drm_of.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "rockchip_drm_drv.h"
 
@@ -30,8 +31,8 @@
 
 #define RK3288_GRF_SOC_CON6		0x025C
 #define RK3288_HDMI_LCDC_SEL		BIT(4)
-#define RK3328_GRF_SOC_CON2		0x0408
 
+#define RK3328_GRF_SOC_CON2		0x0408
 #define RK3328_HDMI_SDAIN_MSK		BIT(11)
 #define RK3328_HDMI_SCLIN_MSK		BIT(10)
 #define RK3328_HDMI_HPD_IOE		BIT(2)
@@ -78,6 +79,7 @@ struct rockchip_hdmi {
 	struct clk *hdmiphy_clk;
 	struct clk *ref_clk;
 	struct clk *grf_clk;
+	struct drm_bridge *bridge;
 	struct dw_hdmi *hdmi;
 	struct phy *phy;
 };
@@ -188,48 +190,13 @@ static const struct dw_hdmi_curr_ctrl rockchip_cur_ctr[] = {
 
 static const struct dw_hdmi_phy_config rockchip_phy_config[] = {
 	/*pixelclk   symbol   term   vlev*/
-	{ 74250000,  0x8009, 0x0004, 0x0272},
-	{ 165000000, 0x802b, 0x0004, 0x0209},
-	{ 297000000, 0x8039, 0x0005, 0x028d},
-	{ 594000000, 0x8039, 0x0000, 0x019d},
-	{ ~0UL,	     0x0000, 0x0000, 0x0000}
+	{ 74250000,  0x8009, 0x0004, 0x0272 },
+	{ 165000000, 0x802b, 0x0004, 0x0209 },
+	{ 297000000, 0x8039, 0x0005, 0x028d },
+	{ 594000000, 0x8039, 0x0000, 0x019d },
+	{ ~0UL,	     0x0000, 0x0000, 0x0000 },
 };
 
-static int rockchip_hdmi_parse_dt(struct rockchip_hdmi *hdmi)
-{
-	struct device_node *np = hdmi->dev->of_node;
-	int ret;
-
-	hdmi->regmap = syscon_regmap_lookup_by_phandle(np, "rockchip,grf");
-	if (IS_ERR(hdmi->regmap)) {
-		dev_err(hdmi->dev, "Unable to get rockchip,grf\n");
-		return PTR_ERR(hdmi->regmap);
-	}
-
-	hdmi->ref_clk = devm_clk_get_optional_enabled(hdmi->dev, "ref");
-	if (!hdmi->ref_clk)
-		hdmi->ref_clk = devm_clk_get_optional_enabled(hdmi->dev, "vpll");
-
-	if (IS_ERR(hdmi->ref_clk)) {
-		ret = PTR_ERR(hdmi->ref_clk);
-		return dev_err_probe(hdmi->dev, ret, "failed to get reference clock\n");
-	}
-
-	hdmi->grf_clk = devm_clk_get_optional(hdmi->dev, "grf");
-	if (IS_ERR(hdmi->grf_clk)) {
-		ret = PTR_ERR(hdmi->grf_clk);
-		return dev_err_probe(hdmi->dev, ret, "failed to get grf clock\n");
-	}
-
-	ret = devm_regulator_get_enable(hdmi->dev, "avdd-0v9");
-	if (ret)
-		return ret;
-
-	ret = devm_regulator_get_enable(hdmi->dev, "avdd-1v8");
-
-	return ret;
-}
-
 static enum drm_mode_status
 dw_hdmi_rockchip_mode_valid(struct dw_hdmi *dw_hdmi, void *data,
 			    const struct drm_display_info *info,
@@ -259,23 +226,22 @@ dw_hdmi_rockchip_mode_valid(struct dw_hdmi *dw_hdmi, void *data,
 	return MODE_OK;
 }
 
-static void dw_hdmi_rockchip_encoder_disable(struct drm_encoder *encoder)
+static void
+dw_hdmi_rockchip_encoder_atomic_mode_set(struct drm_encoder *encoder,
+					 struct drm_crtc_state *crtc_state,
+					 struct drm_connector_state *conn_state)
 {
-}
+	struct rockchip_hdmi *hdmi = to_rockchip_hdmi(encoder);
+	struct drm_display_mode *adj_mode = &crtc_state->adjusted_mode;
 
-static bool
-dw_hdmi_rockchip_encoder_mode_fixup(struct drm_encoder *encoder,
-				    const struct drm_display_mode *mode,
-				    struct drm_display_mode *adj_mode)
-{
-	return true;
-}
+	if (hdmi->phy && conn_state->hdmi.tmds_char_rate) {
+		union phy_configure_opts opts = {};
 
-static void dw_hdmi_rockchip_encoder_mode_set(struct drm_encoder *encoder,
-					      struct drm_display_mode *mode,
-					      struct drm_display_mode *adj_mode)
-{
-	struct rockchip_hdmi *hdmi = to_rockchip_hdmi(encoder);
+		opts.hdmi.bpc = conn_state->hdmi.output_bpc;
+		opts.hdmi.tmds_char_rate = conn_state->hdmi.tmds_char_rate;
+
+		phy_configure(hdmi->phy, &opts);
+	}
 
 	clk_set_rate(hdmi->ref_clk, adj_mode->clock * 1000);
 }
@@ -309,24 +275,71 @@ static void dw_hdmi_rockchip_encoder_enable(struct drm_encoder *encoder)
 	dev_dbg(hdmi->dev, "vop %s output to hdmi\n", ret ? "LIT" : "BIG");
 }
 
+static u32 dw_hdmi_rockchip_get_bus_format(struct drm_encoder *encoder,
+					   struct drm_connector_state *conn_state)
+{
+	struct drm_bridge *bridge __free(drm_bridge_put) = NULL;
+	struct drm_bridge_state *bridge_state;
+
+	bridge = drm_bridge_chain_get_first_bridge(encoder);
+	if (!bridge)
+		return 0;
+
+	bridge_state = drm_atomic_get_bridge_state(conn_state->state, bridge);
+	if (!bridge_state)
+		return 0;
+
+	if (bridge_state->input_bus_cfg.format != MEDIA_BUS_FMT_FIXED)
+		return bridge_state->input_bus_cfg.format;
+
+	return bridge_state->output_bus_cfg.format;
+}
+
 static int
 dw_hdmi_rockchip_encoder_atomic_check(struct drm_encoder *encoder,
 				      struct drm_crtc_state *crtc_state,
 				      struct drm_connector_state *conn_state)
 {
 	struct rockchip_crtc_state *s = to_rockchip_crtc_state(crtc_state);
+	struct rockchip_hdmi *hdmi = to_rockchip_hdmi(encoder);
+	union phy_configure_opts opts = {};
+	u32 bus_format;
+
+	bus_format = dw_hdmi_rockchip_get_bus_format(encoder, conn_state);
+
+	switch (bus_format) {
+	case MEDIA_BUS_FMT_FIXED:
+		bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+		fallthrough;
+	case MEDIA_BUS_FMT_RGB888_1X24:
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+	case MEDIA_BUS_FMT_YUV8_1X24:
+	case MEDIA_BUS_FMT_YUV10_1X30:
+		s->output_mode = ROCKCHIP_OUT_MODE_AAAA;
+		break;
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+		s->output_mode = ROCKCHIP_OUT_MODE_YUV420;
+		break;
+	default:
+		return -EINVAL;
+	}
 
-	s->output_mode = ROCKCHIP_OUT_MODE_AAAA;
 	s->output_type = DRM_MODE_CONNECTOR_HDMIA;
+	s->bus_format = bus_format;
 
-	return 0;
+	if (!hdmi->phy || !conn_state->hdmi.tmds_char_rate)
+		return 0;
+
+	opts.hdmi.bpc = conn_state->hdmi.output_bpc;
+	opts.hdmi.tmds_char_rate = conn_state->hdmi.tmds_char_rate;
+
+	return phy_validate(hdmi->phy, PHY_MODE_HDMI, PHY_HDMI_MODE_TMDS, &opts);
 }
 
 static const struct drm_encoder_helper_funcs dw_hdmi_rockchip_encoder_helper_funcs = {
-	.mode_fixup = dw_hdmi_rockchip_encoder_mode_fixup,
-	.mode_set   = dw_hdmi_rockchip_encoder_mode_set,
-	.enable     = dw_hdmi_rockchip_encoder_enable,
-	.disable    = dw_hdmi_rockchip_encoder_disable,
+	.atomic_mode_set = dw_hdmi_rockchip_encoder_atomic_mode_set,
+	.enable = dw_hdmi_rockchip_encoder_enable,
 	.atomic_check = dw_hdmi_rockchip_encoder_atomic_check,
 };
 
@@ -508,6 +521,7 @@ static const struct dw_hdmi_plat_data rk3568_hdmi_drv_data = {
 	.phy_config = rockchip_phy_config,
 	.phy_data = &rk3568_chip_data,
 	.use_drm_infoframe = true,
+	.output_port = 1,
 };
 
 static const struct of_device_id dw_hdmi_rockchip_dt_ids[] = {
@@ -538,8 +552,8 @@ static int dw_hdmi_rockchip_bind(struct device *dev, struct device *master,
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct device_node *np = dev_of_node(dev);
+	const struct dw_hdmi_plat_data *drv_data;
 	struct dw_hdmi_plat_data *plat_data;
-	const struct of_device_id *match;
 	struct drm_device *drm = data;
 	struct drm_encoder *encoder;
 	struct rockchip_hdmi *hdmi;
@@ -548,17 +562,20 @@ static int dw_hdmi_rockchip_bind(struct device *dev, struct device *master,
 	if (!np)
 		return -ENODEV;
 
-	hdmi = devm_kzalloc(&pdev->dev, sizeof(*hdmi), GFP_KERNEL);
+	drv_data = of_device_get_match_data(dev);
+	if (!drv_data)
+		return -ENODEV;
+
+	hdmi = drmm_kzalloc(drm, sizeof(*hdmi), GFP_KERNEL);
 	if (!hdmi)
 		return -ENOMEM;
 
-	match = of_match_node(dw_hdmi_rockchip_dt_ids, np);
-	plat_data = devm_kmemdup(&pdev->dev, match->data,
-					     sizeof(*plat_data), GFP_KERNEL);
+	plat_data = drmm_kzalloc(drm, sizeof(*drv_data), GFP_KERNEL);
 	if (!plat_data)
 		return -ENOMEM;
+	memcpy(plat_data, drv_data, sizeof(*drv_data));
 
-	hdmi->dev = &pdev->dev;
+	hdmi->dev = dev;
 	hdmi->plat_data = plat_data;
 	hdmi->chip_data = plat_data->phy_data;
 	plat_data->phy_data = hdmi;
@@ -576,18 +593,39 @@ static int dw_hdmi_rockchip_bind(struct device *dev, struct device *master,
 	 * the required CRTC is added later.
 	 */
 	if (encoder->possible_crtcs == 0)
-		return -EPROBE_DEFER;
+		return dev_err_probe(dev, -EPROBE_DEFER,
+				     "failed to find possible crtcs\n");
 
-	ret = rockchip_hdmi_parse_dt(hdmi);
-	if (ret) {
-		return dev_err_probe(hdmi->dev, ret, "Unable to parse OF data\n");
-	}
+	hdmi->regmap = syscon_regmap_lookup_by_phandle(np, "rockchip,grf");
+	if (IS_ERR(hdmi->regmap))
+		return dev_err_probe(dev, PTR_ERR(hdmi->regmap),
+				     "failed to get rockchip,grf\n");
+
+	hdmi->ref_clk = devm_clk_get_optional_enabled(dev, "ref");
+	if (!hdmi->ref_clk)
+		hdmi->ref_clk = devm_clk_get_optional_enabled(dev, "vpll");
+
+	if (IS_ERR(hdmi->ref_clk))
+		return dev_err_probe(dev, PTR_ERR(hdmi->ref_clk),
+				     "failed to get reference clock\n");
+
+	hdmi->grf_clk = devm_clk_get_optional(dev, "grf");
+	if (IS_ERR(hdmi->grf_clk))
+		return dev_err_probe(dev, PTR_ERR(hdmi->grf_clk),
+				     "failed to get grf clock\n");
+
+	ret = devm_regulator_get_enable(dev, "avdd-0v9");
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to enable avdd-0v9\n");
+
+	ret = devm_regulator_get_enable(dev, "avdd-1v8");
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to enable avdd-1v8\n");
 
 	hdmi->phy = devm_phy_optional_get(dev, "hdmi");
-	if (IS_ERR(hdmi->phy)) {
-		ret = PTR_ERR(hdmi->phy);
-		return dev_err_probe(hdmi->dev, ret, "failed to get phy\n");
-	}
+	if (IS_ERR(hdmi->phy))
+		return dev_err_probe(dev, PTR_ERR(hdmi->phy),
+				     "failed to get phy\n");
 
 	index = of_property_match_string(np, "phy-names", "hdmi");
 	if (index >= 0) {
@@ -608,28 +646,27 @@ static int dw_hdmi_rockchip_bind(struct device *dev, struct device *master,
 			     FIELD_PREP_WM16(RK3568_HDMI_SCLIN_MSK, 1));
 	}
 
+	ret = drmm_encoder_init(drm, encoder, NULL, DRM_MODE_ENCODER_TMDS, NULL);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to init encoder\n");
+
 	drm_encoder_helper_add(encoder, &dw_hdmi_rockchip_encoder_helper_funcs);
-	drm_simple_encoder_init(drm, encoder, DRM_MODE_ENCODER_TMDS);
 
 	platform_set_drvdata(pdev, hdmi);
 
 	hdmi->hdmi = dw_hdmi_bind(pdev, encoder, plat_data);
-
-	/*
-	 * If dw_hdmi_bind() fails we'll never call dw_hdmi_unbind(),
-	 * which would have called the encoder cleanup.  Do it manually.
-	 */
-	if (IS_ERR(hdmi->hdmi)) {
-		ret = PTR_ERR(hdmi->hdmi);
-		goto err_bind;
+	if (IS_ERR(hdmi->hdmi))
+		return dev_err_probe(dev, PTR_ERR(hdmi->hdmi),
+				     "failed to probe dw-hdmi bridge\n");
+
+	hdmi->bridge = of_drm_find_and_get_bridge(np);
+	if (!hdmi->bridge) {
+		dw_hdmi_unbind(hdmi->hdmi);
+		return dev_err_probe(dev, -ENODEV,
+				     "failed to find dw-hdmi bridge\n");
 	}
 
 	return 0;
-
-err_bind:
-	drm_encoder_cleanup(encoder);
-
-	return ret;
 }
 
 static void dw_hdmi_rockchip_unbind(struct device *dev, struct device *master,
@@ -637,8 +674,8 @@ static void dw_hdmi_rockchip_unbind(struct device *dev, struct device *master,
 {
 	struct rockchip_hdmi *hdmi = dev_get_drvdata(dev);
 
+	drm_bridge_put(hdmi->bridge);
 	dw_hdmi_unbind(hdmi->hdmi);
-	drm_encoder_cleanup(&hdmi->encoder.encoder);
 }
 
 static const struct component_ops dw_hdmi_rockchip_ops = {
@@ -656,17 +693,18 @@ static void dw_hdmi_rockchip_remove(struct platform_device *pdev)
 	component_del(&pdev->dev, &dw_hdmi_rockchip_ops);
 }
 
-static int __maybe_unused dw_hdmi_rockchip_resume(struct device *dev)
+static int __maybe_unused dw_hdmi_rockchip_resume_early(struct device *dev)
 {
 	struct rockchip_hdmi *hdmi = dev_get_drvdata(dev);
 
-	dw_hdmi_resume(hdmi->hdmi);
+	if (hdmi)
+		dw_hdmi_resume(hdmi->hdmi);
 
 	return 0;
 }
 
 static const struct dev_pm_ops dw_hdmi_rockchip_pm = {
-	SET_SYSTEM_SLEEP_PM_OPS(NULL, dw_hdmi_rockchip_resume)
+	SET_LATE_SYSTEM_SLEEP_PM_OPS(NULL, dw_hdmi_rockchip_resume_early)
 };
 
 struct platform_driver dw_hdmi_rockchip_pltfm_driver = {
diff --git a/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c b/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c
index f35484715c2d1..9c4cf68d79d50 100644
--- a/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c
+++ b/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c
@@ -11,6 +11,7 @@
 #include <linux/gpio/consumer.h>
 #include <linux/hw_bitfield.h>
 #include <linux/mfd/syscon.h>
+#include <linux/media-bus-format.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/phy/phy.h>
@@ -24,7 +25,6 @@
 #include <drm/drm_managed.h>
 #include <drm/drm_of.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "rockchip_drm_drv.h"
 
@@ -44,10 +44,6 @@
 #define RK3576_8BPC			0x0
 #define RK3576_10BPC			0x6
 #define RK3576_COLOR_FORMAT_MASK	GENMASK(7, 4)
-#define RK3576_RGB			0x9
-#define RK3576_YUV422			0x1
-#define RK3576_YUV444			0x2
-#define RK3576_YUV420			0x3
 #define RK3576_CECIN_MASK		BIT(3)
 
 #define RK3576_VO0_GRF_SOC_CON14	0x0038
@@ -75,8 +71,6 @@
 #define RK3588_8BPC			0x0
 #define RK3588_10BPC			0x6
 #define RK3588_COLOR_FORMAT_MASK	GENMASK(3, 0)
-#define RK3588_RGB			0x0
-#define RK3588_YUV420			0x3
 #define RK3588_SCLIN_MASK		BIT(9)
 #define RK3588_SDAIN_MASK		BIT(10)
 #define RK3588_MODE_MASK		BIT(11)
@@ -88,6 +82,11 @@
 #define HOTPLUG_DEBOUNCE_MS		150
 #define MAX_HDMI_PORT_NUM		2
 
+#define RK_COLOR_FMT_RGB		0x0
+#define RK_COLOR_FMT_YUV422		0x1
+#define RK_COLOR_FMT_YUV444		0x2
+#define RK_COLOR_FMT_YUV420		0x3
+
 struct rockchip_hdmi_qp {
 	struct device *dev;
 	struct regmap *regmap;
@@ -116,6 +115,33 @@ static struct rockchip_hdmi_qp *to_rockchip_hdmi_qp(struct drm_encoder *encoder)
 	return container_of(rkencoder, struct rockchip_hdmi_qp, encoder);
 }
 
+/**
+ * dw_hdmi_qp_rockchip_bus_fmt_to_reg - converts a bus format to a GRF reg value
+ * @bus_fmt: One of the MEDIA_BUS_FMT_s allowed by this driver's atomic_check
+ *
+ * Returns: an unshifted value to be written to the COLOR_FORMAT GRF register
+ * on success, or %-EINVAL if the bus format is not supported.
+ */
+static int __pure dw_hdmi_qp_rockchip_bus_fmt_to_reg(u32 bus_fmt)
+{
+	switch (bus_fmt) {
+	case MEDIA_BUS_FMT_RGB888_1X24:
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+		return RK_COLOR_FMT_RGB;
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+	case MEDIA_BUS_FMT_UYVY10_1X20:
+		return RK_COLOR_FMT_YUV422;
+	case MEDIA_BUS_FMT_YUV8_1X24:
+	case MEDIA_BUS_FMT_YUV10_1X30:
+		return RK_COLOR_FMT_YUV444;
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+		return RK_COLOR_FMT_YUV420;
+	}
+
+	return -EINVAL;
+}
+
 static void dw_hdmi_qp_rockchip_encoder_enable(struct drm_encoder *encoder)
 {
 	struct rockchip_hdmi_qp *hdmi = to_rockchip_hdmi_qp(encoder);
@@ -131,29 +157,83 @@ static void dw_hdmi_qp_rockchip_encoder_enable(struct drm_encoder *encoder)
 		hdmi->ctrl_ops->enc_init(hdmi, to_rockchip_crtc_state(crtc->state));
 }
 
+/**
+ * dw_hdmi_qp_rockchip_get_vop_format - get the bus format VOP should output
+ * @encoder: pointer to a &struct drm_encoder
+ * @conn_state: pointer to the current atomic &struct drm_connector_state
+ *
+ * Determines which bus format the Rockchip video processor should output as
+ * to feed into the bridge chain.
+ *
+ * Returns a MEDIA_BUS_FMT_* on success, or negative errno on error.
+ */
+static int dw_hdmi_qp_rockchip_get_vop_format(struct drm_encoder *encoder,
+					      struct drm_connector_state *conn_state)
+{
+	struct drm_bridge *bridge __free(drm_bridge_put) = NULL;
+	struct drm_bridge_state *bstate;
+
+	bridge = drm_bridge_chain_get_first_bridge(encoder);
+	if (!bridge)
+		return -ENODEV;
+
+	bstate = drm_atomic_get_bridge_state(conn_state->state, bridge);
+	if (IS_ERR(bstate))
+		return PTR_ERR(bstate);
+
+	if (bstate->input_bus_cfg.format != MEDIA_BUS_FMT_FIXED)
+		return bstate->input_bus_cfg.format;
+
+	return bstate->output_bus_cfg.format;
+}
+
 static int
 dw_hdmi_qp_rockchip_encoder_atomic_check(struct drm_encoder *encoder,
 					 struct drm_crtc_state *crtc_state,
 					 struct drm_connector_state *conn_state)
 {
-	struct rockchip_hdmi_qp *hdmi = to_rockchip_hdmi_qp(encoder);
 	struct rockchip_crtc_state *s = to_rockchip_crtc_state(crtc_state);
+	struct rockchip_hdmi_qp *hdmi = to_rockchip_hdmi_qp(encoder);
 	union phy_configure_opts phy_cfg = {};
+	int ingest_fmt;
 	int ret;
 
+	ingest_fmt = dw_hdmi_qp_rockchip_get_vop_format(encoder, conn_state);
+	if (ingest_fmt < 0)
+		return -EINVAL;
+
 	if (hdmi->tmds_char_rate == conn_state->hdmi.tmds_char_rate &&
-	    s->output_bpc == conn_state->hdmi.output_bpc)
+	    s->output_bpc == conn_state->hdmi.output_bpc &&
+	    s->bus_format == ingest_fmt)
 		return 0;
 
+	switch (ingest_fmt) {
+	case MEDIA_BUS_FMT_RGB888_1X24:
+	case MEDIA_BUS_FMT_RGB101010_1X30:
+	case MEDIA_BUS_FMT_YUV8_1X24:
+	case MEDIA_BUS_FMT_YUV10_1X30:
+		s->output_mode = ROCKCHIP_OUT_MODE_AAAA;
+		break;
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+		s->output_mode = ROCKCHIP_OUT_MODE_YUV422;
+		break;
+	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
+	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
+		s->output_mode = ROCKCHIP_OUT_MODE_YUV420;
+		break;
+	default:
+		return -EINVAL;
+	}
+
 	phy_cfg.hdmi.tmds_char_rate = conn_state->hdmi.tmds_char_rate;
 	phy_cfg.hdmi.bpc = conn_state->hdmi.output_bpc;
 
 	ret = phy_configure(hdmi->phy, &phy_cfg);
 	if (!ret) {
 		hdmi->tmds_char_rate = conn_state->hdmi.tmds_char_rate;
-		s->output_mode = ROCKCHIP_OUT_MODE_AAAA;
 		s->output_type = DRM_MODE_CONNECTOR_HDMIA;
 		s->output_bpc = conn_state->hdmi.output_bpc;
+		s->bus_format = ingest_fmt;
 	} else {
 		dev_err(hdmi->dev, "Failed to configure phy: %d\n", ret);
 	}
@@ -383,6 +463,7 @@ static void dw_hdmi_qp_rk3588_io_init(struct rockchip_hdmi_qp *hdmi)
 static void dw_hdmi_qp_rk3576_enc_init(struct rockchip_hdmi_qp *hdmi,
 				       struct rockchip_crtc_state *state)
 {
+	int color = dw_hdmi_qp_rockchip_bus_fmt_to_reg(state->bus_format);
 	u32 val;
 
 	if (state->output_bpc == 10)
@@ -390,12 +471,16 @@ static void dw_hdmi_qp_rk3576_enc_init(struct rockchip_hdmi_qp *hdmi,
 	else
 		val = FIELD_PREP_WM16(RK3576_COLOR_DEPTH_MASK, RK3576_8BPC);
 
+	if (likely(color >= 0))
+		val |= FIELD_PREP_WM16(RK3576_COLOR_FORMAT_MASK, color);
+
 	regmap_write(hdmi->vo_regmap, RK3576_VO0_GRF_SOC_CON8, val);
 }
 
 static void dw_hdmi_qp_rk3588_enc_init(struct rockchip_hdmi_qp *hdmi,
 				       struct rockchip_crtc_state *state)
 {
+	int color = dw_hdmi_qp_rockchip_bus_fmt_to_reg(state->bus_format);
 	u32 val;
 
 	if (state->output_bpc == 10)
@@ -403,6 +488,9 @@ static void dw_hdmi_qp_rk3588_enc_init(struct rockchip_hdmi_qp *hdmi,
 	else
 		val = FIELD_PREP_WM16(RK3588_COLOR_DEPTH_MASK, RK3588_8BPC);
 
+	if (likely(color >= 0))
+		val |= FIELD_PREP_WM16(RK3588_COLOR_FORMAT_MASK, color);
+
 	regmap_write(hdmi->vo_regmap,
 		     hdmi->port_id ? RK3588_GRF_VO1_CON6 : RK3588_GRF_VO1_CON3,
 		     val);
@@ -513,6 +601,10 @@ static int dw_hdmi_qp_rockchip_bind(struct device *dev, struct device *master,
 	plat_data.phy_data = hdmi;
 	plat_data.max_bpc = 10;
 
+	plat_data.supported_formats = BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444) |
+				      BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444) |
+				      BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422);
+
 	encoder = &hdmi->encoder.encoder;
 	encoder->possible_crtcs = drm_of_find_possible_crtcs(drm, dev->of_node);
 
diff --git a/drivers/gpu/drm/rockchip/rk3066_hdmi.c b/drivers/gpu/drm/rockchip/rk3066_hdmi.c
index 1996f8656612d..f28d6a883e099 100644
--- a/drivers/gpu/drm/rockchip/rk3066_hdmi.c
+++ b/drivers/gpu/drm/rockchip/rk3066_hdmi.c
@@ -12,7 +12,6 @@
 #include <drm/drm_of.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include <linux/clk.h>
 #include <linux/mfd/syscon.h>
@@ -454,6 +453,10 @@ rk3066_hdmi_encoder_atomic_check(struct drm_encoder *encoder,
 	return 0;
 }
 
+static const struct drm_encoder_funcs rk3066_hdmi_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const
 struct drm_encoder_helper_funcs rk3066_hdmi_encoder_helper_funcs = {
 	.atomic_check   = rk3066_hdmi_encoder_atomic_check,
@@ -497,7 +500,7 @@ rk3066_hdmi_bridge_mode_valid(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs rk3066_hdmi_bridge_funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_enable = rk3066_hdmi_bridge_atomic_enable,
 	.atomic_disable = rk3066_hdmi_bridge_atomic_disable,
 	.detect = rk3066_hdmi_bridge_detect,
@@ -696,7 +699,8 @@ rk3066_hdmi_register(struct drm_device *drm, struct rk3066_hdmi *hdmi)
 		return -EPROBE_DEFER;
 
 	drm_encoder_helper_add(encoder, &rk3066_hdmi_encoder_helper_funcs);
-	drm_simple_encoder_init(drm, encoder, DRM_MODE_ENCODER_TMDS);
+	drm_encoder_init(drm, encoder, &rk3066_hdmi_encoder_funcs,
+			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	hdmi->bridge.driver_private = hdmi;
 	hdmi->bridge.funcs = &rk3066_hdmi_bridge_funcs;
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.h b/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
index 2e86ad00979c4..4705dc6b8bd7d 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
@@ -30,10 +30,14 @@
 #define ROCKCHIP_OUT_MODE_P565		2
 #define ROCKCHIP_OUT_MODE_BT656		5
 #define ROCKCHIP_OUT_MODE_S888		8
+#define ROCKCHIP_OUT_MODE_YUV422	9
 #define ROCKCHIP_OUT_MODE_S888_DUMMY	12
 #define ROCKCHIP_OUT_MODE_YUV420	14
 /* for use special outface */
 #define ROCKCHIP_OUT_MODE_AAAA		15
+/* SoC specific output modes */
+#define ROCKCHIP_OUT_MODE_YUV422_RK3576_DP	12
+#define ROCKCHIP_OUT_MODE_YUV422_RK3576_HDMI	13
 
 /* output flags */
 #define ROCKCHIP_OUTPUT_DSI_DUAL	BIT(0)
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
index b188539dca0b2..9a1dc9f120729 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
@@ -255,7 +255,7 @@ static int rockchip_drm_gem_object_mmap(struct drm_gem_object *obj,
 	 */
 	vm_flags_mod(vma, VM_IO | VM_DONTEXPAND | VM_DONTDUMP, VM_PFNMAP);
 
-	vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+	vma->vm_page_prot = pgprot_writecombine(vma_get_page_prot(vma));
 	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
 
 	if (rk_obj->pages)
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
index a160077a507f2..4cce3e336f5b3 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
@@ -337,7 +337,8 @@ static bool vop2_output_uv_swap(u32 bus_format, u32 output_mode)
 
 static bool vop2_output_rg_swap(struct vop2 *vop2, u32 bus_format)
 {
-	if (vop2->version == VOP_VERSION_RK3588) {
+	if (vop2->version == VOP_VERSION_RK3588 ||
+	    vop2->version == VOP_VERSION_RK3576) {
 		if (bus_format == MEDIA_BUS_FMT_YUV8_1X24 ||
 		    bus_format == MEDIA_BUS_FMT_YUV10_1X30)
 			return true;
@@ -351,6 +352,8 @@ static bool is_yuv_output(u32 bus_format)
 	switch (bus_format) {
 	case MEDIA_BUS_FMT_YUV8_1X24:
 	case MEDIA_BUS_FMT_YUV10_1X30:
+	case MEDIA_BUS_FMT_YUYV10_1X20:
+	case MEDIA_BUS_FMT_UYVY10_1X20:
 	case MEDIA_BUS_FMT_UYYVYY8_0_5X24:
 	case MEDIA_BUS_FMT_UYYVYY10_0_5X30:
 	case MEDIA_BUS_FMT_YUYV8_2X8:
@@ -658,7 +661,7 @@ static int vop2_convert_csc_mode(int csc_mode)
 	case V4L2_COLORSPACE_JPEG:
 		return CSC_BT601F;
 	case V4L2_COLORSPACE_BT2020:
-		return CSC_BT2020;
+		return CSC_BT2020L;
 	default:
 		return CSC_BT709L;
 	}
@@ -728,6 +731,89 @@ static void vop2_setup_csc_mode(struct vop2_video_port *vp,
 	vop2_win_write(win, VOP2_WIN_CSC_MODE, csc_mode);
 }
 
+/*
+ * RGB-to-YCbCr conversion based on color_to_ycbcr() and rgb2ycbcr() from
+ * drivers/media/common/v4l2-tpg/v4l2-tpg-core.c.
+ *
+ * Limited-range Y offset & chroma midpoint are expressed in 16-bit space.
+ */
+#define RGB2YUV_LIMITED_Y_OFFSET	(16 << 8)
+#define RGB2YUV_CHROMA_OFFSET		(128 << 8)
+#define COEFF(v, r)			((s32)(0.5 + (v) * (r) * 256.0))
+
+struct rgb2yuv_matrix {
+	s32 y_r,  y_g,  y_b;
+	s32 cb_r, cb_g, cb_b;
+	s32 cr_r, cr_g, cr_b;
+	s32 y_offset;
+};
+
+/* BT.601 Limited range */
+static const struct rgb2yuv_matrix rgb2yuv_bt601l = {
+	.y_r  = COEFF(0.299, 219),   .y_g  = COEFF(0.587, 219),   .y_b  = COEFF(0.114, 219),
+	.cb_r = COEFF(-0.1687, 224), .cb_g = COEFF(-0.3313, 224), .cb_b = COEFF(0.5, 224),
+	.cr_r = COEFF(0.5, 224),     .cr_g = COEFF(-0.4187, 224), .cr_b = COEFF(-0.0813, 224),
+	.y_offset = RGB2YUV_LIMITED_Y_OFFSET,
+};
+
+/* BT.601 Full range */
+static const struct rgb2yuv_matrix rgb2yuv_bt601f = {
+	.y_r  = COEFF(0.299, 255),   .y_g  = COEFF(0.587, 255),   .y_b  = COEFF(0.114, 255),
+	.cb_r = COEFF(-0.1687, 255), .cb_g = COEFF(-0.3313, 255), .cb_b = COEFF(0.5, 255),
+	.cr_r = COEFF(0.5, 255),     .cr_g = COEFF(-0.4187, 255), .cr_b = COEFF(-0.0813, 255),
+	.y_offset = 0,
+};
+
+/* BT.709 Limited range */
+static const struct rgb2yuv_matrix rgb2yuv_bt709l = {
+	.y_r  = COEFF(0.2126, 219),  .y_g  = COEFF(0.7152, 219),  .y_b  = COEFF(0.0722, 219),
+	.cb_r = COEFF(-0.1146, 224), .cb_g = COEFF(-0.3854, 224), .cb_b = COEFF(0.5, 224),
+	.cr_r = COEFF(0.5, 224),     .cr_g = COEFF(-0.4542, 224), .cr_b = COEFF(-0.0458, 224),
+	.y_offset = RGB2YUV_LIMITED_Y_OFFSET,
+};
+
+/* BT.2020 Limited range */
+static const struct rgb2yuv_matrix rgb2yuv_bt2020l = {
+	.y_r  = COEFF(0.2627, 219),  .y_g  = COEFF(0.6780, 219),  .y_b  = COEFF(0.0593, 219),
+	.cb_r = COEFF(-0.1396, 224), .cb_g = COEFF(-0.3604, 224), .cb_b = COEFF(0.5, 224),
+	.cr_r = COEFF(0.5, 224),     .cr_g = COEFF(-0.4598, 224), .cr_b = COEFF(-0.0402, 224),
+	.y_offset = RGB2YUV_LIMITED_Y_OFFSET,
+};
+
+static const struct rgb2yuv_matrix *
+vop2_rgb2yuv_get_matrix(enum vop_csc_format csc)
+{
+	switch (csc) {
+	case CSC_BT601L:
+		return &rgb2yuv_bt601l;
+	case CSC_BT601F:
+		return &rgb2yuv_bt601f;
+	case CSC_BT2020L:
+		return &rgb2yuv_bt2020l;
+	case CSC_BT709L:
+	default:
+		return &rgb2yuv_bt709l;
+	}
+}
+
+/* Convert an RGB (16bpc) to YUV444 (16bpc). */
+static void vop2_rgb16_to_yuv16(int v4l2_cs, u16 r, u16 g, u16 b,
+				u16 *y, u16 *cb, u16 *cr)
+{
+	enum vop_csc_format csc = vop2_convert_csc_mode(v4l2_cs);
+	const struct rgb2yuv_matrix *m = vop2_rgb2yuv_get_matrix(csc);
+	s64 rs = r, gs = g, bs = b;
+	s64 ys, cbs, crs;
+
+	ys  = m->y_r  * rs + m->y_g  * gs + m->y_b  * bs;
+	cbs = m->cb_r * rs + m->cb_g * gs + m->cb_b * bs;
+	crs = m->cr_r * rs + m->cr_g * gs + m->cr_b * bs;
+
+	*y  = (ys  >> 16) + m->y_offset;
+	*cb = (cbs >> 16) + RGB2YUV_CHROMA_OFFSET;
+	*cr = (crs >> 16) + RGB2YUV_CHROMA_OFFSET;
+}
+
 static void vop2_crtc_enable_irq(struct vop2_video_port *vp, u32 irq)
 {
 	struct vop2 *vop2 = vp->vop2;
@@ -1554,12 +1640,58 @@ static void vop2_dither_setup(struct drm_crtc *crtc, u32 *dsp_ctrl)
 				DITHER_DOWN_ALLEGRO);
 }
 
-static void vop2_post_config(struct drm_crtc *crtc)
+static void vop2_bgcolor_setup(struct drm_crtc *crtc, bool force,
+			       struct drm_crtc_state *new_crtc_state,
+			       struct drm_crtc_state *old_crtc_state)
+{
+	struct rockchip_crtc_state *new_vcstate = to_rockchip_crtc_state(new_crtc_state);
+	struct rockchip_crtc_state *old_vcstate = to_rockchip_crtc_state(old_crtc_state);
+	struct vop2_video_port *vp = to_vop2_video_port(crtc);
+	u64 bgcolor = new_crtc_state->background_color;
+	u16 y, cb, cr;
+	u32 val;
+
+	if (!force && old_crtc_state->background_color == bgcolor &&
+	    old_vcstate->color_space == new_vcstate->color_space)
+		return;
+
+	/*
+	 * Background color is programmed with 10 bits of precision, using YUV
+	 * format when operating in YUV overlay mode, and RGB otherwise.
+	 */
+	if (new_vcstate->yuv_overlay) {
+		vop2_rgb16_to_yuv16(new_vcstate->color_space,
+				    DRM_ARGB64_GETR(bgcolor),
+				    DRM_ARGB64_GETG(bgcolor),
+				    DRM_ARGB64_GETB(bgcolor),
+				    &y, &cb, &cr);
+
+		val = FIELD_PREP(RK3568_VP_DSP_BG__DSP_BG_RED, cr >> 6);
+		FIELD_MODIFY(RK3568_VP_DSP_BG__DSP_BG_GREEN, &val, y >> 6);
+		FIELD_MODIFY(RK3568_VP_DSP_BG__DSP_BG_BLUE, &val, cb >> 6);
+	} else {
+		/*
+		 * Since performance is more important than accuracy here, make
+		 * use of the DRM_ARGB64_GET*_BPCS() helpers.
+		 */
+		val = FIELD_PREP(RK3568_VP_DSP_BG__DSP_BG_RED,
+				 DRM_ARGB64_GETR_BPCS(bgcolor, 10));
+		FIELD_MODIFY(RK3568_VP_DSP_BG__DSP_BG_GREEN, &val,
+			     DRM_ARGB64_GETG_BPCS(bgcolor, 10));
+		FIELD_MODIFY(RK3568_VP_DSP_BG__DSP_BG_BLUE, &val,
+			     DRM_ARGB64_GETB_BPCS(bgcolor, 10));
+	}
+
+	vop2_vp_write(vp, RK3568_VP_DSP_BG, val);
+}
+
+static void vop2_post_config(struct drm_crtc *crtc, bool force,
+			     struct drm_crtc_state *new_crtc_state,
+			     struct drm_crtc_state *old_crtc_state)
 {
 	struct vop2_video_port *vp = to_vop2_video_port(crtc);
 	struct vop2 *vop2 = vp->vop2;
-	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
-	u64 bgcolor = crtc->state->background_color;
+	struct drm_display_mode *mode = &new_crtc_state->adjusted_mode;
 	u16 vtotal = mode->crtc_vtotal;
 	u16 hdisplay = mode->crtc_hdisplay;
 	u16 hact_st = mode->crtc_htotal - mode->crtc_hsync_start;
@@ -1605,15 +1737,7 @@ static void vop2_post_config(struct drm_crtc *crtc)
 		vop2_vp_write(vp, RK3568_VP_POST_DSP_VACT_INFO_F1, val);
 	}
 
-	/*
-	 * Background color is programmed with 10 bits of precision.
-	 * Since performance is more important than accuracy here,
-	 * make use of the DRM_ARGB64_GET*_BPCS() helpers.
-	 */
-	val = FIELD_PREP(RK3568_VP_DSP_BG__DSP_BG_RED, DRM_ARGB64_GETR_BPCS(bgcolor, 10));
-	FIELD_MODIFY(RK3568_VP_DSP_BG__DSP_BG_GREEN, &val, DRM_ARGB64_GETG_BPCS(bgcolor, 10));
-	FIELD_MODIFY(RK3568_VP_DSP_BG__DSP_BG_BLUE, &val, DRM_ARGB64_GETB_BPCS(bgcolor, 10));
-	vop2_vp_write(vp, RK3568_VP_DSP_BG, val);
+	vop2_bgcolor_setup(crtc, force, new_crtc_state, old_crtc_state);
 }
 
 static int us_to_vertical_line(struct drm_display_mode *mode, int us)
@@ -1628,8 +1752,9 @@ static void vop2_crtc_atomic_enable(struct drm_crtc *crtc,
 	struct vop2 *vop2 = vp->vop2;
 	const struct vop2_data *vop2_data = vop2->data;
 	const struct vop2_video_port_data *vp_data = &vop2_data->vp[vp->id];
+	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state, crtc);
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
-	struct rockchip_crtc_state *vcstate = to_rockchip_crtc_state(crtc->state);
+	struct rockchip_crtc_state *vcstate = to_rockchip_crtc_state(crtc_state);
 	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
 	unsigned long clock = mode->crtc_clock * 1000;
 	u16 hsync_len = mode->crtc_hsync_end - mode->crtc_hsync_start;
@@ -1699,6 +1824,22 @@ static void vop2_crtc_atomic_enable(struct drm_crtc *crtc,
 	if (vcstate->output_mode == ROCKCHIP_OUT_MODE_AAAA &&
 	    !(vp_data->feature & VOP2_VP_FEATURE_OUTPUT_10BIT))
 		out_mode = ROCKCHIP_OUT_MODE_P888;
+	else if (vcstate->output_mode == ROCKCHIP_OUT_MODE_YUV422 &&
+		 vop2->version == VOP_VERSION_RK3576)
+		switch (vcstate->output_type) {
+		case DRM_MODE_CONNECTOR_DisplayPort:
+		case DRM_MODE_CONNECTOR_eDP:
+			out_mode = ROCKCHIP_OUT_MODE_YUV422_RK3576_DP;
+			break;
+		case DRM_MODE_CONNECTOR_HDMIA:
+			out_mode = ROCKCHIP_OUT_MODE_YUV422_RK3576_HDMI;
+			break;
+		default:
+			drm_err(vop2->drm, "Unknown DRM_MODE_CONNECTOR %d\n",
+				vcstate->output_type);
+			vop2_unlock(vop2);
+			return;
+		}
 	else
 		out_mode = vcstate->output_mode;
 
@@ -1799,7 +1940,7 @@ static void vop2_crtc_atomic_enable(struct drm_crtc *crtc,
 
 	clk_set_rate(vp->dclk, clock);
 
-	vop2_post_config(crtc);
+	vop2_post_config(crtc, true, crtc_state, old_crtc_state);
 
 	vop2_cfg_done(vp);
 
@@ -1874,6 +2015,7 @@ static void vop2_crtc_atomic_flush(struct drm_crtc *crtc,
 				   struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
+	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state, crtc);
 	struct vop2_video_port *vp = to_vop2_video_port(crtc);
 	struct vop2 *vop2 = vp->vop2;
 
@@ -1881,7 +2023,7 @@ static void vop2_crtc_atomic_flush(struct drm_crtc *crtc,
 	if (!drm_atomic_crtc_needs_modeset(crtc_state) && crtc_state->color_mgmt_changed)
 		vop2_crtc_atomic_try_set_gamma_locked(vop2, vp, crtc, crtc_state);
 
-	vop2_post_config(crtc);
+	vop2_post_config(crtc, false, crtc_state, old_crtc_state);
 
 	vop2_cfg_done(vp);
 
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.h b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.h
index 37722652844a9..ffcb39c130aa1 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.h
@@ -373,7 +373,7 @@ enum vop_csc_format {
 	CSC_BT601L,
 	CSC_BT709L,
 	CSC_BT601F,
-	CSC_BT2020,
+	CSC_BT2020L,
 };
 
 enum src_factor_mode {
diff --git a/drivers/gpu/drm/rockchip/rockchip_lvds.c b/drivers/gpu/drm/rockchip/rockchip_lvds.c
index 7a0c4fa29f2f0..262f818752781 100644
--- a/drivers/gpu/drm/rockchip/rockchip_lvds.c
+++ b/drivers/gpu/drm/rockchip/rockchip_lvds.c
@@ -24,7 +24,6 @@
 #include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "rockchip_drm_drv.h"
 #include "rockchip_lvds.h"
@@ -119,7 +118,7 @@ static const
 struct drm_bridge_funcs rockchip_lvds_bridge_funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.get_modes = rockchip_lvds_bridge_get_modes,
 };
 
@@ -427,6 +426,10 @@ static void px30_lvds_encoder_disable(struct drm_encoder *encoder)
 	drm_panel_unprepare(lvds->panel);
 }
 
+static const struct drm_encoder_funcs rockchip_lvds_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const
 struct drm_encoder_helper_funcs rk3288_lvds_encoder_helper_funcs = {
 	.enable = rk3288_lvds_encoder_enable,
@@ -594,7 +597,9 @@ static int rockchip_lvds_bind(struct device *dev, struct device *master,
 	encoder->possible_crtcs = drm_of_find_possible_crtcs(drm_dev,
 							     dev->of_node);
 
-	ret = drm_simple_encoder_init(drm_dev, encoder, DRM_MODE_ENCODER_LVDS);
+	ret = drm_encoder_init(drm_dev, encoder,
+			       &rockchip_lvds_encoder_funcs,
+			       DRM_MODE_ENCODER_LVDS, NULL);
 	if (ret < 0) {
 		drm_err(drm_dev,
 			"failed to initialize encoder: %d\n", ret);
diff --git a/drivers/gpu/drm/rockchip/rockchip_rgb.c b/drivers/gpu/drm/rockchip/rockchip_rgb.c
index add3123e5ce70..2ad24b914989f 100644
--- a/drivers/gpu/drm/rockchip/rockchip_rgb.c
+++ b/drivers/gpu/drm/rockchip/rockchip_rgb.c
@@ -17,7 +17,6 @@
 #include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "rockchip_drm_drv.h"
 #include "rockchip_rgb.h"
@@ -65,6 +64,10 @@ rockchip_rgb_encoder_atomic_check(struct drm_encoder *encoder,
 	return 0;
 }
 
+static const struct drm_encoder_funcs rockchip_rgb_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const
 struct drm_encoder_helper_funcs rockchip_rgb_encoder_helper_funcs = {
 	.atomic_check = rockchip_rgb_encoder_atomic_check,
@@ -127,7 +130,8 @@ struct rockchip_rgb *rockchip_rgb_init(struct device *dev,
 	encoder = &rgb->encoder.encoder;
 	encoder->possible_crtcs = drm_crtc_mask(crtc);
 
-	ret = drm_simple_encoder_init(drm_dev, encoder, DRM_MODE_ENCODER_NONE);
+	ret = drm_encoder_init(drm_dev, encoder, &rockchip_rgb_encoder_funcs,
+			       DRM_MODE_ENCODER_NONE, NULL);
 	if (ret < 0) {
 		DRM_DEV_ERROR(drm_dev->dev,
 			      "failed to initialize encoder: %d\n", ret);
diff --git a/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c
index 17eda592b1833..c44648e1bc0d6 100644
--- a/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c
+++ b/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c
@@ -2114,7 +2114,7 @@ static u32 rk3568_vop2_read_layer_cfg(struct vop2 *vop2)
 	return vop2_readl(vop2, RK3568_OVL_LAYER_SEL);
 }
 
-static void rk3568_vop2_wait_for_layer_cfg_done(struct vop2 *vop2, u32 cfg)
+static void rk3568_vop2_wait_for_layer_cfg_done(struct vop2 *vop2)
 {
 	u32 atv_layer_cfg;
 	int ret;
@@ -2123,21 +2123,19 @@ static void rk3568_vop2_wait_for_layer_cfg_done(struct vop2 *vop2, u32 cfg)
 	 * Spin until the previous layer configuration is done.
 	 */
 	ret = readx_poll_timeout_atomic(rk3568_vop2_read_layer_cfg, vop2, atv_layer_cfg,
-					atv_layer_cfg == cfg, 10, 50 * 1000);
+					atv_layer_cfg == vop2->old_layer_sel, 10, 50 * 1000);
 	if (ret)
 		drm_err_ratelimited(vop2->drm, "wait layer cfg done timeout: 0x%x--0x%x\n",
-				    atv_layer_cfg, cfg);
+				    atv_layer_cfg, vop2->old_layer_sel);
 }
 
 static void rk3568_vop2_setup_layer_mixer(struct vop2_video_port *vp)
 {
 	struct vop2 *vop2 = vp->vop2;
 	struct drm_plane *plane;
-	u32 layer_sel = 0;
+	u32 layer_sel;
 	u32 port_sel;
-	u32 old_layer_sel = 0;
-	u32 atv_layer_sel = 0;
-	u32 old_port_sel = 0;
+	u32 atv_layer_sel;
 	u8 layer_id;
 	u8 old_layer_id;
 	u8 layer_sel_id;
@@ -2160,8 +2158,7 @@ static void rk3568_vop2_setup_layer_mixer(struct vop2_video_port *vp)
 	else
 		ovl_ctrl &= ~RK3568_OVL_CTRL__YUV_MODE(vp->id);
 
-	old_port_sel = vop2->old_port_sel;
-	port_sel = old_port_sel;
+	port_sel = vop2->old_port_sel;
 	port_sel &= RK3568_OVL_PORT_SEL__SEL_PORT;
 
 	if (vp0->nlayers)
@@ -2187,8 +2184,7 @@ static void rk3568_vop2_setup_layer_mixer(struct vop2_video_port *vp)
 		port_sel |= FIELD_PREP(RK3588_OVL_PORT_SET__PORT3_MUX, 7);
 
 	atv_layer_sel = vop2_readl(vop2, RK3568_OVL_LAYER_SEL);
-	old_layer_sel = vop2->old_layer_sel;
-	layer_sel = old_layer_sel;
+	layer_sel = vop2->old_layer_sel;
 
 	ofs = 0;
 	for (i = 0; i < vp->id; i++)
@@ -2272,8 +2268,6 @@ static void rk3568_vop2_setup_layer_mixer(struct vop2_video_port *vp)
 			     old_win->data->layer_sel_id[vp->id]);
 	}
 
-	vop2->old_layer_sel = layer_sel;
-	vop2->old_port_sel = port_sel;
 	/*
 	 * As the RK3568_OVL_LAYER_SEL and RK3568_OVL_PORT_SEL are shared by all Video Ports,
 	 * and the configuration take effect by one Video Port's vsync.
@@ -2288,17 +2282,8 @@ static void rk3568_vop2_setup_layer_mixer(struct vop2_video_port *vp)
 	 *    lead to the configuration of the previous VP being take effect along with the VSYNC
 	 *    of the new VP.
 	 */
-	if (layer_sel != old_layer_sel || port_sel != old_port_sel)
-		ovl_ctrl |= FIELD_PREP(RK3568_OVL_CTRL__LAYERSEL_REGDONE_SEL, vp->id);
-	vop2_writel(vop2, RK3568_OVL_CTRL, ovl_ctrl);
 
-	if (port_sel != old_port_sel) {
-		vop2_writel(vop2, RK3568_OVL_PORT_SEL, port_sel);
-		vop2_cfg_done(vp);
-		rk3568_vop2_wait_for_port_mux_done(vop2);
-	}
-
-	if (layer_sel != old_layer_sel && atv_layer_sel != old_layer_sel) {
+	if (layer_sel != vop2->old_layer_sel && atv_layer_sel != vop2->old_layer_sel) {
 		cfg_done = vop2_readl(vop2, RK3568_REG_CFG_DONE);
 		cfg_done &= (BIT(vop2->data->nr_vps) - 1);
 		cfg_done &= ~BIT(vp->id);
@@ -2306,10 +2291,23 @@ static void rk3568_vop2_setup_layer_mixer(struct vop2_video_port *vp)
 		 * Changes of other VPs' overlays have not taken effect
 		 */
 		if (cfg_done)
-			rk3568_vop2_wait_for_layer_cfg_done(vop2, vop2->old_layer_sel);
+			rk3568_vop2_wait_for_layer_cfg_done(vop2);
+	}
+
+	if (layer_sel != vop2->old_layer_sel || port_sel != vop2->old_port_sel)
+		ovl_ctrl |= FIELD_PREP(RK3568_OVL_CTRL__LAYERSEL_REGDONE_SEL, vp->id);
+	vop2_writel(vop2, RK3568_OVL_CTRL, ovl_ctrl);
+
+	if (port_sel != vop2->old_port_sel) {
+		vop2->old_port_sel = port_sel;
+		vop2_writel(vop2, RK3568_OVL_PORT_SEL, port_sel);
+		vop2_cfg_done(vp);
+		rk3568_vop2_wait_for_port_mux_done(vop2);
 	}
 
+	vop2->old_layer_sel = layer_sel;
 	vop2_writel(vop2, RK3568_OVL_LAYER_SEL, layer_sel);
+
 	mutex_unlock(&vop2->ovl_lock);
 }
 
diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c
index 4ebb513255edd..c51101ec70c16 100644
--- a/drivers/gpu/drm/scheduler/sched_entity.c
+++ b/drivers/gpu/drm/scheduler/sched_entity.c
@@ -129,7 +129,6 @@ int drm_sched_entity_init(struct drm_sched_entity *entity,
 		return -ENOMEM;
 
 	INIT_LIST_HEAD(&entity->list);
-	entity->rq = NULL;
 	entity->guilty = guilty;
 	entity->priority = priority;
 	entity->last_user = current->group_leader;
@@ -280,9 +279,6 @@ void drm_sched_entity_kill(struct drm_sched_entity *entity)
 	struct drm_sched_job *job;
 	struct dma_fence *prev;
 
-	if (!entity->rq)
-		return;
-
 	spin_lock(&entity->lock);
 	entity->stopped = true;
 	drm_sched_rq_remove_entity(entity->rq, entity);
@@ -329,14 +325,11 @@ EXPORT_SYMBOL(drm_sched_entity_kill);
  */
 long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout)
 {
-	struct drm_gpu_scheduler *sched;
+	struct drm_gpu_scheduler *sched =
+		container_of(entity->rq, typeof(*sched), rq);
 	struct task_struct *last_user;
 	long ret = timeout;
 
-	if (!entity->rq)
-		return 0;
-
-	sched = container_of(entity->rq, typeof(*sched), rq);
 	/*
 	 * The client will not queue more jobs during this fini - consume
 	 * existing queued ones, or discard them on SIGKILL.
diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
index 818d3d4434b57..d2ca01b31ee49 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -588,15 +588,6 @@ int drm_sched_job_init(struct drm_sched_job *job,
 		       u32 credits, void *owner,
 		       uint64_t drm_client_id)
 {
-	if (!entity->rq) {
-		/* This will most likely be followed by missing frames
-		 * or worse--a blank screen--leave a trail in the
-		 * logs, so this can be debugged easier.
-		 */
-		dev_err(job->sched->dev, "%s: entity has no rq!\n", __func__);
-		return -ENOENT;
-	}
-
 	if (unlikely(!credits)) {
 		pr_err("*ERROR* %s: credits cannot be 0!\n", __func__);
 		return -EINVAL;
diff --git a/drivers/gpu/drm/scheduler/tests/tests_basic.c b/drivers/gpu/drm/scheduler/tests/tests_basic.c
index a5a5a35a87b0d..da5c4952780f3 100644
--- a/drivers/gpu/drm/scheduler/tests/tests_basic.c
+++ b/drivers/gpu/drm/scheduler/tests/tests_basic.c
@@ -1,7 +1,11 @@
 // SPDX-License-Identifier: GPL-2.0
 /* Copyright (c) 2025 Valve Corporation */
 
+#include <linux/completion.h>
 #include <linux/delay.h>
+#include <linux/minmax.h>
+#include <linux/time.h>
+#include <linux/workqueue.h>
 
 #include "sched_tests.h"
 
@@ -235,6 +239,360 @@ static void drm_sched_basic_cancel(struct kunit *test)
 	KUNIT_ASSERT_EQ(test, job->hw_fence.error, -ECANCELED);
 }
 
+struct sched_concurrent_context {
+	struct drm_mock_scheduler *sched;
+	struct workqueue_struct *sub_wq;
+	struct kunit *test;
+	struct completion wait_go;
+};
+
+KUNIT_DEFINE_ACTION_WRAPPER(drm_mock_sched_fini_wrap, drm_mock_sched_fini,
+			    struct drm_mock_scheduler *);
+
+KUNIT_DEFINE_ACTION_WRAPPER(drm_mock_sched_entity_free_wrap, drm_mock_sched_entity_free,
+			    struct drm_mock_sched_entity *);
+
+static void complete_destroy_workqueue(void *context)
+{
+	struct sched_concurrent_context *ctx = context;
+
+	complete_all(&ctx->wait_go);
+
+	destroy_workqueue(ctx->sub_wq);
+}
+
+static int drm_sched_concurrent_init(struct kunit *test)
+{
+	struct sched_concurrent_context *ctx;
+	int ret;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	init_completion(&ctx->wait_go);
+
+	ctx->sched = drm_mock_sched_new(test, MAX_SCHEDULE_TIMEOUT);
+
+	ret = kunit_add_action_or_reset(test, drm_mock_sched_fini_wrap, ctx->sched);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	/* Use an unbounded workqueue to maximize job submission concurrency */
+	ctx->sub_wq = alloc_workqueue("drm-sched-submitters-wq", WQ_UNBOUND,
+				      WQ_UNBOUND_MAX_ACTIVE);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->sub_wq);
+
+	ret = kunit_add_action_or_reset(test, complete_destroy_workqueue, ctx);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	ctx->test = test;
+	test->priv = ctx;
+
+	return 0;
+}
+
+struct drm_sched_parallel_params {
+	const char *description;
+	unsigned int num_jobs;
+	unsigned int num_workers;
+};
+
+static const struct drm_sched_parallel_params drm_sched_parallel_cases[] = {
+	{
+		.description = "Parallel submission of multiple jobs per worker",
+		.num_jobs = 8,
+		.num_workers = 16,
+	},
+};
+
+static void
+drm_sched_parallel_desc(const struct drm_sched_parallel_params *params, char *desc)
+{
+	strscpy(desc, params->description, KUNIT_PARAM_DESC_SIZE);
+}
+
+KUNIT_ARRAY_PARAM(drm_sched_parallel, drm_sched_parallel_cases, drm_sched_parallel_desc);
+
+struct parallel_worker {
+	struct work_struct work;
+	struct sched_concurrent_context *ctx;
+	struct drm_mock_sched_entity *entity;
+	struct drm_mock_sched_job **jobs;
+	unsigned int id;
+};
+
+static void drm_sched_parallel_worker(struct work_struct *work)
+{
+	const struct drm_sched_parallel_params *params;
+	struct sched_concurrent_context *test_ctx;
+	struct parallel_worker *worker;
+	unsigned int i;
+
+	worker = container_of(work, struct parallel_worker, work);
+	test_ctx = worker->ctx;
+	params = test_ctx->test->param_value;
+
+	wait_for_completion(&test_ctx->wait_go);
+
+	kunit_info(test_ctx->test, "Parallel worker %u submitting %u jobs started\n",
+		   worker->id, params->num_jobs);
+
+	for (i = 0; i < params->num_jobs; i++)
+		drm_mock_sched_job_submit(worker->jobs[i]);
+}
+
+/*
+ * Spawns workers that submit a sequence of jobs to the mock scheduler.
+ * Once all jobs are submitted, the timeline is manually advanced.
+ */
+static void drm_sched_parallel_submit_test(struct kunit *test)
+{
+	struct sched_concurrent_context *ctx = test->priv;
+	const struct drm_sched_parallel_params *params = test->param_value;
+	struct parallel_worker *workers, *worker;
+	struct drm_mock_sched_job *job;
+	unsigned int i, j, completed_jobs, total_jobs;
+	bool done;
+	int ret;
+
+	KUNIT_ASSERT_GT(test, params->num_workers, 0);
+	KUNIT_ASSERT_GT(test, params->num_jobs, 0);
+
+	workers = kunit_kcalloc(test, params->num_workers, sizeof(*workers),
+				GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, workers);
+
+	/*
+	 * Init workers only after all jobs and entities have been successfully
+	 * allocated. In this way, the cleanup logic for when an assertion fail
+	 * can be simplified.
+	 */
+	for (i = 0; i < params->num_workers; i++) {
+		worker = &workers[i];
+		worker->id = i;
+		worker->ctx = ctx;
+		worker->entity = drm_mock_sched_entity_new(test,
+							   DRM_SCHED_PRIORITY_NORMAL,
+							   ctx->sched);
+
+		worker->jobs = kunit_kcalloc(test, params->num_jobs,
+					     sizeof(*worker->jobs), GFP_KERNEL);
+		KUNIT_ASSERT_NOT_NULL(test, worker->jobs);
+
+		for (j = 0; j < params->num_jobs; j++) {
+			job = drm_mock_sched_job_new(test, worker->entity);
+			worker->jobs[j] = job;
+		}
+
+		ret = kunit_add_action_or_reset(test, drm_mock_sched_entity_free_wrap,
+						worker->entity);
+		KUNIT_ASSERT_EQ(test, ret, 0);
+	}
+
+	for (i = 0; i < params->num_workers; i++) {
+		worker = &workers[i];
+		INIT_WORK(&worker->work, drm_sched_parallel_worker);
+		queue_work(ctx->sub_wq, &worker->work);
+	}
+
+	complete_all(&ctx->wait_go);
+	flush_workqueue(ctx->sub_wq);
+
+	for (i = 0; i < params->num_workers; i++) {
+		worker = &workers[i];
+		for (j = 0; j < params->num_jobs; j++) {
+			job = worker->jobs[j];
+			done = drm_mock_sched_job_wait_scheduled(job, HZ);
+			KUNIT_EXPECT_TRUE(test, done);
+		}
+	}
+
+	total_jobs = params->num_workers * params->num_jobs;
+	completed_jobs = drm_mock_sched_advance(ctx->sched, total_jobs);
+	KUNIT_EXPECT_EQ(test, completed_jobs, total_jobs);
+
+	for (i = 0; i < params->num_workers; i++) {
+		worker = &workers[i];
+		for (j = 0; j < params->num_jobs; j++) {
+			job = worker->jobs[j];
+			done = drm_mock_sched_job_wait_finished(job, HZ);
+			KUNIT_EXPECT_TRUE(test, done);
+		}
+	}
+}
+
+struct drm_sched_interleaved_params {
+	const char *description;
+	unsigned int test_duration_ms;
+	unsigned int job_base_duration_us;
+	unsigned int num_workers;
+	unsigned int num_in_flight_jobs;
+};
+
+static const struct drm_sched_interleaved_params drm_sched_interleaved_cases[] = {
+	{
+		.description = "Interleaved submission of multiple jobs per worker",
+		.test_duration_ms = 1000,
+		.job_base_duration_us = 100,
+		.num_workers = 16,
+		.num_in_flight_jobs = 8,
+	},
+};
+
+static void
+drm_sched_interleaved_desc(const struct drm_sched_interleaved_params *params, char *desc)
+{
+	strscpy(desc, params->description, KUNIT_PARAM_DESC_SIZE);
+}
+
+KUNIT_ARRAY_PARAM(drm_sched_interleaved, drm_sched_interleaved_cases,
+		  drm_sched_interleaved_desc);
+
+struct interleaved_worker {
+	struct work_struct work;
+	struct sched_concurrent_context *ctx;
+	struct drm_mock_sched_entity *entity;
+	struct drm_mock_sched_job **jobs;
+	unsigned int id;
+	unsigned int job_count;
+	unsigned int job_duration_us;
+};
+
+static void drm_sched_interleaved_worker(struct work_struct *work)
+{
+	struct sched_concurrent_context *test_ctx;
+	const struct drm_sched_interleaved_params *params;
+	struct interleaved_worker *worker;
+	unsigned int i, j, max_in_flight_job;
+	unsigned long timeout;
+	bool done;
+
+	worker = container_of(work, struct interleaved_worker, work);
+	test_ctx = worker->ctx;
+	params = test_ctx->test->param_value;
+
+	wait_for_completion(&test_ctx->wait_go);
+
+	kunit_info(test_ctx->test, "Worker %u submitting %u jobs of %u us started\n",
+		   worker->id, worker->job_count, worker->job_duration_us);
+
+	timeout = msecs_to_jiffies(params->test_duration_ms * 2);
+
+	/* Fill the submission window */
+	max_in_flight_job = min(worker->job_count, params->num_in_flight_jobs);
+	for (i = 0; i < max_in_flight_job; i++)
+		drm_mock_sched_job_submit(worker->jobs[i]);
+
+	/* Keep the window full by submitting a new job at once until done */
+	for (i = 0; i < worker->job_count; i++) {
+		done = drm_mock_sched_job_wait_finished(worker->jobs[i], timeout);
+		if (!done)
+			kunit_info(test_ctx->test, "Job %u of worker %u timed out\n",
+				   i, worker->id);
+
+		j = i + max_in_flight_job;
+		if (j < worker->job_count)
+			drm_mock_sched_job_submit(worker->jobs[j]);
+	}
+}
+
+/*
+ * Spawns workers that submit a sequence of jobs to the mock scheduler. Job
+ * durations are chosen as multiples of a base duration value specified as
+ * a test parameter. Since the scheduler serializes jobs from all workers,
+ * the total test duration budget is divided into equal shares among workers.
+ * These shares are then used to compute the number of jobs that each worker
+ * can submit.
+ */
+static void drm_sched_interleaved_submit_test(struct kunit *test)
+{
+	const struct drm_sched_interleaved_params *params = test->param_value;
+	struct sched_concurrent_context *ctx = test->priv;
+	struct interleaved_worker *workers, *worker;
+	struct drm_mock_sched_job *job;
+	unsigned int worker_share_us;
+	unsigned int i, j;
+	bool done;
+	int ret;
+
+	KUNIT_ASSERT_GT(test, params->num_workers, 0);
+	KUNIT_ASSERT_GT(test, params->job_base_duration_us, 0);
+
+	workers = kunit_kcalloc(test, params->num_workers, sizeof(*workers),
+				GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, workers);
+
+	/* Divide the available test time into equal shares among the workers */
+	worker_share_us = (params->test_duration_ms * USEC_PER_MSEC) /
+			  params->num_workers;
+
+	/*
+	 * Init workers only after all jobs and entities have been successfully
+	 * allocated. In this way, the cleanup logic for when an assertion fails
+	 * can be simplified.
+	 */
+	for (i = 0; i < params->num_workers; i++) {
+		worker = &workers[i];
+		worker->id = i;
+		worker->ctx = ctx;
+
+		worker->job_duration_us = params->job_base_duration_us * (i + 1);
+		worker->job_count = worker_share_us / worker->job_duration_us;
+		worker->job_count = max(1U, worker->job_count);
+
+		worker->entity = drm_mock_sched_entity_new(test,
+							   DRM_SCHED_PRIORITY_NORMAL,
+							   ctx->sched);
+
+		worker->jobs = kunit_kcalloc(test, worker->job_count,
+					     sizeof(*worker->jobs), GFP_KERNEL);
+		KUNIT_ASSERT_NOT_NULL(test, worker->jobs);
+
+		for (j = 0; j < worker->job_count; j++) {
+			job = drm_mock_sched_job_new(test, worker->entity);
+			drm_mock_sched_job_set_duration_us(job, worker->job_duration_us);
+
+			worker->jobs[j] = job;
+		}
+
+		ret = kunit_add_action_or_reset(test, drm_mock_sched_entity_free_wrap,
+						worker->entity);
+		KUNIT_ASSERT_EQ(test, ret, 0);
+	}
+
+	for (i = 0; i < params->num_workers; i++) {
+		worker = &workers[i];
+		INIT_WORK(&worker->work, drm_sched_interleaved_worker);
+		queue_work(ctx->sub_wq, &worker->work);
+	}
+
+	complete_all(&ctx->wait_go);
+	flush_workqueue(ctx->sub_wq);
+
+	for (i = 0; i < params->num_workers; i++) {
+		worker = &workers[i];
+		for (j = 0; j < worker->job_count; j++) {
+			job = worker->jobs[j];
+			done = drm_mock_sched_job_is_finished(job);
+			KUNIT_EXPECT_TRUE(test, done);
+		}
+	}
+}
+
+static struct kunit_case drm_sched_concurrent_tests[] = {
+	KUNIT_CASE_PARAM(drm_sched_parallel_submit_test, drm_sched_parallel_gen_params),
+	KUNIT_CASE_PARAM(drm_sched_interleaved_submit_test, drm_sched_interleaved_gen_params),
+	{}
+};
+
+static struct kunit_suite drm_sched_concurrent = {
+	.name = "drm_sched_concurrent_tests",
+	.init = drm_sched_concurrent_init,
+	.test_cases = drm_sched_concurrent_tests,
+	.attr = {
+		.speed = KUNIT_SPEED_SLOW,
+	},
+};
+
 static struct kunit_case drm_sched_cancel_tests[] = {
 	KUNIT_CASE(drm_sched_basic_cancel),
 	{}
@@ -556,6 +914,7 @@ static struct kunit_suite drm_sched_credits = {
 };
 
 kunit_test_suites(&drm_sched_basic,
+		  &drm_sched_concurrent,
 		  &drm_sched_timeout,
 		  &drm_sched_cancel,
 		  &drm_sched_priority,
diff --git a/drivers/gpu/drm/scheduler/tests/tests_scheduler.c b/drivers/gpu/drm/scheduler/tests/tests_scheduler.c
index 90d31888cf924..285546a2218f8 100644
--- a/drivers/gpu/drm/scheduler/tests/tests_scheduler.c
+++ b/drivers/gpu/drm/scheduler/tests/tests_scheduler.c
@@ -599,7 +599,7 @@ static void drm_sched_scheduler_two_clients_test(struct kunit *test)
 	 *  -  qd: Number of outstanding jobs in the client/entity
 	 */
 
-	pr_info(" [pct] - Job sumission progress\n"
+	pr_info(" [pct] - Job submission progress\n"
 " [cps] - Cycles per second\n"
 "  [qd] - Number of outstanding jobs in the client/entity\n");
 	pr_info("%s:\n\t            pct1 cps1 qd1;  pct2 cps2 qd2\n",
diff --git a/drivers/gpu/drm/solomon/ssd130x.c b/drivers/gpu/drm/solomon/ssd130x.c
index cae92a3ae8a41..0940eae7a2e49 100644
--- a/drivers/gpu/drm/solomon/ssd130x.c
+++ b/drivers/gpu/drm/solomon/ssd130x.c
@@ -276,6 +276,49 @@ static int ssd130x_write_cmd(struct ssd130x_device *ssd130x, int count,
 	return ret;
 }
 
+/*
+ * Write a command byte sequence from a buffer.
+ *
+ * Like ssd130x_write_cmd() but takes a pre-built byte array instead of
+ * variadic arguments, handy when the command is already in an array or
+ * when the caller wants to use sizeof() for the length.
+ */
+static int ssd130x_write_cmds(struct ssd130x_device *ssd130x, const u8 *cmd,
+			      size_t len)
+{
+	unsigned int i;
+	int ret;
+
+	for (i = 0; i < len; i++) {
+		ret = regmap_write(ssd130x->regmap, SSD13XX_COMMAND, cmd[i]);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/*
+ * Run a packed command sequence.  The format is a flat byte array where each
+ * entry starts with a length byte followed by that many command bytes.  A
+ * zero length byte terminates the sequence.
+ *
+ * Example: { 2, 0x81, 0x80, 1, 0xAF, 0 }
+ *   sends command {0x81, 0x80}, then command {0xAF}, then stops.
+ */
+static int ssd130x_run_cmd_seq(struct ssd130x_device *ssd130x, const u8 *seq)
+{
+	while (*seq) {
+		u8 len = *seq++;
+		int ret = ssd130x_write_cmds(ssd130x, seq, len);
+
+		if (ret)
+			return ret;
+		seq += len;
+	}
+
+	return 0;
+}
 /* Set address range for horizontal/vertical addressing modes */
 static int ssd130x_set_col_range(struct ssd130x_device *ssd130x,
 				 u8 col_start, u8 cols)
@@ -408,39 +451,26 @@ static int ssd130x_init(struct ssd130x_device *ssd130x)
 	bool scan_mode;
 	int ret;
 
-	/* Set initial contrast */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD13XX_CONTRAST, ssd130x->contrast);
-	if (ret < 0)
-		return ret;
-
 	/* Set segment re-map */
 	seg_remap = (SSD13XX_SET_SEG_REMAP |
 		     SSD13XX_SET_SEG_REMAP_SET(ssd130x->seg_remap));
-	ret = ssd130x_write_cmd(ssd130x, 1, seg_remap);
-	if (ret < 0)
-		return ret;
-
 	/* Set COM direction */
 	com_invdir = (SSD130X_SET_COM_SCAN_DIR |
 		      SSD130X_SET_COM_SCAN_DIR_SET(ssd130x->com_invdir));
-	ret = ssd130x_write_cmd(ssd130x,  1, com_invdir);
-	if (ret < 0)
-		return ret;
-
-	/* Set multiplex ratio value */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD13XX_SET_MULTIPLEX_RATIO, ssd130x->height - 1);
-	if (ret < 0)
-		return ret;
-
-	/* set display offset value */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD130X_SET_DISPLAY_OFFSET, ssd130x->com_offset);
-	if (ret < 0)
-		return ret;
-
 	/* Set clock frequency */
 	dclk = (SSD130X_SET_CLOCK_DIV_SET(ssd130x->dclk_div - 1) |
 		SSD130X_SET_CLOCK_FREQ_SET(ssd130x->dclk_frq));
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD130X_SET_CLOCK_FREQ, dclk);
+
+	const u8 *cmds = (const u8[]) {
+		2, SSD13XX_CONTRAST, ssd130x->contrast,
+		1, seg_remap,
+		1, com_invdir, 0,
+		2, SSD13XX_SET_MULTIPLEX_RATIO, ssd130x->height - 1,
+		2, SSD130X_SET_DISPLAY_OFFSET, ssd130x->com_offset,
+		2, SSD130X_SET_CLOCK_FREQ, dclk,
+		0,
+	};
+	ret = ssd130x_run_cmd_seq(ssd130x, cmds);
 	if (ret < 0)
 		return ret;
 
@@ -462,9 +492,6 @@ static int ssd130x_init(struct ssd130x_device *ssd130x)
 	/* Set precharge period in number of ticks from the internal clock */
 	precharge = (SSD130X_SET_PRECHARGE_PERIOD1_SET(ssd130x->prechargep1) |
 		     SSD130X_SET_PRECHARGE_PERIOD2_SET(ssd130x->prechargep2));
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD130X_SET_PRECHARGE_PERIOD, precharge);
-	if (ret < 0)
-		return ret;
 
 	/* Set COM pins configuration */
 	compins = BIT(1);
@@ -476,22 +503,20 @@ static int ssd130x_init(struct ssd130x_device *ssd130x)
 	scan_mode = !ssd130x->com_seq;
 	compins |= (SSD130X_SET_COM_PINS_CONFIG1_SET(scan_mode) |
 		    SSD130X_SET_COM_PINS_CONFIG2_SET(ssd130x->com_lrremap));
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD130X_SET_COM_PINS_CONFIG, compins);
-	if (ret < 0)
-		return ret;
-
-	/* Set VCOMH */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD130X_SET_VCOMH, ssd130x->vcomh);
-	if (ret < 0)
-		return ret;
 
 	/* Turn on the DC-DC Charge Pump */
 	chargepump = BIT(4);
-
 	if (ssd130x->device_info->need_chargepump)
 		chargepump |= BIT(2);
 
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD130X_CHARGE_PUMP, chargepump);
+	cmds = (const u8[]) {
+		2, SSD130X_SET_PRECHARGE_PERIOD, precharge,
+		2, SSD130X_SET_COM_PINS_CONFIG, compins,
+		2, SSD130X_SET_VCOMH, ssd130x->vcomh,
+		2, SSD130X_CHARGE_PUMP, chargepump,
+		0
+	};
+	ret = ssd130x_run_cmd_seq(ssd130x, cmds);
 	if (ret < 0)
 		return ret;
 
@@ -528,203 +553,68 @@ static int ssd130x_init(struct ssd130x_device *ssd130x)
 
 static int ssd132x_init(struct ssd130x_device *ssd130x)
 {
-	int ret;
-
-	/* Set initial contrast */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD13XX_CONTRAST, 0x80);
-	if (ret < 0)
-		return ret;
-
-	/* Set column start and end */
-	ret = ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_COL_RANGE, 0x00,
-				ssd130x->width / SSD132X_SEGMENT_WIDTH - 1);
-	if (ret < 0)
-		return ret;
-
-	/* Set row start and end */
-	ret = ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_ROW_RANGE, 0x00, ssd130x->height - 1);
-	if (ret < 0)
-		return ret;
-	/*
-	 * Horizontal Address Increment
-	 * Re-map for Column Address, Nibble and COM
-	 * COM Split Odd Even
-	 */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD13XX_SET_SEG_REMAP, 0x53);
-	if (ret < 0)
-		return ret;
-
-	/* Set display start and offset */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD132X_SET_DISPLAY_START, 0x00);
-	if (ret < 0)
-		return ret;
-
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD132X_SET_DISPLAY_OFFSET, 0x00);
-	if (ret < 0)
-		return ret;
-
-	/* Set display mode normal */
-	ret = ssd130x_write_cmd(ssd130x, 1, SSD132X_SET_DISPLAY_NORMAL);
-	if (ret < 0)
-		return ret;
-
-	/* Set multiplex ratio value */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD13XX_SET_MULTIPLEX_RATIO, ssd130x->height - 1);
-	if (ret < 0)
-		return ret;
-
-	/* Set phase length */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD132X_SET_PHASE_LENGTH, 0x55);
-	if (ret < 0)
-		return ret;
-
-	/* Select default linear gray scale table */
-	ret = ssd130x_write_cmd(ssd130x, 1, SSD132X_SELECT_DEFAULT_TABLE);
-	if (ret < 0)
-		return ret;
-
-	/* Set clock frequency */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD132X_SET_CLOCK_FREQ, 0x01);
-	if (ret < 0)
-		return ret;
-
-	/* Enable internal VDD regulator */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD132X_SET_FUNCTION_SELECT_A, 0x1);
-	if (ret < 0)
-		return ret;
-
-	/* Set pre-charge period */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD132X_SET_PRECHARGE_PERIOD, 0x01);
-	if (ret < 0)
-		return ret;
-
-	/* Set pre-charge voltage */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD132X_SET_PRECHARGE_VOLTAGE, 0x08);
-	if (ret < 0)
-		return ret;
-
-	/* Set VCOMH voltage */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD130X_SET_VCOMH_VOLTAGE, 0x07);
-	if (ret < 0)
-		return ret;
-
-	/* Enable second pre-charge and internal VSL */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD132X_SET_FUNCTION_SELECT_B, 0x62);
-	if (ret < 0)
-		return ret;
-
-	return 0;
+	const u8 cmds[] = {
+		2, SSD13XX_CONTRAST, 0x80,
+		3, SSD132X_SET_COL_RANGE, 0x00, ssd130x->width / SSD132X_SEGMENT_WIDTH - 1,
+		3, SSD132X_SET_ROW_RANGE, 0x00, ssd130x->height - 1,
+		/*
+		 * Horizontal Address Increment
+		 * Re-map for Column Address, Nibble and COM
+		 * COM Split Odd Even
+		 */
+		2, SSD13XX_SET_SEG_REMAP, 0x53,
+		2, SSD132X_SET_DISPLAY_START, 0x00,
+		2, SSD132X_SET_DISPLAY_OFFSET, 0x00,
+		1, SSD132X_SET_DISPLAY_NORMAL,
+		2, SSD13XX_SET_MULTIPLEX_RATIO, ssd130x->height - 1,
+		2, SSD132X_SET_PHASE_LENGTH, 0x55,
+		1, SSD132X_SELECT_DEFAULT_TABLE,
+		2, SSD132X_SET_CLOCK_FREQ, 0x01,
+		2, SSD132X_SET_FUNCTION_SELECT_A, 0x1,
+		2, SSD132X_SET_PRECHARGE_PERIOD, 0x01,
+		2, SSD132X_SET_PRECHARGE_VOLTAGE, 0x08,
+		2, SSD130X_SET_VCOMH_VOLTAGE, 0x07,
+		/* Enable second pre-charge and internal VSL */
+		2, SSD132X_SET_FUNCTION_SELECT_B, 0x62,
+		0,
+	};
+
+	return ssd130x_run_cmd_seq(ssd130x, cmds);
 }
 
 static int ssd133x_init(struct ssd130x_device *ssd130x)
 {
-	int ret;
-
-	/* Set color A contrast */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD133X_CONTRAST_A, 0x91);
-	if (ret < 0)
-		return ret;
-
-	/* Set color B contrast */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD133X_CONTRAST_B, 0x50);
-	if (ret < 0)
-		return ret;
-
-	/* Set color C contrast */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD133X_CONTRAST_C, 0x7d);
-	if (ret < 0)
-		return ret;
-
-	/* Set master current */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD133X_SET_MASTER_CURRENT, 0x06);
-	if (ret < 0)
-		return ret;
-
-	/* Set column start and end */
-	ret = ssd130x_write_cmd(ssd130x, 3, SSD133X_SET_COL_RANGE, 0x00, ssd130x->width - 1);
-	if (ret < 0)
-		return ret;
-
-	/* Set row start and end */
-	ret = ssd130x_write_cmd(ssd130x, 3, SSD133X_SET_ROW_RANGE, 0x00, ssd130x->height - 1);
-	if (ret < 0)
-		return ret;
-
-	/*
-	 * Horizontal Address Increment
-	 * Normal order SA,SB,SC (e.g. RGB)
-	 * COM Split Odd Even
-	 * 256 color format
-	 */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD13XX_SET_SEG_REMAP, 0x20);
-	if (ret < 0)
-		return ret;
-
-	/* Set display start and offset */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD133X_SET_DISPLAY_START, 0x00);
-	if (ret < 0)
-		return ret;
-
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD133X_SET_DISPLAY_OFFSET, 0x00);
-	if (ret < 0)
-		return ret;
-
-	/* Set display mode normal */
-	ret = ssd130x_write_cmd(ssd130x, 1, SSD133X_SET_DISPLAY_NORMAL);
-	if (ret < 0)
-		return ret;
-
-	/* Set multiplex ratio value */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD13XX_SET_MULTIPLEX_RATIO, ssd130x->height - 1);
-	if (ret < 0)
-		return ret;
-
-	/* Set master configuration */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD133X_SET_MASTER_CONFIG, 0x8e);
-	if (ret < 0)
-		return ret;
-
-	/* Set power mode */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD133X_POWER_SAVE_MODE, 0x0b);
-	if (ret < 0)
-		return ret;
-
-	/* Set Phase 1 and 2 period */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD133X_PHASES_PERIOD, 0x31);
-	if (ret < 0)
-		return ret;
-
-	/* Set clock divider */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD133X_SET_CLOCK_FREQ, 0xf0);
-	if (ret < 0)
-		return ret;
-
-	/* Set pre-charge A */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD132X_SET_PRECHARGE_A, 0x64);
-	if (ret < 0)
-		return ret;
-
-	/* Set pre-charge B */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD132X_SET_PRECHARGE_B, 0x78);
-	if (ret < 0)
-		return ret;
-
-	/* Set pre-charge C */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD132X_SET_PRECHARGE_C, 0x64);
-	if (ret < 0)
-		return ret;
-
-	/* Set pre-charge level */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD133X_SET_PRECHARGE_VOLTAGE, 0x3a);
-	if (ret < 0)
-		return ret;
-
-	/* Set VCOMH voltage */
-	ret = ssd130x_write_cmd(ssd130x, 2, SSD133X_SET_VCOMH_VOLTAGE, 0x3e);
-	if (ret < 0)
-		return ret;
-
-	return 0;
+	const u8 cmds[] = {
+		2, SSD133X_CONTRAST_A, 0x91,
+		2, SSD133X_CONTRAST_B, 0x50,
+		2, SSD133X_CONTRAST_C, 0x7d,
+		2, SSD133X_SET_MASTER_CURRENT, 0x06,
+		3, SSD133X_SET_COL_RANGE, 0x00, ssd130x->width - 1,
+		3, SSD133X_SET_ROW_RANGE, 0x00, ssd130x->height - 1,
+		/*
+		 * Horizontal Address Increment
+		 * Normal order SA,SB,SC (e.g. RGB)
+		 * COM Split Odd Even
+		 * 256 color format
+		 */
+		2, SSD13XX_SET_SEG_REMAP, 0x20,
+		2, SSD133X_SET_DISPLAY_START, 0x00,
+		2, SSD133X_SET_DISPLAY_OFFSET, 0x00,
+		1, SSD133X_SET_DISPLAY_NORMAL,
+		2, SSD13XX_SET_MULTIPLEX_RATIO, ssd130x->height - 1,
+		2, SSD133X_SET_MASTER_CONFIG, 0x8e,
+		2, SSD133X_POWER_SAVE_MODE, 0x0b,
+		2, SSD133X_PHASES_PERIOD, 0x31,
+		2, SSD133X_SET_CLOCK_FREQ, 0xf0,
+		2, SSD132X_SET_PRECHARGE_A, 0x64,
+		2, SSD132X_SET_PRECHARGE_B, 0x78,
+		2, SSD132X_SET_PRECHARGE_C, 0x64,
+		2, SSD133X_SET_PRECHARGE_VOLTAGE, 0x3a,
+		2, SSD133X_SET_VCOMH_VOLTAGE, 0x3e,
+		0,
+	};
+
+	return ssd130x_run_cmd_seq(ssd130x, cmds);
 }
 
 static int ssd130x_update_rect(struct ssd130x_device *ssd130x,
@@ -835,9 +725,9 @@ static int ssd132x_update_rect(struct ssd130x_device *ssd130x,
 			       struct drm_rect *rect, u8 *buf,
 			       u8 *data_array)
 {
-	unsigned int x = rect->x1;
-	unsigned int y = rect->y1;
 	unsigned int segment_width = SSD132X_SEGMENT_WIDTH;
+	unsigned int col = rect->x1 / segment_width;
+	unsigned int row = rect->y1;
 	unsigned int width = drm_rect_width(rect);
 	unsigned int height = drm_rect_height(rect);
 	unsigned int columns = DIV_ROUND_UP(width, segment_width);
@@ -847,7 +737,7 @@ static int ssd132x_update_rect(struct ssd130x_device *ssd130x,
 	unsigned int i, j;
 	int ret;
 
-	drm_WARN_ONCE(drm, x % segment_width != 0, "x must be aligned to screen segment\n");
+	drm_WARN_ONCE(drm, rect->x1 % segment_width != 0, "x must be aligned to screen segment\n");
 
 	/*
 	 * The screen is divided in Segment and Common outputs, where
@@ -864,12 +754,12 @@ static int ssd132x_update_rect(struct ssd130x_device *ssd130x,
 	 */
 
 	/* Set column start and end */
-	ret = ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_COL_RANGE, x / segment_width, columns - 1);
+	ret = ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_COL_RANGE, col, col + columns - 1);
 	if (ret < 0)
 		return ret;
 
 	/* Set row start and end */
-	ret = ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_ROW_RANGE, y, rows - 1);
+	ret = ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_ROW_RANGE, row, row + rows - 1);
 	if (ret < 0)
 		return ret;
 
@@ -915,12 +805,12 @@ static int ssd133x_update_rect(struct ssd130x_device *ssd130x,
 	 */
 
 	/* Set column start and end */
-	ret = ssd130x_write_cmd(ssd130x, 3, SSD133X_SET_COL_RANGE, x, columns - 1);
+	ret = ssd130x_write_cmd(ssd130x, 3, SSD133X_SET_COL_RANGE, x, x + columns - 1);
 	if (ret < 0)
 		return ret;
 
 	/* Set row start and end */
-	ret = ssd130x_write_cmd(ssd130x, 3, SSD133X_SET_ROW_RANGE, y, rows - 1);
+	ret = ssd130x_write_cmd(ssd130x, 3, SSD133X_SET_ROW_RANGE, y, y + rows - 1);
 	if (ret < 0)
 		return ret;
 
@@ -1007,7 +897,6 @@ static int ssd130x_fb_blit_rect(struct drm_framebuffer *fb,
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(fb->dev);
 	struct iosys_map dst;
 	unsigned int dst_pitch;
-	int ret = 0;
 
 	/* Align y to display page boundaries */
 	rect->y1 = round_down(rect->y1, SSD130X_PAGE_HEIGHT);
@@ -1020,7 +909,7 @@ static int ssd130x_fb_blit_rect(struct drm_framebuffer *fb,
 
 	ssd130x_update_rect(ssd130x, rect, buf, data_array);
 
-	return ret;
+	return 0;
 }
 
 static int ssd132x_fb_blit_rect(struct drm_framebuffer *fb,
@@ -1032,7 +921,6 @@ static int ssd132x_fb_blit_rect(struct drm_framebuffer *fb,
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(fb->dev);
 	unsigned int dst_pitch;
 	struct iosys_map dst;
-	int ret = 0;
 
 	/* Align x to display segment boundaries */
 	rect->x1 = round_down(rect->x1, SSD132X_SEGMENT_WIDTH);
@@ -1046,7 +934,7 @@ static int ssd132x_fb_blit_rect(struct drm_framebuffer *fb,
 
 	ssd132x_update_rect(ssd130x, rect, buf, data_array);
 
-	return ret;
+	return 0;
 }
 
 static int ssd133x_fb_blit_rect(struct drm_framebuffer *fb,
@@ -1058,7 +946,6 @@ static int ssd133x_fb_blit_rect(struct drm_framebuffer *fb,
 	const struct drm_format_info *fi = drm_format_info(DRM_FORMAT_RGB332);
 	unsigned int dst_pitch;
 	struct iosys_map dst;
-	int ret = 0;
 
 	if (!fi)
 		return -EINVAL;
@@ -1070,7 +957,7 @@ static int ssd133x_fb_blit_rect(struct drm_framebuffer *fb,
 
 	ssd133x_update_rect(ssd130x, rect, data_array, dst_pitch);
 
-	return ret;
+	return 0;
 }
 
 static int ssd130x_primary_plane_atomic_check(struct drm_plane *plane,
diff --git a/drivers/gpu/drm/sti/sti_dvo.c b/drivers/gpu/drm/sti/sti_dvo.c
index 7484d3c3f4ed5..b48099468eb97 100644
--- a/drivers/gpu/drm/sti/sti_dvo.c
+++ b/drivers/gpu/drm/sti/sti_dvo.c
@@ -209,7 +209,8 @@ static void dvo_debugfs_init(struct sti_dvo *dvo, struct drm_minor *minor)
 				 minor->debugfs_root, minor);
 }
 
-static void sti_dvo_disable(struct drm_bridge *bridge)
+static void sti_dvo_disable(struct drm_bridge *bridge,
+			    struct drm_atomic_commit *commit)
 {
 	struct sti_dvo *dvo = bridge->driver_private;
 
@@ -232,7 +233,8 @@ static void sti_dvo_disable(struct drm_bridge *bridge)
 	dvo->enabled = false;
 }
 
-static void sti_dvo_pre_enable(struct drm_bridge *bridge)
+static void sti_dvo_pre_enable(struct drm_bridge *bridge,
+			       struct drm_atomic_commit *commit)
 {
 	struct sti_dvo *dvo = bridge->driver_private;
 	struct dvo_config *config = dvo->config;
@@ -320,16 +322,20 @@ static void sti_dvo_set_mode(struct drm_bridge *bridge,
 	dvo->config = &rgb_24bit_de_cfg;
 }
 
-static void sti_dvo_bridge_nope(struct drm_bridge *bridge)
+static void sti_dvo_bridge_nope(struct drm_bridge *bridge,
+				struct drm_atomic_commit *commit)
 {
 	/* do nothing */
 }
 
 static const struct drm_bridge_funcs sti_dvo_bridge_funcs = {
-	.pre_enable = sti_dvo_pre_enable,
-	.enable = sti_dvo_bridge_nope,
-	.disable = sti_dvo_disable,
-	.post_disable = sti_dvo_bridge_nope,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+	.atomic_pre_enable = sti_dvo_pre_enable,
+	.atomic_enable = sti_dvo_bridge_nope,
+	.atomic_disable = sti_dvo_disable,
+	.atomic_post_disable = sti_dvo_bridge_nope,
 	.mode_set = sti_dvo_set_mode,
 };
 
diff --git a/drivers/gpu/drm/sti/sti_hda.c b/drivers/gpu/drm/sti/sti_hda.c
index b9f8f68f01f99..b4def2b25cec7 100644
--- a/drivers/gpu/drm/sti/sti_hda.c
+++ b/drivers/gpu/drm/sti/sti_hda.c
@@ -404,7 +404,8 @@ static void sti_hda_configure_awg(struct sti_hda *hda, u32 *awg_instr, int nb)
 		hda_write(hda, 0, HDA_SYNC_AWGI + i * 4);
 }
 
-static void sti_hda_disable(struct drm_bridge *bridge)
+static void sti_hda_disable(struct drm_bridge *bridge,
+			    struct drm_atomic_commit *commit)
 {
 	struct sti_hda *hda = drm_bridge_to_sti_hda(bridge);
 	u32 val;
@@ -429,7 +430,8 @@ static void sti_hda_disable(struct drm_bridge *bridge)
 	hda->enabled = false;
 }
 
-static void sti_hda_pre_enable(struct drm_bridge *bridge)
+static void sti_hda_pre_enable(struct drm_bridge *bridge,
+			       struct drm_atomic_commit *commit)
 {
 	struct sti_hda *hda = drm_bridge_to_sti_hda(bridge);
 	u32 val, i, mode_idx;
@@ -563,16 +565,20 @@ static void sti_hda_set_mode(struct drm_bridge *bridge,
 			  mode->clock * 1000);
 }
 
-static void sti_hda_bridge_nope(struct drm_bridge *bridge)
+static void sti_hda_bridge_nope(struct drm_bridge *bridge,
+				struct drm_atomic_commit *commit)
 {
 	/* do nothing */
 }
 
 static const struct drm_bridge_funcs sti_hda_bridge_funcs = {
-	.pre_enable = sti_hda_pre_enable,
-	.enable = sti_hda_bridge_nope,
-	.disable = sti_hda_disable,
-	.post_disable = sti_hda_bridge_nope,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+	.atomic_pre_enable = sti_hda_pre_enable,
+	.atomic_enable = sti_hda_bridge_nope,
+	.atomic_disable = sti_hda_disable,
+	.atomic_post_disable = sti_hda_bridge_nope,
 	.mode_set = sti_hda_set_mode,
 };
 
diff --git a/drivers/gpu/drm/sti/sti_hdmi.c b/drivers/gpu/drm/sti/sti_hdmi.c
index f8222e60b1e01..7ba9e0b7ba6c0 100644
--- a/drivers/gpu/drm/sti/sti_hdmi.c
+++ b/drivers/gpu/drm/sti/sti_hdmi.c
@@ -752,7 +752,8 @@ static void hdmi_debugfs_init(struct sti_hdmi *hdmi, struct drm_minor *minor)
 				 minor->debugfs_root, minor);
 }
 
-static void sti_hdmi_disable(struct drm_bridge *bridge)
+static void sti_hdmi_disable(struct drm_bridge *bridge,
+			     struct drm_atomic_commit *commit)
 {
 	struct sti_hdmi *hdmi = drm_bridge_to_sti_hdmi(bridge);
 
@@ -884,7 +885,8 @@ static int hdmi_audio_configure(struct sti_hdmi *hdmi)
 	return hdmi_audio_infoframe_config(hdmi);
 }
 
-static void sti_hdmi_pre_enable(struct drm_bridge *bridge)
+static void sti_hdmi_pre_enable(struct drm_bridge *bridge,
+				struct drm_atomic_commit *commit)
 {
 	struct sti_hdmi *hdmi = drm_bridge_to_sti_hdmi(bridge);
 
@@ -964,16 +966,20 @@ static void sti_hdmi_set_mode(struct drm_bridge *bridge,
 	}
 }
 
-static void sti_hdmi_bridge_nope(struct drm_bridge *bridge)
+static void sti_hdmi_bridge_nope(struct drm_bridge *bridge,
+				 struct drm_atomic_commit *commit)
 {
 	/* do nothing */
 }
 
 static const struct drm_bridge_funcs sti_hdmi_bridge_funcs = {
-	.pre_enable = sti_hdmi_pre_enable,
-	.enable = sti_hdmi_bridge_nope,
-	.disable = sti_hdmi_disable,
-	.post_disable = sti_hdmi_bridge_nope,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+	.atomic_pre_enable = sti_hdmi_pre_enable,
+	.atomic_enable = sti_hdmi_bridge_nope,
+	.atomic_disable = sti_hdmi_disable,
+	.atomic_post_disable = sti_hdmi_bridge_nope,
 	.mode_set = sti_hdmi_set_mode,
 };
 
diff --git a/drivers/gpu/drm/stm/lvds.c b/drivers/gpu/drm/stm/lvds.c
index 50a878688e477..90a44e7220573 100644
--- a/drivers/gpu/drm/stm/lvds.c
+++ b/drivers/gpu/drm/stm/lvds.c
@@ -1038,7 +1038,7 @@ static const struct drm_bridge_funcs lvds_bridge_funcs = {
 	.atomic_disable = lvds_atomic_disable,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 };
 
 static int lvds_probe(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.c b/drivers/gpu/drm/sun4i/sun4i_backend.c
index 03e46a2af6463..adf5b6e09ca64 100644
--- a/drivers/gpu/drm/sun4i/sun4i_backend.c
+++ b/drivers/gpu/drm/sun4i/sun4i_backend.c
@@ -686,7 +686,7 @@ static int sun4i_backend_init_sat(struct device *dev) {
 	ret = clk_prepare_enable(backend->sat_clk);
 	if (ret) {
 		dev_err(dev, "Couldn't enable the SAT clock\n");
-		return ret;
+		goto err_assert_reset;
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
index 74c7c3720ba82..e99f52ebb26f0 100644
--- a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
+++ b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
@@ -24,7 +24,6 @@
 #include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include <drm/display/drm_hdmi_helper.h>
 #include <drm/display/drm_hdmi_state_helper.h>
@@ -172,6 +171,10 @@ static void sun4i_hdmi_enable(struct drm_encoder *encoder,
 	writel(val, hdmi->base + SUN4I_HDMI_VID_CTRL_REG);
 }
 
+static const struct drm_encoder_funcs sun4i_hdmi_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs sun4i_hdmi_helper_funcs = {
 	.atomic_disable	= sun4i_hdmi_disable,
 	.atomic_enable	= sun4i_hdmi_enable,
@@ -285,7 +288,7 @@ sun4i_hdmi_connector_detect(struct drm_connector *connector, bool force)
 static void sun4i_hdmi_connector_reset(struct drm_connector *connector)
 {
 	drm_atomic_helper_connector_reset(connector);
-	__drm_atomic_helper_connector_hdmi_reset(connector, connector->state);
+	__drm_atomic_helper_connector_hdmi_state_init(connector, connector->state);
 }
 
 static const struct drm_connector_funcs sun4i_hdmi_connector_funcs = {
@@ -624,8 +627,8 @@ static int sun4i_hdmi_bind(struct device *dev, struct device *master,
 
 	drm_encoder_helper_add(&hdmi->encoder,
 			       &sun4i_hdmi_helper_funcs);
-	ret = drm_simple_encoder_init(drm, &hdmi->encoder,
-				      DRM_MODE_ENCODER_TMDS);
+	ret = drm_encoder_init(drm, &hdmi->encoder, &sun4i_hdmi_funcs,
+			       DRM_MODE_ENCODER_TMDS, NULL);
 	if (ret) {
 		dev_err(dev, "Couldn't initialise the HDMI encoder\n");
 		goto err_put_ddc_i2c;
diff --git a/drivers/gpu/drm/sun4i/sun4i_lvds.c b/drivers/gpu/drm/sun4i/sun4i_lvds.c
index 6716e895ae8a4..35a3f987c37a6 100644
--- a/drivers/gpu/drm/sun4i/sun4i_lvds.c
+++ b/drivers/gpu/drm/sun4i/sun4i_lvds.c
@@ -8,11 +8,11 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "sun4i_crtc.h"
 #include "sun4i_tcon.h"
@@ -89,6 +89,10 @@ static void sun4i_lvds_encoder_disable(struct drm_encoder *encoder)
 	}
 }
 
+static const struct drm_encoder_funcs sun4i_lvds_enc_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs sun4i_lvds_enc_helper_funcs = {
 	.disable	= sun4i_lvds_encoder_disable,
 	.enable		= sun4i_lvds_encoder_enable,
@@ -115,8 +119,8 @@ int sun4i_lvds_init(struct drm_device *drm, struct sun4i_tcon *tcon)
 
 	drm_encoder_helper_add(&lvds->encoder,
 			       &sun4i_lvds_enc_helper_funcs);
-	ret = drm_simple_encoder_init(drm, &lvds->encoder,
-				      DRM_MODE_ENCODER_LVDS);
+	ret = drm_encoder_init(drm, &lvds->encoder, &sun4i_lvds_enc_funcs,
+			       DRM_MODE_ENCODER_LVDS, NULL);
 	if (ret) {
 		dev_err(drm->dev, "Couldn't initialise the lvds encoder\n");
 		goto err_out;
diff --git a/drivers/gpu/drm/sun4i/sun4i_rgb.c b/drivers/gpu/drm/sun4i/sun4i_rgb.c
index dfb6acc42f02e..9c3fbf1b949ee 100644
--- a/drivers/gpu/drm/sun4i/sun4i_rgb.c
+++ b/drivers/gpu/drm/sun4i/sun4i_rgb.c
@@ -10,11 +10,11 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "sun4i_crtc.h"
 #include "sun4i_tcon.h"
@@ -180,6 +180,10 @@ static void sun4i_rgb_encoder_disable(struct drm_encoder *encoder)
 	}
 }
 
+static const struct drm_encoder_funcs sun4i_rgb_enc_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs sun4i_rgb_enc_helper_funcs = {
 	.disable	= sun4i_rgb_encoder_disable,
 	.enable		= sun4i_rgb_encoder_enable,
@@ -207,8 +211,8 @@ int sun4i_rgb_init(struct drm_device *drm, struct sun4i_tcon *tcon)
 
 	drm_encoder_helper_add(&rgb->encoder,
 			       &sun4i_rgb_enc_helper_funcs);
-	ret = drm_simple_encoder_init(drm, &rgb->encoder,
-				      DRM_MODE_ENCODER_NONE);
+	ret = drm_encoder_init(drm, &rgb->encoder, &sun4i_rgb_enc_funcs,
+			       DRM_MODE_ENCODER_NONE, NULL);
 	if (ret) {
 		dev_err(drm->dev, "Couldn't initialise the rgb encoder\n");
 		goto err_out;
diff --git a/drivers/gpu/drm/sun4i/sun4i_tv.c b/drivers/gpu/drm/sun4i/sun4i_tv.c
index 814b77f278f69..ce22f1662c0ce 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tv.c
+++ b/drivers/gpu/drm/sun4i/sun4i_tv.c
@@ -16,11 +16,11 @@
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "sun4i_crtc.h"
 #include "sun4i_drv.h"
@@ -391,6 +391,10 @@ static void sun4i_tv_enable(struct drm_encoder *encoder,
 			   SUN4I_TVE_EN_ENABLE);
 }
 
+static const struct drm_encoder_funcs sun4i_tv_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs sun4i_tv_helper_funcs = {
 	.atomic_disable	= sun4i_tv_disable,
 	.atomic_enable	= sun4i_tv_enable,
@@ -474,8 +478,8 @@ static int sun4i_tv_bind(struct device *dev, struct device *master,
 
 	drm_encoder_helper_add(&tv->encoder,
 			       &sun4i_tv_helper_funcs);
-	ret = drm_simple_encoder_init(drm, &tv->encoder,
-				      DRM_MODE_ENCODER_TVDAC);
+	ret = drm_encoder_init(drm, &tv->encoder, &sun4i_tv_funcs,
+			       DRM_MODE_ENCODER_TVDAC, NULL);
 	if (ret) {
 		dev_err(dev, "Couldn't initialise the TV encoder\n");
 		goto err_disable_clk;
diff --git a/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c b/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c
index c35b70d83e53b..f0c9f0e573d2e 100644
--- a/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c
+++ b/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c
@@ -20,11 +20,11 @@
 #include <linux/slab.h>
 
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "sun4i_crtc.h"
 #include "sun4i_tcon.h"
@@ -842,6 +842,10 @@ static const struct drm_connector_funcs sun6i_dsi_connector_funcs = {
 	.atomic_destroy_state	= drm_atomic_helper_connector_destroy_state,
 };
 
+static const struct drm_encoder_funcs sun6i_dsi_enc_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs sun6i_dsi_enc_helper_funcs = {
 	.disable	= sun6i_dsi_encoder_disable,
 	.enable		= sun6i_dsi_encoder_enable,
@@ -1056,8 +1060,8 @@ static int sun6i_dsi_bind(struct device *dev, struct device *master,
 
 	drm_encoder_helper_add(&dsi->encoder,
 			       &sun6i_dsi_enc_helper_funcs);
-	ret = drm_simple_encoder_init(drm, &dsi->encoder,
-				      DRM_MODE_ENCODER_DSI);
+	ret = drm_encoder_init(drm, &dsi->encoder, &sun6i_dsi_enc_funcs,
+			       DRM_MODE_ENCODER_DSI, NULL);
 	if (ret) {
 		dev_err(dsi->dev, "Couldn't initialise the DSI encoder\n");
 		return ret;
diff --git a/drivers/gpu/drm/sun4i/sun8i_dw_hdmi.c b/drivers/gpu/drm/sun4i/sun8i_dw_hdmi.c
index 96532709c2a7e..9694ad142a7de 100644
--- a/drivers/gpu/drm/sun4i/sun8i_dw_hdmi.c
+++ b/drivers/gpu/drm/sun4i/sun8i_dw_hdmi.c
@@ -9,8 +9,8 @@
 #include <linux/platform_device.h>
 
 #include <drm/drm_modeset_helper_vtables.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_of.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "sun8i_dw_hdmi.h"
 #include "sun8i_tcon_top.h"
@@ -23,6 +23,9 @@ static void sun8i_dw_hdmi_encoder_mode_set(struct drm_encoder *encoder,
 
 	clk_set_rate(hdmi->clk_tmds, mode->crtc_clock * 1000);
 }
+static const struct drm_encoder_funcs sun8i_dw_hdmi_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
 
 static const struct drm_encoder_helper_funcs
 sun8i_dw_hdmi_encoder_helper_funcs = {
@@ -180,7 +183,8 @@ static int sun8i_dw_hdmi_bind(struct device *dev, struct device *master,
 		goto err_disable_clk_tmds;
 
 	drm_encoder_helper_add(encoder, &sun8i_dw_hdmi_encoder_helper_funcs);
-	drm_simple_encoder_init(drm, encoder, DRM_MODE_ENCODER_TMDS);
+	drm_encoder_init(drm, encoder, &sun8i_dw_hdmi_encoder_funcs,
+			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	plat_data->mode_valid = hdmi->quirks->mode_valid;
 	plat_data->use_drm_infoframe = hdmi->quirks->use_drm_infoframe;
diff --git a/drivers/gpu/drm/sysfb/Kconfig b/drivers/gpu/drm/sysfb/Kconfig
index 2559ead6cf1f8..f7e48178885ee 100644
--- a/drivers/gpu/drm/sysfb/Kconfig
+++ b/drivers/gpu/drm/sysfb/Kconfig
@@ -67,11 +67,8 @@ config DRM_SIMPLEDRM
 
 	  This driver assumes that the display hardware has been initialized
 	  by the firmware or bootloader before the kernel boots. Scanout
-	  buffer, size, and display format must be provided via device tree,
-	  UEFI, VESA, etc.
-
-	  On x86 BIOS or UEFI systems, you should also select SYSFB_SIMPLEFB
-	  to use UEFI and VESA framebuffers.
+	  buffer, size, and display format must be provided via device tree's
+	  simple-framebuffer node.
 
 config DRM_VESADRM
 	tristate "VESA framebuffer driver"
diff --git a/drivers/gpu/drm/sysfb/simpledrm.c b/drivers/gpu/drm/sysfb/simpledrm.c
index fc168920f2c64..21ddf4502ecc8 100644
--- a/drivers/gpu/drm/sysfb/simpledrm.c
+++ b/drivers/gpu/drm/sysfb/simpledrm.c
@@ -6,6 +6,7 @@
 #include <linux/of_address.h>
 #include <linux/of_clk.h>
 #include <linux/of_reserved_mem.h>
+#include <linux/overflow.h>
 #include <linux/platform_data/simplefb.h>
 #include <linux/platform_device.h>
 #include <linux/pm.h>
@@ -27,6 +28,7 @@
 #include <drm/drm_managed.h>
 #include <drm/drm_modeset_helper.h>
 #include <drm/drm_modeset_helper_vtables.h>
+#include <drm/drm_of.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 
@@ -41,20 +43,6 @@
  * Helpers for simplefb
  */
 
-static int
-simplefb_get_validated_int(struct drm_device *dev, const char *name,
-			   uint32_t value)
-{
-	return drm_sysfb_get_validated_int(dev, name, value, INT_MAX);
-}
-
-static int
-simplefb_get_validated_int0(struct drm_device *dev, const char *name,
-			    uint32_t value)
-{
-	return drm_sysfb_get_validated_int0(dev, name, value, INT_MAX);
-}
-
 static const struct drm_format_info *
 simplefb_get_validated_format(struct drm_device *dev, const char *format_name)
 {
@@ -88,21 +76,21 @@ static int
 simplefb_get_width_pd(struct drm_device *dev,
 		      const struct simplefb_platform_data *pd)
 {
-	return simplefb_get_validated_int0(dev, "width", pd->width);
+	return drm_sysfb_get_validated_int0(dev, "width", pd->width, U16_MAX);
 }
 
 static int
 simplefb_get_height_pd(struct drm_device *dev,
 		       const struct simplefb_platform_data *pd)
 {
-	return simplefb_get_validated_int0(dev, "height", pd->height);
+	return drm_sysfb_get_validated_int0(dev, "height", pd->height, U16_MAX);
 }
 
 static int
 simplefb_get_stride_pd(struct drm_device *dev,
 		       const struct simplefb_platform_data *pd)
 {
-	return simplefb_get_validated_int(dev, "stride", pd->stride);
+	return drm_sysfb_get_validated_int(dev, "stride", pd->stride, INT_MAX);
 }
 
 static const struct drm_format_info *
@@ -144,7 +132,7 @@ simplefb_get_width_of(struct drm_device *dev, struct device_node *of_node)
 
 	if (ret)
 		return ret;
-	return simplefb_get_validated_int0(dev, "width", width);
+	return drm_sysfb_get_validated_int0(dev, "width", width, U16_MAX);
 }
 
 static int
@@ -155,7 +143,7 @@ simplefb_get_height_of(struct drm_device *dev, struct device_node *of_node)
 
 	if (ret)
 		return ret;
-	return simplefb_get_validated_int0(dev, "height", height);
+	return drm_sysfb_get_validated_int0(dev, "height", height, U16_MAX);
 }
 
 static int
@@ -166,7 +154,7 @@ simplefb_get_stride_of(struct drm_device *dev, struct device_node *of_node)
 
 	if (ret)
 		return ret;
-	return simplefb_get_validated_int(dev, "stride", stride);
+	return drm_sysfb_get_validated_int(dev, "stride", stride, INT_MAX);
 }
 
 static const struct drm_format_info *
@@ -200,6 +188,39 @@ simplefb_get_memory_of(struct drm_device *dev, struct device_node *of_node)
 	return res;
 }
 
+static int __simplefb_get_panel_size_mm_of(struct drm_device *dev,
+					   struct device_node *of_panel_node,
+					   const char *name)
+{
+	int ret;
+	u32 value;
+
+	ret = of_property_read_u32(of_panel_node, name, &value);
+	if (ret) {
+		drm_dbg(dev, "simplefb: cannot parse panel %s: error %d\n",
+			name, ret);
+		return ret;
+	} else if (value > U16_MAX) {
+		drm_dbg(dev, "simplefb: panel %s of %u exceeds maximum value\n",
+			name, value);
+		return -EINVAL;
+	}
+
+	return value;
+}
+
+static int simplefb_get_panel_width_mm_of(struct drm_device *dev,
+					  struct device_node *of_panel_node)
+{
+	return __simplefb_get_panel_size_mm_of(dev, of_panel_node, "width-mm");
+}
+
+static int simplefb_get_panel_height_mm_of(struct drm_device *dev,
+					   struct device_node *of_panel_node)
+{
+	return __simplefb_get_panel_size_mm_of(dev, of_panel_node, "height-mm");
+}
+
 /*
  * Simple Framebuffer device
  */
@@ -601,9 +622,11 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv,
 	struct drm_sysfb_device *sysfb;
 	struct drm_device *dev;
 	int width, height, stride;
-	int width_mm = 0, height_mm = 0;
+	u16 width_mm = 0, height_mm = 0;
 	struct device_node *panel_node;
+	enum drm_panel_orientation orientation = DRM_MODE_PANEL_ORIENTATION_UNKNOWN;
 	const struct drm_format_info *format;
+	u64 size;
 	struct resource *res, *mem = NULL;
 	struct drm_plane *primary_plane;
 	struct drm_crtc *crtc;
@@ -665,8 +688,24 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv,
 			return ERR_CAST(mem);
 		panel_node = of_parse_phandle(of_node, "panel", 0);
 		if (panel_node) {
-			simplefb_read_u32_of(dev, panel_node, "width-mm", &width_mm);
-			simplefb_read_u32_of(dev, panel_node, "height-mm", &height_mm);
+			/*
+			 * Ignore errors from parsing the physical panel
+			 * size. Using the pre-initialized sizes of 0 will
+			 * make drm_sysfb_mode() calculate a default physical
+			 * size based on a resolution of 96 dpi.
+			 */
+			ret = simplefb_get_panel_width_mm_of(dev, panel_node);
+			if (ret > 0)
+				width_mm = ret;
+			ret = simplefb_get_panel_height_mm_of(dev, panel_node);
+			if (ret > 0)
+				height_mm = ret;
+			/*
+			 * Ignore errors from parsing the panel orientation. With
+			 * the orientation initialized to UNKNOWN, the connector
+			 * helpers will do the right thing.
+			 */
+			drm_of_get_panel_orientation(panel_node, &orientation);
 			of_node_put(panel_node);
 		}
 	} else {
@@ -674,9 +713,23 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv,
 		return ERR_PTR(-ENODEV);
 	}
 	if (!stride) {
-		stride = drm_format_info_min_pitch(format, 0, width);
-		if (drm_WARN_ON(dev, !stride))
+		u64 pitch = drm_format_info_min_pitch(format, 0, width);
+
+		if (drm_WARN_ON(dev, !pitch)) {
+			return ERR_PTR(-EINVAL); /* driver bug */
+		} else if (pitch > INT_MAX) {
+			drm_warn(dev, "stride of %llu exceeds maximum\n", pitch);
 			return ERR_PTR(-EINVAL);
+		}
+		stride = pitch;
+	}
+	if (check_mul_overflow(height, stride, &size)) {
+		drm_err(dev, "framebuffer size exceeds maximum\n");
+		return ERR_PTR(-EINVAL);
+	}
+	if (ALIGN(size, PAGE_SIZE) < PAGE_SIZE) {
+		drm_err(dev, "page-aligned framebuffer exceeds maximum\n");
+		return ERR_PTR(-EINVAL);
 	}
 
 	sysfb->fb_mode = drm_sysfb_mode(width, height, width_mm, height_mm);
@@ -703,6 +756,13 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv,
 
 		drm_dbg(dev, "using system memory framebuffer at %pr\n", mem);
 
+		if (size > resource_size(mem)) {
+			drm_err(dev,
+				"framebuffer size of %llu exceeds memory range %pr\n",
+				size, mem);
+			return ERR_PTR(-EINVAL);
+		}
+
 		screen_base = devm_memremap(dev->dev, mem->start, resource_size(mem), MEMREMAP_WC);
 		if (IS_ERR(screen_base))
 			return screen_base;
@@ -736,6 +796,13 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv,
 			mem = res;
 		}
 
+		if (size > resource_size(mem)) {
+			drm_err(dev,
+				"framebuffer size of %llu exceeds memory range %pr\n",
+				size, mem);
+			return ERR_PTR(-EINVAL);
+		}
+
 		screen_base = devm_ioremap_wc(&pdev->dev, mem->start, resource_size(mem));
 		if (!screen_base)
 			return ERR_PTR(-ENOMEM);
@@ -802,8 +869,7 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv,
 	if (ret)
 		return ERR_PTR(ret);
 	drm_connector_helper_add(connector, &simpledrm_connector_helper_funcs);
-	drm_connector_set_panel_orientation_with_quirk(connector,
-						       DRM_MODE_PANEL_ORIENTATION_UNKNOWN,
+	drm_connector_set_panel_orientation_with_quirk(connector, orientation,
 						       width, height);
 
 	ret = drm_connector_attach_encoder(connector, encoder);
diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c
index dd6c564ff408b..324480c87fb1f 100644
--- a/drivers/gpu/drm/tegra/drm.c
+++ b/drivers/gpu/drm/tegra/drm.c
@@ -1358,10 +1358,12 @@ static SIMPLE_DEV_PM_OPS(host1x_drm_pm_ops, host1x_drm_suspend,
 
 static const struct of_device_id host1x_drm_subdevs[] = {
 	{ .compatible = "nvidia,tegra20-dc", },
+	{ .compatible = "nvidia,tegra20-dsi", },
 	{ .compatible = "nvidia,tegra20-hdmi", },
 	{ .compatible = "nvidia,tegra20-gr2d", },
 	{ .compatible = "nvidia,tegra20-gr3d", },
 	{ .compatible = "nvidia,tegra30-dc", },
+	{ .compatible = "nvidia,tegra30-dsi", },
 	{ .compatible = "nvidia,tegra30-hdmi", },
 	{ .compatible = "nvidia,tegra30-gr2d", },
 	{ .compatible = "nvidia,tegra30-gr3d", },
@@ -1396,6 +1398,7 @@ static const struct of_device_id host1x_drm_subdevs[] = {
 	{ .compatible = "nvidia,tegra194-nvdec", },
 	{ .compatible = "nvidia,tegra234-vic", },
 	{ .compatible = "nvidia,tegra234-nvdec", },
+	{ .compatible = "nvidia,tegra264-vic", },
 	{ /* sentinel */ }
 };
 
diff --git a/drivers/gpu/drm/tegra/dsi.c b/drivers/gpu/drm/tegra/dsi.c
index 7f25c50621c94..e7fdd8c7ac12a 100644
--- a/drivers/gpu/drm/tegra/dsi.c
+++ b/drivers/gpu/drm/tegra/dsi.c
@@ -54,6 +54,11 @@ to_dsi_state(struct drm_connector_state *state)
 	return container_of(state, struct tegra_dsi_state, base);
 }
 
+struct tegra_dsi_config {
+	bool has_multiple_pad_controls;
+	bool has_mux_parent_clk;
+};
+
 struct tegra_dsi {
 	struct host1x_client client;
 	struct tegra_output output;
@@ -83,6 +88,8 @@ struct tegra_dsi {
 	/* for ganged-mode support */
 	struct tegra_dsi *master;
 	struct tegra_dsi *slave;
+
+	const struct tegra_dsi_config *config;
 };
 
 static inline struct tegra_dsi *
@@ -665,39 +672,46 @@ static int tegra_dsi_pad_enable(struct tegra_dsi *dsi)
 {
 	u32 value;
 
-	value = DSI_PAD_CONTROL_VS1_PULLDN(0) | DSI_PAD_CONTROL_VS1_PDIO(0);
-	tegra_dsi_writel(dsi, value, DSI_PAD_CONTROL_0);
+	if (dsi->config->has_multiple_pad_controls) {
+		/*
+		 * XXX Is this still needed? The module reset is deasserted right
+		 * before this function is called.
+		 */
+		tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_0);
+		tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_1);
+		tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_2);
+		tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_3);
+		tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_4);
+
+		value = DSI_PAD_CONTROL_VS1_PULLDN(0) | DSI_PAD_CONTROL_VS1_PDIO(0);
+		tegra_dsi_writel(dsi, value, DSI_PAD_CONTROL_0);
+
+		value = DSI_PAD_SLEW_UP(0x7) | DSI_PAD_SLEW_DN(0x7) |
+			DSI_PAD_LP_UP(0x1) | DSI_PAD_LP_DN(0x1) |
+			DSI_PAD_OUT_CLK(0x0);
+		tegra_dsi_writel(dsi, value, DSI_PAD_CONTROL_2);
+
+		value = DSI_PAD_PREEMP_PD_CLK(0x3) | DSI_PAD_PREEMP_PU_CLK(0x3) |
+			DSI_PAD_PREEMP_PD(0x03) | DSI_PAD_PREEMP_PU(0x3);
+		tegra_dsi_writel(dsi, value, DSI_PAD_CONTROL_3);
+	} else {
+		value = DSI_PAD_CONTROL_LPUPADJ(0x1) | DSI_PAD_CONTROL_LPDNADJ(0x1) |
+			DSI_PAD_CONTROL_PREEMP_EN(0x1) | DSI_PAD_CONTROL_SLEWDNADJ(0x6) |
+			DSI_PAD_CONTROL_SLEWUPADJ(0x6) | DSI_PAD_CONTROL_PDIO(0) |
+			DSI_PAD_CONTROL_PDIO_CLK(0) | DSI_PAD_CONTROL_PULLDN_ENAB(0);
+		tegra_dsi_writel(dsi, value, DSI_PAD_CONTROL_0);
+	}
 
 	return 0;
 }
 
 static int tegra_dsi_pad_calibrate(struct tegra_dsi *dsi)
 {
-	u32 value;
 	int err;
 
-	/*
-	 * XXX Is this still needed? The module reset is deasserted right
-	 * before this function is called.
-	 */
-	tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_0);
-	tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_1);
-	tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_2);
-	tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_3);
-	tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_4);
-
 	/* start calibration */
 	tegra_dsi_pad_enable(dsi);
 
-	value = DSI_PAD_SLEW_UP(0x7) | DSI_PAD_SLEW_DN(0x7) |
-		DSI_PAD_LP_UP(0x1) | DSI_PAD_LP_DN(0x1) |
-		DSI_PAD_OUT_CLK(0x0);
-	tegra_dsi_writel(dsi, value, DSI_PAD_CONTROL_2);
-
-	value = DSI_PAD_PREEMP_PD_CLK(0x3) | DSI_PAD_PREEMP_PU_CLK(0x3) |
-		DSI_PAD_PREEMP_PD(0x03) | DSI_PAD_PREEMP_PU(0x3);
-	tegra_dsi_writel(dsi, value, DSI_PAD_CONTROL_3);
-
 	err = tegra_mipi_start_calibration(dsi->mipi);
 	if (err < 0)
 		return err;
@@ -922,6 +936,15 @@ static void tegra_dsi_encoder_enable(struct drm_encoder *encoder)
 		return;
 	}
 
+	/* If the bootloader enabled DSI it needs to be disabled
+	 * in order for the panel initialization commands to be
+	 * properly sent.
+	 */
+	value = tegra_dsi_readl(dsi, DSI_POWER_CONTROL);
+
+	if (value & DSI_POWER_CONTROL_ENABLE)
+		tegra_dsi_disable(dsi);
+
 	state = tegra_dsi_get_state(dsi);
 
 	tegra_dsi_set_timeout(dsi, state->bclk, state->vrefresh);
@@ -1174,6 +1197,12 @@ static int tegra_dsi_setup_clocks(struct tegra_dsi *dsi)
 	struct clk *parent;
 	int err;
 
+	/*
+	 * Tegra124+ uses a clock gate, not a mux, so this step
+	 * should be redundant for configuration; yet, DSI refuses
+	 * to work without it.
+	 */
+
 	parent = clk_get_parent(dsi->clk);
 	if (!parent)
 		return -EINVAL;
@@ -1562,6 +1591,10 @@ static int tegra_dsi_probe(struct platform_device *pdev)
 	if (!dsi)
 		return -ENOMEM;
 
+	dsi->config = device_get_match_data(&pdev->dev);
+	if (!dsi->config)
+		return -ENODEV;
+
 	dsi->output.dev = dsi->dev = &pdev->dev;
 	dsi->video_fifo_depth = 1920;
 	dsi->host_fifo_depth = 64;
@@ -1600,7 +1633,7 @@ static int tegra_dsi_probe(struct platform_device *pdev)
 		goto remove;
 	}
 
-	dsi->clk_lp = devm_clk_get(&pdev->dev, "lp");
+	dsi->clk_lp = devm_clk_get_optional(&pdev->dev, "lp");
 	if (IS_ERR(dsi->clk_lp)) {
 		err = dev_err_probe(&pdev->dev, PTR_ERR(dsi->clk_lp),
 				    "cannot get low-power clock\n");
@@ -1621,10 +1654,12 @@ static int tegra_dsi_probe(struct platform_device *pdev)
 		goto remove;
 	}
 
-	err = tegra_dsi_setup_clocks(dsi);
-	if (err < 0) {
-		dev_err(&pdev->dev, "cannot setup clocks\n");
-		goto remove;
+	if (dsi->config->has_mux_parent_clk) {
+		err = tegra_dsi_setup_clocks(dsi);
+		if (err < 0) {
+			dev_err(&pdev->dev, "cannot setup clocks\n");
+			goto remove;
+		}
 	}
 
 	dsi->regs = devm_platform_ioremap_resource(pdev, 0);
@@ -1688,11 +1723,40 @@ static void tegra_dsi_remove(struct platform_device *pdev)
 	tegra_mipi_free(dsi->mipi);
 }
 
+static const struct tegra_dsi_config tegra20_dsi_config = {
+	.has_multiple_pad_controls = false,
+	.has_mux_parent_clk = false,
+};
+
+/*
+ * Tegra30 allows DSIA/DSIB to be muxed to either PLL_D or PLL_D2; this is
+ * simply not modeled in the clock driver yet. If this functionality is
+ * required, the has_mux_parent_clk flag can be set to true once the clock
+ * driver is patched.
+ */
+static const struct tegra_dsi_config tegra30_dsi_config = {
+	.has_multiple_pad_controls = false,
+	.has_mux_parent_clk = false,
+};
+
+static const struct tegra_dsi_config tegra114_dsi_config = {
+	.has_multiple_pad_controls = true,
+	.has_mux_parent_clk = true,
+};
+
+/* TODO: figure out why has_mux_parent_clk = true is necessary on Tegra124+ */
+static const struct tegra_dsi_config tegra124_dsi_config = {
+	.has_multiple_pad_controls = true,
+	.has_mux_parent_clk = true,
+};
+
 static const struct of_device_id tegra_dsi_of_match[] = {
-	{ .compatible = "nvidia,tegra210-dsi", },
-	{ .compatible = "nvidia,tegra132-dsi", },
-	{ .compatible = "nvidia,tegra124-dsi", },
-	{ .compatible = "nvidia,tegra114-dsi", },
+	{ .compatible = "nvidia,tegra210-dsi", .data = &tegra124_dsi_config },
+	{ .compatible = "nvidia,tegra132-dsi", .data = &tegra124_dsi_config },
+	{ .compatible = "nvidia,tegra124-dsi", .data = &tegra124_dsi_config },
+	{ .compatible = "nvidia,tegra114-dsi", .data = &tegra114_dsi_config },
+	{ .compatible = "nvidia,tegra30-dsi", .data = &tegra30_dsi_config },
+	{ .compatible = "nvidia,tegra20-dsi", .data = &tegra20_dsi_config },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, tegra_dsi_of_match);
diff --git a/drivers/gpu/drm/tegra/dsi.h b/drivers/gpu/drm/tegra/dsi.h
index f39594e65e974..d834ac0c47ab9 100644
--- a/drivers/gpu/drm/tegra/dsi.h
+++ b/drivers/gpu/drm/tegra/dsi.h
@@ -95,6 +95,16 @@
 #define DSI_TALLY_LRX(x)		(((x) & 0xff) <<  8)
 #define DSI_TALLY_HTX(x)		(((x) & 0xff) <<  0)
 #define DSI_PAD_CONTROL_0		0x4b
+/* Tegra20/Tegra30 */
+#define DSI_PAD_CONTROL_PULLDN_ENAB(x)	(((x) & 0x1) << 28)
+#define DSI_PAD_CONTROL_SLEWUPADJ(x)	(((x) & 0x7) << 24)
+#define DSI_PAD_CONTROL_SLEWDNADJ(x)	(((x) & 0x7) << 20)
+#define DSI_PAD_CONTROL_PREEMP_EN(x)	(((x) & 0x1) << 19)
+#define DSI_PAD_CONTROL_PDIO_CLK(x)	(((x) & 0x1) << 18)
+#define DSI_PAD_CONTROL_PDIO(x)		(((x) & 0x3) << 16)
+#define DSI_PAD_CONTROL_LPUPADJ(x)	(((x) & 0x3) << 14)
+#define DSI_PAD_CONTROL_LPDNADJ(x)	(((x) & 0x3) << 12)
+/* Tegra114+ */
 #define DSI_PAD_CONTROL_VS1_PDIO(x)	(((x) & 0xf) <<  0)
 #define DSI_PAD_CONTROL_VS1_PDIO_CLK	(1 <<  8)
 #define DSI_PAD_CONTROL_VS1_PULLDN(x)	(((x) & 0xf) << 16)
diff --git a/drivers/gpu/drm/tegra/falcon.c b/drivers/gpu/drm/tegra/falcon.c
index 17f616bbcb45b..1172356b6af3b 100644
--- a/drivers/gpu/drm/tegra/falcon.c
+++ b/drivers/gpu/drm/tegra/falcon.c
@@ -26,8 +26,12 @@ int falcon_wait_idle(struct falcon *falcon)
 {
 	u32 value;
 
-	return readl_poll_timeout(falcon->regs + FALCON_IDLESTATE, value,
-				  (value == 0), 10, 100000);
+	if (falcon->riscv)
+		return readl_poll_timeout(falcon->regs + RISCV_CPUCTL, value,
+					  (value & RISCV_CPUCTL_ACTIVE_STAT_ACTIVE), 10, 100000);
+	else
+		return readl_poll_timeout(falcon->regs + FALCON_IDLESTATE, value,
+					  (value == 0), 10, 100000);
 }
 
 static int falcon_dma_wait_not_full(struct falcon *falcon)
@@ -122,6 +126,17 @@ static int falcon_parse_firmware_image(struct falcon *falcon)
 	return 0;
 }
 
+static void falcon_parse_firmware_desc(struct falcon *falcon)
+{
+	struct falcon_fw_riscv_desc *desc =
+		(struct falcon_fw_riscv_desc *)falcon->firmware.desc_firmware->data;
+
+	falcon->firmware.code.offset = desc->code_offset;
+	falcon->firmware.code.size = desc->code_size;
+	falcon->firmware.data.offset = desc->data_offset;
+	falcon->firmware.data.size = desc->data_size;
+}
+
 int falcon_read_firmware(struct falcon *falcon, const char *name)
 {
 	int err;
@@ -133,7 +148,23 @@ int falcon_read_firmware(struct falcon *falcon, const char *name)
 
 	falcon->firmware.size = falcon->firmware.firmware->size;
 
+	if (falcon->riscv) {
+		/* Load separate descriptor */
+		char desc_name[128];
+
+		scnprintf(desc_name, sizeof(desc_name), "%s.desc", name);
+		err = request_firmware(&falcon->firmware.desc_firmware, desc_name, falcon->dev);
+		if (err < 0)
+			goto release_firmware;
+	}
+
 	return 0;
+
+release_firmware:
+	release_firmware(falcon->firmware.firmware);
+	falcon->firmware.firmware = NULL;
+
+	return err;
 }
 
 int falcon_load_firmware(struct falcon *falcon)
@@ -144,16 +175,22 @@ int falcon_load_firmware(struct falcon *falcon)
 	/* copy firmware image into local area. this also ensures endianness */
 	falcon_copy_firmware_image(falcon, firmware);
 
-	/* parse the image data */
-	err = falcon_parse_firmware_image(falcon);
-	if (err < 0) {
-		dev_err(falcon->dev, "failed to parse firmware image\n");
-		return err;
+	if (falcon->riscv) {
+		falcon_parse_firmware_desc(falcon);
+	} else {
+		err = falcon_parse_firmware_image(falcon);
+		if (err < 0) {
+			dev_err(falcon->dev, "failed to parse firmware image\n");
+			return err;
+		}
 	}
 
 	release_firmware(firmware);
 	falcon->firmware.firmware = NULL;
 
+	release_firmware(falcon->firmware.desc_firmware);
+	falcon->firmware.desc_firmware = NULL;
+
 	return 0;
 }
 
@@ -168,6 +205,9 @@ void falcon_exit(struct falcon *falcon)
 {
 	if (falcon->firmware.firmware)
 		release_firmware(falcon->firmware.firmware);
+
+	if (falcon->firmware.desc_firmware)
+		release_firmware(falcon->firmware.desc_firmware);
 }
 
 int falcon_boot(struct falcon *falcon)
@@ -229,9 +269,15 @@ int falcon_boot(struct falcon *falcon)
 			      FALCON_ITFEN_CTXEN,
 		      FALCON_ITFEN);
 
-	/* boot falcon */
-	falcon_writel(falcon, 0x00000000, FALCON_BOOTVEC);
-	falcon_writel(falcon, FALCON_CPUCTL_STARTCPU, FALCON_CPUCTL);
+	if (falcon->riscv) {
+		falcon_writel(falcon, RISCV_BCR_CTRL_CORE_SELECT_RISCV, RISCV_BCR_CTRL);
+		falcon_writel(falcon, 0x0, RISCV_BOOT_VECTOR_HI);
+		falcon_writel(falcon, 0x100000, RISCV_BOOT_VECTOR_LO);
+		falcon_writel(falcon, RISCV_CPUCTL_STARTCPU, RISCV_CPUCTL);
+	} else {
+		falcon_writel(falcon, 0x00000000, FALCON_BOOTVEC);
+		falcon_writel(falcon, FALCON_CPUCTL_STARTCPU, FALCON_CPUCTL);
+	}
 
 	err = falcon_wait_idle(falcon);
 	if (err < 0) {
diff --git a/drivers/gpu/drm/tegra/falcon.h b/drivers/gpu/drm/tegra/falcon.h
index 902bb7e4fd0fc..37a17c6136b3e 100644
--- a/drivers/gpu/drm/tegra/falcon.h
+++ b/drivers/gpu/drm/tegra/falcon.h
@@ -55,6 +55,16 @@
 
 #define FALCON_DMATRFFBOFFS			0x0000111c
 
+#define RISCV_BOOT_VECTOR_LO			0x00001780
+#define RISCV_BOOT_VECTOR_HI			0x00001784
+
+#define RISCV_CPUCTL				0x00001788
+#define RISCV_CPUCTL_STARTCPU			(1 << 0)
+#define RISCV_CPUCTL_ACTIVE_STAT_ACTIVE		(1 << 7)
+
+#define RISCV_BCR_CTRL				0x00001a68
+#define RISCV_BCR_CTRL_CORE_SELECT_RISCV	(1 << 4)
+
 struct falcon_fw_bin_header_v1 {
 	u32 magic;		/* 0x10de */
 	u32 version;		/* version of bin format (1) */
@@ -76,6 +86,14 @@ struct falcon_fw_os_header_v1 {
 	u32 data_size;
 };
 
+struct falcon_fw_riscv_desc {
+	u32 reserved[74];
+	u32 data_offset;
+	u32 data_size;
+	u32 code_offset;
+	u32 code_size;
+};
+
 struct falcon_firmware_section {
 	unsigned long offset;
 	size_t size;
@@ -84,6 +102,8 @@ struct falcon_firmware_section {
 struct falcon_firmware {
 	/* Firmware after it is read but not loaded */
 	const struct firmware *firmware;
+	/* RISC-V firmware descriptor */
+	const struct firmware *desc_firmware;
 
 	/* Raw firmware data */
 	dma_addr_t iova;
@@ -102,6 +122,9 @@ struct falcon {
 	struct device *dev;
 	void __iomem *regs;
 
+	/* Peregrine falcon, external boot */
+	bool riscv;
+
 	struct falcon_firmware firmware;
 };
 
diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c
index 436394e048125..f76af733ea793 100644
--- a/drivers/gpu/drm/tegra/gem.c
+++ b/drivers/gpu/drm/tegra/gem.c
@@ -13,6 +13,7 @@
 #include <linux/dma-buf.h>
 #include <linux/iommu.h>
 #include <linux/module.h>
+#include <linux/pagemap.h>
 #include <linux/vmalloc.h>
 
 #include <drm/drm_drv.h>
@@ -564,7 +565,7 @@ static vm_fault_t tegra_bo_fault(struct vm_fault *vmf)
 	if (!bo->pages)
 		return VM_FAULT_SIGBUS;
 
-	offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
+	offset = linear_page_delta(vma, vmf->address);
 	page = bo->pages[offset];
 
 	return vmf_insert_page(vma, vmf->address, page);
@@ -601,7 +602,7 @@ int __tegra_gem_mmap(struct drm_gem_object *gem, struct vm_area_struct *vma)
 
 		vma->vm_pgoff = vm_pgoff;
 	} else {
-		pgprot_t prot = vm_get_page_prot(vma->vm_flags);
+		pgprot_t prot = vma_get_page_prot(vma);
 
 		vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP);
 
diff --git a/drivers/gpu/drm/tegra/sor.c b/drivers/gpu/drm/tegra/sor.c
index 4032c6ad45bc1..e384cbd0cbf72 100644
--- a/drivers/gpu/drm/tegra/sor.c
+++ b/drivers/gpu/drm/tegra/sor.c
@@ -3764,10 +3764,8 @@ static int tegra_sor_probe(struct platform_device *pdev)
 	sor->num_settings = sor->soc->num_settings;
 
 	sor->pmc = devm_tegra_pmc_get(&pdev->dev);
-	if (IS_ERR(sor->pmc)) {
-		err = PTR_ERR(sor->pmc);
-		goto put_aux;
-	}
+	if (IS_ERR(sor->pmc))
+		return PTR_ERR(sor->pmc);
 
 	np = of_parse_phandle(pdev->dev.of_node, "nvidia,dpaux", 0);
 	if (np) {
diff --git a/drivers/gpu/drm/tegra/vic.c b/drivers/gpu/drm/tegra/vic.c
index 332c9b563d3f4..fe097e9675516 100644
--- a/drivers/gpu/drm/tegra/vic.c
+++ b/drivers/gpu/drm/tegra/vic.c
@@ -8,6 +8,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/host1x.h>
 #include <linux/iommu.h>
+#include <linux/iopoll.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
@@ -20,10 +21,16 @@
 #include "falcon.h"
 #include "vic.h"
 
+#define VIC_FALCON_DEBUGINFO			0x1094
+#define VIC_DEBUGINFO_DUMMY			0xabcd1234
+#define VIC_DEBUGINFO_CLEAR			0x0
+
 struct vic_config {
 	const char *firmware;
 	unsigned int version;
 	bool supports_sid;
+	bool has_riscv;
+	unsigned int transcfg_offset;
 };
 
 struct vic {
@@ -54,8 +61,8 @@ static void vic_writel(struct vic *vic, u32 value, unsigned int offset)
 
 static int vic_boot(struct vic *vic)
 {
-	u32 fce_ucode_size, fce_bin_data_offset, stream_id;
-	void *hdr;
+	u32 stream_id;
+	u32 val;
 	int err = 0;
 
 	if (vic->config->supports_sid && tegra_dev_iommu_get_stream_id(vic->dev, &stream_id)) {
@@ -63,7 +70,7 @@ static int vic_boot(struct vic *vic)
 
 		value = TRANSCFG_ATT(1, TRANSCFG_SID_FALCON) |
 			TRANSCFG_ATT(0, TRANSCFG_SID_HW);
-		vic_writel(vic, value, VIC_TFBIF_TRANSCFG);
+		vic_writel(vic, value, vic->config->transcfg_offset);
 
 		/*
 		 * STREAMID0 is used for input/output buffers. Initialize it to SID_VIC in case
@@ -85,31 +92,51 @@ static int vic_boot(struct vic *vic)
 			CG_WAKEUP_DLY_CNT(4),
 		   NV_PVIC_MISC_PRI_VIC_CG);
 
+	if (vic->config->has_riscv) {
+		/* Write a known pattern into DEBUGINFO register */
+		vic_writel(vic, VIC_DEBUGINFO_DUMMY, VIC_FALCON_DEBUGINFO);
+	}
+
 	err = falcon_boot(&vic->falcon);
 	if (err < 0)
 		return err;
 
-	hdr = vic->falcon.firmware.virt;
-	fce_bin_data_offset = *(u32 *)(hdr + VIC_UCODE_FCE_DATA_OFFSET);
-
-	/* Old VIC firmware needs kernel help with setting up FCE microcode. */
-	if (fce_bin_data_offset != 0x0 && fce_bin_data_offset != 0xa5a5a5a5) {
-		hdr = vic->falcon.firmware.virt +
-			*(u32 *)(hdr + VIC_UCODE_FCE_HEADER_OFFSET);
-		fce_ucode_size = *(u32 *)(hdr + FCE_UCODE_SIZE_OFFSET);
-
-		falcon_execute_method(&vic->falcon, VIC_SET_FCE_UCODE_SIZE,
-				      fce_ucode_size);
-		falcon_execute_method(
-			&vic->falcon, VIC_SET_FCE_UCODE_OFFSET,
-			(vic->falcon.firmware.iova + fce_bin_data_offset) >> 8);
-	}
+	if (vic->config->has_riscv) {
+		/* Check VIC has reached a proper initialized state */
+		err = readl_poll_timeout(vic->regs + VIC_FALCON_DEBUGINFO, val,
+					 val == VIC_DEBUGINFO_CLEAR,
+					 1000, 2000000);
+		if (err) {
+			dev_err(vic->dev, "VIC not initialized, timeout, val=0x%x\n", val);
+			return err;
+		}
+	} else {
+		u32 fce_ucode_size, fce_bin_data_offset;
+		dma_addr_t iova;
+		void *hdr;
+
+		iova = vic->falcon.firmware.iova;
+		hdr = vic->falcon.firmware.virt;
+		fce_bin_data_offset = *(u32 *)(hdr + VIC_UCODE_FCE_DATA_OFFSET);
+
+		/* Old VIC firmware needs kernel help with setting up FCE microcode. */
+		if (fce_bin_data_offset != 0x0 && fce_bin_data_offset != 0xa5a5a5a5) {
+			hdr = vic->falcon.firmware.virt +
+				*(u32 *)(hdr + VIC_UCODE_FCE_HEADER_OFFSET);
+			fce_ucode_size = *(u32 *)(hdr + FCE_UCODE_SIZE_OFFSET);
+
+			falcon_execute_method(&vic->falcon, VIC_SET_FCE_UCODE_SIZE,
+					      fce_ucode_size);
+			falcon_execute_method(&vic->falcon, VIC_SET_FCE_UCODE_OFFSET,
+					      (iova + fce_bin_data_offset) >> 8);
+		}
 
-	err = falcon_wait_idle(&vic->falcon);
-	if (err < 0) {
-		dev_err(vic->dev,
-			"failed to set application ID and FCE base\n");
-		return err;
+		err = falcon_wait_idle(&vic->falcon);
+		if (err < 0) {
+			dev_err(vic->dev,
+				"failed to set application ID and FCE base\n");
+			return err;
+		}
 	}
 
 	return 0;
@@ -277,6 +304,8 @@ static int vic_load_firmware(struct vic *vic)
 
 	if (!vic->config->supports_sid) {
 		vic->can_use_context = false;
+	} else if (vic->config->has_riscv) {
+		vic->can_use_context = true;
 	} else if (fce_bin_data_offset != 0x0 && fce_bin_data_offset != 0xa5a5a5a5) {
 		/*
 		 * Firmware will access FCE through STREAMID0, so context
@@ -302,7 +331,6 @@ static int vic_load_firmware(struct vic *vic)
 	return err;
 }
 
-
 static int __maybe_unused vic_runtime_resume(struct device *dev)
 {
 	struct vic *vic = dev_get_drvdata(dev);
@@ -417,6 +445,7 @@ static const struct vic_config vic_t186_config = {
 	.firmware = NVIDIA_TEGRA_186_VIC_FIRMWARE,
 	.version = 0x18,
 	.supports_sid = true,
+	.transcfg_offset = 0x2044,
 };
 
 #define NVIDIA_TEGRA_194_VIC_FIRMWARE "nvidia/tegra194/vic.bin"
@@ -425,6 +454,7 @@ static const struct vic_config vic_t194_config = {
 	.firmware = NVIDIA_TEGRA_194_VIC_FIRMWARE,
 	.version = 0x19,
 	.supports_sid = true,
+	.transcfg_offset = 0x2044,
 };
 
 #define NVIDIA_TEGRA_234_VIC_FIRMWARE "nvidia/tegra234/vic.bin"
@@ -433,6 +463,18 @@ static const struct vic_config vic_t234_config = {
 	.firmware = NVIDIA_TEGRA_234_VIC_FIRMWARE,
 	.version = 0x23,
 	.supports_sid = true,
+	.transcfg_offset = 0x2044,
+};
+
+#define NVIDIA_TEGRA_264_VIC_FIRMWARE "nvidia/tegra264/vic.bin"
+#define NVIDIA_TEGRA_264_VIC_DESC "nvidia/tegra264/vic.bin.desc"
+
+static const struct vic_config vic_t264_config = {
+	.firmware = NVIDIA_TEGRA_264_VIC_FIRMWARE,
+	.version = 0x264,
+	.supports_sid = true,
+	.has_riscv = true,
+	.transcfg_offset = 0x2244,
 };
 
 static const struct of_device_id tegra_vic_of_match[] = {
@@ -441,6 +483,7 @@ static const struct of_device_id tegra_vic_of_match[] = {
 	{ .compatible = "nvidia,tegra186-vic", .data = &vic_t186_config },
 	{ .compatible = "nvidia,tegra194-vic", .data = &vic_t194_config },
 	{ .compatible = "nvidia,tegra234-vic", .data = &vic_t234_config },
+	{ .compatible = "nvidia,tegra264-vic", .data = &vic_t264_config },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, tegra_vic_of_match);
@@ -495,6 +538,7 @@ static int vic_probe(struct platform_device *pdev)
 
 	vic->falcon.dev = dev;
 	vic->falcon.regs = vic->regs;
+	vic->falcon.riscv = vic->config->has_riscv;
 
 	err = falcon_init(&vic->falcon);
 	if (err < 0)
@@ -571,3 +615,7 @@ MODULE_FIRMWARE(NVIDIA_TEGRA_194_VIC_FIRMWARE);
 #if IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC)
 MODULE_FIRMWARE(NVIDIA_TEGRA_234_VIC_FIRMWARE);
 #endif
+#if IS_ENABLED(CONFIG_ARCH_TEGRA_264_SOC)
+MODULE_FIRMWARE(NVIDIA_TEGRA_264_VIC_FIRMWARE);
+MODULE_FIRMWARE(NVIDIA_TEGRA_264_VIC_DESC);
+#endif
diff --git a/drivers/gpu/drm/tegra/vic.h b/drivers/gpu/drm/tegra/vic.h
index acf35aac948b2..aca98a09c9fb3 100644
--- a/drivers/gpu/drm/tegra/vic.h
+++ b/drivers/gpu/drm/tegra/vic.h
@@ -21,11 +21,10 @@
 #define CG_IDLE_CG_EN				(1 << 6)
 #define CG_WAKEUP_DLY_CNT(val)			((val & 0xf) << 16)
 
-#define VIC_TFBIF_TRANSCFG	0x00002044
-#define  TRANSCFG_ATT(i, v)	(((v) & 0x3) << (i * 4))
-#define  TRANSCFG_SID_HW	0
-#define  TRANSCFG_SID_PHY	1
-#define  TRANSCFG_SID_FALCON	2
+#define TRANSCFG_ATT(i, v)	(((v) & 0x3) << ((i) * 4))
+#define TRANSCFG_SID_HW		0
+#define TRANSCFG_SID_PHY	1
+#define TRANSCFG_SID_FALCON	2
 
 /* Firmware offsets */
 
diff --git a/drivers/gpu/drm/tests/Makefile b/drivers/gpu/drm/tests/Makefile
index 3b83e6616fb25..e7193b49cf91f 100644
--- a/drivers/gpu/drm/tests/Makefile
+++ b/drivers/gpu/drm/tests/Makefile
@@ -24,6 +24,7 @@ obj-$(CONFIG_DRM_KUNIT_TEST) += \
 	drm_probe_helper_test.o \
 	drm_rect_test.o \
 	drm_sysfb_modeset_test.o \
-	drm_fixp_test.o
+	drm_fixp_test.o \
+	drm_kunit_edid.o
 
 CFLAGS_drm_mm_test.o := $(DISABLE_STRUCTLEAK_PLUGIN)
diff --git a/drivers/gpu/drm/tests/drm_bridge_test.c b/drivers/gpu/drm/tests/drm_bridge_test.c
index 64b665580a885..15583ade84946 100644
--- a/drivers/gpu/drm/tests/drm_bridge_test.c
+++ b/drivers/gpu/drm/tests/drm_bridge_test.c
@@ -2,15 +2,24 @@
 /*
  * Kunit test for drm_bridge functions
  */
+#include <linux/cleanup.h>
+#include <linux/media-bus-format.h>
+
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_atomic_state_helper.h>
+#include <drm/drm_atomic_uapi.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_bridge_connector.h>
 #include <drm/drm_bridge_helper.h>
+#include <drm/drm_edid.h>
 #include <drm/drm_kunit_helpers.h>
+#include <drm/drm_managed.h>
 
 #include <kunit/device.h>
 #include <kunit/test.h>
 
+#include "drm_kunit_edid.h"
+
 /*
  * Mimick the typical "private" struct defined by a bridge driver, which
  * embeds a bridge plus other fields.
@@ -37,6 +46,21 @@ struct drm_bridge_init_priv {
 	bool destroyed;
 };
 
+struct drm_bridge_chain_priv {
+	struct drm_device drm;
+	struct drm_encoder encoder;
+	struct drm_plane *plane;
+	struct drm_crtc *crtc;
+	struct drm_connector *connector;
+	unsigned int num_bridges;
+
+	/**
+	 * @test_bridges: array of pointers to &struct drm_bridge_priv entries
+	 *                of which the first @num_bridges entries are valid.
+	 */
+	struct drm_bridge_priv **test_bridges;
+};
+
 static struct drm_bridge_priv *bridge_to_priv(struct drm_bridge *bridge)
 {
 	return container_of(bridge, struct drm_bridge_priv, bridge);
@@ -50,26 +74,6 @@ static void drm_test_bridge_priv_destroy(struct drm_bridge *bridge)
 	priv->destroyed = true;
 }
 
-static void drm_test_bridge_enable(struct drm_bridge *bridge)
-{
-	struct drm_bridge_priv *priv = bridge_to_priv(bridge);
-
-	priv->enable_count++;
-}
-
-static void drm_test_bridge_disable(struct drm_bridge *bridge)
-{
-	struct drm_bridge_priv *priv = bridge_to_priv(bridge);
-
-	priv->disable_count++;
-}
-
-static const struct drm_bridge_funcs drm_test_bridge_legacy_funcs = {
-	.destroy		= drm_test_bridge_priv_destroy,
-	.enable			= drm_test_bridge_enable,
-	.disable		= drm_test_bridge_disable,
-};
-
 static void drm_test_bridge_atomic_enable(struct drm_bridge *bridge,
 					  struct drm_atomic_commit *state)
 {
@@ -92,9 +96,254 @@ static const struct drm_bridge_funcs drm_test_bridge_atomic_funcs = {
 	.atomic_disable		= drm_test_bridge_atomic_disable,
 	.atomic_destroy_state	= drm_atomic_helper_bridge_destroy_state,
 	.atomic_duplicate_state	= drm_atomic_helper_bridge_duplicate_state,
-	.atomic_reset		= drm_atomic_helper_bridge_reset,
+	.atomic_create_state	= drm_atomic_helper_bridge_create_state,
+};
+
+static int dummy_clear_infoframe(struct drm_bridge *bridge)
+{
+	return 0;
+}
+
+static int dummy_write_infoframe(struct drm_bridge *bridge, const u8 *buffer,
+				 size_t len)
+{
+	return 0;
+}
+
+static const struct drm_bridge_funcs drm_test_bridge_bus_fmts_funcs = {
+	.atomic_get_output_bus_fmts	= drm_atomic_helper_bridge_get_hdmi_output_bus_fmts,
+	.atomic_destroy_state		= drm_atomic_helper_bridge_destroy_state,
+	.atomic_duplicate_state		= drm_atomic_helper_bridge_duplicate_state,
+	.atomic_create_state		= drm_atomic_helper_bridge_create_state,
+	.hdmi_write_avi_infoframe	= dummy_write_infoframe,
+	.hdmi_write_hdmi_infoframe	= dummy_write_infoframe,
+	.hdmi_clear_avi_infoframe	= dummy_clear_infoframe,
+	.hdmi_clear_hdmi_infoframe	= dummy_clear_infoframe,
+};
+
+/**
+ * struct fmt_tuple - a tuple of input/output MEDIA_BUS_FMT_*
+ */
+struct fmt_tuple {
+	u32 in_fmt;
+	u32 out_fmt;
+};
+
+/*
+ * Format mapping that only accepts RGB888, and outputs only RGB888
+ */
+static const struct fmt_tuple rgb8_passthrough[] = {
+	{ MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
+};
+
+/*
+ * Format mapping that only accepts YUV444, and outputs only YUV444
+ */
+static const struct fmt_tuple yuv8_passthrough[] = {
+	{ MEDIA_BUS_FMT_YUV8_1X24,   MEDIA_BUS_FMT_YUV8_1X24 },
+};
+
+/*
+ * Format mapping where 8bpc RGB -> 8bpc YUV444, or ID(RGB) or ID(YUV444)
+ */
+static const struct fmt_tuple rgb8_to_yuv8_or_id[] = {
+	{ MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
+	{ MEDIA_BUS_FMT_YUV8_1X24,   MEDIA_BUS_FMT_YUV8_1X24 },
+	{ MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+};
+
+static const struct fmt_tuple rgb8_to_id_yuv8_or_yuv8_to_yuv422_yuv420[] = {
+	{ MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
+	{ MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+	{ MEDIA_BUS_FMT_YUV8_1X24,   MEDIA_BUS_FMT_UYVY8_1X16 },
+	{ MEDIA_BUS_FMT_YUV8_1X24,   MEDIA_BUS_FMT_UYYVYY8_0_5X24 },
+};
+
+/*
+ * Format mapping where 8bpc YUV444 -> 8bpc RGB, or ID(YUV444)
+ */
+static const struct fmt_tuple yuv8_to_rgb8_or_id[] = {
+	{ MEDIA_BUS_FMT_YUV8_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+	{ MEDIA_BUS_FMT_YUV8_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
+};
+
+/*
+ * A format mapping that acts like a video processor that generates an RGB signal
+ */
+static const struct fmt_tuple rgb_producer[] = {
+	{ MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_RGB888_1X24 },
+	{ MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_RGB101010_1X30 },
+	{ MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_RGB121212_1X36 },
+};
+
+/*
+ * A format mapping that acts like a video processor that generates an 8-bit RGB,
+ * YUV444 or YUV420 signal
+ */
+static const struct fmt_tuple rgb_yuv444_yuv420_producer[] = {
+	{ MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_RGB888_1X24 },
+	{ MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_YUV8_1X24 },
+	{ MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_UYYVYY8_0_5X24 },
+};
+
+static const struct fmt_tuple rgb8_yuv444_yuv422_passthrough[] = {
+	{ MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
+	{ MEDIA_BUS_FMT_YUV8_1X24,   MEDIA_BUS_FMT_YUV8_1X24 },
+	{ MEDIA_BUS_FMT_UYVY8_1X16,  MEDIA_BUS_FMT_UYVY8_1X16 },
+};
+
+static const struct fmt_tuple yuv444_yuv422_rgb8_passthrough[] = {
+	{ MEDIA_BUS_FMT_YUV8_1X24,   MEDIA_BUS_FMT_YUV8_1X24 },
+	{ MEDIA_BUS_FMT_UYVY8_1X16,  MEDIA_BUS_FMT_UYVY8_1X16 },
+	{ MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
 };
 
+static bool fmt_in_list(const u32 fmt, const u32 *out_fmts, const size_t num_fmts)
+{
+	size_t i;
+
+	for (i = 0; i < num_fmts; i++)
+		if (out_fmts[i] == fmt)
+			return true;
+
+	return false;
+}
+
+/**
+ * get_tuples_out_fmts - Get unique output formats of a &struct fmt_tuple list
+ * @fmt_tuples: array of &struct fmt_tuple
+ * @num_fmt_tuples: number of entries in @fmt_tuples
+ * @out_fmts: target array to store the unique output bus formats
+ *
+ * Returns the number of unique output formats, i.e. the number of entries in
+ * @out_fmts that were populated with sensible values.
+ */
+static size_t get_tuples_out_fmts(const struct fmt_tuple *fmt_tuples,
+				  const size_t num_fmt_tuples, u32 *out_fmts)
+{
+	size_t num_unique = 0;
+	size_t i;
+
+	for (i = 0; i < num_fmt_tuples; i++)
+		if (!fmt_in_list(fmt_tuples[i].out_fmt, out_fmts, num_unique))
+			out_fmts[num_unique++] = fmt_tuples[i].out_fmt;
+
+	return num_unique;
+}
+
+#define DEFINE_FMT_FUNCS_FROM_TUPLES(name) \
+static u32 *drm_test_bridge_ ## name ## _out_fmts(struct drm_bridge *bridge,			\
+						  struct drm_bridge_state *bridge_state,	\
+						  struct drm_crtc_state *crtc_state,		\
+						  struct drm_connector_state *conn_state,	\
+						  unsigned int *num_output_fmts)		\
+{												\
+	u32 *out_fmts = kcalloc(ARRAY_SIZE((name)), sizeof(u32), GFP_KERNEL);			\
+												\
+	if (out_fmts)										\
+		*num_output_fmts = get_tuples_out_fmts((name), ARRAY_SIZE((name)), out_fmts);	\
+	else											\
+		*num_output_fmts = 0;								\
+												\
+	return out_fmts;									\
+}												\
+												\
+static u32 *drm_test_bridge_ ## name ## _in_fmts(struct drm_bridge *bridge,			\
+						 struct drm_bridge_state *bridge_state,		\
+						 struct drm_crtc_state *crtc_state,		\
+						 struct drm_connector_state *conn_state,	\
+						 u32 output_fmt,				\
+						 unsigned int *num_input_fmts)			\
+{												\
+	u32 *in_fmts = kcalloc(ARRAY_SIZE((name)), sizeof(u32), GFP_KERNEL);			\
+	unsigned int num_fmts = 0;								\
+	size_t i;										\
+												\
+	if (!in_fmts) {										\
+		*num_input_fmts = 0;								\
+		return NULL;									\
+	}											\
+												\
+	for (i = 0; i < ARRAY_SIZE((name)); i++)						\
+		if ((name)[i].out_fmt == output_fmt)						\
+			in_fmts[num_fmts++] = (name)[i].in_fmt;					\
+												\
+	*num_input_fmts = num_fmts;								\
+												\
+	return in_fmts;										\
+}
+
+#define DRM_BRIDGE_ATOMIC_WITH_BUS_FMT_HDMI_FUNC(ident, input_fmts_func, output_fmts_func,	\
+						 hdmi_write_infoframe_func,			\
+						 hdmi_clear_infoframe_func)			\
+static const struct drm_bridge_funcs (ident) = {						\
+	.atomic_enable			= drm_test_bridge_atomic_enable,			\
+	.atomic_disable			= drm_test_bridge_atomic_disable,			\
+	.atomic_destroy_state		= drm_atomic_helper_bridge_destroy_state,		\
+	.atomic_duplicate_state		= drm_atomic_helper_bridge_duplicate_state,		\
+	.atomic_create_state		= drm_atomic_helper_bridge_create_state,		\
+	.atomic_get_input_bus_fmts	= (input_fmts_func),					\
+	.atomic_get_output_bus_fmts	= (output_fmts_func),					\
+	.hdmi_write_avi_infoframe	= (hdmi_write_infoframe_func),				\
+	.hdmi_clear_avi_infoframe	= (hdmi_clear_infoframe_func),				\
+	.hdmi_write_hdmi_infoframe	= (hdmi_write_infoframe_func),				\
+	.hdmi_clear_hdmi_infoframe	= (hdmi_clear_infoframe_func),				\
+}
+
+#define DRM_BRIDGE_ATOMIC_WITH_BUS_FMT_FUNC(ident, input_fmts_func, output_fmts_func)		\
+	DRM_BRIDGE_ATOMIC_WITH_BUS_FMT_HDMI_FUNC(ident, input_fmts_func, output_fmts_func,	\
+						 NULL, NULL)
+
+#define DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(_name)						\
+	DRM_BRIDGE_ATOMIC_WITH_BUS_FMT_FUNC(_name ## _funcs,				\
+					    drm_test_bridge_ ## _name ## _in_fmts,	\
+					    drm_test_bridge_ ## _name ## _out_fmts)
+
+static int drm_test_bridge_write_infoframe_stub(struct drm_bridge *bridge,
+						const u8 *buffer, size_t len)
+{
+	return 0;
+}
+
+static int drm_test_bridge_clear_infoframe_stub(struct drm_bridge *bridge)
+{
+	return 0;
+}
+
+#define DRM_BRIDGE_ATOMIC_WITH_BUS_FMT_HDMI(_name)						\
+	DRM_BRIDGE_ATOMIC_WITH_BUS_FMT_HDMI_FUNC(_name ## _hdmi ## _funcs,			\
+						 drm_test_bridge_ ## _name ## _in_fmts,		\
+						 drm_test_bridge_ ## _name ## _out_fmts,	\
+						 drm_test_bridge_write_infoframe_stub,		\
+						 drm_test_bridge_clear_infoframe_stub)
+DEFINE_FMT_FUNCS_FROM_TUPLES(rgb8_passthrough)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(rgb8_passthrough);
+
+DEFINE_FMT_FUNCS_FROM_TUPLES(yuv8_passthrough)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(yuv8_passthrough);
+
+DEFINE_FMT_FUNCS_FROM_TUPLES(rgb8_to_yuv8_or_id)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(rgb8_to_yuv8_or_id);
+
+DEFINE_FMT_FUNCS_FROM_TUPLES(yuv8_to_rgb8_or_id)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(yuv8_to_rgb8_or_id);
+
+DEFINE_FMT_FUNCS_FROM_TUPLES(rgb_producer)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(rgb_producer);
+
+DEFINE_FMT_FUNCS_FROM_TUPLES(rgb_yuv444_yuv420_producer)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(rgb_yuv444_yuv420_producer);
+
+DEFINE_FMT_FUNCS_FROM_TUPLES(rgb8_to_id_yuv8_or_yuv8_to_yuv422_yuv420)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(rgb8_to_id_yuv8_or_yuv8_to_yuv422_yuv420);
+
+DEFINE_FMT_FUNCS_FROM_TUPLES(rgb8_yuv444_yuv422_passthrough)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(rgb8_yuv444_yuv422_passthrough);
+
+DEFINE_FMT_FUNCS_FROM_TUPLES(yuv444_yuv422_rgb8_passthrough)
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT(yuv444_yuv422_rgb8_passthrough);
+DRM_BRIDGE_ATOMIC_WITH_BUS_FMT_HDMI(yuv444_yuv422_rgb8_passthrough);
+
 KUNIT_DEFINE_ACTION_WRAPPER(drm_bridge_remove_wrapper,
 			    drm_bridge_remove,
 			    struct drm_bridge *);
@@ -178,6 +427,196 @@ drm_test_bridge_init(struct kunit *test, const struct drm_bridge_funcs *funcs)
 	return priv;
 }
 
+static struct drm_bridge_chain_priv *
+drm_test_bridge_chain_init(struct kunit *test, unsigned int num_bridges,
+			   const struct drm_bridge_funcs **funcs)
+{
+	struct drm_bridge_chain_priv *priv;
+	const struct drm_edid *edid;
+	struct drm_bridge *prev;
+	struct drm_encoder *enc;
+	struct drm_bridge *bridge;
+	struct drm_device *drm;
+	bool has_hdmi = false;
+	struct device *dev;
+	unsigned int i;
+	int ret;
+
+	dev = drm_kunit_helper_alloc_device(test);
+	if (IS_ERR(dev))
+		return ERR_CAST(dev);
+
+	priv = drm_kunit_helper_alloc_drm_device(test, dev, struct drm_bridge_chain_priv,
+						 drm, DRIVER_MODESET | DRIVER_ATOMIC);
+	if (IS_ERR(priv))
+		return ERR_CAST(priv);
+
+	drm = &priv->drm;
+
+	priv->test_bridges = drmm_kmalloc_array(drm, num_bridges, sizeof(*priv->test_bridges),
+						GFP_KERNEL);
+	if (!priv->test_bridges)
+		return ERR_PTR(-ENOMEM);
+
+	priv->num_bridges = num_bridges;
+
+	for (i = 0; i < num_bridges; i++) {
+		priv->test_bridges[i] = devm_drm_bridge_alloc(dev, struct drm_bridge_priv,
+							      bridge, funcs[i]);
+		if (IS_ERR(priv->test_bridges[i]))
+			return ERR_CAST(priv->test_bridges[i]);
+
+		priv->test_bridges[i]->data = priv;
+	}
+
+	priv->plane = drm_kunit_helper_create_primary_plane(test, drm, NULL, NULL,
+							    NULL, 0, NULL);
+	if (IS_ERR(priv->plane))
+		return ERR_CAST(priv->plane);
+
+	priv->crtc = drm_kunit_helper_create_crtc(test, drm, priv->plane, NULL,
+						  NULL, NULL);
+	if (IS_ERR(priv->crtc))
+		return ERR_CAST(priv->crtc);
+
+	enc = &priv->encoder;
+	ret = drmm_encoder_init(drm, enc, NULL, DRM_MODE_ENCODER_TMDS, NULL);
+	if (ret)
+		return ERR_PTR(ret);
+
+	enc->possible_crtcs = drm_crtc_mask(priv->crtc);
+
+	prev = NULL;
+	for (i = 0; i < num_bridges; i++) {
+		bridge = &priv->test_bridges[i]->bridge;
+		bridge->type = DRM_MODE_CONNECTOR_VIRTUAL;
+
+		if (bridge->funcs->hdmi_write_hdmi_infoframe) {
+			has_hdmi = true;
+			bridge->ops |= DRM_BRIDGE_OP_HDMI;
+			bridge->type = DRM_MODE_CONNECTOR_HDMIA;
+			bridge->vendor = "LNX";
+			bridge->product = "KUnit";
+			bridge->supported_formats = (BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444) |
+						     BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444) |
+						     BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422) |
+						     BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420));
+		}
+
+		ret = drm_kunit_bridge_add(test, bridge);
+		if (ret)
+			return ERR_PTR(ret);
+
+		ret = drm_bridge_attach(enc, bridge, prev, 0);
+		if (ret)
+			return ERR_PTR(ret);
+
+		prev = bridge;
+	}
+
+	priv->connector = drm_bridge_connector_init(drm, enc);
+	if (IS_ERR(priv->connector))
+		return ERR_CAST(priv->connector);
+
+	drm_connector_attach_encoder(priv->connector, enc);
+
+	drm_mode_config_reset(drm);
+
+	if (!has_hdmi)
+		return priv;
+
+	scoped_guard(mutex, &drm->mode_config.mutex) {
+		edid = drm_edid_alloc(test_edid_hdmi_1080p_rgb_yuv_4k_yuv420_dc_max_200mhz,
+				ARRAY_SIZE(test_edid_hdmi_1080p_rgb_yuv_4k_yuv420_dc_max_200mhz));
+		if (!edid)
+			return ERR_PTR(-EINVAL);
+
+		drm_edid_connector_update(priv->connector, edid);
+		KUNIT_ASSERT_GT(test, drm_edid_connector_add_modes(priv->connector), 0);
+
+		ret = priv->connector->funcs->fill_modes(priv->connector, 4096, 4096);
+	}
+
+	return priv;
+}
+
+static struct drm_bridge_init_priv *
+drm_test_bridge_hdmi_init(struct kunit *test, const struct drm_bridge_funcs *funcs,
+			  unsigned int supported_formats, int max_bpc)
+{
+	struct drm_bridge_init_priv *priv;
+	struct drm_encoder *enc;
+	struct drm_bridge *bridge;
+	struct drm_device *drm;
+	struct device *dev;
+	int ret;
+
+	dev = drm_kunit_helper_alloc_device(test);
+	if (IS_ERR(dev))
+		return ERR_CAST(dev);
+
+	priv = drm_kunit_helper_alloc_drm_device(test, dev,
+						 struct drm_bridge_init_priv, drm,
+						 DRIVER_MODESET | DRIVER_ATOMIC);
+	if (IS_ERR(priv))
+		return ERR_CAST(priv);
+
+	priv->test_bridge = devm_drm_bridge_alloc(dev, struct drm_bridge_priv, bridge, funcs);
+	if (IS_ERR(priv->test_bridge))
+		return ERR_CAST(priv->test_bridge);
+
+	priv->test_bridge->data = priv;
+
+	drm = &priv->drm;
+	priv->plane = drm_kunit_helper_create_primary_plane(test, drm,
+							    NULL,
+							    NULL,
+							    NULL, 0,
+							    NULL);
+	if (IS_ERR(priv->plane))
+		return ERR_CAST(priv->plane);
+
+	priv->crtc = drm_kunit_helper_create_crtc(test, drm,
+						  priv->plane, NULL,
+						  NULL,
+						  NULL);
+	if (IS_ERR(priv->crtc))
+		return ERR_CAST(priv->crtc);
+
+	enc = &priv->encoder;
+	ret = drmm_encoder_init(drm, enc, NULL, DRM_MODE_ENCODER_TMDS, NULL);
+	if (ret)
+		return ERR_PTR(ret);
+
+	enc->possible_crtcs = drm_crtc_mask(priv->crtc);
+
+	bridge = &priv->test_bridge->bridge;
+	bridge->type = DRM_MODE_CONNECTOR_HDMIA;
+	bridge->supported_formats = supported_formats;
+	bridge->max_bpc = max_bpc;
+	bridge->ops |= DRM_BRIDGE_OP_HDMI;
+	bridge->vendor = "LNX";
+	bridge->product = "KUnit";
+
+	ret = drm_kunit_bridge_add(test, bridge);
+	if (ret)
+		return ERR_PTR(ret);
+
+	ret = drm_bridge_attach(enc, bridge, NULL, 0);
+	if (ret)
+		return ERR_PTR(ret);
+
+	priv->connector = drm_bridge_connector_init(drm, enc);
+	if (IS_ERR(priv->connector))
+		return ERR_CAST(priv->connector);
+
+	drm_connector_attach_encoder(priv->connector, enc);
+
+	drm_mode_config_reset(drm);
+
+	return priv;
+}
+
 /*
  * Test that drm_bridge_get_current_state() returns the last committed
  * state for an atomic bridge.
@@ -236,35 +675,8 @@ static void drm_test_drm_bridge_get_current_state_atomic(struct kunit *test)
 	drm_modeset_acquire_fini(&ctx);
 }
 
-/*
- * Test that drm_bridge_get_current_state() returns NULL for a
- * non-atomic bridge.
- */
-static void drm_test_drm_bridge_get_current_state_legacy(struct kunit *test)
-{
-	struct drm_bridge_init_priv *priv;
-	struct drm_bridge *bridge;
-
-	priv = drm_test_bridge_init(test, &drm_test_bridge_legacy_funcs);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv);
-
-	/*
-	 * NOTE: Strictly speaking, we should take the bridge->base.lock
-	 * before calling that function. However, bridge->base is only
-	 * initialized if the bridge is atomic, while we explicitly
-	 * initialize one that isn't there.
-	 *
-	 * In order to avoid unnecessary warnings, let's skip the
-	 * locking. The function would return NULL in all cases anyway,
-	 * so we don't really have any concurrency to worry about.
-	 */
-	bridge = &priv->test_bridge->bridge;
-	KUNIT_EXPECT_NULL(test, drm_bridge_get_current_state(bridge));
-}
-
 static struct kunit_case drm_bridge_get_current_state_tests[] = {
 	KUNIT_CASE(drm_test_drm_bridge_get_current_state_atomic),
-	KUNIT_CASE(drm_test_drm_bridge_get_current_state_legacy),
 	{ }
 };
 
@@ -370,65 +782,92 @@ static void drm_test_drm_bridge_helper_reset_crtc_atomic_disabled(struct kunit *
 }
 
 /*
- * Test that a non-atomic bridge is properly power-cycled when calling
- * drm_bridge_helper_reset_crtc().
+ * Test that a bridge using the drm_atomic_helper_bridge_get_hdmi_output_bus_fmts()
+ * function for &drm_bridge_funcs.atomic_get_output_bus_fmts behaves as expected
+ * for an HDMI connector bridge. Does so by creating an HDMI bridge connector
+ * with RGB444, YCBCR444, and YCBCR420 (but not YCBCR422) as supported formats,
+ * sets the output depth to 8 bits per component, and then validates the returned
+ * list of bus formats.
  */
-static void drm_test_drm_bridge_helper_reset_crtc_legacy(struct kunit *test)
+static void drm_test_drm_bridge_helper_hdmi_output_bus_fmts(struct kunit *test)
 {
+	struct drm_connector_state *conn_state;
+	struct drm_bridge_state *bridge_state;
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_bridge_init_priv *priv;
+	struct drm_crtc_state *crtc_state;
+	struct drm_atomic_commit *state;
 	struct drm_display_mode *mode;
-	struct drm_bridge_priv *bridge_priv;
+	unsigned int num_output_fmts;
+	struct drm_bridge *bridge;
+	u32 *out_bus_fmts;
 	int ret;
 
-	priv = drm_test_bridge_init(test, &drm_test_bridge_legacy_funcs);
+	priv = drm_test_bridge_hdmi_init(test, &drm_test_bridge_bus_fmts_funcs,
+					 BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444) |
+					 BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444) |
+					 BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420),
+					 12);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv);
 
-	mode = drm_kunit_display_mode_from_cea_vic(test, &priv->drm, 16);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mode);
+	bridge = &priv->test_bridge->bridge;
 
 	drm_modeset_acquire_init(&ctx, 0);
 
+	state = drm_kunit_helper_atomic_state_alloc(test, &priv->drm, &ctx);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
 retry_commit:
-	ret = drm_kunit_helper_enable_crtc_connector(test,
-						     &priv->drm, priv->crtc,
-						     priv->connector,
-						     mode,
-						     &ctx);
+	conn_state = drm_atomic_get_connector_state(state, priv->connector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+
+	conn_state->hdmi.output_bpc = 8;
+
+	mode = drm_kunit_display_mode_from_cea_vic(test, &priv->drm, 16);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mode);
+
+	ret = drm_atomic_set_crtc_for_connector(conn_state, priv->crtc);
 	if (ret == -EDEADLK) {
 		drm_modeset_backoff(&ctx);
 		goto retry_commit;
 	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
-	drm_modeset_drop_locks(&ctx);
-	drm_modeset_acquire_fini(&ctx);
-
-	bridge_priv = priv->test_bridge;
-	KUNIT_ASSERT_EQ(test, bridge_priv->enable_count, 1);
-	KUNIT_ASSERT_EQ(test, bridge_priv->disable_count, 0);
+	crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
 
-	drm_modeset_acquire_init(&ctx, 0);
-
-retry_reset:
-	ret = drm_bridge_helper_reset_crtc(&bridge_priv->bridge, &ctx);
+	ret = drm_atomic_set_mode_for_crtc(crtc_state, mode);
 	if (ret == -EDEADLK) {
 		drm_modeset_backoff(&ctx);
-		goto retry_reset;
+		goto retry_commit;
 	}
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
+	crtc_state->enable = true;
+	crtc_state->active = true;
+
+	bridge_state = drm_atomic_get_bridge_state(state, bridge);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bridge_state);
+
+	out_bus_fmts = bridge->funcs->atomic_get_output_bus_fmts(
+		bridge, bridge_state, crtc_state, conn_state, &num_output_fmts);
+	KUNIT_EXPECT_NOT_NULL(test, out_bus_fmts);
+	KUNIT_EXPECT_EQ(test, num_output_fmts, 3);
+
+	KUNIT_EXPECT_EQ(test, out_bus_fmts[0], MEDIA_BUS_FMT_RGB888_1X24);
+	KUNIT_EXPECT_EQ(test, out_bus_fmts[1], MEDIA_BUS_FMT_YUV8_1X24);
+	KUNIT_EXPECT_EQ(test, out_bus_fmts[2], MEDIA_BUS_FMT_UYYVYY8_0_5X24);
+
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
 
-	KUNIT_EXPECT_EQ(test, bridge_priv->enable_count, 2);
-	KUNIT_EXPECT_EQ(test, bridge_priv->disable_count, 1);
+	kfree(out_bus_fmts);
 }
 
 static struct kunit_case drm_bridge_helper_reset_crtc_tests[] = {
 	KUNIT_CASE(drm_test_drm_bridge_helper_reset_crtc_atomic),
 	KUNIT_CASE(drm_test_drm_bridge_helper_reset_crtc_atomic_disabled),
-	KUNIT_CASE(drm_test_drm_bridge_helper_reset_crtc_legacy),
+	KUNIT_CASE(drm_test_drm_bridge_helper_hdmi_output_bus_fmts),
 	{ }
 };
 
@@ -506,14 +945,442 @@ static struct kunit_suite drm_bridge_alloc_test_suite = {
 	.test_cases = drm_bridge_alloc_tests,
 };
 
+/**
+ * drm_test_bridge_chain_verify_fmt - Verify bridge chain format selection
+ * @test: pointer to KUnit test object
+ * @priv: pointer to a &struct drm_bridge_chain_priv for this chain
+ * @expected: constant array of &struct fmt_tuple describing the expected
+ *            input and output bus formats
+ * @num_expected: number of entries in @expected
+ *
+ * Runs the KUNIT_EXPECT clauses to verify the bridge chain format selection
+ * resulted in the expected formats. If %0 is given as a format in a
+ * &struct fmt_tuple, then it is understood to mean "any".
+ *
+ * Must be called with the modeset lock held.
+ */
+static void drm_test_bridge_chain_verify_fmt(struct kunit *test,
+					     struct drm_bridge_chain_priv *priv,
+					     const struct fmt_tuple *const expected,
+					     const unsigned int num_expected)
+{
+	struct drm_bridge_state *bstate;
+	unsigned int i = 0;
+
+	drm_for_each_bridge_in_chain(&priv->encoder, bridge) {
+		KUNIT_ASSERT_LT(test, i, num_expected);
+
+		bstate = drm_bridge_get_current_state(bridge);
+		if (expected[i].in_fmt)
+			KUNIT_EXPECT_EQ(test, bstate->input_bus_cfg.format,
+					expected[i].in_fmt);
+		if (expected[i].out_fmt)
+			KUNIT_EXPECT_EQ(test, bstate->output_bus_cfg.format,
+					expected[i].out_fmt);
+
+		i++;
+	}
+
+	KUNIT_ASSERT_EQ_MSG(test, i, num_expected,
+			    "Fewer bridges (%u) than expected (%u)\n", i, num_expected);
+}
+
+/*
+ * Test that constructs a bridge chain in which an RGB888 producer is chained to
+ * two bridges that will convert from RGB to YUV and from YUV to RGB respectively.
+ *
+ * The test requests an output color_format of RGB using the color_format property,
+ * so to satisfy this request, the bridge chain must take a detour over YUV.
+ */
+static void drm_test_bridge_rgb_yuv_rgb(struct kunit *test)
+{
+	static const struct drm_bridge_funcs *funcs[] = {
+		&rgb_producer_funcs,
+		&rgb8_to_yuv8_or_id_funcs,
+		&yuv8_to_rgb8_or_id_funcs,
+	};
+	static const struct fmt_tuple expected[] = {
+		{ MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_RGB888_1X24 },
+		{ MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+		{ MEDIA_BUS_FMT_YUV8_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
+	};
+	struct drm_connector_state *conn_state;
+	struct drm_modeset_acquire_ctx ctx;
+	struct drm_bridge_chain_priv *priv;
+	struct drm_crtc_state *crtc_state;
+	struct drm_atomic_commit *state;
+	struct drm_display_mode *mode;
+	int ret;
+
+	priv = drm_test_bridge_chain_init(test, ARRAY_SIZE(funcs), funcs);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv);
+
+	drm_modeset_acquire_init(&ctx, 0);
+
+	state = drm_kunit_helper_atomic_state_alloc(test, &priv->drm, &ctx);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+retry_commit:
+	conn_state = drm_atomic_get_connector_state(state, priv->connector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+
+	mode = drm_kunit_display_mode_from_cea_vic(test, &priv->drm, 16);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mode);
+
+	conn_state->color_format = DRM_CONNECTOR_COLOR_FORMAT_RGB444;
+
+	ret = drm_atomic_set_crtc_for_connector(conn_state, priv->crtc);
+	if (ret == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry_commit;
+	}
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+
+	ret = drm_atomic_set_mode_for_crtc(crtc_state, mode);
+	if (ret == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry_commit;
+	}
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	crtc_state->enable = true;
+	crtc_state->active = true;
+
+	ret = drm_atomic_commit(state);
+	if (ret == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry_commit;
+	}
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	drm_test_bridge_chain_verify_fmt(test, priv, expected, ARRAY_SIZE(expected));
+
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+}
+
+/*
+ * Test in which a bridge capable of producing RGB, YUV444 and YUV420 has to
+ * produce RGB and convert with a downstream bridge in the chain to reach the
+ * requested YUV444 color format, as no direct path exists between its YUV444
+ * and the last bridge.
+ *
+ * The rationale behind this test is to devise a scenario in which naively
+ * assuming any format the video processor can output, and the connector
+ * requests, is the right format to pick, does not work.
+ */
+static void drm_test_bridge_must_convert_to_yuv444(struct kunit *test)
+{
+	static const struct drm_bridge_funcs *funcs[] = {
+		&rgb_yuv444_yuv420_producer_funcs,
+		&rgb8_passthrough_funcs,
+		&rgb8_to_id_yuv8_or_yuv8_to_yuv422_yuv420_funcs,
+		&rgb8_yuv444_yuv422_passthrough_funcs,
+	};
+	static const struct fmt_tuple expected[] = {
+		{ MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_RGB888_1X24 },
+		{ MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
+		{ MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+		{ MEDIA_BUS_FMT_YUV8_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+	};
+	struct drm_connector_state *conn_state;
+	struct drm_modeset_acquire_ctx ctx;
+	struct drm_bridge_chain_priv *priv;
+	struct drm_crtc_state *crtc_state;
+	struct drm_atomic_commit *state;
+	struct drm_display_mode *mode;
+	int ret;
+
+	priv = drm_test_bridge_chain_init(test, ARRAY_SIZE(funcs), funcs);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv);
+
+	drm_modeset_acquire_init(&ctx, 0);
+
+	state = drm_kunit_helper_atomic_state_alloc(test, &priv->drm, &ctx);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+retry_commit:
+	conn_state = drm_atomic_get_connector_state(state, priv->connector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+
+	mode = drm_kunit_display_mode_from_cea_vic(test, &priv->drm, 16);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mode);
+
+	conn_state->color_format = DRM_CONNECTOR_COLOR_FORMAT_YCBCR444;
+
+	ret = drm_atomic_set_crtc_for_connector(conn_state, priv->crtc);
+	if (ret == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry_commit;
+	}
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+
+	ret = drm_atomic_set_mode_for_crtc(crtc_state, mode);
+	if (ret == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry_commit;
+	}
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	crtc_state->enable = true;
+	crtc_state->active = true;
+
+	ret = drm_atomic_commit(state);
+	if (ret == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry_commit;
+	}
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	drm_test_bridge_chain_verify_fmt(test, priv, expected, ARRAY_SIZE(expected));
+
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+}
+
+/*
+ * Test which checks that no matter the order of bus formats returned by an
+ * HDMI bridge, RGB is preferred on DRM_CONNECTOR_COLOR_FORMAT_AUTO if it's
+ * available.
+ */
+static void drm_test_bridge_hdmi_auto_rgb(struct kunit *test)
+{
+	static const struct drm_bridge_funcs *funcs[] = {
+		&rgb_yuv444_yuv420_producer_funcs,
+		&yuv444_yuv422_rgb8_passthrough_hdmi_funcs,
+	};
+	static const struct fmt_tuple expected[] = {
+		{ MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_RGB888_1X24 },
+		{ MEDIA_BUS_FMT_RGB888_1X24, MEDIA_BUS_FMT_RGB888_1X24 },
+	};
+	struct drm_connector_state *conn_state;
+	struct drm_modeset_acquire_ctx ctx;
+	struct drm_bridge_chain_priv *priv;
+	struct drm_crtc_state *crtc_state;
+	struct drm_atomic_commit *state;
+	struct drm_display_mode *mode;
+	int ret;
+
+	priv = drm_test_bridge_chain_init(test, ARRAY_SIZE(funcs), funcs);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv);
+
+	drm_modeset_acquire_init(&ctx, 0);
+
+	state = drm_kunit_helper_atomic_state_alloc(test, &priv->drm, &ctx);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+retry_commit:
+	conn_state = drm_atomic_get_connector_state(state, priv->connector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+
+	mode = drm_kunit_display_mode_from_cea_vic(test, &priv->drm, 16);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mode);
+
+	KUNIT_ASSERT_EQ(test, conn_state->color_format, DRM_CONNECTOR_COLOR_FORMAT_AUTO);
+
+	ret = drm_atomic_set_crtc_for_connector(conn_state, priv->crtc);
+	if (ret == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry_commit;
+	}
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+
+	ret = drm_atomic_set_mode_for_crtc(crtc_state, mode);
+	if (ret == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry_commit;
+	}
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	crtc_state->enable = true;
+	crtc_state->active = true;
+
+	ret = drm_atomic_commit(state);
+	if (ret == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry_commit;
+	}
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	KUNIT_EXPECT_EQ(test, conn_state->hdmi.output_format, DRM_OUTPUT_COLOR_FORMAT_RGB444);
+
+	drm_test_bridge_chain_verify_fmt(test, priv, expected, ARRAY_SIZE(expected));
+
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+}
+
+/*
+ * Test which checks that DRM_CONNECTOR_COLOR_FORMAT_AUTO on non-HDMI connectors
+ * will result in the first bus format on the output to be picked.
+ */
+static void drm_test_bridge_auto_first(struct kunit *test)
+{
+	static const struct drm_bridge_funcs *funcs[] = {
+		&rgb_yuv444_yuv420_producer_funcs,
+		&yuv444_yuv422_rgb8_passthrough_funcs,
+	};
+	static const struct fmt_tuple expected[] = {
+		{ MEDIA_BUS_FMT_FIXED, MEDIA_BUS_FMT_YUV8_1X24 },
+		{ MEDIA_BUS_FMT_YUV8_1X24, MEDIA_BUS_FMT_YUV8_1X24 },
+	};
+	struct drm_connector_state *conn_state;
+	struct drm_modeset_acquire_ctx ctx;
+	struct drm_bridge_chain_priv *priv;
+	struct drm_crtc_state *crtc_state;
+	struct drm_atomic_commit *state;
+	struct drm_display_mode *mode;
+	int ret;
+
+	priv = drm_test_bridge_chain_init(test, ARRAY_SIZE(funcs), funcs);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv);
+
+	drm_modeset_acquire_init(&ctx, 0);
+
+	state = drm_kunit_helper_atomic_state_alloc(test, &priv->drm, &ctx);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+retry_commit:
+	conn_state = drm_atomic_get_connector_state(state, priv->connector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+
+	mode = drm_kunit_display_mode_from_cea_vic(test, &priv->drm, 16);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mode);
+
+	KUNIT_ASSERT_EQ(test, conn_state->color_format, DRM_CONNECTOR_COLOR_FORMAT_AUTO);
+
+	ret = drm_atomic_set_crtc_for_connector(conn_state, priv->crtc);
+	if (ret == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry_commit;
+	}
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+
+	ret = drm_atomic_set_mode_for_crtc(crtc_state, mode);
+	if (ret == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry_commit;
+	}
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	crtc_state->enable = true;
+	crtc_state->active = true;
+
+	ret = drm_atomic_commit(state);
+	if (ret == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry_commit;
+	}
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	drm_test_bridge_chain_verify_fmt(test, priv, expected, ARRAY_SIZE(expected));
+
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+}
+
+/*
+ * Test which checks that in a configuration of bridge chains where an RGB
+ * producer is hooked to a YUV444-only pass-through, the atomic commit fails as
+ * the bridge format selection cannot find a valid sequence of bus formats.
+ */
+static void drm_test_bridge_rgb_yuv_no_path(struct kunit *test)
+{
+	static const struct drm_bridge_funcs *funcs[] = {
+		&rgb_producer_funcs,
+		&yuv8_passthrough_funcs,
+		&rgb8_yuv444_yuv422_passthrough_funcs,
+	};
+	struct drm_connector_state *conn_state;
+	struct drm_modeset_acquire_ctx ctx;
+	struct drm_bridge_chain_priv *priv;
+	struct drm_crtc_state *crtc_state;
+	struct drm_atomic_commit *state;
+	struct drm_display_mode *mode;
+	int ret;
+
+	priv = drm_test_bridge_chain_init(test, ARRAY_SIZE(funcs), funcs);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv);
+
+	drm_modeset_acquire_init(&ctx, 0);
+
+	state = drm_kunit_helper_atomic_state_alloc(test, &priv->drm, &ctx);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+retry_commit:
+	conn_state = drm_atomic_get_connector_state(state, priv->connector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+
+	mode = drm_kunit_display_mode_from_cea_vic(test, &priv->drm, 16);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mode);
+
+	ret = drm_atomic_set_crtc_for_connector(conn_state, priv->crtc);
+	if (ret == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry_commit;
+	}
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+
+	ret = drm_atomic_set_mode_for_crtc(crtc_state, mode);
+	if (ret == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry_commit;
+	}
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	crtc_state->enable = true;
+	crtc_state->active = true;
+
+	ret = drm_atomic_commit(state);
+	if (ret == -EDEADLK) {
+		drm_modeset_backoff(&ctx);
+		goto retry_commit;
+	}
+	KUNIT_EXPECT_EQ(test, ret, -ENOTSUPP);
+
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+}
+
+static struct kunit_case drm_bridge_bus_fmt_tests[] = {
+	KUNIT_CASE(drm_test_bridge_rgb_yuv_rgb),
+	KUNIT_CASE(drm_test_bridge_must_convert_to_yuv444),
+	KUNIT_CASE(drm_test_bridge_hdmi_auto_rgb),
+	KUNIT_CASE(drm_test_bridge_auto_first),
+	KUNIT_CASE(drm_test_bridge_rgb_yuv_no_path),
+	{ }
+};
+
+static struct kunit_suite drm_bridge_bus_fmt_test_suite = {
+	.name = "drm_bridge_bus_fmt",
+	.test_cases = drm_bridge_bus_fmt_tests,
+};
+
 kunit_test_suites(
 	&drm_bridge_get_current_state_test_suite,
 	&drm_bridge_helper_reset_crtc_test_suite,
 	&drm_bridge_alloc_test_suite,
+	&drm_bridge_bus_fmt_test_suite,
 );
 
 MODULE_AUTHOR("Maxime Ripard <mripard@kernel.org>");
 MODULE_AUTHOR("Luca Ceresoli <luca.ceresoli@bootlin.com>");
+MODULE_AUTHOR("Nicolas Frattaroli <nicolas.frattaroli@collabora.com>");
 
 MODULE_DESCRIPTION("Kunit test for drm_bridge functions");
 MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c b/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
index c9819c3fc635a..353a261d42dac 100644
--- a/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
+++ b/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
@@ -60,6 +60,40 @@ static struct drm_display_mode *find_preferred_mode(struct drm_connector *connec
 	return preferred;
 }
 
+static struct drm_display_mode *find_420_only_mode(struct drm_connector *connector)
+{
+	struct drm_device *drm = connector->dev;
+	struct drm_display_mode *mode;
+
+	mutex_lock(&drm->mode_config.mutex);
+	list_for_each_entry(mode, &connector->modes, head) {
+		if (drm_mode_is_420_only(&connector->display_info, mode)) {
+			mutex_unlock(&drm->mode_config.mutex);
+			return mode;
+		}
+	}
+	mutex_unlock(&drm->mode_config.mutex);
+
+	return NULL;
+}
+
+static struct drm_display_mode *find_420_also_mode(struct drm_connector *connector)
+{
+	struct drm_device *drm = connector->dev;
+	struct drm_display_mode *mode;
+
+	mutex_lock(&drm->mode_config.mutex);
+	list_for_each_entry(mode, &connector->modes, head) {
+		if (drm_mode_is_420_also(&connector->display_info, mode)) {
+			mutex_unlock(&drm->mode_config.mutex);
+			return mode;
+		}
+	}
+	mutex_unlock(&drm->mode_config.mutex);
+
+	return NULL;
+}
+
 static int set_connector_edid(struct kunit *test, struct drm_connector *connector,
 			      const void *edid, size_t edid_len)
 {
@@ -168,7 +202,7 @@ static const struct drm_connector_helper_funcs dummy_connector_helper_funcs = {
 static void dummy_hdmi_connector_reset(struct drm_connector *connector)
 {
 	drm_atomic_helper_connector_reset(connector);
-	__drm_atomic_helper_connector_hdmi_reset(connector, connector->state);
+	__drm_atomic_helper_connector_hdmi_state_init(connector, connector->state);
 }
 
 static const struct drm_connector_funcs dummy_connector_funcs = {
@@ -1547,6 +1581,7 @@ static void drm_test_check_max_tmds_rate_bpc_fallback_yuv420(struct kunit *test)
  *   RGB/10bpc
  * - The chosen mode has a TMDS character rate lower than the display
  *   supports in YUV422/12bpc.
+ * - The HDMI connector state's color format property is unset (i.e. AUTO)
  *
  * Then we will prefer to keep the RGB format with a lower bpc over
  * picking YUV422.
@@ -1609,6 +1644,7 @@ static void drm_test_check_max_tmds_rate_bpc_fallback_ignore_yuv422(struct kunit
 
 	conn_state = conn->state;
 	KUNIT_ASSERT_NOT_NULL(test, conn_state);
+	KUNIT_ASSERT_EQ(test, conn_state->color_format, DRM_CONNECTOR_COLOR_FORMAT_AUTO);
 
 	KUNIT_EXPECT_EQ(test, conn_state->hdmi.output_bpc, 10);
 	KUNIT_EXPECT_EQ(test, conn_state->hdmi.output_format, DRM_OUTPUT_COLOR_FORMAT_RGB444);
@@ -1626,6 +1662,7 @@ static void drm_test_check_max_tmds_rate_bpc_fallback_ignore_yuv422(struct kunit
  *   RGB/8bpc
  * - The chosen mode has a TMDS character rate lower than the display
  *   supports in YUV420/12bpc.
+ * - The HDMI connector state's color format property is unset (i.e. AUTO)
  *
  * Then we will prefer to keep the RGB format with a lower bpc over
  * picking YUV420.
@@ -1687,6 +1724,7 @@ static void drm_test_check_max_tmds_rate_bpc_fallback_ignore_yuv420(struct kunit
 
 	conn_state = conn->state;
 	KUNIT_ASSERT_NOT_NULL(test, conn_state);
+	KUNIT_ASSERT_EQ(test, conn_state->color_format, DRM_CONNECTOR_COLOR_FORMAT_AUTO);
 
 	KUNIT_EXPECT_EQ(test, conn_state->hdmi.output_bpc, 8);
 	KUNIT_EXPECT_EQ(test, conn_state->hdmi.output_format, DRM_OUTPUT_COLOR_FORMAT_RGB444);
@@ -2198,6 +2236,217 @@ static void drm_test_check_disable_connector(struct kunit *test)
 	drm_modeset_acquire_fini(&ctx);
 }
 
+struct color_format_test_param {
+	enum drm_connector_color_format fmt;
+	enum drm_output_color_format expected;
+	int expected_ret;
+	const char *desc;
+};
+
+/* Test that if:
+ * - an HDMI connector supports RGB, YUV444, YUV422, and YUV420
+ * - the display supports RGB, YUV444, YUV422, and YUV420
+ * - the "color format" property is set
+ * then, for the preferred mode, for a given "color format" option:
+ * - DRM_CONNECTOR_COLOR_FORMAT_AUTO results in an output format of RGB
+ * - DRM_CONNECTOR_COLOR_FORMAT_YCBCR422 results in an output format of YUV422
+ * - DRM_CONNECTOR_COLOR_FORMAT_YCBCR420 results in an output format of YUV420
+ * - DRM_CONNECTOR_COLOR_FORMAT_YCBCR444 results in an output format of YUV444
+ * - DRM_CONNECTOR_COLOR_FORMAT_RGB results in an HDMI output format of RGB
+ */
+static void drm_test_check_hdmi_color_format(struct kunit *test)
+{
+	const struct color_format_test_param *param = test->param_value;
+	struct drm_atomic_helper_connector_hdmi_priv *priv;
+	struct drm_connector_state *conn_state;
+	struct drm_modeset_acquire_ctx ctx;
+	struct drm_crtc_state *crtc_state;
+	struct drm_atomic_commit *state;
+	struct drm_display_info *info;
+	struct drm_display_mode *preferred;
+	int ret;
+
+	priv = drm_kunit_helper_connector_hdmi_init_with_edid_funcs(test,
+				BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444) |
+				BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422) |
+				BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420) |
+				BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444),
+				12,
+				&dummy_connector_hdmi_funcs,
+				test_edid_hdmi_4k_rgb_yuv420_dc_max_340mhz);
+	KUNIT_ASSERT_NOT_NULL(test, priv);
+
+	drm_modeset_acquire_init(&ctx, 0);
+
+	KUNIT_ASSERT_TRUE(test, priv->connector.ycbcr_420_allowed);
+
+	info = &priv->connector.display_info;
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, info);
+	preferred = find_preferred_mode(&priv->connector);
+	KUNIT_ASSERT_TRUE(test, drm_mode_is_420(info, preferred));
+
+	state = drm_kunit_helper_atomic_state_alloc(test, &priv->drm, &ctx);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+	conn_state = drm_atomic_get_connector_state(state, &priv->connector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+
+	conn_state->color_format = param->fmt;
+
+	ret = drm_atomic_set_crtc_for_connector(conn_state, priv->crtc);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+
+	ret = drm_atomic_set_mode_for_crtc(crtc_state, preferred);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	crtc_state->enable = true;
+	crtc_state->active = true;
+
+	ret = drm_atomic_check_only(state);
+	KUNIT_EXPECT_EQ(test, ret, param->expected_ret);
+	KUNIT_EXPECT_EQ(test, conn_state->hdmi.output_format, param->expected);
+
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+}
+
+static const struct color_format_test_param hdmi_color_format_params[] = {
+	{
+		.fmt = DRM_CONNECTOR_COLOR_FORMAT_AUTO,
+		.expected = DRM_OUTPUT_COLOR_FORMAT_RGB444,
+		.expected_ret = 0,
+		.desc = "AUTO -> RGB"
+	},
+	{
+		.fmt = DRM_CONNECTOR_COLOR_FORMAT_YCBCR422,
+		.expected = DRM_OUTPUT_COLOR_FORMAT_YCBCR422,
+		.expected_ret = 0,
+		.desc = "YCBCR422 -> YUV422"
+	},
+	{
+		.fmt = DRM_CONNECTOR_COLOR_FORMAT_YCBCR420,
+		.expected = DRM_OUTPUT_COLOR_FORMAT_YCBCR420,
+		.expected_ret = 0,
+		.desc = "YCBCR420 -> YUV420"
+	},
+	{
+		.fmt = DRM_CONNECTOR_COLOR_FORMAT_YCBCR444,
+		.expected = DRM_OUTPUT_COLOR_FORMAT_YCBCR444,
+		.expected_ret = 0,
+		.desc = "YCBCR444 -> YUV444"
+	},
+	{
+		.fmt = DRM_CONNECTOR_COLOR_FORMAT_RGB444,
+		.expected = DRM_OUTPUT_COLOR_FORMAT_RGB444,
+		.expected_ret = 0,
+		.desc = "RGB -> RGB"
+	},
+};
+
+KUNIT_ARRAY_PARAM_DESC(check_hdmi_color_format, hdmi_color_format_params, desc);
+
+/* Test that if:
+ * - the HDMI connector supports RGB, YUV422, YUV420, and YUV444
+ * - the display has a YUV420-only mode
+ * - the "color format" property is explicitly set (i.e. !AUTO)
+ * then:
+ * - color format DRM_CONNECTOR_COLOR_FORMAT_RGB444 will fail
+ *   drm_atomic_check_only for the YUV420-only mode with -EINVAL
+ * - color format DRM_CONNECTOR_COLOR_FORMAT_YCBCR444 will fail
+ *   drm_atomic_check_only for the YUV420-only mode with -EINVAL
+ * - color format DRM_CONNECTOR_COLOR_FORMAT_YCBCR422 will fail
+ *   drm_atomic_check_only for the YUV420-only mode with -EINVAL
+ * - color format DRM_CONNECTOR_COLOR_FORMAT_YCBCR420 passes
+ *   drm_atomic_check_only for the YUV420-only mode
+ */
+static void drm_test_check_hdmi_color_format_420_only(struct kunit *test)
+{
+	const struct color_format_test_param *param = test->param_value;
+	struct drm_atomic_helper_connector_hdmi_priv *priv;
+	struct drm_connector_state *conn_state;
+	struct drm_modeset_acquire_ctx ctx;
+	struct drm_crtc_state *crtc_state;
+	struct drm_atomic_commit *state;
+	struct drm_display_mode *dank;
+	int ret;
+
+	priv = drm_kunit_helper_connector_hdmi_init_with_edid_funcs(test,
+				BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444) |
+				BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422) |
+				BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420) |
+				BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444),
+				12,
+				&dummy_connector_hdmi_funcs,
+				test_edid_hdmi_1080p_rgb_yuv_4k_yuv420_dc_max_200mhz);
+	KUNIT_ASSERT_NOT_NULL(test, priv);
+
+	drm_modeset_acquire_init(&ctx, 0);
+
+	dank = find_420_only_mode(&priv->connector);
+	KUNIT_ASSERT_NOT_NULL(test, dank);
+
+	state = drm_kunit_helper_atomic_state_alloc(test, &priv->drm, &ctx);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+	conn_state = drm_atomic_get_connector_state(state, &priv->connector);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+
+	conn_state->color_format = param->fmt;
+
+	ret = drm_atomic_set_crtc_for_connector(conn_state, priv->crtc);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+
+	ret = drm_atomic_set_mode_for_crtc(crtc_state, dank);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	crtc_state->enable = true;
+	crtc_state->active = true;
+
+	ret = drm_atomic_check_only(state);
+	KUNIT_EXPECT_EQ(test, ret, param->expected_ret);
+	if (!param->expected_ret)
+		KUNIT_EXPECT_EQ(test, conn_state->hdmi.output_format, param->expected);
+
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+};
+
+static const struct color_format_test_param hdmi_color_format_420_only_params[] = {
+	{
+		.fmt = DRM_CONNECTOR_COLOR_FORMAT_RGB444,
+		.expected = DRM_OUTPUT_COLOR_FORMAT_RGB444,
+		.expected_ret = -EINVAL,
+		.desc = "RGB should fail"
+	},
+	{
+		.fmt = DRM_CONNECTOR_COLOR_FORMAT_YCBCR444,
+		.expected = DRM_OUTPUT_COLOR_FORMAT_YCBCR444,
+		.expected_ret = -EINVAL,
+		.desc = "YUV444 should fail"
+	},
+	{
+		.fmt = DRM_CONNECTOR_COLOR_FORMAT_YCBCR422,
+		.expected = DRM_OUTPUT_COLOR_FORMAT_YCBCR422,
+		.expected_ret = -EINVAL,
+		.desc = "YUV422 should fail"
+	},
+	{
+		.fmt = DRM_CONNECTOR_COLOR_FORMAT_YCBCR420,
+		.expected = DRM_OUTPUT_COLOR_FORMAT_YCBCR420,
+		.expected_ret = 0,
+		.desc = "YUV420 should work"
+	},
+};
+
+KUNIT_ARRAY_PARAM_DESC(check_hdmi_color_format_420_only,
+		       hdmi_color_format_420_only_params, desc);
+
 static struct kunit_case drm_atomic_helper_connector_hdmi_check_tests[] = {
 	KUNIT_CASE(drm_test_check_broadcast_rgb_auto_cea_mode),
 	KUNIT_CASE(drm_test_check_broadcast_rgb_auto_cea_mode_vic_1),
@@ -2227,6 +2476,10 @@ static struct kunit_case drm_atomic_helper_connector_hdmi_check_tests[] = {
 	KUNIT_CASE(drm_test_check_tmds_char_rate_rgb_8bpc),
 	KUNIT_CASE(drm_test_check_tmds_char_rate_rgb_10bpc),
 	KUNIT_CASE(drm_test_check_tmds_char_rate_rgb_12bpc),
+	KUNIT_CASE_PARAM(drm_test_check_hdmi_color_format,
+			 check_hdmi_color_format_gen_params),
+	KUNIT_CASE_PARAM(drm_test_check_hdmi_color_format_420_only,
+			 check_hdmi_color_format_420_only_gen_params),
 	/*
 	 * TODO: We should have tests to check that a change in the
 	 * format triggers a CRTC mode change just like we do for the
@@ -2509,11 +2762,103 @@ static void drm_test_check_mode_valid_reject_max_clock(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, preferred->clock, 25200);
 }
 
+/*
+ * Test that drm_hdmi_connector_mode_valid() will accept modes that require a
+ * 4:2:0 chroma subsampling, even if said mode would violate maximum clock
+ * constraints if it used RGB 4:4:4.
+ */
+static void drm_test_check_mode_valid_yuv420_only_max_clock(struct kunit *test)
+{
+	struct drm_atomic_helper_connector_hdmi_priv *priv;
+	struct drm_display_mode *dank;
+	struct drm_connector *conn;
+
+	priv = drm_kunit_helper_connector_hdmi_init_with_edid_funcs(test,
+				BIT(HDMI_COLORSPACE_RGB) |
+				BIT(HDMI_COLORSPACE_YUV420),
+				8,
+				&dummy_connector_hdmi_funcs,
+				test_edid_hdmi_1080p_rgb_yuv_4k_yuv420_dc_max_200mhz);
+	KUNIT_ASSERT_NOT_NULL(test, priv);
+
+	conn = &priv->connector;
+	KUNIT_ASSERT_EQ(test, conn->display_info.max_tmds_clock, 200 * 1000);
+
+	dank = find_420_only_mode(conn);
+	KUNIT_ASSERT_NOT_NULL(test, dank);
+	KUNIT_EXPECT_EQ(test, dank->hdisplay, 3840);
+	KUNIT_EXPECT_EQ(test, dank->vdisplay, 2160);
+
+	/*
+	 * Note: The mode's "clock" here is not accurate to the actual TMDS
+	 * clock that HDMI will use for a subsampled mode. Hence, why the mode's
+	 * clock is above the .max_tmds_clock of 200MHz.
+	 */
+	KUNIT_EXPECT_EQ(test, dank->clock, 297000);
+}
+
+/*
+ * Test that drm_hdmi_connector_mode_valid() will reject modes that require
+ * 4:2:0 chroma subsampling, if the connector does not support 4:2:0.
+ */
+static void
+drm_test_check_mode_valid_reject_yuv420_only_connector(struct kunit *test)
+{
+	struct drm_atomic_helper_connector_hdmi_priv *priv;
+	struct drm_display_mode *dank;
+	struct drm_connector *conn;
+
+	priv = drm_kunit_helper_connector_hdmi_init_with_edid_funcs(test,
+				BIT(HDMI_COLORSPACE_RGB),
+				8,
+				&dummy_connector_hdmi_funcs,
+				test_edid_hdmi_1080p_rgb_yuv_4k_yuv420_dc_max_200mhz);
+	KUNIT_ASSERT_NOT_NULL(test, priv);
+
+	conn = &priv->connector;
+	KUNIT_ASSERT_EQ(test, conn->display_info.max_tmds_clock, 200 * 1000);
+
+	dank = find_420_only_mode(conn);
+	KUNIT_EXPECT_NULL(test, dank);
+}
+
+/*
+ * Test that drm_hdmi_connector_mode_valid() will accept modes that allow (among
+ * other color formats) 4:2:0 chroma subsampling, even if the connector does not
+ * support 4:2:0, but the mode's clock works for RGB 4:4:4.
+ */
+static void
+drm_test_check_mode_valid_accept_yuv420_also_connector_rgb(struct kunit *test)
+{
+	struct drm_atomic_helper_connector_hdmi_priv *priv;
+	struct drm_display_mode *mode;
+	struct drm_connector *conn;
+
+	priv = drm_kunit_helper_connector_hdmi_init_with_edid_funcs(test,
+				BIT(HDMI_COLORSPACE_RGB),
+				8,
+				&dummy_connector_hdmi_funcs,
+				test_edid_hdmi_4k_rgb_yuv420_dc_max_340mhz);
+	KUNIT_ASSERT_NOT_NULL(test, priv);
+
+	conn = &priv->connector;
+	KUNIT_ASSERT_EQ(test, conn->display_info.max_tmds_clock, 340 * 1000);
+
+	mode = find_420_also_mode(conn);
+	KUNIT_ASSERT_NOT_NULL(test, mode);
+	KUNIT_EXPECT_EQ(test, mode->hdisplay, 3840);
+	KUNIT_EXPECT_EQ(test, mode->vdisplay, 2160);
+	KUNIT_EXPECT_EQ(test, mode->clock, 297000);
+}
+
 static struct kunit_case drm_atomic_helper_connector_hdmi_mode_valid_tests[] = {
 	KUNIT_CASE(drm_test_check_mode_valid),
 	KUNIT_CASE(drm_test_check_mode_valid_reject),
 	KUNIT_CASE(drm_test_check_mode_valid_reject_rate),
 	KUNIT_CASE(drm_test_check_mode_valid_reject_max_clock),
+	KUNIT_CASE(drm_test_check_mode_valid_yuv420_only_max_clock),
+	KUNIT_CASE(drm_test_check_mode_valid_reject_yuv420_only_connector),
+	KUNIT_CASE(drm_test_check_mode_valid_accept_yuv420_also_connector_rgb),
 	{ }
 };
 
diff --git a/drivers/gpu/drm/tests/drm_kunit_edid.c b/drivers/gpu/drm/tests/drm_kunit_edid.c
new file mode 100644
index 0000000000000..c807ae0d9faac
--- /dev/null
+++ b/drivers/gpu/drm/tests/drm_kunit_edid.c
@@ -0,0 +1,995 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/module.h>
+
+#include "drm_kunit_edid.h"
+
+/*
+ * edid-decode (hex):
+ *
+ * 00 ff ff ff ff ff ff 00 31 d8 2a 00 00 00 00 00
+ * 00 21 01 03 81 a0 5a 78 0a 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 01 01 01 01 01 01 01 01 01 01
+ * 01 01 01 01 01 01 02 3a 80 18 71 38 2d 40 58 2c
+ * 45 00 40 84 63 00 00 1e 00 00 00 fc 00 54 65 73
+ * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 32
+ * 46 1e 46 0f 00 0a 20 20 20 20 20 20 00 00 00 10
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ab
+ *
+ * ----------------
+ *
+ * Block 0, Base EDID:
+ *   EDID Structure Version & Revision: 1.3
+ *   Vendor & Product Identification:
+ *     Manufacturer: LNX
+ *     Model: 42
+ *     Made in: 2023
+ *   Basic Display Parameters & Features:
+ *     Digital display
+ *     DFP 1.x compatible TMDS
+ *     Maximum image size: 160 cm x 90 cm
+ *     Gamma: 2.20
+ *     RGB color display
+ *     First detailed timing is the preferred timing
+ *   Color Characteristics:
+ *     Red  : 0.0000, 0.0000
+ *     Green: 0.0000, 0.0000
+ *     Blue : 0.0000, 0.0000
+ *     White: 0.0000, 0.0000
+ *   Established Timings I & II: none
+ *   Standard Timings: none
+ *   Detailed Timing Descriptors:
+ *     DTD 1:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (1600 mm x 900 mm)
+ *                  Hfront   88 Hsync  44 Hback  148 Hpol P
+ *                  Vfront    4 Vsync   5 Vback   36 Vpol P
+ *     Display Product Name: 'Test EDID'
+ *     Display Range Limits:
+ *       Monitor ranges (GTF): 50-70 Hz V, 30-70 kHz H, max dotclock 150 MHz
+ *     Dummy Descriptor:
+ * Checksum: 0xab
+ *
+ * ----------------
+ *
+ * edid-decode 1.30.0-5367
+ * edid-decode SHA: 41ebf7135691 2025-05-01 10:19:22
+ *
+ * EDID conformity: PASS
+ */
+const unsigned char test_edid_dvi_1080p[] = {
+	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x2a, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x01, 0x03, 0x81, 0xa0, 0x5a, 0x78,
+	0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3a, 0x80, 0x18, 0x71, 0x38,
+	0x2d, 0x40, 0x58, 0x2c, 0x45, 0x00, 0x40, 0x84, 0x63, 0x00, 0x00, 0x1e,
+	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
+	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x32,
+	0x46, 0x1e, 0x46, 0x0f, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xab
+};
+EXPORT_SYMBOL(test_edid_dvi_1080p);
+
+/*
+ *
+ * This edid is intentionally broken with the 100MHz limit. It's meant
+ * to be used only with tests in unusual situations.
+ *
+ * edid-decode (hex):
+ *
+ * 00 ff ff ff ff ff ff 00 31 d8 2a 00 00 00 00 00
+ * 00 21 01 03 81 a0 5a 78 02 00 00 00 00 00 00 00
+ * 00 00 00 20 00 00 01 01 01 01 01 01 01 01 01 01
+ * 01 01 01 01 01 01 02 3a 80 18 71 38 2d 40 58 2c
+ * 45 00 40 84 63 00 00 1e 00 00 00 fc 00 54 65 73
+ * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 32
+ * 46 1e 46 0f 00 0a 20 20 20 20 20 20 00 00 00 10
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 92
+ *
+ * 02 03 15 81 e3 05 00 20 41 10 e2 00 4a 67 03 0c
+ * 00 12 34 00 14 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 10
+ *
+ * ----------------
+ *
+ * Block 0, Base EDID:
+ *   EDID Structure Version & Revision: 1.3
+ *   Vendor & Product Identification:
+ *     Manufacturer: LNX
+ *     Model: 42
+ *     Made in: 2023
+ *   Basic Display Parameters & Features:
+ *     Digital display
+ *     DFP 1.x compatible TMDS
+ *     Maximum image size: 160 cm x 90 cm
+ *     Gamma: 2.20
+ *     Monochrome or grayscale display
+ *     First detailed timing is the preferred timing
+ *   Color Characteristics:
+ *     Red  : 0.0000, 0.0000
+ *     Green: 0.0000, 0.0000
+ *     Blue : 0.0000, 0.0000
+ *     White: 0.0000, 0.0000
+ *   Established Timings I & II:
+ *     DMT 0x04:   640x480    59.940476 Hz   4:3     31.469 kHz     25.175000 MHz
+ *   Standard Timings: none
+ *   Detailed Timing Descriptors:
+ *     DTD 1:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (1600 mm x 900 mm)
+ *                  Hfront   88 Hsync  44 Hback  148 Hpol P
+ *                  Vfront    4 Vsync   5 Vback   36 Vpol P
+ *     Display Product Name: 'Test EDID'
+ *     Display Range Limits:
+ *       Monitor ranges (GTF): 50-70 Hz V, 30-70 kHz H, max dotclock 150 MHz
+ *     Dummy Descriptor:
+ *   Extension blocks: 1
+ * Checksum: 0x92
+ *
+ * ----------------
+ *
+ * Block 1, CTA-861 Extension Block:
+ *   Revision: 3
+ *   Underscans IT Video Formats by default
+ *   Native detailed modes: 1
+ *   Colorimetry Data Block:
+ *     sRGB
+ *   Video Data Block:
+ *     VIC  16:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz
+ *   Video Capability Data Block:
+ *     YCbCr quantization: No Data
+ *     RGB quantization: Selectable (via AVI Q)
+ *     PT scan behavior: No Data
+ *     IT scan behavior: Always Underscanned
+ *     CE scan behavior: Always Underscanned
+ *   Vendor-Specific Data Block (HDMI), OUI 00-0C-03:
+ *     Source physical address: 1.2.3.4
+ *     Maximum TMDS clock: 100 MHz
+ * Checksum: 0x10  Unused space in Extension Block: 106 bytes
+ *
+ * ----------------
+ *
+ * edid-decode 1.30.0-5367
+ * edid-decode SHA: 41ebf7135691 2025-05-01 10:19:22
+ *
+ * Failures:
+ *
+ * EDID:
+ *   CTA-861: The maximum HDMI TMDS clock is 100000 kHz, but one or more video timings go up to 148500 kHz.
+ *
+ * EDID conformity: FAIL
+ */
+const unsigned char test_edid_hdmi_1080p_rgb_max_100mhz[] = {
+	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x2a, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x01, 0x03, 0x81, 0xa0, 0x5a, 0x78,
+	0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20,
+	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3a, 0x80, 0x18, 0x71, 0x38,
+	0x2d, 0x40, 0x58, 0x2c, 0x45, 0x00, 0x40, 0x84, 0x63, 0x00, 0x00, 0x1e,
+	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
+	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x32,
+	0x46, 0x1e, 0x46, 0x0f, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x92, 0x02, 0x03, 0x15, 0x81,
+	0xe3, 0x05, 0x00, 0x20, 0x41, 0x10, 0xe2, 0x00, 0x4a, 0x67, 0x03, 0x0c,
+	0x00, 0x12, 0x34, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x10
+};
+EXPORT_SYMBOL(test_edid_hdmi_1080p_rgb_max_100mhz);
+
+/*
+ * edid-decode (hex):
+ *
+ * 00 ff ff ff ff ff ff 00 31 d8 2a 00 00 00 00 00
+ * 00 21 01 03 81 a0 5a 78 02 00 00 00 00 00 00 00
+ * 00 00 00 20 00 00 01 01 01 01 01 01 01 01 01 01
+ * 01 01 01 01 01 01 02 3a 80 18 71 38 2d 40 58 2c
+ * 45 00 40 84 63 00 00 1e 00 00 00 fc 00 54 65 73
+ * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 32
+ * 46 1e 46 0f 00 0a 20 20 20 20 20 20 00 00 00 10
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 92
+ *
+ * 02 03 15 81 e3 05 00 20 41 10 e2 00 4a 67 03 0c
+ * 00 12 34 00 28 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 fc
+ *
+ * ----------------
+ *
+ * Block 0, Base EDID:
+ *   EDID Structure Version & Revision: 1.3
+ *   Vendor & Product Identification:
+ *     Manufacturer: LNX
+ *     Model: 42
+ *     Made in: 2023
+ *   Basic Display Parameters & Features:
+ *     Digital display
+ *     DFP 1.x compatible TMDS
+ *     Maximum image size: 160 cm x 90 cm
+ *     Gamma: 2.20
+ *     Monochrome or grayscale display
+ *     First detailed timing is the preferred timing
+ *   Color Characteristics:
+ *     Red  : 0.0000, 0.0000
+ *     Green: 0.0000, 0.0000
+ *     Blue : 0.0000, 0.0000
+ *     White: 0.0000, 0.0000
+ *   Established Timings I & II:
+ *     DMT 0x04:   640x480    59.940476 Hz   4:3     31.469 kHz     25.175000 MHz
+ *   Standard Timings: none
+ *   Detailed Timing Descriptors:
+ *     DTD 1:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (1600 mm x 900 mm)
+ *                  Hfront   88 Hsync  44 Hback  148 Hpol P
+ *                  Vfront    4 Vsync   5 Vback   36 Vpol P
+ *     Display Product Name: 'Test EDID'
+ *     Display Range Limits:
+ *       Monitor ranges (GTF): 50-70 Hz V, 30-70 kHz H, max dotclock 150 MHz
+ *     Dummy Descriptor:
+ *   Extension blocks: 1
+ * Checksum: 0x92
+ *
+ * ----------------
+ *
+ * Block 1, CTA-861 Extension Block:
+ *   Revision: 3
+ *   Underscans IT Video Formats by default
+ *   Native detailed modes: 1
+ *   Colorimetry Data Block:
+ *     sRGB
+ *   Video Data Block:
+ *     VIC  16:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz
+ *   Video Capability Data Block:
+ *     YCbCr quantization: No Data
+ *     RGB quantization: Selectable (via AVI Q)
+ *     PT scan behavior: No Data
+ *     IT scan behavior: Always Underscanned
+ *     CE scan behavior: Always Underscanned
+ *   Vendor-Specific Data Block (HDMI), OUI 00-0C-03:
+ *     Source physical address: 1.2.3.4
+ *     Maximum TMDS clock: 200 MHz
+ * Checksum: 0xfc  Unused space in Extension Block: 106 bytes
+ *
+ * ----------------
+ *
+ * edid-decode 1.30.0-5367
+ * edid-decode SHA: 41ebf7135691 2025-05-01 10:19:22
+ *
+ * EDID conformity: PASS
+ */
+const unsigned char test_edid_hdmi_1080p_rgb_max_200mhz[] = {
+	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x2a, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x01, 0x03, 0x81, 0xa0, 0x5a, 0x78,
+	0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20,
+	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3a, 0x80, 0x18, 0x71, 0x38,
+	0x2d, 0x40, 0x58, 0x2c, 0x45, 0x00, 0x40, 0x84, 0x63, 0x00, 0x00, 0x1e,
+	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
+	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x32,
+	0x46, 0x1e, 0x46, 0x0f, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x92, 0x02, 0x03, 0x15, 0x81,
+	0xe3, 0x05, 0x00, 0x20, 0x41, 0x10, 0xe2, 0x00, 0x4a, 0x67, 0x03, 0x0c,
+	0x00, 0x12, 0x34, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0xfc
+};
+EXPORT_SYMBOL(test_edid_hdmi_1080p_rgb_max_200mhz);
+
+/*
+ * edid-decode (hex):
+ *
+ * 00 ff ff ff ff ff ff 00 31 d8 2a 00 00 00 00 00
+ * 00 21 01 03 81 a0 5a 78 02 00 00 00 00 00 00 00
+ * 00 00 00 20 00 00 01 01 01 01 01 01 01 01 01 01
+ * 01 01 01 01 01 01 02 3a 80 18 71 38 2d 40 58 2c
+ * 45 00 40 84 63 00 00 1e 00 00 00 fc 00 54 65 73
+ * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 32
+ * 46 1e 46 0f 00 0a 20 20 20 20 20 20 00 00 00 10
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 92
+ *
+ * 02 03 1c 81 e3 05 c0 20 41 10 e2 00 4a 67 03 0c
+ * 00 12 34 00 28 e6 06 05 01 52 52 51 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4e
+ *
+ * ----------------
+ *
+ * Block 0, Base EDID:
+ *   EDID Structure Version & Revision: 1.3
+ *   Vendor & Product Identification:
+ *     Manufacturer: LNX
+ *     Model: 42
+ *     Made in: 2023
+ *   Basic Display Parameters & Features:
+ *     Digital display
+ *     DFP 1.x compatible TMDS
+ *     Maximum image size: 160 cm x 90 cm
+ *     Gamma: 2.20
+ *     Monochrome or grayscale display
+ *     First detailed timing is the preferred timing
+ *   Color Characteristics:
+ *     Red  : 0.0000, 0.0000
+ *     Green: 0.0000, 0.0000
+ *     Blue : 0.0000, 0.0000
+ *     White: 0.0000, 0.0000
+ *   Established Timings I & II:
+ *     DMT 0x04:   640x480    59.940476 Hz   4:3     31.469 kHz     25.175000 MHz
+ *   Standard Timings: none
+ *   Detailed Timing Descriptors:
+ *     DTD 1:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (1600 mm x 900 mm)
+ *                  Hfront   88 Hsync  44 Hback  148 Hpol P
+ *                  Vfront    4 Vsync   5 Vback   36 Vpol P
+ *     Display Product Name: 'Test EDID'
+ *     Display Range Limits:
+ *       Monitor ranges (GTF): 50-70 Hz V, 30-70 kHz H, max dotclock 150 MHz
+ *     Dummy Descriptor:
+ *   Extension blocks: 1
+ * Checksum: 0x92
+ *
+ * ----------------
+ *
+ * Block 1, CTA-861 Extension Block:
+ *   Revision: 3
+ *   Underscans IT Video Formats by default
+ *   Native detailed modes: 1
+ *   Colorimetry Data Block:
+ *     BT2020YCC
+ *     BT2020RGB
+ *     sRGB
+ *   Video Data Block:
+ *     VIC  16:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz
+ *   Video Capability Data Block:
+ *     YCbCr quantization: No Data
+ *     RGB quantization: Selectable (via AVI Q)
+ *     PT scan behavior: No Data
+ *     IT scan behavior: Always Underscanned
+ *     CE scan behavior: Always Underscanned
+ *   Vendor-Specific Data Block (HDMI), OUI 00-0C-03:
+ *     Source physical address: 1.2.3.4
+ *     Maximum TMDS clock: 200 MHz
+ *   HDR Static Metadata Data Block:
+ *     Electro optical transfer functions:
+ *       Traditional gamma - SDR luminance range
+ *       SMPTE ST2084
+ *     Supported static metadata descriptors:
+ *       Static metadata type 1
+ *     Desired content max luminance: 82 (295.365 cd/m^2)
+ *     Desired content max frame-average luminance: 82 (295.365 cd/m^2)
+ *     Desired content min luminance: 81 (0.298 cd/m^2)
+ * Checksum: 0x4e  Unused space in Extension Block: 99 bytes
+ *
+ * ----------------
+ *
+ * edid-decode 1.31.0-5387
+ * edid-decode SHA: 5508bc4301ac 2025-08-25 08:14:22
+ *
+ * EDID conformity: PASS
+ */
+const unsigned char test_edid_hdmi_1080p_rgb_max_200mhz_hdr[] = {
+	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x2a, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x01, 0x03, 0x81, 0xa0, 0x5a, 0x78,
+	0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20,
+	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3a, 0x80, 0x18, 0x71, 0x38,
+	0x2d, 0x40, 0x58, 0x2c, 0x45, 0x00, 0x40, 0x84, 0x63, 0x00, 0x00, 0x1e,
+	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
+	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x32,
+	0x46, 0x1e, 0x46, 0x0f, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x92, 0x02, 0x03, 0x1c, 0x81,
+	0xe3, 0x05, 0xc0, 0x20, 0x41, 0x10, 0xe2, 0x00, 0x4a, 0x67, 0x03, 0x0c,
+	0x00, 0x12, 0x34, 0x78, 0x28, 0xe6, 0x06, 0x05, 0x01, 0x52, 0x52, 0x51,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0xd6,
+};
+EXPORT_SYMBOL(test_edid_hdmi_1080p_rgb_max_200mhz_hdr);
+
+/*
+ * edid-decode (hex):
+ *
+ * 00 ff ff ff ff ff ff 00 31 d8 2a 00 00 00 00 00
+ * 00 21 01 03 81 a0 5a 78 02 00 00 00 00 00 00 00
+ * 00 00 00 20 00 00 01 01 01 01 01 01 01 01 01 01
+ * 01 01 01 01 01 01 02 3a 80 18 71 38 2d 40 58 2c
+ * 45 00 40 84 63 00 00 1e 00 00 00 fc 00 54 65 73
+ * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 32
+ * 46 1e 46 0f 00 0a 20 20 20 20 20 20 00 00 00 10
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 92
+ *
+ * 02 03 15 81 e3 05 00 20 41 10 e2 00 4a 67 03 0c
+ * 00 12 34 00 44 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 e0
+ *
+ * ----------------
+ *
+ * Block 0, Base EDID:
+ *   EDID Structure Version & Revision: 1.3
+ *   Vendor & Product Identification:
+ *     Manufacturer: LNX
+ *     Model: 42
+ *     Made in: 2023
+ *   Basic Display Parameters & Features:
+ *     Digital display
+ *     DFP 1.x compatible TMDS
+ *     Maximum image size: 160 cm x 90 cm
+ *     Gamma: 2.20
+ *     Monochrome or grayscale display
+ *     First detailed timing is the preferred timing
+ *   Color Characteristics:
+ *     Red  : 0.0000, 0.0000
+ *     Green: 0.0000, 0.0000
+ *     Blue : 0.0000, 0.0000
+ *     White: 0.0000, 0.0000
+ *   Established Timings I & II:
+ *     DMT 0x04:   640x480    59.940476 Hz   4:3     31.469 kHz     25.175000 MHz
+ *   Standard Timings: none
+ *   Detailed Timing Descriptors:
+ *     DTD 1:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (1600 mm x 900 mm)
+ *                  Hfront   88 Hsync  44 Hback  148 Hpol P
+ *                  Vfront    4 Vsync   5 Vback   36 Vpol P
+ *     Display Product Name: 'Test EDID'
+ *     Display Range Limits:
+ *       Monitor ranges (GTF): 50-70 Hz V, 30-70 kHz H, max dotclock 150 MHz
+ *     Dummy Descriptor:
+ *   Extension blocks: 1
+ * Checksum: 0x92
+ *
+ * ----------------
+ *
+ * Block 1, CTA-861 Extension Block:
+ *   Revision: 3
+ *   Underscans IT Video Formats by default
+ *   Native detailed modes: 1
+ *   Colorimetry Data Block:
+ *     sRGB
+ *   Video Data Block:
+ *     VIC  16:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz
+ *   Video Capability Data Block:
+ *     YCbCr quantization: No Data
+ *     RGB quantization: Selectable (via AVI Q)
+ *     PT scan behavior: No Data
+ *     IT scan behavior: Always Underscanned
+ *     CE scan behavior: Always Underscanned
+ *   Vendor-Specific Data Block (HDMI), OUI 00-0C-03:
+ *     Source physical address: 1.2.3.4
+ *     Maximum TMDS clock: 340 MHz
+ * Checksum: 0xe0  Unused space in Extension Block: 106 bytes
+ *
+ * ----------------
+ *
+ * edid-decode 1.30.0-5367
+ * edid-decode SHA: 41ebf7135691 2025-05-01 10:19:22
+ *
+ * EDID conformity: PASS
+ */
+const unsigned char test_edid_hdmi_1080p_rgb_max_340mhz[] = {
+	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x2a, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x01, 0x03, 0x81, 0xa0, 0x5a, 0x78,
+	0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20,
+	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3a, 0x80, 0x18, 0x71, 0x38,
+	0x2d, 0x40, 0x58, 0x2c, 0x45, 0x00, 0x40, 0x84, 0x63, 0x00, 0x00, 0x1e,
+	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
+	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x32,
+	0x46, 0x1e, 0x46, 0x0f, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x92, 0x02, 0x03, 0x15, 0x81,
+	0xe3, 0x05, 0x00, 0x20, 0x41, 0x10, 0xe2, 0x00, 0x4a, 0x67, 0x03, 0x0c,
+	0x00, 0x12, 0x34, 0x00, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0xe0
+};
+EXPORT_SYMBOL(test_edid_hdmi_1080p_rgb_max_340mhz);
+
+/*
+ * edid-decode (hex):
+ *
+ * 00 ff ff ff ff ff ff 00 31 d8 2a 00 00 00 00 00
+ * 00 21 01 03 81 a0 5a 78 1a 00 00 00 00 00 00 00
+ * 00 00 00 20 00 00 01 01 01 01 01 01 01 01 01 01
+ * 01 01 01 01 01 01 02 3a 80 18 71 38 2d 40 58 2c
+ * 45 00 40 84 63 00 00 1e 00 00 00 fc 00 54 65 73
+ * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 32
+ * 46 1e 46 0f 00 0a 20 20 20 20 20 20 00 00 00 10
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 7a
+ *
+ * 02 03 15 b1 e3 05 00 20 41 10 e2 00 ca 67 03 0c
+ * 00 12 34 78 28 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 d4
+ *
+ * ----------------
+ *
+ * Block 0, Base EDID:
+ *   EDID Structure Version & Revision: 1.3
+ *   Vendor & Product Identification:
+ *     Manufacturer: LNX
+ *     Model: 42
+ *     Made in: 2023
+ *   Basic Display Parameters & Features:
+ *     Digital display
+ *     DFP 1.x compatible TMDS
+ *     Maximum image size: 160 cm x 90 cm
+ *     Gamma: 2.20
+ *     Undefined display color type
+ *     First detailed timing is the preferred timing
+ *   Color Characteristics:
+ *     Red  : 0.0000, 0.0000
+ *     Green: 0.0000, 0.0000
+ *     Blue : 0.0000, 0.0000
+ *     White: 0.0000, 0.0000
+ *   Established Timings I & II:
+ *     DMT 0x04:   640x480    59.940476 Hz   4:3     31.469 kHz     25.175000 MHz
+ *   Standard Timings: none
+ *   Detailed Timing Descriptors:
+ *     DTD 1:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (1600 mm x 900 mm)
+ *                  Hfront   88 Hsync  44 Hback  148 Hpol P
+ *                  Vfront    4 Vsync   5 Vback   36 Vpol P
+ *     Display Product Name: 'Test EDID'
+ *     Display Range Limits:
+ *       Monitor ranges (GTF): 50-70 Hz V, 30-70 kHz H, max dotclock 150 MHz
+ *     Dummy Descriptor:
+ *   Extension blocks: 1
+ * Checksum: 0x7a
+ *
+ * ----------------
+ *
+ * Block 1, CTA-861 Extension Block:
+ *   Revision: 3
+ *   Underscans IT Video Formats by default
+ *   Supports YCbCr 4:4:4
+ *   Supports YCbCr 4:2:2
+ *   Native detailed modes: 1
+ *   Colorimetry Data Block:
+ *     sRGB
+ *   Video Data Block:
+ *     VIC  16:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz
+ *   Video Capability Data Block:
+ *     YCbCr quantization: Selectable (via AVI YQ)
+ *     RGB quantization: Selectable (via AVI Q)
+ *     PT scan behavior: No Data
+ *     IT scan behavior: Always Underscanned
+ *     CE scan behavior: Always Underscanned
+ *   Vendor-Specific Data Block (HDMI), OUI 00-0C-03:
+ *     Source physical address: 1.2.3.4
+ *     DC_48bit
+ *     DC_36bit
+ *     DC_30bit
+ *     DC_Y444
+ *     Maximum TMDS clock: 200 MHz
+ * Checksum: 0xd4  Unused space in Extension Block: 106 bytes
+ *
+ * ----------------
+ *
+ * edid-decode 1.30.0-5367
+ * edid-decode SHA: 41ebf7135691 2025-05-01 10:19:22
+ *
+ * EDID conformity: PASS
+ */
+const unsigned char test_edid_hdmi_1080p_rgb_yuv_dc_max_200mhz[] = {
+	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x2a, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x01, 0x03, 0x81, 0xa0, 0x5a, 0x78,
+	0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20,
+	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3a, 0x80, 0x18, 0x71, 0x38,
+	0x2d, 0x40, 0x58, 0x2c, 0x45, 0x00, 0x40, 0x84, 0x63, 0x00, 0x00, 0x1e,
+	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
+	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x32,
+	0x46, 0x1e, 0x46, 0x0f, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x7a, 0x02, 0x03, 0x15, 0xb1,
+	0xe3, 0x05, 0x00, 0x20, 0x41, 0x10, 0xe2, 0x00, 0xca, 0x67, 0x03, 0x0c,
+	0x00, 0x12, 0x34, 0x78, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0xd4
+};
+EXPORT_SYMBOL(test_edid_hdmi_1080p_rgb_yuv_dc_max_200mhz);
+
+/*
+ * edid-decode (hex):
+ *
+ * 00 ff ff ff ff ff ff 00 31 d8 2a 00 00 00 00 00
+ * 00 21 01 03 81 a0 5a 78 0a 00 00 00 00 00 00 00
+ * 00 00 00 20 00 00 01 01 01 01 01 01 01 01 01 01
+ * 01 01 01 01 01 01 02 3a 80 18 71 38 2d 40 58 2c
+ * 45 00 40 84 63 00 00 1e 00 00 00 fc 00 54 65 73
+ * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 32
+ * 46 1e 46 0f 00 0a 20 20 20 20 20 20 00 00 00 10
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 8a
+ *
+ * 02 03 15 b1 e3 05 00 20 41 10 e2 00 ca 67 03 0c
+ * 00 12 34 78 44 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 b8
+ *
+ * ----------------
+ *
+ * Block 0, Base EDID:
+ *   EDID Structure Version & Revision: 1.3
+ *   Vendor & Product Identification:
+ *     Manufacturer: LNX
+ *     Model: 42
+ *     Made in: 2023
+ *   Basic Display Parameters & Features:
+ *     Digital display
+ *     DFP 1.x compatible TMDS
+ *     Maximum image size: 160 cm x 90 cm
+ *     Gamma: 2.20
+ *     RGB color display
+ *     First detailed timing is the preferred timing
+ *   Color Characteristics:
+ *     Red  : 0.0000, 0.0000
+ *     Green: 0.0000, 0.0000
+ *     Blue : 0.0000, 0.0000
+ *     White: 0.0000, 0.0000
+ *   Established Timings I & II:
+ *     DMT 0x04:   640x480    59.940476 Hz   4:3     31.469 kHz     25.175000 MHz
+ *   Standard Timings: none
+ *   Detailed Timing Descriptors:
+ *     DTD 1:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (1600 mm x 900 mm)
+ *                  Hfront   88 Hsync  44 Hback  148 Hpol P
+ *                  Vfront    4 Vsync   5 Vback   36 Vpol P
+ *     Display Product Name: 'Test EDID'
+ *     Display Range Limits:
+ *       Monitor ranges (GTF): 50-70 Hz V, 30-70 kHz H, max dotclock 150 MHz
+ *     Dummy Descriptor:
+ *   Extension blocks: 1
+ * Checksum: 0x8a
+ *
+ * ----------------
+ *
+ * Block 1, CTA-861 Extension Block:
+ *   Revision: 3
+ *   Underscans IT Video Formats by default
+ *   Supports YCbCr 4:4:4
+ *   Supports YCbCr 4:2:2
+ *   Native detailed modes: 1
+ *   Colorimetry Data Block:
+ *     sRGB
+ *   Video Data Block:
+ *     VIC  16:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz
+ *   Video Capability Data Block:
+ *     YCbCr quantization: Selectable (via AVI YQ)
+ *     RGB quantization: Selectable (via AVI Q)
+ *     PT scan behavior: No Data
+ *     IT scan behavior: Always Underscanned
+ *     CE scan behavior: Always Underscanned
+ *   Vendor-Specific Data Block (HDMI), OUI 00-0C-03:
+ *     Source physical address: 1.2.3.4
+ *     DC_48bit
+ *     DC_36bit
+ *     DC_30bit
+ *     DC_Y444
+ *     Maximum TMDS clock: 340 MHz
+ * Checksum: 0xb8  Unused space in Extension Block: 106 bytes
+ *
+ * ----------------
+ *
+ * edid-decode 1.30.0-5367
+ * edid-decode SHA: 41ebf7135691 2025-05-01 10:19:22
+ *
+ * EDID conformity: PASS
+ */
+const unsigned char test_edid_hdmi_1080p_rgb_yuv_dc_max_340mhz[] = {
+	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x2a, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x01, 0x03, 0x81, 0xa0, 0x5a, 0x78,
+	0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20,
+	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3a, 0x80, 0x18, 0x71, 0x38,
+	0x2d, 0x40, 0x58, 0x2c, 0x45, 0x00, 0x40, 0x84, 0x63, 0x00, 0x00, 0x1e,
+	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
+	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x32,
+	0x46, 0x1e, 0x46, 0x0f, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x8a, 0x02, 0x03, 0x15, 0xb1,
+	0xe3, 0x05, 0x00, 0x20, 0x41, 0x10, 0xe2, 0x00, 0xca, 0x67, 0x03, 0x0c,
+	0x00, 0x12, 0x34, 0x78, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0xb8
+};
+EXPORT_SYMBOL(test_edid_hdmi_1080p_rgb_yuv_dc_max_340mhz);
+
+/*
+ * Max resolution:
+ * - 1920x1080@60Hz with RGB, YUV444, YUV422
+ * - 3840x2160@30Hz with YUV420 only
+ * Max BPC:         16 for all modes
+ * Max TMDS clock:  200 MHz
+ *
+ * edid-decode (hex):
+ *
+ * 00 ff ff ff ff ff ff 00 31 d8 34 00 00 00 00 00
+ * ff 23 01 03 80 60 36 78 0f ee 91 a3 54 4c 99 26
+ * 0f 50 54 20 00 00 01 01 01 01 01 01 01 01 01 01
+ * 01 01 01 01 01 01 02 3a 80 18 71 38 2d 40 58 2c
+ * 45 00 c0 1c 32 00 00 1e 00 00 00 fc 00 54 65 73
+ * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 18
+ * 55 18 5e 11 00 0a 20 20 20 20 20 20 00 00 00 10
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 bb
+ *
+ * 02 03 29 31 42 90 5f 6c 03 0c 00 10 00 78 28 20
+ * 00 00 01 03 6d d8 5d c4 01 28 80 07 00 00 00 00
+ * 00 00 e3 0f 00 00 e2 0e 5f 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ca
+ *
+ * ----------------
+ *
+ * Block 0, Base EDID:
+ *   EDID Structure Version & Revision: 1.3
+ *   Vendor & Product Identification:
+ *     Manufacturer: LNX
+ *     Model: 52
+ *     Model year: 2025
+ *   Basic Display Parameters & Features:
+ *     Digital display
+ *     Maximum image size: 96 cm x 54 cm
+ *     Gamma: 2.20
+ *     RGB color display
+ *     Default (sRGB) color space is primary color space
+ *     First detailed timing is the preferred timing
+ *     Supports GTF timings within operating range
+ *   Color Characteristics:
+ *     Red  : 0.6396, 0.3300
+ *     Green: 0.2998, 0.5996
+ *     Blue : 0.1503, 0.0595
+ *     White: 0.3125, 0.3291
+ *   Established Timings I & II:
+ *     DMT 0x04:   640x480    59.940476 Hz   4:3     31.469 kHz     25.175000 MHz
+ *   Standard Timings: none
+ *   Detailed Timing Descriptors:
+ *     DTD 1:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (960 mm x 540 mm)
+ *                  Hfront   88 Hsync  44 Hback  148 Hpol P
+ *                  Vfront    4 Vsync   5 Vback   36 Vpol P
+ *     Display Product Name: 'Test EDID'
+ *     Display Range Limits:
+ *       Monitor ranges (GTF): 24-85 Hz V, 24-94 kHz H, max dotclock 170 MHz
+ *     Dummy Descriptor:
+ *   Extension blocks: 1
+ * Checksum: 0xbb
+ *
+ * ----------------
+ *
+ * Block 1, CTA-861 Extension Block:
+ *   Revision: 3
+ *   Supports YCbCr 4:4:4
+ *   Supports YCbCr 4:2:2
+ *   Native detailed modes: 1
+ *   Video Data Block:
+ *     VIC  16:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (native)
+ *     VIC  95:  3840x2160   30.000000 Hz  16:9     67.500 kHz    297.000000 MHz
+ *   Vendor-Specific Data Block (HDMI), OUI 00-0C-03:
+ *     Source physical address: 1.0.0.0
+ *     DC_48bit
+ *     DC_36bit
+ *     DC_30bit
+ *     DC_Y444
+ *     Maximum TMDS clock: 200 MHz
+ *     Extended HDMI video details:
+ *   Vendor-Specific Data Block (HDMI Forum), OUI C4-5D-D8:
+ *     Version: 1
+ *     Maximum TMDS Character Rate: 200 MHz
+ *     SCDC Present
+ *     Supports 16-bits/component Deep Color 4:2:0 Pixel Encoding
+ *     Supports 12-bits/component Deep Color 4:2:0 Pixel Encoding
+ *     Supports 10-bits/component Deep Color 4:2:0 Pixel Encoding
+ *   YCbCr 4:2:0 Capability Map Data Block:
+ *     Empty Capability Map
+ *   YCbCr 4:2:0 Video Data Block:
+ *     VIC  95:  3840x2160   30.000000 Hz  16:9     67.500 kHz    297.000000 MHz
+ * Checksum: 0xca
+ */
+const unsigned char test_edid_hdmi_1080p_rgb_yuv_4k_yuv420_dc_max_200mhz[] = {
+	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x34, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0x23, 0x01, 0x03, 0x80, 0x60, 0x36, 0x78,
+	0x0f, 0xee, 0x91, 0xa3, 0x54, 0x4c, 0x99, 0x26, 0x0f, 0x50, 0x54, 0x20,
+	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3a, 0x80, 0x18, 0x71, 0x38,
+	0x2d, 0x40, 0x58, 0x2c, 0x45, 0x00, 0xc0, 0x1c, 0x32, 0x00, 0x00, 0x1e,
+	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
+	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x18,
+	0x55, 0x18, 0x5e, 0x11, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xbb, 0x02, 0x03, 0x29, 0x31,
+	0x42, 0x90, 0x5f, 0x6c, 0x03, 0x0c, 0x00, 0x10, 0x00, 0x78, 0x28, 0x20,
+	0x00, 0x00, 0x01, 0x03, 0x6d, 0xd8, 0x5d, 0xc4, 0x01, 0x28, 0x80, 0x07,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe3, 0x0f, 0x00, 0x00, 0xe2, 0x0e,
+	0x5f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0xca
+};
+EXPORT_SYMBOL(test_edid_hdmi_1080p_rgb_yuv_4k_yuv420_dc_max_200mhz);
+
+/*
+ * Max resolution: 3840x2160@30Hz with RGB, YUV444, YUV422, YUV420
+ * Max BPC:        16 for all modes
+ * Max TMDS clock: 340 MHz
+ *
+ * edid-decode (hex):
+ *
+ * 00 ff ff ff ff ff ff 00 31 d8 34 00 00 00 00 00
+ * ff 23 01 03 80 60 36 78 0f ee 91 a3 54 4c 99 26
+ * 0f 50 54 20 00 00 01 01 01 01 01 01 01 01 01 01
+ * 01 01 01 01 01 01 04 74 00 30 f2 70 5a 80 b0 58
+ * 8a 00 40 84 63 00 00 1e 00 00 00 fc 00 54 65 73
+ * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 18
+ * 55 18 5e 22 00 0a 20 20 20 20 20 20 00 00 00 10
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ce
+ *
+ * 02 03 27 31 41 5f 6c 03 0c 00 10 00 78 44 20 00
+ * 00 01 03 6d d8 5d c4 01 44 80 07 00 00 00 00 00
+ * 00 e3 0f 01 00 e1 0e 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 84
+ *
+ * ----------------
+ *
+ * Block 0, Base EDID:
+ *   EDID Structure Version & Revision: 1.3
+ *   Vendor & Product Identification:
+ *     Manufacturer: LNX
+ *     Model: 52
+ *     Model year: 2025
+ *   Basic Display Parameters & Features:
+ *     Digital display
+ *     Maximum image size: 96 cm x 54 cm
+ *     Gamma: 2.20
+ *     RGB color display
+ *     Default (sRGB) color space is primary color space
+ *     First detailed timing is the preferred timing
+ *     Supports GTF timings within operating range
+ *   Color Characteristics:
+ *     Red  : 0.6396, 0.3300
+ *     Green: 0.2998, 0.5996
+ *     Blue : 0.1503, 0.0595
+ *     White: 0.3125, 0.3291
+ *   Established Timings I & II:
+ *     DMT 0x04:   640x480    59.940476 Hz   4:3     31.469 kHz     25.175000 MHz
+ *   Standard Timings: none
+ *   Detailed Timing Descriptors:
+ *     DTD 1:  3840x2160   30.000000 Hz  16:9     67.500 kHz    297.000000 MHz (1600 mm x 900 mm)
+ *                  Hfront  176 Hsync  88 Hback  296 Hpol P
+ *                  Vfront    8 Vsync  10 Vback   72 Vpol P
+ *     Display Product Name: 'Test EDID'
+ *     Display Range Limits:
+ *       Monitor ranges (GTF): 24-85 Hz V, 24-94 kHz H, max dotclock 340 MHz
+ *     Dummy Descriptor:
+ *   Extension blocks: 1
+ * Checksum: 0xce
+ *
+ * ----------------
+ *
+ * Block 1, CTA-861 Extension Block:
+ *   Revision: 3
+ *   Supports YCbCr 4:4:4
+ *   Supports YCbCr 4:2:2
+ *   Native detailed modes: 1
+ *   Video Data Block:
+ *     VIC  95:  3840x2160   30.000000 Hz  16:9     67.500 kHz    297.000000 MHz
+ *   Vendor-Specific Data Block (HDMI), OUI 00-0C-03:
+ *     Source physical address: 1.0.0.0
+ *     DC_48bit
+ *     DC_36bit
+ *     DC_30bit
+ *     DC_Y444
+ *     Maximum TMDS clock: 340 MHz
+ *     Extended HDMI video details:
+ *   Vendor-Specific Data Block (HDMI Forum), OUI C4-5D-D8:
+ *     Version: 1
+ *     Maximum TMDS Character Rate: 340 MHz
+ *     SCDC Present
+ *     Supports 16-bits/component Deep Color 4:2:0 Pixel Encoding
+ *     Supports 12-bits/component Deep Color 4:2:0 Pixel Encoding
+ *     Supports 10-bits/component Deep Color 4:2:0 Pixel Encoding
+ *   YCbCr 4:2:0 Capability Map Data Block:
+ *     VIC  95:  3840x2160   30.000000 Hz  16:9     67.500 kHz    297.000000 MHz
+ *   YCbCr 4:2:0 Video Data Block:
+ * Checksum: 0x84
+ */
+const unsigned char test_edid_hdmi_4k_rgb_yuv420_dc_max_340mhz[] = {
+	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x34, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0xff, 0x23, 0x01, 0x03, 0x80, 0x60, 0x36, 0x78,
+	0x0f, 0xee, 0x91, 0xa3, 0x54, 0x4c, 0x99, 0x26, 0x0f, 0x50, 0x54, 0x20,
+	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x74, 0x00, 0x30, 0xf2, 0x70,
+	0x5a, 0x80, 0xb0, 0x58, 0x8a, 0x00, 0x40, 0x84, 0x63, 0x00, 0x00, 0x1e,
+	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
+	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x18,
+	0x55, 0x18, 0x5e, 0x22, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
+	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xce, 0x02, 0x03, 0x27, 0x31,
+	0x41, 0x5f, 0x6c, 0x03, 0x0c, 0x00, 0x10, 0x00, 0x78, 0x44, 0x20, 0x00,
+	0x00, 0x01, 0x03, 0x6d, 0xd8, 0x5d, 0xc4, 0x01, 0x44, 0x80, 0x07, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0xe3, 0x0f, 0x01, 0x00, 0xe1, 0x0e, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x84
+};
+EXPORT_SYMBOL(test_edid_hdmi_4k_rgb_yuv420_dc_max_340mhz);
+
+MODULE_AUTHOR("Maxime Ripard <mripard@kernel.org>");
+MODULE_DESCRIPTION("EDID Definitions for KUnit tests");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/tests/drm_kunit_edid.h b/drivers/gpu/drm/tests/drm_kunit_edid.h
index f4923157f5bf9..28b4df93a5554 100644
--- a/drivers/gpu/drm/tests/drm_kunit_edid.h
+++ b/drivers/gpu/drm/tests/drm_kunit_edid.h
@@ -3,981 +3,14 @@
 #ifndef DRM_KUNIT_EDID_H_
 #define DRM_KUNIT_EDID_H_
 
-/*
- * edid-decode (hex):
- *
- * 00 ff ff ff ff ff ff 00 31 d8 2a 00 00 00 00 00
- * 00 21 01 03 81 a0 5a 78 0a 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 01 01 01 01 01 01 01 01 01 01
- * 01 01 01 01 01 01 02 3a 80 18 71 38 2d 40 58 2c
- * 45 00 40 84 63 00 00 1e 00 00 00 fc 00 54 65 73
- * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 32
- * 46 1e 46 0f 00 0a 20 20 20 20 20 20 00 00 00 10
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ab
- *
- * ----------------
- *
- * Block 0, Base EDID:
- *   EDID Structure Version & Revision: 1.3
- *   Vendor & Product Identification:
- *     Manufacturer: LNX
- *     Model: 42
- *     Made in: 2023
- *   Basic Display Parameters & Features:
- *     Digital display
- *     DFP 1.x compatible TMDS
- *     Maximum image size: 160 cm x 90 cm
- *     Gamma: 2.20
- *     RGB color display
- *     First detailed timing is the preferred timing
- *   Color Characteristics:
- *     Red  : 0.0000, 0.0000
- *     Green: 0.0000, 0.0000
- *     Blue : 0.0000, 0.0000
- *     White: 0.0000, 0.0000
- *   Established Timings I & II: none
- *   Standard Timings: none
- *   Detailed Timing Descriptors:
- *     DTD 1:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (1600 mm x 900 mm)
- *                  Hfront   88 Hsync  44 Hback  148 Hpol P
- *                  Vfront    4 Vsync   5 Vback   36 Vpol P
- *     Display Product Name: 'Test EDID'
- *     Display Range Limits:
- *       Monitor ranges (GTF): 50-70 Hz V, 30-70 kHz H, max dotclock 150 MHz
- *     Dummy Descriptor:
- * Checksum: 0xab
- *
- * ----------------
- *
- * edid-decode 1.30.0-5367
- * edid-decode SHA: 41ebf7135691 2025-05-01 10:19:22
- *
- * EDID conformity: PASS
- */
-static const unsigned char test_edid_dvi_1080p[] = {
-	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x2a, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x01, 0x03, 0x81, 0xa0, 0x5a, 0x78,
-	0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3a, 0x80, 0x18, 0x71, 0x38,
-	0x2d, 0x40, 0x58, 0x2c, 0x45, 0x00, 0x40, 0x84, 0x63, 0x00, 0x00, 0x1e,
-	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
-	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x32,
-	0x46, 0x1e, 0x46, 0x0f, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
-	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xab
-};
-
-/*
- *
- * This edid is intentionally broken with the 100MHz limit. It's meant
- * to be used only with tests in unusual situations.
- *
- * edid-decode (hex):
- *
- * 00 ff ff ff ff ff ff 00 31 d8 2a 00 00 00 00 00
- * 00 21 01 03 81 a0 5a 78 02 00 00 00 00 00 00 00
- * 00 00 00 20 00 00 01 01 01 01 01 01 01 01 01 01
- * 01 01 01 01 01 01 02 3a 80 18 71 38 2d 40 58 2c
- * 45 00 40 84 63 00 00 1e 00 00 00 fc 00 54 65 73
- * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 32
- * 46 1e 46 0f 00 0a 20 20 20 20 20 20 00 00 00 10
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 92
- *
- * 02 03 15 81 e3 05 00 20 41 10 e2 00 4a 67 03 0c
- * 00 12 34 00 14 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 10
- *
- * ----------------
- *
- * Block 0, Base EDID:
- *   EDID Structure Version & Revision: 1.3
- *   Vendor & Product Identification:
- *     Manufacturer: LNX
- *     Model: 42
- *     Made in: 2023
- *   Basic Display Parameters & Features:
- *     Digital display
- *     DFP 1.x compatible TMDS
- *     Maximum image size: 160 cm x 90 cm
- *     Gamma: 2.20
- *     Monochrome or grayscale display
- *     First detailed timing is the preferred timing
- *   Color Characteristics:
- *     Red  : 0.0000, 0.0000
- *     Green: 0.0000, 0.0000
- *     Blue : 0.0000, 0.0000
- *     White: 0.0000, 0.0000
- *   Established Timings I & II:
- *     DMT 0x04:   640x480    59.940476 Hz   4:3     31.469 kHz     25.175000 MHz
- *   Standard Timings: none
- *   Detailed Timing Descriptors:
- *     DTD 1:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (1600 mm x 900 mm)
- *                  Hfront   88 Hsync  44 Hback  148 Hpol P
- *                  Vfront    4 Vsync   5 Vback   36 Vpol P
- *     Display Product Name: 'Test EDID'
- *     Display Range Limits:
- *       Monitor ranges (GTF): 50-70 Hz V, 30-70 kHz H, max dotclock 150 MHz
- *     Dummy Descriptor:
- *   Extension blocks: 1
- * Checksum: 0x92
- *
- * ----------------
- *
- * Block 1, CTA-861 Extension Block:
- *   Revision: 3
- *   Underscans IT Video Formats by default
- *   Native detailed modes: 1
- *   Colorimetry Data Block:
- *     sRGB
- *   Video Data Block:
- *     VIC  16:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz
- *   Video Capability Data Block:
- *     YCbCr quantization: No Data
- *     RGB quantization: Selectable (via AVI Q)
- *     PT scan behavior: No Data
- *     IT scan behavior: Always Underscanned
- *     CE scan behavior: Always Underscanned
- *   Vendor-Specific Data Block (HDMI), OUI 00-0C-03:
- *     Source physical address: 1.2.3.4
- *     Maximum TMDS clock: 100 MHz
- * Checksum: 0x10  Unused space in Extension Block: 106 bytes
- *
- * ----------------
- *
- * edid-decode 1.30.0-5367
- * edid-decode SHA: 41ebf7135691 2025-05-01 10:19:22
- *
- * Failures:
- *
- * EDID:
- *   CTA-861: The maximum HDMI TMDS clock is 100000 kHz, but one or more video timings go up to 148500 kHz.
- *
- * EDID conformity: FAIL
- */
-static const unsigned char test_edid_hdmi_1080p_rgb_max_100mhz[] = {
-	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x2a, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x01, 0x03, 0x81, 0xa0, 0x5a, 0x78,
-	0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20,
-	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3a, 0x80, 0x18, 0x71, 0x38,
-	0x2d, 0x40, 0x58, 0x2c, 0x45, 0x00, 0x40, 0x84, 0x63, 0x00, 0x00, 0x1e,
-	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
-	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x32,
-	0x46, 0x1e, 0x46, 0x0f, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
-	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x92, 0x02, 0x03, 0x15, 0x81,
-	0xe3, 0x05, 0x00, 0x20, 0x41, 0x10, 0xe2, 0x00, 0x4a, 0x67, 0x03, 0x0c,
-	0x00, 0x12, 0x34, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x10
-};
-
-/*
- * edid-decode (hex):
- *
- * 00 ff ff ff ff ff ff 00 31 d8 2a 00 00 00 00 00
- * 00 21 01 03 81 a0 5a 78 02 00 00 00 00 00 00 00
- * 00 00 00 20 00 00 01 01 01 01 01 01 01 01 01 01
- * 01 01 01 01 01 01 02 3a 80 18 71 38 2d 40 58 2c
- * 45 00 40 84 63 00 00 1e 00 00 00 fc 00 54 65 73
- * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 32
- * 46 1e 46 0f 00 0a 20 20 20 20 20 20 00 00 00 10
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 92
- *
- * 02 03 15 81 e3 05 00 20 41 10 e2 00 4a 67 03 0c
- * 00 12 34 00 28 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 fc
- *
- * ----------------
- *
- * Block 0, Base EDID:
- *   EDID Structure Version & Revision: 1.3
- *   Vendor & Product Identification:
- *     Manufacturer: LNX
- *     Model: 42
- *     Made in: 2023
- *   Basic Display Parameters & Features:
- *     Digital display
- *     DFP 1.x compatible TMDS
- *     Maximum image size: 160 cm x 90 cm
- *     Gamma: 2.20
- *     Monochrome or grayscale display
- *     First detailed timing is the preferred timing
- *   Color Characteristics:
- *     Red  : 0.0000, 0.0000
- *     Green: 0.0000, 0.0000
- *     Blue : 0.0000, 0.0000
- *     White: 0.0000, 0.0000
- *   Established Timings I & II:
- *     DMT 0x04:   640x480    59.940476 Hz   4:3     31.469 kHz     25.175000 MHz
- *   Standard Timings: none
- *   Detailed Timing Descriptors:
- *     DTD 1:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (1600 mm x 900 mm)
- *                  Hfront   88 Hsync  44 Hback  148 Hpol P
- *                  Vfront    4 Vsync   5 Vback   36 Vpol P
- *     Display Product Name: 'Test EDID'
- *     Display Range Limits:
- *       Monitor ranges (GTF): 50-70 Hz V, 30-70 kHz H, max dotclock 150 MHz
- *     Dummy Descriptor:
- *   Extension blocks: 1
- * Checksum: 0x92
- *
- * ----------------
- *
- * Block 1, CTA-861 Extension Block:
- *   Revision: 3
- *   Underscans IT Video Formats by default
- *   Native detailed modes: 1
- *   Colorimetry Data Block:
- *     sRGB
- *   Video Data Block:
- *     VIC  16:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz
- *   Video Capability Data Block:
- *     YCbCr quantization: No Data
- *     RGB quantization: Selectable (via AVI Q)
- *     PT scan behavior: No Data
- *     IT scan behavior: Always Underscanned
- *     CE scan behavior: Always Underscanned
- *   Vendor-Specific Data Block (HDMI), OUI 00-0C-03:
- *     Source physical address: 1.2.3.4
- *     Maximum TMDS clock: 200 MHz
- * Checksum: 0xfc  Unused space in Extension Block: 106 bytes
- *
- * ----------------
- *
- * edid-decode 1.30.0-5367
- * edid-decode SHA: 41ebf7135691 2025-05-01 10:19:22
- *
- * EDID conformity: PASS
- */
-static const unsigned char test_edid_hdmi_1080p_rgb_max_200mhz[] = {
-	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x2a, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x01, 0x03, 0x81, 0xa0, 0x5a, 0x78,
-	0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20,
-	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3a, 0x80, 0x18, 0x71, 0x38,
-	0x2d, 0x40, 0x58, 0x2c, 0x45, 0x00, 0x40, 0x84, 0x63, 0x00, 0x00, 0x1e,
-	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
-	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x32,
-	0x46, 0x1e, 0x46, 0x0f, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
-	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x92, 0x02, 0x03, 0x15, 0x81,
-	0xe3, 0x05, 0x00, 0x20, 0x41, 0x10, 0xe2, 0x00, 0x4a, 0x67, 0x03, 0x0c,
-	0x00, 0x12, 0x34, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0xfc
-};
-
-/*
- * edid-decode (hex):
- *
- * 00 ff ff ff ff ff ff 00 31 d8 2a 00 00 00 00 00
- * 00 21 01 03 81 a0 5a 78 02 00 00 00 00 00 00 00
- * 00 00 00 20 00 00 01 01 01 01 01 01 01 01 01 01
- * 01 01 01 01 01 01 02 3a 80 18 71 38 2d 40 58 2c
- * 45 00 40 84 63 00 00 1e 00 00 00 fc 00 54 65 73
- * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 32
- * 46 1e 46 0f 00 0a 20 20 20 20 20 20 00 00 00 10
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 92
- *
- * 02 03 1c 81 e3 05 c0 20 41 10 e2 00 4a 67 03 0c
- * 00 12 34 00 28 e6 06 05 01 52 52 51 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4e
- *
- * ----------------
- *
- * Block 0, Base EDID:
- *   EDID Structure Version & Revision: 1.3
- *   Vendor & Product Identification:
- *     Manufacturer: LNX
- *     Model: 42
- *     Made in: 2023
- *   Basic Display Parameters & Features:
- *     Digital display
- *     DFP 1.x compatible TMDS
- *     Maximum image size: 160 cm x 90 cm
- *     Gamma: 2.20
- *     Monochrome or grayscale display
- *     First detailed timing is the preferred timing
- *   Color Characteristics:
- *     Red  : 0.0000, 0.0000
- *     Green: 0.0000, 0.0000
- *     Blue : 0.0000, 0.0000
- *     White: 0.0000, 0.0000
- *   Established Timings I & II:
- *     DMT 0x04:   640x480    59.940476 Hz   4:3     31.469 kHz     25.175000 MHz
- *   Standard Timings: none
- *   Detailed Timing Descriptors:
- *     DTD 1:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (1600 mm x 900 mm)
- *                  Hfront   88 Hsync  44 Hback  148 Hpol P
- *                  Vfront    4 Vsync   5 Vback   36 Vpol P
- *     Display Product Name: 'Test EDID'
- *     Display Range Limits:
- *       Monitor ranges (GTF): 50-70 Hz V, 30-70 kHz H, max dotclock 150 MHz
- *     Dummy Descriptor:
- *   Extension blocks: 1
- * Checksum: 0x92
- *
- * ----------------
- *
- * Block 1, CTA-861 Extension Block:
- *   Revision: 3
- *   Underscans IT Video Formats by default
- *   Native detailed modes: 1
- *   Colorimetry Data Block:
- *     BT2020YCC
- *     BT2020RGB
- *     sRGB
- *   Video Data Block:
- *     VIC  16:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz
- *   Video Capability Data Block:
- *     YCbCr quantization: No Data
- *     RGB quantization: Selectable (via AVI Q)
- *     PT scan behavior: No Data
- *     IT scan behavior: Always Underscanned
- *     CE scan behavior: Always Underscanned
- *   Vendor-Specific Data Block (HDMI), OUI 00-0C-03:
- *     Source physical address: 1.2.3.4
- *     Maximum TMDS clock: 200 MHz
- *   HDR Static Metadata Data Block:
- *     Electro optical transfer functions:
- *       Traditional gamma - SDR luminance range
- *       SMPTE ST2084
- *     Supported static metadata descriptors:
- *       Static metadata type 1
- *     Desired content max luminance: 82 (295.365 cd/m^2)
- *     Desired content max frame-average luminance: 82 (295.365 cd/m^2)
- *     Desired content min luminance: 81 (0.298 cd/m^2)
- * Checksum: 0x4e  Unused space in Extension Block: 99 bytes
- *
- * ----------------
- *
- * edid-decode 1.31.0-5387
- * edid-decode SHA: 5508bc4301ac 2025-08-25 08:14:22
- *
- * EDID conformity: PASS
- */
-static const unsigned char test_edid_hdmi_1080p_rgb_max_200mhz_hdr[] = {
-	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x2a, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x01, 0x03, 0x81, 0xa0, 0x5a, 0x78,
-	0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20,
-	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3a, 0x80, 0x18, 0x71, 0x38,
-	0x2d, 0x40, 0x58, 0x2c, 0x45, 0x00, 0x40, 0x84, 0x63, 0x00, 0x00, 0x1e,
-	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
-	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x32,
-	0x46, 0x1e, 0x46, 0x0f, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
-	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x92, 0x02, 0x03, 0x1c, 0x81,
-	0xe3, 0x05, 0xc0, 0x20, 0x41, 0x10, 0xe2, 0x00, 0x4a, 0x67, 0x03, 0x0c,
-	0x00, 0x12, 0x34, 0x78, 0x28, 0xe6, 0x06, 0x05, 0x01, 0x52, 0x52, 0x51,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0xd6,
-};
-
-/*
- * edid-decode (hex):
- *
- * 00 ff ff ff ff ff ff 00 31 d8 2a 00 00 00 00 00
- * 00 21 01 03 81 a0 5a 78 02 00 00 00 00 00 00 00
- * 00 00 00 20 00 00 01 01 01 01 01 01 01 01 01 01
- * 01 01 01 01 01 01 02 3a 80 18 71 38 2d 40 58 2c
- * 45 00 40 84 63 00 00 1e 00 00 00 fc 00 54 65 73
- * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 32
- * 46 1e 46 0f 00 0a 20 20 20 20 20 20 00 00 00 10
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 92
- *
- * 02 03 15 81 e3 05 00 20 41 10 e2 00 4a 67 03 0c
- * 00 12 34 00 44 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 e0
- *
- * ----------------
- *
- * Block 0, Base EDID:
- *   EDID Structure Version & Revision: 1.3
- *   Vendor & Product Identification:
- *     Manufacturer: LNX
- *     Model: 42
- *     Made in: 2023
- *   Basic Display Parameters & Features:
- *     Digital display
- *     DFP 1.x compatible TMDS
- *     Maximum image size: 160 cm x 90 cm
- *     Gamma: 2.20
- *     Monochrome or grayscale display
- *     First detailed timing is the preferred timing
- *   Color Characteristics:
- *     Red  : 0.0000, 0.0000
- *     Green: 0.0000, 0.0000
- *     Blue : 0.0000, 0.0000
- *     White: 0.0000, 0.0000
- *   Established Timings I & II:
- *     DMT 0x04:   640x480    59.940476 Hz   4:3     31.469 kHz     25.175000 MHz
- *   Standard Timings: none
- *   Detailed Timing Descriptors:
- *     DTD 1:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (1600 mm x 900 mm)
- *                  Hfront   88 Hsync  44 Hback  148 Hpol P
- *                  Vfront    4 Vsync   5 Vback   36 Vpol P
- *     Display Product Name: 'Test EDID'
- *     Display Range Limits:
- *       Monitor ranges (GTF): 50-70 Hz V, 30-70 kHz H, max dotclock 150 MHz
- *     Dummy Descriptor:
- *   Extension blocks: 1
- * Checksum: 0x92
- *
- * ----------------
- *
- * Block 1, CTA-861 Extension Block:
- *   Revision: 3
- *   Underscans IT Video Formats by default
- *   Native detailed modes: 1
- *   Colorimetry Data Block:
- *     sRGB
- *   Video Data Block:
- *     VIC  16:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz
- *   Video Capability Data Block:
- *     YCbCr quantization: No Data
- *     RGB quantization: Selectable (via AVI Q)
- *     PT scan behavior: No Data
- *     IT scan behavior: Always Underscanned
- *     CE scan behavior: Always Underscanned
- *   Vendor-Specific Data Block (HDMI), OUI 00-0C-03:
- *     Source physical address: 1.2.3.4
- *     Maximum TMDS clock: 340 MHz
- * Checksum: 0xe0  Unused space in Extension Block: 106 bytes
- *
- * ----------------
- *
- * edid-decode 1.30.0-5367
- * edid-decode SHA: 41ebf7135691 2025-05-01 10:19:22
- *
- * EDID conformity: PASS
- */
-static const unsigned char test_edid_hdmi_1080p_rgb_max_340mhz[] = {
-	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x2a, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x01, 0x03, 0x81, 0xa0, 0x5a, 0x78,
-	0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20,
-	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3a, 0x80, 0x18, 0x71, 0x38,
-	0x2d, 0x40, 0x58, 0x2c, 0x45, 0x00, 0x40, 0x84, 0x63, 0x00, 0x00, 0x1e,
-	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
-	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x32,
-	0x46, 0x1e, 0x46, 0x0f, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
-	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x92, 0x02, 0x03, 0x15, 0x81,
-	0xe3, 0x05, 0x00, 0x20, 0x41, 0x10, 0xe2, 0x00, 0x4a, 0x67, 0x03, 0x0c,
-	0x00, 0x12, 0x34, 0x00, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0xe0
-};
-
-/*
- * edid-decode (hex):
- *
- * 00 ff ff ff ff ff ff 00 31 d8 2a 00 00 00 00 00
- * 00 21 01 03 81 a0 5a 78 1a 00 00 00 00 00 00 00
- * 00 00 00 20 00 00 01 01 01 01 01 01 01 01 01 01
- * 01 01 01 01 01 01 02 3a 80 18 71 38 2d 40 58 2c
- * 45 00 40 84 63 00 00 1e 00 00 00 fc 00 54 65 73
- * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 32
- * 46 1e 46 0f 00 0a 20 20 20 20 20 20 00 00 00 10
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 7a
- *
- * 02 03 15 b1 e3 05 00 20 41 10 e2 00 ca 67 03 0c
- * 00 12 34 78 28 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 d4
- *
- * ----------------
- *
- * Block 0, Base EDID:
- *   EDID Structure Version & Revision: 1.3
- *   Vendor & Product Identification:
- *     Manufacturer: LNX
- *     Model: 42
- *     Made in: 2023
- *   Basic Display Parameters & Features:
- *     Digital display
- *     DFP 1.x compatible TMDS
- *     Maximum image size: 160 cm x 90 cm
- *     Gamma: 2.20
- *     Undefined display color type
- *     First detailed timing is the preferred timing
- *   Color Characteristics:
- *     Red  : 0.0000, 0.0000
- *     Green: 0.0000, 0.0000
- *     Blue : 0.0000, 0.0000
- *     White: 0.0000, 0.0000
- *   Established Timings I & II:
- *     DMT 0x04:   640x480    59.940476 Hz   4:3     31.469 kHz     25.175000 MHz
- *   Standard Timings: none
- *   Detailed Timing Descriptors:
- *     DTD 1:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (1600 mm x 900 mm)
- *                  Hfront   88 Hsync  44 Hback  148 Hpol P
- *                  Vfront    4 Vsync   5 Vback   36 Vpol P
- *     Display Product Name: 'Test EDID'
- *     Display Range Limits:
- *       Monitor ranges (GTF): 50-70 Hz V, 30-70 kHz H, max dotclock 150 MHz
- *     Dummy Descriptor:
- *   Extension blocks: 1
- * Checksum: 0x7a
- *
- * ----------------
- *
- * Block 1, CTA-861 Extension Block:
- *   Revision: 3
- *   Underscans IT Video Formats by default
- *   Supports YCbCr 4:4:4
- *   Supports YCbCr 4:2:2
- *   Native detailed modes: 1
- *   Colorimetry Data Block:
- *     sRGB
- *   Video Data Block:
- *     VIC  16:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz
- *   Video Capability Data Block:
- *     YCbCr quantization: Selectable (via AVI YQ)
- *     RGB quantization: Selectable (via AVI Q)
- *     PT scan behavior: No Data
- *     IT scan behavior: Always Underscanned
- *     CE scan behavior: Always Underscanned
- *   Vendor-Specific Data Block (HDMI), OUI 00-0C-03:
- *     Source physical address: 1.2.3.4
- *     DC_48bit
- *     DC_36bit
- *     DC_30bit
- *     DC_Y444
- *     Maximum TMDS clock: 200 MHz
- * Checksum: 0xd4  Unused space in Extension Block: 106 bytes
- *
- * ----------------
- *
- * edid-decode 1.30.0-5367
- * edid-decode SHA: 41ebf7135691 2025-05-01 10:19:22
- *
- * EDID conformity: PASS
- */
-static const unsigned char test_edid_hdmi_1080p_rgb_yuv_dc_max_200mhz[] = {
-	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x2a, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x01, 0x03, 0x81, 0xa0, 0x5a, 0x78,
-	0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20,
-	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3a, 0x80, 0x18, 0x71, 0x38,
-	0x2d, 0x40, 0x58, 0x2c, 0x45, 0x00, 0x40, 0x84, 0x63, 0x00, 0x00, 0x1e,
-	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
-	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x32,
-	0x46, 0x1e, 0x46, 0x0f, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
-	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x7a, 0x02, 0x03, 0x15, 0xb1,
-	0xe3, 0x05, 0x00, 0x20, 0x41, 0x10, 0xe2, 0x00, 0xca, 0x67, 0x03, 0x0c,
-	0x00, 0x12, 0x34, 0x78, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0xd4
-};
-
-/*
- * edid-decode (hex):
- *
- * 00 ff ff ff ff ff ff 00 31 d8 2a 00 00 00 00 00
- * 00 21 01 03 81 a0 5a 78 0a 00 00 00 00 00 00 00
- * 00 00 00 20 00 00 01 01 01 01 01 01 01 01 01 01
- * 01 01 01 01 01 01 02 3a 80 18 71 38 2d 40 58 2c
- * 45 00 40 84 63 00 00 1e 00 00 00 fc 00 54 65 73
- * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 32
- * 46 1e 46 0f 00 0a 20 20 20 20 20 20 00 00 00 10
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 8a
- *
- * 02 03 15 b1 e3 05 00 20 41 10 e2 00 ca 67 03 0c
- * 00 12 34 78 44 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 b8
- *
- * ----------------
- *
- * Block 0, Base EDID:
- *   EDID Structure Version & Revision: 1.3
- *   Vendor & Product Identification:
- *     Manufacturer: LNX
- *     Model: 42
- *     Made in: 2023
- *   Basic Display Parameters & Features:
- *     Digital display
- *     DFP 1.x compatible TMDS
- *     Maximum image size: 160 cm x 90 cm
- *     Gamma: 2.20
- *     RGB color display
- *     First detailed timing is the preferred timing
- *   Color Characteristics:
- *     Red  : 0.0000, 0.0000
- *     Green: 0.0000, 0.0000
- *     Blue : 0.0000, 0.0000
- *     White: 0.0000, 0.0000
- *   Established Timings I & II:
- *     DMT 0x04:   640x480    59.940476 Hz   4:3     31.469 kHz     25.175000 MHz
- *   Standard Timings: none
- *   Detailed Timing Descriptors:
- *     DTD 1:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (1600 mm x 900 mm)
- *                  Hfront   88 Hsync  44 Hback  148 Hpol P
- *                  Vfront    4 Vsync   5 Vback   36 Vpol P
- *     Display Product Name: 'Test EDID'
- *     Display Range Limits:
- *       Monitor ranges (GTF): 50-70 Hz V, 30-70 kHz H, max dotclock 150 MHz
- *     Dummy Descriptor:
- *   Extension blocks: 1
- * Checksum: 0x8a
- *
- * ----------------
- *
- * Block 1, CTA-861 Extension Block:
- *   Revision: 3
- *   Underscans IT Video Formats by default
- *   Supports YCbCr 4:4:4
- *   Supports YCbCr 4:2:2
- *   Native detailed modes: 1
- *   Colorimetry Data Block:
- *     sRGB
- *   Video Data Block:
- *     VIC  16:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz
- *   Video Capability Data Block:
- *     YCbCr quantization: Selectable (via AVI YQ)
- *     RGB quantization: Selectable (via AVI Q)
- *     PT scan behavior: No Data
- *     IT scan behavior: Always Underscanned
- *     CE scan behavior: Always Underscanned
- *   Vendor-Specific Data Block (HDMI), OUI 00-0C-03:
- *     Source physical address: 1.2.3.4
- *     DC_48bit
- *     DC_36bit
- *     DC_30bit
- *     DC_Y444
- *     Maximum TMDS clock: 340 MHz
- * Checksum: 0xb8  Unused space in Extension Block: 106 bytes
- *
- * ----------------
- *
- * edid-decode 1.30.0-5367
- * edid-decode SHA: 41ebf7135691 2025-05-01 10:19:22
- *
- * EDID conformity: PASS
- */
-static const unsigned char test_edid_hdmi_1080p_rgb_yuv_dc_max_340mhz[] = {
-	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x2a, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x01, 0x03, 0x81, 0xa0, 0x5a, 0x78,
-	0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20,
-	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3a, 0x80, 0x18, 0x71, 0x38,
-	0x2d, 0x40, 0x58, 0x2c, 0x45, 0x00, 0x40, 0x84, 0x63, 0x00, 0x00, 0x1e,
-	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
-	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x32,
-	0x46, 0x1e, 0x46, 0x0f, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
-	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x8a, 0x02, 0x03, 0x15, 0xb1,
-	0xe3, 0x05, 0x00, 0x20, 0x41, 0x10, 0xe2, 0x00, 0xca, 0x67, 0x03, 0x0c,
-	0x00, 0x12, 0x34, 0x78, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0xb8
-};
-
-/*
- * Max resolution:
- * - 1920x1080@60Hz with RGB, YUV444, YUV422
- * - 3840x2160@30Hz with YUV420 only
- * Max BPC:         16 for all modes
- * Max TMDS clock:  200 MHz
- *
- * edid-decode (hex):
- *
- * 00 ff ff ff ff ff ff 00 31 d8 34 00 00 00 00 00
- * ff 23 01 03 80 60 36 78 0f ee 91 a3 54 4c 99 26
- * 0f 50 54 20 00 00 01 01 01 01 01 01 01 01 01 01
- * 01 01 01 01 01 01 02 3a 80 18 71 38 2d 40 58 2c
- * 45 00 c0 1c 32 00 00 1e 00 00 00 fc 00 54 65 73
- * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 18
- * 55 18 5e 11 00 0a 20 20 20 20 20 20 00 00 00 10
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 bb
- *
- * 02 03 29 31 42 90 5f 6c 03 0c 00 10 00 78 28 20
- * 00 00 01 03 6d d8 5d c4 01 28 80 07 00 00 00 00
- * 00 00 e3 0f 00 00 e2 0e 5f 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ca
- *
- * ----------------
- *
- * Block 0, Base EDID:
- *   EDID Structure Version & Revision: 1.3
- *   Vendor & Product Identification:
- *     Manufacturer: LNX
- *     Model: 52
- *     Model year: 2025
- *   Basic Display Parameters & Features:
- *     Digital display
- *     Maximum image size: 96 cm x 54 cm
- *     Gamma: 2.20
- *     RGB color display
- *     Default (sRGB) color space is primary color space
- *     First detailed timing is the preferred timing
- *     Supports GTF timings within operating range
- *   Color Characteristics:
- *     Red  : 0.6396, 0.3300
- *     Green: 0.2998, 0.5996
- *     Blue : 0.1503, 0.0595
- *     White: 0.3125, 0.3291
- *   Established Timings I & II:
- *     DMT 0x04:   640x480    59.940476 Hz   4:3     31.469 kHz     25.175000 MHz
- *   Standard Timings: none
- *   Detailed Timing Descriptors:
- *     DTD 1:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (960 mm x 540 mm)
- *                  Hfront   88 Hsync  44 Hback  148 Hpol P
- *                  Vfront    4 Vsync   5 Vback   36 Vpol P
- *     Display Product Name: 'Test EDID'
- *     Display Range Limits:
- *       Monitor ranges (GTF): 24-85 Hz V, 24-94 kHz H, max dotclock 170 MHz
- *     Dummy Descriptor:
- *   Extension blocks: 1
- * Checksum: 0xbb
- *
- * ----------------
- *
- * Block 1, CTA-861 Extension Block:
- *   Revision: 3
- *   Supports YCbCr 4:4:4
- *   Supports YCbCr 4:2:2
- *   Native detailed modes: 1
- *   Video Data Block:
- *     VIC  16:  1920x1080   60.000000 Hz  16:9     67.500 kHz    148.500000 MHz (native)
- *     VIC  95:  3840x2160   30.000000 Hz  16:9     67.500 kHz    297.000000 MHz
- *   Vendor-Specific Data Block (HDMI), OUI 00-0C-03:
- *     Source physical address: 1.0.0.0
- *     DC_48bit
- *     DC_36bit
- *     DC_30bit
- *     DC_Y444
- *     Maximum TMDS clock: 200 MHz
- *     Extended HDMI video details:
- *   Vendor-Specific Data Block (HDMI Forum), OUI C4-5D-D8:
- *     Version: 1
- *     Maximum TMDS Character Rate: 200 MHz
- *     SCDC Present
- *     Supports 16-bits/component Deep Color 4:2:0 Pixel Encoding
- *     Supports 12-bits/component Deep Color 4:2:0 Pixel Encoding
- *     Supports 10-bits/component Deep Color 4:2:0 Pixel Encoding
- *   YCbCr 4:2:0 Capability Map Data Block:
- *     Empty Capability Map
- *   YCbCr 4:2:0 Video Data Block:
- *     VIC  95:  3840x2160   30.000000 Hz  16:9     67.500 kHz    297.000000 MHz
- * Checksum: 0xca
- */
-static const unsigned char test_edid_hdmi_1080p_rgb_yuv_4k_yuv420_dc_max_200mhz[] = {
-	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x34, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0xff, 0x23, 0x01, 0x03, 0x80, 0x60, 0x36, 0x78,
-	0x0f, 0xee, 0x91, 0xa3, 0x54, 0x4c, 0x99, 0x26, 0x0f, 0x50, 0x54, 0x20,
-	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x3a, 0x80, 0x18, 0x71, 0x38,
-	0x2d, 0x40, 0x58, 0x2c, 0x45, 0x00, 0xc0, 0x1c, 0x32, 0x00, 0x00, 0x1e,
-	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
-	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x18,
-	0x55, 0x18, 0x5e, 0x11, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
-	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xbb, 0x02, 0x03, 0x29, 0x31,
-	0x42, 0x90, 0x5f, 0x6c, 0x03, 0x0c, 0x00, 0x10, 0x00, 0x78, 0x28, 0x20,
-	0x00, 0x00, 0x01, 0x03, 0x6d, 0xd8, 0x5d, 0xc4, 0x01, 0x28, 0x80, 0x07,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe3, 0x0f, 0x00, 0x00, 0xe2, 0x0e,
-	0x5f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0xca
-};
-
-/*
- * Max resolution: 3840x2160@30Hz with RGB, YUV444, YUV422, YUV420
- * Max BPC:        16 for all modes
- * Max TMDS clock: 340 MHz
- *
- * edid-decode (hex):
- *
- * 00 ff ff ff ff ff ff 00 31 d8 34 00 00 00 00 00
- * ff 23 01 03 80 60 36 78 0f ee 91 a3 54 4c 99 26
- * 0f 50 54 20 00 00 01 01 01 01 01 01 01 01 01 01
- * 01 01 01 01 01 01 04 74 00 30 f2 70 5a 80 b0 58
- * 8a 00 40 84 63 00 00 1e 00 00 00 fc 00 54 65 73
- * 74 20 45 44 49 44 0a 20 20 20 00 00 00 fd 00 18
- * 55 18 5e 22 00 0a 20 20 20 20 20 20 00 00 00 10
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 ce
- *
- * 02 03 27 31 41 5f 6c 03 0c 00 10 00 78 44 20 00
- * 00 01 03 6d d8 5d c4 01 44 80 07 00 00 00 00 00
- * 00 e3 0f 01 00 e1 0e 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
- * 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 84
- *
- * ----------------
- *
- * Block 0, Base EDID:
- *   EDID Structure Version & Revision: 1.3
- *   Vendor & Product Identification:
- *     Manufacturer: LNX
- *     Model: 52
- *     Model year: 2025
- *   Basic Display Parameters & Features:
- *     Digital display
- *     Maximum image size: 96 cm x 54 cm
- *     Gamma: 2.20
- *     RGB color display
- *     Default (sRGB) color space is primary color space
- *     First detailed timing is the preferred timing
- *     Supports GTF timings within operating range
- *   Color Characteristics:
- *     Red  : 0.6396, 0.3300
- *     Green: 0.2998, 0.5996
- *     Blue : 0.1503, 0.0595
- *     White: 0.3125, 0.3291
- *   Established Timings I & II:
- *     DMT 0x04:   640x480    59.940476 Hz   4:3     31.469 kHz     25.175000 MHz
- *   Standard Timings: none
- *   Detailed Timing Descriptors:
- *     DTD 1:  3840x2160   30.000000 Hz  16:9     67.500 kHz    297.000000 MHz (1600 mm x 900 mm)
- *                  Hfront  176 Hsync  88 Hback  296 Hpol P
- *                  Vfront    8 Vsync  10 Vback   72 Vpol P
- *     Display Product Name: 'Test EDID'
- *     Display Range Limits:
- *       Monitor ranges (GTF): 24-85 Hz V, 24-94 kHz H, max dotclock 340 MHz
- *     Dummy Descriptor:
- *   Extension blocks: 1
- * Checksum: 0xce
- *
- * ----------------
- *
- * Block 1, CTA-861 Extension Block:
- *   Revision: 3
- *   Supports YCbCr 4:4:4
- *   Supports YCbCr 4:2:2
- *   Native detailed modes: 1
- *   Video Data Block:
- *     VIC  95:  3840x2160   30.000000 Hz  16:9     67.500 kHz    297.000000 MHz
- *   Vendor-Specific Data Block (HDMI), OUI 00-0C-03:
- *     Source physical address: 1.0.0.0
- *     DC_48bit
- *     DC_36bit
- *     DC_30bit
- *     DC_Y444
- *     Maximum TMDS clock: 340 MHz
- *     Extended HDMI video details:
- *   Vendor-Specific Data Block (HDMI Forum), OUI C4-5D-D8:
- *     Version: 1
- *     Maximum TMDS Character Rate: 340 MHz
- *     SCDC Present
- *     Supports 16-bits/component Deep Color 4:2:0 Pixel Encoding
- *     Supports 12-bits/component Deep Color 4:2:0 Pixel Encoding
- *     Supports 10-bits/component Deep Color 4:2:0 Pixel Encoding
- *   YCbCr 4:2:0 Capability Map Data Block:
- *     VIC  95:  3840x2160   30.000000 Hz  16:9     67.500 kHz    297.000000 MHz
- *   YCbCr 4:2:0 Video Data Block:
- * Checksum: 0x84
- */
-static const unsigned char test_edid_hdmi_4k_rgb_yuv420_dc_max_340mhz[] = {
-	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x31, 0xd8, 0x34, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0xff, 0x23, 0x01, 0x03, 0x80, 0x60, 0x36, 0x78,
-	0x0f, 0xee, 0x91, 0xa3, 0x54, 0x4c, 0x99, 0x26, 0x0f, 0x50, 0x54, 0x20,
-	0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-	0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x74, 0x00, 0x30, 0xf2, 0x70,
-	0x5a, 0x80, 0xb0, 0x58, 0x8a, 0x00, 0x40, 0x84, 0x63, 0x00, 0x00, 0x1e,
-	0x00, 0x00, 0x00, 0xfc, 0x00, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x44,
-	0x49, 0x44, 0x0a, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x18,
-	0x55, 0x18, 0x5e, 0x22, 0x00, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
-	0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xce, 0x02, 0x03, 0x27, 0x31,
-	0x41, 0x5f, 0x6c, 0x03, 0x0c, 0x00, 0x10, 0x00, 0x78, 0x44, 0x20, 0x00,
-	0x00, 0x01, 0x03, 0x6d, 0xd8, 0x5d, 0xc4, 0x01, 0x44, 0x80, 0x07, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0xe3, 0x0f, 0x01, 0x00, 0xe1, 0x0e, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x84
-};
+extern const unsigned char test_edid_dvi_1080p[128];
+extern const unsigned char test_edid_hdmi_1080p_rgb_max_100mhz[256];
+extern const unsigned char test_edid_hdmi_1080p_rgb_max_200mhz[256];
+extern const unsigned char test_edid_hdmi_1080p_rgb_max_200mhz_hdr[256];
+extern const unsigned char test_edid_hdmi_1080p_rgb_max_340mhz[256];
+extern const unsigned char test_edid_hdmi_1080p_rgb_yuv_dc_max_200mhz[256];
+extern const unsigned char test_edid_hdmi_1080p_rgb_yuv_dc_max_340mhz[256];
+extern const unsigned char test_edid_hdmi_1080p_rgb_yuv_4k_yuv420_dc_max_200mhz[256];
+extern const unsigned char test_edid_hdmi_4k_rgb_yuv420_dc_max_340mhz[256];
 
 #endif // DRM_KUNIT_EDID_H_
diff --git a/drivers/gpu/drm/tidss/tidss_crtc.c b/drivers/gpu/drm/tidss/tidss_crtc.c
index acab9307bdf31..400329aa22004 100644
--- a/drivers/gpu/drm/tidss/tidss_crtc.c
+++ b/drivers/gpu/drm/tidss/tidss_crtc.c
@@ -357,20 +357,17 @@ static void tidss_crtc_destroy_state(struct drm_crtc *crtc,
 	kfree(tstate);
 }
 
-static void tidss_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *tidss_crtc_create_state(struct drm_crtc *crtc)
 {
 	struct tidss_crtc_state *tstate;
 
-	if (crtc->state)
-		tidss_crtc_destroy_state(crtc, crtc->state);
-
 	tstate = kzalloc_obj(*tstate);
-	if (!tstate) {
-		crtc->state = NULL;
-		return;
-	}
+	if (!tstate)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_crtc_state_init(&tstate->base, crtc);
 
-	__drm_atomic_helper_crtc_reset(crtc, &tstate->base);
+	return &tstate->base;
 }
 
 static struct drm_crtc_state *tidss_crtc_duplicate_state(struct drm_crtc *crtc)
@@ -405,10 +402,10 @@ static void tidss_crtc_destroy(struct drm_crtc *crtc)
 }
 
 static const struct drm_crtc_funcs tidss_crtc_funcs = {
-	.reset = tidss_crtc_reset,
 	.destroy = tidss_crtc_destroy,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
+	.atomic_create_state = tidss_crtc_create_state,
 	.atomic_duplicate_state = tidss_crtc_duplicate_state,
 	.atomic_destroy_state = tidss_crtc_destroy_state,
 	.enable_vblank = tidss_crtc_enable_vblank,
diff --git a/drivers/gpu/drm/tidss/tidss_encoder.c b/drivers/gpu/drm/tidss/tidss_encoder.c
index 34db8d2a3792e..698f8d964ca00 100644
--- a/drivers/gpu/drm/tidss/tidss_encoder.c
+++ b/drivers/gpu/drm/tidss/tidss_encoder.c
@@ -76,7 +76,7 @@ static int tidss_bridge_atomic_check(struct drm_bridge *bridge,
 static const struct drm_bridge_funcs tidss_bridge_funcs = {
 	.attach				= tidss_bridge_attach,
 	.atomic_check			= tidss_bridge_atomic_check,
-	.atomic_reset			= drm_atomic_helper_bridge_reset,
+	.atomic_create_state			= drm_atomic_helper_bridge_create_state,
 	.atomic_duplicate_state		= drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state		= drm_atomic_helper_bridge_destroy_state,
 };
diff --git a/drivers/gpu/drm/tidss/tidss_oldi.c b/drivers/gpu/drm/tidss/tidss_oldi.c
index 9c9725aac8290..5eb749f495d59 100644
--- a/drivers/gpu/drm/tidss/tidss_oldi.c
+++ b/drivers/gpu/drm/tidss/tidss_oldi.c
@@ -335,7 +335,7 @@ static const struct drm_bridge_funcs tidss_oldi_bridge_funcs = {
 	.atomic_get_input_bus_fmts = tidss_oldi_atomic_get_input_bus_fmts,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.mode_valid = tidss_oldi_mode_valid,
 };
 
diff --git a/drivers/gpu/drm/tidss/tidss_plane.c b/drivers/gpu/drm/tidss/tidss_plane.c
index 1a8b44fb45f8f..6d82976c2db1b 100644
--- a/drivers/gpu/drm/tidss/tidss_plane.c
+++ b/drivers/gpu/drm/tidss/tidss_plane.c
@@ -178,8 +178,8 @@ static const struct drm_plane_helper_funcs tidss_primary_plane_helper_funcs = {
 static const struct drm_plane_funcs tidss_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
-	.reset = drm_atomic_helper_plane_reset,
 	.destroy = drm_plane_destroy,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 };
diff --git a/drivers/gpu/drm/tiny/gm12u320.c b/drivers/gpu/drm/tiny/gm12u320.c
index d73dfebb4353f..4ad074337af07 100644
--- a/drivers/gpu/drm/tiny/gm12u320.c
+++ b/drivers/gpu/drm/tiny/gm12u320.c
@@ -388,7 +388,7 @@ static void gm12u320_fb_update_work(struct work_struct *work)
 	 * We must draw a frame every 2s otherwise the projector
 	 * switches back to showing its logo.
 	 */
-	queue_delayed_work(system_long_wq, &gm12u320->fb_update.work,
+	queue_delayed_work(system_dfl_long_wq, &gm12u320->fb_update.work,
 			   msecs_to_jiffies(IDLE_TIMEOUT));
 
 	return;
@@ -427,7 +427,8 @@ static void gm12u320_fb_mark_dirty(struct drm_framebuffer *fb,
 	mutex_unlock(&gm12u320->fb_update.lock);
 
 	if (wakeup)
-		mod_delayed_work(system_long_wq, &gm12u320->fb_update.work, 0);
+		mod_delayed_work(system_dfl_long_wq,
+				 &gm12u320->fb_update.work, 0);
 
 	if (old_fb)
 		drm_framebuffer_put(old_fb);
diff --git a/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c
index 2db221f6fc3a1..56ad8ef325840 100644
--- a/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c
+++ b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c
@@ -69,7 +69,7 @@ static void dma_resv_kunit_active_fence_init(struct kunit *test,
 	struct dma_fence *fence;
 
 	fence = alloc_mock_fence(test);
-	dma_fence_enable_sw_signaling(fence);
+	dma_fence_enable_signaling(fence);
 
 	dma_resv_lock(resv, NULL);
 	dma_resv_reserve_fences(resv, 1);
diff --git a/drivers/gpu/drm/ttm/ttm_backup.c b/drivers/gpu/drm/ttm/ttm_backup.c
index 81df4cb5606b4..c5b813a563e7f 100644
--- a/drivers/gpu/drm/ttm/ttm_backup.c
+++ b/drivers/gpu/drm/ttm/ttm_backup.c
@@ -117,7 +117,7 @@ ttm_backup_backup_page(struct file *backup, struct page *page,
 	if (writeback && !folio_mapped(to_folio) &&
 	    folio_clear_dirty_for_io(to_folio)) {
 		folio_set_reclaim(to_folio);
-		ret = shmem_writeout(to_folio, NULL, NULL);
+		ret = shmem_write_folio(to_folio);
 		if (!folio_test_writeback(to_folio))
 			folio_clear_reclaim(to_folio);
 		/*
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index bcd76f6bb7f02..3980f376e3ba4 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -224,7 +224,7 @@ static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 
 	dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP);
 	dma_resv_for_each_fence_unlocked(&cursor, fence)
-		dma_fence_enable_sw_signaling(fence);
+		dma_fence_enable_signaling(fence);
 	dma_resv_iter_end(&cursor);
 }
 
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index a80510489c456..88babf435ac23 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -32,6 +32,7 @@
 #define pr_fmt(fmt) "[TTM] " fmt
 
 #include <linux/export.h>
+#include <linux/pagemap.h>
 
 #include <drm/ttm/ttm_bo.h>
 #include <drm/ttm/ttm_placement.h>
@@ -208,9 +209,9 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,
 	if (unlikely(err != 0))
 		return VM_FAULT_SIGBUS;
 
-	page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) +
-		vma->vm_pgoff - drm_vma_node_start(&bo->base.vma_node);
-	page_last = vma_pages(vma) + vma->vm_pgoff -
+	page_offset = linear_page_index(vma, address) -
+		drm_vma_node_start(&bo->base.vma_node);
+	page_last = vma_end_pgoff(vma) -
 		drm_vma_node_start(&bo->base.vma_node);
 
 	if (unlikely(page_offset >= PFN_UP(bo->base.size)))
diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c
index 278bbe7a11add..80d0ad41456a2 100644
--- a/drivers/gpu/drm/ttm/ttm_pool.c
+++ b/drivers/gpu/drm/ttm/ttm_pool.c
@@ -165,8 +165,13 @@ static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags,
 	 * Do not add latency to the allocation path for allocations orders
 	 * device tolds us do not bring them additional performance gains.
 	 */
-	if (beneficial_order && order > beneficial_order)
-		gfp_flags &= ~__GFP_DIRECT_RECLAIM;
+	if (order && beneficial_order && order != beneficial_order)
+		gfp_flags &= ~__GFP_RECLAIM;
+
+	if (beneficial_order && order == beneficial_order) {
+		gfp_flags &= ~__GFP_NORETRY;
+		gfp_flags |= __GFP_RETRY_MAYFAIL;
+	}
 
 	if (!ttm_pool_uses_dma_alloc(pool)) {
 		p = alloc_pages_node(pool->nid, gfp_flags, order);
@@ -902,6 +907,7 @@ int ttm_pool_restore_and_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
 {
 	struct ttm_pool_tt_restore *restore = tt->restore;
 	struct ttm_pool_alloc_state alloc;
+	int ret;
 
 	if (WARN_ON(!ttm_tt_is_backed_up(tt)))
 		return -EINVAL;
@@ -925,14 +931,22 @@ int ttm_pool_restore_and_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
 	} else {
 		alloc = restore->snapshot_alloc;
 		if (ttm_pool_restore_valid(restore)) {
-			int ret = ttm_pool_restore_commit(restore, tt->backup,
-							  ctx, &alloc);
+			ret = ttm_pool_restore_commit(restore, tt->backup,
+						      ctx, &alloc);
 
 			if (ret)
 				return ret;
 		}
-		if (!alloc.remaining_pages)
+		if (!alloc.remaining_pages) {
+			ret = ttm_pool_apply_caching(&alloc);
+			if (ret)
+				return ret;
+
+			kfree(tt->restore);
+			tt->restore = NULL;
+
 			return 0;
+		}
 	}
 
 	return __ttm_pool_alloc(pool, tt, ctx, &alloc, restore);
@@ -1051,9 +1065,31 @@ long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt,
 		return -EBUSY;
 
 #ifdef CONFIG_X86
-	/* Anything returned to the system needs to be cached. */
-	if (tt->caching != ttm_cached)
-		set_pages_array_wb(tt->pages, tt->num_pages);
+	/* Anything returned to the system needs to be cached. Walk allocations
+	 * skipping NULL pages and issue set_pages_array_wb() per contiguous run.
+	 */
+	if (tt->caching != ttm_cached) {
+		pgoff_t run_start = 0, run_count = 0;
+
+		for (i = 0; i < tt->num_pages; i += num_pages) {
+			page = tt->pages[i];
+			if (unlikely(!page || ttm_backup_page_ptr_is_handle(page))) {
+				if (run_count) {
+					set_pages_array_wb(&tt->pages[run_start],
+							   run_count);
+					run_count = 0;
+				}
+				num_pages = 1;
+				continue;
+			}
+			num_pages = 1UL << ttm_pool_page_order(pool, page);
+			if (!run_count)
+				run_start = i;
+			run_count += num_pages;
+		}
+		if (run_count)
+			set_pages_array_wb(&tt->pages[run_start], run_count);
+	}
 #endif
 
 	if (tt->dma_address || flags->purge) {
@@ -1061,7 +1097,7 @@ long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt,
 			unsigned int order;
 
 			page = tt->pages[i];
-			if (unlikely(!page)) {
+			if (unlikely(!page || ttm_backup_page_ptr_is_handle(page))) {
 				num_pages = 1;
 				continue;
 			}
@@ -1104,6 +1140,10 @@ long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt,
 		if (unlikely(!page))
 			continue;
 
+		/* Already-handled entry from a previous attempt. */
+		if (unlikely(ttm_backup_page_ptr_is_handle(page)))
+			continue;
+
 		ttm_pool_split_for_swap(pool, page);
 
 		shandle = ttm_backup_backup_page(backup, page, flags->writeback, i,
@@ -1341,15 +1381,22 @@ static int ttm_pool_debugfs_shrink_show(struct seq_file *m, void *data)
 		.gfp_mask = GFP_NOFS,
 		.nr_to_scan = TTM_SHRINKER_BATCH,
 	};
-	unsigned long count;
+	unsigned long count, scanned;
 	int nid;
 
 	fs_reclaim_acquire(GFP_KERNEL);
 	for_each_node(nid) {
 		sc.nid = nid;
 		count = ttm_pool_shrinker_count(mm_shrinker, &sc);
-		seq_printf(m, "%d: %lu/%lu\n", nid, count,
-			   ttm_pool_shrinker_scan(mm_shrinker, &sc));
+		scanned = ttm_pool_shrinker_scan(mm_shrinker, &sc);
+
+		/* Convert shrinker API sentinel values to 0 for debugfs output */
+		if (count == SHRINK_EMPTY)
+			count = 0;
+		if (scanned == SHRINK_STOP)
+			scanned = 0;
+
+		seq_printf(m, "%d: %lu/%lu\n", nid, count, scanned);
 	}
 	fs_reclaim_release(GFP_KERNEL);
 
diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs
index d063bc664cc11..8348c6cd39295 100644
--- a/drivers/gpu/drm/tyr/driver.rs
+++ b/drivers/gpu/drm/tyr/driver.rs
@@ -7,7 +7,8 @@
     },
     device::{
         Core,
-        Device, //
+        Device,
+        DeviceContext, //
     },
     dma::{
         Device as DmaDevice,
@@ -46,13 +47,14 @@
 pub(crate) struct TyrDrmDriver;
 
 /// Convenience type alias for the DRM device type for this driver.
-pub(crate) type TyrDrmDevice<Ctx = drm::Registered> = drm::Device<TyrDrmDriver, Ctx>;
+pub(crate) type TyrDrmDevice<Ctx = drm::Normal> = drm::Device<TyrDrmDriver, Ctx>;
 
 pub(crate) struct TyrPlatformDriver;
 
 #[pin_data(PinnedDrop)]
-pub(crate) struct TyrPlatformDriverData {
+pub(crate) struct TyrPlatformDriverData<'bound> {
     _device: ARef<TyrDrmDevice>,
+    _reg: drm::Registration<'bound, TyrDrmDriver>,
 }
 
 #[pin_data]
@@ -97,7 +99,7 @@ fn issue_soft_reset(dev: &Device, iomem: &IoMem<'_>) -> Result {
 
 impl platform::Driver for TyrPlatformDriver {
     type IdInfo = ();
-    type Data<'bound> = TyrPlatformDriverData;
+    type Data<'bound> = TyrPlatformDriverData<'bound>;
     const OF_ID_TABLE: Option<of::IdTable<Self::IdInfo>> = Some(&OF_TABLE);
 
     fn probe<'bound>(
@@ -148,11 +150,14 @@ fn probe<'bound>(
                 gpu_info,
         });
 
-        let tdev = drm::UnregisteredDevice::<TyrDrmDriver>::new(pdev.as_ref(), data)?;
-        let tdev = drm::driver::Registration::new_foreign_owned(tdev, pdev.as_ref(), 0)?;
+        let tdev = drm::UnregisteredDevice::<TyrDrmDriver>::new(pdev, data)?;
+        // SAFETY: `reg` is stored in `TyrPlatformDriverData` and dropped when the driver is
+        // unbound; it is never forgotten.
+        let reg = unsafe { drm::Registration::new(pdev.as_ref(), tdev, (), 0)? };
 
         let driver = TyrPlatformDriverData {
-            _device: tdev.into(),
+            _device: reg.device().into(),
+            _reg: reg,
         };
 
         // We need this to be dev_info!() because dev_dbg!() does not work at
@@ -163,7 +168,7 @@ fn probe<'bound>(
 }
 
 #[pinned_drop]
-impl PinnedDrop for TyrPlatformDriverData {
+impl PinnedDrop for TyrPlatformDriverData<'_> {
     fn drop(self: Pin<&mut Self>) {}
 }
 
@@ -180,8 +185,10 @@ fn drop(self: Pin<&mut Self>) {}
 #[vtable]
 impl drm::Driver for TyrDrmDriver {
     type Data = TyrDrmDeviceData;
+    type RegistrationData<'a> = ();
     type File = TyrDrmFileData;
-    type Object<R: drm::DeviceContext> = drm::gem::shmem::Object<BoData, R>;
+    type Object = drm::gem::shmem::Object<BoData>;
+    type ParentDevice<Ctx: DeviceContext> = platform::Device<Ctx>;
 
     const INFO: drm::DriverInfo = INFO;
     const FEAT_RENDER: bool = true;
diff --git a/drivers/gpu/drm/tyr/file.rs b/drivers/gpu/drm/tyr/file.rs
index 31411da203c5d..b686041d5d6b4 100644
--- a/drivers/gpu/drm/tyr/file.rs
+++ b/drivers/gpu/drm/tyr/file.rs
@@ -1,7 +1,10 @@
 // SPDX-License-Identifier: GPL-2.0 or MIT
 
 use kernel::{
-    drm,
+    drm::{
+        self,
+        Registered, //
+    },
     prelude::*,
     uaccess::UserSlice,
     uapi, //
@@ -28,7 +31,8 @@ fn open(_dev: &drm::Device<Self::Driver>) -> Result<Pin<KBox<Self>>> {
 
 impl TyrDrmFileData {
     pub(crate) fn dev_query(
-        ddev: &TyrDrmDevice,
+        ddev: &TyrDrmDevice<Registered>,
+        _reg_data: &(),
         devquery: &mut uapi::drm_panthor_dev_query,
         _file: &TyrDrmFile,
     ) -> Result<u32> {
diff --git a/drivers/gpu/drm/tyr/gem.rs b/drivers/gpu/drm/tyr/gem.rs
index c6d4d6f9bae39..1640a161754b2 100644
--- a/drivers/gpu/drm/tyr/gem.rs
+++ b/drivers/gpu/drm/tyr/gem.rs
@@ -5,10 +5,7 @@
 //! DRM's GEM subsystem with shmem backing.
 
 use kernel::{
-    drm::{
-        gem,
-        DeviceContext, //
-    },
+    drm::gem,
     prelude::*, //
 };
 
@@ -33,11 +30,7 @@ impl gem::DriverObject for BoData {
     type Driver = TyrDrmDriver;
     type Args = BoCreateArgs;
 
-    fn new<Ctx: DeviceContext>(
-        _dev: &TyrDrmDevice<Ctx>,
-        _size: usize,
-        args: BoCreateArgs,
-    ) -> impl PinInit<Self, Error> {
+    fn new(_dev: &TyrDrmDevice, _size: usize, args: BoCreateArgs) -> impl PinInit<Self, Error> {
         try_pin_init!(Self { flags: args.flags })
     }
 }
diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs
index 562023e5df2f3..831357a8ef87d 100644
--- a/drivers/gpu/drm/tyr/regs.rs
+++ b/drivers/gpu/drm/tyr/regs.rs
@@ -48,17 +48,12 @@ pub(crate) fn read_u64_no_tearing(lo_read: impl Fn() -> u32, hi_read: impl Fn()
 /// These registers correspond to the GPU_CONTROL register page.
 /// They are involved in GPU configuration and control.
 pub(crate) mod gpu_control {
-    use core::convert::TryFrom;
     use kernel::{
-        error::{
-            code::EINVAL,
-            Error, //
-        },
         num::Bounded,
+        prelude::*,
         register,
         uapi, //
     };
-    use pin_init::Zeroable;
 
     register! {
         /// GPU identification register.
@@ -964,14 +959,9 @@ pub(crate) mod mmu_control {
     ///
     /// This array contains 16 instances of the MMU_AS_CONTROL register page.
     pub(crate) mod mmu_as_control {
-        use core::convert::TryFrom;
-
         use kernel::{
-            error::{
-                code::EINVAL,
-                Error, //
-            },
             num::Bounded,
+            prelude::*,
             register, //
         };
 
diff --git a/drivers/gpu/drm/v3d/Kconfig b/drivers/gpu/drm/v3d/Kconfig
index ce62c5908e1db..6a33e0ab30de6 100644
--- a/drivers/gpu/drm/v3d/Kconfig
+++ b/drivers/gpu/drm/v3d/Kconfig
@@ -5,6 +5,7 @@ config DRM_V3D
 	depends on DRM
 	depends on COMMON_CLK
 	depends on MMU
+	select DRM_EXEC
 	select DRM_SCHED
 	select DRM_GEM_SHMEM_HELPER
 	help
diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c
index fc81dd1247e33..8d914dbc3315c 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.c
+++ b/drivers/gpu/drm/v3d/v3d_drv.c
@@ -4,7 +4,11 @@
 /**
  * DOC: Broadcom V3D Graphics Driver
  *
- * This driver supports the Broadcom V3D 3.3 and 4.1 OpenGL ES GPUs.
+ * This driver supports the Broadcom V3D 4.2 and 7.1 GPUs.
+ *
+ * Support for V3D 3.3 and 4.1 GPUs is deprecated and it will be removed
+ * in the next kernel release.
+ *
  * For V3D 2.x support, see the VC4 driver.
  *
  * The V3D GPU includes a tiled render (composed of a bin and render
@@ -414,13 +418,25 @@ static int v3d_platform_drm_probe(struct platform_device *pdev)
 	 */
 	WARN_ON(v3d->ver != gen);
 
+	/* V3D 3.3 and V3D 4.1 has had no in-tree userspace since Mesa
+	 * dropped support in 2024 (MR#25851) and they have no known users.
+	 * Due to that, support is scheduled for removal in the next release.
+	 */
+	if (v3d->ver <= V3D_GEN_41) {
+		dev_warn(dev,
+			 "V3D %u.%u support is deprecated and will be removed "
+			 "in the next kernel release. If you rely on this hardware, "
+			 "please report it to dri-devel@lists.freedesktop.org.\n",
+			 v3d->ver / 10, v3d->ver % 10);
+	}
+
 	v3d->cores = V3D_GET_FIELD(ident1, V3D_HUB_IDENT1_NCORES);
 	WARN_ON(v3d->cores > 1); /* multicore not yet implemented */
 
 	ident3 = V3D_READ(V3D_HUB_IDENT3);
 	v3d->rev = V3D_GET_FIELD(ident3, V3D_HUB_IDENT3_IPREV);
 
-	pm_runtime_set_autosuspend_delay(dev, 100);
+	pm_runtime_set_autosuspend_delay(dev, 50);
 	pm_runtime_use_autosuspend(dev);
 
 	ret = drm_dev_register(drm, 0);
diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h
index 4ebe175a8c6b0..1b88b3dff7574 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.h
+++ b/drivers/gpu/drm/v3d/v3d_drv.h
@@ -7,7 +7,8 @@
 #include <linux/spinlock_types.h>
 #include <linux/workqueue.h>
 
-#include <drm/drm_encoder.h>
+#include <drm/drm_device.h>
+#include <drm/drm_exec.h>
 #include <drm/drm_gem.h>
 #include <drm/drm_gem_shmem_helper.h>
 #include <drm/gpu_scheduler.h>
@@ -74,11 +75,13 @@ struct v3d_queue_state {
 	spinlock_t queue_lock;
 };
 
-/* Performance monitor object. The perform lifetime is controlled by userspace
- * using perfmon related ioctls. A perfmon can be attached to a submit_cl
- * request, and when this is the case, HW perf counters will be activated just
- * before the submit_cl is submitted to the GPU and disabled when the job is
- * done. This way, only events related to a specific job will be counted.
+/* Performance monitor object
+ *
+ * The performance monitor (perfmon) lifetime is controlled by userspace using
+ * perfmon related ioctls. A perfmon can be attached to a CL or CSD submission
+ * request, and when it is, HW performance counters will be activated just
+ * before the job is submitted to the GPU and disabled when the job is done.
+ * This way, only events related to a specific submission will be counted.
  */
 struct v3d_perfmon {
 	/* Tracks the number of users of the perfmon, when this counter reaches
@@ -86,9 +89,6 @@ struct v3d_perfmon {
 	 */
 	refcount_t refcnt;
 
-	/* Protects perfmon stop, as it can be invoked from multiple places. */
-	struct mutex lock;
-
 	/* Number of counters activated in this perfmon instance
 	 * (should be less than DRM_V3D_MAX_PERF_COUNTERS).
 	 */
@@ -170,8 +170,32 @@ struct v3d_dev {
 
 	struct v3d_queue_state queue[V3D_MAX_QUEUES];
 
-	/* Used to track the active perfmon if any. */
-	struct v3d_perfmon *active_perfmon;
+	/*
+	 * Tracks the performance monitor state and consistency.
+	 *
+	 * When a non-global perfmon is attached to a job, the scheduler must
+	 * not run any other job on the HW concurrently (otherwise, the
+	 * counters would be polluted by unrelated work).
+	 */
+	struct {
+		/* Protects @active. */
+		spinlock_t lock;
+
+		/* Perfmon currently programmed in HW (or NULL if none). */
+		struct v3d_perfmon *active;
+
+		/* Finished fence of the most recently submitted job that
+		 * opened a serialization window (i.e. a job with a non-global
+		 * perfmon attached).
+		 */
+		struct dma_fence *fence;
+
+		/* Finished fence of the most recently submitted job on each HW
+		 * queue. Used so that a new perfmon-carrying job can depend on
+		 * every job currently in-flight across all queues.
+		 */
+		struct dma_fence *last_hw_fence[V3D_MAX_QUEUES];
+	} perfmon_state;
 
 	/* Protects bo_stats */
 	struct mutex bo_lock;
@@ -288,6 +312,27 @@ to_v3d_fence(struct dma_fence *fence)
 #define V3D_CORE_READ(core, offset) readl(v3d->core_regs[core] + offset)
 #define V3D_CORE_WRITE(core, offset, val) writel(val, v3d->core_regs[core] + offset)
 
+#define V3D_MAX_JOBS_PER_SUBMISSION 3
+
+/* Per-ioctl submission context */
+struct v3d_submit {
+	struct v3d_dev *v3d;
+
+	struct drm_file *file_priv;
+
+	/* DRM exec context for this submission. */
+	struct drm_exec exec;
+
+	/* Ordered array of jobs forming the submission chain. Jobs are
+	 * appended via v3d_submit_add_job(), then chained and pushed to
+	 * the scheduler by v3d_submit_jobs().
+	 */
+	struct v3d_job *jobs[V3D_MAX_JOBS_PER_SUBMISSION];
+
+	/* Number of jobs currently in @jobs. */
+	u32 job_count;
+};
+
 struct v3d_job {
 	struct drm_sched_job base;
 
@@ -295,6 +340,9 @@ struct v3d_job {
 
 	struct v3d_dev *v3d;
 
+	/* The queue that the job was submitted on. */
+	enum v3d_queue queue;
+
 	/* This is the array of BOs that were looked up at the start
 	 * of submission.
 	 */
@@ -401,8 +449,10 @@ struct v3d_indirect_csd_info {
 	/* Indirect CSD */
 	struct v3d_csd_job *job;
 
-	/* Clean cache job associated to the Indirect CSD job */
-	struct v3d_job *clean_job;
+	/* Indirect CSD args, stashed by the extension parser and later used
+	 * to create the CSD job from them.
+	 */
+	struct drm_v3d_submit_csd args;
 
 	/* Offset within the BO where the workgroup counts are stored */
 	u32 offset;
@@ -417,9 +467,6 @@ struct v3d_indirect_csd_info {
 
 	/* Indirect BO */
 	struct drm_gem_object *indirect;
-
-	/* Context of the Indirect CSD job */
-	struct ww_acquire_ctx acquire_ctx;
 };
 
 struct v3d_timestamp_query_info {
@@ -643,6 +690,10 @@ void v3d_perfmon_put(struct v3d_perfmon *perfmon);
 void v3d_perfmon_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon);
 void v3d_perfmon_stop(struct v3d_dev *v3d, struct v3d_perfmon *perfmon,
 		      bool capture);
+void v3d_perfmon_stop_locked(struct v3d_dev *v3d, struct v3d_perfmon *perfmon,
+			     bool capture);
+void v3d_perfmon_suspend(struct v3d_dev *v3d);
+void v3d_perfmon_resume(struct v3d_dev *v3d);
 struct v3d_perfmon *v3d_perfmon_find(struct v3d_file_priv *v3d_priv, int id);
 void v3d_perfmon_open_file(struct v3d_file_priv *v3d_priv);
 void v3d_perfmon_close_file(struct v3d_file_priv *v3d_priv);
diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
index c43d9af41374b..18c941f15e2e4 100644
--- a/drivers/gpu/drm/v3d/v3d_gem.c
+++ b/drivers/gpu/drm/v3d/v3d_gem.c
@@ -137,7 +137,8 @@ v3d_reset(struct v3d_dev *v3d)
 	v3d_mmu_set_page_table(v3d);
 	v3d_irq_reset(v3d);
 
-	v3d_perfmon_stop(v3d, v3d->active_perfmon, false);
+	/* Re-arm the global perfmon HW counters that the reset zeroed. */
+	v3d_perfmon_resume(v3d);
 
 	trace_v3d_reset_end(dev);
 }
@@ -307,6 +308,7 @@ v3d_gem_init(struct drm_device *dev)
 	}
 
 	spin_lock_init(&v3d->mm_lock);
+	spin_lock_init(&v3d->perfmon_state.lock);
 	ret = drmm_mutex_init(dev, &v3d->bo_lock);
 	if (ret)
 		goto err_stats;
@@ -369,8 +371,11 @@ v3d_gem_destroy(struct drm_device *dev)
 	for (q = 0; q < V3D_MAX_QUEUES; q++) {
 		WARN_ON(v3d->queue[q].active_job);
 		v3d_stats_put(v3d->queue[q].stats);
+		dma_fence_put(v3d->perfmon_state.last_hw_fence[q]);
 	}
 
+	dma_fence_put(v3d->perfmon_state.fence);
+
 	drm_mm_takedown(&v3d->mm);
 
 	dma_free_coherent(v3d->drm.dev, 4096 * 1024, (void *)v3d->pt,
diff --git a/drivers/gpu/drm/v3d/v3d_irq.c b/drivers/gpu/drm/v3d/v3d_irq.c
index 86efaef2722c3..963d711dc16ba 100644
--- a/drivers/gpu/drm/v3d/v3d_irq.c
+++ b/drivers/gpu/drm/v3d/v3d_irq.c
@@ -90,9 +90,12 @@ v3d_irq_signal_fence(struct v3d_dev *v3d, enum v3d_queue q,
 		     void (*trace_irq)(struct drm_device *, uint64_t))
 {
 	struct v3d_queue_state *queue = &v3d->queue[q];
-	struct v3d_fence *fence = to_v3d_fence(queue->active_job->irq_fence);
+	struct v3d_job *job = queue->active_job;
+	struct v3d_fence *fence = to_v3d_fence(job->irq_fence);
 
-	v3d_job_update_stats(queue->active_job);
+	v3d_perfmon_stop(v3d, job->perfmon, true);
+
+	v3d_job_update_stats(job);
 	trace_irq(&v3d->drm, fence->seqno);
 
 	queue->active_job = NULL;
diff --git a/drivers/gpu/drm/v3d/v3d_perfmon.c b/drivers/gpu/drm/v3d/v3d_perfmon.c
index 48ae748247be8..07dab7fb30607 100644
--- a/drivers/gpu/drm/v3d/v3d_perfmon.c
+++ b/drivers/gpu/drm/v3d/v3d_perfmon.c
@@ -217,26 +217,15 @@ void v3d_perfmon_get(struct v3d_perfmon *perfmon)
 
 void v3d_perfmon_put(struct v3d_perfmon *perfmon)
 {
-	if (perfmon && refcount_dec_and_test(&perfmon->refcnt)) {
-		mutex_destroy(&perfmon->lock);
+	if (perfmon && refcount_dec_and_test(&perfmon->refcnt))
 		kfree(perfmon);
-	}
 }
 
-void v3d_perfmon_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon)
+static void v3d_perfmon_hw_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon)
 {
+	u8 ncounters = perfmon->ncounters;
+	u32 mask = GENMASK(ncounters - 1, 0);
 	unsigned int i;
-	u32 mask;
-	u8 ncounters;
-
-	if (WARN_ON_ONCE(!perfmon || v3d->active_perfmon))
-		return;
-
-	if (!pm_runtime_get_if_active(v3d->drm.dev))
-		return;
-
-	ncounters = perfmon->ncounters;
-	mask = GENMASK(ncounters - 1, 0);
 
 	for (i = 0; i < ncounters; i++) {
 		u32 source = i / 4;
@@ -258,39 +247,112 @@ void v3d_perfmon_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon)
 	V3D_CORE_WRITE(0, V3D_V4_PCTR_0_EN, mask);
 	V3D_CORE_WRITE(0, V3D_V4_PCTR_0_CLR, mask);
 	V3D_CORE_WRITE(0, V3D_PCTR_0_OVERFLOW, mask);
+}
+
+static void v3d_perfmon_hw_capture(struct v3d_dev *v3d, struct v3d_perfmon *perfmon)
+{
+	u32 mask = GENMASK(perfmon->ncounters - 1, 0);
 
-	v3d->active_perfmon = perfmon;
+	for (int i = 0; i < perfmon->ncounters; i++)
+		perfmon->values[i] += V3D_CORE_READ(0, V3D_PCTR_0_PCTRX(i));
 
-	v3d_pm_runtime_put(v3d);
+	V3D_CORE_WRITE(0, V3D_V4_PCTR_0_CLR, mask);
 }
 
-void v3d_perfmon_stop(struct v3d_dev *v3d, struct v3d_perfmon *perfmon,
-		      bool capture)
+static void v3d_perfmon_hw_stop(struct v3d_dev *v3d, struct v3d_perfmon *perfmon,
+				bool capture)
 {
-	unsigned int i;
+	if (capture)
+		v3d_perfmon_hw_capture(v3d, perfmon);
+
+	V3D_CORE_WRITE(0, V3D_V4_PCTR_0_EN, 0);
+}
 
-	if (!perfmon || !v3d->active_perfmon)
+void v3d_perfmon_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon)
+{
+	guard(spinlock_irqsave)(&v3d->perfmon_state.lock);
+
+	if (!perfmon || v3d->global_perfmon)
 		return;
 
-	mutex_lock(&perfmon->lock);
-	if (perfmon != v3d->active_perfmon)
-		goto out;
+	/* Cross-queue serialization should have drained any previous perfmon
+	 * job before this one runs.
+	 */
+	if (WARN_ON_ONCE(v3d->perfmon_state.active))
+		return;
 
 	if (!pm_runtime_get_if_active(v3d->drm.dev))
-		goto out_clear;
+		return;
 
-	if (capture)
-		for (i = 0; i < perfmon->ncounters; i++)
-			perfmon->values[i] += V3D_CORE_READ(0, V3D_PCTR_0_PCTRX(i));
+	v3d_perfmon_hw_start(v3d, perfmon);
+	v3d->perfmon_state.active = perfmon;
 
-	V3D_CORE_WRITE(0, V3D_V4_PCTR_0_EN, 0);
+	v3d_pm_runtime_put(v3d);
+}
 
+static void v3d_perfmon_capture_locked(struct v3d_dev *v3d,
+				       struct v3d_perfmon *perfmon)
+{
+	lockdep_assert_held(&v3d->perfmon_state.lock);
+
+	if (!perfmon || perfmon != v3d->perfmon_state.active)
+		return;
+
+	if (!pm_runtime_get_if_active(v3d->drm.dev))
+		return;
+
+	v3d_perfmon_hw_capture(v3d, perfmon);
 	v3d_pm_runtime_put(v3d);
+}
+
+void v3d_perfmon_stop_locked(struct v3d_dev *v3d, struct v3d_perfmon *perfmon,
+			     bool capture)
+{
+	lockdep_assert_held(&v3d->perfmon_state.lock);
 
-out_clear:
-	v3d->active_perfmon = NULL;
-out:
-	mutex_unlock(&perfmon->lock);
+	if (!perfmon || perfmon != v3d->perfmon_state.active)
+		return;
+
+	v3d->perfmon_state.active = NULL;
+
+	/* If the device is suspended, the HW has already stopped counting. */
+	if (!pm_runtime_get_if_active(v3d->drm.dev))
+		return;
+
+	v3d_perfmon_hw_stop(v3d, perfmon, capture);
+	v3d_pm_runtime_put(v3d);
+}
+
+void v3d_perfmon_stop(struct v3d_dev *v3d, struct v3d_perfmon *perfmon,
+		      bool capture)
+{
+	if (!perfmon)
+		return;
+
+	guard(spinlock_irqsave)(&v3d->perfmon_state.lock);
+	v3d_perfmon_stop_locked(v3d, perfmon, capture);
+}
+
+void
+v3d_perfmon_suspend(struct v3d_dev *v3d)
+{
+	guard(spinlock_irqsave)(&v3d->perfmon_state.lock);
+
+	if (!v3d->perfmon_state.active)
+		return;
+
+	v3d_perfmon_hw_stop(v3d, v3d->perfmon_state.active, true);
+}
+
+void
+v3d_perfmon_resume(struct v3d_dev *v3d)
+{
+	guard(spinlock_irqsave)(&v3d->perfmon_state.lock);
+
+	if (!v3d->perfmon_state.active)
+		return;
+
+	v3d_perfmon_hw_start(v3d, v3d->perfmon_state.active);
 }
 
 struct v3d_perfmon *v3d_perfmon_find(struct v3d_file_priv *v3d_priv, int id)
@@ -316,14 +378,17 @@ static void v3d_perfmon_delete(struct v3d_file_priv *v3d_priv,
 	struct v3d_dev *v3d = v3d_priv->v3d;
 
 	/* If the active perfmon is being destroyed, stop it first */
-	if (perfmon == v3d->active_perfmon)
-		v3d_perfmon_stop(v3d, perfmon, false);
+	scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) {
+		v3d_perfmon_stop_locked(v3d, perfmon, false);
 
-	/* If the global perfmon is being destroyed, clean it and release
-	 * the reference stashed in v3d_perfmon_set_global_ioctl().
-	 */
-	if (cmpxchg(&v3d->global_perfmon, perfmon, NULL) == perfmon)
-		v3d_perfmon_put(perfmon);
+		/* If the global perfmon is being destroyed, clean it and release
+		 * the reference stashed in v3d_perfmon_set_global_ioctl().
+		 */
+		if (v3d->global_perfmon == perfmon) {
+			v3d_perfmon_put(v3d->global_perfmon);
+			v3d->global_perfmon = NULL;
+		}
+	}
 
 	v3d_perfmon_put(perfmon);
 }
@@ -371,12 +436,10 @@ int v3d_perfmon_create_ioctl(struct drm_device *dev, void *data,
 	perfmon->ncounters = req->ncounters;
 
 	refcount_set(&perfmon->refcnt, 1);
-	mutex_init(&perfmon->lock);
 
 	ret = xa_alloc(&v3d_priv->perfmons, &id, perfmon, xa_limit_32b,
 		       GFP_KERNEL);
 	if (ret < 0) {
-		mutex_destroy(&perfmon->lock);
 		kfree(perfmon);
 		return ret;
 	}
@@ -408,7 +471,9 @@ int v3d_perfmon_get_values_ioctl(struct drm_device *dev, void *data,
 	struct v3d_dev *v3d = to_v3d_dev(dev);
 	struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
 	struct drm_v3d_perfmon_get_values *req = data;
+	u64 values[DRM_V3D_MAX_PERF_COUNTERS];
 	struct v3d_perfmon *perfmon;
+	size_t size;
 	int ret = 0;
 
 	if (req->pad != 0)
@@ -418,10 +483,14 @@ int v3d_perfmon_get_values_ioctl(struct drm_device *dev, void *data,
 	if (!perfmon)
 		return -EINVAL;
 
-	v3d_perfmon_stop(v3d, perfmon, true);
+	size = perfmon->ncounters * sizeof(u64);
+
+	scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) {
+		v3d_perfmon_capture_locked(v3d, perfmon);
+		memcpy(values, perfmon->values, size);
+	}
 
-	if (copy_to_user(u64_to_user_ptr(req->values_ptr), perfmon->values,
-			 perfmon->ncounters * sizeof(u64)))
+	if (copy_to_user(u64_to_user_ptr(req->values_ptr), values, size))
 		ret = -EFAULT;
 
 	v3d_perfmon_put(perfmon);
@@ -482,18 +551,36 @@ int v3d_perfmon_set_global_ioctl(struct drm_device *dev, void *data,
 		 */
 		v3d_perfmon_put(perfmon);
 
-		old = xchg(&v3d->global_perfmon, NULL);
-		if (!old)
-			return -EINVAL;
+		scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) {
+			old = v3d->global_perfmon;
+			if (!old)
+				return -EINVAL;
+
+			v3d_perfmon_stop_locked(v3d, old, true);
+			v3d->global_perfmon = NULL;
+		}
 
 		v3d_perfmon_put(old);
 
 		return 0;
 	}
 
-	if (cmpxchg(&v3d->global_perfmon, NULL, perfmon)) {
-		v3d_perfmon_put(perfmon);
-		return -EBUSY;
+	scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) {
+		if (v3d->perfmon_state.active || v3d->global_perfmon) {
+			v3d_perfmon_put(perfmon);
+			return -EBUSY;
+		}
+
+		v3d->global_perfmon = perfmon;
+		v3d->perfmon_state.active = perfmon;
+
+		/* If the device is suspended, v3d_perfmon_resume() will
+		 * program the HW on the next resume.
+		 */
+		if (pm_runtime_get_if_active(v3d->drm.dev)) {
+			v3d_perfmon_hw_start(v3d, perfmon);
+			v3d_pm_runtime_put(v3d);
+		}
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/v3d/v3d_power.c b/drivers/gpu/drm/v3d/v3d_power.c
index f7df6393d38fe..ade8e932fb9c5 100644
--- a/drivers/gpu/drm/v3d/v3d_power.c
+++ b/drivers/gpu/drm/v3d/v3d_power.c
@@ -50,6 +50,8 @@ int v3d_power_suspend(struct device *dev)
 	struct v3d_dev *v3d = to_v3d_dev(drm);
 	int ret;
 
+	v3d_perfmon_suspend(v3d);
+
 	v3d_irq_disable(v3d);
 
 	v3d_clean_caches(v3d);
@@ -85,5 +87,7 @@ int v3d_power_resume(struct device *dev)
 	v3d_mmu_set_page_table(v3d);
 	v3d_irq_enable(v3d);
 
+	v3d_perfmon_resume(v3d);
+
 	return 0;
 }
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
index c35f9f70cf8d6..c49b71b6e6ee9 100644
--- a/drivers/gpu/drm/v3d/v3d_sched.c
+++ b/drivers/gpu/drm/v3d/v3d_sched.c
@@ -125,24 +125,6 @@ v3d_performance_query_info_free(struct v3d_performance_query_info *query_info,
 	}
 }
 
-static void
-v3d_switch_perfmon(struct v3d_dev *v3d, struct v3d_job *job)
-{
-	struct v3d_perfmon *perfmon = v3d->global_perfmon;
-
-	if (!perfmon)
-		perfmon = job->perfmon;
-
-	if (perfmon == v3d->active_perfmon)
-		return;
-
-	if (perfmon != v3d->active_perfmon)
-		v3d_perfmon_stop(v3d, v3d->active_perfmon, true);
-
-	if (perfmon && v3d->active_perfmon != perfmon)
-		v3d_perfmon_start(v3d, perfmon);
-}
-
 static void
 v3d_stats_start(struct v3d_stats *stats, u64 now)
 {
@@ -189,15 +171,11 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
 	struct v3d_dev *v3d = job->base.v3d;
 	struct v3d_queue_state *queue = &v3d->queue[V3D_BIN];
 	struct drm_device *dev = &v3d->drm;
-	struct dma_fence *fence;
+	struct dma_fence *fence = NULL;
 	unsigned long irqflags;
 
-	if (unlikely(job->base.base.s_fence->finished.error)) {
-		spin_lock_irqsave(&queue->queue_lock, irqflags);
-		queue->active_job = NULL;
-		spin_unlock_irqrestore(&queue->queue_lock, irqflags);
-		return NULL;
-	}
+	if (unlikely(job->base.base.s_fence->finished.error))
+		goto out_clean_job;
 
 	/* Lock required around bin_job update vs
 	 * v3d_overflow_mem_work().
@@ -214,7 +192,7 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
 
 	fence = v3d_fence_create(v3d, V3D_BIN);
 	if (IS_ERR(fence))
-		return NULL;
+		goto out_clean_job;
 
 	if (job->base.irq_fence)
 		dma_fence_put(job->base.irq_fence);
@@ -224,7 +202,7 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
 			    job->start, job->end);
 
 	v3d_job_start_stats(&job->base);
-	v3d_switch_perfmon(v3d, &job->base);
+	v3d_perfmon_start(v3d, job->base.perfmon);
 
 	/* Set the current and end address of the control list.
 	 * Writing the end register is what starts the job.
@@ -242,6 +220,12 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
 	V3D_CORE_WRITE(0, V3D_CLE_CT0QEA, job->end);
 
 	return fence;
+
+out_clean_job:
+	spin_lock_irqsave(&queue->queue_lock, irqflags);
+	queue->active_job = NULL;
+	spin_unlock_irqrestore(&queue->queue_lock, irqflags);
+	return fence;
 }
 
 static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
@@ -249,12 +233,10 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
 	struct v3d_render_job *job = to_render_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
 	struct drm_device *dev = &v3d->drm;
-	struct dma_fence *fence;
+	struct dma_fence *fence = NULL;
 
-	if (unlikely(job->base.base.s_fence->finished.error)) {
-		v3d->queue[V3D_RENDER].active_job = NULL;
-		return NULL;
-	}
+	if (unlikely(job->base.base.s_fence->finished.error))
+		goto out_clean_job;
 
 	v3d->queue[V3D_RENDER].active_job = &job->base;
 
@@ -268,7 +250,7 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
 
 	fence = v3d_fence_create(v3d, V3D_RENDER);
 	if (IS_ERR(fence))
-		return NULL;
+		goto out_clean_job;
 
 	if (job->base.irq_fence)
 		dma_fence_put(job->base.irq_fence);
@@ -278,7 +260,7 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
 			    job->start, job->end);
 
 	v3d_job_start_stats(&job->base);
-	v3d_switch_perfmon(v3d, &job->base);
+	v3d_perfmon_start(v3d, job->base.perfmon);
 
 	/* XXX: Set the QCFG */
 
@@ -289,6 +271,10 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
 	V3D_CORE_WRITE(0, V3D_CLE_CT1QEA, job->end);
 
 	return fence;
+
+out_clean_job:
+	v3d->queue[V3D_RENDER].active_job = NULL;
+	return fence;
 }
 
 static struct dma_fence *
@@ -297,18 +283,16 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job)
 	struct v3d_tfu_job *job = to_tfu_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
 	struct drm_device *dev = &v3d->drm;
-	struct dma_fence *fence;
+	struct dma_fence *fence = NULL;
 
-	if (unlikely(job->base.base.s_fence->finished.error)) {
-		v3d->queue[V3D_TFU].active_job = NULL;
-		return NULL;
-	}
+	if (unlikely(job->base.base.s_fence->finished.error))
+		goto out_clean_job;
 
 	v3d->queue[V3D_TFU].active_job = &job->base;
 
 	fence = v3d_fence_create(v3d, V3D_TFU);
 	if (IS_ERR(fence))
-		return NULL;
+		goto out_clean_job;
 
 	if (job->base.irq_fence)
 		dma_fence_put(job->base.irq_fence);
@@ -336,6 +320,10 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job)
 	V3D_WRITE(V3D_TFU_ICFG(v3d->ver), job->args.icfg | V3D_TFU_ICFG_IOC);
 
 	return fence;
+
+out_clean_job:
+	v3d->queue[V3D_TFU].active_job = NULL;
+	return fence;
 }
 
 static struct dma_fence *
@@ -344,13 +332,11 @@ v3d_csd_job_run(struct drm_sched_job *sched_job)
 	struct v3d_csd_job *job = to_csd_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
 	struct drm_device *dev = &v3d->drm;
-	struct dma_fence *fence;
+	struct dma_fence *fence = NULL;
 	int i, csd_cfg0_reg;
 
-	if (unlikely(job->base.base.s_fence->finished.error)) {
-		v3d->queue[V3D_CSD].active_job = NULL;
-		return NULL;
-	}
+	if (unlikely(job->base.base.s_fence->finished.error))
+		goto out_clean_job;
 
 	/* The HW interprets a workgroup size of 0 as 65536; however, the
 	 * user-space driver exposes a maximum of 65535. Therefore, a 0 in
@@ -368,7 +354,7 @@ v3d_csd_job_run(struct drm_sched_job *sched_job)
 
 	fence = v3d_fence_create(v3d, V3D_CSD);
 	if (IS_ERR(fence))
-		return NULL;
+		goto out_clean_job;
 
 	if (job->base.irq_fence)
 		dma_fence_put(job->base.irq_fence);
@@ -377,7 +363,7 @@ v3d_csd_job_run(struct drm_sched_job *sched_job)
 	trace_v3d_submit_csd(dev, to_v3d_fence(fence)->seqno);
 
 	v3d_job_start_stats(&job->base);
-	v3d_switch_perfmon(v3d, &job->base);
+	v3d_perfmon_start(v3d, job->base.perfmon);
 
 	csd_cfg0_reg = V3D_CSD_QUEUED_CFG0(v3d->ver);
 	for (i = 1; i <= 6; i++)
@@ -395,6 +381,10 @@ v3d_csd_job_run(struct drm_sched_job *sched_job)
 	V3D_CORE_WRITE(0, csd_cfg0_reg, job->args.cfg[0]);
 
 	return fence;
+
+out_clean_job:
+	v3d->queue[V3D_CSD].active_job = NULL;
+	return fence;
 }
 
 static void
@@ -597,15 +587,8 @@ v3d_write_performance_query_result(struct v3d_cpu_job *job, void *data,
 
 		v3d_perfmon_stop(v3d, perfmon, true);
 
-		if (job->copy.do_64bit) {
-			for (j = 0; j < perfmon->ncounters; j++)
-				write_to_buffer_64(data, offset + j,
-						   perfmon->values[j]);
-		} else {
-			for (j = 0; j < perfmon->ncounters; j++)
-				write_to_buffer_32(data, offset + j,
-						   perfmon->values[j]);
-		}
+		for (j = 0; j < perfmon->ncounters; j++)
+			write_to_buffer(data, offset + j, job->copy.do_64bit, perfmon->values[j]);
 
 		v3d_perfmon_put(perfmon);
 	}
@@ -660,6 +643,9 @@ v3d_cpu_job_run(struct drm_sched_job *sched_job)
 	struct v3d_cpu_job *job = to_cpu_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
 
+	if (unlikely(job->base.base.s_fence->finished.error))
+		return NULL;
+
 	if (job->job_type >= ARRAY_SIZE(cpu_job_function)) {
 		drm_dbg(&v3d->drm, "Unknown CPU job: %d\n", job->job_type);
 		return NULL;
@@ -683,6 +669,9 @@ v3d_cache_clean_job_run(struct drm_sched_job *sched_job)
 	struct v3d_job *job = to_v3d_job(sched_job);
 	struct v3d_dev *v3d = job->v3d;
 
+	if (unlikely(job->base.s_fence->finished.error))
+		return NULL;
+
 	v3d_job_start_stats(job);
 
 	v3d_clean_caches(v3d);
@@ -709,6 +698,8 @@ v3d_gpu_reset_for_timeout(struct v3d_dev *v3d, struct drm_sched_job *sched_job,
 	if (sched_job)
 		drm_sched_increase_karma(sched_job);
 
+	v3d_perfmon_stop(v3d, job->perfmon, false);
+
 	/* get the GPU back into the init state */
 	v3d_reset(v3d);
 
diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c
index 7682b24f13ec5..119a81c9e1c46 100644
--- a/drivers/gpu/drm/v3d/v3d_submit.c
+++ b/drivers/gpu/drm/v3d/v3d_submit.c
@@ -4,6 +4,9 @@
  * Copyright (C) 2023 Raspberry Pi
  */
 
+#include <linux/dma-fence-unwrap.h>
+#include <linux/overflow.h>
+
 #include <drm/drm_print.h>
 #include <drm/drm_syncobj.h>
 
@@ -20,33 +23,51 @@
  * to v3d, so we don't attach dma-buf fences to them.
  */
 static int
-v3d_lock_bo_reservations(struct v3d_job *job,
-			 struct ww_acquire_ctx *acquire_ctx)
+v3d_submit_lock_reservations(struct v3d_submit *submit)
 {
-	int i, ret;
-
-	ret = drm_gem_lock_reservations(job->bo, job->bo_count, acquire_ctx);
-	if (ret)
-		return ret;
-
-	for (i = 0; i < job->bo_count; i++) {
-		ret = dma_resv_reserve_fences(job->bo[i]->resv, 1);
+	int i, j, ret;
+
+	drm_exec_init(&submit->exec,
+		      DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES, 0);
+	drm_exec_until_all_locked(&submit->exec) {
+		for (i = 0; i < submit->job_count; i++) {
+			struct v3d_job *job = submit->jobs[i];
+
+			ret = drm_exec_prepare_array(&submit->exec, job->bo,
+						     job->bo_count, 1);
+			if (ret)
+				break;
+		}
+		drm_exec_retry_on_contention(&submit->exec);
 		if (ret)
 			goto fail;
+	}
 
-		ret = drm_sched_job_add_implicit_dependencies(&job->base,
-							      job->bo[i], true);
-		if (ret)
-			goto fail;
+	for (i = 0; i < submit->job_count; i++) {
+		struct v3d_job *job = submit->jobs[i];
+
+		for (j = 0; j < job->bo_count; j++) {
+			ret = drm_sched_job_add_implicit_dependencies(&job->base,
+								      job->bo[j],
+								      true);
+			if (ret)
+				goto fail;
+		}
 	}
 
 	return 0;
 
 fail:
-	drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx);
+	drm_exec_fini(&submit->exec);
 	return ret;
 }
 
+static void
+v3d_submit_unlock_reservations(struct v3d_submit *submit)
+{
+	drm_exec_fini(&submit->exec);
+}
+
 /**
  * v3d_lookup_bos() - Sets up job->bo[] with the GEM objects
  * referenced by the job.
@@ -64,25 +85,23 @@ v3d_lock_bo_reservations(struct v3d_job *job,
  * failure, because that will happen at `v3d_job_free()`.
  */
 static int
-v3d_lookup_bos(struct drm_device *dev,
-	       struct drm_file *file_priv,
-	       struct v3d_job *job,
-	       u64 bo_handles,
-	       u32 bo_count)
+v3d_lookup_bos(struct v3d_submit *submit, u64 bo_handles, u32 bo_count)
 {
-	job->bo_count = bo_count;
+	struct v3d_job *last_job = submit->jobs[submit->job_count - 1];
 
-	if (!job->bo_count) {
+	last_job->bo_count = bo_count;
+
+	if (!last_job->bo_count) {
 		/* See comment on bo_index for why we have to check
 		 * this.
 		 */
-		drm_warn(dev, "Rendering requires BOs\n");
+		drm_warn(&submit->v3d->drm, "Rendering requires BOs\n");
 		return -EINVAL;
 	}
 
-	return drm_gem_objects_lookup(file_priv,
+	return drm_gem_objects_lookup(submit->file_priv,
 				      (void __user *)(uintptr_t)bo_handles,
-				      job->bo_count, &job->bo);
+				      last_job->bo_count, &last_job->bo);
 }
 
 static void
@@ -162,74 +181,87 @@ void v3d_job_put(struct v3d_job *job)
 }
 
 static int
-v3d_job_allocate(struct v3d_dev *v3d, void **container, size_t size)
+v3d_job_add_syncobjs(struct v3d_job *job, struct drm_file *file_priv,
+		     u32 in_sync, struct v3d_submit_ext *se)
 {
-	*container = kcalloc(1, size, GFP_KERNEL);
-	if (!*container) {
-		drm_err(&v3d->drm, "Cannot allocate memory for V3D job.\n");
-		return -ENOMEM;
+	bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC);
+	struct v3d_dev *v3d = job->v3d;
+	int ret = 0;
+
+	if (!has_multisync) {
+		/* Ignore syncobj if its handle is zero */
+		if (in_sync)
+			ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv,
+								   in_sync, 0);
+		return ret;
+	}
+
+	if (se->in_sync_count && se->wait_stage == job->queue) {
+		struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs);
+
+		for (int i = 0; i < se->in_sync_count; i++) {
+			struct drm_v3d_sem in;
+
+			if (copy_from_user(&in, handle++, sizeof(in))) {
+				drm_dbg(&v3d->drm, "Failed to copy wait dep handle.\n");
+				return -EFAULT;
+			}
+
+			/* Ignore syncobj if its handle is zero */
+			if (in.handle) {
+				ret = drm_sched_job_add_syncobj_dependency(&job->base,
+									   file_priv, in.handle, 0);
+				if (ret)
+					return ret;
+			}
+		}
 	}
 
 	return 0;
 }
 
-static void
-v3d_job_deallocate(void **container)
-{
-	kfree(*container);
-	*container = NULL;
-}
+static const struct {
+	size_t size;
+	void (*free)(struct kref *ref);
+} v3d_job_types[] = {
+	[V3D_BIN]		= { sizeof(struct v3d_bin_job), v3d_job_free },
+	[V3D_RENDER]		= { sizeof(struct v3d_render_job), v3d_render_job_free },
+	[V3D_TFU]		= { sizeof(struct v3d_tfu_job), v3d_job_free },
+	[V3D_CSD]		= { sizeof(struct v3d_csd_job), v3d_job_free },
+	[V3D_CACHE_CLEAN]	= { sizeof(struct v3d_job), v3d_job_free },
+	[V3D_CPU]		= { sizeof(struct v3d_cpu_job), v3d_cpu_job_free },
+};
 
-static int
-v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv,
-	     struct v3d_job *job, void (*free)(struct kref *ref),
-	     u32 in_sync, struct v3d_submit_ext *se, enum v3d_queue queue)
+static struct v3d_job *
+v3d_submit_add_job(struct v3d_submit *submit, enum v3d_queue queue)
 {
-	struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
-	bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC);
-	int ret, i;
+	struct v3d_file_priv *v3d_priv = submit->file_priv->driver_priv;
+	struct v3d_dev *v3d = submit->v3d;
+	struct v3d_job *job;
+	int ret;
+
+	if (queue >= V3D_MAX_QUEUES)
+		return ERR_PTR(-EINVAL);
+
+	job = kzalloc(v3d_job_types[queue].size, GFP_KERNEL);
+	if (!job)
+		return ERR_PTR(-ENOMEM);
 
 	job->v3d = v3d;
-	job->free = free;
+	job->queue = queue;
 	job->file_priv = v3d_priv;
+	job->free = v3d_job_types[queue].free;
 
 	ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue],
-				 1, v3d_priv, file_priv->client_id);
+				 1, v3d_priv, submit->file_priv->client_id);
 	if (ret)
-		return ret;
-
-	if (has_multisync) {
-		if (se->in_sync_count && se->wait_stage == queue) {
-			struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs);
-
-			for (i = 0; i < se->in_sync_count; i++) {
-				struct drm_v3d_sem in;
-
-				if (copy_from_user(&in, handle++, sizeof(in))) {
-					ret = -EFAULT;
-					drm_dbg(&v3d->drm, "Failed to copy wait dep handle.\n");
-					goto fail_job_init;
-				}
-				ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in.handle, 0);
-
-				// TODO: Investigate why this was filtered out for the IOCTL.
-				if (ret && ret != -ENOENT)
-					goto fail_job_init;
-			}
-		}
-	} else {
-		ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in_sync, 0);
-
-		// TODO: Investigate why this was filtered out for the IOCTL.
-		if (ret && ret != -ENOENT)
-			goto fail_job_init;
-	}
+		goto fail_free;
 
 	/* CPU jobs don't require hardware resources */
 	if (queue != V3D_CPU) {
 		ret = v3d_pm_runtime_get(v3d);
 		if (ret)
-			goto fail_job_init;
+			goto fail_sched_job;
 		job->has_pm_ref = true;
 	}
 
@@ -238,52 +270,160 @@ v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv,
 	job->client_stats = v3d_stats_get(v3d_priv->stats[queue]);
 	job->global_stats = v3d_stats_get(v3d->queue[queue].stats);
 
-	return 0;
+	submit->jobs[submit->job_count++] = job;
 
-fail_job_init:
+	return job;
+
+fail_sched_job:
 	drm_sched_job_cleanup(&job->base);
-	return ret;
+fail_free:
+	kfree(job);
+	return ERR_PTR(ret);
 }
 
 static void
-v3d_push_job(struct v3d_job *job)
+v3d_submit_put_jobs(struct v3d_submit *submit)
 {
-	drm_sched_job_arm(&job->base);
+	for (int i = 0; i < submit->job_count; i++)
+		v3d_job_put(submit->jobs[i]);
+}
 
-	job->done_fence = dma_fence_get(&job->base.s_fence->finished);
+static void
+v3d_submit_cleanup_jobs(struct v3d_submit *submit)
+{
+	for (int i = 0; i < submit->job_count; i++)
+		v3d_job_cleanup(submit->jobs[i]);
+}
 
-	/* put by scheduler job completion */
-	kref_get(&job->refcount);
+static int
+v3d_attach_perfmon_to_jobs(struct v3d_submit *submit, u32 perfmon_id)
+{
+	struct v3d_file_priv *v3d_priv = submit->file_priv->driver_priv;
+	struct v3d_dev *v3d = submit->v3d;
+	struct v3d_perfmon *perfmon;
+
+	if (!perfmon_id)
+		return 0;
 
-	drm_sched_entity_push_job(&job->base);
+	scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) {
+		if (v3d->global_perfmon)
+			return -EAGAIN;
+	}
+
+	perfmon = v3d_perfmon_find(v3d_priv, perfmon_id);
+	if (!perfmon)
+		return -ENOENT;
+
+	for (int i = 0; i < submit->job_count; i++) {
+		submit->jobs[i]->perfmon = perfmon;
+		if (i != 0)
+			v3d_perfmon_get(perfmon);
+	}
+
+	return 0;
 }
 
-static void
-v3d_attach_fences_and_unlock_reservation(struct drm_file *file_priv,
-					 struct v3d_job *job,
-					 struct ww_acquire_ctx *acquire_ctx,
-					 u32 out_sync,
-					 struct v3d_submit_ext *se,
-					 struct dma_fence *done_fence)
+/*
+ * Prepare fences to enforce job serialization when a perfmon is active. A job
+ * that carries a non-global perfmon must wait for every job currently in-flight
+ * across all HW queues to finish, otherwise concurrent unrelated work on the
+ * same core would pollute the performance counters. Symmetrically, while such a
+ * job is still in-flight, all subsequently submitted jobs must wait for it.
+ *
+ * We don't serialize the jobs when using a global perfmon as it's expected to
+ * track concurrent activity from all jobs.
+ */
+static int
+v3d_serialize_for_perfmon(struct v3d_job *job)
 {
-	struct drm_syncobj *sync_out;
-	bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC);
-	int i;
+	struct v3d_dev *v3d = job->v3d;
+	struct dma_fence *merged;
+	bool is_global_perfmon;
+	int ret;
+
+	lockdep_assert_held(&v3d->sched_lock);
+
+	scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock)
+		is_global_perfmon = !!v3d->global_perfmon;
+
+	if (is_global_perfmon)
+		goto publish;
+
+	if (job->perfmon) {
+		for (enum v3d_queue q = 0; q < V3D_MAX_QUEUES; q++) {
+			struct dma_fence *f = v3d->perfmon_state.last_hw_fence[q];
+
+			if (!f || dma_fence_is_signaled(f))
+				continue;
+
+			ret = drm_sched_job_add_dependency(&job->base, dma_fence_get(f));
+			if (ret)
+				return ret;
+		}
+	} else if (v3d->perfmon_state.fence &&
+		   !dma_fence_is_signaled(v3d->perfmon_state.fence)) {
+		ret = drm_sched_job_add_dependency(&job->base,
+						   dma_fence_get(v3d->perfmon_state.fence));
+		if (ret)
+			return ret;
+	}
+
+publish:
+	/*
+	 * Accumulate every in-flight job on this queue into one merged fence.
+	 * A HW queue is fed by several scheduler entities (one per-fd), so jobs
+	 * on it can complete out of order.
+	 */
+	merged = dma_fence_unwrap_merge(v3d->perfmon_state.last_hw_fence[job->queue],
+					job->done_fence);
+	if (!merged)
+		return -ENOMEM;
+
+	dma_fence_put(v3d->perfmon_state.last_hw_fence[job->queue]);
+	v3d->perfmon_state.last_hw_fence[job->queue] = merged;
 
-	for (i = 0; i < job->bo_count; i++) {
-		/* XXX: Use shared fences for read-only objects. */
-		dma_resv_add_fence(job->bo[i]->resv, job->done_fence,
-				   DMA_RESV_USAGE_WRITE);
+	if (job->perfmon && !is_global_perfmon) {
+		dma_fence_put(v3d->perfmon_state.fence);
+		v3d->perfmon_state.fence = dma_fence_get(job->done_fence);
+	}
+
+	return 0;
+}
+
+static void
+v3d_submit_attach_object_fences(struct v3d_submit *submit)
+{
+	struct v3d_job *last_job = submit->jobs[submit->job_count - 1];
+
+	/* The submission's last fence covers the entire submission. Attach it
+	 * to every BO touched by any job in the submission.
+	 */
+	for (int i = 0; i < submit->job_count; i++) {
+		struct v3d_job *job = submit->jobs[i];
+
+		for (int j = 0; j < job->bo_count; j++) {
+			/* XXX: Use shared fences for read-only objects. */
+			dma_resv_add_fence(job->bo[j]->resv, last_job->done_fence,
+					   DMA_RESV_USAGE_WRITE);
+		}
 	}
+}
 
-	drm_gem_unlock_reservations(job->bo, job->bo_count, acquire_ctx);
+static void
+v3d_submit_process_post_deps(struct v3d_submit *submit, struct drm_syncobj *sync_out,
+			     struct v3d_submit_ext *se)
+{
+	bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC);
+	struct v3d_job *last_job = submit->jobs[submit->job_count - 1];
+
+	/* Make sure single syncobj and multisync are mutually exclusive */
+	WARN_ON_ONCE(sync_out && has_multisync);
 
 	/* Update the return sync object for the job */
 	/* If it only supports a single signal semaphore*/
 	if (!has_multisync) {
-		sync_out = drm_syncobj_find(file_priv, out_sync);
 		if (sync_out) {
-			drm_syncobj_replace_fence(sync_out, done_fence);
+			drm_syncobj_replace_fence(sync_out, last_job->done_fence);
 			drm_syncobj_put(sync_out);
 		}
 		return;
@@ -291,9 +431,9 @@ v3d_attach_fences_and_unlock_reservation(struct drm_file *file_priv,
 
 	/* If multiple semaphores extension is supported */
 	if (se->out_sync_count) {
-		for (i = 0; i < se->out_sync_count; i++) {
+		for (int i = 0; i < se->out_sync_count; i++) {
 			drm_syncobj_replace_fence(se->out_syncs[i].syncobj,
-						  done_fence);
+						  last_job->done_fence);
 			drm_syncobj_put(se->out_syncs[i].syncobj);
 		}
 		kvfree(se->out_syncs);
@@ -301,53 +441,100 @@ v3d_attach_fences_and_unlock_reservation(struct drm_file *file_priv,
 }
 
 static int
-v3d_setup_csd_jobs_and_bos(struct drm_file *file_priv,
-			   struct v3d_dev *v3d,
-			   struct drm_v3d_submit_csd *args,
-			   struct v3d_csd_job **job,
-			   struct v3d_job **clean_job,
-			   struct v3d_submit_ext *se,
-			   struct ww_acquire_ctx *acquire_ctx)
+v3d_submit_jobs(struct v3d_submit *submit, struct drm_syncobj *sync_out,
+		struct v3d_submit_ext *se)
 {
-	int ret;
+	struct v3d_dev *v3d = submit->v3d;
+	int ret = 0;
 
-	ret = v3d_job_allocate(v3d, (void *)job, sizeof(**job));
-	if (ret)
-		return ret;
+	mutex_lock(&v3d->sched_lock);
 
-	ret = v3d_job_init(v3d, file_priv, &(*job)->base,
-			   v3d_job_free, args->in_sync, se, V3D_CSD);
-	if (ret) {
-		v3d_job_deallocate((void *)job);
-		return ret;
+	for (int i = 0; i < submit->job_count; i++) {
+		struct v3d_job *job = submit->jobs[i];
+
+		drm_sched_job_arm(&job->base);
+		job->done_fence = dma_fence_get(&job->base.s_fence->finished);
+
+		/* put by scheduler job completion */
+		kref_get(&job->refcount);
 	}
 
-	ret = v3d_job_allocate(v3d, (void *)clean_job, sizeof(**clean_job));
-	if (ret)
-		return ret;
+	for (int i = 1; i < submit->job_count; i++) {
+		ret = drm_sched_job_add_dependency(&submit->jobs[i]->base,
+						   dma_fence_get(submit->jobs[i - 1]->done_fence));
+		if (ret)
+			goto err;
+	}
 
-	ret = v3d_job_init(v3d, file_priv, *clean_job,
-			   v3d_job_free, 0, NULL, V3D_CACHE_CLEAN);
-	if (ret) {
-		v3d_job_deallocate((void *)clean_job);
-		return ret;
+	for (int i = 0; i < submit->job_count; i++) {
+		ret = v3d_serialize_for_perfmon(submit->jobs[i]);
+		if (ret)
+			goto err;
 	}
 
-	(*job)->args = *args;
+	for (int i = 0; i < submit->job_count; i++)
+		drm_sched_entity_push_job(&submit->jobs[i]->base);
+
+	mutex_unlock(&v3d->sched_lock);
+
+	v3d_submit_attach_object_fences(submit);
+	v3d_submit_unlock_reservations(submit);
+	v3d_submit_process_post_deps(submit, sync_out, se);
+
+	v3d_submit_put_jobs(submit);
+
+	return 0;
 
-	ret = v3d_lookup_bos(&v3d->drm, file_priv, *clean_job,
-			     args->bo_handles, args->bo_handle_count);
+err:
+	/* Mark every armed job as failed so run_job() skips execution */
+	for (int i = 0; i < submit->job_count; i++)
+		dma_fence_set_error(&submit->jobs[i]->base.s_fence->finished, ret);
+
+	for (int i = 0; i < submit->job_count; i++)
+		drm_sched_entity_push_job(&submit->jobs[i]->base);
+
+	mutex_unlock(&v3d->sched_lock);
+
+	v3d_submit_unlock_reservations(submit);
+	v3d_submit_put_jobs(submit);
+
+	return ret;
+}
+
+static int
+v3d_setup_csd_jobs_and_bos(struct v3d_submit *submit,
+			   struct drm_v3d_submit_csd *args,
+			   struct v3d_submit_ext *se)
+{
+	struct v3d_csd_job *job;
+	struct v3d_job *clean_job;
+	int ret;
+
+	job = (struct v3d_csd_job *)v3d_submit_add_job(submit, V3D_CSD);
+	if (IS_ERR(job))
+		return PTR_ERR(job);
+
+	ret = v3d_job_add_syncobjs(&job->base, submit->file_priv, args->in_sync, se);
 	if (ret)
 		return ret;
 
-	return v3d_lock_bo_reservations(*clean_job, acquire_ctx);
+	job->args = *args;
+
+	clean_job = v3d_submit_add_job(submit, V3D_CACHE_CLEAN);
+	if (IS_ERR(clean_job))
+		return PTR_ERR(clean_job);
+
+	return v3d_lookup_bos(submit, args->bo_handles, args->bo_handle_count);
 }
 
 static void
-v3d_put_multisync_post_deps(struct v3d_submit_ext *se)
+v3d_submit_put_post_deps(struct drm_syncobj *sync_out, struct v3d_submit_ext *se)
 {
 	unsigned int i;
 
+	if (sync_out)
+		drm_syncobj_put(sync_out);
+
 	if (!(se && se->out_sync_count))
 		return;
 
@@ -489,6 +676,7 @@ v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv,
 	}
 
 	job->job_type = V3D_CPU_JOB_TYPE_INDIRECT_CSD;
+	info->args = indirect_csd.submit;
 	info->offset = indirect_csd.offset;
 	info->wg_size = indirect_csd.wg_size;
 	memcpy(&info->wg_uniform_offsets, &indirect_csd.wg_uniform_offsets,
@@ -498,9 +686,7 @@ v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv,
 	if (!info->indirect)
 		return -ENOENT;
 
-	return v3d_setup_csd_jobs_and_bos(file_priv, v3d, &indirect_csd.submit,
-					  &info->job, &info->clean_job,
-					  NULL, &info->acquire_ctx);
+	return 0;
 }
 
 /* Get data for the query timestamp job submission. */
@@ -907,18 +1093,16 @@ int
 v3d_submit_cl_ioctl(struct drm_device *dev, void *data,
 		    struct drm_file *file_priv)
 {
-	struct v3d_dev *v3d = to_v3d_dev(dev);
-	struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
+	struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv };
 	struct drm_v3d_submit_cl *args = data;
+	struct drm_syncobj *sync_out = NULL;
 	struct v3d_submit_ext se = {0};
 	struct v3d_bin_job *bin = NULL;
-	struct v3d_render_job *render = NULL;
-	struct v3d_job *clean_job = NULL;
-	struct v3d_job *last_job;
-	struct ww_acquire_ctx acquire_ctx;
-	int ret = 0;
+	struct v3d_render_job *render;
+	struct v3d_job *clean_job;
+	int ret;
 
-	trace_v3d_submit_cl_ioctl(&v3d->drm, args->rcl_start, args->rcl_end);
+	trace_v3d_submit_cl_ioctl(dev, args->rcl_start, args->rcl_end);
 
 	if (args->pad)
 		return -EINVAL;
@@ -938,30 +1122,17 @@ v3d_submit_cl_ioctl(struct drm_device *dev, void *data,
 		}
 	}
 
-	ret = v3d_job_allocate(v3d, (void *)&render, sizeof(*render));
-	if (ret)
-		return ret;
-
-	ret = v3d_job_init(v3d, file_priv, &render->base,
-			   v3d_render_job_free, args->in_sync_rcl, &se, V3D_RENDER);
-	if (ret) {
-		v3d_job_deallocate((void *)&render);
-		goto fail;
+	/* If multisync is configured, give priority to it and ignore out_sync. */
+	if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) {
+		sync_out = drm_syncobj_find(file_priv, args->out_sync);
+		if (!sync_out)
+			return -ENOENT;
 	}
 
-	render->start = args->rcl_start;
-	render->end = args->rcl_end;
-	INIT_LIST_HEAD(&render->unref_list);
-
 	if (args->bcl_start != args->bcl_end) {
-		ret = v3d_job_allocate(v3d, (void *)&bin, sizeof(*bin));
-		if (ret)
-			goto fail;
-
-		ret = v3d_job_init(v3d, file_priv, &bin->base,
-				   v3d_job_free, args->in_sync_bcl, &se, V3D_BIN);
-		if (ret) {
-			v3d_job_deallocate((void *)&bin);
+		bin = (struct v3d_bin_job *)v3d_submit_add_job(&submit, V3D_BIN);
+		if (IS_ERR(bin)) {
+			ret = PTR_ERR(bin);
 			goto fail;
 		}
 
@@ -970,101 +1141,60 @@ v3d_submit_cl_ioctl(struct drm_device *dev, void *data,
 		bin->qma = args->qma;
 		bin->qms = args->qms;
 		bin->qts = args->qts;
-		bin->render = render;
-	}
 
-	if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) {
-		ret = v3d_job_allocate(v3d, (void *)&clean_job, sizeof(*clean_job));
+		ret = v3d_job_add_syncobjs(&bin->base, file_priv, args->in_sync_bcl,
+					   &se);
 		if (ret)
 			goto fail;
-
-		ret = v3d_job_init(v3d, file_priv, clean_job,
-				   v3d_job_free, 0, NULL, V3D_CACHE_CLEAN);
-		if (ret) {
-			v3d_job_deallocate((void *)&clean_job);
-			goto fail;
-		}
-
-		last_job = clean_job;
-	} else {
-		last_job = &render->base;
 	}
 
-	ret = v3d_lookup_bos(dev, file_priv, last_job,
-			     args->bo_handles, args->bo_handle_count);
-	if (ret)
+	render = (struct v3d_render_job *)v3d_submit_add_job(&submit, V3D_RENDER);
+	if (IS_ERR(render)) {
+		ret = PTR_ERR(render);
 		goto fail;
+	}
 
-	ret = v3d_lock_bo_reservations(last_job, &acquire_ctx);
-	if (ret)
-		goto fail;
+	INIT_LIST_HEAD(&render->unref_list);
+	render->start = args->rcl_start;
+	render->end = args->rcl_end;
 
-	if (args->perfmon_id) {
-		if (v3d->global_perfmon) {
-			ret = -EAGAIN;
-			goto fail_perfmon;
-		}
+	if (bin)
+		bin->render = render;
 
-		render->base.perfmon = v3d_perfmon_find(v3d_priv,
-							args->perfmon_id);
+	ret = v3d_job_add_syncobjs(&render->base, file_priv, args->in_sync_rcl, &se);
+	if (ret)
+		goto fail;
 
-		if (!render->base.perfmon) {
-			ret = -ENOENT;
-			goto fail_perfmon;
+	if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) {
+		clean_job = v3d_submit_add_job(&submit, V3D_CACHE_CLEAN);
+		if (IS_ERR(clean_job)) {
+			ret = PTR_ERR(clean_job);
+			goto fail;
 		}
 	}
 
-	mutex_lock(&v3d->sched_lock);
-	if (bin) {
-		bin->base.perfmon = render->base.perfmon;
-		v3d_perfmon_get(bin->base.perfmon);
-		v3d_push_job(&bin->base);
-
-		ret = drm_sched_job_add_dependency(&render->base.base,
-						   dma_fence_get(bin->base.done_fence));
-		if (ret)
-			goto fail_unreserve;
-	}
-
-	v3d_push_job(&render->base);
-
-	if (clean_job) {
-		struct dma_fence *render_fence =
-			dma_fence_get(render->base.done_fence);
-		ret = drm_sched_job_add_dependency(&clean_job->base,
-						   render_fence);
-		if (ret)
-			goto fail_unreserve;
-		clean_job->perfmon = render->base.perfmon;
-		v3d_perfmon_get(clean_job->perfmon);
-		v3d_push_job(clean_job);
-	}
+	ret = v3d_attach_perfmon_to_jobs(&submit, args->perfmon_id);
+	if (ret)
+		goto fail;
 
-	mutex_unlock(&v3d->sched_lock);
+	ret = v3d_lookup_bos(&submit, args->bo_handles, args->bo_handle_count);
+	if (ret)
+		goto fail;
 
-	v3d_attach_fences_and_unlock_reservation(file_priv,
-						 last_job,
-						 &acquire_ctx,
-						 args->out_sync,
-						 &se,
-						 last_job->done_fence);
+	ret = v3d_submit_lock_reservations(&submit);
+	if (ret)
+		goto fail;
 
-	v3d_job_put(&bin->base);
-	v3d_job_put(&render->base);
-	v3d_job_put(clean_job);
+	ret = v3d_submit_jobs(&submit, sync_out, &se);
+	if (ret)
+		goto fail_submit;
 
 	return 0;
 
-fail_unreserve:
-	mutex_unlock(&v3d->sched_lock);
-fail_perfmon:
-	drm_gem_unlock_reservations(last_job->bo,
-				    last_job->bo_count, &acquire_ctx);
 fail:
-	v3d_job_cleanup((void *)bin);
-	v3d_job_cleanup((void *)render);
-	v3d_job_cleanup(clean_job);
-	v3d_put_multisync_post_deps(&se);
+	v3d_submit_cleanup_jobs(&submit);
+fail_submit:
+	v3d_submit_put_post_deps(sync_out, &se);
 
 	return ret;
 }
@@ -1082,14 +1212,14 @@ int
 v3d_submit_tfu_ioctl(struct drm_device *dev, void *data,
 		     struct drm_file *file_priv)
 {
-	struct v3d_dev *v3d = to_v3d_dev(dev);
+	struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv };
 	struct drm_v3d_submit_tfu *args = data;
+	struct drm_syncobj *sync_out = NULL;
 	struct v3d_submit_ext se = {0};
-	struct v3d_tfu_job *job = NULL;
-	struct ww_acquire_ctx acquire_ctx;
+	struct v3d_tfu_job *job;
 	int ret = 0;
 
-	trace_v3d_submit_tfu_ioctl(&v3d->drm, args->iia);
+	trace_v3d_submit_tfu_ioctl(dev, args->iia);
 
 	if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) {
 		drm_dbg(dev, "invalid flags: %d\n", args->flags);
@@ -1104,17 +1234,23 @@ v3d_submit_tfu_ioctl(struct drm_device *dev, void *data,
 		}
 	}
 
-	ret = v3d_job_allocate(v3d, (void *)&job, sizeof(*job));
-	if (ret)
-		return ret;
+	/* If multisync is configured, give priority to it and ignore out_sync. */
+	if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) {
+		sync_out = drm_syncobj_find(file_priv, args->out_sync);
+		if (!sync_out)
+			return -ENOENT;
+	}
 
-	ret = v3d_job_init(v3d, file_priv, &job->base,
-			   v3d_job_free, args->in_sync, &se, V3D_TFU);
-	if (ret) {
-		v3d_job_deallocate((void *)&job);
+	job = (struct v3d_tfu_job *)v3d_submit_add_job(&submit, V3D_TFU);
+	if (IS_ERR(job)) {
+		ret = PTR_ERR(job);
 		goto fail;
 	}
 
+	ret = v3d_job_add_syncobjs(&job->base, file_priv, args->in_sync, &se);
+	if (ret)
+		goto fail;
+
 	job->base.bo = kzalloc_objs(*job->base.bo, ARRAY_SIZE(args->bo_handles));
 	if (!job->base.bo) {
 		ret = -ENOMEM;
@@ -1142,27 +1278,20 @@ v3d_submit_tfu_ioctl(struct drm_device *dev, void *data,
 		job->base.bo[job->base.bo_count] = bo;
 	}
 
-	ret = v3d_lock_bo_reservations(&job->base, &acquire_ctx);
+	ret = v3d_submit_lock_reservations(&submit);
 	if (ret)
 		goto fail;
 
-	mutex_lock(&v3d->sched_lock);
-	v3d_push_job(&job->base);
-	mutex_unlock(&v3d->sched_lock);
-
-	v3d_attach_fences_and_unlock_reservation(file_priv,
-						 &job->base, &acquire_ctx,
-						 args->out_sync,
-						 &se,
-						 job->base.done_fence);
-
-	v3d_job_put(&job->base);
+	ret = v3d_submit_jobs(&submit, sync_out, &se);
+	if (ret)
+		goto fail_submit;
 
 	return 0;
 
 fail:
-	v3d_job_cleanup((void *)job);
-	v3d_put_multisync_post_deps(&se);
+	v3d_submit_cleanup_jobs(&submit);
+fail_submit:
+	v3d_submit_put_post_deps(sync_out, &se);
 
 	return ret;
 }
@@ -1180,21 +1309,18 @@ int
 v3d_submit_csd_ioctl(struct drm_device *dev, void *data,
 		     struct drm_file *file_priv)
 {
-	struct v3d_dev *v3d = to_v3d_dev(dev);
-	struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
+	struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv };
 	struct drm_v3d_submit_csd *args = data;
+	struct drm_syncobj *sync_out = NULL;
 	struct v3d_submit_ext se = {0};
-	struct v3d_csd_job *job = NULL;
-	struct v3d_job *clean_job = NULL;
-	struct ww_acquire_ctx acquire_ctx;
 	int ret;
 
-	trace_v3d_submit_csd_ioctl(&v3d->drm, args->cfg[5], args->cfg[6]);
+	trace_v3d_submit_csd_ioctl(dev, args->cfg[5], args->cfg[6]);
 
 	if (args->pad)
 		return -EINVAL;
 
-	if (!v3d_has_csd(v3d)) {
+	if (!v3d_has_csd(submit.v3d)) {
 		drm_warn(dev, "Attempting CSD submit on non-CSD hardware\n");
 		return -EINVAL;
 	}
@@ -1212,58 +1338,35 @@ v3d_submit_csd_ioctl(struct drm_device *dev, void *data,
 		}
 	}
 
-	ret = v3d_setup_csd_jobs_and_bos(file_priv, v3d, args,
-					 &job, &clean_job, &se,
-					 &acquire_ctx);
-	if (ret)
-		goto fail;
-
-	if (args->perfmon_id) {
-		if (v3d->global_perfmon) {
-			ret = -EAGAIN;
-			goto fail_perfmon;
-		}
-
-		job->base.perfmon = v3d_perfmon_find(v3d_priv,
-						     args->perfmon_id);
-		if (!job->base.perfmon) {
-			ret = -ENOENT;
-			goto fail_perfmon;
-		}
+	/* If multisync is configured, give priority to it and ignore out_sync. */
+	if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) {
+		sync_out = drm_syncobj_find(file_priv, args->out_sync);
+		if (!sync_out)
+			return -ENOENT;
 	}
 
-	mutex_lock(&v3d->sched_lock);
-	v3d_push_job(&job->base);
-
-	ret = drm_sched_job_add_dependency(&clean_job->base,
-					   dma_fence_get(job->base.done_fence));
+	ret = v3d_setup_csd_jobs_and_bos(&submit, args, &se);
 	if (ret)
-		goto fail_unreserve;
+		goto fail;
 
-	v3d_push_job(clean_job);
-	mutex_unlock(&v3d->sched_lock);
+	ret = v3d_attach_perfmon_to_jobs(&submit, args->perfmon_id);
+	if (ret)
+		goto fail;
 
-	v3d_attach_fences_and_unlock_reservation(file_priv,
-						 clean_job,
-						 &acquire_ctx,
-						 args->out_sync,
-						 &se,
-						 clean_job->done_fence);
+	ret = v3d_submit_lock_reservations(&submit);
+	if (ret)
+		goto fail;
 
-	v3d_job_put(&job->base);
-	v3d_job_put(clean_job);
+	ret = v3d_submit_jobs(&submit, sync_out, &se);
+	if (ret)
+		goto fail_submit;
 
 	return 0;
 
-fail_unreserve:
-	mutex_unlock(&v3d->sched_lock);
-fail_perfmon:
-	drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count,
-				    &acquire_ctx);
 fail:
-	v3d_job_cleanup((void *)job);
-	v3d_job_cleanup(clean_job);
-	v3d_put_multisync_post_deps(&se);
+	v3d_submit_cleanup_jobs(&submit);
+fail_submit:
+	v3d_submit_put_post_deps(sync_out, &se);
 
 	return ret;
 }
@@ -1277,6 +1380,126 @@ static const unsigned int cpu_job_bo_handle_count[] = {
 	[V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY] = 1,
 };
 
+/* Reject offset + (count - 1) * stride + write_size if it leaves the BO. */
+static int
+v3d_check_copy_extent(struct drm_device *dev, size_t bo_size,
+		      u32 offset, u32 stride, u32 count, u64 write_size)
+{
+	u64 last;
+
+	if (!count)
+		return 0;
+
+	/*
+	 * The executors walk a u8 * cursor, so the furthest written byte is
+	 * offset + (count - 1) * stride + write_size, matching the pointer
+	 * arithmetic in v3d_copy_query_results()/v3d_copy_performance_query().
+	 * (count - 1) * stride is a u32 * u32 product that is exact in u64,
+	 * and offset + write_size stays far below the u64 range, so a single
+	 * overflow check guards the total.
+	 */
+	last = write_size + offset;
+	if (check_add_overflow((u64)(count - 1) * stride, last, &last) ||
+	    last > bo_size) {
+		drm_dbg(dev, "CPU job copy buffer exceeds the destination BO.\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/* Reject a query CPU job whose writes would land outside their BO. */
+static int
+v3d_cpu_job_bounds_check(struct v3d_cpu_job *job)
+{
+	struct drm_device *dev = &job->base.v3d->drm;
+	struct v3d_timestamp_query_info *tquery = &job->timestamp_query;
+	struct v3d_copy_query_results_info *copy = &job->copy;
+	u32 elem = copy->do_64bit ? sizeof(u64) : sizeof(u32);
+	struct v3d_bo *dst, *src;
+	u64 slots, write_size;
+	u32 i;
+
+	switch (job->job_type) {
+	case V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY:
+	case V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY:
+		/* Each query writes one u64 timestamp slot into bo[0]. */
+		dst = to_v3d_bo(job->base.bo[0]);
+
+		for (i = 0; i < tquery->count; i++) {
+			if ((u64)tquery->queries[i].offset + sizeof(u64) >
+			    dst->base.base.size)
+				goto err_range;
+		}
+		return 0;
+	case V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY:
+		/* Copies one u64 per query from bo[1] into bo[0]. */
+		dst = to_v3d_bo(job->base.bo[0]);
+		src = to_v3d_bo(job->base.bo[1]);
+
+		for (i = 0; i < tquery->count; i++) {
+			if ((u64)tquery->queries[i].offset + sizeof(u64) >
+			    src->base.base.size)
+				goto err_range;
+		}
+
+		write_size = (copy->availability_bit ? 2 : 1) * elem;
+		return v3d_check_copy_extent(dev, dst->base.base.size,
+					     copy->offset, copy->stride,
+					     tquery->count, write_size);
+	case V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY:
+		/*
+		 * Each query writes nperfmons * DRM_V3D_MAX_PERF_COUNTERS
+		 * counter slots into bo[0], plus an availability slot at
+		 * index ncounters. nperfmons and ncounters are user values,
+		 * so the slot count is computed overflow-safe.
+		 */
+		dst = to_v3d_bo(job->base.bo[0]);
+
+		slots = (u64)job->performance_query.nperfmons *
+			DRM_V3D_MAX_PERF_COUNTERS;
+		if (copy->availability_bit)
+			slots = max(slots,
+				    (u64)job->performance_query.ncounters + 1);
+
+		write_size = slots * elem;
+		return v3d_check_copy_extent(dev, dst->base.base.size,
+					     copy->offset, copy->stride,
+					     job->performance_query.count,
+					     write_size);
+	case V3D_CPU_JOB_TYPE_INDIRECT_CSD: {
+		struct v3d_indirect_csd_info *indirect_csd = &job->indirect_csd;
+
+		/* 3 is the three dimensions (x, y, z) of the workgroup counts. */
+		src = to_v3d_bo(job->base.bo[0]);
+		if ((u64)indirect_csd->offset + 3 * sizeof(u32) >
+		    src->base.base.size)
+			goto err_range;
+
+		dst = to_v3d_bo(indirect_csd->indirect);
+		for (i = 0; i < 3; i++) {
+			u32 uidx = indirect_csd->wg_uniform_offsets[i];
+
+			/*
+			 * 0xffffffff means "skip this rewrite", so the exec
+			 * path never writes that index and it needs no check.
+			 */
+			if (uidx != 0xffffffff &&
+			    (u64)uidx * sizeof(u32) + sizeof(u32) >
+			    dst->base.base.size)
+				goto err_range;
+		}
+		return 0;
+	}
+	default:
+		return 0;
+	}
+
+err_range:
+	drm_dbg(dev, "CPU job query offset exceeds the BO.\n");
+	return -EINVAL;
+}
+
 /**
  * v3d_submit_cpu_ioctl() - Submits a CPU job to the V3D.
  * @dev: DRM device
@@ -1290,14 +1513,10 @@ int
 v3d_submit_cpu_ioctl(struct drm_device *dev, void *data,
 		     struct drm_file *file_priv)
 {
-	struct v3d_dev *v3d = to_v3d_dev(dev);
+	struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv };
 	struct drm_v3d_submit_cpu *args = data;
 	struct v3d_submit_ext se = {0};
-	struct v3d_submit_ext *out_se = NULL;
 	struct v3d_cpu_job *cpu_job = NULL;
-	struct v3d_csd_job *csd_job = NULL;
-	struct v3d_job *clean_job = NULL;
-	struct ww_acquire_ctx acquire_ctx;
 	int ret;
 
 	if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) {
@@ -1305,9 +1524,9 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data,
 		return -EINVAL;
 	}
 
-	ret = v3d_job_allocate(v3d, (void *)&cpu_job, sizeof(*cpu_job));
-	if (ret)
-		return ret;
+	cpu_job = (struct v3d_cpu_job *)v3d_submit_add_job(&submit, V3D_CPU);
+	if (IS_ERR(cpu_job))
+		return PTR_ERR(cpu_job);
 
 	if (args->flags & DRM_V3D_SUBMIT_EXTENSION) {
 		ret = v3d_get_extensions(file_priv, args->extensions, &se, cpu_job);
@@ -1330,92 +1549,55 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data,
 		goto fail;
 	}
 
-	trace_v3d_submit_cpu_ioctl(&v3d->drm, cpu_job->job_type);
+	trace_v3d_submit_cpu_ioctl(dev, cpu_job->job_type);
 
-	ret = v3d_job_init(v3d, file_priv, &cpu_job->base,
-			   v3d_cpu_job_free, 0, &se, V3D_CPU);
-	if (ret) {
-		v3d_job_deallocate((void *)&cpu_job);
+	ret = v3d_job_add_syncobjs(&cpu_job->base, file_priv, 0, &se);
+	if (ret)
 		goto fail;
-	}
-
-	clean_job = cpu_job->indirect_csd.clean_job;
-	csd_job = cpu_job->indirect_csd.job;
 
+	/* Look up the CPU jobs' BOs before v3d_setup_csd_jobs_and_bos() appends
+	 * the CSD and clean jobs in the case of indirect CSD job.
+	 */
 	if (args->bo_handle_count) {
-		ret = v3d_lookup_bos(dev, file_priv, &cpu_job->base,
-				     args->bo_handles, args->bo_handle_count);
+		ret = v3d_lookup_bos(&submit, args->bo_handles, args->bo_handle_count);
 		if (ret)
 			goto fail;
 
-		ret = v3d_lock_bo_reservations(&cpu_job->base, &acquire_ctx);
+		ret = v3d_cpu_job_bounds_check(cpu_job);
 		if (ret)
 			goto fail;
 	}
 
-	mutex_lock(&v3d->sched_lock);
-	v3d_push_job(&cpu_job->base);
-
-	switch (cpu_job->job_type) {
-	case V3D_CPU_JOB_TYPE_INDIRECT_CSD:
-		ret = drm_sched_job_add_dependency(&csd_job->base.base,
-						   dma_fence_get(cpu_job->base.done_fence));
-		if (ret)
-			goto fail_unreserve;
-
-		v3d_push_job(&csd_job->base);
-
-		ret = drm_sched_job_add_dependency(&clean_job->base,
-						   dma_fence_get(csd_job->base.done_fence));
+	if (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) {
+		ret = v3d_setup_csd_jobs_and_bos(&submit, &cpu_job->indirect_csd.args,
+						 NULL);
 		if (ret)
-			goto fail_unreserve;
+			goto fail;
 
-		v3d_push_job(clean_job);
+		/* The CSD job was appended at jobs[1] */
+		if (WARN_ON(submit.jobs[1]->queue != V3D_CSD)) {
+			ret = -EINVAL;
+			goto fail;
+		}
 
-		break;
-	default:
-		break;
+		cpu_job->indirect_csd.job = container_of(submit.jobs[1], struct v3d_csd_job,
+							 base);
 	}
-	mutex_unlock(&v3d->sched_lock);
-
-	out_se = (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) ? NULL : &se;
 
-	v3d_attach_fences_and_unlock_reservation(file_priv,
-						 &cpu_job->base,
-						 &acquire_ctx, 0,
-						 out_se, cpu_job->base.done_fence);
-
-	switch (cpu_job->job_type) {
-	case V3D_CPU_JOB_TYPE_INDIRECT_CSD:
-		v3d_attach_fences_and_unlock_reservation(file_priv,
-							 clean_job,
-							 &cpu_job->indirect_csd.acquire_ctx,
-							 0, &se, clean_job->done_fence);
-		break;
-	default:
-		break;
-	}
+	ret = v3d_submit_lock_reservations(&submit);
+	if (ret)
+		goto fail;
 
-	v3d_job_put(&cpu_job->base);
-	v3d_job_put(&csd_job->base);
-	v3d_job_put(clean_job);
+	ret = v3d_submit_jobs(&submit, NULL, &se);
+	if (ret)
+		goto fail_submit;
 
 	return 0;
 
-fail_unreserve:
-	mutex_unlock(&v3d->sched_lock);
-
-	drm_gem_unlock_reservations(cpu_job->base.bo, cpu_job->base.bo_count,
-				    &acquire_ctx);
-
-	drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count,
-				    &cpu_job->indirect_csd.acquire_ctx);
-
 fail:
-	v3d_job_cleanup((void *)cpu_job);
-	v3d_job_cleanup((void *)csd_job);
-	v3d_job_cleanup(clean_job);
-	v3d_put_multisync_post_deps(&se);
+	v3d_submit_cleanup_jobs(&submit);
+fail_submit:
+	v3d_submit_put_post_deps(NULL, &se);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/vc4/vc4_dsi.c b/drivers/gpu/drm/vc4/vc4_dsi.c
index fc76018b044cd..0148f2befed14 100644
--- a/drivers/gpu/drm/vc4/vc4_dsi.c
+++ b/drivers/gpu/drm/vc4/vc4_dsi.c
@@ -1411,7 +1411,7 @@ static const struct mipi_dsi_host_ops vc4_dsi_host_ops = {
 static const struct drm_bridge_funcs vc4_dsi_bridge_funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_pre_enable = vc4_dsi_bridge_pre_enable,
 	.atomic_enable = vc4_dsi_bridge_enable,
 	.atomic_disable = vc4_dsi_bridge_disable,
diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c
index a161d3b00a255..74dce4be0c008 100644
--- a/drivers/gpu/drm/vc4/vc4_hdmi.c
+++ b/drivers/gpu/drm/vc4/vc4_hdmi.c
@@ -508,7 +508,7 @@ static int vc4_hdmi_connector_atomic_check(struct drm_connector *connector,
 static void vc4_hdmi_connector_reset(struct drm_connector *connector)
 {
 	drm_atomic_helper_connector_reset(connector);
-	__drm_atomic_helper_connector_hdmi_reset(connector, connector->state);
+	__drm_atomic_helper_connector_hdmi_state_init(connector, connector->state);
 	drm_atomic_helper_connector_tv_margins_reset(connector);
 }
 
diff --git a/drivers/gpu/drm/verisilicon/vs_bridge.c b/drivers/gpu/drm/verisilicon/vs_bridge.c
index 7a93049368db2..dc7c85b07fe32 100644
--- a/drivers/gpu/drm/verisilicon/vs_bridge.c
+++ b/drivers/gpu/drm/verisilicon/vs_bridge.c
@@ -246,7 +246,7 @@ static const struct drm_bridge_funcs vs_dpi_bridge_funcs = {
 	.atomic_get_output_bus_fmts = vs_bridge_atomic_get_output_bus_fmts_dpi,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 };
 
 static const struct drm_bridge_funcs vs_dp_bridge_funcs = {
@@ -258,7 +258,7 @@ static const struct drm_bridge_funcs vs_dp_bridge_funcs = {
 	.atomic_get_output_bus_fmts = vs_bridge_atomic_get_output_bus_fmts_dp,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 };
 
 static int vs_bridge_detect_output_interface(struct device_node *of_node,
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.c b/drivers/gpu/drm/virtio/virtgpu_drv.c
index 405097e388a53..2aaa7cb080858 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.c
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.c
@@ -119,10 +119,24 @@ static int virtio_gpu_probe(struct virtio_device *vdev)
 	return ret;
 }
 
+/*
+ * Release pending virtqueue waits so the drm_dev_enter/exit() critical
+ * sections complete before drm_dev_unplug() blocks on synchronize_srcu().
+ */
+static void virtio_gpu_release_vqs(struct drm_device *dev)
+{
+	struct virtio_gpu_device *vgdev = dev->dev_private;
+
+	vgdev->vqs_released = true;
+	wake_up_all(&vgdev->ctrlq.ack_queue);
+	wake_up_all(&vgdev->cursorq.ack_queue);
+}
+
 static void virtio_gpu_remove(struct virtio_device *vdev)
 {
 	struct drm_device *dev = vdev->priv;
 
+	virtio_gpu_release_vqs(dev);
 	drm_dev_unplug(dev);
 
 	if (drm_core_check_feature(dev, DRIVER_ATOMIC))
@@ -136,6 +150,7 @@ static void virtio_gpu_shutdown(struct virtio_device *vdev)
 {
 	struct drm_device *dev = vdev->priv;
 
+	virtio_gpu_release_vqs(dev);
 	/* stop talking to the device */
 	drm_dev_unplug(dev);
 }
@@ -168,6 +183,86 @@ static unsigned int features[] = {
 	VIRTIO_GPU_F_CONTEXT_INIT,
 	VIRTIO_GPU_F_BLOB_ALIGNMENT,
 };
+
+#ifdef CONFIG_PM_SLEEP
+static void virtgpu_hibernation_restore(struct virtio_gpu_device *vgdev)
+{
+	if (vgdev->hibernated) {
+		vgdev->hibernated = false;
+		virtio_gpu_object_restore_all(vgdev);
+	}
+}
+
+static int virtgpu_freeze(struct virtio_device *vdev)
+{
+	struct drm_device *dev = vdev->priv;
+	struct virtio_gpu_device *vgdev = dev->dev_private;
+	int error;
+
+	error = drm_mode_config_helper_suspend(dev);
+	if (error) {
+		DRM_ERROR("suspend error: %d\n", error);
+		return error;
+	}
+
+	if (vgdev->hibernated)
+		virtio_gpu_object_unref_all(vgdev);
+
+	flush_work(&vgdev->obj_free_work);
+	flush_work(&vgdev->ctrlq.dequeue_work);
+	flush_work(&vgdev->cursorq.dequeue_work);
+	flush_work(&vgdev->config_changed_work);
+
+	error = virtio_gpu_wait_queue(&vgdev->ctrlq, vgdev->ctrlq.vq->num_max);
+	if (error) {
+		DRM_ERROR("ctrlq wait failed: %d\n", error);
+		goto err_resume;
+	}
+
+	error = virtio_gpu_wait_queue(&vgdev->cursorq, vgdev->cursorq.vq->num_max);
+	if (error) {
+		DRM_ERROR("cursorq wait failed: %d\n", error);
+		goto err_resume;
+	}
+
+	vdev->config->del_vqs(vdev);
+
+	return 0;
+
+err_resume:
+	virtgpu_hibernation_restore(vgdev);
+
+	drm_mode_config_helper_resume(dev);
+
+	return error;
+}
+
+static int virtgpu_restore(struct virtio_device *vdev)
+{
+	struct drm_device *dev = vdev->priv;
+	struct virtio_gpu_device *vgdev = dev->dev_private;
+	int error;
+
+	error = virtio_gpu_find_vqs(vgdev);
+	if (error) {
+		DRM_ERROR("failed to find virt queues\n");
+		return error;
+	}
+
+	virtio_device_ready(vdev);
+
+	virtgpu_hibernation_restore(vgdev);
+
+	error = drm_mode_config_helper_resume(dev);
+	if (error) {
+		DRM_ERROR("resume error: %d\n", error);
+		return error;
+	}
+
+	return 0;
+}
+#endif
+
 static struct virtio_driver virtio_gpu_driver = {
 	.feature_table = features,
 	.feature_table_size = ARRAY_SIZE(features),
@@ -176,7 +271,11 @@ static struct virtio_driver virtio_gpu_driver = {
 	.probe = virtio_gpu_probe,
 	.remove = virtio_gpu_remove,
 	.shutdown = virtio_gpu_shutdown,
-	.config_changed = virtio_gpu_config_changed
+	.config_changed = virtio_gpu_config_changed,
+#ifdef CONFIG_PM_SLEEP
+	.freeze = virtgpu_freeze,
+	.restore = virtgpu_restore,
+#endif
 };
 
 static int __init virtio_gpu_driver_init(void)
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h
index 7449907754a43..17a6a4d265167 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.h
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.h
@@ -99,6 +99,10 @@ struct virtio_gpu_object {
 
 	int uuid_state;
 	uuid_t uuid;
+
+	/* for restoration of objects after hibernation */
+	struct virtio_gpu_object_params params;
+	struct list_head restore_node;
 };
 #define gem_to_virtio_gpu_obj(gobj) \
 	container_of((gobj), struct virtio_gpu_object, base.base)
@@ -236,6 +240,7 @@ struct virtio_gpu_device {
 
 	struct virtio_gpu_queue ctrlq;
 	struct virtio_gpu_queue cursorq;
+	bool vqs_released;
 	struct kmem_cache *vbufs;
 
 	atomic_t pending_commands;
@@ -259,6 +264,7 @@ struct virtio_gpu_device {
 	bool has_host_visible;
 	bool has_context_init;
 	bool has_blob_alignment;
+	bool hibernated;
 	struct virtio_shm_region host_visible_region;
 	struct drm_mm host_visible_mm;
 
@@ -267,6 +273,8 @@ struct virtio_gpu_device {
 	struct work_struct obj_free_work;
 	spinlock_t obj_free_lock;
 	struct list_head obj_free_list;
+	struct mutex obj_restore_lock;
+	struct list_head obj_restore_list;
 
 	struct virtio_gpu_drv_capset *capsets;
 	uint32_t num_capsets;
@@ -274,6 +282,8 @@ struct virtio_gpu_device {
 	struct list_head cap_cache;
 	uint32_t blob_alignment;
 
+	struct notifier_block pm_nb;
+
 	/* protects uuid state when exporting */
 	spinlock_t resource_export_lock;
 	/* protects map state and host_visible_mm */
@@ -303,6 +313,7 @@ void virtio_gpu_deinit(struct drm_device *dev);
 void virtio_gpu_release(struct drm_device *dev);
 int virtio_gpu_driver_open(struct drm_device *dev, struct drm_file *file);
 void virtio_gpu_driver_postclose(struct drm_device *dev, struct drm_file *file);
+int virtio_gpu_find_vqs(struct virtio_gpu_device *vgdev);
 
 /* virtgpu_gem.c */
 int virtio_gpu_gem_object_open(struct drm_gem_object *obj,
@@ -338,7 +349,8 @@ void virtio_gpu_cmd_create_resource(struct virtio_gpu_device *vgdev,
 				    struct virtio_gpu_object_array *objs,
 				    struct virtio_gpu_fence *fence);
 void virtio_gpu_cmd_unref_resource(struct virtio_gpu_device *vgdev,
-				   struct virtio_gpu_object *bo);
+				   struct virtio_gpu_object *bo,
+				   bool no_cb);
 int virtio_gpu_panic_cmd_transfer_to_host_2d(struct virtio_gpu_device *vgdev,
 					     uint64_t offset,
 					     uint32_t width, uint32_t height,
@@ -424,6 +436,7 @@ void virtio_gpu_dequeue_ctrl_func(struct work_struct *work);
 void virtio_gpu_dequeue_cursor_func(struct work_struct *work);
 void virtio_gpu_panic_notify(struct virtio_gpu_device *vgdev);
 void virtio_gpu_notify(struct virtio_gpu_device *vgdev);
+int virtio_gpu_wait_queue(struct virtio_gpu_queue *vgvq, unsigned int num_elem);
 
 int
 virtio_gpu_cmd_resource_assign_uuid(struct virtio_gpu_device *vgdev,
@@ -470,6 +483,7 @@ void virtio_gpu_fence_event_process(struct virtio_gpu_device *vdev,
 				    u64 fence_id);
 
 /* virtgpu_object.c */
+void virtio_gpu_remove_from_restore_list(struct virtio_gpu_object *bo);
 void virtio_gpu_cleanup_object(struct virtio_gpu_object *bo);
 struct drm_gem_object *virtio_gpu_create_object(struct drm_device *dev,
 						size_t size);
@@ -482,6 +496,14 @@ bool virtio_gpu_is_shmem(struct virtio_gpu_object *bo);
 
 int virtio_gpu_resource_id_get(struct virtio_gpu_device *vgdev,
 			       uint32_t *resid);
+
+void virtio_gpu_add_object_to_restore_list(struct virtio_gpu_device *vgdev,
+					   struct virtio_gpu_object *bo);
+
+int virtio_gpu_object_restore_all(struct virtio_gpu_device *vgdev);
+
+void virtio_gpu_object_unref_all(struct virtio_gpu_device *vgdev);
+
 /* virtgpu_prime.c */
 int virtio_gpu_resource_assign_uuid(struct virtio_gpu_device *vgdev,
 				    struct virtio_gpu_object *bo);
@@ -496,6 +518,8 @@ int virtgpu_dma_buf_import_sgt(struct virtio_gpu_mem_entry **ents,
 			       unsigned int *nents,
 			       struct virtio_gpu_object *bo,
 			       struct dma_buf_attachment *attach);
+int virtgpu_dma_buf_obj_resubmit(struct virtio_gpu_device *vgdev,
+				 struct virtio_gpu_object *bo);
 
 /* virtgpu_debugfs.c */
 void virtio_gpu_debugfs_init(struct drm_minor *minor);
diff --git a/drivers/gpu/drm/virtio/virtgpu_gem.c b/drivers/gpu/drm/virtio/virtgpu_gem.c
index 435d37d360340..66c3f6f74e9c6 100644
--- a/drivers/gpu/drm/virtio/virtgpu_gem.c
+++ b/drivers/gpu/drm/virtio/virtgpu_gem.c
@@ -139,13 +139,15 @@ void virtio_gpu_gem_object_close(struct drm_gem_object *obj,
 	if (!vgdev->has_virgl_3d)
 		return;
 
-	objs = virtio_gpu_array_alloc(1);
-	if (!objs)
-		return;
-	virtio_gpu_array_add_obj(objs, obj);
+	if (vfpriv->context_created) {
+		objs = virtio_gpu_array_alloc(1);
+		if (!objs)
+			return;
+		virtio_gpu_array_add_obj(objs, obj);
 
-	virtio_gpu_cmd_context_detach_resource(vgdev, vfpriv->ctx_id,
-					       objs);
+		virtio_gpu_cmd_context_detach_resource(vgdev, vfpriv->ctx_id,
+						       objs);
+	}
 	virtio_gpu_notify(vgdev);
 }
 
diff --git a/drivers/gpu/drm/virtio/virtgpu_kms.c b/drivers/gpu/drm/virtio/virtgpu_kms.c
index cfde9f573df6e..1ffe1e431f659 100644
--- a/drivers/gpu/drm/virtio/virtgpu_kms.c
+++ b/drivers/gpu/drm/virtio/virtgpu_kms.c
@@ -23,6 +23,7 @@
  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#include <linux/suspend.h>
 #include <linux/virtio.h>
 #include <linux/virtio_config.h>
 #include <linux/virtio_ring.h>
@@ -49,7 +50,10 @@ static void virtio_gpu_config_changed_work_func(struct work_struct *work)
 				virtio_gpu_cmd_get_edids(vgdev);
 			virtio_gpu_cmd_get_display_info(vgdev);
 			virtio_gpu_notify(vgdev);
-			drm_helper_hpd_irq_event(vgdev->ddev);
+			wait_event_timeout(vgdev->resp_wq,
+					   !vgdev->display_info_pending,
+					   5 * HZ);
+			drm_kms_helper_hotplug_event(vgdev->ddev);
 		}
 		events_clear |= VIRTIO_GPU_EVENT_DISPLAY;
 	}
@@ -115,15 +119,53 @@ static void virtio_gpu_get_capsets(struct virtio_gpu_device *vgdev,
 	vgdev->num_capsets = num_capsets;
 }
 
-int virtio_gpu_init(struct virtio_device *vdev, struct drm_device *dev)
+int virtio_gpu_find_vqs(struct virtio_gpu_device *vgdev)
 {
 	struct virtqueue_info vqs_info[] = {
 		{ "control", virtio_gpu_ctrl_ack },
 		{ "cursor", virtio_gpu_cursor_ack },
 	};
-	struct virtio_gpu_device *vgdev;
-	/* this will expand later */
 	struct virtqueue *vqs[2];
+	int ret;
+
+	ret = virtio_find_vqs(vgdev->vdev, 2, vqs, vqs_info, NULL);
+	if (ret)
+		return ret;
+
+	vgdev->ctrlq.vq = vqs[0];
+	vgdev->cursorq.vq = vqs[1];
+
+	return 0;
+}
+
+static int virtio_gpu_pm_notifier(struct notifier_block *nb, unsigned long mode,
+				  void *data)
+{
+	struct virtio_gpu_device *vgdev = container_of(nb,
+						struct virtio_gpu_device,
+						pm_nb);
+
+	switch (mode) {
+	case PM_HIBERNATION_PREPARE:
+		if (vgdev->has_virgl_3d) {
+			DRM_ERROR("S4 not allowed when VIRGL is enabled\n");
+			return notifier_from_errno(-EPERM);
+		}
+
+		vgdev->hibernated = true;
+		break;
+
+	case PM_POST_HIBERNATION:
+		vgdev->hibernated = false;
+		break;
+	}
+
+	return NOTIFY_DONE;
+}
+
+int virtio_gpu_init(struct virtio_device *vdev, struct drm_device *dev)
+{
+	struct virtio_gpu_device *vgdev;
 	u32 num_scanouts, num_capsets, blob_alignment;
 	int ret = 0;
 
@@ -158,6 +200,8 @@ int virtio_gpu_init(struct virtio_device *vdev, struct drm_device *dev)
 		  virtio_gpu_array_put_free_work);
 	INIT_LIST_HEAD(&vgdev->obj_free_list);
 	spin_lock_init(&vgdev->obj_free_lock);
+	INIT_LIST_HEAD(&vgdev->obj_restore_list);
+	mutex_init(&vgdev->obj_restore_lock);
 
 #ifdef __LITTLE_ENDIAN
 	if (virtio_has_feature(vgdev->vdev, VIRTIO_GPU_F_VIRGL))
@@ -215,13 +259,11 @@ int virtio_gpu_init(struct virtio_device *vdev, struct drm_device *dev)
 		 vgdev->has_context_init ? '+' : '-',
 		 vgdev->has_blob_alignment ? '+' : '-');
 
-	ret = virtio_find_vqs(vgdev->vdev, 2, vqs, vqs_info, NULL);
+	ret = virtio_gpu_find_vqs(vgdev);
 	if (ret) {
 		DRM_ERROR("failed to find virt queues\n");
 		goto err_vqs;
 	}
-	vgdev->ctrlq.vq = vqs[0];
-	vgdev->cursorq.vq = vqs[1];
 	ret = virtio_gpu_alloc_vbufs(vgdev);
 	if (ret) {
 		DRM_ERROR("failed to alloc vbufs\n");
@@ -262,11 +304,25 @@ int virtio_gpu_init(struct virtio_device *vdev, struct drm_device *dev)
 			virtio_gpu_cmd_get_edids(vgdev);
 		virtio_gpu_cmd_get_display_info(vgdev);
 		virtio_gpu_notify(vgdev);
-		wait_event_timeout(vgdev->resp_wq, !vgdev->display_info_pending,
-				   5 * HZ);
+		if (!wait_event_timeout(vgdev->resp_wq,
+					!vgdev->display_info_pending,
+					5 * HZ)) {
+			DRM_ERROR("timed out waiting for display info\n");
+			ret = -ETIMEDOUT;
+			goto err_reset_device;
+		}
 	}
+
+	vgdev->pm_nb.notifier_call = virtio_gpu_pm_notifier;
+	ret = register_pm_notifier(&vgdev->pm_nb);
+	if (ret)
+		goto err_reset_device;
+
 	return 0;
 
+err_reset_device:
+	virtio_reset_device(vgdev->vdev);
+	virtio_gpu_modeset_fini(vgdev);
 err_scanouts:
 	virtio_gpu_free_vbufs(vgdev);
 err_vbufs:
@@ -290,12 +346,14 @@ void virtio_gpu_deinit(struct drm_device *dev)
 {
 	struct virtio_gpu_device *vgdev = dev->dev_private;
 
+	unregister_pm_notifier(&vgdev->pm_nb);
 	flush_work(&vgdev->obj_free_work);
 	flush_work(&vgdev->ctrlq.dequeue_work);
 	flush_work(&vgdev->cursorq.dequeue_work);
 	flush_work(&vgdev->config_changed_work);
 	virtio_reset_device(vgdev->vdev);
 	vgdev->vdev->config->del_vqs(vgdev->vdev);
+	mutex_destroy(&vgdev->obj_restore_lock);
 }
 
 void virtio_gpu_release(struct drm_device *dev)
diff --git a/drivers/gpu/drm/virtio/virtgpu_object.c b/drivers/gpu/drm/virtio/virtgpu_object.c
index ec9efacc69195..9bc0bd68c3142 100644
--- a/drivers/gpu/drm/virtio/virtgpu_object.c
+++ b/drivers/gpu/drm/virtio/virtgpu_object.c
@@ -63,6 +63,15 @@ static void virtio_gpu_resource_id_put(struct virtio_gpu_device *vgdev, uint32_t
 		ida_free(&vgdev->resource_ida, id - 1);
 }
 
+void virtio_gpu_remove_from_restore_list(struct virtio_gpu_object *bo)
+{
+	struct virtio_gpu_device *vgdev = bo->base.base.dev->dev_private;
+
+	mutex_lock(&vgdev->obj_restore_lock);
+	list_del_init(&bo->restore_node);
+	mutex_unlock(&vgdev->obj_restore_lock);
+}
+
 void virtio_gpu_cleanup_object(struct virtio_gpu_object *bo)
 {
 	struct virtio_gpu_device *vgdev = bo->base.base.dev->dev_private;
@@ -94,7 +103,8 @@ static void virtio_gpu_free_object(struct drm_gem_object *obj)
 	struct virtio_gpu_device *vgdev = bo->base.base.dev->dev_private;
 
 	if (bo->created) {
-		virtio_gpu_cmd_unref_resource(vgdev, bo);
+		virtio_gpu_remove_from_restore_list(bo);
+		virtio_gpu_cmd_unref_resource(vgdev, bo, false);
 		virtio_gpu_notify(vgdev);
 		/* completion handler calls virtio_gpu_cleanup_object() */
 		return;
@@ -220,6 +230,8 @@ int virtio_gpu_object_create(struct virtio_gpu_device *vgdev,
 		return PTR_ERR(shmem_obj);
 	bo = gem_to_virtio_gpu_obj(&shmem_obj->base);
 
+	INIT_LIST_HEAD(&bo->restore_node);
+
 	ret = virtio_gpu_resource_id_get(vgdev, &bo->hw_res_handle);
 	if (ret < 0)
 		goto err_free_gem;
@@ -258,6 +270,12 @@ int virtio_gpu_object_create(struct virtio_gpu_device *vgdev,
 		virtio_gpu_object_attach(vgdev, bo, ents, nents);
 	}
 
+	if (!params->virgl) {
+		/* store non-virgl object with its param to the restore list */
+		bo->params = *params;
+		virtio_gpu_add_object_to_restore_list(vgdev, bo);
+	}
+
 	*bo_ptr = bo;
 	return 0;
 
@@ -271,3 +289,73 @@ int virtio_gpu_object_create(struct virtio_gpu_device *vgdev,
 	drm_gem_shmem_free(shmem_obj);
 	return ret;
 }
+
+void virtio_gpu_add_object_to_restore_list(struct virtio_gpu_device *vgdev,
+					   struct virtio_gpu_object *bo)
+{
+	mutex_lock(&vgdev->obj_restore_lock);
+	list_add_tail(&bo->restore_node, &vgdev->obj_restore_list);
+	mutex_unlock(&vgdev->obj_restore_lock);
+}
+
+int virtio_gpu_object_restore_all(struct virtio_gpu_device *vgdev)
+{
+	struct virtio_gpu_object *bo, *tmp;
+	struct virtio_gpu_mem_entry *ents;
+	unsigned int nents;
+	int ret = 0;
+
+	mutex_lock(&vgdev->obj_restore_lock);
+	list_for_each_entry_safe(bo, tmp, &vgdev->obj_restore_list,
+				 restore_node) {
+		if (drm_gem_is_imported(&bo->base.base)) {
+			ret = virtgpu_dma_buf_obj_resubmit(vgdev, bo);
+			if (ret)
+				break;
+
+			continue;
+		}
+
+		if (bo->params.blob || bo->attached) {
+			ret = virtio_gpu_object_shmem_init(vgdev, bo, &ents,
+							   &nents);
+			if (ret)
+				break;
+		}
+
+		if (bo->params.blob) {
+			virtio_gpu_cmd_resource_create_blob(vgdev, bo,
+							    &bo->params,
+							    ents, nents);
+		} else {
+			virtio_gpu_cmd_create_resource(vgdev, bo, &bo->params,
+						       NULL, NULL);
+			if (bo->attached) {
+				bo->attached = false;
+				virtio_gpu_object_attach(vgdev, bo, ents,
+							 nents);
+			}
+		}
+	}
+	mutex_unlock(&vgdev->obj_restore_lock);
+
+	if (ret)
+		DRM_ERROR("failed to restore virtio-gpu objects: %d\n", ret);
+
+	return ret;
+}
+
+void virtio_gpu_object_unref_all(struct virtio_gpu_device *vgdev)
+{
+	struct virtio_gpu_object *bo, *tmp;
+
+	mutex_lock(&vgdev->obj_restore_lock);
+	list_for_each_entry_safe(bo, tmp, &vgdev->obj_restore_list,
+				 restore_node)
+		if (bo->created) {
+			virtio_gpu_cmd_unref_resource(vgdev, bo, true);
+			virtio_gpu_notify(vgdev);
+		}
+
+	mutex_unlock(&vgdev->obj_restore_lock);
+}
diff --git a/drivers/gpu/drm/virtio/virtgpu_prime.c b/drivers/gpu/drm/virtio/virtgpu_prime.c
index 70b3b836e1c99..216c77cd0d21b 100644
--- a/drivers/gpu/drm/virtio/virtgpu_prime.c
+++ b/drivers/gpu/drm/virtio/virtgpu_prime.c
@@ -23,6 +23,7 @@
  */
 
 #include <drm/drm_prime.h>
+#include <drm/drm_print.h>
 #include <linux/virtio_dma_buf.h>
 
 #include "virtgpu_drv.h"
@@ -216,7 +217,8 @@ static void virtgpu_dma_buf_free_obj(struct drm_gem_object *obj)
 	}
 
 	if (bo->created) {
-		virtio_gpu_cmd_unref_resource(vgdev, bo);
+		virtio_gpu_remove_from_restore_list(bo);
+		virtio_gpu_cmd_unref_resource(vgdev, bo, false);
 		virtio_gpu_notify(vgdev);
 		return;
 	}
@@ -262,6 +264,13 @@ static int virtgpu_dma_buf_init_obj(struct drm_device *dev,
 	dma_buf_unpin(attach);
 	dma_resv_unlock(resv);
 
+	/*
+	 * Store the dmabuf imported object with its params to the
+	 * restore list.
+	 */
+	bo->params = params;
+	virtio_gpu_add_object_to_restore_list(vgdev, bo);
+
 	return 0;
 
 err_import:
@@ -272,6 +281,38 @@ static int virtgpu_dma_buf_init_obj(struct drm_device *dev,
 	return ret;
 }
 
+int virtgpu_dma_buf_obj_resubmit(struct virtio_gpu_device *vgdev,
+				 struct virtio_gpu_object *bo)
+{
+	struct virtio_gpu_mem_entry *ents;
+	struct scatterlist *sl;
+	int i;
+
+	if (!bo->sgt) {
+		DRM_ERROR("no sgt bound to virtio_gpu_object\n");
+		return -ENOMEM;
+	}
+
+	ents = kvmalloc_array(bo->sgt->nents,
+			      sizeof(struct virtio_gpu_mem_entry),
+			      GFP_KERNEL);
+	if (!ents) {
+		DRM_ERROR("failed to allocate ent list\n");
+		return -ENOMEM;
+	}
+
+	for_each_sgtable_dma_sg(bo->sgt, sl, i) {
+		ents[i].addr = cpu_to_le64(sg_dma_address(sl));
+		ents[i].length = cpu_to_le32(sg_dma_len(sl));
+		ents[i].padding = 0;
+	}
+
+	virtio_gpu_cmd_resource_create_blob(vgdev, bo, &bo->params,
+					    ents, bo->sgt->nents);
+
+	return 0;
+}
+
 static const struct drm_gem_object_funcs virtgpu_gem_dma_buf_funcs = {
 	.free = virtgpu_dma_buf_free_obj,
 };
@@ -317,6 +358,8 @@ struct drm_gem_object *virtgpu_gem_prime_import(struct drm_device *dev,
 	if (!bo)
 		return ERR_PTR(-ENOMEM);
 
+	INIT_LIST_HEAD(&bo->restore_node);
+
 	obj = &bo->base.base;
 	obj->resv = buf->resv;
 	obj->funcs = &virtgpu_gem_dma_buf_funcs;
diff --git a/drivers/gpu/drm/virtio/virtgpu_vq.c b/drivers/gpu/drm/virtio/virtgpu_vq.c
index c8b9475a7472e..f2b0ab0f610ee 100644
--- a/drivers/gpu/drm/virtio/virtgpu_vq.c
+++ b/drivers/gpu/drm/virtio/virtgpu_vq.c
@@ -369,6 +369,20 @@ static int virtio_gpu_panic_queue_ctrl_sgs(struct virtio_gpu_device *vgdev,
 	return 0;
 }
 
+int virtio_gpu_wait_queue(struct virtio_gpu_queue *vgvq, unsigned int num_elem)
+{
+	int ret;
+
+	/* Wait up to 5 seconds for enough free slots to become available */
+	ret = wait_event_timeout(vgvq->ack_queue,
+				 vgvq->vq->num_free >= num_elem,
+				 5 * HZ);
+	if (ret == 0)
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
 static int virtio_gpu_queue_ctrl_sgs(struct virtio_gpu_device *vgdev,
 				     struct virtio_gpu_vbuffer *vbuf,
 				     struct virtio_gpu_fence *fence,
@@ -396,7 +410,19 @@ static int virtio_gpu_queue_ctrl_sgs(struct virtio_gpu_device *vgdev,
 	if (vq->num_free < elemcnt) {
 		spin_unlock(&vgdev->ctrlq.qlock);
 		virtio_gpu_notify(vgdev);
-		wait_event(vgdev->ctrlq.ack_queue, vq->num_free >= elemcnt);
+		wait_event(vgdev->ctrlq.ack_queue,
+			   vq->num_free >= elemcnt || vgdev->vqs_released);
+		/*
+		 * Set by virtio_gpu_release_vqs() to unblock
+		 * synchronize_srcu() wait in drm_dev_unplug().
+		 */
+		if (vgdev->vqs_released) {
+			if (fence && vbuf->objs)
+				virtio_gpu_array_unlock_resv(vbuf->objs);
+			free_vbuf(vgdev, vbuf);
+			drm_dev_exit(idx);
+			return -ENODEV;
+		}
 		goto again;
 	}
 
@@ -566,7 +592,14 @@ static void virtio_gpu_queue_cursor(struct virtio_gpu_device *vgdev,
 	ret = virtqueue_add_sgs(vq, sgs, outcnt, 0, vbuf, GFP_ATOMIC);
 	if (ret == -ENOSPC) {
 		spin_unlock(&vgdev->cursorq.qlock);
-		wait_event(vgdev->cursorq.ack_queue, vq->num_free >= outcnt);
+		wait_event(vgdev->cursorq.ack_queue,
+			   vq->num_free >= outcnt || vgdev->vqs_released);
+		/* See comment in virtio_gpu_queue_ctrl_sgs(). */
+		if (vgdev->vqs_released) {
+			free_vbuf(vgdev, vbuf);
+			drm_dev_exit(idx);
+			return;
+		}
 		spin_lock(&vgdev->cursorq.qlock);
 		goto retry;
 	} else {
@@ -626,14 +659,21 @@ static void virtio_gpu_cmd_unref_cb(struct virtio_gpu_device *vgdev,
 }
 
 void virtio_gpu_cmd_unref_resource(struct virtio_gpu_device *vgdev,
-				   struct virtio_gpu_object *bo)
+				   struct virtio_gpu_object *bo,
+				   bool no_cb)
 {
 	struct virtio_gpu_resource_unref *cmd_p;
 	struct virtio_gpu_vbuffer *vbuf;
 	int ret;
 
-	cmd_p = virtio_gpu_alloc_cmd_cb(vgdev, &vbuf, sizeof(*cmd_p),
-					virtio_gpu_cmd_unref_cb);
+	if (no_cb) {
+		cmd_p = virtio_gpu_alloc_cmd_cb(vgdev, &vbuf, sizeof(*cmd_p),
+						NULL);
+	} else {
+		cmd_p = virtio_gpu_alloc_cmd_cb(vgdev, &vbuf, sizeof(*cmd_p),
+						virtio_gpu_cmd_unref_cb);
+	}
+
 	memset(cmd_p, 0, sizeof(*cmd_p));
 
 	cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_RESOURCE_UNREF);
@@ -840,9 +880,6 @@ static void virtio_gpu_cmd_get_display_info_cb(struct virtio_gpu_device *vgdev,
 	vgdev->display_info_pending = false;
 	spin_unlock(&vgdev->display_info_lock);
 	wake_up(&vgdev->resp_wq);
-
-	if (!drm_helper_hpd_irq_event(vgdev->ddev))
-		drm_kms_helper_hotplug_event(vgdev->ddev);
 }
 
 static void virtio_gpu_cmd_get_capset_info_cb(struct virtio_gpu_device *vgdev,
diff --git a/drivers/gpu/drm/virtio/virtgpu_vram.c b/drivers/gpu/drm/virtio/virtgpu_vram.c
index 4ae3cbc35dd36..5b4a3ab81cd5b 100644
--- a/drivers/gpu/drm/virtio/virtgpu_vram.c
+++ b/drivers/gpu/drm/virtio/virtgpu_vram.c
@@ -20,7 +20,8 @@ static void virtio_gpu_vram_free(struct drm_gem_object *obj)
 		if (unmap)
 			virtio_gpu_cmd_unmap(vgdev, bo);
 
-		virtio_gpu_cmd_unref_resource(vgdev, bo);
+		virtio_gpu_remove_from_restore_list(bo);
+		virtio_gpu_cmd_unref_resource(vgdev, bo, false);
 		virtio_gpu_notify(vgdev);
 		return;
 	}
@@ -55,7 +56,7 @@ static int virtio_gpu_vram_mmap(struct drm_gem_object *obj,
 
 	vma->vm_pgoff -= drm_vma_node_start(&obj->vma_node);
 	vm_flags_set(vma, VM_MIXEDMAP | VM_DONTEXPAND);
-	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+	vma->vm_page_prot = vma_get_page_prot(vma);
 	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
 	vma->vm_ops = &virtio_gpu_vram_vm_ops;
 
@@ -207,6 +208,8 @@ int virtio_gpu_vram_create(struct virtio_gpu_device *vgdev,
 	obj = &vram->base.base.base;
 	obj->funcs = &virtio_gpu_vram_funcs;
 
+	INIT_LIST_HEAD(&vram->base.restore_node);
+
 	params->size = PAGE_ALIGN(params->size);
 	drm_gem_private_object_init(vgdev->ddev, obj, params->size);
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c
index 45561bc1c9eff..a9fd4015a0caa 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c
@@ -481,7 +481,7 @@ vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf)
 	if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE)
 		prot = vm_get_page_prot(vma->vm_flags & ~VM_SHARED);
 	else
-		prot = vm_get_page_prot(vma->vm_flags);
+		prot = vma_get_page_prot(vma);
 
 	ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault);
 	if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
diff --git a/drivers/gpu/drm/xe/.gitignore b/drivers/gpu/drm/xe/.gitignore
index 8778bf132674d..6dad8a5a21355 100644
--- a/drivers/gpu/drm/xe/.gitignore
+++ b/drivers/gpu/drm/xe/.gitignore
@@ -2,3 +2,4 @@
 *.hdrtest
 /generated
 /xe_gen_wa_oob
+!.kunitconfig-display
diff --git a/drivers/gpu/drm/xe/.kunitconfig-display b/drivers/gpu/drm/xe/.kunitconfig-display
new file mode 100644
index 0000000000000..17020aa4ded3a
--- /dev/null
+++ b/drivers/gpu/drm/xe/.kunitconfig-display
@@ -0,0 +1,11 @@
+CONFIG_EXPERT=y
+CONFIG_MODULES=y
+CONFIG_KUNIT=y
+CONFIG_PCI=y
+CONFIG_DEBUG_FS=y
+CONFIG_BLK_DEV_INITRD=y
+CONFIG_MAGIC_SYSRQ=y
+CONFIG_DRM=m
+CONFIG_DRM_XE=m
+CONFIG_DRM_XE_DISPLAY=y
+CONFIG_DRM_XE_KUNIT_TEST=m
diff --git a/drivers/gpu/drm/xe/Kconfig.profile b/drivers/gpu/drm/xe/Kconfig.profile
index 7530df998148f..e07517d120e09 100644
--- a/drivers/gpu/drm/xe/Kconfig.profile
+++ b/drivers/gpu/drm/xe/Kconfig.profile
@@ -1,50 +1,71 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config DRM_XE_JOB_TIMEOUT_MAX
-	int "Default max job timeout (ms)"
+	int "Hard upper limit for job timeout (ms)"
 	default 10000 # milliseconds
 	help
-	  Configures the default max job timeout after which job will
-	  be forcefully taken away from scheduler.
+	  Absolute upper bound (in milliseconds) for the per-engine-class job
+	  timeout. This is the maximum value that can be written to the sysfs
+	  job_timeout_ms knob, regardless of privileges. To raise this ceiling,
+	  increase this value and rebuild the kernel.
 config DRM_XE_JOB_TIMEOUT_MIN
-	int "Default min job timeout (ms)"
+	int "Hard lower limit for job timeout (ms)"
 	default 1 # milliseconds
 	help
-	  Configures the default min job timeout after which job will
-	  be forcefully taken away from scheduler.
+	  Absolute lower bound (in milliseconds) for the per-engine-class job
+	  timeout. This is the minimum value that can be written to the sysfs
+	  job_timeout_ms knob, regardless of privileges.
+
+	  Note: the job timeout default (5000 ms) is hardcoded in the driver
+	  and is not configurable here. Use the sysfs job_timeout_ms knob at
+	  runtime to change the engine-class default.
 config DRM_XE_TIMESLICE_MAX
-	int "Default max timeslice duration (us)"
+	int "Hard upper limit for timeslice duration (us)"
 	default 10000000 # microseconds
 	help
-	  Configures the default max timeslice duration between multiple
-	  contexts by guc scheduling.
+	  Absolute upper bound (in microseconds) for the timeslice duration.
+	  This caps both the sysfs timeslice_duration_us knob and the value
+	  accepted via the DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE UAPI for
+	  processes with CAP_SYS_NICE when DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT
+	  is enabled.
 config DRM_XE_TIMESLICE_MIN
-	int "Default min timeslice duration (us)"
+	int "Hard lower limit for timeslice duration (us)"
 	default 1 # microseconds
 	help
-	  Configures the default min timeslice duration between multiple
-	  contexts by guc scheduling.
+	  Absolute lower bound (in microseconds) for the timeslice duration.
+	  This caps both the sysfs timeslice_duration_us knob and the value
+	  accepted via the DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE UAPI for
+	  processes with CAP_SYS_NICE when DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT
+	  is enabled.
 config DRM_XE_PREEMPT_TIMEOUT
-	int "Preempt timeout (us, jiffy granularity)"
+	int "Default preempt timeout (us, jiffy granularity)"
 	default 640000 # microseconds
 	help
-	  How long to wait (in microseconds) for a preemption event to occur
-	  when submitting a new context. If the current context does not hit
-	  an arbitration point and yield to HW before the timer expires, the
-	  HW will be reset to allow the more important context to execute.
+	  Initial per-engine-class preemption timeout (in microseconds). This
+	  is the value the driver programs at boot; it can be changed at
+	  runtime via the sysfs preempt_timeout_us knob.
+
+	  This is how long the driver waits for the current context to reach
+	  an arbitration point and yield the GPU voluntarily when a
+	  higher-priority context becomes runnable. If the context does not
+	  yield before the timer expires, the HW is reset to allow the
+	  higher-priority context to execute.
+
+	  The range userspace may write via sysfs is bounded by
+	  DRM_XE_PREEMPT_TIMEOUT_MIN and DRM_XE_PREEMPT_TIMEOUT_MAX.
 config DRM_XE_PREEMPT_TIMEOUT_MAX
-	int "Default max preempt timeout (us)"
+	int "Hard upper limit for preempt timeout (us)"
 	default 10000000 # microseconds
 	help
-	  Configures the default max preempt timeout after which context
-	  will be forcefully taken away and higher priority context will
-	  run.
+	  Absolute upper bound (in microseconds) for the per-engine-class
+	  preemption timeout. This is the maximum value that can be written to
+	  the sysfs preempt_timeout_us knob, regardless of privileges.
 config DRM_XE_PREEMPT_TIMEOUT_MIN
-	int "Default min preempt timeout (us)"
+	int "Hard lower limit for preempt timeout (us)"
 	default 1 # microseconds
 	help
-	  Configures the default min preempt timeout after which context
-	  will be forcefully taken away and higher priority context will
-	  run.
+	  Absolute lower bound (in microseconds) for the per-engine-class
+	  preemption timeout. This is the minimum value that can be written to
+	  the sysfs preempt_timeout_us knob, regardless of privileges.
 config DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT
 	bool "Default configuration of limitation on scheduler timeout"
 	default y
diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile
index 8e7b146880f46..67ada1d6c2fb9 100644
--- a/drivers/gpu/drm/xe/Makefile
+++ b/drivers/gpu/drm/xe/Makefile
@@ -101,6 +101,7 @@ xe-y += xe_bb.o \
 	xe_page_reclaim.o \
 	xe_pat.o \
 	xe_pci.o \
+	xe_pci_error.o \
 	xe_pci_rebar.o \
 	xe_pcode.o \
 	xe_pm.o \
@@ -278,6 +279,7 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \
 	i915-display/intel_dp_aux.o \
 	i915-display/intel_dp_aux_backlight.o \
 	i915-display/intel_dp_hdcp.o \
+	i915-display/intel_dp_link_caps.o \
 	i915-display/intel_dp_link_training.o \
 	i915-display/intel_dp_mst.o \
 	i915-display/intel_dp_test.o \
@@ -372,6 +374,9 @@ xe-$(CONFIG_DRM_XE_DP_TUNNEL) += \
 
 obj-$(CONFIG_DRM_XE) += xe.o
 obj-$(CONFIG_DRM_XE_KUNIT_TEST) += tests/
+ifeq ($(CONFIG_DRM_XE_DISPLAY),y)
+	obj-$(CONFIG_DRM_XE_KUNIT_TEST) += display/tests/
+endif
 
 # header test
 hdrtest_find_args := -not -path xe_rtp_helpers.h
diff --git a/drivers/gpu/drm/xe/abi/guc_actions_abi.h b/drivers/gpu/drm/xe/abi/guc_actions_abi.h
index 83a6e7794982e..f5c9b37038d46 100644
--- a/drivers/gpu/drm/xe/abi/guc_actions_abi.h
+++ b/drivers/gpu/drm/xe/abi/guc_actions_abi.h
@@ -152,6 +152,7 @@ enum xe_guc_action {
 	XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC = 0x6002,
 	XE_GUC_ACTION_PAGE_FAULT_RES_DESC = 0x6003,
 	XE_GUC_ACTION_ACCESS_COUNTER_NOTIFY = 0x6004,
+	XE_GUC_ACTION_NOTIFY_UNCORRECTABLE_LOCAL_ERROR = 0x6005,
 	XE_GUC_ACTION_TLB_INVALIDATION = 0x7000,
 	XE_GUC_ACTION_TLB_INVALIDATION_DONE = 0x7001,
 	XE_GUC_ACTION_TLB_INVALIDATION_ALL = 0x7002,
diff --git a/drivers/gpu/drm/xe/abi/guc_capture_abi.h b/drivers/gpu/drm/xe/abi/guc_capture_abi.h
index dd41175537396..ff9c0ae34a288 100644
--- a/drivers/gpu/drm/xe/abi/guc_capture_abi.h
+++ b/drivers/gpu/drm/xe/abi/guc_capture_abi.h
@@ -32,9 +32,10 @@ enum guc_capture_list_class_type {
 	GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE = 2,
 	GUC_CAPTURE_LIST_CLASS_BLITTER = 3,
 	GUC_CAPTURE_LIST_CLASS_GSC_OTHER = 4,
+	GUC_CAPTURE_LIST_CLASS_PAGING = 5,
 };
 
-#define GUC_CAPTURE_LIST_CLASS_MAX	(GUC_CAPTURE_LIST_CLASS_GSC_OTHER + 1)
+#define GUC_CAPTURE_LIST_CLASS_MAX	(GUC_CAPTURE_LIST_CLASS_PAGING + 1)
 
 /**
  * struct guc_mmio_reg - GuC MMIO reg state struct
diff --git a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h
index 644f5a4226d7e..685c4ef17b732 100644
--- a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h
+++ b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h
@@ -22,6 +22,7 @@
  *  |   |       |   - `GuC Scheduling Policies KLVs`_                          |
  *  |   |       |   - `GuC VGT Policy KLVs`_                                   |
  *  |   |       |   - `GuC VF Configuration KLVs`_                             |
+ *  |   |       |   - `GuC Reserved KLVs`_                                     |
  *  |   |       |                                                              |
  *  |   +-------+--------------------------------------------------------------+
  *  |   |  15:0 | **LEN** - length of VALUE (in 32bit dwords)                  |
@@ -52,6 +53,12 @@
  * _`GUC_KLV_GLOBAL_CFG_GROUP_SCHEDULING_AVAILABLE` : 0x3001
  *      Tells the driver whether scheduler groups are enabled or not.
  *      Requires GuC ABI 1.26+
+ *
+ * _`GUC_KLV_GLOBAL_CFG_NUM_PAGING_ENGINE_INSTANCES` : 0x3003
+ *      Tells the driver the paging engine configuration.
+ *      Paging engine logical instances are guaranteed to be dense starting at
+ *      index 0.
+ *      Requires GuC ABI 1.36+
  */
 
 #define GUC_KLV_GLOBAL_CFG_GMD_ID_KEY			0x3000u
@@ -60,6 +67,9 @@
 #define GUC_KLV_GLOBAL_CFG_GROUP_SCHEDULING_AVAILABLE_KEY	0x3001u
 #define GUC_KLV_GLOBAL_CFG_GROUP_SCHEDULING_AVAILABLE_LEN	1u
 
+#define GUC_KLV_GLOBAL_CFG_NUM_PAGING_ENGINE_INSTANCES_KEY	0x3003u
+#define GUC_KLV_GLOBAL_CFG_NUM_PAGING_ENGINE_INSTANCES_LEN	1u
+
 /**
  * DOC: GuC Self Config KLVs
  *
@@ -154,6 +164,11 @@ enum  {
  *      (instead of waiting the full timeslice duration). The bit is instead set
  *      to one if a single context is queued on the engine, to avoid it being
  *      switched out if there isn't another context that can run in its place.
+ *
+ * _`GUC_KLV_OPT_IN_FEATURE_UNCORRECTABLE_LOCAL_ERROR_NOTIFICATION` : 0x4004
+ *      This flag will enable notification from GuC to KMD via G2H message
+ *      GUC_ACTION_GUC2HOST_NOTIFY_UNCORRECTABLE_LOCAL_ERROR upon receiving the
+ *      same interrupt from the CS.
  */
 
 #define GUC_KLV_OPT_IN_FEATURE_EXT_CAT_ERR_TYPE_KEY 0x4001
@@ -162,6 +177,9 @@ enum  {
 #define GUC_KLV_OPT_IN_FEATURE_DYNAMIC_INHIBIT_CONTEXT_SWITCH_KEY 0x4003
 #define GUC_KLV_OPT_IN_FEATURE_DYNAMIC_INHIBIT_CONTEXT_SWITCH_LEN 0u
 
+#define GUC_KLV_OPT_IN_FEATURE_UNCORRECTABLE_LOCAL_ERROR_NOTIFICATION_KEY 0x4004
+#define GUC_KLV_OPT_IN_FEATURE_UNCORRECTABLE_LOCAL_ERROR_NOTIFICATION_LEN 0u
+
 /**
  * DOC: GuC Scheduling Policies KLVs
  *
@@ -491,9 +509,10 @@ enum  {
 #define GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_MIN_LEN	1u
 #define GUC_KLV_VF_CFG_ENGINE_GROUP_PREEMPT_TIMEOUT_MAX_LEN	GUC_MAX_SCHED_GROUPS
 /*
- * Workaround keys:
+ * Feature and Workaround keys:
  */
 enum xe_guc_klv_ids {
+	GUC_FEATURE_KLV_DISABLE_MULTI_QUEUE						= 0x5001,
 	GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED				= 0x9002,
 	GUC_WORKAROUND_KLV_DISABLE_PSMI_INTERRUPTS_AT_C6_ENTRY_RESTORE_AT_EXIT		= 0x9004,
 	GUC_WORKAROUND_KLV_ID_GAM_PFQ_SHADOW_TAIL_POLLING				= 0x9005,
@@ -504,6 +523,24 @@ enum xe_guc_klv_ids {
 	GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH					= 0x900b,
 	GUC_WA_KLV_RESTORE_UNSAVED_MEDIA_CONTROL_REG					= 0x900c,
 	GUC_WA_KLV_CLR_CS_INDIRECT_RING_STATE_IF_IDLE_AT_CTX_REG			= 0x900e,
+	GUC_WA_KLV_REMAP_RANGED_TLB_INV							= 0x900f,
+	GUC_WA_KLV_IGNORE_MMIO_READ_SEM_TOKEN_64					= 0x9010,
 };
 
+/**
+ * DOC: GuC Reserved KLVs
+ *
+ * Range of `GuC KLV`_ keys reserved for internal use by the GuC that will
+ * never be part of the offcial GuC ABI and can be reused by the drivers.
+ *
+ * Currently this range includes 1024 keys starting from:
+ *
+ * _`GUC_KLV_RESERVED_RANGE_START` : 0xF000
+ *
+ * See `Xe Driver KLVs`_ for the KLVs that the Xe driver is currently using.
+ */
+
+#define GUC_KLV_RESERVED_RANGE_START	0xf000u
+#define GUC_KLV_RESERVED_RANGE_LEN	1024u
+
 #endif
diff --git a/drivers/gpu/drm/xe/abi/guc_scheduler_abi.h b/drivers/gpu/drm/xe/abi/guc_scheduler_abi.h
index 19ec89bf39c53..85bba34277ed5 100644
--- a/drivers/gpu/drm/xe/abi/guc_scheduler_abi.h
+++ b/drivers/gpu/drm/xe/abi/guc_scheduler_abi.h
@@ -21,7 +21,8 @@
 #define GUC_BLITTER_CLASS		3
 #define GUC_COMPUTE_CLASS		4
 #define GUC_GSC_OTHER_CLASS		5
-#define GUC_LAST_ENGINE_CLASS		GUC_GSC_OTHER_CLASS
+#define GUC_PAGING_CLASS		6
+#define GUC_LAST_ENGINE_CLASS		GUC_PAGING_CLASS
 #define GUC_MAX_ENGINE_CLASSES		16
 #define GUC_MAX_INSTANCES_PER_CLASS	32
 
diff --git a/drivers/gpu/drm/xe/abi/xe_driver_klvs_abi.h b/drivers/gpu/drm/xe/abi/xe_driver_klvs_abi.h
new file mode 100644
index 0000000000000..3b557e56892a5
--- /dev/null
+++ b/drivers/gpu/drm/xe/abi/xe_driver_klvs_abi.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _ABI_XE_DRIVER_KLVS_ABI_H
+#define _ABI_XE_DRIVER_KLVS_ABI_H
+
+#include "abi/guc_klvs_abi.h"
+
+/**
+ * DOC: Xe Driver KLVs
+ *
+ * The Xe driver uses the following keys from the `GuC Reserved KLVs`_ range:
+ *
+ * _`MIGRATION_KLV_DEVICE_DEVID_KEY` :
+ *      PCI device ID of the migrated VF.
+ * _`MIGRATION_KLV_DEVICE_REVID_KEY` :
+ *      PCI device revision ID of the migrated VF.
+ */
+
+#define MIGRATION_KLV_DEVICE_DEVID_KEY		0xf001u
+#define MIGRATION_KLV_DEVICE_DEVID_LEN		1u
+#define MIGRATION_KLV_DEVICE_REVID_KEY		0xf002u
+#define MIGRATION_KLV_DEVICE_REVID_LEN		1u
+
+#endif
diff --git a/drivers/gpu/drm/xe/display/tests/Makefile b/drivers/gpu/drm/xe/display/tests/Makefile
new file mode 100644
index 0000000000000..c716864580721
--- /dev/null
+++ b/drivers/gpu/drm/xe/display/tests/Makefile
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-2.0
+
+subdir-ccflags-$(CONFIG_DRM_XE_DISPLAY) += \
+	-I$(srctree)/drivers/gpu/drm/i915/display/
+
+# Rule to build display code shared with i915
+$(obj)/i915-display/tests/%.o: $(srctree)/drivers/gpu/drm/i915/display/tests/%.c FORCE
+	$(call cmd,force_checksrc)
+	$(call if_changed_rule,cc_o_c)
+
+obj-$(CONFIG_DRM_XE_KUNIT_TEST) += xe_display_test.o
+xe_display_test-y = \
+		i915-display/tests/intel_dp_link_test.o
diff --git a/drivers/gpu/drm/xe/display/xe_display.c b/drivers/gpu/drm/xe/display/xe_display.c
index 810d93fefcbcc..8da4c457a4a0f 100644
--- a/drivers/gpu/drm/xe/display/xe_display.c
+++ b/drivers/gpu/drm/xe/display/xe_display.c
@@ -19,7 +19,6 @@
 
 #include "intel_acpi.h"
 #include "intel_audio.h"
-#include "intel_bw.h"
 #include "intel_display.h"
 #include "intel_display_core.h"
 #include "intel_display_device.h"
@@ -29,8 +28,6 @@
 #include "intel_dmc.h"
 #include "intel_dmc_wl.h"
 #include "intel_dp.h"
-#include "intel_dram.h"
-#include "intel_encoder.h"
 #include "intel_fbdev.h"
 #include "intel_hdcp.h"
 #include "intel_hotplug.h"
@@ -84,10 +81,8 @@ static void xe_display_fini_early(void *arg)
 	if (!xe->info.probe_display)
 		return;
 
-	intel_hpd_cancel_work(display);
-	intel_display_driver_remove_nogem(display);
 	intel_display_driver_remove_noirq(display);
-	intel_opregion_cleanup(display);
+	intel_display_driver_remove_nogem(display);
 	intel_display_power_cleanup(display);
 }
 
@@ -113,22 +108,9 @@ int xe_display_init_early(struct xe_device *xe)
 		return 0;
 	}
 
-	/* Early display init.. */
-	intel_opregion_setup(display);
-
-	/*
-	 * Fill the dram structure to get the system dram info. This will be
-	 * used for memory latency calculation.
-	 */
-	err = intel_dram_detect(display);
-	if (err)
-		goto err_opregion;
-
-	intel_bw_init_hw(display);
-
 	err = intel_display_driver_probe_noirq(display);
 	if (err)
-		goto err_opregion;
+		return err;
 
 	err = intel_display_driver_probe_nogem(display);
 	if (err)
@@ -138,8 +120,7 @@ int xe_display_init_early(struct xe_device *xe)
 err_noirq:
 	intel_display_driver_remove_noirq(display);
 	intel_display_power_cleanup(display);
-err_opregion:
-	intel_opregion_cleanup(display);
+
 	return err;
 }
 
@@ -177,7 +158,7 @@ void xe_display_register(struct xe_device *xe)
 		return;
 
 	intel_display_driver_register(display);
-	intel_display_power_enable(display);
+	intel_display_driver_runtime_pm_enable(display);
 }
 
 void xe_display_unregister(struct xe_device *xe)
@@ -187,10 +168,34 @@ void xe_display_unregister(struct xe_device *xe)
 	if (!xe->info.probe_display)
 		return;
 
-	intel_display_power_disable(display);
+	intel_display_driver_runtime_pm_disable(display);
 	intel_display_driver_unregister(display);
 }
 
+void xe_display_shutdown(struct xe_device *xe)
+{
+	struct intel_display *display = xe->display;
+
+	if (!xe->info.probe_display)
+		return;
+
+	intel_display_driver_shutdown(display);
+
+	intel_opregion_suspend(display, PCI_D3cold);
+
+	intel_dmc_suspend(display);
+}
+
+void xe_display_shutdown_late(struct xe_device *xe)
+{
+	struct intel_display *display = xe->display;
+
+	if (!xe->info.probe_display)
+		return;
+
+	intel_display_driver_shutdown_late(display);
+}
+
 /* IRQ-related functions */
 
 void xe_display_irq_handler(struct xe_device *xe, u32 master_ctl)
@@ -244,57 +249,62 @@ static bool suspend_to_idle(void)
 	return false;
 }
 
-static void xe_display_enable_d3cold(struct xe_device *xe)
+void xe_display_pm_suspend(struct xe_device *xe)
 {
 	struct intel_display *display = xe->display;
+	bool s2idle = suspend_to_idle();
 
 	if (!xe->info.probe_display)
 		return;
 
-	/*
-	 * We do a lot of poking in a lot of registers, make sure they work
-	 * properly.
-	 */
-	intel_display_power_disable(display);
-
-	intel_display_flush_cleanup_work(display);
+	intel_display_driver_pm_suspend(display);
 
-	intel_opregion_suspend(display, PCI_D3cold);
+	intel_opregion_suspend(display, s2idle ? PCI_D1 : PCI_D3cold);
 
 	intel_dmc_suspend(display);
+}
 
-	if (intel_display_device_present(display))
-		intel_hpd_poll_enable(display);
+void xe_display_pm_suspend_late(struct xe_device *xe)
+{
+	struct intel_display *display = xe->display;
+	bool s2idle = suspend_to_idle();
+
+	if (!xe->info.probe_display)
+		return;
+
+	intel_display_driver_pm_suspend_late(display, s2idle);
 }
 
-static void xe_display_disable_d3cold(struct xe_device *xe)
+void xe_display_pm_resume_early(struct xe_device *xe)
 {
 	struct intel_display *display = xe->display;
 
 	if (!xe->info.probe_display)
 		return;
 
-	intel_dmc_resume(display);
+	intel_display_driver_pm_resume_early(display);
+}
 
-	if (intel_display_device_present(display))
-		drm_mode_config_reset(&xe->drm);
+void xe_display_pm_resume(struct xe_device *xe)
+{
+	struct intel_display *display = xe->display;
 
-	intel_display_driver_init_hw(display);
+	if (!xe->info.probe_display)
+		return;
 
-	intel_hpd_init(display);
+	intel_dmc_resume(display);
 
 	if (intel_display_device_present(display))
-		intel_hpd_poll_disable(display);
+		drm_mode_config_reset(&xe->drm);
 
-	intel_opregion_resume(display);
+	intel_display_driver_init_hw(display);
 
-	intel_display_power_enable(display);
+	intel_display_driver_pm_resume(display);
 }
 
-void xe_display_pm_suspend(struct xe_device *xe)
+static void xe_display_enable_d3cold(struct xe_device *xe)
 {
 	struct intel_display *display = xe->display;
-	bool s2idle = suspend_to_idle();
 
 	if (!xe->info.probe_display)
 		return;
@@ -303,63 +313,43 @@ void xe_display_pm_suspend(struct xe_device *xe)
 	 * We do a lot of poking in a lot of registers, make sure they work
 	 * properly.
 	 */
-	intel_display_power_disable(display);
-	drm_client_dev_suspend(&xe->drm);
-
-	if (intel_display_device_present(display)) {
-		drm_kms_helper_poll_disable(&xe->drm);
-		intel_display_driver_disable_user_access(display);
-		intel_display_driver_suspend(display);
-	}
+	intel_display_driver_runtime_pm_disable(display);
 
 	intel_display_flush_cleanup_work(display);
 
-	intel_encoder_block_all_hpds(display);
-
-	intel_hpd_cancel_work(display);
-
-	if (intel_display_device_present(display)) {
-		intel_display_driver_suspend_access(display);
-		intel_encoder_suspend_all(display);
-	}
-
-	intel_opregion_suspend(display, s2idle ? PCI_D1 : PCI_D3cold);
+	intel_opregion_suspend(display, PCI_D3cold);
 
 	intel_dmc_suspend(display);
+
+	if (intel_display_device_present(display))
+		intel_hpd_poll_enable(display);
 }
 
-void xe_display_pm_shutdown(struct xe_device *xe)
+static void xe_display_disable_d3cold(struct xe_device *xe)
 {
 	struct intel_display *display = xe->display;
 
 	if (!xe->info.probe_display)
 		return;
 
-	intel_display_power_disable(display);
-	drm_client_dev_suspend(&xe->drm);
+	intel_dmc_resume(display);
 
-	if (intel_display_device_present(display)) {
-		drm_kms_helper_poll_disable(&xe->drm);
-		intel_display_driver_disable_user_access(display);
-		intel_display_driver_suspend(display);
-	}
+	if (intel_display_device_present(display))
+		drm_mode_config_reset(&xe->drm);
 
-	intel_display_flush_cleanup_work(display);
-	intel_dp_mst_suspend(display);
-	intel_encoder_block_all_hpds(display);
-	intel_hpd_cancel_work(display);
+	intel_display_driver_init_hw(display);
 
-	if (intel_display_device_present(display))
-		intel_display_driver_suspend_access(display);
+	intel_hpd_init(display);
 
-	intel_encoder_suspend_all(display);
-	intel_encoder_shutdown_all(display);
+	if (intel_display_device_present(display))
+		intel_hpd_poll_disable(display);
 
-	intel_opregion_suspend(display, PCI_D3cold);
+	intel_opregion_resume(display);
 
-	intel_dmc_suspend(display);
+	intel_display_driver_runtime_pm_enable(display);
 }
 
+/* before irq suspend */
 void xe_display_pm_runtime_suspend(struct xe_device *xe)
 {
 	struct intel_display *display = xe->display;
@@ -372,20 +362,10 @@ void xe_display_pm_runtime_suspend(struct xe_device *xe)
 		return;
 	}
 
-	intel_hpd_poll_enable(display);
-}
-
-void xe_display_pm_suspend_late(struct xe_device *xe)
-{
-	struct intel_display *display = xe->display;
-	bool s2idle = suspend_to_idle();
-
-	if (!xe->info.probe_display)
-		return;
-
-	intel_display_power_suspend_late(display, s2idle);
+	intel_display_driver_pm_runtime_suspend(display);
 }
 
+/* after irq suspend */
 void xe_display_pm_runtime_suspend_late(struct xe_device *xe)
 {
 	struct intel_display *display = xe->display;
@@ -393,79 +373,31 @@ void xe_display_pm_runtime_suspend_late(struct xe_device *xe)
 	if (!xe->info.probe_display)
 		return;
 
-	if (xe->d3cold.allowed)
+	if (xe->d3cold.allowed) {
 		xe_display_pm_suspend_late(xe);
-
-	/*
-	 * If xe_display_pm_suspend_late() is not called, it is likely
-	 * that we will be on dynamic DC states with DMC wakelock enabled. We
-	 * need to flush the release work in that case.
-	 */
-	intel_dmc_wl_flush_release_work(display);
-}
-
-void xe_display_pm_shutdown_late(struct xe_device *xe)
-{
-	struct intel_display *display = xe->display;
-
-	if (!xe->info.probe_display)
+		/* Ensure the wakelock release work gets flushed */
+		intel_dmc_wl_flush_release_work(display);
 		return;
+	}
 
-	/*
-	 * The only requirement is to reboot with display DC states disabled,
-	 * for now leaving all display power wells in the INIT power domain
-	 * enabled.
-	 */
-	intel_display_power_driver_remove(display);
+	intel_display_driver_pm_runtime_suspend_late(display);
 }
 
-void xe_display_pm_resume_early(struct xe_device *xe)
+/* before irq resume */
+void xe_display_pm_runtime_resume_early(struct xe_device *xe)
 {
 	struct intel_display *display = xe->display;
 
 	if (!xe->info.probe_display)
 		return;
 
-	intel_display_power_resume_early(display);
-}
-
-void xe_display_pm_resume(struct xe_device *xe)
-{
-	struct intel_display *display = xe->display;
-
-	if (!xe->info.probe_display)
+	if (xe->d3cold.allowed)
 		return;
 
-	intel_dmc_resume(display);
-
-	if (intel_display_device_present(display))
-		drm_mode_config_reset(&xe->drm);
-
-	intel_display_driver_init_hw(display);
-
-	if (intel_display_device_present(display))
-		intel_display_driver_resume_access(display);
-
-	intel_hpd_init(display);
-
-	intel_encoder_unblock_all_hpds(display);
-
-	if (intel_display_device_present(display)) {
-		intel_display_driver_resume(display);
-		drm_kms_helper_poll_enable(&xe->drm);
-		intel_display_driver_enable_user_access(display);
-	}
-
-	if (intel_display_device_present(display))
-		intel_hpd_poll_disable(display);
-
-	intel_opregion_resume(display);
-
-	drm_client_dev_resume(&xe->drm);
-
-	intel_display_power_enable(display);
+	intel_display_driver_pm_runtime_resume_early(display);
 }
 
+/* after irq resume */
 void xe_display_pm_runtime_resume(struct xe_device *xe)
 {
 	struct intel_display *display = xe->display;
@@ -478,9 +410,7 @@ void xe_display_pm_runtime_resume(struct xe_device *xe)
 		return;
 	}
 
-	intel_hpd_init(display);
-	intel_hpd_poll_disable(display);
-	skl_watermark_ipc_update(display);
+	intel_display_driver_pm_runtime_resume(display);
 }
 
 
diff --git a/drivers/gpu/drm/xe/display/xe_display.h b/drivers/gpu/drm/xe/display/xe_display.h
index 60291cb154df7..0babb50bfc777 100644
--- a/drivers/gpu/drm/xe/display/xe_display.h
+++ b/drivers/gpu/drm/xe/display/xe_display.h
@@ -29,19 +29,21 @@ int xe_display_init(struct xe_device *xe);
 void xe_display_register(struct xe_device *xe);
 void xe_display_unregister(struct xe_device *xe);
 
+void xe_display_shutdown(struct xe_device *xe);
+void xe_display_shutdown_late(struct xe_device *xe);
+
 void xe_display_irq_handler(struct xe_device *xe, u32 master_ctl);
 void xe_display_irq_enable(struct xe_device *xe, u32 gu_misc_iir);
 void xe_display_irq_reset(struct xe_device *xe);
 void xe_display_irq_postinstall(struct xe_device *xe);
 
 void xe_display_pm_suspend(struct xe_device *xe);
-void xe_display_pm_shutdown(struct xe_device *xe);
 void xe_display_pm_suspend_late(struct xe_device *xe);
-void xe_display_pm_shutdown_late(struct xe_device *xe);
 void xe_display_pm_resume_early(struct xe_device *xe);
 void xe_display_pm_resume(struct xe_device *xe);
 void xe_display_pm_runtime_suspend(struct xe_device *xe);
 void xe_display_pm_runtime_suspend_late(struct xe_device *xe);
+void xe_display_pm_runtime_resume_early(struct xe_device *xe);
 void xe_display_pm_runtime_resume(struct xe_device *xe);
 
 #define XE_DISPLAY_DRIVER_FEATURES	(DRIVER_MODESET | DRIVER_ATOMIC)
@@ -65,19 +67,21 @@ static inline int xe_display_init(struct xe_device *xe) { return 0; }
 static inline void xe_display_register(struct xe_device *xe) {}
 static inline void xe_display_unregister(struct xe_device *xe) {}
 
+static inline void xe_display_shutdown(struct xe_device *xe) {}
+static inline void xe_display_shutdown_late(struct xe_device *xe) {}
+
 static inline void xe_display_irq_handler(struct xe_device *xe, u32 master_ctl) {}
 static inline void xe_display_irq_enable(struct xe_device *xe, u32 gu_misc_iir) {}
 static inline void xe_display_irq_reset(struct xe_device *xe) {}
 static inline void xe_display_irq_postinstall(struct xe_device *xe) {}
 
 static inline void xe_display_pm_suspend(struct xe_device *xe) {}
-static inline void xe_display_pm_shutdown(struct xe_device *xe) {}
 static inline void xe_display_pm_suspend_late(struct xe_device *xe) {}
-static inline void xe_display_pm_shutdown_late(struct xe_device *xe) {}
 static inline void xe_display_pm_resume_early(struct xe_device *xe) {}
 static inline void xe_display_pm_resume(struct xe_device *xe) {}
 static inline void xe_display_pm_runtime_suspend(struct xe_device *xe) {}
 static inline void xe_display_pm_runtime_suspend_late(struct xe_device *xe) {}
+static inline void xe_display_pm_runtime_resume_early(struct xe_device *xe) {}
 static inline void xe_display_pm_runtime_resume(struct xe_device *xe) {}
 
 #endif /* CONFIG_DRM_XE_DISPLAY */
diff --git a/drivers/gpu/drm/xe/display/xe_display_bo.c b/drivers/gpu/drm/xe/display/xe_display_bo.c
index 8953da0136dcb..f5102cd0e95d6 100644
--- a/drivers/gpu/drm/xe/display/xe_display_bo.c
+++ b/drivers/gpu/drm/xe/display/xe_display_bo.c
@@ -147,33 +147,12 @@ static struct drm_gem_object *xe_display_bo_fbdev_create(struct drm_device *drm,
 	struct xe_device *xe = to_xe_device(drm);
 	struct xe_bo *obj;
 
-	obj = ERR_PTR(-ENODEV);
-
-	if (xe_display_bo_fbdev_prefer_stolen(xe, size)) {
-		obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe),
-						size,
-						ttm_bo_type_kernel,
-						XE_BO_FLAG_FORCE_WC |
-						XE_BO_FLAG_STOLEN |
-						XE_BO_FLAG_GGTT,
-						false);
-		if (!IS_ERR(obj))
-			drm_info(&xe->drm, "Allocated fbdev into stolen\n");
-		else
-			drm_info(&xe->drm, "Allocated fbdev into stolen failed: %li\n", PTR_ERR(obj));
-	} else {
-		drm_info(&xe->drm, "Allocating fbdev: Stolen memory not preferred.\n");
-	}
-
-	if (IS_ERR(obj)) {
-		obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), size,
-						ttm_bo_type_kernel,
-						XE_BO_FLAG_FORCE_WC |
-						XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) |
-						XE_BO_FLAG_GGTT,
-						false);
-	}
-
+	obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), size,
+					ttm_bo_type_kernel,
+					XE_BO_FLAG_FORCE_WC |
+					XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) |
+					XE_BO_FLAG_GGTT,
+					false);
 	if (IS_ERR(obj)) {
 		drm_err(&xe->drm, "failed to allocate framebuffer (%pe)\n", obj);
 		return ERR_PTR(-ENOMEM);
diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c
index 5f4a0cd8deca0..73469ea5f333d 100644
--- a/drivers/gpu/drm/xe/display/xe_fb_pin.c
+++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c
@@ -164,31 +164,14 @@ static int __xe_pin_fb_vma_dpt(struct drm_gem_object *obj,
 		dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8,
 				 XE_PAGE_SIZE);
 
-	if (IS_DGFX(xe))
-		dpt = xe_bo_create_pin_map_at_novm(xe, tile0,
-						   dpt_size, ~0ull,
-						   ttm_bo_type_kernel,
-						   XE_BO_FLAG_VRAM0 |
-						   XE_BO_FLAG_GGTT |
-						   XE_BO_FLAG_PAGETABLE,
-						   pin_params->alignment, false);
-	else
-		dpt = xe_bo_create_pin_map_at_novm(xe, tile0,
-						   dpt_size,  ~0ull,
-						   ttm_bo_type_kernel,
-						   XE_BO_FLAG_STOLEN |
-						   XE_BO_FLAG_GGTT |
-						   XE_BO_FLAG_PAGETABLE,
-						   pin_params->alignment, false);
-	if (IS_ERR(dpt))
-		dpt = xe_bo_create_pin_map_at_novm(xe, tile0,
-						   dpt_size,  ~0ull,
-						   ttm_bo_type_kernel,
-						   XE_BO_FLAG_SYSTEM |
-						   XE_BO_FLAG_GGTT |
-						   XE_BO_FLAG_PAGETABLE |
-						   XE_BO_FLAG_FORCE_WC,
-						   pin_params->alignment, false);
+	dpt = xe_bo_create_pin_map_at_novm(xe, tile0,
+					   dpt_size,  ~0ull,
+					   ttm_bo_type_kernel,
+					   XE_BO_FLAG_VRAM_IF_DGFX(tile0) |
+					   XE_BO_FLAG_GGTT |
+					   XE_BO_FLAG_PAGETABLE |
+					   XE_BO_FLAG_FORCE_WC,
+					   pin_params->alignment, false);
 	if (IS_ERR(dpt))
 		return PTR_ERR(dpt);
 
diff --git a/drivers/gpu/drm/xe/display/xe_initial_plane.c b/drivers/gpu/drm/xe/display/xe_initial_plane.c
index 02b46cc3b6df9..0f86b73036d03 100644
--- a/drivers/gpu/drm/xe/display/xe_initial_plane.c
+++ b/drivers/gpu/drm/xe/display/xe_initial_plane.c
@@ -19,8 +19,24 @@
 #include "xe_fb_pin.h"
 #include "xe_ggtt.h"
 #include "xe_mmio.h"
+#include "xe_ttm_stolen_mgr.h"
 #include "xe_vram_types.h"
 
+static bool is_pte_local(u64 pte)
+{
+	return pte & XE_GGTT_PTE_DM;
+}
+
+static bool has_lmembar(struct xe_device *xe)
+{
+	return GRAPHICS_VERx100(xe) >= 1270;
+}
+
+static bool need_pte_local(struct xe_device *xe)
+{
+	return IS_DGFX(xe) || has_lmembar(xe);
+}
+
 static struct xe_bo *
 initial_plane_bo(struct xe_device *xe,
 		 struct intel_initial_plane_config *plane_config)
@@ -37,16 +53,20 @@ initial_plane_bo(struct xe_device *xe,
 	flags = XE_BO_FLAG_FORCE_WC | XE_BO_FLAG_GGTT;
 
 	base = round_down(plane_config->base, page_size);
+	size = round_up(plane_config->base + plane_config->size,
+			page_size);
+	size -= base;
+
 	if (IS_DGFX(xe)) {
 		u64 pte = xe_ggtt_read_pte(tile0->mem.ggtt, base);
 
-		if (!(pte & XE_GGTT_PTE_DM)) {
-			drm_err(&xe->drm,
-				"Initial plane programming missing DM bit\n");
+		if (is_pte_local(pte) != need_pte_local(xe)) {
+			drm_err(&xe->drm, "Initial plane PTE has bad local memory bit\n");
 			return NULL;
 		}
 
 		phys_base = pte & ~(page_size - 1);
+
 		flags |= XE_BO_FLAG_VRAM0;
 
 		/*
@@ -60,14 +80,26 @@ initial_plane_bo(struct xe_device *xe,
 			return NULL;
 		}
 
-		drm_dbg(&xe->drm,
-			"Using phys_base=%pa, based on initial plane programming\n",
-			&phys_base);
+		drm_dbg_kms(&xe->drm,
+			    "Using phys_base=%pa, based on initial plane programming\n",
+			    &phys_base);
 	} else {
-		struct ttm_resource_manager *stolen = ttm_manager_type(&xe->ttm, XE_PL_STOLEN);
+		struct ttm_resource_manager *stolen;
+		u64 pte;
+
+		stolen = ttm_manager_type(&xe->ttm, XE_PL_STOLEN);
+		if (!stolen) {
+			drm_dbg_kms(&xe->drm, "No stolen for initial FB\n");
+			return NULL;
+		}
+
+		pte = xe_ggtt_read_pte(tile0->mem.ggtt, base);
 
-		if (!stolen)
+		if (is_pte_local(pte) != need_pte_local(xe)) {
+			drm_err(&xe->drm, "Initial plane PTE has bad local memory bit\n");
 			return NULL;
+		}
+
 		phys_base = base;
 		flags |= XE_BO_FLAG_STOLEN;
 
@@ -79,16 +111,12 @@ initial_plane_bo(struct xe_device *xe,
 		}
 	}
 
-	size = round_up(plane_config->base + plane_config->size,
-			page_size);
-	size -= base;
-
 	bo = xe_bo_create_pin_map_at_novm(xe, tile0, size, phys_base,
 					  ttm_bo_type_kernel, flags, 0, false);
 	if (IS_ERR(bo)) {
-		drm_dbg(&xe->drm,
-			"Failed to create bo phys_base=%pa size %u with flags %x: %li\n",
-			&phys_base, size, flags, PTR_ERR(bo));
+		drm_dbg_kms(&xe->drm,
+			    "Failed to create bo phys_base=%pa size %u with flags %x: %li\n",
+			    &phys_base, size, flags, PTR_ERR(bo));
 		return NULL;
 	}
 
diff --git a/drivers/gpu/drm/xe/display/xe_panic.c b/drivers/gpu/drm/xe/display/xe_panic.c
index bebb21d617f08..12c6fb99015dd 100644
--- a/drivers/gpu/drm/xe/display/xe_panic.c
+++ b/drivers/gpu/drm/xe/display/xe_panic.c
@@ -5,8 +5,6 @@
 #include <drm/drm_panic.h>
 #include <drm/intel/display_parent_interface.h>
 
-#include "intel_display_types.h"
-#include "intel_fb.h"
 #include "xe_bo.h"
 #include "xe_panic.h"
 #include "xe_res_cursor.h"
@@ -16,6 +14,9 @@ struct intel_panic {
 	struct iosys_map vmap;
 
 	int page;
+
+	struct xe_bo *bo;
+	unsigned int (*tiling)(unsigned int x, unsigned int y, unsigned int width);
 };
 
 static void xe_panic_kunmap(struct intel_panic *panic)
@@ -36,14 +37,13 @@ static void xe_panic_kunmap(struct intel_panic *panic)
 static void xe_panic_page_set_pixel(struct drm_scanout_buffer *sb, unsigned int x,
 				    unsigned int y, u32 color)
 {
-	struct intel_framebuffer *fb = (struct intel_framebuffer *)sb->private;
-	struct intel_panic *panic = fb->panic;
-	struct xe_bo *bo = gem_to_xe_bo(intel_fb_bo(&fb->base));
+	struct intel_panic *panic = sb->private;
+	struct xe_bo *bo = panic->bo;
 	unsigned int new_page;
 	unsigned int offset;
 
-	if (fb->panic_tiling)
-		offset = fb->panic_tiling(sb->width, x, y);
+	if (panic->tiling)
+		offset = panic->tiling(sb->width, x, y);
 	else
 		offset = y * sb->pitch[0] + x * sb->format->cpp[0];
 
@@ -84,16 +84,22 @@ static struct intel_panic *xe_panic_alloc(void)
 	return panic;
 }
 
-static int xe_panic_setup(struct intel_panic *panic, struct drm_scanout_buffer *sb)
+static int xe_panic_setup(struct intel_panic *panic, struct drm_scanout_buffer *sb,
+			  struct drm_gem_object *obj,
+			  unsigned int (*tiling)(unsigned int x, unsigned int y, unsigned int width))
 {
-	struct intel_framebuffer *fb = (struct intel_framebuffer *)sb->private;
-	struct xe_bo *bo = gem_to_xe_bo(intel_fb_bo(&fb->base));
+	struct xe_bo *bo = gem_to_xe_bo(obj);
 
 	if (xe_bo_is_vram(bo) && !xe_bo_is_visible_vram(bo))
 		return -ENODEV;
 
 	panic->page = -1;
+	panic->bo = bo;
+	panic->tiling = tiling;
+
+	sb->private = panic;
 	sb->set_pixel = xe_panic_page_set_pixel;
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/xe/regs/xe_pmt.h b/drivers/gpu/drm/xe/regs/xe_pmt.h
index 240d57993ea6f..a62ab05c6b4cb 100644
--- a/drivers/gpu/drm/xe/regs/xe_pmt.h
+++ b/drivers/gpu/drm/xe/regs/xe_pmt.h
@@ -15,6 +15,8 @@
 #define	ENERGY_PKG			REG_GENMASK64(31, 0)
 #define	ENERGY_CARD			REG_GENMASK64(63, 32)
 
+#define PUNIT_VERSION_OFFSET		0xA0
+
 #define BMG_TELEMETRY_BASE_OFFSET	0xE0000
 #define BMG_TELEMETRY_OFFSET		(SOC_BASE + BMG_TELEMETRY_BASE_OFFSET)
 
diff --git a/drivers/gpu/drm/xe/tests/Makefile b/drivers/gpu/drm/xe/tests/Makefile
index 0e3408f4952ce..f7aa47f11a368 100644
--- a/drivers/gpu/drm/xe/tests/Makefile
+++ b/drivers/gpu/drm/xe/tests/Makefile
@@ -9,5 +9,6 @@ obj-$(CONFIG_DRM_XE_KUNIT_TEST) += xe_test.o
 xe_test-y = xe_test_mod.o \
 	xe_args_test.o \
 	xe_pci_test.o \
+	xe_rtp_tables_test.o \
 	xe_rtp_test.o \
 	xe_wa_test.o
diff --git a/drivers/gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c b/drivers/gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c
new file mode 100644
index 0000000000000..82869363ab7e4
--- /dev/null
+++ b/drivers/gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c
@@ -0,0 +1,429 @@
+// SPDX-License-Identifier: GPL-2.0 AND MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <kunit/test.h>
+#include <kunit/test-bug.h>
+
+#define TEST_KEY	(GUC_KLV_RESERVED_RANGE_START + 0x3de)
+#define TEST_GROUP_KEY	(GUC_KLV_RESERVED_RANGE_START + 0x3f0)
+#define TEST_PAD	0xdeadbeef
+
+static bool fake_is_group_key(u16 key)
+{
+	return is_reserved_key(key) && key >= TEST_GROUP_KEY;
+}
+
+static void test_count(struct kunit *test)
+{
+	u32 value = 0x12345678;
+	u16 key = TEST_KEY;
+	u32 klvs[] = {
+		PREP_GUC_KLV(key + 0, 0),
+		PREP_GUC_KLV(key + 1, 1), value,
+		PREP_GUC_KLV(key + 2, 2), value, value,
+		PREP_GUC_KLV(key + 3, 0),
+		0, /* padding */
+	};
+
+	KUNIT_EXPECT_EQ(test, 0, xe_guc_klv_count(klvs, 0));
+	KUNIT_EXPECT_EQ(test, 1, xe_guc_klv_count(klvs, 1));
+	KUNIT_EXPECT_GT(test, 0, xe_guc_klv_count(klvs, 2));
+	KUNIT_EXPECT_EQ(test, 2, xe_guc_klv_count(klvs, 3));
+	KUNIT_EXPECT_GT(test, 0, xe_guc_klv_count(klvs, 4));
+	KUNIT_EXPECT_GT(test, 0, xe_guc_klv_count(klvs, 5));
+	KUNIT_EXPECT_EQ(test, 3, xe_guc_klv_count(klvs, 6));
+	KUNIT_EXPECT_EQ(test, 4, xe_guc_klv_count(klvs, 7));
+
+	/* 0 is treated as reserved KLV { KEY=0, LEN=0 } */
+	KUNIT_EXPECT_EQ(test, 5, xe_guc_klv_count(klvs, 8));
+}
+
+static void test_encode_u32(struct kunit *test)
+{
+	u32 *fail = ERR_PTR(-ENOMEM);
+	u32 value = 0x12345678;
+	u16 key = TEST_KEY;
+	u32 klvs[16];
+
+	memset32(klvs, TEST_PAD, ARRAY_SIZE(klvs));
+
+	KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-ENOSPC), xe_guc_klv_encode_u32(klvs, 0, key, value));
+	KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-ENOSPC), xe_guc_klv_encode_u32(klvs, 1, key, value));
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xe_guc_klv_encode_u32(klvs, 2, key, value));
+	KUNIT_EXPECT_EQ(test, klvs[0], PREP_GUC_KLV(key, 1));
+	KUNIT_EXPECT_EQ(test, klvs[1], value);
+	KUNIT_EXPECT_EQ(test, klvs[2], TEST_PAD);
+	KUNIT_EXPECT_PTR_EQ(test, &klvs[2], xe_guc_klv_encode_u32(klvs, 2, key, value));
+	KUNIT_EXPECT_PTR_EQ(test,
+			    xe_guc_klv_encode_u32(klvs, 2, key, value),
+			    xe_guc_klv_encode_u32(klvs, ARRAY_SIZE(klvs), key, value));
+
+	KUNIT_ASSERT_PTR_EQ(test, fail, xe_guc_klv_encode_u32(fail, ARRAY_SIZE(klvs), key, value));
+}
+
+static void test_encode_u64(struct kunit *test)
+{
+	u64 value = 0x123456789abcdef0;
+	u32 *fail = ERR_PTR(-ENOMEM);
+	u16 key = TEST_KEY;
+	u32 klvs[16];
+
+	memset32(klvs, TEST_PAD, ARRAY_SIZE(klvs));
+
+	KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-ENOSPC), xe_guc_klv_encode_u64(klvs, 0, key, value));
+	KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-ENOSPC), xe_guc_klv_encode_u64(klvs, 1, key, value));
+	KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-ENOSPC), xe_guc_klv_encode_u64(klvs, 2, key, value));
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xe_guc_klv_encode_u64(klvs, 3, key, value));
+	KUNIT_EXPECT_EQ(test, klvs[0], PREP_GUC_KLV(key, 2));
+	KUNIT_EXPECT_EQ(test, klvs[1], lower_32_bits(value));
+	KUNIT_EXPECT_EQ(test, klvs[2], upper_32_bits(value));
+	KUNIT_EXPECT_EQ(test, klvs[3], TEST_PAD);
+	KUNIT_EXPECT_PTR_EQ(test, &klvs[3], xe_guc_klv_encode_u64(klvs, 3, key, value));
+	KUNIT_EXPECT_PTR_EQ(test,
+			    xe_guc_klv_encode_u64(klvs, 3, key, value),
+			    xe_guc_klv_encode_u64(klvs, ARRAY_SIZE(klvs), key, value));
+
+	KUNIT_ASSERT_PTR_EQ(test, fail, xe_guc_klv_encode_u64(fail, ARRAY_SIZE(klvs), key, value));
+}
+
+static u32 str_klv_size(const char *string)
+{
+	return GUC_KLV_LEN_MIN + to_num_dwords(strlen(string) + 1);
+}
+
+static void test_encode_string(struct kunit *test)
+{
+	size_t longest_str = to_num_bytes(FIELD_MAX(GUC_KLV_0_LEN)) - 1;
+	u32 avail = GUC_KLV_LEN_MIN + FIELD_MAX(GUC_KLV_0_LEN) + 1;
+	const char *string = "abcdefghijklmnopqrstvwxyz";
+	u16 key = TEST_KEY;
+	u32 *klvs;
+	u32 *next;
+	char *buf;
+	u32 n;
+
+	klvs = kunit_kcalloc(test, avail, sizeof(u32), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, klvs);
+
+	buf = kunit_kzalloc(test, longest_str + 2, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf);
+
+	/* empty string, no space, must fail */
+	for (n = 0; n < str_klv_size(""); n++) {
+		klvs[0] = TEST_PAD;
+		KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-ENOSPC),
+				    xe_guc_klv_encode_string(klvs, n, key, ""));
+		KUNIT_EXPECT_EQ(test, klvs[0], TEST_PAD);
+	}
+
+	/* empty string, must pass */
+	KUNIT_EXPECT_PTR_EQ(test, klvs + str_klv_size(""),
+			    xe_guc_klv_encode_string(klvs, str_klv_size(""), key, ""));
+	KUNIT_EXPECT_PTR_EQ(test, klvs + str_klv_size(""),
+			    xe_guc_klv_encode_string(klvs, avail, key, ""));
+
+	/* demo string, no space, must fail */
+	for (n = 0; n < str_klv_size(string); n++) {
+		klvs[0] = TEST_PAD;
+		KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-ENOSPC),
+				    xe_guc_klv_encode_string(klvs, n, key, string));
+		KUNIT_EXPECT_EQ(test, klvs[0], TEST_PAD);
+	}
+
+	/* different string len, must pass */
+	for (n = 0; n <= strlen(string); n++) {
+		strscpy(buf, string, n + 1);
+		kunit_info(test, "%u: '%s'\n", n, buf);
+		KUNIT_ASSERT_EQ(test, n, strlen(buf));
+		memset32(klvs, TEST_PAD, avail);
+
+		next = xe_guc_klv_encode_string(klvs, str_klv_size(buf), key, buf);
+		KUNIT_ASSERT_NOT_ERR_OR_NULL(test, next);
+		KUNIT_EXPECT_PTR_EQ(test, next, klvs + str_klv_size(buf));
+		KUNIT_EXPECT_STREQ_MSG(test, buf, (char *)(klvs + GUC_KLV_LEN_MIN), "n=%u", n);
+		kunit_info(test, "%u: %*ph\n", n, (int)to_num_bytes(next - klvs), klvs);
+		KUNIT_EXPECT_NE(test, *(next - 1), TEST_PAD);
+		KUNIT_ASSERT_EQ(test, *next, TEST_PAD);
+
+		/* bigger buf doesn't matter */
+		KUNIT_EXPECT_PTR_EQ(test,
+				    xe_guc_klv_encode_string(klvs, str_klv_size(buf), key, buf),
+				    xe_guc_klv_encode_string(klvs, avail, key, buf));
+	}
+
+	/* don't crash if already failed */
+	KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-EROFS),
+			    xe_guc_klv_encode_string(ERR_PTR(-EROFS), avail, key, ""));
+
+	/* too long string, must fail */
+	memset(buf, 'X', longest_str + 1);
+	buf[longest_str + 1] = '\0';
+	KUNIT_EXPECT_LT(test, longest_str, strlen(buf));
+	KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-E2BIG),
+			    xe_guc_klv_encode_string(klvs, avail, key, buf));
+
+	/* longest string, should pass */
+	buf[longest_str] = '\0';
+	KUNIT_EXPECT_EQ(test, longest_str, strlen(buf));
+	KUNIT_EXPECT_PTR_EQ(test, klvs + str_klv_size(buf),
+			    xe_guc_klv_encode_string(klvs, avail, key, buf));
+}
+
+struct some_object {
+	u32 value1;
+	u64 value2;
+} __packed;
+
+static u32 *obj_raw_encoder(u32 *klvs, u32 avail, const void *arg)
+{
+	const struct some_object *obj = arg;
+	size_t sz = sizeof(*obj);
+	u32 dwords = to_num_dwords(sz);
+
+	if (IS_ERR(klvs))
+		return klvs;
+	if (dwords > avail)
+		return ERR_PTR(-ENOSPC);
+	memcpy(klvs, obj, sz);
+	return klvs + dwords;
+}
+
+static u32 *obj_klv_encoder(u32 *klvs, u32 avail, const void *arg)
+{
+	const struct some_object *obj = arg;
+	u32 *end = klvs + avail;
+
+	klvs = xe_guc_klv_encode_u32(klvs, end - klvs, TEST_KEY + 1, obj->value1);
+	klvs = xe_guc_klv_encode_u64(klvs, end - klvs, TEST_KEY + 2, obj->value2);
+	return klvs;
+}
+
+static u32 *obj_nested_encoder(u32 *klvs, u32 avail, const void *arg)
+{
+	u32 *end = klvs + avail;
+
+	klvs = xe_guc_klv_encode_object(klvs, end - klvs, TEST_GROUP_KEY + 1,
+					arg, obj_klv_encoder);
+	klvs = xe_guc_klv_encode_object(klvs, end - klvs, TEST_GROUP_KEY + 2,
+					arg, obj_klv_encoder);
+	return klvs;
+}
+
+static void test_encode_object_raw(struct kunit *test)
+{
+	const struct some_object obj = {
+		.value1 = 0xdead1234,
+		.value2 = 0xdead87654321dead,
+	};
+	u32 payload = to_num_dwords(sizeof(obj));
+	u32 *err = ERR_PTR(-ENOSPC);
+	u16 key = TEST_KEY;
+	u32 klvs[16];
+	u32 n;
+
+	/* too small, must fail */
+	for (n = 0; n < GUC_KLV_LEN_MIN + payload; n++) {
+		memset32(klvs, TEST_PAD, ARRAY_SIZE(klvs));
+		KUNIT_EXPECT_PTR_EQ_MSG(test, ERR_PTR(-ENOSPC),
+					xe_guc_klv_encode_object(klvs, n, key, &obj,
+								 obj_raw_encoder),
+					"buf size=%u dwords", n);
+	}
+
+	/* must pass */
+	memset32(klvs, TEST_PAD, ARRAY_SIZE(klvs));
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test,
+				     xe_guc_klv_encode_object(klvs, GUC_KLV_LEN_MIN + payload,
+							      key, &obj, obj_raw_encoder));
+	KUNIT_EXPECT_EQ(test, klvs[0], PREP_GUC_KLV(key, payload));
+	KUNIT_EXPECT_MEMEQ(test, &klvs[1], &obj, sizeof(obj));
+
+	/* already failed, must fail */
+	KUNIT_ASSERT_PTR_EQ(test, err,
+			    xe_guc_klv_encode_object(err, ARRAY_SIZE(klvs), key,
+						     &obj, obj_raw_encoder));
+}
+
+static void test_encode_object_klv(struct kunit *test)
+{
+	const struct some_object obj = {
+		.value1 = 0xdead1234,
+		.value2 = 0xdead87654321dead,
+	};
+	u16 key = TEST_GROUP_KEY;
+	u32 payload = 0;
+	u32 klvs[16];
+	u32 n;
+
+	payload += GUC_KLV_LEN_MIN + to_num_dwords(sizeof(obj.value1));
+	payload += GUC_KLV_LEN_MIN + to_num_dwords(sizeof(obj.value2));
+
+	/* too small, must fail */
+	for (n = 0; n < GUC_KLV_LEN_MIN + payload; n++) {
+		memset32(klvs, TEST_PAD, ARRAY_SIZE(klvs));
+		KUNIT_EXPECT_PTR_EQ_MSG(test, ERR_PTR(-ENOSPC),
+					xe_guc_klv_encode_object(klvs, n, key, &obj,
+								 obj_klv_encoder),
+					"buf size=%u dwords", n);
+	}
+
+	/* must pass */
+	memset32(klvs, TEST_PAD, ARRAY_SIZE(klvs));
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test,
+				     xe_guc_klv_encode_object(klvs, GUC_KLV_LEN_MIN + payload,
+							      key, &obj, obj_klv_encoder));
+	KUNIT_EXPECT_EQ(test, klvs[0], PREP_GUC_KLV(key, payload));
+	KUNIT_EXPECT_EQ(test, klvs[1], PREP_GUC_KLV(TEST_KEY + 1, 1));
+	KUNIT_EXPECT_EQ(test, klvs[2], obj.value1);
+	KUNIT_EXPECT_EQ(test, klvs[3], PREP_GUC_KLV(TEST_KEY + 2, 2));
+	KUNIT_EXPECT_EQ(test, klvs[4], lower_32_bits(obj.value2));
+	KUNIT_EXPECT_EQ(test, klvs[5], upper_32_bits(obj.value2));
+	KUNIT_EXPECT_EQ(test, klvs[6], TEST_PAD);
+}
+
+static void test_encode_object_nested(struct kunit *test)
+{
+	const struct some_object obj = {
+		.value1 = 0xdead1234,
+		.value2 = 0xdead87654321dead,
+	};
+	u16 key = TEST_GROUP_KEY;
+	u32 payload = 0;
+	u32 klvs[16];
+	u32 n;
+
+	payload += GUC_KLV_LEN_MIN;
+	payload += GUC_KLV_LEN_MIN + to_num_dwords(sizeof(obj.value1));
+	payload += GUC_KLV_LEN_MIN + to_num_dwords(sizeof(obj.value2));
+	payload *= 2;
+
+	/* too small, must fail */
+	for (n = 0; n < GUC_KLV_LEN_MIN + payload; n++) {
+		memset32(klvs, TEST_PAD, ARRAY_SIZE(klvs));
+		KUNIT_EXPECT_PTR_EQ_MSG(test, ERR_PTR(-ENOSPC),
+					xe_guc_klv_encode_object(klvs, n, key, &obj,
+								 obj_nested_encoder),
+					"buf size=%u dwords", n);
+	}
+
+	/* must pass */
+	memset32(klvs, TEST_PAD, ARRAY_SIZE(klvs));
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test,
+				     xe_guc_klv_encode_object(klvs, GUC_KLV_LEN_MIN + payload,
+							      key, &obj, obj_nested_encoder));
+	KUNIT_EXPECT_EQ(test, klvs[0], PREP_GUC_KLV(key, payload));
+	KUNIT_EXPECT_EQ(test, klvs[1], PREP_GUC_KLV(TEST_GROUP_KEY + 1, 5));
+	KUNIT_EXPECT_EQ(test, klvs[2], PREP_GUC_KLV(TEST_KEY + 1, 1));
+	KUNIT_EXPECT_EQ(test, klvs[3], obj.value1);
+	KUNIT_EXPECT_EQ(test, klvs[4], PREP_GUC_KLV(TEST_KEY + 2, 2));
+	KUNIT_EXPECT_EQ(test, klvs[5], lower_32_bits(obj.value2));
+	KUNIT_EXPECT_EQ(test, klvs[6], upper_32_bits(obj.value2));
+	KUNIT_EXPECT_EQ(test, klvs[7], PREP_GUC_KLV(TEST_GROUP_KEY + 2, 5));
+	KUNIT_EXPECT_EQ(test, klvs[8], PREP_GUC_KLV(TEST_KEY + 1, 1));
+	KUNIT_EXPECT_EQ(test, klvs[9], obj.value1);
+	KUNIT_EXPECT_EQ(test, klvs[10], PREP_GUC_KLV(TEST_KEY + 2, 2));
+	KUNIT_EXPECT_EQ(test, klvs[11], lower_32_bits(obj.value2));
+	KUNIT_EXPECT_EQ(test, klvs[12], upper_32_bits(obj.value2));
+	KUNIT_EXPECT_EQ(test, klvs[13], TEST_PAD);
+}
+
+static u32 *obj_echo_encoder(u32 *klvs, u32 avail, const void *arg)
+{
+	return ERR_CAST(arg);
+}
+
+static void test_encode_object_basic(struct kunit *test)
+{
+	u32 longest = GUC_KLV_LEN_MIN + FIELD_MAX(GUC_KLV_0_LEN);
+	u32 avail = GUC_KLV_LEN_MIN + longest;
+	u16 key = TEST_GROUP_KEY;
+	u32 *klvs;
+
+	klvs = kunit_kcalloc(test, avail, sizeof(u32), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, klvs);
+
+	/* smallest */
+	KUNIT_EXPECT_PTR_EQ(test, klvs + GUC_KLV_LEN_MIN,
+			    xe_guc_klv_encode_object(klvs, avail, key,
+						     klvs + GUC_KLV_LEN_MIN,
+						     obj_echo_encoder));
+	/* largest */
+	KUNIT_EXPECT_PTR_EQ(test, klvs + longest,
+			    xe_guc_klv_encode_object(klvs, avail, key,
+						     klvs + longest,
+						     obj_echo_encoder));
+	/* already failed */
+	KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-EROFS),
+			    xe_guc_klv_encode_object(ERR_PTR(-EROFS), avail, key,
+						     klvs + GUC_KLV_LEN_MIN,
+						     obj_echo_encoder));
+	/* encoding error */
+	KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-EUCLEAN),
+			    xe_guc_klv_encode_object(klvs, avail, key,
+						     ERR_PTR(-EUCLEAN),
+						     obj_echo_encoder));
+	/* no space */
+	KUNIT_EXPECT_PTR_EQ(test, ERR_PTR(-ENOSPC),
+			    xe_guc_klv_encode_object(klvs, 0, key,
+						     klvs + GUC_KLV_LEN_MIN,
+						     obj_echo_encoder));
+}
+
+static void __drm_printfn_kunit(struct drm_printer *p, struct va_format *vaf)
+{
+	struct kunit *test = p->arg;
+
+	kunit_info(test, "%pV", vaf);
+}
+
+static struct drm_printer drm_kunit_printer(void)
+{
+	struct drm_printer p = {
+		.printfn = __drm_printfn_kunit,
+		.arg = kunit_get_current_test(),
+	};
+	return p;
+}
+
+static void test_print(struct kunit *test)
+{
+	struct drm_printer p = drm_kunit_printer();
+	u32 zeros[] = { 0, 0, 0, /* padding */ };
+	u32 klvs[] = {
+		PREP_GUC_KLV(GUC_KLV_OPT_IN_FEATURE_EXT_CAT_ERR_TYPE_KEY, 0),
+		PREP_GUC_KLV(GUC_KLV_VF_CFG_NUM_CONTEXTS_KEY, 1), 1234,
+		PREP_GUC_KLV(GUC_KLV_VF_CFG_GGTT_SIZE_KEY, 2), 0x4000, 0x0123,
+		PREP_GUC_KLV(TEST_KEY, 3), 1, 2, 3,
+		PREP_GUC_KLV(TEST_GROUP_KEY, 5),
+		PREP_GUC_KLV(TEST_KEY + 1, 1), 1,
+		PREP_GUC_KLV(TEST_KEY + 2, 2), 1, 2,
+	};
+
+	kunit_activate_static_stub(test, is_group_key, fake_is_group_key);
+	xe_guc_klv_print(zeros, ARRAY_SIZE(zeros), &p);
+	xe_guc_klv_print(klvs, ARRAY_SIZE(klvs), &p);
+}
+
+static struct kunit_case guc_klv_helpers_test_cases[] = {
+	KUNIT_CASE(test_count),
+	KUNIT_CASE(test_encode_u32),
+	KUNIT_CASE(test_encode_u64),
+	KUNIT_CASE(test_encode_string),
+	KUNIT_CASE(test_encode_object_raw),
+	KUNIT_CASE(test_encode_object_klv),
+	KUNIT_CASE(test_encode_object_nested),
+	KUNIT_CASE(test_encode_object_basic),
+	KUNIT_CASE(test_print),
+	{}
+};
+
+static struct kunit_suite guc_klv_helpers_suite = {
+	.name = "guc_klv_helpers",
+	.test_cases = guc_klv_helpers_test_cases,
+};
+
+kunit_test_suite(guc_klv_helpers_suite);
diff --git a/drivers/gpu/drm/xe/tests/xe_pci.c b/drivers/gpu/drm/xe/tests/xe_pci.c
index c2c686aed1cb8..bb0393475524a 100644
--- a/drivers/gpu/drm/xe/tests/xe_pci.c
+++ b/drivers/gpu/drm/xe/tests/xe_pci.c
@@ -310,40 +310,44 @@ const void *xe_pci_id_gen_param(struct kunit *test, const void *prev, char *desc
 }
 EXPORT_SYMBOL_IF_KUNIT(xe_pci_id_gen_param);
 
-static void fake_init_devid(struct xe_device *xe)
+static int fake_probe_info(struct xe_device *xe,
+			   const struct xe_device_desc *desc,
+			   struct xe_pci_fake_data *data,
+			   struct xe_probed_info *probed_info)
 {
-	/* Nothing to do, just keep zero. */
-}
-
-static int fake_read_gmdid(struct xe_device *xe, enum xe_gmdid_type type,
-			   u32 *ver, u32 *revid)
-{
-	struct kunit *test = kunit_get_current_test();
-	struct xe_pci_fake_data *data = test->priv;
+	probed_info->tile_count = 1 + desc->max_remote_tiles;
 
-	if (type == GMDID_MEDIA) {
-		*ver = data->media_verx100;
-		*revid = xe_step_to_gmdid(data->step.media);
+	if (!data || desc->pre_gmdid_graphics_ip) {
+		probed_info->graphics_ip = desc->pre_gmdid_graphics_ip;
+		probed_info->media_ip = desc->pre_gmdid_media_ip;
 	} else {
-		*ver = data->graphics_verx100;
-		*revid = xe_step_to_gmdid(data->step.graphics);
+		probed_info->graphics_ip = find_graphics_ip(data->graphics_verx100);
+
+		if (data->media_verx100) {
+			probed_info->media_ip = find_media_ip(data->media_verx100);
+			xe_assert(xe, probed_info->media_ip);
+		}
 	}
 
-	return 0;
-}
+	xe_assert(xe, probed_info->graphics_ip);
+	if (!probed_info->graphics_ip)
+		return -ENODEV;
 
-static void fake_xe_info_probe_tile_count(struct xe_device *xe)
-{
-	/* Nothing to do, just use the statically defined value. */
+	if (data)
+		probed_info->step = data->step;
+
+	return 0;
 }
 
 int xe_pci_fake_device_init(struct xe_device *xe)
 {
 	struct kunit *test = kunit_get_current_test();
 	struct xe_pci_fake_data *data = test->priv;
+	struct xe_probed_info probed_info = {};
 	const struct pci_device_id *ent = pciidlist;
 	const struct xe_device_desc *desc;
 	const struct xe_subplatform_desc *subplatform_desc;
+	int err;
 
 	if (!data) {
 		desc = (const void *)ent->driver_data;
@@ -373,13 +377,12 @@ int xe_pci_fake_device_init(struct xe_device *xe)
 	xe->sriov.__mode = data && data->sriov_mode ?
 			   data->sriov_mode : XE_SRIOV_MODE_NONE;
 
-	kunit_activate_static_stub(test, init_devid, fake_init_devid);
-	kunit_activate_static_stub(test, read_gmdid, fake_read_gmdid);
-	kunit_activate_static_stub(test, xe_info_probe_tile_count,
-				   fake_xe_info_probe_tile_count);
+	err = fake_probe_info(xe, desc, data, &probed_info);
+	if (err)
+		return err;
 
-	xe_info_init_early(xe, desc, subplatform_desc);
-	xe_info_init(xe, desc);
+	xe_info_init_early(xe, desc, subplatform_desc, &probed_info);
+	xe_info_init(xe, desc, &probed_info);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/xe/tests/xe_rtp_tables_test.c b/drivers/gpu/drm/xe/tests/xe_rtp_tables_test.c
new file mode 100644
index 0000000000000..7e2fc39ac62cd
--- /dev/null
+++ b/drivers/gpu/drm/xe/tests/xe_rtp_tables_test.c
@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <kunit/test.h>
+
+#include "xe_reg_whitelist.h"
+#include "xe_rtp_types.h"
+#include "xe_tuning.h"
+#include "xe_wa.h"
+
+#define RTP_TABLE_PARAM(table)									\
+	static const void *table##_gen_params(struct kunit *test,				\
+					     const void *prev, char *desc)			\
+	{											\
+		typeof((table.entries)[0]) *__next = prev ?					\
+			((typeof(__next))prev) + 1 : (table.entries);				\
+		if (__next - table.entries < table.n_entries) {					\
+			scnprintf(desc, KUNIT_PARAM_DESC_SIZE, #table "/%s", __next->name);	\
+			return __next;								\
+		}										\
+		return NULL;									\
+	}
+
+static void xe_rtp_table_gt_test(struct kunit *test)
+{
+	const struct xe_rtp_entry_sr *entry = test->param_value;
+
+	for (int i = 0; i < entry->n_rules; i++) {
+		KUNIT_EXPECT_TRUE(test,
+				  entry->rules[i].match_type != XE_RTP_MATCH_ENGINE_CLASS ||
+				  entry->flags & XE_RTP_ENTRY_FLAG_FOREACH_ENGINE);
+		KUNIT_EXPECT_TRUE(test,
+				  entry->rules[i].match_type != XE_RTP_MATCH_NOT_ENGINE_CLASS ||
+				  entry->flags & XE_RTP_ENTRY_FLAG_FOREACH_ENGINE);
+	}
+}
+
+RTP_TABLE_PARAM(gt_was);
+RTP_TABLE_PARAM(gt_tunings);
+
+static void xe_rtp_table_oob_test(struct kunit *test)
+{
+	const struct xe_rtp_entry *entry = test->param_value;
+
+	for (int i = 0; i < entry->n_rules; i++) {
+		u8 match_type = entry->rules[i].match_type;
+
+		KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_ENGINE_CLASS);
+		KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_NOT_ENGINE_CLASS);
+	}
+}
+
+RTP_TABLE_PARAM(oob_was);
+
+static void xe_rtp_table_dev_oob_test(struct kunit *test)
+{
+	const struct xe_rtp_entry *entry = test->param_value;
+
+	for (int i = 0; i < entry->n_rules; i++) {
+		u8 match_type = entry->rules[i].match_type;
+
+		KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_ENGINE_CLASS);
+		KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_NOT_ENGINE_CLASS);
+		KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_GRAPHICS_VERSION);
+		KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_GRAPHICS_VERSION_RANGE);
+		KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_GRAPHICS_VERSION_ANY_GT);
+		KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_GRAPHICS_STEP);
+		KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_MEDIA_VERSION);
+		KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_MEDIA_VERSION_RANGE);
+		KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_MEDIA_VERSION_ANY_GT);
+		KUNIT_EXPECT_NE(test, match_type, XE_RTP_MATCH_MEDIA_STEP);
+	}
+}
+
+RTP_TABLE_PARAM(device_oob_was);
+
+static void xe_rtp_table_missing_upper_bound_test(struct kunit *test)
+{
+	const struct xe_rtp_entry_sr *entry = test->param_value;
+
+	for (int i = 0; i < entry->n_rules; i++) {
+		u8 match_type = entry->rules[i].match_type;
+
+		KUNIT_EXPECT_FALSE(test,
+				   match_type == XE_RTP_MATCH_GRAPHICS_VERSION_RANGE &&
+				   entry->rules[i].ver_end == XE_RTP_END_VERSION_UNDEFINED);
+		KUNIT_EXPECT_FALSE(test,
+				   match_type == XE_RTP_MATCH_MEDIA_VERSION_RANGE &&
+				   entry->rules[i].ver_end == XE_RTP_END_VERSION_UNDEFINED);
+	}
+}
+
+RTP_TABLE_PARAM(register_whitelist);
+
+static struct kunit_case xe_rtp_table_tests[] = {
+	KUNIT_CASE_PARAM(xe_rtp_table_gt_test, gt_was_gen_params),
+	KUNIT_CASE_PARAM(xe_rtp_table_gt_test, gt_tunings_gen_params),
+	KUNIT_CASE_PARAM(xe_rtp_table_oob_test, oob_was_gen_params),
+	KUNIT_CASE_PARAM(xe_rtp_table_dev_oob_test, device_oob_was_gen_params),
+	KUNIT_CASE_PARAM(xe_rtp_table_missing_upper_bound_test,
+			 register_whitelist_gen_params),
+	{}
+};
+
+static struct kunit_suite xe_rtp_tables_test_suite = {
+	.name = "xe_rtp_tables_test",
+	.test_cases = xe_rtp_table_tests,
+};
+
+kunit_test_suite(xe_rtp_tables_test_suite);
diff --git a/drivers/gpu/drm/xe/tests/xe_rtp_test.c b/drivers/gpu/drm/xe/tests/xe_rtp_test.c
index 3d0688d058d97..367811621880f 100644
--- a/drivers/gpu/drm/xe/tests/xe_rtp_test.c
+++ b/drivers/gpu/drm/xe/tests/xe_rtp_test.c
@@ -280,6 +280,11 @@ static void xe_rtp_rules_tests(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, err, param->expected_err);
 }
 
+static u32 bits_2_3_set(struct xe_gt *gt, struct xe_hw_engine *hwe)
+{
+	return REG_BIT(2) | REG_BIT(3);
+}
+
 static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = {
 	{
 		.name = "coalesce-same-reg",
@@ -300,6 +305,29 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = {
 			},
 		),
 	},
+	{
+		.name = "coalesce-same-reg-literal-and-func",
+		.expected_reg = REGULAR_REG1,
+		.expected_set_bits = REG_BIT(0) | REG_BIT(1) | REG_BIT(2) | REG_BIT(3),
+		.expected_clr_bits = REG_BIT(0) | REG_BIT(1) | REG_BIT(2) | REG_BIT(3),
+		.expected_active = BIT(0) | BIT(1),
+		.expected_count_sr_entries = 1,
+		/* Different bits on the same register: create a single entry */
+		.table = XE_RTP_TABLE_SR(
+			{ XE_RTP_NAME("basic-1"),
+			  XE_RTP_RULES(FUNC(match_yes)),
+			  XE_RTP_ACTIONS(FIELD_SET(REGULAR_REG1,
+						   REG_BIT(0) | REG_BIT(1),
+						   REG_BIT(0) | REG_BIT(1)))
+			},
+			{ XE_RTP_NAME("basic-2"),
+			  XE_RTP_RULES(FUNC(match_yes)),
+			  XE_RTP_ACTIONS(FIELD_SET_FUNC(REGULAR_REG1,
+							REG_BIT(2) | REG_BIT(3),
+							bits_2_3_set))
+			},
+		),
+	},
 	{
 		.name = "no-match-no-add",
 		.expected_reg = REGULAR_REG1,
@@ -417,6 +445,30 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = {
 			},
 		),
 	},
+	{
+		.name = "conflict-not-disjoint-literal-and-func",
+		.expected_reg = REGULAR_REG1,
+		.expected_set_bits = REG_BIT(1) | REG_BIT(2),
+		.expected_clr_bits = REG_BIT(1) | REG_BIT(2),
+		.expected_active = BIT(0) | BIT(1),
+		.expected_count_sr_entries = 1,
+		.expected_sr_errors = 1,
+		.table = XE_RTP_TABLE_SR(
+			{ XE_RTP_NAME("basic-1"),
+			  XE_RTP_RULES(FUNC(match_yes)),
+			  XE_RTP_ACTIONS(FIELD_SET(REGULAR_REG1,
+						   REG_BIT(1) | REG_BIT(2),
+						   REG_BIT(1) | REG_BIT(2)))
+			},
+			/* drop: bits are not disjoint with previous entries */
+			{ XE_RTP_NAME("basic-2"),
+			  XE_RTP_RULES(FUNC(match_yes)),
+			  XE_RTP_ACTIONS(FIELD_SET_FUNC(REGULAR_REG1,
+							REG_BIT(2) | REG_BIT(3),
+							bits_2_3_set))
+			},
+		),
+	},
 	{
 		.name = "conflict-reg-type",
 		.expected_reg = REGULAR_REG1,
diff --git a/drivers/gpu/drm/xe/tests/xe_sriov_packet_kunit.c b/drivers/gpu/drm/xe/tests/xe_sriov_packet_kunit.c
new file mode 100644
index 0000000000000..c2720b461dee7
--- /dev/null
+++ b/drivers/gpu/drm/xe/tests/xe_sriov_packet_kunit.c
@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: GPL-2.0 AND MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <kunit/test.h>
+#include <kunit/test-bug.h>
+
+#include "xe_device.h"
+#include "xe_guc_klv_helpers.h"
+#include "xe_kunit_helpers.h"
+#include "xe_pci_test.h"
+
+#define TEST_VF		VFID(1)
+
+static int sriov_packet_test_init(struct kunit *test)
+{
+	struct xe_pci_fake_data fake = {
+		.sriov_mode = XE_SRIOV_MODE_PF,
+		.platform = XE_PANTHERLAKE, /* we need MEMIRQ */
+		.subplatform = XE_SUBPLATFORM_NONE,
+		.graphics_verx100 = 3000,
+		.media_verx100 = 3000,
+	};
+	struct xe_device *xe;
+
+	test->priv = &fake;
+	xe_kunit_helper_xe_device_test_init(test);
+	xe = test->priv;
+
+	/* pretend we can support at least VF1 */
+	xe->sriov.pf.device_total_vfs = 1;
+	xe->sriov.pf.driver_max_vfs = 1;
+
+	KUNIT_ASSERT_EQ(test, 0, xe_sriov_init(xe));
+	KUNIT_ASSERT_TRUE(test, xe_sriov_pf_migration_supported(xe));
+
+	return 0;
+}
+
+static void test_descriptor_init(struct kunit *test)
+{
+	struct xe_device *xe = test->priv;
+	struct xe_sriov_packet **desc;
+
+	/* note: with lock held we should avoid KUNIT_ASSERT() */
+	guard(mutex)(pf_migration_mutex(xe, TEST_VF));
+
+	KUNIT_EXPECT_EQ(test, 0, pf_descriptor_init(xe, TEST_VF));
+	desc = pf_pick_descriptor(xe, TEST_VF);
+	KUNIT_EXPECT_NOT_ERR_OR_NULL(test, *desc);
+	if (!*desc)
+		return;
+	KUNIT_EXPECT_NE(test, (*desc)->hdr.version, 0);
+	KUNIT_EXPECT_EQ(test, (*desc)->hdr.version, XE_SRIOV_PACKET_SUPPORTED_VERSION);
+	KUNIT_EXPECT_EQ(test, (*desc)->hdr.type, XE_SRIOV_PACKET_TYPE_DESCRIPTOR);
+	KUNIT_EXPECT_NE(test, (*desc)->hdr.size, 0);
+	KUNIT_EXPECT_NOT_ERR_OR_NULL(test, (*desc)->vaddr);
+	if (!(*desc)->vaddr)
+		return;
+	KUNIT_EXPECT_EQ(test, 0, xe_sriov_packet_process_descriptor(xe, TEST_VF, *desc));
+
+	switch ((*desc)->hdr.version) {
+	case 1:
+		/* v1 is KLV based */
+		KUNIT_EXPECT_TRUE(test, IS_ALIGNED((*desc)->hdr.size, sizeof(u32)));
+		/* v1 has at least DEVID and REVID KLVs */
+		KUNIT_EXPECT_LE(test, 2,
+				xe_guc_klv_count((*desc)->vaddr,
+						 (*desc)->hdr.size / sizeof(u32)));
+		break;
+	default:
+		kunit_mark_skipped(test, "no test code for version %u\n", (*desc)->hdr.version);
+		return;
+	}
+}
+
+static struct kunit_case sriov_packet_test_cases[] = {
+	KUNIT_CASE(test_descriptor_init),
+	{}
+};
+
+static struct kunit_suite sriov_packet_suite = {
+	.name = "sriov_packet",
+	.test_cases = sriov_packet_test_cases,
+	.init = sriov_packet_test_init,
+};
+
+kunit_test_suite(sriov_packet_suite);
diff --git a/drivers/gpu/drm/xe/tests/xe_wa_test.c b/drivers/gpu/drm/xe/tests/xe_wa_test.c
index ff0e2502b39f9..21601e9df3533 100644
--- a/drivers/gpu/drm/xe/tests/xe_wa_test.c
+++ b/drivers/gpu/drm/xe/tests/xe_wa_test.c
@@ -43,9 +43,6 @@ static int xe_wa_test_init(struct kunit *test)
 		xe_gt_mmio_init(gt);
 	}
 
-	if (!param->graphics_verx100)
-		xe->info.step = param->step;
-
 	/* TODO: init hw engines for engine/LRC WAs */
 	xe->drm.dev = dev;
 	test->priv = xe;
diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 4c80bac676229..c266fa6bade1b 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -670,7 +670,7 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
 		dma_resv_iter_begin(&cursor, bo->ttm.base.resv,
 				    DMA_RESV_USAGE_BOOKKEEP);
 		dma_resv_for_each_fence_unlocked(&cursor, fence)
-			dma_fence_enable_sw_signaling(fence);
+			dma_fence_enable_signaling(fence);
 		dma_resv_iter_end(&cursor);
 	}
 
@@ -1102,6 +1102,21 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,
 		xe_pm_runtime_get_noresume(xe);
 	}
 
+	/*
+	 * Attach CCS BBs before submitting the copy job below so a VF
+	 * migration racing the copy sees valid, up to date attach state.
+	 */
+	if (IS_VF_CCS_READY(xe) &&
+	    ((move_lacks_source && new_mem->mem_type == XE_PL_TT) ||
+	     (old_mem_type == XE_PL_SYSTEM && new_mem->mem_type == XE_PL_TT)) &&
+	    handle_system_ccs) {
+		ret = xe_sriov_vf_ccs_attach_bo(bo, new_mem);
+		if (ret) {
+			xe_pm_runtime_put(xe);
+			goto out;
+		}
+	}
+
 	if (move_lacks_source) {
 		u32 flags = 0;
 
@@ -1139,22 +1154,19 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,
 		ttm_bo_move_null(ttm_bo, new_mem);
 	}
 
-	dma_fence_put(fence);
-	xe_pm_runtime_put(xe);
-
 	/*
-	 * CCS meta data is migrated from TT -> SMEM. So, let us detach the
-	 * BBs from BO as it is no longer needed.
+	 * Detach must wait for the copy above to complete: a VF migration
+	 * racing an in-flight copy must still see valid CCS BBs, so don't
+	 * tear them down until the copy fence has signaled.
 	 */
 	if (IS_VF_CCS_READY(xe) && old_mem_type == XE_PL_TT &&
-	    new_mem->mem_type == XE_PL_SYSTEM)
+	    new_mem->mem_type == XE_PL_SYSTEM) {
+		dma_fence_wait(fence, false);
 		xe_sriov_vf_ccs_detach_bo(bo);
+	}
 
-	if (IS_VF_CCS_READY(xe) &&
-	    ((move_lacks_source && new_mem->mem_type == XE_PL_TT) ||
-	     (old_mem_type == XE_PL_SYSTEM && new_mem->mem_type == XE_PL_TT)) &&
-	    handle_system_ccs)
-		ret = xe_sriov_vf_ccs_attach_bo(bo);
+	dma_fence_put(fence);
+	xe_pm_runtime_put(xe);
 
 out:
 	if ((!ttm_bo->resource || ttm_bo->resource->mem_type == XE_PL_SYSTEM) &&
@@ -1349,7 +1361,7 @@ int xe_bo_notifier_prepare_pinned(struct xe_bo *bo)
 		backup = xe_bo_init_locked(xe, NULL, NULL, bo->ttm.base.resv, NULL, xe_bo_size(bo),
 					   DRM_XE_GEM_CPU_CACHING_WB, ttm_bo_type_kernel,
 					   XE_BO_FLAG_SYSTEM | XE_BO_FLAG_NEEDS_CPU_ACCESS |
-					   XE_BO_FLAG_PINNED, &exec);
+					   XE_BO_FLAG_PINNED, NULL, &exec);
 		if (IS_ERR(backup)) {
 			drm_exec_retry_on_contention(&exec);
 			ret = PTR_ERR(backup);
@@ -1490,7 +1502,7 @@ int xe_bo_evict_pinned(struct xe_bo *bo)
 						   xe_bo_size(bo),
 						   DRM_XE_GEM_CPU_CACHING_WB, ttm_bo_type_kernel,
 						   XE_BO_FLAG_SYSTEM | XE_BO_FLAG_NEEDS_CPU_ACCESS |
-						   XE_BO_FLAG_PINNED, &exec);
+						   XE_BO_FLAG_PINNED, NULL, &exec);
 			if (IS_ERR(backup)) {
 				drm_exec_retry_on_contention(&exec);
 				ret = PTR_ERR(backup);
@@ -1826,6 +1838,8 @@ static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo)
 
 	if (bo->ttm.base.import_attach)
 		drm_prime_gem_destroy(&bo->ttm.base, NULL);
+	if (bo->dma_buf)
+		dma_buf_put(bo->dma_buf);
 	drm_gem_object_release(&bo->ttm.base);
 
 	xe_assert(xe, list_empty(&ttm_bo->base.gpuva.list));
@@ -2283,6 +2297,8 @@ void xe_bo_free(struct xe_bo *bo)
  * @cpu_caching: The cpu caching used for system memory backing store.
  * @type: The TTM buffer object type.
  * @flags: XE_BO_FLAG_ flags.
+ * @dma_buf: The dma-buf to reference for the BO lifetime (imported BOs),
+ * or NULL.
  * @exec: The drm_exec transaction to use for exhaustive eviction.
  *
  * Initialize or create an xe buffer object. On failure, any allocated buffer
@@ -2294,7 +2310,8 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo,
 				struct xe_tile *tile, struct dma_resv *resv,
 				struct ttm_lru_bulk_move *bulk, size_t size,
 				u16 cpu_caching, enum ttm_bo_type type,
-				u32 flags, struct drm_exec *exec)
+				u32 flags, struct dma_buf *dma_buf,
+				struct drm_exec *exec)
 {
 	struct ttm_operation_ctx ctx = {
 		.interruptible = true,
@@ -2383,6 +2400,17 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo,
 	placement = (type == ttm_bo_type_sg ||
 		     bo->flags & XE_BO_FLAG_DEFER_BACKING) ? &sys_placement :
 		&bo->placement;
+
+	/*
+	 * For imported BOs, keep the exporter dma-buf alive for the BO
+	 * lifetime. Taken before ttm_bo_init_reserved() to also cover a
+	 * creation failure there. Released in xe_ttm_bo_destroy().
+	 */
+	if (dma_buf) {
+		get_dma_buf(dma_buf);
+		bo->dma_buf = dma_buf;
+	}
+
 	err = ttm_bo_init_reserved(&xe->ttm, &bo->ttm, type,
 				   placement, alignment,
 				   &ctx, NULL, resv, xe_ttm_bo_destroy);
@@ -2500,7 +2528,7 @@ __xe_bo_create_locked(struct xe_device *xe,
 			       vm && !xe_vm_in_fault_mode(vm) &&
 			       flags & XE_BO_FLAG_USER ?
 			       &vm->lru_bulk_move : NULL, size,
-			       cpu_caching, type, flags, exec);
+			       cpu_caching, type, flags, NULL, exec);
 	if (IS_ERR(bo))
 		return bo;
 
diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h
index 6340317f7d2e6..7ae1d9ac05743 100644
--- a/drivers/gpu/drm/xe/xe_bo.h
+++ b/drivers/gpu/drm/xe/xe_bo.h
@@ -118,7 +118,8 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo,
 				struct xe_tile *tile, struct dma_resv *resv,
 				struct ttm_lru_bulk_move *bulk, size_t size,
 				u16 cpu_caching, enum ttm_bo_type type,
-				u32 flags, struct drm_exec *exec);
+				u32 flags, struct dma_buf *dma_buf,
+				struct drm_exec *exec);
 struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile,
 				  struct xe_vm *vm, size_t size,
 				  enum ttm_bo_type type, u32 flags,
diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h
index fcc63ae3f4551..e45f24301050c 100644
--- a/drivers/gpu/drm/xe/xe_bo_types.h
+++ b/drivers/gpu/drm/xe/xe_bo_types.h
@@ -36,6 +36,8 @@ struct xe_bo {
 	struct xe_bo *backup_obj;
 	/** @parent_obj: Ref to parent bo if this a backup_obj */
 	struct xe_bo *parent_obj;
+	/** @dma_buf: Imported dma-buf ref to keep its resv alive. */
+	struct dma_buf *dma_buf;
 	/** @flags: flags for this buffer object */
 	u32 flags;
 	/** @vm: VM this BO is attached to, for extobj this will be NULL */
diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c
index 32102600a1481..052cce9621611 100644
--- a/drivers/gpu/drm/xe/xe_configfs.c
+++ b/drivers/gpu/drm/xe/xe_configfs.c
@@ -237,6 +237,18 @@
  *
  * This setting only takes effect when probing the device.
  *
+ * Enable multi-queue
+ * ------------------
+ *
+ * Multi-queue support on the device is enabled by default where the
+ * hardware supports it. Writing 0 force-disables multi-queue support:
+ * multi-queue exec-queue group creation via ioctl is refused, and the
+ * GuC feature is disabled::
+ *
+ *	# echo 0 > /sys/kernel/config/xe/0000:03:00.0/enable_multi_queue
+ *
+ * This attribute can only be set before binding to the device.
+ *
  * Remove devices
  * ==============
  *
@@ -262,6 +274,7 @@ struct xe_config_group_device {
 		struct wa_bb ctx_restore_mid_bb[XE_ENGINE_CLASS_MAX];
 		bool survivability_mode;
 		bool enable_psmi;
+		bool enable_multi_queue;
 		struct {
 			unsigned int max_vfs;
 			bool admin_only_pf;
@@ -281,6 +294,7 @@ static const struct xe_config_device device_defaults = {
 	.engines_allowed = U64_MAX,
 	.survivability_mode = false,
 	.enable_psmi = false,
+	.enable_multi_queue = true,
 	.sriov = {
 		.max_vfs = XE_DEFAULT_MAX_VFS,
 		.admin_only_pf = XE_DEFAULT_ADMIN_ONLY_PF,
@@ -575,6 +589,33 @@ static ssize_t enable_psmi_store(struct config_item *item, const char *page, siz
 	return len;
 }
 
+static ssize_t enable_multi_queue_show(struct config_item *item, char *page)
+{
+	struct xe_config_device *dev = to_xe_config_device(item);
+
+	return sprintf(page, "%d\n", dev->enable_multi_queue);
+}
+
+static ssize_t enable_multi_queue_store(struct config_item *item, const char *page,
+					size_t len)
+{
+	struct xe_config_group_device *dev = to_xe_config_group_device(item);
+	bool val;
+	int ret;
+
+	ret = kstrtobool(page, &val);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&dev->lock);
+	if (is_bound(dev))
+		return -EBUSY;
+
+	dev->config.enable_multi_queue = val;
+
+	return len;
+}
+
 static bool wa_bb_read_advance(bool dereference, char **p,
 			       const char *append, size_t len,
 			       size_t *max_size)
@@ -812,6 +853,7 @@ static ssize_t ctx_restore_post_bb_store(struct config_item *item,
 
 CONFIGFS_ATTR(, ctx_restore_mid_bb);
 CONFIGFS_ATTR(, ctx_restore_post_bb);
+CONFIGFS_ATTR(, enable_multi_queue);
 CONFIGFS_ATTR(, enable_psmi);
 CONFIGFS_ATTR(, engines_allowed);
 CONFIGFS_ATTR(, gt_types_allowed);
@@ -820,6 +862,7 @@ CONFIGFS_ATTR(, survivability_mode);
 static struct configfs_attribute *xe_config_device_attrs[] = {
 	&attr_ctx_restore_mid_bb,
 	&attr_ctx_restore_post_bb,
+	&attr_enable_multi_queue,
 	&attr_enable_psmi,
 	&attr_engines_allowed,
 	&attr_gt_types_allowed,
@@ -1097,6 +1140,7 @@ static void dump_custom_dev_config(struct pci_dev *pdev,
 
 	PRI_CUSTOM_ATTR("%llx", gt_types_allowed);
 	PRI_CUSTOM_ATTR("%llx", engines_allowed);
+	PRI_CUSTOM_ATTR("%d", enable_multi_queue);
 	PRI_CUSTOM_ATTR("%d", enable_psmi);
 	PRI_CUSTOM_ATTR("%d", survivability_mode);
 	PRI_CUSTOM_ATTR("%u", sriov.admin_only_pf);
@@ -1225,6 +1269,27 @@ bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev)
 	return ret;
 }
 
+/**
+ * xe_configfs_get_enable_multi_queue - get configfs enable_multi_queue setting
+ * @pdev: pci device
+ *
+ * Return: true if multi-queue is enabled for this device (the default),
+ * false if it has been force-disabled via configfs.
+ */
+bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev)
+{
+	struct xe_config_group_device *dev = find_xe_config_group_device(pdev);
+	bool ret;
+
+	if (!dev)
+		return true;
+
+	ret = dev->config.enable_multi_queue;
+	config_group_put(&dev->group);
+
+	return ret;
+}
+
 /**
  * xe_configfs_get_ctx_restore_mid_bb - get configfs ctx_restore_mid_bb setting
  * @pdev: pci device
diff --git a/drivers/gpu/drm/xe/xe_configfs.h b/drivers/gpu/drm/xe/xe_configfs.h
index 07d62bf0c1524..4fbbeafba4734 100644
--- a/drivers/gpu/drm/xe/xe_configfs.h
+++ b/drivers/gpu/drm/xe/xe_configfs.h
@@ -23,6 +23,7 @@ bool xe_configfs_primary_gt_allowed(struct pci_dev *pdev);
 bool xe_configfs_media_gt_allowed(struct pci_dev *pdev);
 u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev);
 bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev);
+bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev);
 u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev,
 				       enum xe_engine_class class,
 				       const u32 **cs);
@@ -42,6 +43,7 @@ static inline bool xe_configfs_primary_gt_allowed(struct pci_dev *pdev) { return
 static inline bool xe_configfs_media_gt_allowed(struct pci_dev *pdev) { return true; }
 static inline u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) { return U64_MAX; }
 static inline bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev) { return false; }
+static inline bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev) { return true; }
 static inline u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev,
 						     enum xe_engine_class class,
 						     const u32 **cs) { return 0; }
diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c
index 22b471303984a..5a3877fcb0f02 100644
--- a/drivers/gpu/drm/xe/xe_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_debugfs.c
@@ -5,6 +5,7 @@
 
 #include "xe_debugfs.h"
 
+#include <linux/bits.h>
 #include <linux/debugfs.h>
 #include <linux/fault-inject.h>
 #include <linux/string_helpers.h>
@@ -21,6 +22,7 @@
 #include "xe_guc_ads.h"
 #include "xe_hw_engine.h"
 #include "xe_mmio.h"
+#include "xe_pcode.h"
 #include "xe_pm.h"
 #include "xe_psmi.h"
 #include "xe_pxp_debugfs.h"
@@ -41,6 +43,55 @@
 DECLARE_FAULT_ATTR(gt_reset_failure);
 DECLARE_FAULT_ATTR(inject_csc_hw_error);
 
+static bool csc_hw_error_available(struct xe_device *xe)
+{
+	return !IS_SRIOV_VF(xe) && xe->info.platform == XE_BATTLEMAGE;
+}
+
+/*
+ * Fault injection table.  Each entry registers a debugfs attribute; add a
+ * matching FAULT_ACTION() below for every entry added here.
+ */
+static struct {
+	const char *name;
+	struct fault_attr *attr;
+	bool (*is_visible)(struct xe_device *xe);
+} xe_fault_inject_entry[] = {
+	{ .name = "fail_gt_reset",
+	  .attr = &gt_reset_failure },
+	{ .name = "inject_csc_hw_error",
+	  .attr = &inject_csc_hw_error,
+	  .is_visible = csc_hw_error_available },
+};
+
+/*
+ * FAULT_ACTION(name, fault_attr) - generate xe_fault_<name>() accessor.
+ * Add one entry per row in xe_fault_inject_entry[].
+ */
+#define FAULT_ACTION(name, fault_attr)			\
+bool xe_fault_##name(void)				\
+{							\
+	return should_fail(&(fault_attr), 1);		\
+}
+
+FAULT_ACTION(gt_reset, gt_reset_failure)
+FAULT_ACTION(csc_hw_error, inject_csc_hw_error)
+
+static void xe_fault_inject_debugfs_register(struct xe_device *xe,
+					     struct dentry *root)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(xe_fault_inject_entry); i++) {
+		if (xe_fault_inject_entry[i].is_visible &&
+		    !xe_fault_inject_entry[i].is_visible(xe))
+			continue;
+
+		fault_create_debugfs_attr(xe_fault_inject_entry[i].name, root,
+					  xe_fault_inject_entry[i].attr);
+	}
+}
+
 static void read_residency_counter(struct xe_device *xe, struct xe_mmio *mmio,
 				   u32 offset, const char *name, struct drm_printer *p)
 {
@@ -117,7 +168,6 @@ static int info(struct seq_file *m, void *data)
 	drm_printf(&p, "revid %d\n", xe->info.revid);
 	drm_printf(&p, "tile_count %d\n", xe->info.tile_count);
 	drm_printf(&p, "vm_max_level %d\n", xe->info.vm_max_level);
-	drm_printf(&p, "force_execlist %s\n", str_yes_no(xe->info.force_execlist));
 	drm_printf(&p, "has_flat_ccs %s\n", str_yes_no(xe->info.has_flat_ccs));
 	drm_printf(&p, "has_usm %s\n", str_yes_no(xe->info.has_usm));
 	drm_printf(&p, "skip_guc_pc %s\n", str_yes_no(xe->info.skip_guc_pc));
@@ -161,6 +211,22 @@ static int workaround_info(struct seq_file *m, void *data)
 	return 0;
 }
 
+static int pcode_info(struct seq_file *m, void *data)
+{
+	struct xe_device *xe = node_to_xe(m->private);
+	struct drm_printer p = drm_seq_file_printer(m);
+	struct xe_pcode_version version;
+	int ret = 0;
+
+	ret = xe_get_pcode_version(xe, &version);
+	if (ret)
+		return ret;
+
+	drm_printf(&p, "pcode version: %u.%u.%u\n", version.major,
+		   version.minor, version.engg);
+	return 0;
+}
+
 static int dgfx_pkg_residencies_show(struct seq_file *m, void *data)
 {
 	struct xe_device *xe;
@@ -221,6 +287,10 @@ static const struct drm_info_list debugfs_list[] = {
 	{ .name = "workarounds", .show = workaround_info, },
 };
 
+static const struct drm_info_list pcode_info_debugfs[] = {
+	{ .name = "pcode_info", .show = pcode_info, },
+};
+
 static const struct drm_info_list debugfs_residencies[] = {
 	{ .name = "dgfx_pkg_residencies", .show = dgfx_pkg_residencies_show, },
 	{ .name = "dgfx_pcie_link_residencies", .show = dgfx_pcie_link_residencies_show, },
@@ -563,10 +633,20 @@ void xe_debugfs_register(struct xe_device *xe)
 		drm_debugfs_create_files(debugfs_residencies,
 					 ARRAY_SIZE(debugfs_residencies),
 					 root, minor);
-		fault_create_debugfs_attr("inject_csc_hw_error", root,
-					  &inject_csc_hw_error);
 	}
 
+	/*
+	 * Pcode version read from PMT is currently only supported on CRI and BMG platforms in PF
+	 * mode, as both platforms support the necessary telemetry read mechanism and have a fixed
+	 * PUNIT_VERSION_OFFSET.
+	 * Attempting this access on other platforms must be verified before enabling support.
+	 */
+	if (!IS_SRIOV_VF(xe) &&
+	    (xe->info.platform == XE_CRESCENTISLAND || xe->info.platform == XE_BATTLEMAGE))
+		drm_debugfs_create_files(pcode_info_debugfs,
+					 ARRAY_SIZE(pcode_info_debugfs),
+					 root, minor);
+
 	debugfs_create_file("forcewake_all", 0400, root, xe,
 			    &forcewake_all_fops);
 
@@ -610,7 +690,7 @@ void xe_debugfs_register(struct xe_device *xe)
 
 	xe_psmi_debugfs_register(xe);
 
-	fault_create_debugfs_attr("fail_gt_reset", root, &gt_reset_failure);
+	xe_fault_inject_debugfs_register(xe, root);
 
 	if (IS_SRIOV_PF(xe))
 		xe_sriov_pf_debugfs_register(xe, root);
diff --git a/drivers/gpu/drm/xe/xe_debugfs.h b/drivers/gpu/drm/xe/xe_debugfs.h
index 17f4c2f1b5e4d..cd56f7442b995 100644
--- a/drivers/gpu/drm/xe/xe_debugfs.h
+++ b/drivers/gpu/drm/xe/xe_debugfs.h
@@ -6,11 +6,17 @@
 #ifndef _XE_DEBUGFS_H_
 #define _XE_DEBUGFS_H_
 
+#include <linux/types.h>
+
 struct xe_device;
 
 #ifdef CONFIG_DEBUG_FS
+bool xe_fault_gt_reset(void);
+bool xe_fault_csc_hw_error(void);
 void xe_debugfs_register(struct xe_device *xe);
 #else
+static inline bool xe_fault_gt_reset(void) { return false; }
+static inline bool xe_fault_csc_hw_error(void) { return false; }
 static inline void xe_debugfs_register(struct xe_device *xe) { }
 #endif
 
diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
index abe25aedeeade..84bab9219d816 100644
--- a/drivers/gpu/drm/xe/xe_device.c
+++ b/drivers/gpu/drm/xe/xe_device.c
@@ -61,6 +61,7 @@
 #include "xe_psmi.h"
 #include "xe_pxp.h"
 #include "xe_query.h"
+#include "xe_ras.h"
 #include "xe_shrinker.h"
 #include "xe_soc_remapper.h"
 #include "xe_survivability_mode.h"
@@ -281,7 +282,7 @@ static vm_fault_t barrier_fault(struct vm_fault *vmf)
 	pgprot_t prot;
 	int idx;
 
-	prot = vm_get_page_prot(vma->vm_flags);
+	prot = vma_get_page_prot(vma);
 
 	if (drm_dev_enter(dev, &idx)) {
 		unsigned long pfn;
@@ -396,7 +397,7 @@ static const struct drm_driver regular_driver = {
 	    XE_DISPLAY_DRIVER_FEATURES |
 	    DRIVER_GEM |
 	    DRIVER_RENDER | DRIVER_SYNCOBJ |
-	    DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA,
+	    DRIVER_SYNCOBJ_TIMELINE,
 	.open = xe_file_open,
 	.postclose = xe_file_close,
 
@@ -426,8 +427,7 @@ static const struct drm_ioctl_desc xe_ioctls_admin_only[] = {
 
 static const struct drm_driver admin_only_driver = {
 	.driver_features =
-	    XE_DISPLAY_DRIVER_FEATURES |
-	    DRIVER_GEM | DRIVER_RENDER | DRIVER_GEM_GPUVA,
+	    DRIVER_GEM | DRIVER_RENDER,
 	.open = xe_file_open,
 	.postclose = xe_file_close,
 	.ioctls = xe_ioctls_admin_only,
@@ -438,7 +438,6 @@ static const struct drm_driver admin_only_driver = {
 	.major = DRIVER_MAJOR,
 	.minor = DRIVER_MINOR,
 	.patchlevel = DRIVER_PATCHLEVEL,
-	XE_DISPLAY_DRIVER_OPS,
 };
 
 /**
@@ -580,7 +579,7 @@ int xe_device_init_early(struct xe_device *xe)
 						       WQ_MEM_RECLAIM);
 	xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0);
 	xe->unordered_wq = alloc_workqueue("xe-unordered-wq", WQ_PERCPU, 0);
-	xe->destroy_wq = alloc_workqueue("xe-destroy-wq", WQ_PERCPU, 0);
+	xe->destroy_wq = alloc_workqueue("xe-destroy-wq", WQ_PERCPU | WQ_MEM_RECLAIM, 0);
 	if (!xe->ordered_wq || !xe->unordered_wq ||
 	    !xe->preempt_fence_wq || !xe->destroy_wq) {
 		/*
@@ -738,9 +737,11 @@ static void vf_update_device_info(struct xe_device *xe)
 	xe->info.probe_display = 0;
 	xe->info.has_heci_cscfi = 0;
 	xe->info.has_heci_gscfi = 0;
+	xe->info.has_i2c = 0;
 	xe->info.has_late_bind = 0;
 	xe->info.skip_guc_pc = 1;
 	xe->info.skip_pcode = 1;
+	xe->info.has_drm_ras = false;
 }
 
 static int xe_device_vram_alloc(struct xe_device *xe)
@@ -949,6 +950,15 @@ int xe_device_probe(struct xe_device *xe)
 			return err;
 	}
 
+	/*
+	 * Wa_16029380221: The affected GT will always use non-coherent
+	 * access to page tables, so we must do uncached writes from the
+	 * CPU.
+	 */
+	for_each_gt(gt, xe, id)
+		if (XE_GT_WA(gt, 16029380221))
+			xe->info.has_cached_pt = false;
+
 	for_each_tile(tile, xe, id) {
 		err = xe_ggtt_init_early(tile->mem.ggtt);
 		if (err)
@@ -989,6 +999,16 @@ int xe_device_probe(struct xe_device *xe)
 	if (err)
 		return err;
 
+	err = xe_soc_remapper_init(xe);
+	if (err)
+		return err;
+
+	err = xe_sysctrl_init(xe);
+	if (err)
+		return err;
+
+	xe_ras_init(xe);
+
 	/*
 	 * Now that GT is initialized (TTM in particular),
 	 * we can try to init display, and inherit the initial fb.
@@ -1029,10 +1049,6 @@ int xe_device_probe(struct xe_device *xe)
 
 	xe_nvm_init(xe);
 
-	err = xe_soc_remapper_init(xe);
-	if (err)
-		return err;
-
 	err = xe_heci_gsc_init(xe);
 	if (err)
 		return err;
@@ -1071,10 +1087,6 @@ int xe_device_probe(struct xe_device *xe)
 	if (err)
 		goto err_unregister_display;
 
-	err = xe_sysctrl_init(xe);
-	if (err)
-		goto err_unregister_display;
-
 	err = xe_device_sysfs_init(xe);
 	if (err)
 		goto err_unregister_display;
@@ -1129,14 +1141,14 @@ void xe_device_shutdown(struct xe_device *xe)
 
 	drm_dbg(&xe->drm, "Shutting down device\n");
 
-	xe_display_pm_shutdown(xe);
+	xe_display_shutdown(xe);
 
 	xe_irq_suspend(xe);
 
 	for_each_gt(gt, xe, id)
 		xe_gt_shutdown(gt);
 
-	xe_display_pm_shutdown_late(xe);
+	xe_display_shutdown_late(xe);
 
 	if (!xe_driver_flr_disabled(xe)) {
 		/* BOOM! */
diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h
index 975768a6a9c83..a03760d0ce386 100644
--- a/drivers/gpu/drm/xe/xe_device.h
+++ b/drivers/gpu/drm/xe/xe_device.h
@@ -116,7 +116,7 @@ static inline struct xe_mmio *xe_root_tile_mmio(struct xe_device *xe)
 
 static inline bool xe_device_uc_enabled(struct xe_device *xe)
 {
-	return !xe->info.force_execlist;
+	return true;
 }
 
 #define for_each_tile(tile__, xe__, id__) \
@@ -181,6 +181,21 @@ static inline bool xe_device_has_mert(const struct xe_device *xe)
 	return xe->info.has_mert;
 }
 
+static inline bool xe_device_is_in_reset(struct xe_device *xe)
+{
+	return atomic_read(&xe->in_reset);
+}
+
+static inline void xe_device_set_in_reset(struct xe_device *xe)
+{
+	atomic_set(&xe->in_reset, 1);
+}
+
+static inline void xe_device_clear_in_reset(struct xe_device *xe)
+{
+	atomic_set(&xe->in_reset, 0);
+}
+
 u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size);
 
 void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p);
diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h
index 32dd2ffbc7963..56c17cca79c0f 100644
--- a/drivers/gpu/drm/xe/xe_device_types.h
+++ b/drivers/gpu/drm/xe/xe_device_types.h
@@ -144,8 +144,6 @@ struct xe_device {
 		 * Keep all flags below alphabetically sorted
 		 */
 
-		/** @info.force_execlist: Forced execlist submission */
-		u8 force_execlist:1;
 		/** @info.has_access_counter: Device supports access counter */
 		u8 has_access_counter:1;
 		/** @info.has_asid: Has address space ID */
@@ -156,6 +154,8 @@ struct xe_device {
 		u8 has_cached_pt:1;
 		/** @info.has_device_atomics_on_smem: Supports device atomics on SMEM */
 		u8 has_device_atomics_on_smem:1;
+		/** @info.has_drm_ras: Device supports drm_ras (Reliability, Availability, Serviceability) */
+		u8 has_drm_ras:1;
 		/** @info.has_fan_control: Device supports fan control */
 		u8 has_fan_control:1;
 		/** @info.has_flat_ccs: Whether flat CCS metadata is used */
@@ -355,7 +355,7 @@ struct xe_device {
 	/** @unordered_wq: used to serialize unordered work */
 	struct workqueue_struct *unordered_wq;
 
-	/** @destroy_wq: used to serialize user destroy work, like queue */
+	/** @destroy_wq: used to serialize SVM pagemap destroy work */
 	struct workqueue_struct *destroy_wq;
 
 	/** @tiles: device tiles */
@@ -483,6 +483,9 @@ struct xe_device {
 	/** @needs_flr_on_fini: requests function-reset on fini */
 	bool needs_flr_on_fini;
 
+	/** @in_reset: Indicates if device is in reset */
+	atomic_t in_reset;
+
 	/** @wedged: Struct to control Wedged States and mode */
 	struct {
 		/** @wedged.flag: Xe device faced a critical error and is now blocked. */
@@ -495,6 +498,9 @@ struct xe_device {
 		bool inconsistent_reset;
 	} wedged;
 
+	/** @devres_group: devres group */
+	void *devres_group;
+
 	/** @bo_device: Struct to control async free of BOs */
 	struct xe_bo_dev {
 		/** @bo_device.async_free: Free worker */
diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c
index 8a920e58245cd..bf0728838ead5 100644
--- a/drivers/gpu/drm/xe/xe_dma_buf.c
+++ b/drivers/gpu/drm/xe/xe_dma_buf.c
@@ -302,7 +302,7 @@ xe_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf)
 
 		bo = xe_bo_init_locked(xe, NULL, NULL, resv, NULL, dma_buf->size,
 				       0, /* Will require 1way or 2way for vm_bind */
-				       ttm_bo_type_sg, XE_BO_FLAG_SYSTEM, &exec);
+				       ttm_bo_type_sg, XE_BO_FLAG_SYSTEM, dma_buf, &exec);
 		drm_exec_retry_on_contention(&exec);
 		if (IS_ERR(bo)) {
 			ret = PTR_ERR(bo);
diff --git a/drivers/gpu/drm/xe/xe_drm_client.c b/drivers/gpu/drm/xe/xe_drm_client.c
index 81020b4b344e0..e116fb562c4c6 100644
--- a/drivers/gpu/drm/xe/xe_drm_client.c
+++ b/drivers/gpu/drm/xe/xe_drm_client.c
@@ -193,7 +193,7 @@ static void bo_meminfo(struct xe_bo *bo,
 		if (!dma_resv_test_signaled(bo->ttm.base.resv,
 					    DMA_RESV_USAGE_BOOKKEEP))
 			stats[mem_type].active += sz;
-		else if (mem_type == XE_PL_SYSTEM)
+		else if (mem_type == XE_PL_SYSTEM || xe_bo_madv_is_dontneed(bo))
 			stats[mem_type].purgeable += sz;
 	}
 }
@@ -273,8 +273,7 @@ static void show_meminfo(struct drm_printer *p, struct drm_file *file)
 					       &stats[mem_type],
 					       DRM_GEM_OBJECT_ACTIVE |
 					       DRM_GEM_OBJECT_RESIDENT |
-					       (mem_type != XE_PL_SYSTEM ? 0 :
-					       DRM_GEM_OBJECT_PURGEABLE),
+					       DRM_GEM_OBJECT_PURGEABLE,
 					       xe_mem_type_to_name[mem_type]);
 		}
 	}
diff --git a/drivers/gpu/drm/xe/xe_drm_ras.c b/drivers/gpu/drm/xe/xe_drm_ras.c
index cd236f53699e0..7937d8ba0ed9c 100644
--- a/drivers/gpu/drm/xe/xe_drm_ras.c
+++ b/drivers/gpu/drm/xe/xe_drm_ras.c
@@ -11,27 +11,46 @@
 
 #include "xe_device_types.h"
 #include "xe_drm_ras.h"
+#include "xe_ras.h"
 
 static const char * const error_components[] = DRM_XE_RAS_ERROR_COMPONENT_NAMES;
 static const char * const error_severity[] = DRM_XE_RAS_ERROR_SEVERITY_NAMES;
 
-static int hw_query_error_counter(struct xe_drm_ras_counter *info,
-				  u32 error_id, const char **name, u32 *val)
+static int query_error_counter(struct xe_device *xe,
+			       enum drm_xe_ras_error_severity severity,
+			       u32 error_id, const char **name, u32 *val)
 {
+	struct xe_drm_ras *ras = &xe->ras;
+	struct xe_drm_ras_counter *info = ras->info[severity];
+
 	if (!info || !info[error_id].name)
 		return -ENOENT;
 
 	*name = info[error_id].name;
+
+	/* Fetch counter from system controller if supported */
+	if (xe->info.has_sysctrl)
+		return xe_ras_get_counter(xe, severity, error_id, val);
+
 	*val = atomic_read(&info[error_id].counter);
 
 	return 0;
 }
 
-static int hw_clear_error_counter(struct xe_drm_ras_counter *info, u32 error_id)
+static int clear_error_counter(struct xe_device *xe,
+			       enum drm_xe_ras_error_severity severity,
+			       u32 error_id)
 {
+	struct xe_drm_ras *ras = &xe->ras;
+	struct xe_drm_ras_counter *info = ras->info[severity];
+
 	if (!info || !info[error_id].name)
 		return -ENOENT;
 
+	/* Clear counter from system controller if supported */
+	if (xe->info.has_sysctrl)
+		return xe_ras_clear_counter(xe, severity, error_id);
+
 	atomic_set(&info[error_id].counter, 0);
 
 	return 0;
@@ -41,38 +60,30 @@ static int query_uncorrectable_error_counter(struct drm_ras_node *ep, u32 error_
 					     const char **name, u32 *val)
 {
 	struct xe_device *xe = ep->priv;
-	struct xe_drm_ras *ras = &xe->ras;
-	struct xe_drm_ras_counter *info = ras->info[DRM_XE_RAS_ERR_SEV_UNCORRECTABLE];
 
-	return hw_query_error_counter(info, error_id, name, val);
+	return query_error_counter(xe, DRM_XE_RAS_ERR_SEV_UNCORRECTABLE, error_id, name, val);
 }
 
 static int clear_uncorrectable_error_counter(struct drm_ras_node *node, u32 error_id)
 {
 	struct xe_device *xe = node->priv;
-	struct xe_drm_ras *ras = &xe->ras;
-	struct xe_drm_ras_counter *info = ras->info[DRM_XE_RAS_ERR_SEV_UNCORRECTABLE];
 
-	return hw_clear_error_counter(info, error_id);
+	return clear_error_counter(xe, DRM_XE_RAS_ERR_SEV_UNCORRECTABLE, error_id);
 }
 
 static int query_correctable_error_counter(struct drm_ras_node *ep, u32 error_id,
 					   const char **name, u32 *val)
 {
 	struct xe_device *xe = ep->priv;
-	struct xe_drm_ras *ras = &xe->ras;
-	struct xe_drm_ras_counter *info = ras->info[DRM_XE_RAS_ERR_SEV_CORRECTABLE];
 
-	return hw_query_error_counter(info, error_id, name, val);
+	return query_error_counter(xe, DRM_XE_RAS_ERR_SEV_CORRECTABLE, error_id, name, val);
 }
 
 static int clear_correctable_error_counter(struct drm_ras_node *node, u32 error_id)
 {
 	struct xe_device *xe = node->priv;
-	struct xe_drm_ras *ras = &xe->ras;
-	struct xe_drm_ras_counter *info = ras->info[DRM_XE_RAS_ERR_SEV_CORRECTABLE];
 
-	return hw_clear_error_counter(info, error_id);
+	return clear_error_counter(xe, DRM_XE_RAS_ERR_SEV_CORRECTABLE, error_id);
 }
 
 static struct xe_drm_ras_counter *allocate_and_copy_counters(struct xe_device *xe)
diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c
index 297be3c42b208..d37770c58c5d3 100644
--- a/drivers/gpu/drm/xe/xe_eu_stall.c
+++ b/drivers/gpu/drm/xe/xe_eu_stall.c
@@ -985,9 +985,10 @@ int xe_eu_stall_stream_open(struct drm_device *dev, u64 data, struct drm_file *f
 		return -ENODEV;
 	}
 
-	if (xe_observation_paranoid && !perfmon_capable()) {
+	ret = xe_observation_paranoid_check();
+	if (ret) {
 		drm_dbg(&xe->drm,  "Insufficient privileges for EU stall monitoring\n");
-		return -EACCES;
+		return ret;
 	}
 
 	/* Initialize and set default values */
diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c
index 1b5ca3ce578a4..38972b6e6d37e 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue.c
+++ b/drivers/gpu/drm/xe/xe_exec_queue.c
@@ -530,10 +530,7 @@ struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe,
 
 	migrate_vm = xe_migrate_get_vm(tile->migrate);
 	if (xe->info.has_usm) {
-		struct xe_hw_engine *hwe = xe_gt_hw_engine(gt,
-							   XE_ENGINE_CLASS_COPY,
-							   gt->usm.reserved_bcs_instance,
-							   false);
+		struct xe_hw_engine *hwe = gt->usm.paging_hwe0;
 
 		if (!hwe) {
 			xe_vm_put(migrate_vm);
@@ -842,6 +839,7 @@ static int xe_exec_queue_group_init(struct xe_device *xe, struct xe_exec_queue *
 	group->primary = q;
 	group->cgp_bo = bo;
 	INIT_LIST_HEAD(&group->list);
+	spin_lock_init(&group->suspend_lock);
 	xa_init_flags(&group->xa, XA_FLAGS_ALLOC1);
 	mutex_init(&group->list_lock);
 	q->multi_queue.group = group;
@@ -1056,6 +1054,7 @@ int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data,
 
 static int exec_queue_user_ext_check(struct xe_exec_queue *q, u64 properties)
 {
+	struct xe_device *xe = gt_to_xe(q->gt);
 	u64 secondary_queue_valid_props = BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP) |
 				  BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY);
 
@@ -1067,6 +1066,16 @@ static int exec_queue_user_ext_check(struct xe_exec_queue *q, u64 properties)
 	    properties & ~secondary_queue_valid_props)
 		return -EINVAL;
 
+	/*
+	 * HWDRM is the only supported PXP type today. It is display related and
+	 * hence can't work with multi-queue. Reject the combination. The secondary
+	 * queue path above already rejects any PXP property, so this also covers
+	 * the multi-queue primary which would otherwise allow it.
+	 */
+	if (XE_IOCTL_DBG(xe, (properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP)) &&
+			 (properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE))))
+		return -EINVAL;
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h
index 2f5ccf294675a..53b6c0bf48499 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue_types.h
+++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h
@@ -62,6 +62,12 @@ struct xe_exec_queue_group {
 	struct list_head list;
 	/** @list_lock: Secondary queue list lock */
 	struct mutex list_lock;
+	/**
+	 * @suspend_lock: Makes a secondary's suspend/resume and its forwarding
+	 * to the primary atomic. Nested outside of the queue's message lock
+	 * (@xe_guc_exec_queue.sched.msg_lock).
+	 */
+	spinlock_t suspend_lock;
 	/** @sync_pending: CGP_SYNC_DONE g2h response pending */
 	bool sync_pending;
 	/** @banned: Group banned */
@@ -200,6 +206,18 @@ struct xe_exec_queue {
 		u32 seqno;
 		/** @lr.link: link into VM's list of exec queues */
 		struct list_head link;
+		/**
+		 * @lr.suspended: Tracks whether the consumer-issued suspend()
+		 * succeeded and a matching resume() is still owed. suspend() can
+		 * fail (e.g. killed/banned/wedged), leaving the queue
+		 * un-suspended, so consumers must only resume() queues that were
+		 * actually suspended. Set by the suspend caller on success and
+		 * cleared by the resume caller. A queue is only ever suspended by
+		 * a single consumer at a time (preempt-fence mode and hw engine
+		 * group fault mode are mutually exclusive), so a single flag is
+		 * sufficient.
+		 */
+		bool suspended;
 	} lr;
 
 #define XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT	0
@@ -310,6 +328,15 @@ struct xe_exec_queue_ops {
 	 * avoidance mechanism.
 	 */
 	int (*suspend_wait)(struct xe_exec_queue *q);
+	/**
+	 * @suspend_wait_blocking: Like @suspend_wait, but waits uninterruptibly
+	 * (does not abort on the calling task's signals). For cleanup/undo paths
+	 * that must complete a suspend on behalf of a queue that may belong to a
+	 * different process than the caller: a signal to the caller must not
+	 * abandon the wait, which would leave the other process's queue
+	 * suspended forever (cross-process DoS). A timeout bans like suspend_wait.
+	 */
+	int (*suspend_wait_blocking)(struct xe_exec_queue *q);
 	/**
 	 * @resume: Resume exec queue execution, exec queue must be in a suspended
 	 * state and dma fence returned from most recent suspend call must be
@@ -318,8 +345,6 @@ struct xe_exec_queue_ops {
 	void (*resume)(struct xe_exec_queue *q);
 	/** @reset_status: check exec queue reset status */
 	bool (*reset_status)(struct xe_exec_queue *q);
-	/** @active: check exec queue is active */
-	bool (*active)(struct xe_exec_queue *q);
 };
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_execlist.c b/drivers/gpu/drm/xe/xe_execlist.c
index 9fb99c038ea88..cc33ae80e8cf8 100644
--- a/drivers/gpu/drm/xe/xe_execlist.c
+++ b/drivers/gpu/drm/xe/xe_execlist.c
@@ -458,12 +458,6 @@ static bool execlist_exec_queue_reset_status(struct xe_exec_queue *q)
 	return false;
 }
 
-static bool execlist_exec_queue_active(struct xe_exec_queue *q)
-{
-	/* NIY */
-	return false;
-}
-
 static const struct xe_exec_queue_ops execlist_exec_queue_ops = {
 	.init = execlist_exec_queue_init,
 	.kill = execlist_exec_queue_kill,
@@ -474,9 +468,9 @@ static const struct xe_exec_queue_ops execlist_exec_queue_ops = {
 	.set_preempt_timeout = execlist_exec_queue_set_preempt_timeout,
 	.suspend = execlist_exec_queue_suspend,
 	.suspend_wait = execlist_exec_queue_suspend_wait,
+	.suspend_wait_blocking = execlist_exec_queue_suspend_wait,
 	.resume = execlist_exec_queue_resume,
 	.reset_status = execlist_exec_queue_reset_status,
-	.active = execlist_exec_queue_active,
 };
 
 int xe_execlist_init(struct xe_gt *gt)
diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c
index a351c578b1705..8ec23862477fc 100644
--- a/drivers/gpu/drm/xe/xe_ggtt.c
+++ b/drivers/gpu/drm/xe/xe_ggtt.c
@@ -111,14 +111,14 @@ struct xe_ggtt_pt_ops {
 struct xe_ggtt {
 	/** @tile: Back pointer to tile where this GGTT belongs */
 	struct xe_tile *tile;
-        /** @start: Start offset of GGTT */
+	/** @start: Start offset of GGTT */
 	u64 start;
 	/** @size: Total usable size of this GGTT */
 	u64 size;
-
 	/**
-	 * @flags: Flags for this GGTT
+	 * @flags: Flags for this GGTT.
 	 * Acceptable flags:
+	 *
 	 * - %XE_GGTT_FLAGS_64K - if PTE size is 64K. Otherwise, regular is 4K.
 	 * - %XE_GGTT_FLAGS_ONLINE - is GGTT online, protected by ggtt->lock
 	 *   after init
@@ -129,7 +129,7 @@ struct xe_ggtt {
 	/** @lock: Mutex lock to protect GGTT data */
 	struct mutex lock;
 	/**
-	 *  @gsm: The iomem pointer to the actual location of the translation
+	 * @gsm: The iomem pointer to the actual location of the translation
 	 * table located in the GSM for easy PTE manipulation
 	 */
 	u64 __iomem *gsm;
diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c
index 783eb6d631b5c..dfdacc0f6de97 100644
--- a/drivers/gpu/drm/xe/xe_gt.c
+++ b/drivers/gpu/drm/xe/xe_gt.c
@@ -21,6 +21,7 @@
 #include "regs/xe_gt_regs.h"
 #include "xe_assert.h"
 #include "xe_bb.h"
+#include "xe_debugfs.h"
 #include "xe_device.h"
 #include "xe_eu_stall.h"
 #include "xe_exec_queue.h"
@@ -917,13 +918,16 @@ static void gt_reset_worker(struct work_struct *w)
 	if (xe_device_wedged(gt_to_xe(gt)))
 		goto err_pm_put;
 
+	if (xe_device_is_in_reset(gt_to_xe(gt)))
+		goto err_pm_put;
+
 	/* We only support GT resets with GuC submission */
 	if (!xe_device_uc_enabled(gt_to_xe(gt)))
 		goto err_pm_put;
 
 	xe_gt_info(gt, "reset started\n");
 
-	if (xe_fault_inject_gt_reset()) {
+	if (xe_fault_gt_reset()) {
 		err = -ECANCELED;
 		goto err_fail;
 	}
@@ -977,18 +981,21 @@ static void gt_reset_worker(struct work_struct *w)
 
 void xe_gt_reset_async(struct xe_gt *gt)
 {
-	xe_gt_info(gt, "trying reset from %ps\n", __builtin_return_address(0));
+	struct xe_device *xe = gt_to_xe(gt);
+
+	if (xe_device_is_in_reset(xe))
+		return;
 
 	/* Don't do a reset while one is already in flight */
-	if (!xe_fault_inject_gt_reset() && xe_uc_reset_prepare(&gt->uc))
+	if (!xe_fault_gt_reset() && xe_uc_reset_prepare(&gt->uc))
 		return;
 
-	xe_gt_info(gt, "reset queued\n");
+	xe_gt_info(gt, "reset queued from %ps\n", __builtin_return_address(0));
 
 	/* Pair with put in gt_reset_worker() if work is enqueued */
-	xe_pm_runtime_get_noresume(gt_to_xe(gt));
+	xe_pm_runtime_get_noresume(xe);
 	if (!queue_work(gt->ordered_wq, &gt->reset.worker))
-		xe_pm_runtime_put(gt_to_xe(gt));
+		xe_pm_runtime_put(xe);
 }
 
 void xe_gt_suspend_prepare(struct xe_gt *gt)
diff --git a/drivers/gpu/drm/xe/xe_gt.h b/drivers/gpu/drm/xe/xe_gt.h
index 4150aa594f05f..65a4655b0994d 100644
--- a/drivers/gpu/drm/xe/xe_gt.h
+++ b/drivers/gpu/drm/xe/xe_gt.h
@@ -6,8 +6,6 @@
 #ifndef _XE_GT_H_
 #define _XE_GT_H_
 
-#include <linux/fault-inject.h>
-
 #include <drm/drm_util.h>
 
 #include "xe_device.h"
@@ -38,12 +36,6 @@
 	xe_gt_is_media_type(gt_) ? MEDIA_VER(xe) : GRAPHICS_VER(xe); \
 })
 
-extern struct fault_attr gt_reset_failure;
-static inline bool xe_fault_inject_gt_reset(void)
-{
-	return IS_ENABLED(CONFIG_DEBUG_FS) && should_fail(&gt_reset_failure, 1);
-}
-
 struct xe_gt *xe_gt_alloc(struct xe_tile *tile);
 int xe_gt_init_early(struct xe_gt *gt);
 int xe_gt_init(struct xe_gt *gt);
@@ -137,10 +129,8 @@ static inline bool xe_gt_is_media_type(struct xe_gt *gt)
 
 static inline bool xe_gt_is_usm_hwe(struct xe_gt *gt, struct xe_hw_engine *hwe)
 {
-	struct xe_device *xe = gt_to_xe(gt);
-
-	return xe->info.has_usm && hwe->class == XE_ENGINE_CLASS_COPY &&
-		hwe->instance == gt->usm.reserved_bcs_instance;
+	return hwe->class == XE_ENGINE_CLASS_COPY &&
+	       (gt->usm.paging_logical_mask & BIT(hwe->logical_instance));
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_gt_mcr.c b/drivers/gpu/drm/xe/xe_gt_mcr.c
index 04f0098070a4f..a97b236dab7c9 100644
--- a/drivers/gpu/drm/xe/xe_gt_mcr.c
+++ b/drivers/gpu/drm/xe/xe_gt_mcr.c
@@ -404,8 +404,7 @@ static unsigned int dss_per_group(struct xe_gt *gt)
 	 * Some older platforms don't have tables or don't have complete tables.
 	 * Newer platforms should always have the required info.
 	 */
-	if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 2000 &&
-	    !gt_to_xe(gt)->info.force_execlist)
+	if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 2000)
 		xe_gt_err(gt, "Slice/Subslice counts missing from hwconfig table; using typical fallback values\n");
 
 	if (gt_to_xe(gt)->info.platform == XE_PVC)
@@ -507,7 +506,7 @@ void xe_gt_mcr_init_early(struct xe_gt *gt)
 	spin_lock_init(&gt->mcr_lock);
 
 	if (gt->info.type == XE_GT_TYPE_MEDIA) {
-		drm_WARN_ON(&xe->drm, MEDIA_VER(xe) < 13);
+		xe_gt_WARN_ON(gt, MEDIA_VER(xe) < 13);
 
 		if (MEDIA_VER(xe) >= 30) {
 			gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table;
@@ -662,9 +661,9 @@ bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt,
 
 	for (int type = 0; type < IMPLICIT_STEERING; type++) {
 		if (reg_in_steering_type_ranges(gt, reg, type)) {
-			drm_WARN(&gt_to_xe(gt)->drm, !gt->steering[type].initialized,
-				 "Uninitialized usage of MCR register %s/%#x\n",
-				 xe_steering_types[type].name, reg.addr);
+			xe_gt_WARN(gt, !gt->steering[type].initialized,
+				   "Uninitialized usage of MCR register %s/%#x\n",
+				   xe_steering_types[type].name, reg.addr);
 
 			*group = gt->steering[type].group_target;
 			*instance = gt->steering[type].instance_target;
@@ -679,9 +678,9 @@ bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt,
 	 * Not found in a steering table and not a register with implicit
 	 * steering. Just steer to 0/0 as a guess and raise a warning.
 	 */
-	drm_WARN(&gt_to_xe(gt)->drm, true,
-		 "Did not find MCR register %#x in any MCR steering table\n",
-		 reg.addr);
+	xe_gt_WARN(gt, true,
+		   "Did not find MCR register %#x in any MCR steering table\n",
+		   reg.addr);
 	*group = 0;
 	*instance = 0;
 
@@ -710,7 +709,7 @@ static void mcr_lock(struct xe_gt *gt) __acquires(&gt->mcr_lock)
 		ret = xe_mmio_wait32(&gt->mmio, STEER_SEMAPHORE, 0x1, 0x1, 10, NULL,
 				     true);
 
-	drm_WARN_ON_ONCE(&xe->drm, ret == -ETIMEDOUT);
+	xe_gt_WARN_ON_ONCE(gt, ret == -ETIMEDOUT);
 }
 
 static void mcr_unlock(struct xe_gt *gt) __releases(&gt->mcr_lock)
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c
index ffa27f66bba78..0f242db775e1c 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c
@@ -381,10 +381,11 @@ static ssize_t sched_group_engines_read(struct file *file, char __user *buf,
 
 	if (group < num_groups) {
 		for_each_hw_engine(hwe, gt, id) {
-			u8 guc_class = xe_engine_class_to_guc_class(hwe->class);
+			u8 guc_class = xe_hwe_to_guc_class(hwe);
+			u16 guc_logical_instance = xe_hwe_guc_logical_instance(hwe);
 			u32 mask = groups[group].engines[guc_class];
 
-			if (mask & BIT(hwe->logical_instance)) {
+			if (mask & BIT(guc_logical_instance)) {
 				strlcat(engines, hwe->name, sizeof(engines));
 				strlcat(engines, " ", sizeof(engines));
 			}
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c
index e8458d63742dd..cdfe194926d3e 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c
@@ -471,7 +471,8 @@ static void pf_sched_group_media_slices(struct xe_gt *gt, struct guc_sched_group
 		return;
 
 	for_each_hw_engine(hwe, gt, id) {
-		u8 guc_class = xe_engine_class_to_guc_class(hwe->class);
+		u8 guc_class = xe_hwe_to_guc_class(hwe);
+		u16 guc_logical_instance = xe_hwe_guc_logical_instance(hwe);
 
 		switch (hwe->class) {
 		case XE_ENGINE_CLASS_VIDEO_DECODE:
@@ -490,7 +491,7 @@ static void pf_sched_group_media_slices(struct xe_gt *gt, struct guc_sched_group
 			slice = 0;
 		}
 
-		values[slice_to_group[slice]].engines[guc_class] |= BIT(hwe->logical_instance);
+		values[slice_to_group[slice]].engines[guc_class] |= BIT(guc_logical_instance);
 	}
 
 	*groups = values;
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
index 0cd9d77f33513..37899fcf5b227 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
@@ -658,6 +658,33 @@ static int vf_cache_sched_groups_status(struct xe_gt *gt)
 	return 0;
 }
 
+static int vf_cache_num_paging_engines(struct xe_gt *gt)
+{
+	struct xe_guc *guc = &gt->uc.guc;
+	struct xe_uc_fw_version guc_version;
+	u32 value = 0;
+	int err;
+
+	xe_gt_sriov_vf_guc_versions(gt, NULL, &guc_version);
+
+	if (MAKE_GUC_VER_STRUCT(guc_version) < MAKE_GUC_VER(1, 36, 0))
+		return 0;
+
+	err = guc_action_query_single_klv32(guc, GUC_KLV_GLOBAL_CFG_NUM_PAGING_ENGINE_INSTANCES_KEY,
+					    &value);
+	if (unlikely(err)) {
+		xe_gt_sriov_err(gt,
+				"Failed to obtain the number of paging instances (%pe)\n",
+				ERR_PTR(err));
+		return err;
+	}
+
+	gt->sriov.vf.runtime.num_paging_engine_instances = value;
+
+	xe_gt_sriov_dbg(gt, "num_paging_engines %u\n", value);
+	return 0;
+}
+
 /**
  * xe_gt_sriov_vf_query_config - Query SR-IOV config data over MMIO.
  * @gt: the &xe_gt
@@ -694,6 +721,10 @@ int xe_gt_sriov_vf_query_config(struct xe_gt *gt)
 	if (has_gmdid(xe))
 		vf_cache_gmdid(gt);
 
+	err = vf_cache_num_paging_engines(gt);
+	if (unlikely(err))
+		return err;
+
 	return 0;
 }
 
@@ -731,6 +762,22 @@ u16 xe_gt_sriov_vf_guc_ids(struct xe_gt *gt)
 	return gt->sriov.vf.self_config.num_ctxs;
 }
 
+/**
+ * xe_gt_sriov_vf_paging_engines - Return the number of paging engine instances
+ * @gt: the &xe_gt
+ *
+ * This function is for VF use only.
+ *
+ * Return: number of GuC paging engine instances configured by the PF.
+ */
+u32 xe_gt_sriov_vf_paging_engines(struct xe_gt *gt)
+{
+	xe_gt_assert(gt, IS_SRIOV_VF(gt_to_xe(gt)));
+	xe_gt_assert(gt, gt->sriov.vf.guc_version.major);
+
+	return gt->sriov.vf.runtime.num_paging_engine_instances;
+}
+
 static int relay_action_handshake(struct xe_gt *gt, u32 *major, u32 *minor)
 {
 	u32 request[VF2PF_HANDSHAKE_REQUEST_MSG_LEN] = {
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
index 79878f21b1dae..d171a8242a342 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
@@ -31,6 +31,7 @@ u32 xe_gt_sriov_vf_gmdid(struct xe_gt *gt);
 u16 xe_gt_sriov_vf_guc_ids(struct xe_gt *gt);
 u64 xe_gt_sriov_vf_lmem(struct xe_gt *gt);
 bool xe_gt_sriov_vf_sched_groups_enabled(struct xe_gt *gt);
+u32 xe_gt_sriov_vf_paging_engines(struct xe_gt *gt);
 
 u32 xe_gt_sriov_vf_read32(struct xe_gt *gt, struct xe_reg reg);
 void xe_gt_sriov_vf_write32(struct xe_gt *gt, struct xe_reg reg, u32 val);
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h
index 80562ffadb169..466f0abd9c281 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf_types.h
@@ -29,6 +29,10 @@ struct xe_gt_sriov_vf_runtime {
 	u32 gmdid;
 	/** @uses_sched_groups: whether PF enabled sched groups or not. */
 	bool uses_sched_groups;
+	/**
+	 * @num_paging_engine_instances: number of configured paging engines.
+	 */
+	u32 num_paging_engine_instances;
 	/** @regs_size: size of runtime register array. */
 	u32 regs_size;
 	/** @num_regs: number of runtime registers in the array. */
diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h
index e5588c88800af..a8bbfbdf38494 100644
--- a/drivers/gpu/drm/xe/xe_gt_types.h
+++ b/drivers/gpu/drm/xe/xe_gt_types.h
@@ -144,6 +144,11 @@ struct xe_gt {
 		u8 id;
 		/** @info.has_indirect_ring_state: GT has indirect ring state support */
 		u8 has_indirect_ring_state:1;
+		/**
+		 * @info.has_uncorrectable_error_reporting: GT has uncorrectable
+		 * error reporting support
+		 */
+		u8 has_uncorrectable_error_reporting:1;
 		/**
 		 * @info.has_xe2_blt_instructions: GT supports Xe2-style MEM_SET
 		 * and MEM_COPY blitter functionality.  Note that despite the
@@ -230,10 +235,16 @@ struct xe_gt {
 		 */
 		struct xe_sa_manager *bb_pool;
 		/**
-		 * @usm.reserved_bcs_instance: reserved BCS instance used for USM
-		 * operations (e.g. migrations, fixing page tables)
+		 * @usm.paging_hwe0: The first designated paging engine.
+		 * This is some reserved BCS instance used for USM operations
+		 * (e.g. migrations, fixing page tables)
+		 */
+		struct xe_hw_engine *paging_hwe0;
+		/**
+		 * @usm.paging_logical_mask: logical mask of paging engines.
+		 * Should be densely populated.
 		 */
-		u16 reserved_bcs_instance;
+		u32 paging_logical_mask;
 	} usm;
 
 	/** @ordered_wq: used to serialize GT resets and TDRs */
diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c
index 4023700ff2a96..4286bd05c6861 100644
--- a/drivers/gpu/drm/xe/xe_guc.c
+++ b/drivers/gpu/drm/xe/xe_guc.c
@@ -12,6 +12,7 @@
 
 #include "abi/guc_actions_abi.h"
 #include "abi/guc_errors_abi.h"
+#include "abi/guc_klvs_abi.h"
 #include "regs/xe_gt_regs.h"
 #include "regs/xe_gtt_defs.h"
 #include "regs/xe_guc_regs.h"
@@ -101,6 +102,14 @@ static u32 guc_ctl_feature_flags(struct xe_guc *guc)
 	if (xe_device_is_l2_flush_optimized(xe) && xe_gt_is_media_type(guc_to_gt(guc)))
 		flags |= GUC_CTL_ENABLE_L2FLUSH_OPT;
 
+	/*
+	 * On GuC firmware 70.66 and above, the GUC_FEATURE_KLV_DISABLE_MULTI_QUEUE
+	 * Feature KLV is used instead.
+	 */
+	if (!xe_configfs_get_enable_multi_queue(to_pci_dev(xe->drm.dev)) &&
+	    !GUC_FIRMWARE_VER_AT_LEAST(guc, 70, 66))
+		flags |= GUC_CTL_DISABLE_MULTI_QUEUE;
+
 	return flags;
 }
 
@@ -641,6 +650,15 @@ int xe_guc_opt_in_features_enable(struct xe_guc *guc)
 	if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 7, 0))
 		klvs[count++] = PREP_GUC_KLV_TAG(OPT_IN_FEATURE_EXT_CAT_ERR_TYPE);
 
+	/*
+	 * The uncorrectable local error notification opt-in was added in
+	 * GuC v70.38.0, which maps to compatibility version v1.18.0.
+	 */
+	if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 18, 0) &&
+	    guc_to_gt(guc)->info.has_uncorrectable_error_reporting)
+		klvs[count++] =
+			PREP_GUC_KLV_TAG(OPT_IN_FEATURE_UNCORRECTABLE_LOCAL_ERROR_NOTIFICATION);
+
 	if (supports_dynamic_ics(guc))
 		klvs[count++] = PREP_GUC_KLV_TAG(OPT_IN_FEATURE_DYNAMIC_INHIBIT_CONTEXT_SWITCH);
 
@@ -1846,6 +1864,58 @@ bool xe_guc_using_main_gamctrl_queues(struct xe_guc *guc)
 	return GT_VER(gt) >= 35;
 }
 
+bool xe_guc_has_paging_engine(struct xe_guc *guc)
+{
+	struct xe_gt *gt = guc_to_gt(guc);
+	struct xe_device *xe = gt_to_xe(gt);
+
+	/*
+	 * On newer platforms the GuC now has a dedicated engine class for the
+	 * special PAGING engine, which is the driver reserved BCS engine used
+	 * for KMD paging/binding operations. GuC requires KMD to refer to this
+	 * using the special PAGING engine class. Note that there is no new hw
+	 * engine here, this is purely a sw view in the GuC itself, which we
+	 * need to respect.
+	 */
+
+	if (IS_SRIOV_VF(xe))
+		return xe_gt_sriov_vf_paging_engines(gt);
+
+	return xe->info.platform >= XE_NOVALAKE_S &&
+	       GUC_FIRMWARE_VER_AT_LEAST(guc, 70, 69, 0);
+}
+
+/**
+ * xe_hwe_guc_logical_instance - Get the GuC-aligned logical instance of a
+ * hardware engine.
+ * @hwe: Hardware engine.
+ *
+ * For GuC backend usage, we should no longer use the raw logical instance
+ * directly. This helper must be used to retrieve the logical instance of the
+ * hardware engine, taking care of any necessary adjustments (such as the GuC
+ * PAGING engine mapping). This is assumed to be used in conjunction with the
+ * GuC engine class.
+ *
+ * Return: Logical instance, taking into account for stuff like GuC PAGING
+ * engine mapping.
+ */
+u16 xe_hwe_guc_logical_instance(struct xe_hw_engine *hwe)
+{
+	struct xe_gt *gt = hwe->gt;
+
+	if (xe_guc_has_paging_engine(&hwe->gt->uc.guc) &&
+	    xe_gt_is_usm_hwe(gt, hwe)) {
+		int shift = gt->usm.paging_hwe0->logical_instance;
+
+		xe_gt_assert(gt, shift <= hwe->logical_instance);
+
+		/* GUC_PAGING_CLASS:guc_logical_instance */
+		return hwe->logical_instance - shift;
+	}
+
+	return hwe->logical_instance;
+}
+
 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
 #include "tests/xe_guc_g2g_test.c"
 #endif
diff --git a/drivers/gpu/drm/xe/xe_guc.h b/drivers/gpu/drm/xe/xe_guc.h
index 02514914f4048..61e3ee19a59bf 100644
--- a/drivers/gpu/drm/xe/xe_guc.h
+++ b/drivers/gpu/drm/xe/xe_guc.h
@@ -62,31 +62,14 @@ void xe_guc_stop(struct xe_guc *guc);
 int xe_guc_start(struct xe_guc *guc);
 void xe_guc_declare_wedged(struct xe_guc *guc);
 bool xe_guc_using_main_gamctrl_queues(struct xe_guc *guc);
+bool xe_guc_has_paging_engine(struct xe_guc *guc);
 
 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
 int xe_guc_g2g_test_notification(struct xe_guc *guc, u32 *payload, u32 len);
 #endif
 
-static inline u16 xe_engine_class_to_guc_class(enum xe_engine_class class)
-{
-	switch (class) {
-	case XE_ENGINE_CLASS_RENDER:
-		return GUC_RENDER_CLASS;
-	case XE_ENGINE_CLASS_VIDEO_DECODE:
-		return GUC_VIDEO_CLASS;
-	case XE_ENGINE_CLASS_VIDEO_ENHANCE:
-		return GUC_VIDEOENHANCE_CLASS;
-	case XE_ENGINE_CLASS_COPY:
-		return GUC_BLITTER_CLASS;
-	case XE_ENGINE_CLASS_COMPUTE:
-		return GUC_COMPUTE_CLASS;
-	case XE_ENGINE_CLASS_OTHER:
-		return GUC_GSC_OTHER_CLASS;
-	default:
-		XE_WARN_ON(class);
-		return -1;
-	}
-}
+u16 xe_hwe_to_guc_class(struct xe_hw_engine *hwe);
+u16 xe_hwe_guc_logical_instance(struct xe_hw_engine *hwe);
 
 static inline struct xe_gt *guc_to_gt(struct xe_guc *guc)
 {
diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c
index c98454545a858..f0ac00586d3ac 100644
--- a/drivers/gpu/drm/xe/xe_guc_ads.c
+++ b/drivers/gpu/drm/xe/xe_guc_ads.c
@@ -16,6 +16,7 @@
 #include "regs/xe_gt_regs.h"
 #include "regs/xe_guc_regs.h"
 #include "xe_bo.h"
+#include "xe_configfs.h"
 #include "xe_gt.h"
 #include "xe_gt_ccs_mode.h"
 #include "xe_gt_mcr.h"
@@ -251,16 +252,42 @@ static size_t calculate_regset_size(struct xe_gt *gt)
 	return count * sizeof(struct guc_mmio_reg);
 }
 
-static u32 engine_enable_mask(struct xe_gt *gt, enum xe_engine_class class)
+static inline enum xe_engine_class guc_class_to_engine_class(u16 guc_class)
+{
+	switch (guc_class) {
+	case GUC_RENDER_CLASS:
+		return XE_ENGINE_CLASS_RENDER;
+	case GUC_VIDEO_CLASS:
+		return XE_ENGINE_CLASS_VIDEO_DECODE;
+	case GUC_VIDEOENHANCE_CLASS:
+		return XE_ENGINE_CLASS_VIDEO_ENHANCE;
+	case GUC_BLITTER_CLASS:
+	case GUC_PAGING_CLASS:
+		return XE_ENGINE_CLASS_COPY;
+	case GUC_COMPUTE_CLASS:
+		return XE_ENGINE_CLASS_COMPUTE;
+	case GUC_GSC_OTHER_CLASS:
+		return XE_ENGINE_CLASS_OTHER;
+	default:
+		XE_WARN_ON(guc_class);
+		return -1;
+	}
+}
+
+static u32 engine_enable_mask(struct xe_gt *gt, u16 guc_class)
 {
 	struct xe_hw_engine *hwe;
 	enum xe_hw_engine_id id;
 	u32 mask = 0;
 
 	for_each_hw_engine(hwe, gt, id)
-		if (hwe->class == class)
+		if (xe_hwe_to_guc_class(hwe) == guc_class)
 			mask |= BIT(hwe->instance);
 
+	/* We expect at most one paging engine per GuC instance, for now */
+	if (guc_class == GUC_PAGING_CLASS)
+		xe_gt_assert(gt, !mask || is_power_of_2(mask));
+
 	return mask;
 }
 
@@ -268,10 +295,13 @@ static size_t calculate_golden_lrc_size(struct xe_guc_ads *ads)
 {
 	struct xe_gt *gt = ads_to_gt(ads);
 	size_t total_size = 0, alloc_size, real_size;
-	int class;
+	u16 guc_class;
+
+	for (guc_class = 0; guc_class <= GUC_LAST_ENGINE_CLASS; ++guc_class) {
+		enum xe_engine_class class =
+			guc_class_to_engine_class(guc_class);
 
-	for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) {
-		if (!engine_enable_mask(gt, class))
+		if (!engine_enable_mask(gt, guc_class))
 			continue;
 
 		real_size = xe_gt_lrc_size(gt, class);
@@ -364,6 +394,28 @@ static void guc_waklv_init(struct xe_guc_ads *ads)
 		guc_waklv_enable(ads, NULL, 0, &offset, &remain,
 				 GUC_WA_KLV_CLR_CS_INDIRECT_RING_STATE_IF_IDLE_AT_CTX_REG);
 
+	if (XE_GT_WA(gt, 22022079272) && GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 62))
+		guc_waklv_enable(ads, NULL, 0, &offset, &remain, GUC_WA_KLV_REMAP_RANGED_TLB_INV);
+
+	/* The GuC does not enable the sem_tok_64 feature on NVL-S */
+	if (XE_GT_WA(gt, 16029897822) && gt_to_xe(gt)->info.platform != XE_NOVALAKE_S &&
+	    GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 69))
+		guc_waklv_enable(ads, NULL, 0, &offset, &remain,
+				 GUC_WA_KLV_IGNORE_MMIO_READ_SEM_TOKEN_64);
+
+	/*
+	 * On GuC firmware 70.66 and above, use the Feature KLV (shared with the
+	 * WA KLV buffer); older firmware uses GUC_CTL_DISABLE_MULTI_QUEUE in
+	 * the init params instead.
+	 */
+	if (!xe_configfs_get_enable_multi_queue(to_pci_dev(gt_to_xe(gt)->drm.dev)) &&
+	    GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 66)) {
+		u32 data = 1;
+
+		guc_waklv_enable(ads, &data, 1, &offset, &remain,
+				 GUC_FEATURE_KLV_DISABLE_MULTI_QUEUE);
+	}
+
 	size = guc_ads_waklv_size(ads) - remain;
 	if (!size)
 		return;
@@ -463,20 +515,36 @@ static void fill_engine_enable_masks(struct xe_gt *gt,
 				     struct iosys_map *info_map)
 {
 	struct xe_device *xe = gt_to_xe(gt);
+	u16 guc_class;
+
+	for (guc_class = 0; guc_class <= GUC_LAST_ENGINE_CLASS; ++guc_class)
+		info_map_write(xe, info_map, engine_enabled_masks[guc_class],
+			       engine_enable_mask(gt, guc_class));
+}
 
-	info_map_write(xe, info_map, engine_enabled_masks[GUC_RENDER_CLASS],
-		       engine_enable_mask(gt, XE_ENGINE_CLASS_RENDER));
-	info_map_write(xe, info_map, engine_enabled_masks[GUC_BLITTER_CLASS],
-		       engine_enable_mask(gt, XE_ENGINE_CLASS_COPY));
-	info_map_write(xe, info_map, engine_enabled_masks[GUC_VIDEO_CLASS],
-		       engine_enable_mask(gt, XE_ENGINE_CLASS_VIDEO_DECODE));
-	info_map_write(xe, info_map,
-		       engine_enabled_masks[GUC_VIDEOENHANCE_CLASS],
-		       engine_enable_mask(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE));
-	info_map_write(xe, info_map, engine_enabled_masks[GUC_COMPUTE_CLASS],
-		       engine_enable_mask(gt, XE_ENGINE_CLASS_COMPUTE));
-	info_map_write(xe, info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS],
-		       engine_enable_mask(gt, XE_ENGINE_CLASS_OTHER));
+u16 xe_hwe_to_guc_class(struct xe_hw_engine *hwe)
+{
+	if (xe_guc_has_paging_engine(&hwe->gt->uc.guc) &&
+	    xe_gt_is_usm_hwe(hwe->gt, hwe))
+		return GUC_PAGING_CLASS;
+
+	switch (hwe->class) {
+	case XE_ENGINE_CLASS_RENDER:
+		return GUC_RENDER_CLASS;
+	case XE_ENGINE_CLASS_VIDEO_DECODE:
+		return GUC_VIDEO_CLASS;
+	case XE_ENGINE_CLASS_VIDEO_ENHANCE:
+		return GUC_VIDEOENHANCE_CLASS;
+	case XE_ENGINE_CLASS_COPY:
+		return GUC_BLITTER_CLASS;
+	case XE_ENGINE_CLASS_COMPUTE:
+		return GUC_COMPUTE_CLASS;
+	case XE_ENGINE_CLASS_OTHER:
+		return GUC_GSC_OTHER_CLASS;
+	default:
+		XE_WARN_ON(hwe->class);
+		return -1;
+	}
 }
 
 /*
@@ -491,15 +559,14 @@ static void guc_golden_lrc_init(struct xe_guc_ads *ads)
 			offsetof(struct __guc_ads_blob, system_info));
 	size_t alloc_size, real_size;
 	u32 addr_ggtt, offset;
-	int class;
+	u16 guc_class;
 
 	offset = guc_ads_golden_lrc_offset(ads);
 	addr_ggtt = xe_bo_ggtt_addr(ads->bo) + offset;
 
-	for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) {
-		u8 guc_class;
-
-		guc_class = xe_engine_class_to_guc_class(class);
+	for (guc_class = 0; guc_class <= GUC_LAST_ENGINE_CLASS; ++guc_class) {
+		enum xe_engine_class class =
+			guc_class_to_engine_class(guc_class);
 
 		if (!info_map_read(xe, &info_map,
 				   engine_enabled_masks[guc_class]))
@@ -548,11 +615,14 @@ static void guc_mapping_table_init(struct xe_gt *gt,
 	guc_mapping_table_init_invalid(gt, info_map);
 
 	for_each_hw_engine(hwe, gt, id) {
+		u16 guc_logical_instance;
 		u8 guc_class;
 
-		guc_class = xe_engine_class_to_guc_class(hwe->class);
+		guc_class = xe_hwe_to_guc_class(hwe);
+		guc_logical_instance = xe_hwe_guc_logical_instance(hwe);
+
 		info_map_write(xe, info_map,
-			       mapping_table[guc_class][hwe->logical_instance],
+			       mapping_table[guc_class][guc_logical_instance],
 			       hwe->instance);
 	}
 }
@@ -580,6 +650,9 @@ static u32 guc_get_capture_engine_mask(struct xe_gt *gt, struct iosys_map *info_
 	case GUC_CAPTURE_LIST_CLASS_GSC_OTHER:
 		mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS]);
 		break;
+	case GUC_CAPTURE_LIST_CLASS_PAGING:
+		mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_PAGING_CLASS]);
+		break;
 	default:
 		mask = 0;
 	}
@@ -805,7 +878,7 @@ static void guc_mmio_reg_state_init(struct xe_guc_ads *ads)
 		 * 2. Record in the header (ads.reg_state_list) the address
 		 * location and number of entries
 		 */
-		gc = xe_engine_class_to_guc_class(hwe->class);
+		gc = xe_hwe_to_guc_class(hwe);
 		ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].address, addr);
 		ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].count, count);
 
@@ -948,14 +1021,13 @@ static void guc_golden_lrc_populate(struct xe_guc_ads *ads)
 			offsetof(struct __guc_ads_blob, system_info));
 	size_t total_size = 0, alloc_size, real_size;
 	u32 offset;
-	int class;
+	u16 guc_class;
 
 	offset = guc_ads_golden_lrc_offset(ads);
 
-	for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) {
-		u8 guc_class;
-
-		guc_class = xe_engine_class_to_guc_class(class);
+	for (guc_class = 0; guc_class <= GUC_LAST_ENGINE_CLASS; ++guc_class) {
+		enum xe_engine_class class =
+			guc_class_to_engine_class(guc_class);
 
 		if (!info_map_read(xe, &info_map,
 				   engine_enabled_masks[guc_class]))
diff --git a/drivers/gpu/drm/xe/xe_guc_capture.c b/drivers/gpu/drm/xe/xe_guc_capture.c
index 1a019137ddf42..82df19b304e17 100644
--- a/drivers/gpu/drm/xe/xe_guc_capture.c
+++ b/drivers/gpu/drm/xe/xe_guc_capture.c
@@ -249,6 +249,8 @@ static const struct __guc_mmio_reg_descr_group xe_hpg_lists[] = {
 	MAKE_REGLIST(xe_blt_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_BLITTER),
 	MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_GSC_OTHER),
 	MAKE_REGLIST(xe_lp_gsc_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_GSC_OTHER),
+	MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_PAGING),
+	MAKE_REGLIST(xe_blt_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_PAGING),
 	{}
 };
 
@@ -265,6 +267,8 @@ static const struct __guc_mmio_reg_descr_group xe3p_lists[] = {
 	MAKE_REGLIST(xe_blt_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_BLITTER),
 	MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_GSC_OTHER),
 	MAKE_REGLIST(xe_lp_gsc_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_GSC_OTHER),
+	MAKE_REGLIST(empty_regs_list, PF, ENGINE_CLASS, GUC_CAPTURE_LIST_CLASS_PAGING),
+	MAKE_REGLIST(xe_blt_inst_regs, PF, ENGINE_INSTANCE, GUC_CAPTURE_LIST_CLASS_PAGING),
 	{}
 };
 static const char * const capture_list_type_names[] = {
@@ -279,6 +283,7 @@ static const char * const capture_engine_class_names[] = {
 	"VideoEnhance",
 	"Blitter",
 	"GSC-Other",
+	"Paging",
 };
 
 struct __guc_capture_ads_cache {
@@ -440,7 +445,7 @@ static void guc_capture_alloc_steered_lists(struct xe_guc *guc)
 	 * to be extended
 	 */
 	for_each_hw_engine(hwe, gt, id) {
-		if (xe_engine_class_to_guc_capture_class(hwe->class) ==
+		if (xe_hwe_to_guc_capture_class(hwe) ==
 		    GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE) {
 			has_rcs_ccs = true;
 			break;
@@ -772,6 +777,10 @@ size_t xe_guc_capture_ads_input_worst_size(struct xe_guc *guc)
 	total_size = PAGE_SIZE;	/* Pad a page in front for empty lists */
 	for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) {
 		for (j = 0; j < GUC_CAPTURE_LIST_CLASS_MAX; j++) {
+			if (!xe_guc_has_paging_engine(guc) &&
+			    j == GUC_CAPTURE_LIST_CLASS_PAGING)
+				continue;
+
 			if (xe_guc_capture_getlistsize(guc, i,
 						       GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS,
 						       j, &class_size) < 0)
@@ -818,7 +827,7 @@ static int guc_capture_output_size_est(struct xe_guc *guc)
 	for_each_hw_engine(hwe, gt, id) {
 		enum guc_capture_list_class_type capture_class;
 
-		capture_class = xe_engine_class_to_guc_capture_class(hwe->class);
+		capture_class = xe_hwe_to_guc_capture_class(hwe);
 		capture_size += sizeof(struct guc_state_capture_group_header_t) +
 					 (3 * sizeof(struct guc_state_capture_header_t));
 
@@ -1626,7 +1635,7 @@ xe_engine_manual_capture(struct xe_hw_engine *hwe, struct xe_hw_engine_snapshot
 	if (!new)
 		return;
 
-	capture_class = xe_engine_class_to_guc_capture_class(hwe->class);
+	capture_class = xe_hwe_to_guc_capture_class(hwe);
 	for (type = GUC_STATE_CAPTURE_TYPE_GLOBAL; type < GUC_STATE_CAPTURE_TYPE_MAX; type++) {
 		struct gcap_reg_list_info *reginfo = &new->reginfo[type];
 		/*
@@ -1668,7 +1677,7 @@ xe_engine_manual_capture(struct xe_hw_engine *hwe, struct xe_hw_engine_snapshot
 		}
 	}
 
-	new->eng_class = xe_engine_class_to_guc_class(hwe->class);
+	new->eng_class = xe_hwe_to_guc_class(hwe);
 	new->eng_inst = hwe->instance;
 	new->guc_id = guc_id;
 	new->lrca = lrca;
@@ -1832,7 +1841,7 @@ void xe_engine_snapshot_print(struct xe_hw_engine_snapshot *snapshot, struct drm
 
 	xe_gt_assert(gt, snapshot->hwe);
 
-	capture_class = xe_engine_class_to_guc_capture_class(snapshot->hwe->class);
+	capture_class = xe_hwe_to_guc_capture_class(snapshot->hwe);
 
 	drm_printf(p, "%s (physical), logical instance=%d\n",
 		   snapshot->name ? snapshot->name : "",
@@ -1904,7 +1913,7 @@ xe_guc_capture_get_matching_and_lock(struct xe_exec_queue *q)
 	for_each_hw_engine(hwe, q->gt, id) {
 		if (hwe != q->hwe)
 			continue;
-		guc_class = xe_engine_class_to_guc_class(hwe->class);
+		guc_class = xe_hwe_to_guc_class(hwe);
 		break;
 	}
 
diff --git a/drivers/gpu/drm/xe/xe_guc_capture.h b/drivers/gpu/drm/xe/xe_guc_capture.h
index dca97d52b192d..fcd4f12985368 100644
--- a/drivers/gpu/drm/xe/xe_guc_capture.h
+++ b/drivers/gpu/drm/xe/xe_guc_capture.h
@@ -28,6 +28,8 @@ static inline enum guc_capture_list_class_type xe_guc_class_to_capture_class(u16
 	case GUC_VIDEOENHANCE_CLASS:
 	case GUC_BLITTER_CLASS:
 		return class;
+	case GUC_PAGING_CLASS:
+		return GUC_CAPTURE_LIST_CLASS_PAGING;
 	default:
 		XE_WARN_ON(class);
 		return GUC_CAPTURE_LIST_CLASS_MAX;
@@ -35,9 +37,9 @@ static inline enum guc_capture_list_class_type xe_guc_class_to_capture_class(u16
 }
 
 static inline enum guc_capture_list_class_type
-xe_engine_class_to_guc_capture_class(enum xe_engine_class class)
+xe_hwe_to_guc_capture_class(struct xe_hw_engine *hwe)
 {
-	return xe_guc_class_to_capture_class(xe_engine_class_to_guc_class(class));
+	return xe_guc_class_to_capture_class(xe_hwe_to_guc_class(hwe));
 }
 
 void xe_guc_capture_process(struct xe_guc *guc);
diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c
index 21e0dad9a4815..fe70c0fd85c52 100644
--- a/drivers/gpu/drm/xe/xe_guc_ct.c
+++ b/drivers/gpu/drm/xe/xe_guc_ct.c
@@ -1065,6 +1065,11 @@ static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action,
 	xe_gt_assert(gt, g2h_len || !num_g2h);
 	lockdep_assert_held(&ct->lock);
 
+	if (xe_device_wedged(ct_to_xe(ct))) {
+		ret = -ENOTRECOVERABLE;
+		goto out;
+	}
+
 	if (unlikely(ct->ctbs.h2g.info.broken)) {
 		ret = -EPIPE;
 		goto out;
@@ -1236,6 +1241,36 @@ static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
 	return ret;
 }
 
+/**
+ * xe_guc_ct_send - Send an HXG message to the GuC over CT
+ * @ct: the &xe_guc_ct
+ * @action: dword array with the HXG message (can't be NULL)
+ * @len: length of the HXG message in dwords (can't be 0)
+ * @g2h_len: G2H response space to reserve in dwords, or 0
+ * @num_g2h: number of G2H messages expected, or 0
+ *
+ * Return codes from the non-blocking send helpers are:
+ *
+ * * -ENOTRECOVERABLE: the xe device is wedged. Stop submitting new GuC work; the
+ *   request cannot make progress until the device is recovered.
+ * * -EPIPE: the H2G CTB is marked broken. The channel stays unusable until the
+ *   CT is restarted, which clears the broken flag.
+ * * -ENODEV: the CT channel is disabled, messages not expected in this state.
+ *   Don't retry until it is enabled again.
+ * * -ECANCELED: the CT channel is stopped or a GT recovery is pending; the
+ *   message was dropped. Often benign. Cancel-tolerant callers (e.g. TLB
+ *   invalidations, GuC submission) rely on the stop/start flow to recover;
+ *   others should retry once the CT is re-enabled or the reset/recovery
+ *   completes.
+ * * -EDEADLK: no CTB room and the wait for space timed out. The send helpers
+ *   have already requested an async GT reset before returning this error.
+ *
+ * -ENOMEM may also be returned if an internal allocation fails; the blocking
+ * xe_guc_ct_send_recv() path retries that allocation. -EBUSY and
+ * -EAGAIN are internal flow-control results handled by the send helpers.
+ *
+ * Return: 0 on success, or a negative error code on failure.
+ */
 int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
 		   u32 g2h_len, u32 num_g2h)
 {
@@ -1388,7 +1423,7 @@ static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
 	if (g2h_fence.fail) {
 		if (g2h_fence.cancel) {
 			xe_gt_dbg(gt, "H2G request %#x canceled!\n", action[0]);
-			ret = -ECANCELED;
+			ret = xe_device_wedged(ct_to_xe(ct)) ? -ENOTRECOVERABLE : -ECANCELED;
 			goto unlock;
 		}
 		xe_gt_err(gt, "H2G request %#x failed: error %#x hint %#x\n",
@@ -1661,6 +1696,9 @@ static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
 		ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload,
 								 adj_len);
 		break;
+	case XE_GUC_ACTION_NOTIFY_UNCORRECTABLE_LOCAL_ERROR:
+		ret = xe_guc_uncorrectable_error_handler(guc, payload, adj_len);
+		break;
 	case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
 		ret = xe_guc_pagefault_handler(guc, payload, adj_len);
 		break;
@@ -1724,6 +1762,9 @@ static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path)
 	xe_gt_assert(gt, xe_guc_ct_initialized(ct));
 	lockdep_assert_held(&ct->fast_lock);
 
+	if (xe_device_wedged(xe))
+		return -ENOTRECOVERABLE;
+
 	if (ct->state == XE_GUC_CT_STATE_DISABLED)
 		return -ENODEV;
 
diff --git a/drivers/gpu/drm/xe/xe_guc_engine_activity.c b/drivers/gpu/drm/xe/xe_guc_engine_activity.c
index 2b99c1ebdd586..a782be57caadb 100644
--- a/drivers/gpu/drm/xe/xe_guc_engine_activity.c
+++ b/drivers/gpu/drm/xe/xe_guc_engine_activity.c
@@ -27,7 +27,8 @@ static struct iosys_map engine_activity_map(struct xe_guc *guc, struct xe_hw_eng
 {
 	struct xe_guc_engine_activity *engine_activity = &guc->engine_activity;
 	struct engine_activity_buffer *buffer;
-	u16 guc_class = xe_engine_class_to_guc_class(hwe->class);
+	u16 guc_class = xe_hwe_to_guc_class(hwe);
+	u16 guc_logical_instance = xe_hwe_guc_logical_instance(hwe);
 	size_t offset;
 
 	if (engine_activity->num_functions) {
@@ -39,7 +40,7 @@ static struct iosys_map engine_activity_map(struct xe_guc *guc, struct xe_hw_eng
 	}
 
 	offset += offsetof(struct guc_engine_activity_data,
-			  engine_activity[guc_class][hwe->logical_instance]);
+			  engine_activity[guc_class][guc_logical_instance]);
 
 	return IOSYS_MAP_INIT_OFFSET(&buffer->activity_bo->vmap, offset);
 }
@@ -150,9 +151,10 @@ static struct engine_activity *hw_engine_to_engine_activity(struct xe_hw_engine
 {
 	struct xe_guc *guc = &hwe->gt->uc.guc;
 	struct engine_activity_group *eag = &guc->engine_activity.eag[index];
-	u16 guc_class = xe_engine_class_to_guc_class(hwe->class);
+	u16 guc_class = xe_hwe_to_guc_class(hwe);
+	u16 guc_logical_instance = xe_hwe_guc_logical_instance(hwe);
 
-	return &eag->engine[guc_class][hwe->logical_instance];
+	return &eag->engine[guc_class][guc_logical_instance];
 }
 
 static u64 cpu_ns_to_guc_tsc_tick(ktime_t ns, u32 freq)
@@ -473,7 +475,7 @@ void xe_guc_engine_activity_enable_stats(struct xe_guc *guc)
 
 	ret = enable_engine_activity_stats(guc);
 	if (ret)
-		xe_gt_err(guc_to_gt(guc), "failed to enable activity stats%d\n", ret);
+		xe_gt_err(guc_to_gt(guc), "failed to enable activity stats: %pe\n", ERR_PTR(ret));
 	else
 		engine_activity_set_cpu_ts(guc, 0);
 }
diff --git a/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h b/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h
index e5e53b421f29f..acdc24d1a6bd2 100644
--- a/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h
@@ -10,6 +10,7 @@
 #include <linux/workqueue.h>
 
 #include "xe_gpu_scheduler_types.h"
+#include "xe_hw_fence_types.h"
 
 struct dma_fence;
 struct xe_exec_queue;
@@ -24,6 +25,10 @@ struct xe_guc_exec_queue {
 	struct rcu_head rcu;
 	/** @sched: GPU scheduler for this xe_exec_queue */
 	struct xe_gpu_scheduler sched;
+	/**
+	 * @name: Scheduler timeline name, kept with @sched until RCU free.
+	 */
+	char name[MAX_FENCE_NAME_LEN];
 	/** @entity: Scheduler entity for this xe_exec_queue */
 	struct xe_sched_entity entity;
 	/**
@@ -49,6 +54,13 @@ struct xe_guc_exec_queue {
 	wait_queue_head_t suspend_wait;
 	/** @suspend_pending: a suspend of the exec_queue is pending */
 	bool suspend_pending;
+	/**
+	 * @suspend_count: Reference count of active suspend requests. The
+	 * exec_queue remains suspended while this is non-zero, allowing
+	 * multiple concurrent callers to independently hold a suspend without
+	 * prematurely re-enabling the queue. Protected by @sched.msg_lock.
+	 */
+	int suspend_count;
 	/**
 	 * @needs_cleanup: Needs a cleanup message during VF post migration
 	 * recovery.
diff --git a/drivers/gpu/drm/xe/xe_guc_fwif.h b/drivers/gpu/drm/xe/xe_guc_fwif.h
index 3fbda4798cff3..971b850f21360 100644
--- a/drivers/gpu/drm/xe/xe_guc_fwif.h
+++ b/drivers/gpu/drm/xe/xe_guc_fwif.h
@@ -68,6 +68,7 @@ struct guc_update_exec_queue_policy {
 #define   GUC_CTL_MAIN_GAMCTRL_QUEUES	BIT(9)
 #define   GUC_CTL_DISABLE_SCHEDULER	BIT(14)
 #define   GUC_CTL_ENABLE_L2FLUSH_OPT	BIT(15)
+#define   GUC_CTL_DISABLE_MULTI_QUEUE	BIT(24)
 
 #define GUC_CTL_DEBUG			3
 #define   GUC_LOG_VERBOSITY		REG_GENMASK(1, 0)
diff --git a/drivers/gpu/drm/xe/xe_guc_klv_helpers.c b/drivers/gpu/drm/xe/xe_guc_klv_helpers.c
index 97600edda837a..dc8612d761e3c 100644
--- a/drivers/gpu/drm/xe/xe_guc_klv_helpers.c
+++ b/drivers/gpu/drm/xe/xe_guc_klv_helpers.c
@@ -4,14 +4,27 @@
  */
 
 #include <linux/bitfield.h>
+#include <kunit/static_stub.h>
 #include <drm/drm_print.h>
 
 #include "abi/guc_klvs_abi.h"
+#include "abi/xe_driver_klvs_abi.h"
 #include "xe_guc_klv_helpers.h"
 #include "xe_guc_klv_thresholds_set.h"
 
 #define make_u64(hi, lo) ((u64)((u64)(u32)(hi) << 32 | (u32)(lo)))
 
+static bool is_group_key(u16 key)
+{
+	KUNIT_STATIC_STUB_REDIRECT(is_group_key, key);
+	return false;
+}
+
+static bool is_reserved_key(u16 key)
+{
+	return in_range(key, GUC_KLV_RESERVED_RANGE_START, GUC_KLV_RESERVED_RANGE_LEN);
+}
+
 /**
  * xe_guc_klv_key_to_string - Convert KLV key into friendly name.
  * @key: the `GuC KLV`_ key
@@ -24,6 +37,8 @@ const char *xe_guc_klv_key_to_string(u16 key)
 	/* GuC Global Config KLVs */
 	case GUC_KLV_GLOBAL_CFG_GROUP_SCHEDULING_AVAILABLE_KEY:
 		return "group_scheduling_available";
+	case GUC_KLV_GLOBAL_CFG_NUM_PAGING_ENGINE_INSTANCES_KEY:
+		return "num_paging_engine_instances";
 	/* VGT POLICY keys */
 	case GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_KEY:
 		return "sched_if_idle";
@@ -71,11 +86,63 @@ const char *xe_guc_klv_key_to_string(u16 key)
 	MAKE_XE_GUC_KLV_THRESHOLDS_SET(define_threshold_key_to_string_case)
 #undef define_threshold_key_to_string_case
 
+	/* driver KLVs */
+	case MIGRATION_KLV_DEVICE_DEVID_KEY:
+		return "migration_devid";
+	case MIGRATION_KLV_DEVICE_REVID_KEY:
+		return "migration_revid";
+
 	default:
+		if (is_reserved_key(key))
+			return "(reserved)";
 		return "(unknown)";
 	}
 }
 
+/**
+ * xe_guc_klv_print_one() - Print single `GuC KLV`_.
+ * @key: KLV key
+ * @len: KLV length (in u32 dwords) of the KLV @value
+ * @value: KLV value (as array of @len u32 dwords)
+ * @p: the &drm_printer
+ *
+ * The buffer may contain more than one KLV.
+ */
+void xe_guc_klv_print_one(u16 key, u16 len, const u32 *value, struct drm_printer *p)
+{
+	const char *name = xe_guc_klv_key_to_string(key);
+
+	if (is_group_key(key)) {
+		struct drm_printer gp = drm_line_printer(p, name, 0);
+
+		drm_printf(p, "{ key %#06x : group %u dwords } # %s\n",
+			   key, len, name);
+
+		/* print group recursively */
+		xe_guc_klv_print(value, len, &gp);
+		return;
+	}
+
+	switch (len) {
+	case 0:
+		drm_printf(p, "{ key %#06x : no value } # %s\n", key, name);
+		break;
+	case 1:
+		drm_printf(p, "{ key %#06x : 32b value %u } # %s\n",
+			   key, value[0], name);
+		break;
+	case 2:
+		drm_printf(p, "{ key %#06x : 64b value %#llx } # %s\n",
+			   key, make_u64(value[1], value[0]), name);
+		break;
+	default:
+		drm_printf(p, "{ key %#06x : %zu bytes %*ph } # %s\n",
+			   key, len * sizeof(u32), (int)(len * sizeof(u32)),
+			   value, name);
+		break;
+	}
+}
+
 /**
  * xe_guc_klv_print - Print content of the buffer with `GuC KLV`_.
  * @klvs: the buffer with KLVs
@@ -101,26 +168,7 @@ void xe_guc_klv_print(const u32 *klvs, u32 num_dwords, struct drm_printer *p)
 			return;
 		}
 
-		switch (len) {
-		case 0:
-			drm_printf(p, "{ key %#06x : no value } # %s\n",
-				   key, xe_guc_klv_key_to_string(key));
-			break;
-		case 1:
-			drm_printf(p, "{ key %#06x : 32b value %u } # %s\n",
-				   key, klvs[0], xe_guc_klv_key_to_string(key));
-			break;
-		case 2:
-			drm_printf(p, "{ key %#06x : 64b value %#llx } # %s\n",
-				   key, make_u64(klvs[1], klvs[0]),
-				   xe_guc_klv_key_to_string(key));
-			break;
-		default:
-			drm_printf(p, "{ key %#06x : %zu bytes %*ph } # %s\n",
-				   key, len * sizeof(u32), (int)(len * sizeof(u32)),
-				   klvs, xe_guc_klv_key_to_string(key));
-			break;
-		}
+		xe_guc_klv_print_one(key, len, klvs, p);
 
 		klvs += len;
 		num_dwords -= len;
@@ -155,3 +203,175 @@ int xe_guc_klv_count(const u32 *klvs, u32 num_dwords)
 
 	return num_dwords ? -ENODATA : num_klvs;
 }
+
+static size_t to_num_bytes(u16 dwords)
+{
+	return dwords * sizeof(u32);
+}
+
+static u16 to_num_dwords(size_t size)
+{
+	return round_up(size, sizeof(u32)) / sizeof(u32);
+}
+
+/**
+ * xe_guc_klv_encode_u32() - Encode 32-bit value as KLV.
+ * @klvs: the buffer where to place KLV
+ * @avail: number of dwords (u32) available in the buffer
+ * @key: key to be used
+ * @value: value to be encoded
+ *
+ * Return: pointer to the buffer location past the encoded KLV or
+ *         an ERR_PTR if there was no space to encode the KLV.
+ */
+u32 *xe_guc_klv_encode_u32(u32 *klvs, u32 avail, u16 key, u32 value)
+{
+	u16 len = to_num_dwords(sizeof(u32));
+
+	if (IS_ERR(klvs))
+		return klvs;
+
+	if (avail < GUC_KLV_LEN_MIN + len)
+		return ERR_PTR(-ENOSPC);
+
+	*klvs++ = PREP_GUC_KLV(key, len);
+	*klvs++ = value;
+	return klvs;
+}
+
+/**
+ * xe_guc_klv_encode_u64() - Encode 64-bit value as KLV.
+ * @klvs: the buffer where to place KLV
+ * @avail: number of dwords (u32) available in the buffer
+ * @key: key to be used
+ * @value: value to be encoded
+ *
+ * Return: pointer to the buffer location past the encoded KLV or
+ *         an ERR_PTR if there was no space to encode the KLV.
+ */
+u32 *xe_guc_klv_encode_u64(u32 *klvs, u32 avail, u16 key, u64 value)
+{
+	u16 len = to_num_dwords(sizeof(u64));
+
+	if (IS_ERR(klvs))
+		return klvs;
+
+	if (avail < GUC_KLV_LEN_MIN + len)
+		return ERR_PTR(-ENOSPC);
+
+	*klvs++ = PREP_GUC_KLV(key, len);
+	*klvs++ = lower_32_bits(value);
+	*klvs++ = upper_32_bits(value);
+	return klvs;
+}
+
+/**
+ * xe_guc_klv_encode_string() - Encode string as KLV.
+ * @klvs: the buffer where to place KLV
+ * @avail: number of dwords (u32) available in the buffer
+ * @key: key to be used
+ * @s: string to be encoded
+ *
+ * Return: pointer to the buffer location past the encoded KLV or
+ *         an ERR_PTR if there was no space to encode the KLV.
+ */
+u32 *xe_guc_klv_encode_string(u32 *klvs, u32 avail, u16 key, const char *s)
+{
+	size_t longest = to_num_bytes(FIELD_MAX(GUC_KLV_0_LEN));
+	size_t size = strnlen(s, longest) + 1; /* \0 */
+	u16 len = to_num_dwords(size);
+
+	if (IS_ERR(klvs))
+		return klvs;
+
+	if (size > longest)
+		return ERR_PTR(-E2BIG);
+
+	if (avail < GUC_KLV_LEN_MIN + len)
+		return ERR_PTR(-ENOSPC);
+
+	*klvs++ = PREP_GUC_KLV(key, len);
+	strscpy_pad((void *)klvs, s, to_num_bytes(len));
+	return klvs + len;
+}
+
+/**
+ * xe_guc_klv_encode_object() - Encode object using custom encoder as single KLV.
+ * @klvs: the buffer where to place KLV
+ * @avail: number of dwords (u32) available in the buffer
+ * @key: key to be used
+ * @obj: opaque object pointer
+ * @encoder: function pointer to the custom encoder
+ *
+ * Return: pointer to the buffer location past the encoded KLV or
+ *         an ERR_PTR if there was no space to encode the KLV.
+ */
+u32 *xe_guc_klv_encode_object(u32 *klvs, u32 avail, u16 key, const void *obj,
+			      u32 *(*encoder)(u32 *klvs, u32 avail, const void *obj))
+{
+	u32 *end;
+
+	if (IS_ERR(klvs))
+		return klvs;
+
+	if (avail < GUC_KLV_LEN_MIN)
+		return ERR_PTR(-ENOSPC);
+
+	if (avail > GUC_KLV_LEN_MIN + FIELD_MAX(GUC_KLV_0_LEN))
+		avail = GUC_KLV_LEN_MIN + FIELD_MAX(GUC_KLV_0_LEN);
+
+	end = encoder(klvs + GUC_KLV_LEN_MIN, avail - GUC_KLV_LEN_MIN, obj);
+	if (IS_ERR(end))
+		return end;
+
+	if (WARN_ON(end < klvs + GUC_KLV_LEN_MIN))
+		return ERR_PTR(-EPIPE);
+
+	if (WARN_ON(end > klvs + avail))
+		return ERR_PTR(-EFBIG);
+
+	*klvs = PREP_GUC_KLV(key, end - (klvs + GUC_KLV_LEN_MIN));
+	return end;
+}
+
+/**
+ * xe_guc_klv_parser() - Parse and decode stream of KLVs.
+ * @klvs: the buffer with KLVs
+ * @num_dwords: number of dwords (u32) available in the buffer
+ * @obj: opaque pointer to be used by the @decoder function
+ * @decoder: pointer to the decoder function
+ *
+ * Return: The sum of all results returned by the decoder or
+ *         an -errno on decoder or buffer failure.
+ */
+int xe_guc_klv_parser(const u32 *klvs, u32 num_dwords, void *obj,
+		      int (*decoder)(void *obj, u16 key, u16 len, const u32 *value))
+{
+	int total = 0;
+	int ret;
+
+	while (num_dwords >= GUC_KLV_LEN_MIN) {
+		u16 key = FIELD_GET(GUC_KLV_0_KEY, klvs[0]);
+		u16 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]);
+
+		klvs += GUC_KLV_LEN_MIN;
+		num_dwords -= GUC_KLV_LEN_MIN;
+
+		if (num_dwords < len)
+			return -ENODATA;
+
+		ret = decoder(obj, key, len, klvs);
+		if (ret < 0)
+			return ret;
+		total += ret;
+
+		klvs += len;
+		num_dwords -= len;
+	}
+
+	return total;
+}
+
+#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST)
+#include "tests/xe_guc_klv_helpers_kunit.c"
+#endif
diff --git a/drivers/gpu/drm/xe/xe_guc_klv_helpers.h b/drivers/gpu/drm/xe/xe_guc_klv_helpers.h
index c676d21c173bb..cf3b29adc9d6b 100644
--- a/drivers/gpu/drm/xe/xe_guc_klv_helpers.h
+++ b/drivers/gpu/drm/xe/xe_guc_klv_helpers.h
@@ -13,9 +13,19 @@ struct drm_printer;
 
 const char *xe_guc_klv_key_to_string(u16 key);
 
+void xe_guc_klv_print_one(u16 key, u16 len, const u32 *value, struct drm_printer *p);
 void xe_guc_klv_print(const u32 *klvs, u32 num_dwords, struct drm_printer *p);
 int xe_guc_klv_count(const u32 *klvs, u32 num_dwords);
 
+u32 *xe_guc_klv_encode_u32(u32 *klvs, u32 avail, u16 key, u32 value);
+u32 *xe_guc_klv_encode_u64(u32 *klvs, u32 avail, u16 key, u64 value);
+u32 *xe_guc_klv_encode_string(u32 *klvs, u32 avail, u16 key, const char *s);
+u32 *xe_guc_klv_encode_object(u32 *klvs, u32 avail, u16 key, const void *obj,
+			      u32 *(*encoder)(u32 *klvs, u32 avail, const void *obj));
+
+int xe_guc_klv_parser(const u32 *klvs, u32 num_dwords, void *obj,
+		      int (*decoder)(void *obj, u16 key, u16 len, const u32 *value));
+
 /**
  * PREP_GUC_KLV - Prepare KLV header value based on provided key and len.
  * @key: KLV key
diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c
index f5c3d8a97ec64..8aaed4fd13ea3 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit.c
+++ b/drivers/gpu/drm/xe/xe_guc_submit.c
@@ -10,6 +10,7 @@
 #include <linux/circ_buf.h>
 #include <linux/dma-fence-array.h>
 
+#include <drm/drm_drv.h>
 #include <drm/drm_managed.h>
 
 #include "abi/guc_actions_abi.h"
@@ -37,6 +38,7 @@
 #include "xe_macros.h"
 #include "xe_map.h"
 #include "xe_mocs.h"
+#include "xe_module.h"
 #include "xe_pm.h"
 #include "xe_ring_ops_types.h"
 #include "xe_sched_job.h"
@@ -232,17 +234,9 @@ static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q)
 static void guc_submit_sw_fini(struct drm_device *drm, void *arg)
 {
 	struct xe_guc *guc = arg;
-	struct xe_device *xe = guc_to_xe(guc);
 	struct xe_gt *gt = guc_to_gt(guc);
-	int ret;
-
-	ret = wait_event_timeout(guc->submission_state.fini_wq,
-				 xa_empty(&guc->submission_state.exec_queue_lookup),
-				 HZ * 5);
 
-	drain_workqueue(xe->destroy_wq);
-
-	xe_gt_assert(gt, ret);
+	xe_gt_assert(gt, xa_empty(&guc->submission_state.exec_queue_lookup));
 
 	xa_destroy(&guc->submission_state.exec_queue_lookup);
 }
@@ -319,8 +313,6 @@ int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids)
 
 	xa_init(&guc->submission_state.exec_queue_lookup);
 
-	init_waitqueue_head(&guc->submission_state.fini_wq);
-
 	primelockdep(guc);
 
 	guc->submission_state.initialized = true;
@@ -411,9 +403,6 @@ static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q,
 	xe_guc_id_mgr_release_locked(&guc->submission_state.idm,
 				     q->guc->id, q->width);
 
-	if (xa_empty(&guc->submission_state.exec_queue_lookup))
-		wake_up(&guc->submission_state.fini_wq);
-
 	mutex_unlock(&guc->submission_state.lock);
 }
 
@@ -972,6 +961,27 @@ static void __register_exec_queue(struct xe_guc *guc,
 	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0);
 }
 
+static u32 xe_hwe_guc_logical_to_submit_mask(struct xe_hw_engine *hwe, u32 logical_mask)
+{
+	struct xe_gt *gt = hwe->gt;
+
+	if (xe_gt_is_usm_hwe(gt, hwe)) {
+		int shift = gt->usm.paging_hwe0->logical_instance;
+		u32 paging_logical_mask = gt->usm.paging_logical_mask;
+
+		xe_gt_assert(gt, (logical_mask & paging_logical_mask) == logical_mask);
+
+		/*
+		 * Remap to GUC_PAGING_CLASS logical instance mask, if
+		 * applicable.
+		 */
+		if (xe_guc_has_paging_engine(&hwe->gt->uc.guc))
+			return logical_mask >> shift;
+	}
+
+	return logical_mask;
+}
+
 static void register_exec_queue(struct xe_exec_queue *q, int ctx_type)
 {
 	struct xe_guc *guc = exec_queue_to_guc(q);
@@ -984,8 +994,9 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type)
 
 	memset(&info, 0, sizeof(info));
 	info.context_idx = q->guc->id;
-	info.engine_class = xe_engine_class_to_guc_class(q->class);
-	info.engine_submit_mask = q->logical_mask;
+	info.engine_class = xe_hwe_to_guc_class(q->hwe);
+	info.engine_submit_mask =
+		xe_hwe_guc_logical_to_submit_mask(q->hwe, q->logical_mask);
 	info.hwlrca_lo = lower_32_bits(xe_lrc_descriptor(lrc));
 	info.hwlrca_hi = upper_32_bits(xe_lrc_descriptor(lrc));
 	info.flags = CONTEXT_REGISTRATION_FLAG_KMD |
@@ -1532,7 +1543,7 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 	 * If devcoredump not captured and GuC capture for the job is not ready
 	 * do manual capture first and decide later if we need to use it
 	 */
-	if (!exec_queue_killed(q) && !xe->devcoredump.captured &&
+	if (!xe_device_is_in_reset(xe) && !exec_queue_killed(q) && !xe->devcoredump.captured &&
 	    !xe_guc_capture_get_matching_and_lock(q)) {
 		/* take force wake before engine register manual capture */
 		CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
@@ -1554,8 +1565,8 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 	set_exec_queue_banned(q);
 
 	/* Kick job / queue off hardware */
-	if (!wedged && (exec_queue_enabled(primary) ||
-			exec_queue_pending_disable(primary))) {
+	if (!xe_device_is_in_reset(xe) && !wedged &&
+	    (exec_queue_enabled(primary) || exec_queue_pending_disable(primary))) {
 		int ret;
 
 		if (exec_queue_reset(primary))
@@ -1623,7 +1634,8 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 
 	trace_xe_sched_job_timedout(job);
 
-	if (!exec_queue_killed(q))
+	/* Do not access device if in reset */
+	if (!xe_device_is_in_reset(xe) && !exec_queue_killed(q))
 		xe_devcoredump(q, job,
 			       "Timedout job - seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx",
 			       xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),
@@ -1681,10 +1693,36 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 	return DRM_GPU_SCHED_STAT_NO_HANG;
 }
 
+static void guc_exec_queue_multi_queue_drop_suspend(struct xe_exec_queue *q);
+static int guc_exec_queue_suspend_wait_blocking(struct xe_exec_queue *q);
+
 static void guc_exec_queue_fini(struct xe_exec_queue *q)
 {
 	struct xe_guc_exec_queue *ge = q->guc;
 	struct xe_guc *guc = exec_queue_to_guc(q);
+	struct drm_device *drm = &guc_to_xe(guc)->drm;
+
+	/*
+	 * A secondary can leave the group while still preempt suspended (e.g.
+	 * xe_vm_remove_compute_exec_queue() forces its preempt fence to signal,
+	 * which suspends it). It holds one forwarded suspend reference on the
+	 * primary, so drop it and resume the primary if it was the last member
+	 * that had it suspended. Primaries forward to nobody, so they don't need
+	 * this.
+	 *
+	 * First make sure the primary's forwarded suspend has completed. If the
+	 * secondary was killed/reset before its preempt fence worker ran, that
+	 * worker skips suspend_wait() (see preempt_fence_work_func()), leaving
+	 * the primary's suspend possibly in flight. drop_suspend() runs under a
+	 * spinlock and cannot wait, so drain it here with the uninterruptible
+	 * blocking wait; otherwise resuming the primary in drop_suspend() could
+	 * trip the !suspend_pending assert.
+	 */
+	if (xe_exec_queue_is_multi_queue_secondary(q)) {
+		if (READ_ONCE(q->guc->suspend_count))
+			guc_exec_queue_suspend_wait_blocking(q);
+		guc_exec_queue_multi_queue_drop_suspend(q);
+	}
 
 	if (xe_exec_queue_is_multi_queue_secondary(q)) {
 		struct xe_exec_queue_group *group = q->multi_queue.group;
@@ -1703,36 +1741,52 @@ static void guc_exec_queue_fini(struct xe_exec_queue *q)
 	 * (timeline name).
 	 */
 	kfree_rcu(ge, rcu);
+
+	drm_dev_put(drm);
 }
 
-static void __guc_exec_queue_destroy_async(struct work_struct *w)
+static void guc_exec_queue_do_destroy(struct xe_exec_queue *q)
 {
-	struct xe_guc_exec_queue *ge =
-		container_of(w, struct xe_guc_exec_queue, destroy_async);
-	struct xe_exec_queue *q = ge->q;
+	struct xe_guc_exec_queue *ge = q->guc;
 	struct xe_guc *guc = exec_queue_to_guc(q);
+	struct xe_device *xe = guc_to_xe(guc);
+	struct drm_device *drm = &xe->drm;
 
-	guard(xe_pm_runtime)(guc_to_xe(guc));
-	trace_xe_exec_queue_destroy(q);
+	/*
+	 * guc_exec_queue_fini() drops the queue's drm_device ref.
+	 * Keep the device alive until the PM-runtime guard unwinds.
+	 */
+	drm_dev_get(drm);
+
+	scoped_guard(xe_pm_runtime, xe) {
+		trace_xe_exec_queue_destroy(q);
 
-	/* Confirm no work left behind accessing device structures */
-	cancel_delayed_work_sync(&ge->sched.base.work_tdr);
+		/* Confirm no work left behind accessing device structures */
+		cancel_delayed_work_sync(&ge->sched.base.work_tdr);
 
-	xe_exec_queue_fini(q);
+		xe_exec_queue_fini(q);
+	}
+
+	drm_dev_put(drm);
 }
 
-static void guc_exec_queue_destroy_async(struct xe_exec_queue *q)
+static void __guc_exec_queue_destroy_async(struct work_struct *w)
 {
-	struct xe_guc *guc = exec_queue_to_guc(q);
-	struct xe_device *xe = guc_to_xe(guc);
+	struct xe_guc_exec_queue *ge =
+		container_of(w, struct xe_guc_exec_queue, destroy_async);
+
+	guc_exec_queue_do_destroy(ge->q);
+}
 
+static void guc_exec_queue_destroy_async(struct xe_exec_queue *q)
+{
 	INIT_WORK(&q->guc->destroy_async, __guc_exec_queue_destroy_async);
 
 	/* We must block on kernel engines so slabs are empty on driver unload */
 	if (q->flags & EXEC_QUEUE_FLAG_PERMANENT || exec_queue_wedged(q))
-		__guc_exec_queue_destroy_async(&q->guc->destroy_async);
+		guc_exec_queue_do_destroy(q);
 	else
-		queue_work(xe->destroy_wq, &q->guc->destroy_async);
+		xe_destroy_wq_queue(&q->guc->destroy_async);
 }
 
 static void __guc_exec_queue_destroy(struct xe_guc *guc, struct xe_exec_queue *q)
@@ -1927,6 +1981,7 @@ static int guc_exec_queue_init(struct xe_exec_queue *q)
 {
 	struct xe_gpu_scheduler *sched;
 	struct xe_guc *guc = exec_queue_to_guc(q);
+	struct drm_device *drm = &guc_to_xe(guc)->drm;
 	struct workqueue_struct *submit_wq = NULL;
 	struct xe_guc_exec_queue *ge;
 	long timeout;
@@ -1938,6 +1993,8 @@ static int guc_exec_queue_init(struct xe_exec_queue *q)
 	if (!ge)
 		return -ENOMEM;
 
+	drm_dev_get(drm);
+
 	q->guc = ge;
 	ge->q = q;
 	init_rcu_head(&ge->rcu);
@@ -1955,6 +2012,8 @@ static int guc_exec_queue_init(struct xe_exec_queue *q)
 
 	xe_exec_queue_assign_name(q, q->guc->id);
 
+	strscpy(ge->name, q->name, sizeof(ge->name));
+
 	/*
 	 * Use primary queue's submit_wq for all secondary queues of a
 	 * multi queue group. This serialization avoids any locking around
@@ -1969,7 +2028,7 @@ static int guc_exec_queue_init(struct xe_exec_queue *q)
 	err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops,
 			    submit_wq, xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, 64,
 			    timeout, guc_to_gt(guc)->ordered_wq, NULL,
-			    q->name, gt_to_xe(q->gt)->drm.dev);
+			    ge->name, gt_to_xe(q->gt)->drm.dev);
 	if (err)
 		goto err_release_id;
 
@@ -2014,6 +2073,7 @@ static int guc_exec_queue_init(struct xe_exec_queue *q)
 	release_guc_id(guc, q);
 err_free:
 	kfree(ge);
+	drm_dev_put(drm);
 
 	return err;
 }
@@ -2163,23 +2223,147 @@ static int guc_exec_queue_set_multi_queue_priority(struct xe_exec_queue *q,
 	return 0;
 }
 
-static int guc_exec_queue_suspend(struct xe_exec_queue *q)
+/*
+ * Core suspend: take a suspend reference on @q and, on the first reference,
+ * disable its GuC context so the GPU is actually preempted. Caller must have
+ * ensured @q is not killed/banned/wedged. Returns true if this was the first
+ * suspend reference (the 0->1 transition).
+ */
+static bool __guc_exec_queue_suspend(struct xe_exec_queue *q)
 {
-	struct xe_gpu_scheduler *sched = &q->guc->sched;
-	struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_SUSPEND;
+	struct xe_guc_exec_queue *ge = q->guc;
+	struct xe_gpu_scheduler *sched = &ge->sched;
+	struct xe_sched_msg *msg = ge->static_msgs + STATIC_MSG_SUSPEND;
+	bool first;
 
-	if (exec_queue_killed_or_banned_or_wedged(q))
-		return -EINVAL;
+	xe_sched_msg_lock(sched);
+	first = (++ge->suspend_count == 1);
+	if (first) {
+		bool added = guc_exec_queue_try_add_msg(q, msg, SUSPEND);
+
+		/* slot must be free at 0->1 */
+		xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), added);
+		ge->suspend_pending = true;
+	}
+	xe_sched_msg_unlock(sched);
+
+	return first;
+}
+
+/*
+ * Core resume: drop a suspend reference on @q and, on the last reference,
+ * re-enable its GuC context. Returns true if this dropped the last suspend
+ * reference (the 1->0 transition).
+ */
+static bool __guc_exec_queue_resume(struct xe_exec_queue *q)
+{
+	struct xe_guc_exec_queue *ge = q->guc;
+	struct xe_gpu_scheduler *sched = &ge->sched;
+	struct xe_sched_msg *msg = ge->static_msgs + STATIC_MSG_RESUME;
+	struct xe_guc *guc = exec_queue_to_guc(q);
+	bool last;
 
 	xe_sched_msg_lock(sched);
-	if (guc_exec_queue_try_add_msg(q, msg, SUSPEND))
-		q->guc->suspend_pending = true;
+	xe_gt_assert(guc_to_gt(guc), !ge->suspend_pending);
+	xe_gt_assert(guc_to_gt(guc), ge->suspend_count > 0);
+	last = (--ge->suspend_count == 0);
+	if (last) {
+		bool added = guc_exec_queue_try_add_msg(q, msg, RESUME);
+
+		/* slot must be free at 1->0 */
+		xe_gt_assert(guc_to_gt(guc), added);
+	}
 	xe_sched_msg_unlock(sched);
 
+	return last;
+}
+
+static int guc_exec_queue_suspend(struct xe_exec_queue *q)
+{
+	if (exec_queue_killed_or_banned_or_wedged(q))
+		return -EINVAL;
+
+	/*
+	 * Non-multi-queue queues and multi-queue primaries suspend themselves
+	 * directly: their own msg_lock makes the suspend_count 0->1 transition
+	 * and the suspend_pending update atomic, so no group level serialization
+	 * is needed.
+	 */
+	if (!xe_exec_queue_is_multi_queue_secondary(q)) {
+		__guc_exec_queue_suspend(q);
+		return 0;
+	}
+
+	/*
+	 * A secondary's suspend is meaningless once the primary - which owns the
+	 * group's GuC context - is gone, so fail it too. This keeps the
+	 * secondary's effective state consistent with guc_exec_queue_reset_status(),
+	 * which already reports the primary's killed/banned/wedged state for
+	 * secondaries. A primary killed *after* this check is still handled at
+	 * message-processing time, where the SUSPEND is a no-op for a killed
+	 * context; this only covers an already-dead primary.
+	 */
+	if (exec_queue_killed_or_banned_or_wedged(xe_exec_queue_multi_queue_primary(q)))
+		return -EINVAL;
+
+	/*
+	 * A secondary doesn't interface with GuC: suspend it like any other
+	 * queue (its own suspend_count drives its internally handled scheduler
+	 * state) and, only on its own 0->1 transition, forward the suspend to the
+	 * primary so the GPU is actually preempted. Hold @suspend_lock so that
+	 * observing the secondary's transition and forwarding it to the primary
+	 * happen atomically; this keeps the primary's refcount paired with member
+	 * transitions even if the same secondary is suspended and resumed
+	 * concurrently across rebind cycles.
+	 */
+	scoped_guard(spinlock, &q->multi_queue.group->suspend_lock) {
+		if (__guc_exec_queue_suspend(q))
+			__guc_exec_queue_suspend(xe_exec_queue_multi_queue_primary(q));
+	}
+
 	return 0;
 }
 
-static int guc_exec_queue_suspend_wait(struct xe_exec_queue *q)
+static void guc_exec_queue_suspend_timeout_ban(struct xe_exec_queue *q)
+{
+	struct xe_guc *guc = exec_queue_to_guc(q);
+
+	xe_gt_warn(guc_to_gt(guc),
+		   "Suspend fence, guc_id=%d, failed to respond, banning queue",
+		   q->guc->id);
+	/*
+	 * The GuC failed to respond to the suspend within the timeout. This is
+	 * not recoverable for this context, so ban it and tear it down via
+	 * cleanup rather than leave it suspended forever. __suspend_fence_signal
+	 * clears suspend_pending and wakes any waiter.
+	 *
+	 * @q is the primary here; it owns the group's GuC context, so a failure
+	 * to suspend it wedges the whole group. Ban and tear down the entire
+	 * group in the multi-queue case.
+	 */
+	if (xe_exec_queue_is_multi_queue(q)) {
+		set_exec_queue_group_banned(q);
+		__suspend_fence_signal(q);
+		xe_guc_exec_queue_group_trigger_cleanup(q);
+	} else {
+		set_exec_queue_banned(q);
+		__suspend_fence_signal(q);
+		xe_guc_exec_queue_trigger_cleanup(q);
+	}
+}
+
+/*
+ * Wait for @q's own suspend to complete: suspend_pending cleared, or the queue
+ * killed / GuC stopped. With @blocking, wait uninterruptibly and do not handle
+ * VF recovery (for callers that must complete on behalf of a possibly
+ * cross-process queue); otherwise wait interruptibly.
+ *
+ * Returns 0 on completion or -ETIME on timeout. Interruptible waits may also
+ * return -EAGAIN (VF recovery in progress, retry) or -ERESTARTSYS (aborted by a
+ * signal; suspend_pending may still be set, so callers must not resume()
+ * without re-confirming the suspend).
+ */
+static int guc_exec_queue_wait_suspend_done(struct xe_exec_queue *q, bool blocking)
 {
 	struct xe_guc *guc = exec_queue_to_guc(q);
 	struct xe_device *xe = guc_to_xe(guc);
@@ -2195,44 +2379,146 @@ static int guc_exec_queue_suspend_wait(struct xe_exec_queue *q)
 	 xe_guc_read_stopped(guc))
 
 retry:
-	if (IS_SRIOV_VF(xe))
+	if (blocking) {
+		if (IS_SRIOV_VF(xe))
+			ret = wait_event_timeout(guc->ct.wq, WAIT_COND, HZ * 5);
+		else
+			ret = wait_event_timeout(q->guc->suspend_wait, WAIT_COND,
+						 HZ * 5);
+	} else if (IS_SRIOV_VF(xe)) {
 		ret = wait_event_interruptible_timeout(guc->ct.wq, WAIT_COND ||
-						       vf_recovery(guc),
-						       HZ * 5);
-	else
+						       vf_recovery(guc), HZ * 5);
+	} else {
 		ret = wait_event_interruptible_timeout(q->guc->suspend_wait,
 						       WAIT_COND, HZ * 5);
+	}
 
-	if (vf_recovery(guc) && !xe_device_wedged((guc_to_xe(guc))))
+	if (!blocking && vf_recovery(guc) && !xe_device_wedged(xe))
 		return -EAGAIN;
 
-	if (!ret) {
-		xe_gt_warn(guc_to_gt(guc),
-			   "Suspend fence, guc_id=%d, failed to respond",
-			   q->guc->id);
-		/* XXX: Trigger GT reset? */
+	if (!ret)
 		return -ETIME;
-	} else if (IS_SRIOV_VF(xe) && !WAIT_COND) {
+	else if (!blocking && IS_SRIOV_VF(xe) && !WAIT_COND)
 		/* Corner case on RESFIX DONE where vf_recovery() changes */
 		goto retry;
-	}
 
 #undef WAIT_COND
 
 	return ret < 0 ? ret : 0;
 }
 
+static int guc_exec_queue_suspend_wait_common(struct xe_exec_queue *q, bool blocking)
+{
+	int ret;
+
+	/*
+	 * A secondary's suspend rides the sched-message worker (short-circuited,
+	 * no GuC round-trip) and so is not synchronous with
+	 * guc_exec_queue_suspend(): its own suspend_pending may still be set
+	 * here. Waiting on the primary alone is not sufficient - if the primary
+	 * was already suspended, the forward is a refcount-only transition that
+	 * queues no new primary SUSPEND and leaves the primary's suspend_pending
+	 * clear, so the primary wait would return immediately while the
+	 * secondary's suspend is still in flight, and a later resume() would trip
+	 * the secondary's !suspend_pending assert. So first wait for the
+	 * secondary's own suspend to complete, then wait on the primary.
+	 *
+	 * A timeout on either bans the queue (being multi-queue, that tears down
+	 * the whole group). A secondary suspend has no real GuC round-trip, so
+	 * its timeout is a software scheduler stall rather than a GuC fault, but
+	 * banning is still the safe recovery: otherwise the queue is left with
+	 * suspend_pending set and a subsequent resume() trips the !suspend_pending
+	 * assert.
+	 */
+	if (xe_exec_queue_is_multi_queue_secondary(q)) {
+		ret = guc_exec_queue_wait_suspend_done(q, blocking);
+		if (ret == -ETIME)
+			guc_exec_queue_suspend_timeout_ban(q);
+		if (ret)
+			return ret;
+	}
+
+	q = xe_exec_queue_multi_queue_primary(q);
+	ret = guc_exec_queue_wait_suspend_done(q, blocking);
+	if (ret == -ETIME)
+		guc_exec_queue_suspend_timeout_ban(q);
+
+	return ret;
+}
+
+static int guc_exec_queue_suspend_wait(struct xe_exec_queue *q)
+{
+	return guc_exec_queue_suspend_wait_common(q, false);
+}
+
+/*
+ * Uninterruptible variant of guc_exec_queue_suspend_wait() for callers that
+ * must complete the wait on behalf of a queue possibly owned by a different
+ * process (e.g. cleanup/undo paths). An interruptible wait could return
+ * -ERESTARTSYS if the calling task is signalled, leaving that queue suspended
+ * forever (cross-process DoS). VF recovery is deliberately not handled (no
+ * -EAGAIN) since a blocking caller cannot retry.
+ */
+static int guc_exec_queue_suspend_wait_blocking(struct xe_exec_queue *q)
+{
+	return guc_exec_queue_suspend_wait_common(q, true);
+}
+
 static void guc_exec_queue_resume(struct xe_exec_queue *q)
 {
-	struct xe_gpu_scheduler *sched = &q->guc->sched;
-	struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_RESUME;
-	struct xe_guc *guc = exec_queue_to_guc(q);
+	/*
+	 * Non-multi-queue queues and multi-queue primaries resume themselves
+	 * directly; their own msg_lock is sufficient.
+	 */
+	if (!xe_exec_queue_is_multi_queue_secondary(q)) {
+		__guc_exec_queue_resume(q);
+		return;
+	}
+
+	/*
+	 * Mirror of guc_exec_queue_suspend(): resume the secondary like any
+	 * other queue and, only on its own 1->0 transition, forward the resume
+	 * to the primary so the primary's GuC context is re-enabled once the
+	 * last member that suspended it resumes. @suspend_lock keeps the
+	 * secondary transition and the primary forward atomic.
+	 */
+	scoped_guard(spinlock, &q->multi_queue.group->suspend_lock) {
+		if (__guc_exec_queue_resume(q))
+			__guc_exec_queue_resume(xe_exec_queue_multi_queue_primary(q));
+	}
+}
 
-	xe_gt_assert(guc_to_gt(guc), !q->guc->suspend_pending);
+/*
+ * Drop a leaving secondary's forwarded suspend reference on the primary and
+ * resume the primary if this was the last member that had it suspended.
+ * See guc_exec_queue_fini().
+ */
+static void guc_exec_queue_multi_queue_drop_suspend(struct xe_exec_queue *q)
+{
+	scoped_guard(spinlock, &q->multi_queue.group->suspend_lock) {
+		struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
 
-	xe_sched_msg_lock(sched);
-	guc_exec_queue_try_add_msg(q, msg, RESUME);
-	xe_sched_msg_unlock(sched);
+		/*
+		 * A suspended secondary holds exactly one suspend reference on the
+		 * primary (forwarded on its 0->1 transition). If it leaves while
+		 * still suspended, release that reference so the primary is not
+		 * kept disabled forever.
+		 */
+		if (!READ_ONCE(q->guc->suspend_count))
+			break;
+
+		if (exec_queue_killed_or_banned_or_wedged(primary))
+			break;
+
+		/*
+		 * No suspend_wait() here (and we can't - suspend_lock is a
+		 * spinlock). guc_exec_queue_fini() has already drained the
+		 * primary's forwarded suspend with the blocking wait, so its
+		 * suspend has completed (suspend_pending cleared) by the time we
+		 * resume it here. __guc_exec_queue_resume() asserts this.
+		 */
+		__guc_exec_queue_resume(primary);
+	}
 }
 
 static bool guc_exec_queue_reset_status(struct xe_exec_queue *q)
@@ -2244,14 +2530,6 @@ static bool guc_exec_queue_reset_status(struct xe_exec_queue *q)
 	return exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q);
 }
 
-static bool guc_exec_queue_active(struct xe_exec_queue *q)
-{
-	struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
-
-	return exec_queue_enabled(primary) &&
-		!exec_queue_pending_disable(primary);
-}
-
 /*
  * All of these functions are an abstraction layer which other parts of Xe can
  * use to trap into the GuC backend. All of these functions, aside from init,
@@ -2269,9 +2547,9 @@ static const struct xe_exec_queue_ops guc_exec_queue_ops = {
 	.set_multi_queue_priority = guc_exec_queue_set_multi_queue_priority,
 	.suspend = guc_exec_queue_suspend,
 	.suspend_wait = guc_exec_queue_suspend_wait,
+	.suspend_wait_blocking = guc_exec_queue_suspend_wait_blocking,
 	.resume = guc_exec_queue_resume,
 	.reset_status = guc_exec_queue_reset_status,
-	.active = guc_exec_queue_active,
 };
 
 static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q)
@@ -3030,6 +3308,38 @@ int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg,
 	return 0;
 }
 
+int xe_guc_uncorrectable_error_handler(struct xe_guc *guc, u32 *msg, u32 len)
+{
+	struct xe_gt *gt = guc_to_gt(guc);
+	struct xe_exec_queue *q;
+	u32 guc_id;
+
+	if (unlikely(!len || len > 1))
+		return -EPROTO;
+
+	guc_id = msg[0];
+
+	if (guc_id == GUC_ID_UNKNOWN) {
+		xe_gt_err(gt, "GuC: Uncorrectable local error with unknown GuC id\n");
+		return 0;
+	}
+
+	q = g2h_exec_queue_lookup(guc, guc_id);
+	if (unlikely(!q))
+		return -EPROTO;
+
+	xe_gt_err(gt,
+		  "GuC: Uncorrectable local error! guc_id=%d class=%s, logical_mask=0x%x",
+		  guc_id, xe_hw_engine_class_to_str(q->class), q->logical_mask);
+
+	trace_xe_guc_uncorrectable_error(q);
+
+	/* Treat the same as engine reset */
+	xe_guc_exec_queue_reset_trigger_cleanup(q);
+
+	return 0;
+}
+
 int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len)
 {
 	struct xe_gt *gt = guc_to_gt(guc);
diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h
index b3839a90c142b..ccade320dc695 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit.h
+++ b/drivers/gpu/drm/xe/xe_guc_submit.h
@@ -34,6 +34,7 @@ int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len);
 int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len);
 int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg,
 					       u32 len);
+int xe_guc_uncorrectable_error_handler(struct xe_guc *guc, u32 *msg, u32 len);
 int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len);
 int xe_guc_error_capture_handler(struct xe_guc *guc, u32 *msg, u32 len);
 int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc *guc, u32 *msg, u32 len);
diff --git a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c
index cf6d106e6036a..046d0655122f1 100644
--- a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c
+++ b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c
@@ -208,9 +208,6 @@ static int send_tlb_inval_asid_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno,
 
 	lockdep_assert_held(&tlb_inval->seqno_lock);
 
-	if (guc_to_xe(guc)->info.force_execlist)
-		return -ECANCELED;
-
 	return send_tlb_inval_ppgtt(guc, seqno, start, end, asid,
 				    XE_GUC_TLB_INVAL_PAGE_SELECTIVE, prl_sa);
 }
@@ -228,9 +225,6 @@ static int send_tlb_inval_ctx_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno,
 
 	lockdep_assert_held(&tlb_inval->seqno_lock);
 
-	if (xe->info.force_execlist)
-		return -ECANCELED;
-
 	vm = xe_device_asid_to_vm(xe, asid);
 	if (IS_ERR(vm))
 		return PTR_ERR(vm);
diff --git a/drivers/gpu/drm/xe/xe_guc_types.h b/drivers/gpu/drm/xe/xe_guc_types.h
index c7b9642b41ba7..31a2acb63ac34 100644
--- a/drivers/gpu/drm/xe/xe_guc_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_types.h
@@ -100,8 +100,6 @@ struct xe_guc {
 		 * even initialized - before that not even the lock is valid
 		 */
 		bool initialized;
-		/** @submission_state.fini_wq: submit fini wait queue */
-		wait_queue_head_t fini_wq;
 	} submission_state;
 
 	/** @hwconfig: Hardware config state */
diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c
index 0b193c451a11d..010499766fce2 100644
--- a/drivers/gpu/drm/xe/xe_hw_engine.c
+++ b/drivers/gpu/drm/xe/xe_hw_engine.c
@@ -337,39 +337,41 @@ static bool xe_rtp_cfeg_wmtp_disabled(const struct xe_device *xe,
 	return xe_mmio_read32(&hwe->gt->mmio, XEHP_FUSE4) & CFEG_WMTP_DISABLE;
 }
 
+static u32 blit_cctl_val(struct xe_gt *gt, struct xe_hw_engine *hwe)
+{
+	return REG_FIELD_PREP(BLIT_CCTL_DST_MOCS_MASK, gt->mocs.uc_index) |
+		REG_FIELD_PREP(BLIT_CCTL_SRC_MOCS_MASK, gt->mocs.uc_index);
+}
+
+static const struct xe_rtp_table_sr lrc_setup = XE_RTP_TABLE_SR(
+	/*
+	 * Some blitter commands do not have a field for MOCS, those
+	 * commands will use MOCS index pointed by BLIT_CCTL.
+	 * BLIT_CCTL registers are needed to be programmed to un-cached.
+	 */
+	{ XE_RTP_NAME("BLIT_CCTL_default_MOCS"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1274),
+		       ENGINE_CLASS(COPY)),
+	  XE_RTP_ACTIONS(FIELD_SET_FUNC(BLIT_CCTL(0),
+					BLIT_CCTL_DST_MOCS_MASK |
+					BLIT_CCTL_SRC_MOCS_MASK,
+					blit_cctl_val,
+					XE_RTP_ACTION_FLAG(ENGINE_BASE)))
+	},
+	/* Disable WMTP if HW doesn't support it */
+	{ XE_RTP_NAME("DISABLE_WMTP_ON_UNSUPPORTED_HW"),
+	  XE_RTP_RULES(FUNC(xe_rtp_cfeg_wmtp_disabled)),
+	  XE_RTP_ACTIONS(FIELD_SET(CS_CHICKEN1(0),
+				   PREEMPT_GPGPU_LEVEL_MASK,
+				   PREEMPT_GPGPU_THREAD_GROUP_LEVEL)),
+	  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE)
+	},
+);
+
 static void
 hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe)
 {
-	struct xe_gt *gt = hwe->gt;
-	const u8 mocs_write_idx = gt->mocs.uc_index;
-	const u8 mocs_read_idx = gt->mocs.uc_index;
-	u32 blit_cctl_val = REG_FIELD_PREP(BLIT_CCTL_DST_MOCS_MASK, mocs_write_idx) |
-			    REG_FIELD_PREP(BLIT_CCTL_SRC_MOCS_MASK, mocs_read_idx);
 	struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe);
-	const struct xe_rtp_table_sr lrc_setup = XE_RTP_TABLE_SR(
-		/*
-		 * Some blitter commands do not have a field for MOCS, those
-		 * commands will use MOCS index pointed by BLIT_CCTL.
-		 * BLIT_CCTL registers are needed to be programmed to un-cached.
-		 */
-		{ XE_RTP_NAME("BLIT_CCTL_default_MOCS"),
-		  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1274),
-			       ENGINE_CLASS(COPY)),
-		  XE_RTP_ACTIONS(FIELD_SET(BLIT_CCTL(0),
-				 BLIT_CCTL_DST_MOCS_MASK |
-				 BLIT_CCTL_SRC_MOCS_MASK,
-				 blit_cctl_val,
-				 XE_RTP_ACTION_FLAG(ENGINE_BASE)))
-		},
-		/* Disable WMTP if HW doesn't support it */
-		{ XE_RTP_NAME("DISABLE_WMTP_ON_UNSUPPORTED_HW"),
-		  XE_RTP_RULES(FUNC(xe_rtp_cfeg_wmtp_disabled)),
-		  XE_RTP_ACTIONS(FIELD_SET(CS_CHICKEN1(0),
-					   PREEMPT_GPGPU_LEVEL_MASK,
-					   PREEMPT_GPGPU_THREAD_GROUP_LEVEL)),
-		  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE)
-		},
-	);
 
 	xe_rtp_process_to_sr(&ctx, &lrc_setup, &hwe->reg_lrc, true);
 }
@@ -385,86 +387,92 @@ void xe_hw_engine_setup_reg_lrc(struct xe_hw_engine *hwe)
 	xe_tuning_process_lrc(hwe);
 }
 
-static void
-hw_engine_setup_default_state(struct xe_hw_engine *hwe)
+/*
+ * RING_CMD_CCTL specifies the default MOCS entry that will be
+ * used by the command streamer when executing commands that
+ * don't have a way to explicitly specify a MOCS setting.
+ * The default should usually reference whichever MOCS entry
+ * corresponds to uncached behavior, although use of a WB cached
+ * entry is recommended by the spec in certain circumstances on
+ * specific platforms.
+ * Bspec: 72161
+ */
+static u32 ring_cmd_cctl_val(struct xe_gt *gt, struct xe_hw_engine *hwe)
 {
-	struct xe_gt *gt = hwe->gt;
 	struct xe_device *xe = gt_to_xe(gt);
+	u8 mocs_read_idx = gt->mocs.uc_index;
+
+	if (hwe->class == XE_ENGINE_CLASS_COMPUTE && IS_DGFX(xe) &&
+	    (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC))
+		mocs_read_idx = gt->mocs.wb_index;
+
+	return REG_FIELD_PREP(CMD_CCTL_WRITE_OVERRIDE_MASK, gt->mocs.uc_index) |
+		REG_FIELD_PREP(CMD_CCTL_READ_OVERRIDE_MASK, mocs_read_idx);
+}
+
+static const struct xe_rtp_table_sr engine_sr = XE_RTP_TABLE_SR(
+	{ XE_RTP_NAME("RING_CMD_CCTL_default_MOCS"),
+	  XE_RTP_RULES(FUNC(xe_rtp_match_always)),
+	  XE_RTP_ACTIONS(FIELD_SET_FUNC(RING_CMD_CCTL(0),
+					CMD_CCTL_WRITE_OVERRIDE_MASK |
+					CMD_CCTL_READ_OVERRIDE_MASK,
+					ring_cmd_cctl_val,
+					XE_RTP_ACTION_FLAG(ENGINE_BASE)))
+	},
+	{ XE_RTP_NAME("Disable HW status page updates for interrupts"),
+	  XE_RTP_RULES(FUNC(xe_rtp_match_always)),
+	  XE_RTP_ACTIONS(SET(RING_HWSTAM(0), ~0x0,
+			     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
+	},
+	{ XE_RTP_NAME("Disable engine 'legacy' mode"),
+	  XE_RTP_RULES(FUNC(xe_rtp_match_always)),
+	  XE_RTP_ACTIONS(SET(GFX_MODE(0), GFX_DISABLE_LEGACY_MODE,
+			     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
+	},
 	/*
-	 * RING_CMD_CCTL specifies the default MOCS entry that will be
-	 * used by the command streamer when executing commands that
-	 * don't have a way to explicitly specify a MOCS setting.
-	 * The default should usually reference whichever MOCS entry
-	 * corresponds to uncached behavior, although use of a WB cached
-	 * entry is recommended by the spec in certain circumstances on
-	 * specific platforms.
-	 * Bspec: 72161
+	 * To allow the GSC engine to go idle on MTL we need to enable
+	 * idle messaging and set the hysteresis value (we use 0xA=5us
+	 * as recommended in spec). On platforms after MTL this is
+	 * enabled by default.
 	 */
-	const u8 mocs_write_idx = gt->mocs.uc_index;
-	const u8 mocs_read_idx = hwe->class == XE_ENGINE_CLASS_COMPUTE && IS_DGFX(xe) &&
-				 (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC) ?
-				 gt->mocs.wb_index : gt->mocs.uc_index;
-	u32 ring_cmd_cctl_val = REG_FIELD_PREP(CMD_CCTL_WRITE_OVERRIDE_MASK, mocs_write_idx) |
-				REG_FIELD_PREP(CMD_CCTL_READ_OVERRIDE_MASK, mocs_read_idx);
+	{ XE_RTP_NAME("MTL GSCCS IDLE MSG enable"),
+	  XE_RTP_RULES(MEDIA_VERSION(1300), ENGINE_CLASS(OTHER)),
+	  XE_RTP_ACTIONS(CLR(RING_PSMI_CTL(0),
+			     IDLE_MSG_DISABLE,
+			     XE_RTP_ACTION_FLAG(ENGINE_BASE)),
+			 FIELD_SET(RING_PWRCTX_MAXCNT(0),
+				   IDLE_WAIT_TIME,
+				   0xA,
+				   XE_RTP_ACTION_FLAG(ENGINE_BASE)))
+	},
+	/* Enable Priority Mem Read */
+	{ XE_RTP_NAME("Priority_Mem_Read"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED)),
+	  XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0), CS_PRIORITY_MEM_READ,
+			     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
+	},
+	{ XE_RTP_NAME("Enable CCS Engine(s)"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1255, XE_RTP_END_VERSION_UNDEFINED),
+		       FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(RCU_MODE, RCU_MODE_CCS_ENABLE))
+	},
+	/* Use Fixed slice CCS mode */
+	{ XE_RTP_NAME("RCU_MODE_FIXED_SLICE_CCS_MODE"),
+	  XE_RTP_RULES(FUNC(xe_hw_engine_match_fixed_cslice_mode)),
+	  XE_RTP_ACTIONS(FIELD_SET(RCU_MODE, RCU_MODE_FIXED_SLICE_CCS_MODE,
+				   RCU_MODE_FIXED_SLICE_CCS_MODE))
+	},
+	{ XE_RTP_NAME("Enable MSI-X interrupt support"),
+	  XE_RTP_RULES(FUNC(xe_rtp_match_has_msix)),
+	  XE_RTP_ACTIONS(SET(GFX_MODE(0), GFX_MSIX_INTERRUPT_ENABLE,
+			     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
+	},
+);
+
+static void
+hw_engine_setup_default_state(struct xe_hw_engine *hwe)
+{
 	struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe);
-	const struct xe_rtp_table_sr engine_sr = XE_RTP_TABLE_SR(
-		{ XE_RTP_NAME("RING_CMD_CCTL_default_MOCS"),
-		  XE_RTP_RULES(FUNC(xe_rtp_match_always)),
-		  XE_RTP_ACTIONS(FIELD_SET(RING_CMD_CCTL(0),
-					   CMD_CCTL_WRITE_OVERRIDE_MASK |
-					   CMD_CCTL_READ_OVERRIDE_MASK,
-					   ring_cmd_cctl_val,
-					   XE_RTP_ACTION_FLAG(ENGINE_BASE)))
-		},
-		{ XE_RTP_NAME("Disable HW status page updates for interrupts"),
-		  XE_RTP_RULES(FUNC(xe_rtp_match_always)),
-		  XE_RTP_ACTIONS(SET(RING_HWSTAM(0), ~0x0,
-				     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
-		},
-		{ XE_RTP_NAME("Disable engine 'legacy' mode"),
-		  XE_RTP_RULES(FUNC(xe_rtp_match_always)),
-		  XE_RTP_ACTIONS(SET(GFX_MODE(0), GFX_DISABLE_LEGACY_MODE,
-				     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
-		},
-		/*
-		 * To allow the GSC engine to go idle on MTL we need to enable
-		 * idle messaging and set the hysteresis value (we use 0xA=5us
-		 * as recommended in spec). On platforms after MTL this is
-		 * enabled by default.
-		 */
-		{ XE_RTP_NAME("MTL GSCCS IDLE MSG enable"),
-		  XE_RTP_RULES(MEDIA_VERSION(1300), ENGINE_CLASS(OTHER)),
-		  XE_RTP_ACTIONS(CLR(RING_PSMI_CTL(0),
-				     IDLE_MSG_DISABLE,
-				     XE_RTP_ACTION_FLAG(ENGINE_BASE)),
-				 FIELD_SET(RING_PWRCTX_MAXCNT(0),
-					   IDLE_WAIT_TIME,
-					   0xA,
-					   XE_RTP_ACTION_FLAG(ENGINE_BASE)))
-		},
-		/* Enable Priority Mem Read */
-		{ XE_RTP_NAME("Priority_Mem_Read"),
-		  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED)),
-		  XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0), CS_PRIORITY_MEM_READ,
-				     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
-		},
-		{ XE_RTP_NAME("Enable CCS Engine(s)"),
-		  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1255, XE_RTP_END_VERSION_UNDEFINED),
-			       FUNC(xe_rtp_match_first_render_or_compute)),
-		  XE_RTP_ACTIONS(SET(RCU_MODE, RCU_MODE_CCS_ENABLE))
-		},
-		/* Use Fixed slice CCS mode */
-		{ XE_RTP_NAME("RCU_MODE_FIXED_SLICE_CCS_MODE"),
-		  XE_RTP_RULES(FUNC(xe_hw_engine_match_fixed_cslice_mode)),
-		  XE_RTP_ACTIONS(FIELD_SET(RCU_MODE, RCU_MODE_FIXED_SLICE_CCS_MODE,
-					   RCU_MODE_FIXED_SLICE_CCS_MODE))
-		},
-		{ XE_RTP_NAME("Enable MSI-X interrupt support"),
-		  XE_RTP_RULES(FUNC(xe_rtp_match_has_msix)),
-		  XE_RTP_ACTIONS(SET(GFX_MODE(0), GFX_MSIX_INTERRUPT_ENABLE,
-				     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
-		},
-	);
 
 	xe_rtp_process_to_sr(&ctx, &engine_sr, &hwe->reg_sr, false);
 }
@@ -639,10 +647,6 @@ static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe,
 			xe_hw_engine_enable_ring(hwe);
 	}
 
-	/* We reserve the highest BCS instance for USM */
-	if (xe->info.has_usm && hwe->class == XE_ENGINE_CLASS_COPY)
-		gt->usm.reserved_bcs_instance = hwe->instance;
-
 	/* Ensure IDLEDLY is lower than MAXCNT */
 	adjust_idledly(hwe);
 
@@ -654,20 +658,80 @@ static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe,
 	return err;
 }
 
-static void hw_engine_setup_logical_mapping(struct xe_gt *gt)
+static int hw_engine_setup_logical_and_paging_mapping(struct xe_gt *gt)
 {
+	struct xe_device *xe = gt_to_xe(gt);
+	unsigned int num_copy_engines = 0, num_paging_engines = 0;
+	unsigned int reserved_logical_bcs_start;
+	struct xe_hw_engine *hwe;
+	enum xe_hw_engine_id id;
 	int class;
 
+	for_each_hw_engine(hwe, gt, id)
+		if (hwe->class == XE_ENGINE_CLASS_COPY)
+			num_copy_engines++;
+
+	if (num_copy_engines && xe->info.has_usm)
+		num_paging_engines = 1;
+
+	if (IS_SRIOV_VF(xe)) {
+		u32 vf_num_paging_engines;
+
+		/*
+		 * PF could in theory reserve multiple paging engines, which
+		 * internally the submission/scheduling backend can load balance
+		 * from. Not something we currently expect, but we are at the
+		 * mercy of the PF, so we just need try our best to mirror the
+		 * paging configuration.
+		 */
+		vf_num_paging_engines = xe_gt_sriov_vf_paging_engines(gt);
+		if (vf_num_paging_engines) {
+			/* This should only be non-zero on NVL-S+ */
+			if (xe_gt_WARN_ON(gt, xe->info.platform < XE_NOVALAKE_S))
+				return -EINVAL;
+
+			num_paging_engines = vf_num_paging_engines;
+		}
+	}
+
+	if (xe_gt_WARN_ON(gt, num_paging_engines > num_copy_engines))
+		return -EINVAL;
+
+	/*
+	 * On PF, we just reserve the highest BCS instance for USM.
+	 *
+	 * Note: This is now a requirement going forward. The PF must ALWAYS
+	 * reserve BCS instances in top-down order, that way the VF has a chance
+	 * of discovering the physical BCS instance mappings for paging engines,
+	 * in conjunction with vf_num_paging_engines. In some places we might
+	 * only have the physical instance, and from hw pov there is no such
+	 * thing as a paging engine. For example, the page fault descriptor,
+	 * which comes directly from the hw, will use the physical engine
+	 * instance.
+	 */
+	reserved_logical_bcs_start = num_copy_engines - num_paging_engines;
+
 	/* FIXME: Doing a simple logical mapping that works for most hardware */
 	for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) {
-		struct xe_hw_engine *hwe;
-		enum xe_hw_engine_id id;
 		int logical_instance = 0;
 
-		for_each_hw_engine(hwe, gt, id)
-			if (hwe->class == class)
+		for_each_hw_engine(hwe, gt, id) {
+			if (hwe->class == class) {
 				hwe->logical_instance = logical_instance++;
+
+				if (class == XE_ENGINE_CLASS_COPY &&
+				    hwe->logical_instance >=
+					    reserved_logical_bcs_start) {
+					if (!gt->usm.paging_hwe0)
+						gt->usm.paging_hwe0 = hwe;
+					gt->usm.paging_logical_mask |=
+						BIT(hwe->logical_instance);
+				}
+			}
+		}
 	}
+
+	return 0;
 }
 
 static void read_media_fuses(struct xe_gt *gt)
@@ -886,7 +950,10 @@ int xe_hw_engines_init(struct xe_gt *gt)
 			return err;
 	}
 
-	hw_engine_setup_logical_mapping(gt);
+	err = hw_engine_setup_logical_and_paging_mapping(gt);
+	if (err)
+		return err;
+
 	err = xe_hw_engine_setup_groups(gt);
 	if (err)
 		return err;
@@ -1035,8 +1102,7 @@ bool xe_hw_engine_is_reserved(struct xe_hw_engine *hwe)
 	    hwe->logical_instance >= gt->ccs_mode)
 		return true;
 
-	return xe->info.has_usm && hwe->class == XE_ENGINE_CLASS_COPY &&
-		hwe->instance == gt->usm.reserved_bcs_instance;
+	return xe_gt_is_usm_hwe(gt, hwe);
 }
 
 const char *xe_hw_engine_class_to_str(enum xe_engine_class class)
diff --git a/drivers/gpu/drm/xe/xe_hw_engine_group.c b/drivers/gpu/drm/xe/xe_hw_engine_group.c
index 02cf32ae5aa9a..0804b426b6f9e 100644
--- a/drivers/gpu/drm/xe/xe_hw_engine_group.c
+++ b/drivers/gpu/drm/xe/xe_hw_engine_group.c
@@ -34,6 +34,15 @@ hw_engine_group_resume_lr_jobs_func(struct work_struct *w)
 		if (!xe_vm_in_fault_mode(q->vm))
 			continue;
 
+		/*
+		 * Only resume queues that were actually suspended. A queue whose
+		 * suspend() failed (e.g. killed/banned/wedged) was never
+		 * suspended, so it must not be resumed.
+		 */
+		if (!READ_ONCE(q->lr.suspended))
+			continue;
+
+		WRITE_ONCE(q->lr.suspended, false);
 		q->ops->resume(q);
 	}
 
@@ -140,7 +149,18 @@ int xe_hw_engine_group_add_exec_queue(struct xe_hw_engine_group *group, struct x
 		return err;
 
 	if (xe_vm_in_fault_mode(q->vm) && group->cur_mode == EXEC_MODE_DMA_FENCE) {
-		q->ops->suspend(q);
+		/*
+		 * suspend() can fail (e.g. killed/banned/wedged), leaving the
+		 * queue un-suspended. Propagate the failure so the queue is not
+		 * added; on failure nothing was suspended, so there is nothing to
+		 * undo. Only record the queue as suspended (and later resume it)
+		 * once suspend() has succeeded.
+		 */
+		err = q->ops->suspend(q);
+		if (err)
+			goto err_suspend;
+
+		WRITE_ONCE(q->lr.suspended, true);
 		err = q->ops->suspend_wait(q);
 		if (err)
 			goto err_suspend;
@@ -216,8 +236,22 @@ static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group
 			return -EAGAIN;
 
 		xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, 1);
+		/*
+		 * suspend() only fails when the queue is killed/banned/wedged.
+		 * Such a queue is being torn down (its removal from HW is handled
+		 * by the kill/ban teardown), so it is not a live fault-mode
+		 * context the mode switch must preempt. Skip it rather than
+		 * failing the switch, otherwise one dying sibling would block a
+		 * dma-fence submission on the healthy queues in the group. Only
+		 * queues recorded as suspended below are later waited on and
+		 * resumed.
+		 */
+		err = q->ops->suspend(q);
+		if (err)
+			continue;
+
+		WRITE_ONCE(q->lr.suspended, true);
 		need_resume = true;
-		q->ops->suspend(q);
 		gt = q->gt;
 	}
 
@@ -225,9 +259,13 @@ static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group
 		if (!xe_vm_in_fault_mode(q->vm))
 			continue;
 
+		/* Only wait on queues that were actually suspended above. */
+		if (!READ_ONCE(q->lr.suspended))
+			continue;
+
 		err = q->ops->suspend_wait(q);
 		if (err)
-			return err;
+			goto err_resume;
 	}
 
 	if (gt) {
@@ -240,6 +278,47 @@ static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group
 		xe_hw_engine_group_resume_faulting_lr_jobs(group);
 
 	return 0;
+
+err_resume:
+	/*
+	 * A suspend_wait() failed partway through the mode switch. Resume the
+	 * sibling queues that were already suspended in this call so they are
+	 * not left suspended forever.
+	 *
+	 * resume() requires the suspend to have completed (suspend_pending
+	 * cleared) or it trips the !suspend_pending assert. So skip the resume
+	 * when either:
+	 *  - suspend_wait_blocking() fails: on a GuC timeout it bans the queue
+	 *    and triggers cleanup, so the queue is being torn down; or
+	 *  - reset_status() is true: the queue was reset/killed/banned/wedged.
+	 *    suspend_wait() can return success in this case via its killed/
+	 *    stopped wait condition while suspend_pending is still set, and the
+	 *    queue is being torn down anyway, so its state is resolved by
+	 *    teardown rather than by a resume here.
+	 * In either case leave the queue marked suspended.
+	 *
+	 * Use the *blocking* (uninterruptible) wait here: the queues resumed on
+	 * this path may belong to a different process than the one that
+	 * triggered the mode switch. An interruptible suspend_wait() would
+	 * return -ERESTARTSYS if the triggering task is signalled, skip the
+	 * resume, and leave the other process's queue suspended forever
+	 * (cross-process DoS).
+	 */
+	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
+		if (!xe_vm_in_fault_mode(q->vm))
+			continue;
+
+		if (!READ_ONCE(q->lr.suspended))
+			continue;
+
+		if (q->ops->suspend_wait_blocking(q) || q->ops->reset_status(q))
+			continue;
+
+		WRITE_ONCE(q->lr.suspended, false);
+		q->ops->resume(q);
+	}
+
+	return err;
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_hw_engine_types.h b/drivers/gpu/drm/xe/xe_hw_engine_types.h
index 84c097da9b6f2..ff115ab429fb7 100644
--- a/drivers/gpu/drm/xe/xe_hw_engine_types.h
+++ b/drivers/gpu/drm/xe/xe_hw_engine_types.h
@@ -114,7 +114,12 @@ struct xe_hw_engine {
 	enum xe_engine_class class;
 	/** @instance: physical instance of this hw engine */
 	u16 instance;
-	/** @logical_instance: logical instance of this hw engine */
+	/**
+	 * @logical_instance: logical instance of this hw engine.
+	 *
+	 * Note: For GuC usage, always use xe_hwe_guc_logical_instance().
+	 * For GuC usage, we should no longer use the raw logical instance.
+	 */
 	u16 logical_instance;
 	/** @irq_offset: IRQ offset of this hw engine */
 	u16 irq_offset;
diff --git a/drivers/gpu/drm/xe/xe_hw_error.c b/drivers/gpu/drm/xe/xe_hw_error.c
index 4b72959b22761..5f2abc9485ff5 100644
--- a/drivers/gpu/drm/xe/xe_hw_error.c
+++ b/drivers/gpu/drm/xe/xe_hw_error.c
@@ -4,12 +4,12 @@
  */
 
 #include <linux/bitmap.h>
-#include <linux/fault-inject.h>
 
 #include "regs/xe_gsc_regs.h"
 #include "regs/xe_hw_error_regs.h"
 #include "regs/xe_irq_regs.h"
 
+#include "xe_debugfs.h"
 #include "xe_device.h"
 #include "xe_drm_ras.h"
 #include "xe_hw_error.h"
@@ -25,8 +25,6 @@
 						 (PVC_COR_ERR_MASK & REG_BIT(err_bit)) : \
 						 (PVC_FAT_ERR_MASK & REG_BIT(err_bit)))
 
-extern struct fault_attr inject_csc_hw_error;
-
 static const char * const error_severity[] = DRM_XE_RAS_ERROR_SEVERITY_NAMES;
 
 static const char * const hec_uncorrected_fw_errors[] = {
@@ -167,11 +165,6 @@ static_assert(ARRAY_SIZE(pvc_master_local_nonfatal_err_reg) == XE_RAS_REG_SIZE);
 						 pvc_master_local_fatal_err_reg : \
 						 pvc_master_local_nonfatal_err_reg)
 
-static bool fault_inject_csc_hw_error(void)
-{
-	return IS_ENABLED(CONFIG_DEBUG_FS) && should_fail(&inject_csc_hw_error, 1);
-}
-
 static void csc_hw_error_work(struct work_struct *work)
 {
 	struct xe_tile *tile = container_of(work, typeof(*tile), csc_hw_error_work);
@@ -437,6 +430,16 @@ static void hw_error_source_handler(struct xe_tile *tile, const enum hardware_er
 	if (!IS_DGFX(xe))
 		return;
 
+	/*
+	 * Hardware errors are reported through System Controller on the platforms that
+	 * support it, and never routed as direct IRQ to SGUnit. So we should never be
+	 * here for those platforms.
+	 */
+	if (xe->info.has_sysctrl) {
+		drm_err_ratelimited(&xe->drm, HW_ERR "Invalid error routing\n");
+		return;
+	}
+
 	spin_lock_irqsave(&xe->irq.lock, flags);
 	err_src = xe_mmio_read32(&tile->mmio, DEV_ERR_STAT_REG(hw_err));
 	if (!err_src) {
@@ -507,7 +510,7 @@ void xe_hw_error_irq_handler(struct xe_tile *tile, const u32 master_ctl)
 {
 	enum hardware_error hw_err;
 
-	if (fault_inject_csc_hw_error())
+	if (xe_fault_csc_hw_error())
 		schedule_work(&tile->csc_hw_error_work);
 
 	for (hw_err = 0; hw_err < HARDWARE_ERROR_MAX; hw_err++) {
@@ -516,14 +519,6 @@ void xe_hw_error_irq_handler(struct xe_tile *tile, const u32 master_ctl)
 	}
 }
 
-static int hw_error_info_init(struct xe_device *xe)
-{
-	if (xe->info.platform != XE_PVC)
-		return 0;
-
-	return xe_drm_ras_init(xe);
-}
-
 /*
  * Process hardware errors during boot
  */
@@ -550,16 +545,11 @@ static void process_hw_errors(struct xe_device *xe)
 void xe_hw_error_init(struct xe_device *xe)
 {
 	struct xe_tile *tile = xe_device_get_root_tile(xe);
-	int ret;
 
 	if (!IS_DGFX(xe) || IS_SRIOV_VF(xe))
 		return;
 
 	INIT_WORK(&tile->csc_hw_error_work, csc_hw_error_work);
 
-	ret = hw_error_info_init(xe);
-	if (ret)
-		drm_err(&xe->drm, "Failed to initialize XE DRM RAS (%pe)\n", ERR_PTR(ret));
-
 	process_hw_errors(xe);
 }
diff --git a/drivers/gpu/drm/xe/xe_i2c.c b/drivers/gpu/drm/xe/xe_i2c.c
index 706783863d07d..3d80eafb9a131 100644
--- a/drivers/gpu/drm/xe/xe_i2c.c
+++ b/drivers/gpu/drm/xe/xe_i2c.c
@@ -123,7 +123,7 @@ static int xe_i2c_register_adapter(struct xe_i2c *i2c)
 	}
 
 	pdev->dev.parent = i2c->drm_dev;
-	pdev->dev.fwnode = fwnode;
+	platform_device_set_fwnode(pdev, fwnode);
 	i2c->adapter_node = fwnode;
 	i2c->pdev = pdev;
 
@@ -334,9 +334,6 @@ int xe_i2c_probe(struct xe_device *xe)
 	if (!xe->info.has_i2c)
 		return 0;
 
-	if (IS_SRIOV_VF(xe))
-		return 0;
-
 	xe_i2c_read_endpoint(xe_root_tile_mmio(xe), &ep);
 	if (ep.cookie != XE_I2C_EP_COOKIE_DEVICE)
 		return 0;
diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c
index a4292a11391d3..3e7c995085d05 100644
--- a/drivers/gpu/drm/xe/xe_lrc.c
+++ b/drivers/gpu/drm/xe/xe_lrc.c
@@ -2618,13 +2618,19 @@ void xe_lrc_snapshot_free(struct xe_lrc_snapshot *snapshot)
 	kfree(snapshot);
 }
 
+static bool engine_valid_for_utilization(struct xe_gt *gt, struct xe_hw_engine *hwe)
+{
+	/* The USM-reserved copy engine runs kernel migrate contexts queried here */
+	return hwe && (!xe_hw_engine_is_reserved(hwe) || xe_gt_is_usm_hwe(gt, hwe));
+}
+
 static struct xe_hw_engine *engine_id_to_hwe(struct xe_gt *gt, u32 engine_id)
 {
 	u16 class = REG_FIELD_GET(ENGINE_CLASS_ID, engine_id);
 	u16 instance = REG_FIELD_GET(ENGINE_INSTANCE_ID, engine_id);
 	struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, class, instance, false);
 
-	if (xe_gt_WARN_ONCE(gt, !hwe || xe_hw_engine_is_reserved(hwe),
+	if (xe_gt_WARN_ONCE(gt, !engine_valid_for_utilization(gt, hwe),
 			    "Unexpected engine class:instance %d:%d for utilization\n",
 			    class, instance))
 		return NULL;
diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c
index 9428dd5e7760c..c84e14e86a823 100644
--- a/drivers/gpu/drm/xe/xe_migrate.c
+++ b/drivers/gpu/drm/xe/xe_migrate.c
@@ -383,27 +383,6 @@ static void xe_migrate_suballoc_manager_init(struct xe_migrate *m, u32 map_ofs)
 				  NUM_VMUSA_UNIT_PER_PAGE, 0);
 }
 
-/*
- * Including the reserved copy engine is required to avoid deadlocks due to
- * migrate jobs servicing the faults gets stuck behind the job that faulted.
- */
-static u32 xe_migrate_usm_logical_mask(struct xe_gt *gt)
-{
-	u32 logical_mask = 0;
-	struct xe_hw_engine *hwe;
-	enum xe_hw_engine_id id;
-
-	for_each_hw_engine(hwe, gt, id) {
-		if (hwe->class != XE_ENGINE_CLASS_COPY)
-			continue;
-
-		if (xe_gt_is_usm_hwe(gt, hwe))
-			logical_mask |= BIT(hwe->logical_instance);
-	}
-
-	return logical_mask;
-}
-
 static bool xe_migrate_needs_ccs_emit(struct xe_device *xe)
 {
 	return xe_device_has_flat_ccs(xe) && !(GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe));
@@ -479,13 +458,10 @@ int xe_migrate_init(struct xe_migrate *m)
 		goto err_out;
 
 	if (xe->info.has_usm) {
-		struct xe_hw_engine *hwe = xe_gt_hw_engine(primary_gt,
-							   XE_ENGINE_CLASS_COPY,
-							   primary_gt->usm.reserved_bcs_instance,
-							   false);
-		u32 logical_mask = xe_migrate_usm_logical_mask(primary_gt);
+		struct xe_hw_engine *hwe0 = primary_gt->usm.paging_hwe0;
+		u32 logical_mask = primary_gt->usm.paging_logical_mask;
 
-		if (!hwe || !logical_mask) {
+		if (!hwe0 || !logical_mask) {
 			err = -EINVAL;
 			goto err_out;
 		}
@@ -494,7 +470,7 @@ int xe_migrate_init(struct xe_migrate *m)
 		 * XXX: Currently only reserving 1 (likely slow) BCS instance on
 		 * PVC, may want to revisit if performance is needed.
 		 */
-		m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe,
+		m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0,
 					    EXEC_QUEUE_FLAG_KERNEL |
 					    EXEC_QUEUE_FLAG_PERMANENT |
 					    EXEC_QUEUE_FLAG_HIGH_PRIORITY |
@@ -1166,6 +1142,8 @@ static int emit_flush_invalidate(u32 *dw, int i, u32 flags)
  * @tile: Tile whose migration context to be used.
  * @q : Execution to be used along with migration context.
  * @src_bo: The buffer object @src is currently bound to.
+ * @new_mem: The (not yet committed) destination resource @src_bo is being
+ *          moved into; src_bo->ttm.resource is still the old resource.
  * @read_write : Creates BB commands for CCS read/write.
  *
  * Creates batch buffer instructions to copy CCS metadata from CCS pool to
@@ -1177,12 +1155,13 @@ static int emit_flush_invalidate(u32 *dw, int i, u32 flags)
  */
 int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q,
 			   struct xe_bo *src_bo,
+			   struct ttm_resource *new_mem,
 			   enum xe_sriov_vf_ccs_rw_ctxs read_write)
 
 {
 	bool src_is_pltt = read_write == XE_SRIOV_VF_CCS_READ_CTX;
 	bool dst_is_pltt = read_write == XE_SRIOV_VF_CCS_WRITE_CTX;
-	struct ttm_resource *src = src_bo->ttm.resource;
+	struct ttm_resource *src = new_mem;
 	struct xe_migrate *m = tile->migrate;
 	struct xe_gt *gt = tile->primary_gt;
 	u32 batch_size, batch_size_allocated;
diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h
index 965c45889c726..78e5b63f3ebea 100644
--- a/drivers/gpu/drm/xe/xe_migrate.h
+++ b/drivers/gpu/drm/xe/xe_migrate.h
@@ -138,6 +138,7 @@ struct dma_fence *xe_migrate_resolve(struct xe_migrate *m,
 
 int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q,
 			   struct xe_bo *src_bo,
+			   struct ttm_resource *new_mem,
 			   enum xe_sriov_vf_ccs_rw_ctxs read_write);
 
 void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo,
diff --git a/drivers/gpu/drm/xe/xe_mmio.c b/drivers/gpu/drm/xe/xe_mmio.c
index 78adb303b6630..7fa18dfcb5a28 100644
--- a/drivers/gpu/drm/xe/xe_mmio.c
+++ b/drivers/gpu/drm/xe/xe_mmio.c
@@ -11,27 +11,18 @@
 #include <linux/pci.h>
 
 #include <drm/drm_managed.h>
-#include <drm/drm_print.h>
 
 #include "regs/xe_bars.h"
 #include "xe_device.h"
 #include "xe_gt_sriov_vf.h"
+#include "xe_printk.h"
 #include "xe_sriov.h"
+#include "xe_tile_printk.h"
 #include "xe_trace.h"
 #include "xe_wa.h"
 
 #include "generated/xe_device_wa_oob.h"
 
-static void tiles_fini(void *arg)
-{
-	struct xe_device *xe = arg;
-	struct xe_tile *tile;
-	int id;
-
-	for_each_remote_tile(tile, xe, id)
-		tile->mmio.regs = NULL;
-}
-
 /*
  * On multi-tile devices, partition the BAR space for MMIO on each tile,
  * possibly accounting for register override on the number of tiles available.
@@ -56,50 +47,71 @@ static void mmio_multi_tile_setup(struct xe_device *xe, size_t tile_mmio_size)
 	struct xe_tile *tile;
 	u8 id;
 
-	/*
-	 * Nothing to be done as tile 0 has already been setup earlier with the
-	 * entire BAR mapped - see xe_mmio_probe_early()
-	 */
-	if (xe->info.tile_count == 1)
-		return;
-
 	for_each_remote_tile(tile, xe, id)
 		xe_mmio_init(&tile->mmio, tile, xe->mmio.regs + id * tile_mmio_size, SZ_4M);
 }
 
+/**
+ * xe_mmio_probe_tiles() - Initialize all tiles' MMIO
+ * @xe: the &xe_device
+ *
+ * Initialize the remaining tiles' MMIO instances.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
 int xe_mmio_probe_tiles(struct xe_device *xe)
 {
 	size_t tile_mmio_size = SZ_16M;
 
-	mmio_multi_tile_setup(xe, tile_mmio_size);
+	/*
+	 * Nothing to be done as tile 0 has already been setup earlier with the
+	 * entire BAR mapped - see xe_mmio_probe_early()
+	 */
+	if (xe->info.tile_count == 1)
+		return 0;
 
-	return devm_add_action_or_reset(xe->drm.dev, tiles_fini, xe);
+	if (xe->mmio.size < xe->info.tile_count * tile_mmio_size) {
+		xe_err(xe, "GTTMMADR_BAR is too small for %d tiles: %zu\n",
+		       xe->info.tile_count, xe->mmio.size);
+		return -EIO;
+	}
+
+	mmio_multi_tile_setup(xe, tile_mmio_size);
+	return 0;
 }
 
 static void mmio_fini(void *arg)
 {
 	struct xe_device *xe = arg;
-	struct xe_tile *root_tile = xe_device_get_root_tile(xe);
 
-	pci_iounmap(to_pci_dev(xe->drm.dev), xe->mmio.regs);
 	xe->mmio.regs = NULL;
-	root_tile->mmio.regs = NULL;
 }
 
+/**
+ * xe_mmio_probe_early() - Probe and initialize device's MMIO
+ * @xe: the &xe_device
+ *
+ * Map the entire GTTMMADR_BAR and initialize the first tile's MMIO instance.
+ *
+ * The first 16MB of the GTTMMADR_BAR always belongs to the root tile, and
+ * includes: registers (0-4MB), reserved space (4MB-8MB) and GGTT (8MB-16MB).
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
 int xe_mmio_probe_early(struct xe_device *xe)
 {
 	struct xe_tile *root_tile = xe_device_get_root_tile(xe);
 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
 
-	/*
-	 * Map the entire BAR.
-	 * The first 16MB of the BAR, belong to the root tile, and include:
-	 * registers (0-4MB), reserved space (4MB-8MB) and GGTT (8MB-16MB).
-	 */
-	xe->mmio.size = pci_resource_len(pdev, GTTMMADR_BAR);
-	xe->mmio.regs = pci_iomap(pdev, GTTMMADR_BAR, 0);
+	xe->mmio.regs = pcim_iomap(pdev, GTTMMADR_BAR, 0);
 	if (!xe->mmio.regs) {
-		drm_err(&xe->drm, "failed to map registers\n");
+		xe_err(xe, "Failed to map GTTMMADR_BAR\n");
+		return -EIO;
+	}
+
+	xe->mmio.size = pci_resource_len(pdev, GTTMMADR_BAR);
+	if (xe->mmio.size < SZ_16M) {
+		xe_err(xe, "GTTMMADR_BAR is too small: %zu\n", xe->mmio.size);
 		return -EIO;
 	}
 
@@ -128,6 +140,11 @@ void xe_mmio_init(struct xe_mmio *mmio, struct xe_tile *tile, void __iomem *ptr,
 	mmio->tile = tile;
 }
 
+static bool mmio_available(struct xe_mmio *mmio)
+{
+	return !xe_tile_WARN_ON_ONCE(mmio->tile, !mmio->tile->xe->mmio.regs);
+}
+
 static void mmio_flush_pending_writes(struct xe_mmio *mmio)
 {
 #define DUMMY_REG_OFFSET	0x130030
@@ -146,6 +163,9 @@ u8 xe_mmio_read8(struct xe_mmio *mmio, struct xe_reg reg)
 	u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr);
 	u8 val;
 
+	if (!mmio_available(mmio))
+		return 0;
+
 	mmio_flush_pending_writes(mmio);
 
 	val = readb(mmio->regs + addr);
@@ -158,6 +178,9 @@ void xe_mmio_write8(struct xe_mmio *mmio, struct xe_reg reg, u8 val)
 {
 	u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr);
 
+	if (!mmio_available(mmio))
+		return;
+
 	trace_xe_reg_rw(mmio, true, addr, val, sizeof(val));
 
 	writeb(val, mmio->regs + addr);
@@ -168,6 +191,9 @@ u16 xe_mmio_read16(struct xe_mmio *mmio, struct xe_reg reg)
 	u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr);
 	u16 val;
 
+	if (!mmio_available(mmio))
+		return 0;
+
 	mmio_flush_pending_writes(mmio);
 
 	val = readw(mmio->regs + addr);
@@ -180,6 +206,9 @@ void xe_mmio_write32(struct xe_mmio *mmio, struct xe_reg reg, u32 val)
 {
 	u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr);
 
+	if (!mmio_available(mmio))
+		return;
+
 	trace_xe_reg_rw(mmio, true, addr, val, sizeof(val));
 
 	if (!reg.vf && IS_SRIOV_VF(mmio->tile->xe))
@@ -194,6 +223,9 @@ u32 xe_mmio_read32(struct xe_mmio *mmio, struct xe_reg reg)
 	u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr);
 	u32 val;
 
+	if (!mmio_available(mmio))
+		return 0;
+
 	mmio_flush_pending_writes(mmio);
 
 	if (!reg.vf && IS_SRIOV_VF(mmio->tile->xe))
@@ -282,8 +314,8 @@ u64 xe_mmio_read64_2x32(struct xe_mmio *mmio, struct xe_reg reg)
 		oldudw = udw;
 	}
 
-	drm_WARN(&mmio->tile->xe->drm, retries == 0,
-		 "64-bit read of %#x did not stabilize\n", reg.addr);
+	xe_tile_WARN(mmio->tile, retries == 0,
+		     "MMIO: 64-bit read of %#x did not stabilize\n", reg.addr);
 
 	return (u64)udw << 32 | ldw;
 }
diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.c b/drivers/gpu/drm/xe/xe_mmio_gem.c
index 8c803ef233cc4..3741ae60f532b 100644
--- a/drivers/gpu/drm/xe/xe_mmio_gem.c
+++ b/drivers/gpu/drm/xe/xe_mmio_gem.c
@@ -149,7 +149,7 @@ static int xe_mmio_gem_mmap(struct drm_gem_object *base, struct vm_area_struct *
 
 	/* Set vm_pgoff (used as a fake buffer offset by DRM) to 0 */
 	vma->vm_pgoff = 0;
-	vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags));
+	vma->vm_page_prot = pgprot_noncached(vma_get_page_prot(vma));
 	vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP |
 		     VM_DONTCOPY | VM_NORESERVE);
 
diff --git a/drivers/gpu/drm/xe/xe_module.c b/drivers/gpu/drm/xe/xe_module.c
index 4cb5781829122..848d652654438 100644
--- a/drivers/gpu/drm/xe/xe_module.c
+++ b/drivers/gpu/drm/xe/xe_module.c
@@ -7,6 +7,7 @@
 
 #include <linux/init.h>
 #include <linux/module.h>
+#include <linux/workqueue.h>
 
 #include <drm/drm_module.h>
 
@@ -36,9 +37,6 @@ module_param_named(svm_notifier_size, xe_modparam.svm_notifier_size, uint, 0600)
 MODULE_PARM_DESC(svm_notifier_size, "Set the svm notifier size in MiB, must be power of 2 "
 		 "[default=" __stringify(XE_DEFAULT_SVM_NOTIFIER_SIZE) "]");
 
-module_param_named_unsafe(force_execlist, xe_modparam.force_execlist, bool, 0444);
-MODULE_PARM_DESC(force_execlist, "Force Execlist submission");
-
 #if IS_ENABLED(CONFIG_DRM_XE_DISPLAY)
 module_param_named(probe_display, xe_modparam.probe_display, bool, 0444);
 MODULE_PARM_DESC(probe_display, "Probe display HW, otherwise it's left untouched "
@@ -91,6 +89,50 @@ static int xe_check_nomodeset(void)
 	return 0;
 }
 
+static struct workqueue_struct *xe_destroy_wq;
+
+static int __init xe_destroy_wq_module_init(void)
+{
+	xe_destroy_wq = alloc_workqueue("xe-guc-destroy-wq", WQ_UNBOUND, 0);
+	if (!xe_destroy_wq)
+		return -ENOMEM;
+	return 0;
+}
+
+static void xe_destroy_wq_module_exit(void)
+{
+	if (xe_destroy_wq)
+		destroy_workqueue(xe_destroy_wq);
+	xe_destroy_wq = NULL;
+}
+
+/**
+ * xe_destroy_wq_queue() - Queue work on the destroy workqueue
+ * @work: work item to queue
+ *
+ * The destroy workqueue has module lifetime and is used for GuC exec queue
+ * teardown that can outlive a single xe_device. SVM pagemap destroy uses the
+ * per-device xe->destroy_wq instead.
+ *
+ * Return: %true if @work was queued, %false if it was already pending.
+ */
+bool xe_destroy_wq_queue(struct work_struct *work)
+{
+	return queue_work(xe_destroy_wq, work);
+}
+
+/**
+ * xe_destroy_wq_flush() - Flush the destroy workqueue
+ *
+ * Drains all pending destroy work. Called from PCI remove to ensure
+ * teardown ordering before the device is destroyed.
+ */
+void xe_destroy_wq_flush(void)
+{
+	if (xe_destroy_wq)
+		flush_workqueue(xe_destroy_wq);
+}
+
 struct init_funcs {
 	int (*init)(void);
 	void (*exit)(void);
@@ -112,6 +154,10 @@ static const struct init_funcs init_funcs[] = {
 		.init = xe_sched_job_module_init,
 		.exit = xe_sched_job_module_exit,
 	},
+	{
+		.init = xe_destroy_wq_module_init,
+		.exit = xe_destroy_wq_module_exit,
+	},
 	{
 		.init = xe_register_pci_driver,
 		.exit = xe_unregister_pci_driver,
diff --git a/drivers/gpu/drm/xe/xe_module.h b/drivers/gpu/drm/xe/xe_module.h
index 79cb9639c0f3d..a0eb7db077704 100644
--- a/drivers/gpu/drm/xe/xe_module.h
+++ b/drivers/gpu/drm/xe/xe_module.h
@@ -8,9 +8,10 @@
 
 #include <linux/types.h>
 
+struct work_struct;
+
 /* Module modprobe variables */
 struct xe_modparam {
-	bool force_execlist;
 	bool probe_display;
 	int force_vram_bar_size;
 	int guc_log_level;
@@ -27,5 +28,8 @@ struct xe_modparam {
 
 extern struct xe_modparam xe_modparam;
 
+bool xe_destroy_wq_queue(struct work_struct *work);
+void xe_destroy_wq_flush(void);
+
 #endif
 
diff --git a/drivers/gpu/drm/xe/xe_nvm.c b/drivers/gpu/drm/xe/xe_nvm.c
index 33487e91f366c..1ea67eaeae243 100644
--- a/drivers/gpu/drm/xe/xe_nvm.c
+++ b/drivers/gpu/drm/xe/xe_nvm.c
@@ -60,35 +60,40 @@ static bool xe_nvm_writable_override(struct xe_device *xe)
 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);
 	bool writable_override;
 	struct xe_reg reg;
-	u32 test_bit;
+	u32 test_bit, test_val;
 
 	switch (xe->info.platform) {
 	case XE_CRESCENTISLAND:
 		reg = PCODE_SCRATCH(0);
 		test_bit = FDO_MODE;
+		test_val = FDO_MODE;
 		break;
 	case XE_BATTLEMAGE:
 		reg = HECI_FWSTS2(DG2_GSC_HECI2_BASE);
 		test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE;
+		test_val = 0;
 		break;
 	case XE_PVC:
 		reg = HECI_FWSTS2(PVC_GSC_HECI2_BASE);
 		test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE;
+		test_val = 0;
 		break;
 	case XE_DG2:
 		reg = HECI_FWSTS2(DG2_GSC_HECI2_BASE);
 		test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE;
+		test_val = 0;
 		break;
 	case XE_DG1:
 		reg = HECI_FWSTS2(DG1_GSC_HECI2_BASE);
 		test_bit = HECI_FW_STATUS_2_NVM_ACCESS_MODE;
+		test_val = 0;
 		break;
 	default:
 		drm_err(&xe->drm, "Unknown platform\n");
 		return true;
 	}
 
-	writable_override = !(xe_mmio_read32(mmio, reg) & test_bit);
+	writable_override = (xe_mmio_read32(mmio, reg) & test_bit) == test_val;
 	if (writable_override)
 		drm_info(&xe->drm, "NVM access overridden by jumper\n");
 	return writable_override;
diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c
index 2dce6a47202ca..b3acbcd678b7c 100644
--- a/drivers/gpu/drm/xe/xe_oa.c
+++ b/drivers/gpu/drm/xe/xe_oa.c
@@ -1702,11 +1702,12 @@ static int xe_oa_release(struct inode *inode, struct file *file)
 static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma)
 {
 	struct xe_oa_stream *stream = file->private_data;
+	int ret = xe_observation_paranoid_check();
 	struct xe_bo *bo = stream->oa_buffer.bo;
 
-	if (xe_observation_paranoid && !perfmon_capable()) {
+	if (ret) {
 		drm_dbg(&stream->oa->xe->drm, "Insufficient privilege to map OA buffer\n");
-		return -EACCES;
+		return ret;
 	}
 
 	/* Can mmap the entire OA buffer or nothing (no partial OA buffer mmaps) */
@@ -2080,10 +2081,12 @@ int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *f
 		privileged_op = true;
 	}
 
-	if (privileged_op && xe_observation_paranoid && !perfmon_capable()) {
-		drm_dbg(&oa->xe->drm, "Insufficient privileges to open xe OA stream\n");
-		ret = -EACCES;
-		goto err_exec_q;
+	if (privileged_op) {
+		ret = xe_observation_paranoid_check();
+		if (ret) {
+			drm_dbg(&oa->xe->drm, "Insufficient privileges to open xe OA stream\n");
+			goto err_exec_q;
+		}
 	}
 
 	if (!param.exec_q && !param.sample) {
@@ -2365,9 +2368,10 @@ int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *fi
 		return -ENODEV;
 	}
 
-	if (xe_observation_paranoid && !perfmon_capable()) {
+	err = xe_observation_paranoid_check();
+	if (err) {
 		drm_dbg(&oa->xe->drm, "Insufficient privileges to add xe OA config\n");
-		return -EACCES;
+		return err;
 	}
 
 	err = copy_from_user(&param, u64_to_user_ptr(data), sizeof(param));
@@ -2467,9 +2471,10 @@ int xe_oa_remove_config_ioctl(struct drm_device *dev, u64 data, struct drm_file
 		return -ENODEV;
 	}
 
-	if (xe_observation_paranoid && !perfmon_capable()) {
+	ret = xe_observation_paranoid_check();
+	if (ret) {
 		drm_dbg(&oa->xe->drm, "Insufficient privileges to remove xe OA config\n");
-		return -EACCES;
+		return ret;
 	}
 
 	ret = get_user(arg, ptr);
diff --git a/drivers/gpu/drm/xe/xe_observation.c b/drivers/gpu/drm/xe/xe_observation.c
index e3f9b546207e4..39e05b9131a74 100644
--- a/drivers/gpu/drm/xe/xe_observation.c
+++ b/drivers/gpu/drm/xe/xe_observation.c
@@ -4,6 +4,7 @@
  */
 
 #include <linux/errno.h>
+#include <linux/perf_event.h>
 #include <linux/sysctl.h>
 
 #include <uapi/drm/xe_drm.h>
@@ -12,9 +13,28 @@
 #include "xe_oa.h"
 #include "xe_observation.h"
 
-u32 xe_observation_paranoid = true;
+static u32 xe_observation_paranoid = true;
 static struct ctl_table_header *sysctl_header;
 
+/**
+ * xe_observation_paranoid_check - Gate access to xe observation streams.
+ *
+ * When the xe-specific observation_paranoid sysctl is enabled (the
+ * default), defer to perf_allow_cpu() so that access is governed by the
+ * same policy as system-wide perf CPU events: kernel.perf_event_paranoid
+ * plus the security_perf_event_open() LSM hook. When the sysctl has been
+ * cleared by a privileged user, observation is open to all callers.
+ *
+ * Return: 0 if access is permitted, a negative errno otherwise.
+ */
+int xe_observation_paranoid_check(void)
+{
+	if (!xe_observation_paranoid)
+		return 0;
+
+	return perf_allow_cpu();
+}
+
 static int xe_oa_ioctl(struct drm_device *dev, struct drm_xe_observation_param *arg,
 		       struct drm_file *file)
 {
@@ -83,11 +103,13 @@ static const struct ctl_table observation_ctl_table[] = {
 };
 
 /**
- * xe_observation_sysctl_register - Register xe_observation_paranoid sysctl
+ * xe_observation_sysctl_register - Register the observation_paranoid sysctl
  *
- * Normally only superuser/root can access observation stream
- * data. However, superuser can set xe_observation_paranoid sysctl to 0 to
- * allow non-privileged users to also access observation data.
+ * When dev.xe.observation_paranoid is set (the default), access to
+ * observation streams follows the system-wide perf_allow_cpu() policy:
+ * kernel.perf_event_paranoid plus the security_perf_event_open() LSM
+ * hook. A privileged user can clear the sysctl to bypass that gate and
+ * allow unprivileged access to observation data.
  *
  * Return: always returns 0
  */
diff --git a/drivers/gpu/drm/xe/xe_observation.h b/drivers/gpu/drm/xe/xe_observation.h
index 17816998e9666..73a03e03c96a7 100644
--- a/drivers/gpu/drm/xe/xe_observation.h
+++ b/drivers/gpu/drm/xe/xe_observation.h
@@ -11,8 +11,7 @@
 struct drm_device;
 struct drm_file;
 
-extern u32 xe_observation_paranoid;
-
+int xe_observation_paranoid_check(void);
 int xe_observation_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
 int xe_observation_sysctl_register(void);
 void xe_observation_sysctl_unregister(void);
diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c
index 3165686e3e04b..691b68c69a2fb 100644
--- a/drivers/gpu/drm/xe/xe_pci.c
+++ b/drivers/gpu/drm/xe/xe_pci.c
@@ -22,10 +22,12 @@
 #include "xe_device.h"
 #include "xe_drv.h"
 #include "xe_gt.h"
+#include "xe_gt_printk.h"
 #include "xe_gt_sriov_vf.h"
 #include "xe_guc.h"
 #include "xe_mmio.h"
 #include "xe_module.h"
+#include "xe_pci_error.h"
 #include "xe_pci_rebar.h"
 #include "xe_pci_sriov.h"
 #include "xe_pci_types.h"
@@ -35,6 +37,7 @@
 #include "xe_step.h"
 #include "xe_survivability_mode.h"
 #include "xe_tile.h"
+#include "xe_tile_printk.h"
 
 enum toggle_d3cold {
 	D3COLD_DISABLE,
@@ -119,6 +122,7 @@ static const struct xe_graphics_desc graphics_xe2 = {
 static const struct xe_graphics_desc graphics_xe3p_lpg = {
 	XE2_GFX_FEATURES,
 	.has_indirect_ring_state = 1,
+	.has_uncorrectable_error_reporting = 1,
 	.multi_queue_engine_class_mask = BIT(XE_ENGINE_CLASS_COPY) | BIT(XE_ENGINE_CLASS_COMPUTE),
 	.num_geometry_xecore_fuse_regs = 3,
 	.num_compute_xecore_fuse_regs = 3,
@@ -128,6 +132,7 @@ static const struct xe_graphics_desc graphics_xe3p_xpc = {
 	XE2_GFX_FEATURES,
 	.has_access_counter = 0,
 	.has_indirect_ring_state = 1,
+	.has_uncorrectable_error_reporting = 1,
 	.hw_engine_mask =
 		GENMASK(XE_HW_ENGINE_BCS8, XE_HW_ENGINE_BCS1) |
 		GENMASK(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0),
@@ -150,6 +155,14 @@ static const struct xe_media_desc media_xelpmp = {
 		BIT(XE_HW_ENGINE_GSCCS0)
 };
 
+static const struct xe_media_desc media_xe3p_hpm = {
+	.has_uncorrectable_error_reporting = 1,
+	.hw_engine_mask =
+		GENMASK(XE_HW_ENGINE_VCS7, XE_HW_ENGINE_VCS0) |
+		GENMASK(XE_HW_ENGINE_VECS3, XE_HW_ENGINE_VECS0) |
+		BIT(XE_HW_ENGINE_GSCCS0)
+};
+
 /* Pre-GMDID Graphics IPs */
 static const struct xe_ip graphics_ip_xelp = { 1200, "Xe_LP", &graphics_xelp };
 static const struct xe_ip graphics_ip_xelpp = { 1210, "Xe_LP+", &graphics_xelp };
@@ -185,7 +198,7 @@ static const struct xe_ip media_ips[] = {
 	{ 3000, "Xe3_LPM", &media_xelpmp },
 	{ 3002, "Xe3_LPM", &media_xelpmp },
 	{ 3500, "Xe3p_LPM", &media_xelpmp },
-	{ 3503, "Xe3p_HPM", &media_xelpmp },
+	{ 3503, "Xe3p_HPM", &media_xe3p_hpm },
 };
 
 #define MULTI_LRC_MASK \
@@ -355,6 +368,7 @@ static const __maybe_unused struct xe_device_desc pvc_desc = {
 	PLATFORM(PVC),
 	.dma_mask_size = 52,
 	.has_display = false,
+	.has_drm_ras = true,
 	.has_gsc_nvm = 1,
 	.has_heci_gscfi = 1,
 	.max_gt_per_tile = 1,
@@ -457,6 +471,7 @@ static const struct xe_device_desc cri_desc = {
 	PLATFORM(CRESCENTISLAND),
 	.dma_mask_size = 52,
 	.has_display = false,
+	.has_drm_ras = true,
 	.has_flat_ccs = false,
 	.has_gsc_nvm = 1,
 	.has_i2c = true,
@@ -599,8 +614,6 @@ static int read_gmdid(struct xe_device *xe, enum xe_gmdid_type type, u32 *ver, u
 	struct xe_reg gmdid_reg = GMD_ID;
 	u32 val;
 
-	KUNIT_STATIC_STUB_REDIRECT(read_gmdid, xe, type, ver, revid);
-
 	if (IS_SRIOV_VF(xe)) {
 		/*
 		 * To get the value of the GMDID register, VFs must obtain it
@@ -726,14 +739,30 @@ static int handle_gmdid(struct xe_device *xe,
 	return 0;
 }
 
-static void init_devid(struct xe_device *xe)
+struct xe_probed_info {
+	u16 devid;
+	u8 revid;
+	u8 tile_count;
+	struct xe_step_info step;
+	const struct xe_ip *graphics_ip;
+	const struct xe_ip *media_ip;
+};
+
+/*
+ * Probe from the hardware the info required by xe_info_init_early().
+ */
+static int xe_probe_info_early(struct xe_device *xe,
+			       const struct xe_device_desc *desc,
+			       struct xe_probed_info *probed_info)
 {
 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
 
-	KUNIT_STATIC_STUB_REDIRECT(init_devid, xe);
+	probed_info->devid = pdev->device;
+	probed_info->revid = pdev->revision;
 
-	xe->info.devid = pdev->device;
-	xe->info.revid = pdev->revision;
+	xe_step_platform_get(desc->platform, probed_info->revid, &probed_info->step);
+
+	return 0;
 }
 
 /*
@@ -742,17 +771,20 @@ static void init_devid(struct xe_device *xe)
  */
 static int xe_info_init_early(struct xe_device *xe,
 			      const struct xe_device_desc *desc,
-			      const struct xe_subplatform_desc *subplatform_desc)
+			      const struct xe_subplatform_desc *subplatform_desc,
+			      struct xe_probed_info *probed_info)
 {
 	int err;
 
+	xe->info.devid = probed_info->devid;
+	xe->info.revid = probed_info->revid;
+	xe->info.step.platform = probed_info->step.platform;
+
 	xe->info.platform_name = desc->platform_name;
 	xe->info.platform = desc->platform;
 	xe->info.subplatform = subplatform_desc ?
 		subplatform_desc->subplatform : XE_SUBPLATFORM_NONE;
 
-	init_devid(xe);
-
 	xe->info.dma_mask_size = desc->dma_mask_size;
 	xe->info.va_bits = desc->va_bits;
 	xe->info.vm_max_level = desc->vm_max_level;
@@ -760,6 +792,7 @@ static int xe_info_init_early(struct xe_device *xe,
 
 	xe->info.is_dgfx = desc->is_dgfx;
 	xe->info.has_cached_pt = desc->has_cached_pt;
+	xe->info.has_drm_ras = desc->has_drm_ras;
 	xe->info.has_fan_control = desc->has_fan_control;
 	/* runtime fusing may force flat_ccs to disabled later */
 	xe->info.has_flat_ccs = desc->has_flat_ccs;
@@ -788,15 +821,12 @@ static int xe_info_init_early(struct xe_device *xe,
 
 	xe->info.probe_display = IS_ENABLED(CONFIG_DRM_XE_DISPLAY) &&
 				 xe_modparam.probe_display &&
-				 desc->has_display;
-	xe->info.force_execlist = xe_modparam.force_execlist;
+				 desc->has_display &&
+				 !xe_device_is_admin_only(xe);
 
 	xe_assert(xe, desc->max_gt_per_tile > 0);
 	xe_assert(xe, desc->max_gt_per_tile <= XE_MAX_GT_PER_TILE);
 	xe->info.max_gt_per_tile = desc->max_gt_per_tile;
-	xe->info.tile_count = 1 + desc->max_remote_tiles;
-
-	xe_step_platform_get(xe);
 
 	err = xe_tile_init_early(xe_device_get_root_tile(xe), xe, 0);
 	if (err)
@@ -805,22 +835,21 @@ static int xe_info_init_early(struct xe_device *xe,
 	return 0;
 }
 
-/*
- * Possibly override number of tile based on configuration register.
- */
-static void xe_info_probe_tile_count(struct xe_device *xe)
+static void xe_probe_tile_count(struct xe_device *xe,
+				const struct xe_device_desc *desc,
+				struct xe_probed_info *probed_info)
 {
 	struct xe_mmio *mmio;
 	u8 tile_count;
 	u32 mtcfg;
 
-	KUNIT_STATIC_STUB_REDIRECT(xe_info_probe_tile_count, xe);
+	probed_info->tile_count = 1 + desc->max_remote_tiles;
 
 	/*
 	 * Probe for tile count only for platforms that support multiple
 	 * tiles.
 	 */
-	if (xe->info.tile_count == 1)
+	if (probed_info->tile_count == 1)
 		return;
 
 	mmio = xe_root_tile_mmio(xe);
@@ -833,10 +862,10 @@ static void xe_info_probe_tile_count(struct xe_device *xe)
 	mtcfg = xe_mmio_read32(mmio, XEHP_MTCFG_ADDR);
 	tile_count = REG_FIELD_GET(TILE_COUNT, mtcfg) + 1;
 
-	if (tile_count < xe->info.tile_count) {
+	if (tile_count < probed_info->tile_count) {
 		drm_info(&xe->drm, "tile_count: %d, reduced_tile_count %d\n",
-			 xe->info.tile_count, tile_count);
-		xe->info.tile_count = tile_count;
+			 probed_info->tile_count, tile_count);
+		probed_info->tile_count = tile_count;
 	}
 }
 
@@ -848,7 +877,7 @@ static struct xe_gt *alloc_primary_gt(struct xe_tile *tile,
 	struct xe_gt *gt;
 
 	if (!xe_configfs_primary_gt_allowed(to_pci_dev(xe->drm.dev))) {
-		xe_info(xe, "Primary GT disabled via configfs\n");
+		xe_tile_info(tile, "Primary GT disabled via configfs\n");
 		return NULL;
 	}
 
@@ -859,7 +888,13 @@ static struct xe_gt *alloc_primary_gt(struct xe_tile *tile,
 	gt->info.type = XE_GT_TYPE_MAIN;
 	gt->info.id = tile->id * xe->info.max_gt_per_tile;
 	gt->info.has_indirect_ring_state = graphics_desc->has_indirect_ring_state;
+	gt->info.has_uncorrectable_error_reporting =
+		graphics_desc->has_uncorrectable_error_reporting;
 	gt->info.multi_queue_engine_class_mask = graphics_desc->multi_queue_engine_class_mask;
+	if (!xe_configfs_get_enable_multi_queue(to_pci_dev(xe->drm.dev))) {
+		xe_gt_info(gt, "Multi-queue disabled via configfs\n");
+		gt->info.multi_queue_engine_class_mask = 0;
+	}
 	gt->info.engine_mask = graphics_desc->hw_engine_mask;
 	gt->info.num_geometry_xecore_fuse_regs = graphics_desc->num_geometry_xecore_fuse_regs;
 	gt->info.num_compute_xecore_fuse_regs = graphics_desc->num_compute_xecore_fuse_regs;
@@ -890,7 +925,7 @@ static struct xe_gt *alloc_media_gt(struct xe_tile *tile,
 	struct xe_gt *gt;
 
 	if (!xe_configfs_media_gt_allowed(to_pci_dev(xe->drm.dev))) {
-		xe_info(xe, "Media GT disabled via configfs\n");
+		xe_tile_info(tile, "Media GT disabled via configfs\n");
 		return NULL;
 	}
 
@@ -904,30 +939,16 @@ static struct xe_gt *alloc_media_gt(struct xe_tile *tile,
 	gt->info.type = XE_GT_TYPE_MEDIA;
 	gt->info.id = tile->id * xe->info.max_gt_per_tile + 1;
 	gt->info.has_indirect_ring_state = media_desc->has_indirect_ring_state;
+	gt->info.has_uncorrectable_error_reporting = media_desc->has_uncorrectable_error_reporting;
 	gt->info.engine_mask = media_desc->hw_engine_mask;
 
 	return gt;
 }
 
-/*
- * Initialize device info content that does require knowledge about
- * graphics / media IP version.
- * Make sure that GT / tile structures allocated by the driver match the data
- * present in device info.
- */
-static int xe_info_init(struct xe_device *xe,
-			const struct xe_device_desc *desc)
+static int xe_probe_ips(struct xe_device *xe,
+			const struct xe_device_desc *desc,
+			struct xe_probed_info *probed_info)
 {
-	u32 graphics_gmdid_revid = 0, media_gmdid_revid = 0;
-	const struct xe_ip *graphics_ip;
-	const struct xe_ip *media_ip;
-	const struct xe_graphics_desc *graphics_desc;
-	const struct xe_media_desc *media_desc;
-	struct xe_tile *tile;
-	struct xe_gt *gt;
-	int ret;
-	u8 id;
-
 	/*
 	 * If this platform supports GMD_ID, we'll detect the proper IP
 	 * descriptor to use from hardware registers.
@@ -936,17 +957,21 @@ static int xe_info_init(struct xe_device *xe,
 	 * versions are simply derived from that.
 	 */
 	if (desc->pre_gmdid_graphics_ip) {
-		graphics_ip = desc->pre_gmdid_graphics_ip;
-		media_ip = desc->pre_gmdid_media_ip;
-		xe_step_pre_gmdid_get(xe);
+		probed_info->graphics_ip = desc->pre_gmdid_graphics_ip;
+		probed_info->media_ip = desc->pre_gmdid_media_ip;
+		xe_step_pre_gmdid_get(xe, &probed_info->step);
 	} else {
+		int err;
+		u32 graphics_revid, media_revid;
+
 		xe_assert(xe, !desc->pre_gmdid_media_ip);
-		ret = handle_gmdid(xe, &graphics_ip, &media_ip,
-				   &graphics_gmdid_revid, &media_gmdid_revid);
-		if (ret)
-			return ret;
 
-		xe_step_gmdid_get(xe, graphics_gmdid_revid, media_gmdid_revid);
+		err = handle_gmdid(xe, &probed_info->graphics_ip, &probed_info->media_ip,
+				   &graphics_revid, &media_revid);
+		if (err)
+			return err;
+
+		xe_step_gmdid_get(xe, graphics_revid, media_revid, &probed_info->step);
 	}
 
 	/*
@@ -954,9 +979,56 @@ static int xe_info_init(struct xe_device *xe,
 	 * error and we should abort driver load.  Failing to detect media
 	 * IP is non-fatal; we'll just proceed without enabling media support.
 	 */
-	if (!graphics_ip)
+	if (!probed_info->graphics_ip)
 		return -ENODEV;
 
+	return 0;
+}
+
+/*
+ * Probe from the hardware the info required by xe_info_init().
+ */
+static int xe_probe_info(struct xe_device *xe,
+			 const struct xe_device_desc *desc,
+			 struct xe_probed_info *probed_info)
+{
+	int err;
+
+	xe_probe_tile_count(xe, desc, probed_info);
+
+	err = xe_probe_ips(xe, desc, probed_info);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+/*
+ * Initialize device info content that does require knowledge about
+ * graphics / media IP version.
+ * Make sure that GT / tile structures allocated by the driver match the data
+ * present in device info.
+ */
+static int xe_info_init(struct xe_device *xe,
+			const struct xe_device_desc *desc,
+			struct xe_probed_info *probed_info)
+{
+	const struct xe_ip *graphics_ip;
+	const struct xe_ip *media_ip;
+	const struct xe_graphics_desc *graphics_desc;
+	const struct xe_media_desc *media_desc;
+	struct xe_tile *tile;
+	struct xe_gt *gt;
+	u8 id;
+
+	graphics_ip = probed_info->graphics_ip;
+	media_ip = probed_info->media_ip;
+
+	xe->info.tile_count = probed_info->tile_count;
+	xe->info.step.basedie = probed_info->step.basedie;
+	xe->info.step.graphics = probed_info->step.graphics;
+	xe->info.step.media = probed_info->step.media;
+
 	xe->info.graphics_verx100 = graphics_ip->verx100;
 	xe->info.graphics_name = graphics_ip->name;
 	graphics_desc = graphics_ip->desc;
@@ -988,8 +1060,6 @@ static int xe_info_init(struct xe_device *xe,
 		xe->info.has_soc_remapper_telem = 0;
 	}
 
-	xe_info_probe_tile_count(xe);
-
 	for_each_remote_tile(tile, xe, id) {
 		int err;
 
@@ -1048,6 +1118,12 @@ static void xe_pci_remove(struct pci_dev *pdev)
 		return;
 
 	xe_device_remove(xe);
+
+	/*
+	 * Preserve remove-time flush after moving destroy work to module
+	 * lifetime.
+	 */
+	xe_destroy_wq_flush();
 	xe_pm_fini(xe);
 }
 
@@ -1072,9 +1148,11 @@ static void xe_pci_remove(struct pci_dev *pdev)
  */
 static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
+	struct xe_probed_info probed_info = {};
 	const struct xe_device_desc *desc = (const void *)ent->driver_data;
 	const struct xe_subplatform_desc *subplatform_desc;
 	struct xe_device *xe;
+	void *group;
 	int err;
 
 	subplatform_desc = find_subplatform(desc, pdev->device);
@@ -1102,6 +1180,11 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (xe_display_driver_probe_defer(pdev))
 		return -EPROBE_DEFER;
 
+	/* Group all devres so xe_pci_error_slot_reset() can release them as a unit. */
+	group = devres_open_group(&pdev->dev, NULL, GFP_KERNEL);
+	if (!group)
+		return -ENOMEM;
+
 	err = pcim_enable_device(pdev);
 	if (err)
 		return err;
@@ -1110,13 +1193,19 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (IS_ERR(xe))
 		return PTR_ERR(xe);
 
+	xe->devres_group = group;
+
 	pci_set_drvdata(pdev, &xe->drm);
 
 	xe_pm_assert_unbounded_bridge(xe);
 
 	pci_set_master(pdev);
 
-	err = xe_info_init_early(xe, desc, subplatform_desc);
+	err = xe_probe_info_early(xe, desc, &probed_info);
+	if (err)
+		return err;
+
+	err = xe_info_init_early(xe, desc, subplatform_desc, &probed_info);
 	if (err)
 		return err;
 
@@ -1135,7 +1224,11 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (err)
 		return err;
 
-	err = xe_info_init(xe, desc);
+	err = xe_probe_info(xe, desc, &probed_info);
+	if (err)
+		return err;
+
+	err = xe_info_init(xe, desc, &probed_info);
 	if (err)
 		return err;
 
@@ -1348,6 +1441,7 @@ static struct pci_driver xe_pci_driver = {
 	.remove = xe_pci_remove,
 	.shutdown = xe_pci_shutdown,
 	.sriov_configure = xe_pci_sriov_configure,
+	.err_handler = &xe_pci_error_handlers,
 #ifdef CONFIG_PM_SLEEP
 	.driver.pm = &xe_pm_ops,
 #endif
diff --git a/drivers/gpu/drm/xe/xe_pci_error.c b/drivers/gpu/drm/xe/xe_pci_error.c
new file mode 100644
index 0000000000000..e41af2ac7f23b
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_pci_error.c
@@ -0,0 +1,139 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <linux/pci.h>
+
+#include "xe_device.h"
+#include "xe_gt.h"
+#include "xe_pci.h"
+#include "xe_pm.h"
+#include "xe_printk.h"
+#include "xe_ras.h"
+#include "xe_survivability_mode.h"
+
+static void prepare_device_for_reset(struct pci_dev *pdev)
+{
+	struct xe_device *xe = pdev_to_xe_device(pdev);
+	struct xe_gt *gt;
+	u8 id;
+
+	/*
+	 * Wedge the device to prevent userspace access but do not send the uevent.
+	 * xe_device_wedged_fini() releases runtime pm if wedged flag is set, so acquire a runtime
+	 * pm reference to avoid underflow.
+	 */
+	if (!atomic_xchg(&xe->wedged.flag, 1))
+		xe_pm_runtime_get_noresume(xe);
+
+	xe_device_set_in_reset(xe);
+
+	for_each_gt(gt, xe, id)
+		xe_gt_declare_wedged(gt);
+
+	pci_disable_device(pdev);
+}
+
+static pci_ers_result_t ras_action_to_pci_result(struct pci_dev *pdev, u8 action)
+{
+	switch (action) {
+	case XE_RAS_RECOVERY_ACTION_RECOVERED:
+		return PCI_ERS_RESULT_RECOVERED;
+	case XE_RAS_RECOVERY_ACTION_RESET:
+		prepare_device_for_reset(pdev);
+		return PCI_ERS_RESULT_NEED_RESET;
+	case XE_RAS_RECOVERY_ACTION_DISCONNECT:
+		return PCI_ERS_RESULT_DISCONNECT;
+	default:
+		return PCI_ERS_RESULT_DISCONNECT;
+	}
+}
+
+static pci_ers_result_t xe_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
+{
+	struct xe_device *xe = pdev_to_xe_device(pdev);
+
+	xe_info(xe, "PCI error: detected state = %d\n", state);
+
+	if (state == pci_channel_io_perm_failure)
+		return PCI_ERS_RESULT_DISCONNECT;
+
+	/* If the device is already wedged or in survivability mode, do not attempt recovery */
+	if (xe_survivability_mode_is_boot_enabled(xe) || xe_device_wedged(xe))
+		return PCI_ERS_RESULT_DISCONNECT;
+
+	switch (state) {
+	case pci_channel_io_normal:
+		return PCI_ERS_RESULT_CAN_RECOVER;
+	case pci_channel_io_frozen:
+		prepare_device_for_reset(pdev);
+		return PCI_ERS_RESULT_NEED_RESET;
+	default:
+		xe_info(xe, "PCI error: unknown state %d\n", state);
+		return PCI_ERS_RESULT_DISCONNECT;
+	}
+}
+
+static pci_ers_result_t xe_pci_error_mmio_enabled(struct pci_dev *pdev)
+{
+	struct xe_device *xe = pdev_to_xe_device(pdev);
+	enum xe_ras_recovery_action action;
+
+	xe_info(xe, "PCI error: MMIO enabled\n");
+	action = xe_ras_process_errors(xe);
+
+	return ras_action_to_pci_result(pdev, action);
+}
+
+static pci_ers_result_t xe_pci_error_slot_reset(struct pci_dev *pdev)
+{
+	const struct pci_device_id *ent = pci_match_id(pdev->driver->id_table, pdev);
+	struct xe_device *xe = pdev_to_xe_device(pdev);
+
+	xe_info(xe, "PCI error: slot reset\n");
+
+	pci_restore_state(pdev);
+
+	if (pci_enable_device(pdev)) {
+		xe_err(xe, "Cannot re-enable PCI device after reset\n");
+		return PCI_ERS_RESULT_DISCONNECT;
+	}
+
+	/*
+	 * Secondary Bus Reset causes all VRAM state to be lost along with
+	 * hardware state. As an initial step, re-probe the device to
+	 * re-initialize the driver and hardware.
+	 * TODO: optimize by re-initializing only the hardware state and re-creating
+	 * kernel BOs.
+	 */
+	xe_device_clear_in_reset(xe);
+	pdev->driver->remove(pdev);
+	devres_release_group(&pdev->dev, xe->devres_group);
+
+	if (pdev->driver->probe(pdev, ent))
+		return PCI_ERS_RESULT_DISCONNECT;
+
+	xe = pdev_to_xe_device(pdev);
+
+	/* Wedge the device to prevent I/O operations till the resume callback */
+	atomic_set(&xe->wedged.flag, 1);
+
+	return PCI_ERS_RESULT_RECOVERED;
+}
+
+static void xe_pci_error_resume(struct pci_dev *pdev)
+{
+	struct xe_device *xe = pdev_to_xe_device(pdev);
+
+	xe_info(xe, "PCI error: resume\n");
+
+	atomic_set(&xe->wedged.flag, 0);
+}
+
+const struct pci_error_handlers xe_pci_error_handlers = {
+	.error_detected	= xe_pci_error_detected,
+	.mmio_enabled	= xe_pci_error_mmio_enabled,
+	.slot_reset	= xe_pci_error_slot_reset,
+	.resume		= xe_pci_error_resume,
+};
diff --git a/drivers/gpu/drm/xe/xe_pci_error.h b/drivers/gpu/drm/xe/xe_pci_error.h
new file mode 100644
index 0000000000000..725ad0214e625
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_pci_error.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _XE_PCI_ERROR_H_
+#define _XE_PCI_ERROR_H_
+
+struct pci_error_handlers;
+
+extern const struct pci_error_handlers xe_pci_error_handlers;
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h
index 5b85e2c24b7b2..fed509ff601e4 100644
--- a/drivers/gpu/drm/xe/xe_pci_types.h
+++ b/drivers/gpu/drm/xe/xe_pci_types.h
@@ -40,6 +40,7 @@ struct xe_device_desc {
 
 	u8 has_cached_pt:1;
 	u8 has_display:1;
+	u8 has_drm_ras:1;
 	u8 has_fan_control:1;
 	u8 has_flat_ccs:1;
 	u8 has_gsc_nvm:1;
@@ -78,12 +79,14 @@ struct xe_graphics_desc {
 	u8 has_ctx_tlb_inval:1;
 	u8 has_usm:1;
 	u8 has_64bit_timestamp:1;
+	u8 has_uncorrectable_error_reporting:1;
 };
 
 struct xe_media_desc {
 	u64 hw_engine_mask;	/* hardware engines provided by media IP */
 
 	u8 has_indirect_ring_state:1;
+	u8 has_uncorrectable_error_reporting:1;
 };
 
 struct xe_ip {
diff --git a/drivers/gpu/drm/xe/xe_pcode.c b/drivers/gpu/drm/xe/xe_pcode.c
index dc66d0c7ee062..ccc3bdeed6bb2 100644
--- a/drivers/gpu/drm/xe/xe_pcode.c
+++ b/drivers/gpu/drm/xe/xe_pcode.c
@@ -11,10 +11,14 @@
 
 #include <drm/drm_managed.h>
 
+#include "regs/xe_pmt.h"
 #include "xe_assert.h"
 #include "xe_device.h"
 #include "xe_mmio.h"
 #include "xe_pcode_api.h"
+#include "xe_pm.h"
+#include "xe_printk.h"
+#include "xe_vsec.h"
 
 /**
  * DOC: PCODE
@@ -323,15 +327,17 @@ int xe_pcode_ready(struct xe_device *xe, bool locked)
 }
 
 /**
- * xe_pcode_init: initialize components of PCODE
+ * xe_pcode_init_early() - Initialize components of PCODE
  * @tile: tile instance
  *
  * This function initializes the xe_pcode component.
  * To be called once only during probe.
+ *
+ * Return: 0 on success or a negative error code on failure.
  */
-void xe_pcode_init(struct xe_tile *tile)
+int xe_pcode_init_early(struct xe_tile *tile)
 {
-	drmm_mutex_init(&tile_to_xe(tile)->drm, &tile->pcode.lock);
+	return drmm_mutex_init(&tile_to_xe(tile)->drm, &tile->pcode.lock);
 }
 
 /**
@@ -348,3 +354,31 @@ int xe_pcode_probe_early(struct xe_device *xe)
 	return xe_pcode_ready(xe, false);
 }
 ALLOW_ERROR_INJECTION(xe_pcode_probe_early, ERRNO); /* See xe_pci_probe */
+
+/**
+ * xe_get_pcode_version - Read pcode version via PMT telemetry
+ * @xe: xe instance
+ * @version: pointer to struct xe_pcode_version to store version info
+ *
+ * Reads the pcode version from PMT telemetry and fills the
+ * provided @version structure.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int xe_get_pcode_version(struct xe_device *xe, struct xe_pcode_version *version)
+{
+	int ret = 0;
+
+	guard(xe_pm_runtime)(xe);
+
+	ret = xe_pmt_telem_read(xe->drm.dev,
+				xe_mmio_read32(xe_root_tile_mmio(xe), PUNIT_TELEMETRY_GUID),
+				(u64 *)version, PUNIT_VERSION_OFFSET, sizeof(*version));
+	if (ret != sizeof(*version)) {
+		xe_warn(xe, "pcode version read from PMT failed, ret %pe\n", ERR_PTR(ret));
+		return ret;
+	}
+	xe_dbg(xe, "pcode version major %u minor %u engg %u\n", version->major,
+	       version->minor, version->engg);
+	return 0;
+}
diff --git a/drivers/gpu/drm/xe/xe_pcode.h b/drivers/gpu/drm/xe/xe_pcode.h
index 490e4f2696078..8fb3e4ba13a6e 100644
--- a/drivers/gpu/drm/xe/xe_pcode.h
+++ b/drivers/gpu/drm/xe/xe_pcode.h
@@ -12,7 +12,13 @@ struct drm_device;
 struct xe_device;
 struct xe_tile;
 
-void xe_pcode_init(struct xe_tile *tile);
+struct xe_pcode_version {
+	u16 minor;
+	u16 major;
+	u32 engg;
+};
+
+int xe_pcode_init_early(struct xe_tile *tile);
 int xe_pcode_probe_early(struct xe_device *xe);
 int xe_pcode_ready(struct xe_device *xe, bool locked);
 int xe_pcode_init_min_freq_table(struct xe_tile *tile, u32 min_gt_freq,
@@ -22,6 +28,7 @@ int xe_pcode_write_timeout(struct xe_tile *tile, u32 mbox, u32 val,
 			   int timeout_ms);
 int xe_pcode_write64_timeout(struct xe_tile *tile, u32 mbox, u32 data0,
 			     u32 data1, int timeout);
+int xe_get_pcode_version(struct xe_device *xe, struct xe_pcode_version *version);
 
 #define xe_pcode_write(tile, mbox, val) \
 	xe_pcode_write_timeout(tile, mbox, val, 1)
diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c
index 99562f691080b..a5289a9df8d25 100644
--- a/drivers/gpu/drm/xe/xe_pm.c
+++ b/drivers/gpu/drm/xe/xe_pm.c
@@ -700,6 +700,8 @@ int xe_pm_runtime_resume(struct xe_device *xe)
 	if (xe->d3cold.allowed)
 		xe_sysctrl_pm_resume(xe);
 
+	xe_display_pm_runtime_resume_early(xe);
+
 	xe_irq_resume(xe);
 
 	for_each_gt(gt, xe, id) {
diff --git a/drivers/gpu/drm/xe/xe_preempt_fence.c b/drivers/gpu/drm/xe/xe_preempt_fence.c
index d6427b473ddda..4aa570fe745d3 100644
--- a/drivers/gpu/drm/xe/xe_preempt_fence.c
+++ b/drivers/gpu/drm/xe/xe_preempt_fence.c
@@ -74,6 +74,13 @@ static bool preempt_fence_enable_signaling(struct dma_fence *fence)
 	struct xe_exec_queue *q = pfence->q;
 
 	pfence->error = q->ops->suspend(q);
+	/*
+	 * Record a successful suspend so the rebind worker only resumes queues
+	 * that were actually suspended; a failed suspend() leaves the queue
+	 * un-suspended and must not be paired with a resume().
+	 */
+	if (!pfence->error)
+		WRITE_ONCE(q->lr.suspended, true);
 	queue_work(q->vm->xe->preempt_fence_wq, &pfence->preempt_work);
 	return true;
 }
diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index e787c0c27c426..148305a6e9a10 100644
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -760,7 +760,7 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,
 			return -EAGAIN;
 		}
 		if (xe_svm_range_has_dma_mapping(range)) {
-			xe_res_first_dma(range->base.pages.dma_addr, 0,
+			xe_res_first_dma(range->pages.dma_addr, 0,
 					 xe_svm_range_size(range),
 					 &curs);
 			xe_svm_range_debug(range, "BIND PREPARE - MIXED");
@@ -2085,6 +2085,9 @@ static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,
 		 * automatically when the context is re-enabled by the rebind worker,
 		 * or in fault mode it was invalidated on PTE zapping.
 		 *
+		 * If rebind, we have to invalidate TLB on context based TLB invalidation
+		 * LR vms, as they cannot be relied on context re-enable.
+		 *
 		 * If !rebind, and scratch enabled VMs, there is a chance the scratch
 		 * PTE is already cached in the TLB so it needs to be invalidated.
 		 * On !LR VMs this is done in the ring ops preceding a batch, but on
@@ -2094,6 +2097,9 @@ static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,
 		if ((!pt_op->rebind && xe_vm_has_scratch(vm) &&
 		     xe_vm_in_lr_mode(vm)))
 			pt_update_ops->needs_invalidation = true;
+		else if (pt_op->rebind && xe_vm_in_preempt_fence_mode(vm) &&
+			 vm->xe->info.has_ctx_tlb_inval)
+			pt_update_ops->needs_invalidation = true;
 		else if (pt_op->rebind && !xe_vm_in_lr_mode(vm))
 			/* We bump also if batch_invalidate_tlb is true */
 			vm->tlb_flush_seqno++;
@@ -2365,8 +2371,11 @@ static void
 xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops)
 {
 	init_llist_head(&pt_update_ops->deferred);
+	pt_update_ops->current_op = 0;
 	pt_update_ops->start = ~0x0ull;
 	pt_update_ops->last = 0x0ull;
+	pt_update_ops->needs_svm_lock = false;
+	pt_update_ops->needs_invalidation = false;
 	xe_page_reclaim_list_init(&pt_update_ops->prl);
 }
 
diff --git a/drivers/gpu/drm/xe/xe_pxp.c b/drivers/gpu/drm/xe/xe_pxp.c
index 968b7e70b3f97..fea3d8ceeddb9 100644
--- a/drivers/gpu/drm/xe/xe_pxp.c
+++ b/drivers/gpu/drm/xe/xe_pxp.c
@@ -59,6 +59,7 @@ bool xe_pxp_is_enabled(const struct xe_pxp *pxp)
 static bool pxp_prerequisites_done(const struct xe_pxp *pxp)
 {
 	struct xe_gt *gt = pxp->gt;
+	bool huc_ok;
 	bool ready;
 
 	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
@@ -73,9 +74,14 @@ static bool pxp_prerequisites_done(const struct xe_pxp *pxp)
 	 */
 	XE_WARN_ON(!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL));
 
-	/* PXP requires both HuC authentication via GSC and GSC proxy initialized */
-	ready = xe_huc_is_authenticated(&gt->uc.huc, XE_HUC_AUTH_VIA_GSC) &&
-		xe_gsc_proxy_init_done(&gt->uc.gsc);
+	/*
+	 * PXP requires GSC proxy to be initialized. On platforms where the HuC
+	 * is loaded by the kernel driver (i.e., pre media 35) PXP also requires
+	 * the HuC to be authenticated by GSC.
+	 */
+	huc_ok = MEDIA_VER(gt_to_xe(gt)) >= 35 ||
+		 xe_huc_is_authenticated(&gt->uc.huc, XE_HUC_AUTH_VIA_GSC);
+	ready = huc_ok && xe_gsc_proxy_init_done(&gt->uc.gsc);
 
 	return ready;
 }
@@ -97,9 +103,13 @@ int xe_pxp_get_readiness_status(struct xe_pxp *pxp)
 	if (!xe_pxp_is_enabled(pxp))
 		return -ENODEV;
 
-	/* if the GSC or HuC FW are in an error state, PXP will never work */
-	if (xe_uc_fw_status_to_error(pxp->gt->uc.huc.fw.status) ||
-	    xe_uc_fw_status_to_error(pxp->gt->uc.gsc.fw.status))
+	/* If the GSC FW is in an error state, PXP will never work */
+	if (xe_uc_fw_status_to_error(pxp->gt->uc.gsc.fw.status))
+		return -EIO;
+
+	/* Same for HuC FW, but only if the kernel owns HuC-loading (i.e. pre-NVL) */
+	if (MEDIA_VER(gt_to_xe(pxp->gt)) < 35 &&
+	    xe_uc_fw_status_to_error(pxp->gt->uc.huc.fw.status))
 		return -EIO;
 
 	guard(xe_pm_runtime)(pxp->xe);
@@ -361,6 +371,7 @@ static void pxp_fini(void *arg)
 int xe_pxp_init(struct xe_device *xe)
 {
 	struct xe_gt *gt = xe->tiles[0].media_gt;
+	bool gsc_ok, huc_ok;
 	struct xe_pxp *pxp;
 	int err;
 
@@ -375,10 +386,14 @@ int xe_pxp_init(struct xe_device *xe)
 	if (!(gt->info.engine_mask & BIT(XE_HW_ENGINE_GSCCS0)))
 		return 0;
 
-	/* PXP requires both GSC and HuC firmwares to be available */
-	if (!xe_uc_fw_is_loadable(&gt->uc.gsc.fw) ||
-	    !xe_uc_fw_is_loadable(&gt->uc.huc.fw)) {
-		drm_info(&xe->drm, "skipping PXP init due to missing FW dependencies");
+	/* PXP requires GSC FW to be available. Pre-NVL it also requires HuC FW */
+	gsc_ok = xe_uc_fw_is_loadable(&gt->uc.gsc.fw);
+	huc_ok = MEDIA_VER(xe) >= 35 || xe_uc_fw_is_loadable(&gt->uc.huc.fw);
+
+	if (!gsc_ok || !huc_ok) {
+		drm_info(&xe->drm, "Skipping PXP due to unsatisfied FW deps - GSC=%s, HuC=%s\n",
+			 str_yes_no(gsc_ok),
+			 MEDIA_VER(xe) >= 35 ? "not needed" : str_yes_no(huc_ok));
 		return 0;
 	}
 
diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c
index 8c7d54498f384..dc975f5953683 100644
--- a/drivers/gpu/drm/xe/xe_query.c
+++ b/drivers/gpu/drm/xe/xe_query.c
@@ -119,6 +119,7 @@ query_engine_cycles(struct xe_device *xe,
 	struct drm_xe_engine_class_instance *eci;
 	struct drm_xe_query_engine_cycles resp;
 	size_t size = sizeof(resp);
+	enum xe_force_wake_domains fw_domain;
 	__ktime_func_t cpu_clock;
 	struct xe_hw_engine *hwe;
 	struct xe_gt *gt;
@@ -154,8 +155,10 @@ query_engine_cycles(struct xe_device *xe,
 	if (!hwe)
 		return -EINVAL;
 
-	xe_with_force_wake(fw_ref, gt_to_fw(gt), XE_FORCEWAKE_ALL) {
-		if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
+	fw_domain = xe_hw_engine_to_fw_domain(hwe);
+
+	xe_with_force_wake(fw_ref, gt_to_fw(gt), fw_domain) {
+		if (!xe_force_wake_ref_has_domain(fw_ref.domains, fw_domain))
 			return -EIO;
 
 		hwe_read_timestamp(hwe, &resp.engine_cycles, &resp.cpu_timestamp,
diff --git a/drivers/gpu/drm/xe/xe_ras.c b/drivers/gpu/drm/xe/xe_ras.c
index 4cb16b419b0c6..ed609912fda14 100644
--- a/drivers/gpu/drm/xe/xe_ras.c
+++ b/drivers/gpu/drm/xe/xe_ras.c
@@ -4,11 +4,24 @@
  */
 
 #include "xe_device.h"
+#include "xe_drm_ras.h"
+#include "xe_pm.h"
 #include "xe_printk.h"
 #include "xe_ras.h"
-#include "xe_ras_types.h"
 #include "xe_sysctrl.h"
 #include "xe_sysctrl_event_types.h"
+#include "xe_sysctrl_mailbox.h"
+#include "xe_sysctrl_mailbox_types.h"
+
+#define CORE_COMPUTE_UNCORR_TYPE	GENMASK(26, 25)
+/*
+ * Uncorrectable error type for core compute errors.
+ * 0 - Correctable Error
+ * 1 - Local Uncorrectable Error
+ * 2 - Global Uncorrectable Error
+ * 3 - Informational Error
+ */
+#define  GLOBAL_UNCORR_ERROR		2
 
 /* Severity of detected errors  */
 enum xe_ras_severity {
@@ -31,6 +44,25 @@ enum xe_ras_component {
 	XE_RAS_COMP_MAX
 };
 
+/* RAS response status codes */
+enum xe_ras_response_status {
+	XE_RAS_STATUS_SUCCESS = 0,
+	XE_RAS_STATUS_INVALID_PARAM,
+	XE_RAS_STATUS_OP_NOT_SUPPORTED,
+	XE_RAS_STATUS_TIMEOUT,
+	XE_RAS_STATUS_HARDWARE_FAILURE,
+	XE_RAS_STATUS_INSUFFICIENT_RESOURCES,
+	XE_RAS_STATUS_MAX
+};
+
+/* GPU health values */
+enum xe_ras_health {
+	XE_RAS_HEALTH_OK = 0,
+	XE_RAS_HEALTH_WARNING,
+	XE_RAS_HEALTH_CRITICAL,
+	XE_RAS_HEALTH_MAX
+};
+
 static const char *const xe_ras_severities[] = {
 	[XE_RAS_SEV_NOT_SUPPORTED]		= "Not Supported",
 	[XE_RAS_SEV_CORRECTABLE]		= "Correctable Error",
@@ -50,6 +82,63 @@ static const char *const xe_ras_components[] = {
 };
 static_assert(ARRAY_SIZE(xe_ras_components) == XE_RAS_COMP_MAX);
 
+static const char * const gpu_health_states[] = {
+	[XE_RAS_HEALTH_OK]		= "ok",
+	[XE_RAS_HEALTH_WARNING]		= "warning",
+	[XE_RAS_HEALTH_CRITICAL]	= "critical",
+};
+static_assert(ARRAY_SIZE(gpu_health_states) == XE_RAS_HEALTH_MAX);
+
+static u8 drm_to_xe_ras_severity(u8 severity)
+{
+	switch (severity) {
+	case DRM_XE_RAS_ERR_SEV_CORRECTABLE:
+		return XE_RAS_SEV_CORRECTABLE;
+	case DRM_XE_RAS_ERR_SEV_UNCORRECTABLE:
+		return XE_RAS_SEV_UNCORRECTABLE;
+	default:
+		return XE_RAS_SEV_NOT_SUPPORTED;
+	}
+}
+
+static u8 drm_to_xe_ras_component(u8 component)
+{
+	switch (component) {
+	case DRM_XE_RAS_ERR_COMP_CORE_COMPUTE:
+		return XE_RAS_COMP_CORE_COMPUTE;
+	case DRM_XE_RAS_ERR_COMP_SOC_INTERNAL:
+		return XE_RAS_COMP_SOC_INTERNAL;
+	case DRM_XE_RAS_ERR_COMP_DEVICE_MEMORY:
+		return XE_RAS_COMP_DEVICE_MEMORY;
+	case DRM_XE_RAS_ERR_COMP_PCIE:
+		return XE_RAS_COMP_PCIE;
+	case DRM_XE_RAS_ERR_COMP_FABRIC:
+		return XE_RAS_COMP_FABRIC;
+	default:
+		return XE_RAS_COMP_NOT_SUPPORTED;
+	}
+}
+
+static int ras_status_to_errno(u32 status)
+{
+	switch (status) {
+	case XE_RAS_STATUS_SUCCESS:
+		return 0;
+	case XE_RAS_STATUS_INVALID_PARAM:
+		return -EINVAL;
+	case XE_RAS_STATUS_OP_NOT_SUPPORTED:
+		return -EOPNOTSUPP;
+	case XE_RAS_STATUS_TIMEOUT:
+		return -ETIMEDOUT;
+	case XE_RAS_STATUS_HARDWARE_FAILURE:
+		return -EIO;
+	case XE_RAS_STATUS_INSUFFICIENT_RESOURCES:
+		return -ENOSPC;
+	default:
+		return -EPROTO;
+	}
+}
+
 static inline const char *sev_to_str(u8 severity)
 {
 	if (severity >= XE_RAS_SEV_MAX)
@@ -66,6 +155,86 @@ static inline const char *comp_to_str(u8 component)
 	return xe_ras_components[component];
 }
 
+static struct pci_dev *find_usp_dev(struct pci_dev *pdev)
+{
+	struct pci_dev *vsp;
+
+	/*
+	 * Device Hierarchy:
+	 *
+	 * Upstream Switch Port (USP) --> Virtual Switch Port (VSP) --> SGunit (GPU endpoint)
+	 */
+	vsp = pci_upstream_bridge(pdev);
+	if (!vsp)
+		return NULL;
+
+	return pci_upstream_bridge(vsp);
+}
+
+static void ras_usp_aer_init(struct xe_device *xe)
+{
+	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
+	struct pci_dev *usp;
+	u16 aer_cap;
+	u32 status;
+
+	usp = find_usp_dev(pdev);
+	if (!usp)
+		return;
+
+	aer_cap = pci_find_ext_capability(usp, PCI_EXT_CAP_ID_ERR);
+	if (!aer_cap) {
+		dev_warn(&usp->dev, "AER capability unavailable\n");
+		return;
+	}
+
+	/*
+	 * Clear any stale Uncorrectable Internal Error Status event in Uncorrectable Error
+	 * Status Register.
+	 */
+	pci_read_config_dword(usp, aer_cap + PCI_ERR_UNCOR_STATUS, &status);
+	if (status & PCI_ERR_UNC_INTN)
+		pci_write_config_dword(usp, aer_cap + PCI_ERR_UNCOR_STATUS, PCI_ERR_UNC_INTN);
+
+	/*
+	 * All errors are steered to USP which is a PCIe AER Compliant device.
+	 * Downgrade all the errors to non-fatal to prevent PCIe bus driver
+	 * from triggering a Secondary Bus Reset (SBR). This allows error
+	 * detection, containment and recovery in the driver.
+	 *
+	 * The Uncorrectable Error Severity Register has the 'Uncorrectable
+	 * Internal Error Severity' set to fatal by default. Set this to
+	 * non-fatal and unmask the error.
+	 */
+
+	/* Downgrade Uncorrectable Internal Error to non-fatal */
+	pci_clear_and_set_config_dword(usp, aer_cap + PCI_ERR_UNCOR_SEVER, PCI_ERR_UNC_INTN, 0);
+
+	/* Unmask Uncorrectable Internal Error */
+	pci_clear_and_set_config_dword(usp, aer_cap + PCI_ERR_UNCOR_MASK, PCI_ERR_UNC_INTN, 0);
+
+	pci_save_state(usp);
+	dev_dbg(&usp->dev, "Uncorrectable Internal Errors downgraded and unmasked\n");
+}
+
+static u8 handle_core_compute_errors(struct xe_ras_error_array *arr)
+{
+	struct xe_ras_compute_error *error_info = (void *)arr->details;
+	u8 uncorr_type;
+
+	uncorr_type = FIELD_GET(CORE_COMPUTE_UNCORR_TYPE, error_info->log_header);
+
+	/* Request a reset if error is global */
+	if (uncorr_type == GLOBAL_UNCORR_ERROR)
+		return XE_RAS_RECOVERY_ACTION_RESET;
+
+	/*
+	 * No action needed for other errors.
+	 * Local errors are recovered using an engine reset by GuC.
+	 */
+	return XE_RAS_RECOVERY_ACTION_RECOVERED;
+}
+
 void xe_ras_counter_threshold_crossed(struct xe_device *xe,
 				      struct xe_sysctrl_event_response *response)
 {
@@ -91,3 +260,368 @@ void xe_ras_counter_threshold_crossed(struct xe_device *xe,
 			comp_to_str(component), sev_to_str(severity));
 	}
 }
+
+static int get_counter(struct xe_device *xe, struct xe_ras_error_class *counter, u32 *value)
+{
+	struct xe_ras_get_counter_response response = {0};
+	struct xe_ras_get_counter_request request = {0};
+	struct xe_sysctrl_mailbox_command command = {0};
+	struct xe_ras_error_common *common;
+	size_t rlen;
+	int ret;
+
+	request.counter = *counter;
+
+	xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, XE_SYSCTRL_CMD_GET_COUNTER,
+				  &request, sizeof(request), &response, sizeof(response));
+
+	ret = xe_sysctrl_send_command(&xe->sc, &command, &rlen);
+	if (ret) {
+		xe_err(xe, "sysctrl: failed to get counter %d\n", ret);
+		return ret;
+	}
+
+	if (rlen != sizeof(response)) {
+		xe_err(xe, "sysctrl: unexpected get counter response length %zu (expected %zu)\n",
+		       rlen, sizeof(response));
+		return -EIO;
+	}
+
+	common = &response.counter.common;
+	*value = response.value;
+
+	xe_dbg(xe, "[RAS]: get counter %u for %s %s\n", *value, comp_to_str(common->component),
+	       sev_to_str(common->severity));
+
+	return 0;
+}
+
+/**
+ * xe_ras_process_errors() - Process and contain hardware errors
+ * @xe: xe device instance
+ *
+ * Get error details from system controller and return recovery
+ * method.
+ *
+ * Returns: recovery action to be taken
+ */
+enum xe_ras_recovery_action xe_ras_process_errors(struct xe_device *xe)
+{
+	struct xe_sysctrl_mailbox_command command = {0};
+	enum xe_ras_recovery_action final_action;
+	u32 remaining = XE_SYSCTRL_FLOOD_LIMIT;
+	struct xe_ras_get_soc_error response;
+	size_t rlen;
+	int ret;
+
+	if (!xe->info.has_sysctrl)
+		return XE_RAS_RECOVERY_ACTION_RESET;
+
+	/* Default action */
+	final_action = XE_RAS_RECOVERY_ACTION_RECOVERED;
+
+	xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, XE_SYSCTRL_CMD_GET_SOC_ERROR,
+				  NULL, 0, &response, sizeof(response));
+
+	do {
+		memset(&response, 0, sizeof(response));
+
+		ret = xe_sysctrl_send_command(&xe->sc, &command, &rlen);
+		if (ret) {
+			xe_err(xe, "sysctrl: failed to get soc error %d\n", ret);
+			goto err;
+		}
+
+		if (rlen != sizeof(response)) {
+			xe_err(xe, "sysctrl: unexpected get soc error response length %zu (expected %zu)\n",
+			       rlen, sizeof(response));
+			goto err;
+		}
+
+		/* Report if number of errors exceeds the maximum errors supported */
+		if (response.num_errors > XE_RAS_NUM_ERROR_ARR)
+			xe_err(xe, "sysctrl: number of errors received %d out of bound (%d)\n",
+			       response.num_errors, XE_RAS_NUM_ERROR_ARR);
+
+		for (int i = 0; i < response.num_errors && i < XE_RAS_NUM_ERROR_ARR; i++) {
+			struct xe_ras_error_array *arr = &response.arr[i];
+			enum xe_ras_recovery_action action;
+			u8 component, severity;
+
+			component = arr->counter.common.component;
+			severity = arr->counter.common.severity;
+
+			xe_info(xe, "[RAS]: %s %s detected\n", comp_to_str(component),
+				sev_to_str(severity));
+
+			switch (component) {
+			case XE_RAS_COMP_CORE_COMPUTE:
+				action = handle_core_compute_errors(arr);
+				break;
+			default:
+				/* For any other component, reset */
+				action = XE_RAS_RECOVERY_ACTION_RESET;
+				break;
+			}
+
+			/* Process and log all errors and then trigger highest recovery action */
+			if (action > final_action)
+				final_action = action;
+		}
+
+		/* Treat flooding as a system controller error */
+		if (!--remaining) {
+			xe_err(xe, "[RAS]: sysctrl: get soc error response flooding\n");
+			goto err;
+		}
+
+	} while (response.additional_errors);
+
+	return final_action;
+
+err:
+	return XE_RAS_RECOVERY_ACTION_RESET;
+}
+
+/**
+ * xe_ras_get_counter() - Get error counter value
+ * @xe: Xe device instance
+ * @severity: Error severity to be queried (&enum drm_xe_ras_error_severity)
+ * @component: Error component to be queried (&enum drm_xe_ras_error_component)
+ * @value: Counter value
+ *
+ * This function retrieves the value of a specific error counter based on
+ * the error severity and component.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int xe_ras_get_counter(struct xe_device *xe, u8 severity, u8 component, u32 *value)
+{
+	struct xe_ras_error_class counter = {0};
+
+	counter.common.severity = drm_to_xe_ras_severity(severity);
+	counter.common.component = drm_to_xe_ras_component(component);
+
+	guard(xe_pm_runtime)(xe);
+	return get_counter(xe, &counter, value);
+}
+
+/**
+ * xe_ras_clear_counter() - Clear error counter value
+ * @xe: Xe device instance
+ * @severity: Error severity to be cleared (&enum drm_xe_ras_error_severity)
+ * @component: Error component to be cleared (&enum drm_xe_ras_error_component)
+ *
+ * This function clears the value of a specific error counter based on
+ * the error severity and component.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int xe_ras_clear_counter(struct xe_device *xe, u8 severity, u8 component)
+{
+	struct xe_ras_clear_counter_response response = {0};
+	struct xe_ras_clear_counter_request request = {0};
+	struct xe_sysctrl_mailbox_command command = {0};
+	struct xe_ras_error_class *counter;
+	size_t rlen;
+	int ret;
+
+	counter = &request.counter;
+	counter->common.severity = drm_to_xe_ras_severity(severity);
+	counter->common.component = drm_to_xe_ras_component(component);
+
+	xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, XE_SYSCTRL_CMD_CLEAR_COUNTER,
+				  &request, sizeof(request), &response, sizeof(response));
+
+	guard(xe_pm_runtime)(xe);
+	ret = xe_sysctrl_send_command(&xe->sc, &command, &rlen);
+	if (ret) {
+		xe_err(xe, "sysctrl: failed to clear counter %d\n", ret);
+		return ret;
+	}
+
+	if (rlen != sizeof(response)) {
+		xe_err(xe, "sysctrl: unexpected clear counter response length %zu (expected %zu)\n",
+		       rlen, sizeof(response));
+		return -EIO;
+	}
+
+	ret = ras_status_to_errno(response.status);
+	if (ret) {
+		xe_err(xe, "sysctrl: clear counter command failed with status %#x\n",
+		       response.status);
+		return ret;
+	}
+
+	counter = &response.counter;
+
+	xe_dbg(xe, "[RAS]: clear counter for %s %s\n", comp_to_str(counter->common.component),
+	       sev_to_str(counter->common.severity));
+
+	return 0;
+}
+
+static ssize_t gpu_health_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct xe_ras_get_health_response response = {0};
+	struct xe_sysctrl_mailbox_command command = {0};
+	struct xe_ras_get_health_request request = {0};
+	struct xe_device *xe = kdev_to_xe_device(dev);
+	const char *health;
+	size_t rlen;
+	int ret;
+
+	xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, XE_SYSCTRL_CMD_GET_HEALTH,
+				  &request, sizeof(request), &response, sizeof(response));
+	guard(xe_pm_runtime)(xe);
+	ret = xe_sysctrl_send_command(&xe->sc, &command, &rlen);
+	if (ret) {
+		xe_err(xe, "sysctrl: failed to get health %d\n", ret);
+		return ret;
+	}
+
+	if (rlen != sizeof(response)) {
+		xe_err(xe, "sysctrl: unexpected get health response length %zu (expected %zu)\n",
+		       rlen, sizeof(response));
+		return -EIO;
+	}
+	if (response.health >= XE_RAS_HEALTH_MAX) {
+		xe_err(xe, "sysctrl: invalid health state %u\n",
+		       response.health);
+		return -EIO;
+	}
+
+	health = gpu_health_states[response.health];
+
+	xe_dbg(xe, "[RAS]: get health: %s\n", health);
+
+	return sysfs_emit(buf, "%s\n", health);
+}
+
+static ssize_t gpu_health_store(struct device *dev, struct device_attribute *attr,
+				const char *buf, size_t count)
+{
+	struct xe_ras_set_health_response response = {0};
+	struct xe_sysctrl_mailbox_command command = {0};
+	struct xe_ras_set_health_request request = {0};
+	struct xe_device *xe = kdev_to_xe_device(dev);
+	const char *health;
+	size_t rlen;
+	int state;
+	int ret;
+
+	state = sysfs_match_string(gpu_health_states, buf);
+	if (state < 0)
+		return -EINVAL;
+
+	request.health = state;
+
+	xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, XE_SYSCTRL_CMD_SET_HEALTH,
+				  &request, sizeof(request), &response, sizeof(response));
+	guard(xe_pm_runtime)(xe);
+	ret = xe_sysctrl_send_command(&xe->sc, &command, &rlen);
+	if (ret) {
+		xe_err(xe, "sysctrl: failed to set health %d\n", ret);
+		return ret;
+	}
+
+	if (rlen != sizeof(response)) {
+		xe_err(xe, "sysctrl: unexpected set health response length %zu (expected %zu)\n",
+		       rlen, sizeof(response));
+		return -EIO;
+	}
+
+	ret = ras_status_to_errno(response.status);
+	if (ret) {
+		xe_err(xe, "sysctrl: set health command failed with status %#x\n",
+		       response.status);
+		return ret;
+	}
+
+	if (response.health >= XE_RAS_HEALTH_MAX) {
+		xe_err(xe, "sysctrl: invalid health state %u\n",
+		       response.health);
+		return -EIO;
+	}
+
+	health = gpu_health_states[response.health];
+
+	xe_dbg(xe, "[RAS]: set health: %s\n", health);
+
+	return count;
+}
+static DEVICE_ATTR_RW(gpu_health);
+
+static struct attribute *gpu_health_attrs[] = {
+	&dev_attr_gpu_health.attr,
+	NULL
+};
+
+/**
+ * DOC: GPU Health Indicator
+ *
+ * On Intel Xe platforms that support the gpu health indicator interface,
+ * the driver exposes this sysfs attribute for in-band access to the gpu
+ * health state::
+ *
+ *     /sys/bus/pci/devices/<device>/gpu_health
+ *
+ * Reading the attribute is available to all users and returns a single
+ * line containing the current gpu health state, whereas writing is
+ * restricted to administrative users and updates the state to one of the
+ * valid values.
+ *
+ * Management tools and administrators use this interface to query the
+ * current gpu health state (e.g. for telemetry/monitoring) and to
+ * update it - for example, to mark the gpu as ``warning`` or ``critical``
+ * after diagnostics, or reset it back to ``ok`` once remediated.
+ *
+ * The valid values for the gpu health state are:
+ *
+ * - ``ok``
+ *     The gpu is healthy and operating within normal parameters.
+ *
+ * - ``warning``
+ *     The gpu is experiencing minor issues but remains operational.
+ *
+ * - ``critical``
+ *     The gpu is in a critical state and may not be operational.
+ *
+ * See Documentation/ABI/testing/sysfs-driver-intel-xe-ras for the ABI
+ * specification.
+ */
+static const struct attribute_group gpu_health_group = {
+	.attrs = gpu_health_attrs,
+};
+
+/**
+ * xe_ras_init - Initialize Xe RAS
+ * @xe: xe device instance
+ *
+ * Initialize Xe RAS
+ */
+void xe_ras_init(struct xe_device *xe)
+{
+	int ret;
+
+	if (!xe->info.has_drm_ras)
+		return;
+
+	xe_drm_ras_init(xe);
+
+	if (!xe->info.has_sysctrl)
+		return;
+
+	if (IS_ENABLED(CONFIG_PCIEAER))
+		ras_usp_aer_init(xe);
+
+	/*
+	 * During probe, process and log any errors detected by firmware while the driver was not
+	 * loaded. Critical errors such as Punit and CSC are reported through Pcode init failure,
+	 * causing the driver to enter survivability mode.
+	 */
+	xe_ras_process_errors(xe);
+	ret = devm_device_add_group(xe->drm.dev, &gpu_health_group);
+	if (ret)
+		xe_err(xe, "Failed to create GPU health sysfs, err=%d\n", ret);
+}
diff --git a/drivers/gpu/drm/xe/xe_ras.h b/drivers/gpu/drm/xe/xe_ras.h
index ea90593b62dc1..6183647340430 100644
--- a/drivers/gpu/drm/xe/xe_ras.h
+++ b/drivers/gpu/drm/xe/xe_ras.h
@@ -6,10 +6,17 @@
 #ifndef _XE_RAS_H_
 #define _XE_RAS_H_
 
+#include <linux/types.h>
+#include "xe_ras_types.h"
+
 struct xe_device;
 struct xe_sysctrl_event_response;
 
 void xe_ras_counter_threshold_crossed(struct xe_device *xe,
 				      struct xe_sysctrl_event_response *response);
+int xe_ras_get_counter(struct xe_device *xe, u8 severity, u8 component, u32 *value);
+int xe_ras_clear_counter(struct xe_device *xe, u8 severity, u8 component);
+void xe_ras_init(struct xe_device *xe);
+enum xe_ras_recovery_action xe_ras_process_errors(struct xe_device *xe);
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_ras_types.h b/drivers/gpu/drm/xe/xe_ras_types.h
index 4e63c67f806a2..766b4b41768ea 100644
--- a/drivers/gpu/drm/xe/xe_ras_types.h
+++ b/drivers/gpu/drm/xe/xe_ras_types.h
@@ -9,6 +9,25 @@
 #include <linux/types.h>
 
 #define XE_RAS_NUM_COUNTERS			16
+#define XE_RAS_NUM_ERROR_ARR			3
+
+/**
+ * enum xe_ras_recovery_action - RAS recovery actions
+ *
+ * @XE_RAS_RECOVERY_ACTION_RECOVERED: Error recovered
+ * @XE_RAS_RECOVERY_ACTION_RESET: Requires reset
+ * @XE_RAS_RECOVERY_ACTION_DISCONNECT: Requires disconnect
+ * @XE_RAS_RECOVERY_ACTION_MAX: Max action value
+ *
+ * This enum defines the possible recovery actions that can be taken in response
+ * to RAS errors.
+ */
+enum xe_ras_recovery_action {
+	XE_RAS_RECOVERY_ACTION_RECOVERED = 0,
+	XE_RAS_RECOVERY_ACTION_RESET,
+	XE_RAS_RECOVERY_ACTION_DISCONNECT,
+	XE_RAS_RECOVERY_ACTION_MAX
+};
 
 /**
  * struct xe_ras_error_common - Error fields that are common across all products
@@ -70,4 +89,133 @@ struct xe_ras_threshold_crossed {
 	struct xe_ras_error_class counters[XE_RAS_NUM_COUNTERS];
 } __packed;
 
+/**
+ * struct xe_ras_get_counter_request - Request structure for get counter
+ */
+struct xe_ras_get_counter_request {
+	/** @counter: Error counter to be queried */
+	struct xe_ras_error_class counter;
+	/** @reserved: Reserved for future use */
+	u32 reserved;
+} __packed;
+
+/**
+ * struct xe_ras_get_counter_response - Response structure for get counter
+ */
+struct xe_ras_get_counter_response {
+	/** @counter: Error counter that was queried */
+	struct xe_ras_error_class counter;
+	/** @value: Current counter value */
+	u32 value;
+	/** @timestamp: Timestamp when counter was last updated */
+	u64 timestamp;
+	/** @threshold: Threshold value for the counter */
+	u32 threshold;
+	/** @reserved: Reserved  */
+	u32 reserved[57];
+} __packed;
+
+/**
+ * struct xe_ras_clear_counter_request - Request structure for clear counter
+ */
+struct xe_ras_clear_counter_request {
+	/** @counter: Counter class to be cleared */
+	struct xe_ras_error_class counter;
+	/** @reserved: Reserved for future use */
+	u32 reserved;
+} __packed;
+
+/**
+ * struct xe_ras_clear_counter_response - Response structure for clear counter
+ */
+struct xe_ras_clear_counter_response {
+	/** @counter: Counter class that was cleared */
+	struct xe_ras_error_class counter;
+	/** @reserved: Reserved */
+	u32 reserved;
+	/** @timestamp: Timestamp when the counter was cleared */
+	u64 timestamp;
+	/** @status: Status of the clear operation */
+	u32 status;
+	/** @reserved1: Reserved for future use */
+	u32 reserved1[3];
+} __packed;
+
+/**
+ * struct xe_ras_error_array - Details of the error types
+ */
+struct xe_ras_error_array {
+	/** @value: Counter value of the detailed error */
+	u32 value;
+	/** @counter: Error counter */
+	struct xe_ras_error_class counter;
+	/** @timestamp: Timestamp */
+	u64 timestamp;
+	/** @details: Error details specific to the counter */
+	u32 details[XE_RAS_NUM_COUNTERS];
+} __packed;
+
+/**
+ * struct xe_ras_get_soc_error - Response from get soc error command
+ */
+struct xe_ras_get_soc_error {
+	/** @num_errors: Number of errors reported in this response */
+	u8 num_errors;
+	/** @additional_errors: Indicates if the errors are pending */
+	u8 additional_errors;
+	/** @arr: Array of up to 3 errors */
+	struct xe_ras_error_array arr[XE_RAS_NUM_ERROR_ARR];
+} __packed;
+
+/**
+ * struct xe_ras_compute_error - Error details of Core Compute error
+ */
+struct xe_ras_compute_error {
+	/** @log_header: Error Source and type */
+	u32 log_header;
+	/** @reserved: Reserved */
+	u32 reserved[15];
+} __packed;
+
+/**
+ * struct xe_ras_get_health_request - Request structure for obtaining gpu health
+ */
+struct xe_ras_get_health_request {
+	/** @reserved: Reserved for future use. */
+	u32 reserved[2];
+} __packed;
+
+/**
+ * struct xe_ras_get_health_response - Response structure for obtaining gpu health
+ */
+struct xe_ras_get_health_response {
+	/** @health: gpu health value */
+	u8 health;
+	/** @reserved: Reserved for future use */
+	u8 reserved[3];
+} __packed;
+
+/**
+ * struct xe_ras_set_health_request - Request structure for setting gpu health
+ */
+struct xe_ras_set_health_request {
+	/** @health: gpu health value */
+	u8 health;
+	/** @reserved: Reserved for future use */
+	u8 reserved[3];
+} __packed;
+
+/**
+ * struct xe_ras_set_health_response - Response structure for setting gpu health
+ */
+struct xe_ras_set_health_response {
+	/** @status: Status of set health operation */
+	u32 status;
+	/** @health: Resulting gpu health value */
+	u8 health;
+	/** @reserved: Reserved for future use */
+	u8 reserved[3];
+	/** @reserved1: Reserved for future use */
+	u32 reserved1[2];
+} __packed;
 #endif
diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.c b/drivers/gpu/drm/xe/xe_reg_whitelist.c
index 526907d2d8246..cab1b578ca0ef 100644
--- a/drivers/gpu/drm/xe/xe_reg_whitelist.c
+++ b/drivers/gpu/drm/xe/xe_reg_whitelist.c
@@ -5,6 +5,8 @@
 
 #include "xe_reg_whitelist.h"
 
+#include <kunit/visibility.h>
+
 #include "regs/xe_engine_regs.h"
 #include "regs/xe_gt_regs.h"
 #include "regs/xe_oa_regs.h"
@@ -41,7 +43,7 @@ static bool match_multi_queue_class(const struct xe_device *xe,
 	return xe_gt_supports_multi_queue(gt, hwe->class);
 }
 
-static const struct xe_rtp_table_sr register_whitelist = XE_RTP_TABLE_SR(
+VISIBLE_IF_KUNIT const struct xe_rtp_table_sr register_whitelist = XE_RTP_TABLE_SR(
 	{ XE_RTP_NAME("WaAllowPMDepthAndInvocationCountAccessFromUMD, 1408556865"),
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(WHITELIST(PS_INVOCATION_COUNT,
@@ -104,6 +106,7 @@ static const struct xe_rtp_table_sr register_whitelist = XE_RTP_TABLE_SR(
 				   RING_FORCE_TO_NONPRIV_ACCESS_RW))
 	},
 );
+EXPORT_SYMBOL_IF_KUNIT(register_whitelist);
 
 static const struct xe_rtp_table_sr oa_whitelist = XE_RTP_TABLE_SR(
 
diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.h b/drivers/gpu/drm/xe/xe_reg_whitelist.h
index e1eb1b7d5480b..c0248063d5153 100644
--- a/drivers/gpu/drm/xe/xe_reg_whitelist.h
+++ b/drivers/gpu/drm/xe/xe_reg_whitelist.h
@@ -14,6 +14,10 @@ struct xe_hw_engine;
 struct xe_reg_sr;
 struct xe_reg_sr_entry;
 
+#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
+extern const struct xe_rtp_table_sr register_whitelist;
+#endif
+
 void xe_reg_whitelist_process_engine(struct xe_hw_engine *hwe);
 
 void xe_reg_whitelist_oa_regs(struct xe_gt *gt);
diff --git a/drivers/gpu/drm/xe/xe_rtp.c b/drivers/gpu/drm/xe/xe_rtp.c
index 83a40e1f9528c..6a8d6ea68f25d 100644
--- a/drivers/gpu/drm/xe/xe_rtp.c
+++ b/drivers/gpu/drm/xe/xe_rtp.c
@@ -227,17 +227,23 @@ static bool rule_matches(const struct xe_device *xe,
 
 static void rtp_add_sr_entry(const struct xe_rtp_action *action,
 			     struct xe_gt *gt,
+			     struct xe_hw_engine *hwe,
 			     u32 mmio_base,
 			     struct xe_reg_sr *sr)
 {
 	struct xe_reg_sr_entry sr_entry = {
 		.reg = action->reg,
 		.clr_bits = action->clr_bits,
-		.set_bits = action->set_bits,
 		.read_mask = action->read_mask,
 	};
 
+	if (action->use_func)
+		sr_entry.set_bits = action->set_func(gt, hwe);
+	else
+		sr_entry.set_bits = action->set_bits;
+
 	sr_entry.reg.addr += mmio_base;
+
 	xe_reg_sr_add(sr, &sr_entry, gt);
 }
 
@@ -259,7 +265,7 @@ static bool rtp_process_one_sr(const struct xe_rtp_entry_sr *entry,
 		else
 			mmio_base = 0;
 
-		rtp_add_sr_entry(action, gt, mmio_base, sr);
+		rtp_add_sr_entry(action, gt, hwe, mmio_base, sr);
 	}
 
 	return true;
diff --git a/drivers/gpu/drm/xe/xe_rtp.h b/drivers/gpu/drm/xe/xe_rtp.h
index 2cc65053cd07f..0032f68ea1873 100644
--- a/drivers/gpu/drm/xe/xe_rtp.h
+++ b/drivers/gpu/drm/xe/xe_rtp.h
@@ -322,6 +322,25 @@ struct xe_reg_sr;
 	  .clr_bits = (mask_bits_), .set_bits = (val_),				\
 	  .read_mask = 0, ##__VA_ARGS__ }
 
+/**
+ * XE_RTP_ACTION_FIELD_SET_FUNC: Set a bit range to the value returned by a function
+ * @reg_: Register
+ * @mask_bits_: Mask of bits to be changed in the register, forming a field
+ * @func_: Function that returns value to set in the field denoted by @mask_bits_
+ * @...: Additional fields to override in the struct xe_rtp_action entry
+ *
+ * This macro works like XE_RTP_ACTION_FIELD_SET(), except that the
+ * field value is evaluated at the time the RTP table is processed.
+ *
+ * @func_ will only be called a single time, when the RTP table is being
+ * processed.  After processing, the value in the reg_sr entry is fixed and
+ * will not be re-evaluated.
+ */
+#define XE_RTP_ACTION_FIELD_SET_FUNC(reg_, mask_bits_, func_, ...)		\
+	{ .reg = XE_RTP_DROP_CAST(reg_),					\
+	  .clr_bits = mask_bits_, .set_func = func_, .use_func = 1,		\
+	  .read_mask = mask_bits_, ##__VA_ARGS__ }
+
 /**
  * XE_RTP_ACTION_WHITELIST - Add register to userspace whitelist
  * @reg_: Register
diff --git a/drivers/gpu/drm/xe/xe_rtp_types.h b/drivers/gpu/drm/xe/xe_rtp_types.h
index 58018ae4f8ccb..b78092fa06e05 100644
--- a/drivers/gpu/drm/xe/xe_rtp_types.h
+++ b/drivers/gpu/drm/xe/xe_rtp_types.h
@@ -22,20 +22,37 @@ struct xe_gt;
  */
 struct xe_rtp_action {
 	/** @reg: Register */
-	struct xe_reg		reg;
+	struct xe_reg reg;
+
 	/**
 	 * @clr_bits: bits to clear when updating register. It's always a
 	 * superset of bits being modified
 	 */
-	u32			clr_bits;
-	/** @set_bits: bits to set when updating register */
-	u32			set_bits;
+	u32 clr_bits;
+
+	union {
+		/** @set_bits: bits to set when updating register */
+		u32 set_bits;
+
+		/** @set_func: function to provide bits to set when updating register */
+		u32 (*set_func)(struct xe_gt *gt,
+				struct xe_hw_engine *hwe);
+	};
+
 #define XE_RTP_NOCHECK		.read_mask = 0
 	/** @read_mask: mask for bits to consider when reading value back */
-	u32			read_mask;
+	u32 read_mask;
+
 #define XE_RTP_ACTION_FLAG_ENGINE_BASE		BIT(0)
 	/** @flags: flags to apply on rule evaluation or action */
-	u8			flags;
+	u8 flags;
+
+	/**
+	 * @use_func:
+	 *   Internal flag indicating @set_func should be called instead of
+	 *   using @set_bits.
+	 */
+	u8 use_func:1;
 };
 
 enum {
@@ -69,6 +86,7 @@ struct xe_rtp_rule {
 			u8 platform;
 			u8 subplatform;
 		};
+
 		/*
 		 * MATCH_GRAPHICS_VERSION / XE_RTP_MATCH_GRAPHICS_VERSION_RANGE /
 		 * MATCH_MEDIA_VERSION  / XE_RTP_MATCH_MEDIA_VERSION_RANGE
@@ -78,15 +96,18 @@ struct xe_rtp_rule {
 #define XE_RTP_END_VERSION_UNDEFINED	U32_MAX
 			u32 ver_end;
 		};
+
 		/* MATCH_STEP */
 		struct {
 			u8 step_start;
 			u8 step_end;
 		};
+
 		/* MATCH_ENGINE_CLASS / MATCH_NOT_ENGINE_CLASS */
 		struct {
 			u8 engine_class;
 		};
+
 		/* MATCH_FUNC */
 		bool (*match_func)(const struct xe_device *xe,
 				   const struct xe_gt *gt,
diff --git a/drivers/gpu/drm/xe/xe_sched_job.c b/drivers/gpu/drm/xe/xe_sched_job.c
index ae5b38b2a884a..a4fa00632a303 100644
--- a/drivers/gpu/drm/xe/xe_sched_job.c
+++ b/drivers/gpu/drm/xe/xe_sched_job.c
@@ -214,7 +214,7 @@ void xe_sched_job_set_error(struct xe_sched_job *job, int error)
 
 	trace_xe_sched_job_set_error(job);
 
-	dma_fence_enable_sw_signaling(job->fence);
+	dma_fence_enable_signaling(job->fence);
 	xe_hw_fence_irq_run(job->q->fence_irq);
 }
 
diff --git a/drivers/gpu/drm/xe/xe_sriov_packet.c b/drivers/gpu/drm/xe/xe_sriov_packet.c
index 2ae9eff2a7c09..e9ae9c9744eab 100644
--- a/drivers/gpu/drm/xe/xe_sriov_packet.c
+++ b/drivers/gpu/drm/xe/xe_sriov_packet.c
@@ -3,6 +3,8 @@
  * Copyright © 2025 Intel Corporation
  */
 
+#include "abi/xe_driver_klvs_abi.h"
+
 #include "xe_bo.h"
 #include "xe_device.h"
 #include "xe_guc_klv_helpers.h"
@@ -352,19 +354,13 @@ ssize_t xe_sriov_packet_write_single(struct xe_device *xe, unsigned int vfid,
 	return copied;
 }
 
-#define MIGRATION_KLV_DEVICE_DEVID_KEY	0xf001u
-#define MIGRATION_KLV_DEVICE_DEVID_LEN	1u
-#define MIGRATION_KLV_DEVICE_REVID_KEY	0xf002u
-#define MIGRATION_KLV_DEVICE_REVID_LEN	1u
-
 #define MIGRATION_DESCRIPTOR_DWORDS	(GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_DEVID_LEN + \
 					 GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_REVID_LEN)
 static int pf_descriptor_init(struct xe_device *xe, unsigned int vfid)
 {
 	struct xe_sriov_packet **desc = pf_pick_descriptor(xe, vfid);
 	struct xe_sriov_packet *data;
-	unsigned int len = 0;
-	u32 *klvs;
+	u32 *klvs, *end;
 	int ret;
 
 	data = xe_sriov_packet_alloc(xe);
@@ -379,20 +375,55 @@ static int pf_descriptor_init(struct xe_device *xe, unsigned int vfid)
 	}
 
 	klvs = data->vaddr;
-	klvs[len++] = PREP_GUC_KLV_CONST(MIGRATION_KLV_DEVICE_DEVID_KEY,
-					 MIGRATION_KLV_DEVICE_DEVID_LEN);
-	klvs[len++] = xe->info.devid;
-	klvs[len++] = PREP_GUC_KLV_CONST(MIGRATION_KLV_DEVICE_REVID_KEY,
-					 MIGRATION_KLV_DEVICE_REVID_LEN);
-	klvs[len++] = xe->info.revid;
+	end = klvs + MIGRATION_DESCRIPTOR_DWORDS;
 
-	xe_assert(xe, len == MIGRATION_DESCRIPTOR_DWORDS);
+	klvs = xe_guc_klv_encode_u32(klvs, end - klvs,
+				     MIGRATION_KLV_DEVICE_DEVID_KEY,
+				     xe->info.devid);
+	klvs = xe_guc_klv_encode_u32(klvs, end - klvs,
+				     MIGRATION_KLV_DEVICE_REVID_KEY,
+				     xe->info.revid);
+	xe_assert(xe, !IS_ERR(klvs));
+	xe_assert(xe, klvs == end);
 
 	*desc = data;
 
 	return 0;
 }
 
+static int descriptor_decoder(void *arg, u16 key, u16 len, const u32 *value)
+{
+	struct xe_device *xe = arg;
+
+	xe_sriov_dbg_verbose(xe, "found KLV %#x %s\n", key, xe_guc_klv_key_to_string(key));
+
+	switch (key) {
+	case MIGRATION_KLV_DEVICE_DEVID_KEY:
+		if (*value != xe->info.devid) {
+			xe_sriov_warn(xe, "Aborting migration, devid mismatch %#06x!=%#06x\n",
+				      *value, xe->info.devid);
+			return -ENODEV;
+		}
+		break;
+	case MIGRATION_KLV_DEVICE_REVID_KEY:
+		if (*value != xe->info.revid) {
+			xe_sriov_warn(xe, "Aborting migration, revid mismatch %#06x!=%#06x\n",
+				      *value, xe->info.revid);
+			return -ENODEV;
+		}
+		break;
+	default:
+		if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
+			struct drm_printer p = xe_dbg_printer(xe);
+
+			xe_sriov_dbg(xe, "unexpected KLV %#x in descriptor!\n", key);
+			xe_guc_klv_print_one(key, len, value, &p);
+		}
+		return 0;
+	}
+	return 1;
+}
+
 /**
  * xe_sriov_packet_process_descriptor() - Process migration data descriptor packet.
  * @xe: the &xe_device
@@ -409,6 +440,7 @@ int xe_sriov_packet_process_descriptor(struct xe_device *xe, unsigned int vfid,
 {
 	u32 num_dwords = data->hdr.size / sizeof(u32);
 	u32 *klvs = data->vaddr;
+	int ret;
 
 	xe_assert(xe, data->hdr.type == XE_SRIOV_PACKET_TYPE_DESCRIPTOR);
 
@@ -418,47 +450,18 @@ int xe_sriov_packet_process_descriptor(struct xe_device *xe, unsigned int vfid,
 		return -EINVAL;
 	}
 
-	while (num_dwords >= GUC_KLV_LEN_MIN) {
-		u32 key = FIELD_GET(GUC_KLV_0_KEY, klvs[0]);
-		u32 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]);
-
-		klvs += GUC_KLV_LEN_MIN;
-		num_dwords -= GUC_KLV_LEN_MIN;
-
-		if (len > num_dwords) {
-			xe_sriov_warn(xe, "Aborting migration, truncated KLV %#x, len %u\n",
-				      key, len);
-			return -EINVAL;
-		}
-
-		switch (key) {
-		case MIGRATION_KLV_DEVICE_DEVID_KEY:
-			if (*klvs != xe->info.devid) {
-				xe_sriov_warn(xe,
-					      "Aborting migration, devid mismatch %#06x!=%#06x\n",
-					      *klvs, xe->info.devid);
-				return -ENODEV;
-			}
-			break;
-		case MIGRATION_KLV_DEVICE_REVID_KEY:
-			if (*klvs != xe->info.revid) {
-				xe_sriov_warn(xe,
-					      "Aborting migration, revid mismatch %#06x!=%#06x\n",
-					      *klvs, xe->info.revid);
-				return -ENODEV;
-			}
-			break;
-		default:
-			xe_sriov_dbg(xe,
-				     "Skipping unknown migration KLV %#x, len=%u\n",
-				     key, len);
-			print_hex_dump_bytes("desc: ", DUMP_PREFIX_OFFSET, klvs,
-					     min(SZ_64, len * sizeof(u32)));
-			break;
-		}
+	ret = xe_guc_klv_count(klvs, num_dwords);
+	if (ret < 0) {
+		xe_sriov_warn(xe, "Aborting migration, corrupted descriptor KLVs (%pe)\n",
+			      ERR_PTR(ret));
+		return ret;
+	}
 
-		klvs += len;
-		num_dwords -= len;
+	ret = xe_guc_klv_parser(klvs, num_dwords, xe, descriptor_decoder);
+	if (ret < 0) {
+		xe_sriov_warn(xe, "Aborting migration, descriptor parsing failed (%pe)\n",
+			      ERR_PTR(ret));
+		return ret;
 	}
 
 	return 0;
@@ -519,3 +522,7 @@ int xe_sriov_packet_save_init(struct xe_device *xe, unsigned int vfid)
 
 	return 0;
 }
+
+#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST)
+#include "tests/xe_sriov_packet_kunit.c"
+#endif
diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c
index 09b99fb2608bc..6787564629c65 100644
--- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c
+++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c
@@ -404,6 +404,8 @@ void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx)
 /**
  * xe_sriov_vf_ccs_attach_bo - Insert CCS read write commands in the BO.
  * @bo: the &buffer object to which batch buffer commands will be added.
+ * @new_mem: the (not yet committed) destination resource @bo is being moved
+ *          into; bo->ttm.resource is still the old resource at this point.
  *
  * This function shall be called only by VF. It inserts the PTEs and copy
  * command instructions in the BO by calling xe_migrate_ccs_rw_copy()
@@ -411,7 +413,7 @@ void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx)
  *
  * Returns: 0 if successful, negative error code on failure.
  */
-int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo)
+int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo, struct ttm_resource *new_mem)
 {
 	struct xe_device *xe = xe_bo_device(bo);
 	enum xe_sriov_vf_ccs_rw_ctxs ctx_id;
@@ -430,7 +432,21 @@ int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo)
 		xe_assert(xe, !bb);
 
 		ctx = &xe->sriov.vf.ccs.contexts[ctx_id];
-		err = xe_migrate_ccs_rw_copy(tile, ctx->mig_q, bo, ctx_id);
+		err = xe_migrate_ccs_rw_copy(tile, ctx->mig_q, bo, new_mem, ctx_id);
+		if (err)
+			goto err_unwind;
+	}
+	return 0;
+
+err_unwind:
+	/*
+	 * Clean up any contexts already attached. Can't reuse
+	 * xe_sriov_vf_ccs_detach_bo() here as it requires both contexts
+	 * attached before cleaning up either one.
+	 */
+	for_each_ccs_rw_ctx(ctx_id) {
+		if (bo->bb_ccs[ctx_id])
+			xe_migrate_ccs_rw_copy_clear(bo, ctx_id);
 	}
 	return err;
 }
diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h
index 00e58b36c510a..e1034d8521040 100644
--- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h
+++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h
@@ -11,11 +11,12 @@
 #include "xe_sriov_vf_ccs_types.h"
 
 struct drm_printer;
+struct ttm_resource;
 struct xe_device;
 struct xe_bo;
 
 int xe_sriov_vf_ccs_init(struct xe_device *xe);
-int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo);
+int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo, struct ttm_resource *new_mem);
 int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo);
 int xe_sriov_vf_ccs_register_context(struct xe_device *xe);
 void xe_sriov_vf_ccs_rebase(struct xe_device *xe);
diff --git a/drivers/gpu/drm/xe/xe_step.c b/drivers/gpu/drm/xe/xe_step.c
index fb9c31613ca7d..55c1996f689e1 100644
--- a/drivers/gpu/drm/xe/xe_step.c
+++ b/drivers/gpu/drm/xe/xe_step.c
@@ -110,12 +110,14 @@ __diag_pop();
 
 /**
  * xe_step_platform_get - Determine platform-level stepping from PCI revid
- * @xe: Xe device
+ * @platform: The Xe platform
+ * @revid: The PCI revid
+ * @step: Pointer to the step struct to update
  *
  * Convert the PCI revid into a platform-level stepping value and store that
- * in the device info.
+ * in @step->platform.
  */
-void xe_step_platform_get(struct xe_device *xe)
+void xe_step_platform_get(enum xe_platform platform, u8 revid, struct xe_step_info *step)
 {
 	/*
 	 * Not all platforms map PCI revid directly into our symbolic stepping
@@ -126,18 +128,21 @@ void xe_step_platform_get(struct xe_device *xe)
 	 * checks.
 	 */
 
-	if (xe->info.platform == XE_NOVALAKE_P)
-		xe->info.step.platform = STEP_A0 + xe->info.revid;
+	if (platform == XE_NOVALAKE_P)
+		step->platform = STEP_A0 + revid;
 }
 
 /**
  * xe_step_pre_gmdid_get - Determine IP steppings from PCI revid
  * @xe: Xe device
+ * @step: Pointer to the step struct to update
+ *
+ * Convert the PCI revid into proper IP steppings and update @step->basedie,
+ * @step->graphics and @step->media accordingly.
  *
- * Convert the PCI revid into proper IP steppings.  This should only be
- * used on platforms that do not have GMD_ID support.
+ * This should only be used on platforms that do not have GMD_ID support.
  */
-void xe_step_pre_gmdid_get(struct xe_device *xe)
+void xe_step_pre_gmdid_get(struct xe_device *xe, struct xe_step_info *step)
 {
 	const struct xe_step_info *revids = NULL;
 	u16 revid = xe->info.revid;
@@ -234,9 +239,9 @@ void xe_step_pre_gmdid_get(struct xe_device *xe)
 	}
 
 done:
-	xe->info.step.graphics = graphics;
-	xe->info.step.media = media;
-	xe->info.step.basedie = basedie;
+	step->graphics = graphics;
+	step->media = media;
+	step->basedie = basedie;
 }
 
 /**
@@ -244,8 +249,10 @@ void xe_step_pre_gmdid_get(struct xe_device *xe)
  * @xe: Xe device
  * @graphics_gmdid_revid: value of graphics GMD_ID register's revid field
  * @media_gmdid_revid: value of media GMD_ID register's revid field
+ * @step: Poninter to the step struct to update.
  *
- * Convert the revid fields of the GMD_ID registers into proper IP steppings.
+ * Convert the revid fields of the GMD_ID registers into proper IP steppings
+ * and update @step->graphics and @step->media accordingly.
  *
  * GMD_ID revid values are currently expected to have consistent meanings on
  * all platforms:  major steppings (A0, B0, etc.) are 4 apart, with minor
@@ -253,7 +260,8 @@ void xe_step_pre_gmdid_get(struct xe_device *xe)
  */
 void xe_step_gmdid_get(struct xe_device *xe,
 		       u32 graphics_gmdid_revid,
-		       u32 media_gmdid_revid)
+		       u32 media_gmdid_revid,
+		       struct xe_step_info *step)
 {
 	u8 graphics = STEP_A0 + graphics_gmdid_revid;
 	u8 media = STEP_A0 + media_gmdid_revid;
@@ -270,8 +278,8 @@ void xe_step_gmdid_get(struct xe_device *xe,
 			media_gmdid_revid);
 	}
 
-	xe->info.step.graphics = graphics;
-	xe->info.step.media = media;
+	step->graphics = graphics;
+	step->media = media;
 }
 
 #define STEP_NAME_CASE(name)	\
diff --git a/drivers/gpu/drm/xe/xe_step.h b/drivers/gpu/drm/xe/xe_step.h
index ea36b22cc297c..5a5845335740c 100644
--- a/drivers/gpu/drm/xe/xe_step.h
+++ b/drivers/gpu/drm/xe/xe_step.h
@@ -10,14 +10,16 @@
 
 #include "xe_step_types.h"
 
+enum xe_platform;
 struct xe_device;
 
-void xe_step_platform_get(struct xe_device *xe);
+void xe_step_platform_get(enum xe_platform platform, u8 revid, struct xe_step_info *step);
 
-void xe_step_pre_gmdid_get(struct xe_device *xe);
+void xe_step_pre_gmdid_get(struct xe_device *xe, struct xe_step_info *step);
 void xe_step_gmdid_get(struct xe_device *xe,
 		       u32 graphics_gmdid_revid,
-		       u32 media_gmdid_revid);
+		       u32 media_gmdid_revid,
+		       struct xe_step_info *step);
 static inline u32 xe_step_to_gmdid(enum intel_step step) { return step - STEP_A0; }
 
 const char *xe_step_name(enum intel_step step);
diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c
index b1e1ac26c66d9..b228a737cfd6d 100644
--- a/drivers/gpu/drm/xe/xe_svm.c
+++ b/drivers/gpu/drm/xe/xe_svm.c
@@ -66,7 +66,7 @@ static bool xe_svm_range_in_vram(struct xe_svm_range *range)
 
 	struct drm_gpusvm_pages_flags flags = {
 		/* Pairs with WRITE_ONCE in drm_gpusvm.c */
-		.__flags = READ_ONCE(range->base.pages.flags.__flags),
+		.__flags = READ_ONCE(range->pages.flags.__flags),
 	};
 
 	return flags.has_devmem_pages;
@@ -96,7 +96,7 @@ static struct xe_vm *range_to_vm(struct drm_gpusvm_range *r)
 	       (r__)->base.gpusvm,					\
 	       xe_svm_range_in_vram((r__)) ? 1 : 0,			\
 	       xe_svm_range_has_vram_binding((r__)) ? 1 : 0,		\
-	       (r__)->base.pages.notifier_seq,				\
+	       (r__)->pages.notifier_seq,				\
 	       xe_svm_range_start((r__)), xe_svm_range_end((r__)),	\
 	       xe_svm_range_size((r__)))
 
@@ -115,6 +115,7 @@ xe_svm_range_alloc(struct drm_gpusvm *gpusvm)
 		return NULL;
 
 	INIT_LIST_HEAD(&range->garbage_collector_link);
+	drm_gpusvm_init_pages(&range->pages, &gpusvm_to_vm(gpusvm)->xe->drm);
 	xe_vm_get(gpusvm_to_vm(gpusvm));
 
 	return &range->base;
@@ -122,8 +123,10 @@ xe_svm_range_alloc(struct drm_gpusvm *gpusvm)
 
 static void xe_svm_range_free(struct drm_gpusvm_range *range)
 {
+	drm_gpusvm_free_pages(range->gpusvm, &(to_xe_range(range)->pages),
+			      drm_gpusvm_range_size(range) >> PAGE_SHIFT);
 	xe_vm_put(range_to_vm(range));
-	kfree(range);
+	kfree(to_xe_range(range));
 }
 
 static void
@@ -134,7 +137,8 @@ xe_svm_garbage_collector_add_range(struct xe_vm *vm, struct xe_svm_range *range,
 
 	range_debug(range, "GARBAGE COLLECTOR ADD");
 
-	drm_gpusvm_range_set_unmapped(&range->base, mmu_range);
+	drm_gpusvm_range_set_unmapped(&range->base, &range->pages, 1,
+				      mmu_range);
 
 	spin_lock(&vm->svm.garbage_collector.lock);
 	if (list_empty(&range->garbage_collector_link))
@@ -166,7 +170,7 @@ xe_svm_range_notifier_event_begin(struct xe_vm *vm, struct drm_gpusvm_range *r,
 	range_debug(range, "NOTIFIER");
 
 	/* Skip if already unmapped or if no binding exist */
-	if (range->base.pages.flags.unmapped || !range->tile_present)
+	if (range->base.flags.unmapped || !range->tile_present)
 		return 0;
 
 	range_debug(range, "NOTIFIER - EXECUTE");
@@ -208,7 +212,8 @@ xe_svm_range_notifier_event_end(struct xe_vm *vm, struct drm_gpusvm_range *r,
 
 	xe_svm_assert_in_notifier(vm);
 
-	drm_gpusvm_range_unmap_pages(&vm->svm.gpusvm, r, &ctx);
+	drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &(to_xe_range(r)->pages),
+			       drm_gpusvm_range_size(r) >> PAGE_SHIFT, &ctx);
 	if (!xe_vm_is_closed(vm) && mmu_range->event == MMU_NOTIFY_UNMAP)
 		xe_svm_garbage_collector_add_range(vm, to_xe_range(r),
 						   mmu_range);
@@ -299,6 +304,7 @@ static void xe_svm_invalidate(struct drm_gpusvm *gpusvm,
 static int __xe_svm_garbage_collector(struct xe_vm *vm,
 				      struct xe_svm_range *range)
 {
+	struct drm_gpusvm_ctx ctx = { .in_notifier = false, };
 	struct dma_fence *fence;
 
 	range_debug(range, "GARBAGE COLLECTOR");
@@ -310,6 +316,10 @@ static int __xe_svm_garbage_collector(struct xe_vm *vm,
 		return PTR_ERR(fence);
 	dma_fence_put(fence);
 
+	drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &range->pages,
+			       drm_gpusvm_range_size(&range->base) >> PAGE_SHIFT,
+			       &ctx);
+
 	drm_gpusvm_range_remove(&vm->svm.gpusvm, &range->base);
 
 	return 0;
@@ -901,7 +911,7 @@ int xe_svm_init(struct xe_vm *vm)
 			return err;
 		}
 
-		err = drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM", &vm->xe->drm,
+		err = drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM",
 				      current->mm, 0, vm->size,
 				      xe_modparam.svm_notifier_size * SZ_1M,
 				      &gpusvm_ops, fault_chunk_sizes,
@@ -915,7 +925,7 @@ int xe_svm_init(struct xe_vm *vm)
 		}
 	} else {
 		err = drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM (simple)",
-				      &vm->xe->drm, NULL, 0, 0, 0, NULL,
+				      NULL, 0, 0, 0, NULL,
 				      NULL, 0);
 	}
 
@@ -944,15 +954,28 @@ void xe_svm_close(struct xe_vm *vm)
  */
 void xe_svm_fini(struct xe_vm *vm)
 {
+	struct drm_gpusvm_notifier *notifier, *next;
+	struct drm_gpusvm_ctx ctx = { .in_notifier = false, };
+
 	xe_assert(vm->xe, xe_vm_is_closed(vm));
 
+	drm_gpusvm_for_each_notifier_safe(notifier, next, &vm->svm.gpusvm, 0, LONG_MAX) {
+		struct drm_gpusvm_range *range, *__next;
+
+		drm_gpusvm_for_each_range_safe(range, __next, notifier, 0, LONG_MAX)
+			drm_gpusvm_unmap_pages(&vm->svm.gpusvm,
+					       &(to_xe_range(range)->pages),
+					       drm_gpusvm_range_size(range) >> PAGE_SHIFT,
+					       &ctx);
+	}
+
 	drm_gpusvm_fini(&vm->svm.gpusvm);
 }
 
 static bool xe_svm_range_has_pagemap_locked(const struct xe_svm_range *range,
 					    const struct drm_pagemap *dpagemap)
 {
-	return range->base.pages.dpagemap == dpagemap;
+	return range->pages.dpagemap == dpagemap;
 }
 
 static bool xe_svm_range_has_pagemap(struct xe_svm_range *range,
@@ -1017,7 +1040,7 @@ bool xe_svm_range_validate(struct xe_vm *vm,
 	if (dpagemap)
 		ret = ret && xe_svm_range_has_pagemap_locked(range, dpagemap);
 	else
-		ret = ret && !range->base.pages.dpagemap;
+		ret = ret && !range->pages.dpagemap;
 
 	xe_svm_notifier_unlock(vm);
 
@@ -1090,7 +1113,7 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap,
 				dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL,
 						      false, MAX_SCHEDULE_TIMEOUT);
 			else if (pre_migrate_fence)
-				dma_fence_enable_sw_signaling(pre_migrate_fence);
+				dma_fence_enable_signaling(pre_migrate_fence);
 		}
 
 		drm_pagemap_devmem_init(&bo->devmem_allocation, dev, mm,
@@ -1135,8 +1158,12 @@ bool xe_svm_range_needs_migrate_to_vram(struct xe_svm_range *range, struct xe_vm
 {
 	struct xe_vm *vm = range_to_vm(&range->base);
 	u64 range_size = xe_svm_range_size(range);
+	struct drm_gpusvm_range_flags flags = {
+		/* READ_ONCE pairs with WRITE_ONCE in drm_gpusvm_range_set_unmapped() */
+		.__flags = READ_ONCE(range->base.flags.__flags),
+	};
 
-	if (!range->base.pages.flags.migrate_devmem || !dpagemap)
+	if (!flags.migrate_devmem || !dpagemap)
 		return false;
 
 	xe_assert(vm->xe, IS_DGFX(vm->xe));
@@ -1220,6 +1247,7 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
 	struct xe_validation_ctx vctx;
 	struct drm_exec exec;
 	struct xe_svm_range *range;
+	struct drm_gpusvm_range_flags range_flags;
 	struct dma_fence *fence;
 	struct drm_pagemap *dpagemap;
 	struct xe_tile *tile = gt_to_tile(gt);
@@ -1248,7 +1276,9 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
 
 	xe_svm_range_fault_count_stats_incr(gt, range);
 
-	if (ctx.devmem_only && !range->base.pages.flags.migrate_devmem)
+	/* READ_ONCE pairs with WRITE_ONCE in drm_gpusvm_range_set_unmapped() */
+	range_flags.__flags = READ_ONCE(range->base.flags.__flags);
+	if (ctx.devmem_only && !range_flags.migrate_devmem)
 		return -EACCES;
 
 	if (xe_svm_range_is_valid(range, tile, ctx.devmem_only, dpagemap)) {
@@ -1505,7 +1535,11 @@ int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range,
 {
 	int err = 0;
 
-	err = drm_gpusvm_range_get_pages(&vm->svm.gpusvm, &range->base, ctx);
+	err = drm_gpusvm_get_pages(&vm->svm.gpusvm, &range->pages,
+				   vm->svm.gpusvm.mm,
+				   &range->base.notifier->notifier,
+				   drm_gpusvm_range_start(&range->base),
+				   drm_gpusvm_range_end(&range->base), ctx);
 	if (err == -EOPNOTSUPP) {
 		range_debug(range, "PAGE FAULT - EVICT PAGES");
 		drm_gpusvm_range_evict(&vm->svm.gpusvm, &range->base);
@@ -1620,8 +1654,12 @@ int xe_svm_alloc_vram(struct xe_svm_range *range, const struct drm_gpusvm_ctx *c
 	struct xe_device *xe = vm->xe;
 	int err, retries = 1;
 	bool write_locked = false;
+	struct drm_gpusvm_range_flags flags = {
+		/* READ_ONCE pairs with WRITE_ONCE in drm_gpusvm_range_set_unmapped() */
+		.__flags = READ_ONCE(range->base.flags.__flags),
+	};
 
-	xe_assert(range_to_vm(&range->base)->xe, range->base.pages.flags.migrate_devmem);
+	xe_assert(range_to_vm(&range->base)->xe, flags.migrate_devmem);
 	range_debug(range, "ALLOCATE VRAM");
 
 	migration_state = drm_gpusvm_scan_mm(&range->base,
diff --git a/drivers/gpu/drm/xe/xe_svm.h b/drivers/gpu/drm/xe/xe_svm.h
index 3ca46a6f98c73..a921556d34662 100644
--- a/drivers/gpu/drm/xe/xe_svm.h
+++ b/drivers/gpu/drm/xe/xe_svm.h
@@ -31,6 +31,8 @@ struct xe_vram_region;
 struct xe_svm_range {
 	/** @base: base drm_gpusvm_range */
 	struct drm_gpusvm_range base;
+	/** @pages: Page/DMA mapping state for this range (single drm_device). */
+	struct drm_gpusvm_pages pages;
 	/**
 	 * @garbage_collector_link: Link into VM's garbage collect SVM range
 	 * list. Protected by VM's garbage collect lock.
@@ -74,7 +76,7 @@ struct xe_pagemap {
  */
 static inline bool xe_svm_range_pages_valid(struct xe_svm_range *range)
 {
-	return drm_gpusvm_range_pages_valid(range->base.gpusvm, &range->base);
+	return drm_gpusvm_pages_valid(range->base.gpusvm, &range->pages);
 }
 
 int xe_devm_add(struct xe_tile *tile, struct xe_vram_region *vr);
@@ -132,7 +134,7 @@ void *xe_svm_private_page_owner(struct xe_vm *vm, bool force_smem);
 static inline bool xe_svm_range_has_dma_mapping(struct xe_svm_range *range)
 {
 	lockdep_assert_held(&range->base.gpusvm->notifier_lock);
-	return range->base.pages.flags.has_dma_mapping;
+	return range->pages.flags.has_dma_mapping;
 }
 
 /**
@@ -210,10 +212,10 @@ struct xe_vram_region;
 struct xe_svm_range {
 	struct {
 		struct interval_tree_node itree;
-		struct {
-			const struct drm_pagemap_addr *dma_addr;
-		} pages;
 	} base;
+	struct {
+		const struct drm_pagemap_addr *dma_addr;
+	} pages;
 	u32 tile_present;
 	u32 tile_invalidated;
 };
@@ -233,7 +235,7 @@ static inline
 int xe_svm_init(struct xe_vm *vm)
 {
 #if IS_ENABLED(CONFIG_DRM_GPUSVM)
-	return drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM (simple)", &vm->xe->drm,
+	return drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM (simple)",
 			       NULL, 0, 0, 0, NULL, NULL, 0);
 #else
 	return 0;
diff --git a/drivers/gpu/drm/xe/xe_sysctrl_event.c b/drivers/gpu/drm/xe/xe_sysctrl_event.c
index b4d17329af6c8..da395148ee9dc 100644
--- a/drivers/gpu/drm/xe/xe_sysctrl_event.c
+++ b/drivers/gpu/drm/xe/xe_sysctrl_event.c
@@ -16,7 +16,7 @@ static void get_pending_event(struct xe_sysctrl *sc, struct xe_sysctrl_mailbox_c
 {
 	struct xe_sysctrl_event_response *response = command->data_out;
 	struct xe_device *xe = sc_to_xe(sc);
-	u32 count = XE_SYSCTRL_EVENT_FLOOD;
+	u32 count = XE_SYSCTRL_FLOOD_LIMIT;
 	size_t len;
 	int ret;
 
diff --git a/drivers/gpu/drm/xe/xe_sysctrl_event_types.h b/drivers/gpu/drm/xe/xe_sysctrl_event_types.h
index c16c66b9fa7f7..348768ca454ab 100644
--- a/drivers/gpu/drm/xe/xe_sysctrl_event_types.h
+++ b/drivers/gpu/drm/xe/xe_sysctrl_event_types.h
@@ -10,9 +10,6 @@
 
 #define XE_SYSCTRL_EVENT_DATA_LEN		59
 
-/* Modify as needed */
-#define XE_SYSCTRL_EVENT_FLOOD			16
-
 /**
  * enum xe_sysctrl_event - Events reported by System Controller
  *
diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c
index 3caa9f15875f5..e13eebaac1d00 100644
--- a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c
+++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c
@@ -293,6 +293,34 @@ static int sysctrl_send_command(struct xe_sysctrl *sc,
 	return 0;
 }
 
+/**
+ * xe_sysctrl_create_command() - Create system controller command
+ * @command: Sysctrl command structure
+ * @group_id: Command group ID
+ * @cmd_id: Command ID
+ * @request: Pointer to request buffer (can be NULL)
+ * @request_len: Size of request buffer
+ * @response: Pointer to response buffer
+ * @response_len: Size of response buffer
+ *
+ * Helper function to create sysctrl command to be sent via %xe_sysctrl_send_command()
+ */
+void xe_sysctrl_create_command(struct xe_sysctrl_mailbox_command *command, u8 group_id, u8 cmd_id,
+			       void *request, size_t request_len, void *response,
+			       size_t response_len)
+{
+	struct xe_sysctrl_app_msg_hdr header = {0};
+
+	header.data = FIELD_PREP(APP_HDR_GROUP_ID_MASK, group_id) |
+		      FIELD_PREP(APP_HDR_COMMAND_MASK, cmd_id);
+
+	command->header = header;
+	command->data_in = request;
+	command->data_in_len = request_len;
+	command->data_out = response;
+	command->data_out_len = response_len;
+}
+
 /**
  * xe_sysctrl_mailbox_init - Initialize System Controller mailbox interface
  * @sc: System controller structure
diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h
index f67e9234de482..fb434cc165b2a 100644
--- a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h
+++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h
@@ -23,6 +23,9 @@ struct xe_sysctrl_mailbox_command;
 #define XE_SYSCTRL_APP_HDR_VERSION(hdr) \
 	FIELD_GET(APP_HDR_VERSION_MASK, (hdr)->data)
 
+void xe_sysctrl_create_command(struct xe_sysctrl_mailbox_command *command, u8 group_id, u8 cmd_id,
+			       void *request, size_t request_len, void *response,
+			       size_t response_len);
 void xe_sysctrl_mailbox_init(struct xe_sysctrl *sc);
 int xe_sysctrl_send_command(struct xe_sysctrl *sc,
 			    struct xe_sysctrl_mailbox_command *cmd,
diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h
index 84d7c647e7432..d0341538ad058 100644
--- a/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h
+++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h
@@ -22,10 +22,20 @@ enum xe_sysctrl_group {
 /**
  * enum xe_sysctrl_gfsp_cmd - Commands supported by GFSP group
  *
+ * @XE_SYSCTRL_CMD_GET_SOC_ERROR: Retrieve basic error information
+ * @XE_SYSCTRL_CMD_GET_COUNTER: Get error counter value
+ * @XE_SYSCTRL_CMD_CLEAR_COUNTER: Clear error counter value
  * @XE_SYSCTRL_CMD_GET_PENDING_EVENT: Retrieve pending event
+ * @XE_SYSCTRL_CMD_GET_HEALTH: Retrieve gpu health
+ * @XE_SYSCTRL_CMD_SET_HEALTH: Set gpu health
  */
 enum xe_sysctrl_gfsp_cmd {
+	XE_SYSCTRL_CMD_GET_SOC_ERROR		= 0x01,
+	XE_SYSCTRL_CMD_GET_COUNTER		= 0x03,
+	XE_SYSCTRL_CMD_CLEAR_COUNTER		= 0x04,
 	XE_SYSCTRL_CMD_GET_PENDING_EVENT	= 0x07,
+	XE_SYSCTRL_CMD_GET_HEALTH		= 0x0B,
+	XE_SYSCTRL_CMD_SET_HEALTH		= 0x0C,
 };
 
 /**
@@ -48,6 +58,9 @@ struct xe_sysctrl_mailbox_command {
 	size_t data_out_len;
 };
 
+/* Modify as needed */
+#define XE_SYSCTRL_FLOOD_LIMIT		16
+
 #define XE_SYSCTRL_MB_FRAME_SIZE	16
 #define XE_SYSCTRL_MB_MAX_FRAMES	64
 #define XE_SYSCTRL_MB_MAX_MESSAGE_SIZE	\
diff --git a/drivers/gpu/drm/xe/xe_tile.c b/drivers/gpu/drm/xe/xe_tile.c
index c465aae7883cc..74d925a337b76 100644
--- a/drivers/gpu/drm/xe/xe_tile.c
+++ b/drivers/gpu/drm/xe/xe_tile.c
@@ -157,7 +157,9 @@ int xe_tile_init_early(struct xe_tile *tile, struct xe_device *xe, u8 id)
 	if (err)
 		return err;
 
-	xe_pcode_init(tile);
+	err = xe_pcode_init_early(tile);
+	if (err)
+		return err;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/xe/xe_trace.h b/drivers/gpu/drm/xe/xe_trace.h
index 750fa32c13b21..2fe8f89a1e34f 100644
--- a/drivers/gpu/drm/xe/xe_trace.h
+++ b/drivers/gpu/drm/xe/xe_trace.h
@@ -213,6 +213,11 @@ DEFINE_EVENT(xe_exec_queue, xe_exec_queue_memory_cat_error,
 	     TP_ARGS(q)
 );
 
+DEFINE_EVENT(xe_exec_queue, xe_guc_uncorrectable_error,
+	     TP_PROTO(struct xe_exec_queue *q),
+	     TP_ARGS(q)
+);
+
 DEFINE_EVENT(xe_exec_queue, xe_exec_queue_cgp_context_error,
 	     TP_PROTO(struct xe_exec_queue *q),
 	     TP_ARGS(q)
diff --git a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c
index 5e9070739e653..60f39efb02d05 100644
--- a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c
+++ b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c
@@ -61,7 +61,7 @@ static u32 get_wopcm_size(struct xe_device *xe)
 	return wopcm_size;
 }
 
-static u64 detect_bar2_dgfx(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr)
+static u64 detect_lmembar_dgfx(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr)
 {
 	struct xe_vram_region *tile_vram = xe_device_get_root_tile(xe)->mem.vram;
 	resource_size_t tile_io_start = xe_vram_region_io_start(tile_vram);
@@ -102,7 +102,7 @@ static u64 detect_bar2_dgfx(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr)
 	return ALIGN_DOWN(stolen_size, SZ_1M);
 }
 
-static u32 detect_bar2_integrated(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr)
+static u32 detect_lmembar_integrated(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr)
 {
 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
 	struct xe_gt *media_gt = xe_device_get_root_tile(xe)->media_gt;
@@ -212,9 +212,9 @@ int xe_ttm_stolen_mgr_init(struct xe_device *xe)
 	if (IS_SRIOV_VF(xe))
 		stolen_size = 0;
 	else if (IS_DGFX(xe))
-		stolen_size = detect_bar2_dgfx(xe, mgr);
+		stolen_size = detect_lmembar_dgfx(xe, mgr);
 	else if (GRAPHICS_VERx100(xe) >= 1270)
-		stolen_size = detect_bar2_integrated(xe, mgr);
+		stolen_size = detect_lmembar_integrated(xe, mgr);
 	else
 		stolen_size = detect_stolen(xe, mgr);
 
@@ -262,9 +262,9 @@ u64 xe_ttm_stolen_io_offset(struct xe_bo *bo, u32 offset)
 	return mgr->io_base + (bo->ttm.resource->start << PAGE_SHIFT) + offset;
 }
 
-static int __xe_ttm_stolen_io_mem_reserve_bar2(struct xe_device *xe,
-					       struct xe_ttm_stolen_mgr *mgr,
-					       struct ttm_resource *mem)
+static int __xe_ttm_stolen_io_mem_reserve_lmembar(struct xe_device *xe,
+						  struct xe_ttm_stolen_mgr *mgr,
+						  struct ttm_resource *mem)
 {
 	if (!mgr->io_base)
 		return -EIO;
@@ -321,7 +321,7 @@ int xe_ttm_stolen_io_mem_reserve(struct xe_device *xe, struct ttm_resource *mem)
 	if (xe_ttm_stolen_cpu_access_needs_ggtt(xe))
 		return __xe_ttm_stolen_io_mem_reserve_stolen(xe, mgr, mem);
 	else
-		return __xe_ttm_stolen_io_mem_reserve_bar2(xe, mgr, mem);
+		return __xe_ttm_stolen_io_mem_reserve_lmembar(xe, mgr, mem);
 }
 
 u64 xe_ttm_stolen_gpu_offset(struct xe_device *xe)
diff --git a/drivers/gpu/drm/xe/xe_tuning.c b/drivers/gpu/drm/xe/xe_tuning.c
index bf3fad9cdbef4..bcec40ca2d35f 100644
--- a/drivers/gpu/drm/xe/xe_tuning.c
+++ b/drivers/gpu/drm/xe/xe_tuning.c
@@ -20,7 +20,7 @@
 #undef XE_REG_MCR
 #define XE_REG_MCR(...)     XE_REG(__VA_ARGS__, .mcr = 1)
 
-static const struct xe_rtp_table_sr gt_tunings = XE_RTP_TABLE_SR(
+VISIBLE_IF_KUNIT const struct xe_rtp_table_sr gt_tunings = XE_RTP_TABLE_SR(
 	{ XE_RTP_NAME("Tuning: Blend Fill Caching Optimization Disable"),
 	  XE_RTP_RULES(PLATFORM(DG2)),
 	  XE_RTP_ACTIONS(SET(XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS))
@@ -101,6 +101,7 @@ static const struct xe_rtp_table_sr gt_tunings = XE_RTP_TABLE_SR(
 				   BANK_HASH_4KB_MODE))
 	},
 );
+EXPORT_SYMBOL_IF_KUNIT(gt_tunings);
 
 static const struct xe_rtp_table_sr engine_tunings = XE_RTP_TABLE_SR(
 	{ XE_RTP_NAME("Tuning: L3 Hashing Mask"),
diff --git a/drivers/gpu/drm/xe/xe_tuning.h b/drivers/gpu/drm/xe/xe_tuning.h
index d18e187debf63..869564e3e992d 100644
--- a/drivers/gpu/drm/xe/xe_tuning.h
+++ b/drivers/gpu/drm/xe/xe_tuning.h
@@ -6,6 +6,8 @@
 #ifndef _XE_TUNING_H_
 #define _XE_TUNING_H_
 
+#include <kunit/visibility.h>
+
 struct drm_printer;
 struct xe_gt;
 struct xe_hw_engine;
@@ -16,4 +18,8 @@ void xe_tuning_process_engine(struct xe_hw_engine *hwe);
 void xe_tuning_process_lrc(struct xe_hw_engine *hwe);
 int xe_tuning_dump(struct xe_gt *gt, struct drm_printer *p);
 
+#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
+extern const struct xe_rtp_table_sr gt_tunings;
+#endif
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_uc_fw.c b/drivers/gpu/drm/xe/xe_uc_fw.c
index 3f08a3b540628..a8e6f18cc9b4a 100644
--- a/drivers/gpu/drm/xe/xe_uc_fw.c
+++ b/drivers/gpu/drm/xe/xe_uc_fw.c
@@ -115,6 +115,7 @@ struct fw_blobs_by_type {
 #define XE_GT_TYPE_ANY XE_GT_TYPE_UNINITIALIZED
 
 #define XE_GUC_FIRMWARE_DEFS(fw_def, mmp_ver, major_ver)					\
+	fw_def(NOVALAKE_S,	GT_TYPE_ANY,	major_ver(xe,	guc,	nvl,	70, 71, 0))	\
 	fw_def(PANTHERLAKE,	GT_TYPE_ANY,	major_ver(xe,	guc,	ptl,	70, 54, 0))	\
 	fw_def(BATTLEMAGE,	GT_TYPE_ANY,	major_ver(xe,	guc,	bmg,	70, 54, 0))	\
 	fw_def(LUNARLAKE,	GT_TYPE_ANY,	major_ver(xe,	guc,	lnl,	70, 53, 0))	\
diff --git a/drivers/gpu/drm/xe/xe_userptr.c b/drivers/gpu/drm/xe/xe_userptr.c
index 6f71bc66b14eb..8b2d461ea0b28 100644
--- a/drivers/gpu/drm/xe/xe_userptr.c
+++ b/drivers/gpu/drm/xe/xe_userptr.c
@@ -180,7 +180,7 @@ xe_vma_userptr_invalidate_pass1(struct xe_vm *vm, struct xe_userptr_vma *uvma)
 	dma_resv_iter_begin(&cursor, xe_vm_resv(vm),
 			    DMA_RESV_USAGE_BOOKKEEP);
 	dma_resv_for_each_fence_unlocked(&cursor, fence) {
-		dma_fence_enable_sw_signaling(fence);
+		dma_fence_enable_signaling(fence);
 		if (signaled && !dma_fence_is_signaled(fence))
 			signaled = false;
 	}
@@ -390,19 +390,20 @@ int xe_userptr_setup(struct xe_userptr_vma *uvma, unsigned long start,
 		     unsigned long range)
 {
 	struct xe_userptr *userptr = &uvma->userptr;
+	struct xe_vm *vm = xe_vma_vm(&uvma->vma);
 	int err;
 
 	INIT_LIST_HEAD(&userptr->invalidate_link);
 	INIT_LIST_HEAD(&userptr->repin_link);
 
+	drm_gpusvm_init_pages(&userptr->pages, &vm->xe->drm);
+
 	err = mmu_interval_notifier_insert(&userptr->notifier, current->mm,
 					   start, range,
 					   &vma_userptr_notifier_ops);
 	if (err)
 		return err;
 
-	userptr->pages.notifier_seq = LONG_MAX;
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 080c2fff0e95c..57bcf0660eb8d 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -206,7 +206,23 @@ static void resume_and_reinstall_preempt_fences(struct xe_vm *vm,
 	xe_vm_assert_held(vm);
 
 	list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) {
-		q->ops->resume(q);
+		/*
+		 * Only resume queues whose suspend() actually succeeded. A
+		 * failed suspend() (e.g. killed/banned/wedged) leaves the queue
+		 * un-suspended, so it must not be resumed.
+		 *
+		 * Also skip queues that have since been reset/killed/banned/
+		 * wedged: their suspend may not have completed (suspend_pending
+		 * can still be set, e.g. a preempt fence signalled with -ENOENT
+		 * without waiting), so resuming would trip the !suspend_pending
+		 * assert in the backend. Such queues are being torn down anyway,
+		 * so leave them marked suspended and let teardown resolve their
+		 * state.
+		 */
+		if (READ_ONCE(q->lr.suspended) && !q->ops->reset_status(q)) {
+			WRITE_ONCE(q->lr.suspended, false);
+			q->ops->resume(q);
+		}
 
 		drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, q->lr.pfence,
 					 DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP);
@@ -256,7 +272,7 @@ int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
 	 */
 	wait = __xe_vm_userptr_needs_repin(vm) || preempt_fences_waiting(vm);
 	if (wait)
-		dma_fence_enable_sw_signaling(pfence);
+		dma_fence_enable_signaling(pfence);
 
 	xe_svm_notifier_unlock(vm);
 
@@ -287,7 +303,7 @@ void xe_vm_remove_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
 		--vm->preempt.num_exec_queues;
 	}
 	if (q->lr.pfence) {
-		dma_fence_enable_sw_signaling(q->lr.pfence);
+		dma_fence_enable_signaling(q->lr.pfence);
 		dma_fence_put(q->lr.pfence);
 		q->lr.pfence = NULL;
 	}
@@ -3255,11 +3271,26 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm,
 						    .request_decompress = false,
 						    .check_purged = true,
 					    });
-		if (!err && !xe_vma_has_no_bo(vma))
-			err = xe_bo_migrate(xe_vma_bo(vma),
-					    region_to_mem_type[region],
-					    NULL,
-					    exec);
+		if (!err && !xe_vma_has_no_bo(vma)) {
+			struct xe_bo *bo = xe_vma_bo(vma);
+			u32 mem_type;
+
+			if (region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC) {
+				unsigned int i;
+
+				mem_type = XE_PL_TT;
+				for (i = 0; i < bo->placement.num_placement; i++) {
+					if (mem_type_is_vram(bo->placements[i].mem_type)) {
+						mem_type = bo->placements[i].mem_type;
+						break;
+					}
+				}
+			} else {
+				mem_type = region_to_mem_type[region];
+			}
+
+			err = xe_bo_migrate(bo, mem_type, NULL, exec);
+		}
 		break;
 	}
 	default:
diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c
index 246fe18431428..0474768a38aaa 100644
--- a/drivers/gpu/drm/xe/xe_vm_madvise.c
+++ b/drivers/gpu/drm/xe/xe_vm_madvise.c
@@ -332,6 +332,20 @@ static int xe_vm_invalidate_madvise_range(struct xe_vm *vm, u64 start, u64 end)
 	return err;
 }
 
+/**
+ * madvise_range_needs_invalidation() - Check whether madvise needs invalidation
+ * @args: madvise ioctl arguments
+ *
+ * Purgeable state updates only touch VMA/BO metadata. PTEs stay valid and are
+ * zapped only if the BO is later purged.
+ *
+ * Return: true when the update needs PTE invalidation.
+ */
+static bool madvise_range_needs_invalidation(const struct drm_xe_madvise *args)
+{
+	return args->type != DRM_XE_VMA_ATTR_PURGEABLE_STATE;
+}
+
 static bool madvise_args_are_sane(struct xe_device *xe, const struct drm_xe_madvise *args)
 {
 	if (XE_IOCTL_DBG(xe, !args))
@@ -708,8 +722,9 @@ int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *fil
 	madvise_funcs[attr_type](xe, vm, madvise_range.vmas, madvise_range.num_vmas, args,
 				 &details);
 
-	err = xe_vm_invalidate_madvise_range(vm, madvise_range.addr,
-					     madvise_range.addr + args->range);
+	if (madvise_range_needs_invalidation(args))
+		err = xe_vm_invalidate_madvise_range(vm, madvise_range.addr,
+						     madvise_range.addr + args->range);
 
 	if (madvise_range.has_svm_userptr_vmas)
 		xe_svm_notifier_unlock(vm);
diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c
index b9d9fe0801aad..139434946f8f8 100644
--- a/drivers/gpu/drm/xe/xe_wa.c
+++ b/drivers/gpu/drm/xe/xe_wa.c
@@ -130,7 +130,7 @@
 __diag_push();
 __diag_ignore_all("-Woverride-init", "Allow field overrides in table");
 
-static const struct xe_rtp_table_sr gt_was = XE_RTP_TABLE_SR(
+VISIBLE_IF_KUNIT const struct xe_rtp_table_sr gt_was = XE_RTP_TABLE_SR(
 	/* Workarounds applying over a range of IPs */
 
 	{ XE_RTP_NAME("14011060649"),
@@ -293,7 +293,7 @@ static const struct xe_rtp_table_sr gt_was = XE_RTP_TABLE_SR(
 	  XE_RTP_ACTIONS(SET(MMIOATSREQLIMIT_GAM_WALK_3D,
 			     DIS_ATS_WRONLY_PG))
 	},
-	{ XE_RTP_NAME("14026144927, 16029437861"),
+	{ XE_RTP_NAME("14026144927, 16029437861, 14026127056"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0)),
 	  XE_RTP_ACTIONS(SET(L3SQCREG2, L3_SQ_DISABLE_COAMA_2WAY_COH |
 			     L3_SQ_DISABLE_COAMA))
@@ -307,6 +307,7 @@ static const struct xe_rtp_table_sr gt_was = XE_RTP_TABLE_SR(
 	  XE_RTP_ACTIONS(SET(GUC_INTR_CHICKEN, DISABLE_SIGNALING_ENGINES))
 	},
 );
+EXPORT_SYMBOL_IF_KUNIT(gt_was);
 
 static const struct xe_rtp_table_sr engine_was = XE_RTP_TABLE_SR(
 	/* Workarounds applying over a range of IPs */
@@ -586,12 +587,12 @@ static const struct xe_rtp_table_sr engine_was = XE_RTP_TABLE_SR(
 
 	/* Xe3p_LPG*/
 
-	{ XE_RTP_NAME("22021149932"),
+	{ XE_RTP_NAME("22021149932, 14026290593"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0),
 		       FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, SAMPLER_LD_LSC_DISABLE))
 	},
-	{ XE_RTP_NAME("14025676848"),
+	{ XE_RTP_NAME("14025676848, 14026270459"),
 	  XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0),
 		       FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(LSC_CHICKEN_BIT_0_UDW, LSCFE_SAME_ADDRESS_ATOMICS_COALESCING_DISABLE))
@@ -802,10 +803,11 @@ static const struct xe_rtp_entry oob_was_entries[] = {
 
 static_assert(ARRAY_SIZE(oob_was_entries)  == _XE_WA_OOB_COUNT);
 
-static __maybe_unused const struct xe_rtp_table oob_was = {
+VISIBLE_IF_KUNIT __maybe_unused const struct xe_rtp_table oob_was = {
 	.entries = oob_was_entries,
 	.n_entries = ARRAY_SIZE(oob_was_entries),
 };
+EXPORT_SYMBOL_IF_KUNIT(oob_was);
 
 static const struct xe_rtp_entry device_oob_was_entries[] = {
 #include <generated/xe_device_wa_oob.c>
@@ -813,10 +815,11 @@ static const struct xe_rtp_entry device_oob_was_entries[] = {
 
 static_assert(ARRAY_SIZE(device_oob_was_entries) == _XE_DEVICE_WA_OOB_COUNT);
 
-static __maybe_unused const struct xe_rtp_table device_oob_was = {
+VISIBLE_IF_KUNIT __maybe_unused const struct xe_rtp_table device_oob_was = {
 	.entries = device_oob_was_entries,
 	.n_entries = ARRAY_SIZE(device_oob_was_entries),
 };
+EXPORT_SYMBOL_IF_KUNIT(device_oob_was);
 
 __diag_pop();
 
diff --git a/drivers/gpu/drm/xe/xe_wa.h b/drivers/gpu/drm/xe/xe_wa.h
index a5f7d33c1b32b..f4da2b2713963 100644
--- a/drivers/gpu/drm/xe/xe_wa.h
+++ b/drivers/gpu/drm/xe/xe_wa.h
@@ -6,6 +6,7 @@
 #ifndef _XE_WA_H_
 #define _XE_WA_H_
 
+#include <kunit/visibility.h>
 #include "xe_assert.h"
 
 struct drm_printer;
@@ -24,6 +25,12 @@ void xe_wa_apply_tile_workarounds(struct xe_tile *tile);
 void xe_wa_device_dump(struct xe_device *xe, struct drm_printer *p);
 int xe_wa_gt_dump(struct xe_gt *gt, struct drm_printer *p);
 
+#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
+extern const struct xe_rtp_table_sr gt_was;
+extern __maybe_unused const struct xe_rtp_table oob_was;
+extern __maybe_unused const struct xe_rtp_table device_oob_was;
+#endif
+
 /**
  * XE_GT_WA - Out-of-band GT workarounds, to be queried and called as needed.
  * @gt__: gt instance
diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules
index f8a185103b805..5d6574ec9deec 100644
--- a/drivers/gpu/drm/xe/xe_wa_oob.rules
+++ b/drivers/gpu/drm/xe/xe_wa_oob.rules
@@ -65,3 +65,9 @@
 
 14025883347	MEDIA_VERSION_RANGE(1301, 3503)
 		GRAPHICS_VERSION_RANGE(2004, 3005)
+16029380221	MEDIA_VERSION(3500)
+22022079272	MEDIA_VERSION(3503)
+		GRAPHICS_VERSION(3510)
+		GRAPHICS_VERSION(3511)
+16029897822	MEDIA_VERSION(3500)
+		GRAPHICS_VERSION(3510)
diff --git a/drivers/gpu/drm/xe/xe_wopcm.c b/drivers/gpu/drm/xe/xe_wopcm.c
index 900daf1d1b1bb..fe65ed246775d 100644
--- a/drivers/gpu/drm/xe/xe_wopcm.c
+++ b/drivers/gpu/drm/xe/xe_wopcm.c
@@ -49,9 +49,9 @@
  */
 
 /* Default WOPCM size is 2MB from Gen11, 1MB on previous platforms */
-/* FIXME: Larger size require for 2 tile PVC, do a proper probe sooner or later */
+/* FIXME: Larger size require for some platforms, do a proper probe sooner or later */
 #define DGFX_WOPCM_SIZE			SZ_4M
-/* FIXME: Larger size require for MTL, do a proper probe sooner or later */
+#define LNL_WOPCM_SIZE			SZ_8M
 #define MTL_WOPCM_SIZE			SZ_4M
 #define WOPCM_SIZE			SZ_2M
 
@@ -179,9 +179,14 @@ static int __wopcm_init_regs(struct xe_device *xe, struct xe_gt *gt,
 
 u32 xe_wopcm_size(struct xe_device *xe)
 {
-	return IS_DGFX(xe) ? DGFX_WOPCM_SIZE :
-		xe->info.platform == XE_METEORLAKE ? MTL_WOPCM_SIZE :
-		WOPCM_SIZE;
+	if (xe->info.platform >= XE_LUNARLAKE)
+		return LNL_WOPCM_SIZE;
+	else if (IS_DGFX(xe))
+		return DGFX_WOPCM_SIZE;
+	else if (xe->info.platform == XE_METEORLAKE)
+		return MTL_WOPCM_SIZE;
+	else
+		return WOPCM_SIZE;
 }
 
 static u32 max_wopcm_size(struct xe_device *xe)
diff --git a/drivers/gpu/drm/xen/xen_drm_front_gem.c b/drivers/gpu/drm/xen/xen_drm_front_gem.c
index eec4c1da3f9e3..dd158443f55f1 100644
--- a/drivers/gpu/drm/xen/xen_drm_front_gem.c
+++ b/drivers/gpu/drm/xen/xen_drm_front_gem.c
@@ -80,7 +80,7 @@ static int xen_drm_front_gem_object_mmap(struct drm_gem_object *gem_obj,
 	 * which is mapped as Normal Inner Write-Back Outer Write-Back
 	 * Inner-Shareable.
 	 */
-	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+	vma->vm_page_prot = vma_get_page_prot(vma);
 
 	/*
 	 * vm_operations_struct.fault handler will be called if CPU access
diff --git a/drivers/gpu/drm/xlnx/zynqmp_dp.c b/drivers/gpu/drm/xlnx/zynqmp_dp.c
index 7fb11b0a44f0c..b209582bc130e 100644
--- a/drivers/gpu/drm/xlnx/zynqmp_dp.c
+++ b/drivers/gpu/drm/xlnx/zynqmp_dp.c
@@ -2280,7 +2280,7 @@ static const struct drm_bridge_funcs zynqmp_dp_bridge_funcs = {
 	.atomic_disable = zynqmp_dp_bridge_atomic_disable,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_check = zynqmp_dp_bridge_atomic_check,
 	.detect = zynqmp_dp_bridge_detect,
 	.edid_read = zynqmp_dp_bridge_edid_read,
diff --git a/drivers/gpu/host1x/Makefile b/drivers/gpu/host1x/Makefile
index fead483af0b4f..b684fbf73841b 100644
--- a/drivers/gpu/host1x/Makefile
+++ b/drivers/gpu/host1x/Makefile
@@ -17,7 +17,8 @@ host1x-y = \
 	hw/host1x05.o \
 	hw/host1x06.o \
 	hw/host1x07.o \
-	hw/host1x08.o
+	hw/host1x08.o \
+	hw/host1x10.o
 
 host1x-$(CONFIG_IOMMU_API) += \
 	context.o
diff --git a/drivers/gpu/host1x/bus.c b/drivers/gpu/host1x/bus.c
index e3884096c2fe1..733f53e69eacc 100644
--- a/drivers/gpu/host1x/bus.c
+++ b/drivers/gpu/host1x/bus.c
@@ -508,7 +508,7 @@ static int host1x_device_add(struct host1x *host1x,
 	 * Add device even if there are no subdevs to ensure syncpoint functionality
 	 * is available regardless of whether any engine subdevices are present
 	 */
-	if (list_empty(&device->subdevs)) {
+	if (list_empty(&device->subdevs) && !device->registered) {
 		err = device_add(&device->dev);
 		if (err < 0)
 			dev_err(&device->dev, "failed to add device: %d\n", err);
@@ -1012,10 +1012,10 @@ void host1x_bo_clear_cached_mappings(struct host1x_bo *bo)
 		if (WARN_ON(!cache))
 			continue;
 
-		mutex_lock(&mapping->cache->lock);
+		mutex_lock(&cache->lock);
 		WARN_ON(kref_read(&mapping->ref) != 1);
 		__host1x_bo_unpin(&mapping->ref);
-		mutex_unlock(&mapping->cache->lock);
+		mutex_unlock(&cache->lock);
 	}
 }
 EXPORT_SYMBOL(host1x_bo_clear_cached_mappings);
diff --git a/drivers/gpu/host1x/cdma.c b/drivers/gpu/host1x/cdma.c
index ba2e572567c03..f6d3db2c8c395 100644
--- a/drivers/gpu/host1x/cdma.c
+++ b/drivers/gpu/host1x/cdma.c
@@ -13,6 +13,7 @@
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/kfifo.h>
+#include <linux/overflow.h>
 #include <linux/slab.h>
 #include <trace/events/host1x.h>
 
@@ -419,7 +420,7 @@ void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
 		/* won't need a timeout when replayed */
 		job->timeout = 0;
 
-		syncpt_incrs = job->syncpt_end - syncpt_val;
+		syncpt_incrs = wrapping_sub(u32, job->syncpt_end, syncpt_val);
 		dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);
 
 		host1x_job_dump(dev, job);
diff --git a/drivers/gpu/host1x/context.c b/drivers/gpu/host1x/context.c
index 512f3d189ebfe..94ecc8769c930 100644
--- a/drivers/gpu/host1x/context.c
+++ b/drivers/gpu/host1x/context.c
@@ -24,7 +24,7 @@ int host1x_memory_context_list_init(struct host1x *host1x)
 	struct host1x_memory_context_list *cdl = &host1x->context_list;
 	struct device_node *node = host1x->dev->of_node;
 	struct host1x_memory_context *ctx;
-	unsigned int i;
+	unsigned int devs, i;
 	int err;
 
 	cdl->devs = NULL;
@@ -35,7 +35,16 @@ int host1x_memory_context_list_init(struct host1x *host1x)
 	if (err < 0)
 		return 0;
 
-	cdl->len = err / 4;
+	devs = 0;
+
+	for (i = 0; i < err / 4; i++) {
+		u32 length;
+
+		of_property_read_u32_index(node, "iommu-map", i * 4 + 3, &length);
+		devs += length;
+	}
+
+	cdl->len = devs;
 	cdl->devs = kzalloc_objs(*cdl->devs, cdl->len);
 	if (!cdl->devs)
 		return -ENOMEM;
diff --git a/drivers/gpu/host1x/debug.c b/drivers/gpu/host1x/debug.c
index 6433c00d5d7e0..b828f773fc065 100644
--- a/drivers/gpu/host1x/debug.c
+++ b/drivers/gpu/host1x/debug.c
@@ -31,7 +31,7 @@ void host1x_debug_output(struct output *o, const char *fmt, ...)
 	int len;
 
 	va_start(args, fmt);
-	len = vsnprintf(o->buf, sizeof(o->buf), fmt, args);
+	len = vscnprintf(o->buf, sizeof(o->buf), fmt, args);
 	va_end(args);
 
 	o->fn(o->ctx, o->buf, len, false);
@@ -43,7 +43,7 @@ void host1x_debug_cont(struct output *o, const char *fmt, ...)
 	int len;
 
 	va_start(args, fmt);
-	len = vsnprintf(o->buf, sizeof(o->buf), fmt, args);
+	len = vscnprintf(o->buf, sizeof(o->buf), fmt, args);
 	va_end(args);
 
 	o->fn(o->ctx, o->buf, len, true);
diff --git a/drivers/gpu/host1x/dev.c b/drivers/gpu/host1x/dev.c
index 3f475f0e65458..d2c64728f8043 100644
--- a/drivers/gpu/host1x/dev.c
+++ b/drivers/gpu/host1x/dev.c
@@ -41,6 +41,7 @@
 #include "hw/host1x06.h"
 #include "hw/host1x07.h"
 #include "hw/host1x08.h"
+#include "hw/host1x10.h"
 
 void host1x_common_writel(struct host1x *host1x, u32 v, u32 r)
 {
@@ -287,7 +288,47 @@ static const struct host1x_info host1x08_info = {
 	.reserve_vblank_syncpts = false,
 };
 
+static const struct host1x_sid_entry tegra264_sid_table[] = {
+	{ /* SE1 MMIO     */  .base = 0x1650, .offset = 0x90,  .limit = 0x90  },
+	{ /* SE2 MMIO     */  .base = 0x1658, .offset = 0x90,  .limit = 0x90  },
+	{ /* SE4 MMIO     */  .base = 0x1660, .offset = 0x90,  .limit = 0x90  },
+	{ /* SE1 ch       */  .base = 0x1738, .offset = 0x90,  .limit = 0x90  },
+	{ /* SE2 ch       */  .base = 0x1740, .offset = 0x90,  .limit = 0x90  },
+	{ /* SE4 ch       */  .base = 0x1748, .offset = 0x90,  .limit = 0x90  },
+	{ /* VIC ch       */  .base = 0x1790, .offset = 0x30,  .limit = 0x30  },
+	{ /* VIC MMIO     */  .base = 0x1688, .offset = 0x34,  .limit = 0x34  },
+	{ /* TSEC MMIO    */  .base = 0x1690, .offset = 0x30,  .limit = 0x34  },
+	{ /* VI MMIO      */  .base = 0x1698, .offset = 0x800, .limit = 0x800 },
+	{ /* VI_THI MMIO  */  .base = 0x16a0, .offset = 0x30,  .limit = 0x34  },
+	{ /* ISP MMIO     */  .base = 0x1680, .offset = 0x800, .limit = 0x800 },
+	{ /* ISP_THI MMIO */  .base = 0x16a8, .offset = 0x30,  .limit = 0x34  },
+	{ /* VI2 MMIO     */  .base = 0x16b8, .offset = 0x800, .limit = 0x800 },
+	{ /* VI2_THI MMIO */  .base = 0x16c0, .offset = 0x30,  .limit = 0x34  },
+	{ /* ISP1 MMIO    */  .base = 0x16c8, .offset = 0x800, .limit = 0x800 },
+	{ /* ISP1_THI MMIO */ .base = 0x16d0, .offset = 0x30,  .limit = 0x34  },
+};
+
+static const struct host1x_info host1x10_info = {
+	.nb_channels = 63,
+	.nb_pts = 1024,
+	.nb_mlocks = 24,
+	.nb_bases = 0,
+	.init = host1x10_init,
+	.sync_offset = 0x0,
+	.dma_mask = DMA_BIT_MASK(40),
+	.has_wide_gather = true,
+	.has_hypervisor = true,
+	.has_common = true,
+	.num_sid_entries = ARRAY_SIZE(tegra264_sid_table),
+	.sid_table = tegra264_sid_table,
+	.streamid_vm_table = { 0x1004, 128 },
+	.classid_vm_table = { 0x1404, 25 },
+	.mmio_vm_table = { 0x1504, 25 },
+	.reserve_vblank_syncpts = false,
+};
+
 static const struct of_device_id host1x_of_match[] = {
+	{ .compatible = "nvidia,tegra264-host1x", .data = &host1x10_info, },
 	{ .compatible = "nvidia,tegra234-host1x", .data = &host1x08_info, },
 	{ .compatible = "nvidia,tegra194-host1x", .data = &host1x07_info, },
 	{ .compatible = "nvidia,tegra186-host1x", .data = &host1x06_info, },
diff --git a/drivers/gpu/host1x/hw/cdma_hw.c b/drivers/gpu/host1x/hw/cdma_hw.c
index 3f3f0018eee0d..08a3cf2b11a92 100644
--- a/drivers/gpu/host1x/hw/cdma_hw.c
+++ b/drivers/gpu/host1x/hw/cdma_hw.c
@@ -246,23 +246,24 @@ static void timeout_release_mlock(struct host1x_cdma *cdma)
 	 * so it turns out that if we don't /actually/ need MLOCKs, we can just
 	 * ignore them.
 	 *
-	 * As such, for now just implement this on Tegra234 where things are
-	 * stricter but also easy to implement.
+	 * As such, for now just implement this on Tegra234 and above where things
+	 * are stricter but also easy to implement.
 	 */
 	struct host1x_channel *ch = cdma_to_channel(cdma);
 	struct host1x *host1x = cdma_to_host1x(cdma);
 	u32 offset;
 
 	switch (ch->client->class) {
+	case HOST1X_CLASS_VIC:
+		offset = HOST1X_COMMON_VIC_MLOCK;
+		break;
+#if HOST1X_HW == 8
 	case HOST1X_CLASS_NVJPG1:
 		offset = HOST1X_COMMON_NVJPG1_MLOCK;
 		break;
 	case HOST1X_CLASS_NVENC:
 		offset = HOST1X_COMMON_NVENC_MLOCK;
 		break;
-	case HOST1X_CLASS_VIC:
-		offset = HOST1X_COMMON_VIC_MLOCK;
-		break;
 	case HOST1X_CLASS_NVJPG:
 		offset = HOST1X_COMMON_NVJPG_MLOCK;
 		break;
@@ -272,6 +273,7 @@ static void timeout_release_mlock(struct host1x_cdma *cdma)
 	case HOST1X_CLASS_OFA:
 		offset = HOST1X_COMMON_OFA_MLOCK;
 		break;
+#endif
 	default:
 		WARN(1, "%s was not updated for class %u", __func__, ch->client->class);
 		return;
@@ -355,7 +357,7 @@ static int cdma_timeout_init(struct host1x_cdma *cdma)
 static void cdma_timeout_destroy(struct host1x_cdma *cdma)
 {
 	if (cdma->timeout.initialized)
-		cancel_delayed_work(&cdma->timeout.wq);
+		cancel_delayed_work_sync(&cdma->timeout.wq);
 
 	cdma->timeout.initialized = false;
 }
diff --git a/drivers/gpu/host1x/hw/channel_hw.c b/drivers/gpu/host1x/hw/channel_hw.c
index 2df6a16d484e0..a8251ec0810ce 100644
--- a/drivers/gpu/host1x/hw/channel_hw.c
+++ b/drivers/gpu/host1x/hw/channel_hw.c
@@ -7,6 +7,7 @@
 
 #include <linux/host1x.h>
 #include <linux/iommu.h>
+#include <linux/overflow.h>
 #include <linux/slab.h>
 
 #include <trace/events/host1x.h>
@@ -36,10 +37,9 @@ static void trace_write_gather(struct host1x_cdma *cdma, struct host1x_bo *bo,
 		for (i = 0; i < words; i += TRACE_MAX_LENGTH) {
 			u32 num_words = min(words - i, TRACE_MAX_LENGTH);
 
-			offset += i * sizeof(u32);
-
 			trace_host1x_cdma_push_gather(dev_name(dev), bo,
-						      num_words, offset,
+						      num_words,
+						      offset + i * sizeof(u32),
 						      mem);
 		}
 
@@ -121,7 +121,8 @@ static void submit_gathers(struct host1x_job *job, struct host1x_job_cmd *cmds,
 
 		if (cmd->is_wait) {
 			if (cmd->wait.relative)
-				threshold = job_syncpt_base + cmd->wait.threshold;
+				threshold = wrapping_add(u32, job_syncpt_base,
+							 cmd->wait.threshold);
 			else
 				threshold = cmd->wait.threshold;
 
@@ -260,7 +261,8 @@ static void channel_program_cdma(struct host1x_job *job)
 
 	/* Submit work. */
 	job->syncpt_end = host1x_syncpt_incr_max(sp, job->syncpt_incrs);
-	submit_gathers(job, job->cmds + i, job->num_cmds - i, job->syncpt_end - job->syncpt_incrs);
+	submit_gathers(job, job->cmds + i, job->num_cmds - i,
+		       wrapping_sub(u32, job->syncpt_end, job->syncpt_incrs));
 
 	/* Before releasing MLOCK, ensure engine is idle again. */
 	fence = host1x_syncpt_incr_max(sp, 1);
@@ -298,7 +300,8 @@ static void channel_program_cdma(struct host1x_job *job)
 
 	job->syncpt_end = host1x_syncpt_incr_max(sp, job->syncpt_incrs);
 
-	submit_gathers(job, job->cmds, job->num_cmds, job->syncpt_end - job->syncpt_incrs);
+	submit_gathers(job, job->cmds, job->num_cmds,
+		       wrapping_sub(u32, job->syncpt_end, job->syncpt_incrs));
 #endif
 }
 
diff --git a/drivers/gpu/host1x/hw/host1x10.c b/drivers/gpu/host1x/hw/host1x10.c
new file mode 100644
index 0000000000000..2800f309bf6f8
--- /dev/null
+++ b/drivers/gpu/host1x/hw/host1x10.c
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Host1x init for Tegra264 SoCs
+ *
+ * Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ */
+
+/* include hw specification */
+#include "host1x10.h"
+#include "host1x10_hardware.h"
+
+/* include code */
+#define HOST1X_HW 10
+
+#include "cdma_hw.c"
+#include "channel_hw.c"
+#include "debug_hw.c"
+#include "intr_hw.c"
+#include "syncpt_hw.c"
+
+#include "../dev.h"
+
+int host1x10_init(struct host1x *host)
+{
+	host->channel_op = &host1x_channel_ops;
+	host->cdma_op = &host1x_cdma_ops;
+	host->cdma_pb_op = &host1x_pushbuffer_ops;
+	host->syncpt_op = &host1x_syncpt_ops;
+	host->intr_op = &host1x_intr_ops;
+	host->debug_op = &host1x_debug_ops;
+
+	return 0;
+}
diff --git a/drivers/gpu/host1x/hw/host1x10.h b/drivers/gpu/host1x/hw/host1x10.h
new file mode 100644
index 0000000000000..577f6ff3dff52
--- /dev/null
+++ b/drivers/gpu/host1x/hw/host1x10.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Host1x init for Tegra264 SoCs
+ *
+ * Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ */
+
+#ifndef HOST1X_HOST1X10_H
+#define HOST1X_HOST1X10_H
+
+struct host1x;
+
+int host1x10_init(struct host1x *host);
+
+#endif
diff --git a/drivers/gpu/host1x/hw/host1x10_hardware.h b/drivers/gpu/host1x/hw/host1x10_hardware.h
new file mode 100644
index 0000000000000..abbead8190b12
--- /dev/null
+++ b/drivers/gpu/host1x/hw/host1x10_hardware.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Tegra host1x Register Offsets for Tegra264
+ *
+ * Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ */
+
+#ifndef __HOST1X_HOST1X10_HARDWARE_H
+#define __HOST1X_HOST1X10_HARDWARE_H
+
+#include <linux/types.h>
+#include <linux/bitops.h>
+
+#include "hw_host1x10_uclass.h"
+#include "hw_host1x10_vm.h"
+#include "hw_host1x10_hypervisor.h"
+#include "hw_host1x10_common.h"
+
+#include "opcodes.h"
+
+#endif
diff --git a/drivers/gpu/host1x/hw/hw_host1x10_common.h b/drivers/gpu/host1x/hw/hw_host1x10_common.h
new file mode 100644
index 0000000000000..48a632672a47b
--- /dev/null
+++ b/drivers/gpu/host1x/hw/hw_host1x10_common.h
@@ -0,0 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ */
+
+#define HOST1X_COMMON_VIC_MLOCK			0x4060
diff --git a/drivers/gpu/host1x/hw/hw_host1x10_hypervisor.h b/drivers/gpu/host1x/hw/hw_host1x10_hypervisor.h
new file mode 100644
index 0000000000000..8c9069caffa8e
--- /dev/null
+++ b/drivers/gpu/host1x/hw/hw_host1x10_hypervisor.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ */
+
+#define HOST1X_HV_SYNCPT_PROT_EN			0x172c
+#define HOST1X_HV_SYNCPT_PROT_EN_CH_EN			BIT(1)
+#define HOST1X_HV_CH_MLOCK_EN(x)			(0x1708 + (x * 4))
+#define HOST1X_HV_CH_KERNEL_FILTER_GBUFFER(x)		(0x1718 + (x * 4))
+#define HOST1X_HV_SYNCPT_VM(x)				(0x0 + 4 * (x))
diff --git a/drivers/gpu/host1x/hw/hw_host1x10_uclass.h b/drivers/gpu/host1x/hw/hw_host1x10_uclass.h
new file mode 100644
index 0000000000000..abe83e67fa837
--- /dev/null
+++ b/drivers/gpu/host1x/hw/hw_host1x10_uclass.h
@@ -0,0 +1,181 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ */
+
+ /*
+  * Function naming determines intended use:
+  *
+  *     <x>_r(void) : Returns the offset for register <x>.
+  *
+  *     <x>_w(void) : Returns the word offset for word (4 byte) element <x>.
+  *
+  *     <x>_<y>_s(void) : Returns size of field <y> of register <x> in bits.
+  *
+  *     <x>_<y>_f(u32 v) : Returns a value based on 'v' which has been shifted
+  *         and masked to place it at field <y> of register <x>.  This value
+  *         can be |'d with others to produce a full register value for
+  *         register <x>.
+  *
+  *     <x>_<y>_m(void) : Returns a mask for field <y> of register <x>.  This
+  *         value can be ~'d and then &'d to clear the value of field <y> for
+  *         register <x>.
+  *
+  *     <x>_<y>_<z>_f(void) : Returns the constant value <z> after being shifted
+  *         to place it at field <y> of register <x>.  This value can be |'d
+  *         with others to produce a full register value for <x>.
+  *
+  *     <x>_<y>_v(u32 r) : Returns the value of field <y> from a full register
+  *         <x> value 'r' after being shifted to place its LSB at bit 0.
+  *         This value is suitable for direct comparison with other unshifted
+  *         values appropriate for use in field <y> of register <x>.
+  *
+  *     <x>_<y>_<z>_v(void) : Returns the constant value for <z> defined for
+  *         field <y> of register <x>.  This value is suitable for direct
+  *         comparison with unshifted values appropriate for use in field <y>
+  *         of register <x>.
+  */
+
+#ifndef HOST1X_HW_HOST1X10_UCLASS_H
+#define HOST1X_HW_HOST1X10_UCLASS_H
+
+static inline u32 host1x_uclass_incr_syncpt_r(void)
+{
+	return 0x0;
+}
+#define HOST1X_UCLASS_INCR_SYNCPT \
+	host1x_uclass_incr_syncpt_r()
+static inline u32 host1x_uclass_incr_syncpt_cond_f(u32 v)
+{
+	return (v & 0xff) << 10;
+}
+#define HOST1X_UCLASS_INCR_SYNCPT_COND_F(v) \
+	host1x_uclass_incr_syncpt_cond_f(v)
+static inline u32 host1x_uclass_incr_syncpt_indx_f(u32 v)
+{
+	return (v & 0x3ff) << 0;
+}
+#define HOST1X_UCLASS_INCR_SYNCPT_INDX_F(v) \
+	host1x_uclass_incr_syncpt_indx_f(v)
+static inline u32 host1x_uclass_wait_syncpt_r(void)
+{
+	return 0x8;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT \
+	host1x_uclass_wait_syncpt_r()
+static inline u32 host1x_uclass_wait_syncpt_indx_f(u32 v)
+{
+	return (v & 0xff) << 24;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_INDX_F(v) \
+	host1x_uclass_wait_syncpt_indx_f(v)
+static inline u32 host1x_uclass_wait_syncpt_thresh_f(u32 v)
+{
+	return (v & 0xffffff) << 0;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_THRESH_F(v) \
+	host1x_uclass_wait_syncpt_thresh_f(v)
+static inline u32 host1x_uclass_wait_syncpt_base_r(void)
+{
+	return 0x9;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_BASE \
+	host1x_uclass_wait_syncpt_base_r()
+static inline u32 host1x_uclass_wait_syncpt_base_indx_f(u32 v)
+{
+	return (v & 0xff) << 24;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_BASE_INDX_F(v) \
+	host1x_uclass_wait_syncpt_base_indx_f(v)
+static inline u32 host1x_uclass_wait_syncpt_base_base_indx_f(u32 v)
+{
+	return (v & 0xff) << 16;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_BASE_BASE_INDX_F(v) \
+	host1x_uclass_wait_syncpt_base_base_indx_f(v)
+static inline u32 host1x_uclass_wait_syncpt_base_offset_f(u32 v)
+{
+	return (v & 0xffff) << 0;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_BASE_OFFSET_F(v) \
+	host1x_uclass_wait_syncpt_base_offset_f(v)
+static inline u32 host1x_uclass_load_syncpt_base_r(void)
+{
+	return 0xb;
+}
+#define HOST1X_UCLASS_LOAD_SYNCPT_BASE \
+	host1x_uclass_load_syncpt_base_r()
+static inline u32 host1x_uclass_load_syncpt_base_base_indx_f(u32 v)
+{
+	return (v & 0xff) << 24;
+}
+#define HOST1X_UCLASS_LOAD_SYNCPT_BASE_BASE_INDX_F(v) \
+	host1x_uclass_load_syncpt_base_base_indx_f(v)
+static inline u32 host1x_uclass_load_syncpt_base_value_f(u32 v)
+{
+	return (v & 0xffffff) << 0;
+}
+#define HOST1X_UCLASS_LOAD_SYNCPT_BASE_VALUE_F(v) \
+	host1x_uclass_load_syncpt_base_value_f(v)
+static inline u32 host1x_uclass_incr_syncpt_base_base_indx_f(u32 v)
+{
+	return (v & 0xff) << 24;
+}
+#define HOST1X_UCLASS_INCR_SYNCPT_BASE_BASE_INDX_F(v) \
+	host1x_uclass_incr_syncpt_base_base_indx_f(v)
+static inline u32 host1x_uclass_incr_syncpt_base_offset_f(u32 v)
+{
+	return (v & 0xffffff) << 0;
+}
+#define HOST1X_UCLASS_INCR_SYNCPT_BASE_OFFSET_F(v) \
+	host1x_uclass_incr_syncpt_base_offset_f(v)
+static inline u32 host1x_uclass_indoff_r(void)
+{
+	return 0x2d;
+}
+#define HOST1X_UCLASS_INDOFF \
+	host1x_uclass_indoff_r()
+static inline u32 host1x_uclass_indoff_indbe_f(u32 v)
+{
+	return (v & 0xf) << 28;
+}
+#define HOST1X_UCLASS_INDOFF_INDBE_F(v) \
+	host1x_uclass_indoff_indbe_f(v)
+static inline u32 host1x_uclass_indoff_autoinc_f(u32 v)
+{
+	return (v & 0x1) << 27;
+}
+#define HOST1X_UCLASS_INDOFF_AUTOINC_F(v) \
+	host1x_uclass_indoff_autoinc_f(v)
+static inline u32 host1x_uclass_indoff_indmodid_f(u32 v)
+{
+	return (v & 0xff) << 18;
+}
+#define HOST1X_UCLASS_INDOFF_INDMODID_F(v) \
+	host1x_uclass_indoff_indmodid_f(v)
+static inline u32 host1x_uclass_indoff_indroffset_f(u32 v)
+{
+	return (v & 0xffff) << 2;
+}
+#define HOST1X_UCLASS_INDOFF_INDROFFSET_F(v) \
+	host1x_uclass_indoff_indroffset_f(v)
+static inline u32 host1x_uclass_indoff_rwn_read_v(void)
+{
+	return 1;
+}
+#define HOST1X_UCLASS_INDOFF_INDROFFSET_F(v) \
+	host1x_uclass_indoff_indroffset_f(v)
+static inline u32 host1x_uclass_load_syncpt_payload_32_r(void)
+{
+	return 0x4e;
+}
+#define HOST1X_UCLASS_LOAD_SYNCPT_PAYLOAD_32 \
+	host1x_uclass_load_syncpt_payload_32_r()
+static inline u32 host1x_uclass_wait_syncpt_32_r(void)
+{
+	return 0x50;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_32 \
+	host1x_uclass_wait_syncpt_32_r()
+
+#endif
diff --git a/drivers/gpu/host1x/hw/hw_host1x10_vm.h b/drivers/gpu/host1x/hw/hw_host1x10_vm.h
new file mode 100644
index 0000000000000..75f5b881c5615
--- /dev/null
+++ b/drivers/gpu/host1x/hw/hw_host1x10_vm.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ */
+
+#define HOST1X_CHANNEL_DMASTART				0x0000
+#define HOST1X_CHANNEL_DMASTART_HI			0x0004
+#define HOST1X_CHANNEL_DMAPUT				0x0008
+#define HOST1X_CHANNEL_DMAPUT_HI			0x000c
+#define HOST1X_CHANNEL_DMAGET				0x0010
+#define HOST1X_CHANNEL_DMAGET_HI			0x0014
+#define HOST1X_CHANNEL_DMAEND				0x0018
+#define HOST1X_CHANNEL_DMAEND_HI			0x001c
+#define HOST1X_CHANNEL_DMACTRL				0x0020
+#define HOST1X_CHANNEL_DMACTRL_DMASTOP			BIT(0)
+#define HOST1X_CHANNEL_DMACTRL_DMAGETRST		BIT(1)
+#define HOST1X_CHANNEL_DMACTRL_DMAINITGET		BIT(2)
+#define HOST1X_CHANNEL_CMDFIFO_STAT			0x0024
+#define HOST1X_CHANNEL_CMDFIFO_STAT_EMPTY		BIT(13)
+#define HOST1X_CHANNEL_CMDFIFO_RDATA			0x0028
+#define HOST1X_CHANNEL_CMDP_OFFSET			0x0030
+#define HOST1X_CHANNEL_CMDP_CLASS			0x0034
+#define HOST1X_CHANNEL_CHANNELSTAT			0x0038
+#define HOST1X_CHANNEL_CMDPROC_STOP			0x0048
+#define HOST1X_CHANNEL_TEARDOWN				0x004c
+#define HOST1X_CHANNEL_SMMU_STREAMID			0x0084
+
+#define HOST1X_SYNC_SYNCPT_CPU_INCR(x)			(0x6400 + 4 * (x))
+#define HOST1X_SYNC_SYNCPT_THRESH_CPU0_INT_STATUS(x)	(0x6600 + 4 * (x))
+#define HOST1X_SYNC_SYNCPT_INTR_DEST(x)			(0x6684 + 4 * (x))
+#define HOST1X_SYNC_SYNCPT_THRESH_INT_ENABLE_CPU0(x)	(0x770c + 4 * (x))
+#define HOST1X_SYNC_SYNCPT_THRESH_INT_DISABLE(x)	(0x7790 + 4 * (x))
+#define HOST1X_SYNC_SYNCPT(x)				(0x8080 + 4 * (x))
+#define HOST1X_SYNC_SYNCPT_INT_THRESH(x)		(0xa088 + 4 * (x))
+#define HOST1X_SYNC_SYNCPT_CH_APP(x)			(0xb090 + 4 * (x))
+#define HOST1X_SYNC_SYNCPT_CH_APP_CH(v)			(((v) & 0x3f) << 8)
diff --git a/drivers/gpu/host1x/intr.c b/drivers/gpu/host1x/intr.c
index 7232972507682..6a383e0c6eb91 100644
--- a/drivers/gpu/host1x/intr.c
+++ b/drivers/gpu/host1x/intr.c
@@ -7,6 +7,7 @@
 
 #include <linux/clk.h>
 #include <linux/interrupt.h>
+#include <linux/overflow.h>
 #include "dev.h"
 #include "fence.h"
 #include "intr.h"
@@ -17,7 +18,7 @@ static void host1x_intr_add_fence_to_list(struct host1x_fence_list *list,
 	struct host1x_syncpt_fence *fence_in_list;
 
 	list_for_each_entry_reverse(fence_in_list, &list->list, list) {
-		if ((s32)(fence_in_list->threshold - fence->threshold) <= 0) {
+		if ((s32)wrapping_sub(u32, fence_in_list->threshold, fence->threshold) <= 0) {
 			/* Fence in list is before us, we can insert here */
 			list_add(&fence->list, &fence_in_list->list);
 			return;
@@ -83,7 +84,7 @@ void host1x_intr_handle_interrupt(struct host1x *host, unsigned int id)
 	spin_lock(&sp->fences.lock);
 
 	list_for_each_entry_safe(fence, tmp, &sp->fences.list, list) {
-		if (((value - fence->threshold) & 0x80000000U) != 0U) {
+		if ((wrapping_sub(u32, value, fence->threshold) & 0x80000000U) != 0U) {
 			/* Fence is not yet expired, we are done */
 			break;
 		}
diff --git a/drivers/gpu/host1x/job.c b/drivers/gpu/host1x/job.c
index 70bda32f1ff48..71411bc344bbc 100644
--- a/drivers/gpu/host1x/job.c
+++ b/drivers/gpu/host1x/job.c
@@ -138,7 +138,7 @@ void host1x_job_add_wait(struct host1x_job *job, u32 id, u32 thresh,
 }
 EXPORT_SYMBOL(host1x_job_add_wait);
 
-static unsigned int pin_job(struct host1x *host, struct host1x_job *job)
+static int pin_job(struct host1x *host, struct host1x_job *job)
 {
 	unsigned long mask = HOST1X_RELOC_READ | HOST1X_RELOC_WRITE;
 	struct host1x_client *client = job->client;
diff --git a/drivers/gpu/host1x/syncpt.c b/drivers/gpu/host1x/syncpt.c
index 807c74fc6a0ac..2c111710f3bfa 100644
--- a/drivers/gpu/host1x/syncpt.c
+++ b/drivers/gpu/host1x/syncpt.c
@@ -8,6 +8,7 @@
 #include <linux/module.h>
 #include <linux/device.h>
 #include <linux/dma-fence.h>
+#include <linux/overflow.h>
 #include <linux/slab.h>
 
 #include <trace/events/host1x.h>
@@ -126,6 +127,10 @@ EXPORT_SYMBOL(host1x_syncpt_id);
  */
 u32 host1x_syncpt_incr_max(struct host1x_syncpt *sp, u32 incrs)
 {
+	/*
+	 * Syncpoint values are intended to be modulo 2^32, so overflow
+	 * here is intended.
+	 */
 	return (u32)atomic_add_return(incrs, &sp->max_val);
 }
 EXPORT_SYMBOL(host1x_syncpt_incr_max);
@@ -279,7 +284,7 @@ bool host1x_syncpt_is_expired(struct host1x_syncpt *sp, u32 thresh)
 
 	current_val = (u32)atomic_read(&sp->min_val);
 
-	return ((current_val - thresh) & 0x80000000U) == 0U;
+	return (wrapping_sub(u32, current_val, thresh) & 0x80000000U) == 0U;
 }
 
 int host1x_syncpt_init(struct host1x *host)
diff --git a/drivers/gpu/host1x/syncpt.h b/drivers/gpu/host1x/syncpt.h
index 4c3f3b2f0e9c9..9eff42efc4459 100644
--- a/drivers/gpu/host1x/syncpt.h
+++ b/drivers/gpu/host1x/syncpt.h
@@ -12,6 +12,7 @@
 #include <linux/host1x.h>
 #include <linux/kernel.h>
 #include <linux/kref.h>
+#include <linux/overflow.h>
 #include <linux/sched.h>
 
 #include "fence.h"
@@ -77,7 +78,7 @@ static inline bool host1x_syncpt_check_max(struct host1x_syncpt *sp, u32 real)
 	if (sp->client_managed)
 		return true;
 	max = host1x_syncpt_read_max(sp);
-	return (s32)(max - real) >= 0;
+	return (s32)wrapping_sub(u32, max, real) >= 0;
 }
 
 /* Return true if sync point is client managed. */
diff --git a/drivers/gpu/nova-core/.gitignore b/drivers/gpu/nova-core/.gitignore
new file mode 100644
index 0000000000000..7cc8318c76b1b
--- /dev/null
+++ b/drivers/gpu/nova-core/.gitignore
@@ -0,0 +1 @@
+exports_nova_core_generated.h
diff --git a/drivers/gpu/nova-core/Makefile b/drivers/gpu/nova-core/Makefile
index 4ae544f808f42..216329760a5b9 100644
--- a/drivers/gpu/nova-core/Makefile
+++ b/drivers/gpu/nova-core/Makefile
@@ -1,4 +1,3 @@
 # SPDX-License-Identifier: GPL-2.0
-
-obj-$(CONFIG_NOVA_CORE) += nova-core.o
-nova-core-y := nova_core.o
+# nova-core is built from drivers/gpu/Makefile.
+# nova_core.o (rust-analyzer marker - DO NOT REMOVE).
diff --git a/drivers/gpu/nova-core/bitfield.rs b/drivers/gpu/nova-core/bitfield.rs
deleted file mode 100644
index 660c3911402de..0000000000000
--- a/drivers/gpu/nova-core/bitfield.rs
+++ /dev/null
@@ -1,329 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-//! Bitfield library for Rust structures
-//!
-//! Support for defining bitfields in Rust structures. Also used by the [`register!`] macro.
-
-/// Defines a struct with accessors to access bits within an inner unsigned integer.
-///
-/// # Syntax
-///
-/// ```rust
-/// use nova_core::bitfield;
-///
-/// #[derive(Debug, Clone, Copy, Default)]
-/// enum Mode {
-///     #[default]
-///     Low = 0,
-///     High = 1,
-///     Auto = 2,
-/// }
-///
-/// impl TryFrom<u8> for Mode {
-///     type Error = u8;
-///     fn try_from(value: u8) -> Result<Self, Self::Error> {
-///         match value {
-///             0 => Ok(Mode::Low),
-///             1 => Ok(Mode::High),
-///             2 => Ok(Mode::Auto),
-///             _ => Err(value),
-///         }
-///     }
-/// }
-///
-/// impl From<Mode> for u8 {
-///     fn from(mode: Mode) -> u8 {
-///         mode as u8
-///     }
-/// }
-///
-/// #[derive(Debug, Clone, Copy, Default)]
-/// enum State {
-///     #[default]
-///     Inactive = 0,
-///     Active = 1,
-/// }
-///
-/// impl From<bool> for State {
-///     fn from(value: bool) -> Self {
-///         if value { State::Active } else { State::Inactive }
-///     }
-/// }
-///
-/// impl From<State> for bool {
-///     fn from(state: State) -> bool {
-///         match state {
-///             State::Inactive => false,
-///             State::Active => true,
-///         }
-///     }
-/// }
-///
-/// bitfield! {
-///     pub struct ControlReg(u32) {
-///         7:7 state as bool => State;
-///         3:0 mode as u8 ?=> Mode;
-///     }
-/// }
-/// ```
-///
-/// This generates a struct with:
-/// - Field accessors: `mode()`, `state()`, etc.
-/// - Field setters: `set_mode()`, `set_state()`, etc. (supports chaining with builder pattern).
-///   Note that the compiler will error out if the size of the setter's arg exceeds the
-///   struct's storage size.
-/// - Debug and Default implementations.
-///
-/// Note: Field accessors and setters inherit the same visibility as the struct itself.
-/// In the example above, both `mode()` and `set_mode()` methods will be `pub`.
-///
-/// Fields are defined as follows:
-///
-/// - `as <type>` simply returns the field value casted to <type>, typically `u32`, `u16`, `u8` or
-///   `bool`. Note that `bool` fields must have a range of 1 bit.
-/// - `as <type> => <into_type>` calls `<into_type>`'s `From::<<type>>` implementation and returns
-///   the result.
-/// - `as <type> ?=> <try_into_type>` calls `<try_into_type>`'s `TryFrom::<<type>>` implementation
-///   and returns the result. This is useful with fields for which not all values are valid.
-macro_rules! bitfield {
-    // Main entry point - defines the bitfield struct with fields
-    ($vis:vis struct $name:ident($storage:ty) $(, $comment:literal)? { $($fields:tt)* }) => {
-        bitfield!(@core $vis $name $storage $(, $comment)? { $($fields)* });
-    };
-
-    // All rules below are helpers.
-
-    // Defines the wrapper `$name` type, as well as its relevant implementations (`Debug`,
-    // `Default`, and conversion to the value type) and field accessor methods.
-    (@core $vis:vis $name:ident $storage:ty $(, $comment:literal)? { $($fields:tt)* }) => {
-        $(
-        #[doc=$comment]
-        )?
-        #[repr(transparent)]
-        #[derive(Clone, Copy)]
-        $vis struct $name($storage);
-
-        impl ::core::convert::From<$name> for $storage {
-            fn from(val: $name) -> $storage {
-                val.0
-            }
-        }
-
-        bitfield!(@fields_dispatcher $vis $name $storage { $($fields)* });
-    };
-
-    // Captures the fields and passes them to all the implementers that require field information.
-    //
-    // Used to simplify the matching rules for implementers, so they don't need to match the entire
-    // complex fields rule even though they only make use of part of it.
-    (@fields_dispatcher $vis:vis $name:ident $storage:ty {
-        $($hi:tt:$lo:tt $field:ident as $type:tt
-            $(?=> $try_into_type:ty)?
-            $(=> $into_type:ty)?
-            $(, $comment:literal)?
-        ;
-        )*
-    }
-    ) => {
-        bitfield!(@field_accessors $vis $name $storage {
-            $(
-                $hi:$lo $field as $type
-                $(?=> $try_into_type)?
-                $(=> $into_type)?
-                $(, $comment)?
-            ;
-            )*
-        });
-        bitfield!(@debug $name { $($field;)* });
-        bitfield!(@default $name { $($field;)* });
-    };
-
-    // Defines all the field getter/setter methods for `$name`.
-    (
-        @field_accessors $vis:vis $name:ident $storage:ty {
-        $($hi:tt:$lo:tt $field:ident as $type:tt
-            $(?=> $try_into_type:ty)?
-            $(=> $into_type:ty)?
-            $(, $comment:literal)?
-        ;
-        )*
-        }
-    ) => {
-        $(
-            bitfield!(@check_field_bounds $hi:$lo $field as $type);
-        )*
-
-        #[allow(dead_code)]
-        impl $name {
-            $(
-            bitfield!(@field_accessor $vis $name $storage, $hi:$lo $field as $type
-                $(?=> $try_into_type)?
-                $(=> $into_type)?
-                $(, $comment)?
-                ;
-            );
-            )*
-        }
-    };
-
-    // Boolean fields must have `$hi == $lo`.
-    (@check_field_bounds $hi:tt:$lo:tt $field:ident as bool) => {
-        #[allow(clippy::eq_op)]
-        const _: () = {
-            ::kernel::build_assert::build_assert!(
-                $hi == $lo,
-                concat!("boolean field `", stringify!($field), "` covers more than one bit")
-            );
-        };
-    };
-
-    // Non-boolean fields must have `$hi >= $lo`.
-    (@check_field_bounds $hi:tt:$lo:tt $field:ident as $type:tt) => {
-        #[allow(clippy::eq_op)]
-        const _: () = {
-            ::kernel::build_assert::build_assert!(
-                $hi >= $lo,
-                concat!("field `", stringify!($field), "`'s MSB is smaller than its LSB")
-            );
-        };
-    };
-
-    // Catches fields defined as `bool` and convert them into a boolean value.
-    (
-        @field_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident as bool
-            => $into_type:ty $(, $comment:literal)?;
-    ) => {
-        bitfield!(
-            @leaf_accessor $vis $name $storage, $hi:$lo $field
-            { |f| <$into_type>::from(f != 0) }
-            bool $into_type => $into_type $(, $comment)?;
-        );
-    };
-
-    // Shortcut for fields defined as `bool` without the `=>` syntax.
-    (
-        @field_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident as bool
-            $(, $comment:literal)?;
-    ) => {
-        bitfield!(
-            @field_accessor $vis $name $storage, $hi:$lo $field as bool => bool $(, $comment)?;
-        );
-    };
-
-    // Catches the `?=>` syntax for non-boolean fields.
-    (
-        @field_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident as $type:tt
-            ?=> $try_into_type:ty $(, $comment:literal)?;
-    ) => {
-        bitfield!(@leaf_accessor $vis $name $storage, $hi:$lo $field
-            { |f| <$try_into_type>::try_from(f as $type) } $type $try_into_type =>
-            ::core::result::Result<
-                $try_into_type,
-                <$try_into_type as ::core::convert::TryFrom<$type>>::Error
-            >
-            $(, $comment)?;);
-    };
-
-    // Catches the `=>` syntax for non-boolean fields.
-    (
-        @field_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident as $type:tt
-            => $into_type:ty $(, $comment:literal)?;
-    ) => {
-        bitfield!(@leaf_accessor $vis $name $storage, $hi:$lo $field
-            { |f| <$into_type>::from(f as $type) } $type $into_type => $into_type $(, $comment)?;);
-    };
-
-    // Shortcut for non-boolean fields defined without the `=>` or `?=>` syntax.
-    (
-        @field_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident as $type:tt
-            $(, $comment:literal)?;
-    ) => {
-        bitfield!(
-            @field_accessor $vis $name $storage, $hi:$lo $field as $type => $type $(, $comment)?;
-        );
-    };
-
-    // Generates the accessor methods for a single field.
-    (
-        @leaf_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident
-            { $process:expr } $prim_type:tt $to_type:ty => $res_type:ty $(, $comment:literal)?;
-    ) => {
-        ::kernel::macros::paste!(
-        const [<$field:upper _RANGE>]: ::core::ops::RangeInclusive<u8> = $lo..=$hi;
-        const [<$field:upper _MASK>]: $storage = {
-            // Generate mask for shifting
-            match ::core::mem::size_of::<$storage>() {
-                1 => ::kernel::bits::genmask_u8($lo..=$hi) as $storage,
-                2 => ::kernel::bits::genmask_u16($lo..=$hi) as $storage,
-                4 => ::kernel::bits::genmask_u32($lo..=$hi) as $storage,
-                8 => ::kernel::bits::genmask_u64($lo..=$hi) as $storage,
-                _ => ::kernel::build_error!("Unsupported storage type size")
-            }
-        };
-        const [<$field:upper _SHIFT>]: u32 = $lo;
-        );
-
-        $(
-        #[doc="Returns the value of this field:"]
-        #[doc=$comment]
-        )?
-        #[inline(always)]
-        $vis fn $field(self) -> $res_type {
-            ::kernel::macros::paste!(
-            const MASK: $storage = $name::[<$field:upper _MASK>];
-            const SHIFT: u32 = $name::[<$field:upper _SHIFT>];
-            );
-            let field = ((self.0 & MASK) >> SHIFT);
-
-            $process(field)
-        }
-
-        ::kernel::macros::paste!(
-        $(
-        #[doc="Sets the value of this field:"]
-        #[doc=$comment]
-        )?
-        #[inline(always)]
-        $vis fn [<set_ $field>](mut self, value: $to_type) -> Self {
-            const MASK: $storage = $name::[<$field:upper _MASK>];
-            const SHIFT: u32 = $name::[<$field:upper _SHIFT>];
-            let value = ($storage::from($prim_type::from(value)) << SHIFT) & MASK;
-            self.0 = (self.0 & !MASK) | value;
-
-            self
-        }
-        );
-    };
-
-    // Generates the `Debug` implementation for `$name`.
-    (@debug $name:ident { $($field:ident;)* }) => {
-        impl ::kernel::fmt::Debug for $name {
-            fn fmt(&self, f: &mut ::kernel::fmt::Formatter<'_>) -> ::kernel::fmt::Result {
-                f.debug_struct(stringify!($name))
-                    .field("<raw>", &::kernel::prelude::fmt!("{:#x}", &self.0))
-                $(
-                    .field(stringify!($field), &self.$field())
-                )*
-                    .finish()
-            }
-        }
-    };
-
-    // Generates the `Default` implementation for `$name`.
-    (@default $name:ident { $($field:ident;)* }) => {
-        /// Returns a value for the bitfield where all fields are set to their default value.
-        impl ::core::default::Default for $name {
-            fn default() -> Self {
-                let value = Self(Default::default());
-
-                ::kernel::macros::paste!(
-                $(
-                let value = value.[<set_ $field>](Default::default());
-                )*
-                );
-
-                value
-            }
-        }
-    };
-}
diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs
index 5738d4ac521b7..48380ac15f68b 100644
--- a/drivers/gpu/nova-core/driver.rs
+++ b/drivers/gpu/nova-core/driver.rs
@@ -15,7 +15,7 @@
         Atomic,
         Relaxed, //
     },
-    types::ForLt,
+    types::CovariantForLt,
 };
 
 use crate::gpu::Gpu;
@@ -29,7 +29,7 @@ pub(crate) struct NovaCore<'bound> {
     pub(crate) gpu: Gpu<'bound>,
     bar: pci::Bar<'bound, BAR0_SIZE>,
     #[allow(clippy::type_complexity)]
-    _reg: auxiliary::Registration<'bound, ForLt!(())>,
+    _reg: auxiliary::Registration<'bound, CovariantForLt!(())>,
 }
 
 pub(crate) struct NovaCoreDriver;
diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs
index 94c7696a64935..78948cc8bff31 100644
--- a/drivers/gpu/nova-core/falcon.rs
+++ b/drivers/gpu/nova-core/falcon.rs
@@ -5,10 +5,7 @@
 use hal::FalconHal;
 
 use kernel::{
-    device::{
-        self,
-        Device, //
-    },
+    device,
     dma::{
         Coherent,
         CoherentBox,
@@ -24,7 +21,6 @@
         Io,
     },
     prelude::*,
-    sync::aref::ARef,
     time::Delta,
 };
 
@@ -358,41 +354,47 @@ pub(crate) trait FalconFirmware {
 }
 
 /// Contains the base parameters common to all Falcon instances.
-pub(crate) struct Falcon<E: FalconEngine> {
+pub(crate) struct Falcon<'a, E: FalconEngine> {
     hal: KBox<dyn FalconHal<E>>,
-    dev: ARef<device::Device>,
+    dev: &'a device::Device<device::Bound>,
+    bar: Bar0<'a>,
 }
 
-impl<E: FalconEngine + 'static> Falcon<E> {
+impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
     /// Create a new falcon instance.
-    pub(crate) fn new(dev: &device::Device, chipset: Chipset) -> Result<Self> {
+    pub(crate) fn new(
+        dev: &'a device::Device<device::Bound>,
+        chipset: Chipset,
+        bar: Bar0<'a>,
+    ) -> Result<Self> {
         Ok(Self {
             hal: hal::falcon_hal(chipset)?,
-            dev: dev.into(),
+            dev,
+            bar,
         })
     }
 
     /// Resets DMA-related registers.
-    pub(crate) fn dma_reset(&self, bar: Bar0<'_>) {
-        bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| {
+    pub(crate) fn dma_reset(&self) {
+        self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| {
             v.with_allow_phys_no_ctx(true)
         });
 
-        bar.write(
+        self.bar.write(
             WithBase::of::<E>(),
             regs::NV_PFALCON_FALCON_DMACTL::zeroed(),
         );
     }
 
     /// Reset the controller, select the falcon core, and wait for memory scrubbing to complete.
-    pub(crate) fn reset(&self, bar: Bar0<'_>) -> Result {
-        self.hal.reset_eng(bar)?;
-        self.hal.select_core(self, bar)?;
-        self.hal.reset_wait_mem_scrubbing(bar)?;
+    pub(crate) fn reset(&self) -> Result {
+        self.hal.reset_eng(self)?;
+        self.hal.select_core(self)?;
+        self.hal.reset_wait_mem_scrubbing(self)?;
 
-        bar.write(
+        self.bar.write(
             WithBase::of::<E>(),
-            regs::NV_PFALCON_FALCON_RM::from(bar.read(regs::NV_PMC_BOOT_0).into_raw()),
+            regs::NV_PFALCON_FALCON_RM::from(self.bar.read(regs::NV_PMC_BOOT_0).into_raw()),
         );
 
         Ok(())
@@ -404,18 +406,14 @@ pub(crate) fn reset(&self, bar: Bar0<'_>) -> Result {
     /// Write a slice to Falcon IMEM memory using programmed I/O (PIO).
     ///
     /// Returns `EINVAL` if `img.len()` is not a multiple of 4.
-    fn pio_wr_imem_slice(
-        &self,
-        bar: Bar0<'_>,
-        load_offsets: FalconPioImemLoadTarget<'_>,
-    ) -> Result {
+    fn pio_wr_imem_slice(&self, load_offsets: FalconPioImemLoadTarget<'_>) -> Result {
         // Rejecting misaligned images here allows us to avoid checking
         // inside the loops.
         if load_offsets.data.len() % 4 != 0 {
             return Err(EINVAL);
         }
 
-        bar.write(
+        self.bar.write(
             WithBase::of::<E>().at(Self::PIO_PORT),
             regs::NV_PFALCON_FALCON_IMEMC::zeroed()
                 .with_secure(load_offsets.secure)
@@ -426,13 +424,13 @@ fn pio_wr_imem_slice(
         for (n, block) in load_offsets.data.chunks(MEM_BLOCK_ALIGNMENT).enumerate() {
             let n = u16::try_from(n)?;
             let tag: u16 = load_offsets.start_tag.checked_add(n).ok_or(ERANGE)?;
-            bar.write(
+            self.bar.write(
                 WithBase::of::<E>().at(Self::PIO_PORT),
                 regs::NV_PFALCON_FALCON_IMEMT::zeroed().with_tag(tag),
             );
             for word in block.chunks_exact(4) {
                 let w = [word[0], word[1], word[2], word[3]];
-                bar.write(
+                self.bar.write(
                     WithBase::of::<E>().at(Self::PIO_PORT),
                     regs::NV_PFALCON_FALCON_IMEMD::zeroed().with_data(u32::from_le_bytes(w)),
                 );
@@ -445,18 +443,14 @@ fn pio_wr_imem_slice(
     /// Write a slice to Falcon DMEM memory using programmed I/O (PIO).
     ///
     /// Returns `EINVAL` if `img.len()` is not a multiple of 4.
-    fn pio_wr_dmem_slice(
-        &self,
-        bar: Bar0<'_>,
-        load_offsets: FalconPioDmemLoadTarget<'_>,
-    ) -> Result {
+    fn pio_wr_dmem_slice(&self, load_offsets: FalconPioDmemLoadTarget<'_>) -> Result {
         // Rejecting misaligned images here allows us to avoid checking
         // inside the loops.
         if load_offsets.data.len() % 4 != 0 {
             return Err(EINVAL);
         }
 
-        bar.write(
+        self.bar.write(
             WithBase::of::<E>().at(Self::PIO_PORT),
             regs::NV_PFALCON_FALCON_DMEMC::zeroed()
                 .with_aincw(true)
@@ -465,7 +459,7 @@ fn pio_wr_dmem_slice(
 
         for word in load_offsets.data.chunks_exact(4) {
             let w = [word[0], word[1], word[2], word[3]];
-            bar.write(
+            self.bar.write(
                 WithBase::of::<E>().at(Self::PIO_PORT),
                 regs::NV_PFALCON_FALCON_DMEMD::zeroed().with_data(u32::from_le_bytes(w)),
             );
@@ -477,29 +471,28 @@ fn pio_wr_dmem_slice(
     /// Perform a PIO copy into `IMEM` and `DMEM` of `fw`, and prepare the falcon to run it.
     pub(crate) fn pio_load<F: FalconFirmware<Target = E> + FalconPioLoadable>(
         &self,
-        bar: Bar0<'_>,
         fw: &F,
     ) -> Result {
-        bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| {
+        self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| {
             v.with_allow_phys_no_ctx(true)
         });
 
-        bar.write(
+        self.bar.write(
             WithBase::of::<E>(),
             regs::NV_PFALCON_FALCON_DMACTL::zeroed(),
         );
 
         if let Some(imem_ns) = fw.imem_ns_load_params() {
-            self.pio_wr_imem_slice(bar, imem_ns)?;
+            self.pio_wr_imem_slice(imem_ns)?;
         }
         if let Some(imem_sec) = fw.imem_sec_load_params() {
-            self.pio_wr_imem_slice(bar, imem_sec)?;
+            self.pio_wr_imem_slice(imem_sec)?;
         }
-        self.pio_wr_dmem_slice(bar, fw.dmem_load_params())?;
+        self.pio_wr_dmem_slice(fw.dmem_load_params())?;
 
-        self.hal.program_brom(self, bar, &fw.brom_params());
+        self.hal.program_brom(self, &fw.brom_params());
 
-        bar.write(
+        self.bar.write(
             WithBase::of::<E>(),
             regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()),
         );
@@ -513,7 +506,6 @@ pub(crate) fn pio_load<F: FalconFirmware<Target = E> + FalconPioLoadable>(
     /// `sec` is set if the loaded firmware is expected to run in secure mode.
     fn dma_wr(
         &self,
-        bar: Bar0<'_>,
         dma_obj: &Coherent<[u8]>,
         target_mem: FalconMem,
         load_offsets: FalconDmaLoadTarget,
@@ -571,7 +563,7 @@ fn dma_wr(
 
         // Set up the base source DMA address.
 
-        bar.write(
+        self.bar.write(
             WithBase::of::<E>(),
             regs::NV_PFALCON_FALCON_DMATRFBASE::zeroed().with_base(
                 // CAST: `as u32` is used on purpose since we do want to strip the upper bits,
@@ -579,7 +571,7 @@ fn dma_wr(
                 (dma_start >> 8) as u32,
             ),
         );
-        bar.write(
+        self.bar.write(
             WithBase::of::<E>(),
             regs::NV_PFALCON_FALCON_DMATRFBASE1::zeroed().try_with_base(dma_start >> 40)?,
         );
@@ -590,23 +582,23 @@ fn dma_wr(
 
         for pos in (0..num_transfers).map(|i| i * DMA_LEN) {
             // Perform a transfer of size `DMA_LEN`.
-            bar.write(
+            self.bar.write(
                 WithBase::of::<E>(),
                 regs::NV_PFALCON_FALCON_DMATRFMOFFS::zeroed()
                     .try_with_offs(load_offsets.dst_start + pos)?,
             );
-            bar.write(
+            self.bar.write(
                 WithBase::of::<E>(),
                 regs::NV_PFALCON_FALCON_DMATRFFBOFFS::zeroed().with_offs(src_start + pos),
             );
 
-            bar.write(WithBase::of::<E>(), cmd);
+            self.bar.write(WithBase::of::<E>(), cmd);
 
             // Wait for the transfer to complete.
             // TIMEOUT: arbitrarily large value, no DMA transfer to the falcon's small memories
             // should ever take that long.
             read_poll_timeout(
-                || Ok(bar.read(regs::NV_PFALCON_FALCON_DMATRFCMD::of::<E>())),
+                || Ok(self.bar.read(regs::NV_PFALCON_FALCON_DMATRFCMD::of::<E>())),
                 |r| r.idle(),
                 Delta::ZERO,
                 Delta::from_secs(2),
@@ -617,12 +609,7 @@ fn dma_wr(
     }
 
     /// Perform a DMA load into `IMEM` and `DMEM` of `fw`, and prepare the falcon to run it.
-    fn dma_load<F: FalconFirmware<Target = E> + FalconDmaLoadable>(
-        &self,
-        dev: &Device<device::Bound>,
-        bar: Bar0<'_>,
-        fw: &F,
-    ) -> Result {
+    fn dma_load<F: FalconFirmware<Target = E> + FalconDmaLoadable>(&self, fw: &F) -> Result {
         // DMA object with firmware content as the source of the DMA engine.
         let dma_obj = {
             let fw_slice = fw.as_slice();
@@ -630,7 +617,7 @@ fn dma_load<F: FalconFirmware<Target = E> + FalconDmaLoadable>(
             // DMA copies are done in chunks of `MEM_BLOCK_ALIGNMENT`, so pad the length
             // accordingly and fill with `0`.
             let mut dma_obj = CoherentBox::zeroed_slice(
-                dev,
+                self.dev,
                 fw_slice.len().next_multiple_of(MEM_BLOCK_ALIGNMENT),
                 GFP_KERNEL,
             )?;
@@ -642,24 +629,20 @@ fn dma_load<F: FalconFirmware<Target = E> + FalconDmaLoadable>(
             dma_obj.into()
         };
 
-        self.dma_reset(bar);
-        bar.update(regs::NV_PFALCON_FBIF_TRANSCFG::of::<E>().at(0), |v| {
-            v.with_target(FalconFbifTarget::CoherentSysmem)
-                .with_mem_type(FalconFbifMemType::Physical)
-        });
+        self.dma_reset();
+        self.bar
+            .update(regs::NV_PFALCON_FBIF_TRANSCFG::of::<E>().at(0), |v| {
+                v.with_target(FalconFbifTarget::CoherentSysmem)
+                    .with_mem_type(FalconFbifMemType::Physical)
+            });
 
-        self.dma_wr(
-            bar,
-            &dma_obj,
-            FalconMem::ImemSecure,
-            fw.imem_sec_load_params(),
-        )?;
-        self.dma_wr(bar, &dma_obj, FalconMem::Dmem, fw.dmem_load_params())?;
+        self.dma_wr(&dma_obj, FalconMem::ImemSecure, fw.imem_sec_load_params())?;
+        self.dma_wr(&dma_obj, FalconMem::Dmem, fw.dmem_load_params())?;
 
-        self.hal.program_brom(self, bar, &fw.brom_params());
+        self.hal.program_brom(self, &fw.brom_params());
 
         // Set `BootVec` to start of non-secure code.
-        bar.write(
+        self.bar.write(
             WithBase::of::<E>(),
             regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()),
         );
@@ -668,10 +651,10 @@ fn dma_load<F: FalconFirmware<Target = E> + FalconDmaLoadable>(
     }
 
     /// Wait until the falcon CPU is halted.
-    pub(crate) fn wait_till_halted(&self, bar: Bar0<'_>) -> Result<()> {
+    pub(crate) fn wait_till_halted(&self) -> Result<()> {
         // TIMEOUT: arbitrarily large value, firmwares should complete in less than 2 seconds.
         read_poll_timeout(
-            || Ok(bar.read(regs::NV_PFALCON_FALCON_CPUCTL::of::<E>())),
+            || Ok(self.bar.read(regs::NV_PFALCON_FALCON_CPUCTL::of::<E>())),
             |r| r.halted(),
             Delta::ZERO,
             Delta::from_secs(2),
@@ -681,16 +664,17 @@ pub(crate) fn wait_till_halted(&self, bar: Bar0<'_>) -> Result<()> {
     }
 
     /// Start the falcon CPU.
-    pub(crate) fn start(&self, bar: Bar0<'_>) -> Result<()> {
-        match bar
+    pub(crate) fn start(&self) -> Result<()> {
+        match self
+            .bar
             .read(regs::NV_PFALCON_FALCON_CPUCTL::of::<E>())
             .alias_en()
         {
-            true => bar.write(
+            true => self.bar.write(
                 WithBase::of::<E>(),
                 regs::NV_PFALCON_FALCON_CPUCTL_ALIAS::zeroed().with_startcpu(true),
             ),
-            false => bar.write(
+            false => self.bar.write(
                 WithBase::of::<E>(),
                 regs::NV_PFALCON_FALCON_CPUCTL::zeroed().with_startcpu(true),
             ),
@@ -700,16 +684,16 @@ pub(crate) fn start(&self, bar: Bar0<'_>) -> Result<()> {
     }
 
     /// Writes values to the mailbox registers if provided.
-    pub(crate) fn write_mailboxes(&self, bar: Bar0<'_>, mbox0: Option<u32>, mbox1: Option<u32>) {
+    pub(crate) fn write_mailboxes(&self, mbox0: Option<u32>, mbox1: Option<u32>) {
         if let Some(mbox0) = mbox0 {
-            bar.write(
+            self.bar.write(
                 WithBase::of::<E>(),
                 regs::NV_PFALCON_FALCON_MAILBOX0::zeroed().with_value(mbox0),
             );
         }
 
         if let Some(mbox1) = mbox1 {
-            bar.write(
+            self.bar.write(
                 WithBase::of::<E>(),
                 regs::NV_PFALCON_FALCON_MAILBOX1::zeroed().with_value(mbox1),
             );
@@ -717,21 +701,23 @@ pub(crate) fn write_mailboxes(&self, bar: Bar0<'_>, mbox0: Option<u32>, mbox1: O
     }
 
     /// Reads the value from `mbox0` register.
-    pub(crate) fn read_mailbox0(&self, bar: Bar0<'_>) -> u32 {
-        bar.read(regs::NV_PFALCON_FALCON_MAILBOX0::of::<E>())
+    pub(crate) fn read_mailbox0(&self) -> u32 {
+        self.bar
+            .read(regs::NV_PFALCON_FALCON_MAILBOX0::of::<E>())
             .value()
     }
 
     /// Reads the value from `mbox1` register.
-    pub(crate) fn read_mailbox1(&self, bar: Bar0<'_>) -> u32 {
-        bar.read(regs::NV_PFALCON_FALCON_MAILBOX1::of::<E>())
+    pub(crate) fn read_mailbox1(&self) -> u32 {
+        self.bar
+            .read(regs::NV_PFALCON_FALCON_MAILBOX1::of::<E>())
             .value()
     }
 
     /// Reads values from both mailbox registers.
-    pub(crate) fn read_mailboxes(&self, bar: Bar0<'_>) -> (u32, u32) {
-        let mbox0 = self.read_mailbox0(bar);
-        let mbox1 = self.read_mailbox1(bar);
+    pub(crate) fn read_mailboxes(&self) -> (u32, u32) {
+        let mbox0 = self.read_mailbox0();
+        let mbox1 = self.read_mailbox1();
 
         (mbox0, mbox1)
     }
@@ -743,54 +729,43 @@ pub(crate) fn read_mailboxes(&self, bar: Bar0<'_>) -> (u32, u32) {
     ///
     /// Wait up to two seconds for the firmware to complete, and return its exit status read from
     /// the `MBOX0` and `MBOX1` registers.
-    pub(crate) fn boot(
-        &self,
-        bar: Bar0<'_>,
-        mbox0: Option<u32>,
-        mbox1: Option<u32>,
-    ) -> Result<(u32, u32)> {
-        self.write_mailboxes(bar, mbox0, mbox1);
-        self.start(bar)?;
-        self.wait_till_halted(bar)?;
-        Ok(self.read_mailboxes(bar))
+    pub(crate) fn boot(&self, mbox0: Option<u32>, mbox1: Option<u32>) -> Result<(u32, u32)> {
+        self.write_mailboxes(mbox0, mbox1);
+        self.start()?;
+        self.wait_till_halted()?;
+        Ok(self.read_mailboxes())
     }
 
     /// Returns the fused version of the signature to use in order to run a HS firmware on this
     /// falcon instance. `engine_id_mask` and `ucode_id` are obtained from the firmware header.
     pub(crate) fn signature_reg_fuse_version(
         &self,
-        bar: Bar0<'_>,
         engine_id_mask: u16,
         ucode_id: u8,
     ) -> Result<u32> {
         self.hal
-            .signature_reg_fuse_version(self, bar, engine_id_mask, ucode_id)
+            .signature_reg_fuse_version(self, engine_id_mask, ucode_id)
     }
 
     /// Check if the RISC-V core is active.
     ///
     /// Returns `true` if the RISC-V core is active, `false` otherwise.
-    pub(crate) fn is_riscv_active(&self, bar: Bar0<'_>) -> bool {
-        self.hal.is_riscv_active(bar)
+    pub(crate) fn is_riscv_active(&self) -> bool {
+        self.hal.is_riscv_active(self)
     }
 
     /// Load a firmware image into Falcon memory, using the preferred method for the current
     /// chipset.
-    pub(crate) fn load<F: FalconFirmware<Target = E> + FalconDmaLoadable>(
-        &self,
-        dev: &Device<device::Bound>,
-        bar: Bar0<'_>,
-        fw: &F,
-    ) -> Result {
+    pub(crate) fn load<F: FalconFirmware<Target = E> + FalconDmaLoadable>(&self, fw: &F) -> Result {
         match self.hal.load_method() {
-            LoadMethod::Dma => self.dma_load(dev, bar, fw),
-            LoadMethod::Pio => self.pio_load(bar, &fw.try_as_pio_loadable()?),
+            LoadMethod::Dma => self.dma_load(fw),
+            LoadMethod::Pio => self.pio_load(&fw.try_as_pio_loadable()?),
         }
     }
 
     /// Write the application version to the OS register.
-    pub(crate) fn write_os_version(&self, bar: Bar0<'_>, app_version: u32) {
-        bar.write(
+    pub(crate) fn write_os_version(&self, app_version: u32) {
+        self.bar.write(
             WithBase::of::<E>(),
             regs::NV_PFALCON_FALCON_OS::zeroed().with_value(app_version),
         );
diff --git a/drivers/gpu/nova-core/falcon/fsp.rs b/drivers/gpu/nova-core/falcon/fsp.rs
index 52cdb84ef0e80..53b1079843ae4 100644
--- a/drivers/gpu/nova-core/falcon/fsp.rs
+++ b/drivers/gpu/nova-core/falcon/fsp.rs
@@ -21,7 +21,6 @@
 };
 
 use crate::{
-    driver::Bar0,
     falcon::{
         Falcon,
         FalconEngine,
@@ -48,18 +47,18 @@ impl RegisterBase<PFalcon2Base> for Fsp {
 
 impl FalconEngine for Fsp {}
 
-impl Falcon<Fsp> {
+impl<'a> Falcon<'a, Fsp> {
     /// Writes `data` to FSP external memory at offset `0`.
     ///
     /// `data` is interpreted as little-endian 32-bit words. Returns `EINVAL`
     /// if the `data` length is not 4-byte aligned.
-    fn write_emem(&mut self, bar: Bar0<'_>, data: &[u8]) -> Result {
+    fn write_emem(&mut self, data: &[u8]) -> Result {
         if data.len() % 4 != 0 {
             return Err(EINVAL);
         }
 
         // Begin a write burst at offset `0`, auto-incrementing on each write.
-        bar.write(
+        self.bar.write(
             WithBase::of::<Fsp>(),
             regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincw(true),
         );
@@ -68,7 +67,7 @@ fn write_emem(&mut self, bar: Bar0<'_>, data: &[u8]) -> Result {
             let value = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]);
 
             // Write the next 32-bit `value`; hardware advances the offset.
-            bar.write(
+            self.bar.write(
                 WithBase::of::<Fsp>(),
                 regs::NV_PFALCON_FALCON_EMEMD::zeroed().with_data(value),
             );
@@ -81,20 +80,23 @@ fn write_emem(&mut self, bar: Bar0<'_>, data: &[u8]) -> Result {
     ///
     /// `data` is stored as little-endian 32-bit words. Returns `EINVAL` if
     /// the `data` length is not 4-byte aligned.
-    fn read_emem(&mut self, bar: Bar0<'_>, data: &mut [u8]) -> Result {
+    fn read_emem(&mut self, data: &mut [u8]) -> Result {
         if data.len() % 4 != 0 {
             return Err(EINVAL);
         }
 
         // Begin a read burst at offset `0`, auto-incrementing on each read.
-        bar.write(
+        self.bar.write(
             WithBase::of::<Fsp>(),
             regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincr(true),
         );
 
         for chunk in data.chunks_exact_mut(4) {
             // Read the next 32-bit word; hardware advances the offset.
-            let value = bar.read(regs::NV_PFALCON_FALCON_EMEMD::of::<Fsp>()).data();
+            let value = self
+                .bar
+                .read(regs::NV_PFALCON_FALCON_EMEMD::of::<Fsp>())
+                .data();
             chunk.copy_from_slice(&value.to_le_bytes());
         }
 
@@ -107,9 +109,9 @@ fn read_emem(&mut self, bar: Bar0<'_>, data: &mut [u8]) -> Result {
     ///
     /// The FSP message queue is not circular. Pointers are reset to 0 after each
     /// message exchange, so `tail >= head` is always true when data is present.
-    fn poll_msgq(&self, bar: Bar0<'_>) -> u32 {
-        let head = bar.read(regs::NV_PFSP_MSGQ_HEAD::at(0)).val();
-        let tail = bar.read(regs::NV_PFSP_MSGQ_TAIL::at(0)).val();
+    fn poll_msgq(&self) -> u32 {
+        let head = self.bar.read(regs::NV_PFSP_MSGQ_HEAD::at(0)).val();
+        let tail = self.bar.read(regs::NV_PFSP_MSGQ_TAIL::at(0)).val();
 
         if head == tail {
             return 0;
@@ -122,20 +124,20 @@ fn poll_msgq(&self, bar: Bar0<'_>) -> u32 {
     /// Writes `packet` to FSP EMEM and updates the queue pointers to notify FSP.
     ///
     /// Returns `EINVAL` if `packet` is empty or its length is not 4-byte aligned.
-    pub(crate) fn send_msg(&mut self, bar: Bar0<'_>, packet: &[u8]) -> Result {
+    pub(crate) fn send_msg(&mut self, packet: &[u8]) -> Result {
         if packet.is_empty() {
             return Err(EINVAL);
         }
 
-        self.write_emem(bar, packet)?;
+        self.write_emem(packet)?;
 
         // Update queue pointers. TAIL points at the last DWORD written.
         let tail_offset = u32::try_from(packet.len() - 4).map_err(|_| EINVAL)?;
-        bar.write(
+        self.bar.write(
             Array::at(0),
             regs::NV_PFSP_QUEUE_TAIL::zeroed().with_address(tail_offset),
         );
-        bar.write(
+        self.bar.write(
             Array::at(0),
             regs::NV_PFSP_QUEUE_HEAD::zeroed().with_address(0),
         );
@@ -148,9 +150,9 @@ pub(crate) fn send_msg(&mut self, bar: Bar0<'_>, packet: &[u8]) -> Result {
     ///
     /// Returns `ETIMEDOUT` if no message was available until timeout, or a regular error code if a
     /// memory allocation error occurred.
-    pub(crate) fn recv_msg(&mut self, bar: Bar0<'_>) -> Result<KVec<u8>> {
+    pub(crate) fn recv_msg(&mut self) -> Result<KVec<u8>> {
         let msg_size = read_poll_timeout(
-            || Ok(self.poll_msgq(bar)),
+            || Ok(self.poll_msgq()),
             |&size| size > 0,
             Delta::from_millis(10),
             Delta::from_millis(FSP_MSG_TIMEOUT_MS),
@@ -160,11 +162,13 @@ pub(crate) fn recv_msg(&mut self, bar: Bar0<'_>) -> Result<KVec<u8>> {
         let mut buffer = KVec::<u8>::new();
         buffer.resize(msg_size, 0, GFP_KERNEL)?;
 
-        self.read_emem(bar, &mut buffer)?;
+        self.read_emem(&mut buffer)?;
 
         // Reset message queue pointers after reading.
-        bar.write(Array::at(0), regs::NV_PFSP_MSGQ_TAIL::zeroed().with_val(0));
-        bar.write(Array::at(0), regs::NV_PFSP_MSGQ_HEAD::zeroed().with_val(0));
+        self.bar
+            .write(Array::at(0), regs::NV_PFSP_MSGQ_TAIL::zeroed().with_val(0));
+        self.bar
+            .write(Array::at(0), regs::NV_PFSP_MSGQ_HEAD::zeroed().with_val(0));
 
         Ok(buffer)
     }
diff --git a/drivers/gpu/nova-core/falcon/gsp.rs b/drivers/gpu/nova-core/falcon/gsp.rs
index d1f6f7fcffff7..ae32f401aeb03 100644
--- a/drivers/gpu/nova-core/falcon/gsp.rs
+++ b/drivers/gpu/nova-core/falcon/gsp.rs
@@ -14,7 +14,6 @@
 };
 
 use crate::{
-    driver::Bar0,
     falcon::{
         Falcon,
         FalconEngine,
@@ -24,10 +23,6 @@
     regs,
 };
 
-/// Pattern returned by GSP register reads while the PRIV target mask still blocks CPU access.
-const GSP_TARGET_MASK_LOCKED_PATTERN: u32 = 0xbadf_4100;
-const GSP_TARGET_MASK_LOCKED_MASK: u32 = 0xffff_ff00;
-
 /// Type specifying the `Gsp` falcon engine. Cannot be instantiated.
 pub(crate) struct Gsp(());
 
@@ -41,20 +36,20 @@ impl RegisterBase<PFalcon2Base> for Gsp {
 
 impl FalconEngine for Gsp {}
 
-impl Falcon<Gsp> {
+impl<'a> Falcon<'a, Gsp> {
     /// Clears the SWGEN0 bit in the Falcon's IRQ status clear register to
     /// allow GSP to signal CPU for processing new messages in message queue.
-    pub(crate) fn clear_swgen0_intr(&self, bar: Bar0<'_>) {
-        bar.write(
+    pub(crate) fn clear_swgen0_intr(&self) {
+        self.bar.write(
             WithBase::of::<Gsp>(),
             regs::NV_PFALCON_FALCON_IRQSCLR::zeroed().with_swgen0(true),
         );
     }
 
     /// Checks if GSP reload/resume has completed during the boot process.
-    pub(crate) fn check_reload_completed(&self, bar: Bar0<'_>, timeout: Delta) -> Result<bool> {
+    pub(crate) fn check_reload_completed(&self, timeout: Delta) -> Result<bool> {
         read_poll_timeout(
-            || Ok(bar.read(regs::NV_PGC6_BSI_SECURE_SCRATCH_14)),
+            || Ok(self.bar.read(regs::NV_PGC6_BSI_SECURE_SCRATCH_14)),
             |val| val.boot_stage_3_handoff(),
             Delta::ZERO,
             timeout,
@@ -63,17 +58,24 @@ pub(crate) fn check_reload_completed(&self, bar: Bar0<'_>, timeout: Delta) -> Re
     }
 
     /// Returns whether the RISC-V branch privilege lockdown bit is set.
-    pub(crate) fn riscv_branch_privilege_lockdown(&self, bar: Bar0<'_>) -> bool {
-        bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<Gsp>())
+    pub(crate) fn riscv_branch_privilege_lockdown(&self) -> bool {
+        self.bar
+            .read(regs::NV_PFALCON_FALCON_HWCFG2::of::<Gsp>())
             .riscv_br_priv_lockdown()
     }
 
     /// Returns whether GSP registers can be read by the CPU.
-    pub(crate) fn priv_target_mask_released(&self, bar: Bar0<'_>) -> bool {
-        let hwcfg2 = bar
+    pub(crate) fn priv_target_mask_released(&self) -> bool {
+        /// Pattern returned by GSP register reads while the PRIV target mask still blocks CPU
+        /// access. The low byte varies; the upper 24 bits are fixed.
+        const LOCKED_PATTERN: u32 = 0xbadf_4100;
+        const LOCKED_MASK: u32 = 0xffff_ff00;
+
+        let hwcfg2 = self
+            .bar
             .read(regs::NV_PFALCON_FALCON_HWCFG2::of::<Gsp>())
             .into_raw();
 
-        hwcfg2 != 0 && (hwcfg2 & GSP_TARGET_MASK_LOCKED_MASK) != GSP_TARGET_MASK_LOCKED_PATTERN
+        hwcfg2 != 0 && (hwcfg2 & LOCKED_MASK) != LOCKED_PATTERN
     }
 }
diff --git a/drivers/gpu/nova-core/falcon/hal.rs b/drivers/gpu/nova-core/falcon/hal.rs
index 89b56823906b5..ee4a017f3a4c6 100644
--- a/drivers/gpu/nova-core/falcon/hal.rs
+++ b/drivers/gpu/nova-core/falcon/hal.rs
@@ -3,7 +3,6 @@
 use kernel::prelude::*;
 
 use crate::{
-    driver::Bar0,
     falcon::{
         Falcon,
         FalconBromParams,
@@ -34,7 +33,7 @@ pub(crate) enum LoadMethod {
 /// registers.
 pub(crate) trait FalconHal<E: FalconEngine>: Send + Sync {
     /// Activates the Falcon core if the engine is a risvc/falcon dual engine.
-    fn select_core(&self, _falcon: &Falcon<E>, _bar: Bar0<'_>) -> Result {
+    fn select_core(&self, _falcon: &Falcon<'_, E>) -> Result {
         Ok(())
     }
 
@@ -42,24 +41,23 @@ fn select_core(&self, _falcon: &Falcon<E>, _bar: Bar0<'_>) -> Result {
     /// falcon instance. `engine_id_mask` and `ucode_id` are obtained from the firmware header.
     fn signature_reg_fuse_version(
         &self,
-        falcon: &Falcon<E>,
-        bar: Bar0<'_>,
+        falcon: &Falcon<'_, E>,
         engine_id_mask: u16,
         ucode_id: u8,
     ) -> Result<u32>;
 
     /// Program the boot ROM registers prior to starting a secure firmware.
-    fn program_brom(&self, falcon: &Falcon<E>, bar: Bar0<'_>, params: &FalconBromParams);
+    fn program_brom(&self, falcon: &Falcon<'_, E>, params: &FalconBromParams);
 
     /// Check if the RISC-V core is active.
     /// Returns `true` if the RISC-V core is active, `false` otherwise.
-    fn is_riscv_active(&self, bar: Bar0<'_>) -> bool;
+    fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool;
 
     /// Wait for memory scrubbing to complete.
-    fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result;
+    fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result;
 
     /// Reset the falcon engine.
-    fn reset_eng(&self, bar: Bar0<'_>) -> Result;
+    fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result;
 
     /// Returns the method used to load data into the falcon's memory.
     ///
diff --git a/drivers/gpu/nova-core/falcon/hal/ga102.rs b/drivers/gpu/nova-core/falcon/hal/ga102.rs
index cf6ce47e6b256..fe821ded5fa12 100644
--- a/drivers/gpu/nova-core/falcon/hal/ga102.rs
+++ b/drivers/gpu/nova-core/falcon/hal/ga102.rs
@@ -115,33 +115,34 @@ pub(super) fn new() -> Self {
 }
 
 impl<E: FalconEngine> FalconHal<E> for Ga102<E> {
-    fn select_core(&self, _falcon: &Falcon<E>, bar: Bar0<'_>) -> Result {
-        select_core_ga102::<E>(bar)
+    fn select_core(&self, falcon: &Falcon<'_, E>) -> Result {
+        select_core_ga102::<E>(falcon.bar)
     }
 
     fn signature_reg_fuse_version(
         &self,
-        falcon: &Falcon<E>,
-        bar: Bar0<'_>,
+        falcon: &Falcon<'_, E>,
         engine_id_mask: u16,
         ucode_id: u8,
     ) -> Result<u32> {
-        signature_reg_fuse_version_ga102(&falcon.dev, bar, engine_id_mask, ucode_id)
+        signature_reg_fuse_version_ga102(falcon.dev, falcon.bar, engine_id_mask, ucode_id)
     }
 
-    fn program_brom(&self, _falcon: &Falcon<E>, bar: Bar0<'_>, params: &FalconBromParams) {
-        program_brom_ga102::<E>(bar, params);
+    fn program_brom(&self, falcon: &Falcon<'_, E>, params: &FalconBromParams) {
+        program_brom_ga102::<E>(falcon.bar, params);
     }
 
-    fn is_riscv_active(&self, bar: Bar0<'_>) -> bool {
-        bar.read(regs::NV_PRISCV_RISCV_CPUCTL::of::<E>())
+    fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool {
+        falcon
+            .bar
+            .read(regs::NV_PRISCV_RISCV_CPUCTL::of::<E>())
             .active_stat()
     }
 
-    fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result {
+    fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result {
         // TIMEOUT: memory scrubbing should complete in less than 20ms.
         read_poll_timeout(
-            || Ok(bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<E>())),
+            || Ok(falcon.bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<E>())),
             |r| r.mem_scrubbing_done(),
             Delta::ZERO,
             Delta::from_millis(20),
@@ -149,7 +150,9 @@ fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result {
         .map(|_| ())
     }
 
-    fn reset_eng(&self, bar: Bar0<'_>) -> Result {
+    fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result {
+        let bar = falcon.bar;
+
         let _ = bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<E>());
 
         // According to OpenRM's `kflcnPreResetWait_GA102` documentation, HW sometimes does not set
@@ -162,7 +165,7 @@ fn reset_eng(&self, bar: Bar0<'_>) -> Result {
         );
 
         regs::NV_PFALCON_FALCON_ENGINE::reset_engine::<E>(bar);
-        self.reset_wait_mem_scrubbing(bar)?;
+        self.reset_wait_mem_scrubbing(falcon)?;
 
         Ok(())
     }
diff --git a/drivers/gpu/nova-core/falcon/hal/tu102.rs b/drivers/gpu/nova-core/falcon/hal/tu102.rs
index 3aaee38693127..34bf9f3f44c73 100644
--- a/drivers/gpu/nova-core/falcon/hal/tu102.rs
+++ b/drivers/gpu/nova-core/falcon/hal/tu102.rs
@@ -13,7 +13,6 @@
 };
 
 use crate::{
-    driver::Bar0,
     falcon::{
         hal::LoadMethod,
         Falcon,
@@ -34,31 +33,32 @@ pub(super) fn new() -> Self {
 }
 
 impl<E: FalconEngine> FalconHal<E> for Tu102<E> {
-    fn select_core(&self, _falcon: &Falcon<E>, _bar: Bar0<'_>) -> Result {
+    fn select_core(&self, _falcon: &Falcon<'_, E>) -> Result {
         Ok(())
     }
 
     fn signature_reg_fuse_version(
         &self,
-        _falcon: &Falcon<E>,
-        _bar: Bar0<'_>,
+        _falcon: &Falcon<'_, E>,
         _engine_id_mask: u16,
         _ucode_id: u8,
     ) -> Result<u32> {
         Ok(0)
     }
 
-    fn program_brom(&self, _falcon: &Falcon<E>, _bar: Bar0<'_>, _params: &FalconBromParams) {}
+    fn program_brom(&self, _falcon: &Falcon<'_, E>, _params: &FalconBromParams) {}
 
-    fn is_riscv_active(&self, bar: Bar0<'_>) -> bool {
-        bar.read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS::of::<E>())
+    fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool {
+        falcon
+            .bar
+            .read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS::of::<E>())
             .active_stat()
     }
 
-    fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result {
+    fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result {
         // TIMEOUT: memory scrubbing should complete in less than 10ms.
         read_poll_timeout(
-            || Ok(bar.read(regs::NV_PFALCON_FALCON_DMACTL::of::<E>())),
+            || Ok(falcon.bar.read(regs::NV_PFALCON_FALCON_DMACTL::of::<E>())),
             |r| r.mem_scrubbing_done(),
             Delta::ZERO,
             Delta::from_millis(10),
@@ -66,9 +66,9 @@ fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result {
         .map(|_| ())
     }
 
-    fn reset_eng(&self, bar: Bar0<'_>) -> Result {
-        regs::NV_PFALCON_FALCON_ENGINE::reset_engine::<E>(bar);
-        self.reset_wait_mem_scrubbing(bar)?;
+    fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result {
+        regs::NV_PFALCON_FALCON_ENGINE::reset_engine::<E>(falcon.bar);
+        self.reset_wait_mem_scrubbing(falcon)?;
 
         Ok(())
     }
diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs
index 725e428154cf0..273cff752fae7 100644
--- a/drivers/gpu/nova-core/fb.rs
+++ b/drivers/gpu/nova-core/fb.rs
@@ -23,11 +23,11 @@
     firmware::gsp::GspFirmware,
     gpu::Chipset,
     gsp,
-    num::FromSafeCast,
-    regs, //
+    num::FromSafeCast, //
 };
 
 mod hal;
+mod regs;
 
 /// Type holding the sysmem flush memory page, a page of memory to be written into the
 /// `NV_PFB_NISO_FLUSH_SYSMEM_ADDR*` registers and used to maintain memory coherency.
diff --git a/drivers/gpu/nova-core/fb/hal/gb202.rs b/drivers/gpu/nova-core/fb/hal/gb202.rs
index 038d1278c6341..b78e0970f66db 100644
--- a/drivers/gpu/nova-core/fb/hal/gb202.rs
+++ b/drivers/gpu/nova-core/fb/hal/gb202.rs
@@ -4,13 +4,7 @@
 //! Blackwell GB20x framebuffer HAL.
 
 use kernel::{
-    io::{
-        register::{
-            RegisterBase,
-            WithBase, //
-        },
-        Io, //
-    },
+    io::Io,
     num::Bounded,
     prelude::*,
     sizes::SizeConstants, //
@@ -24,35 +18,29 @@
 
 struct Gb202;
 
-impl RegisterBase<regs::Fbhub0Base> for Gb202 {
-    const BASE: usize = 0x008a_0000;
-}
-
 fn read_sysmem_flush_page_gb202(bar: Bar0<'_>) -> u64 {
     let lo = u64::from(
-        bar.read(regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb202>())
+        bar.read(regs::NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_LO)
             .adr(),
     );
     let hi = u64::from(
-        bar.read(regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb202>())
+        bar.read(regs::NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_HI)
             .adr(),
     );
 
-    lo | (hi << 32)
+    (hi << 32) | lo
 }
 
 /// Write the sysmem flush page address through the GB20x FBHUB0 registers.
 fn write_sysmem_flush_page_gb202(bar: Bar0<'_>, addr: Bounded<u64, 52>) {
     // Write HI first. The hardware will trigger the flush on the LO write.
-    bar.write(
-        regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb202>(),
-        regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed()
+    bar.write_reg(
+        regs::NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed()
             .with_adr(addr.shr::<32, 20>().cast::<u32>()),
     );
-    bar.write(
-        regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb202>(),
+    bar.write_reg(
         // CAST: lower 32 bits. Hardware ignores bits 7:0.
-        regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(*addr as u32),
+        regs::NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(*addr as u32),
     );
 }
 
diff --git a/drivers/gpu/nova-core/fb/hal/gh100.rs b/drivers/gpu/nova-core/fb/hal/gh100.rs
index 5450c7254dadb..d39fe99537ed1 100644
--- a/drivers/gpu/nova-core/fb/hal/gh100.rs
+++ b/drivers/gpu/nova-core/fb/hal/gh100.rs
@@ -2,24 +2,49 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
 use kernel::{
+    io::Io,
+    num::Bounded,
     prelude::*,
     sizes::SizeConstants, //
 };
 
 use crate::{
     driver::Bar0,
-    fb::hal::FbHal, //
+    fb::hal::FbHal,
+    regs, //
 };
 
 struct Gh100;
 
+fn read_sysmem_flush_page_gh100(bar: Bar0<'_>) -> u64 {
+    let lo = u64::from(bar.read(regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO).adr());
+    let hi = u64::from(bar.read(regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI).adr());
+
+    (hi << 32) | lo
+}
+
+/// Write the sysmem flush page address through the Hopper FBHUB registers.
+fn write_sysmem_flush_page_gh100(bar: Bar0<'_>, addr: Bounded<u64, 52>) {
+    // Write HI first. The hardware will trigger the flush on the LO write.
+    bar.write_reg(
+        regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed()
+            .with_adr(addr.shr::<32, 20>().cast::<u32>()),
+    );
+    bar.write_reg(
+        // CAST: lower 32 bits. Hardware ignores bits 7:0.
+        regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(*addr as u32),
+    );
+}
+
 impl FbHal for Gh100 {
     fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 {
-        super::ga100::read_sysmem_flush_page_ga100(bar)
+        read_sysmem_flush_page_gh100(bar)
     }
 
     fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result {
-        super::ga100::write_sysmem_flush_page_ga100(bar, addr);
+        let addr = Bounded::<u64, 52>::try_new(addr).ok_or(EINVAL)?;
+
+        write_sysmem_flush_page_gh100(bar, addr);
 
         Ok(())
     }
diff --git a/drivers/gpu/nova-core/fb/regs.rs b/drivers/gpu/nova-core/fb/regs.rs
new file mode 100644
index 0000000000000..b2ec02f584bee
--- /dev/null
+++ b/drivers/gpu/nova-core/fb/regs.rs
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use kernel::io::register;
+
+// PDISP
+
+register! {
+    pub(super) NV_PDISP_VGA_WORKSPACE_BASE(u32) @ 0x00625f04 {
+        /// VGA workspace base address divided by 0x10000.
+        31:8    addr;
+        /// Set if the `addr` field is valid.
+        3:3     status_valid => bool;
+    }
+}
+
+impl NV_PDISP_VGA_WORKSPACE_BASE {
+    /// Returns the base address of the VGA workspace, or `None` if none exists.
+    pub(super) fn vga_workspace_addr(self) -> Option<u64> {
+        if self.status_valid() {
+            Some(u64::from(self.addr()) << 16)
+        } else {
+            None
+        }
+    }
+}
diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs
index 1e89390209f56..20eff987c5d6a 100644
--- a/drivers/gpu/nova-core/firmware.rs
+++ b/drivers/gpu/nova-core/firmware.rs
@@ -21,6 +21,7 @@
         FalconFirmware, //
     },
     gpu,
+    gsp::boot_firmware_files,
     num::{
         FromSafeCast,
         IntoSafeCast, //
@@ -88,7 +89,7 @@ pub(crate) struct FalconUCodeDescV2 {
 
 /// Structure used to describe some firmwares, notably FWSEC-FRTS.
 #[repr(C)]
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, FromBytes)]
 pub(crate) struct FalconUCodeDescV3 {
     /// Header defined by `NV_BIT_FALCON_UCODE_DESC_HEADER_VDESC*` in OpenRM.
     hdr: u32,
@@ -119,10 +120,6 @@ pub(crate) struct FalconUCodeDescV3 {
     _reserved: u16,
 }
 
-// SAFETY: all bit patterns are valid for this type, and it doesn't use
-// interior mutability.
-unsafe impl FromBytes for FalconUCodeDescV3 {}
-
 /// Enum wrapping the different versions of Falcon microcode descriptors.
 ///
 /// This allows handling both V2 and V3 descriptor formats through a
@@ -423,23 +420,20 @@ const fn make_entry_file(self, chipset: &str, fw: &str) -> Self {
     const fn make_entry_chipset(self, chipset: gpu::Chipset) -> Self {
         let name = chipset.name();
 
-        let this = self
-            .make_entry_file(name, "booter_load")
-            .make_entry_file(name, "booter_unload")
+        // GSP firmware files are always present.
+        let mut this = self
             .make_entry_file(name, "bootloader")
             .make_entry_file(name, "gsp");
 
-        let this = if chipset.needs_fwsec_bootloader() {
-            this.make_entry_file(name, "gen_bootloader")
-        } else {
-            this
-        };
-
-        if chipset.uses_fsp() {
-            this.make_entry_file(name, "fmc")
-        } else {
-            this
+        // Add the firmware files specific to the GSP boot method of `chipset`.
+        let boot_files = boot_firmware_files(chipset);
+        let mut i = 0;
+        while i < boot_files.len() {
+            this = this.make_entry_file(name, boot_files[i]);
+            i += 1;
         }
+
+        this
     }
 
     pub(crate) const fn create(
@@ -464,11 +458,9 @@ pub(crate) const fn create(
 /// that scheme before nova-core becomes stable, which means this module will eventually be
 /// removed.
 mod elf {
-    use core::mem::size_of;
-
     use kernel::{
         bindings,
-        str::CStr,
+        prelude::*,
         transmute::FromBytes, //
     };
 
diff --git a/drivers/gpu/nova-core/firmware/booter.rs b/drivers/gpu/nova-core/firmware/booter.rs
index d9313ac361af3..acb7f4d8a5327 100644
--- a/drivers/gpu/nova-core/firmware/booter.rs
+++ b/drivers/gpu/nova-core/firmware/booter.rs
@@ -15,7 +15,6 @@
 };
 
 use crate::{
-    driver::Bar0,
     falcon::{
         sec2::Sec2,
         Falcon,
@@ -293,8 +292,7 @@ pub(crate) fn new(
         kind: BooterKind,
         chipset: Chipset,
         ver: &str,
-        falcon: &Falcon<<Self as FalconFirmware>::Target>,
-        bar: Bar0<'_>,
+        falcon: &Falcon<'_, <Self as FalconFirmware>::Target>,
     ) -> Result<Self> {
         let fw_name = match kind {
             BooterKind::Loader => "booter_load",
@@ -339,11 +337,8 @@ pub(crate) fn new(
             } else {
                 // Obtain the version from the fuse register, and extract the corresponding
                 // signature.
-                let reg_fuse_version = falcon.signature_reg_fuse_version(
-                    bar,
-                    brom_params.engine_id_mask,
-                    brom_params.ucode_id,
-                )?;
+                let reg_fuse_version = falcon
+                    .signature_reg_fuse_version(brom_params.engine_id_mask, brom_params.ucode_id)?;
 
                 // `0` means the last signature should be used.
                 const FUSE_VERSION_USE_LAST_SIG: u32 = 0;
@@ -405,18 +400,14 @@ pub(crate) fn new(
     pub(crate) fn run<T>(
         &self,
         dev: &device::Device<device::Bound>,
-        bar: Bar0<'_>,
-        sec2_falcon: &Falcon<Sec2>,
+        sec2_falcon: &Falcon<'_, Sec2>,
         wpr_meta: &Coherent<T>,
     ) -> Result {
-        sec2_falcon.reset(bar)?;
-        sec2_falcon.load(dev, bar, self)?;
+        sec2_falcon.reset()?;
+        sec2_falcon.load(self)?;
         let wpr_handle = wpr_meta.dma_handle();
-        let (mbox0, mbox1) = sec2_falcon.boot(
-            bar,
-            Some(wpr_handle as u32),
-            Some((wpr_handle >> 32) as u32),
-        )?;
+        let (mbox0, mbox1) =
+            sec2_falcon.boot(Some(wpr_handle as u32), Some((wpr_handle >> 32) as u32))?;
         dev_dbg!(dev, "SEC2 MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1);
 
         if mbox0 != 0 {
diff --git a/drivers/gpu/nova-core/firmware/fwsec.rs b/drivers/gpu/nova-core/firmware/fwsec.rs
index 199ae2adb664c..7a931f22f6298 100644
--- a/drivers/gpu/nova-core/firmware/fwsec.rs
+++ b/drivers/gpu/nova-core/firmware/fwsec.rs
@@ -27,7 +27,6 @@
 };
 
 use crate::{
-    driver::Bar0,
     falcon::{
         gsp::Gsp,
         Falcon,
@@ -320,8 +319,7 @@ impl FwsecFirmware {
     /// command.
     pub(crate) fn new(
         dev: &Device<device::Bound>,
-        falcon: &Falcon<Gsp>,
-        bar: Bar0<'_>,
+        falcon: &Falcon<'_, Gsp>,
         bios: &Vbios,
         cmd: FwsecCommand,
     ) -> Result<Self> {
@@ -337,7 +335,7 @@ pub(crate) fn new(
                 .ok_or(EINVAL)?;
             let desc_sig_versions = u32::from(desc.signature_versions());
             let reg_fuse_version =
-                falcon.signature_reg_fuse_version(bar, desc.engine_id_mask(), desc.ucode_id())?;
+                falcon.signature_reg_fuse_version(desc.engine_id_mask(), desc.ucode_id())?;
             dev_dbg!(
                 dev,
                 "desc_sig_versions: {:#x}, reg_fuse_version: {}\n",
@@ -387,24 +385,18 @@ pub(crate) fn new(
 
     /// Loads the FWSEC firmware into `falcon` and execute it.
     ///
-    /// This must only be called on chipsets that do not need the FWSEC bootloader (i.e., where
-    /// [`Chipset::needs_fwsec_bootloader()`](crate::gpu::Chipset::needs_fwsec_bootloader) returns
-    /// `false`). On chipsets that do, use [`bootloader::FwsecFirmwareWithBl`] instead.
-    pub(crate) fn run(
-        &self,
-        dev: &Device<device::Bound>,
-        falcon: &Falcon<Gsp>,
-        bar: Bar0<'_>,
-    ) -> Result<()> {
+    /// This must only be called on chipsets that do not need the FWSEC bootloader. On chipsets
+    /// where the bootloader is required, use [`bootloader::FwsecFirmwareWithBl`] instead.
+    pub(crate) fn run(&self, dev: &Device<device::Bound>, falcon: &Falcon<'_, Gsp>) -> Result<()> {
         // Reset falcon, load the firmware, and run it.
         falcon
-            .reset(bar)
+            .reset()
             .inspect_err(|e| dev_err!(dev, "Failed to reset GSP falcon: {:?}\n", e))?;
         falcon
-            .load(dev, bar, self)
+            .load(self)
             .inspect_err(|e| dev_err!(dev, "Failed to load FWSEC firmware: {:?}\n", e))?;
         let (mbox0, _) = falcon
-            .boot(bar, Some(0), None)
+            .boot(Some(0), None)
             .inspect_err(|e| dev_err!(dev, "Failed to boot FWSEC firmware: {:?}\n", e))?;
         if mbox0 != 0 {
             dev_err!(dev, "FWSEC firmware returned error {}\n", mbox0);
diff --git a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs
index 039920dc340b8..d9fafd2eea5b1 100644
--- a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs
+++ b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs
@@ -7,16 +7,12 @@
 //! be loaded using PIO.
 
 use kernel::{
-    alloc::KVec,
     device::{
         self,
         Device, //
     },
     dma::Coherent,
-    io::{
-        register::WithBase, //
-        Io,
-    },
+    io::{register::WithBase, Io},
     prelude::*,
     ptr::{
         Alignable,
@@ -51,7 +47,7 @@
         FIRMWARE_VERSION, //
     },
     gpu::Chipset,
-    num::FromSafeCast,
+    num::FromSafeCast, //
     regs,
 };
 
@@ -279,15 +275,15 @@ pub(crate) fn new(
     pub(crate) fn run(
         &self,
         dev: &Device<device::Bound>,
-        falcon: &Falcon<Gsp>,
+        falcon: &Falcon<'_, Gsp>,
         bar: Bar0<'_>,
     ) -> Result<()> {
         // Reset falcon, load the firmware, and run it.
         falcon
-            .reset(bar)
+            .reset()
             .inspect_err(|e| dev_err!(dev, "Failed to reset GSP falcon: {:?}\n", e))?;
         falcon
-            .pio_load(bar, self)
+            .pio_load(self)
             .inspect_err(|e| dev_err!(dev, "Failed to load FWSEC firmware: {:?}\n", e))?;
 
         // Configure DMA index for the bootloader to fetch the FWSEC firmware from system memory.
@@ -302,7 +298,7 @@ pub(crate) fn run(
         );
 
         let (mbox0, _) = falcon
-            .boot(bar, Some(0), None)
+            .boot(Some(0), None)
             .inspect_err(|e| dev_err!(dev, "Failed to boot FWSEC firmware: {:?}\n", e))?;
         if mbox0 != 0 {
             dev_err!(dev, "FWSEC firmware returned error {}\n", mbox0);
diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs
index 8fc243c66e356..afbd75879d169 100644
--- a/drivers/gpu/nova-core/fsp.rs
+++ b/drivers/gpu/nova-core/fsp.rs
@@ -11,6 +11,7 @@
     device,
     dma::Coherent,
     io::poll::read_poll_timeout,
+    num::TryIntoBounded,
     prelude::*,
     ptr::{
         Alignable,
@@ -31,9 +32,12 @@
         Falcon, //
     },
     fb::FbLayout,
-    firmware::fsp::{
-        FmcSignatures,
-        FspFirmware, //
+    firmware::{
+        fsp::{
+            FmcSignatures,
+            FspFirmware, //
+        },
+        FIRMWARE_VERSION, //
     },
     gpu::Chipset,
     gsp::GspFmcBootParams,
@@ -57,17 +61,40 @@ struct NvdmPayloadCommandResponse {
     error_code: u32,
 }
 
-/// Complete FSP response structure with MCTP and NVDM headers.
+/// Common MCTP and NVDM headers shared by all FSP messages.
 #[repr(C, packed)]
 #[derive(Clone, Copy)]
-struct FspResponse {
+struct FspMessageHeader {
     mctp_header: MctpHeader,
     nvdm_header: NvdmHeader,
+}
+
+// SAFETY: FspMessageHeader is a packed C struct with only integral fields.
+unsafe impl AsBytes for FspMessageHeader {}
+
+// SAFETY: FspMessageHeader is a packed C struct with only integral fields.
+unsafe impl FromBytes for FspMessageHeader {}
+
+impl FspMessageHeader {
+    /// Construct a standard FSP message header for the given NVDM type.
+    fn new(nvdm_type: NvdmType) -> Self {
+        Self {
+            mctp_header: MctpHeader::single_packet(),
+            nvdm_header: NvdmHeader::new(nvdm_type),
+        }
+    }
+}
+
+/// Common FSP response header with MCTP, NVDM and command response payloads.
+#[repr(C, packed)]
+#[derive(Clone, Copy)]
+struct FspResponseHeader {
+    header: FspMessageHeader,
     response: NvdmPayloadCommandResponse,
 }
 
-// SAFETY: FspResponse is a packed C struct with only integral fields.
-unsafe impl FromBytes for FspResponse {}
+// SAFETY: FspResponseHeader is a packed C struct with only integral fields.
+unsafe impl FromBytes for FspResponseHeader {}
 
 /// Trait implemented by types representing a message to send to FSP.
 ///
@@ -94,23 +121,23 @@ struct NvdmPayloadCot {
     gsp_boot_args_sysmem_offset: u64,
 }
 
-/// Complete FSP message structure with MCTP and NVDM headers.
+/// Complete FSP COT (Chain of Trust) message structure.
 #[repr(C)]
 #[derive(Clone, Copy)]
-struct FspMessage {
-    mctp_header: MctpHeader,
-    nvdm_header: NvdmHeader,
+struct FspCotMessage {
+    header: FspMessageHeader,
     cot: NvdmPayloadCot,
 }
 
-impl FspMessage {
-    /// Returns an in-place initializer for [`FspMessage`].
+impl FspCotMessage {
+    /// Returns an in-place initializer for [`FspCotMessage`].
     fn new<'a>(
         fb_layout: &FbLayout,
         fsp_fw: &'a FspFirmware,
         args: &'a FmcBootArgs,
     ) -> Result<impl Init<Self> + 'a> {
-        // frts_offset is relative to FB end: FRTS_location = FB_END - frts_offset
+        // frts_vidmem_offset is measured from the end of FB, so FRTS sits at
+        // (end of FB) - frts_vidmem_offset.
         let frts_vidmem_offset = if !args.resume {
             let frts_reserved_size = fb_layout.heap.len() + u64::from(fb_layout.pmu_reserved_size);
 
@@ -131,8 +158,7 @@ fn new<'a>(
         let size = num::usize_into_u16::<{ core::mem::size_of::<NvdmPayloadCot>() }>();
 
         Ok(init!(Self {
-            mctp_header: MctpHeader::single_packet(),
-            nvdm_header: NvdmHeader::new(NvdmType::Cot),
+            header: FspMessageHeader::new(NvdmType::Cot),
             // The payload is packed, so we cannot use `init!`. Initialize it member-by-member using
             // `chain`.
             cot <- pin_init::init_zeroed(),
@@ -143,8 +169,8 @@ fn new<'a>(
             msg.cot.gsp_fmc_sysmem_offset = fsp_fw.fmc_image.dma_handle();
             msg.cot.frts_vidmem_offset = frts_vidmem_offset;
             msg.cot.frts_vidmem_size = frts_size;
-            // frts_sysmem_* intentionally left at zero for now, but will be needed for e.g.
-            // systems without VRAM.
+            // frts_sysmem_* are left at zero because this path places FRTS in vidmem. The sysmem
+            // fields point to an FRTS buffer in sysmem instead, for systems without VRAM.
             msg.cot.gsp_boot_args_sysmem_offset = args.fmc_boot_params.dma_handle();
             msg.cot.sigs = *fsp_fw.fmc_sigs;
 
@@ -153,11 +179,11 @@ fn new<'a>(
     }
 }
 
-// SAFETY: `FspMessage` is `#[repr(C)]` with no padding, so all of its
+// SAFETY: `FspCotMessage` is `#[repr(C)]` with no padding, so all of its
 // bytes are initialized.
-unsafe impl AsBytes for FspMessage {}
+unsafe impl AsBytes for FspCotMessage {}
 
-impl MessageToFsp for FspMessage {
+impl MessageToFsp for FspCotMessage {
     const NVDM_TYPE: NvdmType = NvdmType::Cot;
 }
 
@@ -199,28 +225,45 @@ pub(crate) fn boot_params_dma_handle(&self) -> u64 {
 /// An `Fsp` is produced by [`Fsp::wait_secure_boot`], which only returns once FSP secure boot
 /// has completed. It owns the FSP falcon and the FMC firmware, which are used for the subsequent
 /// Chain of Trust boot.
-pub(crate) struct Fsp {
-    falcon: Falcon<FspEngine>,
+pub(crate) struct Fsp<'a> {
+    falcon: Falcon<'a, FspEngine>,
     fsp_fw: FspFirmware,
 }
 
-impl Fsp {
+impl<'a> Fsp<'a> {
+    /// Attempts to create a `Fsp` instance.
+    ///
+    /// This can involve waiting for FSP secure boot completion, but should be instantaneous in
+    /// practice.
+    ///
+    /// If `chipset` doesn't support FSP, `Ok(None)` is returned.
+    pub(crate) fn try_new(
+        dev: &'a device::Device<device::Bound>,
+        bar: Bar0<'a>,
+        chipset: Chipset,
+    ) -> Result<Option<Self>> {
+        match hal::fsp_hal(chipset) {
+            None => Ok(None),
+            Some(hal) => Self::wait_secure_boot(dev, bar, chipset, hal).map(Option::Some),
+        }
+    }
+
     /// Waits for FSP secure boot completion, then returns the [`Fsp`] interface.
     ///
     /// Polls the thermal scratch register until FSP signals boot completion or the timeout
     /// elapses. Returning an [`Fsp`] only on success guarantees, at the API level, that the
     /// interface is not used before secure boot has completed.
-    pub(crate) fn wait_secure_boot(
-        dev: &device::Device<device::Bound>,
-        bar: Bar0<'_>,
+    fn wait_secure_boot(
+        dev: &'a device::Device<device::Bound>,
+        bar: Bar0<'a>,
         chipset: Chipset,
-        fsp_fw: FspFirmware,
-    ) -> Result<Fsp> {
+        hal: &'static dyn hal::FspHal,
+    ) -> Result<Fsp<'a>> {
         /// FSP secure boot completion timeout in milliseconds.
         const FSP_SECURE_BOOT_TIMEOUT_MS: i64 = 5000;
 
-        let hal = hal::fsp_hal(chipset).ok_or(ENOTSUPP)?;
-        let falcon = Falcon::<FspEngine>::new(dev, chipset)?;
+        let falcon = Falcon::<FspEngine>::new(dev, chipset, bar)?;
+        let fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?;
 
         read_poll_timeout(
             || Ok(hal.fsp_boot_status(bar)),
@@ -236,23 +279,25 @@ pub(crate) fn wait_secure_boot(
     }
 
     /// Sends a message to FSP and waits for the response.
-    fn send_sync_fsp<M>(&mut self, dev: &device::Device, bar: Bar0<'_>, msg: &M) -> Result
+    /// Returns the full response buffer on success.
+    fn send_sync_fsp<M>(&mut self, dev: &device::Device, msg: &M) -> Result<KVec<u8>>
     where
         M: MessageToFsp,
     {
-        self.falcon.send_msg(bar, msg.as_bytes())?;
+        self.falcon.send_msg(msg.as_bytes())?;
 
-        let response_buf = self.falcon.recv_msg(bar).inspect_err(|e| {
+        let response_buf = self.falcon.recv_msg().inspect_err(|e| {
             dev_err!(dev, "FSP response error: {:?}\n", e);
         })?;
 
-        let (response, _) = FspResponse::from_bytes_prefix(&response_buf[..]).ok_or_else(|| {
-            dev_err!(dev, "FSP response too small: {}\n", response_buf.len());
-            EIO
-        })?;
+        let (response, _) =
+            FspResponseHeader::from_bytes_prefix(&response_buf[..]).ok_or_else(|| {
+                dev_err!(dev, "FSP response too small: {}\n", response_buf.len());
+                EIO
+            })?;
 
-        let mctp_header = response.mctp_header;
-        let nvdm_header = response.nvdm_header;
+        let mctp_header = response.header.mctp_header;
+        let nvdm_header = response.header.nvdm_header;
         let command_nvdm_type = response.response.command_nvdm_type;
         let error_code = response.response.error_code;
 
@@ -274,7 +319,7 @@ fn send_sync_fsp<M>(&mut self, dev: &device::Device, bar: Bar0<'_>, msg: &M) ->
             return Err(EIO);
         }
 
-        if command_nvdm_type != u8::from(M::NVDM_TYPE).into() {
+        if command_nvdm_type.try_into_bounded() != Some(M::NVDM_TYPE.into()) {
             dev_err!(
                 dev,
                 "Expected NVDM type {:?} in reply, got {:#x}\n",
@@ -294,7 +339,7 @@ fn send_sync_fsp<M>(&mut self, dev: &device::Device, bar: Bar0<'_>, msg: &M) ->
             return Err(EIO);
         }
 
-        Ok(())
+        Ok(response_buf)
     }
 
     /// Boots GSP FMC via FSP Chain of Trust.
@@ -304,15 +349,17 @@ fn send_sync_fsp<M>(&mut self, dev: &device::Device, bar: Bar0<'_>, msg: &M) ->
     pub(crate) fn boot_fmc(
         &mut self,
         dev: &device::Device<device::Bound>,
-        bar: Bar0<'_>,
         fb_layout: &FbLayout,
         args: &FmcBootArgs,
     ) -> Result {
         dev_dbg!(dev, "Starting FSP boot sequence for {}\n", args.chipset);
 
-        let msg = KBox::init(FspMessage::new(fb_layout, &self.fsp_fw, args)?, GFP_KERNEL)?;
+        let msg = KBox::init(
+            FspCotMessage::new(fb_layout, &self.fsp_fw, args)?,
+            GFP_KERNEL,
+        )?;
 
-        self.send_sync_fsp(dev, bar, &*msg)?;
+        let _response_buf = self.send_sync_fsp(dev, &*msg)?;
 
         dev_dbg!(dev, "FSP Chain of Trust completed successfully\n");
         Ok(())
diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs
index b3c91731db45d..442c0979f9c6f 100644
--- a/drivers/gpu/nova-core/gpu.rs
+++ b/drivers/gpu/nova-core/gpu.rs
@@ -9,7 +9,8 @@
     io::Io,
     num::Bounded,
     pci,
-    prelude::*, //
+    prelude::*,
+    sizes::SizeConstants, //
 };
 
 use crate::{
@@ -21,9 +22,12 @@
         Falcon, //
     },
     fb::SysmemFlush,
+    fsp::Fsp,
     gsp::{
         self,
-        Gsp, //
+        commands::GetGspStaticInfoReply,
+        Gsp,
+        GspBootContext, //
     },
     regs,
 };
@@ -130,22 +134,6 @@ pub(crate) const fn arch(self) -> Architecture {
         }
     }
 
-    /// Returns `true` if this chipset requires the PIO-loaded bootloader in order to boot FWSEC.
-    ///
-    /// This includes all chipsets < GA102.
-    pub(crate) const fn needs_fwsec_bootloader(self) -> bool {
-        matches!(self.arch(), Architecture::Turing) || matches!(self, Self::GA100)
-    }
-
-    /// Returns `true` if this chipset boots via FSP (Hopper and later), which requires the FMC
-    /// firmware image.
-    pub(crate) const fn uses_fsp(self) -> bool {
-        matches!(
-            self.arch(),
-            Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x
-        )
-    }
-
     /// Returns the address range of the PCI config mirror space.
     pub(crate) fn pci_config_mirror_range(self) -> Range<u32> {
         hal::gpu_hal(self).pci_config_mirror_range()
@@ -262,37 +250,84 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
     }
 }
 
-/// Structure holding the resources required to operate the GPU.
+/// Self-contained resources to operate and drop the GSP.
 #[pin_data(PinnedDrop)]
-pub(crate) struct Gpu<'gpu> {
+struct GspResources<'gpu> {
     /// Device owning the GPU.
-    device: &'gpu device::Device<device::Bound>,
+    device: &'gpu pci::Device<device::Bound>,
+    /// Details about the chipset.
     spec: Spec,
     /// MMIO mapping of PCI BAR 0.
     bar: Bar0<'gpu>,
-    /// System memory page required for flushing all pending GPU-side memory writes done through
-    /// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation).
-    sysmem_flush: SysmemFlush<'gpu>,
     /// GSP falcon instance, used for GSP boot up and cleanup.
-    gsp_falcon: Falcon<GspFalcon>,
+    gsp_falcon: Falcon<'gpu, GspFalcon>,
     /// SEC2 falcon instance, used for GSP boot up and cleanup.
-    sec2_falcon: Falcon<Sec2Falcon>,
-    /// GSP runtime data. Temporarily an empty placeholder.
+    sec2_falcon: Falcon<'gpu, Sec2Falcon>,
+    /// FSP instance, if on an arch that supports it.
+    // TODO: use different resource types for each boot method, and make the relevant Gsp methods
+    // generic against them.
+    fsp: Option<Fsp<'gpu>>,
+    /// GSP runtime data.
     #[pin]
     gsp: Gsp,
     /// GSP unload firmware bundle, if any.
     unload_bundle: Option<gsp::UnloadBundle>,
 }
 
+/// Structure holding the resources required to operate the GPU.
+#[pin_data]
+pub(crate) struct Gpu<'gpu> {
+    spec: Spec,
+    /// Static GPU information as provided by the GSP.
+    gsp_static_info: GetGspStaticInfoReply,
+    /// GSP and its resources.
+    #[pin]
+    gsp_resources: GspResources<'gpu>,
+    /// System memory page required for flushing all pending GPU-side memory writes done through
+    /// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation).
+    ///
+    /// Must be kept declared *after* `gsp_resources`, as the latter's `PinnedDrop` implementation
+    /// requires the sysmem flush page to be in place.
+    sysmem_flush: SysmemFlush<'gpu>,
+}
+
+#[pinned_drop]
+impl PinnedDrop for GspResources<'_> {
+    fn drop(self: Pin<&mut Self>) {
+        let this = self.project();
+        let device = *this.device;
+        let bar = *this.bar;
+        let bundle = this.unload_bundle.take();
+
+        let _ = this
+            .gsp
+            .as_ref()
+            .get_ref()
+            .unload(
+                GspBootContext {
+                    pdev: device,
+                    bar,
+                    chipset: this.spec.chipset,
+                    gsp_falcon: &*this.gsp_falcon,
+                    sec2_falcon: &*this.sec2_falcon,
+                    fsp: this.fsp.as_mut(),
+                },
+                bundle,
+            )
+            .inspect_err(|e| dev_err!(device, "failed to unload GSP: {:?}\n", e));
+    }
+}
+
 impl<'gpu> Gpu<'gpu> {
     pub(crate) fn new(
         pdev: &'gpu pci::Device<device::Core<'_>>,
         bar: Bar0<'gpu>,
     ) -> impl PinInit<Self, Error> + 'gpu {
+        let dev = pdev.as_ref();
+
         try_pin_init!(Self {
-            device: pdev.as_ref(),
-            spec: Spec::new(pdev.as_ref(), bar).inspect(|spec| {
-                dev_info!(pdev,"NVIDIA ({})\n", spec);
+            spec: Spec::new(dev, bar).inspect(|spec| {
+                dev_info!(dev,"NVIDIA ({})\n", spec);
             })?,
 
             // We must wait for GFW_BOOT completion before doing any significant setup on the GPU.
@@ -305,43 +340,70 @@ pub(crate) fn new(
                 unsafe { pdev.dma_set_mask_and_coherent(dma_mask)? };
 
                 hal.wait_gfw_boot_completion(bar)
-                    .inspect_err(|_| dev_err!(pdev, "GFW boot did not complete\n"))?;
+                    .inspect_err(|_| dev_err!(dev, "GFW boot did not complete\n"))?;
             },
 
-            sysmem_flush: SysmemFlush::register(pdev.as_ref(), bar, spec.chipset)?,
+            // Initialize this early because `gsp_resources` depends on it.
+            sysmem_flush: SysmemFlush::register(dev, bar, spec.chipset)?,
 
-            gsp_falcon: Falcon::new(
-                pdev.as_ref(),
-                spec.chipset,
-            )
-            .inspect(|falcon| falcon.clear_swgen0_intr(bar))?,
+            gsp_resources <- try_pin_init!(GspResources {
+                device: pdev,
 
-            sec2_falcon: Falcon::new(pdev.as_ref(), spec.chipset)?,
+                spec: *spec,
 
-            gsp <- Gsp::new(pdev),
+                bar,
 
-            // This member must be initialized last, so the `UnloadBundle` can never be dropped from
-            // outside of the constructed `Gpu`, ensuring that the unload sequence is properly run
-            // in case of failure.
-            unload_bundle: gsp.boot(pdev, bar, spec.chipset, gsp_falcon, sec2_falcon)?,
-            bar,
-        })
-    }
-}
+                gsp_falcon: Falcon::new(
+                    dev,
+                    spec.chipset,
+                    bar
+                )
+                .inspect(|falcon| falcon.clear_swgen0_intr())?,
 
-#[pinned_drop]
-impl PinnedDrop for Gpu<'_> {
-    fn drop(self: Pin<&mut Self>) {
-        let this = self.project();
-        let device = *this.device;
-        let bar = *this.bar;
-        let bundle = this.unload_bundle.take();
+                sec2_falcon: Falcon::new(dev, spec.chipset, bar)?,
 
-        let _ = this
-            .gsp
-            .as_ref()
-            .get_ref()
-            .unload(device, bar, &*this.gsp_falcon, &*this.sec2_falcon, bundle)
-            .inspect_err(|e| dev_err!(device, "failed to unload GSP: {:?}\n", e));
+                fsp: Fsp::try_new(dev, bar, spec.chipset)?,
+
+                gsp <- Gsp::new(pdev),
+
+                // This member must be initialized last, so the `UnloadBundle` can never be dropped
+                // from outside of the constructed `GspResources`, ensuring that the unload sequence
+                // is properly run in case of failure.
+                unload_bundle: gsp.boot(GspBootContext {
+                    pdev,
+                    bar,
+                    chipset: spec.chipset,
+                    gsp_falcon,
+                    sec2_falcon,
+                    fsp: fsp.as_mut(),
+                })?,
+            }),
+
+            gsp_static_info: {
+                // Obtain and display basic GPU information.
+                let info = gsp_resources.gsp.get_static_info(bar)?;
+                match info.gpu_name() {
+                    Ok(name) => dev_info!(dev, "GPU name: {}\n", name),
+                    Err(e) => dev_warn!(dev, "GPU name unavailable: {:?}\n", e),
+                }
+
+                if !info.usable_fb_regions.is_empty() {
+                    dev_dbg!(dev, "Usable FB regions:\n");
+                    for region in &info.usable_fb_regions {
+                        dev_dbg!(dev, "  - {:#x?}\n", region);
+                    }
+
+                    dev_dbg!(
+                        dev,
+                        "Total usable VRAM: {} MiB\n",
+                        info.usable_fb_regions.iter().fold(0u64, |res, region| res
+                            .saturating_add(region.end - region.start))
+                            / u64::SZ_1M
+                    );
+                }
+
+                info
+            }
+        })
     }
 }
diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs
index 69175ca3315c5..f38630026e5db 100644
--- a/drivers/gpu/nova-core/gsp.rs
+++ b/drivers/gpu/nova-core/gsp.rs
@@ -9,19 +9,22 @@
     dma::{
         Coherent,
         CoherentBox,
+        CoherentView,
         DmaAddress, //
     },
+    io::{
+        io_project,
+        io_write,
+        Io, //
+    },
     pci,
-    prelude::*,
-    transmute::{
-        AsBytes,
-        FromBytes, //
-    }, //
+    prelude::*, //
 };
 
 pub(crate) mod cmdq;
 pub(crate) mod commands;
 mod fw;
+mod regs;
 mod sequencer;
 
 pub(crate) use fw::{
@@ -29,12 +32,23 @@
     GspFwWprMeta,
     LibosParams, //
 };
+pub(crate) use hal::boot_firmware_files;
 
 use crate::{
-    gsp::cmdq::Cmdq,
-    gsp::fw::{
-        GspArgumentsPadded,
-        LibosMemoryRegionInitArgument, //
+    driver::Bar0,
+    falcon::{
+        gsp::Gsp as GspFalcon,
+        sec2::Sec2 as Sec2Falcon,
+        Falcon, //
+    },
+    fsp::Fsp,
+    gpu::Chipset,
+    gsp::{
+        cmdq::Cmdq,
+        fw::{
+            GspArgumentsPadded,
+            LibosMemoryRegionInitArgument, //
+        },
     },
     num,
 };
@@ -42,27 +56,48 @@
 pub(crate) const GSP_PAGE_SHIFT: usize = 12;
 pub(crate) const GSP_PAGE_SIZE: usize = 1 << GSP_PAGE_SHIFT;
 
+/// Common context for the GSP boot process.
+///
+/// It carries two distinct lifetimes:
+///
+/// - `'gpu` is the lifetime of the bound GPU device, as captured by the GPU subdevices.
+/// - `'ctx` is a shorter lifetime during which this context borrows those subdevices.
+pub(crate) struct GspBootContext<'ctx, 'gpu> {
+    pub(crate) pdev: &'gpu pci::Device<device::Bound>,
+    pub(crate) bar: Bar0<'gpu>,
+    pub(crate) chipset: Chipset,
+    pub(crate) gsp_falcon: &'ctx Falcon<'gpu, GspFalcon>,
+    pub(crate) sec2_falcon: &'ctx Falcon<'gpu, Sec2Falcon>,
+    pub(crate) fsp: Option<&'ctx mut Fsp<'gpu>>,
+}
+
+impl<'ctx, 'gpu> GspBootContext<'ctx, 'gpu> {
+    pub(crate) fn dev(&self) -> &'gpu device::Device<device::Bound> {
+        self.pdev.as_ref()
+    }
+}
+
 /// Number of GSP pages to use in a RM log buffer.
 const RM_LOG_BUFFER_NUM_PAGES: usize = 0x10;
 const LOG_BUFFER_SIZE: usize = RM_LOG_BUFFER_NUM_PAGES * GSP_PAGE_SIZE;
 
 /// Array of page table entries, as understood by the GSP bootloader.
 #[repr(C)]
+#[derive(FromBytes, IntoBytes)]
 struct PteArray<const NUM_ENTRIES: usize>([u64; NUM_ENTRIES]);
 
-/// SAFETY: arrays of `u64` implement `FromBytes` and we are but a wrapper around one.
-unsafe impl<const NUM_ENTRIES: usize> FromBytes for PteArray<NUM_ENTRIES> {}
-
-/// SAFETY: arrays of `u64` implement `AsBytes` and we are but a wrapper around one.
-unsafe impl<const NUM_ENTRIES: usize> AsBytes for PteArray<NUM_ENTRIES> {}
-
 impl<const NUM_PAGES: usize> PteArray<NUM_PAGES> {
-    /// Returns the page table entry for `index`, for a mapping starting at `start`.
-    // TODO: Replace with `IoView` projection once available.
-    fn entry(start: DmaAddress, index: usize) -> Result<u64> {
-        start
-            .checked_add(num::usize_as_u64(index) << GSP_PAGE_SHIFT)
-            .ok_or(EOVERFLOW)
+    /// Initialize a new page table array mapping `NUM_PAGES` GSP pages starting at address `start`.
+    fn init(view: CoherentView<'_, Self>, start: DmaAddress) -> Result<()> {
+        for i in 0..NUM_PAGES {
+            io_write!(view, .0[build: i],
+                start
+                    .checked_add(num::usize_as_u64(i) << GSP_PAGE_SHIFT)
+                    .ok_or(EOVERFLOW)?
+            );
+        }
+
+        Ok(())
     }
 }
 
@@ -89,17 +124,12 @@ fn new(dev: &device::Device<device::Bound>) -> Result<Self> {
 
         let start_addr = obj.0.dma_handle();
 
-        // SAFETY: `obj` has just been created and we are its sole user.
-        let pte_region = unsafe {
-            &mut obj.0.as_mut()[size_of::<u64>()..][..RM_LOG_BUFFER_NUM_PAGES * size_of::<u64>()]
-        };
-
-        // Write values one by one to avoid an on-stack instance of `PteArray`.
-        for (i, chunk) in pte_region.chunks_exact_mut(size_of::<u64>()).enumerate() {
-            let pte_value = PteArray::<0>::entry(start_addr, i)?;
-
-            chunk.copy_from_slice(&pte_value.to_ne_bytes());
-        }
+        let pte_view = io_project!(
+            obj.0,
+            [build: size_of::<u64>()..][build: ..RM_LOG_BUFFER_NUM_PAGES * size_of::<u64>()]
+        )
+        .try_cast::<PteArray<RM_LOG_BUFFER_NUM_PAGES>>()?;
+        PteArray::init(pte_view, start_addr)?;
 
         Ok(obj)
     }
@@ -185,6 +215,11 @@ pub(crate) fn new(pdev: &pci::Device<device::Bound>) -> impl PinInit<Self, Error
             }))
         })
     }
+
+    /// Query the GSP for the static GPU information.
+    pub(crate) fn get_static_info(&self, bar: Bar0<'_>) -> Result<commands::GetGspStaticInfoReply> {
+        self.cmdq.send_command(bar, commands::GetGspStaticInfo)
+    }
 }
 
 /// Opaque bundle required to unload the GSP. Created by [`Gsp::boot`], consumed by [`Gsp::unload`].
diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs
index 8afb62d689cb4..6b7d002050002 100644
--- a/drivers/gpu/nova-core/gsp/boot.rs
+++ b/drivers/gpu/nova-core/gsp/boot.rs
@@ -3,10 +3,8 @@
 
 use kernel::{
     bits,
-    device,
     dma::Coherent,
     io::poll::read_poll_timeout,
-    pci,
     prelude::*,
     time::Delta,
     types::ScopeGuard, //
@@ -16,7 +14,6 @@
     driver::Bar0,
     falcon::{
         gsp::Gsp,
-        sec2::Sec2,
         Falcon, //
     },
     fb::FbLayout,
@@ -24,7 +21,6 @@
         gsp::GspFirmware,
         FIRMWARE_VERSION, //
     },
-    gpu::Chipset,
     gsp::{
         cmdq::Cmdq,
         commands,
@@ -32,66 +28,6 @@
     },
 };
 
-/// Arguments required to call [`Gsp::unload`](super::Gsp::unload).
-///
-/// Stored as their own type to avoid repeating a long and tedious list in [`BootUnloadGuard`].
-pub(super) struct BootUnloadArgs<'a> {
-    gsp: &'a super::Gsp,
-    dev: &'a device::Device<device::Bound>,
-    bar: Bar0<'a>,
-    gsp_falcon: &'a Falcon<Gsp>,
-    sec2_falcon: &'a Falcon<Sec2>,
-    unload_bundle: Option<super::UnloadBundle>,
-}
-
-/// Guard that calls [`Gsp::unload`](super::Gsp::unload) with a
-/// [`UnloadBundle`](super::UnloadBundle) when dropped.
-///
-/// Used to ensure the `UnloadBundle` is run during failure paths.
-pub(super) struct BootUnloadGuard<'a> {
-    guard: ScopeGuard<BootUnloadArgs<'a>, fn(BootUnloadArgs<'a>)>,
-}
-
-impl<'a> BootUnloadGuard<'a> {
-    /// Wraps `unload_bundle` into a guard that executes it when dropped.
-    pub(super) fn new(
-        gsp: &'a super::Gsp,
-        dev: &'a device::Device<device::Bound>,
-        bar: Bar0<'a>,
-        gsp_falcon: &'a Falcon<Gsp>,
-        sec2_falcon: &'a Falcon<Sec2>,
-        unload_bundle: Option<super::UnloadBundle>,
-    ) -> Self {
-        Self {
-            guard: ScopeGuard::new_with_data(
-                BootUnloadArgs {
-                    gsp,
-                    dev,
-                    bar,
-                    gsp_falcon,
-                    sec2_falcon,
-                    unload_bundle,
-                },
-                |args| {
-                    let _ = super::Gsp::unload(
-                        args.gsp,
-                        args.dev,
-                        args.bar,
-                        args.gsp_falcon,
-                        args.sec2_falcon,
-                        args.unload_bundle,
-                    );
-                },
-            ),
-        }
-    }
-
-    /// Disarms the guard and returns the [`UnloadBundle`](super::UnloadBundle) it contains.
-    pub(super) fn dismiss(self) -> Option<super::UnloadBundle> {
-        self.guard.dismiss().unload_bundle
-    }
-}
-
 impl super::Gsp {
     /// Attempt to boot the GSP.
     ///
@@ -103,12 +39,12 @@ impl super::Gsp {
     /// [`Self::unload`]) returned.
     pub(crate) fn boot(
         self: Pin<&mut Self>,
-        pdev: &pci::Device<device::Bound>,
-        bar: Bar0<'_>,
-        chipset: Chipset,
-        gsp_falcon: &Falcon<Gsp>,
-        sec2_falcon: &Falcon<Sec2>,
+        mut ctx: super::GspBootContext<'_, '_>,
     ) -> Result<Option<super::UnloadBundle>> {
+        let pdev = ctx.pdev;
+        let bar = ctx.bar;
+        let chipset = ctx.chipset;
+        let gsp_falcon = ctx.gsp_falcon;
         let dev = pdev.as_ref();
         let hal = super::hal::gsp_hal(chipset);
 
@@ -120,54 +56,54 @@ pub(crate) fn boot(
         let wpr_meta = Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::new(&gsp_fw, &fb_layout))?;
 
         // Perform the chipset-specific boot sequence, and retrieve the unload bundle.
-        let unload_guard = hal.boot(
-            &self,
-            dev,
-            bar,
-            chipset,
-            &fb_layout,
-            &wpr_meta,
-            gsp_falcon,
-            sec2_falcon,
-        )?;
+        let unload_bundle = hal
+            .boot(&self, &mut ctx, &fb_layout, &wpr_meta)?
+            .or_else(|| {
+                dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n");
+                dev_warn!(
+                    dev,
+                    "The GPU will need to be reset before the driver can bind again.\n"
+                );
+
+                None
+            });
 
-        gsp_falcon.write_os_version(bar, gsp_fw.bootloader.app_version);
+        let mut unload_guard =
+            ScopeGuard::new_with_data((ctx, unload_bundle), |(ctx, unload_bundle)| {
+                let _ = self.unload(ctx, unload_bundle);
+            });
+        let ctx = &mut unload_guard.0;
+
+        gsp_falcon.write_os_version(gsp_fw.bootloader.app_version);
 
         // Poll for RISC-V to become active before continuing.
         read_poll_timeout(
-            || Ok(gsp_falcon.is_riscv_active(bar)),
+            || Ok(gsp_falcon.is_riscv_active()),
             |val: &bool| *val,
             Delta::from_millis(10),
             Delta::from_secs(5),
         )?;
 
-        dev_dbg!(pdev, "RISC-V active? {}\n", gsp_falcon.is_riscv_active(bar),);
+        dev_dbg!(pdev, "RISC-V active? {}\n", gsp_falcon.is_riscv_active(),);
 
         self.cmdq
             .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset))?;
         self.cmdq
-            .send_command_no_wait(bar, commands::SetRegistry::new())?;
+            .send_command_no_wait(bar, commands::SetRegistry::new()?)?;
 
-        hal.post_boot(&self, dev, bar, &gsp_fw, gsp_falcon, sec2_falcon)?;
+        hal.post_boot(&self, ctx, &gsp_fw)?;
 
         // Wait until GSP is fully initialized.
         commands::wait_gsp_init_done(&self.cmdq)?;
 
-        // Obtain and display basic GPU information.
-        let info = self.cmdq.send_command(bar, commands::GetGspStaticInfo)?;
-        match info.gpu_name() {
-            Ok(name) => dev_info!(pdev, "GPU name: {}\n", name),
-            Err(e) => dev_warn!(pdev, "GPU name unavailable: {:?}\n", e),
-        }
-
-        Ok(unload_guard.dismiss())
+        Ok(unload_guard.dismiss().1)
     }
 
     /// Shut down the GSP and wait until it is offline.
     fn shutdown_gsp(
         cmdq: &Cmdq,
         bar: Bar0<'_>,
-        gsp_falcon: &Falcon<Gsp>,
+        gsp_falcon: &Falcon<'_, Gsp>,
         mode: commands::PowerStateLevel,
     ) -> Result {
         // Command to shut the GSP down.
@@ -176,7 +112,7 @@ fn shutdown_gsp(
         // Wait until GSP signals it is suspended.
         const LIBOS_INTERRUPT_PROCESSOR_SUSPENDED: u32 = bits::bit_u32(31);
         read_poll_timeout(
-            || Ok(gsp_falcon.read_mailbox0(bar)),
+            || Ok(gsp_falcon.read_mailbox0()),
             |&mb0| mb0 & LIBOS_INTERRUPT_PROCESSOR_SUSPENDED != 0,
             Delta::from_millis(10),
             Delta::from_secs(5),
@@ -189,17 +125,16 @@ fn shutdown_gsp(
     /// This stops all activity on the GSP.
     pub(crate) fn unload(
         &self,
-        dev: &device::Device<device::Bound>,
-        bar: Bar0<'_>,
-        gsp_falcon: &Falcon<Gsp>,
-        sec2_falcon: &Falcon<Sec2>,
+        mut ctx: super::GspBootContext<'_, '_>,
         unload_bundle: Option<super::UnloadBundle>,
     ) -> Result {
+        let dev = ctx.dev();
+
         // Shut down the GSP. Keep going even in case of error.
         let mut res = Self::shutdown_gsp(
             &self.cmdq,
-            bar,
-            gsp_falcon,
+            ctx.bar,
+            ctx.gsp_falcon,
             commands::PowerStateLevel::Level0,
         )
         .inspect_err(|e| dev_err!(dev, "GSP shutdown failed: {:?}\n", e));
@@ -209,7 +144,7 @@ pub(crate) fn unload(
             res = res.and(
                 unload_bundle
                     .0
-                    .run(dev, bar, gsp_falcon, sec2_falcon)
+                    .run(&mut ctx)
                     .inspect_err(|e| dev_err!(dev, "Unload bundle failed: {:?}\n", e)),
             );
         } else {
diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs
index 070de0731e950..cd844fe48f059 100644
--- a/drivers/gpu/nova-core/gsp/cmdq.rs
+++ b/drivers/gpu/nova-core/gsp/cmdq.rs
@@ -2,16 +2,23 @@
 
 mod continuation;
 
-use core::mem;
+use core::{
+    mem,
+    sync::atomic::{
+        fence,
+        Ordering, //
+    },
+};
 
 use kernel::{
     device,
     dma::{
         Coherent,
+        CoherentBox,
         DmaAddress, //
     },
-    dma_write,
     io::{
+        io_project,
         poll::read_poll_timeout,
         Io, //
     },
@@ -51,10 +58,11 @@
         GSP_PAGE_SIZE, //
     },
     num,
-    regs,
     sbuffer::SBufferIter, //
 };
 
+use super::regs;
+
 /// Marker type representing the absence of a reply for a command. Commands using this as their
 /// reply type are sent using [`Cmdq::send_command_no_wait`].
 pub(crate) struct NoReply;
@@ -171,20 +179,18 @@ struct MsgqData {
 #[repr(C)]
 // There is no struct defined for this in the open-gpu-kernel-source headers.
 // Instead it is defined by code in `GspMsgQueuesInit()`.
-// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module.
-pub(super) struct Msgq {
+struct Msgq {
     /// Header for sending messages, including the write pointer.
-    pub(super) tx: MsgqTxHeader,
+    tx: MsgqTxHeader,
     /// Header for receiving messages, including the read pointer.
-    pub(super) rx: MsgqRxHeader,
+    rx: MsgqRxHeader,
     /// The message queue proper.
     msgq: MsgqData,
 }
 
 /// Structure shared between the driver and the GSP and containing the command and message queues.
 #[repr(C)]
-// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module.
-pub(super) struct GspMem {
+struct GspMem {
     /// Self-mapping page table entries.
     ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>,
     /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the
@@ -192,13 +198,13 @@ pub(super) struct GspMem {
     /// index into the GSP queue.
     ///
     /// This member is read-only for the GSP.
-    pub(super) cpuq: Msgq,
+    cpuq: Msgq,
     /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the
     /// write and read pointers that the GSP updates. This means that the read pointer here is an
     /// index into the CPU queue.
     ///
     /// This member is read-only for the driver.
-    pub(super) gspq: Msgq,
+    gspq: Msgq,
 }
 
 impl GspMem {
@@ -232,20 +238,12 @@ fn new(dev: &device::Device<device::Bound>) -> Result<Self> {
         const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::<Msgq>() }>();
         const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>();
 
-        let gsp_mem = Coherent::<GspMem>::zeroed(dev, GFP_KERNEL)?;
+        let mut gsp_mem = CoherentBox::<GspMem>::zeroed(dev, GFP_KERNEL)?;
+        gsp_mem.cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES);
+        gsp_mem.cpuq.rx = MsgqRxHeader::new();
 
-        let start = gsp_mem.dma_handle();
-        // Write values one by one to avoid an on-stack instance of `PteArray`.
-        for i in 0..GspMem::PTE_ARRAY_SIZE {
-            dma_write!(gsp_mem, .ptes.0[build: i], PteArray::<0>::entry(start, i)?);
-        }
-
-        dma_write!(
-            gsp_mem,
-            .cpuq.tx,
-            MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES)
-        );
-        dma_write!(gsp_mem, .cpuq.rx, MsgqRxHeader::new());
+        let gsp_mem: Coherent<_> = gsp_mem.into();
+        PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_handle())?;
 
         Ok(Self(gsp_mem))
     }
@@ -406,7 +404,7 @@ fn allocate_command(&mut self, size: usize, timeout: Delta) -> Result<GspCommand
     //
     // - The returned value is within `0..MSGQ_NUM_PAGES`.
     fn gsp_write_ptr(&self) -> u32 {
-        super::fw::gsp_mem::gsp_write_ptr(&self.0)
+        MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES
     }
 
     // Returns the index of the memory page the GSP will read the next command from.
@@ -415,7 +413,7 @@ fn gsp_write_ptr(&self) -> u32 {
     //
     // - The returned value is within `0..MSGQ_NUM_PAGES`.
     fn gsp_read_ptr(&self) -> u32 {
-        super::fw::gsp_mem::gsp_read_ptr(&self.0)
+        MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES
     }
 
     // Returns the index of the memory page the CPU can read the next message from.
@@ -424,12 +422,18 @@ fn gsp_read_ptr(&self) -> u32 {
     //
     // - The returned value is within `0..MSGQ_NUM_PAGES`.
     fn cpu_read_ptr(&self) -> u32 {
-        super::fw::gsp_mem::cpu_read_ptr(&self.0)
+        MsgqRxHeader::read_ptr(io_project!(self.0, .cpuq.rx)) % MSGQ_NUM_PAGES
     }
 
     // Informs the GSP that it can send `elem_count` new pages into the message queue.
     fn advance_cpu_read_ptr(&mut self, elem_count: u32) {
-        super::fw::gsp_mem::advance_cpu_read_ptr(&self.0, elem_count)
+        let rx = io_project!(self.0, .cpuq.rx);
+        let rptr = MsgqRxHeader::read_ptr(rx).wrapping_add(elem_count) % MSGQ_NUM_PAGES;
+
+        // Ensure read pointer is properly ordered.
+        fence(Ordering::SeqCst);
+
+        MsgqRxHeader::set_read_ptr(rx, rptr)
     }
 
     // Returns the index of the memory page the CPU can write the next command to.
@@ -438,12 +442,17 @@ fn advance_cpu_read_ptr(&mut self, elem_count: u32) {
     //
     // - The returned value is within `0..MSGQ_NUM_PAGES`.
     fn cpu_write_ptr(&self) -> u32 {
-        super::fw::gsp_mem::cpu_write_ptr(&self.0)
+        MsgqTxHeader::write_ptr(io_project!(self.0, .cpuq.tx)) % MSGQ_NUM_PAGES
     }
 
     // Informs the GSP that it can process `elem_count` new pages from the command queue.
     fn advance_cpu_write_ptr(&mut self, elem_count: u32) {
-        super::fw::gsp_mem::advance_cpu_write_ptr(&self.0, elem_count)
+        let tx = io_project!(self.0, .cpuq.tx);
+        let wptr = MsgqTxHeader::write_ptr(tx).wrapping_add(elem_count) % MSGQ_NUM_PAGES;
+        MsgqTxHeader::set_write_ptr(tx, wptr);
+
+        // Ensure all command data is visible before triggering the GSP read.
+        fence(Ordering::SeqCst);
     }
 }
 
diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs
index f84de9f4f0450..08380de39048a 100644
--- a/drivers/gpu/nova-core/gsp/commands.rs
+++ b/drivers/gpu/nova-core/gsp/commands.rs
@@ -5,6 +5,7 @@
     array,
     convert::Infallible,
     ffi::FromBytesUntilNulError,
+    ops::Range,
     str::Utf8Error, //
 };
 
@@ -66,37 +67,44 @@ struct RegistryEntry {
 
 /// The `SetRegistry` command.
 pub(crate) struct SetRegistry {
-    entries: [RegistryEntry; Self::NUM_ENTRIES],
+    entries: KVec<RegistryEntry>,
 }
 
 impl SetRegistry {
-    // For now we hard-code the registry entries. Future work will allow others to
-    // be added as module parameters.
-    const NUM_ENTRIES: usize = 3;
-
     /// Creates a new `SetRegistry` command, using a set of hardcoded entries.
-    pub(crate) fn new() -> Self {
-        Self {
-            entries: [
-                // RMSecBusResetEnable - enables PCI secondary bus reset
-                RegistryEntry {
-                    key: "RMSecBusResetEnable",
-                    value: 1,
-                },
-                // RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration registers on
-                // any PCI reset.
-                RegistryEntry {
-                    key: "RMForcePcieConfigSave",
-                    value: 1,
-                },
-                // RMDevidCheckIgnore - allows GSP-RM to boot even if the PCI dev ID is not found
-                // in the internal product name database.
-                RegistryEntry {
-                    key: "RMDevidCheckIgnore",
-                    value: 1,
-                },
-            ],
-        }
+    pub(crate) fn new() -> Result<Self> {
+        let mut entries = KVec::new();
+
+        // RMSecBusResetEnable - enables PCI secondary bus reset
+        entries.push(
+            RegistryEntry {
+                key: "RMSecBusResetEnable",
+                value: 1,
+            },
+            GFP_KERNEL,
+        )?;
+
+        // RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration registers on
+        // any PCI reset.
+        entries.push(
+            RegistryEntry {
+                key: "RMForcePcieConfigSave",
+                value: 1,
+            },
+            GFP_KERNEL,
+        )?;
+
+        // RMDevidCheckIgnore - allows GSP-RM to boot even if the PCI dev ID is not found
+        // in the internal product name database.
+        entries.push(
+            RegistryEntry {
+                key: "RMDevidCheckIgnore",
+                value: 1,
+            },
+            GFP_KERNEL,
+        )?;
+
+        Ok(Self { entries })
     }
 }
 
@@ -107,15 +115,18 @@ impl CommandToGsp for SetRegistry {
     type InitError = Infallible;
 
     fn init(&self) -> impl Init<Self::Command, Self::InitError> {
-        Self::Command::init(Self::NUM_ENTRIES as u32, self.variable_payload_len() as u32)
+        Self::Command::init(
+            self.entries.len() as u32,
+            self.variable_payload_len() as u32,
+        )
     }
 
     fn variable_payload_len(&self) -> usize {
         let mut key_size = 0;
-        for i in 0..Self::NUM_ENTRIES {
-            key_size += self.entries[i].key.len() + 1; // +1 for NULL terminator
+        for entry in self.entries.iter() {
+            key_size += entry.key.len() + 1; // +1 for NULL terminator
         }
-        Self::NUM_ENTRIES * size_of::<fw::commands::PackedRegistryEntry>() + key_size
+        self.entries.len() * size_of::<fw::commands::PackedRegistryEntry>() + key_size
     }
 
     fn init_variable_payload(
@@ -123,12 +134,12 @@ fn init_variable_payload(
         dst: &mut SBufferIter<core::array::IntoIter<&mut [u8], 2>>,
     ) -> Result {
         let string_data_start_offset = size_of::<Self::Command>()
-            + Self::NUM_ENTRIES * size_of::<fw::commands::PackedRegistryEntry>();
+            + self.entries.len() * size_of::<fw::commands::PackedRegistryEntry>();
 
         // Array for string data.
         let mut string_data = KVec::new();
 
-        for entry in self.entries.iter().take(Self::NUM_ENTRIES) {
+        for entry in self.entries.iter() {
             dst.write_all(
                 fw::commands::PackedRegistryEntry::new(
                     (string_data_start_offset + string_data.len()) as u32,
@@ -191,22 +202,30 @@ fn init(&self) -> impl Init<Self::Command, Self::InitError> {
     }
 }
 
-/// The reply from the GSP to the [`GetGspInfo`] command.
+/// The reply from the GSP to the [`GetGspStaticInfo`] command.
 pub(crate) struct GetGspStaticInfoReply {
     gpu_name: [u8; 64],
+    /// Usable FB (VRAM) regions for driver memory allocation.
+    pub(crate) usable_fb_regions: KVec<Range<u64>>,
 }
 
 impl MessageFromGsp for GetGspStaticInfoReply {
     const FUNCTION: MsgFunction = MsgFunction::GetGspStaticInfo;
     type Message = fw::commands::GspStaticConfigInfo;
-    type InitError = Infallible;
+    type InitError = Error;
 
     fn read(
         msg: &Self::Message,
         _sbuffer: &mut SBufferIter<array::IntoIter<&[u8], 2>>,
     ) -> Result<Self, Self::InitError> {
+        let mut usable_fb_regions = KVec::new();
+        for region in msg.usable_fb_regions() {
+            usable_fb_regions.push(region, GFP_KERNEL)?;
+        }
+
         Ok(GetGspStaticInfoReply {
             gpu_name: msg.gpu_name_str(),
+            usable_fb_regions,
         })
     }
 }
diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs
index 4db0cfa4dc4d1..166b0c69edb7f 100644
--- a/drivers/gpu/nova-core/gsp/fw.rs
+++ b/drivers/gpu/nova-core/gsp/fw.rs
@@ -10,7 +10,15 @@
 use core::ops::Range;
 
 use kernel::{
-    dma::Coherent,
+    bitfield,
+    dma::{
+        Coherent,
+        CoherentView, //
+    },
+    io::{
+        io_read,
+        io_write, //
+    },
     prelude::*,
     ptr::{
         Alignable,
@@ -44,59 +52,6 @@
     },
 };
 
-// TODO: Replace with `IoView` projections once available.
-pub(super) mod gsp_mem {
-    use core::sync::atomic::{
-        fence,
-        Ordering, //
-    };
-
-    use kernel::{
-        dma::Coherent,
-        dma_read,
-        dma_write, //
-    };
-
-    use crate::gsp::cmdq::{
-        GspMem,
-        MSGQ_NUM_PAGES, //
-    };
-
-    pub(in crate::gsp) fn gsp_write_ptr(qs: &Coherent<GspMem>) -> u32 {
-        dma_read!(qs, .gspq.tx.0.writePtr) % MSGQ_NUM_PAGES
-    }
-
-    pub(in crate::gsp) fn gsp_read_ptr(qs: &Coherent<GspMem>) -> u32 {
-        dma_read!(qs, .gspq.rx.0.readPtr) % MSGQ_NUM_PAGES
-    }
-
-    pub(in crate::gsp) fn cpu_read_ptr(qs: &Coherent<GspMem>) -> u32 {
-        dma_read!(qs, .cpuq.rx.0.readPtr) % MSGQ_NUM_PAGES
-    }
-
-    pub(in crate::gsp) fn advance_cpu_read_ptr(qs: &Coherent<GspMem>, count: u32) {
-        let rptr = cpu_read_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES;
-
-        // Ensure read pointer is properly ordered.
-        fence(Ordering::SeqCst);
-
-        dma_write!(qs, .cpuq.rx.0.readPtr, rptr);
-    }
-
-    pub(in crate::gsp) fn cpu_write_ptr(qs: &Coherent<GspMem>) -> u32 {
-        dma_read!(qs, .cpuq.tx.0.writePtr) % MSGQ_NUM_PAGES
-    }
-
-    pub(in crate::gsp) fn advance_cpu_write_ptr(qs: &Coherent<GspMem>, count: u32) {
-        let wptr = cpu_write_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES;
-
-        dma_write!(qs, .cpuq.tx.0.writePtr, wptr);
-
-        // Ensure all command data is visible before triggering the GSP read.
-        fence(Ordering::SeqCst);
-    }
-}
-
 /// Maximum size of a single GSP message queue element in bytes.
 pub(crate) const GSP_MSG_QUEUE_ELEMENT_SIZE_MAX: usize =
     num::u32_as_usize(bindings::GSP_MSG_QUEUE_ELEMENT_SIZE_MAX);
@@ -219,7 +174,6 @@ pub(crate) fn new<'a>(
         gsp_firmware: &'a GspFirmware,
         fb_layout: &'a FbLayout,
     ) -> impl Init<Self> + 'a {
-        #[allow(non_snake_case)]
         let init_inner = init!(bindings::GspFwWprMeta {
             // CAST: we want to store the bits of `GSP_FW_WPR_META_MAGIC` unmodified.
             magic: bindings::GSP_FW_WPR_META_MAGIC as u64,
@@ -674,7 +628,6 @@ fn id8(name: &str) -> u64 {
             u64::from_ne_bytes(bytes)
         }
 
-        #[allow(non_snake_case)]
         let init_inner = init!(bindings::LibosMemoryRegionInitArgument {
             id8: id8(name),
             pa: obj.dma_handle(),
@@ -720,6 +673,16 @@ pub(crate) fn new(msgq_size: u32, rx_hdr_offset: u32, msg_count: u32) -> Self {
             entryOff: num::usize_into_u32::<GSP_PAGE_SIZE>(),
         })
     }
+
+    /// Returns the value of the write pointer for this queue.
+    pub(crate) fn write_ptr(this: CoherentView<'_, Self>) -> u32 {
+        io_read!(this, .0.writePtr)
+    }
+
+    /// Sets the value of the write pointer for this queue.
+    pub(crate) fn set_write_ptr(this: CoherentView<'_, Self>, val: u32) {
+        io_write!(this, .0.writePtr, val)
+    }
 }
 
 // SAFETY: Padding is explicit and does not contain uninitialized data.
@@ -735,6 +698,16 @@ impl MsgqRxHeader {
     pub(crate) fn new() -> Self {
         Self(Default::default())
     }
+
+    /// Returns the value of the read pointer for this queue.
+    pub(crate) fn read_ptr(this: CoherentView<'_, Self>) -> u32 {
+        io_read!(this, .0.readPtr)
+    }
+
+    /// Sets the value of the read pointer for this queue.
+    pub(crate) fn set_read_ptr(this: CoherentView<'_, Self>, val: u32) {
+        io_write!(this, .0.readPtr, val)
+    }
 }
 
 // SAFETY: Padding is explicit and does not contain uninitialized data.
@@ -742,8 +715,8 @@ unsafe impl AsBytes for MsgqRxHeader {}
 
 bitfield! {
     struct MsgHeaderVersion(u32) {
-        31:24 major as u8;
-        23:16 minor as u8;
+        31:24 major;
+        23:16 minor;
     }
 }
 
@@ -752,9 +725,9 @@ impl MsgHeaderVersion {
     const MINOR_TOT: u8 = 0;
 
     fn new() -> Self {
-        Self::default()
-            .set_major(Self::MAJOR_TOT)
-            .set_minor(Self::MINOR_TOT)
+        Self::zeroed()
+            .with_major(Self::MAJOR_TOT)
+            .with_minor(Self::MINOR_TOT)
     }
 }
 
@@ -793,7 +766,6 @@ impl GspMsgElement {
     /// * `sequence` - Sequence number of the message.
     /// * `cmd_size` - Size of the command (not including the message element), in bytes.
     /// * `function` - Function of the message.
-    #[allow(non_snake_case)]
     pub(crate) fn init(
         sequence: u32,
         cmd_size: usize,
@@ -876,7 +848,6 @@ pub(crate) struct GspArgumentsCached {
 impl GspArgumentsCached {
     /// Creates the arguments for starting the GSP up using `cmdq` as its command queue.
     pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ {
-        #[allow(non_snake_case)]
         let init_inner = init!(bindings::GSP_ARGUMENTS_CACHED {
             messageQueueInitArguments <- MessageQueueInitArguments::new(cmdq),
             bDmemStack: 1,
@@ -923,7 +894,6 @@ unsafe impl FromBytes for GspArgumentsPadded {}
 
 impl MessageQueueInitArguments {
     /// Creates a new init arguments structure for `cmdq`.
-    #[allow(non_snake_case)]
     fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ {
         init!(MessageQueueInitArguments {
             sharedMemPhysAddr: cmdq.dma_handle,
@@ -947,7 +917,6 @@ pub(crate) enum GspDmaTarget {
 
 impl GspAcrBootGspRmParams {
     fn new(target: GspDmaTarget, wpr_meta_addr: u64) -> impl Init<Self> {
-        #[allow(non_snake_case)]
         let params = init!(Self {
             target: target as u32,
             gspRmDescSize: num::usize_into_u32::<{ size_of::<GspFwWprMeta>() }>(),
@@ -966,7 +935,6 @@ fn new(target: GspDmaTarget, wpr_meta_addr: u64) -> impl Init<Self> {
 
 impl GspRmParams {
     fn new(target: GspDmaTarget, libos_addr: u64) -> impl Init<Self> {
-        #[allow(non_snake_case)]
         let params = init!(Self {
             target: target as u32,
             bootArgsOffset: libos_addr,
@@ -986,7 +954,6 @@ unsafe impl FromBytes for GspFmcBootParams {}
 
 impl GspFmcBootParams {
     pub(crate) fn new(wpr_meta_addr: u64, libos_addr: u64) -> impl Init<Self> {
-        #[allow(non_snake_case)]
         let init = init!(Self {
             // Blackwell FSP obtains WPR info from other sources, so
             // wprCarveoutOffset and wprCarveoutSize are left zero.
diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs
index 7bcc41fc7fa07..6dc31d1bf5ae4 100644
--- a/drivers/gpu/nova-core/gsp/fw/commands.rs
+++ b/drivers/gpu/nova-core/gsp/fw/commands.rs
@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: GPL-2.0
 // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
+use core::ops::Range;
+
 use kernel::{
     device,
     pci,
@@ -13,7 +15,8 @@
 
 use crate::{
     gpu::Chipset,
-    gsp::GSP_PAGE_SIZE, //
+    gsp::GSP_PAGE_SIZE,
+    num::IntoSafeCast, //
 };
 
 use super::bindings;
@@ -27,7 +30,6 @@ pub(crate) struct GspSetSystemInfo {
 
 impl GspSetSystemInfo {
     /// Returns an in-place initializer for the `GspSetSystemInfo` command.
-    #[allow(non_snake_case)]
     pub(crate) fn init<'a>(
         dev: &'a pci::Device<device::Bound>,
         chipset: Chipset,
@@ -99,7 +101,6 @@ pub(crate) struct PackedRegistryTable {
 }
 
 impl PackedRegistryTable {
-    #[allow(non_snake_case)]
     pub(crate) fn init(num_entries: u32, size: u32) -> impl Init<Self> {
         type InnerPackedRegistryTable = bindings::PACKED_REGISTRY_TABLE;
         let init_inner = init!(InnerPackedRegistryTable {
@@ -129,6 +130,41 @@ impl GspStaticConfigInfo {
     pub(crate) fn gpu_name_str(&self) -> [u8; 64] {
         self.0.gpuNameString
     }
+
+    /// Returns an iterator over valid FB regions from GSP firmware data.
+    fn fb_regions(
+        &self,
+    ) -> impl Iterator<Item = &bindings::NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO> {
+        let fb_info = &self.0.fbRegionInfoParams;
+        fb_info
+            .fbRegion
+            .iter()
+            .take(fb_info.numFBRegions.into_safe_cast())
+            .filter(|reg| reg.limit >= reg.base)
+    }
+
+    /// Iterates over usable FB regions from GSP firmware data.
+    ///
+    /// Each yielded region is a [`Range<u64>`] suitable for driver memory allocation.
+    /// Usable regions are those that satisfy all the following properties:
+    /// - Are not reserved for firmware internal use.
+    /// - Are not protected (hardware-enforced access restrictions).
+    /// - Support compression (can use GPU memory compression for bandwidth).
+    /// - Support ISO (isochronous memory for display requiring guaranteed bandwidth).
+    pub(crate) fn usable_fb_regions(&self) -> impl Iterator<Item = Range<u64>> + '_ {
+        self.fb_regions().filter_map(|reg| {
+            // Filter: not reserved, not protected, supports compression and ISO.
+            if reg.reserved == 0
+                && reg.bProtected == 0
+                && reg.supportCompressed != 0
+                && reg.supportISO != 0
+            {
+                reg.limit.checked_add(1).map(|end| reg.base..end)
+            } else {
+                None
+            }
+        })
+    }
 }
 
 // SAFETY: Padding is explicit and will not contain uninitialized data.
diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs
index 04f004856c605..34b4bb82a999d 100644
--- a/drivers/gpu/nova-core/gsp/hal.rs
+++ b/drivers/gpu/nova-core/gsp/hal.rs
@@ -1,23 +1,16 @@
 // SPDX-License-Identifier: GPL-2.0
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
+mod ga102;
 mod gh100;
 mod tu102;
 
-use kernel::prelude::*;
-
 use kernel::{
-    device,
-    dma::Coherent, //
+    dma::Coherent,
+    prelude::*, //
 };
 
 use crate::{
-    driver::Bar0,
-    falcon::{
-        gsp::Gsp as GspEngine,
-        sec2::Sec2,
-        Falcon, //
-    },
     fb::FbLayout,
     firmware::gsp::GspFirmware,
     gpu::{
@@ -25,8 +18,8 @@
         Chipset, //
     },
     gsp::{
-        boot::BootUnloadGuard,
         Gsp,
+        GspBootContext,
         GspFwWprMeta, //
     },
 };
@@ -38,33 +31,22 @@
 /// required for unloading is prepared at load time, and stored here until it needs to be run.
 pub(super) trait UnloadBundle: Send {
     /// Performs the steps required to properly reset the GSP after it has been stopped.
-    fn run(
-        &self,
-        dev: &device::Device<device::Bound>,
-        bar: Bar0<'_>,
-        gsp_falcon: &Falcon<GspEngine>,
-        sec2_falcon: &Falcon<Sec2>,
-    ) -> Result;
+    fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result;
 }
 
 /// Trait implemented by GSP HALs.
 pub(super) trait GspHal: Send {
     /// Performs the GSP boot process, loading and running the required firmwares as needed.
     ///
-    /// Upon success, returns a guard that runs the GSP unload sequence if GSP boot does not
-    /// complete.
-    #[allow(clippy::too_many_arguments)]
-    fn boot<'a>(
+    /// Upon success, returns the [`crate::gsp::UnloadBundle`] to use with [`Gsp::unload`], if one
+    /// could be created.
+    fn boot(
         &self,
-        gsp: &'a Gsp,
-        dev: &'a device::Device<device::Bound>,
-        bar: Bar0<'a>,
-        chipset: Chipset,
+        gsp: &Gsp,
+        ctx: &mut GspBootContext<'_, '_>,
         fb_layout: &FbLayout,
         wpr_meta: &Coherent<GspFwWprMeta>,
-        gsp_falcon: &'a Falcon<GspEngine>,
-        sec2_falcon: &'a Falcon<Sec2>,
-    ) -> Result<BootUnloadGuard<'a>>;
+    ) -> Result<Option<crate::gsp::UnloadBundle>>;
 
     /// Performs HAL-specific post-GSP boot tasks.
     ///
@@ -73,20 +55,38 @@ fn boot<'a>(
     fn post_boot(
         &self,
         _gsp: &Gsp,
-        _dev: &device::Device<device::Bound>,
-        _bar: Bar0<'_>,
+        _ctx: &mut GspBootContext<'_, '_>,
         _gsp_fw: &GspFirmware,
-        _gsp_falcon: &Falcon<GspEngine>,
-        _sec2_falcon: &Falcon<Sec2>,
     ) -> Result {
         Ok(())
     }
 }
 
+/// Returns the names of the firmware files required to boot the GSP of `chipset`, in addition to
+/// the "bootloader" and "gsp" images required by all chipsets.
+pub(crate) const fn boot_firmware_files(chipset: Chipset) -> &'static [&'static str] {
+    match chipset.arch() {
+        // Turing chipsets boot the GSP via the SEC2 Booter, and require the FWSEC bootloader.
+        Architecture::Turing => &["booter_load", "booter_unload", "gen_bootloader"],
+        // GA100 also requires the FWSEC bootloader.
+        Architecture::Ampere if matches!(chipset, Chipset::GA100) => {
+            &["booter_load", "booter_unload", "gen_bootloader"]
+        }
+        // Other Ampere chipsets, as well as Ada chipsets, run FWSEC directly.
+        Architecture::Ampere | Architecture::Ada => &["booter_load", "booter_unload"],
+        // Hopper and later chipsets boot the GSP via the FMC image loaded by FSP.
+        Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => {
+            &["fmc"]
+        }
+    }
+}
+
 /// Returns the GSP HAL to be used for `chipset`.
 pub(super) fn gsp_hal(chipset: Chipset) -> &'static dyn GspHal {
     match chipset.arch() {
-        Architecture::Turing | Architecture::Ampere | Architecture::Ada => tu102::TU102_HAL,
+        Architecture::Turing => tu102::TU102_HAL,
+        Architecture::Ampere if matches!(chipset, Chipset::GA100) => tu102::TU102_HAL,
+        Architecture::Ampere | Architecture::Ada => ga102::GA102_HAL,
         Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => {
             gh100::GH100_HAL
         }
diff --git a/drivers/gpu/nova-core/gsp/hal/ga102.rs b/drivers/gpu/nova-core/gsp/hal/ga102.rs
new file mode 100644
index 0000000000000..ceb3eb39d138b
--- /dev/null
+++ b/drivers/gpu/nova-core/gsp/hal/ga102.rs
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+use crate::gsp::hal::{
+    tu102::Tu102,
+    GspHal, //
+};
+
+/// The GA102 HAL is like the TU102 one, except it doesn't use the bootloader.
+const GA102: Tu102 = Tu102 {
+    needs_fwsec_bootloader: false,
+};
+
+pub(super) const GA102_HAL: &dyn GspHal = &GA102;
diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs
index 98f5ce197d13b..ad04904b5af41 100644
--- a/drivers/gpu/nova-core/gsp/hal/gh100.rs
+++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs
@@ -7,33 +7,24 @@
     device,
     dma::Coherent,
     io::poll::read_poll_timeout,
-    time::Delta, //
+    time::Delta,
+    types::ScopeGuard, //
 };
 
 use crate::{
-    driver::Bar0,
     falcon::{
         gsp::Gsp as GspEngine,
-        sec2::Sec2,
         Falcon, //
     },
     fb::FbLayout,
-    firmware::{
-        fsp::FspFirmware,
-        FIRMWARE_VERSION, //
-    },
-    fsp::{
-        FmcBootArgs,
-        Fsp, //
-    },
-    gpu::Chipset,
+    fsp::FmcBootArgs,
     gsp::{
-        boot::BootUnloadGuard,
         hal::{
             GspHal,
             UnloadBundle, //
         },
         Gsp,
+        GspBootContext,
         GspFwWprMeta, //
     },
 };
@@ -46,10 +37,10 @@ struct GspMbox {
 
 impl GspMbox {
     /// Reads both mailboxes from the GSP falcon.
-    fn read(gsp_falcon: &Falcon<GspEngine>, bar: Bar0<'_>) -> Self {
+    fn read(gsp_falcon: &Falcon<'_, GspEngine>) -> Self {
         Self {
-            mbox0: gsp_falcon.read_mailbox0(bar),
-            mbox1: gsp_falcon.read_mailbox1(bar),
+            mbox0: gsp_falcon.read_mailbox0(),
+            mbox1: gsp_falcon.read_mailbox1(),
         }
     }
 
@@ -64,8 +55,7 @@ fn combined_addr(&self) -> u64 {
     /// either condition should stop the poll loop.
     fn lockdown_released_or_error(
         &self,
-        gsp_falcon: &Falcon<GspEngine>,
-        bar: Bar0<'_>,
+        gsp_falcon: &Falcon<'_, GspEngine>,
         fmc_boot_params_addr: u64,
     ) -> bool {
         // GSP-FMC normally clears the boot parameters address from the mailboxes early during
@@ -75,15 +65,14 @@ fn lockdown_released_or_error(
             return self.combined_addr() != fmc_boot_params_addr;
         }
 
-        !gsp_falcon.riscv_branch_privilege_lockdown(bar)
+        !gsp_falcon.riscv_branch_privilege_lockdown()
     }
 }
 
 /// Waits for GSP lockdown to be released after FSP Chain of Trust.
 fn wait_for_gsp_lockdown_release(
     dev: &device::Device<device::Bound>,
-    bar: Bar0<'_>,
-    gsp_falcon: &Falcon<GspEngine>,
+    gsp_falcon: &Falcon<'_, GspEngine>,
     fmc_boot_params_addr: u64,
 ) -> Result {
     dev_dbg!(dev, "Waiting for GSP lockdown release\n");
@@ -92,14 +81,14 @@ fn wait_for_gsp_lockdown_release(
         || {
             // While the PRIV target mask is still locked to FSP, GSP register and mailbox reads
             // are not meaningful. Wait until HWCFG2 says the CPU can read them.
-            Ok(match gsp_falcon.priv_target_mask_released(bar) {
+            Ok(match gsp_falcon.priv_target_mask_released() {
                 false => None,
-                true => Some(GspMbox::read(gsp_falcon, bar)),
+                true => Some(GspMbox::read(gsp_falcon)),
             })
         },
         |mbox| match mbox {
             None => false,
-            Some(mbox) => mbox.lockdown_released_or_error(gsp_falcon, bar, fmc_boot_params_addr),
+            Some(mbox) => mbox.lockdown_released_or_error(gsp_falcon, fmc_boot_params_addr),
         },
         Delta::from_millis(10),
         Delta::from_secs(30),
@@ -123,22 +112,16 @@ fn wait_for_gsp_lockdown_release(
 struct FspUnloadBundle;
 
 impl UnloadBundle for FspUnloadBundle {
-    fn run(
-        &self,
-        dev: &device::Device<device::Bound>,
-        bar: Bar0<'_>,
-        gsp_falcon: &Falcon<GspEngine>,
-        _sec2_falcon: &Falcon<Sec2>,
-    ) -> Result {
+    fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result {
         // GSP falcon does most of the work of resetting, so just wait for it to finish.
         read_poll_timeout(
-            || Ok(gsp_falcon.is_riscv_active(bar)),
+            || Ok(ctx.gsp_falcon.is_riscv_active()),
             |&active| !active,
             Delta::from_millis(10),
             Delta::from_secs(5),
         )
         .map(|_| ())
-        .inspect_err(|_| dev_err!(dev, "GSP falcon failed to halt\n"))
+        .inspect_err(|_| dev_err!(ctx.dev(), "GSP falcon failed to halt\n"))
     }
 }
 
@@ -149,29 +132,21 @@ impl GspHal for Gh100 {
     ///
     /// This path uses FSP to establish a chain of trust and boot GSP-FMC. FSP handles
     /// the GSP boot internally - no manual GSP reset/boot is needed.
-    fn boot<'a>(
+    fn boot(
         &self,
-        gsp: &'a Gsp,
-        dev: &'a device::Device<device::Bound>,
-        bar: Bar0<'a>,
-        chipset: Chipset,
+        gsp: &Gsp,
+        ctx: &mut GspBootContext<'_, '_>,
         fb_layout: &FbLayout,
         wpr_meta: &Coherent<GspFwWprMeta>,
-        gsp_falcon: &'a Falcon<GspEngine>,
-        sec2_falcon: &'a Falcon<Sec2>,
-    ) -> Result<BootUnloadGuard<'a>> {
-        let fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?;
+    ) -> Result<Option<crate::gsp::UnloadBundle>> {
+        let dev = ctx.dev();
+        let chipset = ctx.chipset;
+        let gsp_falcon = ctx.gsp_falcon;
 
         let unload_bundle = crate::gsp::UnloadBundle(
             KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox<dyn UnloadBundle>
         );
 
-        // Wrap the unload bundle into a drop guard so it is automatically run upon failure.
-        let unload_guard =
-            BootUnloadGuard::new(gsp, dev, bar, gsp_falcon, sec2_falcon, Some(unload_bundle));
-
-        let mut fsp = Fsp::wait_secure_boot(dev, bar, chipset, fsp_fw)?;
-
         let args = FmcBootArgs::new(
             dev,
             chipset,
@@ -180,11 +155,23 @@ fn boot<'a>(
             false,
         )?;
 
-        fsp.boot_fmc(dev, bar, fb_layout, &args)?;
+        // Wait for the GSP RISC-V core to halt in case of error. We create this guard after `args`
+        // to make sure that boot args are kept alive until halt, in case they are still being
+        // accessed.
+        let mut unload_guard =
+            ScopeGuard::new_with_data((unload_bundle, ctx), |(unload_bundle, ctx)| {
+                let _ = unload_bundle.0.run(ctx);
+            });
+
+        let fsp = unload_guard.1.fsp.as_mut().ok_or(ENODEV)?;
+
+        fsp.boot_fmc(dev, fb_layout, &args)?;
 
-        wait_for_gsp_lockdown_release(dev, bar, gsp_falcon, args.boot_params_dma_handle())?;
+        // Wait for GSP-FMC to release the GSP lockdown, indicating that `args` is not accessed
+        // anymore.
+        wait_for_gsp_lockdown_release(dev, gsp_falcon, args.boot_params_dma_handle())?;
 
-        Ok(unload_guard)
+        Ok(Some(unload_guard.dismiss().0))
     }
 }
 
diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs
index 2f6301af71131..29bb17171f56e 100644
--- a/drivers/gpu/nova-core/gsp/hal/tu102.rs
+++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs
@@ -6,7 +6,8 @@
 use kernel::{
     device,
     dma::Coherent,
-    io::Io, //
+    io::Io,
+    types::ScopeGuard, //
 };
 
 use crate::{
@@ -32,16 +33,13 @@
     },
     gpu::Chipset,
     gsp::{
-        boot::BootUnloadGuard,
         hal::{
             GspHal,
             UnloadBundle, //
         },
-        sequencer::{
-            GspSequencer,
-            GspSequencerParams, //
-        },
+        sequencer::GspSequencer,
         Gsp,
+        GspBootContext,
         GspFwWprMeta, //
     },
     regs,
@@ -58,32 +56,15 @@ enum FwsecUnloadFirmware {
 }
 
 impl FwsecUnloadFirmware {
-    /// Loads the FWSEC SB firmware, as well as its bootloader if `chipset` requires it.
-    fn new(
-        dev: &device::Device<device::Bound>,
-        bar: Bar0<'_>,
-        chipset: Chipset,
-        bios: &Vbios,
-        gsp_falcon: &Falcon<GspEngine>,
-    ) -> Result<Self> {
-        let fwsec_sb = FwsecFirmware::new(dev, gsp_falcon, bar, bios, FwsecCommand::Sb)?;
-
-        Ok(if chipset.needs_fwsec_bootloader() {
-            Self::WithBl(FwsecFirmwareWithBl::new(fwsec_sb, dev, chipset)?)
-        } else {
-            Self::WithoutBl(fwsec_sb)
-        })
-    }
-
     /// Runs the FWSEC SB firmware.
     fn run(
         &self,
         dev: &device::Device<device::Bound>,
         bar: Bar0<'_>,
-        gsp_falcon: &Falcon<GspEngine>,
+        gsp_falcon: &Falcon<'_, GspEngine>,
     ) -> Result {
         match self {
-            Self::WithoutBl(fw) => fw.run(dev, gsp_falcon, bar),
+            Self::WithoutBl(fw) => fw.run(dev, gsp_falcon),
             Self::WithBl(fw) => fw.run(dev, gsp_falcon, bar),
         }
     }
@@ -96,56 +77,33 @@ struct Sec2UnloadBundle {
     booter_unloader: BooterFirmware,
 }
 
-impl Sec2UnloadBundle {
-    /// Load and prepare the resources required to properly reset the GSP after it has been stopped.
-    fn build(
-        dev: &device::Device<device::Bound>,
-        bar: Bar0<'_>,
-        chipset: Chipset,
-        bios: &Vbios,
-        gsp_falcon: &Falcon<GspEngine>,
-        sec2_falcon: &Falcon<Sec2>,
-    ) -> Result<KBox<dyn UnloadBundle>> {
-        KBox::new(
-            Self {
-                fwsec_sb: FwsecUnloadFirmware::new(dev, bar, chipset, bios, gsp_falcon)?,
-                booter_unloader: BooterFirmware::new(
-                    dev,
-                    BooterKind::Unloader,
-                    chipset,
-                    FIRMWARE_VERSION,
-                    sec2_falcon,
-                    bar,
-                )?,
-            },
-            GFP_KERNEL,
-        )
-        .map(|b| b as KBox<dyn UnloadBundle>)
-        .map_err(Into::into)
-    }
-}
-
 impl UnloadBundle for Sec2UnloadBundle {
-    fn run(
-        &self,
-        dev: &device::Device<device::Bound>,
-        bar: Bar0<'_>,
-        gsp_falcon: &Falcon<GspEngine>,
-        sec2_falcon: &Falcon<Sec2>,
-    ) -> Result {
+    fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result {
+        let dev = ctx.dev();
+        let bar = ctx.bar;
+
         // Run FWSEC-SB to reset the GSP falcon to its pre-libos state.
-        self.fwsec_sb.run(dev, bar, gsp_falcon)?;
+        // Log errors but keep going if it fails.
+        let fwsec_sb_res = self
+            .fwsec_sb
+            .run(dev, bar, ctx.gsp_falcon)
+            .inspect_err(|e| dev_err!(dev, "FWSEC-SB failed to run: {:?}\n", e));
 
         // Remove WPR2 region if set.
         let wpr2_hi = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI);
-        if wpr2_hi.is_wpr2_set() {
-            sec2_falcon.reset(bar)?;
-            sec2_falcon.load(dev, bar, &self.booter_unloader)?;
+        let booter_unloader_res = (|| {
+            if !wpr2_hi.is_wpr2_set() {
+                return Ok(());
+            }
+
+            ctx.sec2_falcon.reset()?;
+            ctx.sec2_falcon.load(&self.booter_unloader)?;
 
             // Sentinel value to confirm that Booter Unloader has run.
             const MAILBOX_SENTINEL: u32 = 0xff;
-            let (mbox0, _) =
-                sec2_falcon.boot(bar, Some(MAILBOX_SENTINEL), Some(MAILBOX_SENTINEL))?;
+            let (mbox0, _) = ctx
+                .sec2_falcon
+                .boot(Some(MAILBOX_SENTINEL), Some(MAILBOX_SENTINEL))?;
             if mbox0 != 0 {
                 dev_err!(dev, "Booter Unloader returned error 0x{:x}\n", mbox0);
                 return Err(EINVAL);
@@ -160,147 +118,183 @@ fn run(
                 );
                 return Err(EBUSY);
             }
-        }
 
-        Ok(())
+            Ok(())
+        })()
+        .inspect_err(|e| dev_err!(dev, "Booter Unloader failed to run: {:?}\n", e));
+
+        fwsec_sb_res.and(booter_unloader_res)
     }
 }
 
-/// Helper function to load and run the FWSEC-FRTS firmware and confirm that it has properly
-/// created the WPR2 region.
-fn run_fwsec_frts(
-    dev: &device::Device<device::Bound>,
-    chipset: Chipset,
-    falcon: &Falcon<GspEngine>,
-    bar: Bar0<'_>,
-    bios: &Vbios,
-    fb_layout: &FbLayout,
-) -> Result {
-    // Check that the WPR2 region does not already exist - if it does, we cannot run
-    // FWSEC-FRTS until the GPU is reset.
-    if bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound() != 0 {
-        dev_err!(
-            dev,
-            "WPR2 region already exists - GPU needs to be reset to proceed\n"
-        );
-        return Err(EBUSY);
-    }
+pub(super) struct Tu102 {
+    /// If `true`, then the FWSEC-FRTS bootloader will be used to load the actual firmware.
+    pub(super) needs_fwsec_bootloader: bool,
+}
 
-    // FWSEC-FRTS will create the WPR2 region.
-    let fwsec_frts = FwsecFirmware::new(
-        dev,
-        falcon,
-        bar,
-        bios,
-        FwsecCommand::Frts {
-            frts_addr: fb_layout.frts.start,
-            frts_size: fb_layout.frts.len(),
-        },
-    )?;
-
-    if chipset.needs_fwsec_bootloader() {
-        let fwsec_frts_bl = FwsecFirmwareWithBl::new(fwsec_frts, dev, chipset)?;
-        // Load and run the bootloader, which will load FWSEC-FRTS and run it.
-        fwsec_frts_bl.run(dev, falcon, bar)?;
-    } else {
-        // Load and run FWSEC-FRTS directly.
-        fwsec_frts.run(dev, falcon, bar)?;
-    }
+impl Tu102 {
+    /// Helper method to load and run the FWSEC-FRTS firmware and confirm that it has properly
+    /// created the WPR2 region.
+    fn run_fwsec_frts(
+        &self,
+        dev: &device::Device<device::Bound>,
+        chipset: Chipset,
+        falcon: &Falcon<'_, GspEngine>,
+        bar: Bar0<'_>,
+        bios: &Vbios,
+        fb_layout: &FbLayout,
+    ) -> Result {
+        // Check that the WPR2 region does not already exist - if it does, we cannot run
+        // FWSEC-FRTS until the GPU is reset.
+        if bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound() != 0 {
+            dev_err!(
+                dev,
+                "WPR2 region already exists - GPU needs to be reset to proceed\n"
+            );
+            return Err(EBUSY);
+        }
 
-    // SCRATCH_E contains the error code for FWSEC-FRTS.
-    let frts_status = bar
-        .read(regs::NV_PBUS_SW_SCRATCH_0E_FRTS_ERR)
-        .frts_err_code();
-    if frts_status != 0 {
-        dev_err!(
+        // FWSEC-FRTS will create the WPR2 region.
+        let fwsec_frts = FwsecFirmware::new(
             dev,
-            "FWSEC-FRTS returned with error code {:#x}\n",
-            frts_status
-        );
-
-        return Err(EIO);
-    }
-
-    // Check that the WPR2 region has been created as we requested.
-    let (wpr2_lo, wpr2_hi) = (
-        bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO).lower_bound(),
-        bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound(),
-    );
-
-    match (wpr2_lo, wpr2_hi) {
-        (_, 0) => {
-            dev_err!(dev, "WPR2 region not created after running FWSEC-FRTS\n");
+            falcon,
+            bios,
+            FwsecCommand::Frts {
+                frts_addr: fb_layout.frts.start,
+                frts_size: fb_layout.frts.len(),
+            },
+        )?;
 
-            Err(EIO)
+        if self.needs_fwsec_bootloader {
+            let fwsec_frts_bl = FwsecFirmwareWithBl::new(fwsec_frts, dev, chipset)?;
+            // Load and run the bootloader, which will load FWSEC-FRTS and run it.
+            fwsec_frts_bl.run(dev, falcon, bar)?;
+        } else {
+            // Load and run FWSEC-FRTS directly.
+            fwsec_frts.run(dev, falcon)?;
         }
-        (wpr2_lo, _) if wpr2_lo != fb_layout.frts.start => {
+
+        // SCRATCH_E contains the error code for FWSEC-FRTS.
+        let frts_status = bar
+            .read(regs::NV_PBUS_SW_SCRATCH_0E_FRTS_ERR)
+            .frts_err_code();
+        if frts_status != 0 {
             dev_err!(
                 dev,
-                "WPR2 region created at unexpected address {:#x}; expected {:#x}\n",
-                wpr2_lo,
-                fb_layout.frts.start,
+                "FWSEC-FRTS returned with error code {:#x}\n",
+                frts_status
             );
 
-            Err(EIO)
+            return Err(EIO);
         }
-        (wpr2_lo, wpr2_hi) => {
-            dev_dbg!(dev, "WPR2: {:#x}-{:#x}\n", wpr2_lo, wpr2_hi);
-            dev_dbg!(dev, "GPU instance built\n");
 
-            Ok(())
+        // Check that the WPR2 region has been created as we requested.
+        let (wpr2_lo, wpr2_hi) = (
+            bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO).lower_bound(),
+            bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound(),
+        );
+
+        match (wpr2_lo, wpr2_hi) {
+            (_, 0) => {
+                dev_err!(dev, "WPR2 region not created after running FWSEC-FRTS\n");
+
+                Err(EIO)
+            }
+            (wpr2_lo, _) if wpr2_lo != fb_layout.frts.start => {
+                dev_err!(
+                    dev,
+                    "WPR2 region created at unexpected address {:#x}; expected {:#x}\n",
+                    wpr2_lo,
+                    fb_layout.frts.start,
+                );
+
+                Err(EIO)
+            }
+            (wpr2_lo, wpr2_hi) => {
+                dev_dbg!(dev, "WPR2: {:#x}-{:#x}\n", wpr2_lo, wpr2_hi);
+                dev_dbg!(dev, "GPU instance built\n");
+
+                Ok(())
+            }
         }
     }
-}
 
-struct Tu102;
+    /// Load and prepare the resources required to properly reset the GSP after it has been stopped.
+    fn build_unload_bundle(
+        &self,
+        dev: &device::Device<device::Bound>,
+        chipset: Chipset,
+        bios: &Vbios,
+        gsp_falcon: &Falcon<'_, GspEngine>,
+        sec2_falcon: &Falcon<'_, Sec2>,
+    ) -> Result<crate::gsp::UnloadBundle> {
+        // Load the FWSEC SB firmware, as well as its bootloader if required.
+        let fwsec_sb = FwsecFirmware::new(dev, gsp_falcon, bios, FwsecCommand::Sb)?;
+        let fwsec_sb = if self.needs_fwsec_bootloader {
+            FwsecUnloadFirmware::WithBl(FwsecFirmwareWithBl::new(fwsec_sb, dev, chipset)?)
+        } else {
+            FwsecUnloadFirmware::WithoutBl(fwsec_sb)
+        };
+
+        KBox::new(
+            Sec2UnloadBundle {
+                fwsec_sb,
+                booter_unloader: BooterFirmware::new(
+                    dev,
+                    BooterKind::Unloader,
+                    chipset,
+                    FIRMWARE_VERSION,
+                    sec2_falcon,
+                )?,
+            },
+            GFP_KERNEL,
+        )
+        .map(|b| crate::gsp::UnloadBundle(b))
+        .map_err(Into::into)
+    }
+}
 
 impl GspHal for Tu102 {
-    fn boot<'a>(
+    fn boot(
         &self,
-        gsp: &'a Gsp,
-        dev: &'a device::Device<device::Bound>,
-        bar: Bar0<'a>,
-        chipset: Chipset,
+        gsp: &Gsp,
+        ctx: &mut GspBootContext<'_, '_>,
         fb_layout: &FbLayout,
         wpr_meta: &Coherent<GspFwWprMeta>,
-        gsp_falcon: &'a Falcon<GspEngine>,
-        sec2_falcon: &'a Falcon<Sec2>,
-    ) -> Result<BootUnloadGuard<'a>> {
+    ) -> Result<Option<crate::gsp::UnloadBundle>> {
+        let dev = ctx.dev();
+        let bar = ctx.bar;
+        let chipset = ctx.chipset;
+        let gsp_falcon = ctx.gsp_falcon;
+        let sec2_falcon = ctx.sec2_falcon;
+
         let bios = Vbios::new(dev, bar)?;
 
         // Try and prepare the unload bundle.
         //
         // If the unload bundle creation fails, the GPU will need to be reset before the driver can
         // be probed again.
-        let unload_bundle =
-            Sec2UnloadBundle::build(dev, bar, chipset, &bios, gsp_falcon, sec2_falcon)
-                .inspect_err(|e| {
-                    dev_warn!(dev, "Failed to prepare unload firmware: {:?}\n", e);
-                    dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n");
-                    dev_warn!(
-                        dev,
-                        "The GPU will need to be reset before the driver can bind again.\n"
-                    );
-                })
-                .ok()
-                .map(crate::gsp::UnloadBundle);
-
-        // Wrap the unload bundle into a drop guard so it is automatically run upon failure.
-        let unload_guard =
-            BootUnloadGuard::new(gsp, dev, bar, gsp_falcon, sec2_falcon, unload_bundle);
+        let unload_bundle = self
+            .build_unload_bundle(dev, chipset, &bios, gsp_falcon, sec2_falcon)
+            .inspect_err(|e| dev_warn!(dev, "Failed to prepare unload firmware: {:?}\n", e))
+            .ok();
+
+        // Run the unload bundle to try and recover the GSP if an error occurs.
+        let unload_guard = ScopeGuard::new_with_data(unload_bundle, |unload_bundle| {
+            if let Some(unload_bundle) = unload_bundle {
+                let _ = unload_bundle.0.run(ctx);
+            }
+        });
 
         // FWSEC-FRTS is not executed on chips where the FRTS region size is 0 (e.g. GA100).
         if !fb_layout.frts.is_empty() {
-            run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, fb_layout)?;
+            self.run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, fb_layout)?;
         }
 
-        gsp_falcon.reset(bar)?;
+        gsp_falcon.reset()?;
         let libos_handle = gsp.libos.dma_handle();
-        let (mbox0, mbox1) = gsp_falcon.boot(
-            bar,
-            Some(libos_handle as u32),
-            Some((libos_handle >> 32) as u32),
-        )?;
+        let (mbox0, mbox1) =
+            gsp_falcon.boot(Some(libos_handle as u32), Some((libos_handle >> 32) as u32))?;
         dev_dbg!(dev, "GSP MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1);
 
         dev_dbg!(
@@ -314,36 +308,32 @@ fn boot<'a>(
             chipset,
             FIRMWARE_VERSION,
             sec2_falcon,
-            bar,
         )?
-        .run(dev, bar, sec2_falcon, wpr_meta)?;
+        .run(dev, sec2_falcon, wpr_meta)?;
 
-        Ok(unload_guard)
+        Ok(unload_guard.dismiss())
     }
 
     fn post_boot(
         &self,
         gsp: &Gsp,
-        dev: &device::Device<device::Bound>,
-        bar: Bar0<'_>,
+        ctx: &mut GspBootContext<'_, '_>,
         gsp_fw: &GspFirmware,
-        gsp_falcon: &Falcon<GspEngine>,
-        sec2_falcon: &Falcon<Sec2>,
     ) -> Result {
-        // Create and run the GSP sequencer.
-        let seq_params = GspSequencerParams {
-            bootloader_app_version: gsp_fw.bootloader.app_version,
-            libos_dma_handle: gsp.libos.dma_handle(),
-            gsp_falcon,
-            sec2_falcon,
-            dev,
-            bar,
-        };
-        GspSequencer::run(&gsp.cmdq, seq_params)?;
+        GspSequencer::run(
+            &gsp.cmdq,
+            ctx,
+            gsp.libos.dma_handle(),
+            gsp_fw.bootloader.app_version,
+        )?;
 
         Ok(())
     }
 }
 
-const TU102: Tu102 = Tu102;
+/// The TU102 HAL requires the use of the FWSEC bootloader.
+const TU102: Tu102 = Tu102 {
+    needs_fwsec_bootloader: true,
+};
+
 pub(super) const TU102_HAL: &dyn GspHal = &TU102;
diff --git a/drivers/gpu/nova-core/gsp/regs.rs b/drivers/gpu/nova-core/gsp/regs.rs
new file mode 100644
index 0000000000000..a76dea3c3ab03
--- /dev/null
+++ b/drivers/gpu/nova-core/gsp/regs.rs
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use kernel::io::register;
+
+// PGSP
+
+register! {
+    pub(super) NV_PGSP_QUEUE_HEAD(u32) @ 0x00110c00 {
+        31:0    address;
+    }
+}
diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs
index e0850d21adca1..422a74f9ecbd5 100644
--- a/drivers/gpu/nova-core/gsp/sequencer.rs
+++ b/drivers/gpu/nova-core/gsp/sequencer.rs
@@ -31,6 +31,7 @@
             MessageFromGsp, //
         },
         fw,
+        GspBootContext, //
     },
     num::FromSafeCast,
     sbuffer::SBufferIter,
@@ -128,14 +129,12 @@ pub(crate) fn new(data: &[u8], dev: &device::Device) -> Result<(Self, usize)> {
 
 /// GSP Sequencer for executing firmware commands during boot.
 pub(crate) struct GspSequencer<'a> {
-    /// Sequencer information with command data.
-    seq_info: GspSequence,
     /// `Bar0` for register access.
     bar: Bar0<'a>,
     /// SEC2 falcon for core operations.
-    sec2_falcon: &'a Falcon<Sec2>,
+    sec2_falcon: &'a Falcon<'a, Sec2>,
     /// GSP falcon for core operations.
-    gsp_falcon: &'a Falcon<Gsp>,
+    gsp_falcon: &'a Falcon<'a, Gsp>,
     /// LibOS DMA handle address.
     libos_dma_handle: u64,
     /// Bootloader application version.
@@ -213,16 +212,16 @@ fn run(&self, seq: &GspSequencer<'_>) -> Result {
             GspSeqCmd::DelayUs(cmd) => cmd.run(seq),
             GspSeqCmd::RegStore(cmd) => cmd.run(seq),
             GspSeqCmd::CoreReset => {
-                seq.gsp_falcon.reset(seq.bar)?;
-                seq.gsp_falcon.dma_reset(seq.bar);
+                seq.gsp_falcon.reset()?;
+                seq.gsp_falcon.dma_reset();
                 Ok(())
             }
             GspSeqCmd::CoreStart => {
-                seq.gsp_falcon.start(seq.bar)?;
+                seq.gsp_falcon.start()?;
                 Ok(())
             }
             GspSeqCmd::CoreWaitForHalt => {
-                seq.gsp_falcon.wait_till_halted(seq.bar)?;
+                seq.gsp_falcon.wait_till_halted()?;
                 Ok(())
             }
             GspSeqCmd::CoreResume => {
@@ -231,35 +230,32 @@ fn run(&self, seq: &GspSequencer<'_>) -> Result {
                 // sequencer will start both.
 
                 // Reset the GSP to prepare it for resuming.
-                seq.gsp_falcon.reset(seq.bar)?;
+                seq.gsp_falcon.reset()?;
 
                 // Write the libOS DMA handle to GSP mailboxes.
                 seq.gsp_falcon.write_mailboxes(
-                    seq.bar,
                     Some(seq.libos_dma_handle as u32),
                     Some((seq.libos_dma_handle >> 32) as u32),
                 );
 
                 // Start the SEC2 falcon which will trigger GSP-RM to resume on the GSP.
-                seq.sec2_falcon.start(seq.bar)?;
+                seq.sec2_falcon.start()?;
 
                 // Poll until GSP-RM reload/resume has completed (up to 2 seconds).
-                seq.gsp_falcon
-                    .check_reload_completed(seq.bar, Delta::from_secs(2))?;
+                seq.gsp_falcon.check_reload_completed(Delta::from_secs(2))?;
 
                 // Verify SEC2 completed successfully by checking its mailbox for errors.
-                let mbox0 = seq.sec2_falcon.read_mailbox0(seq.bar);
+                let mbox0 = seq.sec2_falcon.read_mailbox0();
                 if mbox0 != 0 {
                     dev_err!(seq.dev, "Sequencer: sec2 errors: {:?}\n", mbox0);
                     return Err(EIO);
                 }
 
                 // Configure GSP with the bootloader version.
-                seq.gsp_falcon
-                    .write_os_version(seq.bar, seq.bootloader_app_version);
+                seq.gsp_falcon.write_os_version(seq.bootloader_app_version);
 
                 // Verify the GSP's RISC-V core is active indicating successful GSP boot.
-                if !seq.gsp_falcon.is_riscv_active(seq.bar) {
+                if !seq.gsp_falcon.is_riscv_active() {
                     dev_err!(seq.dev, "Sequencer: RISC-V core is not active\n");
                     return Err(EIO);
                 }
@@ -270,7 +266,7 @@ fn run(&self, seq: &GspSequencer<'_>) -> Result {
 }
 
 /// Iterator over GSP sequencer commands.
-pub(crate) struct GspSeqIter<'a> {
+struct GspSeqIter<'a> {
     /// Command data buffer.
     cmd_data: &'a [u8],
     /// Current position in the buffer.
@@ -283,6 +279,18 @@ pub(crate) struct GspSeqIter<'a> {
     dev: &'a device::Device,
 }
 
+impl<'a> GspSeqIter<'a> {
+    fn new(seq: &'a GspSequence, dev: &'a device::Device) -> Self {
+        Self {
+            cmd_data: &seq.cmd_data,
+            current_offset: 0,
+            total_cmds: seq.cmd_index,
+            cmds_processed: 0,
+            dev,
+        }
+    }
+}
+
 impl<'a> Iterator for GspSeqIter<'a> {
     type Item = Result<GspSeqCmd>;
 
@@ -325,37 +333,12 @@ fn next(&mut self) -> Option<Self::Item> {
 }
 
 impl<'a> GspSequencer<'a> {
-    fn iter(&self) -> GspSeqIter<'_> {
-        let cmd_data = &self.seq_info.cmd_data[..];
-
-        GspSeqIter {
-            cmd_data,
-            current_offset: 0,
-            total_cmds: self.seq_info.cmd_index,
-            cmds_processed: 0,
-            dev: self.dev,
-        }
-    }
-}
-
-/// Parameters for running the GSP sequencer.
-pub(crate) struct GspSequencerParams<'a> {
-    /// Bootloader application version.
-    pub(crate) bootloader_app_version: u32,
-    /// LibOS DMA handle address.
-    pub(crate) libos_dma_handle: u64,
-    /// GSP falcon for core operations.
-    pub(crate) gsp_falcon: &'a Falcon<Gsp>,
-    /// SEC2 falcon for core operations.
-    pub(crate) sec2_falcon: &'a Falcon<Sec2>,
-    /// Device for logging.
-    pub(crate) dev: &'a device::Device,
-    /// BAR0 for register access.
-    pub(crate) bar: Bar0<'a>,
-}
-
-impl<'a> GspSequencer<'a> {
-    pub(crate) fn run(cmdq: &Cmdq, params: GspSequencerParams<'a>) -> Result {
+    pub(crate) fn run(
+        cmdq: &Cmdq,
+        ctx: &'a GspBootContext<'_, '_>,
+        libos_dma_handle: u64,
+        bootloader_app_version: u32,
+    ) -> Result {
         let seq_info = loop {
             match cmdq.receive_msg::<GspSequence>(Cmdq::RECEIVE_TIMEOUT) {
                 Ok(seq_info) => break seq_info,
@@ -365,25 +348,24 @@ pub(crate) fn run(cmdq: &Cmdq, params: GspSequencerParams<'a>) -> Result {
         };
 
         let sequencer = GspSequencer {
-            seq_info,
-            bar: params.bar,
-            sec2_falcon: params.sec2_falcon,
-            gsp_falcon: params.gsp_falcon,
-            libos_dma_handle: params.libos_dma_handle,
-            bootloader_app_version: params.bootloader_app_version,
-            dev: params.dev,
+            bar: ctx.bar,
+            sec2_falcon: ctx.sec2_falcon,
+            gsp_falcon: ctx.gsp_falcon,
+            libos_dma_handle,
+            bootloader_app_version,
+            dev: ctx.dev(),
         };
 
         dev_dbg!(sequencer.dev, "Running CPU Sequencer commands\n");
 
-        for cmd_result in sequencer.iter() {
+        for cmd_result in GspSeqIter::new(&seq_info, sequencer.dev) {
             match cmd_result {
                 Ok(cmd) => cmd.run(&sequencer)?,
                 Err(e) => {
                     dev_err!(
                         sequencer.dev,
                         "Error running command at index {}\n",
-                        sequencer.seq_info.cmd_index
+                        seq_info.cmd_index
                     );
                     return Err(e);
                 }
diff --git a/drivers/gpu/nova-core/mctp.rs b/drivers/gpu/nova-core/mctp.rs
index 482786e07bc7e..acc2abbd4b0cd 100644
--- a/drivers/gpu/nova-core/mctp.rs
+++ b/drivers/gpu/nova-core/mctp.rs
@@ -7,55 +7,51 @@
 //! Data Model) messages between the kernel driver and GPU firmware processors
 //! such as FSP and GSP.
 
-use kernel::pci::Vendor;
+use kernel::{
+    bitfield,
+    pci::Vendor,
+    prelude::*, //
+};
 
-/// NVDM message type identifiers carried over MCTP.
-#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
-#[repr(u8)]
-pub(crate) enum NvdmType {
-    #[default]
-    /// Chain of Trust boot message.
-    Cot = 0x14,
-    /// FSP command response.
-    FspResponse = 0x15,
-}
-
-impl TryFrom<u8> for NvdmType {
-    type Error = u8;
-
-    fn try_from(value: u8) -> Result<Self, Self::Error> {
-        match value {
-            x if x == u8::from(Self::Cot) => Ok(Self::Cot),
-            x if x == u8::from(Self::FspResponse) => Ok(Self::FspResponse),
-            _ => Err(value),
-        }
-    }
-}
+use crate::{
+    bounded_enum,
+    num, //
+};
 
-impl From<NvdmType> for u8 {
-    fn from(value: NvdmType) -> Self {
-        value as u8
+bounded_enum! {
+    /// NVDM message type identifiers carried over MCTP.
+    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
+    pub(crate) enum NvdmType with TryFrom<Bounded<u32, 8>> {
+        /// Chain of Trust boot message.
+        Cot = 0x14,
+        /// FSP command response.
+        FspResponse = 0x15,
     }
 }
 
 bitfield! {
-    pub(crate) struct MctpHeader(u32), "MCTP transport header for NVIDIA firmware messages." {
-        31:31 som as bool, "Start-of-message bit.";
-        30:30 eom as bool, "End-of-message bit.";
-        29:28 seq as u8, "Packet sequence number.";
-        23:16 seid as u8, "Source endpoint ID.";
+    /// MCTP transport header for NVIDIA firmware messages.
+    pub(crate) struct MctpHeader(u32) {
+        /// Start-of-message bit.
+        31:31 som;
+        /// End-of-message bit.
+        30:30 eom;
+        /// Packet sequence number.
+        29:28 seq;
+        /// Source endpoint ID.
+        23:16 seid;
     }
 }
 
 impl MctpHeader {
     /// Builds a single-packet MCTP header (`SOM=1`, `EOM=1`, `SEQ=0`, `SEID=0`).
     pub(crate) fn single_packet() -> Self {
-        Self::default().set_som(true).set_eom(true)
+        Self::zeroed().with_som(true).with_eom(true)
     }
 
     /// Returns whether this is a complete single-packet message (`SOM=1` and `EOM=1`).
     pub(crate) fn is_single_packet(self) -> bool {
-        self.som() && self.eom()
+        self.som().into_bool() && self.eom().into_bool()
     }
 }
 
@@ -63,26 +59,30 @@ pub(crate) fn is_single_packet(self) -> bool {
 const MSG_TYPE_VENDOR_PCI: u8 = 0x7e;
 
 bitfield! {
-    pub(crate) struct NvdmHeader(u32), "NVIDIA Vendor-Defined Message header over MCTP." {
-        31:24 nvdm_type as u8 ?=> NvdmType, "NVDM message type.";
-        23:8 vendor_id as u16, "PCI vendor ID.";
-        6:0 msg_type as u8, "MCTP vendor-defined message type.";
+    /// NVIDIA Vendor-Defined Message header over MCTP.
+    pub(crate) struct NvdmHeader(u32) {
+        /// NVDM message type.
+        31:24 nvdm_type ?=> NvdmType;
+        /// PCI vendor ID.
+        23:8 vendor_id;
+        /// MCTP vendor-defined message type.
+        6:0 msg_type;
     }
 }
 
 impl NvdmHeader {
     /// Builds an NVDM header for the given message type.
     pub(crate) fn new(nvdm_type: NvdmType) -> Self {
-        Self::default()
-            .set_msg_type(MSG_TYPE_VENDOR_PCI)
-            .set_vendor_id(Vendor::NVIDIA.as_raw())
-            .set_nvdm_type(nvdm_type)
+        Self::zeroed()
+            .with_const_msg_type::<{ num::u8_as_u32(MSG_TYPE_VENDOR_PCI) }>()
+            .with_vendor_id(Vendor::NVIDIA.as_raw())
+            .with_nvdm_type(nvdm_type)
     }
 
     /// Validates this header against the expected NVIDIA NVDM format and type.
     pub(crate) fn validate(self, expected_type: NvdmType) -> bool {
-        self.msg_type() == MSG_TYPE_VENDOR_PCI
-            && self.vendor_id() == Vendor::NVIDIA.as_raw()
+        u8::from(self.msg_type()) == MSG_TYPE_VENDOR_PCI
+            && u16::from(self.vendor_id()) == Vendor::NVIDIA.as_raw()
             && matches!(self.nvdm_type(), Ok(nvdm_type) if nvdm_type == expected_type)
     }
 }
diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs
index 9f0199f7b38cc..a61406ba5c0bd 100644
--- a/drivers/gpu/nova-core/nova_core.rs
+++ b/drivers/gpu/nova-core/nova_core.rs
@@ -10,9 +10,6 @@
     InPlaceModule, //
 };
 
-#[macro_use]
-mod bitfield;
-
 mod driver;
 mod falcon;
 mod fb;
@@ -54,7 +51,7 @@ struct NovaCoreModule {
 
 impl InPlaceModule for NovaCoreModule {
     fn init(module: &'static kernel::ThisModule) -> impl PinInit<Self, Error> {
-        let dir = debugfs::Dir::new(kernel::c_str!("nova-core"));
+        let dir = debugfs::Dir::new(c"nova-core");
 
         // SAFETY: We are the only driver code running during init, so there
         // cannot be any concurrent access to `DEBUGFS_ROOT`.
diff --git a/drivers/gpu/nova-core/nova_core_exports.c b/drivers/gpu/nova-core/nova_core_exports.c
new file mode 100644
index 0000000000000..6e80ca9792ee5
--- /dev/null
+++ b/drivers/gpu/nova-core/nova_core_exports.c
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+/*
+ * Exports Rust symbols from the `nova_core` crate for use by dependent modules.
+ *
+ * This is a workaround until the build system supports Rust cross-module
+ * dependencies natively.
+ */
+
+#include <linux/export.h>
+
+#define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym)
+
+#include "exports_nova_core_generated.h"
diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs
index 0f49c1ab83ad4..397124f245ee9 100644
--- a/drivers/gpu/nova-core/regs.rs
+++ b/drivers/gpu/nova-core/regs.rs
@@ -126,7 +126,7 @@ fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result {
     }
 
     /// High bits of the physical system memory address used by the GPU to perform sysmembar
-    /// operations (see [`crate::fb::SysmemFlush`]).
+    /// operations.
     pub(crate) NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x00100c40 {
         23:0    adr_63_40;
     }
@@ -153,11 +153,6 @@ fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result {
 /// The base is provided by the GB10x framebuffer HAL.
 pub(crate) struct Hshub0Base(());
 
-/// Base of the GB20x FBHUB0 register window (`NV_FBHUB0_PRI_BASE` in Open RM).
-///
-/// The base is provided by the GB20x framebuffer HAL.
-pub(crate) struct Fbhub0Base(());
-
 register! {
     // GB10x sysmem flush registers, relative to the HSHUB0 base. GB10x routes sysmembar
     // through a primary and an EG (egress) pair that must both be programmed to the same
@@ -178,16 +173,37 @@ fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result {
     pub(crate) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x000006c4 {
         19:0    adr;
     }
+}
 
-    // GB20x sysmem flush registers, relative to the FBHUB0 base. Unlike the older
-    // NV_PFB_NISO_FLUSH_SYSMEM_ADDR registers which encode the address with an 8-bit
-    // right-shift, these take the raw address split into lower and upper halves. Hardware
-    // ignores bits 7:0 of the LO register.
-    pub(crate) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Fbhub0Base + 0x00001d58 {
+register! {
+    // GB20x FBHUB0 sysmem flush registers. Unlike the older
+    // NV_PFB_NISO_FLUSH_SYSMEM_ADDR registers, which encode the address with an
+    // 8-bit right-shift, these take the raw address split into lower and upper
+    // halves. Hardware ignores bits 7:0 of the LO register.
+    pub(crate) NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x008a1d58 {
         31:0    adr => u32;
     }
 
-    pub(crate) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Fbhub0Base + 0x00001d5c {
+    pub(crate) NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x008a1d5c {
+        19:0    adr;
+    }
+}
+
+register! {
+    /// Low bits of the physical system memory address used by the GPU to perform
+    /// sysmembar operations on Hopper.
+    ///
+    /// Like the GB20x FBHUB0 registers, and unlike the Ampere
+    /// `NV_PFB_NISO_FLUSH_SYSMEM_ADDR` registers (which encode the address with an
+    /// 8-bit right-shift), these take the raw address split into lower and upper
+    /// halves. Hardware ignores bits 7:0 of the LO register.
+    pub(crate) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x00100a34 {
+        31:0    adr => u32;
+    }
+
+    /// High bits of the physical system memory address used by the GPU to perform
+    /// sysmembar operations on Hopper.
+    pub(crate) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x00100a38 {
         19:0    adr;
     }
 }
@@ -227,14 +243,6 @@ pub(crate) fn is_wpr2_set(self) -> bool {
     }
 }
 
-// PGSP
-
-register! {
-    pub(crate) NV_PGSP_QUEUE_HEAD(u32) @ 0x00110c00 {
-        31:0    address;
-    }
-}
-
 // PGC6 register space.
 //
 // `GC6` is a GPU low-power state where VRAM is in self-refresh and the GPU is powered down (except
@@ -294,28 +302,6 @@ pub(crate) fn usable_fb_size(self) -> u64 {
     }
 }
 
-// PDISP
-
-register! {
-    pub(crate) NV_PDISP_VGA_WORKSPACE_BASE(u32) @ 0x00625f04 {
-        /// VGA workspace base address divided by 0x10000.
-        31:8    addr;
-        /// Set if the `addr` field is valid.
-        3:3     status_valid => bool;
-    }
-}
-
-impl NV_PDISP_VGA_WORKSPACE_BASE {
-    /// Returns the base address of the VGA workspace, or `None` if none exists.
-    pub(crate) fn vga_workspace_addr(self) -> Option<u64> {
-        if self.status_valid() {
-            Some(u64::from(self.addr()) << 16)
-        } else {
-            None
-        }
-    }
-}
-
 // FUSE
 
 pub(crate) const NV_FUSE_OPT_FPF_SIZE: usize = 16;
diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs
index c6e6bfcd6a1f8..c03650ee52265 100644
--- a/drivers/gpu/nova-core/vbios.rs
+++ b/drivers/gpu/nova-core/vbios.rs
@@ -13,11 +13,8 @@
     register,
     sizes::SZ_4K,
     sync::aref::ARef,
-    transmute::FromBytes,
 };
 
-use zerocopy::FromBytes as _;
-
 use crate::{
     driver::Bar0,
     firmware::{
@@ -359,7 +356,7 @@ pub(crate) fn fwsec_image(&self) -> &FwSecBiosImage {
 }
 
 /// PCI Data Structure as defined in PCI Firmware Specification
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, FromBytes)]
 #[repr(C)]
 struct PcirStruct {
     /// PCI Data Structure signature ("PCIR" or "NPDS")
@@ -388,15 +385,12 @@ struct PcirStruct {
     max_runtime_image_len: u16,
 }
 
-// SAFETY: all bit patterns are valid for `PcirStruct`.
-unsafe impl FromBytes for PcirStruct {}
-
 impl PcirStruct {
     /// The bit in `last_image` that indicates the last image.
     const LAST_IMAGE_BIT_MASK: u8 = 0x80;
 
     fn new(dev: &device::Device, data: &[u8]) -> Result<Self> {
-        let (pcir, _) = PcirStruct::from_bytes_copy_prefix(data).ok_or(EINVAL)?;
+        let (pcir, _) = PcirStruct::read_from_prefix(data).map_err(|_| EINVAL)?;
 
         // Signature should be "PCIR" (0x52494350) or "NPDS" (0x5344504e).
         if &pcir.signature != b"PCIR" && &pcir.signature != b"NPDS" {
@@ -432,7 +426,7 @@ fn image_size_bytes(&self) -> usize {
 /// This is the head of the BIT table, that is used to locate the Falcon data. The BIT table (with
 /// its header) is in the [`PciAtBiosImage`] and the falcon data it is pointing to is in the
 /// [`FwSecBiosImage`].
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, FromBytes)]
 #[repr(C)]
 struct BitHeader {
     /// 0h: BIT Header Identifier (BMP=0x7FFF/BIT=0xB8FF)
@@ -451,12 +445,9 @@ struct BitHeader {
     checksum: u8,
 }
 
-// SAFETY: all bit patterns are valid for `BitHeader`.
-unsafe impl FromBytes for BitHeader {}
-
 impl BitHeader {
     fn new(data: &[u8]) -> Result<Self> {
-        let (header, _) = BitHeader::from_bytes_copy_prefix(data).ok_or(EINVAL)?;
+        let (header, _) = BitHeader::read_from_prefix(data).map_err(|_| EINVAL)?;
 
         // Check header ID and signature
         if header.id != 0xB8FF || &header.signature != b"BIT\0" {
@@ -468,7 +459,7 @@ fn new(data: &[u8]) -> Result<Self> {
 }
 
 /// BIT Token Entry: Records in the BIT table followed by the BIT header.
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, FromBytes)]
 #[repr(C)]
 struct BitToken {
     /// 00h: Token identifier
@@ -481,9 +472,6 @@ struct BitToken {
     data_offset: u16,
 }
 
-// SAFETY: all bit patterns are valid for `BitToken`.
-unsafe impl FromBytes for BitToken {}
-
 impl BitToken {
     /// BIT token ID for Falcon data.
     const ID_FALCON_DATA: u8 = 0x70;
@@ -508,7 +496,7 @@ fn from_id(image: &PciAtBiosImage, token_id: u8) -> Result<Self> {
                 .and_then(|data| data.get(..entry_size))
                 .ok_or(EINVAL)?;
 
-            let (token, _) = BitToken::from_bytes_copy_prefix(entry).ok_or(EINVAL)?;
+            let (token, _) = BitToken::read_from_prefix(entry).map_err(|_| EINVAL)?;
 
             // Check if this token has the requested ID
             if token.id == token_id {
@@ -525,7 +513,7 @@ fn from_id(image: &PciAtBiosImage, token_id: u8) -> Result<Self> {
 ///
 /// This header is at the beginning of every image in the set of images in the ROM. It contains a
 /// pointer to the PCI Data Structure which describes the image.
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, FromBytes)]
 #[repr(C)]
 struct PciRomHeader {
     /// 00h: Signature (0xAA55)
@@ -536,13 +524,10 @@ struct PciRomHeader {
     pci_data_struct_offset: u16,
 }
 
-// SAFETY: all bit patterns are valid for `PciRomHeader`.
-unsafe impl FromBytes for PciRomHeader {}
-
 impl PciRomHeader {
     fn new(dev: &device::Device, data: &[u8]) -> Result<Self> {
-        let (rom_header, _) = PciRomHeader::from_bytes_copy_prefix(data)
-            .ok_or(EINVAL)
+        let (rom_header, _) = PciRomHeader::read_from_prefix(data)
+            .map_err(|_| EINVAL)
             .inspect_err(|_| dev_err!(dev, "Not enough data for ROM header\n"))?;
 
         // Check for valid ROM signatures.
@@ -564,7 +549,7 @@ fn new(dev: &device::Device, data: &[u8]) -> Result<Self> {
 /// PCI Data Structure. It contains some fields that are redundant with the PCI Data Structure, but
 /// are needed for traversing the BIOS images. It is expected to be present in all BIOS images
 /// except for NBSI images.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, FromBytes)]
 #[repr(C)]
 struct NpdeStruct {
     /// 00h: Signature ("NPDE")
@@ -579,15 +564,12 @@ struct NpdeStruct {
     last_image: u8,
 }
 
-// SAFETY: all bit patterns are valid for `NpdeStruct`.
-unsafe impl FromBytes for NpdeStruct {}
-
 impl NpdeStruct {
     /// The bit in `last_image` that indicates the last image.
     const LAST_IMAGE_BIT_MASK: u8 = 0x80;
 
     fn new(dev: &device::Device, data: &[u8]) -> Option<Self> {
-        let (npde, _) = NpdeStruct::from_bytes_copy_prefix(data)?;
+        let (npde, _) = NpdeStruct::read_from_prefix(data).ok()?;
 
         // Signature should be "NPDE" (0x4544504E).
         if &npde.signature != b"NPDE" {
@@ -784,7 +766,7 @@ fn falcon_data_offset(&self, dev: &device::Device) -> Result<usize> {
         let data = &self.base.data;
         let (ptr, _) = data
             .get(offset..)
-            .and_then(u32::from_bytes_copy_prefix)
+            .and_then(|p| u32::read_from_prefix(p).ok())
             .ok_or(EINVAL)?;
 
         usize::from_safe_cast(ptr)
@@ -814,6 +796,7 @@ fn try_from(base: BiosImage) -> Result<Self> {
 /// The [`PmuLookupTableEntry`] structure is a single entry in the [`PmuLookupTable`].
 ///
 /// See the [`PmuLookupTable`] description for more information.
+#[derive(FromBytes)]
 #[repr(C, packed)]
 struct PmuLookupTableEntry {
     application_id: u8,
@@ -821,9 +804,6 @@ struct PmuLookupTableEntry {
     data: u32,
 }
 
-// SAFETY: all bit patterns are valid for `PmuLookupTableEntry`.
-unsafe impl FromBytes for PmuLookupTableEntry {}
-
 impl PmuLookupTableEntry {
     /// PMU lookup table application ID for firmware security license ucode.
     #[expect(dead_code)]
@@ -836,6 +816,7 @@ impl PmuLookupTableEntry {
 }
 
 #[repr(C)]
+#[derive(FromBytes)]
 struct PmuLookupTableHeader {
     version: u8,
     header_len: u8,
@@ -843,9 +824,6 @@ struct PmuLookupTableHeader {
     entry_count: u8,
 }
 
-// SAFETY: all bit patterns are valid for `PmuLookupTableHeader`.
-unsafe impl FromBytes for PmuLookupTableHeader {}
-
 /// The [`PmuLookupTableEntry`] structure is used to find the [`PmuLookupTableEntry`] for a given
 /// application ID.
 ///
@@ -857,7 +835,7 @@ struct PmuLookupTable {
 
 impl PmuLookupTable {
     fn new(dev: &device::Device, data: &[u8]) -> Result<Self> {
-        let (header, _) = PmuLookupTableHeader::from_bytes_copy_prefix(data).ok_or(EINVAL)?;
+        let (header, _) = PmuLookupTableHeader::read_from_prefix(data).map_err(|_| EINVAL)?;
 
         let header_len = usize::from(header.header_len);
         let entry_len = usize::from(header.entry_len);
@@ -872,8 +850,8 @@ fn new(dev: &device::Device, data: &[u8]) -> Result<Self> {
 
         let mut entries = KVVec::with_capacity(entry_count, GFP_KERNEL)?;
         for i in 0..entry_count {
-            let (entry, _) = PmuLookupTableEntry::from_bytes_copy_prefix(&data[i * entry_len..])
-                .ok_or(EINVAL)?;
+            let (entry, _) = PmuLookupTableEntry::read_from_prefix(&data[i * entry_len..])
+                .map_err(|_| EINVAL)?;
             entries.push(entry, GFP_KERNEL)?;
         }
 
@@ -929,15 +907,11 @@ pub(crate) fn header(&self) -> Result<FalconUCodeDesc> {
         let ver = data.get(1).copied().ok_or(EINVAL)?;
         match ver {
             2 => {
-                let v2 = FalconUCodeDescV2::read_from_prefix(data)
-                    .map_err(|_| EINVAL)?
-                    .0;
+                let (v2, _) = FalconUCodeDescV2::read_from_prefix(data).map_err(|_| EINVAL)?;
                 Ok(FalconUCodeDesc::V2(v2))
             }
             3 => {
-                let v3 = FalconUCodeDescV3::from_bytes_copy_prefix(data)
-                    .ok_or(EINVAL)?
-                    .0;
+                let (v3, _) = FalconUCodeDescV3::read_from_prefix(data).map_err(|_| EINVAL)?;
                 Ok(FalconUCodeDesc::V3(v3))
             }
             _ => {
diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig
index f9bcaeb66385b..03f36899e458a 100644
--- a/drivers/hid/Kconfig
+++ b/drivers/hid/Kconfig
@@ -1079,6 +1079,15 @@ config HID_ROCCAT
 	Say Y here if you have a Roccat mouse or keyboard and want
 	support for its special functionalities.
 
+config HID_ROCCAT_KONE_KUNIT_TEST
+	bool "KUnit tests for the Roccat Kone driver" if !KUNIT_ALL_TESTS
+	depends on HID_ROCCAT=y && KUNIT=y
+	default KUNIT_ALL_TESTS
+	help
+	  Enable the KUnit regression tests for the Roccat Kone driver,
+	  covering bounds checking of device-supplied profile indices.
+	  If unsure, say N.
+
 config HID_SAITEK
 	tristate "Saitek (Mad Catz) non-fully HID-compliant devices"
 	help
diff --git a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.c b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.c
index 837d59e7a6610..8ac44368a37d5 100644
--- a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.c
+++ b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.c
@@ -7,6 +7,7 @@
  *
  * Author: Basavaraj Natikar <Basavaraj.Natikar@amd.com>
  */
+#include <linux/amd-pmf.h>
 #include <linux/amd-pmf-io.h>
 #include <linux/io-64-nonatomic-lo-hi.h>
 #include <linux/iopoll.h>
@@ -102,20 +103,20 @@ static int amd_sfh_mode_info(u32 *platform_type, u32 *laptop_placement)
 	*platform_type = mode.op_mode.devicemode;
 
 	if (mode.op_mode.ontablestate == 1) {
-		*laptop_placement = ON_TABLE;
+		*laptop_placement = AMD_PMF_ON_TABLE;
 	} else if (mode.op_mode.ontablestate == 2) {
-		*laptop_placement = ON_LAP_MOTION;
+		*laptop_placement = AMD_PMF_ON_LAP_MOTION;
 	} else if (mode.op_mode.inbagstate == 1) {
-		*laptop_placement = IN_BAG;
+		*laptop_placement = AMD_PMF_IN_BAG;
 	} else if (mode.op_mode.outbagstate == 1) {
-		*laptop_placement = OUT_OF_BAG;
+		*laptop_placement = AMD_PMF_OUT_OF_BAG;
 	} else if (mode.op_mode.ontablestate == 0 || mode.op_mode.inbagstate == 0 ||
 		 mode.op_mode.outbagstate == 0) {
-		*laptop_placement = LP_UNKNOWN;
+		*laptop_placement = AMD_PMF_LP_UNKNOWN;
 		pr_warn_once("Unknown laptop placement\n");
 	} else if (mode.op_mode.ontablestate == 3 || mode.op_mode.inbagstate == 3 ||
 		 mode.op_mode.outbagstate == 3) {
-		*laptop_placement = LP_UNDEFINED;
+		*laptop_placement = AMD_PMF_LP_UNDEFINED;
 		pr_warn_once("Undefined laptop placement\n");
 	}
 
diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c
index 3f5e96900b67a..d34d74df3dc0b 100644
--- a/drivers/hid/hid-asus.c
+++ b/drivers/hid/hid-asus.c
@@ -965,6 +965,12 @@ static int asus_input_configured(struct hid_device *hdev, struct hid_input *hi)
 		}
 	}
 
+	if (drvdata->quirks & QUIRK_HID_FN_LOCK) {
+		drvdata->fn_lock = true;
+		INIT_WORK(&drvdata->fn_lock_sync_work, asus_sync_fn_lock);
+		asus_kbd_set_fn_lock(hdev, true);
+	}
+
 	if (drvdata->tp) {
 		int ret;
 
diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c
index cf123347a2af7..f107f5103b358 100644
--- a/drivers/hid/hid-core.c
+++ b/drivers/hid/hid-core.c
@@ -2447,9 +2447,16 @@ EXPORT_SYMBOL_GPL(hid_hw_start);
  *
  * This is usually called from remove function or from probe when something
  * failed and hid_hw_start was called already.
+ *
+ * If the caller enabled HID input via hid_device_io_start() and is unwinding
+ * without an explicit hid_device_io_stop(), quiesce input first so that
+ * in-flight reports cannot reach handlers (e.g. hidraw_report_event) whose
+ * backing objects hid_disconnect() is about to free.
  */
 void hid_hw_stop(struct hid_device *hdev)
 {
+	if (hdev->io_started)
+		hid_device_io_stop(hdev);
 	hid_disconnect(hdev);
 	hdev->ll_driver->stop(hdev);
 }
diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h
index 1059922baaacd..b70f719b3b075 100644
--- a/drivers/hid/hid-ids.h
+++ b/drivers/hid/hid-ids.h
@@ -1379,6 +1379,8 @@
 #define USB_DEVICE_ID_STEELSERIES_SRWS1	0x1410
 #define USB_DEVICE_ID_STEELSERIES_ARCTIS_1  0x12b6
 #define USB_DEVICE_ID_STEELSERIES_ARCTIS_9  0x12c2
+#define USB_DEVICE_ID_STEELSERIES_MSI_KLC   0x1122
+#define USB_DEVICE_ID_STEELSERIES_MSI_ALC   0x1161
 
 #define USB_VENDOR_ID_SUN		0x0430
 #define USB_DEVICE_ID_RARITAN_KVM_DONGLE	0xcdab
diff --git a/drivers/hid/hid-nintendo.c b/drivers/hid/hid-nintendo.c
index e7302ec01ff1f..acf0adac6b84b 100644
--- a/drivers/hid/hid-nintendo.c
+++ b/drivers/hid/hid-nintendo.c
@@ -1474,7 +1474,6 @@ static void joycon_parse_imu_report(struct joycon_ctlr *ctlr,
 		dropped_threshold = ctlr->imu_avg_delta_ms * 3 / 2;
 		dropped_pkts = (delta - min(delta, dropped_threshold)) /
 				ctlr->imu_avg_delta_ms;
-		ctlr->imu_timestamp_us += 1000 * ctlr->imu_avg_delta_ms;
 		if (dropped_pkts > JC_IMU_DROPPED_PKT_WARNING) {
 			hid_warn_ratelimited(ctlr->hdev,
 				 "compensating for %u dropped IMU reports\n",
diff --git a/drivers/hid/hid-roccat-kone.c b/drivers/hid/hid-roccat-kone.c
index 58654cf78f0df..3dae9eaa0b6f4 100644
--- a/drivers/hid/hid-roccat-kone.c
+++ b/drivers/hid/hid-roccat-kone.c
@@ -36,6 +36,8 @@ static uint profile_numbers[5] = {0, 1, 2, 3, 4};
 
 static void kone_profile_activated(struct kone_device *kone, uint new_profile)
 {
+	if (new_profile < 1 || new_profile > ARRAY_SIZE(kone->profiles))
+		new_profile = 1;
 	kone->actual_profile = new_profile;
 	kone->actual_dpi = kone->profiles[new_profile - 1].startup_dpi;
 }
@@ -793,8 +795,10 @@ static void kone_keep_values_up_to_date(struct kone_device *kone,
 {
 	switch (event->event) {
 	case kone_mouse_event_switch_profile:
-		kone->actual_dpi = kone->profiles[event->value - 1].
-				startup_dpi;
+		if (event->value >= 1 &&
+		    event->value <= ARRAY_SIZE(kone->profiles))
+			kone->actual_dpi =
+				kone->profiles[event->value - 1].startup_dpi;
 		fallthrough;
 	case kone_mouse_event_osd_profile:
 		kone->actual_profile = event->value;
@@ -915,3 +919,60 @@ module_exit(kone_exit);
 MODULE_AUTHOR("Stefan Achatz");
 MODULE_DESCRIPTION("USB Roccat Kone driver");
 MODULE_LICENSE("GPL v2");
+
+#if IS_ENABLED(CONFIG_HID_ROCCAT_KONE_KUNIT_TEST)
+#include <kunit/test.h>
+
+/*
+ * Regression test for the out-of-bounds read in
+ * kone_keep_values_up_to_date(): a malicious USB device sends a
+ * "switch profile" HID event (event == kone_mouse_event_switch_profile)
+ * with an attacker-chosen value in 0..255, which is used unbounded as
+ * profiles[value - 1].  On an unpatched kernel the attack case triggers a
+ * KASAN slab-out-of-bounds read; the fix must leave actual_dpi unchanged.
+ */
+static void kone_profile_index_oob_test(struct kunit *test)
+{
+	struct kone_device *kone;
+	struct kone_mouse_event ev = {};
+	/*
+	 * Allocate only up to the end of profiles[] so that any index past
+	 * the 5-element array is IMMEDIATELY out of bounds and lands in the
+	 * KASAN redzone (a far over-read would hit unrelated valid memory and
+	 * escape KASAN).
+	 */
+	size_t sz = offsetof(struct kone_device, profiles) +
+		    sizeof(kone->profiles);
+
+	kone = kunit_kzalloc(test, sz, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, kone);
+	kone->profiles[0].startup_dpi = 0x42;
+
+	/* benign control: a valid in-range value drives the SAME path and
+	 * must succeed (proves the trigger reaches the real code).
+	 */
+	ev.event = kone_mouse_event_switch_profile;
+	ev.value = 1;
+	kone_keep_values_up_to_date(kone, &ev);
+	KUNIT_EXPECT_EQ(test, kone->actual_dpi, 0x42);
+
+	/* attack: value == ARRAY_SIZE(profiles) + 1 reads profiles[5], one
+	 * element past the array end -> KASAN slab-out-of-bounds read on an
+	 * unpatched kernel. The fix must reject it (actual_dpi unchanged).
+	 */
+	ev.value = ARRAY_SIZE(kone->profiles) + 1;
+	kone_keep_values_up_to_date(kone, &ev);
+	KUNIT_EXPECT_EQ(test, kone->actual_dpi, 0x42);
+}
+
+static struct kunit_case kone_test_cases[] = {
+	KUNIT_CASE(kone_profile_index_oob_test),
+	{}
+};
+
+static struct kunit_suite kone_test_suite = {
+	.name = "hid-roccat-kone",
+	.test_cases = kone_test_cases,
+};
+kunit_test_suite(kone_test_suite);
+#endif /* CONFIG_HID_ROCCAT_KONE_KUNIT_TEST */
diff --git a/drivers/hid/hid-sony.c b/drivers/hid/hid-sony.c
index e75246d29e16d..253fff4066ebd 100644
--- a/drivers/hid/hid-sony.c
+++ b/drivers/hid/hid-sony.c
@@ -29,6 +29,7 @@
  * There will be no PIN request from the device.
  */
 
+#include <linux/cleanup.h>
 #include <linux/device.h>
 #include <linux/hid.h>
 #include <linux/module.h>
@@ -81,6 +82,7 @@
 #define SONY_FF_SUPPORT (SIXAXIS_CONTROLLER | MOTION_CONTROLLER)
 #define SONY_BT_DEVICE (SIXAXIS_CONTROLLER_BT | MOTION_CONTROLLER_BT | NAVIGATION_CONTROLLER_BT)
 #define NSG_MRXU_REMOTE (NSG_MR5U_REMOTE_BT | NSG_MR7U_REMOTE_BT)
+#define RB4_GUITAR_PS4 (RB4_GUITAR_PS4_USB | RB4_GUITAR_PS4_BT)
 
 #define MAX_LEDS 4
 #define NSG_MRXU_MAX_X 1667
@@ -521,10 +523,6 @@ static DEFINE_SPINLOCK(sony_dev_list_lock);
 static LIST_HEAD(sony_device_list);
 static DEFINE_IDA(sony_device_id_allocator);
 
-enum sony_worker {
-	SONY_WORKER_STATE
-};
-
 struct sony_sc {
 	spinlock_t lock;
 	struct list_head list_node;
@@ -534,6 +532,7 @@ struct sony_sc {
 	struct input_dev *sensor_dev;
 	struct led_classdev *leds[MAX_LEDS];
 	unsigned long quirks;
+	int (*raw_event)(struct sony_sc *sc, u8 *rd, int size);
 	struct work_struct state_worker;
 	void (*send_output_report)(struct sony_sc *sc);
 	struct power_supply *battery;
@@ -566,19 +565,11 @@ struct sony_sc {
 
 static void sony_set_leds(struct sony_sc *sc);
 
-static inline void sony_schedule_work(struct sony_sc *sc,
-				      enum sony_worker which)
+static inline void sony_schedule_work(struct sony_sc *sc)
 {
-	unsigned long flags;
-
-	switch (which) {
-	case SONY_WORKER_STATE:
-		spin_lock_irqsave(&sc->lock, flags);
-		if (!sc->defer_initialization && sc->state_worker_initialized)
-			schedule_work(&sc->state_worker);
-		spin_unlock_irqrestore(&sc->lock, flags);
-		break;
-	}
+	guard(spinlock_irqsave)(&sc->lock);
+	if (!sc->defer_initialization && sc->state_worker_initialized)
+		schedule_work(&sc->state_worker);
 }
 
 static void ghl_magic_poke_cb(struct urb *urb)
@@ -946,15 +937,39 @@ static const u8 *sony_report_fixup(struct hid_device *hdev, u8 *rdesc,
 	return rdesc;
 }
 
-static void sixaxis_parse_report(struct sony_sc *sc, u8 *rd, int size)
+static int sixaxis_raw_event(struct sony_sc *sc, u8 *rd, int size)
 {
 	static const u8 sixaxis_battery_capacity[] = { 0, 1, 25, 50, 75, 100 };
-	unsigned long flags;
 	int offset;
 	u8 index;
 	u8 battery_capacity;
 	int battery_status;
 
+	if (unlikely(size != 49 || rd[0] != 0x01))
+		return 0;
+
+	if (sc->quirks & SIXAXIS_CONTROLLER) {
+		/*
+		 * When connected via Bluetooth the Sixaxis occasionally sends
+		 * a report with the second byte 0xff and the rest zeroed.
+		 *
+		 * This report does not reflect the actual state of the
+		 * controller must be ignored to avoid generating false input
+		 * events.
+		 */
+		if (rd[1] == 0xff)
+			return -EINVAL;
+
+		/*
+		 * Sixaxis HID report has acclerometers/gyro with MSByte first, this
+		 * has to be BYTE_SWAPPED before passing up to joystick interface
+		 */
+		swap(rd[41], rd[42]);
+		swap(rd[43], rd[44]);
+		swap(rd[45], rd[46]);
+		swap(rd[47], rd[48]);
+	}
+
 	/*
 	 * The sixaxis is charging if the battery value is 0xee
 	 * and it is fully charged if the value is 0xef.
@@ -972,10 +987,10 @@ static void sixaxis_parse_report(struct sony_sc *sc, u8 *rd, int size)
 		battery_status = POWER_SUPPLY_STATUS_DISCHARGING;
 	}
 
-	spin_lock_irqsave(&sc->lock, flags);
-	sc->battery_capacity = battery_capacity;
-	sc->battery_status = battery_status;
-	spin_unlock_irqrestore(&sc->lock, flags);
+	scoped_guard(spinlock_irqsave, &sc->lock) {
+		sc->battery_capacity = battery_capacity;
+		sc->battery_status = battery_status;
+	}
 
 	if (sc->quirks & SIXAXIS_CONTROLLER) {
 		int val;
@@ -993,13 +1008,18 @@ static void sixaxis_parse_report(struct sony_sc *sc, u8 *rd, int size)
 
 		input_sync(sc->sensor_dev);
 	}
+
+	return 0;
 }
 
-static void nsg_mrxu_parse_report(struct sony_sc *sc, u8 *rd, int size)
+static int nsg_mrxu_raw_event(struct sony_sc *sc, u8 *rd, int size)
 {
 	int n, offset, relx, rely;
 	u8 active;
 
+	if (unlikely(size < 12 || rd[0] != 0x02))
+		return 0;
+
 	/*
 	 * The NSG-MRxU multi-touch trackpad data starts at offset 1 and
 	 *   the touch-related data starts at offset 2.
@@ -1067,10 +1087,33 @@ static void nsg_mrxu_parse_report(struct sony_sc *sc, u8 *rd, int size)
 	input_mt_sync_frame(sc->touchpad);
 
 	input_sync(sc->touchpad);
+	return 0;
+}
+
+static int rb3_pro_instrument_raw_event(struct sony_sc *sc, u8 *rd, int size)
+{
+	/* Rock Band 3 PS3 Pro instruments set rd[24] to 0xE0 when they're
+	 * sending full reports, and 0x02 when only sending navigation.
+	 */
+	if (size < 25 || rd[24] != 0x02)
+		return 0;
+
+	/* Only attempt to enable full report every 8 seconds */
+	if (time_after(jiffies, sc->rb3_pro_poke_jiffies)) {
+		sc->rb3_pro_poke_jiffies = jiffies + secs_to_jiffies(8);
+		rb3_pro_instrument_enable_full_report(sc);
+	}
+
+	return 0;
 }
 
-static void rb4_ps4_guitar_parse_report(struct sony_sc *sc, u8 *rd, int size)
+static int rb4_ps4_guitar_raw_event(struct sony_sc *sc, u8 *rd, int size)
 {
+	const int expected_size = (sc->quirks & RB4_GUITAR_PS4_BT) ? 78 : 64;
+
+	if (unlikely(size != expected_size || rd[0] != 0x01))
+		return 0;
+
 	/*
 	 * Rock Band 4 PS4 guitars have whammy and
 	 * tilt functionality, they're located at
@@ -1084,15 +1127,18 @@ static void rb4_ps4_guitar_parse_report(struct sony_sc *sc, u8 *rd, int size)
 	input_report_abs(sc->input_dev, ABS_RZ, rd[45]);
 
 	input_sync(sc->input_dev);
+	return 0;
 }
 
-static void rb4_ps5_guitar_parse_report(struct sony_sc *sc, u8 *rd, int size)
+static int rb4_ps5_guitar_raw_event(struct sony_sc *sc, u8 *rd, int size)
 {
 	u8 charging_status;
 	u8 battery_data;
 	u8 battery_capacity;
 	u8 battery_status;
-	unsigned long flags;
+
+	if (unlikely(size != 64 || rd[0] != 0x01))
+		return 0;
 
 	/*
 	 * Rock Band 4 PS5 guitars have whammy and
@@ -1132,73 +1178,30 @@ static void rb4_ps5_guitar_parse_report(struct sony_sc *sc, u8 *rd, int size)
 		break;
 	}
 
-	spin_lock_irqsave(&sc->lock, flags);
-	sc->battery_capacity = battery_capacity;
-	sc->battery_status = battery_status;
-	spin_unlock_irqrestore(&sc->lock, flags);
+	scoped_guard(spinlock_irqsave, &sc->lock) {
+		sc->battery_capacity = battery_capacity;
+		sc->battery_status = battery_status;
+	}
 
 	input_sync(sc->input_dev);
+	return 0;
 }
 
 static int sony_raw_event(struct hid_device *hdev, struct hid_report *report,
 		u8 *rd, int size)
 {
 	struct sony_sc *sc = hid_get_drvdata(hdev);
+	int ret;
 
-	/*
-	 * Sixaxis HID report has acclerometers/gyro with MSByte first, this
-	 * has to be BYTE_SWAPPED before passing up to joystick interface
-	 */
-	if ((sc->quirks & SIXAXIS_CONTROLLER) && rd[0] == 0x01 && size == 49) {
-		/*
-		 * When connected via Bluetooth the Sixaxis occasionally sends
-		 * a report with the second byte 0xff and the rest zeroed.
-		 *
-		 * This report does not reflect the actual state of the
-		 * controller must be ignored to avoid generating false input
-		 * events.
-		 */
-		if (rd[1] == 0xff)
-			return -EINVAL;
-
-		swap(rd[41], rd[42]);
-		swap(rd[43], rd[44]);
-		swap(rd[45], rd[46]);
-		swap(rd[47], rd[48]);
-
-		sixaxis_parse_report(sc, rd, size);
-	} else if ((sc->quirks & MOTION_CONTROLLER_BT) && rd[0] == 0x01 && size == 49) {
-		sixaxis_parse_report(sc, rd, size);
-	} else if ((sc->quirks & NAVIGATION_CONTROLLER) && rd[0] == 0x01 && size == 49) {
-		sixaxis_parse_report(sc, rd, size);
-	} else if ((sc->quirks & NSG_MRXU_REMOTE) && rd[0] == 0x02 && size >= 12) {
-		nsg_mrxu_parse_report(sc, rd, size);
-		return 1;
-	} else if ((sc->quirks & RB4_GUITAR_PS4_USB) && rd[0] == 0x01 && size == 64) {
-		rb4_ps4_guitar_parse_report(sc, rd, size);
-		return 1;
-	} else if ((sc->quirks & RB4_GUITAR_PS4_BT) && rd[0] == 0x01 && size == 78) {
-		rb4_ps4_guitar_parse_report(sc, rd, size);
-		return 1;
-	} else if ((sc->quirks & RB4_GUITAR_PS5) && rd[0] == 0x01 && size == 64) {
-		rb4_ps5_guitar_parse_report(sc, rd, size);
-		return 1;
-	}
-
-	/* Rock Band 3 PS3 Pro instruments set rd[24] to 0xE0 when they're
-	 * sending full reports, and 0x02 when only sending navigation.
-	 */
-	if ((sc->quirks & RB3_PRO_INSTRUMENT) && size >= 25 && rd[24] == 0x02) {
-		/* Only attempt to enable full report every 8 seconds */
-		if (time_after(jiffies, sc->rb3_pro_poke_jiffies)) {
-			sc->rb3_pro_poke_jiffies = jiffies + secs_to_jiffies(8);
-			rb3_pro_instrument_enable_full_report(sc);
-		}
+	if (sc->raw_event) {
+		ret = sc->raw_event(sc, rd, size);
+		if (unlikely(ret < 0))
+			return ret;
 	}
 
-	if (sc->defer_initialization) {
+	if (unlikely(sc->defer_initialization)) {
 		sc->defer_initialization = 0;
-		sony_schedule_work(sc, SONY_WORKER_STATE);
+		sony_schedule_work(sc);
 	}
 
 	return 0;
@@ -1256,7 +1259,7 @@ static int sony_mapping(struct hid_device *hdev, struct hid_input *hi,
 	if (sc->quirks & DJH_TURNTABLE)
 		return djh_turntable_mapping(hdev, hi, field, usage, bit, max);
 
-	if (sc->quirks & (RB4_GUITAR_PS4_USB | RB4_GUITAR_PS4_BT))
+	if (sc->quirks & RB4_GUITAR_PS4)
 		return rb4_guitar_mapping(hdev, hi, field, usage, bit, max);
 
 	if (sc->quirks & RB4_GUITAR_PS5)
@@ -1269,8 +1272,6 @@ static int sony_mapping(struct hid_device *hdev, struct hid_input *hi,
 static int sony_register_touchpad(struct sony_sc *sc, int touch_count,
 		int w, int h, int touch_major, int touch_minor, int orientation)
 {
-	size_t name_sz;
-	char *name;
 	int ret;
 
 	sc->touchpad = devm_input_allocate_device(&sc->hdev->dev);
@@ -1292,12 +1293,10 @@ static int sony_register_touchpad(struct sony_sc *sc, int touch_count,
 	 * a suffix. Other devices which were added later like Sony TV remotes
 	 * inhirited this suffix.
 	 */
-	name_sz = strlen(sc->hdev->name) + sizeof(TOUCHPAD_SUFFIX);
-	name = devm_kzalloc(&sc->hdev->dev, name_sz, GFP_KERNEL);
-	if (!name)
+	sc->touchpad->name = devm_kasprintf(&sc->hdev->dev, GFP_KERNEL, "%s" TOUCHPAD_SUFFIX,
+										sc->hdev->name);
+	if (!sc->touchpad->name)
 		return -ENOMEM;
-	snprintf(name, name_sz, "%s" TOUCHPAD_SUFFIX, sc->hdev->name);
-	sc->touchpad->name = name;
 
 	/* We map the button underneath the touchpad to BTN_LEFT. */
 	__set_bit(EV_KEY, sc->touchpad->evbit);
@@ -1334,8 +1333,6 @@ static int sony_register_touchpad(struct sony_sc *sc, int touch_count,
 
 static int sony_register_sensors(struct sony_sc *sc)
 {
-	size_t name_sz;
-	char *name;
 	int ret;
 
 	sc->sensor_dev = devm_input_allocate_device(&sc->hdev->dev);
@@ -1354,12 +1351,10 @@ static int sony_register_sensors(struct sony_sc *sc)
 	/* Append a suffix to the controller name as there are various
 	 * DS4 compatible non-Sony devices with different names.
 	 */
-	name_sz = strlen(sc->hdev->name) + sizeof(SENSOR_SUFFIX);
-	name = devm_kzalloc(&sc->hdev->dev, name_sz, GFP_KERNEL);
-	if (!name)
+	sc->sensor_dev->name = devm_kasprintf(&sc->hdev->dev, GFP_KERNEL, "%s" SENSOR_SUFFIX,
+										sc->hdev->name);
+	if (!sc->sensor_dev->name)
 		return -ENOMEM;
-	snprintf(name, name_sz, "%s" SENSOR_SUFFIX, sc->hdev->name);
-	sc->sensor_dev->name = name;
 
 	if (sc->quirks & SIXAXIS_CONTROLLER) {
 		/* For the DS3 we only support the accelerometer, which works
@@ -1507,7 +1502,7 @@ static void buzz_set_leds(struct sony_sc *sc)
 static void sony_set_leds(struct sony_sc *sc)
 {
 	if (!(sc->quirks & BUZZ_CONTROLLER))
-		sony_schedule_work(sc, SONY_WORKER_STATE);
+		sony_schedule_work(sc);
 	else
 		buzz_set_leds(sc);
 }
@@ -1618,7 +1613,7 @@ static int sony_led_blink_set(struct led_classdev *led, unsigned long *delay_on,
 		new_off != drv_data->led_delay_off[n]) {
 		drv_data->led_delay_on[n] = new_on;
 		drv_data->led_delay_off[n] = new_off;
-		sony_schedule_work(drv_data, SONY_WORKER_STATE);
+		sony_schedule_work(drv_data);
 	}
 
 	return 0;
@@ -1846,7 +1841,7 @@ static int sony_play_effect(struct input_dev *dev, void *data,
 	sc->left = effect->u.rumble.strong_magnitude / 256;
 	sc->right = effect->u.rumble.weak_magnitude / 256;
 
-	sony_schedule_work(sc, SONY_WORKER_STATE);
+	sony_schedule_work(sc);
 	return 0;
 }
 
@@ -1869,15 +1864,14 @@ static int sony_battery_get_property(struct power_supply *psy,
 				     union power_supply_propval *val)
 {
 	struct sony_sc *sc = power_supply_get_drvdata(psy);
-	unsigned long flags;
 	int ret = 0;
 	u8 battery_capacity;
 	int battery_status;
 
-	spin_lock_irqsave(&sc->lock, flags);
-	battery_capacity = sc->battery_capacity;
-	battery_status = sc->battery_status;
-	spin_unlock_irqrestore(&sc->lock, flags);
+	scoped_guard(spinlock_irqsave, &sc->lock) {
+		battery_capacity = sc->battery_capacity;
+		battery_status = sc->battery_status;
+	}
 
 	switch (psp) {
 	case POWER_SUPPLY_PROP_PRESENT:
@@ -1959,10 +1953,9 @@ static inline int sony_compare_connection_type(struct sony_sc *sc0,
 static int sony_check_add_dev_list(struct sony_sc *sc)
 {
 	struct sony_sc *entry;
-	unsigned long flags;
 	int ret;
 
-	spin_lock_irqsave(&sony_dev_list_lock, flags);
+	guard(spinlock_irqsave)(&sony_dev_list_lock);
 
 	list_for_each_entry(entry, &sony_device_list, list_node) {
 		ret = memcmp(sc->mac_address, entry->mac_address,
@@ -1976,26 +1969,23 @@ static int sony_check_add_dev_list(struct sony_sc *sc)
 				"controller with MAC address %pMR already connected\n",
 				sc->mac_address);
 			}
-			goto unlock;
+			goto out;
 		}
 	}
 
 	ret = 0;
 	list_add(&(sc->list_node), &sony_device_list);
 
-unlock:
-	spin_unlock_irqrestore(&sony_dev_list_lock, flags);
+out:
 	return ret;
 }
 
 static void sony_remove_dev_list(struct sony_sc *sc)
 {
-	unsigned long flags;
-
 	if (sc->list_node.next) {
-		spin_lock_irqsave(&sony_dev_list_lock, flags);
-		list_del(&(sc->list_node));
-		spin_unlock_irqrestore(&sony_dev_list_lock, flags);
+		scoped_guard(spinlock_irqsave, &sony_dev_list_lock) {
+			list_del(&(sc->list_node));
+		}
 	}
 }
 
@@ -2110,6 +2100,12 @@ static void sony_release_device_id(struct sony_sc *sc)
 	}
 }
 
+static inline void sony_init_raw_event_handler(struct sony_sc *sc,
+				int (*raw_event)(struct sony_sc *, u8 *, int))
+{
+	sc->raw_event = raw_event;
+}
+
 static inline void sony_init_output_report(struct sony_sc *sc,
 				void (*send_output_report)(struct sony_sc *))
 {
@@ -2123,12 +2119,10 @@ static inline void sony_init_output_report(struct sony_sc *sc,
 
 static inline void sony_cancel_work_sync(struct sony_sc *sc)
 {
-	unsigned long flags;
-
 	if (sc->state_worker_initialized) {
-		spin_lock_irqsave(&sc->lock, flags);
-		sc->state_worker_initialized = 0;
-		spin_unlock_irqrestore(&sc->lock, flags);
+		scoped_guard(spinlock_irqsave, &sc->lock) {
+			sc->state_worker_initialized = 0;
+		}
 		cancel_work_sync(&sc->state_worker);
 	}
 }
@@ -2185,6 +2179,7 @@ static int sony_input_configured(struct hid_device *hdev,
 			goto err_stop;
 		}
 
+		sony_init_raw_event_handler(sc, sixaxis_raw_event);
 		sony_init_output_report(sc, sixaxis_send_output_report);
 	} else if (sc->quirks & NAVIGATION_CONTROLLER_BT) {
 		/*
@@ -2199,6 +2194,7 @@ static int sony_input_configured(struct hid_device *hdev,
 			goto err_stop;
 		}
 
+		sony_init_raw_event_handler(sc, sixaxis_raw_event);
 		sony_init_output_report(sc, sixaxis_send_output_report);
 	} else if (sc->quirks & RB3_PRO_INSTRUMENT) {
 		/*
@@ -2213,6 +2209,8 @@ static int sony_input_configured(struct hid_device *hdev,
 		 */
 		hdev->quirks |= HID_QUIRK_NO_OUTPUT_REPORTS_ON_INTR_EP;
 		hdev->quirks |= HID_QUIRK_SKIP_OUTPUT_REPORT_ID;
+
+		sony_init_raw_event_handler(sc, rb3_pro_instrument_raw_event);
 	} else if (sc->quirks & SIXAXIS_CONTROLLER_USB) {
 		/*
 		 * The Sony Sixaxis does not handle HID Output Reports on the
@@ -2237,6 +2235,7 @@ static int sony_input_configured(struct hid_device *hdev,
 			goto err_stop;
 		}
 
+		sony_init_raw_event_handler(sc, sixaxis_raw_event);
 		sony_init_output_report(sc, sixaxis_send_output_report);
 	} else if (sc->quirks & SIXAXIS_CONTROLLER_BT) {
 		/*
@@ -2258,6 +2257,7 @@ static int sony_input_configured(struct hid_device *hdev,
 			goto err_stop;
 		}
 
+		sony_init_raw_event_handler(sc, sixaxis_raw_event);
 		sony_init_output_report(sc, sixaxis_send_output_report);
 	} else if (sc->quirks & NSG_MRXU_REMOTE) {
 		/*
@@ -2273,8 +2273,15 @@ static int sony_input_configured(struct hid_device *hdev,
 			goto err_stop;
 		}
 
+		sony_init_raw_event_handler(sc, nsg_mrxu_raw_event);
 	} else if (sc->quirks & MOTION_CONTROLLER) {
+		if (sc->quirks & MOTION_CONTROLLER_BT)
+			sony_init_raw_event_handler(sc, sixaxis_raw_event);
 		sony_init_output_report(sc, motion_send_output_report);
+	} else if (sc->quirks & RB4_GUITAR_PS4) {
+		sony_init_raw_event_handler(sc, rb4_ps4_guitar_raw_event);
+	} else if (sc->quirks & RB4_GUITAR_PS5) {
+		sony_init_raw_event_handler(sc, rb4_ps5_guitar_raw_event);
 	}
 
 	if (sc->quirks & SONY_LED_SUPPORT) {
diff --git a/drivers/hid/hid-steelseries.c b/drivers/hid/hid-steelseries.c
index f98435631aa18..73f77dd071103 100644
--- a/drivers/hid/hid-steelseries.c
+++ b/drivers/hid/hid-steelseries.c
@@ -10,16 +10,30 @@
  */
 
 #include <linux/device.h>
+#include <linux/dmi.h>
 #include <linux/hid.h>
 #include <linux/module.h>
 #include <linux/usb.h>
 #include <linux/leds.h>
+#include <linux/led-class-multicolor.h>
+#include <linux/slab.h>
 
 #include "hid-ids.h"
 
 #define STEELSERIES_SRWS1		BIT(0)
 #define STEELSERIES_ARCTIS_1		BIT(1)
 #define STEELSERIES_ARCTIS_9		BIT(2)
+#define STEELSERIES_MSI_RGB		BIT(3)
+
+#define STEELSERIES_MSI_RGB_WVALUE 0x0300 /* Feature report, ID 0 */
+#define STEELSERIES_MSI_RGB_REPORT_LEN 524
+#define STEELSERIES_MSI_RGB_OPCODE 0x0c
+#define STEELSERIES_MSI_RGB_KLC_MODE 0x66
+#define STEELSERIES_MSI_RGB_ALC_MODE 0x06
+
+#define STEELSERIES_HAS_LEDS_MULTICOLOR \
+	(IS_BUILTIN(CONFIG_LEDS_CLASS_MULTICOLOR) || \
+	 (IS_MODULE(CONFIG_LEDS_CLASS_MULTICOLOR) && IS_MODULE(CONFIG_HID_STEELSERIES)))
 
 struct steelseries_device {
 	struct hid_device *hdev;
@@ -34,6 +48,14 @@ struct steelseries_device {
 	uint8_t battery_capacity;
 	bool headset_connected;
 	bool battery_charging;
+	bool battery_registered;
+
+#if STEELSERIES_HAS_LEDS_MULTICOLOR
+	struct led_classdev_mc mc_cdev;
+	struct mc_subled subled_info[3];
+	struct mutex rgb_lock; /* protects rgb_buf */
+	u8 *rgb_buf;
+#endif
 };
 
 #if IS_BUILTIN(CONFIG_LEDS_CLASS) || \
@@ -510,6 +532,8 @@ static int steelseries_headset_battery_register(struct steelseries_device *sd)
 	power_supply_powers(sd->battery, &sd->hdev->dev);
 
 	INIT_DELAYED_WORK(&sd->battery_work, steelseries_headset_battery_timer_tick);
+	/* Pairs with smp_load_acquire() in raw_event and remove paths */
+	smp_store_release(&sd->battery_registered, true);
 	steelseries_headset_fetch_battery(sd->hdev);
 
 	if (sd->quirks & STEELSERIES_ARCTIS_9) {
@@ -523,11 +547,230 @@ static int steelseries_headset_battery_register(struct steelseries_device *sd)
 
 static bool steelseries_is_vendor_usage_page(struct hid_device *hdev, uint8_t usage_page)
 {
+	if (hdev->rsize < 3)
+		return false;
+
 	return hdev->rdesc[0] == 0x06 &&
 		hdev->rdesc[1] == usage_page &&
 		hdev->rdesc[2] == 0xff;
 }
 
+static const struct dmi_system_id steelseries_msi_rgb_dmi_table[] = {
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Micro-Star International Co., Ltd."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Raider A18 HX A9WJG"),
+			DMI_MATCH(DMI_BOARD_NAME, "MS-182L"),
+		},
+	},
+	{ }
+};
+
+static struct usb_interface *steelseries_hid_to_usb_intf(struct hid_device *hdev)
+{
+	if (!hid_is_usb(hdev))
+		return NULL;
+
+	return to_usb_interface(hdev->dev.parent);
+}
+
+static bool steelseries_msi_rgb_is_interface0(struct hid_device *hdev)
+{
+	struct usb_interface *intf = steelseries_hid_to_usb_intf(hdev);
+	struct usb_device *udev;
+
+	if (!intf)
+		return false;
+
+	udev = interface_to_usbdev(intf);
+
+	return intf == usb_ifnum_to_if(udev, 0);
+}
+
+#if STEELSERIES_HAS_LEDS_MULTICOLOR
+
+static struct usb_device *steelseries_hid_to_usb_dev(struct hid_device *hdev)
+{
+	struct usb_interface *intf = steelseries_hid_to_usb_intf(hdev);
+
+	if (!intf)
+		return NULL;
+
+	return interface_to_usbdev(intf);
+}
+
+static int steelseries_msi_rgb_set_blocking(struct led_classdev *led_cdev,
+					    enum led_brightness brightness)
+{
+	struct led_classdev_mc *mc_cdev = lcdev_to_mccdev(led_cdev);
+	struct steelseries_device *sd = container_of(mc_cdev,
+						    struct steelseries_device,
+						    mc_cdev);
+	struct hid_device *hdev = sd->hdev;
+	struct usb_device *udev = steelseries_hid_to_usb_dev(hdev);
+	int i, ret;
+	u8 r, g, b;
+
+	static const u8 keys[] = {
+		0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b,
+		0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13,
+		0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b,
+		0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23,
+		0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b,
+		0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x33, 0x34,
+		0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c,
+		0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44,
+		0x45, 0x46, 0x47, 0x49, 0x4b, 0x4c, 0x4e, 0x4f,
+		0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57,
+		0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f,
+		0x60, 0x61, 0x62, 0x63, 0x64, 0x66, 0xe0, 0xe1,
+		0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xf0
+	};
+	static const u8 alc_zones[] = { 0x00, 0x01, 0x02, 0x03 };
+
+	if (!udev)
+		return -ENODEV;
+
+	mutex_lock(&sd->rgb_lock);
+
+	led_mc_calc_color_components(mc_cdev, brightness);
+
+	r = mc_cdev->subled_info[0].brightness;
+	g = mc_cdev->subled_info[1].brightness;
+	b = mc_cdev->subled_info[2].brightness;
+
+	/*
+	 * Report layout (524 bytes):
+	 * Byte 0: Opcode (0x0c)
+	 * Byte 1: 0x00
+	 * Byte 2: Mode (0x66 for Keyboard, 0x06 for Lightbar)
+	 * Byte 3: 0x00
+	 * Bytes 4+: 4-byte chunks per LED (Index, R, G, B)
+	 */
+	memset(sd->rgb_buf, 0, STEELSERIES_MSI_RGB_REPORT_LEN);
+	sd->rgb_buf[0] = STEELSERIES_MSI_RGB_OPCODE;
+	sd->rgb_buf[1] = 0x00;
+	sd->rgb_buf[3] = 0x00;
+
+	for (i = 0; i < (STEELSERIES_MSI_RGB_REPORT_LEN - 4) / 4; i++)
+		sd->rgb_buf[4 + i * 4] = 0xff;
+
+	if (hdev->product == USB_DEVICE_ID_STEELSERIES_MSI_KLC) {
+		sd->rgb_buf[2] = STEELSERIES_MSI_RGB_KLC_MODE;
+		for (i = 0; i < ARRAY_SIZE(keys); i++) {
+			sd->rgb_buf[4 + i * 4] = keys[i];
+			sd->rgb_buf[5 + i * 4] = r;
+			sd->rgb_buf[6 + i * 4] = g;
+			sd->rgb_buf[7 + i * 4] = b;
+		}
+	} else {
+		sd->rgb_buf[2] = STEELSERIES_MSI_RGB_ALC_MODE;
+		for (i = 0; i < ARRAY_SIZE(alc_zones); i++) {
+			sd->rgb_buf[4 + i * 4] = alc_zones[i];
+			sd->rgb_buf[5 + i * 4] = r;
+			sd->rgb_buf[6 + i * 4] = g;
+			sd->rgb_buf[7 + i * 4] = b;
+		}
+	}
+
+	/*
+	 * Send the vendor report verbatim with usb_control_msg(): byte 0 is a
+	 * protocol opcode (0x0c), not a HID report ID, and the controller
+	 * expects it under report ID 0 (wValue 0x0300). hid_hw_raw_request()
+	 * would write the report number into byte 0, so the direct control
+	 * transfer is used to keep the payload byte-identical to the tested
+	 * userspace implementation.
+	 */
+	ret = hid_hw_power(hdev, PM_HINT_FULLON);
+	if (ret < 0)
+		goto out_unlock;
+
+	ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
+			      HID_REQ_SET_REPORT,
+			      USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE,
+			      STEELSERIES_MSI_RGB_WVALUE, 0,
+			      sd->rgb_buf, STEELSERIES_MSI_RGB_REPORT_LEN,
+			      USB_CTRL_SET_TIMEOUT);
+
+	hid_hw_power(hdev, PM_HINT_NORMAL);
+
+out_unlock:
+	mutex_unlock(&sd->rgb_lock);
+	return ret < 0 ? ret : 0;
+}
+
+static void steelseries_msi_rgb_free_buf(void *data)
+{
+	kfree(data);
+}
+
+static int steelseries_msi_rgb_register(struct steelseries_device *sd)
+{
+	struct hid_device *hdev = sd->hdev;
+	struct led_classdev *led_cdev;
+	int ret;
+
+	sd->rgb_buf = kzalloc(STEELSERIES_MSI_RGB_REPORT_LEN, GFP_KERNEL);
+	if (!sd->rgb_buf)
+		return -ENOMEM;
+
+	ret = devm_add_action_or_reset(&hdev->dev,
+				       steelseries_msi_rgb_free_buf,
+				       sd->rgb_buf);
+	if (ret) {
+		sd->rgb_buf = NULL;
+		return ret;
+	}
+
+	ret = devm_mutex_init(&hdev->dev, &sd->rgb_lock);
+	if (ret) {
+		devm_remove_action(&hdev->dev, steelseries_msi_rgb_free_buf,
+				   sd->rgb_buf);
+		kfree(sd->rgb_buf);
+		sd->rgb_buf = NULL;
+		return ret;
+	}
+
+	sd->subled_info[0].color_index = LED_COLOR_ID_RED;
+	sd->subled_info[1].color_index = LED_COLOR_ID_GREEN;
+	sd->subled_info[2].color_index = LED_COLOR_ID_BLUE;
+	sd->subled_info[0].intensity = 255;
+	sd->subled_info[1].intensity = 255;
+	sd->subled_info[2].intensity = 255;
+	sd->subled_info[0].channel = 0;
+	sd->subled_info[1].channel = 1;
+	sd->subled_info[2].channel = 2;
+
+	sd->mc_cdev.subled_info = sd->subled_info;
+	sd->mc_cdev.num_colors = 3;
+
+	led_cdev = &sd->mc_cdev.led_cdev;
+	if (hdev->product == USB_DEVICE_ID_STEELSERIES_MSI_KLC)
+		led_cdev->name = "steelseries::kbd_backlight";
+	else
+		led_cdev->name = "steelseries::lightbar";
+
+	led_cdev->max_brightness = 255;
+	led_cdev->brightness_set_blocking = steelseries_msi_rgb_set_blocking;
+
+	ret = devm_led_classdev_multicolor_register(&hdev->dev, &sd->mc_cdev);
+	if (ret) {
+		devm_remove_action(&hdev->dev, steelseries_msi_rgb_free_buf,
+				   sd->rgb_buf);
+		kfree(sd->rgb_buf);
+		sd->rgb_buf = NULL;
+		return ret;
+	}
+
+	return 0;
+}
+#else
+static int steelseries_msi_rgb_register(struct steelseries_device *sd)
+{
+	return -ENODEV;
+}
+#endif
+
 static int steelseries_probe(struct hid_device *hdev, const struct hid_device_id *id)
 {
 	struct steelseries_device *sd;
@@ -549,6 +792,14 @@ static int steelseries_probe(struct hid_device *hdev, const struct hid_device_id
 	sd->hdev = hdev;
 	sd->quirks = id->driver_data;
 
+	if (sd->quirks & STEELSERIES_MSI_RGB) {
+		if (!dmi_check_system(steelseries_msi_rgb_dmi_table) ||
+		    !steelseries_msi_rgb_is_interface0(hdev)) {
+			hid_dbg(hdev, "MSI RGB quirk not applicable, using generic HID path\n");
+			sd->quirks &= ~STEELSERIES_MSI_RGB;
+		}
+	}
+
 	ret = hid_parse(hdev);
 	if (ret)
 		return ret;
@@ -565,12 +816,28 @@ static int steelseries_probe(struct hid_device *hdev, const struct hid_device_id
 
 	ret = hid_hw_open(hdev);
 	if (ret)
-		return ret;
+		goto err_stop;
+
+	if (sd->quirks & STEELSERIES_MSI_RGB) {
+		ret = steelseries_msi_rgb_register(sd);
+		if (ret) {
+			hid_warn(hdev,
+				 "Failed to register MSI RGB LEDs: %d, continuing without RGB support\n",
+				 ret);
+			sd->quirks &= ~STEELSERIES_MSI_RGB;
+		}
+		return 0;
+	}
 
-	if (steelseries_headset_battery_register(sd) < 0)
+	if ((sd->quirks & (STEELSERIES_ARCTIS_1 | STEELSERIES_ARCTIS_9)) &&
+	    steelseries_headset_battery_register(sd) < 0)
 		hid_err(sd->hdev,
 			"Failed to register battery for headset\n");
 
+	return 0;
+
+err_stop:
+	hid_hw_stop(hdev);
 	return ret;
 }
 
@@ -588,12 +855,16 @@ static void steelseries_remove(struct hid_device *hdev)
 	}
 
 	sd = hid_get_drvdata(hdev);
+	if (!sd)
+		return;
 
 	spin_lock_irqsave(&sd->lock, flags);
 	sd->removed = true;
 	spin_unlock_irqrestore(&sd->lock, flags);
 
-	cancel_delayed_work_sync(&sd->battery_work);
+	/* Pairs with smp_store_release() in steelseries_headset_battery_register() */
+	if (smp_load_acquire(&sd->battery_registered))
+		cancel_delayed_work_sync(&sd->battery_work);
 
 	hid_hw_close(hdev);
 	hid_hw_stop(hdev);
@@ -624,20 +895,34 @@ static uint8_t steelseries_headset_map_capacity(uint8_t capacity, uint8_t min_in
 	return (capacity - min_in) * 100 / (max_in - min_in);
 }
 
+static bool steelseries_is_headset(struct hid_device *hdev)
+{
+	return hdev->product == USB_DEVICE_ID_STEELSERIES_ARCTIS_1 ||
+	       hdev->product == USB_DEVICE_ID_STEELSERIES_ARCTIS_9;
+}
+
 static int steelseries_headset_raw_event(struct hid_device *hdev,
 					struct hid_report *report, u8 *read_buf,
 					int size)
 {
-	struct steelseries_device *sd = hid_get_drvdata(hdev);
-	int capacity = sd->battery_capacity;
-	bool connected = sd->headset_connected;
-	bool charging = sd->battery_charging;
+	struct steelseries_device *sd;
+	int capacity;
+	bool connected;
+	bool charging;
 	unsigned long flags;
 
-	/* Not a headset */
-	if (hdev->product == USB_DEVICE_ID_STEELSERIES_SRWS1)
+	if (!steelseries_is_headset(hdev))
 		return 0;
 
+	sd = hid_get_drvdata(hdev);
+	/* Pairs with smp_store_release() in steelseries_headset_battery_register() */
+	if (!sd || !smp_load_acquire(&sd->battery_registered))
+		return 0;
+
+	capacity = sd->battery_capacity;
+	connected = sd->headset_connected;
+	charging = sd->battery_charging;
+
 	if (hdev->product == USB_DEVICE_ID_STEELSERIES_ARCTIS_1) {
 		hid_dbg(sd->hdev,
 			"Parsing raw event for Arctis 1 headset (%*ph)\n", size, read_buf);
@@ -732,6 +1017,16 @@ static const struct hid_device_id steelseries_devices[] = {
 	  HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_ARCTIS_9),
 	  .driver_data = STEELSERIES_ARCTIS_9 },
 
+#if STEELSERIES_HAS_LEDS_MULTICOLOR
+	{ /* MSI Raider A18 KLC */
+	  HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_MSI_KLC),
+	  .driver_data = STEELSERIES_MSI_RGB },
+
+	{ /* MSI Raider A18 ALC */
+	  HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_MSI_ALC),
+	  .driver_data = STEELSERIES_MSI_RGB },
+#endif
+
 	{ }
 };
 MODULE_DEVICE_TABLE(hid, steelseries_devices);
diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c
index da1f0ea85625d..a29bf051ada7c 100644
--- a/drivers/hid/wacom_wac.c
+++ b/drivers/hid/wacom_wac.c
@@ -1192,8 +1192,11 @@ static int int_dist(int x1, int y1, int x2, int y2)
 static void wacom_intuos_bt_process_data(struct wacom_wac *wacom,
 		unsigned char *data)
 {
-	memcpy(wacom->data, data, 10);
+	u8 *saved_data = wacom->data;
+
+	wacom->data = data;
 	wacom_intuos_irq(wacom);
+	wacom->data = saved_data;
 
 	input_sync(wacom->pen_input);
 	if (wacom->pad_input)
@@ -1202,7 +1205,7 @@ static void wacom_intuos_bt_process_data(struct wacom_wac *wacom,
 
 static int wacom_intuos_bt_irq(struct wacom_wac *wacom, size_t len)
 {
-	u8 *data = kmemdup(wacom->data, len, GFP_KERNEL);
+	u8 *data = wacom->data;
 	int i = 1;
 	unsigned power_raw, battery_capacity, bat_charging, ps_connected;
 
@@ -1242,7 +1245,6 @@ static int wacom_intuos_bt_irq(struct wacom_wac *wacom, size_t len)
 		break;
 	}
 
-	kfree(data);
 	return 0;
 }
 
diff --git a/drivers/hv/mshv_regions.c b/drivers/hv/mshv_regions.c
index 6d65e5b421529..dddaade31b5dc 100644
--- a/drivers/hv/mshv_regions.c
+++ b/drivers/hv/mshv_regions.c
@@ -381,46 +381,6 @@ int mshv_region_get(struct mshv_mem_region *region)
 	return kref_get_unless_zero(&region->mreg_refcount);
 }
 
-/**
- * mshv_region_hmm_fault_and_lock - Handle HMM faults and lock the memory region
- * @region: Pointer to the memory region structure
- * @range: Pointer to the HMM range structure
- *
- * This function performs the following steps:
- * 1. Reads the notifier sequence for the HMM range.
- * 2. Acquires a read lock on the memory map.
- * 3. Handles HMM faults for the specified range.
- * 4. Releases the read lock on the memory map.
- * 5. If successful, locks the memory region mutex.
- * 6. Verifies if the notifier sequence has changed during the operation.
- *    If it has, releases the mutex and returns -EBUSY to match with
- *    hmm_range_fault() return code for repeating.
- *
- * Return: 0 on success, a negative error code otherwise.
- */
-static int mshv_region_hmm_fault_and_lock(struct mshv_mem_region *region,
-					  struct hmm_range *range)
-{
-	int ret;
-
-	range->notifier_seq = mmu_interval_read_begin(range->notifier);
-	mmap_read_lock(region->mreg_mni.mm);
-	ret = hmm_range_fault(range);
-	mmap_read_unlock(region->mreg_mni.mm);
-	if (ret)
-		return ret;
-
-	mutex_lock(&region->mreg_mutex);
-
-	if (mmu_interval_read_retry(range->notifier, range->notifier_seq)) {
-		mutex_unlock(&region->mreg_mutex);
-		cond_resched();
-		return -EBUSY;
-	}
-
-	return 0;
-}
-
 /**
  * mshv_region_range_fault - Handle memory range faults for a given region.
  * @region: Pointer to the memory region structure.
@@ -452,13 +412,19 @@ static int mshv_region_range_fault(struct mshv_mem_region *region,
 	range.start = region->start_uaddr + page_offset * HV_HYP_PAGE_SIZE;
 	range.end = range.start + page_count * HV_HYP_PAGE_SIZE;
 
-	do {
-		ret = mshv_region_hmm_fault_and_lock(region, &range);
-	} while (ret == -EBUSY);
-
+again:
+	ret = hmm_range_fault_unlocked_timeout(&range, 0);
 	if (ret)
 		goto out;
 
+	mutex_lock(&region->mreg_mutex);
+
+	if (mmu_interval_read_retry(range.notifier, range.notifier_seq)) {
+		mutex_unlock(&region->mreg_mutex);
+		cond_resched();
+		goto again;
+	}
+
 	for (i = 0; i < page_count; i++)
 		region->mreg_pages[page_offset + i] = hmm_pfn_to_page(pfns[i]);
 
diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig
index 2bfbcc033d599..08c29685126ad 100644
--- a/drivers/hwmon/Kconfig
+++ b/drivers/hwmon/Kconfig
@@ -942,6 +942,19 @@ config SENSORS_JC42
 	  This driver can also be built as a module. If so, the module
 	  will be called jc42.
 
+config SENSORS_POLARFIRE_SOC_TVS
+	tristate "PolarFire SoC (MPFS) temperature and voltage sensor"
+	depends on POLARFIRE_SOC_SYSCONS || COMPILE_TEST
+	depends on MFD_SYSCON
+	help
+	  This driver adds support for the PolarFire SoC (MPFS) Temperature and
+	  Voltage Sensor.
+
+	  To compile this driver as a module, choose M here. the
+	  module will be called tvs-mpfs.
+
+	  If unsure, say N.
+
 config SENSORS_POWERZ
 	tristate "ChargerLAB POWER-Z USB-C tester"
 	depends on USB
@@ -2070,6 +2083,17 @@ config SENSORS_DME1737
 	  This driver can also be built as a module. If so, the module
 	  will be called dme1737.
 
+config SENSORS_EIC7700_PVT
+	tristate "Eswin EIC7700 Voltage, Temperature sensor driver"
+	depends on ARCH_ESWIN || COMPILE_TEST
+	select POLYNOMIAL
+	help
+	  If you say yes here you get support for Eswin EIC7700 PVT sensor
+	  embedded into the SoC.
+
+	  This driver can also be built as a module. If so, the module will be
+	  called eic7700-pvt.
+
 config SENSORS_EMC1403
 	tristate "SMSC EMC1403/23 thermal sensor"
 	depends on I2C
diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile
index 63effc0ab8d11..63809eeec2f49 100644
--- a/drivers/hwmon/Makefile
+++ b/drivers/hwmon/Makefile
@@ -72,6 +72,7 @@ obj-$(CONFIG_SENSORS_DME1737)	+= dme1737.o
 obj-$(CONFIG_SENSORS_DRIVETEMP)	+= drivetemp.o
 obj-$(CONFIG_SENSORS_DS620)	+= ds620.o
 obj-$(CONFIG_SENSORS_DS1621)	+= ds1621.o
+obj-$(CONFIG_SENSORS_EIC7700_PVT) += eic7700-pvt.o
 obj-$(CONFIG_SENSORS_EMC1403)	+= emc1403.o
 obj-$(CONFIG_SENSORS_EMC1812)	+= emc1812.o
 obj-$(CONFIG_SENSORS_EMC2103)	+= emc2103.o
@@ -197,6 +198,7 @@ obj-$(CONFIG_SENSORS_NZXT_SMART2) += nzxt-smart2.o
 obj-$(CONFIG_SENSORS_PC87360)	+= pc87360.o
 obj-$(CONFIG_SENSORS_PC87427)	+= pc87427.o
 obj-$(CONFIG_SENSORS_PCF8591)	+= pcf8591.o
+obj-$(CONFIG_SENSORS_POLARFIRE_SOC_TVS)  += tvs-mpfs.o
 obj-$(CONFIG_SENSORS_POWERZ)	+= powerz.o
 obj-$(CONFIG_SENSORS_POWR1220)  += powr1220.o
 obj-$(CONFIG_SENSORS_PROM21_XHCI)	+= prom21-xhci.o
diff --git a/drivers/hwmon/acpi_power_meter.c b/drivers/hwmon/acpi_power_meter.c
index 0c9b9f4180fb7..8a539e8d1334b 100644
--- a/drivers/hwmon/acpi_power_meter.c
+++ b/drivers/hwmon/acpi_power_meter.c
@@ -21,8 +21,6 @@
 #include <linux/platform_device.h>
 
 #define ACPI_POWER_METER_NAME		"power_meter"
-#define ACPI_POWER_METER_DEVICE_NAME	"Power Meter"
-#define ACPI_POWER_METER_CLASS		"pwr_meter_resource"
 
 #define NUM_SENSORS			17
 
@@ -877,7 +875,7 @@ static void acpi_power_meter_notify(acpi_handle handle, u32 event, void *data)
 		break;
 	}
 
-	acpi_bus_generate_netlink_event(ACPI_POWER_METER_CLASS,
+	acpi_bus_generate_netlink_event("pwr_meter_resource",
 					dev_name(&resource->acpi_dev->dev),
 					event, 0);
 }
@@ -899,8 +897,6 @@ static int acpi_power_meter_probe(struct platform_device *pdev)
 	resource->sensors_valid = 0;
 	resource->acpi_dev = device;
 	mutex_init(&resource->lock);
-	strscpy(acpi_device_name(device), ACPI_POWER_METER_DEVICE_NAME);
-	strscpy(acpi_device_class(device), ACPI_POWER_METER_CLASS);
 
 	platform_set_drvdata(pdev, resource);
 
diff --git a/drivers/hwmon/asus-ec-sensors.c b/drivers/hwmon/asus-ec-sensors.c
index 29a23484cbe7b..33df42cfb5750 100644
--- a/drivers/hwmon/asus-ec-sensors.c
+++ b/drivers/hwmon/asus-ec-sensors.c
@@ -386,6 +386,8 @@ static const struct ec_sensor_info sensors_family_intel_600[] = {
 	[ec_sensor_temp_t_sensor] =
 		EC_SENSOR("T_Sensor", hwmon_temp, 1, 0x00, 0x3d),
 	[ec_sensor_temp_vrm] = EC_SENSOR("VRM", hwmon_temp, 1, 0x00, 0x3e),
+	[ec_sensor_fan_cpu_opt] =
+		EC_SENSOR("CPU_Opt", hwmon_fan, 2, 0x00, 0xb0),
 	[ec_sensor_fan_water_flow] =
 		EC_SENSOR("Water_Flow", hwmon_fan, 2, 0x00, 0xbe),
 	[ec_sensor_temp_water_in] =
@@ -487,6 +489,14 @@ static const struct ec_board_info board_info_crosshair_x670e_hero = {
 	.family = family_amd_600_series,
 };
 
+static const struct ec_board_info board_info_crosshair_x870e_hero = {
+	.sensors = SENSOR_TEMP_CPU | SENSOR_TEMP_CPU_PACKAGE |
+		SENSOR_TEMP_MB | SENSOR_TEMP_VRM |
+		SENSOR_TEMP_T_SENSOR | SENSOR_FAN_CPU_OPT,
+	.mutex_path = ASUS_HW_ACCESS_MUTEX_SB_PCI0_SBRG_SIO1_MUT0,
+	.family = family_amd_800_series,
+};
+
 static const struct ec_board_info board_info_maximus_vi_hero = {
 	.sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB |
 		SENSOR_TEMP_T_SENSOR |
@@ -527,6 +537,14 @@ static const struct ec_board_info board_info_maximus_z790_extreme = {
 	.family = family_intel_700_series,
 };
 
+static const struct ec_board_info board_info_maximus_z790_hero = {
+	.sensors = SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM |
+		SENSOR_SET_TEMP_WATER | SENSOR_FAN_WATER_FLOW |
+		SENSOR_FAN_CPU_OPT,
+	.mutex_path = ASUS_HW_ACCESS_MUTEX_RMTW_ASMX,
+	.family = family_intel_700_series,
+};
+
 static const struct ec_board_info board_info_prime_x470_pro = {
 	.sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB |
 		SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM |
@@ -590,6 +608,13 @@ static const struct ec_board_info board_info_pro_art_x870E_creator_wifi = {
 	.family = family_amd_800_series,
 };
 
+static const struct ec_board_info board_info_pro_art_z690_creator_wifi = {
+	.sensors = SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM |
+		SENSOR_FAN_CPU_OPT,
+	.mutex_path = ASUS_HW_ACCESS_MUTEX_SB_PC00_LPCB_SIO1_MUT0,
+	.family = family_intel_600_series,
+};
+
 static const struct ec_board_info board_info_pro_ws_trx50_sage_wifi = {
 	/* Board also has a nct6798 */
 	.sensors = SENSOR_TEMP_CPU | SENSOR_TEMP_CPU_PACKAGE | SENSOR_TEMP_VRME |
@@ -859,6 +884,8 @@ static const struct dmi_system_id dmi_table[] = {
 					&board_info_pro_art_x670E_creator_wifi),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ProArt X870E-CREATOR WIFI",
 					&board_info_pro_art_x870E_creator_wifi),
+	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ProArt Z690-CREATOR WIFI",
+					&board_info_pro_art_z690_creator_wifi),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("Pro WS TRX50-SAGE WIFI",
 					&board_info_pro_ws_trx50_sage_wifi),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("Pro WS TRX50-SAGE WIFI A",
@@ -883,6 +910,8 @@ static const struct dmi_system_id dmi_table[] = {
 					&board_info_crosshair_x670e_gene),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR X670E HERO",
 					&board_info_crosshair_x670e_hero),
+	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR X870E HERO",
+					&board_info_crosshair_x870e_hero),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG MAXIMUS XI HERO",
 					&board_info_maximus_xi_hero),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG MAXIMUS XI HERO (WI-FI)",
@@ -893,6 +922,8 @@ static const struct dmi_system_id dmi_table[] = {
 					&board_info_maximus_z690_formula),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG MAXIMUS Z790 EXTREME",
 					&board_info_maximus_z790_extreme),
+	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG MAXIMUS Z790 HERO",
+					&board_info_maximus_z790_hero),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX B550-E GAMING",
 					&board_info_strix_b550_e_gaming),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX B550-I GAMING",
@@ -927,6 +958,8 @@ static const struct dmi_system_id dmi_table[] = {
 					&board_info_strix_x870_i_gaming_wifi),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX X870E-E GAMING WIFI",
 					&board_info_strix_x870e_e_gaming_wifi),
+	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX X870E-E GAMING WIFI7 R2",
+					&board_info_strix_x870e_e_gaming_wifi),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX X870E-H GAMING WIFI7",
 					&board_info_strix_x870e_h_gaming_wifi7),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX Z390-F GAMING",
diff --git a/drivers/hwmon/chipcap2.c b/drivers/hwmon/chipcap2.c
index 4aecf463180fd..086571d556b7e 100644
--- a/drivers/hwmon/chipcap2.c
+++ b/drivers/hwmon/chipcap2.c
@@ -22,6 +22,8 @@
 #include <linux/irq.h>
 #include <linux/module.h>
 #include <linux/regulator/consumer.h>
+#include <linux/mod_devicetable.h>
+#include <linux/property.h>
 
 #define CC2_START_CM			0xA0
 #define CC2_START_NOM			0x80
@@ -83,6 +85,7 @@ struct cc2_data {
 	struct i2c_client *client;
 	struct regulator *regulator;
 	const char *name;
+	const char *label;
 	int irq_ready;
 	int irq_low;
 	int irq_high;
@@ -449,6 +452,8 @@ static umode_t cc2_is_visible(const void *data, enum hwmon_sensor_types type,
 		switch (attr) {
 		case hwmon_humidity_input:
 			return 0444;
+		case hwmon_humidity_label:
+			return cc2->label ? 0444 : 0;
 		case hwmon_humidity_min_alarm:
 			return cc2->rh_alarm.low_alarm_visible ? 0444 : 0;
 		case hwmon_humidity_max_alarm:
@@ -466,6 +471,8 @@ static umode_t cc2_is_visible(const void *data, enum hwmon_sensor_types type,
 		switch (attr) {
 		case hwmon_temp_input:
 			return 0444;
+		case hwmon_temp_label:
+			return cc2->label ? 0444 : 0;
 		default:
 			return 0;
 		}
@@ -552,6 +559,16 @@ static int cc2_humidity_max_alarm_status(struct cc2_data *data, long *val)
 	return 0;
 }
 
+static int cc2_read_string(struct device *dev, enum hwmon_sensor_types type,
+			   u32 attr, int channel, const char **str)
+{
+	struct cc2_data *data = dev_get_drvdata(dev);
+
+	*str = data->label;
+
+	return 0;
+}
+
 static int cc2_read(struct device *dev, enum hwmon_sensor_types type, u32 attr,
 		    int channel, long *val)
 {
@@ -670,8 +687,9 @@ static int cc2_request_alarm_irqs(struct cc2_data *data, struct device *dev)
 }
 
 static const struct hwmon_channel_info *cc2_info[] = {
-	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT),
-	HWMON_CHANNEL_INFO(humidity, HWMON_H_INPUT | HWMON_H_MIN | HWMON_H_MAX |
+	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT | HWMON_T_LABEL),
+	HWMON_CHANNEL_INFO(humidity, HWMON_H_INPUT | HWMON_H_LABEL |
+			   HWMON_H_MIN | HWMON_H_MAX |
 			   HWMON_H_MIN_HYST | HWMON_H_MAX_HYST |
 			   HWMON_H_MIN_ALARM | HWMON_H_MAX_ALARM),
 	NULL
@@ -680,6 +698,7 @@ static const struct hwmon_channel_info *cc2_info[] = {
 static const struct hwmon_ops cc2_hwmon_ops = {
 	.is_visible = cc2_is_visible,
 	.read = cc2_read,
+	.read_string = cc2_read_string,
 	.write = cc2_write,
 };
 
@@ -710,6 +729,8 @@ static int cc2_probe(struct i2c_client *client)
 		return dev_err_probe(dev, PTR_ERR(data->regulator),
 				     "Failed to get regulator\n");
 
+	device_property_read_string(dev, "label", &data->label);
+
 	ret = cc2_request_ready_irq(data, dev);
 	if (ret)
 		return dev_err_probe(dev, ret, "Failed to request ready irq\n");
diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c
index 6215ea49faaa9..d47ba64788e1f 100644
--- a/drivers/hwmon/coretemp.c
+++ b/drivers/hwmon/coretemp.c
@@ -68,7 +68,7 @@ enum coretemp_attr_index {
  *		This value is passed as "id" field to rdmsr/wrmsr functions.
  * @status_reg: One of IA32_THERM_STATUS or IA32_PACKAGE_THERM_STATUS,
  *		from where the temperature values should be read.
- * @attr_size:  Total number of pre-core attrs displayed in the sysfs.
+ * @attr_size:  Number of sysfs attributes in this temperature data group.
  */
 struct temp_data {
 	int temp;
diff --git a/drivers/hwmon/corsair-cpro.c b/drivers/hwmon/corsair-cpro.c
index b6e508e43fa17..8354a002f4c5e 100644
--- a/drivers/hwmon/corsair-cpro.c
+++ b/drivers/hwmon/corsair-cpro.c
@@ -645,6 +645,7 @@ static int ccp_probe(struct hid_device *hdev, const struct hid_device_id *id)
 
 out_hw_close:
 	hid_hw_close(hdev);
+	hid_device_io_stop(hdev);
 out_hw_stop:
 	hid_hw_stop(hdev);
 	return ret;
diff --git a/drivers/hwmon/corsair-psu.c b/drivers/hwmon/corsair-psu.c
index 76f3e1da68d09..ce958cdaef58a 100644
--- a/drivers/hwmon/corsair-psu.c
+++ b/drivers/hwmon/corsair-psu.c
@@ -822,6 +822,7 @@ static int corsairpsu_probe(struct hid_device *hdev, const struct hid_device_id
 
 fail_and_close:
 	hid_hw_close(hdev);
+	hid_device_io_stop(hdev);
 fail_and_stop:
 	hid_hw_stop(hdev);
 	return ret;
diff --git a/drivers/hwmon/cros_ec_hwmon.c b/drivers/hwmon/cros_ec_hwmon.c
index 03bfcc40bb7c3..e91e2103a6c76 100644
--- a/drivers/hwmon/cros_ec_hwmon.c
+++ b/drivers/hwmon/cros_ec_hwmon.c
@@ -146,9 +146,17 @@ static bool cros_ec_hwmon_is_error_temp(u8 temp)
 	       temp == EC_TEMP_SENSOR_NOT_CALIBRATED;
 }
 
+/* This differs slightly from the variant in units.h to avoid rounding inconsistencies. */
+#define CROS_EC_HWMON_ABSOLUTE_ZERO_MILLICELSIUS (-273000)
+
+static long cros_ec_hwmon_kelvin_to_millicelsius(long t)
+{
+	return t * MILLIDEGREE_PER_DEGREE + CROS_EC_HWMON_ABSOLUTE_ZERO_MILLICELSIUS;
+}
+
 static long cros_ec_hwmon_temp_to_millicelsius(u8 temp)
 {
-	return kelvin_to_millicelsius((((long)temp) + EC_TEMP_SENSOR_OFFSET));
+	return cros_ec_hwmon_kelvin_to_millicelsius((((long)temp) + EC_TEMP_SENSOR_OFFSET));
 }
 
 static bool cros_ec_hwmon_attr_is_temp_threshold(u32 attr)
@@ -227,7 +235,7 @@ static int cros_ec_hwmon_read(struct device *dev, enum hwmon_sensor_types type,
 								cros_ec_hwmon_attr_to_thres(attr),
 								&threshold);
 			if (ret == 0)
-				*val = kelvin_to_millicelsius(threshold);
+				*val = cros_ec_hwmon_kelvin_to_millicelsius(threshold);
 		}
 	}
 
diff --git a/drivers/hwmon/eic7700-pvt.c b/drivers/hwmon/eic7700-pvt.c
new file mode 100644
index 0000000000000..d7403ab33f4e6
--- /dev/null
+++ b/drivers/hwmon/eic7700-pvt.c
@@ -0,0 +1,509 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ESWIN EIC7700 Voltage, Temperature sensor driver
+ *
+ * Copyright 2026, Beijing ESWIN Computing Technology Co., Ltd.
+ *
+ * Authors:
+ *   Yulin Lu <luyulin@eswincomputing.com>
+ *   Huan He <hehuan1@eswincomputing.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/polynomial.h>
+#include <linux/reset.h>
+#include "eic7700-pvt.h"
+
+static const struct pvt_sensor_info pvt_info[] = {
+	PVT_SENSOR_INFO(0, "Temperature", hwmon_temp, TEMP),
+	PVT_SENSOR_INFO(0, "Voltage", hwmon_in, VOLT),
+};
+
+static const char * const pvt_clk_names[PVT_CLK_NUM] = {"enable", "apb"};
+
+/*
+ * The original translation formulae of the temperature (in degrees of Celsius)
+ * to PVT data and vice-versa are following:
+ * N = 6.0818e-8*(T^4) +1.2873e-5*(T^3) + 7.2244e-3*(T^2) + 3.6484*(T^1) +
+ *     1.6198e2,
+ * T = -1.8439e-11*(N^4) + 8.0705e-8*(N^3) + -1.8501e-4*(N^2) +
+ *     3.2843e-1*(N^1) - 4.8690e1,
+ * where T = [-40, 125]C and N = [27, 771].
+ * They must be accordingly altered to be suitable for the integer arithmetics.
+ * The technique is called 'factor redistribution', which just makes sure the
+ * multiplications and divisions are made so to have a result of the operations
+ * within the integer numbers limit. In addition we need to translate the
+ * formulae to accept millidegrees of Celsius. Here what they look like after
+ * the alterations:
+ * N = (60818e-20*(T^4) + 12873e-14*(T^3) + 72244e-9*(T^2) + 36484e-3*T +
+ *     16198e2) / 1e4,
+ * T = -18439e-12*(N^4) + 80705e-9*(N^3) - 185010e-6*(N^2) + 328430e-3*N -
+ *     48690,
+ * where T = [-40000, 125000] mC and N = [27, 771].
+ */
+static const struct polynomial poly_N_to_temp = {
+	.total_divider = 1,
+	.terms = {
+		{4, -18439, 1000, 1},
+		{3, 80705, 1000, 1},
+		{2, -185010, 1000, 1},
+		{1, 328430, 1000, 1},
+		{0, -48690, 1, 1}
+	}
+};
+
+/*
+ * Similar alterations are performed for the voltage conversion equations.
+ * The original formulae are:
+ * N = 1.3905e3*V - 5.7685e2,
+ * V = (N + 5.7685e2) / 1.3905e3,
+ * where V = [0.72, 0.88] V and N = [424, 646].
+ * After the optimization they looks as follows:
+ * N = (13905e-3*V - 5768.5) / 10,
+ * V = (N * 10^5 / 13905 + 57685 * 10^3 / 13905) / 10.
+ * where V = [720, 880] mV and N = [424, 646].
+ */
+static const struct polynomial poly_N_to_volt = {
+	.total_divider = 10,
+	.terms = {
+		{1, 100000, 13905, 1},
+		{0, 57685000, 1, 13905}
+	}
+};
+
+static inline u32 eic7700_pvt_update(void __iomem *reg, u32 mask, u32 data)
+{
+	u32 old;
+
+	old = readl_relaxed(reg);
+	writel((old & ~mask) | (data & mask), reg);
+
+	return old & mask;
+}
+
+static inline void eic7700_pvt_set_mode(struct pvt_hwmon *pvt, u32 mode)
+{
+	u32 old;
+
+	mode = FIELD_PREP(PVT_MODE_MASK, mode);
+
+	old = eic7700_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, 0);
+	eic7700_pvt_update(pvt->regs + PVT_MODE, PVT_MODE_MASK, mode);
+	eic7700_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, old);
+}
+
+static inline void eic7700_pvt_set_trim(struct pvt_hwmon *pvt, u32 val)
+{
+	u32 old;
+
+	old = eic7700_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, 0);
+	writel(val, pvt->regs + PVT_TRIM);
+	eic7700_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, old);
+}
+
+static irqreturn_t eic7700_pvt_hard_isr(int irq, void *data)
+{
+	struct pvt_hwmon *pvt = data;
+	u32 stat, val;
+	int active;
+
+	if (IS_ENABLED(CONFIG_PM)) {
+		active = pm_runtime_get_if_active(pvt->dev);
+		if (active <= 0)
+			return IRQ_NONE;
+	}
+
+	stat = readl(pvt->regs + PVT_INT);
+	if (!(stat & PVT_INT_STAT)) {
+		if (IS_ENABLED(CONFIG_PM))
+			pm_runtime_put(pvt->dev);
+		return IRQ_NONE;
+	}
+
+	eic7700_pvt_update(pvt->regs + PVT_INT, PVT_INT_CLR, PVT_INT_CLR);
+	/*
+	 * Read the data, update the cache and notify a waiter of this event.
+	 */
+	val = readl(pvt->regs + PVT_DATA);
+	WRITE_ONCE(pvt->data_cache, FIELD_GET(PVT_DATA_OUT, val));
+	complete(&pvt->conversion);
+
+	if (IS_ENABLED(CONFIG_PM))
+		pm_runtime_put(pvt->dev);
+
+	return IRQ_HANDLED;
+}
+
+static int eic7700_pvt_read_data(struct pvt_hwmon *pvt,
+				 enum pvt_sensor_type type, long *val)
+{
+	unsigned long timeout;
+	u32 data;
+	int ret;
+
+	/*
+	 * Wait for PVT conversion to complete and update the data cache. The
+	 * data read procedure is following: set the requested PVT sensor mode,
+	 * enable conversion, wait until conversion is finished, then disable
+	 * conversion and IRQ, and read the cached data.
+	 */
+	reinit_completion(&pvt->conversion);
+
+	eic7700_pvt_set_mode(pvt, pvt_info[type].mode);
+	eic7700_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, PVT_ENA_EN);
+
+	/*
+	 * Wait with timeout since in case if the sensor is suddenly powered
+	 * down the request won't be completed and the caller will hang up on
+	 * this procedure until the power is back up again. Multiply the
+	 * timeout by the factor of two to prevent a false timeout.
+	 */
+	timeout = 2 * usecs_to_jiffies(ktime_to_us(pvt->timeout));
+	ret = wait_for_completion_timeout(&pvt->conversion, timeout);
+
+	eic7700_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, 0);
+	eic7700_pvt_update(pvt->regs + PVT_INT, PVT_INT_CLR, PVT_INT_CLR);
+
+	if (!ret)
+		synchronize_irq(pvt->irq);
+
+	data = READ_ONCE(pvt->data_cache);
+
+	if (!ret)
+		return -ETIMEDOUT;
+
+	if (type == PVT_TEMP)
+		*val = polynomial_calc(&poly_N_to_temp, data);
+	else
+		*val = polynomial_calc(&poly_N_to_volt, data);
+
+	return 0;
+}
+
+static const struct hwmon_channel_info *pvt_channel_info[] = {
+	HWMON_CHANNEL_INFO(chip, HWMON_C_REGISTER_TZ),
+	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT | HWMON_T_LABEL),
+	HWMON_CHANNEL_INFO(in, HWMON_I_INPUT | HWMON_I_LABEL),
+	NULL
+};
+
+static umode_t eic7700_pvt_hwmon_is_visible(const void *data,
+					    enum hwmon_sensor_types type,
+					    u32 attr, int ch)
+{
+	switch (type) {
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_input:
+		case hwmon_temp_label:
+			return 0444;
+		}
+		break;
+	case hwmon_in:
+		switch (attr) {
+		case hwmon_in_input:
+		case hwmon_in_label:
+			return 0444;
+		}
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int eic7700_pvt_hwmon_read(struct device *dev,
+				  enum hwmon_sensor_types type, u32 attr,
+				  int ch, long *val)
+{
+	struct pvt_hwmon *pvt = dev_get_drvdata(dev);
+	int ret;
+
+	ret = pm_runtime_get_sync(pvt->dev);
+	if (ret < 0) {
+		dev_err(pvt->dev, "Failed to resume PVT device: %d\n", ret);
+		pm_runtime_put_noidle(pvt->dev);
+		return ret;
+	}
+
+	switch (type) {
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_input:
+			ret = eic7700_pvt_read_data(pvt, ch, val);
+			break;
+		default:
+			ret = -EOPNOTSUPP;
+		}
+		break;
+	case hwmon_in:
+		if (attr == hwmon_in_input)
+			ret = eic7700_pvt_read_data(pvt, PVT_VOLT + ch, val);
+		else
+			ret = -EOPNOTSUPP;
+		break;
+	default:
+		ret = -EOPNOTSUPP;
+	}
+
+	pm_runtime_mark_last_busy(pvt->dev);
+	pm_runtime_put_autosuspend(pvt->dev);
+	return ret;
+}
+
+static int eic7700_pvt_hwmon_read_string(struct device *dev,
+					 enum hwmon_sensor_types type, u32 attr,
+					 int ch, const char **str)
+{
+	switch (type) {
+	case hwmon_temp:
+		if (attr == hwmon_temp_label) {
+			*str = pvt_info[ch].label;
+			return 0;
+		}
+		break;
+	case hwmon_in:
+		if (attr == hwmon_in_label) {
+			*str = pvt_info[PVT_VOLT + ch].label;
+			return 0;
+		}
+		break;
+	default:
+		break;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static const struct hwmon_ops pvt_hwmon_ops = {
+	.is_visible = eic7700_pvt_hwmon_is_visible,
+	.read = eic7700_pvt_hwmon_read,
+	.read_string = eic7700_pvt_hwmon_read_string
+};
+
+static const struct hwmon_chip_info pvt_hwmon_info = {
+	.ops = &pvt_hwmon_ops,
+	.info = pvt_channel_info
+};
+
+static struct pvt_hwmon *eic7700_pvt_create_data(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct pvt_hwmon *pvt;
+
+	pvt = devm_kzalloc(dev, sizeof(*pvt), GFP_KERNEL);
+	if (!pvt)
+		return ERR_PTR(-ENOMEM);
+
+	pvt->dev = dev;
+	init_completion(&pvt->conversion);
+
+	return pvt;
+}
+
+static int eic7700_pvt_init_iface(struct pvt_hwmon *pvt)
+{
+	/*
+	 * Make sure controller are disabled so not to accidentally have ISR
+	 * executed before the driver data is fully initialized. Clear the IRQ
+	 * status as well.
+	 */
+	eic7700_pvt_update(pvt->regs + PVT_ENA, PVT_ENA_EN, 0);
+	eic7700_pvt_update(pvt->regs + PVT_INT, PVT_INT_CLR, PVT_INT_CLR);
+	readl(pvt->regs + PVT_INT);
+	readl(pvt->regs + PVT_DATA);
+
+	/* Setup default sensor mode and temperature trim. */
+	eic7700_pvt_set_mode(pvt, pvt_info[PVT_TEMP].mode);
+
+	/*
+	 * Max conversion latency (~333 µs) derived from PVT spec:
+	 * maximum sampling rate = 3000 samples/sec.
+	 */
+	pvt->timeout = ns_to_ktime(PVT_TOUT_MIN);
+
+	eic7700_pvt_set_trim(pvt, PVT_TRIM_DEF);
+
+	return 0;
+}
+
+static int eic7700_pvt_request_irq(struct pvt_hwmon *pvt)
+{
+	struct platform_device *pdev = to_platform_device(pvt->dev);
+	int ret;
+
+	pvt->irq = platform_get_irq(pdev, 0);
+	if (pvt->irq < 0)
+		return pvt->irq;
+
+	ret = devm_request_threaded_irq(pvt->dev, pvt->irq,
+					eic7700_pvt_hard_isr, NULL,
+					IRQF_TRIGGER_HIGH, "pvt", pvt);
+	if (ret) {
+		dev_err(pvt->dev, "Couldn't request PVT IRQ\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int eic7700_pvt_create_hwmon(struct pvt_hwmon *pvt)
+{
+	pvt->hwmon = devm_hwmon_device_register_with_info(pvt->dev, "pvt",
+							  pvt, &pvt_hwmon_info,
+							  NULL);
+	if (IS_ERR(pvt->hwmon)) {
+		dev_err(pvt->dev, "Couldn't create hwmon device\n");
+		return PTR_ERR(pvt->hwmon);
+	}
+
+	return 0;
+}
+
+static void eic7700_pvt_disable_pm_runtime(void *data)
+{
+	struct pvt_hwmon *pvt = data;
+
+	pm_runtime_dont_use_autosuspend(pvt->dev);
+	pm_runtime_disable(pvt->dev);
+
+	if (!pm_runtime_status_suspended(pvt->dev)) {
+		clk_bulk_disable_unprepare(PVT_CLK_NUM, pvt->clks);
+		pm_runtime_set_suspended(pvt->dev);
+	}
+}
+
+static int eic7700_pvt_probe(struct platform_device *pdev)
+{
+	struct reset_control *rst;
+	struct pvt_hwmon *pvt;
+	int i, ret;
+
+	pvt = eic7700_pvt_create_data(pdev);
+	if (IS_ERR(pvt))
+		return PTR_ERR(pvt);
+
+	platform_set_drvdata(pdev, pvt);
+
+	pvt->regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(pvt->regs))
+		return PTR_ERR(pvt->regs);
+
+	for (i = 0; i < PVT_CLK_NUM; i++)
+		pvt->clks[i].id = pvt_clk_names[i];
+
+	ret = devm_clk_bulk_get(&pdev->dev, PVT_CLK_NUM, pvt->clks);
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret,
+				     "Couldn't get clock descriptors\n");
+
+	rst = devm_reset_control_get_exclusive_deasserted(&pdev->dev, NULL);
+	if (IS_ERR(rst))
+		return dev_err_probe(pvt->dev, PTR_ERR(rst),
+				     "Couldn't get reset control\n");
+
+	ret = clk_bulk_prepare_enable(PVT_CLK_NUM, pvt->clks);
+	if (ret)
+		return dev_err_probe(pvt->dev, ret,
+				     "Failed to enable clocks\n");
+
+	ret = eic7700_pvt_init_iface(pvt);
+	if (ret) {
+		clk_bulk_disable_unprepare(PVT_CLK_NUM, pvt->clks);
+		return ret;
+	}
+
+	if (IS_ENABLED(CONFIG_PM))
+		clk_bulk_disable_unprepare(PVT_CLK_NUM, pvt->clks);
+
+	pm_runtime_enable(&pdev->dev);
+	pm_runtime_set_autosuspend_delay(&pdev->dev, 3000);
+	pm_runtime_use_autosuspend(&pdev->dev);
+	pm_runtime_get_noresume(&pdev->dev);
+
+	ret = devm_add_action_or_reset(pvt->dev, eic7700_pvt_disable_pm_runtime,
+				       pvt);
+	if (ret) {
+		pm_runtime_put_noidle(&pdev->dev);
+		return dev_err_probe(&pdev->dev, ret,
+				     "Can't register PM cleanup\n");
+	}
+
+	ret = eic7700_pvt_request_irq(pvt);
+	if (ret)
+		goto err_put_pm_runtime;
+
+	ret = eic7700_pvt_create_hwmon(pvt);
+	if (ret)
+		goto err_put_pm_runtime;
+
+	pm_runtime_put_autosuspend(&pdev->dev);
+
+	return 0;
+
+err_put_pm_runtime:
+	pm_runtime_put_noidle(&pdev->dev);
+	return ret;
+}
+
+static int __maybe_unused eic7700_pvt_runtime_resume(struct device *dev)
+{
+	struct pvt_hwmon *pvt = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_bulk_prepare_enable(PVT_CLK_NUM, pvt->clks);
+	if (ret) {
+		dev_err(dev, "Failed to enable clocks: %d\n", ret);
+		return ret;
+	}
+
+	eic7700_pvt_set_trim(pvt, PVT_TRIM_DEF);
+
+	return 0;
+}
+
+static int __maybe_unused eic7700_pvt_runtime_suspend(struct device *dev)
+{
+	struct pvt_hwmon *pvt = dev_get_drvdata(dev);
+
+	clk_bulk_disable_unprepare(PVT_CLK_NUM, pvt->clks);
+
+	return 0;
+}
+
+static const struct dev_pm_ops eic7700_pvt_pm_ops = {
+	SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
+	RUNTIME_PM_OPS(eic7700_pvt_runtime_suspend, eic7700_pvt_runtime_resume,
+		       NULL)
+};
+
+static const struct of_device_id pvt_of_match[] = {
+	{ .compatible = "eswin,eic7700-pvt"},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, pvt_of_match);
+
+static struct platform_driver pvt_driver = {
+	.probe = eic7700_pvt_probe,
+	.driver = {
+		.name = "eic7700-pvt",
+		.of_match_table = pvt_of_match,
+		.pm = pm_ptr(&eic7700_pvt_pm_ops),
+	},
+};
+module_platform_driver(pvt_driver);
+
+MODULE_AUTHOR("Yulin Lu <luyulin@eswincomputing.com>");
+MODULE_AUTHOR("Huan He <hehuan1@eswincomputing.com>");
+MODULE_DESCRIPTION("Eswin eic7700 PVT driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/eic7700-pvt.h b/drivers/hwmon/eic7700-pvt.h
new file mode 100644
index 0000000000000..fb10f9e4e93a5
--- /dev/null
+++ b/drivers/hwmon/eic7700-pvt.h
@@ -0,0 +1,99 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * ESWIN EIC7700 Voltage, Temperature sensor driver
+ *
+ * Copyright 2026, Beijing ESWIN Computing Technology Co., Ltd.
+ */
+#ifndef __HWMON_EIC7700_PVT_H__
+#define __HWMON_EIC7700_PVT_H__
+
+#include <linux/completion.h>
+#include <linux/hwmon.h>
+#include <linux/kernel.h>
+#include <linux/time.h>
+
+/* ESWIN EIC7700 PVT registers and their bitfields */
+#define PVT_TRIM		0x04
+#define PVT_MODE		0x08
+#define PVT_MODE_MASK		GENMASK(2, 0)
+#define PVT_CTRL_MODE_TEMP	0x0
+#define PVT_CTRL_MODE_VOLT	0x4
+#define PVT_ENA			0x0c
+#define PVT_ENA_EN		BIT(0)
+#define PVT_INT			0x10
+#define PVT_INT_STAT		BIT(0)
+#define PVT_INT_CLR		BIT(1)
+#define PVT_DATA		0x14
+#define PVT_DATA_OUT		GENMASK(9, 0)
+
+/*
+ * PVT sensors-related limits and default values
+ * @PVT_TEMP_CHS: Number of temperature hwmon channels.
+ * @PVT_VOLT_CHS: Number of voltage hwmon channels.
+ * @PVT_TRIM_DEF: Default temperature sensor trim value (set a proper value
+ *		  when one is determined for ESWIN EIC7700 SoC).
+ * @PVT_TOUT_MIN: Minimal timeout between samples in nanoseconds.
+ */
+#define PVT_TEMP_CHS		1
+#define PVT_VOLT_CHS		1
+#define PVT_TRIM_DEF		0
+#define PVT_TOUT_MIN		(NSEC_PER_SEC / 3000)
+
+/*
+ * enum pvt_sensor_type - ESWIN EIC7700 PVT sensor types (correspond to each PVT
+ *			  sampling mode)
+ * @PVT_TEMP: PVT Temperature sensor.
+ * @PVT_VOLT: PVT Voltage sensor.
+ */
+enum pvt_sensor_type {
+	PVT_TEMP = 0,
+	PVT_VOLT
+};
+
+#define PVT_CLK_NUM		2
+
+/*
+ * struct pvt_sensor_info - ESWIN EIC7700 PVT sensor informational structure
+ * @channel: Sensor channel ID.
+ * @label: hwmon sensor label.
+ * @mode: PVT mode corresponding to the channel.
+ * @type: Sensor type.
+ */
+struct pvt_sensor_info {
+	int channel;
+	const char *label;
+	u32 mode;
+	enum hwmon_sensor_types type;
+};
+
+#define PVT_SENSOR_INFO(_ch, _label, _type, _mode)	\
+	{						\
+		.channel = _ch,				\
+		.label = _label,			\
+		.mode = PVT_CTRL_MODE_ ##_mode,		\
+		.type = _type,				\
+	}
+
+/*
+ * struct pvt_hwmon - Eswin EIC7700 PVT private data
+ * @dev: device structure of the PVT platform device.
+ * @hwmon: hwmon device structure.
+ * @regs: pointer to the Eswin EIC7700 PVT registers region.
+ * @irq: PVT events IRQ number.
+ * @clks: PVT clock descriptors.
+ * @data_cache: data cache in raw format.
+ * @conversion: data conversion completion.
+ * @timeout: conversion timeout.
+ */
+struct pvt_hwmon {
+	struct device *dev;
+	struct device *hwmon;
+	void __iomem *regs;
+	int irq;
+	struct clk_bulk_data clks[PVT_CLK_NUM];
+	u32 data_cache;
+	struct completion conversion;
+	ktime_t timeout;
+};
+
+#endif /* __HWMON_EIC7700_PVT_H__ */
diff --git a/drivers/hwmon/gigabyte_waterforce.c b/drivers/hwmon/gigabyte_waterforce.c
index 27487e215bddf..4eea05f8b569c 100644
--- a/drivers/hwmon/gigabyte_waterforce.c
+++ b/drivers/hwmon/gigabyte_waterforce.c
@@ -371,13 +371,15 @@ static int waterforce_probe(struct hid_device *hdev, const struct hid_device_id
 	if (IS_ERR(priv->hwmon_dev)) {
 		ret = PTR_ERR(priv->hwmon_dev);
 		hid_err(hdev, "hwmon registration failed with %d\n", ret);
-		goto fail_and_close;
+		goto fail_and_io_stop;
 	}
 
 	waterforce_debugfs_init(priv);
 
 	return 0;
 
+fail_and_io_stop:
+	hid_device_io_stop(hdev);
 fail_and_close:
 	hid_hw_close(hdev);
 fail_and_stop:
diff --git a/drivers/hwmon/hwmon-vid.c b/drivers/hwmon/hwmon-vid.c
index 2df4956296ed5..dee42c163d929 100644
--- a/drivers/hwmon/hwmon-vid.c
+++ b/drivers/hwmon/hwmon-vid.c
@@ -237,16 +237,17 @@ static struct vrm_model vrm_models[] = {
  */
 static u8 get_via_model_d_vrm(void)
 {
-	unsigned int vid, brand, __maybe_unused dummy;
+	u64 msr;
+	unsigned int vid, brand;
 	static const char *brands[4] = {
 		"C7-M", "C7", "Eden", "C7-D"
 	};
 
-	rdmsr(0x198, dummy, vid);
-	vid &= 0xff;
+	rdmsrq(0x198, msr);
+	vid = (msr >> 32) & 0xff;
 
-	rdmsr(0x1154, brand, dummy);
-	brand = ((brand >> 4) ^ (brand >> 2)) & 0x03;
+	rdmsrq(0x1154, msr);
+	brand = ((msr >> 4) ^ (msr >> 2)) & 0x03;
 
 	if (vid > 0x3f) {
 		pr_info("Using %d-bit VID table for VIA %s CPU\n",
diff --git a/drivers/hwmon/iio_hwmon.c b/drivers/hwmon/iio_hwmon.c
index fc17ad93a3768..6502e20517a81 100644
--- a/drivers/hwmon/iio_hwmon.c
+++ b/drivers/hwmon/iio_hwmon.c
@@ -221,3 +221,4 @@ module_platform_driver(iio_hwmon_driver);
 MODULE_AUTHOR("Jonathan Cameron <jic23@kernel.org>");
 MODULE_DESCRIPTION("IIO to hwmon driver");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/hwmon/ina2xx.c b/drivers/hwmon/ina2xx.c
index c4742e84b999d..a533e30539866 100644
--- a/drivers/hwmon/ina2xx.c
+++ b/drivers/hwmon/ina2xx.c
@@ -122,6 +122,7 @@ static const struct regmap_config ina2xx_regmap_config = {
 enum ina2xx_ids {
 	ina219,
 	ina226,
+	ina232,
 	ina234,
 	ina260,
 	sy24655
@@ -196,6 +197,20 @@ static const struct ina2xx_config ina2xx_config[] = {
 		.current_shift = 4,
 		.has_update_interval = true,
 	},
+	[ina232] = {
+		.config_default = INA226_CONFIG_DEFAULT,
+		.calibration_value = 2048,
+		.shunt_div = 400,
+		.shunt_voltage_shift = 0,
+		.bus_voltage_shift = 0,
+		.bus_voltage_lsb = 1600,
+		.power_lsb_factor = 32,
+		.has_alerts = true,
+		.has_ishunt = false,
+		.has_power_average = false,
+		.current_shift = 0,
+		.has_update_interval = true,
+	},
 	[ina260] = {
 		.config_default = INA260_CONFIG_DEFAULT,
 		.shunt_div = 400,
@@ -1005,6 +1020,7 @@ static const struct i2c_device_id ina2xx_id[] = {
 	{ .name = "ina226", .driver_data = ina226 },
 	{ .name = "ina230", .driver_data = ina226 },
 	{ .name = "ina231", .driver_data = ina226 },
+	{ .name = "ina232", .driver_data = ina232 },
 	{ .name = "ina234", .driver_data = ina234 },
 	{ .name = "ina260", .driver_data = ina260 },
 	{ .name = "sy24655", .driver_data = sy24655 },
@@ -1037,6 +1053,10 @@ static const struct of_device_id __maybe_unused ina2xx_of_match[] = {
 		.compatible = "ti,ina231",
 		.data = (void *)ina226
 	},
+	{
+		.compatible = "ti,ina232",
+		.data = (void *)ina232
+	},
 	{
 		.compatible = "ti,ina234",
 		.data = (void *)ina234
diff --git a/drivers/hwmon/ntc_thermistor.c b/drivers/hwmon/ntc_thermistor.c
index 1ac0288fbbd80..4fa7f522f2451 100644
--- a/drivers/hwmon/ntc_thermistor.c
+++ b/drivers/hwmon/ntc_thermistor.c
@@ -748,3 +748,4 @@ MODULE_DESCRIPTION("NTC Thermistor Driver");
 MODULE_AUTHOR("MyungJoo Ham <myungjoo.ham@samsung.com>");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("platform:ntc-thermistor");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/hwmon/nzxt-kraken3.c b/drivers/hwmon/nzxt-kraken3.c
index d00409bcab93a..05525406c5fbb 100644
--- a/drivers/hwmon/nzxt-kraken3.c
+++ b/drivers/hwmon/nzxt-kraken3.c
@@ -948,7 +948,7 @@ static int kraken3_probe(struct hid_device *hdev, const struct hid_device_id *id
 	ret = kraken3_init_device(hdev);
 	if (ret < 0) {
 		hid_err(hdev, "device init failed with %d\n", ret);
-		goto fail_and_close;
+		goto fail_and_stop_io;
 	}
 
 	ret = kraken3_get_fw_ver(hdev);
@@ -960,13 +960,15 @@ static int kraken3_probe(struct hid_device *hdev, const struct hid_device_id *id
 	if (IS_ERR(priv->hwmon_dev)) {
 		ret = PTR_ERR(priv->hwmon_dev);
 		hid_err(hdev, "hwmon registration failed with %d\n", ret);
-		goto fail_and_close;
+		goto fail_and_stop_io;
 	}
 
 	kraken3_debugfs_init(priv, device_name);
 
 	return 0;
 
+fail_and_stop_io:
+	hid_device_io_stop(hdev);
 fail_and_close:
 	hid_hw_close(hdev);
 fail_and_stop:
diff --git a/drivers/hwmon/nzxt-smart2.c b/drivers/hwmon/nzxt-smart2.c
index 58ef9fa0184be..e2316c46629d6 100644
--- a/drivers/hwmon/nzxt-smart2.c
+++ b/drivers/hwmon/nzxt-smart2.c
@@ -768,7 +768,7 @@ static int nzxt_smart2_hid_probe(struct hid_device *hdev,
 
 out_hw_close:
 	hid_hw_close(hdev);
-
+	hid_device_io_stop(hdev);
 out_hw_stop:
 	hid_hw_stop(hdev);
 	return ret;
diff --git a/drivers/hwmon/pmbus/Kconfig b/drivers/hwmon/pmbus/Kconfig
index c8cda160b5f8e..ae003a6eac30a 100644
--- a/drivers/hwmon/pmbus/Kconfig
+++ b/drivers/hwmon/pmbus/Kconfig
@@ -52,8 +52,8 @@ config SENSORS_ADM1275
 	help
 	  If you say yes here you get hardware monitoring support for Analog
 	  Devices ADM1075, ADM1272, ADM1273, ADM1275, ADM1276, ADM1278, ADM1281,
-	  ADM1293, ADM1294 and SQ24905C Hot-Swap Controller and
-	  Digital Power Monitors.
+	  ADM1293, ADM1294, ROHM BD12780, ROHM BD12790, and SQ24905C
+	  Hot-Swap Controller and Digital Power Monitors.
 
 	  This driver can also be built as a module. If so, the module will
 	  be called adm1275.
@@ -677,6 +677,25 @@ config SENSORS_STEF48H28
 	  This driver can also be built as a module. If so, the module will
 	  be called stef48h28.
 
+config SENSORS_SQ24860
+	tristate "Silergy SQ24860"
+	help
+	  If you say yes here you get hardware monitoring support for Silergy
+	  SQ24860 eFuse.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called sq24860.
+
+config SENSORS_SQ24860_REGULATOR
+	bool "Regulator support for SQ24860"
+	depends on SENSORS_SQ24860 && REGULATOR
+	default SENSORS_SQ24860
+	help
+	  If you say yes here you get regulator support for Silergy SQ24860.
+	  The regulator is registered through the PMBus regulator framework and
+	  can be used to control the output exposed by the device.
+	  This option is only useful if regulator framework support is needed.
+
 config SENSORS_STPDDC60
 	tristate "ST STPDDC60"
 	help
@@ -803,6 +822,16 @@ config SENSORS_XDPE1A2G7B
 	  This driver can also be built as a module. If so, the module will
 	  be called xdpe1a2g7b.
 
+config SENSORS_XDPE1A2G7B_REGULATOR
+	bool "Regulator support for XDPE1A2G7B and compatibles"
+	depends on SENSORS_XDPE1A2G7B && REGULATOR
+	help
+	  If you say yes here you get regulator support for Infineon
+	  XDPE1A2G5B and XDPE1A2G7B multi-phase digital controllers.
+
+	  This enables the controllers to be used as regulator devices,
+	  providing voltage control through the regulator framework.
+
 config SENSORS_XDPE122
 	tristate "Infineon XDPE122 family"
 	help
diff --git a/drivers/hwmon/pmbus/Makefile b/drivers/hwmon/pmbus/Makefile
index ffc05f4932131..e01adacc8b764 100644
--- a/drivers/hwmon/pmbus/Makefile
+++ b/drivers/hwmon/pmbus/Makefile
@@ -65,6 +65,7 @@ obj-$(CONFIG_SENSORS_PM6764TR)	+= pm6764tr.o
 obj-$(CONFIG_SENSORS_PXE1610)	+= pxe1610.o
 obj-$(CONFIG_SENSORS_Q54SJ108A2)	+= q54sj108a2.o
 obj-$(CONFIG_SENSORS_STEF48H28)	+= stef48h28.o
+obj-$(CONFIG_SENSORS_SQ24860)	+= sq24860.o
 obj-$(CONFIG_SENSORS_STPDDC60)	+= stpddc60.o
 obj-$(CONFIG_SENSORS_TDA38640)	+= tda38640.o
 obj-$(CONFIG_SENSORS_TPS25990)	+= tps25990.o
diff --git a/drivers/hwmon/pmbus/adm1275.c b/drivers/hwmon/pmbus/adm1275.c
index d3c3ff85dba33..174f9bcba7e26 100644
--- a/drivers/hwmon/pmbus/adm1275.c
+++ b/drivers/hwmon/pmbus/adm1275.c
@@ -19,7 +19,7 @@
 #include "pmbus.h"
 
 enum chips { adm1075, adm1272, adm1273, adm1275, adm1276, adm1278, adm1281,
-	 adm1293, adm1294, sq24905c };
+	 adm1293, adm1294, bd12780, bd12790, sq24905c };
 
 #define ADM1275_MFR_STATUS_IOUT_WARN2	BIT(0)
 #define ADM1293_MFR_STATUS_VAUX_UV_WARN	BIT(5)
@@ -47,6 +47,8 @@ enum chips { adm1075, adm1272, adm1273, adm1275, adm1276, adm1278, adm1281,
 #define ADM1278_VOUT_EN			BIT(1)
 
 #define ADM1278_PMON_DEFCONFIG		(ADM1278_VOUT_EN | ADM1278_TEMP1_EN | ADM1278_TSFILT)
+/* The BD127[89]0 data sheets mark TSFILT bit as reserved. */
+#define BD12780_PMON_DEFCONFIG		(ADM1278_VOUT_EN | ADM1278_TEMP1_EN)
 
 #define ADM1293_IRANGE_25		0
 #define ADM1293_IRANGE_50		BIT(6)
@@ -134,6 +136,30 @@ static const struct coefficients adm1272_coefficients[] = {
 
 };
 
+/*
+ * BD12790 coefficients derived from preliminary datasheet, Table 1 (p.18)
+ * and the PMBus direct-format relationship X = (Y * 10^(-R) - b) / m.
+ *
+ * Voltage: V[V] = 14.77e-3 * code (60V) / 24.62e-3 * code (100V)
+ *   -> m = 6770, R=-2 / m = 4062, R=-2
+ * Current: code = I[A] * RS * 132802.1 + 2048 (15mV) / * 66401.06 + 2048 (30mV)
+ *   -> m = 1328, b = 2048 * 10^(-R) = 20480, R=-1 / m = 664, same b and R
+ * Power: code = k * RS * PIN, k = 35119.94 / 17559.97 / 21071.44 / 10535.72
+ *   -> m = round(k * 10^(-3-R)), R=-2 for 60V/15mV, R=-3 for the other three
+ * Temperature: code = 4.2 * T + 3188 -> m = 42, b = 3188 * 10 = 31880, R=-1
+ */
+static const struct coefficients bd12790_coefficients[] = {
+	[0] = { 6770, 0, -2 },		/* voltage, vrange 60V */
+	[1] = { 4062, 0, -2 },		/* voltage, vrange 100V */
+	[2] = { 1328, 20480, -1 },	/* current, vsense range 15mV */
+	[3] = { 664, 20480, -1 },	/* current, vsense range 30mV */
+	[4] = { 3512, 0, -2 },		/* power, vrange 60V, irange 15mV */
+	[5] = { 21071, 0, -3 },		/* power, vrange 100V, irange 15mV */
+	[6] = { 17560, 0, -3 },		/* power, vrange 60V, irange 30mV */
+	[7] = { 10536, 0, -3 },		/* power, vrange 100V, irange 30mV */
+	[8] = { 42, 31880, -1 },	/* temperature */
+};
+
 static const struct coefficients adm1275_coefficients[] = {
 	[0] = { 19199, 0, -2 },		/* voltage, vrange set */
 	[1] = { 6720, 0, -1 },		/* voltage, vrange not set */
@@ -487,6 +513,8 @@ static const struct i2c_device_id adm1275_id[] = {
 	{ .name = "adm1281", .driver_data = adm1281 },
 	{ .name = "adm1293", .driver_data = adm1293 },
 	{ .name = "adm1294", .driver_data = adm1294 },
+	{ .name = "bd12780", .driver_data = bd12780 },
+	{ .name = "bd12790", .driver_data = bd12790 },
 	{ .name = "mc09c", .driver_data = sq24905c },
 	{ }
 };
@@ -494,12 +522,13 @@ MODULE_DEVICE_TABLE(i2c, adm1275_id);
 
 /* Enable VOUT & TEMP1 if not enabled (disabled by default) */
 static int adm1275_enable_vout_temp(struct adm1275_data *data,
-				    struct i2c_client *client, int config)
+				    struct i2c_client *client, int config,
+				    u16 defconfig)
 {
 	int ret;
 
-	if ((config & ADM1278_PMON_DEFCONFIG) != ADM1278_PMON_DEFCONFIG) {
-		config |= ADM1278_PMON_DEFCONFIG;
+	if ((config & defconfig) != defconfig) {
+		config |= defconfig;
 		ret = adm1275_write_pmon_config(data, client, config);
 		if (ret < 0) {
 			dev_err(&client->dev, "Failed to enable VOUT/TEMP1 monitoring\n");
@@ -535,7 +564,8 @@ static int adm1275_probe(struct i2c_client *client)
 		return ret;
 	}
 	if ((ret != 3 || strncmp(block_buffer, "ADI", 3)) &&
-	    (ret != 2 || strncmp(block_buffer, "SY", 2))) {
+	    (ret != 2 || strncmp(block_buffer, "SY", 2)) &&
+	    (ret != 4 || strncmp(block_buffer, "ROHM", 4))) {
 		dev_err(&client->dev, "Unsupported Manufacturer ID\n");
 		return -ENODEV;
 	}
@@ -562,6 +592,7 @@ static int adm1275_probe(struct i2c_client *client)
 	if (mid->driver_data == adm1272 || mid->driver_data == adm1273 ||
 	    mid->driver_data == adm1278 || mid->driver_data == adm1281 ||
 	    mid->driver_data == adm1293 || mid->driver_data == adm1294 ||
+	    mid->driver_data == bd12780 || mid->driver_data == bd12790 ||
 	    mid->driver_data == sq24905c)
 		config_read_fn = i2c_smbus_read_word_data;
 	else
@@ -642,6 +673,7 @@ static int adm1275_probe(struct i2c_client *client)
 		data->have_power_sampling = true;
 
 		coefficients = adm1272_coefficients;
+
 		vindex = (config & ADM1275_VRANGE) ? 1 : 0;
 		cindex = (config & ADM1272_IRANGE) ? 3 : 2;
 		/* pindex depends on the combination of the above */
@@ -666,7 +698,53 @@ static int adm1275_probe(struct i2c_client *client)
 			PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT |
 			PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP;
 
-		ret = adm1275_enable_vout_temp(data, client, config);
+		ret = adm1275_enable_vout_temp(data, client, config,
+					       ADM1278_PMON_DEFCONFIG);
+		if (ret)
+			return ret;
+
+		if (config & ADM1278_VIN_EN)
+			info->func[0] |= PMBUS_HAVE_VIN;
+		break;
+
+	/*
+	 * The BD12790 is almost identical to the adm1272. Only the defconfig
+	 * and coefficients have minor differences.
+	 */
+	case bd12790:
+		data->have_vout = true;
+		data->have_pin_max = true;
+		data->have_temp_max = true;
+		data->have_power_sampling = true;
+
+		coefficients = bd12790_coefficients;
+
+		vindex = (config & ADM1275_VRANGE) ? 1 : 0;
+		cindex = (config & ADM1272_IRANGE) ? 3 : 2;
+		/* pindex depends on the combination of the above */
+		switch (config & (ADM1275_VRANGE | ADM1272_IRANGE)) {
+		case 0:
+		default:
+			pindex = 4;
+			break;
+		case ADM1275_VRANGE:
+			pindex = 5;
+			break;
+		case ADM1272_IRANGE:
+			pindex = 6;
+			break;
+		case ADM1275_VRANGE | ADM1272_IRANGE:
+			pindex = 7;
+			break;
+		}
+		tindex = 8;
+
+		info->func[0] |= PMBUS_HAVE_PIN | PMBUS_HAVE_STATUS_INPUT |
+			PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT |
+			PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP;
+
+		ret = adm1275_enable_vout_temp(data, client, config,
+					       BD12780_PMON_DEFCONFIG);
 		if (ret)
 			return ret;
 
@@ -728,12 +806,44 @@ static int adm1275_probe(struct i2c_client *client)
 			PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT |
 			PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP;
 
-		ret = adm1275_enable_vout_temp(data, client, config);
+		ret = adm1275_enable_vout_temp(data, client, config,
+					       ADM1278_PMON_DEFCONFIG);
+		if (ret)
+			return ret;
+
+		if (config & ADM1278_VIN_EN)
+			info->func[0] |= PMBUS_HAVE_VIN;
+		break;
+
+	/*
+	 * The BD12780 is almost functionally identical with the adm1278 above.
+	 * Only differences visible to the driver are lack of TSFILT bits and
+	 * different identification register contents.
+	 */
+	case bd12780:
+		data->have_vout = true;
+		data->have_pin_max = true;
+		data->have_temp_max = true;
+		data->have_power_sampling = true;
+
+		coefficients = adm1278_coefficients;
+		vindex = 0;
+		cindex = 1;
+		pindex = 2;
+		tindex = 3;
+
+		info->func[0] |= PMBUS_HAVE_PIN | PMBUS_HAVE_STATUS_INPUT |
+			PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT |
+			PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP;
+
+		ret = adm1275_enable_vout_temp(data, client, config,
+					       BD12780_PMON_DEFCONFIG);
 		if (ret)
 			return ret;
 
 		if (config & ADM1278_VIN_EN)
 			info->func[0] |= PMBUS_HAVE_VIN;
+
 		break;
 	case adm1293:
 	case adm1294:
@@ -870,9 +980,27 @@ static int adm1275_probe(struct i2c_client *client)
 	return pmbus_do_probe(client, info);
 }
 
+static const struct of_device_id adm1275_of_match[] = {
+	{ .compatible = "adi,adm1075", },
+	{ .compatible = "adi,adm1272", },
+	{ .compatible = "adi,adm1273", },
+	{ .compatible = "adi,adm1275", },
+	{ .compatible = "adi,adm1276", },
+	{ .compatible = "adi,adm1278", },
+	{ .compatible = "adi,adm1281", },
+	{ .compatible = "adi,adm1293", },
+	{ .compatible = "adi,adm1294", },
+	{ .compatible = "rohm,bd12780", },
+	{ .compatible = "rohm,bd12790", },
+	{ .compatible = "silergy,mc09c", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, adm1275_of_match);
+
 static struct i2c_driver adm1275_driver = {
 	.driver = {
 		   .name = "adm1275",
+		   .of_match_table = adm1275_of_match,
 		   },
 	.probe = adm1275_probe,
 	.id_table = adm1275_id,
diff --git a/drivers/hwmon/pmbus/lm25066.c b/drivers/hwmon/pmbus/lm25066.c
index 2be50cf1bbdea..f20991001727e 100644
--- a/drivers/hwmon/pmbus/lm25066.c
+++ b/drivers/hwmon/pmbus/lm25066.c
@@ -34,6 +34,7 @@ enum chips { lm25056, lm25066, lm5064, lm5066, lm5066i };
 #define LM25066_READ_AVG_PIN		0xdf
 
 #define LM25066_DEV_SETUP_CL		BIT(4)	/* Current limit */
+#define LM25066_DEV_SETUP_CL_CFG	BIT(2)	/* Current limit configuration */
 
 #define LM25066_SAMPLES_FOR_AVG_MAX	4096
 
@@ -484,6 +485,42 @@ static int lm25066_probe(struct i2c_client *client)
 
 	data->id = (enum chips)(unsigned long)i2c_get_match_data(client);
 
+	if (data->id != lm25056) {
+		int config_new = config;
+		const char *cl_setting;
+		int ret;
+
+		if (!of_property_read_string(client->dev.of_node,
+					     "ti,current-range", &cl_setting)) {
+			config_new |= LM25066_DEV_SETUP_CL_CFG;
+			if (strcmp(cl_setting, "high") == 0) {
+				if (data->id == lm25066)
+					config_new |= LM25066_DEV_SETUP_CL;
+				else
+					config_new &= ~LM25066_DEV_SETUP_CL;
+			} else if (strcmp(cl_setting, "low") == 0) {
+				if (data->id == lm25066)
+					config_new &= ~LM25066_DEV_SETUP_CL;
+				else
+					config_new |= LM25066_DEV_SETUP_CL;
+			} else {
+				dev_err(&client->dev,
+					"invalid current-range setting: %s\n",
+					cl_setting);
+				return -EINVAL;
+			}
+		}
+
+		if (config_new != config) {
+			ret = i2c_smbus_write_byte_data(client,
+							LM25066_DEVICE_SETUP,
+							config_new);
+			if (ret < 0)
+				return ret;
+			config = config_new;
+		}
+	}
+
 	info = &data->info;
 
 	info->pages = 1;
diff --git a/drivers/hwmon/pmbus/sq24860.c b/drivers/hwmon/pmbus/sq24860.c
new file mode 100644
index 0000000000000..30202a4b34cf6
--- /dev/null
+++ b/drivers/hwmon/pmbus/sq24860.c
@@ -0,0 +1,430 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Author: Ziming Zhu <ziming.zhu@silergycorp.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/math64.h>
+
+#include "pmbus.h"
+
+#define SQ24860_IIN_CAL_GAIN		0x38
+#define SQ24860_READ_VAUX		0xd0
+#define SQ24860_READ_VIN_MIN		0xd1
+#define SQ24860_READ_VIN_PEAK		0xd2
+#define SQ24860_READ_IIN_PEAK		0xd4
+#define SQ24860_READ_PIN_PEAK		0xd5
+#define SQ24860_READ_TEMP_AVG		0xd6
+#define SQ24860_READ_TEMP_PEAK		0xd7
+#define SQ24860_READ_VOUT_MIN		0xda
+#define SQ24860_READ_VIN_AVG		0xdc
+#define SQ24860_READ_VOUT_AVG		0xdd
+#define SQ24860_READ_IIN_AVG		0xde
+#define SQ24860_READ_PIN_AVG		0xdf
+#define SQ24860_VIREF			0xe0
+#define SQ24860_PK_MIN_AVG		0xea
+#define PK_MIN_AVG_RST_PEAK		BIT(7)
+#define PK_MIN_AVG_RST_AVG		BIT(6)
+#define PK_MIN_AVG_RST_MIN		BIT(5)
+#define PK_MIN_AVG_AVG_CNT		GENMASK(2, 0)
+#define SQ24860_MFR_WRITE_PROTECT	0xf8
+#define SQ24860_UNLOCKED		BIT(7)
+
+#define SQ24860_8B_SHIFT		2
+#define SQ24860_IIN_OCF_NUM		1000000
+#define SQ24860_IIN_OCF_DIV		129278
+#define SQ24860_IIN_OCF_OFF		165
+
+#define PK_MIN_AVG_RST_MASK		(PK_MIN_AVG_RST_PEAK | \
+					 PK_MIN_AVG_RST_AVG  | \
+					 PK_MIN_AVG_RST_MIN)
+#define SQ24860_MAX_SAMPLES		BIT(FIELD_MAX(PK_MIN_AVG_AVG_CNT))
+/*
+ * Arbitrary default Rimon value: 1.6kOhm
+ */
+#define SQ24860_DEFAULT_RIMON		1600000000
+#define SQ24860_GIMON			18180
+
+#define SQ24860_VAUX_DIV		20
+
+static int sq24860_write_iin_cal_gain(struct i2c_client *client, u32 rimon)
+{
+	u64 temp = 6400ULL * 1000000000ULL * 1000ULL;
+	u64 denom;
+	u64 word;
+
+	if (!rimon)
+		return -EINVAL;
+
+	denom = (u64)rimon * SQ24860_GIMON;
+	word = div64_u64(temp, denom);
+	if (!word || word > U16_MAX)
+		return -EINVAL;
+
+	return i2c_smbus_write_word_data(client, SQ24860_IIN_CAL_GAIN,
+					(u16)word);
+}
+
+static int sq24860_mfr_write_protect_set(struct i2c_client *client,
+					 u8 protect)
+{
+	u8 val;
+
+	switch (protect) {
+	case 0:
+		val = 0xa2;
+		break;
+	case PB_WP_ALL:
+		val = 0x0;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return pmbus_write_byte_data(client, -1, SQ24860_MFR_WRITE_PROTECT,
+				     val);
+}
+
+static int sq24860_mfr_write_protect_get(struct i2c_client *client)
+{
+	int ret = pmbus_read_byte_data(client, -1, SQ24860_MFR_WRITE_PROTECT);
+
+	if (ret < 0)
+		return ret;
+
+	return (ret & SQ24860_UNLOCKED) ? 0 : PB_WP_ALL;
+}
+
+static int sq24860_read_word_data(struct i2c_client *client,
+				  int page, int phase, int reg)
+{
+	int ret;
+
+	switch (reg) {
+	case PMBUS_VIRT_READ_VIN_MAX:
+		ret = pmbus_read_word_data(client, page, phase,
+					   SQ24860_READ_VIN_PEAK);
+		break;
+
+	case PMBUS_VIRT_READ_VIN_MIN:
+		ret = pmbus_read_word_data(client, page, phase,
+					   SQ24860_READ_VIN_MIN);
+		break;
+
+	case PMBUS_VIRT_READ_VIN_AVG:
+		ret = pmbus_read_word_data(client, page, phase,
+					   SQ24860_READ_VIN_AVG);
+		break;
+
+	case PMBUS_VIRT_READ_VOUT_MIN:
+		ret = pmbus_read_word_data(client, page, phase,
+					   SQ24860_READ_VOUT_MIN);
+		break;
+
+	case PMBUS_VIRT_READ_VOUT_AVG:
+		ret = pmbus_read_word_data(client, page, phase,
+					   SQ24860_READ_VOUT_AVG);
+		break;
+
+	case PMBUS_VIRT_READ_IIN_AVG:
+		ret = pmbus_read_word_data(client, page, phase,
+					   SQ24860_READ_IIN_AVG);
+		break;
+
+	case PMBUS_VIRT_READ_IIN_MAX:
+		ret = pmbus_read_word_data(client, page, phase,
+					   SQ24860_READ_IIN_PEAK);
+		break;
+
+	case PMBUS_VIRT_READ_TEMP_AVG:
+		ret = pmbus_read_word_data(client, page, phase,
+					   SQ24860_READ_TEMP_AVG);
+		break;
+
+	case PMBUS_VIRT_READ_TEMP_MAX:
+		ret = pmbus_read_word_data(client, page, phase,
+					   SQ24860_READ_TEMP_PEAK);
+		break;
+
+	case PMBUS_VIRT_READ_PIN_AVG:
+		ret = pmbus_read_word_data(client, page, phase,
+					   SQ24860_READ_PIN_AVG);
+		break;
+
+	case PMBUS_VIRT_READ_PIN_MAX:
+		ret = pmbus_read_word_data(client, page, phase,
+					   SQ24860_READ_PIN_PEAK);
+		break;
+
+	case PMBUS_VIRT_READ_VMON:
+		ret = pmbus_read_word_data(client, page, phase,
+					   SQ24860_READ_VAUX);
+		if (ret < 0)
+			break;
+		ret = DIV_ROUND_CLOSEST(ret, SQ24860_VAUX_DIV);
+		break;
+
+	case PMBUS_VIN_UV_WARN_LIMIT:
+	case PMBUS_VIN_UV_FAULT_LIMIT:
+	case PMBUS_VIN_OV_WARN_LIMIT:
+	case PMBUS_VIN_OV_FAULT_LIMIT:
+	case PMBUS_VOUT_UV_WARN_LIMIT:
+	case PMBUS_IIN_OC_WARN_LIMIT:
+	case PMBUS_OT_WARN_LIMIT:
+	case PMBUS_OT_FAULT_LIMIT:
+	case PMBUS_PIN_OP_WARN_LIMIT:
+		/*
+		 * These registers provide an 8 bits value instead of a
+		 * 10bits one. Just shifting twice the register value is
+		 * enough to make the sensor type conversion work, even
+		 * if the datasheet provides different m, b and R for
+		 * those.
+		 */
+		ret = pmbus_read_word_data(client, page, phase, reg);
+		if (ret < 0)
+			break;
+		ret <<= SQ24860_8B_SHIFT;
+		break;
+
+	case PMBUS_IIN_OC_FAULT_LIMIT:
+		/*
+		 * VIREF directly sets the over-current limit at which the eFuse
+		 * will turn the FET off and trigger a fault. Expose it through
+		 * this generic property instead of a manufacturer specific one.
+		 */
+		ret = pmbus_read_byte_data(client, page, SQ24860_VIREF);
+		if (ret < 0)
+			break;
+		ret = DIV_ROUND_CLOSEST(ret * SQ24860_IIN_OCF_NUM,
+					SQ24860_IIN_OCF_DIV);
+		ret += SQ24860_IIN_OCF_OFF;
+		break;
+
+	case PMBUS_VIRT_SAMPLES:
+		ret = pmbus_read_byte_data(client, page, SQ24860_PK_MIN_AVG);
+		if (ret < 0)
+			break;
+		ret = BIT(FIELD_GET(PK_MIN_AVG_AVG_CNT, ret));
+		break;
+
+	case PMBUS_VIRT_RESET_TEMP_HISTORY:
+	case PMBUS_VIRT_RESET_VIN_HISTORY:
+	case PMBUS_VIRT_RESET_IIN_HISTORY:
+	case PMBUS_VIRT_RESET_PIN_HISTORY:
+	case PMBUS_VIRT_RESET_VOUT_HISTORY:
+		ret = 0;
+		break;
+
+	default:
+		ret = -ENODATA;
+		break;
+	}
+
+	return ret;
+}
+
+static int sq24860_write_word_data(struct i2c_client *client,
+				   int page, int reg, u16 value)
+{
+	int ret;
+
+	switch (reg) {
+	case PMBUS_VIN_UV_WARN_LIMIT:
+	case PMBUS_VIN_UV_FAULT_LIMIT:
+	case PMBUS_VIN_OV_WARN_LIMIT:
+	case PMBUS_VIN_OV_FAULT_LIMIT:
+	case PMBUS_VOUT_UV_WARN_LIMIT:
+	case PMBUS_IIN_OC_WARN_LIMIT:
+	case PMBUS_OT_WARN_LIMIT:
+	case PMBUS_OT_FAULT_LIMIT:
+	case PMBUS_PIN_OP_WARN_LIMIT:
+		value = max_t(s16, (s16)value, 0);
+		value >>= SQ24860_8B_SHIFT;
+		value = clamp_val(value, 0, 0xff);
+		ret = pmbus_write_word_data(client, page, reg, value);
+		break;
+
+	case PMBUS_IIN_OC_FAULT_LIMIT:
+		value = max_t(s16, (s16)value, SQ24860_IIN_OCF_OFF);
+		value -= SQ24860_IIN_OCF_OFF;
+		value = DIV_ROUND_CLOSEST(((unsigned int)value) * SQ24860_IIN_OCF_DIV,
+					  SQ24860_IIN_OCF_NUM);
+		value = clamp_val(value, 0, 0x3f);
+		ret = pmbus_write_byte_data(client, page, SQ24860_VIREF, value);
+		break;
+
+	case PMBUS_VIRT_SAMPLES:
+		value = clamp_val(value, 1, SQ24860_MAX_SAMPLES);
+		value = ilog2(value);
+		ret = pmbus_update_byte_data(client, page, SQ24860_PK_MIN_AVG,
+					     PK_MIN_AVG_AVG_CNT,
+					     FIELD_PREP(PK_MIN_AVG_AVG_CNT, value));
+		break;
+
+	case PMBUS_VIRT_RESET_TEMP_HISTORY:
+	case PMBUS_VIRT_RESET_VIN_HISTORY:
+	case PMBUS_VIRT_RESET_IIN_HISTORY:
+	case PMBUS_VIRT_RESET_PIN_HISTORY:
+	case PMBUS_VIRT_RESET_VOUT_HISTORY:
+		/*
+		 * SQ24860 has history resets based on MIN/AVG/PEAK instead of per
+		 * sensor type. Exposing this quirk in hwmon is not desirable so
+		 * reset MIN, AVG and PEAK together. Even is there effectively only
+		 * one reset, which resets everything, expose the 5 entries so
+		 * userspace is not required map a sensor type to another to trigger
+		 * a reset
+		 */
+		ret = pmbus_update_byte_data(client, 0, SQ24860_PK_MIN_AVG,
+					     PK_MIN_AVG_RST_MASK,
+					     PK_MIN_AVG_RST_MASK);
+		break;
+
+	default:
+		ret = -ENODATA;
+		break;
+	}
+
+	return ret;
+}
+
+static int sq24860_read_byte_data(struct i2c_client *client,
+				  int page, int reg)
+{
+	int ret;
+
+	switch (reg) {
+	case PMBUS_WRITE_PROTECT:
+		ret = sq24860_mfr_write_protect_get(client);
+		break;
+
+	default:
+		ret = -ENODATA;
+		break;
+	}
+
+	return ret;
+}
+
+static int sq24860_write_byte_data(struct i2c_client *client,
+				   int page, int reg, u8 byte)
+{
+	int ret;
+
+	switch (reg) {
+	case PMBUS_WRITE_PROTECT:
+		ret = sq24860_mfr_write_protect_set(client, byte);
+		break;
+
+	default:
+		ret = -ENODATA;
+		break;
+	}
+
+	return ret;
+}
+
+#if IS_ENABLED(CONFIG_SENSORS_SQ24860_REGULATOR)
+static const struct regulator_desc sq24860_reg_desc[] = {
+	PMBUS_REGULATOR_ONE_NODE("vout"),
+};
+#endif
+
+static const struct pmbus_driver_info sq24860_base_info = {
+	.pages = 1,
+	.format[PSC_VOLTAGE_IN] = direct,
+	.m[PSC_VOLTAGE_IN] = 64,
+	.b[PSC_VOLTAGE_IN] = 0,
+	.R[PSC_VOLTAGE_IN] = 0,
+	.format[PSC_VOLTAGE_OUT] = direct,
+	.m[PSC_VOLTAGE_OUT] = 64,
+	.b[PSC_VOLTAGE_OUT] = 0,
+	.R[PSC_VOLTAGE_OUT] = 0,
+	.format[PSC_TEMPERATURE] = direct,
+	.m[PSC_TEMPERATURE] = 1,
+	.b[PSC_TEMPERATURE] = 0,
+	.R[PSC_TEMPERATURE] = 0,
+	/*
+	 * Current and power measurements depend on the calibration gain
+	 * programmed from the board-specific IMON resistor value.
+	 */
+	.format[PSC_CURRENT_IN] = direct,
+	.m[PSC_CURRENT_IN] = 16,
+	.b[PSC_CURRENT_IN] = 0,
+	.R[PSC_CURRENT_IN] = 0,
+	.format[PSC_POWER] = direct,
+	.m[PSC_POWER] = 2,
+	.b[PSC_POWER] = 0,
+	.R[PSC_POWER] = 0,
+	.func[0] = PMBUS_HAVE_VIN |
+		   PMBUS_HAVE_VOUT |
+		   PMBUS_HAVE_VMON |
+		   PMBUS_HAVE_IIN |
+		   PMBUS_HAVE_PIN |
+		   PMBUS_HAVE_TEMP |
+		   PMBUS_HAVE_STATUS_VOUT |
+		   PMBUS_HAVE_STATUS_IOUT |
+		   PMBUS_HAVE_STATUS_INPUT |
+		   PMBUS_HAVE_STATUS_TEMP |
+		   PMBUS_HAVE_SAMPLES,
+	.read_word_data = sq24860_read_word_data,
+	.write_word_data = sq24860_write_word_data,
+	.read_byte_data = sq24860_read_byte_data,
+	.write_byte_data = sq24860_write_byte_data,
+
+#if IS_ENABLED(CONFIG_SENSORS_SQ24860_REGULATOR)
+	.reg_desc = sq24860_reg_desc,
+	.num_regulators = ARRAY_SIZE(sq24860_reg_desc),
+#endif
+};
+
+static const struct i2c_device_id sq24860_i2c_id[] = {
+	{ "sq24860" },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, sq24860_i2c_id);
+
+static const struct of_device_id sq24860_of_match[] = {
+	{ .compatible = "silergy,sq24860" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, sq24860_of_match);
+
+static int sq24860_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct pmbus_driver_info *info;
+	u32 rimon;
+	int ret;
+
+	if (device_property_read_u32(dev, "silergy,rimon-micro-ohms", &rimon))
+		rimon = SQ24860_DEFAULT_RIMON;
+	ret = sq24860_write_iin_cal_gain(client, rimon);
+	if (ret < 0)
+		return dev_err_probe(&client->dev, ret,
+					     "Failed to set gain\n");
+	info = devm_kmemdup(dev, &sq24860_base_info, sizeof(*info), GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	return pmbus_do_probe(client, info);
+}
+
+static struct i2c_driver sq24860_driver = {
+	.driver = {
+		.name = "sq24860",
+		.of_match_table = sq24860_of_match,
+	},
+	.probe = sq24860_probe,
+	.id_table = sq24860_i2c_id,
+};
+module_i2c_driver(sq24860_driver);
+
+MODULE_AUTHOR("Ziming Zhu <ziming.zhu@silergycorp.com>");
+MODULE_DESCRIPTION("PMBUS driver for SQ24860 eFuse");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("PMBUS");
diff --git a/drivers/hwmon/pmbus/xdpe1a2g7b.c b/drivers/hwmon/pmbus/xdpe1a2g7b.c
index 971e7b73752ea..9455ef443b2b3 100644
--- a/drivers/hwmon/pmbus/xdpe1a2g7b.c
+++ b/drivers/hwmon/pmbus/xdpe1a2g7b.c
@@ -57,6 +57,13 @@ static int xdpe1a2g7b_identify(struct i2c_client *client,
 	return 0;
 }
 
+#if IS_ENABLED(CONFIG_SENSORS_XDPE1A2G7B_REGULATOR)
+static const struct regulator_desc xdpe1a2g7b_reg_desc[] = {
+	PMBUS_REGULATOR("vout", 0),
+	PMBUS_REGULATOR("vout", 1),
+};
+#endif
+
 static struct pmbus_driver_info xdpe1a2g7b_info = {
 	.pages = XDPE1A2G7B_PAGE_NUM,
 	.identify = xdpe1a2g7b_identify,
@@ -72,6 +79,10 @@ static struct pmbus_driver_info xdpe1a2g7b_info = {
 	.func[1] = PMBUS_HAVE_VIN | PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT |
 		   PMBUS_HAVE_IIN | PMBUS_HAVE_IOUT | PMBUS_HAVE_STATUS_IOUT |
 		   PMBUS_HAVE_PIN | PMBUS_HAVE_POUT | PMBUS_HAVE_STATUS_INPUT,
+#if IS_ENABLED(CONFIG_SENSORS_XDPE1A2G7B_REGULATOR)
+	.num_regulators = XDPE1A2G7B_PAGE_NUM,
+	.reg_desc = xdpe1a2g7b_reg_desc,
+#endif
 };
 
 static int xdpe1a2g7b_probe(struct i2c_client *client)
diff --git a/drivers/hwmon/tvs-mpfs.c b/drivers/hwmon/tvs-mpfs.c
new file mode 100644
index 0000000000000..aad00676518b7
--- /dev/null
+++ b/drivers/hwmon/tvs-mpfs.c
@@ -0,0 +1,390 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Author: Lars Randers <lranders@mail.dk>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/err.h>
+#include <linux/freezer.h>
+#include <linux/hwmon.h>
+#include <linux/io.h>
+#include <linux/kthread.h>
+#include <linux/math.h>
+#include <linux/mfd/syscon.h>
+#include <linux/minmax.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#define MPFS_TVS_CTRL 0x08
+#define MPFS_TVS_OUTPUT0 0x24
+#define MPFS_TVS_OUTPUT1 0x28
+
+#define MPFS_TVS_CTRL_TEMP_VALID	BIT(19)
+#define MPFS_TVS_CTRL_V2P5_VALID	BIT(18)
+#define MPFS_TVS_CTRL_V1P8_VALID	BIT(17)
+#define MPFS_TVS_CTRL_V1P05_VALID	BIT(16)
+
+#define MPFS_TVS_CTRL_TEMP_ENABLE	BIT(3)
+#define MPFS_TVS_CTRL_V2P5_ENABLE	BIT(2)
+#define MPFS_TVS_CTRL_V1P8_ENABLE	BIT(1)
+#define MPFS_TVS_CTRL_V1P05_ENABLE	BIT(0)
+#define MPFS_TVS_CTRL_ENABLE_ALL	GENMASK(3, 0)
+
+/*
+ * For all of these the value in millivolts is stored in 16 bits, with an upper
+ * sign bit and a lower 3 bits of decimal. These masks discard the sign bit and
+ * decimal places, because if Linux is running these voltages cannot be negative
+ * and so avoid having to convert to two's complement.
+ */
+#define MPFS_OUTPUT0_V1P8_MASK	GENMASK(30, 19)
+#define MPFS_OUTPUT0_V1P05_MASK	GENMASK(14, 3)
+#define MPFS_OUTPUT1_V2P5_MASK	GENMASK(14, 3)
+
+/*
+ * The register map claims that the temperature is stored in bits 31:16, but
+ * application note "AN4682: PolarFire FPGA Temperature and Voltage Sensor"
+ * says that 31 is reserved. Temperature is in kelvin, so what's probably a
+ * sign bit has no value anyway.
+ */
+#define MPFS_OUTPUT1_TEMP_MASK GENMASK(30, 16)
+
+#define MPFS_TVS_INTERVAL_MASK GENMASK(15, 8)
+#define MPFS_TVS_INTERVAL_OFFSET 8
+/* The interval register is in increments of 32 us */
+#define MPFS_TVS_INTERVAL_SCALE 32
+/* with 254 usable increments of 32 us available, 8 ms is the integer limit */
+#define MPFS_TVS_INTERVAL_MAX_MS 8
+
+/* 273.1875 in 11.4 fixed-point notation */
+#define MPFS_TVS_K_TO_C 0x1113
+
+enum mpfs_tvs_sensors {
+	SENSOR_V1P05 = 0,
+	SENSOR_V1P8,
+	SENSOR_V2P5,
+};
+
+static const char * const mpfs_tvs_voltage_labels[] = { "1P05", "1P8", "2P5" };
+
+struct mpfs_tvs {
+	struct regmap *regmap;
+};
+
+static int mpfs_tvs_voltage_read(struct mpfs_tvs *data, u32 attr,
+				 int channel, long *val)
+{
+	u32 tmp, control;
+
+	if (attr != hwmon_in_input && attr != hwmon_in_enable)
+		return -EOPNOTSUPP;
+
+	regmap_read(data->regmap, MPFS_TVS_CTRL, &control);
+
+	switch (channel) {
+	case SENSOR_V2P5:
+		if (attr == hwmon_in_enable) {
+			*val = FIELD_GET(MPFS_TVS_CTRL_V2P5_ENABLE, control);
+			break;
+		}
+
+		if (!(control & MPFS_TVS_CTRL_V2P5_VALID))
+			return -ENODATA;
+
+		regmap_read(data->regmap, MPFS_TVS_OUTPUT1, &tmp);
+		*val = FIELD_GET(MPFS_OUTPUT1_V2P5_MASK, tmp);
+		break;
+	case SENSOR_V1P8:
+		if (attr == hwmon_in_enable) {
+			*val = FIELD_GET(MPFS_TVS_CTRL_V1P8_ENABLE, control);
+			break;
+		}
+
+		if (!(control & MPFS_TVS_CTRL_V1P8_VALID))
+			return -ENODATA;
+
+		regmap_read(data->regmap, MPFS_TVS_OUTPUT0, &tmp);
+		*val = FIELD_GET(MPFS_OUTPUT0_V1P8_MASK, tmp);
+		break;
+	case SENSOR_V1P05:
+		if (attr == hwmon_in_enable) {
+			*val = FIELD_GET(MPFS_TVS_CTRL_V1P05_ENABLE, control);
+			break;
+		}
+
+		if (!(control & MPFS_TVS_CTRL_V1P05_VALID))
+			return -ENODATA;
+
+		regmap_read(data->regmap, MPFS_TVS_OUTPUT0, &tmp);
+		*val = FIELD_GET(MPFS_OUTPUT0_V1P05_MASK, tmp);
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int mpfs_tvs_voltage_write(struct mpfs_tvs *data, u32 attr,
+				  int channel, long val)
+{
+	u32 tmp;
+
+	if (attr != hwmon_in_enable)
+		return -EOPNOTSUPP;
+
+	if (val > 1 || val < 0)
+		return -EINVAL;
+
+	switch (channel) {
+	case SENSOR_V2P5:
+		tmp = FIELD_PREP(MPFS_TVS_CTRL_V2P5_ENABLE, val);
+		regmap_update_bits(data->regmap, MPFS_TVS_CTRL,
+				   MPFS_TVS_CTRL_V2P5_ENABLE, tmp);
+		break;
+	case SENSOR_V1P8:
+		tmp = FIELD_PREP(MPFS_TVS_CTRL_V1P8_ENABLE, val);
+		regmap_update_bits(data->regmap, MPFS_TVS_CTRL,
+				   MPFS_TVS_CTRL_V1P8_ENABLE, tmp);
+		break;
+	case SENSOR_V1P05:
+		tmp = FIELD_PREP(MPFS_TVS_CTRL_V1P05_ENABLE, val);
+		regmap_update_bits(data->regmap, MPFS_TVS_CTRL,
+				   MPFS_TVS_CTRL_V1P05_ENABLE, tmp);
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int mpfs_tvs_temp_read(struct mpfs_tvs *data, u32 attr, long *val)
+{
+	u32 tmp, control;
+
+	if (attr != hwmon_temp_input && attr != hwmon_temp_enable)
+		return -EOPNOTSUPP;
+
+	regmap_read(data->regmap, MPFS_TVS_CTRL, &control);
+
+	if (attr == hwmon_temp_enable) {
+		*val = FIELD_GET(MPFS_TVS_CTRL_TEMP_ENABLE, control);
+		return 0;
+	}
+
+	if (!(control & MPFS_TVS_CTRL_TEMP_VALID))
+		return -ENODATA;
+
+	regmap_read(data->regmap, MPFS_TVS_OUTPUT1, &tmp);
+	*val = FIELD_GET(MPFS_OUTPUT1_TEMP_MASK, tmp);
+	*val -= MPFS_TVS_K_TO_C;
+	*val = (1000 * *val) >> 4; /* fixed point (11.4) to millidegrees */
+
+	return 0;
+}
+
+static int mpfs_tvs_temp_write(struct mpfs_tvs *data, u32 attr, long val)
+{
+	u32 tmp;
+
+	if (attr != hwmon_temp_enable)
+		return -EOPNOTSUPP;
+
+	if (val > 1 || val < 0)
+		return -EINVAL;
+
+	tmp = FIELD_PREP(MPFS_TVS_CTRL_TEMP_ENABLE, val);
+	regmap_update_bits(data->regmap, MPFS_TVS_CTRL,
+			   MPFS_TVS_CTRL_TEMP_ENABLE, tmp);
+
+	return 0;
+}
+
+static int mpfs_tvs_interval_read(struct mpfs_tvs *data, u32 attr, long *val)
+{
+	u32 tmp;
+
+	if (attr != hwmon_chip_update_interval)
+		return -EOPNOTSUPP;
+
+	regmap_read(data->regmap, MPFS_TVS_CTRL, &tmp);
+	*val = FIELD_GET(MPFS_TVS_INTERVAL_MASK, tmp);
+	*val *= MPFS_TVS_INTERVAL_SCALE;
+	*val = roundup(*val, 1000);
+	*val /= 1000;
+
+	return 0;
+}
+
+static int mpfs_tvs_interval_write(struct mpfs_tvs *data, u32 attr, long val)
+{
+	long temp = val;
+
+	if (attr != hwmon_chip_update_interval)
+		return -EOPNOTSUPP;
+
+	temp = clamp(temp, 0, MPFS_TVS_INTERVAL_MAX_MS);
+
+	temp *= 1000;
+	temp /= MPFS_TVS_INTERVAL_SCALE;
+
+	temp <<= MPFS_TVS_INTERVAL_OFFSET;
+	regmap_update_bits(data->regmap, MPFS_TVS_CTRL,
+			   MPFS_TVS_INTERVAL_MASK, temp);
+
+	return 0;
+}
+
+static umode_t mpfs_tvs_is_visible(const void *data,
+				   enum hwmon_sensor_types type,
+				   u32 attr, int channel)
+{
+	if (type == hwmon_chip && attr == hwmon_chip_update_interval)
+		return 0644;
+
+	if (type == hwmon_temp) {
+		switch (attr) {
+		case hwmon_temp_enable:
+			return 0644;
+		case hwmon_temp_input:
+		case hwmon_temp_label:
+			return 0444;
+		default:
+			return 0;
+		}
+	}
+
+	if (type == hwmon_in) {
+		switch (attr) {
+		case hwmon_in_enable:
+			return 0644;
+		case hwmon_in_input:
+		case hwmon_in_label:
+			return 0444;
+		default:
+			return 0;
+		}
+	}
+
+	return 0;
+}
+
+static int mpfs_tvs_read(struct device *dev, enum hwmon_sensor_types type,
+			 u32 attr, int channel, long *val)
+{
+	struct mpfs_tvs *data = dev_get_drvdata(dev);
+
+	switch (type) {
+	case hwmon_temp:
+		return mpfs_tvs_temp_read(data, attr, val);
+	case hwmon_in:
+		return mpfs_tvs_voltage_read(data, attr, channel, val);
+	case hwmon_chip:
+		return mpfs_tvs_interval_read(data, attr, val);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int mpfs_tvs_write(struct device *dev, enum hwmon_sensor_types type,
+			  u32 attr, int channel, long val)
+{
+	struct mpfs_tvs *data = dev_get_drvdata(dev);
+
+	switch (type) {
+	case hwmon_temp:
+		return mpfs_tvs_temp_write(data, attr, val);
+	case hwmon_in:
+		return mpfs_tvs_voltage_write(data, attr, channel, val);
+	case hwmon_chip:
+		return mpfs_tvs_interval_write(data, attr, val);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int mpfs_tvs_read_labels(struct device *dev,
+				enum hwmon_sensor_types type,
+				u32 attr, int channel,
+				const char **str)
+{
+	switch (type) {
+	case hwmon_temp:
+		*str = "Die Temp";
+		return 0;
+	case hwmon_in:
+		*str = mpfs_tvs_voltage_labels[channel];
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static const struct hwmon_ops mpfs_tvs_ops = {
+	.is_visible = mpfs_tvs_is_visible,
+	.read_string = mpfs_tvs_read_labels,
+	.read = mpfs_tvs_read,
+	.write = mpfs_tvs_write,
+};
+
+static const struct hwmon_channel_info *mpfs_tvs_info[] = {
+	HWMON_CHANNEL_INFO(chip,
+			   HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL),
+	HWMON_CHANNEL_INFO(temp,
+			   HWMON_T_INPUT | HWMON_T_LABEL | HWMON_T_ENABLE),
+	HWMON_CHANNEL_INFO(in,
+			   HWMON_I_INPUT | HWMON_I_LABEL | HWMON_I_ENABLE,
+			   HWMON_I_INPUT | HWMON_I_LABEL | HWMON_I_ENABLE,
+			   HWMON_I_INPUT | HWMON_I_LABEL | HWMON_I_ENABLE),
+	NULL
+};
+
+static const struct hwmon_chip_info mpfs_tvs_chip_info = {
+	.ops = &mpfs_tvs_ops,
+	.info = mpfs_tvs_info,
+};
+
+static int mpfs_tvs_probe(struct platform_device *pdev)
+{
+	struct device *hwmon_dev;
+	struct mpfs_tvs *data;
+
+	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->regmap = device_node_to_regmap(pdev->dev.parent->of_node);
+	if (IS_ERR(data->regmap))
+		return dev_err_probe(&pdev->dev, PTR_ERR(data->regmap),
+				     "Failed to find syscon regmap\n");
+
+	/*
+	 * It's an MMIO regmap with no resources, there's nothing that can fail
+	 * and return an error
+	 */
+	regmap_write(data->regmap, MPFS_TVS_CTRL, MPFS_TVS_CTRL_ENABLE_ALL);
+
+	hwmon_dev = devm_hwmon_device_register_with_info(&pdev->dev, "mpfs_tvs",
+							 data,
+							 &mpfs_tvs_chip_info,
+							 NULL);
+	if (IS_ERR(hwmon_dev))
+		return dev_err_probe(&pdev->dev, PTR_ERR(hwmon_dev),
+				     "hwmon device registration failed.\n");
+
+	return 0;
+}
+
+static struct platform_driver mpfs_tvs_driver = {
+	.probe = mpfs_tvs_probe,
+	.driver = {
+		.name = "mpfs-tvs",
+	},
+};
+module_platform_driver(mpfs_tvs_driver);
+
+MODULE_AUTHOR("Lars Randers <lranders@mail.dk>");
+MODULE_DESCRIPTION("PolarFire SoC temperature & voltage sensor driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/xgene-hwmon.c b/drivers/hwmon/xgene-hwmon.c
index 11c5d80428cdf..38b140c23c88e 100644
--- a/drivers/hwmon/xgene-hwmon.c
+++ b/drivers/hwmon/xgene-hwmon.c
@@ -133,10 +133,6 @@ static int xgene_hwmon_pcc_rd(struct xgene_hwmon_dev *ctx, u32 *msg)
 	init_completion(&ctx->rd_complete);
 	ctx->resp_pending = true;
 
-	/* Write signature for subspace */
-	WRITE_ONCE(generic_comm_base->signature,
-		   cpu_to_le32(PCC_SIGNATURE | ctx->mbox_idx));
-
 	/* Write to the shared command region */
 	WRITE_ONCE(generic_comm_base->command,
 		   cpu_to_le16(MSG_TYPE(msg[0]) | PCC_CMD_GENERATE_DB_INTR));
diff --git a/drivers/hwmon/yogafan.c b/drivers/hwmon/yogafan.c
index 605cc928f21f3..0a41b8672b56e 100644
--- a/drivers/hwmon/yogafan.c
+++ b/drivers/hwmon/yogafan.c
@@ -71,6 +71,12 @@ static const struct yogafan_config legion_16bit_dual_cfg = {
 	.paths = { "\\_SB.PCI0.LPC0.EC0.FANS", "\\_SB.PCI0.LPC0.EC0.FA2S" }
 };
 
+static const struct yogafan_config loq_15iax9_8bit_dual_cfg = {
+	.multiplier = 100,
+	.fan_count = 2,
+	.paths = { "\\_SB.PC00.LPCB.EC0.FA1S", "\\_SB.PC00.LPCB.EC0.FA2S" }
+};
+
 static void apply_rllag_filter(struct yoga_fan_data *data, int idx, long raw_rpm)
 {
 	ktime_t now = ktime_get_boottime();
@@ -194,6 +200,14 @@ static const struct dmi_system_id yogafan_quirks[] = {
 		},
 		.driver_data = (void *)&ideapad_8bit_fan0_cfg,
 	},
+	{
+		.ident = "Lenovo LOQ 15IAX9",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_FAMILY, "LOQ 15IAX9"),
+		},
+		.driver_data = (void *)&loq_15iax9_8bit_dual_cfg,
+	},
 	{ }
 };
 MODULE_DEVICE_TABLE(dmi, yogafan_quirks);
diff --git a/drivers/hwspinlock/hwspinlock_core.c b/drivers/hwspinlock/hwspinlock_core.c
index cc8e952a67727..a509b73da190d 100644
--- a/drivers/hwspinlock/hwspinlock_core.c
+++ b/drivers/hwspinlock/hwspinlock_core.c
@@ -472,7 +472,7 @@ static int hwspin_lock_register_single(struct hwspinlock *hwlock, int id)
 
 out:
 	mutex_unlock(&hwspinlock_tree_lock);
-	return 0;
+	return ret;
 }
 
 static struct hwspinlock *hwspin_lock_unregister_single(unsigned int id)
diff --git a/drivers/hwtracing/ptt/hisi_ptt.c b/drivers/hwtracing/ptt/hisi_ptt.c
index 94c371c491357..233c4c32513cb 100644
--- a/drivers/hwtracing/ptt/hisi_ptt.c
+++ b/drivers/hwtracing/ptt/hisi_ptt.c
@@ -780,7 +780,7 @@ static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
 	struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev));
 	const cpumask_t *cpumask = cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev));
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask));
 }
 static DEVICE_ATTR_RO(cpumask);
 
diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig
index d354569942803..d7b89508311ff 100644
--- a/drivers/i2c/busses/Kconfig
+++ b/drivers/i2c/busses/Kconfig
@@ -793,7 +793,7 @@ config I2C_JZ4780
 config I2C_K1
 	tristate "SpacemiT K1 I2C adapter"
 	depends on ARCH_SPACEMIT || COMPILE_TEST
-	depends on OF
+	depends on OF && COMMON_CLK
 	default ARCH_SPACEMIT
 	help
 	  This option enables support for the I2C interface on the SpacemiT K1
diff --git a/drivers/i2c/busses/i2c-amd-asf-plat.c b/drivers/i2c/busses/i2c-amd-asf-plat.c
index 82cbc8fb5c182..5968953e483a0 100644
--- a/drivers/i2c/busses/i2c-amd-asf-plat.c
+++ b/drivers/i2c/busses/i2c-amd-asf-plat.c
@@ -89,6 +89,10 @@ static void amd_asf_process_target(struct work_struct *work)
 		outb_p(reg, ASFDATABNKSEL);
 		cmd = inb_p(ASFINDEX);
 		len = inb_p(ASFDATARWPTR);
+
+		if (len > ASF_BLOCK_MAX_BYTES)
+			return;
+
 		for (idx = 0; idx < len; idx++)
 			data[idx] = inb_p(ASFINDEX);
 
diff --git a/drivers/i2c/busses/i2c-davinci.c b/drivers/i2c/busses/i2c-davinci.c
index 0617f416cb0b5..e15eef163a8f8 100644
--- a/drivers/i2c/busses/i2c-davinci.c
+++ b/drivers/i2c/busses/i2c-davinci.c
@@ -117,8 +117,6 @@
 /* timeout for pm runtime autosuspend */
 #define DAVINCI_I2C_PM_TIMEOUT	1000	/* ms */
 
-#define DAVINCI_I2C_DEFAULT_BUS_FREQ	100000
-
 struct davinci_i2c_dev {
 	struct device           *dev;
 	void __iomem		*base;
@@ -760,7 +758,7 @@ static int davinci_i2c_probe(struct platform_device *pdev)
 
 	r = device_property_read_u32(&pdev->dev, "clock-frequency", &prop);
 	if (r)
-		prop = DAVINCI_I2C_DEFAULT_BUS_FREQ;
+		prop = I2C_MAX_STANDARD_MODE_FREQ;
 
 	dev->bus_freq = prop / 1000;
 
diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c
index 28313d0fad37e..d5e6e2eca3b34 100644
--- a/drivers/i2c/busses/i2c-imx.c
+++ b/drivers/i2c/busses/i2c-imx.c
@@ -1061,11 +1061,28 @@ static inline enum imx_i2c_state i2c_imx_isr_read_continue(struct imx_i2c_struct
 static inline void i2c_imx_isr_read_block_data_len(struct imx_i2c_struct *i2c_imx)
 {
 	u8 len = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2DR);
+	unsigned int temp;
 
 	if (len == 0 || len > I2C_SMBUS_BLOCK_MAX) {
+		/*
+		 * SMBus 3.1 6.5.7: support count byte of 0.
+		 * I2C_SMBUS_BLOCK_MAX case should not hold the SDA either.
+		 * So NACK it (TXAK) to not hold the bus.
+		 */
+		temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR);
+		temp |= I2CR_TXAK;
+		imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR);
+
+		if (len == 0) {
+			i2c_imx->msg->buf[i2c_imx->msg_buf_idx++] = 0;
+			i2c_imx->msg->len = 2;
+			return;
+		}
+
 		i2c_imx->isr_result = -EPROTO;
 		i2c_imx->state = IMX_I2C_STATE_FAILED;
 		wake_up(&i2c_imx->queue);
+		return;
 	}
 	i2c_imx->msg->len += len;
 	i2c_imx->msg->buf[i2c_imx->msg_buf_idx++] = len;
@@ -1415,6 +1432,7 @@ static int i2c_imx_atomic_read(struct imx_i2c_struct *i2c_imx,
 	int i, result;
 	unsigned int temp;
 	int block_data = msgs->flags & I2C_M_RECV_LEN;
+	int block_err = 0;
 
 	result = i2c_imx_prepare_read(i2c_imx, msgs, false);
 	if (result)
@@ -1436,8 +1454,20 @@ static int i2c_imx_atomic_read(struct imx_i2c_struct *i2c_imx,
 		 */
 		if ((!i) && block_data) {
 			len = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2DR);
-			if ((len == 0) || (len > I2C_SMBUS_BLOCK_MAX))
-				return -EPROTO;
+			if ((len == 0) || (len > I2C_SMBUS_BLOCK_MAX)) {
+				/*
+				 * SMBus 3.1 6.5.7: support count byte of 0.
+				 * I2C_SMBUS_BLOCK_MAX case should not hold the SDA either.
+				 */
+				if (len > I2C_SMBUS_BLOCK_MAX)
+					block_err = -EPROTO;
+				temp = imx_i2c_read_reg(i2c_imx, IMX_I2C_I2CR);
+				temp |= I2CR_TXAK;
+				imx_i2c_write_reg(temp, i2c_imx, IMX_I2C_I2CR);
+				msgs->buf[0] = 0;
+				msgs->len = 2;
+				continue;
+			}
 			dev_dbg(&i2c_imx->adapter.dev,
 				"<%s> read length: 0x%X\n",
 				__func__, len);
@@ -1485,7 +1515,7 @@ static int i2c_imx_atomic_read(struct imx_i2c_struct *i2c_imx,
 			"<%s> read byte: B%d=0x%X\n",
 			__func__, i, msgs->buf[i]);
 	}
-	return 0;
+	return block_err;
 }
 
 static int i2c_imx_read(struct imx_i2c_struct *i2c_imx, struct i2c_msg *msgs,
diff --git a/drivers/i2c/busses/i2c-k1.c b/drivers/i2c/busses/i2c-k1.c
index 9152cf436bea0..487f23fde7250 100644
--- a/drivers/i2c/busses/i2c-k1.c
+++ b/drivers/i2c/busses/i2c-k1.c
@@ -4,7 +4,9 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/bits.h>
 #include <linux/clk.h>
+#include <linux/clk-provider.h>
 #include <linux/i2c.h>
 #include <linux/iopoll.h>
 #include <linux/module.h>
@@ -17,6 +19,8 @@
 #define SPACEMIT_ISR		 0x4		/* Status register */
 #define SPACEMIT_IDBR		 0xc		/* Data buffer register */
 #define SPACEMIT_IRCR		 0x18		/* Reset cycle counter */
+#define SPACEMIT_ILCR		 0x10		/* Load Count Register */
+#define SPACEMIT_IWCR		 0x14		/* Wait Count Register */
 #define SPACEMIT_IBMR		 0x1c		/* Bus monitor register */
 
 /* SPACEMIT_ICR register fields */
@@ -88,12 +92,22 @@
 #define SPACEMIT_BMR_SDA         BIT(0)		/* SDA line level */
 #define SPACEMIT_BMR_SCL         BIT(1)		/* SCL line level */
 
+#define SPACEMIT_LCR_LV_STANDARD_MASK		GENMASK(8, 0)
+#define SPACEMIT_LCR_LV_FAST_MASK		GENMASK(17, 9)
+
+/* SPACEMIT_IWCR register fields */
+#define SPACEMIT_WCR_COUNT			GENMASK(4, 0)
+#define SPACEMIT_WCR_HS_COUNT1			GENMASK(9, 5)
+#define SPACEMIT_WCR_HS_COUNT2			GENMASK(14, 10)
+
+/* Required by I2C IP for correct SCL timing */
+#define SPACEMIT_IWCR_INIT_VALUE		(FIELD_PREP(SPACEMIT_WCR_COUNT, 10) | \
+						 FIELD_PREP(SPACEMIT_WCR_HS_COUNT1, 1) | \
+						 FIELD_PREP(SPACEMIT_WCR_HS_COUNT2, 5))
+
 /* i2c bus recover timeout: us */
 #define SPACEMIT_I2C_BUS_BUSY_TIMEOUT		100000
 
-#define SPACEMIT_I2C_MAX_STANDARD_MODE_FREQ	100000	/* Hz */
-#define SPACEMIT_I2C_MAX_FAST_MODE_FREQ		400000	/* Hz */
-
 #define SPACEMIT_SR_ERR	(SPACEMIT_SR_BED | SPACEMIT_SR_RXOV | SPACEMIT_SR_ALD)
 
 #define SPACEMIT_BUS_RESET_CLK_CNT_MAX		9
@@ -109,11 +123,20 @@ enum spacemit_i2c_state {
 	SPACEMIT_STATE_WRITE,
 };
 
+enum spacemit_i2c_mode {
+	SPACEMIT_MODE_STANDARD,
+	SPACEMIT_MODE_FAST
+};
+
 /* i2c-spacemit driver's main struct */
 struct spacemit_i2c_dev {
 	struct device *dev;
 	struct i2c_adapter adapt;
 
+	struct clk_hw scl_clk_hw;
+	struct clk *scl_clk;
+	enum spacemit_i2c_mode mode;
+
 	/* hardware resources */
 	void __iomem *base;
 	int irq;
@@ -135,6 +158,85 @@ struct spacemit_i2c_dev {
 	u32 status;
 };
 
+static void spacemit_i2c_scl_clk_disable_unprepare(void *data)
+{
+	clk_disable_unprepare(data);
+}
+
+/*
+ * Calculate the ILCR divider value (lv) from the target SCL rate.
+ *
+ * Hardware timing formulas:
+ * - standard mode: SCL = FCLK / (2 * SLV + 8)
+ * - fast mode:     SCL = FCLK / (2 * FLV + 10)
+ */
+static u32 spacemit_i2c_calc_lv(struct spacemit_i2c_dev *i2c,
+				unsigned long parent_rate,
+				unsigned long target_rate)
+{
+	u32 offset, denom;
+
+	offset = (i2c->mode == SPACEMIT_MODE_STANDARD) ? 8 : 10;
+	denom = DIV_ROUND_CLOSEST(parent_rate, target_rate);
+
+	return (denom <= offset) ? 0 : DIV_ROUND_CLOSEST(denom - offset, 2);
+}
+
+static int spacemit_i2c_clk_set_rate(struct clk_hw *hw, unsigned long rate,
+				     unsigned long parent_rate)
+{
+	struct spacemit_i2c_dev *i2c = container_of(hw, struct spacemit_i2c_dev, scl_clk_hw);
+	u32 lv, lcr, mask;
+
+	lv = spacemit_i2c_calc_lv(i2c, parent_rate, rate);
+
+	mask = (i2c->mode == SPACEMIT_MODE_STANDARD) ?
+		SPACEMIT_LCR_LV_STANDARD_MASK : SPACEMIT_LCR_LV_FAST_MASK;
+
+	lcr = readl(i2c->base + SPACEMIT_ILCR);
+	lcr &= ~mask;
+	lcr |= field_prep(mask, lv);
+	writel(lcr, i2c->base + SPACEMIT_ILCR);
+
+	return 0;
+}
+
+static int spacemit_i2c_clk_determine_rate(struct clk_hw *hw,
+					   struct clk_rate_request *req)
+{
+	struct spacemit_i2c_dev *i2c = container_of(hw, struct spacemit_i2c_dev, scl_clk_hw);
+	u32 lv, offset;
+
+	lv = spacemit_i2c_calc_lv(i2c, req->best_parent_rate, req->rate);
+	offset = (i2c->mode == SPACEMIT_MODE_STANDARD) ? 8 : 10;
+	req->rate = DIV_ROUND_CLOSEST(req->best_parent_rate, lv * 2 + offset);
+
+	return 0;
+}
+
+static unsigned long spacemit_i2c_clk_recalc_rate(struct clk_hw *hw,
+						  unsigned long parent_rate)
+{
+	struct spacemit_i2c_dev *i2c = container_of(hw, struct spacemit_i2c_dev, scl_clk_hw);
+	u32 lcr, lv = 0;
+
+	lcr = readl(i2c->base + SPACEMIT_ILCR);
+
+	if (i2c->mode == SPACEMIT_MODE_STANDARD) {
+		lv = FIELD_GET(SPACEMIT_LCR_LV_STANDARD_MASK, lcr);
+		return DIV_ROUND_CLOSEST(parent_rate, lv * 2 + 8);
+	}
+
+	lv = FIELD_GET(SPACEMIT_LCR_LV_FAST_MASK, lcr);
+	return DIV_ROUND_CLOSEST(parent_rate, lv * 2 + 10);
+}
+
+static const struct clk_ops spacemit_i2c_clk_ops = {
+	.set_rate = spacemit_i2c_clk_set_rate,
+	.determine_rate = spacemit_i2c_clk_determine_rate,
+	.recalc_rate = spacemit_i2c_clk_recalc_rate,
+};
+
 static void spacemit_i2c_enable(struct spacemit_i2c_dev *i2c)
 {
 	u32 val;
@@ -153,6 +255,28 @@ static void spacemit_i2c_disable(struct spacemit_i2c_dev *i2c)
 	writel(val, i2c->base + SPACEMIT_ICR);
 }
 
+static int spacemit_i2c_register_scl_clk(struct spacemit_i2c_dev *i2c)
+{
+	struct clk_init_data init = {};
+	char name[64];
+	int ret;
+
+	ret = snprintf(name, sizeof(name), "%s_scl_clk", dev_name(i2c->dev));
+	if (ret >= ARRAY_SIZE(name))
+		dev_warn(i2c->dev, "scl clock name truncated");
+
+	init.name = name;
+	init.ops = &spacemit_i2c_clk_ops;
+	init.parent_data = (struct clk_parent_data[]) {
+		{ .fw_name = "func" },
+	};
+	init.num_parents = 1;
+
+	i2c->scl_clk_hw.init = &init;
+
+	return devm_clk_hw_register(i2c->dev, &i2c->scl_clk_hw);
+}
+
 static void spacemit_i2c_reset(struct spacemit_i2c_dev *i2c)
 {
 	writel(SPACEMIT_CR_UR, i2c->base + SPACEMIT_ICR);
@@ -286,7 +410,7 @@ static void spacemit_i2c_init(struct spacemit_i2c_dev *i2c)
 		val |= SPACEMIT_CR_MSDIE;
 	}
 
-	if (i2c->clock_freq == SPACEMIT_I2C_MAX_FAST_MODE_FREQ)
+	if (i2c->mode == SPACEMIT_MODE_FAST)
 		val |= SPACEMIT_CR_MODE_FAST;
 
 	/* disable response to general call */
@@ -309,6 +433,14 @@ static void spacemit_i2c_init(struct spacemit_i2c_dev *i2c)
 	writel(val, i2c->base + SPACEMIT_IRCR);
 
 	spacemit_i2c_clear_int_status(i2c, SPACEMIT_I2C_INT_STATUS_MASK);
+
+	/*
+	 * Initialize IWCR to the value specified by the I2C IP designer.
+	 * The SCL frequency formulas (SCL = FCLK / (2*SLV+8) for standard
+	 * mode, SCL = FCLK / (2*FLV+10) for fast mode) are only valid when
+	 * IWCR contains this specific value.
+	 */
+	writel(SPACEMIT_IWCR_INIT_VALUE, i2c->base + SPACEMIT_IWCR);
 }
 
 static void spacemit_i2c_start(struct spacemit_i2c_dev *i2c)
@@ -596,7 +728,7 @@ static irqreturn_t spacemit_i2c_irq_handler(int irq, void *devid)
 
 	status = readl(i2c->base + SPACEMIT_ISR);
 	if (!status)
-		return IRQ_HANDLED;
+		return IRQ_NONE;
 
 	i2c->status = status;
 
@@ -698,19 +830,18 @@ static int spacemit_i2c_probe(struct platform_device *pdev)
 	if (!i2c)
 		return -ENOMEM;
 
-	ret = of_property_read_u32(of_node, "clock-frequency", &i2c->clock_freq);
-	if (ret && ret != -EINVAL)
-		dev_warn(dev, "failed to read clock-frequency property: %d\n", ret);
+	of_property_read_u32(of_node, "clock-frequency", &i2c->clock_freq);
 
 	/* For now, this driver doesn't support high-speed. */
-	if (!i2c->clock_freq || i2c->clock_freq > SPACEMIT_I2C_MAX_FAST_MODE_FREQ) {
-		dev_warn(dev, "unsupported clock frequency %u; using %u\n",
-			 i2c->clock_freq, SPACEMIT_I2C_MAX_FAST_MODE_FREQ);
-		i2c->clock_freq = SPACEMIT_I2C_MAX_FAST_MODE_FREQ;
-	} else if (i2c->clock_freq < SPACEMIT_I2C_MAX_STANDARD_MODE_FREQ) {
-		dev_warn(dev, "unsupported clock frequency %u; using %u\n",
-			 i2c->clock_freq,  SPACEMIT_I2C_MAX_STANDARD_MODE_FREQ);
-		i2c->clock_freq = SPACEMIT_I2C_MAX_STANDARD_MODE_FREQ;
+	if (i2c->clock_freq > I2C_MAX_STANDARD_MODE_FREQ &&
+	    i2c->clock_freq <= I2C_MAX_FAST_MODE_FREQ) {
+		i2c->mode = SPACEMIT_MODE_FAST;
+	} else if (i2c->clock_freq && i2c->clock_freq <= I2C_MAX_STANDARD_MODE_FREQ) {
+		i2c->mode = SPACEMIT_MODE_STANDARD;
+	} else {
+		dev_info(dev, "clock-frequency not set or out of range, using fast mode\n");
+		i2c->mode = SPACEMIT_MODE_FAST;
+		i2c->clock_freq = I2C_MAX_FAST_MODE_FREQ;
 	}
 
 	i2c->dev = &pdev->dev;
@@ -732,6 +863,15 @@ static int spacemit_i2c_probe(struct platform_device *pdev)
 	if (IS_ERR(clk))
 		return dev_err_probe(dev, PTR_ERR(clk), "failed to enable func clock");
 
+	ret = spacemit_i2c_register_scl_clk(i2c);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to register scl clock\n");
+
+	i2c->scl_clk = devm_clk_hw_get_clk(dev, &i2c->scl_clk_hw, "scl");
+	if (IS_ERR(i2c->scl_clk))
+		return dev_err_probe(dev, PTR_ERR(i2c->scl_clk),
+				     "failed to get scl clock\n");
+
 	clk = devm_clk_get_enabled(dev, "bus");
 	if (IS_ERR(clk))
 		return dev_err_probe(dev, PTR_ERR(clk), "failed to enable bus clock");
@@ -741,6 +881,19 @@ static int spacemit_i2c_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, PTR_ERR(rst),
 				     "failed to acquire deasserted reset\n");
 
+	ret = clk_set_rate(i2c->scl_clk, i2c->clock_freq);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to set rate for SCL clock");
+
+	ret = clk_prepare_enable(i2c->scl_clk);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to prepare and enable clock");
+
+	ret = devm_add_action_or_reset(dev, spacemit_i2c_scl_clk_disable_unprepare,
+				       i2c->scl_clk);
+	if (ret)
+		return ret;
+
 	spacemit_i2c_reset(i2c);
 
 	i2c_set_adapdata(&i2c->adapt, i2c);
diff --git a/drivers/i2c/busses/i2c-microchip-corei2c.c b/drivers/i2c/busses/i2c-microchip-corei2c.c
index c8599733633e7..330e150ef6d5f 100644
--- a/drivers/i2c/busses/i2c-microchip-corei2c.c
+++ b/drivers/i2c/busses/i2c-microchip-corei2c.c
@@ -565,10 +565,10 @@ static int mchp_corei2c_probe(struct platform_device *pdev)
 				       &idev->bus_clk_rate);
 	if (ret || !idev->bus_clk_rate) {
 		dev_info(&pdev->dev, "default to 100kHz\n");
-		idev->bus_clk_rate = 100000;
+		idev->bus_clk_rate = I2C_MAX_STANDARD_MODE_FREQ;
 	}
 
-	if (idev->bus_clk_rate > 400000)
+	if (idev->bus_clk_rate > I2C_MAX_FAST_MODE_FREQ)
 		return dev_err_probe(&pdev->dev, -EINVAL,
 				     "clock-frequency too high: %d\n",
 				     idev->bus_clk_rate);
diff --git a/drivers/i2c/busses/i2c-mlxbf.c b/drivers/i2c/busses/i2c-mlxbf.c
index 6c1cfe9ec8acc..e33512b253531 100644
--- a/drivers/i2c/busses/i2c-mlxbf.c
+++ b/drivers/i2c/busses/i2c-mlxbf.c
@@ -1051,8 +1051,10 @@ static int mlxbf_i2c_init_resource(struct platform_device *pdev,
 
 	tmp_res->io = devm_platform_get_and_ioremap_resource(pdev, type, &tmp_res->params);
 	if (IS_ERR(tmp_res->io)) {
+		int ret = PTR_ERR(tmp_res->io);
+
 		devm_kfree(dev, tmp_res);
-		return PTR_ERR(tmp_res->io);
+		return ret;
 	}
 
 	tmp_res->type = type;
diff --git a/drivers/i2c/busses/i2c-mt65xx.c b/drivers/i2c/busses/i2c-mt65xx.c
index 126040ca05f14..307925fb78e36 100644
--- a/drivers/i2c/busses/i2c-mt65xx.c
+++ b/drivers/i2c/busses/i2c-mt65xx.c
@@ -1258,7 +1258,7 @@ static int mtk_i2c_transfer(struct i2c_adapter *adap,
 	i2c->auto_restart = i2c->dev_comp->auto_restart;
 
 	/* checking if we can skip restart and optimize using WRRD mode */
-	if (i2c->auto_restart && num == 2) {
+	if (num == 2) {
 		if (!(msgs[0].flags & I2C_M_RD) && (msgs[1].flags & I2C_M_RD) &&
 		    msgs[0].addr == msgs[1].addr) {
 			i2c->auto_restart = 0;
diff --git a/drivers/i2c/busses/i2c-nomadik.c b/drivers/i2c/busses/i2c-nomadik.c
index b63ee51c16520..404709179d737 100644
--- a/drivers/i2c/busses/i2c-nomadik.c
+++ b/drivers/i2c/busses/i2c-nomadik.c
@@ -1050,9 +1050,9 @@ static int nmk_i2c_eyeq5_probe(struct nmk_i2c_dev *priv)
 	if (id >= ARRAY_SIZE(nmk_i2c_eyeq5_masks))
 		return -ENOENT;
 
-	if (priv->clk_freq <= 400000)
+	if (priv->clk_freq <= I2C_MAX_FAST_MODE_FREQ)
 		speed_mode = I2C_EYEQ5_SPEED_FAST;
-	else if (priv->clk_freq <= 1000000)
+	else if (priv->clk_freq <= I2C_MAX_FAST_MODE_PLUS_FREQ)
 		speed_mode = I2C_EYEQ5_SPEED_FAST_PLUS;
 	else
 		speed_mode = I2C_EYEQ5_SPEED_HIGH_SPEED;
diff --git a/drivers/i2c/busses/i2c-octeon-core.h b/drivers/i2c/busses/i2c-octeon-core.h
index 32a44f2d6274f..aba81477d7d4f 100644
--- a/drivers/i2c/busses/i2c-octeon-core.h
+++ b/drivers/i2c/busses/i2c-octeon-core.h
@@ -235,7 +235,7 @@ static inline void octeon_i2c_write_int(struct octeon_i2c *i2c, u64 data)
 	octeon_i2c_writeq_flush(data, i2c->twsi_base + OCTEON_REG_TWSI_INT(i2c));
 }
 
-#define IS_LS_FREQ(twsi_freq)	((twsi_freq) <= 400000)
+#define IS_LS_FREQ(twsi_freq)	((twsi_freq) <= I2C_MAX_FAST_MODE_FREQ)
 #define PCI_SUBSYS_DEVID_9XXX	0xB
 #define PCI_SUBSYS_MASK		GENMASK(15, 12)
 /**
diff --git a/drivers/i2c/busses/i2c-pnx.c b/drivers/i2c/busses/i2c-pnx.c
index 8daa0008bd052..e1cc2b2bd628f 100644
--- a/drivers/i2c/busses/i2c-pnx.c
+++ b/drivers/i2c/busses/i2c-pnx.c
@@ -24,7 +24,6 @@
 #include <linux/of.h>
 
 #define I2C_PNX_TIMEOUT_DEFAULT		10 /* msec */
-#define I2C_PNX_SPEED_KHZ_DEFAULT	100
 #define I2C_PNX_REGION_SIZE		0x100
 
 struct i2c_pnx_mif {
@@ -606,12 +605,12 @@ static DEFINE_SIMPLE_DEV_PM_OPS(i2c_pnx_pm,
 
 static int i2c_pnx_probe(struct platform_device *pdev)
 {
+	u32 speed = I2C_MAX_STANDARD_MODE_FREQ;
 	unsigned long tmp;
 	int ret = 0;
 	struct i2c_pnx_algo_data *alg_data;
 	unsigned long freq;
 	struct resource *res;
-	u32 speed = I2C_PNX_SPEED_KHZ_DEFAULT * 1000;
 
 	alg_data = devm_kzalloc(&pdev->dev, sizeof(*alg_data), GFP_KERNEL);
 	if (!alg_data)
diff --git a/drivers/i2c/busses/i2c-pxa-pci.c b/drivers/i2c/busses/i2c-pxa-pci.c
index dbd542300f800..92a0647f08c69 100644
--- a/drivers/i2c/busses/i2c-pxa-pci.c
+++ b/drivers/i2c/busses/i2c-pxa-pci.c
@@ -76,7 +76,8 @@ static struct platform_device *add_i2c_device(struct pci_dev *dev, int bar)
 		goto out;
 	}
 	pdev->dev.parent = &dev->dev;
-	pdev->dev.of_node = child;
+
+	platform_device_set_of_node(pdev, child);
 
 	ret = platform_device_add_resources(pdev, res, ARRAY_SIZE(res));
 	if (ret)
diff --git a/drivers/i2c/busses/i2c-qcom-geni.c b/drivers/i2c/busses/i2c-qcom-geni.c
index d2f5055b0b108..96dbf04138bee 100644
--- a/drivers/i2c/busses/i2c-qcom-geni.c
+++ b/drivers/i2c/busses/i2c-qcom-geni.c
@@ -77,6 +77,15 @@ enum geni_i2c_err_code {
 #define XFER_TIMEOUT		HZ
 #define RST_TIMEOUT		HZ
 
+struct geni_i2c_desc {
+	bool no_dma_support;
+	unsigned int tx_fifo_depth;
+	int (*resources_init)(struct geni_se *se);
+	int (*set_rate)(struct geni_se *se, unsigned long freq);
+	int (*power_on)(struct geni_se *se);
+	int (*power_off)(struct geni_se *se);
+};
+
 #define QCOM_I2C_MIN_NUM_OF_MSGS_MULTI_DESC	2
 
 /**
@@ -107,7 +116,6 @@ struct geni_i2c_dev {
 	int cur_wr;
 	int cur_rd;
 	spinlock_t lock;
-	struct clk *core_clk;
 	u32 clk_freq_out;
 	const struct geni_i2c_clk_fld *clk_fld;
 	void *dma_buf;
@@ -121,13 +129,7 @@ struct geni_i2c_dev {
 	bool is_tx_multi_desc_xfer;
 	u32 num_msgs;
 	struct geni_i2c_gpi_multi_desc_xfer i2c_multi_desc_config;
-};
-
-struct geni_i2c_desc {
-	bool has_core_clk;
-	char *icc_ddr;
-	bool no_dma_support;
-	unsigned int tx_fifo_depth;
+	const struct geni_i2c_desc *dev_data;
 };
 
 struct geni_i2c_err_log {
@@ -202,8 +204,9 @@ static int geni_i2c_clk_map_idx(struct geni_i2c_dev *gi2c)
 	return -EINVAL;
 }
 
-static void qcom_geni_i2c_conf(struct geni_i2c_dev *gi2c)
+static int qcom_geni_i2c_conf(struct geni_se *se, unsigned long freq)
 {
+	struct geni_i2c_dev *gi2c = dev_get_drvdata(se->dev);
 	const struct geni_i2c_clk_fld *itr = gi2c->clk_fld;
 	u32 val;
 
@@ -216,6 +219,7 @@ static void qcom_geni_i2c_conf(struct geni_i2c_dev *gi2c)
 	val |= itr->t_low_cnt << LOW_COUNTER_SHFT;
 	val |= itr->t_cycle_cnt;
 	writel_relaxed(val, gi2c->se.base + SE_I2C_SCL_COUNTERS);
+	return 0;
 }
 
 static void geni_i2c_err_misc(struct geni_i2c_dev *gi2c)
@@ -921,7 +925,9 @@ static int geni_i2c_xfer(struct i2c_adapter *adap,
 		return ret;
 	}
 
-	qcom_geni_i2c_conf(gi2c);
+	ret = gi2c->dev_data->set_rate(&gi2c->se, gi2c->clk_freq_out);
+	if (ret)
+		return ret;
 
 	if (gi2c->gpi_mode)
 		ret = geni_i2c_gpi_xfer(gi2c, msgs, num);
@@ -944,15 +950,6 @@ static const struct i2c_algorithm geni_i2c_algo = {
 	.functionality = geni_i2c_func,
 };
 
-#ifdef CONFIG_ACPI
-static const struct acpi_device_id geni_i2c_acpi_match[] = {
-	{ "QCOM0220"},
-	{ "QCOM0411" },
-	{ }
-};
-MODULE_DEVICE_TABLE(acpi, geni_i2c_acpi_match);
-#endif
-
 static void release_gpi_dma(struct geni_i2c_dev *gi2c)
 {
 	if (gi2c->rx_c)
@@ -990,13 +987,95 @@ static int setup_gpi_dma(struct geni_i2c_dev *gi2c)
 	return ret;
 }
 
+static int geni_i2c_init(struct geni_i2c_dev *gi2c)
+{
+	u32 proto, tx_depth;
+	bool fifo_disable;
+	int ret;
+
+	ret = pm_runtime_resume_and_get(gi2c->se.dev);
+	if (ret < 0) {
+		dev_err(gi2c->se.dev, "error turning on device :%d\n", ret);
+		return ret;
+	}
+
+	proto = geni_se_read_proto(&gi2c->se);
+	if (proto == GENI_SE_INVALID_PROTO) {
+		ret = geni_load_se_firmware(&gi2c->se, GENI_SE_I2C);
+		if (ret) {
+			dev_err_probe(gi2c->se.dev, ret, "i2c firmware load failed ret: %d\n", ret);
+			goto err;
+		}
+	} else if (proto != GENI_SE_I2C) {
+		ret = dev_err_probe(gi2c->se.dev, -ENXIO, "Invalid proto %d\n", proto);
+		goto err;
+	}
+
+	if (gi2c->dev_data->no_dma_support) {
+		fifo_disable = false;
+		gi2c->no_dma = true;
+	} else {
+		fifo_disable = readl_relaxed(gi2c->se.base + GENI_IF_DISABLE_RO) & FIFO_IF_DISABLE;
+	}
+
+	if (fifo_disable) {
+		/* FIFO is disabled, so we can only use GPI DMA */
+		gi2c->gpi_mode = true;
+		ret = setup_gpi_dma(gi2c);
+		if (ret)
+			goto err;
+
+		dev_dbg(gi2c->se.dev, "Using GPI DMA mode for I2C\n");
+	} else {
+		gi2c->gpi_mode = false;
+		tx_depth = geni_se_get_tx_fifo_depth(&gi2c->se);
+
+		/* I2C Master Hub Serial Elements doesn't have the HW_PARAM_0 register */
+		if (!tx_depth && gi2c->se.core_clk)
+			tx_depth = gi2c->dev_data->tx_fifo_depth;
+
+		if (!tx_depth) {
+			ret = dev_err_probe(gi2c->se.dev, -EINVAL,
+					    "Invalid TX FIFO depth\n");
+			goto err;
+		}
+
+		gi2c->tx_wm = tx_depth - 1;
+		geni_se_init(&gi2c->se, gi2c->tx_wm, tx_depth);
+		geni_se_config_packing(&gi2c->se, BITS_PER_BYTE,
+				       PACKING_BYTES_PW, true, true, true);
+
+		dev_dbg(gi2c->se.dev, "i2c fifo/se-dma mode. fifo depth:%d\n", tx_depth);
+	}
+
+err:
+	pm_runtime_put(gi2c->se.dev);
+	return ret;
+}
+
+static int geni_i2c_resources_init(struct geni_se *se)
+{
+	struct geni_i2c_dev *gi2c = dev_get_drvdata(se->dev);
+	int ret;
+
+	ret = geni_se_resources_init(&gi2c->se);
+	if (ret)
+		return ret;
+
+	ret = geni_i2c_clk_map_idx(gi2c);
+	if (ret)
+		return dev_err_probe(gi2c->se.dev, ret, "Invalid clk frequency %d Hz\n",
+				     gi2c->clk_freq_out);
+
+	return geni_icc_set_bw_ab(&gi2c->se, GENI_DEFAULT_BW, GENI_DEFAULT_BW,
+				  Bps_to_icc(gi2c->clk_freq_out));
+}
+
 static int geni_i2c_probe(struct platform_device *pdev)
 {
 	struct geni_i2c_dev *gi2c;
-	u32 proto, tx_depth, fifo_disable;
 	int ret;
 	struct device *dev = &pdev->dev;
-	const struct geni_i2c_desc *desc = NULL;
 
 	gi2c = devm_kzalloc(dev, sizeof(*gi2c), GFP_KERNEL);
 	if (!gi2c)
@@ -1008,17 +1087,9 @@ static int geni_i2c_probe(struct platform_device *pdev)
 	if (IS_ERR(gi2c->se.base))
 		return PTR_ERR(gi2c->se.base);
 
-	desc = device_get_match_data(&pdev->dev);
-
-	if (desc && desc->has_core_clk) {
-		gi2c->core_clk = devm_clk_get(dev, "core");
-		if (IS_ERR(gi2c->core_clk))
-			return PTR_ERR(gi2c->core_clk);
-	}
-
-	gi2c->se.clk = devm_clk_get(dev, "se");
-	if (IS_ERR(gi2c->se.clk) && !has_acpi_companion(dev))
-		return PTR_ERR(gi2c->se.clk);
+	gi2c->dev_data = device_get_match_data(&pdev->dev);
+	if (!gi2c->dev_data)
+		return -EINVAL;
 
 	ret = device_property_read_u32(dev, "clock-frequency",
 				       &gi2c->clk_freq_out);
@@ -1034,16 +1105,15 @@ static int geni_i2c_probe(struct platform_device *pdev)
 	if (gi2c->irq < 0)
 		return gi2c->irq;
 
-	ret = geni_i2c_clk_map_idx(gi2c);
-	if (ret)
-		return dev_err_probe(dev, ret, "Invalid clk frequency %d Hz\n",
-				     gi2c->clk_freq_out);
-
 	gi2c->adap.algo = &geni_i2c_algo;
 	init_completion(&gi2c->done);
 	spin_lock_init(&gi2c->lock);
 	platform_set_drvdata(pdev, gi2c);
 
+	ret = gi2c->dev_data->resources_init(&gi2c->se);
+	if (ret)
+		return ret;
+
 	/* Keep interrupts disabled initially to allow for low-power modes */
 	ret = devm_request_irq(dev, gi2c->irq, geni_i2c_irq, IRQF_NO_AUTOEN,
 			       dev_name(dev), gi2c);
@@ -1056,118 +1126,26 @@ static int geni_i2c_probe(struct platform_device *pdev)
 	gi2c->adap.dev.of_node = dev->of_node;
 	strscpy(gi2c->adap.name, "Geni-I2C", sizeof(gi2c->adap.name));
 
-	ret = geni_icc_get(&gi2c->se, desc ? desc->icc_ddr : "qup-memory");
-	if (ret)
-		return ret;
-	/*
-	 * Set the bus quota for core and cpu to a reasonable value for
-	 * register access.
-	 * Set quota for DDR based on bus speed.
-	 */
-	gi2c->se.icc_paths[GENI_TO_CORE].avg_bw = GENI_DEFAULT_BW;
-	gi2c->se.icc_paths[CPU_TO_GENI].avg_bw = GENI_DEFAULT_BW;
-	if (!desc || desc->icc_ddr)
-		gi2c->se.icc_paths[GENI_TO_DDR].avg_bw = Bps_to_icc(gi2c->clk_freq_out);
-
-	ret = geni_icc_set_bw(&gi2c->se);
-	if (ret)
-		return ret;
-
-	ret = clk_prepare_enable(gi2c->core_clk);
-	if (ret)
-		return ret;
-
-	ret = geni_se_resources_on(&gi2c->se);
-	if (ret) {
-		dev_err_probe(dev, ret, "Error turning on resources\n");
-		goto err_clk;
-	}
-	proto = geni_se_read_proto(&gi2c->se);
-	if (proto == GENI_SE_INVALID_PROTO) {
-		ret = geni_load_se_firmware(&gi2c->se, GENI_SE_I2C);
-		if (ret) {
-			dev_err_probe(dev, ret, "i2c firmware load failed ret: %d\n", ret);
-			goto err_resources;
-		}
-	} else if (proto != GENI_SE_I2C) {
-		ret = dev_err_probe(dev, -ENXIO, "Invalid proto %d\n", proto);
-		goto err_resources;
-	}
-
-	if (desc && desc->no_dma_support) {
-		fifo_disable = false;
-		gi2c->no_dma = true;
-	} else {
-		fifo_disable = readl_relaxed(gi2c->se.base + GENI_IF_DISABLE_RO) & FIFO_IF_DISABLE;
-	}
-
-	if (fifo_disable) {
-		/* FIFO is disabled, so we can only use GPI DMA */
-		gi2c->gpi_mode = true;
-		ret = setup_gpi_dma(gi2c);
-		if (ret)
-			goto err_resources;
-
-		dev_dbg(dev, "Using GPI DMA mode for I2C\n");
-	} else {
-		gi2c->gpi_mode = false;
-		tx_depth = geni_se_get_tx_fifo_depth(&gi2c->se);
-
-		/* I2C Master Hub Serial Elements doesn't have the HW_PARAM_0 register */
-		if (!tx_depth && desc)
-			tx_depth = desc->tx_fifo_depth;
-
-		if (!tx_depth) {
-			ret = dev_err_probe(dev, -EINVAL,
-					    "Invalid TX FIFO depth\n");
-			goto err_resources;
-		}
-
-		gi2c->tx_wm = tx_depth - 1;
-		geni_se_init(&gi2c->se, gi2c->tx_wm, tx_depth);
-		geni_se_config_packing(&gi2c->se, BITS_PER_BYTE,
-				       PACKING_BYTES_PW, true, true, true);
-
-		dev_dbg(dev, "i2c fifo/se-dma mode. fifo depth:%d\n", tx_depth);
-	}
-
-	clk_disable_unprepare(gi2c->core_clk);
-	ret = geni_se_resources_off(&gi2c->se);
-	if (ret) {
-		dev_err_probe(dev, ret, "Error turning off resources\n");
-		goto err_dma;
-	}
-
-	ret = geni_icc_disable(&gi2c->se);
-	if (ret)
-		goto err_dma;
-
 	pm_runtime_set_suspended(gi2c->se.dev);
 	pm_runtime_set_autosuspend_delay(gi2c->se.dev, I2C_AUTO_SUSPEND_DELAY);
 	pm_runtime_use_autosuspend(gi2c->se.dev);
 	pm_runtime_enable(gi2c->se.dev);
 
+	ret = geni_i2c_init(gi2c);
+	if (ret < 0) {
+		pm_runtime_disable(gi2c->se.dev);
+		return ret;
+	}
+
 	ret = i2c_add_adapter(&gi2c->adap);
 	if (ret) {
 		dev_err_probe(dev, ret, "Error adding i2c adapter\n");
 		pm_runtime_disable(gi2c->se.dev);
-		goto err_dma;
+		return ret;
 	}
 
 	dev_dbg(dev, "Geni-I2C adaptor successfully added\n");
 
-	return ret;
-
-err_resources:
-	geni_se_resources_off(&gi2c->se);
-err_clk:
-	clk_disable_unprepare(gi2c->core_clk);
-
-	return ret;
-
-err_dma:
-	release_gpi_dma(gi2c);
-
 	return ret;
 }
 
@@ -1190,48 +1168,36 @@ static void geni_i2c_shutdown(struct platform_device *pdev)
 
 static int __maybe_unused geni_i2c_runtime_suspend(struct device *dev)
 {
-	int ret;
+	int ret = 0;
 	struct geni_i2c_dev *gi2c = dev_get_drvdata(dev);
 
 	disable_irq(gi2c->irq);
-	ret = geni_se_resources_off(&gi2c->se);
-	if (ret) {
-		enable_irq(gi2c->irq);
-		return ret;
-	}
 
-	clk_disable_unprepare(gi2c->core_clk);
+	if (gi2c->dev_data->power_off) {
+		ret = gi2c->dev_data->power_off(&gi2c->se);
+		if (ret) {
+			enable_irq(gi2c->irq);
+			return ret;
+		}
+	}
 
-	return geni_icc_disable(&gi2c->se);
+	return 0;
 }
 
 static int __maybe_unused geni_i2c_runtime_resume(struct device *dev)
 {
-	int ret;
+	int ret = 0;
 	struct geni_i2c_dev *gi2c = dev_get_drvdata(dev);
 
-	ret = geni_icc_enable(&gi2c->se);
-	if (ret)
-		return ret;
-
-	ret = clk_prepare_enable(gi2c->core_clk);
-	if (ret)
-		goto out_icc_disable;
-
-	ret = geni_se_resources_on(&gi2c->se);
-	if (ret)
-		goto out_clk_disable;
+	if (gi2c->dev_data->power_on) {
+		ret = gi2c->dev_data->power_on(&gi2c->se);
+		if (ret)
+			return ret;
+	}
 
 	enable_irq(gi2c->irq);
 
 	return 0;
-
-out_clk_disable:
-	clk_disable_unprepare(gi2c->core_clk);
-out_icc_disable:
-	geni_icc_disable(&gi2c->se);
-
-	return ret;
 }
 
 static int __maybe_unused geni_i2c_suspend_noirq(struct device *dev)
@@ -1267,16 +1233,40 @@ static const struct dev_pm_ops geni_i2c_pm_ops = {
 									NULL)
 };
 
+static const struct geni_i2c_desc geni_i2c = {
+	.resources_init = geni_i2c_resources_init,
+	.set_rate = qcom_geni_i2c_conf,
+	.power_on = geni_se_resources_activate,
+	.power_off = geni_se_resources_deactivate,
+};
+
 static const struct geni_i2c_desc i2c_master_hub = {
-	.has_core_clk = true,
-	.icc_ddr = NULL,
 	.no_dma_support = true,
 	.tx_fifo_depth = 16,
+	.resources_init = geni_i2c_resources_init,
+	.set_rate = qcom_geni_i2c_conf,
+	.power_on = geni_se_resources_activate,
+	.power_off = geni_se_resources_deactivate,
+};
+
+static const struct geni_i2c_desc sa8255p_geni_i2c = {
+	.resources_init = geni_se_domain_attach,
+	.set_rate = geni_se_set_perf_opp,
+};
+
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id geni_i2c_acpi_match[] = {
+	{ "QCOM0220", (kernel_ulong_t)&geni_i2c},
+	{ "QCOM0411", (kernel_ulong_t)&geni_i2c},
+	{ }
 };
+MODULE_DEVICE_TABLE(acpi, geni_i2c_acpi_match);
+#endif
 
 static const struct of_device_id geni_i2c_dt_match[] = {
-	{ .compatible = "qcom,geni-i2c" },
+	{ .compatible = "qcom,geni-i2c", .data = &geni_i2c },
 	{ .compatible = "qcom,geni-i2c-master-hub", .data = &i2c_master_hub },
+	{ .compatible = "qcom,sa8255p-geni-i2c", .data = &sa8255p_geni_i2c },
 	{}
 };
 MODULE_DEVICE_TABLE(of, geni_i2c_dt_match);
diff --git a/drivers/i2c/i2c-core-acpi.c b/drivers/i2c/i2c-core-acpi.c
index e5fddacae9a4b..10cdceaba4759 100644
--- a/drivers/i2c/i2c-core-acpi.c
+++ b/drivers/i2c/i2c-core-acpi.c
@@ -168,9 +168,12 @@ static int i2c_acpi_do_lookup(struct acpi_device *adev,
 	INIT_LIST_HEAD(&resource_list);
 	ret = acpi_dev_get_resources(adev, &resource_list,
 				     i2c_acpi_fill_info, lookup);
+	if (ret < 0)
+		return ret;
+
 	acpi_dev_free_resource_list(&resource_list);
 
-	if (ret < 0 || !info->addr)
+	if (!info->addr)
 		return -EINVAL;
 
 	return 0;
@@ -376,6 +379,7 @@ static const struct acpi_device_id i2c_acpi_force_100khz_device_ids[] = {
 	{ "DLL0945", 0 },
 	{ "ELAN0678", 0 },
 	{ "ELAN06FA", 0 },
+	{ "ELAN1300", 0 },
 	{}
 };
 
diff --git a/drivers/i3c/master/adi-i3c-master.c b/drivers/i3c/master/adi-i3c-master.c
index 047081c9f0643..e29aac2869577 100644
--- a/drivers/i3c/master/adi-i3c-master.c
+++ b/drivers/i3c/master/adi-i3c-master.c
@@ -964,17 +964,9 @@ static int adi_i3c_master_probe(struct platform_device *pdev)
 	writel(0x00, master->regs + REG_ENABLE);
 	writel(0x00, master->regs + REG_IRQ_MASK);
 
-	ret = devm_request_irq(&pdev->dev, irq, adi_i3c_master_irq, 0,
-			       dev_name(&pdev->dev), master);
-	if (ret)
-		return ret;
-
 	platform_set_drvdata(pdev, master);
 
 	master->free_rr_slots = GENMASK(ADI_MAX_DEVS, 1);
-
-	writel(REG_IRQ_PENDING_CMDR, master->regs + REG_IRQ_MASK);
-
 	spin_lock_init(&master->ibi.lock);
 	master->ibi.num_slots = 15;
 	master->ibi.slots = devm_kcalloc(&pdev->dev, master->ibi.num_slots,
@@ -986,6 +978,13 @@ static int adi_i3c_master_probe(struct platform_device *pdev)
 	spin_lock_init(&master->xferqueue.lock);
 	INIT_LIST_HEAD(&master->xferqueue.list);
 
+	ret = devm_request_irq(&pdev->dev, irq, adi_i3c_master_irq, 0,
+			       dev_name(&pdev->dev), master);
+	if (ret)
+		return ret;
+
+	writel(REG_IRQ_PENDING_CMDR, master->regs + REG_IRQ_MASK);
+
 	return i3c_master_register(&master->base, &pdev->dev,
 				   &adi_i3c_master_ops, false);
 }
diff --git a/drivers/i3c/master/dw-i3c-master.c b/drivers/i3c/master/dw-i3c-master.c
index 2f8c0c4683e06..dc3b74822f8ea 100644
--- a/drivers/i3c/master/dw-i3c-master.c
+++ b/drivers/i3c/master/dw-i3c-master.c
@@ -888,7 +888,15 @@ static int dw_i3c_master_daa(struct i3c_master_controller *m)
 	if (!wait_for_completion_timeout(&xfer->comp, XFER_TIMEOUT))
 		dw_i3c_master_dequeue_xfer(master, xfer);
 
-	newdevs = GENMASK(master->maxdevs - cmd->rx_len - 1, 0);
+	/*
+	 * cmd->rx_len holds the number of addresses ENTDAA left unassigned.
+	 * On an empty bus rx_len == maxdevs, so avoid GENMASK(-1, 0).
+	 */
+	if (cmd->rx_len >= master->maxdevs)
+		newdevs = 0;
+	else
+		newdevs = GENMASK(master->maxdevs - cmd->rx_len - 1, 0);
+
 	newdevs &= ~olddevs;
 
 	for (pos = 0; pos < master->maxdevs; pos++) {
diff --git a/drivers/i3c/master/svc-i3c-master.c b/drivers/i3c/master/svc-i3c-master.c
index 93805df8a9406..486c22840ea9a 100644
--- a/drivers/i3c/master/svc-i3c-master.c
+++ b/drivers/i3c/master/svc-i3c-master.c
@@ -455,14 +455,22 @@ static int svc_i3c_master_handle_ibi(struct svc_i3c_master *master,
 	buf = slot->data;
 
 	while (SVC_I3C_MSTATUS_RXPEND(readl(master->regs + SVC_I3C_MSTATUS))  &&
-	       slot->len < SVC_I3C_FIFO_SIZE) {
+	       slot->len < dev->ibi->max_payload_len) {
 		mdatactrl = readl(master->regs + SVC_I3C_MDATACTRL);
 		count = SVC_I3C_MDATACTRL_RXCOUNT(mdatactrl);
+		count = min(count, dev->ibi->max_payload_len - slot->len);
 		readsb(master->regs + SVC_I3C_MRDATAB, buf, count);
 		slot->len += count;
 		buf += count;
 	}
 
+	/*
+	 * The device may have sent more than the requested payload. Drop the
+	 * extra bytes so they do not leak into the next transfer.
+	 */
+	if (SVC_I3C_MSTATUS_RXPEND(readl(master->regs + SVC_I3C_MSTATUS)))
+		writel(SVC_I3C_MDATACTRL_FLUSHRB, master->regs + SVC_I3C_MDATACTRL);
+
 	master->ibi.tbq_slot = slot;
 
 	return 0;
@@ -1488,8 +1496,11 @@ static int svc_i3c_master_xfer(struct svc_i3c_master *master,
 			svc_i3c_master_emit_force_exit(master);
 
 		/* Wait idle if stop is sent. */
-		readl_poll_timeout(master->regs + SVC_I3C_MSTATUS, reg,
-				   SVC_I3C_MSTATUS_STATE_IDLE(reg), 0, 1000);
+		ret = readl_poll_timeout(master->regs + SVC_I3C_MSTATUS, reg,
+					 SVC_I3C_MSTATUS_STATE_IDLE(reg),
+					 0, 1000);
+		if (ret)
+			goto cleanup;
 	}
 
 	return 0;
@@ -1500,6 +1511,7 @@ static int svc_i3c_master_xfer(struct svc_i3c_master *master,
 	else
 		svc_i3c_master_emit_force_exit(master);
 
+cleanup:
 	svc_i3c_master_clear_merrwarn(master);
 	svc_i3c_master_flush_fifo(master);
 
diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c
index d74b478db2804..bcc2725769c5b 100644
--- a/drivers/idle/intel_idle.c
+++ b/drivers/idle/intel_idle.c
@@ -1779,6 +1779,7 @@ module_param_named(no_native, no_native, bool, 0444);
 MODULE_PARM_DESC(no_native, "Ignore cpu specific (native) idle states in lieu of ACPI idle states");
 
 static struct acpi_processor_power acpi_state_table __initdata;
+static bool acpi_lpi_available __initdata;
 
 /**
  * intel_idle_cst_usable - Check if the _CST information can be used.
@@ -1803,18 +1804,37 @@ static bool __init intel_idle_cst_usable(void)
 	return true;
 }
 
-static bool __init intel_idle_acpi_cst_extract(void)
+static bool __init intel_idle_acpi_extract_lpi_cstates(void)
 {
 	unsigned int cpu;
 
-	if (no_acpi) {
-		pr_debug("Not allowed to use ACPI _CST\n");
-		return false;
+	for_each_possible_cpu(cpu) {
+		struct acpi_processor *pr;
+
+		pr = per_cpu(processors, cpu);
+		if (!pr)
+			continue;
+
+		if (acpi_processor_extract_lpi_info(pr->handle,
+						    &acpi_state_table, true))
+			continue;
+
+		acpi_lpi_available = true;
+		return true;
 	}
 
+	pr_debug("No ACPI _LPI idle states\n");
+	return false;
+}
+
+static bool __init intel_idle_acpi_extract_cst_cstates(void)
+{
+	unsigned int cpu;
+
 	for_each_possible_cpu(cpu) {
-		struct acpi_processor *pr = per_cpu(processors, cpu);
+		struct acpi_processor *pr;
 
+		pr = per_cpu(processors, cpu);
 		if (!pr)
 			continue;
 
@@ -1826,18 +1846,96 @@ static bool __init intel_idle_acpi_cst_extract(void)
 		if (!intel_idle_cst_usable())
 			continue;
 
-		if (!acpi_processor_claim_cst_control())
-			break;
+		return true;
+	}
+
+	pr_debug("ACPI _CST not found or not usable\n");
+	return false;
+}
+
+static bool __init intel_idle_acpi_extract_cstates(void)
+{
+	if (intel_idle_acpi_extract_lpi_cstates())
+		return true;
 
+	if (intel_idle_acpi_extract_cst_cstates())
 		return true;
+
+	return false;
+}
+
+static bool __init intel_idle_acpi_probe(void)
+{
+	if (no_acpi) {
+		pr_debug("Not allowed to use ACPI for C-states extraction\n");
+		return false;
 	}
 
+	if (intel_idle_acpi_extract_cstates() &&
+	    acpi_processor_claim_cst_control())
+		return true;
+
 	acpi_state_table.count = 0;
-	pr_debug("ACPI _CST not found or not usable\n");
 	return false;
 }
 
-static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)
+static void __init intel_idle_complete_state_init(struct cpuidle_state *state)
+{
+	if (intel_idle_state_needs_timer_stop(state))
+		state->flags |= CPUIDLE_FLAG_TIMER_STOP;
+
+	state->enter = intel_idle;
+	state->enter_dead = intel_idle_enter_dead;
+	state->enter_s2idle = intel_idle_s2idle;
+}
+
+static void __init intel_idle_init_cstates_acpi_lpi(struct cpuidle_driver *drv)
+{
+	int index;
+
+	for (index = 0; index < acpi_state_table.count; index++) {
+		struct acpi_lpi_state *lpi_state;
+		struct cpuidle_state *state;
+
+		if (intel_idle_max_cstate_reached(index))
+			break;
+
+		lpi_state = &acpi_state_table.lpi_states[index];
+
+		state = &drv->states[drv->state_count++];
+
+		scnprintf(state->name, CPUIDLE_NAME_LEN, "C%d_LPI", index + 1);
+		strscpy(state->desc, lpi_state->desc, CPUIDLE_DESC_LEN);
+		state->exit_latency = lpi_state->wake_latency;
+		state->target_residency = lpi_state->min_residency;
+		state->flags = MWAIT2flg(lpi_state->address);
+		/*
+		 * Assume that entering any of the idle states extracted from
+		 * _LPI except for the first two will cause the TLB to be
+		 * flushed and let the core call leave_mm() for them upfront
+		 * to avoid unnecessary wakeups due to TLB shootdowns.
+		 */
+		if (index > 1)
+			state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
+
+		if (disabled_states_mask & BIT(index + 1))
+			state->flags |= CPUIDLE_FLAG_OFF;
+
+		intel_idle_complete_state_init(state);
+
+		pr_info("%s: MWAIT hint 0x%x\n", state->name, flg2MWAIT(state->flags));
+	}
+
+	/*
+	 * Assume the first idle state in the table to be C1 and if any deeper
+	 * idle states are exposed while X86_FEATURE_NONSTOP_TSC is unset, mark
+	 * the TSC as unstable.
+	 */
+	if (index > 1 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
+		mark_tsc_unstable("TSC halts in idle");
+}
+
+static void __init intel_idle_init_cstates_acpi_cst(struct cpuidle_driver *drv)
 {
 	int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
 
@@ -1879,49 +1977,76 @@ static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)
 		if (disabled_states_mask & BIT(cstate))
 			state->flags |= CPUIDLE_FLAG_OFF;
 
-		if (intel_idle_state_needs_timer_stop(state))
-			state->flags |= CPUIDLE_FLAG_TIMER_STOP;
-
 		if (cx->type > ACPI_STATE_C1 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
 			mark_tsc_unstable("TSC halts in idle");
 
-		state->enter = intel_idle;
-		state->enter_dead = intel_idle_enter_dead;
-		state->enter_s2idle = intel_idle_s2idle;
+		intel_idle_complete_state_init(state);
 	}
 }
 
-static bool __init intel_idle_off_by_default(unsigned int flags, u32 mwait_hint)
+static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)
 {
-	int cstate, limit;
+	if (acpi_lpi_available)
+		intel_idle_init_cstates_acpi_lpi(drv);
+	else
+		intel_idle_init_cstates_acpi_cst(drv);
+}
 
-	/*
-	 * If there are no _CST C-states, do not disable any C-states by
-	 * default.
-	 */
-	if (!acpi_state_table.count)
-		return false;
+static bool __init intel_idle_acpi_hint_match(unsigned int flags, u32 acpi_hint,
+					      u32 table_hint)
+{
+	if (flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH) {
+		acpi_hint &= ~MWAIT_SUBSTATE_MASK;
+		table_hint &= ~MWAIT_SUBSTATE_MASK;
+	}
+	return acpi_hint == table_hint;
+}
+
+static bool __init intel_idle_off_by_default_lpi(unsigned int flags, u32 mwait_hint)
+{
+	int index;
+
+	for (index = 0; index < acpi_state_table.count; index++) {
+		u32 acpi_hint = acpi_state_table.lpi_states[index].address;
+
+		if (intel_idle_acpi_hint_match(flags, acpi_hint, mwait_hint))
+			return false;
+	}
+	return true;
+}
+
+static bool __init intel_idle_off_by_default_cst(unsigned int flags, u32 mwait_hint)
+{
+	int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
 
-	limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
 	/*
 	 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
 	 * the interesting states are ACPI_CSTATE_FFH.
 	 */
 	for (cstate = 1; cstate < limit; cstate++) {
 		u32 acpi_hint = acpi_state_table.states[cstate].address;
-		u32 table_hint = mwait_hint;
 
-		if (flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH) {
-			acpi_hint &= ~MWAIT_SUBSTATE_MASK;
-			table_hint &= ~MWAIT_SUBSTATE_MASK;
-		}
-
-		if (acpi_hint == table_hint)
+		if (intel_idle_acpi_hint_match(flags, acpi_hint, mwait_hint))
 			return false;
 	}
 	return true;
 }
 
+static bool __init intel_idle_off_by_default(unsigned int flags, u32 mwait_hint)
+{
+	/*
+	 * If there is no C-states information in the ACPI tables, do not
+	 * disable any C-states by default.
+	 */
+	if (!acpi_state_table.count)
+		return false;
+
+	if (acpi_lpi_available)
+		return intel_idle_off_by_default_lpi(flags, mwait_hint);
+
+	return intel_idle_off_by_default_cst(flags, mwait_hint);
+}
+
 static inline bool ignore_native(void)
 {
 	return no_native && !no_acpi;
@@ -1929,7 +2054,7 @@ static inline bool ignore_native(void)
 #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */
 #define force_use_acpi	(false)
 
-static inline bool intel_idle_acpi_cst_extract(void) { return false; }
+static inline bool intel_idle_acpi_probe(void) { return false; }
 static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { }
 static inline bool intel_idle_off_by_default(unsigned int flags, u32 mwait_hint)
 {
@@ -2741,7 +2866,7 @@ static int __init intel_idle_init(void)
 	if (icpu) {
 		if (icpu->state_table)
 			cpuidle_state_table = icpu->state_table;
-		else if (!intel_idle_acpi_cst_extract())
+		else if (!intel_idle_acpi_probe())
 			return -ENODEV;
 
 		auto_demotion_disable_flags = icpu->auto_demotion_disable_flags;
@@ -2750,8 +2875,8 @@ static int __init intel_idle_init(void)
 		if (icpu->c1_demotion_supported)
 			c1_demotion_supported = true;
 		if (icpu->use_acpi || force_use_acpi)
-			intel_idle_acpi_cst_extract();
-	} else if (!intel_idle_acpi_cst_extract()) {
+			intel_idle_acpi_probe();
+	} else if (!intel_idle_acpi_probe()) {
 		return -ENODEV;
 	}
 
diff --git a/drivers/iio/TODO b/drivers/iio/TODO
index 2ace27d1ac62a..7b05308248d7c 100644
--- a/drivers/iio/TODO
+++ b/drivers/iio/TODO
@@ -5,12 +5,17 @@ Documentation
 tree
   - Yaml conversions for abandoned drivers
   - ABI Documentation
-  - Audit driviers/iio/staging/Documentation
+  - Audit drivers/staging/iio/Documentation
 
-- Converting drivers from device tree centric to more generic
-property handlers.
+- Converting drivers from device tree centric to generic property handlers.
 
 - Refactor old platform_data constructs from drivers and convert it
-to state struct and using property handlers and readers.
+  to state struct and using property handlers and readers.
+
+- Convert selected IIO drivers to use fully device managed resource APIs.
+
+- Evaluate and possibly adopt cleanup.h helpers (guard()(), __free(), et cetera)
+  in drivers where manual resource management is still used and there is a
+  significant improvement to the code flow and complexity to be had.
 
 Mailing list: linux-iio@vger.kernel.org
diff --git a/drivers/iio/accel/adxl345_spi.c b/drivers/iio/accel/adxl345_spi.c
index 0315f4bfd69a7..4fa1b04f75b63 100644
--- a/drivers/iio/accel/adxl345_spi.c
+++ b/drivers/iio/accel/adxl345_spi.c
@@ -60,8 +60,8 @@ static const struct adxl345_chip_info adxl375_spi_info = {
 };
 
 static const struct spi_device_id adxl345_spi_id[] = {
-	{ "adxl345", (kernel_ulong_t)&adxl345_spi_info },
-	{ "adxl375", (kernel_ulong_t)&adxl375_spi_info },
+	{ .name = "adxl345", .driver_data = (kernel_ulong_t)&adxl345_spi_info },
+	{ .name = "adxl375", .driver_data = (kernel_ulong_t)&adxl375_spi_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adxl345_spi_id);
diff --git a/drivers/iio/accel/adxl355_spi.c b/drivers/iio/accel/adxl355_spi.c
index 437be2f1d53ce..d2209ca8ca6e8 100644
--- a/drivers/iio/accel/adxl355_spi.c
+++ b/drivers/iio/accel/adxl355_spi.c
@@ -40,8 +40,8 @@ static int adxl355_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id adxl355_spi_id[] = {
-	{ "adxl355", (kernel_ulong_t)&adxl35x_chip_info[ADXL355] },
-	{ "adxl359", (kernel_ulong_t)&adxl35x_chip_info[ADXL359] },
+	{ .name = "adxl355", .driver_data = (kernel_ulong_t)&adxl35x_chip_info[ADXL355] },
+	{ .name = "adxl359", .driver_data = (kernel_ulong_t)&adxl35x_chip_info[ADXL359] },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adxl355_spi_id);
diff --git a/drivers/iio/accel/adxl367_spi.c b/drivers/iio/accel/adxl367_spi.c
index 785652c7fc92e..1d1418452bdc3 100644
--- a/drivers/iio/accel/adxl367_spi.c
+++ b/drivers/iio/accel/adxl367_spi.c
@@ -137,7 +137,7 @@ static int adxl367_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id adxl367_spi_id[] = {
-	{ "adxl367", 0 },
+	{ .name = "adxl367" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adxl367_spi_id);
diff --git a/drivers/iio/accel/adxl372_spi.c b/drivers/iio/accel/adxl372_spi.c
index 25fdb42543727..128e148c4e96b 100644
--- a/drivers/iio/accel/adxl372_spi.c
+++ b/drivers/iio/accel/adxl372_spi.c
@@ -34,8 +34,8 @@ static int adxl372_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id adxl372_spi_id[] = {
-	{ "adxl371", (kernel_ulong_t)&adxl371_chip_info },
-	{ "adxl372", (kernel_ulong_t)&adxl372_chip_info },
+	{ .name = "adxl371", .driver_data = (kernel_ulong_t)&adxl371_chip_info },
+	{ .name = "adxl372", .driver_data = (kernel_ulong_t)&adxl372_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adxl372_spi_id);
diff --git a/drivers/iio/accel/adxl380_spi.c b/drivers/iio/accel/adxl380_spi.c
index ee3e6338b53d7..ae8467f4e6c01 100644
--- a/drivers/iio/accel/adxl380_spi.c
+++ b/drivers/iio/accel/adxl380_spi.c
@@ -34,10 +34,10 @@ static int adxl380_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id adxl380_spi_id[] = {
-	{ "adxl318", (kernel_ulong_t)&adxl318_chip_info },
-	{ "adxl319", (kernel_ulong_t)&adxl319_chip_info },
-	{ "adxl380", (kernel_ulong_t)&adxl380_chip_info },
-	{ "adxl382", (kernel_ulong_t)&adxl382_chip_info },
+	{ .name = "adxl318", .driver_data = (kernel_ulong_t)&adxl318_chip_info },
+	{ .name = "adxl319", .driver_data = (kernel_ulong_t)&adxl319_chip_info },
+	{ .name = "adxl380", .driver_data = (kernel_ulong_t)&adxl380_chip_info },
+	{ .name = "adxl382", .driver_data = (kernel_ulong_t)&adxl382_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adxl380_spi_id);
diff --git a/drivers/iio/accel/bma220_spi.c b/drivers/iio/accel/bma220_spi.c
index d897aec1c9f21..d6c1087cc3c0f 100644
--- a/drivers/iio/accel/bma220_spi.c
+++ b/drivers/iio/accel/bma220_spi.c
@@ -25,7 +25,7 @@ static int bma220_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id bma220_spi_id[] = {
-	{ "bma220", 0 },
+	{ .name = "bma220" },
 	{ }
 };
 
diff --git a/drivers/iio/accel/bma400_spi.c b/drivers/iio/accel/bma400_spi.c
index 4b89e313704f5..7ac0a7881b593 100644
--- a/drivers/iio/accel/bma400_spi.c
+++ b/drivers/iio/accel/bma400_spi.c
@@ -87,7 +87,7 @@ static int bma400_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id bma400_spi_ids[] = {
-	{ "bma400", 0 },
+	{ .name = "bma400" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, bma400_spi_ids);
diff --git a/drivers/iio/accel/bmc150-accel-core.c b/drivers/iio/accel/bmc150-accel-core.c
index dc8a6285cf3df..46bb36d9366ee 100644
--- a/drivers/iio/accel/bmc150-accel-core.c
+++ b/drivers/iio/accel/bmc150-accel-core.c
@@ -988,8 +988,10 @@ static int __bmc150_accel_fifo_flush(struct iio_dev *indio_dev,
 	do_div(sample_period, count);
 	tstamp = data->timestamp - (count - 1) * sample_period;
 
-	if (samples && count > samples)
-		count = samples;
+	if (samples)
+		count = min3(count, samples, BMC150_ACCEL_FIFO_LENGTH);
+	else
+		count = min(count, BMC150_ACCEL_FIFO_LENGTH);
 
 	count = min_t(u8, count, BMC150_ACCEL_FIFO_LENGTH);
 
diff --git a/drivers/iio/accel/bmc150-accel-spi.c b/drivers/iio/accel/bmc150-accel-spi.c
index 7d2be6f635384..405bd7b2d1997 100644
--- a/drivers/iio/accel/bmc150-accel-spi.c
+++ b/drivers/iio/accel/bmc150-accel-spi.c
@@ -52,15 +52,15 @@ static const struct acpi_device_id bmc150_accel_acpi_match[] = {
 MODULE_DEVICE_TABLE(acpi, bmc150_accel_acpi_match);
 
 static const struct spi_device_id bmc150_accel_id[] = {
-	{"bma222"},
-	{"bma222e"},
-	{"bma250e"},
-	{"bma253"},
-	{"bma255"},
-	{"bma280"},
-	{"bmc150_accel"},
-	{"bmc156_accel", BOSCH_BMC156},
-	{"bmi055_accel"},
+	{ .name = "bma222",       .driver_data = BOSCH_UNKNOWN },
+	{ .name = "bma222e",      .driver_data = BOSCH_UNKNOWN },
+	{ .name = "bma250e",      .driver_data = BOSCH_UNKNOWN },
+	{ .name = "bma253",       .driver_data = BOSCH_UNKNOWN },
+	{ .name = "bma255",       .driver_data = BOSCH_UNKNOWN },
+	{ .name = "bma280",       .driver_data = BOSCH_UNKNOWN },
+	{ .name = "bmc150_accel", .driver_data = BOSCH_UNKNOWN },
+	{ .name = "bmc156_accel", .driver_data = BOSCH_BMC156 },
+	{ .name = "bmi055_accel", .driver_data = BOSCH_UNKNOWN },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, bmc150_accel_id);
diff --git a/drivers/iio/accel/bmi088-accel-spi.c b/drivers/iio/accel/bmi088-accel-spi.c
index 44cb50c76cb1b..ae89e162e0979 100644
--- a/drivers/iio/accel/bmi088-accel-spi.c
+++ b/drivers/iio/accel/bmi088-accel-spi.c
@@ -72,9 +72,9 @@ static const struct of_device_id bmi088_of_match[] = {
 MODULE_DEVICE_TABLE(of, bmi088_of_match);
 
 static const struct spi_device_id bmi088_accel_id[] = {
-	{"bmi085-accel",  BOSCH_BMI085},
-	{"bmi088-accel",  BOSCH_BMI088},
-	{"bmi090l-accel", BOSCH_BMI090L},
+	{ .name = "bmi085-accel", .driver_data = BOSCH_BMI085 },
+	{ .name = "bmi088-accel", .driver_data = BOSCH_BMI088 },
+	{ .name = "bmi090l-accel", .driver_data = BOSCH_BMI090L },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, bmi088_accel_id);
diff --git a/drivers/iio/accel/fxls8962af-core.c b/drivers/iio/accel/fxls8962af-core.c
index d0c2a8daef0db..18d7b09bddd2b 100644
--- a/drivers/iio/accel/fxls8962af-core.c
+++ b/drivers/iio/accel/fxls8962af-core.c
@@ -969,6 +969,8 @@ static int fxls8962af_fifo_flush(struct iio_dev *indio_dev)
 	if (!count)
 		return 0;
 
+	count = min(count, FXLS8962AF_FIFO_LENGTH);
+
 	data->old_timestamp = data->timestamp;
 	data->timestamp = iio_get_time_ns(indio_dev);
 
diff --git a/drivers/iio/accel/fxls8962af-spi.c b/drivers/iio/accel/fxls8962af-spi.c
index 8936032526a0d..baf3c4352f146 100644
--- a/drivers/iio/accel/fxls8962af-spi.c
+++ b/drivers/iio/accel/fxls8962af-spi.c
@@ -34,8 +34,8 @@ static const struct of_device_id fxls8962af_spi_of_match[] = {
 MODULE_DEVICE_TABLE(of, fxls8962af_spi_of_match);
 
 static const struct spi_device_id fxls8962af_spi_id_table[] = {
-	{ "fxls8962af", fxls8962af },
-	{ "fxls8964af", fxls8964af },
+	{ .name = "fxls8962af", .driver_data = fxls8962af },
+	{ .name = "fxls8964af", .driver_data = fxls8964af },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, fxls8962af_spi_id_table);
diff --git a/drivers/iio/accel/hid-sensor-accel-3d.c b/drivers/iio/accel/hid-sensor-accel-3d.c
index 737572bb44d95..eb521125f4eba 100644
--- a/drivers/iio/accel/hid-sensor-accel-3d.c
+++ b/drivers/iio/accel/hid-sensor-accel-3d.c
@@ -121,9 +121,8 @@ static const struct iio_chan_spec gravity_channels[] = {
 
 /* Channel read_raw handler */
 static int accel_3d_read_raw(struct iio_dev *indio_dev,
-			      struct iio_chan_spec const *chan,
-			      int *val, int *val2,
-			      long mask)
+			     struct iio_chan_spec const *chan,
+			     int *val, int *val2, long mask)
 {
 	struct accel_3d_state *accel_state = iio_priv(indio_dev);
 	int report_id = -1;
@@ -150,7 +149,7 @@ static int accel_3d_read_raw(struct iio_dev *indio_dev,
 		else {
 			*val = 0;
 			hid_sensor_power_state(&accel_state->common_attributes,
-						 false);
+					       false);
 			return -EINVAL;
 		}
 		hid_sensor_power_state(&accel_state->common_attributes, false);
@@ -183,10 +182,8 @@ static int accel_3d_read_raw(struct iio_dev *indio_dev,
 
 /* Channel write_raw handler */
 static int accel_3d_write_raw(struct iio_dev *indio_dev,
-			       struct iio_chan_spec const *chan,
-			       int val,
-			       int val2,
-			       long mask)
+			      struct iio_chan_spec const *chan,
+			      int val, int val2, long mask)
 {
 	struct accel_3d_state *accel_state = iio_priv(indio_dev);
 	int ret = 0;
@@ -222,8 +219,7 @@ static void hid_sensor_push_data(struct iio_dev *indio_dev, void *data,
 
 /* Callback handler to send event after all samples are received and captured */
 static int accel_3d_proc_event(struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
-				void *priv)
+			       u32 usage_id, void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
 	struct accel_3d_state *accel_state = iio_priv(indio_dev);
@@ -246,9 +242,9 @@ static int accel_3d_proc_event(struct hid_sensor_hub_device *hsdev,
 
 /* Capture samples in local storage */
 static int accel_3d_capture_sample(struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
-				size_t raw_len, char *raw_data,
-				void *priv)
+				   u32 usage_id,
+				   size_t raw_len, char *raw_data,
+				   void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
 	struct accel_3d_state *accel_state = iio_priv(indio_dev);
@@ -280,10 +276,10 @@ static int accel_3d_capture_sample(struct hid_sensor_hub_device *hsdev,
 
 /* Parse report which is specific to an usage id*/
 static int accel_3d_parse_report(struct platform_device *pdev,
-				struct hid_sensor_hub_device *hsdev,
-				struct iio_chan_spec *channels,
-				unsigned usage_id,
-				struct accel_3d_state *st)
+				 struct hid_sensor_hub_device *hsdev,
+				 struct iio_chan_spec *channels,
+				 u32 usage_id,
+				 struct accel_3d_state *st)
 {
 	int ret;
 
@@ -302,10 +298,10 @@ static int accel_3d_parse_report(struct platform_device *pdev,
 		};
 	}
 	dev_dbg(&pdev->dev, "accel_3d %x:%x, %x:%x, %x:%x\n",
-			st->accel[0].index,
-			st->accel[0].report_id,
-			st->accel[1].index, st->accel[1].report_id,
-			st->accel[2].index, st->accel[2].report_id);
+		st->accel[0].index,
+		st->accel[0].report_id,
+		st->accel[1].index, st->accel[1].report_id,
+		st->accel[2].index, st->accel[2].report_id);
 
 	st->scale_precision = hid_sensor_format_scale(
 				hsdev->usage,
@@ -328,7 +324,7 @@ static int hid_accel_3d_probe(struct platform_device *pdev)
 
 	indio_dev = devm_iio_device_alloc(&pdev->dev,
 					  sizeof(struct accel_3d_state));
-	if (indio_dev == NULL)
+	if (!indio_dev)
 		return -ENOMEM;
 
 	platform_set_drvdata(pdev, indio_dev);
@@ -365,8 +361,8 @@ static int hid_accel_3d_probe(struct platform_device *pdev)
 		return -ENOMEM;
 	}
 	ret = accel_3d_parse_report(pdev, hsdev,
-				(struct iio_chan_spec *)indio_dev->channels,
-				hsdev->usage, accel_state);
+				    (struct iio_chan_spec *)indio_dev->channels,
+				    hsdev->usage, accel_state);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to setup attributes\n");
 		return ret;
@@ -379,32 +375,32 @@ static int hid_accel_3d_probe(struct platform_device *pdev)
 	atomic_set(&accel_state->common_attributes.data_ready, 0);
 
 	ret = hid_sensor_setup_trigger(indio_dev, name,
-					&accel_state->common_attributes);
+				       &accel_state->common_attributes);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "trigger setup failed\n");
 		return ret;
 	}
 
-	ret = iio_device_register(indio_dev);
-	if (ret) {
-		dev_err(&pdev->dev, "device register failed\n");
-		goto error_remove_trigger;
-	}
-
 	accel_state->callbacks.send_event = accel_3d_proc_event;
 	accel_state->callbacks.capture_sample = accel_3d_capture_sample;
 	accel_state->callbacks.pdev = pdev;
 	ret = sensor_hub_register_callback(hsdev, hsdev->usage,
-					&accel_state->callbacks);
+					   &accel_state->callbacks);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "callback reg failed\n");
-		goto error_iio_unreg;
+		goto error_remove_trigger;
+	}
+
+	ret = iio_device_register(indio_dev);
+	if (ret) {
+		dev_err(&pdev->dev, "device register failed\n");
+		goto error_remove_callback;
 	}
 
 	return ret;
 
-error_iio_unreg:
-	iio_device_unregister(indio_dev);
+error_remove_callback:
+	sensor_hub_remove_callback(hsdev, hsdev->usage);
 error_remove_trigger:
 	hid_sensor_remove_trigger(indio_dev, &accel_state->common_attributes);
 	return ret;
@@ -417,8 +413,8 @@ static void hid_accel_3d_remove(struct platform_device *pdev)
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
 	struct accel_3d_state *accel_state = iio_priv(indio_dev);
 
-	sensor_hub_remove_callback(hsdev, hsdev->usage);
 	iio_device_unregister(indio_dev);
+	sensor_hub_remove_callback(hsdev, hsdev->usage);
 	hid_sensor_remove_trigger(indio_dev, &accel_state->common_attributes);
 }
 
diff --git a/drivers/iio/accel/kxsd9-spi.c b/drivers/iio/accel/kxsd9-spi.c
index 31a26baba9f7e..71da958d40514 100644
--- a/drivers/iio/accel/kxsd9-spi.c
+++ b/drivers/iio/accel/kxsd9-spi.c
@@ -36,7 +36,7 @@ static void kxsd9_spi_remove(struct spi_device *spi)
 }
 
 static const struct spi_device_id kxsd9_spi_id[] = {
-	{"kxsd9", 0},
+	{ .name = "kxsd9" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, kxsd9_spi_id);
diff --git a/drivers/iio/accel/mma7455_spi.c b/drivers/iio/accel/mma7455_spi.c
index aca02e83f789d..872c47acf8822 100644
--- a/drivers/iio/accel/mma7455_spi.c
+++ b/drivers/iio/accel/mma7455_spi.c
@@ -28,8 +28,8 @@ static void mma7455_spi_remove(struct spi_device *spi)
 }
 
 static const struct spi_device_id mma7455_spi_ids[] = {
-	{ "mma7455", 0 },
-	{ "mma7456", 0 },
+	{ .name = "mma7455" },
+	{ .name = "mma7456" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, mma7455_spi_ids);
diff --git a/drivers/iio/accel/sca3000.c b/drivers/iio/accel/sca3000.c
index 573831199bbaa..a92a563f6a4bf 100644
--- a/drivers/iio/accel/sca3000.c
+++ b/drivers/iio/accel/sca3000.c
@@ -1504,10 +1504,10 @@ static int sca3000_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id sca3000_id[] = {
-	{ "sca3000_d01", (kernel_ulong_t)&sca3000_chip_info_d01 },
-	{ "sca3000_e02", (kernel_ulong_t)&sca3000_chip_info_e02 },
-	{ "sca3000_e04", (kernel_ulong_t)&sca3000_chip_info_e04 },
-	{ "sca3000_e05", (kernel_ulong_t)&sca3000_chip_info_e05 },
+	{ .name = "sca3000_d01", .driver_data = (kernel_ulong_t)&sca3000_chip_info_d01 },
+	{ .name = "sca3000_e02", .driver_data = (kernel_ulong_t)&sca3000_chip_info_e02 },
+	{ .name = "sca3000_e04", .driver_data = (kernel_ulong_t)&sca3000_chip_info_e04 },
+	{ .name = "sca3000_e05", .driver_data = (kernel_ulong_t)&sca3000_chip_info_e05 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, sca3000_id);
diff --git a/drivers/iio/accel/sca3300.c b/drivers/iio/accel/sca3300.c
index 8380b237831ce..67582cc34149a 100644
--- a/drivers/iio/accel/sca3300.c
+++ b/drivers/iio/accel/sca3300.c
@@ -664,8 +664,8 @@ static const struct of_device_id sca3300_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, sca3300_dt_ids);
 
 static const struct spi_device_id sca3300_ids[] = {
-	{ "sca3300" },
-	{ "scl3300" },
+	{ .name = "sca3300" },
+	{ .name = "scl3300" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, sca3300_ids);
diff --git a/drivers/iio/accel/st_accel_spi.c b/drivers/iio/accel/st_accel_spi.c
index e6bf8d0d4ff52..171f241c0a30f 100644
--- a/drivers/iio/accel/st_accel_spi.c
+++ b/drivers/iio/accel/st_accel_spi.c
@@ -146,26 +146,26 @@ static int st_accel_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id st_accel_id_table[] = {
-	{ LIS3DH_ACCEL_DEV_NAME },
-	{ LSM330D_ACCEL_DEV_NAME },
-	{ LSM330DL_ACCEL_DEV_NAME },
-	{ LSM330DLC_ACCEL_DEV_NAME },
-	{ LIS331DLH_ACCEL_DEV_NAME },
-	{ LSM330_ACCEL_DEV_NAME },
-	{ LSM303AGR_ACCEL_DEV_NAME },
-	{ LIS2DH12_ACCEL_DEV_NAME },
-	{ LIS2DS12_ACCEL_DEV_NAME },
-	{ LIS3L02DQ_ACCEL_DEV_NAME },
-	{ LNG2DM_ACCEL_DEV_NAME },
-	{ H3LIS331DL_ACCEL_DEV_NAME },
-	{ LIS331DL_ACCEL_DEV_NAME },
-	{ LIS3LV02DL_ACCEL_DEV_NAME },
-	{ LIS2DW12_ACCEL_DEV_NAME },
-	{ LIS3DHH_ACCEL_DEV_NAME },
-	{ LIS3DE_ACCEL_DEV_NAME },
-	{ LIS302DL_ACCEL_DEV_NAME },
-	{ LSM303C_ACCEL_DEV_NAME },
-	{ IIS328DQ_ACCEL_DEV_NAME },
+	{ .name = LIS3DH_ACCEL_DEV_NAME },
+	{ .name = LSM330D_ACCEL_DEV_NAME },
+	{ .name = LSM330DL_ACCEL_DEV_NAME },
+	{ .name = LSM330DLC_ACCEL_DEV_NAME },
+	{ .name = LIS331DLH_ACCEL_DEV_NAME },
+	{ .name = LSM330_ACCEL_DEV_NAME },
+	{ .name = LSM303AGR_ACCEL_DEV_NAME },
+	{ .name = LIS2DH12_ACCEL_DEV_NAME },
+	{ .name = LIS2DS12_ACCEL_DEV_NAME },
+	{ .name = LIS3L02DQ_ACCEL_DEV_NAME },
+	{ .name = LNG2DM_ACCEL_DEV_NAME },
+	{ .name = H3LIS331DL_ACCEL_DEV_NAME },
+	{ .name = LIS331DL_ACCEL_DEV_NAME },
+	{ .name = LIS3LV02DL_ACCEL_DEV_NAME },
+	{ .name = LIS2DW12_ACCEL_DEV_NAME },
+	{ .name = LIS3DHH_ACCEL_DEV_NAME },
+	{ .name = LIS3DE_ACCEL_DEV_NAME },
+	{ .name = LIS302DL_ACCEL_DEV_NAME },
+	{ .name = LSM303C_ACCEL_DEV_NAME },
+	{ .name = IIS328DQ_ACCEL_DEV_NAME },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, st_accel_id_table);
diff --git a/drivers/iio/accel/stk8312.c b/drivers/iio/accel/stk8312.c
index 8b5a3e0489e97..ad4d11337f4bd 100644
--- a/drivers/iio/accel/stk8312.c
+++ b/drivers/iio/accel/stk8312.c
@@ -7,12 +7,20 @@
  * IIO driver for STK8312; 7-bit I2C address: 0x3D.
  */
 
+#include <linux/array_size.h>
+#include <linux/bits.h>
+#include <linux/delay.h>
+#include <linux/dev_printk.h>
+#include <linux/errno.h>
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
-#include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/delay.h>
+#include <linux/mod_devicetable.h>
+#include <linux/mutex.h>
+#include <linux/pm.h>
+#include <linux/sysfs.h>
 #include <linux/types.h>
+
 #include <linux/iio/buffer.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
diff --git a/drivers/iio/accel/stk8ba50.c b/drivers/iio/accel/stk8ba50.c
index d0c53b8ac850a..54638cd75ea74 100644
--- a/drivers/iio/accel/stk8ba50.c
+++ b/drivers/iio/accel/stk8ba50.c
@@ -7,11 +7,18 @@
  * STK8BA50 7-bit I2C address: 0x18.
  */
 
+#include <linux/array_size.h>
+#include <linux/bitops.h>
+#include <linux/dev_printk.h>
+#include <linux/errno.h>
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
-#include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/pm.h>
+#include <linux/sysfs.h>
 #include <linux/types.h>
+
 #include <linux/iio/buffer.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig
index 3755a81c1efdb..d1d6ca27cce10 100644
--- a/drivers/iio/adc/Kconfig
+++ b/drivers/iio/adc/Kconfig
@@ -33,7 +33,8 @@ config AB8500_GPADC
 	  and USB voltages integral to the U8500 platform.
 
 config AD_SIGMA_DELTA
-	tristate
+	tristate "Analog Devices Sigma-Delta Modulator support" if COMPILE_TEST
+	depends on SPI
 	select IIO_BUFFER
 	select IIO_BUFFER_DMAENGINE
 	select IIO_TRIGGERED_BUFFER
@@ -1076,6 +1077,7 @@ config MAX1363
 config MAX14001
 	tristate "Analog Devices MAX14001/MAX14002 ADC driver"
 	depends on SPI
+	select REGMAP
 	help
 	  Say yes here to build support for Analog Devices MAX14001/MAX14002
 	  Configurable, Isolated 10-bit ADCs for Multi-Range Binary Inputs.
@@ -1086,6 +1088,7 @@ config MAX14001
 config MAX34408
 	tristate "Maxim max34408/max344089 ADC driver"
 	depends on I2C
+	select REGMAP_I2C
 	help
 	  Say yes here to build support for Maxim max34408/max34409 current sense
 	  monitor with 8-bits ADC interface with overcurrent delay/threshold and
@@ -1165,6 +1168,17 @@ config MCP3911
 	  This driver can also be built as a module. If so, the module will be
 	  called mcp3911.
 
+config MEDIATEK_MT6323_AUXADC
+	tristate "MediaTek MT6323 PMIC AUXADC driver"
+	depends on MFD_MT6397
+	help
+	  Say yes here to enable support for MediaTek MT6323 PMIC Auxiliary ADC.
+	  This driver provides multiple channels for system monitoring,
+	  such as battery voltage, PMIC temperature, and others.
+
+	  This driver can also be built as a module. If so, the module will be
+	  called mt6323-auxadc.
+
 config MEDIATEK_MT6359_AUXADC
 	tristate "MediaTek MT6359 PMIC AUXADC driver"
 	depends on MFD_MT6397
@@ -1964,6 +1978,39 @@ config TWL6030_GPADC
 	  This driver can also be built as a module. If so, the module will be
 	  called twl6030-gpadc.
 
+config VERSAL_SYSMON_CORE
+	tristate
+	select REGMAP
+
+config VERSAL_SYSMON
+	tristate "AMD Versal SysMon driver"
+	depends on ARCH_ZYNQMP || COMPILE_TEST
+	depends on HAS_IOMEM
+	select VERSAL_SYSMON_CORE
+	help
+	  Say yes here to have support for the AMD/Xilinx Versal System
+	  Monitor (SysMon). This driver provides voltage and temperature
+	  monitoring through the IIO subsystem.
+
+	  The SysMon measures up to 160 supply voltages and reads up to
+	  64 temperature satellites distributed across the SoC.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called versal-sysmon.
+
+config VERSAL_SYSMON_I2C
+	tristate "AMD Versal SysMon I2C driver"
+	depends on I2C
+	select VERSAL_SYSMON_CORE
+	help
+	  Say yes here to have support for the AMD/Xilinx Versal System
+	  Monitor (SysMon) via I2C interface. This driver enables voltage
+	  and temperature monitoring when the Versal chip has SysMon
+	  configured with I2C access.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called versal-sysmon-i2c.
+
 config VF610_ADC
 	tristate "Freescale vf610 ADC driver"
 	depends on HAS_IOMEM
diff --git a/drivers/iio/adc/Makefile b/drivers/iio/adc/Makefile
index 707dd708912fe..0f90b75577ff8 100644
--- a/drivers/iio/adc/Makefile
+++ b/drivers/iio/adc/Makefile
@@ -100,6 +100,7 @@ obj-$(CONFIG_MCP320X) += mcp320x.o
 obj-$(CONFIG_MCP3422) += mcp3422.o
 obj-$(CONFIG_MCP3564) += mcp3564.o
 obj-$(CONFIG_MCP3911) += mcp3911.o
+obj-$(CONFIG_MEDIATEK_MT6323_AUXADC) += mt6323-auxadc.o
 obj-$(CONFIG_MEDIATEK_MT6359_AUXADC) += mt6359-auxadc.o
 obj-$(CONFIG_MEDIATEK_MT6360_ADC) += mt6360-adc.o
 obj-$(CONFIG_MEDIATEK_MT6370_ADC) += mt6370-adc.o
@@ -168,6 +169,9 @@ obj-$(CONFIG_TI_TLC4541) += ti-tlc4541.o
 obj-$(CONFIG_TI_TSC2046) += ti-tsc2046.o
 obj-$(CONFIG_TWL4030_MADC) += twl4030-madc.o
 obj-$(CONFIG_TWL6030_GPADC) += twl6030-gpadc.o
+obj-$(CONFIG_VERSAL_SYSMON_CORE) += versal-sysmon-core.o
+obj-$(CONFIG_VERSAL_SYSMON) += versal-sysmon.o
+obj-$(CONFIG_VERSAL_SYSMON_I2C) += versal-sysmon-i2c.o
 obj-$(CONFIG_VF610_ADC) += vf610_adc.o
 obj-$(CONFIG_VIPERBOARD_ADC) += viperboard_adc.o
 obj-$(CONFIG_XILINX_AMS) += xilinx-ams.o
diff --git a/drivers/iio/adc/ad4000.c b/drivers/iio/adc/ad4000.c
index c2f6f5f9812c7..a33896b2cf974 100644
--- a/drivers/iio/adc/ad4000.c
+++ b/drivers/iio/adc/ad4000.c
@@ -1176,37 +1176,37 @@ static int ad4000_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad4000_id[] = {
-	{ "ad4000", (kernel_ulong_t)&ad4000_chip_info },
-	{ "ad4001", (kernel_ulong_t)&ad4001_chip_info },
-	{ "ad4002", (kernel_ulong_t)&ad4002_chip_info },
-	{ "ad4003", (kernel_ulong_t)&ad4003_chip_info },
-	{ "ad4004", (kernel_ulong_t)&ad4004_chip_info },
-	{ "ad4005", (kernel_ulong_t)&ad4005_chip_info },
-	{ "ad4006", (kernel_ulong_t)&ad4006_chip_info },
-	{ "ad4007", (kernel_ulong_t)&ad4007_chip_info },
-	{ "ad4008", (kernel_ulong_t)&ad4008_chip_info },
-	{ "ad4010", (kernel_ulong_t)&ad4010_chip_info },
-	{ "ad4011", (kernel_ulong_t)&ad4011_chip_info },
-	{ "ad4020", (kernel_ulong_t)&ad4020_chip_info },
-	{ "ad4021", (kernel_ulong_t)&ad4021_chip_info },
-	{ "ad4022", (kernel_ulong_t)&ad4022_chip_info },
-	{ "adaq4001", (kernel_ulong_t)&adaq4001_chip_info },
-	{ "adaq4003", (kernel_ulong_t)&adaq4003_chip_info },
-	{ "ad7685", (kernel_ulong_t)&ad7685_chip_info },
-	{ "ad7686", (kernel_ulong_t)&ad7686_chip_info },
-	{ "ad7687", (kernel_ulong_t)&ad7687_chip_info },
-	{ "ad7688", (kernel_ulong_t)&ad7688_chip_info },
-	{ "ad7690", (kernel_ulong_t)&ad7690_chip_info },
-	{ "ad7691", (kernel_ulong_t)&ad7691_chip_info },
-	{ "ad7693", (kernel_ulong_t)&ad7693_chip_info },
-	{ "ad7942", (kernel_ulong_t)&ad7942_chip_info },
-	{ "ad7946", (kernel_ulong_t)&ad7946_chip_info },
-	{ "ad7980", (kernel_ulong_t)&ad7980_chip_info },
-	{ "ad7982", (kernel_ulong_t)&ad7982_chip_info },
-	{ "ad7983", (kernel_ulong_t)&ad7983_chip_info },
-	{ "ad7984", (kernel_ulong_t)&ad7984_chip_info },
-	{ "ad7988-1", (kernel_ulong_t)&ad7988_1_chip_info },
-	{ "ad7988-5", (kernel_ulong_t)&ad7988_5_chip_info },
+	{ .name = "ad4000", .driver_data = (kernel_ulong_t)&ad4000_chip_info },
+	{ .name = "ad4001", .driver_data = (kernel_ulong_t)&ad4001_chip_info },
+	{ .name = "ad4002", .driver_data = (kernel_ulong_t)&ad4002_chip_info },
+	{ .name = "ad4003", .driver_data = (kernel_ulong_t)&ad4003_chip_info },
+	{ .name = "ad4004", .driver_data = (kernel_ulong_t)&ad4004_chip_info },
+	{ .name = "ad4005", .driver_data = (kernel_ulong_t)&ad4005_chip_info },
+	{ .name = "ad4006", .driver_data = (kernel_ulong_t)&ad4006_chip_info },
+	{ .name = "ad4007", .driver_data = (kernel_ulong_t)&ad4007_chip_info },
+	{ .name = "ad4008", .driver_data = (kernel_ulong_t)&ad4008_chip_info },
+	{ .name = "ad4010", .driver_data = (kernel_ulong_t)&ad4010_chip_info },
+	{ .name = "ad4011", .driver_data = (kernel_ulong_t)&ad4011_chip_info },
+	{ .name = "ad4020", .driver_data = (kernel_ulong_t)&ad4020_chip_info },
+	{ .name = "ad4021", .driver_data = (kernel_ulong_t)&ad4021_chip_info },
+	{ .name = "ad4022", .driver_data = (kernel_ulong_t)&ad4022_chip_info },
+	{ .name = "adaq4001", .driver_data = (kernel_ulong_t)&adaq4001_chip_info },
+	{ .name = "adaq4003", .driver_data = (kernel_ulong_t)&adaq4003_chip_info },
+	{ .name = "ad7685", .driver_data = (kernel_ulong_t)&ad7685_chip_info },
+	{ .name = "ad7686", .driver_data = (kernel_ulong_t)&ad7686_chip_info },
+	{ .name = "ad7687", .driver_data = (kernel_ulong_t)&ad7687_chip_info },
+	{ .name = "ad7688", .driver_data = (kernel_ulong_t)&ad7688_chip_info },
+	{ .name = "ad7690", .driver_data = (kernel_ulong_t)&ad7690_chip_info },
+	{ .name = "ad7691", .driver_data = (kernel_ulong_t)&ad7691_chip_info },
+	{ .name = "ad7693", .driver_data = (kernel_ulong_t)&ad7693_chip_info },
+	{ .name = "ad7942", .driver_data = (kernel_ulong_t)&ad7942_chip_info },
+	{ .name = "ad7946", .driver_data = (kernel_ulong_t)&ad7946_chip_info },
+	{ .name = "ad7980", .driver_data = (kernel_ulong_t)&ad7980_chip_info },
+	{ .name = "ad7982", .driver_data = (kernel_ulong_t)&ad7982_chip_info },
+	{ .name = "ad7983", .driver_data = (kernel_ulong_t)&ad7983_chip_info },
+	{ .name = "ad7984", .driver_data = (kernel_ulong_t)&ad7984_chip_info },
+	{ .name = "ad7988-1", .driver_data = (kernel_ulong_t)&ad7988_1_chip_info },
+	{ .name = "ad7988-5", .driver_data = (kernel_ulong_t)&ad7988_5_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad4000_id);
diff --git a/drivers/iio/adc/ad4030.c b/drivers/iio/adc/ad4030.c
index 9c5f19321e3b8..e97400a1aa481 100644
--- a/drivers/iio/adc/ad4030.c
+++ b/drivers/iio/adc/ad4030.c
@@ -1763,13 +1763,13 @@ static const struct ad4030_chip_info adaq4224_chip_info = {
 };
 
 static const struct spi_device_id ad4030_id_table[] = {
-	{ "ad4030-24", (kernel_ulong_t)&ad4030_24_chip_info },
-	{ "ad4630-16", (kernel_ulong_t)&ad4630_16_chip_info },
-	{ "ad4630-24", (kernel_ulong_t)&ad4630_24_chip_info },
-	{ "ad4632-16", (kernel_ulong_t)&ad4632_16_chip_info },
-	{ "ad4632-24", (kernel_ulong_t)&ad4632_24_chip_info },
-	{ "adaq4216", (kernel_ulong_t)&adaq4216_chip_info },
-	{ "adaq4224", (kernel_ulong_t)&adaq4224_chip_info },
+	{ .name = "ad4030-24", .driver_data = (kernel_ulong_t)&ad4030_24_chip_info },
+	{ .name = "ad4630-16", .driver_data = (kernel_ulong_t)&ad4630_16_chip_info },
+	{ .name = "ad4630-24", .driver_data = (kernel_ulong_t)&ad4630_24_chip_info },
+	{ .name = "ad4632-16", .driver_data = (kernel_ulong_t)&ad4632_16_chip_info },
+	{ .name = "ad4632-24", .driver_data = (kernel_ulong_t)&ad4632_24_chip_info },
+	{ .name = "adaq4216", .driver_data = (kernel_ulong_t)&adaq4216_chip_info },
+	{ .name = "adaq4224", .driver_data = (kernel_ulong_t)&adaq4224_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad4030_id_table);
diff --git a/drivers/iio/adc/ad4080.c b/drivers/iio/adc/ad4080.c
index 0797d64aeaeca..c7ecf6971be45 100644
--- a/drivers/iio/adc/ad4080.c
+++ b/drivers/iio/adc/ad4080.c
@@ -848,17 +848,17 @@ static int ad4080_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad4080_id[] = {
-	{ "ad4080", (kernel_ulong_t)&ad4080_chip_info },
-	{ "ad4081", (kernel_ulong_t)&ad4081_chip_info },
-	{ "ad4082", (kernel_ulong_t)&ad4082_chip_info },
-	{ "ad4083", (kernel_ulong_t)&ad4083_chip_info },
-	{ "ad4084", (kernel_ulong_t)&ad4084_chip_info },
-	{ "ad4085", (kernel_ulong_t)&ad4085_chip_info },
-	{ "ad4086", (kernel_ulong_t)&ad4086_chip_info },
-	{ "ad4087", (kernel_ulong_t)&ad4087_chip_info },
-	{ "ad4088", (kernel_ulong_t)&ad4088_chip_info },
-	{ "ad4880", (kernel_ulong_t)&ad4880_chip_info },
-	{ "ad4884", (kernel_ulong_t)&ad4884_chip_info },
+	{ .name = "ad4080", .driver_data = (kernel_ulong_t)&ad4080_chip_info },
+	{ .name = "ad4081", .driver_data = (kernel_ulong_t)&ad4081_chip_info },
+	{ .name = "ad4082", .driver_data = (kernel_ulong_t)&ad4082_chip_info },
+	{ .name = "ad4083", .driver_data = (kernel_ulong_t)&ad4083_chip_info },
+	{ .name = "ad4084", .driver_data = (kernel_ulong_t)&ad4084_chip_info },
+	{ .name = "ad4085", .driver_data = (kernel_ulong_t)&ad4085_chip_info },
+	{ .name = "ad4086", .driver_data = (kernel_ulong_t)&ad4086_chip_info },
+	{ .name = "ad4087", .driver_data = (kernel_ulong_t)&ad4087_chip_info },
+	{ .name = "ad4088", .driver_data = (kernel_ulong_t)&ad4088_chip_info },
+	{ .name = "ad4880", .driver_data = (kernel_ulong_t)&ad4880_chip_info },
+	{ .name = "ad4884", .driver_data = (kernel_ulong_t)&ad4884_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad4080_id);
diff --git a/drivers/iio/adc/ad4130.c b/drivers/iio/adc/ad4130.c
index 7f39f3484062a..3ffc4b765e0e6 100644
--- a/drivers/iio/adc/ad4130.c
+++ b/drivers/iio/adc/ad4130.c
@@ -2425,12 +2425,12 @@ static const struct of_device_id ad4130_of_match[] = {
 MODULE_DEVICE_TABLE(of, ad4130_of_match);
 
 static const struct spi_device_id ad4130_id_table[] = {
-	{ "ad4129-4", (kernel_ulong_t)&ad4129_4_chip_info },
-	{ "ad4129-8", (kernel_ulong_t)&ad4129_8_chip_info },
-	{ "ad4130-4", (kernel_ulong_t)&ad4130_4_chip_info },
-	{ "ad4130", (kernel_ulong_t)&ad4130_8_chip_info },
-	{ "ad4131-4", (kernel_ulong_t)&ad4131_4_chip_info },
-	{ "ad4131-8", (kernel_ulong_t)&ad4131_8_chip_info },
+	{ .name = "ad4129-4", .driver_data = (kernel_ulong_t)&ad4129_4_chip_info },
+	{ .name = "ad4129-8", .driver_data = (kernel_ulong_t)&ad4129_8_chip_info },
+	{ .name = "ad4130-4", .driver_data = (kernel_ulong_t)&ad4130_4_chip_info },
+	{ .name = "ad4130", .driver_data = (kernel_ulong_t)&ad4130_8_chip_info },
+	{ .name = "ad4131-4", .driver_data = (kernel_ulong_t)&ad4131_4_chip_info },
+	{ .name = "ad4131-8", .driver_data = (kernel_ulong_t)&ad4131_8_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad4130_id_table);
diff --git a/drivers/iio/adc/ad4134.c b/drivers/iio/adc/ad4134.c
index cb9d74316f6a9..0490218bb0e9b 100644
--- a/drivers/iio/adc/ad4134.c
+++ b/drivers/iio/adc/ad4134.c
@@ -472,7 +472,7 @@ static int ad4134_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad4134_id[] = {
-	{ "ad4134" },
+	{ .name = "ad4134" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad4134_id);
diff --git a/drivers/iio/adc/ad4170-4.c b/drivers/iio/adc/ad4170-4.c
index 627cbf5a37b0b..f7757b3a931c3 100644
--- a/drivers/iio/adc/ad4170-4.c
+++ b/drivers/iio/adc/ad4170-4.c
@@ -2979,9 +2979,9 @@ static int ad4170_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad4170_id_table[] = {
-	{ "ad4170-4", (kernel_ulong_t)&ad4170_chip_info },
-	{ "ad4190-4", (kernel_ulong_t)&ad4190_chip_info },
-	{ "ad4195-4", (kernel_ulong_t)&ad4195_chip_info },
+	{ .name = "ad4170-4", .driver_data = (kernel_ulong_t)&ad4170_chip_info },
+	{ .name = "ad4190-4", .driver_data = (kernel_ulong_t)&ad4190_chip_info },
+	{ .name = "ad4195-4", .driver_data = (kernel_ulong_t)&ad4195_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad4170_id_table);
diff --git a/drivers/iio/adc/ad4851.c b/drivers/iio/adc/ad4851.c
index 940b042d743ac..480d1798f17b6 100644
--- a/drivers/iio/adc/ad4851.c
+++ b/drivers/iio/adc/ad4851.c
@@ -1285,15 +1285,15 @@ static const struct of_device_id ad4851_of_match[] = {
 };
 
 static const struct spi_device_id ad4851_spi_id[] = {
-	{ "ad4851", (kernel_ulong_t)&ad4851_info },
-	{ "ad4852", (kernel_ulong_t)&ad4852_info },
-	{ "ad4853", (kernel_ulong_t)&ad4853_info },
-	{ "ad4854", (kernel_ulong_t)&ad4854_info },
-	{ "ad4855", (kernel_ulong_t)&ad4855_info },
-	{ "ad4856", (kernel_ulong_t)&ad4856_info },
-	{ "ad4857", (kernel_ulong_t)&ad4857_info },
-	{ "ad4858", (kernel_ulong_t)&ad4858_info },
-	{ "ad4858i", (kernel_ulong_t)&ad4858i_info },
+	{ .name = "ad4851", .driver_data = (kernel_ulong_t)&ad4851_info },
+	{ .name = "ad4852", .driver_data = (kernel_ulong_t)&ad4852_info },
+	{ .name = "ad4853", .driver_data = (kernel_ulong_t)&ad4853_info },
+	{ .name = "ad4854", .driver_data = (kernel_ulong_t)&ad4854_info },
+	{ .name = "ad4855", .driver_data = (kernel_ulong_t)&ad4855_info },
+	{ .name = "ad4856", .driver_data = (kernel_ulong_t)&ad4856_info },
+	{ .name = "ad4857", .driver_data = (kernel_ulong_t)&ad4857_info },
+	{ .name = "ad4858", .driver_data = (kernel_ulong_t)&ad4858_info },
+	{ .name = "ad4858i", .driver_data = (kernel_ulong_t)&ad4858i_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad4851_spi_id);
diff --git a/drivers/iio/adc/ad7091r8.c b/drivers/iio/adc/ad7091r8.c
index e93b8bb60e8ea..465a8ba01a225 100644
--- a/drivers/iio/adc/ad7091r8.c
+++ b/drivers/iio/adc/ad7091r8.c
@@ -249,9 +249,9 @@ static const struct of_device_id ad7091r8_of_match[] = {
 MODULE_DEVICE_TABLE(of, ad7091r8_of_match);
 
 static const struct spi_device_id ad7091r8_spi_id[] = {
-	{ "ad7091r2", (kernel_ulong_t)&ad7091r2_init_info },
-	{ "ad7091r4", (kernel_ulong_t)&ad7091r4_init_info },
-	{ "ad7091r8", (kernel_ulong_t)&ad7091r8_init_info },
+	{ .name = "ad7091r2", .driver_data = (kernel_ulong_t)&ad7091r2_init_info },
+	{ .name = "ad7091r4", .driver_data = (kernel_ulong_t)&ad7091r4_init_info },
+	{ .name = "ad7091r8", .driver_data = (kernel_ulong_t)&ad7091r8_init_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7091r8_spi_id);
diff --git a/drivers/iio/adc/ad7124.c b/drivers/iio/adc/ad7124.c
index 19058d0814183..fc5dbc624856d 100644
--- a/drivers/iio/adc/ad7124.c
+++ b/drivers/iio/adc/ad7124.c
@@ -1683,8 +1683,8 @@ static const struct of_device_id ad7124_of_match[] = {
 MODULE_DEVICE_TABLE(of, ad7124_of_match);
 
 static const struct spi_device_id ad71124_ids[] = {
-	{ "ad7124-4", (kernel_ulong_t)&ad7124_4_chip_info },
-	{ "ad7124-8", (kernel_ulong_t)&ad7124_8_chip_info },
+	{ .name = "ad7124-4", .driver_data = (kernel_ulong_t)&ad7124_4_chip_info },
+	{ .name = "ad7124-8", .driver_data = (kernel_ulong_t)&ad7124_8_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad71124_ids);
diff --git a/drivers/iio/adc/ad7173.c b/drivers/iio/adc/ad7173.c
index 9ee65d63c5252..c940f51ad7446 100644
--- a/drivers/iio/adc/ad7173.c
+++ b/drivers/iio/adc/ad7173.c
@@ -2065,19 +2065,19 @@ static const struct of_device_id ad7173_of_match[] = {
 MODULE_DEVICE_TABLE(of, ad7173_of_match);
 
 static const struct spi_device_id ad7173_id_table[] = {
-	{ "ad4111",   (kernel_ulong_t)&ad4111_device_info },
-	{ "ad4112",   (kernel_ulong_t)&ad4112_device_info },
-	{ "ad4113",   (kernel_ulong_t)&ad4113_device_info },
-	{ "ad4114",   (kernel_ulong_t)&ad4114_device_info },
-	{ "ad4115",   (kernel_ulong_t)&ad4115_device_info },
-	{ "ad4116",   (kernel_ulong_t)&ad4116_device_info },
-	{ "ad7172-2", (kernel_ulong_t)&ad7172_2_device_info },
-	{ "ad7172-4", (kernel_ulong_t)&ad7172_4_device_info },
-	{ "ad7173-8", (kernel_ulong_t)&ad7173_8_device_info },
-	{ "ad7175-2", (kernel_ulong_t)&ad7175_2_device_info },
-	{ "ad7175-8", (kernel_ulong_t)&ad7175_8_device_info },
-	{ "ad7176-2", (kernel_ulong_t)&ad7176_2_device_info },
-	{ "ad7177-2", (kernel_ulong_t)&ad7177_2_device_info },
+	{ .name = "ad4111",   .driver_data = (kernel_ulong_t)&ad4111_device_info },
+	{ .name = "ad4112",   .driver_data = (kernel_ulong_t)&ad4112_device_info },
+	{ .name = "ad4113",   .driver_data = (kernel_ulong_t)&ad4113_device_info },
+	{ .name = "ad4114",   .driver_data = (kernel_ulong_t)&ad4114_device_info },
+	{ .name = "ad4115",   .driver_data = (kernel_ulong_t)&ad4115_device_info },
+	{ .name = "ad4116",   .driver_data = (kernel_ulong_t)&ad4116_device_info },
+	{ .name = "ad7172-2", .driver_data = (kernel_ulong_t)&ad7172_2_device_info },
+	{ .name = "ad7172-4", .driver_data = (kernel_ulong_t)&ad7172_4_device_info },
+	{ .name = "ad7173-8", .driver_data = (kernel_ulong_t)&ad7173_8_device_info },
+	{ .name = "ad7175-2", .driver_data = (kernel_ulong_t)&ad7175_2_device_info },
+	{ .name = "ad7175-8", .driver_data = (kernel_ulong_t)&ad7175_8_device_info },
+	{ .name = "ad7176-2", .driver_data = (kernel_ulong_t)&ad7176_2_device_info },
+	{ .name = "ad7177-2", .driver_data = (kernel_ulong_t)&ad7177_2_device_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7173_id_table);
diff --git a/drivers/iio/adc/ad7191.c b/drivers/iio/adc/ad7191.c
index 94b172dce866d..071d26191b3c6 100644
--- a/drivers/iio/adc/ad7191.c
+++ b/drivers/iio/adc/ad7191.c
@@ -535,7 +535,7 @@ static const struct of_device_id ad7191_of_match[] = {
 MODULE_DEVICE_TABLE(of, ad7191_of_match);
 
 static const struct spi_device_id ad7191_id_table[] = {
-	{ "ad7191" },
+	{ .name = "ad7191" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7191_id_table);
diff --git a/drivers/iio/adc/ad7192.c b/drivers/iio/adc/ad7192.c
index 2cfad0bda7523..d8f5886fcf694 100644
--- a/drivers/iio/adc/ad7192.c
+++ b/drivers/iio/adc/ad7192.c
@@ -1448,11 +1448,11 @@ static const struct of_device_id ad7192_of_match[] = {
 MODULE_DEVICE_TABLE(of, ad7192_of_match);
 
 static const struct spi_device_id ad7192_ids[] = {
-	{ "ad7190", (kernel_ulong_t)&ad7192_chip_info_tbl[ID_AD7190] },
-	{ "ad7192", (kernel_ulong_t)&ad7192_chip_info_tbl[ID_AD7192] },
-	{ "ad7193", (kernel_ulong_t)&ad7192_chip_info_tbl[ID_AD7193] },
-	{ "ad7194", (kernel_ulong_t)&ad7192_chip_info_tbl[ID_AD7194] },
-	{ "ad7195", (kernel_ulong_t)&ad7192_chip_info_tbl[ID_AD7195] },
+	{ .name = "ad7190", .driver_data = (kernel_ulong_t)&ad7192_chip_info_tbl[ID_AD7190] },
+	{ .name = "ad7192", .driver_data = (kernel_ulong_t)&ad7192_chip_info_tbl[ID_AD7192] },
+	{ .name = "ad7193", .driver_data = (kernel_ulong_t)&ad7192_chip_info_tbl[ID_AD7193] },
+	{ .name = "ad7194", .driver_data = (kernel_ulong_t)&ad7192_chip_info_tbl[ID_AD7194] },
+	{ .name = "ad7195", .driver_data = (kernel_ulong_t)&ad7192_chip_info_tbl[ID_AD7195] },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7192_ids);
diff --git a/drivers/iio/adc/ad7266.c b/drivers/iio/adc/ad7266.c
index 0ef36c249ab80..594a070c3745a 100644
--- a/drivers/iio/adc/ad7266.c
+++ b/drivers/iio/adc/ad7266.c
@@ -453,8 +453,8 @@ static int ad7266_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad7266_id[] = {
-	{ "ad7265", 0 },
-	{ "ad7266", 0 },
+	{ .name = "ad7265" },
+	{ .name = "ad7266" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7266_id);
diff --git a/drivers/iio/adc/ad7280a.c b/drivers/iio/adc/ad7280a.c
index 2972e706de922..505fa68ee539c 100644
--- a/drivers/iio/adc/ad7280a.c
+++ b/drivers/iio/adc/ad7280a.c
@@ -1082,7 +1082,7 @@ static int ad7280_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad7280_id[] = {
-	{ "ad7280a", 0 },
+	{ .name = "ad7280a" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7280_id);
diff --git a/drivers/iio/adc/ad7292.c b/drivers/iio/adc/ad7292.c
index 0ba0fbad4f703..1d7bf46b80e24 100644
--- a/drivers/iio/adc/ad7292.c
+++ b/drivers/iio/adc/ad7292.c
@@ -299,7 +299,7 @@ static int ad7292_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad7292_id_table[] = {
-	{ "ad7292", 0 },
+	{ .name = "ad7292" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7292_id_table);
diff --git a/drivers/iio/adc/ad7298.c b/drivers/iio/adc/ad7298.c
index 5fa76aec360e4..c669022b5dec4 100644
--- a/drivers/iio/adc/ad7298.c
+++ b/drivers/iio/adc/ad7298.c
@@ -353,7 +353,7 @@ static const struct acpi_device_id ad7298_acpi_ids[] = {
 MODULE_DEVICE_TABLE(acpi, ad7298_acpi_ids);
 
 static const struct spi_device_id ad7298_id[] = {
-	{ "ad7298", 0 },
+	{ .name = "ad7298" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7298_id);
diff --git a/drivers/iio/adc/ad7380.c b/drivers/iio/adc/ad7380.c
index 9f77990a03f93..2e90f125a7974 100644
--- a/drivers/iio/adc/ad7380.c
+++ b/drivers/iio/adc/ad7380.c
@@ -2116,24 +2116,24 @@ static const struct of_device_id ad7380_of_match_table[] = {
 };
 
 static const struct spi_device_id ad7380_id_table[] = {
-	{ "ad7380", (kernel_ulong_t)&ad7380_chip_info },
-	{ "ad7381", (kernel_ulong_t)&ad7381_chip_info },
-	{ "ad7383", (kernel_ulong_t)&ad7383_chip_info },
-	{ "ad7384", (kernel_ulong_t)&ad7384_chip_info },
-	{ "ad7386", (kernel_ulong_t)&ad7386_chip_info },
-	{ "ad7387", (kernel_ulong_t)&ad7387_chip_info },
-	{ "ad7388", (kernel_ulong_t)&ad7388_chip_info },
-	{ "ad7380-4", (kernel_ulong_t)&ad7380_4_chip_info },
-	{ "ad7381-4", (kernel_ulong_t)&ad7381_4_chip_info },
-	{ "ad7383-4", (kernel_ulong_t)&ad7383_4_chip_info },
-	{ "ad7384-4", (kernel_ulong_t)&ad7384_4_chip_info },
-	{ "ad7386-4", (kernel_ulong_t)&ad7386_4_chip_info },
-	{ "ad7387-4", (kernel_ulong_t)&ad7387_4_chip_info },
-	{ "ad7388-4", (kernel_ulong_t)&ad7388_4_chip_info },
-	{ "ad7389-4", (kernel_ulong_t)&ad7389_4_chip_info },
-	{ "adaq4370-4", (kernel_ulong_t)&adaq4370_4_chip_info },
-	{ "adaq4380-4", (kernel_ulong_t)&adaq4380_4_chip_info },
-	{ "adaq4381-4", (kernel_ulong_t)&adaq4381_4_chip_info },
+	{ .name = "ad7380", .driver_data = (kernel_ulong_t)&ad7380_chip_info },
+	{ .name = "ad7381", .driver_data = (kernel_ulong_t)&ad7381_chip_info },
+	{ .name = "ad7383", .driver_data = (kernel_ulong_t)&ad7383_chip_info },
+	{ .name = "ad7384", .driver_data = (kernel_ulong_t)&ad7384_chip_info },
+	{ .name = "ad7386", .driver_data = (kernel_ulong_t)&ad7386_chip_info },
+	{ .name = "ad7387", .driver_data = (kernel_ulong_t)&ad7387_chip_info },
+	{ .name = "ad7388", .driver_data = (kernel_ulong_t)&ad7388_chip_info },
+	{ .name = "ad7380-4", .driver_data = (kernel_ulong_t)&ad7380_4_chip_info },
+	{ .name = "ad7381-4", .driver_data = (kernel_ulong_t)&ad7381_4_chip_info },
+	{ .name = "ad7383-4", .driver_data = (kernel_ulong_t)&ad7383_4_chip_info },
+	{ .name = "ad7384-4", .driver_data = (kernel_ulong_t)&ad7384_4_chip_info },
+	{ .name = "ad7386-4", .driver_data = (kernel_ulong_t)&ad7386_4_chip_info },
+	{ .name = "ad7387-4", .driver_data = (kernel_ulong_t)&ad7387_4_chip_info },
+	{ .name = "ad7388-4", .driver_data = (kernel_ulong_t)&ad7388_4_chip_info },
+	{ .name = "ad7389-4", .driver_data = (kernel_ulong_t)&ad7389_4_chip_info },
+	{ .name = "adaq4370-4", .driver_data = (kernel_ulong_t)&adaq4370_4_chip_info },
+	{ .name = "adaq4380-4", .driver_data = (kernel_ulong_t)&adaq4380_4_chip_info },
+	{ .name = "adaq4381-4", .driver_data = (kernel_ulong_t)&adaq4381_4_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7380_id_table);
diff --git a/drivers/iio/adc/ad7476.c b/drivers/iio/adc/ad7476.c
index 21d3f6aae972a..c007f453ae18a 100644
--- a/drivers/iio/adc/ad7476.c
+++ b/drivers/iio/adc/ad7476.c
@@ -411,42 +411,42 @@ static int ad7476_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad7476_id[] = {
-	{ "ad7091", (kernel_ulong_t)&ad7091_chip_info },
-	{ "ad7091r", (kernel_ulong_t)&ad7091r_chip_info },
-	{ "ad7273", (kernel_ulong_t)&ad7273_chip_info },
-	{ "ad7274", (kernel_ulong_t)&ad7274_chip_info },
-	{ "ad7276", (kernel_ulong_t)&ad7276_chip_info },
-	{ "ad7277", (kernel_ulong_t)&ad7277_chip_info },
-	{ "ad7278", (kernel_ulong_t)&ad7278_chip_info },
-	{ "ad7466", (kernel_ulong_t)&ad7466_chip_info },
-	{ "ad7467", (kernel_ulong_t)&ad7467_chip_info },
-	{ "ad7468", (kernel_ulong_t)&ad7468_chip_info },
-	{ "ad7475", (kernel_ulong_t)&ad7475_chip_info },
-	{ "ad7476", (kernel_ulong_t)&ad7466_chip_info },
-	{ "ad7476a", (kernel_ulong_t)&ad7466_chip_info },
-	{ "ad7477", (kernel_ulong_t)&ad7467_chip_info },
-	{ "ad7477a", (kernel_ulong_t)&ad7467_chip_info },
-	{ "ad7478", (kernel_ulong_t)&ad7468_chip_info },
-	{ "ad7478a", (kernel_ulong_t)&ad7468_chip_info },
-	{ "ad7495", (kernel_ulong_t)&ad7495_chip_info },
-	{ "ad7910", (kernel_ulong_t)&ad7467_chip_info },
-	{ "ad7920", (kernel_ulong_t)&ad7466_chip_info },
-	{ "ad7940", (kernel_ulong_t)&ad7940_chip_info },
-	{ "adc081s", (kernel_ulong_t)&adc081s_chip_info },
-	{ "adc101s", (kernel_ulong_t)&adc101s_chip_info },
-	{ "adc121s", (kernel_ulong_t)&adc121s_chip_info },
-	{ "ads7866", (kernel_ulong_t)&ads7866_chip_info },
-	{ "ads7867", (kernel_ulong_t)&ads7867_chip_info },
-	{ "ads7868", (kernel_ulong_t)&ads7868_chip_info },
-	{ "bd79105", (kernel_ulong_t)&bd79105_chip_info },
+	{ .name = "ad7091", .driver_data = (kernel_ulong_t)&ad7091_chip_info },
+	{ .name = "ad7091r", .driver_data = (kernel_ulong_t)&ad7091r_chip_info },
+	{ .name = "ad7273", .driver_data = (kernel_ulong_t)&ad7273_chip_info },
+	{ .name = "ad7274", .driver_data = (kernel_ulong_t)&ad7274_chip_info },
+	{ .name = "ad7276", .driver_data = (kernel_ulong_t)&ad7276_chip_info },
+	{ .name = "ad7277", .driver_data = (kernel_ulong_t)&ad7277_chip_info },
+	{ .name = "ad7278", .driver_data = (kernel_ulong_t)&ad7278_chip_info },
+	{ .name = "ad7466", .driver_data = (kernel_ulong_t)&ad7466_chip_info },
+	{ .name = "ad7467", .driver_data = (kernel_ulong_t)&ad7467_chip_info },
+	{ .name = "ad7468", .driver_data = (kernel_ulong_t)&ad7468_chip_info },
+	{ .name = "ad7475", .driver_data = (kernel_ulong_t)&ad7475_chip_info },
+	{ .name = "ad7476", .driver_data = (kernel_ulong_t)&ad7466_chip_info },
+	{ .name = "ad7476a", .driver_data = (kernel_ulong_t)&ad7466_chip_info },
+	{ .name = "ad7477", .driver_data = (kernel_ulong_t)&ad7467_chip_info },
+	{ .name = "ad7477a", .driver_data = (kernel_ulong_t)&ad7467_chip_info },
+	{ .name = "ad7478", .driver_data = (kernel_ulong_t)&ad7468_chip_info },
+	{ .name = "ad7478a", .driver_data = (kernel_ulong_t)&ad7468_chip_info },
+	{ .name = "ad7495", .driver_data = (kernel_ulong_t)&ad7495_chip_info },
+	{ .name = "ad7910", .driver_data = (kernel_ulong_t)&ad7467_chip_info },
+	{ .name = "ad7920", .driver_data = (kernel_ulong_t)&ad7466_chip_info },
+	{ .name = "ad7940", .driver_data = (kernel_ulong_t)&ad7940_chip_info },
+	{ .name = "adc081s", .driver_data = (kernel_ulong_t)&adc081s_chip_info },
+	{ .name = "adc101s", .driver_data = (kernel_ulong_t)&adc101s_chip_info },
+	{ .name = "adc121s", .driver_data = (kernel_ulong_t)&adc121s_chip_info },
+	{ .name = "ads7866", .driver_data = (kernel_ulong_t)&ads7866_chip_info },
+	{ .name = "ads7867", .driver_data = (kernel_ulong_t)&ads7867_chip_info },
+	{ .name = "ads7868", .driver_data = (kernel_ulong_t)&ads7868_chip_info },
+	{ .name = "bd79105", .driver_data = (kernel_ulong_t)&bd79105_chip_info },
 	/*
 	 * The ROHM BU79100G is identical to the TI's ADS7866 from the software
 	 * point of view. The binding document mandates the ADS7866 to be
 	 * marked as a fallback for the BU79100G, but we still need the SPI ID
 	 * here to make the module loading work.
 	 */
-	{ "bu79100g", (kernel_ulong_t)&ads7866_chip_info },
-	{ "ltc2314-14", (kernel_ulong_t)&ltc2314_14_chip_info },
+	{ .name = "bu79100g", .driver_data = (kernel_ulong_t)&ads7866_chip_info },
+	{ .name = "ltc2314-14", .driver_data = (kernel_ulong_t)&ltc2314_14_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7476_id);
diff --git a/drivers/iio/adc/ad7606_spi.c b/drivers/iio/adc/ad7606_spi.c
index 7e17ccbcedd0c..1ebdfb8580ab5 100644
--- a/drivers/iio/adc/ad7606_spi.c
+++ b/drivers/iio/adc/ad7606_spi.c
@@ -463,17 +463,17 @@ static int ad7606_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad7606_id_table[] = {
-	{ "ad7605-4", (kernel_ulong_t)&ad7605_4_bus_info },
-	{ "ad7606-4", (kernel_ulong_t)&ad7606_4_bus_info },
-	{ "ad7606-6", (kernel_ulong_t)&ad7606_6_bus_info },
-	{ "ad7606-8", (kernel_ulong_t)&ad7606_8_bus_info },
-	{ "ad7606b",  (kernel_ulong_t)&ad7606b_bus_info },
-	{ "ad7606c-16", (kernel_ulong_t)&ad7606c_16_bus_info },
-	{ "ad7606c-18", (kernel_ulong_t)&ad7606c_18_bus_info },
-	{ "ad7607",   (kernel_ulong_t)&ad7607_bus_info },
-	{ "ad7608",   (kernel_ulong_t)&ad7608_bus_info },
-	{ "ad7609",   (kernel_ulong_t)&ad7609_bus_info },
-	{ "ad7616",   (kernel_ulong_t)&ad7616_bus_info },
+	{ .name = "ad7605-4",   .driver_data = (kernel_ulong_t)&ad7605_4_bus_info },
+	{ .name = "ad7606-4",   .driver_data = (kernel_ulong_t)&ad7606_4_bus_info },
+	{ .name = "ad7606-6",   .driver_data = (kernel_ulong_t)&ad7606_6_bus_info },
+	{ .name = "ad7606-8",   .driver_data = (kernel_ulong_t)&ad7606_8_bus_info },
+	{ .name = "ad7606b",    .driver_data = (kernel_ulong_t)&ad7606b_bus_info },
+	{ .name = "ad7606c-16", .driver_data = (kernel_ulong_t)&ad7606c_16_bus_info },
+	{ .name = "ad7606c-18", .driver_data = (kernel_ulong_t)&ad7606c_18_bus_info },
+	{ .name = "ad7607",     .driver_data = (kernel_ulong_t)&ad7607_bus_info },
+	{ .name = "ad7608",     .driver_data = (kernel_ulong_t)&ad7608_bus_info },
+	{ .name = "ad7609",     .driver_data = (kernel_ulong_t)&ad7609_bus_info },
+	{ .name = "ad7616",     .driver_data = (kernel_ulong_t)&ad7616_bus_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7606_id_table);
diff --git a/drivers/iio/adc/ad7766.c b/drivers/iio/adc/ad7766.c
index 9e4a66477d2d0..92957196de598 100644
--- a/drivers/iio/adc/ad7766.c
+++ b/drivers/iio/adc/ad7766.c
@@ -285,12 +285,12 @@ static int ad7766_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad7766_id[] = {
-	{ "ad7766", ID_AD7766 },
-	{ "ad7766-1", ID_AD7766_1 },
-	{ "ad7766-2", ID_AD7766_2 },
-	{ "ad7767", ID_AD7766 },
-	{ "ad7767-1", ID_AD7766_1 },
-	{ "ad7767-2", ID_AD7766_2 },
+	{ .name = "ad7766", .driver_data = ID_AD7766 },
+	{ .name = "ad7766-1", .driver_data = ID_AD7766_1 },
+	{ .name = "ad7766-2", .driver_data = ID_AD7766_2 },
+	{ .name = "ad7767", .driver_data = ID_AD7766 },
+	{ .name = "ad7767-1", .driver_data = ID_AD7766_1 },
+	{ .name = "ad7767-2", .driver_data = ID_AD7766_2 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7766_id);
diff --git a/drivers/iio/adc/ad7768-1.c b/drivers/iio/adc/ad7768-1.c
index 598936e47fd2d..e9060c1bbe6fb 100644
--- a/drivers/iio/adc/ad7768-1.c
+++ b/drivers/iio/adc/ad7768-1.c
@@ -1918,10 +1918,10 @@ static int ad7768_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad7768_id_table[] = {
-	{ "ad7768-1", (kernel_ulong_t)&ad7768_chip_info },
-	{ "adaq7767-1", (kernel_ulong_t)&adaq7767_chip_info },
-	{ "adaq7768-1", (kernel_ulong_t)&adaq7768_chip_info },
-	{ "adaq7769-1", (kernel_ulong_t)&adaq7769_chip_info },
+	{ .name = "ad7768-1", .driver_data = (kernel_ulong_t)&ad7768_chip_info },
+	{ .name = "adaq7767-1", .driver_data = (kernel_ulong_t)&adaq7767_chip_info },
+	{ .name = "adaq7768-1", .driver_data = (kernel_ulong_t)&adaq7768_chip_info },
+	{ .name = "adaq7769-1", .driver_data = (kernel_ulong_t)&adaq7769_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7768_id_table);
diff --git a/drivers/iio/adc/ad7779.c b/drivers/iio/adc/ad7779.c
index 003e23d6e2427..79fb38d0f400a 100644
--- a/drivers/iio/adc/ad7779.c
+++ b/drivers/iio/adc/ad7779.c
@@ -142,7 +142,6 @@ struct ad7779_state {
 	const struct ad7779_chip_info *chip_info;
 	struct clk *mclk;
 	struct iio_trigger *trig;
-	struct completion completion;
 	unsigned int sampling_freq;
 	enum ad7779_filter filter_enabled;
 	struct iio_backend *back;
@@ -842,8 +841,7 @@ static int ad7779_setup_without_backend(struct ad7779_state *st, struct iio_dev
 			       IRQF_NO_THREAD | IRQF_NO_AUTOEN, indio_dev->name,
 			       st->trig);
 	if (ret)
-		return dev_err_probe(dev, ret, "request IRQ %d failed\n",
-				     st->spi->irq);
+		return ret;
 
 	ret = devm_iio_trigger_register(dev, st->trig);
 	if (ret)
@@ -851,8 +849,6 @@ static int ad7779_setup_without_backend(struct ad7779_state *st, struct iio_dev
 
 	indio_dev->trig = iio_trigger_get(st->trig);
 
-	init_completion(&st->completion);
-
 	ret = devm_iio_triggered_buffer_setup(dev, indio_dev,
 					      &iio_pollfunc_store_time,
 					      &ad7779_trigger_handler,
diff --git a/drivers/iio/adc/ad7780.c b/drivers/iio/adc/ad7780.c
index 26382b1f8c9e4..1cfad70320feb 100644
--- a/drivers/iio/adc/ad7780.c
+++ b/drivers/iio/adc/ad7780.c
@@ -347,10 +347,10 @@ static int ad7780_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad7780_id[] = {
-	{ "ad7170", ID_AD7170 },
-	{ "ad7171", ID_AD7171 },
-	{ "ad7780", ID_AD7780 },
-	{ "ad7781", ID_AD7781 },
+	{ .name = "ad7170", .driver_data = ID_AD7170 },
+	{ .name = "ad7171", .driver_data = ID_AD7171 },
+	{ .name = "ad7780", .driver_data = ID_AD7780 },
+	{ .name = "ad7781", .driver_data = ID_AD7781 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7780_id);
diff --git a/drivers/iio/adc/ad7791.c b/drivers/iio/adc/ad7791.c
index bcdc19e799aaf..c2688307bbba1 100644
--- a/drivers/iio/adc/ad7791.c
+++ b/drivers/iio/adc/ad7791.c
@@ -458,11 +458,11 @@ static int ad7791_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad7791_spi_ids[] = {
-	{ "ad7787", AD7787 },
-	{ "ad7788", AD7788 },
-	{ "ad7789", AD7789 },
-	{ "ad7790", AD7790 },
-	{ "ad7791", AD7791 },
+	{ .name = "ad7787", .driver_data = AD7787 },
+	{ .name = "ad7788", .driver_data = AD7788 },
+	{ .name = "ad7789", .driver_data = AD7789 },
+	{ .name = "ad7790", .driver_data = AD7790 },
+	{ .name = "ad7791", .driver_data = AD7791 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7791_spi_ids);
diff --git a/drivers/iio/adc/ad7793.c b/drivers/iio/adc/ad7793.c
index ea1c192e21421..b0f44c144f20e 100644
--- a/drivers/iio/adc/ad7793.c
+++ b/drivers/iio/adc/ad7793.c
@@ -829,15 +829,15 @@ static int ad7793_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad7793_id[] = {
-	{ "ad7785", ID_AD7785 },
-	{ "ad7792", ID_AD7792 },
-	{ "ad7793", ID_AD7793 },
-	{ "ad7794", ID_AD7794 },
-	{ "ad7795", ID_AD7795 },
-	{ "ad7796", ID_AD7796 },
-	{ "ad7797", ID_AD7797 },
-	{ "ad7798", ID_AD7798 },
-	{ "ad7799", ID_AD7799 },
+	{ .name = "ad7785", .driver_data = ID_AD7785 },
+	{ .name = "ad7792", .driver_data = ID_AD7792 },
+	{ .name = "ad7793", .driver_data = ID_AD7793 },
+	{ .name = "ad7794", .driver_data = ID_AD7794 },
+	{ .name = "ad7795", .driver_data = ID_AD7795 },
+	{ .name = "ad7796", .driver_data = ID_AD7796 },
+	{ .name = "ad7797", .driver_data = ID_AD7797 },
+	{ .name = "ad7798", .driver_data = ID_AD7798 },
+	{ .name = "ad7799", .driver_data = ID_AD7799 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7793_id);
diff --git a/drivers/iio/adc/ad7887.c b/drivers/iio/adc/ad7887.c
index 068171d54596a..62aaedbce7483 100644
--- a/drivers/iio/adc/ad7887.c
+++ b/drivers/iio/adc/ad7887.c
@@ -328,7 +328,7 @@ static int ad7887_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad7887_id[] = {
-	{ "ad7887", ID_AD7887 },
+	{ .name = "ad7887", .driver_data = ID_AD7887 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7887_id);
diff --git a/drivers/iio/adc/ad7923.c b/drivers/iio/adc/ad7923.c
index acc87d486aa42..44e014da97222 100644
--- a/drivers/iio/adc/ad7923.c
+++ b/drivers/iio/adc/ad7923.c
@@ -360,13 +360,13 @@ static int ad7923_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad7923_id[] = {
-	{ "ad7904", AD7904 },
-	{ "ad7914", AD7914 },
-	{ "ad7923", AD7924 },
-	{ "ad7924", AD7924 },
-	{ "ad7908", AD7908 },
-	{ "ad7918", AD7918 },
-	{ "ad7928", AD7928 },
+	{ .name = "ad7904", .driver_data = AD7904 },
+	{ .name = "ad7914", .driver_data = AD7914 },
+	{ .name = "ad7923", .driver_data = AD7924 },
+	{ .name = "ad7924", .driver_data = AD7924 },
+	{ .name = "ad7908", .driver_data = AD7908 },
+	{ .name = "ad7918", .driver_data = AD7918 },
+	{ .name = "ad7928", .driver_data = AD7928 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7923_id);
diff --git a/drivers/iio/adc/ad7944.c b/drivers/iio/adc/ad7944.c
index 7722cf9e82143..12656d3cdcec8 100644
--- a/drivers/iio/adc/ad7944.c
+++ b/drivers/iio/adc/ad7944.c
@@ -865,9 +865,9 @@ static const struct of_device_id ad7944_of_match[] = {
 MODULE_DEVICE_TABLE(of, ad7944_of_match);
 
 static const struct spi_device_id ad7944_spi_id[] = {
-	{ "ad7944", (kernel_ulong_t)&ad7944_chip_info },
-	{ "ad7985", (kernel_ulong_t)&ad7985_chip_info },
-	{ "ad7986", (kernel_ulong_t)&ad7986_chip_info },
+	{ .name = "ad7944", .driver_data = (kernel_ulong_t)&ad7944_chip_info },
+	{ .name = "ad7985", .driver_data = (kernel_ulong_t)&ad7985_chip_info },
+	{ .name = "ad7986", .driver_data = (kernel_ulong_t)&ad7986_chip_info },
 	{ }
 
 };
diff --git a/drivers/iio/adc/ad7949.c b/drivers/iio/adc/ad7949.c
index ebc629bcfd4da..93a57a3d59b83 100644
--- a/drivers/iio/adc/ad7949.c
+++ b/drivers/iio/adc/ad7949.c
@@ -410,9 +410,9 @@ static const struct of_device_id ad7949_spi_of_id[] = {
 MODULE_DEVICE_TABLE(of, ad7949_spi_of_id);
 
 static const struct spi_device_id ad7949_spi_id[] = {
-	{ "ad7949", ID_AD7949  },
-	{ "ad7682", ID_AD7682 },
-	{ "ad7689", ID_AD7689 },
+	{ .name = "ad7949", .driver_data = ID_AD7949 },
+	{ .name = "ad7682", .driver_data = ID_AD7682 },
+	{ .name = "ad7689", .driver_data = ID_AD7689 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7949_spi_id);
diff --git a/drivers/iio/adc/ad9467.c b/drivers/iio/adc/ad9467.c
index 0c377f9a7f256..34b7b1256f0df 100644
--- a/drivers/iio/adc/ad9467.c
+++ b/drivers/iio/adc/ad9467.c
@@ -1406,13 +1406,13 @@ static const struct of_device_id ad9467_of_match[] = {
 MODULE_DEVICE_TABLE(of, ad9467_of_match);
 
 static const struct spi_device_id ad9467_ids[] = {
-	{ "ad9211", (kernel_ulong_t)&ad9211_chip_tbl },
-	{ "ad9265", (kernel_ulong_t)&ad9265_chip_tbl },
-	{ "ad9434", (kernel_ulong_t)&ad9434_chip_tbl },
-	{ "ad9467", (kernel_ulong_t)&ad9467_chip_tbl },
-	{ "ad9643", (kernel_ulong_t)&ad9643_chip_tbl },
-	{ "ad9649", (kernel_ulong_t)&ad9649_chip_tbl, },
-	{ "ad9652", (kernel_ulong_t)&ad9652_chip_tbl, },
+	{ .name = "ad9211", .driver_data = (kernel_ulong_t)&ad9211_chip_tbl },
+	{ .name = "ad9265", .driver_data = (kernel_ulong_t)&ad9265_chip_tbl },
+	{ .name = "ad9434", .driver_data = (kernel_ulong_t)&ad9434_chip_tbl },
+	{ .name = "ad9467", .driver_data = (kernel_ulong_t)&ad9467_chip_tbl },
+	{ .name = "ad9643", .driver_data = (kernel_ulong_t)&ad9643_chip_tbl },
+	{ .name = "ad9649", .driver_data = (kernel_ulong_t)&ad9649_chip_tbl },
+	{ .name = "ad9652", .driver_data = (kernel_ulong_t)&ad9652_chip_tbl },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad9467_ids);
diff --git a/drivers/iio/adc/ade9000.c b/drivers/iio/adc/ade9000.c
index c0a9babfa519a..c6c3ea953feae 100644
--- a/drivers/iio/adc/ade9000.c
+++ b/drivers/iio/adc/ade9000.c
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0-only
-/**
+/*
  * ADE9000 driver
  *
  * Copyright 2025 Analog Devices Inc.
@@ -1768,7 +1768,7 @@ static int ade9000_probe(struct spi_device *spi)
 };
 
 static const struct spi_device_id ade9000_id[] = {
-	{ "ade9000", 0 },
+	{ .name = "ade9000" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ade9000_id);
diff --git a/drivers/iio/adc/envelope-detector.c b/drivers/iio/adc/envelope-detector.c
index 30672e584c108..aace7e1037702 100644
--- a/drivers/iio/adc/envelope-detector.c
+++ b/drivers/iio/adc/envelope-detector.c
@@ -405,3 +405,4 @@ module_platform_driver(envelope_detector_driver);
 MODULE_DESCRIPTION("Envelope detector using a DAC and a comparator");
 MODULE_AUTHOR("Peter Rosin <peda@axentia.se>");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/iio/adc/hi8435.c b/drivers/iio/adc/hi8435.c
index b01ba86c29458..9fa63de4b9cac 100644
--- a/drivers/iio/adc/hi8435.c
+++ b/drivers/iio/adc/hi8435.c
@@ -526,7 +526,7 @@ static const struct of_device_id hi8435_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, hi8435_dt_ids);
 
 static const struct spi_device_id hi8435_id[] = {
-	{ "hi8435", 0 },
+	{ .name = "hi8435" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, hi8435_id);
diff --git a/drivers/iio/adc/hx711.c b/drivers/iio/adc/hx711.c
index 17e9badf1d1c9..00adafa7cbd79 100644
--- a/drivers/iio/adc/hx711.c
+++ b/drivers/iio/adc/hx711.c
@@ -4,6 +4,8 @@
  *
  * Copyright (c) 2016 Andreas Klinger <ak@it-klinger.de>
  */
+#include <linux/array_size.h>
+#include <linux/dev_printk.h>
 #include <linux/err.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -20,6 +22,8 @@
 #include <linux/gpio/consumer.h>
 #include <linux/regulator/consumer.h>
 
+#define HX711_DATA_BITS		24
+
 /* gain to pulse and scale conversion */
 #define HX711_GAIN_MAX		3
 #define HX711_RESET_GAIN	128
@@ -73,6 +77,20 @@ static int hx711_get_scale_to_gain(const int *gain_scale, int scale)
 	return -EINVAL;
 }
 
+/**
+ * struct hx711_chip_info - per-variant static configuration
+ * @name: IIO device name
+ * @channels: channel specification array
+ * @num_channels: number of entries in @channels
+ * @iio_info: IIO info ops for this variant
+ */
+struct hx711_chip_info {
+	const char			*name;
+	const struct iio_chan_spec	*channels __counted_by_ptr(num_channels);
+	unsigned int			num_channels;
+	const struct iio_info		*iio_info;
+};
+
 struct hx711_data {
 	struct device		*dev;
 	struct gpio_desc	*gpiod_pd_sck;
@@ -80,6 +98,7 @@ struct hx711_data {
 	int			gain_set;	/* gain set on device */
 	int			gain_chan_a;	/* gain for channel A */
 	int			gain_scale[HX711_GAIN_MAX];
+	const struct hx711_chip_info	*chip_info;
 	struct mutex		lock;
 	/*
 	 * triggered buffer
@@ -137,17 +156,18 @@ static int hx711_cycle(struct hx711_data *hx711_data)
 	return gpiod_get_value(hx711_data->gpiod_dout);
 }
 
-static int hx711_read(struct hx711_data *hx711_data)
+static int hx711_read(struct hx711_data *hx711_data, int trailing_pulses)
 {
-	int i, ret;
 	int value = 0;
-	int val = gpiod_get_value(hx711_data->gpiod_dout);
+	int val;
+	int ret;
 
 	/* we double check if it's really down */
+	val = gpiod_get_value(hx711_data->gpiod_dout);
 	if (val)
 		return -EIO;
 
-	for (i = 0; i < 24; i++) {
+	for (int i = 0; i < HX711_DATA_BITS; i++) {
 		value <<= 1;
 		ret = hx711_cycle(hx711_data);
 		if (ret)
@@ -156,7 +176,7 @@ static int hx711_read(struct hx711_data *hx711_data)
 
 	value ^= 0x800000;
 
-	for (i = 0; i < hx711_get_gain_to_pulse(hx711_data->gain_set); i++)
+	for (int i = 0; i < trailing_pulses; i++)
 		hx711_cycle(hx711_data);
 
 	return value;
@@ -186,8 +206,9 @@ static int hx711_wait_for_ready(struct hx711_data *hx711_data)
 
 static int hx711_reset(struct hx711_data *hx711_data)
 {
-	int val = hx711_wait_for_ready(hx711_data);
+	int val;
 
+	val = hx711_wait_for_ready(hx711_data);
 	if (val) {
 		/*
 		 * an examination with the oszilloscope indicated
@@ -219,7 +240,8 @@ static int hx711_set_gain_for_channel(struct hx711_data *hx711_data, int chan)
 		if (hx711_data->gain_set == 32) {
 			hx711_data->gain_set = hx711_data->gain_chan_a;
 
-			ret = hx711_read(hx711_data);
+			ret = hx711_read(hx711_data,
+					 hx711_get_gain_to_pulse(hx711_data->gain_set));
 			if (ret < 0)
 				return ret;
 
@@ -231,7 +253,8 @@ static int hx711_set_gain_for_channel(struct hx711_data *hx711_data, int chan)
 		if (hx711_data->gain_set != 32) {
 			hx711_data->gain_set = 32;
 
-			ret = hx711_read(hx711_data);
+			ret = hx711_read(hx711_data,
+					 hx711_get_gain_to_pulse(hx711_data->gain_set));
 			if (ret < 0)
 				return ret;
 
@@ -244,10 +267,26 @@ static int hx711_set_gain_for_channel(struct hx711_data *hx711_data, int chan)
 	return 0;
 }
 
-static int hx711_reset_read(struct hx711_data *hx711_data, int chan)
+static int hx711_set_hx711_channel(struct hx711_data *hx711_data,
+				   const struct iio_chan_spec *chan,
+				   int *trailing_pulses)
 {
 	int ret;
-	int val;
+
+	ret = hx711_set_gain_for_channel(hx711_data, chan->channel);
+	if (ret < 0)
+		return ret;
+
+	*trailing_pulses = hx711_get_gain_to_pulse(hx711_data->gain_set);
+
+	return 0;
+}
+
+static int hx711_reset_read(struct hx711_data *hx711_data,
+			    const struct iio_chan_spec *chan)
+{
+	int trailing_pulses;
+	int ret;
 
 	/*
 	 * hx711_reset() must be called from here
@@ -258,13 +297,11 @@ static int hx711_reset_read(struct hx711_data *hx711_data, int chan)
 		return -EIO;
 	}
 
-	ret = hx711_set_gain_for_channel(hx711_data, chan);
+	ret = hx711_set_hx711_channel(hx711_data, chan, &trailing_pulses);
 	if (ret < 0)
 		return ret;
 
-	val = hx711_read(hx711_data);
-
-	return val;
+	return hx711_read(hx711_data, trailing_pulses);
 }
 
 static int hx711_read_raw(struct iio_dev *indio_dev,
@@ -277,7 +314,7 @@ static int hx711_read_raw(struct iio_dev *indio_dev,
 	case IIO_CHAN_INFO_RAW:
 		mutex_lock(&hx711_data->lock);
 
-		*val = hx711_reset_read(hx711_data, chan->channel);
+		*val = hx711_reset_read(hx711_data, chan);
 
 		mutex_unlock(&hx711_data->lock);
 
@@ -331,7 +368,8 @@ static int hx711_write_raw(struct iio_dev *indio_dev,
 			if (gain != 32)
 				hx711_data->gain_chan_a = gain;
 
-			ret = hx711_read(hx711_data);
+			ret = hx711_read(hx711_data,
+					 hx711_get_gain_to_pulse(hx711_data->gain_set));
 			if (ret < 0) {
 				mutex_unlock(&hx711_data->lock);
 				return ret;
@@ -366,8 +404,8 @@ static irqreturn_t hx711_trigger(int irq, void *p)
 	memset(&hx711_data->buffer, 0, sizeof(hx711_data->buffer));
 
 	iio_for_each_active_channel(indio_dev, i) {
-		hx711_data->buffer.channel[j] = hx711_reset_read(hx711_data,
-					indio_dev->channels[i].channel);
+		hx711_data->buffer.channel[j] =
+			hx711_reset_read(hx711_data, &indio_dev->channels[i]);
 		j++;
 	}
 
@@ -455,8 +493,16 @@ static const struct iio_chan_spec hx711_chan_spec[] = {
 	IIO_CHAN_SOFT_TIMESTAMP(2),
 };
 
+static const struct hx711_chip_info hx711_chip = {
+	.name		= "hx711",
+	.channels	= hx711_chan_spec,
+	.iio_info	= &hx711_iio_info,
+	.num_channels	= ARRAY_SIZE(hx711_chan_spec),
+};
+
 static int hx711_probe(struct platform_device *pdev)
 {
+	const struct hx711_chip_info *chip_info;
 	struct device *dev = &pdev->dev;
 	struct hx711_data *hx711_data;
 	struct iio_dev *indio_dev;
@@ -472,6 +518,12 @@ static int hx711_probe(struct platform_device *pdev)
 
 	mutex_init(&hx711_data->lock);
 
+	chip_info = device_get_match_data(dev);
+	if (!chip_info)
+		return dev_err_probe(dev, -ENODEV, "missing driver data\n");
+
+	hx711_data->chip_info = chip_info;
+
 	/*
 	 * PD_SCK stands for power down and serial clock input of HX711
 	 * in the driver it is an output
@@ -533,11 +585,11 @@ static int hx711_probe(struct platform_device *pdev)
 	hx711_data->data_ready_delay_ns =
 				1000000000 / hx711_data->clock_frequency;
 
-	indio_dev->name = "hx711";
-	indio_dev->info = &hx711_iio_info;
+	indio_dev->name = chip_info->name;
+	indio_dev->info = chip_info->iio_info;
 	indio_dev->modes = INDIO_DIRECT_MODE;
-	indio_dev->channels = hx711_chan_spec;
-	indio_dev->num_channels = ARRAY_SIZE(hx711_chan_spec);
+	indio_dev->channels = chip_info->channels;
+	indio_dev->num_channels = chip_info->num_channels;
 
 	ret = devm_iio_triggered_buffer_setup(dev, indio_dev,
 					      iio_pollfunc_store_time,
@@ -554,7 +606,7 @@ static int hx711_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id of_hx711_match[] = {
-	{ .compatible = "avia,hx711", },
+	{ .compatible = "avia,hx711", .data = &hx711_chip },
 	{ }
 };
 
diff --git a/drivers/iio/adc/max1027.c b/drivers/iio/adc/max1027.c
index 8b2c7b2a9c8f4..fb6f12316724b 100644
--- a/drivers/iio/adc/max1027.c
+++ b/drivers/iio/adc/max1027.c
@@ -72,12 +72,12 @@ enum max1027_id {
 };
 
 static const struct spi_device_id max1027_id[] = {
-	{ "max1027", max1027 },
-	{ "max1029", max1029 },
-	{ "max1031", max1031 },
-	{ "max1227", max1227 },
-	{ "max1229", max1229 },
-	{ "max1231", max1231 },
+	{ .name = "max1027", .driver_data = max1027 },
+	{ .name = "max1029", .driver_data = max1029 },
+	{ .name = "max1031", .driver_data = max1031 },
+	{ .name = "max1227", .driver_data = max1227 },
+	{ .name = "max1229", .driver_data = max1229 },
+	{ .name = "max1231", .driver_data = max1231 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, max1027_id);
diff --git a/drivers/iio/adc/max1118.c b/drivers/iio/adc/max1118.c
index d394e03cc4e82..75e51ce3077e1 100644
--- a/drivers/iio/adc/max1118.c
+++ b/drivers/iio/adc/max1118.c
@@ -256,9 +256,9 @@ static int max1118_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id max1118_id[] = {
-	{ "max1117", max1117 },
-	{ "max1118", max1118 },
-	{ "max1119", max1119 },
+	{ .name = "max1117", .driver_data = max1117 },
+	{ .name = "max1118", .driver_data = max1118 },
+	{ .name = "max1119", .driver_data = max1119 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, max1118_id);
diff --git a/drivers/iio/adc/max11205.c b/drivers/iio/adc/max11205.c
index 6c803df220b6e..63b6561dcf475 100644
--- a/drivers/iio/adc/max11205.c
+++ b/drivers/iio/adc/max11205.c
@@ -145,8 +145,8 @@ static int max11205_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id max11205_spi_ids[] = {
-	{ "max11205a", (kernel_ulong_t)&max11205_chip_info[TYPE_MAX11205A] },
-	{ "max11205b", (kernel_ulong_t)&max11205_chip_info[TYPE_MAX11205B] },
+	{ .name = "max11205a", .driver_data = (kernel_ulong_t)&max11205_chip_info[TYPE_MAX11205A] },
+	{ .name = "max11205b", .driver_data = (kernel_ulong_t)&max11205_chip_info[TYPE_MAX11205B] },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, max11205_spi_ids);
diff --git a/drivers/iio/adc/max11410.c b/drivers/iio/adc/max11410.c
index dc1b963565920..23ae684cf08b2 100644
--- a/drivers/iio/adc/max11410.c
+++ b/drivers/iio/adc/max11410.c
@@ -1025,7 +1025,7 @@ static const struct of_device_id max11410_spi_of_id[] = {
 MODULE_DEVICE_TABLE(of, max11410_spi_of_id);
 
 static const struct spi_device_id max11410_id[] = {
-	{ "max11410" },
+	{ .name = "max11410" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, max11410_id);
diff --git a/drivers/iio/adc/max1241.c b/drivers/iio/adc/max1241.c
index d62c1a0116593..b3cecbf9a89ed 100644
--- a/drivers/iio/adc/max1241.c
+++ b/drivers/iio/adc/max1241.c
@@ -15,10 +15,6 @@
 #define MAX1241_VAL_MASK GENMASK(11, 0)
 #define MAX1241_SHUTDOWN_DELAY_USEC 4
 
-enum max1241_id {
-	max1241,
-};
-
 struct max1241 {
 	struct spi_device *spi;
 	struct mutex lock;
@@ -166,7 +162,7 @@ static int max1241_probe(struct spi_device *spi)
 	else
 		dev_dbg(dev, "no shutdown pin passed, low-power mode disabled");
 
-	indio_dev->name = spi_get_device_id(spi)->name;
+	indio_dev->name = "max1241";
 	indio_dev->info = &max1241_info;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->channels = max1241_channels;
@@ -176,9 +172,10 @@ static int max1241_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id max1241_id[] = {
-	{ "max1241", max1241 },
+	{ .name = "max1241" },
 	{ }
 };
+MODULE_DEVICE_TABLE(spi, max1241_id);
 
 static const struct of_device_id max1241_dt_ids[] = {
 	{ .compatible = "maxim,max1241" },
diff --git a/drivers/iio/adc/max14001.c b/drivers/iio/adc/max14001.c
index 09017163b1918..58adeac62dde4 100644
--- a/drivers/iio/adc/max14001.c
+++ b/drivers/iio/adc/max14001.c
@@ -362,8 +362,8 @@ static struct max14001_chip_info max14002_chip_info = {
 };
 
 static const struct spi_device_id max14001_id_table[] = {
-	{ "max14001", (kernel_ulong_t)&max14001_chip_info },
-	{ "max14002", (kernel_ulong_t)&max14002_chip_info },
+	{ .name = "max14001", .driver_data = (kernel_ulong_t)&max14001_chip_info },
+	{ .name = "max14002", .driver_data = (kernel_ulong_t)&max14002_chip_info },
 	{ }
 };
 
diff --git a/drivers/iio/adc/mcp320x.c b/drivers/iio/adc/mcp320x.c
index 686c519aa3d25..18cb957e70958 100644
--- a/drivers/iio/adc/mcp320x.c
+++ b/drivers/iio/adc/mcp320x.c
@@ -476,19 +476,19 @@ static const struct of_device_id mcp320x_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, mcp320x_dt_ids);
 
 static const struct spi_device_id mcp320x_id[] = {
-	{ "mcp3001", mcp3001 },
-	{ "mcp3002", mcp3002 },
-	{ "mcp3004", mcp3004 },
-	{ "mcp3008", mcp3008 },
-	{ "mcp3201", mcp3201 },
-	{ "mcp3202", mcp3202 },
-	{ "mcp3204", mcp3204 },
-	{ "mcp3208", mcp3208 },
-	{ "mcp3301", mcp3301 },
-	{ "mcp3550-50", mcp3550_50 },
-	{ "mcp3550-60", mcp3550_60 },
-	{ "mcp3551", mcp3551 },
-	{ "mcp3553", mcp3553 },
+	{ .name = "mcp3001", .driver_data = mcp3001 },
+	{ .name = "mcp3002", .driver_data = mcp3002 },
+	{ .name = "mcp3004", .driver_data = mcp3004 },
+	{ .name = "mcp3008", .driver_data = mcp3008 },
+	{ .name = "mcp3201", .driver_data = mcp3201 },
+	{ .name = "mcp3202", .driver_data = mcp3202 },
+	{ .name = "mcp3204", .driver_data = mcp3204 },
+	{ .name = "mcp3208", .driver_data = mcp3208 },
+	{ .name = "mcp3301", .driver_data = mcp3301 },
+	{ .name = "mcp3550-50", .driver_data = mcp3550_50 },
+	{ .name = "mcp3550-60", .driver_data = mcp3550_60 },
+	{ .name = "mcp3551", .driver_data = mcp3551 },
+	{ .name = "mcp3553", .driver_data = mcp3553 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, mcp320x_id);
diff --git a/drivers/iio/adc/mcp3564.c b/drivers/iio/adc/mcp3564.c
index 36675563829ef..259ac4ebdedf1 100644
--- a/drivers/iio/adc/mcp3564.c
+++ b/drivers/iio/adc/mcp3564.c
@@ -1451,18 +1451,18 @@ static const struct of_device_id mcp3564_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, mcp3564_dt_ids);
 
 static const struct spi_device_id mcp3564_id[] = {
-	{ "mcp3461", (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3461] },
-	{ "mcp3462", (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3462] },
-	{ "mcp3464", (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3464] },
-	{ "mcp3561", (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3561] },
-	{ "mcp3562", (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3562] },
-	{ "mcp3564", (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3564] },
-	{ "mcp3461r", (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3461r] },
-	{ "mcp3462r", (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3462r] },
-	{ "mcp3464r", (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3464r] },
-	{ "mcp3561r", (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3561r] },
-	{ "mcp3562r", (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3562r] },
-	{ "mcp3564r", (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3564r] },
+	{ .name = "mcp3461", .driver_data = (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3461] },
+	{ .name = "mcp3462", .driver_data = (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3462] },
+	{ .name = "mcp3464", .driver_data = (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3464] },
+	{ .name = "mcp3561", .driver_data = (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3561] },
+	{ .name = "mcp3562", .driver_data = (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3562] },
+	{ .name = "mcp3564", .driver_data = (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3564] },
+	{ .name = "mcp3461r", .driver_data = (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3461r] },
+	{ .name = "mcp3462r", .driver_data = (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3462r] },
+	{ .name = "mcp3464r", .driver_data = (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3464r] },
+	{ .name = "mcp3561r", .driver_data = (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3561r] },
+	{ .name = "mcp3562r", .driver_data = (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3562r] },
+	{ .name = "mcp3564r", .driver_data = (kernel_ulong_t)&mcp3564_chip_infos_tbl[mcp3564r] },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, mcp3564_id);
diff --git a/drivers/iio/adc/mcp3911.c b/drivers/iio/adc/mcp3911.c
index 5bf74b49bdb53..797424f59f74b 100644
--- a/drivers/iio/adc/mcp3911.c
+++ b/drivers/iio/adc/mcp3911.c
@@ -927,13 +927,13 @@ static const struct of_device_id mcp3911_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, mcp3911_dt_ids);
 
 static const struct spi_device_id mcp3911_id[] = {
-	{ "mcp3910", (kernel_ulong_t)&mcp3911_chip_info[MCP3910] },
-	{ "mcp3911", (kernel_ulong_t)&mcp3911_chip_info[MCP3911] },
-	{ "mcp3912", (kernel_ulong_t)&mcp3911_chip_info[MCP3912] },
-	{ "mcp3913", (kernel_ulong_t)&mcp3911_chip_info[MCP3913] },
-	{ "mcp3914", (kernel_ulong_t)&mcp3911_chip_info[MCP3914] },
-	{ "mcp3918", (kernel_ulong_t)&mcp3911_chip_info[MCP3918] },
-	{ "mcp3919", (kernel_ulong_t)&mcp3911_chip_info[MCP3919] },
+	{ .name = "mcp3910", .driver_data = (kernel_ulong_t)&mcp3911_chip_info[MCP3910] },
+	{ .name = "mcp3911", .driver_data = (kernel_ulong_t)&mcp3911_chip_info[MCP3911] },
+	{ .name = "mcp3912", .driver_data = (kernel_ulong_t)&mcp3911_chip_info[MCP3912] },
+	{ .name = "mcp3913", .driver_data = (kernel_ulong_t)&mcp3911_chip_info[MCP3913] },
+	{ .name = "mcp3914", .driver_data = (kernel_ulong_t)&mcp3911_chip_info[MCP3914] },
+	{ .name = "mcp3918", .driver_data = (kernel_ulong_t)&mcp3911_chip_info[MCP3918] },
+	{ .name = "mcp3919", .driver_data = (kernel_ulong_t)&mcp3911_chip_info[MCP3919] },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, mcp3911_id);
diff --git a/drivers/iio/adc/mt6323-auxadc.c b/drivers/iio/adc/mt6323-auxadc.c
new file mode 100644
index 0000000000000..c450fb6f09cb5
--- /dev/null
+++ b/drivers/iio/adc/mt6323-auxadc.c
@@ -0,0 +1,314 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2026 Roman Vivchar <rva333@protonmail.com>
+ *
+ * Based on drivers/iio/adc/mt6359-auxadc.c
+ */
+
+#include <linux/array_size.h>
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/cleanup.h>
+#include <linux/delay.h>
+#include <linux/iio/iio.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/stringify.h>
+#include <linux/time.h>
+#include <linux/types.h>
+
+#include <linux/mfd/mt6323/registers.h>
+
+#include <dt-bindings/iio/adc/mediatek,mt6323-auxadc.h>
+
+#define AUXADC_STRUP_CON10_RSTB_SEL	BIT(7)
+#define AUXADC_STRUP_CON10_RSTB_SW	BIT(5)
+
+#define AUXADC_TOP_CKPDN2_CTL_CK	BIT(5)
+
+#define AUXADC_TRIM_CH2_MASK		GENMASK(11, 10)
+#define AUXADC_TRIM_CH4_MASK		GENMASK(9, 8)
+#define AUXADC_TRIM_CH5_MASK		GENMASK(5, 4)
+#define AUXADC_TRIM_CH6_MASK		GENMASK(3, 2)
+
+#define AUXADC_CON27_VREF18_ENB_MD	BIT(15)
+#define AUXADC_CON27_MD_STATUS		BIT(0)
+
+#define AUXADC_CON19_GPS_STATUS		BIT(1)
+
+#define AUXADC_CON26_VREF18_SELB	BIT(1)
+#define AUXADC_CON26_DECI_GDLY_SEL	BIT(0)
+
+#define AUXADC_CON11_VBUF_EN		BIT(4)
+
+#define AUXADC_CON19_DECI_GDLY_MASK	GENMASK(15, 14)
+#define AUXADC_ADC19_BUSY_MASK		GENMASK(15, 1)
+#define AUXADC_READY_MASK		BIT(15)
+#define AUXADC_DATA_MASK		GENMASK(14, 0)
+
+#define AUXADC_CON9_OSR_MASK		GENMASK(12, 10)
+#define AUXADC_DEFAULT_OSR		3
+
+#define MTK_PMIC_IIO_CHAN(_name, _chan, _addr)                  \
+{                                                               \
+	.type = IIO_VOLTAGE,                                    \
+	.indexed = 1,                                           \
+	.channel = _chan,                                       \
+	.address = _addr,                                       \
+	.datasheet_name = __stringify(_name),                   \
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |          \
+			      BIT(IIO_CHAN_INFO_SCALE),         \
+}
+
+/*
+ * AUXADC reports everything in mV, including temperature and
+ * current channels. Channel macros are mapped such that their
+ * ID matches their respective hardware bit position in CON22.
+ */
+static const struct iio_chan_spec mt6323_auxadc_channels[] = {
+	MTK_PMIC_IIO_CHAN(baton2,    MT6323_AUXADC_BATON2,    MT6323_AUXADC_ADC6),
+	MTK_PMIC_IIO_CHAN(ch6,       MT6323_AUXADC_CH6,       MT6323_AUXADC_ADC11),
+	MTK_PMIC_IIO_CHAN(bat_temp,  MT6323_AUXADC_BAT_TEMP,  MT6323_AUXADC_ADC5),
+	MTK_PMIC_IIO_CHAN(chip_temp, MT6323_AUXADC_CHIP_TEMP, MT6323_AUXADC_ADC4),
+	MTK_PMIC_IIO_CHAN(vcdt,      MT6323_AUXADC_VCDT,      MT6323_AUXADC_ADC2),
+	MTK_PMIC_IIO_CHAN(baton1,    MT6323_AUXADC_BATON1,    MT6323_AUXADC_ADC3),
+	MTK_PMIC_IIO_CHAN(isense,    MT6323_AUXADC_ISENSE,    MT6323_AUXADC_ADC1),
+	MTK_PMIC_IIO_CHAN(batsns,    MT6323_AUXADC_BATSNS,    MT6323_AUXADC_ADC0),
+	MTK_PMIC_IIO_CHAN(accdet,    MT6323_AUXADC_ACCDET,    MT6323_AUXADC_ADC7),
+};
+
+/*
+ * The MediaTek MT6323 (as well as a lot of other PMICs) has the following hierarchy:
+ * PMIC AUXADC <- PMIC MFD <- SoC PWRAP (wrapper for PWRAP FSM)
+ *
+ * Therefore, PWRAP regmap should be obtained using dev->parent->parent.
+ */
+struct mt6323_auxadc {
+	struct regmap *regmap;
+	/* AUXADC doesn't support reading multiple channels simultaneously. */
+	struct mutex lock;
+};
+
+static int mt6323_auxadc_prepare_channel(struct mt6323_auxadc *auxadc)
+{
+	struct regmap *map = auxadc->regmap;
+	u32 val;
+	int ret;
+
+	ret = regmap_read(map, MT6323_AUXADC_CON19, &val);
+	if (ret)
+		return ret;
+
+	/* The ADC is idle. */
+	if (!(val & AUXADC_CON19_DECI_GDLY_MASK))
+		return 0;
+
+	ret = regmap_read_poll_timeout(map, MT6323_AUXADC_ADC19,
+				       val, !(val & AUXADC_ADC19_BUSY_MASK),
+				       10, 500);
+	if (ret)
+		return ret;
+
+	return regmap_clear_bits(map, MT6323_AUXADC_CON19,
+				 AUXADC_CON19_DECI_GDLY_MASK);
+}
+
+static int mt6323_auxadc_request(struct mt6323_auxadc *auxadc,
+				 unsigned long channel)
+{
+	struct regmap *map = auxadc->regmap;
+	int ret;
+
+	ret = regmap_set_bits(map, MT6323_AUXADC_CON11, AUXADC_CON11_VBUF_EN);
+	if (ret)
+		return ret;
+
+	return regmap_set_bits(map, MT6323_AUXADC_CON22, BIT(channel));
+}
+
+static int mt6323_auxadc_release(struct mt6323_auxadc *auxadc,
+				 unsigned long channel)
+{
+	struct regmap *map = auxadc->regmap;
+	int ret;
+
+	ret = regmap_clear_bits(map, MT6323_AUXADC_CON22, BIT(channel));
+	if (ret)
+		return ret;
+
+	return regmap_clear_bits(map, MT6323_AUXADC_CON11, AUXADC_CON11_VBUF_EN);
+}
+
+static int mt6323_auxadc_read(struct mt6323_auxadc *auxadc,
+			      const struct iio_chan_spec *chan, int *out)
+{
+	struct regmap *map = auxadc->regmap;
+	u32 val;
+	int ret;
+
+	ret = regmap_read_poll_timeout(map, chan->address,
+				       val, (val & AUXADC_READY_MASK),
+				       1 * USEC_PER_MSEC, 100 * USEC_PER_MSEC);
+	if (ret)
+		return ret;
+
+	*out = FIELD_GET(AUXADC_DATA_MASK, val);
+
+	return 0;
+}
+
+static int mt6323_auxadc_read_raw(struct iio_dev *indio_dev,
+				  const struct iio_chan_spec *chan,
+				  int *val, int *val2, long mask)
+{
+	struct mt6323_auxadc *auxadc = iio_priv(indio_dev);
+	int ret, mult;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SCALE:
+		if (chan->channel == MT6323_AUXADC_ISENSE ||
+		    chan->channel == MT6323_AUXADC_BATSNS)
+			mult = 4;
+		else
+			mult = 1;
+
+		/* 1800mV full range with 15-bit resolution. */
+		*val = mult * 1800;
+		*val2 = 15;
+
+		return IIO_VAL_FRACTIONAL_LOG2;
+	case IIO_CHAN_INFO_RAW: {
+		guard(mutex)(&auxadc->lock);
+
+		ret = mt6323_auxadc_prepare_channel(auxadc);
+		if (ret)
+			return ret;
+
+		ret = mt6323_auxadc_request(auxadc, chan->channel);
+		if (ret)
+			return ret;
+
+		/* Hardware limitation: the AUXADC needs a delay to become ready. */
+		fsleep(300);
+
+		ret = mt6323_auxadc_read(auxadc, chan, val);
+
+		if (mt6323_auxadc_release(auxadc, chan->channel))
+			dev_err(&indio_dev->dev,
+				"failed to release channel %d\n", chan->channel);
+
+		if (ret)
+			return ret;
+
+		return IIO_VAL_INT;
+	}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int mt6323_auxadc_init(struct mt6323_auxadc *auxadc)
+{
+	struct regmap *map = auxadc->regmap;
+	int ret;
+
+	ret = regmap_set_bits(map, MT6323_STRUP_CON10,
+			      AUXADC_STRUP_CON10_RSTB_SW |
+			      AUXADC_STRUP_CON10_RSTB_SEL);
+	if (ret)
+		return ret;
+
+	ret = regmap_set_bits(map, MT6323_TOP_CKPDN2, AUXADC_TOP_CKPDN2_CTL_CK);
+	if (ret)
+		return ret;
+
+	ret = regmap_update_bits(map, MT6323_AUXADC_CON10,
+				 AUXADC_TRIM_CH2_MASK | AUXADC_TRIM_CH4_MASK |
+				 AUXADC_TRIM_CH5_MASK | AUXADC_TRIM_CH6_MASK,
+				 FIELD_PREP(AUXADC_TRIM_CH2_MASK, 1) |
+				 FIELD_PREP(AUXADC_TRIM_CH4_MASK, 1) |
+				 FIELD_PREP(AUXADC_TRIM_CH5_MASK, 1) |
+				 FIELD_PREP(AUXADC_TRIM_CH6_MASK, 1));
+	if (ret)
+		return ret;
+
+	ret = regmap_set_bits(map, MT6323_AUXADC_CON27,
+			      AUXADC_CON27_VREF18_ENB_MD |
+			      AUXADC_CON27_MD_STATUS);
+	if (ret)
+		return ret;
+
+	ret = regmap_set_bits(map, MT6323_AUXADC_CON19, AUXADC_CON19_GPS_STATUS);
+	if (ret)
+		return ret;
+
+	ret = regmap_set_bits(map, MT6323_AUXADC_CON26,
+			      AUXADC_CON26_VREF18_SELB |
+			      AUXADC_CON26_DECI_GDLY_SEL);
+	if (ret)
+		return ret;
+
+	return regmap_update_bits(map, MT6323_AUXADC_CON9, AUXADC_CON9_OSR_MASK,
+				  FIELD_PREP(AUXADC_CON9_OSR_MASK, AUXADC_DEFAULT_OSR));
+}
+
+static const struct iio_info mt6323_auxadc_iio_info = {
+	.read_raw = mt6323_auxadc_read_raw,
+};
+
+static int mt6323_auxadc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct mt6323_auxadc *auxadc;
+	struct regmap *regmap;
+	struct iio_dev *iio;
+	int ret;
+
+	regmap = dev_get_regmap(dev->parent->parent, NULL);
+	if (!regmap)
+		return dev_err_probe(dev, -ENODEV, "failed to get regmap\n");
+
+	iio = devm_iio_device_alloc(dev, sizeof(*auxadc));
+	if (!iio)
+		return -ENOMEM;
+
+	auxadc = iio_priv(iio);
+	auxadc->regmap = regmap;
+
+	ret = devm_mutex_init(dev, &auxadc->lock);
+	if (ret)
+		return ret;
+
+	ret = mt6323_auxadc_init(auxadc);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to initialize auxadc\n");
+
+	iio->name = "mt6323-auxadc";
+	iio->info = &mt6323_auxadc_iio_info;
+	iio->modes = INDIO_DIRECT_MODE;
+	iio->channels = mt6323_auxadc_channels;
+	iio->num_channels = ARRAY_SIZE(mt6323_auxadc_channels);
+
+	return devm_iio_device_register(dev, iio);
+}
+
+static const struct of_device_id mt6323_auxadc_of_match[] = {
+	{ .compatible = "mediatek,mt6323-auxadc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, mt6323_auxadc_of_match);
+
+static struct platform_driver mt6323_auxadc_driver = {
+	.driver = {
+		.name = "mt6323-auxadc",
+		.of_match_table = mt6323_auxadc_of_match,
+	},
+	.probe	= mt6323_auxadc_probe,
+};
+module_platform_driver(mt6323_auxadc_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("MediaTek MT6323 PMIC AUXADC Driver");
diff --git a/drivers/iio/adc/rohm-bd79112.c b/drivers/iio/adc/rohm-bd79112.c
index c4b0bec807946..2506fec7be845 100644
--- a/drivers/iio/adc/rohm-bd79112.c
+++ b/drivers/iio/adc/rohm-bd79112.c
@@ -529,7 +529,7 @@ static const struct of_device_id bd79112_of_match[] = {
 MODULE_DEVICE_TABLE(of, bd79112_of_match);
 
 static const struct spi_device_id bd79112_id[] = {
-	{ "bd79112" },
+	{ .name = "bd79112" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, bd79112_id);
diff --git a/drivers/iio/adc/rtq6056.c b/drivers/iio/adc/rtq6056.c
index ba525a3c5cc28..2083c4351e43d 100644
--- a/drivers/iio/adc/rtq6056.c
+++ b/drivers/iio/adc/rtq6056.c
@@ -871,8 +871,8 @@ static const struct richtek_dev_data rtq6059_devdata = {
 };
 
 static const struct i2c_device_id rtq6056_id[] = {
-	{ "rtq6056", (kernel_ulong_t)&rtq6056_devdata },
-	{ "rtq6059", (kernel_ulong_t)&rtq6059_devdata },
+	{ .name = "rtq6056", .driver_data = (kernel_ulong_t)&rtq6056_devdata },
+	{ .name = "rtq6059", .driver_data = (kernel_ulong_t)&rtq6059_devdata },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, rtq6056_id);
diff --git a/drivers/iio/adc/stm32-dfsdm-adc.c b/drivers/iio/adc/stm32-dfsdm-adc.c
index 9664b9bd75d41..00f05e167afc9 100644
--- a/drivers/iio/adc/stm32-dfsdm-adc.c
+++ b/drivers/iio/adc/stm32-dfsdm-adc.c
@@ -660,11 +660,8 @@ static int stm32_dfsdm_channel_parse_of(struct stm32_dfsdm *dfsdm,
 	}
 	df_ch->src = val;
 
-	ret = of_property_read_u32_index(indio_dev->dev.of_node,
-					 "st,adc-alt-channel", chan_idx,
-					 &df_ch->alt_si);
-	if (ret < 0)
-		df_ch->alt_si = 0;
+	df_ch->alt_si = of_property_present(indio_dev->dev.of_node,
+					    "st,adc-alt-channel");
 
 	return 0;
 }
@@ -1815,9 +1812,8 @@ static int stm32_dfsdm_adc_probe(struct platform_device *pdev)
 
 	adc->dfsdm->fl_list[adc->fl_id].ford = val;
 
-	ret = of_property_read_u32(dev->of_node, "st,filter0-sync", &val);
-	if (!ret)
-		adc->dfsdm->fl_list[adc->fl_id].sync_mode = val;
+	adc->dfsdm->fl_list[adc->fl_id].sync_mode =
+		of_property_read_bool(dev->of_node, "st,filter0-sync");
 
 	adc->dev_data = dev_data;
 	ret = dev_data->init(dev, iio);
diff --git a/drivers/iio/adc/ti-adc081c.c b/drivers/iio/adc/ti-adc081c.c
index e33a2f4bf66ca..2df8f1c21c2a3 100644
--- a/drivers/iio/adc/ti-adc081c.c
+++ b/drivers/iio/adc/ti-adc081c.c
@@ -189,10 +189,8 @@ static int adc081c_probe(struct i2c_client *client)
 
 	err = devm_iio_triggered_buffer_setup(&client->dev, iio, NULL,
 					      adc081c_trigger_handler, NULL);
-	if (err < 0) {
-		dev_err(&client->dev, "iio triggered buffer setup failed\n");
-		return err;
-	}
+	if (err < 0)
+		return dev_err_probe(&client->dev, err, "iio triggered buffer setup failed\n");
 
 	return devm_iio_device_register(&client->dev, iio);
 }
diff --git a/drivers/iio/adc/ti-adc0832.c b/drivers/iio/adc/ti-adc0832.c
index 63d712d5d1113..a522d86760349 100644
--- a/drivers/iio/adc/ti-adc0832.c
+++ b/drivers/iio/adc/ti-adc0832.c
@@ -313,10 +313,10 @@ static const struct of_device_id adc0832_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, adc0832_dt_ids);
 
 static const struct spi_device_id adc0832_id[] = {
-	{ "adc0831", adc0831 },
-	{ "adc0832", adc0832 },
-	{ "adc0834", adc0834 },
-	{ "adc0838", adc0838 },
+	{ .name = "adc0831", .driver_data = adc0831 },
+	{ .name = "adc0832", .driver_data = adc0832 },
+	{ .name = "adc0834", .driver_data = adc0834 },
+	{ .name = "adc0838", .driver_data = adc0838 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adc0832_id);
diff --git a/drivers/iio/adc/ti-adc084s021.c b/drivers/iio/adc/ti-adc084s021.c
index 51596b500a906..cf2f6ee59411d 100644
--- a/drivers/iio/adc/ti-adc084s021.c
+++ b/drivers/iio/adc/ti-adc084s021.c
@@ -228,10 +228,8 @@ static int adc084s021_probe(struct spi_device *spi)
 	ret = devm_iio_triggered_buffer_setup(&spi->dev, indio_dev, NULL,
 					    adc084s021_buffer_trigger_handler,
 					    &adc084s021_buffer_setup_ops);
-	if (ret) {
-		dev_err(&spi->dev, "Failed to setup triggered buffer\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(&spi->dev, ret, "Failed to setup triggered buffer\n");
 
 	return devm_iio_device_register(&spi->dev, indio_dev);
 }
@@ -243,7 +241,7 @@ static const struct of_device_id adc084s021_of_match[] = {
 MODULE_DEVICE_TABLE(of, adc084s021_of_match);
 
 static const struct spi_device_id adc084s021_id[] = {
-	{ ADC084S021_DRIVER_NAME, 0 },
+	{ .name = ADC084S021_DRIVER_NAME },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adc084s021_id);
diff --git a/drivers/iio/adc/ti-adc108s102.c b/drivers/iio/adc/ti-adc108s102.c
index d1f61b440959f..10959e835cb8a 100644
--- a/drivers/iio/adc/ti-adc108s102.c
+++ b/drivers/iio/adc/ti-adc108s102.c
@@ -276,7 +276,7 @@ static const struct acpi_device_id adc108s102_acpi_ids[] = {
 MODULE_DEVICE_TABLE(acpi, adc108s102_acpi_ids);
 
 static const struct spi_device_id adc108s102_id[] = {
-	{ "adc108s102", 0 },
+	{ .name = "adc108s102" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adc108s102_id);
diff --git a/drivers/iio/adc/ti-adc12138.c b/drivers/iio/adc/ti-adc12138.c
index e5ec4b073daae..279e4999eb49e 100644
--- a/drivers/iio/adc/ti-adc12138.c
+++ b/drivers/iio/adc/ti-adc12138.c
@@ -516,9 +516,9 @@ static const struct of_device_id adc12138_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, adc12138_dt_ids);
 
 static const struct spi_device_id adc12138_id[] = {
-	{ "adc12130", adc12130 },
-	{ "adc12132", adc12132 },
-	{ "adc12138", adc12138 },
+	{ .name = "adc12130", .driver_data = adc12130 },
+	{ .name = "adc12132", .driver_data = adc12132 },
+	{ .name = "adc12138", .driver_data = adc12138 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adc12138_id);
diff --git a/drivers/iio/adc/ti-adc128s052.c b/drivers/iio/adc/ti-adc128s052.c
index 2cb68582ee6a4..1899813c1aee3 100644
--- a/drivers/iio/adc/ti-adc128s052.c
+++ b/drivers/iio/adc/ti-adc128s052.c
@@ -247,18 +247,18 @@ static const struct of_device_id adc128_of_match[] = {
 MODULE_DEVICE_TABLE(of, adc128_of_match);
 
 static const struct spi_device_id adc128_id[] = {
-	{ "adc128s052", (kernel_ulong_t)&adc128s_config },
-	{ "adc122s021",	(kernel_ulong_t)&adc122s_config },
-	{ "adc122s051",	(kernel_ulong_t)&adc122s_config },
-	{ "adc122s101",	(kernel_ulong_t)&adc122s_config },
-	{ "adc124s021", (kernel_ulong_t)&adc124s_config },
-	{ "adc124s051", (kernel_ulong_t)&adc124s_config },
-	{ "adc124s101", (kernel_ulong_t)&adc124s_config },
-	{ "bd79100", (kernel_ulong_t)&bd79100_config },
-	{ "bd79101", (kernel_ulong_t)&bd79101_config },
-	{ "bd79102", (kernel_ulong_t)&bd79102_config },
-	{ "bd79103", (kernel_ulong_t)&bd79104_config },
-	{ "bd79104", (kernel_ulong_t)&bd79104_config },
+	{ .name = "adc128s052", .driver_data = (kernel_ulong_t)&adc128s_config },
+	{ .name = "adc122s021", .driver_data = (kernel_ulong_t)&adc122s_config },
+	{ .name = "adc122s051", .driver_data = (kernel_ulong_t)&adc122s_config },
+	{ .name = "adc122s101", .driver_data = (kernel_ulong_t)&adc122s_config },
+	{ .name = "adc124s021", .driver_data = (kernel_ulong_t)&adc124s_config },
+	{ .name = "adc124s051", .driver_data = (kernel_ulong_t)&adc124s_config },
+	{ .name = "adc124s101", .driver_data = (kernel_ulong_t)&adc124s_config },
+	{ .name = "bd79100", .driver_data = (kernel_ulong_t)&bd79100_config },
+	{ .name = "bd79101", .driver_data = (kernel_ulong_t)&bd79101_config },
+	{ .name = "bd79102", .driver_data = (kernel_ulong_t)&bd79102_config },
+	{ .name = "bd79103", .driver_data = (kernel_ulong_t)&bd79104_config },
+	{ .name = "bd79104", .driver_data = (kernel_ulong_t)&bd79104_config },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adc128_id);
diff --git a/drivers/iio/adc/ti-adc161s626.c b/drivers/iio/adc/ti-adc161s626.c
index 08aa32bd5e4b5..422df0685580a 100644
--- a/drivers/iio/adc/ti-adc161s626.c
+++ b/drivers/iio/adc/ti-adc161s626.c
@@ -229,8 +229,8 @@ static const struct of_device_id ti_adc_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, ti_adc_dt_ids);
 
 static const struct spi_device_id ti_adc_id[] = {
-	{ "adc141s626", TI_ADC141S626 },
-	{ "adc161s626", TI_ADC161S626 },
+	{ .name = "adc141s626", .driver_data = TI_ADC141S626 },
+	{ .name = "adc161s626", .driver_data = TI_ADC161S626 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ti_adc_id);
diff --git a/drivers/iio/adc/ti-ads1015.c b/drivers/iio/adc/ti-ads1015.c
index 8a272af69f7d2..0fbfa4e499aa8 100644
--- a/drivers/iio/adc/ti-ads1015.c
+++ b/drivers/iio/adc/ti-ads1015.c
@@ -930,6 +930,17 @@ static int ads1015_set_conv_mode(struct ads1015_data *data, int mode)
 				  mode << ADS1015_CFG_MOD_SHIFT);
 }
 
+static void ads1015_power_down(void *p)
+{
+	struct ads1015_data *data = p;
+	int ret;
+
+	ret = ads1015_set_conv_mode(data, ADS1015_SINGLESHOT);
+	if (ret)
+		dev_warn(regmap_get_device(data->regmap),
+			 "Failed to power down (%pe)\n", ERR_PTR(ret));
+}
+
 static int ads1015_probe(struct i2c_client *client)
 {
 	const struct ads1015_chip_data *chip;
@@ -1030,6 +1041,10 @@ static int ads1015_probe(struct i2c_client *client)
 	if (ret)
 		return ret;
 
+	ret = devm_add_action_or_reset(&client->dev, ads1015_power_down, data);
+	if (ret)
+		return ret;
+
 	data->conv_invalid = true;
 
 	ret = pm_runtime_set_active(&client->dev);
@@ -1037,33 +1052,11 @@ static int ads1015_probe(struct i2c_client *client)
 		return ret;
 	pm_runtime_set_autosuspend_delay(&client->dev, ADS1015_SLEEP_DELAY_MS);
 	pm_runtime_use_autosuspend(&client->dev);
-	pm_runtime_enable(&client->dev);
-
-	ret = iio_device_register(indio_dev);
-	if (ret < 0) {
-		dev_err(&client->dev, "Failed to register IIO device\n");
+	ret = devm_pm_runtime_enable(&client->dev);
+	if (ret)
 		return ret;
-	}
-
-	return 0;
-}
 
-static void ads1015_remove(struct i2c_client *client)
-{
-	struct iio_dev *indio_dev = i2c_get_clientdata(client);
-	struct ads1015_data *data = iio_priv(indio_dev);
-	int ret;
-
-	iio_device_unregister(indio_dev);
-
-	pm_runtime_disable(&client->dev);
-	pm_runtime_set_suspended(&client->dev);
-
-	/* power down single shot mode */
-	ret = ads1015_set_conv_mode(data, ADS1015_SINGLESHOT);
-	if (ret)
-		dev_warn(&client->dev, "Failed to power down (%pe)\n",
-			 ERR_PTR(ret));
+	return devm_iio_device_register(&client->dev, indio_dev);
 }
 
 #ifdef CONFIG_PM
@@ -1150,7 +1143,6 @@ static struct i2c_driver ads1015_driver = {
 		.pm = &ads1015_pm_ops,
 	},
 	.probe		= ads1015_probe,
-	.remove		= ads1015_remove,
 	.id_table	= ads1015_id,
 };
 
diff --git a/drivers/iio/adc/ti-ads1018.c b/drivers/iio/adc/ti-ads1018.c
index d6624c71a3744..d9ec4a8d7ce54 100644
--- a/drivers/iio/adc/ti-ads1018.c
+++ b/drivers/iio/adc/ti-ads1018.c
@@ -717,8 +717,8 @@ static const struct of_device_id ads1018_of_match[] = {
 MODULE_DEVICE_TABLE(of, ads1018_of_match);
 
 static const struct spi_device_id ads1018_spi_match[] = {
-	{ "ads1018", (kernel_ulong_t)&ads1018_chip_info },
-	{ "ads1118", (kernel_ulong_t)&ads1118_chip_info },
+	{ .name = "ads1018", .driver_data = (kernel_ulong_t)&ads1018_chip_info },
+	{ .name = "ads1118", .driver_data = (kernel_ulong_t)&ads1118_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ads1018_spi_match);
diff --git a/drivers/iio/adc/ti-ads124s08.c b/drivers/iio/adc/ti-ads124s08.c
index 45de477788090..49fbc63508acb 100644
--- a/drivers/iio/adc/ti-ads124s08.c
+++ b/drivers/iio/adc/ti-ads124s08.c
@@ -348,8 +348,8 @@ static int ads124s_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ads124s_id[] = {
-	{ "ads124s06", ADS124S06_ID },
-	{ "ads124s08", ADS124S08_ID },
+	{ .name = "ads124s06", .driver_data = ADS124S06_ID },
+	{ .name = "ads124s08", .driver_data = ADS124S08_ID },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ads124s_id);
diff --git a/drivers/iio/adc/ti-ads1298.c b/drivers/iio/adc/ti-ads1298.c
index 9e47ace16acd0..06c6353aa1b2e 100644
--- a/drivers/iio/adc/ti-ads1298.c
+++ b/drivers/iio/adc/ti-ads1298.c
@@ -743,7 +743,7 @@ static int ads1298_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ads1298_id[] = {
-	{ "ads1298" },
+	{ .name = "ads1298" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ads1298_id);
diff --git a/drivers/iio/adc/ti-ads131e08.c b/drivers/iio/adc/ti-ads131e08.c
index a585621b0bc30..1d25f887cadb9 100644
--- a/drivers/iio/adc/ti-ads131e08.c
+++ b/drivers/iio/adc/ti-ads131e08.c
@@ -917,9 +917,9 @@ static const struct of_device_id ads131e08_of_match[] = {
 MODULE_DEVICE_TABLE(of, ads131e08_of_match);
 
 static const struct spi_device_id ads131e08_ids[] = {
-	{ "ads131e04", (kernel_ulong_t)&ads131e08_info_tbl[ads131e04] },
-	{ "ads131e06", (kernel_ulong_t)&ads131e08_info_tbl[ads131e06] },
-	{ "ads131e08", (kernel_ulong_t)&ads131e08_info_tbl[ads131e08] },
+	{ .name = "ads131e04", .driver_data = (kernel_ulong_t)&ads131e08_info_tbl[ads131e04] },
+	{ .name = "ads131e06", .driver_data = (kernel_ulong_t)&ads131e08_info_tbl[ads131e06] },
+	{ .name = "ads131e08", .driver_data = (kernel_ulong_t)&ads131e08_info_tbl[ads131e08] },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ads131e08_ids);
diff --git a/drivers/iio/adc/ti-ads131m02.c b/drivers/iio/adc/ti-ads131m02.c
index 36203ce37c89b..2f8f75c8216b2 100644
--- a/drivers/iio/adc/ti-ads131m02.c
+++ b/drivers/iio/adc/ti-ads131m02.c
@@ -943,11 +943,11 @@ static const struct of_device_id ads131m_of_match[] = {
 MODULE_DEVICE_TABLE(of, ads131m_of_match);
 
 static const struct spi_device_id ads131m_id[] = {
-	{ "ads131m02", (kernel_ulong_t)&ads131m02_config },
-	{ "ads131m03", (kernel_ulong_t)&ads131m03_config },
-	{ "ads131m04", (kernel_ulong_t)&ads131m04_config },
-	{ "ads131m06", (kernel_ulong_t)&ads131m06_config },
-	{ "ads131m08", (kernel_ulong_t)&ads131m08_config },
+	{ .name = "ads131m02", .driver_data = (kernel_ulong_t)&ads131m02_config },
+	{ .name = "ads131m03", .driver_data = (kernel_ulong_t)&ads131m03_config },
+	{ .name = "ads131m04", .driver_data = (kernel_ulong_t)&ads131m04_config },
+	{ .name = "ads131m06", .driver_data = (kernel_ulong_t)&ads131m06_config },
+	{ .name = "ads131m08", .driver_data = (kernel_ulong_t)&ads131m08_config },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ads131m_id);
diff --git a/drivers/iio/adc/ti-ads7950.c b/drivers/iio/adc/ti-ads7950.c
index 39a074bce6d55..d27c1f2885fd6 100644
--- a/drivers/iio/adc/ti-ads7950.c
+++ b/drivers/iio/adc/ti-ads7950.c
@@ -619,18 +619,18 @@ static int ti_ads7950_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ti_ads7950_id[] = {
-	{ "ads7950", (kernel_ulong_t)&ti_ads7950_chip_info },
-	{ "ads7951", (kernel_ulong_t)&ti_ads7951_chip_info },
-	{ "ads7952", (kernel_ulong_t)&ti_ads7952_chip_info },
-	{ "ads7953", (kernel_ulong_t)&ti_ads7953_chip_info },
-	{ "ads7954", (kernel_ulong_t)&ti_ads7954_chip_info },
-	{ "ads7955", (kernel_ulong_t)&ti_ads7955_chip_info },
-	{ "ads7956", (kernel_ulong_t)&ti_ads7956_chip_info },
-	{ "ads7957", (kernel_ulong_t)&ti_ads7957_chip_info },
-	{ "ads7958", (kernel_ulong_t)&ti_ads7958_chip_info },
-	{ "ads7959", (kernel_ulong_t)&ti_ads7959_chip_info },
-	{ "ads7960", (kernel_ulong_t)&ti_ads7960_chip_info },
-	{ "ads7961", (kernel_ulong_t)&ti_ads7961_chip_info },
+	{ .name = "ads7950", .driver_data = (kernel_ulong_t)&ti_ads7950_chip_info },
+	{ .name = "ads7951", .driver_data = (kernel_ulong_t)&ti_ads7951_chip_info },
+	{ .name = "ads7952", .driver_data = (kernel_ulong_t)&ti_ads7952_chip_info },
+	{ .name = "ads7953", .driver_data = (kernel_ulong_t)&ti_ads7953_chip_info },
+	{ .name = "ads7954", .driver_data = (kernel_ulong_t)&ti_ads7954_chip_info },
+	{ .name = "ads7955", .driver_data = (kernel_ulong_t)&ti_ads7955_chip_info },
+	{ .name = "ads7956", .driver_data = (kernel_ulong_t)&ti_ads7956_chip_info },
+	{ .name = "ads7957", .driver_data = (kernel_ulong_t)&ti_ads7957_chip_info },
+	{ .name = "ads7958", .driver_data = (kernel_ulong_t)&ti_ads7958_chip_info },
+	{ .name = "ads7959", .driver_data = (kernel_ulong_t)&ti_ads7959_chip_info },
+	{ .name = "ads7960", .driver_data = (kernel_ulong_t)&ti_ads7960_chip_info },
+	{ .name = "ads7961", .driver_data = (kernel_ulong_t)&ti_ads7961_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ti_ads7950_id);
diff --git a/drivers/iio/adc/ti-ads8688.c b/drivers/iio/adc/ti-ads8688.c
index ba5e240aa41a7..8ad070b5ec4ca 100644
--- a/drivers/iio/adc/ti-ads8688.c
+++ b/drivers/iio/adc/ti-ads8688.c
@@ -453,8 +453,8 @@ static int ads8688_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ads8688_id[] = {
-	{ "ads8684", ID_ADS8684 },
-	{ "ads8688", ID_ADS8688 },
+	{ .name = "ads8684", .driver_data = ID_ADS8684 },
+	{ .name = "ads8688", .driver_data = ID_ADS8688 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ads8688_id);
diff --git a/drivers/iio/adc/ti-lmp92064.c b/drivers/iio/adc/ti-lmp92064.c
index 7e57006a8a126..d248bbea0d318 100644
--- a/drivers/iio/adc/ti-lmp92064.c
+++ b/drivers/iio/adc/ti-lmp92064.c
@@ -357,7 +357,7 @@ static int lmp92064_adc_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id lmp92064_id_table[] = {
-	{ "lmp92064" },
+	{ .name = "lmp92064" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, lmp92064_id_table);
diff --git a/drivers/iio/adc/ti-tlc4541.c b/drivers/iio/adc/ti-tlc4541.c
index 94bbf5afe30e6..ea776f80de721 100644
--- a/drivers/iio/adc/ti-tlc4541.c
+++ b/drivers/iio/adc/ti-tlc4541.c
@@ -240,8 +240,8 @@ static const struct of_device_id tlc4541_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, tlc4541_dt_ids);
 
 static const struct spi_device_id tlc4541_id[] = {
-	{ "tlc3541", TLC3541 },
-	{ "tlc4541", TLC4541 },
+	{ .name = "tlc3541", .driver_data = TLC3541 },
+	{ .name = "tlc4541", .driver_data = TLC4541 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, tlc4541_id);
diff --git a/drivers/iio/adc/ti-tsc2046.c b/drivers/iio/adc/ti-tsc2046.c
index aba4b10a17acc..125f6d7e2d84a 100644
--- a/drivers/iio/adc/ti-tsc2046.c
+++ b/drivers/iio/adc/ti-tsc2046.c
@@ -121,11 +121,6 @@ struct tsc2046_adc_group_layout {
 	unsigned int skip;
 };
 
-struct tsc2046_adc_dcfg {
-	const struct iio_chan_spec *channels;
-	unsigned int num_channels;
-};
-
 struct tsc2046_adc_ch_cfg {
 	unsigned int settling_time_us;
 	unsigned int oversampling_ratio;
@@ -141,7 +136,6 @@ enum tsc2046_state {
 
 struct tsc2046_adc_priv {
 	struct spi_device *spi;
-	const struct tsc2046_adc_dcfg *dcfg;
 	bool internal_vref;
 
 	struct iio_trigger *trig;
@@ -214,11 +208,6 @@ const struct iio_chan_spec name ## _channels[] = { \
 
 static DECLARE_TI_TSC2046_8_CHANNELS(tsc2046_adc, 12);
 
-static const struct tsc2046_adc_dcfg tsc2046_adc_dcfg_tsc2046e = {
-	.channels = tsc2046_adc_channels,
-	.num_channels = ARRAY_SIZE(tsc2046_adc_channels),
-};
-
 /*
  * Convert time to a number of samples which can be transferred within this
  * time.
@@ -739,7 +728,6 @@ static void tsc2046_adc_parse_fwnode(struct tsc2046_adc_priv *priv)
 
 static int tsc2046_adc_probe(struct spi_device *spi)
 {
-	const struct tsc2046_adc_dcfg *dcfg;
 	struct device *dev = &spi->dev;
 	struct tsc2046_adc_priv *priv;
 	struct iio_dev *indio_dev;
@@ -752,10 +740,6 @@ static int tsc2046_adc_probe(struct spi_device *spi)
 		return -EINVAL;
 	}
 
-	dcfg = spi_get_device_match_data(spi);
-	if (!dcfg)
-		return -EINVAL;
-
 	spi->mode &= ~SPI_MODE_X_MASK;
 	spi->mode |= SPI_MODE_0;
 	ret = spi_setup(spi);
@@ -767,14 +751,13 @@ static int tsc2046_adc_probe(struct spi_device *spi)
 		return -ENOMEM;
 
 	priv = iio_priv(indio_dev);
-	priv->dcfg = dcfg;
 
 	priv->spi = spi;
 
 	indio_dev->name = TI_TSC2046_NAME;
 	indio_dev->modes = INDIO_DIRECT_MODE;
-	indio_dev->channels = dcfg->channels;
-	indio_dev->num_channels = dcfg->num_channels;
+	indio_dev->channels = tsc2046_adc_channels;
+	indio_dev->num_channels = ARRAY_SIZE(tsc2046_adc_channels);
 	indio_dev->info = &tsc2046_adc_info;
 
 	ret = devm_regulator_get_enable_read_voltage(dev, "vref");
@@ -829,13 +812,13 @@ static int tsc2046_adc_probe(struct spi_device *spi)
 }
 
 static const struct of_device_id ads7950_of_table[] = {
-	{ .compatible = "ti,tsc2046e-adc", .data = &tsc2046_adc_dcfg_tsc2046e },
+	{ .compatible = "ti,tsc2046e-adc" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, ads7950_of_table);
 
 static const struct spi_device_id tsc2046_adc_spi_ids[] = {
-	{ "tsc2046e-adc", (unsigned long)&tsc2046_adc_dcfg_tsc2046e },
+	{ .name = "tsc2046e-adc" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, tsc2046_adc_spi_ids);
diff --git a/drivers/iio/adc/versal-sysmon-core.c b/drivers/iio/adc/versal-sysmon-core.c
new file mode 100644
index 0000000000000..1b55d343982ef
--- /dev/null
+++ b/drivers/iio/adc/versal-sysmon-core.c
@@ -0,0 +1,1052 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * AMD Versal SysMon core driver
+ *
+ * Copyright (C) 2019 - 2022, Xilinx, Inc.
+ * Copyright (C) 2022 - 2026, Advanced Micro Devices, Inc.
+ */
+
+#include <linux/array_size.h>
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/cleanup.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/limits.h>
+#include <linux/minmax.h>
+#include <linux/module.h>
+#include <linux/overflow.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/string.h>
+#include <linux/sysfs.h>
+#include <linux/units.h>
+
+#include <linux/iio/events.h>
+#include <linux/iio/iio.h>
+
+#include "versal-sysmon.h"
+
+/*
+ * Oversampling ratio values exposed to userspace via IIO.
+ * Actual number of samples averaged: 1=none, 2=2x, 4=4x, 8=8x, 16=16x.
+ */
+static const int sysmon_oversampling_avail[] = { 1, 2, 4, 8, 16 };
+
+/* TEMP hysteresis mode bit in SYSMON_TEMP_EV_CFG */
+#define SYSMON_TEMP_HYST_MASK		BIT(1)
+
+/* Compute alarm register offset from a channel address */
+#define SYSMON_ALARM_OFFSET(addr) \
+	(SYSMON_ALARM_REG + ((addr) / SYSMON_ALARM_BITS_PER_REG) * SYSMON_REG_STRIDE)
+
+#define SYSMON_CHAN_TEMP(_chan, _address, _name)		\
+{								\
+	.type = IIO_TEMP,					\
+	.indexed = 1,						\
+	.address = _address,					\
+	.channel = _chan,					\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),		\
+	.info_mask_shared_by_type =				\
+		BIT(IIO_CHAN_INFO_SCALE) |			\
+		BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),		\
+	.info_mask_shared_by_type_available =			\
+		BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),		\
+	.datasheet_name = _name,				\
+}
+
+enum sysmon_alarm_bit {
+	SYSMON_BIT_ALARM0 = 0,
+	SYSMON_BIT_ALARM1 = 1,
+	SYSMON_BIT_ALARM2 = 2,
+	SYSMON_BIT_ALARM3 = 3,
+	SYSMON_BIT_ALARM4 = 4,
+	SYSMON_BIT_TEMP = 9,
+};
+
+/* Temperature event specification: rising threshold + hysteresis only */
+static const struct iio_event_spec sysmon_temp_events[] = {
+	{
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_RISING,
+		.mask_separate = BIT(IIO_EV_INFO_ENABLE) |
+				 BIT(IIO_EV_INFO_VALUE) |
+				 BIT(IIO_EV_INFO_HYSTERESIS),
+	},
+};
+
+/* Supply event specifications */
+static const struct iio_event_spec sysmon_supply_events[] = {
+	{
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_RISING,
+		.mask_separate = BIT(IIO_EV_INFO_VALUE),
+	},
+	{
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_FALLING,
+		.mask_separate = BIT(IIO_EV_INFO_VALUE),
+	},
+	{
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_EITHER,
+		.mask_separate = BIT(IIO_EV_INFO_ENABLE),
+	},
+};
+
+/*
+ * Static temperature channels (always present).
+ *
+ * These are hardware-computed aggregate registers across all active
+ * temperature satellites:
+ *   temp:     current max temperature across all active satellites
+ *   min:      current min temperature across all active satellites
+ *   max_max:  highest peak recorded since last hardware reset
+ *   min_min:  lowest trough recorded since last hardware reset
+ */
+static const struct iio_chan_spec temp_channels[] = {
+	SYSMON_CHAN_TEMP(0, SYSMON_TEMP_MAX, "temp"),
+	SYSMON_CHAN_TEMP(1, SYSMON_TEMP_MIN, "min"),
+	SYSMON_CHAN_TEMP(2, SYSMON_TEMP_MAX_MAX, "max_max"),
+	SYSMON_CHAN_TEMP(3, SYSMON_TEMP_MIN_MIN, "min_min"),
+};
+
+static void sysmon_q8p7_to_millicelsius(s16 raw_data, int *val)
+{
+	*val = (raw_data * MILLIDEGREE_PER_DEGREE) >> SYSMON_FRACTIONAL_SHIFT;
+}
+
+static void sysmon_millicelsius_to_q8p7(u32 *raw_data, int val)
+{
+	*raw_data = (val << SYSMON_FRACTIONAL_SHIFT) / MILLIDEGREE_PER_DEGREE;
+}
+
+static void sysmon_supply_rawtoprocessed(int raw_data, int *val)
+{
+	int mantissa, format, exponent;
+
+	mantissa = FIELD_GET(SYSMON_MANTISSA_MASK, raw_data);
+	exponent = SYSMON_SUPPLY_MANTISSA_BITS - FIELD_GET(SYSMON_MODE_MASK, raw_data);
+	format = FIELD_GET(SYSMON_FMT_MASK, raw_data);
+	/*
+	 * When format bit is set the mantissa is two's complement
+	 * (per hardware spec); sign-extend to int for correct arithmetic.
+	 */
+	if (format)
+		mantissa = sign_extend32(mantissa, 15);
+
+	*val = (mantissa * (int)MILLI) >> exponent;
+}
+
+static void sysmon_supply_processedtoraw(int val, u32 reg_val, u32 *raw_data)
+{
+	int exponent = FIELD_GET(SYSMON_MODE_MASK, reg_val);
+	int format = FIELD_GET(SYSMON_FMT_MASK, reg_val);
+	int scale, tmp;
+
+	scale = BIT(SYSMON_SUPPLY_MANTISSA_BITS - exponent);
+	tmp = (val * scale) / (int)MILLI;
+
+	if (format)
+		tmp = clamp(tmp, S16_MIN, S16_MAX);
+	else
+		tmp = clamp(tmp, 0, U16_MAX);
+
+	*raw_data = (u16)tmp;
+}
+
+static int sysmon_supply_thresh_offset(unsigned long address, enum iio_event_direction dir)
+{
+	if (dir == IIO_EV_DIR_RISING)
+		return (address * SYSMON_REG_STRIDE) + SYSMON_SUPPLY_TH_UP;
+	if (dir == IIO_EV_DIR_FALLING)
+		return (address * SYSMON_REG_STRIDE) + SYSMON_SUPPLY_TH_LOW;
+
+	return -EINVAL;
+}
+
+static int sysmon_read_raw(struct iio_dev *indio_dev,
+			   struct iio_chan_spec const *chan,
+			   int *val, int *val2, long mask)
+{
+	struct sysmon *sysmon = iio_priv(indio_dev);
+	unsigned int regval;
+	int ret;
+
+	guard(mutex)(&sysmon->lock);
+
+	if (mask == IIO_CHAN_INFO_OVERSAMPLING_RATIO) {
+		*val = (chan->type == IIO_TEMP) ? sysmon->temp_oversampling :
+						 sysmon->supply_oversampling;
+		return IIO_VAL_INT;
+	}
+
+	switch (chan->type) {
+	case IIO_TEMP:
+		if (mask == IIO_CHAN_INFO_SCALE) {
+			/* Q8.7 to millicelsius: raw * 1000 / 128 */
+			*val = MILLIDEGREE_PER_DEGREE;
+			*val2 = BIT(SYSMON_FRACTIONAL_SHIFT);
+			return IIO_VAL_FRACTIONAL;
+		}
+		if (mask != IIO_CHAN_INFO_RAW)
+			return -EINVAL;
+
+		ret = regmap_read(sysmon->regmap, chan->address, &regval);
+		if (ret)
+			return ret;
+
+		*val = sign_extend32(regval, 15);
+		return IIO_VAL_INT;
+
+	case IIO_VOLTAGE:
+		if (mask != IIO_CHAN_INFO_PROCESSED)
+			return -EINVAL;
+
+		ret = regmap_read(sysmon->regmap,
+				  chan->address * SYSMON_REG_STRIDE +
+				  SYSMON_SUPPLY_BASE, &regval);
+		if (ret)
+			return ret;
+
+		sysmon_supply_rawtoprocessed(regval, val);
+		return IIO_VAL_INT;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static u32 sysmon_get_event_mask(const struct iio_chan_spec *chan)
+{
+	if (chan->type == IIO_TEMP)
+		return BIT(SYSMON_BIT_TEMP);
+
+	return BIT(chan->address / SYSMON_ALARM_BITS_PER_REG);
+}
+
+static int sysmon_read_alarm_config(struct sysmon *sysmon,
+				    unsigned long address)
+{
+	u32 shift = address % SYSMON_ALARM_BITS_PER_REG;
+	u32 offset = SYSMON_ALARM_OFFSET(address);
+
+	return regmap_test_bits(sysmon->regmap, offset, BIT(shift));
+}
+
+static int sysmon_write_alarm_config(struct sysmon *sysmon,
+				     unsigned long address, bool enable)
+{
+	u32 shift = address % SYSMON_ALARM_BITS_PER_REG;
+	u32 offset = SYSMON_ALARM_OFFSET(address);
+
+	return regmap_assign_bits(sysmon->regmap, offset, BIT(shift), enable);
+}
+
+static int sysmon_read_event_config(struct iio_dev *indio_dev,
+				    const struct iio_chan_spec *chan,
+				    enum iio_event_type type,
+				    enum iio_event_direction dir)
+{
+	struct sysmon *sysmon = iio_priv(indio_dev);
+	u32 mask = sysmon_get_event_mask(chan);
+	unsigned int imr;
+	int config_value;
+	int ret;
+
+	ret = regmap_read(sysmon->regmap, SYSMON_IMR, &imr);
+	if (ret)
+		return ret;
+
+	/* IMR bits are 1=masked, invert to get 1=enabled */
+	imr = ~imr;
+
+	switch (chan->type) {
+	case IIO_VOLTAGE:
+		config_value = sysmon_read_alarm_config(sysmon, chan->address);
+		if (config_value < 0)
+			return config_value;
+		return config_value && (imr & mask);
+
+	case IIO_TEMP:
+		/*
+		 * Return the administrative state, not the hardware IMR.
+		 * The IRQ handler temporarily masks the interrupt during
+		 * the polling window; reading IMR would show it as disabled.
+		 * temp_mask bit is set when administratively disabled.
+		 */
+		return !(sysmon->temp_mask & mask);
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int sysmon_write_event_config(struct iio_dev *indio_dev,
+				     const struct iio_chan_spec *chan,
+				     enum iio_event_type type,
+				     enum iio_event_direction dir,
+				     bool state)
+{
+	u32 offset = SYSMON_ALARM_OFFSET(chan->address);
+	struct sysmon *sysmon = iio_priv(indio_dev);
+	u32 mask = sysmon_get_event_mask(chan);
+	unsigned int alarm_config;
+	int ret;
+
+	guard(mutex)(&sysmon->lock);
+
+	switch (chan->type) {
+	case IIO_VOLTAGE:
+		ret = sysmon_write_alarm_config(sysmon, chan->address, state);
+		if (ret)
+			return ret;
+
+		ret = regmap_read(sysmon->regmap, offset, &alarm_config);
+		if (ret)
+			return ret;
+
+		if (alarm_config)
+			return regmap_write(sysmon->regmap, SYSMON_IER, mask);
+
+		return regmap_write(sysmon->regmap, SYSMON_IDR, mask);
+
+	case IIO_TEMP:
+		if (state) {
+			ret = regmap_write(sysmon->regmap, SYSMON_IER, mask);
+			if (ret)
+				return ret;
+
+			scoped_guard(spinlock_irq, &sysmon->irq_lock)
+				sysmon->temp_mask &= ~mask;
+		} else {
+			ret = regmap_write(sysmon->regmap, SYSMON_IDR, mask);
+			if (ret)
+				return ret;
+
+			scoped_guard(spinlock_irq, &sysmon->irq_lock)
+				sysmon->temp_mask |= mask;
+		}
+		return 0;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+/*
+ * Recompute the lower threshold register from upper threshold and
+ * cached hysteresis. Called when either upper threshold or hysteresis
+ * is written.
+ */
+static int sysmon_update_temp_lower(struct sysmon *sysmon)
+{
+	unsigned int upper_reg;
+	int upper_mc, lower_mc;
+	u32 raw_val;
+	int ret;
+
+	ret = regmap_read(sysmon->regmap, SYSMON_TEMP_TH_UP, &upper_reg);
+	if (ret)
+		return ret;
+
+	sysmon_q8p7_to_millicelsius(upper_reg, &upper_mc);
+	lower_mc = clamp(upper_mc - sysmon->temp_hysteresis, -256000, 255992);
+	sysmon_millicelsius_to_q8p7(&raw_val, lower_mc);
+
+	return regmap_write(sysmon->regmap, SYSMON_TEMP_TH_LOW, raw_val);
+}
+
+static int sysmon_read_event_value(struct iio_dev *indio_dev,
+				   const struct iio_chan_spec *chan,
+				   enum iio_event_type type,
+				   enum iio_event_direction dir,
+				   enum iio_event_info info,
+				   int *val, int *val2)
+{
+	struct sysmon *sysmon = iio_priv(indio_dev);
+	unsigned int reg_val;
+	int offset;
+	int ret;
+
+	guard(mutex)(&sysmon->lock);
+
+	switch (chan->type) {
+	case IIO_TEMP:
+		switch (info) {
+		case IIO_EV_INFO_VALUE:
+			ret = regmap_read(sysmon->regmap, SYSMON_TEMP_TH_UP, &reg_val);
+			if (ret)
+				return ret;
+
+			sysmon_q8p7_to_millicelsius(reg_val, val);
+
+			return IIO_VAL_INT;
+
+		case IIO_EV_INFO_HYSTERESIS:
+			*val = sysmon->temp_hysteresis;
+			return IIO_VAL_INT;
+
+		default:
+			return -EINVAL;
+		}
+
+	case IIO_VOLTAGE:
+		offset = sysmon_supply_thresh_offset(chan->address, dir);
+		if (offset < 0)
+			return offset;
+
+		ret = regmap_read(sysmon->regmap, offset, &reg_val);
+		if (ret)
+			return ret;
+
+		sysmon_supply_rawtoprocessed(reg_val, val);
+
+		return IIO_VAL_INT;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int sysmon_write_event_value(struct iio_dev *indio_dev,
+				    const struct iio_chan_spec *chan,
+				    enum iio_event_type type,
+				    enum iio_event_direction dir,
+				    enum iio_event_info info,
+				    int val, int val2)
+{
+	struct sysmon *sysmon = iio_priv(indio_dev);
+	unsigned int reg_val;
+	u32 raw_val;
+	int offset;
+	int ret;
+
+	guard(mutex)(&sysmon->lock);
+
+	switch (chan->type) {
+	case IIO_TEMP:
+		switch (info) {
+		case IIO_EV_INFO_VALUE:
+			/* Q8.7 signed range: -256000 to +255992 mC */
+			if (val < -256000 || val > 255992)
+				return -EINVAL;
+
+			sysmon_millicelsius_to_q8p7(&raw_val, val);
+
+			ret = regmap_write(sysmon->regmap, SYSMON_TEMP_TH_UP, raw_val);
+			if (ret)
+				return ret;
+
+			/* Recompute lower = upper - hysteresis */
+			return sysmon_update_temp_lower(sysmon);
+
+		case IIO_EV_INFO_HYSTERESIS:
+			if (val < 0)
+				return -EINVAL;
+
+			sysmon->temp_hysteresis = val;
+
+			return sysmon_update_temp_lower(sysmon);
+
+		default:
+			return -EINVAL;
+		}
+
+	case IIO_VOLTAGE:
+		offset = sysmon_supply_thresh_offset(chan->address, dir);
+		if (offset < 0)
+			return offset;
+
+		ret = regmap_read(sysmon->regmap, offset, &reg_val);
+		if (ret)
+			return ret;
+
+		/* Clamp to prevent overflow in processedtoraw conversion */
+		if (val < -32768 || val > 32767)
+			return -EINVAL;
+
+		sysmon_supply_processedtoraw(val, reg_val, &raw_val);
+
+		/*
+		 * The hardware threshold register returns FMT and MODE
+		 * bits in the upper 16 bits on read, but only the lower
+		 * 16-bit mantissa is used on write.
+		 */
+		return regmap_write(sysmon->regmap, offset, raw_val);
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int sysmon_set_avg_enable(struct sysmon *sysmon,
+				 u32 base, u32 count, u32 val)
+{
+	struct regmap *map = sysmon->regmap;
+	int ret;
+
+	for (unsigned int i = 0; i < count; i++) {
+		ret = regmap_write(map, base + i * SYSMON_REG_STRIDE, val);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int sysmon_osr_write_temp(struct sysmon *sysmon, unsigned int val)
+{
+	/*
+	 * HW register encoding is sample_count / 2:
+	 * 0=none, 1=2x, 2=4x, 4=8x, 8=16x (not log2-based).
+	 */
+	unsigned int hw_val = val >> 1;
+	unsigned int readback;
+	int ret;
+
+	ret = regmap_update_bits(sysmon->regmap, SYSMON_CONFIG,
+				SYSMON_CONFIG_TEMP_SAT_OSR,
+				FIELD_PREP(SYSMON_CONFIG_TEMP_SAT_OSR, hw_val));
+	if (ret)
+		return ret;
+
+	/*
+	 * Readback fence: the SysMon CONFIG register resides in the
+	 * PMC domain behind the NoC. A posted write may not reach the
+	 * hardware before the next MMIO access. Reading the register
+	 * back forces the interconnect to complete the write, preventing
+	 * a bus hang on the subsequent access.
+	 */
+	regmap_read(sysmon->regmap, SYSMON_CONFIG, &readback);
+
+	return sysmon_set_avg_enable(sysmon, SYSMON_TEMP_EN_AVG_BASE,
+				     SYSMON_TEMP_EN_AVG_COUNT,
+				     hw_val ? ~0 : 0);
+}
+
+static int sysmon_osr_write_supply(struct sysmon *sysmon, unsigned int val)
+{
+	/* HW encoding: sample_count / 2 (see sysmon_osr_write_temp) */
+	unsigned int hw_val = val >> 1;
+	unsigned int readback;
+	int ret;
+
+	ret = regmap_update_bits(sysmon->regmap, SYSMON_CONFIG,
+				SYSMON_CONFIG_SUPPLY_OSR,
+				FIELD_PREP(SYSMON_CONFIG_SUPPLY_OSR, hw_val));
+	if (ret)
+		return ret;
+
+	/* Readback fence -- see sysmon_osr_write_temp for details */
+	regmap_read(sysmon->regmap, SYSMON_CONFIG, &readback);
+
+	return sysmon_set_avg_enable(sysmon, SYSMON_SUPPLY_EN_AVG_BASE,
+				     SYSMON_SUPPLY_EN_AVG_COUNT,
+				     hw_val ? ~0 : 0);
+}
+
+static int sysmon_write_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int val, int val2, long mask)
+{
+	struct sysmon *sysmon = iio_priv(indio_dev);
+	unsigned int i;
+	int ret;
+
+	if (mask != IIO_CHAN_INFO_OVERSAMPLING_RATIO)
+		return -EINVAL;
+
+	for (i = 0; i < ARRAY_SIZE(sysmon_oversampling_avail); i++) {
+		if (val == sysmon_oversampling_avail[i])
+			break;
+	}
+	if (i == ARRAY_SIZE(sysmon_oversampling_avail))
+		return -EINVAL;
+
+	guard(mutex)(&sysmon->lock);
+
+	if (chan->type == IIO_TEMP) {
+		ret = sysmon_osr_write_temp(sysmon, val);
+		if (ret)
+			return ret;
+		sysmon->temp_oversampling = val;
+	} else {
+		ret = sysmon_osr_write_supply(sysmon, val);
+		if (ret)
+			return ret;
+		sysmon->supply_oversampling = val;
+	}
+
+	return 0;
+}
+
+static int sysmon_write_raw_get_fmt(struct iio_dev *indio_dev,
+				    struct iio_chan_spec const *chan,
+				    long mask)
+{
+	if (mask == IIO_CHAN_INFO_OVERSAMPLING_RATIO)
+		return IIO_VAL_INT;
+
+	return -EINVAL;
+}
+
+static int sysmon_read_avail(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     const int **vals, int *type,
+			     int *length, long mask)
+{
+	if (mask != IIO_CHAN_INFO_OVERSAMPLING_RATIO)
+		return -EINVAL;
+
+	*vals = sysmon_oversampling_avail;
+	*type = IIO_VAL_INT;
+	*length = ARRAY_SIZE(sysmon_oversampling_avail);
+
+	return IIO_AVAIL_LIST;
+}
+
+static int sysmon_read_label(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     char *label)
+{
+	if (chan->datasheet_name)
+		return sysfs_emit(label, "%s\n", chan->datasheet_name);
+
+	return -EINVAL;
+}
+
+static const struct iio_info sysmon_iio_info = {
+	.read_raw = sysmon_read_raw,
+	.write_raw = sysmon_write_raw,
+	.write_raw_get_fmt = sysmon_write_raw_get_fmt,
+	.read_avail = sysmon_read_avail,
+	.read_label = sysmon_read_label,
+	.read_event_config = sysmon_read_event_config,
+	.write_event_config = sysmon_write_event_config,
+	.read_event_value = sysmon_read_event_value,
+	.write_event_value = sysmon_write_event_value,
+};
+
+static void sysmon_push_event(struct iio_dev *indio_dev, u32 address)
+{
+	const struct iio_chan_spec *chan;
+	enum iio_event_direction dir;
+
+	for (unsigned int i = 0; i < indio_dev->num_channels; i++) {
+		if (indio_dev->channels[i].address != address)
+			continue;
+
+		chan = &indio_dev->channels[i];
+		/* Temp uses hysteresis mode (rising only), voltage uses window */
+		dir = (chan->type == IIO_TEMP) ? IIO_EV_DIR_RISING :
+						 IIO_EV_DIR_EITHER;
+		iio_push_event(indio_dev,
+			       IIO_UNMOD_EVENT_CODE(chan->type,
+						    chan->channel,
+						    IIO_EV_TYPE_THRESH,
+						    dir),
+			       iio_get_time_ns(indio_dev));
+	}
+}
+
+static int sysmon_handle_event(struct iio_dev *indio_dev, u32 event)
+{
+	u32 alarm_flag_offset = SYSMON_ALARM_FLAG + event * SYSMON_REG_STRIDE;
+	u32 alarm_reg_offset = SYSMON_ALARM_REG + event * SYSMON_REG_STRIDE;
+	struct sysmon *sysmon = iio_priv(indio_dev);
+	unsigned long alarm_flag_reg;
+	unsigned int reg_val;
+	u32 address, bit;
+	int ret;
+
+	switch (event) {
+	case SYSMON_BIT_TEMP:
+		sysmon_push_event(indio_dev, SYSMON_TEMP_MAX);
+
+		ret = regmap_write(sysmon->regmap, SYSMON_IDR, BIT(SYSMON_BIT_TEMP));
+		if (ret)
+			return ret;
+
+		sysmon->masked_temp |= BIT(SYSMON_BIT_TEMP);
+		return 0;
+
+	case SYSMON_BIT_ALARM0:
+	case SYSMON_BIT_ALARM1:
+	case SYSMON_BIT_ALARM2:
+	case SYSMON_BIT_ALARM3:
+	case SYSMON_BIT_ALARM4:
+		ret = regmap_read(sysmon->regmap, alarm_flag_offset, &reg_val);
+		if (ret)
+			return ret;
+
+		alarm_flag_reg = reg_val;
+
+		for_each_set_bit(bit, &alarm_flag_reg, SYSMON_ALARM_BITS_PER_REG) {
+			address = bit + SYSMON_ALARM_BITS_PER_REG * event;
+			sysmon_push_event(indio_dev, address);
+			ret = regmap_clear_bits(sysmon->regmap, alarm_reg_offset, BIT(bit));
+			if (ret)
+				return ret;
+		}
+
+		return regmap_write(sysmon->regmap, alarm_flag_offset, alarm_flag_reg);
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static void sysmon_handle_events(struct iio_dev *indio_dev,
+				 unsigned long events)
+{
+	unsigned int bit;
+
+	for_each_set_bit(bit, &events, SYSMON_NO_OF_EVENTS)
+		sysmon_handle_event(indio_dev, bit);
+}
+
+static void sysmon_unmask_temp(struct sysmon *sysmon, unsigned int isr)
+{
+	unsigned int status;
+	u32 ier;
+
+	status = isr & SYSMON_TEMP_INTR_MASK;
+
+	ier = ~status & sysmon->masked_temp;
+	sysmon->masked_temp &= status;
+
+	/* Only unmask if not administratively disabled by userspace */
+	ier &= ~sysmon->temp_mask;
+
+	regmap_write(sysmon->regmap, SYSMON_IER, ier);
+}
+
+/*
+ * Versal threshold interrupts are level-sensitive. Active threshold
+ * interrupts are masked in the handler and polled via delayed work
+ * until the condition clears, then unmasked.
+ */
+static void sysmon_unmask_worker(struct work_struct *work)
+{
+	struct sysmon *sysmon =
+		container_of(work, struct sysmon, sysmon_unmask_work.work);
+	unsigned int isr;
+
+	/*
+	 * If the ISR read fails, skip processing to avoid acting
+	 * on undefined data.
+	 */
+	scoped_guard(spinlock_irq, &sysmon->irq_lock) {
+		if (regmap_read(sysmon->regmap, SYSMON_ISR, &isr))
+			break;
+		regmap_write(sysmon->regmap, SYSMON_ISR, isr);
+		sysmon_unmask_temp(sysmon, isr);
+	}
+
+	if (sysmon->masked_temp)
+		schedule_delayed_work(&sysmon->sysmon_unmask_work,
+				      msecs_to_jiffies(SYSMON_UNMASK_WORK_DELAY_MS));
+	else
+		regmap_write(sysmon->regmap, SYSMON_STATUS_RESET, 1);
+}
+
+static irqreturn_t sysmon_iio_irq(int irq, void *data)
+{
+	struct iio_dev *indio_dev = data;
+	struct sysmon *sysmon = iio_priv(indio_dev);
+	unsigned int isr, imr;
+
+	guard(spinlock)(&sysmon->irq_lock);
+
+	if (regmap_read(sysmon->regmap, SYSMON_ISR, &isr) ||
+	    regmap_read(sysmon->regmap, SYSMON_IMR, &imr))
+		return IRQ_NONE;
+
+	isr &= ~imr;
+	if (!isr)
+		return IRQ_NONE;
+
+	regmap_write(sysmon->regmap, SYSMON_ISR, isr);
+
+	sysmon_handle_events(indio_dev, isr);
+	schedule_delayed_work(&sysmon->sysmon_unmask_work,
+			      msecs_to_jiffies(SYSMON_UNMASK_WORK_DELAY_MS));
+
+	return IRQ_HANDLED;
+}
+
+static void sysmon_disable_interrupts(void *data)
+{
+	struct sysmon *sysmon = data;
+
+	regmap_write(sysmon->regmap, SYSMON_IDR, SYSMON_INTR_ALL_MASK);
+
+	scoped_guard(spinlock_irq, &sysmon->irq_lock)
+		sysmon->masked_temp = 0;
+
+	cancel_delayed_work_sync(&sysmon->sysmon_unmask_work);
+}
+
+static int sysmon_init_interrupt(struct sysmon *sysmon,
+				 struct device *dev,
+				 struct iio_dev *indio_dev,
+				 int irq)
+{
+	unsigned int imr;
+	int ret;
+
+	/* Events not supported without IRQ (e.g. I2C path) */
+	if (!irq)
+		return 0;
+
+	INIT_DELAYED_WORK(&sysmon->sysmon_unmask_work, sysmon_unmask_worker);
+
+	ret = regmap_read(sysmon->regmap, SYSMON_IMR, &imr);
+	if (ret)
+		return ret;
+	sysmon->temp_mask = imr & SYSMON_TEMP_INTR_MASK;
+
+	ret = devm_request_irq(dev, irq, sysmon_iio_irq, 0, "sysmon-irq", indio_dev);
+	if (ret)
+		return ret;
+
+	return devm_add_action_or_reset(dev, sysmon_disable_interrupts, sysmon);
+}
+
+/*
+ * Initialize the cached hysteresis for a temperature channel from the
+ * current hardware threshold registers: hysteresis = upper - lower.
+ */
+static int sysmon_init_hysteresis(struct sysmon *sysmon, int *hysteresis)
+{
+	unsigned int upper_reg, lower_reg;
+	int upper_mc, lower_mc;
+	int ret;
+
+	ret = regmap_read(sysmon->regmap, SYSMON_TEMP_TH_UP, &upper_reg);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(sysmon->regmap, SYSMON_TEMP_TH_LOW, &lower_reg);
+	if (ret)
+		return ret;
+
+	sysmon_q8p7_to_millicelsius(upper_reg, &upper_mc);
+	sysmon_q8p7_to_millicelsius(lower_reg, &lower_mc);
+	*hysteresis = upper_mc - lower_mc;
+
+	return 0;
+}
+
+/**
+ * sysmon_parse_fw() - Parse firmware nodes and configure IIO channels.
+ * @indio_dev: IIO device instance
+ * @dev: Parent device
+ * @irq: IRQ number (positive enables event channels, 0 disables)
+ *
+ * Reads voltage-channels and temperature-channels container nodes from
+ * firmware and builds the IIO channel array. Static temperature channels
+ * and event channels are prepended, followed by supply and satellite
+ * channels from DT.
+ *
+ * Event channels and per-channel event specs are only added when the
+ * device has an IRQ. I2C devices have no interrupt line, and the I2C
+ * regmap cannot be called from atomic context, so events are not
+ * supported on that path.
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+static int sysmon_parse_fw(struct iio_dev *indio_dev, struct device *dev, int irq)
+{
+	unsigned int num_chan, num_static, num_supply, num_temp;
+	unsigned int idx, temp_chan_idx, volt_chan_idx;
+	struct iio_chan_spec *sysmon_channels;
+	const char *label;
+	u32 reg;
+	int ret;
+
+	struct fwnode_handle *supply_node __free(fwnode_handle) =
+		device_get_named_child_node(dev, "voltage-channels");
+	num_supply = fwnode_get_child_node_count(supply_node);
+
+	struct fwnode_handle *temp_node __free(fwnode_handle) =
+		device_get_named_child_node(dev, "temperature-channels");
+	num_temp = fwnode_get_child_node_count(temp_node);
+
+	num_static = ARRAY_SIZE(temp_channels);
+	num_chan = size_add(num_temp, size_add(num_static, num_supply));
+	sysmon_channels = devm_kcalloc(dev, num_chan, sizeof(*sysmon_channels), GFP_KERNEL);
+	if (!sysmon_channels)
+		return -ENOMEM;
+
+	memcpy(sysmon_channels, temp_channels, sizeof(temp_channels));
+
+	/* Attach event spec to channel 0 when IRQ is available */
+	if (irq > 0) {
+		sysmon_channels[0].event_spec = sysmon_temp_events;
+		sysmon_channels[0].num_event_specs = ARRAY_SIZE(sysmon_temp_events);
+	}
+
+	idx = num_static;
+
+	/* Supply channels from DT */
+	fwnode_for_each_child_node_scoped(supply_node, child) {
+		ret = fwnode_property_read_u32(child, "reg", &reg);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "missing reg for supply channel\n");
+
+		if (reg > SYSMON_SUPPLY_IDX_MAX)
+			return dev_err_probe(dev, -EINVAL,
+					     "supply reg %u exceeds max %u\n",
+					     reg, SYSMON_SUPPLY_IDX_MAX);
+
+		ret = fwnode_property_read_string(child, "label", &label);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "missing label for supply channel\n");
+
+		sysmon_channels[idx++] = (struct iio_chan_spec) {
+			.type = IIO_VOLTAGE,
+			.indexed = 1,
+			.address = reg,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED),
+			.info_mask_shared_by_type =
+				BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+			.info_mask_shared_by_type_available =
+				BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+			.event_spec = irq > 0 ?
+				sysmon_supply_events : NULL,
+			.num_event_specs = irq > 0 ?
+				ARRAY_SIZE(sysmon_supply_events) : 0,
+			.datasheet_name = label,
+		};
+	}
+
+	/* Temperature satellite channels from DT */
+	fwnode_for_each_child_node_scoped(temp_node, child) {
+		ret = fwnode_property_read_u32(child, "reg", &reg);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "missing reg for temp channel\n");
+
+		if (reg < 1 || reg > SYSMON_TEMP_SAT_MAX)
+			return dev_err_probe(dev, -EINVAL,
+					     "temp reg %u out of range [1..%u]\n",
+					     reg, SYSMON_TEMP_SAT_MAX);
+
+		ret = fwnode_property_read_string(child, "label", &label);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "missing label for temp channel\n");
+
+		sysmon_channels[idx++] = (struct iio_chan_spec) {
+			.type = IIO_TEMP,
+			.indexed = 1,
+			.address = SYSMON_TEMP_SAT_BASE +
+				   (reg - 1) * SYSMON_REG_STRIDE,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+			.info_mask_shared_by_type =
+				BIT(IIO_CHAN_INFO_SCALE) |
+				BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+			.info_mask_shared_by_type_available =
+				BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+			.datasheet_name = label,
+		};
+	}
+
+	indio_dev->num_channels = idx;
+	indio_dev->info = &sysmon_iio_info;
+
+	/*
+	 * Assign per-type sequential channel numbers.
+	 * IIO sysfs uses type prefix (in_tempN, in_voltageN)
+	 * so numbers only need to be unique within each type.
+	 */
+	temp_chan_idx = 0;
+	volt_chan_idx = 0;
+	for (unsigned int idx = 0; idx < indio_dev->num_channels; idx++) {
+		if (sysmon_channels[idx].type == IIO_TEMP)
+			sysmon_channels[idx].channel = temp_chan_idx++;
+		else
+			sysmon_channels[idx].channel = volt_chan_idx++;
+	}
+
+	indio_dev->channels = sysmon_channels;
+
+	return 0;
+}
+
+/**
+ * devm_versal_sysmon_core_probe() - Initialize Versal SysMon core
+ * @dev: Parent device
+ * @regmap: Register map for hardware access
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int devm_versal_sysmon_core_probe(struct device *dev, struct regmap *regmap)
+{
+	struct iio_dev *indio_dev;
+	struct sysmon *sysmon;
+	int irq;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*sysmon));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	sysmon = iio_priv(indio_dev);
+	sysmon->regmap = regmap;
+	sysmon->temp_oversampling = 1;
+	sysmon->supply_oversampling = 1;
+
+	ret = devm_mutex_init(dev, &sysmon->lock);
+	if (ret)
+		return ret;
+	spin_lock_init(&sysmon->irq_lock);
+
+	/* Disable all interrupts and clear pending status */
+	ret = regmap_write(sysmon->regmap, SYSMON_IDR, SYSMON_INTR_ALL_MASK);
+	if (ret)
+		return ret;
+	ret = regmap_write(sysmon->regmap, SYSMON_ISR, SYSMON_INTR_ALL_MASK);
+	if (ret)
+		return ret;
+
+	irq = fwnode_irq_get(dev_fwnode(dev), 0);
+	if (irq == -EPROBE_DEFER)
+		return dev_err_probe(dev, irq, "failed to get IRQ\n");
+
+	indio_dev->name = "versal-sysmon";
+	indio_dev->modes = INDIO_DIRECT_MODE;
+
+	ret = sysmon_parse_fw(indio_dev, dev, irq);
+	if (ret)
+		return ret;
+
+	if (irq > 0) {
+		/* Set hysteresis mode for temperature threshold */
+		ret = regmap_set_bits(sysmon->regmap, SYSMON_TEMP_EV_CFG,
+				      SYSMON_TEMP_HYST_MASK);
+		if (ret)
+			return ret;
+
+		/* Initialize cached hysteresis from hardware registers */
+		ret = sysmon_init_hysteresis(sysmon, &sysmon->temp_hysteresis);
+		if (ret)
+			return ret;
+
+		ret = sysmon_init_interrupt(sysmon, dev, indio_dev, irq);
+		if (ret)
+			return ret;
+	}
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+EXPORT_SYMBOL_NS_GPL(devm_versal_sysmon_core_probe, "VERSAL_SYSMON");
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("AMD Versal SysMon Core Driver");
+MODULE_AUTHOR("Salih Erim <salih.erim@amd.com>");
diff --git a/drivers/iio/adc/versal-sysmon-i2c.c b/drivers/iio/adc/versal-sysmon-i2c.c
new file mode 100644
index 0000000000000..e9a7629159ab1
--- /dev/null
+++ b/drivers/iio/adc/versal-sysmon-i2c.c
@@ -0,0 +1,134 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * AMD Versal SysMon I2C driver
+ *
+ * Copyright (C) 2023 - 2026, Advanced Micro Devices, Inc.
+ */
+
+#include <linux/bits.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/types.h>
+#include <linux/unaligned.h>
+
+#include "versal-sysmon.h"
+
+#define SYSMON_I2C_INSTR_READ	BIT(2)
+#define SYSMON_I2C_INSTR_WRITE	BIT(3)
+
+/*
+ * I2C command frame layout (8 bytes):
+ *   [0..3] data payload (little-endian u32)
+ *   [4..5] register offset >> 2 (little-endian u16)
+ *   [6]    instruction (read/write)
+ *   [7]    reserved
+ */
+#define SYSMON_I2C_DATA_OFS	0
+#define SYSMON_I2C_REG_OFS	4
+#define SYSMON_I2C_INSTR_OFS	6
+
+static int sysmon_i2c_reg_read(void *context, unsigned int reg,
+			       unsigned int *val)
+{
+	struct i2c_client *client = context;
+	u8 write_buf[8] = { };
+	u8 read_buf[4];
+	int ret;
+
+	put_unaligned_le16(reg >> 2, &write_buf[SYSMON_I2C_REG_OFS]);
+	write_buf[SYSMON_I2C_INSTR_OFS] = SYSMON_I2C_INSTR_READ;
+
+	ret = i2c_master_send(client, write_buf, sizeof(write_buf));
+	if (ret < 0)
+		return ret;
+	if (ret != sizeof(write_buf))
+		return -EIO;
+
+	ret = i2c_master_recv(client, read_buf, sizeof(read_buf));
+	if (ret < 0)
+		return ret;
+	if (ret != sizeof(read_buf))
+		return -EIO;
+
+	*val = get_unaligned_le32(read_buf);
+
+	return 0;
+}
+
+static int sysmon_i2c_reg_write(void *context, unsigned int reg,
+				unsigned int val)
+{
+	struct i2c_client *client = context;
+	u8 write_buf[8] = { };
+	int ret;
+
+	put_unaligned_le32(val, &write_buf[SYSMON_I2C_DATA_OFS]);
+	put_unaligned_le16(reg >> 2, &write_buf[SYSMON_I2C_REG_OFS]);
+	write_buf[SYSMON_I2C_INSTR_OFS] = SYSMON_I2C_INSTR_WRITE;
+
+	ret = i2c_master_send(client, write_buf, sizeof(write_buf));
+	if (ret < 0)
+		return ret;
+	if (ret != sizeof(write_buf))
+		return -EIO;
+
+	return 0;
+}
+
+/*
+ * Almost all registers are volatile (live ADC readings, interrupt
+ * status). The rest are not accessed often enough to benefit from
+ * caching.
+ */
+static const struct regmap_config sysmon_i2c_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = SYSMON_REG_STRIDE,
+	.max_register = SYSMON_MAX_REG,
+	.reg_read = sysmon_i2c_reg_read,
+	.reg_write = sysmon_i2c_reg_write,
+};
+
+static int sysmon_i2c_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct regmap *regmap;
+
+	regmap = devm_regmap_init(dev, NULL, client, &sysmon_i2c_regmap_config);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	/* I2C has no IRQ connection; events are not supported */
+	return devm_versal_sysmon_core_probe(dev, regmap);
+}
+
+static const struct of_device_id sysmon_i2c_of_match_table[] = {
+	{ .compatible = "xlnx,versal-sysmon" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, sysmon_i2c_of_match_table);
+
+static const struct i2c_device_id sysmon_i2c_id_table[] = {
+	{ .name = "versal-sysmon" },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, sysmon_i2c_id_table);
+
+static struct i2c_driver sysmon_i2c_driver = {
+	.probe = sysmon_i2c_probe,
+	.driver = {
+		.name = "versal-sysmon-i2c",
+		.of_match_table = sysmon_i2c_of_match_table,
+	},
+	.id_table = sysmon_i2c_id_table,
+};
+module_i2c_driver(sysmon_i2c_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("AMD Versal SysMon I2C Driver");
+MODULE_IMPORT_NS("VERSAL_SYSMON");
+MODULE_AUTHOR("Conall O'Griofa <conall.ogriofa@amd.com>");
+MODULE_AUTHOR("Salih Erim <salih.erim@amd.com>");
diff --git a/drivers/iio/adc/versal-sysmon.c b/drivers/iio/adc/versal-sysmon.c
new file mode 100644
index 0000000000000..529d0486c9f9e
--- /dev/null
+++ b/drivers/iio/adc/versal-sysmon.c
@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * AMD Versal SysMon MMIO platform driver
+ *
+ * Copyright (C) 2019 - 2022, Xilinx, Inc.
+ * Copyright (C) 2022 - 2026, Advanced Micro Devices, Inc.
+ */
+
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/types.h>
+
+#include "versal-sysmon.h"
+
+struct sysmon_mmio {
+	void __iomem *base;
+};
+
+static int sysmon_mmio_reg_read(void *context, unsigned int reg, unsigned int *val)
+{
+	struct sysmon_mmio *mmio = context;
+
+	*val = readl(mmio->base + reg);
+	return 0;
+}
+
+static int sysmon_mmio_reg_write(void *context, unsigned int reg, unsigned int val)
+{
+	struct sysmon_mmio *mmio = context;
+
+	/* NPI must be unlocked before any register write except to NPI_LOCK */
+	if (reg != SYSMON_NPI_LOCK)
+		writel(SYSMON_NPI_UNLOCK_CODE, mmio->base + SYSMON_NPI_LOCK);
+	writel(val, mmio->base + reg);
+
+	return 0;
+}
+
+static const struct regmap_config sysmon_mmio_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = SYSMON_REG_STRIDE,
+	.max_register = SYSMON_MAX_REG,
+	.reg_read = sysmon_mmio_reg_read,
+	.reg_write = sysmon_mmio_reg_write,
+	.fast_io = true,
+};
+
+static int sysmon_platform_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct sysmon_mmio *mmio;
+	struct regmap *regmap;
+
+	mmio = devm_kzalloc(dev, sizeof(*mmio), GFP_KERNEL);
+	if (!mmio)
+		return -ENOMEM;
+
+	mmio->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(mmio->base))
+		return PTR_ERR(mmio->base);
+
+	regmap = devm_regmap_init(dev, NULL, mmio, &sysmon_mmio_regmap_config);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	return devm_versal_sysmon_core_probe(dev, regmap);
+}
+
+static const struct of_device_id sysmon_of_match_table[] = {
+	{ .compatible = "xlnx,versal-sysmon" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, sysmon_of_match_table);
+
+static struct platform_driver sysmon_platform_driver = {
+	.probe = sysmon_platform_probe,
+	.driver = {
+		.name = "versal-sysmon",
+		.of_match_table = sysmon_of_match_table,
+	},
+};
+module_platform_driver(sysmon_platform_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("AMD Versal SysMon Platform Driver");
+MODULE_IMPORT_NS("VERSAL_SYSMON");
+MODULE_AUTHOR("Salih Erim <salih.erim@amd.com>");
diff --git a/drivers/iio/adc/versal-sysmon.h b/drivers/iio/adc/versal-sysmon.h
new file mode 100644
index 0000000000000..bb9a75bf71c0d
--- /dev/null
+++ b/drivers/iio/adc/versal-sysmon.h
@@ -0,0 +1,120 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * AMD Versal SysMon driver
+ *
+ * Copyright (C) 2019 - 2022, Xilinx, Inc.
+ * Copyright (C) 2022 - 2026, Advanced Micro Devices, Inc.
+ */
+
+#ifndef _VERSAL_SYSMON_H_
+#define _VERSAL_SYSMON_H_
+
+#include <linux/bits.h>
+#include <linux/mutex.h>
+#include <linux/spinlock_types.h>
+#include <linux/workqueue.h>
+
+struct device;
+struct regmap;
+
+/* Register offsets (sorted by address) */
+#define SYSMON_NPI_LOCK			0x000C
+#define SYSMON_ISR			0x0044
+#define SYSMON_IMR			0x0048
+#define SYSMON_IER			0x004C
+#define SYSMON_IDR			0x0050
+#define SYSMON_CONFIG			0x0100
+#define SYSMON_TEMP_MAX			0x1030
+#define SYSMON_TEMP_MIN			0x1034
+#define SYSMON_SUPPLY_BASE		0x1040
+#define SYSMON_ALARM_FLAG		0x1018
+#define SYSMON_ALARM_REG		0x1940
+#define SYSMON_SUPPLY_EN_AVG_BASE	0x1958
+#define SYSMON_TEMP_TH_LOW		0x1970
+#define SYSMON_TEMP_TH_UP		0x1974
+#define SYSMON_SUPPLY_TH_LOW		0x1980
+#define SYSMON_SUPPLY_TH_UP		0x1C80
+#define SYSMON_TEMP_EV_CFG		0x1F84
+#define SYSMON_TEMP_MIN_MIN		0x1F8C
+#define SYSMON_TEMP_MAX_MAX		0x1F90
+#define SYSMON_STATUS_RESET		0x1F94
+#define SYSMON_TEMP_SAT_BASE		0x1FAC
+#define SYSMON_TEMP_EN_AVG_BASE		0x24B4
+#define SYSMON_MAX_REG			0x24C0
+
+/* NPI unlock value written to SYSMON_NPI_LOCK */
+#define SYSMON_NPI_UNLOCK_CODE		0xF9E8D7C6
+
+/* Register stride: 4 bytes per 32-bit register */
+#define SYSMON_REG_STRIDE		4
+
+#define SYSMON_SUPPLY_IDX_MAX		159
+#define SYSMON_TEMP_SAT_MAX		64
+#define SYSMON_NO_OF_EVENTS		32
+#define SYSMON_INTR_ALL_MASK		GENMASK(31, 0)
+
+/* ISR/IMR temperature alarm mask (bit 9) */
+#define SYSMON_TEMP_INTR_MASK		BIT(9)
+
+/* SYSMON_CONFIG: supply oversampling ratio */
+#define SYSMON_CONFIG_SUPPLY_OSR	GENMASK(17, 14)
+
+/* SYSMON_CONFIG: temperature satellite oversampling ratio */
+#define SYSMON_CONFIG_TEMP_SAT_OSR	GENMASK(27, 24)
+
+/* Per-channel averaging enable register counts */
+#define SYSMON_SUPPLY_EN_AVG_COUNT	5
+#define SYSMON_TEMP_EN_AVG_COUNT	2
+
+/* Supply voltage conversion register fields */
+#define SYSMON_MANTISSA_MASK		GENMASK(15, 0)
+#define SYSMON_FMT_MASK			BIT(16)
+#define SYSMON_MODE_MASK		GENMASK(18, 17)
+
+/* Q8.7 fractional shift */
+#define SYSMON_FRACTIONAL_SHIFT		7U
+#define SYSMON_SUPPLY_MANTISSA_BITS	16
+
+/* Bits per alarm register */
+#define SYSMON_ALARM_BITS_PER_REG	32
+
+#define SYSMON_UNMASK_WORK_DELAY_MS	500
+
+/**
+ * struct sysmon - Driver data for Versal SysMon
+ * @regmap: register map for hardware access
+ * @lock: protects read-modify-write sequences on threshold registers
+ *        and cached state that spans multiple regmap calls
+ * @irq_lock: protects interrupt mask register updates (MMIO path only)
+ * @masked_temp: currently masked temperature alarm bits
+ * @temp_mask: temperature interrupt configuration mask
+ * @temp_hysteresis: cached DEVICE_TEMP hysteresis in millicelsius
+ * @sysmon_unmask_work: re-enables events after alarm condition clears
+ * @temp_oversampling: current temp oversampling ratio
+ * @supply_oversampling: current supply oversampling ratio
+ */
+struct sysmon {
+	struct regmap *regmap;
+	/*
+	 * Protects read-modify-write sequences on threshold registers
+	 * and cached state (oversampling ratios, hysteresis values)
+	 * that spans multiple regmap calls.
+	 */
+	struct mutex lock;
+	/*
+	 * Protects interrupt mask register updates.  Only used on the
+	 * MMIO path (fast_io regmap); I2C has no IRQ and never reaches
+	 * the event code that takes this lock.
+	 */
+	spinlock_t irq_lock;
+	unsigned int masked_temp;
+	unsigned int temp_mask;
+	int temp_hysteresis;
+	struct delayed_work sysmon_unmask_work;
+	unsigned int temp_oversampling;
+	unsigned int supply_oversampling;
+};
+
+int devm_versal_sysmon_core_probe(struct device *dev, struct regmap *regmap);
+
+#endif /* _VERSAL_SYSMON_H_ */
diff --git a/drivers/iio/addac/ad74115.c b/drivers/iio/addac/ad74115.c
index 41e0b1d334cc2..9b9d7a12ce878 100644
--- a/drivers/iio/addac/ad74115.c
+++ b/drivers/iio/addac/ad74115.c
@@ -1904,7 +1904,7 @@ static int __init ad74115_register_driver(struct spi_driver *spi)
 }
 
 static const struct spi_device_id ad74115_spi_id[] = {
-	{ "ad74115h" },
+	{ .name = "ad74115h" },
 	{ }
 };
 
diff --git a/drivers/iio/afe/iio-rescale.c b/drivers/iio/afe/iio-rescale.c
index 654a3a50eb4f1..3a6a09ffa5267 100644
--- a/drivers/iio/afe/iio-rescale.c
+++ b/drivers/iio/afe/iio-rescale.c
@@ -608,3 +608,4 @@ module_platform_driver(rescale_driver);
 MODULE_DESCRIPTION("IIO rescale driver");
 MODULE_AUTHOR("Peter Rosin <peda@axentia.se>");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/iio/amplifiers/ad8366.c b/drivers/iio/amplifiers/ad8366.c
index affc9c9d8488e..e12f8604f59f4 100644
--- a/drivers/iio/amplifiers/ad8366.c
+++ b/drivers/iio/amplifiers/ad8366.c
@@ -351,19 +351,19 @@ static int ad8366_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad8366_id[] = {
-	{ "ad8366", (kernel_ulong_t)&ad8366_chip_info },
-	{ "ada4961", (kernel_ulong_t)&ada4961_chip_info },
-	{ "adl5240", (kernel_ulong_t)&adl5240_chip_info },
-	{ "adrf5702", (kernel_ulong_t)&adrf5702_chip_info },
-	{ "adrf5703", (kernel_ulong_t)&adrf5703_chip_info },
-	{ "adrf5720", (kernel_ulong_t)&adrf5720_chip_info },
-	{ "adrf5730", (kernel_ulong_t)&adrf5730_chip_info },
-	{ "adrf5731", (kernel_ulong_t)&adrf5731_chip_info },
-	{ "hmc271a", (kernel_ulong_t)&hmc271_chip_info },
-	{ "hmc792a", (kernel_ulong_t)&hmc792_chip_info },
-	{ "hmc1018a", (kernel_ulong_t)&hmc1018_chip_info },
-	{ "hmc1019a", (kernel_ulong_t)&hmc1019_chip_info },
-	{ "hmc1119", (kernel_ulong_t)&hmc1119_chip_info },
+	{ .name = "ad8366", .driver_data = (kernel_ulong_t)&ad8366_chip_info },
+	{ .name = "ada4961", .driver_data = (kernel_ulong_t)&ada4961_chip_info },
+	{ .name = "adl5240", .driver_data = (kernel_ulong_t)&adl5240_chip_info },
+	{ .name = "adrf5702", .driver_data = (kernel_ulong_t)&adrf5702_chip_info },
+	{ .name = "adrf5703", .driver_data = (kernel_ulong_t)&adrf5703_chip_info },
+	{ .name = "adrf5720", .driver_data = (kernel_ulong_t)&adrf5720_chip_info },
+	{ .name = "adrf5730", .driver_data = (kernel_ulong_t)&adrf5730_chip_info },
+	{ .name = "adrf5731", .driver_data = (kernel_ulong_t)&adrf5731_chip_info },
+	{ .name = "hmc271a", .driver_data = (kernel_ulong_t)&hmc271_chip_info },
+	{ .name = "hmc792a", .driver_data = (kernel_ulong_t)&hmc792_chip_info },
+	{ .name = "hmc1018a", .driver_data = (kernel_ulong_t)&hmc1018_chip_info },
+	{ .name = "hmc1019a", .driver_data = (kernel_ulong_t)&hmc1019_chip_info },
+	{ .name = "hmc1119", .driver_data = (kernel_ulong_t)&hmc1119_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad8366_id);
diff --git a/drivers/iio/amplifiers/ada4250.c b/drivers/iio/amplifiers/ada4250.c
index 71e361af2074f..9702de3a50742 100644
--- a/drivers/iio/amplifiers/ada4250.c
+++ b/drivers/iio/amplifiers/ada4250.c
@@ -358,7 +358,7 @@ static int ada4250_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ada4250_id[] = {
-	{ "ada4250", 0 },
+	{ .name = "ada4250" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ada4250_id);
diff --git a/drivers/iio/buffer/industrialio-buffer-cb.c b/drivers/iio/buffer/industrialio-buffer-cb.c
index 0c266c216525e..b58263bcf3cb9 100644
--- a/drivers/iio/buffer/industrialio-buffer-cb.c
+++ b/drivers/iio/buffer/industrialio-buffer-cb.c
@@ -154,3 +154,4 @@ EXPORT_SYMBOL_GPL(iio_channel_cb_get_iio_dev);
 MODULE_AUTHOR("Jonathan Cameron <jic23@kernel.org>");
 MODULE_DESCRIPTION("Industrial I/O callback buffer");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/iio/buffer/industrialio-hw-consumer.c b/drivers/iio/buffer/industrialio-hw-consumer.c
index 10e912bbf0c5f..d035aa33b3570 100644
--- a/drivers/iio/buffer/industrialio-hw-consumer.c
+++ b/drivers/iio/buffer/industrialio-hw-consumer.c
@@ -216,3 +216,4 @@ EXPORT_SYMBOL_GPL(iio_hw_consumer_disable);
 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
 MODULE_DESCRIPTION("Hardware consumer buffer the IIO framework");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/iio/chemical/bme680_spi.c b/drivers/iio/chemical/bme680_spi.c
index 785200a6fd659..c55c6c5145946 100644
--- a/drivers/iio/chemical/bme680_spi.c
+++ b/drivers/iio/chemical/bme680_spi.c
@@ -130,7 +130,7 @@ static int bme680_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id bme680_spi_id[] = {
-	{"bme680", 0},
+	{ .name = "bme680" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, bme680_spi_id);
diff --git a/drivers/iio/chemical/ens160_spi.c b/drivers/iio/chemical/ens160_spi.c
index a674c0e1bf4b4..7c11d19291c99 100644
--- a/drivers/iio/chemical/ens160_spi.c
+++ b/drivers/iio/chemical/ens160_spi.c
@@ -39,7 +39,7 @@ static const struct of_device_id ens160_spi_of_match[] = {
 MODULE_DEVICE_TABLE(of, ens160_spi_of_match);
 
 static const struct spi_device_id ens160_spi_id[] = {
-	{ "ens160" },
+	{ .name = "ens160" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ens160_spi_id);
diff --git a/drivers/iio/chemical/scd30_core.c b/drivers/iio/chemical/scd30_core.c
index c5425136814bb..f85cdd8bd84f7 100644
--- a/drivers/iio/chemical/scd30_core.c
+++ b/drivers/iio/chemical/scd30_core.c
@@ -2,7 +2,7 @@
 /*
  * Sensirion SCD30 carbon dioxide sensor core driver
  *
- * Copyright (c) 2020 Tomasz Duszynski <tomasz.duszynski@octakon.com>
+ * Copyright (c) 2020 Tomasz Duszynski <tduszyns@gmail.com>
  */
 
 #include <linux/bitfield.h>
@@ -711,7 +711,11 @@ int scd30_probe(struct device *dev, int irq, const char *name, void *priv,
 	state->pressure_comp = SCD30_PRESSURE_COMP_DEFAULT;
 	state->meas_interval = SCD30_MEAS_INTERVAL_DEFAULT;
 	state->command = command;
-	mutex_init(&state->lock);
+
+	ret = devm_mutex_init(dev, &state->lock);
+	if (ret)
+		return ret;
+
 	init_completion(&state->meas_ready);
 
 	dev_set_drvdata(dev, indio_dev);
@@ -770,6 +774,6 @@ int scd30_probe(struct device *dev, int irq, const char *name, void *priv,
 }
 EXPORT_SYMBOL_NS(scd30_probe, "IIO_SCD30");
 
-MODULE_AUTHOR("Tomasz Duszynski <tomasz.duszynski@octakon.com>");
+MODULE_AUTHOR("Tomasz Duszynski <tduszyns@gmail.com>");
 MODULE_DESCRIPTION("Sensirion SCD30 carbon dioxide sensor core driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/chemical/scd30_i2c.c b/drivers/iio/chemical/scd30_i2c.c
index abceccdddc71f..bcef1e473e344 100644
--- a/drivers/iio/chemical/scd30_i2c.c
+++ b/drivers/iio/chemical/scd30_i2c.c
@@ -2,7 +2,7 @@
 /*
  * Sensirion SCD30 carbon dioxide sensor i2c driver
  *
- * Copyright (c) 2020 Tomasz Duszynski <tomasz.duszynski@octakon.com>
+ * Copyright (c) 2020 Tomasz Duszynski <tduszyns@gmail.com>
  *
  * I2C slave address: 0x61
  */
@@ -136,7 +136,7 @@ static struct i2c_driver scd30_i2c_driver = {
 };
 module_i2c_driver(scd30_i2c_driver);
 
-MODULE_AUTHOR("Tomasz Duszynski <tomasz.duszynski@octakon.com>");
+MODULE_AUTHOR("Tomasz Duszynski <tduszyns@gmail.com>");
 MODULE_DESCRIPTION("Sensirion SCD30 carbon dioxide sensor i2c driver");
 MODULE_LICENSE("GPL v2");
 MODULE_IMPORT_NS("IIO_SCD30");
diff --git a/drivers/iio/chemical/scd30_serial.c b/drivers/iio/chemical/scd30_serial.c
index 7fefdadbfa0a8..31b0fc957b086 100644
--- a/drivers/iio/chemical/scd30_serial.c
+++ b/drivers/iio/chemical/scd30_serial.c
@@ -2,7 +2,7 @@
 /*
  * Sensirion SCD30 carbon dioxide sensor serial driver
  *
- * Copyright (c) 2020 Tomasz Duszynski <tomasz.duszynski@octakon.com>
+ * Copyright (c) 2020 Tomasz Duszynski <tduszyns@gmail.com>
  */
 #include <linux/crc16.h>
 #include <linux/device.h>
@@ -257,7 +257,7 @@ static struct serdev_device_driver scd30_serdev_driver = {
 };
 module_serdev_device_driver(scd30_serdev_driver);
 
-MODULE_AUTHOR("Tomasz Duszynski <tomasz.duszynski@octakon.com>");
+MODULE_AUTHOR("Tomasz Duszynski <tduszyns@gmail.com>");
 MODULE_DESCRIPTION("Sensirion SCD30 carbon dioxide sensor serial driver");
 MODULE_LICENSE("GPL v2");
 MODULE_IMPORT_NS("IIO_SCD30");
diff --git a/drivers/iio/chemical/sps30_i2c.c b/drivers/iio/chemical/sps30_i2c.c
index 90f1adb8c89f2..2d3a876cc06ed 100644
--- a/drivers/iio/chemical/sps30_i2c.c
+++ b/drivers/iio/chemical/sps30_i2c.c
@@ -2,7 +2,7 @@
 /*
  * Sensirion SPS30 particulate matter sensor i2c driver
  *
- * Copyright (c) 2020 Tomasz Duszynski <tomasz.duszynski@octakon.com>
+ * Copyright (c) 2020 Tomasz Duszynski <tduszyns@gmail.com>
  *
  * I2C slave address: 0x69
  */
@@ -252,7 +252,7 @@ static struct i2c_driver sps30_i2c_driver = {
 };
 module_i2c_driver(sps30_i2c_driver);
 
-MODULE_AUTHOR("Tomasz Duszynski <tomasz.duszynski@octakon.com>");
+MODULE_AUTHOR("Tomasz Duszynski <tduszyns@gmail.com>");
 MODULE_DESCRIPTION("Sensirion SPS30 particulate matter sensor i2c driver");
 MODULE_LICENSE("GPL v2");
 MODULE_IMPORT_NS("IIO_SPS30");
diff --git a/drivers/iio/chemical/sps30_serial.c b/drivers/iio/chemical/sps30_serial.c
index 80eab9b2e4bf1..0cdba8703e938 100644
--- a/drivers/iio/chemical/sps30_serial.c
+++ b/drivers/iio/chemical/sps30_serial.c
@@ -2,7 +2,7 @@
 /*
  * Sensirion SPS30 particulate matter sensor serial driver
  *
- * Copyright (c) 2021 Tomasz Duszynski <tomasz.duszynski@octakon.com>
+ * Copyright (c) 2021 Tomasz Duszynski <tduszyns@gmail.com>
  */
 #include <linux/completion.h>
 #include <linux/device.h>
@@ -425,7 +425,7 @@ static struct serdev_device_driver sps30_serial_driver = {
 };
 module_serdev_device_driver(sps30_serial_driver);
 
-MODULE_AUTHOR("Tomasz Duszynski <tomasz.duszynski@octakon.com>");
+MODULE_AUTHOR("Tomasz Duszynski <tduszyns@gmail.com>");
 MODULE_DESCRIPTION("Sensirion SPS30 particulate matter sensor serial driver");
 MODULE_LICENSE("GPL v2");
 MODULE_IMPORT_NS("IIO_SPS30");
diff --git a/drivers/iio/common/hid-sensors/hid-sensor-attributes.c b/drivers/iio/common/hid-sensors/hid-sensor-attributes.c
index c115a72832b2e..9e7c26e37880a 100644
--- a/drivers/iio/common/hid-sensors/hid-sensor-attributes.c
+++ b/drivers/iio/common/hid-sensors/hid-sensor-attributes.c
@@ -72,8 +72,7 @@ static const struct {
 	{HID_USAGE_SENSOR_HUMAN_ATTENTION, 0, 1, 0},
 };
 
-static void simple_div(int dividend, int divisor, int *whole,
-				int *micro_frac)
+static void simple_div(int dividend, int divisor, int *whole, int *micro_frac)
 {
 	int rem;
 	int exp = 0;
@@ -111,7 +110,7 @@ for 10^-2.
 Negative numbers are 2's complement
 */
 static void convert_from_vtf_format(u32 value, int size, int exp,
-					int *val1, int *val2)
+				    int *val1, int *val2)
 {
 	int sign = 1;
 
@@ -176,7 +175,7 @@ s32 hid_sensor_read_poll_value(struct hid_sensor_common *st)
 EXPORT_SYMBOL_NS(hid_sensor_read_poll_value, "IIO_HID_ATTRIBUTES");
 
 int hid_sensor_read_samp_freq_value(struct hid_sensor_common *st,
-				int *val1, int *val2)
+				    int *val1, int *val2)
 {
 	s32 value;
 	int ret;
@@ -203,7 +202,7 @@ int hid_sensor_read_samp_freq_value(struct hid_sensor_common *st,
 EXPORT_SYMBOL_NS(hid_sensor_read_samp_freq_value, "IIO_HID");
 
 int hid_sensor_write_samp_freq_value(struct hid_sensor_common *st,
-				int val1, int val2)
+				     int val1, int val2)
 {
 	s32 value;
 	int ret;
@@ -238,15 +237,15 @@ int hid_sensor_write_samp_freq_value(struct hid_sensor_common *st,
 EXPORT_SYMBOL_NS(hid_sensor_write_samp_freq_value, "IIO_HID");
 
 int hid_sensor_read_raw_hyst_value(struct hid_sensor_common *st,
-				int *val1, int *val2)
+				   int *val1, int *val2)
 {
 	s32 value;
 	int ret;
 
 	ret = sensor_hub_get_feature(st->hsdev,
 				     st->sensitivity.report_id,
-				     st->sensitivity.index, sizeof(value),
-				     &value);
+				     st->sensitivity.index,
+				     sizeof(value), &value);
 	if (ret < 0 || value < 0) {
 		*val1 = *val2 = 0;
 		return -EINVAL;
@@ -268,8 +267,8 @@ int hid_sensor_read_raw_hyst_rel_value(struct hid_sensor_common *st, int *val1,
 
 	ret = sensor_hub_get_feature(st->hsdev,
 				     st->sensitivity_rel.report_id,
-				     st->sensitivity_rel.index, sizeof(value),
-				     &value);
+				     st->sensitivity_rel.index,
+				     sizeof(value), &value);
 	if (ret < 0 || value < 0) {
 		*val1 = *val2 = 0;
 		return -EINVAL;
@@ -282,9 +281,8 @@ int hid_sensor_read_raw_hyst_rel_value(struct hid_sensor_common *st, int *val1,
 }
 EXPORT_SYMBOL_NS(hid_sensor_read_raw_hyst_rel_value, "IIO_HID");
 
-
 int hid_sensor_write_raw_hyst_value(struct hid_sensor_common *st,
-					int val1, int val2)
+				    int val1, int val2)
 {
 	s32 value;
 	int ret;
@@ -293,8 +291,8 @@ int hid_sensor_write_raw_hyst_value(struct hid_sensor_common *st,
 		return -EINVAL;
 
 	value = convert_to_vtf_format(st->sensitivity.size,
-				st->sensitivity.unit_expo,
-				val1, val2);
+				      st->sensitivity.unit_expo,
+				      val1, val2);
 	ret = sensor_hub_set_feature(st->hsdev, st->sensitivity.report_id,
 				     st->sensitivity.index, sizeof(value),
 				     &value);
@@ -303,8 +301,8 @@ int hid_sensor_write_raw_hyst_value(struct hid_sensor_common *st,
 
 	ret = sensor_hub_get_feature(st->hsdev,
 				     st->sensitivity.report_id,
-				     st->sensitivity.index, sizeof(value),
-				     &value);
+				     st->sensitivity.index,
+				     sizeof(value), &value);
 	if (ret < 0 || value < 0)
 		return -EINVAL;
 
@@ -324,8 +322,8 @@ int hid_sensor_write_raw_hyst_rel_value(struct hid_sensor_common *st,
 		return -EINVAL;
 
 	value = convert_to_vtf_format(st->sensitivity_rel.size,
-				st->sensitivity_rel.unit_expo,
-				val1, val2);
+				      st->sensitivity_rel.unit_expo,
+				      val1, val2);
 	ret = sensor_hub_set_feature(st->hsdev, st->sensitivity_rel.report_id,
 				     st->sensitivity_rel.index, sizeof(value),
 				     &value);
@@ -334,8 +332,8 @@ int hid_sensor_write_raw_hyst_rel_value(struct hid_sensor_common *st,
 
 	ret = sensor_hub_get_feature(st->hsdev,
 				     st->sensitivity_rel.report_id,
-				     st->sensitivity_rel.index, sizeof(value),
-				     &value);
+				     st->sensitivity_rel.index,
+				     sizeof(value), &value);
 	if (ret < 0 || value < 0)
 		return -EINVAL;
 
@@ -356,8 +354,8 @@ EXPORT_SYMBOL_NS(hid_sensor_write_raw_hyst_rel_value, "IIO_HID");
  * 1.001745329 ->exp:4-> val0[10017]val1[453290000]
  * 9.806650000 ->exp:-2-> val0[0]val1[98066500]
  */
-static void adjust_exponent_nano(int *val0, int *val1, int scale0,
-				  int scale1, int exp)
+static void adjust_exponent_nano(int *val0, int *val1,
+				 int scale0, int scale1, int exp)
 {
 	int divisor;
 	int i;
@@ -404,8 +402,8 @@ static void adjust_exponent_nano(int *val0, int *val1, int scale0,
 }
 
 int hid_sensor_format_scale(u32 usage_id,
-			struct hid_sensor_hub_attribute_info *attr_info,
-			int *val0, int *val1)
+			    struct hid_sensor_hub_attribute_info *attr_info,
+			    int *val0, int *val1)
 {
 	int i;
 	int exp;
@@ -415,12 +413,11 @@ int hid_sensor_format_scale(u32 usage_id,
 
 	for (i = 0; i < ARRAY_SIZE(unit_conversion); ++i) {
 		if (unit_conversion[i].usage_id == usage_id &&
-			unit_conversion[i].unit == attr_info->units) {
-			exp  = hid_sensor_convert_exponent(
-						attr_info->unit_expo);
+		    unit_conversion[i].unit == attr_info->units) {
+			exp  = hid_sensor_convert_exponent(attr_info->unit_expo);
 			adjust_exponent_nano(val0, val1,
-					unit_conversion[i].scale_val0,
-					unit_conversion[i].scale_val1, exp);
+					     unit_conversion[i].scale_val0,
+					     unit_conversion[i].scale_val1, exp);
 			break;
 		}
 	}
@@ -438,8 +435,8 @@ EXPORT_SYMBOL_NS(hid_sensor_convert_timestamp, "IIO_HID");
 
 static
 int hid_sensor_get_reporting_interval(struct hid_sensor_hub_device *hsdev,
-					u32 usage_id,
-					struct hid_sensor_common *st)
+				      u32 usage_id,
+				      struct hid_sensor_common *st)
 {
 	sensor_hub_input_get_attribute_info(hsdev,
 					HID_FEATURE_REPORT, usage_id,
@@ -473,9 +470,10 @@ int hid_sensor_get_report_latency(struct hid_sensor_common *st)
 	int ret;
 	int value;
 
-	ret = sensor_hub_get_feature(st->hsdev, st->report_latency.report_id,
-				     st->report_latency.index, sizeof(value),
-				     &value);
+	ret = sensor_hub_get_feature(st->hsdev,
+				     st->report_latency.report_id,
+				     st->report_latency.index,
+				     sizeof(value), &value);
 	if (ret < 0)
 		return ret;
 
@@ -498,10 +496,10 @@ bool hid_sensor_batch_mode_supported(struct hid_sensor_common *st)
 EXPORT_SYMBOL_NS(hid_sensor_batch_mode_supported, "IIO_HID_ATTRIBUTES");
 
 int hid_sensor_parse_common_attributes(struct hid_sensor_hub_device *hsdev,
-					u32 usage_id,
-					struct hid_sensor_common *st,
-					const u32 *sensitivity_addresses,
-					u32 sensitivity_addresses_len)
+				       u32 usage_id,
+				       struct hid_sensor_common *st,
+				       const u32 *sensitivity_addresses,
+				       u32 sensitivity_addresses_len)
 {
 
 	struct hid_sensor_hub_attribute_info timestamp;
@@ -527,7 +525,7 @@ int hid_sensor_parse_common_attributes(struct hid_sensor_hub_device *hsdev,
 	sensor_hub_input_get_attribute_info(hsdev,
 			HID_FEATURE_REPORT, usage_id,
 			HID_USAGE_SENSOR_PROP_SENSITIVITY_ABS,
-			 &st->sensitivity);
+			&st->sensitivity);
 
 	sensor_hub_input_get_attribute_info(hsdev,
 			HID_FEATURE_REPORT, usage_id,
@@ -578,8 +576,9 @@ int hid_sensor_parse_common_attributes(struct hid_sensor_hub_device *hsdev,
 		timestamp.index, timestamp.report_id);
 
 	ret = sensor_hub_get_feature(hsdev,
-				st->power_state.report_id,
-				st->power_state.index, sizeof(value), &value);
+				     st->power_state.report_id,
+				     st->power_state.index,
+				     sizeof(value), &value);
 	if (ret < 0)
 		return ret;
 	if (value < 0)
diff --git a/drivers/iio/common/hid-sensors/hid-sensor-trigger.c b/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
index 417c4ab8c1b27..60808e2430ca0 100644
--- a/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
+++ b/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
@@ -233,7 +233,7 @@ void hid_sensor_remove_trigger(struct iio_dev *indio_dev,
 EXPORT_SYMBOL_NS(hid_sensor_remove_trigger, "IIO_HID");
 
 int hid_sensor_setup_trigger(struct iio_dev *indio_dev, const char *name,
-				struct hid_sensor_common *attrb)
+			     struct hid_sensor_common *attrb)
 {
 	const struct iio_dev_attr **fifo_attrs;
 	int ret;
@@ -266,7 +266,7 @@ int hid_sensor_setup_trigger(struct iio_dev *indio_dev, const char *name,
 
 	trig = iio_trigger_alloc(indio_dev->dev.parent,
 				 "%s-dev%d", name, iio_device_id(indio_dev));
-	if (trig == NULL) {
+	if (!trig) {
 		dev_err(&indio_dev->dev, "Trigger Allocate Failed\n");
 		return -ENOMEM;
 	}
@@ -314,7 +314,9 @@ static int __maybe_unused hid_sensor_resume(struct device *dev)
 {
 	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct hid_sensor_common *attrb = iio_device_get_drvdata(indio_dev);
+
 	schedule_work(&attrb->work);
+
 	return 0;
 }
 
@@ -322,6 +324,7 @@ static int __maybe_unused hid_sensor_runtime_resume(struct device *dev)
 {
 	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct hid_sensor_common *attrb = iio_device_get_drvdata(indio_dev);
+
 	return _hid_sensor_power_state(attrb, true);
 }
 
diff --git a/drivers/iio/common/hid-sensors/hid-sensor-trigger.h b/drivers/iio/common/hid-sensors/hid-sensor-trigger.h
index f94fca4f1edfa..589de858e3694 100644
--- a/drivers/iio/common/hid-sensors/hid-sensor-trigger.h
+++ b/drivers/iio/common/hid-sensors/hid-sensor-trigger.h
@@ -15,7 +15,7 @@ struct iio_dev;
 extern const struct dev_pm_ops hid_sensor_pm_ops;
 
 int hid_sensor_setup_trigger(struct iio_dev *indio_dev, const char *name,
-				struct hid_sensor_common *attrb);
+			     struct hid_sensor_common *attrb);
 void hid_sensor_remove_trigger(struct iio_dev *indio_dev,
 			       struct hid_sensor_common *attrb);
 int hid_sensor_power_state(struct hid_sensor_common *st, bool state);
diff --git a/drivers/iio/dac/Kconfig b/drivers/iio/dac/Kconfig
index 657c68e75542e..17529509da9d6 100644
--- a/drivers/iio/dac/Kconfig
+++ b/drivers/iio/dac/Kconfig
@@ -11,8 +11,11 @@ config AD3530R
 	depends on SPI
 	select REGMAP_SPI
 	help
-	  Say yes here to build support for Analog Devices AD3530R, AD3531R
-	  Digital to Analog Converter.
+	  Say yes here to build support for the following Analog Devices
+	  Digital to Analog Converters:
+	  - AD3530/AD3530R (8-channel)
+	  - AD3531/AD3531R (4-channel)
+	  - AD3532/AD3532R (16-channel)
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called ad3530r.
@@ -527,6 +530,17 @@ config MAX5821
 	  Say yes here to build support for Maxim MAX5821
 	  10 bits DAC.
 
+config MCF54415_DAC
+	tristate "NXP MCF54415 DAC driver"
+	depends on M5441x || COMPILE_TEST
+	select REGMAP_MMIO
+	help
+	  Say yes here if you want to build support for NXP ColdFire
+	  MCF54415/6/7/8 12-bit DAC module.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called mcf54415_dac.
+
 config MCP4725
 	tristate "MCP4725/6 DAC driver"
 	depends on I2C
diff --git a/drivers/iio/dac/Makefile b/drivers/iio/dac/Makefile
index 0034317984985..5d20d37e44ce5 100644
--- a/drivers/iio/dac/Makefile
+++ b/drivers/iio/dac/Makefile
@@ -52,6 +52,7 @@ obj-$(CONFIG_MAX517) += max517.o
 obj-$(CONFIG_MAX22007) += max22007.o
 obj-$(CONFIG_MAX5522) += max5522.o
 obj-$(CONFIG_MAX5821) += max5821.o
+obj-$(CONFIG_MCF54415_DAC) += mcf54415_dac.o
 obj-$(CONFIG_MCP4725) += mcp4725.o
 obj-$(CONFIG_MCP4728) += mcp4728.o
 obj-$(CONFIG_MCP47FEB02) += mcp47feb02.o
diff --git a/drivers/iio/dac/ad3530r.c b/drivers/iio/dac/ad3530r.c
index 4e911bfb6fc5f..06b7f280f61bf 100644
--- a/drivers/iio/dac/ad3530r.c
+++ b/drivers/iio/dac/ad3530r.c
@@ -2,6 +2,7 @@
 /*
  * AD3530R/AD3530 8-channel, 16-bit Voltage Output DAC Driver
  * AD3531R/AD3531 4-channel, 16-bit Voltage Output DAC Driver
+ * AD3532R/AD3532 16-channel, 16-bit Voltage Output DAC Driver
  *
  * Copyright 2025 Analog Devices Inc.
  */
@@ -38,6 +39,25 @@
 #define AD3531R_SW_LDAC_TRIG_A			0xDD
 #define AD3531R_INPUT_CH			0xE3
 
+/* AD3532R/AD3532 bank 0 registers (channels 0-7) */
+#define AD3532R_INTERFACE_CONFIG_A_0		0x1000
+#define AD3532R_OUTPUT_OPERATING_MODE_0		0x1020
+#define AD3532R_OUTPUT_OPERATING_MODE_1		0x1021
+#define AD3532R_OUTPUT_CONTROL_0		0x102A
+#define AD3532R_REFERENCE_CONTROL_0		0x103C
+#define AD3532R_SW_LDAC_TRIG_0			0x10E5
+#define AD3532R_INPUT_CH_0			0x10EB
+
+/* AD3532R/AD3532 bank 1 registers (channels 8-15) */
+#define AD3532R_INTERFACE_CONFIG_A_1		0x3000
+#define AD3532R_OUTPUT_OPERATING_MODE_2		0x3020
+#define AD3532R_OUTPUT_OPERATING_MODE_3		0x3021
+#define AD3532R_OUTPUT_CONTROL_1		0x302A
+#define AD3532R_REFERENCE_CONTROL_1		0x303C
+#define AD3532R_SW_LDAC_TRIG_1			0x30E5
+#define AD3532R_INPUT_CH_1			0x30EB
+#define AD3532R_MAX_REG_ADDR			0x30F9
+
 #define AD3530R_SLD_TRIG_A			BIT(7)
 #define AD3530R_OUTPUT_CONTROL_RANGE		BIT(2)
 #define AD3530R_REFERENCE_CONTROL_SEL		BIT(0)
@@ -49,8 +69,10 @@
 #define AD3530R_LDAC_PULSE_US			100
 
 #define AD3530R_DAC_MAX_VAL			GENMASK(15, 0)
-#define AD3530R_MAX_CHANNELS			8
+#define AD3530R_CH_PER_REG			4
+#define AD3530R_CH_PER_BANK			8
 #define AD3531R_MAX_CHANNELS			4
+#define AD3532R_MAX_CHANNELS			16
 
 enum ad3530r_mode {
 	AD3530R_NORMAL_OP,
@@ -67,9 +89,16 @@ struct ad3530r_chan {
 struct ad3530r_chip_info {
 	const char *name;
 	const struct iio_chan_spec *channels;
+	const struct regmap_config *regmap_config;
 	int (*input_ch_reg)(unsigned int channel);
+	int (*sw_ldac_trig_reg)(unsigned int channel);
+	const unsigned int *interface_config_a;
+	const unsigned int *output_control;
+	const unsigned int *reference_control;
+	const unsigned int *op_mode;
 	unsigned int num_channels;
-	unsigned int sw_ldac_trig_reg;
+	unsigned int num_banks;
+	unsigned int num_op_mode_regs;
 	bool internal_ref_support;
 };
 
@@ -77,7 +106,7 @@ struct ad3530r_state {
 	struct regmap *regmap;
 	/* lock to protect against multiple access to the device and shared data */
 	struct mutex lock;
-	struct ad3530r_chan chan[AD3530R_MAX_CHANNELS];
+	struct ad3530r_chan chan[AD3532R_MAX_CHANNELS];
 	const struct ad3530r_chip_info *chip_info;
 	struct gpio_desc *ldac_gpio;
 	int vref_mV;
@@ -98,6 +127,14 @@ static int ad3531r_input_ch_reg(unsigned int channel)
 	return 2 * channel + AD3531R_INPUT_CH;
 }
 
+static int ad3532r_input_ch_reg(unsigned int channel)
+{
+	unsigned int bank = channel / AD3530R_CH_PER_BANK;
+	unsigned int local_ch = channel % AD3530R_CH_PER_BANK;
+
+	return 2 * local_ch + (bank ? AD3532R_INPUT_CH_1 : AD3532R_INPUT_CH_0);
+}
+
 static const char * const ad3530r_powerdown_modes[] = {
 	"1kohm_to_gnd",
 	"7.7kohm_to_gnd",
@@ -110,6 +147,12 @@ static const char * const ad3531r_powerdown_modes[] = {
 	"16kohm_to_gnd",
 };
 
+static const char * const ad3532r_powerdown_modes[] = {
+	"1kohm_to_gnd",
+	"10kohm_to_gnd",
+	"three_state",
+};
+
 static int ad3530r_get_powerdown_mode(struct iio_dev *indio_dev,
 				      const struct iio_chan_spec *chan)
 {
@@ -145,6 +188,13 @@ static const struct iio_enum ad3531r_powerdown_mode_enum = {
 	.set = ad3530r_set_powerdown_mode,
 };
 
+static const struct iio_enum ad3532r_powerdown_mode_enum = {
+	.items = ad3532r_powerdown_modes,
+	.num_items = ARRAY_SIZE(ad3532r_powerdown_modes),
+	.get = ad3530r_get_powerdown_mode,
+	.set = ad3530r_set_powerdown_mode,
+};
+
 static ssize_t ad3530r_get_dac_powerdown(struct iio_dev *indio_dev,
 					 uintptr_t private,
 					 const struct iio_chan_spec *chan,
@@ -189,6 +239,62 @@ static ssize_t ad3530r_set_dac_powerdown(struct iio_dev *indio_dev,
 	return len;
 }
 
+static ssize_t ad3532r_set_dac_powerdown(struct iio_dev *indio_dev,
+					 uintptr_t private,
+					 const struct iio_chan_spec *chan,
+					 const char *buf, size_t len)
+{
+	struct ad3530r_state *st = iio_priv(indio_dev);
+	unsigned int bank, local_ch, reg_in_bank, ch_in_reg;
+	unsigned int reg, mask, val;
+	bool powerdown;
+	int ret;
+
+	ret = kstrtobool(buf, &powerdown);
+	if (ret)
+		return ret;
+
+	bank = chan->channel / AD3530R_CH_PER_BANK;
+	local_ch = chan->channel % AD3530R_CH_PER_BANK;
+	reg_in_bank = local_ch / AD3530R_CH_PER_REG;
+	ch_in_reg = local_ch % AD3530R_CH_PER_REG;
+
+	reg = reg_in_bank + (bank ? AD3532R_OUTPUT_OPERATING_MODE_2 :
+				    AD3532R_OUTPUT_OPERATING_MODE_0);
+	mask = AD3530R_OP_MODE_CHAN_MSK(ch_in_reg);
+
+	guard(mutex)(&st->lock);
+	if (powerdown) {
+		val = field_prep(mask, st->chan[chan->channel].powerdown_mode);
+		ret = regmap_update_bits(st->regmap, reg, mask, val);
+	} else {
+		ret = regmap_clear_bits(st->regmap, reg, mask);
+	}
+	if (ret)
+		return ret;
+
+	st->chan[chan->channel].powerdown = powerdown;
+
+	return len;
+}
+
+static int ad3530r_trigger_sw_ldac_reg(unsigned int channel)
+{
+	return AD3530R_SW_LDAC_TRIG_A;
+}
+
+static int ad3531r_trigger_sw_ldac_reg(unsigned int channel)
+{
+	return AD3531R_SW_LDAC_TRIG_A;
+}
+
+static int ad3532r_trigger_sw_ldac_reg(unsigned int channel)
+{
+	unsigned int bank = channel / AD3530R_CH_PER_BANK;
+
+	return bank ? AD3532R_SW_LDAC_TRIG_1 : AD3532R_SW_LDAC_TRIG_0;
+}
+
 static int ad3530r_trigger_hw_ldac(struct gpio_desc *ldac_gpio)
 {
 	gpiod_set_value_cansleep(ldac_gpio, 1);
@@ -214,7 +320,7 @@ static int ad3530r_dac_write(struct ad3530r_state *st, unsigned int chan,
 	if (st->ldac_gpio)
 		return ad3530r_trigger_hw_ldac(st->ldac_gpio);
 
-	return regmap_set_bits(st->regmap, st->chip_info->sw_ldac_trig_reg,
+	return regmap_set_bits(st->regmap, st->chip_info->sw_ldac_trig_reg(chan),
 			       AD3530R_SLD_TRIG_A);
 }
 
@@ -301,6 +407,19 @@ static const struct iio_chan_spec_ext_info ad3531r_ext_info[] = {
 	{ }
 };
 
+static const struct iio_chan_spec_ext_info ad3532r_ext_info[] = {
+	{
+		.name = "powerdown",
+		.shared = IIO_SEPARATE,
+		.read = ad3530r_get_dac_powerdown,
+		.write = ad3532r_set_dac_powerdown,
+	},
+	IIO_ENUM("powerdown_mode", IIO_SEPARATE, &ad3532r_powerdown_mode_enum),
+	IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE,
+			   &ad3532r_powerdown_mode_enum),
+	{ }
+};
+
 #define AD3530R_CHAN(_chan, _ext_info)				\
 {								\
 	.type = IIO_VOLTAGE,					\
@@ -330,48 +449,215 @@ static const struct iio_chan_spec ad3531r_channels[] = {
 	AD3530R_CHAN(3, ad3531r_ext_info),
 };
 
+static const struct iio_chan_spec ad3532r_channels[] = {
+	AD3530R_CHAN(0, ad3532r_ext_info),
+	AD3530R_CHAN(1, ad3532r_ext_info),
+	AD3530R_CHAN(2, ad3532r_ext_info),
+	AD3530R_CHAN(3, ad3532r_ext_info),
+	AD3530R_CHAN(4, ad3532r_ext_info),
+	AD3530R_CHAN(5, ad3532r_ext_info),
+	AD3530R_CHAN(6, ad3532r_ext_info),
+	AD3530R_CHAN(7, ad3532r_ext_info),
+	AD3530R_CHAN(8, ad3532r_ext_info),
+	AD3530R_CHAN(9, ad3532r_ext_info),
+	AD3530R_CHAN(10, ad3532r_ext_info),
+	AD3530R_CHAN(11, ad3532r_ext_info),
+	AD3530R_CHAN(12, ad3532r_ext_info),
+	AD3530R_CHAN(13, ad3532r_ext_info),
+	AD3530R_CHAN(14, ad3532r_ext_info),
+	AD3530R_CHAN(15, ad3532r_ext_info),
+};
+
+static const unsigned int ad3530r_if_config[] = {
+	AD3530R_INTERFACE_CONFIG_A,
+};
+
+static const unsigned int ad3530r_out_ctrl[] = {
+	AD3530R_OUTPUT_CONTROL_0,
+};
+
+static const unsigned int ad3530r_ref_ctrl[] = {
+	AD3530R_REFERENCE_CONTROL_0,
+};
+
+static const unsigned int ad3530r_op_mode[] = {
+	AD3530R_OUTPUT_OPERATING_MODE_0,
+	AD3530R_OUTPUT_OPERATING_MODE_1,
+};
+
+static const unsigned int ad3531r_op_mode[] = {
+	AD3530R_OUTPUT_OPERATING_MODE_0,
+};
+
+static const unsigned int ad3532r_if_config[] = {
+	AD3532R_INTERFACE_CONFIG_A_0,
+	AD3532R_INTERFACE_CONFIG_A_1,
+};
+
+static const unsigned int ad3532r_out_ctrl[] = {
+	AD3532R_OUTPUT_CONTROL_0,
+	AD3532R_OUTPUT_CONTROL_1,
+};
+
+static const unsigned int ad3532r_ref_ctrl[] = {
+	AD3532R_REFERENCE_CONTROL_0,
+	AD3532R_REFERENCE_CONTROL_1,
+};
+
+static const unsigned int ad3532r_op_mode[] = {
+	AD3532R_OUTPUT_OPERATING_MODE_0,
+	AD3532R_OUTPUT_OPERATING_MODE_1,
+	AD3532R_OUTPUT_OPERATING_MODE_2,
+	AD3532R_OUTPUT_OPERATING_MODE_3,
+};
+
+static const struct regmap_config ad3530r_regmap_config = {
+	.reg_bits = 16,
+	.val_bits = 8,
+	.max_register = AD3530R_MAX_REG_ADDR,
+};
+
+static const struct regmap_config ad3532r_regmap_config = {
+	.reg_bits = 16,
+	.val_bits = 8,
+	.max_register = AD3532R_MAX_REG_ADDR,
+};
+
 static const struct ad3530r_chip_info ad3530_chip = {
 	.name = "ad3530",
 	.channels = ad3530r_channels,
+	.regmap_config = &ad3530r_regmap_config,
 	.num_channels = ARRAY_SIZE(ad3530r_channels),
-	.sw_ldac_trig_reg = AD3530R_SW_LDAC_TRIG_A,
+	.sw_ldac_trig_reg = ad3530r_trigger_sw_ldac_reg,
 	.input_ch_reg = ad3530r_input_ch_reg,
+	.interface_config_a = ad3530r_if_config,
+	.output_control = ad3530r_out_ctrl,
+	.reference_control = ad3530r_ref_ctrl,
+	.op_mode = ad3530r_op_mode,
+	.num_banks = ARRAY_SIZE(ad3530r_if_config),
+	.num_op_mode_regs = ARRAY_SIZE(ad3530r_op_mode),
 	.internal_ref_support = false,
 };
 
 static const struct ad3530r_chip_info ad3530r_chip = {
 	.name = "ad3530r",
 	.channels = ad3530r_channels,
+	.regmap_config = &ad3530r_regmap_config,
 	.num_channels = ARRAY_SIZE(ad3530r_channels),
-	.sw_ldac_trig_reg = AD3530R_SW_LDAC_TRIG_A,
+	.sw_ldac_trig_reg = ad3530r_trigger_sw_ldac_reg,
 	.input_ch_reg = ad3530r_input_ch_reg,
+	.interface_config_a = ad3530r_if_config,
+	.output_control = ad3530r_out_ctrl,
+	.reference_control = ad3530r_ref_ctrl,
+	.op_mode = ad3530r_op_mode,
+	.num_banks = ARRAY_SIZE(ad3530r_if_config),
+	.num_op_mode_regs = ARRAY_SIZE(ad3530r_op_mode),
 	.internal_ref_support = true,
 };
 
 static const struct ad3530r_chip_info ad3531_chip = {
 	.name = "ad3531",
 	.channels = ad3531r_channels,
+	.regmap_config = &ad3530r_regmap_config,
 	.num_channels = ARRAY_SIZE(ad3531r_channels),
-	.sw_ldac_trig_reg = AD3531R_SW_LDAC_TRIG_A,
+	.sw_ldac_trig_reg = ad3531r_trigger_sw_ldac_reg,
 	.input_ch_reg = ad3531r_input_ch_reg,
+	.interface_config_a = ad3530r_if_config,
+	.output_control = ad3530r_out_ctrl,
+	.reference_control = ad3530r_ref_ctrl,
+	.op_mode = ad3531r_op_mode,
+	.num_banks = ARRAY_SIZE(ad3530r_if_config),
+	.num_op_mode_regs = ARRAY_SIZE(ad3531r_op_mode),
 	.internal_ref_support = false,
 };
 
 static const struct ad3530r_chip_info ad3531r_chip = {
 	.name = "ad3531r",
 	.channels = ad3531r_channels,
+	.regmap_config = &ad3530r_regmap_config,
 	.num_channels = ARRAY_SIZE(ad3531r_channels),
-	.sw_ldac_trig_reg = AD3531R_SW_LDAC_TRIG_A,
+	.sw_ldac_trig_reg = ad3531r_trigger_sw_ldac_reg,
 	.input_ch_reg = ad3531r_input_ch_reg,
+	.interface_config_a = ad3530r_if_config,
+	.output_control = ad3530r_out_ctrl,
+	.reference_control = ad3530r_ref_ctrl,
+	.op_mode = ad3531r_op_mode,
+	.num_banks = ARRAY_SIZE(ad3530r_if_config),
+	.num_op_mode_regs = ARRAY_SIZE(ad3531r_op_mode),
+	.internal_ref_support = true,
+};
+
+static const struct ad3530r_chip_info ad3532_chip = {
+	.name = "ad3532",
+	.channels = ad3532r_channels,
+	.regmap_config = &ad3532r_regmap_config,
+	.num_channels = ARRAY_SIZE(ad3532r_channels),
+	.sw_ldac_trig_reg = ad3532r_trigger_sw_ldac_reg,
+	.input_ch_reg = ad3532r_input_ch_reg,
+	.interface_config_a = ad3532r_if_config,
+	.output_control = ad3532r_out_ctrl,
+	.reference_control = ad3532r_ref_ctrl,
+	.op_mode = ad3532r_op_mode,
+	.num_banks = ARRAY_SIZE(ad3532r_if_config),
+	.num_op_mode_regs = ARRAY_SIZE(ad3532r_op_mode),
+	.internal_ref_support = false,
+};
+
+static const struct ad3530r_chip_info ad3532r_chip = {
+	.name = "ad3532r",
+	.channels = ad3532r_channels,
+	.regmap_config = &ad3532r_regmap_config,
+	.num_channels = ARRAY_SIZE(ad3532r_channels),
+	.sw_ldac_trig_reg = ad3532r_trigger_sw_ldac_reg,
+	.input_ch_reg = ad3532r_input_ch_reg,
+	.interface_config_a = ad3532r_if_config,
+	.output_control = ad3532r_out_ctrl,
+	.reference_control = ad3532r_ref_ctrl,
+	.op_mode = ad3532r_op_mode,
+	.num_banks = ARRAY_SIZE(ad3532r_if_config),
+	.num_op_mode_regs = ARRAY_SIZE(ad3532r_op_mode),
 	.internal_ref_support = true,
 };
 
+static int ad3530r_set_reg_bank_bits(const struct ad3530r_state *st,
+				     const unsigned int *regs,
+				     unsigned int num_regs,
+				     unsigned int mask)
+{
+	int ret;
+
+	for (unsigned int i = 0; i < num_regs; i++) {
+		ret = regmap_set_bits(st->regmap, regs[i], mask);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int ad3530r_write_reg_banks(const struct ad3530r_state *st,
+				   const unsigned int *regs,
+				   unsigned int num_regs,
+				   unsigned int val)
+{
+	int ret;
+
+	for (unsigned int i = 0; i < num_regs; i++) {
+		ret = regmap_write(st->regmap, regs[i], val);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 static int ad3530r_setup(struct ad3530r_state *st, int external_vref_uV)
 {
+	const struct ad3530r_chip_info *chip_info = st->chip_info;
 	struct device *dev = regmap_get_device(st->regmap);
 	struct gpio_desc *reset_gpio;
-	int i, ret;
 	u8 range_multiplier, val;
+	int ret;
 
 	reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
 	if (IS_ERR(reset_gpio))
@@ -384,8 +670,9 @@ static int ad3530r_setup(struct ad3530r_state *st, int external_vref_uV)
 		gpiod_set_value_cansleep(reset_gpio, 0);
 	} else {
 		/* Perform software reset */
-		ret = regmap_update_bits(st->regmap, AD3530R_INTERFACE_CONFIG_A,
-					 AD3530R_SW_RESET, AD3530R_SW_RESET);
+		ret = ad3530r_set_reg_bank_bits(st, chip_info->interface_config_a,
+						chip_info->num_banks,
+						AD3530R_SW_RESET);
 		if (ret)
 			return ret;
 	}
@@ -394,8 +681,9 @@ static int ad3530r_setup(struct ad3530r_state *st, int external_vref_uV)
 
 	range_multiplier = 1;
 	if (device_property_read_bool(dev, "adi,range-double")) {
-		ret = regmap_set_bits(st->regmap, AD3530R_OUTPUT_CONTROL_0,
-				      AD3530R_OUTPUT_CONTROL_RANGE);
+		ret = ad3530r_set_reg_bank_bits(st, chip_info->output_control,
+						chip_info->num_banks,
+						AD3530R_OUTPUT_CONTROL_RANGE);
 		if (ret)
 			return ret;
 
@@ -405,8 +693,9 @@ static int ad3530r_setup(struct ad3530r_state *st, int external_vref_uV)
 	if (external_vref_uV) {
 		st->vref_mV = range_multiplier * external_vref_uV / MILLI;
 	} else {
-		ret = regmap_set_bits(st->regmap, AD3530R_REFERENCE_CONTROL_0,
-				      AD3530R_REFERENCE_CONTROL_SEL);
+		ret = ad3530r_set_reg_bank_bits(st, chip_info->reference_control,
+						chip_info->num_banks,
+						AD3530R_REFERENCE_CONTROL_SEL);
 		if (ret)
 			return ret;
 
@@ -419,18 +708,12 @@ static int ad3530r_setup(struct ad3530r_state *st, int external_vref_uV)
 	      FIELD_PREP(AD3530R_OP_MODE_CHAN_MSK(2), AD3530R_NORMAL_OP) |
 	      FIELD_PREP(AD3530R_OP_MODE_CHAN_MSK(3), AD3530R_NORMAL_OP);
 
-	ret = regmap_write(st->regmap, AD3530R_OUTPUT_OPERATING_MODE_0, val);
+	ret = ad3530r_write_reg_banks(st, st->chip_info->op_mode,
+				      st->chip_info->num_op_mode_regs, val);
 	if (ret)
 		return ret;
 
-	if (st->chip_info->num_channels > 4) {
-		ret = regmap_write(st->regmap, AD3530R_OUTPUT_OPERATING_MODE_1,
-				   val);
-		if (ret)
-			return ret;
-	}
-
-	for (i = 0; i < st->chip_info->num_channels; i++)
+	for (unsigned int i = 0; i < st->chip_info->num_channels; i++)
 		st->chan[i].powerdown_mode = AD3530R_POWERDOWN_32K;
 
 	st->ldac_gpio = devm_gpiod_get_optional(dev, "ldac", GPIOD_OUT_LOW);
@@ -441,12 +724,6 @@ static int ad3530r_setup(struct ad3530r_state *st, int external_vref_uV)
 	return 0;
 }
 
-static const struct regmap_config ad3530r_regmap_config = {
-	.reg_bits = 16,
-	.val_bits = 8,
-	.max_register = AD3530R_MAX_REG_ADDR,
-};
-
 static const struct iio_info ad3530r_info = {
 	.read_raw = ad3530r_read_raw,
 	.write_raw = ad3530r_write_raw,
@@ -467,7 +744,11 @@ static int ad3530r_probe(struct spi_device *spi)
 
 	st = iio_priv(indio_dev);
 
-	st->regmap = devm_regmap_init_spi(spi, &ad3530r_regmap_config);
+	st->chip_info = spi_get_device_match_data(spi);
+	if (!st->chip_info)
+		return -ENODEV;
+
+	st->regmap = devm_regmap_init_spi(spi, st->chip_info->regmap_config);
 	if (IS_ERR(st->regmap))
 		return dev_err_probe(dev, PTR_ERR(st->regmap),
 				     "Failed to init regmap");
@@ -476,10 +757,6 @@ static int ad3530r_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	st->chip_info = spi_get_device_match_data(spi);
-	if (!st->chip_info)
-		return -ENODEV;
-
 	ret = devm_regulator_bulk_get_enable(dev, ARRAY_SIZE(regulators),
 					     regulators);
 	if (ret)
@@ -509,10 +786,12 @@ static int ad3530r_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad3530r_id[] = {
-	{ "ad3530", (kernel_ulong_t)&ad3530_chip },
-	{ "ad3530r", (kernel_ulong_t)&ad3530r_chip },
-	{ "ad3531", (kernel_ulong_t)&ad3531_chip },
-	{ "ad3531r", (kernel_ulong_t)&ad3531r_chip },
+	{ .name = "ad3530", .driver_data = (kernel_ulong_t)&ad3530_chip },
+	{ .name = "ad3530r", .driver_data = (kernel_ulong_t)&ad3530r_chip },
+	{ .name = "ad3531", .driver_data = (kernel_ulong_t)&ad3531_chip },
+	{ .name = "ad3531r", .driver_data = (kernel_ulong_t)&ad3531r_chip },
+	{ .name = "ad3532", .driver_data = (kernel_ulong_t)&ad3532_chip },
+	{ .name = "ad3532r", .driver_data = (kernel_ulong_t)&ad3532r_chip },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad3530r_id);
@@ -522,6 +801,8 @@ static const struct of_device_id ad3530r_of_match[] = {
 	{ .compatible = "adi,ad3530r", .data = &ad3530r_chip },
 	{ .compatible = "adi,ad3531", .data = &ad3531_chip },
 	{ .compatible = "adi,ad3531r", .data = &ad3531r_chip },
+	{ .compatible = "adi,ad3532", .data = &ad3532_chip },
+	{ .compatible = "adi,ad3532r", .data = &ad3532r_chip },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, ad3530r_of_match);
diff --git a/drivers/iio/dac/ad5064.c b/drivers/iio/dac/ad5064.c
index b5ec33f5c3170..298f3a7035714 100644
--- a/drivers/iio/dac/ad5064.c
+++ b/drivers/iio/dac/ad5064.c
@@ -920,22 +920,22 @@ static int ad5064_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad5064_spi_ids[] = {
-	{"ad5024", ID_AD5024},
-	{"ad5025", ID_AD5025},
-	{"ad5044", ID_AD5044},
-	{"ad5045", ID_AD5045},
-	{"ad5064", ID_AD5064},
-	{"ad5064-1", ID_AD5064_1},
-	{"ad5065", ID_AD5065},
-	{"ad5628-1", ID_AD5628_1},
-	{"ad5628-2", ID_AD5628_2},
-	{"ad5648-1", ID_AD5648_1},
-	{"ad5648-2", ID_AD5648_2},
-	{"ad5666-1", ID_AD5666_1},
-	{"ad5666-2", ID_AD5666_2},
-	{"ad5668-1", ID_AD5668_1},
-	{"ad5668-2", ID_AD5668_2},
-	{"ad5668-3", ID_AD5668_2}, /* similar enough to ad5668-2 */
+	{ .name = "ad5024", .driver_data = ID_AD5024 },
+	{ .name = "ad5025", .driver_data = ID_AD5025 },
+	{ .name = "ad5044", .driver_data = ID_AD5044 },
+	{ .name = "ad5045", .driver_data = ID_AD5045 },
+	{ .name = "ad5064", .driver_data = ID_AD5064 },
+	{ .name = "ad5064-1", .driver_data = ID_AD5064_1 },
+	{ .name = "ad5065", .driver_data = ID_AD5065 },
+	{ .name = "ad5628-1", .driver_data = ID_AD5628_1 },
+	{ .name = "ad5628-2", .driver_data = ID_AD5628_2 },
+	{ .name = "ad5648-1", .driver_data = ID_AD5648_1 },
+	{ .name = "ad5648-2", .driver_data = ID_AD5648_2 },
+	{ .name = "ad5666-1", .driver_data = ID_AD5666_1 },
+	{ .name = "ad5666-2", .driver_data = ID_AD5666_2 },
+	{ .name = "ad5668-1", .driver_data = ID_AD5668_1 },
+	{ .name = "ad5668-2", .driver_data = ID_AD5668_2 },
+	{ .name = "ad5668-3", .driver_data = ID_AD5668_2 }, /* similar enough to ad5668-2 */
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad5064_spi_ids);
diff --git a/drivers/iio/dac/ad5360.c b/drivers/iio/dac/ad5360.c
index 20316fd568e62..c1553f18a26ad 100644
--- a/drivers/iio/dac/ad5360.c
+++ b/drivers/iio/dac/ad5360.c
@@ -523,14 +523,14 @@ static void ad5360_remove(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad5360_ids[] = {
-	{ "ad5360", ID_AD5360 },
-	{ "ad5361", ID_AD5361 },
-	{ "ad5362", ID_AD5362 },
-	{ "ad5363", ID_AD5363 },
-	{ "ad5370", ID_AD5370 },
-	{ "ad5371", ID_AD5371 },
-	{ "ad5372", ID_AD5372 },
-	{ "ad5373", ID_AD5373 },
+	{ .name = "ad5360", .driver_data = ID_AD5360 },
+	{ .name = "ad5361", .driver_data = ID_AD5361 },
+	{ .name = "ad5362", .driver_data = ID_AD5362 },
+	{ .name = "ad5363", .driver_data = ID_AD5363 },
+	{ .name = "ad5370", .driver_data = ID_AD5370 },
+	{ .name = "ad5371", .driver_data = ID_AD5371 },
+	{ .name = "ad5372", .driver_data = ID_AD5372 },
+	{ .name = "ad5373", .driver_data = ID_AD5373 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad5360_ids);
diff --git a/drivers/iio/dac/ad5380.c b/drivers/iio/dac/ad5380.c
index 2e587bdd32143..4a5ad419f4eaa 100644
--- a/drivers/iio/dac/ad5380.c
+++ b/drivers/iio/dac/ad5380.c
@@ -446,22 +446,22 @@ static int ad5380_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad5380_spi_ids[] = {
-	{ "ad5380-3", ID_AD5380_3 },
-	{ "ad5380-5", ID_AD5380_5 },
-	{ "ad5381-3", ID_AD5381_3 },
-	{ "ad5381-5", ID_AD5381_5 },
-	{ "ad5382-3", ID_AD5382_3 },
-	{ "ad5382-5", ID_AD5382_5 },
-	{ "ad5383-3", ID_AD5383_3 },
-	{ "ad5383-5", ID_AD5383_5 },
-	{ "ad5384-3", ID_AD5380_3 },
-	{ "ad5384-5", ID_AD5380_5 },
-	{ "ad5390-3", ID_AD5390_3 },
-	{ "ad5390-5", ID_AD5390_5 },
-	{ "ad5391-3", ID_AD5391_3 },
-	{ "ad5391-5", ID_AD5391_5 },
-	{ "ad5392-3", ID_AD5392_3 },
-	{ "ad5392-5", ID_AD5392_5 },
+	{ .name = "ad5380-3", .driver_data = ID_AD5380_3 },
+	{ .name = "ad5380-5", .driver_data = ID_AD5380_5 },
+	{ .name = "ad5381-3", .driver_data = ID_AD5381_3 },
+	{ .name = "ad5381-5", .driver_data = ID_AD5381_5 },
+	{ .name = "ad5382-3", .driver_data = ID_AD5382_3 },
+	{ .name = "ad5382-5", .driver_data = ID_AD5382_5 },
+	{ .name = "ad5383-3", .driver_data = ID_AD5383_3 },
+	{ .name = "ad5383-5", .driver_data = ID_AD5383_5 },
+	{ .name = "ad5384-3", .driver_data = ID_AD5380_3 },
+	{ .name = "ad5384-5", .driver_data = ID_AD5380_5 },
+	{ .name = "ad5390-3", .driver_data = ID_AD5390_3 },
+	{ .name = "ad5390-5", .driver_data = ID_AD5390_5 },
+	{ .name = "ad5391-3", .driver_data = ID_AD5391_3 },
+	{ .name = "ad5391-5", .driver_data = ID_AD5391_5 },
+	{ .name = "ad5392-3", .driver_data = ID_AD5392_3 },
+	{ .name = "ad5392-5", .driver_data = ID_AD5392_5 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad5380_spi_ids);
diff --git a/drivers/iio/dac/ad5446-spi.c b/drivers/iio/dac/ad5446-spi.c
index 54bd0e113f40d..85bb8829046ae 100644
--- a/drivers/iio/dac/ad5446-spi.c
+++ b/drivers/iio/dac/ad5446-spi.c
@@ -163,38 +163,38 @@ static const struct ad5446_chip_info ad5662_chip_info = {
 };
 
 static const struct spi_device_id ad5446_spi_ids[] = {
-	{"ad5300", (kernel_ulong_t)&ad5300_chip_info},
-	{"ad5310", (kernel_ulong_t)&ad5310_chip_info},
-	{"ad5320", (kernel_ulong_t)&ad5320_chip_info},
-	{"ad5444", (kernel_ulong_t)&ad5444_chip_info},
-	{"ad5446", (kernel_ulong_t)&ad5446_chip_info},
-	{"ad5450", (kernel_ulong_t)&ad5450_chip_info},
-	{"ad5451", (kernel_ulong_t)&ad5451_chip_info},
-	{"ad5452", (kernel_ulong_t)&ad5444_chip_info}, /* ad5452 is compatible to the ad5444 */
-	{"ad5453", (kernel_ulong_t)&ad5446_chip_info}, /* ad5453 is compatible to the ad5446 */
-	{"ad5512a", (kernel_ulong_t)&ad5512a_chip_info},
-	{"ad5541a", (kernel_ulong_t)&ad5541a_chip_info},
-	{"ad5542", (kernel_ulong_t)&ad5541a_chip_info}, /* ad5541a and ad5542 are compatible */
-	{"ad5542a", (kernel_ulong_t)&ad5541a_chip_info}, /* ad5541a and ad5542a are compatible */
-	{"ad5543", (kernel_ulong_t)&ad5541a_chip_info}, /* ad5541a and ad5543 are compatible */
-	{"ad5553", (kernel_ulong_t)&ad5553_chip_info},
-	{"ad5600", (kernel_ulong_t)&ad5541a_chip_info}, /* ad5541a and ad5600 are compatible */
-	{"ad5601", (kernel_ulong_t)&ad5601_chip_info},
-	{"ad5611", (kernel_ulong_t)&ad5611_chip_info},
-	{"ad5621", (kernel_ulong_t)&ad5621_chip_info},
-	{"ad5641", (kernel_ulong_t)&ad5641_chip_info},
-	{"ad5620-2500", (kernel_ulong_t)&ad5620_2500_chip_info}, /* AD5620/40/60: */
+	{ .name = "ad5300", .driver_data = (kernel_ulong_t)&ad5300_chip_info },
+	{ .name = "ad5310", .driver_data = (kernel_ulong_t)&ad5310_chip_info },
+	{ .name = "ad5320", .driver_data = (kernel_ulong_t)&ad5320_chip_info },
+	{ .name = "ad5444", .driver_data = (kernel_ulong_t)&ad5444_chip_info },
+	{ .name = "ad5446", .driver_data = (kernel_ulong_t)&ad5446_chip_info },
+	{ .name = "ad5450", .driver_data = (kernel_ulong_t)&ad5450_chip_info },
+	{ .name = "ad5451", .driver_data = (kernel_ulong_t)&ad5451_chip_info },
+	{ .name = "ad5452", .driver_data = (kernel_ulong_t)&ad5444_chip_info }, /* ad5452 is compatible to the ad5444 */
+	{ .name = "ad5453", .driver_data = (kernel_ulong_t)&ad5446_chip_info }, /* ad5453 is compatible to the ad5446 */
+	{ .name = "ad5512a", .driver_data = (kernel_ulong_t)&ad5512a_chip_info },
+	{ .name = "ad5541a", .driver_data = (kernel_ulong_t)&ad5541a_chip_info },
+	{ .name = "ad5542", .driver_data = (kernel_ulong_t)&ad5541a_chip_info }, /* ad5541a and ad5542 are compatible */
+	{ .name = "ad5542a", .driver_data = (kernel_ulong_t)&ad5541a_chip_info }, /* ad5541a and ad5542a are compatible */
+	{ .name = "ad5543", .driver_data = (kernel_ulong_t)&ad5541a_chip_info }, /* ad5541a and ad5543 are compatible */
+	{ .name = "ad5553", .driver_data = (kernel_ulong_t)&ad5553_chip_info },
+	{ .name = "ad5600", .driver_data = (kernel_ulong_t)&ad5541a_chip_info }, /* ad5541a and ad5600 are compatible */
+	{ .name = "ad5601", .driver_data = (kernel_ulong_t)&ad5601_chip_info },
+	{ .name = "ad5611", .driver_data = (kernel_ulong_t)&ad5611_chip_info },
+	{ .name = "ad5621", .driver_data = (kernel_ulong_t)&ad5621_chip_info },
+	{ .name = "ad5641", .driver_data = (kernel_ulong_t)&ad5641_chip_info },
+	{ .name = "ad5620-2500", .driver_data = (kernel_ulong_t)&ad5620_2500_chip_info }, /* AD5620/40/60: */
 	/* part numbers may look differently */
-	{"ad5620-1250", (kernel_ulong_t)&ad5620_1250_chip_info},
-	{"ad5640-2500", (kernel_ulong_t)&ad5640_2500_chip_info},
-	{"ad5640-1250", (kernel_ulong_t)&ad5640_1250_chip_info},
-	{"ad5660-2500", (kernel_ulong_t)&ad5660_2500_chip_info},
-	{"ad5660-1250", (kernel_ulong_t)&ad5660_1250_chip_info},
-	{"ad5662", (kernel_ulong_t)&ad5662_chip_info},
-	{"dac081s101", (kernel_ulong_t)&ad5300_chip_info}, /* compatible Texas Instruments chips */
-	{"dac101s101", (kernel_ulong_t)&ad5310_chip_info},
-	{"dac121s101", (kernel_ulong_t)&ad5320_chip_info},
-	{"dac7512", (kernel_ulong_t)&ad5320_chip_info},
+	{ .name = "ad5620-1250", .driver_data = (kernel_ulong_t)&ad5620_1250_chip_info },
+	{ .name = "ad5640-2500", .driver_data = (kernel_ulong_t)&ad5640_2500_chip_info },
+	{ .name = "ad5640-1250", .driver_data = (kernel_ulong_t)&ad5640_1250_chip_info },
+	{ .name = "ad5660-2500", .driver_data = (kernel_ulong_t)&ad5660_2500_chip_info },
+	{ .name = "ad5660-1250", .driver_data = (kernel_ulong_t)&ad5660_1250_chip_info },
+	{ .name = "ad5662", .driver_data = (kernel_ulong_t)&ad5662_chip_info },
+	{ .name = "dac081s101", .driver_data = (kernel_ulong_t)&ad5300_chip_info }, /* compatible Texas Instruments chips */
+	{ .name = "dac101s101", .driver_data = (kernel_ulong_t)&ad5310_chip_info },
+	{ .name = "dac121s101", .driver_data = (kernel_ulong_t)&ad5320_chip_info },
+	{ .name = "dac7512", .driver_data = (kernel_ulong_t)&ad5320_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad5446_spi_ids);
diff --git a/drivers/iio/dac/ad5449.c b/drivers/iio/dac/ad5449.c
index d8c3252602593..19e66bc696766 100644
--- a/drivers/iio/dac/ad5449.c
+++ b/drivers/iio/dac/ad5449.c
@@ -330,13 +330,13 @@ static void ad5449_spi_remove(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad5449_spi_ids[] = {
-	{ "ad5415", ID_AD5449 },
-	{ "ad5426", ID_AD5426 },
-	{ "ad5429", ID_AD5429 },
-	{ "ad5432", ID_AD5432 },
-	{ "ad5439", ID_AD5439 },
-	{ "ad5443", ID_AD5443 },
-	{ "ad5449", ID_AD5449 },
+	{ .name = "ad5415", .driver_data = ID_AD5449 },
+	{ .name = "ad5426", .driver_data = ID_AD5426 },
+	{ .name = "ad5429", .driver_data = ID_AD5429 },
+	{ .name = "ad5432", .driver_data = ID_AD5432 },
+	{ .name = "ad5439", .driver_data = ID_AD5439 },
+	{ .name = "ad5443", .driver_data = ID_AD5443 },
+	{ .name = "ad5449", .driver_data = ID_AD5449 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad5449_spi_ids);
diff --git a/drivers/iio/dac/ad5504.c b/drivers/iio/dac/ad5504.c
index 5e586185d8579..ad1a74678cf21 100644
--- a/drivers/iio/dac/ad5504.c
+++ b/drivers/iio/dac/ad5504.c
@@ -319,8 +319,8 @@ static int ad5504_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad5504_id[] = {
-	{"ad5504", ID_AD5504},
-	{"ad5501", ID_AD5501},
+	{ .name = "ad5504", .driver_data = ID_AD5504 },
+	{ .name = "ad5501", .driver_data = ID_AD5501 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad5504_id);
diff --git a/drivers/iio/dac/ad5624r_spi.c b/drivers/iio/dac/ad5624r_spi.c
index 13aefe769bad8..56aea4b1c167c 100644
--- a/drivers/iio/dac/ad5624r_spi.c
+++ b/drivers/iio/dac/ad5624r_spi.c
@@ -260,12 +260,12 @@ static int ad5624r_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad5624r_id[] = {
-	{"ad5624r3", ID_AD5624R3},
-	{"ad5644r3", ID_AD5644R3},
-	{"ad5664r3", ID_AD5664R3},
-	{"ad5624r5", ID_AD5624R5},
-	{"ad5644r5", ID_AD5644R5},
-	{"ad5664r5", ID_AD5664R5},
+	{ .name = "ad5624r3", .driver_data = ID_AD5624R3 },
+	{ .name = "ad5644r3", .driver_data = ID_AD5644R3 },
+	{ .name = "ad5664r3", .driver_data = ID_AD5664R3 },
+	{ .name = "ad5624r5", .driver_data = ID_AD5624R5 },
+	{ .name = "ad5644r5", .driver_data = ID_AD5644R5 },
+	{ .name = "ad5664r5", .driver_data = ID_AD5664R5 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad5624r_id);
diff --git a/drivers/iio/dac/ad5686.c b/drivers/iio/dac/ad5686.c
index 5840fda4b0118..713fe71ad1e7b 100644
--- a/drivers/iio/dac/ad5686.c
+++ b/drivers/iio/dac/ad5686.c
@@ -8,11 +8,15 @@
 #include <linux/array_size.h>
 #include <linux/bitfield.h>
 #include <linux/bitops.h>
+#include <linux/delay.h>
+#include <linux/dev_printk.h>
 #include <linux/errno.h>
 #include <linux/export.h>
+#include <linux/gpio/consumer.h>
 #include <linux/kstrtox.h>
 #include <linux/module.h>
 #include <linux/regulator/consumer.h>
+#include <linux/reset.h>
 #include <linux/sysfs.h>
 #include <linux/wordpart.h>
 
@@ -470,6 +474,7 @@ int ad5686_probe(struct device *dev,
 		 const struct ad5686_chip_info *chip_info,
 		 const char *name, const struct ad5686_bus_ops *ops)
 {
+	struct reset_control *rstc;
 	struct ad5686_state *st;
 	struct iio_dev *indio_dev;
 	int ret, i;
@@ -484,12 +489,43 @@ int ad5686_probe(struct device *dev,
 	st->ops = ops;
 	st->chip_info = chip_info;
 
-	ret = devm_regulator_get_enable_read_voltage(dev, "vcc");
+	rstc = devm_reset_control_get_optional_exclusive(dev, NULL);
+	if (IS_ERR(rstc))
+		return dev_err_probe(dev, PTR_ERR(rstc),
+				     "Failed to get reset control\n");
+
+	ret = devm_regulator_get_enable(dev, "vdd");
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to enable vdd supply\n");
+
+	ret = devm_regulator_get_enable(dev, "vlogic");
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to enable vlogic supply\n");
+
+	ret = devm_regulator_get_enable_read_voltage(dev, "vref");
+	if (ret == -ENODEV) /* vcc-supply is deprecated, but supported still */
+		ret = devm_regulator_get_enable_read_voltage(dev, "vcc");
 	if (ret < 0 && ret != -ENODEV)
-		return ret;
+		return dev_err_probe(dev, ret, "failed to read vref voltage\n");
 
 	st->use_internal_vref = ret == -ENODEV;
 	st->vref_mv = st->use_internal_vref ? st->chip_info->int_vref_mv : ret / 1000;
+	if (!st->vref_mv)
+		return dev_err_probe(dev, -EINVAL,
+				     "invalid or not provided vref voltage\n");
+
+	/* 4.5us power-up time: Datasheet Table 4: Timing Characteristics */
+	fsleep(5);
+
+	/* 1us >> 30ns reset pulse activation time: Datasheet Table 4 */
+	reset_control_assert(rstc);
+	fsleep(1);
+	reset_control_deassert(rstc);
+
+	st->ldac_gpio = devm_gpiod_get_optional(dev, "ldac", GPIOD_OUT_HIGH);
+	if (IS_ERR(st->ldac_gpio))
+		return dev_err_probe(dev, PTR_ERR(st->ldac_gpio),
+				     "Failed to get LDAC GPIO\n");
 
 	/* Initialize masks to all ones */
 	st->pwr_down_mask = ~0;
diff --git a/drivers/iio/dac/ad5686.h b/drivers/iio/dac/ad5686.h
index a06fe7d893057..c424720f8f72f 100644
--- a/drivers/iio/dac/ad5686.h
+++ b/drivers/iio/dac/ad5686.h
@@ -60,6 +60,8 @@ enum ad5686_regmap_type {
 	AD5686_REGMAP,
 };
 
+struct gpio_desc;
+
 struct ad5686_state;
 
 /**
@@ -119,6 +121,7 @@ extern const struct ad5686_chip_info ad5679r_chip_info;
  * @dev:		device instance
  * @chip_info:		chip model specific constants, available modes etc
  * @ops:		bus specific operations
+ * @ldac_gpio:		LDAC pin GPIO descriptor
  * @vref_mv:		actual reference voltage used
  * @pwr_down_mask:	power down mask
  * @pwr_down_mode:	current power down mode
@@ -131,6 +134,7 @@ struct ad5686_state {
 	struct device			*dev;
 	const struct ad5686_chip_info	*chip_info;
 	const struct ad5686_bus_ops	*ops;
+	struct gpio_desc		*ldac_gpio;
 	unsigned short			vref_mv;
 	unsigned int			pwr_down_mask;
 	unsigned int			pwr_down_mode;
diff --git a/drivers/iio/dac/ad5706r.c b/drivers/iio/dac/ad5706r.c
index e4e48ca7ad34b..06c4860a7ebac 100644
--- a/drivers/iio/dac/ad5706r.c
+++ b/drivers/iio/dac/ad5706r.c
@@ -232,7 +232,7 @@ static const struct of_device_id ad5706r_of_match[] = {
 MODULE_DEVICE_TABLE(of, ad5706r_of_match);
 
 static const struct spi_device_id ad5706r_id[] = {
-	{ "ad5706r" },
+	{ .name = "ad5706r" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad5706r_id);
diff --git a/drivers/iio/dac/ad5755.c b/drivers/iio/dac/ad5755.c
index cc6d56140d667..0828b99090990 100644
--- a/drivers/iio/dac/ad5755.c
+++ b/drivers/iio/dac/ad5755.c
@@ -849,11 +849,11 @@ static int ad5755_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad5755_id[] = {
-	{ "ad5755", (kernel_ulong_t)&ad5755_chip_info_tbl[ID_AD5755] },
-	{ "ad5755-1", (kernel_ulong_t)&ad5755_chip_info_tbl[ID_AD5755] },
-	{ "ad5757", (kernel_ulong_t)&ad5755_chip_info_tbl[ID_AD5757] },
-	{ "ad5735", (kernel_ulong_t)&ad5755_chip_info_tbl[ID_AD5735] },
-	{ "ad5737", (kernel_ulong_t)&ad5755_chip_info_tbl[ID_AD5737] },
+	{ .name = "ad5755", .driver_data = (kernel_ulong_t)&ad5755_chip_info_tbl[ID_AD5755] },
+	{ .name = "ad5755-1", .driver_data = (kernel_ulong_t)&ad5755_chip_info_tbl[ID_AD5755] },
+	{ .name = "ad5757", .driver_data = (kernel_ulong_t)&ad5755_chip_info_tbl[ID_AD5757] },
+	{ .name = "ad5735", .driver_data = (kernel_ulong_t)&ad5755_chip_info_tbl[ID_AD5735] },
+	{ .name = "ad5737", .driver_data = (kernel_ulong_t)&ad5755_chip_info_tbl[ID_AD5737] },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad5755_id);
diff --git a/drivers/iio/dac/ad5758.c b/drivers/iio/dac/ad5758.c
index bb30842e70803..44efa373cd80a 100644
--- a/drivers/iio/dac/ad5758.c
+++ b/drivers/iio/dac/ad5758.c
@@ -878,7 +878,7 @@ static int ad5758_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad5758_id[] = {
-	{ "ad5758", 0 },
+	{ .name = "ad5758" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad5758_id);
diff --git a/drivers/iio/dac/ad5761.c b/drivers/iio/dac/ad5761.c
index b5d20f04f070a..c49568b943d6f 100644
--- a/drivers/iio/dac/ad5761.c
+++ b/drivers/iio/dac/ad5761.c
@@ -342,10 +342,10 @@ static int ad5761_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad5761_id[] = {
-	{"ad5721", ID_AD5721},
-	{"ad5721r", ID_AD5721R},
-	{"ad5761", ID_AD5761},
-	{"ad5761r", ID_AD5761R},
+	{ .name = "ad5721", .driver_data = ID_AD5721 },
+	{ .name = "ad5721r", .driver_data = ID_AD5721R },
+	{ .name = "ad5761", .driver_data = ID_AD5761 },
+	{ .name = "ad5761r", .driver_data = ID_AD5761R },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad5761_id);
diff --git a/drivers/iio/dac/ad5764.c b/drivers/iio/dac/ad5764.c
index fbbd7105a80c7..2be4bee7eb013 100644
--- a/drivers/iio/dac/ad5764.c
+++ b/drivers/iio/dac/ad5764.c
@@ -342,10 +342,10 @@ static void ad5764_remove(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad5764_ids[] = {
-	{ "ad5744", ID_AD5744 },
-	{ "ad5744r", ID_AD5744R },
-	{ "ad5764", ID_AD5764 },
-	{ "ad5764r", ID_AD5764R },
+	{ .name = "ad5744", .driver_data = ID_AD5744 },
+	{ .name = "ad5744r", .driver_data = ID_AD5744R },
+	{ .name = "ad5764", .driver_data = ID_AD5764 },
+	{ .name = "ad5764r", .driver_data = ID_AD5764R },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad5764_ids);
diff --git a/drivers/iio/dac/ad5766.c b/drivers/iio/dac/ad5766.c
index f6a0a0d84fefd..061b43fc03450 100644
--- a/drivers/iio/dac/ad5766.c
+++ b/drivers/iio/dac/ad5766.c
@@ -651,8 +651,8 @@ static const struct of_device_id ad5766_dt_match[] = {
 MODULE_DEVICE_TABLE(of, ad5766_dt_match);
 
 static const struct spi_device_id ad5766_spi_ids[] = {
-	{ "ad5766", ID_AD5766 },
-	{ "ad5767", ID_AD5767 },
+	{ .name = "ad5766", .driver_data = ID_AD5766 },
+	{ .name = "ad5767", .driver_data = ID_AD5767 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad5766_spi_ids);
diff --git a/drivers/iio/dac/ad5770r.c b/drivers/iio/dac/ad5770r.c
index 6027e8d88b278..bdc48e57013f4 100644
--- a/drivers/iio/dac/ad5770r.c
+++ b/drivers/iio/dac/ad5770r.c
@@ -642,7 +642,7 @@ static const struct of_device_id ad5770r_of_id[] = {
 MODULE_DEVICE_TABLE(of, ad5770r_of_id);
 
 static const struct spi_device_id ad5770r_id[] = {
-	{ "ad5770r", 0 },
+	{ .name = "ad5770r" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad5770r_id);
diff --git a/drivers/iio/dac/ad5791.c b/drivers/iio/dac/ad5791.c
index ae7297f083983..a5eda135c9bd2 100644
--- a/drivers/iio/dac/ad5791.c
+++ b/drivers/iio/dac/ad5791.c
@@ -586,11 +586,11 @@ static const struct of_device_id ad5791_of_match[] = {
 MODULE_DEVICE_TABLE(of, ad5791_of_match);
 
 static const struct spi_device_id ad5791_id[] = {
-	{ "ad5760", (kernel_ulong_t)&ad5760_chip_info },
-	{ "ad5780", (kernel_ulong_t)&ad5780_chip_info },
-	{ "ad5781", (kernel_ulong_t)&ad5781_chip_info },
-	{ "ad5790", (kernel_ulong_t)&ad5790_chip_info },
-	{ "ad5791", (kernel_ulong_t)&ad5791_chip_info },
+	{ .name = "ad5760", .driver_data = (kernel_ulong_t)&ad5760_chip_info },
+	{ .name = "ad5780", .driver_data = (kernel_ulong_t)&ad5780_chip_info },
+	{ .name = "ad5781", .driver_data = (kernel_ulong_t)&ad5781_chip_info },
+	{ .name = "ad5790", .driver_data = (kernel_ulong_t)&ad5790_chip_info },
+	{ .name = "ad5791", .driver_data = (kernel_ulong_t)&ad5791_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad5791_id);
diff --git a/drivers/iio/dac/ad7293.c b/drivers/iio/dac/ad7293.c
index 03acf7c114b04..33a49813ece06 100644
--- a/drivers/iio/dac/ad7293.c
+++ b/drivers/iio/dac/ad7293.c
@@ -869,7 +869,7 @@ static int ad7293_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad7293_id[] = {
-	{ "ad7293", 0 },
+	{ .name = "ad7293" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7293_id);
diff --git a/drivers/iio/dac/ad7303.c b/drivers/iio/dac/ad7303.c
index 6451fc586a67e..83153e7d7fc8d 100644
--- a/drivers/iio/dac/ad7303.c
+++ b/drivers/iio/dac/ad7303.c
@@ -270,7 +270,7 @@ static const struct of_device_id ad7303_spi_of_match[] = {
 MODULE_DEVICE_TABLE(of, ad7303_spi_of_match);
 
 static const struct spi_device_id ad7303_spi_ids[] = {
-	{ "ad7303", 0 },
+	{ .name = "ad7303" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad7303_spi_ids);
diff --git a/drivers/iio/dac/ad8460.c b/drivers/iio/dac/ad8460.c
index ddec62b6e57bd..a2292810e4cff 100644
--- a/drivers/iio/dac/ad8460.c
+++ b/drivers/iio/dac/ad8460.c
@@ -954,3 +954,4 @@ MODULE_AUTHOR("Mariel Tinaco <mariel.tinaco@analog.com");
 MODULE_DESCRIPTION("AD8460 DAC driver");
 MODULE_LICENSE("GPL");
 MODULE_IMPORT_NS("IIO_DMAENGINE_BUFFER");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/iio/dac/ad8801.c b/drivers/iio/dac/ad8801.c
index 60e663af1cc18..08d7d74e999ea 100644
--- a/drivers/iio/dac/ad8801.c
+++ b/drivers/iio/dac/ad8801.c
@@ -151,8 +151,8 @@ static int ad8801_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad8801_ids[] = {
-	{"ad8801", ID_AD8801},
-	{"ad8803", ID_AD8803},
+	{ .name = "ad8801", .driver_data = ID_AD8801 },
+	{ .name = "ad8803", .driver_data = ID_AD8803 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad8801_ids);
diff --git a/drivers/iio/dac/ad9739a.c b/drivers/iio/dac/ad9739a.c
index ccd6a3b1e891a..5263a0c968f56 100644
--- a/drivers/iio/dac/ad9739a.c
+++ b/drivers/iio/dac/ad9739a.c
@@ -446,7 +446,7 @@ static const struct of_device_id ad9739a_of_match[] = {
 MODULE_DEVICE_TABLE(of, ad9739a_of_match);
 
 static const struct spi_device_id ad9739a_id[] = {
-	{"ad9739a"},
+	{ .name = "ad9739a" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad9739a_id);
diff --git a/drivers/iio/dac/dpot-dac.c b/drivers/iio/dac/dpot-dac.c
index cf6d94e7af847..de36e2e27330f 100644
--- a/drivers/iio/dac/dpot-dac.c
+++ b/drivers/iio/dac/dpot-dac.c
@@ -253,3 +253,4 @@ module_platform_driver(dpot_dac_driver);
 MODULE_DESCRIPTION("DAC emulation driver using a digital potentiometer");
 MODULE_AUTHOR("Peter Rosin <peda@axentia.se>");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/iio/dac/ltc1660.c b/drivers/iio/dac/ltc1660.c
index 6e80b49f46655..90ffb58e79276 100644
--- a/drivers/iio/dac/ltc1660.c
+++ b/drivers/iio/dac/ltc1660.c
@@ -224,8 +224,8 @@ static const struct of_device_id ltc1660_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, ltc1660_dt_ids);
 
 static const struct spi_device_id ltc1660_id[] = {
-	{"ltc1660", ID_LTC1660},
-	{"ltc1665", ID_LTC1665},
+	{ .name = "ltc1660", .driver_data = ID_LTC1660 },
+	{ .name = "ltc1665", .driver_data = ID_LTC1665 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ltc1660_id);
diff --git a/drivers/iio/dac/ltc2632.c b/drivers/iio/dac/ltc2632.c
index d6a3d290e7a81..3800aee8dd6e4 100644
--- a/drivers/iio/dac/ltc2632.c
+++ b/drivers/iio/dac/ltc2632.c
@@ -362,28 +362,28 @@ static int ltc2632_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ltc2632_id[] = {
-	{ "ltc2632-l12", (kernel_ulong_t)&ltc2632l12_chip_info },
-	{ "ltc2632-l10", (kernel_ulong_t)&ltc2632l10_chip_info },
-	{ "ltc2632-l8",  (kernel_ulong_t)&ltc2632l8_chip_info  },
-	{ "ltc2632-h12", (kernel_ulong_t)&ltc2632h12_chip_info },
-	{ "ltc2632-h10", (kernel_ulong_t)&ltc2632h10_chip_info },
-	{ "ltc2632-h8",  (kernel_ulong_t)&ltc2632h8_chip_info  },
-	{ "ltc2634-l12", (kernel_ulong_t)&ltc2634l12_chip_info },
-	{ "ltc2634-l10", (kernel_ulong_t)&ltc2634l10_chip_info },
-	{ "ltc2634-l8",  (kernel_ulong_t)&ltc2634l8_chip_info  },
-	{ "ltc2634-h12", (kernel_ulong_t)&ltc2634h12_chip_info },
-	{ "ltc2634-h10", (kernel_ulong_t)&ltc2634h10_chip_info },
-	{ "ltc2634-h8",  (kernel_ulong_t)&ltc2634h8_chip_info  },
-	{ "ltc2636-l12", (kernel_ulong_t)&ltc2636l12_chip_info },
-	{ "ltc2636-l10", (kernel_ulong_t)&ltc2636l10_chip_info },
-	{ "ltc2636-l8",  (kernel_ulong_t)&ltc2636l8_chip_info  },
-	{ "ltc2636-h12", (kernel_ulong_t)&ltc2636h12_chip_info },
-	{ "ltc2636-h10", (kernel_ulong_t)&ltc2636h10_chip_info },
-	{ "ltc2636-h8",  (kernel_ulong_t)&ltc2636h8_chip_info  },
-	{ "ltc2654-l16", (kernel_ulong_t)&ltc2654l16_chip_info },
-	{ "ltc2654-l12", (kernel_ulong_t)&ltc2634l12_chip_info },
-	{ "ltc2654-h16", (kernel_ulong_t)&ltc2654h16_chip_info },
-	{ "ltc2654-h12", (kernel_ulong_t)&ltc2634h12_chip_info },
+	{ .name = "ltc2632-l12", .driver_data = (kernel_ulong_t)&ltc2632l12_chip_info },
+	{ .name = "ltc2632-l10", .driver_data = (kernel_ulong_t)&ltc2632l10_chip_info },
+	{ .name = "ltc2632-l8", .driver_data = (kernel_ulong_t)&ltc2632l8_chip_info },
+	{ .name = "ltc2632-h12", .driver_data = (kernel_ulong_t)&ltc2632h12_chip_info },
+	{ .name = "ltc2632-h10", .driver_data = (kernel_ulong_t)&ltc2632h10_chip_info },
+	{ .name = "ltc2632-h8", .driver_data = (kernel_ulong_t)&ltc2632h8_chip_info },
+	{ .name = "ltc2634-l12", .driver_data = (kernel_ulong_t)&ltc2634l12_chip_info },
+	{ .name = "ltc2634-l10", .driver_data = (kernel_ulong_t)&ltc2634l10_chip_info },
+	{ .name = "ltc2634-l8", .driver_data = (kernel_ulong_t)&ltc2634l8_chip_info },
+	{ .name = "ltc2634-h12", .driver_data = (kernel_ulong_t)&ltc2634h12_chip_info },
+	{ .name = "ltc2634-h10", .driver_data = (kernel_ulong_t)&ltc2634h10_chip_info },
+	{ .name = "ltc2634-h8", .driver_data = (kernel_ulong_t)&ltc2634h8_chip_info },
+	{ .name = "ltc2636-l12", .driver_data = (kernel_ulong_t)&ltc2636l12_chip_info },
+	{ .name = "ltc2636-l10", .driver_data = (kernel_ulong_t)&ltc2636l10_chip_info },
+	{ .name = "ltc2636-l8", .driver_data = (kernel_ulong_t)&ltc2636l8_chip_info },
+	{ .name = "ltc2636-h12", .driver_data = (kernel_ulong_t)&ltc2636h12_chip_info },
+	{ .name = "ltc2636-h10", .driver_data = (kernel_ulong_t)&ltc2636h10_chip_info },
+	{ .name = "ltc2636-h8", .driver_data = (kernel_ulong_t)&ltc2636h8_chip_info },
+	{ .name = "ltc2654-l16", .driver_data = (kernel_ulong_t)&ltc2654l16_chip_info },
+	{ .name = "ltc2654-l12", .driver_data = (kernel_ulong_t)&ltc2634l12_chip_info },
+	{ .name = "ltc2654-h16", .driver_data = (kernel_ulong_t)&ltc2654h16_chip_info },
+	{ .name = "ltc2654-h12", .driver_data = (kernel_ulong_t)&ltc2634h12_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ltc2632_id);
diff --git a/drivers/iio/dac/ltc2664.c b/drivers/iio/dac/ltc2664.c
index c48b9efc82804..35bec55ebde1f 100644
--- a/drivers/iio/dac/ltc2664.c
+++ b/drivers/iio/dac/ltc2664.c
@@ -708,8 +708,8 @@ static int ltc2664_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ltc2664_id[] = {
-	{ "ltc2664", (kernel_ulong_t)&ltc2664_chip },
-	{ "ltc2672", (kernel_ulong_t)&ltc2672_chip },
+	{ .name = "ltc2664", .driver_data = (kernel_ulong_t)&ltc2664_chip },
+	{ .name = "ltc2672", .driver_data = (kernel_ulong_t)&ltc2672_chip },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ltc2664_id);
diff --git a/drivers/iio/dac/ltc2688.c b/drivers/iio/dac/ltc2688.c
index a575ef8371dc4..0301bb7dce313 100644
--- a/drivers/iio/dac/ltc2688.c
+++ b/drivers/iio/dac/ltc2688.c
@@ -991,7 +991,7 @@ static const struct of_device_id ltc2688_of_id[] = {
 MODULE_DEVICE_TABLE(of, ltc2688_of_id);
 
 static const struct spi_device_id ltc2688_id[] = {
-	{ "ltc2688" },
+	{ .name = "ltc2688" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ltc2688_id);
diff --git a/drivers/iio/dac/m62332.c b/drivers/iio/dac/m62332.c
index 7e80c0eb5cc1a..60bb672b70e20 100644
--- a/drivers/iio/dac/m62332.c
+++ b/drivers/iio/dac/m62332.c
@@ -32,6 +32,7 @@ static int m62332_set_value(struct iio_dev *indio_dev, u8 val, int channel)
 {
 	struct m62332_data *data = iio_priv(indio_dev);
 	struct i2c_client *client = data->client;
+	bool enabling, disabling;
 	u8 outbuf[2];
 	int res;
 
@@ -43,7 +44,10 @@ static int m62332_set_value(struct iio_dev *indio_dev, u8 val, int channel)
 
 	mutex_lock(&data->mutex);
 
-	if (val) {
+	enabling = val && !data->raw[channel];
+	disabling = !val && data->raw[channel];
+
+	if (enabling) {
 		res = regulator_enable(data->vcc);
 		if (res)
 			goto out;
@@ -52,14 +56,17 @@ static int m62332_set_value(struct iio_dev *indio_dev, u8 val, int channel)
 	res = i2c_master_send(client, outbuf, ARRAY_SIZE(outbuf));
 	if (res >= 0 && res != ARRAY_SIZE(outbuf))
 		res = -EIO;
-	if (res < 0)
+	if (res < 0) {
+		if (enabling)
+			regulator_disable(data->vcc);
 		goto out;
+	}
 
-	data->raw[channel] = val;
-
-	if (!val)
+	if (disabling)
 		regulator_disable(data->vcc);
 
+	data->raw[channel] = val;
+
 	mutex_unlock(&data->mutex);
 
 	return 0;
diff --git a/drivers/iio/dac/max22007.c b/drivers/iio/dac/max22007.c
index d747901df5a32..e2c7e2ad37ba6 100644
--- a/drivers/iio/dac/max22007.c
+++ b/drivers/iio/dac/max22007.c
@@ -464,7 +464,7 @@ static int max22007_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id max22007_id[] = {
-	{ "max22007" },
+	{ .name = "max22007" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, max22007_id);
diff --git a/drivers/iio/dac/max5522.c b/drivers/iio/dac/max5522.c
index 1459ba132df6b..f3415748c765f 100644
--- a/drivers/iio/dac/max5522.c
+++ b/drivers/iio/dac/max5522.c
@@ -24,15 +24,8 @@
 
 #define MAX5522_REG_DATA(x)	((x) + MAX5522_CTRL_LOAD_IN_A)
 
-struct max5522_chip_info {
-	const char *name;
-	const struct iio_chan_spec *channels;
-	unsigned int num_channels;
-};
-
 struct max5522_state {
 	struct regmap *regmap;
-	const struct max5522_chip_info *chip_info;
 	unsigned short dac_cache[2];
 	int vref_mV;
 };
@@ -57,18 +50,6 @@ static const struct iio_chan_spec max5522_channels[] = {
 	MAX5522_CHANNEL(1),
 };
 
-enum max5522_type {
-	ID_MAX5522,
-};
-
-static const struct max5522_chip_info max5522_chip_info_tbl[] = {
-	[ID_MAX5522] = {
-		.name = "max5522",
-		.channels = max5522_channels,
-		.num_channels = 2,
-	},
-};
-
 static inline int max5522_info_to_reg(struct iio_chan_spec const *chan)
 {
 	return MAX5522_REG_DATA(chan->channel);
@@ -139,9 +120,6 @@ static int max5522_spi_probe(struct spi_device *spi)
 	}
 
 	state = iio_priv(indio_dev);
-	state->chip_info = spi_get_device_match_data(spi);
-	if (!state->chip_info)
-		return -EINVAL;
 
 	ret = devm_regulator_get_enable_read_voltage(&spi->dev, "vrefin");
 	if (ret < 0)
@@ -158,22 +136,19 @@ static int max5522_spi_probe(struct spi_device *spi)
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->channels = max5522_channels;
 	indio_dev->num_channels = ARRAY_SIZE(max5522_channels);
-	indio_dev->name = max5522_chip_info_tbl[ID_MAX5522].name;
+	indio_dev->name = "max5522";
 
 	return devm_iio_device_register(&spi->dev, indio_dev);
 }
 
 static const struct spi_device_id max5522_ids[] = {
-	{ "max5522", (kernel_ulong_t)&max5522_chip_info_tbl[ID_MAX5522] },
+	{ .name = "max5522" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, max5522_ids);
 
 static const struct of_device_id max5522_of_match[] = {
-	{
-		.compatible = "maxim,max5522",
-		.data = &max5522_chip_info_tbl[ID_MAX5522],
-	},
+	{ .compatible = "maxim,max5522" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, max5522_of_match);
diff --git a/drivers/iio/dac/mcf54415_dac.c b/drivers/iio/dac/mcf54415_dac.c
new file mode 100644
index 0000000000000..e2c12241a5347
--- /dev/null
+++ b/drivers/iio/dac/mcf54415_dac.c
@@ -0,0 +1,180 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * NXP mcf54415 DAC driver
+ *
+ * Copyright 2026 BayLibre - adureghello@baylibre.com
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/types.h>
+
+#include <linux/iio/iio.h>
+
+#define MCF54415_DAC_CR			0x00
+#define MCF54415_DAC_CR_PDN		BIT(0)
+#define MCF54415_DAC_CR_HSLS		BIT(6)
+#define MCF54415_DAC_CR_WMLVL		GENMASK(9, 8)
+#define MCF54415_DAC_CR_FILT		BIT(12)
+
+#define MCF54415_DAC_DATA		0x02
+
+struct mcf54415_dac {
+	struct regmap *map;
+	struct clk *clk;
+};
+
+static const struct regmap_config mcf54415_dac_regmap_config = {
+	.reg_bits = 16,
+	.reg_stride = 2,
+	.val_bits = 16,
+	.max_register = 0x0c, /* DACX_FILTCNT,  R.M. Table 30-2 */
+	.val_format_endian = REGMAP_ENDIAN_BIG,
+	.reg_format_endian = REGMAP_ENDIAN_BIG,
+};
+
+static int mcf54415_dac_init(struct mcf54415_dac *info)
+{
+	u16 val = MCF54415_DAC_CR_FILT | FIELD_PREP(MCF54415_DAC_CR_WMLVL, 1);
+	int ret;
+
+	/* Fixed defaults and enable DAC (bit 0 set to 0) */
+	ret = regmap_write(info->map, MCF54415_DAC_CR, val);
+	if (ret)
+		return ret;
+
+	/* DAC is ready after 12us, from RM table 40-3  */
+	fsleep(12);
+
+	return 0;
+}
+
+static void mcf54415_dac_exit(void *data)
+{
+	struct mcf54415_dac *info = data;
+
+	regmap_set_bits(info->map, MCF54415_DAC_CR, MCF54415_DAC_CR_PDN);
+}
+
+static const struct iio_chan_spec mcf54415_dac_iio_channel = {
+	.type = IIO_VOLTAGE,
+	.output = 1,
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),
+};
+
+static int mcf54415_read_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     int *val, int *val2, long mask)
+{
+	struct mcf54415_dac *info = iio_priv(indio_dev);
+	unsigned int reg;
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = regmap_read(info->map, MCF54415_DAC_DATA, &reg);
+		if (ret)
+			return ret;
+		*val = reg & GENMASK(11, 0);
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		/* Reference voltage as per ColdFire datasheet is 3.3V */
+		*val = 3300 /* mV */;
+		*val2 = 12;
+		return IIO_VAL_FRACTIONAL_LOG2;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int mcf54415_write_raw(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan,
+			      int val, int val2, long mask)
+{
+	struct mcf54415_dac *info = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		/* Check based on RM 30.3.2 (DACn_DATA) reg. resolution */
+		if (val < 0 || val > 4095)
+			return -EINVAL;
+		return regmap_write(info->map, MCF54415_DAC_DATA, val);
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info mcf54415_dac_iio_info = {
+	.read_raw = &mcf54415_read_raw,
+	.write_raw = &mcf54415_write_raw,
+};
+
+static int mcf54415_dac_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct iio_dev *indio_dev;
+	struct mcf54415_dac *info;
+	void __iomem *regs;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*info));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	info = iio_priv(indio_dev);
+
+	regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(regs))
+		return dev_err_probe(dev, PTR_ERR(regs), "failed to get io regs\n");
+
+	info->map = devm_regmap_init_mmio(dev, regs, &mcf54415_dac_regmap_config);
+	if (IS_ERR(info->map))
+		return PTR_ERR(info->map);
+
+	info->clk = devm_clk_get_enabled(dev, "dac");
+	if (IS_ERR(info->clk))
+		return dev_err_probe(dev, PTR_ERR(info->clk), "failed getting clock\n");
+
+	indio_dev->name = "mcf54415";
+	indio_dev->info = &mcf54415_dac_iio_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = &mcf54415_dac_iio_channel;
+	indio_dev->num_channels = 1;
+
+	ret = mcf54415_dac_init(info);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(dev, mcf54415_dac_exit, info);
+	if (ret)
+		return ret;
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static const struct platform_device_id mcf54415_dac_ids[] = {
+	{ .name = "mcfdac" },
+	{ }
+};
+MODULE_DEVICE_TABLE(platform, mcf54415_dac_ids);
+
+static struct platform_driver mcf54415_dac_driver = {
+	.driver = {
+		.name = "mcf54415_dac",
+	},
+	.probe = mcf54415_dac_probe,
+	.id_table = mcf54415_dac_ids,
+};
+module_platform_driver(mcf54415_dac_driver);
+
+MODULE_AUTHOR("Angelo Dureghello <angelo@kernel-space.org>");
+MODULE_DESCRIPTION("NXP MCF54415 DAC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/dac/mcp4821.c b/drivers/iio/dac/mcp4821.c
index d2e7c930c8483..8d8c00cd60e98 100644
--- a/drivers/iio/dac/mcp4821.c
+++ b/drivers/iio/dac/mcp4821.c
@@ -297,12 +297,12 @@ static const struct of_device_id mcp4821_of_table[] = {
 MODULE_DEVICE_TABLE(of, mcp4821_of_table);
 
 static const struct spi_device_id mcp4821_id_table[] = {
-	{ "mcp4801", (kernel_ulong_t)&mcp4821_chip_info_table[ID_MCP4801]},
-	{ "mcp4802", (kernel_ulong_t)&mcp4821_chip_info_table[ID_MCP4802]},
-	{ "mcp4811", (kernel_ulong_t)&mcp4821_chip_info_table[ID_MCP4811]},
-	{ "mcp4812", (kernel_ulong_t)&mcp4821_chip_info_table[ID_MCP4812]},
-	{ "mcp4821", (kernel_ulong_t)&mcp4821_chip_info_table[ID_MCP4821]},
-	{ "mcp4822", (kernel_ulong_t)&mcp4821_chip_info_table[ID_MCP4822]},
+	{ .name = "mcp4801", .driver_data = (kernel_ulong_t)&mcp4821_chip_info_table[ID_MCP4801] },
+	{ .name = "mcp4802", .driver_data = (kernel_ulong_t)&mcp4821_chip_info_table[ID_MCP4802] },
+	{ .name = "mcp4811", .driver_data = (kernel_ulong_t)&mcp4821_chip_info_table[ID_MCP4811] },
+	{ .name = "mcp4812", .driver_data = (kernel_ulong_t)&mcp4821_chip_info_table[ID_MCP4812] },
+	{ .name = "mcp4821", .driver_data = (kernel_ulong_t)&mcp4821_chip_info_table[ID_MCP4821] },
+	{ .name = "mcp4822", .driver_data = (kernel_ulong_t)&mcp4821_chip_info_table[ID_MCP4822] },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, mcp4821_id_table);
diff --git a/drivers/iio/dac/mcp4922.c b/drivers/iio/dac/mcp4922.c
index 74f338afcab98..15072a9081ffa 100644
--- a/drivers/iio/dac/mcp4922.c
+++ b/drivers/iio/dac/mcp4922.c
@@ -157,10 +157,10 @@ static int mcp4922_probe(struct spi_device *spi)
 
 
 static const struct spi_device_id mcp4922_id[] = {
-	{"mcp4902", ID_MCP4902},
-	{"mcp4912", ID_MCP4912},
-	{"mcp4921", ID_MCP4921},
-	{"mcp4922", ID_MCP4922},
+	{ .name = "mcp4902", .driver_data = ID_MCP4902 },
+	{ .name = "mcp4912", .driver_data = ID_MCP4912 },
+	{ .name = "mcp4921", .driver_data = ID_MCP4921 },
+	{ .name = "mcp4922", .driver_data = ID_MCP4922 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, mcp4922_id);
diff --git a/drivers/iio/dac/rohm-bd79703.c b/drivers/iio/dac/rohm-bd79703.c
index e91090e4a66d9..6e2e7787f8a47 100644
--- a/drivers/iio/dac/rohm-bd79703.c
+++ b/drivers/iio/dac/rohm-bd79703.c
@@ -214,10 +214,10 @@ static int bd79703_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id bd79703_id[] = {
-	{ "bd79700", (kernel_ulong_t)&bd79700_chip_data },
-	{ "bd79701", (kernel_ulong_t)&bd79701_chip_data },
-	{ "bd79702", (kernel_ulong_t)&bd79702_chip_data },
-	{ "bd79703", (kernel_ulong_t)&bd79703_chip_data },
+	{ .name = "bd79700", .driver_data = (kernel_ulong_t)&bd79700_chip_data },
+	{ .name = "bd79701", .driver_data = (kernel_ulong_t)&bd79701_chip_data },
+	{ .name = "bd79702", .driver_data = (kernel_ulong_t)&bd79702_chip_data },
+	{ .name = "bd79703", .driver_data = (kernel_ulong_t)&bd79703_chip_data },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, bd79703_id);
diff --git a/drivers/iio/dac/ti-dac082s085.c b/drivers/iio/dac/ti-dac082s085.c
index 6e62c1f302c82..afa7d9bad5deb 100644
--- a/drivers/iio/dac/ti-dac082s085.c
+++ b/drivers/iio/dac/ti-dac082s085.c
@@ -334,12 +334,12 @@ static const struct of_device_id ti_dac_of_id[] = {
 MODULE_DEVICE_TABLE(of, ti_dac_of_id);
 
 static const struct spi_device_id ti_dac_spi_id[] = {
-	{ "dac082s085", dual_8bit  },
-	{ "dac102s085", dual_10bit },
-	{ "dac122s085", dual_12bit },
-	{ "dac084s085", quad_8bit  },
-	{ "dac104s085", quad_10bit },
-	{ "dac124s085", quad_12bit },
+	{ .name = "dac082s085", .driver_data = dual_8bit },
+	{ .name = "dac102s085", .driver_data = dual_10bit },
+	{ .name = "dac122s085", .driver_data = dual_12bit },
+	{ .name = "dac084s085", .driver_data = quad_8bit },
+	{ .name = "dac104s085", .driver_data = quad_10bit },
+	{ .name = "dac124s085", .driver_data = quad_12bit },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ti_dac_spi_id);
diff --git a/drivers/iio/dac/ti-dac7311.c b/drivers/iio/dac/ti-dac7311.c
index 5c1c5213962ff..e5843bd352590 100644
--- a/drivers/iio/dac/ti-dac7311.c
+++ b/drivers/iio/dac/ti-dac7311.c
@@ -311,9 +311,9 @@ static const struct of_device_id ti_dac_of_id[] = {
 MODULE_DEVICE_TABLE(of, ti_dac_of_id);
 
 static const struct spi_device_id ti_dac_spi_id[] = {
-	{ "dac5311", ID_DAC5311  },
-	{ "dac6311", ID_DAC6311 },
-	{ "dac7311", ID_DAC7311 },
+	{ .name = "dac5311", .driver_data = ID_DAC5311 },
+	{ .name = "dac6311", .driver_data = ID_DAC6311 },
+	{ .name = "dac7311", .driver_data = ID_DAC7311 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ti_dac_spi_id);
diff --git a/drivers/iio/dac/ti-dac7612.c b/drivers/iio/dac/ti-dac7612.c
index c308eca02b88b..9065663cd6098 100644
--- a/drivers/iio/dac/ti-dac7612.c
+++ b/drivers/iio/dac/ti-dac7612.c
@@ -165,7 +165,7 @@ static int dac7612_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id dac7612_id[] = {
-	{"ti-dac7612"},
+	{ .name = "ti-dac7612" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, dac7612_id);
diff --git a/drivers/iio/filter/admv8818.c b/drivers/iio/filter/admv8818.c
index 8bcd298e1f3dd..b88689e60cc88 100644
--- a/drivers/iio/filter/admv8818.c
+++ b/drivers/iio/filter/admv8818.c
@@ -789,7 +789,7 @@ static int admv8818_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id admv8818_id[] = {
-	{ "admv8818", 0 },
+	{ .name = "admv8818" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, admv8818_id);
diff --git a/drivers/iio/frequency/Kconfig b/drivers/iio/frequency/Kconfig
index 583cbdf4e8cda..90c6304c4bcda 100644
--- a/drivers/iio/frequency/Kconfig
+++ b/drivers/iio/frequency/Kconfig
@@ -29,6 +29,16 @@ endmenu
 
 menu "Phase-Locked Loop (PLL) frequency synthesizers"
 
+config ADF41513
+	tristate "Analog Devices ADF41513 PLL Frequency Synthesizer"
+	depends on SPI
+	help
+	  Say yes here to build support for Analog Devices ADF41513
+	  26.5 GHz Integer-N/Fractional-N PLL Frequency Synthesizer.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called adf41513.
+
 config ADF4350
 	tristate "Analog Devices ADF4350/ADF4351 Wideband Synthesizers"
 	depends on SPI
diff --git a/drivers/iio/frequency/Makefile b/drivers/iio/frequency/Makefile
index 70d0e0b70e802..53b4d01414d8d 100644
--- a/drivers/iio/frequency/Makefile
+++ b/drivers/iio/frequency/Makefile
@@ -5,6 +5,7 @@
 
 # When adding new entries keep the list in alphabetical order
 obj-$(CONFIG_AD9523) += ad9523.o
+obj-$(CONFIG_ADF41513) += adf41513.o
 obj-$(CONFIG_ADF4350) += adf4350.o
 obj-$(CONFIG_ADF4371) += adf4371.o
 obj-$(CONFIG_ADF4377) += adf4377.o
diff --git a/drivers/iio/frequency/ad9523.c b/drivers/iio/frequency/ad9523.c
index ea4d2763564a3..1e393f40a8c86 100644
--- a/drivers/iio/frequency/ad9523.c
+++ b/drivers/iio/frequency/ad9523.c
@@ -990,7 +990,6 @@ static int ad9523_probe(struct spi_device *spi)
 	if (IS_ERR(st->sync_gpio))
 		return PTR_ERR(st->sync_gpio);
 
-	spi_set_drvdata(spi, indio_dev);
 	st->spi = spi;
 	st->pdata = pdata;
 
@@ -1009,7 +1008,7 @@ static int ad9523_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad9523_id[] = {
-	{"ad9523-1", 9523},
+	{ .name = "ad9523-1" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad9523_id);
diff --git a/drivers/iio/frequency/adf41513.c b/drivers/iio/frequency/adf41513.c
new file mode 100644
index 0000000000000..c497027012ba0
--- /dev/null
+++ b/drivers/iio/frequency/adf41513.c
@@ -0,0 +1,1246 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * ADF41513 SPI PLL Frequency Synthesizer driver
+ *
+ * Copyright 2026 Analog Devices Inc.
+ */
+
+#include <linux/array_size.h>
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/cleanup.h>
+#include <linux/clk.h>
+#include <linux/dev_printk.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/gpio/consumer.h>
+#include <linux/kstrtox.h>
+#include <linux/log2.h>
+#include <linux/math64.h>
+#include <linux/minmax.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/pm.h>
+#include <linux/property.h>
+#include <linux/regulator/consumer.h>
+#include <linux/spi/spi.h>
+#include <linux/sysfs.h>
+#include <linux/time64.h>
+#include <linux/types.h>
+#include <linux/units.h>
+
+#include <linux/iio/iio.h>
+
+/* Registers */
+#define ADF41513_REG0		0
+#define ADF41513_REG1		1
+#define ADF41513_REG2		2
+#define ADF41513_REG3		3
+#define ADF41513_REG4		4
+#define ADF41513_REG5		5
+#define ADF41513_REG6		6
+#define ADF41513_REG7		7
+#define ADF41513_REG8		8
+#define ADF41513_REG9		9
+#define ADF41513_REG10		10
+#define ADF41513_REG11		11
+#define ADF41513_REG12		12
+#define ADF41513_REG13		13
+#define ADF41513_REG_NUM	14
+
+#define ADF41513_SYNC_REG0	BIT(ADF41513_REG0)
+#define ADF41513_SYNC_REG1	BIT(ADF41513_REG1)
+#define ADF41513_SYNC_REG2	BIT(ADF41513_REG2)
+#define ADF41513_SYNC_REG3	BIT(ADF41513_REG3)
+#define ADF41513_SYNC_REG4	BIT(ADF41513_REG4)
+#define ADF41513_SYNC_REG5	BIT(ADF41513_REG5)
+#define ADF41513_SYNC_REG6	BIT(ADF41513_REG6)
+#define ADF41513_SYNC_REG7	BIT(ADF41513_REG7)
+#define ADF41513_SYNC_REG9	BIT(ADF41513_REG9)
+#define ADF41513_SYNC_REG11	BIT(ADF41513_REG11)
+#define ADF41513_SYNC_REG12	BIT(ADF41513_REG12)
+#define ADF41513_SYNC_REG13	BIT(ADF41513_REG13)
+#define ADF41513_SYNC_DIFF	0
+#define ADF41513_SYNC_ALL	GENMASK(ADF41513_REG13, ADF41513_REG0)
+
+/* REG0 Bit Definitions */
+#define ADF41513_REG0_CTRL_BITS_MSK		GENMASK(3, 0)
+#define ADF41513_REG0_INT_MSK			GENMASK(19, 4)
+#define ADF41513_REG0_VAR_MOD_MSK		BIT(28)
+
+/* REG1 Bit Definitions */
+#define ADF41513_REG1_FRAC1_MSK			GENMASK(28, 4)
+#define ADF41513_REG1_DITHER2_MSK		BIT(31)
+
+/* REG2 Bit Definitions */
+#define ADF41513_REG2_PHASE_VAL_MSK		GENMASK(15, 4)
+#define ADF41513_REG2_PHASE_ADJ_MSK		BIT(31)
+
+/* REG3 Bit Definitions */
+#define ADF41513_REG3_FRAC2_MSK			GENMASK(27, 4)
+
+/* REG4 Bit Definitions */
+#define ADF41513_REG4_MOD2_MSK			GENMASK(27, 4)
+
+/* REG5 Bit Definitions */
+#define ADF41513_REG5_CLK1_DIV_MSK		GENMASK(15, 4)
+#define ADF41513_REG5_R_CNT_MSK			GENMASK(20, 16)
+#define ADF41513_REG5_REF_DOUBLER_MSK		BIT(21)
+#define ADF41513_REG5_RDIV2_MSK			BIT(22)
+#define ADF41513_REG5_PRESCALER_MSK		BIT(23)
+#define ADF41513_REG5_LSB_P1_MSK		BIT(24)
+#define ADF41513_REG5_CP_CURRENT_MSK		GENMASK(28, 25)
+#define ADF41513_REG5_DLD_MODES_MSK		GENMASK(31, 30)
+
+/* REG6 Bit Definitions */
+#define ADF41513_REG6_COUNTER_RESET_MSK		BIT(4)
+#define ADF41513_REG6_CP_TRISTATE_MSK		BIT(5)
+#define ADF41513_REG6_POWER_DOWN_MSK		BIT(6)
+#define ADF41513_REG6_PD_POLARITY_MSK		BIT(7)
+#define ADF41513_REG6_LDP_MSK			GENMASK(9, 8)
+#define ADF41513_REG6_CP_TRISTATE_PD_ON_MSK	BIT(16)
+#define ADF41513_REG6_SD_RESET_MSK		BIT(17)
+#define ADF41513_REG6_LOL_ENABLE_MSK		BIT(18)
+#define ADF41513_REG6_ABP_MSK			BIT(19)
+#define ADF41513_REG6_INT_MODE_MSK		BIT(20)
+#define ADF41513_REG6_BLEED_ENABLE_MSK		BIT(22)
+#define ADF41513_REG6_BLEED_POLARITY_MSK	BIT(23)
+#define ADF41513_REG6_BLEED_CURRENT_MSK		GENMASK(31, 24)
+
+/* REG7 Bit Definitions */
+#define ADF41513_REG7_CLK2_DIV_MSK		GENMASK(17, 6)
+#define ADF41513_REG7_CLK_DIV_MODE_MSK		GENMASK(19, 18)
+#define ADF41513_REG7_PS_BIAS_MSK		GENMASK(21, 20)
+#define ADF41513_REG7_N_DELAY_MSK		GENMASK(23, 22)
+#define ADF41513_REG7_LD_CLK_SEL_MSK		BIT(26)
+#define ADF41513_REG7_LD_COUNT_MSK		GENMASK(29, 27)
+
+/* REG9 Bit Definitions */
+#define ADF41513_REG9_LD_BIAS_MSK		GENMASK(31, 30)
+
+/* REG11 Bit Definitions */
+#define ADF41513_REG11_POWER_DOWN_SEL_MSK	BIT(31)
+
+/* REG12 Bit Definitions */
+#define ADF41513_REG12_READBACK_SEL_MSK		GENMASK(19, 14)
+#define ADF41513_REG12_LE_SELECT_MSK		BIT(20)
+#define ADF41513_REG12_MASTER_RESET_MSK		BIT(22)
+#define ADF41513_REG12_LOGIC_LEVEL_MSK		BIT(27)
+#define ADF41513_REG12_MUXOUT_MSK		GENMASK(31, 28)
+
+/* MUXOUT Selection */
+#define ADF41513_MUXOUT_TRISTATE		0x0
+#define ADF41513_MUXOUT_DVDD			0x1
+#define ADF41513_MUXOUT_DGND			0x2
+#define ADF41513_MUXOUT_R_DIV			0x3
+#define ADF41513_MUXOUT_N_DIV			0x4
+#define ADF41513_MUXOUT_DIG_LD			0x6
+#define ADF41513_MUXOUT_SDO			0x7
+#define ADF41513_MUXOUT_READBACK		0x8
+#define ADF41513_MUXOUT_CLK1_DIV		0xA
+#define ADF41513_MUXOUT_R_DIV2			0xD
+#define ADF41513_MUXOUT_N_DIV2			0xE
+
+/* DLD Mode Selection */
+#define ADF41513_DLD_TRISTATE			0x0
+#define ADF41513_DLD_DIG_LD			0x1
+#define ADF41513_DLD_LOW			0x2
+#define ADF41513_DLD_HIGH			0x3
+
+/* Prescaler Selection */
+#define ADF41513_PRESCALER_4_5			0
+#define ADF41513_PRESCALER_8_9			1
+#define ADF41513_PRESCALER_AUTO			2
+
+/* CLK Divider mode */
+#define ADF41513_CLK_DIV_MODE_OFF		0
+#define ADF41513_CLK_DIV_MODE_PHASE_RESYNC	2
+
+/* Specifications */
+#define ADF41510_MAX_RF_FREQ_HZ			(10ULL * HZ_PER_GHZ)
+#define ADF41513_MIN_RF_FREQ_HZ			(1ULL * HZ_PER_GHZ)
+#define ADF41513_MAX_RF_FREQ_HZ			(26500ULL * HZ_PER_MHZ)
+
+#define ADF41513_MIN_REF_FREQ_HZ		(10 * HZ_PER_MHZ)
+#define ADF41513_MAX_REF_FREQ_HZ		(800 * HZ_PER_MHZ)
+#define ADF41513_MAX_REF_FREQ_DOUBLER_HZ	(225 * HZ_PER_MHZ)
+
+#define ADF41513_MAX_PFD_FREQ_INT_N_UHZ		(250ULL * MEGA * MICROHZ_PER_HZ)
+#define ADF41513_MAX_PFD_FREQ_FRAC_N_UHZ	(125ULL * MEGA * MICROHZ_PER_HZ)
+#define ADF41513_MAX_FREQ_RESOLUTION_UHZ	(100ULL * KILO * MICROHZ_PER_HZ)
+
+#define ADF41513_MIN_INT_4_5			20
+#define ADF41513_MAX_INT_4_5			511
+#define ADF41513_MIN_INT_8_9			64
+#define ADF41513_MAX_INT_8_9			1023
+
+#define ADF41513_MIN_INT_FRAC_4_5		23
+#define ADF41513_MIN_INT_FRAC_8_9		75
+
+#define ADF41513_MIN_R_CNT			1
+#define ADF41513_MAX_R_CNT			32
+
+#define ADF41513_MIN_R_SET			1800
+#define ADF41513_DEFAULT_R_SET			2700
+#define ADF41513_MAX_R_SET			10000
+
+#define ADF41513_MIN_CP_VOLTAGE_mV		810
+#define ADF41513_DEFAULT_CP_VOLTAGE_mV		6480
+#define ADF41513_MAX_CP_VOLTAGE_mV		12960
+
+#define ADF41513_MIN_CP_CURRENT_uA		81
+#define ADF41513_MAX_CP_CURRENT_uA		7200
+
+#define ADF41513_LD_COUNT_FAST_MIN		2
+#define ADF41513_LD_COUNT_FAST_LIMIT		64
+#define ADF41513_LD_COUNT_MIN			64
+#define ADF41513_LD_COUNT_MAX			8192
+
+#define ADF41513_FIXED_MODULUS			BIT(25)
+#define ADF41513_MAX_MOD2			(BIT(24) - 1)
+#define ADF41513_MAX_PHASE_VAL			(BIT(12) - 1)
+#define ADF41513_MAX_CLK_DIVIDER		(BIT(12) - 1)
+
+#define ADF41513_HZ_DECIMAL_SCALE		6
+#define ADF41513_PS_BIAS_INIT			0x2
+#define ADF41513_MAX_PHASE_MICRORAD		((2 * 314159265UL) / 100)
+
+enum adf41513_pll_mode {
+	ADF41513_MODE_INVALID,
+	ADF41513_MODE_INTEGER_N,
+	ADF41513_MODE_FIXED_MODULUS,
+	ADF41513_MODE_VARIABLE_MODULUS,
+};
+
+struct adf41513_chip_info {
+	const char *name;
+	u64 max_rf_freq_hz;
+	bool has_prescaler_8_9;
+};
+
+struct adf41513_data {
+	u64 power_up_frequency_hz;
+	u64 freq_resolution_uhz;
+	u32 phase_resync_period_ns;
+	u32 charge_pump_voltage_mv;
+	u32 lock_detect_count;
+
+	u8 ref_div_factor;
+	bool ref_doubler_en;
+	bool ref_div2_en;
+	bool phase_detector_polarity;
+
+	bool logic_lvl_1v8_en;
+	bool le_sync_en;
+};
+
+struct adf41513_pll_settings {
+	enum adf41513_pll_mode mode;
+
+	/* reference path parameters */
+	u8 r_counter;
+	u8 ref_doubler;
+	u8 ref_div2;
+	u8 prescaler;
+
+	/* frequency parameters */
+	u64 target_frequency_uhz;
+	u64 actual_frequency_uhz;
+	u64 pfd_frequency_uhz;
+
+	/* pll parameters */
+	u32 frac1;
+	u32 frac2;
+	u32 mod2;
+	u16 int_val;
+};
+
+struct adf41513_state {
+	const struct adf41513_chip_info *chip_info;
+	struct spi_device *spi;
+	struct gpio_desc *lock_detect;
+	struct clk *ref_clk;
+	u32 ref_freq_hz;
+
+	/*
+	 * Lock for accessing device registers. Some operations require
+	 * multiple consecutive R/W operations, during which the device
+	 * shouldn't be interrupted. The buffers are also shared across
+	 * all operations so need to be protected on stand alone reads and
+	 * writes.
+	 */
+	struct mutex lock;
+
+	/* Cached register values */
+	u32 regs[ADF41513_REG_NUM];
+	u32 regs_hw[ADF41513_REG_NUM];
+
+	struct adf41513_data data;
+	struct adf41513_pll_settings settings;
+
+	bool powerdown;
+};
+
+static const u16 adf41513_ld_window_x10_ns[] = {
+	9, 12, 16, 17, 21, 28, 29, 35,			/* 0 - 7 */
+	43, 47, 49, 52, 70, 79, 115,			/* 8 - 14 */
+};
+
+static const u8 adf41513_ldp_bias[] = {
+	0xC, 0xD, 0xE, 0x8, 0x9, 0x4, 0xA, 0x5,		/* 0 - 7 */
+	0x0, 0x6, 0xB, 0x1, 0x2, 0x7, 0x3,		/* 8 - 14 */
+};
+
+static const char * const adf41513_power_supplies[] = {
+	"avdd1", "avdd2", "avdd3", "avdd4", "avdd5", "vp",
+};
+
+static int adf41513_sync_config(struct adf41513_state *st, u16 sync_mask)
+{
+	__be32 d32;
+	int ret;
+
+	/* write registers in reverse order (R13 to R0)*/
+	for (int i = ADF41513_REG13; i >= ADF41513_REG0; i--) {
+		if (st->regs_hw[i] == st->regs[i] && !(sync_mask & BIT(i)))
+			continue;
+
+		d32 = cpu_to_be32(st->regs[i] | i);
+		ret = spi_write_then_read(st->spi, &d32, sizeof(d32), NULL, 0);
+		if (ret < 0)
+			return ret;
+		st->regs_hw[i] = st->regs[i];
+		dev_dbg(&st->spi->dev, "REG%d <= 0x%08X\n", i, st->regs[i] | i);
+	}
+
+	return 0;
+}
+
+static u64 adf41513_pll_get_rate(struct adf41513_state *st)
+{
+	struct adf41513_pll_settings *cfg = &st->settings;
+
+	if (cfg->mode != ADF41513_MODE_INVALID)
+		return cfg->actual_frequency_uhz;
+
+	/* get pll settings from regs_hw */
+	cfg->int_val = FIELD_GET(ADF41513_REG0_INT_MSK, st->regs_hw[ADF41513_REG0]);
+	cfg->frac1 = FIELD_GET(ADF41513_REG1_FRAC1_MSK, st->regs_hw[ADF41513_REG1]);
+	cfg->frac2 = FIELD_GET(ADF41513_REG3_FRAC2_MSK, st->regs_hw[ADF41513_REG3]);
+	cfg->mod2 = FIELD_GET(ADF41513_REG4_MOD2_MSK, st->regs_hw[ADF41513_REG4]);
+	cfg->r_counter = FIELD_GET(ADF41513_REG5_R_CNT_MSK, st->regs_hw[ADF41513_REG5]);
+	cfg->ref_doubler = FIELD_GET(ADF41513_REG5_REF_DOUBLER_MSK, st->regs_hw[ADF41513_REG5]);
+	cfg->ref_div2 = FIELD_GET(ADF41513_REG5_RDIV2_MSK, st->regs_hw[ADF41513_REG5]);
+	cfg->prescaler = FIELD_GET(ADF41513_REG5_PRESCALER_MSK, st->regs_hw[ADF41513_REG5]);
+
+	if (!cfg->mod2)
+		cfg->mod2 = 1;
+	if (!cfg->r_counter)
+		cfg->r_counter = ADF41513_MAX_R_CNT;
+
+	/* calculate pfd frequency */
+	cfg->pfd_frequency_uhz = (u64)st->ref_freq_hz * MICRO;
+	if (cfg->ref_doubler)
+		cfg->pfd_frequency_uhz <<= 1;
+	if (cfg->ref_div2)
+		cfg->pfd_frequency_uhz >>= 1;
+	cfg->pfd_frequency_uhz = div_u64(cfg->pfd_frequency_uhz, cfg->r_counter);
+	cfg->actual_frequency_uhz = (u64)cfg->int_val * cfg->pfd_frequency_uhz;
+
+	/* check if int mode is selected */
+	if (FIELD_GET(ADF41513_REG6_INT_MODE_MSK, st->regs_hw[ADF41513_REG6])) {
+		cfg->mode = ADF41513_MODE_INTEGER_N;
+	} else {
+		cfg->actual_frequency_uhz += mul_u64_u32_div(cfg->pfd_frequency_uhz,
+							     cfg->frac1,
+							     ADF41513_FIXED_MODULUS);
+
+		/* check if variable modulus is selected */
+		if (FIELD_GET(ADF41513_REG0_VAR_MOD_MSK, st->regs_hw[ADF41513_REG0])) {
+			cfg->actual_frequency_uhz +=
+				mul_u64_u64_div_u64(cfg->frac2,
+						    cfg->pfd_frequency_uhz,
+						    (u64)cfg->mod2 * ADF41513_FIXED_MODULUS);
+
+			cfg->mode = ADF41513_MODE_VARIABLE_MODULUS;
+		} else {
+			/* LSB_P1 offset */
+			if (!FIELD_GET(ADF41513_REG5_LSB_P1_MSK, st->regs_hw[ADF41513_REG5]))
+				cfg->actual_frequency_uhz +=
+					div_u64(cfg->pfd_frequency_uhz,
+						2 * ADF41513_FIXED_MODULUS);
+			cfg->mode = ADF41513_MODE_FIXED_MODULUS;
+		}
+	}
+
+	cfg->target_frequency_uhz = cfg->actual_frequency_uhz;
+
+	return cfg->actual_frequency_uhz;
+}
+
+static int adf41513_calc_pfd_frequency(struct adf41513_state *st,
+				       struct adf41513_pll_settings *result,
+				       u64 fpfd_limit_uhz)
+{
+	result->ref_div2 = st->data.ref_div2_en;
+	result->ref_doubler = st->data.ref_doubler_en;
+	result->r_counter = st->data.ref_div_factor - 1;
+
+	do {
+		result->r_counter++;
+		/* f_PFD = REF_IN × ((1 + D)/(R × (1 + T))) */
+		result->pfd_frequency_uhz = (u64)st->ref_freq_hz * MICRO;
+		if (result->ref_doubler)
+			result->pfd_frequency_uhz <<= 1;
+		if (result->ref_div2)
+			result->pfd_frequency_uhz >>= 1;
+		result->pfd_frequency_uhz = div_u64(result->pfd_frequency_uhz,
+						    result->r_counter);
+	} while (result->pfd_frequency_uhz > fpfd_limit_uhz);
+
+	if (result->r_counter > ADF41513_MAX_R_CNT) {
+		dev_err(&st->spi->dev, "Cannot optimize PFD frequency\n");
+		return -ERANGE;
+	}
+
+	return 0;
+}
+
+static int adf41513_calc_integer_n(struct adf41513_state *st,
+				   struct adf41513_pll_settings *result)
+{
+	u32 max_int = st->chip_info->has_prescaler_8_9 ?
+		      ADF41513_MAX_INT_8_9 : ADF41513_MAX_INT_4_5;
+	u64 freq_error_uhz;
+	u32 int_val = div64_u64_rem(result->target_frequency_uhz, result->pfd_frequency_uhz,
+				    &freq_error_uhz);
+
+	/* check if freq error is within a tolerance of 1/2 resolution */
+	if (freq_error_uhz > (result->pfd_frequency_uhz >> 1) && int_val < max_int) {
+		int_val++;
+		freq_error_uhz = result->pfd_frequency_uhz - freq_error_uhz;
+	}
+
+	if (freq_error_uhz > st->data.freq_resolution_uhz)
+		return -ERANGE;
+
+	/* set prescaler */
+	if (st->chip_info->has_prescaler_8_9 && int_val >= ADF41513_MIN_INT_8_9 &&
+	    int_val <= ADF41513_MAX_INT_8_9)
+		result->prescaler = 1;
+	else if (int_val >= ADF41513_MIN_INT_4_5 && int_val <= ADF41513_MAX_INT_4_5)
+		result->prescaler = 0;
+	else
+		return -ERANGE;
+
+	result->actual_frequency_uhz = (u64)int_val * result->pfd_frequency_uhz;
+	result->mode = ADF41513_MODE_INTEGER_N;
+	result->int_val = int_val;
+	result->frac1 = 0;
+	result->frac2 = 0;
+	result->mod2 = 0;
+
+	return 0;
+}
+
+static int adf41513_calc_fixed_mod(struct adf41513_state *st,
+				   struct adf41513_pll_settings *result)
+{
+	u64 resolution_uhz = div_u64(result->pfd_frequency_uhz, ADF41513_FIXED_MODULUS);
+	u64 target_frequency_uhz = result->target_frequency_uhz;
+	u64 freq_error_uhz;
+	u32 int_val, frac1;
+	bool lsb_p1_offset = !FIELD_GET(ADF41513_REG5_LSB_P1_MSK, st->regs[ADF41513_REG5]);
+
+	/* LSB_P1 adds a frequency offset of f_pfd/2^26 */
+	if (lsb_p1_offset)
+		target_frequency_uhz -= resolution_uhz >> 1;
+
+	int_val = div64_u64_rem(target_frequency_uhz, result->pfd_frequency_uhz,
+				&freq_error_uhz);
+
+	if (st->chip_info->has_prescaler_8_9 && int_val >= ADF41513_MIN_INT_FRAC_8_9 &&
+	    int_val <= ADF41513_MAX_INT_8_9)
+		result->prescaler = 1;
+	else if (int_val >= ADF41513_MIN_INT_FRAC_4_5 && int_val <= ADF41513_MAX_INT_4_5)
+		result->prescaler = 0;
+	else
+		return -ERANGE;
+
+	/* compute frac1 and fixed modulus error */
+	frac1 = mul_u64_u64_div_u64(freq_error_uhz, ADF41513_FIXED_MODULUS,
+				    result->pfd_frequency_uhz);
+	freq_error_uhz -= mul_u64_u32_div(result->pfd_frequency_uhz, frac1,
+					  ADF41513_FIXED_MODULUS);
+
+	/* check if freq error is within a tolerance of 1/2 resolution */
+	if (freq_error_uhz > (resolution_uhz >> 1) && frac1 < (ADF41513_FIXED_MODULUS - 1)) {
+		frac1++;
+		freq_error_uhz = freq_error_uhz < resolution_uhz ?
+				 resolution_uhz - freq_error_uhz : 0;
+	}
+
+	if (freq_error_uhz > st->data.freq_resolution_uhz)
+		return -ERANGE;
+
+	/* integer part */
+	result->actual_frequency_uhz = (u64)int_val * result->pfd_frequency_uhz;
+	/* fractional part */
+	if (lsb_p1_offset)
+		result->actual_frequency_uhz +=	(resolution_uhz >> 1);
+	result->actual_frequency_uhz += mul_u64_u32_div(result->pfd_frequency_uhz, frac1,
+							ADF41513_FIXED_MODULUS);
+	result->mode = ADF41513_MODE_FIXED_MODULUS;
+	result->int_val = int_val;
+	result->frac1 = frac1;
+	result->frac2 = 0;
+	result->mod2 = 0;
+
+	return 0;
+}
+
+static int adf41513_calc_variable_mod(struct adf41513_state *st,
+				      struct adf41513_pll_settings *result)
+{
+	u64 freq_error_uhz, mod2;
+	u32 frac1, frac2;
+	u32 int_val = div64_u64_rem(result->target_frequency_uhz,
+				    result->pfd_frequency_uhz, &freq_error_uhz);
+
+	if (st->chip_info->has_prescaler_8_9 && int_val >= ADF41513_MIN_INT_FRAC_8_9 &&
+	    int_val <= ADF41513_MAX_INT_8_9)
+		result->prescaler = 1;
+	else if (int_val >= ADF41513_MIN_INT_FRAC_4_5 && int_val <= ADF41513_MAX_INT_4_5)
+		result->prescaler = 0;
+	else
+		return -ERANGE;
+
+	/* calculate required mod2 based on target resolution / 2 */
+	mod2 = DIV64_U64_ROUND_CLOSEST(result->pfd_frequency_uhz << 1,
+				       st->data.freq_resolution_uhz * ADF41513_FIXED_MODULUS);
+	/* ensure mod2 is at least 2 for meaningful operation */
+	mod2 = clamp(mod2, 2, ADF41513_MAX_MOD2);
+
+	/* calculate frac1 and frac2 */
+	frac1 = mul_u64_u64_div_u64(freq_error_uhz, ADF41513_FIXED_MODULUS,
+				    result->pfd_frequency_uhz);
+	frac2 = mul_u64_u64_div_u64(freq_error_uhz, mod2 * ADF41513_FIXED_MODULUS,
+				    result->pfd_frequency_uhz) - mod2 * frac1;
+
+	/* integer part */
+	result->actual_frequency_uhz = (u64)int_val * result->pfd_frequency_uhz;
+	/* fractional part */
+	result->actual_frequency_uhz += mul_u64_u64_div_u64(mod2 * frac1 + frac2,
+							    result->pfd_frequency_uhz,
+							    mod2 * ADF41513_FIXED_MODULUS);
+	result->mode = ADF41513_MODE_VARIABLE_MODULUS;
+	result->int_val = int_val;
+	result->frac1 = frac1;
+	result->frac2 = frac2;
+	result->mod2 = mod2;
+
+	return 0;
+}
+
+static int adf41513_calc_pll_settings(struct adf41513_state *st,
+				      struct adf41513_pll_settings *result,
+				      u64 rf_out_uhz)
+{
+	u64 max_rf_freq_uhz = st->chip_info->max_rf_freq_hz * MICRO;
+	u64 min_rf_freq_uhz = ADF41513_MIN_RF_FREQ_HZ * MICRO;
+	u64 pfd_freq_limit_uhz;
+	int ret;
+
+	if (rf_out_uhz < min_rf_freq_uhz || rf_out_uhz > max_rf_freq_uhz) {
+		dev_err(&st->spi->dev, "RF frequency %llu uHz out of range [%llu, %llu] uHz\n",
+			rf_out_uhz, min_rf_freq_uhz, max_rf_freq_uhz);
+		return -EINVAL;
+	}
+
+	result->target_frequency_uhz = rf_out_uhz;
+
+	/* try integer-N first (best phase noise performance) */
+	pfd_freq_limit_uhz = min(div_u64(rf_out_uhz, ADF41513_MIN_INT_4_5),
+				 ADF41513_MAX_PFD_FREQ_INT_N_UHZ);
+	ret = adf41513_calc_pfd_frequency(st, result, pfd_freq_limit_uhz);
+	if (ret)
+		return ret;
+
+	if (adf41513_calc_integer_n(st, result) == 0)
+		return 0;
+
+	/* try fractional-N: recompute pfd frequency if necessary */
+	pfd_freq_limit_uhz = min(div_u64(rf_out_uhz, ADF41513_MIN_INT_FRAC_4_5),
+				 ADF41513_MAX_PFD_FREQ_FRAC_N_UHZ);
+	if (pfd_freq_limit_uhz < result->pfd_frequency_uhz) {
+		ret = adf41513_calc_pfd_frequency(st, result, pfd_freq_limit_uhz);
+		if (ret)
+			return ret;
+	}
+
+	/* fixed-modulus attempt */
+	if (adf41513_calc_fixed_mod(st, result) == 0)
+		return 0;
+
+	/* variable-modulus attempt */
+	ret = adf41513_calc_variable_mod(st, result);
+	if (ret) {
+		dev_err(&st->spi->dev,
+			"no valid PLL configuration found for %llu uHz\n",
+			rf_out_uhz);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void adf41513_set_bleed_val(struct adf41513_state *st)
+{
+	u32 bleed_value, cp_index;
+
+	if (!(st->regs[ADF41513_REG6] & ADF41513_REG6_BLEED_ENABLE_MSK))
+		return;
+
+	if (st->data.phase_detector_polarity)
+		bleed_value = 90;
+	else
+		bleed_value = 144;
+
+	cp_index = 1 + FIELD_GET(ADF41513_REG5_CP_CURRENT_MSK,
+				 st->regs[ADF41513_REG5]);
+	bleed_value = div64_u64(st->settings.pfd_frequency_uhz * cp_index * bleed_value,
+				1600ULL * MEGA * MICROHZ_PER_HZ);
+
+	FIELD_MODIFY(ADF41513_REG6_BLEED_CURRENT_MSK, &st->regs[ADF41513_REG6],
+		     bleed_value);
+}
+
+static void adf41513_set_ld_window(struct adf41513_state *st)
+{
+	/*
+	 * The ideal lock detector window size is halfway between the max
+	 * window, set by the phase comparison period t_PFD = (1 / f_PFD),
+	 * and the minimum is set by (I_BLEED/I_CP) × t_PFD
+	 */
+	u16 ld_window_10x_ns = div64_u64(10ULL * NSEC_PER_SEC * MICROHZ_PER_HZ,
+					 st->settings.pfd_frequency_uhz << 1);
+	u8 ld_idx, ldp, ld_bias;
+
+	if (st->settings.mode != ADF41513_MODE_INTEGER_N) {
+		/* account for bleed current (deduced from eq.6 and eq.7) */
+		if (st->data.phase_detector_polarity)
+			ld_window_10x_ns += 4;
+		else
+			ld_window_10x_ns += 6;
+	}
+
+	ld_idx = find_closest(ld_window_10x_ns, adf41513_ld_window_x10_ns,
+			      ARRAY_SIZE(adf41513_ld_window_x10_ns));
+	ldp = (adf41513_ldp_bias[ld_idx] >> 2) & 0x3;
+	ld_bias = adf41513_ldp_bias[ld_idx] & 0x3;
+
+	FIELD_MODIFY(ADF41513_REG6_LDP_MSK, &st->regs[ADF41513_REG6], ldp);
+	FIELD_MODIFY(ADF41513_REG9_LD_BIAS_MSK, &st->regs[ADF41513_REG9], ld_bias);
+}
+
+static void adf41513_set_phase_resync(struct adf41513_state *st)
+{
+	u32 total_div, clk1_div, clk2_div;
+
+	if (!st->data.phase_resync_period_ns)
+		return;
+
+	/* assuming both clock dividers hold similar values */
+	total_div = mul_u64_u64_div_u64(st->settings.pfd_frequency_uhz,
+					st->data.phase_resync_period_ns,
+					1ULL * MICROHZ_PER_HZ * NSEC_PER_SEC);
+	clk1_div = clamp(int_sqrt(total_div), 1,
+			 ADF41513_MAX_CLK_DIVIDER);
+	clk2_div = clamp(DIV_ROUND_CLOSEST(total_div, clk1_div), 1,
+			 ADF41513_MAX_CLK_DIVIDER);
+
+	FIELD_MODIFY(ADF41513_REG5_CLK1_DIV_MSK, &st->regs[ADF41513_REG5],
+		     clk1_div);
+	FIELD_MODIFY(ADF41513_REG7_CLK2_DIV_MSK, &st->regs[ADF41513_REG7],
+		     clk2_div);
+
+	/* enable phase resync */
+	FIELD_MODIFY(ADF41513_REG7_CLK_DIV_MODE_MSK, &st->regs[ADF41513_REG7],
+		     ADF41513_CLK_DIV_MODE_PHASE_RESYNC);
+}
+
+static int adf41513_set_frequency(struct adf41513_state *st, u64 freq_uhz, u16 sync_mask)
+{
+	struct adf41513_pll_settings result;
+	bool pfd_change = false;
+	bool mode_change = false;
+	int ret;
+
+	ret = adf41513_calc_pll_settings(st, &result, freq_uhz);
+	if (ret < 0)
+		return ret;
+
+	/* apply computed results to pll settings */
+	pfd_change = st->settings.pfd_frequency_uhz != result.pfd_frequency_uhz;
+	mode_change = st->settings.mode != result.mode;
+	st->settings = result;
+
+	dev_dbg(&st->spi->dev,
+		"%s mode: int=%u, frac1=%u, frac2=%u, mod2=%u, fpdf=%llu Hz, prescaler=%s\n",
+		(result.mode == ADF41513_MODE_INTEGER_N) ? "integer-n" :
+		(result.mode == ADF41513_MODE_FIXED_MODULUS) ? "fixed-modulus" : "variable-modulus",
+		result.int_val, result.frac1, result.frac2, result.mod2,
+		div64_u64(result.pfd_frequency_uhz, MICRO),
+		result.prescaler ? "8/9" : "4/5");
+
+	st->regs[ADF41513_REG0] = FIELD_PREP(ADF41513_REG0_INT_MSK,
+					     st->settings.int_val);
+	if (st->settings.mode == ADF41513_MODE_VARIABLE_MODULUS)
+		st->regs[ADF41513_REG0] |= ADF41513_REG0_VAR_MOD_MSK;
+
+	st->regs[ADF41513_REG1] = FIELD_PREP(ADF41513_REG1_FRAC1_MSK,
+					     st->settings.frac1);
+	if (st->settings.mode != ADF41513_MODE_INTEGER_N)
+		st->regs[ADF41513_REG1] |= ADF41513_REG1_DITHER2_MSK;
+
+	st->regs[ADF41513_REG3] = FIELD_PREP(ADF41513_REG3_FRAC2_MSK,
+					     st->settings.frac2);
+	FIELD_MODIFY(ADF41513_REG4_MOD2_MSK, &st->regs[ADF41513_REG4],
+		     st->settings.mod2);
+	FIELD_MODIFY(ADF41513_REG5_R_CNT_MSK, &st->regs[ADF41513_REG5],
+		     st->settings.r_counter % ADF41513_MAX_R_CNT);
+	FIELD_MODIFY(ADF41513_REG5_REF_DOUBLER_MSK, &st->regs[ADF41513_REG5],
+		     st->settings.ref_doubler);
+	FIELD_MODIFY(ADF41513_REG5_RDIV2_MSK, &st->regs[ADF41513_REG5],
+		     st->settings.ref_div2);
+	FIELD_MODIFY(ADF41513_REG5_PRESCALER_MSK, &st->regs[ADF41513_REG5],
+		     st->settings.prescaler);
+
+	if (st->settings.mode == ADF41513_MODE_INTEGER_N) {
+		st->regs[ADF41513_REG6] |= ADF41513_REG6_INT_MODE_MSK;
+		st->regs[ADF41513_REG6] &= ~ADF41513_REG6_BLEED_ENABLE_MSK;
+	} else {
+		st->regs[ADF41513_REG6] &= ~ADF41513_REG6_INT_MODE_MSK;
+		st->regs[ADF41513_REG6] |= ADF41513_REG6_BLEED_ENABLE_MSK;
+	}
+
+	if (pfd_change)
+		adf41513_set_phase_resync(st);
+
+	if (pfd_change || mode_change) {
+		adf41513_set_bleed_val(st);
+		adf41513_set_ld_window(st);
+	}
+
+	return adf41513_sync_config(st, sync_mask | ADF41513_SYNC_REG0);
+}
+
+static int adf41513_suspend(struct adf41513_state *st)
+{
+	st->regs[ADF41513_REG6] |= FIELD_PREP(ADF41513_REG6_POWER_DOWN_MSK, 1);
+	st->regs[ADF41513_REG12] &= ~ADF41513_REG12_LE_SELECT_MSK;
+	return adf41513_sync_config(st, ADF41513_SYNC_DIFF);
+}
+
+static int adf41513_resume(struct adf41513_state *st)
+{
+	int ret;
+
+	st->regs[ADF41513_REG6] &= ~ADF41513_REG6_POWER_DOWN_MSK;
+	st->regs[ADF41513_REG12] &= ~ADF41513_REG12_LE_SELECT_MSK;
+	ret = adf41513_sync_config(st, ADF41513_SYNC_ALL);
+	if (ret)
+		return ret;
+
+	if (st->data.le_sync_en) {
+		st->regs[ADF41513_REG12] |= ADF41513_REG12_LE_SELECT_MSK;
+		ret = adf41513_sync_config(st, ADF41513_SYNC_DIFF);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static ssize_t adf41513_read_resolution(struct iio_dev *indio_dev,
+					uintptr_t private,
+					const struct iio_chan_spec *chan,
+					char *buf)
+{
+	struct adf41513_state *st = iio_priv(indio_dev);
+	int vals[2];
+
+	guard(mutex)(&st->lock);
+
+	iio_val_s64_decompose(st->data.freq_resolution_uhz, &vals[0], &vals[1]);
+	return iio_format_value(buf, IIO_VAL_DECIMAL64_MICRO, ARRAY_SIZE(vals), vals);
+}
+
+static ssize_t adf41513_read_powerdown(struct iio_dev *indio_dev,
+				       uintptr_t private,
+				       const struct iio_chan_spec *chan,
+				       char *buf)
+{
+	struct adf41513_state *st = iio_priv(indio_dev);
+	u32 val;
+
+	guard(mutex)(&st->lock);
+
+	val = FIELD_GET(ADF41513_REG6_POWER_DOWN_MSK, st->regs_hw[ADF41513_REG6]);
+	return sysfs_emit(buf, "%u\n", val);
+}
+
+static ssize_t adf41513_write_resolution(struct iio_dev *indio_dev,
+					 uintptr_t private,
+					 const struct iio_chan_spec *chan,
+					 const char *buf, size_t len)
+{
+	struct adf41513_state *st = iio_priv(indio_dev);
+	u64 freq_uhz;
+	int ret;
+
+	ret = kstrtoudec64(buf, ADF41513_HZ_DECIMAL_SCALE, &freq_uhz);
+	if (ret)
+		return ret;
+
+	if (freq_uhz == 0 || freq_uhz > ADF41513_MAX_FREQ_RESOLUTION_UHZ)
+		return -EINVAL;
+
+	guard(mutex)(&st->lock);
+
+	st->data.freq_resolution_uhz = freq_uhz;
+	return len;
+}
+
+static ssize_t adf41513_write_powerdown(struct iio_dev *indio_dev,
+					uintptr_t private,
+					const struct iio_chan_spec *chan,
+					const char *buf, size_t len)
+{
+	struct adf41513_state *st = iio_priv(indio_dev);
+	bool val;
+	int ret;
+
+	ret = kstrtobool(buf, &val);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&st->lock);
+
+	if (val)
+		ret = adf41513_suspend(st);
+	else
+		ret = adf41513_resume(st);
+	if (ret)
+		return ret;
+
+	st->powerdown = val;
+	return len;
+}
+
+static const struct iio_chan_spec_ext_info adf41513_ext_info[] = {
+	{
+		.name = "frequency_resolution",
+		.read = adf41513_read_resolution,
+		.write = adf41513_write_resolution,
+		.shared = IIO_SEPARATE,
+	},
+	{
+		.name = "powerdown",
+		.read = adf41513_read_powerdown,
+		.write = adf41513_write_powerdown,
+		.shared = IIO_SEPARATE,
+	},
+	{ }
+};
+
+static const struct iio_chan_spec adf41513_chan = {
+	.type = IIO_ALTVOLTAGE,
+	.indexed = 1,
+	.output = 1,
+	.channel = 0,
+	.info_mask_separate = BIT(IIO_CHAN_INFO_FREQUENCY) |
+			      BIT(IIO_CHAN_INFO_PHASE),
+	.ext_info = adf41513_ext_info,
+};
+
+static int adf41513_read_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     int *val, int *val2, long info)
+{
+	struct adf41513_state *st = iio_priv(indio_dev);
+	u64 tmp64;
+
+	guard(mutex)(&st->lock);
+
+	switch (info) {
+	case IIO_CHAN_INFO_FREQUENCY:
+		if (st->lock_detect &&
+		    !gpiod_get_value_cansleep(st->lock_detect)) {
+			dev_dbg(&st->spi->dev, "PLL un-locked\n");
+			return -EBUSY;
+		}
+		tmp64 = adf41513_pll_get_rate(st);
+		iio_val_s64_decompose(tmp64, val, val2);
+		return IIO_VAL_DECIMAL64_MICRO;
+	case IIO_CHAN_INFO_PHASE:
+		tmp64 = FIELD_GET(ADF41513_REG2_PHASE_VAL_MSK,
+				  st->regs_hw[ADF41513_REG2]);
+		tmp64 = (tmp64 * ADF41513_MAX_PHASE_MICRORAD) >> 12;
+		iio_val_s64_decompose(tmp64, val, val2);
+		return IIO_VAL_DECIMAL64_MICRO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int adf41513_write_raw(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan,
+			      int val, int val2, long info)
+{
+	struct adf41513_state *st = iio_priv(indio_dev);
+	u64 tmp64 = iio_val_s64_compose(val, val2);
+	u16 phase_val;
+	int ret;
+
+	guard(mutex)(&st->lock);
+
+	switch (info) {
+	case IIO_CHAN_INFO_FREQUENCY:
+		return adf41513_set_frequency(st, tmp64, ADF41513_SYNC_DIFF);
+	case IIO_CHAN_INFO_PHASE:
+		if (tmp64 >= ADF41513_MAX_PHASE_MICRORAD)
+			return -EINVAL;
+
+		phase_val = DIV_U64_ROUND_CLOSEST(tmp64 << 12,
+						  ADF41513_MAX_PHASE_MICRORAD);
+		phase_val = min(phase_val, ADF41513_MAX_PHASE_VAL);
+		st->regs[ADF41513_REG2] |= ADF41513_REG2_PHASE_ADJ_MSK;
+		FIELD_MODIFY(ADF41513_REG2_PHASE_VAL_MSK,
+			     &st->regs[ADF41513_REG2], phase_val);
+		ret = adf41513_sync_config(st, ADF41513_SYNC_REG0);
+		/* clear phase adjust for the next sync */
+		st->regs[ADF41513_REG2] &= ~ADF41513_REG2_PHASE_ADJ_MSK;
+		return ret;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int adf41513_write_raw_get_fmt(struct iio_dev *indio_dev,
+				      struct iio_chan_spec const *chan,
+				      long mask)
+{
+	switch (mask) {
+	case IIO_CHAN_INFO_FREQUENCY:
+	case IIO_CHAN_INFO_PHASE:
+		return IIO_VAL_DECIMAL64_MICRO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int adf41513_reg_access(struct iio_dev *indio_dev, unsigned int reg,
+			       unsigned int writeval, unsigned int *readval)
+{
+	struct adf41513_state *st = iio_priv(indio_dev);
+
+	if (reg > ADF41513_REG13)
+		return -EINVAL;
+
+	guard(mutex)(&st->lock);
+
+	if (!readval) {
+		if (reg <= ADF41513_REG6)
+			st->settings.mode = ADF41513_MODE_INVALID;
+		st->regs[reg] = writeval & ~0xF; /* Clear control bits */
+		return adf41513_sync_config(st, BIT(reg));
+	}
+
+	*readval = st->regs_hw[reg];
+	return 0;
+}
+
+static const struct iio_info adf41513_info = {
+	.read_raw = adf41513_read_raw,
+	.write_raw = adf41513_write_raw,
+	.write_raw_get_fmt = adf41513_write_raw_get_fmt,
+	.debugfs_reg_access = &adf41513_reg_access,
+};
+
+static int adf41513_parse_fw(struct adf41513_state *st)
+{
+	struct device *dev = &st->spi->dev;
+	u32 tmp, cp_resistance, cp_current;
+	int ret;
+
+	tmp = ADF41510_MAX_RF_FREQ_HZ / MEGA;
+	device_property_read_u32(dev, "adi,power-up-frequency-mhz", &tmp);
+	st->data.power_up_frequency_hz = (u64)tmp * MEGA;
+	if (st->data.power_up_frequency_hz < ADF41513_MIN_RF_FREQ_HZ ||
+	    st->data.power_up_frequency_hz > st->chip_info->max_rf_freq_hz)
+		return dev_err_probe(dev, -ERANGE,
+				     "power-up frequency %llu Hz out of range\n",
+				     st->data.power_up_frequency_hz);
+
+	tmp = ADF41513_MIN_R_CNT;
+	device_property_read_u32(dev, "adi,reference-div-factor", &tmp);
+	if (tmp < ADF41513_MIN_R_CNT || tmp > ADF41513_MAX_R_CNT)
+		return dev_err_probe(dev, -ERANGE,
+				     "invalid reference div factor %u\n", tmp);
+	st->data.ref_div_factor = tmp;
+
+	st->data.ref_div2_en = device_property_read_bool(dev, "adi,reference-div2-enable");
+	st->data.ref_doubler_en = device_property_read_bool(dev, "adi,reference-doubler-enable");
+
+	if (st->data.ref_doubler_en &&
+	    st->ref_freq_hz > ADF41513_MAX_REF_FREQ_DOUBLER_HZ) {
+		return dev_err_probe(dev, -ERANGE,
+				     "Ref frequency not supported with doubler enabled\n");
+	}
+
+	cp_resistance = ADF41513_DEFAULT_R_SET;
+	device_property_read_u32(dev, "adi,charge-pump-resistor-ohms", &cp_resistance);
+	if (cp_resistance < ADF41513_MIN_R_SET || cp_resistance > ADF41513_MAX_R_SET)
+		return dev_err_probe(dev, -ERANGE, "R_SET %u Ohms out of range\n", cp_resistance);
+
+	st->data.charge_pump_voltage_mv = ADF41513_DEFAULT_CP_VOLTAGE_mV;
+	ret = device_property_read_u32(dev, "adi,charge-pump-current-microamp", &cp_current);
+	if (!ret) {
+		if (cp_current < ADF41513_MIN_CP_CURRENT_uA ||
+		    cp_current > ADF41513_MAX_CP_CURRENT_uA)
+			return dev_err_probe(dev, -ERANGE,
+					     "I_CP %u uA out of range\n", cp_current);
+
+		tmp = DIV_ROUND_CLOSEST(cp_current * cp_resistance, MILLI);
+		if (tmp < ADF41513_MIN_CP_VOLTAGE_mV || tmp > ADF41513_MAX_CP_VOLTAGE_mV)
+			return dev_err_probe(dev, -ERANGE, "I_CP %u uA (%u Ohms) out of range\n",
+					     cp_current, cp_resistance);
+		st->data.charge_pump_voltage_mv = tmp;
+	}
+
+	st->data.phase_detector_polarity =
+		device_property_read_bool(dev, "adi,phase-detector-polarity-positive-enable");
+
+	st->data.phase_resync_period_ns = 0;
+	ret = device_property_read_u32(dev, "adi,phase-resync-period-ns", &tmp);
+	if (!ret)
+		st->data.phase_resync_period_ns = tmp;
+
+	st->data.logic_lvl_1v8_en = device_property_read_bool(dev, "adi,logic-level-1v8-enable");
+
+	tmp = ADF41513_LD_COUNT_MIN;
+	device_property_read_u32(dev, "adi,lock-detector-count", &tmp);
+	if (tmp < ADF41513_LD_COUNT_FAST_MIN || tmp > ADF41513_LD_COUNT_MAX ||
+	    !is_power_of_2(tmp))
+		return dev_err_probe(dev, -ERANGE,
+				     "invalid lock detect count: %u\n", tmp);
+	st->data.lock_detect_count = tmp;
+
+	/* load enable sync */
+	st->data.le_sync_en = device_property_read_bool(dev, "adi,le-sync-enable");
+	st->data.freq_resolution_uhz = MICROHZ_PER_HZ;
+
+	return 0;
+}
+
+static void adf41513_chip_disable(void *data)
+{
+	gpiod_set_value_cansleep(data, 0);
+}
+
+static void adf41513_close(void *data)
+{
+	adf41513_suspend(data);
+}
+
+static int adf41513_setup(struct device *dev, struct adf41513_state *st)
+{
+	u32 tmp;
+	int ret;
+
+	memset(st->regs_hw, 0xFF, sizeof(st->regs_hw));
+
+	/* assuming DLD pin is used for lock detection */
+	st->regs[ADF41513_REG5] = FIELD_PREP(ADF41513_REG5_DLD_MODES_MSK,
+					     ADF41513_DLD_DIG_LD);
+
+	tmp = DIV_ROUND_CLOSEST(st->data.charge_pump_voltage_mv, ADF41513_MIN_CP_VOLTAGE_mV);
+	st->regs[ADF41513_REG5] |= FIELD_PREP(ADF41513_REG5_CP_CURRENT_MSK, tmp - 1);
+
+	st->regs[ADF41513_REG6] = ADF41513_REG6_ABP_MSK |
+				  ADF41513_REG6_LOL_ENABLE_MSK |
+				  ADF41513_REG6_SD_RESET_MSK;
+	if (st->data.phase_detector_polarity)
+		st->regs[ADF41513_REG6] |= ADF41513_REG6_PD_POLARITY_MSK;
+
+	st->regs[ADF41513_REG7] = FIELD_PREP(ADF41513_REG7_PS_BIAS_MSK,
+					     ADF41513_PS_BIAS_INIT);
+	tmp = ilog2(st->data.lock_detect_count);
+	if (st->data.lock_detect_count < ADF41513_LD_COUNT_FAST_LIMIT) {
+		tmp -= const_ilog2(ADF41513_LD_COUNT_FAST_MIN);
+		st->regs[ADF41513_REG7] |= ADF41513_REG7_LD_CLK_SEL_MSK;
+	} else {
+		tmp -= const_ilog2(ADF41513_LD_COUNT_MIN);
+	}
+	st->regs[ADF41513_REG7] |= FIELD_PREP(ADF41513_REG7_LD_COUNT_MSK, tmp);
+
+	st->regs[ADF41513_REG11] = ADF41513_REG11_POWER_DOWN_SEL_MSK;
+	st->regs[ADF41513_REG12] = FIELD_PREP(ADF41513_REG12_LOGIC_LEVEL_MSK,
+					      st->data.logic_lvl_1v8_en ? 0 : 1);
+
+	/* perform initialization sequence with power-up frequency */
+	ret = adf41513_set_frequency(st, st->data.power_up_frequency_hz * MICRO,
+				     ADF41513_SYNC_ALL);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(dev, adf41513_close, st);
+	if (ret)
+		return ret;
+
+	if (st->data.le_sync_en) {
+		st->regs[ADF41513_REG12] |= ADF41513_REG12_LE_SELECT_MSK;
+		ret = adf41513_sync_config(st, ADF41513_SYNC_DIFF);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int adf41513_pm_suspend(struct device *dev)
+{
+	struct adf41513_state *st = dev_get_drvdata(dev);
+
+	guard(mutex)(&st->lock);
+	return adf41513_suspend(st);
+}
+
+static int adf41513_pm_resume(struct device *dev)
+{
+	struct adf41513_state *st = dev_get_drvdata(dev);
+
+	guard(mutex)(&st->lock);
+	if (st->powerdown)
+		return 0; /* nothing to do */
+
+	return adf41513_resume(st);
+}
+
+static const struct adf41513_chip_info adf41510_chip_info = {
+	.name = "adf41510",
+	.max_rf_freq_hz = ADF41510_MAX_RF_FREQ_HZ,
+	.has_prescaler_8_9 = false,
+};
+
+static const struct adf41513_chip_info adf41513_chip_info = {
+	.name = "adf41513",
+	.max_rf_freq_hz = ADF41513_MAX_RF_FREQ_HZ,
+	.has_prescaler_8_9 = true,
+};
+
+static int adf41513_probe(struct spi_device *spi)
+{
+	struct device *dev = &spi->dev;
+	struct gpio_desc *chip_enable;
+	struct iio_dev *indio_dev;
+	struct adf41513_state *st;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	st = iio_priv(indio_dev);
+	st->spi = spi;
+	st->chip_info = spi_get_device_match_data(spi);
+	if (!st->chip_info)
+		return -EINVAL;
+
+	spi_set_drvdata(spi, st);
+
+	st->ref_clk = devm_clk_get_enabled(dev, NULL);
+	if (IS_ERR(st->ref_clk))
+		return PTR_ERR(st->ref_clk);
+
+	st->ref_freq_hz = clk_get_rate(st->ref_clk);
+	if (st->ref_freq_hz < ADF41513_MIN_REF_FREQ_HZ ||
+	    st->ref_freq_hz > ADF41513_MAX_REF_FREQ_HZ)
+		return dev_err_probe(dev, -ERANGE,
+				     "reference frequency %u Hz out of range\n",
+				     st->ref_freq_hz);
+
+	ret = adf41513_parse_fw(st);
+	if (ret)
+		return ret;
+
+	ret = devm_regulator_bulk_get_enable(dev,
+					     ARRAY_SIZE(adf41513_power_supplies),
+					     adf41513_power_supplies);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to get and enable regulators\n");
+
+	st->lock_detect = devm_gpiod_get_optional(dev, "lock-detect", GPIOD_IN);
+	if (IS_ERR(st->lock_detect))
+		return dev_err_probe(dev, PTR_ERR(st->lock_detect),
+				     "fail to request lock detect GPIO\n");
+
+	chip_enable = devm_gpiod_get_optional(dev, "enable", GPIOD_OUT_HIGH);
+	if (IS_ERR(chip_enable))
+		return dev_err_probe(dev, PTR_ERR(chip_enable),
+				     "fail to request chip enable GPIO\n");
+
+	ret = devm_add_action_or_reset(dev, adf41513_chip_disable, chip_enable);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to add disable action\n");
+
+	ret = devm_mutex_init(dev, &st->lock);
+	if (ret)
+		return ret;
+
+	indio_dev->name = st->chip_info->name;
+	indio_dev->info = &adf41513_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = &adf41513_chan;
+	indio_dev->num_channels = 1;
+
+	ret = adf41513_setup(dev, st);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to setup device\n");
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static const struct spi_device_id adf41513_id[] = {
+	{ .name = "adf41510", .driver_data = (kernel_ulong_t)&adf41510_chip_info },
+	{ .name = "adf41513", .driver_data = (kernel_ulong_t)&adf41513_chip_info },
+	{ }
+};
+MODULE_DEVICE_TABLE(spi, adf41513_id);
+
+static const struct of_device_id adf41513_of_match[] = {
+	{ .compatible = "adi,adf41510", .data = &adf41510_chip_info },
+	{ .compatible = "adi,adf41513", .data = &adf41513_chip_info },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, adf41513_of_match);
+
+static DEFINE_SIMPLE_DEV_PM_OPS(adf41513_pm_ops, adf41513_pm_suspend, adf41513_pm_resume);
+
+static struct spi_driver adf41513_driver = {
+	.driver = {
+		.name = "adf41513",
+		.pm = pm_ptr(&adf41513_pm_ops),
+		.of_match_table = adf41513_of_match,
+	},
+	.probe = adf41513_probe,
+	.id_table = adf41513_id,
+};
+module_spi_driver(adf41513_driver);
+
+MODULE_AUTHOR("Rodrigo Alencar <rodrigo.alencar@analog.com>");
+MODULE_DESCRIPTION("Analog Devices ADF41513 PLL Frequency Synthesizer");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/frequency/adf4350.c b/drivers/iio/frequency/adf4350.c
index 639cac522b44b..315317d6eec41 100644
--- a/drivers/iio/frequency/adf4350.c
+++ b/drivers/iio/frequency/adf4350.c
@@ -691,8 +691,8 @@ static const struct of_device_id adf4350_of_match[] = {
 MODULE_DEVICE_TABLE(of, adf4350_of_match);
 
 static const struct spi_device_id adf4350_id[] = {
-	{"adf4350", 4350},
-	{"adf4351", 4351},
+	{ .name = "adf4350", .driver_data = 4350 },
+	{ .name = "adf4351", .driver_data = 4351 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adf4350_id);
diff --git a/drivers/iio/frequency/adf4371.c b/drivers/iio/frequency/adf4371.c
index d6dc7827fb410..e20ed718d01d1 100644
--- a/drivers/iio/frequency/adf4371.c
+++ b/drivers/iio/frequency/adf4371.c
@@ -624,8 +624,8 @@ static int adf4371_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id adf4371_id_table[] = {
-	{ "adf4371", (kernel_ulong_t)&adf4371_chip_info },
-	{ "adf4372", (kernel_ulong_t)&adf4372_chip_info },
+	{ .name = "adf4371", .driver_data = (kernel_ulong_t)&adf4371_chip_info },
+	{ .name = "adf4372", .driver_data = (kernel_ulong_t)&adf4372_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adf4371_id_table);
diff --git a/drivers/iio/frequency/adf4377.c b/drivers/iio/frequency/adf4377.c
index ff6077e29c739..4dd19a9aa9943 100644
--- a/drivers/iio/frequency/adf4377.c
+++ b/drivers/iio/frequency/adf4377.c
@@ -7,6 +7,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/bits.h>
+#include <linux/cleanup.h>
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
 #include <linux/clkdev.h>
@@ -518,14 +519,15 @@ static int adf4377_get_freq(struct adf4377_state *st, u64 *freq)
 	u64 clkin_freq;
 	int ret;
 
-	mutex_lock(&st->lock);
+	guard(mutex)(&st->lock);
+
 	ret = regmap_read(st->regmap, 0x12, &ref_div_factor);
 	if (ret)
-		goto exit;
+		return ret;
 
 	ret = regmap_bulk_read(st->regmap, 0x10, st->buf, sizeof(st->buf));
 	if (ret)
-		goto exit;
+		return ret;
 
 	clkin_freq = clk_get_rate(st->clkin);
 	ref_div_factor = FIELD_GET(ADF4377_0012_R_DIV_MSK, ref_div_factor);
@@ -533,10 +535,8 @@ static int adf4377_get_freq(struct adf4377_state *st, u64 *freq)
 			  get_unaligned_le16(&st->buf));
 
 	*freq = div_u64(clkin_freq, ref_div_factor) * n_int;
-exit:
-	mutex_unlock(&st->lock);
 
-	return ret;
+	return 0;
 }
 
 static int adf4377_set_freq(struct adf4377_state *st, u64 freq)
@@ -545,26 +545,24 @@ static int adf4377_set_freq(struct adf4377_state *st, u64 freq)
 	u64 f_vco;
 	int ret;
 
-	mutex_lock(&st->lock);
+	guard(mutex)(&st->lock);
 
-	if (freq > ADF4377_MAX_CLKPN_FREQ || freq < ADF4377_MIN_CLKPN_FREQ) {
-		ret = -EINVAL;
-		goto exit;
-	}
+	if (freq > ADF4377_MAX_CLKPN_FREQ || freq < ADF4377_MIN_CLKPN_FREQ)
+		return -EINVAL;
 
 	ret = regmap_update_bits(st->regmap, 0x1C, ADF4377_001C_EN_DNCLK_MSK |
 				 ADF4377_001C_EN_DRCLK_MSK,
 				 FIELD_PREP(ADF4377_001C_EN_DNCLK_MSK, 1) |
 				 FIELD_PREP(ADF4377_001C_EN_DRCLK_MSK, 1));
 	if (ret)
-		goto exit;
+		return ret;
 
 	ret = regmap_update_bits(st->regmap, 0x11, ADF4377_0011_EN_AUTOCAL_MSK |
 				 ADF4377_0011_DCLK_DIV2_MSK,
 				 FIELD_PREP(ADF4377_0011_EN_AUTOCAL_MSK, 1) |
 				 FIELD_PREP(ADF4377_0011_DCLK_DIV2_MSK, st->dclk_div2));
 	if (ret)
-		goto exit;
+		return ret;
 
 	ret = regmap_update_bits(st->regmap, 0x2E, ADF4377_002E_EN_ADC_CNV_MSK |
 				 ADF4377_002E_EN_ADC_MSK |
@@ -574,56 +572,56 @@ static int adf4377_set_freq(struct adf4377_state *st, u64 freq)
 				 FIELD_PREP(ADF4377_002E_ADC_A_CONV_MSK,
 					    ADF4377_002E_ADC_A_CONV_VCO_CALIB));
 	if (ret)
-		goto exit;
+		return ret;
 
 	ret = regmap_update_bits(st->regmap, 0x20, ADF4377_0020_EN_ADC_CLK_MSK,
 				 FIELD_PREP(ADF4377_0020_EN_ADC_CLK_MSK, 1));
 	if (ret)
-		goto exit;
+		return ret;
 
 	ret = regmap_update_bits(st->regmap, 0x2F, ADF4377_002F_DCLK_DIV1_MSK,
 				 FIELD_PREP(ADF4377_002F_DCLK_DIV1_MSK, st->dclk_div1));
 	if (ret)
-		goto exit;
+		return ret;
 
 	ret = regmap_update_bits(st->regmap, 0x24, ADF4377_0024_DCLK_MODE_MSK,
 				 FIELD_PREP(ADF4377_0024_DCLK_MODE_MSK, st->dclk_mode));
 	if (ret)
-		goto exit;
+		return ret;
 
 	ret = regmap_write(st->regmap, 0x27,
 			   FIELD_PREP(ADF4377_0027_SYNTH_LOCK_TO_LSB_MSK,
 				      st->synth_lock_timeout));
 	if (ret)
-		goto exit;
+		return ret;
 
 	ret = regmap_update_bits(st->regmap, 0x28, ADF4377_0028_SYNTH_LOCK_TO_MSB_MSK,
 				 FIELD_PREP(ADF4377_0028_SYNTH_LOCK_TO_MSB_MSK,
 					    st->synth_lock_timeout >> 8));
 	if (ret)
-		goto exit;
+		return ret;
 
 	ret = regmap_write(st->regmap, 0x29,
 			   FIELD_PREP(ADF4377_0029_VCO_ALC_TO_LSB_MSK,
 				      st->vco_alc_timeout));
 	if (ret)
-		goto exit;
+		return ret;
 
 	ret = regmap_update_bits(st->regmap, 0x2A, ADF4377_002A_VCO_ALC_TO_MSB_MSK,
 				 FIELD_PREP(ADF4377_002A_VCO_ALC_TO_MSB_MSK,
 					    st->vco_alc_timeout >> 8));
 	if (ret)
-		goto exit;
+		return ret;
 
 	ret = regmap_write(st->regmap, 0x26,
 			   FIELD_PREP(ADF4377_0026_VCO_BAND_DIV_MSK, st->vco_band_div));
 	if (ret)
-		goto exit;
+		return ret;
 
 	ret = regmap_write(st->regmap, 0x2D,
 			   FIELD_PREP(ADF4377_002D_ADC_CLK_DIV_MSK, st->adc_clk_div));
 	if (ret)
-		goto exit;
+		return ret;
 
 	st->clkout_div_sel = 0;
 
@@ -641,24 +639,24 @@ static int adf4377_set_freq(struct adf4377_state *st, u64 freq)
 				 FIELD_PREP(ADF4377_0011_EN_RDBLR_MSK, 0) |
 				 FIELD_PREP(ADF4377_0011_N_INT_MSB_MSK, st->n_int >> 8));
 	if (ret)
-		goto exit;
+		return ret;
 
 	ret = regmap_update_bits(st->regmap, 0x12, ADF4377_0012_R_DIV_MSK |
 				 ADF4377_0012_CLKOUT_DIV_MSK,
 				 FIELD_PREP(ADF4377_0012_CLKOUT_DIV_MSK, st->clkout_div_sel) |
 				 FIELD_PREP(ADF4377_0012_R_DIV_MSK, st->ref_div_factor));
 	if (ret)
-		goto exit;
+		return ret;
 
 	ret = regmap_write(st->regmap, 0x10,
 			   FIELD_PREP(ADF4377_0010_N_INT_LSB_MSK, st->n_int));
 	if (ret)
-		goto exit;
+		return ret;
 
 	ret = regmap_read_poll_timeout(st->regmap, 0x49, read_val,
 				       !(read_val & (ADF4377_0049_FSM_BUSY_MSK)), 200, 200 * 100);
 	if (ret)
-		goto exit;
+		return ret;
 
 	/* Disable EN_DNCLK, EN_DRCLK */
 	ret = regmap_update_bits(st->regmap, 0x1C, ADF4377_001C_EN_DNCLK_MSK |
@@ -666,26 +664,21 @@ static int adf4377_set_freq(struct adf4377_state *st, u64 freq)
 				 FIELD_PREP(ADF4377_001C_EN_DNCLK_MSK, 0) |
 				 FIELD_PREP(ADF4377_001C_EN_DRCLK_MSK, 0));
 	if (ret)
-		goto exit;
+		return ret;
 
 	/* Disable EN_ADC_CLK */
 	ret = regmap_update_bits(st->regmap, 0x20, ADF4377_0020_EN_ADC_CLK_MSK,
 				 FIELD_PREP(ADF4377_0020_EN_ADC_CLK_MSK, 0));
 	if (ret)
-		goto exit;
+		return ret;
 
 	/* Set output Amplitude */
-	ret = regmap_update_bits(st->regmap, 0x19, ADF4377_0019_CLKOUT2_OP_MSK |
-				 ADF4377_0019_CLKOUT1_OP_MSK,
-				 FIELD_PREP(ADF4377_0019_CLKOUT1_OP_MSK,
-					    ADF4377_0019_CLKOUT_420MV) |
-				 FIELD_PREP(ADF4377_0019_CLKOUT2_OP_MSK,
-					    ADF4377_0019_CLKOUT_420MV));
-
-exit:
-	mutex_unlock(&st->lock);
-
-	return ret;
+	return regmap_update_bits(st->regmap, 0x19, ADF4377_0019_CLKOUT2_OP_MSK |
+				  ADF4377_0019_CLKOUT1_OP_MSK,
+				  FIELD_PREP(ADF4377_0019_CLKOUT1_OP_MSK,
+					     ADF4377_0019_CLKOUT_420MV) |
+				  FIELD_PREP(ADF4377_0019_CLKOUT2_OP_MSK,
+					     ADF4377_0019_CLKOUT_420MV));
 }
 
 static void adf4377_gpio_init(struct adf4377_state *st)
@@ -919,13 +912,11 @@ static int adf4377_properties_parse(struct adf4377_state *st)
 static int adf4377_freq_change(struct notifier_block *nb, unsigned long action, void *data)
 {
 	struct adf4377_state *st = container_of(nb, struct adf4377_state, nb);
-	int ret;
 
 	if (action == POST_RATE_CHANGE) {
-		mutex_lock(&st->lock);
-		ret = notifier_from_errno(adf4377_init(st));
-		mutex_unlock(&st->lock);
-		return ret;
+		guard(mutex)(&st->lock);
+
+		return notifier_from_errno(adf4377_init(st));
 	}
 
 	return NOTIFY_OK;
@@ -1097,8 +1088,8 @@ static int adf4377_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id adf4377_id[] = {
-	{ "adf4377", (kernel_ulong_t)&adf4377_chip_info },
-	{ "adf4378", (kernel_ulong_t)&adf4378_chip_info },
+	{ .name = "adf4377", .driver_data = (kernel_ulong_t)&adf4377_chip_info },
+	{ .name = "adf4378", .driver_data = (kernel_ulong_t)&adf4378_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adf4377_id);
diff --git a/drivers/iio/frequency/admv1013.c b/drivers/iio/frequency/admv1013.c
index b823adfb0f70f..c9baaab574e3f 100644
--- a/drivers/iio/frequency/admv1013.c
+++ b/drivers/iio/frequency/admv1013.c
@@ -619,7 +619,7 @@ static int admv1013_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id admv1013_id[] = {
-	{ "admv1013", 0 },
+	{ .name = "admv1013" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, admv1013_id);
diff --git a/drivers/iio/frequency/admv1014.c b/drivers/iio/frequency/admv1014.c
index 5d36ac4bb6662..d1d528975229d 100644
--- a/drivers/iio/frequency/admv1014.c
+++ b/drivers/iio/frequency/admv1014.c
@@ -781,7 +781,7 @@ static int admv1014_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id admv1014_id[] = {
-	{ "admv1014", 0 },
+	{ .name = "admv1014" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, admv1014_id);
diff --git a/drivers/iio/frequency/adrf6780.c b/drivers/iio/frequency/adrf6780.c
index c2dc06f05c21f..79a6150621725 100644
--- a/drivers/iio/frequency/adrf6780.c
+++ b/drivers/iio/frequency/adrf6780.c
@@ -486,7 +486,7 @@ static int adrf6780_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id adrf6780_id[] = {
-	{ "adrf6780", 0 },
+	{ .name = "adrf6780" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adrf6780_id);
diff --git a/drivers/iio/gyro/adis16080.c b/drivers/iio/gyro/adis16080.c
index 178bba95a7094..0c8bc1e569fe1 100644
--- a/drivers/iio/gyro/adis16080.c
+++ b/drivers/iio/gyro/adis16080.c
@@ -212,8 +212,8 @@ static int adis16080_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id adis16080_ids[] = {
-	{ "adis16080", ID_ADIS16080 },
-	{ "adis16100", ID_ADIS16100 },
+	{ .name = "adis16080", .driver_data = ID_ADIS16080 },
+	{ .name = "adis16100", .driver_data = ID_ADIS16100 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adis16080_ids);
diff --git a/drivers/iio/gyro/adis16136.c b/drivers/iio/gyro/adis16136.c
index 369c7428e1efe..ff335f81e9ae9 100644
--- a/drivers/iio/gyro/adis16136.c
+++ b/drivers/iio/gyro/adis16136.c
@@ -563,10 +563,10 @@ static int adis16136_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id adis16136_ids[] = {
-	{ "adis16133", ID_ADIS16133 },
-	{ "adis16135", ID_ADIS16135 },
-	{ "adis16136", ID_ADIS16136 },
-	{ "adis16137", ID_ADIS16137 },
+	{ .name = "adis16133", .driver_data = ID_ADIS16133 },
+	{ .name = "adis16135", .driver_data = ID_ADIS16135 },
+	{ .name = "adis16136", .driver_data = ID_ADIS16136 },
+	{ .name = "adis16137", .driver_data = ID_ADIS16137 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adis16136_ids);
diff --git a/drivers/iio/gyro/adis16260.c b/drivers/iio/gyro/adis16260.c
index 91b9c5f18ec40..a79a8edf4c989 100644
--- a/drivers/iio/gyro/adis16260.c
+++ b/drivers/iio/gyro/adis16260.c
@@ -411,12 +411,12 @@ static int adis16260_probe(struct spi_device *spi)
  * support for the on chip filtering.
  */
 static const struct spi_device_id adis16260_id[] = {
-	{"adis16260", ADIS16260},
-	{"adis16265", ADIS16260},
-	{"adis16266", ADIS16266},
-	{"adis16250", ADIS16260},
-	{"adis16255", ADIS16260},
-	{"adis16251", ADIS16251},
+	{ .name = "adis16260", .driver_data = ADIS16260 },
+	{ .name = "adis16265", .driver_data = ADIS16260 },
+	{ .name = "adis16266", .driver_data = ADIS16266 },
+	{ .name = "adis16250", .driver_data = ADIS16260 },
+	{ .name = "adis16255", .driver_data = ADIS16260 },
+	{ .name = "adis16251", .driver_data = ADIS16251 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adis16260_id);
diff --git a/drivers/iio/gyro/adxrs450.c b/drivers/iio/gyro/adxrs450.c
index a1d8d3cb301b2..030d042cab8bc 100644
--- a/drivers/iio/gyro/adxrs450.c
+++ b/drivers/iio/gyro/adxrs450.c
@@ -439,8 +439,8 @@ static int adxrs450_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id adxrs450_id[] = {
-	{"adxrs450", ID_ADXRS450},
-	{"adxrs453", ID_ADXRS453},
+	{ .name = "adxrs450", .driver_data = ID_ADXRS450 },
+	{ .name = "adxrs453", .driver_data = ID_ADXRS453 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adxrs450_id);
diff --git a/drivers/iio/gyro/bmg160_spi.c b/drivers/iio/gyro/bmg160_spi.c
index 6aecc5eb83472..14ae1d03be3d9 100644
--- a/drivers/iio/gyro/bmg160_spi.c
+++ b/drivers/iio/gyro/bmg160_spi.c
@@ -33,12 +33,11 @@ static void bmg160_spi_remove(struct spi_device *spi)
 }
 
 static const struct spi_device_id bmg160_spi_id[] = {
-	{"bmg160", 0},
-	{"bmi055_gyro", 0},
-	{"bmi088_gyro", 0},
+	{ .name = "bmg160" },
+	{ .name = "bmi055_gyro" },
+	{ .name = "bmi088_gyro" },
 	{ }
 };
-
 MODULE_DEVICE_TABLE(spi, bmg160_spi_id);
 
 static const struct of_device_id bmg160_of_match[] = {
diff --git a/drivers/iio/gyro/fxas21002c_spi.c b/drivers/iio/gyro/fxas21002c_spi.c
index bd5b8678da131..40b1e02b2a5c5 100644
--- a/drivers/iio/gyro/fxas21002c_spi.c
+++ b/drivers/iio/gyro/fxas21002c_spi.c
@@ -39,7 +39,7 @@ static void fxas21002c_spi_remove(struct spi_device *spi)
 }
 
 static const struct spi_device_id fxas21002c_spi_id[] = {
-	{ "fxas21002c", 0 },
+	{ .name = "fxas21002c" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, fxas21002c_spi_id);
diff --git a/drivers/iio/gyro/hid-sensor-gyro-3d.c b/drivers/iio/gyro/hid-sensor-gyro-3d.c
index adc52a5267f73..58250a972567b 100644
--- a/drivers/iio/gyro/hid-sensor-gyro-3d.c
+++ b/drivers/iio/gyro/hid-sensor-gyro-3d.c
@@ -84,9 +84,8 @@ static const struct iio_chan_spec gyro_3d_channels[] = {
 
 /* Channel read_raw handler */
 static int gyro_3d_read_raw(struct iio_dev *indio_dev,
-			      struct iio_chan_spec const *chan,
-			      int *val, int *val2,
-			      long mask)
+			    struct iio_chan_spec const *chan,
+			    int *val, int *val2, long mask)
 {
 	struct gyro_3d_state *gyro_state = iio_priv(indio_dev);
 	int report_id = -1;
@@ -111,8 +110,7 @@ static int gyro_3d_read_raw(struct iio_dev *indio_dev,
 					min < 0);
 		else {
 			*val = 0;
-			hid_sensor_power_state(&gyro_state->common_attributes,
-						false);
+			hid_sensor_power_state(&gyro_state->common_attributes, false);
 			return -EINVAL;
 		}
 		hid_sensor_power_state(&gyro_state->common_attributes, false);
@@ -145,10 +143,8 @@ static int gyro_3d_read_raw(struct iio_dev *indio_dev,
 
 /* Channel write_raw handler */
 static int gyro_3d_write_raw(struct iio_dev *indio_dev,
-			       struct iio_chan_spec const *chan,
-			       int val,
-			       int val2,
-			       long mask)
+			     struct iio_chan_spec const *chan,
+			     int val, int val2, long mask)
 {
 	struct gyro_3d_state *gyro_state = iio_priv(indio_dev);
 	int ret = 0;
@@ -176,8 +172,7 @@ static const struct iio_info gyro_3d_info = {
 
 /* Callback handler to send event after all samples are received and captured */
 static int gyro_3d_proc_event(struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
-				void *priv)
+			      u32 usage_id, void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
 	struct gyro_3d_state *gyro_state = iio_priv(indio_dev);
@@ -198,9 +193,9 @@ static int gyro_3d_proc_event(struct hid_sensor_hub_device *hsdev,
 
 /* Capture samples in local storage */
 static int gyro_3d_capture_sample(struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
-				size_t raw_len, char *raw_data,
-				void *priv)
+				  u32 usage_id,
+				  size_t raw_len, char *raw_data,
+				  void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
 	struct gyro_3d_state *gyro_state = iio_priv(indio_dev);
@@ -233,7 +228,7 @@ static int gyro_3d_capture_sample(struct hid_sensor_hub_device *hsdev,
 static int gyro_3d_parse_report(struct platform_device *pdev,
 				struct hid_sensor_hub_device *hsdev,
 				struct iio_chan_spec *channels,
-				unsigned usage_id,
+				u32 usage_id,
 				struct gyro_3d_state *st)
 {
 	int ret;
@@ -253,10 +248,10 @@ static int gyro_3d_parse_report(struct platform_device *pdev,
 		};
 	}
 	dev_dbg(&pdev->dev, "gyro_3d %x:%x, %x:%x, %x:%x\n",
-			st->gyro[0].index,
-			st->gyro[0].report_id,
-			st->gyro[1].index, st->gyro[1].report_id,
-			st->gyro[2].index, st->gyro[2].report_id);
+		st->gyro[0].index,
+		st->gyro[0].report_id,
+		st->gyro[1].index, st->gyro[1].report_id,
+		st->gyro[2].index, st->gyro[2].report_id);
 
 	st->scale_precision = hid_sensor_format_scale(
 				HID_USAGE_SENSOR_GYRO_3D,
@@ -317,32 +312,32 @@ static int hid_gyro_3d_probe(struct platform_device *pdev)
 	atomic_set(&gyro_state->common_attributes.data_ready, 0);
 
 	ret = hid_sensor_setup_trigger(indio_dev, name,
-					&gyro_state->common_attributes);
+				       &gyro_state->common_attributes);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "trigger setup failed\n");
 		return ret;
 	}
 
-	ret = iio_device_register(indio_dev);
-	if (ret) {
-		dev_err(&pdev->dev, "device register failed\n");
-		goto error_remove_trigger;
-	}
-
 	gyro_state->callbacks.send_event = gyro_3d_proc_event;
 	gyro_state->callbacks.capture_sample = gyro_3d_capture_sample;
 	gyro_state->callbacks.pdev = pdev;
 	ret = sensor_hub_register_callback(hsdev, HID_USAGE_SENSOR_GYRO_3D,
-					&gyro_state->callbacks);
+					   &gyro_state->callbacks);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "callback reg failed\n");
-		goto error_iio_unreg;
+		goto error_remove_trigger;
+	}
+
+	ret = iio_device_register(indio_dev);
+	if (ret) {
+		dev_err(&pdev->dev, "device register failed\n");
+		goto error_remove_callback;
 	}
 
 	return ret;
 
-error_iio_unreg:
-	iio_device_unregister(indio_dev);
+error_remove_callback:
+	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_GYRO_3D);
 error_remove_trigger:
 	hid_sensor_remove_trigger(indio_dev, &gyro_state->common_attributes);
 	return ret;
@@ -355,8 +350,8 @@ static void hid_gyro_3d_remove(struct platform_device *pdev)
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
 	struct gyro_3d_state *gyro_state = iio_priv(indio_dev);
 
-	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_GYRO_3D);
 	iio_device_unregister(indio_dev);
+	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_GYRO_3D);
 	hid_sensor_remove_trigger(indio_dev, &gyro_state->common_attributes);
 }
 
diff --git a/drivers/iio/gyro/st_gyro_spi.c b/drivers/iio/gyro/st_gyro_spi.c
index adfbbc0d37dc6..c56331755b5c2 100644
--- a/drivers/iio/gyro/st_gyro_spi.c
+++ b/drivers/iio/gyro/st_gyro_spi.c
@@ -97,15 +97,15 @@ static int st_gyro_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id st_gyro_id_table[] = {
-	{ L3G4200D_GYRO_DEV_NAME },
-	{ LSM330D_GYRO_DEV_NAME },
-	{ LSM330DL_GYRO_DEV_NAME },
-	{ LSM330DLC_GYRO_DEV_NAME },
-	{ L3GD20_GYRO_DEV_NAME },
-	{ L3GD20H_GYRO_DEV_NAME },
-	{ L3G4IS_GYRO_DEV_NAME },
-	{ LSM330_GYRO_DEV_NAME },
-	{ LSM9DS0_GYRO_DEV_NAME },
+	{ .name = L3G4200D_GYRO_DEV_NAME },
+	{ .name = LSM330D_GYRO_DEV_NAME },
+	{ .name = LSM330DL_GYRO_DEV_NAME },
+	{ .name = LSM330DLC_GYRO_DEV_NAME },
+	{ .name = L3GD20_GYRO_DEV_NAME },
+	{ .name = L3GD20H_GYRO_DEV_NAME },
+	{ .name = L3G4IS_GYRO_DEV_NAME },
+	{ .name = LSM330_GYRO_DEV_NAME },
+	{ .name = LSM9DS0_GYRO_DEV_NAME },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, st_gyro_id_table);
diff --git a/drivers/iio/health/afe4403.c b/drivers/iio/health/afe4403.c
index d358f4d5e5da8..27e32707f089e 100644
--- a/drivers/iio/health/afe4403.c
+++ b/drivers/iio/health/afe4403.c
@@ -568,7 +568,7 @@ static int afe4403_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id afe4403_ids[] = {
-	{ "afe4403", 0 },
+	{ .name = "afe4403" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, afe4403_ids);
diff --git a/drivers/iio/humidity/hid-sensor-humidity.c b/drivers/iio/humidity/hid-sensor-humidity.c
index 5267a14d73ec6..7cec81ff56856 100644
--- a/drivers/iio/humidity/hid-sensor-humidity.c
+++ b/drivers/iio/humidity/hid-sensor-humidity.c
@@ -44,7 +44,7 @@ static const struct iio_chan_spec humidity_channels[] = {
 
 /* Adjust channel real bits based on report descriptor */
 static void humidity_adjust_channel_bit_mask(struct iio_chan_spec *channels,
-					int channel, int size)
+					     int channel, int size)
 {
 	channels[channel].scan_type.sign = 's';
 	/* Real storage bits will change based on the report desc. */
@@ -54,8 +54,8 @@ static void humidity_adjust_channel_bit_mask(struct iio_chan_spec *channels,
 }
 
 static int humidity_read_raw(struct iio_dev *indio_dev,
-				struct iio_chan_spec const *chan,
-				int *val, int *val2, long mask)
+			     struct iio_chan_spec const *chan,
+			     int *val, int *val2, long mask)
 {
 	struct hid_humidity_state *humid_st = iio_priv(indio_dev);
 
@@ -100,8 +100,8 @@ static int humidity_read_raw(struct iio_dev *indio_dev,
 }
 
 static int humidity_write_raw(struct iio_dev *indio_dev,
-				struct iio_chan_spec const *chan,
-				int val, int val2, long mask)
+			      struct iio_chan_spec const *chan,
+			      int val, int val2, long mask)
 {
 	struct hid_humidity_state *humid_st = iio_priv(indio_dev);
 
@@ -126,7 +126,7 @@ static const struct iio_info humidity_info = {
 
 /* Callback handler to send event after all samples are received and captured */
 static int humidity_proc_event(struct hid_sensor_hub_device *hsdev,
-				unsigned int usage_id, void *pdev)
+			       u32 usage_id, void *pdev)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
 	struct hid_humidity_state *humid_st = iio_priv(indio_dev);
@@ -140,8 +140,9 @@ static int humidity_proc_event(struct hid_sensor_hub_device *hsdev,
 
 /* Capture samples in local storage */
 static int humidity_capture_sample(struct hid_sensor_hub_device *hsdev,
-				unsigned int usage_id, size_t raw_len,
-				char *raw_data, void *pdev)
+				   u32 usage_id,
+				   size_t raw_len, char *raw_data,
+				   void *pdev)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
 	struct hid_humidity_state *humid_st = iio_priv(indio_dev);
@@ -158,10 +159,10 @@ static int humidity_capture_sample(struct hid_sensor_hub_device *hsdev,
 
 /* Parse report which is specific to an usage id */
 static int humidity_parse_report(struct platform_device *pdev,
-				struct hid_sensor_hub_device *hsdev,
-				struct iio_chan_spec *channels,
-				unsigned int usage_id,
-				struct hid_humidity_state *st)
+				 struct hid_sensor_hub_device *hsdev,
+				 struct iio_chan_spec *channels,
+				 u32 usage_id,
+				 struct hid_humidity_state *st)
 {
 	int ret;
 
@@ -214,13 +215,13 @@ static int hid_humidity_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	humid_chans = devm_kmemdup(&indio_dev->dev, humidity_channels,
-					sizeof(humidity_channels), GFP_KERNEL);
+	humid_chans = devm_kmemdup(&pdev->dev, humidity_channels,
+				   sizeof(humidity_channels), GFP_KERNEL);
 	if (!humid_chans)
 		return -ENOMEM;
 
 	ret = humidity_parse_report(pdev, hsdev, humid_chans,
-				HID_USAGE_SENSOR_HUMIDITY, humid_st);
+				    HID_USAGE_SENSOR_HUMIDITY, humid_st);
 	if (ret)
 		return ret;
 
@@ -233,7 +234,7 @@ static int hid_humidity_probe(struct platform_device *pdev)
 	atomic_set(&humid_st->common_attributes.data_ready, 0);
 
 	ret = hid_sensor_setup_trigger(indio_dev, name,
-				&humid_st->common_attributes);
+				       &humid_st->common_attributes);
 	if (ret)
 		return ret;
 
@@ -241,7 +242,7 @@ static int hid_humidity_probe(struct platform_device *pdev)
 
 	humidity_callbacks.pdev = pdev;
 	ret = sensor_hub_register_callback(hsdev, HID_USAGE_SENSOR_HUMIDITY,
-					&humidity_callbacks);
+					   &humidity_callbacks);
 	if (ret)
 		goto error_remove_trigger;
 
diff --git a/drivers/iio/humidity/hts221_spi.c b/drivers/iio/humidity/hts221_spi.c
index e6fef2acd523e..f962842cc71df 100644
--- a/drivers/iio/humidity/hts221_spi.c
+++ b/drivers/iio/humidity/hts221_spi.c
@@ -47,7 +47,7 @@ static const struct of_device_id hts221_spi_of_match[] = {
 MODULE_DEVICE_TABLE(of, hts221_spi_of_match);
 
 static const struct spi_device_id hts221_spi_id_table[] = {
-	{ HTS221_DEV_NAME },
+	{ .name = HTS221_DEV_NAME },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, hts221_spi_id_table);
diff --git a/drivers/iio/iio_core_trigger.h b/drivers/iio/iio_core_trigger.h
index e1a56824e07fe..4698a857739fc 100644
--- a/drivers/iio/iio_core_trigger.h
+++ b/drivers/iio/iio_core_trigger.h
@@ -56,4 +56,4 @@ static inline int iio_trigger_detach_poll_func(struct iio_trigger *trig,
 	return 0;
 }
 
-#endif /* CONFIG_TRIGGER_CONSUMER */
+#endif /* CONFIG_IIO_TRIGGER */
diff --git a/drivers/iio/imu/adis16400.c b/drivers/iio/imu/adis16400.c
index 36323ad149e04..4842346f9f0c0 100644
--- a/drivers/iio/imu/adis16400.c
+++ b/drivers/iio/imu/adis16400.c
@@ -1189,21 +1189,21 @@ static int adis16400_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id adis16400_id[] = {
-	{ "adis16300", (kernel_ulong_t)&adis16300_chip_info },
-	{ "adis16305", (kernel_ulong_t)&adis16300_chip_info },
-	{ "adis16334", (kernel_ulong_t)&adis16334_chip_info },
-	{ "adis16350", (kernel_ulong_t)&adis16350_chip_info },
-	{ "adis16354", (kernel_ulong_t)&adis16350_chip_info },
-	{ "adis16355", (kernel_ulong_t)&adis16350_chip_info },
-	{ "adis16360", (kernel_ulong_t)&adis16360_chip_info },
-	{ "adis16362", (kernel_ulong_t)&adis16362_chip_info },
-	{ "adis16364", (kernel_ulong_t)&adis16364_chip_info },
-	{ "adis16365", (kernel_ulong_t)&adis16360_chip_info },
-	{ "adis16367", (kernel_ulong_t)&adis16367_chip_info },
-	{ "adis16400", (kernel_ulong_t)&adis16400_chip_info },
-	{ "adis16405", (kernel_ulong_t)&adis16400_chip_info },
-	{ "adis16445", (kernel_ulong_t)&adis16445_chip_info },
-	{ "adis16448", (kernel_ulong_t)&adis16448_chip_info },
+	{ .name = "adis16300", .driver_data = (kernel_ulong_t)&adis16300_chip_info },
+	{ .name = "adis16305", .driver_data = (kernel_ulong_t)&adis16300_chip_info },
+	{ .name = "adis16334", .driver_data = (kernel_ulong_t)&adis16334_chip_info },
+	{ .name = "adis16350", .driver_data = (kernel_ulong_t)&adis16350_chip_info },
+	{ .name = "adis16354", .driver_data = (kernel_ulong_t)&adis16350_chip_info },
+	{ .name = "adis16355", .driver_data = (kernel_ulong_t)&adis16350_chip_info },
+	{ .name = "adis16360", .driver_data = (kernel_ulong_t)&adis16360_chip_info },
+	{ .name = "adis16362", .driver_data = (kernel_ulong_t)&adis16362_chip_info },
+	{ .name = "adis16364", .driver_data = (kernel_ulong_t)&adis16364_chip_info },
+	{ .name = "adis16365", .driver_data = (kernel_ulong_t)&adis16360_chip_info },
+	{ .name = "adis16367", .driver_data = (kernel_ulong_t)&adis16367_chip_info },
+	{ .name = "adis16400", .driver_data = (kernel_ulong_t)&adis16400_chip_info },
+	{ .name = "adis16405", .driver_data = (kernel_ulong_t)&adis16400_chip_info },
+	{ .name = "adis16445", .driver_data = (kernel_ulong_t)&adis16445_chip_info },
+	{ .name = "adis16448", .driver_data = (kernel_ulong_t)&adis16448_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adis16400_id);
diff --git a/drivers/iio/imu/adis16460.c b/drivers/iio/imu/adis16460.c
index ba1887d36577c..cadf21c05bbfc 100644
--- a/drivers/iio/imu/adis16460.c
+++ b/drivers/iio/imu/adis16460.c
@@ -394,7 +394,7 @@ static int adis16460_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id adis16460_ids[] = {
-	{ "adis16460", 0 },
+	{ .name = "adis16460" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adis16460_ids);
diff --git a/drivers/iio/imu/adis16475.c b/drivers/iio/imu/adis16475.c
index 17335386d8e3a..5f21dceda8f7e 100644
--- a/drivers/iio/imu/adis16475.c
+++ b/drivers/iio/imu/adis16475.c
@@ -2061,33 +2061,33 @@ static const struct of_device_id adis16475_of_match[] = {
 MODULE_DEVICE_TABLE(of, adis16475_of_match);
 
 static const struct spi_device_id adis16475_ids[] = {
-	{ "adis16470", (kernel_ulong_t)&adis16475_chip_info[ADIS16470] },
-	{ "adis16475-1", (kernel_ulong_t)&adis16475_chip_info[ADIS16475_1] },
-	{ "adis16475-2", (kernel_ulong_t)&adis16475_chip_info[ADIS16475_2] },
-	{ "adis16475-3", (kernel_ulong_t)&adis16475_chip_info[ADIS16475_3] },
-	{ "adis16477-1", (kernel_ulong_t)&adis16475_chip_info[ADIS16477_1] },
-	{ "adis16477-2", (kernel_ulong_t)&adis16475_chip_info[ADIS16477_2] },
-	{ "adis16477-3", (kernel_ulong_t)&adis16475_chip_info[ADIS16477_3] },
-	{ "adis16465-1", (kernel_ulong_t)&adis16475_chip_info[ADIS16465_1] },
-	{ "adis16465-2", (kernel_ulong_t)&adis16475_chip_info[ADIS16465_2] },
-	{ "adis16465-3", (kernel_ulong_t)&adis16475_chip_info[ADIS16465_3] },
-	{ "adis16467-1", (kernel_ulong_t)&adis16475_chip_info[ADIS16467_1] },
-	{ "adis16467-2", (kernel_ulong_t)&adis16475_chip_info[ADIS16467_2] },
-	{ "adis16467-3", (kernel_ulong_t)&adis16475_chip_info[ADIS16467_3] },
-	{ "adis16500", (kernel_ulong_t)&adis16475_chip_info[ADIS16500] },
-	{ "adis16501", (kernel_ulong_t)&adis16475_chip_info[ADIS16501] },
-	{ "adis16505-1", (kernel_ulong_t)&adis16475_chip_info[ADIS16505_1] },
-	{ "adis16505-2", (kernel_ulong_t)&adis16475_chip_info[ADIS16505_2] },
-	{ "adis16505-3", (kernel_ulong_t)&adis16475_chip_info[ADIS16505_3] },
-	{ "adis16507-1", (kernel_ulong_t)&adis16475_chip_info[ADIS16507_1] },
-	{ "adis16507-2", (kernel_ulong_t)&adis16475_chip_info[ADIS16507_2] },
-	{ "adis16507-3", (kernel_ulong_t)&adis16475_chip_info[ADIS16507_3] },
-	{ "adis16575-2", (kernel_ulong_t)&adis16475_chip_info[ADIS16575_2] },
-	{ "adis16575-3", (kernel_ulong_t)&adis16475_chip_info[ADIS16575_3] },
-	{ "adis16576-2", (kernel_ulong_t)&adis16475_chip_info[ADIS16576_2] },
-	{ "adis16576-3", (kernel_ulong_t)&adis16475_chip_info[ADIS16576_3] },
-	{ "adis16577-2", (kernel_ulong_t)&adis16475_chip_info[ADIS16577_2] },
-	{ "adis16577-3", (kernel_ulong_t)&adis16475_chip_info[ADIS16577_3] },
+	{ .name = "adis16470", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16470] },
+	{ .name = "adis16475-1", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16475_1] },
+	{ .name = "adis16475-2", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16475_2] },
+	{ .name = "adis16475-3", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16475_3] },
+	{ .name = "adis16477-1", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16477_1] },
+	{ .name = "adis16477-2", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16477_2] },
+	{ .name = "adis16477-3", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16477_3] },
+	{ .name = "adis16465-1", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16465_1] },
+	{ .name = "adis16465-2", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16465_2] },
+	{ .name = "adis16465-3", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16465_3] },
+	{ .name = "adis16467-1", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16467_1] },
+	{ .name = "adis16467-2", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16467_2] },
+	{ .name = "adis16467-3", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16467_3] },
+	{ .name = "adis16500", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16500] },
+	{ .name = "adis16501", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16501] },
+	{ .name = "adis16505-1", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16505_1] },
+	{ .name = "adis16505-2", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16505_2] },
+	{ .name = "adis16505-3", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16505_3] },
+	{ .name = "adis16507-1", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16507_1] },
+	{ .name = "adis16507-2", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16507_2] },
+	{ .name = "adis16507-3", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16507_3] },
+	{ .name = "adis16575-2", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16575_2] },
+	{ .name = "adis16575-3", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16575_3] },
+	{ .name = "adis16576-2", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16576_2] },
+	{ .name = "adis16576-3", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16576_3] },
+	{ .name = "adis16577-2", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16577_2] },
+	{ .name = "adis16577-3", .driver_data = (kernel_ulong_t)&adis16475_chip_info[ADIS16577_3] },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adis16475_ids);
diff --git a/drivers/iio/imu/adis16480.c b/drivers/iio/imu/adis16480.c
index e009f3824768c..5187566c18763 100644
--- a/drivers/iio/imu/adis16480.c
+++ b/drivers/iio/imu/adis16480.c
@@ -110,7 +110,7 @@
 
 #define ADIS16480_REG_SERIAL_NUM		ADIS16480_REG(0x04, 0x20)
 
-/* Each filter coefficent bank spans two pages */
+/* Each filter coefficient bank spans two pages */
 #define ADIS16480_FIR_COEF(page) (x < 60 ? ADIS16480_REG(page, (x) + 8) : \
 		ADIS16480_REG((page) + 1, (x) - 60 + 8))
 #define ADIS16480_FIR_COEF_A(x)			ADIS16480_FIR_COEF(0x05, (x))
@@ -1467,7 +1467,7 @@ static irqreturn_t adis16480_trigger_handler(int irq, void *p)
 			 * We need to perform the padding to have the buffer
 			 * elements naturally aligned in case there are any
 			 * 32-bit storage size channels enabled which are added
-			 * in the buffer after the temprature data. In case
+			 * in the buffer after the temperature data. In case
 			 * there is no data being added after the temperature
 			 * data, the padding is harmless.
 			 */
@@ -1806,26 +1806,26 @@ static int adis16480_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id adis16480_ids[] = {
-	{ "adis16375", ADIS16375 },
-	{ "adis16480", ADIS16480 },
-	{ "adis16485", ADIS16485 },
-	{ "adis16486", ADIS16486 },
-	{ "adis16487", ADIS16487 },
-	{ "adis16488", ADIS16488 },
-	{ "adis16489", ADIS16489 },
-	{ "adis16490", ADIS16490 },
-	{ "adis16495-1", ADIS16495_1 },
-	{ "adis16495-2", ADIS16495_2 },
-	{ "adis16495-3", ADIS16495_3 },
-	{ "adis16497-1", ADIS16497_1 },
-	{ "adis16497-2", ADIS16497_2 },
-	{ "adis16497-3", ADIS16497_3 },
-	{ "adis16545-1", ADIS16545_1 },
-	{ "adis16545-2", ADIS16545_2 },
-	{ "adis16545-3", ADIS16545_3 },
-	{ "adis16547-1", ADIS16547_1 },
-	{ "adis16547-2", ADIS16547_2 },
-	{ "adis16547-3", ADIS16547_3 },
+	{ .name = "adis16375", .driver_data = ADIS16375 },
+	{ .name = "adis16480", .driver_data = ADIS16480 },
+	{ .name = "adis16485", .driver_data = ADIS16485 },
+	{ .name = "adis16486", .driver_data = ADIS16486 },
+	{ .name = "adis16487", .driver_data = ADIS16487 },
+	{ .name = "adis16488", .driver_data = ADIS16488 },
+	{ .name = "adis16489", .driver_data = ADIS16489 },
+	{ .name = "adis16490", .driver_data = ADIS16490 },
+	{ .name = "adis16495-1", .driver_data = ADIS16495_1 },
+	{ .name = "adis16495-2", .driver_data = ADIS16495_2 },
+	{ .name = "adis16495-3", .driver_data = ADIS16495_3 },
+	{ .name = "adis16497-1", .driver_data = ADIS16497_1 },
+	{ .name = "adis16497-2", .driver_data = ADIS16497_2 },
+	{ .name = "adis16497-3", .driver_data = ADIS16497_3 },
+	{ .name = "adis16545-1", .driver_data = ADIS16545_1 },
+	{ .name = "adis16545-2", .driver_data = ADIS16545_2 },
+	{ .name = "adis16545-3", .driver_data = ADIS16545_3 },
+	{ .name = "adis16547-1", .driver_data = ADIS16547_1 },
+	{ .name = "adis16547-2", .driver_data = ADIS16547_2 },
+	{ .name = "adis16547-3", .driver_data = ADIS16547_3 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adis16480_ids);
diff --git a/drivers/iio/imu/adis16550.c b/drivers/iio/imu/adis16550.c
index 1e435d60cd6d1..096a612865c42 100644
--- a/drivers/iio/imu/adis16550.c
+++ b/drivers/iio/imu/adis16550.c
@@ -88,25 +88,7 @@ struct adis16550_sync {
 	u16 max_rate;
 };
 
-struct adis16550_chip_info {
-	const struct iio_chan_spec *channels;
-	const struct adis16550_sync *sync_mode;
-	char *name;
-	u32 num_channels;
-	u32 gyro_max_val;
-	u32 gyro_max_scale;
-	u32 accel_max_val;
-	u32 accel_max_scale;
-	u32 temp_scale;
-	u32 deltang_max_val;
-	u32 deltvel_max_val;
-	u32 int_clk;
-	u16 max_dec;
-	u16 num_sync;
-};
-
 struct adis16550 {
-	const struct adis16550_chip_info *info;
 	struct adis adis;
 	unsigned long clk_freq_hz;
 	u32 sync_mode;
@@ -449,8 +431,8 @@ static int adis16550_set_freq_hz(struct adis16550 *st, u32 freq_hz)
 	 * The optimal sample rate for the supported IMUs is between
 	 * int_clk - 1000 and int_clk + 500.
 	 */
-	u32 max_sample_rate = st->info->int_clk * 1000 + 500000;
-	u32 min_sample_rate = st->info->int_clk * 1000 - 1000000;
+	u32 max_sample_rate = 4000 * 1000 + 500000;
+	u32 min_sample_rate = 4000 * 1000 - 1000000;
 
 	if (!freq_hz)
 		return -EINVAL;
@@ -483,7 +465,7 @@ static int adis16550_set_freq_hz(struct adis16550 *st, u32 freq_hz)
 	if (dec)
 		dec--;
 
-	dec = min(dec, st->info->max_dec);
+	dec = min(dec, 4095);
 
 	return __adis_write_reg_16(&st->adis, ADIS16550_REG_DEC_RATE, dec);
 }
@@ -591,30 +573,30 @@ static int adis16550_read_raw(struct iio_dev *indio_dev,
 	case IIO_CHAN_INFO_SCALE:
 		switch (chan->type) {
 		case IIO_ANGL_VEL:
-			*val = st->info->gyro_max_val;
-			*val2 = st->info->gyro_max_scale;
+			*val = 1;
+			*val2 = IIO_RAD_TO_DEGREE(80 << 16);
 			return IIO_VAL_FRACTIONAL;
 		case IIO_ACCEL:
-			*val = st->info->accel_max_val;
-			*val2 = st->info->accel_max_scale;
+			*val = 1;
+			*val2 = IIO_M_S_2_TO_G(102400000);
 			return IIO_VAL_FRACTIONAL;
 		case IIO_TEMP:
-			*val = st->info->temp_scale;
+			*val = 4;
 			return IIO_VAL_INT;
 		case IIO_DELTA_ANGL:
-			*val = st->info->deltang_max_val;
+			*val = IIO_DEGREE_TO_RAD(720);
 			*val2 = 31;
 			return IIO_VAL_FRACTIONAL_LOG2;
 		case IIO_DELTA_VELOCITY:
-			*val = st->info->deltvel_max_val;
+			*val = 125;
 			*val2 = 31;
 			return IIO_VAL_FRACTIONAL_LOG2;
 		default:
 			return -EINVAL;
 		}
 	case IIO_CHAN_INFO_OFFSET:
-		/* temperature centered at 25°C */
-		*val = DIV_ROUND_CLOSEST(25000, st->info->temp_scale);
+		/* temperature centered at 25°C divided by temp scale */
+		*val = 25000 / 4;
 		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_CALIBBIAS:
 		ret = adis_read_reg_32(&st->adis,
@@ -792,23 +774,6 @@ static const struct adis16550_sync adis16550_sync_modes[] = {
 	{ ADIS16550_SYNC_MODE_SCALED, 1, 128 },
 };
 
-static const struct adis16550_chip_info adis16550_chip_info = {
-	.num_channels = ARRAY_SIZE(adis16550_channels),
-	.channels = adis16550_channels,
-	.name = "adis16550",
-	.gyro_max_val = 1,
-	.gyro_max_scale = IIO_RAD_TO_DEGREE(80 << 16),
-	.accel_max_val = 1,
-	.accel_max_scale = IIO_M_S_2_TO_G(102400000),
-	.temp_scale = 4,
-	.deltang_max_val = IIO_DEGREE_TO_RAD(720),
-	.deltvel_max_val = 125,
-	.int_clk = 4000,
-	.max_dec = 4095,
-	.sync_mode = adis16550_sync_modes,
-	.num_sync = ARRAY_SIZE(adis16550_sync_modes),
-};
-
 static u32 adis16550_validate_crc(__be32 *buffer, const u8 n_elem)
 {
 	int i;
@@ -917,21 +882,21 @@ static int adis16550_config_sync(struct adis16550 *st)
 	if (IS_ERR(clk))
 		return PTR_ERR(clk);
 	if (!clk) {
-		st->clk_freq_hz = st->info->int_clk * 1000;
+		st->clk_freq_hz = 4000000;
 		return 0;
 	}
 
 	st->clk_freq_hz = clk_get_rate(clk);
 
-	for (i = 0; i < st->info->num_sync; i++) {
-		if (st->clk_freq_hz >= st->info->sync_mode[i].min_rate &&
-		    st->clk_freq_hz <= st->info->sync_mode[i].max_rate) {
-			sync_mode_data = &st->info->sync_mode[i];
+	for (i = 0; i < ARRAY_SIZE(adis16550_sync_modes); i++) {
+		if (st->clk_freq_hz >= adis16550_sync_modes[i].min_rate &&
+		    st->clk_freq_hz <= adis16550_sync_modes[i].max_rate) {
+			sync_mode_data = &adis16550_sync_modes[i];
 			break;
 		}
 	}
 
-	if (i == st->info->num_sync)
+	if (i == ARRAY_SIZE(adis16550_sync_modes))
 		return dev_err_probe(dev, -EINVAL, "Clk rate: %lu not in a valid range",
 				     st->clk_freq_hz);
 
@@ -942,7 +907,7 @@ static int adis16550_config_sync(struct adis16550 *st)
 		 * of [3000 4500].
 		 */
 
-		sync_scale = DIV_ROUND_CLOSEST(st->info->int_clk, st->clk_freq_hz);
+		sync_scale = DIV_ROUND_CLOSEST(4000, st->clk_freq_hz);
 
 		if (3000 > sync_scale || 4500 < sync_scale)
 			return dev_err_probe(dev, -EINVAL,
@@ -954,7 +919,7 @@ static int adis16550_config_sync(struct adis16550 *st)
 		if (ret)
 			return ret;
 
-		st->clk_freq_hz = st->info->int_clk;
+		st->clk_freq_hz = 4000;
 	}
 
 	st->clk_freq_hz *= 1000;
@@ -1063,13 +1028,11 @@ static int adis16550_probe(struct spi_device *spi)
 		return -ENOMEM;
 
 	st = iio_priv(indio_dev);
-	st->info = spi_get_device_match_data(spi);
-	if (!st->info)
-		return -EINVAL;
+
 	adis = &st->adis;
-	indio_dev->name = st->info->name;
-	indio_dev->channels = st->info->channels;
-	indio_dev->num_channels = st->info->num_channels;
+	indio_dev->name = "adis16550";
+	indio_dev->channels = adis16550_channels;
+	indio_dev->num_channels = ARRAY_SIZE(adis16550_channels);
 	indio_dev->available_scan_masks = adis16550_channel_masks;
 	indio_dev->info = &adis16550_info;
 	indio_dev->modes = INDIO_DIRECT_MODE;
@@ -1116,13 +1079,13 @@ static int adis16550_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id adis16550_id[] = {
-	{ "adis16550",  (kernel_ulong_t)&adis16550_chip_info},
+	{ .name = "adis16550" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, adis16550_id);
 
 static const struct of_device_id adis16550_of_match[] = {
-	{ .compatible = "adi,adis16550", .data = &adis16550_chip_info },
+	{ .compatible = "adi,adis16550" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, adis16550_of_match);
diff --git a/drivers/iio/imu/bmi160/bmi160_core.c b/drivers/iio/imu/bmi160/bmi160_core.c
index 86f6ecfd64aa2..1e73a7f1f0170 100644
--- a/drivers/iio/imu/bmi160/bmi160_core.c
+++ b/drivers/iio/imu/bmi160/bmi160_core.c
@@ -5,7 +5,7 @@
  * Copyright (c) 2016, Intel Corporation.
  * Copyright (c) 2019, Martin Kelly.
  *
- * IIO core driver for BMI160, with support for I2C/SPI busses
+ * IIO core driver for BMI160, with support for I2C/SPI buses
  *
  * TODO: magnetometer, hardware FIFO
  */
diff --git a/drivers/iio/imu/bmi160/bmi160_spi.c b/drivers/iio/imu/bmi160/bmi160_spi.c
index 2f0a578ee40f2..107eb33749229 100644
--- a/drivers/iio/imu/bmi160/bmi160_spi.c
+++ b/drivers/iio/imu/bmi160/bmi160_spi.c
@@ -34,8 +34,8 @@ static int bmi160_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id bmi160_spi_id[] = {
-	{"bmi120", 0},
-	{"bmi160", 0},
+	{ .name = "bmi120" },
+	{ .name = "bmi160" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, bmi160_spi_id);
diff --git a/drivers/iio/imu/bmi270/bmi270_spi.c b/drivers/iio/imu/bmi270/bmi270_spi.c
index dc7fa01421bc9..759b56b59e544 100644
--- a/drivers/iio/imu/bmi270/bmi270_spi.c
+++ b/drivers/iio/imu/bmi270/bmi270_spi.c
@@ -65,8 +65,8 @@ static int bmi270_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id bmi270_spi_id[] = {
-	{ "bmi260", (kernel_ulong_t)&bmi260_chip_info },
-	{ "bmi270", (kernel_ulong_t)&bmi270_chip_info },
+	{ .name = "bmi260", .driver_data = (kernel_ulong_t)&bmi260_chip_info },
+	{ .name = "bmi270", .driver_data = (kernel_ulong_t)&bmi270_chip_info },
 	{ }
 };
 
@@ -75,6 +75,7 @@ static const struct of_device_id bmi270_of_match[] = {
 	{ .compatible = "bosch,bmi270", .data = &bmi270_chip_info },
 	{ }
 };
+MODULE_DEVICE_TABLE(of, bmi270_of_match);
 
 static struct spi_driver bmi270_spi_driver = {
 	.driver = {
diff --git a/drivers/iio/imu/bmi323/bmi323_core.c b/drivers/iio/imu/bmi323/bmi323_core.c
index f3d4994233998..72955a697a930 100644
--- a/drivers/iio/imu/bmi323/bmi323_core.c
+++ b/drivers/iio/imu/bmi323/bmi323_core.c
@@ -1128,7 +1128,7 @@ static int bmi323_set_watermark(struct iio_dev *indio_dev, unsigned int val)
 {
 	struct bmi323_data *data = iio_priv(indio_dev);
 
-	val = min(val, (u32)BMI323_FIFO_FULL_IN_FRAMES);
+	val = min(val, BMI323_FIFO_FULL_IN_FRAMES);
 
 	guard(mutex)(&data->mutex);
 	data->watermark = val;
diff --git a/drivers/iio/imu/bmi323/bmi323_spi.c b/drivers/iio/imu/bmi323/bmi323_spi.c
index 92f1c4bcc1929..ed0235cc40da2 100644
--- a/drivers/iio/imu/bmi323/bmi323_spi.c
+++ b/drivers/iio/imu/bmi323/bmi323_spi.c
@@ -64,7 +64,7 @@ static int bmi323_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id bmi323_spi_ids[] = {
-	{ "bmi323" },
+	{ .name = "bmi323" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, bmi323_spi_ids);
diff --git a/drivers/iio/imu/fxos8700_core.c b/drivers/iio/imu/fxos8700_core.c
index 281ebfd9c15a8..9d2eb6fbab4a2 100644
--- a/drivers/iio/imu/fxos8700_core.c
+++ b/drivers/iio/imu/fxos8700_core.c
@@ -2,7 +2,7 @@
 /*
  * FXOS8700 - NXP IMU (accelerometer plus magnetometer)
  *
- * IIO core driver for FXOS8700, with support for I2C/SPI busses
+ * IIO core driver for FXOS8700, with support for I2C/SPI buses
  *
  * TODO: Buffer, trigger, and IRQ support
  */
diff --git a/drivers/iio/imu/fxos8700_spi.c b/drivers/iio/imu/fxos8700_spi.c
index 3edf90220bfa5..56ad0c5914d98 100644
--- a/drivers/iio/imu/fxos8700_spi.c
+++ b/drivers/iio/imu/fxos8700_spi.c
@@ -23,7 +23,7 @@ static int fxos8700_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id fxos8700_spi_id[] = {
-	{"fxos8700", 0},
+	{ .name = "fxos8700" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, fxos8700_spi_id);
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600.h b/drivers/iio/imu/inv_icm42600/inv_icm42600.h
index c8b48a5c5ed06..b55d993f02643 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600.h
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600.h
@@ -6,13 +6,15 @@
 #ifndef INV_ICM42600_H_
 #define INV_ICM42600_H_
 
-#include <linux/bits.h>
 #include <linux/bitfield.h>
-#include <linux/regmap.h>
+#include <linux/bits.h>
 #include <linux/mutex.h>
-#include <linux/regulator/consumer.h>
 #include <linux/pm.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+
 #include <linux/iio/iio.h>
+
 #include <linux/iio/common/inv_sensors_timestamp.h>
 
 #include "inv_icm42600_buffer.h"
@@ -354,6 +356,8 @@ struct inv_icm42600_sensor_state {
 		cpu_to_le16((_wm) & GENMASK(11, 0))
 /* FIFO is 2048 bytes, let 12 samples for reading latency */
 #define INV_ICM42600_FIFO_WATERMARK_MAX			(2048 - 12 * 16)
+/* INV_ICM42600_FIFO_WATERMARK_MAX / 8 = 232 */
+#define INV_ICM42600_FIFO_WATERMARK_MAX_SAMPLES		232
 
 #define INV_ICM42600_REG_INT_CONFIG1			0x0064
 #define INV_ICM42600_INT_CONFIG1_TPULSE_DURATION	BIT(6)
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c
index 7df920ef3cf08..4b0e3cd8a5067 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c
@@ -3,25 +3,26 @@
  * Copyright (C) 2020 Invensense, Inc.
  */
 
-#include <linux/kernel.h>
+#include <linux/delay.h>
 #include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/math64.h>
+#include <linux/minmax.h>
 #include <linux/mutex.h>
 #include <linux/pm_runtime.h>
 #include <linux/regmap.h>
-#include <linux/delay.h>
-#include <linux/math64.h>
-#include <linux/minmax.h>
 #include <linux/units.h>
 
 #include <linux/iio/buffer.h>
-#include <linux/iio/common/inv_sensors_timestamp.h>
 #include <linux/iio/events.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/kfifo_buf.h>
 
+#include <linux/iio/common/inv_sensors_timestamp.h>
+
 #include "inv_icm42600.h"
-#include "inv_icm42600_temp.h"
 #include "inv_icm42600_buffer.h"
+#include "inv_icm42600_temp.h"
 
 #define INV_ICM42600_ACCEL_CHAN(_modifier, _index, _ext_info)		\
 	{								\
@@ -1186,8 +1187,9 @@ struct iio_dev *inv_icm42600_accel_init(struct inv_icm42600_state *st)
 	indio_dev->num_channels = ARRAY_SIZE(inv_icm42600_accel_channels);
 	indio_dev->available_scan_masks = inv_icm42600_accel_scan_masks;
 
-	ret = devm_iio_kfifo_buffer_setup(dev, indio_dev,
-					  &inv_icm42600_buffer_ops);
+	ret = devm_iio_kfifo_buffer_setup_ext(dev, indio_dev,
+					      &inv_icm42600_buffer_ops,
+					      inv_icm42600_buffer_attrs);
 	if (ret)
 		return ERR_PTR(ret);
 
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c
index 5c3840acf0857..998d312f7bde3 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c
@@ -3,17 +3,20 @@
  * Copyright (C) 2020 Invensense, Inc.
  */
 
-#include <linux/kernel.h>
+#include <linux/delay.h>
 #include <linux/device.h>
+#include <linux/kernel.h>
 #include <linux/minmax.h>
 #include <linux/mutex.h>
 #include <linux/pm_runtime.h>
 #include <linux/regmap.h>
-#include <linux/delay.h>
+#include <linux/stringify.h>
 
 #include <linux/iio/buffer.h>
-#include <linux/iio/common/inv_sensors_timestamp.h>
 #include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+
+#include <linux/iio/common/inv_sensors_timestamp.h>
 
 #include "inv_icm42600.h"
 #include "inv_icm42600_buffer.h"
@@ -186,7 +189,7 @@ static unsigned int inv_icm42600_wm_truncate(unsigned int watermark,
  * smallest latency but this is not as simple as choosing the smallest watermark
  * value. Latency depends on watermark and ODR. It requires several steps:
  * 1) compute gyro and accel latencies and choose the smallest value.
- * 2) adapt the choosen latency so that it is a multiple of both gyro and accel
+ * 2) adapt the chosen latency so that it is a multiple of both gyro and accel
  *    ones. Otherwise it is possible that you don't meet a requirement. (for
  *    example with gyro @100Hz wm 4 and accel @100Hz with wm 6, choosing the
  *    value of 4 will not meet accel latency requirement because 6 is not a
@@ -438,6 +441,40 @@ const struct iio_buffer_setup_ops inv_icm42600_buffer_ops = {
 	.postdisable = inv_icm42600_buffer_postdisable,
 };
 
+static ssize_t hwfifo_watermark_show(struct device *dev,
+				     struct device_attribute *attr,
+				     char *buf)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct inv_icm42600_state *st = iio_device_get_drvdata(indio_dev);
+	unsigned int wm;
+
+	guard(mutex)(&st->lock);
+
+	if (indio_dev == st->indio_accel)
+		wm = st->fifo.watermark.eff_accel;
+	else if (indio_dev == st->indio_gyro)
+		wm = st->fifo.watermark.eff_gyro;
+	else
+		return -EINVAL;
+
+	return sysfs_emit(buf, "%u\n", wm);
+}
+
+IIO_STATIC_CONST_DEVICE_ATTR(hwfifo_watermark_min, "1");
+IIO_STATIC_CONST_DEVICE_ATTR(hwfifo_watermark_max,
+			     __stringify(INV_ICM42600_FIFO_WATERMARK_MAX_SAMPLES));
+static IIO_DEVICE_ATTR_RO(hwfifo_watermark, 0);
+IIO_STATIC_CONST_DEVICE_ATTR(hwfifo_enabled, "1");
+
+const struct iio_dev_attr *inv_icm42600_buffer_attrs[] = {
+	&iio_dev_attr_hwfifo_watermark_min,
+	&iio_dev_attr_hwfifo_watermark_max,
+	&iio_dev_attr_hwfifo_watermark,
+	&iio_dev_attr_hwfifo_enabled,
+	NULL
+};
+
 int inv_icm42600_buffer_fifo_read(struct inv_icm42600_state *st,
 				  unsigned int max)
 {
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.h b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.h
index 88b8b9f780af3..e37ee34163378 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.h
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.h
@@ -6,8 +6,8 @@
 #ifndef INV_ICM42600_BUFFER_H_
 #define INV_ICM42600_BUFFER_H_
 
-#include <linux/kernel.h>
 #include <linux/bits.h>
+#include <linux/kernel.h>
 
 struct inv_icm42600_state;
 
@@ -80,6 +80,7 @@ ssize_t inv_icm42600_fifo_decode_packet(const void *packet, const void **accel,
 					const void **timestamp, unsigned int *odr);
 
 extern const struct iio_buffer_setup_ops inv_icm42600_buffer_ops;
+extern const struct iio_dev_attr *inv_icm42600_buffer_attrs[];
 
 int inv_icm42600_buffer_init(struct inv_icm42600_state *st);
 
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_core.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_core.c
index 76eb22488e5f3..dc97d8a274e36 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_core.c
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_core.c
@@ -3,18 +3,18 @@
  * Copyright (C) 2020 Invensense, Inc.
  */
 
-#include <linux/kernel.h>
-#include <linux/device.h>
-#include <linux/module.h>
-#include <linux/slab.h>
 #include <linux/delay.h>
-#include <linux/mutex.h>
+#include <linux/device.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
-#include <linux/regulator/consumer.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/pm_runtime.h>
 #include <linux/property.h>
 #include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/slab.h>
 
 #include <linux/iio/iio.h>
 
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c
index a18dcac939298..253bf571439d9 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c
@@ -3,22 +3,23 @@
  * Copyright (C) 2020 Invensense, Inc.
  */
 
-#include <linux/kernel.h>
+#include <linux/delay.h>
 #include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/math64.h>
 #include <linux/mutex.h>
 #include <linux/pm_runtime.h>
 #include <linux/regmap.h>
-#include <linux/delay.h>
-#include <linux/math64.h>
 
 #include <linux/iio/buffer.h>
-#include <linux/iio/common/inv_sensors_timestamp.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/kfifo_buf.h>
 
+#include <linux/iio/common/inv_sensors_timestamp.h>
+
 #include "inv_icm42600.h"
-#include "inv_icm42600_temp.h"
 #include "inv_icm42600_buffer.h"
+#include "inv_icm42600_temp.h"
 
 #define INV_ICM42600_GYRO_CHAN(_modifier, _index, _ext_info)		\
 	{								\
@@ -772,8 +773,9 @@ struct iio_dev *inv_icm42600_gyro_init(struct inv_icm42600_state *st)
 	indio_dev->available_scan_masks = inv_icm42600_gyro_scan_masks;
 	indio_dev->setup_ops = &inv_icm42600_buffer_ops;
 
-	ret = devm_iio_kfifo_buffer_setup(dev, indio_dev,
-					  &inv_icm42600_buffer_ops);
+	ret = devm_iio_kfifo_buffer_setup_ext(dev, indio_dev,
+					      &inv_icm42600_buffer_ops,
+					      inv_icm42600_buffer_attrs);
 	if (ret)
 		return ERR_PTR(ret);
 
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_i2c.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_i2c.c
index 1013aff4f0abd..28552d2db91d7 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_i2c.c
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_i2c.c
@@ -3,12 +3,12 @@
  * Copyright (C) 2020 InvenSense, Inc.
  */
 
-#include <linux/kernel.h>
 #include <linux/device.h>
-#include <linux/module.h>
 #include <linux/i2c.h>
-#include <linux/regmap.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
 #include <linux/property.h>
+#include <linux/regmap.h>
 
 #include "inv_icm42600.h"
 
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_spi.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_spi.c
index 57e3c448dca13..faf743bc64444 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_spi.c
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_spi.c
@@ -3,12 +3,12 @@
  * Copyright (C) 2020 InvenSense, Inc.
  */
 
-#include <linux/kernel.h>
 #include <linux/device.h>
+#include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/spi/spi.h>
-#include <linux/regmap.h>
 #include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/spi/spi.h>
 
 #include "inv_icm42600.h"
 
@@ -71,13 +71,13 @@ static int inv_icm42600_probe(struct spi_device *spi)
  * supported by this driver
  */
 static const struct spi_device_id inv_icm42600_id[] = {
-	{ "icm42600", INV_CHIP_ICM42600 },
-	{ "icm42602", INV_CHIP_ICM42602 },
-	{ "icm42605", INV_CHIP_ICM42605 },
-	{ "icm42686", INV_CHIP_ICM42686 },
-	{ "icm42622", INV_CHIP_ICM42622 },
-	{ "icm42688", INV_CHIP_ICM42688 },
-	{ "icm42631", INV_CHIP_ICM42631 },
+	{ .name = "icm42600", .driver_data = INV_CHIP_ICM42600 },
+	{ .name = "icm42602", .driver_data = INV_CHIP_ICM42602 },
+	{ .name = "icm42605", .driver_data = INV_CHIP_ICM42605 },
+	{ .name = "icm42686", .driver_data = INV_CHIP_ICM42686 },
+	{ .name = "icm42622", .driver_data = INV_CHIP_ICM42622 },
+	{ .name = "icm42688", .driver_data = INV_CHIP_ICM42688 },
+	{ .name = "icm42631", .driver_data = INV_CHIP_ICM42631 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, inv_icm42600_id);
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_temp.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_temp.c
index 727b03d541a58..7f80bda471c39 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_temp.c
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_temp.c
@@ -3,11 +3,12 @@
  * Copyright (C) 2020 Invensense, Inc.
  */
 
-#include <linux/kernel.h>
 #include <linux/device.h>
+#include <linux/kernel.h>
 #include <linux/mutex.h>
 #include <linux/pm_runtime.h>
 #include <linux/regmap.h>
+
 #include <linux/iio/iio.h>
 
 #include "inv_icm42600.h"
diff --git a/drivers/iio/imu/inv_icm45600/inv_icm45600.h b/drivers/iio/imu/inv_icm45600/inv_icm45600.h
index 1c796d4b2a403..95fa934a42b46 100644
--- a/drivers/iio/imu/inv_icm45600/inv_icm45600.h
+++ b/drivers/iio/imu/inv_icm45600/inv_icm45600.h
@@ -190,7 +190,7 @@ struct inv_icm45600_sensor_state {
 #define INV_ICM45600_REG_IREG_ADDR			0x7C
 #define INV_ICM45600_REG_IREG_DATA			0x7E
 
-/* Direct acces registers */
+/* Direct access registers */
 #define INV_ICM45600_REG_MISC2				0x007F
 #define INV_ICM45600_MISC2_SOFT_RESET			BIT(1)
 
diff --git a/drivers/iio/imu/inv_icm45600/inv_icm45600_buffer.c b/drivers/iio/imu/inv_icm45600/inv_icm45600_buffer.c
index 2b9ea317385ce..42111c543d3c5 100644
--- a/drivers/iio/imu/inv_icm45600/inv_icm45600_buffer.c
+++ b/drivers/iio/imu/inv_icm45600/inv_icm45600_buffer.c
@@ -422,8 +422,11 @@ int inv_icm45600_buffer_fifo_read(struct inv_icm45600_state *st,
 	if (max > 0 && fifo_nb > max)
 		fifo_nb = max;
 
-	/* Try to read all FIFO data in internal buffer. */
-	st->fifo.count = fifo_nb * packet_size;
+	/*
+	 * Read all FIFO data into the internal buffer, clamping the
+	 * device-reported count to the buffer capacity.
+	 */
+	st->fifo.count = min(fifo_nb * packet_size, INV_ICM45600_FIFO_SIZE_MAX);
 	ret = regmap_noinc_read(st->map, INV_ICM45600_REG_FIFO_DATA,
 				st->fifo.data, st->fifo.count);
 	if (ret == -ENOTSUPP || ret == -EFBIG) {
diff --git a/drivers/iio/imu/inv_icm45600/inv_icm45600_core.c b/drivers/iio/imu/inv_icm45600/inv_icm45600_core.c
index d49053161a657..c1d7aa7e950de 100644
--- a/drivers/iio/imu/inv_icm45600/inv_icm45600_core.c
+++ b/drivers/iio/imu/inv_icm45600/inv_icm45600_core.c
@@ -716,7 +716,7 @@ int inv_icm45600_core_probe(struct regmap *regmap, const struct inv_icm45600_chi
 
 	dev_set_drvdata(dev, st);
 
-	st->fifo.data = devm_kzalloc(dev, 8192, GFP_KERNEL);
+	st->fifo.data = devm_kzalloc(dev, INV_ICM45600_FIFO_SIZE_MAX, GFP_KERNEL);
 	if (!st->fifo.data)
 		return -ENOMEM;
 
diff --git a/drivers/iio/imu/inv_icm45600/inv_icm45600_i2c.c b/drivers/iio/imu/inv_icm45600/inv_icm45600_i2c.c
index 81ba1b60f04bb..69acf9bac2db6 100644
--- a/drivers/iio/imu/inv_icm45600/inv_icm45600_i2c.c
+++ b/drivers/iio/imu/inv_icm45600/inv_icm45600_i2c.c
@@ -22,7 +22,7 @@ static int inv_icm45600_probe(struct i2c_client *client)
 	if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_I2C_BLOCK))
 		return -ENODEV;
 
-	chip_info = device_get_match_data(&client->dev);
+	chip_info = i2c_get_match_data(client);
 	if (!chip_info)
 		return -ENODEV;
 
diff --git a/drivers/iio/imu/inv_icm45600/inv_icm45600_spi.c b/drivers/iio/imu/inv_icm45600/inv_icm45600_spi.c
index 450a0f2abaaaa..087e590e04c20 100644
--- a/drivers/iio/imu/inv_icm45600/inv_icm45600_spi.c
+++ b/drivers/iio/imu/inv_icm45600/inv_icm45600_spi.c
@@ -48,14 +48,14 @@ static int inv_icm45600_probe(struct spi_device *spi)
  * supported by this driver.
  */
 static const struct spi_device_id inv_icm45600_id[] = {
-	{ "icm45605", (kernel_ulong_t)&inv_icm45605_chip_info },
-	{ "icm45606", (kernel_ulong_t)&inv_icm45606_chip_info },
-	{ "icm45608", (kernel_ulong_t)&inv_icm45608_chip_info },
-	{ "icm45634", (kernel_ulong_t)&inv_icm45634_chip_info },
-	{ "icm45686", (kernel_ulong_t)&inv_icm45686_chip_info },
-	{ "icm45687", (kernel_ulong_t)&inv_icm45687_chip_info },
-	{ "icm45688p", (kernel_ulong_t)&inv_icm45688p_chip_info },
-	{ "icm45689", (kernel_ulong_t)&inv_icm45689_chip_info },
+	{ .name = "icm45605", .driver_data = (kernel_ulong_t)&inv_icm45605_chip_info },
+	{ .name = "icm45606", .driver_data = (kernel_ulong_t)&inv_icm45606_chip_info },
+	{ .name = "icm45608", .driver_data = (kernel_ulong_t)&inv_icm45608_chip_info },
+	{ .name = "icm45634", .driver_data = (kernel_ulong_t)&inv_icm45634_chip_info },
+	{ .name = "icm45686", .driver_data = (kernel_ulong_t)&inv_icm45686_chip_info },
+	{ .name = "icm45687", .driver_data = (kernel_ulong_t)&inv_icm45687_chip_info },
+	{ .name = "icm45688p", .driver_data = (kernel_ulong_t)&inv_icm45688p_chip_info },
+	{ .name = "icm45689", .driver_data = (kernel_ulong_t)&inv_icm45689_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, inv_icm45600_id);
diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_magn.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_magn.c
index 47394594d17ab..6b858fdfd1c6c 100644
--- a/drivers/iio/imu/inv_mpu6050/inv_mpu_magn.c
+++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_magn.c
@@ -106,9 +106,9 @@ static int inv_magn_init(struct inv_mpu6050_state *st)
 		return ret;
 
 	/*
-	 * Sensor sentivity
+	 * Sensor sensitivity
 	 * 1 uT = 0.01 G and value is in micron (1e6)
-	 * sensitvity = x uT * 0.01 * 1e6
+	 * sensitivity = x uT * 0.01 * 1e6
 	 */
 	switch (st->chip_type) {
 	case INV_MPU9150:
diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c
index b8204eb0b4c61..71e90ddc19dd4 100644
--- a/drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c
+++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c
@@ -65,23 +65,23 @@ static int inv_mpu_probe(struct spi_device *spi)
  * supported by this driver
  */
 static const struct spi_device_id inv_mpu_id[] = {
-	{"mpu6000", INV_MPU6000},
-	{"mpu6500", INV_MPU6500},
-	{"mpu6515", INV_MPU6515},
-	{"mpu6880", INV_MPU6880},
-	{"mpu9250", INV_MPU9250},
-	{"mpu9255", INV_MPU9255},
-	{"icm20608", INV_ICM20608},
-	{"icm20608d", INV_ICM20608D},
-	{"icm20609", INV_ICM20609},
-	{"icm20689", INV_ICM20689},
-	{"icm20600", INV_ICM20600},
-	{"icm20602", INV_ICM20602},
-	{"icm20690", INV_ICM20690},
-	{"iam20380", INV_IAM20380},
-	{"iam20680", INV_IAM20680},
-	{"iam20680hp", INV_IAM20680HP},
-	{"iam20680ht", INV_IAM20680HT},
+	{ .name = "mpu6000", .driver_data = INV_MPU6000 },
+	{ .name = "mpu6500", .driver_data = INV_MPU6500 },
+	{ .name = "mpu6515", .driver_data = INV_MPU6515 },
+	{ .name = "mpu6880", .driver_data = INV_MPU6880 },
+	{ .name = "mpu9250", .driver_data = INV_MPU9250 },
+	{ .name = "mpu9255", .driver_data = INV_MPU9255 },
+	{ .name = "icm20608", .driver_data = INV_ICM20608 },
+	{ .name = "icm20608d", .driver_data = INV_ICM20608D },
+	{ .name = "icm20609", .driver_data = INV_ICM20609 },
+	{ .name = "icm20689", .driver_data = INV_ICM20689 },
+	{ .name = "icm20600", .driver_data = INV_ICM20600 },
+	{ .name = "icm20602", .driver_data = INV_ICM20602 },
+	{ .name = "icm20690", .driver_data = INV_ICM20690 },
+	{ .name = "iam20380", .driver_data = INV_IAM20380 },
+	{ .name = "iam20680", .driver_data = INV_IAM20680 },
+	{ .name = "iam20680hp", .driver_data = INV_IAM20680HP },
+	{ .name = "iam20680ht", .driver_data = INV_IAM20680HT },
 	{ }
 };
 
diff --git a/drivers/iio/imu/smi240.c b/drivers/iio/imu/smi240.c
index d159ee59acdd4..f419296bba88f 100644
--- a/drivers/iio/imu/smi240.c
+++ b/drivers/iio/imu/smi240.c
@@ -593,7 +593,7 @@ static int smi240_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id smi240_spi_id[] = {
-	{ "smi240" },
+	{ .name = "smi240" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, smi240_spi_id);
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c
index 3389b15df0bc0..ed47d5a6f11c5 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c
@@ -138,30 +138,30 @@ static const struct of_device_id st_lsm6dsx_spi_of_match[] = {
 MODULE_DEVICE_TABLE(of, st_lsm6dsx_spi_of_match);
 
 static const struct spi_device_id st_lsm6dsx_spi_id_table[] = {
-	{ ST_LSM6DS3_DEV_NAME, ST_LSM6DS3_ID },
-	{ ST_LSM6DS3H_DEV_NAME, ST_LSM6DS3H_ID },
-	{ ST_LSM6DSL_DEV_NAME, ST_LSM6DSL_ID },
-	{ ST_LSM6DSM_DEV_NAME, ST_LSM6DSM_ID },
-	{ ST_ISM330DLC_DEV_NAME, ST_ISM330DLC_ID },
-	{ ST_LSM6DSO_DEV_NAME, ST_LSM6DSO_ID },
-	{ ST_ASM330LHH_DEV_NAME, ST_ASM330LHH_ID },
-	{ ST_LSM6DSOX_DEV_NAME, ST_LSM6DSOX_ID },
-	{ ST_LSM6DSR_DEV_NAME, ST_LSM6DSR_ID },
-	{ ST_LSM6DS3TRC_DEV_NAME, ST_LSM6DS3TRC_ID },
-	{ ST_ISM330DHCX_DEV_NAME, ST_ISM330DHCX_ID },
-	{ ST_LSM9DS1_DEV_NAME, ST_LSM9DS1_ID },
-	{ ST_LSM6DS0_DEV_NAME, ST_LSM6DS0_ID },
-	{ ST_LSM6DSRX_DEV_NAME, ST_LSM6DSRX_ID },
-	{ ST_LSM6DST_DEV_NAME, ST_LSM6DST_ID },
-	{ ST_LSM6DSOP_DEV_NAME, ST_LSM6DSOP_ID },
-	{ ST_ASM330LHHX_DEV_NAME, ST_ASM330LHHX_ID },
-	{ ST_LSM6DSTX_DEV_NAME, ST_LSM6DSTX_ID },
-	{ ST_LSM6DSV_DEV_NAME, ST_LSM6DSV_ID },
-	{ ST_LSM6DSV16X_DEV_NAME, ST_LSM6DSV16X_ID },
-	{ ST_LSM6DSO16IS_DEV_NAME, ST_LSM6DSO16IS_ID },
-	{ ST_ISM330IS_DEV_NAME, ST_ISM330IS_ID },
-	{ ST_ASM330LHB_DEV_NAME, ST_ASM330LHB_ID },
-	{ ST_ASM330LHHXG1_DEV_NAME, ST_ASM330LHHXG1_ID },
+	{ .name = ST_LSM6DS3_DEV_NAME, .driver_data = ST_LSM6DS3_ID },
+	{ .name = ST_LSM6DS3H_DEV_NAME, .driver_data = ST_LSM6DS3H_ID },
+	{ .name = ST_LSM6DSL_DEV_NAME, .driver_data = ST_LSM6DSL_ID },
+	{ .name = ST_LSM6DSM_DEV_NAME, .driver_data = ST_LSM6DSM_ID },
+	{ .name = ST_ISM330DLC_DEV_NAME, .driver_data = ST_ISM330DLC_ID },
+	{ .name = ST_LSM6DSO_DEV_NAME, .driver_data = ST_LSM6DSO_ID },
+	{ .name = ST_ASM330LHH_DEV_NAME, .driver_data = ST_ASM330LHH_ID },
+	{ .name = ST_LSM6DSOX_DEV_NAME, .driver_data = ST_LSM6DSOX_ID },
+	{ .name = ST_LSM6DSR_DEV_NAME, .driver_data = ST_LSM6DSR_ID },
+	{ .name = ST_LSM6DS3TRC_DEV_NAME, .driver_data = ST_LSM6DS3TRC_ID },
+	{ .name = ST_ISM330DHCX_DEV_NAME, .driver_data = ST_ISM330DHCX_ID },
+	{ .name = ST_LSM9DS1_DEV_NAME, .driver_data = ST_LSM9DS1_ID },
+	{ .name = ST_LSM6DS0_DEV_NAME, .driver_data = ST_LSM6DS0_ID },
+	{ .name = ST_LSM6DSRX_DEV_NAME, .driver_data = ST_LSM6DSRX_ID },
+	{ .name = ST_LSM6DST_DEV_NAME, .driver_data = ST_LSM6DST_ID },
+	{ .name = ST_LSM6DSOP_DEV_NAME, .driver_data = ST_LSM6DSOP_ID },
+	{ .name = ST_ASM330LHHX_DEV_NAME, .driver_data = ST_ASM330LHHX_ID },
+	{ .name = ST_LSM6DSTX_DEV_NAME, .driver_data = ST_LSM6DSTX_ID },
+	{ .name = ST_LSM6DSV_DEV_NAME, .driver_data = ST_LSM6DSV_ID },
+	{ .name = ST_LSM6DSV16X_DEV_NAME, .driver_data = ST_LSM6DSV16X_ID },
+	{ .name = ST_LSM6DSO16IS_DEV_NAME, .driver_data = ST_LSM6DSO16IS_ID },
+	{ .name = ST_ISM330IS_DEV_NAME, .driver_data = ST_ISM330IS_ID },
+	{ .name = ST_ASM330LHB_DEV_NAME, .driver_data = ST_ASM330LHB_ID },
+	{ .name = ST_ASM330LHHXG1_DEV_NAME, .driver_data = ST_ASM330LHHXG1_ID },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, st_lsm6dsx_spi_id_table);
diff --git a/drivers/iio/imu/st_lsm9ds0/st_lsm9ds0_spi.c b/drivers/iio/imu/st_lsm9ds0/st_lsm9ds0_spi.c
index 8c5d8535e54cb..e67f09e1f093f 100644
--- a/drivers/iio/imu/st_lsm9ds0/st_lsm9ds0_spi.c
+++ b/drivers/iio/imu/st_lsm9ds0/st_lsm9ds0_spi.c
@@ -32,8 +32,8 @@ static const struct of_device_id st_lsm9ds0_of_match[] = {
 MODULE_DEVICE_TABLE(of, st_lsm9ds0_of_match);
 
 static const struct spi_device_id st_lsm9ds0_id_table[] = {
-	{ LSM303D_IMU_DEV_NAME },
-	{ LSM9DS0_IMU_DEV_NAME },
+	{ .name = LSM303D_IMU_DEV_NAME },
+	{ .name = LSM9DS0_IMU_DEV_NAME },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, st_lsm9ds0_id_table);
diff --git a/drivers/iio/industrialio-backend.c b/drivers/iio/industrialio-backend.c
index 4763e224ebc68..f7a4be8ec3207 100644
--- a/drivers/iio/industrialio-backend.c
+++ b/drivers/iio/industrialio-backend.c
@@ -649,7 +649,7 @@ EXPORT_SYMBOL_NS_GPL(iio_backend_ext_info_get, "IIO_BACKEND");
  * @len: Buffer length
  *
  * This helper is intended to be used by backends that extend an IIO channel
- * (trough iio_backend_extend_chan_spec()) with extended info. In that case,
+ * (through iio_backend_extend_chan_spec()) with extended info. In that case,
  * backends are not supposed to give their own callbacks (as they would not have
  * a way to get the backend from indio_dev). This is the setter.
  *
@@ -851,7 +851,7 @@ EXPORT_SYMBOL_NS_GPL(iio_backend_filter_type_set, "IIO_BACKEND");
  * @back: Backend device
  * @timeout_us: Timeout value in us.
  *
- * When activated, it initates a proccess that aligns the sample's most
+ * When activated, it initiates a process that aligns the sample's most
  * significant bit (MSB) based solely on the captured data, without
  * considering any other external signals.
  *
@@ -1017,7 +1017,7 @@ static struct iio_backend *__devm_iio_backend_fwnode_get(struct device *dev, con
  * @dev: Consumer device for the backend
  * @name: Backend name
  *
- * Get's the backend associated with @dev.
+ * Gets the backend associated with @dev.
  *
  * RETURNS:
  * A backend pointer, negative error pointer otherwise.
@@ -1050,7 +1050,7 @@ EXPORT_SYMBOL_NS_GPL(devm_iio_backend_get_by_index, "IIO_BACKEND");
  * @name: Backend name
  * @fwnode: Firmware node of the backend consumer
  *
- * Get's the backend associated with a firmware node.
+ * Gets the backend associated with a firmware node.
  *
  * RETURNS:
  * A backend pointer, negative error pointer otherwise.
diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c
index 7e3d3872e2e65..bdf3d4c063317 100644
--- a/drivers/iio/industrialio-core.c
+++ b/drivers/iio/industrialio-core.c
@@ -19,6 +19,7 @@
 #include <linux/idr.h>
 #include <linux/kdev_t.h>
 #include <linux/kernel.h>
+#include <linux/math64.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/poll.h>
@@ -26,7 +27,6 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/wait.h>
-#include <linux/wordpart.h>
 
 #include <linux/iio/buffer.h>
 #include <linux/iio/buffer_impl.h>
@@ -656,6 +656,7 @@ static ssize_t __iio_format_value(char *buf, size_t offset, unsigned int type,
 				  int size, const int *vals)
 {
 	int tmp0, tmp1;
+	int l = 0;
 	s64 tmp2;
 	bool scale_db = false;
 
@@ -699,7 +700,6 @@ static ssize_t __iio_format_value(char *buf, size_t offset, unsigned int type,
 	case IIO_VAL_INT_MULTIPLE:
 	{
 		int i;
-		int l = 0;
 
 		for (i = 0; i < size; ++i)
 			l += sysfs_emit_at(buf, offset + l, "%d ", vals[i]);
@@ -708,8 +708,25 @@ static ssize_t __iio_format_value(char *buf, size_t offset, unsigned int type,
 	case IIO_VAL_CHAR:
 		return sysfs_emit_at(buf, offset, "%c", (char)vals[0]);
 	case IIO_VAL_INT_64:
-		tmp2 = (s64)((((u64)vals[1]) << 32) | (u32)vals[0]);
-		return sysfs_emit_at(buf, offset, "%lld", tmp2);
+		return sysfs_emit_at(buf, offset, "%lld",
+				     iio_val_s64_compose(vals[0], vals[1]));
+	case IIO_VAL_DECIMAL64_MILLI:
+	case IIO_VAL_DECIMAL64_MICRO:
+	case IIO_VAL_DECIMAL64_NANO:
+	case IIO_VAL_DECIMAL64_PICO:
+	{
+		int scale = type - IIO_VAL_DECIMAL64_BASE;
+		s64 frac;
+
+		tmp2 = div64_s64_rem(iio_val_s64_compose(vals[0], vals[1]),
+				     int_pow(10, scale), &frac);
+		if (tmp2 == 0 && frac < 0)
+			l += sysfs_emit_at(buf, offset, "-");
+
+		l += sysfs_emit_at(buf, offset + l, "%lld.%0*lld", tmp2, scale,
+				   abs(frac));
+		return l;
+	}
 	default:
 		return 0;
 	}
@@ -979,6 +996,7 @@ static ssize_t iio_write_channel_info(struct device *dev,
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
 	int ret, fract_mult = 100000;
+	int type, dec_scale = 0;
 	int integer, fract = 0;
 	long long integer64;
 	bool is_char = false;
@@ -989,9 +1007,11 @@ static ssize_t iio_write_channel_info(struct device *dev,
 	if (!indio_dev->info->write_raw)
 		return -EINVAL;
 
-	if (indio_dev->info->write_raw_get_fmt)
-		switch (indio_dev->info->write_raw_get_fmt(indio_dev,
-			this_attr->c, this_attr->address)) {
+	if (indio_dev->info->write_raw_get_fmt) {
+		type = indio_dev->info->write_raw_get_fmt(indio_dev,
+							  this_attr->c,
+							  this_attr->address);
+		switch (type) {
 		case IIO_VAL_INT:
 			fract_mult = 0;
 			break;
@@ -1007,12 +1027,19 @@ static ssize_t iio_write_channel_info(struct device *dev,
 		case IIO_VAL_CHAR:
 			is_char = true;
 			break;
+		case IIO_VAL_DECIMAL64_MILLI:
+		case IIO_VAL_DECIMAL64_MICRO:
+		case IIO_VAL_DECIMAL64_NANO:
+		case IIO_VAL_DECIMAL64_PICO:
+			dec_scale = type - IIO_VAL_DECIMAL64_BASE;
+			fallthrough;
 		case IIO_VAL_INT_64:
 			is_64bit = true;
 			break;
 		default:
 			return -EINVAL;
 		}
+	}
 
 	if (is_char) {
 		char ch;
@@ -1021,12 +1048,14 @@ static ssize_t iio_write_channel_info(struct device *dev,
 			return -EINVAL;
 		integer = ch;
 	} else if (is_64bit) {
-		ret = kstrtoll(buf, 0, &integer64);
+		if (dec_scale)
+			ret = kstrtodec64(buf, dec_scale, &integer64);
+		else
+			ret = kstrtoll(buf, 0, &integer64);
 		if (ret)
 			return ret;
 
-		fract = upper_32_bits(integer64);
-		integer = lower_32_bits(integer64);
+		iio_val_s64_decompose(integer64, &integer, &fract);
 	} else {
 		ret = __iio_str_to_fixpoint(buf, fract_mult, &integer, &fract,
 					    scale_db);
diff --git a/drivers/iio/inkern.c b/drivers/iio/inkern.c
index 9ce20cb05a9b6..fb08e4f02520d 100644
--- a/drivers/iio/inkern.c
+++ b/drivers/iio/inkern.c
@@ -281,7 +281,7 @@ struct iio_channel *fwnode_iio_channel_get_by_name(struct fwnode_handle *fwnode,
 
 	return ERR_PTR(-ENODEV);
 }
-EXPORT_SYMBOL_GPL(fwnode_iio_channel_get_by_name);
+EXPORT_SYMBOL_NS_GPL(fwnode_iio_channel_get_by_name, "IIO_CONSUMER");
 
 static struct iio_channel *fwnode_iio_channel_get_all(struct device *dev)
 {
@@ -386,7 +386,7 @@ struct iio_channel *iio_channel_get(struct device *dev,
 
 	return iio_channel_get_sys(name, channel_name);
 }
-EXPORT_SYMBOL_GPL(iio_channel_get);
+EXPORT_SYMBOL_NS_GPL(iio_channel_get, "IIO_CONSUMER");
 
 void iio_channel_release(struct iio_channel *channel)
 {
@@ -395,7 +395,7 @@ void iio_channel_release(struct iio_channel *channel)
 	iio_device_put(channel->indio_dev);
 	kfree(channel);
 }
-EXPORT_SYMBOL_GPL(iio_channel_release);
+EXPORT_SYMBOL_NS_GPL(iio_channel_release, "IIO_CONSUMER");
 
 static void devm_iio_channel_free(void *iio_channel)
 {
@@ -418,7 +418,7 @@ struct iio_channel *devm_iio_channel_get(struct device *dev,
 
 	return channel;
 }
-EXPORT_SYMBOL_GPL(devm_iio_channel_get);
+EXPORT_SYMBOL_NS_GPL(devm_iio_channel_get, "IIO_CONSUMER");
 
 struct iio_channel *devm_fwnode_iio_channel_get_by_name(struct device *dev,
 							struct fwnode_handle *fwnode,
@@ -437,7 +437,7 @@ struct iio_channel *devm_fwnode_iio_channel_get_by_name(struct device *dev,
 
 	return channel;
 }
-EXPORT_SYMBOL_GPL(devm_fwnode_iio_channel_get_by_name);
+EXPORT_SYMBOL_NS_GPL(devm_fwnode_iio_channel_get_by_name, "IIO_CONSUMER");
 
 struct iio_channel *iio_channel_get_all(struct device *dev)
 {
@@ -506,7 +506,7 @@ struct iio_channel *iio_channel_get_all(struct device *dev)
 		iio_device_put(chans[i].indio_dev);
 	return ERR_PTR(ret);
 }
-EXPORT_SYMBOL_GPL(iio_channel_get_all);
+EXPORT_SYMBOL_NS_GPL(iio_channel_get_all, "IIO_CONSUMER");
 
 void iio_channel_release_all(struct iio_channel *channels)
 {
@@ -518,7 +518,7 @@ void iio_channel_release_all(struct iio_channel *channels)
 	}
 	kfree(channels);
 }
-EXPORT_SYMBOL_GPL(iio_channel_release_all);
+EXPORT_SYMBOL_NS_GPL(iio_channel_release_all, "IIO_CONSUMER");
 
 static void devm_iio_channel_free_all(void *iio_channels)
 {
@@ -541,7 +541,7 @@ struct iio_channel *devm_iio_channel_get_all(struct device *dev)
 
 	return channels;
 }
-EXPORT_SYMBOL_GPL(devm_iio_channel_get_all);
+EXPORT_SYMBOL_NS_GPL(devm_iio_channel_get_all, "IIO_CONSUMER");
 
 static int iio_channel_read(struct iio_channel *chan, int *val, int *val2,
 			    enum iio_chan_info_enum info)
@@ -585,7 +585,7 @@ int iio_read_channel_raw(struct iio_channel *chan, int *val)
 
 	return iio_channel_read(chan, val, NULL, IIO_CHAN_INFO_RAW);
 }
-EXPORT_SYMBOL_GPL(iio_read_channel_raw);
+EXPORT_SYMBOL_NS_GPL(iio_read_channel_raw, "IIO_CONSUMER");
 
 int iio_read_channel_average_raw(struct iio_channel *chan, int *val)
 {
@@ -597,7 +597,7 @@ int iio_read_channel_average_raw(struct iio_channel *chan, int *val)
 
 	return iio_channel_read(chan, val, NULL, IIO_CHAN_INFO_AVERAGE_RAW);
 }
-EXPORT_SYMBOL_GPL(iio_read_channel_average_raw);
+EXPORT_SYMBOL_NS_GPL(iio_read_channel_average_raw, "IIO_CONSUMER");
 
 int iio_multiply_value(int *result, s64 multiplier,
 		       unsigned int type, int val, int val2)
@@ -701,7 +701,7 @@ int iio_convert_raw_to_processed(struct iio_channel *chan, int raw,
 	return iio_convert_raw_to_processed_unlocked(chan, raw, processed,
 						     scale);
 }
-EXPORT_SYMBOL_GPL(iio_convert_raw_to_processed);
+EXPORT_SYMBOL_NS_GPL(iio_convert_raw_to_processed, "IIO_CONSUMER");
 
 int iio_read_channel_attribute(struct iio_channel *chan, int *val, int *val2,
 			       enum iio_chan_info_enum attribute)
@@ -714,13 +714,13 @@ int iio_read_channel_attribute(struct iio_channel *chan, int *val, int *val2,
 
 	return iio_channel_read(chan, val, val2, attribute);
 }
-EXPORT_SYMBOL_GPL(iio_read_channel_attribute);
+EXPORT_SYMBOL_NS_GPL(iio_read_channel_attribute, "IIO_CONSUMER");
 
 int iio_read_channel_offset(struct iio_channel *chan, int *val, int *val2)
 {
 	return iio_read_channel_attribute(chan, val, val2, IIO_CHAN_INFO_OFFSET);
 }
-EXPORT_SYMBOL_GPL(iio_read_channel_offset);
+EXPORT_SYMBOL_NS_GPL(iio_read_channel_offset, "IIO_CONSUMER");
 
 int iio_read_channel_processed_scale(struct iio_channel *chan, int *val,
 				     unsigned int scale)
@@ -752,20 +752,20 @@ int iio_read_channel_processed_scale(struct iio_channel *chan, int *val,
 							     scale);
 	}
 }
-EXPORT_SYMBOL_GPL(iio_read_channel_processed_scale);
+EXPORT_SYMBOL_NS_GPL(iio_read_channel_processed_scale, "IIO_CONSUMER");
 
 int iio_read_channel_processed(struct iio_channel *chan, int *val)
 {
 	/* This is just a special case with scale factor 1 */
 	return iio_read_channel_processed_scale(chan, val, 1);
 }
-EXPORT_SYMBOL_GPL(iio_read_channel_processed);
+EXPORT_SYMBOL_NS_GPL(iio_read_channel_processed, "IIO_CONSUMER");
 
 int iio_read_channel_scale(struct iio_channel *chan, int *val, int *val2)
 {
 	return iio_read_channel_attribute(chan, val, val2, IIO_CHAN_INFO_SCALE);
 }
-EXPORT_SYMBOL_GPL(iio_read_channel_scale);
+EXPORT_SYMBOL_NS_GPL(iio_read_channel_scale, "IIO_CONSUMER");
 
 static int iio_channel_read_avail(struct iio_channel *chan,
 				  const int **vals, int *type, int *length,
@@ -794,7 +794,7 @@ int iio_read_avail_channel_attribute(struct iio_channel *chan,
 
 	return iio_channel_read_avail(chan, vals, type, length, attribute);
 }
-EXPORT_SYMBOL_GPL(iio_read_avail_channel_attribute);
+EXPORT_SYMBOL_NS_GPL(iio_read_avail_channel_attribute, "IIO_CONSUMER");
 
 int iio_read_avail_channel_raw(struct iio_channel *chan,
 			       const int **vals, int *length)
@@ -811,7 +811,7 @@ int iio_read_avail_channel_raw(struct iio_channel *chan,
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(iio_read_avail_channel_raw);
+EXPORT_SYMBOL_NS_GPL(iio_read_avail_channel_raw, "IIO_CONSUMER");
 
 static int iio_channel_read_max(struct iio_channel *chan,
 				int *val, int *val2, int *type,
@@ -867,7 +867,7 @@ int iio_read_max_channel_raw(struct iio_channel *chan, int *val)
 
 	return iio_channel_read_max(chan, val, NULL, &type, IIO_CHAN_INFO_RAW);
 }
-EXPORT_SYMBOL_GPL(iio_read_max_channel_raw);
+EXPORT_SYMBOL_NS_GPL(iio_read_max_channel_raw, "IIO_CONSUMER");
 
 static int iio_channel_read_min(struct iio_channel *chan,
 				int *val, int *val2, int *type,
@@ -923,7 +923,7 @@ int iio_read_min_channel_raw(struct iio_channel *chan, int *val)
 
 	return iio_channel_read_min(chan, val, NULL, &type, IIO_CHAN_INFO_RAW);
 }
-EXPORT_SYMBOL_GPL(iio_read_min_channel_raw);
+EXPORT_SYMBOL_NS_GPL(iio_read_min_channel_raw, "IIO_CONSUMER");
 
 int iio_get_channel_type(struct iio_channel *chan, enum iio_chan_type *type)
 {
@@ -937,7 +937,7 @@ int iio_get_channel_type(struct iio_channel *chan, enum iio_chan_type *type)
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(iio_get_channel_type);
+EXPORT_SYMBOL_NS_GPL(iio_get_channel_type, "IIO_CONSUMER");
 
 static int iio_channel_write(struct iio_channel *chan, int val, int val2,
 			     enum iio_chan_info_enum info)
@@ -961,13 +961,13 @@ int iio_write_channel_attribute(struct iio_channel *chan, int val, int val2,
 
 	return iio_channel_write(chan, val, val2, attribute);
 }
-EXPORT_SYMBOL_GPL(iio_write_channel_attribute);
+EXPORT_SYMBOL_NS_GPL(iio_write_channel_attribute, "IIO_CONSUMER");
 
 int iio_write_channel_raw(struct iio_channel *chan, int val)
 {
 	return iio_write_channel_attribute(chan, val, 0, IIO_CHAN_INFO_RAW);
 }
-EXPORT_SYMBOL_GPL(iio_write_channel_raw);
+EXPORT_SYMBOL_NS_GPL(iio_write_channel_raw, "IIO_CONSUMER");
 
 unsigned int iio_get_channel_ext_info_count(struct iio_channel *chan)
 {
@@ -982,7 +982,7 @@ unsigned int iio_get_channel_ext_info_count(struct iio_channel *chan)
 
 	return i;
 }
-EXPORT_SYMBOL_GPL(iio_get_channel_ext_info_count);
+EXPORT_SYMBOL_NS_GPL(iio_get_channel_ext_info_count, "IIO_CONSUMER");
 
 static const struct iio_chan_spec_ext_info *
 iio_lookup_ext_info(const struct iio_channel *chan, const char *attr)
@@ -1017,7 +1017,7 @@ ssize_t iio_read_channel_ext_info(struct iio_channel *chan,
 	return ext_info->read(chan->indio_dev, ext_info->private,
 			      chan->channel, buf);
 }
-EXPORT_SYMBOL_GPL(iio_read_channel_ext_info);
+EXPORT_SYMBOL_NS_GPL(iio_read_channel_ext_info, "IIO_CONSUMER");
 
 ssize_t iio_write_channel_ext_info(struct iio_channel *chan, const char *attr,
 				   const char *buf, size_t len)
@@ -1031,7 +1031,7 @@ ssize_t iio_write_channel_ext_info(struct iio_channel *chan, const char *attr,
 	return ext_info->write(chan->indio_dev, ext_info->private,
 			       chan->channel, buf, len);
 }
-EXPORT_SYMBOL_GPL(iio_write_channel_ext_info);
+EXPORT_SYMBOL_NS_GPL(iio_write_channel_ext_info, "IIO_CONSUMER");
 
 ssize_t iio_read_channel_label(struct iio_channel *chan, char *buf)
 {
@@ -1042,4 +1042,4 @@ ssize_t iio_read_channel_label(struct iio_channel *chan, char *buf)
 
 	return do_iio_read_channel_label(chan->indio_dev, chan->channel, buf);
 }
-EXPORT_SYMBOL_GPL(iio_read_channel_label);
+EXPORT_SYMBOL_NS_GPL(iio_read_channel_label, "IIO_CONSUMER");
diff --git a/drivers/iio/light/cm3605.c b/drivers/iio/light/cm3605.c
index 98c84f33db60b..9f799041edf92 100644
--- a/drivers/iio/light/cm3605.c
+++ b/drivers/iio/light/cm3605.c
@@ -324,3 +324,4 @@ module_platform_driver(cm3605_driver);
 MODULE_AUTHOR("Linus Walleij <linus.walleij@linaro.org>");
 MODULE_DESCRIPTION("CM3605 ambient light and proximity sensor driver");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/iio/light/gp2ap002.c b/drivers/iio/light/gp2ap002.c
index a8db514cca5ef..02818e2aed680 100644
--- a/drivers/iio/light/gp2ap002.c
+++ b/drivers/iio/light/gp2ap002.c
@@ -669,7 +669,7 @@ static int gp2ap002_runtime_resume(struct device *dev)
 	ret = regulator_enable(gp2ap002->vio);
 	if (ret) {
 		dev_err(dev, "failed to enable VIO regulator in resume path\n");
-		return ret;
+		goto out_disable_vdd;
 	}
 
 	msleep(20);
@@ -677,13 +677,19 @@ static int gp2ap002_runtime_resume(struct device *dev)
 	ret = gp2ap002_init(gp2ap002);
 	if (ret) {
 		dev_err(dev, "re-initialization failed\n");
-		return ret;
+		goto out_disable_vio;
 	}
 
 	/* Re-activate the IRQ */
 	enable_irq(gp2ap002->irq);
 
 	return 0;
+
+out_disable_vio:
+	regulator_disable(gp2ap002->vio);
+out_disable_vdd:
+	regulator_disable(gp2ap002->vdd);
+	return ret;
 }
 
 static DEFINE_RUNTIME_DEV_PM_OPS(gp2ap002_dev_pm_ops, gp2ap002_runtime_suspend,
@@ -717,3 +723,4 @@ module_i2c_driver(gp2ap002_driver);
 MODULE_AUTHOR("Linus Walleij <linus.walleij@linaro.org>");
 MODULE_DESCRIPTION("GP2AP002 ambient light and proximity sensor driver");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/iio/light/hid-sensor-als.c b/drivers/iio/light/hid-sensor-als.c
index 6bf1bc9a38fe6..6fb6ace26834b 100644
--- a/drivers/iio/light/hid-sensor-als.c
+++ b/drivers/iio/light/hid-sensor-als.c
@@ -119,9 +119,8 @@ static const struct iio_chan_spec als_channels[] = {
 
 /* Channel read_raw handler */
 static int als_read_raw(struct iio_dev *indio_dev,
-			      struct iio_chan_spec const *chan,
-			      int *val, int *val2,
-			      long mask)
+			struct iio_chan_spec const *chan,
+			int *val, int *val2, long mask)
 {
 	struct als_state *als_state = iio_priv(indio_dev);
 	struct hid_sensor_hub_device *hsdev = als_state->common_attributes.hsdev;
@@ -162,12 +161,12 @@ static int als_read_raw(struct iio_dev *indio_dev,
 		}
 		if (report_id >= 0) {
 			hid_sensor_power_state(&als_state->common_attributes,
-						true);
+					       true);
 			*val = sensor_hub_input_attr_get_raw_value(
 					hsdev, hsdev->usage, address, report_id,
 					SENSOR_HUB_SYNC, min < 0);
 			hid_sensor_power_state(&als_state->common_attributes,
-						false);
+					       false);
 		} else {
 			*val = 0;
 			return -EINVAL;
@@ -205,10 +204,8 @@ static int als_read_raw(struct iio_dev *indio_dev,
 
 /* Channel write_raw handler */
 static int als_write_raw(struct iio_dev *indio_dev,
-			       struct iio_chan_spec const *chan,
-			       int val,
-			       int val2,
-			       long mask)
+			 struct iio_chan_spec const *chan,
+			 int val, int val2, long mask)
 {
 	struct als_state *als_state = iio_priv(indio_dev);
 	int ret = 0;
@@ -240,8 +237,7 @@ static const struct iio_info als_info = {
 
 /* Callback handler to send event after all samples are received and captured */
 static int als_proc_event(struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
-				void *priv)
+			  u32 usage_id, void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
 	struct als_state *als_state = iio_priv(indio_dev);
@@ -262,9 +258,9 @@ static int als_proc_event(struct hid_sensor_hub_device *hsdev,
 
 /* Capture samples in local storage */
 static int als_capture_sample(struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
-				size_t raw_len, char *raw_data,
-				void *priv)
+			      u32 usage_id,
+			      size_t raw_len, char *raw_data,
+			      void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
 	struct als_state *als_state = iio_priv(indio_dev);
@@ -303,9 +299,9 @@ static int als_capture_sample(struct hid_sensor_hub_device *hsdev,
 
 /* Parse report which is specific to an usage id*/
 static int als_parse_report(struct platform_device *pdev,
-				struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
-				struct als_state *st)
+			    struct hid_sensor_hub_device *hsdev,
+			    u32 usage_id,
+			    struct als_state *st)
 {
 	struct iio_chan_spec *channels;
 	int ret, index = 0;
@@ -399,31 +395,31 @@ static int hid_als_probe(struct platform_device *pdev)
 	atomic_set(&als_state->common_attributes.data_ready, 0);
 
 	ret = hid_sensor_setup_trigger(indio_dev, name,
-				&als_state->common_attributes);
+				       &als_state->common_attributes);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "trigger setup failed\n");
 		return ret;
 	}
 
-	ret = iio_device_register(indio_dev);
-	if (ret) {
-		dev_err(&pdev->dev, "device register failed\n");
-		goto error_remove_trigger;
-	}
-
 	als_state->callbacks.send_event = als_proc_event;
 	als_state->callbacks.capture_sample = als_capture_sample;
 	als_state->callbacks.pdev = pdev;
 	ret = sensor_hub_register_callback(hsdev, hsdev->usage, &als_state->callbacks);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "callback reg failed\n");
-		goto error_iio_unreg;
+		goto error_remove_trigger;
+	}
+
+	ret = iio_device_register(indio_dev);
+	if (ret) {
+		dev_err(&pdev->dev, "device register failed\n");
+		goto error_remove_callback;
 	}
 
 	return ret;
 
-error_iio_unreg:
-	iio_device_unregister(indio_dev);
+error_remove_callback:
+	sensor_hub_remove_callback(hsdev, hsdev->usage);
 error_remove_trigger:
 	hid_sensor_remove_trigger(indio_dev, &als_state->common_attributes);
 	return ret;
@@ -436,8 +432,8 @@ static void hid_als_remove(struct platform_device *pdev)
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
 	struct als_state *als_state = iio_priv(indio_dev);
 
-	sensor_hub_remove_callback(hsdev, hsdev->usage);
 	iio_device_unregister(indio_dev);
+	sensor_hub_remove_callback(hsdev, hsdev->usage);
 	hid_sensor_remove_trigger(indio_dev, &als_state->common_attributes);
 }
 
diff --git a/drivers/iio/light/hid-sensor-prox.c b/drivers/iio/light/hid-sensor-prox.c
index 95bf2bfd86ee8..63fd3eff171c1 100644
--- a/drivers/iio/light/hid-sensor-prox.c
+++ b/drivers/iio/light/hid-sensor-prox.c
@@ -69,9 +69,8 @@ static const struct iio_chan_spec prox_channels[] = {
 
 /* Channel read_raw handler */
 static int prox_read_raw(struct iio_dev *indio_dev,
-			      struct iio_chan_spec const *chan,
-			      int *val, int *val2,
-			      long mask)
+			 struct iio_chan_spec const *chan,
+			 int *val, int *val2, long mask)
 {
 	struct prox_state *prox_state = iio_priv(indio_dev);
 	struct hid_sensor_hub_device *hsdev;
@@ -134,10 +133,8 @@ static int prox_read_raw(struct iio_dev *indio_dev,
 
 /* Channel write_raw handler */
 static int prox_write_raw(struct iio_dev *indio_dev,
-			       struct iio_chan_spec const *chan,
-			       int val,
-			       int val2,
-			       long mask)
+			  struct iio_chan_spec const *chan,
+			  int val, int val2, long mask)
 {
 	struct prox_state *prox_state = iio_priv(indio_dev);
 	int ret = 0;
@@ -165,8 +162,7 @@ static const struct iio_info prox_info = {
 
 /* Callback handler to send event after all samples are received and captured */
 static int prox_proc_event(struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
-				void *priv)
+			   u32 usage_id, void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
 	struct prox_state *prox_state = iio_priv(indio_dev);
@@ -182,9 +178,9 @@ static int prox_proc_event(struct hid_sensor_hub_device *hsdev,
 
 /* Capture samples in local storage */
 static int prox_capture_sample(struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
-				size_t raw_len, char *raw_data,
-				void *priv)
+			       u32 usage_id,
+			       size_t raw_len, char *raw_data,
+			       void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
 	struct prox_state *prox_state = iio_priv(indio_dev);
@@ -217,8 +213,8 @@ static int prox_capture_sample(struct hid_sensor_hub_device *hsdev,
 
 /* Parse report which is specific to an usage id*/
 static int prox_parse_report(struct platform_device *pdev,
-				struct hid_sensor_hub_device *hsdev,
-				struct prox_state *st)
+			     struct hid_sensor_hub_device *hsdev,
+			     struct prox_state *st)
 {
 	struct iio_chan_spec *channels = st->channels;
 	int index = 0;
@@ -270,8 +266,7 @@ static int hid_prox_probe(struct platform_device *pdev)
 	struct iio_dev *indio_dev;
 	struct prox_state *prox_state;
 
-	indio_dev = devm_iio_device_alloc(&pdev->dev,
-				sizeof(struct prox_state));
+	indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(struct prox_state));
 	if (!indio_dev)
 		return -ENOMEM;
 	platform_set_drvdata(pdev, indio_dev);
@@ -305,18 +300,12 @@ static int hid_prox_probe(struct platform_device *pdev)
 	atomic_set(&prox_state->common_attributes.data_ready, 0);
 
 	ret = hid_sensor_setup_trigger(indio_dev, name,
-				&prox_state->common_attributes);
+				       &prox_state->common_attributes);
 	if (ret) {
 		dev_err(&pdev->dev, "trigger setup failed\n");
 		return ret;
 	}
 
-	ret = iio_device_register(indio_dev);
-	if (ret) {
-		dev_err(&pdev->dev, "device register failed\n");
-		goto error_remove_trigger;
-	}
-
 	prox_state->callbacks.send_event = prox_proc_event;
 	prox_state->callbacks.capture_sample = prox_capture_sample;
 	prox_state->callbacks.pdev = pdev;
@@ -324,13 +313,19 @@ static int hid_prox_probe(struct platform_device *pdev)
 					   &prox_state->callbacks);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "callback reg failed\n");
-		goto error_iio_unreg;
+		goto error_remove_trigger;
+	}
+
+	ret = iio_device_register(indio_dev);
+	if (ret) {
+		dev_err(&pdev->dev, "device register failed\n");
+		goto error_remove_callback;
 	}
 
 	return ret;
 
-error_iio_unreg:
-	iio_device_unregister(indio_dev);
+error_remove_callback:
+	sensor_hub_remove_callback(hsdev, hsdev->usage);
 error_remove_trigger:
 	hid_sensor_remove_trigger(indio_dev, &prox_state->common_attributes);
 	return ret;
@@ -343,8 +338,8 @@ static void hid_prox_remove(struct platform_device *pdev)
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
 	struct prox_state *prox_state = iio_priv(indio_dev);
 
-	sensor_hub_remove_callback(hsdev, hsdev->usage);
 	iio_device_unregister(indio_dev);
+	sensor_hub_remove_callback(hsdev, hsdev->usage);
 	hid_sensor_remove_trigger(indio_dev, &prox_state->common_attributes);
 }
 
diff --git a/drivers/iio/light/opt3001.c b/drivers/iio/light/opt3001.c
index c5bd7fa23fb75..434ef7e034fe3 100644
--- a/drivers/iio/light/opt3001.c
+++ b/drivers/iio/light/opt3001.c
@@ -10,6 +10,7 @@
 
 #include <linux/array_size.h>
 #include <linux/bits.h>
+#include <linux/cleanup.h>
 #include <linux/delay.h>
 #include <linux/dev_printk.h>
 #include <linux/errno.h>
@@ -100,9 +101,14 @@ struct opt3001_chip_info {
 
 struct opt3001 {
 	struct i2c_client	*client;
-	struct device		*dev;
 
+	/*
+	 * Ensure data capture and read-modify-write sequences are
+	 * not interrupted.
+	 */
 	struct mutex		lock;
+
+	/* Allows for IRQs to bypass locking mechanism */
 	bool			ok_to_ignore_lock;
 	bool			result_ready;
 	wait_queue_head_t	result_ready_queue;
@@ -223,11 +229,10 @@ static const struct opt3001_scale opt3002_scales[] = {
 	},
 };
 
-static int opt3001_find_scale(const struct opt3001 *opt, int val,
-		int val2, u8 *exponent)
+static int opt3001_find_scale(const struct opt3001 *opt, int val, int val2,
+			      u8 *exponent)
 {
-	int i;
-	for (i = 0; i < ARRAY_SIZE(*opt->chip_info->scales); i++) {
+	for (unsigned int i = 0; i < ARRAY_SIZE(*opt->chip_info->scales); i++) {
 		const struct opt3001_scale *scale = &(*opt->chip_info->scales)[i];
 		/*
 		 * Compare the integer and micro parts to determine value scale.
@@ -242,8 +247,8 @@ static int opt3001_find_scale(const struct opt3001 *opt, int val,
 	return -EINVAL;
 }
 
-static void opt3001_to_iio_ret(struct opt3001 *opt, u8 exponent,
-		u16 mantissa, int *val, int *val2)
+static void opt3001_to_iio_ret(struct opt3001 *opt, u8 exponent, u16 mantissa,
+			       int *val, int *val2)
 {
 	int ret;
 	int whole = opt->chip_info->factor_whole;
@@ -312,6 +317,8 @@ static const struct iio_chan_spec opt3002_channels[] = {
 
 static int opt3001_get_processed(struct opt3001 *opt, int *val, int *val2)
 {
+	struct i2c_client *client = opt->client;
+	struct device *dev = &client->dev;
 	int ret;
 	u16 mantissa;
 	u16 reg;
@@ -325,12 +332,12 @@ static int opt3001_get_processed(struct opt3001 *opt, int *val, int *val2)
 		 * doing so will overwrite the low-level limit value however we
 		 * will restore this value later on.
 		 */
-		ret = i2c_smbus_write_word_swapped(opt->client,
-					OPT3001_LOW_LIMIT,
-					OPT3001_LOW_LIMIT_EOC_ENABLE);
+		ret = i2c_smbus_write_word_swapped(client,
+						   OPT3001_LOW_LIMIT,
+						   OPT3001_LOW_LIMIT_EOC_ENABLE);
 		if (ret < 0) {
-			dev_err(opt->dev, "failed to write register %02x\n",
-					OPT3001_LOW_LIMIT);
+			dev_err(dev, "failed to write register %02x\n",
+				OPT3001_LOW_LIMIT);
 			return ret;
 		}
 
@@ -342,29 +349,28 @@ static int opt3001_get_processed(struct opt3001 *opt, int *val, int *val2)
 	opt->result_ready = false;
 
 	/* Configure for single-conversion mode and start a new conversion */
-	ret = i2c_smbus_read_word_swapped(opt->client, OPT3001_CONFIGURATION);
+	ret = i2c_smbus_read_word_swapped(client, OPT3001_CONFIGURATION);
 	if (ret < 0) {
-		dev_err(opt->dev, "failed to read register %02x\n",
-				OPT3001_CONFIGURATION);
+		dev_err(dev, "failed to read register %02x\n",
+			OPT3001_CONFIGURATION);
 		goto err;
 	}
 
 	reg = ret;
 	opt3001_set_mode(opt, &reg, OPT3001_CONFIGURATION_M_SINGLE);
 
-	ret = i2c_smbus_write_word_swapped(opt->client, OPT3001_CONFIGURATION,
-			reg);
+	ret = i2c_smbus_write_word_swapped(client, OPT3001_CONFIGURATION, reg);
 	if (ret < 0) {
-		dev_err(opt->dev, "failed to write register %02x\n",
-				OPT3001_CONFIGURATION);
+		dev_err(dev, "failed to write register %02x\n",
+			OPT3001_CONFIGURATION);
 		goto err;
 	}
 
 	if (opt->use_irq) {
 		/* Wait for the IRQ to indicate the conversion is complete */
 		ret = wait_event_timeout(opt->result_ready_queue,
-				opt->result_ready,
-				msecs_to_jiffies(OPT3001_RESULT_READY_LONG));
+					 opt->result_ready,
+					 msecs_to_jiffies(OPT3001_RESULT_READY_LONG));
 		if (ret == 0) {
 			ret = -ETIMEDOUT;
 			goto err;
@@ -376,10 +382,9 @@ static int opt3001_get_processed(struct opt3001 *opt, int *val, int *val2)
 		msleep(timeout);
 
 		/* Check result ready flag */
-		ret = i2c_smbus_read_word_swapped(opt->client,
-						  OPT3001_CONFIGURATION);
+		ret = i2c_smbus_read_word_swapped(client, OPT3001_CONFIGURATION);
 		if (ret < 0) {
-			dev_err(opt->dev, "failed to read register %02x\n",
+			dev_err(dev, "failed to read register %02x\n",
 				OPT3001_CONFIGURATION);
 			goto err;
 		}
@@ -390,9 +395,9 @@ static int opt3001_get_processed(struct opt3001 *opt, int *val, int *val2)
 		}
 
 		/* Obtain value */
-		ret = i2c_smbus_read_word_swapped(opt->client, OPT3001_RESULT);
+		ret = i2c_smbus_read_word_swapped(client, OPT3001_RESULT);
 		if (ret < 0) {
-			dev_err(opt->dev, "failed to read register %02x\n",
+			dev_err(dev, "failed to read register %02x\n",
 				OPT3001_RESULT);
 			goto err;
 		}
@@ -417,12 +422,12 @@ static int opt3001_get_processed(struct opt3001 *opt, int *val, int *val2)
 		 * bit-overlap and therefore can't be done.
 		 */
 		value = (opt->low_thresh_exp << 12) | opt->low_thresh_mantissa;
-		ret = i2c_smbus_write_word_swapped(opt->client,
+		ret = i2c_smbus_write_word_swapped(client,
 						   OPT3001_LOW_LIMIT,
 						   value);
 		if (ret < 0) {
-			dev_err(opt->dev, "failed to write register %02x\n",
-					OPT3001_LOW_LIMIT);
+			dev_err(dev, "failed to write register %02x\n",
+				OPT3001_LOW_LIMIT);
 			return ret;
 		}
 	}
@@ -445,13 +450,15 @@ static int opt3001_get_int_time(struct opt3001 *opt, int *val, int *val2)
 
 static int opt3001_set_int_time(struct opt3001 *opt, int time)
 {
+	struct i2c_client *client = opt->client;
+	struct device *dev = &client->dev;
 	int ret;
 	u16 reg;
 
-	ret = i2c_smbus_read_word_swapped(opt->client, OPT3001_CONFIGURATION);
+	ret = i2c_smbus_read_word_swapped(client, OPT3001_CONFIGURATION);
 	if (ret < 0) {
-		dev_err(opt->dev, "failed to read register %02x\n",
-				OPT3001_CONFIGURATION);
+		dev_err(dev, "failed to read register %02x\n",
+			OPT3001_CONFIGURATION);
 		return ret;
 	}
 
@@ -470,16 +477,14 @@ static int opt3001_set_int_time(struct opt3001 *opt, int time)
 		return -EINVAL;
 	}
 
-	return i2c_smbus_write_word_swapped(opt->client, OPT3001_CONFIGURATION,
-			reg);
+	return i2c_smbus_write_word_swapped(client, OPT3001_CONFIGURATION, reg);
 }
 
 static int opt3001_read_raw(struct iio_dev *iio,
-		struct iio_chan_spec const *chan, int *val, int *val2,
-		long mask)
+			    struct iio_chan_spec const *chan,
+			    int *val, int *val2, long mask)
 {
 	struct opt3001 *opt = iio_priv(iio);
-	int ret;
 
 	if (opt->mode == OPT3001_CONFIGURATION_M_CONTINUOUS)
 		return -EBUSY;
@@ -487,31 +492,24 @@ static int opt3001_read_raw(struct iio_dev *iio,
 	if (chan->type != opt->chip_info->chan_type)
 		return -EINVAL;
 
-	mutex_lock(&opt->lock);
+	guard(mutex)(&opt->lock);
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
 	case IIO_CHAN_INFO_PROCESSED:
-		ret = opt3001_get_processed(opt, val, val2);
-		break;
+		return opt3001_get_processed(opt, val, val2);
 	case IIO_CHAN_INFO_INT_TIME:
-		ret = opt3001_get_int_time(opt, val, val2);
-		break;
+		return opt3001_get_int_time(opt, val, val2);
 	default:
-		ret = -EINVAL;
+		return -EINVAL;
 	}
-
-	mutex_unlock(&opt->lock);
-
-	return ret;
 }
 
 static int opt3001_write_raw(struct iio_dev *iio,
-		struct iio_chan_spec const *chan, int val, int val2,
-		long mask)
+			     struct iio_chan_spec const *chan,
+			     int val, int val2, long mask)
 {
 	struct opt3001 *opt = iio_priv(iio);
-	int ret;
 
 	if (opt->mode == OPT3001_CONFIGURATION_M_CONTINUOUS)
 		return -EBUSY;
@@ -525,47 +523,46 @@ static int opt3001_write_raw(struct iio_dev *iio,
 	if (val != 0)
 		return -EINVAL;
 
-	mutex_lock(&opt->lock);
-	ret = opt3001_set_int_time(opt, val2);
-	mutex_unlock(&opt->lock);
+	guard(mutex)(&opt->lock);
 
-	return ret;
+	return opt3001_set_int_time(opt, val2);
 }
 
 static int opt3001_read_event_value(struct iio_dev *iio,
-		const struct iio_chan_spec *chan, enum iio_event_type type,
-		enum iio_event_direction dir, enum iio_event_info info,
-		int *val, int *val2)
+				    const struct iio_chan_spec *chan,
+				    enum iio_event_type type,
+				    enum iio_event_direction dir,
+				    enum iio_event_info info,
+				    int *val, int *val2)
 {
 	struct opt3001 *opt = iio_priv(iio);
-	int ret = IIO_VAL_INT_PLUS_MICRO;
 
-	mutex_lock(&opt->lock);
+	guard(mutex)(&opt->lock);
 
 	switch (dir) {
 	case IIO_EV_DIR_RISING:
 		opt3001_to_iio_ret(opt, opt->high_thresh_exp,
-				opt->high_thresh_mantissa, val, val2);
-		break;
+				   opt->high_thresh_mantissa, val, val2);
+		return IIO_VAL_INT_PLUS_MICRO;
 	case IIO_EV_DIR_FALLING:
 		opt3001_to_iio_ret(opt, opt->low_thresh_exp,
-				opt->low_thresh_mantissa, val, val2);
-		break;
+				   opt->low_thresh_mantissa, val, val2);
+		return IIO_VAL_INT_PLUS_MICRO;
 	default:
-		ret = -EINVAL;
+		return -EINVAL;
 	}
-
-	mutex_unlock(&opt->lock);
-
-	return ret;
 }
 
 static int opt3001_write_event_value(struct iio_dev *iio,
-		const struct iio_chan_spec *chan, enum iio_event_type type,
-		enum iio_event_direction dir, enum iio_event_info info,
-		int val, int val2)
+				     const struct iio_chan_spec *chan,
+				     enum iio_event_type type,
+				     enum iio_event_direction dir,
+				     enum iio_event_info info,
+				     int val, int val2)
 {
 	struct opt3001 *opt = iio_priv(iio);
+	struct i2c_client *client = opt->client;
+	struct device *dev = &client->dev;
 	int ret;
 	int whole;
 	int integer;
@@ -580,12 +577,12 @@ static int opt3001_write_event_value(struct iio_dev *iio,
 	if (val < 0)
 		return -EINVAL;
 
-	mutex_lock(&opt->lock);
+	guard(mutex)(&opt->lock);
 
 	ret = opt3001_find_scale(opt, val, val2, &exponent);
 	if (ret < 0) {
-		dev_err(opt->dev, "can't find scale for %d.%06u\n", val, val2);
-		goto err;
+		dev_err(dev, "can't find scale for %d.%06u\n", val, val2);
+		return ret;
 	}
 
 	whole = opt->chip_info->factor_whole;
@@ -608,25 +605,22 @@ static int opt3001_write_event_value(struct iio_dev *iio,
 		opt->low_thresh_exp = exponent;
 		break;
 	default:
-		ret = -EINVAL;
-		goto err;
+		return -EINVAL;
 	}
 
-	ret = i2c_smbus_write_word_swapped(opt->client, reg, value);
+	ret = i2c_smbus_write_word_swapped(client, reg, value);
 	if (ret < 0) {
-		dev_err(opt->dev, "failed to write register %02x\n", reg);
-		goto err;
+		dev_err(dev, "failed to write register %02x\n", reg);
+		return ret;
 	}
 
-err:
-	mutex_unlock(&opt->lock);
-
-	return ret;
+	return 0;
 }
 
 static int opt3001_read_event_config(struct iio_dev *iio,
-		const struct iio_chan_spec *chan, enum iio_event_type type,
-		enum iio_event_direction dir)
+				     const struct iio_chan_spec *chan,
+				     enum iio_event_type type,
+				     enum iio_event_direction dir)
 {
 	struct opt3001 *opt = iio_priv(iio);
 
@@ -634,10 +628,14 @@ static int opt3001_read_event_config(struct iio_dev *iio,
 }
 
 static int opt3001_write_event_config(struct iio_dev *iio,
-		const struct iio_chan_spec *chan, enum iio_event_type type,
-		enum iio_event_direction dir, bool state)
+				      const struct iio_chan_spec *chan,
+				      enum iio_event_type type,
+				      enum iio_event_direction dir,
+				      bool state)
 {
 	struct opt3001 *opt = iio_priv(iio);
+	struct i2c_client *client = opt->client;
+	struct device *dev = &client->dev;
 	int ret;
 	u16 mode;
 	u16 reg;
@@ -648,33 +646,29 @@ static int opt3001_write_event_config(struct iio_dev *iio,
 	if (!state && opt->mode == OPT3001_CONFIGURATION_M_SHUTDOWN)
 		return 0;
 
-	mutex_lock(&opt->lock);
+	guard(mutex)(&opt->lock);
 
 	mode = state ? OPT3001_CONFIGURATION_M_CONTINUOUS
 		: OPT3001_CONFIGURATION_M_SHUTDOWN;
 
-	ret = i2c_smbus_read_word_swapped(opt->client, OPT3001_CONFIGURATION);
+	ret = i2c_smbus_read_word_swapped(client, OPT3001_CONFIGURATION);
 	if (ret < 0) {
-		dev_err(opt->dev, "failed to read register %02x\n",
-				OPT3001_CONFIGURATION);
-		goto err;
+		dev_err(dev, "failed to read register %02x\n",
+			OPT3001_CONFIGURATION);
+		return ret;
 	}
 
 	reg = ret;
 	opt3001_set_mode(opt, &reg, mode);
 
-	ret = i2c_smbus_write_word_swapped(opt->client, OPT3001_CONFIGURATION,
-			reg);
+	ret = i2c_smbus_write_word_swapped(client, OPT3001_CONFIGURATION, reg);
 	if (ret < 0) {
-		dev_err(opt->dev, "failed to write register %02x\n",
-				OPT3001_CONFIGURATION);
-		goto err;
+		dev_err(dev, "failed to write register %02x\n",
+			OPT3001_CONFIGURATION);
+		return ret;
 	}
 
-err:
-	mutex_unlock(&opt->lock);
-
-	return ret;
+	return 0;
 }
 
 static const struct iio_info opt3001_info = {
@@ -689,47 +683,69 @@ static const struct iio_info opt3001_info = {
 
 static int opt3001_read_id(struct opt3001 *opt)
 {
+	struct i2c_client *client = opt->client;
+	struct device *dev = &client->dev;
 	char manufacturer[2];
 	u16 device_id;
 	int ret;
 
-	ret = i2c_smbus_read_word_swapped(opt->client, OPT3001_MANUFACTURER_ID);
-	if (ret < 0) {
-		dev_err(opt->dev, "failed to read register %02x\n",
-				OPT3001_MANUFACTURER_ID);
-		return ret;
-	}
+	ret = i2c_smbus_read_word_swapped(client, OPT3001_MANUFACTURER_ID);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to read register %02x\n",
+				     OPT3001_MANUFACTURER_ID);
 
 	manufacturer[0] = ret >> 8;
 	manufacturer[1] = ret & 0xff;
 
-	ret = i2c_smbus_read_word_swapped(opt->client, OPT3001_DEVICE_ID);
-	if (ret < 0) {
-		dev_err(opt->dev, "failed to read register %02x\n",
-			OPT3001_DEVICE_ID);
-		return ret;
-	}
+	ret = i2c_smbus_read_word_swapped(client, OPT3001_DEVICE_ID);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to read register %02x\n",
+				     OPT3001_DEVICE_ID);
 
 	device_id = ret;
 
-	dev_info(opt->dev, "Found %c%c OPT%04x\n", manufacturer[0],
-			manufacturer[1], device_id);
+	dev_info(dev, "Found %c%c OPT%04x\n", manufacturer[0], manufacturer[1],
+		 device_id);
 
 	return 0;
 }
 
-static int opt3001_configure(struct opt3001 *opt)
+static void opt3001_power_off(void *data)
 {
+	struct opt3001 *opt = data;
+	struct i2c_client *client = opt->client;
+	struct device *dev = &client->dev;
+	u16 reg_val;
 	int ret;
-	u16 reg;
 
-	ret = i2c_smbus_read_word_swapped(opt->client, OPT3001_CONFIGURATION);
+	ret = i2c_smbus_read_word_swapped(client, OPT3001_CONFIGURATION);
 	if (ret < 0) {
-		dev_err(opt->dev, "failed to read register %02x\n",
-				OPT3001_CONFIGURATION);
-		return ret;
+		dev_err(dev, "failed to read register %02x\n",
+			OPT3001_CONFIGURATION);
+		return;
 	}
 
+	reg_val = ret;
+	opt3001_set_mode(opt, &reg_val, OPT3001_CONFIGURATION_M_SHUTDOWN);
+
+	ret = i2c_smbus_write_word_swapped(client, OPT3001_CONFIGURATION, reg_val);
+	if (ret < 0)
+		dev_err(dev, "failed to write to register %02x\n",
+			OPT3001_CONFIGURATION);
+}
+
+static int opt3001_configure(struct opt3001 *opt)
+{
+	struct i2c_client *client = opt->client;
+	struct device *dev = &client->dev;
+	int ret;
+	u16 reg;
+
+	ret = i2c_smbus_read_word_swapped(client, OPT3001_CONFIGURATION);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to read register %02x\n",
+				     OPT3001_CONFIGURATION);
+
 	reg = ret;
 
 	/* Enable automatic full-scale setting mode */
@@ -751,30 +767,28 @@ static int opt3001_configure(struct opt3001 *opt)
 	reg &= ~OPT3001_CONFIGURATION_ME;
 	reg &= ~OPT3001_CONFIGURATION_FC_MASK;
 
-	ret = i2c_smbus_write_word_swapped(opt->client, OPT3001_CONFIGURATION,
-			reg);
-	if (ret < 0) {
-		dev_err(opt->dev, "failed to write register %02x\n",
-				OPT3001_CONFIGURATION);
-		return ret;
-	}
+	ret = i2c_smbus_write_word_swapped(client, OPT3001_CONFIGURATION, reg);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to write register %02x\n",
+				     OPT3001_CONFIGURATION);
 
-	ret = i2c_smbus_read_word_swapped(opt->client, OPT3001_LOW_LIMIT);
-	if (ret < 0) {
-		dev_err(opt->dev, "failed to read register %02x\n",
-				OPT3001_LOW_LIMIT);
-		return ret;
-	}
+	ret = devm_add_action_or_reset(dev, opt3001_power_off, opt);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to register power off function\n");
+
+	ret = i2c_smbus_read_word_swapped(client, OPT3001_LOW_LIMIT);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to read register %02x\n",
+				     OPT3001_LOW_LIMIT);
 
 	opt->low_thresh_mantissa = OPT3001_REG_MANTISSA(ret);
 	opt->low_thresh_exp = OPT3001_REG_EXPONENT(ret);
 
-	ret = i2c_smbus_read_word_swapped(opt->client, OPT3001_HIGH_LIMIT);
-	if (ret < 0) {
-		dev_err(opt->dev, "failed to read register %02x\n",
-				OPT3001_HIGH_LIMIT);
-		return ret;
-	}
+	ret = i2c_smbus_read_word_swapped(client, OPT3001_HIGH_LIMIT);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to read register %02x\n",
+				     OPT3001_HIGH_LIMIT);
 
 	opt->high_thresh_mantissa = OPT3001_REG_MANTISSA(ret);
 	opt->high_thresh_exp = OPT3001_REG_EXPONENT(ret);
@@ -786,6 +800,8 @@ static irqreturn_t opt3001_irq(int irq, void *_iio)
 {
 	struct iio_dev *iio = _iio;
 	struct opt3001 *opt = iio_priv(iio);
+	struct i2c_client *client = opt->client;
+	struct device *dev = &client->dev;
 	int ret;
 	bool wake_result_ready_queue = false;
 	enum iio_chan_type chan_type = opt->chip_info->chan_type;
@@ -794,10 +810,10 @@ static irqreturn_t opt3001_irq(int irq, void *_iio)
 	if (!ok_to_ignore_lock)
 		mutex_lock(&opt->lock);
 
-	ret = i2c_smbus_read_word_swapped(opt->client, OPT3001_CONFIGURATION);
+	ret = i2c_smbus_read_word_swapped(client, OPT3001_CONFIGURATION);
 	if (ret < 0) {
-		dev_err(opt->dev, "failed to read register %02x\n",
-				OPT3001_CONFIGURATION);
+		dev_err(dev, "failed to read register %02x\n",
+			OPT3001_CONFIGURATION);
 		goto out;
 	}
 
@@ -805,21 +821,21 @@ static irqreturn_t opt3001_irq(int irq, void *_iio)
 			OPT3001_CONFIGURATION_M_CONTINUOUS) {
 		if (ret & OPT3001_CONFIGURATION_FH)
 			iio_push_event(iio,
-					IIO_UNMOD_EVENT_CODE(chan_type, 0,
-							IIO_EV_TYPE_THRESH,
-							IIO_EV_DIR_RISING),
-					iio_get_time_ns(iio));
+				       IIO_UNMOD_EVENT_CODE(chan_type, 0,
+							    IIO_EV_TYPE_THRESH,
+							    IIO_EV_DIR_RISING),
+				       iio_get_time_ns(iio));
 		if (ret & OPT3001_CONFIGURATION_FL)
 			iio_push_event(iio,
-					IIO_UNMOD_EVENT_CODE(chan_type, 0,
-							IIO_EV_TYPE_THRESH,
-							IIO_EV_DIR_FALLING),
-					iio_get_time_ns(iio));
+				       IIO_UNMOD_EVENT_CODE(chan_type, 0,
+							    IIO_EV_TYPE_THRESH,
+							    IIO_EV_DIR_FALLING),
+				       iio_get_time_ns(iio));
 	} else if (ret & OPT3001_CONFIGURATION_CRF) {
-		ret = i2c_smbus_read_word_swapped(opt->client, OPT3001_RESULT);
+		ret = i2c_smbus_read_word_swapped(client, OPT3001_RESULT);
 		if (ret < 0) {
-			dev_err(opt->dev, "failed to read register %02x\n",
-					OPT3001_RESULT);
+			dev_err(dev, "failed to read register %02x\n",
+				OPT3001_RESULT);
 			goto out;
 		}
 		opt->result = ret;
@@ -852,12 +868,13 @@ static int opt3001_probe(struct i2c_client *client)
 
 	opt = iio_priv(iio);
 	opt->client = client;
-	opt->dev = dev;
 	opt->chip_info = i2c_get_match_data(client);
 
-	mutex_init(&opt->lock);
+	ret = devm_mutex_init(dev, &opt->lock);
+	if (ret)
+		return ret;
+
 	init_waitqueue_head(&opt->result_ready_queue);
-	i2c_set_clientdata(client, iio);
 
 	if (opt->chip_info->has_id) {
 		ret = opt3001_read_id(opt);
@@ -875,55 +892,20 @@ static int opt3001_probe(struct i2c_client *client)
 	iio->modes = INDIO_DIRECT_MODE;
 	iio->info = &opt3001_info;
 
-	ret = devm_iio_device_register(dev, iio);
-	if (ret) {
-		dev_err(dev, "failed to register IIO device\n");
-		return ret;
-	}
-
 	/* Make use of INT pin only if valid IRQ no. is given */
 	if (irq > 0) {
-		ret = request_threaded_irq(irq, NULL, opt3001_irq,
-				IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
-				"opt3001", iio);
-		if (ret) {
-			dev_err(dev, "failed to request IRQ #%d\n", irq);
+		ret = devm_request_threaded_irq(dev, irq, NULL, opt3001_irq,
+						IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
+						"opt3001", iio);
+		if (ret)
 			return ret;
-		}
+
 		opt->use_irq = true;
 	} else {
-		dev_dbg(opt->dev, "enabling interrupt-less operation\n");
+		dev_dbg(dev, "enabling interrupt-less operation\n");
 	}
 
-	return 0;
-}
-
-static void opt3001_remove(struct i2c_client *client)
-{
-	struct iio_dev *iio = i2c_get_clientdata(client);
-	struct opt3001 *opt = iio_priv(iio);
-	int ret;
-	u16 reg;
-
-	if (opt->use_irq)
-		free_irq(client->irq, iio);
-
-	ret = i2c_smbus_read_word_swapped(opt->client, OPT3001_CONFIGURATION);
-	if (ret < 0) {
-		dev_err(opt->dev, "failed to read register %02x\n",
-				OPT3001_CONFIGURATION);
-		return;
-	}
-
-	reg = ret;
-	opt3001_set_mode(opt, &reg, OPT3001_CONFIGURATION_M_SHUTDOWN);
-
-	ret = i2c_smbus_write_word_swapped(opt->client, OPT3001_CONFIGURATION,
-			reg);
-	if (ret < 0) {
-		dev_err(opt->dev, "failed to write register %02x\n",
-				OPT3001_CONFIGURATION);
-	}
+	return devm_iio_device_register(dev, iio);
 }
 
 static const struct opt3001_chip_info opt3001_chip_information = {
@@ -964,7 +946,6 @@ MODULE_DEVICE_TABLE(of, opt3001_of_match);
 
 static struct i2c_driver opt3001_driver = {
 	.probe = opt3001_probe,
-	.remove = opt3001_remove,
 	.id_table = opt3001_id,
 
 	.driver = {
diff --git a/drivers/iio/light/st_uvis25_spi.c b/drivers/iio/light/st_uvis25_spi.c
index c4c15093e9e5b..0f78a0abfafb6 100644
--- a/drivers/iio/light/st_uvis25_spi.c
+++ b/drivers/iio/light/st_uvis25_spi.c
@@ -46,7 +46,7 @@ static const struct of_device_id st_uvis25_spi_of_match[] = {
 MODULE_DEVICE_TABLE(of, st_uvis25_spi_of_match);
 
 static const struct spi_device_id st_uvis25_spi_id_table[] = {
-	{ ST_UVIS25_DEV_NAME },
+	{ .name = ST_UVIS25_DEV_NAME },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, st_uvis25_spi_id_table);
diff --git a/drivers/iio/light/tcs3472.c b/drivers/iio/light/tcs3472.c
index b61442c5c1bef..b392943e51c8b 100644
--- a/drivers/iio/light/tcs3472.c
+++ b/drivers/iio/light/tcs3472.c
@@ -9,8 +9,6 @@
  * TCS34727)
  *
  * Datasheet: http://ams.com/eng/content/download/319364/1117183/file/TCS3472_Datasheet_EN_v2.pdf
- *
- * TODO: wait time
  */
 
 #include <linux/cleanup.h>
@@ -19,6 +17,7 @@
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/pm.h>
+#include <linux/units.h>
 
 #include <linux/iio/buffer.h>
 #include <linux/iio/events.h>
@@ -53,19 +52,30 @@
 #define TCS3472_STATUS_AINT BIT(4)
 #define TCS3472_STATUS_AVALID BIT(0)
 #define TCS3472_ENABLE_AIEN BIT(4)
+#define TCS3472_ENABLE_WEN BIT(3)
 #define TCS3472_ENABLE_AEN BIT(1)
 #define TCS3472_ENABLE_PON BIT(0)
+#define TCS3472_ENABLE_RUN						\
+	(TCS3472_ENABLE_AEN | TCS3472_ENABLE_PON | TCS3472_ENABLE_WEN)
 #define TCS3472_CONTROL_AGAIN_MASK (BIT(0) | BIT(1))
+#define TCS3472_CONFIG_WLONG BIT(1)
+
+#define TCS3472_ATIME_TO_US(atime) (((256) - (atime)) * 2400)
 
 struct tcs3472_data {
 	struct i2c_client *client;
 	struct mutex lock;
+	int target_freq_hz;
+	int target_freq_uhz;
 	u16 low_thresh;
 	u16 high_thresh;
 	u8 enable;
+	u8 enable_pre_suspend;
 	u8 control;
 	u8 atime;
 	u8 apers;
+	u8 wtime;
+	bool wlong;
 };
 
 static const struct iio_event_spec tcs3472_events[] = {
@@ -91,6 +101,7 @@ static const struct iio_event_spec tcs3472_events[] = {
 	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \
 	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_CALIBSCALE) | \
 		BIT(IIO_CHAN_INFO_INT_TIME), \
+	.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), \
 	.channel2 = IIO_MOD_LIGHT_##_color, \
 	.address = _addr, \
 	.scan_index = _si, \
@@ -114,9 +125,65 @@ static const struct iio_chan_spec tcs3472_channels[] = {
 	IIO_CHAN_SOFT_TIMESTAMP(4),
 };
 
+/*
+ * The chip's cycle time is the sum of three components:
+ *   - ATIME: the programmable RGBC integration time.
+ *   - The fixed RGBC initialization time (2400 us).
+ *   - WTIME: the wait time, used only if WEN is set. If WLONG is active,
+ *     the wait step is multiplied by 12 (2400 us -> 28800 us).
+ */
+static unsigned int tcs3472_cycle_time_us(struct tcs3472_data *data)
+{
+	unsigned int atime_us = TCS3472_ATIME_TO_US(data->atime);
+	unsigned int init_us = 2400;
+	unsigned int wtime_us;
+
+	if (!(data->enable & TCS3472_ENABLE_WEN))
+		wtime_us = 0;
+	else if (data->wlong)
+		wtime_us = (256 - data->wtime) * 28800;
+	else
+		wtime_us = (256 - data->wtime) * 2400;
+
+	return atime_us + init_us + wtime_us;
+}
+
+/*
+ * Convert a cycle time in microseconds to a frequency in Hz and microhertz.
+ *
+ * Given cycle_us = T (the cycle period in microseconds), the corresponding
+ * frequency is:
+ *   f = 1e6 / T  [Hz]
+ *
+ * The result is split into the IIO_VAL_INT_PLUS_MICRO format:
+ *   val  = floor(1e6 / T)                [Hz]
+ *   val2 = (1e6 mod T) * 1e6 / T         [microhertz]
+ */
+static void tcs3472_cycle_to_freq(unsigned int cycle_us, int *val, int *val2)
+{
+	*val = USEC_PER_SEC / cycle_us;
+	*val2 = div_u64((u64)(USEC_PER_SEC % cycle_us) * USEC_PER_SEC,
+			cycle_us);
+}
+
 static int tcs3472_req_data(struct tcs3472_data *data)
 {
-	int tries = 50;
+	/*
+	 * The worst-case cycle time is reached with ATIME=0x00, WTIME=0x00
+	 * and WLONG=1. So:
+	 *   614 ms (Max Integration Time)
+	 * + 2.4 ms (RGBC Init)
+	 * + 7.37 s (Max Wait Time)
+	 * = ~ 8 s (Total Max cycle time).
+	 * Use that as a polling upper bound; in normal operation the loop
+	 * exits as soon as AVALID is set. So the total number of tries in 8
+	 * seconds considering a polling period of 20 ms is 400.
+	 * Considering a 20% margin due to oscillator tolerance, the total
+	 * duration becomes approximately 9.8 seconds, which corresponds to
+	 * about 480 steps. Therefore, setting it to 500 appears to be a
+	 * reasonable and safe trade-off.
+	 */
+	int tries = 500;
 	int ret;
 
 	while (tries--) {
@@ -164,18 +231,165 @@ static int tcs3472_read_raw(struct iio_dev *indio_dev,
 		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_INT_TIME:
 		*val = 0;
-		*val2 = (256 - data->atime) * 2400;
+		*val2 = TCS3472_ATIME_TO_US(data->atime);
 		return IIO_VAL_INT_PLUS_MICRO;
+	case IIO_CHAN_INFO_SAMP_FREQ: {
+		unsigned int cycle_us;
+
+		guard(mutex)(&data->lock);
+		cycle_us = tcs3472_cycle_time_us(data);
+		tcs3472_cycle_to_freq(cycle_us, val, val2);
+		return IIO_VAL_INT_PLUS_MICRO;
+	}
 	default:
 		return -EINVAL;
 	}
 }
 
+/*
+ * __tcs3472_set_sampling_freq() - implementation of sampling frequency
+ * configuration. The caller must hold data->lock.
+ */
+static int __tcs3472_set_sampling_freq(struct tcs3472_data *data,
+				       int val, int val2)
+{
+	unsigned int atime_us;
+	unsigned int init_us = 2400;
+	u64 cycle_us;
+	s64 wait_us;
+	int wtime;
+	bool wlong = false;
+	u8 config;
+	int ret;
+
+	if (val < 0 || val2 < 0 || (val == 0 && val2 == 0))
+		return -EINVAL;
+
+	atime_us = TCS3472_ATIME_TO_US(data->atime);
+
+	/*
+	 * cycle_us = 1 / freq, expressed in microseconds.
+	 * Numerator: 1 [s] = PSEC_PER_SEC [ps]
+	 * Denominator: freq [Hz] * MICROHZ_PER_HZ + val2 [uHz] = freq in [uHz]
+	 * Result: ps / uHz = us
+	 */
+	cycle_us = div64_u64(PSEC_PER_SEC, (u64)val * MICROHZ_PER_HZ + val2);
+
+	/*
+	 * wait_us can be negative when the requested frequency is too high
+	 * to be reached, or very large when the requested frequency is
+	 * close to zero. Use s64 to cover the full range:
+	 *
+	 *   cycle_us = PSEC_PER_SEC / (val * MICROHZ_PER_HZ + val2)
+	 *
+	 * The divisor of the formula above reaches its maximum when
+	 * val = val2 = INT_MAX:
+	 *  INT_MAX * MICROHZ_PER_HZ + INT_MAX = ~2.15e18
+	 * so cycle_us_min = floor(1e12 / 2.15e18) = 0.
+	 *
+	 * The divisor reaches its minimum (1) when val = 0 and val2 = 1,
+	 * so cycle_us_max = 1e12 / 1 = 1e12.
+	 *
+	 * Therefore:
+	 *   wait_us_min = 0 - 2400 - 612000 = -616800
+	 *   wait_us_max = 1e12 - 2400 - 2400 = 999999995200
+	 *
+	 * Both fit comfortably in s64.
+	 */
+	wait_us = (s64)cycle_us - init_us - atime_us;
+	if (wait_us < 2400) {
+		if (data->enable & TCS3472_ENABLE_WEN) {
+			u8 enable = data->enable & ~TCS3472_ENABLE_WEN;
+
+			ret = i2c_smbus_write_byte_data(data->client,
+							TCS3472_ENABLE, enable);
+			if (ret)
+				return ret;
+
+			data->enable = enable;
+		}
+
+		data->target_freq_hz = val;
+		data->target_freq_uhz = val2;
+		return 0;
+	}
+
+	/*
+	 * Wait state is needed: make sure WEN is active before programming
+	 * WTIME (and possibly WLONG).
+	 */
+	if (!(data->enable & TCS3472_ENABLE_WEN)) {
+		u8 enable = data->enable | TCS3472_ENABLE_WEN;
+
+		ret = i2c_smbus_write_byte_data(data->client, TCS3472_ENABLE,
+						enable);
+		if (ret)
+			return ret;
+
+		data->enable = enable;
+	}
+
+	wtime = 256 - DIV_ROUND_CLOSEST_ULL(wait_us, 2400);
+	if (wtime < 0) {
+		/*
+		 * If wait_us is too high (so the requested frequency is too
+		 * low), the resulting wait exceeds what WTIME can represent
+		 * (max 614 ms without WLONG). Enable WLONG, whose step is 12x
+		 * longer (28.8 ms instead of 2.4 ms), and recompute.
+		 */
+		wlong = true;
+		wtime = 256 - DIV_ROUND_CLOSEST_ULL(wait_us, 28800);
+	}
+
+	if (wlong != data->wlong) {
+		ret = i2c_smbus_read_byte_data(data->client, TCS3472_CONFIG);
+		if (ret < 0)
+			return ret;
+
+		config = ret;
+		if (wlong)
+			config |= TCS3472_CONFIG_WLONG;
+		else
+			config &= ~TCS3472_CONFIG_WLONG;
+
+		ret = i2c_smbus_write_byte_data(data->client, TCS3472_CONFIG,
+						config);
+		if (ret)
+			return ret;
+
+		data->wlong = wlong;
+	}
+
+	/*
+	 * If the requested wait is so long that even WLONG cannot
+	 * cover it, wtime may still be negative. Saturate to 0,
+	 * which is the largest possible wait (256 * 28.8 ms = 7.37 s).
+	 */
+	wtime = clamp(wtime, 0, 255);
+	ret = i2c_smbus_write_byte_data(data->client, TCS3472_WTIME, wtime);
+	if (ret)
+		return ret;
+
+	data->wtime = wtime;
+	data->target_freq_hz = val;
+	data->target_freq_uhz = val2;
+
+	return 0;
+}
+
+static int tcs3472_set_sampling_freq(struct tcs3472_data *data,
+				     int val, int val2)
+{
+	guard(mutex)(&data->lock);
+	return __tcs3472_set_sampling_freq(data, val, val2);
+}
+
 static int tcs3472_write_raw(struct iio_dev *indio_dev,
 			       struct iio_chan_spec const *chan,
 			       int val, int val2, long mask)
 {
 	struct tcs3472_data *data = iio_priv(indio_dev);
+	int ret;
 	int i;
 
 	switch (mask) {
@@ -196,15 +410,31 @@ static int tcs3472_write_raw(struct iio_dev *indio_dev,
 		if (val != 0)
 			return -EINVAL;
 		for (i = 0; i < 256; i++) {
-			if (val2 == (256 - i) * 2400) {
-				data->atime = i;
-				return i2c_smbus_write_byte_data(
-					data->client, TCS3472_ATIME,
-					data->atime);
-			}
-
+			if (val2 != (256 - i) * 2400)
+				continue;
+
+			guard(mutex)(&data->lock);
+
+			ret = i2c_smbus_write_byte_data(data->client,
+							TCS3472_ATIME, i);
+			if (ret)
+				return ret;
+
+			data->atime = i;
+
+			/*
+			 * ATIME just changed, so the cycle time changed too.
+			 * Re-run the sampling frequency logic to recompute
+			 * WTIME and preserve the user's last requested
+			 * frequency. Lock is already held.
+			 */
+			return __tcs3472_set_sampling_freq(data,
+							 data->target_freq_hz,
+							 data->target_freq_uhz);
 		}
 		return -EINVAL;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		return tcs3472_set_sampling_freq(data, val, val2);
 	default:
 		return -EINVAL;
 	}
@@ -234,7 +464,7 @@ static int tcs3472_read_event(struct iio_dev *indio_dev,
 			data->high_thresh : data->low_thresh;
 		return IIO_VAL_INT;
 	case IIO_EV_INFO_PERIOD:
-		period = (256 - data->atime) * 2400 *
+		period = tcs3472_cycle_time_us(data) *
 			tcs3472_intr_pers[data->apers];
 		*val = period / USEC_PER_SEC;
 		*val2 = period % USEC_PER_SEC;
@@ -279,11 +509,13 @@ static int tcs3472_write_event(struct iio_dev *indio_dev,
 			data->low_thresh = val;
 
 		return 0;
-	case IIO_EV_INFO_PERIOD:
+	case IIO_EV_INFO_PERIOD:{
+		unsigned int cycle_us;
+
 		period = val * USEC_PER_SEC + val2;
+		cycle_us = tcs3472_cycle_time_us(data);
 		for (i = 1; i < ARRAY_SIZE(tcs3472_intr_pers) - 1; i++) {
-			if (period <= (256 - data->atime) * 2400 *
-					tcs3472_intr_pers[i])
+			if (period <= cycle_us * tcs3472_intr_pers[i])
 				break;
 		}
 		ret = i2c_smbus_write_byte_data(data->client, TCS3472_PERS, i);
@@ -293,6 +525,7 @@ static int tcs3472_write_event(struct iio_dev *indio_dev,
 		data->apers = i;
 
 		return 0;
+	}
 	default:
 		return -EINVAL;
 	}
@@ -434,17 +667,16 @@ static const struct iio_info tcs3472_info = {
 static int tcs3472_powerdown(struct tcs3472_data *data)
 {
 	int ret;
-	u8 enable_mask = TCS3472_ENABLE_AEN | TCS3472_ENABLE_PON;
 
 	guard(mutex)(&data->lock);
 
+	data->enable_pre_suspend = data->enable;
+
 	ret = i2c_smbus_write_byte_data(data->client, TCS3472_ENABLE,
-					data->enable & ~enable_mask);
+					data->enable & ~TCS3472_ENABLE_RUN);
 	if (ret)
 		return ret;
 
-	data->enable &= ~enable_mask;
-
 	return 0;
 }
 
@@ -458,7 +690,9 @@ static int tcs3472_probe(struct i2c_client *client)
 	struct device *dev = &client->dev;
 	struct tcs3472_data *data;
 	struct iio_dev *indio_dev;
+	unsigned int cycle_us;
 	int ret;
+	u8 enable;
 
 	indio_dev = devm_iio_device_alloc(dev, sizeof(*data));
 	if (!indio_dev)
@@ -498,6 +732,16 @@ static int tcs3472_probe(struct i2c_client *client)
 		return ret;
 	data->atime = ret;
 
+	ret = i2c_smbus_read_byte_data(data->client, TCS3472_WTIME);
+	if (ret < 0)
+		return ret;
+	data->wtime = ret;
+
+	ret = i2c_smbus_read_byte_data(data->client, TCS3472_CONFIG);
+	if (ret < 0)
+		return ret;
+	data->wlong = (ret & TCS3472_CONFIG_WLONG) ? 1 : 0;
+
 	ret = i2c_smbus_read_word_data(data->client, TCS3472_AILT);
 	if (ret < 0)
 		return ret;
@@ -518,14 +762,30 @@ static int tcs3472_probe(struct i2c_client *client)
 	if (ret < 0)
 		return ret;
 
-	/* enable device */
-	data->enable = ret | TCS3472_ENABLE_PON | TCS3472_ENABLE_AEN;
-	data->enable &= ~TCS3472_ENABLE_AIEN;
-	ret = i2c_smbus_write_byte_data(data->client, TCS3472_ENABLE,
-		data->enable);
+	/*
+	 * Enable the chip in its full running state, including WEN. The
+	 * actual wait time is controlled by the WTIME and WLONG registers,
+	 * which retain their power-on defaults until userspace writes to
+	 * sampling_frequency.
+	 */
+	enable = (ret | TCS3472_ENABLE_RUN) & ~TCS3472_ENABLE_AIEN;
+
+	ret = i2c_smbus_write_byte_data(data->client, TCS3472_ENABLE, enable);
 	if (ret < 0)
 		return ret;
 
+	data->enable = enable;
+
+	/*
+	 * Initialize target frequency from the chip's current state so that
+	 * subsequent integration_time changes via IIO_CHAN_INFO_INT_TIME can
+	 * preserve a meaningful sampling rate, even before userspace writes
+	 * sampling_frequency for the first time.
+	 */
+	cycle_us = tcs3472_cycle_time_us(data);
+	tcs3472_cycle_to_freq(cycle_us, &data->target_freq_hz,
+			      &data->target_freq_uhz);
+
 	ret = devm_add_action_or_reset(dev, tcs3472_powerdown_action, data);
 	if (ret)
 		return ret;
@@ -561,16 +821,21 @@ static int tcs3472_resume(struct device *dev)
 	struct tcs3472_data *data = iio_priv(i2c_get_clientdata(
 		to_i2c_client(dev)));
 	int ret;
-	u8 enable_mask = TCS3472_ENABLE_AEN | TCS3472_ENABLE_PON;
 
 	guard(mutex)(&data->lock);
 
+	/*
+	 * Restore the full ENABLE register from the snapshot taken in
+	 * tcs3472_powerdown(). This preserves the user's last
+	 * sampling_frequency configuration (in particular the WEN bit)
+	 * across suspend/resume.
+	 */
 	ret = i2c_smbus_write_byte_data(data->client, TCS3472_ENABLE,
-		data->enable | enable_mask);
+					data->enable_pre_suspend);
 	if (ret)
 		return ret;
 
-	data->enable |= enable_mask;
+	data->enable = data->enable_pre_suspend;
 
 	return 0;
 }
diff --git a/drivers/iio/magnetometer/Kconfig b/drivers/iio/magnetometer/Kconfig
index ea2697fb5ab61..2dee566ba7033 100644
--- a/drivers/iio/magnetometer/Kconfig
+++ b/drivers/iio/magnetometer/Kconfig
@@ -209,6 +209,17 @@ config INFINEON_TLV493D
 	  To compile this driver as a module, choose M here: the module
 	  will be called tlv493d.
 
+config QMC5883L
+	tristate "QST QMC5883L 3-Axis Magnetic Sensor"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  Say Y here to add support driver for QST QMC5883L 3-Axis
+	  Magnetic Sensor.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called qmc5883l.
+
 config SENSORS_HMC5843
 	tristate
 	select IIO_BUFFER
diff --git a/drivers/iio/magnetometer/Makefile b/drivers/iio/magnetometer/Makefile
index 7fd9b3fd914e3..b9478e6513e47 100644
--- a/drivers/iio/magnetometer/Makefile
+++ b/drivers/iio/magnetometer/Makefile
@@ -27,6 +27,8 @@ obj-$(CONFIG_IIO_ST_MAGN_SPI_3AXIS) += st_magn_spi.o
 
 obj-$(CONFIG_INFINEON_TLV493D)		+= tlv493d.o
 
+obj-$(CONFIG_QMC5883L)			+= qmc5883l.o
+
 obj-$(CONFIG_SENSORS_HMC5843)		+= hmc5843_core.o
 obj-$(CONFIG_SENSORS_HMC5843_I2C)	+= hmc5843_i2c.o
 obj-$(CONFIG_SENSORS_HMC5843_SPI)	+= hmc5843_spi.o
diff --git a/drivers/iio/magnetometer/bmc150_magn.c b/drivers/iio/magnetometer/bmc150_magn.c
index bf25519880089..32dff9cdd0f70 100644
--- a/drivers/iio/magnetometer/bmc150_magn.c
+++ b/drivers/iio/magnetometer/bmc150_magn.c
@@ -9,23 +9,25 @@
  * (C) Copyright 2011~2014 Bosch Sensortec GmbH All Rights Reserved
  */
 
-#include <linux/module.h>
-#include <linux/i2c.h>
-#include <linux/interrupt.h>
+#include <linux/bitfield.h>
 #include <linux/cleanup.h>
 #include <linux/delay.h>
-#include <linux/slab.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
 #include <linux/pm.h>
 #include <linux/pm_runtime.h>
-#include <linux/iio/iio.h>
-#include <linux/iio/sysfs.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/slab.h>
+
 #include <linux/iio/buffer.h>
 #include <linux/iio/events.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
 #include <linux/iio/trigger.h>
 #include <linux/iio/trigger_consumer.h>
 #include <linux/iio/triggered_buffer.h>
-#include <linux/regmap.h>
-#include <linux/regulator/consumer.h>
 
 #include "bmc150_magn.h"
 
@@ -245,14 +247,14 @@ static int bmc150_magn_set_power_mode(struct bmc150_magn_data *data,
 		return regmap_update_bits(data->regmap,
 					  BMC150_MAGN_REG_OPMODE_ODR,
 					  BMC150_MAGN_MASK_OPMODE,
-					  BMC150_MAGN_MODE_SLEEP <<
-					  BMC150_MAGN_SHIFT_OPMODE);
+					  FIELD_PREP(BMC150_MAGN_MASK_OPMODE,
+						     BMC150_MAGN_MODE_SLEEP));
 	case BMC150_MAGN_POWER_MODE_NORMAL:
 		return regmap_update_bits(data->regmap,
 					  BMC150_MAGN_REG_OPMODE_ODR,
 					  BMC150_MAGN_MASK_OPMODE,
-					  BMC150_MAGN_MODE_NORMAL <<
-					  BMC150_MAGN_SHIFT_OPMODE);
+					  FIELD_PREP(BMC150_MAGN_MASK_OPMODE,
+						     BMC150_MAGN_MODE_NORMAL));
 	}
 
 	return -EINVAL;
@@ -290,7 +292,7 @@ static int bmc150_magn_get_odr(struct bmc150_magn_data *data, int *val)
 	ret = regmap_read(data->regmap, BMC150_MAGN_REG_OPMODE_ODR, &reg_val);
 	if (ret < 0)
 		return ret;
-	odr_val = (reg_val & BMC150_MAGN_MASK_ODR) >> BMC150_MAGN_SHIFT_ODR;
+	odr_val = FIELD_GET(BMC150_MAGN_MASK_ODR, reg_val);
 
 	for (i = 0; i < ARRAY_SIZE(bmc150_magn_samp_freq_table); i++)
 		if (bmc150_magn_samp_freq_table[i].reg_val == odr_val) {
@@ -303,21 +305,17 @@ static int bmc150_magn_get_odr(struct bmc150_magn_data *data, int *val)
 
 static int bmc150_magn_set_odr(struct bmc150_magn_data *data, int val)
 {
-	int ret;
 	u8 i;
 
 	for (i = 0; i < ARRAY_SIZE(bmc150_magn_samp_freq_table); i++) {
-		if (bmc150_magn_samp_freq_table[i].freq == val) {
-			ret = regmap_update_bits(data->regmap,
-						 BMC150_MAGN_REG_OPMODE_ODR,
-						 BMC150_MAGN_MASK_ODR,
-						 bmc150_magn_samp_freq_table[i].
-						 reg_val <<
-						 BMC150_MAGN_SHIFT_ODR);
-			if (ret < 0)
-				return ret;
-			return 0;
-		}
+		if (bmc150_magn_samp_freq_table[i].freq != val)
+			continue;
+
+		return regmap_update_bits(data->regmap,
+					  BMC150_MAGN_REG_OPMODE_ODR,
+					  BMC150_MAGN_MASK_ODR,
+					  FIELD_PREP(BMC150_MAGN_MASK_ODR,
+						     bmc150_magn_samp_freq_table[i].reg_val));
 	}
 
 	return -EINVAL;
@@ -800,7 +798,7 @@ static int bmc150_magn_data_rdy_trigger_set_state(struct iio_trigger *trig,
 
 	ret = regmap_update_bits(data->regmap, BMC150_MAGN_REG_INT_DRDY,
 				 BMC150_MAGN_MASK_DRDY_EN,
-				 state << BMC150_MAGN_SHIFT_DRDY_EN);
+				 FIELD_PREP(BMC150_MAGN_MASK_DRDY_EN, state));
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/iio/magnetometer/bmc150_magn_spi.c b/drivers/iio/magnetometer/bmc150_magn_spi.c
index 8af8d3528c435..27ab845fb2578 100644
--- a/drivers/iio/magnetometer/bmc150_magn_spi.c
+++ b/drivers/iio/magnetometer/bmc150_magn_spi.c
@@ -33,9 +33,9 @@ static void bmc150_magn_spi_remove(struct spi_device *spi)
 }
 
 static const struct spi_device_id bmc150_magn_spi_id[] = {
-	{"bmc150_magn", 0},
-	{"bmc156_magn", 0},
-	{"bmm150_magn", 0},
+	{ .name = "bmc150_magn" },
+	{ .name = "bmc156_magn" },
+	{ .name = "bmm150_magn" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, bmc150_magn_spi_id);
diff --git a/drivers/iio/magnetometer/hid-sensor-magn-3d.c b/drivers/iio/magnetometer/hid-sensor-magn-3d.c
index d8b8bcc865c31..ad18f233ee166 100644
--- a/drivers/iio/magnetometer/hid-sensor-magn-3d.c
+++ b/drivers/iio/magnetometer/hid-sensor-magn-3d.c
@@ -134,9 +134,8 @@ static const struct iio_chan_spec magn_3d_channels[] = {
 
 /* Channel read_raw handler */
 static int magn_3d_read_raw(struct iio_dev *indio_dev,
-			      struct iio_chan_spec const *chan,
-			      int *val, int *val2,
-			      long mask)
+			    struct iio_chan_spec const *chan,
+			    int *val, int *val2, long mask)
 {
 	struct magn_3d_state *magn_state = iio_priv(indio_dev);
 	int report_id = -1;
@@ -166,8 +165,7 @@ static int magn_3d_read_raw(struct iio_dev *indio_dev,
 				false);
 			return -EINVAL;
 		}
-		hid_sensor_power_state(&magn_state->magn_flux_attributes,
-					false);
+		hid_sensor_power_state(&magn_state->magn_flux_attributes, false);
 		ret_type = IIO_VAL_INT;
 		break;
 	case IIO_CHAN_INFO_SCALE:
@@ -228,10 +226,8 @@ static int magn_3d_read_raw(struct iio_dev *indio_dev,
 
 /* Channel write_raw handler */
 static int magn_3d_write_raw(struct iio_dev *indio_dev,
-			       struct iio_chan_spec const *chan,
-			       int val,
-			       int val2,
-			       long mask)
+			     struct iio_chan_spec const *chan,
+			     int val, int val2, long mask)
 {
 	struct magn_3d_state *magn_state = iio_priv(indio_dev);
 	int ret = 0;
@@ -269,8 +265,7 @@ static const struct iio_info magn_3d_info = {
 
 /* Callback handler to send event after all samples are received and captured */
 static int magn_3d_proc_event(struct hid_sensor_hub_device *hsdev,
-				u32 usage_id,
-				void *priv)
+			      u32 usage_id, void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
 	struct magn_3d_state *magn_state = iio_priv(indio_dev);
@@ -291,9 +286,9 @@ static int magn_3d_proc_event(struct hid_sensor_hub_device *hsdev,
 
 /* Capture samples in local storage */
 static int magn_3d_capture_sample(struct hid_sensor_hub_device *hsdev,
-				u32 usage_id,
-				size_t raw_len, char *raw_data,
-				void *priv)
+				  u32 usage_id,
+				  size_t raw_len, char *raw_data,
+				  void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
 	struct magn_3d_state *magn_state = iio_priv(indio_dev);
@@ -326,7 +321,7 @@ static int magn_3d_capture_sample(struct hid_sensor_hub_device *hsdev,
 
 	iio_val = magn_state->magn_val_addr[offset];
 
-	if (iio_val != NULL)
+	if (iio_val)
 		*iio_val = *((u32 *)raw_data);
 	else
 		ret = -EINVAL;
@@ -367,18 +362,17 @@ static int magn_3d_parse_report(struct platform_device *pdev,
 		return  -EINVAL;
 	}
 
-	dev_dbg(&pdev->dev, "magn_3d Found %d usage attributes\n",
-			attr_count);
+	dev_dbg(&pdev->dev, "magn_3d Found %d usage attributes\n", attr_count);
 	dev_dbg(&pdev->dev, "magn_3d X: %x:%x Y: %x:%x Z: %x:%x\n",
-			st->magn[0].index,
-			st->magn[0].report_id,
-			st->magn[1].index, st->magn[1].report_id,
-			st->magn[2].index, st->magn[2].report_id);
+		st->magn[0].index,
+		st->magn[0].report_id,
+		st->magn[1].index, st->magn[1].report_id,
+		st->magn[2].index, st->magn[2].report_id);
 
 	/* Setup IIO channel array */
 	_channels = devm_kcalloc(&pdev->dev, attr_count,
-				sizeof(struct iio_chan_spec),
-				GFP_KERNEL);
+				 sizeof(struct iio_chan_spec),
+				 GFP_KERNEL);
 	if (!_channels) {
 		dev_err(&pdev->dev,
 			"failed to allocate space for iio channels\n");
@@ -425,8 +419,7 @@ static int magn_3d_parse_report(struct platform_device *pdev,
 
 	*channels = _channels;
 
-	dev_dbg(&pdev->dev, "magn_3d Setup %d IIO channels\n",
-			*chan_count);
+	dev_dbg(&pdev->dev, "magn_3d Setup %d IIO channels\n", *chan_count);
 
 	st->magn_flux_attr.scale_precision = hid_sensor_format_scale(
 				HID_USAGE_SENSOR_COMPASS_3D,
@@ -467,7 +460,7 @@ static int hid_magn_3d_probe(struct platform_device *pdev)
 
 	indio_dev = devm_iio_device_alloc(&pdev->dev,
 					  sizeof(struct magn_3d_state));
-	if (indio_dev == NULL)
+	if (!indio_dev)
 		return -ENOMEM;
 
 	platform_set_drvdata(pdev, indio_dev);
@@ -490,8 +483,8 @@ static int hid_magn_3d_probe(struct platform_device *pdev)
 	magn_state->rot_attributes.sensitivity.index = -1;
 
 	ret = magn_3d_parse_report(pdev, hsdev,
-				&channels, &chan_count,
-				HID_USAGE_SENSOR_COMPASS_3D, magn_state);
+				   &channels, &chan_count,
+				   HID_USAGE_SENSOR_COMPASS_3D, magn_state);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to parse report\n");
 		return ret;
@@ -506,32 +499,32 @@ static int hid_magn_3d_probe(struct platform_device *pdev)
 	atomic_set(&magn_state->magn_flux_attributes.data_ready, 0);
 
 	ret = hid_sensor_setup_trigger(indio_dev, name,
-					&magn_state->magn_flux_attributes);
+				       &magn_state->magn_flux_attributes);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "trigger setup failed\n");
 		return ret;
 	}
 
-	ret = iio_device_register(indio_dev);
-	if (ret) {
-		dev_err(&pdev->dev, "device register failed\n");
-		goto error_remove_trigger;
-	}
-
 	magn_state->callbacks.send_event = magn_3d_proc_event;
 	magn_state->callbacks.capture_sample = magn_3d_capture_sample;
 	magn_state->callbacks.pdev = pdev;
 	ret = sensor_hub_register_callback(hsdev, HID_USAGE_SENSOR_COMPASS_3D,
-					&magn_state->callbacks);
+					   &magn_state->callbacks);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "callback reg failed\n");
-		goto error_iio_unreg;
+		goto error_remove_trigger;
+	}
+
+	ret = iio_device_register(indio_dev);
+	if (ret) {
+		dev_err(&pdev->dev, "device register failed\n");
+		goto error_remove_callback;
 	}
 
 	return ret;
 
-error_iio_unreg:
-	iio_device_unregister(indio_dev);
+error_remove_callback:
+	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_COMPASS_3D);
 error_remove_trigger:
 	hid_sensor_remove_trigger(indio_dev, &magn_state->magn_flux_attributes);
 	return ret;
@@ -544,8 +537,8 @@ static void hid_magn_3d_remove(struct platform_device *pdev)
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
 	struct magn_3d_state *magn_state = iio_priv(indio_dev);
 
-	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_COMPASS_3D);
 	iio_device_unregister(indio_dev);
+	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_COMPASS_3D);
 	hid_sensor_remove_trigger(indio_dev, &magn_state->magn_flux_attributes);
 }
 
diff --git a/drivers/iio/magnetometer/hmc5843_spi.c b/drivers/iio/magnetometer/hmc5843_spi.c
index 6a55c1559b0d8..29dee5dce2aea 100644
--- a/drivers/iio/magnetometer/hmc5843_spi.c
+++ b/drivers/iio/magnetometer/hmc5843_spi.c
@@ -56,7 +56,6 @@ static int hmc5843_spi_probe(struct spi_device *spi)
 {
 	int ret;
 	struct regmap *regmap;
-	const struct spi_device_id *id = spi_get_device_id(spi);
 
 	spi->mode = SPI_MODE_3;
 	spi->max_speed_hz = 8000000;
@@ -69,8 +68,7 @@ static int hmc5843_spi_probe(struct spi_device *spi)
 		return PTR_ERR(regmap);
 
 	return hmc5843_common_probe(&spi->dev,
-			regmap,
-			id->driver_data, id->name);
+			regmap, HMC5983_ID, "hmc5983");
 }
 
 static void hmc5843_spi_remove(struct spi_device *spi)
@@ -79,7 +77,7 @@ static void hmc5843_spi_remove(struct spi_device *spi)
 }
 
 static const struct spi_device_id hmc5843_id[] = {
-	{ "hmc5983", HMC5983_ID },
+	{ .name = "hmc5983" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, hmc5843_id);
diff --git a/drivers/iio/magnetometer/mmc5983.c b/drivers/iio/magnetometer/mmc5983.c
index 03f46e8c16114..d10a6288e147f 100644
--- a/drivers/iio/magnetometer/mmc5983.c
+++ b/drivers/iio/magnetometer/mmc5983.c
@@ -327,7 +327,7 @@ static const struct of_device_id mmc5983_of_match[] = {
 MODULE_DEVICE_TABLE(of, mmc5983_of_match);
 
 static const struct i2c_device_id mmc5983_id[] = {
-	{ "mmc5983" },
+	{ .name = "mmc5983" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mmc5983_id);
diff --git a/drivers/iio/magnetometer/qmc5883l.c b/drivers/iio/magnetometer/qmc5883l.c
new file mode 100644
index 0000000000000..c3469715dc63f
--- /dev/null
+++ b/drivers/iio/magnetometer/qmc5883l.c
@@ -0,0 +1,516 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+/*
+ * Support for QST QMC5883L 3-Axis Magnetic Sensor on I2C bus.
+ *
+ * Copyright (C) 2026 Siratul Islam <siratul.islam@linux.dev>
+ *
+ * Datasheet available at
+ * <https://www.qstcorp.com/upload/pdf/202512/13-52-04%20QMC5883L%20Datasheet%20Rev.%20B.pdf>
+ */
+
+#include <linux/array_size.h>
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/cleanup.h>
+#include <linux/delay.h>
+#include <linux/dev_printk.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/time.h>
+#include <linux/types.h>
+
+#include <asm/byteorder.h>
+
+#include <linux/iio/iio.h>
+
+#define QMC5883L_REG_X_LSB	0x00
+#define QMC5883L_REG_STATUS1	0x06
+#define QMC5883L_REG_CTRL1	0x09
+#define QMC5883L_REG_CTRL2	0x0A
+#define QMC5883L_REG_SET_RESET	0x0B
+#define QMC5883L_REG_ID		0x0D
+
+#define QMC5883L_CHIP_ID	0xFF
+
+#define QMC5883L_MODE_MASK	GENMASK(1, 0)
+#define QMC5883L_ODR_MASK	GENMASK(3, 2)
+#define QMC5883L_RNG_MASK	GENMASK(5, 4)
+#define QMC5883L_OSR_MASK	GENMASK(7, 6)
+
+#define QMC5883L_MODE_STANDBY	0x00
+#define QMC5883L_MODE_CONT	0x01
+
+#define QMC5883L_ODR_10HZ	0x00
+#define QMC5883L_ODR_50HZ	0x01
+#define QMC5883L_ODR_100HZ	0x02
+#define QMC5883L_ODR_200HZ	0x03
+
+#define QMC5883L_RNG_2G		0x00
+#define QMC5883L_RNG_8G		0x01
+
+#define QMC5883L_OSR_512	0x00
+#define QMC5883L_OSR_256	0x01
+#define QMC5883L_OSR_128	0x02
+#define QMC5883L_OSR_64		0x03
+
+#define QMC5883L_STATUS_DRDY	BIT(0)
+#define QMC5883L_STATUS_OVL	BIT(1)
+
+#define QMC5883L_SET_RESET_VAL	BIT(0)
+#define QMC5883L_INT_DISABLE	BIT(0)
+#define QMC5883L_SOFT_RESET	BIT(7)
+
+#define QMC5883L_PORT_US	350
+
+struct qmc5883l_data {
+	struct regmap *regmap;
+	/*
+	 * Protect data->range/odr/osr.
+	 * Protect poll and read during measurement.
+	 */
+	struct mutex mutex;
+	u8 range;
+	u8 odr;
+	u8 osr;
+};
+
+enum qmc5883l_chan {
+	QMC5883L_AXIS_X,
+	QMC5883L_AXIS_Y,
+	QMC5883L_AXIS_Z,
+};
+
+static const int qmc5883l_odr_avail[] = { 10, 50, 100, 200 };
+
+static const int qmc5883l_osr_avail[] = { 512, 256, 128, 64 };
+
+static const int qmc5883l_scales[][2] = {
+	[QMC5883L_RNG_2G] = { 0, 83333 },
+	[QMC5883L_RNG_8G] = { 0, 333333 },
+};
+
+static int qmc5883l_take_measurement(struct iio_dev *indio_dev, int index,
+				     int *val)
+{
+	struct qmc5883l_data *data = iio_priv(indio_dev);
+	struct regmap *map = data->regmap;
+	unsigned int status;
+	__le16 buf[3];
+	int ret;
+
+	guard(mutex) (&data->mutex);
+
+	/* 50ms headroom over the slowest ODR (10Hz) */
+	ret = regmap_read_poll_timeout(map, QMC5883L_REG_STATUS1,
+				       status, (status & QMC5883L_STATUS_DRDY),
+				       2 * USEC_PER_MSEC, 150 * USEC_PER_MSEC);
+	if (ret)
+		return ret;
+
+	ret = regmap_bulk_read(map, QMC5883L_REG_X_LSB, buf, sizeof(buf));
+	if (ret)
+		return ret;
+
+	if (status & QMC5883L_STATUS_OVL)
+		return -ERANGE;
+
+	*val = (s16)le16_to_cpu(buf[index]);
+
+	return 0;
+}
+
+static int qmc5883l_read_raw(struct iio_dev *indio_dev,
+			     const struct iio_chan_spec *chan,
+			     int *val, int *val2, long mask)
+{
+	struct qmc5883l_data *data = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = qmc5883l_take_measurement(indio_dev, chan->address, val);
+		if (ret)
+			return ret;
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE: {
+		guard(mutex)(&data->mutex);
+
+		*val = qmc5883l_scales[data->range][0];
+		*val2 = qmc5883l_scales[data->range][1];
+
+		return IIO_VAL_INT_PLUS_NANO;
+	}
+	case IIO_CHAN_INFO_SAMP_FREQ: {
+		guard(mutex)(&data->mutex);
+
+		switch (data->odr) {
+		case QMC5883L_ODR_200HZ:
+			*val = 200;
+			break;
+		case QMC5883L_ODR_100HZ:
+			*val = 100;
+			break;
+		case QMC5883L_ODR_50HZ:
+			*val = 50;
+			break;
+		case QMC5883L_ODR_10HZ:
+			*val = 10;
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		return IIO_VAL_INT;
+	}
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO: {
+		guard(mutex)(&data->mutex);
+
+		switch (data->osr) {
+		case QMC5883L_OSR_64:
+			*val = 64;
+			break;
+		case QMC5883L_OSR_128:
+			*val = 128;
+			break;
+		case QMC5883L_OSR_256:
+			*val = 256;
+			break;
+		case QMC5883L_OSR_512:
+			*val = 512;
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		return IIO_VAL_INT;
+	}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int qmc5883l_write_raw(struct iio_dev *indio_dev,
+			      const struct iio_chan_spec *chan,
+			      int val, int val2, long mask)
+{
+	struct qmc5883l_data *data = iio_priv(indio_dev);
+	u8 rng, osr, odr;
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SCALE: {
+		if (val != 0)
+			return -EINVAL;
+
+		if (val2 == qmc5883l_scales[QMC5883L_RNG_2G][1])
+			rng = QMC5883L_RNG_2G;
+		else if (val2 == qmc5883l_scales[QMC5883L_RNG_8G][1])
+			rng = QMC5883L_RNG_8G;
+		else
+			return -EINVAL;
+
+		guard(mutex)(&data->mutex);
+
+		ret = regmap_update_bits(data->regmap, QMC5883L_REG_CTRL1,
+					 QMC5883L_RNG_MASK,
+					 FIELD_PREP(QMC5883L_RNG_MASK, rng));
+		if (ret)
+			return ret;
+
+		data->range = rng;
+
+		return 0;
+	}
+	case IIO_CHAN_INFO_SAMP_FREQ: {
+		switch (val) {
+		case 200:
+			odr = QMC5883L_ODR_200HZ;
+			break;
+		case 100:
+			odr = QMC5883L_ODR_100HZ;
+			break;
+		case 50:
+			odr = QMC5883L_ODR_50HZ;
+			break;
+		case 10:
+			odr = QMC5883L_ODR_10HZ;
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		guard(mutex)(&data->mutex);
+
+		ret = regmap_update_bits(data->regmap, QMC5883L_REG_CTRL1,
+					 QMC5883L_ODR_MASK,
+					 FIELD_PREP(QMC5883L_ODR_MASK, odr));
+		if (ret)
+			return ret;
+
+		data->odr = odr;
+
+		return 0;
+	}
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO: {
+		switch (val) {
+		case 64:
+			osr = QMC5883L_OSR_64;
+			break;
+		case 128:
+			osr = QMC5883L_OSR_128;
+			break;
+		case 256:
+			osr = QMC5883L_OSR_256;
+			break;
+		case 512:
+			osr = QMC5883L_OSR_512;
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		guard(mutex)(&data->mutex);
+
+		ret = regmap_update_bits(data->regmap, QMC5883L_REG_CTRL1,
+					 QMC5883L_OSR_MASK,
+					 FIELD_PREP(QMC5883L_OSR_MASK, osr));
+		if (ret)
+			return ret;
+
+		data->osr = osr;
+
+		return 0;
+	}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int qmc5883l_read_avail(struct iio_dev *indio_dev,
+			       struct iio_chan_spec const *chan,
+			       const int **vals, int *type, int *length,
+			       long mask)
+{
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*vals = qmc5883l_odr_avail;
+		*type = IIO_VAL_INT;
+		*length = ARRAY_SIZE(qmc5883l_odr_avail);
+		return IIO_AVAIL_LIST;
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
+		*vals = qmc5883l_osr_avail;
+		*type = IIO_VAL_INT;
+		*length = ARRAY_SIZE(qmc5883l_osr_avail);
+		return IIO_AVAIL_LIST;
+	case IIO_CHAN_INFO_SCALE:
+		*vals = (const int *)qmc5883l_scales;
+		*type = IIO_VAL_INT_PLUS_NANO;
+		*length = ARRAY_SIZE(qmc5883l_scales) * 2;
+		return IIO_AVAIL_LIST;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int qmc5883l_write_raw_get_fmt(struct iio_dev *indio_dev,
+				      struct iio_chan_spec const *chan,
+				      long mask)
+{
+	switch (mask) {
+	case IIO_CHAN_INFO_SCALE:
+		return IIO_VAL_INT_PLUS_NANO;
+	default:
+		return IIO_VAL_INT;
+	}
+}
+
+static const struct iio_info qmc5883l_info = {
+	.read_raw = qmc5883l_read_raw,
+	.write_raw = qmc5883l_write_raw,
+	.read_avail = qmc5883l_read_avail,
+	.write_raw_get_fmt = qmc5883l_write_raw_get_fmt,
+};
+
+static int qmc5883l_init(struct qmc5883l_data *data)
+{
+	struct regmap *map = data->regmap;
+	unsigned int reg;
+	int ret;
+
+	ret = regmap_read(map, QMC5883L_REG_ID, &reg);
+	if (ret)
+		return ret;
+
+	/* Not failing because rev 1.0 had this register reserved */
+	if (reg != QMC5883L_CHIP_ID)
+		dev_warn(regmap_get_device(map),
+			 "Unknown chip id: 0x%02x, continuing\n", reg);
+
+	ret = regmap_write(map, QMC5883L_REG_CTRL2, QMC5883L_SOFT_RESET);
+	if (ret)
+		return ret;
+
+	/* Use POR completion time as a conservative bound */
+	fsleep(QMC5883L_PORT_US);
+
+	/* DRDY pin not used in this version of the driver */
+	ret = regmap_write(map, QMC5883L_REG_CTRL2, QMC5883L_INT_DISABLE);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(map, QMC5883L_REG_SET_RESET, QMC5883L_SET_RESET_VAL);
+	if (ret)
+		return ret;
+
+	data->odr = QMC5883L_ODR_50HZ;
+	data->range = QMC5883L_RNG_2G;
+	data->osr = QMC5883L_OSR_64;
+
+	return regmap_write(map, QMC5883L_REG_CTRL1,
+			    FIELD_PREP(QMC5883L_MODE_MASK, QMC5883L_MODE_CONT) |
+			    FIELD_PREP(QMC5883L_ODR_MASK, data->odr) |
+			    FIELD_PREP(QMC5883L_RNG_MASK, data->range) |
+			    FIELD_PREP(QMC5883L_OSR_MASK, data->osr));
+}
+
+static void qmc5883l_power_down_action(void *priv)
+{
+	struct qmc5883l_data *data = priv;
+
+	regmap_update_bits(data->regmap, QMC5883L_REG_CTRL1,
+			   QMC5883L_MODE_MASK,
+			   FIELD_PREP(QMC5883L_MODE_MASK, QMC5883L_MODE_STANDBY));
+}
+
+static bool qmc5883l_volatile_reg(struct device *dev, unsigned int reg)
+{
+	return reg <= QMC5883L_REG_STATUS1;
+}
+
+static bool qmc5883l_writable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case QMC5883L_REG_CTRL1:
+	case QMC5883L_REG_CTRL2:
+	case QMC5883L_REG_SET_RESET:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static const struct regmap_config qmc5883l_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = QMC5883L_REG_ID,
+	.cache_type = REGCACHE_MAPLE,
+	.volatile_reg = qmc5883l_volatile_reg,
+	.writeable_reg = qmc5883l_writable_reg,
+};
+
+#define QMC5883L_CHANNEL(_axis)                                \
+	{                                                      \
+		.type = IIO_MAGN,                              \
+		.modified = 1,                                 \
+		.channel2 = IIO_MOD_##_axis,                   \
+		.address = QMC5883L_AXIS_##_axis,              \
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),  \
+		.info_mask_shared_by_type =                    \
+			BIT(IIO_CHAN_INFO_SCALE) |             \
+			BIT(IIO_CHAN_INFO_SAMP_FREQ) |         \
+			BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \
+		.info_mask_shared_by_type_available =          \
+			BIT(IIO_CHAN_INFO_SCALE) |             \
+			BIT(IIO_CHAN_INFO_SAMP_FREQ) |         \
+			BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \
+	}
+
+static const struct iio_chan_spec qmc5883l_channels[] = {
+	QMC5883L_CHANNEL(X),
+	QMC5883L_CHANNEL(Y),
+	QMC5883L_CHANNEL(Z),
+};
+
+static int qmc5883l_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct qmc5883l_data *data;
+	struct iio_dev *indio_dev;
+	struct regmap *map;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	map = devm_regmap_init_i2c(client, &qmc5883l_regmap_config);
+	if (IS_ERR(map))
+		return dev_err_probe(dev, PTR_ERR(map),
+				     "regmap initialization failed\n");
+
+	ret = devm_regulator_get_enable(dev, "vdd");
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to enable VDD regulator\n");
+
+	ret = devm_regulator_get_enable(dev, "vddio");
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to enable VDDIO regulator\n");
+
+	/* POR completion time max per datasheet Table 7 */
+	fsleep(QMC5883L_PORT_US);
+
+	data = iio_priv(indio_dev);
+	data->regmap = map;
+
+	ret = devm_mutex_init(dev, &data->mutex);
+	if (ret)
+		return ret;
+
+	indio_dev->name = "qmc5883l";
+	indio_dev->info = &qmc5883l_info;
+	indio_dev->channels = qmc5883l_channels;
+	indio_dev->num_channels = ARRAY_SIZE(qmc5883l_channels);
+	indio_dev->modes = INDIO_DIRECT_MODE;
+
+	ret = qmc5883l_init(data);
+	if (ret)
+		return dev_err_probe(dev, ret, "qmc5883l init failed\n");
+
+	ret = devm_add_action_or_reset(dev, qmc5883l_power_down_action, data);
+	if (ret)
+		return ret;
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static const struct of_device_id qmc5883l_match[] = {
+	{ .compatible = "qstcorp,qmc5883l" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, qmc5883l_match);
+
+static const struct i2c_device_id qmc5883l_id[] = {
+	{ .name = "qmc5883l" },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, qmc5883l_id);
+
+static struct i2c_driver qmc5883l_driver = {
+	.driver = {
+		.name = "qmc5883l",
+		.of_match_table = qmc5883l_match,
+	},
+	.id_table = qmc5883l_id,
+	.probe = qmc5883l_probe,
+};
+module_i2c_driver(qmc5883l_driver);
+
+MODULE_DESCRIPTION("QST QMC5883L 3-Axis Magnetic Sensor driver");
+MODULE_AUTHOR("Siratul Islam <siratul.islam@linux.dev>");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/iio/magnetometer/st_magn_spi.c b/drivers/iio/magnetometer/st_magn_spi.c
index 69693e836ad74..af4642b6c3c00 100644
--- a/drivers/iio/magnetometer/st_magn_spi.c
+++ b/drivers/iio/magnetometer/st_magn_spi.c
@@ -87,12 +87,12 @@ static int st_magn_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id st_magn_id_table[] = {
-	{ LIS3MDL_MAGN_DEV_NAME },
-	{ LSM303AGR_MAGN_DEV_NAME },
-	{ LIS2MDL_MAGN_DEV_NAME },
-	{ LSM9DS1_MAGN_DEV_NAME },
-	{ IIS2MDC_MAGN_DEV_NAME },
-	{ LSM303C_MAGN_DEV_NAME },
+	{ .name = LIS3MDL_MAGN_DEV_NAME },
+	{ .name = LSM303AGR_MAGN_DEV_NAME },
+	{ .name = LIS2MDL_MAGN_DEV_NAME },
+	{ .name = LSM9DS1_MAGN_DEV_NAME },
+	{ .name = IIS2MDC_MAGN_DEV_NAME },
+	{ .name = LSM303C_MAGN_DEV_NAME },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, st_magn_id_table);
diff --git a/drivers/iio/multiplexer/iio-mux.c b/drivers/iio/multiplexer/iio-mux.c
index 4421dafcf94e8..562457a673826 100644
--- a/drivers/iio/multiplexer/iio-mux.c
+++ b/drivers/iio/multiplexer/iio-mux.c
@@ -463,3 +463,4 @@ module_platform_driver(mux_driver);
 MODULE_DESCRIPTION("IIO multiplexer driver");
 MODULE_AUTHOR("Peter Rosin <peda@axentia.se>");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/iio/orientation/hid-sensor-incl-3d.c b/drivers/iio/orientation/hid-sensor-incl-3d.c
index ea60611192f34..aea28321db95c 100644
--- a/drivers/iio/orientation/hid-sensor-incl-3d.c
+++ b/drivers/iio/orientation/hid-sensor-incl-3d.c
@@ -85,8 +85,7 @@ static const struct iio_chan_spec incl_3d_channels[] = {
 };
 
 /* Adjust channel real bits based on report descriptor */
-static void incl_3d_adjust_channel_bit_mask(struct iio_chan_spec *chan,
-						int size)
+static void incl_3d_adjust_channel_bit_mask(struct iio_chan_spec *chan, int size)
 {
 	chan->scan_type.sign = 's';
 	/* Real storage bits will change based on the report desc. */
@@ -97,9 +96,8 @@ static void incl_3d_adjust_channel_bit_mask(struct iio_chan_spec *chan,
 
 /* Channel read_raw handler */
 static int incl_3d_read_raw(struct iio_dev *indio_dev,
-			      struct iio_chan_spec const *chan,
-			      int *val, int *val2,
-			      long mask)
+			    struct iio_chan_spec const *chan,
+			    int *val, int *val2, long mask)
 {
 	struct incl_3d_state *incl_state = iio_priv(indio_dev);
 	int report_id = -1;
@@ -124,7 +122,7 @@ static int incl_3d_read_raw(struct iio_dev *indio_dev,
 				min < 0);
 		else {
 			hid_sensor_power_state(&incl_state->common_attributes,
-						false);
+					       false);
 			return -EINVAL;
 		}
 		hid_sensor_power_state(&incl_state->common_attributes, false);
@@ -157,10 +155,8 @@ static int incl_3d_read_raw(struct iio_dev *indio_dev,
 
 /* Channel write_raw handler */
 static int incl_3d_write_raw(struct iio_dev *indio_dev,
-			       struct iio_chan_spec const *chan,
-			       int val,
-			       int val2,
-			       long mask)
+			     struct iio_chan_spec const *chan,
+			     int val, int val2, long mask)
 {
 	struct incl_3d_state *incl_state = iio_priv(indio_dev);
 	int ret;
@@ -188,8 +184,7 @@ static const struct iio_info incl_3d_info = {
 
 /* Callback handler to send event after all samples are received and captured */
 static int incl_3d_proc_event(struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
-				void *priv)
+			      u32 usage_id, void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
 	struct incl_3d_state *incl_state = iio_priv(indio_dev);
@@ -211,9 +206,9 @@ static int incl_3d_proc_event(struct hid_sensor_hub_device *hsdev,
 
 /* Capture samples in local storage */
 static int incl_3d_capture_sample(struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
-				size_t raw_len, char *raw_data,
-				void *priv)
+				  u32 usage_id,
+				  size_t raw_len, char *raw_data,
+				  void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
 	struct incl_3d_state *incl_state = iio_priv(indio_dev);
@@ -246,7 +241,7 @@ static int incl_3d_capture_sample(struct hid_sensor_hub_device *hsdev,
 static int incl_3d_parse_report(struct platform_device *pdev,
 				struct hid_sensor_hub_device *hsdev,
 				struct iio_chan_spec *channels,
-				unsigned usage_id,
+				u32 usage_id,
 				struct incl_3d_state *st)
 {
 	int ret;
@@ -306,7 +301,7 @@ static int hid_incl_3d_probe(struct platform_device *pdev)
 
 	indio_dev = devm_iio_device_alloc(&pdev->dev,
 					  sizeof(struct incl_3d_state));
-	if (indio_dev == NULL)
+	if (!indio_dev)
 		return -ENOMEM;
 
 	platform_set_drvdata(pdev, indio_dev);
@@ -349,33 +344,33 @@ static int hid_incl_3d_probe(struct platform_device *pdev)
 	atomic_set(&incl_state->common_attributes.data_ready, 0);
 
 	ret = hid_sensor_setup_trigger(indio_dev, name,
-					&incl_state->common_attributes);
+				       &incl_state->common_attributes);
 	if (ret) {
 		dev_err(&pdev->dev, "trigger setup failed\n");
 		return ret;
 	}
 
-	ret = iio_device_register(indio_dev);
-	if (ret) {
-		dev_err(&pdev->dev, "device register failed\n");
-		goto error_remove_trigger;
-	}
-
 	incl_state->callbacks.send_event = incl_3d_proc_event;
 	incl_state->callbacks.capture_sample = incl_3d_capture_sample;
 	incl_state->callbacks.pdev = pdev;
 	ret = sensor_hub_register_callback(hsdev,
-					HID_USAGE_SENSOR_INCLINOMETER_3D,
-					&incl_state->callbacks);
+					   HID_USAGE_SENSOR_INCLINOMETER_3D,
+					   &incl_state->callbacks);
 	if (ret) {
 		dev_err(&pdev->dev, "callback reg failed\n");
-		goto error_iio_unreg;
+		goto error_remove_trigger;
+	}
+
+	ret = iio_device_register(indio_dev);
+	if (ret) {
+		dev_err(&pdev->dev, "device register failed\n");
+		goto error_remove_callback;
 	}
 
 	return 0;
 
-error_iio_unreg:
-	iio_device_unregister(indio_dev);
+error_remove_callback:
+	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_INCLINOMETER_3D);
 error_remove_trigger:
 	hid_sensor_remove_trigger(indio_dev, &incl_state->common_attributes);
 	return ret;
@@ -388,8 +383,8 @@ static void hid_incl_3d_remove(struct platform_device *pdev)
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
 	struct incl_3d_state *incl_state = iio_priv(indio_dev);
 
-	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_INCLINOMETER_3D);
 	iio_device_unregister(indio_dev);
+	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_INCLINOMETER_3D);
 	hid_sensor_remove_trigger(indio_dev, &incl_state->common_attributes);
 }
 
diff --git a/drivers/iio/orientation/hid-sensor-rotation.c b/drivers/iio/orientation/hid-sensor-rotation.c
index cc3e66dbb90fc..f8c98f866cb0c 100644
--- a/drivers/iio/orientation/hid-sensor-rotation.c
+++ b/drivers/iio/orientation/hid-sensor-rotation.c
@@ -80,9 +80,8 @@ static const struct iio_chan_spec dev_rot_channels[] = {
 
 /* Channel read_raw handler */
 static int dev_rot_read_raw(struct iio_dev *indio_dev,
-				struct iio_chan_spec const *chan,
-				int size, int *vals, int *val_len,
-				long mask)
+			    struct iio_chan_spec const *chan,
+			    int size, int *vals, int *val_len, long mask)
 {
 	struct dev_rot_state *rot_state = iio_priv(indio_dev);
 	struct hid_sensor_hub_device *hsdev = rot_state->common_attributes.hsdev;
@@ -164,10 +163,8 @@ static int dev_rot_read_raw(struct iio_dev *indio_dev,
 
 /* Channel write_raw handler */
 static int dev_rot_write_raw(struct iio_dev *indio_dev,
-			       struct iio_chan_spec const *chan,
-			       int val,
-			       int val2,
-			       long mask)
+			     struct iio_chan_spec const *chan,
+			     int val, int val2, long mask)
 {
 	struct dev_rot_state *rot_state = iio_priv(indio_dev);
 	int ret;
@@ -211,8 +208,7 @@ static const struct iio_info dev_rot_info = {
 
 /* Callback handler to send event after all samples are received and captured */
 static int dev_rot_proc_event(struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
-				void *priv)
+			      u32 usage_id, void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
 	struct dev_rot_state *rot_state = iio_priv(indio_dev);
@@ -244,9 +240,9 @@ static int dev_rot_proc_event(struct hid_sensor_hub_device *hsdev,
 
 /* Capture samples in local storage */
 static int dev_rot_capture_sample(struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
-				size_t raw_len, char *raw_data,
-				void *priv)
+				  u32 usage_id,
+				  size_t raw_len, char *raw_data,
+				  void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
 	struct dev_rot_state *rot_state = iio_priv(indio_dev);
@@ -275,7 +271,7 @@ static int dev_rot_capture_sample(struct hid_sensor_hub_device *hsdev,
 /* Parse report which is specific to an usage id*/
 static int dev_rot_parse_report(struct platform_device *pdev,
 				struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
+				u32 usage_id,
 				struct dev_rot_state *st)
 {
 	int ret;
@@ -313,7 +309,7 @@ static int hid_dev_rot_probe(struct platform_device *pdev)
 
 	indio_dev = devm_iio_device_alloc(&pdev->dev,
 					  sizeof(struct dev_rot_state));
-	if (indio_dev == NULL)
+	if (!indio_dev)
 		return -ENOMEM;
 
 	platform_set_drvdata(pdev, indio_dev);
@@ -361,32 +357,32 @@ static int hid_dev_rot_probe(struct platform_device *pdev)
 	atomic_set(&rot_state->common_attributes.data_ready, 0);
 
 	ret = hid_sensor_setup_trigger(indio_dev, name,
-					&rot_state->common_attributes);
+				       &rot_state->common_attributes);
 	if (ret) {
 		dev_err(&pdev->dev, "trigger setup failed\n");
 		return ret;
 	}
 
-	ret = iio_device_register(indio_dev);
-	if (ret) {
-		dev_err(&pdev->dev, "device register failed\n");
-		goto error_remove_trigger;
-	}
-
 	rot_state->callbacks.send_event = dev_rot_proc_event;
 	rot_state->callbacks.capture_sample = dev_rot_capture_sample;
 	rot_state->callbacks.pdev = pdev;
 	ret = sensor_hub_register_callback(hsdev, hsdev->usage,
-					&rot_state->callbacks);
+					   &rot_state->callbacks);
 	if (ret) {
 		dev_err(&pdev->dev, "callback reg failed\n");
-		goto error_iio_unreg;
+		goto error_remove_trigger;
+	}
+
+	ret = iio_device_register(indio_dev);
+	if (ret) {
+		dev_err(&pdev->dev, "device register failed\n");
+		goto error_remove_callback;
 	}
 
 	return 0;
 
-error_iio_unreg:
-	iio_device_unregister(indio_dev);
+error_remove_callback:
+	sensor_hub_remove_callback(hsdev, hsdev->usage);
 error_remove_trigger:
 	hid_sensor_remove_trigger(indio_dev, &rot_state->common_attributes);
 	return ret;
@@ -399,8 +395,8 @@ static void hid_dev_rot_remove(struct platform_device *pdev)
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
 	struct dev_rot_state *rot_state = iio_priv(indio_dev);
 
-	sensor_hub_remove_callback(hsdev, hsdev->usage);
 	iio_device_unregister(indio_dev);
+	sensor_hub_remove_callback(hsdev, hsdev->usage);
 	hid_sensor_remove_trigger(indio_dev, &rot_state->common_attributes);
 }
 
diff --git a/drivers/iio/position/hid-sensor-custom-intel-hinge.c b/drivers/iio/position/hid-sensor-custom-intel-hinge.c
index bffa8fe4fe446..d275bc1413feb 100644
--- a/drivers/iio/position/hid-sensor-custom-intel-hinge.c
+++ b/drivers/iio/position/hid-sensor-custom-intel-hinge.c
@@ -106,8 +106,8 @@ static void hinge_adjust_channel_realbits(struct iio_chan_spec *channels,
 
 /* Channel read_raw handler */
 static int hinge_read_raw(struct iio_dev *indio_dev,
-			  struct iio_chan_spec const *chan, int *val, int *val2,
-			  long mask)
+			  struct iio_chan_spec const *chan,
+			  int *val, int *val2, long mask)
 {
 	struct hinge_state *st = iio_priv(indio_dev);
 	struct hid_sensor_hub_device *hsdev;
@@ -153,8 +153,8 @@ static int hinge_read_raw(struct iio_dev *indio_dev,
 
 /* Channel write_raw handler */
 static int hinge_write_raw(struct iio_dev *indio_dev,
-			   struct iio_chan_spec const *chan, int val, int val2,
-			   long mask)
+			   struct iio_chan_spec const *chan,
+			   int val, int val2, long mask)
 {
 	struct hinge_state *st = iio_priv(indio_dev);
 
@@ -189,7 +189,7 @@ static const struct iio_info hinge_info = {
  * and captured.
  */
 static int hinge_proc_event(struct hid_sensor_hub_device *hsdev,
-			    unsigned int usage_id, void *priv)
+			    u32 usage_id, void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
 	struct hinge_state *st = iio_priv(indio_dev);
@@ -208,8 +208,9 @@ static int hinge_proc_event(struct hid_sensor_hub_device *hsdev,
 
 /* Capture samples in local storage */
 static int hinge_capture_sample(struct hid_sensor_hub_device *hsdev,
-				unsigned int usage_id, size_t raw_len,
-				char *raw_data, void *priv)
+				u32 usage_id,
+				size_t raw_len, char *raw_data,
+				void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
 	struct hinge_state *st = iio_priv(indio_dev);
@@ -235,7 +236,7 @@ static int hinge_capture_sample(struct hid_sensor_hub_device *hsdev,
 static int hinge_parse_report(struct platform_device *pdev,
 			      struct hid_sensor_hub_device *hsdev,
 			      struct iio_chan_spec *channels,
-			      unsigned int usage_id, struct hinge_state *st)
+			      u32 usage_id, struct hinge_state *st)
 {
 	int ret;
 	int i;
@@ -291,7 +292,7 @@ static int hid_hinge_probe(struct platform_device *pdev)
 	}
 
 	indio_dev->num_channels = ARRAY_SIZE(hinge_channels);
-	indio_dev->channels = devm_kmemdup(&indio_dev->dev, hinge_channels,
+	indio_dev->channels = devm_kmemdup(&pdev->dev, hinge_channels,
 					   sizeof(hinge_channels), GFP_KERNEL);
 	if (!indio_dev->channels)
 		return -ENOMEM;
diff --git a/drivers/iio/potentiometer/max5481.c b/drivers/iio/potentiometer/max5481.c
index ddf54a1df976a..ec1b7bdc706b5 100644
--- a/drivers/iio/potentiometer/max5481.c
+++ b/drivers/iio/potentiometer/max5481.c
@@ -169,10 +169,10 @@ static int max5481_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id max5481_id_table[] = {
-	{ "max5481", max5481 },
-	{ "max5482", max5482 },
-	{ "max5483", max5483 },
-	{ "max5484", max5484 },
+	{ .name = "max5481", .driver_data = max5481 },
+	{ .name = "max5482", .driver_data = max5482 },
+	{ .name = "max5483", .driver_data = max5483 },
+	{ .name = "max5484", .driver_data = max5484 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, max5481_id_table);
diff --git a/drivers/iio/potentiometer/max5487.c b/drivers/iio/potentiometer/max5487.c
index 9541695af474d..1f0d0056f4128 100644
--- a/drivers/iio/potentiometer/max5487.c
+++ b/drivers/iio/potentiometer/max5487.c
@@ -125,9 +125,9 @@ static void max5487_spi_remove(struct spi_device *spi)
 }
 
 static const struct spi_device_id max5487_id[] = {
-	{ "MAX5487", 10 },
-	{ "MAX5488", 50 },
-	{ "MAX5489", 100 },
+	{ .name = "MAX5487", .driver_data = 10 },
+	{ .name = "MAX5488", .driver_data = 50 },
+	{ .name = "MAX5489", .driver_data = 100 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, max5487_id);
diff --git a/drivers/iio/potentiometer/mcp41010.c b/drivers/iio/potentiometer/mcp41010.c
index ed07642312242..cfd5d0f6d3689 100644
--- a/drivers/iio/potentiometer/mcp41010.c
+++ b/drivers/iio/potentiometer/mcp41010.c
@@ -175,12 +175,12 @@ static const struct of_device_id mcp41010_match[] = {
 MODULE_DEVICE_TABLE(of, mcp41010_match);
 
 static const struct spi_device_id mcp41010_id[] = {
-	{ "mcp41010", MCP41010 },
-	{ "mcp41050", MCP41050 },
-	{ "mcp41100", MCP41100 },
-	{ "mcp42010", MCP42010 },
-	{ "mcp42050", MCP42050 },
-	{ "mcp42100", MCP42100 },
+	{ .name = "mcp41010", .driver_data = MCP41010 },
+	{ .name = "mcp41050", .driver_data = MCP41050 },
+	{ .name = "mcp41100", .driver_data = MCP41100 },
+	{ .name = "mcp42010", .driver_data = MCP42010 },
+	{ .name = "mcp42050", .driver_data = MCP42050 },
+	{ .name = "mcp42100", .driver_data = MCP42100 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, mcp41010_id);
diff --git a/drivers/iio/potentiometer/mcp4131.c b/drivers/iio/potentiometer/mcp4131.c
index dad09ce6a75d4..fadf9fcc18174 100644
--- a/drivers/iio/potentiometer/mcp4131.c
+++ b/drivers/iio/potentiometer/mcp4131.c
@@ -406,70 +406,70 @@ static const struct of_device_id mcp4131_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, mcp4131_dt_ids);
 
 static const struct spi_device_id mcp4131_id[] = {
-	{ "mcp4131-502", MCP413x_502 },
-	{ "mcp4131-103", MCP413x_103 },
-	{ "mcp4131-503", MCP413x_503 },
-	{ "mcp4131-104", MCP413x_104 },
-	{ "mcp4132-502", MCP413x_502 },
-	{ "mcp4132-103", MCP413x_103 },
-	{ "mcp4132-503", MCP413x_503 },
-	{ "mcp4132-104", MCP413x_104 },
-	{ "mcp4141-502", MCP414x_502 },
-	{ "mcp4141-103", MCP414x_103 },
-	{ "mcp4141-503", MCP414x_503 },
-	{ "mcp4141-104", MCP414x_104 },
-	{ "mcp4142-502", MCP414x_502 },
-	{ "mcp4142-103", MCP414x_103 },
-	{ "mcp4142-503", MCP414x_503 },
-	{ "mcp4142-104", MCP414x_104 },
-	{ "mcp4151-502", MCP415x_502 },
-	{ "mcp4151-103", MCP415x_103 },
-	{ "mcp4151-503", MCP415x_503 },
-	{ "mcp4151-104", MCP415x_104 },
-	{ "mcp4152-502", MCP415x_502 },
-	{ "mcp4152-103", MCP415x_103 },
-	{ "mcp4152-503", MCP415x_503 },
-	{ "mcp4152-104", MCP415x_104 },
-	{ "mcp4161-502", MCP416x_502 },
-	{ "mcp4161-103", MCP416x_103 },
-	{ "mcp4161-503", MCP416x_503 },
-	{ "mcp4161-104", MCP416x_104 },
-	{ "mcp4162-502", MCP416x_502 },
-	{ "mcp4162-103", MCP416x_103 },
-	{ "mcp4162-503", MCP416x_503 },
-	{ "mcp4162-104", MCP416x_104 },
-	{ "mcp4231-502", MCP423x_502 },
-	{ "mcp4231-103", MCP423x_103 },
-	{ "mcp4231-503", MCP423x_503 },
-	{ "mcp4231-104", MCP423x_104 },
-	{ "mcp4232-502", MCP423x_502 },
-	{ "mcp4232-103", MCP423x_103 },
-	{ "mcp4232-503", MCP423x_503 },
-	{ "mcp4232-104", MCP423x_104 },
-	{ "mcp4241-502", MCP424x_502 },
-	{ "mcp4241-103", MCP424x_103 },
-	{ "mcp4241-503", MCP424x_503 },
-	{ "mcp4241-104", MCP424x_104 },
-	{ "mcp4242-502", MCP424x_502 },
-	{ "mcp4242-103", MCP424x_103 },
-	{ "mcp4242-503", MCP424x_503 },
-	{ "mcp4242-104", MCP424x_104 },
-	{ "mcp4251-502", MCP425x_502 },
-	{ "mcp4251-103", MCP425x_103 },
-	{ "mcp4251-503", MCP425x_503 },
-	{ "mcp4251-104", MCP425x_104 },
-	{ "mcp4252-502", MCP425x_502 },
-	{ "mcp4252-103", MCP425x_103 },
-	{ "mcp4252-503", MCP425x_503 },
-	{ "mcp4252-104", MCP425x_104 },
-	{ "mcp4261-502", MCP426x_502 },
-	{ "mcp4261-103", MCP426x_103 },
-	{ "mcp4261-503", MCP426x_503 },
-	{ "mcp4261-104", MCP426x_104 },
-	{ "mcp4262-502", MCP426x_502 },
-	{ "mcp4262-103", MCP426x_103 },
-	{ "mcp4262-503", MCP426x_503 },
-	{ "mcp4262-104", MCP426x_104 },
+	{ .name = "mcp4131-502", .driver_data = MCP413x_502 },
+	{ .name = "mcp4131-103", .driver_data = MCP413x_103 },
+	{ .name = "mcp4131-503", .driver_data = MCP413x_503 },
+	{ .name = "mcp4131-104", .driver_data = MCP413x_104 },
+	{ .name = "mcp4132-502", .driver_data = MCP413x_502 },
+	{ .name = "mcp4132-103", .driver_data = MCP413x_103 },
+	{ .name = "mcp4132-503", .driver_data = MCP413x_503 },
+	{ .name = "mcp4132-104", .driver_data = MCP413x_104 },
+	{ .name = "mcp4141-502", .driver_data = MCP414x_502 },
+	{ .name = "mcp4141-103", .driver_data = MCP414x_103 },
+	{ .name = "mcp4141-503", .driver_data = MCP414x_503 },
+	{ .name = "mcp4141-104", .driver_data = MCP414x_104 },
+	{ .name = "mcp4142-502", .driver_data = MCP414x_502 },
+	{ .name = "mcp4142-103", .driver_data = MCP414x_103 },
+	{ .name = "mcp4142-503", .driver_data = MCP414x_503 },
+	{ .name = "mcp4142-104", .driver_data = MCP414x_104 },
+	{ .name = "mcp4151-502", .driver_data = MCP415x_502 },
+	{ .name = "mcp4151-103", .driver_data = MCP415x_103 },
+	{ .name = "mcp4151-503", .driver_data = MCP415x_503 },
+	{ .name = "mcp4151-104", .driver_data = MCP415x_104 },
+	{ .name = "mcp4152-502", .driver_data = MCP415x_502 },
+	{ .name = "mcp4152-103", .driver_data = MCP415x_103 },
+	{ .name = "mcp4152-503", .driver_data = MCP415x_503 },
+	{ .name = "mcp4152-104", .driver_data = MCP415x_104 },
+	{ .name = "mcp4161-502", .driver_data = MCP416x_502 },
+	{ .name = "mcp4161-103", .driver_data = MCP416x_103 },
+	{ .name = "mcp4161-503", .driver_data = MCP416x_503 },
+	{ .name = "mcp4161-104", .driver_data = MCP416x_104 },
+	{ .name = "mcp4162-502", .driver_data = MCP416x_502 },
+	{ .name = "mcp4162-103", .driver_data = MCP416x_103 },
+	{ .name = "mcp4162-503", .driver_data = MCP416x_503 },
+	{ .name = "mcp4162-104", .driver_data = MCP416x_104 },
+	{ .name = "mcp4231-502", .driver_data = MCP423x_502 },
+	{ .name = "mcp4231-103", .driver_data = MCP423x_103 },
+	{ .name = "mcp4231-503", .driver_data = MCP423x_503 },
+	{ .name = "mcp4231-104", .driver_data = MCP423x_104 },
+	{ .name = "mcp4232-502", .driver_data = MCP423x_502 },
+	{ .name = "mcp4232-103", .driver_data = MCP423x_103 },
+	{ .name = "mcp4232-503", .driver_data = MCP423x_503 },
+	{ .name = "mcp4232-104", .driver_data = MCP423x_104 },
+	{ .name = "mcp4241-502", .driver_data = MCP424x_502 },
+	{ .name = "mcp4241-103", .driver_data = MCP424x_103 },
+	{ .name = "mcp4241-503", .driver_data = MCP424x_503 },
+	{ .name = "mcp4241-104", .driver_data = MCP424x_104 },
+	{ .name = "mcp4242-502", .driver_data = MCP424x_502 },
+	{ .name = "mcp4242-103", .driver_data = MCP424x_103 },
+	{ .name = "mcp4242-503", .driver_data = MCP424x_503 },
+	{ .name = "mcp4242-104", .driver_data = MCP424x_104 },
+	{ .name = "mcp4251-502", .driver_data = MCP425x_502 },
+	{ .name = "mcp4251-103", .driver_data = MCP425x_103 },
+	{ .name = "mcp4251-503", .driver_data = MCP425x_503 },
+	{ .name = "mcp4251-104", .driver_data = MCP425x_104 },
+	{ .name = "mcp4252-502", .driver_data = MCP425x_502 },
+	{ .name = "mcp4252-103", .driver_data = MCP425x_103 },
+	{ .name = "mcp4252-503", .driver_data = MCP425x_503 },
+	{ .name = "mcp4252-104", .driver_data = MCP425x_104 },
+	{ .name = "mcp4261-502", .driver_data = MCP426x_502 },
+	{ .name = "mcp4261-103", .driver_data = MCP426x_103 },
+	{ .name = "mcp4261-503", .driver_data = MCP426x_503 },
+	{ .name = "mcp4261-104", .driver_data = MCP426x_104 },
+	{ .name = "mcp4262-502", .driver_data = MCP426x_502 },
+	{ .name = "mcp4262-103", .driver_data = MCP426x_103 },
+	{ .name = "mcp4262-503", .driver_data = MCP426x_503 },
+	{ .name = "mcp4262-104", .driver_data = MCP426x_104 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, mcp4131_id);
diff --git a/drivers/iio/potentiometer/x9250.c b/drivers/iio/potentiometer/x9250.c
index 7353484926997..974e4447fc2cc 100644
--- a/drivers/iio/potentiometer/x9250.c
+++ b/drivers/iio/potentiometer/x9250.c
@@ -198,8 +198,8 @@ static const struct of_device_id x9250_of_match[] = {
 MODULE_DEVICE_TABLE(of, x9250_of_match);
 
 static const struct spi_device_id x9250_id_table[] = {
-	{ "x9250t", (kernel_ulong_t)&x9250_cfg[X9250T] },
-	{ "x9250u", (kernel_ulong_t)&x9250_cfg[X9250U] },
+	{ .name = "x9250t", .driver_data = (kernel_ulong_t)&x9250_cfg[X9250T] },
+	{ .name = "x9250u", .driver_data = (kernel_ulong_t)&x9250_cfg[X9250U] },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, x9250_id_table);
diff --git a/drivers/iio/potentiostat/lmp91000.c b/drivers/iio/potentiostat/lmp91000.c
index 1984d990438c2..00cb3fea97ca2 100644
--- a/drivers/iio/potentiostat/lmp91000.c
+++ b/drivers/iio/potentiostat/lmp91000.c
@@ -422,3 +422,4 @@ module_i2c_driver(lmp91000_driver);
 MODULE_AUTHOR("Matt Ranostay <matt.ranostay@konsulko.com>");
 MODULE_DESCRIPTION("LMP91000 digital potentiostat");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/iio/pressure/abp2030pa_spi.c b/drivers/iio/pressure/abp2030pa_spi.c
index 8bc59eb499aa0..60d0785748276 100644
--- a/drivers/iio/pressure/abp2030pa_spi.c
+++ b/drivers/iio/pressure/abp2030pa_spi.c
@@ -45,7 +45,7 @@ static const struct of_device_id abp2_spi_match[] = {
 MODULE_DEVICE_TABLE(of, abp2_spi_match);
 
 static const struct spi_device_id abp2_spi_id[] = {
-	{ "abp2030pa" },
+	{ .name = "abp2030pa" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, abp2_spi_id);
diff --git a/drivers/iio/pressure/bmp280-core.c b/drivers/iio/pressure/bmp280-core.c
index 990340a9b10c5..c89153dcc3230 100644
--- a/drivers/iio/pressure/bmp280-core.c
+++ b/drivers/iio/pressure/bmp280-core.c
@@ -750,7 +750,10 @@ static int bmp280_read_raw(struct iio_dev *indio_dev,
 	struct bmp280_data *data = iio_priv(indio_dev);
 	int ret;
 
-	pm_runtime_get_sync(data->dev);
+	ret = pm_runtime_resume_and_get(data->dev);
+	if (ret < 0)
+		return ret;
+
 	ret = bmp280_read_raw_impl(indio_dev, chan, val, val2, mask);
 	pm_runtime_put_autosuspend(data->dev);
 
@@ -924,7 +927,10 @@ static int bmp280_write_raw(struct iio_dev *indio_dev,
 	struct bmp280_data *data = iio_priv(indio_dev);
 	int ret;
 
-	pm_runtime_get_sync(data->dev);
+	ret = pm_runtime_resume_and_get(data->dev);
+	if (ret < 0)
+		return ret;
+
 	ret = bmp280_write_raw_impl(indio_dev, chan, val, val2, mask);
 	pm_runtime_put_autosuspend(data->dev);
 
@@ -2254,7 +2260,10 @@ static int bmp580_nvmem_read(void *priv, unsigned int offset, void *val,
 	struct bmp280_data *data = priv;
 	int ret;
 
-	pm_runtime_get_sync(data->dev);
+	ret = pm_runtime_resume_and_get(data->dev);
+	if (ret < 0)
+		return ret;
+
 	ret = bmp580_nvmem_read_impl(priv, offset, val, bytes);
 	pm_runtime_put_autosuspend(data->dev);
 
@@ -2328,7 +2337,10 @@ static int bmp580_nvmem_write(void *priv, unsigned int offset, void *val,
 	struct bmp280_data *data = priv;
 	int ret;
 
-	pm_runtime_get_sync(data->dev);
+	ret = pm_runtime_resume_and_get(data->dev);
+	if (ret < 0)
+		return ret;
+
 	ret = bmp580_nvmem_write_impl(priv, offset, val, bytes);
 	pm_runtime_put_autosuspend(data->dev);
 
@@ -3108,11 +3120,17 @@ EXPORT_SYMBOL_NS(bmp085_chip_info, "IIO_BMP280");
 static int bmp280_buffer_preenable(struct iio_dev *indio_dev)
 {
 	struct bmp280_data *data = iio_priv(indio_dev);
+	int ret;
 
-	pm_runtime_get_sync(data->dev);
-	data->chip_info->set_mode(data, BMP280_NORMAL);
+	ret = pm_runtime_resume_and_get(data->dev);
+	if (ret < 0)
+		return ret;
 
-	return 0;
+	ret = data->chip_info->set_mode(data, BMP280_NORMAL);
+	if (ret)
+		pm_runtime_put_autosuspend(data->dev);
+
+	return ret;
 }
 
 static int bmp280_buffer_postdisable(struct iio_dev *indio_dev)
diff --git a/drivers/iio/pressure/bmp280-spi.c b/drivers/iio/pressure/bmp280-spi.c
index 04bf2f5be5b11..a02a621b80150 100644
--- a/drivers/iio/pressure/bmp280-spi.c
+++ b/drivers/iio/pressure/bmp280-spi.c
@@ -118,13 +118,13 @@ static const struct of_device_id bmp280_of_spi_match[] = {
 MODULE_DEVICE_TABLE(of, bmp280_of_spi_match);
 
 static const struct spi_device_id bmp280_spi_id[] = {
-	{ "bmp085", (kernel_ulong_t)&bmp085_chip_info },
-	{ "bmp180", (kernel_ulong_t)&bmp180_chip_info },
-	{ "bmp181", (kernel_ulong_t)&bmp180_chip_info },
-	{ "bmp280", (kernel_ulong_t)&bmp280_chip_info },
-	{ "bme280", (kernel_ulong_t)&bme280_chip_info },
-	{ "bmp380", (kernel_ulong_t)&bmp380_chip_info },
-	{ "bmp580", (kernel_ulong_t)&bmp580_chip_info },
+	{ .name = "bmp085", .driver_data = (kernel_ulong_t)&bmp085_chip_info },
+	{ .name = "bmp180", .driver_data = (kernel_ulong_t)&bmp180_chip_info },
+	{ .name = "bmp181", .driver_data = (kernel_ulong_t)&bmp180_chip_info },
+	{ .name = "bmp280", .driver_data = (kernel_ulong_t)&bmp280_chip_info },
+	{ .name = "bme280", .driver_data = (kernel_ulong_t)&bme280_chip_info },
+	{ .name = "bmp380", .driver_data = (kernel_ulong_t)&bmp380_chip_info },
+	{ .name = "bmp580", .driver_data = (kernel_ulong_t)&bmp580_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, bmp280_spi_id);
diff --git a/drivers/iio/pressure/hid-sensor-press.c b/drivers/iio/pressure/hid-sensor-press.c
index ae499d197555a..e688b0776547f 100644
--- a/drivers/iio/pressure/hid-sensor-press.c
+++ b/drivers/iio/pressure/hid-sensor-press.c
@@ -55,9 +55,8 @@ static const struct iio_chan_spec press_channels[] = {
 
 /* Channel read_raw handler */
 static int press_read_raw(struct iio_dev *indio_dev,
-			      struct iio_chan_spec const *chan,
-			      int *val, int *val2,
-			      long mask)
+			  struct iio_chan_spec const *chan,
+			  int *val, int *val2, long mask)
 {
 	struct press_state *press_state = iio_priv(indio_dev);
 	int report_id = -1;
@@ -81,7 +80,7 @@ static int press_read_raw(struct iio_dev *indio_dev,
 		}
 		if (report_id >= 0) {
 			hid_sensor_power_state(&press_state->common_attributes,
-						true);
+					       true);
 			*val = sensor_hub_input_attr_get_raw_value(
 				press_state->common_attributes.hsdev,
 				HID_USAGE_SENSOR_PRESSURE, address,
@@ -89,7 +88,7 @@ static int press_read_raw(struct iio_dev *indio_dev,
 				SENSOR_HUB_SYNC,
 				min < 0);
 			hid_sensor_power_state(&press_state->common_attributes,
-						false);
+					       false);
 		} else {
 			*val = 0;
 			return -EINVAL;
@@ -123,10 +122,8 @@ static int press_read_raw(struct iio_dev *indio_dev,
 
 /* Channel write_raw handler */
 static int press_write_raw(struct iio_dev *indio_dev,
-			       struct iio_chan_spec const *chan,
-			       int val,
-			       int val2,
-			       long mask)
+			   struct iio_chan_spec const *chan,
+			   int val, int val2, long mask)
 {
 	struct press_state *press_state = iio_priv(indio_dev);
 	int ret = 0;
@@ -154,8 +151,7 @@ static const struct iio_info press_info = {
 
 /* Callback handler to send event after all samples are received and captured */
 static int press_proc_event(struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
-				void *priv)
+			    u32 usage_id, void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
 	struct press_state *press_state = iio_priv(indio_dev);
@@ -175,7 +171,7 @@ static int press_proc_event(struct hid_sensor_hub_device *hsdev,
 
 /* Capture samples in local storage */
 static int press_capture_sample(struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
+				u32 usage_id,
 				size_t raw_len, char *raw_data,
 				void *priv)
 {
@@ -201,10 +197,10 @@ static int press_capture_sample(struct hid_sensor_hub_device *hsdev,
 
 /* Parse report which is specific to an usage id*/
 static int press_parse_report(struct platform_device *pdev,
-				struct hid_sensor_hub_device *hsdev,
-				struct iio_chan_spec *channels,
-				unsigned usage_id,
-				struct press_state *st)
+			      struct hid_sensor_hub_device *hsdev,
+			      struct iio_chan_spec *channels,
+			      u32 usage_id,
+			      struct press_state *st)
 {
 	int ret;
 
@@ -221,7 +217,7 @@ static int press_parse_report(struct platform_device *pdev,
 	};
 
 	dev_dbg(&pdev->dev, "press %x:%x\n", st->press_attr.index,
-			st->press_attr.report_id);
+		st->press_attr.report_id);
 
 	st->scale_precision = hid_sensor_format_scale(
 				HID_USAGE_SENSOR_PRESSURE,
@@ -240,8 +236,7 @@ static int hid_press_probe(struct platform_device *pdev)
 	struct iio_dev *indio_dev;
 	struct press_state *press_state;
 
-	indio_dev = devm_iio_device_alloc(&pdev->dev,
-				sizeof(struct press_state));
+	indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(struct press_state));
 	if (!indio_dev)
 		return -ENOMEM;
 	platform_set_drvdata(pdev, indio_dev);
@@ -284,32 +279,32 @@ static int hid_press_probe(struct platform_device *pdev)
 	atomic_set(&press_state->common_attributes.data_ready, 0);
 
 	ret = hid_sensor_setup_trigger(indio_dev, name,
-				&press_state->common_attributes);
+				       &press_state->common_attributes);
 	if (ret) {
 		dev_err(&pdev->dev, "trigger setup failed\n");
 		return ret;
 	}
 
-	ret = iio_device_register(indio_dev);
-	if (ret) {
-		dev_err(&pdev->dev, "device register failed\n");
-		goto error_remove_trigger;
-	}
-
 	press_state->callbacks.send_event = press_proc_event;
 	press_state->callbacks.capture_sample = press_capture_sample;
 	press_state->callbacks.pdev = pdev;
 	ret = sensor_hub_register_callback(hsdev, HID_USAGE_SENSOR_PRESSURE,
-					&press_state->callbacks);
+					   &press_state->callbacks);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "callback reg failed\n");
-		goto error_iio_unreg;
+		goto error_remove_trigger;
+	}
+
+	ret = iio_device_register(indio_dev);
+	if (ret) {
+		dev_err(&pdev->dev, "device register failed\n");
+		goto error_remove_callback;
 	}
 
 	return ret;
 
-error_iio_unreg:
-	iio_device_unregister(indio_dev);
+error_remove_callback:
+	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_PRESSURE);
 error_remove_trigger:
 	hid_sensor_remove_trigger(indio_dev, &press_state->common_attributes);
 	return ret;
@@ -322,8 +317,8 @@ static void hid_press_remove(struct platform_device *pdev)
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
 	struct press_state *press_state = iio_priv(indio_dev);
 
-	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_PRESSURE);
 	iio_device_unregister(indio_dev);
+	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_PRESSURE);
 	hid_sensor_remove_trigger(indio_dev, &press_state->common_attributes);
 }
 
diff --git a/drivers/iio/pressure/hsc030pa_spi.c b/drivers/iio/pressure/hsc030pa_spi.c
index be58e550cc12b..02021b6f799a2 100644
--- a/drivers/iio/pressure/hsc030pa_spi.c
+++ b/drivers/iio/pressure/hsc030pa_spi.c
@@ -39,7 +39,7 @@ static const struct of_device_id hsc_spi_match[] = {
 MODULE_DEVICE_TABLE(of, hsc_spi_match);
 
 static const struct spi_device_id hsc_spi_id[] = {
-	{ "hsc030pa" },
+	{ .name = "hsc030pa" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, hsc_spi_id);
diff --git a/drivers/iio/pressure/mpl115_spi.c b/drivers/iio/pressure/mpl115_spi.c
index 4e1d24beff949..3cff616e8d2f4 100644
--- a/drivers/iio/pressure/mpl115_spi.c
+++ b/drivers/iio/pressure/mpl115_spi.c
@@ -84,7 +84,7 @@ static int mpl115_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id mpl115_spi_ids[] = {
-	{ "mpl115", 0 },
+	{ .name = "mpl115" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, mpl115_spi_ids);
diff --git a/drivers/iio/pressure/mprls0025pa_spi.c b/drivers/iio/pressure/mprls0025pa_spi.c
index 23f47c04ed45e..6270370383c31 100644
--- a/drivers/iio/pressure/mprls0025pa_spi.c
+++ b/drivers/iio/pressure/mprls0025pa_spi.c
@@ -59,7 +59,7 @@ static const struct of_device_id mpr_spi_match[] = {
 MODULE_DEVICE_TABLE(of, mpr_spi_match);
 
 static const struct spi_device_id mpr_spi_id[] = {
-	{ "mprls0025pa" },
+	{ .name = "mprls0025pa" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, mpr_spi_id);
diff --git a/drivers/iio/pressure/ms5611_spi.c b/drivers/iio/pressure/ms5611_spi.c
index c41aaa6244cd7..7133e6744e3f3 100644
--- a/drivers/iio/pressure/ms5611_spi.c
+++ b/drivers/iio/pressure/ms5611_spi.c
@@ -113,8 +113,8 @@ static const struct of_device_id ms5611_spi_matches[] = {
 MODULE_DEVICE_TABLE(of, ms5611_spi_matches);
 
 static const struct spi_device_id ms5611_id[] = {
-	{ "ms5611", MS5611 },
-	{ "ms5607", MS5607 },
+	{ .name = "ms5611", .driver_data = MS5611 },
+	{ .name = "ms5607", .driver_data = MS5607 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ms5611_id);
diff --git a/drivers/iio/pressure/rohm-bm1390.c b/drivers/iio/pressure/rohm-bm1390.c
index b3be9de036788..9d72ae64126c6 100644
--- a/drivers/iio/pressure/rohm-bm1390.c
+++ b/drivers/iio/pressure/rohm-bm1390.c
@@ -621,17 +621,15 @@ static const struct iio_buffer_setup_ops bm1390_buffer_ops = {
 	.predisable = bm1390_buffer_predisable,
 };
 
-static irqreturn_t bm1390_trigger_handler(int irq, void *p)
+static bool bm1390_handle_trigger(struct iio_dev *idev)
 {
-	struct iio_poll_func *pf = p;
-	struct iio_dev *idev = pf->indio_dev;
 	struct bm1390_data *data = iio_priv(idev);
 	int ret, status;
 
 	/* DRDY is acked by reading status reg */
 	ret = regmap_read(data->regmap, BM1390_REG_STATUS, &status);
 	if (ret || !status)
-		return IRQ_NONE;
+		return false;
 
 	dev_dbg(data->dev, "DRDY trig status 0x%x\n", status);
 
@@ -639,7 +637,7 @@ static irqreturn_t bm1390_trigger_handler(int irq, void *p)
 		ret = bm1390_pressure_read(data, &data->buf.pressure);
 		if (ret) {
 			dev_warn(data->dev, "sample read failed %d\n", ret);
-			return IRQ_NONE;
+			return false;
 		}
 	}
 
@@ -648,15 +646,26 @@ static irqreturn_t bm1390_trigger_handler(int irq, void *p)
 				       &data->buf.temp, sizeof(data->buf.temp));
 		if (ret) {
 			dev_warn(data->dev, "temp read failed %d\n", ret);
-			return IRQ_HANDLED;
+			return true;
 		}
 	}
 
 	iio_push_to_buffers_with_ts(idev, &data->buf, sizeof(data->buf),
 				    data->timestamp);
+
+	return true;
+}
+
+static irqreturn_t bm1390_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *idev = pf->indio_dev;
+	bool result;
+
+	result = bm1390_handle_trigger(idev);
 	iio_trigger_notify_done(idev->trig);
 
-	return IRQ_HANDLED;
+	return IRQ_RETVAL(result);
 }
 
 /* Get timestamps and wake the thread if we need to read data */
diff --git a/drivers/iio/pressure/st_pressure_spi.c b/drivers/iio/pressure/st_pressure_spi.c
index d843186b44afc..8add0c392c6ab 100644
--- a/drivers/iio/pressure/st_pressure_spi.c
+++ b/drivers/iio/pressure/st_pressure_spi.c
@@ -93,18 +93,18 @@ static int st_press_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id st_press_id_table[] = {
-	{ LPS001WP_PRESS_DEV_NAME },
-	{ LPS25H_PRESS_DEV_NAME },
-	{ LPS331AP_PRESS_DEV_NAME },
-	{ LPS22HB_PRESS_DEV_NAME },
-	{ LPS33HW_PRESS_DEV_NAME },
-	{ LPS35HW_PRESS_DEV_NAME },
-	{ LPS22HH_PRESS_DEV_NAME },
-	{ LPS22DF_PRESS_DEV_NAME },
-	{ "lps001wp-press" },
-	{ "lps25h-press", },
-	{ "lps331ap-press" },
-	{ "lps22hb-press" },
+	{ .name = LPS001WP_PRESS_DEV_NAME },
+	{ .name = LPS25H_PRESS_DEV_NAME },
+	{ .name = LPS331AP_PRESS_DEV_NAME },
+	{ .name = LPS22HB_PRESS_DEV_NAME },
+	{ .name = LPS33HW_PRESS_DEV_NAME },
+	{ .name = LPS35HW_PRESS_DEV_NAME },
+	{ .name = LPS22HH_PRESS_DEV_NAME },
+	{ .name = LPS22DF_PRESS_DEV_NAME },
+	{ .name = "lps001wp-press" },
+	{ .name = "lps25h-press" },
+	{ .name = "lps331ap-press" },
+	{ .name = "lps22hb-press" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, st_press_id_table);
diff --git a/drivers/iio/pressure/zpa2326_spi.c b/drivers/iio/pressure/zpa2326_spi.c
index 73e37a77c933f..dfbb74a58dc04 100644
--- a/drivers/iio/pressure/zpa2326_spi.c
+++ b/drivers/iio/pressure/zpa2326_spi.c
@@ -60,7 +60,7 @@ static void zpa2326_remove_spi(struct spi_device *spi)
 }
 
 static const struct spi_device_id zpa2326_spi_ids[] = {
-	{ "zpa2326", 0 },
+	{ .name = "zpa2326" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, zpa2326_spi_ids);
diff --git a/drivers/iio/proximity/as3935.c b/drivers/iio/proximity/as3935.c
index 3406232822cbc..93833037bc944 100644
--- a/drivers/iio/proximity/as3935.c
+++ b/drivers/iio/proximity/as3935.c
@@ -448,7 +448,7 @@ static const struct of_device_id as3935_of_match[] = {
 MODULE_DEVICE_TABLE(of, as3935_of_match);
 
 static const struct spi_device_id as3935_id[] = {
-	{"as3935", 0},
+	{ .name = "as3935" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, as3935_id);
diff --git a/drivers/iio/proximity/vl53l1x-i2c.c b/drivers/iio/proximity/vl53l1x-i2c.c
index dc4ffbc95d1f9..7ca0f2e652f4c 100644
--- a/drivers/iio/proximity/vl53l1x-i2c.c
+++ b/drivers/iio/proximity/vl53l1x-i2c.c
@@ -42,6 +42,7 @@
 #define VL53L1X_REG_SOFT_RESET						0x0000
 #define VL53L1X_REG_VHV_CONFIG__TIMEOUT_MACROP_LOOP_BOUND		0x0008
 #define VL53L1X_REG_VHV_CONFIG__INIT					0x000B
+#define VL53L1X_REG_DEFAULT_CONFIG					0x002D
 #define VL53L1X_REG_GPIO_HV_MUX__CTRL					0x0030
 #define VL53L1X_REG_GPIO__TIO_HV_STATUS					0x0031
 #define VL53L1X_REG_SYSTEM__INTERRUPT_CONFIG_GPIO			0x0046
@@ -63,7 +64,6 @@
 #define VL53L1X_REG_RESULT__OSC_CALIBRATE_VAL				0x00DE
 #define VL53L1X_REG_FIRMWARE__SYSTEM_STATUS				0x00E5
 #define VL53L1X_REG_IDENTIFICATION__MODEL_ID				0x010F
-#define VL53L1X_REG_DEFAULT_CONFIG					0x002D
 
 #define VL53L1X_MODEL_ID_VAL		0xEACC
 
@@ -81,6 +81,9 @@
 
 #define VL53L1X_OSC_CALIBRATE_MASK	GENMASK(9, 0)
 
+#define VL53L1X_FIRMWARE__SYSTEM_STATUS_BOOTED	BIT(0)
+#define VL53L1X_GPIO__TIO_HV_STATUS_DATA_READY	BIT(0)
+
 /* Inter-measurement period uses PLL divider with 1.075 oscillator correction */
 static const struct u32_fract vl53l1x_osc_correction = {
 	.numerator = 1075,
@@ -190,6 +193,17 @@ static int vl53l1x_stop_ranging(struct vl53l1x_data *data)
 			    VL53L1X_MODE_START_STOP);
 }
 
+static int vl53l1x_wait_data_ready(struct vl53l1x_data *data)
+{
+	unsigned int val;
+
+	/* 1ms poll, 1s timeout covers max timing budgets (per ST Ultra Lite Driver) */
+	return regmap_read_poll_timeout(data->regmap,
+		VL53L1X_REG_GPIO__TIO_HV_STATUS, val,
+		(val & VL53L1X_GPIO__TIO_HV_STATUS_DATA_READY) != data->gpio_polarity,
+		1 * USEC_PER_MSEC, 1 * USEC_PER_SEC);
+}
+
 /*
  * Default configuration blob from ST's VL53L1X Ultra Lite Driver
  * (STSW-IMG009).
@@ -229,10 +243,9 @@ static int vl53l1x_chip_init(struct vl53l1x_data *data)
 	}
 
 	ret = regmap_read_poll_timeout(data->regmap,
-				       VL53L1X_REG_FIRMWARE__SYSTEM_STATUS, val,
-				       val & BIT(0),
-				       1 * USEC_PER_MSEC,
-				       100 * USEC_PER_MSEC);
+				       VL53L1X_REG_FIRMWARE__SYSTEM_STATUS,
+				       val, val & VL53L1X_FIRMWARE__SYSTEM_STATUS_BOOTED,
+				       1 * USEC_PER_MSEC, 100 * USEC_PER_MSEC);
 	if (ret)
 		return dev_err_probe(dev, ret, "firmware boot timeout\n");
 
@@ -260,12 +273,7 @@ static int vl53l1x_chip_init(struct vl53l1x_data *data)
 	if (ret)
 		return ret;
 
-	/* 1ms poll, 1s timeout covers max timing budgets (per ST Ultra Lite Driver) */
-	ret = regmap_read_poll_timeout(data->regmap,
-				       VL53L1X_REG_GPIO__TIO_HV_STATUS, val,
-				       (val & 1) != data->gpio_polarity,
-				       1 * USEC_PER_MSEC,
-				       1000 * USEC_PER_MSEC);
+	ret = vl53l1x_wait_data_ready(data);
 	if (ret)
 		return ret;
 
@@ -460,14 +468,7 @@ static int vl53l1x_read_proximity(struct vl53l1x_data *data, int *val)
 		if (!wait_for_completion_timeout(&data->completion, HZ))
 			return -ETIMEDOUT;
 	} else {
-		unsigned int rdy;
-
-		/* 1ms poll, 1s timeout covers max timing budgets (per ST Ultra Lite Driver) */
-		ret = regmap_read_poll_timeout(data->regmap,
-					       VL53L1X_REG_GPIO__TIO_HV_STATUS, rdy,
-					       (rdy & 1) != data->gpio_polarity,
-					       1 * USEC_PER_MSEC,
-					       1000 * USEC_PER_MSEC);
+		ret = vl53l1x_wait_data_ready(data);
 		if (ret)
 			return ret;
 	}
diff --git a/drivers/iio/resolver/ad2s1200.c b/drivers/iio/resolver/ad2s1200.c
index 55bcbbd4021a1..ec7afad8d45c2 100644
--- a/drivers/iio/resolver/ad2s1200.c
+++ b/drivers/iio/resolver/ad2s1200.c
@@ -182,8 +182,8 @@ static const struct of_device_id ad2s1200_of_match[] = {
 MODULE_DEVICE_TABLE(of, ad2s1200_of_match);
 
 static const struct spi_device_id ad2s1200_id[] = {
-	{ "ad2s1200" },
-	{ "ad2s1205" },
+	{ .name = "ad2s1200" },
+	{ .name = "ad2s1205" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad2s1200_id);
diff --git a/drivers/iio/resolver/ad2s1210.c b/drivers/iio/resolver/ad2s1210.c
index 1be19fe8aa3f6..3b5ec21e3446f 100644
--- a/drivers/iio/resolver/ad2s1210.c
+++ b/drivers/iio/resolver/ad2s1210.c
@@ -1276,10 +1276,8 @@ static int ad2s1210_debugfs_reg_access(struct iio_dev *indio_dev,
 	return regmap_write(st->regmap, reg, writeval);
 }
 
-static irqreturn_t ad2s1210_trigger_handler(int irq, void *p)
+static void ad2s1210_scan_to_buffers(struct iio_dev *indio_dev, s64 timestamp)
 {
-	struct iio_poll_func *pf = p;
-	struct iio_dev *indio_dev = pf->indio_dev;
 	struct ad2s1210_state *st = iio_priv(indio_dev);
 	size_t chan = 0;
 	int ret;
@@ -1295,15 +1293,15 @@ static irqreturn_t ad2s1210_trigger_handler(int irq, void *p)
 					       AD2S1210_REG_POSITION_MSB,
 					       &st->sample.raw, 2);
 			if (ret < 0)
-				goto error_ret;
+				return;
 		} else {
 			ret = ad2s1210_set_mode(st, MOD_POS);
 			if (ret < 0)
-				goto error_ret;
+				return;
 
 			ret = spi_read(st->sdev, &st->sample, 3);
 			if (ret < 0)
-				goto error_ret;
+				return;
 		}
 
 		memcpy(&st->scan.chan[chan++], &st->sample.raw, 2);
@@ -1315,15 +1313,15 @@ static irqreturn_t ad2s1210_trigger_handler(int irq, void *p)
 					       AD2S1210_REG_VELOCITY_MSB,
 					       &st->sample.raw, 2);
 			if (ret < 0)
-				goto error_ret;
+				return;
 		} else {
 			ret = ad2s1210_set_mode(st, MOD_VEL);
 			if (ret < 0)
-				goto error_ret;
+				return;
 
 			ret = spi_read(st->sdev, &st->sample, 3);
 			if (ret < 0)
-				goto error_ret;
+				return;
 		}
 
 		memcpy(&st->scan.chan[chan++], &st->sample.raw, 2);
@@ -1334,16 +1332,22 @@ static irqreturn_t ad2s1210_trigger_handler(int irq, void *p)
 
 		ret = regmap_read(st->regmap, AD2S1210_REG_FAULT, &reg_val);
 		if (ret < 0)
-			goto error_ret;
+			return;
 
 		st->sample.fault = reg_val;
 	}
 
-	ad2s1210_push_events(indio_dev, st->sample.fault, pf->timestamp);
+	ad2s1210_push_events(indio_dev, st->sample.fault, timestamp);
 	iio_push_to_buffers_with_ts(indio_dev, &st->scan, sizeof(st->scan),
-				    pf->timestamp);
+				    timestamp);
+}
+
+static irqreturn_t ad2s1210_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
 
-error_ret:
+	ad2s1210_scan_to_buffers(indio_dev, pf->timestamp);
 	iio_trigger_notify_done(indio_dev->trig);
 
 	return IRQ_HANDLED;
@@ -1597,7 +1601,7 @@ static const struct of_device_id ad2s1210_of_match[] = {
 MODULE_DEVICE_TABLE(of, ad2s1210_of_match);
 
 static const struct spi_device_id ad2s1210_id[] = {
-	{ "ad2s1210" },
+	{ .name = "ad2s1210" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad2s1210_id);
diff --git a/drivers/iio/resolver/ad2s90.c b/drivers/iio/resolver/ad2s90.c
index 18f1c905eeacf..b7a2dcc428711 100644
--- a/drivers/iio/resolver/ad2s90.c
+++ b/drivers/iio/resolver/ad2s90.c
@@ -110,7 +110,7 @@ static const struct of_device_id ad2s90_of_match[] = {
 MODULE_DEVICE_TABLE(of, ad2s90_of_match);
 
 static const struct spi_device_id ad2s90_id[] = {
-	{ "ad2s90" },
+	{ .name = "ad2s90" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad2s90_id);
diff --git a/drivers/iio/temperature/hid-sensor-temperature.c b/drivers/iio/temperature/hid-sensor-temperature.c
index 54a00253de114..5e3262e461f46 100644
--- a/drivers/iio/temperature/hid-sensor-temperature.c
+++ b/drivers/iio/temperature/hid-sensor-temperature.c
@@ -44,7 +44,7 @@ static const struct iio_chan_spec temperature_channels[] = {
 
 /* Adjust channel real bits based on report descriptor */
 static void temperature_adjust_channel_bit_mask(struct iio_chan_spec *channels,
-					int channel, int size)
+						int channel, int size)
 {
 	channels[channel].scan_type.sign = 's';
 	/* Real storage bits will change based on the report desc. */
@@ -100,8 +100,8 @@ static int temperature_read_raw(struct iio_dev *indio_dev,
 }
 
 static int temperature_write_raw(struct iio_dev *indio_dev,
-				struct iio_chan_spec const *chan,
-				int val, int val2, long mask)
+				 struct iio_chan_spec const *chan,
+				 int val, int val2, long mask)
 {
 	struct temperature_state *temp_st = iio_priv(indio_dev);
 
@@ -124,7 +124,7 @@ static const struct iio_info temperature_info = {
 
 /* Callback handler to send event after all samples are received and captured */
 static int temperature_proc_event(struct hid_sensor_hub_device *hsdev,
-				unsigned int usage_id, void *pdev)
+				  u32 usage_id, void *pdev)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
 	struct temperature_state *temp_st = iio_priv(indio_dev);
@@ -139,8 +139,9 @@ static int temperature_proc_event(struct hid_sensor_hub_device *hsdev,
 
 /* Capture samples in local storage */
 static int temperature_capture_sample(struct hid_sensor_hub_device *hsdev,
-				unsigned int usage_id, size_t raw_len,
-				char *raw_data, void *pdev)
+				      u32 usage_id,
+				      size_t raw_len, char *raw_data,
+				      void *pdev)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
 	struct temperature_state *temp_st = iio_priv(indio_dev);
@@ -156,10 +157,10 @@ static int temperature_capture_sample(struct hid_sensor_hub_device *hsdev,
 
 /* Parse report which is specific to an usage id*/
 static int temperature_parse_report(struct platform_device *pdev,
-				struct hid_sensor_hub_device *hsdev,
-				struct iio_chan_spec *channels,
-				unsigned int usage_id,
-				struct temperature_state *st)
+				    struct hid_sensor_hub_device *hsdev,
+				    struct iio_chan_spec *channels,
+				    u32 usage_id,
+				    struct temperature_state *st)
 {
 	int ret;
 
@@ -170,8 +171,7 @@ static int temperature_parse_report(struct platform_device *pdev,
 	if (ret < 0)
 		return ret;
 
-	temperature_adjust_channel_bit_mask(channels, 0,
-					st->temperature_attr.size);
+	temperature_adjust_channel_bit_mask(channels, 0, st->temperature_attr.size);
 
 	st->scale_precision = hid_sensor_format_scale(
 				HID_USAGE_SENSOR_TEMPERATURE,
@@ -212,13 +212,13 @@ static int hid_temperature_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	temp_chans = devm_kmemdup(&indio_dev->dev, temperature_channels,
-				sizeof(temperature_channels), GFP_KERNEL);
+	temp_chans = devm_kmemdup(&pdev->dev, temperature_channels,
+				  sizeof(temperature_channels), GFP_KERNEL);
 	if (!temp_chans)
 		return -ENOMEM;
 
 	ret = temperature_parse_report(pdev, hsdev, temp_chans,
-				HID_USAGE_SENSOR_TEMPERATURE, temp_st);
+				       HID_USAGE_SENSOR_TEMPERATURE, temp_st);
 	if (ret)
 		return ret;
 
@@ -231,7 +231,7 @@ static int hid_temperature_probe(struct platform_device *pdev)
 	atomic_set(&temp_st->common_attributes.data_ready, 0);
 
 	ret = hid_sensor_setup_trigger(indio_dev, name,
-				&temp_st->common_attributes);
+				       &temp_st->common_attributes);
 	if (ret)
 		return ret;
 
@@ -239,11 +239,11 @@ static int hid_temperature_probe(struct platform_device *pdev)
 
 	temperature_callbacks.pdev = pdev;
 	ret = sensor_hub_register_callback(hsdev, HID_USAGE_SENSOR_TEMPERATURE,
-					&temperature_callbacks);
+					   &temperature_callbacks);
 	if (ret)
 		goto error_remove_trigger;
 
-	ret = devm_iio_device_register(indio_dev->dev.parent, indio_dev);
+	ret = iio_device_register(indio_dev);
 	if (ret)
 		goto error_remove_callback;
 
@@ -263,6 +263,7 @@ static void hid_temperature_remove(struct platform_device *pdev)
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
 	struct temperature_state *temp_st = iio_priv(indio_dev);
 
+	iio_device_unregister(indio_dev);
 	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_TEMPERATURE);
 	hid_sensor_remove_trigger(indio_dev, &temp_st->common_attributes);
 }
diff --git a/drivers/iio/temperature/ltc2983.c b/drivers/iio/temperature/ltc2983.c
index f8c4917bb1188..a9d38ed45da69 100644
--- a/drivers/iio/temperature/ltc2983.c
+++ b/drivers/iio/temperature/ltc2983.c
@@ -2070,11 +2070,11 @@ static const struct ltc2983_chip_info ltm2985_chip_info_data = {
 };
 
 static const struct spi_device_id ltc2983_id_table[] = {
-	{ "adt7604", (kernel_ulong_t)&adt7604_chip_info_data },
-	{ "ltc2983", (kernel_ulong_t)&ltc2983_chip_info_data },
-	{ "ltc2984", (kernel_ulong_t)&ltc2984_chip_info_data },
-	{ "ltc2986", (kernel_ulong_t)&ltc2986_chip_info_data },
-	{ "ltm2985", (kernel_ulong_t)&ltm2985_chip_info_data },
+	{ .name = "adt7604", .driver_data = (kernel_ulong_t)&adt7604_chip_info_data },
+	{ .name = "ltc2983", .driver_data = (kernel_ulong_t)&ltc2983_chip_info_data },
+	{ .name = "ltc2984", .driver_data = (kernel_ulong_t)&ltc2984_chip_info_data },
+	{ .name = "ltc2986", .driver_data = (kernel_ulong_t)&ltc2986_chip_info_data },
+	{ .name = "ltm2985", .driver_data = (kernel_ulong_t)&ltm2985_chip_info_data },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ltc2983_id_table);
diff --git a/drivers/iio/temperature/max31856.c b/drivers/iio/temperature/max31856.c
index 13405f3ba829f..061f6fe6de5e5 100644
--- a/drivers/iio/temperature/max31856.c
+++ b/drivers/iio/temperature/max31856.c
@@ -461,7 +461,7 @@ static int max31856_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id max31856_id[] = {
-	{ "max31856", 0 },
+	{ .name = "max31856" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, max31856_id);
diff --git a/drivers/iio/temperature/max31865.c b/drivers/iio/temperature/max31865.c
index 82cf06c4d2723..f9c93f95d80b4 100644
--- a/drivers/iio/temperature/max31865.c
+++ b/drivers/iio/temperature/max31865.c
@@ -324,7 +324,7 @@ static int max31865_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id max31865_id[] = {
-	{ "max31865", 0 },
+	{ .name = "max31865" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, max31865_id);
diff --git a/drivers/iio/temperature/maxim_thermocouple.c b/drivers/iio/temperature/maxim_thermocouple.c
index 015afeab19149..5f216e238d550 100644
--- a/drivers/iio/temperature/maxim_thermocouple.c
+++ b/drivers/iio/temperature/maxim_thermocouple.c
@@ -262,15 +262,15 @@ static int maxim_thermocouple_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id maxim_thermocouple_id[] = {
-	{"max6675", MAX6675},
-	{"max31855", MAX31855},
-	{"max31855k", MAX31855K},
-	{"max31855j", MAX31855J},
-	{"max31855n", MAX31855N},
-	{"max31855s", MAX31855S},
-	{"max31855t", MAX31855T},
-	{"max31855e", MAX31855E},
-	{"max31855r", MAX31855R},
+	{ .name = "max6675", .driver_data = MAX6675 },
+	{ .name = "max31855", .driver_data = MAX31855 },
+	{ .name = "max31855k", .driver_data = MAX31855K },
+	{ .name = "max31855j", .driver_data = MAX31855J },
+	{ .name = "max31855n", .driver_data = MAX31855N },
+	{ .name = "max31855s", .driver_data = MAX31855S },
+	{ .name = "max31855t", .driver_data = MAX31855T },
+	{ .name = "max31855e", .driver_data = MAX31855E },
+	{ .name = "max31855r", .driver_data = MAX31855R },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, maxim_thermocouple_id);
diff --git a/drivers/iio/temperature/tmp117.c b/drivers/iio/temperature/tmp117.c
index 6bc18616ad15e..74cb8d62bef3c 100644
--- a/drivers/iio/temperature/tmp117.c
+++ b/drivers/iio/temperature/tmp117.c
@@ -9,6 +9,7 @@
  * Note: This driver assumes that the sensor has been calibrated beforehand.
  */
 
+#include <linux/array_size.h>
 #include <linux/delay.h>
 #include <linux/err.h>
 #include <linux/i2c.h>
@@ -38,6 +39,7 @@
 
 #define TMP116_DEVICE_ID		0x1116
 #define TMP117_DEVICE_ID		0x0117
+#define TMP119_DEVICE_ID		0x2117
 
 struct tmp117_data {
 	struct i2c_client *client;
@@ -139,6 +141,12 @@ static const struct tmp11x_info tmp117_channels_info = {
 	.num_channels = ARRAY_SIZE(tmp117_channels)
 };
 
+static const struct tmp11x_info tmp119_channels_info = {
+	.name = "tmp119",
+	.channels = tmp117_channels,
+	.num_channels = ARRAY_SIZE(tmp117_channels)
+};
+
 static const struct iio_info tmp117_info = {
 	.read_raw = tmp117_read_raw,
 	.write_raw = tmp117_write_raw,
@@ -172,6 +180,9 @@ static int tmp117_probe(struct i2c_client *client)
 	case TMP117_DEVICE_ID:
 		match_data = &tmp117_channels_info;
 		break;
+	case TMP119_DEVICE_ID:
+		match_data = &tmp119_channels_info;
+		break;
 	default:
 		dev_info(&client->dev,
 			 "Unknown device id (0x%x), use fallback compatible\n",
@@ -204,6 +215,7 @@ static int tmp117_probe(struct i2c_client *client)
 static const struct of_device_id tmp117_of_match[] = {
 	{ .compatible = "ti,tmp116", .data = &tmp116_channels_info },
 	{ .compatible = "ti,tmp117", .data = &tmp117_channels_info },
+	{ .compatible = "ti,tmp119", .data = &tmp119_channels_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, tmp117_of_match);
@@ -211,6 +223,7 @@ MODULE_DEVICE_TABLE(of, tmp117_of_match);
 static const struct i2c_device_id tmp117_id[] = {
 	{ .name = "tmp116", .driver_data = (kernel_ulong_t)&tmp116_channels_info },
 	{ .name = "tmp117", .driver_data = (kernel_ulong_t)&tmp117_channels_info },
+	{ .name = "tmp119", .driver_data = (kernel_ulong_t)&tmp119_channels_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tmp117_id);
diff --git a/drivers/iio/test/iio-test-format.c b/drivers/iio/test/iio-test-format.c
index 872dd8582003f..e5d0a2ac46702 100644
--- a/drivers/iio/test/iio-test-format.c
+++ b/drivers/iio/test/iio-test-format.c
@@ -200,56 +200,108 @@ static void iio_test_iio_format_value_multiple(struct kunit *test)
 static void iio_test_iio_format_value_integer_64(struct kunit *test)
 {
 	int values[2];
-	s64 value;
 	char *buf;
 	int ret;
 
 	buf = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf);
 
-	value = 24;
-	values[0] = lower_32_bits(value);
-	values[1] = upper_32_bits(value);
+	iio_val_s64_decompose(24, &values[0], &values[1]);
 	ret = iio_format_value(buf, IIO_VAL_INT_64, ARRAY_SIZE(values), values);
 	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "24\n");
 
-	value = -24;
-	values[0] = lower_32_bits(value);
-	values[1] = upper_32_bits(value);
+	iio_val_s64_decompose(-24, &values[0], &values[1]);
 	ret = iio_format_value(buf, IIO_VAL_INT_64, ARRAY_SIZE(values), values);
 	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "-24\n");
 
-	value = 0;
-	values[0] = lower_32_bits(value);
-	values[1] = upper_32_bits(value);
+	iio_val_s64_decompose(0, &values[0], &values[1]);
 	ret = iio_format_value(buf, IIO_VAL_INT_64, ARRAY_SIZE(values), values);
 	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "0\n");
 
-	value = UINT_MAX;
-	values[0] = lower_32_bits(value);
-	values[1] = upper_32_bits(value);
+	iio_val_s64_decompose(UINT_MAX, &values[0], &values[1]);
 	ret = iio_format_value(buf, IIO_VAL_INT_64, ARRAY_SIZE(values), values);
 	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "4294967295\n");
 
-	value = -((s64)UINT_MAX);
-	values[0] = lower_32_bits(value);
-	values[1] = upper_32_bits(value);
+	iio_val_s64_decompose(-((s64)UINT_MAX), &values[0], &values[1]);
 	ret = iio_format_value(buf, IIO_VAL_INT_64, ARRAY_SIZE(values), values);
 	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "-4294967295\n");
 
-	value = LLONG_MAX;
-	values[0] = lower_32_bits(value);
-	values[1] = upper_32_bits(value);
+	iio_val_s64_decompose(LLONG_MAX, &values[0], &values[1]);
 	ret = iio_format_value(buf, IIO_VAL_INT_64, ARRAY_SIZE(values), values);
 	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "9223372036854775807\n");
 
-	value = LLONG_MIN;
-	values[0] = lower_32_bits(value);
-	values[1] = upper_32_bits(value);
+	iio_val_s64_decompose(LLONG_MIN, &values[0], &values[1]);
 	ret = iio_format_value(buf, IIO_VAL_INT_64, ARRAY_SIZE(values), values);
 	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "-9223372036854775808\n");
 }
 
+static void iio_test_iio_format_value_decimal_64(struct kunit *test)
+{
+	int values[2];
+	char *buf;
+	int ret;
+
+	buf = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf);
+
+	/* DECIMAL64_MILLI: positive >= 1, value 1.234 */
+	iio_val_s64_decompose(1234, &values[0], &values[1]);
+	ret = iio_format_value(buf, IIO_VAL_DECIMAL64_MILLI, ARRAY_SIZE(values), values);
+	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "1.234\n");
+
+	/* DECIMAL64_MICRO: positive >= 1, value 3.141592 */
+	iio_val_s64_decompose(3141592, &values[0], &values[1]);
+	ret = iio_format_value(buf, IIO_VAL_DECIMAL64_MICRO, ARRAY_SIZE(values), values);
+	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "3.141592\n");
+
+	/* DECIMAL64_MILLI: positive < 1, value 0.042 */
+	iio_val_s64_decompose(42, &values[0], &values[1]);
+	ret = iio_format_value(buf, IIO_VAL_DECIMAL64_MILLI, ARRAY_SIZE(values), values);
+	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "0.042\n");
+
+	/* DECIMAL64_MILLI: negative <= -1, value -1.234 */
+	iio_val_s64_decompose(-1234, &values[0], &values[1]);
+	ret = iio_format_value(buf, IIO_VAL_DECIMAL64_MILLI, ARRAY_SIZE(values), values);
+	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "-1.234\n");
+
+	/* DECIMAL64_MILLI: negative > -1, value -0.123 */
+	iio_val_s64_decompose(-123, &values[0], &values[1]);
+	ret = iio_format_value(buf, IIO_VAL_DECIMAL64_MILLI, ARRAY_SIZE(values), values);
+	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "-0.123\n");
+
+	/* DECIMAL64_MILLI: zero */
+	iio_val_s64_decompose(0, &values[0], &values[1]);
+	ret = iio_format_value(buf, IIO_VAL_DECIMAL64_MILLI, ARRAY_SIZE(values), values);
+	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "0.000\n");
+
+	/* DECIMAL64_NANO: value 1.000000001 */
+	iio_val_s64_decompose(1000000001, &values[0], &values[1]);
+	ret = iio_format_value(buf, IIO_VAL_DECIMAL64_NANO, ARRAY_SIZE(values), values);
+	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "1.000000001\n");
+
+	/* DECIMAL64_MICRO: large value using upper 32 bits */
+	iio_val_s64_decompose(5000000000000042LL, &values[0], &values[1]);
+	ret = iio_format_value(buf, IIO_VAL_DECIMAL64_MICRO, ARRAY_SIZE(values), values);
+	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "5000000000.000042\n");
+
+	/* limits */
+	iio_val_s64_decompose(LLONG_MAX, &values[0], &values[1]);
+	ret = iio_format_value(buf, IIO_VAL_DECIMAL64_PICO, ARRAY_SIZE(values), values);
+	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "9223372.036854775807\n");
+	ret = iio_format_value(buf, IIO_VAL_DECIMAL64_NANO, ARRAY_SIZE(values), values);
+	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "9223372036.854775807\n");
+	ret = iio_format_value(buf, IIO_VAL_DECIMAL64_MICRO, ARRAY_SIZE(values), values);
+	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "9223372036854.775807\n");
+
+	iio_val_s64_decompose(LLONG_MIN, &values[0], &values[1]);
+	ret = iio_format_value(buf, IIO_VAL_DECIMAL64_PICO, ARRAY_SIZE(values), values);
+	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "-9223372.036854775808\n");
+	ret = iio_format_value(buf, IIO_VAL_DECIMAL64_NANO, ARRAY_SIZE(values), values);
+	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "-9223372036.854775808\n");
+	ret = iio_format_value(buf, IIO_VAL_DECIMAL64_MICRO, ARRAY_SIZE(values), values);
+	IIO_TEST_FORMAT_EXPECT_EQ(test, buf, ret, "-9223372036854.775808\n");
+}
+
 static struct kunit_case iio_format_test_cases[] = {
 		KUNIT_CASE(iio_test_iio_format_value_integer),
 		KUNIT_CASE(iio_test_iio_format_value_fixedpoint),
@@ -257,6 +309,7 @@ static struct kunit_case iio_format_test_cases[] = {
 		KUNIT_CASE(iio_test_iio_format_value_fractional_log2),
 		KUNIT_CASE(iio_test_iio_format_value_multiple),
 		KUNIT_CASE(iio_test_iio_format_value_integer_64),
+		KUNIT_CASE(iio_test_iio_format_value_decimal_64),
 		{ }
 };
 
diff --git a/drivers/iio/test/iio-test-rescale.c b/drivers/iio/test/iio-test-rescale.c
index ac6942cf1e448..bcd4a607be192 100644
--- a/drivers/iio/test/iio-test-rescale.c
+++ b/drivers/iio/test/iio-test-rescale.c
@@ -575,7 +575,7 @@ static const struct rescale_tc_data offset_cases[] = {
 
 static void case_to_desc(const struct rescale_tc_data *t, char *desc)
 {
-	strcpy(desc, t->name);
+	strscpy(desc, t->name, KUNIT_PARAM_DESC_SIZE);
 }
 
 KUNIT_ARRAY_PARAM(iio_rescale_scale, scale_cases, case_to_desc);
diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c
index e9fb7ad4c377c..bdaa3eb83bccd 100644
--- a/drivers/infiniband/core/addr.c
+++ b/drivers/infiniband/core/addr.c
@@ -769,7 +769,7 @@ void rdma_addr_cancel(struct rdma_dev_addr *addr)
 
 	/*
 	 * sync canceling the work after removing it from the req_list
-	 * guarentees no work is running and none will be started.
+	 * guarantees no work is running and none will be started.
 	 */
 	cancel_delayed_work_sync(&found->work);
 	kfree(found);
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 9480d1a51c116..cc24fddf9aaa9 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -2104,7 +2104,7 @@ static void destroy_id_handler_unlock(struct rdma_id_private *id_priv)
 	/*
 	 * Setting the state to destroyed under the handler mutex provides a
 	 * fence against calling handler callbacks. If this is invoked due to
-	 * the failure of a handler callback then it guarentees that no future
+	 * the failure of a handler callback then it guarantees that no future
 	 * handlers will be called.
 	 */
 	lockdep_assert_held(&id_priv->handler_mutex);
@@ -5270,7 +5270,7 @@ static int cma_netevent_callback(struct notifier_block *self,
 
 	list_for_each_entry(current_id, &ips_node->id_list, id_list_entry) {
 		if (!memcmp(current_id->id.route.addr.dev_addr.dst_dev_addr,
-			   neigh->ha, ETH_ALEN))
+			   neigh->ha, neigh->dev->addr_len))
 			continue;
 		cma_id_get(current_id);
 		if (!queue_work(cma_wq, &current_id->id.net_work))
diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c
index 3d7b6cddd131c..ee98188e57fb5 100644
--- a/drivers/infiniband/core/cq.c
+++ b/drivers/infiniband/core/cq.c
@@ -393,8 +393,7 @@ static int ib_alloc_cqs(struct ib_device *dev, unsigned int nr_cqes,
 	 * a reasonable batch size so that we can share CQs between
 	 * multiple users instead of allocating a larger number of CQs.
 	 */
-	nr_cqes = min_t(unsigned int, dev->attrs.max_cqe,
-			max(nr_cqes, IB_MAX_SHARED_CQ_SZ));
+	nr_cqes = min(dev->attrs.max_cqe, max(nr_cqes, IB_MAX_SHARED_CQ_SZ));
 	nr_cqs = min_t(unsigned int, dev->num_comp_vectors, num_online_cpus());
 	for (i = 0; i < nr_cqs; i++) {
 		cq = ib_alloc_cq(dev, NULL, nr_cqes, i, poll_ctx);
diff --git a/drivers/infiniband/core/iwpm_msg.c b/drivers/infiniband/core/iwpm_msg.c
index 4625abd29ac06..1b10f2973ad9d 100644
--- a/drivers/infiniband/core/iwpm_msg.c
+++ b/drivers/infiniband/core/iwpm_msg.c
@@ -268,7 +268,7 @@ int iwpm_add_and_query_mapping(struct iwpm_sa_data *pm_msg, u8 nl_client)
 	if (ret)
 		goto query_mapping_error;
 
-	/* If flags are required and we're not V4, then return a quite error */
+	/* If flags are required and we're not V4, then return a quiet error */
 	if (pm_msg->flags && iwpm_ulib_version == IWPM_UABI_VERSION_MIN) {
 		ret = -EINVAL;
 		goto query_mapping_error_nowarn;
diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
index 8d19613179e3e..e0b3b36b8b149 100644
--- a/drivers/infiniband/core/mad.c
+++ b/drivers/infiniband/core/mad.c
@@ -2031,6 +2031,24 @@ void ib_mark_mad_done(struct ib_mad_send_wr_private *mad_send_wr)
 		change_mad_state(mad_send_wr, IB_MAD_STATE_EARLY_RESP);
 }
 
+static bool is_kernel_rmpp_data_response(struct ib_mad_agent_private *agent,
+					 struct ib_mad_recv_wc *mad_recv_wc)
+{
+	const struct ib_mad_hdr *mad_hdr = &mad_recv_wc->recv_buf.mad->mad_hdr;
+	struct ib_rmpp_mad *rmpp_mad;
+
+	if (!ib_mad_kernel_rmpp_agent(&agent->agent) ||
+	    !ib_response_mad(mad_hdr) ||
+	    !ib_is_mad_class_rmpp(mad_hdr->mgmt_class))
+		return false;
+
+	rmpp_mad = (struct ib_rmpp_mad *)mad_recv_wc->recv_buf.mad;
+
+	return (ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) &
+		IB_MGMT_RMPP_FLAG_ACTIVE) &&
+	       rmpp_mad->rmpp_hdr.rmpp_type == IB_MGMT_RMPP_TYPE_DATA;
+}
+
 static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
 				 struct ib_mad_recv_wc *mad_recv_wc)
 {
@@ -2050,6 +2068,18 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
 	}
 
 	list_add(&mad_recv_wc->recv_buf.list, &mad_recv_wc->rmpp_list);
+	if (is_kernel_rmpp_data_response(mad_agent_priv, mad_recv_wc)) {
+		spin_lock_irqsave(&mad_agent_priv->lock, flags);
+		mad_send_wr = ib_find_send_mad(mad_agent_priv, mad_recv_wc);
+		spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
+
+		if (!mad_send_wr) {
+			ib_free_recv_mad(mad_recv_wc);
+			deref_mad_agent(mad_agent_priv);
+			return;
+		}
+	}
+
 	if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) {
 		mad_recv_wc = ib_process_rmpp_recv_wc(mad_agent_priv,
 						      mad_recv_wc);
diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c
index 02a0a9c0a4a6a..aae4f3f6bcba1 100644
--- a/drivers/infiniband/core/nldev.c
+++ b/drivers/infiniband/core/nldev.c
@@ -188,6 +188,7 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
 	[RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD] = { .type = NLA_U32 },
 	[RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES] = { .type = NLA_U32 },
 	[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_KERNEL_VENDOR_KEY] = { .type = NLA_U64 },
+	[RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_MAX]	= { .type = NLA_U64 },
 };
 
 static int put_driver_name_print_type(struct sk_buff *msg, const char *name,
@@ -413,7 +414,7 @@ static int fill_port_info(struct sk_buff *msg,
 }
 
 static int fill_res_info_entry(struct sk_buff *msg,
-			       const char *name, u64 curr)
+			       const char *name, u64 curr, u64 max)
 {
 	struct nlattr *entry_attr;
 
@@ -427,6 +428,9 @@ static int fill_res_info_entry(struct sk_buff *msg,
 	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR, curr,
 			      RDMA_NLDEV_ATTR_PAD))
 		goto err;
+	if (max && nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_MAX,
+				     max, RDMA_NLDEV_ATTR_PAD))
+		goto err;
 
 	nla_nest_end(msg, entry_attr);
 	return 0;
@@ -450,7 +454,8 @@ static int fill_res_info(struct sk_buff *msg, struct ib_device *device,
 	};
 
 	struct nlattr *table_attr;
-	int ret, i, curr;
+	u64 curr, max;
+	int ret, i;
 
 	if (fill_nldev_handle(msg, device))
 		return -EMSGSIZE;
@@ -463,7 +468,26 @@ static int fill_res_info(struct sk_buff *msg, struct ib_device *device,
 		if (!names[i])
 			continue;
 		curr = rdma_restrack_count(device, i, show_details);
-		ret = fill_res_info_entry(msg, names[i], curr);
+		switch (i) {
+		case RDMA_RESTRACK_QP:
+			max = device->attrs.max_qp;
+			break;
+		case RDMA_RESTRACK_CQ:
+			max = device->attrs.max_cq;
+			break;
+		case RDMA_RESTRACK_MR:
+			max = device->attrs.max_mr;
+			break;
+		case RDMA_RESTRACK_PD:
+			max = device->attrs.max_pd;
+			break;
+		case RDMA_RESTRACK_SRQ:
+			max = device->attrs.max_srq;
+			break;
+		default:
+			max = 0;
+		}
+		ret = fill_res_info_entry(msg, names[i], curr, max);
 		if (ret)
 			goto err;
 	}
diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c
index cfee2071586c1..1b2f9df49e289 100644
--- a/drivers/infiniband/core/restrack.c
+++ b/drivers/infiniband/core/restrack.c
@@ -61,7 +61,7 @@ void rdma_restrack_clean(struct ib_device *dev)
  * @type: actual type of object to operate
  * @show_details: count driver specific objects
  */
-int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type,
+u32 rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type,
 			bool show_details)
 {
 	struct rdma_restrack_root *rt = &dev->res[type];
diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c
index 73498723a5d55..81f44dadfa522 100644
--- a/drivers/infiniband/core/umem.c
+++ b/drivers/infiniband/core/umem.c
@@ -209,7 +209,8 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device,
 
 	mmgrab(mm);
 
-	page_list = (struct page **) __get_free_page(GFP_KERNEL);
+	/* TODO: switch to "fast and as large as possible" allocation helper */
+	page_list = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!page_list) {
 		ret = -ENOMEM;
 		goto umem_kfree;
@@ -269,7 +270,7 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device,
 	__ib_umem_release(device, umem, 0);
 	atomic64_sub(ib_umem_num_pages(umem), &mm->pinned_vm);
 out:
-	free_page((unsigned long) page_list);
+	kfree(page_list);
 umem_kfree:
 	if (ret) {
 		mmdrop(umem->owning_mm);
diff --git a/drivers/infiniband/core/umem_dmabuf.c b/drivers/infiniband/core/umem_dmabuf.c
index ad023c2d84d85..39b5564a4c35e 100644
--- a/drivers/infiniband/core/umem_dmabuf.c
+++ b/drivers/infiniband/core/umem_dmabuf.c
@@ -181,7 +181,7 @@ struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device,
 }
 EXPORT_SYMBOL(ib_umem_dmabuf_get);
 
-static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = {
+static const struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = {
 	.allow_peer2peer = true,
 };
 
@@ -205,7 +205,7 @@ static void ib_umem_dmabuf_revoke_locked(struct dma_buf_attachment *attach)
 	umem_dmabuf->revoked = 1;
 }
 
-static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_revocable_ops = {
+static const struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_revocable_ops = {
 	.allow_peer2peer = true,
 	.invalidate_mappings = ib_umem_dmabuf_revoke_locked,
 };
diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c
index 404fa1cc32542..9cc21cd762d98 100644
--- a/drivers/infiniband/core/umem_odp.c
+++ b/drivers/infiniband/core/umem_odp.c
@@ -329,7 +329,7 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt,
 	struct mm_struct *owning_mm = umem_odp->umem.owning_mm;
 	int pfn_index, dma_index, ret = 0, start_idx;
 	unsigned int page_shift, hmm_order, pfn_start_idx;
-	unsigned long num_pfns, current_seq;
+	unsigned long num_pfns;
 	struct hmm_range range = {};
 	unsigned long timeout;
 
@@ -363,26 +363,18 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt,
 	}
 
 	range.hmm_pfns = &(umem_odp->map.pfn_list[pfn_start_idx]);
-	timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
+	timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
 
 retry:
-	current_seq = range.notifier_seq =
-		mmu_interval_read_begin(&umem_odp->notifier);
-
-	mmap_read_lock(owning_mm);
-	ret = hmm_range_fault(&range);
-	mmap_read_unlock(owning_mm);
-	if (unlikely(ret)) {
-		if (ret == -EBUSY && !time_after(jiffies, timeout))
-			goto retry;
+	ret = hmm_range_fault_unlocked_timeout(&range, timeout);
+	if (unlikely(ret))
 		goto out_put_mm;
-	}
 
 	start_idx = (range.start - ib_umem_start(umem_odp)) >> page_shift;
 	dma_index = start_idx;
 
 	mutex_lock(&umem_odp->umem_mutex);
-	if (mmu_interval_read_retry(&umem_odp->notifier, current_seq)) {
+	if (mmu_interval_read_retry(&umem_odp->notifier, range.notifier_seq)) {
 		mutex_unlock(&umem_odp->umem_mutex);
 		goto retry;
 	}
diff --git a/drivers/infiniband/core/uverbs_std_types_mr.c b/drivers/infiniband/core/uverbs_std_types_mr.c
index 570b9656801d7..0c72f801e0d32 100644
--- a/drivers/infiniband/core/uverbs_std_types_mr.c
+++ b/drivers/infiniband/core/uverbs_std_types_mr.c
@@ -364,7 +364,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_REG_MR)(
 							dmah, attrs);
 	else
 		mr = pd->device->ops.reg_user_mr(pd, addr, length, iova,
-						 access_flags, dmah, NULL);
+						 access_flags, dmah,
+						 &attrs->driver_udata);
 
 	if (IS_ERR(mr))
 		return PTR_ERR(mr);
@@ -527,7 +528,8 @@ DECLARE_UVERBS_NAMED_METHOD(
 			    UA_MANDATORY),
 	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_REG_MR_RESP_RKEY,
 			    UVERBS_ATTR_TYPE(u32),
-			    UA_MANDATORY));
+			    UA_MANDATORY),
+	UVERBS_ATTR_UHW());
 
 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
 	UVERBS_METHOD_MR_DESTROY,
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 3b613b57e2697..86811d31092ce 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -2196,13 +2196,13 @@ struct ib_cq *__ib_create_cq(struct ib_device *device,
 	struct ib_cq *cq;
 	int ret;
 
+	if (WARN_ON_ONCE(!cq_attr->cqe))
+		return ERR_PTR(-EINVAL);
+
 	cq = rdma_zalloc_drv_obj(device, ib_cq);
 	if (!cq)
 		return ERR_PTR(-ENOMEM);
 
-	if (WARN_ON_ONCE(!cq_attr->cqe))
-		return ERR_PTR(-EINVAL);
-
 	cq->device = device;
 	cq->comp_handler = comp_handler;
 	cq->event_handler = event_handler;
diff --git a/drivers/infiniband/hw/bng_re/bng_fw.c b/drivers/infiniband/hw/bng_re/bng_fw.c
index 50156c300b335..ab6a2d2e95b53 100644
--- a/drivers/infiniband/hw/bng_re/bng_fw.c
+++ b/drivers/infiniband/hw/bng_re/bng_fw.c
@@ -401,14 +401,15 @@ static int __wait_for_resp(struct bng_re_rcfw *rcfw, u16 cookie)
 {
 	struct bng_re_cmdq_ctx *cmdq;
 	struct bng_re_crsqe *crsqe;
+	unsigned long time_left;
 
 	cmdq = &rcfw->cmdq;
 	crsqe = &rcfw->crsqe_tbl[cookie];
 
 	do {
-		wait_event_timeout(cmdq->waitq,
-				   !crsqe->is_in_used,
-				   secs_to_jiffies(rcfw->max_timeout));
+		time_left = wait_event_timeout(cmdq->waitq,
+					       !crsqe->is_in_used,
+					       secs_to_jiffies(rcfw->max_timeout));
 
 		if (!crsqe->is_in_used)
 			return 0;
@@ -417,6 +418,9 @@ static int __wait_for_resp(struct bng_re_rcfw *rcfw, u16 cookie)
 
 		if (!crsqe->is_in_used)
 			return 0;
+
+		if (!time_left)
+			return -ENODEV;
 	} while (true);
 };
 
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index 5657625290076..90138d64adee2 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -193,7 +193,6 @@ int bnxt_re_query_device(struct ib_device *ibdev,
 	if (rc)
 		return rc;
 
-	memset(ib_attr, 0, sizeof(*ib_attr));
 	memcpy(&ib_attr->fw_ver, dev_attr->fw_ver,
 	       min(sizeof(dev_attr->fw_ver),
 		   sizeof(ib_attr->fw_ver)));
diff --git a/drivers/infiniband/hw/efa/Makefile b/drivers/infiniband/hw/efa/Makefile
index 6e83083af0bc3..a6a433b0ba2f1 100644
--- a/drivers/infiniband/hw/efa/Makefile
+++ b/drivers/infiniband/hw/efa/Makefile
@@ -1,9 +1,9 @@
 # SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
-# Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+# Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
 #
 # Makefile for Amazon Elastic Fabric Adapter (EFA) device driver.
 #
 
 obj-$(CONFIG_INFINIBAND_EFA) += efa.o
 
-efa-y := efa_com_cmd.o efa_com.o efa_main.o efa_verbs.o
+efa-y := efa_com_cmd.o efa_ah_cache.o efa_com.o efa_main.o efa_verbs.o
diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
index 826790ca9d839..3eb3a4de89129 100644
--- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
+++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
@@ -367,10 +367,10 @@ struct efa_admin_reg_mr_cmd {
 
 	/*
 	 * flags and page size
-	 * 4:0 : phys_page_size_shift - page size is (1 <<
+	 * 5:0 : phys_page_size_shift - page size is (1 <<
 	 *    phys_page_size_shift). Page size is used for
 	 *    building the Virtual to Physical address mapping
-	 * 6:5 : reserved - MBZ
+	 * 6 : reserved - MBZ
 	 * 7 : mem_addr_phy_mode_en - Enable bit for physical
 	 *    memory registration (no translation), can be used
 	 *    only by privileged clients. If set, PBL must
@@ -1103,7 +1103,7 @@ struct efa_admin_host_info {
 #define EFA_ADMIN_MODIFY_QP_CMD_RNR_RETRY_MASK              BIT(5)
 
 /* reg_mr_cmd */
-#define EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK      GENMASK(4, 0)
+#define EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK      GENMASK(5, 0)
 #define EFA_ADMIN_REG_MR_CMD_MEM_ADDR_PHY_MODE_EN_MASK      BIT(7)
 #define EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE_MASK        BIT(0)
 #define EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE_MASK       BIT(1)
diff --git a/drivers/infiniband/hw/efa/efa_ah_cache.c b/drivers/infiniband/hw/efa/efa_ah_cache.c
new file mode 100644
index 0000000000000..6219529fa889c
--- /dev/null
+++ b/drivers/infiniband/hw/efa/efa_ah_cache.c
@@ -0,0 +1,135 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+/*
+ * Copyright 2026 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#include <linux/slab.h>
+
+#include "efa_ah_cache.h"
+
+static const struct rhashtable_params ah_cache_params = {
+	.key_len = sizeof(struct efa_ah_cache_key),
+	.key_offset = offsetof(struct efa_ah_cache_entry, key),
+	.head_offset = offsetof(struct efa_ah_cache_entry, linkage),
+};
+
+int efa_ah_cache_init(struct efa_ah_cache *ah_cache)
+{
+	int err;
+
+	mutex_init(&ah_cache->lock);
+	err = rhashtable_init(&ah_cache->hashtable, &ah_cache_params);
+	if (err)
+		mutex_destroy(&ah_cache->lock);
+
+	return err;
+}
+
+static void efa_ah_cache_entry_free(void *ptr, void *arg)
+{
+	struct efa_ah_cache_entry *entry = ptr;
+
+	WARN_ON(entry->usecnt);
+	mutex_destroy(&entry->lock);
+	kfree(entry);
+}
+
+void efa_ah_cache_destroy(struct efa_ah_cache *ah_cache)
+{
+	rhashtable_free_and_destroy(&ah_cache->hashtable, efa_ah_cache_entry_free, NULL);
+	mutex_destroy(&ah_cache->lock);
+}
+
+static struct efa_ah_cache_entry *efa_ah_cache_lookup_locked(struct efa_ah_cache *ah_cache, u16 pd,
+							     u8 *gid)
+	__must_hold(&ah_cache->lock)
+{
+	struct efa_ah_cache_key key = {};
+
+	memcpy(key.gid, gid, sizeof(key.gid));
+	key.pd = pd;
+
+	return rhashtable_lookup_fast(&ah_cache->hashtable, &key, ah_cache_params);
+}
+
+struct efa_ah_cache_entry *efa_ah_cache_lookup(struct efa_ah_cache *ah_cache, u16 pd, u8 *gid)
+{
+	struct efa_ah_cache_entry *entry;
+
+	mutex_lock(&ah_cache->lock);
+	entry = efa_ah_cache_lookup_locked(ah_cache, pd, gid);
+	mutex_unlock(&ah_cache->lock);
+
+	return entry;
+}
+
+/**
+ * efa_ah_cache_get - Get or create an AH cache entry
+ * @ah_cache: AH cache
+ * @pd: Protection domain number
+ * @gid: GID address
+ *
+ * Look up an AH cache entry by PD and GID. If found, take a reference and
+ * return it. If not found, allocate a new entry and insert it. The caller must lock
+ * the entry mutex and check usecnt to determine whether a device create
+ * command is needed.
+ *
+ * Return: Pointer to the entry on success, ERR_PTR on failure.
+ */
+struct efa_ah_cache_entry *efa_ah_cache_get(struct efa_ah_cache *ah_cache, u16 pd, u8 *gid)
+{
+	struct efa_ah_cache_entry *entry;
+	int err;
+
+	mutex_lock(&ah_cache->lock);
+
+	entry = efa_ah_cache_lookup_locked(ah_cache, pd, gid);
+	if (entry) {
+		refcount_inc(&entry->refcount);
+		mutex_unlock(&ah_cache->lock);
+		return entry;
+	}
+
+	entry = kzalloc_obj(*entry);
+	if (!entry) {
+		mutex_unlock(&ah_cache->lock);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	memcpy(entry->key.gid, gid, sizeof(entry->key.gid));
+	entry->key.pd = pd;
+	refcount_set(&entry->refcount, 1);
+	mutex_init(&entry->lock);
+
+	err = rhashtable_insert_fast(&ah_cache->hashtable, &entry->linkage, ah_cache_params);
+	if (err) {
+		mutex_destroy(&entry->lock);
+		kfree(entry);
+		mutex_unlock(&ah_cache->lock);
+		return ERR_PTR(err);
+	}
+
+	mutex_unlock(&ah_cache->lock);
+	return entry;
+}
+
+/**
+ * efa_ah_cache_put - Put a refcount of an AH cache entry
+ * @ah_cache: AH cache
+ * @entry: AH cache entry
+ *
+ * Drop the refcount. If it reaches zero, remove the entry from the hashtable
+ * and free it.
+ */
+void efa_ah_cache_put(struct efa_ah_cache *ah_cache, struct efa_ah_cache_entry *entry)
+{
+	if (!refcount_dec_and_mutex_lock(&entry->refcount, &ah_cache->lock))
+		return;
+
+	/* AH cache lock is held here */
+	rhashtable_remove_fast(&ah_cache->hashtable, &entry->linkage, ah_cache_params);
+	mutex_unlock(&ah_cache->lock);
+
+	mutex_destroy(&entry->lock);
+	kfree(entry);
+}
diff --git a/drivers/infiniband/hw/efa/efa_ah_cache.h b/drivers/infiniband/hw/efa/efa_ah_cache.h
new file mode 100644
index 0000000000000..e7cdcbb640700
--- /dev/null
+++ b/drivers/infiniband/hw/efa/efa_ah_cache.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright 2026 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#ifndef _EFA_AH_CACHE_H_
+#define _EFA_AH_CACHE_H_
+
+#include <linux/refcount.h>
+#include <linux/rhashtable.h>
+
+#define EFA_AH_GID_SIZE 16
+
+struct efa_ah_cache_key {
+	u8 gid[EFA_AH_GID_SIZE];
+	u16 pd;
+};
+
+struct efa_ah_cache_entry {
+	struct efa_ah_cache_key key;
+	u16 ah;
+	unsigned int usecnt;
+	refcount_t refcount;
+	struct rhash_head linkage;
+	struct mutex lock; /* Serializes device commands per cache entry */
+};
+
+struct efa_ah_cache {
+	struct rhashtable hashtable;
+	struct mutex lock; /* Protects AH cache hashtable */
+};
+
+int efa_ah_cache_init(struct efa_ah_cache *ah_cache);
+void efa_ah_cache_destroy(struct efa_ah_cache *ah_cache);
+struct efa_ah_cache_entry *efa_ah_cache_get(struct efa_ah_cache *ah_cache, u16 pd, u8 *gid);
+struct efa_ah_cache_entry *efa_ah_cache_lookup(struct efa_ah_cache *ah_cache, u16 pd, u8 *gid);
+void efa_ah_cache_put(struct efa_ah_cache *ah_cache, struct efa_ah_cache_entry *entry);
+
+#endif /* _EFA_AH_CACHE_H_ */
diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c
index 7cc3f4af0bb9d..7097d1c2f23df 100644
--- a/drivers/infiniband/hw/efa/efa_com.c
+++ b/drivers/infiniband/hw/efa/efa_com.c
@@ -724,6 +724,8 @@ void efa_com_admin_destroy(struct efa_com_dev *edev)
 
 	size = aenq->depth * sizeof(*aenq->entries);
 	dma_free_coherent(edev->dmadev, size, aenq->entries, aenq->dma_addr);
+
+	efa_ah_cache_destroy(&edev->ah_cache);
 }
 
 /**
@@ -782,6 +784,12 @@ int efa_com_admin_init(struct efa_com_dev *edev,
 		return -ENODEV;
 	}
 
+	err = efa_ah_cache_init(&edev->ah_cache);
+	if (err) {
+		ibdev_err(edev->efa_dev, "Failed to init AH cache\n");
+		return err;
+	}
+
 	aq->depth = EFA_ADMIN_QUEUE_DEPTH;
 
 	aq->dmadev = edev->dmadev;
@@ -794,7 +802,7 @@ int efa_com_admin_init(struct efa_com_dev *edev,
 
 	err = efa_com_init_comp_ctxt(aq);
 	if (err)
-		return err;
+		goto err_destroy_ah_cache;
 
 	err = efa_com_admin_init_sq(edev);
 	if (err)
@@ -832,6 +840,8 @@ int efa_com_admin_init(struct efa_com_dev *edev,
 			  aq->sq.entries, aq->sq.dma_addr);
 err_destroy_comp_ctxt:
 	devm_kfree(edev->dmadev, aq->comp_ctx);
+err_destroy_ah_cache:
+	efa_ah_cache_destroy(&edev->ah_cache);
 
 	return err;
 }
diff --git a/drivers/infiniband/hw/efa/efa_com.h b/drivers/infiniband/hw/efa/efa_com.h
index f8c692b0e092e..599db9d583bf5 100644
--- a/drivers/infiniband/hw/efa/efa_com.h
+++ b/drivers/infiniband/hw/efa/efa_com.h
@@ -14,6 +14,7 @@
 
 #include <rdma/ib_verbs.h>
 
+#include "efa_ah_cache.h"
 #include "efa_common_defs.h"
 #include "efa_admin_defs.h"
 #include "efa_admin_cmds_defs.h"
@@ -113,6 +114,8 @@ struct efa_com_dev {
 	u32 supported_features;
 	u32 dma_addr_bits;
 
+	struct efa_ah_cache ah_cache;
+
 	u32 dev_api_ver;
 	struct efa_com_mmio_read mmio_read;
 };
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c
index 5db4f5805b599..0b96862c27870 100644
--- a/drivers/infiniband/hw/efa/efa_com_cmd.c
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.c
@@ -322,8 +322,21 @@ int efa_com_create_ah(struct efa_com_dev *edev,
 	struct efa_admin_create_ah_resp cmd_completion;
 	struct efa_com_admin_queue *aq = &edev->aq;
 	struct efa_admin_create_ah_cmd ah_cmd = {};
+	struct efa_ah_cache_entry *entry;
 	int err;
 
+	entry = efa_ah_cache_get(&edev->ah_cache, params->pdn, params->dest_addr);
+	if (IS_ERR(entry))
+		return PTR_ERR(entry);
+
+	mutex_lock(&entry->lock);
+	if (entry->usecnt) {
+		result->ah = entry->ah;
+		entry->usecnt++;
+		mutex_unlock(&entry->lock);
+		return 0;
+	}
+
 	ah_cmd.aq_common_desc.opcode = EFA_ADMIN_CREATE_AH;
 
 	memcpy(ah_cmd.dest_addr, params->dest_addr, sizeof(ah_cmd.dest_addr));
@@ -335,13 +348,18 @@ int efa_com_create_ah(struct efa_com_dev *edev,
 			       (struct efa_admin_acq_entry *)&cmd_completion,
 			       sizeof(cmd_completion));
 	if (err) {
+		mutex_unlock(&entry->lock);
+		efa_ah_cache_put(&edev->ah_cache, entry);
 		ibdev_err_ratelimited(edev->efa_dev,
 				      "Failed to create ah for %pI6 [%d]\n",
 				      ah_cmd.dest_addr, err);
 		return err;
 	}
 
+	entry->ah = cmd_completion.ah;
 	result->ah = cmd_completion.ah;
+	entry->usecnt++;
+	mutex_unlock(&entry->lock);
 
 	return 0;
 }
@@ -352,11 +370,20 @@ int efa_com_destroy_ah(struct efa_com_dev *edev,
 	struct efa_admin_destroy_ah_resp cmd_completion;
 	struct efa_admin_destroy_ah_cmd ah_cmd = {};
 	struct efa_com_admin_queue *aq = &edev->aq;
-	int err;
+	struct efa_ah_cache_entry *entry;
+	int err = 0;
+
+	entry = efa_ah_cache_lookup(&edev->ah_cache, params->pdn, params->gid);
+	if (!entry)
+		return -EINVAL;
+
+	mutex_lock(&entry->lock);
+	if (entry->usecnt > 1)
+		goto out_put;
 
 	ah_cmd.aq_common_desc.opcode = EFA_ADMIN_DESTROY_AH;
-	ah_cmd.ah = params->ah;
-	ah_cmd.pd = params->pdn;
+	ah_cmd.ah = entry->ah;
+	ah_cmd.pd = entry->key.pd;
 
 	err = efa_com_cmd_exec(aq,
 			       (struct efa_admin_aq_entry *)&ah_cmd,
@@ -364,13 +391,19 @@ int efa_com_destroy_ah(struct efa_com_dev *edev,
 			       (struct efa_admin_acq_entry *)&cmd_completion,
 			       sizeof(cmd_completion));
 	if (err) {
+		mutex_unlock(&entry->lock);
 		ibdev_err_ratelimited(edev->efa_dev,
 				      "Failed to destroy ah-%d pd-%d [%d]\n",
 				      ah_cmd.ah, ah_cmd.pd, err);
 		return err;
 	}
 
-	return 0;
+out_put:
+	entry->usecnt--;
+	mutex_unlock(&entry->lock);
+	efa_ah_cache_put(&edev->ah_cache, entry);
+
+	return err;
 }
 
 bool
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h
index ef15b3c38429c..39bd4e06684a5 100644
--- a/drivers/infiniband/hw/efa/efa_com_cmd.h
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.h
@@ -106,6 +106,7 @@ struct efa_com_create_ah_result {
 
 struct efa_com_destroy_ah_params {
 	u16 ah;
+	u8 gid[EFA_GID_SIZE];
 	u16 pdn;
 };
 
diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c
index 97da8e828e340..ee09dc1e0b437 100644
--- a/drivers/infiniband/hw/efa/efa_main.c
+++ b/drivers/infiniband/hw/efa/efa_main.c
@@ -17,12 +17,14 @@
 #define PCI_DEV_ID_EFA1_VF 0xefa1
 #define PCI_DEV_ID_EFA2_VF 0xefa2
 #define PCI_DEV_ID_EFA3_VF 0xefa3
+#define PCI_DEV_ID_EFA4_VF 0xefa4
 
 static const struct pci_device_id efa_pci_tbl[] = {
 	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA0_VF) },
 	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA1_VF) },
 	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA2_VF) },
 	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA3_VF) },
+	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA4_VF) },
 	{ }
 };
 
diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c
index 06d3365aeb569..6be8ec53dcb34 100644
--- a/drivers/infiniband/hw/efa/efa_verbs.c
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -222,7 +222,6 @@ int efa_query_device(struct ib_device *ibdev,
 
 	dev_attr = &dev->dev_attr;
 
-	memset(props, 0, sizeof(*props));
 	props->max_mr_size = dev_attr->max_mr_pages * PAGE_SIZE;
 	props->page_size_cap = dev_attr->page_size_cap;
 	props->vendor_id = dev->pdev->vendor;
@@ -2054,10 +2053,11 @@ int efa_mmap(struct ib_ucontext *ibucontext,
 
 static int efa_ah_destroy(struct efa_dev *dev, struct efa_ah *ah)
 {
-	struct efa_com_destroy_ah_params params = {
-		.ah = ah->ah,
-		.pdn = to_epd(ah->ibah.pd)->pdn,
-	};
+	struct efa_com_destroy_ah_params params = {};
+
+	params.ah = ah->ah;
+	memcpy(params.gid, ah->id, sizeof(params.gid));
+	params.pdn = to_epd(ah->ibah.pd)->pdn;
 
 	return efa_com_destroy_ah(&dev->edev, &params);
 }
diff --git a/drivers/infiniband/hw/erdma/erdma_qp.c b/drivers/infiniband/hw/erdma/erdma_qp.c
index 25f6c49aec779..e002343832f74 100644
--- a/drivers/infiniband/hw/erdma/erdma_qp.c
+++ b/drivers/infiniband/hw/erdma/erdma_qp.c
@@ -734,7 +734,7 @@ int erdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *recv_wr,
 	const struct ib_recv_wr *wr = recv_wr;
 	struct erdma_qp *qp = to_eqp(ibqp);
 	unsigned long flags;
-	int ret;
+	int ret = 0;
 
 	spin_lock_irqsave(&qp->lock, flags);
 
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c
index 74afe6eb18b0b..9491cbab69b34 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.c
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.c
@@ -324,8 +324,6 @@ int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
 	if (err)
 		return err;
 
-	memset(attr, 0, sizeof(*attr));
-
 	attr->max_mr_size = dev->attrs.max_mr_size;
 	attr->vendor_id = PCI_VENDOR_ID_ALIBABA;
 	attr->vendor_part_id = dev->pdev->device;
diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c
index 44c524e453968..592e330e74bf1 100644
--- a/drivers/infiniband/hw/hfi1/chip.c
+++ b/drivers/infiniband/hw/hfi1/chip.c
@@ -11929,26 +11929,27 @@ u8 encode_rcv_header_entry_size(u8 size)
 
 /**
  * hfi1_validate_rcvhdrcnt - validate hdrcnt
- * @dd: the device data
+ * @pdev: the PCI device
  * @thecnt: the header count
  */
-int hfi1_validate_rcvhdrcnt(struct hfi1_devdata *dd, uint thecnt)
+int hfi1_validate_rcvhdrcnt(struct pci_dev *pdev, uint thecnt)
 {
 	if (thecnt <= HFI1_MIN_HDRQ_EGRBUF_CNT) {
-		dd_dev_err(dd, "Receive header queue count too small\n");
+		dev_err(&pdev->dev, "Receive header queue count too small\n");
 		return -EINVAL;
 	}
 
 	if (thecnt > HFI1_MAX_HDRQ_EGRBUF_CNT) {
-		dd_dev_err(dd,
-			   "Receive header queue count cannot be greater than %u\n",
-			   HFI1_MAX_HDRQ_EGRBUF_CNT);
+		dev_err(&pdev->dev,
+			"Receive header queue count cannot be greater than %u\n",
+			HFI1_MAX_HDRQ_EGRBUF_CNT);
 		return -EINVAL;
 	}
 
 	if (thecnt % HDRQ_INCREMENT) {
-		dd_dev_err(dd, "Receive header queue count %d must be divisible by %lu\n",
-			   thecnt, HDRQ_INCREMENT);
+		dev_err(&pdev->dev,
+			"Receive header queue count %u must be divisible by %lu\n",
+			thecnt, HDRQ_INCREMENT);
 		return -EINVAL;
 	}
 
@@ -15007,7 +15008,7 @@ int hfi1_init_dd(struct hfi1_devdata *dd)
 	 */
 	ret = hfi1_pcie_ddinit(dd, pdev);
 	if (ret < 0)
-		goto bail_free;
+		goto bail;
 
 	/* Save PCI space registers to rewrite after device reset */
 	ret = save_pci_variables(dd);
@@ -15262,8 +15263,6 @@ int hfi1_init_dd(struct hfi1_devdata *dd)
 bail_cleanup:
 	hfi1_free_rx(dd);
 	hfi1_pcie_ddcleanup(dd);
-bail_free:
-	hfi1_free_devdata(dd);
 bail:
 	return ret;
 }
diff --git a/drivers/infiniband/hw/hfi1/chip.h b/drivers/infiniband/hw/hfi1/chip.h
index 56e03d486acef..bc1c9e1c3172c 100644
--- a/drivers/infiniband/hw/hfi1/chip.h
+++ b/drivers/infiniband/hw/hfi1/chip.h
@@ -660,7 +660,7 @@ static inline u32 chip_rcv_array_count(struct hfi1_devdata *dd)
 }
 
 u8 encode_rcv_header_entry_size(u8 size);
-int hfi1_validate_rcvhdrcnt(struct hfi1_devdata *dd, uint thecnt);
+int hfi1_validate_rcvhdrcnt(struct pci_dev *pdev, uint thecnt);
 void set_hdrq_regs(struct hfi1_devdata *dd, u8 ctxt, u8 entsize, u16 hdrcnt);
 
 u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl,
diff --git a/drivers/infiniband/hw/hfi1/device.c b/drivers/infiniband/hw/hfi1/device.c
index a98a4175e53bb..adcfb80d52d48 100644
--- a/drivers/infiniband/hw/hfi1/device.c
+++ b/drivers/infiniband/hw/hfi1/device.c
@@ -10,18 +10,6 @@
 #include "hfi.h"
 #include "device.h"
 
-static char *hfi1_devnode(const struct device *dev, umode_t *mode)
-{
-	if (mode)
-		*mode = 0600;
-	return kasprintf(GFP_KERNEL, "%s", dev_name(dev));
-}
-
-static const struct class class = {
-	.name = "hfi1",
-	.devnode = hfi1_devnode,
-};
-
 static char *hfi1_user_devnode(const struct device *dev, umode_t *mode)
 {
 	if (mode)
@@ -38,7 +26,6 @@ static dev_t hfi1_dev;
 int hfi1_cdev_init(int minor, const char *name,
 		   const struct file_operations *fops,
 		   struct cdev *cdev, struct device **devp,
-		   bool user_accessible,
 		   struct kobject *parent)
 {
 	const dev_t dev = MKDEV(MAJOR(hfi1_dev), minor);
@@ -57,10 +44,7 @@ int hfi1_cdev_init(int minor, const char *name,
 		goto done;
 	}
 
-	if (user_accessible)
-		device = device_create(&user_class, NULL, dev, NULL, "%s", name);
-	else
-		device = device_create(&class, NULL, dev, NULL, "%s", name);
+	device = device_create(&user_class, NULL, dev, NULL, "%s", name);
 
 	if (IS_ERR(device)) {
 		ret = PTR_ERR(device);
@@ -100,33 +84,21 @@ int __init dev_init(void)
 	ret = alloc_chrdev_region(&hfi1_dev, 0, HFI1_NMINORS, DRIVER_NAME);
 	if (ret < 0) {
 		pr_err("Could not allocate chrdev region (err %d)\n", -ret);
-		goto done;
-	}
-
-	ret = class_register(&class);
-	if (ret) {
-		pr_err("Could not create device class (err %d)\n", -ret);
-		unregister_chrdev_region(hfi1_dev, HFI1_NMINORS);
-		goto done;
+		return ret;
 	}
 
 	ret = class_register(&user_class);
 	if (ret) {
 		pr_err("Could not create device class for user accessible files (err %d)\n",
 		       -ret);
-		class_unregister(&class);
 		unregister_chrdev_region(hfi1_dev, HFI1_NMINORS);
-		goto done;
 	}
 
-done:
 	return ret;
 }
 
 void dev_cleanup(void)
 {
-	class_unregister(&class);
 	class_unregister(&user_class);
-
 	unregister_chrdev_region(hfi1_dev, HFI1_NMINORS);
 }
diff --git a/drivers/infiniband/hw/hfi1/device.h b/drivers/infiniband/hw/hfi1/device.h
index a91bea426ba5a..3e2d21770e6c0 100644
--- a/drivers/infiniband/hw/hfi1/device.h
+++ b/drivers/infiniband/hw/hfi1/device.h
@@ -9,7 +9,6 @@
 int hfi1_cdev_init(int minor, const char *name,
 		   const struct file_operations *fops,
 		   struct cdev *cdev, struct device **devp,
-		   bool user_accessible,
 		   struct kobject *parent);
 void hfi1_cdev_cleanup(struct cdev *cdev, struct device **devp);
 const char *class_name(void);
diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c
index 56031becb2731..dc548e6802e24 100644
--- a/drivers/infiniband/hw/hfi1/file_ops.c
+++ b/drivers/infiniband/hw/hfi1/file_ops.c
@@ -1689,7 +1689,7 @@ static int user_add(struct hfi1_devdata *dd)
 	snprintf(name, sizeof(name), "%s_%d", class_name(), dd->unit);
 	ret = hfi1_cdev_init(dd->unit, name, &hfi1_file_ops,
 			     &dd->user_cdev, &dd->user_device,
-			     true, &dd->verbs_dev.rdi.ibdev.dev.kobj);
+			     &dd->verbs_dev.rdi.ibdev.dev.kobj);
 	if (ret)
 		user_remove(dd);
 
diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h
index 5a0310f758dc2..80d480c4fc6a8 100644
--- a/drivers/infiniband/hw/hfi1/hfi.h
+++ b/drivers/infiniband/hw/hfi1/hfi.h
@@ -2024,9 +2024,7 @@ struct cc_state *get_cc_state_protected(struct hfi1_pportdata *ppd)
 		/* waiting for an urgent packet to arrive */
 #define HFI1_CTXT_WAITING_URG 4
 
-/* free up any allocated data at closes */
 int hfi1_init_dd(struct hfi1_devdata *dd);
-void hfi1_free_devdata(struct hfi1_devdata *dd);
 
 /* LED beaconing functions */
 void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon,
@@ -2132,7 +2130,7 @@ void hfi1_verbs_unregister_sysfs(struct hfi1_devdata *dd);
 /* Hook for sysfs read of QSFP */
 int qsfp_dump(struct hfi1_pportdata *ppd, char *buf, int len);
 
-int hfi1_pcie_init(struct hfi1_devdata *dd);
+int hfi1_pcie_init(struct pci_dev *pdev);
 void hfi1_pcie_cleanup(struct pci_dev *pdev);
 int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev);
 void hfi1_pcie_ddcleanup(struct hfi1_devdata *);
diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c
index b7fd8b1fbbbde..f94d896e7212d 100644
--- a/drivers/infiniband/hw/hfi1/init.c
+++ b/drivers/infiniband/hw/hfi1/init.c
@@ -629,8 +629,6 @@ void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd,
 	ppd->sm_trap_qp = 0x0;
 	ppd->sa_qp = 0x1;
 
-	ppd->hfi1_wq = NULL;
-
 	spin_lock_init(&ppd->cca_timer_lock);
 
 	for (i = 0; i < OPA_MAX_SLS; i++) {
@@ -740,31 +738,27 @@ static int create_workqueues(struct hfi1_devdata *dd)
 
 	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
 		ppd = dd->pport + pidx;
-		if (!ppd->hfi1_wq) {
-			ppd->hfi1_wq =
-				alloc_workqueue(
-				    "hfi%d_%d",
-				    WQ_SYSFS | WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM |
-				    WQ_PERCPU,
-				    HFI1_MAX_ACTIVE_WORKQUEUE_ENTRIES,
-				    dd->unit, pidx);
-			if (!ppd->hfi1_wq)
-				goto wq_error;
-		}
-		if (!ppd->link_wq) {
-			/*
-			 * Make the link workqueue single-threaded to enforce
-			 * serialization.
-			 */
-			ppd->link_wq =
-				alloc_workqueue(
-				    "hfi_link_%d_%d",
-				    WQ_SYSFS | WQ_MEM_RECLAIM | WQ_UNBOUND,
-				    1, /* max_active */
-				    dd->unit, pidx);
-			if (!ppd->link_wq)
-				goto wq_error;
-		}
+		ppd->hfi1_wq =
+			alloc_workqueue(
+			    "hfi%d_%d",
+			    WQ_SYSFS | WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM |
+			    WQ_PERCPU,
+			    HFI1_MAX_ACTIVE_WORKQUEUE_ENTRIES,
+			    dd->unit, pidx);
+		if (!ppd->hfi1_wq)
+			goto wq_error;
+		/*
+		 * Make the link workqueue single-threaded to enforce
+		 * serialization.
+		 */
+		ppd->link_wq =
+			alloc_workqueue(
+			    "hfi_link_%d_%d",
+			    WQ_SYSFS | WQ_MEM_RECLAIM | WQ_UNBOUND,
+			    1, /* max_active */
+			    dd->unit, pidx);
+		if (!ppd->link_wq)
+			goto wq_error;
 	}
 	return 0;
 wq_error:
@@ -1161,7 +1155,7 @@ static void finalize_asic_data(struct hfi1_devdata *dd,
  * It cleans up and frees all data structures set up by
  * by hfi1_alloc_devdata().
  */
-void hfi1_free_devdata(struct hfi1_devdata *dd)
+static void hfi1_free_devdata(struct hfi1_devdata *dd)
 {
 	struct hfi1_asic_data *ad;
 	unsigned long flags;
@@ -1225,8 +1219,9 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev,
 			GFP_KERNEL);
 	if (ret < 0) {
 		dev_err(&pdev->dev,
-			"Could not allocate unit ID: error %d\n", -ret);
-		goto bail;
+			"Could not allocate unit ID: error %pe\n", ERR_PTR(ret));
+		rvt_dealloc_device(&dd->verbs_dev.rdi);
+		return ERR_PTR(ret);
 	}
 
 	/*
@@ -1554,50 +1549,28 @@ static void postinit_cleanup(struct hfi1_devdata *dd)
 	hfi1_dev_affinity_clean_up(dd);
 
 	hfi1_pcie_ddcleanup(dd);
-	hfi1_pcie_cleanup(dd->pcidev);
 
 	cleanup_device_data(dd);
-
-	hfi1_free_devdata(dd);
 }
 
 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
-	int ret = 0, j, pidx, initfail;
+	int ret;
 	struct hfi1_devdata *dd;
-	struct hfi1_pportdata *ppd;
 
 	/* First, lock the non-writable module parameters */
 	HFI1_CAP_LOCK();
 
-	/* Validate dev ids */
-	if (!(ent->device == PCI_DEVICE_ID_INTEL0 ||
-	      ent->device == PCI_DEVICE_ID_INTEL1)) {
-		dev_err(&pdev->dev, "Failing on unknown Intel deviceid 0x%x\n",
-			ent->device);
-		ret = -ENODEV;
-		goto bail;
-	}
-
-	/* Allocate the dd so we can get to work */
-	dd = hfi1_alloc_devdata(pdev, NUM_IB_PORTS *
-				sizeof(struct hfi1_pportdata));
-	if (IS_ERR(dd)) {
-		ret = PTR_ERR(dd);
-		goto bail;
-	}
-
 	/* Validate some global module parameters */
-	ret = hfi1_validate_rcvhdrcnt(dd, rcvhdrcnt);
+	ret = hfi1_validate_rcvhdrcnt(pdev, rcvhdrcnt);
 	if (ret)
-		goto bail;
+		return ret;
 
 	/* use the encoding function as a sanitization check */
 	if (!encode_rcv_header_entry_size(hfi1_hdrq_entsize)) {
-		dd_dev_err(dd, "Invalid HdrQ Entry size %u\n",
-			   hfi1_hdrq_entsize);
-		ret = -EINVAL;
-		goto bail;
+		dev_err(&pdev->dev, "Invalid HdrQ Entry size %u\n",
+			hfi1_hdrq_entsize);
+		return -EINVAL;
 	}
 
 	/* The receive eager buffer size must be set before the receive
@@ -1617,87 +1590,75 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 			clamp_val(eager_buffer_size,
 				  MIN_EAGER_BUFFER * 8,
 				  MAX_EAGER_BUFFER_TOTAL);
-		dd_dev_info(dd, "Eager buffer size %u\n",
-			    eager_buffer_size);
+		pci_info(pdev, "Eager buffer size %u\n", eager_buffer_size);
 	} else {
-		dd_dev_err(dd, "Invalid Eager buffer size of 0\n");
-		ret = -EINVAL;
-		goto bail;
+		dev_err(&pdev->dev, "Invalid Eager buffer size of 0\n");
+		return -EINVAL;
 	}
 
 	/* restrict value of hfi1_rcvarr_split */
 	hfi1_rcvarr_split = clamp_val(hfi1_rcvarr_split, 0, 100);
 
-	ret = hfi1_pcie_init(dd);
+	ret = hfi1_pcie_init(pdev);
 	if (ret)
-		goto bail;
+		return ret;
+
+	/* Allocate the dd so we can get to work */
+	dd = hfi1_alloc_devdata(pdev, NUM_IB_PORTS *
+				sizeof(struct hfi1_pportdata));
+	if (IS_ERR(dd)) {
+		ret = PTR_ERR(dd);
+		goto clean_pcie;
+	}
+
+	ret = create_workqueues(dd);
+	if (ret)
+		goto free_devdata;
 
 	/*
-	 * Do device-specific initialization, function table setup, dd
-	 * allocation, etc.
+	 * Do device-specific initialization, function table setup, etc.
 	 */
 	ret = hfi1_init_dd(dd);
 	if (ret)
-		goto clean_bail; /* error already printed */
-
-	ret = create_workqueues(dd);
-	if (ret)
-		goto clean_bail;
+		goto destroy_workqueues; /* error already printed */
 
 	/* do the generic initialization */
-	initfail = hfi1_init(dd, 0);
+	ret = hfi1_init(dd, 0);
+	if (ret)
+		goto free_rx;
 
 	ret = hfi1_register_ib_device(dd);
+	if (ret)
+		goto free_rx;
 
 	/*
 	 * Now ready for use.  this should be cleared whenever we
-	 * detect a reset, or initiate one.  If earlier failure,
-	 * we still create devices, so diags, etc. can be used
-	 * to determine cause of problem.
+	 * detect a reset, or initiate one.
 	 */
-	if (!initfail && !ret) {
-		dd->flags |= HFI1_INITTED;
-		/* create debufs files after init and ib register */
-		hfi1_dbg_ibdev_init(&dd->verbs_dev);
-	}
+	dd->flags |= HFI1_INITTED;
 
-	j = hfi1_device_create(dd);
-	if (j)
-		dd_dev_err(dd, "Failed to create /dev devices: %d\n", -j);
-
-	if (initfail || ret) {
-		msix_clean_up_interrupts(dd);
-		stop_timers(dd);
-		flush_workqueue(ib_wq);
-		for (pidx = 0; pidx < dd->num_pports; ++pidx) {
-			hfi1_quiet_serdes(dd->pport + pidx);
-			ppd = dd->pport + pidx;
-			if (ppd->hfi1_wq) {
-				destroy_workqueue(ppd->hfi1_wq);
-				ppd->hfi1_wq = NULL;
-			}
-			if (ppd->link_wq) {
-				destroy_workqueue(ppd->link_wq);
-				ppd->link_wq = NULL;
-			}
-		}
-		if (!j)
-			hfi1_device_remove(dd);
-		if (!ret)
-			hfi1_unregister_ib_device(dd);
-		postinit_cleanup(dd);
-		if (initfail)
-			ret = initfail;
-		goto bail;	/* everything already cleaned */
-	}
+	ret = hfi1_device_create(dd);
+	if (ret)
+		dd_dev_err(dd, "Failed to create /dev devices: %pe\n",
+			   ERR_PTR(ret));
 
 	sdma_start(dd);
+	hfi1_dbg_ibdev_init(&dd->verbs_dev);
 
 	return 0;
 
-clean_bail:
+free_rx:
+	hfi1_free_rx(dd);
+	shutdown_device(dd);
+	stop_timers(dd);
+	postinit_cleanup(dd);
+
+destroy_workqueues:
+	destroy_workqueues(dd);
+free_devdata:
+	hfi1_free_devdata(dd);
+clean_pcie:
 	hfi1_pcie_cleanup(pdev);
-bail:
 	return ret;
 }
 
@@ -1737,14 +1698,11 @@ static void remove_one(struct pci_dev *pdev)
 	 * clear dma engines, etc.
 	 */
 	shutdown_device(dd);
-	destroy_workqueues(dd);
-
 	stop_timers(dd);
-
-	/* wait until all of our (qsfp) queue_work() calls complete */
-	flush_workqueue(ib_wq);
-
 	postinit_cleanup(dd);
+	destroy_workqueues(dd);
+	hfi1_free_devdata(dd);
+	hfi1_pcie_cleanup(pdev);
 }
 
 static void shutdown_one(struct pci_dev *pdev)
diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c
index 7133964749f80..1154b8cc713ce 100644
--- a/drivers/infiniband/hw/hfi1/pcie.c
+++ b/drivers/infiniband/hw/hfi1/pcie.c
@@ -21,10 +21,9 @@
 /*
  * Do all the common PCIe setup and initialization.
  */
-int hfi1_pcie_init(struct hfi1_devdata *dd)
+int hfi1_pcie_init(struct pci_dev *pdev)
 {
 	int ret;
-	struct pci_dev *pdev = dd->pcidev;
 
 	ret = pci_enable_device(pdev);
 	if (ret) {
@@ -40,13 +39,15 @@ int hfi1_pcie_init(struct hfi1_devdata *dd)
 		 * about that, it appears.  If the original BAR was retained
 		 * in the kernel data structures, this may be OK.
 		 */
-		dd_dev_err(dd, "pci enable failed: error %d\n", -ret);
+		dev_err(&pdev->dev, "pci enable failed: error %pe\n",
+			ERR_PTR(ret));
 		return ret;
 	}
 
 	ret = pci_request_regions(pdev, DRIVER_NAME);
 	if (ret) {
-		dd_dev_err(dd, "pci_request_regions fails: err %d\n", -ret);
+		dev_err(&pdev->dev, "pci_request_regions fails: err %pe\n",
+			ERR_PTR(ret));
 		goto bail;
 	}
 
@@ -59,7 +60,8 @@ int hfi1_pcie_init(struct hfi1_devdata *dd)
 		 */
 		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 		if (ret) {
-			dd_dev_err(dd, "Unable to set DMA mask: %d\n", ret);
+			dev_err(&pdev->dev, "Unable to set DMA mask: %pe\n",
+				ERR_PTR(ret));
 			goto bail;
 		}
 	}
@@ -512,29 +514,28 @@ pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
 
 	switch (state) {
 	case pci_channel_io_normal:
-		dd_dev_info(dd, "State Normal, ignoring\n");
+		dev_info(&pdev->dev, "State Normal, ignoring\n");
 		break;
 
 	case pci_channel_io_frozen:
-		dd_dev_info(dd, "State Frozen, requesting reset\n");
+		dev_info(&pdev->dev, "State Frozen, requesting reset\n");
 		pci_disable_device(pdev);
 		ret = PCI_ERS_RESULT_NEED_RESET;
 		break;
 
 	case pci_channel_io_perm_failure:
+		dev_info(&pdev->dev, "State Permanent Failure, disabling\n");
 		if (dd) {
-			dd_dev_info(dd, "State Permanent Failure, disabling\n");
 			/* no more register accesses! */
 			dd->flags &= ~HFI1_PRESENT;
 			hfi1_disable_after_error(dd);
 		}
-		 /* else early, or other problem */
 		ret =  PCI_ERS_RESULT_DISCONNECT;
 		break;
 
 	default: /* shouldn't happen */
-		dd_dev_info(dd, "HFI1 PCI errors detected (state %d)\n",
-			    state);
+		dev_info(&pdev->dev, "HFI1 PCI errors detected (state %d)\n",
+			 state);
 		break;
 	}
 	return ret;
@@ -561,9 +562,7 @@ pci_mmio_enabled(struct pci_dev *pdev)
 static pci_ers_result_t
 pci_slot_reset(struct pci_dev *pdev)
 {
-	struct hfi1_devdata *dd = pci_get_drvdata(pdev);
-
-	dd_dev_info(dd, "HFI1 slot_reset function called, ignored\n");
+	dev_info(&pdev->dev, "HFI1 slot_reset function called, ignored\n");
 	return PCI_ERS_RESULT_CAN_RECOVER;
 }
 
@@ -572,7 +571,10 @@ pci_resume(struct pci_dev *pdev)
 {
 	struct hfi1_devdata *dd = pci_get_drvdata(pdev);
 
-	dd_dev_info(dd, "HFI1 resume function called\n");
+	dev_info(&pdev->dev, "HFI1 resume function called\n");
+	if (!dd)
+		return;
+
 	/*
 	 * Running jobs will fail, since it's asynchronous
 	 * unlike sysfs-requested reset.   Better than
diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c
index cfd9dd0f7e817..f253c8ee182d5 100644
--- a/drivers/infiniband/hw/hfi1/sdma.c
+++ b/drivers/infiniband/hw/hfi1/sdma.c
@@ -11,6 +11,7 @@
 #include <linux/timer.h>
 #include <linux/vmalloc.h>
 #include <linux/highmem.h>
+#include <linux/sysfs.h>
 
 #include "hfi.h"
 #include "common.h"
@@ -1049,7 +1050,7 @@ ssize_t sdma_get_cpu_to_sde_map(struct sdma_engine *sde, char *buf)
 	if (cpumask_empty(&sde->cpu_mask))
 		snprintf(buf, PAGE_SIZE, "%s\n", "empty");
 	else
-		cpumap_print_to_pagebuf(true, buf, &sde->cpu_mask);
+		sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&sde->cpu_mask));
 	mutex_unlock(&process_to_sde_mutex);
 	return strnlen(buf, PAGE_SIZE);
 }
diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c
index 7041a8e9134b2..92edec4fa61b6 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hem.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hem.c
@@ -836,7 +836,7 @@ static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,
 					mhop.bt_chunk_size;
 
 	for (i = 0; i < table->num_hem; ++i) {
-		obj = i * buf_chunk_size / table->obj_size;
+		obj = (u64)i * buf_chunk_size / table->obj_size;
 		if (table->hem[i])
 			hns_roce_table_mhop_put(hr_dev, table, obj, 0);
 	}
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index 2b3a1cafd1b21..27cc7df55ee70 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -4393,7 +4393,7 @@ static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
 	struct hns_roce_hem_mhop mhop;
 	struct hns_roce_hem *hem;
 	unsigned long mhop_obj = obj;
-	int i, j, k;
+	u64 i, j, k;
 	int ret = 0;
 	u64 hem_idx = 0;
 	u64 l1_idx = 0;
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index c6f633bd5a340..09c07de5f0222 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -227,8 +227,6 @@ static int hns_roce_query_device(struct ib_device *ib_dev,
 	if (ret)
 		return ret;
 
-	memset(props, 0, sizeof(*props));
-
 	props->fw_ver = hr_dev->caps.fw_ver;
 	props->sys_image_guid = cpu_to_be64(hr_dev->sys_image_guid);
 	props->max_mr_size = (u64)(~(0ULL));
diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.h b/drivers/infiniband/hw/ionic/ionic_ibdev.h
index 63828240d6590..53dab2d548441 100644
--- a/drivers/infiniband/hw/ionic/ionic_ibdev.h
+++ b/drivers/infiniband/hw/ionic/ionic_ibdev.h
@@ -33,7 +33,6 @@
 #define IONIC_MAX_QPID 0xffffff
 #define IONIC_SPEC_HIGH 8
 #define IONIC_MAX_PD 1024
-#define IONIC_SPEC_HIGH 8
 #define IONIC_SQCMB_ORDER 5
 #define IONIC_RQCMB_ORDER 0
 
diff --git a/drivers/infiniband/hw/irdma/i40iw_hw.c b/drivers/infiniband/hw/irdma/i40iw_hw.c
index 60c1f2b1811de..8301938b45432 100644
--- a/drivers/infiniband/hw/irdma/i40iw_hw.c
+++ b/drivers/infiniband/hw/irdma/i40iw_hw.c
@@ -29,7 +29,9 @@ static u32 i40iw_regs[IRDMA_MAX_REGS] = {
 	I40E_PFHMC_PDINV,
 	I40E_GLHMC_VFPDINV(0),
 	I40E_GLPE_CRITERR,
-	0xffffffff      /* PFINT_RATEN not used in FPK */
+	0xffffffff,     /* PFINT_RATEN not used in FPK */
+	0xffffffff,     /* PFHMC_ERRORINFO not used in FPK */
+	0xffffffff      /* PFHMC_ERRORDATA not used in FPK */
 };
 
 static u32 i40iw_stat_offsets[] = {
diff --git a/drivers/infiniband/hw/irdma/icrdma_hw.c b/drivers/infiniband/hw/irdma/icrdma_hw.c
index 32f26284a788b..b1f1b54857626 100644
--- a/drivers/infiniband/hw/irdma/icrdma_hw.c
+++ b/drivers/infiniband/hw/irdma/icrdma_hw.c
@@ -29,6 +29,8 @@ static u32 icrdma_regs[IRDMA_MAX_REGS] = {
 	GLHMC_VFPDINV(0),
 	GLPE_CRITERR,
 	GLINT_RATE(0),
+	PFHMC_ERRORINFO,
+	PFHMC_ERRORDATA,
 };
 
 static u64 icrdma_masks[IRDMA_MAX_MASKS] = {
diff --git a/drivers/infiniband/hw/irdma/icrdma_hw.h b/drivers/infiniband/hw/irdma/icrdma_hw.h
index d97944ab45dac..0acdeda1236d1 100644
--- a/drivers/infiniband/hw/irdma/icrdma_hw.h
+++ b/drivers/infiniband/hw/irdma/icrdma_hw.h
@@ -40,6 +40,8 @@
 #define GLHMC_VFPDINV(_i)	(0x00528300 + ((_i) * 4)) /* _i=0...31 */
 #define GLPE_CRITERR		0x00534000
 #define GLINT_RATE(_INT)	(0x0015A000 + ((_INT) * 4)) /* _i=0...2047 */ /* Reset Source: CORER */
+#define PFHMC_ERRORINFO		0x00520400
+#define PFHMC_ERRORDATA		0x00520500
 
 #define ICRDMA_DB_ADDR_OFFSET		(8 * 1024 * 1024 - 64 * 1024)
 
diff --git a/drivers/infiniband/hw/irdma/icrdma_if.c b/drivers/infiniband/hw/irdma/icrdma_if.c
index 2172a2092e3f9..4b451d8482a48 100644
--- a/drivers/infiniband/hw/irdma/icrdma_if.c
+++ b/drivers/infiniband/hw/irdma/icrdma_if.c
@@ -91,8 +91,12 @@ static void icrdma_iidc_event_handler(struct iidc_rdma_core_dev_info *cdev_info,
 			}
 		}
 		if (event->reg & IRDMAPFINT_OICR_HMC_ERR_M) {
-			ibdev_err(&iwdev->ibdev, "HMC Error\n");
-			iwdev->rf->reset = true;
+			u32 hmc_errinfo = readl(iwdev->rf->sc_dev.hw_regs[IRDMA_PFHMC_ERRORINFO]);
+			u32 hmc_errdata = readl(iwdev->rf->sc_dev.hw_regs[IRDMA_PFHMC_ERRORDATA]);
+
+			/* Log diagnostics; do not reset here. */
+			ibdev_warn(&iwdev->ibdev, "HMC Error: errinfo=0x%08x errdata=0x%08x\n",
+				   hmc_errinfo, hmc_errdata);
 		}
 		if (event->reg & IRDMAPFINT_OICR_PE_PUSH_M) {
 			ibdev_err(&iwdev->ibdev, "PE Push Error\n");
diff --git a/drivers/infiniband/hw/irdma/irdma.h b/drivers/infiniband/hw/irdma/irdma.h
index b5ce515f4ee83..d4530520e39fd 100644
--- a/drivers/infiniband/hw/irdma/irdma.h
+++ b/drivers/infiniband/hw/irdma/irdma.h
@@ -66,6 +66,8 @@ enum irdma_registers {
 	IRDMA_GLHMC_VFPDINV,
 	IRDMA_GLPE_CRITERR,
 	IRDMA_GLINT_RATE,
+	IRDMA_PFHMC_ERRORINFO,
+	IRDMA_PFHMC_ERRORDATA,
 	IRDMA_MAX_REGS, /* Must be last entry */
 };
 
diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c
index e4037d5ef8993..290ad02ed6575 100644
--- a/drivers/infiniband/hw/irdma/utils.c
+++ b/drivers/infiniband/hw/irdma/utils.c
@@ -1168,6 +1168,12 @@ void irdma_free_qp_rsrc(struct irdma_qp *iwqp)
 	iwqp->kqp.dma_mem.va = NULL;
 	kfree(iwqp->kqp.sq_wrid_mem);
 	kfree(iwqp->kqp.rq_wrid_mem);
+
+	if (iwqp->user_mode && iwqp->iwpbl) {
+		struct irdma_mr *iwmr = iwqp->iwpbl->iwmr;
+
+		refcount_dec(&iwmr->user_ring_refs);
+	}
 }
 
 /**
diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c
index cb54c7c8fcd8f..f9a6042bd4c32 100644
--- a/drivers/infiniband/hw/irdma/verbs.c
+++ b/drivers/infiniband/hw/irdma/verbs.c
@@ -22,7 +22,6 @@ static int irdma_query_device(struct ib_device *ibdev,
 	if (err)
 		return err;
 
-	memset(props, 0, sizeof(*props));
 	addrconf_addr_eui48((u8 *)&props->sys_image_guid,
 			    iwdev->netdev->dev_addr);
 	props->fw_ver = (u64)irdma_fw_major_ver(&rf->sc_dev) << 32 |
@@ -464,6 +463,9 @@ static struct irdma_pbl *irdma_get_pbl(unsigned long va,
 
 	list_for_each_entry (iwpbl, pbl_list, list) {
 		if (iwpbl->user_base == va) {
+			struct irdma_mr *iwmr = iwpbl->iwmr;
+
+			refcount_inc(&iwmr->user_ring_refs);
 			list_del(&iwpbl->list);
 			iwpbl->on_list = false;
 			return iwpbl;
@@ -633,17 +635,16 @@ static int irdma_setup_umode_qp(struct ib_udata *udata,
 
 	iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx;
 	iwqp->user_mode = 1;
-	if (req.user_wqe_bufs) {
-		spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
-		iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs,
-					    &ucontext->qp_reg_mem_list);
-		spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
-
-		if (!iwqp->iwpbl) {
-			ret = -ENODATA;
-			ibdev_dbg(&iwdev->ibdev, "VERBS: no pbl info\n");
-			return ret;
-		}
+
+	spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
+	iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs,
+				    &ucontext->qp_reg_mem_list);
+	spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
+
+	if (!iwqp->iwpbl) {
+		ret = -ENODATA;
+		ibdev_dbg(&iwdev->ibdev, "VERBS: no pbl info\n");
+		return ret;
 	}
 
 	if (!ucontext->use_raw_attrs) {
@@ -1881,6 +1882,11 @@ static void irdma_srq_free_rsrc(struct irdma_pci_f *rf, struct irdma_srq *iwsrq)
 		dma_free_coherent(rf->sc_dev.hw->device, iwsrq->kmem.size,
 				  iwsrq->kmem.va, iwsrq->kmem.pa);
 		iwsrq->kmem.va = NULL;
+	} else {
+		/* Not called in any failure path, so iwpbl is valid. */
+		struct irdma_mr *iwmr = iwsrq->iwpbl->iwmr;
+
+		refcount_dec(&iwmr->user_ring_refs);
 	}
 
 	irdma_free_rsrc(rf, rf->allocated_srqs, srq->srq_uk.srq_id);
@@ -1903,6 +1909,21 @@ static void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq)
 				  iwcq->kmem_shadow.size,
 				  iwcq->kmem_shadow.va, iwcq->kmem_shadow.pa);
 		iwcq->kmem_shadow.va = NULL;
+	} else {
+		struct irdma_mr *iwmr;
+
+		/* May be called in a failure path before iwpbl is valid. */
+		if (iwcq->iwpbl) {
+			iwmr = iwcq->iwpbl->iwmr;
+
+			refcount_dec(&iwmr->user_ring_refs);
+		}
+
+		if (iwcq->iwpbl_shadow) {
+			iwmr = iwcq->iwpbl_shadow->iwmr;
+
+			refcount_dec(&iwmr->user_ring_refs);
+		}
 	}
 
 	irdma_free_rsrc(rf, rf->allocated_cqs, cq->cq_uk.cq_id);
@@ -2018,7 +2039,7 @@ static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries,
 	struct irdma_modify_cq_info info = {};
 	struct irdma_dma_mem kmem_buf;
 	struct irdma_cq_mr *cqmr_buf;
-	struct irdma_pbl *iwpbl_buf;
+	struct irdma_pbl *iwpbl_buf = NULL;
 	struct irdma_device *iwdev;
 	struct irdma_pci_f *rf;
 	struct irdma_cq_buf *cq_buf = NULL;
@@ -2129,6 +2150,19 @@ static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries,
 		goto error;
 
 	spin_lock_irqsave(&iwcq->lock, flags);
+	if (udata) {
+		struct irdma_pbl *old_iwpbl = iwcq->iwpbl;
+
+		/* Only update if the resize was successful. Otherwise, HW is
+		 * still pointing to the old PBL.
+		 */
+		iwcq->iwpbl = iwpbl_buf;
+		if (old_iwpbl) {
+			struct irdma_mr *old_iwmr = old_iwpbl->iwmr;
+
+			refcount_dec(&old_iwmr->user_ring_refs);
+		}
+	}
 	if (cq_buf) {
 		cq_buf->kmem_buf = iwcq->kmem;
 		cq_buf->hw = dev->hw;
@@ -2144,6 +2178,11 @@ static int irdma_resize_cq(struct ib_cq *ibcq, unsigned int entries,
 
 	return 0;
 error:
+	if (iwpbl_buf) {
+		struct irdma_mr *iwmr = iwpbl_buf->iwmr;
+
+		refcount_dec(&iwmr->user_ring_refs);
+	}
 	if (!udata) {
 		dma_free_coherent(dev->hw->device, kmem_buf.size, kmem_buf.va,
 				  kmem_buf.pa);
@@ -2420,6 +2459,11 @@ static int irdma_create_srq(struct ib_srq *ibsrq,
 		dma_free_coherent(rf->hw.device, iwsrq->kmem.size,
 				  iwsrq->kmem.va, iwsrq->kmem.pa);
 free_rsrc:
+	if (iwsrq->user_mode && iwsrq->iwpbl) {
+		struct irdma_mr *iwmr = iwsrq->iwpbl->iwmr;
+
+		refcount_dec(&iwmr->user_ring_refs);
+	}
 	irdma_free_rsrc(rf, rf->allocated_srqs, iwsrq->srq_num);
 	return err_code;
 }
@@ -2499,6 +2543,8 @@ static int irdma_create_cq(struct ib_cq *ibcq,
 	INIT_LIST_HEAD(&iwcq->resize_list);
 	INIT_LIST_HEAD(&iwcq->cmpl_generated);
 	iwcq->cq_num = cq_num;
+	iwcq->iwpbl = NULL;
+	iwcq->iwpbl_shadow = NULL;
 	info.dev = dev;
 	ukinfo->cq_size = max(entries, 4);
 	ukinfo->cq_id = cq_num;
@@ -2518,8 +2564,6 @@ static int irdma_create_cq(struct ib_cq *ibcq,
 		struct irdma_ucontext *ucontext;
 		struct irdma_create_cq_req req = {};
 		struct irdma_cq_mr *cqmr;
-		struct irdma_pbl *iwpbl;
-		struct irdma_pbl *iwpbl_shadow;
 		struct irdma_cq_mr *cqmr_shadow;
 
 		iwcq->user_mode = true;
@@ -2533,34 +2577,34 @@ static int irdma_create_cq(struct ib_cq *ibcq,
 		}
 
 		spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
-		iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf,
-				      &ucontext->cq_reg_mem_list);
+		iwcq->iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf,
+					    &ucontext->cq_reg_mem_list);
 		spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
-		if (!iwpbl) {
+		if (!iwcq->iwpbl) {
 			err_code = -EPROTO;
 			goto cq_free_rsrc;
 		}
 
-		cqmr = &iwpbl->cq_mr;
+		cqmr = &iwcq->iwpbl->cq_mr;
 
 		if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags &
 		    IRDMA_FEATURE_CQ_RESIZE) {
 			spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
-			iwpbl_shadow = irdma_get_pbl(
+			iwcq->iwpbl_shadow = irdma_get_pbl(
 					(unsigned long)req.user_shadow_area,
 					&ucontext->cq_reg_mem_list);
 			spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
 
-			if (!iwpbl_shadow) {
+			if (!iwcq->iwpbl_shadow) {
 				err_code = -EPROTO;
 				goto cq_free_rsrc;
 			}
-			cqmr_shadow = &iwpbl_shadow->cq_mr;
+			cqmr_shadow = &iwcq->iwpbl_shadow->cq_mr;
 			info.shadow_area_pa = cqmr_shadow->cq_pbl.addr;
 		} else {
 			info.shadow_area_pa = cqmr->shadow;
 		}
-		if (iwpbl->pbl_allocated) {
+		if (iwcq->iwpbl->pbl_allocated) {
 			info.virtual_map = true;
 			info.pbl_chunk_size = 1;
 			info.first_pm_pbl_idx = cqmr->cq_pbl.idx;
@@ -2802,7 +2846,7 @@ static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size)
 	u32 pg_idx;
 
 	for (pg_idx = 0; pg_idx < npages; pg_idx++) {
-		if ((*arr + (pg_size * pg_idx)) != arr[pg_idx])
+		if ((*arr + ((u64)pg_size * pg_idx)) != arr[pg_idx])
 			return false;
 	}
 
@@ -2835,7 +2879,7 @@ static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc,
 
 	for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) {
 		arr = leaf->addr;
-		if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr)
+		if ((*start_addr + ((u64)i * pg_size * PBLE_PER_PAGE)) != *arr)
 			return false;
 		ret = irdma_check_mem_contiguous(arr, leaf->cnt, pg_size);
 		if (!ret)
@@ -3363,6 +3407,7 @@ static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region,
 	if (!iwmr)
 		return ERR_PTR(-ENOMEM);
 
+	refcount_set(&iwmr->user_ring_refs, 1);
 	iwpbl = &iwmr->iwpbl;
 	iwpbl->iwmr = iwmr;
 	iwmr->region = region;
@@ -3791,6 +3836,9 @@ static struct ib_mr *irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags,
 	if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS))
 		return ERR_PTR(-EOPNOTSUPP);
 
+	if (iwmr->type != IRDMA_MEMREG_TYPE_MEM)
+	     return ERR_PTR(-EINVAL);
+
 	ret = ib_umem_check_rereg(iwmr->region, flags, new_access);
 	if (ret)
 		return ERR_PTR(ret);
@@ -3927,41 +3975,41 @@ static struct ib_mr *irdma_get_dma_mr(struct ib_pd *pd, int acc)
  * irdma_del_memlist - Deleting pbl list entries for CQ/QP
  * @iwmr: iwmr for IB's user page addresses
  * @ucontext: ptr to user context
+ *
+ * Return: True if the MR is currently in-use by a QP/CQ/SRQ ring.
  */
-static void irdma_del_memlist(struct irdma_mr *iwmr,
+static bool irdma_del_memlist(struct irdma_mr *iwmr,
 			      struct irdma_ucontext *ucontext)
 {
 	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
 	unsigned long flags;
+	spinlock_t *lock;
+	bool in_use = false;
 
 	switch (iwmr->type) {
 	case IRDMA_MEMREG_TYPE_CQ:
-		spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
-		if (iwpbl->on_list) {
-			iwpbl->on_list = false;
-			list_del(&iwpbl->list);
-		}
-		spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
+		lock = &ucontext->cq_reg_mem_list_lock;
 		break;
 	case IRDMA_MEMREG_TYPE_QP:
-		spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
-		if (iwpbl->on_list) {
-			iwpbl->on_list = false;
-			list_del(&iwpbl->list);
-		}
-		spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
+		lock = &ucontext->qp_reg_mem_list_lock;
 		break;
 	case IRDMA_MEMREG_TYPE_SRQ:
-		spin_lock_irqsave(&ucontext->srq_reg_mem_list_lock, flags);
-		if (iwpbl->on_list) {
-			iwpbl->on_list = false;
-			list_del(&iwpbl->list);
-		}
-		spin_unlock_irqrestore(&ucontext->srq_reg_mem_list_lock, flags);
+		lock = &ucontext->srq_reg_mem_list_lock;
 		break;
 	default:
-		break;
+		return false;
+	}
+
+	spin_lock_irqsave(lock, flags);
+	if (!refcount_dec_if_one(&iwmr->user_ring_refs)) {
+		in_use = true;
+	} else if (iwpbl->on_list) {
+		iwpbl->on_list = false;
+		list_del(&iwpbl->list);
 	}
+	spin_unlock_irqrestore(lock, flags);
+
+	return in_use;
 }
 
 /**
@@ -3984,7 +4032,12 @@ static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
 			ucontext = rdma_udata_to_drv_context(udata,
 						struct irdma_ucontext,
 						ibucontext);
-			irdma_del_memlist(iwmr, ucontext);
+
+			/* Do not allow the MR to be unpinned if it is still
+			 * backing a user ring.
+			 */
+			if (irdma_del_memlist(iwmr, ucontext))
+				return -EBUSY;
 		}
 		goto done;
 	}
diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h
index 289ebc9b23ca7..a1651641eb714 100644
--- a/drivers/infiniband/hw/irdma/verbs.h
+++ b/drivers/infiniband/hw/irdma/verbs.h
@@ -120,6 +120,7 @@ struct irdma_mr {
 	u64 len;
 	u64 pgaddrmem[IRDMA_MAX_SAVED_PHY_PGADDR];
 	struct irdma_pbl iwpbl;
+	refcount_t user_ring_refs;
 };
 
 struct irdma_srq {
@@ -152,6 +153,8 @@ struct irdma_cq {
 	struct list_head resize_list;
 	struct irdma_cq_poll_info cur_cqe;
 	struct list_head cmpl_generated;
+	struct irdma_pbl *iwpbl;
+	struct irdma_pbl *iwpbl_shadow;
 };
 
 struct irdma_cmpl_gen {
diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c
index efe2935bda29f..a5b3606a1dd5a 100644
--- a/drivers/infiniband/hw/mana/main.c
+++ b/drivers/infiniband/hw/mana/main.c
@@ -579,7 +579,6 @@ int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
 	if (err)
 		return err;
 
-	memset(props, 0, sizeof(*props));
 	props->vendor_id = pdev->vendor;
 	props->vendor_part_id = dev->gdma_dev->dev_id.type;
 	props->max_mr_size = MANA_IB_MAX_MR_SIZE;
diff --git a/drivers/infiniband/hw/mana/wr.c b/drivers/infiniband/hw/mana/wr.c
index 1813567d3b16c..36a1d506f08f6 100644
--- a/drivers/infiniband/hw/mana/wr.c
+++ b/drivers/infiniband/hw/mana/wr.c
@@ -144,7 +144,7 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr
 int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 		      const struct ib_send_wr **bad_wr)
 {
-	int err;
+	int err = 0;
 	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
 
 	for (; wr; wr = wr->next) {
diff --git a/drivers/infiniband/hw/mlx4/cm.c b/drivers/infiniband/hw/mlx4/cm.c
index 63a868a3822f2..202fd5365e351 100644
--- a/drivers/infiniband/hw/mlx4/cm.c
+++ b/drivers/infiniband/hw/mlx4/cm.c
@@ -315,14 +315,20 @@ int mlx4_ib_multiplex_cm_handler(struct ib_device *ibdev, int port, int slave_id
 		id = id_map_get(ibdev, &pv_cm_id, slave_id, sl_cm_id);
 		if (id)
 			goto cont;
+		if (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID)
+			return 0;
 		id = id_map_alloc(ibdev, slave_id, sl_cm_id);
 		if (IS_ERR(id)) {
 			mlx4_ib_warn(ibdev, "%s: id{slave: %d, sl_cm_id: 0x%x} Failed to id_map_alloc\n",
 				__func__, slave_id, sl_cm_id);
 			return PTR_ERR(id);
 		}
-	} else if (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID ||
-		   mad->mad_hdr.attr_id == CM_SIDR_REP_ATTR_ID) {
+	} else if (mad->mad_hdr.attr_id == CM_REJ_ATTR_ID) {
+		sl_cm_id = get_local_comm_id(mad);
+		id = id_map_get(ibdev, &pv_cm_id, slave_id, sl_cm_id);
+		if (!id)
+			return 0;
+	} else if (mad->mad_hdr.attr_id == CM_SIDR_REP_ATTR_ID) {
 		return 0;
 	} else {
 		sl_cm_id = get_local_comm_id(mad);
@@ -338,7 +344,8 @@ int mlx4_ib_multiplex_cm_handler(struct ib_device *ibdev, int port, int slave_id
 cont:
 	set_local_comm_id(mad, id->pv_cm_id);
 
-	if (mad->mad_hdr.attr_id == CM_DREQ_ATTR_ID)
+	if (mad->mad_hdr.attr_id == CM_DREQ_ATTR_ID ||
+	    mad->mad_hdr.attr_id == CM_REJ_ATTR_ID)
 		schedule_delayed(ibdev, id);
 	return 0;
 }
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index 17073e8f105aa..7266a61419440 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -471,8 +471,6 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
 	if (err)
 		goto out;
 
-	memset(props, 0, sizeof *props);
-
 	have_ib_ports = num_ib_ports(dev->dev);
 
 	props->fw_ver = dev->dev->caps.fw_ver;
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 02809114fc79a..e8bba5a76d4ee 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -969,7 +969,6 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	if (err)
 		return err;
 
-	memset(props, 0, sizeof(*props));
 	err = mlx5_query_system_image_guid(ibdev,
 					   &props->sys_image_guid);
 	if (err)
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index e6b74955d95dc..00e13028762ae 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -339,7 +339,7 @@ static int mlx5r_build_frmr_key(struct ib_device *device,
 	return 0;
 }
 
-static struct ib_frmr_pool_ops mlx5r_frmr_pool_ops = {
+static const struct ib_frmr_pool_ops mlx5r_frmr_pool_ops = {
 	.create_frmrs = mlx5r_create_mkeys,
 	.destroy_frmrs = mlx5r_destroy_mkeys,
 	.build_key = mlx5r_build_frmr_key,
@@ -898,8 +898,8 @@ static void mlx5_ib_dmabuf_invalidate_cb(struct dma_buf_attachment *attach)
 	ib_umem_dmabuf_unmap_pages(umem_dmabuf);
 }
 
-static struct dma_buf_attach_ops mlx5_ib_dmabuf_attach_ops = {
-	.allow_peer2peer = 1,
+static const struct dma_buf_attach_ops mlx5_ib_dmabuf_attach_ops = {
+	.allow_peer2peer = true,
 	.invalidate_mappings = mlx5_ib_dmabuf_invalidate_cb,
 };
 
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 1badec9bf5270..b8618610737aa 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -37,6 +37,7 @@
 #include <linux/hmm.h>
 #include <linux/hmm-dma.h>
 #include <linux/pci-p2pdma.h>
+#include <linux/slab.h>
 
 #include "mlx5_ib.h"
 #include "cmd.h"
@@ -1414,7 +1415,8 @@ static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev,
 		goto resolve_page_fault;
 	}
 
-	wqe_start = (void *)__get_free_page(GFP_KERNEL);
+	/* TODO: switch to "fast and as large as possible" allocation helper */
+	wqe_start = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!wqe_start) {
 		mlx5_ib_err(dev, "Error allocating memory for IO page fault handling.\n");
 		goto resolve_page_fault;
@@ -1475,7 +1477,7 @@ static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev,
 		    pfault->wqe.wq_num, resume_with_error,
 		    pfault->type);
 	mlx5_core_res_put(res);
-	free_page((unsigned long)wqe_start);
+	kfree(wqe_start);
 }
 
 static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev,
diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c
index 48cae5cc1c1be..c595b85b428c3 100644
--- a/drivers/infiniband/hw/mlx5/umr.c
+++ b/drivers/infiniband/hw/mlx5/umr.c
@@ -978,7 +978,7 @@ static inline int _mlx5r_dmabuf_umr_update_pas(struct mlx5_ib_mr *mr,
 						     start_block, nblocks);
 }
 
-/**
+/*
  * This function makes an mkey non-present by zapping the translation entries of
  * the mkey by zapping (zeroing out) the first N entries, where N is determined
  * by the largest page size supported by the device and the MR length.
diff --git a/drivers/infiniband/hw/mlx5/wr.c b/drivers/infiniband/hw/mlx5/wr.c
index 9947feb7fb8a0..fca9e1d9d5e9b 100644
--- a/drivers/infiniband/hw/mlx5/wr.c
+++ b/drivers/infiniband/hw/mlx5/wr.c
@@ -840,13 +840,15 @@ static int handle_psv(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	return err;
 }
 
-static int handle_reg_mr_integrity(struct mlx5_ib_dev *dev,
-				   struct mlx5_ib_qp *qp,
-				   const struct ib_send_wr *wr,
-				   struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
-				   int *size, void **cur_edge,
-				   unsigned int *idx, int nreq, u8 fence,
-				   u8 next_fence)
+static noinline_for_stack int handle_reg_mr_integrity(struct mlx5_ib_dev *dev,
+						      struct mlx5_ib_qp *qp,
+						      const struct ib_send_wr *wr,
+						      struct mlx5_wqe_ctrl_seg **ctrl,
+						      void **seg,
+						      int *size, void **cur_edge,
+						      unsigned int *idx, int nreq,
+						      u8 fence,
+						      u8 next_fence)
 {
 	struct mlx5_ib_mr *mr;
 	struct mlx5_ib_mr *pi_mr;
diff --git a/drivers/infiniband/hw/mthca/mthca_allocator.c b/drivers/infiniband/hw/mthca/mthca_allocator.c
index dedc301235a0d..117a070e784ea 100644
--- a/drivers/infiniband/hw/mthca/mthca_allocator.c
+++ b/drivers/infiniband/hw/mthca/mthca_allocator.c
@@ -126,7 +126,7 @@ int mthca_array_set(struct mthca_array *array, int index, void *value)
 
 	/* Allocate with GFP_ATOMIC because we'll be called with locks held. */
 	if (!array->page_list[p].page)
-		array->page_list[p].page = (void **) get_zeroed_page(GFP_ATOMIC);
+		array->page_list[p].page = kzalloc(PAGE_SIZE, GFP_ATOMIC);
 
 	if (!array->page_list[p].page)
 		return -ENOMEM;
@@ -142,7 +142,7 @@ void mthca_array_clear(struct mthca_array *array, int index)
 	int p = (index * sizeof (void *)) >> PAGE_SHIFT;
 
 	if (--array->page_list[p].used == 0) {
-		free_page((unsigned long) array->page_list[p].page);
+		kfree(array->page_list[p].page);
 		array->page_list[p].page = NULL;
 	} else
 		array->page_list[p].page[index & MTHCA_ARRAY_MASK] = NULL;
@@ -174,7 +174,7 @@ void mthca_array_cleanup(struct mthca_array *array, int nent)
 	int i;
 
 	for (i = 0; i < (nent * sizeof (void *) + PAGE_SIZE - 1) / PAGE_SIZE; ++i)
-		free_page((unsigned long) array->page_list[i].page);
+		kfree(array->page_list[i].page);
 
 	kfree(array->page_list);
 }
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index f90f67afc8fab..05cca36f6a4bd 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -69,8 +69,6 @@ static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *pr
 		goto out;
 	}
 
-	memset(props, 0, sizeof *props);
-
 	props->fw_ver              = mdev->fw_ver;
 
 	ib_init_query_mad(in_mad);
@@ -895,7 +893,8 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 		goto err_umem;
 	}
 
-	pages = (u64 *) __get_free_page(GFP_KERNEL);
+	/* TODO: switch to "fast and as large as possible" allocation helper */
+	pages = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!pages) {
 		err = -ENOMEM;
 		goto err_mtt;
@@ -924,7 +923,7 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	if (i)
 		err = mthca_write_mtt(dev, mr->mtt, n, pages, i);
 mtt_done:
-	free_page((unsigned long) pages);
+	kfree(pages);
 	if (err)
 		goto err_mtt;
 
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index 53ba32d168a17..de83dc0ea79c4 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -74,7 +74,6 @@ int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
 	if (err)
 		return err;
 
-	memset(attr, 0, sizeof *attr);
 	memcpy(&attr->fw_ver, &dev->attr.fw_ver[0],
 	       min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver)));
 	addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c
index c90a1b5c8ee7d..1cf502579ad1f 100644
--- a/drivers/infiniband/hw/qedr/verbs.c
+++ b/drivers/infiniband/hw/qedr/verbs.c
@@ -118,8 +118,6 @@ int qedr_query_device(struct ib_device *ibdev,
 	if (rc)
 		return rc;
 
-	memset(attr, 0, sizeof(*attr));
-
 	attr->fw_ver = qattr->fw_ver;
 	attr->sys_image_guid = qattr->sys_image_guid;
 	attr->max_mr_size = qattr->max_mr_size;
@@ -148,7 +146,7 @@ int qedr_query_device(struct ib_device *ibdev,
 	attr->max_qp_init_rd_atom =
 	    1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
 	attr->max_qp_rd_atom =
-	    min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
+	    min(1U << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
 		attr->max_qp_init_rd_atom);
 
 	attr->max_srq = qattr->max_srq;
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
index dc355b00f61ce..3402c84eb9049 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
@@ -284,7 +284,6 @@ int usnic_ib_query_device(struct ib_device *ibdev,
 
 	mutex_lock(&us_ibdev->usdev_lock);
 	us_ibdev->netdev->ethtool_ops->get_drvinfo(us_ibdev->netdev, &info);
-	memset(props, 0, sizeof(*props));
 	usnic_mac_ip_to_gid(us_ibdev->ufdev->mac, us_ibdev->ufdev->inaddr,
 			&gid.raw[0]);
 	memcpy(&props->sys_image_guid, &gid.global.interface_id,
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index 70e7d08fdce69..c40cce69e9457 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -263,7 +263,7 @@ static inline bool wss_exceeds_threshold(struct rvt_wss *wss)
 static void get_map_page(struct rvt_qpn_table *qpt,
 			 struct rvt_qpn_map *map)
 {
-	unsigned long page = get_zeroed_page(GFP_KERNEL);
+	void *page = kzalloc(PAGE_SIZE, GFP_KERNEL);
 
 	/*
 	 * Free the page if someone raced with us installing it.
@@ -271,9 +271,9 @@ static void get_map_page(struct rvt_qpn_table *qpt,
 
 	spin_lock(&qpt->lock);
 	if (map->page)
-		free_page(page);
+		kfree(page);
 	else
-		map->page = (void *)page;
+		map->page = page;
 	spin_unlock(&qpt->lock);
 }
 
@@ -343,7 +343,7 @@ static void free_qpn_table(struct rvt_qpn_table *qpt)
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(qpt->map); i++)
-		free_page((unsigned long)qpt->map[i].page);
+		kfree(qpt->map[i].page);
 }
 
 /**
diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c
index 5fa3a1f333268..f37d6d64adb9d 100644
--- a/drivers/infiniband/sw/rdmavt/vt.c
+++ b/drivers/infiniband/sw/rdmavt/vt.c
@@ -55,8 +55,10 @@ struct rvt_dev_info *rvt_alloc_device(size_t size, int nports)
 		return rdi;
 
 	rdi->ports = kzalloc_objs(*rdi->ports, nports);
-	if (!rdi->ports)
+	if (!rdi->ports) {
 		ib_dealloc_device(&rdi->ibdev);
+		return NULL;
+	}
 
 	return rdi;
 }
diff --git a/drivers/infiniband/sw/rxe/rxe_mw.c b/drivers/infiniband/sw/rxe/rxe_mw.c
index 379e65bfcd49a..bddb7a2578313 100644
--- a/drivers/infiniband/sw/rxe/rxe_mw.c
+++ b/drivers/infiniband/sw/rxe/rxe_mw.c
@@ -72,13 +72,6 @@ static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
 			return -EINVAL;
 		}
 
-		/* C10-72 */
-		if (unlikely(qp->pd != to_rpd(mw->ibmw.pd))) {
-			rxe_dbg_mw(mw,
-				"attempt to bind type 2 MW with qp with different PD\n");
-			return -EINVAL;
-		}
-
 		/* o10-37.2.40 */
 		if (unlikely(!mr || wqe->wr.wr.mw.length == 0)) {
 			rxe_dbg_mw(mw,
@@ -87,10 +80,21 @@ static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
 		}
 	}
 
-	/* remaining checks only apply to a nonzero MR */
+	/* C10-72 */
+	if (unlikely(qp->pd != rxe_mw_pd(mw))) {
+		rxe_dbg_mw(mw, "attempt to bind MW with qp with different PD\n");
+		return -EINVAL;
+	}
+
 	if (!mr)
 		return 0;
 
+	/* remaining checks only apply to a nonzero MR */
+	if (unlikely(qp->pd != mr_pd(mr))) {
+		rxe_dbg_mw(mw, "attempt to bind MW/QP to MR with different PD\n");
+		return -EINVAL;
+	}
+
 	if (unlikely(mr->access & IB_ZERO_BASED)) {
 		rxe_dbg_mw(mw, "attempt to bind MW to zero based MR\n");
 		return -EINVAL;
diff --git a/drivers/infiniband/sw/rxe/rxe_odp.c b/drivers/infiniband/sw/rxe/rxe_odp.c
index ff904d5e54a73..c189c033175da 100644
--- a/drivers/infiniband/sw/rxe/rxe_odp.c
+++ b/drivers/infiniband/sw/rxe/rxe_odp.c
@@ -87,14 +87,9 @@ int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length,
 
 	rxe_mr_init(access_flags, mr);
 
-	if (!start && length == U64_MAX) {
-		if (iova != 0)
-			return -EINVAL;
-		if (!(rxe->attr.odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))
-			return -EINVAL;
-
-		/* Never reach here, for implicit ODP is not implemented. */
-	}
+	/* Implicit ODP (start=0, length=U64_MAX) is not implemented. */
+	if (!start && length == U64_MAX)
+		return -EOPNOTSUPP;
 
 	umem_odp = ib_umem_odp_get(&rxe->ib_dev, start, length, access_flags,
 				   &rxe_mn_ops);
diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c
index f3dff1aea96af..7a0529a17992c 100644
--- a/drivers/infiniband/sw/rxe/rxe_qp.c
+++ b/drivers/infiniband/sw/rxe/rxe_qp.c
@@ -67,27 +67,27 @@ static int rxe_qp_chk_cap(struct rxe_dev *rxe, struct ib_qp_cap *cap,
 			  int has_srq)
 {
 	if (cap->max_send_wr > rxe->attr.max_qp_wr) {
-		rxe_dbg_dev(rxe, "invalid send wr = %u > %d\n",
-			 cap->max_send_wr, rxe->attr.max_qp_wr);
+		rxe_dbg_dev(rxe, "invalid send wr = %u > %u\n",
+			    cap->max_send_wr, rxe->attr.max_qp_wr);
 		goto err1;
 	}
 
 	if (cap->max_send_sge > rxe->attr.max_send_sge) {
-		rxe_dbg_dev(rxe, "invalid send sge = %u > %d\n",
-			 cap->max_send_sge, rxe->attr.max_send_sge);
+		rxe_dbg_dev(rxe, "invalid send sge = %u > %u\n",
+			    cap->max_send_sge, rxe->attr.max_send_sge);
 		goto err1;
 	}
 
 	if (!has_srq) {
 		if (cap->max_recv_wr > rxe->attr.max_qp_wr) {
-			rxe_dbg_dev(rxe, "invalid recv wr = %u > %d\n",
-				 cap->max_recv_wr, rxe->attr.max_qp_wr);
+			rxe_dbg_dev(rxe, "invalid recv wr = %u > %u\n",
+				    cap->max_recv_wr, rxe->attr.max_qp_wr);
 			goto err1;
 		}
 
 		if (cap->max_recv_sge > rxe->attr.max_recv_sge) {
-			rxe_dbg_dev(rxe, "invalid recv sge = %u > %d\n",
-				 cap->max_recv_sge, rxe->attr.max_recv_sge);
+			rxe_dbg_dev(rxe, "invalid recv sge = %u > %u\n",
+				    cap->max_recv_sge, rxe->attr.max_recv_sge);
 			goto err1;
 		}
 	}
@@ -537,9 +537,9 @@ int rxe_qp_chk_attr(struct rxe_dev *rxe, struct rxe_qp *qp,
 
 	if (mask & IB_QP_MAX_QP_RD_ATOMIC) {
 		if (attr->max_rd_atomic > rxe->attr.max_qp_rd_atom) {
-			rxe_dbg_qp(qp, "invalid max_rd_atomic %d > %d\n",
-				 attr->max_rd_atomic,
-				 rxe->attr.max_qp_rd_atom);
+			rxe_dbg_qp(qp, "invalid max_rd_atomic %u > %u\n",
+				   attr->max_rd_atomic,
+				   rxe->attr.max_qp_rd_atom);
 			goto err1;
 		}
 	}
diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
index d8cbdfa70cdbd..02b16e2b49b8f 100644
--- a/drivers/infiniband/sw/rxe/rxe_resp.c
+++ b/drivers/infiniband/sw/rxe/rxe_resp.c
@@ -1217,7 +1217,14 @@ static enum resp_states do_complete(struct rxe_qp *qp,
 	spin_lock_irqsave(&qp->state_lock, flags);
 	if (unlikely(qp_state(qp) == IB_QPS_ERR)) {
 		spin_unlock_irqrestore(&qp->state_lock, flags);
-		return RESPST_CHK_RESOURCE;
+		/* The packet was executed and completed before the QP
+		 * moved to ERROR; it must be consumed exactly once.
+		 * Re-entering the request chain with the stale packet
+		 * would copy it into every remaining recv WQE as a new
+		 * completion.  Remaining WQEs are flushed by the drain
+		 * path at rxe_receiver() entry.
+		 */
+		return pkt ? RESPST_CLEANUP : RESPST_CHK_RESOURCE;
 	}
 	spin_unlock_irqrestore(&qp->state_lock, flags);
 
diff --git a/drivers/infiniband/sw/rxe/rxe_srq.c b/drivers/infiniband/sw/rxe/rxe_srq.c
index c9a7cd38953d3..74904a6fdf2b2 100644
--- a/drivers/infiniband/sw/rxe/rxe_srq.c
+++ b/drivers/infiniband/sw/rxe/rxe_srq.c
@@ -13,8 +13,8 @@ int rxe_srq_chk_init(struct rxe_dev *rxe, struct ib_srq_init_attr *init)
 	struct ib_srq_attr *attr = &init->attr;
 
 	if (attr->max_wr > rxe->attr.max_srq_wr) {
-		rxe_dbg_dev(rxe, "max_wr(%d) > max_srq_wr(%d)\n",
-			attr->max_wr, rxe->attr.max_srq_wr);
+		rxe_dbg_dev(rxe, "max_wr(%u) > max_srq_wr(%u)\n",
+			    attr->max_wr, rxe->attr.max_srq_wr);
 		goto err1;
 	}
 
@@ -27,8 +27,8 @@ int rxe_srq_chk_init(struct rxe_dev *rxe, struct ib_srq_init_attr *init)
 		attr->max_wr = RXE_MIN_SRQ_WR;
 
 	if (attr->max_sge > rxe->attr.max_srq_sge) {
-		rxe_dbg_dev(rxe, "max_sge(%d) > max_srq_sge(%d)\n",
-			attr->max_sge, rxe->attr.max_srq_sge);
+		rxe_dbg_dev(rxe, "max_sge(%u) > max_srq_sge(%u)\n",
+			    attr->max_sge, rxe->attr.max_srq_sge);
 		goto err1;
 	}
 
@@ -107,8 +107,8 @@ int rxe_srq_chk_attr(struct rxe_dev *rxe, struct rxe_srq *srq,
 
 	if (mask & IB_SRQ_MAX_WR) {
 		if (attr->max_wr > rxe->attr.max_srq_wr) {
-			rxe_dbg_srq(srq, "max_wr(%d) > max_srq_wr(%d)\n",
-				attr->max_wr, rxe->attr.max_srq_wr);
+			rxe_dbg_srq(srq, "max_wr(%u) > max_srq_wr(%u)\n",
+				    attr->max_wr, rxe->attr.max_srq_wr);
 			goto err1;
 		}
 
@@ -129,8 +129,8 @@ int rxe_srq_chk_attr(struct rxe_dev *rxe, struct rxe_srq *srq,
 
 	if (mask & IB_SRQ_LIMIT) {
 		if (attr->srq_limit > rxe->attr.max_srq_wr) {
-			rxe_dbg_srq(srq, "srq_limit(%d) > max_srq_wr(%d)\n",
-				attr->srq_limit, rxe->attr.max_srq_wr);
+			rxe_dbg_srq(srq, "srq_limit(%u) > max_srq_wr(%u)\n",
+				    attr->srq_limit, rxe->attr.max_srq_wr);
 			goto err1;
 		}
 
diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c
index b74ac85c1b8b8..30fc14037fe35 100644
--- a/drivers/infiniband/sw/siw/siw_verbs.c
+++ b/drivers/infiniband/sw/siw/siw_verbs.c
@@ -136,8 +136,6 @@ int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr,
 	if (rv)
 		return rv;
 
-	memset(attr, 0, sizeof(*attr));
-
 	/* Revisit atomic caps if RFC 7306 gets supported */
 	attr->atomic_cap = 0;
 	attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS;
@@ -318,6 +316,7 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
 	struct siw_ucontext *uctx =
 		rdma_udata_to_drv_context(udata, struct siw_ucontext,
 					  base_ucontext);
+	struct siw_uresp_create_qp uresp = {};
 	unsigned long flags;
 	int num_sqe, num_rqe, rv = 0;
 	size_t length;
@@ -371,11 +370,6 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
 	spin_lock_init(&qp->rq_lock);
 	spin_lock_init(&qp->orq_lock);
 
-	rv = siw_qp_add(sdev, qp);
-	if (rv)
-		goto err_atomic;
-
-
 	/* All queue indices are derived from modulo operations
 	 * on a free running 'get' (consumer) and 'put' (producer)
 	 * unsigned counter. Having queue sizes at power of two
@@ -393,14 +387,14 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
 
 	if (qp->sendq == NULL) {
 		rv = -ENOMEM;
-		goto err_out_xa;
+		goto err_out;
 	}
 	if (attrs->sq_sig_type != IB_SIGNAL_REQ_WR) {
 		if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
 			qp->attrs.flags |= SIW_SIGNAL_ALL_WR;
 		else {
 			rv = -EINVAL;
-			goto err_out_xa;
+			goto err_out;
 		}
 	}
 	qp->pd = pd;
@@ -426,7 +420,7 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
 
 		if (qp->recvq == NULL) {
 			rv = -ENOMEM;
-			goto err_out_xa;
+			goto err_out;
 		}
 		qp->attrs.rq_size = num_rqe;
 	}
@@ -441,11 +435,8 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
 	qp->attrs.state = SIW_QP_STATE_IDLE;
 
 	if (udata) {
-		struct siw_uresp_create_qp uresp = {};
-
 		uresp.num_sqe = num_sqe;
 		uresp.num_rqe = num_rqe;
-		uresp.qp_id = qp_id(qp);
 
 		if (qp->sendq) {
 			length = num_sqe * sizeof(struct siw_sqe);
@@ -454,7 +445,7 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
 						      length, &uresp.sq_key);
 			if (!qp->sq_entry) {
 				rv = -ENOMEM;
-				goto err_out_xa;
+				goto err_out;
 			}
 		}
 
@@ -466,9 +457,23 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
 			if (!qp->rq_entry) {
 				uresp.sq_key = SIW_INVAL_UOBJ_KEY;
 				rv = -ENOMEM;
-				goto err_out_xa;
+				goto err_out;
 			}
 		}
+	}
+	qp->tx_cpu = siw_get_tx_cpu(sdev);
+	if (qp->tx_cpu < 0) {
+		rv = -EINVAL;
+		goto err_out;
+	}
+	init_completion(&qp->qp_free);
+
+	rv = siw_qp_add(sdev, qp);
+	if (rv)
+		goto err_out_tx;
+
+	if (udata) {
+		uresp.qp_id = qp_id(qp);
 
 		if (udata->outlen < sizeof(uresp)) {
 			rv = -EINVAL;
@@ -478,22 +483,19 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
 		if (rv)
 			goto err_out_xa;
 	}
-	qp->tx_cpu = siw_get_tx_cpu(sdev);
-	if (qp->tx_cpu < 0) {
-		rv = -EINVAL;
-		goto err_out_xa;
-	}
+
 	INIT_LIST_HEAD(&qp->devq);
 	spin_lock_irqsave(&sdev->lock, flags);
 	list_add_tail(&qp->devq, &sdev->qp_list);
 	spin_unlock_irqrestore(&sdev->lock, flags);
 
-	init_completion(&qp->qp_free);
-
 	return 0;
 
 err_out_xa:
 	xa_erase(&sdev->qp_xa, qp_id(qp));
+err_out_tx:
+	siw_put_tx_cpu(qp->tx_cpu);
+err_out:
 	if (uctx) {
 		rdma_user_mmap_entry_remove(qp->sq_entry);
 		rdma_user_mmap_entry_remove(qp->rq_entry);
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
index 57fec88a16290..ed0592898384d 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
@@ -1071,8 +1071,7 @@ static struct ib_qp *ipoib_cm_create_tx_qp(struct net_device *dev, struct ipoib_
 	struct ib_qp *tx_qp;
 
 	if (dev->features & NETIF_F_SG)
-		attr.cap.max_send_sge = min_t(u32, priv->ca->attrs.max_send_sge,
-					      MAX_SKB_FRAGS + 1);
+		attr.cap.max_send_sge = min(priv->ca->attrs.max_send_sge, MAX_SKB_FRAGS + 1);
 
 	tx_qp = ib_create_qp(priv->pd, &attr);
 	tx->max_send_sge = attr.cap.max_send_sge;
@@ -1582,7 +1581,8 @@ static void ipoib_cm_create_srq(struct net_device *dev, int max_sge)
 int ipoib_cm_dev_init(struct net_device *dev)
 {
 	struct ipoib_dev_priv *priv = ipoib_priv(dev);
-	int max_srq_sge, i;
+	u32 max_srq_sge;
+	int i;
 	u8 addr;
 
 	INIT_LIST_HEAD(&priv->cm.passive_ids);
@@ -1600,9 +1600,9 @@ int ipoib_cm_dev_init(struct net_device *dev)
 
 	skb_queue_head_init(&priv->cm.skb_queue);
 
-	ipoib_dbg(priv, "max_srq_sge=%d\n", priv->ca->attrs.max_srq_sge);
+	ipoib_dbg(priv, "max_srq_sge=%u\n", priv->ca->attrs.max_srq_sge);
 
-	max_srq_sge = min_t(int, IPOIB_CM_RX_SG, priv->ca->attrs.max_srq_sge);
+	max_srq_sge = min(priv->ca->attrs.max_srq_sge, IPOIB_CM_RX_SG);
 	ipoib_cm_create_srq(dev, max_srq_sge);
 	if (ipoib_cm_has_srq(dev)) {
 		priv->cm.max_cm_mtu = max_srq_sge * PAGE_SIZE - 0x10;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
index 3ed1ea566690d..2490696a1aabc 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
@@ -147,8 +147,7 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca)
 		.cap = {
 			.max_send_wr  = ipoib_sendq_size,
 			.max_recv_wr  = ipoib_recvq_size,
-			.max_send_sge = min_t(u32, priv->ca->attrs.max_send_sge,
-					      MAX_SKB_FRAGS + 1),
+			.max_send_sge = min(priv->ca->attrs.max_send_sge, MAX_SKB_FRAGS + 1),
 			.max_recv_sge = IPOIB_UD_RX_SG
 		},
 		.sq_sig_type = IB_SIGNAL_ALL_WR,
diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index f03b3bb3c0c4c..55fe68e5b8375 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -244,8 +244,7 @@ static int iser_create_ib_conn_res(struct ib_conn *ib_conn)
 		max_send_wr = ISER_QP_SIG_MAX_REQ_DTOS + 1;
 	else
 		max_send_wr = ISER_QP_MAX_REQ_DTOS + 1;
-	max_send_wr = min_t(unsigned int, max_send_wr,
-			    (unsigned int)ib_dev->attrs.max_qp_wr);
+	max_send_wr = min(max_send_wr, ib_dev->attrs.max_qp_wr);
 
 	cq_size = max_send_wr + ISER_QP_MAX_RECV_DTOS;
 	ib_conn->cq = ib_cq_pool_get(ib_dev, cq_size, -1, IB_POLL_SOFTIRQ);
@@ -589,7 +588,7 @@ static void iser_route_handler(struct rdma_cm_id *cma_id)
 		goto failure;
 
 	memset(&conn_param, 0, sizeof conn_param);
-	conn_param.responder_resources = ib_dev->attrs.max_qp_rd_atom;
+	conn_param.responder_resources = min(ib_dev->attrs.max_qp_rd_atom, U8_MAX);
 	conn_param.initiator_depth = 1;
 	conn_param.retry_count = 7;
 	conn_param.rnr_retry_count = 6;
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index 1015a51f750af..4691845bf815f 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -214,9 +214,9 @@ isert_create_device_ib_res(struct isert_device *device)
 	struct ib_device *ib_dev = device->ib_device;
 	int ret;
 
-	isert_dbg("devattr->max_send_sge: %d devattr->max_recv_sge %d\n",
+	isert_dbg("devattr->max_send_sge: %u devattr->max_recv_sge %u\n",
 		  ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge);
-	isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd);
+	isert_dbg("devattr->max_sge_rd: %u\n", ib_dev->attrs.max_sge_rd);
 
 	device->pd = ib_alloc_pd(ib_dev, 0);
 	if (IS_ERR(device->pd)) {
@@ -381,8 +381,7 @@ isert_set_nego_params(struct isert_conn *isert_conn,
 	struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs;
 
 	/* Set max inflight RDMA READ requests */
-	isert_conn->initiator_depth = min_t(u8, param->initiator_depth,
-				attr->max_qp_init_rd_atom);
+	isert_conn->initiator_depth = min(param->initiator_depth, attr->max_qp_init_rd_atom);
 	isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth);
 
 	if (param->private_data) {
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
index d34d7e5f34d6f..7b2c51ae614f8 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
@@ -1675,8 +1675,7 @@ static int create_con_cq_qp(struct rtrs_clt_con *con)
 		 * + 2 for drain and heartbeat
 		 * in case qp gets into error state.
 		 */
-		max_send_wr =
-			min_t(int, wr_limit, SERVICE_CON_QUEUE_DEPTH * 2 + 2);
+		max_send_wr = min(wr_limit, SERVICE_CON_QUEUE_DEPTH * 2 + 2);
 		max_recv_wr = max_send_wr;
 	} else {
 		/*
@@ -1692,11 +1691,9 @@ static int create_con_cq_qp(struct rtrs_clt_con *con)
 		wr_limit = clt_path->s.dev->ib_dev->attrs.max_qp_wr;
 		/* Shared between connections */
 		clt_path->s.dev_ref++;
-		max_send_wr = min_t(int, wr_limit,
-			      /* QD * (REQ + RSP + FR REGS or INVS) + drain */
-			      clt_path->queue_depth * 4 + 1);
-		max_recv_wr = min_t(int, wr_limit,
-			      clt_path->queue_depth * 3 + 1);
+		/* QD * (REQ + RSP + FR REGS or INVS) + drain */
+		max_send_wr = min(wr_limit, clt_path->queue_depth * 4 + 1);
+		max_recv_wr = min(wr_limit, clt_path->queue_depth * 3 + 1);
 		max_send_sge = 2;
 	}
 	atomic_set(&con->c.sq_wr_avail, max_send_wr);
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
index 7d8e4422cc573..5314599d29fe1 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
@@ -1740,21 +1740,16 @@ static int create_con(struct rtrs_srv_path *srv_path,
 		 * All receive and all send (each requiring invalidate)
 		 * + 2 for drain and heartbeat
 		 */
-		max_send_wr = min_t(int, wr_limit,
-				    SERVICE_CON_QUEUE_DEPTH * 2 + 2);
+		max_send_wr = min(wr_limit, SERVICE_CON_QUEUE_DEPTH * 2 + 2);
 		max_recv_wr = max_send_wr;
 		s->signal_interval = min_not_zero(srv->queue_depth,
 						  (size_t)SERVICE_CON_QUEUE_DEPTH);
 	} else {
 		/* when always_invlaidate enalbed, we need linv+rinv+mr+imm */
 		if (always_invalidate)
-			max_send_wr =
-				min_t(int, wr_limit,
-				      srv->queue_depth * (1 + 4) + 1);
+			max_send_wr = min(wr_limit, srv->queue_depth * (1 + 4) + 1);
 		else
-			max_send_wr =
-				min_t(int, wr_limit,
-				      srv->queue_depth * (1 + 2) + 1);
+			max_send_wr = min(wr_limit, srv->queue_depth * (1 + 2) + 1);
 
 		max_recv_wr = srv->queue_depth + 1;
 	}
diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index acbd787de265b..0caebbc2810ff 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -557,7 +557,7 @@ static int srp_create_ch_ib(struct srp_rdma_ch *ch)
 	init_attr->cap.max_send_wr     = m * target->queue_size;
 	init_attr->cap.max_recv_wr     = target->queue_size + 1;
 	init_attr->cap.max_recv_sge    = 1;
-	init_attr->cap.max_send_sge    = min(SRP_MAX_SGE, attr->max_send_sge);
+	init_attr->cap.max_send_sge    = min(attr->max_send_sge, SRP_MAX_SGE);
 	init_attr->sq_sig_type         = IB_SIGNAL_REQ_WR;
 	init_attr->qp_type             = IB_QPT_RC;
 	init_attr->send_cq             = send_cq;
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index f66cfd70c2636..a760b4fbee90f 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -77,8 +77,8 @@ module_param(srp_max_req_size, int, 0444);
 MODULE_PARM_DESC(srp_max_req_size,
 		 "Maximum size of SRP request messages in bytes.");
 
-static int srpt_srq_size = DEFAULT_SRPT_SRQ_SIZE;
-module_param(srpt_srq_size, int, 0444);
+static unsigned int srpt_srq_size = DEFAULT_SRPT_SRQ_SIZE;
+module_param(srpt_srq_size, uint, 0444);
 MODULE_PARM_DESC(srpt_srq_size,
 		 "Shared receive queue (SRQ) size.");
 
@@ -405,8 +405,7 @@ static void srpt_get_ioc(struct srpt_port *sport, u32 slot,
 	if (sdev->use_srq)
 		send_queue_depth = sdev->srq_size;
 	else
-		send_queue_depth = min(MAX_SRPT_RQ_SIZE,
-				       sdev->device->attrs.max_qp_wr);
+		send_queue_depth = min(sdev->device->attrs.max_qp_wr, MAX_SRPT_RQ_SIZE);
 
 	memset(iocp, 0, sizeof(*iocp));
 	strcpy(iocp->id_string, SRPT_ID_STRING);
@@ -1851,7 +1850,7 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
 	struct srpt_port *sport = ch->sport;
 	struct srpt_device *sdev = sport->sdev;
 	const struct ib_device_attr *attrs = &sdev->device->attrs;
-	int sq_size = sport->port_attrib.srp_sq_size;
+	u32 sq_size = sport->port_attrib.srp_sq_size;
 	int i, ret;
 
 	WARN_ON(ch->rq_size < 1);
@@ -1912,13 +1911,13 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
 		bool retry = sq_size > MIN_SRPT_SQ_SIZE;
 
 		if (retry) {
-			pr_debug("failed to create queue pair with sq_size = %d (%d) - retrying\n",
+			pr_debug("failed to create queue pair with sq_size = %u (%d) - retrying\n",
 				 sq_size, ret);
 			ib_cq_pool_put(ch->cq, ch->cq_size);
 			sq_size = max(sq_size / 2, MIN_SRPT_SQ_SIZE);
 			goto retry;
 		} else {
-			pr_err("failed to create queue pair with sq_size = %d (%d)\n",
+			pr_err("failed to create queue pair with sq_size = %u (%d)\n",
 			       sq_size, ret);
 			goto err_destroy_cq;
 		}
@@ -1926,7 +1925,7 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
 
 	atomic_set(&ch->sq_wr_avail, qp_init->cap.max_send_wr);
 
-	pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d ch= %p\n",
+	pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %u ch= %p\n",
 		 __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
 		 qp_init->cap.max_send_wr, ch);
 
@@ -2299,7 +2298,7 @@ static int srpt_cm_req_recv(struct srpt_device *const sdev,
 	 * depth to avoid that the initiator driver has to report QUEUE_FULL
 	 * to the SCSI mid-layer.
 	 */
-	ch->rq_size = min(MAX_SRPT_RQ_SIZE, sdev->device->attrs.max_qp_wr);
+	ch->rq_size = min(sdev->device->attrs.max_qp_wr, MAX_SRPT_RQ_SIZE);
 	spin_lock_init(&ch->spinlock);
 	ch->state = CH_CONNECTING;
 	INIT_LIST_HEAD(&ch->cmd_wait_list);
@@ -3137,7 +3136,7 @@ static int srpt_alloc_srq(struct srpt_device *sdev)
 		return PTR_ERR(srq);
 	}
 
-	pr_debug("create SRQ #wr= %d max_allow=%d dev= %s\n", sdev->srq_size,
+	pr_debug("create SRQ #wr= %d max_allow=%u dev= %s\n", sdev->srq_size,
 		 sdev->device->attrs.max_srq_wr, dev_name(&device->dev));
 
 	sdev->req_buf_cache = srpt_cache_get(srp_max_req_size);
@@ -3952,7 +3951,7 @@ static int __init srpt_init_module(void)
 
 	if (srpt_srq_size < MIN_SRPT_SRQ_SIZE
 	    || srpt_srq_size > MAX_SRPT_SRQ_SIZE) {
-		pr_err("invalid value %d for kernel module parameter srpt_srq_size -- must be in the range [%d..%d].\n",
+		pr_err("invalid value %u for kernel module parameter srpt_srq_size -- must be in the range [%d..%d].\n",
 		       srpt_srq_size, MIN_SRPT_SRQ_SIZE, MAX_SRPT_SRQ_SIZE);
 		goto out;
 	}
diff --git a/drivers/input/joystick/adc-joystick.c b/drivers/input/joystick/adc-joystick.c
index ff44f9978b719..4fa42f88bcfa4 100644
--- a/drivers/input/joystick/adc-joystick.c
+++ b/drivers/input/joystick/adc-joystick.c
@@ -329,3 +329,4 @@ module_platform_driver(adc_joystick_driver);
 MODULE_DESCRIPTION("Input driver for joysticks connected over ADC");
 MODULE_AUTHOR("Artur Rojek <contact@artur-rojek.eu>");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/input/keyboard/adc-keys.c b/drivers/input/keyboard/adc-keys.c
index f1753207429db..d687459a0c803 100644
--- a/drivers/input/keyboard/adc-keys.c
+++ b/drivers/input/keyboard/adc-keys.c
@@ -202,3 +202,4 @@ module_platform_driver(adc_keys_driver);
 MODULE_AUTHOR("Alexandre Belloni <alexandre.belloni@free-electrons.com>");
 MODULE_DESCRIPTION("Input driver for resistor ladder connected on ADC");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/input/keyboard/cap11xx.c b/drivers/input/keyboard/cap11xx.c
index 2447c1ae21661..25b6f7bd97ee7 100644
--- a/drivers/input/keyboard/cap11xx.c
+++ b/drivers/input/keyboard/cap11xx.c
@@ -5,6 +5,8 @@
  * (c) 2014 Daniel Mack <linux@zonque.org>
  */
 
+#include <linux/bits.h>
+#include <linux/delay.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/interrupt.h>
@@ -16,63 +18,43 @@
 #include <linux/gpio/consumer.h>
 #include <linux/bitfield.h>
 
+#define CAP1114_REG_BUTTON_STATUS1	0x03
+#define CAP1114_REG_BUTTON_STATUS2	0x04
+#define CAP1114_REG_CONFIG2			0x40
+#define CAP1114_REG_CONFIG2_VOL_UP_DOWN	BIT(1)
+#define CAP1114_REG_LED_OUTPUT_CONTROL1	0x73
+
 #define CAP11XX_REG_MAIN_CONTROL	0x00
 #define CAP11XX_REG_MAIN_CONTROL_GAIN_SHIFT	(6)
 #define CAP11XX_REG_MAIN_CONTROL_GAIN_MASK	(0xc0)
 #define CAP11XX_REG_MAIN_CONTROL_DLSEEP		BIT(4)
-#define CAP11XX_REG_GENERAL_STATUS	0x02
 #define CAP11XX_REG_SENSOR_INPUT	0x03
-#define CAP11XX_REG_NOISE_FLAG_STATUS	0x0a
+#define CAP1114_REG_BUTTON_STATUS2	0x04
 #define CAP11XX_REG_SENOR_DELTA(X)	(0x10 + (X))
 #define CAP11XX_REG_SENSITIVITY_CONTROL	0x1f
 #define CAP11XX_REG_SENSITIVITY_CONTROL_DELTA_SENSE_MASK	0x70
-#define CAP11XX_REG_CONFIG		0x20
-#define CAP11XX_REG_SENSOR_ENABLE	0x21
-#define CAP11XX_REG_SENSOR_CONFIG	0x22
-#define CAP11XX_REG_SENSOR_CONFIG2	0x23
-#define CAP11XX_REG_SAMPLING_CONFIG	0x24
-#define CAP11XX_REG_CALIBRATION		0x26
-#define CAP11XX_REG_INT_ENABLE		0x27
 #define CAP11XX_REG_REPEAT_RATE		0x28
 #define CAP11XX_REG_SIGNAL_GUARD_ENABLE	0x29
-#define CAP11XX_REG_MT_CONFIG		0x2a
-#define CAP11XX_REG_MT_PATTERN_CONFIG	0x2b
-#define CAP11XX_REG_MT_PATTERN		0x2d
-#define CAP11XX_REG_RECALIB_CONFIG	0x2f
 #define CAP11XX_REG_SENSOR_THRESH(X)	(0x30 + (X))
-#define CAP11XX_REG_SENSOR_NOISE_THRESH	0x38
-#define CAP11XX_REG_STANDBY_CHANNEL	0x40
-#define CAP11XX_REG_STANDBY_CONFIG	0x41
-#define CAP11XX_REG_STANDBY_SENSITIVITY	0x42
-#define CAP11XX_REG_STANDBY_THRESH	0x43
 #define CAP11XX_REG_CONFIG2		0x44
 #define CAP11XX_REG_CONFIG2_ALT_POL	BIT(6)
-#define CAP11XX_REG_SENSOR_BASE_CNT(X)	(0x50 + (X))
-#define CAP11XX_REG_LED_POLARITY	0x73
 #define CAP11XX_REG_LED_OUTPUT_CONTROL	0x74
 #define CAP11XX_REG_CALIB_SENSITIVITY_CONFIG	0x80
 #define CAP11XX_REG_CALIB_SENSITIVITY_CONFIG2	0x81
-
-#define CAP11XX_REG_LED_DUTY_CYCLE_1	0x90
-#define CAP11XX_REG_LED_DUTY_CYCLE_2	0x91
-#define CAP11XX_REG_LED_DUTY_CYCLE_3	0x92
 #define CAP11XX_REG_LED_DUTY_CYCLE_4	0x93
 
-#define CAP11XX_REG_LED_DUTY_MIN_MASK	(0x0f)
-#define CAP11XX_REG_LED_DUTY_MIN_MASK_SHIFT	(0)
 #define CAP11XX_REG_LED_DUTY_MAX_MASK	(0xf0)
-#define CAP11XX_REG_LED_DUTY_MAX_MASK_SHIFT	(4)
 #define CAP11XX_REG_LED_DUTY_MAX_VALUE	(15)
 
-#define CAP11XX_REG_SENSOR_CALIB	(0xb1 + (X))
-#define CAP11XX_REG_SENSOR_CALIB_LSB1	0xb9
-#define CAP11XX_REG_SENSOR_CALIB_LSB2	0xba
 #define CAP11XX_REG_PRODUCT_ID		0xfd
 #define CAP11XX_REG_MANUFACTURER_ID	0xfe
 #define CAP11XX_REG_REVISION		0xff
 
 #define CAP11XX_MANUFACTURER_ID	0x5d
 
+#define CAP11XX_T_RST_FILT_MIN_US	10000
+#define CAP11XX_T_RST_ON_MIN_MS		400
+
 #ifdef CONFIG_LEDS_CLASS
 struct cap11xx_led {
 	struct cap11xx_priv *priv;
@@ -85,6 +67,7 @@ struct cap11xx_priv {
 	struct regmap *regmap;
 	struct device *dev;
 	struct input_dev *idev;
+	struct gpio_desc *reset_gpio;
 	const struct cap11xx_hw_model *model;
 
 	struct cap11xx_led *leds;
@@ -101,47 +84,32 @@ struct cap11xx_priv {
 
 struct cap11xx_hw_model {
 	u8 product_id;
+	u8 led_output_control_reg_base;
+	u8 sensor_input_reg_base;
 	unsigned int num_channels;
 	unsigned int num_leds;
+	unsigned int num_sensor_thresholds;
 	bool has_gain;
+	bool has_grouped_sensors;
 	bool has_irq_config;
+	bool has_repeat_en;
 	bool has_sensitivity_control;
 	bool has_signal_guard;
 };
 
 static const struct reg_default cap11xx_reg_defaults[] = {
 	{ CAP11XX_REG_MAIN_CONTROL,		0x00 },
-	{ CAP11XX_REG_GENERAL_STATUS,		0x00 },
-	{ CAP11XX_REG_SENSOR_INPUT,		0x00 },
-	{ CAP11XX_REG_NOISE_FLAG_STATUS,	0x00 },
 	{ CAP11XX_REG_SENSITIVITY_CONTROL,	0x2f },
-	{ CAP11XX_REG_CONFIG,			0x20 },
-	{ CAP11XX_REG_SENSOR_ENABLE,		0x3f },
-	{ CAP11XX_REG_SENSOR_CONFIG,		0xa4 },
-	{ CAP11XX_REG_SENSOR_CONFIG2,		0x07 },
-	{ CAP11XX_REG_SAMPLING_CONFIG,		0x39 },
-	{ CAP11XX_REG_CALIBRATION,		0x00 },
-	{ CAP11XX_REG_INT_ENABLE,		0x3f },
 	{ CAP11XX_REG_REPEAT_RATE,		0x3f },
-	{ CAP11XX_REG_MT_CONFIG,		0x80 },
-	{ CAP11XX_REG_MT_PATTERN_CONFIG,	0x00 },
-	{ CAP11XX_REG_MT_PATTERN,		0x3f },
-	{ CAP11XX_REG_RECALIB_CONFIG,		0x8a },
 	{ CAP11XX_REG_SENSOR_THRESH(0),		0x40 },
 	{ CAP11XX_REG_SENSOR_THRESH(1),		0x40 },
 	{ CAP11XX_REG_SENSOR_THRESH(2),		0x40 },
 	{ CAP11XX_REG_SENSOR_THRESH(3),		0x40 },
 	{ CAP11XX_REG_SENSOR_THRESH(4),		0x40 },
 	{ CAP11XX_REG_SENSOR_THRESH(5),		0x40 },
-	{ CAP11XX_REG_SENSOR_NOISE_THRESH,	0x01 },
-	{ CAP11XX_REG_STANDBY_CHANNEL,		0x00 },
-	{ CAP11XX_REG_STANDBY_CONFIG,		0x39 },
-	{ CAP11XX_REG_STANDBY_SENSITIVITY,	0x02 },
-	{ CAP11XX_REG_STANDBY_THRESH,		0x40 },
+	{ CAP11XX_REG_SENSOR_THRESH(6),		0x40 },
+	{ CAP11XX_REG_SENSOR_THRESH(7),		0x40 },
 	{ CAP11XX_REG_CONFIG2,			0x40 },
-	{ CAP11XX_REG_LED_POLARITY,		0x00 },
-	{ CAP11XX_REG_SENSOR_CALIB_LSB1,	0x00 },
-	{ CAP11XX_REG_SENSOR_CALIB_LSB2,	0x00 },
 };
 
 static bool cap11xx_volatile_reg(struct device *dev, unsigned int reg)
@@ -149,6 +117,12 @@ static bool cap11xx_volatile_reg(struct device *dev, unsigned int reg)
 	switch (reg) {
 	case CAP11XX_REG_MAIN_CONTROL:
 	case CAP11XX_REG_SENSOR_INPUT:
+	/*
+	 * CAP1114_REG_BUTTON_STATUS1 (CAP11XX_REG_SENSOR_INPUT) and
+	 * CAP1114_REG_BUTTON_STATUS2 is volatile for the CAP1114,
+	 * which supports more than 8 touch channels.
+	 */
+	case CAP1114_REG_BUTTON_STATUS2:
 	case CAP11XX_REG_SENOR_DELTA(0):
 	case CAP11XX_REG_SENOR_DELTA(1):
 	case CAP11XX_REG_SENOR_DELTA(2):
@@ -257,8 +231,8 @@ static int cap11xx_init_keys(struct cap11xx_priv *priv)
 	}
 
 	if (!of_property_read_u32_array(node, "microchip,input-threshold",
-					priv->thresholds, priv->model->num_channels)) {
-		for (i = 0; i < priv->model->num_channels; i++) {
+					priv->thresholds, priv->model->num_sensor_thresholds)) {
+		for (i = 0; i < priv->model->num_sensor_thresholds; i++) {
 			if (priv->thresholds[i] > 127) {
 				dev_err(dev, "Invalid input-threshold value %u\n",
 					priv->thresholds[i]);
@@ -273,10 +247,13 @@ static int cap11xx_init_keys(struct cap11xx_priv *priv)
 		}
 	}
 
-	if (!of_property_read_u32_array(node, "microchip,calib-sensitivity",
-					priv->calib_sensitivities,
-					priv->model->num_channels)) {
-		if (priv->model->has_sensitivity_control) {
+	if (of_property_present(node, "microchip,calib-sensitivity")) {
+		if (!priv->model->has_sensitivity_control) {
+			dev_warn(dev,
+				 "This model doesn't support 'calib-sensitivity'\n");
+		} else if (!of_property_read_u32_array(node, "microchip,calib-sensitivity",
+						       priv->calib_sensitivities,
+						       priv->model->num_channels)) {
 			for (i = 0; i < priv->model->num_channels; i++) {
 				if (!is_power_of_2(priv->calib_sensitivities[i]) ||
 				    priv->calib_sensitivities[i] > 4) {
@@ -296,32 +273,31 @@ static int cap11xx_init_keys(struct cap11xx_priv *priv)
 				if (error)
 					return error;
 			}
-		} else {
-			dev_warn(dev,
-				 "This model doesn't support 'calib-sensitivity'\n");
 		}
 	}
 
-	for (i = 0; i < priv->model->num_channels; i++) {
-		if (!of_property_read_u32_index(node, "microchip,signal-guard",
-						i, &u32_val)) {
-			if (u32_val > 1)
-				return -EINVAL;
-			if (u32_val)
-				priv->signal_guard_inputs_mask |= 0x01 << i;
-		}
-	}
-
-	if (priv->signal_guard_inputs_mask) {
-		if (priv->model->has_signal_guard) {
-			error = regmap_write(priv->regmap,
-					     CAP11XX_REG_SIGNAL_GUARD_ENABLE,
-					     priv->signal_guard_inputs_mask);
-			if (error)
-				return error;
-		} else {
+	if (of_property_present(node, "microchip,signal-guard")) {
+		if (!priv->model->has_signal_guard) {
 			dev_warn(dev,
 				 "This model doesn't support 'signal-guard'\n");
+		} else {
+			for (i = 0; i < priv->model->num_channels; i++) {
+				if (!of_property_read_u32_index(node, "microchip,signal-guard",
+								i, &u32_val)) {
+					if (u32_val > 1)
+						return -EINVAL;
+					if (u32_val)
+						priv->signal_guard_inputs_mask |= 0x01 << i;
+				}
+			}
+
+			if (priv->signal_guard_inputs_mask) {
+				error = regmap_write(priv->regmap,
+						     CAP11XX_REG_SIGNAL_GUARD_ENABLE,
+						     priv->signal_guard_inputs_mask);
+				if (error)
+					return error;
+			}
 		}
 	}
 
@@ -332,10 +308,23 @@ static int cap11xx_init_keys(struct cap11xx_priv *priv)
 	of_property_read_u32_array(node, "linux,keycodes",
 				   priv->keycodes, priv->model->num_channels);
 
-	/* Disable autorepeat. The Linux input system has its own handling. */
-	error = regmap_write(priv->regmap, CAP11XX_REG_REPEAT_RATE, 0);
-	if (error)
-		return error;
+	/*
+	 * CAP1114 needs dedicated configuration to split
+	 * grouped sensors into independent inputs.
+	 */
+	if (priv->model->has_grouped_sensors) {
+		error = regmap_set_bits(priv->regmap, CAP1114_REG_CONFIG2,
+					CAP1114_REG_CONFIG2_VOL_UP_DOWN);
+		if (error)
+			return error;
+	}
+
+	if (priv->model->has_repeat_en) {
+		/* Disable autorepeat. The Linux input system has its own handling. */
+		error = regmap_write(priv->regmap, CAP11XX_REG_REPEAT_RATE, 0);
+		if (error)
+			return error;
+	}
 
 	return 0;
 }
@@ -354,10 +343,25 @@ static irqreturn_t cap11xx_thread_func(int irq_num, void *data)
 	if (ret < 0)
 		goto out;
 
-	ret = regmap_read(priv->regmap, CAP11XX_REG_SENSOR_INPUT, &status);
+	ret = regmap_read(priv->regmap, priv->model->sensor_input_reg_base, &status);
 	if (ret < 0)
 		goto out;
 
+	if (priv->model->num_channels > 8) {
+		unsigned int status2;
+
+		ret = regmap_read(priv->regmap, priv->model->sensor_input_reg_base + 1, &status2);
+		if (ret < 0)
+			goto out;
+
+		/*
+		 * CAP1114 STATUS1 register only contains data for the first 6 channels.
+		 * the remaining channels is stored in STATUS2.
+		 */
+		status &= GENMASK(5, 0);
+		status |= FIELD_PREP(GENMASK(13, 6), status2);
+	}
+
 	for (i = 0; i < priv->idev->keycodemax; i++)
 		input_report_key(priv->idev, priv->keycodes[i],
 				 status & (1 << i));
@@ -407,10 +411,16 @@ static int cap11xx_led_set(struct led_classdev *cdev,
 	 * limitation. Brightness levels per LED are either
 	 * 0 (OFF) and 1 (ON).
 	 */
-	return regmap_update_bits(priv->regmap,
-				  CAP11XX_REG_LED_OUTPUT_CONTROL,
-				  BIT(led->reg),
-				  value ? BIT(led->reg) : 0);
+	if (led->reg >= 8)
+		return regmap_update_bits(priv->regmap,
+					  priv->model->led_output_control_reg_base + 1,
+					  BIT(led->reg - 8),
+					  value ? BIT(led->reg - 8) : 0);
+	else
+		return regmap_update_bits(priv->regmap,
+					  priv->model->led_output_control_reg_base,
+					  BIT(led->reg),
+					  value ? BIT(led->reg) : 0);
 }
 
 static int cap11xx_init_leds(struct device *dev,
@@ -420,6 +430,7 @@ static int cap11xx_init_leds(struct device *dev,
 	struct cap11xx_led *led;
 	int cnt = of_get_child_count(node);
 	int error;
+	u32 duty_val;
 
 	if (!num_leds || !cnt)
 		return 0;
@@ -433,15 +444,26 @@ static int cap11xx_init_leds(struct device *dev,
 
 	priv->leds = led;
 
+	/* Set all LEDs to off */
 	error = regmap_update_bits(priv->regmap,
-				CAP11XX_REG_LED_OUTPUT_CONTROL, 0xff, 0);
+				   priv->model->led_output_control_reg_base,
+				   GENMASK(min(num_leds, 8) - 1, 0), 0);
 	if (error)
 		return error;
 
+	if (num_leds > 8) {
+		error = regmap_update_bits(priv->regmap,
+					   priv->model->led_output_control_reg_base + 1,
+					   GENMASK(num_leds - 8 - 1, 0), 0);
+		if (error)
+			return error;
+	}
+
+	duty_val = FIELD_PREP(CAP11XX_REG_LED_DUTY_MAX_MASK,
+			      CAP11XX_REG_LED_DUTY_MAX_VALUE);
+
 	error = regmap_update_bits(priv->regmap, CAP11XX_REG_LED_DUTY_CYCLE_4,
-				CAP11XX_REG_LED_DUTY_MAX_MASK,
-				CAP11XX_REG_LED_DUTY_MAX_VALUE <<
-				CAP11XX_REG_LED_DUTY_MAX_MASK_SHIFT);
+				   CAP11XX_REG_LED_DUTY_MAX_MASK, duty_val);
 	if (error)
 		return error;
 
@@ -510,9 +532,20 @@ static int cap11xx_i2c_probe(struct i2c_client *i2c_client)
 	if (IS_ERR(priv->regmap))
 		return PTR_ERR(priv->regmap);
 
+	priv->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(priv->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(priv->reset_gpio),
+				     "Failed to get 'reset' GPIO\n");
+
+	if (priv->reset_gpio) {
+		usleep_range(CAP11XX_T_RST_FILT_MIN_US, CAP11XX_T_RST_FILT_MIN_US * 2);
+		gpiod_set_value_cansleep(priv->reset_gpio, 0);
+		msleep(CAP11XX_T_RST_ON_MIN_MS);
+	}
+
 	error = regmap_read(priv->regmap, CAP11XX_REG_PRODUCT_ID, &val);
 	if (error)
-		return error;
+		return dev_err_probe(dev, error, "Failed to read product ID\n");
 
 	if (val != cap->product_id) {
 		dev_err(dev, "Product ID: Got 0x%02x, expected 0x%02x\n",
@@ -522,7 +555,7 @@ static int cap11xx_i2c_probe(struct i2c_client *i2c_client)
 
 	error = regmap_read(priv->regmap, CAP11XX_REG_MANUFACTURER_ID, &val);
 	if (error)
-		return error;
+		return dev_err_probe(dev, error, "Failed to read manufacturer ID\n");
 
 	if (val != CAP11XX_MANUFACTURER_ID) {
 		dev_err(dev, "Manufacturer ID: Got 0x%02x, expected 0x%02x\n",
@@ -531,11 +564,8 @@ static int cap11xx_i2c_probe(struct i2c_client *i2c_client)
 	}
 
 	error = regmap_read(priv->regmap, CAP11XX_REG_REVISION, &rev);
-	if (error < 0)
-		return error;
-
-	dev_info(dev, "CAP11XX detected, model %s, revision 0x%02x\n",
-			 id->name, rev);
+	if (error)
+		return dev_err_probe(dev, error, "Failed to read revision\n");
 
 	priv->model = cap;
 
@@ -599,47 +629,79 @@ static int cap11xx_i2c_probe(struct i2c_client *i2c_client)
 }
 
 static const struct cap11xx_hw_model cap1106_model = {
-	.product_id = 0x55, .num_channels = 6, .num_leds = 0,
+	.product_id = 0x55,
+	.num_channels = 6, .num_leds = 0, .num_sensor_thresholds = 6,
+	.sensor_input_reg_base = CAP11XX_REG_SENSOR_INPUT,
 	.has_gain = true,
 	.has_irq_config = true,
+	.has_repeat_en = true,
+};
+
+static const struct cap11xx_hw_model cap1114_model = {
+	.product_id = 0x3a,
+	.num_channels = 14, .num_leds = 11, .num_sensor_thresholds = 8,
+	.led_output_control_reg_base = CAP1114_REG_LED_OUTPUT_CONTROL1,
+	.sensor_input_reg_base = CAP1114_REG_BUTTON_STATUS1,
+	.has_grouped_sensors = true,
 };
 
 static const struct cap11xx_hw_model cap1126_model = {
-	.product_id = 0x53, .num_channels = 6, .num_leds = 2,
+	.product_id = 0x53,
+	.num_channels = 6, .num_leds = 2, .num_sensor_thresholds = 6,
+	.led_output_control_reg_base = CAP11XX_REG_LED_OUTPUT_CONTROL,
+	.sensor_input_reg_base = CAP11XX_REG_SENSOR_INPUT,
 	.has_gain = true,
 	.has_irq_config = true,
+	.has_repeat_en = true,
 };
 
 static const struct cap11xx_hw_model cap1188_model = {
-	.product_id = 0x50, .num_channels = 8, .num_leds = 8,
+	.product_id = 0x50,
+	.num_channels = 8, .num_leds = 8, .num_sensor_thresholds = 8,
+	.led_output_control_reg_base = CAP11XX_REG_LED_OUTPUT_CONTROL,
+	.sensor_input_reg_base = CAP11XX_REG_SENSOR_INPUT,
 	.has_gain = true,
 	.has_irq_config = true,
+	.has_repeat_en = true,
 };
 
 static const struct cap11xx_hw_model cap1203_model = {
-	.product_id = 0x6d, .num_channels = 3, .num_leds = 0,
+	.product_id = 0x6d,
+	.num_channels = 3, .num_leds = 0, .num_sensor_thresholds = 3,
+	.sensor_input_reg_base = CAP11XX_REG_SENSOR_INPUT,
+	.has_repeat_en = true,
 };
 
 static const struct cap11xx_hw_model cap1206_model = {
-	.product_id = 0x67, .num_channels = 6, .num_leds = 0,
+	.product_id = 0x67,
+	.num_channels = 6, .num_leds = 0, .num_sensor_thresholds = 6,
+	.sensor_input_reg_base = CAP11XX_REG_SENSOR_INPUT,
+	.has_repeat_en = true,
 };
 
 static const struct cap11xx_hw_model cap1293_model = {
-	.product_id = 0x6f, .num_channels = 3, .num_leds = 0,
+	.product_id = 0x6f,
+	.num_channels = 3, .num_leds = 0, .num_sensor_thresholds = 3,
+	.sensor_input_reg_base = CAP11XX_REG_SENSOR_INPUT,
 	.has_gain = true,
+	.has_repeat_en = true,
 	.has_sensitivity_control = true,
 	.has_signal_guard = true,
 };
 
 static const struct cap11xx_hw_model cap1298_model = {
-	.product_id = 0x71, .num_channels = 8, .num_leds = 0,
+	.product_id = 0x71,
+	.num_channels = 8, .num_leds = 0, .num_sensor_thresholds = 8,
+	.sensor_input_reg_base = CAP11XX_REG_SENSOR_INPUT,
 	.has_gain = true,
+	.has_repeat_en = true,
 	.has_sensitivity_control = true,
 	.has_signal_guard = true,
 };
 
 static const struct of_device_id cap11xx_dt_ids[] = {
 	{ .compatible = "microchip,cap1106", .data = &cap1106_model },
+	{ .compatible = "microchip,cap1114", .data = &cap1114_model },
 	{ .compatible = "microchip,cap1126", .data = &cap1126_model },
 	{ .compatible = "microchip,cap1188", .data = &cap1188_model },
 	{ .compatible = "microchip,cap1203", .data = &cap1203_model },
@@ -652,6 +714,7 @@ MODULE_DEVICE_TABLE(of, cap11xx_dt_ids);
 
 static const struct i2c_device_id cap11xx_i2c_ids[] = {
 	{ .name = "cap1106", .driver_data = (kernel_ulong_t)&cap1106_model },
+	{ .name = "cap1114", .driver_data = (kernel_ulong_t)&cap1114_model },
 	{ .name = "cap1126", .driver_data = (kernel_ulong_t)&cap1126_model },
 	{ .name = "cap1188", .driver_data = (kernel_ulong_t)&cap1188_model },
 	{ .name = "cap1203", .driver_data = (kernel_ulong_t)&cap1203_model },
diff --git a/drivers/input/keyboard/matrix_keypad.c b/drivers/input/keyboard/matrix_keypad.c
index e50a6fea9a602..8863b741d1a32 100644
--- a/drivers/input/keyboard/matrix_keypad.c
+++ b/drivers/input/keyboard/matrix_keypad.c
@@ -16,7 +16,6 @@
 #include <linux/interrupt.h>
 #include <linux/jiffies.h>
 #include <linux/module.h>
-#include <linux/gpio.h>
 #include <linux/input/matrix_keypad.h>
 #include <linux/slab.h>
 #include <linux/of.h>
diff --git a/drivers/input/keyboard/tca8418_keypad.c b/drivers/input/keyboard/tca8418_keypad.c
index eb5be644f236d..4616afa0286c3 100644
--- a/drivers/input/keyboard/tca8418_keypad.c
+++ b/drivers/input/keyboard/tca8418_keypad.c
@@ -254,7 +254,8 @@ static int tca8418_configure(struct tca8418_keypad *keypad_data,
 		return error;
 
 	error = tca8418_write_byte(keypad_data, REG_CFG,
-				CFG_INT_CFG | CFG_OVR_FLOW_IEN | CFG_KE_IEN);
+				   CFG_INT_CFG | CFG_OVR_FLOW_IEN |
+					CFG_OVR_FLOW_M | CFG_KE_IEN);
 
 	return error;
 }
diff --git a/drivers/input/misc/Kconfig b/drivers/input/misc/Kconfig
index 1f6c57dba0308..7154eaf5a60b1 100644
--- a/drivers/input/misc/Kconfig
+++ b/drivers/input/misc/Kconfig
@@ -842,6 +842,18 @@ config INPUT_IQS7222
 	  To compile this driver as a module, choose M here: the
 	  module will be called iqs7222.
 
+config INPUT_ISA1200_HAPTIC
+	tristate "Imagis ISA1200 haptic feedback unit"
+	depends on I2C
+	select INPUT_FF_MEMLESS
+	select REGMAP_I2C
+	help
+	  Say Y to enable support for the Imagis ISA1200 haptic
+	  feedback unit.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called isa1200.
+
 config INPUT_CMA3000
 	tristate "VTI CMA3000 Tri-axis accelerometer"
 	help
diff --git a/drivers/input/misc/Makefile b/drivers/input/misc/Makefile
index 2281d6803fce9..e9f85ca20c335 100644
--- a/drivers/input/misc/Makefile
+++ b/drivers/input/misc/Makefile
@@ -49,6 +49,7 @@ obj-$(CONFIG_INPUT_IMS_PCU)		+= ims-pcu.o
 obj-$(CONFIG_INPUT_IQS269A)		+= iqs269a.o
 obj-$(CONFIG_INPUT_IQS626A)		+= iqs626a.o
 obj-$(CONFIG_INPUT_IQS7222)		+= iqs7222.o
+obj-$(CONFIG_INPUT_ISA1200_HAPTIC)	+= isa1200.o
 obj-$(CONFIG_INPUT_KEYSPAN_REMOTE)	+= keyspan_remote.o
 obj-$(CONFIG_INPUT_KXTJ9)		+= kxtj9.o
 obj-$(CONFIG_INPUT_M68K_BEEP)		+= m68kspkr.o
diff --git a/drivers/input/misc/ims-pcu.c b/drivers/input/misc/ims-pcu.c
index b1ff8c70877fe..b1a0edcc49b4e 100644
--- a/drivers/input/misc/ims-pcu.c
+++ b/drivers/input/misc/ims-pcu.c
@@ -2234,6 +2234,7 @@ static const struct usb_device_id ims_pcu_id_table[] = {
 	},
 	{ }
 };
+MODULE_DEVICE_TABLE(usb, ims_pcu_id_table);
 
 static const struct attribute_group *ims_pcu_sysfs_groups[] = {
 	&ims_pcu_attr_group,
diff --git a/drivers/input/misc/isa1200.c b/drivers/input/misc/isa1200.c
new file mode 100644
index 0000000000000..926cffcd38d67
--- /dev/null
+++ b/drivers/input/misc/isa1200.c
@@ -0,0 +1,533 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <linux/array_size.h>
+#include <linux/bitmap.h>
+#include <linux/bits.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/devm-helpers.h>
+#include <linux/err.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/input.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/pwm.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/units.h>
+
+/*
+ * System control (LDO regulator)
+ *
+ * LDO voltage to register mapping is linear, but it is split in two parts:
+ * 2.3V - 3.0V map to 0x08 - 0x0f; 3.1V - 3.8V map to 0x00 - 0x7
+ */
+
+#define ISA1200_SCTRL			0x00
+#define ISA1200_LDO_VOLTAGE_BASE	0x08
+#define ISA1200_LDO_VOLTAGE_STEP	100000
+#define ISA1200_LDO_VOLTAGE_2V3		23
+#define ISA1200_LDO_VOLTAGE_3V1		31
+#define ISA1200_LDO_VOLTAGE_MIN		2300000
+#define ISA1200_LDO_VOLTAGE_MAX		3800000
+
+/*
+ * The output frequency is calculated with this formula:
+ *
+ *                 base clock frequency
+ * fout = -----------------------------------------
+ *        (128 - PWM_FREQ) * 2 * PLLDIV * PWM_PERIOD
+ *
+ * The base clock frequency is the clock frequency provided on the
+ * clock input to the chip, divided by the value in HCTRL0
+ *
+ * PWM_FREQ is configured in register HCTRL4, it is common to set this
+ * to 0 to get only two variables to calculate.
+ *
+ * PLLDIV is configured in register HCTRL3 (bits 7..4, so 0..15)
+ * PWM_PERIOD is configured in register HCTRL6
+ * Further the duty cycle can be configured in HCTRL5
+ */
+
+/*
+ * HCTRL0 configures clock or PWM input and selects the divider for
+ * the clock input.
+ */
+#define ISA1200_HCTRL0			0x30
+#define ISA1200_HCTRL0_HAP_ENABLE	BIT(7)
+#define ISA1200_HCTRL0_PWM_GEN_MODE	BIT(4)
+#define ISA1200_HCTRL0_PWM_INPUT_MODE	BIT(3)
+#define ISA1200_HCTRL0_CLKDIV_128	128
+
+/*
+ * HCTRL1 configures the motor type and clock sourse
+ */
+#define ISA1200_HCTRL1			0x31
+#define ISA1200_HCTRL1_EXT_CLOCK	BIT(7)
+#define ISA1200_HCTRL1_DAC_INVERT	BIT(6)
+#define ISA1200_HCTRL1_MODE(n)		(((n) & 1) << 5)
+
+/* HCTRL2 controls software reset of the chip */
+#define ISA1200_HCTRL2			0x32
+#define ISA1200_HCTRL2_SW_RESET		BIT(0)
+
+/*
+ * HCTRL3 controls the PLL divisor
+ *
+ * Bits [0,1] are always set to 1 (we don't know what they are
+ * used for) and bit 4 and upward control the PLL divisor.
+ */
+#define ISA1200_HCTRL3			0x33
+#define ISA1200_HCTRL3_DEFAULT		0x03
+#define ISA1200_HCTRL3_PLLDIV(n)	(((n) & 0xf) << 4)
+
+/* HCTRL4 controls the PWM frequency of external channel */
+#define ISA1200_HCTRL4			0x34
+
+/* HCTRL5 controls the PWM high duty cycle of internal channel */
+#define ISA1200_HCTRL5			0x35
+
+/* HCTRL6 controls the PWM period of internal channel */
+#define ISA1200_HCTRL6			0x36
+#define ISA1200_HCTRL6_PERIOD_SCALE	100
+
+/* The use for these registers is unknown but they exist */
+#define ISA1200_HCTRL7			0x37
+#define ISA1200_HCTRL8			0x38
+#define ISA1200_HCTRL9			0x39
+#define ISA1200_HCTRLA			0x3a
+#define ISA1200_HCTRLB			0x3b
+#define ISA1200_HCTRLC			0x3c
+#define ISA1200_HCTRLD			0x3d
+
+#define ISA1200_EN_PINS_MAX		2
+
+static const struct regulator_bulk_data isa1200_supplies[] = {
+	{ .supply = "vdd" }, { .supply = "vddp" },
+};
+
+struct isa1200_config {
+	u32 ldo_voltage;
+	u32 mode;
+	u32 clkdiv;
+	u32 plldiv;
+	u32 freq;
+	u32 period;
+	u32 duty;
+};
+
+struct isa1200 {
+	struct input_dev *input;
+	struct regmap *map;
+
+	struct clk *clk;
+	struct pwm_device *pwm;
+	struct gpio_descs *enable_gpios;
+	struct regulator_bulk_data *supplies;
+
+	struct work_struct play_work;
+	struct isa1200_config config;
+
+	int level;
+	bool suspended;
+	bool active;
+};
+
+static const struct regmap_config isa1200_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = ISA1200_HCTRLD,
+};
+
+static void isa1200_start(struct isa1200 *isa)
+{
+	struct isa1200_config *config = &isa->config;
+	struct device *dev = &isa->input->dev;
+	struct pwm_state state;
+	u8 hctrl0 = 0, hctrl1 = 0;
+	DECLARE_BITMAP(values, ISA1200_EN_PINS_MAX);
+	int err;
+
+	if (!isa->active) {
+		err = regulator_bulk_enable(ARRAY_SIZE(isa1200_supplies),
+					    isa->supplies);
+		if (err) {
+			dev_err(dev, "failed to enable supplies (%d)\n", err);
+			return;
+		}
+
+		err = clk_prepare_enable(isa->clk);
+		if (err) {
+			dev_err(dev, "failed to enable clock (%d)\n", err);
+			regulator_bulk_disable(ARRAY_SIZE(isa1200_supplies),
+					       isa->supplies);
+			return;
+		}
+
+		bitmap_fill(values, ISA1200_EN_PINS_MAX);
+		gpiod_multi_set_value_cansleep(isa->enable_gpios, values);
+
+		usleep_range(200, 300);
+	}
+
+	regmap_write(isa->map, ISA1200_SCTRL, config->ldo_voltage);
+
+	if (isa->clk) {
+		hctrl0 = ISA1200_HCTRL0_PWM_GEN_MODE;
+		hctrl1 = ISA1200_HCTRL1_EXT_CLOCK;
+	}
+
+	if (isa->pwm) {
+		hctrl0 = ISA1200_HCTRL0_PWM_INPUT_MODE;
+		hctrl1 = 0;
+	}
+
+	hctrl0 |= __ffs(config->clkdiv / ISA1200_HCTRL0_CLKDIV_128);
+	hctrl1 |= ISA1200_HCTRL1_DAC_INVERT;
+	hctrl1 |= ISA1200_HCTRL1_MODE(config->mode);
+
+	regmap_write(isa->map, ISA1200_HCTRL0, hctrl0);
+	regmap_write(isa->map, ISA1200_HCTRL1, hctrl1);
+
+	/* Make sure to de-assert software reset */
+	regmap_write(isa->map, ISA1200_HCTRL2, 0x00);
+
+	/* PLL divisor */
+	regmap_write(isa->map, ISA1200_HCTRL3,
+		     ISA1200_HCTRL3_PLLDIV(config->plldiv) |
+		     ISA1200_HCTRL3_DEFAULT);
+
+	/* Frequency */
+	regmap_write(isa->map, ISA1200_HCTRL4, config->freq);
+	/* Duty cycle */
+	regmap_write(isa->map, ISA1200_HCTRL5, config->period >> 1);
+	/* Period */
+	regmap_write(isa->map, ISA1200_HCTRL6, config->period);
+
+	hctrl0 |= ISA1200_HCTRL0_HAP_ENABLE;
+	regmap_write(isa->map, ISA1200_HCTRL0, hctrl0);
+
+	if (isa->clk)
+		regmap_write(isa->map, ISA1200_HCTRL5, config->duty);
+
+	if (isa->pwm) {
+		pwm_get_state(isa->pwm, &state);
+		state.duty_cycle = config->duty;
+		state.enabled = true;
+		pwm_apply_might_sleep(isa->pwm, &state);
+	}
+
+	isa->active = true;
+}
+
+static void isa1200_stop(struct isa1200 *isa)
+{
+	struct pwm_state state;
+	DECLARE_BITMAP(values, ISA1200_EN_PINS_MAX);
+
+	if (!isa->active)
+		return;
+
+	if (isa->pwm) {
+		pwm_get_state(isa->pwm, &state);
+		state.duty_cycle = 0;
+		state.enabled = false;
+		pwm_apply_might_sleep(isa->pwm, &state);
+	}
+
+	regmap_write(isa->map, ISA1200_HCTRL0, 0x00);
+
+	bitmap_zero(values, ISA1200_EN_PINS_MAX);
+	gpiod_multi_set_value_cansleep(isa->enable_gpios, values);
+
+	clk_disable_unprepare(isa->clk);
+	regulator_bulk_disable(ARRAY_SIZE(isa1200_supplies),
+			       isa->supplies);
+
+	isa->active = false;
+}
+
+static void isa1200_play_work(struct work_struct *work)
+{
+	struct isa1200 *isa = container_of(work, struct isa1200, play_work);
+
+	if (!READ_ONCE(isa->suspended)) {
+		if (isa->level)
+			isa1200_start(isa);
+		else
+			isa1200_stop(isa);
+	}
+}
+
+static int isa1200_vibrator_play_effect(struct input_dev *input, void *data,
+					struct ff_effect *effect)
+{
+	struct isa1200 *isa = input_get_drvdata(input);
+	int level;
+
+	/*
+	 * TODO: we currently only support rumble.
+	 * The ISA1200 can control two motors and some devices
+	 * also have two motors mounted.
+	 */
+	level = effect->u.rumble.strong_magnitude;
+	if (!level)
+		level = effect->u.rumble.weak_magnitude;
+
+	dev_dbg(&input->dev, "FF effect type %d level %d\n",
+		effect->type, level);
+
+	if (isa->level != level) {
+		isa->level = level;
+		if (!READ_ONCE(isa->suspended))
+			schedule_work(&isa->play_work);
+	}
+
+	return 0;
+}
+
+static void isa1200_vibrator_close(struct input_dev *input)
+{
+	struct isa1200 *isa = input_get_drvdata(input);
+
+	cancel_work_sync(&isa->play_work);
+	isa1200_stop(isa);
+	isa->level = 0;
+}
+
+static int isa1200_of_probe(struct i2c_client *client)
+{
+	struct isa1200 *isa = i2c_get_clientdata(client);
+	struct isa1200_config *config = &isa->config;
+	struct device *dev = &client->dev;
+	struct fwnode_handle *ldo_node;
+	int err;
+
+	isa->clk = devm_clk_get_optional(dev, NULL);
+	if (IS_ERR(isa->clk))
+		return dev_err_probe(dev, PTR_ERR(isa->clk),
+				     "failed to get clock\n");
+
+	isa->pwm = devm_pwm_get(dev, NULL);
+	if (IS_ERR(isa->pwm)) {
+		err = PTR_ERR(isa->pwm);
+		if (err == -ENODEV || err == -EINVAL)
+			isa->pwm = NULL;
+		else
+			return dev_err_probe(dev, err, "getting PWM\n");
+	}
+
+	if (!isa->clk && !isa->pwm)
+		return dev_err_probe(dev, -EINVAL,
+				     "clock or PWM are required, none were provided\n");
+
+	err = devm_regulator_bulk_get_const(dev, ARRAY_SIZE(isa1200_supplies),
+					    isa1200_supplies, &isa->supplies);
+	if (err)
+		return dev_err_probe(dev, err, "failed to get supplies\n");
+
+	isa->enable_gpios = devm_gpiod_get_array_optional(dev, "control",
+							  GPIOD_OUT_LOW);
+	if (IS_ERR(isa->enable_gpios))
+		return dev_err_probe(dev, PTR_ERR(isa->enable_gpios),
+				     "failed to get enable gpios\n");
+
+	if (isa->enable_gpios && isa->enable_gpios->ndescs > ISA1200_EN_PINS_MAX)
+		return dev_err_probe(dev, -EINVAL, "too many enable gpios\n");
+
+	ldo_node = device_get_named_child_node(dev, "ldo");
+	if (!ldo_node)
+		return dev_err_probe(dev, -ENODEV,
+				     "failed to get embedded LDO node\n");
+
+	err = fwnode_property_read_u32(ldo_node, "regulator-min-microvolt",
+				       &config->ldo_voltage);
+	fwnode_handle_put(ldo_node);
+	if (err)
+		return dev_err_probe(dev, err,
+				     "failed to get ldo voltage\n");
+
+	config->ldo_voltage = clamp(config->ldo_voltage,
+				    ISA1200_LDO_VOLTAGE_MIN,
+				    ISA1200_LDO_VOLTAGE_MAX);
+
+	config->ldo_voltage /= ISA1200_LDO_VOLTAGE_STEP;
+	if (config->ldo_voltage < ISA1200_LDO_VOLTAGE_3V1)
+		config->ldo_voltage = config->ldo_voltage -
+				      ISA1200_LDO_VOLTAGE_2V3 +
+				      ISA1200_LDO_VOLTAGE_BASE;
+	else
+		config->ldo_voltage -= ISA1200_LDO_VOLTAGE_3V1;
+
+	config->mode = 0; /* LRA_MODE */
+	device_property_read_u32(dev, "imagis,mode", &config->mode);
+
+	config->clkdiv = ISA1200_HCTRL0_CLKDIV_128;
+	device_property_read_u32(dev, "imagis,clk-div", &config->clkdiv);
+	if (!config->clkdiv)
+		return dev_err_probe(dev, -EINVAL, "clk-div cannot be zero\n");
+
+	config->clkdiv = clamp(config->clkdiv, ISA1200_HCTRL0_CLKDIV_128,
+			       ISA1200_HCTRL0_CLKDIV_128 << 3);
+
+	err = device_property_read_u32(dev, "imagis,pll-div", &config->plldiv);
+	if (err || !config->plldiv)
+		config->plldiv = 1;
+
+	config->period = 0;
+	config->freq = 0;
+	config->duty = 0;
+
+	if (isa->clk) {
+		err = device_property_read_u32(dev, "imagis,period-ns",
+					       &config->period);
+		if (err)
+			return dev_err_probe(dev, err,
+					     "failed to get period\n");
+
+		/*
+		 * TODO: The scale value is arbitrary, but it fits observations
+		 * quite well, and the exact conversion method is unknown.
+		 * The period property value returned above is the HCTRL6
+		 * register value set by the vendor code, multiplied by 100.
+		 */
+		config->period /= ISA1200_HCTRL6_PERIOD_SCALE;
+		config->duty = config->period >> 1;
+	}
+
+	if (isa->pwm) {
+		struct pwm_state state;
+
+		pwm_init_state(isa->pwm, &state);
+
+		if (!state.period)
+			return dev_err_probe(dev, -EINVAL,
+					     "PWM period cannot be zero\n");
+
+		config->freq = div64_u64(NANO, state.period * config->clkdiv);
+		config->duty = state.period >> 1;
+
+		err = pwm_apply_might_sleep(isa->pwm, &state);
+		if (err)
+			return dev_err_probe(dev, err,
+					     "failed to apply initial PWM state\n");
+	}
+
+	/*
+	 * TODO: If device is using a clock, this property should return the
+	 * value written to the HCTRL5 register by downstrem code. It likely
+	 * needs to be converted into a meaningful duty cycle value, though
+	 * unfortunately the exact conversion mechanism is unknown. If the
+	 * device uses PWM, this property will return the correct duty cycle
+	 * in nanoseconds.
+	 */
+	device_property_read_u32(dev, "imagis,duty-cycle-ns", &config->duty);
+
+	return 0;
+}
+
+static int isa1200_probe(struct i2c_client *client)
+{
+	struct isa1200 *isa;
+	struct device *dev = &client->dev;
+	int err;
+
+	isa = devm_kzalloc(dev, sizeof(*isa), GFP_KERNEL);
+	if (!isa)
+		return -ENOMEM;
+
+	isa->input = devm_input_allocate_device(dev);
+	if (!isa->input)
+		return -ENOMEM;
+
+	i2c_set_clientdata(client, isa);
+
+	err = isa1200_of_probe(client);
+	if (err)
+		return err;
+
+	isa->map = devm_regmap_init_i2c(client, &isa1200_regmap_config);
+	if (IS_ERR(isa->map))
+		return dev_err_probe(dev, PTR_ERR(isa->map),
+				     "failed to initialize register map\n");
+
+	INIT_WORK(&isa->play_work, isa1200_play_work);
+
+	isa->input->name = "isa1200-haptic";
+	isa->input->id.bustype = BUS_I2C;
+	isa->input->close = isa1200_vibrator_close;
+
+	isa->active = false;
+
+	input_set_drvdata(isa->input, isa);
+
+	/* TODO: this hardware can likely support more than rumble */
+	input_set_capability(isa->input, EV_FF, FF_RUMBLE);
+
+	err = input_ff_create_memless(isa->input, NULL,
+				      isa1200_vibrator_play_effect);
+	if (err)
+		return dev_err_probe(dev, err, "failed to create FF dev\n");
+
+	err = input_register_device(isa->input);
+	if (err)
+		return dev_err_probe(dev, err, "failed to register input dev\n");
+
+	return 0;
+}
+
+static int isa1200_suspend(struct device *dev)
+{
+	struct isa1200 *isa = dev_get_drvdata(dev);
+
+	guard(mutex)(&isa->input->mutex);
+
+	if (input_device_enabled(isa->input)) {
+		WRITE_ONCE(isa->suspended, true);
+		cancel_work_sync(&isa->play_work);
+		isa1200_stop(isa);
+	}
+
+	return 0;
+}
+
+static int isa1200_resume(struct device *dev)
+{
+	struct isa1200 *isa = dev_get_drvdata(dev);
+
+	guard(mutex)(&isa->input->mutex);
+
+	if (input_device_enabled(isa->input)) {
+		WRITE_ONCE(isa->suspended, false);
+		if (isa->level)
+			schedule_work(&isa->play_work);
+	}
+
+	return 0;
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(isa1200_pm_ops, isa1200_suspend, isa1200_resume);
+
+static const struct of_device_id isa1200_of_match[] = {
+	{ .compatible = "imagis,isa1200" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, isa1200_of_match);
+
+static struct i2c_driver isa1200_i2c_driver = {
+	.driver = {
+		.name = "isa1200",
+		.of_match_table = isa1200_of_match,
+		.pm = pm_sleep_ptr(&isa1200_pm_ops),
+	},
+	.probe = isa1200_probe,
+};
+module_i2c_driver(isa1200_i2c_driver);
+
+MODULE_AUTHOR("Linus Walleij <linusw@kernel.org>");
+MODULE_AUTHOR("Svyatoslav Ryhel <clamor95@gmail.com>");
+MODULE_DESCRIPTION("Imagis ISA1200 haptic feedback unit");
+MODULE_LICENSE("GPL");
diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c
index c70502e24031a..2170bbe4c5897 100644
--- a/drivers/input/mouse/synaptics.c
+++ b/drivers/input/mouse/synaptics.c
@@ -164,6 +164,7 @@ static const char * const topbuttonpad_pnp_ids[] = {
 #ifdef CONFIG_MOUSE_PS2_SYNAPTICS_SMBUS
 static const char * const smbus_pnp_ids[] = {
 	/* all of the topbuttonpad_pnp_ids are valid, we just add some extras */
+	"DLL0597", /* Dell Inspiron 3521 */
 	"DLL060d", /* Dell Precision M3800 */
 	"LEN0048", /* X1 Carbon 3 */
 	"LEN0046", /* X250 */
diff --git a/drivers/input/serio/i8042-acpipnpio.h b/drivers/input/serio/i8042-acpipnpio.h
index 412f82d7a303b..9ecb0eed48c4b 100644
--- a/drivers/input/serio/i8042-acpipnpio.h
+++ b/drivers/input/serio/i8042-acpipnpio.h
@@ -3,6 +3,7 @@
 #define _I8042_ACPIPNPIO_H
 
 #include <linux/acpi.h>
+#include <linux/seq_buf.h>
 
 #ifdef CONFIG_X86
 #include <asm/x86_init.h>
@@ -1479,17 +1480,21 @@ static char i8042_pnp_aux_name[32];
 
 static void i8042_pnp_id_to_string(struct pnp_id *id, char *dst, int dst_size)
 {
-	strscpy(dst, "PNP:", dst_size);
+	struct seq_buf sb;
+
+	seq_buf_init(&sb, dst, dst_size);
+	seq_buf_printf(&sb, "PNP:");
 
 	while (id) {
-		strlcat(dst, " ", dst_size);
-		strlcat(dst, id->id, dst_size);
+		seq_buf_printf(&sb, " %s", id->id);
 		id = id->next;
 	}
 }
 
 static int i8042_pnp_kbd_probe(struct pnp_dev *dev, const struct pnp_device_id *did)
 {
+	const char *name = pnp_dev_name(dev);
+
 	if (pnp_port_valid(dev, 0) && pnp_port_len(dev, 0) == 1)
 		i8042_pnp_data_reg = pnp_port_start(dev,0);
 
@@ -1499,11 +1504,8 @@ static int i8042_pnp_kbd_probe(struct pnp_dev *dev, const struct pnp_device_id *
 	if (pnp_irq_valid(dev,0))
 		i8042_pnp_kbd_irq = pnp_irq(dev, 0);
 
-	strscpy(i8042_pnp_kbd_name, did->id, sizeof(i8042_pnp_kbd_name));
-	if (strlen(pnp_dev_name(dev))) {
-		strlcat(i8042_pnp_kbd_name, ":", sizeof(i8042_pnp_kbd_name));
-		strlcat(i8042_pnp_kbd_name, pnp_dev_name(dev), sizeof(i8042_pnp_kbd_name));
-	}
+	scnprintf(i8042_pnp_kbd_name, sizeof(i8042_pnp_kbd_name), "%s%s%s",
+		  did->id, strlen(name) ? ":" : "", name);
 	i8042_pnp_id_to_string(dev->id, i8042_kbd_firmware_id,
 			       sizeof(i8042_kbd_firmware_id));
 	i8042_kbd_fwnode = dev_fwnode(&dev->dev);
@@ -1517,6 +1519,8 @@ static int i8042_pnp_kbd_probe(struct pnp_dev *dev, const struct pnp_device_id *
 
 static int i8042_pnp_aux_probe(struct pnp_dev *dev, const struct pnp_device_id *did)
 {
+	const char *name = pnp_dev_name(dev);
+
 	if (pnp_port_valid(dev, 0) && pnp_port_len(dev, 0) == 1)
 		i8042_pnp_data_reg = pnp_port_start(dev,0);
 
@@ -1526,11 +1530,8 @@ static int i8042_pnp_aux_probe(struct pnp_dev *dev, const struct pnp_device_id *
 	if (pnp_irq_valid(dev, 0))
 		i8042_pnp_aux_irq = pnp_irq(dev, 0);
 
-	strscpy(i8042_pnp_aux_name, did->id, sizeof(i8042_pnp_aux_name));
-	if (strlen(pnp_dev_name(dev))) {
-		strlcat(i8042_pnp_aux_name, ":", sizeof(i8042_pnp_aux_name));
-		strlcat(i8042_pnp_aux_name, pnp_dev_name(dev), sizeof(i8042_pnp_aux_name));
-	}
+	scnprintf(i8042_pnp_aux_name, sizeof(i8042_pnp_aux_name), "%s%s%s",
+		  did->id, strlen(name) ? ":" : "", name);
 	i8042_pnp_id_to_string(dev->id, i8042_aux_firmware_id,
 			       sizeof(i8042_aux_firmware_id));
 
diff --git a/drivers/input/touchscreen/colibri-vf50-ts.c b/drivers/input/touchscreen/colibri-vf50-ts.c
index 98d5b2ba63fb9..89c4d7b2b89e8 100644
--- a/drivers/input/touchscreen/colibri-vf50-ts.c
+++ b/drivers/input/touchscreen/colibri-vf50-ts.c
@@ -372,3 +372,4 @@ module_platform_driver(vf50_touch_driver);
 MODULE_AUTHOR("Sanchayan Maity");
 MODULE_DESCRIPTION("Colibri VF50 Touchscreen driver");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/input/touchscreen/mms114.c b/drivers/input/touchscreen/mms114.c
index 5cef97246a158..27911a9f4e9eb 100644
--- a/drivers/input/touchscreen/mms114.c
+++ b/drivers/input/touchscreen/mms114.c
@@ -4,6 +4,8 @@
 // Copyright (c) 2012 Samsung Electronics Co., Ltd.
 // Author: Joonyoung Shim <jy0922.shim@samsung.com>
 
+#include <linux/bitfield.h>
+#include <linux/bits.h>
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/of.h>
@@ -52,21 +54,13 @@
 #define MMS114_TYPE_TOUCHSCREEN		1
 #define MMS114_TYPE_TOUCHKEY		2
 
-enum mms_type {
-	TYPE_MMS114	= 114,
-	TYPE_MMS134S	= 134,
-	TYPE_MMS136	= 136,
-	TYPE_MMS152	= 152,
-	TYPE_MMS345L	= 345,
-};
-
 struct mms114_data {
+	const struct mms_chip	*chip;
 	struct i2c_client	*client;
 	struct input_dev	*input_dev;
 	struct regulator	*core_reg;
 	struct regulator	*io_reg;
 	struct touchscreen_properties props;
-	enum mms_type		type;
 	unsigned int		contact_threshold;
 	unsigned int		moving_threshold;
 
@@ -77,9 +71,23 @@ struct mms114_data {
 	u8			cache_mode_control;
 };
 
+struct mms_chip {
+	const char *name;
+	int event_size;
+	bool has_config_regs;
+	int (*get_version)(struct mms114_data *data);
+};
+
+#define MMS114_FLAGS_ID_MASK		GENMASK(3, 0)
+#define MMS114_FLAGS_TYPE_MASK		GENMASK(6, 5)
+#define MMS114_FLAGS_PRESSED_MASK	BIT(7)
+
+#define MMS114_XY_HI_X_MASK		GENMASK(3, 0)
+#define MMS114_XY_HI_Y_MASK		GENMASK(7, 4)
+
 struct mms114_touch {
-	u8 id:4, reserved_bit4:1, type:2, pressed:1;
-	u8 x_hi:4, y_hi:4;
+	u8 flags;
+	u8 xy_hi;
 	u8 x_lo;
 	u8 y_lo;
 	u8 width;
@@ -87,16 +95,16 @@ struct mms114_touch {
 	u8 reserved[2];
 } __packed;
 
-static int __mms114_read_reg(struct mms114_data *data, unsigned int reg,
-			     unsigned int len, u8 *val)
+static int __mms114_read_reg(struct mms114_data *data, u8 reg,
+			     unsigned int len, void *val)
 {
 	struct i2c_client *client = data->client;
 	struct i2c_msg xfer[2];
-	u8 buf = reg & 0xff;
+	u8 buf = reg;
 	int error;
 
-	if (reg <= MMS114_MODE_CONTROL && reg + len > MMS114_MODE_CONTROL)
-		BUG();
+	if (WARN_ON(reg <= MMS114_MODE_CONTROL && reg + len > MMS114_MODE_CONTROL))
+		return -EINVAL;
 
 	/* Write register */
 	xfer[0].addr = client->addr;
@@ -116,12 +124,12 @@ static int __mms114_read_reg(struct mms114_data *data, unsigned int reg,
 			"%s: i2c transfer failed (%d)\n", __func__, error);
 		return error < 0 ? error : -EIO;
 	}
-	udelay(MMS114_I2C_DELAY);
+	usleep_range(MMS114_I2C_DELAY, MMS114_I2C_DELAY + 50);
 
 	return 0;
 }
 
-static int mms114_read_reg(struct mms114_data *data, unsigned int reg)
+static int mms114_read_reg(struct mms114_data *data, u8 reg)
 {
 	u8 val;
 	int error;
@@ -133,15 +141,14 @@ static int mms114_read_reg(struct mms114_data *data, unsigned int reg)
 	return error < 0 ? error : val;
 }
 
-static int mms114_write_reg(struct mms114_data *data, unsigned int reg,
-			    unsigned int val)
+static int mms114_write_reg(struct mms114_data *data, u8 reg, u8 val)
 {
 	struct i2c_client *client = data->client;
 	u8 buf[2];
 	int error;
 
-	buf[0] = reg & 0xff;
-	buf[1] = val & 0xff;
+	buf[0] = reg;
+	buf[1] = val;
 
 	error = i2c_master_send(client, buf, 2);
 	if (error != 2) {
@@ -149,7 +156,7 @@ static int mms114_write_reg(struct mms114_data *data, unsigned int reg,
 			"%s: i2c send failed (%d)\n", __func__, error);
 		return error < 0 ? error : -EIO;
 	}
-	udelay(MMS114_I2C_DELAY);
+	usleep_range(MMS114_I2C_DELAY, MMS114_I2C_DELAY + 50);
 
 	if (reg == MMS114_MODE_CONTROL)
 		data->cache_mode_control = val;
@@ -157,32 +164,119 @@ static int mms114_write_reg(struct mms114_data *data, unsigned int reg,
 	return 0;
 }
 
+static int mms114_get_version(struct mms114_data *data)
+{
+	struct device *dev = &data->client->dev;
+	u8 buf[6];
+	int error;
+
+	error = __mms114_read_reg(data, MMS114_TSP_REV, 6, buf);
+	if (error)
+		return error;
+
+	dev_info(dev, "TSP Rev: 0x%x, HW Rev: 0x%x, Firmware Ver: 0x%x\n",
+		 buf[0], buf[1], buf[3]);
+	return 0;
+}
+
+static int mms152_get_version(struct mms114_data *data)
+{
+	struct device *dev = &data->client->dev;
+	u8 buf[3];
+	int group;
+	int error;
+
+	error = __mms114_read_reg(data, MMS152_FW_REV, 3, buf);
+	if (error)
+		return error;
+
+	group = i2c_smbus_read_byte_data(data->client, MMS152_COMPAT_GROUP);
+	if (group < 0)
+		return group;
+
+	dev_info(dev, "TSP FW Rev: bootloader 0x%x / core 0x%x / config 0x%x, Compat group: %c\n",
+		 buf[0], buf[1], buf[2], group);
+	return 0;
+}
+
+static int mms345l_get_version(struct mms114_data *data)
+{
+	struct device *dev = &data->client->dev;
+	u8 buf[3];
+	int error;
+
+	error = __mms114_read_reg(data, MMS152_FW_REV, 3, buf);
+	if (error)
+		return error;
+
+	dev_info(dev, "TSP FW Rev: bootloader 0x%x / core 0x%x / config 0x%x\n",
+		 buf[0], buf[1], buf[2]);
+	return 0;
+}
+
+static const struct mms_chip mms114_descriptor = {
+	.name = "MMS114",
+	.event_size = MMS114_EVENT_SIZE,
+	.has_config_regs = true,
+	.get_version = mms114_get_version,
+};
+
+static const struct mms_chip mms134s_descriptor = {
+	.name = "MMS134S",
+	.event_size = MMS136_EVENT_SIZE,
+	.has_config_regs = true,
+	.get_version = mms114_get_version,
+};
+
+static const struct mms_chip mms136_descriptor = {
+	.name = "MMS136",
+	.event_size = MMS136_EVENT_SIZE,
+	.has_config_regs = true,
+	.get_version = mms114_get_version,
+};
+
+static const struct mms_chip mms152_descriptor = {
+	.name = "MMS152",
+	.event_size = MMS114_EVENT_SIZE,
+	.has_config_regs = false,
+	.get_version = mms152_get_version,
+};
+
+static const struct mms_chip mms345l_descriptor = {
+	.name = "MMS345L",
+	.event_size = MMS114_EVENT_SIZE,
+	.has_config_regs = false,
+	.get_version = mms345l_get_version,
+};
+
 static void mms114_process_mt(struct mms114_data *data, struct mms114_touch *touch)
 {
 	struct i2c_client *client = data->client;
 	struct input_dev *input_dev = data->input_dev;
-	unsigned int id;
+	unsigned int id = FIELD_GET(MMS114_FLAGS_ID_MASK, touch->flags);
+	unsigned int type = FIELD_GET(MMS114_FLAGS_TYPE_MASK, touch->flags);
+	bool pressed = FIELD_GET(MMS114_FLAGS_PRESSED_MASK, touch->flags);
 	unsigned int x;
 	unsigned int y;
 
-	if (touch->id == 0 || touch->id > MMS114_MAX_TOUCH) {
-		dev_err(&client->dev, "Wrong touch id (%d)\n", touch->id);
+	if (id == 0 || id > MMS114_MAX_TOUCH) {
+		dev_err(&client->dev, "Wrong touch id (%d)\n", id);
 		return;
 	}
 
-	id = touch->id - 1;
-	x = touch->x_lo | touch->x_hi << 8;
-	y = touch->y_lo | touch->y_hi << 8;
+	id--;
+	x = touch->x_lo | FIELD_GET(MMS114_XY_HI_X_MASK, touch->xy_hi) << 8;
+	y = touch->y_lo | FIELD_GET(MMS114_XY_HI_Y_MASK, touch->xy_hi) << 8;
 
 	dev_dbg(&client->dev,
 		"id: %d, type: %d, pressed: %d, x: %d, y: %d, width: %d, strength: %d\n",
-		id, touch->type, touch->pressed,
+		id, type, pressed,
 		x, y, touch->width, touch->strength);
 
 	input_mt_slot(input_dev, id);
-	input_mt_report_slot_state(input_dev, MT_TOOL_FINGER, touch->pressed);
+	input_mt_report_slot_state(input_dev, MT_TOOL_FINGER, pressed);
 
-	if (touch->pressed) {
+	if (pressed) {
 		touchscreen_report_pos(input_dev, &data->props, x, y, true);
 		input_report_abs(input_dev, ABS_MT_TOUCH_MAJOR, touch->width);
 		input_report_abs(input_dev, ABS_MT_PRESSURE, touch->strength);
@@ -195,21 +289,23 @@ static void mms114_process_touchkey(struct mms114_data *data,
 	struct i2c_client *client = data->client;
 	struct input_dev *input_dev = data->input_dev;
 	unsigned int keycode_id;
+	unsigned int id = FIELD_GET(MMS114_FLAGS_ID_MASK, touch->flags);
+	bool pressed = FIELD_GET(MMS114_FLAGS_PRESSED_MASK, touch->flags);
 
-	if (touch->id == 0)
+	if (id == 0)
 		return;
 
-	if (touch->id > data->num_keycodes) {
+	if (id > data->num_keycodes) {
 		dev_err(&client->dev, "Wrong touch id for touchkey (%d)\n",
-			touch->id);
+			id);
 		return;
 	}
 
-	keycode_id = touch->id - 1;
+	keycode_id = id - 1;
 	dev_dbg(&client->dev, "keycode id: %d, pressed: %d\n", keycode_id,
-		touch->pressed);
+		pressed);
 
-	input_report_key(input_dev, data->keycodes[keycode_id], touch->pressed);
+	input_report_key(input_dev, data->keycodes[keycode_id], pressed);
 }
 
 static irqreturn_t mms114_interrupt(int irq, void *dev_id)
@@ -218,8 +314,8 @@ static irqreturn_t mms114_interrupt(int irq, void *dev_id)
 	struct i2c_client *client = data->client;
 	struct mms114_touch touch[MMS114_MAX_TOUCH];
 	struct mms114_touch *t;
+	int event_size = data->chip->event_size;
 	int packet_size;
-	int event_size;
 	int touch_size;
 	int index;
 	int error;
@@ -234,23 +330,17 @@ static irqreturn_t mms114_interrupt(int irq, void *dev_id)
 		goto out;
 	}
 
-	/* MMS136 has slightly different event size */
-	if (data->type == TYPE_MMS134S || data->type == TYPE_MMS136)
-		event_size = MMS136_EVENT_SIZE;
-	else
-		event_size = MMS114_EVENT_SIZE;
-
 	touch_size = packet_size / event_size;
 
-	error = __mms114_read_reg(data, MMS114_INFORMATION, packet_size,
-			(u8 *)touch);
-	if (error < 0)
+	error = __mms114_read_reg(data, MMS114_INFORMATION, packet_size, touch);
+	if (error)
 		goto out;
 
 	for (index = 0; index < touch_size; index++) {
 		t = (struct mms114_touch *)((u8 *)touch + index * event_size);
+		unsigned int type = FIELD_GET(MMS114_FLAGS_TYPE_MASK, t->flags);
 
-		switch (t->type) {
+		switch (type) {
 		case MMS114_TYPE_TOUCHSCREEN:
 			mms114_process_mt(data, t);
 			break;
@@ -261,7 +351,7 @@ static irqreturn_t mms114_interrupt(int irq, void *dev_id)
 
 		default:
 			dev_err(&client->dev, "Wrong touch type (%d)\n",
-				t->type);
+				type);
 			break;
 		}
 	}
@@ -290,65 +380,17 @@ static int mms114_set_active(struct mms114_data *data, bool active)
 	return mms114_write_reg(data, MMS114_MODE_CONTROL, val);
 }
 
-static int mms114_get_version(struct mms114_data *data)
-{
-	struct device *dev = &data->client->dev;
-	u8 buf[6];
-	int group;
-	int error;
-
-	switch (data->type) {
-	case TYPE_MMS345L:
-		error = __mms114_read_reg(data, MMS152_FW_REV, 3, buf);
-		if (error)
-			return error;
-
-		dev_info(dev, "TSP FW Rev: bootloader 0x%x / core 0x%x / config 0x%x\n",
-			 buf[0], buf[1], buf[2]);
-		break;
-
-	case TYPE_MMS152:
-		error = __mms114_read_reg(data, MMS152_FW_REV, 3, buf);
-		if (error)
-			return error;
-
-		group = i2c_smbus_read_byte_data(data->client,
-						  MMS152_COMPAT_GROUP);
-		if (group < 0)
-			return group;
-
-		dev_info(dev, "TSP FW Rev: bootloader 0x%x / core 0x%x / config 0x%x, Compat group: %c\n",
-			 buf[0], buf[1], buf[2], group);
-		break;
-
-	case TYPE_MMS114:
-	case TYPE_MMS134S:
-	case TYPE_MMS136:
-		error = __mms114_read_reg(data, MMS114_TSP_REV, 6, buf);
-		if (error)
-			return error;
-
-		dev_info(dev, "TSP Rev: 0x%x, HW Rev: 0x%x, Firmware Ver: 0x%x\n",
-			 buf[0], buf[1], buf[3]);
-		break;
-	}
-
-	return 0;
-}
-
 static int mms114_setup_regs(struct mms114_data *data)
 {
 	const struct touchscreen_properties *props = &data->props;
 	int val;
 	int error;
 
-	error = mms114_get_version(data);
-	if (error < 0)
+	error = data->chip->get_version(data);
+	if (error)
 		return error;
 
-	/* MMS114, MMS134S and MMS136 have configuration and power on registers */
-	if (data->type != TYPE_MMS114 && data->type != TYPE_MMS134S &&
-	    data->type != TYPE_MMS136)
+	if (!data->chip->has_config_regs)
 		return 0;
 
 	error = mms114_set_active(data, true);
@@ -366,21 +408,21 @@ static int mms114_setup_regs(struct mms114_data *data)
 	if (error < 0)
 		return error;
 
-	val = props->max_x & 0xff;
+	val = props->max_y & 0xff;
 	error = mms114_write_reg(data, MMS114_Y_RESOLUTION, val);
 	if (error < 0)
 		return error;
 
 	if (data->contact_threshold) {
 		error = mms114_write_reg(data, MMS114_CONTACT_THRESHOLD,
-				data->contact_threshold);
+					 data->contact_threshold);
 		if (error < 0)
 			return error;
 	}
 
 	if (data->moving_threshold) {
 		error = mms114_write_reg(data, MMS114_MOVING_THRESHOLD,
-				data->moving_threshold);
+					 data->moving_threshold);
 		if (error < 0)
 			return error;
 	}
@@ -466,9 +508,9 @@ static int mms114_parse_legacy_bindings(struct mms114_data *data)
 	}
 
 	device_property_read_u32(dev, "contact-threshold",
-				&data->contact_threshold);
+				 &data->contact_threshold);
 	device_property_read_u32(dev, "moving-threshold",
-				&data->moving_threshold);
+				 &data->moving_threshold);
 
 	if (device_property_read_bool(dev, "x-invert"))
 		props->invert_x = true;
@@ -484,7 +526,6 @@ static int mms114_probe(struct i2c_client *client)
 {
 	struct mms114_data *data;
 	struct input_dev *input_dev;
-	const void *match_data;
 	int error;
 	int i;
 
@@ -504,12 +545,10 @@ static int mms114_probe(struct i2c_client *client)
 	data->client = client;
 	data->input_dev = input_dev;
 
-	match_data = device_get_match_data(&client->dev);
-	if (!match_data)
+	data->chip = i2c_get_match_data(client);
+	if (!data->chip)
 		return -EINVAL;
 
-	data->type = (enum mms_type)match_data;
-
 	data->num_keycodes = device_property_count_u32(&client->dev,
 						       "linux,keycodes");
 	if (data->num_keycodes == -EINVAL) {
@@ -521,7 +560,7 @@ static int mms114_probe(struct i2c_client *client)
 		return data->num_keycodes;
 	} else if (data->num_keycodes > MMS114_MAX_TOUCHKEYS) {
 		dev_warn(&client->dev,
-			"Found %d linux,keycodes but max is %d, ignoring the rest\n",
+			 "Found %d linux,keycodes but max is %d, ignoring the rest\n",
 			 data->num_keycodes, MMS114_MAX_TOUCHKEYS);
 		data->num_keycodes = MMS114_MAX_TOUCHKEYS;
 	}
@@ -566,8 +605,7 @@ static int mms114_probe(struct i2c_client *client)
 				     0, data->props.max_y, 0, 0);
 	}
 
-	if (data->type == TYPE_MMS114 || data->type == TYPE_MMS134S ||
-	    data->type == TYPE_MMS136) {
+	if (data->chip->has_config_regs) {
 		/*
 		 * The firmware handles movement and pressure fuzz, so
 		 * don't duplicate that in software.
@@ -582,8 +620,8 @@ static int mms114_probe(struct i2c_client *client)
 	}
 
 	input_dev->name = devm_kasprintf(&client->dev, GFP_KERNEL,
-					 "MELFAS MMS%d Touchscreen",
-					 data->type);
+					 "MELFAS %s Touchscreen",
+					 data->chip->name);
 	if (!input_dev->name)
 		return -ENOMEM;
 
@@ -679,29 +717,22 @@ static int mms114_resume(struct device *dev)
 static DEFINE_SIMPLE_DEV_PM_OPS(mms114_pm_ops, mms114_suspend, mms114_resume);
 
 static const struct i2c_device_id mms114_id[] = {
-	{ .name = "mms114" },
+	{ .name = "mms114", .driver_data = (kernel_ulong_t)&mms114_descriptor },
+	{ .name = "mms134s", .driver_data = (kernel_ulong_t)&mms134s_descriptor },
+	{ .name = "mms136", .driver_data = (kernel_ulong_t)&mms136_descriptor },
+	{ .name = "mms152", .driver_data = (kernel_ulong_t)&mms152_descriptor },
+	{ .name = "mms345l", .driver_data = (kernel_ulong_t)&mms345l_descriptor },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mms114_id);
 
 #ifdef CONFIG_OF
 static const struct of_device_id mms114_dt_match[] = {
-	{
-		.compatible = "melfas,mms114",
-		.data = (void *)TYPE_MMS114,
-	}, {
-		.compatible = "melfas,mms134s",
-		.data = (void *)TYPE_MMS134S,
-	}, {
-		.compatible = "melfas,mms136",
-		.data = (void *)TYPE_MMS136,
-	}, {
-		.compatible = "melfas,mms152",
-		.data = (void *)TYPE_MMS152,
-	}, {
-		.compatible = "melfas,mms345l",
-		.data = (void *)TYPE_MMS345L,
-	},
+	{ .compatible = "melfas,mms114", .data = &mms114_descriptor },
+	{ .compatible = "melfas,mms134s", .data = &mms134s_descriptor },
+	{ .compatible = "melfas,mms136", .data = &mms136_descriptor },
+	{ .compatible = "melfas,mms152", .data = &mms152_descriptor },
+	{ .compatible = "melfas,mms345l", .data = &mms345l_descriptor },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, mms114_dt_match);
@@ -722,4 +753,4 @@ module_i2c_driver(mms114_driver);
 /* Module information */
 MODULE_AUTHOR("Joonyoung Shim <jy0922.shim@samsung.com>");
 MODULE_DESCRIPTION("MELFAS mms114 Touchscreen driver");
-MODULE_LICENSE("GPL v2");
+MODULE_LICENSE("GPL");
diff --git a/drivers/input/touchscreen/resistive-adc-touch.c b/drivers/input/touchscreen/resistive-adc-touch.c
index 68f7e2e28f37a..c89edc1280fe7 100644
--- a/drivers/input/touchscreen/resistive-adc-touch.c
+++ b/drivers/input/touchscreen/resistive-adc-touch.c
@@ -300,3 +300,4 @@ module_platform_driver(grts_driver);
 MODULE_AUTHOR("Eugen Hristev <eugen.hristev@microchip.com>");
 MODULE_DESCRIPTION("Generic ADC Resistive Touch Driver");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/input/touchscreen/sur40.c b/drivers/input/touchscreen/sur40.c
index fe63d53d56db0..77ec2c94b91fd 100644
--- a/drivers/input/touchscreen/sur40.c
+++ b/drivers/input/touchscreen/sur40.c
@@ -128,8 +128,8 @@ struct sur40_image_header {
 /* polling interval (ms) */
 #define POLL_INTERVAL 1
 
-/* maximum number of contacts FIXME: this is a guess? */
-#define MAX_CONTACTS 64
+/* maximum number of contacts */
+#define MAX_CONTACTS 52
 
 /* control commands */
 #define SUR40_GET_VERSION 0xb0 /* 12 bytes string    */
diff --git a/drivers/input/touchscreen/wacom_w8001.c b/drivers/input/touchscreen/wacom_w8001.c
index 45930d7318733..d8d1cdc3f09e3 100644
--- a/drivers/input/touchscreen/wacom_w8001.c
+++ b/drivers/input/touchscreen/wacom_w8001.c
@@ -417,6 +417,13 @@ static int w8001_detect(struct w8001 *w8001)
 	return 0;
 }
 
+static void w8001_append_suffix(char *dest, const char *suffix, size_t dest_sz)
+{
+	size_t used = strnlen(dest, dest_sz);
+
+	strscpy(dest + used, suffix, dest_sz - used);
+}
+
 static int w8001_setup_pen(struct w8001 *w8001, char *basename,
 			   size_t basename_sz)
 {
@@ -453,7 +460,7 @@ static int w8001_setup_pen(struct w8001 *w8001, char *basename,
 	}
 
 	w8001->id = 0x90;
-	strlcat(basename, " Penabled", basename_sz);
+	w8001_append_suffix(basename, " Penabled", basename_sz);
 
 	return 0;
 }
@@ -503,14 +510,14 @@ static int w8001_setup_touch(struct w8001 *w8001, char *basename,
 	case 2:
 		w8001->pktlen = W8001_PKTLEN_TOUCH93;
 		w8001->id = 0x93;
-		strlcat(basename, " 1FG", basename_sz);
+		w8001_append_suffix(basename, " 1FG", basename_sz);
 		break;
 
 	case 1:
 	case 3:
 	case 4:
 		w8001->pktlen = W8001_PKTLEN_TOUCH9A;
-		strlcat(basename, " 1FG", basename_sz);
+		w8001_append_suffix(basename, " 1FG", basename_sz);
 		w8001->id = 0x9a;
 		break;
 
@@ -534,7 +541,7 @@ static int w8001_setup_touch(struct w8001 *w8001, char *basename,
 		input_abs_set_res(dev, ABS_MT_POSITION_X, touch.panel_res);
 		input_abs_set_res(dev, ABS_MT_POSITION_Y, touch.panel_res);
 
-		strlcat(basename, " 2FG", basename_sz);
+		w8001_append_suffix(basename, " 2FG", basename_sz);
 		if (w8001->max_pen_x && w8001->max_pen_y)
 			w8001->id = 0xE3;
 		else
@@ -542,7 +549,7 @@ static int w8001_setup_touch(struct w8001 *w8001, char *basename,
 		break;
 	}
 
-	strlcat(basename, " Touchscreen", basename_sz);
+	w8001_append_suffix(basename, " Touchscreen", basename_sz);
 
 	return 0;
 }
diff --git a/drivers/interconnect/qcom/Kconfig b/drivers/interconnect/qcom/Kconfig
index 5b8a094ca4ede..e4806bd7929cf 100644
--- a/drivers/interconnect/qcom/Kconfig
+++ b/drivers/interconnect/qcom/Kconfig
@@ -47,6 +47,16 @@ config INTERCONNECT_QCOM_KAANAPALI
 	  This is a driver for the Qualcomm Network-on-Chip on kaanapali-based
 	  platforms.
 
+config INTERCONNECT_QCOM_MAILI
+	tristate "Qualcomm MAILI interconnect driver"
+	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	select INTERCONNECT_QCOM_RPMH
+	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
+	help
+	  This is a driver for the Qualcomm Network-on-Chip on maili-based
+	  platforms.
+
 config INTERCONNECT_QCOM_MSM8909
 	tristate "Qualcomm MSM8909 interconnect driver"
 	depends on INTERCONNECT_QCOM
@@ -333,6 +343,7 @@ config INTERCONNECT_QCOM_SHIKRA
 	tristate "Qualcomm SHIKRA interconnect driver"
 	depends on INTERCONNECT_QCOM
 	depends on QCOM_SMD_RPM
+	default ARCH_QCOM
 	select INTERCONNECT_QCOM_SMD_RPM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on shikra-based
diff --git a/drivers/interconnect/qcom/Makefile b/drivers/interconnect/qcom/Makefile
index 1c7d410b40cc9..5f139a370bbea 100644
--- a/drivers/interconnect/qcom/Makefile
+++ b/drivers/interconnect/qcom/Makefile
@@ -8,6 +8,7 @@ qnoc-eliza-objs				:= eliza.o
 qnoc-glymur-objs			:= glymur.o
 qnoc-hawi-objs				:= hawi.o
 qnoc-kaanapali-objs			:= kaanapali.o
+qnoc-maili-objs				:= maili.o
 qnoc-milos-objs				:= milos.o
 qnoc-msm8909-objs			:= msm8909.o
 qnoc-msm8916-objs			:= msm8916.o
@@ -56,6 +57,7 @@ obj-$(CONFIG_INTERCONNECT_QCOM_ELIZA) += qnoc-eliza.o
 obj-$(CONFIG_INTERCONNECT_QCOM_GLYMUR) += qnoc-glymur.o
 obj-$(CONFIG_INTERCONNECT_QCOM_HAWI) += qnoc-hawi.o
 obj-$(CONFIG_INTERCONNECT_QCOM_KAANAPALI) += qnoc-kaanapali.o
+obj-$(CONFIG_INTERCONNECT_QCOM_MAILI) += qnoc-maili.o
 obj-$(CONFIG_INTERCONNECT_QCOM_MILOS) += qnoc-milos.o
 obj-$(CONFIG_INTERCONNECT_QCOM_MSM8909) += qnoc-msm8909.o
 obj-$(CONFIG_INTERCONNECT_QCOM_MSM8916) += qnoc-msm8916.o
diff --git a/drivers/interconnect/qcom/maili.c b/drivers/interconnect/qcom/maili.c
new file mode 100644
index 0000000000000..e5eed7f998f92
--- /dev/null
+++ b/drivers/interconnect/qcom/maili.c
@@ -0,0 +1,2091 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ *
+ */
+
+#include <linux/device.h>
+#include <linux/interconnect.h>
+#include <linux/interconnect-provider.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <dt-bindings/interconnect/qcom,maili-rpmh.h>
+
+#include "bcm-voter.h"
+#include "icc-rpmh.h"
+
+static struct qcom_icc_node ddr_eff_veto_slave = {
+	.name = "ddr_eff_veto_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qup0_core_slave = {
+	.name = "qup0_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qup1_core_slave = {
+	.name = "qup1_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qup2_core_slave = {
+	.name = "qup2_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qup3_core_slave = {
+	.name = "qup3_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qup4_core_slave = {
+	.name = "qup4_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qup5_core_slave = {
+	.name = "qup5_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_aoss = {
+	.name = "qhs_aoss",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ipa = {
+	.name = "qhs_ipa",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ipc_router_fence = {
+	.name = "qhs_ipc_router_fence",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_soccp = {
+	.name = "qhs_soccp",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_tme_cfg = {
+	.name = "qhs_tme_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qss_ddrss_cfg = {
+	.name = "qss_ddrss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qxs_imem = {
+	.name = "qxs_imem",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node xs_pcie = {
+	.name = "xs_pcie",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node xs_pcie_g4x1 = {
+	.name = "xs_pcie_g4x1",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node qhs_lpi_cc = {
+	.name = "qhs_lpi_cc",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qns_lb = {
+	.name = "qns_lb",
+	.channels = 4,
+	.buswidth = 16,
+};
+
+static struct qcom_icc_node srvc_llclpi_noc = {
+	.name = "srvc_llclpi_noc",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node srvc_llclpi_noc_chipcx = {
+	.name = "srvc_llclpi_noc_chipcx",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node ebi = {
+	.name = "ebi",
+	.channels = 4,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node ddr_rt_slave = {
+	.name = "ddr_rt_slave",
+	.channels = 4,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node srvc_pcie_aggre_noc = {
+	.name = "srvc_pcie_aggre_noc",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ahb2phy0 = {
+	.name = "qhs_ahb2phy0",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_boot_rom = {
+	.name = "qhs_boot_rom",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_camera_cfg = {
+	.name = "qhs_camera_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_clk_ctl = {
+	.name = "qhs_clk_ctl",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_crypto_cfg = {
+	.name = "qhs_crypto_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_display_cfg = {
+	.name = "qhs_display_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_eva_cfg = {
+	.name = "qhs_eva_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_gpuss_cfg = {
+	.name = "qhs_gpuss_cfg",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node qhs_i2c = {
+	.name = "qhs_i2c",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_imem_cfg = {
+	.name = "qhs_imem_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ipc_router = {
+	.name = "qhs_ipc_router",
+	.channels = 4,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_iris_cfg = {
+	.name = "qhs_iris_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_mss_cfg = {
+	.name = "qhs_mss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_pcie_cfg = {
+	.name = "qhs_pcie_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_pcie_g4x1_cfg = {
+	.name = "qhs_pcie_g4x1_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_prng = {
+	.name = "qhs_prng",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qspi = {
+	.name = "qhs_qspi",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qup1 = {
+	.name = "qhs_qup1",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qup2 = {
+	.name = "qhs_qup2",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qup3 = {
+	.name = "qhs_qup3",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qup4 = {
+	.name = "qhs_qup4",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qup5 = {
+	.name = "qhs_qup5",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_sdc2 = {
+	.name = "qhs_sdc2",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_sdc4 = {
+	.name = "qhs_sdc4",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_tlmm = {
+	.name = "qhs_tlmm",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ufs_mem_cfg = {
+	.name = "qhs_ufs_mem_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_usb3 = {
+	.name = "qhs_usb3",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_vsense_ctrl_cfg = {
+	.name = "qhs_vsense_ctrl_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qss_qdss_cfg = {
+	.name = "qss_qdss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qss_qdss_stm = {
+	.name = "qss_qdss_stm",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qss_tcsr = {
+	.name = "qss_tcsr",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node xs_sys_tcu_cfg = {
+	.name = "xs_sys_tcu_cfg",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node ddr_eff_veto_master = {
+	.name = "ddr_eff_veto_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &ddr_eff_veto_slave },
+};
+
+static struct qcom_icc_node qup0_core_master = {
+	.name = "qup0_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup0_core_slave },
+};
+
+static struct qcom_icc_node qup1_core_master = {
+	.name = "qup1_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup1_core_slave },
+};
+
+static struct qcom_icc_node qup2_core_master = {
+	.name = "qup2_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup2_core_slave },
+};
+
+static struct qcom_icc_node qup3_core_master = {
+	.name = "qup3_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup3_core_slave },
+};
+
+static struct qcom_icc_node qup4_core_master = {
+	.name = "qup4_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup4_core_slave },
+};
+
+static struct qcom_icc_node qup5_core_master = {
+	.name = "qup5_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup5_core_slave },
+};
+
+static struct qcom_icc_node qnm_gemnoc_pcie = {
+	.name = "qnm_gemnoc_pcie",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 2,
+	.link_nodes = { &xs_pcie, &xs_pcie_g4x1 },
+};
+
+static struct qcom_icc_node qnm_lpiaon_noc_llclpi_noc = {
+	.name = "qnm_lpiaon_noc_llclpi_noc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 4,
+	.link_nodes = { &qhs_lpi_cc, &qns_lb,
+			&srvc_llclpi_noc, &srvc_llclpi_noc_chipcx },
+};
+
+static struct qcom_icc_node llcc_mc = {
+	.name = "llcc_mc",
+	.channels = 4,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &ebi },
+};
+
+static struct qcom_icc_node ddr_rt_mc = {
+	.name = "ddr_rt_mc",
+	.channels = 4,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &ddr_rt_slave },
+};
+
+static struct qcom_icc_node qsm_pcie_anoc_cfg = {
+	.name = "qsm_pcie_anoc_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &srvc_pcie_aggre_noc },
+};
+
+static struct qcom_icc_node qsm_cfg_center = {
+	.name = "qsm_cfg_center",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 14,
+	.link_nodes = { &qhs_boot_rom, &qhs_clk_ctl,
+			&qhs_crypto_cfg, &qhs_gpuss_cfg,
+			&qhs_imem_cfg, &qhs_ipc_router,
+			&qhs_mss_cfg, &qhs_prng,
+			&qhs_tlmm, &qhs_vsense_ctrl_cfg,
+			&qss_qdss_cfg, &qss_qdss_stm,
+			&qss_tcsr, &xs_sys_tcu_cfg },
+};
+
+static struct qcom_icc_node qsm_cfg_east = {
+	.name = "qsm_cfg_east",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 2,
+	.link_nodes = { &qhs_qup2, &qhs_qup4 },
+};
+
+static struct qcom_icc_node qsm_cfg_mm_hf = {
+	.name = "qsm_cfg_mm_hf",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qhs_display_cfg },
+};
+
+static struct qcom_icc_node qsm_cfg_mm_sf = {
+	.name = "qsm_cfg_mm_sf",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 3,
+	.link_nodes = { &qhs_camera_cfg, &qhs_eva_cfg,
+			&qhs_iris_cfg },
+};
+
+static struct qcom_icc_node qsm_cfg_north = {
+	.name = "qsm_cfg_north",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 3,
+	.link_nodes = { &qhs_ahb2phy0, &qhs_ufs_mem_cfg,
+			&qhs_usb3 },
+};
+
+static struct qcom_icc_node qsm_cfg_west = {
+	.name = "qsm_cfg_west",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 2,
+	.link_nodes = { &qhs_qup1, &qhs_sdc2 },
+};
+
+static struct qcom_icc_node qns_llcc = {
+	.name = "qns_llcc",
+	.channels = 4,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &llcc_mc },
+};
+
+static struct qcom_icc_node qns_pcie = {
+	.name = "qns_pcie",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &qnm_gemnoc_pcie },
+};
+
+static struct qcom_icc_node qns_llc_lpinoc = {
+	.name = "qns_llc_lpinoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_lpiaon_noc_llclpi_noc },
+};
+
+static struct qcom_icc_node qss_pcie_anoc_cfg = {
+	.name = "qss_pcie_anoc_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_pcie_anoc_cfg },
+};
+
+static struct qcom_icc_node qss_stdst_center_cfg = {
+	.name = "qss_stdst_center_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_cfg_center },
+};
+
+static struct qcom_icc_node qss_stdst_east_cfg = {
+	.name = "qss_stdst_east_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_cfg_east },
+};
+
+static struct qcom_icc_node qss_stdst_mm_hf_cfg = {
+	.name = "qss_stdst_mm_hf_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_cfg_mm_hf },
+};
+
+static struct qcom_icc_node qss_stdst_mm_sf_cfg = {
+	.name = "qss_stdst_mm_sf_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_cfg_mm_sf },
+};
+
+static struct qcom_icc_node qss_stdst_north_cfg = {
+	.name = "qss_stdst_north_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_cfg_north },
+};
+
+static struct qcom_icc_node qss_stdst_west_cfg = {
+	.name = "qss_stdst_west_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_cfg_west },
+};
+
+static struct qcom_icc_node alm_gic = {
+	.name = "alm_gic",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x14d000 },
+		.prio = 4,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_llcc },
+};
+
+static struct qcom_icc_node qnm_qpace = {
+	.name = "qnm_qpace",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x153000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_llcc },
+};
+
+static struct qcom_icc_node qsm_cfg_south = {
+	.name = "qsm_cfg_south",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 8,
+	.link_nodes = { &qhs_i2c, &qhs_pcie_cfg,
+			&qhs_pcie_g4x1_cfg, &qhs_qspi,
+			&qhs_qup3, &qhs_qup5,
+			&qhs_sdc4, &qss_pcie_anoc_cfg },
+};
+
+static struct qcom_icc_node qss_stdst_south_cfg = {
+	.name = "qss_stdst_south_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_cfg_south },
+};
+
+static struct qcom_icc_node qsm_cnoc_main = {
+	.name = "qsm_cnoc_main",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 7,
+	.link_nodes = { &qss_stdst_center_cfg, &qss_stdst_east_cfg,
+			&qss_stdst_mm_hf_cfg, &qss_stdst_mm_sf_cfg,
+			&qss_stdst_north_cfg, &qss_stdst_south_cfg,
+			&qss_stdst_west_cfg },
+};
+
+static struct qcom_icc_node qss_cfg = {
+	.name = "qss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_cnoc_main },
+};
+
+static struct qcom_icc_node qnm_gemnoc_cnoc = {
+	.name = "qnm_gemnoc_cnoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 8,
+	.link_nodes = { &qhs_aoss, &qhs_ipa,
+			&qhs_ipc_router_fence, &qhs_soccp,
+			&qhs_tme_cfg, &qss_cfg,
+			&qss_ddrss_cfg, &qxs_imem },
+};
+
+static struct qcom_icc_node qns_gem_noc_cnoc = {
+	.name = "qns_gem_noc_cnoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_gemnoc_cnoc },
+};
+
+static struct qcom_icc_node alm_gpu_tcu = {
+	.name = "alm_gpu_tcu",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x145000 },
+		.prio = 1,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 2,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_node alm_sys_tcu = {
+	.name = "alm_sys_tcu",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x147000 },
+		.prio = 6,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 2,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_node chm_apps = {
+	.name = "chm_apps",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_gpu = {
+	.name = "qnm_gpu",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x51000, 0xd1000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_lpass_gemnoc = {
+	.name = "qnm_lpass_gemnoc",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x149000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_mdsp = {
+	.name = "qnm_mdsp",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_mnoc_hf = {
+	.name = "qnm_mnoc_hf",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x55000, 0xd5000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_mnoc_sf = {
+	.name = "qnm_mnoc_sf",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x57000, 0xd7000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_nsp_gemnoc = {
+	.name = "qnm_nsp_gemnoc",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x59000, 0xd9000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_pcie = {
+	.name = "qnm_pcie",
+	.channels = 1,
+	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x14b000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 2,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_node qnm_snoc_sf = {
+	.name = "qnm_snoc_sf",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x14f000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_wlan_q6 = {
+	.name = "qnm_wlan_q6",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qns_lpass_ag_noc_gemnoc = {
+	.name = "qns_lpass_ag_noc_gemnoc",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_lpass_gemnoc },
+};
+
+static struct qcom_icc_node qns_mem_noc_hf = {
+	.name = "qns_mem_noc_hf",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_mnoc_hf },
+};
+
+static struct qcom_icc_node qns_mem_noc_sf = {
+	.name = "qns_mem_noc_sf",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_mnoc_sf },
+};
+
+static struct qcom_icc_node qns_nsp_gemnoc = {
+	.name = "qns_nsp_gemnoc",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_nsp_gemnoc },
+};
+
+static struct qcom_icc_node qns_pcie_gemnoc = {
+	.name = "qns_pcie_gemnoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_pcie },
+};
+
+static struct qcom_icc_node qns_gemnoc_sf = {
+	.name = "qns_gemnoc_sf",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_snoc_sf },
+};
+
+static struct qcom_icc_node qnm_lpiaon_noc = {
+	.name = "qnm_lpiaon_noc",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qns_lpass_ag_noc_gemnoc },
+};
+
+static struct qcom_icc_node qnm_camnoc_hf = {
+	.name = "qnm_camnoc_hf",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x50000, 0x51000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_hf },
+};
+
+static struct qcom_icc_node qnm_mdp = {
+	.name = "qnm_mdp",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x52000, 0x53000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_hf },
+};
+
+static struct qcom_icc_node qnm_camnoc_nrt_icp_sf = {
+	.name = "qnm_camnoc_nrt_icp_sf",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x20000 },
+		.prio = 4,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_camnoc_rt_cdm_sf = {
+	.name = "qnm_camnoc_rt_cdm_sf",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x28000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_camnoc_sf = {
+	.name = "qnm_camnoc_sf",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x21000, 0x22000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_mdss_dcp = {
+	.name = "qnm_mdss_dcp",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x29000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_vapss_hcp = {
+	.name = "qnm_vapss_hcp",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_video_cv_cpu = {
+	.name = "qnm_video_cv_cpu",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x24000 },
+		.prio = 4,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_video_eva = {
+	.name = "qnm_video_eva",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x25000, 0x26000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_video_mvp = {
+	.name = "qnm_video_mvp",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x23000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_video_v_cpu = {
+	.name = "qnm_video_v_cpu",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x27000 },
+		.prio = 4,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_nsp = {
+	.name = "qnm_nsp",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qns_nsp_gemnoc },
+};
+
+static struct qcom_icc_node xm_pcie = {
+	.name = "xm_pcie",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xc000 },
+		.prio = 3,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_pcie_gemnoc },
+};
+
+static struct qcom_icc_node xm_pcie_g4x1 = {
+	.name = "xm_pcie_g4x1",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xd000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_pcie_gemnoc },
+};
+
+static struct qcom_icc_node qnm_aggre_noc = {
+	.name = "qnm_aggre_noc",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x20000 },
+		.prio = 2,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_gemnoc_sf },
+};
+
+static struct qcom_icc_node qnm_apss_noc = {
+	.name = "qnm_apss_noc",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1e000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_gemnoc_sf },
+};
+
+static struct qcom_icc_node qnm_cnoc_data = {
+	.name = "qnm_cnoc_data",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1f000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_gemnoc_sf },
+};
+
+static struct qcom_icc_node qns_a1noc_snoc = {
+	.name = "qns_a1noc_snoc",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_aggre_noc },
+};
+
+static struct qcom_icc_node qns_lpass_aggnoc = {
+	.name = "qns_lpass_aggnoc",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_lpiaon_noc },
+};
+
+static struct qcom_icc_node qhm_qspi = {
+	.name = "qhm_qspi",
+	.channels = 1,
+	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x49000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qhm_qup2 = {
+	.name = "qhm_qup2",
+	.channels = 1,
+	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x48000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qhm_qup3 = {
+	.name = "qhm_qup3",
+	.channels = 1,
+	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x46000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qhm_qup4 = {
+	.name = "qhm_qup4",
+	.channels = 1,
+	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x47000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qhm_qup5 = {
+	.name = "qhm_qup5",
+	.channels = 1,
+	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x4e000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qxm_crypto = {
+	.name = "qxm_crypto",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x40000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qxm_ipa = {
+	.name = "qxm_ipa",
+	.channels = 1,
+	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x41000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qxm_qup1 = {
+	.name = "qxm_qup1",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x4d000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qxm_soccp = {
+	.name = "qxm_soccp",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x45000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node xm_qdss_etr_0 = {
+	.name = "xm_qdss_etr_0",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x42000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node xm_qdss_etr_1 = {
+	.name = "xm_qdss_etr_1",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x43000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node xm_sdc2 = {
+	.name = "xm_sdc2",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x44000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node xm_sdc4 = {
+	.name = "xm_sdc4",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node xm_ufs_mem = {
+	.name = "xm_ufs_mem",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x4b000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node xm_usb3 = {
+	.name = "xm_usb3",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x4c000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qnm_lpass_lpinoc = {
+	.name = "qnm_lpass_lpinoc",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 2,
+	.link_nodes = { &qns_llc_lpinoc, &qns_lpass_aggnoc },
+};
+
+static struct qcom_icc_node qns_lpi_aon_noc = {
+	.name = "qns_lpi_aon_noc",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_lpass_lpinoc },
+};
+
+static struct qcom_icc_node qnm_lpinoc_dsp_qns4m = {
+	.name = "qnm_lpinoc_dsp_qns4m",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qns_lpi_aon_noc },
+};
+
+static struct qcom_icc_bcm bcm_acv = {
+	.name = "ACV",
+	.enable_mask = BIT(3),
+	.num_nodes = 1,
+	.nodes = { &ebi },
+};
+
+static struct qcom_icc_bcm bcm_ce0 = {
+	.name = "CE0",
+	.num_nodes = 1,
+	.nodes = { &qxm_crypto },
+};
+
+static struct qcom_icc_bcm bcm_cn0 = {
+	.name = "CN0",
+	.enable_mask = BIT(0),
+	.keepalive = true,
+	.num_nodes = 25,
+	.nodes = { &qnm_gemnoc_cnoc, &qnm_gemnoc_pcie,
+		   &qhs_aoss, &qhs_ipa,
+		   &qhs_ipc_router_fence, &qhs_soccp,
+		   &qhs_tme_cfg, &qss_cfg,
+		   &qss_ddrss_cfg, &qxs_imem,
+		   &xs_pcie, &xs_pcie_g4x1,
+		   &qsm_cfg_center, &qsm_cfg_east,
+		   &qsm_cfg_mm_hf, &qsm_cfg_mm_sf,
+		   &qsm_cfg_north, &qsm_cfg_south,
+		   &qsm_cfg_west, &qhs_ahb2phy0,
+		   &qhs_boot_rom, &qhs_camera_cfg,
+		   &qhs_clk_ctl, &qhs_crypto_cfg,
+		   &qhs_eva_cfg },
+};
+
+static struct qcom_icc_bcm bcm_cn1 = {
+	.name = "CN1",
+	.num_nodes = 6,
+	.nodes = { &qhs_display_cfg, &qhs_qup1,
+		   &qhs_qup2, &qhs_qup3,
+		   &qhs_qup4, &qhs_qup5 },
+};
+
+static struct qcom_icc_bcm bcm_co0 = {
+	.name = "CO0",
+	.enable_mask = BIT(0),
+	.num_nodes = 2,
+	.nodes = { &qnm_nsp, &qns_nsp_gemnoc },
+};
+
+static struct qcom_icc_bcm bcm_de0 = {
+	.name = "DE0",
+	.enable_mask = BIT(0),
+	.num_nodes = 1,
+	.nodes = { &ddr_eff_veto_slave },
+};
+
+static struct qcom_icc_bcm bcm_lp0 = {
+	.name = "LP0",
+	.num_nodes = 5,
+	.nodes = { &qnm_lpiaon_noc_llclpi_noc, &qns_lb,
+		   &qnm_lpass_lpinoc, &qns_llc_lpinoc,
+		   &qns_lpass_aggnoc },
+};
+
+static struct qcom_icc_bcm bcm_mc0 = {
+	.name = "MC0",
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &ebi },
+};
+
+static struct qcom_icc_bcm bcm_mc5 = {
+	.name = "MC5",
+	.num_nodes = 1,
+	.nodes = { &ddr_rt_slave },
+};
+
+static struct qcom_icc_bcm bcm_mm0 = {
+	.name = "MM0",
+	.num_nodes = 1,
+	.nodes = { &qns_mem_noc_hf },
+};
+
+static struct qcom_icc_bcm bcm_mm1 = {
+	.name = "MM1",
+	.enable_mask = BIT(0),
+	.num_nodes = 9,
+	.nodes = { &qnm_camnoc_hf, &qnm_camnoc_nrt_icp_sf,
+		   &qnm_camnoc_rt_cdm_sf, &qnm_camnoc_sf,
+		   &qnm_vapss_hcp, &qnm_video_cv_cpu,
+		   &qnm_video_mvp, &qnm_video_v_cpu,
+		   &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_bcm bcm_qpc0 = {
+	.name = "QPC0",
+	.num_nodes = 1,
+	.nodes = { &qnm_qpace },
+};
+
+static struct qcom_icc_bcm bcm_qup0 = {
+	.name = "QUP0",
+	.vote_scale = 1,
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qup0_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_qup1 = {
+	.name = "QUP1",
+	.vote_scale = 1,
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qup1_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_qup2 = {
+	.name = "QUP2",
+	.vote_scale = 1,
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qup2_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_qup3 = {
+	.name = "QUP3",
+	.vote_scale = 1,
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qup3_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_qup4 = {
+	.name = "QUP4",
+	.vote_scale = 1,
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qup4_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_qup5 = {
+	.name = "QUP5",
+	.vote_scale = 1,
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qup5_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_sh0 = {
+	.name = "SH0",
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qns_llcc },
+};
+
+static struct qcom_icc_bcm bcm_sh1 = {
+	.name = "SH1",
+	.enable_mask = BIT(0),
+	.num_nodes = 15,
+	.nodes = { &alm_gic, &alm_gpu_tcu,
+		   &alm_sys_tcu, &chm_apps,
+		   &qnm_gpu, &qnm_lpass_gemnoc,
+		   &qnm_mdsp, &qnm_mnoc_hf,
+		   &qnm_mnoc_sf, &qnm_nsp_gemnoc,
+		   &qnm_pcie, &qnm_snoc_sf,
+		   &qnm_wlan_q6, &qns_gem_noc_cnoc,
+		   &qns_pcie },
+};
+
+static struct qcom_icc_bcm bcm_sn0 = {
+	.name = "SN0",
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qns_gemnoc_sf },
+};
+
+static struct qcom_icc_bcm bcm_sn2 = {
+	.name = "SN2",
+	.num_nodes = 1,
+	.nodes = { &qnm_aggre_noc },
+};
+
+static struct qcom_icc_bcm bcm_sn3 = {
+	.name = "SN3",
+	.num_nodes = 1,
+	.nodes = { &qns_pcie_gemnoc },
+};
+
+static struct qcom_icc_bcm * const aggre1_noc_bcms[] = {
+	&bcm_ce0,
+};
+
+static struct qcom_icc_node * const aggre1_noc_nodes[] = {
+	[MASTER_QSPI_0] = &qhm_qspi,
+	[MASTER_QUP_2] = &qhm_qup2,
+	[MASTER_QUP_3] = &qhm_qup3,
+	[MASTER_QUP_4] = &qhm_qup4,
+	[MASTER_QUP_5] = &qhm_qup5,
+	[MASTER_CRYPTO] = &qxm_crypto,
+	[MASTER_IPA] = &qxm_ipa,
+	[MASTER_QUP_1] = &qxm_qup1,
+	[MASTER_SOCCP_PROC] = &qxm_soccp,
+	[MASTER_QDSS_ETR] = &xm_qdss_etr_0,
+	[MASTER_QDSS_ETR_1] = &xm_qdss_etr_1,
+	[MASTER_SDCC_2] = &xm_sdc2,
+	[MASTER_SDCC_4] = &xm_sdc4,
+	[MASTER_UFS_MEM] = &xm_ufs_mem,
+	[MASTER_USB3] = &xm_usb3,
+	[SLAVE_A1NOC_SNOC] = &qns_a1noc_snoc,
+};
+
+static const struct regmap_config maili_aggre1_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x56200,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc maili_aggre1_noc = {
+	.config = &maili_aggre1_noc_regmap_config,
+	.nodes = aggre1_noc_nodes,
+	.num_nodes = ARRAY_SIZE(aggre1_noc_nodes),
+	.bcms = aggre1_noc_bcms,
+	.num_bcms = ARRAY_SIZE(aggre1_noc_bcms),
+	.qos_requires_clocks = true,
+};
+
+static struct qcom_icc_bcm * const clk_virt_bcms[] = {
+	&bcm_de0,
+	&bcm_qup0,
+	&bcm_qup1,
+	&bcm_qup2,
+	&bcm_qup3,
+	&bcm_qup4,
+	&bcm_qup5,
+};
+
+static struct qcom_icc_node * const clk_virt_nodes[] = {
+	[MASTER_DDR_EFF_VETO] = &ddr_eff_veto_master,
+	[MASTER_QUP_CORE_0] = &qup0_core_master,
+	[MASTER_QUP_CORE_1] = &qup1_core_master,
+	[MASTER_QUP_CORE_2] = &qup2_core_master,
+	[MASTER_QUP_CORE_3] = &qup3_core_master,
+	[MASTER_QUP_CORE_4] = &qup4_core_master,
+	[MASTER_QUP_CORE_5] = &qup5_core_master,
+	[SLAVE_DDR_EFF_VETO] = &ddr_eff_veto_slave,
+	[SLAVE_QUP_CORE_0] = &qup0_core_slave,
+	[SLAVE_QUP_CORE_1] = &qup1_core_slave,
+	[SLAVE_QUP_CORE_2] = &qup2_core_slave,
+	[SLAVE_QUP_CORE_3] = &qup3_core_slave,
+	[SLAVE_QUP_CORE_4] = &qup4_core_slave,
+	[SLAVE_QUP_CORE_5] = &qup5_core_slave,
+};
+
+static const struct qcom_icc_desc maili_clk_virt = {
+	.nodes = clk_virt_nodes,
+	.num_nodes = ARRAY_SIZE(clk_virt_nodes),
+	.bcms = clk_virt_bcms,
+	.num_bcms = ARRAY_SIZE(clk_virt_bcms),
+};
+
+static struct qcom_icc_bcm * const cnoc_main_bcms[] = {
+	&bcm_cn0,
+};
+
+static struct qcom_icc_node * const cnoc_main_nodes[] = {
+	[MASTER_GEM_NOC_CNOC] = &qnm_gemnoc_cnoc,
+	[MASTER_GEM_NOC_PCIE_SNOC] = &qnm_gemnoc_pcie,
+	[SLAVE_AOSS] = &qhs_aoss,
+	[SLAVE_IPA_CFG] = &qhs_ipa,
+	[SLAVE_IPC_ROUTER_FENCE] = &qhs_ipc_router_fence,
+	[SLAVE_SOCCP] = &qhs_soccp,
+	[SLAVE_TME_CFG] = &qhs_tme_cfg,
+	[SLAVE_CNOC_CFG] = &qss_cfg,
+	[SLAVE_DDRSS_CFG] = &qss_ddrss_cfg,
+	[SLAVE_IMEM] = &qxs_imem,
+	[SLAVE_PCIE_0] = &xs_pcie,
+	[SLAVE_PCIE_1] = &xs_pcie_g4x1,
+};
+
+static const struct regmap_config maili_cnoc_main_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x20000,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc maili_cnoc_main = {
+	.config = &maili_cnoc_main_regmap_config,
+	.nodes = cnoc_main_nodes,
+	.num_nodes = ARRAY_SIZE(cnoc_main_nodes),
+	.bcms = cnoc_main_bcms,
+	.num_bcms = ARRAY_SIZE(cnoc_main_bcms),
+};
+
+static struct qcom_icc_bcm * const gem_noc_bcms[] = {
+	&bcm_qpc0,
+	&bcm_sh0,
+	&bcm_sh1,
+};
+
+static struct qcom_icc_node * const gem_noc_nodes[] = {
+	[MASTER_GIC] = &alm_gic,
+	[MASTER_GPU_TCU] = &alm_gpu_tcu,
+	[MASTER_SYS_TCU] = &alm_sys_tcu,
+	[MASTER_APPSS_PROC] = &chm_apps,
+	[MASTER_GFX3D] = &qnm_gpu,
+	[MASTER_LPASS_GEM_NOC] = &qnm_lpass_gemnoc,
+	[MASTER_MSS_PROC] = &qnm_mdsp,
+	[MASTER_MNOC_HF_MEM_NOC] = &qnm_mnoc_hf,
+	[MASTER_MNOC_SF_MEM_NOC] = &qnm_mnoc_sf,
+	[MASTER_COMPUTE_NOC] = &qnm_nsp_gemnoc,
+	[MASTER_ANOC_PCIE_GEM_NOC] = &qnm_pcie,
+	[MASTER_QPACE] = &qnm_qpace,
+	[MASTER_SNOC_SF_MEM_NOC] = &qnm_snoc_sf,
+	[MASTER_WLAN_Q6] = &qnm_wlan_q6,
+	[SLAVE_GEM_NOC_CNOC] = &qns_gem_noc_cnoc,
+	[SLAVE_LLCC] = &qns_llcc,
+	[SLAVE_MEM_NOC_PCIE_SNOC] = &qns_pcie,
+};
+
+static const struct regmap_config maili_gem_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x160200,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc maili_gem_noc = {
+	.config = &maili_gem_noc_regmap_config,
+	.nodes = gem_noc_nodes,
+	.num_nodes = ARRAY_SIZE(gem_noc_nodes),
+	.bcms = gem_noc_bcms,
+	.num_bcms = ARRAY_SIZE(gem_noc_bcms),
+};
+
+static struct qcom_icc_bcm * const llclpi_noc_bcms[] = {
+	&bcm_lp0,
+};
+
+static struct qcom_icc_node * const llclpi_noc_nodes[] = {
+	[MASTER_LPIAON_NOC_LLCLPI_NOC] = &qnm_lpiaon_noc_llclpi_noc,
+	[SLAVE_LPASS_LPI_CC] = &qhs_lpi_cc,
+	[SLAVE_LLCC_ISLAND] = &qns_lb,
+	[SLAVE_SERVICE_LLCLPI_NOC] = &srvc_llclpi_noc,
+	[SLAVE_SERVICE_LLCLPI_NOC_CHIPCX] = &srvc_llclpi_noc_chipcx,
+};
+
+static const struct regmap_config maili_llclpi_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x17200,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc maili_llclpi_noc = {
+	.config = &maili_llclpi_noc_regmap_config,
+	.nodes = llclpi_noc_nodes,
+	.num_nodes = ARRAY_SIZE(llclpi_noc_nodes),
+	.bcms = llclpi_noc_bcms,
+	.num_bcms = ARRAY_SIZE(llclpi_noc_bcms),
+};
+
+static struct qcom_icc_node * const lpass_ag_noc_nodes[] = {
+	[MASTER_LPIAON_NOC] = &qnm_lpiaon_noc,
+	[SLAVE_LPASS_GEM_NOC] = &qns_lpass_ag_noc_gemnoc,
+};
+
+static const struct regmap_config maili_lpass_ag_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xc080,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc maili_lpass_ag_noc = {
+	.config = &maili_lpass_ag_noc_regmap_config,
+	.nodes = lpass_ag_noc_nodes,
+	.num_nodes = ARRAY_SIZE(lpass_ag_noc_nodes),
+};
+
+static struct qcom_icc_bcm * const lpass_lpiaon_noc_bcms[] = {
+	&bcm_lp0,
+};
+
+static struct qcom_icc_node * const lpass_lpiaon_noc_nodes[] = {
+	[MASTER_LPASS_LPINOC] = &qnm_lpass_lpinoc,
+	[SLAVE_LPIAON_NOC_LLCLPI_NOC] = &qns_llc_lpinoc,
+	[SLAVE_LPIAON_NOC_LPASS_AG_NOC] = &qns_lpass_aggnoc,
+};
+
+static const struct regmap_config maili_lpass_lpiaon_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x19080,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc maili_lpass_lpiaon_noc = {
+	.config = &maili_lpass_lpiaon_noc_regmap_config,
+	.nodes = lpass_lpiaon_noc_nodes,
+	.num_nodes = ARRAY_SIZE(lpass_lpiaon_noc_nodes),
+	.bcms = lpass_lpiaon_noc_bcms,
+	.num_bcms = ARRAY_SIZE(lpass_lpiaon_noc_bcms),
+};
+
+static struct qcom_icc_node * const lpass_lpicx_noc_nodes[] = {
+	[MASTER_LPASS_PROC] = &qnm_lpinoc_dsp_qns4m,
+	[SLAVE_LPICX_NOC_LPIAON_NOC] = &qns_lpi_aon_noc,
+};
+
+static const struct regmap_config maili_lpass_lpicx_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x46080,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc maili_lpass_lpicx_noc = {
+	.config = &maili_lpass_lpicx_noc_regmap_config,
+	.nodes = lpass_lpicx_noc_nodes,
+	.num_nodes = ARRAY_SIZE(lpass_lpicx_noc_nodes),
+};
+
+static struct qcom_icc_bcm * const mc_virt_bcms[] = {
+	&bcm_acv,
+	&bcm_mc0,
+	&bcm_mc5,
+};
+
+static struct qcom_icc_node * const mc_virt_nodes[] = {
+	[MASTER_LLCC] = &llcc_mc,
+	[MASTER_DDR_RT] = &ddr_rt_mc,
+	[SLAVE_EBI1] = &ebi,
+	[SLAVE_DDR_RT] = &ddr_rt_slave,
+};
+
+static const struct qcom_icc_desc maili_mc_virt = {
+	.nodes = mc_virt_nodes,
+	.num_nodes = ARRAY_SIZE(mc_virt_nodes),
+	.bcms = mc_virt_bcms,
+	.num_bcms = ARRAY_SIZE(mc_virt_bcms),
+};
+
+static struct qcom_icc_bcm * const mmss_noc_bcms[] = {
+	&bcm_mm0,
+	&bcm_mm1,
+};
+
+static struct qcom_icc_node * const mmss_noc_nodes[] = {
+	[MASTER_CAMNOC_HF] = &qnm_camnoc_hf,
+	[MASTER_CAMNOC_NRT_ICP_SF] = &qnm_camnoc_nrt_icp_sf,
+	[MASTER_CAMNOC_RT_CDM_SF] = &qnm_camnoc_rt_cdm_sf,
+	[MASTER_CAMNOC_SF] = &qnm_camnoc_sf,
+	[MASTER_MDP] = &qnm_mdp,
+	[MASTER_MDSS_DCP] = &qnm_mdss_dcp,
+	[MASTER_CDSP_HCP] = &qnm_vapss_hcp,
+	[MASTER_VIDEO_CV_PROC] = &qnm_video_cv_cpu,
+	[MASTER_VIDEO_EVA] = &qnm_video_eva,
+	[MASTER_VIDEO_MVP] = &qnm_video_mvp,
+	[MASTER_VIDEO_V_PROC] = &qnm_video_v_cpu,
+	[SLAVE_MNOC_HF_MEM_NOC] = &qns_mem_noc_hf,
+	[SLAVE_MNOC_SF_MEM_NOC] = &qns_mem_noc_sf,
+};
+
+static const struct regmap_config maili_mmss_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x5f800,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc maili_mmss_noc = {
+	.config = &maili_mmss_noc_regmap_config,
+	.nodes = mmss_noc_nodes,
+	.num_nodes = ARRAY_SIZE(mmss_noc_nodes),
+	.bcms = mmss_noc_bcms,
+	.num_bcms = ARRAY_SIZE(mmss_noc_bcms),
+};
+
+static struct qcom_icc_bcm * const nsp_noc_bcms[] = {
+	&bcm_co0,
+};
+
+static struct qcom_icc_node * const nsp_noc_nodes[] = {
+	[MASTER_CDSP_PROC] = &qnm_nsp,
+	[SLAVE_CDSP_MEM_NOC] = &qns_nsp_gemnoc,
+};
+
+static const struct regmap_config maili_nsp_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x21280,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc maili_nsp_noc = {
+	.config = &maili_nsp_noc_regmap_config,
+	.nodes = nsp_noc_nodes,
+	.num_nodes = ARRAY_SIZE(nsp_noc_nodes),
+	.bcms = nsp_noc_bcms,
+	.num_bcms = ARRAY_SIZE(nsp_noc_bcms),
+};
+
+static struct qcom_icc_bcm * const pcie_anoc_bcms[] = {
+	&bcm_sn3,
+};
+
+static struct qcom_icc_node * const pcie_anoc_nodes[] = {
+	[MASTER_PCIE_ANOC_CFG] = &qsm_pcie_anoc_cfg,
+	[MASTER_PCIE_0] = &xm_pcie,
+	[MASTER_PCIE_1] = &xm_pcie_g4x1,
+	[SLAVE_ANOC_PCIE_GEM_NOC] = &qns_pcie_gemnoc,
+	[SLAVE_SERVICE_PCIE_ANOC] = &srvc_pcie_aggre_noc,
+};
+
+static const struct regmap_config maili_pcie_anoc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x12400,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc maili_pcie_anoc = {
+	.config = &maili_pcie_anoc_regmap_config,
+	.nodes = pcie_anoc_nodes,
+	.num_nodes = ARRAY_SIZE(pcie_anoc_nodes),
+	.bcms = pcie_anoc_bcms,
+	.num_bcms = ARRAY_SIZE(pcie_anoc_bcms),
+	.qos_requires_clocks = true,
+};
+
+static struct qcom_icc_bcm * const stdst_cfg_bcms[] = {
+	&bcm_cn0,
+	&bcm_cn1,
+};
+
+static struct qcom_icc_node * const stdst_cfg_nodes[] = {
+	[MASTER_CFG_CENTER] = &qsm_cfg_center,
+	[MASTER_CFG_EAST] = &qsm_cfg_east,
+	[MASTER_CFG_MM_HF] = &qsm_cfg_mm_hf,
+	[MASTER_CFG_MM_SF] = &qsm_cfg_mm_sf,
+	[MASTER_CFG_NORTH] = &qsm_cfg_north,
+	[MASTER_CFG_SOUTH] = &qsm_cfg_south,
+	[MASTER_CFG_WEST] = &qsm_cfg_west,
+	[SLAVE_AHB2PHY_SOUTH] = &qhs_ahb2phy0,
+	[SLAVE_BOOT_ROM] = &qhs_boot_rom,
+	[SLAVE_CAMERA_CFG] = &qhs_camera_cfg,
+	[SLAVE_CLK_CTL] = &qhs_clk_ctl,
+	[SLAVE_CRYPTO_CFG] = &qhs_crypto_cfg,
+	[SLAVE_DISPLAY_CFG] = &qhs_display_cfg,
+	[SLAVE_EVA_CFG] = &qhs_eva_cfg,
+	[SLAVE_GFX3D_CFG] = &qhs_gpuss_cfg,
+	[SLAVE_I2C] = &qhs_i2c,
+	[SLAVE_IMEM_CFG] = &qhs_imem_cfg,
+	[SLAVE_IPC_ROUTER_CFG] = &qhs_ipc_router,
+	[SLAVE_IRIS_CFG] = &qhs_iris_cfg,
+	[SLAVE_CNOC_MSS] = &qhs_mss_cfg,
+	[SLAVE_PCIE_0_CFG] = &qhs_pcie_cfg,
+	[SLAVE_PCIE_1_CFG] = &qhs_pcie_g4x1_cfg,
+	[SLAVE_PRNG] = &qhs_prng,
+	[SLAVE_QSPI_0] = &qhs_qspi,
+	[SLAVE_QUP_1] = &qhs_qup1,
+	[SLAVE_QUP_2] = &qhs_qup2,
+	[SLAVE_QUP_3] = &qhs_qup3,
+	[SLAVE_QUP_4] = &qhs_qup4,
+	[SLAVE_QUP_5] = &qhs_qup5,
+	[SLAVE_SDCC_2] = &qhs_sdc2,
+	[SLAVE_SDCC_4] = &qhs_sdc4,
+	[SLAVE_TLMM] = &qhs_tlmm,
+	[SLAVE_UFS_MEM_CFG] = &qhs_ufs_mem_cfg,
+	[SLAVE_USB3] = &qhs_usb3,
+	[SLAVE_VSENSE_CTRL_CFG] = &qhs_vsense_ctrl_cfg,
+	[SLAVE_PCIE_ANOC_CFG] = &qss_pcie_anoc_cfg,
+	[SLAVE_QDSS_CFG] = &qss_qdss_cfg,
+	[SLAVE_QDSS_STM] = &qss_qdss_stm,
+	[SLAVE_TCSR] = &qss_tcsr,
+	[SLAVE_TCU] = &xs_sys_tcu_cfg,
+};
+
+static const struct regmap_config maili_stdst_cfg_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x9000,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc maili_stdst_cfg = {
+	.config = &maili_stdst_cfg_regmap_config,
+	.nodes = stdst_cfg_nodes,
+	.num_nodes = ARRAY_SIZE(stdst_cfg_nodes),
+	.bcms = stdst_cfg_bcms,
+	.num_bcms = ARRAY_SIZE(stdst_cfg_bcms),
+};
+
+static struct qcom_icc_node * const stdst_main_nodes[] = {
+	[MASTER_CNOC_STARDUST] = &qsm_cnoc_main,
+	[SLAVE_STARDUST_CENTER_CFG] = &qss_stdst_center_cfg,
+	[SLAVE_STARDUST_EAST_CFG] = &qss_stdst_east_cfg,
+	[SLAVE_STARDUST_MM_HF_CFG] = &qss_stdst_mm_hf_cfg,
+	[SLAVE_STARDUST_MM_SF_CFG] = &qss_stdst_mm_sf_cfg,
+	[SLAVE_STARDUST_NORTH_CFG] = &qss_stdst_north_cfg,
+	[SLAVE_STARDUST_SOUTH_CFG] = &qss_stdst_south_cfg,
+	[SLAVE_STARDUST_WEST_CFG] = &qss_stdst_west_cfg,
+};
+
+static const struct regmap_config maili_stdst_main_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xf9000,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc maili_stdst_main = {
+	.config = &maili_stdst_main_regmap_config,
+	.nodes = stdst_main_nodes,
+	.num_nodes = ARRAY_SIZE(stdst_main_nodes),
+};
+
+static struct qcom_icc_bcm * const system_noc_bcms[] = {
+	&bcm_sn0,
+	&bcm_sn2,
+};
+
+static struct qcom_icc_node * const system_noc_nodes[] = {
+	[MASTER_A1NOC_SNOC] = &qnm_aggre_noc,
+	[MASTER_APSS_NOC] = &qnm_apss_noc,
+	[MASTER_CNOC_SNOC] = &qnm_cnoc_data,
+	[SLAVE_SNOC_GEM_NOC_SF] = &qns_gemnoc_sf,
+};
+
+static const struct regmap_config maili_system_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x20080,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc maili_system_noc = {
+	.config = &maili_system_noc_regmap_config,
+	.nodes = system_noc_nodes,
+	.num_nodes = ARRAY_SIZE(system_noc_nodes),
+	.bcms = system_noc_bcms,
+	.num_bcms = ARRAY_SIZE(system_noc_bcms),
+};
+
+static const struct of_device_id qnoc_of_match[] = {
+	{ .compatible = "qcom,maili-aggre-noc", .data = &maili_aggre1_noc },
+	{ .compatible = "qcom,maili-clk-virt", .data = &maili_clk_virt },
+	{ .compatible = "qcom,maili-cnoc-main", .data = &maili_cnoc_main },
+	{ .compatible = "qcom,maili-gem-noc", .data = &maili_gem_noc },
+	{ .compatible = "qcom,maili-llclpi-noc", .data = &maili_llclpi_noc },
+	{ .compatible = "qcom,maili-lpass-ag-noc", .data = &maili_lpass_ag_noc },
+	{ .compatible = "qcom,maili-lpass-lpiaon-noc", .data = &maili_lpass_lpiaon_noc },
+	{ .compatible = "qcom,maili-lpass-lpicx-noc", .data = &maili_lpass_lpicx_noc },
+	{ .compatible = "qcom,maili-mc-virt", .data = &maili_mc_virt },
+	{ .compatible = "qcom,maili-mmss-noc", .data = &maili_mmss_noc },
+	{ .compatible = "qcom,maili-nsp-noc", .data = &maili_nsp_noc },
+	{ .compatible = "qcom,maili-pcie-anoc", .data = &maili_pcie_anoc },
+	{ .compatible = "qcom,maili-stdst-cfg", .data = &maili_stdst_cfg },
+	{ .compatible = "qcom,maili-stdst-main", .data = &maili_stdst_main },
+	{ .compatible = "qcom,maili-system-noc", .data = &maili_system_noc },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, qnoc_of_match);
+
+static struct platform_driver qnoc_driver = {
+	.probe = qcom_icc_rpmh_probe,
+	.remove = qcom_icc_rpmh_remove,
+	.driver = {
+		.name = "qnoc-maili",
+		.of_match_table = qnoc_of_match,
+		.sync_state = icc_sync_state,
+	},
+};
+
+static int __init qnoc_driver_init(void)
+{
+	return platform_driver_register(&qnoc_driver);
+}
+core_initcall(qnoc_driver_init);
+
+static void __exit qnoc_driver_exit(void)
+{
+	platform_driver_unregister(&qnoc_driver);
+}
+module_exit(qnoc_driver_exit);
+
+MODULE_DESCRIPTION("Qualcomm Maili NoC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/interconnect/qcom/osm-l3.c b/drivers/interconnect/qcom/osm-l3.c
index b33f00da1880d..ecad636b53e06 100644
--- a/drivers/interconnect/qcom/osm-l3.c
+++ b/drivers/interconnect/qcom/osm-l3.c
@@ -60,6 +60,7 @@ struct qcom_osm_l3_desc {
 	unsigned int lut_row_size;
 	unsigned int reg_freq_lut;
 	unsigned int reg_perf_state;
+	unsigned int lut_max_entries;
 };
 
 #define DEFINE_QNODE(_name, _buswidth)					\
@@ -90,6 +91,7 @@ static const struct qcom_osm_l3_desc osm_l3 = {
 	.lut_row_size = OSM_LUT_ROW_SIZE,
 	.reg_freq_lut = OSM_REG_FREQ_LUT,
 	.reg_perf_state = OSM_REG_PERF_STATE,
+	.lut_max_entries = LUT_MAX_ENTRIES,
 };
 
 static const struct qcom_osm_l3_desc epss_l3_perf_state = {
@@ -98,6 +100,16 @@ static const struct qcom_osm_l3_desc epss_l3_perf_state = {
 	.lut_row_size = EPSS_LUT_ROW_SIZE,
 	.reg_freq_lut = EPSS_REG_FREQ_LUT,
 	.reg_perf_state = EPSS_REG_PERF_STATE,
+	.lut_max_entries = LUT_MAX_ENTRIES,
+};
+
+static const struct qcom_osm_l3_desc shikra_epss_l3_perf_state = {
+	.nodes = epss_l3_nodes,
+	.num_nodes = ARRAY_SIZE(epss_l3_nodes),
+	.lut_row_size = EPSS_LUT_ROW_SIZE,
+	.reg_freq_lut = EPSS_REG_FREQ_LUT,
+	.reg_perf_state = EPSS_REG_PERF_STATE,
+	.lut_max_entries = 12,
 };
 
 static const struct qcom_osm_l3_desc epss_l3_l3_vote = {
@@ -106,6 +118,7 @@ static const struct qcom_osm_l3_desc epss_l3_l3_vote = {
 	.lut_row_size = EPSS_LUT_ROW_SIZE,
 	.reg_freq_lut = EPSS_REG_FREQ_LUT,
 	.reg_perf_state = EPSS_REG_L3_VOTE,
+	.lut_max_entries = LUT_MAX_ENTRIES,
 };
 
 static int qcom_osm_l3_set(struct icc_node *src, struct icc_node *dst)
@@ -189,7 +202,7 @@ static int qcom_osm_l3_probe(struct platform_device *pdev)
 
 	qp->reg_perf_state = desc->reg_perf_state;
 
-	for (i = 0; i < LUT_MAX_ENTRIES; i++) {
+	for (i = 0; i < desc->lut_max_entries; i++) {
 		info = readl_relaxed(qp->base + desc->reg_freq_lut +
 				     i * desc->lut_row_size);
 		src = FIELD_GET(LUT_SRC, info);
@@ -272,6 +285,7 @@ static const struct of_device_id osm_l3_of_match[] = {
 	{ .compatible = "qcom,sc7180-osm-l3", .data = &osm_l3 },
 	{ .compatible = "qcom,sc7280-epss-l3", .data = &epss_l3_perf_state },
 	{ .compatible = "qcom,sdm845-osm-l3", .data = &osm_l3 },
+	{ .compatible = "qcom,shikra-epss-l3", .data = &shikra_epss_l3_perf_state },
 	{ .compatible = "qcom,sm8150-osm-l3", .data = &osm_l3 },
 	{ .compatible = "qcom,sc8180x-osm-l3", .data = &osm_l3 },
 	{ .compatible = "qcom,sm8250-epss-l3", .data = &epss_l3_perf_state },
diff --git a/drivers/interconnect/qcom/sc8280xp.c b/drivers/interconnect/qcom/sc8280xp.c
index 4110536664d08..15e284fcd89a8 100644
--- a/drivers/interconnect/qcom/sc8280xp.c
+++ b/drivers/interconnect/qcom/sc8280xp.c
@@ -215,6 +215,11 @@ static struct qcom_icc_node qhm_qspi = {
 	.name = "qhm_qspi",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x5000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -223,6 +228,11 @@ static struct qcom_icc_node qhm_qup1 = {
 	.name = "qhm_qup1",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x6000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -231,6 +241,11 @@ static struct qcom_icc_node qhm_qup2 = {
 	.name = "qhm_qup2",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xc000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -247,6 +262,12 @@ static struct qcom_icc_node qxm_ipa = {
 	.name = "qxm_ipa",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x8000 },
+		.prio = 2,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -255,6 +276,11 @@ static struct qcom_icc_node xm_emac_1 = {
 	.name = "xm_emac_1",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xd000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -263,6 +289,11 @@ static struct qcom_icc_node xm_sdc4 = {
 	.name = "xm_sdc4",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x7000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -271,6 +302,11 @@ static struct qcom_icc_node xm_ufs_mem = {
 	.name = "xm_ufs_mem",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x9000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -279,6 +315,11 @@ static struct qcom_icc_node xm_usb3_0 = {
 	.name = "xm_usb3_0",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xb000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_snoc },
 };
@@ -287,6 +328,11 @@ static struct qcom_icc_node xm_usb3_1 = {
 	.name = "xm_usb3_1",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_snoc },
 };
@@ -295,6 +341,11 @@ static struct qcom_icc_node xm_usb3_mp = {
 	.name = "xm_usb3_mp",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xe000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_snoc },
 };
@@ -303,6 +354,11 @@ static struct qcom_icc_node xm_usb4_host0 = {
 	.name = "xm_usb4_host0",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xf000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_snoc },
 };
@@ -311,6 +367,11 @@ static struct qcom_icc_node xm_usb4_host1 = {
 	.name = "xm_usb4_host1",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x10000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_snoc },
 };
@@ -319,6 +380,11 @@ static struct qcom_icc_node qhm_qdss_bam = {
 	.name = "qhm_qdss_bam",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -327,6 +393,11 @@ static struct qcom_icc_node qhm_qup0 = {
 	.name = "qhm_qup0",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xb000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -343,6 +414,12 @@ static struct qcom_icc_node qxm_crypto = {
 	.name = "qxm_crypto",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xe000 },
+		.prio = 2,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -367,6 +444,11 @@ static struct qcom_icc_node xm_emac_0 = {
 	.name = "xm_emac_0",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x11000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -375,6 +457,11 @@ static struct qcom_icc_node xm_pcie3_0 = {
 	.name = "xm_pcie3_0",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x12000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_gem_noc },
 };
@@ -383,6 +470,11 @@ static struct qcom_icc_node xm_pcie3_1 = {
 	.name = "xm_pcie3_1",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xf000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_gem_noc },
 };
@@ -391,6 +483,11 @@ static struct qcom_icc_node xm_pcie3_2a = {
 	.name = "xm_pcie3_2a",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x13000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_gem_noc },
 };
@@ -399,6 +496,11 @@ static struct qcom_icc_node xm_pcie3_2b = {
 	.name = "xm_pcie3_2b",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x14000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_gem_noc },
 };
@@ -407,6 +509,11 @@ static struct qcom_icc_node xm_pcie3_3a = {
 	.name = "xm_pcie3_3a",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x15000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_gem_noc },
 };
@@ -415,6 +522,11 @@ static struct qcom_icc_node xm_pcie3_3b = {
 	.name = "xm_pcie3_3b",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x16000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_gem_noc },
 };
@@ -423,6 +535,11 @@ static struct qcom_icc_node xm_pcie3_4 = {
 	.name = "xm_pcie3_4",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x10000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_gem_noc },
 };
@@ -431,6 +548,11 @@ static struct qcom_icc_node xm_qdss_etr = {
 	.name = "xm_qdss_etr",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x7000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -439,6 +561,11 @@ static struct qcom_icc_node xm_sdc2 = {
 	.name = "xm_sdc2",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x8000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -447,6 +574,11 @@ static struct qcom_icc_node xm_ufs_card = {
 	.name = "xm_ufs_card",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x9000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -586,6 +718,11 @@ static struct qcom_icc_node alm_gpu_tcu = {
 	.name = "alm_gpu_tcu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa0000 },
+		.prio = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc,
 			&qns_llcc },
@@ -595,6 +732,11 @@ static struct qcom_icc_node alm_pcie_tcu = {
 	.name = "alm_pcie_tcu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa1000 },
+		.prio = 3,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc,
 			&qns_llcc },
@@ -604,6 +746,11 @@ static struct qcom_icc_node alm_sys_tcu = {
 	.name = "alm_sys_tcu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa2000 },
+		.prio = 6,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc,
 			&qns_llcc },
@@ -623,6 +770,11 @@ static struct qcom_icc_node qnm_cmpnoc0 = {
 	.name = "qnm_cmpnoc0",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x1b000, 0x66000 },
+		.urg_fwd = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc,
 			&qns_llcc },
@@ -632,6 +784,11 @@ static struct qcom_icc_node qnm_cmpnoc1 = {
 	.name = "qnm_cmpnoc1",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x24000, 0x6f000 },
+		.urg_fwd = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc,
 			&qns_llcc },
@@ -651,6 +808,11 @@ static struct qcom_icc_node qnm_gpu = {
 	.name = "qnm_gpu",
 	.channels = 4,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 4,
+		.port_offsets = { 0x1c000, 0x1d000, 0x67000, 0x68000 },
+		.urg_fwd = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc,
 			&qns_llcc },
@@ -660,6 +822,11 @@ static struct qcom_icc_node qnm_mnoc_hf = {
 	.name = "qnm_mnoc_hf",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x1e000, 0x69000 },
+		.urg_fwd = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_llcc,
 			&qns_pcie },
@@ -669,6 +836,11 @@ static struct qcom_icc_node qnm_mnoc_sf = {
 	.name = "qnm_mnoc_sf",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x1f000, 0x6a000 },
+		.urg_fwd = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc,
 			&qns_llcc },
@@ -678,6 +850,12 @@ static struct qcom_icc_node qnm_pcie = {
 	.name = "qnm_pcie",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa3000 },
+		.prio = 2,
+		.urg_fwd = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc,
 			&qns_llcc },
@@ -687,6 +865,11 @@ static struct qcom_icc_node qnm_snoc_gc = {
 	.name = "qnm_snoc_gc",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa4000 },
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_llcc },
 };
@@ -695,6 +878,11 @@ static struct qcom_icc_node qnm_snoc_sf = {
 	.name = "qnm_snoc_sf",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa5000 },
+		.urg_fwd = 1,
+	},
 	.num_links = 3,
 	.link_nodes = { &qns_gem_noc_cnoc,
 			&qns_llcc,
@@ -737,6 +925,11 @@ static struct qcom_icc_node qnm_camnoc_hf = {
 	.name = "qnm_camnoc_hf",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x10000, 0x10180 },
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_hf },
 };
@@ -745,6 +938,11 @@ static struct qcom_icc_node qnm_mdp0_0 = {
 	.name = "qnm_mdp0_0",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x16000 },
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_hf },
 };
@@ -753,6 +951,11 @@ static struct qcom_icc_node qnm_mdp0_1 = {
 	.name = "qnm_mdp0_1",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x16080 },
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_hf },
 };
@@ -761,6 +964,11 @@ static struct qcom_icc_node qnm_mdp1_0 = {
 	.name = "qnm_mdp1_0",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xf000 },
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_hf },
 };
@@ -769,6 +977,11 @@ static struct qcom_icc_node qnm_mdp1_1 = {
 	.name = "qnm_mdp1_1",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xf800 },
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_hf },
 };
@@ -785,6 +998,11 @@ static struct qcom_icc_node qnm_rot_0 = {
 	.name = "qnm_rot_0",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x17000 },
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -793,6 +1011,11 @@ static struct qcom_icc_node qnm_rot_1 = {
 	.name = "qnm_rot_1",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xe000 },
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -801,6 +1024,11 @@ static struct qcom_icc_node qnm_video0 = {
 	.name = "qnm_video0",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x14000 },
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -809,6 +1037,11 @@ static struct qcom_icc_node qnm_video1 = {
 	.name = "qnm_video1",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x14080 },
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -817,6 +1050,11 @@ static struct qcom_icc_node qnm_video_cvp = {
 	.name = "qnm_video_cvp",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x15000 },
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -825,6 +1063,11 @@ static struct qcom_icc_node qxm_camnoc_icp = {
 	.name = "qxm_camnoc_icp",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x11000 },
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -833,6 +1076,11 @@ static struct qcom_icc_node qxm_camnoc_sf = {
 	.name = "qxm_camnoc_sf",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x12000 },
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -899,6 +1147,11 @@ static struct qcom_icc_node qnm_lpass_noc = {
 	.name = "qnm_lpass_noc",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xe000 },
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_gemnoc_sf },
 };
@@ -915,6 +1168,12 @@ static struct qcom_icc_node qxm_pimem = {
 	.name = "qxm_pimem",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x10000 },
+		.prio = 2,
+		.urg_fwd = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_gemnoc_gc },
 };
@@ -923,6 +1182,11 @@ static struct qcom_icc_node xm_gic = {
 	.name = "xm_gic",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x12000 },
+		.prio = 2,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_gemnoc_gc },
 };
@@ -1996,11 +2260,21 @@ static struct qcom_icc_node * const aggre1_noc_nodes[] = {
 	[SLAVE_SERVICE_A1NOC] = &srvc_aggre1_noc,
 };
 
+static const struct regmap_config sc8280xp_aggre1_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x3af80,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc sc8280xp_aggre1_noc = {
+	.config = &sc8280xp_aggre1_noc_regmap_config,
 	.nodes = aggre1_noc_nodes,
 	.num_nodes = ARRAY_SIZE(aggre1_noc_nodes),
 	.bcms = aggre1_noc_bcms,
 	.num_bcms = ARRAY_SIZE(aggre1_noc_bcms),
+	.qos_requires_clocks = true,
 };
 
 static struct qcom_icc_bcm * const aggre2_noc_bcms[] = {
@@ -2032,11 +2306,21 @@ static struct qcom_icc_node * const aggre2_noc_nodes[] = {
 	[SLAVE_SERVICE_A2NOC] = &srvc_aggre2_noc,
 };
 
+static const struct regmap_config sc8280xp_aggre2_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x3af80,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc sc8280xp_aggre2_noc = {
+	.config = &sc8280xp_aggre2_noc_regmap_config,
 	.nodes = aggre2_noc_nodes,
 	.num_nodes = ARRAY_SIZE(aggre2_noc_nodes),
 	.bcms = aggre2_noc_bcms,
 	.num_bcms = ARRAY_SIZE(aggre2_noc_bcms),
+	.qos_requires_clocks = true,
 };
 
 static struct qcom_icc_bcm * const clk_virt_bcms[] = {
@@ -2158,7 +2442,16 @@ static struct qcom_icc_node * const config_noc_nodes[] = {
 	[SLAVE_TCU] = &xs_sys_tcu_cfg,
 };
 
+static const struct regmap_config sc8280xp_config_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x2c000,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc sc8280xp_config_noc = {
+	.config = &sc8280xp_config_noc_regmap_config,
 	.nodes = config_noc_nodes,
 	.num_nodes = ARRAY_SIZE(config_noc_nodes),
 	.bcms = config_noc_bcms,
@@ -2174,7 +2467,16 @@ static struct qcom_icc_node * const dc_noc_nodes[] = {
 	[SLAVE_GEM_NOC_CFG] = &qns_gemnoc,
 };
 
+static const struct regmap_config sc8280xp_dc_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x5080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc sc8280xp_dc_noc = {
+	.config = &sc8280xp_dc_noc_regmap_config,
 	.nodes = dc_noc_nodes,
 	.num_nodes = ARRAY_SIZE(dc_noc_nodes),
 	.bcms = dc_noc_bcms,
@@ -2208,7 +2510,16 @@ static struct qcom_icc_node * const gem_noc_nodes[] = {
 	[SLAVE_SERVICE_GEM_NOC] = &srvc_sys_gemnoc,
 };
 
+static const struct regmap_config sc8280xp_gem_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xb8400,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc sc8280xp_gem_noc = {
+	.config = &sc8280xp_gem_noc_regmap_config,
 	.nodes = gem_noc_nodes,
 	.num_nodes = ARRAY_SIZE(gem_noc_nodes),
 	.bcms = gem_noc_bcms,
@@ -2231,7 +2542,16 @@ static struct qcom_icc_node * const lpass_ag_noc_nodes[] = {
 	[SLAVE_SERVICE_LPASS_AG_NOC] = &srvc_niu_lpass_agnoc,
 };
 
+static const struct regmap_config sc8280xp_lpass_ag_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xf080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc sc8280xp_lpass_ag_noc = {
+	.config = &sc8280xp_lpass_ag_noc_regmap_config,
 	.nodes = lpass_ag_noc_nodes,
 	.num_nodes = ARRAY_SIZE(lpass_ag_noc_nodes),
 	.bcms = lpass_ag_noc_bcms,
@@ -2279,7 +2599,16 @@ static struct qcom_icc_node * const mmss_noc_nodes[] = {
 	[SLAVE_SERVICE_MNOC] = &srvc_mnoc,
 };
 
+static const struct regmap_config sc8280xp_mmss_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1fa80,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc sc8280xp_mmss_noc = {
+	.config = &sc8280xp_mmss_noc_regmap_config,
 	.nodes = mmss_noc_nodes,
 	.num_nodes = ARRAY_SIZE(mmss_noc_nodes),
 	.bcms = mmss_noc_bcms,
@@ -2299,7 +2628,16 @@ static struct qcom_icc_node * const nspa_noc_nodes[] = {
 	[SLAVE_SERVICE_NSP_NOC] = &service_nsp_noc,
 };
 
+static const struct regmap_config sc8280xp_nspa_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x10000,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc sc8280xp_nspa_noc = {
+	.config = &sc8280xp_nspa_noc_regmap_config,
 	.nodes = nspa_noc_nodes,
 	.num_nodes = ARRAY_SIZE(nspa_noc_nodes),
 	.bcms = nspa_noc_bcms,
@@ -2319,7 +2657,16 @@ static struct qcom_icc_node * const nspb_noc_nodes[] = {
 	[SLAVE_SERVICE_NSPB_NOC] = &service_nspb_noc,
 };
 
+static const struct regmap_config sc8280xp_nspb_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x10000,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc sc8280xp_nspb_noc = {
+	.config = &sc8280xp_nspb_noc_regmap_config,
 	.nodes = nspb_noc_nodes,
 	.num_nodes = ARRAY_SIZE(nspb_noc_nodes),
 	.bcms = nspb_noc_bcms,
@@ -2348,7 +2695,16 @@ static struct qcom_icc_node * const system_noc_main_nodes[] = {
 	[SLAVE_SERVICE_SNOC] = &srvc_snoc,
 };
 
+static const struct regmap_config sc8280xp_system_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1a400,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc sc8280xp_system_noc_main = {
+	.config = &sc8280xp_system_noc_regmap_config,
 	.nodes = system_noc_main_nodes,
 	.num_nodes = ARRAY_SIZE(system_noc_main_nodes),
 	.bcms = system_noc_main_bcms,
diff --git a/drivers/interconnect/qcom/x1e80100.c b/drivers/interconnect/qcom/x1e80100.c
index 2ba2823c7860e..8075e0ff20597 100644
--- a/drivers/interconnect/qcom/x1e80100.c
+++ b/drivers/interconnect/qcom/x1e80100.c
@@ -173,6 +173,13 @@ static struct qcom_icc_node qhm_qspi = {
 	.name = "qhm_qspi",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xb000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -181,6 +188,13 @@ static struct qcom_icc_node qhm_qup1 = {
 	.name = "qhm_qup1",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xc000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -189,6 +203,13 @@ static struct qcom_icc_node xm_sdc4 = {
 	.name = "xm_sdc4",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xd000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -197,6 +218,13 @@ static struct qcom_icc_node xm_ufs_mem = {
 	.name = "xm_ufs_mem",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xe000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -205,6 +233,13 @@ static struct qcom_icc_node qhm_qup0 = {
 	.name = "qhm_qup0",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x16000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -213,6 +248,13 @@ static struct qcom_icc_node qhm_qup2 = {
 	.name = "qhm_qup2",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x11000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -221,6 +263,13 @@ static struct qcom_icc_node qxm_crypto = {
 	.name = "qxm_crypto",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x12000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -237,6 +286,13 @@ static struct qcom_icc_node xm_qdss_etr_0 = {
 	.name = "xm_qdss_etr_0",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x13000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -245,6 +301,13 @@ static struct qcom_icc_node xm_qdss_etr_1 = {
 	.name = "xm_qdss_etr_1",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x14000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -253,6 +316,13 @@ static struct qcom_icc_node xm_sdc2 = {
 	.name = "xm_sdc2",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x15000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -337,6 +407,13 @@ static struct qcom_icc_node alm_gpu_tcu = {
 	.name = "alm_gpu_tcu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x271000 },
+		.prio = 1,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -345,6 +422,13 @@ static struct qcom_icc_node alm_pcie_tcu = {
 	.name = "alm_pcie_tcu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x27d000 },
+		.prio = 3,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -353,6 +437,13 @@ static struct qcom_icc_node alm_sys_tcu = {
 	.name = "alm_sys_tcu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x273000 },
+		.prio = 6,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -370,6 +461,13 @@ static struct qcom_icc_node qnm_gpu = {
 	.name = "qnm_gpu",
 	.channels = 4,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 4,
+		.port_offsets = { 0x51000, 0x58000, 0xd1000, 0xd8000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -378,6 +476,13 @@ static struct qcom_icc_node qnm_lpass = {
 	.name = "qnm_lpass",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x275000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 3,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
 			&qns_pcie },
@@ -387,6 +492,13 @@ static struct qcom_icc_node qnm_mnoc_hf = {
 	.name = "qnm_mnoc_hf",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x53000, 0xd3000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -395,6 +507,13 @@ static struct qcom_icc_node qnm_mnoc_sf = {
 	.name = "qnm_mnoc_sf",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x55000, 0xd5000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -403,6 +522,13 @@ static struct qcom_icc_node qnm_nsp_noc = {
 	.name = "qnm_nsp_noc",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x57000, 0xd7000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 3,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
 			&qns_pcie },
@@ -412,6 +538,13 @@ static struct qcom_icc_node qnm_pcie = {
 	.name = "qnm_pcie",
 	.channels = 1,
 	.buswidth = 64,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x277000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -420,6 +553,13 @@ static struct qcom_icc_node qnm_snoc_sf = {
 	.name = "qnm_snoc_sf",
 	.channels = 1,
 	.buswidth = 64,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x27b000 },
+		.prio = 2,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 3,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
 			&qns_pcie },
@@ -429,6 +569,13 @@ static struct qcom_icc_node xm_gic = {
 	.name = "xm_gic",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x27f000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_llcc },
 };
@@ -469,6 +616,13 @@ static struct qcom_icc_node qnm_av1_enc = {
 	.name = "qnm_av1_enc",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x2f000 },
+		.prio = 4,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -477,6 +631,13 @@ static struct qcom_icc_node qnm_camnoc_hf = {
 	.name = "qnm_camnoc_hf",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x28000, 0x29000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_hf },
 };
@@ -485,6 +646,13 @@ static struct qcom_icc_node qnm_camnoc_icp = {
 	.name = "qnm_camnoc_icp",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x2a000 },
+		.prio = 4,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -493,6 +661,13 @@ static struct qcom_icc_node qnm_camnoc_sf = {
 	.name = "qnm_camnoc_sf",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x2b000, 0x2c000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -501,6 +676,13 @@ static struct qcom_icc_node qnm_eva = {
 	.name = "qnm_eva",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x33000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -509,6 +691,13 @@ static struct qcom_icc_node qnm_mdp = {
 	.name = "qnm_mdp",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x2d000, 0x2e000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_hf },
 };
@@ -517,6 +706,13 @@ static struct qcom_icc_node qnm_video = {
 	.name = "qnm_video",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x30000, 0x31000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -525,6 +721,13 @@ static struct qcom_icc_node qnm_video_cv_cpu = {
 	.name = "qnm_video_cv_cpu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x32000 },
+		.prio = 4,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -533,6 +736,13 @@ static struct qcom_icc_node qnm_video_v_cpu = {
 	.name = "qnm_video_v_cpu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x34000 },
+		.prio = 4,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -573,6 +783,13 @@ static struct qcom_icc_node xm_pcie_3 = {
 	.name = "xm_pcie_3",
 	.channels = 1,
 	.buswidth = 64,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x7000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_north_gem_noc },
 };
@@ -581,6 +798,13 @@ static struct qcom_icc_node xm_pcie_4 = {
 	.name = "xm_pcie_4",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x8000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_north_gem_noc },
 };
@@ -589,6 +813,13 @@ static struct qcom_icc_node xm_pcie_5 = {
 	.name = "xm_pcie_5",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x9000 },
+		.prio = 3,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_north_gem_noc },
 };
@@ -597,6 +828,13 @@ static struct qcom_icc_node xm_pcie_0 = {
 	.name = "xm_pcie_0",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x9000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_south_gem_noc },
 };
@@ -605,6 +843,13 @@ static struct qcom_icc_node xm_pcie_1 = {
 	.name = "xm_pcie_1",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_south_gem_noc },
 };
@@ -613,6 +858,13 @@ static struct qcom_icc_node xm_pcie_2 = {
 	.name = "xm_pcie_2",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xb000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_south_gem_noc },
 };
@@ -621,6 +873,13 @@ static struct qcom_icc_node xm_pcie_6a = {
 	.name = "xm_pcie_6a",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xc000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_south_gem_noc },
 };
@@ -629,6 +888,13 @@ static struct qcom_icc_node xm_pcie_6b = {
 	.name = "xm_pcie_6b",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xd000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_south_gem_noc },
 };
@@ -653,6 +919,13 @@ static struct qcom_icc_node qnm_gic = {
 	.name = "qnm_gic",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1c000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_gemnoc_sf },
 };
@@ -685,6 +958,13 @@ static struct qcom_icc_node xm_usb2_0 = {
 	.name = "xm_usb2_0",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x6000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_north_snoc },
 };
@@ -693,6 +973,13 @@ static struct qcom_icc_node xm_usb3_mp = {
 	.name = "xm_usb3_mp",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x7000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_north_snoc },
 };
@@ -701,6 +988,13 @@ static struct qcom_icc_node xm_usb3_0 = {
 	.name = "xm_usb3_0",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -709,6 +1003,13 @@ static struct qcom_icc_node xm_usb3_1 = {
 	.name = "xm_usb3_1",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xb000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -717,6 +1018,13 @@ static struct qcom_icc_node xm_usb3_2 = {
 	.name = "xm_usb3_2",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xc000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -725,6 +1033,13 @@ static struct qcom_icc_node xm_usb4_0 = {
 	.name = "xm_usb4_0",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xd000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -733,6 +1048,13 @@ static struct qcom_icc_node xm_usb4_1 = {
 	.name = "xm_usb4_1",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xe000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -741,6 +1063,13 @@ static struct qcom_icc_node xm_usb4_2 = {
 	.name = "xm_usb4_2",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xf000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -1466,11 +1795,21 @@ static struct qcom_icc_node * const aggre1_noc_nodes[] = {
 	[SLAVE_A1NOC_SNOC] = &qns_a1noc_snoc,
 };
 
+static const struct regmap_config x1e80100_aggre1_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x14400,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_aggre1_noc = {
+	.config = &x1e80100_aggre1_noc_regmap_config,
 	.nodes = aggre1_noc_nodes,
 	.num_nodes = ARRAY_SIZE(aggre1_noc_nodes),
 	.bcms = aggre1_noc_bcms,
 	.num_bcms = ARRAY_SIZE(aggre1_noc_bcms),
+	.qos_requires_clocks = true,
 };
 
 static struct qcom_icc_bcm * const aggre2_noc_bcms[] = {
@@ -1488,7 +1827,16 @@ static struct qcom_icc_node * const aggre2_noc_nodes[] = {
 	[SLAVE_A2NOC_SNOC] = &qns_a2noc_snoc,
 };
 
+static const struct regmap_config x1e80100_aggre2_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1c400,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_aggre2_noc = {
+	.config = &x1e80100_aggre2_noc_regmap_config,
 	.nodes = aggre2_noc_nodes,
 	.num_nodes = ARRAY_SIZE(aggre2_noc_nodes),
 	.bcms = aggre2_noc_bcms,
@@ -1573,7 +1921,16 @@ static struct qcom_icc_node * const cnoc_cfg_nodes[] = {
 	[SLAVE_TCU] = &xs_sys_tcu_cfg,
 };
 
+static const struct regmap_config x1e80100_cnoc_cfg_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x6600,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_cnoc_cfg = {
+	.config = &x1e80100_cnoc_cfg_regmap_config,
 	.nodes = cnoc_cfg_nodes,
 	.num_nodes = ARRAY_SIZE(cnoc_cfg_nodes),
 	.bcms = cnoc_cfg_bcms,
@@ -1603,7 +1960,16 @@ static struct qcom_icc_node * const cnoc_main_nodes[] = {
 	[SLAVE_PCIE_6B] = &xs_pcie_6b,
 };
 
+static const struct regmap_config x1e80100_cnoc_main_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x14400,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_cnoc_main = {
+	.config = &x1e80100_cnoc_main_regmap_config,
 	.nodes = cnoc_main_nodes,
 	.num_nodes = ARRAY_SIZE(cnoc_main_nodes),
 	.bcms = cnoc_main_bcms,
@@ -1633,7 +1999,16 @@ static struct qcom_icc_node * const gem_noc_nodes[] = {
 	[SLAVE_MEM_NOC_PCIE_SNOC] = &qns_pcie,
 };
 
+static const struct regmap_config x1e80100_gem_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x311200,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_gem_noc = {
+	.config = &x1e80100_gem_noc_regmap_config,
 	.nodes = gem_noc_nodes,
 	.num_nodes = ARRAY_SIZE(gem_noc_nodes),
 	.bcms = gem_noc_bcms,
@@ -1648,7 +2023,16 @@ static struct qcom_icc_node * const lpass_ag_noc_nodes[] = {
 	[SLAVE_LPASS_GEM_NOC] = &qns_lpass_ag_noc_gemnoc,
 };
 
+static const struct regmap_config x1e80100_lpass_ag_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xe080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_lpass_ag_noc = {
+	.config = &x1e80100_lpass_ag_noc_regmap_config,
 	.nodes = lpass_ag_noc_nodes,
 	.num_nodes = ARRAY_SIZE(lpass_ag_noc_nodes),
 	.bcms = lpass_ag_noc_bcms,
@@ -1664,7 +2048,16 @@ static struct qcom_icc_node * const lpass_lpiaon_noc_nodes[] = {
 	[SLAVE_LPIAON_NOC_LPASS_AG_NOC] = &qns_lpass_aggnoc,
 };
 
+static const struct regmap_config x1e80100_lpass_lpiaon_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x19080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_lpass_lpiaon_noc = {
+	.config = &x1e80100_lpass_lpiaon_noc_regmap_config,
 	.nodes = lpass_lpiaon_noc_nodes,
 	.num_nodes = ARRAY_SIZE(lpass_lpiaon_noc_nodes),
 	.bcms = lpass_lpiaon_noc_bcms,
@@ -1679,7 +2072,16 @@ static struct qcom_icc_node * const lpass_lpicx_noc_nodes[] = {
 	[SLAVE_LPICX_NOC_LPIAON_NOC] = &qns_lpi_aon_noc,
 };
 
+static const struct regmap_config x1e80100_lpass_lpicx_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x3a200,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_lpass_lpicx_noc = {
+	.config = &x1e80100_lpass_lpicx_noc_regmap_config,
 	.nodes = lpass_lpicx_noc_nodes,
 	.num_nodes = ARRAY_SIZE(lpass_lpicx_noc_nodes),
 	.bcms = lpass_lpicx_noc_bcms,
@@ -1724,7 +2126,16 @@ static struct qcom_icc_node * const mmss_noc_nodes[] = {
 	[SLAVE_SERVICE_MNOC] = &srvc_mnoc,
 };
 
+static const struct regmap_config x1e80100_mmss_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x5b800,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_mmss_noc = {
+	.config = &x1e80100_mmss_noc_regmap_config,
 	.nodes = mmss_noc_nodes,
 	.num_nodes = ARRAY_SIZE(mmss_noc_nodes),
 	.bcms = mmss_noc_bcms,
@@ -1740,7 +2151,16 @@ static struct qcom_icc_node * const nsp_noc_nodes[] = {
 	[SLAVE_CDSP_MEM_NOC] = &qns_nsp_gemnoc,
 };
 
+static const struct regmap_config x1e80100_nsp_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xe080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_nsp_noc = {
+	.config = &x1e80100_nsp_noc_regmap_config,
 	.nodes = nsp_noc_nodes,
 	.num_nodes = ARRAY_SIZE(nsp_noc_nodes),
 	.bcms = nsp_noc_bcms,
@@ -1757,7 +2177,16 @@ static struct qcom_icc_node * const pcie_center_anoc_nodes[] = {
 	[SLAVE_ANOC_PCIE_GEM_NOC] = &qns_pcie_mem_noc,
 };
 
+static const struct regmap_config x1e80100_pcie_center_anoc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x7000,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_pcie_center_anoc = {
+	.config = &x1e80100_pcie_center_anoc_regmap_config,
 	.nodes = pcie_center_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(pcie_center_anoc_nodes),
 	.bcms = pcie_center_anoc_bcms,
@@ -1774,7 +2203,16 @@ static struct qcom_icc_node * const pcie_north_anoc_nodes[] = {
 	[SLAVE_PCIE_NORTH] = &qns_pcie_north_gem_noc,
 };
 
+static const struct regmap_config x1e80100_pcie_north_anoc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x9080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_pcie_north_anoc = {
+	.config = &x1e80100_pcie_north_anoc_regmap_config,
 	.nodes = pcie_north_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(pcie_north_anoc_nodes),
 	.bcms = pcie_north_anoc_bcms,
@@ -1793,7 +2231,16 @@ static struct qcom_icc_node * const pcie_south_anoc_nodes[] = {
 	[SLAVE_PCIE_SOUTH] = &qns_pcie_south_gem_noc,
 };
 
+static const struct regmap_config x1e80100_pcie_south_anoc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xd080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_pcie_south_anoc = {
+	.config = &x1e80100_pcie_south_anoc_regmap_config,
 	.nodes = pcie_south_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(pcie_south_anoc_nodes),
 	.bcms = pcie_south_anoc_bcms,
@@ -1815,7 +2262,16 @@ static struct qcom_icc_node * const system_noc_nodes[] = {
 	[SLAVE_SNOC_GEM_NOC_SF] = &qns_gemnoc_sf,
 };
 
+static const struct regmap_config x1e80100_system_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1c080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_system_noc = {
+	.config = &x1e80100_system_noc_regmap_config,
 	.nodes = system_noc_nodes,
 	.num_nodes = ARRAY_SIZE(system_noc_nodes),
 	.bcms = system_noc_bcms,
@@ -1831,7 +2287,16 @@ static struct qcom_icc_node * const usb_center_anoc_nodes[] = {
 	[SLAVE_USB_NOC_SNOC] = &qns_aggre_usb_snoc,
 };
 
+static const struct regmap_config x1e80100_usb_center_anoc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x8800,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_usb_center_anoc = {
+	.config = &x1e80100_usb_center_anoc_regmap_config,
 	.nodes = usb_center_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(usb_center_anoc_nodes),
 	.bcms = usb_center_anoc_bcms,
@@ -1847,11 +2312,21 @@ static struct qcom_icc_node * const usb_north_anoc_nodes[] = {
 	[SLAVE_AGGRE_USB_NORTH] = &qns_aggre_usb_north_snoc,
 };
 
+static const struct regmap_config x1e80100_usb_north_anoc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x7080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_usb_north_anoc = {
+	.config = &x1e80100_usb_north_anoc_regmap_config,
 	.nodes = usb_north_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(usb_north_anoc_nodes),
 	.bcms = usb_north_anoc_bcms,
 	.num_bcms = ARRAY_SIZE(usb_north_anoc_bcms),
+	.qos_requires_clocks = true,
 };
 
 static struct qcom_icc_bcm * const usb_south_anoc_bcms[] = {
@@ -1867,11 +2342,21 @@ static struct qcom_icc_node * const usb_south_anoc_nodes[] = {
 	[SLAVE_AGGRE_USB_SOUTH] = &qns_aggre_usb_south_snoc,
 };
 
+static const struct regmap_config x1e80100_usb_south_anoc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xf080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_usb_south_anoc = {
+	.config = &x1e80100_usb_south_anoc_regmap_config,
 	.nodes = usb_south_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(usb_south_anoc_nodes),
 	.bcms = usb_south_anoc_bcms,
 	.num_bcms = ARRAY_SIZE(usb_south_anoc_bcms),
+	.qos_requires_clocks = true,
 };
 
 static const struct of_device_id qnoc_of_match[] = {
diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index e93bcb5eef702..c0748521dff88 100644
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -3085,20 +3085,44 @@ static void __init free_iommu_resources(void)
 	free_pci_segments();
 }
 
-/* SB IOAPIC is always on this device in AMD systems */
-#define IOAPIC_SB_DEVID		((0x00 << 8) | PCI_DEVFN(0x14, 0))
+static bool __init check_sb_ioapic(int devid)
+{
+	u8 bus = PCI_BUS_NUM(devid);
+	u8 devfn = devid & 0xff;
+	u16 val;
+
+	val = read_pci_config_16(bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
+				 PCI_CLASS_DEVICE);
+
+	/*
+	 * The SB IOAPIC is integrated into the FCH (Southbridge), which is
+	 * exposed as an SMBus or ISA bridge in PCI config space.
+	 */
+	return val == PCI_CLASS_SERIAL_SMBUS || val == PCI_CLASS_BRIDGE_ISA;
+}
 
-/* SB IOAPIC for Hygon family 18h model 4h is on the device 0xb */
-#define IOAPIC_SB_DEVID_FAM18H_M4H	((0x00 << 8) | PCI_DEVFN(0xb, 0))
+/*
+ * The Southbridge IOAPIC is assigned a GSI Base of 0 (handling interrupts
+ * 0 through 23).
+ */
+static int __init get_sb_ioapic_id(void)
+{
+	int idx = mp_find_ioapic(0);
+
+	if (idx < 0)
+		return -ENODEV;
+
+	return mpc_ioapic_id(idx);
+}
 
 static bool __init check_ioapic_information(void)
 {
 	const char *fw_bug = FW_BUG;
 	bool ret, has_sb_ioapic;
-	int idx;
+	int idx, sb_apicid;
 
 	has_sb_ioapic = false;
-	ret           = false;
+	ret           = true;
 
 	/*
 	 * If we have map overrides on the kernel command line the
@@ -3108,6 +3132,16 @@ static bool __init check_ioapic_information(void)
 	if (cmdline_maps)
 		fw_bug = "";
 
+	sb_apicid = get_sb_ioapic_id();
+	if (sb_apicid < 0) {
+		/*
+		 * Lack of SB IOAPIC registration is not a firmware bug,
+		 * e.g. kernel booted with noapic or noacpi.
+		 */
+		fw_bug = "";
+		goto out;
+	}
+
 	for (idx = 0; idx < nr_ioapics; idx++) {
 		int devid, id = mpc_ioapic_id(idx);
 
@@ -3116,17 +3150,11 @@ static bool __init check_ioapic_information(void)
 			pr_err("%s: IOAPIC[%d] not in IVRS table\n",
 				fw_bug, id);
 			ret = false;
-		} else if (devid == IOAPIC_SB_DEVID ||
-			   (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON &&
-			    boot_cpu_data.x86 == 0x18 &&
-			    boot_cpu_data.x86_model >= 0x4 &&
-			    boot_cpu_data.x86_model <= 0xf &&
-			    devid == IOAPIC_SB_DEVID_FAM18H_M4H)) {
+		} else if (id == sb_apicid && check_sb_ioapic(devid)) {
 			has_sb_ioapic = true;
-			ret           = true;
 		}
 	}
-
+out:
 	if (!has_sb_ioapic) {
 		/*
 		 * We expect the SB IOAPIC to be listed in the IVRS
@@ -3137,6 +3165,7 @@ static bool __init check_ioapic_information(void)
 		 * device id for the IOAPIC in the system.
 		 */
 		pr_err("%s: No southbridge IOAPIC found\n", fw_bug);
+		ret = false;
 	}
 
 	if (!ret)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c
index 1e9f7d2de3441..85ebfdb3d2a7a 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c
@@ -297,6 +297,20 @@ static int arm_vsmmu_vsid_to_sid(struct arm_vsmmu *vsmmu, u32 vsid, u32 *sid)
 	return ret;
 }
 
+static int arm_vsmmu_vdevice_init(struct iommufd_vdevice *vdev)
+{
+	struct device *dev = iommufd_vdevice_to_device(vdev);
+	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
+
+	/*
+	 * arm_vsmmu_vsid_to_sid() maps a vSID to master->streams[0] alone, so
+	 * more streams would leave the rest stale and none reads out of bounds.
+	 */
+	if (master->num_streams != 1)
+		return -EOPNOTSUPP;
+	return 0;
+}
+
 /* This is basically iommu_viommu_arm_smmuv3_invalidate in u64 for conversion */
 struct arm_vsmmu_invalidation_cmd {
 	union {
@@ -403,6 +417,7 @@ int arm_vsmmu_cache_invalidate(struct iommufd_viommu *viommu,
 static const struct iommufd_viommu_ops arm_vsmmu_ops = {
 	.alloc_domain_nested = arm_vsmmu_alloc_domain_nested,
 	.cache_invalidate = arm_vsmmu_cache_invalidate,
+	.vdevice_init = arm_vsmmu_vdevice_init,
 };
 
 size_t arm_smmu_get_viommu_size(struct device *dev,
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index a10affb483a4f..57b750ebcd3d6 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -2956,8 +2956,13 @@ static void arm_smmu_enable_ats(struct arm_smmu_master *master)
 	 * ATC invalidation of PASID 0 causes the entire ATC to be flushed.
 	 */
 	arm_smmu_atc_inv_master(master, IOMMU_NO_PASID);
-	if (pci_enable_ats(pdev, stu))
-		dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
+
+	 /*
+	  * Since pci_prepare_ats() has already verified the HW capability
+	  * and programmed the STE, pci_enable_ats() should not fail here.
+	  */
+	WARN(pci_enable_ats(pdev, stu),
+	     "%s: Failed to enable ATS (STU %zu)\n", dev_name(master->dev), stu);
 }
 
 static int arm_smmu_enable_pasid(struct arm_smmu_master *master)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 9abaec0703efb..5598ed4bff72e 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -2068,10 +2068,20 @@ static void iommu_dma_iova_unlink_range_slow(struct device *dev,
 		arch_sync_dma_flush();
 }
 
-static void __iommu_dma_iova_unlink(struct device *dev,
-		struct dma_iova_state *state, size_t offset, size_t size,
-		enum dma_data_direction dir, unsigned long attrs,
-		bool free_iova)
+/**
+ * dma_iova_unlink - Unlink a range of IOVA space
+ * @dev: DMA device
+ * @state: IOVA state
+ * @offset: offset into the IOVA state to unlink
+ * @size: size of the buffer
+ * @dir: DMA direction
+ * @attrs: attributes of mapping properties
+ *
+ * Unlink a range of IOVA space for the given IOVA state.
+ */
+void dma_iova_unlink(struct device *dev, struct dma_iova_state *state,
+		size_t offset, size_t size, enum dma_data_direction dir,
+		unsigned long attrs)
 {
 	struct iommu_domain *domain = iommu_get_dma_domain(dev);
 	struct iommu_dma_cookie *cookie = domain->iova_cookie;
@@ -2087,35 +2097,13 @@ static void __iommu_dma_iova_unlink(struct device *dev,
 		iommu_dma_iova_unlink_range_slow(dev, addr, size, dir, attrs);
 
 	iommu_iotlb_gather_init(&iotlb_gather);
-	iotlb_gather.queued = free_iova && READ_ONCE(cookie->fq_domain);
 
 	size = iova_align(iovad, size + iova_start_pad);
 	addr -= iova_start_pad;
 	unmapped = iommu_unmap_fast(domain, addr, size, &iotlb_gather);
 	WARN_ON(unmapped != size);
 
-	if (!iotlb_gather.queued)
-		iommu_iotlb_sync(domain, &iotlb_gather);
-	if (free_iova)
-		iommu_dma_free_iova(domain, addr, size, &iotlb_gather);
-}
-
-/**
- * dma_iova_unlink - Unlink a range of IOVA space
- * @dev: DMA device
- * @state: IOVA state
- * @offset: offset into the IOVA state to unlink
- * @size: size of the buffer
- * @dir: DMA direction
- * @attrs: attributes of mapping properties
- *
- * Unlink a range of IOVA space for the given IOVA state.
- */
-void dma_iova_unlink(struct device *dev, struct dma_iova_state *state,
-		size_t offset, size_t size, enum dma_data_direction dir,
-		unsigned long attrs)
-{
-	 __iommu_dma_iova_unlink(dev, state, offset, size, dir, attrs, false);
+	iommu_iotlb_sync(domain, &iotlb_gather);
 }
 EXPORT_SYMBOL_GPL(dma_iova_unlink);
 
@@ -2136,14 +2124,13 @@ void dma_iova_destroy(struct device *dev, struct dma_iova_state *state,
 		unsigned long attrs)
 {
 	if (mapped_len)
-		__iommu_dma_iova_unlink(dev, state, 0, mapped_len, dir, attrs,
-				true);
-	else
-		/*
-		 * We can be here if first call to dma_iova_link() failed and
-		 * there is nothing to unlink, so let's be more clear.
-		 */
-		dma_iova_free(dev, state);
+		dma_iova_unlink(dev, state, 0, mapped_len, dir, attrs);
+
+	/*
+	 * We can be here if the first call to dma_iova_link() failed and
+	 * there is nothing to unlink, so let's be more clear.
+	 */
+	dma_iova_free(dev, state);
 }
 EXPORT_SYMBOL_GPL(dma_iova_destroy);
 
diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c
index 25aa477a95a95..c83bbc3faad56 100644
--- a/drivers/iommu/fsl_pamu.c
+++ b/drivers/iommu/fsl_pamu.c
@@ -8,6 +8,7 @@
 
 #include "fsl_pamu.h"
 
+#include <linux/cleanup.h>
 #include <linux/fsl/guts.h>
 #include <linux/interrupt.h>
 #include <linux/genalloc.h>
@@ -933,7 +934,6 @@ static struct platform_driver fsl_of_pamu_driver = {
 static __init int fsl_pamu_init(void)
 {
 	struct platform_device *pdev = NULL;
-	struct device_node *np;
 	int ret;
 
 	/*
@@ -955,7 +955,8 @@ static __init int fsl_pamu_init(void)
 	 * PAMU node would require significant changes to a lot of code.
 	 */
 
-	np = of_find_compatible_node(NULL, NULL, "fsl,pamu");
+	struct device_node *np __free(device_node) =
+			of_find_compatible_node(NULL, NULL, "fsl,pamu");
 	if (!np) {
 		pr_err("could not find a PAMU node\n");
 		return -ENODEV;
@@ -964,7 +965,7 @@ static __init int fsl_pamu_init(void)
 	ret = platform_driver_register(&fsl_of_pamu_driver);
 	if (ret) {
 		pr_err("could not register driver (err=%i)\n", ret);
-		goto error_driver_register;
+		return ret;
 	}
 
 	pdev = platform_device_alloc("fsl-of-pamu", 0);
@@ -973,7 +974,8 @@ static __init int fsl_pamu_init(void)
 		ret = -ENOMEM;
 		goto error_device_alloc;
 	}
-	pdev->dev.of_node = of_node_get(np);
+
+	platform_device_set_of_node(pdev, np);
 
 	ret = pamu_domain_init();
 	if (ret)
@@ -988,17 +990,11 @@ static __init int fsl_pamu_init(void)
 	return 0;
 
 error_device_add:
-	of_node_put(pdev->dev.of_node);
-	pdev->dev.of_node = NULL;
-
 	platform_device_put(pdev);
 
 error_device_alloc:
 	platform_driver_unregister(&fsl_of_pamu_driver);
 
-error_driver_register:
-	of_node_put(np);
-
 	return ret;
 }
 arch_initcall(fsl_pamu_init);
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index 849d06dfe1aec..f394513235533 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -876,8 +876,14 @@ static void iommu_enable_pci_ats(struct device_domain_info *info)
 	if (!pci_ats_page_aligned(pdev))
 		return;
 
-	if (!pci_enable_ats(pdev, VTD_PAGE_SHIFT))
-		info->ats_enabled = 1;
+	/*
+	 * pci_enable_ats() should not fail here because earlier checks
+	 * have already verified support and configuration.
+	 */
+	if (WARN_ON(pci_enable_ats(pdev, VTD_PAGE_SHIFT)))
+		return;
+
+	info->ats_enabled = 1;
 }
 
 static void iommu_disable_pci_ats(struct device_domain_info *info)
@@ -3292,7 +3298,10 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)
 
 	dev_iommu_priv_set(dev, info);
 	if (pdev && pci_ats_supported(pdev)) {
-		pci_prepare_ats(pdev, VTD_PAGE_SHIFT);
+		ret = pci_prepare_ats(pdev, VTD_PAGE_SHIFT);
+		if (ret)
+			goto free;
+
 		ret = device_rbtree_insert(iommu, info);
 		if (ret)
 			goto free;
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index e8f13dcebbde5..23a531595835f 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -390,7 +390,7 @@ int iommu_mock_device_add(struct device *dev, struct iommu_device *iommu)
 
 	rc = device_add(dev);
 	if (rc)
-		iommu_fwspec_free(dev);
+		dev_iommu_free(dev);
 	return rc;
 }
 EXPORT_SYMBOL_GPL(iommu_mock_device_add);
diff --git a/drivers/iommu/iommufd/Kconfig b/drivers/iommu/iommufd/Kconfig
index 455bac0351f28..6c3bea83631b5 100644
--- a/drivers/iommu/iommufd/Kconfig
+++ b/drivers/iommu/iommufd/Kconfig
@@ -16,6 +16,18 @@ config IOMMUFD
 	  If you don't know what to do here, say N.
 
 if IOMMUFD
+config IOMMUFD_NOIOMMU
+	bool
+	depends on !GENERIC_ATOMIC64 # IOMMU_PT_AMDV1 requires cmpxchg64
+	select GENERIC_PT
+	select IOMMU_PT
+	select IOMMU_PT_AMDV1
+	help
+	  Provides a SW-only IO page table for devices without hardware
+	  IOMMU backing. This uses the AMDV1 page table format for
+	  IOVA-to-PA lookups only, not for hardware DMA translation.
+	  To be selected by VFIO_NOIOMMU when VFIO_DEVICE_CDEV is enabled.
+
 config IOMMUFD_VFIO_CONTAINER
 	bool "IOMMUFD provides the VFIO container /dev/vfio/vfio"
 	depends on VFIO_GROUP && !VFIO_CONTAINER
diff --git a/drivers/iommu/iommufd/Makefile b/drivers/iommu/iommufd/Makefile
index 71d692c9a8f49..67207914bb6e0 100644
--- a/drivers/iommu/iommufd/Makefile
+++ b/drivers/iommu/iommufd/Makefile
@@ -10,6 +10,7 @@ iommufd-y := \
 	vfio_compat.o \
 	viommu.o
 
+iommufd-$(CONFIG_IOMMUFD_NOIOMMU) += hwpt_noiommu.o
 iommufd-$(CONFIG_IOMMUFD_TEST) += selftest.o
 
 obj-$(CONFIG_IOMMUFD) += iommufd.o
diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c
index 170a7005f0bc8..11944f4cac1a1 100644
--- a/drivers/iommu/iommufd/device.c
+++ b/drivers/iommu/iommufd/device.c
@@ -23,6 +23,19 @@ struct iommufd_attach {
 	struct xarray device_array;
 };
 
+/*
+ * Detect a noiommu device for the cdev path. We check dev->iommu rather than
+ * using device_iommu_mapped() (which checks dev->iommu_group) because when
+ * both group and cdev interfaces coexist, the group path assigns a fake
+ * noiommu iommu_group to the device. That would cause device_iommu_mapped()
+ * to return true and hide the noiommu case from the cdev path. dev->iommu is
+ * reliably NULL when no IOMMU driver is managing the device.
+ */
+static bool iommufd_device_is_noiommu(struct iommufd_device *idev)
+{
+	return IS_ENABLED(CONFIG_IOMMUFD_NOIOMMU) && !idev->dev->iommu;
+}
+
 static void iommufd_group_release(struct kref *kref)
 {
 	struct iommufd_group *igroup =
@@ -30,9 +43,11 @@ static void iommufd_group_release(struct kref *kref)
 
 	WARN_ON(!xa_empty(&igroup->pasid_attach));
 
-	xa_cmpxchg(&igroup->ictx->groups, iommu_group_id(igroup->group), igroup,
-		   NULL, GFP_KERNEL);
-	iommu_group_put(igroup->group);
+	if (igroup->group) {
+		xa_cmpxchg(&igroup->ictx->groups, iommu_group_id(igroup->group),
+			   igroup, NULL, GFP_KERNEL);
+		iommu_group_put(igroup->group);
+	}
 	mutex_destroy(&igroup->lock);
 	kfree(igroup);
 }
@@ -56,6 +71,30 @@ static bool iommufd_group_try_get(struct iommufd_group *igroup,
 	return kref_get_unless_zero(&igroup->ref);
 }
 
+static struct iommufd_group *iommufd_alloc_group(struct iommufd_ctx *ictx,
+						 struct iommu_group *group)
+{
+	struct iommufd_group *new_igroup;
+
+	new_igroup = kzalloc_obj(*new_igroup, GFP_KERNEL);
+	if (!new_igroup)
+		return ERR_PTR(-ENOMEM);
+
+	kref_init(&new_igroup->ref);
+	mutex_init(&new_igroup->lock);
+	xa_init(&new_igroup->pasid_attach);
+	new_igroup->sw_msi_start = PHYS_ADDR_MAX;
+	/* group reference moves into new_igroup */
+	new_igroup->group = group;
+
+	/*
+	 * The ictx is not additionally refcounted here because all objects using
+	 * an igroup must put it before their destroy completes.
+	 */
+	new_igroup->ictx = ictx;
+	return new_igroup;
+}
+
 /*
  * iommufd needs to store some more data for each iommu_group, we keep a
  * parallel xarray indexed by iommu_group id to hold this instead of putting it
@@ -87,25 +126,12 @@ static struct iommufd_group *iommufd_get_group(struct iommufd_ctx *ictx,
 	}
 	xa_unlock(&ictx->groups);
 
-	new_igroup = kzalloc_obj(*new_igroup);
-	if (!new_igroup) {
+	new_igroup = iommufd_alloc_group(ictx, group);
+	if (IS_ERR(new_igroup)) {
 		iommu_group_put(group);
-		return ERR_PTR(-ENOMEM);
+		return new_igroup;
 	}
 
-	kref_init(&new_igroup->ref);
-	mutex_init(&new_igroup->lock);
-	xa_init(&new_igroup->pasid_attach);
-	new_igroup->sw_msi_start = PHYS_ADDR_MAX;
-	/* group reference moves into new_igroup */
-	new_igroup->group = group;
-
-	/*
-	 * The ictx is not additionally refcounted here becase all objects using
-	 * an igroup must put it before their destroy completes.
-	 */
-	new_igroup->ictx = ictx;
-
 	/*
 	 * We dropped the lock so igroup is invalid. NULL is a safe and likely
 	 * value to assume for the xa_cmpxchg algorithm.
@@ -148,29 +174,22 @@ static void iommufd_device_remove_vdev(struct iommufd_device *idev)
 	if (!idev->vdev)
 		goto out_unlock;
 
-	vdev = iommufd_get_vdevice(idev->ictx, idev->vdev->obj.id);
+	vdev = idev->vdev;
+
 	/*
 	 * An ongoing vdev destroy ioctl has removed the vdev from the object
 	 * xarray, but has not finished iommufd_vdevice_destroy() yet as it
 	 * needs the same mutex. We exit the locking then wait on wait_cnt
 	 * reference for the vdev destruction.
 	 */
-	if (IS_ERR(vdev))
+	if (iommufd_try_inc_users(idev->ictx, &vdev->obj))
 		goto out_unlock;
 
-	/* Should never happen */
-	if (WARN_ON(vdev != idev->vdev)) {
-		iommufd_put_object(idev->ictx, &vdev->obj);
-		goto out_unlock;
-	}
-
 	/*
 	 * vdev is still alive. Hold a users refcount to prevent racing with
 	 * userspace destruction, then use iommufd_object_tombstone_user() to
 	 * destroy it and leave a tombstone.
 	 */
-	refcount_inc(&vdev->obj.users);
-	iommufd_put_object(idev->ictx, &vdev->obj);
 	mutex_unlock(&idev->igroup->lock);
 	iommufd_object_tombstone_user(idev->ictx, &vdev->obj);
 	return;
@@ -193,32 +212,20 @@ void iommufd_device_destroy(struct iommufd_object *obj)
 	struct iommufd_device *idev =
 		container_of(obj, struct iommufd_device, obj);
 
-	iommu_device_release_dma_owner(idev->dev);
+	/* igroup is NULL when destroy called during bind error cleanup */
+	if (!idev->igroup)
+		return;
+	if (!iommufd_device_is_noiommu(idev))
+		iommu_device_release_dma_owner(idev->dev);
 	iommufd_put_group(idev->igroup);
 	if (!iommufd_selftest_is_mock_dev(idev->dev))
 		iommufd_ctx_put(idev->ictx);
 }
 
-/**
- * iommufd_device_bind - Bind a physical device to an iommu fd
- * @ictx: iommufd file descriptor
- * @dev: Pointer to a physical device struct
- * @id: Output ID number to return to userspace for this device
- *
- * A successful bind establishes an ownership over the device and returns
- * struct iommufd_device pointer, otherwise returns error pointer.
- *
- * A driver using this API must set driver_managed_dma and must not touch
- * the device until this routine succeeds and establishes ownership.
- *
- * Binding a PCI device places the entire RID under iommufd control.
- *
- * The caller must undo this with iommufd_device_unbind()
- */
-struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx,
-					   struct device *dev, u32 *id)
+static int iommufd_bind_iommu(struct iommufd_device *idev)
 {
-	struct iommufd_device *idev;
+	struct iommufd_ctx *ictx = idev->ictx;
+	struct device *dev = idev->dev;
 	struct iommufd_group *igroup;
 	int rc;
 
@@ -227,11 +234,11 @@ struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx,
 	 * to restore cache coherency.
 	 */
 	if (!device_iommu_capable(dev, IOMMU_CAP_CACHE_COHERENCY))
-		return ERR_PTR(-EINVAL);
+		return -EINVAL;
 
 	igroup = iommufd_get_group(ictx, dev);
 	if (IS_ERR(igroup))
-		return ERR_CAST(igroup);
+		return PTR_ERR(igroup);
 
 	/*
 	 * For historical compat with VFIO the insecure interrupt path is
@@ -257,21 +264,77 @@ struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx,
 	if (rc)
 		goto out_group_put;
 
+	/* igroup refcount moves into iommufd_device */
+	idev->igroup = igroup;
+	idev->enforce_cache_coherency =
+		device_iommu_capable(dev, IOMMU_CAP_ENFORCE_CACHE_COHERENCY);
+	return 0;
+
+out_group_put:
+	iommufd_put_group(igroup);
+	return rc;
+}
+
+/*
+ * Noiommu devices have no real IOMMU group. Create a dummy igroup so that
+ * internal code paths that expect idev->igroup to be present still work.
+ * A NULL igroup->group distinguishes this from a real IOMMU-backed group.
+ */
+static int iommufd_bind_noiommu(struct iommufd_device *idev)
+{
+	struct iommufd_group *igroup;
+
+	igroup = iommufd_alloc_group(idev->ictx, NULL);
+	if (IS_ERR(igroup))
+		return PTR_ERR(igroup);
+	idev->igroup = igroup;
+	return 0;
+}
+
+/**
+ * iommufd_device_bind - Bind a physical device to an iommu fd
+ * @ictx: iommufd file descriptor
+ * @dev: Pointer to a physical device struct
+ * @id: Output ID number to return to userspace for this device
+ *
+ * A successful bind establishes an ownership over the device and returns
+ * struct iommufd_device pointer, otherwise returns error pointer.
+ *
+ * A driver using this API must set driver_managed_dma and must not touch
+ * the device until this routine succeeds and establishes ownership.
+ *
+ * Binding a PCI device places the entire RID under iommufd control.
+ *
+ * The caller must undo this with iommufd_device_unbind()
+ */
+struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx,
+					   struct device *dev, u32 *id)
+{
+	struct iommufd_device *idev;
+	int rc;
+
 	idev = iommufd_object_alloc(ictx, idev, IOMMUFD_OBJ_DEVICE);
-	if (IS_ERR(idev)) {
-		rc = PTR_ERR(idev);
-		goto out_release_owner;
-	}
+	if (IS_ERR(idev))
+		return idev;
+
 	idev->ictx = ictx;
+	idev->dev = dev;
+
+	if (!iommufd_device_is_noiommu(idev))
+		rc = iommufd_bind_iommu(idev);
+	else
+		rc = iommufd_bind_noiommu(idev);
+	if (rc)
+		goto err_out;
+
+	/*
+	 * Take a ctx reference after bind succeeds. This must happen here
+	 * so that iommufd_device_destroy() can handle partial initialization
+	 */
 	if (!iommufd_selftest_is_mock_dev(dev))
 		iommufd_ctx_get(ictx);
-	idev->dev = dev;
-	idev->enforce_cache_coherency =
-		device_iommu_capable(dev, IOMMU_CAP_ENFORCE_CACHE_COHERENCY);
 	/* The calling driver is a user until iommufd_device_unbind() */
 	refcount_inc(&idev->obj.users);
-	/* igroup refcount moves into iommufd_device */
-	idev->igroup = igroup;
 
 	/*
 	 * If the caller fails after this success it must call
@@ -283,11 +346,14 @@ struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx,
 	*id = idev->obj.id;
 	return idev;
 
-out_release_owner:
-	iommu_device_release_dma_owner(dev);
-out_group_put:
-	iommufd_put_group(igroup);
+err_out:
+	/*
+	 * iommufd_device_destroy() handles partially initialized idev,
+	 * so iommufd_object_abort_and_destroy() is safe to call here.
+	 */
+	iommufd_object_abort_and_destroy(ictx, &idev->obj);
 	return ERR_PTR(rc);
+
 }
 EXPORT_SYMBOL_NS_GPL(iommufd_device_bind, "IOMMUFD");
 
@@ -508,6 +574,9 @@ static int iommufd_hwpt_attach_device(struct iommufd_hw_pagetable *hwpt,
 	if (rc)
 		return rc;
 
+	if (iommufd_device_is_noiommu(idev))
+		return 0;
+
 	handle = kzalloc_obj(*handle);
 	if (!handle)
 		return -ENOMEM;
@@ -548,6 +617,9 @@ static void iommufd_hwpt_detach_device(struct iommufd_hw_pagetable *hwpt,
 {
 	struct iommufd_attach_handle *handle;
 
+	if (iommufd_device_is_noiommu(idev))
+		return;
+
 	handle = iommufd_device_get_attach_handle(idev, pasid);
 	if (pasid == IOMMU_NO_PASID)
 		iommu_detach_group_handle(hwpt->domain, idev->igroup->group);
@@ -573,6 +645,9 @@ static int iommufd_hwpt_replace_device(struct iommufd_device *idev,
 	if (rc)
 		return rc;
 
+	if (iommufd_device_is_noiommu(idev))
+		return 0;
+
 	old_handle = iommufd_device_get_attach_handle(idev, pasid);
 
 	handle = kzalloc_obj(*handle);
@@ -589,7 +664,7 @@ static int iommufd_hwpt_replace_device(struct iommufd_device *idev,
 	if (rc)
 		goto out_free_handle;
 
-	iommufd_auto_response_faults(hwpt, old_handle);
+	iommufd_auto_response_faults(old, old_handle);
 	kfree(old_handle);
 
 	return 0;
@@ -603,7 +678,8 @@ int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt,
 				struct iommufd_device *idev, ioasid_t pasid)
 {
 	struct iommufd_hwpt_paging *hwpt_paging = find_hwpt_paging(hwpt);
-	bool attach_resv = hwpt_paging && pasid == IOMMU_NO_PASID;
+	bool attach_resv = hwpt_paging && pasid == IOMMU_NO_PASID &&
+			   !iommufd_device_is_noiommu(idev);
 	struct iommufd_group *igroup = idev->igroup;
 	struct iommufd_hw_pagetable *old_hwpt;
 	struct iommufd_attach *attach;
@@ -782,7 +858,8 @@ iommufd_device_do_replace(struct iommufd_device *idev, ioasid_t pasid,
 			  struct iommufd_hw_pagetable *hwpt)
 {
 	struct iommufd_hwpt_paging *hwpt_paging = find_hwpt_paging(hwpt);
-	bool attach_resv = hwpt_paging && pasid == IOMMU_NO_PASID;
+	bool attach_resv = hwpt_paging && pasid == IOMMU_NO_PASID &&
+			   !iommufd_device_is_noiommu(idev);
 	struct iommufd_hwpt_paging *old_hwpt_paging;
 	struct iommufd_group *igroup = idev->igroup;
 	struct iommufd_hw_pagetable *old_hwpt;
@@ -989,7 +1066,7 @@ static int iommufd_device_change_pt(struct iommufd_device *idev,
  * iommufd_device_attach - Connect a device/pasid to an iommu_domain
  * @idev: device to attach
  * @pasid: pasid to attach
- * @pt_id: Input a IOMMUFD_OBJ_IOAS, or IOMMUFD_OBJ_HWPT_PAGING
+ * @pt_id: Input an IOMMUFD_OBJ_IOAS, or IOMMUFD_OBJ_HWPT_PAGING
  *         Output the IOMMUFD_OBJ_HWPT_PAGING ID
  *
  * This connects the device/pasid to an iommu_domain, either automatically
@@ -1022,7 +1099,7 @@ EXPORT_SYMBOL_NS_GPL(iommufd_device_attach, "IOMMUFD");
  * iommufd_device_replace - Change the device/pasid's iommu_domain
  * @idev: device to change
  * @pasid: pasid to change
- * @pt_id: Input a IOMMUFD_OBJ_IOAS, or IOMMUFD_OBJ_HWPT_PAGING
+ * @pt_id: Input an IOMMUFD_OBJ_IOAS, or IOMMUFD_OBJ_HWPT_PAGING
  *         Output the IOMMUFD_OBJ_HWPT_PAGING ID
  *
  * This is the same as::
@@ -1044,7 +1121,7 @@ int iommufd_device_replace(struct iommufd_device *idev, ioasid_t pasid,
 EXPORT_SYMBOL_NS_GPL(iommufd_device_replace, "IOMMUFD");
 
 /**
- * iommufd_device_detach - Disconnect a device/device to an iommu_domain
+ * iommufd_device_detach - Disconnect a device/pasid from an iommu_domain
  * @idev: device to detach
  * @pasid: pasid to detach
  *
@@ -1574,6 +1651,11 @@ int iommufd_get_hw_info(struct iommufd_ucmd *ucmd)
 	if (IS_ERR(idev))
 		return PTR_ERR(idev);
 
+	if (iommufd_device_is_noiommu(idev)) {
+		rc = -EOPNOTSUPP;
+		goto out_put;
+	}
+
 	ops = dev_iommu_ops(idev->dev);
 	if (ops->hw_info) {
 		data = ops->hw_info(idev->dev, &data_len, &cmd->out_data_type);
diff --git a/drivers/iommu/iommufd/hw_pagetable.c b/drivers/iommu/iommufd/hw_pagetable.c
index 623cc608ca0cd..ef6e119c2a752 100644
--- a/drivers/iommu/iommufd/hw_pagetable.c
+++ b/drivers/iommu/iommufd/hw_pagetable.c
@@ -8,6 +8,15 @@
 #include "../iommu-priv.h"
 #include "iommufd_private.h"
 
+static const struct iommu_ops *get_iommu_ops(struct iommufd_device *idev)
+{
+	if (IS_ENABLED(CONFIG_IOMMUFD_NOIOMMU) && !idev->igroup->group)
+		return &iommufd_noiommu_ops;
+	if (WARN_ON_ONCE(!idev->dev->iommu))
+		return NULL;
+	return dev_iommu_ops(idev->dev);
+}
+
 static void __iommufd_hwpt_destroy(struct iommufd_hw_pagetable *hwpt)
 {
 	if (hwpt->domain)
@@ -114,11 +123,13 @@ iommufd_hwpt_paging_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
 				IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
 				IOMMU_HWPT_FAULT_ID_VALID |
 				IOMMU_HWPT_ALLOC_PASID;
-	const struct iommu_ops *ops = dev_iommu_ops(idev->dev);
+	const struct iommu_ops *ops = get_iommu_ops(idev);
 	struct iommufd_hwpt_paging *hwpt_paging;
 	struct iommufd_hw_pagetable *hwpt;
 	int rc;
 
+	if (!ops)
+		return ERR_PTR(-ENODEV);
 	lockdep_assert_held(&ioas->mutex);
 
 	if ((flags || user_data) && !ops->domain_alloc_paging_flags)
@@ -229,7 +240,7 @@ iommufd_hwpt_nested_alloc(struct iommufd_ctx *ictx,
 			  struct iommufd_device *idev, u32 flags,
 			  const struct iommu_user_data *user_data)
 {
-	const struct iommu_ops *ops = dev_iommu_ops(idev->dev);
+	const struct iommu_ops *ops = get_iommu_ops(idev);
 	struct iommufd_hwpt_nested *hwpt_nested;
 	struct iommufd_hw_pagetable *hwpt;
 	int rc;
@@ -389,10 +400,12 @@ int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd)
 		hwpt = &hwpt_nested->common;
 	} else if (pt_obj->type == IOMMUFD_OBJ_VIOMMU) {
 		struct iommufd_hwpt_nested *hwpt_nested;
+		struct iommu_device *iommu_dev;
 		struct iommufd_viommu *viommu;
 
 		viommu = container_of(pt_obj, struct iommufd_viommu, obj);
-		if (viommu->iommu_dev != __iommu_get_iommu_dev(idev->dev)) {
+		iommu_dev = iommufd_device_get_iommu_dev(idev);
+		if (!iommu_dev || viommu->iommu_dev != iommu_dev) {
 			rc = -EINVAL;
 			goto out_unlock;
 		}
diff --git a/drivers/iommu/iommufd/hwpt_noiommu.c b/drivers/iommu/iommufd/hwpt_noiommu.c
new file mode 100644
index 0000000000000..9b8b5eb714913
--- /dev/null
+++ b/drivers/iommu/iommufd/hwpt_noiommu.c
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES
+ */
+#include <linux/generic_pt/iommu.h>
+#include <linux/iommu.h>
+#include "../iommu-pages.h"
+#include "iommufd_private.h"
+
+static const struct iommu_domain_ops noiommu_amdv1_ops;
+
+struct noiommu_domain {
+	union {
+		struct iommu_domain domain;
+		struct pt_iommu_amdv1 amdv1;
+	};
+	spinlock_t lock;
+};
+PT_IOMMU_CHECK_DOMAIN(struct noiommu_domain, amdv1.iommu, domain);
+
+static void noiommu_change_top(struct pt_iommu *iommu_table,
+			       phys_addr_t top_paddr, unsigned int top_level)
+{
+}
+
+static spinlock_t *noiommu_get_top_lock(struct pt_iommu *iommupt)
+{
+	struct noiommu_domain *domain =
+		container_of(iommupt, struct noiommu_domain, amdv1.iommu);
+
+	return &domain->lock;
+}
+
+static const struct pt_iommu_driver_ops noiommu_driver_ops = {
+	.get_top_lock = noiommu_get_top_lock,
+	.change_top = noiommu_change_top,
+};
+
+static struct iommu_domain *
+noiommu_alloc_paging_flags(struct device *dev, u32 flags,
+			   const struct iommu_user_data *user_data)
+{
+	struct pt_iommu_amdv1_cfg cfg = {};
+	struct noiommu_domain *dom;
+	int rc;
+
+	if (flags || user_data)
+		return ERR_PTR(-EOPNOTSUPP);
+
+	cfg.common.hw_max_vasz_lg2 = 64;
+	cfg.common.hw_max_oasz_lg2 = 52;
+	cfg.starting_level = 2;
+	cfg.common.features =
+		(BIT(PT_FEAT_DYNAMIC_TOP) | BIT(PT_FEAT_AMDV1_ENCRYPT_TABLES) |
+		 BIT(PT_FEAT_AMDV1_FORCE_COHERENCE));
+
+	dom = kzalloc(sizeof(*dom), GFP_KERNEL);
+	if (!dom)
+		return ERR_PTR(-ENOMEM);
+
+	spin_lock_init(&dom->lock);
+	dom->amdv1.iommu.nid = NUMA_NO_NODE;
+	dom->amdv1.iommu.driver_ops = &noiommu_driver_ops;
+	dom->domain.ops = &noiommu_amdv1_ops;
+
+	/* Use SW-only page table which is based on AMDV1 */
+	rc = pt_iommu_amdv1_init(&dom->amdv1, &cfg, GFP_KERNEL);
+	if (rc) {
+		kfree(dom);
+		return ERR_PTR(rc);
+	}
+
+	return &dom->domain;
+}
+
+static void noiommu_domain_free(struct iommu_domain *iommu_domain)
+{
+	struct noiommu_domain *domain =
+		container_of(iommu_domain, struct noiommu_domain, domain);
+
+	pt_iommu_deinit(&domain->amdv1.iommu);
+	kfree(domain);
+}
+
+static void noiommu_iotlb_sync(struct iommu_domain *domain,
+			       struct iommu_iotlb_gather *gather)
+{
+	iommu_put_pages_list(&gather->freelist);
+}
+
+/*
+ * Domain ops for iommufd no-IOMMU mode. Uses AMDV1 format as a
+ * SW-only IOPT because it has the best multi-page size options
+ * of all the formats. IOVAs serve only for IOVA-to-PA lookups,
+ * not for hardware DMA translation.
+ */
+static const struct iommu_domain_ops noiommu_amdv1_ops = {
+	IOMMU_PT_DOMAIN_OPS(amdv1),
+	.iotlb_sync = noiommu_iotlb_sync,
+	.free = noiommu_domain_free,
+};
+
+const struct iommu_ops iommufd_noiommu_ops = {
+	.domain_alloc_paging_flags = noiommu_alloc_paging_flags,
+};
diff --git a/drivers/iommu/iommufd/io_pagetable.c b/drivers/iommu/iommufd/io_pagetable.c
index 24d4917105d94..4e447ce74cf6c 100644
--- a/drivers/iommu/iommufd/io_pagetable.c
+++ b/drivers/iommu/iommufd/io_pagetable.c
@@ -859,6 +859,84 @@ int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova,
 	return iopt_unmap_iova_range(iopt, iova, iova_last, unmapped);
 }
 
+#ifdef CONFIG_IOMMUFD_NOIOMMU
+int iopt_get_phys(struct io_pagetable *iopt, unsigned long iova, u64 *paddr,
+		  u64 *length)
+{
+	struct iopt_area *area;
+	struct iopt_pages *pages;
+	u64 max_length = *length;
+	u64 tmp_length = 0;
+	u64 tmp_paddr = 0;
+	int rc = 0;
+
+	if (!max_length)
+		return -EINVAL;
+
+	down_read(&iopt->iova_rwsem);
+	area = iopt_area_iter_first(iopt, iova, iova);
+	if (!area || !area->pages) {
+		rc = -ENOENT;
+		goto unlock_exit;
+	}
+
+	pages = area->pages;
+	mutex_lock(&pages->mutex);
+	if (iopt_dmabuf_revoked(pages)) {
+		rc = -EINVAL;
+		goto unlock_pages;
+	}
+
+	if (!area->storage_domain ||
+	    area->storage_domain->owner != &iommufd_noiommu_ops) {
+		rc = -EOPNOTSUPP;
+		goto unlock_pages;
+	}
+
+	*paddr = iommu_iova_to_phys(area->storage_domain, iova);
+	tmp_length = min_t(u64, PAGE_SIZE - offset_in_page(iova),
+			   iopt_area_last_iova(area) - iova + 1);
+	tmp_paddr = *paddr;
+	/*
+	 * Return the physically contiguous length, capped by the caller
+	 * supplied range length.
+	 */
+	while (iova < iopt_area_last_iova(area)) {
+		unsigned long next_iova;
+		u64 next_paddr;
+
+		if (tmp_length >= max_length)
+			break;
+
+		if (check_add_overflow(iova, PAGE_SIZE, &next_iova))
+			break;
+
+		if (next_iova > iopt_area_last_iova(area))
+			break;
+
+		next_paddr = iommu_iova_to_phys(area->storage_domain, next_iova);
+
+		if (!next_paddr || next_paddr != tmp_paddr + PAGE_SIZE)
+			break;
+
+		iova = next_iova;
+		tmp_paddr += PAGE_SIZE;
+		tmp_length += PAGE_SIZE;
+	}
+
+	if (tmp_length > max_length)
+		tmp_length = max_length;
+	*length = tmp_length;
+
+unlock_pages:
+	mutex_unlock(&pages->mutex);
+unlock_exit:
+	up_read(&iopt->iova_rwsem);
+
+	return rc;
+}
+#endif
+
 int iopt_unmap_all(struct io_pagetable *iopt, unsigned long *unmapped)
 {
 	/* If the IOVAs are empty then unmap all succeeds */
@@ -1513,7 +1591,7 @@ int iopt_table_enforce_dev_resv_regions(struct io_pagetable *iopt,
 		return -EINVAL;
 
 	down_write(&iopt->iova_rwsem);
-	/* FIXME: drivers allocate memory but there is no failure propogated */
+	/* FIXME: drivers allocate memory but there is no failure propagated */
 	iommu_get_resv_regions(dev, &resv_regions);
 
 	list_for_each_entry(resv, &resv_regions, list) {
diff --git a/drivers/iommu/iommufd/ioas.c b/drivers/iommu/iommufd/ioas.c
index fed06c2b728e2..1a18cee7456bc 100644
--- a/drivers/iommu/iommufd/ioas.c
+++ b/drivers/iommu/iommufd/ioas.c
@@ -375,6 +375,42 @@ int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd)
 	return rc;
 }
 
+#ifdef CONFIG_IOMMUFD_NOIOMMU
+int iommufd_ioas_noiommu_get_pa(struct iommufd_ucmd *ucmd)
+{
+	struct iommu_ioas_noiommu_get_pa *cmd = ucmd->cmd;
+	struct iommufd_ioas *ioas;
+	int rc;
+
+	if (!capable(CAP_SYS_RAWIO))
+		return -EPERM;
+
+	if (cmd->flags || cmd->__reserved)
+		return -EOPNOTSUPP;
+
+	if (!cmd->length)
+		return -EINVAL;
+
+	if (cmd->iova >= ULONG_MAX)
+		return -EOVERFLOW;
+
+	ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id);
+	if (IS_ERR(ioas))
+		return PTR_ERR(ioas);
+
+	rc = iopt_get_phys(&ioas->iopt, cmd->iova, &cmd->out_phys,
+			   &cmd->length);
+	if (rc)
+		goto out_put;
+
+	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
+out_put:
+	iommufd_put_object(ucmd->ictx, &ioas->obj);
+
+	return rc;
+}
+#endif
+
 static void iommufd_release_all_iova_rwsem(struct iommufd_ctx *ictx,
 					   struct xarray *ioas_list)
 {
diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h
index 43fbc5bed8de3..eb2e85b27e429 100644
--- a/drivers/iommu/iommufd/iommufd_private.h
+++ b/drivers/iommu/iommufd/iommufd_private.h
@@ -118,6 +118,16 @@ int iopt_map_pages(struct io_pagetable *iopt, struct list_head *pages_list,
 int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova,
 		    unsigned long length, unsigned long *unmapped);
 int iopt_unmap_all(struct io_pagetable *iopt, unsigned long *unmapped);
+#ifdef CONFIG_IOMMUFD_NOIOMMU
+int iopt_get_phys(struct io_pagetable *iopt, unsigned long iova, u64 *paddr,
+		  u64 *length);
+#else
+static inline int iopt_get_phys(struct io_pagetable *iopt, unsigned long iova,
+				u64 *paddr, u64 *length)
+{
+	return -EOPNOTSUPP;
+}
+#endif
 
 int iopt_read_and_clear_dirty_data(struct io_pagetable *iopt,
 				   struct iommu_domain *domain,
@@ -182,6 +192,7 @@ static inline bool iommufd_lock_obj(struct iommufd_object *obj)
 	return true;
 }
 
+int iommufd_try_inc_users(struct iommufd_ctx *ictx, struct iommufd_object *obj);
 struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id,
 					  enum iommufd_object_type type);
 static inline void iommufd_put_object(struct iommufd_ctx *ictx,
@@ -346,6 +357,14 @@ int iommufd_ioas_map_file(struct iommufd_ucmd *ucmd);
 int iommufd_ioas_change_process(struct iommufd_ucmd *ucmd);
 int iommufd_ioas_copy(struct iommufd_ucmd *ucmd);
 int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd);
+#ifdef CONFIG_IOMMUFD_NOIOMMU
+int iommufd_ioas_noiommu_get_pa(struct iommufd_ucmd *ucmd);
+#else
+static inline int iommufd_ioas_noiommu_get_pa(struct iommufd_ucmd *ucmd)
+{
+	return -EOPNOTSUPP;
+}
+#endif
 int iommufd_ioas_option(struct iommufd_ucmd *ucmd);
 int iommufd_option_rlimit_mode(struct iommu_option *cmd,
 			       struct iommufd_ctx *ictx);
@@ -464,6 +483,8 @@ static inline void iommufd_hw_pagetable_put(struct iommufd_ctx *ictx,
 	refcount_dec(&hwpt->obj.users);
 }
 
+extern const struct iommu_ops iommufd_noiommu_ops;
+
 struct iommufd_attach;
 
 struct iommufd_group {
@@ -501,6 +522,16 @@ iommufd_get_device(struct iommufd_ucmd *ucmd, u32 id)
 			    struct iommufd_device, obj);
 }
 
+static inline struct iommu_device *
+iommufd_device_get_iommu_dev(struct iommufd_device *idev)
+{
+	if (IS_ENABLED(CONFIG_IOMMUFD_NOIOMMU) && !idev->igroup->group)
+		return NULL;
+	if (WARN_ON_ONCE(!idev->dev->iommu))
+		return NULL;
+	return __iommu_get_iommu_dev(idev->dev);
+}
+
 void iommufd_device_pre_destroy(struct iommufd_object *obj);
 void iommufd_device_destroy(struct iommufd_object *obj);
 int iommufd_get_hw_info(struct iommufd_ucmd *ucmd);
@@ -698,14 +729,6 @@ void iommufd_vdevice_abort(struct iommufd_object *obj);
 int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd);
 void iommufd_hw_queue_destroy(struct iommufd_object *obj);
 
-static inline struct iommufd_vdevice *
-iommufd_get_vdevice(struct iommufd_ctx *ictx, u32 id)
-{
-	return container_of(iommufd_get_object(ictx, id,
-					       IOMMUFD_OBJ_VDEVICE),
-			    struct iommufd_vdevice, obj);
-}
-
 #ifdef CONFIG_IOMMUFD_TEST
 int iommufd_test(struct iommufd_ucmd *ucmd);
 void iommufd_selftest_destroy(struct iommufd_object *obj);
diff --git a/drivers/iommu/iommufd/main.c b/drivers/iommu/iommufd/main.c
index 8c6d43601afbe..9a921b1531620 100644
--- a/drivers/iommu/iommufd/main.c
+++ b/drivers/iommu/iommufd/main.c
@@ -180,6 +180,26 @@ struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id,
 	return obj;
 }
 
+/*
+ * Increment the users count of an object outside the context of an ioctl that
+ * has already locked it. The users refcount cannot be increased on an already
+ * created object unless the object is installed in the xarray, otherwise things
+ * are racing with a parallel destruction.
+ */
+int iommufd_try_inc_users(struct iommufd_ctx *ictx, struct iommufd_object *obj)
+{
+	struct iommufd_object *cur;
+
+	xa_lock(&ictx->objects);
+	cur = xa_load(&ictx->objects, obj->id);
+	if (cur == obj)
+		refcount_inc(&obj->users);
+	xa_unlock(&ictx->objects);
+	if (cur != obj)
+		return -EBUSY;
+	return 0;
+}
+
 static int iommufd_object_dec_wait(struct iommufd_ctx *ictx,
 				   struct iommufd_object *to_destroy)
 {
@@ -424,6 +444,7 @@ union ucmd_buffer {
 	struct iommu_ioas_alloc alloc;
 	struct iommu_ioas_allow_iovas allow_iovas;
 	struct iommu_ioas_copy ioas_copy;
+	struct iommu_ioas_noiommu_get_pa noiommu_get_pa;
 	struct iommu_ioas_iova_ranges iova_ranges;
 	struct iommu_ioas_map map;
 	struct iommu_ioas_unmap unmap;
@@ -482,6 +503,8 @@ static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = {
 	IOCTL_OP(IOMMU_IOAS_MAP, iommufd_ioas_map, struct iommu_ioas_map, iova),
 	IOCTL_OP(IOMMU_IOAS_MAP_FILE, iommufd_ioas_map_file,
 		 struct iommu_ioas_map_file, iova),
+	IOCTL_OP(IOMMU_IOAS_NOIOMMU_GET_PA, iommufd_ioas_noiommu_get_pa, struct iommu_ioas_noiommu_get_pa,
+		 out_phys),
 	IOCTL_OP(IOMMU_IOAS_UNMAP, iommufd_ioas_unmap, struct iommu_ioas_unmap,
 		 length),
 	IOCTL_OP(IOMMU_OPTION, iommufd_option, struct iommu_option, val64),
@@ -804,5 +827,6 @@ MODULE_ALIAS("devname:vfio/vfio");
 MODULE_IMPORT_NS("IOMMUFD_INTERNAL");
 MODULE_IMPORT_NS("IOMMUFD");
 MODULE_IMPORT_NS("DMA_BUF");
+MODULE_IMPORT_NS("GENERIC_PT_IOMMU");
 MODULE_DESCRIPTION("I/O Address Space Management for passthrough devices");
 MODULE_LICENSE("GPL");
diff --git a/drivers/iommu/iommufd/pages.c b/drivers/iommu/iommufd/pages.c
index 03c8379bbc347..404f31d8f7291 100644
--- a/drivers/iommu/iommufd/pages.c
+++ b/drivers/iommu/iommufd/pages.c
@@ -2451,6 +2451,9 @@ int iopt_area_add_access(struct iopt_area *area, unsigned long start_index,
 	if ((flags & IOMMUFD_ACCESS_RW_WRITE) && !pages->writable)
 		return -EPERM;
 
+	if (iopt_is_dmabuf(pages))
+		return -EINVAL;
+
 	mutex_lock(&pages->mutex);
 	access = iopt_pages_get_exact_access(pages, start_index, last_index);
 	if (access) {
diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c
index af07c642a5260..b021fc6bc5f5f 100644
--- a/drivers/iommu/iommufd/selftest.c
+++ b/drivers/iommu/iommufd/selftest.c
@@ -12,6 +12,7 @@
 #include <linux/iommu.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
+#include <linux/sizes.h>
 #include <linux/xarray.h>
 #include <uapi/linux/iommufd.h>
 #include <linux/generic_pt/iommu.h>
@@ -1695,6 +1696,9 @@ static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id,
 	if (!page_size || !length || iova % page_size || length % page_size ||
 	    !uptr)
 		return -EINVAL;
+	max = length / page_size;
+	if (max > SZ_16M * BITS_PER_BYTE)
+		return -EOVERFLOW;
 
 	hwpt = get_md_pagetable(ucmd, mockpt_id, &mock);
 	if (IS_ERR(hwpt))
@@ -1705,7 +1709,6 @@ static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id,
 		goto out_put;
 	}
 
-	max = length / page_size;
 	tmp = kvzalloc(DIV_ROUND_UP(max, BITS_PER_LONG) * sizeof(unsigned long),
 		       GFP_KERNEL_ACCOUNT);
 	if (!tmp) {
@@ -2023,7 +2026,12 @@ static int iommufd_test_dmabuf_get(struct iommufd_ucmd *ucmd,
 		goto err_free;
 	}
 
-	return dma_buf_fd(dmabuf, open_flags);
+	rc = dma_buf_fd(dmabuf, open_flags);
+	if (rc < 0) {
+		dma_buf_put(dmabuf);
+		return rc;
+	}
+	return 0;
 
 err_free:
 	kfree(priv->memory);
diff --git a/drivers/iommu/iommufd/viommu.c b/drivers/iommu/iommufd/viommu.c
index 4081deda9b33d..f7951057a1e56 100644
--- a/drivers/iommu/iommufd/viommu.c
+++ b/drivers/iommu/iommufd/viommu.c
@@ -25,6 +25,7 @@ int iommufd_viommu_alloc_ioctl(struct iommufd_ucmd *ucmd)
 	struct iommufd_hwpt_paging *hwpt_paging;
 	struct iommufd_viommu *viommu;
 	struct iommufd_device *idev;
+	struct iommu_device *iommu_dev;
 	const struct iommu_ops *ops;
 	size_t viommu_size;
 	int rc;
@@ -36,7 +37,12 @@ int iommufd_viommu_alloc_ioctl(struct iommufd_ucmd *ucmd)
 	if (IS_ERR(idev))
 		return PTR_ERR(idev);
 
-	ops = dev_iommu_ops(idev->dev);
+	iommu_dev = iommufd_device_get_iommu_dev(idev);
+	if (!iommu_dev) {
+		rc = -EOPNOTSUPP;
+		goto out_put_idev;
+	}
+	ops = iommu_dev->ops;
 	if (!ops->get_viommu_size || !ops->viommu_init) {
 		rc = -EOPNOTSUPP;
 		goto out_put_idev;
@@ -87,7 +93,7 @@ int iommufd_viommu_alloc_ioctl(struct iommufd_ucmd *ucmd)
 	 * pluggable IOMMU instance (if exists) is responsible for refcounting
 	 * on its own.
 	 */
-	viommu->iommu_dev = __iommu_get_iommu_dev(idev->dev);
+	viommu->iommu_dev = iommu_dev;
 
 	rc = ops->viommu_init(viommu, hwpt_paging->common.domain,
 			      user_data.len ? &user_data : NULL);
@@ -143,9 +149,10 @@ void iommufd_vdevice_destroy(struct iommufd_object *obj)
 int iommufd_vdevice_alloc_ioctl(struct iommufd_ucmd *ucmd)
 {
 	struct iommu_vdevice_alloc *cmd = ucmd->cmd;
-	struct iommufd_vdevice *vdev, *curr;
+	struct iommufd_vdevice *vdev;
 	size_t vdev_size = sizeof(*vdev);
 	struct iommufd_viommu *viommu;
+	struct iommu_device *iommu_dev;
 	struct iommufd_device *idev;
 	u64 virt_id = cmd->virt_id;
 	int rc = 0;
@@ -164,7 +171,8 @@ int iommufd_vdevice_alloc_ioctl(struct iommufd_ucmd *ucmd)
 		goto out_put_viommu;
 	}
 
-	if (viommu->iommu_dev != __iommu_get_iommu_dev(idev->dev)) {
+	iommu_dev = iommufd_device_get_iommu_dev(idev);
+	if (!iommu_dev || viommu->iommu_dev != iommu_dev) {
 		rc = -EINVAL;
 		goto out_put_idev;
 	}
@@ -189,7 +197,7 @@ int iommufd_vdevice_alloc_ioctl(struct iommufd_ucmd *ucmd)
 		if (WARN_ON_ONCE(viommu->ops->vdevice_size < vdev_size ||
 				 !viommu->ops->vdevice_init)) {
 			rc = -EOPNOTSUPP;
-			goto out_put_idev;
+			goto out_unlock_igroup;
 		}
 		vdev_size = viommu->ops->vdevice_size;
 	}
@@ -218,18 +226,28 @@ int iommufd_vdevice_alloc_ioctl(struct iommufd_ucmd *ucmd)
 	 */
 	idev->vdev = vdev;
 
-	curr = xa_cmpxchg(&viommu->vdevs, virt_id, NULL, vdev, GFP_KERNEL);
-	if (curr) {
-		rc = xa_err(curr) ?: -EEXIST;
+	/*
+	 * Reserve the slot with a zero entry (reads back as NULL) until the
+	 * vdevice_init() op accepts the vDEVICE. Only the xa_* helpers hide a
+	 * reserved entry, so never use a raw xas_* iterator on this xarray.
+	 */
+	rc = xa_insert(&viommu->vdevs, virt_id, NULL, GFP_KERNEL);
+	if (rc) {
+		if (rc == -EBUSY)
+			rc = -EEXIST;
 		goto out_abort;
 	}
 
 	if (viommu->ops && viommu->ops->vdevice_init) {
 		rc = viommu->ops->vdevice_init(vdev);
-		if (rc)
+		if (rc) {
+			xa_release(&viommu->vdevs, virt_id);
 			goto out_abort;
+		}
 	}
 
+	xa_store(&viommu->vdevs, virt_id, vdev, GFP_KERNEL);
+
 	cmd->out_vdevice_id = vdev->obj.id;
 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
 	if (rc)
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index 0ad5ff431d5b6..708baeb29c036 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -720,7 +720,7 @@ static int msm_iommu_probe(struct platform_device *pdev)
 
 	iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);
 	if (!iommu)
-		return -ENODEV;
+		return -ENOMEM;
 
 	iommu->dev = &pdev->dev;
 	INIT_LIST_HEAD(&iommu->ctx_list);
diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c
index ac97dd2868d4b..e907c99531423 100644
--- a/drivers/iommu/mtk_iommu_v1.c
+++ b/drivers/iommu/mtk_iommu_v1.c
@@ -88,7 +88,7 @@ struct dma_iommu_mapping {
 /* MTK generation one iommu HW only support 4K size mapping */
 #define MT2701_IOMMU_PAGE_SHIFT			12
 #define MT2701_IOMMU_PAGE_SIZE			(1UL << MT2701_IOMMU_PAGE_SHIFT)
-#define MT2701_LARB_NR_MAX			3
+#define MT2701_LARB_NR_MAX			4
 
 /*
  * MTK m4u support 4GB iova address space, and only support 4K page
diff --git a/drivers/iommu/riscv/iommu-pci.c b/drivers/iommu/riscv/iommu-pci.c
index d82d2b00904c7..8abf52dfb4c9b 100644
--- a/drivers/iommu/riscv/iommu-pci.c
+++ b/drivers/iommu/riscv/iommu-pci.c
@@ -109,9 +109,9 @@ static void riscv_iommu_pci_shutdown(struct pci_dev *pdev)
 }
 
 static const struct pci_device_id riscv_iommu_pci_tbl[] = {
-	{PCI_VDEVICE(REDHAT, PCI_DEVICE_ID_REDHAT_RISCV_IOMMU), 0},
-	{PCI_VDEVICE(RIVOS, PCI_DEVICE_ID_RIVOS_RISCV_IOMMU_GA), 0},
-	{0,}
+	{ PCI_VDEVICE(REDHAT, PCI_DEVICE_ID_REDHAT_RISCV_IOMMU) },
+	{ PCI_VDEVICE(RIVOS, PCI_DEVICE_ID_RIVOS_RISCV_IOMMU_GA) },
+	{ }
 };
 
 static struct pci_driver riscv_iommu_pci_driver = {
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index 42f2278a702d0..20b77fbc51ee0 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -212,11 +212,6 @@ config HISILICON_IRQ_MBIGEN
 	select ARM_GIC_V3
 	select ARM_GIC_V3_ITS
 
-config IMGPDC_IRQ
-	bool
-	select GENERIC_IRQ_CHIP
-	select IRQ_DOMAIN
-
 config IXP4XX_IRQ
 	bool
 	select IRQ_DOMAIN
diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
index 74912429c59f2..ab33cccd84719 100644
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -44,7 +44,6 @@ obj-$(CONFIG_ARMADA_370_XP_IRQ)		+= irq-armada-370-xp.o
 obj-$(CONFIG_ATMEL_AIC_IRQ)		+= irq-atmel-aic-common.o irq-atmel-aic.o
 obj-$(CONFIG_ATMEL_AIC5_IRQ)	+= irq-atmel-aic-common.o irq-atmel-aic5.o
 obj-$(CONFIG_I8259)			+= irq-i8259.o
-obj-$(CONFIG_IMGPDC_IRQ)		+= irq-imgpdc.o
 obj-$(CONFIG_IRQ_MIPS_CPU)		+= irq-mips-cpu.o
 obj-$(CONFIG_IXP4XX_IRQ)		+= irq-ixp4xx.o
 obj-$(CONFIG_JCORE_AIC)			+= irq-jcore-aic.o
diff --git a/drivers/irqchip/irq-gic-its-msi-parent.c b/drivers/irqchip/irq-gic-its-msi-parent.c
index b9257103a9997..b2b9d2068bb14 100644
--- a/drivers/irqchip/irq-gic-its-msi-parent.c
+++ b/drivers/irqchip/irq-gic-its-msi-parent.c
@@ -18,7 +18,8 @@
 
 #define ITS_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK |	\
 				 MSI_FLAG_PCI_MSIX      |	\
-				 MSI_FLAG_MULTI_PCI_MSI)
+				 MSI_FLAG_MULTI_PCI_MSI |	\
+				 MSI_FLAG_PCI_MSIX_ALLOC_DYN)
 
 static int its_translate_frame_address(struct fwnode_handle *msi_node, phys_addr_t *pa)
 {
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index 6f5811aae59c1..f2576191c0675 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -4888,10 +4888,22 @@ static bool __maybe_unused its_enable_quirk_hip09_162100801(void *data)
 	return true;
 }
 
-static bool __maybe_unused its_enable_rk3568002(void *data)
+static const char * const dma_32bit_impaired_platforms[] = {
+#ifdef CONFIG_RENESAS_ERRATUM_GEN4GICITS1
+	"renesas,r8a779f0",
+	"renesas,r8a779g0",
+	"renesas,r8a779h0",
+#endif
+#ifdef CONFIG_ROCKCHIP_ERRATUM_3568002
+	"rockchip,rk3566",
+	"rockchip,rk3568",
+#endif
+	NULL,
+};
+
+static bool its_enable_dma32(void *data)
 {
-	if (!of_machine_is_compatible("rockchip,rk3566") &&
-	    !of_machine_is_compatible("rockchip,rk3568"))
+	if (!of_machine_compatible_match(dma_32bit_impaired_platforms))
 		return false;
 
 	gfp_flags_quirk |= GFP_DMA32;
@@ -4966,14 +4978,12 @@ static const struct gic_quirk its_quirks[] = {
 		.property = "dma-noncoherent",
 		.init   = its_set_non_coherent,
 	},
-#ifdef CONFIG_ROCKCHIP_ERRATUM_3568002
 	{
-		.desc   = "ITS: Rockchip erratum RK3568002",
+		.desc   = "ITS: Broken GIC600 integration limited to 32bit PA",
 		.iidr   = 0x0201743b,
 		.mask   = 0xffffffff,
-		.init   = its_enable_rk3568002,
+		.init   = its_enable_dma32,
 	},
-#endif
 	{
 	}
 };
@@ -5320,7 +5330,7 @@ static int __init its_probe_one(struct its_node *its)
 
 	err = its_init_domain(its);
 	if (err)
-		goto out_free_tables;
+		goto out_free_collection;
 
 	raw_spin_lock(&its_lock);
 	list_add(&its->entry, &its_nodes);
@@ -5328,6 +5338,8 @@ static int __init its_probe_one(struct its_node *its)
 
 	return 0;
 
+out_free_collection:
+	kfree(its->collections);
 out_free_tables:
 	its_free_tables(its);
 out_free_cmd:
@@ -5739,9 +5751,13 @@ static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header,
 		its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
 
 	err = its_probe_one(its);
-	if (!err)
-		return 0;
+	if (err)
+		goto probe_err;
+
+	return 0;
 
+probe_err:
+	its_node_destroy(its);
 node_err:
 	iort_deregister_domain_token(its_entry->translation_id);
 dom_err:
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index 99444a1b2ffa2..2673954d45770 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -2588,7 +2588,7 @@ gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end)
 	if (err)
 		goto out_fwhandle_free;
 
-	acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, gic_v3_get_gsi_domain_id);
+	acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, gic_v3_get_gsi_domain_id, NULL);
 
 	if (static_branch_likely(&supports_deactivate_key))
 		gic_acpi_setup_kvm_info();
diff --git a/drivers/irqchip/irq-gic-v5-iwb.c b/drivers/irqchip/irq-gic-v5-iwb.c
index 9103feb70ce8b..442c20614dbe4 100644
--- a/drivers/irqchip/irq-gic-v5-iwb.c
+++ b/drivers/irqchip/irq-gic-v5-iwb.c
@@ -143,7 +143,7 @@ static int gicv5_iwb_irq_domain_translate(struct irq_domain *d, struct irq_fwspe
 			return -EINVAL;
 
 		/*
-		 * param[0] is be the wire
+		 * param[0] is the wire interrupt number
 		 * param[1] is the interrupt type
 		 */
 		*hwirq = fwspec->param[0];
@@ -269,6 +269,8 @@ static int gicv5_iwb_device_probe(struct platform_device *pdev)
 	if (IS_ERR(iwb_node))
 		return PTR_ERR(iwb_node);
 
+	acpi_device_clear_deps(&pdev->dev);
+
 	return 0;
 }
 
diff --git a/drivers/irqchip/irq-gic-v5.c b/drivers/irqchip/irq-gic-v5.c
index e9d1795235a66..5dcf404ab4c8f 100644
--- a/drivers/irqchip/irq-gic-v5.c
+++ b/drivers/irqchip/irq-gic-v5.c
@@ -1226,11 +1226,19 @@ static struct fwnode_handle *gsi_domain_handle;
 static struct fwnode_handle *gic_v5_get_gsi_domain_id(u32 gsi)
 {
 	if (FIELD_GET(GICV5_GSI_IC_TYPE, gsi) == GICV5_GSI_IWB_TYPE)
-		return iort_iwb_handle(FIELD_GET(GICV5_GSI_IWB_FRAME_ID, gsi));
+		return iort_iwb_handle_fwnode(FIELD_GET(GICV5_GSI_IWB_FRAME_ID, gsi));
 
 	return gsi_domain_handle;
 }
 
+static acpi_handle gic_v5_get_gsi_handle(u32 gsi)
+{
+	if (FIELD_GET(GICV5_GSI_IC_TYPE, gsi) == GICV5_GSI_IWB_TYPE)
+		return iort_iwb_handle(FIELD_GET(GICV5_GSI_IWB_FRAME_ID, gsi));
+
+	return NULL;
+}
+
 static int __init gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end)
 {
 	struct acpi_madt_gicv5_irs *irs = (struct acpi_madt_gicv5_irs *)header;
@@ -1251,7 +1259,8 @@ static int __init gic_acpi_init(union acpi_subtable_headers *header, const unsig
 	if (ret)
 		goto out_irs;
 
-	acpi_set_irq_model(ACPI_IRQ_MODEL_GIC_V5, gic_v5_get_gsi_domain_id);
+	acpi_set_irq_model(ACPI_IRQ_MODEL_GIC_V5, gic_v5_get_gsi_domain_id,
+			   gic_v5_get_gsi_handle);
 
 	return 0;
 
diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c
index ec70c84e9f91d..f6bc29f515fb7 100644
--- a/drivers/irqchip/irq-gic.c
+++ b/drivers/irqchip/irq-gic.c
@@ -1690,7 +1690,7 @@ static int __init gic_v2_acpi_init(union acpi_subtable_headers *header,
 		return ret;
 	}
 
-	acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, gic_v2_get_gsi_domain_id);
+	acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, gic_v2_get_gsi_domain_id, NULL);
 
 	if (IS_ENABLED(CONFIG_ARM_GIC_V2M))
 		gicv2m_init(NULL, gic_data[0].domain);
diff --git a/drivers/irqchip/irq-imgpdc.c b/drivers/irqchip/irq-imgpdc.c
deleted file mode 100644
index 4feef4ab5fecc..0000000000000
--- a/drivers/irqchip/irq-imgpdc.c
+++ /dev/null
@@ -1,495 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * IMG PowerDown Controller (PDC)
- *
- * Copyright 2010-2013 Imagination Technologies Ltd.
- *
- * Exposes the syswake and PDC peripheral wake interrupts to the system.
- *
- */
-
-#include <linux/bitops.h>
-#include <linux/interrupt.h>
-#include <linux/irqdomain.h>
-#include <linux/io.h>
-#include <linux/kernel.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
-#include <linux/spinlock.h>
-
-/* PDC interrupt register numbers */
-
-#define PDC_IRQ_STATUS			0x310
-#define PDC_IRQ_ENABLE			0x314
-#define PDC_IRQ_CLEAR			0x318
-#define PDC_IRQ_ROUTE			0x31c
-#define PDC_SYS_WAKE_BASE		0x330
-#define PDC_SYS_WAKE_STRIDE		0x8
-#define PDC_SYS_WAKE_CONFIG_BASE	0x334
-#define PDC_SYS_WAKE_CONFIG_STRIDE	0x8
-
-/* PDC interrupt register field masks */
-
-#define PDC_IRQ_SYS3			0x08
-#define PDC_IRQ_SYS2			0x04
-#define PDC_IRQ_SYS1			0x02
-#define PDC_IRQ_SYS0			0x01
-#define PDC_IRQ_ROUTE_WU_EN_SYS3	0x08000000
-#define PDC_IRQ_ROUTE_WU_EN_SYS2	0x04000000
-#define PDC_IRQ_ROUTE_WU_EN_SYS1	0x02000000
-#define PDC_IRQ_ROUTE_WU_EN_SYS0	0x01000000
-#define PDC_IRQ_ROUTE_WU_EN_WD		0x00040000
-#define PDC_IRQ_ROUTE_WU_EN_IR		0x00020000
-#define PDC_IRQ_ROUTE_WU_EN_RTC		0x00010000
-#define PDC_IRQ_ROUTE_EXT_EN_SYS3	0x00000800
-#define PDC_IRQ_ROUTE_EXT_EN_SYS2	0x00000400
-#define PDC_IRQ_ROUTE_EXT_EN_SYS1	0x00000200
-#define PDC_IRQ_ROUTE_EXT_EN_SYS0	0x00000100
-#define PDC_IRQ_ROUTE_EXT_EN_WD		0x00000004
-#define PDC_IRQ_ROUTE_EXT_EN_IR		0x00000002
-#define PDC_IRQ_ROUTE_EXT_EN_RTC	0x00000001
-#define PDC_SYS_WAKE_RESET		0x00000010
-#define PDC_SYS_WAKE_INT_MODE		0x0000000e
-#define PDC_SYS_WAKE_INT_MODE_SHIFT	1
-#define PDC_SYS_WAKE_PIN_VAL		0x00000001
-
-/* PDC interrupt constants */
-
-#define PDC_SYS_WAKE_INT_LOW		0x0
-#define PDC_SYS_WAKE_INT_HIGH		0x1
-#define PDC_SYS_WAKE_INT_DOWN		0x2
-#define PDC_SYS_WAKE_INT_UP		0x3
-#define PDC_SYS_WAKE_INT_CHANGE		0x6
-#define PDC_SYS_WAKE_INT_NONE		0x4
-
-/**
- * struct pdc_intc_priv - private pdc interrupt data.
- * @nr_perips:		Number of peripheral interrupt signals.
- * @nr_syswakes:	Number of syswake signals.
- * @perip_irqs:		List of peripheral IRQ numbers handled.
- * @syswake_irq:	Shared PDC syswake IRQ number.
- * @domain:		IRQ domain for PDC peripheral and syswake IRQs.
- * @pdc_base:		Base of PDC registers.
- * @irq_route:		Cached version of PDC_IRQ_ROUTE register.
- * @lock:		Lock to protect the PDC syswake registers and the cached
- *			values of those registers in this struct.
- */
-struct pdc_intc_priv {
-	unsigned int		nr_perips;
-	unsigned int		nr_syswakes;
-	unsigned int		*perip_irqs;
-	unsigned int		syswake_irq;
-	struct irq_domain	*domain;
-	void __iomem		*pdc_base;
-
-	u32			irq_route;
-	raw_spinlock_t		lock;
-};
-
-static void pdc_write(struct pdc_intc_priv *priv, unsigned int reg_offs,
-		      unsigned int data)
-{
-	iowrite32(data, priv->pdc_base + reg_offs);
-}
-
-static unsigned int pdc_read(struct pdc_intc_priv *priv,
-			     unsigned int reg_offs)
-{
-	return ioread32(priv->pdc_base + reg_offs);
-}
-
-/* Generic IRQ callbacks */
-
-#define SYS0_HWIRQ	8
-
-static unsigned int hwirq_is_syswake(irq_hw_number_t hw)
-{
-	return hw >= SYS0_HWIRQ;
-}
-
-static unsigned int hwirq_to_syswake(irq_hw_number_t hw)
-{
-	return hw - SYS0_HWIRQ;
-}
-
-static irq_hw_number_t syswake_to_hwirq(unsigned int syswake)
-{
-	return SYS0_HWIRQ + syswake;
-}
-
-static struct pdc_intc_priv *irqd_to_priv(struct irq_data *data)
-{
-	return (struct pdc_intc_priv *)data->domain->host_data;
-}
-
-/*
- * perip_irq_mask() and perip_irq_unmask() use IRQ_ROUTE which also contains
- * wake bits, therefore we cannot use the generic irqchip mask callbacks as they
- * cache the mask.
- */
-
-static void perip_irq_mask(struct irq_data *data)
-{
-	struct pdc_intc_priv *priv = irqd_to_priv(data);
-
-	raw_spin_lock(&priv->lock);
-	priv->irq_route &= ~data->mask;
-	pdc_write(priv, PDC_IRQ_ROUTE, priv->irq_route);
-	raw_spin_unlock(&priv->lock);
-}
-
-static void perip_irq_unmask(struct irq_data *data)
-{
-	struct pdc_intc_priv *priv = irqd_to_priv(data);
-
-	raw_spin_lock(&priv->lock);
-	priv->irq_route |= data->mask;
-	pdc_write(priv, PDC_IRQ_ROUTE, priv->irq_route);
-	raw_spin_unlock(&priv->lock);
-}
-
-static int syswake_irq_set_type(struct irq_data *data, unsigned int flow_type)
-{
-	struct pdc_intc_priv *priv = irqd_to_priv(data);
-	unsigned int syswake = hwirq_to_syswake(data->hwirq);
-	unsigned int irq_mode;
-	unsigned int soc_sys_wake_regoff, soc_sys_wake;
-
-	/* translate to syswake IRQ mode */
-	switch (flow_type) {
-	case IRQ_TYPE_EDGE_BOTH:
-		irq_mode = PDC_SYS_WAKE_INT_CHANGE;
-		break;
-	case IRQ_TYPE_EDGE_RISING:
-		irq_mode = PDC_SYS_WAKE_INT_UP;
-		break;
-	case IRQ_TYPE_EDGE_FALLING:
-		irq_mode = PDC_SYS_WAKE_INT_DOWN;
-		break;
-	case IRQ_TYPE_LEVEL_HIGH:
-		irq_mode = PDC_SYS_WAKE_INT_HIGH;
-		break;
-	case IRQ_TYPE_LEVEL_LOW:
-		irq_mode = PDC_SYS_WAKE_INT_LOW;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	raw_spin_lock(&priv->lock);
-
-	/* set the IRQ mode */
-	soc_sys_wake_regoff = PDC_SYS_WAKE_BASE + syswake*PDC_SYS_WAKE_STRIDE;
-	soc_sys_wake = pdc_read(priv, soc_sys_wake_regoff);
-	soc_sys_wake &= ~PDC_SYS_WAKE_INT_MODE;
-	soc_sys_wake |= irq_mode << PDC_SYS_WAKE_INT_MODE_SHIFT;
-	pdc_write(priv, soc_sys_wake_regoff, soc_sys_wake);
-
-	/* and update the handler */
-	irq_setup_alt_chip(data, flow_type);
-
-	raw_spin_unlock(&priv->lock);
-
-	return 0;
-}
-
-/* applies to both peripheral and syswake interrupts */
-static int pdc_irq_set_wake(struct irq_data *data, unsigned int on)
-{
-	struct pdc_intc_priv *priv = irqd_to_priv(data);
-	irq_hw_number_t hw = data->hwirq;
-	unsigned int mask = (1 << 16) << hw;
-	unsigned int dst_irq;
-
-	raw_spin_lock(&priv->lock);
-	if (on)
-		priv->irq_route |= mask;
-	else
-		priv->irq_route &= ~mask;
-	pdc_write(priv, PDC_IRQ_ROUTE, priv->irq_route);
-	raw_spin_unlock(&priv->lock);
-
-	/* control the destination IRQ wakeup too for standby mode */
-	if (hwirq_is_syswake(hw))
-		dst_irq = priv->syswake_irq;
-	else
-		dst_irq = priv->perip_irqs[hw];
-	irq_set_irq_wake(dst_irq, on);
-
-	return 0;
-}
-
-static void pdc_intc_perip_isr(struct irq_desc *desc)
-{
-	unsigned int irq = irq_desc_get_irq(desc);
-	struct pdc_intc_priv *priv;
-	unsigned int i;
-
-	priv = (struct pdc_intc_priv *)irq_desc_get_handler_data(desc);
-
-	/* find the peripheral number */
-	for (i = 0; i < priv->nr_perips; ++i)
-		if (irq == priv->perip_irqs[i])
-			goto found;
-
-	/* should never get here */
-	return;
-found:
-
-	/* pass on the interrupt */
-	generic_handle_domain_irq(priv->domain, i);
-}
-
-static void pdc_intc_syswake_isr(struct irq_desc *desc)
-{
-	struct pdc_intc_priv *priv;
-	unsigned int syswake;
-	unsigned int status;
-
-	priv = (struct pdc_intc_priv *)irq_desc_get_handler_data(desc);
-
-	status = pdc_read(priv, PDC_IRQ_STATUS) &
-		 pdc_read(priv, PDC_IRQ_ENABLE);
-	status &= (1 << priv->nr_syswakes) - 1;
-
-	for (syswake = 0; status; status >>= 1, ++syswake) {
-		/* Has this sys_wake triggered? */
-		if (!(status & 1))
-			continue;
-
-		generic_handle_domain_irq(priv->domain, syswake_to_hwirq(syswake));
-	}
-}
-
-static void pdc_intc_setup(struct pdc_intc_priv *priv)
-{
-	int i;
-	unsigned int soc_sys_wake_regoff;
-	unsigned int soc_sys_wake;
-
-	/*
-	 * Mask all syswake interrupts before routing, or we could receive an
-	 * interrupt before we're ready to handle it.
-	 */
-	pdc_write(priv, PDC_IRQ_ENABLE, 0);
-
-	/*
-	 * Enable routing of all syswakes
-	 * Disable all wake sources
-	 */
-	priv->irq_route = ((PDC_IRQ_ROUTE_EXT_EN_SYS0 << priv->nr_syswakes) -
-				PDC_IRQ_ROUTE_EXT_EN_SYS0);
-	pdc_write(priv, PDC_IRQ_ROUTE, priv->irq_route);
-
-	/* Initialise syswake IRQ */
-	for (i = 0; i < priv->nr_syswakes; ++i) {
-		/* set the IRQ mode to none */
-		soc_sys_wake_regoff = PDC_SYS_WAKE_BASE + i*PDC_SYS_WAKE_STRIDE;
-		soc_sys_wake = PDC_SYS_WAKE_INT_NONE
-				<< PDC_SYS_WAKE_INT_MODE_SHIFT;
-		pdc_write(priv, soc_sys_wake_regoff, soc_sys_wake);
-	}
-}
-
-static int pdc_intc_probe(struct platform_device *pdev)
-{
-	struct pdc_intc_priv *priv;
-	struct device_node *node = pdev->dev.of_node;
-	struct resource *res_regs;
-	struct irq_chip_generic *gc;
-	unsigned int i;
-	int irq, ret;
-	u32 val;
-
-	if (!node)
-		return -ENOENT;
-
-	/* Get registers */
-	res_regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (res_regs == NULL) {
-		dev_err(&pdev->dev, "cannot find registers resource\n");
-		return -ENOENT;
-	}
-
-	/* Allocate driver data */
-	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-	raw_spin_lock_init(&priv->lock);
-	platform_set_drvdata(pdev, priv);
-
-	/* Ioremap the registers */
-	priv->pdc_base = devm_ioremap(&pdev->dev, res_regs->start,
-				      resource_size(res_regs));
-	if (!priv->pdc_base)
-		return -EIO;
-
-	/* Get number of peripherals */
-	ret = of_property_read_u32(node, "num-perips", &val);
-	if (ret) {
-		dev_err(&pdev->dev, "No num-perips node property found\n");
-		return -EINVAL;
-	}
-	if (val > SYS0_HWIRQ) {
-		dev_err(&pdev->dev, "num-perips (%u) out of range\n", val);
-		return -EINVAL;
-	}
-	priv->nr_perips = val;
-
-	/* Get number of syswakes */
-	ret = of_property_read_u32(node, "num-syswakes", &val);
-	if (ret) {
-		dev_err(&pdev->dev, "No num-syswakes node property found\n");
-		return -EINVAL;
-	}
-	if (val > SYS0_HWIRQ) {
-		dev_err(&pdev->dev, "num-syswakes (%u) out of range\n", val);
-		return -EINVAL;
-	}
-	priv->nr_syswakes = val;
-
-	/* Get peripheral IRQ numbers */
-	priv->perip_irqs = devm_kcalloc(&pdev->dev, 4, priv->nr_perips,
-					GFP_KERNEL);
-	if (!priv->perip_irqs)
-		return -ENOMEM;
-	for (i = 0; i < priv->nr_perips; ++i) {
-		irq = platform_get_irq(pdev, 1 + i);
-		if (irq < 0)
-			return irq;
-		priv->perip_irqs[i] = irq;
-	}
-	/* check if too many were provided */
-	if (platform_get_irq(pdev, 1 + i) >= 0) {
-		dev_err(&pdev->dev, "surplus perip IRQs detected\n");
-		return -EINVAL;
-	}
-
-	/* Get syswake IRQ number */
-	irq = platform_get_irq(pdev, 0);
-	if (irq < 0)
-		return irq;
-	priv->syswake_irq = irq;
-
-	/* Set up an IRQ domain */
-	priv->domain = irq_domain_create_linear(dev_fwnode(&pdev->dev), 16, &irq_generic_chip_ops,
-						priv);
-	if (unlikely(!priv->domain)) {
-		dev_err(&pdev->dev, "cannot add IRQ domain\n");
-		return -ENOMEM;
-	}
-	priv->domain->flags |= IRQ_DOMAIN_FLAG_DESTROY_GC;
-
-	/*
-	 * Set up 2 generic irq chips with 2 chip types.
-	 * The first one for peripheral irqs (only 1 chip type used)
-	 * The second one for syswake irqs (edge and level chip types)
-	 */
-	ret = irq_alloc_domain_generic_chips(priv->domain, 8, 2, "pdc",
-					     handle_level_irq, 0, 0,
-					     IRQ_GC_INIT_NESTED_LOCK);
-	if (ret)
-		goto err_generic;
-
-	/* peripheral interrupt chip */
-
-	gc = irq_get_domain_generic_chip(priv->domain, 0);
-	gc->unused	= ~(BIT(priv->nr_perips) - 1);
-	gc->reg_base	= priv->pdc_base;
-	/*
-	 * IRQ_ROUTE contains wake bits, so we can't use the generic versions as
-	 * they cache the mask
-	 */
-	gc->chip_types[0].regs.mask		= PDC_IRQ_ROUTE;
-	gc->chip_types[0].chip.irq_mask		= perip_irq_mask;
-	gc->chip_types[0].chip.irq_unmask	= perip_irq_unmask;
-	gc->chip_types[0].chip.irq_set_wake	= pdc_irq_set_wake;
-
-	/* syswake interrupt chip */
-
-	gc = irq_get_domain_generic_chip(priv->domain, 8);
-	gc->unused	= ~(BIT(priv->nr_syswakes) - 1);
-	gc->reg_base	= priv->pdc_base;
-
-	/* edge interrupts */
-	gc->chip_types[0].type			= IRQ_TYPE_EDGE_BOTH;
-	gc->chip_types[0].handler		= handle_edge_irq;
-	gc->chip_types[0].regs.ack		= PDC_IRQ_CLEAR;
-	gc->chip_types[0].regs.mask		= PDC_IRQ_ENABLE;
-	gc->chip_types[0].chip.irq_ack		= irq_gc_ack_set_bit;
-	gc->chip_types[0].chip.irq_mask		= irq_gc_mask_clr_bit;
-	gc->chip_types[0].chip.irq_unmask	= irq_gc_mask_set_bit;
-	gc->chip_types[0].chip.irq_set_type	= syswake_irq_set_type;
-	gc->chip_types[0].chip.irq_set_wake	= pdc_irq_set_wake;
-	/* for standby we pass on to the shared syswake IRQ */
-	gc->chip_types[0].chip.flags		= IRQCHIP_MASK_ON_SUSPEND;
-
-	/* level interrupts */
-	gc->chip_types[1].type			= IRQ_TYPE_LEVEL_MASK;
-	gc->chip_types[1].handler		= handle_level_irq;
-	gc->chip_types[1].regs.ack		= PDC_IRQ_CLEAR;
-	gc->chip_types[1].regs.mask		= PDC_IRQ_ENABLE;
-	gc->chip_types[1].chip.irq_ack		= irq_gc_ack_set_bit;
-	gc->chip_types[1].chip.irq_mask		= irq_gc_mask_clr_bit;
-	gc->chip_types[1].chip.irq_unmask	= irq_gc_mask_set_bit;
-	gc->chip_types[1].chip.irq_set_type	= syswake_irq_set_type;
-	gc->chip_types[1].chip.irq_set_wake	= pdc_irq_set_wake;
-	/* for standby we pass on to the shared syswake IRQ */
-	gc->chip_types[1].chip.flags		= IRQCHIP_MASK_ON_SUSPEND;
-
-	/* Set up the hardware to enable interrupt routing */
-	pdc_intc_setup(priv);
-
-	/* Setup chained handlers for the peripheral IRQs */
-	for (i = 0; i < priv->nr_perips; ++i) {
-		irq = priv->perip_irqs[i];
-		irq_set_chained_handler_and_data(irq, pdc_intc_perip_isr,
-						 priv);
-	}
-
-	/* Setup chained handler for the syswake IRQ */
-	irq_set_chained_handler_and_data(priv->syswake_irq,
-					 pdc_intc_syswake_isr, priv);
-
-	dev_info(&pdev->dev,
-		 "PDC IRQ controller initialised (%u perip IRQs, %u syswake IRQs)\n",
-		 priv->nr_perips,
-		 priv->nr_syswakes);
-
-	return 0;
-err_generic:
-	irq_domain_remove(priv->domain);
-	return ret;
-}
-
-static void pdc_intc_remove(struct platform_device *pdev)
-{
-	struct pdc_intc_priv *priv = platform_get_drvdata(pdev);
-
-	for (unsigned int i = 0; i < priv->nr_perips; ++i)
-		irq_set_chained_handler_and_data(priv->perip_irqs[i], NULL, NULL);
-
-	irq_set_chained_handler_and_data(priv->syswake_irq, NULL, NULL);
-
-	irq_domain_remove(priv->domain);
-}
-
-static const struct of_device_id pdc_intc_match[] = {
-	{ .compatible = "img,pdc-intc" },
-	{}
-};
-
-static struct platform_driver pdc_intc_driver = {
-	.driver = {
-		.name		= "pdc-intc",
-		.of_match_table	= pdc_intc_match,
-	},
-	.probe		= pdc_intc_probe,
-	.remove		= pdc_intc_remove,
-};
-
-static int __init pdc_intc_init(void)
-{
-	return platform_driver_register(&pdc_intc_driver);
-}
-core_initcall(pdc_intc_init);
diff --git a/drivers/irqchip/irq-loongarch-cpu.c b/drivers/irqchip/irq-loongarch-cpu.c
index 950bc087e3880..84ce248894889 100644
--- a/drivers/irqchip/irq-loongarch-cpu.c
+++ b/drivers/irqchip/irq-loongarch-cpu.c
@@ -168,7 +168,7 @@ static int __init cpuintc_acpi_init(union acpi_subtable_headers *header,
 		panic("Failed to add irqdomain for LoongArch CPU");
 
 	set_handle_irq(&handle_cpu_irq);
-	acpi_set_irq_model(ACPI_IRQ_MODEL_LPIC, lpic_get_gsi_domain_id);
+	acpi_set_irq_model(ACPI_IRQ_MODEL_LPIC, lpic_get_gsi_domain_id, NULL);
 	acpi_set_gsi_to_irq_fallback(lpic_gsi_to_irq);
 	ret = acpi_cascade_irqdomain_init();
 
diff --git a/drivers/irqchip/irq-qcom-mpm.c b/drivers/irqchip/irq-qcom-mpm.c
index 181320528a47a..8c575d7ce8a17 100644
--- a/drivers/irqchip/irq-qcom-mpm.c
+++ b/drivers/irqchip/irq-qcom-mpm.c
@@ -464,10 +464,8 @@ static int qcom_mpm_probe(struct platform_device *pdev, struct device_node *pare
 
 	ret = devm_request_irq(dev, irq, qcom_mpm_handler, IRQF_NO_SUSPEND,
 			       "qcom_mpm", priv);
-	if (ret) {
-		dev_err(dev, "failed to request irq: %d\n", ret);
+	if (ret)
 		goto remove_domain;
-	}
 
 	return 0;
 
diff --git a/drivers/irqchip/irq-realtek-rtl.c b/drivers/irqchip/irq-realtek-rtl.c
index 2ae3be7fa6338..c8becb458da23 100644
--- a/drivers/irqchip/irq-realtek-rtl.c
+++ b/drivers/irqchip/irq-realtek-rtl.c
@@ -25,6 +25,15 @@
 
 #define REG(cpu, x)		(realtek_ictl_base[cpu] + x)
 
+struct realtek_ictl_output {
+	struct fwnode_handle	*fwnode;
+	struct irq_domain	*domain;
+	unsigned int		parent_irq;
+	unsigned int		parent_hwirq;
+	unsigned int		index;
+	u32			mask;
+};
+
 static DEFINE_RAW_SPINLOCK(irq_lock);
 static void __iomem *realtek_ictl_base[NR_CPUS];
 
@@ -105,53 +114,141 @@ static struct irq_chip realtek_ictl_irq = {
 	.irq_set_affinity	= realtek_ictl_irq_affinity,
 };
 
-static int intc_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw)
+static int intc_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw_irq)
 {
+	struct realtek_ictl_output *output = d->host_data;
 	unsigned int cpu;
 
 	irq_set_chip_and_handler(irq, &realtek_ictl_irq, handle_level_irq);
 
 	guard(raw_spinlock_irqsave)(&irq_lock);
+	output->mask |= BIT(hw_irq);
 	for_each_present_cpu(cpu)
-		write_irr(cpu, hw, 1);
+		write_irr(cpu, hw_irq, output->parent_hwirq - 1);
 
 	return 0;
 }
 
+static int intc_select(struct irq_domain *d, struct irq_fwspec *fwspec,
+		       enum irq_domain_bus_token bus_token)
+{
+	struct realtek_ictl_output *output = d->host_data;
+	unsigned int index = 0;
+
+	if (fwspec->fwnode != output->fwnode)
+		return false;
+
+	if (fwspec->param_count == 2)
+		index = fwspec->param[1];
+
+	return index == output->index;
+}
+
 static const struct irq_domain_ops irq_domain_ops = {
-	.map = intc_map,
-	.xlate = irq_domain_xlate_onecell,
+	.map	= intc_map,
+	.select	= intc_select,
+	.xlate	= irq_domain_xlate_onecell,
 };
 
 static void realtek_irq_dispatch(struct irq_desc *desc)
 {
+	struct realtek_ictl_output *output = irq_desc_get_handler_data(desc);
 	struct irq_chip *chip = irq_desc_get_chip(desc);
 	unsigned int cpu = smp_processor_id();
-	struct irq_domain *domain;
 	unsigned long pending;
-	unsigned int soc_int;
+	unsigned int hw_irq;
 
 	chained_irq_enter(chip, desc);
-	pending = readl(REG(cpu, RTL_ICTL_GIMR)) & readl(REG(cpu, RTL_ICTL_GISR));
+	pending = readl(REG(cpu, RTL_ICTL_GIMR)) & readl(REG(cpu, RTL_ICTL_GISR)) & output->mask;
 
 	if (unlikely(!pending)) {
 		spurious_interrupt();
 		goto out;
 	}
 
-	domain = irq_desc_get_handler_data(desc);
-	for_each_set_bit(soc_int, &pending, RTL_ICTL_NUM_INPUTS)
-		generic_handle_domain_irq(domain, soc_int);
+	for_each_set_bit(hw_irq, &pending, RTL_ICTL_NUM_INPUTS)
+		generic_handle_domain_irq(output->domain, hw_irq);
 
 out:
 	chained_irq_exit(chip, desc);
 }
 
-static int __init realtek_rtl_of_init(struct device_node *node, struct device_node *parent)
+static int __init realtek_setup_parents(struct device_node *node)
 {
+	int p, cnt, err, parent_irq, num_parents = of_irq_count(node);
+	struct realtek_ictl_output *output;
+	struct irq_data *parent_data;
 	struct of_phandle_args oirq;
 	struct irq_domain *domain;
-	int cpu, parent_irq;
+
+	cnt = max(1, num_parents);
+	output = kcalloc(cnt, sizeof(*output), GFP_KERNEL);
+	if (!output)
+		return -ENOMEM;
+
+	for (p = 0; p < cnt; p++) {
+		if (WARN_ON(!num_parents)) {
+			/*
+			 * If DT contains no parent interrupts, assume MIPS IRQ 2 (HW0) is
+			 * connected to the first output. This is the case for all known hardware.
+			 */
+			oirq.np = of_find_compatible_node(NULL, NULL,
+							  "mti,cpu-interrupt-controller");
+			if (!oirq.np) {
+				err = -EINVAL;
+				goto err_out;
+			}
+
+			oirq.args_count = 1;
+			oirq.args[0] = 2;
+			parent_irq = irq_create_of_mapping(&oirq);
+			of_node_put(oirq.np);
+		} else {
+			parent_irq = of_irq_get(node, p);
+		}
+
+		if (parent_irq <= 0) {
+			err = parent_irq ? parent_irq : -ENODEV;
+			goto err_out;
+		}
+
+		parent_data = irq_get_irq_data(parent_irq);
+		if (!parent_data) {
+			err = -EINVAL;
+			goto err_out;
+		}
+
+		domain = irq_domain_create_linear(of_fwnode_handle(node), RTL_ICTL_NUM_INPUTS,
+						  &irq_domain_ops, &output[p]);
+		if (!domain) {
+			err = -ENOMEM;
+			goto err_out;
+		}
+
+		output[p].domain = domain;
+		output[p].fwnode = of_fwnode_handle(node);
+		output[p].index = p;
+		output[p].parent_irq = parent_irq;
+		output[p].parent_hwirq = irqd_to_hwirq(parent_data);
+		irq_set_chained_handler_and_data(parent_irq, realtek_irq_dispatch, &output[p]);
+	}
+
+	return 0;
+
+err_out:
+	while (p--) {
+		irq_set_chained_handler_and_data(output[p].parent_irq, NULL, NULL);
+		irq_domain_remove(output[p].domain);
+	}
+
+	kfree(output);
+
+	return err;
+}
+
+static int __init realtek_rtl_of_init(struct device_node *node, struct device_node *parent)
+{
+	unsigned int cpu;
 
 	for_each_present_cpu(cpu) {
 		realtek_ictl_base[cpu] = of_iomap(node, cpu);
@@ -165,36 +262,7 @@ static int __init realtek_rtl_of_init(struct device_node *node, struct device_no
 		}
 	}
 
-	if (WARN_ON(!of_irq_count(node))) {
-		/*
-		 * If DT contains no parent interrupts, assume MIPS CPU IRQ 2
-		 * (HW0) is connected to the first output. This is the case for
-		 * all known hardware anyway. "interrupt-map" is deprecated, so
-		 * don't bother trying to parse that.
-		 */
-		oirq.np = of_find_compatible_node(NULL, NULL, "mti,cpu-interrupt-controller");
-		oirq.args_count = 1;
-		oirq.args[0] = 2;
-
-		parent_irq = irq_create_of_mapping(&oirq);
-
-		of_node_put(oirq.np);
-	} else {
-		parent_irq = of_irq_get(node, 0);
-	}
-
-	if (parent_irq < 0)
-		return parent_irq;
-	else if (!parent_irq)
-		return -ENODEV;
-
-	domain = irq_domain_create_linear(of_fwnode_handle(node), RTL_ICTL_NUM_INPUTS, &irq_domain_ops, NULL);
-	if (!domain)
-		return -ENOMEM;
-
-	irq_set_chained_handler_and_data(parent_irq, realtek_irq_dispatch, domain);
-
-	return 0;
+	return realtek_setup_parents(node);
 }
 
 IRQCHIP_DECLARE(realtek_rtl_intc, "realtek,rtl-intc", realtek_rtl_of_init);
diff --git a/drivers/irqchip/irq-renesas-irqc.c b/drivers/irqchip/irq-renesas-irqc.c
index a20a6471b0e48..1ff3535a4617f 100644
--- a/drivers/irqchip/irq-renesas-irqc.c
+++ b/drivers/irqchip/irq-renesas-irqc.c
@@ -176,6 +176,7 @@ static int irqc_probe(struct platform_device *pdev)
 		goto err_runtime_pm_disable;
 	}
 
+	p->irq_domain->flags |= IRQ_DOMAIN_FLAG_DESTROY_GC;
 	ret = irq_alloc_domain_generic_chips(p->irq_domain, p->number_of_irqs,
 					     1, "irqc", handle_level_irq,
 					     0, 0, IRQ_GC_INIT_NESTED_LOCK);
diff --git a/drivers/irqchip/irq-renesas-rzt2h.c b/drivers/irqchip/irq-renesas-rzt2h.c
index e06264add3cc0..9086fc3a2043d 100644
--- a/drivers/irqchip/irq-renesas-rzt2h.c
+++ b/drivers/irqchip/irq-renesas-rzt2h.c
@@ -398,7 +398,7 @@ static int rzt2h_icu_request_irqs(struct platform_device *pdev, struct irq_domai
 		ret = devm_request_irq(dev, virq, handler, 0, dev_name(dev),
 				       data ?: (void *)(uintptr_t)offset);
 		if (ret)
-			return dev_err_probe(dev, ret, "Failed to request IRQ %u\n", offset);
+			return ret;
 	}
 
 	return 0;
diff --git a/drivers/irqchip/irq-renesas-rzv2h.c b/drivers/irqchip/irq-renesas-rzv2h.c
index 971ac83eee90e..1a4703a353e95 100644
--- a/drivers/irqchip/irq-renesas-rzv2h.c
+++ b/drivers/irqchip/irq-renesas-rzv2h.c
@@ -816,7 +816,7 @@ static int rzv2h_icu_setup_irqs(struct platform_device *pdev, struct irq_domain
 		ret = devm_request_irq(dev, virq, rzv2h_icu_swint_irq, 0, dev_name(dev),
 				       (void *)(uintptr_t)i);
 		if (ret)
-			return dev_err_probe(dev, ret, "Failed to request int-ca55-%u IRQ\n", i);
+			return ret;
 	}
 
 	/* Unmask and clear all IP/CA55 error interrupts */
@@ -844,7 +844,7 @@ static int rzv2h_icu_setup_irqs(struct platform_device *pdev, struct irq_domain
 
 	ret = devm_request_irq(dev, virq, rzv2h_icu_error_irq, 0, dev_name(dev), rzv2h_icu_data);
 	if (ret)
-		return dev_err_probe(dev, ret, "Failed to request icu-error-ca55 IRQ\n");
+		return ret;
 
 	return 0;
 }
diff --git a/drivers/irqchip/irq-riscv-intc.c b/drivers/irqchip/irq-riscv-intc.c
index 84418dbd5a270..0595144116e2a 100644
--- a/drivers/irqchip/irq-riscv-intc.c
+++ b/drivers/irqchip/irq-riscv-intc.c
@@ -384,7 +384,8 @@ static int __init riscv_intc_acpi_init(union acpi_subtable_headers *header,
 	if (rc)
 		irq_domain_free_fwnode(fn);
 	else
-		acpi_set_irq_model(ACPI_IRQ_MODEL_RINTC, riscv_acpi_get_gsi_domain_id);
+		acpi_set_irq_model(ACPI_IRQ_MODEL_RINTC, riscv_acpi_get_gsi_domain_id,
+				   acpi_get_riscv_gsi_handle);
 
 	return rc;
 }
diff --git a/drivers/irqchip/qcom-pdc.c b/drivers/irqchip/qcom-pdc.c
index 2014dbb0bc43a..ce6d80c7f17a3 100644
--- a/drivers/irqchip/qcom-pdc.c
+++ b/drivers/irqchip/qcom-pdc.c
@@ -20,22 +20,19 @@
 #include <linux/spinlock.h>
 #include <linux/slab.h>
 #include <linux/types.h>
+#include <linux/firmware/qcom/qcom_scm.h>
 
-#define PDC_MAX_GPIO_IRQS	256
-#define PDC_DRV_SIZE		0x10000
-
-/* Valid only on HW version < 3.2 */
-#define IRQ_ENABLE_BANK		0x10
-#define IRQ_ENABLE_BANK_MAX	(IRQ_ENABLE_BANK + BITS_TO_BYTES(PDC_MAX_GPIO_IRQS))
+#define PDC_MAX_IRQS			256
+#define IRQ_ENABLE_BANK_MAX		BITS_TO_BYTES(PDC_MAX_IRQS)
 #define IRQ_ENABLE_BANK_INDEX_MASK	GENMASK(31, 5)
 #define IRQ_ENABLE_BANK_BIT_MASK	GENMASK(4, 0)
-#define IRQ_i_CFG		0x110
-
-/* Valid only on HW version >= 3.2 */
-#define IRQ_i_CFG_IRQ_ENABLE	3
 
-#define IRQ_i_CFG_TYPE_MASK		GENMASK(2, 0)
+/* Secure DRV register to configure the PDC mode via qcom_scm_io_writel() */
+#define PDC_GPIO_INT_CTL_ENABLE		0xb2045e8
+#define PDC_PASS_THROUGH_MODE		0x0
+#define PDC_SECONDARY_MODE		0x1
 
+#define PDC_DRV_SIZE			0x10000
 #define PDC_VERSION_REG			0x1000
 #define PDC_VERSION_MAJOR		GENMASK(23, 16)
 #define PDC_VERSION_MINOR		GENMASK(15, 8)
@@ -46,22 +43,149 @@
 
 /* Notable PDC versions */
 #define PDC_VERSION_3_2			PDC_VERSION(3, 2, 0)
+#define PDC_VERSION_3_0			PDC_VERSION(3, 0, 0)
+#define PDC_VERSION_2_7			PDC_VERSION(2, 7, 0)
+
+/*
+ * PDC Hardware registers layout per version:
+ *
+ * IRQ_ENABLE_BANK[b], b = 0....BITS_TO_BYTES(PDC_MAX_IRQS)
+ * IRQ_CFG[n], n = 0....PDC_MAX_IRQS
+ *
+ * +---------------------------------------------------------------+
+ * |       v2.7        |       v3.0        |       v3.2            |
+ * |---------------------------------------------------------------|
+ * |       BASE        |       BASE        |       BASE            |
+ * |---------------------------------------------------------------|
+ * |                                                               |
+ * |   IRQ_ENABLE_BANK | IRQ_ENABLE_BANK   |       NA              |
+ * |---------------------------------------------------------------|
+ * |   IRQ_CFG         | IRQ_CFG           | IRQ_CFG               |
+ * |                   |                   |                       |
+ * |                   |                   | [31:6] Unused         |
+ * |                   | [31:5] Unused     |    [5] GPIO_STATUS    |
+ * |                   |    [4] GPIO_STATUS|    [4] GPIO_MASK      |
+ * |   [31:3] Unused   |    [3] GPIO_MASK  |    [3] IRQ_ENABLE     |
+ * |    [0:2] Type     |  [0:2] Type       |  [0:2] Type           |
+ * |---------------------------------------------------------------|
+ * |   IRQ_PARAM       | IRQ_PARAM         | IRQ_PARAM             |
+ * |                   |                                           |
+ * |   [15:8] NUM_GPIO | [15:8] NUM_GPIO   | [15:8] NUM_GPIO       |
+ * |    [7:0] NUM_SPI  |  [7:0] NUM_SPI    |  [7:0] NUM_SPI        |
+ * +---------------------------------------------------------------+
+ */
+
+/**
+ * struct pdc_regs: PDC registers location
+ *
+ * @irq_en_reg:     IRQ_ENABLE_BANK register location
+ * @irq_cfg_reg:    IRQ_CFG register location
+ * @irq_param_reg:  IRQ_PARAM register location
+ */
+struct pdc_regs {
+	u32	irq_en_reg;
+	u32	irq_cfg_reg;
+	u32	irq_param_reg;
+};
+
+/**
+ * struct pdc_irq_cfg: bit fields for PDC IRQ_CFG register
+ *
+ * @gpio_irq_sts:   bit number for GPIO_STATUS field
+ * @gpio_irq_mask:  bit number for GPIO_MASK field
+ * @irq_enable:     bit number for IRQ_ENABLE field
+ * @irq_type:       GENMASK for IRQ_TYPE field
+ */
+struct pdc_irq_cfg {
+	u32	gpio_irq_sts;
+	u32	gpio_irq_mask;
+	u32	irq_enable;
+	u32	irq_type;
+};
+
+/**
+ * struct pdc_desc: PDC driver state
+ *
+ * @base:           PDC base register for DRV2 / HLOS
+ * @prev_base:      PDC DRV1 base, applicable only for x1e RTL bug.
+ * @version:        PDC version
+ * @num_spis:       Total number of direct SPI interrupts
+ * @region:         PDC interrupt continuous range
+ * @region_cnt:     Total PDC ranges
+ * @mode:           PDC_PASS_THROUGH_MODE or PDC_SECONDARY_MODE
+ * @x1e_quirk:      x1e H/W Bug handling
+ * @lock:           lock for IRQ_ENABLE_BANK protection
+ * @regs:           PDC regs (IRQ_ENABLE_BANK and IRQ_CFG)
+ * @cfg_fields:     Fields of IRQ_CFG reg
+ * @enable_intr:    pointer to enable function based on PDC version
+ * @unmask_gpio:    pointer to GPIO irq unmask function
+ * @clear_gpio:     pointer to GPIO irq clear function
+ */
+struct pdc_desc {
+	void __iomem			*base;
+	void __iomem			*prev_base;
+	u32				version;
+	u32				num_spis;
+
+	struct pdc_pin_region		*region;
+	int				region_cnt;
+
+	bool				x1e_quirk;
+
+	u8				mode;
+	raw_spinlock_t			lock;
+
+	const struct pdc_regs		*regs;
+	const struct pdc_irq_cfg	*cfg_fields;
+
+	void (*enable_intr)(int pin_out, bool on);
+	void (*unmask_gpio)(int pin_out, bool on);
+	void (*clear_gpio)(int pin_out);
+};
+
+static const struct pdc_regs pdc_v3_2 = {
+	.irq_cfg_reg	= 0x110,
+	.irq_param_reg	= 0x100c,
+};
+
+static const struct pdc_irq_cfg pdc_cfg_v3_2 = {
+	.gpio_irq_sts	= 5,
+	.gpio_irq_mask	= 4,
+	.irq_enable	= 3,
+	.irq_type	= GENMASK(2, 0),
+};
+
+static const struct pdc_regs pdc_v3_0 = {
+	.irq_en_reg	= 0x10,
+	.irq_cfg_reg	= 0x110,
+	.irq_param_reg	= 0x100c,
+};
+
+static const struct pdc_irq_cfg pdc_cfg_v3_0 = {
+	.gpio_irq_sts	= 4,
+	.gpio_irq_mask	= 3,
+	.irq_type	= GENMASK(2, 0),
+};
+
+static const struct pdc_regs pdc_v2_7 = {
+	.irq_en_reg	= 0x10,
+	.irq_cfg_reg	= 0x110,
+	.irq_param_reg	= 0x100c,
+};
+
+static const struct pdc_irq_cfg pdc_cfg_v2_7 = {
+	.irq_type	= GENMASK(2, 0),
+};
 
 struct pdc_pin_region {
-	u32 pin_base;
-	u32 parent_base;
-	u32 cnt;
+	u32	pin_base;
+	u32	parent_base;
+	u32	cnt;
 };
 
 #define pin_to_hwirq(r, p)	((r)->parent_base + (p) - (r)->pin_base)
 
-static DEFINE_RAW_SPINLOCK(pdc_lock);
-static void __iomem *pdc_base;
-static void __iomem *pdc_prev_base;
-static struct pdc_pin_region *pdc_region;
-static int pdc_region_cnt;
-static unsigned int pdc_version;
-static bool pdc_x1e_quirk;
+static struct pdc_desc *pdc;
 
 static void pdc_base_reg_write(void __iomem *base, int reg, u32 i, u32 val)
 {
@@ -70,12 +194,21 @@ static void pdc_base_reg_write(void __iomem *base, int reg, u32 i, u32 val)
 
 static void pdc_reg_write(int reg, u32 i, u32 val)
 {
-	pdc_base_reg_write(pdc_base, reg, i, val);
+	pdc_base_reg_write(pdc->base, reg, i, val);
 }
 
 static u32 pdc_reg_read(int reg, u32 i)
 {
-	return readl_relaxed(pdc_base + reg + i * sizeof(u32));
+	return readl_relaxed(pdc->base + reg + i * sizeof(u32));
+}
+
+static inline bool pdc_pin_is_gpio(int pin_out)
+{
+	/*
+	 * PDC allocates direct SPIs at the beginning and
+	 * all GPIOs as SPIs are allocated after direct SPIs.
+	 */
+	return pin_out >= pdc->num_spis;
 }
 
 static void pdc_x1e_irq_enable_write(u32 bank, u32 enable)
@@ -86,24 +219,24 @@ static void pdc_x1e_irq_enable_write(u32 bank, u32 enable)
 	switch (bank) {
 	case 0 ... 1:
 		/* Use previous DRV (client) region and shift to bank 3-4 */
-		base = pdc_prev_base;
+		base = pdc->prev_base;
 		bank += 3;
 		break;
 	case 2 ... 4:
 		/* Use our own region and shift to bank 0-2 */
-		base = pdc_base;
+		base = pdc->base;
 		bank -= 2;
 		break;
 	case 5:
 		/* No fixup required for bank 5 */
-		base = pdc_base;
+		base = pdc->base;
 		break;
 	default:
 		WARN_ON(1);
 		return;
 	}
 
-	pdc_base_reg_write(base, IRQ_ENABLE_BANK, bank, enable);
+	pdc_base_reg_write(base, pdc->regs->irq_en_reg, bank, enable);
 }
 
 static void pdc_enable_intr_bank(int pin_out, bool on)
@@ -114,52 +247,97 @@ static void pdc_enable_intr_bank(int pin_out, bool on)
 	index = FIELD_GET(IRQ_ENABLE_BANK_INDEX_MASK, pin_out);
 	mask = FIELD_GET(IRQ_ENABLE_BANK_BIT_MASK, pin_out);
 
-	enable = pdc_reg_read(IRQ_ENABLE_BANK, index);
+	guard(raw_spinlock_irqsave)(&pdc->lock);
+
+	enable = pdc_reg_read(pdc->regs->irq_en_reg, index);
 	__assign_bit(mask, &enable, on);
 
-	if (pdc_x1e_quirk)
+	if (pdc->x1e_quirk)
 		pdc_x1e_irq_enable_write(index, enable);
 	else
-		pdc_reg_write(IRQ_ENABLE_BANK, index, enable);
+		pdc_reg_write(pdc->regs->irq_en_reg, index, enable);
 }
 
-static void pdc_enable_intr_cfg(int pin_out, bool on)
+static void pdc_clear_gpio_cfg(int pin_out)
 {
-	unsigned long enable = pdc_reg_read(IRQ_i_CFG, pin_out);
+	unsigned long gpio_sts;
 
-	__assign_bit(IRQ_i_CFG_IRQ_ENABLE, &enable, on);
-	pdc_reg_write(IRQ_i_CFG, pin_out, enable);
+	gpio_sts = pdc_reg_read(pdc->regs->irq_cfg_reg, pin_out);
+	__clear_bit(pdc->cfg_fields->gpio_irq_sts, &gpio_sts);
+	pdc_reg_write(pdc->regs->irq_cfg_reg, pin_out, gpio_sts);
 }
 
-static void __pdc_enable_intr(int pin_out, bool on)
+static void pdc_unmask_gpio_cfg(int pin_out, bool unmask)
 {
-	if (pdc_version < PDC_VERSION_3_2)
-		pdc_enable_intr_bank(pin_out, on);
-	else
-		pdc_enable_intr_cfg(pin_out, on);
+	unsigned long gpio_mask;
+
+	gpio_mask = pdc_reg_read(pdc->regs->irq_cfg_reg, pin_out);
+	__assign_bit(pdc->cfg_fields->gpio_irq_mask, &gpio_mask, !unmask);
+	pdc_reg_write(pdc->regs->irq_cfg_reg, pin_out, gpio_mask);
 }
 
-static void pdc_enable_intr(struct irq_data *d, bool on)
+static void pdc_enable_intr_cfg(int pin_out, bool on)
 {
-	unsigned long flags;
+	unsigned long enable = pdc_reg_read(pdc->regs->irq_cfg_reg, pin_out);
 
-	raw_spin_lock_irqsave(&pdc_lock, flags);
-	__pdc_enable_intr(d->hwirq, on);
-	raw_spin_unlock_irqrestore(&pdc_lock, flags);
+	__assign_bit(pdc->cfg_fields->irq_enable, &enable, on);
+	pdc_reg_write(pdc->regs->irq_cfg_reg, pin_out, enable);
+}
+
+static void qcom_pdc_gic_secondary_disable(struct irq_data *d)
+{
+	pdc->enable_intr(d->hwirq, false);
+	pdc->unmask_gpio(d->hwirq, false);
+	irq_chip_disable_parent(d);
 }
 
 static void qcom_pdc_gic_disable(struct irq_data *d)
 {
-	pdc_enable_intr(d, false);
+	pdc->enable_intr(d->hwirq, false);
 	irq_chip_disable_parent(d);
 }
 
 static void qcom_pdc_gic_enable(struct irq_data *d)
 {
-	pdc_enable_intr(d, true);
+	pdc->enable_intr(d->hwirq, true);
+	irq_chip_enable_parent(d);
+}
+
+static void qcom_pdc_gic_secondary_enable(struct irq_data *d)
+{
+	pdc->enable_intr(d->hwirq, true);
+	pdc->unmask_gpio(d->hwirq, true);
 	irq_chip_enable_parent(d);
 }
 
+static void qcom_pdc_secondary_ack(struct irq_data *d)
+{
+	if (!irqd_is_level_type(d))
+		pdc->clear_gpio(d->hwirq);
+}
+
+static void qcom_pdc_gic_secondary_eoi(struct irq_data *d)
+{
+	if (irqd_is_level_type(d))
+		pdc->clear_gpio(d->hwirq);
+
+	irq_chip_eoi_parent(d);
+}
+
+static void qcom_pdc_secondary_mask(struct irq_data *d)
+{
+	pdc->enable_intr(d->hwirq, false);
+	pdc->unmask_gpio(d->hwirq, false);
+	irq_chip_mask_parent(d);
+}
+
+static void qcom_pdc_secondary_unmask(struct irq_data *d)
+{
+	pdc->enable_intr(d->hwirq, true);
+	pdc->unmask_gpio(d->hwirq, true);
+	irq_chip_unmask_parent(d);
+}
+
 /*
  * GIC does not handle falling edge or active low. To allow falling edge and
  * active low interrupts to be handled at GIC, PDC has an inverter that inverts
@@ -186,18 +364,18 @@ enum pdc_irq_config_bits {
 /**
  * qcom_pdc_gic_set_type: Configure PDC for the interrupt
  *
- * @d: the interrupt data
+ * @d:    the interrupt data
  * @type: the interrupt type
  *
- * If @type is edge triggered, forward that as Rising edge as PDC
- * takes care of converting falling edge to rising edge signal
+ * If @type is edge triggered, forward that as rising edge as PDC
+ * takes care of converting all edge types to rising edge signal
  * If @type is level, then forward that as level high as PDC
- * takes care of converting falling edge to rising edge signal
+ * takes care of converting all level types to level high signal
  */
 static int qcom_pdc_gic_set_type(struct irq_data *d, unsigned int type)
 {
-	enum pdc_irq_config_bits pdc_type;
 	enum pdc_irq_config_bits old_pdc_type;
+	enum pdc_irq_config_bits pdc_type;
 	int ret;
 
 	switch (type) {
@@ -224,9 +402,75 @@ static int qcom_pdc_gic_set_type(struct irq_data *d, unsigned int type)
 		return -EINVAL;
 	}
 
-	old_pdc_type = pdc_reg_read(IRQ_i_CFG, d->hwirq);
-	pdc_type |= (old_pdc_type & ~IRQ_i_CFG_TYPE_MASK);
-	pdc_reg_write(IRQ_i_CFG, d->hwirq, pdc_type);
+	old_pdc_type = pdc_reg_read(pdc->regs->irq_cfg_reg, d->hwirq);
+	pdc_type |= (old_pdc_type & ~pdc->cfg_fields->irq_type);
+	pdc_reg_write(pdc->regs->irq_cfg_reg, d->hwirq, pdc_type);
+
+	ret = irq_chip_set_type_parent(d, type);
+	if (ret)
+		return ret;
+
+	/*
+	 * When we change types the PDC can give a phantom interrupt.
+	 * Clear it.  Specifically the phantom shows up when reconfiguring
+	 * polarity of interrupt without changing the state of the signal
+	 * but let's be consistent and clear it always.
+	 *
+	 * Doing this works because we have IRQCHIP_SET_TYPE_MASKED so the
+	 * interrupt will be cleared before the rest of the system sees it.
+	 */
+	if (old_pdc_type != pdc_type)
+		irq_chip_set_parent_state(d, IRQCHIP_STATE_PENDING, false);
+
+	return 0;
+}
+
+/**
+ * qcom_pdc_gic_secondary_set_type: Configure PDC for the interrupt in secondary mode
+ *
+ * @d:    the interrupt data
+ * @type: the interrupt type
+ *
+ * All @type are forwarded as level high type to parent GIC
+ */
+static int qcom_pdc_gic_secondary_set_type(struct irq_data *d, unsigned int type)
+{
+	enum pdc_irq_config_bits old_pdc_type;
+	enum pdc_irq_config_bits pdc_type;
+	int ret;
+
+	switch (type) {
+	case IRQ_TYPE_EDGE_RISING:
+		pdc_type = PDC_EDGE_RISING;
+		break;
+	case IRQ_TYPE_EDGE_FALLING:
+		pdc_type = PDC_EDGE_FALLING;
+		break;
+	case IRQ_TYPE_EDGE_BOTH:
+		pdc_type = PDC_EDGE_DUAL;
+		break;
+	case IRQ_TYPE_LEVEL_HIGH:
+		pdc_type = PDC_LEVEL_HIGH;
+		break;
+	case IRQ_TYPE_LEVEL_LOW:
+		pdc_type = PDC_LEVEL_LOW;
+		break;
+	default:
+		WARN_ON(1);
+		return -EINVAL;
+	}
+
+	old_pdc_type = pdc_reg_read(pdc->regs->irq_cfg_reg, d->hwirq);
+	pdc_type |= (old_pdc_type & ~pdc->cfg_fields->irq_type);
+	pdc_reg_write(pdc->regs->irq_cfg_reg, d->hwirq, pdc_type);
+
+	/*
+	 * PDC forwards GPIOs as level high to GIC in secondary
+	 * mode. Update the type and clear any previously latched
+	 * phantom interrupt at PDC.
+	 */
+	type = IRQ_TYPE_LEVEL_HIGH;
+	pdc->clear_gpio(d->hwirq);
 
 	ret = irq_chip_set_type_parent(d, type);
 	if (ret)
@@ -266,14 +510,32 @@ static struct irq_chip qcom_pdc_gic_chip = {
 	.irq_set_affinity	= irq_chip_set_affinity_parent,
 };
 
+static struct irq_chip qcom_pdc_gic_secondary_chip = {
+	.name			= "PDC",
+	.irq_ack		= qcom_pdc_secondary_ack,
+	.irq_eoi		= qcom_pdc_gic_secondary_eoi,
+	.irq_mask		= qcom_pdc_secondary_mask,
+	.irq_unmask		= qcom_pdc_secondary_unmask,
+	.irq_disable		= qcom_pdc_gic_secondary_disable,
+	.irq_enable		= qcom_pdc_gic_secondary_enable,
+	.irq_get_irqchip_state	= irq_chip_get_parent_state,
+	.irq_set_irqchip_state	= irq_chip_set_parent_state,
+	.irq_retrigger		= irq_chip_retrigger_hierarchy,
+	.irq_set_type		= qcom_pdc_gic_secondary_set_type,
+	.flags			= IRQCHIP_MASK_ON_SUSPEND |
+				  IRQCHIP_SET_TYPE_MASKED |
+				  IRQCHIP_SKIP_SET_WAKE |
+				  IRQCHIP_ENABLE_WAKEUP_ON_SUSPEND,
+	.irq_set_vcpu_affinity	= irq_chip_set_vcpu_affinity_parent,
+	.irq_set_affinity	= irq_chip_set_affinity_parent,
+};
+
 static struct pdc_pin_region *get_pin_region(int pin)
 {
-	int i;
-
-	for (i = 0; i < pdc_region_cnt; i++) {
-		if (pin >= pdc_region[i].pin_base &&
-		    pin < pdc_region[i].pin_base + pdc_region[i].cnt)
-			return &pdc_region[i];
+	for (int i = 0; i < pdc->region_cnt; i++) {
+		if (pin >= pdc->region[i].pin_base &&
+		    pin < pdc->region[i].pin_base + pdc->region[i].cnt)
+			return &pdc->region[i];
 	}
 
 	return NULL;
@@ -301,24 +563,45 @@ static int qcom_pdc_alloc(struct irq_domain *domain, unsigned int virq,
 	if (ret)
 		return ret;
 
+	/*
+	 * PDC secondary chip is only set for the GPIO interrupts as SPIs.
+	 * Direct SPI interrupts are still in pass through mode (no latching
+	 * at PDC).
+	 */
+	if (pdc->mode == PDC_SECONDARY_MODE && pdc_pin_is_gpio(hwirq)) {
+		ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq,
+						    &qcom_pdc_gic_secondary_chip,
+						    NULL);
+		if (ret)
+			return ret;
+
+		/* Secondary mode converts all interrupts to LEVEL HIGH type */
+		type = IRQ_TYPE_LEVEL_HIGH;
+	} else {
+		ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq,
+						    &qcom_pdc_gic_chip,
+						    NULL);
+		if (ret)
+			return ret;
+
+		if (type & IRQ_TYPE_EDGE_BOTH)
+			type = IRQ_TYPE_EDGE_RISING;
+
+		if (type & IRQ_TYPE_LEVEL_MASK)
+			type = IRQ_TYPE_LEVEL_HIGH;
+	}
+
 	region = get_pin_region(hwirq);
 	if (!region)
 		return irq_domain_disconnect_hierarchy(domain->parent, virq);
 
-	if (type & IRQ_TYPE_EDGE_BOTH)
-		type = IRQ_TYPE_EDGE_RISING;
-
-	if (type & IRQ_TYPE_LEVEL_MASK)
-		type = IRQ_TYPE_LEVEL_HIGH;
-
 	parent_fwspec.fwnode      = domain->parent->fwnode;
 	parent_fwspec.param_count = 3;
 	parent_fwspec.param[0]    = 0;
 	parent_fwspec.param[1]    = pin_to_hwirq(region, hwirq);
 	parent_fwspec.param[2]    = type;
 
-	return irq_domain_alloc_irqs_parent(domain, virq, nr_irqs,
-					    &parent_fwspec);
+	return irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &parent_fwspec);
 }
 
 static const struct irq_domain_ops qcom_pdc_ops = {
@@ -327,52 +610,56 @@ static const struct irq_domain_ops qcom_pdc_ops = {
 	.free		= irq_domain_free_irqs_common,
 };
 
-static int pdc_setup_pin_mapping(struct device_node *np)
+static int pdc_setup_pin_mapping(struct device *dev)
 {
-	int ret, n, i;
+	struct device_node *np = dev->of_node;
+	int ret, n;
 
 	n = of_property_count_elems_of_size(np, "qcom,pdc-ranges", sizeof(u32));
 	if (n <= 0 || n % 3)
 		return -EINVAL;
 
-	pdc_region_cnt = n / 3;
-	pdc_region = kzalloc_objs(*pdc_region, pdc_region_cnt);
-	if (!pdc_region) {
-		pdc_region_cnt = 0;
+	pdc->region_cnt = n / 3;
+	pdc->region = devm_kcalloc(dev, pdc->region_cnt, sizeof(*pdc->region), GFP_KERNEL);
+	if (!pdc->region) {
+		pdc->region_cnt = 0;
 		return -ENOMEM;
 	}
 
-	for (n = 0; n < pdc_region_cnt; n++) {
-		ret = of_property_read_u32_index(np, "qcom,pdc-ranges",
-						 n * 3 + 0,
-						 &pdc_region[n].pin_base);
+	for (n = 0; n < pdc->region_cnt; n++) {
+		ret = of_property_read_u32_index(np, "qcom,pdc-ranges", n * 3 + 0,
+						 &pdc->region[n].pin_base);
 		if (ret)
 			return ret;
-		ret = of_property_read_u32_index(np, "qcom,pdc-ranges",
-						 n * 3 + 1,
-						 &pdc_region[n].parent_base);
+		ret = of_property_read_u32_index(np, "qcom,pdc-ranges", n * 3 + 1,
+						 &pdc->region[n].parent_base);
 		if (ret)
 			return ret;
-		ret = of_property_read_u32_index(np, "qcom,pdc-ranges",
-						 n * 3 + 2,
-						 &pdc_region[n].cnt);
+		ret = of_property_read_u32_index(np, "qcom,pdc-ranges", n * 3 + 2,
+						 &pdc->region[n].cnt);
 		if (ret)
 			return ret;
 
-		for (i = 0; i < pdc_region[n].cnt; i++)
-			__pdc_enable_intr(i + pdc_region[n].pin_base, 0);
+		for (int i = 0; i < pdc->region[n].cnt; i++) {
+			if (pdc_pin_is_gpio(i + pdc->region[n].pin_base) &&
+			    pdc->mode == PDC_SECONDARY_MODE)
+				pdc->clear_gpio(i + pdc->region[n].pin_base);
+
+			pdc->enable_intr(i + pdc->region[n].pin_base, false);
+		}
 	}
 
 	return 0;
 }
 
-
 static int qcom_pdc_probe(struct platform_device *pdev, struct device_node *parent)
 {
 	struct irq_domain *parent_domain, *pdc_domain;
 	struct device_node *node = pdev->dev.of_node;
+	struct device *dev = &pdev->dev;
 	resource_size_t res_size;
 	struct resource res;
+	u32 irq_param;
 	int ret;
 
 	/* compat with old sm8150 DT which had very small region for PDC */
@@ -383,6 +670,34 @@ static int qcom_pdc_probe(struct platform_device *pdev, struct device_node *pare
 	if (res_size > resource_size(&res))
 		pr_warn("%pOF: invalid reg size, please fix DT\n", node);
 
+	pdc = devm_kzalloc(dev, sizeof(*pdc), GFP_KERNEL);
+	if (!pdc)
+		return -ENOMEM;
+
+	pdc->base = devm_ioremap(dev, res.start, res_size);
+	if (!pdc->base) {
+		pr_err("%pOF: unable to map PDC registers\n", node);
+		return -ENXIO;
+	}
+
+	pdc->version = pdc_reg_read(PDC_VERSION_REG, 0);
+
+	if (pdc->version >= PDC_VERSION_3_2) {
+		pdc->cfg_fields = &pdc_cfg_v3_2;
+		pdc->regs = &pdc_v3_2;
+		pdc->enable_intr = pdc_enable_intr_cfg;
+	} else if (pdc->version >= PDC_VERSION_3_0) {
+		pdc->cfg_fields = &pdc_cfg_v3_0;
+		pdc->regs = &pdc_v3_0;
+		pdc->enable_intr = pdc_enable_intr_bank;
+	} else {
+		pdc->cfg_fields = &pdc_cfg_v2_7;
+		pdc->regs = &pdc_v2_7;
+		pdc->enable_intr = pdc_enable_intr_bank;
+	}
+
+	pdc->mode = PDC_PASS_THROUGH_MODE;
+
 	/*
 	 * PDC has multiple DRV regions, each one provides the same set of
 	 * registers for a particular client in the system. Due to a hardware
@@ -392,57 +707,54 @@ static int qcom_pdc_probe(struct platform_device *pdev, struct device_node *pare
 	 * region with the expected offset to preserve support for old DTs.
 	 */
 	if (of_device_is_compatible(node, "qcom,x1e80100-pdc")) {
-		pdc_prev_base = ioremap(res.start - PDC_DRV_SIZE, IRQ_ENABLE_BANK_MAX);
-		if (!pdc_prev_base) {
+		pdc->prev_base = devm_ioremap(dev, res.start - PDC_DRV_SIZE,
+					      pdc->regs->irq_en_reg + IRQ_ENABLE_BANK_MAX);
+		if (!pdc->prev_base) {
 			pr_err("%pOF: unable to map previous PDC DRV region\n", node);
 			return -ENXIO;
 		}
 
-		pdc_x1e_quirk = true;
-	}
+		pdc->x1e_quirk = true;
 
-	pdc_base = ioremap(res.start, res_size);
-	if (!pdc_base) {
-		pr_err("%pOF: unable to map PDC registers\n", node);
-		ret = -ENXIO;
-		goto fail;
+		if (!qcom_scm_is_available())
+			return -EPROBE_DEFER;
+
+		ret = qcom_scm_io_writel(PDC_GPIO_INT_CTL_ENABLE, PDC_PASS_THROUGH_MODE);
+		if (ret) {
+			pdc->mode = PDC_SECONDARY_MODE;
+			pdc->unmask_gpio = pdc_unmask_gpio_cfg;
+			pdc->clear_gpio = pdc_clear_gpio_cfg;
+		}
 	}
 
-	pdc_version = pdc_reg_read(PDC_VERSION_REG, 0);
+	irq_param = pdc_reg_read(pdc->regs->irq_param_reg, 0);
+	pdc->num_spis = FIELD_GET(GENMASK(7, 0), irq_param);
 
 	parent_domain = irq_find_host(parent);
 	if (!parent_domain) {
 		pr_err("%pOF: unable to find PDC's parent domain\n", node);
-		ret = -ENXIO;
-		goto fail;
+		return -ENXIO;
 	}
 
-	ret = pdc_setup_pin_mapping(node);
+	raw_spin_lock_init(&pdc->lock);
+
+	ret = pdc_setup_pin_mapping(dev);
 	if (ret) {
 		pr_err("%pOF: failed to init PDC pin-hwirq mapping\n", node);
-		goto fail;
+		return ret;
 	}
 
-	pdc_domain = irq_domain_create_hierarchy(parent_domain,
-					IRQ_DOMAIN_FLAG_QCOM_PDC_WAKEUP,
-					PDC_MAX_GPIO_IRQS,
-					of_fwnode_handle(node),
-					&qcom_pdc_ops, NULL);
+	pdc_domain = irq_domain_create_hierarchy(parent_domain, IRQ_DOMAIN_FLAG_QCOM_PDC_WAKEUP,
+						 PDC_MAX_IRQS, of_fwnode_handle(node),
+						 &qcom_pdc_ops, NULL);
 	if (!pdc_domain) {
 		pr_err("%pOF: PDC domain add failed\n", node);
-		ret = -ENOMEM;
-		goto fail;
+		return -ENOMEM;
 	}
 
 	irq_domain_update_bus_token(pdc_domain, DOMAIN_BUS_WAKEUP);
 
 	return 0;
-
-fail:
-	kfree(pdc_region);
-	iounmap(pdc_base);
-	iounmap(pdc_prev_base);
-	return ret;
 }
 
 IRQCHIP_PLATFORM_DRIVER_BEGIN(qcom_pdc)
diff --git a/drivers/leds/leds-pca9532.c b/drivers/leds/leds-pca9532.c
index f3bf59495b68d..2d37e00e459de 100644
--- a/drivers/leds/leds-pca9532.c
+++ b/drivers/leds/leds-pca9532.c
@@ -327,9 +327,9 @@ static int pca9532_gpio_set_value(struct gpio_chip *gc, unsigned int offset,
 	struct pca9532_led *led = &data->leds[offset];
 
 	if (val)
-		led->state = PCA9532_ON;
-	else
 		led->state = PCA9532_OFF;
+	else
+		led->state = PCA9532_ON;
 
 	pca9532_setled(led);
 
@@ -349,7 +349,7 @@ static int pca9532_gpio_get_value(struct gpio_chip *gc, unsigned offset)
 static int pca9532_gpio_direction_input(struct gpio_chip *gc, unsigned offset)
 {
 	/* To use as input ensure pin is not driven */
-	pca9532_gpio_set_value(gc, offset, 0);
+	pca9532_gpio_set_value(gc, offset, 1);
 
 	return 0;
 }
diff --git a/drivers/leds/leds-st1202.c b/drivers/leds/leds-st1202.c
index 7f68d956f6948..168df5ecf27b7 100644
--- a/drivers/leds/leds-st1202.c
+++ b/drivers/leds/leds-st1202.c
@@ -31,10 +31,11 @@
 #define ST1202_ILED_REG0                   0x09
 #define ST1202_MAX_LEDS                    12
 #define ST1202_MAX_PATTERNS                8
-#define ST1202_MILLIS_PATTERN_DUR_MAX      5660
+#define ST1202_MILLIS_PATTERN_DUR_MAX      (ST1202_MILLIS_PATTERN_DUR_MIN * U8_MAX)
 #define ST1202_MILLIS_PATTERN_DUR_MIN      22
 #define ST1202_PATTERN_DUR                 0x16
 #define ST1202_PATTERN_PWM                 0x1E
+#define ST1202_PATTERN_PWM_FULL            0x0FFF
 #define ST1202_PATTERN_REP                 0x15
 
 struct st1202_led {
@@ -85,7 +86,7 @@ static int st1202_write_reg(struct st1202_chip *chip, int reg, uint8_t val)
 
 static uint8_t st1202_prescalar_to_miliseconds(unsigned int value)
 {
-	return value / ST1202_MILLIS_PATTERN_DUR_MIN - 1;
+	return value / ST1202_MILLIS_PATTERN_DUR_MIN;
 }
 
 static int st1202_pwm_pattern_write(struct st1202_chip *chip, int led_num,
@@ -127,37 +128,11 @@ static int st1202_duration_pattern_write(struct st1202_chip *chip, int pattern,
 				st1202_prescalar_to_miliseconds(value));
 }
 
-static void st1202_brightness_set(struct led_classdev *led_cdev,
-				enum led_brightness value)
-{
-	struct st1202_led *led = cdev_to_st1202_led(led_cdev);
-	struct st1202_chip *chip = led->chip;
-
-	guard(mutex)(&chip->lock);
-
-	st1202_write_reg(chip, ST1202_ILED_REG0 + led->led_num, value);
-}
-
-static enum led_brightness st1202_brightness_get(struct led_classdev *led_cdev)
-{
-	struct st1202_led *led = cdev_to_st1202_led(led_cdev);
-	struct st1202_chip *chip = led->chip;
-	u8 value = 0;
-
-	guard(mutex)(&chip->lock);
-
-	st1202_read_reg(chip, ST1202_ILED_REG0 + led->led_num, &value);
-
-	return value;
-}
-
-static int st1202_channel_set(struct st1202_chip *chip, int led_num, bool active)
+static int __st1202_channel_set(struct st1202_chip *chip, int led_num, bool active)
 {
 	u8 chan_low, chan_high;
 	int ret;
 
-	guard(mutex)(&chip->lock);
-
 	if (led_num <= 7) {
 		ret = st1202_read_reg(chip, ST1202_CHAN_ENABLE_LOW, &chan_low);
 		if (ret < 0)
@@ -185,6 +160,40 @@ static int st1202_channel_set(struct st1202_chip *chip, int led_num, bool active
 	return 0;
 }
 
+static int st1202_channel_set(struct st1202_chip *chip, int led_num, bool active)
+{
+	guard(mutex)(&chip->lock);
+
+	return __st1202_channel_set(chip, led_num, active);
+}
+
+static void st1202_brightness_set(struct led_classdev *led_cdev,
+				enum led_brightness value)
+{
+	struct st1202_led *led = cdev_to_st1202_led(led_cdev);
+	struct st1202_chip *chip = led->chip;
+
+	guard(mutex)(&chip->lock);
+
+	for (int patt = 0; patt < ST1202_MAX_PATTERNS; patt++)
+		st1202_pwm_pattern_write(chip, led->led_num, patt, ST1202_PATTERN_PWM_FULL);
+	st1202_write_reg(chip, ST1202_ILED_REG0 + led->led_num, value);
+	__st1202_channel_set(chip, led->led_num, !!value);
+}
+
+static enum led_brightness st1202_brightness_get(struct led_classdev *led_cdev)
+{
+	struct st1202_led *led = cdev_to_st1202_led(led_cdev);
+	struct st1202_chip *chip = led->chip;
+	u8 value = 0;
+
+	guard(mutex)(&chip->lock);
+
+	st1202_read_reg(chip, ST1202_ILED_REG0 + led->led_num, &value);
+
+	return value;
+}
+
 static int st1202_led_set(struct led_classdev *ldev, enum led_brightness value)
 {
 	struct st1202_led *led = cdev_to_st1202_led(ldev);
@@ -200,12 +209,16 @@ static int st1202_led_pattern_clear(struct led_classdev *ldev)
 
 	guard(mutex)(&chip->lock);
 
+	ret = st1202_write_reg(chip, ST1202_CONFIG_REG, ST1202_CONFIG_REG_SHFT);
+	if (ret != 0)
+		return ret;
+
 	for (int patt = 0; patt < ST1202_MAX_PATTERNS; patt++) {
-		ret = st1202_pwm_pattern_write(chip, led->led_num, patt, LED_OFF);
+		ret = st1202_pwm_pattern_write(chip, led->led_num, patt, ST1202_PATTERN_PWM_FULL);
 		if (ret != 0)
 			return ret;
 
-		ret = st1202_duration_pattern_write(chip, patt, ST1202_MILLIS_PATTERN_DUR_MIN);
+		ret = st1202_write_reg(chip, ST1202_PATTERN_DUR + patt, 0);
 		if (ret != 0)
 			return ret;
 	}
@@ -224,13 +237,19 @@ static int st1202_led_pattern_set(struct led_classdev *ldev,
 	if (len > ST1202_MAX_PATTERNS)
 		return -EINVAL;
 
-	guard(mutex)(&chip->lock);
-
 	for (int patt = 0; patt < len; patt++) {
 		if (pattern[patt].delta_t < ST1202_MILLIS_PATTERN_DUR_MIN ||
 				pattern[patt].delta_t > ST1202_MILLIS_PATTERN_DUR_MAX)
 			return -EINVAL;
+	}
 
+	guard(mutex)(&chip->lock);
+
+	ret = st1202_write_reg(chip, ST1202_CONFIG_REG, ST1202_CONFIG_REG_SHFT);
+	if (ret != 0)
+		return ret;
+
+	for (int patt = 0; patt < len; patt++) {
 		ret = st1202_pwm_pattern_write(chip, led->led_num, patt, pattern[patt].brightness);
 		if (ret != 0)
 			return ret;
@@ -244,6 +263,10 @@ static int st1202_led_pattern_set(struct led_classdev *ldev,
 	if (ret != 0)
 		return ret;
 
+	ret = __st1202_channel_set(chip, led->led_num, true);
+	if (ret != 0)
+		return ret;
+
 	ret = st1202_write_reg(chip, ST1202_CONFIG_REG, (ST1202_CONFIG_REG_PATSR |
 							ST1202_CONFIG_REG_PATS | ST1202_CONFIG_REG_SHFT));
 	if (ret != 0)
@@ -256,13 +279,19 @@ static int st1202_dt_init(struct st1202_chip *chip)
 {
 	struct device *dev = &chip->client->dev;
 	struct st1202_led *led;
-	int err, reg;
+	int err;
+	u32 reg;
 
 	for_each_available_child_of_node_scoped(dev_of_node(dev), child) {
 		err = of_property_read_u32(child, "reg", &reg);
 		if (err)
 			return dev_err_probe(dev, err, "Invalid register\n");
 
+		if (reg >= ST1202_MAX_LEDS)
+			return dev_err_probe(dev, -EINVAL,
+					"LED reg %u out of range [0, %d]\n",
+					reg, ST1202_MAX_LEDS - 1);
+
 		led = &chip->leds[reg];
 		led->is_active = true;
 		led->fwnode = of_fwnode_handle(child);
@@ -322,11 +351,6 @@ static int st1202_setup(struct st1202_chip *chip)
 	if (ret < 0)
 		return ret;
 
-	ret = st1202_write_reg(chip, ST1202_CONFIG_REG,
-				ST1202_CONFIG_REG_PATS | ST1202_CONFIG_REG_PATSR);
-	if (ret < 0)
-		return ret;
-
 	return 0;
 }
 
diff --git a/drivers/leds/rgb/leds-lp5860-core.c b/drivers/leds/rgb/leds-lp5860-core.c
index fd0e2f6e6e0f4..e21d5f2302be0 100644
--- a/drivers/leds/rgb/leds-lp5860-core.c
+++ b/drivers/leds/rgb/leds-lp5860-core.c
@@ -204,9 +204,9 @@ int lp5860_device_init(struct device *dev)
 	mutex_lock(&lp->lock);
 	ret = regmap_update_bits(lp->regmap, LP5860_REG_DEV_INITIAL, LP5860_MODE_MASK,
 				 LP5860_MODE_1 << LP5860_MODE_SHIFT);
+	mutex_unlock(&lp->lock);
 	if (ret)
 		goto err_disable;
-	mutex_unlock(&lp->lock);
 
 	ret = lp5860_init_dt(lp);
 	if (ret)
@@ -215,7 +215,6 @@ int lp5860_device_init(struct device *dev)
 	return 0;
 
 err_disable:
-	mutex_unlock(&lp->lock);
 	lp5860_chip_enable(lp, LP5860_CHIP_DISABLE);
 	return ret;
 }
diff --git a/drivers/leds/rgb/leds-lp5860-spi.c b/drivers/leds/rgb/leds-lp5860-spi.c
index 5e0c44854a680..6bf6a625c28af 100644
--- a/drivers/leds/rgb/leds-lp5860-spi.c
+++ b/drivers/leds/rgb/leds-lp5860-spi.c
@@ -38,6 +38,7 @@ static int lp5860_probe(struct spi_device *spi)
 	struct device *dev = &spi->dev;
 	struct lp5860 *lp5860;
 	unsigned int multi_leds;
+	int ret;
 
 	multi_leds = device_get_child_node_count(dev);
 	if (!multi_leds) {
@@ -61,7 +62,10 @@ static int lp5860_probe(struct spi_device *spi)
 				     "Failed to initialise Regmap.\n");
 
 	lp5860->dev = dev;
-	mutex_init(&lp5860->lock);
+
+	ret = devm_mutex_init(dev, &lp5860->lock);
+	if (ret)
+		return ret;
 
 	spi_set_drvdata(spi, lp5860);
 
@@ -70,10 +74,6 @@ static int lp5860_probe(struct spi_device *spi)
 
 static void lp5860_remove(struct spi_device *spi)
 {
-	struct lp5860 *lp5860 = spi_get_drvdata(spi);
-
-	mutex_destroy(&lp5860->lock);
-
 	lp5860_device_remove(&spi->dev);
 }
 
diff --git a/drivers/leds/rgb/leds-pwm-multicolor.c b/drivers/leds/rgb/leds-pwm-multicolor.c
index d5b303aab5d61..6365459ec27f1 100644
--- a/drivers/leds/rgb/leds-pwm-multicolor.c
+++ b/drivers/leds/rgb/leds-pwm-multicolor.c
@@ -94,6 +94,8 @@ static int iterate_subleds(struct device *dev, struct pwm_mc_led *priv,
 		}
 
 		subled[priv->mc_cdev.num_colors].color_index = color;
+		fwnode_property_read_u32(fwnode, "default-intensity",
+					 &subled[priv->mc_cdev.num_colors].intensity);
 		priv->mc_cdev.num_colors++;
 	}
 
diff --git a/drivers/leds/trigger/ledtrig-netdev.c b/drivers/leds/trigger/ledtrig-netdev.c
index 64c078e997f28..5b4e92c14dbb4 100644
--- a/drivers/leds/trigger/ledtrig-netdev.c
+++ b/drivers/leds/trigger/ledtrig-netdev.c
@@ -129,6 +129,22 @@ static void set_baseline_state(struct led_netdev_data *trigger_data)
 		    trigger_data->link_speed == SPEED_10000)
 			blink_on = true;
 
+		if (test_bit(TRIGGER_NETDEV_LINK_25000, &trigger_data->mode) &&
+		    trigger_data->link_speed == SPEED_25000)
+			blink_on = true;
+
+		if (test_bit(TRIGGER_NETDEV_LINK_40000, &trigger_data->mode) &&
+		    trigger_data->link_speed == SPEED_40000)
+			blink_on = true;
+
+		if (test_bit(TRIGGER_NETDEV_LINK_50000, &trigger_data->mode) &&
+		    trigger_data->link_speed == SPEED_50000)
+			blink_on = true;
+
+		if (test_bit(TRIGGER_NETDEV_LINK_100000, &trigger_data->mode) &&
+		    trigger_data->link_speed == SPEED_100000)
+			blink_on = true;
+
 		if (test_bit(TRIGGER_NETDEV_HALF_DUPLEX, &trigger_data->mode) &&
 		    trigger_data->duplex == DUPLEX_HALF)
 			blink_on = true;
@@ -342,6 +358,10 @@ static ssize_t netdev_led_attr_show(struct device *dev, char *buf,
 	case TRIGGER_NETDEV_LINK_2500:
 	case TRIGGER_NETDEV_LINK_5000:
 	case TRIGGER_NETDEV_LINK_10000:
+	case TRIGGER_NETDEV_LINK_25000:
+	case TRIGGER_NETDEV_LINK_40000:
+	case TRIGGER_NETDEV_LINK_50000:
+	case TRIGGER_NETDEV_LINK_100000:
 	case TRIGGER_NETDEV_HALF_DUPLEX:
 	case TRIGGER_NETDEV_FULL_DUPLEX:
 	case TRIGGER_NETDEV_TX:
@@ -378,6 +398,10 @@ static ssize_t netdev_led_attr_store(struct device *dev, const char *buf,
 	case TRIGGER_NETDEV_LINK_2500:
 	case TRIGGER_NETDEV_LINK_5000:
 	case TRIGGER_NETDEV_LINK_10000:
+	case TRIGGER_NETDEV_LINK_25000:
+	case TRIGGER_NETDEV_LINK_40000:
+	case TRIGGER_NETDEV_LINK_50000:
+	case TRIGGER_NETDEV_LINK_100000:
 	case TRIGGER_NETDEV_HALF_DUPLEX:
 	case TRIGGER_NETDEV_FULL_DUPLEX:
 	case TRIGGER_NETDEV_TX:
@@ -401,7 +425,11 @@ static ssize_t netdev_led_attr_store(struct device *dev, const char *buf,
 	     test_bit(TRIGGER_NETDEV_LINK_1000, &mode) ||
 	     test_bit(TRIGGER_NETDEV_LINK_2500, &mode) ||
 	     test_bit(TRIGGER_NETDEV_LINK_5000, &mode) ||
-	     test_bit(TRIGGER_NETDEV_LINK_10000, &mode)))
+	     test_bit(TRIGGER_NETDEV_LINK_10000, &mode) ||
+	     test_bit(TRIGGER_NETDEV_LINK_25000, &mode) ||
+	     test_bit(TRIGGER_NETDEV_LINK_40000, &mode) ||
+	     test_bit(TRIGGER_NETDEV_LINK_50000, &mode) ||
+	     test_bit(TRIGGER_NETDEV_LINK_100000, &mode)))
 		return -EINVAL;
 
 	cancel_delayed_work_sync(&trigger_data->work);
@@ -438,6 +466,10 @@ DEFINE_NETDEV_TRIGGER(link_1000, TRIGGER_NETDEV_LINK_1000);
 DEFINE_NETDEV_TRIGGER(link_2500, TRIGGER_NETDEV_LINK_2500);
 DEFINE_NETDEV_TRIGGER(link_5000, TRIGGER_NETDEV_LINK_5000);
 DEFINE_NETDEV_TRIGGER(link_10000, TRIGGER_NETDEV_LINK_10000);
+DEFINE_NETDEV_TRIGGER(link_25000, TRIGGER_NETDEV_LINK_25000);
+DEFINE_NETDEV_TRIGGER(link_40000, TRIGGER_NETDEV_LINK_40000);
+DEFINE_NETDEV_TRIGGER(link_50000, TRIGGER_NETDEV_LINK_50000);
+DEFINE_NETDEV_TRIGGER(link_100000, TRIGGER_NETDEV_LINK_100000);
 DEFINE_NETDEV_TRIGGER(half_duplex, TRIGGER_NETDEV_HALF_DUPLEX);
 DEFINE_NETDEV_TRIGGER(full_duplex, TRIGGER_NETDEV_FULL_DUPLEX);
 DEFINE_NETDEV_TRIGGER(tx, TRIGGER_NETDEV_TX);
@@ -526,6 +558,10 @@ static umode_t netdev_trig_link_speed_visible(struct kobject *kobj,
 		CHECK_LINK_MODE_ATTR(2500);
 		CHECK_LINK_MODE_ATTR(5000);
 		CHECK_LINK_MODE_ATTR(10000);
+		CHECK_LINK_MODE_ATTR(25000);
+		CHECK_LINK_MODE_ATTR(40000);
+		CHECK_LINK_MODE_ATTR(50000);
+		CHECK_LINK_MODE_ATTR(100000);
 	}
 
 	return 0;
@@ -538,6 +574,10 @@ static struct attribute *netdev_trig_link_speed_attrs[] = {
 	&dev_attr_link_2500.attr,
 	&dev_attr_link_5000.attr,
 	&dev_attr_link_10000.attr,
+	&dev_attr_link_25000.attr,
+	&dev_attr_link_40000.attr,
+	&dev_attr_link_50000.attr,
+	&dev_attr_link_100000.attr,
 	NULL
 };
 
@@ -673,6 +713,10 @@ static void netdev_trig_work(struct work_struct *work)
 			 test_bit(TRIGGER_NETDEV_LINK_2500, &trigger_data->mode) ||
 			 test_bit(TRIGGER_NETDEV_LINK_5000, &trigger_data->mode) ||
 			 test_bit(TRIGGER_NETDEV_LINK_10000, &trigger_data->mode) ||
+			 test_bit(TRIGGER_NETDEV_LINK_25000, &trigger_data->mode) ||
+			 test_bit(TRIGGER_NETDEV_LINK_40000, &trigger_data->mode) ||
+			 test_bit(TRIGGER_NETDEV_LINK_50000, &trigger_data->mode) ||
+			 test_bit(TRIGGER_NETDEV_LINK_100000, &trigger_data->mode) ||
 			 test_bit(TRIGGER_NETDEV_HALF_DUPLEX, &trigger_data->mode) ||
 			 test_bit(TRIGGER_NETDEV_FULL_DUPLEX, &trigger_data->mode);
 		interval = jiffies_to_msecs(
diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c
index 097315a9bf0f1..33dbc71b730ff 100644
--- a/drivers/md/dm-cache-target.c
+++ b/drivers/md/dm-cache-target.c
@@ -16,6 +16,7 @@
 #include <linux/dm-kcopyd.h>
 #include <linux/jiffies.h>
 #include <linux/init.h>
+#include <linux/kstrtox.h>
 #include <linux/mempool.h>
 #include <linux/module.h>
 #include <linux/rwsem.h>
@@ -3311,42 +3312,46 @@ struct cblock_range {
 	dm_cblock_t end;
 };
 
+static inline dm_cblock_t cblock_succ(dm_cblock_t b)
+{
+	return to_cblock(from_cblock(b) + 1);
+}
+
 /*
  * A cache block range can take two forms:
  *
  * i) A single cblock, eg. '3456'
  * ii) A begin and end cblock with a dash between, eg. 123-234
  */
-static int parse_cblock_range(struct cache *cache, const char *str,
+static int parse_cblock_range(struct cache *cache, char *str,
 			      struct cblock_range *result)
 {
-	char dummy;
-	uint64_t b, e;
+	char *blocknr = strsep(&str, "-");
+	unsigned int b, e;
 	int r;
 
-	/*
-	 * Try and parse form (ii) first.
-	 */
-	r = sscanf(str, "%llu-%llu%c", &b, &e, &dummy);
+	r = kstrtouint(blocknr, 10, &b);
+	if (r)
+		goto bad;
 
-	if (r == 2) {
-		result->begin = to_cblock(b);
-		result->end = to_cblock(e);
-		return 0;
-	}
+	result->begin = to_cblock(b);
 
-	/*
-	 * That didn't work, try form (i).
-	 */
-	r = sscanf(str, "%llu%c", &b, &dummy);
+	if (str) {
+		blocknr = str;
 
-	if (r == 1) {
-		result->begin = to_cblock(b);
-		result->end = to_cblock(from_cblock(result->begin) + 1u);
-		return 0;
+		r = kstrtouint(blocknr, 10, &e);
+		if (r)
+			goto bad;
+
+		result->end = to_cblock(e);
+	} else {
+		result->end = cblock_succ(result->begin);
 	}
 
-	DMERR("%s: invalid cblock range '%s'", cache_device_name(cache), str);
+	return 0;
+
+bad:
+	DMERR("%s: invalid cblock range '%s'", cache_device_name(cache), blocknr);
 	return -EINVAL;
 }
 
@@ -3377,11 +3382,6 @@ static int validate_cblock_range(struct cache *cache, struct cblock_range *range
 	return 0;
 }
 
-static inline dm_cblock_t cblock_succ(dm_cblock_t b)
-{
-	return to_cblock(from_cblock(b) + 1);
-}
-
 static int request_invalidation(struct cache *cache, struct cblock_range *range)
 {
 	int r = 0;
@@ -3405,7 +3405,7 @@ static int request_invalidation(struct cache *cache, struct cblock_range *range)
 }
 
 static int process_invalidate_cblocks_message(struct cache *cache, unsigned int count,
-					      const char **cblock_ranges)
+					      char **cblock_ranges)
 {
 	int r = 0;
 	unsigned int i;
@@ -3460,7 +3460,7 @@ static int cache_message(struct dm_target *ti, unsigned int argc, char **argv,
 	}
 
 	if (!strcasecmp(argv[0], "invalidate_cblocks"))
-		return process_invalidate_cblocks_message(cache, argc - 1, (const char **) argv + 1);
+		return process_invalidate_cblocks_message(cache, argc - 1, argv + 1);
 
 	if (argc != 2)
 		return -EINVAL;
diff --git a/drivers/md/dm-inlinecrypt.c b/drivers/md/dm-inlinecrypt.c
index 41293c18d10f2..863cb471e95e2 100644
--- a/drivers/md/dm-inlinecrypt.c
+++ b/drivers/md/dm-inlinecrypt.c
@@ -347,7 +347,6 @@ static int inlinecrypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	err = get_key_size(&argv[1]);
 	if (err < 0) {
 		ti->error = "Cannot parse key size";
-		err = -EINVAL;
 		goto bad;
 	}
 	ctx->key_size = err;
diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c
index 1f2593f113f64..5156079785dcf 100644
--- a/drivers/md/dm-integrity.c
+++ b/drivers/md/dm-integrity.c
@@ -2782,7 +2782,8 @@ static void integrity_bio_wait(struct work_struct *w)
 		switch (r) {
 			case DM_MAPIO_KILL:
 				bio->bi_status = BLK_STS_IOERR;
-				fallthrough;
+				bio_endio(bio);
+				return;
 			case DM_MAPIO_REMAPPED:
 				submit_bio_noacct(bio);
 				fallthrough;
@@ -3040,7 +3041,7 @@ static void do_journal_write(struct dm_integrity_c *ic, unsigned int write_start
 				r = dm_integrity_rw_tag(ic, journal_entry_tag(ic, je2), &metadata_block, &metadata_offset,
 							ic->tag_size, TAG_WRITE);
 				if (unlikely(r))
-					dm_integrity_io_error(ic, "reading tags", r);
+					dm_integrity_io_error(ic, "writing tags", r);
 			}
 
 			atomic_inc(&comp.in_flight);
@@ -4634,6 +4635,7 @@ static int create_journal(struct dm_integrity_c *ic, char **error)
 	if (!ic->journal_tree) {
 		*error = "Could not allocate memory for journal tree";
 		r = -ENOMEM;
+		goto bad;
 	}
 bad:
 	kfree(crypt_data);
diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c
index 61af2a437a050..a6b8e97755cd5 100644
--- a/drivers/md/dm-ioctl.c
+++ b/drivers/md/dm-ioctl.c
@@ -54,10 +54,8 @@ struct hash_cell {
 };
 
 struct vers_iter {
-	size_t param_size;
 	struct dm_target_versions *vers, *old_vers;
 	char *end;
-	uint32_t flags;
 };
 
 
@@ -694,118 +692,108 @@ static int list_devices(struct file *filp, struct dm_ioctl *param, size_t param_
 {
 	struct rb_node *n;
 	struct hash_cell *hc;
-	size_t len, needed = 0;
-	struct gendisk *disk;
-	struct dm_name_list *orig_nl, *nl, *old_nl = NULL;
+	size_t len;
+	struct dm_name_list *nl, *old_nl = NULL;
+	void *result_start, *result_limit;
 	uint32_t *event_nr;
 
-	down_write(&_hash_lock);
-
-	/*
-	 * Loop through all the devices working out how much
-	 * space we need.
-	 */
-	for (n = rb_first(&name_rb_tree); n; n = rb_next(n)) {
-		hc = container_of(n, struct hash_cell, name_node);
-		if (!filter_device(hc, param->name, param->uuid))
-			continue;
-		needed += align_val(offsetof(struct dm_name_list, name) + strlen(hc->name) + 1);
-		needed += align_val(sizeof(uint32_t) * 2);
-		if (param->flags & DM_UUID_FLAG && hc->uuid)
-			needed += align_val(strlen(hc->uuid) + 1);
-	}
-
 	/*
 	 * Grab our output buffer.
 	 */
-	nl = orig_nl = get_result_buffer(param, param_size, &len);
-	if (len < needed || len < sizeof(nl->dev)) {
-		param->flags |= DM_BUFFER_FULL_FLAG;
-		goto out;
-	}
-	param->data_size = param->data_start + needed;
+	nl = result_start = get_result_buffer(param, param_size, &len);
+	result_limit = result_start + len;
 
-	nl->dev = 0;	/* Flags no data */
+	if (len >= sizeof(*nl))
+		nl->dev = 0;	/* Flags no data */
+
+	down_write(&_hash_lock);
 
 	/*
-	 * Now loop through filling out the names.
+	 * Loop through filling out the names.
 	 */
 	for (n = rb_first(&name_rb_tree); n; n = rb_next(n)) {
-		void *uuid_ptr;
+		void *next_nl;
 
 		hc = container_of(n, struct hash_cell, name_node);
 		if (!filter_device(hc, param->name, param->uuid))
 			continue;
-		if (old_nl)
-			old_nl->next = (uint32_t) ((void *) nl -
-						   (void *) old_nl);
-		disk = dm_disk(hc->md);
-		nl->dev = huge_encode_dev(disk_devt(disk));
-		nl->next = 0;
-		strcpy(nl->name, hc->name);
 
-		old_nl = nl;
-		event_nr = align_ptr(nl->name + strlen(hc->name) + 1);
+		len = strlen(hc->name);
+		event_nr = align_ptr(nl->name + len + 1);
+		next_nl = event_nr + 2;
+		if (next_nl > result_limit)
+			break;
+
+		((u64 *)event_nr)[-1] = 0;
+		memcpy(nl->name, hc->name, len);
+
+		nl->dev = huge_encode_dev(disk_devt(dm_disk(hc->md)));
+
 		event_nr[0] = dm_get_event_nr(hc->md);
 		event_nr[1] = 0;
-		uuid_ptr = align_ptr(event_nr + 2);
+
 		if (param->flags & DM_UUID_FLAG) {
 			if (hc->uuid) {
+				len = strlen(hc->uuid);
+				next_nl = align_ptr(next_nl + len + 1);
+				if (next_nl > result_limit)
+					break;
 				event_nr[1] |= DM_NAME_LIST_FLAG_HAS_UUID;
-				strcpy(uuid_ptr, hc->uuid);
-				uuid_ptr = align_ptr(uuid_ptr + strlen(hc->uuid) + 1);
+				((u64 *)next_nl)[-1] = 0;
+				memcpy(event_nr + 2, hc->uuid, len);
 			} else {
 				event_nr[1] |= DM_NAME_LIST_FLAG_DOESNT_HAVE_UUID;
 			}
 		}
-		nl = uuid_ptr;
+		nl->next = next_nl - (void *)nl;
+		old_nl = nl;
+		nl = next_nl;
 	}
-	/*
-	 * If mismatch happens, security may be compromised due to buffer
-	 * overflow, so it's better to crash.
-	 */
-	BUG_ON((char *)nl - (char *)orig_nl != needed);
 
- out:
-	up_write(&_hash_lock);
-	return 0;
-}
+	if (old_nl)
+		old_nl->next = 0;
 
-static void list_version_get_needed(struct target_type *tt, void *needed_param)
-{
-	size_t *needed = needed_param;
+	if (n)
+		param->flags |= DM_BUFFER_FULL_FLAG;
+	else
+		param->data_size = param->data_start + ((void *)nl - result_start);
 
-	*needed += sizeof(struct dm_target_versions);
-	*needed += strlen(tt->name) + 1;
-	*needed += ALIGN_MASK;
+	up_write(&_hash_lock);
+	return 0;
 }
 
 static void list_version_get_info(struct target_type *tt, void *param)
 {
 	struct vers_iter *info = param;
+	struct dm_target_versions *vers = info->vers;
+	size_t name_len = strlen(tt->name);
+
+	if (!vers)
+		return;
+
+	info->old_vers = vers;
+	info->vers = align_ptr((void *)(info->vers + 1) + name_len + 1);
 
-	/* Check space - it might have changed since the first iteration */
-	if ((char *)info->vers + sizeof(struct dm_target_versions) + strlen(tt->name) + 1 > info->end) {
-		info->flags = DM_BUFFER_FULL_FLAG;
+	/* Check space */
+	if ((char *)info->vers > info->end) {
+		info->vers = NULL;
 		return;
 	}
 
-	if (info->old_vers)
-		info->old_vers->next = (uint32_t) ((void *)info->vers - (void *)info->old_vers);
+	/* Zero padding and terminate vers->name[] */
+	((u64 *)info->vers)[-1] = 0;
 
-	info->vers->version[0] = tt->version[0];
-	info->vers->version[1] = tt->version[1];
-	info->vers->version[2] = tt->version[2];
-	info->vers->next = 0;
-	strcpy(info->vers->name, tt->name);
+	vers->next = (char *)info->vers - (char *)vers;
 
-	info->old_vers = info->vers;
-	info->vers = align_ptr((void *)(info->vers + 1) + strlen(tt->name) + 1);
+	vers->version[0] = tt->version[0];
+	vers->version[1] = tt->version[1];
+	vers->version[2] = tt->version[2];
+	memcpy(vers->name, tt->name, name_len);
 }
 
 static int __list_versions(struct dm_ioctl *param, size_t param_size, const char *name)
 {
-	size_t len, needed = 0;
+	size_t len;
 	struct dm_target_versions *vers;
 	struct vers_iter iter_info;
 	struct target_type *tt = NULL;
@@ -816,41 +804,31 @@ static int __list_versions(struct dm_ioctl *param, size_t param_size, const char
 			return -EINVAL;
 	}
 
-	/*
-	 * Loop through all the devices working out how much
-	 * space we need.
-	 */
-	if (!tt)
-		dm_target_iterate(list_version_get_needed, &needed);
-	else
-		list_version_get_needed(tt, &needed);
-
 	/*
 	 * Grab our output buffer.
 	 */
 	vers = get_result_buffer(param, param_size, &len);
-	if (len < needed) {
-		param->flags |= DM_BUFFER_FULL_FLAG;
-		goto out;
-	}
-	param->data_size = param->data_start + needed;
 
-	iter_info.param_size = param_size;
 	iter_info.old_vers = NULL;
 	iter_info.vers = vers;
-	iter_info.flags = 0;
-	iter_info.end = (char *)vers + needed;
+	iter_info.end = (char *)vers + len;
 
 	/*
-	 * Now loop through filling out the names & versions.
+	 * Loop through filling out the names & versions.
 	 */
 	if (!tt)
 		dm_target_iterate(list_version_get_info, &iter_info);
 	else
 		list_version_get_info(tt, &iter_info);
-	param->flags |= iter_info.flags;
 
- out:
+	if (iter_info.vers) {
+		if (iter_info.old_vers)
+			iter_info.old_vers->next = 0;
+		param->data_size = param->data_start + ((char *)iter_info.vers - (char *)vers);
+	} else {
+		param->flags |= DM_BUFFER_FULL_FLAG;
+	}
+
 	if (tt)
 		dm_put_target_type(tt);
 	return 0;
@@ -1444,10 +1422,6 @@ static void retrieve_status(struct dm_table *table,
 
 		outptr += sizeof(struct dm_target_spec);
 		remaining = len - (outptr - outbuf);
-		if (remaining <= 0) {
-			param->flags |= DM_BUFFER_FULL_FLAG;
-			break;
-		}
 
 		/* Get the status/table string from the target driver */
 		if (ti->type->status) {
@@ -1998,33 +1972,32 @@ static int target_message(struct file *filp, struct dm_ioctl *param, size_t para
 static ioctl_fn lookup_ioctl(unsigned int cmd, int *ioctl_flags)
 {
 	static const struct {
-		int cmd;
 		int flags;
 		ioctl_fn fn;
 	} _ioctls[] = {
-		{DM_VERSION_CMD, 0, NULL}, /* version is dealt with elsewhere */
-		{DM_REMOVE_ALL_CMD, IOCTL_FLAGS_NO_PARAMS | IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, remove_all},
-		{DM_LIST_DEVICES_CMD, 0, list_devices},
-
-		{DM_DEV_CREATE_CMD, IOCTL_FLAGS_NO_PARAMS | IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, dev_create},
-		{DM_DEV_REMOVE_CMD, IOCTL_FLAGS_NO_PARAMS | IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, dev_remove},
-		{DM_DEV_RENAME_CMD, IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, dev_rename},
-		{DM_DEV_SUSPEND_CMD, IOCTL_FLAGS_NO_PARAMS, dev_suspend},
-		{DM_DEV_STATUS_CMD, IOCTL_FLAGS_NO_PARAMS, dev_status},
-		{DM_DEV_WAIT_CMD, 0, dev_wait},
-
-		{DM_TABLE_LOAD_CMD, 0, table_load},
-		{DM_TABLE_CLEAR_CMD, IOCTL_FLAGS_NO_PARAMS, table_clear},
-		{DM_TABLE_DEPS_CMD, 0, table_deps},
-		{DM_TABLE_STATUS_CMD, 0, table_status},
-
-		{DM_LIST_VERSIONS_CMD, 0, list_versions},
-
-		{DM_TARGET_MSG_CMD, 0, target_message},
-		{DM_DEV_SET_GEOMETRY_CMD, 0, dev_set_geometry},
-		{DM_DEV_ARM_POLL_CMD, IOCTL_FLAGS_NO_PARAMS, dev_arm_poll},
-		{DM_GET_TARGET_VERSION_CMD, 0, get_target_version},
-		{DM_MPATH_PROBE_PATHS_CMD, 0, NULL}, /* block device ioctl */
+		[DM_VERSION_CMD] = {0, NULL}, /* version is dealt with elsewhere */
+		[DM_REMOVE_ALL_CMD] = {IOCTL_FLAGS_NO_PARAMS | IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, remove_all},
+		[DM_LIST_DEVICES_CMD] = {0, list_devices},
+
+		[DM_DEV_CREATE_CMD] = {IOCTL_FLAGS_NO_PARAMS | IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, dev_create},
+		[DM_DEV_REMOVE_CMD] = {IOCTL_FLAGS_NO_PARAMS | IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, dev_remove},
+		[DM_DEV_RENAME_CMD] = {IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, dev_rename},
+		[DM_DEV_SUSPEND_CMD] = {IOCTL_FLAGS_NO_PARAMS, dev_suspend},
+		[DM_DEV_STATUS_CMD] = {IOCTL_FLAGS_NO_PARAMS, dev_status},
+		[DM_DEV_WAIT_CMD] = {0, dev_wait},
+
+		[DM_TABLE_LOAD_CMD] = {0, table_load},
+		[DM_TABLE_CLEAR_CMD] = {IOCTL_FLAGS_NO_PARAMS, table_clear},
+		[DM_TABLE_DEPS_CMD] = {0, table_deps},
+		[DM_TABLE_STATUS_CMD] = {0, table_status},
+
+		[DM_LIST_VERSIONS_CMD] = {0, list_versions},
+
+		[DM_TARGET_MSG_CMD] = {0, target_message},
+		[DM_DEV_SET_GEOMETRY_CMD] = {0, dev_set_geometry},
+		[DM_DEV_ARM_POLL_CMD] = {IOCTL_FLAGS_NO_PARAMS, dev_arm_poll},
+		[DM_GET_TARGET_VERSION_CMD] = {0, get_target_version},
+		[DM_MPATH_PROBE_PATHS_CMD] = {0, NULL}, /* block device ioctl */
 	};
 
 	if (unlikely(cmd >= ARRAY_SIZE(_ioctls)))
@@ -2273,12 +2246,9 @@ static long dm_compat_ctl_ioctl(struct file *file, uint command, ulong u)
 
 static int dm_open(struct inode *inode, struct file *filp)
 {
-	int r;
 	struct dm_file *priv;
 
-	r = nonseekable_open(inode, filp);
-	if (unlikely(r))
-		return r;
+	nonseekable_open(inode, filp);
 
 	priv = filp->private_data = kmalloc_obj(struct dm_file);
 	if (!priv)
diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c
index 1489fda9d24a0..c8a9ca1eeae76 100644
--- a/drivers/md/dm-snap.c
+++ b/drivers/md/dm-snap.c
@@ -911,9 +911,7 @@ static int init_hash_tables(struct dm_snapshot *s)
 
 static void merge_shutdown(struct dm_snapshot *s)
 {
-	clear_bit_unlock(RUNNING_MERGE, &s->state_bits);
-	smp_mb__after_atomic();
-	wake_up_bit(&s->state_bits, RUNNING_MERGE);
+	clear_and_wake_up_bit(RUNNING_MERGE, &s->state_bits);
 }
 
 static struct bio *__release_queued_bios_after_merge(struct dm_snapshot *s)
diff --git a/drivers/md/dm-switch.c b/drivers/md/dm-switch.c
index 5952f02de1e62..e5b507b4fa7bf 100644
--- a/drivers/md/dm-switch.c
+++ b/drivers/md/dm-switch.c
@@ -184,7 +184,7 @@ static void switch_region_table_write(struct switch_ctx *sctx, unsigned long reg
 	pte = sctx->region_table[region_index];
 	pte &= ~((((region_table_slot_t)1 << sctx->region_table_entry_bits) - 1) << bit);
 	pte |= (region_table_slot_t)value << bit;
-	sctx->region_table[region_index] = pte;
+	WRITE_ONCE(sctx->region_table[region_index], pte);
 }
 
 /*
diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c
index dc2eff6b739df..a483f49dd4ce6 100644
--- a/drivers/md/dm-table.c
+++ b/drivers/md/dm-table.c
@@ -2035,7 +2035,7 @@ int dm_table_set_restrictions(struct dm_table *t, struct request_queue *q,
 		limits->features &= ~BLK_FEAT_NOWAIT;
 
 	/*
-	 * The current polling impementation does not support request based
+	 * The current polling implementation does not support request based
 	 * stacking.
 	 */
 	if (!__table_type_bio_based(t->type))
diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c
index 1b07630912540..241d3f3747e77 100644
--- a/drivers/md/dm-verity-target.c
+++ b/drivers/md/dm-verity-target.c
@@ -316,9 +316,7 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
 		else if (verity_handle_err(v,
 					   DM_VERITY_BLOCK_TYPE_METADATA,
 					   hash_block)) {
-			struct bio *bio;
 			io->had_mismatch = true;
-			bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
 			dm_audit_log_bio(DM_MSG_PREFIX, "verify-metadata", bio,
 					 block, 0);
 			r = -EIO;
diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c
index 493f5202ad042..302b60793da29 100644
--- a/drivers/md/dm-writecache.c
+++ b/drivers/md/dm-writecache.c
@@ -1227,7 +1227,7 @@ static void memcpy_flushcache_optimized(void *dest, void *source, size_t size)
 	 * advantage seen with cache-allocating-writes plus flushing.
 	 */
 #ifdef CONFIG_X86
-	if (static_cpu_has(X86_FEATURE_CLFLUSHOPT) &&
+	if (cpu_feature_enabled(X86_FEATURE_CLFLUSHOPT) &&
 	    likely(boot_cpu_data.x86_clflush_size == 64) &&
 	    likely(size >= 768)) {
 		do {
diff --git a/drivers/md/dm-zoned-metadata.c b/drivers/md/dm-zoned-metadata.c
index f4f81c79a0804..094b632158825 100644
--- a/drivers/md/dm-zoned-metadata.c
+++ b/drivers/md/dm-zoned-metadata.c
@@ -519,9 +519,7 @@ static void dmz_mblock_bio_end_io(struct bio *bio)
 	else
 		flag = DMZ_META_READING;
 
-	clear_bit_unlock(flag, &mblk->state);
-	smp_mb__after_atomic();
-	wake_up_bit(&mblk->state, flag);
+	clear_and_wake_up_bit(flag, &mblk->state);
 
 	bio_put(bio);
 }
@@ -1910,9 +1908,7 @@ void dmz_unlock_zone_reclaim(struct dm_zone *zone)
 	WARN_ON(dmz_is_active(zone));
 	WARN_ON(!dmz_in_reclaim(zone));
 
-	clear_bit_unlock(DMZ_RECLAIM, &zone->flags);
-	smp_mb__after_atomic();
-	wake_up_bit(&zone->flags, DMZ_RECLAIM);
+	clear_and_wake_up_bit(DMZ_RECLAIM, &zone->flags);
 }
 
 /*
diff --git a/drivers/md/dm-zoned-reclaim.c b/drivers/md/dm-zoned-reclaim.c
index ad9c7bc21d541..c041413c729ef 100644
--- a/drivers/md/dm-zoned-reclaim.c
+++ b/drivers/md/dm-zoned-reclaim.c
@@ -106,9 +106,7 @@ static void dmz_reclaim_kcopy_end(int read_err, unsigned long write_err,
 	else
 		zrc->kc_err = 0;
 
-	clear_bit_unlock(DMZ_RECLAIM_KCOPY, &zrc->flags);
-	smp_mb__after_atomic();
-	wake_up_bit(&zrc->flags, DMZ_RECLAIM_KCOPY);
+	clear_and_wake_up_bit(DMZ_RECLAIM_KCOPY, &zrc->flags);
 }
 
 /*
diff --git a/drivers/media/cec/core/cec-adap.c b/drivers/media/cec/core/cec-adap.c
index a90cb84a4b4d0..ee8a200b9803f 100644
--- a/drivers/media/cec/core/cec-adap.c
+++ b/drivers/media/cec/core/cec-adap.c
@@ -80,9 +80,9 @@ void cec_queue_event_fh(struct cec_fh *fh,
 			const struct cec_event *new_ev, u64 ts)
 {
 	static const u16 max_events[CEC_NUM_EVENTS] = {
-		1, 1, 800, 800, 8, 8, 8, 8
+		3, 1, 800, 800, 8, 8, 8, 8
 	};
-	struct cec_event_entry *entry;
+	struct cec_event_entry *new_entry, *entry;
 	unsigned int ev_idx = new_ev->event - 1;
 
 	if (WARN_ON(ev_idx >= ARRAY_SIZE(fh->events)))
@@ -92,36 +92,57 @@ void cec_queue_event_fh(struct cec_fh *fh,
 		ts = ktime_get_ns();
 
 	mutex_lock(&fh->lock);
-	if (ev_idx < CEC_NUM_CORE_EVENTS)
-		entry = &fh->core_events[ev_idx];
-	else
-		entry = kmalloc_obj(*entry);
-	if (entry) {
+	new_entry = kmalloc_obj(*new_entry);
+	if (new_entry) {
 		if (new_ev->event == CEC_EVENT_LOST_MSGS &&
 		    fh->queued_events[ev_idx]) {
+			entry = list_first_entry(&fh->events[ev_idx],
+						 struct cec_event_entry, list);
 			entry->ev.lost_msgs.lost_msgs +=
 				new_ev->lost_msgs.lost_msgs;
+			kfree(new_entry);
 			goto unlock;
 		}
-		entry->ev = *new_ev;
-		entry->ev.ts = ts;
+
+		new_entry->ev = *new_ev;
+		new_entry->ev.ts = ts;
+
+		/*
+		 * If the physical address becomes invalid (HPD went low),
+		 * then just flush all pending STATE_CHANGE events since
+		 * those are all obsoleted.
+		 *
+		 * This ensures you will not see stale STATE_CHANGE events.
+		 */
+		if (new_ev->event == CEC_EVENT_STATE_CHANGE &&
+		    new_ev->state_change.phys_addr == CEC_PHYS_ADDR_INVALID &&
+		    fh->queued_events[ev_idx]) {
+			/* drop all events */
+			while (!list_empty(&fh->events[ev_idx])) {
+				entry = list_first_entry(&fh->events[ev_idx],
+						struct cec_event_entry, list);
+				list_del(&entry->list);
+				kfree(entry);
+				fh->total_queued_events--;
+				fh->queued_events[ev_idx]--;
+			}
+			new_entry->ev.flags |= CEC_EVENT_FL_DROPPED_EVENTS;
+		}
 
 		if (fh->queued_events[ev_idx] < max_events[ev_idx]) {
 			/* Add new msg at the end of the queue */
-			list_add_tail(&entry->list, &fh->events[ev_idx]);
+			list_add_tail(&new_entry->list, &fh->events[ev_idx]);
 			fh->queued_events[ev_idx]++;
 			fh->total_queued_events++;
 			goto unlock;
 		}
 
-		if (ev_idx >= CEC_NUM_CORE_EVENTS) {
-			list_add_tail(&entry->list, &fh->events[ev_idx]);
-			/* drop the oldest event */
-			entry = list_first_entry(&fh->events[ev_idx],
-						 struct cec_event_entry, list);
-			list_del(&entry->list);
-			kfree(entry);
-		}
+		list_add_tail(&new_entry->list, &fh->events[ev_idx]);
+		/* drop the oldest event */
+		entry = list_first_entry(&fh->events[ev_idx],
+					 struct cec_event_entry, list);
+		list_del(&entry->list);
+		kfree(entry);
 	}
 	/* Mark that events were lost */
 	entry = list_first_entry_or_null(&fh->events[ev_idx],
@@ -608,6 +629,13 @@ void cec_transmit_done_ts(struct cec_adapter *adap, u8 status,
 		attempts_made = 1;
 
 	mutex_lock(&adap->lock);
+	if (adap->error_inj_tx_timeouts) {
+		dprintk(2, "%s: error_inj_tx_timeouts %u\n",
+			__func__, adap->error_inj_tx_timeouts);
+		adap->error_inj_tx_timeouts--;
+		mutex_unlock(&adap->lock);
+		return;
+	}
 	data = adap->transmitting;
 	if (!data) {
 		/*
@@ -965,7 +993,7 @@ int cec_transmit_msg_fh(struct cec_adapter *adap, struct cec_msg *msg,
 	 */
 	mutex_unlock(&adap->lock);
 	err = wait_for_completion_killable(&data->c);
-	cancel_delayed_work_sync(&data->work);
+	disable_delayed_work_sync(&data->work);
 	mutex_lock(&adap->lock);
 
 	if (err)
@@ -1317,7 +1345,7 @@ static int cec_config_log_addr(struct cec_adapter *adap,
 {
 	struct cec_log_addrs *las = &adap->log_addrs;
 	struct cec_msg msg = { };
-	const unsigned int max_retries = 2;
+	const unsigned int max_attempts = 3;
 	unsigned int i;
 	int err;
 
@@ -1328,7 +1356,7 @@ static int cec_config_log_addr(struct cec_adapter *adap,
 	msg.len = 1;
 	msg.msg[0] = (log_addr << 4) | log_addr;
 
-	for (i = 0; i < max_retries; i++) {
+	for (i = 0; i < max_attempts; i++) {
 		err = cec_transmit_msg_fh(adap, &msg, NULL, true);
 
 		/*
@@ -1345,31 +1373,24 @@ static int cec_config_log_addr(struct cec_adapter *adap,
 		if (err)
 			return err;
 
-		/*
-		 * The message was aborted or timed out due to a disconnect or
-		 * unconfigure, just bail out.
-		 */
-		if (msg.tx_status &
-		    (CEC_TX_STATUS_ABORTED | CEC_TX_STATUS_TIMEOUT))
-			return -EINTR;
 		if (msg.tx_status & CEC_TX_STATUS_OK)
 			return 0;
 		if (msg.tx_status & CEC_TX_STATUS_NACK)
 			break;
 		/*
-		 * Retry up to max_retries times if the message was neither
+		 * Do up to max_attempts if the message was neither
 		 * OKed or NACKed. This can happen due to e.g. a Lost
 		 * Arbitration condition.
 		 */
 	}
 
 	/*
-	 * If we are unable to get an OK or a NACK after max_retries attempts
+	 * If we are unable to get an OK or a NACK after max_attempts
 	 * (and note that each attempt already consists of four polls), then
 	 * we assume that something is really weird and that it is not a
 	 * good idea to try and claim this logical address.
 	 */
-	if (i == max_retries) {
+	if (i == max_attempts) {
 		dprintk(0, "polling for LA %u failed with tx_status=0x%04x\n",
 			log_addr, msg.tx_status);
 		return 0;
@@ -1468,6 +1489,7 @@ static int cec_config_thread_func(void *arg)
 	dprintk(1, "physical address: %x.%x.%x.%x, claim %d logical addresses\n",
 		cec_phys_addr_exp(adap->phys_addr), las->num_log_addrs);
 	las->log_addr_mask = 0;
+	las->flags &= ~CEC_LOG_ADDRS_FL_CONFIG_FAILED;
 
 	if (las->log_addr_type[0] == CEC_LOG_ADDR_TYPE_UNREGISTERED)
 		goto configured;
@@ -1600,6 +1622,8 @@ static int cec_config_thread_func(void *arg)
 unconfigure:
 	for (i = 0; i < las->num_log_addrs; i++)
 		las->log_addr[i] = CEC_LOG_ADDR_INVALID;
+	if (adap->phys_addr != CEC_PHYS_ADDR_INVALID)
+		las->flags |= CEC_LOG_ADDRS_FL_CONFIG_FAILED;
 	cec_adap_unconfigure(adap);
 	adap->is_configuring = false;
 	adap->must_reconfigure = false;
@@ -1717,7 +1741,6 @@ void __cec_s_phys_addr(struct cec_adapter *adap, u16 phys_addr, bool block)
 		cec_phys_addr_exp(phys_addr));
 	if (becomes_invalid || !is_invalid) {
 		adap->phys_addr = CEC_PHYS_ADDR_INVALID;
-		cec_post_state_event(adap);
 		cec_adap_unconfigure(adap);
 		if (becomes_invalid) {
 			cec_adap_enable(adap);
@@ -2219,9 +2242,13 @@ static int cec_receive_notify(struct cec_adapter *adap, struct cec_msg *msg,
 		 * Unprocessed messages are aborted if userspace isn't doing
 		 * any processing either.
 		 */
+		mutex_lock(&adap->lock);
 		if (!is_broadcast && !is_reply && !adap->follower_cnt &&
-		    !adap->cec_follower && msg->msg[1] != CEC_MSG_FEATURE_ABORT)
+		    !adap->cec_follower && msg->msg[1] != CEC_MSG_FEATURE_ABORT) {
+			mutex_unlock(&adap->lock);
 			return cec_feature_abort(adap, msg);
+		}
+		mutex_unlock(&adap->lock);
 		break;
 	}
 
@@ -2234,10 +2261,12 @@ static int cec_receive_notify(struct cec_adapter *adap, struct cec_msg *msg,
 	 * Send to the exclusive follower if there is one, otherwise send
 	 * to all followers.
 	 */
+	mutex_lock(&adap->lock);
 	if (adap->cec_follower)
 		cec_queue_msg_fh(adap->cec_follower, msg);
 	else
 		cec_queue_msg_followers(adap, msg);
+	mutex_unlock(&adap->lock);
 	return 0;
 }
 
diff --git a/drivers/media/cec/core/cec-api.c b/drivers/media/cec/core/cec-api.c
index 103ded79526f1..a491d7fc8a71a 100644
--- a/drivers/media/cec/core/cec-api.c
+++ b/drivers/media/cec/core/cec-api.c
@@ -345,8 +345,7 @@ static long cec_dqevent(struct cec_adapter *adap, struct cec_fh *fh,
 
 	if (copy_to_user(parg, &ev->ev, sizeof(ev->ev)))
 		err = -EFAULT;
-	if (ev_idx >= CEC_NUM_CORE_EVENTS)
-		kfree(ev);
+	kfree(ev);
 	fh->queued_events[ev_idx]--;
 	fh->total_queued_events--;
 
@@ -673,7 +672,7 @@ static int cec_release(struct inode *inode, struct file *filp)
 		list_del(&entry->list);
 		kfree(entry);
 	}
-	for (i = CEC_NUM_CORE_EVENTS; i < CEC_NUM_EVENTS; i++) {
+	for (i = 0; i < CEC_NUM_EVENTS; i++) {
 		while (!list_empty(&fh->events[i])) {
 			struct cec_event_entry *entry =
 				list_first_entry(&fh->events[i],
diff --git a/drivers/media/cec/core/cec-core.c b/drivers/media/cec/core/cec-core.c
index 0fcd3b5e60c8d..230f8a413e74c 100644
--- a/drivers/media/cec/core/cec-core.c
+++ b/drivers/media/cec/core/cec-core.c
@@ -5,16 +5,33 @@
  * Copyright 2016 Cisco Systems, Inc. and/or its affiliates. All rights reserved.
  */
 
+#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
+
+#include <linux/bitops.h>
+#include <linux/bug.h>
+#include <linux/cdev.h>
+#include <linux/container_of.h>
 #include <linux/debugfs.h>
-#include <linux/errno.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/fs.h>
 #include <linux/init.h>
-#include <linux/kernel.h>
-#include <linux/kmod.h>
-#include <linux/mm.h>
+#include <linux/kobject.h>
+#include <linux/kthread.h>
+#include <linux/list.h>
+#include <linux/minmax.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/printk.h>
+#include <linux/seq_file.h>
 #include <linux/slab.h>
+#include <linux/sprintf.h>
 #include <linux/string.h>
+#include <linux/time.h>
 #include <linux/types.h>
+#include <linux/wait.h>
+
+#include <asm/page.h>
 
 #include "cec-priv.h"
 
@@ -93,7 +110,7 @@ static int __must_check cec_devnode_register(struct cec_devnode *devnode,
 	minor = find_first_zero_bit(cec_devnode_nums, CEC_NUM_DEVICES);
 	if (minor == CEC_NUM_DEVICES) {
 		mutex_unlock(&cec_devnode_lock);
-		pr_err("could not get a free minor\n");
+		pr_err("Could not get a free minor\n");
 		return -ENFILE;
 	}
 
@@ -104,19 +121,19 @@ static int __must_check cec_devnode_register(struct cec_devnode *devnode,
 	devnode->dev.bus = &cec_bus_type;
 	devnode->dev.devt = MKDEV(MAJOR(cec_dev_t), minor);
 	devnode->dev.release = cec_devnode_release;
-	dev_set_name(&devnode->dev, "cec%d", devnode->minor);
+	dev_set_name(&devnode->dev, "%s%d", CEC_NAME, devnode->minor);
 	device_initialize(&devnode->dev);
 
 	/* Part 2: Initialize and register the character device */
 	cdev_init(&devnode->cdev, &cec_devnode_fops);
 	devnode->cdev.owner = owner;
-	kobject_set_name(&devnode->cdev.kobj, "cec%d", devnode->minor);
+	kobject_set_name(&devnode->cdev.kobj, "%s%d", CEC_NAME, devnode->minor);
 
 	devnode->registered = true;
 	ret = cdev_device_add(&devnode->cdev, &devnode->dev);
 	if (ret) {
 		devnode->registered = false;
-		pr_err("%s: cdev_device_add failed\n", __func__);
+		pr_err("cdev_device_add() failed\n");
 		goto clr_bit;
 	}
 
@@ -205,19 +222,35 @@ static int cec_error_inj_show(struct seq_file *sf, void *unused)
 
 	return call_op(adap, error_inj_show, sf);
 }
+DEFINE_SHOW_STORE_ATTRIBUTE(cec_error_inj);
 
-static int cec_error_inj_open(struct inode *inode, struct file *file)
+static ssize_t cec_error_inj_tx_timeouts_write(struct file *file,
+			const char __user *ubuf, size_t count, loff_t *ppos)
 {
-	return single_open(file, cec_error_inj_show, inode->i_private);
+	struct seq_file *sf = file->private_data;
+	struct cec_adapter *adap = sf->private;
+	int ret;
+
+	if (count > 5)
+		return -EINVAL;
+
+	mutex_lock(&adap->lock);
+	ret = kstrtou32_from_user(ubuf, count, 0, &adap->error_inj_tx_timeouts);
+	if (ret)
+		adap->error_inj_tx_timeouts = 0;
+	mutex_unlock(&adap->lock);
+	return ret ? : count;
 }
 
-static const struct file_operations cec_error_inj_fops = {
-	.open = cec_error_inj_open,
-	.write = cec_error_inj_write,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
+static int cec_error_inj_tx_timeouts_show(struct seq_file *sf, void *unused)
+{
+	struct cec_adapter *adap = sf->private;
+
+	seq_printf(sf, "%u", adap->error_inj_tx_timeouts);
+	return 0;
+}
+
+DEFINE_SHOW_STORE_ATTRIBUTE(cec_error_inj_tx_timeouts);
 #endif
 
 struct cec_adapter *cec_allocate_adapter(const struct cec_adap_ops *ops,
@@ -263,12 +296,11 @@ struct cec_adapter *cec_allocate_adapter(const struct cec_adap_ops *ops,
 	mutex_init(&adap->devnode.lock_fhs);
 	mutex_init(&adap->devnode.lock);
 
-	adap->kthread = kthread_run(cec_thread_func, adap, "cec-%s", name);
+	adap->kthread = kthread_run(cec_thread_func, adap, "%s-%s", CEC_NAME, name);
 	if (IS_ERR(adap->kthread)) {
-		pr_err("cec-%s: kernel_thread() failed\n", name);
+		pr_err("%s: kthread_run() failed\n", name);
 		res = PTR_ERR(adap->kthread);
-		kfree(adap);
-		return ERR_PTR(res);
+		goto err_free_adap;
 	}
 
 #ifdef CONFIG_MEDIA_CEC_RC
@@ -278,11 +310,10 @@ struct cec_adapter *cec_allocate_adapter(const struct cec_adap_ops *ops,
 	/* Prepare the RC input device */
 	adap->rc = rc_allocate_device(RC_DRIVER_SCANCODE);
 	if (!adap->rc) {
-		pr_err("cec-%s: failed to allocate memory for rc_dev\n",
-		       name);
+		pr_err("%s: failed to allocate memory for rc_dev\n", name);
 		kthread_stop(adap->kthread);
-		kfree(adap);
-		return ERR_PTR(-ENOMEM);
+		res = -ENOMEM;
+		goto err_free_adap;
 	}
 
 	snprintf(adap->input_phys, sizeof(adap->input_phys),
@@ -298,9 +329,18 @@ struct cec_adapter *cec_allocate_adapter(const struct cec_adap_ops *ops,
 	adap->rc->allowed_protocols = RC_PROTO_BIT_CEC;
 	adap->rc->priv = adap;
 	adap->rc->map_name = RC_MAP_CEC;
-	adap->rc->timeout = MS_TO_US(550);
+	adap->rc->timeout = 550 * USEC_PER_MSEC;
 #endif
 	return adap;
+
+err_free_adap:
+	mutex_destroy(&adap->devnode.lock);
+	mutex_destroy(&adap->devnode.lock_fhs);
+
+	mutex_destroy(&adap->lock);
+
+	kfree(adap);
+	return ERR_PTR(res);
 }
 EXPORT_SYMBOL_GPL(cec_allocate_adapter);
 
@@ -326,8 +366,7 @@ int cec_register_adapter(struct cec_adapter *adap,
 		res = rc_register_device(adap->rc);
 
 		if (res) {
-			pr_err("cec-%s: failed to prepare input device\n",
-			       adap->name);
+			pr_err("%s: failed to prepare input device\n", adap->name);
 			rc_free_device(adap->rc);
 			adap->rc = NULL;
 			return res;
@@ -356,6 +395,9 @@ int cec_register_adapter(struct cec_adapter *adap,
 	debugfs_create_devm_seqfile(&adap->devnode.dev, "status", adap->cec_dir,
 				    cec_adap_status);
 
+	debugfs_create_file("error-inj-tx-timeouts", 0644, adap->cec_dir, adap,
+			    &cec_error_inj_tx_timeouts_fops);
+
 	if (!adap->ops->error_inj_show || !adap->ops->error_inj_parse_line)
 		return 0;
 	debugfs_create_file("error-inj", 0644, adap->cec_dir, adap,
@@ -371,9 +413,7 @@ void cec_unregister_adapter(struct cec_adapter *adap)
 		return;
 
 #ifdef CONFIG_MEDIA_CEC_RC
-	/* Note: rc_unregister also calls rc_free */
 	rc_unregister_device(adap->rc);
-	adap->rc = NULL;
 #endif
 	debugfs_remove_recursive(adap->cec_dir);
 #ifdef CONFIG_CEC_NOTIFIER
@@ -387,14 +427,23 @@ void cec_delete_adapter(struct cec_adapter *adap)
 {
 	if (IS_ERR_OR_NULL(adap))
 		return;
+
 	if (adap->kthread_config)
 		kthread_stop(adap->kthread_config);
 	kthread_stop(adap->kthread);
+
 	if (adap->ops->adap_free)
 		adap->ops->adap_free(adap);
+
 #ifdef CONFIG_MEDIA_CEC_RC
 	rc_free_device(adap->rc);
 #endif
+
+	mutex_destroy(&adap->devnode.lock);
+	mutex_destroy(&adap->devnode.lock_fhs);
+
+	mutex_destroy(&adap->lock);
+
 	kfree(adap);
 }
 EXPORT_SYMBOL_GPL(cec_delete_adapter);
@@ -407,14 +456,14 @@ static int __init cec_devnode_init(void)
 	int ret = alloc_chrdev_region(&cec_dev_t, 0, CEC_NUM_DEVICES, CEC_NAME);
 
 	if (ret < 0) {
-		pr_warn("cec: unable to allocate major\n");
+		pr_warn("Unable to allocate major\n");
 		return ret;
 	}
 
 #ifdef CONFIG_DEBUG_FS
-	top_cec_dir = debugfs_create_dir("cec", NULL);
+	top_cec_dir = debugfs_create_dir(CEC_NAME, NULL);
 	if (IS_ERR_OR_NULL(top_cec_dir)) {
-		pr_warn("cec: Failed to create debugfs cec dir\n");
+		pr_warn("Failed to create debugfs " CEC_NAME " dir\n");
 		top_cec_dir = NULL;
 	}
 #endif
@@ -423,7 +472,7 @@ static int __init cec_devnode_init(void)
 	if (ret < 0) {
 		debugfs_remove_recursive(top_cec_dir);
 		unregister_chrdev_region(cec_dev_t, CEC_NUM_DEVICES);
-		pr_warn("cec: bus_register failed\n");
+		pr_warn("bus_register() failed\n");
 		return -EIO;
 	}
 
diff --git a/drivers/media/cec/core/cec-pin.c b/drivers/media/cec/core/cec-pin.c
index 6e1c391028322..6a0ee32e8401e 100644
--- a/drivers/media/cec/core/cec-pin.c
+++ b/drivers/media/cec/core/cec-pin.c
@@ -115,7 +115,7 @@ static void cec_pin_update(struct cec_pin *pin, bool v, bool force)
 		return;
 
 	pin->adap->cec_pin_is_high = v;
-	if (atomic_read(&pin->work_pin_num_events) < CEC_NUM_PIN_EVENTS) {
+	if (atomic_read_acquire(&pin->work_pin_num_events) < CEC_NUM_PIN_EVENTS) {
 		u8 ev = v;
 
 		if (pin->work_pin_events_dropped) {
@@ -126,7 +126,7 @@ static void cec_pin_update(struct cec_pin *pin, bool v, bool force)
 		pin->work_pin_ts[pin->work_pin_events_wr] = ktime_get();
 		pin->work_pin_events_wr =
 			(pin->work_pin_events_wr + 1) % CEC_NUM_PIN_EVENTS;
-		atomic_inc(&pin->work_pin_num_events);
+		atomic_inc_return_release(&pin->work_pin_num_events);
 	} else {
 		pin->work_pin_events_dropped = true;
 		pin->work_pin_events_dropped_cnt++;
@@ -692,7 +692,6 @@ static void cec_pin_rx_states(struct cec_pin *pin, ktime_t ts)
 		v = cec_pin_read(pin);
 		if (!v)
 			break;
-		pin->state = CEC_ST_RX_START_BIT_HIGH;
 		delta = ktime_us_delta(ts, pin->ts);
 		/* Start bit low is too short, go back to idle */
 		if (delta < CEC_TIM_START_BIT_LOW_MIN - CEC_TIM_IDLE_SAMPLE) {
@@ -703,7 +702,16 @@ static void cec_pin_rx_states(struct cec_pin *pin, ktime_t ts)
 			cec_pin_to_idle(pin);
 			break;
 		}
+		pin->state = CEC_ST_RX_START_BIT_HIGH;
 		if (rx_arb_lost(pin, &poll)) {
+			/*
+			 * Normally rx_toggle is toggled in cec_pin_to_idle()
+			 * when we're in an RX state, but here we switch to TX
+			 * mode, so cec_pin_to_idle() sees a TX mode and never
+			 * toggles rx_toggle. So toggle it here as a special
+			 * corner case.
+			 */
+			pin->rx_toggle ^= 1;
 			cec_msg_init(&pin->tx_msg, poll >> 4, poll & 0xf);
 			pin->tx_generated_poll = true;
 			pin->tx_extra_bytes = 0;
@@ -1101,7 +1109,7 @@ static int cec_pin_thread_func(void *_adap)
 						     pin->work_tx_ts);
 		}
 
-		while (atomic_read(&pin->work_pin_num_events)) {
+		while (atomic_read_acquire(&pin->work_pin_num_events)) {
 			unsigned int idx = pin->work_pin_events_rd;
 			u8 v = pin->work_pin_events[idx];
 
@@ -1110,7 +1118,7 @@ static int cec_pin_thread_func(void *_adap)
 						v & CEC_PIN_EVENT_FL_DROPPED,
 						pin->work_pin_ts[idx]);
 			pin->work_pin_events_rd = (idx + 1) % CEC_NUM_PIN_EVENTS;
-			atomic_dec(&pin->work_pin_num_events);
+			atomic_dec_return_release(&pin->work_pin_num_events);
 		}
 
 		switch (atomic_xchg(&pin->work_irq_change,
diff --git a/drivers/media/cec/platform/meson/ao-cec-g12a.c b/drivers/media/cec/platform/meson/ao-cec-g12a.c
index 41f5b8669cb01..2c914f0008b50 100644
--- a/drivers/media/cec/platform/meson/ao-cec-g12a.c
+++ b/drivers/media/cec/platform/meson/ao-cec-g12a.c
@@ -405,6 +405,7 @@ static int meson_ao_cec_g12a_write(void *context, unsigned int addr,
 }
 
 static const struct regmap_config meson_ao_cec_g12a_cec_regmap_conf = {
+	.name = "core",
 	.reg_bits = 8,
 	.val_bits = 8,
 	.reg_read = meson_ao_cec_g12a_read,
diff --git a/drivers/media/cec/platform/seco/seco-cec.c b/drivers/media/cec/platform/seco/seco-cec.c
index 97ed9654c78a1..654c70503dd57 100644
--- a/drivers/media/cec/platform/seco/seco-cec.c
+++ b/drivers/media/cec/platform/seco/seco-cec.c
@@ -7,14 +7,15 @@
  * Copyright (C) 2018, Aidilab Srl.
  */
 
-#include <linux/module.h>
 #include <linux/acpi.h>
 #include <linux/delay.h>
 #include <linux/dmi.h>
 #include <linux/gpio/consumer.h>
 #include <linux/interrupt.h>
+#include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/platform_device.h>
+#include <linux/time.h>
 
 /* CEC Framework */
 #include <media/cec-notifier.h>
@@ -356,7 +357,7 @@ static int secocec_ir_probe(void *priv)
 	cec->ir->allowed_protocols = RC_PROTO_BIT_RC5;
 	cec->ir->priv = cec;
 	cec->ir->map_name = RC_MAP_HAUPPAUGE;
-	cec->ir->timeout = MS_TO_US(100);
+	cec->ir->timeout = 100 * USEC_PER_MSEC;
 
 	/* Clear the status register */
 	status = smb_rd16(SECOCEC_STATUS_REG_1, &val);
diff --git a/drivers/media/cec/platform/stm32/stm32-cec.c b/drivers/media/cec/platform/stm32/stm32-cec.c
index 1ec0cece0a5b7..29418966a98ef 100644
--- a/drivers/media/cec/platform/stm32/stm32-cec.c
+++ b/drivers/media/cec/platform/stm32/stm32-cec.c
@@ -132,7 +132,8 @@ static void stm32_rx_done(struct stm32_cec *cec, u32 status)
 		u32 val;
 
 		regmap_read(cec->regmap, CEC_RXDR, &val);
-		cec->rx_msg.msg[cec->rx_msg.len++] = val & 0xFF;
+		if (cec->rx_msg.len < CEC_MAX_MSG_SIZE)
+			cec->rx_msg.msg[cec->rx_msg.len++] = val & 0xFF;
 	}
 
 	if (cec->irq_status & RXEND) {
@@ -193,18 +194,24 @@ static int stm32_cec_adap_enable(struct cec_adapter *adap, bool enable)
 static int stm32_cec_adap_log_addr(struct cec_adapter *adap, u8 logical_addr)
 {
 	struct stm32_cec *cec = adap->priv;
-	u32 oar = (1 << logical_addr) << 16;
 	u32 val;
+	int ret;
 
 	/* Poll every 100µs the register CEC_CR to wait end of transmission */
-	regmap_read_poll_timeout(cec->regmap, CEC_CR, val, !(val & TXSOM),
-				 100, CEC_XFER_TIMEOUT_MS * 1000);
+	ret = regmap_read_poll_timeout(cec->regmap, CEC_CR, val, !(val & TXSOM),
+				       100, CEC_XFER_TIMEOUT_MS * 1000);
+	if (ret)
+		return ret;
+
 	regmap_update_bits(cec->regmap, CEC_CR, CECEN, 0);
 
-	if (logical_addr == CEC_LOG_ADDR_INVALID)
+	if (logical_addr == CEC_LOG_ADDR_INVALID) {
 		regmap_update_bits(cec->regmap, CEC_CFGR, OAR, 0);
-	else
+	} else {
+		u32 oar = BIT(logical_addr) << 16;
+
 		regmap_update_bits(cec->regmap, CEC_CFGR, oar, oar);
+	}
 
 	regmap_update_bits(cec->regmap, CEC_CR, CECEN, CECEN);
 
diff --git a/drivers/media/cec/platform/tegra/tegra_cec.c b/drivers/media/cec/platform/tegra/tegra_cec.c
index 3ed50097262f6..e22a4625f1922 100644
--- a/drivers/media/cec/platform/tegra/tegra_cec.c
+++ b/drivers/media/cec/platform/tegra/tegra_cec.c
@@ -24,6 +24,7 @@
 #include <linux/of.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
+#include <linux/seq_file.h>
 #include <linux/clk/tegra.h>
 
 #include <media/cec-notifier.h>
@@ -47,6 +48,7 @@ struct tegra_cec {
 	u32			tx_buf[CEC_MAX_MSG_SIZE];
 	u8			tx_buf_cur;
 	u8			tx_buf_cnt;
+	u32			rx_total_low_drives;
 };
 
 static inline u32 cec_read(struct tegra_cec *cec, u32 reg)
@@ -116,6 +118,13 @@ static irqreturn_t tegra_cec_irq_handler(int irq, void *data)
 		return IRQ_WAKE_THREAD;
 	}
 
+	if (status & TEGRA_CEC_INT_STAT_RX_BUS_ERROR_DETECTED) {
+		dev_warn_ratelimited(dev, "RX bus error detected, generated low drive\n");
+		cec->rx_total_low_drives++;
+		cec_write(cec, TEGRA_CEC_INT_STAT,
+			  TEGRA_CEC_INT_STAT_RX_BUS_ERROR_DETECTED);
+	}
+
 	if ((status & TEGRA_CEC_INT_STAT_TX_ARBITRATION_FAILED) ||
 		   (status & TEGRA_CEC_INT_STAT_TX_BUS_ANOMALY_DETECTED)) {
 		tegra_cec_error_recovery(cec);
@@ -241,9 +250,21 @@ static int tegra_cec_adap_enable(struct cec_adapter *adap, bool enable)
 		  TEGRA_CEC_INT_MASK_TX_BUS_ANOMALY_DETECTED |
 		  TEGRA_CEC_INT_MASK_TX_FRAME_TRANSMITTED |
 		  TEGRA_CEC_INT_MASK_RX_REGISTER_FULL |
+		  TEGRA_CEC_INT_MASK_RX_BUS_ERROR_DETECTED |
 		  TEGRA_CEC_INT_MASK_RX_START_BIT_DETECTED);
 
-	cec_write(cec, TEGRA_CEC_HW_CONTROL, TEGRA_CEC_HWCTRL_TX_RX_MODE);
+	/*
+	 * TX_NAK_MODE ensures that the whole message is transmitted even
+	 * if each byte is NACKed. Without this flag the retransmit of the
+	 * messages after a NACK can be corrupt. This is a bug in the hardware.
+	 *
+	 * While less efficient, in practice you rarely transmit messages
+	 * that can be NACKed, with the exception of POLL messages which
+	 * are just one byte anyway.
+	 */
+	cec_write(cec, TEGRA_CEC_HW_CONTROL,
+		  TEGRA_CEC_HWCTRL_TX_RX_MODE |
+		  TEGRA_CEC_HWCTRL_TX_NAK_MODE);
 	return 0;
 }
 
@@ -307,11 +328,20 @@ static int tegra_cec_adap_transmit(struct cec_adapter *adap, u8 attempts,
 	return 0;
 }
 
+static void tegra_cec_adap_status(struct cec_adapter *adap, struct seq_file *file)
+{
+	struct tegra_cec *cec = adap->priv;
+
+	seq_printf(file, "receive low drive count: %u\n",
+		   cec->rx_total_low_drives);
+}
+
 static const struct cec_adap_ops tegra_cec_ops = {
 	.adap_enable = tegra_cec_adap_enable,
 	.adap_log_addr = tegra_cec_adap_log_addr,
 	.adap_transmit = tegra_cec_adap_transmit,
 	.adap_monitor_all_enable = tegra_cec_adap_monitor_all_enable,
+	.adap_status = tegra_cec_adap_status,
 };
 
 static int tegra_cec_probe(struct platform_device *pdev)
@@ -458,6 +488,7 @@ static const struct of_device_id tegra_cec_of_match[] = {
 	{ .compatible = "nvidia,tegra210-cec", },
 	{},
 };
+MODULE_DEVICE_TABLE(of, tegra_cec_of_match);
 
 static struct platform_driver tegra_cec_driver = {
 	.driver = {
diff --git a/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c b/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c
index 3381d86096a1f..3c6ce6f3d93e7 100644
--- a/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c
+++ b/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c
@@ -657,7 +657,8 @@ static void extron_process_received(struct extron_port *port, const char *data)
 	if (!port || port->disconnected)
 		return;
 
-	if (len < 5 || (len - 2) % 3 || data[len - 2] != '*')
+	if (len < 5 || ((len - 2) / 3 > sizeof(msg.msg)) ||
+	    (len - 2) % 3 || data[len - 2] != '*')
 		goto malformed;
 
 	while (*data != '*') {
diff --git a/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c b/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c
index 931e5dc453b90..e1d5c220f7381 100644
--- a/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c
+++ b/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c
@@ -2346,9 +2346,11 @@ static void tpg_fill_params_extras(const struct tpg_data *tpg,
 			(params->is_60hz ? V4L2_FIELD_TOP : V4L2_FIELD_BOTTOM);
 }
 
-static void tpg_fill_plane_extras(const struct tpg_data *tpg,
-				  const struct tpg_draw_params *params,
-				  unsigned p, unsigned h, u8 *vbuf)
+/* noinline to work around clang KASAN issues */
+static noinline_for_stack void
+tpg_fill_plane_extras(const struct tpg_data *tpg,
+		      const struct tpg_draw_params *params,
+		      unsigned p, unsigned h, u8 *vbuf)
 {
 	unsigned twopixsize = params->twopixsize;
 	unsigned img_width = params->img_width;
@@ -2483,9 +2485,9 @@ static void tpg_fill_plane_extras(const struct tpg_data *tpg,
 	}
 }
 
-static void tpg_fill_plane_pattern(const struct tpg_data *tpg,
-				   const struct tpg_draw_params *params,
-				   unsigned p, unsigned h, u8 *vbuf)
+static noinline_for_stack void
+tpg_fill_plane_pattern(const struct tpg_data *tpg, const struct tpg_draw_params *params,
+		       unsigned p, unsigned h, u8 *vbuf)
 {
 	unsigned twopixsize = params->twopixsize;
 	unsigned img_width = params->img_width;
diff --git a/drivers/media/common/videobuf2/videobuf2-v4l2.c b/drivers/media/common/videobuf2/videobuf2-v4l2.c
index 4baded4fd3b8c..5a1ee5eb9c0f9 100644
--- a/drivers/media/common/videobuf2/videobuf2-v4l2.c
+++ b/drivers/media/common/videobuf2/videobuf2-v4l2.c
@@ -635,19 +635,6 @@ struct vb2_buffer *vb2_find_buffer(struct vb2_queue *q, u64 timestamp)
 }
 EXPORT_SYMBOL_GPL(vb2_find_buffer);
 
-/*
- * vb2_querybuf() - query video buffer information
- * @q:		vb2 queue
- * @b:		buffer struct passed from userspace to vidioc_querybuf handler
- *		in driver
- *
- * Should be called from vidioc_querybuf ioctl handler in driver.
- * This function will verify the passed v4l2_buffer structure and fill the
- * relevant information for the userspace.
- *
- * The return values from this function are intended to be directly returned
- * from vidioc_querybuf handler in driver.
- */
 int vb2_querybuf(struct vb2_queue *q, struct v4l2_buffer *b)
 {
 	struct vb2_buffer *vb;
diff --git a/drivers/media/dvb-core/dvb_frontend.c b/drivers/media/dvb-core/dvb_frontend.c
index d082b6c57c76a..0286da57f3827 100644
--- a/drivers/media/dvb-core/dvb_frontend.c
+++ b/drivers/media/dvb-core/dvb_frontend.c
@@ -30,6 +30,7 @@
 #include <linux/kthread.h>
 #include <linux/ktime.h>
 #include <linux/compat.h>
+#include <linux/lockdep.h>
 #include <asm/processor.h>
 
 #include <media/dvb_frontend.h>
@@ -2760,77 +2761,25 @@ static __poll_t dvb_frontend_poll(struct file *file, struct poll_table_struct *w
 	return 0;
 }
 
-static int dvb_frontend_open(struct inode *inode, struct file *file)
+static int __dvb_frontend_open(struct inode *inode, struct file *file)
 {
 	struct dvb_device *dvbdev = file->private_data;
 	struct dvb_frontend *fe = dvbdev->priv;
 	struct dvb_frontend_private *fepriv = fe->frontend_priv;
-	struct dvb_adapter *adapter = fe->dvb;
 	int ret;
 
-	dev_dbg(fe->dvb->device, "%s:\n", __func__);
-	if (fe->exit == DVB_FE_DEVICE_REMOVED)
-		return -ENODEV;
-
-	if (adapter->mfe_shared == 2) {
-		mutex_lock(&adapter->mfe_lock);
-		if ((file->f_flags & O_ACCMODE) != O_RDONLY) {
-			if (adapter->mfe_dvbdev &&
-			    !adapter->mfe_dvbdev->writers) {
-				mutex_unlock(&adapter->mfe_lock);
-				return -EBUSY;
-			}
-			adapter->mfe_dvbdev = dvbdev;
-		}
-	} else if (adapter->mfe_shared) {
-		mutex_lock(&adapter->mfe_lock);
-
-		if (!adapter->mfe_dvbdev)
-			adapter->mfe_dvbdev = dvbdev;
-
-		else if (adapter->mfe_dvbdev != dvbdev) {
-			struct dvb_device
-				*mfedev = adapter->mfe_dvbdev;
-			struct dvb_frontend
-				*mfe = mfedev->priv;
-			struct dvb_frontend_private
-				*mfepriv = mfe->frontend_priv;
-			int mferetry = (dvb_mfe_wait_time << 1);
-
-			mutex_unlock(&adapter->mfe_lock);
-			while (mferetry-- && (mfedev->users != -1 ||
-					      mfepriv->thread)) {
-				if (msleep_interruptible(500)) {
-					if (signal_pending(current))
-						return -EINTR;
-				}
-			}
-
-			mutex_lock(&adapter->mfe_lock);
-			if (adapter->mfe_dvbdev != dvbdev) {
-				mfedev = adapter->mfe_dvbdev;
-				mfe = mfedev->priv;
-				mfepriv = mfe->frontend_priv;
-				if (mfedev->users != -1 ||
-				    mfepriv->thread) {
-					mutex_unlock(&adapter->mfe_lock);
-					return -EBUSY;
-				}
-				adapter->mfe_dvbdev = dvbdev;
-			}
-		}
-	}
-
 	if (dvbdev->users == -1 && fe->ops.ts_bus_ctrl) {
 		if ((ret = fe->ops.ts_bus_ctrl(fe, 1)) < 0)
-			goto err0;
-
-		/* If we took control of the bus, we need to force
-		   reinitialization.  This is because many ts_bus_ctrl()
-		   functions strobe the RESET pin on the demod, and if the
-		   frontend thread already exists then the dvb_init() routine
-		   won't get called (which is what usually does initial
-		   register configuration). */
+			return ret;
+
+		/*
+		 * If we took control of the bus, we need to force
+		 * reinitialization.  This is because many ts_bus_ctrl()
+		 * functions strobe the RESET pin on the demod, and if the
+		 * frontend thread already exists then the dvb_init() routine
+		 * won't get called (which is what usually does initial
+		 * register configuration).
+		 */
 		fepriv->reinitialise = 1;
 	}
 
@@ -2871,8 +2820,6 @@ static int dvb_frontend_open(struct inode *inode, struct file *file)
 
 	dvb_frontend_get(fe);
 
-	if (adapter->mfe_shared)
-		mutex_unlock(&adapter->mfe_lock);
 	return ret;
 
 err3:
@@ -2891,12 +2838,88 @@ static int dvb_frontend_open(struct inode *inode, struct file *file)
 err1:
 	if (dvbdev->users == -1 && fe->ops.ts_bus_ctrl)
 		fe->ops.ts_bus_ctrl(fe, 0);
-err0:
-	if (adapter->mfe_shared)
-		mutex_unlock(&adapter->mfe_lock);
+
+	return ret;
+}
+
+static int wait_dvb_frontend(struct dvb_adapter *adapter,
+			     struct dvb_device *mfedev)
+{
+	struct dvb_frontend *mfe = mfedev->priv;
+	struct dvb_frontend_private *mfepriv = mfe->frontend_priv;
+	int mferetry = (dvb_mfe_wait_time << 1);
+	int ret = 0;
+
+	lockdep_assert_held(&adapter->mfe_lock);
+
+	if (mfedev->users == -1 && !mfepriv->thread)
+		return 0;
+
+	mutex_unlock(&adapter->mfe_lock);
+
+	while (mferetry-- && (mfedev->users != -1 || mfepriv->thread)) {
+		if (msleep_interruptible(500))
+			if (signal_pending(current)) {
+				ret = -EINTR;
+				break;
+			}
+	}
+
+	mutex_lock(&adapter->mfe_lock);
+
 	return ret;
 }
 
+static int dvb_frontend_open(struct inode *inode, struct file *file)
+{
+	struct dvb_device *dvbdev = file->private_data;
+	struct dvb_frontend *fe = dvbdev->priv;
+	struct dvb_adapter *adapter = fe->dvb;
+
+	dev_dbg(fe->dvb->device, "%s:\n", __func__);
+	if (fe->exit == DVB_FE_DEVICE_REMOVED)
+		return -ENODEV;
+
+	if (!adapter->mfe_shared)
+		return __dvb_frontend_open(inode, file);
+
+	guard(mutex)(&adapter->mfe_lock);
+
+	if (adapter->mfe_shared == 2) {
+		if ((file->f_flags & O_ACCMODE) != O_RDONLY) {
+			if (adapter->mfe_dvbdev &&
+			    !adapter->mfe_dvbdev->writers)
+				return -EBUSY;
+			adapter->mfe_dvbdev = dvbdev;
+		}
+		return __dvb_frontend_open(inode, file);
+	}
+
+	if (!adapter->mfe_dvbdev) {
+		adapter->mfe_dvbdev = dvbdev;
+	} else if (adapter->mfe_dvbdev != dvbdev) {
+		struct dvb_device *mfedev = adapter->mfe_dvbdev;
+		struct dvb_frontend *mfe = mfedev->priv;
+		struct dvb_frontend_private *mfepriv = mfe->frontend_priv;
+		int ret;
+
+		ret = wait_dvb_frontend(adapter, mfedev);
+		if (ret)
+			return ret;
+
+		if (adapter->mfe_dvbdev != dvbdev) {
+			mfedev = adapter->mfe_dvbdev;
+			mfe = mfedev->priv;
+			mfepriv = mfe->frontend_priv;
+			if (mfedev->users != -1 || mfepriv->thread)
+				return -EBUSY;
+			adapter->mfe_dvbdev = dvbdev;
+		}
+	}
+
+	return __dvb_frontend_open(inode, file);
+}
+
 static int dvb_frontend_release(struct inode *inode, struct file *file)
 {
 	struct dvb_device *dvbdev = file->private_data;
diff --git a/drivers/media/dvb-frontends/helene.c b/drivers/media/dvb-frontends/helene.c
index 993280fefc2ce..5fbb466cc8af9 100644
--- a/drivers/media/dvb-frontends/helene.c
+++ b/drivers/media/dvb-frontends/helene.c
@@ -995,22 +995,22 @@ struct dvb_frontend *helene_attach_s(struct dvb_frontend *fe,
 		const struct helene_config *config,
 		struct i2c_adapter *i2c)
 {
-	struct helene_priv *priv = NULL;
+	struct helene_priv *pr = NULL;
 
-	priv = kzalloc_obj(struct helene_priv);
-	if (priv == NULL)
+	pr = kzalloc_obj(struct helene_priv);
+	if (!pr)
 		return NULL;
-	priv->i2c_address = (config->i2c_address >> 1);
-	priv->i2c = i2c;
-	priv->set_tuner_data = config->set_tuner_priv;
-	priv->set_tuner = config->set_tuner_callback;
-	priv->xtal = config->xtal;
+	pr->i2c_address = (config->i2c_address >> 1);
+	pr->i2c = i2c;
+	pr->set_tuner_data = config->set_tuner_priv;
+	pr->set_tuner = config->set_tuner_callback;
+	pr->xtal = config->xtal;
 
 	if (fe->ops.i2c_gate_ctrl)
 		fe->ops.i2c_gate_ctrl(fe, 1);
 
-	if (helene_x_pon(priv) != 0) {
-		kfree(priv);
+	if (helene_x_pon(pr) != 0) {
+		kfree(pr);
 		return NULL;
 	}
 
@@ -1019,10 +1019,10 @@ struct dvb_frontend *helene_attach_s(struct dvb_frontend *fe,
 
 	memcpy(&fe->ops.tuner_ops, &helene_tuner_ops_s,
 			sizeof(struct dvb_tuner_ops));
-	fe->tuner_priv = priv;
-	dev_info(&priv->i2c->dev,
-			"Sony HELENE Sat attached on addr=%x at I2C adapter %p\n",
-			priv->i2c_address, priv->i2c);
+	fe->tuner_priv = pr;
+	dev_info(&pr->i2c->dev,
+		 "Sony HELENE Sat attached on addr=%x at I2C adapter %p\n",
+		 pr->i2c_address, pr->i2c);
 	return fe;
 }
 EXPORT_SYMBOL_GPL(helene_attach_s);
@@ -1031,22 +1031,22 @@ struct dvb_frontend *helene_attach(struct dvb_frontend *fe,
 		const struct helene_config *config,
 		struct i2c_adapter *i2c)
 {
-	struct helene_priv *priv = NULL;
+	struct helene_priv *pr = NULL;
 
-	priv = kzalloc_obj(struct helene_priv);
-	if (priv == NULL)
+	pr = kzalloc_obj(struct helene_priv);
+	if (!pr)
 		return NULL;
-	priv->i2c_address = (config->i2c_address >> 1);
-	priv->i2c = i2c;
-	priv->set_tuner_data = config->set_tuner_priv;
-	priv->set_tuner = config->set_tuner_callback;
-	priv->xtal = config->xtal;
+	pr->i2c_address = (config->i2c_address >> 1);
+	pr->i2c = i2c;
+	pr->set_tuner_data = config->set_tuner_priv;
+	pr->set_tuner = config->set_tuner_callback;
+	pr->xtal = config->xtal;
 
 	if (fe->ops.i2c_gate_ctrl)
 		fe->ops.i2c_gate_ctrl(fe, 1);
 
-	if (helene_x_pon(priv) != 0) {
-		kfree(priv);
+	if (helene_x_pon(pr) != 0) {
+		kfree(pr);
 		return NULL;
 	}
 
@@ -1055,10 +1055,10 @@ struct dvb_frontend *helene_attach(struct dvb_frontend *fe,
 
 	memcpy(&fe->ops.tuner_ops, &helene_tuner_ops_t,
 			sizeof(struct dvb_tuner_ops));
-	fe->tuner_priv = priv;
-	dev_info(&priv->i2c->dev,
-			"Sony HELENE Ter attached on addr=%x at I2C adapter %p\n",
-			priv->i2c_address, priv->i2c);
+	fe->tuner_priv = pr;
+	dev_info(&pr->i2c->dev,
+		 "Sony HELENE Ter attached on addr=%x at I2C adapter %p\n",
+		 pr->i2c_address, pr->i2c);
 	return fe;
 }
 EXPORT_SYMBOL_GPL(helene_attach);
diff --git a/drivers/media/dvb-frontends/rtl2832_sdr.c b/drivers/media/dvb-frontends/rtl2832_sdr.c
index c564485e3bbb5..0330e7f0881a7 100644
--- a/drivers/media/dvb-frontends/rtl2832_sdr.c
+++ b/drivers/media/dvb-frontends/rtl2832_sdr.c
@@ -906,9 +906,12 @@ static int rtl2832_sdr_start_streaming(struct vb2_queue *vq, unsigned int count)
 		goto err;
 
 	mutex_unlock(&dev->v4l2_lock);
+
 	return 0;
 
 err:
+	rtl2832_sdr_free_urbs(dev);
+	rtl2832_sdr_free_stream_bufs(dev);
 	rtl2832_sdr_cleanup_queued_bufs(dev, VB2_BUF_STATE_QUEUED);
 	mutex_unlock(&dev->v4l2_lock);
 
@@ -1477,14 +1480,22 @@ static void rtl2832_sdr_remove(struct platform_device *pdev)
 
 	dev_dbg(&pdev->dev, "\n");
 
-	mutex_lock(&dev->vb_queue_lock);
+	/*
+	 * vb2_video_unregister_device() releases the vb2 queue, which
+	 * triggers rtl2832_sdr_stop_streaming() if streaming is active.
+	 * stop_streaming() uses dev->udev to free URBs and coherent DMA
+	 * stream buffers via usb_free_coherent(), so it must run before
+	 * dev->udev is cleared. vb2_video_unregister_device() locks
+	 * vb_queue_lock internally and stop_streaming() locks v4l2_lock,
+	 * so neither may be held by the caller.
+	 */
+	v4l2_device_disconnect(&dev->v4l2_dev);
+	vb2_video_unregister_device(&dev->vdev);
+
 	mutex_lock(&dev->v4l2_lock);
-	/* No need to keep the urbs around after disconnection */
 	dev->udev = NULL;
-	v4l2_device_disconnect(&dev->v4l2_dev);
-	video_unregister_device(&dev->vdev);
 	mutex_unlock(&dev->v4l2_lock);
-	mutex_unlock(&dev->vb_queue_lock);
+
 	v4l2_device_put(&dev->v4l2_dev);
 	module_put(pdev->dev.parent->driver->owner);
 }
diff --git a/drivers/media/i2c/Kconfig b/drivers/media/i2c/Kconfig
index 5d173e0ecf424..5c52007f9cbeb 100644
--- a/drivers/media/i2c/Kconfig
+++ b/drivers/media/i2c/Kconfig
@@ -259,6 +259,7 @@ config VIDEO_IMX335
 
 config VIDEO_IMX355
 	tristate "Sony IMX355 sensor support"
+	select V4L2_CCI_I2C
 	help
 	  This is a Video4Linux2 sensor driver for the Sony
 	  IMX355 camera.
@@ -269,6 +270,7 @@ config VIDEO_IMX355
 config VIDEO_IMX412
 	tristate "Sony IMX412 sensor support"
 	depends on OF
+	select V4L2_CCI_I2C
 	help
 	  This is a Video4Linux2 sensor driver for the Sony
 	  IMX412 camera.
@@ -287,6 +289,27 @@ config VIDEO_IMX415
 	  To compile this driver as a module, choose M here: the
 	  module will be called imx415.
 
+config VIDEO_IMX471
+	tristate "Sony IMX471 sensor support"
+	select V4L2_CCI_I2C
+	help
+	  This is a Video4Linux2 sensor driver for the Sony
+	  IMX471 camera.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called imx471.
+
+config VIDEO_IMX678
+	tristate "Sony IMX678 sensor support"
+	depends on GPIOLIB
+	select V4L2_CCI_I2C
+	help
+	  This is a Video4Linux2 sensor driver for the Sony
+	  IMX678 camera.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called imx678.
+
 config VIDEO_MAX9271_LIB
 	tristate
 
diff --git a/drivers/media/i2c/Makefile b/drivers/media/i2c/Makefile
index e45359efe0e41..d04bd5724552e 100644
--- a/drivers/media/i2c/Makefile
+++ b/drivers/media/i2c/Makefile
@@ -61,6 +61,8 @@ obj-$(CONFIG_VIDEO_IMX335) += imx335.o
 obj-$(CONFIG_VIDEO_IMX355) += imx355.o
 obj-$(CONFIG_VIDEO_IMX412) += imx412.o
 obj-$(CONFIG_VIDEO_IMX415) += imx415.o
+obj-$(CONFIG_VIDEO_IMX678) += imx678.o
+obj-$(CONFIG_VIDEO_IMX471) += imx471.o
 obj-$(CONFIG_VIDEO_IR_I2C) += ir-kbd-i2c.o
 obj-$(CONFIG_VIDEO_ISL7998X) += isl7998x.o
 obj-$(CONFIG_VIDEO_KS0127) += ks0127.o
diff --git a/drivers/media/i2c/alvium-csi2.c b/drivers/media/i2c/alvium-csi2.c
index dd991c2ee7002..f51f9b9877592 100644
--- a/drivers/media/i2c/alvium-csi2.c
+++ b/drivers/media/i2c/alvium-csi2.c
@@ -1290,7 +1290,7 @@ static int alvium_set_ctrl_auto_exposure(struct alvium_dev *alvium, bool on)
 	struct device *dev = &alvium->i2c_client->dev;
 	int ret;
 
-	ret = alvium_write_hshake(alvium, REG_BCRM_WHITE_BALANCE_AUTO_RW,
+	ret = alvium_write_hshake(alvium, REG_BCRM_EXPOSURE_AUTO_RW,
 				  on ? 0x02 : 0x00);
 	if (ret) {
 		dev_err(dev, "Fail to set autoexposure reg\n");
diff --git a/drivers/media/i2c/cvs/core.c b/drivers/media/i2c/cvs/core.c
index fe9e59ac311ca..d4a3b9c3bab1e 100644
--- a/drivers/media/i2c/cvs/core.c
+++ b/drivers/media/i2c/cvs/core.c
@@ -31,6 +31,7 @@
 
 #define PCI_DEVICE_ID_INTEL_IPU7		0x645d	/* MTL / LNL */
 #define PCI_DEVICE_ID_INTEL_IPU7P5		0xb05d	/* ARL / PTL */
+#define PCI_DEVICE_ID_INTEL_IPU8		0xd719	/* NVL */
 
 /*
  * IPU7 PCI device IDs not covered by ipu6_pci_tbl in ipu6-pci-table.h.
@@ -39,6 +40,7 @@
 static const struct pci_device_id icvs_ipu7_tbl[] = {
 	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IPU7) },
 	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IPU7P5) },
+	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IPU8) },
 	{ }
 };
 
@@ -962,6 +964,7 @@ static const struct acpi_device_id intel_cvs_acpi_match[] = {
 	{ "INTC10DE" }, /* LNL */
 	{ "INTC10E0" }, /* ARL */
 	{ "INTC10E1" }, /* PTL */
+	{ "INTC10FA" }, /* NVL */
 	{ }
 };
 MODULE_DEVICE_TABLE(acpi, intel_cvs_acpi_match);
diff --git a/drivers/media/i2c/gc0310.c b/drivers/media/i2c/gc0310.c
index 7af4d66f42a0f..754e82ad50ae4 100644
--- a/drivers/media/i2c/gc0310.c
+++ b/drivers/media/i2c/gc0310.c
@@ -6,6 +6,7 @@
  * Copyright (c) 2023-2025 Hans de Goede <hansg@kernel.org>
  */
 
+#include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/errno.h>
 #include <linux/gpio/consumer.h>
@@ -84,6 +85,8 @@
 #define to_gc0310_sensor(x) container_of(x, struct gc0310_device, sd)
 
 struct gc0310_device {
+	struct clk *clk;
+
 	struct v4l2_subdev sd;
 	struct media_pad pad;
 
@@ -635,7 +638,6 @@ static int gc0310_check_hwcfg(struct device *dev)
 	};
 	struct fwnode_handle *ep_fwnode;
 	unsigned long link_freq_bitmap;
-	u32 mclk;
 	int ret;
 
 	/*
@@ -647,21 +649,6 @@ static int gc0310_check_hwcfg(struct device *dev)
 		return dev_err_probe(dev, -EPROBE_DEFER,
 				     "waiting for fwnode graph endpoint\n");
 
-	ret = fwnode_property_read_u32(dev_fwnode(dev), "clock-frequency",
-				       &mclk);
-	if (ret) {
-		fwnode_handle_put(ep_fwnode);
-		return dev_err_probe(dev, ret,
-				     "reading clock-frequency property\n");
-	}
-
-	if (mclk != GC0310_MCLK_FREQ) {
-		fwnode_handle_put(ep_fwnode);
-		return dev_err_probe(dev, -EINVAL,
-				     "external clock %u is not supported\n",
-				     mclk);
-	}
-
 	ret = v4l2_fwnode_endpoint_alloc_parse(ep_fwnode, &bus_cfg);
 	fwnode_handle_put(ep_fwnode);
 	if (ret)
@@ -685,6 +672,7 @@ static int gc0310_check_hwcfg(struct device *dev)
 static int gc0310_probe(struct i2c_client *client)
 {
 	struct gc0310_device *sensor;
+	unsigned long freq;
 	int ret;
 
 	ret = gc0310_check_hwcfg(&client->dev);
@@ -695,6 +683,16 @@ static int gc0310_probe(struct i2c_client *client)
 	if (!sensor)
 		return -ENOMEM;
 
+	sensor->clk = devm_v4l2_sensor_clk_get(&client->dev, NULL);
+	if (IS_ERR(sensor->clk))
+		return dev_err_probe(&client->dev, PTR_ERR(sensor->clk),
+				     "failed to get clock\n");
+
+	freq = clk_get_rate(sensor->clk);
+	if (freq != GC0310_MCLK_FREQ)
+		return dev_err_probe(&client->dev, -EINVAL,
+				     "external clock %lu is not supported\n", freq);
+
 	sensor->reset = devm_gpiod_get(&client->dev, "reset", GPIOD_OUT_HIGH);
 	if (IS_ERR(sensor->reset)) {
 		return dev_err_probe(&client->dev, PTR_ERR(sensor->reset),
diff --git a/drivers/media/i2c/imx355.c b/drivers/media/i2c/imx355.c
index 27a5c212a527f..8eb8588cb71bb 100644
--- a/drivers/media/i2c/imx355.c
+++ b/drivers/media/i2c/imx355.c
@@ -9,74 +9,97 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/pm_runtime.h>
+#include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
 #include <linux/unaligned.h>
 
+#include <media/v4l2-cci.h>
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-device.h>
 #include <media/v4l2-event.h>
 #include <media/v4l2-fwnode.h>
 
-#define IMX355_REG_MODE_SELECT		0x0100
+#define IMX355_REG_MODE_SELECT		CCI_REG8(0x0100)
 #define IMX355_MODE_STANDBY		0x00
 #define IMX355_MODE_STREAMING		0x01
 
 /* Chip ID */
-#define IMX355_REG_CHIP_ID		0x0016
+#define IMX355_REG_CHIP_ID		CCI_REG16(0x0016)
 #define IMX355_CHIP_ID			0x0355
 
+#define IMX355_REG_LANE_SEL		CCI_REG8(0x0114)
+
+/* PLL registers that depend on the external clock frequency */
+#define IMX355_REG_EXTCLK_FREQ		CCI_REG16(0x0136)
+#define IMX355_REG_PLL_OP_PREDIV	CCI_REG8(0x030d)
+#define IMX355_REG_PLL_OP_MUL		CCI_REG16(0x030e)
+#define IMX355_REG_PLL_IVT_PCK_DIV	CCI_REG8(0x0301)
+#define IMX355_REG_PLL_IVT_SYSCK_DIV	CCI_REG8(0x0303)
+#define IMX355_PLL_OP_PREDIV		2
+#define IMX355_PLL_IVT_PCK_DIV		5
+
 /* V_TIMING internal */
-#define IMX355_REG_FLL			0x0340
+#define IMX355_REG_FLL			CCI_REG16(0x0340)
 #define IMX355_FLL_MAX			0xffff
+#define IMX355_VBLANK_MIN		20
+
+#define IMX355_REG_LLP			CCI_REG16(0x0342)
+#define IMX355_LLP_MAX			0xffff
+
+#define IMX355_REG_X_ADD_START		CCI_REG16(0x0344)
+#define IMX355_REG_Y_ADD_START		CCI_REG16(0x0346)
+#define IMX355_REG_X_ADD_END		CCI_REG16(0x0348)
+#define IMX355_REG_Y_ADD_END		CCI_REG16(0x034a)
+#define IMX355_REG_X_OUT_SIZE		CCI_REG16(0x034c)
+#define IMX355_REG_Y_OUT_SIZE		CCI_REG16(0x034e)
 
 /* Exposure control */
-#define IMX355_REG_EXPOSURE		0x0202
+#define IMX355_REG_EXPOSURE		CCI_REG16(0x0202)
 #define IMX355_EXPOSURE_MIN		1
 #define IMX355_EXPOSURE_STEP		1
 #define IMX355_EXPOSURE_DEFAULT		0x0282
+#define IMX355_EXPOSURE_OFFSET		10
 
 /* Analog gain control */
-#define IMX355_REG_ANALOG_GAIN		0x0204
+#define IMX355_REG_ANALOG_GAIN		CCI_REG16(0x0204)
 #define IMX355_ANA_GAIN_MIN		0
 #define IMX355_ANA_GAIN_MAX		960
 #define IMX355_ANA_GAIN_STEP		1
 #define IMX355_ANA_GAIN_DEFAULT		0
 
 /* Digital gain control */
-#define IMX355_REG_DPGA_USE_GLOBAL_GAIN	0x3070
-#define IMX355_REG_DIG_GAIN_GLOBAL	0x020e
+#define IMX355_REG_DPGA_USE_GLOBAL_GAIN	CCI_REG8(0x3070)
+#define IMX355_REG_DIG_GAIN_GLOBAL	CCI_REG16(0x020e)
 #define IMX355_DGTL_GAIN_MIN		256
 #define IMX355_DGTL_GAIN_MAX		4095
 #define IMX355_DGTL_GAIN_STEP		1
 #define IMX355_DGTL_GAIN_DEFAULT	256
 
 /* Test Pattern Control */
-#define IMX355_REG_TEST_PATTERN		0x0600
+#define IMX355_REG_TEST_PATTERN		CCI_REG16(0x0600)
 #define IMX355_TEST_PATTERN_DISABLED		0
 #define IMX355_TEST_PATTERN_SOLID_COLOR		1
 #define IMX355_TEST_PATTERN_COLOR_BARS		2
 #define IMX355_TEST_PATTERN_GRAY_COLOR_BARS	3
 #define IMX355_TEST_PATTERN_PN9			4
 
-/* Flip Control */
-#define IMX355_REG_ORIENTATION		0x0101
+#define IMX355_REG_REQ_LINK_BIT_RATE	CCI_REG16(0x0820)
 
-/* default link frequency and external clock */
-#define IMX355_LINK_FREQ_DEFAULT	360000000LL
-#define IMX355_EXT_CLK			19200000
-#define IMX355_LINK_FREQ_INDEX		0
+#define IMX355_REG_BINNING_MODE		CCI_REG8(0x0900)
+#define IMX355_REG_BINNING_TYPE		CCI_REG8(0x0901)
+#define IMX355_REG_BINNING_WEIGHTING	CCI_REG8(0x0902)
 
-/* number of data lanes */
-#define IMX355_DATA_LANES		4
+/* Flip Control */
+#define IMX355_REG_ORIENTATION		CCI_REG8(0x0101)
 
-struct imx355_reg {
-	u16 address;
-	u8 val;
-};
+#define IMX355_PIXEL_ARRAY_TOP		0
+#define IMX355_PIXEL_ARRAY_LEFT		0
+#define IMX355_PIXEL_ARRAY_WIDTH	3280
+#define IMX355_PIXEL_ARRAY_HEIGHT	2464
 
 struct imx355_reg_list {
 	u32 num_of_regs;
-	const struct imx355_reg *regs;
+	const struct cci_reg_sequence *regs;
 };
 
 /* Mode : resolution and related config&values */
@@ -85,28 +108,58 @@ struct imx355_mode {
 	u32 width;
 	/* Frame height */
 	u32 height;
+	struct v4l2_rect crop;
 
 	/* V-timing */
 	u32 fll_def;
-	u32 fll_min;
 
 	/* H-timing */
 	u32 llp;
 
-	/* index of link frequency */
-	u32 link_freq_index;
-
 	/* Default register values */
 	struct imx355_reg_list reg_list;
 };
 
+struct imx355_clk_params {
+	u32 ext_clk;
+	u16 extclk_freq;	/* External clock (MHz) in 8.8 fixed point) */
+	u16 pll_op_mpy[2];	/* OP system PLL multiplier */
+	u8 pll_op_prediv[2];	/* OP system pre PLL d */
+};
+
+/*
+ * The clock tree is in single PLL mode, so PREDIV_VT and MPY_IVT do nothing.
+ * In 4 lane mode the MIPI rate is 360Mhz (720Mbit/s) and pixel rate is
+ * 288MPix/s.
+ * In 2 lane mode the MIPI rate is 444MHz (888Mbit/s) and pixel rate
+ * 177.6MPix/s with a 24MHz clock, and 441.6MHz (883.2Mbit/s) and 176.6MPix/s
+ * with a 19.2MHz clock.
+ */
+static const struct imx355_clk_params imx355_clk_params[] = {
+	{
+		.ext_clk = 19200000,
+		.extclk_freq = 0x1333,
+		.pll_op_mpy = { 75, 92 },
+		.pll_op_prediv = { 2, 2 }
+	},
+	{
+		.ext_clk = 24000000,
+		.extclk_freq = 0x1800,
+		.pll_op_mpy = { 60, 111 },
+		.pll_op_prediv = { 2, 3 }
+	},
+};
+
 struct imx355_hwcfg {
+	s64 link_freq_menu;
 	unsigned long link_freq_bitmap;
+	unsigned int num_lanes;
 };
 
 struct imx355 {
 	struct device *dev;
 	struct clk *clk;
+	struct regmap *regmap;
 
 	struct v4l2_subdev sd;
 	struct media_pad pad;
@@ -114,24 +167,14 @@ struct imx355 {
 	struct v4l2_ctrl_handler ctrl_handler;
 	/* V4L2 Controls */
 	struct v4l2_ctrl *link_freq;
-	struct v4l2_ctrl *pixel_rate;
 	struct v4l2_ctrl *vblank;
 	struct v4l2_ctrl *hblank;
 	struct v4l2_ctrl *exposure;
 	struct v4l2_ctrl *vflip;
 	struct v4l2_ctrl *hflip;
 
-	/* Current mode */
-	const struct imx355_mode *cur_mode;
-
 	struct imx355_hwcfg *hwcfg;
-
-	/*
-	 * Mutex for serialized access:
-	 * Protect sensor set pad format and start/stop streaming safely.
-	 * Protect access to sensor v4l2 controls.
-	 */
-	struct mutex mutex;
+	const struct imx355_clk_params *clk_params;
 
 	struct gpio_desc *reset_gpio;
 	struct regulator_bulk_data *supplies;
@@ -143,750 +186,147 @@ static const struct regulator_bulk_data imx355_supplies[] = {
 	{ .supply = "dovdd" },
 };
 
-static const struct imx355_reg imx355_global_regs[] = {
-	{ 0x0136, 0x13 },
-	{ 0x0137, 0x33 },
-	{ 0x304e, 0x03 },
-	{ 0x4348, 0x16 },
-	{ 0x4350, 0x19 },
-	{ 0x4408, 0x0a },
-	{ 0x440c, 0x0b },
-	{ 0x4411, 0x5f },
-	{ 0x4412, 0x2c },
-	{ 0x4623, 0x00 },
-	{ 0x462c, 0x0f },
-	{ 0x462d, 0x00 },
-	{ 0x462e, 0x00 },
-	{ 0x4684, 0x54 },
-	{ 0x480a, 0x07 },
-	{ 0x4908, 0x07 },
-	{ 0x4909, 0x07 },
-	{ 0x490d, 0x0a },
-	{ 0x491e, 0x0f },
-	{ 0x4921, 0x06 },
-	{ 0x4923, 0x28 },
-	{ 0x4924, 0x28 },
-	{ 0x4925, 0x29 },
-	{ 0x4926, 0x29 },
-	{ 0x4927, 0x1f },
-	{ 0x4928, 0x20 },
-	{ 0x4929, 0x20 },
-	{ 0x492a, 0x20 },
-	{ 0x492c, 0x05 },
-	{ 0x492d, 0x06 },
-	{ 0x492e, 0x06 },
-	{ 0x492f, 0x06 },
-	{ 0x4930, 0x03 },
-	{ 0x4931, 0x04 },
-	{ 0x4932, 0x04 },
-	{ 0x4933, 0x05 },
-	{ 0x595e, 0x01 },
-	{ 0x5963, 0x01 },
-	{ 0x3030, 0x01 },
-	{ 0x3031, 0x01 },
-	{ 0x3045, 0x01 },
-	{ 0x4010, 0x00 },
-	{ 0x4011, 0x00 },
-	{ 0x4012, 0x00 },
-	{ 0x4013, 0x01 },
-	{ 0x68a8, 0xfe },
-	{ 0x68a9, 0xff },
-	{ 0x6888, 0x00 },
-	{ 0x6889, 0x00 },
-	{ 0x68b0, 0x00 },
-	{ 0x3058, 0x00 },
-	{ 0x305a, 0x00 },
+static const struct cci_reg_sequence imx355_global_regs[] = {
+	{ CCI_REG8(0x304e), 0x03 },
+	{ CCI_REG8(0x4348), 0x16 },
+	{ CCI_REG8(0x4350), 0x19 },
+	{ CCI_REG8(0x4408), 0x0a },
+	{ CCI_REG8(0x440c), 0x0b },
+	{ CCI_REG8(0x4411), 0x5f },
+	{ CCI_REG8(0x4412), 0x2c },
+	{ CCI_REG8(0x4623), 0x00 },
+	{ CCI_REG8(0x462c), 0x0f },
+	{ CCI_REG8(0x462d), 0x00 },
+	{ CCI_REG8(0x462e), 0x00 },
+	{ CCI_REG8(0x4684), 0x54 },
+	{ CCI_REG8(0x480a), 0x07 },
+	{ CCI_REG8(0x4908), 0x07 },
+	{ CCI_REG8(0x4909), 0x07 },
+	{ CCI_REG8(0x490d), 0x0a },
+	{ CCI_REG8(0x491e), 0x0f },
+	{ CCI_REG8(0x4921), 0x06 },
+	{ CCI_REG8(0x4923), 0x28 },
+	{ CCI_REG8(0x4924), 0x28 },
+	{ CCI_REG8(0x4925), 0x29 },
+	{ CCI_REG8(0x4926), 0x29 },
+	{ CCI_REG8(0x4927), 0x1f },
+	{ CCI_REG8(0x4928), 0x20 },
+	{ CCI_REG8(0x4929), 0x20 },
+	{ CCI_REG8(0x492a), 0x20 },
+	{ CCI_REG8(0x492c), 0x05 },
+	{ CCI_REG8(0x492d), 0x06 },
+	{ CCI_REG8(0x492e), 0x06 },
+	{ CCI_REG8(0x492f), 0x06 },
+	{ CCI_REG8(0x4930), 0x03 },
+	{ CCI_REG8(0x4931), 0x04 },
+	{ CCI_REG8(0x4932), 0x04 },
+	{ CCI_REG8(0x4933), 0x05 },
+	{ CCI_REG8(0x595e), 0x01 },
+	{ CCI_REG8(0x5963), 0x01 },
+	{ CCI_REG8(0x3030), 0x01 },
+	{ CCI_REG8(0x3031), 0x01 },
+	{ CCI_REG8(0x3045), 0x01 },
+	{ CCI_REG8(0x4010), 0x00 },
+	{ CCI_REG8(0x4011), 0x00 },
+	{ CCI_REG8(0x4012), 0x00 },
+	{ CCI_REG8(0x4013), 0x01 },
+	{ CCI_REG8(0x68a8), 0xfe },
+	{ CCI_REG8(0x68a9), 0xff },
+	{ CCI_REG8(0x6888), 0x00 },
+	{ CCI_REG8(0x6889), 0x00 },
+	{ CCI_REG8(0x68b0), 0x00 },
+	{ CCI_REG8(0x3058), 0x00 },
+	{ CCI_REG8(0x305a), 0x00 },
+	{ CCI_REG8(0x0112), 0x0a },
+	{ CCI_REG8(0x0113), 0x0a },
+	{ IMX355_REG_PLL_IVT_PCK_DIV, IMX355_PLL_IVT_PCK_DIV },
+	{ CCI_REG8(0x0303), 0x01 },
+	{ CCI_REG8(0x0305), 0x02 },
+	{ CCI_REG8(0x0306), 0x00 },
+	{ CCI_REG8(0x0307), 0x78 },
+	{ CCI_REG8(0x030b), 0x01 },
+	{ IMX355_REG_PLL_OP_PREDIV, IMX355_PLL_OP_PREDIV },
+	{ CCI_REG8(0x0310), 0x00 },
+	{ CCI_REG8(0x0220), 0x00 },
+	{ CCI_REG8(0x0222), 0x01 },
+	{ CCI_REG8(0x3088), 0x04 },
+	{ CCI_REG8(0x6813), 0x02 },
+	{ CCI_REG8(0x6835), 0x07 },
+	{ CCI_REG8(0x6836), 0x01 },
+	{ CCI_REG8(0x6837), 0x04 },
+	{ CCI_REG8(0x684d), 0x07 },
+	{ CCI_REG8(0x684e), 0x01 },
+	{ CCI_REG8(0x684f), 0x04 },
 };
 
-static const struct imx355_reg_list imx355_global_setting = {
-	.num_of_regs = ARRAY_SIZE(imx355_global_regs),
-	.regs = imx355_global_regs,
+static const struct cci_reg_sequence mode_3268x2448_regs[] = {
+	{ CCI_REG8(0x0700), 0x00 },
+	{ CCI_REG8(0x0701), 0x10 },
 };
 
-static const struct imx355_reg mode_3268x2448_regs[] = {
-	{ 0x0112, 0x0a },
-	{ 0x0113, 0x0a },
-	{ 0x0114, 0x03 },
-	{ 0x0342, 0x0e },
-	{ 0x0343, 0x58 },
-	{ 0x0340, 0x0a },
-	{ 0x0341, 0x37 },
-	{ 0x0344, 0x00 },
-	{ 0x0345, 0x08 },
-	{ 0x0346, 0x00 },
-	{ 0x0347, 0x08 },
-	{ 0x0348, 0x0c },
-	{ 0x0349, 0xcb },
-	{ 0x034a, 0x09 },
-	{ 0x034b, 0x97 },
-	{ 0x0220, 0x00 },
-	{ 0x0222, 0x01 },
-	{ 0x0900, 0x00 },
-	{ 0x0901, 0x11 },
-	{ 0x0902, 0x00 },
-	{ 0x034c, 0x0c },
-	{ 0x034d, 0xc4 },
-	{ 0x034e, 0x09 },
-	{ 0x034f, 0x90 },
-	{ 0x0301, 0x05 },
-	{ 0x0303, 0x01 },
-	{ 0x0305, 0x02 },
-	{ 0x0306, 0x00 },
-	{ 0x0307, 0x78 },
-	{ 0x030b, 0x01 },
-	{ 0x030d, 0x02 },
-	{ 0x030e, 0x00 },
-	{ 0x030f, 0x4b },
-	{ 0x0310, 0x00 },
-	{ 0x0700, 0x00 },
-	{ 0x0701, 0x10 },
-	{ 0x0820, 0x0b },
-	{ 0x0821, 0x40 },
-	{ 0x3088, 0x04 },
-	{ 0x6813, 0x02 },
-	{ 0x6835, 0x07 },
-	{ 0x6836, 0x01 },
-	{ 0x6837, 0x04 },
-	{ 0x684d, 0x07 },
-	{ 0x684e, 0x01 },
-	{ 0x684f, 0x04 },
+static const struct cci_reg_sequence mode_3264x2448_regs[] = {
+	{ CCI_REG8(0x0700), 0x00 },
+	{ CCI_REG8(0x0701), 0x10 },
 };
 
-static const struct imx355_reg mode_3264x2448_regs[] = {
-	{ 0x0112, 0x0a },
-	{ 0x0113, 0x0a },
-	{ 0x0114, 0x03 },
-	{ 0x0342, 0x0e },
-	{ 0x0343, 0x58 },
-	{ 0x0340, 0x0a },
-	{ 0x0341, 0x37 },
-	{ 0x0344, 0x00 },
-	{ 0x0345, 0x08 },
-	{ 0x0346, 0x00 },
-	{ 0x0347, 0x08 },
-	{ 0x0348, 0x0c },
-	{ 0x0349, 0xc7 },
-	{ 0x034a, 0x09 },
-	{ 0x034b, 0x97 },
-	{ 0x0220, 0x00 },
-	{ 0x0222, 0x01 },
-	{ 0x0900, 0x00 },
-	{ 0x0901, 0x11 },
-	{ 0x0902, 0x00 },
-	{ 0x034c, 0x0c },
-	{ 0x034d, 0xc0 },
-	{ 0x034e, 0x09 },
-	{ 0x034f, 0x90 },
-	{ 0x0301, 0x05 },
-	{ 0x0303, 0x01 },
-	{ 0x0305, 0x02 },
-	{ 0x0306, 0x00 },
-	{ 0x0307, 0x78 },
-	{ 0x030b, 0x01 },
-	{ 0x030d, 0x02 },
-	{ 0x030e, 0x00 },
-	{ 0x030f, 0x4b },
-	{ 0x0310, 0x00 },
-	{ 0x0700, 0x00 },
-	{ 0x0701, 0x10 },
-	{ 0x0820, 0x0b },
-	{ 0x0821, 0x40 },
-	{ 0x3088, 0x04 },
-	{ 0x6813, 0x02 },
-	{ 0x6835, 0x07 },
-	{ 0x6836, 0x01 },
-	{ 0x6837, 0x04 },
-	{ 0x684d, 0x07 },
-	{ 0x684e, 0x01 },
-	{ 0x684f, 0x04 },
+static const struct cci_reg_sequence mode_3280x2464_regs[] = {
+	{ CCI_REG8(0x0700), 0x00 },
+	{ CCI_REG8(0x0701), 0x10 },
 };
 
-static const struct imx355_reg mode_3280x2464_regs[] = {
-	{ 0x0112, 0x0a },
-	{ 0x0113, 0x0a },
-	{ 0x0114, 0x03 },
-	{ 0x0342, 0x0e },
-	{ 0x0343, 0x58 },
-	{ 0x0340, 0x0a },
-	{ 0x0341, 0x37 },
-	{ 0x0344, 0x00 },
-	{ 0x0345, 0x00 },
-	{ 0x0346, 0x00 },
-	{ 0x0347, 0x00 },
-	{ 0x0348, 0x0c },
-	{ 0x0349, 0xcf },
-	{ 0x034a, 0x09 },
-	{ 0x034b, 0x9f },
-	{ 0x0220, 0x00 },
-	{ 0x0222, 0x01 },
-	{ 0x0900, 0x00 },
-	{ 0x0901, 0x11 },
-	{ 0x0902, 0x00 },
-	{ 0x034c, 0x0c },
-	{ 0x034d, 0xd0 },
-	{ 0x034e, 0x09 },
-	{ 0x034f, 0xa0 },
-	{ 0x0301, 0x05 },
-	{ 0x0303, 0x01 },
-	{ 0x0305, 0x02 },
-	{ 0x0306, 0x00 },
-	{ 0x0307, 0x78 },
-	{ 0x030b, 0x01 },
-	{ 0x030d, 0x02 },
-	{ 0x030e, 0x00 },
-	{ 0x030f, 0x4b },
-	{ 0x0310, 0x00 },
-	{ 0x0700, 0x00 },
-	{ 0x0701, 0x10 },
-	{ 0x0820, 0x0b },
-	{ 0x0821, 0x40 },
-	{ 0x3088, 0x04 },
-	{ 0x6813, 0x02 },
-	{ 0x6835, 0x07 },
-	{ 0x6836, 0x01 },
-	{ 0x6837, 0x04 },
-	{ 0x684d, 0x07 },
-	{ 0x684e, 0x01 },
-	{ 0x684f, 0x04 },
+static const struct cci_reg_sequence mode_1940x1096_regs[] = {
+	{ CCI_REG8(0x0700), 0x00 },
+	{ CCI_REG8(0x0701), 0x10 },
 };
 
-static const struct imx355_reg mode_1940x1096_regs[] = {
-	{ 0x0112, 0x0a },
-	{ 0x0113, 0x0a },
-	{ 0x0114, 0x03 },
-	{ 0x0342, 0x0e },
-	{ 0x0343, 0x58 },
-	{ 0x0340, 0x05 },
-	{ 0x0341, 0x1a },
-	{ 0x0344, 0x02 },
-	{ 0x0345, 0xa0 },
-	{ 0x0346, 0x02 },
-	{ 0x0347, 0xac },
-	{ 0x0348, 0x0a },
-	{ 0x0349, 0x33 },
-	{ 0x034a, 0x06 },
-	{ 0x034b, 0xf3 },
-	{ 0x0220, 0x00 },
-	{ 0x0222, 0x01 },
-	{ 0x0900, 0x00 },
-	{ 0x0901, 0x11 },
-	{ 0x0902, 0x00 },
-	{ 0x034c, 0x07 },
-	{ 0x034d, 0x94 },
-	{ 0x034e, 0x04 },
-	{ 0x034f, 0x48 },
-	{ 0x0301, 0x05 },
-	{ 0x0303, 0x01 },
-	{ 0x0305, 0x02 },
-	{ 0x0306, 0x00 },
-	{ 0x0307, 0x78 },
-	{ 0x030b, 0x01 },
-	{ 0x030d, 0x02 },
-	{ 0x030e, 0x00 },
-	{ 0x030f, 0x4b },
-	{ 0x0310, 0x00 },
-	{ 0x0700, 0x00 },
-	{ 0x0701, 0x10 },
-	{ 0x0820, 0x0b },
-	{ 0x0821, 0x40 },
-	{ 0x3088, 0x04 },
-	{ 0x6813, 0x02 },
-	{ 0x6835, 0x07 },
-	{ 0x6836, 0x01 },
-	{ 0x6837, 0x04 },
-	{ 0x684d, 0x07 },
-	{ 0x684e, 0x01 },
-	{ 0x684f, 0x04 },
+static const struct cci_reg_sequence mode_1936x1096_regs[] = {
+	{ CCI_REG8(0x0700), 0x00 },
+	{ CCI_REG8(0x0701), 0x10 },
 };
 
-static const struct imx355_reg mode_1936x1096_regs[] = {
-	{ 0x0112, 0x0a },
-	{ 0x0113, 0x0a },
-	{ 0x0114, 0x03 },
-	{ 0x0342, 0x0e },
-	{ 0x0343, 0x58 },
-	{ 0x0340, 0x05 },
-	{ 0x0341, 0x1a },
-	{ 0x0344, 0x02 },
-	{ 0x0345, 0xa0 },
-	{ 0x0346, 0x02 },
-	{ 0x0347, 0xac },
-	{ 0x0348, 0x0a },
-	{ 0x0349, 0x2f },
-	{ 0x034a, 0x06 },
-	{ 0x034b, 0xf3 },
-	{ 0x0220, 0x00 },
-	{ 0x0222, 0x01 },
-	{ 0x0900, 0x00 },
-	{ 0x0901, 0x11 },
-	{ 0x0902, 0x00 },
-	{ 0x034c, 0x07 },
-	{ 0x034d, 0x90 },
-	{ 0x034e, 0x04 },
-	{ 0x034f, 0x48 },
-	{ 0x0301, 0x05 },
-	{ 0x0303, 0x01 },
-	{ 0x0305, 0x02 },
-	{ 0x0306, 0x00 },
-	{ 0x0307, 0x78 },
-	{ 0x030b, 0x01 },
-	{ 0x030d, 0x02 },
-	{ 0x030e, 0x00 },
-	{ 0x030f, 0x4b },
-	{ 0x0310, 0x00 },
-	{ 0x0700, 0x00 },
-	{ 0x0701, 0x10 },
-	{ 0x0820, 0x0b },
-	{ 0x0821, 0x40 },
-	{ 0x3088, 0x04 },
-	{ 0x6813, 0x02 },
-	{ 0x6835, 0x07 },
-	{ 0x6836, 0x01 },
-	{ 0x6837, 0x04 },
-	{ 0x684d, 0x07 },
-	{ 0x684e, 0x01 },
-	{ 0x684f, 0x04 },
+static const struct cci_reg_sequence mode_1924x1080_regs[] = {
+	{ CCI_REG8(0x0700), 0x00 },
+	{ CCI_REG8(0x0701), 0x10 },
 };
 
-static const struct imx355_reg mode_1924x1080_regs[] = {
-	{ 0x0112, 0x0a },
-	{ 0x0113, 0x0a },
-	{ 0x0114, 0x03 },
-	{ 0x0342, 0x0e },
-	{ 0x0343, 0x58 },
-	{ 0x0340, 0x05 },
-	{ 0x0341, 0x1a },
-	{ 0x0344, 0x02 },
-	{ 0x0345, 0xa8 },
-	{ 0x0346, 0x02 },
-	{ 0x0347, 0xb4 },
-	{ 0x0348, 0x0a },
-	{ 0x0349, 0x2b },
-	{ 0x034a, 0x06 },
-	{ 0x034b, 0xeb },
-	{ 0x0220, 0x00 },
-	{ 0x0222, 0x01 },
-	{ 0x0900, 0x00 },
-	{ 0x0901, 0x11 },
-	{ 0x0902, 0x00 },
-	{ 0x034c, 0x07 },
-	{ 0x034d, 0x84 },
-	{ 0x034e, 0x04 },
-	{ 0x034f, 0x38 },
-	{ 0x0301, 0x05 },
-	{ 0x0303, 0x01 },
-	{ 0x0305, 0x02 },
-	{ 0x0306, 0x00 },
-	{ 0x0307, 0x78 },
-	{ 0x030b, 0x01 },
-	{ 0x030d, 0x02 },
-	{ 0x030e, 0x00 },
-	{ 0x030f, 0x4b },
-	{ 0x0310, 0x00 },
-	{ 0x0700, 0x00 },
-	{ 0x0701, 0x10 },
-	{ 0x0820, 0x0b },
-	{ 0x0821, 0x40 },
-	{ 0x3088, 0x04 },
-	{ 0x6813, 0x02 },
-	{ 0x6835, 0x07 },
-	{ 0x6836, 0x01 },
-	{ 0x6837, 0x04 },
-	{ 0x684d, 0x07 },
-	{ 0x684e, 0x01 },
-	{ 0x684f, 0x04 },
+static const struct cci_reg_sequence mode_1920x1080_regs[] = {
+	{ CCI_REG8(0x0700), 0x00 },
+	{ CCI_REG8(0x0701), 0x10 },
 };
 
-static const struct imx355_reg mode_1920x1080_regs[] = {
-	{ 0x0112, 0x0a },
-	{ 0x0113, 0x0a },
-	{ 0x0114, 0x03 },
-	{ 0x0342, 0x0e },
-	{ 0x0343, 0x58 },
-	{ 0x0340, 0x05 },
-	{ 0x0341, 0x1a },
-	{ 0x0344, 0x02 },
-	{ 0x0345, 0xa8 },
-	{ 0x0346, 0x02 },
-	{ 0x0347, 0xb4 },
-	{ 0x0348, 0x0a },
-	{ 0x0349, 0x27 },
-	{ 0x034a, 0x06 },
-	{ 0x034b, 0xeb },
-	{ 0x0220, 0x00 },
-	{ 0x0222, 0x01 },
-	{ 0x0900, 0x00 },
-	{ 0x0901, 0x11 },
-	{ 0x0902, 0x00 },
-	{ 0x034c, 0x07 },
-	{ 0x034d, 0x80 },
-	{ 0x034e, 0x04 },
-	{ 0x034f, 0x38 },
-	{ 0x0301, 0x05 },
-	{ 0x0303, 0x01 },
-	{ 0x0305, 0x02 },
-	{ 0x0306, 0x00 },
-	{ 0x0307, 0x78 },
-	{ 0x030b, 0x01 },
-	{ 0x030d, 0x02 },
-	{ 0x030e, 0x00 },
-	{ 0x030f, 0x4b },
-	{ 0x0310, 0x00 },
-	{ 0x0700, 0x00 },
-	{ 0x0701, 0x10 },
-	{ 0x0820, 0x0b },
-	{ 0x0821, 0x40 },
-	{ 0x3088, 0x04 },
-	{ 0x6813, 0x02 },
-	{ 0x6835, 0x07 },
-	{ 0x6836, 0x01 },
-	{ 0x6837, 0x04 },
-	{ 0x684d, 0x07 },
-	{ 0x684e, 0x01 },
-	{ 0x684f, 0x04 },
+static const struct cci_reg_sequence mode_1640x1232_regs[] = {
+	{ CCI_REG8(0x0700), 0x00 },
+	{ CCI_REG8(0x0701), 0x10 },
 };
 
-static const struct imx355_reg mode_1640x1232_regs[] = {
-	{ 0x0112, 0x0a },
-	{ 0x0113, 0x0a },
-	{ 0x0114, 0x03 },
-	{ 0x0342, 0x07 },
-	{ 0x0343, 0x2c },
-	{ 0x0340, 0x05 },
-	{ 0x0341, 0x1a },
-	{ 0x0344, 0x00 },
-	{ 0x0345, 0x00 },
-	{ 0x0346, 0x00 },
-	{ 0x0347, 0x00 },
-	{ 0x0348, 0x0c },
-	{ 0x0349, 0xcf },
-	{ 0x034a, 0x09 },
-	{ 0x034b, 0x9f },
-	{ 0x0220, 0x00 },
-	{ 0x0222, 0x01 },
-	{ 0x0900, 0x01 },
-	{ 0x0901, 0x22 },
-	{ 0x0902, 0x00 },
-	{ 0x034c, 0x06 },
-	{ 0x034d, 0x68 },
-	{ 0x034e, 0x04 },
-	{ 0x034f, 0xd0 },
-	{ 0x0301, 0x05 },
-	{ 0x0303, 0x01 },
-	{ 0x0305, 0x02 },
-	{ 0x0306, 0x00 },
-	{ 0x0307, 0x78 },
-	{ 0x030b, 0x01 },
-	{ 0x030d, 0x02 },
-	{ 0x030e, 0x00 },
-	{ 0x030f, 0x4b },
-	{ 0x0310, 0x00 },
-	{ 0x0700, 0x00 },
-	{ 0x0701, 0x10 },
-	{ 0x0820, 0x0b },
-	{ 0x0821, 0x40 },
-	{ 0x3088, 0x04 },
-	{ 0x6813, 0x02 },
-	{ 0x6835, 0x07 },
-	{ 0x6836, 0x01 },
-	{ 0x6837, 0x04 },
-	{ 0x684d, 0x07 },
-	{ 0x684e, 0x01 },
-	{ 0x684f, 0x04 },
+static const struct cci_reg_sequence mode_1640x922_regs[] = {
+	{ CCI_REG8(0x0700), 0x00 },
+	{ CCI_REG8(0x0701), 0x10 },
 };
 
-static const struct imx355_reg mode_1640x922_regs[] = {
-	{ 0x0112, 0x0a },
-	{ 0x0113, 0x0a },
-	{ 0x0114, 0x03 },
-	{ 0x0342, 0x07 },
-	{ 0x0343, 0x2c },
-	{ 0x0340, 0x05 },
-	{ 0x0341, 0x1a },
-	{ 0x0344, 0x00 },
-	{ 0x0345, 0x00 },
-	{ 0x0346, 0x01 },
-	{ 0x0347, 0x30 },
-	{ 0x0348, 0x0c },
-	{ 0x0349, 0xcf },
-	{ 0x034a, 0x08 },
-	{ 0x034b, 0x63 },
-	{ 0x0220, 0x00 },
-	{ 0x0222, 0x01 },
-	{ 0x0900, 0x01 },
-	{ 0x0901, 0x22 },
-	{ 0x0902, 0x00 },
-	{ 0x034c, 0x06 },
-	{ 0x034d, 0x68 },
-	{ 0x034e, 0x03 },
-	{ 0x034f, 0x9a },
-	{ 0x0301, 0x05 },
-	{ 0x0303, 0x01 },
-	{ 0x0305, 0x02 },
-	{ 0x0306, 0x00 },
-	{ 0x0307, 0x78 },
-	{ 0x030b, 0x01 },
-	{ 0x030d, 0x02 },
-	{ 0x030e, 0x00 },
-	{ 0x030f, 0x4b },
-	{ 0x0310, 0x00 },
-	{ 0x0700, 0x00 },
-	{ 0x0701, 0x10 },
-	{ 0x0820, 0x0b },
-	{ 0x0821, 0x40 },
-	{ 0x3088, 0x04 },
-	{ 0x6813, 0x02 },
-	{ 0x6835, 0x07 },
-	{ 0x6836, 0x01 },
-	{ 0x6837, 0x04 },
-	{ 0x684d, 0x07 },
-	{ 0x684e, 0x01 },
-	{ 0x684f, 0x04 },
+static const struct cci_reg_sequence mode_1300x736_regs[] = {
+	{ CCI_REG8(0x0700), 0x00 },
+	{ CCI_REG8(0x0701), 0x10 },
 };
 
-static const struct imx355_reg mode_1300x736_regs[] = {
-	{ 0x0112, 0x0a },
-	{ 0x0113, 0x0a },
-	{ 0x0114, 0x03 },
-	{ 0x0342, 0x07 },
-	{ 0x0343, 0x2c },
-	{ 0x0340, 0x05 },
-	{ 0x0341, 0x1a },
-	{ 0x0344, 0x01 },
-	{ 0x0345, 0x58 },
-	{ 0x0346, 0x01 },
-	{ 0x0347, 0xf0 },
-	{ 0x0348, 0x0b },
-	{ 0x0349, 0x7f },
-	{ 0x034a, 0x07 },
-	{ 0x034b, 0xaf },
-	{ 0x0220, 0x00 },
-	{ 0x0222, 0x01 },
-	{ 0x0900, 0x01 },
-	{ 0x0901, 0x22 },
-	{ 0x0902, 0x00 },
-	{ 0x034c, 0x05 },
-	{ 0x034d, 0x14 },
-	{ 0x034e, 0x02 },
-	{ 0x034f, 0xe0 },
-	{ 0x0301, 0x05 },
-	{ 0x0303, 0x01 },
-	{ 0x0305, 0x02 },
-	{ 0x0306, 0x00 },
-	{ 0x0307, 0x78 },
-	{ 0x030b, 0x01 },
-	{ 0x030d, 0x02 },
-	{ 0x030e, 0x00 },
-	{ 0x030f, 0x4b },
-	{ 0x0310, 0x00 },
-	{ 0x0700, 0x00 },
-	{ 0x0701, 0x10 },
-	{ 0x0820, 0x0b },
-	{ 0x0821, 0x40 },
-	{ 0x3088, 0x04 },
-	{ 0x6813, 0x02 },
-	{ 0x6835, 0x07 },
-	{ 0x6836, 0x01 },
-	{ 0x6837, 0x04 },
-	{ 0x684d, 0x07 },
-	{ 0x684e, 0x01 },
-	{ 0x684f, 0x04 },
+static const struct cci_reg_sequence mode_1296x736_regs[] = {
+	{ CCI_REG8(0x0700), 0x00 },
+	{ CCI_REG8(0x0701), 0x10 },
 };
 
-static const struct imx355_reg mode_1296x736_regs[] = {
-	{ 0x0112, 0x0a },
-	{ 0x0113, 0x0a },
-	{ 0x0114, 0x03 },
-	{ 0x0342, 0x07 },
-	{ 0x0343, 0x2c },
-	{ 0x0340, 0x05 },
-	{ 0x0341, 0x1a },
-	{ 0x0344, 0x01 },
-	{ 0x0345, 0x58 },
-	{ 0x0346, 0x01 },
-	{ 0x0347, 0xf0 },
-	{ 0x0348, 0x0b },
-	{ 0x0349, 0x77 },
-	{ 0x034a, 0x07 },
-	{ 0x034b, 0xaf },
-	{ 0x0220, 0x00 },
-	{ 0x0222, 0x01 },
-	{ 0x0900, 0x01 },
-	{ 0x0901, 0x22 },
-	{ 0x0902, 0x00 },
-	{ 0x034c, 0x05 },
-	{ 0x034d, 0x10 },
-	{ 0x034e, 0x02 },
-	{ 0x034f, 0xe0 },
-	{ 0x0301, 0x05 },
-	{ 0x0303, 0x01 },
-	{ 0x0305, 0x02 },
-	{ 0x0306, 0x00 },
-	{ 0x0307, 0x78 },
-	{ 0x030b, 0x01 },
-	{ 0x030d, 0x02 },
-	{ 0x030e, 0x00 },
-	{ 0x030f, 0x4b },
-	{ 0x0310, 0x00 },
-	{ 0x0700, 0x00 },
-	{ 0x0701, 0x10 },
-	{ 0x0820, 0x0b },
-	{ 0x0821, 0x40 },
-	{ 0x3088, 0x04 },
-	{ 0x6813, 0x02 },
-	{ 0x6835, 0x07 },
-	{ 0x6836, 0x01 },
-	{ 0x6837, 0x04 },
-	{ 0x684d, 0x07 },
-	{ 0x684e, 0x01 },
-	{ 0x684f, 0x04 },
+static const struct cci_reg_sequence mode_1284x720_regs[] = {
+	{ CCI_REG8(0x0700), 0x00 },
+	{ CCI_REG8(0x0701), 0x10 },
 };
 
-static const struct imx355_reg mode_1284x720_regs[] = {
-	{ 0x0112, 0x0a },
-	{ 0x0113, 0x0a },
-	{ 0x0114, 0x03 },
-	{ 0x0342, 0x07 },
-	{ 0x0343, 0x2c },
-	{ 0x0340, 0x05 },
-	{ 0x0341, 0x1a },
-	{ 0x0344, 0x01 },
-	{ 0x0345, 0x68 },
-	{ 0x0346, 0x02 },
-	{ 0x0347, 0x00 },
-	{ 0x0348, 0x0b },
-	{ 0x0349, 0x6f },
-	{ 0x034a, 0x07 },
-	{ 0x034b, 0x9f },
-	{ 0x0220, 0x00 },
-	{ 0x0222, 0x01 },
-	{ 0x0900, 0x01 },
-	{ 0x0901, 0x22 },
-	{ 0x0902, 0x00 },
-	{ 0x034c, 0x05 },
-	{ 0x034d, 0x04 },
-	{ 0x034e, 0x02 },
-	{ 0x034f, 0xd0 },
-	{ 0x0301, 0x05 },
-	{ 0x0303, 0x01 },
-	{ 0x0305, 0x02 },
-	{ 0x0306, 0x00 },
-	{ 0x0307, 0x78 },
-	{ 0x030b, 0x01 },
-	{ 0x030d, 0x02 },
-	{ 0x030e, 0x00 },
-	{ 0x030f, 0x4b },
-	{ 0x0310, 0x00 },
-	{ 0x0700, 0x00 },
-	{ 0x0701, 0x10 },
-	{ 0x0820, 0x0b },
-	{ 0x0821, 0x40 },
-	{ 0x3088, 0x04 },
-	{ 0x6813, 0x02 },
-	{ 0x6835, 0x07 },
-	{ 0x6836, 0x01 },
-	{ 0x6837, 0x04 },
-	{ 0x684d, 0x07 },
-	{ 0x684e, 0x01 },
-	{ 0x684f, 0x04 },
+static const struct cci_reg_sequence mode_1280x720_regs[] = {
+	{ CCI_REG8(0x0700), 0x00 },
+	{ CCI_REG8(0x0701), 0x10 },
 };
 
-static const struct imx355_reg mode_1280x720_regs[] = {
-	{ 0x0112, 0x0a },
-	{ 0x0113, 0x0a },
-	{ 0x0114, 0x03 },
-	{ 0x0342, 0x07 },
-	{ 0x0343, 0x2c },
-	{ 0x0340, 0x05 },
-	{ 0x0341, 0x1a },
-	{ 0x0344, 0x01 },
-	{ 0x0345, 0x68 },
-	{ 0x0346, 0x02 },
-	{ 0x0347, 0x00 },
-	{ 0x0348, 0x0b },
-	{ 0x0349, 0x67 },
-	{ 0x034a, 0x07 },
-	{ 0x034b, 0x9f },
-	{ 0x0220, 0x00 },
-	{ 0x0222, 0x01 },
-	{ 0x0900, 0x01 },
-	{ 0x0901, 0x22 },
-	{ 0x0902, 0x00 },
-	{ 0x034c, 0x05 },
-	{ 0x034d, 0x00 },
-	{ 0x034e, 0x02 },
-	{ 0x034f, 0xd0 },
-	{ 0x0301, 0x05 },
-	{ 0x0303, 0x01 },
-	{ 0x0305, 0x02 },
-	{ 0x0306, 0x00 },
-	{ 0x0307, 0x78 },
-	{ 0x030b, 0x01 },
-	{ 0x030d, 0x02 },
-	{ 0x030e, 0x00 },
-	{ 0x030f, 0x4b },
-	{ 0x0310, 0x00 },
-	{ 0x0700, 0x00 },
-	{ 0x0701, 0x10 },
-	{ 0x0820, 0x0b },
-	{ 0x0821, 0x40 },
-	{ 0x3088, 0x04 },
-	{ 0x6813, 0x02 },
-	{ 0x6835, 0x07 },
-	{ 0x6836, 0x01 },
-	{ 0x6837, 0x04 },
-	{ 0x684d, 0x07 },
-	{ 0x684e, 0x01 },
-	{ 0x684f, 0x04 },
-};
-
-static const struct imx355_reg mode_820x616_regs[] = {
-	{ 0x0112, 0x0a },
-	{ 0x0113, 0x0a },
-	{ 0x0114, 0x03 },
-	{ 0x0342, 0x0e },
-	{ 0x0343, 0x58 },
-	{ 0x0340, 0x02 },
-	{ 0x0341, 0x8c },
-	{ 0x0344, 0x00 },
-	{ 0x0345, 0x00 },
-	{ 0x0346, 0x00 },
-	{ 0x0347, 0x00 },
-	{ 0x0348, 0x0c },
-	{ 0x0349, 0xcf },
-	{ 0x034a, 0x09 },
-	{ 0x034b, 0x9f },
-	{ 0x0220, 0x00 },
-	{ 0x0222, 0x01 },
-	{ 0x0900, 0x01 },
-	{ 0x0901, 0x44 },
-	{ 0x0902, 0x00 },
-	{ 0x034c, 0x03 },
-	{ 0x034d, 0x34 },
-	{ 0x034e, 0x02 },
-	{ 0x034f, 0x68 },
-	{ 0x0301, 0x05 },
-	{ 0x0303, 0x01 },
-	{ 0x0305, 0x02 },
-	{ 0x0306, 0x00 },
-	{ 0x0307, 0x78 },
-	{ 0x030b, 0x01 },
-	{ 0x030d, 0x02 },
-	{ 0x030e, 0x00 },
-	{ 0x030f, 0x4b },
-	{ 0x0310, 0x00 },
-	{ 0x0700, 0x02 },
-	{ 0x0701, 0x78 },
-	{ 0x0820, 0x0b },
-	{ 0x0821, 0x40 },
-	{ 0x3088, 0x04 },
-	{ 0x6813, 0x02 },
-	{ 0x6835, 0x07 },
-	{ 0x6836, 0x01 },
-	{ 0x6837, 0x04 },
-	{ 0x684d, 0x07 },
-	{ 0x684e, 0x01 },
-	{ 0x684f, 0x04 },
+static const struct cci_reg_sequence mode_820x616_regs[] = {
+	{ CCI_REG8(0x0700), 0x02 },
+	{ CCI_REG8(0x0701), 0x78 },
 };
 
 static const char * const imx355_test_pattern_menu[] = {
@@ -897,23 +337,19 @@ static const char * const imx355_test_pattern_menu[] = {
 	"Pseudorandom Sequence (PN9)",
 };
 
-/*
- * When adding more than the one below, make sure the disallowed ones will
- * actually be disabled in the LINK_FREQ control.
- */
-static const s64 link_freq_menu_items[] = {
-	IMX355_LINK_FREQ_DEFAULT,
-};
-
 /* Mode configs */
 static const struct imx355_mode supported_modes[] = {
 	{
 		.width = 3280,
 		.height = 2464,
+		.crop = {
+			.width = 3280,
+			.height = 2464,
+			.left = 0,
+			.top = 0,
+		},
 		.fll_def = 2615,
-		.fll_min = 2615,
 		.llp = 3672,
-		.link_freq_index = IMX355_LINK_FREQ_INDEX,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_3280x2464_regs),
 			.regs = mode_3280x2464_regs,
@@ -922,10 +358,14 @@ static const struct imx355_mode supported_modes[] = {
 	{
 		.width = 3268,
 		.height = 2448,
+		.crop = {
+			.width = 3268,
+			.height = 2448,
+			.left = 8,
+			.top = 8,
+		},
 		.fll_def = 2615,
-		.fll_min = 2615,
 		.llp = 3672,
-		.link_freq_index = IMX355_LINK_FREQ_INDEX,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_3268x2448_regs),
 			.regs = mode_3268x2448_regs,
@@ -934,10 +374,14 @@ static const struct imx355_mode supported_modes[] = {
 	{
 		.width = 3264,
 		.height = 2448,
+		.crop = {
+			.width = 3264,
+			.height = 2448,
+			.left = 8,
+			.top = 8,
+		},
 		.fll_def = 2615,
-		.fll_min = 2615,
 		.llp = 3672,
-		.link_freq_index = IMX355_LINK_FREQ_INDEX,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_3264x2448_regs),
 			.regs = mode_3264x2448_regs,
@@ -946,10 +390,14 @@ static const struct imx355_mode supported_modes[] = {
 	{
 		.width = 1940,
 		.height = 1096,
+		.crop = {
+			.width = 1940,
+			.height = 1096,
+			.left = 672,
+			.top = 684,
+		},
 		.fll_def = 1306,
-		.fll_min = 1306,
 		.llp = 3672,
-		.link_freq_index = IMX355_LINK_FREQ_INDEX,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_1940x1096_regs),
 			.regs = mode_1940x1096_regs,
@@ -958,10 +406,14 @@ static const struct imx355_mode supported_modes[] = {
 	{
 		.width = 1936,
 		.height = 1096,
+		.crop = {
+			.width = 1936,
+			.height = 1096,
+			.left = 672,
+			.top = 684,
+		},
 		.fll_def = 1306,
-		.fll_min = 1306,
 		.llp = 3672,
-		.link_freq_index = IMX355_LINK_FREQ_INDEX,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_1936x1096_regs),
 			.regs = mode_1936x1096_regs,
@@ -970,10 +422,14 @@ static const struct imx355_mode supported_modes[] = {
 	{
 		.width = 1924,
 		.height = 1080,
+		.crop = {
+			.width = 1924,
+			.height = 1080,
+			.left = 680,
+			.top = 692,
+		},
 		.fll_def = 1306,
-		.fll_min = 1306,
 		.llp = 3672,
-		.link_freq_index = IMX355_LINK_FREQ_INDEX,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_1924x1080_regs),
 			.regs = mode_1924x1080_regs,
@@ -982,10 +438,14 @@ static const struct imx355_mode supported_modes[] = {
 	{
 		.width = 1920,
 		.height = 1080,
+		.crop = {
+			.width = 1920,
+			.height = 1080,
+			.left = 680,
+			.top = 692,
+		},
 		.fll_def = 1306,
-		.fll_min = 1306,
 		.llp = 3672,
-		.link_freq_index = IMX355_LINK_FREQ_INDEX,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_1920x1080_regs),
 			.regs = mode_1920x1080_regs,
@@ -994,10 +454,14 @@ static const struct imx355_mode supported_modes[] = {
 	{
 		.width = 1640,
 		.height = 1232,
+		.crop = {
+			.width = 3280,
+			.height = 2464,
+			.left = 0,
+			.top = 0,
+		},
 		.fll_def = 1306,
-		.fll_min = 1306,
 		.llp = 1836,
-		.link_freq_index = IMX355_LINK_FREQ_INDEX,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_1640x1232_regs),
 			.regs = mode_1640x1232_regs,
@@ -1006,10 +470,14 @@ static const struct imx355_mode supported_modes[] = {
 	{
 		.width = 1640,
 		.height = 922,
+		.crop = {
+			.width = 3280,
+			.height = 1844,
+			.left = 0,
+			.top = 304,
+		},
 		.fll_def = 1306,
-		.fll_min = 1306,
 		.llp = 1836,
-		.link_freq_index = IMX355_LINK_FREQ_INDEX,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_1640x922_regs),
 			.regs = mode_1640x922_regs,
@@ -1018,10 +486,14 @@ static const struct imx355_mode supported_modes[] = {
 	{
 		.width = 1300,
 		.height = 736,
+		.crop = {
+			.width = 2600,
+			.height = 1472,
+			.left = 344,
+			.top = 496,
+		},
 		.fll_def = 1306,
-		.fll_min = 1306,
 		.llp = 1836,
-		.link_freq_index = IMX355_LINK_FREQ_INDEX,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_1300x736_regs),
 			.regs = mode_1300x736_regs,
@@ -1030,10 +502,14 @@ static const struct imx355_mode supported_modes[] = {
 	{
 		.width = 1296,
 		.height = 736,
+		.crop = {
+			.width = 2592,
+			.height = 1472,
+			.left = 344,
+			.top = 496,
+		},
 		.fll_def = 1306,
-		.fll_min = 1306,
 		.llp = 1836,
-		.link_freq_index = IMX355_LINK_FREQ_INDEX,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_1296x736_regs),
 			.regs = mode_1296x736_regs,
@@ -1042,10 +518,14 @@ static const struct imx355_mode supported_modes[] = {
 	{
 		.width = 1284,
 		.height = 720,
+		.crop = {
+			.width = 2568,
+			.height = 1440,
+			.left = 360,
+			.top = 512,
+		},
 		.fll_def = 1306,
-		.fll_min = 1306,
 		.llp = 1836,
-		.link_freq_index = IMX355_LINK_FREQ_INDEX,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_1284x720_regs),
 			.regs = mode_1284x720_regs,
@@ -1054,10 +534,14 @@ static const struct imx355_mode supported_modes[] = {
 	{
 		.width = 1280,
 		.height = 720,
+		.crop = {
+			.width = 2560,
+			.height = 1440,
+			.left = 360,
+			.top = 512,
+		},
 		.fll_def = 1306,
-		.fll_min = 1306,
 		.llp = 1836,
-		.link_freq_index = IMX355_LINK_FREQ_INDEX,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_1280x720_regs),
 			.regs = mode_1280x720_regs,
@@ -1066,10 +550,14 @@ static const struct imx355_mode supported_modes[] = {
 	{
 		.width = 820,
 		.height = 616,
+		.crop = {
+			.width = 3280,
+			.height = 2464,
+			.left = 0,
+			.top = 0,
+		},
 		.fll_def = 652,
-		.fll_min = 652,
 		.llp = 3672,
-		.link_freq_index = IMX355_LINK_FREQ_INDEX,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_820x616_regs),
 			.regs = mode_820x616_regs,
@@ -1095,116 +583,28 @@ static u32 imx355_get_format_code(struct imx355 *imx355)
 		{ MEDIA_BUS_FMT_SGBRG10_1X10, MEDIA_BUS_FMT_SBGGR10_1X10, },
 	};
 
-	lockdep_assert_held(&imx355->mutex);
 	code = codes[imx355->vflip->val][imx355->hflip->val];
 
 	return code;
 }
 
-/* Read registers up to 4 at a time */
-static int imx355_read_reg(struct imx355 *imx355, u16 reg, u32 len, u32 *val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&imx355->sd);
-	struct i2c_msg msgs[2];
-	u8 addr_buf[2];
-	u8 data_buf[4] = { 0 };
-	int ret;
-
-	if (len > 4)
-		return -EINVAL;
-
-	put_unaligned_be16(reg, addr_buf);
-	/* Write register address */
-	msgs[0].addr = client->addr;
-	msgs[0].flags = 0;
-	msgs[0].len = ARRAY_SIZE(addr_buf);
-	msgs[0].buf = addr_buf;
-
-	/* Read data from register */
-	msgs[1].addr = client->addr;
-	msgs[1].flags = I2C_M_RD;
-	msgs[1].len = len;
-	msgs[1].buf = &data_buf[4 - len];
-
-	ret = i2c_transfer(client->adapter, msgs, ARRAY_SIZE(msgs));
-	if (ret != ARRAY_SIZE(msgs))
-		return -EIO;
-
-	*val = get_unaligned_be32(data_buf);
-
-	return 0;
-}
-
-/* Write registers up to 4 at a time */
-static int imx355_write_reg(struct imx355 *imx355, u16 reg, u32 len, u32 val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&imx355->sd);
-	u8 buf[6];
-
-	if (len > 4)
-		return -EINVAL;
-
-	put_unaligned_be16(reg, buf);
-	put_unaligned_be32(val << (8 * (4 - len)), buf + 2);
-	if (i2c_master_send(client, buf, len + 2) != len + 2)
-		return -EIO;
-
-	return 0;
-}
-
-/* Write a list of registers */
-static int imx355_write_regs(struct imx355 *imx355,
-			     const struct imx355_reg *regs, u32 len)
-{
-	int ret;
-	u32 i;
-
-	for (i = 0; i < len; i++) {
-		ret = imx355_write_reg(imx355, regs[i].address, 1, regs[i].val);
-		if (ret) {
-			dev_err_ratelimited(imx355->dev,
-					    "write reg 0x%4.4x return err %d",
-					    regs[i].address, ret);
-
-			return ret;
-		}
-	}
-
-	return 0;
-}
-
-/* Open sub-device */
-static int imx355_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
-{
-	struct imx355 *imx355 = to_imx355(sd);
-	struct v4l2_mbus_framefmt *try_fmt =
-		v4l2_subdev_state_get_format(fh->state, 0);
-
-	mutex_lock(&imx355->mutex);
-
-	/* Initialize try_fmt */
-	try_fmt->width = imx355->cur_mode->width;
-	try_fmt->height = imx355->cur_mode->height;
-	try_fmt->code = imx355_get_format_code(imx355);
-	try_fmt->field = V4L2_FIELD_NONE;
-
-	mutex_unlock(&imx355->mutex);
-
-	return 0;
-}
-
 static int imx355_set_ctrl(struct v4l2_ctrl *ctrl)
 {
 	struct imx355 *imx355 = container_of(ctrl->handler,
 					     struct imx355, ctrl_handler);
+	const struct v4l2_mbus_framefmt *format = NULL;
+	struct v4l2_subdev_state *state;
 	s64 max;
 	int ret;
 
+	state = v4l2_subdev_get_locked_active_state(&imx355->sd);
+	format = v4l2_subdev_state_get_format(state, 0);
+
 	/* Propagate change of current control to all related controls */
 	switch (ctrl->id) {
 	case V4L2_CID_VBLANK:
 		/* Update max exposure while meeting expected vblanking */
-		max = imx355->cur_mode->height + ctrl->val - 10;
+		max = format->height + ctrl->val - IMX355_EXPOSURE_OFFSET;
 		__v4l2_ctrl_modify_range(imx355->exposure,
 					 imx355->exposure->minimum,
 					 max, imx355->exposure->step, max);
@@ -1221,31 +621,31 @@ static int imx355_set_ctrl(struct v4l2_ctrl *ctrl)
 	switch (ctrl->id) {
 	case V4L2_CID_ANALOGUE_GAIN:
 		/* Analog gain = 1024/(1024 - ctrl->val) times */
-		ret = imx355_write_reg(imx355, IMX355_REG_ANALOG_GAIN, 2,
-				       ctrl->val);
+		ret = cci_write(imx355->regmap, IMX355_REG_ANALOG_GAIN,
+				ctrl->val, NULL);
 		break;
 	case V4L2_CID_DIGITAL_GAIN:
-		ret = imx355_write_reg(imx355, IMX355_REG_DIG_GAIN_GLOBAL, 2,
-				       ctrl->val);
+		ret = cci_write(imx355->regmap, IMX355_REG_DIG_GAIN_GLOBAL,
+				ctrl->val, NULL);
 		break;
 	case V4L2_CID_EXPOSURE:
-		ret = imx355_write_reg(imx355, IMX355_REG_EXPOSURE, 2,
-				       ctrl->val);
+		ret = cci_write(imx355->regmap, IMX355_REG_EXPOSURE,
+				ctrl->val, NULL);
 		break;
 	case V4L2_CID_VBLANK:
 		/* Update FLL that meets expected vertical blanking */
-		ret = imx355_write_reg(imx355, IMX355_REG_FLL, 2,
-				       imx355->cur_mode->height + ctrl->val);
+		ret = cci_write(imx355->regmap, IMX355_REG_FLL,
+				format->height + ctrl->val, NULL);
 		break;
 	case V4L2_CID_TEST_PATTERN:
-		ret = imx355_write_reg(imx355, IMX355_REG_TEST_PATTERN,
-				       2, ctrl->val);
+		ret = cci_write(imx355->regmap, IMX355_REG_TEST_PATTERN,
+				ctrl->val, NULL);
 		break;
 	case V4L2_CID_HFLIP:
 	case V4L2_CID_VFLIP:
-		ret = imx355_write_reg(imx355, IMX355_REG_ORIENTATION, 1,
-				       imx355->hflip->val |
-				       imx355->vflip->val << 1);
+		ret = cci_write(imx355->regmap, IMX355_REG_ORIENTATION,
+				imx355->hflip->val | imx355->vflip->val << 1,
+				NULL);
 		break;
 	default:
 		ret = -EINVAL;
@@ -1272,9 +672,7 @@ static int imx355_enum_mbus_code(struct v4l2_subdev *sd,
 	if (code->index > 0)
 		return -EINVAL;
 
-	mutex_lock(&imx355->mutex);
 	code->code = imx355_get_format_code(imx355);
-	mutex_unlock(&imx355->mutex);
 
 	return 0;
 }
@@ -1288,12 +686,9 @@ static int imx355_enum_frame_size(struct v4l2_subdev *sd,
 	if (fse->index >= ARRAY_SIZE(supported_modes))
 		return -EINVAL;
 
-	mutex_lock(&imx355->mutex);
 	if (fse->code != imx355_get_format_code(imx355)) {
-		mutex_unlock(&imx355->mutex);
 		return -EINVAL;
 	}
-	mutex_unlock(&imx355->mutex);
 
 	fse->min_width = supported_modes[fse->index].width;
 	fse->max_width = fse->min_width;
@@ -1311,36 +706,10 @@ static void imx355_update_pad_format(struct imx355 *imx355,
 	fmt->format.height = mode->height;
 	fmt->format.code = imx355_get_format_code(imx355);
 	fmt->format.field = V4L2_FIELD_NONE;
-}
-
-static int imx355_do_get_pad_format(struct imx355 *imx355,
-				    struct v4l2_subdev_state *sd_state,
-				    struct v4l2_subdev_format *fmt)
-{
-	struct v4l2_mbus_framefmt *framefmt;
-
-	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
-		framefmt = v4l2_subdev_state_get_format(sd_state, fmt->pad);
-		fmt->format = *framefmt;
-	} else {
-		imx355_update_pad_format(imx355, imx355->cur_mode, fmt);
-	}
-
-	return 0;
-}
-
-static int imx355_get_pad_format(struct v4l2_subdev *sd,
-				 struct v4l2_subdev_state *sd_state,
-				 struct v4l2_subdev_format *fmt)
-{
-	struct imx355 *imx355 = to_imx355(sd);
-	int ret;
-
-	mutex_lock(&imx355->mutex);
-	ret = imx355_do_get_pad_format(imx355, sd_state, fmt);
-	mutex_unlock(&imx355->mutex);
-
-	return ret;
+	fmt->format.colorspace = V4L2_COLORSPACE_RAW;
+	fmt->format.ycbcr_enc = V4L2_YCBCR_ENC_601;
+	fmt->format.quantization = V4L2_QUANTIZATION_FULL_RANGE;
+	fmt->format.xfer_func = V4L2_XFER_FUNC_NONE;
 }
 
 static int
@@ -1351,13 +720,8 @@ imx355_set_pad_format(struct v4l2_subdev *sd,
 	struct imx355 *imx355 = to_imx355(sd);
 	const struct imx355_mode *mode;
 	struct v4l2_mbus_framefmt *framefmt;
-	s32 vblank_def;
-	s32 vblank_min;
+	struct v4l2_rect *crop;
 	s64 h_blank;
-	u64 pixel_rate;
-	u32 height;
-
-	mutex_lock(&imx355->mutex);
 
 	/*
 	 * Only one bayer order is supported.
@@ -1370,23 +734,25 @@ imx355_set_pad_format(struct v4l2_subdev *sd,
 				      width, height,
 				      fmt->format.width, fmt->format.height);
 	imx355_update_pad_format(imx355, mode, fmt);
-	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
-		framefmt = v4l2_subdev_state_get_format(sd_state, fmt->pad);
-		*framefmt = fmt->format;
-	} else {
-		imx355->cur_mode = mode;
-		pixel_rate = IMX355_LINK_FREQ_DEFAULT * 2 * 4;
-		do_div(pixel_rate, 10);
-		__v4l2_ctrl_s_ctrl_int64(imx355->pixel_rate, pixel_rate);
+	framefmt = v4l2_subdev_state_get_format(sd_state, 0);
+
+	*framefmt = fmt->format;
+
+	crop = v4l2_subdev_state_get_crop(sd_state, 0);
+	crop->width = mode->crop.width;
+	crop->height = mode->crop.height;
+	crop->left = mode->crop.left;
+	crop->top = mode->crop.top;
+
+	if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
 		/* Update limits and set FPS to default */
-		height = imx355->cur_mode->height;
-		vblank_def = imx355->cur_mode->fll_def - height;
-		vblank_min = imx355->cur_mode->fll_min - height;
-		height = IMX355_FLL_MAX - height;
-		__v4l2_ctrl_modify_range(imx355->vblank, vblank_min, height, 1,
-					 vblank_def);
-		__v4l2_ctrl_s_ctrl(imx355->vblank, vblank_def);
-		h_blank = mode->llp - imx355->cur_mode->width;
+		__v4l2_ctrl_modify_range(imx355->vblank, IMX355_VBLANK_MIN,
+					 IMX355_FLL_MAX - mode->height, 1,
+					 mode->fll_def - mode->height);
+		__v4l2_ctrl_s_ctrl(imx355->vblank, mode->fll_def - mode->height);
+
+		h_blank = mode->llp - mode->width;
+
 		/*
 		 * Currently hblank is not changeable.
 		 * So FPS control is done only by vblank.
@@ -1395,7 +761,42 @@ imx355_set_pad_format(struct v4l2_subdev *sd,
 					 h_blank, 1, h_blank);
 	}
 
-	mutex_unlock(&imx355->mutex);
+	return 0;
+}
+
+static int imx355_get_selection(struct v4l2_subdev *sd,
+				struct v4l2_subdev_state *sd_state,
+				struct v4l2_subdev_selection *sel)
+{
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP:
+		sel->r = *v4l2_subdev_state_get_crop(sd_state, 0);
+		return 0;
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_NATIVE_SIZE:
+		sel->r.top = IMX355_PIXEL_ARRAY_TOP;
+		sel->r.left = IMX355_PIXEL_ARRAY_LEFT;
+		sel->r.width = IMX355_PIXEL_ARRAY_WIDTH;
+		sel->r.height = IMX355_PIXEL_ARRAY_HEIGHT;
+
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static int imx355_entity_init_state(struct v4l2_subdev *subdev,
+				    struct v4l2_subdev_state *sd_state)
+{
+	struct v4l2_subdev_format fmt = { };
+
+	fmt.which = sd_state ? V4L2_SUBDEV_FORMAT_TRY : V4L2_SUBDEV_FORMAT_ACTIVE;
+	fmt.format.code = MEDIA_BUS_FMT_SRGGB10_1X10;
+	fmt.format.width = supported_modes[0].width;
+	fmt.format.height = supported_modes[0].height;
+
+	imx355_set_pad_format(subdev, sd_state, &fmt);
 
 	return 0;
 }
@@ -1403,52 +804,98 @@ imx355_set_pad_format(struct v4l2_subdev *sd,
 /* Start streaming */
 static int imx355_start_streaming(struct imx355 *imx355)
 {
-	const struct imx355_reg_list *reg_list;
-	int ret;
+	const struct v4l2_mbus_framefmt *fmt;
+	struct v4l2_subdev_state *state;
+	const struct imx355_mode *mode;
+	int lane_idx = imx355->hwcfg->num_lanes == 4 ? 0 : 1;
+	struct v4l2_rect *crop;
+	u64 link_bitrate;
+	u8 binning_mode;
+	int ret = 0;
 
 	/* Global Setting */
-	reg_list = &imx355_global_setting;
-	ret = imx355_write_regs(imx355, reg_list->regs, reg_list->num_of_regs);
-	if (ret) {
-		dev_err(imx355->dev, "failed to set global settings");
-		return ret;
-	}
+	cci_multi_reg_write(imx355->regmap, imx355_global_regs,
+			    ARRAY_SIZE(imx355_global_regs), &ret);
 
-	/* Apply default values of current mode */
-	reg_list = &imx355->cur_mode->reg_list;
-	ret = imx355_write_regs(imx355, reg_list->regs, reg_list->num_of_regs);
-	if (ret) {
-		dev_err(imx355->dev, "failed to set mode");
-		return ret;
-	}
+	/* Apply values of current mode */
+	state = v4l2_subdev_get_locked_active_state(&imx355->sd);
+	fmt = v4l2_subdev_state_get_format(state, 0);
+	crop = v4l2_subdev_state_get_crop(state, 0);
+	mode = v4l2_find_nearest_size(supported_modes,
+				      ARRAY_SIZE(supported_modes),
+				      width, height, fmt->width, fmt->height);
+	cci_multi_reg_write(imx355->regmap, mode->reg_list.regs,
+			    mode->reg_list.num_of_regs, &ret);
+
+	/* Set readout crop and size registers  */
+	cci_write(imx355->regmap, IMX355_REG_X_ADD_START, crop->left,
+		  &ret);
+	cci_write(imx355->regmap, IMX355_REG_Y_ADD_START, crop->top, &ret);
+	cci_write(imx355->regmap, IMX355_REG_X_ADD_END,
+		  crop->width + crop->left - 1, &ret);
+	cci_write(imx355->regmap, IMX355_REG_Y_ADD_END,
+		  crop->height + crop->top - 1, &ret);
+	cci_write(imx355->regmap, IMX355_REG_X_OUT_SIZE, fmt->width, &ret);
+	cci_write(imx355->regmap, IMX355_REG_Y_OUT_SIZE, fmt->height, &ret);
+
+	binning_mode = ((crop->width / fmt->width) << 4) |
+			(crop->height / fmt->height);
+	cci_write(imx355->regmap, IMX355_REG_BINNING_MODE,
+		  binning_mode == 0x11 ? 0x00 : 0x01, &ret);
+	cci_write(imx355->regmap, IMX355_REG_BINNING_TYPE, binning_mode, &ret);
+	cci_write(imx355->regmap, IMX355_REG_BINNING_WEIGHTING, 0x00, &ret);
+
+	/* Set PLL registers for the external clock frequency */
+	cci_write(imx355->regmap, IMX355_REG_EXTCLK_FREQ,
+		  imx355->clk_params->extclk_freq, &ret);
+	cci_write(imx355->regmap, IMX355_REG_PLL_OP_MUL,
+		  imx355->clk_params->pll_op_mpy[lane_idx], &ret);
+	cci_write(imx355->regmap, IMX355_REG_PLL_OP_PREDIV,
+		  imx355->clk_params->pll_op_prediv[lane_idx], &ret);
+	cci_write(imx355->regmap, IMX355_REG_PLL_IVT_SYSCK_DIV,
+		  lane_idx ? 2 : 1, &ret);
+
+	/* Set MIPI configuration */
+	cci_write(imx355->regmap, IMX355_REG_LANE_SEL,
+		  imx355->hwcfg->num_lanes - 1, &ret);
+
+	link_bitrate = imx355->link_freq->qmenu_int[imx355->link_freq->val] *
+		       imx355->hwcfg->num_lanes * 2;
+	do_div(link_bitrate, 1000000);
+	cci_write(imx355->regmap, IMX355_REG_REQ_LINK_BIT_RATE, link_bitrate,
+		  &ret);
 
 	/* set digital gain control to all color mode */
-	ret = imx355_write_reg(imx355, IMX355_REG_DPGA_USE_GLOBAL_GAIN, 1, 1);
-	if (ret)
-		return ret;
+	cci_write(imx355->regmap, IMX355_REG_DPGA_USE_GLOBAL_GAIN, 1, &ret);
+
+	/* set line length */
+	cci_write(imx355->regmap, IMX355_REG_LLP,
+		  imx355->hblank->val + fmt->width, &ret);
 
 	/* Apply customized values from user */
-	ret =  __v4l2_ctrl_handler_setup(imx355->sd.ctrl_handler);
-	if (ret)
-		return ret;
+	if (!ret)
+		ret = __v4l2_ctrl_handler_setup(imx355->sd.ctrl_handler);
+
+	cci_write(imx355->regmap, IMX355_REG_MODE_SELECT, IMX355_MODE_STREAMING,
+		  &ret);
 
-	return imx355_write_reg(imx355, IMX355_REG_MODE_SELECT,
-				1, IMX355_MODE_STREAMING);
+	return ret;
 }
 
 /* Stop streaming */
 static int imx355_stop_streaming(struct imx355 *imx355)
 {
-	return imx355_write_reg(imx355, IMX355_REG_MODE_SELECT,
-				1, IMX355_MODE_STANDBY);
+	return cci_write(imx355->regmap, IMX355_REG_MODE_SELECT,
+			 IMX355_MODE_STANDBY, NULL);
 }
 
 static int imx355_set_stream(struct v4l2_subdev *sd, int enable)
 {
 	struct imx355 *imx355 = to_imx355(sd);
+	struct v4l2_subdev_state *state;
 	int ret = 0;
 
-	mutex_lock(&imx355->mutex);
+	state = v4l2_subdev_lock_and_get_active_state(sd);
 
 	if (enable) {
 		ret = pm_runtime_resume_and_get(imx355->dev);
@@ -1464,21 +911,21 @@ static int imx355_set_stream(struct v4l2_subdev *sd, int enable)
 			goto err_rpm_put;
 	} else {
 		imx355_stop_streaming(imx355);
-		pm_runtime_put(imx355->dev);
+		pm_runtime_put_autosuspend(imx355->dev);
 	}
 
 	/* vflip and hflip cannot change during streaming */
 	__v4l2_ctrl_grab(imx355->vflip, enable);
 	__v4l2_ctrl_grab(imx355->hflip, enable);
 
-	mutex_unlock(&imx355->mutex);
+	v4l2_subdev_unlock_state(state);
 
 	return ret;
 
 err_rpm_put:
-	pm_runtime_put(imx355->dev);
+	pm_runtime_put_autosuspend(imx355->dev);
 err_unlock:
-	mutex_unlock(&imx355->mutex);
+	v4l2_subdev_unlock_state(state);
 
 	return ret;
 }
@@ -1487,14 +934,14 @@ static int imx355_set_stream(struct v4l2_subdev *sd, int enable)
 static int imx355_identify_module(struct imx355 *imx355)
 {
 	int ret;
-	u32 val;
+	u64 val;
 
-	ret = imx355_read_reg(imx355, IMX355_REG_CHIP_ID, 2, &val);
+	ret = cci_read(imx355->regmap, IMX355_REG_CHIP_ID, &val, NULL);
 	if (ret)
 		return ret;
 
 	if (val != IMX355_CHIP_ID) {
-		dev_err(imx355->dev, "chip id mismatch: %x!=%x",
+		dev_err(imx355->dev, "chip id mismatch: %x!=%llx",
 			IMX355_CHIP_ID, val);
 		return -EIO;
 	}
@@ -1512,9 +959,10 @@ static const struct v4l2_subdev_video_ops imx355_video_ops = {
 
 static const struct v4l2_subdev_pad_ops imx355_pad_ops = {
 	.enum_mbus_code = imx355_enum_mbus_code,
-	.get_fmt = imx355_get_pad_format,
+	.get_fmt = v4l2_subdev_get_fmt,
 	.set_fmt = imx355_set_pad_format,
 	.enum_frame_size = imx355_enum_frame_size,
+	.get_selection = imx355_get_selection,
 };
 
 static const struct v4l2_subdev_ops imx355_subdev_ops = {
@@ -1528,7 +976,7 @@ static const struct media_entity_operations imx355_subdev_entity_ops = {
 };
 
 static const struct v4l2_subdev_internal_ops imx355_internal_ops = {
-	.open = imx355_open,
+	.init_state = imx355_entity_init_state,
 };
 
 static int imx355_power_off(struct device *dev)
@@ -1582,13 +1030,11 @@ static int imx355_init_controls(struct imx355 *imx355)
 {
 	struct v4l2_fwnode_device_properties props;
 	struct v4l2_ctrl_handler *ctrl_hdlr;
+	const struct imx355_mode *mode = &supported_modes[0];
 	s64 exposure_max;
 	s64 vblank_def;
-	s64 vblank_min;
 	s64 hblank;
 	u64 pixel_rate;
-	const struct imx355_mode *mode;
-	u32 max;
 	int ret;
 
 	ctrl_hdlr = &imx355->ctrl_handler;
@@ -1596,40 +1042,34 @@ static int imx355_init_controls(struct imx355 *imx355)
 	if (ret)
 		return ret;
 
-	ctrl_hdlr->lock = &imx355->mutex;
-	max = ARRAY_SIZE(link_freq_menu_items) - 1;
 	imx355->link_freq = v4l2_ctrl_new_int_menu(ctrl_hdlr, &imx355_ctrl_ops,
-						   V4L2_CID_LINK_FREQ, max, 0,
-						   link_freq_menu_items);
+						   V4L2_CID_LINK_FREQ, 0, 0,
+						   &imx355->hwcfg->link_freq_menu);
 	if (imx355->link_freq)
 		imx355->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY;
 
 	/* pixel_rate = link_freq * 2 * nr_of_lanes / bits_per_sample */
-	pixel_rate = IMX355_LINK_FREQ_DEFAULT * 2 * 4;
+	pixel_rate = imx355->hwcfg->link_freq_menu * 2 * imx355->hwcfg->num_lanes;
 	do_div(pixel_rate, 10);
-	/* By default, PIXEL_RATE is read only */
-	imx355->pixel_rate = v4l2_ctrl_new_std(ctrl_hdlr, &imx355_ctrl_ops,
-					       V4L2_CID_PIXEL_RATE, pixel_rate,
-					       pixel_rate, 1, pixel_rate);
+
+	v4l2_ctrl_new_std(ctrl_hdlr, &imx355_ctrl_ops, V4L2_CID_PIXEL_RATE,
+			  pixel_rate, pixel_rate, 1, pixel_rate);
 
 	/* Initialize vblank/hblank/exposure parameters based on current mode */
-	mode = imx355->cur_mode;
 	vblank_def = mode->fll_def - mode->height;
-	vblank_min = mode->fll_min - mode->height;
 	imx355->vblank = v4l2_ctrl_new_std(ctrl_hdlr, &imx355_ctrl_ops,
-					   V4L2_CID_VBLANK, vblank_min,
+					   V4L2_CID_VBLANK, IMX355_VBLANK_MIN,
 					   IMX355_FLL_MAX - mode->height,
 					   1, vblank_def);
 
 	hblank = mode->llp - mode->width;
-	imx355->hblank = v4l2_ctrl_new_std(ctrl_hdlr, &imx355_ctrl_ops,
-					   V4L2_CID_HBLANK, hblank, hblank,
-					   1, hblank);
+	imx355->hblank = v4l2_ctrl_new_std(ctrl_hdlr, NULL, V4L2_CID_HBLANK,
+					   hblank, hblank, 1, hblank);
 	if (imx355->hblank)
 		imx355->hblank->flags |= V4L2_CTRL_FLAG_READ_ONLY;
 
 	/* fll >= exposure time + adjust parameter (default value is 10) */
-	exposure_max = mode->fll_def - 10;
+	exposure_max = mode->fll_def - IMX355_EXPOSURE_OFFSET;
 	imx355->exposure = v4l2_ctrl_new_std(ctrl_hdlr, &imx355_ctrl_ops,
 					     V4L2_CID_EXPOSURE,
 					     IMX355_EXPOSURE_MIN, exposure_max,
@@ -1683,14 +1123,17 @@ static int imx355_init_controls(struct imx355 *imx355)
 	return ret;
 }
 
-static struct imx355_hwcfg *imx355_get_hwcfg(struct device *dev)
+static struct imx355_hwcfg *imx355_get_hwcfg(struct imx355 *imx355)
 {
+	struct device *dev = imx355->dev;
 	struct imx355_hwcfg *cfg;
 	struct v4l2_fwnode_endpoint bus_cfg = {
 		.bus_type = V4L2_MBUS_CSI2_DPHY
 	};
+	const struct imx355_clk_params *clk = imx355->clk_params;
 	struct fwnode_handle *ep;
 	struct fwnode_handle *fwnode = dev_fwnode(dev);
+	int lane_idx;
 	int ret;
 
 	if (!fwnode)
@@ -1708,13 +1151,18 @@ static struct imx355_hwcfg *imx355_get_hwcfg(struct device *dev)
 	if (!cfg)
 		goto out_err;
 
-	if (bus_cfg.bus.mipi_csi2.num_data_lanes != IMX355_DATA_LANES)
+	if (bus_cfg.bus.mipi_csi2.num_data_lanes != 2 &&
+	    bus_cfg.bus.mipi_csi2.num_data_lanes != 4)
 		goto out_err;
 
+	cfg->num_lanes = bus_cfg.bus.mipi_csi2.num_data_lanes;
+
+	lane_idx = cfg->num_lanes == 4 ? 0 : 1;
+	cfg->link_freq_menu = (clk->ext_clk * clk->pll_op_mpy[lane_idx]) /
+			      (clk->pll_op_prediv[lane_idx] * 2);
 	ret = v4l2_link_freq_to_bitmap(dev, bus_cfg.link_frequencies,
 				       bus_cfg.nr_of_link_frequencies,
-				       link_freq_menu_items,
-				       ARRAY_SIZE(link_freq_menu_items),
+				       &cfg->link_freq_menu, 1,
 				       &cfg->link_freq_bitmap);
 	if (ret)
 		goto out_err;
@@ -1741,7 +1189,10 @@ static int imx355_probe(struct i2c_client *client)
 
 	imx355->dev = &client->dev;
 
-	mutex_init(&imx355->mutex);
+	imx355->regmap = devm_cci_regmap_init_i2c(client, 16);
+	if (IS_ERR(imx355->regmap))
+		return dev_err_probe(imx355->dev, PTR_ERR(imx355->regmap),
+				     "Unable to initialize I2C\n");
 
 	imx355->clk = devm_v4l2_sensor_clk_get(imx355->dev, NULL);
 	if (IS_ERR(imx355->clk))
@@ -1749,7 +1200,13 @@ static int imx355_probe(struct i2c_client *client)
 				     "failed to get clock\n");
 
 	freq = clk_get_rate(imx355->clk);
-	if (freq != IMX355_EXT_CLK)
+	for (unsigned int i = 0; i < ARRAY_SIZE(imx355_clk_params); i++) {
+		if (freq == imx355_clk_params[i].ext_clk) {
+			imx355->clk_params = &imx355_clk_params[i];
+			break;
+		}
+	}
+	if (!imx355->clk_params)
 		return dev_err_probe(imx355->dev, -EINVAL,
 				     "external clock %lu is not supported\n",
 				     freq);
@@ -1760,7 +1217,7 @@ static int imx355_probe(struct i2c_client *client)
 					    &imx355->supplies);
 	if (ret) {
 		dev_err_probe(imx355->dev, ret, "could not get regulators");
-		goto error_probe;
+		return ret;
 	}
 
 	imx355->reset_gpio = devm_gpiod_get_optional(imx355->dev, "reset",
@@ -1768,22 +1225,21 @@ static int imx355_probe(struct i2c_client *client)
 	if (IS_ERR(imx355->reset_gpio)) {
 		ret = dev_err_probe(imx355->dev, PTR_ERR(imx355->reset_gpio),
 				    "failed to get gpios");
-		goto error_probe;
+		return ret;
 	}
 
 	/* Initialize subdev */
 	v4l2_i2c_subdev_init(&imx355->sd, client, &imx355_subdev_ops);
 
-	imx355->hwcfg = imx355_get_hwcfg(imx355->dev);
+	imx355->hwcfg = imx355_get_hwcfg(imx355);
 	if (!imx355->hwcfg) {
 		dev_err(imx355->dev, "failed to get hwcfg");
-		ret = -ENODEV;
-		goto error_probe;
+		return -ENODEV;
 	}
 
 	ret = imx355_power_on(imx355->dev);
 	if (ret)
-		goto error_probe;
+		return ret;
 
 	/* Check module identity */
 	ret = imx355_identify_module(imx355);
@@ -1792,9 +1248,6 @@ static int imx355_probe(struct i2c_client *client)
 		goto error_power_off;
 	}
 
-	/* Set default mode to max resolution */
-	imx355->cur_mode = &supported_modes[0];
-
 	ret = imx355_init_controls(imx355);
 	if (ret) {
 		dev_err(imx355->dev, "failed to init controls: %d", ret);
@@ -1816,23 +1269,36 @@ static int imx355_probe(struct i2c_client *client)
 		goto error_handler_free;
 	}
 
+	imx355->sd.state_lock = imx355->ctrl_handler.lock;
+	ret = v4l2_subdev_init_finalize(&imx355->sd);
+	if (ret < 0) {
+		dev_err_probe(imx355->dev, ret, "subdev init error\n");
+		goto error_media_entity_free;
+	}
+
 	/*
 	 * Device is already turned on by i2c-core with ACPI domain PM.
 	 * Enable runtime PM and turn off the device.
 	 */
 	pm_runtime_set_active(imx355->dev);
 	pm_runtime_enable(imx355->dev);
-	pm_runtime_idle(imx355->dev);
+	pm_runtime_set_autosuspend_delay(imx355->dev, 1000);
+	pm_runtime_use_autosuspend(imx355->dev);
 
 	ret = v4l2_async_register_subdev_sensor(&imx355->sd);
 	if (ret < 0)
-		goto error_media_entity_runtime_pm;
+		goto error_subdev_cleanup_runtime_pm;
+
+	pm_runtime_idle(imx355->dev);
 
 	return 0;
 
-error_media_entity_runtime_pm:
+error_subdev_cleanup_runtime_pm:
 	pm_runtime_disable(imx355->dev);
 	pm_runtime_set_suspended(imx355->dev);
+	pm_runtime_dont_use_autosuspend(imx355->dev);
+	v4l2_subdev_cleanup(&imx355->sd);
+error_media_entity_free:
 	media_entity_cleanup(&imx355->sd.entity);
 
 error_handler_free:
@@ -1841,9 +1307,6 @@ static int imx355_probe(struct i2c_client *client)
 error_power_off:
 	imx355_power_off(imx355->dev);
 
-error_probe:
-	mutex_destroy(&imx355->mutex);
-
 	return ret;
 }
 
@@ -1853,6 +1316,7 @@ static void imx355_remove(struct i2c_client *client)
 	struct imx355 *imx355 = to_imx355(sd);
 
 	v4l2_async_unregister_subdev(sd);
+	v4l2_subdev_cleanup(sd);
 	media_entity_cleanup(&sd->entity);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 
@@ -1863,7 +1327,7 @@ static void imx355_remove(struct i2c_client *client)
 		pm_runtime_set_suspended(imx355->dev);
 	}
 
-	mutex_destroy(&imx355->mutex);
+	pm_runtime_dont_use_autosuspend(imx355->dev);
 }
 
 static const struct acpi_device_id imx355_acpi_ids[] __maybe_unused = {
diff --git a/drivers/media/i2c/imx412.c b/drivers/media/i2c/imx412.c
index 2705af2f16c0b..d2760e9e6194e 100644
--- a/drivers/media/i2c/imx412.c
+++ b/drivers/media/i2c/imx412.c
@@ -13,38 +13,39 @@
 #include <linux/pm_runtime.h>
 #include <linux/regulator/consumer.h>
 
+#include <media/v4l2-cci.h>
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-fwnode.h>
 #include <media/v4l2-subdev.h>
 
 /* Streaming Mode */
-#define IMX412_REG_MODE_SELECT	0x0100
+#define IMX412_REG_MODE_SELECT	CCI_REG8(0x0100)
 #define IMX412_MODE_STANDBY	0x00
 #define IMX412_MODE_STREAMING	0x01
 
 /* Lines per frame */
-#define IMX412_REG_LPFR		0x0340
+#define IMX412_REG_LPFR		CCI_REG16(0x0340)
 
 /* Chip ID */
-#define IMX412_REG_ID		0x0016
+#define IMX412_REG_ID		CCI_REG16(0x0016)
 #define IMX412_ID		0x577
 
 /* Exposure control */
-#define IMX412_REG_EXPOSURE_CIT	0x0202
+#define IMX412_REG_EXPOSURE_CIT	CCI_REG16(0x0202)
 #define IMX412_EXPOSURE_MIN	8
 #define IMX412_EXPOSURE_OFFSET	22
 #define IMX412_EXPOSURE_STEP	1
 #define IMX412_EXPOSURE_DEFAULT	0x0648
 
 /* Analog gain control */
-#define IMX412_REG_AGAIN	0x0204
+#define IMX412_REG_AGAIN	CCI_REG16(0x0204)
 #define IMX412_AGAIN_MIN	0
 #define IMX412_AGAIN_MAX	978
 #define IMX412_AGAIN_STEP	1
 #define IMX412_AGAIN_DEFAULT	0
 
 /* Group hold register */
-#define IMX412_REG_HOLD		0x0104
+#define IMX412_REG_HOLD		CCI_REG8(0x0104)
 
 /* Input clock rate */
 #define IMX412_INCLK_RATE	24000000
@@ -56,16 +57,6 @@
 #define IMX412_REG_MIN		0x00
 #define IMX412_REG_MAX		0xffff
 
-/**
- * struct imx412_reg - imx412 sensor register
- * @address: Register address
- * @val: Register value
- */
-struct imx412_reg {
-	u16 address;
-	u8 val;
-};
-
 /**
  * struct imx412_reg_list - imx412 sensor register list
  * @num_of_regs: Number of registers in the list
@@ -73,7 +64,7 @@ struct imx412_reg {
  */
 struct imx412_reg_list {
 	u32 num_of_regs;
-	const struct imx412_reg *regs;
+	const struct cci_reg_sequence *regs;
 };
 
 /**
@@ -111,6 +102,7 @@ static const char * const imx412_supply_names[] = {
 /**
  * struct imx412 - imx412 sensor device structure
  * @dev: Pointer to generic device
+ * @cci: CCI register map
  * @client: Pointer to i2c client
  * @sd: V4L2 sub-device
  * @pad: Media pad. Only one pad supported
@@ -126,10 +118,10 @@ static const char * const imx412_supply_names[] = {
  * @again_ctrl: Pointer to analog gain control
  * @vblank: Vertical blanking in lines
  * @cur_mode: Pointer to current selected sensor mode
- * @mutex: Mutex for serializing sensor controls
  */
 struct imx412 {
 	struct device *dev;
+	struct regmap *cci;
 	struct i2c_client *client;
 	struct v4l2_subdev sd;
 	struct media_pad pad;
@@ -147,7 +139,6 @@ struct imx412 {
 	};
 	u32 vblank;
 	const struct imx412_mode *cur_mode;
-	struct mutex mutex;
 };
 
 static const s64 link_freq[] = {
@@ -155,238 +146,238 @@ static const s64 link_freq[] = {
 };
 
 /* Sensor mode registers */
-static const struct imx412_reg mode_4056x3040_regs[] = {
-	{0x0136, 0x18},
-	{0x0137, 0x00},
-	{0x3c7e, 0x08},
-	{0x3c7f, 0x02},
-	{0x38a8, 0x1f},
-	{0x38a9, 0xff},
-	{0x38aa, 0x1f},
-	{0x38ab, 0xff},
-	{0x55d4, 0x00},
-	{0x55d5, 0x00},
-	{0x55d6, 0x07},
-	{0x55d7, 0xff},
-	{0x55e8, 0x07},
-	{0x55e9, 0xff},
-	{0x55ea, 0x00},
-	{0x55eb, 0x00},
-	{0x575c, 0x07},
-	{0x575d, 0xff},
-	{0x575e, 0x00},
-	{0x575f, 0x00},
-	{0x5764, 0x00},
-	{0x5765, 0x00},
-	{0x5766, 0x07},
-	{0x5767, 0xff},
-	{0x5974, 0x04},
-	{0x5975, 0x01},
-	{0x5f10, 0x09},
-	{0x5f11, 0x92},
-	{0x5f12, 0x32},
-	{0x5f13, 0x72},
-	{0x5f14, 0x16},
-	{0x5f15, 0xba},
-	{0x5f17, 0x13},
-	{0x5f18, 0x24},
-	{0x5f19, 0x60},
-	{0x5f1a, 0xe3},
-	{0x5f1b, 0xad},
-	{0x5f1c, 0x74},
-	{0x5f2d, 0x25},
-	{0x5f5c, 0xd0},
-	{0x6a22, 0x00},
-	{0x6a23, 0x1d},
-	{0x7ba8, 0x00},
-	{0x7ba9, 0x00},
-	{0x886b, 0x00},
-	{0x9002, 0x0a},
-	{0x9004, 0x1a},
-	{0x9214, 0x93},
-	{0x9215, 0x69},
-	{0x9216, 0x93},
-	{0x9217, 0x6b},
-	{0x9218, 0x93},
-	{0x9219, 0x6d},
-	{0x921a, 0x57},
-	{0x921b, 0x58},
-	{0x921c, 0x57},
-	{0x921d, 0x59},
-	{0x921e, 0x57},
-	{0x921f, 0x5a},
-	{0x9220, 0x57},
-	{0x9221, 0x5b},
-	{0x9222, 0x93},
-	{0x9223, 0x02},
-	{0x9224, 0x93},
-	{0x9225, 0x03},
-	{0x9226, 0x93},
-	{0x9227, 0x04},
-	{0x9228, 0x93},
-	{0x9229, 0x05},
-	{0x922a, 0x98},
-	{0x922b, 0x21},
-	{0x922c, 0xb2},
-	{0x922d, 0xdb},
-	{0x922e, 0xb2},
-	{0x922f, 0xdc},
-	{0x9230, 0xb2},
-	{0x9231, 0xdd},
-	{0x9232, 0xe2},
-	{0x9233, 0xe1},
-	{0x9234, 0xb2},
-	{0x9235, 0xe2},
-	{0x9236, 0xb2},
-	{0x9237, 0xe3},
-	{0x9238, 0xb7},
-	{0x9239, 0xb9},
-	{0x923a, 0xb7},
-	{0x923b, 0xbb},
-	{0x923c, 0xb7},
-	{0x923d, 0xbc},
-	{0x923e, 0xb7},
-	{0x923f, 0xc5},
-	{0x9240, 0xb7},
-	{0x9241, 0xc7},
-	{0x9242, 0xb7},
-	{0x9243, 0xc9},
-	{0x9244, 0x98},
-	{0x9245, 0x56},
-	{0x9246, 0x98},
-	{0x9247, 0x55},
-	{0x9380, 0x00},
-	{0x9381, 0x62},
-	{0x9382, 0x00},
-	{0x9383, 0x56},
-	{0x9384, 0x00},
-	{0x9385, 0x52},
-	{0x9388, 0x00},
-	{0x9389, 0x55},
-	{0x938a, 0x00},
-	{0x938b, 0x55},
-	{0x938c, 0x00},
-	{0x938d, 0x41},
-	{0x5078, 0x01},
-	{0x0112, 0x0a},
-	{0x0113, 0x0a},
-	{0x0114, 0x03},
-	{0x0342, 0x11},
-	{0x0343, 0xa0},
-	{0x0340, 0x0d},
-	{0x0341, 0xda},
-	{0x3210, 0x00},
-	{0x0344, 0x00},
-	{0x0345, 0x00},
-	{0x0346, 0x00},
-	{0x0347, 0x00},
-	{0x0348, 0x0f},
-	{0x0349, 0xd7},
-	{0x034a, 0x0b},
-	{0x034b, 0xdf},
-	{0x00e3, 0x00},
-	{0x00e4, 0x00},
-	{0x00e5, 0x01},
-	{0x00fc, 0x0a},
-	{0x00fd, 0x0a},
-	{0x00fe, 0x0a},
-	{0x00ff, 0x0a},
-	{0xe013, 0x00},
-	{0x0220, 0x00},
-	{0x0221, 0x11},
-	{0x0381, 0x01},
-	{0x0383, 0x01},
-	{0x0385, 0x01},
-	{0x0387, 0x01},
-	{0x0900, 0x00},
-	{0x0901, 0x11},
-	{0x0902, 0x00},
-	{0x3140, 0x02},
-	{0x3241, 0x11},
-	{0x3250, 0x03},
-	{0x3e10, 0x00},
-	{0x3e11, 0x00},
-	{0x3f0d, 0x00},
-	{0x3f42, 0x00},
-	{0x3f43, 0x00},
-	{0x0401, 0x00},
-	{0x0404, 0x00},
-	{0x0405, 0x10},
-	{0x0408, 0x00},
-	{0x0409, 0x00},
-	{0x040a, 0x00},
-	{0x040b, 0x00},
-	{0x040c, 0x0f},
-	{0x040d, 0xd8},
-	{0x040e, 0x0b},
-	{0x040f, 0xe0},
-	{0x034c, 0x0f},
-	{0x034d, 0xd8},
-	{0x034e, 0x0b},
-	{0x034f, 0xe0},
-	{0x0301, 0x05},
-	{0x0303, 0x02},
-	{0x0305, 0x04},
-	{0x0306, 0x00},
-	{0x0307, 0xc8},
-	{0x0309, 0x0a},
-	{0x030b, 0x01},
-	{0x030d, 0x02},
-	{0x030e, 0x01},
-	{0x030f, 0x5e},
-	{0x0310, 0x00},
-	{0x0820, 0x12},
-	{0x0821, 0xc0},
-	{0x0822, 0x00},
-	{0x0823, 0x00},
-	{0x3e20, 0x01},
-	{0x3e37, 0x00},
-	{0x3f50, 0x00},
-	{0x3f56, 0x00},
-	{0x3f57, 0xe2},
-	{0x3c0a, 0x5a},
-	{0x3c0b, 0x55},
-	{0x3c0c, 0x28},
-	{0x3c0d, 0x07},
-	{0x3c0e, 0xff},
-	{0x3c0f, 0x00},
-	{0x3c10, 0x00},
-	{0x3c11, 0x02},
-	{0x3c12, 0x00},
-	{0x3c13, 0x03},
-	{0x3c14, 0x00},
-	{0x3c15, 0x00},
-	{0x3c16, 0x0c},
-	{0x3c17, 0x0c},
-	{0x3c18, 0x0c},
-	{0x3c19, 0x0a},
-	{0x3c1a, 0x0a},
-	{0x3c1b, 0x0a},
-	{0x3c1c, 0x00},
-	{0x3c1d, 0x00},
-	{0x3c1e, 0x00},
-	{0x3c1f, 0x00},
-	{0x3c20, 0x00},
-	{0x3c21, 0x00},
-	{0x3c22, 0x3f},
-	{0x3c23, 0x0a},
-	{0x3e35, 0x01},
-	{0x3f4a, 0x03},
-	{0x3f4b, 0xbf},
-	{0x3f26, 0x00},
-	{0x0202, 0x0d},
-	{0x0203, 0xc4},
-	{0x0204, 0x00},
-	{0x0205, 0x00},
-	{0x020e, 0x01},
-	{0x020f, 0x00},
-	{0x0210, 0x01},
-	{0x0211, 0x00},
-	{0x0212, 0x01},
-	{0x0213, 0x00},
-	{0x0214, 0x01},
-	{0x0215, 0x00},
-	{0xbcf1, 0x00},
+static const struct cci_reg_sequence mode_4056x3040_regs[] = {
+	{ CCI_REG8(0x0136), 0x18 },
+	{ CCI_REG8(0x0137), 0x00 },
+	{ CCI_REG8(0x3c7e), 0x08 },
+	{ CCI_REG8(0x3c7f), 0x02 },
+	{ CCI_REG8(0x38a8), 0x1f },
+	{ CCI_REG8(0x38a9), 0xff },
+	{ CCI_REG8(0x38aa), 0x1f },
+	{ CCI_REG8(0x38ab), 0xff },
+	{ CCI_REG8(0x55d4), 0x00 },
+	{ CCI_REG8(0x55d5), 0x00 },
+	{ CCI_REG8(0x55d6), 0x07 },
+	{ CCI_REG8(0x55d7), 0xff },
+	{ CCI_REG8(0x55e8), 0x07 },
+	{ CCI_REG8(0x55e9), 0xff },
+	{ CCI_REG8(0x55ea), 0x00 },
+	{ CCI_REG8(0x55eb), 0x00 },
+	{ CCI_REG8(0x575c), 0x07 },
+	{ CCI_REG8(0x575d), 0xff },
+	{ CCI_REG8(0x575e), 0x00 },
+	{ CCI_REG8(0x575f), 0x00 },
+	{ CCI_REG8(0x5764), 0x00 },
+	{ CCI_REG8(0x5765), 0x00 },
+	{ CCI_REG8(0x5766), 0x07 },
+	{ CCI_REG8(0x5767), 0xff },
+	{ CCI_REG8(0x5974), 0x04 },
+	{ CCI_REG8(0x5975), 0x01 },
+	{ CCI_REG8(0x5f10), 0x09 },
+	{ CCI_REG8(0x5f11), 0x92 },
+	{ CCI_REG8(0x5f12), 0x32 },
+	{ CCI_REG8(0x5f13), 0x72 },
+	{ CCI_REG8(0x5f14), 0x16 },
+	{ CCI_REG8(0x5f15), 0xba },
+	{ CCI_REG8(0x5f17), 0x13 },
+	{ CCI_REG8(0x5f18), 0x24 },
+	{ CCI_REG8(0x5f19), 0x60 },
+	{ CCI_REG8(0x5f1a), 0xe3 },
+	{ CCI_REG8(0x5f1b), 0xad },
+	{ CCI_REG8(0x5f1c), 0x74 },
+	{ CCI_REG8(0x5f2d), 0x25 },
+	{ CCI_REG8(0x5f5c), 0xd0 },
+	{ CCI_REG8(0x6a22), 0x00 },
+	{ CCI_REG8(0x6a23), 0x1d },
+	{ CCI_REG8(0x7ba8), 0x00 },
+	{ CCI_REG8(0x7ba9), 0x00 },
+	{ CCI_REG8(0x886b), 0x00 },
+	{ CCI_REG8(0x9002), 0x0a },
+	{ CCI_REG8(0x9004), 0x1a },
+	{ CCI_REG8(0x9214), 0x93 },
+	{ CCI_REG8(0x9215), 0x69 },
+	{ CCI_REG8(0x9216), 0x93 },
+	{ CCI_REG8(0x9217), 0x6b },
+	{ CCI_REG8(0x9218), 0x93 },
+	{ CCI_REG8(0x9219), 0x6d },
+	{ CCI_REG8(0x921a), 0x57 },
+	{ CCI_REG8(0x921b), 0x58 },
+	{ CCI_REG8(0x921c), 0x57 },
+	{ CCI_REG8(0x921d), 0x59 },
+	{ CCI_REG8(0x921e), 0x57 },
+	{ CCI_REG8(0x921f), 0x5a },
+	{ CCI_REG8(0x9220), 0x57 },
+	{ CCI_REG8(0x9221), 0x5b },
+	{ CCI_REG8(0x9222), 0x93 },
+	{ CCI_REG8(0x9223), 0x02 },
+	{ CCI_REG8(0x9224), 0x93 },
+	{ CCI_REG8(0x9225), 0x03 },
+	{ CCI_REG8(0x9226), 0x93 },
+	{ CCI_REG8(0x9227), 0x04 },
+	{ CCI_REG8(0x9228), 0x93 },
+	{ CCI_REG8(0x9229), 0x05 },
+	{ CCI_REG8(0x922a), 0x98 },
+	{ CCI_REG8(0x922b), 0x21 },
+	{ CCI_REG8(0x922c), 0xb2 },
+	{ CCI_REG8(0x922d), 0xdb },
+	{ CCI_REG8(0x922e), 0xb2 },
+	{ CCI_REG8(0x922f), 0xdc },
+	{ CCI_REG8(0x9230), 0xb2 },
+	{ CCI_REG8(0x9231), 0xdd },
+	{ CCI_REG8(0x9232), 0xe2 },
+	{ CCI_REG8(0x9233), 0xe1 },
+	{ CCI_REG8(0x9234), 0xb2 },
+	{ CCI_REG8(0x9235), 0xe2 },
+	{ CCI_REG8(0x9236), 0xb2 },
+	{ CCI_REG8(0x9237), 0xe3 },
+	{ CCI_REG8(0x9238), 0xb7 },
+	{ CCI_REG8(0x9239), 0xb9 },
+	{ CCI_REG8(0x923a), 0xb7 },
+	{ CCI_REG8(0x923b), 0xbb },
+	{ CCI_REG8(0x923c), 0xb7 },
+	{ CCI_REG8(0x923d), 0xbc },
+	{ CCI_REG8(0x923e), 0xb7 },
+	{ CCI_REG8(0x923f), 0xc5 },
+	{ CCI_REG8(0x9240), 0xb7 },
+	{ CCI_REG8(0x9241), 0xc7 },
+	{ CCI_REG8(0x9242), 0xb7 },
+	{ CCI_REG8(0x9243), 0xc9 },
+	{ CCI_REG8(0x9244), 0x98 },
+	{ CCI_REG8(0x9245), 0x56 },
+	{ CCI_REG8(0x9246), 0x98 },
+	{ CCI_REG8(0x9247), 0x55 },
+	{ CCI_REG8(0x9380), 0x00 },
+	{ CCI_REG8(0x9381), 0x62 },
+	{ CCI_REG8(0x9382), 0x00 },
+	{ CCI_REG8(0x9383), 0x56 },
+	{ CCI_REG8(0x9384), 0x00 },
+	{ CCI_REG8(0x9385), 0x52 },
+	{ CCI_REG8(0x9388), 0x00 },
+	{ CCI_REG8(0x9389), 0x55 },
+	{ CCI_REG8(0x938a), 0x00 },
+	{ CCI_REG8(0x938b), 0x55 },
+	{ CCI_REG8(0x938c), 0x00 },
+	{ CCI_REG8(0x938d), 0x41 },
+	{ CCI_REG8(0x5078), 0x01 },
+	{ CCI_REG8(0x0112), 0x0a },
+	{ CCI_REG8(0x0113), 0x0a },
+	{ CCI_REG8(0x0114), 0x03 },
+	{ CCI_REG8(0x0342), 0x11 },
+	{ CCI_REG8(0x0343), 0xa0 },
+	{ CCI_REG8(0x0340), 0x0d },
+	{ CCI_REG8(0x0341), 0xda },
+	{ CCI_REG8(0x3210), 0x00 },
+	{ CCI_REG8(0x0344), 0x00 },
+	{ CCI_REG8(0x0345), 0x00 },
+	{ CCI_REG8(0x0346), 0x00 },
+	{ CCI_REG8(0x0347), 0x00 },
+	{ CCI_REG8(0x0348), 0x0f },
+	{ CCI_REG8(0x0349), 0xd7 },
+	{ CCI_REG8(0x034a), 0x0b },
+	{ CCI_REG8(0x034b), 0xdf },
+	{ CCI_REG8(0x00e3), 0x00 },
+	{ CCI_REG8(0x00e4), 0x00 },
+	{ CCI_REG8(0x00e5), 0x01 },
+	{ CCI_REG8(0x00fc), 0x0a },
+	{ CCI_REG8(0x00fd), 0x0a },
+	{ CCI_REG8(0x00fe), 0x0a },
+	{ CCI_REG8(0x00ff), 0x0a },
+	{ CCI_REG8(0xe013), 0x00 },
+	{ CCI_REG8(0x0220), 0x00 },
+	{ CCI_REG8(0x0221), 0x11 },
+	{ CCI_REG8(0x0381), 0x01 },
+	{ CCI_REG8(0x0383), 0x01 },
+	{ CCI_REG8(0x0385), 0x01 },
+	{ CCI_REG8(0x0387), 0x01 },
+	{ CCI_REG8(0x0900), 0x00 },
+	{ CCI_REG8(0x0901), 0x11 },
+	{ CCI_REG8(0x0902), 0x00 },
+	{ CCI_REG8(0x3140), 0x02 },
+	{ CCI_REG8(0x3241), 0x11 },
+	{ CCI_REG8(0x3250), 0x03 },
+	{ CCI_REG8(0x3e10), 0x00 },
+	{ CCI_REG8(0x3e11), 0x00 },
+	{ CCI_REG8(0x3f0d), 0x00 },
+	{ CCI_REG8(0x3f42), 0x00 },
+	{ CCI_REG8(0x3f43), 0x00 },
+	{ CCI_REG8(0x0401), 0x00 },
+	{ CCI_REG8(0x0404), 0x00 },
+	{ CCI_REG8(0x0405), 0x10 },
+	{ CCI_REG8(0x0408), 0x00 },
+	{ CCI_REG8(0x0409), 0x00 },
+	{ CCI_REG8(0x040a), 0x00 },
+	{ CCI_REG8(0x040b), 0x00 },
+	{ CCI_REG8(0x040c), 0x0f },
+	{ CCI_REG8(0x040d), 0xd8 },
+	{ CCI_REG8(0x040e), 0x0b },
+	{ CCI_REG8(0x040f), 0xe0 },
+	{ CCI_REG8(0x034c), 0x0f },
+	{ CCI_REG8(0x034d), 0xd8 },
+	{ CCI_REG8(0x034e), 0x0b },
+	{ CCI_REG8(0x034f), 0xe0 },
+	{ CCI_REG8(0x0301), 0x05 },
+	{ CCI_REG8(0x0303), 0x02 },
+	{ CCI_REG8(0x0305), 0x04 },
+	{ CCI_REG8(0x0306), 0x00 },
+	{ CCI_REG8(0x0307), 0xc8 },
+	{ CCI_REG8(0x0309), 0x0a },
+	{ CCI_REG8(0x030b), 0x01 },
+	{ CCI_REG8(0x030d), 0x02 },
+	{ CCI_REG8(0x030e), 0x01 },
+	{ CCI_REG8(0x030f), 0x5e },
+	{ CCI_REG8(0x0310), 0x00 },
+	{ CCI_REG8(0x0820), 0x12 },
+	{ CCI_REG8(0x0821), 0xc0 },
+	{ CCI_REG8(0x0822), 0x00 },
+	{ CCI_REG8(0x0823), 0x00 },
+	{ CCI_REG8(0x3e20), 0x01 },
+	{ CCI_REG8(0x3e37), 0x00 },
+	{ CCI_REG8(0x3f50), 0x00 },
+	{ CCI_REG8(0x3f56), 0x00 },
+	{ CCI_REG8(0x3f57), 0xe2 },
+	{ CCI_REG8(0x3c0a), 0x5a },
+	{ CCI_REG8(0x3c0b), 0x55 },
+	{ CCI_REG8(0x3c0c), 0x28 },
+	{ CCI_REG8(0x3c0d), 0x07 },
+	{ CCI_REG8(0x3c0e), 0xff },
+	{ CCI_REG8(0x3c0f), 0x00 },
+	{ CCI_REG8(0x3c10), 0x00 },
+	{ CCI_REG8(0x3c11), 0x02 },
+	{ CCI_REG8(0x3c12), 0x00 },
+	{ CCI_REG8(0x3c13), 0x03 },
+	{ CCI_REG8(0x3c14), 0x00 },
+	{ CCI_REG8(0x3c15), 0x00 },
+	{ CCI_REG8(0x3c16), 0x0c },
+	{ CCI_REG8(0x3c17), 0x0c },
+	{ CCI_REG8(0x3c18), 0x0c },
+	{ CCI_REG8(0x3c19), 0x0a },
+	{ CCI_REG8(0x3c1a), 0x0a },
+	{ CCI_REG8(0x3c1b), 0x0a },
+	{ CCI_REG8(0x3c1c), 0x00 },
+	{ CCI_REG8(0x3c1d), 0x00 },
+	{ CCI_REG8(0x3c1e), 0x00 },
+	{ CCI_REG8(0x3c1f), 0x00 },
+	{ CCI_REG8(0x3c20), 0x00 },
+	{ CCI_REG8(0x3c21), 0x00 },
+	{ CCI_REG8(0x3c22), 0x3f },
+	{ CCI_REG8(0x3c23), 0x0a },
+	{ CCI_REG8(0x3e35), 0x01 },
+	{ CCI_REG8(0x3f4a), 0x03 },
+	{ CCI_REG8(0x3f4b), 0xbf },
+	{ CCI_REG8(0x3f26), 0x00 },
+	{ CCI_REG8(0x0202), 0x0d },
+	{ CCI_REG8(0x0203), 0xc4 },
+	{ CCI_REG8(0x0204), 0x00 },
+	{ CCI_REG8(0x0205), 0x00 },
+	{ CCI_REG8(0x020e), 0x01 },
+	{ CCI_REG8(0x020f), 0x00 },
+	{ CCI_REG8(0x0210), 0x01 },
+	{ CCI_REG8(0x0211), 0x00 },
+	{ CCI_REG8(0x0212), 0x01 },
+	{ CCI_REG8(0x0213), 0x00 },
+	{ CCI_REG8(0x0214), 0x01 },
+	{ CCI_REG8(0x0215), 0x00 },
+	{ CCI_REG8(0xbcf1), 0x00 },
 };
 
 /* Supported sensor mode configurations */
@@ -417,97 +408,6 @@ static inline struct imx412 *to_imx412(struct v4l2_subdev *subdev)
 	return container_of(subdev, struct imx412, sd);
 }
 
-/**
- * imx412_read_reg() - Read registers.
- * @imx412: pointer to imx412 device
- * @reg: register address
- * @len: length of bytes to read. Max supported bytes is 4
- * @val: pointer to register value to be filled.
- *
- * Return: 0 if successful, error code otherwise.
- */
-static int imx412_read_reg(struct imx412 *imx412, u16 reg, u32 len, u32 *val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&imx412->sd);
-	struct i2c_msg msgs[2] = {0};
-	u8 addr_buf[2] = {0};
-	u8 data_buf[4] = {0};
-	int ret;
-
-	if (WARN_ON(len > 4))
-		return -EINVAL;
-
-	put_unaligned_be16(reg, addr_buf);
-
-	/* Write register address */
-	msgs[0].addr = client->addr;
-	msgs[0].flags = 0;
-	msgs[0].len = ARRAY_SIZE(addr_buf);
-	msgs[0].buf = addr_buf;
-
-	/* Read data from register */
-	msgs[1].addr = client->addr;
-	msgs[1].flags = I2C_M_RD;
-	msgs[1].len = len;
-	msgs[1].buf = &data_buf[4 - len];
-
-	ret = i2c_transfer(client->adapter, msgs, ARRAY_SIZE(msgs));
-	if (ret != ARRAY_SIZE(msgs))
-		return -EIO;
-
-	*val = get_unaligned_be32(data_buf);
-
-	return 0;
-}
-
-/**
- * imx412_write_reg() - Write register
- * @imx412: pointer to imx412 device
- * @reg: register address
- * @len: length of bytes. Max supported bytes is 4
- * @val: register value
- *
- * Return: 0 if successful, error code otherwise.
- */
-static int imx412_write_reg(struct imx412 *imx412, u16 reg, u32 len, u32 val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&imx412->sd);
-	u8 buf[6] = {0};
-
-	if (WARN_ON(len > 4))
-		return -EINVAL;
-
-	put_unaligned_be16(reg, buf);
-	put_unaligned_be32(val << (8 * (4 - len)), buf + 2);
-	if (i2c_master_send(client, buf, len + 2) != len + 2)
-		return -EIO;
-
-	return 0;
-}
-
-/**
- * imx412_write_regs() - Write a list of registers
- * @imx412: pointer to imx412 device
- * @regs: list of registers to be written
- * @len: length of registers array
- *
- * Return: 0 if successful, error code otherwise.
- */
-static int imx412_write_regs(struct imx412 *imx412,
-			     const struct imx412_reg *regs, u32 len)
-{
-	unsigned int i;
-	int ret;
-
-	for (i = 0; i < len; i++) {
-		ret = imx412_write_reg(imx412, regs[i].address, 1, regs[i].val);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
 /**
  * imx412_update_controls() - Update control ranges based on streaming mode
  * @imx412: pointer to imx412 device
@@ -543,29 +443,25 @@ static int imx412_update_controls(struct imx412 *imx412,
 static int imx412_update_exp_gain(struct imx412 *imx412, u32 exposure, u32 gain)
 {
 	u32 lpfr;
-	int ret;
+	int ret = 0;
+	int ret_hold;
 
 	lpfr = imx412->vblank + imx412->cur_mode->height;
 
 	dev_dbg(imx412->dev, "Set exp %u, analog gain %u, lpfr %u\n",
 		exposure, gain, lpfr);
 
-	ret = imx412_write_reg(imx412, IMX412_REG_HOLD, 1, 1);
-	if (ret)
-		return ret;
+	cci_write(imx412->cci, IMX412_REG_HOLD, 1, &ret);
 
-	ret = imx412_write_reg(imx412, IMX412_REG_LPFR, 2, lpfr);
-	if (ret)
-		goto error_release_group_hold;
+	cci_write(imx412->cci, IMX412_REG_LPFR, lpfr, &ret);
 
-	ret = imx412_write_reg(imx412, IMX412_REG_EXPOSURE_CIT, 2, exposure);
-	if (ret)
-		goto error_release_group_hold;
+	cci_write(imx412->cci, IMX412_REG_EXPOSURE_CIT, exposure, &ret);
 
-	ret = imx412_write_reg(imx412, IMX412_REG_AGAIN, 2, gain);
+	cci_write(imx412->cci, IMX412_REG_AGAIN, gain, &ret);
 
-error_release_group_hold:
-	imx412_write_reg(imx412, IMX412_REG_HOLD, 1, 0);
+	ret_hold = cci_write(imx412->cci, IMX412_REG_HOLD, 0, NULL);
+	if (ret_hold)
+		return ret_hold;
 
 	return ret;
 }
@@ -679,8 +575,6 @@ static int imx412_get_pad_format(struct v4l2_subdev *sd,
 {
 	struct imx412 *imx412 = to_imx412(sd);
 
-	mutex_lock(&imx412->mutex);
-
 	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
 		struct v4l2_mbus_framefmt *framefmt;
 
@@ -690,8 +584,6 @@ static int imx412_get_pad_format(struct v4l2_subdev *sd,
 		imx412_fill_pad_format(imx412, imx412->cur_mode, fmt);
 	}
 
-	mutex_unlock(&imx412->mutex);
-
 	return 0;
 }
 
@@ -703,8 +595,6 @@ static int imx412_set_pad_format(struct v4l2_subdev *sd,
 	const struct imx412_mode *mode;
 	int ret = 0;
 
-	mutex_lock(&imx412->mutex);
-
 	mode = &supported_mode;
 	imx412_fill_pad_format(imx412, mode, fmt);
 
@@ -719,8 +609,6 @@ static int imx412_set_pad_format(struct v4l2_subdev *sd,
 			imx412->cur_mode = mode;
 	}
 
-	mutex_unlock(&imx412->mutex);
-
 	return ret;
 }
 
@@ -737,88 +625,86 @@ static int imx412_init_state(struct v4l2_subdev *sd,
 }
 
 /**
- * imx412_start_streaming() - Start sensor stream
- * @imx412: pointer to imx412 device
+ * imx412_enable_streams() - Enable specified streams for the sensor
+ * @sd: pointer to the V4L2 subdevice
+ * @state: pointer to the subdevice state
+ * @pad: pad number for which streams are enabled
+ * @streams_mask: bitmask specifying the streams to enable
  *
  * Return: 0 if successful, error code otherwise.
  */
-static int imx412_start_streaming(struct imx412 *imx412)
+static int imx412_enable_streams(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *state,
+				 u32 pad, u64 streams_mask)
 {
 	const struct imx412_reg_list *reg_list;
+	struct imx412 *imx412 = to_imx412(sd);
 	int ret;
 
+	ret = pm_runtime_resume_and_get(imx412->dev);
+	if (ret < 0)
+		return ret;
+
 	/* Write sensor mode registers */
 	reg_list = &imx412->cur_mode->reg_list;
-	ret = imx412_write_regs(imx412, reg_list->regs,
-				reg_list->num_of_regs);
+	ret = cci_multi_reg_write(imx412->cci, reg_list->regs,
+				  reg_list->num_of_regs, NULL);
 	if (ret) {
 		dev_err(imx412->dev, "fail to write initial registers\n");
-		return ret;
+		goto err_rpm_put;
 	}
 
 	/* Setup handler will write actual exposure and gain */
 	ret =  __v4l2_ctrl_handler_setup(imx412->sd.ctrl_handler);
 	if (ret) {
 		dev_err(imx412->dev, "fail to setup handler\n");
-		return ret;
+		goto err_rpm_put;
 	}
 
 	/* Delay is required before streaming*/
 	usleep_range(7400, 8000);
 
 	/* Start streaming */
-	ret = imx412_write_reg(imx412, IMX412_REG_MODE_SELECT,
-			       1, IMX412_MODE_STREAMING);
+	ret = cci_write(imx412->cci, IMX412_REG_MODE_SELECT,
+			IMX412_MODE_STREAMING, NULL);
 	if (ret) {
 		dev_err(imx412->dev, "fail to start streaming\n");
-		return ret;
+		goto err_rpm_put;
 	}
 
 	return 0;
+
+err_rpm_put:
+	pm_runtime_put(imx412->dev);
+
+	return ret;
 }
 
 /**
- * imx412_stop_streaming() - Stop sensor stream
- * @imx412: pointer to imx412 device
+ * imx412_disable_streams() - Enable specified streams for the sensor
+ * @sd: pointer to the V4L2 subdevice
+ * @state: pointer to the subdevice state
+ * @pad: pad number for which streams are disabled
+ * @streams_mask: bitmask specifying the streams to disable
  *
  * Return: 0 if successful, error code otherwise.
  */
-static int imx412_stop_streaming(struct imx412 *imx412)
-{
-	return imx412_write_reg(imx412, IMX412_REG_MODE_SELECT,
-				1, IMX412_MODE_STANDBY);
-}
-
-static int imx412_set_stream(struct v4l2_subdev *sd, int enable)
+static int imx412_disable_streams(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_state *state,
+				  u32 pad, u64 streams_mask)
 {
 	struct imx412 *imx412 = to_imx412(sd);
 	int ret;
 
-	mutex_lock(&imx412->mutex);
+	ret = cci_write(imx412->cci, IMX412_REG_MODE_SELECT,
+			IMX412_MODE_STANDBY, NULL);
 
-	if (enable) {
-		ret = pm_runtime_resume_and_get(imx412->dev);
-		if (ret)
-			goto error_unlock;
-
-		ret = imx412_start_streaming(imx412);
-		if (ret)
-			goto error_power_off;
-	} else {
-		imx412_stop_streaming(imx412);
-		pm_runtime_put(imx412->dev);
-	}
-
-	mutex_unlock(&imx412->mutex);
-
-	return 0;
+	if (ret)
+		dev_err(imx412->dev, "failed to set stream off\n");
 
-error_power_off:
 	pm_runtime_put(imx412->dev);
-error_unlock:
-	mutex_unlock(&imx412->mutex);
 
-	return ret;
+	return 0;
 }
 
 /**
@@ -830,16 +716,18 @@ static int imx412_set_stream(struct v4l2_subdev *sd, int enable)
 static int imx412_detect(struct imx412 *imx412)
 {
 	int ret;
-	u32 val;
+	u64 val;
 
-	ret = imx412_read_reg(imx412, IMX412_REG_ID, 2, &val);
+	ret = cci_read(imx412->cci, IMX412_REG_ID, &val, NULL);
 	if (ret)
-		return ret;
+		return dev_err_probe(imx412->dev, ret,
+				     "failed to read chip id %x\n",
+				     IMX412_ID);
 
 	if (val != IMX412_ID) {
-		dev_err(imx412->dev, "chip id mismatch: %x!=%x\n",
-			IMX412_ID, val);
-		return -ENXIO;
+		return dev_err_probe(imx412->dev, -ENODEV,
+				     "chip id mismatch: %x!=%llx",
+				     IMX412_ID, val);
 	}
 
 	return 0;
@@ -933,7 +821,7 @@ static int imx412_parse_hw_config(struct imx412 *imx412)
 
 /* V4l2 subdevice ops */
 static const struct v4l2_subdev_video_ops imx412_video_ops = {
-	.s_stream = imx412_set_stream,
+	.s_stream = v4l2_subdev_s_stream_helper,
 };
 
 static const struct v4l2_subdev_pad_ops imx412_pad_ops = {
@@ -941,6 +829,8 @@ static const struct v4l2_subdev_pad_ops imx412_pad_ops = {
 	.enum_frame_size = imx412_enum_frame_size,
 	.get_fmt = imx412_get_pad_format,
 	.set_fmt = imx412_set_pad_format,
+	.enable_streams = imx412_enable_streams,
+	.disable_streams = imx412_disable_streams,
 };
 
 static const struct v4l2_subdev_ops imx412_subdev_ops = {
@@ -1021,9 +911,6 @@ static int imx412_init_controls(struct imx412 *imx412)
 	if (ret)
 		return ret;
 
-	/* Serialize controls with sensor device */
-	ctrl_hdlr->lock = &imx412->mutex;
-
 	/* Initialize exposure and gain */
 	lpfr = mode->vblank + mode->height;
 	imx412->exp_ctrl = v4l2_ctrl_new_std(ctrl_hdlr,
@@ -1104,6 +991,11 @@ static int imx412_probe(struct i2c_client *client)
 	if (!name)
 		return -ENODEV;
 
+	imx412->cci = devm_cci_regmap_init_i2c(client, 16);
+	if (IS_ERR(imx412->cci))
+		return dev_err_probe(imx412->dev, PTR_ERR(imx412->cci),
+				     "Failed to init CCI\n");
+
 	/* Initialize subdev */
 	v4l2_i2c_subdev_init(&imx412->sd, client, &imx412_subdev_ops);
 	imx412->sd.internal_ops = &imx412_internal_ops;
@@ -1114,13 +1006,10 @@ static int imx412_probe(struct i2c_client *client)
 		return ret;
 	}
 
-	mutex_init(&imx412->mutex);
-
 	ret = imx412_power_on(imx412->dev);
-	if (ret) {
-		dev_err(imx412->dev, "failed to power-on the sensor\n");
-		goto error_mutex_destroy;
-	}
+	if (ret)
+		return dev_err_probe(imx412->dev, ret,
+				     "failed to power-on the sensor\n");
 
 	/* Check module identity */
 	ret = imx412_detect(imx412);
@@ -1153,27 +1042,37 @@ static int imx412_probe(struct i2c_client *client)
 		goto error_handler_free;
 	}
 
-	ret = v4l2_async_register_subdev_sensor(&imx412->sd);
+	imx412->sd.state_lock = imx412->ctrl_handler.lock;
+	ret = v4l2_subdev_init_finalize(&imx412->sd);
 	if (ret < 0) {
-		dev_err(imx412->dev,
-			"failed to register async subdev: %d\n", ret);
+		dev_err_probe(imx412->dev, ret, "subdev init error\n");
 		goto error_media_entity;
 	}
 
 	pm_runtime_set_active(imx412->dev);
 	pm_runtime_enable(imx412->dev);
+
+	ret = v4l2_async_register_subdev_sensor(&imx412->sd);
+	if (ret < 0) {
+		dev_err_probe(imx412->dev, ret,
+			      "failed to register sub-device\n");
+		goto error_subdev_cleanup;
+	}
+
 	pm_runtime_idle(imx412->dev);
 
 	return 0;
 
+error_subdev_cleanup:
+	v4l2_subdev_cleanup(&imx412->sd);
+	pm_runtime_disable(imx412->dev);
+	pm_runtime_set_suspended(imx412->dev);
 error_media_entity:
 	media_entity_cleanup(&imx412->sd.entity);
 error_handler_free:
 	v4l2_ctrl_handler_free(imx412->sd.ctrl_handler);
 error_power_off:
 	imx412_power_off(imx412->dev);
-error_mutex_destroy:
-	mutex_destroy(&imx412->mutex);
 
 	return ret;
 }
@@ -1181,9 +1080,9 @@ static int imx412_probe(struct i2c_client *client)
 static void imx412_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
-	struct imx412 *imx412 = to_imx412(sd);
 
 	v4l2_async_unregister_subdev(sd);
+	v4l2_subdev_cleanup(sd);
 	media_entity_cleanup(&sd->entity);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 
@@ -1191,8 +1090,6 @@ static void imx412_remove(struct i2c_client *client)
 	if (!pm_runtime_status_suspended(&client->dev))
 		imx412_power_off(&client->dev);
 	pm_runtime_set_suspended(&client->dev);
-
-	mutex_destroy(&imx412->mutex);
 }
 
 static const struct dev_pm_ops imx412_pm_ops = {
diff --git a/drivers/media/i2c/imx471.c b/drivers/media/i2c/imx471.c
new file mode 100644
index 0000000000000..6d358b11e96dd
--- /dev/null
+++ b/drivers/media/i2c/imx471.c
@@ -0,0 +1,957 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * imx471.c - imx471 sensor driver
+ *
+ * Copyright (C) 2025 Intel Corporation
+ * Copyright (C) 2026 Kate Hsuan <hpa@redhat.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/pm_runtime.h>
+#include <linux/regulator/consumer.h>
+#include <media/v4l2-cci.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-subdev.h>
+#include <media/v4l2-fwnode.h>
+
+#define IMX471_REG_MODE_SELECT			CCI_REG8(0x0100)
+#define IMX471_MODE_STANDBY			0x00
+#define IMX471_MODE_STREAMING			0x01
+
+/* Chip ID */
+#define IMX471_REG_CHIP_ID			CCI_REG16(0x0016)
+#define IMX471_CHIP_ID				0x0471
+
+/* V_TIMING internal */
+#define IMX471_REG_FLL				CCI_REG16(0x0340)
+#define IMX471_FLL_MAX				0xffff
+
+/* Exposure control */
+#define IMX471_REG_EXPOSURE			CCI_REG16(0x0202)
+#define IMX471_EXPOSURE_MIN			1
+#define IMX471_EXPOSURE_STEP			1
+#define IMX471_EXPOSURE_DEFAULT			1270
+
+/* Default exposure margin */
+#define IMX471_EXPOSURE_MARGIN			18
+
+/* Analog gain control */
+#define IMX471_REG_ANALOG_GAIN			CCI_REG16(0x0204)
+#define IMX471_ANA_GAIN_MIN			0
+#define IMX471_ANA_GAIN_MAX			800
+#define IMX471_ANA_GAIN_STEP			1
+#define IMX471_ANA_GAIN_DEFAULT			0
+
+/* Digital gain control */
+#define IMX471_REG_DPGA_USE_GLOBAL_GAIN		CCI_REG16(0x3ff9)
+#define IMX471_REG_DIG_GAIN_GLOBAL		CCI_REG16(0x020e)
+#define IMX471_DGTL_GAIN_MIN			256
+#define IMX471_DGTL_GAIN_MAX			4095
+#define IMX471_DGTL_GAIN_STEP			1
+#define IMX471_DGTL_GAIN_DEFAULT		256
+
+/* HFLIP and VFLIP control */
+#define IMX471_REG_ORIENTATION			CCI_REG8(0x0101)
+
+/* Test Pattern Control */
+#define IMX471_REG_TEST_PATTERN			CCI_REG8(0x0600)
+
+/* default link frequency and external clock */
+#define IMX471_LINK_FREQ_DEFAULT		200000000LL
+#define IMX471_EXT_CLK				19200000
+
+/* PLL */
+#define IMX471_REG_VTPXCK_DIV			CCI_REG8(0x0301)
+#define IMX471_REG_VTSYCK_DIV			CCI_REG8(0x0303)
+#define IMX471_REG_PREPLLCK_VT_DIV		CCI_REG8(0x0305)
+#define IMX471_REG_PLL_VT_MPY			CCI_REG16(0x0306)
+#define IMX471_REG_OPPXCK_DIV			CCI_REG8(0x0309)
+#define IMX471_REG_OPSYCK_DIV			CCI_REG8(0x030b)
+#define IMX471_REG_PLL_MULT_DRIV		CCI_REG8(0x0310)
+#define IMX471_PLL_SINGLE			0
+#define IMX471_PLL_DUAL				1
+
+/* IMX471 native and active pixel array size */
+#define IMX471_NATIVE_WIDTH			4672
+#define IMX471_NATIVE_HEIGHT			3512
+#define IMX471_PIXEL_ARRAY_LEFT			8
+#define IMX471_PIXEL_ARRAY_TOP			8
+#define IMX471_PIXEL_ARRAY_WIDTH		4656
+#define IMX471_PIXEL_ARRAY_HEIGHT		3496
+
+#define IMX471_REG_EXCK_FREQ			CCI_REG16(0x0136)
+
+#define IMX471_REG_CSI_DATA_FORMAT		CCI_REG16(0x0112)
+#define IMX471_CSI_DATA_FORMAT_RAW10		0x0a0a
+
+#define IMX471_REG_CSI_LANE_MODE		CCI_REG8(0x0114)
+#define IMX471_CSI_2_LANE_MODE			1
+#define IMX471_CSI_4_LANE_MODE			3
+
+#define IMX471_REG_X_ADD_STA			CCI_REG16(0x0344)
+#define IMX471_REG_Y_ADD_STA			CCI_REG16(0x0346)
+#define IMX471_REG_X_ADD_END			CCI_REG16(0x0348)
+#define IMX471_REG_Y_ADD_END			CCI_REG16(0x034a)
+#define IMX471_REG_X_OUTPUT_SIZE		CCI_REG16(0x034c)
+#define IMX471_REG_Y_OUTPUT_SIZE		CCI_REG16(0x034e)
+#define IMX471_REG_X_EVEN_INC			CCI_REG8(0x0381)
+#define IMX471_REG_X_ODD_INC			CCI_REG8(0x0383)
+#define IMX471_REG_Y_EVEN_INC			CCI_REG8(0x0385)
+#define IMX471_REG_Y_ODD_INC			CCI_REG8(0x0387)
+
+#define IMX471_REG_DIG_CROP_X_OFFSET		CCI_REG16(0x0408)
+#define IMX471_REG_DIG_CROP_Y_OFFSET		CCI_REG16(0x040a)
+#define IMX471_REG_DIG_CROP_WIDTH		CCI_REG16(0x040c)
+#define IMX471_REG_DIG_CROP_HEIGHT		CCI_REG16(0x040e)
+
+/* Binning mode */
+#define IMX471_REG_BINNING_MODE			CCI_REG8(0x0900)
+#define IMX471_BINNING_NONE			0
+#define IMX471_BINNING_ENABLE			1
+#define IMX471_REG_BINNING_TYPE			CCI_REG8(0x0901)
+#define IMX471_REG_BINNING_WEIGHTING		CCI_REG8(0x0902)
+
+#define to_imx471(_sd) container_of_const(_sd, struct imx471, sd)
+
+static const char * const imx471_supply_name[] = {
+	"vana",
+};
+
+struct imx471_mode {
+	u32 width;
+	u32 height;
+
+	/* V-timing */
+	u32 fll_def;
+	u32 fll_min;
+
+	/* H-timing */
+	u32 llp;
+
+	const struct cci_reg_sequence *default_mode_regs;
+	unsigned int default_mode_regs_length;
+};
+
+struct imx471 {
+	struct v4l2_subdev sd;
+	struct media_pad pad;
+
+	struct v4l2_ctrl_handler ctrl_handler;
+	struct v4l2_ctrl *vblank;
+	struct v4l2_ctrl *hblank;
+	struct v4l2_ctrl *vflip;
+	struct v4l2_ctrl *hflip;
+	struct v4l2_ctrl *exposure;
+
+	struct gpio_desc *reset_gpio;
+	struct regulator_bulk_data supplies[ARRAY_SIZE(imx471_supply_name)];
+	struct clk *img_clk;
+
+	struct device *dev;
+	struct regmap *regmap;
+};
+
+static const struct cci_reg_sequence imx471_global_regs[] = {
+	{ IMX471_REG_EXCK_FREQ, 0x1333 },
+	{ CCI_REG8(0x3c7e), 0x08 },
+	{ CCI_REG8(0x3c7f), 0x05 },
+	{ CCI_REG8(0x3e35), 0x00 },
+	{ CCI_REG8(0x3e36), 0x00 },
+	{ CCI_REG8(0x3e37), 0x00 },
+	{ CCI_REG8(0x3f7f), 0x01 },
+	{ CCI_REG8(0x4431), 0x04 },
+	{ CCI_REG8(0x531c), 0x01 },
+	{ CCI_REG8(0x531d), 0x02 },
+	{ CCI_REG8(0x531e), 0x04 },
+	{ CCI_REG8(0x5928), 0x00 },
+	{ CCI_REG8(0x5929), 0x2f },
+	{ CCI_REG8(0x592a), 0x00 },
+	{ CCI_REG8(0x592b), 0x85 },
+	{ CCI_REG8(0x592c), 0x00 },
+	{ CCI_REG8(0x592d), 0x32 },
+	{ CCI_REG8(0x592e), 0x00 },
+	{ CCI_REG8(0x592f), 0x88 },
+	{ CCI_REG8(0x5930), 0x00 },
+	{ CCI_REG8(0x5931), 0x3d },
+	{ CCI_REG8(0x5932), 0x00 },
+	{ CCI_REG8(0x5933), 0x93 },
+	{ CCI_REG8(0x5938), 0x00 },
+	{ CCI_REG8(0x5939), 0x24 },
+	{ CCI_REG8(0x593a), 0x00 },
+	{ CCI_REG8(0x593b), 0x7a },
+	{ CCI_REG8(0x593c), 0x00 },
+	{ CCI_REG8(0x593d), 0x24 },
+	{ CCI_REG8(0x593e), 0x00 },
+	{ CCI_REG8(0x593f), 0x7a },
+	{ CCI_REG8(0x5940), 0x00 },
+	{ CCI_REG8(0x5941), 0x2f },
+	{ CCI_REG8(0x5942), 0x00 },
+	{ CCI_REG8(0x5943), 0x85 },
+	{ CCI_REG8(0x5f0e), 0x6e },
+	{ CCI_REG8(0x5f11), 0xc6 },
+	{ CCI_REG8(0x5f17), 0x5e },
+	{ CCI_REG8(0x7990), 0x01 },
+	{ CCI_REG8(0x7993), 0x5d },
+	{ CCI_REG8(0x7994), 0x5d },
+	{ CCI_REG8(0x7995), 0xa1 },
+	{ CCI_REG8(0x799a), 0x01 },
+	{ CCI_REG8(0x799d), 0x00 },
+	{ CCI_REG8(0x8169), 0x01 },
+	{ CCI_REG8(0x8359), 0x01 },
+	{ CCI_REG8(0x9302), 0x1e },
+	{ CCI_REG8(0x9306), 0x1f },
+	{ CCI_REG8(0x930a), 0x26 },
+	{ CCI_REG8(0x930e), 0x23 },
+	{ CCI_REG8(0x9312), 0x23 },
+	{ CCI_REG8(0x9316), 0x2c },
+	{ CCI_REG8(0x9317), 0x19 },
+	{ CCI_REG8(0xb046), 0x01 },
+	{ CCI_REG8(0xb048), 0x01 },
+};
+
+static const struct cci_reg_sequence mode_1928x1088_regs[] = {
+	{ IMX471_REG_CSI_DATA_FORMAT, IMX471_CSI_DATA_FORMAT_RAW10 },
+	{ IMX471_REG_CSI_LANE_MODE, IMX471_CSI_4_LANE_MODE },
+	{ IMX471_REG_X_ADD_STA, 8 },
+	{ IMX471_REG_Y_ADD_STA, 408 },
+	{ IMX471_REG_X_ADD_END, 4647 },
+	{ IMX471_REG_Y_ADD_END, 3051 },
+	{ IMX471_REG_X_EVEN_INC, 1 },
+	{ IMX471_REG_X_ODD_INC, 1 },
+	{ IMX471_REG_Y_EVEN_INC, 1 },
+	{ IMX471_REG_Y_ODD_INC, 1 },
+	{ IMX471_REG_BINNING_MODE, IMX471_BINNING_ENABLE },
+	{ IMX471_REG_BINNING_TYPE, 0x22 },
+	{ IMX471_REG_BINNING_WEIGHTING, 0x08 },
+	{ IMX471_REG_DIG_CROP_X_OFFSET, 208 },
+	{ IMX471_REG_DIG_CROP_Y_OFFSET, 108 },
+	{ IMX471_REG_DIG_CROP_WIDTH, 1928 },
+	{ IMX471_REG_DIG_CROP_HEIGHT, 1088 },
+	{ IMX471_REG_X_OUTPUT_SIZE, 1928 },
+	{ IMX471_REG_Y_OUTPUT_SIZE, 1088 },
+	{ IMX471_REG_VTPXCK_DIV, 0x06 },
+	{ IMX471_REG_VTSYCK_DIV, 0x02 },
+	{ IMX471_REG_PREPLLCK_VT_DIV, 0x02 },
+	{ IMX471_REG_PLL_VT_MPY, 0x0079 },
+	{ IMX471_REG_OPSYCK_DIV, 0x01 },
+	{ CCI_REG8(0x030d), 0x02 },
+	{ CCI_REG8(0x030e), 0x00 },
+	{ CCI_REG8(0x030f), 0x53 },
+	{ IMX471_REG_PLL_MULT_DRIV, IMX471_PLL_DUAL },
+	{ CCI_REG8(0x3f4c), 0x81 },
+	{ CCI_REG8(0x3f4d), 0x81 },
+	{ CCI_REG8(0x3f78), 0x01 },
+	{ CCI_REG8(0x3f79), 0x31 },
+	{ CCI_REG8(0x3ffe), 0x00 },
+	{ CCI_REG8(0x3fff), 0x8a },
+	{ CCI_REG8(0x5f0a), 0xb6 },
+};
+
+static const char * const imx471_test_pattern_menu[] = {
+	"Disabled",
+	"Solid Colour",
+	"Eight Vertical Colour Bars",
+	"Colour Bars With Fade to Grey",
+	"Pseudorandom Sequence (PN9)",
+};
+
+static const s64 link_freq_menu_items[] = {
+	IMX471_LINK_FREQ_DEFAULT,
+};
+
+/*
+ * The Bayer formats for the flipping.
+ * - no flip
+ * - h flip
+ * - v flip
+ * - h and v flips
+ */
+static const u32 imx471_hv_flips_bayer_order[] = {
+	MEDIA_BUS_FMT_SRGGB10_1X10,
+	MEDIA_BUS_FMT_SGRBG10_1X10,
+	MEDIA_BUS_FMT_SGBRG10_1X10,
+	MEDIA_BUS_FMT_SBGGR10_1X10,
+};
+
+static const struct imx471_mode imx471_modes[] = {
+	{
+		.width = 1928,
+		.height = 1088,
+		.fll_def = 1308,
+		.fll_min = 1308,
+		.llp = 2328,
+		.default_mode_regs = mode_1928x1088_regs,
+		.default_mode_regs_length = ARRAY_SIZE(mode_1928x1088_regs),
+	},
+};
+
+static int imx471_get_regulators(struct device *dev, struct imx471 *sensor)
+{
+	for (unsigned int i = 0; i < ARRAY_SIZE(imx471_supply_name); i++)
+		sensor->supplies[i].supply = imx471_supply_name[i];
+
+	return devm_regulator_bulk_get(dev, ARRAY_SIZE(imx471_supply_name),
+				       sensor->supplies);
+}
+
+static int imx471_set_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct imx471 *sensor = container_of_const(ctrl->handler,
+						   struct imx471,
+						   ctrl_handler);
+	struct v4l2_subdev_state *state =
+			v4l2_subdev_get_locked_active_state(&sensor->sd);
+	const struct v4l2_mbus_framefmt *format =
+			v4l2_subdev_state_get_format(state, 0);
+	int ret;
+
+	if (ctrl->id == V4L2_CID_VBLANK) {
+		s64 exposure_max = format->height + ctrl->val -
+				   IMX471_EXPOSURE_MARGIN;
+		ret = __v4l2_ctrl_modify_range(sensor->exposure,
+					       sensor->exposure->minimum,
+					       exposure_max,
+					       sensor->exposure->step,
+					       exposure_max);
+		if (ret)
+			return ret;
+	}
+
+	if (!pm_runtime_get_if_in_use(sensor->dev))
+		return 0;
+
+	switch (ctrl->id) {
+	case V4L2_CID_ANALOGUE_GAIN:
+		ret = cci_write(sensor->regmap, IMX471_REG_ANALOG_GAIN,
+				ctrl->val, NULL);
+		break;
+	case V4L2_CID_DIGITAL_GAIN:
+		ret = cci_write(sensor->regmap, IMX471_REG_DIG_GAIN_GLOBAL,
+				ctrl->val, NULL);
+		break;
+	case V4L2_CID_EXPOSURE:
+		ret = cci_write(sensor->regmap, IMX471_REG_EXPOSURE,
+				ctrl->val, &ret);
+		break;
+	case V4L2_CID_VBLANK:
+		/* Update FLL that meets expected vertical blanking */
+		ret = cci_write(sensor->regmap, IMX471_REG_FLL,
+				format->height + ctrl->val, &ret);
+		break;
+	case V4L2_CID_TEST_PATTERN:
+		ret = cci_write(sensor->regmap, IMX471_REG_TEST_PATTERN,
+				ctrl->val, NULL);
+		break;
+	case V4L2_CID_HFLIP:
+	case V4L2_CID_VFLIP:
+		ret = cci_write(sensor->regmap, IMX471_REG_ORIENTATION,
+				sensor->hflip->val | sensor->vflip->val << 1,
+				NULL);
+		break;
+	default:
+		ret = -EINVAL;
+		dev_err(sensor->dev, "ctrl(id:0x%x,val:0x%x) is not handled\n",
+			ctrl->id, ctrl->val);
+		break;
+	}
+
+	pm_runtime_put(sensor->dev);
+
+	return ret;
+}
+
+static const struct v4l2_ctrl_ops imx471_ctrl_ops = {
+	.s_ctrl = imx471_set_ctrl,
+};
+
+static u32 imx471_get_format_code(struct imx471 *sensor)
+{
+	unsigned int i;
+
+	i = (sensor->vflip->val ? 2 : 0) | (sensor->hflip->val ? 1 : 0);
+
+	return imx471_hv_flips_bayer_order[i];
+}
+
+static int imx471_enum_mbus_code(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *sd_state,
+				 struct v4l2_subdev_mbus_code_enum *code)
+{
+	struct imx471 *sensor = to_imx471(sd);
+
+	if (code->index >= (ARRAY_SIZE(imx471_hv_flips_bayer_order) / 4))
+		return -EINVAL;
+
+	code->code = imx471_get_format_code(sensor);
+
+	return 0;
+}
+
+static int imx471_enum_frame_size(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_state *sd_state,
+				  struct v4l2_subdev_frame_size_enum *fse)
+{
+	if (fse->index >= ARRAY_SIZE(imx471_modes))
+		return -EINVAL;
+
+	fse->min_width = imx471_modes[fse->index].width;
+	fse->max_width = fse->min_width;
+	fse->min_height = imx471_modes[fse->index].height;
+	fse->max_height = fse->min_height;
+
+	return 0;
+}
+
+static void imx471_update_pad_format(struct imx471 *sensor,
+				     const struct imx471_mode *mode,
+				     struct v4l2_subdev_format *fmt)
+{
+	fmt->format.code = imx471_get_format_code(sensor);
+	fmt->format.width = mode->width;
+	fmt->format.height = mode->height;
+	fmt->format.field = V4L2_FIELD_NONE;
+}
+
+static int imx471_set_pad_format(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *sd_state,
+				 struct v4l2_subdev_format *fmt)
+{
+	struct imx471 *sensor = to_imx471(sd);
+	const struct imx471_mode *mode;
+	int h_blank, ret;
+
+	mode = v4l2_find_nearest_size(imx471_modes, ARRAY_SIZE(imx471_modes),
+				      width, height, fmt->format.width,
+				      fmt->format.height);
+
+	imx471_update_pad_format(sensor, mode, fmt);
+
+	*v4l2_subdev_state_get_format(sd_state, fmt->pad) = fmt->format;
+
+	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY)
+		return 0;
+
+	if (media_entity_is_streaming(&sensor->sd.entity))
+		return -EBUSY;
+
+	ret = __v4l2_ctrl_modify_range(sensor->vblank,
+				       mode->fll_min - mode->height,
+				       IMX471_FLL_MAX - mode->height,
+				       1,
+				       mode->fll_def - mode->height);
+	if (ret)
+		return ret;
+
+	h_blank = mode->llp - mode->width;
+	/*
+	 * Currently hblank is not changeable.
+	 * So FPS control is done only by vblank.
+	 */
+	return __v4l2_ctrl_modify_range(sensor->hblank, h_blank,
+					h_blank, 1, h_blank);
+}
+
+static int imx471_get_selection(struct v4l2_subdev *sd,
+				struct v4l2_subdev_state *sd_state,
+				struct v4l2_subdev_selection *sel)
+{
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP:
+		sel->r = *v4l2_subdev_state_get_crop(sd_state, sel->pad);
+		break;
+
+	case V4L2_SEL_TGT_NATIVE_SIZE:
+		sel->r.top = 0;
+		sel->r.left = 0;
+		sel->r.width = IMX471_NATIVE_WIDTH;
+		sel->r.height = IMX471_NATIVE_HEIGHT;
+		return 0;
+
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		sel->r.top = IMX471_PIXEL_ARRAY_TOP;
+		sel->r.left = IMX471_PIXEL_ARRAY_LEFT;
+		sel->r.width = IMX471_PIXEL_ARRAY_WIDTH;
+		sel->r.height = IMX471_PIXEL_ARRAY_HEIGHT;
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static int imx471_init_state(struct v4l2_subdev *sd,
+			     struct v4l2_subdev_state *sd_state)
+{
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_TRY,
+		.format = {
+			.code = MEDIA_BUS_FMT_SRGGB10_1X10,
+			.width = imx471_modes[0].width,
+			.height = imx471_modes[0].height,
+		},
+	};
+
+	return imx471_set_pad_format(sd, sd_state, &fmt);
+}
+
+static int imx471_identify_module(struct imx471 *sensor)
+{
+	int ret;
+	u64 val;
+
+	ret = cci_read(sensor->regmap, IMX471_REG_CHIP_ID, &val, NULL);
+	if (ret)
+		return dev_err_probe(sensor->dev, ret,
+				     "failed to read chip id\n");
+
+	if (val != IMX471_CHIP_ID)
+		return dev_err_probe(sensor->dev, -EIO,
+				     "chip id mismatch: %x!=%llx\n",
+				     IMX471_CHIP_ID, val);
+
+	return 0;
+}
+
+static int imx471_power_off(struct device *dev)
+{
+	struct v4l2_subdev *sd = dev_get_drvdata(dev);
+	struct imx471 *sensor = to_imx471(sd);
+
+	clk_disable_unprepare(sensor->img_clk);
+	gpiod_set_value_cansleep(sensor->reset_gpio, 1);
+
+	regulator_bulk_disable(ARRAY_SIZE(imx471_supply_name),
+			       sensor->supplies);
+
+	return 0;
+}
+
+static int imx471_power_on(struct device *dev)
+{
+	struct v4l2_subdev *sd = dev_get_drvdata(dev);
+	struct imx471 *sensor = to_imx471(sd);
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(imx471_supply_name),
+				    sensor->supplies);
+	if (ret < 0) {
+		dev_err(dev, "failed to enable regulators: %d\n", ret);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(sensor->img_clk);
+	if (ret < 0) {
+		regulator_bulk_disable(ARRAY_SIZE(imx471_supply_name),
+				       sensor->supplies);
+		dev_err(dev, "failed to enable imaging clock: %d\n", ret);
+		return ret;
+	}
+
+	gpiod_set_value_cansleep(sensor->reset_gpio, 0);
+
+	usleep_range(10000, 15000);
+
+	return 0;
+}
+
+static int imx471_enable_stream(struct v4l2_subdev *sd,
+				struct v4l2_subdev_state *state,
+				u32 pad, u64 streams_mask)
+{
+	struct imx471 *sensor = to_imx471(sd);
+	const struct imx471_mode *mode;
+	struct v4l2_mbus_framefmt *fmt;
+	int ret;
+
+	ret = pm_runtime_resume_and_get(sensor->dev);
+	if (ret)
+		return ret;
+
+	ret = imx471_identify_module(sensor);
+	if (ret)
+		goto error_powerdown;
+
+	ret = cci_multi_reg_write(sensor->regmap, imx471_global_regs,
+				  ARRAY_SIZE(imx471_global_regs), NULL);
+	if (ret) {
+		dev_err(sensor->dev, "failed to set global settings: %d\n",
+			ret);
+		goto error_powerdown;
+	}
+
+	fmt = v4l2_subdev_state_get_format(state, 0);
+	mode = v4l2_find_nearest_size(imx471_modes, ARRAY_SIZE(imx471_modes),
+				      width, height, fmt->width, fmt->height);
+
+	ret = cci_multi_reg_write(sensor->regmap, mode->default_mode_regs,
+				  mode->default_mode_regs_length, NULL);
+	if (ret) {
+		dev_err(sensor->dev, "failed to set mode: %d\n", ret);
+		goto error_powerdown;
+	}
+
+	ret = cci_write(sensor->regmap, IMX471_REG_DPGA_USE_GLOBAL_GAIN, 1,
+			NULL);
+	if (ret)
+		goto error_powerdown;
+
+	ret = __v4l2_ctrl_handler_setup(&sensor->ctrl_handler);
+	if (ret)
+		goto error_powerdown;
+
+	ret = cci_write(sensor->regmap, IMX471_REG_MODE_SELECT,
+			IMX471_MODE_STREAMING, NULL);
+	if (ret)
+		goto error_powerdown;
+
+	__v4l2_ctrl_grab(sensor->vflip, true);
+	__v4l2_ctrl_grab(sensor->hflip, true);
+
+	return ret;
+
+error_powerdown:
+	pm_runtime_put(sensor->dev);
+
+	return ret;
+}
+
+static int imx471_disable_stream(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *state,
+				 u32 pad, u64 streams_mask)
+{
+	struct imx471 *sensor = to_imx471(sd);
+	int ret;
+
+	ret = cci_write(sensor->regmap, IMX471_REG_MODE_SELECT,
+			IMX471_MODE_STANDBY, NULL);
+	pm_runtime_put(sensor->dev);
+
+	if (ret)
+		dev_err(sensor->dev,
+			"failed to disable stream with return value: %d\n",
+			ret);
+
+	__v4l2_ctrl_grab(sensor->vflip, false);
+	__v4l2_ctrl_grab(sensor->hflip, false);
+
+	return 0;
+}
+
+static const struct v4l2_subdev_video_ops imx471_video_ops = {
+	.s_stream = v4l2_subdev_s_stream_helper,
+};
+
+static const struct v4l2_subdev_pad_ops imx471_pad_ops = {
+	.enum_mbus_code = imx471_enum_mbus_code,
+	.get_fmt = v4l2_subdev_get_fmt,
+	.set_fmt = imx471_set_pad_format,
+	.get_selection = imx471_get_selection,
+	.enum_frame_size = imx471_enum_frame_size,
+	.enable_streams = imx471_enable_stream,
+	.disable_streams = imx471_disable_stream,
+};
+
+static const struct v4l2_subdev_ops imx471_subdev_ops = {
+	.video = &imx471_video_ops,
+	.pad = &imx471_pad_ops,
+};
+
+static const struct v4l2_subdev_internal_ops imx471_internal_ops = {
+	.init_state = imx471_init_state,
+};
+
+static int imx471_init_controls(struct imx471 *sensor)
+{
+	const struct imx471_mode *mode = &imx471_modes[0];
+	struct v4l2_fwnode_device_properties props;
+	struct v4l2_ctrl_handler *ctrl_hdlr;
+	struct v4l2_ctrl *link_freq;
+	s64 exposure_max, hblank;
+	u64 pixel_rate;
+	int ret;
+
+	ret = v4l2_fwnode_device_parse(sensor->dev, &props);
+	if (ret) {
+		dev_err(sensor->dev, "failed to parse fwnode: %d\n", ret);
+		return ret;
+	}
+
+	ctrl_hdlr = &sensor->ctrl_handler;
+	v4l2_ctrl_handler_init(ctrl_hdlr, 12);
+
+	v4l2_ctrl_new_fwnode_properties(ctrl_hdlr, &imx471_ctrl_ops, &props);
+
+	link_freq = v4l2_ctrl_new_int_menu(ctrl_hdlr,
+					   &imx471_ctrl_ops,
+					   V4L2_CID_LINK_FREQ,
+					   ARRAY_SIZE(link_freq_menu_items) - 1,
+					   0,
+					   link_freq_menu_items);
+
+	/* pixel_rate = link_freq * 2 * nr_of_lanes / bits_per_sample */
+	pixel_rate = div_u64(IMX471_LINK_FREQ_DEFAULT * 2 * 4, 10);
+
+	v4l2_ctrl_new_std(ctrl_hdlr, &imx471_ctrl_ops,
+			  V4L2_CID_PIXEL_RATE, pixel_rate,
+			  pixel_rate, 1, pixel_rate);
+
+	sensor->vblank = v4l2_ctrl_new_std(ctrl_hdlr,
+					   &imx471_ctrl_ops,
+					   V4L2_CID_VBLANK,
+					   mode->fll_min - mode->height,
+					   IMX471_FLL_MAX - mode->height,
+					   1,
+					   mode->fll_def - mode->height);
+
+	hblank = mode->llp - mode->width;
+	sensor->hblank = v4l2_ctrl_new_std(ctrl_hdlr, &imx471_ctrl_ops,
+					   V4L2_CID_HBLANK, hblank, hblank,
+					   1, hblank);
+
+	/* fll >= exposure time + adjust parameter (default value is 18) */
+	exposure_max = mode->fll_def - IMX471_EXPOSURE_MARGIN;
+	sensor->exposure = v4l2_ctrl_new_std(ctrl_hdlr, &imx471_ctrl_ops,
+					     V4L2_CID_EXPOSURE,
+					     IMX471_EXPOSURE_MIN, exposure_max,
+					     IMX471_EXPOSURE_STEP,
+					     IMX471_EXPOSURE_DEFAULT);
+
+	v4l2_ctrl_new_std(ctrl_hdlr, &imx471_ctrl_ops, V4L2_CID_ANALOGUE_GAIN,
+			  IMX471_ANA_GAIN_MIN, IMX471_ANA_GAIN_MAX,
+			  IMX471_ANA_GAIN_STEP, IMX471_ANA_GAIN_DEFAULT);
+
+	v4l2_ctrl_new_std(ctrl_hdlr, &imx471_ctrl_ops, V4L2_CID_DIGITAL_GAIN,
+			  IMX471_DGTL_GAIN_MIN, IMX471_DGTL_GAIN_MAX,
+			  IMX471_DGTL_GAIN_STEP, IMX471_DGTL_GAIN_DEFAULT);
+
+	v4l2_ctrl_new_std_menu_items(ctrl_hdlr, &imx471_ctrl_ops,
+				     V4L2_CID_TEST_PATTERN,
+				     ARRAY_SIZE(imx471_test_pattern_menu) - 1,
+				     0, 0, imx471_test_pattern_menu);
+
+	sensor->hflip = v4l2_ctrl_new_std(ctrl_hdlr, &imx471_ctrl_ops,
+					  V4L2_CID_HFLIP, 0, 1, 1, 0);
+
+	sensor->vflip = v4l2_ctrl_new_std(ctrl_hdlr, &imx471_ctrl_ops,
+					  V4L2_CID_VFLIP, 0, 1, 1, 0);
+
+	if (ctrl_hdlr->error) {
+		dev_err(sensor->dev, "%s control init failed: %d\n",
+			__func__, ctrl_hdlr->error);
+		goto error;
+	}
+
+	link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+	sensor->hblank->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+	sensor->hflip->flags |= V4L2_CTRL_FLAG_MODIFY_LAYOUT;
+	sensor->vflip->flags |= V4L2_CTRL_FLAG_MODIFY_LAYOUT;
+
+	sensor->sd.ctrl_handler = ctrl_hdlr;
+
+	return 0;
+
+error:
+	v4l2_ctrl_handler_free(ctrl_hdlr);
+
+	return ctrl_hdlr->error;
+}
+
+static int imx471_check_hwcfg(struct imx471 *sensor)
+{
+	struct v4l2_fwnode_endpoint bus_cfg = {
+		.bus_type = V4L2_MBUS_CSI2_DPHY,
+	};
+	struct fwnode_handle *ep, *fwnode = dev_fwnode(sensor->dev);
+	unsigned long link_freq_bitmap;
+	struct clk *clk;
+	int ret;
+
+	clk = devm_v4l2_sensor_clk_get(sensor->dev, NULL);
+	if (IS_ERR(clk))
+		return dev_err_probe(sensor->dev, PTR_ERR(clk),
+				     "can't get clock frequency\n");
+
+	if (clk_get_rate(clk) != IMX471_EXT_CLK)
+		return dev_err_probe(sensor->dev, -EINVAL,
+				     "external clock %lu is not supported\n",
+				     clk_get_rate(clk));
+
+	ep = fwnode_graph_get_endpoint_by_id(fwnode, 0, 0, 0);
+	ret = v4l2_fwnode_endpoint_alloc_parse(ep, &bus_cfg);
+	fwnode_handle_put(ep);
+	if (ret)
+		return dev_err_probe(sensor->dev, ret,
+				     "parsing endpoint failed\n");
+
+	if (bus_cfg.bus.mipi_csi2.num_data_lanes != 4) {
+		ret = dev_err_probe(sensor->dev, -EINVAL,
+				    "number of CSI2 data lanes %u is not supported\n",
+				    bus_cfg.bus.mipi_csi2.num_data_lanes);
+		goto done_endpoint_free;
+	}
+
+	ret = v4l2_link_freq_to_bitmap(sensor->dev, bus_cfg.link_frequencies,
+				       bus_cfg.nr_of_link_frequencies,
+				       link_freq_menu_items,
+				       ARRAY_SIZE(link_freq_menu_items),
+				       &link_freq_bitmap);
+
+done_endpoint_free:
+	v4l2_fwnode_endpoint_free(&bus_cfg);
+
+	return ret;
+}
+
+static int imx471_probe(struct i2c_client *client)
+{
+	struct imx471 *sensor;
+	int ret;
+
+	sensor = devm_kzalloc(&client->dev, sizeof(*sensor), GFP_KERNEL);
+	if (!sensor)
+		return dev_err_probe(&client->dev, -ENOMEM,
+				     "failed to allocate memory\n");
+
+	sensor->dev = &client->dev;
+
+	ret = imx471_check_hwcfg(sensor);
+	if (ret)
+		return dev_err_probe(sensor->dev, ret,
+				     "failed to check hwcfg: %d\n", ret);
+
+	ret = imx471_get_regulators(sensor->dev, sensor);
+	if (ret)
+		return dev_err_probe(sensor->dev, ret,
+				     "failed to get regulators\n");
+
+	sensor->reset_gpio = devm_gpiod_get_optional(sensor->dev, "reset",
+						     GPIOD_OUT_HIGH);
+	if (IS_ERR(sensor->reset_gpio))
+		return dev_err_probe(sensor->dev, PTR_ERR(sensor->reset_gpio),
+				     "failed to get reset gpio\n");
+
+	sensor->img_clk = devm_v4l2_sensor_clk_get(sensor->dev, NULL);
+	if (IS_ERR(sensor->img_clk))
+		return dev_err_probe(sensor->dev, PTR_ERR(sensor->img_clk),
+				     "failed to get imaging clock\n");
+
+	v4l2_i2c_subdev_init(&sensor->sd, client, &imx471_subdev_ops);
+
+	sensor->regmap = devm_cci_regmap_init_i2c(client, 16);
+	if (IS_ERR(sensor->regmap))
+		return dev_err_probe(sensor->dev, PTR_ERR(sensor->regmap),
+				     "failed to initialize CCI\n");
+
+	ret = imx471_power_on(sensor->dev);
+	if (ret)
+		return dev_err_probe(sensor->dev, ret,
+				     "failed to power on\n");
+
+	ret = imx471_identify_module(sensor);
+	if (ret) {
+		dev_err_probe(sensor->dev, ret, "failed to find sensor: %d\n",
+			      ret);
+		goto error_power_off;
+	}
+
+	ret = imx471_init_controls(sensor);
+	if (ret) {
+		dev_err_probe(sensor->dev, ret, "failed to init controls: %d\n",
+			      ret);
+		goto error_power_off;
+	}
+
+	sensor->sd.internal_ops = &imx471_internal_ops;
+	sensor->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	sensor->pad.flags = MEDIA_PAD_FL_SOURCE;
+	sensor->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
+
+	ret = media_entity_pads_init(&sensor->sd.entity, 1, &sensor->pad);
+	if (ret) {
+		dev_err_probe(sensor->dev, ret,
+			      "failed to init entity pads: %d\n", ret);
+		goto error_v4l2_ctrl_handler_free;
+	}
+
+	sensor->sd.state_lock = sensor->ctrl_handler.lock;
+	ret = v4l2_subdev_init_finalize(&sensor->sd);
+	if (ret < 0) {
+		dev_err_probe(sensor->dev, ret, "failed to init subdev: %d\n",
+			      ret);
+		goto error_media_entity_pm;
+	}
+
+	pm_runtime_set_active(sensor->dev);
+	pm_runtime_enable(sensor->dev);
+
+	ret = v4l2_async_register_subdev_sensor(&sensor->sd);
+	if (ret < 0)
+		goto error_v4l2_subdev_cleanup;
+
+	pm_runtime_idle(sensor->dev);
+
+	return 0;
+
+error_v4l2_subdev_cleanup:
+	pm_runtime_disable(sensor->dev);
+	pm_runtime_set_suspended(sensor->dev);
+	v4l2_subdev_cleanup(&sensor->sd);
+
+error_media_entity_pm:
+	media_entity_cleanup(&sensor->sd.entity);
+
+error_v4l2_ctrl_handler_free:
+	v4l2_ctrl_handler_free(sensor->sd.ctrl_handler);
+
+error_power_off:
+	imx471_power_off(sensor->dev);
+
+	return ret;
+}
+
+static void imx471_remove(struct i2c_client *client)
+{
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+
+	v4l2_async_unregister_subdev(sd);
+	v4l2_subdev_cleanup(sd);
+	media_entity_cleanup(&sd->entity);
+	v4l2_ctrl_handler_free(sd->ctrl_handler);
+
+	pm_runtime_disable(&client->dev);
+
+	if (!pm_runtime_status_suspended(&client->dev)) {
+		imx471_power_off(&client->dev);
+		pm_runtime_set_suspended(&client->dev);
+	}
+}
+
+static DEFINE_RUNTIME_DEV_PM_OPS(imx471_pm_ops, imx471_power_off,
+				 imx471_power_on, NULL);
+
+static const struct acpi_device_id imx471_acpi_ids[] __maybe_unused = {
+	{ "SONY471A" },
+	{ "TBE20A0" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(acpi, imx471_acpi_ids);
+
+static struct i2c_driver imx471_i2c_driver = {
+	.driver = {
+		.name = "imx471",
+		.acpi_match_table = ACPI_PTR(imx471_acpi_ids),
+		.pm = pm_sleep_ptr(&imx471_pm_ops),
+	},
+	.probe = imx471_probe,
+	.remove = imx471_remove,
+};
+module_i2c_driver(imx471_i2c_driver);
+
+MODULE_AUTHOR("Jimmy Su <jimmy.su@intel.com>");
+MODULE_AUTHOR("Serin Yeh <serin.yeh@intel.com>");
+MODULE_AUTHOR("Kate Hsuan <hpa@redhat.com>");
+MODULE_DESCRIPTION("Sony imx471 sensor driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/i2c/imx678.c b/drivers/media/i2c/imx678.c
new file mode 100644
index 0000000000000..0efbf43d2fe6c
--- /dev/null
+++ b/drivers/media/i2c/imx678.c
@@ -0,0 +1,1447 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * V4L2 driver for Sony IMX678
+ *
+ * Diagonal 8.86 mm (Type 1/1.8) CMOS image sensor with 8.40 M effective pixels.
+ *
+ * Copyright (C) 2026 Ideas On Board Oy.
+ *
+ * Based on Sony IMX678 driver prepared by Will Whang & Soho Enterprise Ltd.
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/media-bus-format.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/pm_runtime.h>
+#include <linux/regulator/consumer.h>
+#include <media/v4l2-cci.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-mediabus.h>
+#include <media/v4l2-rect.h>
+#include <media/v4l2-subdev.h>
+
+/* Standby or streaming mode */
+#define IMX678_REG_MODE_SELECT          CCI_REG8(0x3000)
+#define IMX678_MODE_STANDBY             0x01
+#define IMX678_MODE_STREAMING           0x00
+#define IMX678_STREAM_DELAY_US          25000
+#define IMX678_STREAM_DELAY_RANGE_US    1000
+
+/* XVS/XHS sync control */
+#define IMX678_REG_XMSTA                CCI_REG8(0x3002)
+#define IMX678_REG_XXS_DRV              CCI_REG8(0x30a6)
+#define IMX678_REG_XXS_OUTSEL           CCI_REG8(0x30a4)
+
+/* Clk selection */
+#define IMX678_REG_INCK_SEL             CCI_REG8(0x3014)
+
+/* Link Speed */
+#define IMX678_REG_DATARATE_SEL         CCI_REG8(0x3015)
+
+/* Lane Count */
+#define IMX678_REG_LANEMODE             CCI_REG8(0x3040)
+
+/*
+ * The internal readout clock runs at 74.25 MHz. In one cycle the AD reads 8
+ * pixels, thus giving us a rate of 74.25 * 8 = 594 MPix/s
+ */
+#define IMX678_PIXEL_RATE		594000000
+#define IMX678_PIX_PER_CLK		8
+
+/* VMAX - Frame Length in Lines */
+#define IMX678_REG_VMAX                 CCI_REG24_LE(0x3028)
+#define IMX678_VMAX_MAX                 0xfffff
+#define IMX678_VMAX_DEFAULT             2250
+
+/* HMAX - Line Length in Cycles (8 Pixels) */
+#define IMX678_REG_HMAX                 CCI_REG16_LE(0x302c)
+#define IMX678_HMAX_MAX                 0xffff
+
+/* SHR internal */
+#define IMX678_REG_SHR                  CCI_REG24_LE(0x3050)
+#define IMX678_SHR_MIN                  8
+
+/* Exposure control */
+#define IMX678_EXPOSURE_MIN             2
+#define IMX678_EXPOSURE_STEP            1
+#define IMX678_EXPOSURE_DEFAULT         1000
+
+/*
+ * Analogue gain control
+ * Range is from 0 to 100 (0dB - 30dB) with 0.3dB step size
+ * Values from 101 to 240 are valid but correspond to additional digital gain
+ * (0.3dB - 42dB) so don't expose it to userspace
+ */
+#define IMX678_REG_GAIN			CCI_REG16_LE(0x3070)
+#define IMX678_ANA_GAIN_MIN_NORMAL      0
+#define IMX678_ANA_GAIN_MAX_NORMAL      100
+#define IMX678_ANA_GAIN_STEP            1
+#define IMX678_ANA_GAIN_DEFAULT         0
+
+/* Crop */
+#define IMX678_REG_WINMODE		CCI_REG8(0x3018)
+#define IMX678_REG_PIX_HST		CCI_REG16_LE(0x303c)
+#define IMX678_REG_PIX_HWIDTH		CCI_REG16_LE(0x303e)
+#define IMX678_REG_PIX_VST		CCI_REG16_LE(0x3044)
+#define IMX678_REG_PIX_VWIDTH		CCI_REG16_LE(0x3046)
+
+/* Flip */
+#define IMX678_REG_WINMODEH             CCI_REG8(0x3020)
+#define IMX678_REG_WINMODEV             CCI_REG8(0x3021)
+
+/* Sensor Identification */
+#define IMX678_REG_MONOCHROME		CCI_REG8(0x4d18)
+#define IMX678_TYPE			BIT(0)
+#define IMX678_REG_MODULE_ID		CCI_REG16_LE(0x4d1c)
+#define IMX678_ID			0x02a6
+#define IMX678_MODULE_ID_DELAY		80000
+
+/* Common configuration registers */
+#define IMX678_REG_WDMODE               CCI_REG8(0x301a)
+#define IMX678_REG_ADDMODE              CCI_REG8(0x301b)
+#define IMX678_REG_THIN_V_EN            CCI_REG8(0x301c)
+#define IMX678_REG_VCMODE               CCI_REG8(0x301e)
+#define IMX678_REG_ADBIT                CCI_REG8(0x3022)
+#define IMX678_REG_MDBIT                CCI_REG8(0x3023)
+#define IMX678_REG_GAIN_PGC_FIDMD       CCI_REG8(0x3400)
+
+/* Test pattern generator */
+#define IMX678_REG_TPG_EN_DUOUT		CCI_REG8(0x30e0)
+#define IMX678_REG_TPG_PATSEL_DUOUT	CCI_REG8(0x30e2)
+#define IMX678_TPG_ALL_000		0
+#define IMX678_TPG_ALL_FFF		1
+#define IMX678_TPG_ALL_555		2
+#define IMX678_TPG_ALL_AAA		3
+#define IMX678_TPG_TOG_555_AAA		4
+#define IMX678_TPG_TOG_AAA_555		5
+#define IMX678_TPG_TOG_000_555		6
+#define IMX678_TPG_TOG_555_000		7
+#define IMX678_TPG_TOG_000_FFF		8
+#define IMX678_TPG_TOG_FFF_000		9
+#define IMX678_TPG_H_COLOR_BARS		10
+#define IMX678_TPG_V_COLOR_BARS		11
+#define IMX678_REG_TPG_COLORWIDTH	CCI_REG8(0x30e4)
+#define IMX678_TPG_COLORWIDTH_80PIX	0
+#define IMX678_TPG_COLORWIDTH_160PIX	1
+#define IMX678_TPG_COLORWIDTH_320PIX	2
+#define IMX678_TPG_COLORWIDTH_640PIX	3
+
+#define IMX678_REG_INTERFACE_SEL	CCI_REG8(0x4e3c)
+#define IMX678_INTERFACE_2L_4L		0x07
+#define IMX678_INTERFACE_8L_2x4L	0x7f
+
+/* Minimum output resolution */
+#define IMX678_PIXEL_ARRAY_MIN_WIDTH	1040
+#define IMX678_PIXEL_ARRAY_MIN_HEIGHT	956
+
+/* Sensor windowing register alignment */
+#define IMX678_CROP_HWIDTH_ALIGN	16
+#define IMX678_CROP_VWIDTH_ALIGN	4
+#define IMX678_CROP_HST_ALIGN		4
+#define IMX678_CROP_VST_ALIGN		4
+
+/* Subdev pads */
+#define IMX678_SOURCE_PAD		0
+
+/* IMX678 native and active pixel array size. */
+static const struct v4l2_rect imx678_native_area = {
+	.top = 0,
+	.left = 0,
+	.width = 3857,
+	.height = 2201,
+};
+
+static const struct v4l2_rect imx678_active_area = {
+	.top = 20,
+	.left = 0,
+	.width = 3856,
+	.height = 2180,
+};
+
+enum imx678_type {
+	IMX678_COLOR = 0,
+	IMX678_MONOCHROME = 1,
+};
+
+struct imx678_model_info {
+	enum imx678_type type;
+	const u32 *codes;
+	unsigned int num_codes;
+};
+
+enum imx678_lanemode {
+	IMX678_LANEMODE_2L = 1,
+	IMX678_LANEMODE_4L = 3,
+};
+
+/* Link frequency setup (DDR: lane rate = 2 x link freq) */
+enum {
+	IMX678_LINK_FREQ_297MHZ,
+	IMX678_LINK_FREQ_360MHZ,
+	IMX678_LINK_FREQ_445MHZ,
+	IMX678_LINK_FREQ_594MHZ,
+	IMX678_LINK_FREQ_720MHZ,
+	IMX678_LINK_FREQ_891MHZ,
+	IMX678_LINK_FREQ_1039MHZ,
+	IMX678_LINK_FREQ_1188MHZ,
+};
+
+static const u8 link_freqs_reg_value[] = {
+	[IMX678_LINK_FREQ_297MHZ]  = 0x07,
+	[IMX678_LINK_FREQ_360MHZ]  = 0x06,
+	[IMX678_LINK_FREQ_445MHZ]  = 0x05,
+	[IMX678_LINK_FREQ_594MHZ]  = 0x04,
+	[IMX678_LINK_FREQ_720MHZ]  = 0x03,
+	[IMX678_LINK_FREQ_891MHZ]  = 0x02,
+	[IMX678_LINK_FREQ_1039MHZ] = 0x01,
+	[IMX678_LINK_FREQ_1188MHZ] = 0x00,
+};
+
+static const u64 link_freqs[] = {
+	[IMX678_LINK_FREQ_297MHZ]  = 297000000,
+	[IMX678_LINK_FREQ_360MHZ]  = 360000000,
+	[IMX678_LINK_FREQ_445MHZ]  = 445500000,
+	[IMX678_LINK_FREQ_594MHZ]  = 594000000,
+	[IMX678_LINK_FREQ_720MHZ]  = 720000000,
+	[IMX678_LINK_FREQ_891MHZ]  = 891000000,
+	[IMX678_LINK_FREQ_1039MHZ] = 1039500000,
+	[IMX678_LINK_FREQ_1188MHZ] = 1188000000,
+};
+
+static const u16 min_hmax_4lane[] = {
+	[IMX678_LINK_FREQ_297MHZ] = 1584,
+	[IMX678_LINK_FREQ_360MHZ] = 1320,
+	[IMX678_LINK_FREQ_445MHZ] = 1100,
+	[IMX678_LINK_FREQ_594MHZ] =  792,
+	[IMX678_LINK_FREQ_720MHZ] =  660,
+	[IMX678_LINK_FREQ_891MHZ] =  550,
+	[IMX678_LINK_FREQ_1039MHZ] = 550,
+	[IMX678_LINK_FREQ_1188MHZ] = 550,
+};
+
+struct imx678_inck_cfg {
+	u32 xclk_hz;   /* platform clock rate  */
+	u8  inck_sel;  /* value for reg        */
+};
+
+static const struct imx678_inck_cfg imx678_inck_table[] = {
+	{ 74250000, 0x00 },
+	{ 37125000, 0x01 },
+	{ 72000000, 0x02 },
+	{ 27000000, 0x03 },
+	{ 24000000, 0x04 },
+	{ 36000000, 0x05 },
+	{ 18000000, 0x06 },
+	{ 13500000, 0x07 },
+};
+
+static const char * const imx678_tpg_menu[] = {
+	"Disabled",
+	"All 000h",
+	"All FFFh",
+	"All 555h",
+	"All AAAh",
+	"Toggle 555/AAAh",
+	"Toggle AAA/555h",
+	"Toggle 000/555h",
+	"Toggle 555/000h",
+	"Toggle 000/FFFh",
+	"Toggle FFF/000h",
+	"Horizontal color bars",
+	"Vertical color bars",
+};
+
+static const int imx678_tpg_val[] = {
+	IMX678_TPG_ALL_000,
+	IMX678_TPG_ALL_000,
+	IMX678_TPG_ALL_FFF,
+	IMX678_TPG_ALL_555,
+	IMX678_TPG_ALL_AAA,
+	IMX678_TPG_TOG_555_AAA,
+	IMX678_TPG_TOG_AAA_555,
+	IMX678_TPG_TOG_000_555,
+	IMX678_TPG_TOG_555_000,
+	IMX678_TPG_TOG_000_FFF,
+	IMX678_TPG_TOG_FFF_000,
+	IMX678_TPG_H_COLOR_BARS,
+	IMX678_TPG_V_COLOR_BARS,
+};
+
+/* Common configuration */
+static const struct cci_reg_sequence common_regs[] = {
+	{ IMX678_REG_THIN_V_EN, 0x00 },
+	{ IMX678_REG_VCMODE, 0x01 },
+	{ CCI_REG8(0x306b), 0x00 },
+	{ IMX678_REG_GAIN_PGC_FIDMD, 0x01 },
+	{ CCI_REG8(0x3460), 0x22 },
+	{ CCI_REG8(0x355a), 0x64 },
+	{ CCI_REG8(0x3a02), 0x7a },
+	{ CCI_REG8(0x3a10), 0xec },
+	{ CCI_REG8(0x3a12), 0x71 },
+	{ CCI_REG8(0x3a14), 0xde },
+	{ CCI_REG8(0x3a20), 0x2b },
+	{ CCI_REG8(0x3a24), 0x22 },
+	{ CCI_REG8(0x3a25), 0x25 },
+	{ CCI_REG8(0x3a26), 0x2a },
+	{ CCI_REG8(0x3a27), 0x2c },
+	{ CCI_REG8(0x3a28), 0x39 },
+	{ CCI_REG8(0x3a29), 0x38 },
+	{ CCI_REG8(0x3a30), 0x04 },
+	{ CCI_REG8(0x3a31), 0x04 },
+	{ CCI_REG8(0x3a32), 0x03 },
+	{ CCI_REG8(0x3a33), 0x03 },
+	{ CCI_REG8(0x3a34), 0x09 },
+	{ CCI_REG8(0x3a35), 0x06 },
+	{ CCI_REG8(0x3a38), 0xcd },
+	{ CCI_REG8(0x3a3a), 0x4c },
+	{ CCI_REG8(0x3a3c), 0xb9 },
+	{ CCI_REG8(0x3a3e), 0x30 },
+	{ CCI_REG8(0x3a40), 0x2c },
+	{ CCI_REG8(0x3a42), 0x39 },
+	{ CCI_REG8(0x3a4e), 0x00 },
+	{ CCI_REG8(0x3a52), 0x00 },
+	{ CCI_REG8(0x3a56), 0x00 },
+	{ CCI_REG8(0x3a5a), 0x00 },
+	{ CCI_REG8(0x3a5e), 0x00 },
+	{ CCI_REG8(0x3a62), 0x00 },
+	{ CCI_REG8(0x3a64), 0x00 },
+	{ CCI_REG8(0x3a6e), 0xa0 },
+	{ CCI_REG8(0x3a70), 0x50 },
+	{ CCI_REG8(0x3a8c), 0x04 },
+	{ CCI_REG8(0x3a8d), 0x03 },
+	{ CCI_REG8(0x3a8e), 0x09 },
+	{ CCI_REG8(0x3a90), 0x38 },
+	{ CCI_REG8(0x3a91), 0x42 },
+	{ CCI_REG8(0x3a92), 0x3c },
+	{ CCI_REG8(0x3b0e), 0xf3 },
+	{ CCI_REG8(0x3b12), 0xe5 },
+	{ CCI_REG8(0x3b27), 0xc0 },
+	{ CCI_REG8(0x3b2e), 0xef },
+	{ CCI_REG8(0x3b30), 0x6a },
+	{ CCI_REG8(0x3b32), 0xf6 },
+	{ CCI_REG8(0x3b36), 0xe1 },
+	{ CCI_REG8(0x3b3a), 0xe8 },
+	{ CCI_REG8(0x3b5a), 0x17 },
+	{ CCI_REG8(0x3b5e), 0xef },
+	{ CCI_REG8(0x3b60), 0x6a },
+	{ CCI_REG8(0x3b62), 0xf6 },
+	{ CCI_REG8(0x3b66), 0xe1 },
+	{ CCI_REG8(0x3b6a), 0xe8 },
+	{ CCI_REG8(0x3b88), 0xec },
+	{ CCI_REG8(0x3b8a), 0xed },
+	{ CCI_REG8(0x3b94), 0x71 },
+	{ CCI_REG8(0x3b96), 0x72 },
+	{ CCI_REG8(0x3b98), 0xde },
+	{ CCI_REG8(0x3b9a), 0xdf },
+	{ CCI_REG8(0x3c0f), 0x06 },
+	{ CCI_REG8(0x3c10), 0x06 },
+	{ CCI_REG8(0x3c11), 0x06 },
+	{ CCI_REG8(0x3c12), 0x06 },
+	{ CCI_REG8(0x3c13), 0x06 },
+	{ CCI_REG8(0x3c18), 0x20 },
+	{ CCI_REG8(0x3c37), 0x10 },
+	{ CCI_REG8(0x3c3a), 0x7a },
+	{ CCI_REG8(0x3c40), 0xf4 },
+	{ CCI_REG8(0x3c48), 0xe6 },
+	{ CCI_REG8(0x3c54), 0xce },
+	{ CCI_REG8(0x3c56), 0xd0 },
+	{ CCI_REG8(0x3c6c), 0x53 },
+	{ CCI_REG8(0x3c6e), 0x55 },
+	{ CCI_REG8(0x3c70), 0xc0 },
+	{ CCI_REG8(0x3c72), 0xc2 },
+	{ CCI_REG8(0x3c7e), 0xce },
+	{ CCI_REG8(0x3c8c), 0xcf },
+	{ CCI_REG8(0x3c8e), 0xeb },
+	{ CCI_REG8(0x3c98), 0x54 },
+	{ CCI_REG8(0x3c9a), 0x70 },
+	{ CCI_REG8(0x3c9c), 0xc1 },
+	{ CCI_REG8(0x3c9e), 0xdd },
+	{ CCI_REG8(0x3cb0), 0x7a },
+	{ CCI_REG8(0x3cb2), 0xba },
+	{ CCI_REG8(0x3cc8), 0xbc },
+	{ CCI_REG8(0x3cca), 0x7c },
+	{ CCI_REG8(0x3cd4), 0xea },
+	{ CCI_REG8(0x3cd5), 0x01 },
+	{ CCI_REG8(0x3cd6), 0x4a },
+	{ CCI_REG8(0x3cd8), 0x00 },
+	{ CCI_REG8(0x3cd9), 0x00 },
+	{ CCI_REG8(0x3cda), 0xff },
+	{ CCI_REG8(0x3cdb), 0x03 },
+	{ CCI_REG8(0x3cdc), 0x00 },
+	{ CCI_REG8(0x3cdd), 0x00 },
+	{ CCI_REG8(0x3cde), 0xff },
+	{ CCI_REG8(0x3cdf), 0x03 },
+	{ CCI_REG8(0x3ce4), 0x4c },
+	{ CCI_REG8(0x3ce6), 0xec },
+	{ CCI_REG8(0x3ce7), 0x01 },
+	{ CCI_REG8(0x3ce8), 0xff },
+	{ CCI_REG8(0x3ce9), 0x03 },
+	{ CCI_REG8(0x3cea), 0x00 },
+	{ CCI_REG8(0x3ceb), 0x00 },
+	{ CCI_REG8(0x3cec), 0xff },
+	{ CCI_REG8(0x3ced), 0x03 },
+	{ CCI_REG8(0x3cee), 0x00 },
+	{ CCI_REG8(0x3cef), 0x00 },
+	{ CCI_REG8(0x3cf2), 0xff },
+	{ CCI_REG8(0x3cf3), 0x03 },
+	{ CCI_REG8(0x3cf4), 0x00 },
+	{ CCI_REG8(0x3e28), 0x82 },
+	{ CCI_REG8(0x3e2a), 0x80 },
+	{ CCI_REG8(0x3e30), 0x85 },
+	{ CCI_REG8(0x3e32), 0x7d },
+	{ CCI_REG8(0x3e5c), 0xce },
+	{ CCI_REG8(0x3e5e), 0xd3 },
+	{ CCI_REG8(0x3e70), 0x53 },
+	{ CCI_REG8(0x3e72), 0x58 },
+	{ CCI_REG8(0x3e74), 0xc0 },
+	{ CCI_REG8(0x3e76), 0xc5 },
+	{ CCI_REG8(0x3e78), 0xc0 },
+	{ CCI_REG8(0x3e79), 0x01 },
+	{ CCI_REG8(0x3e7a), 0xd4 },
+	{ CCI_REG8(0x3e7b), 0x01 },
+	{ CCI_REG8(0x3eb4), 0x0b },
+	{ CCI_REG8(0x3eb5), 0x02 },
+	{ CCI_REG8(0x3eb6), 0x4d },
+	{ CCI_REG8(0x3eb7), 0x42 },
+	{ CCI_REG8(0x3eec), 0xf3 },
+	{ CCI_REG8(0x3eee), 0xe7 },
+	{ CCI_REG8(0x3f01), 0x01 },
+	{ CCI_REG8(0x3f24), 0x10 },
+	{ CCI_REG8(0x3f28), 0x2d },
+	{ CCI_REG8(0x3f2a), 0x2d },
+	{ CCI_REG8(0x3f2c), 0x2d },
+	{ CCI_REG8(0x3f2e), 0x2d },
+	{ CCI_REG8(0x3f30), 0x23 },
+	{ CCI_REG8(0x3f38), 0x2d },
+	{ CCI_REG8(0x3f3a), 0x2d },
+	{ CCI_REG8(0x3f3c), 0x2d },
+	{ CCI_REG8(0x3f3e), 0x28 },
+	{ CCI_REG8(0x3f40), 0x1e },
+	{ CCI_REG8(0x3f48), 0x2d },
+	{ CCI_REG8(0x3f4a), 0x2d },
+	{ CCI_REG8(0x3f4c), 0x00 },
+	{ CCI_REG8(0x4004), 0xe4 },
+	{ CCI_REG8(0x4006), 0xff },
+	{ CCI_REG8(0x4018), 0x69 },
+	{ CCI_REG8(0x401a), 0x84 },
+	{ CCI_REG8(0x401c), 0xd6 },
+	{ CCI_REG8(0x401e), 0xf1 },
+	{ CCI_REG8(0x4038), 0xde },
+	{ CCI_REG8(0x403a), 0x00 },
+	{ CCI_REG8(0x403b), 0x01 },
+	{ CCI_REG8(0x404c), 0x63 },
+	{ CCI_REG8(0x404e), 0x85 },
+	{ CCI_REG8(0x4050), 0xd0 },
+	{ CCI_REG8(0x4052), 0xf2 },
+	{ CCI_REG8(0x4108), 0xdd },
+	{ CCI_REG8(0x410a), 0xf7 },
+	{ CCI_REG8(0x411c), 0x62 },
+	{ CCI_REG8(0x411e), 0x7c },
+	{ CCI_REG8(0x4120), 0xcf },
+	{ CCI_REG8(0x4122), 0xe9 },
+	{ CCI_REG8(0x4138), 0xe6 },
+	{ CCI_REG8(0x413a), 0xf1 },
+	{ CCI_REG8(0x414c), 0x6b },
+	{ CCI_REG8(0x414e), 0x76 },
+	{ CCI_REG8(0x4150), 0xd8 },
+	{ CCI_REG8(0x4152), 0xe3 },
+	{ CCI_REG8(0x417e), 0x03 },
+	{ CCI_REG8(0x417f), 0x01 },
+	{ CCI_REG8(0x4186), 0xe0 },
+	{ CCI_REG8(0x4190), 0xf3 },
+	{ CCI_REG8(0x4192), 0xf7 },
+	{ CCI_REG8(0x419c), 0x78 },
+	{ CCI_REG8(0x419e), 0x7c },
+	{ CCI_REG8(0x41a0), 0xe5 },
+	{ CCI_REG8(0x41a2), 0xe9 },
+	{ CCI_REG8(0x41c8), 0xe2 },
+	{ CCI_REG8(0x41ca), 0xfd },
+	{ CCI_REG8(0x41dc), 0x67 },
+	{ CCI_REG8(0x41de), 0x82 },
+	{ CCI_REG8(0x41e0), 0xd4 },
+	{ CCI_REG8(0x41e2), 0xef },
+	{ CCI_REG8(0x4200), 0xde },
+	{ CCI_REG8(0x4202), 0xda },
+	{ CCI_REG8(0x4218), 0x63 },
+	{ CCI_REG8(0x421a), 0x5f },
+	{ CCI_REG8(0x421c), 0xd0 },
+	{ CCI_REG8(0x421e), 0xcc },
+	{ CCI_REG8(0x425a), 0x82 },
+	{ CCI_REG8(0x425c), 0xef },
+	{ CCI_REG8(0x4348), 0xfe },
+	{ CCI_REG8(0x4349), 0x06 },
+	{ CCI_REG8(0x4352), 0xce },
+	{ CCI_REG8(0x4420), 0x0b },
+	{ CCI_REG8(0x4421), 0x02 },
+	{ CCI_REG8(0x4422), 0x4d },
+	{ CCI_REG8(0x4423), 0x0a },
+	{ CCI_REG8(0x4426), 0xf5 },
+	{ CCI_REG8(0x442a), 0xe7 },
+	{ CCI_REG8(0x4432), 0xf5 },
+	{ CCI_REG8(0x4436), 0xe7 },
+	{ CCI_REG8(0x4466), 0xb4 },
+	{ CCI_REG8(0x446e), 0x32 },
+	{ CCI_REG8(0x449f), 0x1c },
+	{ CCI_REG8(0x44a4), 0x2c },
+	{ CCI_REG8(0x44a6), 0x2c },
+	{ CCI_REG8(0x44a8), 0x2c },
+	{ CCI_REG8(0x44aa), 0x2c },
+	{ CCI_REG8(0x44b4), 0x2c },
+	{ CCI_REG8(0x44b6), 0x2c },
+	{ CCI_REG8(0x44b8), 0x2c },
+	{ CCI_REG8(0x44ba), 0x2c },
+	{ CCI_REG8(0x44c4), 0x2c },
+	{ CCI_REG8(0x44c6), 0x2c },
+	{ CCI_REG8(0x44c8), 0x2c },
+	{ CCI_REG8(0x4506), 0xf3 },
+	{ CCI_REG8(0x450e), 0xe5 },
+	{ CCI_REG8(0x4516), 0xf3 },
+	{ CCI_REG8(0x4522), 0xe5 },
+	{ CCI_REG8(0x4524), 0xf3 },
+	{ CCI_REG8(0x452c), 0xe5 },
+	{ CCI_REG8(0x453c), 0x22 },
+	{ CCI_REG8(0x453d), 0x1b },
+	{ CCI_REG8(0x453e), 0x1b },
+	{ CCI_REG8(0x453f), 0x15 },
+	{ CCI_REG8(0x4540), 0x15 },
+	{ CCI_REG8(0x4541), 0x15 },
+	{ CCI_REG8(0x4542), 0x15 },
+	{ CCI_REG8(0x4543), 0x15 },
+	{ CCI_REG8(0x4544), 0x15 },
+	{ CCI_REG8(0x4548), 0x00 },
+	{ CCI_REG8(0x4549), 0x01 },
+	{ CCI_REG8(0x454a), 0x01 },
+	{ CCI_REG8(0x454b), 0x06 },
+	{ CCI_REG8(0x454c), 0x06 },
+	{ CCI_REG8(0x454d), 0x06 },
+	{ CCI_REG8(0x454e), 0x06 },
+	{ CCI_REG8(0x454f), 0x06 },
+	{ CCI_REG8(0x4550), 0x06 },
+	{ CCI_REG8(0x4554), 0x55 },
+	{ CCI_REG8(0x4555), 0x02 },
+	{ CCI_REG8(0x4556), 0x42 },
+	{ CCI_REG8(0x4557), 0x05 },
+	{ CCI_REG8(0x4558), 0xfd },
+	{ CCI_REG8(0x4559), 0x05 },
+	{ CCI_REG8(0x455a), 0x94 },
+	{ CCI_REG8(0x455b), 0x06 },
+	{ CCI_REG8(0x455d), 0x06 },
+	{ CCI_REG8(0x455e), 0x49 },
+	{ CCI_REG8(0x455f), 0x07 },
+	{ CCI_REG8(0x4560), 0x7f },
+	{ CCI_REG8(0x4561), 0x07 },
+	{ CCI_REG8(0x4562), 0xa5 },
+	{ CCI_REG8(0x4564), 0x55 },
+	{ CCI_REG8(0x4565), 0x02 },
+	{ CCI_REG8(0x4566), 0x42 },
+	{ CCI_REG8(0x4567), 0x05 },
+	{ CCI_REG8(0x4568), 0xfd },
+	{ CCI_REG8(0x4569), 0x05 },
+	{ CCI_REG8(0x456a), 0x94 },
+	{ CCI_REG8(0x456b), 0x06 },
+	{ CCI_REG8(0x456d), 0x06 },
+	{ CCI_REG8(0x456e), 0x49 },
+	{ CCI_REG8(0x456f), 0x07 },
+	{ CCI_REG8(0x4572), 0xa5 },
+	{ CCI_REG8(0x460c), 0x7d },
+	{ CCI_REG8(0x460e), 0xb1 },
+	{ CCI_REG8(0x4614), 0xa8 },
+	{ CCI_REG8(0x4616), 0xb2 },
+	{ CCI_REG8(0x461c), 0x7e },
+	{ CCI_REG8(0x461e), 0xa7 },
+	{ CCI_REG8(0x4624), 0xa8 },
+	{ CCI_REG8(0x4626), 0xb2 },
+	{ CCI_REG8(0x462c), 0x7e },
+	{ CCI_REG8(0x462e), 0x8a },
+	{ CCI_REG8(0x4630), 0x94 },
+	{ CCI_REG8(0x4632), 0xa7 },
+	{ CCI_REG8(0x4634), 0xfb },
+	{ CCI_REG8(0x4636), 0x2f },
+	{ CCI_REG8(0x4638), 0x81 },
+	{ CCI_REG8(0x4639), 0x01 },
+	{ CCI_REG8(0x463a), 0xb5 },
+	{ CCI_REG8(0x463b), 0x01 },
+	{ CCI_REG8(0x463c), 0x26 },
+	{ CCI_REG8(0x463e), 0x30 },
+	{ CCI_REG8(0x4640), 0xac },
+	{ CCI_REG8(0x4641), 0x01 },
+	{ CCI_REG8(0x4642), 0xb6 },
+	{ CCI_REG8(0x4643), 0x01 },
+	{ CCI_REG8(0x4644), 0xfc },
+	{ CCI_REG8(0x4646), 0x25 },
+	{ CCI_REG8(0x4648), 0x82 },
+	{ CCI_REG8(0x4649), 0x01 },
+	{ CCI_REG8(0x464a), 0xab },
+	{ CCI_REG8(0x464b), 0x01 },
+	{ CCI_REG8(0x464c), 0x26 },
+	{ CCI_REG8(0x464e), 0x30 },
+	{ CCI_REG8(0x4654), 0xfc },
+	{ CCI_REG8(0x4656), 0x08 },
+	{ CCI_REG8(0x4658), 0x12 },
+	{ CCI_REG8(0x465a), 0x25 },
+	{ CCI_REG8(0x4662), 0xfc },
+	{ CCI_REG8(0x46a2), 0xfb },
+	{ CCI_REG8(0x46d6), 0xf3 },
+	{ CCI_REG8(0x46e6), 0x00 },
+	{ CCI_REG8(0x46e8), 0xff },
+	{ CCI_REG8(0x46e9), 0x03 },
+	{ CCI_REG8(0x46ec), 0x7a },
+	{ CCI_REG8(0x46ee), 0xe5 },
+	{ CCI_REG8(0x46f4), 0xee },
+	{ CCI_REG8(0x46f6), 0xf2 },
+	{ CCI_REG8(0x470c), 0xff },
+	{ CCI_REG8(0x470d), 0x03 },
+	{ CCI_REG8(0x470e), 0x00 },
+	{ CCI_REG8(0x4714), 0xe0 },
+	{ CCI_REG8(0x4716), 0xe4 },
+	{ CCI_REG8(0x471e), 0xed },
+	{ CCI_REG8(0x472e), 0x00 },
+	{ CCI_REG8(0x4730), 0xff },
+	{ CCI_REG8(0x4731), 0x03 },
+	{ CCI_REG8(0x4734), 0x7b },
+	{ CCI_REG8(0x4736), 0xdf },
+	{ CCI_REG8(0x4754), 0x7d },
+	{ CCI_REG8(0x4756), 0x8b },
+	{ CCI_REG8(0x4758), 0x93 },
+	{ CCI_REG8(0x475a), 0xb1 },
+	{ CCI_REG8(0x475c), 0xfb },
+	{ CCI_REG8(0x475e), 0x09 },
+	{ CCI_REG8(0x4760), 0x11 },
+	{ CCI_REG8(0x4762), 0x2f },
+	{ CCI_REG8(0x4766), 0xcc },
+	{ CCI_REG8(0x4776), 0xcb },
+	{ CCI_REG8(0x477e), 0x4a },
+	{ CCI_REG8(0x478e), 0x49 },
+	{ CCI_REG8(0x4794), 0x7c },
+	{ CCI_REG8(0x4796), 0x8f },
+	{ CCI_REG8(0x4798), 0xb3 },
+	{ CCI_REG8(0x4799), 0x00 },
+	{ CCI_REG8(0x479a), 0xcc },
+	{ CCI_REG8(0x479c), 0xc1 },
+	{ CCI_REG8(0x479e), 0xcb },
+	{ CCI_REG8(0x47a4), 0x7d },
+	{ CCI_REG8(0x47a6), 0x8e },
+	{ CCI_REG8(0x47a8), 0xb4 },
+	{ CCI_REG8(0x47a9), 0x00 },
+	{ CCI_REG8(0x47aa), 0xc0 },
+	{ CCI_REG8(0x47ac), 0xfa },
+	{ CCI_REG8(0x47ae), 0x0d },
+	{ CCI_REG8(0x47b0), 0x31 },
+	{ CCI_REG8(0x47b1), 0x01 },
+	{ CCI_REG8(0x47b2), 0x4a },
+	{ CCI_REG8(0x47b3), 0x01 },
+	{ CCI_REG8(0x47b4), 0x3f },
+	{ CCI_REG8(0x47b6), 0x49 },
+	{ CCI_REG8(0x47bc), 0xfb },
+	{ CCI_REG8(0x47be), 0x0c },
+	{ CCI_REG8(0x47c0), 0x32 },
+	{ CCI_REG8(0x47c1), 0x01 },
+	{ CCI_REG8(0x47c2), 0x3e },
+	{ CCI_REG8(0x47c3), 0x01 },
+	{ IMX678_REG_WDMODE, 0x00 },
+	{ IMX678_REG_MDBIT, 0x01 },
+	{ IMX678_REG_XXS_DRV, 0x00 },
+};
+
+static const u32 codes_bayer[] = {
+	MEDIA_BUS_FMT_SRGGB12_1X12,
+};
+
+static const u32 codes_monochrome[] = {
+	MEDIA_BUS_FMT_Y12_1X12,
+};
+
+static const struct imx678_model_info imx678_aaqr_info = {
+	.type = IMX678_COLOR,
+	.codes = codes_bayer,
+	.num_codes = ARRAY_SIZE(codes_bayer),
+};
+
+static const struct imx678_model_info imx678_aamr_info = {
+	.type = IMX678_MONOCHROME,
+	.codes = codes_monochrome,
+	.num_codes = ARRAY_SIZE(codes_monochrome),
+};
+
+static const char * const imx678_supply_name[] = {
+	"avdd",  /* Analog (3.3V) supply */
+	"dvdd",  /* Digital Core (1.1V) supply */
+	"ovdd",  /* IF (1.8V) supply */
+};
+
+struct imx678 {
+	struct v4l2_subdev sd;
+	struct media_pad pad;
+	struct regmap *cci;
+
+	const struct imx678_model_info *info;
+
+	struct clk *xclk;
+	u32 xclk_freq;
+
+	/* chosen INCK_SEL register value */
+	u8 inck_sel_val;
+
+	/* Link configurations */
+	enum imx678_lanemode lane_mode;
+	unsigned long link_freq_bitmap;
+
+	struct gpio_desc *reset_gpio;
+	struct regulator_bulk_data supplies[ARRAY_SIZE(imx678_supply_name)];
+
+	struct v4l2_ctrl_handler ctrl_handler;
+
+	/* V4L2 Controls */
+	struct v4l2_ctrl *exposure;
+	struct v4l2_ctrl *vblank;
+	struct v4l2_ctrl *hblank;
+
+	/* Track VMAX for exposure updates */
+	u32 vmax;
+};
+
+static inline struct imx678 *to_imx678(struct v4l2_subdev *_sd)
+{
+	return container_of_const(_sd, struct imx678, sd);
+}
+
+static u32 imx678_default_mbus_code(struct imx678 *imx678)
+{
+	return imx678->info->codes[0];
+}
+
+static bool imx678_mbus_code_supported(struct imx678 *imx678, u32 code)
+{
+	for (unsigned int i = 0; i < imx678->info->num_codes; i++) {
+		if (imx678->info->codes[i] == code)
+			return true;
+	}
+
+	return false;
+}
+
+static int imx678_set_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct imx678 *imx678 = container_of_const(ctrl->handler, struct
+						   imx678, ctrl_handler);
+	struct i2c_client *client = v4l2_get_subdevdata(&imx678->sd);
+	const struct v4l2_mbus_framefmt *format;
+	struct v4l2_subdev_state *state;
+	int ret = 0;
+
+	state = v4l2_subdev_get_locked_active_state(&imx678->sd);
+	format = v4l2_subdev_state_get_format(state, IMX678_SOURCE_PAD);
+
+	if (ctrl->id == V4L2_CID_VBLANK) {
+		u32 current_exposure = imx678->exposure->cur.val;
+
+		imx678->vmax = format->height + ctrl->val;
+
+		current_exposure = clamp_t(u32, current_exposure,
+					   IMX678_EXPOSURE_MIN,
+					   imx678->vmax - IMX678_SHR_MIN);
+		ret = __v4l2_ctrl_modify_range(imx678->exposure,
+					       IMX678_EXPOSURE_MIN,
+					       imx678->vmax - IMX678_SHR_MIN,
+					       1, current_exposure);
+		if (ret)
+			return ret;
+	}
+
+	/*
+	 * Only apply control values when device is powered on (RPM ACTIVE)
+	 * and streaming (usage count != 0)
+	 */
+	if (!pm_runtime_get_if_in_use(&client->dev))
+		return 0;
+
+	switch (ctrl->id) {
+	case V4L2_CID_VBLANK:
+		cci_write(imx678->cci, IMX678_REG_VMAX, imx678->vmax, &ret);
+		fallthrough; /* SHR = VMAX - exposure, so update it */
+	case V4L2_CID_EXPOSURE: {
+		u32 shr = imx678->vmax - imx678->exposure->val;
+
+		cci_write(imx678->cci, IMX678_REG_SHR, shr, &ret);
+		break;
+	}
+	case V4L2_CID_ANALOGUE_GAIN:
+		cci_write(imx678->cci, IMX678_REG_GAIN, ctrl->val, &ret);
+		break;
+	case V4L2_CID_HBLANK: {
+		u32 hmax = (format->width + ctrl->val) / IMX678_PIX_PER_CLK;
+
+		cci_write(imx678->cci, IMX678_REG_HMAX, hmax, &ret);
+		break;
+	}
+	case V4L2_CID_TEST_PATTERN: {
+		cci_write(imx678->cci, IMX678_REG_TPG_COLORWIDTH,
+			  IMX678_TPG_COLORWIDTH_160PIX, &ret);
+		cci_write(imx678->cci, IMX678_REG_TPG_PATSEL_DUOUT,
+			  imx678_tpg_val[ctrl->val], &ret);
+		cci_write(imx678->cci, IMX678_REG_TPG_EN_DUOUT,
+			  (ctrl->val) ? 1 : 0,
+			  &ret);
+		break;
+	}
+	case V4L2_CID_HFLIP:
+		cci_write(imx678->cci, IMX678_REG_WINMODEH, ctrl->val, &ret);
+		break;
+	case V4L2_CID_VFLIP:
+		cci_write(imx678->cci, IMX678_REG_WINMODEV, ctrl->val, &ret);
+		break;
+	default:
+		dev_warn(&client->dev,
+			 "ctrl(id:0x%x,val:0x%x) is not handled\n",
+			 ctrl->id, ctrl->val);
+		break;
+	}
+
+	pm_runtime_put(&client->dev);
+
+	return ret;
+}
+
+static const struct v4l2_ctrl_ops imx678_ctrl_ops = {
+	.s_ctrl = imx678_set_ctrl,
+};
+
+static int imx678_enum_mbus_code(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *sd_state,
+				 struct v4l2_subdev_mbus_code_enum *code)
+{
+	struct imx678 *imx678 = to_imx678(sd);
+
+	if (code->index >= imx678->info->num_codes)
+		return -EINVAL;
+
+	code->code = imx678->info->codes[code->index];
+
+	return 0;
+}
+
+static int imx678_enum_frame_size(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_state *sd_state,
+				  struct v4l2_subdev_frame_size_enum *fse)
+{
+	struct imx678 *imx678 = to_imx678(sd);
+	const struct v4l2_rect *crop;
+
+	if (fse->index)
+		return -EINVAL;
+
+	if (!imx678_mbus_code_supported(imx678, fse->code))
+		return -EINVAL;
+
+	crop = v4l2_subdev_state_get_crop(sd_state, fse->pad);
+
+	fse->min_width = crop->width;
+	fse->max_width = fse->min_width;
+	fse->min_height = crop->height;
+	fse->max_height = fse->min_height;
+
+	return 0;
+}
+
+static int imx678_get_selection(struct v4l2_subdev *sd,
+				struct v4l2_subdev_state *sd_state,
+				struct v4l2_subdev_selection *sel)
+{
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP:
+		sel->r = *v4l2_subdev_state_get_crop(sd_state, sel->pad);
+		return 0;
+
+	case V4L2_SEL_TGT_NATIVE_SIZE:
+		sel->r = imx678_native_area;
+		return 0;
+
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		sel->r = imx678_active_area;
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static int imx678_init_state(struct v4l2_subdev *sd,
+			     struct v4l2_subdev_state *state)
+{
+	struct imx678 *imx678 = to_imx678(sd);
+	struct v4l2_mbus_framefmt *format;
+	struct v4l2_rect *crop;
+
+	crop = v4l2_subdev_state_get_crop(state, IMX678_SOURCE_PAD);
+	*crop = imx678_active_area;
+
+	format = v4l2_subdev_state_get_format(state, IMX678_SOURCE_PAD);
+	format->code = imx678_default_mbus_code(imx678);
+	format->width = imx678_active_area.width;
+	format->height = imx678_active_area.height;
+	format->field = V4L2_FIELD_NONE;
+	format->colorspace = V4L2_COLORSPACE_RAW;
+	format->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
+	format->quantization = V4L2_QUANTIZATION_FULL_RANGE;
+	format->xfer_func = V4L2_XFER_FUNC_NONE;
+
+	return 0;
+}
+
+static int imx678_write_common(struct imx678 *imx678)
+{
+	int ret = 0;
+
+	cci_multi_reg_write(imx678->cci, common_regs, ARRAY_SIZE(common_regs),
+			    &ret);
+
+	cci_write(imx678->cci, IMX678_REG_INCK_SEL, imx678->inck_sel_val, &ret);
+	cci_write(imx678->cci, IMX678_REG_DATARATE_SEL,
+		  link_freqs_reg_value[__ffs(imx678->link_freq_bitmap)], &ret);
+	cci_write(imx678->cci, IMX678_REG_LANEMODE, imx678->lane_mode, &ret);
+
+	cci_write(imx678->cci, IMX678_REG_INTERFACE_SEL, IMX678_INTERFACE_2L_4L,
+		  &ret);
+
+	return ret;
+}
+
+static int imx678_program_window(struct imx678 *imx678,
+				 const struct v4l2_rect *crop)
+{
+	int ret = 0;
+
+	cci_write(imx678->cci, IMX678_REG_ADDMODE, 0x00, &ret);
+	cci_write(imx678->cci, IMX678_REG_WINMODE,
+		  v4l2_rect_equal(crop, &imx678_active_area) ? 0x00 : 0x04,
+		  &ret);
+	cci_write(imx678->cci, IMX678_REG_PIX_HST,
+		  crop->left - imx678_active_area.left, &ret);
+	cci_write(imx678->cci, IMX678_REG_PIX_HWIDTH, crop->width, &ret);
+	cci_write(imx678->cci, IMX678_REG_PIX_VST,
+		  crop->top - imx678_active_area.top, &ret);
+	cci_write(imx678->cci, IMX678_REG_PIX_VWIDTH, crop->height, &ret);
+	cci_write(imx678->cci, IMX678_REG_ADBIT, 0x01, &ret);
+
+	return ret;
+}
+
+static int imx678_enable_streams(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *state, u32 pad,
+				 u64 mask)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	struct imx678 *imx678 = to_imx678(sd);
+	const struct v4l2_rect *crop;
+	int ret;
+
+	ret = pm_runtime_resume_and_get(&client->dev);
+	if (ret < 0)
+		return ret;
+
+	crop = v4l2_subdev_state_get_crop(state, pad);
+	ret = imx678_program_window(imx678, crop);
+	if (ret) {
+		dev_err(&client->dev, "%s failed to set mode\n", __func__);
+		goto err_rpm_put;
+	}
+
+	ret = __v4l2_ctrl_handler_setup(imx678->sd.ctrl_handler);
+	if (ret) {
+		dev_err(&client->dev, "%s failed to apply user values\n",
+			__func__);
+		goto err_rpm_put;
+	}
+
+	cci_write(imx678->cci, IMX678_REG_MODE_SELECT, IMX678_MODE_STREAMING,
+		  &ret);
+	usleep_range(IMX678_STREAM_DELAY_US, IMX678_STREAM_DELAY_US +
+		     IMX678_STREAM_DELAY_RANGE_US);
+	cci_write(imx678->cci, IMX678_REG_XMSTA, 0x00, &ret);
+
+	if (ret) {
+		dev_err(&client->dev, "%s failed to start streaming\n",
+			__func__);
+		goto err_rpm_put;
+	}
+
+	return 0;
+
+err_rpm_put:
+	pm_runtime_put(&client->dev);
+
+	return ret;
+}
+
+static int imx678_disable_streams(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_state *state,
+				  u32 pad, u64 mask)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	struct imx678 *imx678 = to_imx678(sd);
+	int ret = 0;
+
+	/* Master mode disable */
+	cci_write(imx678->cci, IMX678_REG_XMSTA, 0x01, &ret);
+	/* Standby */
+	cci_write(imx678->cci, IMX678_REG_MODE_SELECT, IMX678_MODE_STANDBY,
+		  &ret);
+	if (ret)
+		dev_err(&client->dev, "%s failed to stop stream\n", __func__);
+
+	pm_runtime_put(&client->dev);
+
+	return ret;
+}
+
+static int imx678_power_on(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct imx678 *imx678 = to_imx678(sd);
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(imx678_supply_name),
+				    imx678->supplies);
+	if (ret) {
+		dev_err(&client->dev, "%s: failed to enable regulators\n",
+			__func__);
+		return ret;
+	}
+
+	fsleep(1); /* Tlow > 500ns */
+
+	gpiod_set_value_cansleep(imx678->reset_gpio, 0);
+
+	fsleep(1); /* T3 > 1us */
+
+	ret = clk_prepare_enable(imx678->xclk);
+	if (ret) {
+		dev_err(&client->dev, "%s: failed to enable clock\n",
+			__func__);
+		goto reg_off;
+	}
+
+	fsleep(20); /* T4 > 20us */
+
+	ret = imx678_write_common(imx678);
+	if (ret) {
+		dev_err(&client->dev, "%s failed to write registers\n",
+			__func__);
+		goto clk_off;
+	}
+
+	return 0;
+
+clk_off:
+	clk_disable_unprepare(imx678->xclk);
+
+reg_off:
+	gpiod_set_value_cansleep(imx678->reset_gpio, 1);
+	regulator_bulk_disable(ARRAY_SIZE(imx678_supply_name),
+			       imx678->supplies);
+
+	return ret;
+}
+
+static int imx678_power_off(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct imx678 *imx678 = to_imx678(sd);
+
+	clk_disable_unprepare(imx678->xclk);
+	gpiod_set_value_cansleep(imx678->reset_gpio, 1);
+	regulator_bulk_disable(ARRAY_SIZE(imx678_supply_name),
+			       imx678->supplies);
+
+	return 0;
+}
+
+static int imx678_identify_model(struct imx678 *imx678)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&imx678->sd);
+	const struct imx678_model_info *info;
+	enum imx678_type detected;
+	int ret = 0;
+	u64 val = 0;
+
+	info = device_get_match_data(&client->dev);
+
+	/*
+	 * This sensor's ID registers become accessible 80ms after coming out
+	 * of STANDBY mode.
+	 */
+	cci_write(imx678->cci, IMX678_REG_MODE_SELECT, 0, &ret);
+	fsleep(IMX678_MODULE_ID_DELAY);
+
+	cci_read(imx678->cci, IMX678_REG_MODULE_ID, &val, &ret);
+
+	if (ret) {
+		dev_err(&client->dev,
+			"I2C transaction failed ret = %d\n", ret);
+		return ret;
+	}
+
+	if (val != IMX678_ID) {
+		dev_err(&client->dev,
+			"Chip ID mismatch: %x!=%llx\n", IMX678_ID, val);
+		return -ENXIO;
+	}
+
+	cci_read(imx678->cci, IMX678_REG_MONOCHROME, &val, &ret);
+
+	if (ret) {
+		dev_err(&client->dev,
+			"I2C transaction failed ret = %d\n", ret);
+		return ret;
+	}
+
+	detected = val & IMX678_TYPE;
+
+	/* Prefer to use sensor type specified in device tree */
+	if (info) {
+		imx678->info = info;
+		if (detected != info->type)
+			dev_err(&client->dev,
+				"detected %s sensor, DT specifies %s; using DT value\n",
+				detected == IMX678_COLOR ? "color" : "mono",
+				info->type == IMX678_COLOR ? "color" : "mono");
+	} else {
+		imx678->info = detected == IMX678_MONOCHROME ?
+			       &imx678_aamr_info : &imx678_aaqr_info;
+		dev_info(&client->dev,
+			 "sensor type missing in DT; detected %s sensor\n",
+			 detected == IMX678_MONOCHROME ? "mono" : "color");
+	}
+
+	return 0;
+}
+
+static const struct v4l2_subdev_video_ops imx678_video_ops = {
+	.s_stream = v4l2_subdev_s_stream_helper,
+};
+
+static const struct v4l2_subdev_pad_ops imx678_pad_ops = {
+	.enum_mbus_code = imx678_enum_mbus_code,
+	.get_fmt = v4l2_subdev_get_fmt,
+	.set_fmt = v4l2_subdev_get_fmt,
+	.get_selection = imx678_get_selection,
+	.enum_frame_size = imx678_enum_frame_size,
+	.enable_streams = imx678_enable_streams,
+	.disable_streams = imx678_disable_streams,
+};
+
+static const struct v4l2_subdev_ops imx678_subdev_ops = {
+	.video = &imx678_video_ops,
+	.pad = &imx678_pad_ops,
+};
+
+static const struct v4l2_subdev_internal_ops imx678_internal_ops = {
+	.init_state = imx678_init_state,
+};
+
+static int imx678_init_controls(struct imx678 *imx678)
+{
+	struct v4l2_ctrl_handler *ctrl_hdlr;
+	const u32 hmax_4lane = min_hmax_4lane[__ffs(imx678->link_freq_bitmap)];
+	const u32 lane_scale = imx678->lane_mode == IMX678_LANEMODE_2L ? 2 : 1;
+	struct i2c_client *client = v4l2_get_subdevdata(&imx678->sd);
+	struct v4l2_fwnode_device_properties props;
+	struct v4l2_ctrl *link_freq;
+	s32 hblank, max_hblank, vblank, max_vblank;
+	u32 hmax;
+	int ret;
+
+	ret = v4l2_fwnode_device_parse(&client->dev, &props);
+	if (ret < 0)
+		return ret;
+
+	ctrl_hdlr = &imx678->ctrl_handler;
+	ret = v4l2_ctrl_handler_init(ctrl_hdlr, 11);
+	if (ret)
+		return ret;
+
+	imx678->vmax = IMX678_VMAX_DEFAULT;
+	hmax = hmax_4lane * lane_scale;
+
+	/* PIXEL_RATE is fixed and read-only */
+	v4l2_ctrl_new_std(ctrl_hdlr, &imx678_ctrl_ops, V4L2_CID_PIXEL_RATE,
+			  IMX678_PIXEL_RATE, IMX678_PIXEL_RATE, 1,
+			  IMX678_PIXEL_RATE);
+
+	/* LINK_FREQ is also read only */
+	link_freq = v4l2_ctrl_new_int_menu(ctrl_hdlr, &imx678_ctrl_ops,
+					   V4L2_CID_LINK_FREQ,
+					   ARRAY_SIZE(link_freqs) - 1,
+					   __ffs(imx678->link_freq_bitmap),
+					   link_freqs);
+
+	if (link_freq)
+		link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+
+	vblank = imx678->vmax - imx678_active_area.height;
+	max_vblank = IMX678_VMAX_MAX - imx678_active_area.height;
+	imx678->vblank = v4l2_ctrl_new_std(ctrl_hdlr, &imx678_ctrl_ops,
+					   V4L2_CID_VBLANK, vblank, max_vblank,
+					   2, vblank);
+
+	hblank = hmax * IMX678_PIX_PER_CLK - imx678_active_area.width;
+	max_hblank = IMX678_HMAX_MAX * IMX678_PIX_PER_CLK -
+		     imx678_active_area.width;
+	imx678->hblank = v4l2_ctrl_new_std(ctrl_hdlr, &imx678_ctrl_ops,
+					   V4L2_CID_HBLANK, hblank, max_hblank,
+					   IMX678_PIX_PER_CLK, hblank);
+
+	imx678->exposure = v4l2_ctrl_new_std(ctrl_hdlr, &imx678_ctrl_ops,
+					     V4L2_CID_EXPOSURE,
+					     IMX678_EXPOSURE_MIN,
+					     IMX678_VMAX_DEFAULT -
+					     IMX678_SHR_MIN,
+					     IMX678_EXPOSURE_STEP,
+					     IMX678_EXPOSURE_DEFAULT);
+
+	v4l2_ctrl_new_std(ctrl_hdlr, &imx678_ctrl_ops, V4L2_CID_ANALOGUE_GAIN,
+			  IMX678_ANA_GAIN_MIN_NORMAL,
+			  IMX678_ANA_GAIN_MAX_NORMAL, IMX678_ANA_GAIN_STEP,
+			  IMX678_ANA_GAIN_DEFAULT);
+
+	v4l2_ctrl_new_std(ctrl_hdlr, &imx678_ctrl_ops, V4L2_CID_HFLIP,
+			  0, 1, 1, 0);
+	v4l2_ctrl_new_std(ctrl_hdlr, &imx678_ctrl_ops, V4L2_CID_VFLIP,
+			  0, 1, 1, 0);
+
+	v4l2_ctrl_new_std_menu_items(ctrl_hdlr, &imx678_ctrl_ops,
+				     V4L2_CID_TEST_PATTERN,
+				     ARRAY_SIZE(imx678_tpg_menu) - 1, 0, 0,
+				     imx678_tpg_menu);
+
+	v4l2_ctrl_new_fwnode_properties(ctrl_hdlr, &imx678_ctrl_ops, &props);
+
+	if (ctrl_hdlr->error) {
+		ret = ctrl_hdlr->error;
+		dev_err(&client->dev, "%s control init failed (%d)\n",
+			__func__, ret);
+		v4l2_ctrl_handler_free(ctrl_hdlr);
+		return ret;
+	}
+
+	imx678->sd.ctrl_handler = ctrl_hdlr;
+
+	return 0;
+}
+
+static int imx678_check_hwcfg(struct device *dev, struct imx678 *imx678)
+{
+	struct fwnode_handle *endpoint;
+	struct v4l2_fwnode_endpoint ep_cfg = {
+		.bus_type = V4L2_MBUS_CSI2_DPHY
+	};
+	int ret = -EINVAL;
+
+	endpoint = fwnode_graph_get_endpoint_by_id(dev_fwnode(dev), 0, 0, 0);
+	if (!endpoint) {
+		dev_err(dev, "endpoint node not found\n");
+		return -EINVAL;
+	}
+
+	if (v4l2_fwnode_endpoint_alloc_parse(endpoint, &ep_cfg)) {
+		dev_err(dev, "could not parse endpoint\n");
+		goto error_out;
+	}
+
+	switch (ep_cfg.bus.mipi_csi2.num_data_lanes) {
+	case 2:
+		imx678->lane_mode = IMX678_LANEMODE_2L;
+		break;
+	case 4:
+		imx678->lane_mode = IMX678_LANEMODE_4L;
+		break;
+	default:
+		dev_err(dev,
+			"only 2 or 4 CSI2 data lanes are currently supported\n");
+		goto error_out;
+	}
+
+	ret = v4l2_link_freq_to_bitmap(dev, ep_cfg.link_frequencies,
+				       ep_cfg.nr_of_link_frequencies,
+				       link_freqs, ARRAY_SIZE(link_freqs),
+				       &imx678->link_freq_bitmap);
+
+error_out:
+	v4l2_fwnode_endpoint_free(&ep_cfg);
+	fwnode_handle_put(endpoint);
+
+	return ret;
+}
+
+static int imx678_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct imx678 *imx678;
+	int ret, i;
+
+	imx678 = devm_kzalloc(&client->dev, sizeof(*imx678), GFP_KERNEL);
+	if (!imx678)
+		return -ENOMEM;
+
+	v4l2_i2c_subdev_init(&imx678->sd, client, &imx678_subdev_ops);
+
+	imx678->cci = devm_cci_regmap_init_i2c(client, 16);
+	if (IS_ERR(imx678->cci))
+		return dev_err_probe(dev, PTR_ERR(imx678->cci),
+				     "failed to init CCI\n");
+
+	if (imx678_check_hwcfg(dev, imx678))
+		return -EINVAL;
+
+	imx678->xclk = devm_v4l2_sensor_clk_get(dev, NULL);
+	if (IS_ERR(imx678->xclk))
+		return dev_err_probe(dev, PTR_ERR(imx678->xclk),
+				     "failed to get xclk\n");
+
+	imx678->xclk_freq = clk_get_rate(imx678->xclk);
+
+	for (i = 0; i < ARRAY_SIZE(imx678_inck_table); ++i) {
+		if (imx678_inck_table[i].xclk_hz == imx678->xclk_freq) {
+			imx678->inck_sel_val = imx678_inck_table[i].inck_sel;
+			break;
+		}
+	}
+
+	if (i == ARRAY_SIZE(imx678_inck_table))
+		return dev_err_probe(dev, -EINVAL,
+				     "unsupported XCLK rate %u Hz\n",
+				     imx678->xclk_freq);
+
+	for (i = 0; i < ARRAY_SIZE(imx678_supply_name); i++)
+		imx678->supplies[i].supply = imx678_supply_name[i];
+
+	ret = devm_regulator_bulk_get(&client->dev,
+				      ARRAY_SIZE(imx678_supply_name),
+				      imx678->supplies);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to get regulators\n");
+
+	imx678->reset_gpio = devm_gpiod_get_optional(dev, "reset",
+						     GPIOD_OUT_HIGH);
+	if (IS_ERR(imx678->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(imx678->reset_gpio),
+				     "failed to get reset GPIO\n");
+
+	ret = imx678_power_on(dev);
+	if (ret)
+		return ret;
+
+	ret = imx678_identify_model(imx678);
+	if (ret)
+		goto error_power_off;
+
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+
+	ret = imx678_init_controls(imx678);
+	if (ret)
+		goto error_pm_runtime;
+
+	imx678->sd.internal_ops = &imx678_internal_ops;
+	imx678->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	imx678->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
+
+	imx678->pad.flags = MEDIA_PAD_FL_SOURCE;
+
+	ret = media_entity_pads_init(&imx678->sd.entity, 1, &imx678->pad);
+	if (ret) {
+		dev_err_probe(dev, ret, "failed to init entity pads\n");
+		goto error_handler_free;
+	}
+
+	imx678->sd.state_lock = imx678->ctrl_handler.lock;
+	ret = v4l2_subdev_init_finalize(&imx678->sd);
+	if (ret < 0) {
+		dev_err_probe(dev, ret, "subdev init error\n");
+		goto error_media_entity;
+	}
+
+	ret = v4l2_async_register_subdev_sensor(&imx678->sd);
+	if (ret < 0) {
+		dev_err_probe(dev, ret,
+			      "failed to register sensor sub-device\n");
+		goto error_subdev_cleanup;
+	}
+
+	pm_runtime_idle(dev);
+
+	return 0;
+
+error_subdev_cleanup:
+	v4l2_subdev_cleanup(&imx678->sd);
+
+error_media_entity:
+	media_entity_cleanup(&imx678->sd.entity);
+
+error_handler_free:
+	v4l2_ctrl_handler_free(imx678->sd.ctrl_handler);
+
+error_pm_runtime:
+	pm_runtime_disable(&client->dev);
+	pm_runtime_set_suspended(&client->dev);
+
+error_power_off:
+	imx678_power_off(&client->dev);
+
+	return ret;
+}
+
+static void imx678_remove(struct i2c_client *client)
+{
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct imx678 *imx678 = to_imx678(sd);
+
+	v4l2_async_unregister_subdev(sd);
+	v4l2_subdev_cleanup(sd);
+	media_entity_cleanup(&sd->entity);
+	v4l2_ctrl_handler_free(imx678->sd.ctrl_handler);
+
+	pm_runtime_disable(&client->dev);
+	if (!pm_runtime_status_suspended(&client->dev))
+		imx678_power_off(&client->dev);
+	pm_runtime_set_suspended(&client->dev);
+}
+
+static const struct dev_pm_ops imx678_pm_ops = {
+	SET_RUNTIME_PM_OPS(imx678_power_off, imx678_power_on, NULL)
+};
+
+static const struct of_device_id imx678_of_match[] = {
+	{ .compatible = "sony,imx678-aamr", .data = &imx678_aamr_info },
+	{ .compatible = "sony,imx678-aaqr", .data = &imx678_aaqr_info },
+	/* for non-conforming DTs that rely on runtime check */
+	{ .compatible = "sony,imx678" },
+	{ /* sentinel */ }
+};
+
+MODULE_DEVICE_TABLE(of, imx678_of_match);
+
+static struct i2c_driver imx678_i2c_driver = {
+	.driver = {
+		.name = "imx678",
+		.of_match_table = imx678_of_match,
+		.pm = pm_ptr(&imx678_pm_ops),
+	},
+	.probe = imx678_probe,
+	.remove = imx678_remove,
+};
+
+module_i2c_driver(imx678_i2c_driver);
+
+MODULE_AUTHOR("Will Whang <will@willwhang.com>");
+MODULE_AUTHOR("Tetsuya NOMURA <tetsuya.nomura@soho-enterprise.com>");
+MODULE_AUTHOR("Jai Luthra <jai.luthra@ideasonboard.com>");
+MODULE_DESCRIPTION("Sony imx678 sensor driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/i2c/ov02a10.c b/drivers/media/i2c/ov02a10.c
index 143dcfe104456..0150e4d296af6 100644
--- a/drivers/media/i2c/ov02a10.c
+++ b/drivers/media/i2c/ov02a10.c
@@ -820,18 +820,16 @@ static int ov02a10_check_hwcfg(struct device *dev, struct ov02a10 *ov02a10)
 	if (!ep)
 		return -ENXIO;
 
+	/* Optional indication of MIPI clock voltage unit */
+	if (!fwnode_property_read_u32(ep, "ovti,mipi-clock-voltage",
+				      &clk_volt))
+		ov02a10->mipi_clock_voltage = clk_volt;
+
 	ret = v4l2_fwnode_endpoint_alloc_parse(ep, &bus_cfg);
 	fwnode_handle_put(ep);
 	if (ret)
 		return ret;
 
-	/* Optional indication of MIPI clock voltage unit */
-	ret = fwnode_property_read_u32(ep, "ovti,mipi-clock-voltage",
-				       &clk_volt);
-
-	if (!ret)
-		ov02a10->mipi_clock_voltage = clk_volt;
-
 	for (i = 0; i < ARRAY_SIZE(link_freq_menu_items); i++) {
 		for (j = 0; j < bus_cfg.nr_of_link_frequencies; j++) {
 			if (link_freq_menu_items[i] ==
diff --git a/drivers/media/i2c/ov7740.c b/drivers/media/i2c/ov7740.c
index c2e02f1918169..b4e14171556f9 100644
--- a/drivers/media/i2c/ov7740.c
+++ b/drivers/media/i2c/ov7740.c
@@ -1116,18 +1116,14 @@ static void ov7740_remove(struct i2c_client *client)
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev);
 
-	mutex_destroy(&ov7740->mutex);
-	v4l2_ctrl_handler_free(ov7740->subdev.ctrl_handler);
-	media_entity_cleanup(&ov7740->subdev.entity);
 	v4l2_async_unregister_subdev(sd);
+	media_entity_cleanup(&ov7740->subdev.entity);
 	ov7740_free_controls(ov7740);
 
-	pm_runtime_get_sync(&client->dev);
 	pm_runtime_disable(&client->dev);
+	if (!pm_runtime_status_suspended(&client->dev))
+		ov7740_set_power(ov7740, 0);
 	pm_runtime_set_suspended(&client->dev);
-	pm_runtime_put_noidle(&client->dev);
-
-	ov7740_set_power(ov7740, 0);
 }
 
 static int __maybe_unused ov7740_runtime_suspend(struct device *dev)
diff --git a/drivers/media/i2c/ov8865.c b/drivers/media/i2c/ov8865.c
index 8c9cd769fa014..289a34a3eeb17 100644
--- a/drivers/media/i2c/ov8865.c
+++ b/drivers/media/i2c/ov8865.c
@@ -2608,7 +2608,7 @@ static int ov8865_s_stream(struct v4l2_subdev *subdev, int enable)
 {
 	struct ov8865_sensor *sensor = ov8865_subdev_sensor(subdev);
 	struct ov8865_state *state = &sensor->state;
-	int ret;
+	int ret = 0;
 
 	if (enable) {
 		ret = pm_runtime_resume_and_get(sensor->dev);
@@ -2617,18 +2617,32 @@ static int ov8865_s_stream(struct v4l2_subdev *subdev, int enable)
 	}
 
 	mutex_lock(&sensor->mutex);
-	ret = ov8865_sw_standby(sensor, !enable);
-	mutex_unlock(&sensor->mutex);
 
-	if (ret)
-		return ret;
+	/*
+	 * The sensor may have been kept powered by something else (e.g. the
+	 * VCM's runtime PM device link on IPU3 platforms), in which case
+	 * runtime resume did not run and the hardware may still be
+	 * configured for a previous mode. Always program the negotiated
+	 * configuration on stream start.
+	 */
+	if (enable) {
+		ret = ov8865_sensor_init(sensor);
+		if (!ret)
+			ret = __v4l2_ctrl_handler_setup(&sensor->ctrls.handler);
+	}
 
-	state->streaming = !!enable;
+	if (!ret)
+		ret = ov8865_sw_standby(sensor, !enable);
+
+	mutex_unlock(&sensor->mutex);
 
-	if (!enable)
+	if (ret || !enable)
 		pm_runtime_put(sensor->dev);
 
-	return 0;
+	if (!ret)
+		state->streaming = enable;
+
+	return ret;
 }
 
 static const struct v4l2_subdev_video_ops ov8865_subdev_video_ops = {
@@ -2915,15 +2929,15 @@ static int ov8865_resume(struct device *dev)
 	if (ret)
 		goto complete;
 
-	ret = ov8865_sensor_init(sensor);
-	if (ret)
-		goto error_power;
+	if (state->streaming) {
+		ret = ov8865_sensor_init(sensor);
+		if (ret)
+			goto error_power;
 
-	ret = __v4l2_ctrl_handler_setup(&sensor->ctrls.handler);
-	if (ret)
-		goto error_power;
+		ret = __v4l2_ctrl_handler_setup(&sensor->ctrls.handler);
+		if (ret)
+			goto error_power;
 
-	if (state->streaming) {
 		ret = ov8865_sw_standby(sensor, false);
 		if (ret)
 			goto error_power;
diff --git a/drivers/media/i2c/ov9282.c b/drivers/media/i2c/ov9282.c
index 5b6f897a74fcd..3ba630ec4f2a2 100644
--- a/drivers/media/i2c/ov9282.c
+++ b/drivers/media/i2c/ov9282.c
@@ -197,7 +197,6 @@ struct ov9282_mode {
  * @noncontinuous_clock: Selection of CSI2 noncontinuous clock mode
  * @cur_mode: Pointer to current selected sensor mode
  * @code: Mbus code currently selected
- * @mutex: Mutex for serializing sensor controls
  */
 struct ov9282 {
 	struct device *dev;
diff --git a/drivers/media/i2c/rdacm21.c b/drivers/media/i2c/rdacm21.c
index bcab462708c70..ece8a410e7ced 100644
--- a/drivers/media/i2c/rdacm21.c
+++ b/drivers/media/i2c/rdacm21.c
@@ -588,10 +588,12 @@ static int rdacm21_probe(struct i2c_client *client)
 
 	ret = v4l2_async_register_subdev(&dev->sd);
 	if (ret)
-		goto error_free_ctrls;
+		goto error_entity_cleanup;
 
 	return 0;
 
+error_entity_cleanup:
+	media_entity_cleanup(&dev->sd.entity);
 error_free_ctrls:
 	v4l2_ctrl_handler_free(&dev->ctrls);
 error:
@@ -606,6 +608,7 @@ static void rdacm21_remove(struct i2c_client *client)
 
 	v4l2_async_unregister_subdev(&dev->sd);
 	v4l2_ctrl_handler_free(&dev->ctrls);
+	media_entity_cleanup(&dev->sd.entity);
 	i2c_unregister_device(dev->isp);
 }
 
diff --git a/drivers/media/i2c/vd55g1.c b/drivers/media/i2c/vd55g1.c
index 78d18c0281541..c4142d771e7c8 100644
--- a/drivers/media/i2c/vd55g1.c
+++ b/drivers/media/i2c/vd55g1.c
@@ -29,11 +29,7 @@
 
 /* Register Map */
 #define VD55G1_REG_MODEL_ID				CCI_REG32_LE(0x0000)
-#define VD55G1_MODEL_ID_VD55G1				0x53354731 /* Mono */
-#define VD55G1_MODEL_ID_VD65G4				0x53354733 /* RGB */
-#define VD55G1_REG_REVISION				CCI_REG16_LE(0x0004)
-#define VD55G1_REVISION_CCB				0x2020
-#define VD55G1_REVISION_BAYER				0x3030
+#define VD55G1_REG_COLOR_VERSION			CCI_REG32_LE(0x0670)
 #define VD55G1_REG_FWPATCH_REVISION			CCI_REG16_LE(0x0012)
 #define VD55G1_REG_FWPATCH_START_ADDR			CCI_REG8(0x2000)
 #define VD55G1_REG_SYSTEM_FSM				CCI_REG8(0x001c)
@@ -60,7 +56,10 @@
 #define VD55G1_PATGEN_ENABLE				BIT(0)
 #define VD55G1_REG_MANUAL_ANALOG_GAIN			CCI_REG8(0x0501)
 #define VD55G1_REG_MANUAL_COARSE_EXPOSURE		CCI_REG16_LE(0x0502)
-#define VD55G1_REG_MANUAL_DIGITAL_GAIN			CCI_REG16_LE(0x0504)
+#define VD55G1_REG_MANUAL_DIGITAL_GAIN_CH0		CCI_REG16_LE(0x0504)
+#define VD55G1_REG_MANUAL_DIGITAL_GAIN_CH1		CCI_REG16_LE(0x0506)
+#define VD55G1_REG_MANUAL_DIGITAL_GAIN_CH2		CCI_REG16_LE(0x0508)
+#define VD55G1_REG_MANUAL_DIGITAL_GAIN_CH3		CCI_REG16_LE(0x050a)
 #define VD55G1_REG_APPLIED_COARSE_EXPOSURE		CCI_REG16_LE(0x00e8)
 #define VD55G1_REG_APPLIED_ANALOG_GAIN			CCI_REG16_LE(0x00ea)
 #define VD55G1_REG_APPLIED_DIGITAL_GAIN			CCI_REG16_LE(0x00ec)
@@ -114,9 +113,8 @@
 
 #define VD55G1_WIDTH					804
 #define VD55G1_HEIGHT					704
-#define VD55G1_MODE_DEF					0
+#define VD55G1_MODE_IDX_DEF				0
 #define VD55G1_NB_GPIOS					4
-#define VD55G1_MBUS_CODE_DEF				0
 #define VD55G1_DGAIN_DEF				256
 #define VD55G1_AGAIN_DEF				19
 #define VD55G1_EXPO_MAX_TERM				64
@@ -135,8 +133,39 @@
 #define VD55G1_MIPI_RATE_MIN				(250 * MEGA)
 #define VD55G1_MIPI_RATE_MAX				(1200 * MEGA)
 
-#define VD55G1_MODEL_ID_NAME(id) \
-	((id) == VD55G1_MODEL_ID_VD55G1 ? "vd55g1" : "vd65g4")
+enum vd55g1_model_id {
+	VD55G1_MODEL_ID_2 = 0x53354731,
+	VD55G1_MODEL_ID_3 = 0x53354733,
+};
+
+enum vd55g1_color_version {
+	VD55G1_COLOR_VERSION_MONO,
+	VD55G1_COLOR_VERSION_BAYER,
+};
+
+struct vd55g1_version {
+	char *name;
+	enum vd55g1_model_id id;
+	enum vd55g1_color_version color;
+};
+
+static const struct vd55g1_version vd55g1_versions[] = {
+	{
+		.name  = "vd55g1",
+		.id    = VD55G1_MODEL_ID_2,
+		.color = VD55G1_COLOR_VERSION_MONO,
+	},
+	{
+		.name  = "vd55g4",
+		.id    = VD55G1_MODEL_ID_3,
+		.color = VD55G1_COLOR_VERSION_MONO,
+	},
+	{
+		.name  = "vd65g4",
+		.id    = VD55G1_MODEL_ID_3,
+		.color = VD55G1_COLOR_VERSION_BAYER,
+	},
+};
 
 static const u8 vd55g1_patch_array[] = {
 	0x44, 0x03, 0x09, 0x02, 0xe6, 0x01, 0x42, 0x00, 0xea, 0x01, 0x42, 0x00,
@@ -532,7 +561,7 @@ struct vd55g1_vblank_limits {
 
 struct vd55g1 {
 	struct device *dev;
-	unsigned int id;
+	const struct vd55g1_version *version;
 	struct v4l2_subdev sd;
 	struct media_pad pad;
 	struct regulator_bulk_data supplies[ARRAY_SIZE(vd55g1_supply_name)];
@@ -623,20 +652,28 @@ static unsigned int vd55g1_get_fmt_data_type(u32 code)
 
 static u32 vd55g1_get_fmt_code(struct vd55g1 *sensor, u32 code)
 {
+	u32 fallback_code;
 	unsigned int i, j;
 
-	if (sensor->id == VD55G1_MODEL_ID_VD55G1)
-		return code;
-
-	for (i = 0; i < ARRAY_SIZE(vd55g1_mbus_formats_bayer); i++) {
-		for (j = 0; j < ARRAY_SIZE(vd55g1_mbus_formats_bayer[i]); j++) {
-			if (vd55g1_mbus_formats_bayer[i][j] == code)
-				goto adapt_bayer_pattern;
+	if (sensor->version->color == VD55G1_COLOR_VERSION_MONO) {
+		fallback_code = vd55g1_mbus_formats_mono[0];
+		for (i = 0; i < ARRAY_SIZE(vd55g1_mbus_formats_mono); i++)
+			if (vd55g1_mbus_formats_mono[i] == code)
+				return code;
+	} else {
+		fallback_code = vd55g1_mbus_formats_bayer[0][0];
+		for (i = 0; i < ARRAY_SIZE(vd55g1_mbus_formats_bayer); i++) {
+			for (j = 0;
+			     j < ARRAY_SIZE(vd55g1_mbus_formats_bayer[i]);
+			     j++) {
+				if (vd55g1_mbus_formats_bayer[i][j] == code)
+					goto adapt_bayer_pattern;
+			}
 		}
 	}
-	dev_warn(sensor->dev, "Unsupported mbus format\n");
+	dev_warn(sensor->dev, "Unsupported mbus format: 0x%x\n", code);
 
-	return code;
+	return fallback_code;
 
 adapt_bayer_pattern:
 	j = 0;
@@ -850,9 +887,16 @@ static int vd55g1_update_expo_cluster(struct vd55g1 *sensor, bool is_auto)
 		vd55g1_write(sensor, VD55G1_REG_MANUAL_ANALOG_GAIN,
 			     sensor->again_ctrl->val, &ret);
 
-	if (!is_auto && sensor->dgain_ctrl->is_new)
-		vd55g1_write(sensor, VD55G1_REG_MANUAL_DIGITAL_GAIN,
+	if (!is_auto && sensor->dgain_ctrl->is_new) {
+		vd55g1_write(sensor, VD55G1_REG_MANUAL_DIGITAL_GAIN_CH0,
 			     sensor->dgain_ctrl->val, &ret);
+		vd55g1_write(sensor, VD55G1_REG_MANUAL_DIGITAL_GAIN_CH1,
+			     sensor->dgain_ctrl->val, &ret);
+		vd55g1_write(sensor, VD55G1_REG_MANUAL_DIGITAL_GAIN_CH2,
+			     sensor->dgain_ctrl->val, &ret);
+		vd55g1_write(sensor, VD55G1_REG_MANUAL_DIGITAL_GAIN_CH3,
+			     sensor->dgain_ctrl->val, &ret);
+	}
 
 	return ret;
 }
@@ -1173,8 +1217,8 @@ static int vd55g1_patch(struct vd55g1 *sensor)
 	u64 patch;
 	int ret = 0;
 
-	/* vd55g1 needs a patch while vd65g4 does not */
-	if (sensor->id == VD55G1_MODEL_ID_VD55G1) {
+	/* Version 2 needs a patch while version 3 does not */
+	if (sensor->version->id == VD55G1_MODEL_ID_2) {
 		vd55g1_write_array(sensor, VD55G1_REG_FWPATCH_START_ADDR,
 				   sizeof(vd55g1_patch_array),
 				   vd55g1_patch_array, &ret);
@@ -1246,7 +1290,7 @@ static int vd55g1_enum_mbus_code(struct v4l2_subdev *sd,
 	struct vd55g1 *sensor = to_vd55g1(sd);
 	u32 base_code;
 
-	if (sensor->id == VD55G1_MODEL_ID_VD55G1) {
+	if (sensor->version->color != VD55G1_COLOR_VERSION_BAYER) {
 		if (code->index >= ARRAY_SIZE(vd55g1_mbus_formats_mono))
 			return -EINVAL;
 		base_code = vd55g1_mbus_formats_mono[code->index];
@@ -1347,6 +1391,7 @@ static int vd55g1_init_state(struct v4l2_subdev *sd,
 {
 	struct vd55g1 *sensor = to_vd55g1(sd);
 	struct v4l2_subdev_format fmt = { 0 };
+	int code;
 	struct v4l2_subdev_route routes[] = {
 		{ .flags = V4L2_SUBDEV_ROUTE_FL_ACTIVE }
 	};
@@ -1361,9 +1406,13 @@ static int vd55g1_init_state(struct v4l2_subdev *sd,
 	if (ret)
 		return ret;
 
-	vd55g1_update_pad_fmt(sensor, &vd55g1_supported_modes[VD55G1_MODE_DEF],
-			      vd55g1_get_fmt_code(sensor, VD55G1_MBUS_CODE_DEF),
-			      &fmt.format);
+	if (sensor->version->color != VD55G1_COLOR_VERSION_BAYER)
+		code = vd55g1_mbus_formats_mono[0];
+	else
+		code = vd55g1_mbus_formats_bayer[0][0];
+	fmt.format.code = vd55g1_get_fmt_code(sensor, code);
+	fmt.format.width = vd55g1_supported_modes[VD55G1_MODE_IDX_DEF].width;
+	fmt.format.height = vd55g1_supported_modes[VD55G1_MODE_IDX_DEF].height;
 
 	return vd55g1_set_pad_fmt(sd, sd_state, &fmt);
 }
@@ -1644,38 +1693,46 @@ static int vd55g1_init_ctrls(struct vd55g1 *sensor)
 	return ret;
 }
 
+static const struct vd55g1_version *
+vd55g1_get_version(enum vd55g1_model_id id,
+		   enum vd55g1_color_version color)
+{
+	for (unsigned int i = 0; i < ARRAY_SIZE(vd55g1_versions); i++) {
+		if (vd55g1_versions[i].id == id &&
+		    vd55g1_versions[i].color == color)
+			return &vd55g1_versions[i];
+	}
+
+	return NULL;
+}
+
 static int vd55g1_detect(struct vd55g1 *sensor)
 {
-	unsigned int dt_id = (uintptr_t)device_get_match_data(sensor->dev);
-	u64 rev, id;
-	int ret;
+	const struct vd55g1_version *dt_version =
+		device_get_match_data(sensor->dev);
+	const struct vd55g1_version *version;
+	u64 color, id;
+	int ret = 0;
 
-	ret = vd55g1_read(sensor, VD55G1_REG_MODEL_ID, &id, NULL);
+	vd55g1_read(sensor, VD55G1_REG_MODEL_ID, &id, &ret);
+	vd55g1_read(sensor, VD55G1_REG_COLOR_VERSION, &color, &ret);
 	if (ret)
 		return ret;
 
-	if (id != VD55G1_MODEL_ID_VD55G1 && id != VD55G1_MODEL_ID_VD65G4) {
-		dev_warn(sensor->dev, "Unsupported sensor id 0x%x\n",
-			 (u32)id);
+	version = vd55g1_get_version(id, color);
+	if (!version) {
+		dev_warn(sensor->dev, "Unsupported sensor version, expected %s\n",
+			 dt_version->name);
 		return -ENODEV;
 	}
-	if (id != dt_id) {
-		dev_err(sensor->dev, "Probed sensor %s and device tree definition (%s) mismatch",
-			VD55G1_MODEL_ID_NAME(id), VD55G1_MODEL_ID_NAME(dt_id));
+	if (version->id != dt_version->id ||
+	    version->color != dt_version->color) {
+		dev_err(sensor->dev, "Probed sensor version %s and device tree definition %s mismatch",
+			version->name, dt_version->name);
 		return -ENODEV;
 	}
-	sensor->id = id;
-
-	ret = vd55g1_read(sensor, VD55G1_REG_REVISION, &rev, NULL);
-	if (ret)
-		return ret;
 
-	if ((id == VD55G1_MODEL_ID_VD55G1 && rev != VD55G1_REVISION_CCB) &&
-	    (id == VD55G1_MODEL_ID_VD65G4 && rev != VD55G1_REVISION_BAYER)) {
-		dev_err(sensor->dev, "Unsupported sensor revision 0x%x for sensor %s\n",
-			(u16)rev, VD55G1_MODEL_ID_NAME(id));
-		return -ENODEV;
-	}
+	sensor->version = version;
 
 	return 0;
 }
@@ -2033,8 +2090,9 @@ static void vd55g1_remove(struct i2c_client *client)
 }
 
 static const struct of_device_id vd55g1_dt_ids[] = {
-	{ .compatible = "st,vd55g1", .data = (void *)VD55G1_MODEL_ID_VD55G1 },
-	{ .compatible = "st,vd65g4", .data = (void *)VD55G1_MODEL_ID_VD65G4 },
+	{ .compatible = "st,vd55g1", .data = (void *)&vd55g1_versions[0] },
+	{ .compatible = "st,vd55g4", .data = (void *)&vd55g1_versions[1] },
+	{ .compatible = "st,vd65g4", .data = (void *)&vd55g1_versions[2] },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, vd55g1_dt_ids);
diff --git a/drivers/media/i2c/video-i2c.c b/drivers/media/i2c/video-i2c.c
index 6b50fb422a614..4811a04895246 100644
--- a/drivers/media/i2c/video-i2c.c
+++ b/drivers/media/i2c/video-i2c.c
@@ -522,8 +522,12 @@ static int start_streaming(struct vb2_queue *vq, unsigned int count)
 	data->kthread_vid_cap = kthread_run(video_i2c_thread_vid_cap, data,
 					    "%s-vid-cap", data->v4l2_dev.name);
 	ret = PTR_ERR_OR_ZERO(data->kthread_vid_cap);
-	if (!ret)
-		return 0;
+	if (ret) {
+		data->kthread_vid_cap = NULL;
+		goto error_rpm_put;
+	}
+
+	return 0;
 
 error_rpm_put:
 	pm_runtime_put_autosuspend(dev);
diff --git a/drivers/media/pci/bt8xx/bttv-driver.c b/drivers/media/pci/bt8xx/bttv-driver.c
index 6634692082712..c631b8bbd3863 100644
--- a/drivers/media/pci/bt8xx/bttv-driver.c
+++ b/drivers/media/pci/bt8xx/bttv-driver.c
@@ -3563,12 +3563,12 @@ static int __maybe_unused bttv_resume(struct device *dev)
 }
 
 static const struct pci_device_id bttv_pci_tbl[] = {
-	{PCI_VDEVICE(BROOKTREE, PCI_DEVICE_ID_BT848), 0},
-	{PCI_VDEVICE(BROOKTREE, PCI_DEVICE_ID_BT849), 0},
-	{PCI_VDEVICE(BROOKTREE, PCI_DEVICE_ID_BT878), 0},
-	{PCI_VDEVICE(BROOKTREE, PCI_DEVICE_ID_BT879), 0},
-	{PCI_VDEVICE(BROOKTREE, PCI_DEVICE_ID_FUSION879), 0},
-	{0,}
+	{ PCI_VDEVICE(BROOKTREE, PCI_DEVICE_ID_BT848) },
+	{ PCI_VDEVICE(BROOKTREE, PCI_DEVICE_ID_BT849) },
+	{ PCI_VDEVICE(BROOKTREE, PCI_DEVICE_ID_BT878) },
+	{ PCI_VDEVICE(BROOKTREE, PCI_DEVICE_ID_BT879) },
+	{ PCI_VDEVICE(BROOKTREE, PCI_DEVICE_ID_FUSION879) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, bttv_pci_tbl);
diff --git a/drivers/media/pci/cobalt/cobalt-alsa-main.c b/drivers/media/pci/cobalt/cobalt-alsa-main.c
index 7bb7f13c70c0d..9ed547cd40af5 100644
--- a/drivers/media/pci/cobalt/cobalt-alsa-main.c
+++ b/drivers/media/pci/cobalt/cobalt-alsa-main.c
@@ -135,7 +135,6 @@ int cobalt_alsa_init(struct cobalt_stream *s)
 err_exit_free:
 	if (sc != NULL)
 		snd_card_free(sc);
-	kfree(cobsc);
 err_exit:
 	return ret;
 }
diff --git a/drivers/media/pci/cobalt/cobalt-driver.c b/drivers/media/pci/cobalt/cobalt-driver.c
index 9b9f69ff40167..7b1ca1238c8d9 100644
--- a/drivers/media/pci/cobalt/cobalt-driver.c
+++ b/drivers/media/pci/cobalt/cobalt-driver.c
@@ -26,9 +26,8 @@
 
 /* add your revision and whatnot here */
 static const struct pci_device_id cobalt_pci_tbl[] = {
-	{PCI_VENDOR_ID_CISCO, PCI_DEVICE_ID_COBALT,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{0,}
+	{ PCI_VDEVICE(CISCO, PCI_DEVICE_ID_COBALT) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, cobalt_pci_tbl);
diff --git a/drivers/media/pci/cx18/cx18-driver.c b/drivers/media/pci/cx18/cx18-driver.c
index f778f79b921d6..214fac7af61e5 100644
--- a/drivers/media/pci/cx18/cx18-driver.c
+++ b/drivers/media/pci/cx18/cx18-driver.c
@@ -40,9 +40,8 @@ EXPORT_SYMBOL(cx18_ext_init);
 
 /* add your revision and whatnot here */
 static const struct pci_device_id cx18_pci_tbl[] = {
-	{PCI_VENDOR_ID_CX, PCI_DEVICE_ID_CX23418,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{0,}
+	{ PCI_VDEVICE(CX, PCI_DEVICE_ID_CX23418) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, cx18_pci_tbl);
diff --git a/drivers/media/pci/cx18/cx18-vbi.c b/drivers/media/pci/cx18/cx18-vbi.c
index 8dc4ce3259353..3a8430f3259eb 100644
--- a/drivers/media/pci/cx18/cx18-vbi.c
+++ b/drivers/media/pci/cx18/cx18-vbi.c
@@ -135,7 +135,7 @@ static u32 compress_sliced_buf(struct cx18 *cx, u8 *buf, u32 size,
 				    : VBI_HBLANK_SAMPLES_50HZ;
 
 	/* find the first valid line */
-	for (i = hdr_size, buf += hdr_size; i < size; i++, buf++) {
+	for (i = hdr_size, buf += hdr_size; i + 3 < size; i++, buf++) {
 		if (buf[0] == 0xff && !buf[1] && !buf[2] &&
 		    (buf[3] == sliced_vbi_eav_rp[0] ||
 		     buf[3] == sliced_vbi_eav_rp[1]))
diff --git a/drivers/media/pci/cx23885/cx23885-core.c b/drivers/media/pci/cx23885/cx23885-core.c
index 9b92e8db494c5..5fb26285e4af7 100644
--- a/drivers/media/pci/cx23885/cx23885-core.c
+++ b/drivers/media/pci/cx23885/cx23885-core.c
@@ -2279,16 +2279,10 @@ static int __maybe_unused cx23885_resume(struct device *dev_d)
 static const struct pci_device_id cx23885_pci_tbl[] = {
 	{
 		/* CX23885 */
-		.vendor       = 0x14f1,
-		.device       = 0x8852,
-		.subvendor    = PCI_ANY_ID,
-		.subdevice    = PCI_ANY_ID,
+		PCI_DEVICE(0x14f1, 0x8852),
 	}, {
 		/* CX23887 Rev 2 */
-		.vendor       = 0x14f1,
-		.device       = 0x8880,
-		.subvendor    = PCI_ANY_ID,
-		.subdevice    = PCI_ANY_ID,
+		PCI_DEVICE(0x14f1, 0x8880),
 	}, {
 		/* --- end of list --- */
 	}
diff --git a/drivers/media/pci/cx25821/cx25821-alsa.c b/drivers/media/pci/cx25821/cx25821-alsa.c
index 4fdc59aaed8ee..f963ae79f685b 100644
--- a/drivers/media/pci/cx25821/cx25821-alsa.c
+++ b/drivers/media/pci/cx25821/cx25821-alsa.c
@@ -681,8 +681,8 @@ static int snd_cx25821_pcm(struct cx25821_audio_dev *chip, int device,
  */
 
 static const struct pci_device_id __maybe_unused cx25821_audio_pci_tbl[] = {
-	{0x14f1, 0x0920, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{0,}
+	{ PCI_DEVICE(0x14f1, 0x0920) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, cx25821_audio_pci_tbl);
diff --git a/drivers/media/pci/cx25821/cx25821-core.c b/drivers/media/pci/cx25821/cx25821-core.c
index 5acb1dc00ae88..f883a779b5706 100644
--- a/drivers/media/pci/cx25821/cx25821-core.c
+++ b/drivers/media/pci/cx25821/cx25821-core.c
@@ -1347,16 +1347,10 @@ static void cx25821_finidev(struct pci_dev *pci_dev)
 static const struct pci_device_id cx25821_pci_tbl[] = {
 	{
 		/* CX25821 Athena */
-		.vendor = 0x14f1,
-		.device = 0x8210,
-		.subvendor = 0x14f1,
-		.subdevice = 0x0920,
+		PCI_DEVICE_SUB(0x14f1, 0x8210, 0x14f1, 0x0920),
 	}, {
 		/* CX25821 No Brand */
-		.vendor = 0x14f1,
-		.device = 0x8210,
-		.subvendor = 0x0000,
-		.subdevice = 0x0000,
+		PCI_DEVICE_SUB(0x14f1, 0x8210, 0x0000, 0x0000),
 	}, {
 		/* --- end of list --- */
 	}
diff --git a/drivers/media/pci/cx88/cx88-alsa.c b/drivers/media/pci/cx88/cx88-alsa.c
index dce041a5e47a2..d2534f90cfbad 100644
--- a/drivers/media/pci/cx88/cx88-alsa.c
+++ b/drivers/media/pci/cx88/cx88-alsa.c
@@ -809,9 +809,9 @@ static const struct snd_kcontrol_new snd_cx88_alc_switch = {
  */
 
 static const struct pci_device_id cx88_audio_pci_tbl[] = {
-	{0x14f1, 0x8801, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{0x14f1, 0x8811, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{0, }
+	{ PCI_DEVICE(0x14f1, 0x8801) },
+	{ PCI_DEVICE(0x14f1, 0x8811) },
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, cx88_audio_pci_tbl);
 
diff --git a/drivers/media/pci/cx88/cx88-mpeg.c b/drivers/media/pci/cx88/cx88-mpeg.c
index 676160e9554da..a399a9bf3b92e 100644
--- a/drivers/media/pci/cx88/cx88-mpeg.c
+++ b/drivers/media/pci/cx88/cx88-mpeg.c
@@ -787,10 +787,7 @@ static void cx8802_remove(struct pci_dev *pci_dev)
 
 static const struct pci_device_id cx8802_pci_tbl[] = {
 	{
-		.vendor       = 0x14f1,
-		.device       = 0x8802,
-		.subvendor    = PCI_ANY_ID,
-		.subdevice    = PCI_ANY_ID,
+		PCI_DEVICE(0x14f1, 0x8802),
 	}, {
 		/* --- end of list --- */
 	}
diff --git a/drivers/media/pci/cx88/cx88-video.c b/drivers/media/pci/cx88/cx88-video.c
index c78b156c5cda0..eaa46a2f92e74 100644
--- a/drivers/media/pci/cx88/cx88-video.c
+++ b/drivers/media/pci/cx88/cx88-video.c
@@ -1610,10 +1610,7 @@ static int __maybe_unused cx8800_resume(struct device *dev_d)
 
 static const struct pci_device_id cx8800_pci_tbl[] = {
 	{
-		.vendor       = 0x14f1,
-		.device       = 0x8800,
-		.subvendor    = PCI_ANY_ID,
-		.subdevice    = PCI_ANY_ID,
+		PCI_DEVICE(0x14f1, 0x8800),
 	}, {
 		/* --- end of list --- */
 	}
diff --git a/drivers/media/pci/ddbridge/ddbridge-main.c b/drivers/media/pci/ddbridge/ddbridge-main.c
index 363badab7cf07..248ace29e5c87 100644
--- a/drivers/media/pci/ddbridge/ddbridge-main.c
+++ b/drivers/media/pci/ddbridge/ddbridge-main.c
@@ -269,7 +269,7 @@ static const struct pci_device_id ddb_id_table[] = {
 	DDB_DEVICE_ANY(0x0323),
 	DDB_DEVICE_ANY(0x0328),
 	DDB_DEVICE_ANY(0x0329),
-	{0}
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, ddb_id_table);
diff --git a/drivers/media/pci/ddbridge/ddbridge.h b/drivers/media/pci/ddbridge/ddbridge.h
index f01ecdb0b6278..cf50898f9a929 100644
--- a/drivers/media/pci/ddbridge/ddbridge.h
+++ b/drivers/media/pci/ddbridge/ddbridge.h
@@ -14,7 +14,7 @@
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/dvb/ca.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
diff --git a/drivers/media/pci/dm1105/dm1105.c b/drivers/media/pci/dm1105/dm1105.c
index 80d2e143384b4..ae4544042a743 100644
--- a/drivers/media/pci/dm1105/dm1105.c
+++ b/drivers/media/pci/dm1105/dm1105.c
@@ -1209,15 +1209,9 @@ static void dm1105_remove(struct pci_dev *pdev)
 
 static const struct pci_device_id dm1105_id_table[] = {
 	{
-		.vendor = PCI_VENDOR_ID_TRIGEM,
-		.device = PCI_DEVICE_ID_DM1105,
-		.subvendor = PCI_ANY_ID,
-		.subdevice = PCI_ANY_ID,
+		PCI_VDEVICE(TRIGEM, PCI_DEVICE_ID_DM1105),
 	}, {
-		.vendor = PCI_VENDOR_ID_AXESS,
-		.device = PCI_DEVICE_ID_DM05,
-		.subvendor = PCI_ANY_ID,
-		.subdevice = PCI_ANY_ID,
+		PCI_VDEVICE(AXESS, PCI_DEVICE_ID_DM05),
 	}, {
 		/* empty */
 	},
diff --git a/drivers/media/pci/dt3155/dt3155.c b/drivers/media/pci/dt3155/dt3155.c
index 7bddcbba4cf1b..660aab6b62e0c 100644
--- a/drivers/media/pci/dt3155/dt3155.c
+++ b/drivers/media/pci/dt3155/dt3155.c
@@ -577,8 +577,8 @@ static void dt3155_remove(struct pci_dev *pdev)
 }
 
 static const struct pci_device_id pci_ids[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, DT3155_DEVICE_ID) },
-	{ 0, /* zero marks the end */ },
+	{ PCI_VDEVICE(INTEL, DT3155_DEVICE_ID) },
+	{ /* zero marks the end */ },
 };
 MODULE_DEVICE_TABLE(pci, pci_ids);
 
diff --git a/drivers/media/pci/hws/hws_v4l2_ioctl.h b/drivers/media/pci/hws/hws_v4l2_ioctl.h
index 53044f78d6fa3..9a81e940e33e5 100644
--- a/drivers/media/pci/hws/hws_v4l2_ioctl.h
+++ b/drivers/media/pci/hws/hws_v4l2_ioctl.h
@@ -17,14 +17,10 @@ int hws_vidioc_g_parm(struct file *file, void *fh, struct v4l2_streamparm *setfp
 int hws_vidioc_enum_input(struct file *file, void *priv, struct v4l2_input *i);
 int hws_vidioc_g_input(struct file *file, void *priv, unsigned int *i);
 int hws_vidioc_s_input(struct file *file, void *priv, unsigned int i);
-int hws_vidioc_g_ctrl(struct file *file, void *fh, struct v4l2_control *a);
-int hws_vidioc_s_ctrl(struct file *file, void *fh, struct v4l2_control *a);
 int hws_vidioc_dv_timings_cap(struct file *file, void *fh,
 			      struct v4l2_dv_timings_cap *cap);
 int hws_vidioc_s_dv_timings(struct file *file, void *fh,
 			    struct v4l2_dv_timings *timings);
-
-int hws_vidioc_queryctrl(struct file *file, void *fh, struct v4l2_queryctrl *a);
 int hws_vidioc_g_dv_timings(struct file *file, void *fh,
 			    struct v4l2_dv_timings *timings);
 int hws_vidioc_enum_dv_timings(struct file *file, void *fh,
diff --git a/drivers/media/pci/intel/ipu-bridge.c b/drivers/media/pci/intel/ipu-bridge.c
index 88581a4c081d6..42a309e7218eb 100644
--- a/drivers/media/pci/intel/ipu-bridge.c
+++ b/drivers/media/pci/intel/ipu-bridge.c
@@ -97,6 +97,8 @@ static const struct ipu_sensor_config ipu_supported_sensors[] = {
 	IPU_SENSOR_CONFIG("OVTI8856", 3, 180000000, 360000000, 720000000),
 	/* Sony IMX471 */
 	IPU_SENSOR_CONFIG("SONY471A", 1, 200000000),
+	/* Sony IMX471 (found on Lenovo X1 Carbon G14) */
+	IPU_SENSOR_CONFIG("TBE20A0", 1, 200000000),
 	/* Toshiba T4KA3 */
 	IPU_SENSOR_CONFIG("XMCC0003", 1, 321468000),
 };
@@ -134,6 +136,52 @@ static const struct dmi_system_id upside_down_sensor_dmi_ids[] = {
 		},
 		.driver_data = "OVTI02C1",
 	},
+	{
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "XPS 16 (Dell 16 Premium) DA16250"),
+		},
+		.driver_data = "OVTI02C1",
+	},
+	/*
+	 * The first four characters of DMI_BOARD_NAME identify the Lenovo
+	 * machine type/model. For example, a DMI_BOARD_NAME starting with
+	 * "21Q6" indicates a ThinkPad X9-15.
+	 *
+	 * Reference: https://psref.lenovo.com/
+	 */
+	{
+		/* Lenovo X9-14 */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_BOARD_NAME, "21QA"),
+		},
+		.driver_data = "SONY471A",
+	},
+	{
+		/* Lenovo X9-14 */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_BOARD_NAME, "21QB"),
+		},
+		.driver_data = "SONY471A",
+	},
+	{
+		/* Lenovo X9-15 */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_BOARD_NAME, "21Q6"),
+		},
+		.driver_data = "SONY471A",
+	},
+	{
+		/* Lenovo X9-15 */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_BOARD_NAME, "21Q7"),
+		},
+		.driver_data = "SONY471A",
+	},
 	{} /* Terminating entry */
 };
 
@@ -171,6 +219,7 @@ static const struct acpi_device_id ivsc_acpi_ids[] = {
 	{ "INTC10DE" }, /* LNL */
 	{ "INTC10E0" }, /* ARL */
 	{ "INTC10E1" }, /* PTL */
+	{ "INTC10FA" }, /* NVL */
 };
 
 static struct acpi_device *ipu_bridge_get_ivsc_acpi_dev(struct acpi_device *adev)
@@ -180,8 +229,8 @@ static struct acpi_device *ipu_bridge_get_ivsc_acpi_dev(struct acpi_device *adev
 	for (i = 0; i < ARRAY_SIZE(ivsc_acpi_ids); i++) {
 		const struct acpi_device_id *acpi_id = &ivsc_acpi_ids[i];
 		struct acpi_device *consumer, *ivsc_adev;
-
 		acpi_handle handle = acpi_device_handle(ACPI_PTR(adev));
+
 		for_each_acpi_dev_match(ivsc_adev, acpi_id->id, NULL, -1)
 			/* camera sensor depends on IVSC in DSDT if exist */
 			for_each_acpi_consumer_dev(ivsc_adev, consumer)
diff --git a/drivers/media/pci/intel/ipu3/ipu3-cio2.c b/drivers/media/pci/intel/ipu3/ipu3-cio2.c
index 986b9afd7cb5b..eb1824ee86fdd 100644
--- a/drivers/media/pci/intel/ipu3/ipu3-cio2.c
+++ b/drivers/media/pci/intel/ipu3/ipu3-cio2.c
@@ -1961,7 +1961,7 @@ static const struct dev_pm_ops cio2_pm_ops = {
 };
 
 static const struct pci_device_id cio2_pci_id_table[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, CIO2_PCI_ID) },
+	{ PCI_VDEVICE(INTEL, CIO2_PCI_ID) },
 	{ }
 };
 
diff --git a/drivers/media/pci/intel/ipu6/ipu6.c b/drivers/media/pci/intel/ipu6/ipu6.c
index d033d46181692..5449a2006bcce 100644
--- a/drivers/media/pci/intel/ipu6/ipu6.c
+++ b/drivers/media/pci/intel/ipu6/ipu6.c
@@ -400,7 +400,6 @@ ipu6_isys_init(struct pci_dev *pdev, struct device *parent,
 				       &ipdata->hw_variant);
 	if (IS_ERR(isys_adev->mmu)) {
 		put_device(&isys_adev->auxdev.dev);
-		kfree(pdata);
 		return dev_err_cast_probe(dev, isys_adev->mmu,
 				"ipu6_mmu_init(isys_adev->mmu) failed\n");
 	}
@@ -408,10 +407,8 @@ ipu6_isys_init(struct pci_dev *pdev, struct device *parent,
 	isys_adev->mmu->dev = &isys_adev->auxdev.dev;
 
 	ret = ipu6_bus_add_device(isys_adev);
-	if (ret) {
-		kfree(pdata);
+	if (ret)
 		return ERR_PTR(ret);
-	}
 
 	return isys_adev;
 }
@@ -444,7 +441,6 @@ ipu6_psys_init(struct pci_dev *pdev, struct device *parent,
 				       &ipdata->hw_variant);
 	if (IS_ERR(psys_adev->mmu)) {
 		put_device(&psys_adev->auxdev.dev);
-		kfree(pdata);
 		return dev_err_cast_probe(&pdev->dev, psys_adev->mmu,
 				"ipu6_mmu_init(psys_adev->mmu) failed\n");
 	}
@@ -452,10 +448,8 @@ ipu6_psys_init(struct pci_dev *pdev, struct device *parent,
 	psys_adev->mmu->dev = &psys_adev->auxdev.dev;
 
 	ret = ipu6_bus_add_device(psys_adev);
-	if (ret) {
-		kfree(pdata);
+	if (ret)
 		return ERR_PTR(ret);
-	}
 
 	return psys_adev;
 }
diff --git a/drivers/media/pci/ivtv/ivtv-driver.c b/drivers/media/pci/ivtv/ivtv-driver.c
index e0091a8e5c888..b9ea56ec9593f 100644
--- a/drivers/media/pci/ivtv/ivtv-driver.c
+++ b/drivers/media/pci/ivtv/ivtv-driver.c
@@ -60,11 +60,9 @@ EXPORT_SYMBOL(ivtv_ext_init);
 
 /* add your revision and whatnot here */
 static const struct pci_device_id ivtv_pci_tbl[] = {
-	{PCI_VENDOR_ID_ICOMP, PCI_DEVICE_ID_IVTV15,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_ICOMP, PCI_DEVICE_ID_IVTV16,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{0,}
+	{ PCI_VDEVICE(ICOMP, PCI_DEVICE_ID_IVTV15) },
+	{ PCI_VDEVICE(ICOMP, PCI_DEVICE_ID_IVTV16) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci,ivtv_pci_tbl);
diff --git a/drivers/media/pci/ivtv/ivtv-vbi.c b/drivers/media/pci/ivtv/ivtv-vbi.c
index ae7a00f462574..af086c4cbe1a7 100644
--- a/drivers/media/pci/ivtv/ivtv-vbi.c
+++ b/drivers/media/pci/ivtv/ivtv-vbi.c
@@ -330,7 +330,7 @@ static u32 compress_sliced_buf(struct ivtv *itv, u32 line, u8 *buf, u32 size, u8
 	unsigned lines = 0;
 
 	/* find the first valid line */
-	for (i = 0; i < size; i++, buf++) {
+	for (i = 0; i + 3 < size; i++, buf++) {
 		if (buf[0] == 0xff && !buf[1] && !buf[2] && buf[3] == sav)
 			break;
 	}
diff --git a/drivers/media/pci/mantis/mantis_common.h b/drivers/media/pci/mantis/mantis_common.h
index 6e563ecd94e88..f357ca268dd1f 100644
--- a/drivers/media/pci/mantis/mantis_common.h
+++ b/drivers/media/pci/mantis/mantis_common.h
@@ -58,11 +58,9 @@
 #define TERRATEC		0x153b
 
 #define MAKE_ENTRY(__subven, __subdev, __configptr, __rc) {		\
-		.vendor		= TWINHAN_TECHNOLOGIES,			\
-		.device		= MANTIS,				\
-		.subvendor	= (__subven),				\
-		.subdevice	= (__subdev),				\
-		.driver_data	= (unsigned long)			\
+		PCI_DEVICE_SUB(TWINHAN_TECHNOLOGIES, MANTIS,		\
+			       (__subven), (__subdev)),			\
+		.driver_data = (unsigned long)				\
 			&(struct mantis_pci_drvdata){__configptr, __rc}	\
 }
 
diff --git a/drivers/media/pci/mgb4/mgb4_core.c b/drivers/media/pci/mgb4/mgb4_core.c
index d23d854581c5f..cbd193e065361 100644
--- a/drivers/media/pci/mgb4/mgb4_core.c
+++ b/drivers/media/pci/mgb4/mgb4_core.c
@@ -694,9 +694,9 @@ static void mgb4_remove(struct pci_dev *pdev)
 }
 
 static const struct pci_device_id mgb4_pci_ids[] = {
-	{ PCI_DEVICE(DIGITEQ_VID, T100_DID), },
-	{ PCI_DEVICE(DIGITEQ_VID, T200_DID), },
-	{ 0, }
+	{ PCI_DEVICE(DIGITEQ_VID, T100_DID) },
+	{ PCI_DEVICE(DIGITEQ_VID, T200_DID) },
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, mgb4_pci_ids);
 
diff --git a/drivers/media/pci/netup_unidvb/netup_unidvb_core.c b/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
index ec08023b5d5e2..00f81e4c8e8d3 100644
--- a/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
+++ b/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
@@ -1014,7 +1014,7 @@ static void netup_unidvb_finidev(struct pci_dev *pci_dev)
 static const struct pci_device_id netup_unidvb_pci_tbl[] = {
 	{ PCI_DEVICE(0x1b55, 0x18f6) }, /* hw rev. 1.3 */
 	{ PCI_DEVICE(0x1b55, 0x18f7) }, /* hw rev. 1.4 */
-	{ 0, }
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, netup_unidvb_pci_tbl);
 
diff --git a/drivers/media/pci/ngene/ngene-cards.c b/drivers/media/pci/ngene/ngene-cards.c
index 7dbc21e1a2cae..a1a83f3bb6b4c 100644
--- a/drivers/media/pci/ngene/ngene-cards.c
+++ b/drivers/media/pci/ngene/ngene-cards.c
@@ -1155,8 +1155,7 @@ static const struct ngene_info ngene_info_terratec = {
 /****************************************************************************/
 
 #define NGENE_ID(_subvend, _subdev, _driverdata) { \
-	.vendor = NGENE_VID, .device = NGENE_PID, \
-	.subvendor = _subvend, .subdevice = _subdev, \
+	PCI_DEVICE_SUB(NGENE_VID, NGENE_PID, (_subvend), (_subdev)), \
 	.driver_data = (unsigned long) &_driverdata }
 
 /****************************************************************************/
@@ -1173,7 +1172,7 @@ static const struct pci_device_id ngene_id_tbl[] = {
 	NGENE_ID(0x18c3, 0xdd20, ngene_info_duoFlex),
 	NGENE_ID(0x1461, 0x062e, ngene_info_m780),
 	NGENE_ID(0x153b, 0x1167, ngene_info_terratec),
-	{0}
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, ngene_id_tbl);
 
diff --git a/drivers/media/pci/pluto2/pluto2.c b/drivers/media/pci/pluto2/pluto2.c
index 22c2222d5c4d7..77b3e4eec54dc 100644
--- a/drivers/media/pci/pluto2/pluto2.c
+++ b/drivers/media/pci/pluto2/pluto2.c
@@ -762,10 +762,7 @@ static void pluto2_remove(struct pci_dev *pdev)
 
 static const struct pci_device_id pluto2_id_table[] = {
 	{
-		.vendor = PCI_VENDOR_ID_SCM,
-		.device = PCI_DEVICE_ID_PLUTO2,
-		.subvendor = PCI_ANY_ID,
-		.subdevice = PCI_ANY_ID,
+		PCI_VDEVICE(SCM, PCI_DEVICE_ID_PLUTO2),
 	}, {
 		/* empty */
 	},
diff --git a/drivers/media/pci/saa7134/saa7134-cards.c b/drivers/media/pci/saa7134/saa7134-cards.c
index e80fb4ebfda61..a7d665407460f 100644
--- a/drivers/media/pci/saa7134/saa7134-cards.c
+++ b/drivers/media/pci/saa7134/saa7134-cards.c
@@ -5800,1317 +5800,758 @@ const unsigned int saa7134_bcount = ARRAY_SIZE(saa7134_boards);
 
 struct pci_device_id saa7134_pci_tbl[] = {
 	{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = PCI_VENDOR_ID_PHILIPS,
-		.subdevice    = 0x2001,
-		.driver_data  = SAA7134_BOARD_PROTEUS_PRO,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = PCI_VENDOR_ID_PHILIPS,
-		.subdevice    = 0x2001,
-		.driver_data  = SAA7134_BOARD_PROTEUS_PRO,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = PCI_VENDOR_ID_PHILIPS,
-		.subdevice    = 0x6752,
-		.driver_data  = SAA7134_BOARD_EMPRESS,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1131,
-		.subdevice    = 0x4e85,
-		.driver_data  = SAA7134_BOARD_MONSTERTV,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x153b,
-		.subdevice    = 0x1142,
-		.driver_data  = SAA7134_BOARD_CINERGY400,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x153b,
-		.subdevice    = 0x1143,
-		.driver_data  = SAA7134_BOARD_CINERGY600,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x153b,
-		.subdevice    = 0x1158,
-		.driver_data  = SAA7134_BOARD_CINERGY600_MK3,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x153b,
-		.subdevice    = 0x1162,
-		.driver_data  = SAA7134_BOARD_CINERGY400_CARDBUS,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x5169,
-		.subdevice    = 0x0138,
-		.driver_data  = SAA7134_BOARD_FLYVIDEO3000_NTSC,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x5168,
-		.subdevice    = 0x0138,
-		.driver_data  = SAA7134_BOARD_FLYVIDEO3000,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x4e42,				/* "Typhoon PCI Capture TV Card" Art.No. 50673 */
-		.subdevice    = 0x0138,
-		.driver_data  = SAA7134_BOARD_FLYVIDEO3000,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x5168,
-		.subdevice    = 0x0138,
-		.driver_data  = SAA7134_BOARD_FLYVIDEO2000,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x4e42,		/* Typhoon */
-		.subdevice    = 0x0138,		/* LifeView FlyTV Prime30 OEM */
-		.driver_data  = SAA7134_BOARD_FLYVIDEO2000,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5168,
-		.subdevice    = 0x0212, /* minipci, LR212 */
-		.driver_data  = SAA7134_BOARD_FLYTVPLATINUM_MINI,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x14c0,
-		.subdevice    = 0x1212, /* minipci, LR1212 */
-		.driver_data  = SAA7134_BOARD_FLYTVPLATINUM_MINI2,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x4e42,
-		.subdevice    = 0x0212, /* OEM minipci, LR212 */
-		.driver_data  = SAA7134_BOARD_FLYTVPLATINUM_MINI,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5168,	/* Animation Technologies (LifeView) */
-		.subdevice    = 0x0214, /* Standard PCI, LR214 Rev E and earlier (SAA7135) */
-		.driver_data  = SAA7134_BOARD_FLYTVPLATINUM_FM,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5168,	/* Animation Technologies (LifeView) */
-		.subdevice    = 0x5214, /* Standard PCI, LR214 Rev F onwards (SAA7131) */
-		.driver_data  = SAA7134_BOARD_FLYTVPLATINUM_FM,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1489, /* KYE */
-		.subdevice    = 0x0214, /* Genius VideoWonder ProTV */
-		.driver_data  = SAA7134_BOARD_FLYTVPLATINUM_FM, /* is an LR214WF actually */
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x16be,
-		.subdevice    = 0x0003,
-		.driver_data  = SAA7134_BOARD_MD7134,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x16be, /* CTX946 analog TV, HW mpeg, DVB-T */
-		.subdevice    = 0x5000, /* only analog TV and DVB-T for now */
-		.driver_data  = SAA7134_BOARD_MD7134,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x1048,
-		.subdevice    = 0x226b,
-		.driver_data  = SAA7134_BOARD_ELSA,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x1048,
-		.subdevice    = 0x226a,
-		.driver_data  = SAA7134_BOARD_ELSA_500TV,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x1048,
-		.subdevice    = 0x226c,
-		.driver_data  = SAA7134_BOARD_ELSA_700TV,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = PCI_VENDOR_ID_ASUSTEK,
-		.subdevice    = 0x4842,
-		.driver_data  = SAA7134_BOARD_ASUSTeK_TVFM7134,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = PCI_VENDOR_ID_ASUSTEK,
-		.subdevice    = 0x4845,
-		.driver_data  = SAA7134_BOARD_ASUSTeK_TVFM7135,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = PCI_VENDOR_ID_ASUSTEK,
-		.subdevice    = 0x4830,
-		.driver_data  = SAA7134_BOARD_ASUSTeK_TVFM7134,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = PCI_VENDOR_ID_ASUSTEK,
-		.subdevice    = 0x4843,
-		.driver_data  = SAA7134_BOARD_ASUSTEK_TVFM7133,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = PCI_VENDOR_ID_ASUSTEK,
-		.subdevice    = 0x4840,
-		.driver_data  = SAA7134_BOARD_ASUSTeK_TVFM7134,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = PCI_VENDOR_ID_PHILIPS,
-		.subdevice    = 0xfe01,
-		.driver_data  = SAA7134_BOARD_TVSTATION_RDS,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1894,
-		.subdevice    = 0xfe01,
-		.driver_data  = SAA7134_BOARD_TVSTATION_RDS,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1894,
-		.subdevice    = 0xa006,
-		.driver_data  = SAA7134_BOARD_TVSTATION_DVR,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1131,
-		.subdevice    = 0x7133,
-		.driver_data  = SAA7134_BOARD_VA1000POWER,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = PCI_VENDOR_ID_PHILIPS,
-		.subdevice    = 0x2001,
-		.driver_data  = SAA7134_BOARD_10MOONSTVMASTER,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x185b,
-		.subdevice    = 0xc100,
-		.driver_data  = SAA7134_BOARD_VIDEOMATE_TV,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x185b,
-		.subdevice    = 0xc100,
-		.driver_data  = SAA7134_BOARD_VIDEOMATE_TV_GOLD_PLUS,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = PCI_VENDOR_ID_MATROX,
-		.subdevice    = 0x48d0,
-		.driver_data  = SAA7134_BOARD_CRONOS_PLUS,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xa70b,
-		.driver_data  = SAA7134_BOARD_MD2819,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xa7a1,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_A700_PRO,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xa7a2,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_A700_HYBRID,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0x2115,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_STUDIO_305,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xa115,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_STUDIO_505,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0x2108,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_305,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0x10ff,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_DVD_EZMAKER,
-	},{
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134,
+				PCI_VENDOR_ID_PHILIPS, 0x2001),
+		.driver_data = SAA7134_BOARD_PROTEUS_PRO,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133,
+				PCI_VENDOR_ID_PHILIPS, 0x2001),
+		.driver_data = SAA7134_BOARD_PROTEUS_PRO,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134,
+				PCI_VENDOR_ID_PHILIPS, 0x6752),
+		.driver_data = SAA7134_BOARD_EMPRESS,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1131, 0x4e85),
+		.driver_data = SAA7134_BOARD_MONSTERTV,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x153b, 0x1142),
+		.driver_data = SAA7134_BOARD_CINERGY400,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x153b, 0x1143),
+		.driver_data = SAA7134_BOARD_CINERGY600,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x153b, 0x1158),
+		.driver_data = SAA7134_BOARD_CINERGY600_MK3,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x153b, 0x1162),
+		.driver_data = SAA7134_BOARD_CINERGY400_CARDBUS,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x5169, 0x0138),
+		.driver_data = SAA7134_BOARD_FLYVIDEO3000_NTSC,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x5168, 0x0138),
+		.driver_data = SAA7134_BOARD_FLYVIDEO3000,
+	}, {
+		/* "Typhoon PCI Capture TV Card" Art.No. 50673 */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x4e42, 0x0138),
+		.driver_data = SAA7134_BOARD_FLYVIDEO3000,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x5168, 0x0138),
+		.driver_data = SAA7134_BOARD_FLYVIDEO2000,
+	}, {
+		/* Typhoon LifeView FlyTV Prime30 OEM */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x4e42, 0x0138),
+		.driver_data = SAA7134_BOARD_FLYVIDEO2000,
+	}, {
+		/* minipci, LR212 */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5168, 0x0212),
+		.driver_data = SAA7134_BOARD_FLYTVPLATINUM_MINI,
+	}, {
+		/* minipci, LR1212 */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x14c0, 0x1212),
+		.driver_data = SAA7134_BOARD_FLYTVPLATINUM_MINI2,
+	}, {
+		/* OEM minipci, LR212 */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x4e42, 0x0212),
+		.driver_data = SAA7134_BOARD_FLYTVPLATINUM_MINI,
+	}, {
+		/*
+		 * Animation Technologies (LifeView)
+		 * Standard PCI, LR214 Rev E and earlier (SAA7135)
+		 */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5168, 0x0214),
+		.driver_data = SAA7134_BOARD_FLYTVPLATINUM_FM,
+	}, {
+		/*
+		 * Animation Technologies (LifeView)
+		 * Standard PCI, LR214 Rev F onwards (SAA7131)
+		 */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5168, 0x5214),
+		.driver_data = SAA7134_BOARD_FLYTVPLATINUM_FM,
+	}, {
+		/* KYE Genius VideoWonder ProTV */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1489, 0x0214),
+		.driver_data = SAA7134_BOARD_FLYTVPLATINUM_FM, /* is an LR214WF actually */
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x16be, 0x0003),
+		.driver_data = SAA7134_BOARD_MD7134,
+	}, {
+		/* CTX946 analog TV, HW mpeg, DVB-T; only analog TV and DVB-T for now */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x16be, 0x5000),
+		.driver_data = SAA7134_BOARD_MD7134,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x1048, 0x226b),
+		.driver_data = SAA7134_BOARD_ELSA,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x1048, 0x226a),
+		.driver_data = SAA7134_BOARD_ELSA_500TV,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x1048, 0x226c),
+		.driver_data = SAA7134_BOARD_ELSA_700TV,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134,
+				PCI_VENDOR_ID_ASUSTEK, 0x4842),
+		.driver_data = SAA7134_BOARD_ASUSTeK_TVFM7134,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133,
+				PCI_VENDOR_ID_ASUSTEK, 0x4845),
+		.driver_data = SAA7134_BOARD_ASUSTeK_TVFM7135,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134,
+				PCI_VENDOR_ID_ASUSTEK, 0x4830),
+		.driver_data = SAA7134_BOARD_ASUSTeK_TVFM7134,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133,
+				PCI_VENDOR_ID_ASUSTEK, 0x4843),
+		.driver_data = SAA7134_BOARD_ASUSTEK_TVFM7133,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134,
+				PCI_VENDOR_ID_ASUSTEK, 0x4840),
+		.driver_data = SAA7134_BOARD_ASUSTeK_TVFM7134,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134,
+				PCI_VENDOR_ID_PHILIPS, 0xfe01),
+		.driver_data = SAA7134_BOARD_TVSTATION_RDS,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1894, 0xfe01),
+		.driver_data = SAA7134_BOARD_TVSTATION_RDS,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1894, 0xa006),
+		.driver_data = SAA7134_BOARD_TVSTATION_DVR,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1131, 0x7133),
+		.driver_data = SAA7134_BOARD_VA1000POWER,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130,
+				PCI_VENDOR_ID_PHILIPS, 0x2001),
+		.driver_data = SAA7134_BOARD_10MOONSTVMASTER,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x185b, 0xc100),
+		.driver_data = SAA7134_BOARD_VIDEOMATE_TV,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x185b, 0xc100),
+		.driver_data = SAA7134_BOARD_VIDEOMATE_TV_GOLD_PLUS,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130,
+				PCI_VENDOR_ID_MATROX, 0x48d0),
+		.driver_data = SAA7134_BOARD_CRONOS_PLUS,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1461, 0xa70b),
+		.driver_data = SAA7134_BOARD_MD2819,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0xa7a1),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_A700_PRO,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0xa7a2),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_A700_HYBRID,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x1461, 0x2115),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_STUDIO_305,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x1461, 0xa115),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_STUDIO_505,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x1461, 0x2108),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_305,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x1461, 0x10ff),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_DVD_EZMAKER,
+	}, {
 		/* AVerMedia CardBus */
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xd6ee,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_CARDBUS,
-	},{
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1461, 0xd6ee),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_CARDBUS,
+	}, {
 		/* AVerMedia CardBus */
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xb7e9,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_CARDBUS_501,
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1461, 0xb7e9),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_CARDBUS_501,
 	}, {
-		/* TransGear 3000TV */
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0x050c,
-		.driver_data  = SAA7134_BOARD_TG3000TV,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x11bd,
-		.subdevice    = 0x002b,
-		.driver_data  = SAA7134_BOARD_PINNACLE_PCTV_STEREO,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x11bd,
-		.subdevice    = 0x002d,
-		.driver_data  = SAA7134_BOARD_PINNACLE_300I_DVBT_PAL,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1019,
-		.subdevice    = 0x4cb4,
-		.driver_data  = SAA7134_BOARD_ECS_TVP3XP,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1019,
-		.subdevice    = 0x4cb5,
-		.driver_data  = SAA7134_BOARD_ECS_TVP3XP_4CB5,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1019,
-		.subdevice    = 0x4cb6,
-		.driver_data  = SAA7134_BOARD_ECS_TVP3XP_4CB6,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x12ab,
-		.subdevice    = 0x0800,
-		.driver_data  = SAA7134_BOARD_UPMOST_PURPLE_TV,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x153b,
-		.subdevice    = 0x1152,
-		.driver_data  = SAA7134_BOARD_CINERGY200,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x185b,
-		.subdevice    = 0xc100,
-		.driver_data  = SAA7134_BOARD_VIDEOMATE_TV_PVR,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0x9715,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_STUDIO_307,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xa70a,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_307,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x185b,
-		.subdevice    = 0xc200,
-		.driver_data  = SAA7134_BOARD_VIDEOMATE_GOLD_PLUS,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1540,
-		.subdevice    = 0x9524,
-		.driver_data  = SAA7134_BOARD_PROVIDEO_PV952,
-
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5168,
-		.subdevice    = 0x0502,                /* Cardbus version */
-		.driver_data  = SAA7134_BOARD_FLYDVBT_DUO_CARDBUS,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5168,
-		.subdevice    = 0x0306,                /* PCI version */
-		.driver_data  = SAA7134_BOARD_FLYDVBTDUO,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xf31f,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_GO_007_FM,
-
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xf11d,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_M135A,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0x4155,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_M733A,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0x4255,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_M733A,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = PCI_VENDOR_ID_PHILIPS,
-		.subdevice    = 0x2004,
-		.driver_data  = SAA7134_BOARD_PHILIPS_TOUGH,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1421,
-		.subdevice    = 0x0350,		/* PCI version */
-		.driver_data  = SAA7134_BOARD_ADS_INSTANT_TV,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1421,
-		.subdevice    = 0x0351,		/* PCI version, new revision */
-		.driver_data  = SAA7134_BOARD_ADS_INSTANT_TV,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1421,
-		.subdevice    = 0x0370,		/* cardbus version */
-		.driver_data  = SAA7134_BOARD_ADS_INSTANT_TV,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1421,
-		.subdevice    = 0x1370,        /* cardbus version */
-		.driver_data  = SAA7134_BOARD_ADS_INSTANT_TV,
-
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x4e42,		/* Typhoon */
-		.subdevice    = 0x0502,		/* LifeView LR502 OEM */
-		.driver_data  = SAA7134_BOARD_FLYDVBT_DUO_CARDBUS,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1043,
-		.subdevice    = 0x0210,		/* mini pci NTSC version */
-		.driver_data  = SAA7134_BOARD_FLYTV_DIGIMATRIX,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1043,
-		.subdevice    = 0x0210,		/* mini pci PAL/SECAM version */
-		.driver_data  = SAA7134_BOARD_ASUSTEK_DIGIMATRIX_TV,
-
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x0000, /* It shouldn't break anything, since subdevice id seems unique */
-		.subdevice    = 0x4091,
-		.driver_data  = SAA7134_BOARD_BEHOLD_409FM,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5456, /* GoTView */
-		.subdevice    = 0x7135,
-		.driver_data  = SAA7134_BOARD_GOTVIEW_7135,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = PCI_VENDOR_ID_PHILIPS,
-		.subdevice    = 0x2004,
-		.driver_data  = SAA7134_BOARD_PHILIPS_EUROPA,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x185b,
-		.subdevice    = 0xc900,
-		.driver_data  = SAA7134_BOARD_VIDEOMATE_DVBT_300,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x185b,
-		.subdevice    = 0xc901,
-		.driver_data  = SAA7134_BOARD_VIDEOMATE_DVBT_200,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1435,
-		.subdevice    = 0x7350,
-		.driver_data  = SAA7134_BOARD_RTD_VFG7350,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1435,
-		.subdevice    = 0x7330,
-		.driver_data  = SAA7134_BOARD_RTD_VFG7330,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461,
-		.subdevice    = 0x1044,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_AVERTVHD_A180,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1131,
-		.subdevice    = 0x4ee9,
-		.driver_data  = SAA7134_BOARD_MONSTERTV_MOBILE,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x11bd,
-		.subdevice    = 0x002e,
-		.driver_data  = SAA7134_BOARD_PINNACLE_PCTV_110i,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1043,
-		.subdevice    = 0x4862,
-		.driver_data  = SAA7134_BOARD_ASUSTeK_P7131_DUAL,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = PCI_VENDOR_ID_PHILIPS,
-		.subdevice    = 0x2018,
-		.driver_data  = SAA7134_BOARD_PHILIPS_TIGER,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1462,
-		.subdevice    = 0x6231, /* tda8275a, ks003 IR */
-		.driver_data  = SAA7134_BOARD_MSI_TVATANYWHERE_PLUS,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1462,
-		.subdevice    = 0x8624, /* tda8275, ks003 IR */
-		.driver_data  = SAA7134_BOARD_MSI_TVATANYWHERE_PLUS,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x153b,
-		.subdevice    = 0x1160,
-		.driver_data  = SAA7134_BOARD_CINERGY250PCI,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,	/* SAA 7131E */
-		.subvendor    = 0x5168,
-		.subdevice    = 0x0319,
-		.driver_data  = SAA7134_BOARD_FLYDVB_TRIO,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1461,
-		.subdevice    = 0x2c05,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_777,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x5168,
-		.subdevice    = 0x0301,
-		.driver_data  = SAA7134_BOARD_FLYDVBT_LR301,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x0331,
-		.subdevice    = 0x1421,
-		.driver_data  = SAA7134_BOARD_ADS_DUO_CARDBUS_PTV331,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x17de,
-		.subdevice    = 0x7201,
-		.driver_data  = SAA7134_BOARD_TEVION_DVBT_220RF,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x17de,
-		.subdevice    = 0x7250,
-		.driver_data  = SAA7134_BOARD_KWORLD_DVBT_210,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133, /* SAA7135HL */
-		.subvendor    = 0x17de,
-		.subdevice    = 0x7350,
-		.driver_data  = SAA7134_BOARD_KWORLD_ATSC110,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133, /* SAA7135HL */
-		.subvendor    = 0x17de,
-		.subdevice    = 0x7352,
-		.driver_data  = SAA7134_BOARD_KWORLD_ATSC110, /* ATSC 115 */
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133, /* SAA7135HL */
-		.subvendor    = 0x17de,
-		.subdevice    = 0xa134,
-		.driver_data  = SAA7134_BOARD_KWORLD_PC150U,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1461,
-		.subdevice    = 0x7360,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_A169_B,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1461,
-		.subdevice    = 0x6360,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_A169_B1,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x16be,
-		.subdevice    = 0x0005,
-		.driver_data  = SAA7134_BOARD_MD7134_BRIDGE_2,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x5168,
-		.subdevice    = 0x0300,
-		.driver_data  = SAA7134_BOARD_FLYDVBS_LR300,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x4e42,
-		.subdevice    = 0x0300,/* LR300 */
-		.driver_data  = SAA7134_BOARD_FLYDVBS_LR300,
-	},{
-		.vendor = PCI_VENDOR_ID_PHILIPS,
-		.device = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor = 0x1489,
-		.subdevice = 0x0301,
+		/* AVerMedia TransGear 3000TV */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x1461, 0x050c),
+		.driver_data = SAA7134_BOARD_TG3000TV,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x11bd, 0x002b),
+		.driver_data = SAA7134_BOARD_PINNACLE_PCTV_STEREO,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x11bd, 0x002d),
+		.driver_data = SAA7134_BOARD_PINNACLE_300I_DVBT_PAL,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1019, 0x4cb4),
+		.driver_data = SAA7134_BOARD_ECS_TVP3XP,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1019, 0x4cb5),
+		.driver_data = SAA7134_BOARD_ECS_TVP3XP_4CB5,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1019, 0x4cb6),
+		.driver_data = SAA7134_BOARD_ECS_TVP3XP_4CB6,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x12ab, 0x0800),
+		.driver_data = SAA7134_BOARD_UPMOST_PURPLE_TV,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x153b, 0x1152),
+		.driver_data = SAA7134_BOARD_CINERGY200,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x185b, 0xc100),
+		.driver_data = SAA7134_BOARD_VIDEOMATE_TV_PVR,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1461, 0x9715),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_STUDIO_307,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1461, 0xa70a),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_307,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x185b, 0xc200),
+		.driver_data = SAA7134_BOARD_VIDEOMATE_GOLD_PLUS,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1540, 0x9524),
+		.driver_data = SAA7134_BOARD_PROVIDEO_PV952,
+	}, {
+		/* Cardbus version */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5168, 0x0502),
+		.driver_data = SAA7134_BOARD_FLYDVBT_DUO_CARDBUS,
+	}, {
+		/* PCI version */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5168, 0x0306),
+		.driver_data = SAA7134_BOARD_FLYDVBTDUO,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0xf31f),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_GO_007_FM,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0xf11d),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_M135A,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0x4155),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_M733A,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0x4255),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_M733A,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130,
+				PCI_VENDOR_ID_PHILIPS, 0x2004),
+		.driver_data = SAA7134_BOARD_PHILIPS_TOUGH,
+	}, {
+		/* PCI version */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1421, 0x0350),
+		.driver_data = SAA7134_BOARD_ADS_INSTANT_TV,
+	}, {
+		/* PCI version, new revision */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1421, 0x0351),
+		.driver_data = SAA7134_BOARD_ADS_INSTANT_TV,
+	}, {
+		/* cardbus version */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1421, 0x0370),
+		.driver_data = SAA7134_BOARD_ADS_INSTANT_TV,
+	}, {
+		/* cardbus version */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1421, 0x1370),
+		.driver_data = SAA7134_BOARD_ADS_INSTANT_TV,
+	}, {
+		/* Typhoon LifeView LR502 OEM */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x4e42, 0x0502),
+		.driver_data = SAA7134_BOARD_FLYDVBT_DUO_CARDBUS,
+	}, {
+		/* mini pci NTSC version */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1043, 0x0210),
+		.driver_data = SAA7134_BOARD_FLYTV_DIGIMATRIX,
+	}, {
+		/* mini pci PAL/SECAM version */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1043, 0x0210),
+		.driver_data = SAA7134_BOARD_ASUSTEK_DIGIMATRIX_TV,
+	}, {
+		/* subvendor == 0 shouldn't break anything, since subdevice id seems unique */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x0000, 0x4091),
+		.driver_data = SAA7134_BOARD_BEHOLD_409FM,
+	}, {
+		/* GoTView */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5456, 0x7135),
+		.driver_data = SAA7134_BOARD_GOTVIEW_7135,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134,
+				PCI_VENDOR_ID_PHILIPS, 0x2004),
+		.driver_data = SAA7134_BOARD_PHILIPS_EUROPA,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x185b, 0xc900),
+		.driver_data = SAA7134_BOARD_VIDEOMATE_DVBT_300,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x185b, 0xc901),
+		.driver_data = SAA7134_BOARD_VIDEOMATE_DVBT_200,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1435, 0x7350),
+		.driver_data = SAA7134_BOARD_RTD_VFG7350,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1435, 0x7330),
+		.driver_data = SAA7134_BOARD_RTD_VFG7330,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0x1044),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_AVERTVHD_A180,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1131, 0x4ee9),
+		.driver_data = SAA7134_BOARD_MONSTERTV_MOBILE,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x11bd, 0x002e),
+		.driver_data = SAA7134_BOARD_PINNACLE_PCTV_110i,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1043, 0x4862),
+		.driver_data = SAA7134_BOARD_ASUSTeK_P7131_DUAL,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133,
+				PCI_VENDOR_ID_PHILIPS, 0x2018),
+		.driver_data = SAA7134_BOARD_PHILIPS_TIGER,
+	}, {
+		/* tda8275a, ks003 IR */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1462, 0x6231),
+		.driver_data = SAA7134_BOARD_MSI_TVATANYWHERE_PLUS,
+	}, {
+		/* tda8275, ks003 IR */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1462, 0x8624),
+		.driver_data = SAA7134_BOARD_MSI_TVATANYWHERE_PLUS,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x153b, 0x1160),
+		.driver_data = SAA7134_BOARD_CINERGY250PCI,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5168, 0x0319),
+		.driver_data = SAA7134_BOARD_FLYDVB_TRIO,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1461, 0x2c05),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_777,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x5168, 0x0301),
 		.driver_data = SAA7134_BOARD_FLYDVBT_LR301,
-	},{
-		.vendor = PCI_VENDOR_ID_PHILIPS,
-		.device = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor = 0x5168, /* Animation Technologies (LifeView) */
-		.subdevice = 0x0304,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x0331, 0x1421),
+		.driver_data = SAA7134_BOARD_ADS_DUO_CARDBUS_PTV331,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x17de, 0x7201),
+		.driver_data = SAA7134_BOARD_TEVION_DVBT_220RF,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x17de, 0x7250),
+		.driver_data = SAA7134_BOARD_KWORLD_DVBT_210,
+	}, {
+		/* SAA7135HL */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x17de, 0x7350),
+		.driver_data = SAA7134_BOARD_KWORLD_ATSC110,
+	}, {
+		/* SAA7135HL */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x17de, 0x7352),
+		.driver_data = SAA7134_BOARD_KWORLD_ATSC110, /* ATSC 115 */
+	}, {
+		/* SAA7135HL */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x17de, 0xa134),
+		.driver_data = SAA7134_BOARD_KWORLD_PC150U,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1461, 0x7360),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_A169_B,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1461, 0x6360),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_A169_B1,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x16be, 0x0005),
+		.driver_data = SAA7134_BOARD_MD7134_BRIDGE_2,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x5168, 0x0300),
+		.driver_data = SAA7134_BOARD_FLYDVBS_LR300,
+	}, {
+		/* LR300 */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x4e42, 0x0300),
+		.driver_data = SAA7134_BOARD_FLYDVBS_LR300,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1489, 0x0301),
+		.driver_data = SAA7134_BOARD_FLYDVBT_LR301,
+	}, {
+		/* Animation Technologies (LifeView) */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5168, 0x0304),
 		.driver_data = SAA7134_BOARD_FLYTVPLATINUM_FM,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5168,
-		.subdevice    = 0x3306,
-		.driver_data  = SAA7134_BOARD_FLYDVBT_HYBRID_CARDBUS,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5168,
-		.subdevice    = 0x3502,  /* what's the difference to 0x3306 ?*/
-		.driver_data  = SAA7134_BOARD_FLYDVBT_HYBRID_CARDBUS,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5168,
-		.subdevice    = 0x3307, /* FlyDVB-T Hybrid Mini PCI */
-		.driver_data  = SAA7134_BOARD_FLYDVBT_HYBRID_CARDBUS,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x16be,
-		.subdevice    = 0x0007,
-		.driver_data  = SAA7134_BOARD_MEDION_MD8800_QUADRO,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x16be,
-		.subdevice    = 0x0008,
-		.driver_data  = SAA7134_BOARD_MEDION_MD8800_QUADRO,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x16be,
-		.subdevice    = 0x000d, /* triple CTX948_V1.1.1 */
-		.driver_data  = SAA7134_BOARD_MEDION_MD8800_QUADRO,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461,
-		.subdevice    = 0x2c05,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_777,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1489,
-		.subdevice    = 0x0502,                /* Cardbus version */
-		.driver_data  = SAA7134_BOARD_FLYDVBT_DUO_CARDBUS,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x0919, /* Philips Proteus PRO 2309 */
-		.subdevice    = 0x2003,
-		.driver_data  = SAA7134_BOARD_PROTEUS_2309,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1461,
-		.subdevice    = 0x2c00,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_A16AR,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1043,
-		.subdevice    = 0x4860,
-		.driver_data  = SAA7134_BOARD_ASUS_EUROPA2_HYBRID,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x11bd,
-		.subdevice    = 0x002f,
-		.driver_data  = SAA7134_BOARD_PINNACLE_PCTV_310i,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0x9715,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_STUDIO_507,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xa11b,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_STUDIO_507UA,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1043,
-		.subdevice    = 0x4876,
-		.driver_data  = SAA7134_BOARD_ASUSTeK_P7131_HYBRID_LNA,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x0070,
-		.subdevice    = 0x6700,
-		.driver_data  = SAA7134_BOARD_HAUPPAUGE_HVR1110,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x0070,
-		.subdevice    = 0x6701,
-		.driver_data  = SAA7134_BOARD_HAUPPAUGE_HVR1110,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x0070,
-		.subdevice    = 0x6702,
-		.driver_data  = SAA7134_BOARD_HAUPPAUGE_HVR1110,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x0070,
-		.subdevice    = 0x6703,
-		.driver_data  = SAA7134_BOARD_HAUPPAUGE_HVR1110,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x0070,
-		.subdevice    = 0x6704,
-		.driver_data  = SAA7134_BOARD_HAUPPAUGE_HVR1110,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x0070,
-		.subdevice    = 0x6705,
-		.driver_data  = SAA7134_BOARD_HAUPPAUGE_HVR1110,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x0070,
-		.subdevice    = 0x6706,
-		.driver_data  = SAA7134_BOARD_HAUPPAUGE_HVR1150,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x0070,
-		.subdevice    = 0x6707,
-		.driver_data  = SAA7134_BOARD_HAUPPAUGE_HVR1120,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x0070,
-		.subdevice    = 0x6708,
-		.driver_data  = SAA7134_BOARD_HAUPPAUGE_HVR1150,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x0070,
-		.subdevice    = 0x6709,
-		.driver_data  = SAA7134_BOARD_HAUPPAUGE_HVR1120,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x0070,
-		.subdevice    = 0x670a,
-		.driver_data  = SAA7134_BOARD_HAUPPAUGE_HVR1120,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x153b,
-		.subdevice    = 0x1172,
-		.driver_data  = SAA7134_BOARD_CINERGY_HT_PCMCIA,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = PCI_VENDOR_ID_PHILIPS,
-		.subdevice    = 0x2342,
-		.driver_data  = SAA7134_BOARD_ENCORE_ENLTV,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x1131,
-		.subdevice    = 0x2341,
-		.driver_data  = SAA7134_BOARD_ENCORE_ENLTV,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x3016,
-		.subdevice    = 0x2344,
-		.driver_data  = SAA7134_BOARD_ENCORE_ENLTV,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x1131,
-		.subdevice    = 0x230f,
-		.driver_data  = SAA7134_BOARD_ENCORE_ENLTV_FM,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x1a7f,
-		.subdevice    = 0x2008,
-		.driver_data  = SAA7134_BOARD_ENCORE_ENLTV_FM53,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1a7f,
-		.subdevice    = 0x2108,
-		.driver_data  = SAA7134_BOARD_ENCORE_ENLTV_FM3,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x153b,
-		.subdevice    = 0x1175,
-		.driver_data  = SAA7134_BOARD_CINERGY_HT_PCI,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xf31e,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_M102,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x4E42,         /* MSI */
-		.subdevice    = 0x0306,         /* TV@nywhere DUO */
-		.driver_data  = SAA7134_BOARD_FLYDVBTDUO,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1043,
-		.subdevice    = 0x4871,
-		.driver_data  = SAA7134_BOARD_ASUS_P7131_4871,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1043,
-		.subdevice    = 0x4857,		/* REV:1.00 */
-		.driver_data  = SAA7134_BOARD_ASUSTeK_TIGER,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x0919, /* SinoVideo PCI 2309 Proteus (7134) */
-		.subdevice    = 0x2003, /* OEM cardbus */
-		.driver_data  = SAA7134_BOARD_SABRENT_TV_PCB05,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = PCI_VENDOR_ID_PHILIPS,
-		.subdevice    = 0x2304,
-		.driver_data  = SAA7134_BOARD_10MOONSTVMASTER3,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xf01d, /* AVerTV DVB-T Super 007 */
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_SUPER_007,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x0000,
-		.subdevice    = 0x4016,
-		.driver_data  = SAA7134_BOARD_BEHOLD_401,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x0000,
-		.subdevice    = 0x4036,
-		.driver_data  = SAA7134_BOARD_BEHOLD_403,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x0000,
-		.subdevice    = 0x4037,
-		.driver_data  = SAA7134_BOARD_BEHOLD_403FM,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x0000,
-		.subdevice    = 0x4050,
-		.driver_data  = SAA7134_BOARD_BEHOLD_405,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x0000,
-		.subdevice    = 0x4051,
-		.driver_data  = SAA7134_BOARD_BEHOLD_405FM,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x0000,
-		.subdevice    = 0x4070,
-		.driver_data  = SAA7134_BOARD_BEHOLD_407,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x0000,
-		.subdevice    = 0x4071,
-		.driver_data  = SAA7134_BOARD_BEHOLD_407FM,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x0000,
-		.subdevice    = 0x4090,
-		.driver_data  = SAA7134_BOARD_BEHOLD_409,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x0000,
-		.subdevice    = 0x505B,
-		.driver_data  = SAA7134_BOARD_BEHOLD_505RDS_MK5,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x0000,
-		.subdevice    = 0x5051,
-		.driver_data  = SAA7134_BOARD_BEHOLD_505RDS_MK3,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x5ace,
-		.subdevice    = 0x5050,
-		.driver_data  = SAA7134_BOARD_BEHOLD_505FM,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x0000,
-		.subdevice    = 0x5071,
-		.driver_data  = SAA7134_BOARD_BEHOLD_507RDS_MK3,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x0000,
-		.subdevice    = 0x507B,
-		.driver_data  = SAA7134_BOARD_BEHOLD_507RDS_MK5,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x5ace,
-		.subdevice    = 0x5070,
-		.driver_data  = SAA7134_BOARD_BEHOLD_507_9FM,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5ace,
-		.subdevice    = 0x5090,
-		.driver_data  = SAA7134_BOARD_BEHOLD_507_9FM,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x0000,
-		.subdevice    = 0x5201,
-		.driver_data  = SAA7134_BOARD_BEHOLD_COLUMBUS_TVFM,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x5ace,
-		.subdevice    = 0x6070,
-		.driver_data  = SAA7134_BOARD_BEHOLD_607FM_MK3,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x5ace,
-		.subdevice    = 0x6071,
-		.driver_data  = SAA7134_BOARD_BEHOLD_607FM_MK5,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x5ace,
-		.subdevice    = 0x6072,
-		.driver_data  = SAA7134_BOARD_BEHOLD_607RDS_MK3,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x5ace,
-		.subdevice    = 0x6073,
-		.driver_data  = SAA7134_BOARD_BEHOLD_607RDS_MK5,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5ace,
-		.subdevice    = 0x6090,
-		.driver_data  = SAA7134_BOARD_BEHOLD_609FM_MK3,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5ace,
-		.subdevice    = 0x6091,
-		.driver_data  = SAA7134_BOARD_BEHOLD_609FM_MK5,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5ace,
-		.subdevice    = 0x6092,
-		.driver_data  = SAA7134_BOARD_BEHOLD_609RDS_MK3,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5ace,
-		.subdevice    = 0x6093,
-		.driver_data  = SAA7134_BOARD_BEHOLD_609RDS_MK5,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5ace,
-		.subdevice    = 0x6190,
-		.driver_data  = SAA7134_BOARD_BEHOLD_M6,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5ace,
-		.subdevice    = 0x6193,
-		.driver_data  = SAA7134_BOARD_BEHOLD_M6_EXTRA,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5ace,
-		.subdevice    = 0x6191,
-		.driver_data  = SAA7134_BOARD_BEHOLD_M63,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x4e42,
-		.subdevice    = 0x3502,
-		.driver_data  = SAA7134_BOARD_FLYDVBT_HYBRID_CARDBUS,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1822, /*Twinhan Technology Co. Ltd*/
-		.subdevice    = 0x0022,
-		.driver_data  = SAA7134_BOARD_TWINHAN_DTV_DVB_3056,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x16be,
-		.subdevice    = 0x0010, /* Medion version CTX953_V.1.4.3 */
-		.driver_data  = SAA7134_BOARD_CREATIX_CTX953,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1462, /* MSI */
-		.subdevice    = 0x8625, /* TV@nywhere A/D v1.1 */
-		.driver_data  = SAA7134_BOARD_MSI_TVANYWHERE_AD11,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xf436,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_CARDBUS_506,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xf936,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_A16D,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xa836,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_M115,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x185b,
-		.subdevice    = 0xc900,
-		.driver_data  = SAA7134_BOARD_VIDEOMATE_T750,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133, /* SAA7135HL */
-		.subvendor    = 0x1421,
-		.subdevice    = 0x0380,
-		.driver_data  = SAA7134_BOARD_ADS_INSTANT_HDTV_PCI,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5169,
-		.subdevice    = 0x1502,
-		.driver_data  = SAA7134_BOARD_FLYTVPLATINUM_MINI,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5ace,
-		.subdevice    = 0x6290,
-		.driver_data  = SAA7134_BOARD_BEHOLD_H6,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xf636,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_M103,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xf736,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_M103,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1043,
-		.subdevice    = 0x4878, /* REV:1.02G */
-		.driver_data  = SAA7134_BOARD_ASUSTeK_TIGER_3IN1,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1043,
-		.subdevice    = 0x48cd,
-		.driver_data  = SAA7134_BOARD_ASUSTeK_PS3_100,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x17de,
-		.subdevice    = 0x7128,
-		.driver_data  = SAA7134_BOARD_KWORLD_PLUS_TV_ANALOG,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x17de,
-		.subdevice    = 0xb136,
-		.driver_data  = SAA7134_BOARD_KWORLD_PCI_SBTVD_FULLSEG,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xf31d,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_GO_007_FM_PLUS,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x185b,
-		.subdevice    = 0xc900,
-		.driver_data  = SAA7134_BOARD_VIDEOMATE_S350,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5ace, /* Beholder Intl. Ltd. */
-		.subdevice    = 0x7595,
-		.driver_data  = SAA7134_BOARD_BEHOLD_X7,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x19d1, /* RoverMedia */
-		.subdevice    = 0x0138, /* LifeView FlyTV Prime30 OEM */
-		.driver_data  = SAA7134_BOARD_ROVERMEDIA_LINK_PRO_FM,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = PCI_VENDOR_ID_PHILIPS,
-		.subdevice    = 0x2004,
-		.driver_data  = SAA7134_BOARD_ZOLID_HYBRID_PCI,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x1043,
-		.subdevice    = 0x4847,
-		.driver_data  = SAA7134_BOARD_ASUS_EUROPA_HYBRID,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x107d,
-		.subdevice    = 0x6655,
-		.driver_data  = SAA7134_BOARD_LEADTEK_WINFAST_DTV1000S,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x13c2,
-		.subdevice    = 0x2804,
-		.driver_data  = SAA7134_BOARD_TECHNOTREND_BUDGET_T3000,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5ace, /* Beholder Intl. Ltd. */
-		.subdevice    = 0x7190,
-		.driver_data  = SAA7134_BOARD_BEHOLD_H7,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5ace, /* Beholder Intl. Ltd. */
-		.subdevice    = 0x7090,
-		.driver_data  = SAA7134_BOARD_BEHOLD_A7,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7135,
-		.subvendor    = 0x185b,
-		.subdevice    = 0xc900,
-		.driver_data  = SAA7134_BOARD_VIDEOMATE_M1F,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x5ace,
-		.subdevice    = 0x5030,
-		.driver_data  = SAA7134_BOARD_BEHOLD_503FM,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x5ace,
-		.subdevice    = 0x5010,
-		.driver_data  = SAA7134_BOARD_BEHOLD_501,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = 0x17de,
-		.subdevice    = 0xd136,
-		.driver_data  = SAA7134_BOARD_MAGICPRO_PROHDTV_PRO2,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x6000,
-		.subdevice    = 0x0811,
-		.driver_data  = SAA7134_BOARD_SENSORAY811_911,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x6000,
-		.subdevice    = 0x0911,
-		.driver_data  = SAA7134_BOARD_SENSORAY811_911,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0x2055, /* AverTV Satellite Hybrid+FM A706 */
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_A706,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1905, /* WIS */
-		.subdevice    = 0x7007,
-		.driver_data  = SAA7134_BOARD_WIS_VOYAGER,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x1461, /* Avermedia Technologies Inc */
-		.subdevice    = 0xa10a,
-		.driver_data  = SAA7134_BOARD_AVERMEDIA_505,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = 0x107d,
-		.subdevice    = 0x6f3a,
-		.driver_data  = SAA7134_BOARD_LEADTEK_WINFAST_TV2100_FM,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x1779, /* V One Multimedia PTE Ltd */
-		.subdevice    = 0x13cf,
-		.driver_data  = SAA7134_BOARD_SNAZIO_TVPVR_PRO,
-	}, {
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = 0x107d,
-		.subdevice    = 0x6f2e,
-		.driver_data  = SAA7134_BOARD_LEADTEK_WINFAST_HDTV200_H,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5168, 0x3306),
+		.driver_data = SAA7134_BOARD_FLYDVBT_HYBRID_CARDBUS,
+	}, {
+		/* what's the difference to .subdevice = 0x3306 above? */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5168, 0x3502),
+		.driver_data = SAA7134_BOARD_FLYDVBT_HYBRID_CARDBUS,
+	}, {
+		/* FlyDVB-T Hybrid Mini PCI */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5168, 0x3307),
+		.driver_data = SAA7134_BOARD_FLYDVBT_HYBRID_CARDBUS,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x16be, 0x0007),
+		.driver_data = SAA7134_BOARD_MEDION_MD8800_QUADRO,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x16be, 0x0008),
+		.driver_data = SAA7134_BOARD_MEDION_MD8800_QUADRO,
+	}, {
+		/* triple CTX948_V1.1.1 */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x16be, 0x000d),
+		.driver_data = SAA7134_BOARD_MEDION_MD8800_QUADRO,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0x2c05),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_777,
+	}, {
+		/* Cardbus version */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1489, 0x0502),
+		.driver_data = SAA7134_BOARD_FLYDVBT_DUO_CARDBUS,
+	}, {
+		/* Philips Proteus PRO 2309 */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x0919, 0x2003),
+		.driver_data = SAA7134_BOARD_PROTEUS_2309,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1461, 0x2c00),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_A16AR,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1043, 0x4860),
+		.driver_data = SAA7134_BOARD_ASUS_EUROPA2_HYBRID,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x11bd, 0x002f),
+		.driver_data = SAA7134_BOARD_PINNACLE_PCTV_310i,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0x9715),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_STUDIO_507,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1461, 0xa11b),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_STUDIO_507UA,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1043, 0x4876),
+		.driver_data = SAA7134_BOARD_ASUSTeK_P7131_HYBRID_LNA,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x0070, 0x6700),
+		.driver_data = SAA7134_BOARD_HAUPPAUGE_HVR1110,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x0070, 0x6701),
+		.driver_data = SAA7134_BOARD_HAUPPAUGE_HVR1110,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x0070, 0x6702),
+		.driver_data = SAA7134_BOARD_HAUPPAUGE_HVR1110,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x0070, 0x6703),
+		.driver_data = SAA7134_BOARD_HAUPPAUGE_HVR1110,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x0070, 0x6704),
+		.driver_data = SAA7134_BOARD_HAUPPAUGE_HVR1110,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x0070, 0x6705),
+		.driver_data = SAA7134_BOARD_HAUPPAUGE_HVR1110,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x0070, 0x6706),
+		.driver_data = SAA7134_BOARD_HAUPPAUGE_HVR1150,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x0070, 0x6707),
+		.driver_data = SAA7134_BOARD_HAUPPAUGE_HVR1120,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x0070, 0x6708),
+		.driver_data = SAA7134_BOARD_HAUPPAUGE_HVR1150,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x0070, 0x6709),
+		.driver_data = SAA7134_BOARD_HAUPPAUGE_HVR1120,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x0070, 0x670a),
+		.driver_data = SAA7134_BOARD_HAUPPAUGE_HVR1120,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x153b, 0x1172),
+		.driver_data = SAA7134_BOARD_CINERGY_HT_PCMCIA,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130,
+				PCI_VENDOR_ID_PHILIPS, 0x2342),
+		.driver_data = SAA7134_BOARD_ENCORE_ENLTV,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x1131, 0x2341),
+		.driver_data = SAA7134_BOARD_ENCORE_ENLTV,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x3016, 0x2344),
+		.driver_data = SAA7134_BOARD_ENCORE_ENLTV,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x1131, 0x230f),
+		.driver_data = SAA7134_BOARD_ENCORE_ENLTV_FM,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x1a7f, 0x2008),
+		.driver_data = SAA7134_BOARD_ENCORE_ENLTV_FM53,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1a7f, 0x2108),
+		.driver_data = SAA7134_BOARD_ENCORE_ENLTV_FM3,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x153b, 0x1175),
+		.driver_data = SAA7134_BOARD_CINERGY_HT_PCI,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0xf31e),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_M102,
+	}, {
+		/* MSI TV@nywhere DUO */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x4E42, 0x0306),
+		.driver_data = SAA7134_BOARD_FLYDVBTDUO,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1043, 0x4871),
+		.driver_data = SAA7134_BOARD_ASUS_P7131_4871,
+	}, {
+		/* REV:1.00 */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1043, 0x4857),
+		.driver_data = SAA7134_BOARD_ASUSTeK_TIGER,
+	}, {
+		/* SinoVideo PCI 2309 Proteus (7134) OEM cardbus */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x0919, 0x2003),
+		.driver_data = SAA7134_BOARD_SABRENT_TV_PCB05,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130,
+				PCI_VENDOR_ID_PHILIPS, 0x2304),
+		.driver_data = SAA7134_BOARD_10MOONSTVMASTER3,
+	}, {
+		/* Avermedia Technologies Inc AVerTV DVB-T Super 007 */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0xf01d),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_SUPER_007,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x0000, 0x4016),
+		.driver_data = SAA7134_BOARD_BEHOLD_401,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x0000, 0x4036),
+		.driver_data = SAA7134_BOARD_BEHOLD_403,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x0000, 0x4037),
+		.driver_data = SAA7134_BOARD_BEHOLD_403FM,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x0000, 0x4050),
+		.driver_data = SAA7134_BOARD_BEHOLD_405,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x0000, 0x4051),
+		.driver_data = SAA7134_BOARD_BEHOLD_405FM,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x0000, 0x4070),
+		.driver_data = SAA7134_BOARD_BEHOLD_407,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x0000, 0x4071),
+		.driver_data = SAA7134_BOARD_BEHOLD_407FM,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x0000, 0x4090),
+		.driver_data = SAA7134_BOARD_BEHOLD_409,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x0000, 0x505B),
+		.driver_data = SAA7134_BOARD_BEHOLD_505RDS_MK5,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x0000, 0x5051),
+		.driver_data = SAA7134_BOARD_BEHOLD_505RDS_MK3,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x5ace, 0x5050),
+		.driver_data = SAA7134_BOARD_BEHOLD_505FM,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x0000, 0x5071),
+		.driver_data = SAA7134_BOARD_BEHOLD_507RDS_MK3,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x0000, 0x507B),
+		.driver_data = SAA7134_BOARD_BEHOLD_507RDS_MK5,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x5ace, 0x5070),
+		.driver_data = SAA7134_BOARD_BEHOLD_507_9FM,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5ace, 0x5090),
+		.driver_data = SAA7134_BOARD_BEHOLD_507_9FM,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x0000, 0x5201),
+		.driver_data = SAA7134_BOARD_BEHOLD_COLUMBUS_TVFM,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x5ace, 0x6070),
+		.driver_data = SAA7134_BOARD_BEHOLD_607FM_MK3,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x5ace, 0x6071),
+		.driver_data = SAA7134_BOARD_BEHOLD_607FM_MK5,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x5ace, 0x6072),
+		.driver_data = SAA7134_BOARD_BEHOLD_607RDS_MK3,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x5ace, 0x6073),
+		.driver_data = SAA7134_BOARD_BEHOLD_607RDS_MK5,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5ace, 0x6090),
+		.driver_data = SAA7134_BOARD_BEHOLD_609FM_MK3,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5ace, 0x6091),
+		.driver_data = SAA7134_BOARD_BEHOLD_609FM_MK5,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5ace, 0x6092),
+		.driver_data = SAA7134_BOARD_BEHOLD_609RDS_MK3,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5ace, 0x6093),
+		.driver_data = SAA7134_BOARD_BEHOLD_609RDS_MK5,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5ace, 0x6190),
+		.driver_data = SAA7134_BOARD_BEHOLD_M6,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5ace, 0x6193),
+		.driver_data = SAA7134_BOARD_BEHOLD_M6_EXTRA,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5ace, 0x6191),
+		.driver_data = SAA7134_BOARD_BEHOLD_M63,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x4e42, 0x3502),
+		.driver_data = SAA7134_BOARD_FLYDVBT_HYBRID_CARDBUS,
+	}, {
+		/*Twinhan Technology Co. Ltd*/
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1822, 0x0022),
+		.driver_data = SAA7134_BOARD_TWINHAN_DTV_DVB_3056,
+	}, {
+		/* Medion version CTX953_V.1.4.3 */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x16be, 0x0010),
+		.driver_data = SAA7134_BOARD_CREATIX_CTX953,
+	}, {
+		/* MSI TV@nywhere A/D v1.1 */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1462, 0x8625),
+		.driver_data = SAA7134_BOARD_MSI_TVANYWHERE_AD11,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0xf436),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_CARDBUS_506,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0xf936),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_A16D,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0xa836),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_M115,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x185b, 0xc900),
+		.driver_data = SAA7134_BOARD_VIDEOMATE_T750,
+	}, {
+		/* SAA7135HL */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1421, 0x0380),
+		.driver_data = SAA7134_BOARD_ADS_INSTANT_HDTV_PCI,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5169, 0x1502),
+		.driver_data = SAA7134_BOARD_FLYTVPLATINUM_MINI,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5ace, 0x6290),
+		.driver_data = SAA7134_BOARD_BEHOLD_H6,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0xf636),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_M103,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0xf736),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_M103,
+	}, {
+		/* REV:1.02G */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1043, 0x4878),
+		.driver_data = SAA7134_BOARD_ASUSTeK_TIGER_3IN1,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1043, 0x48cd),
+		.driver_data = SAA7134_BOARD_ASUSTeK_PS3_100,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x17de, 0x7128),
+		.driver_data = SAA7134_BOARD_KWORLD_PLUS_TV_ANALOG,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x17de, 0xb136),
+		.driver_data = SAA7134_BOARD_KWORLD_PCI_SBTVD_FULLSEG,
+	}, {
+		/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0xf31d),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_GO_007_FM_PLUS,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x185b, 0xc900),
+		.driver_data = SAA7134_BOARD_VIDEOMATE_S350,
+	}, {
+		/* Beholder Intl. Ltd. */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5ace, 0x7595),
+		.driver_data = SAA7134_BOARD_BEHOLD_X7,
+	}, {
+		/* RoverMedia LifeView FlyTV Prime30 OEM */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x19d1, 0x0138),
+		.driver_data = SAA7134_BOARD_ROVERMEDIA_LINK_PRO_FM,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133,
+				PCI_VENDOR_ID_PHILIPS, 0x2004),
+		.driver_data = SAA7134_BOARD_ZOLID_HYBRID_PCI,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x1043, 0x4847),
+		.driver_data = SAA7134_BOARD_ASUS_EUROPA_HYBRID,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x107d, 0x6655),
+		.driver_data = SAA7134_BOARD_LEADTEK_WINFAST_DTV1000S,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x13c2, 0x2804),
+		.driver_data = SAA7134_BOARD_TECHNOTREND_BUDGET_T3000,
+	}, {
+		/* Beholder Intl. Ltd. */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5ace, 0x7190),
+		.driver_data = SAA7134_BOARD_BEHOLD_H7,
+	}, {
+		/* Beholder Intl. Ltd. */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5ace, 0x7090),
+		.driver_data = SAA7134_BOARD_BEHOLD_A7,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7135, 0x185b, 0xc900),
+		.driver_data = SAA7134_BOARD_VIDEOMATE_M1F,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x5ace, 0x5030),
+		.driver_data = SAA7134_BOARD_BEHOLD_503FM,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x5ace, 0x5010),
+		.driver_data = SAA7134_BOARD_BEHOLD_501,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134, 0x17de, 0xd136),
+		.driver_data = SAA7134_BOARD_MAGICPRO_PROHDTV_PRO2,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x6000, 0x0811),
+		.driver_data = SAA7134_BOARD_SENSORAY811_911,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x6000, 0x0911),
+		.driver_data = SAA7134_BOARD_SENSORAY811_911,
+	}, {
+		/* Avermedia Technologies Inc AverTV Satellite Hybrid+FM A706 */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1461, 0x2055),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_A706,
+	}, {
+		/* WIS */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1905, 0x7007),
+		.driver_data = SAA7134_BOARD_WIS_VOYAGER,
+	}, {
+			/* Avermedia Technologies Inc */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x1461, 0xa10a),
+		.driver_data = SAA7134_BOARD_AVERMEDIA_505,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130, 0x107d, 0x6f3a),
+		.driver_data = SAA7134_BOARD_LEADTEK_WINFAST_TV2100_FM,
+	}, {
+		/* V One Multimedia PTE Ltd */
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x1779, 0x13cf),
+		.driver_data = SAA7134_BOARD_SNAZIO_TVPVR_PRO,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133, 0x107d, 0x6f2e),
+		.driver_data = SAA7134_BOARD_LEADTEK_WINFAST_HDTV200_H,
 	}, {
 		/* --- boards without eeprom + subsystem ID --- */
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = PCI_VENDOR_ID_PHILIPS,
-		.subdevice    = 0,
-		.driver_data  = SAA7134_BOARD_NOAUTO,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = PCI_VENDOR_ID_PHILIPS,
-		.subdevice    = 0,
-		.driver_data  = SAA7134_BOARD_NOAUTO,
-	},{
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134,
+				PCI_VENDOR_ID_PHILIPS, 0),
+		.driver_data = SAA7134_BOARD_NOAUTO,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130,
+				PCI_VENDOR_ID_PHILIPS, 0),
+		.driver_data = SAA7134_BOARD_NOAUTO,
+	}, {
 		/* --- default catch --- */
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7130,
-		.subvendor    = PCI_ANY_ID,
-		.subdevice    = PCI_ANY_ID,
-		.driver_data  = SAA7134_BOARD_UNKNOWN,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7133,
-		.subvendor    = PCI_ANY_ID,
-		.subdevice    = PCI_ANY_ID,
-		.driver_data  = SAA7134_BOARD_UNKNOWN,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7134,
-		.subvendor    = PCI_ANY_ID,
-		.subdevice    = PCI_ANY_ID,
-		.driver_data  = SAA7134_BOARD_UNKNOWN,
-	},{
-		.vendor       = PCI_VENDOR_ID_PHILIPS,
-		.device       = PCI_DEVICE_ID_PHILIPS_SAA7135,
-		.subvendor    = PCI_ANY_ID,
-		.subdevice    = PCI_ANY_ID,
-		.driver_data  = SAA7134_BOARD_UNKNOWN,
-	},{
+		PCI_DEVICE(PCI_VENDOR_ID_PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7130),
+		.driver_data = SAA7134_BOARD_UNKNOWN,
+	}, {
+		PCI_DEVICE(PCI_VENDOR_ID_PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7133),
+		.driver_data = SAA7134_BOARD_UNKNOWN,
+	}, {
+		PCI_DEVICE(PCI_VENDOR_ID_PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7134),
+		.driver_data = SAA7134_BOARD_UNKNOWN,
+	}, {
+		PCI_DEVICE(PCI_VENDOR_ID_PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7135),
+		.driver_data = SAA7134_BOARD_UNKNOWN,
+	}, {
 		/* --- end of list --- */
 	}
 };
diff --git a/drivers/media/pci/saa7146/hexium_gemini.c b/drivers/media/pci/saa7146/hexium_gemini.c
index 1eef4e1029728..f2298e8bb96d6 100644
--- a/drivers/media/pci/saa7146/hexium_gemini.c
+++ b/drivers/media/pci/saa7146/hexium_gemini.c
@@ -361,22 +361,14 @@ static struct saa7146_pci_extension_data hexium_gemini_dual_4bnc = {
 
 static const struct pci_device_id pci_tbl[] = {
 	{
-	 .vendor = PCI_VENDOR_ID_PHILIPS,
-	 .device = PCI_DEVICE_ID_PHILIPS_SAA7146,
-	 .subvendor = 0x17c8,
-	 .subdevice = 0x2401,
-	 .driver_data = (unsigned long) &hexium_gemini_4bnc,
-	 },
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7146, 0x17c8, 0x2401),
+		.driver_data = (unsigned long)&hexium_gemini_4bnc,
+	},
 	{
-	 .vendor = PCI_VENDOR_ID_PHILIPS,
-	 .device = PCI_DEVICE_ID_PHILIPS_SAA7146,
-	 .subvendor = 0x17c8,
-	 .subdevice = 0x2402,
-	 .driver_data = (unsigned long) &hexium_gemini_dual_4bnc,
-	 },
-	{
-	 .vendor = 0,
-	 }
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7146, 0x17c8, 0x2402),
+		.driver_data = (unsigned long)&hexium_gemini_dual_4bnc,
+	},
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, pci_tbl);
diff --git a/drivers/media/pci/saa7146/hexium_orion.c b/drivers/media/pci/saa7146/hexium_orion.c
index 6f1b4bff75964..8ef2a4e8235d1 100644
--- a/drivers/media/pci/saa7146/hexium_orion.c
+++ b/drivers/media/pci/saa7146/hexium_orion.c
@@ -425,29 +425,16 @@ static struct saa7146_pci_extension_data hexium_orion_4bnc = {
 
 static const struct pci_device_id pci_tbl[] = {
 	{
-	 .vendor = PCI_VENDOR_ID_PHILIPS,
-	 .device = PCI_DEVICE_ID_PHILIPS_SAA7146,
-	 .subvendor = 0x0000,
-	 .subdevice = 0x0000,
-	 .driver_data = (unsigned long) &hexium_hv_pci6,
-	 },
-	{
-	 .vendor = PCI_VENDOR_ID_PHILIPS,
-	 .device = PCI_DEVICE_ID_PHILIPS_SAA7146,
-	 .subvendor = 0x17c8,
-	 .subdevice = 0x0101,
-	 .driver_data = (unsigned long) &hexium_orion_1svhs_3bnc,
-	 },
-	{
-	 .vendor = PCI_VENDOR_ID_PHILIPS,
-	 .device = PCI_DEVICE_ID_PHILIPS_SAA7146,
-	 .subvendor = 0x17c8,
-	 .subdevice = 0x2101,
-	 .driver_data = (unsigned long) &hexium_orion_4bnc,
-	 },
-	{
-	 .vendor = 0,
-	 }
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7146, 0x0000, 0x0000),
+		.driver_data = (unsigned long)&hexium_hv_pci6,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7146, 0x17c8, 0x0101),
+		.driver_data = (unsigned long)&hexium_orion_1svhs_3bnc,
+	}, {
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7146, 0x17c8, 0x2101),
+		.driver_data = (unsigned long)&hexium_orion_4bnc,
+	},
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, pci_tbl);
diff --git a/drivers/media/pci/saa7146/mxb.c b/drivers/media/pci/saa7146/mxb.c
index d931b4e3052fc..d759e8a87e247 100644
--- a/drivers/media/pci/saa7146/mxb.c
+++ b/drivers/media/pci/saa7146/mxb.c
@@ -821,14 +821,10 @@ static struct saa7146_pci_extension_data mxb = {
 
 static const struct pci_device_id pci_tbl[] = {
 	{
-		.vendor    = PCI_VENDOR_ID_PHILIPS,
-		.device	   = PCI_DEVICE_ID_PHILIPS_SAA7146,
-		.subvendor = 0x0000,
-		.subdevice = 0x0000,
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7146, 0x0000, 0x0000),
 		.driver_data = (unsigned long)&mxb,
-	}, {
-		.vendor	= 0,
-	}
+	},
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, pci_tbl);
diff --git a/drivers/media/pci/saa7164/saa7164-core.c b/drivers/media/pci/saa7164/saa7164-core.c
index 6bcde506adf50..683bd78288b33 100644
--- a/drivers/media/pci/saa7164/saa7164-core.c
+++ b/drivers/media/pci/saa7164/saa7164-core.c
@@ -1535,10 +1535,7 @@ static void saa7164_finidev(struct pci_dev *pci_dev)
 static const struct pci_device_id saa7164_pci_tbl[] = {
 	{
 		/* SAA7164 */
-		.vendor       = 0x1131,
-		.device       = 0x7164,
-		.subvendor    = PCI_ANY_ID,
-		.subdevice    = PCI_ANY_ID,
+		PCI_DEVICE(0x1131, 0x7164),
 	}, {
 		/* --- end of list --- */
 	}
diff --git a/drivers/media/pci/smipcie/smipcie-main.c b/drivers/media/pci/smipcie/smipcie-main.c
index 387c29958c986..5f18abb48b17d 100644
--- a/drivers/media/pci/smipcie/smipcie-main.c
+++ b/drivers/media/pci/smipcie/smipcie-main.c
@@ -1097,8 +1097,7 @@ static const struct smi_cfg_info technotrend_s2_4200_cfg = {
 
 /* PCI IDs */
 #define SMI_ID(_subvend, _subdev, _driverdata) {	\
-	.vendor      = SMI_VID,    .device    = SMI_PID, \
-	.subvendor   = _subvend, .subdevice = _subdev, \
+	PCI_DEVICE_SUB(SMI_VID, SMI_PID, (_subvend), (_subdev)), \
 	.driver_data = (unsigned long)&_driverdata }
 
 static const struct pci_device_id smi_id_table[] = {
@@ -1106,7 +1105,7 @@ static const struct pci_device_id smi_id_table[] = {
 	SMI_ID(0x4254, 0x0552, dvbsky_s952_cfg),
 	SMI_ID(0x4254, 0x5580, dvbsky_t9580_cfg),
 	SMI_ID(0x13c2, 0x3016, technotrend_s2_4200_cfg),
-	{0}
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, smi_id_table);
 
diff --git a/drivers/media/pci/solo6x10/solo6x10-core.c b/drivers/media/pci/solo6x10/solo6x10-core.c
index 11cddf4b43125..282c2c87151fe 100644
--- a/drivers/media/pci/solo6x10/solo6x10-core.c
+++ b/drivers/media/pci/solo6x10/solo6x10-core.c
@@ -656,7 +656,7 @@ static const struct pci_device_id solo_id_table[] = {
 	  .driver_data = SOLO_DEV_6110 },
 	{ PCI_DEVICE(PCI_VENDOR_ID_BLUECHERRY, PCI_DEVICE_ID_BC_6110_16),
 	  .driver_data = SOLO_DEV_6110 },
-	{0,}
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, solo_id_table);
diff --git a/drivers/media/pci/ttpci/budget-ci.c b/drivers/media/pci/ttpci/budget-ci.c
index 8b496b959d7ea..eb82a670c5bf2 100644
--- a/drivers/media/pci/ttpci/budget-ci.c
+++ b/drivers/media/pci/ttpci/budget-ci.c
@@ -1537,9 +1537,7 @@ static const struct pci_device_id pci_tbl[] = {
 	MAKE_EXTENSION_PCI(ttc1501, 0x13c2, 0x101a),
 	MAKE_EXTENSION_PCI(tt3200, 0x13c2, 0x1019),
 	MAKE_EXTENSION_PCI(ttbs1500b, 0x13c2, 0x101b),
-	{
-	 .vendor = 0,
-	}
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, pci_tbl);
diff --git a/drivers/media/pci/ttpci/budget.c b/drivers/media/pci/ttpci/budget.c
index 95370156aff55..c9f419fc07ad1 100644
--- a/drivers/media/pci/ttpci/budget.c
+++ b/drivers/media/pci/ttpci/budget.c
@@ -858,9 +858,7 @@ static const struct pci_device_id pci_tbl[] = {
 	MAKE_EXTENSION_PCI(fsact, 0x1131, 0x5f61),
 	MAKE_EXTENSION_PCI(omicom, 0x14c4, 0x1020),
 	MAKE_EXTENSION_PCI(sylt, 0x1131, 0x4f52),
-	{
-		.vendor    = 0,
-	}
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, pci_tbl);
diff --git a/drivers/media/pci/tw5864/tw5864-core.c b/drivers/media/pci/tw5864/tw5864-core.c
index 832788603f885..6ec8bce4ac4fe 100644
--- a/drivers/media/pci/tw5864/tw5864-core.c
+++ b/drivers/media/pci/tw5864/tw5864-core.c
@@ -72,8 +72,8 @@ MODULE_PARM_DESC(video_nr, "video devices numbers array");
  * added under vendor 0x1797 (Techwell Inc.) as subsystem IDs.
  */
 static const struct pci_device_id tw5864_pci_tbl[] = {
-	{PCI_DEVICE(PCI_VENDOR_ID_TECHWELL, PCI_DEVICE_ID_TECHWELL_5864)},
-	{0,}
+	{ PCI_VDEVICE(TECHWELL, PCI_DEVICE_ID_TECHWELL_5864) },
+	{ }
 };
 
 void tw5864_irqmask_apply(struct tw5864_dev *dev)
diff --git a/drivers/media/pci/tw68/tw68-core.c b/drivers/media/pci/tw68/tw68-core.c
index 08b7ce1043aaf..509d7ddec1505 100644
--- a/drivers/media/pci/tw68/tw68-core.c
+++ b/drivers/media/pci/tw68/tw68-core.c
@@ -62,14 +62,14 @@ static atomic_t tw68_instance = ATOMIC_INIT(0);
  * added under vendor 0x1797 (Techwell Inc.) as subsystem IDs.
  */
 static const struct pci_device_id tw68_pci_tbl[] = {
-	{PCI_DEVICE(PCI_VENDOR_ID_TECHWELL, PCI_DEVICE_ID_TECHWELL_6800)},
-	{PCI_DEVICE(PCI_VENDOR_ID_TECHWELL, PCI_DEVICE_ID_TECHWELL_6801)},
-	{PCI_DEVICE(PCI_VENDOR_ID_TECHWELL, PCI_DEVICE_ID_TECHWELL_6804)},
-	{PCI_DEVICE(PCI_VENDOR_ID_TECHWELL, PCI_DEVICE_ID_TECHWELL_6816_1)},
-	{PCI_DEVICE(PCI_VENDOR_ID_TECHWELL, PCI_DEVICE_ID_TECHWELL_6816_2)},
-	{PCI_DEVICE(PCI_VENDOR_ID_TECHWELL, PCI_DEVICE_ID_TECHWELL_6816_3)},
-	{PCI_DEVICE(PCI_VENDOR_ID_TECHWELL, PCI_DEVICE_ID_TECHWELL_6816_4)},
-	{0,}
+	{ PCI_VDEVICE(TECHWELL, PCI_DEVICE_ID_TECHWELL_6800) },
+	{ PCI_VDEVICE(TECHWELL, PCI_DEVICE_ID_TECHWELL_6801) },
+	{ PCI_VDEVICE(TECHWELL, PCI_DEVICE_ID_TECHWELL_6804) },
+	{ PCI_VDEVICE(TECHWELL, PCI_DEVICE_ID_TECHWELL_6816_1) },
+	{ PCI_VDEVICE(TECHWELL, PCI_DEVICE_ID_TECHWELL_6816_2) },
+	{ PCI_VDEVICE(TECHWELL, PCI_DEVICE_ID_TECHWELL_6816_3) },
+	{ PCI_VDEVICE(TECHWELL, PCI_DEVICE_ID_TECHWELL_6816_4) },
+	{ }
 };
 
 /* ------------------------------------------------------------------ */
diff --git a/drivers/media/pci/tw686x/tw686x-core.c b/drivers/media/pci/tw686x/tw686x-core.c
index a10e382218171..5a4ab329c06e9 100644
--- a/drivers/media/pci/tw686x/tw686x-core.c
+++ b/drivers/media/pci/tw686x/tw686x-core.c
@@ -416,25 +416,24 @@ static void tw686x_remove(struct pci_dev *pci_dev)
 /* driver_data is number of A/V channels */
 static const struct pci_device_id tw686x_pci_tbl[] = {
 	{
-		PCI_DEVICE(PCI_VENDOR_ID_TECHWELL, 0x6864),
-		.driver_data = 4
-	},
-	{
-		PCI_DEVICE(PCI_VENDOR_ID_TECHWELL, 0x6865), /* not tested */
-		.driver_data = 4 | TYPE_SECOND_GEN
+		PCI_VDEVICE(TECHWELL, 0x6864),
+		.driver_data = 4,
+	}, {
+		PCI_VDEVICE(TECHWELL, 0x6865), /* not tested */
+		.driver_data = 4 | TYPE_SECOND_GEN,
 	},
 	/*
 	 * TW6868 supports 8 A/V channels with an external TW2865 chip;
 	 * not supported by the driver.
 	 */
 	{
-		PCI_DEVICE(PCI_VENDOR_ID_TECHWELL, 0x6868), /* not tested */
-		.driver_data = 4
+		PCI_VDEVICE(TECHWELL, 0x6868), /* not tested */
+		.driver_data = 4,
+	}, {
+		PCI_VDEVICE(TECHWELL, 0x6869),
+		.driver_data = 8 | TYPE_SECOND_GEN
 	},
-	{
-		PCI_DEVICE(PCI_VENDOR_ID_TECHWELL, 0x6869),
-		.driver_data = 8 | TYPE_SECOND_GEN},
-	{}
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, tw686x_pci_tbl);
 
diff --git a/drivers/media/pci/zoran/zoran_card.c b/drivers/media/pci/zoran/zoran_card.c
index f707bdc1fb0f1..8bcf355a54185 100644
--- a/drivers/media/pci/zoran/zoran_card.c
+++ b/drivers/media/pci/zoran/zoran_card.c
@@ -75,8 +75,9 @@ MODULE_LICENSE("GPL");
 MODULE_VERSION(ZORAN_VERSION);
 
 #define ZR_DEVICE(subven, subdev, data)	{ \
-	.vendor = PCI_VENDOR_ID_ZORAN, .device = PCI_DEVICE_ID_ZORAN_36057, \
-	.subvendor = (subven), .subdevice = (subdev), .driver_data = (data) }
+	PCI_DEVICE_SUB(PCI_VENDOR_ID_ZORAN, PCI_DEVICE_ID_ZORAN_36057, \
+		       (subven), (subdev)), \
+	.driver_data = (data) }
 
 static const struct pci_device_id zr36067_pci_tbl[] = {
 	ZR_DEVICE(PCI_VENDOR_ID_MIRO, PCI_DEVICE_ID_MIRO_DC10PLUS, DC10_PLUS),
@@ -84,7 +85,7 @@ static const struct pci_device_id zr36067_pci_tbl[] = {
 	ZR_DEVICE(PCI_VENDOR_ID_ELECTRONICDESIGNGMBH, PCI_DEVICE_ID_LML_33R10, LML33R10),
 	ZR_DEVICE(PCI_VENDOR_ID_IOMEGA, PCI_DEVICE_ID_IOMEGA_BUZ, BUZ),
 	ZR_DEVICE(PCI_ANY_ID, PCI_ANY_ID, NUM_CARDS),
-	{0}
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, zr36067_pci_tbl);
 
diff --git a/drivers/media/platform/amd/isp4/Kconfig b/drivers/media/platform/amd/isp4/Kconfig
index 9d1927af1cb8c..77b61fae82bab 100644
--- a/drivers/media/platform/amd/isp4/Kconfig
+++ b/drivers/media/platform/amd/isp4/Kconfig
@@ -5,6 +5,7 @@ config VIDEO_AMD_ISP4_CAPTURE
 	depends on DRM_AMDGPU && DRM_AMD_ISP
 	depends on HAS_DMA
 	depends on VIDEO_DEV
+	select MEDIA_CONTROLLER
 	select VIDEOBUF2_CORE
 	select VIDEOBUF2_MEMOPS
 	select VIDEOBUF2_V4L2
diff --git a/drivers/media/platform/amd/isp4/isp4_interface.c b/drivers/media/platform/amd/isp4/isp4_interface.c
index 8d73f66bb42cb..00a8179092921 100644
--- a/drivers/media/platform/amd/isp4/isp4_interface.c
+++ b/drivers/media/platform/amd/isp4/isp4_interface.c
@@ -375,7 +375,7 @@ static int isp4if_send_fw_cmd(struct isp4_interface *ispif, u32 cmd_id,
 			return -ENOMEM;
 
 		/* Get two references: one for the resp thread, one for us */
-		atomic_set(&ele->refcnt, 2);
+		refcount_set(&ele->refcnt, 2);
 		init_completion(&ele->cmd_done);
 	}
 
@@ -455,7 +455,7 @@ static int isp4if_send_fw_cmd(struct isp4_interface *ispif, u32 cmd_id,
 
 put_ele_ref:
 	/* Don't free the command if we didn't put the last reference */
-	if (ele && atomic_dec_return(&ele->refcnt))
+	if (ele && !refcount_dec_and_test(&ele->refcnt))
 		ele = NULL;
 
 free_ele:
diff --git a/drivers/media/platform/amd/isp4/isp4_interface.h b/drivers/media/platform/amd/isp4/isp4_interface.h
index ce3ac9b9e5cda..04db71cd54e6c 100644
--- a/drivers/media/platform/amd/isp4/isp4_interface.h
+++ b/drivers/media/platform/amd/isp4/isp4_interface.h
@@ -68,7 +68,7 @@ struct isp4if_cmd_element {
 	u32 seq_num;
 	u32 cmd_id;
 	struct completion cmd_done;
-	atomic_t refcnt;
+	refcount_t refcnt;
 };
 
 struct isp4_interface {
diff --git a/drivers/media/platform/amd/isp4/isp4_subdev.c b/drivers/media/platform/amd/isp4/isp4_subdev.c
index 48deea79ce6c2..2a8bc12078434 100644
--- a/drivers/media/platform/amd/isp4/isp4_subdev.c
+++ b/drivers/media/platform/amd/isp4/isp4_subdev.c
@@ -391,7 +391,7 @@ static void isp4sd_fw_resp_cmd_done(struct isp4_subdev *isp_subdev,
 
 	if (ele) {
 		complete(&ele->cmd_done);
-		if (atomic_dec_and_test(&ele->refcnt))
+		if (refcount_dec_and_test(&ele->refcnt))
 			kfree(ele);
 	}
 }
diff --git a/drivers/media/platform/amphion/venc.c b/drivers/media/platform/amphion/venc.c
index 0b3d58b9f2f7a..2ba4ae20f8292 100644
--- a/drivers/media/platform/amphion/venc.c
+++ b/drivers/media/platform/amphion/venc.c
@@ -973,8 +973,6 @@ static int venc_start_session(struct vpu_inst *inst, u32 type)
 	venc->ready_count = 0;
 	venc->stopped = false;
 	vpu_process_output_buffer(inst);
-	if (venc->frame_count == 0)
-		dev_err(inst->dev, "[%d] there is no input when starting\n", inst->id);
 
 	return 0;
 error:
diff --git a/drivers/media/platform/broadcom/bcm2835-unicam.c b/drivers/media/platform/broadcom/bcm2835-unicam.c
index 8d28ba0b59a39..1508843ae58cb 100644
--- a/drivers/media/platform/broadcom/bcm2835-unicam.c
+++ b/drivers/media/platform/broadcom/bcm2835-unicam.c
@@ -2613,6 +2613,7 @@ static int unicam_async_nf_init(struct unicam_device *unicam)
 	return 0;
 
 error:
+	v4l2_async_nf_cleanup(&unicam->notifier);
 	fwnode_handle_put(ep_handle);
 	return ret;
 }
@@ -2745,6 +2746,7 @@ static void unicam_remove(struct platform_device *pdev)
 	v4l2_device_unregister(&unicam->v4l2_dev);
 	media_device_unregister(&unicam->mdev);
 	v4l2_async_nf_unregister(&unicam->notifier);
+	v4l2_async_nf_cleanup(&unicam->notifier);
 
 	unicam_subdev_cleanup(unicam);
 
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
index bb2ba9204a836..6564cf3ec739b 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2021-2023 CHIPS&MEDIA INC
  */
 
+#include <linux/delay.h>
 #include <linux/pm_runtime.h>
 #include "wave5-helper.h"
 
@@ -474,7 +475,10 @@ static void wave5_vpu_dec_finish_decode(struct vpu_instance *inst)
 			v4l2_m2m_job_finish(inst->v4l2_m2m_dev, m2m_ctx);
 	}
 
-	inst->queuing_fail = false;
+	if (inst->queuing_fail) {
+		inst->queuing_fail = false;
+		v4l2_m2m_try_schedule(m2m_ctx);
+	}
 }
 
 static int wave5_vpu_dec_querycap(struct file *file, void *fh, struct v4l2_capability *cap)
@@ -819,7 +823,15 @@ static int wave5_vpu_dec_stop(struct vpu_instance *inst)
 		 * calls do not block on a mutex while inside this spinlock.
 		 */
 		spin_unlock_irqrestore(&inst->state_spinlock, flags);
+		/*
+		 * V4L2_DEC_CMD_STOP can arrive while the device is runtime
+		 * suspended (e.g. on pipeline teardown). Setting the EOS flag
+		 * accesses VPU registers via send_firmware_command(), so the
+		 * device must be resumed first to avoid an asynchronous SError.
+		 */
+		pm_runtime_resume_and_get(inst->dev->dev);
 		ret = wave5_vpu_dec_set_eos_on_firmware(inst);
+		pm_runtime_put_autosuspend(inst->dev->dev);
 		if (ret)
 			return ret;
 
@@ -1403,6 +1415,7 @@ static int wave5_vpu_dec_start_streaming(struct vb2_queue *q, unsigned int count
 	} else if (q->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) {
 		struct dec_initial_info *initial_info =
 			&inst->codec_info->dec_info.initial_info;
+		struct dec_info *p_dec_info = &inst->codec_info->dec_info;
 
 		if (inst->state == VPU_INST_STATE_STOP)
 			ret = switch_state(inst, VPU_INST_STATE_INIT_SEQ);
@@ -1410,6 +1423,7 @@ static int wave5_vpu_dec_start_streaming(struct vb2_queue *q, unsigned int count
 			goto return_buffers;
 
 		if (inst->state == VPU_INST_STATE_INIT_SEQ &&
+		    p_dec_info->initial_info_obtained &&
 		    inst->dev->product_code == WAVE521C_CODE) {
 			if (initial_info->luma_bitdepth != 8) {
 				dev_info(inst->dev->dev, "%s: no support for %d bit depth",
@@ -1418,7 +1432,6 @@ static int wave5_vpu_dec_start_streaming(struct vb2_queue *q, unsigned int count
 				goto return_buffers;
 			}
 		}
-
 	}
 	pm_runtime_put_autosuspend(inst->dev->dev);
 	return ret;
@@ -1536,15 +1549,15 @@ static void wave5_vpu_dec_stop_streaming(struct vb2_queue *q)
 {
 	struct vpu_instance *inst = vb2_get_drv_priv(q);
 	struct v4l2_m2m_ctx *m2m_ctx = inst->v4l2_fh.m2m_ctx;
-
-	bool check_cmd = TRUE;
+	unsigned long timeout;
 
 	dev_dbg(inst->dev->dev, "%s: type: %u\n", __func__, q->type);
 	pm_runtime_resume_and_get(inst->dev->dev);
 	inst->empty_queue = true;
-	while (check_cmd) {
+
+	timeout = jiffies + msecs_to_jiffies(VPU_DEC_STOP_TIMEOUT);
+	while (true) {
 		struct queue_status_info q_status;
-		struct dec_output_info dec_output_info;
 
 		wave5_vpu_dec_give_command(inst, DEC_GET_QUEUE_STATUS, &q_status);
 		if ((inst->state == VPU_INST_STATE_STOP ||
@@ -1553,8 +1566,10 @@ static void wave5_vpu_dec_stop_streaming(struct vb2_queue *q)
 			q_status.report_queue_count == 0)
 			break;
 
-		if (wave5_vpu_dec_get_output_info(inst, &dec_output_info))
-			dev_dbg(inst->dev->dev, "there is no output info\n");
+		if (time_after(jiffies, timeout))
+			break;
+
+		usleep_range(1000, 2000);
 	}
 
 	v4l2_m2m_update_stop_streaming_state(m2m_ctx, q);
@@ -1651,6 +1666,7 @@ static void wave5_vpu_dec_device_run(void *priv)
 	struct queue_status_info q_status;
 	u32 fail_res = 0;
 	int ret = 0;
+	bool cmd_issued = false;
 
 	dev_dbg(inst->dev->dev, "%s: Fill the ring buffer with new bitstream data", __func__);
 	pm_runtime_resume_and_get(inst->dev->dev);
@@ -1662,9 +1678,13 @@ static void wave5_vpu_dec_device_run(void *priv)
 		} else if (!inst->eos &&
 				inst->queuing_num == 0 &&
 				inst->state == VPU_INST_STATE_PIC_RUN) {
-			dev_dbg(inst->dev->dev, "%s: no bitstream for feeding, so skip ", __func__);
-			inst->empty_queue = true;
-			goto finish_job_and_return;
+			wave5_vpu_dec_give_command(inst, DEC_GET_QUEUE_STATUS, &q_status);
+			if (q_status.instance_queue_count == v4l2_m2m_num_src_bufs_ready(m2m_ctx)) {
+				dev_dbg(inst->dev->dev, "%s: no bitstream, skip\n",
+					__func__);
+				inst->empty_queue = true;
+				goto finish_job_and_return;
+			}
 		}
 	}
 
@@ -1744,6 +1764,7 @@ static void wave5_vpu_dec_device_run(void *priv)
 			inst->retry = false;
 			if (!inst->eos)
 				inst->queuing_num--;
+			cmd_issued = true;
 		}
 		break;
 	default:
@@ -1761,8 +1782,16 @@ static void wave5_vpu_dec_device_run(void *priv)
 	 * in power and CPU time.
 	 * If EOS is passed, device_run will not call job_finish no more, it is called
 	 * only if HW is idle status in order to reduce overhead.
+	 *
+	 * Deferring job_finish() is only safe when this run actually queued a
+	 * DEC_PIC command (cmd_issued): that guarantees a completion IRQ, and
+	 * thus a later finish_decode(), will release the shared job slot. When
+	 * device_run() is entered with no command to issue (e.g. a job that was
+	 * queued while draining but reached the STOP state by the time it ran),
+	 * no IRQ follows, so finish the job here to avoid leaking the slot and
+	 * stalling every instance sharing the VPU.
 	 */
-	if (!inst->sent_eos)
+	if (!inst->sent_eos || !cmd_issued)
 		v4l2_m2m_job_finish(inst->v4l2_m2m_dev, m2m_ctx);
 }
 
@@ -1776,11 +1805,22 @@ static void wave5_vpu_dec_job_abort(void *priv)
 	if (ret)
 		return;
 
+	/*
+	 * job_abort() runs from the STREAMOFF path and may be called while the
+	 * device is runtime suspended. Setting the EOS flag talks to the
+	 * firmware (send_firmware_command() accesses VPU registers), so the
+	 * device must be resumed first; otherwise the register access faults
+	 * with an asynchronous SError.
+	 */
+	pm_runtime_resume_and_get(inst->dev->dev);
+
 	ret = wave5_vpu_dec_set_eos_on_firmware(inst);
 	if (ret)
 		dev_warn(inst->dev->dev,
 			 "Setting EOS for the bitstream, fail: %d\n", ret);
 
+	pm_runtime_put_autosuspend(inst->dev->dev);
+
 	v4l2_m2m_job_finish(inst->v4l2_m2m_dev, m2m_ctx);
 }
 
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
index e6c94b6f26711..f9fcdf4c224b3 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
@@ -1494,7 +1494,8 @@ static const struct vb2_ops wave5_vpu_enc_vb2_ops = {
 	.stop_streaming = wave5_vpu_enc_stop_streaming,
 };
 
-static void wave5_set_default_format(struct v4l2_pix_format_mplane *src_fmt,
+static void wave5_set_default_format(struct vpu_instance *inst,
+				     struct v4l2_pix_format_mplane *src_fmt,
 				     struct v4l2_pix_format_mplane *dst_fmt)
 {
 	src_fmt->pixelformat = enc_fmt_list[VPU_FMT_TYPE_RAW][0].v4l2_pix_fmt;
@@ -1506,6 +1507,7 @@ static void wave5_set_default_format(struct v4l2_pix_format_mplane *src_fmt,
 	wave5_update_pix_fmt(dst_fmt, VPU_FMT_TYPE_CODEC,
 			     W5_DEF_ENC_PIC_WIDTH, W5_DEF_ENC_PIC_HEIGHT,
 			     &enc_frmsize[VPU_FMT_TYPE_CODEC]);
+	inst->std = wave5_to_vpu_std(dst_fmt->pixelformat, inst->type);
 }
 
 static int wave5_vpu_enc_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_vq)
@@ -1770,7 +1772,7 @@ static int wave5_vpu_open_enc(struct file *filp)
 	inst->v4l2_fh.ctrl_handler = v4l2_ctrl_hdl;
 	v4l2_ctrl_handler_setup(v4l2_ctrl_hdl);
 
-	wave5_set_default_format(&inst->src_fmt, &inst->dst_fmt);
+	wave5_set_default_format(inst, &inst->src_fmt, &inst->dst_fmt);
 	inst->conf_win.width = inst->dst_fmt.width;
 	inst->conf_win.height = inst->dst_fmt.height;
 	inst->colorspace = V4L2_COLORSPACE_REC709;
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuconfig.h b/drivers/media/platform/chips-media/wave5/wave5-vpuconfig.h
index 4ebd48d5550ea..e04f2dbf3b655 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpuconfig.h
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpuconfig.h
@@ -59,7 +59,7 @@
 //  application specific configuration
 #define VPU_ENC_TIMEOUT                 60000
 #define VPU_DEC_TIMEOUT                 60000
-#define VPU_DEC_STOP_TIMEOUT            10
+#define VPU_DEC_STOP_TIMEOUT            300
 
 // for WAVE encoder
 #define USE_SRC_PRP_AXI         0
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-cmdq.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-cmdq.c
index d30a05782ab9e..96a66aadf0cdc 100644
--- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-cmdq.c
+++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-cmdq.c
@@ -321,7 +321,7 @@ static int mdp_path_config_subfrm(struct mdp_cmdq_cmd *cmd,
 	/* Enable mux settings */
 	for (index = 0; index < ctrl->num_sets; index++) {
 		set = &ctrl->sets[index];
-		cmdq_pkt_write(&cmd->pkt, set->subsys_id, set->reg, set->value);
+		cmdq_pkt_write_subsys(&cmd->pkt, set->subsys_id, set->reg, set->reg, set->value);
 	}
 	/* Config sub-frame information */
 	for (index = (num_comp - 1); index >= 0; index--) {
@@ -376,7 +376,7 @@ static int mdp_path_config_subfrm(struct mdp_cmdq_cmd *cmd,
 	/* Disable mux settings */
 	for (index = 0; index < ctrl->num_sets; index++) {
 		set = &ctrl->sets[index];
-		cmdq_pkt_write(&cmd->pkt, set->subsys_id, set->reg, 0);
+		cmdq_pkt_write_subsys(&cmd->pkt, set->subsys_id, set->reg, set->reg, 0);
 	}
 
 	return 0;
@@ -628,7 +628,7 @@ static struct mdp_cmdq_cmd *mdp_cmdq_prepare(struct mdp_dev *mdp,
 		goto err_free_path;
 	}
 	cmdq_pkt_eoc(&cmd->pkt);
-	cmdq_pkt_jump_rel(&cmd->pkt, CMDQ_INST_SIZE, mdp->cmdq_shift_pa[pp_idx]);
+	cmdq_pkt_jump_rel_temp(&cmd->pkt, CMDQ_INST_SIZE, mdp->cmdq_shift_pa[pp_idx]);
 
 	for (i = 0; i < num_comp; i++) {
 		s32 inner_id = MDP_COMP_NONE;
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.h b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.h
index 681906c164194..c6fc180950f24 100644
--- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.h
+++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.h
@@ -12,14 +12,14 @@
 #define MM_REG_WRITE_MASK(cmd, id, base, ofst, val, mask)	\
 do {								\
 	typeof(mask) (m) = (mask);				\
-	cmdq_pkt_write_mask(&((cmd)->pkt), id, (base) + (ofst),	\
-			    (val),				\
+	cmdq_pkt_write_mask_subsys(&((cmd)->pkt), (id), (base),	\
+				   (base) + (ofst), (val),	\
 		(((m) & (ofst##_MASK)) == (ofst##_MASK)) ?	\
 			(0xffffffff) : (m));			\
 } while (0)
 
 #define MM_REG_WRITE(cmd, id, base, ofst, val)			\
-	cmdq_pkt_write(&((cmd)->pkt), id, (base) + (ofst), (val))
+	cmdq_pkt_write_subsys(&((cmd)->pkt), (id), (base), (base) + (ofst), (val))
 
 #define MM_REG_WAIT(cmd, evt)					\
 do {								\
diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c
index 8f4da4cf55d26..f412aadb5996b 100644
--- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c
+++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c
@@ -294,7 +294,11 @@ static int mdp_probe(struct platform_device *pdev)
 			goto err_destroy_clock_wq;
 		}
 		mdp->scp = platform_get_drvdata(mm_pdev);
-		put_device(&mm_pdev->dev);
+	}
+
+	if (!mdp->scp) {
+		ret = -EPROBE_DEFER;
+		goto err_destroy_clock_wq;
 	}
 
 	mdp->rproc_handle = scp_get_rproc(mdp->scp);
diff --git a/drivers/media/platform/mediatek/vcodec/decoder/vdec/vdec_vp9_req_lat_if.c b/drivers/media/platform/mediatek/vcodec/decoder/vdec/vdec_vp9_req_lat_if.c
index adbacabdbebc1..ba7a1cd998b95 100644
--- a/drivers/media/platform/mediatek/vcodec/decoder/vdec/vdec_vp9_req_lat_if.c
+++ b/drivers/media/platform/mediatek/vcodec/decoder/vdec/vdec_vp9_req_lat_if.c
@@ -974,10 +974,10 @@ static int vdec_vp9_slice_setup_pfc(struct vdec_vp9_slice_instance *instance,
 	return 0;
 }
 
-static int vdec_vp9_slice_setup_lat_buffer(struct vdec_vp9_slice_instance *instance,
-					   struct vdec_vp9_slice_vsi *vsi,
-					   struct mtk_vcodec_mem *bs,
-					   struct vdec_lat_buf *lat_buf)
+static void vdec_vp9_slice_setup_lat_buffer(struct vdec_vp9_slice_instance *instance,
+					    struct vdec_vp9_slice_vsi *vsi,
+					    struct mtk_vcodec_mem *bs,
+					    struct vdec_lat_buf *lat_buf)
 {
 	int i;
 
@@ -1011,12 +1011,10 @@ static int vdec_vp9_slice_setup_lat_buffer(struct vdec_vp9_slice_instance *insta
 
 	vsi->row_info.buf = 0;
 	vsi->row_info.size = 0;
-
-	return 0;
 }
 
-static int vdec_vp9_slice_setup_prob_buffer(struct vdec_vp9_slice_instance *instance,
-					    struct vdec_vp9_slice_vsi *vsi)
+static void vdec_vp9_slice_setup_prob_buffer(struct vdec_vp9_slice_instance *instance,
+					     struct vdec_vp9_slice_vsi *vsi)
 {
 	struct vdec_vp9_slice_frame_ctx *frame_ctx;
 	struct vdec_vp9_slice_uncompressed_header *uh;
@@ -1032,8 +1030,6 @@ static int vdec_vp9_slice_setup_prob_buffer(struct vdec_vp9_slice_instance *inst
 	else
 		frame_ctx = vdec_vp9_slice_default_frame_ctx;
 	memcpy(instance->prob.va, frame_ctx, sizeof(*frame_ctx));
-
-	return 0;
 }
 
 static void vdec_vp9_slice_setup_seg_buffer(struct vdec_vp9_slice_instance *instance,
@@ -1166,17 +1162,13 @@ static int vdec_vp9_slice_setup_lat(struct vdec_vp9_slice_instance *instance,
 	if (ret)
 		goto err;
 
-	ret = vdec_vp9_slice_setup_lat_buffer(instance, vsi, bs, lat_buf);
-	if (ret)
-		goto err;
+	vdec_vp9_slice_setup_lat_buffer(instance, vsi, bs, lat_buf);
 
 	vdec_vp9_slice_setup_seg_buffer(instance, vsi, &instance->seg[0]);
 
 	/* setup prob/tile buffers for LAT */
 
-	ret = vdec_vp9_slice_setup_prob_buffer(instance, vsi);
-	if (ret)
-		goto err;
+	vdec_vp9_slice_setup_prob_buffer(instance, vsi);
 
 	ret = vdec_vp9_slice_setup_tile_buffer(instance, vsi, bs);
 	if (ret)
@@ -1808,10 +1800,7 @@ static int vdec_vp9_slice_setup_single(struct vdec_vp9_slice_instance *instance,
 
 	vdec_vp9_slice_setup_single_buffer(instance, pfc, vsi, bs, fb);
 	vdec_vp9_slice_setup_seg_buffer(instance, vsi, &instance->seg[0]);
-
-	ret = vdec_vp9_slice_setup_prob_buffer(instance, vsi);
-	if (ret)
-		goto err;
+	vdec_vp9_slice_setup_prob_buffer(instance, vsi);
 
 	ret = vdec_vp9_slice_setup_tile_buffer(instance, vsi, bs);
 	if (ret)
diff --git a/drivers/media/platform/qcom/iris/iris_vpu_buffer.c b/drivers/media/platform/qcom/iris/iris_vpu_buffer.c
index fb6f1016415e2..faebb54728660 100644
--- a/drivers/media/platform/qcom/iris/iris_vpu_buffer.c
+++ b/drivers/media/platform/qcom/iris/iris_vpu_buffer.c
@@ -580,7 +580,7 @@ static u32 hfi_buffer_line_av1d(u32 frame_width, u32 frame_height,
 		ALIGN(size_av1d_qp(frame_width, frame_height), DMA_ALIGNMENT);
 	opbwr8 = size_av1d_lb_opb_wr1_nv12_ubwc(frame_width, frame_height);
 	opbwr10 = size_av1d_lb_opb_wr1_tp10_ubwc(frame_width, frame_height);
-	opbwrbufsize = opbwr8 >= opbwr10 ? opbwr8 : opbwr10;
+	opbwrbufsize = max(opbwr8, opbwr10);
 	size = ALIGN((size + opbwrbufsize), DMA_ALIGNMENT);
 	if (is_opb) {
 		vpss_lb_size = size_vpss_lb(frame_width, frame_height);
@@ -700,7 +700,7 @@ static u32 hfi_buffer_ibc_av1d(u32 frame_width, u32 frame_height)
 
 	ibc8 = size_av1d_ibc_nv12_ubwc(frame_width, frame_height);
 	ibc10 = size_av1d_ibc_tp10_ubwc(frame_width, frame_height);
-	size = ibc8 >= ibc10 ? ibc8 : ibc10;
+	size = max(ibc8, ibc10);
 
 	return ALIGN(size, DMA_ALIGNMENT);
 }
diff --git a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru.h b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru.h
index 5bf334e173d2b..b426bc7898bf7 100644
--- a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru.h
+++ b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru.h
@@ -116,11 +116,9 @@ struct rzg2l_cru_info {
  * @scratch_phys:	physical address of the scratch buffer
  *
  * @qlock:		protects @queue_buf, @buf_list, @sequence
- *			@state
  * @queue_buf:		Keeps track of buffers given to HW slot
  * @buf_list:		list of queued buffers
  * @sequence:		V4L2 buffers sequence number
- * @state:		keeps track of operation state
  *
  * @format:		active V4L2 pixel format
  */
diff --git a/drivers/media/platform/rockchip/rga/rga3-hw.c b/drivers/media/platform/rockchip/rga/rga3-hw.c
index ca1c268303dd4..c4a71306278ba 100644
--- a/drivers/media/platform/rockchip/rga/rga3-hw.c
+++ b/drivers/media/platform/rockchip/rga/rga3-hw.c
@@ -450,10 +450,14 @@ static void *rga3_adjust_and_map_format(struct rga_ctx *ctx,
 	other_format = is_output ? &ctx->out.pix : &ctx->in.pix;
 	other_format_info = v4l2_format_info(other_format->pixelformat);
 
-	if ((v4l2_is_format_rgb(format_info) &&
-	     v4l2_is_format_yuv(other_format_info)) ||
-	    (v4l2_is_format_yuv(format_info) &&
-	     v4l2_is_format_rgb(other_format_info))) {
+	/*
+	 * Only apply the quantization restrictions when we need to
+	 * convert between RGB and YUV. Otherwise there is no point
+	 * to limit the quantization for operations like scaling or
+	 * rotations.
+	 */
+	if (v4l2_is_format_yuv(format_info) &&
+	    v4l2_is_format_rgb(other_format_info)) {
 		/*
 		 * The RGA3 only supports BT601, BT709 and BT2020 RGB<->YUV conversions
 		 * Additionally BT709 and BT2020 only support limited range YUV.
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c
index f89602075121b..9c4a6093af324 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c
@@ -209,6 +209,9 @@ static void st_ref_pic_set_prediction(struct rkvdec_hevc_run *run, int idx,
 	int i, j;
 	int dPoc;
 
+	if ((unsigned int)rps_data->delta_idx_minus1 + 1 > idx)
+		return;
+
 	ref_rps_idx = st_rps_idx - (rps_data->delta_idx_minus1 + 1); /* 7-59 */
 	delta_rps = (1 - 2 * rps_data->delta_rps_sign) *
 		   (rps_data->abs_delta_rps_minus1 + 1); /* 7-60 */
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc.c
index 87abf93dfd5e2..ff3942f91c5df 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc.c
@@ -258,9 +258,9 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 		for (i = 0; i <= pps->num_tile_rows_minus1; i++)
 			WRITE_PPS(pps->row_height_minus1[i], ROW_HEIGHT(i));
 	} else {
-		WRITE_PPS(((sps->pic_width_in_luma_samples + ctb_size_y - 1) / ctb_size_y) - 1,
+		WRITE_PPS(DIV_ROUND_UP(sps->pic_width_in_luma_samples, ctb_size_y) - 1,
 			  COLUMN_WIDTH(0));
-		WRITE_PPS(((sps->pic_height_in_luma_samples + ctb_size_y - 1) / ctb_size_y) - 1,
+		WRITE_PPS(DIV_ROUND_UP(sps->pic_height_in_luma_samples, ctb_size_y) - 1,
 			  ROW_HEIGHT(0));
 	}
 
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu381-hevc.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu381-hevc.c
index fe6414a175510..d07c74679552c 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu381-hevc.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu381-hevc.c
@@ -261,8 +261,8 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 	memset(row_height, 0, sizeof(row_height));
 
 	max_cu_width = 1 << (sps->log2_diff_max_min_luma_coding_block_size + log2_min_cb_size);
-	pic_in_cts_width = (width + max_cu_width - 1) / max_cu_width;
-	pic_in_cts_height = (height + max_cu_width - 1) / max_cu_width;
+	pic_in_cts_width = DIV_ROUND_UP(width, max_cu_width);
+	pic_in_cts_height = DIV_ROUND_UP(height, max_cu_width);
 
 	if (pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED) {
 		if (pps->flags & V4L2_HEVC_PPS_FLAG_UNIFORM_SPACING) {
@@ -275,8 +275,8 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 						  column_width, row_height);
 		}
 	} else {
-		column_width[0] = (width + max_cu_width - 1) / max_cu_width;
-		row_height[0] = (height + max_cu_width - 1) / max_cu_width;
+		column_width[0] = DIV_ROUND_UP(width, max_cu_width);
+		row_height[0] = DIV_ROUND_UP(height, max_cu_width);
 	}
 
 	for (i = 0; i < 20; i++) {
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c
index 3575338a531a4..3462d995d4cf0 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c
@@ -287,8 +287,8 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 	memset(row_height, 0, sizeof(row_height));
 
 	max_cu_width = 1 << (sps->log2_diff_max_min_luma_coding_block_size + log2_min_cb_size);
-	pic_in_cts_width = (width + max_cu_width - 1) / max_cu_width;
-	pic_in_cts_height = (height + max_cu_width - 1) / max_cu_width;
+	pic_in_cts_width = DIV_ROUND_UP(width, max_cu_width);
+	pic_in_cts_height = DIV_ROUND_UP(height, max_cu_width);
 
 	if (tiles_enabled) {
 		if (pps->flags & V4L2_HEVC_PPS_FLAG_UNIFORM_SPACING) {
@@ -301,8 +301,8 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 						  column_width, row_height);
 		}
 	} else {
-		column_width[0] = (width + max_cu_width - 1) / max_cu_width;
-		row_height[0] = (height + max_cu_width - 1) / max_cu_width;
+		column_width[0] = DIV_ROUND_UP(width, max_cu_width);
+		row_height[0] = DIV_ROUND_UP(height, max_cu_width);
 	}
 
 	for (i = 0; i < 20; i++)
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec.c b/drivers/media/platform/rockchip/rkvdec/rkvdec.c
index 1d1e9bfef8e96..061281f903f3c 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec.c
@@ -278,12 +278,12 @@ static const struct rkvdec_ctrl_desc vdpu38x_hevc_ctrl_descs[] = {
 	{
 		.cfg.id = V4L2_CID_STATELESS_HEVC_EXT_SPS_ST_RPS,
 		.cfg.ops = &rkvdec_ctrl_ops,
-		.cfg.dims = { 65 },
+		.cfg.dims = { 64 },
 	},
 	{
 		.cfg.id = V4L2_CID_STATELESS_HEVC_EXT_SPS_LT_RPS,
 		.cfg.ops = &rkvdec_ctrl_ops,
-		.cfg.dims = { 65 },
+		.cfg.dims = { 32 },
 	},
 };
 
@@ -1818,8 +1818,8 @@ static int rkvdec_probe(struct platform_device *pdev)
 	vb2_dma_contig_set_max_seg_size(&pdev->dev, DMA_BIT_MASK(32));
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq <= 0)
-		return -ENXIO;
+	if (irq < 0)
+		return irq;
 
 	ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
 					rkvdec_irq_handler, IRQF_ONESHOT,
diff --git a/drivers/media/platform/samsung/s3c-camif/camif-core.c b/drivers/media/platform/samsung/s3c-camif/camif-core.c
index 221e3c447f361..14eedd1ceb278 100644
--- a/drivers/media/platform/samsung/s3c-camif/camif-core.c
+++ b/drivers/media/platform/samsung/s3c-camif/camif-core.c
@@ -12,7 +12,6 @@
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/errno.h>
-#include <linux/gpio.h>
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
@@ -412,7 +411,7 @@ static int s3c_camif_probe(struct platform_device *pdev)
 
 	camif->dev = dev;
 
-	if (!pdata || !pdata->gpio_get || !pdata->gpio_put) {
+	if (!pdata) {
 		dev_err(dev, "wrong platform data\n");
 		return -EINVAL;
 	}
@@ -429,9 +428,7 @@ static int s3c_camif_probe(struct platform_device *pdev)
 	if (ret < 0)
 		return ret;
 
-	ret = pdata->gpio_get();
-	if (ret < 0)
-		return ret;
+	/* FIXME: get GPIOs here */
 
 	ret = s3c_camif_create_subdev(camif);
 	if (ret < 0)
@@ -504,14 +501,12 @@ static int s3c_camif_probe(struct platform_device *pdev)
 err_clk:
 	s3c_camif_unregister_subdev(camif);
 err_sd:
-	pdata->gpio_put();
 	return ret;
 }
 
 static void s3c_camif_remove(struct platform_device *pdev)
 {
 	struct camif_dev *camif = platform_get_drvdata(pdev);
-	struct s3c_camif_plat_data *pdata = &camif->pdata;
 
 	media_device_unregister(&camif->media_dev);
 	media_device_cleanup(&camif->media_dev);
@@ -521,7 +516,6 @@ static void s3c_camif_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 	camif_clk_put(camif);
 	s3c_camif_unregister_subdev(camif);
-	pdata->gpio_put();
 }
 
 static int s3c_camif_runtime_resume(struct device *dev)
diff --git a/drivers/media/platform/st/stm32/stm32-dcmi.c b/drivers/media/platform/st/stm32/stm32-dcmi.c
index eeb0199864dd1..b1f9af8edf6e0 100644
--- a/drivers/media/platform/st/stm32/stm32-dcmi.c
+++ b/drivers/media/platform/st/stm32/stm32-dcmi.c
@@ -2273,9 +2273,7 @@ static int dcmi_resume(struct device *dev)
 	pinctrl_pm_select_default_state(dev);
 
 	/* clock enable */
-	pm_runtime_force_resume(dev);
-
-	return 0;
+	return pm_runtime_force_resume(dev);
 }
 
 static const struct dev_pm_ops dcmi_pm_ops = {
diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c
index 49398d0777646..516334541b2c8 100644
--- a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c
+++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c
@@ -649,9 +649,7 @@ static int dcmipp_resume(struct device *dev)
 	pinctrl_pm_select_default_state(dev);
 
 	/* clock enable */
-	pm_runtime_force_resume(dev);
-
-	return 0;
+	return pm_runtime_force_resume(dev);
 }
 
 static const struct dev_pm_ops dcmipp_pm_ops = {
diff --git a/drivers/media/platform/ti/Kconfig b/drivers/media/platform/ti/Kconfig
index d0cb05481bd85..1a020b2bbb4f2 100644
--- a/drivers/media/platform/ti/Kconfig
+++ b/drivers/media/platform/ti/Kconfig
@@ -50,6 +50,7 @@ config VIDEO_TI_VIP
         select VIDEO_TI_VPDMA
         select VIDEO_TI_SC
         select VIDEO_TI_CSC
+        select V4L2_FWNODE
         help
         Driver support for VIP module on certain TI SoC's
         VIP = Video Input Port.
diff --git a/drivers/media/platform/verisilicon/hantro_drv.c b/drivers/media/platform/verisilicon/hantro_drv.c
index 2e81877f640fb..ad71c0402ef3b 100644
--- a/drivers/media/platform/verisilicon/hantro_drv.c
+++ b/drivers/media/platform/verisilicon/hantro_drv.c
@@ -771,8 +771,10 @@ static int hantro_register_entity(struct media_device *mdev,
 		return ret;
 
 	ret = media_device_register_entity(mdev, entity);
-	if (ret)
+	if (ret) {
+		media_entity_cleanup(entity);
 		return ret;
+	}
 
 	return 0;
 }
@@ -860,12 +862,13 @@ static int hantro_attach_func(struct hantro_dev *vpu,
 
 err_rel_entity2:
 	media_device_unregister_entity(&func->sink);
-
+	media_entity_cleanup(&func->sink);
 err_rel_entity1:
 	media_device_unregister_entity(&func->proc);
-
+	media_entity_cleanup(&func->proc);
 err_rel_entity0:
 	media_device_unregister_entity(&func->vdev.entity);
+	media_entity_cleanup(&func->vdev.entity);
 	return ret;
 }
 
@@ -878,6 +881,9 @@ static void hantro_detach_func(struct hantro_func *func)
 	media_device_unregister_entity(&func->sink);
 	media_device_unregister_entity(&func->proc);
 	media_device_unregister_entity(&func->vdev.entity);
+	media_entity_cleanup(&func->sink);
+	media_entity_cleanup(&func->proc);
+	media_entity_cleanup(&func->vdev.entity);
 }
 
 static int hantro_add_func(struct hantro_dev *vpu, unsigned int funcid)
diff --git a/drivers/media/radio/radio-cadet.c b/drivers/media/radio/radio-cadet.c
index 5110754e1a317..53c1d1061cf47 100644
--- a/drivers/media/radio/radio-cadet.c
+++ b/drivers/media/radio/radio-cadet.c
@@ -527,8 +527,8 @@ static const struct v4l2_ctrl_ops cadet_ctrl_ops = {
 
 static const struct pnp_device_id cadet_pnp_devices[] = {
 	/* ADS Cadet AM/FM Radio Card */
-	{.id = "MSM0c24", .driver_data = 0},
-	{.id = ""}
+	{ .id = "MSM0c24" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pnp, cadet_pnp_devices);
diff --git a/drivers/media/radio/radio-gemtek.c b/drivers/media/radio/radio-gemtek.c
index 3d1d0b46195b8..7e4257e4b71d1 100644
--- a/drivers/media/radio/radio-gemtek.c
+++ b/drivers/media/radio/radio-gemtek.c
@@ -284,8 +284,8 @@ static const int gemtek_ioports[] = { 0x20c, 0x30c, 0x24c, 0x34c, 0x248, 0x28c }
 #ifdef CONFIG_PNP
 static const struct pnp_device_id gemtek_pnp_devices[] = {
 	/* AOpen FX-3D/Pro Radio */
-	{.id = "ADS7183", .driver_data = 0},
-	{.id = ""}
+	{ .id = "ADS7183" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pnp, gemtek_pnp_devices);
diff --git a/drivers/media/rc/fintek-cir.c b/drivers/media/rc/fintek-cir.c
index 5055dfc3f4651..c196ee923ecd3 100644
--- a/drivers/media/rc/fintek-cir.c
+++ b/drivers/media/rc/fintek-cir.c
@@ -642,8 +642,8 @@ static void fintek_shutdown(struct pnp_dev *pdev)
 }
 
 static const struct pnp_device_id fintek_ids[] = {
-	{ "FIT0002", 0 },   /* CIR */
-	{ "", 0 },
+	{ .id = "FIT0002" },   /* CIR */
+	{ }
 };
 
 static struct pnp_driver fintek_driver = {
diff --git a/drivers/media/rc/ite-cir.c b/drivers/media/rc/ite-cir.c
index bde2a70512310..1fbafcd8219e5 100644
--- a/drivers/media/rc/ite-cir.c
+++ b/drivers/media/rc/ite-cir.c
@@ -1203,9 +1203,16 @@ static void ite_close(struct rc_dev *rcdev)
 	spin_unlock_irqrestore(&dev->lock, flags);
 }
 
+enum ite_model_type {
+	ITE8704,
+	ITE8713,
+	ITE8708,
+	ITE8709,
+};
+
 /* supported models and their parameters */
 static const struct ite_dev_params ite_dev_descs[] = {
-	{	/* 0: ITE8704 */
+	[ITE8704] = {
 	       .model = "ITE8704 CIR transceiver",
 	       .io_region_size = IT87_IOREG_LENGTH,
 	       .io_rsrc_no = 0,
@@ -1224,7 +1231,7 @@ static const struct ite_dev_params ite_dev_descs[] = {
 	       .init_hardware = it87_init_hardware,
 	       .set_carrier_params = it87_set_carrier_params,
 	       },
-	{	/* 1: ITE8713 */
+	[ITE8713] = {
 	       .model = "ITE8713 CIR transceiver",
 	       .io_region_size = IT87_IOREG_LENGTH,
 	       .io_rsrc_no = 0,
@@ -1243,7 +1250,7 @@ static const struct ite_dev_params ite_dev_descs[] = {
 	       .init_hardware = it87_init_hardware,
 	       .set_carrier_params = it87_set_carrier_params,
 	       },
-	{	/* 2: ITE8708 */
+	[ITE8708] = {
 	       .model = "ITE8708 CIR transceiver",
 	       .io_region_size = IT8708_IOREG_LENGTH,
 	       .io_rsrc_no = 0,
@@ -1263,7 +1270,7 @@ static const struct ite_dev_params ite_dev_descs[] = {
 	       .init_hardware = it8708_init_hardware,
 	       .set_carrier_params = it8708_set_carrier_params,
 	       },
-	{	/* 3: ITE8709 */
+	[ITE8709] = {
 	       .model = "ITE8709 CIR transceiver",
 	       .io_region_size = IT8709_IOREG_LENGTH,
 	       .io_rsrc_no = 2,
@@ -1286,11 +1293,11 @@ static const struct ite_dev_params ite_dev_descs[] = {
 };
 
 static const struct pnp_device_id ite_ids[] = {
-	{"ITE8704", 0},		/* Default model */
-	{"ITE8713", 1},		/* CIR found in EEEBox 1501U */
-	{"ITE8708", 2},		/* Bridged IT8512 */
-	{"ITE8709", 3},		/* SRAM-Bridged IT8512 */
-	{"", 0},
+	{ .id = "ITE8704", .driver_data = ITE8704 },		/* Default model */
+	{ .id = "ITE8713", .driver_data = ITE8713 },		/* CIR found in EEEBox 1501U */
+	{ .id = "ITE8708", .driver_data = ITE8708 },		/* Bridged IT8512 */
+	{ .id = "ITE8709", .driver_data = ITE8709 },		/* SRAM-Bridged IT8512 */
+	{ }
 };
 
 /* allocate memory, probe hardware, and initialize everything */
@@ -1301,7 +1308,7 @@ static int ite_probe(struct pnp_dev *pdev, const struct pnp_device_id
 	struct ite_dev *itdev = NULL;
 	struct rc_dev *rdev = NULL;
 	int ret = -ENOMEM;
-	int model_no;
+	enum ite_model_type model_no;
 	int io_rsrc_no;
 
 	itdev = kzalloc_obj(struct ite_dev);
@@ -1317,7 +1324,7 @@ static int ite_probe(struct pnp_dev *pdev, const struct pnp_device_id
 	ret = -ENODEV;
 
 	/* get the model number */
-	model_no = (int)dev_id->driver_data;
+	model_no = dev_id->driver_data;
 	dev_dbg(&pdev->dev, "Auto-detected model: %s\n",
 		ite_dev_descs[model_no].model);
 
diff --git a/drivers/media/rc/keymaps/Makefile b/drivers/media/rc/keymaps/Makefile
index d04572627cdd0..38b84e34b2255 100644
--- a/drivers/media/rc/keymaps/Makefile
+++ b/drivers/media/rc/keymaps/Makefile
@@ -129,7 +129,7 @@ obj-$(CONFIG_RC_MAP) += \
 			rc-twinhan1027.o \
 			rc-twinhan-dtv-cab-ci.o \
 			rc-vega-s9x.o \
-			rc-videomate-m1f.o \
+			rc-videomate-k100.o \
 			rc-videomate-s350.o \
 			rc-videomate-tv-pvr.o \
 			rc-videostrong-kii-pro.o \
diff --git a/drivers/media/rc/keymaps/rc-videomate-k100.c b/drivers/media/rc/keymaps/rc-videomate-k100.c
new file mode 100644
index 0000000000000..6664cfd0b7f3a
--- /dev/null
+++ b/drivers/media/rc/keymaps/rc-videomate-k100.c
@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/* videomate-k100.h - Keytable for videomate_k100 Remote Controller
+ *
+ * keymap imported from ir-keymaps.c
+ *
+ * Copyright (c) 2010 by Pavel Osnova <pvosnova@gmail.com>
+ */
+
+#include <media/rc-map.h>
+#include <linux/module.h>
+
+static struct rc_map_table videomate_k100[] = {
+	{ 0x01, KEY_POWER },
+	{ 0x31, KEY_TUNER },
+	{ 0x33, KEY_VIDEO },
+	{ 0x2f, KEY_RADIO },
+	{ 0x30, KEY_CAMERA },
+	{ 0x2d, KEY_NEW }, /* TV record button */
+	{ 0x17, KEY_CYCLEWINDOWS },
+	{ 0x2c, KEY_ANGLE },
+	{ 0x2b, KEY_LANGUAGE },
+	{ 0x32, KEY_SEARCH }, /* '...' button */
+	{ 0x11, KEY_UP },
+	{ 0x13, KEY_LEFT },
+	{ 0x15, KEY_OK },
+	{ 0x14, KEY_RIGHT },
+	{ 0x12, KEY_DOWN },
+	{ 0x16, KEY_BACKSPACE },
+	{ 0x02, KEY_ZOOM }, /* WIN key */
+	{ 0x04, KEY_INFO },
+	{ 0x05, KEY_VOLUMEUP },
+	{ 0x03, KEY_MUTE },
+	{ 0x07, KEY_CHANNELUP },
+	{ 0x06, KEY_VOLUMEDOWN },
+	{ 0x08, KEY_CHANNELDOWN },
+	{ 0x0c, KEY_RECORD },
+	{ 0x0e, KEY_STOP },
+	{ 0x0a, KEY_BACK },
+	{ 0x0b, KEY_PLAY },
+	{ 0x09, KEY_FORWARD },
+	{ 0x10, KEY_PREVIOUS },
+	{ 0x0d, KEY_PAUSE },
+	{ 0x0f, KEY_NEXT },
+	{ 0x1e, KEY_NUMERIC_1 },
+	{ 0x1f, KEY_NUMERIC_2 },
+	{ 0x20, KEY_NUMERIC_3 },
+	{ 0x21, KEY_NUMERIC_4 },
+	{ 0x22, KEY_NUMERIC_5 },
+	{ 0x23, KEY_NUMERIC_6 },
+	{ 0x24, KEY_NUMERIC_7 },
+	{ 0x25, KEY_NUMERIC_8 },
+	{ 0x26, KEY_NUMERIC_9 },
+	{ 0x2a, KEY_NUMERIC_STAR }, /* * key */
+	{ 0x1d, KEY_NUMERIC_0 },
+	{ 0x29, KEY_SUBTITLE }, /* # key */
+	{ 0x27, KEY_CLEAR },
+	{ 0x34, KEY_SCREEN },
+	{ 0x28, KEY_ENTER },
+	{ 0x19, KEY_RED },
+	{ 0x1a, KEY_GREEN },
+	{ 0x1b, KEY_YELLOW },
+	{ 0x1c, KEY_BLUE },
+	{ 0x18, KEY_TEXT },
+};
+
+static struct rc_map_list videomate_k100_map = {
+	.map = {
+		.scan     = videomate_k100,
+		.size     = ARRAY_SIZE(videomate_k100),
+		.rc_proto = RC_PROTO_UNKNOWN,     /* Legacy IR type */
+		.name     = RC_MAP_VIDEOMATE_K100,
+	}
+};
+
+static int __init init_rc_map_videomate_k100(void)
+{
+	return rc_map_register(&videomate_k100_map);
+}
+
+static void __exit exit_rc_map_videomate_k100(void)
+{
+	rc_map_unregister(&videomate_k100_map);
+}
+
+module_init(init_rc_map_videomate_k100)
+module_exit(exit_rc_map_videomate_k100)
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Pavel Osnova <pvosnova@gmail.com>");
+MODULE_DESCRIPTION("videomate-k100 remote controller keytable");
diff --git a/drivers/media/rc/keymaps/rc-videomate-m1f.c b/drivers/media/rc/keymaps/rc-videomate-m1f.c
deleted file mode 100644
index 1f9be84ff27b2..0000000000000
--- a/drivers/media/rc/keymaps/rc-videomate-m1f.c
+++ /dev/null
@@ -1,90 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/* videomate-k100.h - Keytable for videomate_k100 Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Pavel Osnova <pvosnova@gmail.com>
- */
-
-#include <media/rc-map.h>
-#include <linux/module.h>
-
-static struct rc_map_table videomate_k100[] = {
-	{ 0x01, KEY_POWER },
-	{ 0x31, KEY_TUNER },
-	{ 0x33, KEY_VIDEO },
-	{ 0x2f, KEY_RADIO },
-	{ 0x30, KEY_CAMERA },
-	{ 0x2d, KEY_NEW }, /* TV record button */
-	{ 0x17, KEY_CYCLEWINDOWS },
-	{ 0x2c, KEY_ANGLE },
-	{ 0x2b, KEY_LANGUAGE },
-	{ 0x32, KEY_SEARCH }, /* '...' button */
-	{ 0x11, KEY_UP },
-	{ 0x13, KEY_LEFT },
-	{ 0x15, KEY_OK },
-	{ 0x14, KEY_RIGHT },
-	{ 0x12, KEY_DOWN },
-	{ 0x16, KEY_BACKSPACE },
-	{ 0x02, KEY_ZOOM }, /* WIN key */
-	{ 0x04, KEY_INFO },
-	{ 0x05, KEY_VOLUMEUP },
-	{ 0x03, KEY_MUTE },
-	{ 0x07, KEY_CHANNELUP },
-	{ 0x06, KEY_VOLUMEDOWN },
-	{ 0x08, KEY_CHANNELDOWN },
-	{ 0x0c, KEY_RECORD },
-	{ 0x0e, KEY_STOP },
-	{ 0x0a, KEY_BACK },
-	{ 0x0b, KEY_PLAY },
-	{ 0x09, KEY_FORWARD },
-	{ 0x10, KEY_PREVIOUS },
-	{ 0x0d, KEY_PAUSE },
-	{ 0x0f, KEY_NEXT },
-	{ 0x1e, KEY_NUMERIC_1 },
-	{ 0x1f, KEY_NUMERIC_2 },
-	{ 0x20, KEY_NUMERIC_3 },
-	{ 0x21, KEY_NUMERIC_4 },
-	{ 0x22, KEY_NUMERIC_5 },
-	{ 0x23, KEY_NUMERIC_6 },
-	{ 0x24, KEY_NUMERIC_7 },
-	{ 0x25, KEY_NUMERIC_8 },
-	{ 0x26, KEY_NUMERIC_9 },
-	{ 0x2a, KEY_NUMERIC_STAR }, /* * key */
-	{ 0x1d, KEY_NUMERIC_0 },
-	{ 0x29, KEY_SUBTITLE }, /* # key */
-	{ 0x27, KEY_CLEAR },
-	{ 0x34, KEY_SCREEN },
-	{ 0x28, KEY_ENTER },
-	{ 0x19, KEY_RED },
-	{ 0x1a, KEY_GREEN },
-	{ 0x1b, KEY_YELLOW },
-	{ 0x1c, KEY_BLUE },
-	{ 0x18, KEY_TEXT },
-};
-
-static struct rc_map_list videomate_k100_map = {
-	.map = {
-		.scan     = videomate_k100,
-		.size     = ARRAY_SIZE(videomate_k100),
-		.rc_proto = RC_PROTO_UNKNOWN,     /* Legacy IR type */
-		.name     = RC_MAP_VIDEOMATE_K100,
-	}
-};
-
-static int __init init_rc_map_videomate_k100(void)
-{
-	return rc_map_register(&videomate_k100_map);
-}
-
-static void __exit exit_rc_map_videomate_k100(void)
-{
-	rc_map_unregister(&videomate_k100_map);
-}
-
-module_init(init_rc_map_videomate_k100)
-module_exit(exit_rc_map_videomate_k100)
-
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Pavel Osnova <pvosnova@gmail.com>");
-MODULE_DESCRIPTION("videomate-m1f remote controller keytable");
diff --git a/drivers/media/rc/nuvoton-cir.c b/drivers/media/rc/nuvoton-cir.c
index 4e5a0c8dc9a0a..918258dadd698 100644
--- a/drivers/media/rc/nuvoton-cir.c
+++ b/drivers/media/rc/nuvoton-cir.c
@@ -1092,9 +1092,9 @@ static void nvt_shutdown(struct pnp_dev *pdev)
 }
 
 static const struct pnp_device_id nvt_ids[] = {
-	{ "WEC0530", 0 },   /* CIR */
-	{ "NTN0530", 0 },   /* CIR for new chip's pnp id*/
-	{ "", 0 },
+	{ .id = "WEC0530" },   /* CIR */
+	{ .id = "NTN0530" },   /* CIR for new chip's pnp id*/
+	{ }
 };
 
 static struct pnp_driver nvt_driver = {
diff --git a/drivers/media/rc/sunxi-cir.c b/drivers/media/rc/sunxi-cir.c
index cb4c56bf0752a..28e840a7e5b8c 100644
--- a/drivers/media/rc/sunxi-cir.c
+++ b/drivers/media/rc/sunxi-cir.c
@@ -344,22 +344,25 @@ static int sunxi_ir_probe(struct platform_device *pdev)
 	ir->irq = platform_get_irq(pdev, 0);
 	if (ir->irq < 0) {
 		ret = ir->irq;
-		goto exit_free_dev;
+		goto exit_unregister_dev;
 	}
 
 	ret = devm_request_irq(dev, ir->irq, sunxi_ir_irq, 0, SUNXI_IR_DEV, ir);
 	if (ret) {
 		dev_err(dev, "failed request irq\n");
-		goto exit_free_dev;
+		goto exit_unregister_dev;
 	}
 
 	ret = sunxi_ir_hw_init(dev);
 	if (ret)
-		goto exit_free_dev;
+		goto exit_unregister_dev;
 
 	dev_info(dev, "initialized sunXi IR driver\n");
 	return 0;
 
+exit_unregister_dev:
+	rc_unregister_device(ir->rc);
+
 exit_free_dev:
 	rc_free_device(ir->rc);
 
diff --git a/drivers/media/rc/winbond-cir.c b/drivers/media/rc/winbond-cir.c
index 8e804661a6215..1ae036f8a2967 100644
--- a/drivers/media/rc/winbond-cir.c
+++ b/drivers/media/rc/winbond-cir.c
@@ -1179,8 +1179,8 @@ wbcir_remove(struct pnp_dev *device)
 }
 
 static const struct pnp_device_id wbcir_ids[] = {
-	{ "WEC1022", 0 },
-	{ "", 0 }
+	{ .id = "WEC1022" },
+	{ }
 };
 MODULE_DEVICE_TABLE(pnp, wbcir_ids);
 
diff --git a/drivers/media/test-drivers/vicodec/vicodec-core.c b/drivers/media/test-drivers/vicodec/vicodec-core.c
index 318e8330f16ae..ff9d50fb05fd1 100644
--- a/drivers/media/test-drivers/vicodec/vicodec-core.c
+++ b/drivers/media/test-drivers/vicodec/vicodec-core.c
@@ -63,11 +63,11 @@ struct pixfmt_info {
 };
 
 static const struct v4l2_fwht_pixfmt_info pixfmt_fwht = {
-	V4L2_PIX_FMT_FWHT, 0, 3, 1, 1, 1, 1, 1, 0, 1
+	V4L2_PIX_FMT_FWHT, 0, 4, 1, 1, 1, 1, 1, 0, 1
 };
 
 static const struct v4l2_fwht_pixfmt_info pixfmt_stateless_fwht = {
-	V4L2_PIX_FMT_FWHT_STATELESS, 0, 3, 1, 1, 1, 1, 1, 0, 1
+	V4L2_PIX_FMT_FWHT_STATELESS, 0, 4, 1, 1, 1, 1, 1, 0, 1
 };
 
 static void vicodec_dev_release(struct device *dev)
diff --git a/drivers/media/test-drivers/vimc/vimc-sensor.c b/drivers/media/test-drivers/vimc/vimc-sensor.c
index 5deebcc78a33b..83dcc9d61ee05 100644
--- a/drivers/media/test-drivers/vimc/vimc-sensor.c
+++ b/drivers/media/test-drivers/vimc/vimc-sensor.c
@@ -92,8 +92,8 @@ static void vimc_sensor_tpg_s_format(struct vimc_sensor_device *vsensor,
 	tpg_s_xfer_func(&vsensor->tpg, format->xfer_func);
 }
 
-static int vimc_sensor_update_frame_timing(struct v4l2_subdev *sd,
-					   u32 width, u32 height)
+static void vimc_sensor_update_frame_timing(struct v4l2_subdev *sd,
+					    u32 width, u32 height)
 {
 	struct vimc_sensor_device *vsensor =
 		container_of(sd, struct vimc_sensor_device, sd);
@@ -103,13 +103,15 @@ static int vimc_sensor_update_frame_timing(struct v4l2_subdev *sd,
 	u64 total_pixels = (u64)hts * vts;
 	u64 frame_interval_ns;
 
+	/* Sanity check, pixel rate is fixed and fits in 32 bits. */
+	if (WARN_ON(pixel_rate >= 0x100000000))
+		return;
+
 	frame_interval_ns = total_pixels * NSEC_PER_SEC;
-	do_div(frame_interval_ns, pixel_rate);
+	do_div(frame_interval_ns, (u32)pixel_rate);
 	vsensor->hw.fps_jiffies = nsecs_to_jiffies(frame_interval_ns);
 	if (vsensor->hw.fps_jiffies == 0)
 		vsensor->hw.fps_jiffies = 1;
-
-	return 0;
 }
 
 static void vimc_sensor_adjust_fmt(struct v4l2_mbus_framefmt *fmt)
diff --git a/drivers/media/test-drivers/vivid/vivid-meta-out.c b/drivers/media/test-drivers/vivid/vivid-meta-out.c
index 55e5e5dec2f2a..ca913b808225b 100644
--- a/drivers/media/test-drivers/vivid/vivid-meta-out.c
+++ b/drivers/media/test-drivers/vivid/vivid-meta-out.c
@@ -17,12 +17,8 @@ static int meta_out_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers,
 				unsigned int *nplanes, unsigned int sizes[],
 				struct device *alloc_devs[])
 {
-	struct vivid_dev *dev = vb2_get_drv_priv(vq);
 	unsigned int size =  sizeof(struct vivid_meta_out_buf);
 
-	if (!vivid_is_webcam(dev))
-		return -EINVAL;
-
 	if (*nplanes) {
 		if (sizes[0] < size)
 			return -EINVAL;
@@ -127,11 +123,6 @@ const struct vb2_ops vivid_meta_out_qops = {
 int vidioc_enum_fmt_meta_out(struct file *file, void  *priv,
 			     struct v4l2_fmtdesc *f)
 {
-	struct vivid_dev *dev = video_drvdata(file);
-
-	if (!vivid_is_webcam(dev))
-		return -EINVAL;
-
 	if (f->index > 0)
 		return -EINVAL;
 
@@ -143,12 +134,8 @@ int vidioc_enum_fmt_meta_out(struct file *file, void  *priv,
 int vidioc_g_fmt_meta_out(struct file *file, void *priv,
 			  struct v4l2_format *f)
 {
-	struct vivid_dev *dev = video_drvdata(file);
 	struct v4l2_meta_format *meta = &f->fmt.meta;
 
-	if (!vivid_is_webcam(dev) || !dev->has_meta_out)
-		return -EINVAL;
-
 	meta->dataformat = V4L2_META_FMT_VIVID;
 	meta->buffersize = sizeof(struct vivid_meta_out_buf);
 	return 0;
@@ -159,10 +146,12 @@ void vivid_meta_out_process(struct vivid_dev *dev,
 {
 	struct vivid_meta_out_buf *meta = vb2_plane_vaddr(&buf->vb.vb2_buf, 0);
 
-	v4l2_ctrl_s_ctrl(dev->brightness, meta->brightness);
-	v4l2_ctrl_s_ctrl(dev->contrast, meta->contrast);
-	v4l2_ctrl_s_ctrl(dev->saturation, meta->saturation);
-	v4l2_ctrl_s_ctrl(dev->hue, meta->hue);
+	if (dev->brightness) {
+		v4l2_ctrl_s_ctrl(dev->brightness, meta->brightness);
+		v4l2_ctrl_s_ctrl(dev->contrast, meta->contrast);
+		v4l2_ctrl_s_ctrl(dev->saturation, meta->saturation);
+		v4l2_ctrl_s_ctrl(dev->hue, meta->hue);
+	}
 
 	dprintk(dev, 2, " %s brightness %u contrast %u saturation %u hue %d\n",
 		__func__, meta->brightness, meta->contrast,
diff --git a/drivers/media/usb/airspy/airspy.c b/drivers/media/usb/airspy/airspy.c
index 57edb42463e81..358a66ab8e48a 100644
--- a/drivers/media/usb/airspy/airspy.c
+++ b/drivers/media/usb/airspy/airspy.c
@@ -464,14 +464,21 @@ static void airspy_disconnect(struct usb_interface *intf)
 
 	dev_dbg(s->dev, "\n");
 
-	mutex_lock(&s->vb_queue_lock);
+	/*
+	 * vb2_video_unregister_device() releases the vb2 queue, which
+	 * triggers airspy_stop_streaming() if streaming is active.
+	 * stop_streaming() dereferences s->udev via airspy_ctrl_msg() and
+	 * airspy_free_stream_bufs(), so it must run before s->udev is
+	 * cleared. vb2_video_unregister_device() locks vb_queue_lock
+	 * internally and stop_streaming() locks v4l2_lock, so neither may
+	 * be held by the caller.
+	 */
+	v4l2_device_disconnect(&s->v4l2_dev);
+	vb2_video_unregister_device(&s->vdev);
+
 	mutex_lock(&s->v4l2_lock);
-	/* No need to keep the urbs around after disconnection */
 	s->udev = NULL;
-	v4l2_device_disconnect(&s->v4l2_dev);
-	video_unregister_device(&s->vdev);
 	mutex_unlock(&s->v4l2_lock);
-	mutex_unlock(&s->vb_queue_lock);
 
 	v4l2_device_put(&s->v4l2_dev);
 }
diff --git a/drivers/media/usb/as102/as102_usb_drv.c b/drivers/media/usb/as102/as102_usb_drv.c
index a11024451cebd..be2f8be560fae 100644
--- a/drivers/media/usb/as102/as102_usb_drv.c
+++ b/drivers/media/usb/as102/as102_usb_drv.c
@@ -24,35 +24,33 @@ static void as102_usb_stop_stream(struct as102_dev_t *dev);
 static int as102_open(struct inode *inode, struct file *file);
 static int as102_release(struct inode *inode, struct file *file);
 
-static const struct usb_device_id as102_usb_id_table[] = {
-	{ USB_DEVICE(AS102_USB_DEVICE_VENDOR_ID, AS102_USB_DEVICE_PID_0001) },
-	{ USB_DEVICE(PCTV_74E_USB_VID, PCTV_74E_USB_PID) },
-	{ USB_DEVICE(ELGATO_EYETV_DTT_USB_VID, ELGATO_EYETV_DTT_USB_PID) },
-	{ USB_DEVICE(NBOX_DVBT_DONGLE_USB_VID, NBOX_DVBT_DONGLE_USB_PID) },
-	{ USB_DEVICE(SKY_IT_DIGITAL_KEY_USB_VID, SKY_IT_DIGITAL_KEY_USB_PID) },
-	{ } /* Terminating entry */
+struct as102_dev_info {
+	const char *name;
+	/*
+	 * eLNA configuration: devices built on the reference design work best
+	 * with 0xA0, while custom designs seem to require 0xC0
+	 */
+	uint8_t elna_cfg;
 };
 
-/* Note that this table must always have the same number of entries as the
-   as102_usb_id_table struct */
-static const char * const as102_device_names[] = {
-	AS102_REFERENCE_DESIGN,
-	AS102_PCTV_74E,
-	AS102_ELGATO_EYETV_DTT_NAME,
-	AS102_NBOX_DVBT_DONGLE_NAME,
-	AS102_SKY_IT_DIGITAL_KEY_NAME,
-	NULL /* Terminating entry */
-};
+#define DRIVER_INFO(dev_name, dev_elna_cfg) \
+	.driver_info = (kernel_ulong_t)&(const struct as102_dev_info){ \
+		.name = (dev_name), \
+		.elna_cfg = (dev_elna_cfg), \
+	}
 
-/* eLNA configuration: devices built on the reference design work best
-   with 0xA0, while custom designs seem to require 0xC0 */
-static uint8_t const as102_elna_cfg[] = {
-	0xA0,
-	0xC0,
-	0xC0,
-	0xA0,
-	0xA0,
-	0x00 /* Terminating entry */
+static const struct usb_device_id as102_usb_id_table[] = {
+	{ USB_DEVICE(AS102_USB_DEVICE_VENDOR_ID, AS102_USB_DEVICE_PID_0001),
+	  DRIVER_INFO(AS102_REFERENCE_DESIGN, 0xA0) },
+	{ USB_DEVICE(PCTV_74E_USB_VID, PCTV_74E_USB_PID),
+	  DRIVER_INFO(AS102_PCTV_74E, 0xC0) },
+	{ USB_DEVICE(ELGATO_EYETV_DTT_USB_VID, ELGATO_EYETV_DTT_USB_PID),
+	  DRIVER_INFO(AS102_ELGATO_EYETV_DTT_NAME, 0xC0) },
+	{ USB_DEVICE(NBOX_DVBT_DONGLE_USB_VID, NBOX_DVBT_DONGLE_USB_PID),
+	  DRIVER_INFO(AS102_NBOX_DVBT_DONGLE_NAME, 0xA0) },
+	{ USB_DEVICE(SKY_IT_DIGITAL_KEY_USB_VID, SKY_IT_DIGITAL_KEY_USB_PID),
+	  DRIVER_INFO(AS102_SKY_IT_DIGITAL_KEY_NAME, 0xA0) },
+	{ } /* Terminating entry */
 };
 
 struct usb_driver as102_usb_driver = {
@@ -336,29 +334,18 @@ static int as102_usb_probe(struct usb_interface *intf,
 {
 	int ret;
 	struct as102_dev_t *as102_dev;
-	int i;
-
-	/* This should never actually happen */
-	if (ARRAY_SIZE(as102_usb_id_table) !=
-	    (sizeof(as102_device_names) / sizeof(const char *))) {
-		pr_err("Device names table invalid size");
-		return -EINVAL;
-	}
+	const struct as102_dev_info *info = (const struct as102_dev_info *)id->driver_info;
 
 	as102_dev = kzalloc_obj(struct as102_dev_t);
 	if (as102_dev == NULL)
 		return -ENOMEM;
 
-	/* Assign the user-friendly device name */
-	for (i = 0; i < ARRAY_SIZE(as102_usb_id_table); i++) {
-		if (id == &as102_usb_id_table[i]) {
-			as102_dev->name = as102_device_names[i];
-			as102_dev->elna_cfg = as102_elna_cfg[i];
-		}
-	}
-
-	if (as102_dev->name == NULL)
+	if (info) {
+		as102_dev->name = info->name;
+		as102_dev->elna_cfg = info->elna_cfg;
+	} else {
 		as102_dev->name = "Unknown AS102 device";
+	}
 
 	/* set private callback functions */
 	as102_dev->bus_adap.ops = &as102_priv_ops;
diff --git a/drivers/media/usb/au0828/au0828-cards.c b/drivers/media/usb/au0828/au0828-cards.c
index 9929ce5dcdcd3..8ab100db01abb 100644
--- a/drivers/media/usb/au0828/au0828-cards.c
+++ b/drivers/media/usb/au0828/au0828-cards.c
@@ -124,6 +124,17 @@ struct au0828_board au0828_boards[] = {
 			},
 		},
 	},
+	[AU0828_BOARD_MONOPRICE_106456] = {
+		/*
+		 * Monoprice 106456 USB ATSC/QAM tuner (board rev TV22AD-A),
+		 * a.k.a. AnyTV AUTV002, USB ID 05e1:0400. Same AU8522 demod +
+		 * NXP TDA18271HDC2 tuner @ 0x60 as the Hauppauge Woodbury.
+		 */
+		.name	= "Monoprice 106456 USB ATSC/QAM (TV22AD-A)",
+		.tuner_type = TUNER_NXP_TDA18271,
+		.tuner_addr = 0x60,
+		.i2c_clk_divider = AU0828_I2C_CLK_250KHZ,
+	},
 	[AU0828_BOARD_HAUPPAUGE_HVR1265] = {
 		.name	= "Hauppauge HVR1265",
 		.tuner_type = TUNER_XC5000,
@@ -294,6 +305,7 @@ void au0828_gpio_setup(struct au0828_dev *dev)
 	case AU0828_BOARD_HAUPPAUGE_WOODBURY:
 	case AU0828_BOARD_HAUPPAUGE_HVR1265:
 	case AU0828_BOARD_HAUPPAUGE_IMPACTVCBE:
+	case AU0828_BOARD_MONOPRICE_106456:
 		/* GPIO's
 		 * 4 - CS5340
 		 * 5 - AU8522 Demodulator
@@ -378,6 +390,8 @@ struct usb_device_id au0828_usb_id_table[] = {
 		.driver_info = AU0828_BOARD_HAUPPAUGE_HVR950Q_MXL },
 	{ USB_DEVICE(0x05e1, 0x0480),
 		.driver_info = AU0828_BOARD_HAUPPAUGE_WOODBURY },
+	{ USB_DEVICE(0x05e1, 0x0400),
+		.driver_info = AU0828_BOARD_MONOPRICE_106456 },
 	{ USB_DEVICE(0x2040, 0x8200),
 		.driver_info = AU0828_BOARD_HAUPPAUGE_WOODBURY },
 	{ USB_DEVICE(0x2040, 0x7260),
diff --git a/drivers/media/usb/au0828/au0828-cards.h b/drivers/media/usb/au0828/au0828-cards.h
index a438aeb334a08..a4a283731f7d6 100644
--- a/drivers/media/usb/au0828/au0828-cards.h
+++ b/drivers/media/usb/au0828/au0828-cards.h
@@ -13,3 +13,4 @@
 #define AU0828_BOARD_HAUPPAUGE_WOODBURY	5
 #define AU0828_BOARD_HAUPPAUGE_IMPACTVCBE	6
 #define AU0828_BOARD_HAUPPAUGE_HVR1265		7
+#define AU0828_BOARD_MONOPRICE_106456		8
diff --git a/drivers/media/usb/au0828/au0828-dvb.c b/drivers/media/usb/au0828/au0828-dvb.c
index 31123e6f9fc31..9c95b7ceaecdc 100644
--- a/drivers/media/usb/au0828/au0828-dvb.c
+++ b/drivers/media/usb/au0828/au0828-dvb.c
@@ -600,6 +600,7 @@ int au0828_dvb_register(struct au0828_dev *dev)
 				   &mxl5007t_hvr950q_config);
 		break;
 	case AU0828_BOARD_HAUPPAUGE_WOODBURY:
+	case AU0828_BOARD_MONOPRICE_106456:
 		dvb->frontend = dvb_attach(au8522_attach,
 				&hauppauge_woodbury_config,
 				&dev->i2c_adap);
diff --git a/drivers/media/usb/cx231xx/cx231xx-audio.c b/drivers/media/usb/cx231xx/cx231xx-audio.c
index 9c71b32552df3..b24ceef497e4c 100644
--- a/drivers/media/usb/cx231xx/cx231xx-audio.c
+++ b/drivers/media/usb/cx231xx/cx231xx-audio.c
@@ -443,6 +443,11 @@ static int snd_cx231xx_pcm_close(struct snd_pcm_substream *substream)
 	int ret;
 	struct cx231xx *dev = snd_pcm_substream_chip(substream);
 
+	if (!dev) {
+		pr_err("cx231xx: called with null device\n");
+		return -ENODEV;
+	}
+
 	dev_dbg(dev->dev, "closing device\n");
 
 	/* inform hardware to stop streaming */
diff --git a/drivers/media/usb/cx231xx/cx231xx-video.c b/drivers/media/usb/cx231xx/cx231xx-video.c
index 2cd4e333bc4b0..70aa99fead278 100644
--- a/drivers/media/usb/cx231xx/cx231xx-video.c
+++ b/drivers/media/usb/cx231xx/cx231xx-video.c
@@ -898,7 +898,7 @@ static int vidioc_s_fmt_vid_cap(struct file *file, void *priv,
 	if (rc)
 		return rc;
 
-	if (vb2_is_busy(&dev->vidq)) {
+	if (vb2_is_busy(&dev->vidq) || vb2_is_busy(&dev->vbiq)) {
 		dev_err(dev->dev, "%s: queue busy\n", __func__);
 		return -EBUSY;
 	}
@@ -933,7 +933,7 @@ static int vidioc_s_std(struct file *file, void *priv, v4l2_std_id norm)
 	if (dev->norm == norm)
 		return 0;
 
-	if (vb2_is_busy(&dev->vidq))
+	if (vb2_is_busy(&dev->vidq) || vb2_is_busy(&dev->vbiq))
 		return -EBUSY;
 
 	dev->norm = norm;
diff --git a/drivers/media/usb/em28xx/em28xx-cards.c b/drivers/media/usb/em28xx/em28xx-cards.c
index fbfb74eab4759..b94b00456bcdb 100644
--- a/drivers/media/usb/em28xx/em28xx-cards.c
+++ b/drivers/media/usb/em28xx/em28xx-cards.c
@@ -2677,6 +2677,28 @@ const struct em28xx_board em28xx_boards[] = {
 			.gpio     = mygica_utv3_tuner_audio_gpio,
 		} },
 	},
+	/* eb1a:8286 StarTech SVID2USB232
+	 * Empia EM28281 with integrated TVP5150-compatible video decoder.
+	 * Composite and S-Video inputs, stereo line-in audio.
+	 */
+	[EM28281_BOARD_STARTECH_SVID2USB232] = {
+		.name         = "StarTech SVID2USB232",
+		.vchannels    = 2,
+		.tuner_type   = TUNER_ABSENT,
+		.has_dvb      = 0,
+		.decoder      = EM28XX_BUILTIN,
+		.i2c_speed    = EM28XX_I2C_CLK_WAIT_ENABLE | EM28XX_I2C_FREQ_400_KHZ,
+		.xclk         = EM28XX_XCLK_FREQUENCY_12MHZ,
+		.input           = { {
+			.type     = EM28XX_VMUX_COMPOSITE,
+			.vmux     = EM2828X_COMPOSITE,
+			.amux     = EM28XX_AMUX_LINE_IN,
+		}, {
+			.type     = EM28XX_VMUX_SVIDEO,
+			.vmux     = EM2828X_SVIDEO,
+			.amux     = EM28XX_AMUX_LINE_IN,
+		} },
+	},
 	[EM2828X_BOARD_HAUPPAUGE_USB_LIVE2] = {
 		.name         = "Hauppauge USB Live2",
 		.vchannels    = 2,
@@ -2946,6 +2968,8 @@ struct usb_device_id em28xx_id_table[] = {
 			.driver_info = EM2874_BOARD_HAUPPAUGE_USB_QUADHD },
 	{ USB_DEVICE(0x2040, 0xc220),
 			.driver_info = EM2828X_BOARD_HAUPPAUGE_USB_LIVE2 },
+	{ USB_DEVICE(0xeb1a, 0x8286),
+			.driver_info = EM28281_BOARD_STARTECH_SVID2USB232 },
 	{ USB_DEVICE(0x2040, 0x0360),
 			.driver_info = EM2828X_BOARD_HAUPPAUGE_935_V2 },
 	{ USB_DEVICE(0x2040, 0x8360),
@@ -3758,7 +3782,7 @@ void em28xx_free_device(struct kref *ref)
 {
 	struct em28xx *dev = kref_to_dev(ref);
 
-	dev_info(&dev->intf->dev, "Freeing device\n");
+	pr_info("%s: Freeing device\n", dev->name);
 
 	if (!dev->disconnected)
 		em28xx_release_resources(dev);
@@ -3859,6 +3883,11 @@ static int em28xx_init_dev(struct em28xx *dev, struct usb_device *udev,
 			dev->wait_after_write = 0;
 			dev->eeprom_addrwidth_16bit = 1;
 			break;
+		case CHIP_ID_EM28281:
+			chip_name = "em28281";
+			dev->wait_after_write = 0;
+			dev->eeprom_addrwidth_16bit = 1;
+			break;
 		case CHIP_ID_EM2883:
 			chip_name = "em2882/3";
 			dev->wait_after_write = 0;
@@ -4082,7 +4111,7 @@ static void em28xx_check_usb_descriptor(struct em28xx *dev,
 			dev->analog_ep_bulk = e->bEndpointAddress;
 		}
 		return;
-	};
+	}
 }
 
 /*
diff --git a/drivers/media/usb/em28xx/em28xx-core.c b/drivers/media/usb/em28xx/em28xx-core.c
index 5bbb082dbed9b..0b998e7cafef8 100644
--- a/drivers/media/usb/em28xx/em28xx-core.c
+++ b/drivers/media/usb/em28xx/em28xx-core.c
@@ -632,7 +632,7 @@ void em2828X_decoder_vmux(struct em28xx *dev, unsigned int vin)
 	default:
 		dev_dbg(&dev->intf->dev, "EM2828X_SVIDEO\n");
 		break;
-	};
+	}
 
 	em28xx_write_reg(dev, 0x24, 0x00);
 	em28xx_write_reg(dev, 0x25, 0x02);
@@ -1265,6 +1265,8 @@ void em28xx_close_extension(struct em28xx *dev)
 			ops->fini(dev);
 		}
 	}
+	if (dev->dev_next)
+		list_del(&dev->dev_next->devlist);
 	list_del(&dev->devlist);
 	mutex_unlock(&em28xx_devlist_mutex);
 }
diff --git a/drivers/media/usb/em28xx/em28xx-dvb.c b/drivers/media/usb/em28xx/em28xx-dvb.c
index 938f1980d448d..8482fc4045ea7 100644
--- a/drivers/media/usb/em28xx/em28xx-dvb.c
+++ b/drivers/media/usb/em28xx/em28xx-dvb.c
@@ -29,7 +29,7 @@
 #include <media/dmxdev.h>
 #include <media/tuner.h>
 #include "tuner-simple.h"
-#include <linux/gpio.h>
+#include <linux/gpio/legacy.h>
 
 #include "lgdt330x.h"
 #include "lgdt3305.h"
diff --git a/drivers/media/usb/em28xx/em28xx-reg.h b/drivers/media/usb/em28xx/em28xx-reg.h
index 68a0fcc2fa720..8931733a8e241 100644
--- a/drivers/media/usb/em28xx/em28xx-reg.h
+++ b/drivers/media/usb/em28xx/em28xx-reg.h
@@ -283,6 +283,7 @@ enum em28xx_chip_id {
 	CHIP_ID_EM2884 = 68,
 	CHIP_ID_EM28174 = 113,
 	CHIP_ID_EM28178 = 114,
+	CHIP_ID_EM28281 = 145,
 	CHIP_ID_EM2828X = 148,
 };
 
diff --git a/drivers/media/usb/em28xx/em28xx-video.c b/drivers/media/usb/em28xx/em28xx-video.c
index da0422c65e5f4..c418add65bb51 100644
--- a/drivers/media/usb/em28xx/em28xx-video.c
+++ b/drivers/media/usb/em28xx/em28xx-video.c
@@ -1213,6 +1213,9 @@ int em28xx_start_analog_streaming(struct vb2_queue *vq, unsigned int count)
 {
 	struct em28xx *dev = vb2_get_drv_priv(vq);
 	struct em28xx_v4l2 *v4l2 = dev->v4l2;
+	struct em28xx_dmaqueue *dmaq = vq->type == V4L2_BUF_TYPE_VBI_CAPTURE ?
+		&dev->vbiq : &dev->vidq;
+	unsigned long flags = 0;
 	struct v4l2_frequency f;
 	struct v4l2_fh *owner;
 	int rc = 0;
@@ -1227,7 +1230,7 @@ int em28xx_start_analog_streaming(struct vb2_queue *vq, unsigned int count)
 	 */
 	rc = res_get(dev, vq->type);
 	if (rc)
-		return rc;
+		goto exit;
 
 	if (v4l2->streaming_users == 0) {
 		/* First active streaming user, so allocate all the URBs */
@@ -1250,7 +1253,7 @@ int em28xx_start_analog_streaming(struct vb2_queue *vq, unsigned int count)
 					  em28xx_urb_data_copy);
 		if (rc < 0) {
 			res_free(dev, vq->type);
-			return rc;
+			goto exit;
 		}
 
 		/*
@@ -1275,7 +1278,18 @@ int em28xx_start_analog_streaming(struct vb2_queue *vq, unsigned int count)
 	}
 
 	v4l2->streaming_users++;
+	return 0;
+
+exit:
+	spin_lock_irqsave(&dev->slock, flags);
+	while (!list_empty(&dmaq->active)) {
+		struct em28xx_buffer *buf;
 
+		buf = list_entry(dmaq->active.next, struct em28xx_buffer, list);
+		list_del(&buf->list);
+		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_QUEUED);
+	}
+	spin_unlock_irqrestore(&dev->slock, flags);
 	return rc;
 }
 
@@ -2275,18 +2289,27 @@ static int radio_s_tuner(struct file *file, void *priv,
 }
 
 /*
- * em28xx_free_v4l2() - Free struct em28xx_v4l2
+ * em28xx_free_v4l2() - v4l2_device release callback
  *
- * @ref: struct kref for struct em28xx_v4l2
+ * @v4l2_dev: pointer to struct v4l2_device embedded in struct em28xx_v4l2
  *
- * Called when all users of struct em28xx_v4l2 are gone
+ * Called by the v4l2 core when the last reference to the v4l2_device is
+ * released. At this point no userspace file handle nor video_device node
+ * keeps the v4l2 instance alive anymore, so it is safe to release all
+ * v4l2-related resources and drop the em28xx device reference taken when
+ * the v4l2 extension was initialized.
  */
-static void em28xx_free_v4l2(struct kref *ref)
+static void em28xx_free_v4l2(struct v4l2_device *v4l2_dev)
 {
-	struct em28xx_v4l2 *v4l2 = container_of(ref, struct em28xx_v4l2, ref);
+	struct em28xx_v4l2 *v4l2 =
+		container_of(v4l2_dev, struct em28xx_v4l2, v4l2_dev);
+	struct em28xx *dev = v4l2->dev;
 
-	v4l2->dev->v4l2 = NULL;
+	v4l2_ctrl_handler_free(&v4l2->ctrl_handler);
+	v4l2_device_unregister(v4l2_dev);
+	dev->v4l2 = NULL;
 	kfree(v4l2);
+	kref_put(&dev->ref, em28xx_free_device);
 }
 
 /*
@@ -2323,9 +2346,8 @@ static int em28xx_v4l2_open(struct file *filp)
 		return -ENODEV;
 	}
 
-	em28xx_videodbg("open dev=%s type=%s users=%d\n",
-			video_device_node_name(vdev), v4l2_type_names[fh_type],
-			v4l2->users);
+	em28xx_videodbg("open dev=%s type=%s\n",
+			video_device_node_name(vdev), v4l2_type_names[fh_type]);
 
 	ret = v4l2_fh_open(filp);
 	if (ret) {
@@ -2336,7 +2358,7 @@ static int em28xx_v4l2_open(struct file *filp)
 		return ret;
 	}
 
-	if (v4l2->users == 0) {
+	if (v4l2_fh_is_singular_file(filp)) {
 		em28xx_set_mode(dev, EM28XX_ANALOG_MODE);
 
 		if (vdev->vfl_type != VFL_TYPE_RADIO)
@@ -2354,10 +2376,6 @@ static int em28xx_v4l2_open(struct file *filp)
 		v4l2_device_call_all(&v4l2->v4l2_dev, 0, tuner, s_radio);
 	}
 
-	kref_get(&dev->ref);
-	kref_get(&v4l2->ref);
-	v4l2->users++;
-
 	mutex_unlock(&dev->lock);
 
 	return 0;
@@ -2398,27 +2416,27 @@ static int em28xx_v4l2_fini(struct em28xx *dev)
 	if (video_is_registered(&v4l2->radio_dev)) {
 		dev_info(&dev->intf->dev, "V4L2 device %s deregistered\n",
 			 video_device_node_name(&v4l2->radio_dev));
-		video_unregister_device(&v4l2->radio_dev);
+		vb2_video_unregister_device(&v4l2->radio_dev);
 	}
 	if (video_is_registered(&v4l2->vbi_dev)) {
 		dev_info(&dev->intf->dev, "V4L2 device %s deregistered\n",
 			 video_device_node_name(&v4l2->vbi_dev));
-		video_unregister_device(&v4l2->vbi_dev);
+		vb2_video_unregister_device(&v4l2->vbi_dev);
 	}
 	if (video_is_registered(&v4l2->vdev)) {
 		dev_info(&dev->intf->dev, "V4L2 device %s deregistered\n",
 			 video_device_node_name(&v4l2->vdev));
-		video_unregister_device(&v4l2->vdev);
+		vb2_video_unregister_device(&v4l2->vdev);
 	}
 
-	v4l2_ctrl_handler_free(&v4l2->ctrl_handler);
-	v4l2_device_unregister(&v4l2->v4l2_dev);
-
-	kref_put(&v4l2->ref, em28xx_free_v4l2);
-
 	mutex_unlock(&dev->lock);
 
-	kref_put(&dev->ref, em28xx_free_device);
+	/*
+	 * Drop the initial reference taken at v4l2_device_register() time.
+	 * The em28xx_free_v4l2() release callback will be invoked once all
+	 * userspace file handles to the video device nodes are closed.
+	 */
+	v4l2_device_put(&v4l2->v4l2_dev);
 
 	return 0;
 }
@@ -2460,13 +2478,13 @@ static int em28xx_v4l2_close(struct file *filp)
 	struct em28xx_v4l2    *v4l2 = dev->v4l2;
 	struct usb_device *udev = interface_to_usbdev(dev->intf);
 	int              err;
+	bool last_user;
 
-	em28xx_videodbg("users=%d\n", v4l2->users);
-
-	vb2_fop_release(filp);
 	mutex_lock(&dev->lock);
+	last_user = v4l2_fh_is_singular_file(filp);
+	_vb2_fop_release(filp, NULL);
 
-	if (v4l2->users == 1) {
+	if (last_user) {
 		/* No sense to try to write to the device */
 		if (dev->disconnected)
 			goto exit;
@@ -2489,10 +2507,7 @@ static int em28xx_v4l2_close(struct file *filp)
 	}
 
 exit:
-	v4l2->users--;
-	kref_put(&v4l2->ref, em28xx_free_v4l2);
 	mutex_unlock(&dev->lock);
-	kref_put(&dev->ref, em28xx_free_device);
 
 	return 0;
 }
@@ -2711,7 +2726,6 @@ static int em28xx_v4l2_init(struct em28xx *dev)
 		mutex_unlock(&dev->lock);
 		return -ENOMEM;
 	}
-	kref_init(&v4l2->ref);
 	v4l2->dev = dev;
 	dev->v4l2 = v4l2;
 
@@ -2722,9 +2736,21 @@ static int em28xx_v4l2_init(struct em28xx *dev)
 	if (ret < 0) {
 		dev_err(&dev->intf->dev,
 			"Call to v4l2_device_register() failed!\n");
-		goto err;
+		dev->v4l2 = NULL;
+		kfree(v4l2);
+		mutex_unlock(&dev->lock);
+		return ret;
 	}
 
+	/*
+	 * From this point on, em28xx_free_v4l2() will be used to release
+	 * v4l2-related resources when the v4l2_device refcount reaches
+	 * zero. Take a reference to the em28xx device so that it cannot
+	 * be freed before the v4l2 instance is released.
+	 */
+	v4l2->v4l2_dev.release = em28xx_free_v4l2;
+	kref_get(&dev->ref);
+
 	hdl = &v4l2->ctrl_handler;
 	v4l2_ctrl_handler_init(hdl, 9);
 	v4l2->v4l2_dev.ctrl_handler = hdl;
@@ -3048,8 +3074,6 @@ static int em28xx_v4l2_init(struct em28xx *dev)
 	dev_info(&dev->intf->dev,
 		 "V4L2 extension successfully initialized\n");
 
-	kref_get(&dev->ref);
-
 	mutex_unlock(&dev->lock);
 	return 0;
 
@@ -3058,27 +3082,29 @@ static int em28xx_v4l2_init(struct em28xx *dev)
 		dev_info(&dev->intf->dev,
 			 "V4L2 device %s deregistered\n",
 			 video_device_node_name(&v4l2->radio_dev));
-		video_unregister_device(&v4l2->radio_dev);
+		vb2_video_unregister_device(&v4l2->radio_dev);
 	}
 	if (video_is_registered(&v4l2->vbi_dev)) {
 		dev_info(&dev->intf->dev,
 			 "V4L2 device %s deregistered\n",
 			 video_device_node_name(&v4l2->vbi_dev));
-		video_unregister_device(&v4l2->vbi_dev);
+		vb2_video_unregister_device(&v4l2->vbi_dev);
 	}
 	if (video_is_registered(&v4l2->vdev)) {
 		dev_info(&dev->intf->dev,
 			 "V4L2 device %s deregistered\n",
 			 video_device_node_name(&v4l2->vdev));
-		video_unregister_device(&v4l2->vdev);
+		vb2_video_unregister_device(&v4l2->vdev);
 	}
 
-	v4l2_ctrl_handler_free(&v4l2->ctrl_handler);
-	v4l2_device_unregister(&v4l2->v4l2_dev);
-err:
-	dev->v4l2 = NULL;
-	kref_put(&v4l2->ref, em28xx_free_v4l2);
 	mutex_unlock(&dev->lock);
+
+	/*
+	 * Drop the initial reference. em28xx_free_v4l2() will be called
+	 * once the last video_device node release has decremented the
+	 * v4l2_device refcount to zero.
+	 */
+	v4l2_device_put(&v4l2->v4l2_dev);
 	return ret;
 }
 
diff --git a/drivers/media/usb/em28xx/em28xx.h b/drivers/media/usb/em28xx/em28xx.h
index 711f281613f5e..8eee639b09407 100644
--- a/drivers/media/usb/em28xx/em28xx.h
+++ b/drivers/media/usb/em28xx/em28xx.h
@@ -150,6 +150,7 @@
 #define EM2828X_BOARD_HAUPPAUGE_955_V2            110
 #define EM2828X_BOARD_HAUPPAUGE_975_V2            111
 #define EM28178_BOARD_PCTV_461E_V3                112
+#define EM28281_BOARD_STARTECH_SVID2USB232        113
 
 /* Limits minimum and default number of buffers */
 #define EM28XX_MIN_BUF 4
@@ -558,7 +559,6 @@ struct em28xx_eeprom {
 #define EM28XX_RESOURCE_VBI   0x02
 
 struct em28xx_v4l2 {
-	struct kref ref;
 	struct em28xx *dev;
 
 	struct v4l2_device v4l2_dev;
@@ -582,7 +582,6 @@ struct em28xx_v4l2 {
 	int sensor_yres;
 	int sensor_xtal;
 
-	int users;		/* user count for exclusive use */
 	int streaming_users;    /* number of actively streaming users */
 
 	u32 frequency;		/* selected tuner frequency */
diff --git a/drivers/media/usb/go7007/go7007-driver.c b/drivers/media/usb/go7007/go7007-driver.c
index 25b3ee25aaa49..453ab5c3aa03a 100644
--- a/drivers/media/usb/go7007/go7007-driver.c
+++ b/drivers/media/usb/go7007/go7007-driver.c
@@ -264,17 +264,17 @@ int go7007_register_encoder(struct go7007 *go, unsigned num_i2c_devs)
 	ret = go7007_init_encoder(go);
 	mutex_unlock(&go->hw_lock);
 	if (ret < 0)
-		return ret;
+		goto err_unregister_v4l2_dev;
 
 	ret = go7007_v4l2_ctrl_init(go);
 	if (ret < 0)
-		return ret;
+		goto err_free_controls;
 
 	if (!go->i2c_adapter_online &&
 			go->board_info->flags & GO7007_BOARD_USE_ONBOARD_I2C) {
 		ret = go7007_i2c_init(go);
 		if (ret < 0)
-			return ret;
+			goto err_free_controls;
 		go->i2c_adapter_online = 1;
 	}
 	if (go->i2c_adapter_online) {
@@ -304,13 +304,19 @@ int go7007_register_encoder(struct go7007 *go, unsigned num_i2c_devs)
 
 	ret = go7007_v4l2_init(go);
 	if (ret < 0)
-		return ret;
+		goto err_free_controls;
 
 	if (go->board_info->flags & GO7007_BOARD_HAS_AUDIO) {
 		go->audio_enabled = 1;
 		go7007_snd_init(go);
 	}
 	return 0;
+
+err_free_controls:
+	v4l2_ctrl_handler_free(&go->hdl);
+err_unregister_v4l2_dev:
+	v4l2_device_unregister(&go->v4l2_dev);
+	return ret;
 }
 EXPORT_SYMBOL(go7007_register_encoder);
 
diff --git a/drivers/media/usb/go7007/go7007-fw.c b/drivers/media/usb/go7007/go7007-fw.c
index 86ce593e0c546..8cf959c20e30c 100644
--- a/drivers/media/usb/go7007/go7007-fw.c
+++ b/drivers/media/usb/go7007/go7007-fw.c
@@ -13,6 +13,7 @@
  */
 
 #include <linux/module.h>
+#include <linux/bitops.h>
 #include <linux/time.h>
 #include <linux/mm.h>
 #include <linux/device.h>
@@ -58,14 +59,16 @@ struct code_gen {
 #define CODE_GEN(name, dest) struct code_gen name = { dest, 0, 32, 0 }
 
 #define CODE_ADD(name, val, length) do { \
-	name.b -= (length); \
-	name.a |= (val) << name.b; \
-	while (name.b <= 24) { \
-		*name.p = name.a >> 24; \
-		++name.p; \
-		name.a <<= 8; \
-		name.b += 8; \
-		name.len += 8; \
+	if (length) { \
+		name.b -= (length); \
+		name.a |= (val) << name.b; \
+		while (name.b <= 24) { \
+			*name.p = name.a >> 24; \
+			++name.p; \
+			name.a <<= 8; \
+			name.b += 8; \
+			name.len += 8; \
+		} \
 	} \
 } while (0)
 
@@ -707,11 +710,10 @@ static int gen_mpeg1hdr_to_package(struct go7007 *go,
 
 static int vti_bitlen(struct go7007 *go)
 {
-	unsigned int i, max_time_incr = go->sensor_framerate / go->fps_scale;
+	unsigned int max_time_incr = go->sensor_framerate / go->fps_scale;
+	int bitlen = fls(max_time_incr);
 
-	for (i = 31; (max_time_incr & ((1 << i) - 1)) == max_time_incr; --i)
-		;
-	return i + 1;
+	return bitlen ?: 1;
 }
 
 static int mpeg4_frame_header(struct go7007 *go, unsigned char *buf,
@@ -1209,7 +1211,7 @@ static int seqhead_to_package(struct go7007 *go, __le16 *code, int space,
 		0xbf08,		fps,
 		0xbf09,		0,
 		0xbff2,		vop_time_increment_bitlength,
-		0xbff3,		(1 << vop_time_increment_bitlength) - 1,
+		0xbff3,		GENMASK(vop_time_increment_bitlength - 1, 0),
 		0xbfe6,		0,
 		0xbfe7,		(fps / 1000) << 8,
 		0,		0,
diff --git a/drivers/media/usb/go7007/snd-go7007.c b/drivers/media/usb/go7007/snd-go7007.c
index e4b9f37be77b4..01d547ad678d4 100644
--- a/drivers/media/usb/go7007/snd-go7007.c
+++ b/drivers/media/usb/go7007/snd-go7007.c
@@ -195,6 +195,13 @@ static const struct snd_device_ops go7007_snd_device_ops = {
 	.dev_free	= go7007_snd_free,
 };
 
+static void go7007_snd_card_free(struct snd_card *card)
+{
+	struct go7007 *go = card->private_data;
+
+	v4l2_device_put(&go->v4l2_dev);
+}
+
 int go7007_snd_init(struct go7007 *go)
 {
 	static int dev;
@@ -245,6 +252,8 @@ int go7007_snd_init(struct go7007 *go)
 	gosnd->substream = NULL;
 	go->snd_context = gosnd;
 	v4l2_device_get(&go->v4l2_dev);
+	gosnd->card->private_data = go;
+	gosnd->card->private_free = go7007_snd_card_free;
 	++dev;
 
 	return 0;
@@ -263,7 +272,6 @@ int go7007_snd_remove(struct go7007 *go)
 
 	snd_card_disconnect(gosnd->card);
 	snd_card_free_when_closed(gosnd->card);
-	v4l2_device_put(&go->v4l2_dev);
 	return 0;
 }
 EXPORT_SYMBOL(go7007_snd_remove);
diff --git a/drivers/media/usb/s2255/s2255drv.c b/drivers/media/usb/s2255/s2255drv.c
index 0b8182edf8e4b..3917913c3b47c 100644
--- a/drivers/media/usb/s2255/s2255drv.c
+++ b/drivers/media/usb/s2255/s2255drv.c
@@ -2277,6 +2277,11 @@ static int s2255_probe(struct usb_interface *interface,
 	}
 	/* check the firmware is valid */
 	fw_size = dev->fw_data->fw->size;
+	if (fw_size < 8) {
+		dev_err(&interface->dev, "Firmware invalid: too small.\n");
+		retval = -ENODEV;
+		goto errorFWMARKER;
+	}
 	pdata = (__le32 *) &dev->fw_data->fw->data[fw_size - 8];
 
 	if (*pdata != S2255_FW_MARKER) {
diff --git a/drivers/media/usb/stk1160/stk1160-core.c b/drivers/media/usb/stk1160/stk1160-core.c
index f9462a9ca7616..5058cada0d0a3 100644
--- a/drivers/media/usb/stk1160/stk1160-core.c
+++ b/drivers/media/usb/stk1160/stk1160-core.c
@@ -263,7 +263,7 @@ static int stk1160_scan_usb(struct usb_interface *intf, struct usb_device *udev,
 static int stk1160_probe(struct usb_interface *interface,
 		const struct usb_device_id *id)
 {
-	int rc = 0;
+	int rc;
 
 	unsigned int *alt_max_pkt_size;	/* array of wMaxPacketSize */
 	struct usb_device *udev;
@@ -290,15 +290,13 @@ static int stk1160_probe(struct usb_interface *interface,
 	 * Also, check if device speed is fast enough.
 	 */
 	rc = stk1160_scan_usb(interface, udev, alt_max_pkt_size);
-	if (rc < 0) {
-		kfree(alt_max_pkt_size);
-		return rc;
-	}
+	if (rc < 0)
+		goto free_array;
 
 	dev = kzalloc_obj(struct stk1160);
 	if (dev == NULL) {
-		kfree(alt_max_pkt_size);
-		return -ENOMEM;
+		rc = -ENOMEM;
+		goto free_array;
 	}
 
 	dev->alt_max_pkt_size = alt_max_pkt_size;
@@ -379,8 +377,9 @@ static int stk1160_probe(struct usb_interface *interface,
 free_ctrl:
 	v4l2_ctrl_handler_free(&dev->ctrl_handler);
 free_err:
-	kfree(alt_max_pkt_size);
 	kfree(dev);
+free_array:
+	kfree(alt_max_pkt_size);
 
 	return rc;
 }
diff --git a/drivers/media/usb/stk1160/stk1160-video.c b/drivers/media/usb/stk1160/stk1160-video.c
index f4baf92632862..020b75fc5b2d3 100644
--- a/drivers/media/usb/stk1160/stk1160-video.c
+++ b/drivers/media/usb/stk1160/stk1160-video.c
@@ -48,6 +48,9 @@ static inline void print_err_status(struct stk1160 *dev,
 	case -EILSEQ:
 		errmsg = "CRC/Timeout (could be anything)";
 		break;
+	case -ESHUTDOWN:
+		errmsg = "host controller removed";
+		break;
 	case -ETIME:
 		errmsg = "Device does not respond";
 		break;
diff --git a/drivers/media/usb/usbtv/usbtv-audio.c b/drivers/media/usb/usbtv/usbtv-audio.c
index 333bd305a4f9f..ae0a14e5ed2ab 100644
--- a/drivers/media/usb/usbtv/usbtv-audio.c
+++ b/drivers/media/usb/usbtv/usbtv-audio.c
@@ -317,6 +317,13 @@ static const struct snd_pcm_ops snd_usbtv_pcm_ops = {
 	.pointer = snd_usbtv_pointer,
 };
 
+static void usbtv_audio_card_free(struct snd_card *card)
+{
+	struct usbtv *usbtv = card->private_data;
+
+	v4l2_device_put(&usbtv->v4l2_dev);
+}
+
 int usbtv_audio_init(struct usbtv *usbtv)
 {
 	int rv;
@@ -331,6 +338,10 @@ int usbtv_audio_init(struct usbtv *usbtv)
 	if (rv < 0)
 		return rv;
 
+	v4l2_device_get(&usbtv->v4l2_dev);
+	card->private_data = usbtv;
+	card->private_free = usbtv_audio_card_free;
+
 	strscpy(card->driver, usbtv->dev->driver->name, sizeof(card->driver));
 	strscpy(card->shortname, "usbtv", sizeof(card->shortname));
 	snprintf(card->longname, sizeof(card->longname),
diff --git a/drivers/media/v4l2-core/v4l2-async.c b/drivers/media/v4l2-core/v4l2-async.c
index 888a2e213b088..f36d60e6ff41e 100644
--- a/drivers/media/v4l2-core/v4l2-async.c
+++ b/drivers/media/v4l2-core/v4l2-async.c
@@ -898,9 +898,18 @@ void v4l2_async_unregister_subdev(struct v4l2_subdev *sd)
 	sd->subdev_notifier = NULL;
 
 	if (sd->asc_list.next) {
-		list_for_each_entry_safe(asc, asc_tmp, &sd->asc_list,
-					 asc_subdev_entry) {
-			v4l2_async_unbind_subdev_one(asc->notifier, asc);
+		if (list_empty(&sd->asc_list)) {
+			/*
+			 * If the sub-device was registered through other means
+			 * than v4l2-async, there are no async connections but
+			 * the sub-device may still well be registered.
+			 * Unregister it now.
+			 */
+			v4l2_device_unregister_subdev(sd);
+		} else {
+			list_for_each_entry_safe(asc, asc_tmp, &sd->asc_list,
+						 asc_subdev_entry)
+				v4l2_async_unbind_subdev_one(asc->notifier, asc);
 		}
 	}
 
diff --git a/drivers/media/v4l2-core/v4l2-ctrls-core.c b/drivers/media/v4l2-core/v4l2-ctrls-core.c
index ba047d7d86010..1214e7744ac01 100644
--- a/drivers/media/v4l2-core/v4l2-ctrls-core.c
+++ b/drivers/media/v4l2-core/v4l2-ctrls-core.c
@@ -16,6 +16,9 @@
 
 static const union v4l2_ctrl_ptr ptr_null;
 
+#define V4L2_HEVC_MAX_SHORT_TERM_REF_PIC_SETS	64
+#define V4L2_HEVC_MAX_LONG_TERM_REF_PICS_SPS	32
+
 static void fill_event(struct v4l2_event *ev, struct v4l2_ctrl *ctrl,
 		       u32 changes)
 {
@@ -1214,6 +1217,10 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx,
 	case V4L2_CTRL_TYPE_HEVC_SPS:
 		p_hevc_sps = p;
 
+		if (p_hevc_sps->num_short_term_ref_pic_sets >
+		    V4L2_HEVC_MAX_SHORT_TERM_REF_PIC_SETS)
+			return -EINVAL;
+
 		if (!(p_hevc_sps->flags & V4L2_HEVC_SPS_FLAG_PCM_ENABLED)) {
 			p_hevc_sps->pcm_sample_bit_depth_luma_minus1 = 0;
 			p_hevc_sps->pcm_sample_bit_depth_chroma_minus1 = 0;
@@ -1224,6 +1231,9 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx,
 		if (!(p_hevc_sps->flags &
 		      V4L2_HEVC_SPS_FLAG_LONG_TERM_REF_PICS_PRESENT))
 			p_hevc_sps->num_long_term_ref_pics_sps = 0;
+		else if (p_hevc_sps->num_long_term_ref_pics_sps >
+			 V4L2_HEVC_MAX_LONG_TERM_REF_PICS_SPS)
+			return -EINVAL;
 		break;
 
 	case V4L2_CTRL_TYPE_HEVC_PPS:
@@ -1280,6 +1290,11 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx,
 
 		if (p_hevc_st_rps->flags & ~V4L2_HEVC_EXT_SPS_ST_RPS_FLAG_INTER_REF_PIC_SET_PRED)
 			return -EINVAL;
+		if (p_hevc_st_rps->num_negative_pics > 16 ||
+		    p_hevc_st_rps->num_positive_pics > 16 ||
+		    p_hevc_st_rps->num_negative_pics +
+		    p_hevc_st_rps->num_positive_pics > 16)
+			return -EINVAL;
 		break;
 
 	case V4L2_CTRL_TYPE_HEVC_EXT_SPS_LT_RPS:
diff --git a/drivers/media/v4l2-core/v4l2-fwnode.c b/drivers/media/v4l2-core/v4l2-fwnode.c
index 62a3a452f7884..570df3c415b0a 100644
--- a/drivers/media/v4l2-core/v4l2-fwnode.c
+++ b/drivers/media/v4l2-core/v4l2-fwnode.c
@@ -633,6 +633,8 @@ int v4l2_fwnode_parse_link(struct fwnode_handle *fwnode,
 	if (!link->remote_node)
 		goto err_put_remote_endpoint;
 
+	fwnode_handle_put(fwnode);
+
 	return 0;
 
 err_put_remote_endpoint:
diff --git a/drivers/media/v4l2-core/v4l2-vp9.c b/drivers/media/v4l2-core/v4l2-vp9.c
index 859589f1fd35f..e965ffbd9b8aa 100644
--- a/drivers/media/v4l2-core/v4l2-vp9.c
+++ b/drivers/media/v4l2-core/v4l2-vp9.c
@@ -1582,25 +1582,25 @@ static inline u8 noncoef_merge_prob(u8 pre_prob, u32 ct0, u32 ct1)
  *	merge_prob(p[9], c[9], [10])
  */
 
-static inline void merge_probs_variant_a(u8 *p, const u32 *c, u16 count_sat, u32 update_factor)
+static noinline_for_stack void merge_probs_variant_a(u8 *p, const u32 *c, u16 count_sat, u32 update_factor)
 {
 	p[1] = merge_prob(p[1], c[0], c[1] + c[2], count_sat, update_factor);
 	p[2] = merge_prob(p[2], c[1], c[2], count_sat, update_factor);
 }
 
-static inline void merge_probs_variant_b(u8 *p, const u32 *c, u16 count_sat, u32 update_factor)
+static noinline_for_stack void merge_probs_variant_b(u8 *p, const u32 *c, u16 count_sat, u32 update_factor)
 {
 	p[0] = merge_prob(p[0], c[0], c[1], count_sat, update_factor);
 }
 
-static inline void merge_probs_variant_c(u8 *p, const u32 *c)
+static noinline_for_stack void merge_probs_variant_c(u8 *p, const u32 *c)
 {
 	p[0] = noncoef_merge_prob(p[0], c[2], c[1] + c[0] + c[3]);
 	p[1] = noncoef_merge_prob(p[1], c[0], c[1] + c[3]);
 	p[2] = noncoef_merge_prob(p[2], c[1], c[3]);
 }
 
-static void merge_probs_variant_d(u8 *p, const u32 *c)
+static noinline_for_stack void merge_probs_variant_d(u8 *p, const u32 *c)
 {
 	u32 sum = 0, s2;
 
@@ -1624,20 +1624,20 @@ static void merge_probs_variant_d(u8 *p, const u32 *c)
 	p[8] = noncoef_merge_prob(p[8], c[6], c[7]);
 }
 
-static inline void merge_probs_variant_e(u8 *p, const u32 *c)
+static noinline_for_stack void merge_probs_variant_e(u8 *p, const u32 *c)
 {
 	p[0] = noncoef_merge_prob(p[0], c[0], c[1] + c[2] + c[3]);
 	p[1] = noncoef_merge_prob(p[1], c[1], c[2] + c[3]);
 	p[2] = noncoef_merge_prob(p[2], c[2], c[3]);
 }
 
-static inline void merge_probs_variant_f(u8 *p, const u32 *c)
+static noinline_for_stack void merge_probs_variant_f(u8 *p, const u32 *c)
 {
 	p[0] = noncoef_merge_prob(p[0], c[0], c[1] + c[2]);
 	p[1] = noncoef_merge_prob(p[1], c[1], c[2]);
 }
 
-static void merge_probs_variant_g(u8 *p, const u32 *c)
+static noinline_for_stack void merge_probs_variant_g(u8 *p, const u32 *c)
 {
 	u32 sum;
 
@@ -1659,12 +1659,12 @@ static void merge_probs_variant_g(u8 *p, const u32 *c)
 }
 
 /* 8.4.3 Coefficient probability adaptation process */
-static inline void adapt_probs_variant_a_coef(u8 *p, const u32 *c, u32 update_factor)
+static noinline_for_stack void adapt_probs_variant_a_coef(u8 *p, const u32 *c, u32 update_factor)
 {
 	merge_probs_variant_a(p, c, 24, update_factor);
 }
 
-static inline void adapt_probs_variant_b_coef(u8 *p, const u32 *c, u32 update_factor)
+static noinline_for_stack void adapt_probs_variant_b_coef(u8 *p, const u32 *c, u32 update_factor)
 {
 	merge_probs_variant_b(p, c, 24, update_factor);
 }
@@ -1724,33 +1724,33 @@ static inline void adapt_probs_variant_b(u8 *p, const u32 *c)
 	merge_probs_variant_b(p, c, 20, 128);
 }
 
-static inline void adapt_probs_variant_c(u8 *p, const u32 *c)
+static noinline_for_stack void adapt_probs_variant_c(u8 *p, const u32 *c)
 {
 	merge_probs_variant_c(p, c);
 }
 
-static inline void adapt_probs_variant_d(u8 *p, const u32 *c)
+static noinline_for_stack void adapt_probs_variant_d(u8 *p, const u32 *c)
 {
 	merge_probs_variant_d(p, c);
 }
 
-static inline void adapt_probs_variant_e(u8 *p, const u32 *c)
+static noinline_for_stack void adapt_probs_variant_e(u8 *p, const u32 *c)
 {
 	merge_probs_variant_e(p, c);
 }
 
-static inline void adapt_probs_variant_f(u8 *p, const u32 *c)
+static noinline_for_stack void adapt_probs_variant_f(u8 *p, const u32 *c)
 {
 	merge_probs_variant_f(p, c);
 }
 
-static inline void adapt_probs_variant_g(u8 *p, const u32 *c)
+static noinline_for_stack void adapt_probs_variant_g(u8 *p, const u32 *c)
 {
 	merge_probs_variant_g(p, c);
 }
 
 /* 8.4.4 Non coefficient probability adaptation process, adapt_prob() */
-static inline u8 adapt_prob(u8 prob, const u32 counts[2])
+static noinline_for_stack u8 adapt_prob(u8 prob, const u32 counts[2])
 {
 	return noncoef_merge_prob(prob, counts[0], counts[1]);
 }
diff --git a/drivers/memory/stm32_omm.c b/drivers/memory/stm32_omm.c
index 0e891396bdb61..84eb23c824f5b 100644
--- a/drivers/memory/stm32_omm.c
+++ b/drivers/memory/stm32_omm.c
@@ -46,7 +46,7 @@ static int stm32_omm_set_amcr(struct device *dev, bool set)
 	struct device_node *node;
 	struct resource res, res1;
 	unsigned int syscon_args[2];
-	int ret, idx;
+	int ret = 0, idx;
 	unsigned int i, amcr, read_amcr;
 
 	for (i = 0; i < omm->nb_child; i++) {
diff --git a/drivers/memory/tegra/mc.c b/drivers/memory/tegra/mc.c
index ec80ea9cc1736..458ce99916ad6 100644
--- a/drivers/memory/tegra/mc.c
+++ b/drivers/memory/tegra/mc.c
@@ -3,6 +3,7 @@
  * Copyright (C) 2014-2026 NVIDIA CORPORATION.  All rights reserved.
  */
 
+#include <linux/cleanup.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/dma-mapping.h>
@@ -10,6 +11,7 @@
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/of.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
@@ -22,6 +24,9 @@
 
 #include "mc.h"
 
+static DEFINE_MUTEX(tegra_mc_debugfs_root_lock);
+static struct dentry *tegra_mc_debugfs_root;
+
 static const struct of_device_id tegra_mc_of_match[] = {
 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
 	{ .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc },
@@ -598,6 +603,13 @@ irqreturn_t tegra30_mc_handle_irq(int irq, void *data)
 	if (!status)
 		return IRQ_NONE;
 
+	if (!mc->soc->regs) {
+		dev_err_ratelimited(mc->dev,
+				    "MC error interrupt 0x%08lx with no error register map, Clearing.\n",
+				    status);
+		goto clear;
+	}
+
 	for_each_set_bit(bit, &status, 32) {
 		const char *error = tegra_mc_status_names[bit] ?: "unknown";
 		const char *client = "unknown", *desc;
@@ -736,6 +748,7 @@ irqreturn_t tegra30_mc_handle_irq(int irq, void *data)
 				    desc, perm);
 	}
 
+clear:
 	/* clear interrupts */
 	if (mc->soc->num_channels) {
 		mc_ch_writel(mc, channel, status, MC_INTSTATUS);
@@ -778,7 +791,7 @@ struct icc_node *tegra_mc_icc_xlate(const struct of_phandle_args *spec, void *da
 	struct icc_node *node;
 
 	list_for_each_entry(node, &mc->provider.nodes, node_list) {
-		if (node->id == spec->args[0])
+		if (tegra_mc_client_id_from_node(node) == spec->args[0])
 			return node;
 	}
 
@@ -834,6 +847,7 @@ const struct tegra_mc_icc_ops tegra_mc_icc_ops = {
  */
 static int tegra_mc_interconnect_setup(struct tegra_mc *mc)
 {
+	int node_id = dev_to_node(mc->dev);
 	struct icc_node *node;
 	unsigned int i;
 	int err;
@@ -854,31 +868,40 @@ static int tegra_mc_interconnect_setup(struct tegra_mc *mc)
 	icc_provider_init(&mc->provider);
 
 	/* create Memory Controller node */
-	node = icc_node_create(TEGRA_ICC_MC);
+	node = tegra_mc_icc_node_create(node_id, TEGRA_ICC_MC);
 	if (IS_ERR(node))
 		return PTR_ERR(node);
 
-	node->name = "Memory Controller";
+	if (node_id == NUMA_NO_NODE)
+		node->name = "Memory Controller";
+	else
+		node->name = dev_name(mc->dev);
+
 	icc_node_add(node, &mc->provider);
 
 	/* link Memory Controller to External Memory Controller */
-	err = icc_link_create(node, TEGRA_ICC_EMC);
+	err = tegra_mc_icc_link_create(node, node_id, TEGRA_ICC_EMC);
 	if (err)
 		goto remove_nodes;
 
 	for (i = 0; i < mc->soc->num_clients; i++) {
 		/* create MC client node */
-		node = icc_node_create(mc->soc->clients[i].id);
+		node = tegra_mc_icc_node_create(node_id, mc->soc->clients[i].id);
 		if (IS_ERR(node)) {
 			err = PTR_ERR(node);
 			goto remove_nodes;
 		}
 
-		node->name = mc->soc->clients[i].name;
+		if (node_id == NUMA_NO_NODE)
+			node->name = mc->soc->clients[i].name;
+		else
+			node->name = devm_kasprintf(mc->dev, GFP_KERNEL, "%d-%s",
+						    node_id, mc->soc->clients[i].name);
+
 		icc_node_add(node, &mc->provider);
 
 		/* link Memory Client to Memory Controller */
-		err = icc_link_create(node, TEGRA_ICC_MC);
+		err = tegra_mc_icc_link_create(node, node_id, TEGRA_ICC_MC);
 		if (err)
 			goto remove_nodes;
 
@@ -957,7 +980,16 @@ static int tegra_mc_probe(struct platform_device *pdev)
 	if (IS_ERR(mc->regs))
 		return PTR_ERR(mc->regs);
 
-	mc->debugfs.root = debugfs_create_dir("mc", NULL);
+	scoped_guard(mutex, &tegra_mc_debugfs_root_lock) {
+		if (!tegra_mc_debugfs_root)
+			tegra_mc_debugfs_root = debugfs_create_dir("mc", NULL);
+
+		if (dev_to_node(mc->dev) == NUMA_NO_NODE)
+			mc->debugfs.root = tegra_mc_debugfs_root;
+		else
+			mc->debugfs.root = debugfs_create_dir(dev_name(mc->dev),
+							      tegra_mc_debugfs_root);
+	}
 
 	if (mc->soc->ops && mc->soc->ops->probe) {
 		err = mc->soc->ops->probe(mc);
diff --git a/drivers/memory/tegra/mc.h b/drivers/memory/tegra/mc.h
index e94d265d7b678..01499292dd5f6 100644
--- a/drivers/memory/tegra/mc.h
+++ b/drivers/memory/tegra/mc.h
@@ -8,6 +8,7 @@
 
 #include <linux/bits.h>
 #include <linux/io.h>
+#include <linux/numa.h>
 #include <linux/types.h>
 
 #include <soc/tegra/mc.h>
@@ -167,6 +168,42 @@ icc_provider_to_tegra_mc(struct icc_provider *provider)
 	return container_of(provider, struct tegra_mc, provider);
 }
 
+/*
+ * Compose a globally-unique ICC node ID. On single-socket
+ * systems (NUMA_NO_NODE), the SoC client ID is returned unchanged.
+ * On multi-socket systems, the NUMA node ID is encoded in the
+ * upper bits of the returned ID.
+ *
+ * The client ID field is sized to keep composed IDs below
+ * ICC_DYN_ID_START (the start of the ICC core's dynamic-ID range).
+ */
+#define TEGRA_MC_CLIENT_ID_BITS	12
+#define TEGRA_MC_CLIENT_ID_MASK	((1U << TEGRA_MC_CLIENT_ID_BITS) - 1)
+
+static inline u32 tegra_mc_get_client_id(int node_id, int id)
+{
+	if (node_id == NUMA_NO_NODE)
+		return id;
+
+	return ((node_id + 1) << TEGRA_MC_CLIENT_ID_BITS) | id;
+}
+
+static inline struct icc_node *tegra_mc_icc_node_create(int node_id, int id)
+{
+	return icc_node_create(tegra_mc_get_client_id(node_id, id));
+}
+
+static inline int tegra_mc_icc_link_create(struct icc_node *node, int node_id, int id)
+{
+	return icc_link_create(node, tegra_mc_get_client_id(node_id, id));
+}
+
+/* Return the SoC client ID encoded in an ICC node ID. */
+static inline u32 tegra_mc_client_id_from_node(const struct icc_node *node)
+{
+	return node->id & TEGRA_MC_CLIENT_ID_MASK;
+}
+
 static inline u32 mc_ch_readl(const struct tegra_mc *mc, int ch,
 			      unsigned long offset)
 {
diff --git a/drivers/memory/tegra/tegra186-emc.c b/drivers/memory/tegra/tegra186-emc.c
index 2a4cb64c4c4c5..c9583b66f18de 100644
--- a/drivers/memory/tegra/tegra186-emc.c
+++ b/drivers/memory/tegra/tegra186-emc.c
@@ -3,15 +3,21 @@
  * Copyright (C) 2019-2025 NVIDIA CORPORATION.  All rights reserved.
  */
 
+#include <linux/cleanup.h>
 #include <linux/clk.h>
 #include <linux/debugfs.h>
+#include <linux/fs.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
 
 #include <soc/tegra/bpmp.h>
 #include "mc.h"
 
+static DEFINE_MUTEX(tegra_emc_debugfs_root_lock);
+static struct dentry *tegra_emc_debugfs_root;
+
 struct tegra186_emc_dvfs {
 	unsigned long latency;
 	unsigned long rate;
@@ -206,7 +212,17 @@ static int tegra186_emc_get_emc_dvfs_latency(struct tegra186_emc *emc)
 		return err;
 	}
 
-	emc->debugfs.root = debugfs_create_dir("emc", NULL);
+	scoped_guard(mutex, &tegra_emc_debugfs_root_lock) {
+		if (!tegra_emc_debugfs_root)
+			tegra_emc_debugfs_root = debugfs_create_dir("emc", NULL);
+
+		if (dev_to_node(emc->dev) == NUMA_NO_NODE)
+			emc->debugfs.root = tegra_emc_debugfs_root;
+		else
+			emc->debugfs.root = debugfs_create_dir(dev_name(emc->dev),
+							       tegra_emc_debugfs_root);
+	}
+
 	debugfs_create_file("available_rates", 0444, emc->debugfs.root, emc,
 			    &tegra186_emc_debug_available_rates_fops);
 	debugfs_create_file("min_rate", 0644, emc->debugfs.root, emc,
@@ -238,7 +254,7 @@ tegra186_emc_of_icc_xlate(const struct of_phandle_args *spec, void *data)
 
 	/* External Memory is the only possible ICC route */
 	list_for_each_entry(node, &provider->nodes, node_list) {
-		if (node->id != TEGRA_ICC_EMEM)
+		if (tegra_mc_client_id_from_node(node) != TEGRA_ICC_EMEM)
 			continue;
 
 		return node;
@@ -257,6 +273,7 @@ static int tegra186_emc_icc_get_init_bw(struct icc_node *node, u32 *avg, u32 *pe
 
 static int tegra186_emc_interconnect_init(struct tegra186_emc *emc)
 {
+	int node_id = dev_to_node(emc->dev->parent);
 	struct icc_node *node;
 	int err;
 
@@ -270,26 +287,36 @@ static int tegra186_emc_interconnect_init(struct tegra186_emc *emc)
 	icc_provider_init(&emc->provider);
 
 	/* create External Memory Controller node */
-	node = icc_node_create(TEGRA_ICC_EMC);
-	if (IS_ERR(node))
-		return PTR_ERR(node);
+	node = tegra_mc_icc_node_create(node_id, TEGRA_ICC_EMC);
+	if (IS_ERR(node)) {
+		err = PTR_ERR(node);
+		goto remove_nodes;
+	}
+
+	if (node_id == NUMA_NO_NODE)
+		node->name = "External Memory Controller";
+	else
+		node->name = dev_name(emc->dev);
 
-	node->name = "External Memory Controller";
 	icc_node_add(node, &emc->provider);
 
 	/* link External Memory Controller to External Memory (DRAM) */
-	err = icc_link_create(node, TEGRA_ICC_EMEM);
+	err = tegra_mc_icc_link_create(node, node_id, TEGRA_ICC_EMEM);
 	if (err)
 		goto remove_nodes;
 
 	/* create External Memory node */
-	node = icc_node_create(TEGRA_ICC_EMEM);
+	node = tegra_mc_icc_node_create(node_id, TEGRA_ICC_EMEM);
 	if (IS_ERR(node)) {
 		err = PTR_ERR(node);
 		goto remove_nodes;
 	}
 
-	node->name = "External Memory (DRAM)";
+	if (node_id == NUMA_NO_NODE)
+		node->name = "External Memory (DRAM)";
+	else
+		node->name = devm_kasprintf(emc->dev, GFP_KERNEL, "%d-dram", node_id);
+
 	icc_node_add(node, &emc->provider);
 
 	err = icc_provider_register(&emc->provider);
@@ -382,6 +409,16 @@ static void tegra186_emc_remove(struct platform_device *pdev)
 
 	debugfs_remove_recursive(emc->debugfs.root);
 
+	scoped_guard(mutex, &tegra_emc_debugfs_root_lock) {
+		if (emc->debugfs.root == tegra_emc_debugfs_root) {
+			tegra_emc_debugfs_root = NULL;
+		} else if (tegra_emc_debugfs_root &&
+			   simple_empty(tegra_emc_debugfs_root)) {
+			debugfs_remove(tegra_emc_debugfs_root);
+			tegra_emc_debugfs_root = NULL;
+		}
+	}
+
 	mc->bpmp = NULL;
 	tegra_bpmp_put(emc->bpmp);
 }
diff --git a/drivers/memstick/core/ms_block.c b/drivers/memstick/core/ms_block.c
index a01fe313558e7..ce33907bfc242 100644
--- a/drivers/memstick/core/ms_block.c
+++ b/drivers/memstick/core/ms_block.c
@@ -1338,6 +1338,10 @@ static int msb_ftl_initialize(struct msb_data *msb)
 		return 0;
 
 	msb->zone_count = msb->block_count / MS_BLOCKS_IN_ZONE;
+	if (msb->block_count > MS_MAX_ZONES * MS_BLOCKS_IN_ZONE) {
+		pr_err("Too many blocks: %d\n", msb->block_count);
+		return -EINVAL;
+	}
 	msb->logical_block_count = msb->zone_count * 496 - 2;
 
 	msb->used_blocks_bitmap = bitmap_zalloc(msb->block_count, GFP_KERNEL);
diff --git a/drivers/mfd/88pm886.c b/drivers/mfd/88pm886.c
index e411d8dee5542..f8401d5e6dbe3 100644
--- a/drivers/mfd/88pm886.c
+++ b/drivers/mfd/88pm886.c
@@ -16,6 +16,12 @@ static const struct regmap_config pm886_regmap_config = {
 	.max_register = PM886_REG_RTC_SPARE6,
 };
 
+static const struct regmap_config pm886_regmap_battery_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = PM886_REG_CLS_CONFIG1,
+};
+
 static const struct regmap_irq pm886_regmap_irqs[] = {
 	REGMAP_IRQ_REG(PM886_IRQ_ONKEY, 0, PM886_INT_ENA1_ONKEY),
 };
@@ -85,10 +91,11 @@ static int pm886_setup_irq(struct pm886_chip *chip,
 
 static int pm886_probe(struct i2c_client *client)
 {
+	struct regmap *regmap, *regmap_battery;
 	struct regmap_irq_chip_data *irq_data;
 	struct device *dev = &client->dev;
+	struct i2c_client *battery_page;
 	struct pm886_chip *chip;
-	struct regmap *regmap;
 	unsigned int chip_id;
 	int err;
 
@@ -112,6 +119,18 @@ static int pm886_probe(struct i2c_client *client)
 	if (chip->chip_id != chip_id)
 		return dev_err_probe(dev, -EINVAL, "Unsupported chip: 0x%x\n", chip_id);
 
+	battery_page = devm_i2c_new_dummy_device(dev, client->adapter,
+						 client->addr + PM886_PAGE_OFFSET_BATTERY);
+	if (IS_ERR(battery_page))
+		return dev_err_probe(dev, PTR_ERR(battery_page),
+				     "Failed to initialize battery page\n");
+
+	regmap_battery = devm_regmap_init_i2c(battery_page, &pm886_regmap_battery_config);
+	if (IS_ERR(regmap_battery))
+		return dev_err_probe(dev, PTR_ERR(regmap_battery),
+				     "Failed to initialize battery regmap\n");
+	chip->regmap_battery = regmap_battery;
+
 	err = pm886_setup_irq(chip, &irq_data);
 	if (err)
 		return err;
diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig
index 763ce6a34782b..35f6e9b76d056 100644
--- a/drivers/mfd/Kconfig
+++ b/drivers/mfd/Kconfig
@@ -1461,7 +1461,6 @@ config MFD_SEC_I2C
 config MFD_SI476X_CORE
 	tristate "Silicon Laboratories 4761/64/68 AM/FM radio."
 	depends on I2C
-	depends on GPIOLIB_LEGACY
 	select MFD_CORE
 	select REGMAP_I2C
 	help
diff --git a/drivers/mfd/ab8500-core.c b/drivers/mfd/ab8500-core.c
index f0bc0b5a6f4ad..86fa99022cb35 100644
--- a/drivers/mfd/ab8500-core.c
+++ b/drivers/mfd/ab8500-core.c
@@ -19,7 +19,7 @@
 #include <linux/mfd/core.h>
 #include <linux/mfd/abx500.h>
 #include <linux/mfd/abx500/ab8500.h>
-#include <linux/mfd/dbx500-prcmu.h>
+#include <linux/mfd/db8500-prcmu.h>
 #include <linux/of.h>
 
 /*
diff --git a/drivers/mfd/altera-a10sr.c b/drivers/mfd/altera-a10sr.c
index d53e433ab5c1d..9c02622286558 100644
--- a/drivers/mfd/altera-a10sr.c
+++ b/drivers/mfd/altera-a10sr.c
@@ -155,7 +155,7 @@ MODULE_DEVICE_TABLE(of, altr_a10sr_spi_of_match);
 
 static const struct spi_device_id altr_a10sr_spi_ids[] = {
 	{ .name = "a10sr" },
-	{ },
+	{ }
 };
 MODULE_DEVICE_TABLE(spi, altr_a10sr_spi_ids);
 
diff --git a/drivers/mfd/arizona-spi.c b/drivers/mfd/arizona-spi.c
index eaa2b2bc5dd01..04baf5a1c6526 100644
--- a/drivers/mfd/arizona-spi.c
+++ b/drivers/mfd/arizona-spi.c
@@ -255,12 +255,12 @@ static void arizona_spi_remove(struct spi_device *spi)
 }
 
 static const struct spi_device_id arizona_spi_ids[] = {
-	{ "wm5102", WM5102 },
-	{ "wm5110", WM5110 },
-	{ "wm8280", WM8280 },
-	{ "wm1831", WM1831 },
-	{ "cs47l24", CS47L24 },
-	{ },
+	{ .name = "wm5102", .driver_data = WM5102 },
+	{ .name = "wm5110", .driver_data = WM5110 },
+	{ .name = "wm8280", .driver_data = WM8280 },
+	{ .name = "wm1831", .driver_data = WM1831 },
+	{ .name = "cs47l24", .driver_data = CS47L24 },
+	{ }
 };
 MODULE_DEVICE_TABLE(spi, arizona_spi_ids);
 
diff --git a/drivers/mfd/axp20x.c b/drivers/mfd/axp20x.c
index 679364189ea58..50df66de24e06 100644
--- a/drivers/mfd/axp20x.c
+++ b/drivers/mfd/axp20x.c
@@ -1253,7 +1253,7 @@ static int axp20x_power_off(struct sys_off_data *data)
 		break;
 	}
 
-	regmap_write(axp20x->regmap, shutdown_reg, AXP20X_OFF);
+	regmap_set_bits(axp20x->regmap, shutdown_reg, AXP20X_OFF);
 
 	/* Give capacitors etc. time to drain to avoid kernel panic msg. */
 	mdelay(500);
diff --git a/drivers/mfd/cs40l50-spi.c b/drivers/mfd/cs40l50-spi.c
index 53526b595a0d1..a635951ca6b76 100644
--- a/drivers/mfd/cs40l50-spi.c
+++ b/drivers/mfd/cs40l50-spi.c
@@ -40,8 +40,8 @@ static void cs40l50_spi_remove(struct spi_device *spi)
 }
 
 static const struct spi_device_id cs40l50_id_spi[] = {
-	{ "cs40l50" },
-	{}
+	{ .name = "cs40l50" },
+	{ }
 };
 MODULE_DEVICE_TABLE(spi, cs40l50_id_spi);
 
diff --git a/drivers/mfd/cs42l43-i2c.c b/drivers/mfd/cs42l43-i2c.c
index 44ad63129b3ff..cbe05c3ea9100 100644
--- a/drivers/mfd/cs42l43-i2c.c
+++ b/drivers/mfd/cs42l43-i2c.c
@@ -44,8 +44,6 @@ static int cs42l43_i2c_probe(struct i2c_client *i2c)
 
 	cs42l43->dev = &i2c->dev;
 	cs42l43->irq = i2c->irq;
-	/* A device on an I2C is always attached by definition. */
-	cs42l43->attached = true;
 	cs42l43->variant_id = (long)device_get_match_data(cs42l43->dev);
 
 	cs42l43->regmap = devm_regmap_init_i2c(i2c, &cs42l43_i2c_regmap);
diff --git a/drivers/mfd/cs42l43-sdw.c b/drivers/mfd/cs42l43-sdw.c
index 1804b942bdb5b..7f0c8d370662d 100644
--- a/drivers/mfd/cs42l43-sdw.c
+++ b/drivers/mfd/cs42l43-sdw.c
@@ -99,17 +99,10 @@ static int cs42l43_sdw_update_status(struct sdw_slave *sdw, enum sdw_slave_statu
 
 		sdw_write_no_pm(sdw, CS42L43_GEN_INT_MASK_1,
 				CS42L43_INT_STAT_GEN1_MASK);
-
-		cs42l43->attached = true;
-
-		complete(&cs42l43->device_attach);
 		break;
 	case SDW_SLAVE_UNATTACHED:
 		dev_dbg(cs42l43->dev, "Device detach\n");
 
-		cs42l43->attached = false;
-
-		reinit_completion(&cs42l43->device_attach);
 		complete(&cs42l43->device_detach);
 		break;
 	default:
diff --git a/drivers/mfd/cs42l43.c b/drivers/mfd/cs42l43.c
index ed6d93893de04..d2bbd2f18af7d 100644
--- a/drivers/mfd/cs42l43.c
+++ b/drivers/mfd/cs42l43.c
@@ -586,15 +586,13 @@ static int cs42l43_soft_reset(struct cs42l43 *cs42l43)
  */
 static int cs42l43_wait_for_attach(struct cs42l43 *cs42l43)
 {
-	if (!cs42l43->attached) {
-		unsigned long timeout = msecs_to_jiffies(CS42L43_SDW_ATTACH_TIMEOUT_MS);
-		unsigned long time;
+	int ret;
 
-		time = wait_for_completion_timeout(&cs42l43->device_attach, timeout);
-		if (!time) {
-			dev_err(cs42l43->dev, "Timed out waiting for device re-attach\n");
-			return -ETIMEDOUT;
-		}
+	if (cs42l43->sdw) {
+		ret = sdw_slave_wait_for_init(cs42l43->sdw,
+					      CS42L43_SDW_ATTACH_TIMEOUT_MS);
+		if (ret)
+			return ret;
 	}
 
 	regcache_cache_only(cs42l43->regmap, false);
@@ -1120,7 +1118,6 @@ int cs42l43_dev_probe(struct cs42l43 *cs42l43)
 	dev_set_drvdata(cs42l43->dev, cs42l43);
 
 	mutex_init(&cs42l43->pll_lock);
-	init_completion(&cs42l43->device_attach);
 	init_completion(&cs42l43->device_detach);
 	init_completion(&cs42l43->firmware_download);
 	INIT_WORK(&cs42l43->boot_work, cs42l43_boot_work);
diff --git a/drivers/mfd/da9052-spi.c b/drivers/mfd/da9052-spi.c
index be5f2b34e18ae..29cf2c17fde1a 100644
--- a/drivers/mfd/da9052-spi.c
+++ b/drivers/mfd/da9052-spi.c
@@ -63,12 +63,12 @@ static void da9052_spi_remove(struct spi_device *spi)
 }
 
 static const struct spi_device_id da9052_spi_id[] = {
-	{"da9052", DA9052},
-	{"da9053-aa", DA9053_AA},
-	{"da9053-ba", DA9053_BA},
-	{"da9053-bb", DA9053_BB},
-	{"da9053-bc", DA9053_BC},
-	{}
+	{ .name = "da9052", .driver_data = DA9052 },
+	{ .name = "da9053-aa", .driver_data = DA9053_AA },
+	{ .name = "da9053-ba", .driver_data = DA9053_BA },
+	{ .name = "da9053-bb", .driver_data = DA9053_BB },
+	{ .name = "da9053-bc", .driver_data = DA9053_BC },
+	{ }
 };
 
 static struct spi_driver da9052_spi_driver = {
diff --git a/drivers/mfd/db8500-prcmu.c b/drivers/mfd/db8500-prcmu.c
index 21e68a382b114..6672c55f2ebcc 100644
--- a/drivers/mfd/db8500-prcmu.c
+++ b/drivers/mfd/db8500-prcmu.c
@@ -32,7 +32,7 @@
 #include <linux/platform_device.h>
 #include <linux/uaccess.h>
 #include <linux/mfd/core.h>
-#include <linux/mfd/dbx500-prcmu.h>
+#include <linux/mfd/db8500-prcmu.h>
 #include <linux/mfd/abx500/ab8500.h>
 #include <linux/regulator/db8500-prcmu.h>
 #include <linux/regulator/machine.h>
@@ -2285,7 +2285,7 @@ void db8500_prcmu_system_reset(u16 reset_code)
 /**
  * db8500_prcmu_get_reset_code - Retrieve SW reset reason code
  *
- * Retrieves the reset reason code stored by prcmu_system_reset() before
+ * Retrieves the reset reason code stored by db8500_prcmu_system_reset() before
  * last restart.
  */
 u16 db8500_prcmu_get_reset_code(void)
@@ -3041,7 +3041,7 @@ static int db8500_prcmu_probe(struct platform_device *pdev)
 
 	db8500_irq_init(np);
 
-	prcmu_config_esram0_deep_sleep(ESRAM0_DEEP_SLEEP_STATE_RET);
+	db8500_prcmu_config_esram0_deep_sleep(ESRAM0_DEEP_SLEEP_STATE_RET);
 
 	err = mfd_add_devices(&pdev->dev, 0, common_prcmu_devs,
 			      ARRAY_SIZE(common_prcmu_devs), NULL, 0, db8500_irq_domain);
diff --git a/drivers/mfd/intel-m10-bmc-spi.c b/drivers/mfd/intel-m10-bmc-spi.c
index cfa620f0c70e1..94b9c99bb4f86 100644
--- a/drivers/mfd/intel-m10-bmc-spi.c
+++ b/drivers/mfd/intel-m10-bmc-spi.c
@@ -160,9 +160,9 @@ static const struct intel_m10bmc_platform_info m10bmc_spi_n5010 = {
 };
 
 static const struct spi_device_id m10bmc_spi_id[] = {
-	{ "m10-n3000", (kernel_ulong_t)&m10bmc_spi_n3000 },
-	{ "m10-d5005", (kernel_ulong_t)&m10bmc_spi_d5005 },
-	{ "m10-n5010", (kernel_ulong_t)&m10bmc_spi_n5010 },
+	{ .name = "m10-n3000", .driver_data = (kernel_ulong_t)&m10bmc_spi_n3000 },
+	{ .name = "m10-d5005", .driver_data = (kernel_ulong_t)&m10bmc_spi_d5005 },
+	{ .name = "m10-n5010", .driver_data = (kernel_ulong_t)&m10bmc_spi_n5010 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, m10bmc_spi_id);
diff --git a/drivers/mfd/ipaq-micro.c b/drivers/mfd/ipaq-micro.c
index 4b757d8472825..5146a6eb0e5a6 100644
--- a/drivers/mfd/ipaq-micro.c
+++ b/drivers/mfd/ipaq-micro.c
@@ -221,7 +221,7 @@ static void ipaq_micro_eeprom_read(struct ipaq_micro *micro,
 
 static char *ipaq_micro_str(u8 *wchar, u8 len)
 {
-	char retstr[256];
+	char retstr[256] = { 0 };
 	u8 i;
 
 	for (i = 0; i < len / 2; i++)
diff --git a/drivers/mfd/madera-spi.c b/drivers/mfd/madera-spi.c
index ce9e90322c9ca..3fffa21ceadc1 100644
--- a/drivers/mfd/madera-spi.c
+++ b/drivers/mfd/madera-spi.c
@@ -112,15 +112,15 @@ static void madera_spi_remove(struct spi_device *spi)
 }
 
 static const struct spi_device_id madera_spi_ids[] = {
-	{ "cs47l15", CS47L15 },
-	{ "cs47l35", CS47L35 },
-	{ "cs47l85", CS47L85 },
-	{ "cs47l90", CS47L90 },
-	{ "cs47l91", CS47L91 },
-	{ "cs42l92", CS42L92 },
-	{ "cs47l92", CS47L92 },
-	{ "cs47l93", CS47L93 },
-	{ "wm1840", WM1840 },
+	{ .name = "cs47l15", .driver_data = CS47L15 },
+	{ .name = "cs47l35", .driver_data = CS47L35 },
+	{ .name = "cs47l85", .driver_data = CS47L85 },
+	{ .name = "cs47l90", .driver_data = CS47L90 },
+	{ .name = "cs47l91", .driver_data = CS47L91 },
+	{ .name = "cs42l92", .driver_data = CS42L92 },
+	{ .name = "cs47l92", .driver_data = CS47L92 },
+	{ .name = "cs47l93", .driver_data = CS47L93 },
+	{ .name = "wm1840", .driver_data = WM1840 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, madera_spi_ids);
diff --git a/drivers/mfd/motorola-cpcap.c b/drivers/mfd/motorola-cpcap.c
index feeccb2c6655f..4ed8704c985ab 100644
--- a/drivers/mfd/motorola-cpcap.c
+++ b/drivers/mfd/motorola-cpcap.c
@@ -202,9 +202,9 @@ static const struct of_device_id cpcap_of_match[] = {
 MODULE_DEVICE_TABLE(of, cpcap_of_match);
 
 static const struct spi_device_id cpcap_spi_ids[] = {
-	{ .name = "cpcap", },
-	{ .name = "6556002", },
-	{},
+	{ .name = "cpcap" },
+	{ .name = "6556002" },
+	{ }
 };
 MODULE_DEVICE_TABLE(spi, cpcap_spi_ids);
 
diff --git a/drivers/mfd/mt6397-core.c b/drivers/mfd/mt6397-core.c
index 1bdacda9a933f..ea1d039477e30 100644
--- a/drivers/mfd/mt6397-core.c
+++ b/drivers/mfd/mt6397-core.c
@@ -125,6 +125,9 @@ static const struct resource mt6323_pwrc_resources[] = {
 
 static const struct mfd_cell mt6323_devs[] = {
 	{
+		.name = "mt6323-efuse",
+		.of_compatible = "mediatek,mt6323-efuse",
+	}, {
 		.name = "mt6323-rtc",
 		.num_resources = ARRAY_SIZE(mt6323_rtc_resources),
 		.resources = mt6323_rtc_resources,
diff --git a/drivers/mfd/ocelot-spi.c b/drivers/mfd/ocelot-spi.c
index fc30663824bb1..d3ab4868a5238 100644
--- a/drivers/mfd/ocelot-spi.c
+++ b/drivers/mfd/ocelot-spi.c
@@ -270,7 +270,7 @@ static int ocelot_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ocelot_spi_ids[] = {
-	{ "vsc7512", 0 },
+	{ .name = "vsc7512" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ocelot_spi_ids);
diff --git a/drivers/mfd/rk8xx-spi.c b/drivers/mfd/rk8xx-spi.c
index 3405fb82ff9fb..bb85fe60518f6 100644
--- a/drivers/mfd/rk8xx-spi.c
+++ b/drivers/mfd/rk8xx-spi.c
@@ -104,7 +104,7 @@ static const struct of_device_id rk8xx_spi_of_match[] = {
 MODULE_DEVICE_TABLE(of, rk8xx_spi_of_match);
 
 static const struct spi_device_id rk8xx_spi_id_table[] = {
-	{ "rk806", 0 },
+	{ .name = "rk806" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, rk8xx_spi_id_table);
diff --git a/drivers/mfd/rohm-bd71828.c b/drivers/mfd/rohm-bd71828.c
index a79f354bf5cb6..5fb6142cf0878 100644
--- a/drivers/mfd/rohm-bd71828.c
+++ b/drivers/mfd/rohm-bd71828.c
@@ -5,7 +5,8 @@
  * ROHM BD718[15/28/79] and BD72720 PMIC driver
  */
 
-#include <linux/gpio_keys.h>
+#include <linux/device/devres.h>
+#include <linux/gfp_types.h>
 #include <linux/i2c.h>
 #include <linux/input.h>
 #include <linux/interrupt.h>
@@ -18,6 +19,7 @@
 #include <linux/mfd/rohm-generic.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/property.h>
 #include <linux/regmap.h>
 #include <linux/types.h>
 
@@ -37,19 +39,6 @@
 		},							   \
 	}
 
-static struct gpio_keys_button button = {
-	.code = KEY_POWER,
-	.gpio = -1,
-	.type = EV_KEY,
-	.wakeup = 1,
-};
-
-static const struct gpio_keys_platform_data bd71828_powerkey_data = {
-	.buttons = &button,
-	.nbuttons = 1,
-	.name = "bd71828-pwrkey",
-};
-
 static const struct resource bd71815_rtc_irqs[] = {
 	DEFINE_RES_IRQ_NAMED(BD71815_INT_RTC0, "bd70528-rtc-alm-0"),
 	DEFINE_RES_IRQ_NAMED(BD71815_INT_RTC1, "bd70528-rtc-alm-1"),
@@ -174,10 +163,6 @@ static struct mfd_cell bd71828_mfd_cells[] = {
 		.name = "bd71828-rtc",
 		.resources = bd71828_rtc_irqs,
 		.num_resources = ARRAY_SIZE(bd71828_rtc_irqs),
-	}, {
-		.name = "gpio-keys",
-		.platform_data = &bd71828_powerkey_data,
-		.pdata_size = sizeof(bd71828_powerkey_data),
 	},
 };
 
@@ -242,11 +227,8 @@ static const struct mfd_cell bd72720_mfd_cells[] = {
 		.name = "bd72720-rtc",
 		.resources = bd72720_rtc_irqs,
 		.num_resources = ARRAY_SIZE(bd72720_rtc_irqs),
-	}, {
-		.name = "gpio-keys",
-		.platform_data = &bd71828_powerkey_data,
-		.pdata_size = sizeof(bd71828_powerkey_data),
 	},
+	/* Power button is registered separately */
 };
 
 static const struct regmap_range bd71815_volatile_ranges[] = {
@@ -877,6 +859,84 @@ static int set_clk_mode(struct device *dev, struct regmap *regmap,
 				  OUT32K_MODE_CMOS);
 }
 
+static const struct property_entry bd71828_powerkey_parent_props[] = {
+	PROPERTY_ENTRY_STRING("label", "bd71828-pwrkey"),
+	{ }
+};
+
+static const struct property_entry bd71828_powerkey_props[] = {
+	PROPERTY_ENTRY_U32("linux,code", KEY_POWER),
+	PROPERTY_ENTRY_BOOL("wakeup-source"),
+	{ }
+};
+
+#define GPIO_KEYS  0	/* Node corresponding to gpio-keys device itself */
+#define PWRON_KEY  1	/* Node describing power button in gpio-keys */
+
+static int bd71828_i2c_register_swnodes(const struct software_node *nodes)
+{
+	const struct software_node * const node_group[] = {
+		&nodes[GPIO_KEYS], &nodes[PWRON_KEY], NULL
+	};
+
+	return software_node_register_node_group(node_group);
+}
+
+static void bd71828_i2c_unregister_swnodes(void *data)
+{
+	const struct software_node *nodes = data;
+	const struct software_node * const node_group[] = {
+		&nodes[GPIO_KEYS], &nodes[PWRON_KEY], NULL
+	};
+
+	software_node_unregister_node_group(node_group);
+}
+
+static int bd71828_i2c_register_pwrbutton(struct device *dev, int button_irq,
+					  struct irq_domain *irq_domain)
+{
+	const struct resource res[] = {
+		DEFINE_RES_IRQ_NAMED(button_irq, "bd71828-pwrkey"),
+	};
+	struct mfd_cell gpio_keys_cell = {
+		.name = "gpio-keys",
+		.resources = res,
+		.num_resources = ARRAY_SIZE(res),
+	};
+	struct software_node *nodes;
+	int ret;
+
+	nodes = devm_kcalloc(dev, 2, sizeof(*nodes), GFP_KERNEL);
+	if (!nodes)
+		return -ENOMEM;
+
+	nodes[GPIO_KEYS].name = devm_kasprintf(dev, GFP_KERNEL, "%s-power-key", dev_name(dev));
+	if (!nodes[GPIO_KEYS].name)
+		return -ENOMEM;
+
+	nodes[GPIO_KEYS].properties = bd71828_powerkey_parent_props;
+
+	nodes[PWRON_KEY].parent = &nodes[GPIO_KEYS];
+	nodes[PWRON_KEY].properties = bd71828_powerkey_props;
+
+	ret = bd71828_i2c_register_swnodes(nodes);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(dev, bd71828_i2c_unregister_swnodes, nodes);
+	if (ret)
+		return ret;
+
+	gpio_keys_cell.swnode = &nodes[GPIO_KEYS];
+
+	ret = devm_mfd_add_devices(dev, PLATFORM_DEVID_AUTO, &gpio_keys_cell, 1,
+				   NULL, 0, irq_domain);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to register power-button");
+
+	return 0;
+}
+
 static struct i2c_client *bd71828_dev;
 static void bd71828_power_off(void)
 {
@@ -929,6 +989,7 @@ static struct regmap *bd72720_do_regmaps(struct i2c_client *i2c)
 static int bd71828_i2c_probe(struct i2c_client *i2c)
 {
 	struct regmap_irq_chip_data *irq_data;
+	struct irq_domain *irq_domain;
 	int ret;
 	struct regmap *regmap = NULL;
 	const struct regmap_config *regmap_config;
@@ -1022,23 +1083,23 @@ static int bd71828_i2c_probe(struct i2c_client *i2c)
 					"Failed to enable main level IRQs\n");
 		}
 	}
-	if (button_irq) {
-		ret = regmap_irq_get_virq(irq_data, button_irq);
-		if (ret < 0)
-			return dev_err_probe(&i2c->dev, ret,
-					     "Failed to get the power-key IRQ\n");
-
-		button.irq = ret;
-	}
 
 	ret = set_clk_mode(&i2c->dev, regmap, clkmode_reg);
 	if (ret)
 		return ret;
 
+	irq_domain = regmap_irq_get_domain(irq_data);
+
 	ret = devm_mfd_add_devices(&i2c->dev, PLATFORM_DEVID_AUTO, mfd, cells,
-				   NULL, 0, regmap_irq_get_domain(irq_data));
+				   NULL, 0, irq_domain);
 	if (ret)
-		return	dev_err_probe(&i2c->dev, ret, "Failed to create subdevices\n");
+		return dev_err_probe(&i2c->dev, ret, "Failed to create subdevices\n");
+
+	if (button_irq) {
+		ret = bd71828_i2c_register_pwrbutton(&i2c->dev, button_irq, irq_domain);
+		if (ret)
+			return ret;
+	}
 
 	if (of_device_is_system_power_controller(i2c->dev.of_node) &&
 	    chip_type == ROHM_CHIP_TYPE_BD71828) {
diff --git a/drivers/mfd/rohm-bd718x7.c b/drivers/mfd/rohm-bd718x7.c
index ff714fd4f54d2..be2acc429fe31 100644
--- a/drivers/mfd/rohm-bd718x7.c
+++ b/drivers/mfd/rohm-bd718x7.c
@@ -7,7 +7,8 @@
 // Datasheet for BD71837MWV available from
 // https://www.rohm.com/datasheet/BD71837MWV/bd71837mwv-e
 
-#include <linux/gpio_keys.h>
+#include <linux/device/devres.h>
+#include <linux/gfp_types.h>
 #include <linux/i2c.h>
 #include <linux/input.h>
 #include <linux/interrupt.h>
@@ -15,37 +16,16 @@
 #include <linux/mfd/core.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/property.h>
 #include <linux/regmap.h>
 #include <linux/types.h>
 
-static struct gpio_keys_button button = {
-	.code = KEY_POWER,
-	.gpio = -1,
-	.type = EV_KEY,
-};
-
-static struct gpio_keys_platform_data bd718xx_powerkey_data = {
-	.buttons = &button,
-	.nbuttons = 1,
-	.name = "bd718xx-pwrkey",
-};
-
 static struct mfd_cell bd71837_mfd_cells[] = {
-	{
-		.name = "gpio-keys",
-		.platform_data = &bd718xx_powerkey_data,
-		.pdata_size = sizeof(bd718xx_powerkey_data),
-	},
 	{ .name = "bd71837-clk", },
 	{ .name = "bd71837-pmic", },
 };
 
 static struct mfd_cell bd71847_mfd_cells[] = {
-	{
-		.name = "gpio-keys",
-		.platform_data = &bd718xx_powerkey_data,
-		.pdata_size = sizeof(bd718xx_powerkey_data),
-	},
 	{ .name = "bd71847-clk", },
 	{ .name = "bd71847-pmic", },
 };
@@ -125,10 +105,89 @@ static int bd718xx_init_press_duration(struct regmap *regmap,
 	return 0;
 }
 
+static const struct property_entry bd718xx_powerkey_parent_props[] = {
+	PROPERTY_ENTRY_STRING("label", "bd718xx-pwrkey"),
+	{ }
+};
+
+static const struct property_entry bd718xx_powerkey_props[] = {
+	PROPERTY_ENTRY_U32("linux,code", KEY_POWER),
+	{ }
+};
+
+static const struct resource bd718xx_powerkey_resources[] = {
+	DEFINE_RES_IRQ_NAMED(BD718XX_INT_PWRBTN_S, "bd718xx-pwrkey"),
+};
+
+#define GPIO_KEYS  0	/* Node corresponding to gpio-keys device itself */
+#define PWRON_KEY  1	/* Node describing power button in gpio-keys */
+
+static int bd718xx_i2c_register_swnodes(const struct software_node *nodes)
+{
+	const struct software_node * const node_group[] = {
+		&nodes[GPIO_KEYS], &nodes[PWRON_KEY], NULL
+	};
+
+	return software_node_register_node_group(node_group);
+}
+
+static void bd718xx_i2c_unregister_swnodes(void *data)
+{
+	const struct software_node *nodes = data;
+	const struct software_node * const node_group[] = {
+		&nodes[GPIO_KEYS], &nodes[PWRON_KEY], NULL
+	};
+
+	software_node_unregister_node_group(node_group);
+}
+
+static int bd718xx_i2c_register_pwrbutton(struct device *dev,
+					  struct irq_domain *irq_domain)
+{
+	struct mfd_cell gpio_keys_cell = {
+		.name = "gpio-keys",
+		.resources = bd718xx_powerkey_resources,
+		.num_resources = ARRAY_SIZE(bd718xx_powerkey_resources),
+	};
+	struct software_node *nodes;
+	int ret;
+
+	nodes = devm_kcalloc(dev, 2, sizeof(*nodes), GFP_KERNEL);
+	if (!nodes)
+		return -ENOMEM;
+
+	nodes[GPIO_KEYS].name = devm_kasprintf(dev, GFP_KERNEL, "%s-power-key", dev_name(dev));
+	if (!nodes[GPIO_KEYS].name)
+		return -ENOMEM;
+
+	nodes[GPIO_KEYS].properties = bd718xx_powerkey_parent_props;
+
+	nodes[PWRON_KEY].parent = &nodes[GPIO_KEYS];
+	nodes[PWRON_KEY].properties = bd718xx_powerkey_props;
+
+	ret = bd718xx_i2c_register_swnodes(nodes);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(dev, bd718xx_i2c_unregister_swnodes, nodes);
+	if (ret)
+		return ret;
+
+	gpio_keys_cell.swnode = &nodes[GPIO_KEYS];
+
+	ret = devm_mfd_add_devices(dev, PLATFORM_DEVID_AUTO, &gpio_keys_cell, 1,
+				   NULL, 0, irq_domain);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to register power-button");
+
+	return 0;
+}
+
 static int bd718xx_i2c_probe(struct i2c_client *i2c)
 {
 	struct regmap *regmap;
 	struct regmap_irq_chip_data *irq_data;
+	struct irq_domain *irq_domain;
 	int ret;
 	unsigned int chip_type;
 	struct mfd_cell *mfd;
@@ -169,20 +228,18 @@ static int bd718xx_i2c_probe(struct i2c_client *i2c)
 	if (ret)
 		return ret;
 
-	ret = regmap_irq_get_virq(irq_data, BD718XX_INT_PWRBTN_S);
-
-	if (ret < 0)
-		return dev_err_probe(&i2c->dev, ret, "Failed to get the IRQ\n");
-
-	button.irq = ret;
+	irq_domain = regmap_irq_get_domain(irq_data);
 
 	ret = devm_mfd_add_devices(&i2c->dev, PLATFORM_DEVID_AUTO,
-				   mfd, cells, NULL, 0,
-				   regmap_irq_get_domain(irq_data));
+				   mfd, cells, NULL, 0, irq_domain);
+	if (ret)
+		return dev_err_probe(&i2c->dev, ret, "Failed to create subdevices\n");
+
+	ret = bd718xx_i2c_register_pwrbutton(&i2c->dev, irq_domain);
 	if (ret)
-		dev_err_probe(&i2c->dev, ret, "Failed to create subdevices\n");
+		return ret;
 
-	return ret;
+	return 0;
 }
 
 static const struct of_device_id bd718xx_of_match[] = {
diff --git a/drivers/mfd/rsmu_spi.c b/drivers/mfd/rsmu_spi.c
index e07f21482439e..cdb0f9797ec6a 100644
--- a/drivers/mfd/rsmu_spi.c
+++ b/drivers/mfd/rsmu_spi.c
@@ -239,12 +239,12 @@ static void rsmu_spi_remove(struct spi_device *client)
 }
 
 static const struct spi_device_id rsmu_spi_id[] = {
-	{ "8a34000",  RSMU_CM },
-	{ "8a34001",  RSMU_CM },
-	{ "8a34002",  RSMU_CM },
-	{ "82p33810", RSMU_SABRE },
-	{ "82p33811", RSMU_SABRE },
-	{}
+	{ .name = "8a34000", .driver_data = RSMU_CM },
+	{ .name = "8a34001", .driver_data = RSMU_CM },
+	{ .name = "8a34002", .driver_data = RSMU_CM },
+	{ .name = "82p33810", .driver_data = RSMU_SABRE },
+	{ .name = "82p33811", .driver_data = RSMU_SABRE },
+	{ }
 };
 MODULE_DEVICE_TABLE(spi, rsmu_spi_id);
 
diff --git a/drivers/mfd/si476x-cmd.c b/drivers/mfd/si476x-cmd.c
index 3bb2decfebd31..58e9bea7e90ab 100644
--- a/drivers/mfd/si476x-cmd.c
+++ b/drivers/mfd/si476x-cmd.c
@@ -15,7 +15,6 @@
 #include <linux/atomic.h>
 #include <linux/i2c.h>
 #include <linux/device.h>
-#include <linux/gpio.h>
 #include <linux/videodev2.h>
 
 #include <linux/mfd/si476x-core.h>
diff --git a/drivers/mfd/si476x-i2c.c b/drivers/mfd/si476x-i2c.c
index 7ddc97dfc9402..55700ce711f49 100644
--- a/drivers/mfd/si476x-i2c.c
+++ b/drivers/mfd/si476x-i2c.c
@@ -13,7 +13,7 @@
 #include <linux/slab.h>
 #include <linux/interrupt.h>
 #include <linux/delay.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/regulator/consumer.h>
 #include <linux/i2c.h>
 #include <linux/err.h>
@@ -130,8 +130,8 @@ int si476x_core_start(struct si476x_core *core, bool soft)
 	int err;
 
 	if (!soft) {
-		if (gpio_is_valid(core->gpio_reset))
-			gpio_set_value_cansleep(core->gpio_reset, 1);
+		if (core->reset)
+			gpiod_set_value_cansleep(core->reset, 0);
 
 		if (client->irq)
 			enable_irq(client->irq);
@@ -197,8 +197,8 @@ int si476x_core_start(struct si476x_core *core, bool soft)
 	else
 		cancel_delayed_work_sync(&core->status_monitor);
 
-	if (gpio_is_valid(core->gpio_reset))
-		gpio_set_value_cansleep(core->gpio_reset, 0);
+	if (core->reset)
+		gpiod_set_value_cansleep(core->reset, 1);
 
 	return err;
 }
@@ -243,8 +243,8 @@ int si476x_core_stop(struct si476x_core *core, bool soft)
 		cancel_delayed_work_sync(&core->status_monitor);
 
 	if (!soft) {
-		if (gpio_is_valid(core->gpio_reset))
-			gpio_set_value_cansleep(core->gpio_reset, 0);
+		if (core->reset)
+			gpiod_set_value_cansleep(core->reset, 1);
 	}
 	return err;
 }
@@ -712,24 +712,18 @@ static int si476x_core_probe(struct i2c_client *client)
 	atomic_set(&core->is_alive, 0);
 	core->power_state = SI476X_POWER_DOWN;
 
+	core->reset = devm_gpiod_get_optional(&client->dev, "reset",
+					      GPIOD_OUT_HIGH);
+	if (IS_ERR(core->reset))
+		return dev_err_probe(&client->dev, PTR_ERR(core->reset),
+				     "error getting reset GPIO\n");
+	gpiod_set_consumer_name(core->reset, "si476x reset");
+
 	pdata = dev_get_platdata(&client->dev);
 	if (pdata) {
 		memcpy(&core->power_up_parameters,
 		       &pdata->power_up_parameters,
 		       sizeof(core->power_up_parameters));
-
-		core->gpio_reset = -1;
-		if (gpio_is_valid(pdata->gpio_reset)) {
-			rval = gpio_request(pdata->gpio_reset, "si476x reset");
-			if (rval) {
-				dev_err(&client->dev,
-					"Failed to request gpio: %d\n", rval);
-				return rval;
-			}
-			core->gpio_reset = pdata->gpio_reset;
-			gpio_direction_output(core->gpio_reset, 0);
-		}
-
 		core->diversity_mode = pdata->diversity_mode;
 		memcpy(&core->pinmux, &pdata->pinmux,
 		       sizeof(struct si476x_pinmux));
@@ -748,7 +742,7 @@ static int si476x_core_probe(struct i2c_client *client)
 				       core->supplies);
 	if (rval) {
 		dev_err(&client->dev, "Failed to get all of the regulators\n");
-		goto free_gpio;
+		return rval;
 	}
 
 	mutex_init(&core->cmd_lock);
@@ -761,7 +755,7 @@ static int si476x_core_probe(struct i2c_client *client)
 			   GFP_KERNEL);
 	if (rval) {
 		dev_err(&client->dev, "Could not allocate the FIFO\n");
-		goto free_gpio;
+		return rval;
 	}
 	mutex_init(&core->rds_drainer_status_lock);
 	init_waitqueue_head(&core->rds_read_queue);
@@ -827,11 +821,6 @@ static int si476x_core_probe(struct i2c_client *client)
 
 free_kfifo:
 	kfifo_free(&core->rds_fifo);
-
-free_gpio:
-	if (gpio_is_valid(core->gpio_reset))
-		gpio_free(core->gpio_reset);
-
 	return rval;
 }
 
@@ -848,9 +837,6 @@ static void si476x_core_remove(struct i2c_client *client)
 		cancel_delayed_work_sync(&core->status_monitor);
 
 	kfifo_free(&core->rds_fifo);
-
-	if (gpio_is_valid(core->gpio_reset))
-		gpio_free(core->gpio_reset);
 }
 
 
diff --git a/drivers/mfd/sprd-sc27xx-spi.c b/drivers/mfd/sprd-sc27xx-spi.c
index aa052f6466231..9a8e6add8fcaa 100644
--- a/drivers/mfd/sprd-sc27xx-spi.c
+++ b/drivers/mfd/sprd-sc27xx-spi.c
@@ -61,6 +61,7 @@ static const struct mfd_cell sc2730_devices[] = {
 	MFD_CELL_OF("sc2730-efuse", NULL, NULL, 0, 0, "sprd,sc2730-efuse"),
 	MFD_CELL_OF("sc2730-eic", NULL, NULL, 0, 0, "sprd,sc2730-eic"),
 	MFD_CELL_OF("sc2730-fgu", NULL, NULL, 0, 0, "sprd,sc2730-fgu"),
+	MFD_CELL_NAME("sc2730-regulator"),
 	MFD_CELL_OF("sc2730-rtc", NULL, NULL, 0, 0, "sprd,sc2730-rtc"),
 	MFD_CELL_OF("sc2730-vibrator", NULL, NULL, 0, 0, "sprd,sc2730-vibrator"),
 };
@@ -294,7 +295,7 @@ MODULE_DEVICE_TABLE(of, sprd_pmic_match);
 static const struct spi_device_id sprd_pmic_spi_ids[] = {
 	{ .name = "sc2730", .driver_data = PMIC_TYPE_SC2730 },
 	{ .name = "sc2731", .driver_data = PMIC_TYPE_SC2731 },
-	{},
+	{ }
 };
 MODULE_DEVICE_TABLE(spi, sprd_pmic_spi_ids);
 
diff --git a/drivers/mfd/stmpe-spi.c b/drivers/mfd/stmpe-spi.c
index dea31efface6e..22a3da062deed 100644
--- a/drivers/mfd/stmpe-spi.c
+++ b/drivers/mfd/stmpe-spi.c
@@ -121,12 +121,12 @@ static const struct of_device_id stmpe_spi_of_match[] = {
 MODULE_DEVICE_TABLE(of, stmpe_spi_of_match);
 
 static const struct spi_device_id stmpe_spi_id[] = {
-	{ "stmpe610", STMPE610 },
-	{ "stmpe801", STMPE801 },
-	{ "stmpe811", STMPE811 },
-	{ "stmpe1601", STMPE1601 },
-	{ "stmpe2401", STMPE2401 },
-	{ "stmpe2403", STMPE2403 },
+	{ .name = "stmpe610", .driver_data = STMPE610 },
+	{ .name = "stmpe801", .driver_data = STMPE801 },
+	{ .name = "stmpe811", .driver_data = STMPE811 },
+	{ .name = "stmpe1601", .driver_data = STMPE1601 },
+	{ .name = "stmpe2401", .driver_data = STMPE2401 },
+	{ .name = "stmpe2403", .driver_data = STMPE2403 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, stmpe_spi_id);
diff --git a/drivers/mfd/tps6586x.c b/drivers/mfd/tps6586x.c
index f5f8054466033..3cfd2f02b62f3 100644
--- a/drivers/mfd/tps6586x.c
+++ b/drivers/mfd/tps6586x.c
@@ -397,7 +397,7 @@ static int tps6586x_add_subdevs(struct tps6586x *tps6586x,
 
 		pdev->dev.parent = tps6586x->dev;
 		pdev->dev.platform_data = subdev->platform_data;
-		pdev->dev.of_node = of_node_get(subdev->of_node);
+		platform_device_set_of_node(pdev, subdev->of_node);
 
 		ret = platform_device_add(pdev);
 		if (ret) {
diff --git a/drivers/mfd/tps65912-spi.c b/drivers/mfd/tps65912-spi.c
index 2a77dccd60599..2442a2e67d676 100644
--- a/drivers/mfd/tps65912-spi.c
+++ b/drivers/mfd/tps65912-spi.c
@@ -43,7 +43,7 @@ static int tps65912_spi_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id tps65912_spi_id_table[] = {
-	{ "tps65912", 0 },
+	{ .name = "tps65912" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(spi, tps65912_spi_id_table);
diff --git a/drivers/mfd/viperboard.c b/drivers/mfd/viperboard.c
index 888737b8e7be8..36be8e69f45c3 100644
--- a/drivers/mfd/viperboard.c
+++ b/drivers/mfd/viperboard.c
@@ -96,8 +96,7 @@ static int vprbrd_probe(struct usb_interface *interface,
 	return 0;
 
 error:
-	if (vb)
-		kfree(vb);
+	kfree(vb);
 
 	return ret;
 }
diff --git a/drivers/mfd/wm831x-spi.c b/drivers/mfd/wm831x-spi.c
index 54c87267917b9..1e519fd9a9e17 100644
--- a/drivers/mfd/wm831x-spi.c
+++ b/drivers/mfd/wm831x-spi.c
@@ -77,14 +77,14 @@ static const struct dev_pm_ops wm831x_spi_pm = {
 };
 
 static const struct spi_device_id wm831x_spi_ids[] = {
-	{ "wm8310", WM8310 },
-	{ "wm8311", WM8311 },
-	{ "wm8312", WM8312 },
-	{ "wm8320", WM8320 },
-	{ "wm8321", WM8321 },
-	{ "wm8325", WM8325 },
-	{ "wm8326", WM8326 },
-	{ },
+	{ .name = "wm8310", .driver_data = WM8310 },
+	{ .name = "wm8311", .driver_data = WM8311 },
+	{ .name = "wm8312", .driver_data = WM8312 },
+	{ .name = "wm8320", .driver_data = WM8320 },
+	{ .name = "wm8321", .driver_data = WM8321 },
+	{ .name = "wm8325", .driver_data = WM8325 },
+	{ .name = "wm8326", .driver_data = WM8326 },
+	{ }
 };
 
 static struct spi_driver wm831x_spi_driver = {
diff --git a/drivers/misc/cardreader/rtsx_usb.c b/drivers/misc/cardreader/rtsx_usb.c
index 1830e9ed25216..a127744918f42 100644
--- a/drivers/misc/cardreader/rtsx_usb.c
+++ b/drivers/misc/cardreader/rtsx_usb.c
@@ -312,6 +312,9 @@ int rtsx_usb_get_card_status(struct rtsx_ucr *ucr, u16 *status)
 	if (ret < 0)
 		return ret;
 
+	ucr->card_status_cache = *status;
+	ucr->card_status_valid = true;
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(rtsx_usb_get_card_status);
@@ -623,6 +626,7 @@ static int rtsx_usb_probe(struct usb_interface *intf,
 {
 	struct usb_device *usb_dev = interface_to_usbdev(intf);
 	struct rtsx_ucr *ucr;
+	u16 status;
 	int ret;
 
 	dev_dbg(&intf->dev,
@@ -659,6 +663,9 @@ static int rtsx_usb_probe(struct usb_interface *intf,
 	if (ret)
 		goto out_init_fail;
 
+	/* Prime cached status for runtime autosuspend decisions. */
+	rtsx_usb_get_card_status(ucr, &status);
+
 	/* initialize USB SG transfer timer */
 	timer_setup(&ucr->sg_timer, rtsx_usb_sg_timed_out, 0);
 
@@ -713,22 +720,29 @@ static int rtsx_usb_suspend(struct usb_interface *intf, pm_message_t message)
 	struct rtsx_ucr *ucr =
 		(struct rtsx_ucr *)usb_get_intfdata(intf);
 	u16 val = 0;
+	bool valid = false;
 
 	dev_dbg(&intf->dev, "%s called with pm message 0x%04x\n",
 			__func__, message.event);
 
 	if (PMSG_IS_AUTO(message)) {
 		if (mutex_trylock(&ucr->dev_mutex)) {
-			rtsx_usb_get_card_status(ucr, &val);
+			valid = ucr->card_status_valid;
+			if (valid)
+				val = ucr->card_status_cache;
 			mutex_unlock(&ucr->dev_mutex);
 
-			/* Defer the autosuspend if card exists */
-			if (val & (SD_CD | MS_CD)) {
+			/*
+			 * Do not issue USB commands from runtime autosuspend.
+			 * Raw SD_CD is not authoritative on tray-based readers,
+			 * while a real SD card is protected by the SD/MMC child
+			 * runtime-PM reference once the card is powered. Keep
+			 * the historical Memory Stick autosuspend deferral when
+			 * the cached status says MS media is present.
+			 */
+			if (valid && (val & MS_CD)) {
 				device_for_each_child(&intf->dev, NULL, rtsx_usb_resume_child);
 				return -EAGAIN;
-			} else {
-				/* if the card does not exists, clear OCP status */
-				rtsx_usb_write_register(ucr, OCPCTL, MS_OCP_CLEAR, MS_OCP_CLEAR);
 			}
 		} else {
 			/* There is an ongoing operation*/
diff --git a/drivers/misc/eeprom/Kconfig b/drivers/misc/eeprom/Kconfig
index 4d0ce47aa282c..d6af02588c667 100644
--- a/drivers/misc/eeprom/Kconfig
+++ b/drivers/misc/eeprom/Kconfig
@@ -1,52 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0-only
 menu "EEPROM support"
 
-config EEPROM_AT24
-	tristate "I2C EEPROMs / RAMs / ROMs from most vendors"
-	depends on I2C && SYSFS
-	select NVMEM
-	select NVMEM_SYSFS
-	select REGMAP
-	select REGMAP_I2C
-	help
-	  Enable this driver to get read/write support to most I2C EEPROMs
-	  and compatible devices like FRAMs, SRAMs, ROMs etc. After you
-	  configure the driver to know about each chip on your target
-	  board.  Use these generic chip names, instead of vendor-specific
-	  ones like at24c64, 24lc02 or fm24c04:
-
-	     24c00, 24c01, 24c02, spd (readonly 24c02), 24c04, 24c08,
-	     24c16, 24c32, 24c64, 24c128, 24c256, 24c512, 24c1024, 24c2048
-
-	  Unless you like data loss puzzles, always be sure that any chip
-	  you configure as a 24c32 (32 kbit) or larger is NOT really a
-	  24c16 (16 kbit) or smaller, and vice versa. Marking the chip
-	  as read-only won't help recover from this. Also, if your chip
-	  has any software write-protect mechanism you may want to review the
-	  code to make sure this driver won't turn it on by accident.
-
-	  If you use this with an SMBus adapter instead of an I2C adapter,
-	  full functionality is not available.  Only smaller devices are
-	  supported (24c16 and below, max 4 kByte).
-
-	  This driver can also be built as a module.  If so, the module
-	  will be called at24.
-
-config EEPROM_AT25
-	tristate "SPI EEPROMs (FRAMs) from most vendors"
-	depends on SPI && SYSFS
-	select NVMEM
-	select NVMEM_SYSFS
-	select SPI_MEM
-	help
-	  Enable this driver to get read/write support to most SPI EEPROMs
-	  and Cypress FRAMs,
-	  after you configure the board init code to know about each eeprom
-	  on your target board.
-
-	  This driver can also be built as a module.  If so, the module
-	  will be called at25.
-
 config EEPROM_MAX6875
 	tristate "Maxim MAX6874/5 power supply supervisor"
 	depends on I2C
@@ -69,22 +23,6 @@ config EEPROM_93CX6
 
 	  If unsure, say N.
 
-config EEPROM_93XX46
-	tristate "Microwire EEPROM 93XX46 support"
-	depends on SPI && SYSFS
-	select REGMAP
-	select NVMEM
-	select NVMEM_SYSFS
-	help
-	  Driver for the microwire EEPROM chipsets 93xx46x. The driver
-	  supports both read and write commands and also the command to
-	  erase the whole EEPROM.
-
-	  This driver can also be built as a module.  If so, the module
-	  will be called eeprom_93xx46.
-
-	  If unsure, say N.
-
 config EEPROM_DIGSY_MTC_CFG
 	bool "DigsyMTC display configuration EEPROMs device"
 	depends on GPIO_MPC5200 && SPI_GPIO
@@ -107,35 +45,4 @@ config EEPROM_IDT_89HPESX
 	  This driver can also be built as a module. If so, the module
 	  will be called idt_89hpesx.
 
-config EEPROM_EE1004
-	tristate "SPD EEPROMs on DDR4 memory modules"
-	depends on I2C && SYSFS
-	select NVMEM
-	select NVMEM_SYSFS
-	help
-	  Enable this driver to get read support to SPD EEPROMs following
-	  the JEDEC EE1004 standard. These are typically found on DDR4
-	  SDRAM memory modules.
-
-	  This driver can also be built as a module.  If so, the module
-	  will be called ee1004.
-
-config EEPROM_M24LR
-	tristate "STMicroelectronics M24LR RFID/NFC EEPROM support"
-	depends on I2C && SYSFS
-	select REGMAP_I2C
-	select NVMEM
-	select NVMEM_SYSFS
-	help
-	  This enables support for STMicroelectronics M24LR RFID/NFC EEPROM
-	  chips. These dual-interface devices expose two I2C addresses:
-	  one for EEPROM memory access and another for control and system
-	  configuration (e.g. UID, password handling).
-
-	  This driver provides a sysfs interface for control functions and
-	  integrates with the nvmem subsystem for EEPROM access.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called m24lr.
-
 endmenu
diff --git a/drivers/misc/eeprom/Makefile b/drivers/misc/eeprom/Makefile
index 8f311fd6a4ce1..44a20cf025a1b 100644
--- a/drivers/misc/eeprom/Makefile
+++ b/drivers/misc/eeprom/Makefile
@@ -1,10 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_EEPROM_AT24)	+= at24.o
-obj-$(CONFIG_EEPROM_AT25)	+= at25.o
 obj-$(CONFIG_EEPROM_MAX6875)	+= max6875.o
 obj-$(CONFIG_EEPROM_93CX6)	+= eeprom_93cx6.o
-obj-$(CONFIG_EEPROM_93XX46)	+= eeprom_93xx46.o
 obj-$(CONFIG_EEPROM_DIGSY_MTC_CFG) += digsy_mtc_eeprom.o
 obj-$(CONFIG_EEPROM_IDT_89HPESX) += idt_89hpesx.o
-obj-$(CONFIG_EEPROM_EE1004)	+= ee1004.o
-obj-$(CONFIG_EEPROM_M24LR) += m24lr.o
diff --git a/drivers/misc/eeprom/at24.c b/drivers/misc/eeprom/at24.c
deleted file mode 100644
index 772c4d9fa6512..0000000000000
--- a/drivers/misc/eeprom/at24.c
+++ /dev/null
@@ -1,871 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * at24.c - handle most I2C EEPROMs
- *
- * Copyright (C) 2005-2007 David Brownell
- * Copyright (C) 2008 Wolfram Sang, Pengutronix
- */
-
-#include <linux/acpi.h>
-#include <linux/bitops.h>
-#include <linux/capability.h>
-#include <linux/delay.h>
-#include <linux/i2c.h>
-#include <linux/init.h>
-#include <linux/jiffies.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/nvmem-provider.h>
-#include <linux/pm_runtime.h>
-#include <linux/property.h>
-#include <linux/regmap.h>
-#include <linux/regulator/consumer.h>
-#include <linux/slab.h>
-
-/* Address pointer is 16 bit. */
-#define AT24_FLAG_ADDR16	BIT(7)
-/* sysfs-entry will be read-only. */
-#define AT24_FLAG_READONLY	BIT(6)
-/* sysfs-entry will be world-readable. */
-#define AT24_FLAG_IRUGO		BIT(5)
-/* Take always 8 addresses (24c00). */
-#define AT24_FLAG_TAKE8ADDR	BIT(4)
-/* Factory-programmed serial number. */
-#define AT24_FLAG_SERIAL	BIT(3)
-/* Factory-programmed mac address. */
-#define AT24_FLAG_MAC		BIT(2)
-/* Does not auto-rollover reads to the next slave address. */
-#define AT24_FLAG_NO_RDROL	BIT(1)
-
-/*
- * I2C EEPROMs from most vendors are inexpensive and mostly interchangeable.
- * Differences between different vendor product lines (like Atmel AT24C or
- * MicroChip 24LC, etc) won't much matter for typical read/write access.
- * There are also I2C RAM chips, likewise interchangeable. One example
- * would be the PCF8570, which acts like a 24c02 EEPROM (256 bytes).
- *
- * However, misconfiguration can lose data. "Set 16-bit memory address"
- * to a part with 8-bit addressing will overwrite data. Writing with too
- * big a page size also loses data. And it's not safe to assume that the
- * conventional addresses 0x50..0x57 only hold eeproms; a PCF8563 RTC
- * uses 0x51, for just one example.
- *
- * Accordingly, explicit board-specific configuration data should be used
- * in almost all cases. (One partial exception is an SMBus used to access
- * "SPD" data for DRAM sticks. Those only use 24c02 EEPROMs.)
- *
- * So this driver uses "new style" I2C driver binding, expecting to be
- * told what devices exist. That may be in arch/X/mach-Y/board-Z.c or
- * similar kernel-resident tables; or, configuration data coming from
- * a bootloader.
- *
- * Other than binding model, current differences from "eeprom" driver are
- * that this one handles write access and isn't restricted to 24c02 devices.
- * It also handles larger devices (32 kbit and up) with two-byte addresses,
- * which won't work on pure SMBus systems.
- */
-
-struct at24_data {
-	/*
-	 * Lock protects against activities from other Linux tasks,
-	 * but not from changes by other I2C masters.
-	 */
-	struct mutex lock;
-
-	unsigned int write_max;
-	unsigned int num_addresses;
-	unsigned int offset_adj;
-
-	u32 byte_len;
-	u16 page_size;
-	u8 flags;
-
-	struct nvmem_device *nvmem;
-	struct regulator *vcc_reg;
-	void (*read_post)(unsigned int off, char *buf, size_t count);
-
-	/*
-	 * Some chips tie up multiple I2C addresses; dummy devices reserve
-	 * them for us.
-	 */
-	u8 bank_addr_shift;
-	struct regmap *client_regmaps[] __counted_by(num_addresses);
-};
-
-/*
- * This parameter is to help this driver avoid blocking other drivers out
- * of I2C for potentially troublesome amounts of time. With a 100 kHz I2C
- * clock, one 256 byte read takes about 1/43 second which is excessive;
- * but the 1/170 second it takes at 400 kHz may be quite reasonable; and
- * at 1 MHz (Fm+) a 1/430 second delay could easily be invisible.
- *
- * This value is forced to be a power of two so that writes align on pages.
- */
-static unsigned int at24_io_limit = 128;
-module_param_named(io_limit, at24_io_limit, uint, 0);
-MODULE_PARM_DESC(at24_io_limit, "Maximum bytes per I/O (default 128)");
-
-/*
- * Specs often allow 5 msec for a page write, sometimes 20 msec;
- * it's important to recover from write timeouts.
- */
-static unsigned int at24_write_timeout = 25;
-module_param_named(write_timeout, at24_write_timeout, uint, 0);
-MODULE_PARM_DESC(at24_write_timeout, "Time (in ms) to try writes (default 25)");
-
-struct at24_chip_data {
-	u32 byte_len;
-	u8 flags;
-	u8 bank_addr_shift;
-	void (*read_post)(unsigned int off, char *buf, size_t count);
-};
-
-#define AT24_CHIP_DATA(_name, _len, _flags)				\
-	static const struct at24_chip_data _name = {			\
-		.byte_len = _len, .flags = _flags,			\
-	}
-
-#define AT24_CHIP_DATA_CB(_name, _len, _flags, _read_post)		\
-	static const struct at24_chip_data _name = {			\
-		.byte_len = _len, .flags = _flags,			\
-		.read_post = _read_post,				\
-	}
-
-#define AT24_CHIP_DATA_BS(_name, _len, _flags, _bank_addr_shift)	\
-	static const struct at24_chip_data _name = {			\
-		.byte_len = _len, .flags = _flags,			\
-		.bank_addr_shift = _bank_addr_shift			\
-	}
-
-static void at24_read_post_vaio(unsigned int off, char *buf, size_t count)
-{
-	int i;
-
-	if (capable(CAP_SYS_ADMIN))
-		return;
-
-	/*
-	 * Hide VAIO private settings to regular users:
-	 * - BIOS passwords: bytes 0x00 to 0x0f
-	 * - UUID: bytes 0x10 to 0x1f
-	 * - Serial number: 0xc0 to 0xdf
-	 */
-	for (i = 0; i < count; i++) {
-		if ((off + i <= 0x1f) ||
-		    (off + i >= 0xc0 && off + i <= 0xdf))
-			buf[i] = 0;
-	}
-}
-
-/* needs 8 addresses as A0-A2 are ignored */
-AT24_CHIP_DATA(at24_data_24c00, 128 / 8, AT24_FLAG_TAKE8ADDR);
-/* old variants can't be handled with this generic entry! */
-AT24_CHIP_DATA(at24_data_24c01, 1024 / 8, 0);
-AT24_CHIP_DATA(at24_data_24cs01, 16,
-	AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
-AT24_CHIP_DATA(at24_data_24c02, 2048 / 8, 0);
-AT24_CHIP_DATA(at24_data_24cs02, 16,
-	AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
-AT24_CHIP_DATA(at24_data_24mac402, 48 / 8,
-	AT24_FLAG_MAC | AT24_FLAG_READONLY);
-AT24_CHIP_DATA(at24_data_24mac602, 64 / 8,
-	AT24_FLAG_MAC | AT24_FLAG_READONLY);
-AT24_CHIP_DATA(at24_data_24aa025e48, 48 / 8,
-	AT24_FLAG_READONLY);
-AT24_CHIP_DATA(at24_data_24aa025e64, 64 / 8,
-	AT24_FLAG_READONLY);
-/* spd is a 24c02 in memory DIMMs */
-AT24_CHIP_DATA(at24_data_spd, 2048 / 8,
-	AT24_FLAG_READONLY | AT24_FLAG_IRUGO);
-/* 24c02_vaio is a 24c02 on some Sony laptops */
-AT24_CHIP_DATA_CB(at24_data_24c02_vaio, 2048 / 8,
-	AT24_FLAG_READONLY | AT24_FLAG_IRUGO,
-	at24_read_post_vaio);
-AT24_CHIP_DATA(at24_data_24c04, 4096 / 8, 0);
-AT24_CHIP_DATA(at24_data_24cs04, 16,
-	AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
-/* 24rf08 quirk is handled at i2c-core */
-AT24_CHIP_DATA(at24_data_24c08, 8192 / 8, 0);
-AT24_CHIP_DATA(at24_data_24cs08, 16,
-	AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
-AT24_CHIP_DATA(at24_data_24c16, 16384 / 8, 0);
-AT24_CHIP_DATA(at24_data_24cs16, 16,
-	AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
-AT24_CHIP_DATA(at24_data_24c32, 32768 / 8, AT24_FLAG_ADDR16);
-/* M24C32-D Additional Write lockable page (M24C32-D order codes) */
-AT24_CHIP_DATA(at24_data_24c32d_wlp, 32, AT24_FLAG_ADDR16);
-AT24_CHIP_DATA(at24_data_24cs32, 16,
-	AT24_FLAG_ADDR16 | AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
-AT24_CHIP_DATA(at24_data_24c64, 65536 / 8, AT24_FLAG_ADDR16);
-/* M24C64-D Additional Write lockable page (M24C64-D order codes) */
-AT24_CHIP_DATA(at24_data_24c64d_wlp, 32, AT24_FLAG_ADDR16);
-AT24_CHIP_DATA(at24_data_24cs64, 16,
-	AT24_FLAG_ADDR16 | AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
-AT24_CHIP_DATA(at24_data_24c128, 131072 / 8, AT24_FLAG_ADDR16);
-AT24_CHIP_DATA(at24_data_24c256, 262144 / 8, AT24_FLAG_ADDR16);
-/* M24256E Additional Write lockable page (M24256E-F order codes) */
-AT24_CHIP_DATA(at24_data_24256e_wlp, 64, AT24_FLAG_ADDR16);
-AT24_CHIP_DATA(at24_data_24c512, 524288 / 8, AT24_FLAG_ADDR16);
-AT24_CHIP_DATA(at24_data_24c1024, 1048576 / 8, AT24_FLAG_ADDR16);
-AT24_CHIP_DATA_BS(at24_data_24c1025, 1048576 / 8, AT24_FLAG_ADDR16, 2);
-AT24_CHIP_DATA(at24_data_24c2048, 2097152 / 8, AT24_FLAG_ADDR16);
-/* identical to 24c08 ? */
-AT24_CHIP_DATA(at24_data_INT3499, 8192 / 8, 0);
-
-static const struct i2c_device_id at24_ids[] = {
-	{ .name = "24c00",	.driver_data = (kernel_ulong_t)&at24_data_24c00 },
-	{ .name = "24c01",	.driver_data = (kernel_ulong_t)&at24_data_24c01 },
-	{ .name = "24cs01",	.driver_data = (kernel_ulong_t)&at24_data_24cs01 },
-	{ .name = "24c02",	.driver_data = (kernel_ulong_t)&at24_data_24c02 },
-	{ .name = "24cs02",	.driver_data = (kernel_ulong_t)&at24_data_24cs02 },
-	{ .name = "24mac402",	.driver_data = (kernel_ulong_t)&at24_data_24mac402 },
-	{ .name = "24mac602",	.driver_data = (kernel_ulong_t)&at24_data_24mac602 },
-	{ .name = "24aa025e48",	.driver_data = (kernel_ulong_t)&at24_data_24aa025e48 },
-	{ .name = "24aa025e64",	.driver_data = (kernel_ulong_t)&at24_data_24aa025e64 },
-	{ .name = "spd",	.driver_data = (kernel_ulong_t)&at24_data_spd },
-	{ .name = "24c02-vaio",	.driver_data = (kernel_ulong_t)&at24_data_24c02_vaio },
-	{ .name = "24c04",	.driver_data = (kernel_ulong_t)&at24_data_24c04 },
-	{ .name = "24cs04",	.driver_data = (kernel_ulong_t)&at24_data_24cs04 },
-	{ .name = "24c08",	.driver_data = (kernel_ulong_t)&at24_data_24c08 },
-	{ .name = "24cs08",	.driver_data = (kernel_ulong_t)&at24_data_24cs08 },
-	{ .name = "24c16",	.driver_data = (kernel_ulong_t)&at24_data_24c16 },
-	{ .name = "24cs16",	.driver_data = (kernel_ulong_t)&at24_data_24cs16 },
-	{ .name = "24c32",	.driver_data = (kernel_ulong_t)&at24_data_24c32 },
-	{ .name = "24c32d-wl",	.driver_data = (kernel_ulong_t)&at24_data_24c32d_wlp },
-	{ .name = "24cs32",	.driver_data = (kernel_ulong_t)&at24_data_24cs32 },
-	{ .name = "24c64",	.driver_data = (kernel_ulong_t)&at24_data_24c64 },
-	{ .name = "24c64-wl",	.driver_data = (kernel_ulong_t)&at24_data_24c64d_wlp },
-	{ .name = "24cs64",	.driver_data = (kernel_ulong_t)&at24_data_24cs64 },
-	{ .name = "24c128",	.driver_data = (kernel_ulong_t)&at24_data_24c128 },
-	{ .name = "24c256",	.driver_data = (kernel_ulong_t)&at24_data_24c256 },
-	{ .name = "24256e-wl",	.driver_data = (kernel_ulong_t)&at24_data_24256e_wlp },
-	{ .name = "24c512",	.driver_data = (kernel_ulong_t)&at24_data_24c512 },
-	{ .name = "24c1024",	.driver_data = (kernel_ulong_t)&at24_data_24c1024 },
-	{ .name = "24c1025",	.driver_data = (kernel_ulong_t)&at24_data_24c1025 },
-	{ .name = "24c2048",	.driver_data = (kernel_ulong_t)&at24_data_24c2048 },
-	{ .name = "at24",	.driver_data = 0 },
-	{ /* END OF LIST */ }
-};
-MODULE_DEVICE_TABLE(i2c, at24_ids);
-
-static const struct of_device_id at24_of_match[] = {
-	{ .compatible = "atmel,24c00",		.data = &at24_data_24c00 },
-	{ .compatible = "atmel,24c01",		.data = &at24_data_24c01 },
-	{ .compatible = "atmel,24cs01",		.data = &at24_data_24cs01 },
-	{ .compatible = "atmel,24c02",		.data = &at24_data_24c02 },
-	{ .compatible = "atmel,24cs02",		.data = &at24_data_24cs02 },
-	{ .compatible = "atmel,24mac402",	.data = &at24_data_24mac402 },
-	{ .compatible = "atmel,24mac602",	.data = &at24_data_24mac602 },
-	{ .compatible = "atmel,spd",		.data = &at24_data_spd },
-	{ .compatible = "atmel,24c04",		.data = &at24_data_24c04 },
-	{ .compatible = "atmel,24cs04",		.data = &at24_data_24cs04 },
-	{ .compatible = "atmel,24c08",		.data = &at24_data_24c08 },
-	{ .compatible = "atmel,24cs08",		.data = &at24_data_24cs08 },
-	{ .compatible = "atmel,24c16",		.data = &at24_data_24c16 },
-	{ .compatible = "atmel,24cs16",		.data = &at24_data_24cs16 },
-	{ .compatible = "atmel,24c32",		.data = &at24_data_24c32 },
-	{ .compatible = "atmel,24c32d-wl",	.data = &at24_data_24c32d_wlp },
-	{ .compatible = "atmel,24cs32",		.data = &at24_data_24cs32 },
-	{ .compatible = "atmel,24c64",		.data = &at24_data_24c64 },
-	{ .compatible = "atmel,24c64d-wl",	.data = &at24_data_24c64d_wlp },
-	{ .compatible = "atmel,24cs64",		.data = &at24_data_24cs64 },
-	{ .compatible = "atmel,24c128",		.data = &at24_data_24c128 },
-	{ .compatible = "atmel,24c256",		.data = &at24_data_24c256 },
-	{ .compatible = "atmel,24c512",		.data = &at24_data_24c512 },
-	{ .compatible = "atmel,24c1024",	.data = &at24_data_24c1024 },
-	{ .compatible = "atmel,24c1025",	.data = &at24_data_24c1025 },
-	{ .compatible = "atmel,24c2048",	.data = &at24_data_24c2048 },
-	{ .compatible = "microchip,24aa025e48",	.data = &at24_data_24aa025e48 },
-	{ .compatible = "microchip,24aa025e64",	.data = &at24_data_24aa025e64 },
-	{ .compatible = "st,24256e-wl",		.data = &at24_data_24256e_wlp },
-	{ /* END OF LIST */ },
-};
-MODULE_DEVICE_TABLE(of, at24_of_match);
-
-static const struct acpi_device_id at24_acpi_ids[] = {
-	{ "INT3499",	(kernel_ulong_t)&at24_data_INT3499 },
-	{ "TPF0001",	(kernel_ulong_t)&at24_data_24c1024 },
-	{ /* END OF LIST */ }
-};
-MODULE_DEVICE_TABLE(acpi, at24_acpi_ids);
-
-/*
- * This routine supports chips which consume multiple I2C addresses. It
- * computes the addressing information to be used for a given r/w request.
- * Assumes that sanity checks for offset happened at sysfs-layer.
- *
- * Slave address and byte offset derive from the offset. Always
- * set the byte address; on a multi-master board, another master
- * may have changed the chip's "current" address pointer.
- */
-static struct regmap *at24_translate_offset(struct at24_data *at24,
-					    unsigned int *offset)
-{
-	unsigned int i;
-
-	if (at24->flags & AT24_FLAG_ADDR16) {
-		i = *offset >> 16;
-		*offset &= 0xffff;
-	} else {
-		i = *offset >> 8;
-		*offset &= 0xff;
-	}
-
-	return at24->client_regmaps[i];
-}
-
-static struct device *at24_base_client_dev(struct at24_data *at24)
-{
-	return regmap_get_device(at24->client_regmaps[0]);
-}
-
-static size_t at24_adjust_read_count(struct at24_data *at24,
-				      unsigned int offset, size_t count)
-{
-	unsigned int bits;
-	size_t remainder;
-
-	/*
-	 * In case of multi-address chips that don't rollover reads to
-	 * the next slave address: truncate the count to the slave boundary,
-	 * so that the read never straddles slaves.
-	 */
-	if (at24->flags & AT24_FLAG_NO_RDROL) {
-		bits = (at24->flags & AT24_FLAG_ADDR16) ? 16 : 8;
-		remainder = BIT(bits) - offset;
-		if (count > remainder)
-			count = remainder;
-	}
-
-	if (count > at24_io_limit)
-		count = at24_io_limit;
-
-	return count;
-}
-
-static ssize_t at24_regmap_read(struct at24_data *at24, char *buf,
-				unsigned int offset, size_t count)
-{
-	unsigned long timeout, read_time;
-	struct regmap *regmap;
-	int ret;
-
-	regmap = at24_translate_offset(at24, &offset);
-	count = at24_adjust_read_count(at24, offset, count);
-
-	/* adjust offset for mac and serial read ops */
-	offset += at24->offset_adj;
-
-	timeout = jiffies + msecs_to_jiffies(at24_write_timeout);
-	do {
-		/*
-		 * The timestamp shall be taken before the actual operation
-		 * to avoid a premature timeout in case of high CPU load.
-		 */
-		read_time = jiffies;
-
-		ret = regmap_bulk_read(regmap, offset, buf, count);
-		dev_dbg(regmap_get_device(regmap), "read %zu@%d --> %d (%ld)\n",
-			count, offset, ret, jiffies);
-		if (!ret)
-			return count;
-
-		usleep_range(1000, 1500);
-	} while (time_before(read_time, timeout));
-
-	return -ETIMEDOUT;
-}
-
-/*
- * Note that if the hardware write-protect pin is pulled high, the whole
- * chip is normally write protected. But there are plenty of product
- * variants here, including OTP fuses and partial chip protect.
- *
- * We only use page mode writes; the alternative is sloooow. These routines
- * write at most one page.
- */
-
-static size_t at24_adjust_write_count(struct at24_data *at24,
-				      unsigned int offset, size_t count)
-{
-	unsigned int next_page;
-
-	/* write_max is at most a page */
-	if (count > at24->write_max)
-		count = at24->write_max;
-
-	/* Never roll over backwards, to the start of this page */
-	next_page = roundup(offset + 1, at24->page_size);
-	if (offset + count > next_page)
-		count = next_page - offset;
-
-	return count;
-}
-
-static ssize_t at24_regmap_write(struct at24_data *at24, const char *buf,
-				 unsigned int offset, size_t count)
-{
-	unsigned long timeout, write_time;
-	struct regmap *regmap;
-	int ret;
-
-	regmap = at24_translate_offset(at24, &offset);
-	count = at24_adjust_write_count(at24, offset, count);
-	timeout = jiffies + msecs_to_jiffies(at24_write_timeout);
-
-	do {
-		/*
-		 * The timestamp shall be taken before the actual operation
-		 * to avoid a premature timeout in case of high CPU load.
-		 */
-		write_time = jiffies;
-
-		ret = regmap_bulk_write(regmap, offset, buf, count);
-		dev_dbg(regmap_get_device(regmap), "write %zu@%d --> %d (%ld)\n",
-			count, offset, ret, jiffies);
-		if (!ret)
-			return count;
-
-		usleep_range(1000, 1500);
-	} while (time_before(write_time, timeout));
-
-	return -ETIMEDOUT;
-}
-
-static int at24_read(void *priv, unsigned int off, void *val, size_t count)
-{
-	struct at24_data *at24;
-	struct device *dev;
-	char *buf = val;
-	int i, ret;
-
-	at24 = priv;
-	dev = at24_base_client_dev(at24);
-
-	if (unlikely(!count))
-		return count;
-
-	if (off + count > at24->byte_len)
-		return -EINVAL;
-
-	ret = pm_runtime_resume_and_get(dev);
-	if (ret)
-		return ret;
-	/*
-	 * Read data from chip, protecting against concurrent updates
-	 * from this host, but not from other I2C masters.
-	 */
-	mutex_lock(&at24->lock);
-
-	for (i = 0; count; i += ret, count -= ret) {
-		ret = at24_regmap_read(at24, buf + i, off + i, count);
-		if (ret < 0) {
-			mutex_unlock(&at24->lock);
-			pm_runtime_put(dev);
-			return ret;
-		}
-	}
-
-	mutex_unlock(&at24->lock);
-
-	pm_runtime_put(dev);
-
-	if (unlikely(at24->read_post))
-		at24->read_post(off, buf, i);
-
-	return 0;
-}
-
-static int at24_write(void *priv, unsigned int off, void *val, size_t count)
-{
-	struct at24_data *at24;
-	struct device *dev;
-	char *buf = val;
-	int ret;
-
-	at24 = priv;
-	dev = at24_base_client_dev(at24);
-
-	if (unlikely(!count))
-		return -EINVAL;
-
-	if (off + count > at24->byte_len)
-		return -EINVAL;
-
-	ret = pm_runtime_resume_and_get(dev);
-	if (ret)
-		return ret;
-	/*
-	 * Write data to chip, protecting against concurrent updates
-	 * from this host, but not from other I2C masters.
-	 */
-	mutex_lock(&at24->lock);
-
-	while (count) {
-		ret = at24_regmap_write(at24, buf, off, count);
-		if (ret < 0) {
-			mutex_unlock(&at24->lock);
-			pm_runtime_put(dev);
-			return ret;
-		}
-		buf += ret;
-		off += ret;
-		count -= ret;
-	}
-
-	mutex_unlock(&at24->lock);
-
-	pm_runtime_put(dev);
-
-	return 0;
-}
-
-static int at24_make_dummy_client(struct at24_data *at24, unsigned int index,
-				  struct i2c_client *base_client,
-				  struct regmap_config *regmap_config)
-{
-	struct i2c_client *dummy_client;
-	struct regmap *regmap;
-
-	dummy_client = devm_i2c_new_dummy_device(&base_client->dev,
-						 base_client->adapter,
-						 base_client->addr +
-						 (index << at24->bank_addr_shift));
-	if (IS_ERR(dummy_client))
-		return PTR_ERR(dummy_client);
-
-	regmap = devm_regmap_init_i2c(dummy_client, regmap_config);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	at24->client_regmaps[index] = regmap;
-
-	return 0;
-}
-
-static unsigned int at24_get_offset_adj(u8 flags, unsigned int byte_len)
-{
-	if (flags & AT24_FLAG_MAC) {
-		/* EUI-48 starts from 0x9a, EUI-64 from 0x98 */
-		return 0xa0 - byte_len;
-	} else if (flags & AT24_FLAG_SERIAL && flags & AT24_FLAG_ADDR16) {
-		/*
-		 * For 16 bit address pointers, the word address must contain
-		 * a '10' sequence in bits 11 and 10 regardless of the
-		 * intended position of the address pointer.
-		 */
-		return 0x0800;
-	} else if (flags & AT24_FLAG_SERIAL) {
-		/*
-		 * Otherwise the word address must begin with a '10' sequence,
-		 * regardless of the intended address.
-		 */
-		return 0x0080;
-	} else {
-		return 0;
-	}
-}
-
-static void at24_probe_temp_sensor(struct i2c_client *client)
-{
-	struct at24_data *at24 = i2c_get_clientdata(client);
-	struct i2c_board_info info = { .type = "jc42" };
-	int ret;
-	u8 val;
-
-	/*
-	 * Byte 2 has value 11 for DDR3, earlier versions don't
-	 * support the thermal sensor present flag
-	 */
-	ret = at24_read(at24, 2, &val, 1);
-	if (ret || val != 11)
-		return;
-
-	/* Byte 32, bit 7 is set if temp sensor is present */
-	ret = at24_read(at24, 32, &val, 1);
-	if (ret || !(val & BIT(7)))
-		return;
-
-	info.addr = 0x18 | (client->addr & 7);
-
-	i2c_new_client_device(client->adapter, &info);
-}
-
-static int at24_probe(struct i2c_client *client)
-{
-	struct regmap_config regmap_config = { };
-	struct nvmem_config nvmem_config = { };
-	u32 byte_len, page_size, flags, addrw;
-	const struct at24_chip_data *cdata;
-	struct device *dev = &client->dev;
-	bool i2c_fn_i2c, i2c_fn_block;
-	unsigned int i, num_addresses;
-	struct at24_data *at24;
-	bool full_power;
-	struct regmap *regmap;
-	bool writable;
-	u8 test_byte;
-	int err;
-
-	i2c_fn_i2c = i2c_check_functionality(client->adapter, I2C_FUNC_I2C);
-	i2c_fn_block = i2c_check_functionality(client->adapter,
-					       I2C_FUNC_SMBUS_WRITE_I2C_BLOCK);
-
-	cdata = i2c_get_match_data(client);
-	if (!cdata)
-		return -ENODEV;
-
-	err = device_property_read_u32(dev, "pagesize", &page_size);
-	if (err)
-		/*
-		 * This is slow, but we can't know all eeproms, so we better
-		 * play safe. Specifying custom eeprom-types via device tree
-		 * or properties is recommended anyhow.
-		 */
-		page_size = 1;
-
-	flags = cdata->flags;
-	if (device_property_present(dev, "read-only"))
-		flags |= AT24_FLAG_READONLY;
-	if (device_property_present(dev, "no-read-rollover"))
-		flags |= AT24_FLAG_NO_RDROL;
-
-	err = device_property_read_u32(dev, "address-width", &addrw);
-	if (!err) {
-		switch (addrw) {
-		case 8:
-			if (flags & AT24_FLAG_ADDR16)
-				dev_warn(dev,
-					 "Override address width to be 8, while default is 16\n");
-			flags &= ~AT24_FLAG_ADDR16;
-			break;
-		case 16:
-			flags |= AT24_FLAG_ADDR16;
-			break;
-		default:
-			dev_warn(dev, "Bad \"address-width\" property: %u\n",
-				 addrw);
-		}
-	}
-
-	err = device_property_read_u32(dev, "size", &byte_len);
-	if (err)
-		byte_len = cdata->byte_len;
-
-	if (!i2c_fn_i2c && !i2c_fn_block)
-		page_size = 1;
-
-	if (!page_size)
-		return dev_err_probe(dev, -EINVAL, "page_size must not be 0!\n");
-
-	if (!is_power_of_2(page_size))
-		dev_warn(dev, "page_size looks suspicious (no power of 2)!\n");
-
-	err = device_property_read_u32(dev, "num-addresses", &num_addresses);
-	if (err) {
-		if (flags & AT24_FLAG_TAKE8ADDR)
-			num_addresses = 8;
-		else
-			num_addresses =	DIV_ROUND_UP(byte_len,
-				(flags & AT24_FLAG_ADDR16) ? 65536 : 256);
-	}
-
-	if ((flags & AT24_FLAG_SERIAL) && (flags & AT24_FLAG_MAC))
-		return dev_err_probe(dev, -EINVAL,
-				     "invalid device data - cannot have both AT24_FLAG_SERIAL & AT24_FLAG_MAC.");
-
-	regmap_config.val_bits = 8;
-	regmap_config.reg_bits = (flags & AT24_FLAG_ADDR16) ? 16 : 8;
-	regmap_config.disable_locking = true;
-
-	regmap = devm_regmap_init_i2c(client, &regmap_config);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	at24 = devm_kzalloc(dev, struct_size(at24, client_regmaps, num_addresses),
-			    GFP_KERNEL);
-	if (!at24)
-		return -ENOMEM;
-
-	mutex_init(&at24->lock);
-	at24->byte_len = byte_len;
-	at24->page_size = page_size;
-	at24->flags = flags;
-	at24->read_post = cdata->read_post;
-	at24->bank_addr_shift = cdata->bank_addr_shift;
-	at24->num_addresses = num_addresses;
-	at24->offset_adj = at24_get_offset_adj(flags, byte_len);
-	at24->client_regmaps[0] = regmap;
-
-	at24->vcc_reg = devm_regulator_get(dev, "vcc");
-	if (IS_ERR(at24->vcc_reg))
-		return PTR_ERR(at24->vcc_reg);
-
-	writable = !(flags & AT24_FLAG_READONLY);
-	if (writable) {
-		at24->write_max = min_t(unsigned int,
-					page_size, at24_io_limit);
-		if (!i2c_fn_i2c && at24->write_max > I2C_SMBUS_BLOCK_MAX)
-			at24->write_max = I2C_SMBUS_BLOCK_MAX;
-	}
-
-	/* use dummy devices for multiple-address chips */
-	for (i = 1; i < num_addresses; i++) {
-		err = at24_make_dummy_client(at24, i, client, &regmap_config);
-		if (err)
-			return err;
-	}
-
-	/*
-	 * We initialize nvmem_config.id to NVMEM_DEVID_AUTO even if the
-	 * label property is set as some platform can have multiple eeproms
-	 * with same label and we can not register each of those with same
-	 * label. Failing to register those eeproms trigger cascade failure
-	 * on such platform.
-	 */
-	nvmem_config.id = NVMEM_DEVID_AUTO;
-
-	if (device_property_present(dev, "label")) {
-		err = device_property_read_string(dev, "label",
-						  &nvmem_config.name);
-		if (err)
-			return err;
-	} else {
-		nvmem_config.name = dev_name(dev);
-	}
-
-	nvmem_config.type = NVMEM_TYPE_EEPROM;
-	nvmem_config.dev = dev;
-	nvmem_config.read_only = !writable;
-	nvmem_config.root_only = !(flags & AT24_FLAG_IRUGO);
-	nvmem_config.owner = THIS_MODULE;
-	nvmem_config.compat = true;
-	nvmem_config.base_dev = dev;
-	nvmem_config.reg_read = at24_read;
-	nvmem_config.reg_write = at24_write;
-	nvmem_config.priv = at24;
-	nvmem_config.stride = 1;
-	nvmem_config.word_size = 1;
-	nvmem_config.size = byte_len;
-
-	i2c_set_clientdata(client, at24);
-
-	full_power = acpi_dev_state_d0(&client->dev);
-	if (full_power) {
-		err = regulator_enable(at24->vcc_reg);
-		if (err)
-			return dev_err_probe(dev, err, "Failed to enable vcc regulator\n");
-
-		pm_runtime_set_active(dev);
-	}
-	pm_runtime_enable(dev);
-
-	/*
-	 * Perform a one-byte test read to verify that the chip is functional,
-	 * unless powering on the device is to be avoided during probe (i.e.
-	 * it's powered off right now).
-	 */
-	if (full_power) {
-		err = at24_read(at24, 0, &test_byte, 1);
-		if (err) {
-			pm_runtime_disable(dev);
-			if (!pm_runtime_status_suspended(dev))
-				regulator_disable(at24->vcc_reg);
-			return -ENODEV;
-		}
-	}
-
-	at24->nvmem = devm_nvmem_register(dev, &nvmem_config);
-	if (IS_ERR(at24->nvmem)) {
-		pm_runtime_disable(dev);
-		if (!pm_runtime_status_suspended(dev))
-			regulator_disable(at24->vcc_reg);
-		return dev_err_probe(dev, PTR_ERR(at24->nvmem),
-				     "failed to register nvmem\n");
-	}
-
-	/* If this a SPD EEPROM, probe for DDR3 thermal sensor */
-	if (cdata == &at24_data_spd)
-		at24_probe_temp_sensor(client);
-
-	pm_runtime_idle(dev);
-
-	if (writable)
-		dev_info(dev, "%u byte %s EEPROM, writable, %u bytes/write\n",
-			 byte_len, client->name, at24->write_max);
-	else
-		dev_info(dev, "%u byte %s EEPROM, read-only\n",
-			 byte_len, client->name);
-
-	return 0;
-}
-
-static void at24_remove(struct i2c_client *client)
-{
-	struct at24_data *at24 = i2c_get_clientdata(client);
-
-	pm_runtime_disable(&client->dev);
-	if (acpi_dev_state_d0(&client->dev)) {
-		if (!pm_runtime_status_suspended(&client->dev))
-			regulator_disable(at24->vcc_reg);
-		pm_runtime_set_suspended(&client->dev);
-	}
-}
-
-static int __maybe_unused at24_suspend(struct device *dev)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	struct at24_data *at24 = i2c_get_clientdata(client);
-
-	return regulator_disable(at24->vcc_reg);
-}
-
-static int __maybe_unused at24_resume(struct device *dev)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	struct at24_data *at24 = i2c_get_clientdata(client);
-
-	return regulator_enable(at24->vcc_reg);
-}
-
-static const struct dev_pm_ops at24_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
-				pm_runtime_force_resume)
-	SET_RUNTIME_PM_OPS(at24_suspend, at24_resume, NULL)
-};
-
-static struct i2c_driver at24_driver = {
-	.driver = {
-		.name = "at24",
-		.pm = &at24_pm_ops,
-		.of_match_table = at24_of_match,
-		.acpi_match_table = at24_acpi_ids,
-	},
-	.probe = at24_probe,
-	.remove = at24_remove,
-	.id_table = at24_ids,
-	.flags = I2C_DRV_ACPI_WAIVE_D0_PROBE,
-};
-
-static int __init at24_init(void)
-{
-	if (!at24_io_limit) {
-		pr_err("at24: at24_io_limit must not be 0!\n");
-		return -EINVAL;
-	}
-
-	at24_io_limit = rounddown_pow_of_two(at24_io_limit);
-	return i2c_add_driver(&at24_driver);
-}
-module_init(at24_init);
-
-static void __exit at24_exit(void)
-{
-	i2c_del_driver(&at24_driver);
-}
-module_exit(at24_exit);
-
-MODULE_DESCRIPTION("Driver for most I2C EEPROMs");
-MODULE_AUTHOR("David Brownell and Wolfram Sang");
-MODULE_LICENSE("GPL");
diff --git a/drivers/misc/eeprom/at25.c b/drivers/misc/eeprom/at25.c
deleted file mode 100644
index bc2cfb75d9bb4..0000000000000
--- a/drivers/misc/eeprom/at25.c
+++ /dev/null
@@ -1,585 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Driver for most of the SPI EEPROMs, such as Atmel AT25 models
- * and Cypress FRAMs FM25 models.
- *
- * Copyright (C) 2006 David Brownell
- */
-
-#include <linux/bits.h>
-#include <linux/cleanup.h>
-#include <linux/delay.h>
-#include <linux/device.h>
-#include <linux/iopoll.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/property.h>
-#include <linux/sched.h>
-#include <linux/slab.h>
-
-#include <linux/spi/eeprom.h>
-#include <linux/spi/spi.h>
-#include <linux/spi/spi-mem.h>
-
-#include <linux/nvmem-provider.h>
-
-/*
- * NOTE: this is an *EEPROM* driver. The vagaries of product naming
- * mean that some AT25 products are EEPROMs, and others are FLASH.
- * Handle FLASH chips with the drivers/mtd/devices/m25p80.c driver,
- * not this one!
- *
- * EEPROMs that can be used with this driver include, for example:
- *   AT25M02, AT25128B
- */
-
-#define	FM25_SN_LEN	8		/* serial number length */
-#define	FM25_MAX_ID_LEN	9		/* ID length */
-#define EE_MAXADDRLEN	3		/* 24 bit addresses, up to 2 MBytes */
-
-struct at25_data {
-	struct spi_eeprom	chip;
-	struct spi_mem		*spimem;
-	struct mutex		lock;
-	unsigned		addrlen;
-	struct nvmem_config	nvmem_config;
-	struct nvmem_device	*nvmem;
-	u8 sernum[FM25_SN_LEN];
-	u8 id[FM25_MAX_ID_LEN];
-	u8 id_len;
-};
-
-#define	AT25_WREN	0x06		/* latch the write enable */
-#define	AT25_WRDI	0x04		/* reset the write enable */
-#define	AT25_RDSR	0x05		/* read status register */
-#define	AT25_WRSR	0x01		/* write status register */
-#define	AT25_READ	0x03		/* read byte(s) */
-#define	AT25_WRITE	0x02		/* write byte(s)/sector */
-#define	FM25_SLEEP	0xb9		/* enter sleep mode */
-#define	FM25_RDID	0x9f		/* read device ID */
-#define	FM25_RDSN	0xc3		/* read S/N */
-
-#define	AT25_SR_nRDY	0x01		/* nRDY = write-in-progress */
-#define	AT25_SR_WEN	0x02		/* write enable (latched) */
-#define	AT25_SR_BP0	0x04		/* BP for software writeprotect */
-#define	AT25_SR_BP1	0x08
-#define	AT25_SR_WPEN	0x80		/* writeprotect enable */
-
-#define	AT25_INSTR_BIT3	0x08		/* additional address bit in instr */
-
-/*
- * Specs often allow 5ms for a page write, sometimes 20ms;
- * it's important to recover from write timeouts.
- */
-#define	EE_TIMEOUT	25
-
-/*-------------------------------------------------------------------------*/
-
-#define	io_limit	PAGE_SIZE	/* bytes */
-
-/* Handle the address MSB as part of instruction byte */
-static u8 at25_instr(struct at25_data *at25, u8 instr, unsigned int off)
-{
-	if (!(at25->chip.flags & EE_INSTR_BIT3_IS_ADDR))
-		return instr;
-	if (off < BIT(at25->addrlen * 8))
-		return instr;
-	return instr | AT25_INSTR_BIT3;
-}
-
-static int at25_ee_read(void *priv, unsigned int offset,
-			void *val, size_t count)
-{
-	u8 *bounce __free(kfree) = kmalloc(min(count, io_limit), GFP_KERNEL);
-	struct at25_data *at25 = priv;
-	char *buf = val;
-	unsigned int msg_offset = offset;
-	size_t bytes_left = count;
-	size_t segment;
-	int status;
-
-	if (!bounce)
-		return -ENOMEM;
-
-	if (unlikely(offset >= at25->chip.byte_len))
-		return -EINVAL;
-	if ((offset + count) > at25->chip.byte_len)
-		count = at25->chip.byte_len - offset;
-	if (unlikely(!count))
-		return -EINVAL;
-
-	do {
-		struct spi_mem_op op;
-
-		segment = min(bytes_left, io_limit);
-
-		op = (struct spi_mem_op)SPI_MEM_OP(SPI_MEM_OP_CMD(at25_instr(at25, AT25_READ,
-									     msg_offset), 1),
-						   SPI_MEM_OP_ADDR(at25->addrlen, msg_offset, 1),
-						   SPI_MEM_OP_NO_DUMMY,
-						   SPI_MEM_OP_DATA_IN(segment, bounce, 1));
-
-		status = spi_mem_adjust_op_size(at25->spimem, &op);
-		if (status)
-			return status;
-		segment = op.data.nbytes;
-
-		mutex_lock(&at25->lock);
-		status = spi_mem_exec_op(at25->spimem, &op);
-		mutex_unlock(&at25->lock);
-		if (status)
-			return status;
-		memcpy(buf, bounce, segment);
-
-		msg_offset += segment;
-		buf += segment;
-		bytes_left -= segment;
-	} while (bytes_left > 0);
-
-	dev_dbg(&at25->spimem->spi->dev, "read %zu bytes at %d\n",
-		count, offset);
-	return 0;
-}
-
-/*
- * Read extra registers as ID or serial number
- *
- * Allow for the callers to provide @buf on stack (not necessary DMA-capable)
- * by allocating a bounce buffer internally.
- */
-static int fm25_aux_read(struct at25_data *at25, u8 *buf, uint8_t command,
-			 int len)
-{
-	u8 *bounce __free(kfree) = kmalloc(len, GFP_KERNEL);
-	struct spi_mem_op op;
-	int status;
-
-	if (!bounce)
-		return -ENOMEM;
-
-	op = (struct spi_mem_op)SPI_MEM_OP(SPI_MEM_OP_CMD(command, 1),
-					   SPI_MEM_OP_NO_ADDR,
-					   SPI_MEM_OP_NO_DUMMY,
-					   SPI_MEM_OP_DATA_IN(len, bounce, 1));
-
-	status = spi_mem_exec_op(at25->spimem, &op);
-	dev_dbg(&at25->spimem->spi->dev, "read %d aux bytes --> %d\n", len, status);
-	if (status)
-		return status;
-
-	memcpy(buf, bounce, len);
-
-	return 0;
-}
-
-static ssize_t sernum_show(struct device *dev, struct device_attribute *attr, char *buf)
-{
-	struct at25_data *at25;
-
-	at25 = dev_get_drvdata(dev);
-	return sysfs_emit(buf, "%*ph\n", (int)sizeof(at25->sernum), at25->sernum);
-}
-static DEVICE_ATTR_RO(sernum);
-
-static ssize_t jedec_id_show(struct device *dev, struct device_attribute *attr, char *buf)
-{
-	struct at25_data *at25;
-
-	at25 = dev_get_drvdata(dev);
-
-	if (!at25->id_len)
-		return -EOPNOTSUPP;
-
-	return sysfs_emit(buf, "%*phN\n", at25->id_len, at25->id);
-}
-static DEVICE_ATTR_RO(jedec_id);
-
-static struct attribute *at25_attrs[] = {
-	&dev_attr_sernum.attr,
-	&dev_attr_jedec_id.attr,
-	NULL,
-};
-ATTRIBUTE_GROUPS(at25);
-
-/*
- * Poll Read Status Register with timeout
- *
- * Return:
- * 0, if the chip is ready
- * [positive] Status Register value as-is, if the chip is busy
- * [negative] error code in case of read failure
- */
-static int at25_wait_ready(struct at25_data *at25)
-{
-	u8 *bounce __free(kfree) = kmalloc(1, GFP_KERNEL);
-	struct spi_mem_op op;
-	int status;
-
-	if (!bounce)
-		return -ENOMEM;
-
-	op = (struct spi_mem_op)SPI_MEM_OP(SPI_MEM_OP_CMD(AT25_RDSR, 1),
-					   SPI_MEM_OP_NO_ADDR,
-					   SPI_MEM_OP_NO_DUMMY,
-					   SPI_MEM_OP_DATA_IN(1, bounce, 1));
-
-	read_poll_timeout(spi_mem_exec_op, status,
-			  status || !(bounce[0] & AT25_SR_nRDY), false,
-			  USEC_PER_MSEC, USEC_PER_MSEC * EE_TIMEOUT,
-			  at25->spimem, &op);
-	if (status < 0)
-		return status;
-	if (!(bounce[0] & AT25_SR_nRDY))
-		return 0;
-
-	return bounce[0];
-}
-
-static int at25_ee_write(void *priv, unsigned int off, void *val, size_t count)
-{
-	u8 *bounce __free(kfree) = kmalloc(min(count, io_limit), GFP_KERNEL);
-	struct at25_data *at25 = priv;
-	const char *buf = val;
-	unsigned int buf_size;
-	int status;
-
-	if (unlikely(off >= at25->chip.byte_len))
-		return -EFBIG;
-	if ((off + count) > at25->chip.byte_len)
-		count = at25->chip.byte_len - off;
-	if (unlikely(!count))
-		return -EINVAL;
-
-	buf_size = at25->chip.page_size;
-
-	if (!bounce)
-		return -ENOMEM;
-
-	/*
-	 * For write, rollover is within the page ... so we write at
-	 * most one page, then manually roll over to the next page.
-	 */
-	guard(mutex)(&at25->lock);
-	do {
-		struct spi_mem_op op = SPI_MEM_OP(SPI_MEM_OP_CMD(AT25_WREN, 1),
-						  SPI_MEM_OP_NO_ADDR,
-						  SPI_MEM_OP_NO_DUMMY,
-						  SPI_MEM_OP_NO_DATA);
-		unsigned int segment;
-
-		status = spi_mem_exec_op(at25->spimem, &op);
-		if (status < 0) {
-			dev_dbg(&at25->spimem->spi->dev, "WREN --> %d\n", status);
-			return status;
-		}
-
-		/* Write as much of a page as we can */
-		segment = buf_size - (off % buf_size);
-		if (segment > count)
-			segment = count;
-		if (segment > io_limit)
-			segment = io_limit;
-
-		op = (struct spi_mem_op)SPI_MEM_OP(SPI_MEM_OP_CMD(at25_instr(at25, AT25_WRITE, off),
-								  1),
-						   SPI_MEM_OP_ADDR(at25->addrlen, off, 1),
-						   SPI_MEM_OP_NO_DUMMY,
-						   SPI_MEM_OP_DATA_OUT(segment, bounce, 1));
-
-		status = spi_mem_adjust_op_size(at25->spimem, &op);
-		if (status)
-			return status;
-		segment = op.data.nbytes;
-
-		memcpy(bounce, buf, segment);
-
-		status = spi_mem_exec_op(at25->spimem, &op);
-		dev_dbg(&at25->spimem->spi->dev, "write %u bytes at %u --> %d\n",
-			segment, off, status);
-		if (status)
-			return status;
-
-		/*
-		 * REVISIT this should detect (or prevent) failed writes
-		 * to read-only sections of the EEPROM...
-		 */
-
-		status = at25_wait_ready(at25);
-		if (status < 0) {
-			dev_err_probe(&at25->spimem->spi->dev, status,
-				      "Read Status Redister command failed\n");
-			return status;
-		}
-		if (status) {
-			dev_dbg(&at25->spimem->spi->dev,
-				"Status %02x\n", status);
-			dev_err(&at25->spimem->spi->dev,
-				"write %u bytes offset %u, timeout after %u msecs\n",
-				segment, off, EE_TIMEOUT);
-			return -ETIMEDOUT;
-		}
-
-		off += segment;
-		buf += segment;
-		count -= segment;
-
-	} while (count > 0);
-
-	return status;
-}
-
-/*-------------------------------------------------------------------------*/
-
-static int at25_fw_to_chip(struct device *dev, struct spi_eeprom *chip)
-{
-	u32 val;
-	int err;
-
-	strscpy(chip->name, "at25", sizeof(chip->name));
-
-	err = device_property_read_u32(dev, "size", &val);
-	if (err)
-		err = device_property_read_u32(dev, "at25,byte-len", &val);
-	if (err) {
-		dev_err(dev, "Error: missing \"size\" property\n");
-		return err;
-	}
-	chip->byte_len = val;
-
-	err = device_property_read_u32(dev, "pagesize", &val);
-	if (err)
-		err = device_property_read_u32(dev, "at25,page-size", &val);
-	if (err) {
-		dev_err(dev, "Error: missing \"pagesize\" property\n");
-		return err;
-	}
-	chip->page_size = val;
-
-	err = device_property_read_u32(dev, "address-width", &val);
-	if (err) {
-		err = device_property_read_u32(dev, "at25,addr-mode", &val);
-		if (err) {
-			dev_err(dev, "Error: missing \"address-width\" property\n");
-			return err;
-		}
-		chip->flags = (u16)val;
-	} else {
-		switch (val) {
-		case 9:
-			chip->flags |= EE_INSTR_BIT3_IS_ADDR;
-			fallthrough;
-		case 8:
-			chip->flags |= EE_ADDR1;
-			break;
-		case 16:
-			chip->flags |= EE_ADDR2;
-			break;
-		case 24:
-			chip->flags |= EE_ADDR3;
-			break;
-		default:
-			dev_err(dev,
-				"Error: bad \"address-width\" property: %u\n",
-				val);
-			return -ENODEV;
-		}
-		if (device_property_present(dev, "read-only"))
-			chip->flags |= EE_READONLY;
-	}
-	return 0;
-}
-
-static int at25_fram_to_chip(struct device *dev, struct spi_eeprom *chip)
-{
-	struct at25_data *at25 = container_of(chip, struct at25_data, chip);
-	u8 sernum[FM25_SN_LEN];
-	u8 id[FM25_MAX_ID_LEN];
-	u32 val;
-	int i;
-
-	strscpy(chip->name, "fm25", sizeof(chip->name));
-
-	if (!device_property_read_u32(dev, "size", &val)) {
-		chip->byte_len = val;
-	} else {
-		/* Get ID of chip */
-		fm25_aux_read(at25, id, FM25_RDID, FM25_MAX_ID_LEN);
-
-		/* Store the unprocessed ID for exposing via sysfs */
-		memcpy(at25->id, id, FM25_MAX_ID_LEN);
-		at25->id_len = FM25_MAX_ID_LEN;
-
-		/* There are inside-out FRAM variations, detect them and reverse the ID bytes */
-		if (id[6] == 0x7f && id[2] == 0xc2)
-			for (i = 0; i < ARRAY_SIZE(id) / 2; i++) {
-				u8 tmp = id[i];
-				int j = ARRAY_SIZE(id) - i - 1;
-
-				id[i] = id[j];
-				id[j] = tmp;
-			}
-
-		if (id[6] == 0xc2) {
-			at25->id_len = 9;
-			switch (id[7]) {
-			case 0x21 ... 0x26:
-				chip->byte_len = BIT(id[7] - 0x21 + 4) * 1024;
-				break;
-			case 0x2a ... 0x30:
-				/* CY15B102QN ... CY15B116QN */
-				chip->byte_len = BIT(((id[7] >> 1) & 0xf) + 13);
-				break;
-			default:
-				dev_err(dev, "Error: unsupported size (id %02x)\n", id[7]);
-				return -ENODEV;
-			}
-		} else if (id[2] == 0x82 && id[3] == 0x06) {
-			at25->id_len = 8;
-			switch (id[1]) {
-			case 0x51 ... 0x54:
-				/* CY15B102QSN ... CY15B204QSN */
-				chip->byte_len = BIT(((id[0] >> 3) & 0x1F) + 9);
-				break;
-			default:
-				dev_err(dev, "Error: unsupported product id %02x\n", id[1]);
-				return -ENODEV;
-			}
-		} else {
-			dev_err(dev, "Error: unrecognized JEDEC ID format: %*ph\n",
-				FM25_MAX_ID_LEN, id);
-			return -ENODEV;
-		}
-
-		fm25_aux_read(at25, sernum, FM25_RDSN, FM25_SN_LEN);
-		/* Swap byte order */
-		for (i = 0; i < FM25_SN_LEN; i++)
-			at25->sernum[i] = sernum[FM25_SN_LEN - 1 - i];
-	}
-
-	if (chip->byte_len > 64 * 1024)
-		chip->flags |= EE_ADDR3;
-	else
-		chip->flags |= EE_ADDR2;
-
-	chip->page_size = PAGE_SIZE;
-	return 0;
-}
-
-static const struct of_device_id at25_of_match[] = {
-	{ .compatible = "atmel,at25" },
-	{ .compatible = "cypress,fm25" },
-	{ }
-};
-MODULE_DEVICE_TABLE(of, at25_of_match);
-
-static const struct spi_device_id at25_spi_ids[] = {
-	{ .name = "at25" },
-	{ .name = "fm25" },
-	{ }
-};
-MODULE_DEVICE_TABLE(spi, at25_spi_ids);
-
-static int at25_probe(struct spi_mem *mem)
-{
-	struct spi_device *spi = mem->spi;
-	struct spi_eeprom *pdata;
-	struct at25_data *at25;
-	bool is_fram;
-	int err;
-
-	at25 = devm_kzalloc(&spi->dev, sizeof(*at25), GFP_KERNEL);
-	if (!at25)
-		return -ENOMEM;
-
-	at25->spimem = mem;
-
-	/*
-	 * Ping the chip ... the status register is pretty portable,
-	 * unlike probing manufacturer IDs.
-	 */
-	err = at25_wait_ready(at25);
-	if (err < 0)
-		return dev_err_probe(&spi->dev, err, "Read Status Register command failed\n");
-	if (err) {
-		dev_err(&spi->dev, "Not ready (%02x)\n", err);
-		return -ENXIO;
-	}
-
-	mutex_init(&at25->lock);
-	spi_set_drvdata(spi, at25);
-
-	is_fram = fwnode_device_is_compatible(dev_fwnode(&spi->dev), "cypress,fm25");
-
-	/* Chip description */
-	pdata = dev_get_platdata(&spi->dev);
-	if (pdata) {
-		at25->chip = *pdata;
-	} else {
-		if (is_fram)
-			err = at25_fram_to_chip(&spi->dev, &at25->chip);
-		else
-			err = at25_fw_to_chip(&spi->dev, &at25->chip);
-		if (err)
-			return err;
-	}
-
-	/* For now we only support 8/16/24 bit addressing */
-	if (at25->chip.flags & EE_ADDR1)
-		at25->addrlen = 1;
-	else if (at25->chip.flags & EE_ADDR2)
-		at25->addrlen = 2;
-	else if (at25->chip.flags & EE_ADDR3)
-		at25->addrlen = 3;
-	else {
-		dev_dbg(&spi->dev, "unsupported address type\n");
-		return -EINVAL;
-	}
-
-	at25->nvmem_config.type = is_fram ? NVMEM_TYPE_FRAM : NVMEM_TYPE_EEPROM;
-	at25->nvmem_config.name = dev_name(&spi->dev);
-	at25->nvmem_config.dev = &spi->dev;
-	at25->nvmem_config.read_only = at25->chip.flags & EE_READONLY;
-	at25->nvmem_config.root_only = true;
-	at25->nvmem_config.owner = THIS_MODULE;
-	at25->nvmem_config.compat = true;
-	at25->nvmem_config.base_dev = &spi->dev;
-	at25->nvmem_config.reg_read = at25_ee_read;
-	at25->nvmem_config.reg_write = at25_ee_write;
-	at25->nvmem_config.priv = at25;
-	at25->nvmem_config.stride = 1;
-	at25->nvmem_config.word_size = 1;
-	at25->nvmem_config.size = at25->chip.byte_len;
-
-	at25->nvmem = devm_nvmem_register(&spi->dev, &at25->nvmem_config);
-	if (IS_ERR(at25->nvmem))
-		return PTR_ERR(at25->nvmem);
-
-	dev_info(&spi->dev, "%d %s %s %s%s, pagesize %u\n",
-		 (at25->chip.byte_len < 1024) ?
-			at25->chip.byte_len : (at25->chip.byte_len / 1024),
-		 (at25->chip.byte_len < 1024) ? "Byte" : "KByte",
-		 at25->chip.name, is_fram ? "fram" : "eeprom",
-		 (at25->chip.flags & EE_READONLY) ? " (readonly)" : "",
-		 at25->chip.page_size);
-	return 0;
-}
-
-/*-------------------------------------------------------------------------*/
-
-static struct spi_mem_driver at25_driver = {
-	.spidrv = {
-		.driver = {
-			.name		= "at25",
-			.of_match_table = at25_of_match,
-			.dev_groups	= at25_groups,
-		},
-		.id_table	= at25_spi_ids,
-	},
-	.probe		= at25_probe,
-};
-
-module_spi_mem_driver(at25_driver);
-
-MODULE_DESCRIPTION("Driver for most SPI EEPROMs");
-MODULE_AUTHOR("David Brownell");
-MODULE_LICENSE("GPL");
diff --git a/drivers/misc/eeprom/ee1004.c b/drivers/misc/eeprom/ee1004.c
deleted file mode 100644
index 923f404a44c03..0000000000000
--- a/drivers/misc/eeprom/ee1004.c
+++ /dev/null
@@ -1,351 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * ee1004 - driver for DDR4 SPD EEPROMs
- *
- * Copyright (C) 2017-2019 Jean Delvare
- *
- * Based on the at24 driver:
- * Copyright (C) 2005-2007 David Brownell
- * Copyright (C) 2008 Wolfram Sang, Pengutronix
- */
-
-#include <linux/device.h>
-#include <linux/i2c.h>
-#include <linux/init.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/nvmem-provider.h>
-
-/*
- * DDR4 memory modules use special EEPROMs following the Jedec EE1004
- * specification. These are 512-byte EEPROMs using a single I2C address
- * in the 0x50-0x57 range for data. One of two 256-byte page is selected
- * by writing a command to I2C address 0x36 or 0x37 on the same I2C bus.
- *
- * Therefore we need to request these 2 additional addresses, and serialize
- * access to all such EEPROMs with a single mutex.
- *
- * We assume it is safe to read up to 32 bytes at once from these EEPROMs.
- * We use SMBus access even if I2C is available, these EEPROMs are small
- * enough, and reading from them infrequent enough, that we favor simplicity
- * over performance.
- */
-
-#define EE1004_MAX_BUSSES		8
-#define EE1004_ADDR_SET_PAGE		0x36
-#define EE1004_NUM_PAGES		2
-#define EE1004_PAGE_SIZE		256
-#define EE1004_PAGE_SHIFT		8
-#define EE1004_EEPROM_SIZE		(EE1004_PAGE_SIZE * EE1004_NUM_PAGES)
-
-/*
- * Mutex protects ee1004_set_page and ee1004_dev_count, and must be held
- * from page selection to end of read.
- */
-static DEFINE_MUTEX(ee1004_bus_lock);
-
-static struct ee1004_bus_data {
-	struct i2c_adapter *adap;
-	struct i2c_client *set_page[EE1004_NUM_PAGES];
-	unsigned int dev_count;
-	int current_page;
-} ee1004_bus_data[EE1004_MAX_BUSSES];
-
-static const struct i2c_device_id ee1004_ids[] = {
-	{ "ee1004" },
-	{ }
-};
-MODULE_DEVICE_TABLE(i2c, ee1004_ids);
-
-/*-------------------------------------------------------------------------*/
-
-static struct ee1004_bus_data *ee1004_get_bus_data(struct i2c_adapter *adap)
-{
-	int i;
-
-	for (i = 0; i < EE1004_MAX_BUSSES; i++)
-		if (ee1004_bus_data[i].adap == adap)
-			return ee1004_bus_data + i;
-
-	/* If not existent yet, create new entry */
-	for (i = 0; i < EE1004_MAX_BUSSES; i++)
-		if (!ee1004_bus_data[i].adap) {
-			ee1004_bus_data[i].adap = adap;
-			return ee1004_bus_data + i;
-		}
-
-	return NULL;
-}
-
-static int ee1004_get_current_page(struct ee1004_bus_data *bd)
-{
-	int err;
-
-	err = i2c_smbus_read_byte(bd->set_page[0]);
-	if (err == -ENXIO) {
-		/* Nack means page 1 is selected */
-		return 1;
-	}
-	if (err < 0) {
-		/* Anything else is a real error, bail out */
-		return err;
-	}
-
-	/* Ack means page 0 is selected, returned value meaningless */
-	return 0;
-}
-
-static int ee1004_set_current_page(struct i2c_client *client, int page)
-{
-	struct ee1004_bus_data *bd = i2c_get_clientdata(client);
-	int ret;
-
-	if (page == bd->current_page)
-		return 0;
-
-	/* Data is ignored */
-	ret = i2c_smbus_write_byte(bd->set_page[page], 0x00);
-	/*
-	 * Don't give up just yet. Some memory modules will select the page
-	 * but not ack the command. Check which page is selected now.
-	 */
-	if (ret == -ENXIO && ee1004_get_current_page(bd) == page)
-		ret = 0;
-	if (ret < 0) {
-		dev_err(&client->dev, "Failed to select page %d (%d)\n", page, ret);
-		return ret;
-	}
-
-	dev_dbg(&client->dev, "Selected page %d\n", page);
-	bd->current_page = page;
-
-	return 0;
-}
-
-static ssize_t ee1004_eeprom_read(struct i2c_client *client, char *buf,
-				  unsigned int offset, size_t count)
-{
-	int status, page;
-
-	page = offset >> EE1004_PAGE_SHIFT;
-	offset &= (1 << EE1004_PAGE_SHIFT) - 1;
-
-	status = ee1004_set_current_page(client, page);
-	if (status)
-		return status;
-
-	/* Can't cross page boundaries */
-	if (offset + count > EE1004_PAGE_SIZE)
-		count = EE1004_PAGE_SIZE - offset;
-
-	if (count > I2C_SMBUS_BLOCK_MAX)
-		count = I2C_SMBUS_BLOCK_MAX;
-
-	return i2c_smbus_read_i2c_block_data_or_emulated(client, offset, count, buf);
-}
-
-static int ee1004_read(void *priv, unsigned int off, void *val, size_t count)
-{
-	struct i2c_client *client = priv;
-	char *buf = val;
-	int ret;
-
-	if (unlikely(!count))
-		return count;
-
-	if (off + count > EE1004_EEPROM_SIZE)
-		return -EINVAL;
-
-	/*
-	 * Read data from chip, protecting against concurrent access to
-	 * other EE1004 SPD EEPROMs on the same adapter.
-	 */
-	mutex_lock(&ee1004_bus_lock);
-
-	while (count) {
-		ret = ee1004_eeprom_read(client, buf, off, count);
-		if (ret < 0) {
-			mutex_unlock(&ee1004_bus_lock);
-			return ret;
-		}
-
-		buf += ret;
-		off += ret;
-		count -= ret;
-	}
-
-	mutex_unlock(&ee1004_bus_lock);
-
-	return 0;
-}
-
-static void ee1004_probe_temp_sensor(struct i2c_client *client)
-{
-	struct i2c_board_info info = { .type = "jc42" };
-	unsigned short addr = 0x18 | (client->addr & 7);
-	unsigned short addr_list[] = { addr, I2C_CLIENT_END };
-	u8 data[2];
-	int ret;
-
-	/* byte 14, bit 7 is set if temp sensor is present */
-	ret = ee1004_eeprom_read(client, data, 14, 1);
-	if (ret != 1)
-		return;
-
-	if (!(data[0] & BIT(7))) {
-		/*
-		 * If the SPD data suggests that there is no temperature
-		 * sensor, it may still be there for SPD revision 1.0.
-		 * See SPD Annex L, Revision 1 and 2, for details.
-		 * Check DIMM type and SPD revision; if it is a DDR4
-		 * with SPD revision 1.0, check the thermal sensor address
-		 * and instantiate the jc42 driver if a chip is found at
-		 * that address.
-		 * It is not necessary to check if there is a chip at the
-		 * temperature sensor address since i2c_new_scanned_device()
-		 * will do that and return silently if no chip is found.
-		 */
-		ret = ee1004_eeprom_read(client, data, 1, 2);
-		if (ret != 2 || data[0] != 0x10 || data[1] != 0x0c)
-			return;
-	}
-	i2c_new_scanned_device(client->adapter, &info, addr_list, NULL);
-}
-
-static void ee1004_cleanup(int idx, struct ee1004_bus_data *bd)
-{
-	if (--bd->dev_count == 0) {
-		while (--idx >= 0)
-			i2c_unregister_device(bd->set_page[idx]);
-		memset(bd, 0, sizeof(struct ee1004_bus_data));
-	}
-}
-
-static void ee1004_cleanup_bus_data(void *data)
-{
-	struct ee1004_bus_data *bd = data;
-
-	/* Remove page select clients if this is the last device */
-	mutex_lock(&ee1004_bus_lock);
-	ee1004_cleanup(EE1004_NUM_PAGES, bd);
-	mutex_unlock(&ee1004_bus_lock);
-}
-
-static int ee1004_init_bus_data(struct i2c_client *client)
-{
-	struct ee1004_bus_data *bd;
-	int err, cnr = 0;
-
-	bd = ee1004_get_bus_data(client->adapter);
-	if (!bd)
-		return dev_err_probe(&client->dev, -ENOSPC, "Only %d busses supported",
-				     EE1004_MAX_BUSSES);
-
-	i2c_set_clientdata(client, bd);
-
-	if (++bd->dev_count == 1) {
-		/* Use 2 dummy devices for page select command */
-		for (cnr = 0; cnr < EE1004_NUM_PAGES; cnr++) {
-			struct i2c_client *cl;
-
-			cl = i2c_new_dummy_device(client->adapter, EE1004_ADDR_SET_PAGE + cnr);
-			if (IS_ERR(cl)) {
-				err = PTR_ERR(cl);
-				goto err_out;
-			}
-
-			bd->set_page[cnr] = cl;
-		}
-
-		/* Remember current page to avoid unneeded page select */
-		err = ee1004_get_current_page(bd);
-		if (err < 0)
-			goto err_out;
-
-		dev_dbg(&client->dev, "Currently selected page: %d\n", err);
-		bd->current_page = err;
-	}
-
-	return 0;
-
-err_out:
-	ee1004_cleanup(cnr, bd);
-
-	return err;
-}
-
-static int ee1004_probe(struct i2c_client *client)
-{
-	struct nvmem_config config = {
-		.dev = &client->dev,
-		.name = dev_name(&client->dev),
-		.id = NVMEM_DEVID_NONE,
-		.owner = THIS_MODULE,
-		.type = NVMEM_TYPE_EEPROM,
-		.read_only = true,
-		.root_only = false,
-		.reg_read = ee1004_read,
-		.size = EE1004_EEPROM_SIZE,
-		.word_size = 1,
-		.stride = 1,
-		.priv = client,
-		.compat = true,
-		.base_dev = &client->dev,
-	};
-	struct nvmem_device *ndev;
-	int err;
-
-	/* Make sure we can operate on this adapter */
-	if (!i2c_check_functionality(client->adapter,
-				     I2C_FUNC_SMBUS_BYTE | I2C_FUNC_SMBUS_READ_I2C_BLOCK) &&
-	    !i2c_check_functionality(client->adapter,
-				     I2C_FUNC_SMBUS_BYTE | I2C_FUNC_SMBUS_READ_BYTE_DATA))
-		return -EPFNOSUPPORT;
-
-	err = i2c_smbus_read_byte(client);
-	if (err < 0)
-		return -ENODEV;
-
-	mutex_lock(&ee1004_bus_lock);
-
-	err = ee1004_init_bus_data(client);
-	if (err < 0) {
-		mutex_unlock(&ee1004_bus_lock);
-		return err;
-	}
-
-	ee1004_probe_temp_sensor(client);
-
-	mutex_unlock(&ee1004_bus_lock);
-
-	err = devm_add_action_or_reset(&client->dev, ee1004_cleanup_bus_data,
-				       i2c_get_clientdata(client));
-	if (err < 0)
-		return err;
-
-	ndev = devm_nvmem_register(&client->dev, &config);
-	if (IS_ERR(ndev))
-		return PTR_ERR(ndev);
-
-	dev_info(&client->dev,
-		 "%u byte EE1004-compliant SPD EEPROM, read-only\n",
-		 EE1004_EEPROM_SIZE);
-
-	return 0;
-}
-
-/*-------------------------------------------------------------------------*/
-
-static struct i2c_driver ee1004_driver = {
-	.driver = {
-		.name = "ee1004",
-	},
-	.probe = ee1004_probe,
-	.id_table = ee1004_ids,
-};
-module_i2c_driver(ee1004_driver);
-
-MODULE_DESCRIPTION("Driver for EE1004-compliant DDR4 SPD EEPROMs");
-MODULE_AUTHOR("Jean Delvare");
-MODULE_LICENSE("GPL");
diff --git a/drivers/misc/eeprom/eeprom_93xx46.c b/drivers/misc/eeprom/eeprom_93xx46.c
deleted file mode 100644
index f9c3ab52c2f92..0000000000000
--- a/drivers/misc/eeprom/eeprom_93xx46.c
+++ /dev/null
@@ -1,560 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Driver for 93xx46 EEPROMs
- *
- * (C) 2011 DENX Software Engineering, Anatolij Gustschin <agust@denx.de>
- */
-
-#include <linux/array_size.h>
-#include <linux/bits.h>
-#include <linux/delay.h>
-#include <linux/device.h>
-#include <linux/gpio/consumer.h>
-#include <linux/kstrtox.h>
-#include <linux/log2.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/property.h>
-#include <linux/slab.h>
-#include <linux/spi/spi.h>
-#include <linux/string_choices.h>
-
-#include <linux/nvmem-provider.h>
-
-struct eeprom_93xx46_platform_data {
-	unsigned char	flags;
-#define EE_ADDR8	0x01		/*  8 bit addr. cfg */
-#define EE_ADDR16	0x02		/* 16 bit addr. cfg */
-#define EE_READONLY	0x08		/* forbid writing */
-#define EE_SIZE1K	0x10		/* 1 kb of data, that is a 93xx46 */
-#define EE_SIZE2K	0x20		/* 2 kb of data, that is a 93xx56 */
-#define EE_SIZE4K	0x40		/* 4 kb of data, that is a 93xx66 */
-
-	unsigned int	quirks;
-/* Single word read transfers only; no sequential read. */
-#define EEPROM_93XX46_QUIRK_SINGLE_WORD_READ		(1 << 0)
-/* Instructions such as EWEN are (addrlen + 2) in length. */
-#define EEPROM_93XX46_QUIRK_INSTRUCTION_LENGTH		(1 << 1)
-/* Add extra cycle after address during a read */
-#define EEPROM_93XX46_QUIRK_EXTRA_READ_CYCLE		BIT(2)
-
-	struct gpio_desc *select;
-};
-
-#define OP_START	0x4
-#define OP_WRITE	(OP_START | 0x1)
-#define OP_READ		(OP_START | 0x2)
-/* The following addresses are offset for the 1K EEPROM variant in 16-bit mode */
-#define ADDR_EWDS	0x00
-#define ADDR_ERAL	0x20
-#define ADDR_EWEN	0x30
-
-struct eeprom_93xx46_devtype_data {
-	unsigned int quirks;
-	unsigned char flags;
-};
-
-static const struct eeprom_93xx46_devtype_data at93c46_data = {
-	.flags = EE_SIZE1K,
-};
-
-static const struct eeprom_93xx46_devtype_data at93c56_data = {
-	.flags = EE_SIZE2K,
-};
-
-static const struct eeprom_93xx46_devtype_data at93c66_data = {
-	.flags = EE_SIZE4K,
-};
-
-static const struct eeprom_93xx46_devtype_data atmel_at93c46d_data = {
-	.flags = EE_SIZE1K,
-	.quirks = EEPROM_93XX46_QUIRK_SINGLE_WORD_READ |
-		  EEPROM_93XX46_QUIRK_INSTRUCTION_LENGTH,
-};
-
-static const struct eeprom_93xx46_devtype_data microchip_93lc46b_data = {
-	.flags = EE_SIZE1K,
-	.quirks = EEPROM_93XX46_QUIRK_EXTRA_READ_CYCLE,
-};
-
-struct eeprom_93xx46_dev {
-	struct spi_device *spi;
-	struct eeprom_93xx46_platform_data *pdata;
-	struct mutex lock;
-	struct nvmem_config nvmem_config;
-	struct nvmem_device *nvmem;
-	int addrlen;
-	int size;
-};
-
-static inline bool has_quirk_single_word_read(struct eeprom_93xx46_dev *edev)
-{
-	return edev->pdata->quirks & EEPROM_93XX46_QUIRK_SINGLE_WORD_READ;
-}
-
-static inline bool has_quirk_instruction_length(struct eeprom_93xx46_dev *edev)
-{
-	return edev->pdata->quirks & EEPROM_93XX46_QUIRK_INSTRUCTION_LENGTH;
-}
-
-static inline bool has_quirk_extra_read_cycle(struct eeprom_93xx46_dev *edev)
-{
-	return edev->pdata->quirks & EEPROM_93XX46_QUIRK_EXTRA_READ_CYCLE;
-}
-
-static int eeprom_93xx46_read(void *priv, unsigned int off,
-			      void *val, size_t count)
-{
-	struct eeprom_93xx46_dev *edev = priv;
-	char *buf = val;
-	int err = 0;
-	int bits;
-
-	if (unlikely(off >= edev->size))
-		return 0;
-	if ((off + count) > edev->size)
-		count = edev->size - off;
-	if (unlikely(!count))
-		return count;
-
-	mutex_lock(&edev->lock);
-
-	gpiod_set_value_cansleep(edev->pdata->select, 1);
-
-	/* The opcode in front of the address is three bits. */
-	bits = edev->addrlen + 3;
-
-	while (count) {
-		struct spi_message m;
-		struct spi_transfer t[2] = {};
-		u16 cmd_addr = OP_READ << edev->addrlen;
-		size_t nbytes = count;
-
-		if (edev->pdata->flags & EE_ADDR8) {
-			cmd_addr |= off;
-			if (has_quirk_single_word_read(edev))
-				nbytes = 1;
-		} else {
-			cmd_addr |= (off >> 1);
-			if (has_quirk_single_word_read(edev))
-				nbytes = 2;
-		}
-
-		dev_dbg(&edev->spi->dev, "read cmd 0x%x, %d Hz\n",
-			cmd_addr, edev->spi->max_speed_hz);
-
-		if (has_quirk_extra_read_cycle(edev)) {
-			cmd_addr <<= 1;
-			bits += 1;
-		}
-
-		t[0].tx_buf = (char *)&cmd_addr;
-		t[0].len = 2;
-		t[0].bits_per_word = bits;
-
-		t[1].rx_buf = buf;
-		t[1].len = count;
-		t[1].bits_per_word = 8;
-
-		spi_message_init_with_transfers(&m, t, ARRAY_SIZE(t));
-
-		err = spi_sync(edev->spi, &m);
-		/* have to wait at least Tcsl ns */
-		ndelay(250);
-
-		if (err) {
-			dev_err(&edev->spi->dev, "read %zu bytes at %u: err. %d\n",
-				nbytes, off, err);
-			break;
-		}
-
-		buf += nbytes;
-		off += nbytes;
-		count -= nbytes;
-	}
-
-	gpiod_set_value_cansleep(edev->pdata->select, 0);
-
-	mutex_unlock(&edev->lock);
-
-	return err;
-}
-
-static int eeprom_93xx46_ew(struct eeprom_93xx46_dev *edev, int is_on)
-{
-	struct spi_message m;
-	struct spi_transfer t = {};
-	int bits, ret;
-	u16 cmd_addr;
-
-	/* The opcode in front of the address is three bits. */
-	bits = edev->addrlen + 3;
-
-	cmd_addr = OP_START << edev->addrlen;
-	cmd_addr |= (is_on ? ADDR_EWEN : ADDR_EWDS) << (edev->addrlen - 6);
-
-	if (has_quirk_instruction_length(edev)) {
-		cmd_addr <<= 2;
-		bits += 2;
-	}
-
-	dev_dbg(&edev->spi->dev, "ew %s cmd 0x%04x, %d bits\n",
-		str_enable_disable(is_on), cmd_addr, bits);
-
-	t.tx_buf = &cmd_addr;
-	t.len = 2;
-	t.bits_per_word = bits;
-
-	spi_message_init_with_transfers(&m, &t, 1);
-
-	mutex_lock(&edev->lock);
-
-	gpiod_set_value_cansleep(edev->pdata->select, 1);
-
-	ret = spi_sync(edev->spi, &m);
-	/* have to wait at least Tcsl ns */
-	ndelay(250);
-	if (ret)
-		dev_err(&edev->spi->dev, "erase/write %s error %d\n",
-			str_enable_disable(is_on), ret);
-
-	gpiod_set_value_cansleep(edev->pdata->select, 0);
-
-	mutex_unlock(&edev->lock);
-	return ret;
-}
-
-static ssize_t
-eeprom_93xx46_write_word(struct eeprom_93xx46_dev *edev,
-			 const char *buf, unsigned int off)
-{
-	struct spi_message m;
-	struct spi_transfer t[2] = {};
-	int bits, data_len, ret;
-	u16 cmd_addr;
-
-	if (unlikely(off >= edev->size))
-		return -EINVAL;
-
-	/* The opcode in front of the address is three bits. */
-	bits = edev->addrlen + 3;
-
-	cmd_addr = OP_WRITE << edev->addrlen;
-
-	if (edev->pdata->flags & EE_ADDR8) {
-		cmd_addr |= off;
-		data_len = 1;
-	} else {
-		cmd_addr |= (off >> 1);
-		data_len = 2;
-	}
-
-	dev_dbg(&edev->spi->dev, "write cmd 0x%x\n", cmd_addr);
-
-	t[0].tx_buf = (char *)&cmd_addr;
-	t[0].len = 2;
-	t[0].bits_per_word = bits;
-
-	t[1].tx_buf = buf;
-	t[1].len = data_len;
-	t[1].bits_per_word = 8;
-
-	spi_message_init_with_transfers(&m, t, ARRAY_SIZE(t));
-
-	ret = spi_sync(edev->spi, &m);
-	/* have to wait program cycle time Twc ms */
-	mdelay(6);
-	return ret;
-}
-
-static int eeprom_93xx46_write(void *priv, unsigned int off,
-				   void *val, size_t count)
-{
-	struct eeprom_93xx46_dev *edev = priv;
-	char *buf = val;
-	int ret, step = 1;
-	unsigned int i;
-
-	if (unlikely(off >= edev->size))
-		return -EFBIG;
-	if ((off + count) > edev->size)
-		count = edev->size - off;
-	if (unlikely(!count))
-		return count;
-
-	/* only write even number of bytes on 16-bit devices */
-	if (edev->pdata->flags & EE_ADDR16) {
-		step = 2;
-		count &= ~1;
-	}
-
-	/* erase/write enable */
-	ret = eeprom_93xx46_ew(edev, 1);
-	if (ret)
-		return ret;
-
-	mutex_lock(&edev->lock);
-
-	gpiod_set_value_cansleep(edev->pdata->select, 1);
-
-	for (i = 0; i < count; i += step) {
-		ret = eeprom_93xx46_write_word(edev, &buf[i], off + i);
-		if (ret) {
-			dev_err(&edev->spi->dev, "write failed at %u: %d\n", off + i, ret);
-			break;
-		}
-	}
-
-	gpiod_set_value_cansleep(edev->pdata->select, 0);
-
-	mutex_unlock(&edev->lock);
-
-	/* erase/write disable */
-	eeprom_93xx46_ew(edev, 0);
-	return ret;
-}
-
-static int eeprom_93xx46_eral(struct eeprom_93xx46_dev *edev)
-{
-	struct spi_message m;
-	struct spi_transfer t = {};
-	int bits, ret;
-	u16 cmd_addr;
-
-	/* The opcode in front of the address is three bits. */
-	bits = edev->addrlen + 3;
-
-	cmd_addr = OP_START << edev->addrlen;
-	cmd_addr |= ADDR_ERAL << (edev->addrlen - 6);
-
-	if (has_quirk_instruction_length(edev)) {
-		cmd_addr <<= 2;
-		bits += 2;
-	}
-
-	dev_dbg(&edev->spi->dev, "eral cmd 0x%04x, %d bits\n", cmd_addr, bits);
-
-	t.tx_buf = &cmd_addr;
-	t.len = 2;
-	t.bits_per_word = bits;
-
-	spi_message_init_with_transfers(&m, &t, 1);
-
-	mutex_lock(&edev->lock);
-
-	gpiod_set_value_cansleep(edev->pdata->select, 1);
-
-	ret = spi_sync(edev->spi, &m);
-	if (ret)
-		dev_err(&edev->spi->dev, "erase error %d\n", ret);
-	/* have to wait erase cycle time Tec ms */
-	mdelay(6);
-
-	gpiod_set_value_cansleep(edev->pdata->select, 0);
-
-	mutex_unlock(&edev->lock);
-	return ret;
-}
-
-static ssize_t erase_store(struct device *dev, struct device_attribute *attr,
-			   const char *buf, size_t count)
-{
-	struct eeprom_93xx46_dev *edev = dev_get_drvdata(dev);
-	bool erase;
-	int ret;
-
-	ret = kstrtobool(buf, &erase);
-	if (ret)
-		return ret;
-
-	if (erase) {
-		ret = eeprom_93xx46_ew(edev, 1);
-		if (ret)
-			return ret;
-		ret = eeprom_93xx46_eral(edev);
-		if (ret)
-			return ret;
-		ret = eeprom_93xx46_ew(edev, 0);
-		if (ret)
-			return ret;
-	}
-	return count;
-}
-static DEVICE_ATTR_WO(erase);
-
-static const struct of_device_id eeprom_93xx46_of_table[] = {
-	{ .compatible = "eeprom-93xx46", .data = &at93c46_data, },
-	{ .compatible = "atmel,at93c46", .data = &at93c46_data, },
-	{ .compatible = "atmel,at93c46d", .data = &atmel_at93c46d_data, },
-	{ .compatible = "atmel,at93c56", .data = &at93c56_data, },
-	{ .compatible = "atmel,at93c66", .data = &at93c66_data, },
-	{ .compatible = "microchip,93lc46b", .data = &microchip_93lc46b_data, },
-	{}
-};
-MODULE_DEVICE_TABLE(of, eeprom_93xx46_of_table);
-
-static const struct spi_device_id eeprom_93xx46_spi_ids[] = {
-	{ .name = "eeprom-93xx46",
-	  .driver_data = (kernel_ulong_t)&at93c46_data, },
-	{ .name = "at93c46",
-	  .driver_data = (kernel_ulong_t)&at93c46_data, },
-	{ .name = "at93c46d",
-	  .driver_data = (kernel_ulong_t)&atmel_at93c46d_data, },
-	{ .name = "at93c56",
-	  .driver_data = (kernel_ulong_t)&at93c56_data, },
-	{ .name = "at93c66",
-	  .driver_data = (kernel_ulong_t)&at93c66_data, },
-	{ .name = "93lc46b",
-	  .driver_data = (kernel_ulong_t)&microchip_93lc46b_data, },
-	{}
-};
-MODULE_DEVICE_TABLE(spi, eeprom_93xx46_spi_ids);
-
-static int eeprom_93xx46_probe_fw(struct device *dev)
-{
-	const struct eeprom_93xx46_devtype_data *data;
-	struct eeprom_93xx46_platform_data *pd;
-	u32 tmp;
-	int ret;
-
-	pd = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL);
-	if (!pd)
-		return -ENOMEM;
-
-	ret = device_property_read_u32(dev, "data-size", &tmp);
-	if (ret < 0) {
-		dev_err(dev, "data-size property not found\n");
-		return ret;
-	}
-
-	if (tmp == 8) {
-		pd->flags |= EE_ADDR8;
-	} else if (tmp == 16) {
-		pd->flags |= EE_ADDR16;
-	} else {
-		dev_err(dev, "invalid data-size (%d)\n", tmp);
-		return -EINVAL;
-	}
-
-	if (device_property_read_bool(dev, "read-only"))
-		pd->flags |= EE_READONLY;
-
-	pd->select = devm_gpiod_get_optional(dev, "select", GPIOD_OUT_LOW);
-	if (IS_ERR(pd->select))
-		return PTR_ERR(pd->select);
-	gpiod_set_consumer_name(pd->select, "93xx46 EEPROMs OE");
-
-	data = spi_get_device_match_data(to_spi_device(dev));
-	if (data) {
-		pd->quirks = data->quirks;
-		pd->flags |= data->flags;
-	}
-
-	dev->platform_data = pd;
-
-	return 0;
-}
-
-static int eeprom_93xx46_probe(struct spi_device *spi)
-{
-	struct eeprom_93xx46_platform_data *pd;
-	struct eeprom_93xx46_dev *edev;
-	struct device *dev = &spi->dev;
-	int err;
-
-	err = eeprom_93xx46_probe_fw(dev);
-	if (err < 0)
-		return err;
-
-	pd = spi->dev.platform_data;
-	if (!pd) {
-		dev_err(&spi->dev, "missing platform data\n");
-		return -ENODEV;
-	}
-
-	edev = devm_kzalloc(&spi->dev, sizeof(*edev), GFP_KERNEL);
-	if (!edev)
-		return -ENOMEM;
-
-	if (pd->flags & EE_SIZE1K)
-		edev->size = 128;
-	else if (pd->flags & EE_SIZE2K)
-		edev->size = 256;
-	else if (pd->flags & EE_SIZE4K)
-		edev->size = 512;
-	else {
-		dev_err(&spi->dev, "unspecified size\n");
-		return -EINVAL;
-	}
-
-	if (pd->flags & EE_ADDR8)
-		edev->addrlen = ilog2(edev->size);
-	else if (pd->flags & EE_ADDR16)
-		edev->addrlen = ilog2(edev->size) - 1;
-	else {
-		dev_err(&spi->dev, "unspecified address type\n");
-		return -EINVAL;
-	}
-
-	mutex_init(&edev->lock);
-
-	edev->spi = spi;
-	edev->pdata = pd;
-
-	edev->nvmem_config.type = NVMEM_TYPE_EEPROM;
-	edev->nvmem_config.name = dev_name(&spi->dev);
-	edev->nvmem_config.dev = &spi->dev;
-	edev->nvmem_config.read_only = pd->flags & EE_READONLY;
-	edev->nvmem_config.root_only = true;
-	edev->nvmem_config.owner = THIS_MODULE;
-	edev->nvmem_config.compat = true;
-	edev->nvmem_config.base_dev = &spi->dev;
-	edev->nvmem_config.reg_read = eeprom_93xx46_read;
-	edev->nvmem_config.reg_write = eeprom_93xx46_write;
-	edev->nvmem_config.priv = edev;
-	edev->nvmem_config.stride = 4;
-	edev->nvmem_config.word_size = 1;
-	edev->nvmem_config.size = edev->size;
-
-	edev->nvmem = devm_nvmem_register(&spi->dev, &edev->nvmem_config);
-	if (IS_ERR(edev->nvmem))
-		return PTR_ERR(edev->nvmem);
-
-	dev_info(&spi->dev, "%d-bit eeprom containing %d bytes %s\n",
-		(pd->flags & EE_ADDR8) ? 8 : 16,
-		edev->size,
-		(pd->flags & EE_READONLY) ? "(readonly)" : "");
-
-	if (!(pd->flags & EE_READONLY)) {
-		if (device_create_file(&spi->dev, &dev_attr_erase))
-			dev_err(&spi->dev, "can't create erase interface\n");
-	}
-
-	spi_set_drvdata(spi, edev);
-	return 0;
-}
-
-static void eeprom_93xx46_remove(struct spi_device *spi)
-{
-	struct eeprom_93xx46_dev *edev = spi_get_drvdata(spi);
-
-	if (!(edev->pdata->flags & EE_READONLY))
-		device_remove_file(&spi->dev, &dev_attr_erase);
-}
-
-static struct spi_driver eeprom_93xx46_driver = {
-	.driver = {
-		.name	= "93xx46",
-		.of_match_table = eeprom_93xx46_of_table,
-	},
-	.probe		= eeprom_93xx46_probe,
-	.remove		= eeprom_93xx46_remove,
-	.id_table	= eeprom_93xx46_spi_ids,
-};
-
-module_spi_driver(eeprom_93xx46_driver);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Driver for 93xx46 EEPROMs");
-MODULE_AUTHOR("Anatolij Gustschin <agust@denx.de>");
-MODULE_ALIAS("spi:93xx46");
diff --git a/drivers/misc/eeprom/m24lr.c b/drivers/misc/eeprom/m24lr.c
deleted file mode 100644
index 7a9fd45a8e460..0000000000000
--- a/drivers/misc/eeprom/m24lr.c
+++ /dev/null
@@ -1,606 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * m24lr.c - Sysfs control interface for ST M24LR series RFID/NFC chips
- *
- * Copyright (c) 2025 Abd-Alrhman Masalkhi <abd.masalkhi@gmail.com>
- *
- * This driver implements both the sysfs-based control interface and EEPROM
- * access for STMicroelectronics M24LR series chips (e.g., M24LR04E-R).
- * It provides access to control registers for features such as password
- * authentication, memory protection, and device configuration. In addition,
- * it manages read and write operations to the EEPROM region of the chip.
- */
-
-#include <linux/device.h>
-#include <linux/i2c.h>
-#include <linux/module.h>
-#include <linux/nvmem-provider.h>
-#include <linux/of.h>
-#include <linux/of_device.h>
-#include <linux/regmap.h>
-
-#define M24LR_WRITE_TIMEOUT	  25u
-#define M24LR_READ_TIMEOUT	  (M24LR_WRITE_TIMEOUT)
-
-/**
- * struct m24lr_chip - describes chip-specific sysfs layout
- * @sss_len:       the length of the sss region
- * @page_size:	   chip-specific limit on the maximum number of bytes allowed
- *		   in a single write operation.
- * @eeprom_size:   size of the EEPROM in byte
- *
- * Supports multiple M24LR chip variants (e.g., M24LRxx) by allowing each
- * to define its own set of sysfs attributes, depending on its available
- * registers and features.
- */
-struct m24lr_chip {
-	unsigned int sss_len;
-	unsigned int page_size;
-	unsigned int eeprom_size;
-};
-
-/**
- * struct m24lr - core driver data for M24LR chip control
- * @uid:           64 bits unique identifier stored in the device
- * @sss_len:       the length of the sss region
- * @page_size:	   chip-specific limit on the maximum number of bytes allowed
- *		   in a single write operation.
- * @eeprom_size:   size of the EEPROM in byte
- * @ctl_regmap:	   regmap interface for accessing the system parameter sector
- * @eeprom_regmap: regmap interface for accessing the EEPROM
- * @lock:	   mutex to synchronize operations to the device
- *
- * Central data structure holding the state and resources used by the
- * M24LR device driver.
- */
-struct m24lr {
-	u64 uid;
-	unsigned int sss_len;
-	unsigned int page_size;
-	unsigned int eeprom_size;
-	struct regmap *ctl_regmap;
-	struct regmap *eeprom_regmap;
-	struct mutex lock;	 /* synchronize operations to the device */
-};
-
-static const struct regmap_range m24lr_ctl_vo_ranges[] = {
-	regmap_reg_range(0, 63),
-};
-
-static const struct regmap_access_table m24lr_ctl_vo_table = {
-	.yes_ranges = m24lr_ctl_vo_ranges,
-	.n_yes_ranges = ARRAY_SIZE(m24lr_ctl_vo_ranges),
-};
-
-static const struct regmap_config m24lr_ctl_regmap_conf = {
-	.name = "m24lr_ctl",
-	.reg_stride = 1,
-	.reg_bits = 16,
-	.val_bits = 8,
-	.disable_locking = false,
-	.cache_type = REGCACHE_RBTREE,/* Flat can't be used, there's huge gap */
-	.volatile_table = &m24lr_ctl_vo_table,
-};
-
-/* Chip descriptor for M24LR04E-R variant */
-static const struct m24lr_chip m24lr04e_r_chip = {
-	.page_size = 4,
-	.eeprom_size = 512,
-	.sss_len = 4,
-};
-
-/* Chip descriptor for M24LR16E-R variant */
-static const struct m24lr_chip m24lr16e_r_chip = {
-	.page_size = 4,
-	.eeprom_size = 2048,
-	.sss_len = 16,
-};
-
-/* Chip descriptor for M24LR64E-R variant */
-static const struct m24lr_chip m24lr64e_r_chip = {
-	.page_size = 4,
-	.eeprom_size = 8192,
-	.sss_len = 64,
-};
-
-static const struct i2c_device_id m24lr_ids[] = {
-	{ "m24lr04e-r", (kernel_ulong_t)&m24lr04e_r_chip},
-	{ "m24lr16e-r", (kernel_ulong_t)&m24lr16e_r_chip},
-	{ "m24lr64e-r", (kernel_ulong_t)&m24lr64e_r_chip},
-	{ }
-};
-MODULE_DEVICE_TABLE(i2c, m24lr_ids);
-
-static const struct of_device_id m24lr_of_match[] = {
-	{ .compatible = "st,m24lr04e-r", .data = &m24lr04e_r_chip},
-	{ .compatible = "st,m24lr16e-r", .data = &m24lr16e_r_chip},
-	{ .compatible = "st,m24lr64e-r", .data = &m24lr64e_r_chip},
-	{ }
-};
-MODULE_DEVICE_TABLE(of, m24lr_of_match);
-
-/**
- * m24lr_regmap_read - read data using regmap with retry on failure
- * @regmap:  regmap instance for the device
- * @buf:     buffer to store the read data
- * @size:    number of bytes to read
- * @offset:  starting register address
- *
- * Attempts to read a block of data from the device with retries and timeout.
- * Some M24LR chips may transiently NACK reads (e.g., during internal write
- * cycles), so this function retries with a short sleep until the timeout
- * expires.
- *
- * Returns:
- *	 Number of bytes read on success,
- *	 -ETIMEDOUT if the read fails within the timeout window.
- */
-static ssize_t m24lr_regmap_read(struct regmap *regmap, u8 *buf,
-				 size_t size, unsigned int offset)
-{
-	int err;
-	unsigned long timeout, read_time;
-	ssize_t ret = -ETIMEDOUT;
-
-	timeout = jiffies + msecs_to_jiffies(M24LR_READ_TIMEOUT);
-	do {
-		read_time = jiffies;
-
-		err = regmap_bulk_read(regmap, offset, buf, size);
-		if (!err) {
-			ret = size;
-			break;
-		}
-
-		usleep_range(1000, 2000);
-	} while (time_before(read_time, timeout));
-
-	return ret;
-}
-
-/**
- * m24lr_regmap_write - write data using regmap with retry on failure
- * @regmap: regmap instance for the device
- * @buf:    buffer containing the data to write
- * @size:   number of bytes to write
- * @offset: starting register address
- *
- * Attempts to write a block of data to the device with retries and a timeout.
- * Some M24LR devices may NACK I2C writes while an internal write operation
- * is in progress. This function retries the write operation with a short delay
- * until it succeeds or the timeout is reached.
- *
- * Returns:
- *	 Number of bytes written on success,
- *	 -ETIMEDOUT if the write fails within the timeout window.
- */
-static ssize_t m24lr_regmap_write(struct regmap *regmap, const u8 *buf,
-				  size_t size, unsigned int offset)
-{
-	int err;
-	unsigned long timeout, write_time;
-	ssize_t ret = -ETIMEDOUT;
-
-	timeout = jiffies + msecs_to_jiffies(M24LR_WRITE_TIMEOUT);
-
-	do {
-		write_time = jiffies;
-
-		err = regmap_bulk_write(regmap, offset, buf, size);
-		if (!err) {
-			ret = size;
-			break;
-		}
-
-		usleep_range(1000, 2000);
-	} while (time_before(write_time, timeout));
-
-	return ret;
-}
-
-static ssize_t m24lr_read(struct m24lr *m24lr, u8 *buf, size_t size,
-			  unsigned int offset, bool is_eeprom)
-{
-	struct regmap *regmap;
-	ssize_t ret;
-
-	if (is_eeprom)
-		regmap = m24lr->eeprom_regmap;
-	else
-		regmap = m24lr->ctl_regmap;
-
-	mutex_lock(&m24lr->lock);
-	ret = m24lr_regmap_read(regmap, buf, size, offset);
-	mutex_unlock(&m24lr->lock);
-
-	return ret;
-}
-
-/**
- * m24lr_write - write buffer to M24LR device with page alignment handling
- * @m24lr:     pointer to driver context
- * @buf:       data buffer to write
- * @size:      number of bytes to write
- * @offset:    target register address in the device
- * @is_eeprom: true if the write should target the EEPROM,
- *             false if it should target the system parameters sector.
- *
- * Writes data to the M24LR device using regmap, split into chunks no larger
- * than page_size to respect device-specific write limitations (e.g., page
- * size or I2C hold-time concerns). Each chunk is aligned to the page boundary
- * defined by page_size.
- *
- * Returns:
- *	 Total number of bytes written on success,
- *	 A negative error code if any write fails.
- */
-static ssize_t m24lr_write(struct m24lr *m24lr, const u8 *buf, size_t size,
-			   unsigned int offset, bool is_eeprom)
-{
-	unsigned int n, next_sector;
-	struct regmap *regmap;
-	ssize_t ret = 0;
-	ssize_t err;
-
-	if (is_eeprom)
-		regmap = m24lr->eeprom_regmap;
-	else
-		regmap = m24lr->ctl_regmap;
-
-	n = min_t(unsigned int, size, m24lr->page_size);
-	next_sector = roundup(offset + 1, m24lr->page_size);
-	if (offset + n > next_sector)
-		n = next_sector - offset;
-
-	mutex_lock(&m24lr->lock);
-	while (n) {
-		err = m24lr_regmap_write(regmap, buf + offset, n, offset);
-		if (IS_ERR_VALUE(err)) {
-			if (!ret)
-				ret = err;
-
-			break;
-		}
-
-		offset += n;
-		size -= n;
-		ret += n;
-		n = min_t(unsigned int, size, m24lr->page_size);
-	}
-	mutex_unlock(&m24lr->lock);
-
-	return ret;
-}
-
-/**
- * m24lr_write_pass - Write password to M24LR043-R using secure format
- * @m24lr: Pointer to device control structure
- * @buf:   Input buffer containing hex-encoded password
- * @count: Number of bytes in @buf
- * @code:  Operation code to embed between password copies
- *
- * This function parses a 4-byte password, encodes it in  big-endian format,
- * and constructs a 9-byte sequence of the form:
- *
- *	  [BE(password), code, BE(password)]
- *
- * The result is written to register 0x0900 (2304), which is the password
- * register in M24LR04E-R chip.
- *
- * Return: Number of bytes written on success, or negative error code on failure
- */
-static ssize_t m24lr_write_pass(struct m24lr *m24lr, const char *buf,
-				size_t count, u8 code)
-{
-	__be32 be_pass;
-	u8 output[9];
-	ssize_t ret;
-	u32 pass;
-	int err;
-
-	if (!count)
-		return -EINVAL;
-
-	if (count > 8)
-		return -EINVAL;
-
-	err = kstrtou32(buf, 16, &pass);
-	if (err)
-		return err;
-
-	be_pass = cpu_to_be32(pass);
-
-	memcpy(output, &be_pass, sizeof(be_pass));
-	output[4] = code;
-	memcpy(output + 5, &be_pass, sizeof(be_pass));
-
-	mutex_lock(&m24lr->lock);
-	ret = m24lr_regmap_write(m24lr->ctl_regmap, output, 9, 2304);
-	mutex_unlock(&m24lr->lock);
-
-	return ret;
-}
-
-static ssize_t m24lr_read_reg_le(struct m24lr *m24lr, u64 *val,
-				 unsigned int reg_addr,
-				 unsigned int reg_size)
-{
-	ssize_t ret;
-	__le64 input = 0;
-
-	ret = m24lr_read(m24lr, (u8 *)&input, reg_size, reg_addr, false);
-	if (IS_ERR_VALUE(ret))
-		return ret;
-
-	if (ret != reg_size)
-		return -EINVAL;
-
-	switch (reg_size) {
-	case 1:
-		*val = *(u8 *)&input;
-		break;
-	case 2:
-		*val = le16_to_cpu((__le16)input);
-		break;
-	case 4:
-		*val = le32_to_cpu((__le32)input);
-		break;
-	case 8:
-		*val = le64_to_cpu((__le64)input);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int m24lr_nvmem_read(void *priv, unsigned int offset, void *val,
-			    size_t bytes)
-{
-	ssize_t err;
-	struct m24lr *m24lr = priv;
-
-	if (!bytes)
-		return bytes;
-
-	if (offset + bytes > m24lr->eeprom_size)
-		return -EINVAL;
-
-	err = m24lr_read(m24lr, val, bytes, offset, true);
-	if (IS_ERR_VALUE(err))
-		return err;
-
-	return 0;
-}
-
-static int m24lr_nvmem_write(void *priv, unsigned int offset, void *val,
-			     size_t bytes)
-{
-	ssize_t err;
-	struct m24lr *m24lr = priv;
-
-	if (!bytes)
-		return -EINVAL;
-
-	if (offset + bytes > m24lr->eeprom_size)
-		return -EINVAL;
-
-	err = m24lr_write(m24lr, val, bytes, offset, true);
-	if (IS_ERR_VALUE(err))
-		return err;
-
-	return 0;
-}
-
-static ssize_t m24lr_ctl_sss_read(struct file *filep, struct kobject *kobj,
-				  const struct bin_attribute *attr, char *buf,
-				  loff_t offset, size_t count)
-{
-	struct m24lr *m24lr = attr->private;
-
-	if (!count)
-		return count;
-
-	if (size_add(offset, count) > m24lr->sss_len)
-		return -EINVAL;
-
-	return m24lr_read(m24lr, buf, count, offset, false);
-}
-
-static ssize_t m24lr_ctl_sss_write(struct file *filep, struct kobject *kobj,
-				   const struct bin_attribute *attr, char *buf,
-				   loff_t offset, size_t count)
-{
-	struct m24lr *m24lr = attr->private;
-
-	if (!count)
-		return -EINVAL;
-
-	if (size_add(offset, count) > m24lr->sss_len)
-		return -EINVAL;
-
-	return m24lr_write(m24lr, buf, count, offset, false);
-}
-static BIN_ATTR(sss, 0600, m24lr_ctl_sss_read, m24lr_ctl_sss_write, 0);
-
-static ssize_t new_pass_store(struct device *dev, struct device_attribute *attr,
-			      const char *buf, size_t count)
-{
-	struct m24lr *m24lr = i2c_get_clientdata(to_i2c_client(dev));
-
-	return m24lr_write_pass(m24lr, buf, count, 7);
-}
-static DEVICE_ATTR_WO(new_pass);
-
-static ssize_t unlock_store(struct device *dev, struct device_attribute *attr,
-			    const char *buf, size_t count)
-{
-	struct m24lr *m24lr = i2c_get_clientdata(to_i2c_client(dev));
-
-	return m24lr_write_pass(m24lr, buf, count, 9);
-}
-static DEVICE_ATTR_WO(unlock);
-
-static ssize_t uid_show(struct device *dev, struct device_attribute *attr,
-			char *buf)
-{
-	struct m24lr *m24lr = i2c_get_clientdata(to_i2c_client(dev));
-
-	return sysfs_emit(buf, "%llx\n", m24lr->uid);
-}
-static DEVICE_ATTR_RO(uid);
-
-static ssize_t total_sectors_show(struct device *dev,
-				  struct device_attribute *attr, char *buf)
-{
-	struct m24lr *m24lr = i2c_get_clientdata(to_i2c_client(dev));
-
-	return sysfs_emit(buf, "%x\n", m24lr->sss_len);
-}
-static DEVICE_ATTR_RO(total_sectors);
-
-static struct attribute *m24lr_ctl_dev_attrs[] = {
-	&dev_attr_unlock.attr,
-	&dev_attr_new_pass.attr,
-	&dev_attr_uid.attr,
-	&dev_attr_total_sectors.attr,
-	NULL,
-};
-
-static const struct m24lr_chip *m24lr_get_chip(struct device *dev)
-{
-	const struct m24lr_chip *ret;
-	const struct i2c_device_id *id;
-
-	id = i2c_match_id(m24lr_ids, to_i2c_client(dev));
-
-	if (dev->of_node && of_match_device(m24lr_of_match, dev))
-		ret = of_device_get_match_data(dev);
-	else if (id)
-		ret = (void *)id->driver_data;
-	else
-		ret = acpi_device_get_match_data(dev);
-
-	return ret;
-}
-
-static int m24lr_probe(struct i2c_client *client)
-{
-	struct regmap_config eeprom_regmap_conf = {0};
-	struct nvmem_config nvmem_conf = {0};
-	struct device *dev = &client->dev;
-	struct i2c_client *eeprom_client;
-	const struct m24lr_chip *chip;
-	struct regmap *eeprom_regmap;
-	struct nvmem_device *nvmem;
-	struct regmap *ctl_regmap;
-	struct m24lr *m24lr;
-	u32 regs[2];
-	long err;
-
-	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C))
-		return -EOPNOTSUPP;
-
-	chip = m24lr_get_chip(dev);
-	if (!chip)
-		return -ENODEV;
-
-	m24lr = devm_kzalloc(dev, sizeof(struct m24lr), GFP_KERNEL);
-	if (!m24lr)
-		return -ENOMEM;
-
-	err = device_property_read_u32_array(dev, "reg", regs, ARRAY_SIZE(regs));
-	if (err)
-		return dev_err_probe(dev, err, "Failed to read 'reg' property\n");
-
-	/* Create a second I2C client for the eeprom interface */
-	eeprom_client = devm_i2c_new_dummy_device(dev, client->adapter, regs[1]);
-	if (IS_ERR(eeprom_client))
-		return dev_err_probe(dev, PTR_ERR(eeprom_client),
-				     "Failed to create dummy I2C client for the EEPROM\n");
-
-	ctl_regmap = devm_regmap_init_i2c(client, &m24lr_ctl_regmap_conf);
-	if (IS_ERR(ctl_regmap))
-		return dev_err_probe(dev, PTR_ERR(ctl_regmap),
-				      "Failed to init regmap\n");
-
-	eeprom_regmap_conf.name = "m24lr_eeprom";
-	eeprom_regmap_conf.reg_bits = 16;
-	eeprom_regmap_conf.val_bits = 8;
-	eeprom_regmap_conf.disable_locking = true;
-	eeprom_regmap_conf.max_register = chip->eeprom_size - 1;
-
-	eeprom_regmap = devm_regmap_init_i2c(eeprom_client,
-					     &eeprom_regmap_conf);
-	if (IS_ERR(eeprom_regmap))
-		return dev_err_probe(dev, PTR_ERR(eeprom_regmap),
-				     "Failed to init regmap\n");
-
-	mutex_init(&m24lr->lock);
-	m24lr->sss_len = chip->sss_len;
-	m24lr->page_size = chip->page_size;
-	m24lr->eeprom_size = chip->eeprom_size;
-	m24lr->eeprom_regmap = eeprom_regmap;
-	m24lr->ctl_regmap = ctl_regmap;
-
-	nvmem_conf.dev = &eeprom_client->dev;
-	nvmem_conf.owner = THIS_MODULE;
-	nvmem_conf.type = NVMEM_TYPE_EEPROM;
-	nvmem_conf.reg_read = m24lr_nvmem_read;
-	nvmem_conf.reg_write = m24lr_nvmem_write;
-	nvmem_conf.size = chip->eeprom_size;
-	nvmem_conf.word_size = 1;
-	nvmem_conf.stride = 1;
-	nvmem_conf.priv = m24lr;
-
-	nvmem = devm_nvmem_register(dev, &nvmem_conf);
-	if (IS_ERR(nvmem))
-		return dev_err_probe(dev, PTR_ERR(nvmem),
-				     "Failed to register nvmem\n");
-
-	i2c_set_clientdata(client, m24lr);
-	i2c_set_clientdata(eeprom_client, m24lr);
-
-	bin_attr_sss.size = chip->sss_len;
-	bin_attr_sss.private = m24lr;
-	err = sysfs_create_bin_file(&dev->kobj, &bin_attr_sss);
-	if (err)
-		return dev_err_probe(dev, err,
-				     "Failed to create sss bin file\n");
-
-	/* test by reading the uid, if success store it */
-	err = m24lr_read_reg_le(m24lr, &m24lr->uid, 2324, sizeof(m24lr->uid));
-	if (IS_ERR_VALUE(err))
-		goto remove_bin_file;
-
-	return 0;
-
-remove_bin_file:
-	sysfs_remove_bin_file(&dev->kobj, &bin_attr_sss);
-
-	return err;
-}
-
-static void m24lr_remove(struct i2c_client *client)
-{
-	sysfs_remove_bin_file(&client->dev.kobj, &bin_attr_sss);
-}
-
-ATTRIBUTE_GROUPS(m24lr_ctl_dev);
-
-static struct i2c_driver m24lr_driver = {
-	.driver = {
-		.name = "m24lr",
-		.of_match_table = m24lr_of_match,
-		.dev_groups = m24lr_ctl_dev_groups,
-	},
-	.probe	  = m24lr_probe,
-	.remove = m24lr_remove,
-	.id_table = m24lr_ids,
-};
-module_i2c_driver(m24lr_driver);
-
-MODULE_AUTHOR("Abd-Alrhman Masalkhi");
-MODULE_DESCRIPTION("st m24lr control driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c
index f3a49384586d1..d86e79134c681 100644
--- a/drivers/misc/fastrpc.c
+++ b/drivers/misc/fastrpc.c
@@ -422,6 +422,9 @@ static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd,
 
 static void fastrpc_buf_free(struct fastrpc_buf *buf)
 {
+	if (!buf)
+		return;
+
 	dma_free_coherent(buf->dev, buf->size, buf->virt,
 			  fastrpc_ipa_to_dma_addr(buf->fl->cctx, buf->dma_addr));
 	kfree(buf);
@@ -514,8 +517,7 @@ static void fastrpc_user_free(struct kref *ref)
 	struct fastrpc_map *map, *m;
 	struct fastrpc_buf *buf, *b;
 
-	if (fl->init_mem)
-		fastrpc_buf_free(fl->init_mem);
+	fastrpc_buf_free(fl->init_mem);
 
 	list_for_each_entry_safe(ctx, n, &fl->pending, node) {
 		list_del(&ctx->node);
@@ -560,8 +562,7 @@ static void fastrpc_context_free(struct kref *ref)
 	for (i = 0; i < ctx->nbufs; i++)
 		fastrpc_map_put(ctx->maps[i]);
 
-	if (ctx->buf)
-		fastrpc_buf_free(ctx->buf);
+	fastrpc_buf_free(ctx->buf);
 
 	spin_lock_irqsave(&cctx->lock, flags);
 	idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4);
@@ -1305,10 +1306,12 @@ static int fastrpc_internal_invoke(struct fastrpc_user *fl,  u32 kernel,
 	}
 
 	if (err == -ERESTARTSYS) {
+		spin_lock(&fl->lock);
 		list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
 			list_del(&buf->node);
 			list_add_tail(&buf->node, &fl->cctx->invoke_interrupted_mmaps);
 		}
+		spin_unlock(&fl->lock);
 	}
 
 	if (err)
@@ -1370,7 +1373,9 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl,
 		err = PTR_ERR(name);
 		goto err;
 	}
-
+	inbuf.client_id = fl->client_id;
+	inbuf.namelen = init.namelen;
+	inbuf.pageslen = 0;
 	if (!fl->cctx->remote_heap) {
 		err = fastrpc_remote_heap_alloc(fl, fl->sctx->dev, init.memlen,
 						&fl->cctx->remote_heap);
@@ -1393,12 +1398,10 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl,
 				goto err_map;
 			}
 			scm_done = true;
+			inbuf.pageslen = 1;
 		}
 	}
 
-	inbuf.client_id = fl->client_id;
-	inbuf.namelen = init.namelen;
-	inbuf.pageslen = 0;
 	fl->pd = USER_PD;
 
 	args[0].ptr = (u64)(uintptr_t)&inbuf;
@@ -1673,7 +1676,7 @@ static int fastrpc_device_open(struct inode *inode, struct file *filp)
 		dev_err(&cctx->rpdev->dev, "No session available\n");
 		mutex_destroy(&fl->mutex);
 		kfree(fl);
-
+		fastrpc_channel_ctx_put(cctx);
 		return -EBUSY;
 	}
 
@@ -1889,9 +1892,6 @@ static int fastrpc_req_munmap_impl(struct fastrpc_user *fl, struct fastrpc_buf *
 				      &args[0]);
 	if (!err) {
 		dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr);
-		spin_lock(&fl->lock);
-		list_del(&buf->node);
-		spin_unlock(&fl->lock);
 		fastrpc_buf_free(buf);
 	} else {
 		dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr);
@@ -1905,6 +1905,7 @@ static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp)
 	struct fastrpc_buf *buf = NULL, *iter, *b;
 	struct fastrpc_req_munmap req;
 	struct device *dev = fl->sctx->dev;
+	int err;
 
 	if (copy_from_user(&req, argp, sizeof(req)))
 		return -EFAULT;
@@ -1912,6 +1913,7 @@ static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp)
 	spin_lock(&fl->lock);
 	list_for_each_entry_safe(iter, b, &fl->mmaps, node) {
 		if ((iter->raddr == req.vaddrout) && (iter->size == req.size)) {
+			list_del(&iter->node);
 			buf = iter;
 			break;
 		}
@@ -1924,7 +1926,14 @@ static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp)
 		return -EINVAL;
 	}
 
-	return fastrpc_req_munmap_impl(fl, buf);
+	err = fastrpc_req_munmap_impl(fl, buf);
+	if (err) {
+		spin_lock(&fl->lock);
+		list_add_tail(&buf->node, &fl->mmaps);
+		spin_unlock(&fl->lock);
+	}
+
+	return err;
 }
 
 static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp)
@@ -2223,19 +2232,22 @@ static int fastrpc_cb_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	int i, sessions = 0;
 	unsigned long flags;
-	int rc;
 	u32 dma_bits;
+	u32 sid = 0;
+	int rc;
 
 	cctx = dev_get_drvdata(dev->parent);
 	if (!cctx)
 		return -EINVAL;
 
 	of_property_read_u32(dev->of_node, "qcom,nsessions", &sessions);
+	if (of_property_read_u32(dev->of_node, "reg", &sid))
+		dev_info(dev, "FastRPC Session ID not specified in DT\n");
 
 	spin_lock_irqsave(&cctx->lock, flags);
 	if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) {
-		dev_err(&pdev->dev, "too many sessions\n");
 		spin_unlock_irqrestore(&cctx->lock, flags);
+		dev_err(&pdev->dev, "too many sessions\n");
 		return -ENOSPC;
 	}
 	dma_bits = cctx->soc_data->dma_addr_bits_default;
@@ -2244,13 +2256,11 @@ static int fastrpc_cb_probe(struct platform_device *pdev)
 	sess->valid = true;
 	sess->dev = dev;
 	dev_set_drvdata(dev, sess);
+	sess->sid = sid;
 
 	if (cctx->domain_id == CDSP_DOMAIN_ID)
 		dma_bits = cctx->soc_data->dma_addr_bits_cdsp;
 
-	if (of_property_read_u32(dev->of_node, "reg", &sess->sid))
-		dev_info(dev, "FastRPC Session ID not specified in DT\n");
-
 	if (sessions > 0) {
 		struct fastrpc_session_ctx *dup_sess;
 
diff --git a/drivers/misc/sgi-xp/xpc_uv.c b/drivers/misc/sgi-xp/xpc_uv.c
index 772c787268932..79c2f00ed4d70 100644
--- a/drivers/misc/sgi-xp/xpc_uv.c
+++ b/drivers/misc/sgi-xp/xpc_uv.c
@@ -170,9 +170,8 @@ xpc_create_gru_mq_uv(unsigned int mq_size, int cpu, char *irq_name,
 	mq->mmr_blade = uv_cpu_to_blade_id(cpu);
 
 	nid = cpu_to_node(cpu);
-	page = __alloc_pages_node(nid,
-				      GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE,
-				      pg_order);
+	page = alloc_pages_node(nid, GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE,
+				pg_order);
 	if (page == NULL) {
 		dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to alloc %d "
 			"bytes of memory on nid=%d for GRU mq\n", mq_size, nid);
diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c
index 0274e8d07660d..e760d13ee325e 100644
--- a/drivers/mmc/core/block.c
+++ b/drivers/mmc/core/block.c
@@ -2715,7 +2715,6 @@ static void mmc_blk_rpmb_device_release(struct device *dev)
 {
 	struct mmc_rpmb_data *rpmb = dev_get_drvdata(dev);
 
-	rpmb_dev_unregister(rpmb->rdev);
 	mmc_blk_put(rpmb->md);
 	ida_free(&mmc_rpmb_ida, rpmb->id);
 	kfree(rpmb);
@@ -2930,8 +2929,8 @@ static int mmc_blk_alloc_rpmb_part(struct mmc_card *card,
 }
 
 static void mmc_blk_remove_rpmb_part(struct mmc_rpmb_data *rpmb)
-
 {
+	rpmb_dev_unregister(rpmb->rdev);
 	cdev_device_del(&rpmb->chrdev, &rpmb->dev);
 	put_device(&rpmb->dev);
 }
@@ -3338,6 +3337,11 @@ static int __init mmc_blk_init(void)
 {
 	int res;
 
+	if (perdev_minors <= 0) {
+		pr_err("mmcblk: invalid minors per device: %d\n", perdev_minors);
+		return -EINVAL;
+	}
+
 	res  = bus_register(&mmc_rpmb_bus_type);
 	if (res < 0) {
 		pr_err("mmcblk: could not register RPMB bus type\n");
diff --git a/drivers/mmc/core/mmc_test.c b/drivers/mmc/core/mmc_test.c
index ab38e4c45a8da..4dc16649e61d4 100644
--- a/drivers/mmc/core/mmc_test.c
+++ b/drivers/mmc/core/mmc_test.c
@@ -318,9 +318,9 @@ static void mmc_test_free_mem(struct mmc_test_mem *mem)
 {
 	if (!mem)
 		return;
-	while (mem->cnt--)
-		__free_pages(mem->arr[mem->cnt].page,
-			     mem->arr[mem->cnt].order);
+	for (unsigned int i = 0; i < mem->cnt; i++)
+		__free_pages(mem->arr[i].page,
+			     mem->arr[i].order);
 	kfree(mem);
 }
 
@@ -341,6 +341,7 @@ static struct mmc_test_mem *mmc_test_alloc_mem(unsigned long min_sz,
 	unsigned long page_cnt = 0;
 	unsigned long limit = nr_free_buffer_pages() >> 4;
 	struct mmc_test_mem *mem;
+	unsigned int idx = 0;
 
 	if (max_page_cnt > limit)
 		max_page_cnt = limit;
@@ -375,23 +376,26 @@ static struct mmc_test_mem *mmc_test_alloc_mem(unsigned long min_sz,
 				goto out_free;
 			break;
 		}
-		mem->arr[mem->cnt].page = page;
-		mem->arr[mem->cnt].order = order;
-		mem->cnt += 1;
+		mem->arr[idx].page = page;
+		mem->arr[idx].order = order;
+		idx += 1;
 		if (max_page_cnt <= (1UL << order))
 			break;
 		max_page_cnt -= 1UL << order;
 		page_cnt += 1UL << order;
-		if (mem->cnt >= max_segs) {
+		if (idx >= mem->cnt) {
 			if (page_cnt < min_page_cnt)
 				goto out_free;
 			break;
 		}
 	}
 
+	mem->cnt = idx;
+
 	return mem;
 
 out_free:
+	mem->cnt = idx;
 	mmc_test_free_mem(mem);
 	return NULL;
 }
diff --git a/drivers/mmc/core/sd.c b/drivers/mmc/core/sd.c
index 948948ca9b4ae..c763efb10f649 100644
--- a/drivers/mmc/core/sd.c
+++ b/drivers/mmc/core/sd.c
@@ -1379,15 +1379,10 @@ static int sd_flush_cache(struct mmc_host *host)
 
 static int sd_enable_cache(struct mmc_card *card)
 {
-	u8 *reg_buf;
 	int err;
 
 	card->ext_perf.feature_enabled &= ~SD_EXT_PERF_CACHE;
 
-	reg_buf = kzalloc(512, GFP_KERNEL);
-	if (!reg_buf)
-		return -ENOMEM;
-
 	/*
 	 * Set Cache Enable at bit 0 in the performance enhancement register at
 	 * 260 bytes offset.
@@ -1397,7 +1392,7 @@ static int sd_enable_cache(struct mmc_card *card)
 	if (err) {
 		pr_warn("%s: error %d writing Cache Enable bit\n",
 			mmc_hostname(card->host), err);
-		goto out;
+		return err;
 	}
 
 	err = mmc_poll_for_busy(card, SD_WRITE_EXTR_SINGLE_TIMEOUT_MS, false,
@@ -1405,8 +1400,6 @@ static int sd_enable_cache(struct mmc_card *card)
 	if (!err)
 		card->ext_perf.feature_enabled |= SD_EXT_PERF_CACHE;
 
-out:
-	kfree(reg_buf);
 	return err;
 }
 
diff --git a/drivers/mmc/host/bcm2835.c b/drivers/mmc/host/bcm2835.c
index ee63835b3ca0a..c949a3eaae794 100644
--- a/drivers/mmc/host/bcm2835.c
+++ b/drivers/mmc/host/bcm2835.c
@@ -188,7 +188,7 @@ struct bcm2835_host {
 	u32			drain_words;
 	struct page		*drain_page;
 	u32			drain_offset;
-	bool			use_dma;
+	struct device		*dma_dev;
 };
 
 static void bcm2835_dumpcmd(struct bcm2835_host *host, struct mmc_command *cmd,
@@ -494,8 +494,7 @@ void bcm2835_prepare_dma(struct bcm2835_host *host, struct mmc_data *data)
 				     &host->dma_cfg_rx :
 				     &host->dma_cfg_tx);
 
-	sg_len = dma_map_sg(dma_chan->device->dev, data->sg, data->sg_len,
-			    dir_data);
+	sg_len = dma_map_sg(host->dma_dev, data->sg, data->sg_len, dir_data);
 	if (!sg_len)
 		return;
 
@@ -503,8 +502,7 @@ void bcm2835_prepare_dma(struct bcm2835_host *host, struct mmc_data *data)
 				       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 
 	if (!desc) {
-		dma_unmap_sg(dma_chan->device->dev, data->sg, data->sg_len,
-			     dir_data);
+		dma_unmap_sg(host->dma_dev, data->sg, data->sg_len, dir_data);
 		return;
 	}
 
@@ -1057,7 +1055,7 @@ static void bcm2835_dma_complete_work(struct work_struct *work)
 	data = host->data;
 
 	if (host->dma_chan) {
-		dma_unmap_sg(host->dma_chan->device->dev,
+		dma_unmap_sg(host->dma_dev,
 			     data->sg, data->sg_len,
 			     host->dma_dir);
 
@@ -1201,7 +1199,7 @@ static void bcm2835_request(struct mmc_host *mmc, struct mmc_request *mrq)
 		return;
 	}
 
-	if (host->use_dma && mrq->data && (mrq->data->blocks > PIO_THRESHOLD))
+	if (host->dma_dev && mrq->data && (mrq->data->blocks > PIO_THRESHOLD))
 		bcm2835_prepare_dma(host, mrq->data);
 
 	host->use_sbc = !!mrq->sbc && host->mrq->data &&
@@ -1281,10 +1279,7 @@ static int bcm2835_add_host(struct bcm2835_host *host)
 
 	if (!host->dma_chan_rxtx) {
 		dev_warn(dev, "unable to initialise DMA channel. Falling back to PIO\n");
-		host->use_dma = false;
 	} else {
-		host->use_dma = true;
-
 		host->dma_cfg_tx.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 		host->dma_cfg_tx.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 		host->dma_cfg_tx.direction = DMA_MEM_TO_DEV;
@@ -1297,15 +1292,21 @@ static int bcm2835_add_host(struct bcm2835_host *host)
 		host->dma_cfg_rx.src_addr = host->phys_addr + SDDATA;
 		host->dma_cfg_rx.dst_addr = 0;
 
-		if (dmaengine_slave_config(host->dma_chan_rxtx,
-					   &host->dma_cfg_tx) != 0 ||
-		    dmaengine_slave_config(host->dma_chan_rxtx,
-					   &host->dma_cfg_rx) != 0)
-			host->use_dma = false;
+		if (!dmaengine_slave_config(host->dma_chan_rxtx,
+					   &host->dma_cfg_tx) &&
+		    !dmaengine_slave_config(host->dma_chan_rxtx,
+					   &host->dma_cfg_rx)) {
+			host->dma_dev =
+				dmaengine_get_dma_device(host->dma_chan_rxtx);
+		}
 	}
 
 	mmc->max_segs = 128;
-	mmc->max_req_size = min_t(size_t, 524288, dma_max_mapping_size(dev));
+	mmc->max_req_size = 524288;
+	if (host->dma_dev) {
+		mmc->max_req_size = min_t(size_t, mmc->max_req_size,
+				dma_max_mapping_size(host->dma_dev));
+	}
 	mmc->max_seg_size = mmc->max_req_size;
 	mmc->max_blk_size = 1024;
 	mmc->max_blk_count =  65535;
@@ -1336,10 +1337,10 @@ static int bcm2835_add_host(struct bcm2835_host *host)
 	}
 
 	pio_limit_string[0] = '\0';
-	if (host->use_dma && (PIO_THRESHOLD > 0))
+	if (host->dma_dev && (PIO_THRESHOLD > 0))
 		sprintf(pio_limit_string, " (>%d)", PIO_THRESHOLD);
 	dev_info(dev, "loaded - DMA %s%s\n",
-		 host->use_dma ? "enabled" : "disabled", pio_limit_string);
+		 host->dma_dev ? "enabled" : "disabled", pio_limit_string);
 
 	return 0;
 }
diff --git a/drivers/mmc/host/cqhci-core.c b/drivers/mmc/host/cqhci-core.c
index 178277d90c31c..a466cd88b043e 100644
--- a/drivers/mmc/host/cqhci-core.c
+++ b/drivers/mmc/host/cqhci-core.c
@@ -484,11 +484,11 @@ void cqhci_set_tran_desc(u8 *desc, dma_addr_t addr, int len, bool end,
 {
 	__le32 *attr = (__le32 __force *)desc;
 
-	*attr = (CQHCI_VALID(1) |
-		 CQHCI_END(end ? 1 : 0) |
-		 CQHCI_INT(0) |
-		 CQHCI_ACT(0x4) |
-		 CQHCI_DAT_LENGTH(len));
+	*attr = cpu_to_le32((CQHCI_VALID(1) |
+			     CQHCI_END(end ? 1 : 0) |
+			     CQHCI_INT(0) |
+			     CQHCI_ACT(0x4) |
+			     CQHCI_DAT_LENGTH(len)));
 
 	if (dma64) {
 		__le64 *dataddr = (__le64 __force *)(desc + 4);
@@ -542,7 +542,7 @@ static int cqhci_prep_tran_desc(struct mmc_request *mrq,
 static void cqhci_prep_dcmd_desc(struct mmc_host *mmc,
 				   struct mmc_request *mrq)
 {
-	u64 *task_desc = NULL;
+	__le64 *task_desc = NULL;
 	u64 data = 0;
 	u8 resp_type;
 	u8 *desc;
@@ -574,7 +574,7 @@ static void cqhci_prep_dcmd_desc(struct mmc_host *mmc,
 		 CQHCI_CMD_TIMING(timing) | CQHCI_RESP_TYPE(resp_type));
 	if (cq_host->ops->update_dcmd_desc)
 		cq_host->ops->update_dcmd_desc(mmc, mrq, &data);
-	*task_desc |= data;
+	*task_desc |= cpu_to_le64(data);
 	desc = (u8 *)task_desc;
 	pr_debug("%s: cqhci: dcmd: cmd: %d timing: %d resp: %d\n",
 		 mmc_hostname(mmc), mrq->cmd->opcode, timing, resp_type);
@@ -819,8 +819,7 @@ static void cqhci_finish_mrq(struct mmc_host *mmc, unsigned int tag)
 	mmc_cqe_request_done(mmc, mrq);
 }
 
-irqreturn_t cqhci_irq(struct mmc_host *mmc, u32 intmask, int cmd_error,
-		      int data_error)
+irqreturn_t cqhci_irq(struct mmc_host *mmc, int cmd_error, int data_error)
 {
 	u32 status;
 	unsigned long tag = 0, comp_status;
diff --git a/drivers/mmc/host/cqhci.h b/drivers/mmc/host/cqhci.h
index 3668856531c14..8fbbc48c3f856 100644
--- a/drivers/mmc/host/cqhci.h
+++ b/drivers/mmc/host/cqhci.h
@@ -315,8 +315,7 @@ static inline u32 cqhci_readl(struct cqhci_host *host, int reg)
 
 struct platform_device;
 
-irqreturn_t cqhci_irq(struct mmc_host *mmc, u32 intmask, int cmd_error,
-		      int data_error);
+irqreturn_t cqhci_irq(struct mmc_host *mmc, int cmd_error, int data_error);
 int cqhci_init(struct cqhci_host *cq_host, struct mmc_host *mmc, bool dma64);
 struct cqhci_host *cqhci_pltfm_init(struct platform_device *pdev);
 int cqhci_deactivate(struct mmc_host *mmc);
diff --git a/drivers/mmc/host/dw_mmc-pltfm.h b/drivers/mmc/host/dw_mmc-pltfm.h
index ef1b05d484c39..f3193662abf1d 100644
--- a/drivers/mmc/host/dw_mmc-pltfm.h
+++ b/drivers/mmc/host/dw_mmc-pltfm.h
@@ -11,6 +11,5 @@
 extern int dw_mci_pltfm_register(struct platform_device *pdev,
 				const struct dw_mci_drv_data *drv_data);
 extern void dw_mci_pltfm_remove(struct platform_device *pdev);
-extern const struct dev_pm_ops dw_mci_pmops;
 
 #endif /* _DW_MMC_PLTFM_H_ */
diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c
index d734d010444dd..cf5cecc5491d5 100644
--- a/drivers/mmc/host/dw_mmc.c
+++ b/drivers/mmc/host/dw_mmc.c
@@ -2057,6 +2057,7 @@ static void dw_mci_work_func(struct work_struct *t)
 			}
 
 			dw_mci_stop_fault_timer(host);
+			dw_mci_stop_dma(host);
 			host->data = NULL;
 			set_bit(EVENT_DATA_COMPLETE, &host->completed_events);
 			err = dw_mci_data_complete(host, data);
diff --git a/drivers/mmc/host/dw_mmc.h b/drivers/mmc/host/dw_mmc.h
index 9ffcd3946cffb..38610c89d54a2 100644
--- a/drivers/mmc/host/dw_mmc.h
+++ b/drivers/mmc/host/dw_mmc.h
@@ -19,6 +19,8 @@
 #include <linux/interrupt.h>
 #include <linux/workqueue.h>
 
+extern const struct dev_pm_ops dw_mci_pmops;
+
 enum dw_mci_state {
 	STATE_IDLE = 0,
 	STATE_SENDING_CMD,
diff --git a/drivers/mmc/host/litex_mmc.c b/drivers/mmc/host/litex_mmc.c
index 06a6f24702e0b..332fef8b66465 100644
--- a/drivers/mmc/host/litex_mmc.c
+++ b/drivers/mmc/host/litex_mmc.c
@@ -28,6 +28,7 @@
 #define LITEX_PHY_CLOCKERDIV  0x04
 #define LITEX_PHY_INITIALIZE  0x08
 #define LITEX_PHY_WRITESTATUS 0x0C
+#define LITEX_PHY_SETTINGS    0x18
 #define LITEX_CORE_CMDARG     0x00
 #define LITEX_CORE_CMDCMD     0x04
 #define LITEX_CORE_CMDSND     0x08
@@ -71,6 +72,10 @@
 #define SD_INIT_DELAY_US  1000
 #define SD_INIT_CLK_HZ    400000
 
+#define SD_PHY_SPEED_1X 0
+#define SD_PHY_SPEED_4X 1
+#define SD_PHY_SPEED_8X 2
+
 #define SDIRQ_CARD_DETECT    1
 #define SDIRQ_SD_TO_MEM_DONE 2
 #define SDIRQ_MEM_TO_SD_DONE 4
@@ -95,11 +100,9 @@ struct litex_mmc_host {
 	unsigned int ref_clk;
 	unsigned int sd_clk;
 
-	u32 resp[4];
-	u16 rca;
+	u8 width;
 
-	bool is_bus_width_set;
-	bool app_cmd;
+	u32 resp[4];
 };
 
 static int litex_mmc_sdcard_wait_done(void __iomem *reg, struct device *dev)
@@ -164,11 +167,6 @@ static int litex_mmc_send_cmd(struct litex_mmc_host *host,
 			      host->sdcore + LITEX_CORE_CMDRSP, 0x10);
 	}
 
-	if (!host->app_cmd && cmd == SD_SEND_RELATIVE_ADDR)
-		host->rca = (host->resp[3] >> 16);
-
-	host->app_cmd = (cmd == MMC_APP_CMD);
-
 	if (transfer == SD_CTL_DATA_XFER_NONE)
 		return ret; /* OK from prior litex_mmc_sdcard_wait_done() */
 
@@ -190,51 +188,6 @@ static int litex_mmc_send_cmd(struct litex_mmc_host *host,
 	return ret;
 }
 
-static int litex_mmc_send_app_cmd(struct litex_mmc_host *host)
-{
-	return litex_mmc_send_cmd(host, MMC_APP_CMD, host->rca << 16,
-				  SD_CTL_RESP_SHORT, SD_CTL_DATA_XFER_NONE);
-}
-
-static int litex_mmc_send_set_bus_w_cmd(struct litex_mmc_host *host, u32 width)
-{
-	return litex_mmc_send_cmd(host, SD_APP_SET_BUS_WIDTH, width,
-				  SD_CTL_RESP_SHORT, SD_CTL_DATA_XFER_NONE);
-}
-
-static int litex_mmc_set_bus_width(struct litex_mmc_host *host)
-{
-	bool app_cmd_sent;
-	int ret;
-
-	if (host->is_bus_width_set)
-		return 0;
-
-	/* Ensure 'app_cmd' precedes 'app_set_bus_width_cmd' */
-	app_cmd_sent = host->app_cmd; /* was preceding command app_cmd? */
-	if (!app_cmd_sent) {
-		ret = litex_mmc_send_app_cmd(host);
-		if (ret)
-			return ret;
-	}
-
-	/* LiteSDCard only supports 4-bit bus width */
-	ret = litex_mmc_send_set_bus_w_cmd(host, MMC_BUS_WIDTH_4);
-	if (ret)
-		return ret;
-
-	/* Re-send 'app_cmd' if necessary */
-	if (app_cmd_sent) {
-		ret = litex_mmc_send_app_cmd(host);
-		if (ret)
-			return ret;
-	}
-
-	host->is_bus_width_set = true;
-
-	return 0;
-}
-
 static int litex_mmc_get_cd(struct mmc_host *mmc)
 {
 	struct litex_mmc_host *host = mmc_priv(mmc);
@@ -247,9 +200,6 @@ static int litex_mmc_get_cd(struct mmc_host *mmc)
 	if (ret)
 		return ret;
 
-	/* Ensure bus width will be set (again) upon card (re)insertion */
-	host->is_bus_width_set = false;
-
 	return 0;
 }
 
@@ -365,39 +315,19 @@ static void litex_mmc_request(struct mmc_host *mmc, struct mmc_request *mrq)
 						litex_mmc_response_len(sbc),
 						SD_CTL_DATA_XFER_NONE);
 		if (sbc->error) {
-			host->is_bus_width_set = false;
 			mmc_request_done(mmc, mrq);
 			return;
 		}
 	}
 
-	if (data) {
-		/*
-		 * LiteSDCard only supports 4-bit bus width; therefore, we MUST
-		 * inject a SET_BUS_WIDTH (acmd6) before the very first data
-		 * transfer, earlier than when the mmc subsystem would normally
-		 * get around to it!
-		 */
-		cmd->error = litex_mmc_set_bus_width(host);
-		if (cmd->error) {
-			dev_err(dev, "Can't set bus width!\n");
-			mmc_request_done(mmc, mrq);
-			return;
-		}
-
+	if (data)
 		litex_mmc_do_dma(host, data, &len, &direct, &transfer);
-	}
 
 	do {
 		cmd->error = litex_mmc_send_cmd(host, cmd->opcode, cmd->arg,
 						response_len, transfer);
 	} while (cmd->error && retries-- > 0);
 
-	if (cmd->error) {
-		/* Card may be gone; don't assume bus width is still set */
-		host->is_bus_width_set = false;
-	}
-
 	if (response_len == SD_CTL_RESP_SHORT) {
 		/* Pull short response fields from appropriate host registers */
 		cmd->resp[0] = host->resp[3];
@@ -410,13 +340,10 @@ static void litex_mmc_request(struct mmc_host *mmc, struct mmc_request *mrq)
 	}
 
 	/* Send stop-transmission command if required */
-	if (stop && (cmd->error || !sbc)) {
+	if (stop && (cmd->error || !sbc))
 		stop->error = litex_mmc_send_cmd(host, stop->opcode, stop->arg,
 						 litex_mmc_response_len(stop),
 						 SD_CTL_DATA_XFER_NONE);
-		if (stop->error)
-			host->is_bus_width_set = false;
-	}
 
 	if (data) {
 		dma_unmap_sg(dev, data->sg, data->sg_len,
@@ -450,6 +377,24 @@ static void litex_mmc_setclk(struct litex_mmc_host *host, unsigned int freq)
 static void litex_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 {
 	struct litex_mmc_host *host = mmc_priv(mmc);
+	unsigned int bus_width = SD_PHY_SPEED_1X;
+
+	switch (ios->bus_width) {
+	case MMC_BUS_WIDTH_1:
+		bus_width = SD_PHY_SPEED_1X;
+		break;
+	case MMC_BUS_WIDTH_4:
+		bus_width = SD_PHY_SPEED_4X;
+		break;
+	case MMC_BUS_WIDTH_8:
+		bus_width = SD_PHY_SPEED_8X;
+		break;
+	}
+
+	if (host->width != ios->bus_width) {
+		litex_write8(host->sdphy + LITEX_PHY_SETTINGS, bus_width);
+		host->width = ios->bus_width;
+	}
 
 	/*
 	 * The SD specification requires at least 74 idle clocks before CMD0.
@@ -462,13 +407,6 @@ static void litex_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 		return;
 	}
 
-	/*
-	 * NOTE: Ignore any ios->bus_width updates; they occur right after
-	 * the mmc core sends its own acmd6 bus-width change notification,
-	 * which is redundant since we snoop on the command flow and inject
-	 * an early acmd6 before the first data transfer command is sent!
-	 */
-
 	/* Update sd_clk */
 	if (ios->clock != host->sd_clk)
 		litex_mmc_setclk(host, ios->clock);
@@ -546,14 +484,7 @@ static int litex_mmc_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, PTR_ERR(clk), "can't get clock\n");
 	host->ref_clk = clk_get_rate(clk);
 	host->sd_clk = 0;
-
-	/*
-	 * LiteSDCard only supports 4-bit bus width; therefore, we MUST inject
-	 * a SET_BUS_WIDTH (acmd6) before the very first data transfer, earlier
-	 * than when the mmc subsystem would normally get around to it!
-	 */
-	host->is_bus_width_set = false;
-	host->app_cmd = false;
+	host->width = MMC_BUS_WIDTH_1;
 
 	/* LiteSDCard can support 64-bit DMA addressing */
 	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
@@ -586,6 +517,9 @@ static int litex_mmc_probe(struct platform_device *pdev)
 	litex_write8(host->sdreader + LITEX_BLK2MEM_ENA, 0);
 	litex_write8(host->sdwriter + LITEX_MEM2BLK_ENA, 0);
 
+	/* Ensure the litex is at bus width x1 */
+	litex_write8(host->sdphy + LITEX_PHY_SETTINGS, SD_PHY_SPEED_1X);
+
 	init_completion(&host->cmd_done);
 	ret = litex_mmc_irq_init(pdev, host);
 	if (ret)
@@ -610,9 +544,8 @@ static int litex_mmc_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	/* Force 4-bit bus_width (only width supported by hardware) */
+	/* Only drop 8-bit bus_width support */
 	mmc->caps &= ~MMC_CAP_8_BIT_DATA;
-	mmc->caps |= MMC_CAP_4_BIT_DATA;
 
 	/* Set default capabilities */
 	mmc->caps |= MMC_CAP_WAIT_WHILE_BUSY |
diff --git a/drivers/mmc/host/moxart-mmc.c b/drivers/mmc/host/moxart-mmc.c
index 3dd8f232052f6..4a4d8b19b18cc 100644
--- a/drivers/mmc/host/moxart-mmc.c
+++ b/drivers/mmc/host/moxart-mmc.c
@@ -264,6 +264,7 @@ static void moxart_transfer_dma(struct mmc_data *data, struct moxart_host *host)
 	u32 len, dir_slave;
 	struct dma_async_tx_descriptor *desc = NULL;
 	struct dma_chan *dma_chan;
+	long timeout;
 
 	if (host->data_len == data->bytes_xfered)
 		return;
@@ -296,11 +297,17 @@ static void moxart_transfer_dma(struct mmc_data *data, struct moxart_host *host)
 		dma_async_issue_pending(dma_chan);
 	}
 
-	wait_for_completion_interruptible_timeout(&host->dma_complete,
-						  host->timeout);
+	timeout = wait_for_completion_interruptible_timeout(&host->dma_complete,
+							    host->timeout);
+	if (timeout <= 0) {
+		dmaengine_terminate_sync(dma_chan);
+		data->error = timeout ?: -ETIMEDOUT;
+		goto unmap;
+	}
 
 	data->bytes_xfered = host->data_len;
 
+unmap:
 	dma_unmap_sg(dma_chan->device->dev,
 		     data->sg, data->sg_len,
 		     mmc_get_dma_dir(data));
diff --git a/drivers/mmc/host/mtk-sd.c b/drivers/mmc/host/mtk-sd.c
index b2680cc054bda..01ea3adbdf3b6 100644
--- a/drivers/mmc/host/mtk-sd.c
+++ b/drivers/mmc/host/mtk-sd.c
@@ -1805,7 +1805,7 @@ static irqreturn_t msdc_cmdq_irq(struct msdc_host *host, u32 intsts)
 			cmd_err, dat_err, intsts);
 	}
 
-	return cqhci_irq(mmc, 0, cmd_err, dat_err);
+	return cqhci_irq(mmc, cmd_err, dat_err);
 }
 
 static irqreturn_t msdc_irq(int irq, void *dev_id)
diff --git a/drivers/mmc/host/pxamci.c b/drivers/mmc/host/pxamci.c
index b5ea058ed4678..f8427f071c009 100644
--- a/drivers/mmc/host/pxamci.c
+++ b/drivers/mmc/host/pxamci.c
@@ -43,8 +43,7 @@
 #define NR_SG	1
 #define CLKRT_OFF	(~0)
 
-#define mmc_has_26MHz()		(cpu_is_pxa300() || cpu_is_pxa310() \
-				|| cpu_is_pxa935())
+#define mmc_has_26MHz()		(cpu_is_pxa300() || cpu_is_pxa310())
 
 struct pxamci_host {
 	struct mmc_host		*mmc;
diff --git a/drivers/mmc/host/rtsx_usb_sdmmc.c b/drivers/mmc/host/rtsx_usb_sdmmc.c
index 84674659a84d4..bf37fab233cf0 100644
--- a/drivers/mmc/host/rtsx_usb_sdmmc.c
+++ b/drivers/mmc/host/rtsx_usb_sdmmc.c
@@ -44,6 +44,7 @@ struct rtsx_usb_sdmmc {
 	bool			double_clk;
 	bool			host_removal;
 	bool			card_exist;
+	bool			suppress_cd;
 	bool			initial_mode;
 	bool			ddr_mode;
 
@@ -774,6 +775,7 @@ static int sdmmc_get_cd(struct mmc_host *mmc)
 	struct rtsx_ucr *ucr = host->ucr;
 	int err;
 	u16 val;
+	bool cd;
 
 	if (host->host_removal)
 		return -ENOMEDIUM;
@@ -791,8 +793,14 @@ static int sdmmc_get_cd(struct mmc_host *mmc)
 
 	/* get OCP status */
 	host->ocp_stat = (val >> 4) & 0x03;
+	cd = val & SD_CD;
 
-	if (val & SD_CD) {
+	if (!cd) {
+		WRITE_ONCE(host->suppress_cd, false);
+		goto no_card;
+	}
+
+	if (!READ_ONCE(host->suppress_cd)) {
 		host->card_exist = true;
 		return 1;
 	}
@@ -874,6 +882,8 @@ static void sdmmc_request(struct mmc_host *mmc, struct mmc_request *mrq)
 		 * detect card when fail to update card existence state and
 		 * speed up card removal when retry
 		 */
+		if (!mmc->card && cmd->error == -ETIMEDOUT)
+			WRITE_ONCE(host->suppress_cd, true);
 		sdmmc_get_cd(mmc);
 		dev_dbg(sdmmc_dev(host), "cmd->error = %d\n", cmd->error);
 	}
@@ -1359,6 +1369,7 @@ static void rtsx_usb_init_host(struct rtsx_usb_sdmmc *host)
 
 	host->power_mode = MMC_POWER_OFF;
 	host->ocp_stat = 0;
+	host->suppress_cd = false;
 }
 
 static int rtsx_usb_sdmmc_drv_probe(struct platform_device *pdev)
diff --git a/drivers/mmc/host/sdhci-brcmstb.c b/drivers/mmc/host/sdhci-brcmstb.c
index 57e45951644e3..1de2f05fd9587 100644
--- a/drivers/mmc/host/sdhci-brcmstb.c
+++ b/drivers/mmc/host/sdhci-brcmstb.c
@@ -430,7 +430,7 @@ static u32 sdhci_brcmstb_cqhci_irq(struct sdhci_host *host, u32 intmask)
 	if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
 		return intmask;
 
-	cqhci_irq(host->mmc, intmask, cmd_error, data_error);
+	cqhci_irq(host->mmc, cmd_error, data_error);
 
 	return 0;
 }
diff --git a/drivers/mmc/host/sdhci-esdhc-imx.c b/drivers/mmc/host/sdhci-esdhc-imx.c
index 18ecddd6df6f2..6ab59d3e346f9 100644
--- a/drivers/mmc/host/sdhci-esdhc-imx.c
+++ b/drivers/mmc/host/sdhci-esdhc-imx.c
@@ -1326,19 +1326,21 @@ static int esdhc_change_pinstate(struct sdhci_host *host,
 
 	dev_dbg(mmc_dev(host->mmc), "change pinctrl state for uhs %d\n", uhs);
 
-	if (IS_ERR(imx_data->pinctrl) ||
-		IS_ERR(imx_data->pins_100mhz) ||
-		IS_ERR(imx_data->pins_200mhz))
+	if (IS_ERR(imx_data->pinctrl))
 		return -EINVAL;
 
 	switch (uhs) {
 	case MMC_TIMING_UHS_SDR50:
 	case MMC_TIMING_UHS_DDR50:
+		if (IS_ERR(imx_data->pins_100mhz))
+			return -EINVAL;
 		pinctrl = imx_data->pins_100mhz;
 		break;
 	case MMC_TIMING_UHS_SDR104:
 	case MMC_TIMING_MMC_HS200:
 	case MMC_TIMING_MMC_HS400:
+		if (IS_ERR(imx_data->pins_200mhz))
+			return -EINVAL;
 		pinctrl = imx_data->pins_200mhz;
 		break;
 	default:
@@ -1349,6 +1351,23 @@ static int esdhc_change_pinstate(struct sdhci_host *host,
 	return pinctrl_select_state(imx_data->pinctrl, pinctrl);
 }
 
+static void esdhc_set_dll_override(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct pltfm_imx_data *imx_data = sdhci_pltfm_priv(pltfm_host);
+	struct esdhc_platform_data *boarddata = &imx_data->boarddata;
+	u32 v;
+
+	if (!boarddata->delay_line)
+		return;
+
+	v = boarddata->delay_line << ESDHC_DLL_OVERRIDE_VAL_SHIFT |
+	    (1 << ESDHC_DLL_OVERRIDE_EN_SHIFT);
+	if (is_imx53_esdhc(imx_data))
+		v <<= 1;
+	writel(v, host->ioaddr + ESDHC_DLL_CTRL);
+}
+
 /*
  * For HS400 eMMC, there is a data_strobe line. This signal is generated
  * by the device and used for data output and CRC status response output
@@ -1404,7 +1423,6 @@ static void esdhc_set_uhs_signaling(struct sdhci_host *host, unsigned timing)
 	u32 m;
 	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 	struct pltfm_imx_data *imx_data = sdhci_pltfm_priv(pltfm_host);
-	struct esdhc_platform_data *boarddata = &imx_data->boarddata;
 
 	/* disable ddr mode and disable HS400 mode */
 	m = readl(host->ioaddr + ESDHC_MIX_CTRL);
@@ -1425,15 +1443,7 @@ static void esdhc_set_uhs_signaling(struct sdhci_host *host, unsigned timing)
 		m |= ESDHC_MIX_CTRL_DDREN;
 		writel(m, host->ioaddr + ESDHC_MIX_CTRL);
 		imx_data->is_ddr = 1;
-		if (boarddata->delay_line) {
-			u32 v;
-			v = boarddata->delay_line <<
-				ESDHC_DLL_OVERRIDE_VAL_SHIFT |
-				(1 << ESDHC_DLL_OVERRIDE_EN_SHIFT);
-			if (is_imx53_esdhc(imx_data))
-				v <<= 1;
-			writel(v, host->ioaddr + ESDHC_DLL_CTRL);
-		}
+		esdhc_set_dll_override(host);
 		break;
 	case MMC_TIMING_MMC_HS400:
 		m |= ESDHC_MIX_CTRL_DDREN | ESDHC_MIX_CTRL_HS400_EN;
@@ -1503,7 +1513,7 @@ static u32 esdhc_cqhci_irq(struct sdhci_host *host, u32 intmask)
 	if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
 		return intmask;
 
-	cqhci_irq(host->mmc, intmask, cmd_error, data_error);
+	cqhci_irq(host->mmc, cmd_error, data_error);
 
 	return 0;
 }
@@ -2051,7 +2061,9 @@ static int sdhci_esdhc_suspend(struct device *dev)
 	 * 2, make sure the pm_runtime_force_resume() in sdhci_esdhc_resume() really
 	 *    invoke its ->runtime_resume callback (needs_force_resume = 1).
 	 */
-	pm_runtime_get_sync(dev);
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret)
+		return ret;
 
 	if ((imx_data->socdata->flags & ESDHC_FLAG_STATE_LOST_IN_LPMODE) &&
 		(host->tuning_mode != SDHCI_TUNING_MODE_1)) {
@@ -2064,15 +2076,14 @@ static int sdhci_esdhc_suspend(struct device *dev)
 	 * to save the tuning delay value just in case the usdhc
 	 * lost power during system PM.
 	 */
-	if (mmc_card_keep_power(host->mmc) && mmc_card_wake_sdio_irq(host->mmc) &&
-	    esdhc_is_usdhc(imx_data))
+	if (mmc_card_keep_power(host->mmc) && esdhc_is_usdhc(imx_data))
 		sdhc_esdhc_tuning_save(host);
 
+	/* The irqs of imx are not shared. It is safe to disable */
+	disable_irq(host->irq);
+
 	if (device_may_wakeup(dev)) {
-		/* The irqs of imx are not shared. It is safe to disable */
-		disable_irq(host->irq);
-		ret = sdhci_enable_irq_wakeups(host);
-		if (!ret)
+		if (!sdhci_enable_irq_wakeups(host))
 			dev_warn(dev, "Failed to enable irq wakeup\n");
 	} else {
 		/*
@@ -2083,12 +2094,12 @@ static int sdhci_esdhc_suspend(struct device *dev)
 		 * other function like GPIO function to save power in PM,
 		 * which finally block the SDIO wakeup function.
 		 */
-		ret = pinctrl_pm_select_sleep_state(dev);
-		if (ret)
-			return ret;
+		if (pinctrl_pm_select_sleep_state(dev))
+			dev_warn(dev, "Failed to select sleep pinctrl state\n");
 	}
 
-	ret = mmc_gpio_set_cd_wake(host->mmc, true);
+	if (mmc_gpio_set_cd_wake(host->mmc, true))
+		dev_warn(dev, "Failed to enable cd wake\n");
 
 	/*
 	 * Make sure invoke runtime_suspend to gate off clock.
@@ -2096,7 +2107,7 @@ static int sdhci_esdhc_suspend(struct device *dev)
 	 */
 	pm_runtime_force_suspend(dev);
 
-	return ret;
+	return 0;
 }
 
 static int sdhci_esdhc_resume(struct device *dev)
@@ -2106,31 +2117,45 @@ static int sdhci_esdhc_resume(struct device *dev)
 	struct pltfm_imx_data *imx_data = sdhci_pltfm_priv(pltfm_host);
 	int ret;
 
-	pm_runtime_force_resume(dev);
+	if (!device_may_wakeup(dev)) {
+		ret = esdhc_change_pinstate(host, host->timing);
+		if (ret)
+			dev_warn(dev, "Failed to restore pinctrl state\n");
+	}
 
-	ret = mmc_gpio_set_cd_wake(host->mmc, false);
+	ret = pm_runtime_force_resume(dev);
 	if (ret)
 		return ret;
 
+	mmc_gpio_set_cd_wake(host->mmc, false);
+
 	/* re-initialize hw state in case it's lost in low power mode */
 	sdhci_esdhc_imx_hwinit(host);
 
-	if (host->irq_wake_enabled) {
+	if (host->irq_wake_enabled)
 		sdhci_disable_irq_wakeups(host);
-		enable_irq(host->irq);
-	}
+
+	enable_irq(host->irq);
 
 	/*
 	 * restore the saved tuning delay value for the device which keep
 	 * power during system PM.
 	 */
-	if (mmc_card_keep_power(host->mmc) && mmc_card_wake_sdio_irq(host->mmc) &&
-	    esdhc_is_usdhc(imx_data))
+	if (mmc_card_keep_power(host->mmc) && esdhc_is_usdhc(imx_data)) {
 		sdhc_esdhc_tuning_restore(host);
 
+		/*
+		 * Restore DLL override for DDR modes. hwinit unconditionally
+		 * clears ESDHC_DLL_CTRL, but the card is still in DDR mode.
+		 */
+		if (host->timing == MMC_TIMING_UHS_DDR50 ||
+		    host->timing == MMC_TIMING_MMC_DDR52)
+			esdhc_set_dll_override(host);
+	}
+
 	pm_runtime_put_autosuspend(dev);
 
-	return ret;
+	return 0;
 }
 
 static int sdhci_esdhc_runtime_suspend(struct device *dev)
diff --git a/drivers/mmc/host/sdhci-esdhc-mcf.c b/drivers/mmc/host/sdhci-esdhc-mcf.c
index 375fce5639d71..6853521e8b2c9 100644
--- a/drivers/mmc/host/sdhci-esdhc-mcf.c
+++ b/drivers/mmc/host/sdhci-esdhc-mcf.c
@@ -55,7 +55,7 @@ static inline void esdhc_clrset_be(struct sdhci_host *host,
 	if (reg == SDHCI_HOST_CONTROL)
 		val |= ESDHC_PROCTL_D3CD;
 
-	writel((readl(base) & ~mask) | val, base);
+	mcf_write32((mcf_read32(base) & ~mask) | val, base);
 }
 
 /*
@@ -71,7 +71,7 @@ static void esdhc_mcf_writeb_be(struct sdhci_host *host, u8 val, int reg)
 	if (reg == SDHCI_HOST_CONTROL) {
 		u32 host_ctrl = ESDHC_DEFAULT_HOST_CONTROL;
 		u8 dma_bits = (val & SDHCI_CTRL_DMA_MASK) >> 3;
-		u8 tmp = readb(host->ioaddr + SDHCI_HOST_CONTROL + 1);
+		u8 tmp = mcf_read8(host->ioaddr + SDHCI_HOST_CONTROL + 1);
 
 		tmp &= ~0x03;
 		tmp |= dma_bits;
@@ -82,12 +82,12 @@ static void esdhc_mcf_writeb_be(struct sdhci_host *host, u8 val, int reg)
 		 */
 		host_ctrl |= val;
 		host_ctrl |= (dma_bits << 8);
-		writel(host_ctrl, host->ioaddr + SDHCI_HOST_CONTROL);
+		mcf_write32(host_ctrl, host->ioaddr + SDHCI_HOST_CONTROL);
 
 		return;
 	}
 
-	writel((readl(base) & mask) | (val << shift), base);
+	mcf_write32((mcf_read32(base) & mask) | (val << shift), base);
 }
 
 static void esdhc_mcf_writew_be(struct sdhci_host *host, u16 val, int reg)
@@ -110,24 +110,24 @@ static void esdhc_mcf_writew_be(struct sdhci_host *host, u16 val, int reg)
 		 * As for the fsl driver,
 		 * we have to set the mode in a single write here.
 		 */
-		writel(val << 16 | mcf_data->aside,
+		mcf_write32(val << 16 | mcf_data->aside,
 		       host->ioaddr + SDHCI_TRANSFER_MODE);
 		return;
 	}
 
-	writel((readl(base) & mask) | (val << shift), base);
+	mcf_write32((mcf_read32(base) & mask) | (val << shift), base);
 }
 
 static void esdhc_mcf_writel_be(struct sdhci_host *host, u32 val, int reg)
 {
-	writel(val, host->ioaddr + reg);
+	mcf_write32(val, host->ioaddr + reg);
 }
 
 static u8 esdhc_mcf_readb_be(struct sdhci_host *host, int reg)
 {
 	if (reg == SDHCI_HOST_CONTROL) {
 		u8 __iomem *base = host->ioaddr + (reg & ~3);
-		u16 val = readw(base + 2);
+		u16 val = mcf_read16(base + 2);
 		u8 dma_bits = (val >> 5) & SDHCI_CTRL_DMA_MASK;
 		u8 host_ctrl = val & 0xff;
 
@@ -137,7 +137,7 @@ static u8 esdhc_mcf_readb_be(struct sdhci_host *host, int reg)
 		return host_ctrl;
 	}
 
-	return readb(host->ioaddr + (reg ^ 0x3));
+	return mcf_read8(host->ioaddr + (reg ^ 0x3));
 }
 
 static u16 esdhc_mcf_readw_be(struct sdhci_host *host, int reg)
@@ -149,14 +149,14 @@ static u16 esdhc_mcf_readw_be(struct sdhci_host *host, int reg)
 	if (reg == SDHCI_HOST_VERSION)
 		reg -= 2;
 
-	return readw(host->ioaddr + (reg ^ 0x2));
+	return mcf_read16(host->ioaddr + (reg ^ 0x2));
 }
 
 static u32 esdhc_mcf_readl_be(struct sdhci_host *host, int reg)
 {
 	u32 val;
 
-	val = readl(host->ioaddr + reg);
+	val = mcf_read32(host->ioaddr + reg);
 
 	/*
 	 * RM (25.3.9) sd pin clock must never exceed 25Mhz.
@@ -245,7 +245,7 @@ static void esdhc_mcf_pltfm_set_clock(struct sdhci_host *host,
 	 * fvco = fsys * outdvi1 + 1
 	 * fshdc = fvco / outdiv3 + 1
 	 */
-	temp = readl(pll_dr);
+	temp = mcf_read32(pll_dr);
 	fsys = pltfm_host->clock;
 	fvco = fsys * ((temp & 0x1f) + 1);
 	fesdhc = fvco / (((temp >> 10) & 0x1f) + 1);
diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c
index 0882ce74e0c9b..ceed47ccfda88 100644
--- a/drivers/mmc/host/sdhci-msm.c
+++ b/drivers/mmc/host/sdhci-msm.c
@@ -2165,7 +2165,7 @@ static u32 sdhci_msm_cqe_irq(struct sdhci_host *host, u32 intmask)
 	if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
 		return intmask;
 
-	cqhci_irq(host->mmc, intmask, cmd_error, data_error);
+	cqhci_irq(host->mmc, cmd_error, data_error);
 	return 0;
 }
 
diff --git a/drivers/mmc/host/sdhci-of-arasan.c b/drivers/mmc/host/sdhci-of-arasan.c
index 785d3acb18c58..4ca73e7d799ea 100644
--- a/drivers/mmc/host/sdhci-of-arasan.c
+++ b/drivers/mmc/host/sdhci-of-arasan.c
@@ -555,7 +555,7 @@ static u32 sdhci_arasan_cqhci_irq(struct sdhci_host *host, u32 intmask)
 	if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
 		return intmask;
 
-	cqhci_irq(host->mmc, intmask, cmd_error, data_error);
+	cqhci_irq(host->mmc, cmd_error, data_error);
 
 	return 0;
 }
diff --git a/drivers/mmc/host/sdhci-of-dwcmshc.c b/drivers/mmc/host/sdhci-of-dwcmshc.c
index eef53455b8ee4..1baa2379c9c39 100644
--- a/drivers/mmc/host/sdhci-of-dwcmshc.c
+++ b/drivers/mmc/host/sdhci-of-dwcmshc.c
@@ -624,7 +624,7 @@ static u32 dwcmshc_cqe_irq_handler(struct sdhci_host *host, u32 intmask)
 	if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
 		return intmask;
 
-	cqhci_irq(host->mmc, intmask, cmd_error, data_error);
+	cqhci_irq(host->mmc, cmd_error, data_error);
 
 	return 0;
 }
@@ -2013,6 +2013,7 @@ static void dwcmshc_bf3_hw_reset(struct sdhci_host *host)
 {
 	struct arm_smccc_res res = { 0 };
 
+	pr_debug("%s: resetting...\n", __func__);
 	arm_smccc_smc(BLUEFIELD_SMC_SET_EMMC_RST_N, 0, 0, 0, 0, 0, 0, 0, &res);
 
 	if (res.a0)
@@ -2433,13 +2434,16 @@ static int dwcmshc_probe(struct platform_device *pdev)
 			return err;
 
 		priv->bus_clk = devm_clk_get(dev, "bus");
-		if (!IS_ERR(priv->bus_clk))
-			clk_prepare_enable(priv->bus_clk);
+		if (!IS_ERR(priv->bus_clk)) {
+			err = clk_prepare_enable(priv->bus_clk);
+			if (err)
+				goto err_clk;
+		}
 	}
 
 	err = mmc_of_parse(host->mmc);
 	if (err)
-		goto err_clk;
+		goto err_bus_clk;
 
 	sdhci_get_of_property(pdev);
 
@@ -2453,7 +2457,7 @@ static int dwcmshc_probe(struct platform_device *pdev)
 	if (pltfm_data->init) {
 		err = pltfm_data->init(&pdev->dev, host, priv);
 		if (err)
-			goto err_clk;
+			goto err_bus_clk;
 	}
 
 #ifdef CONFIG_ACPI
@@ -2499,9 +2503,10 @@ static int dwcmshc_probe(struct platform_device *pdev)
 err_rpm:
 	pm_runtime_disable(dev);
 	pm_runtime_put_noidle(dev);
+err_bus_clk:
+	clk_disable_unprepare(priv->bus_clk);
 err_clk:
 	clk_disable_unprepare(pltfm_host->clk);
-	clk_disable_unprepare(priv->bus_clk);
 	clk_bulk_disable_unprepare(priv->num_other_clks, priv->other_clks);
 	return err;
 }
diff --git a/drivers/mmc/host/sdhci-of-ma35d1.c b/drivers/mmc/host/sdhci-of-ma35d1.c
index a3b6768948386..2a826de97779e 100644
--- a/drivers/mmc/host/sdhci-of-ma35d1.c
+++ b/drivers/mmc/host/sdhci-of-ma35d1.c
@@ -290,6 +290,7 @@ static const struct of_device_id sdhci_ma35_dt_ids[] = {
 	{ .compatible = "nuvoton,ma35d1-sdhci" },
 	{}
 };
+MODULE_DEVICE_TABLE(of, sdhci_ma35_dt_ids);
 
 static struct platform_driver sdhci_ma35_driver = {
 	.driver	= {
diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c
index c347fac24515a..b121d896a8049 100644
--- a/drivers/mmc/host/sdhci-pci-core.c
+++ b/drivers/mmc/host/sdhci-pci-core.c
@@ -215,7 +215,7 @@ static u32 sdhci_cqhci_irq(struct sdhci_host *host, u32 intmask)
 	if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
 		return intmask;
 
-	cqhci_irq(host->mmc, intmask, cmd_error, data_error);
+	cqhci_irq(host->mmc, cmd_error, data_error);
 
 	return 0;
 }
diff --git a/drivers/mmc/host/sdhci-pci-gli.c b/drivers/mmc/host/sdhci-pci-gli.c
index 6e4084407662a..b55618566d651 100644
--- a/drivers/mmc/host/sdhci-pci-gli.c
+++ b/drivers/mmc/host/sdhci-pci-gli.c
@@ -1760,7 +1760,7 @@ static u32 sdhci_gl9763e_cqhci_irq(struct sdhci_host *host, u32 intmask)
 	if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
 		return intmask;
 
-	cqhci_irq(host->mmc, intmask, cmd_error, data_error);
+	cqhci_irq(host->mmc, cmd_error, data_error);
 
 	return 0;
 }
diff --git a/drivers/mmc/host/sdhci-tegra.c b/drivers/mmc/host/sdhci-tegra.c
index 820ce4dae58ba..221e48b59f480 100644
--- a/drivers/mmc/host/sdhci-tegra.c
+++ b/drivers/mmc/host/sdhci-tegra.c
@@ -1280,7 +1280,7 @@ static u32 sdhci_tegra_cqhci_irq(struct sdhci_host *host, u32 intmask)
 	if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
 		return intmask;
 
-	cqhci_irq(host->mmc, intmask, cmd_error, data_error);
+	cqhci_irq(host->mmc, cmd_error, data_error);
 
 	return 0;
 }
diff --git a/drivers/mmc/host/sdhci_am654.c b/drivers/mmc/host/sdhci_am654.c
index d235b0aecfdb1..2a27db2f558bb 100644
--- a/drivers/mmc/host/sdhci_am654.c
+++ b/drivers/mmc/host/sdhci_am654.c
@@ -462,7 +462,7 @@ static u32 sdhci_am654_cqhci_irq(struct sdhci_host *host, u32 intmask)
 	if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
 		return intmask;
 
-	cqhci_irq(host->mmc, intmask, cmd_error, data_error);
+	cqhci_irq(host->mmc, cmd_error, data_error);
 
 	return 0;
 }
diff --git a/drivers/mmc/host/vub300.c b/drivers/mmc/host/vub300.c
index 6c3cb2f1c9d32..2dae474dcd067 100644
--- a/drivers/mmc/host/vub300.c
+++ b/drivers/mmc/host/vub300.c
@@ -1586,7 +1586,7 @@ static int __command_write_data(struct vub300_mmc_host *vub300,
 	return linear_length;
 }
 
-static void __vub300_command_response(struct vub300_mmc_host *vub300,
+static bool __vub300_command_response(struct vub300_mmc_host *vub300,
 				      struct mmc_command *cmd,
 				      struct mmc_data *data, int data_length)
 {
@@ -1598,17 +1598,11 @@ static void __vub300_command_response(struct vub300_mmc_host *vub300,
 					    msecs_to_jiffies(msec_timeout));
 	if (respretval == 0) { /* TIMED OUT */
 		/* we don't know which of "out" and "res" if any failed */
-		int result;
 		vub300->usb_timed_out = 1;
 		usb_kill_urb(vub300->command_out_urb);
 		usb_kill_urb(vub300->command_res_urb);
 		cmd->error = -ETIMEDOUT;
-		result = usb_lock_device_for_reset(vub300->udev,
-						   vub300->interface);
-		if (result == 0) {
-			result = usb_reset_device(vub300->udev);
-			usb_unlock_device(vub300->udev);
-		}
+		return true;
 	} else if (respretval < 0) {
 		/* we don't know which of "out" and "res" if any failed */
 		usb_kill_urb(vub300->command_out_urb);
@@ -1704,6 +1698,8 @@ static void __vub300_command_response(struct vub300_mmc_host *vub300,
 	} else {
 		cmd->error = -EINVAL;
 	}
+
+	return false;
 }
 
 static void construct_request_response(struct vub300_mmc_host *vub300,
@@ -1749,6 +1745,7 @@ static void vub300_cmndwork_thread(struct work_struct *work)
 		struct mmc_request *req = vub300->req;
 		struct mmc_command *cmd = vub300->cmd;
 		struct mmc_data *data = vub300->data;
+		bool reset_device;
 		int data_length;
 		mutex_lock(&vub300->cmd_mutex);
 		init_completion(&vub300->command_complete);
@@ -1771,7 +1768,8 @@ static void vub300_cmndwork_thread(struct work_struct *work)
 			data_length = __command_read_data(vub300, cmd, data);
 		else
 			data_length = __command_write_data(vub300, cmd, data);
-		__vub300_command_response(vub300, cmd, data, data_length);
+		reset_device = __vub300_command_response(vub300, cmd,
+							 data, data_length);
 		vub300->req = NULL;
 		vub300->cmd = NULL;
 		vub300->data = NULL;
@@ -1779,6 +1777,16 @@ static void vub300_cmndwork_thread(struct work_struct *work)
 			if (cmd->error == -ENOMEDIUM)
 				check_vub300_port_status(vub300);
 			mutex_unlock(&vub300->cmd_mutex);
+			if (reset_device) {
+				int result;
+
+				result = usb_lock_device_for_reset(vub300->udev,
+								   vub300->interface);
+				if (result == 0) {
+					result = usb_reset_device(vub300->udev);
+					usb_unlock_device(vub300->udev);
+				}
+			}
 			mmc_request_done(vub300->mmc, req);
 			kref_put(&vub300->kref, vub300_delete);
 			return;
@@ -2336,12 +2344,16 @@ static int vub300_probe(struct usb_interface *interface,
 			 interface_to_InterfaceNumber(interface));
 	retval = mmc_add_host(mmc);
 	if (retval)
-		goto err_delete_timer;
+		goto err_stop_io;
 
 	return 0;
 
-err_delete_timer:
-	timer_delete_sync(&vub300->inactivity_timer);
+err_stop_io:
+	vub300->interface = NULL;
+	kref_put(&vub300->kref, vub300_delete);
+
+	return retval;
+
 err_free_host:
 	mmc_free_host(mmc);
 	/*
diff --git a/drivers/mmc/host/wbsd.c b/drivers/mmc/host/wbsd.c
index c33a0223ce7f4..51706898239e8 100644
--- a/drivers/mmc/host/wbsd.c
+++ b/drivers/mmc/host/wbsd.c
@@ -52,9 +52,9 @@
 #ifdef CONFIG_PNP
 
 static const struct pnp_device_id pnp_dev_table[] = {
-	{ "WEC0517", 0 },
-	{ "WEC0518", 0 },
-	{ "", 0 },
+	{ .id = "WEC0517" },
+	{ .id = "WEC0518" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pnp, pnp_dev_table);
diff --git a/drivers/mtd/chips/cfi_cmdset_0001.c b/drivers/mtd/chips/cfi_cmdset_0001.c
index b73596a8e021f..6049ba2d6bcb6 100644
--- a/drivers/mtd/chips/cfi_cmdset_0001.c
+++ b/drivers/mtd/chips/cfi_cmdset_0001.c
@@ -2565,6 +2565,12 @@ static int cfi_intelext_suspend(struct mtd_info *mtd)
 			ret = -EAGAIN;
 			break;
 		case FL_PM_SUSPENDED:
+		case FL_SHUTDOWN:
+			/*
+			 * Already suspended, or put into array mode by the
+			 * reboot notifier ahead of an imminent power-off.
+			 * Either way there is nothing to do.
+			 */
 			break;
 		}
 		mutex_unlock(&chip->mutex);
diff --git a/drivers/mtd/devices/mchp23k256.c b/drivers/mtd/devices/mchp23k256.c
index cef5f9677d39e..66e058df4c328 100644
--- a/drivers/mtd/devices/mchp23k256.c
+++ b/drivers/mtd/devices/mchp23k256.c
@@ -188,7 +188,7 @@ static int mchp23k256_probe(struct spi_device *spi)
 
 	data = dev_get_platdata(&spi->dev);
 
-	flash->caps = of_device_get_match_data(&spi->dev);
+	flash->caps = spi_get_device_match_data(spi);
 	if (!flash->caps)
 		flash->caps = &mchp23k256_caps;
 
diff --git a/drivers/mtd/maps/Kconfig b/drivers/mtd/maps/Kconfig
index dce5e67ce3c2b..f447902d707e1 100644
--- a/drivers/mtd/maps/Kconfig
+++ b/drivers/mtd/maps/Kconfig
@@ -277,6 +277,12 @@ config MTD_PCMCIA_ANONYMOUS
 
 	  If unsure, say N.
 
+config MTD_UCLINUX
+	bool "Generic uClinux RAM/ROM filesystem support"
+	depends on (MTD_RAM=y || MTD_ROM=y) && (!MMU || COLDFIRE)
+	help
+	  Map driver to support image based filesystems for uClinux.
+
 config MTD_PLATRAM
 	tristate "Map driver for platform device RAM (mtd-ram)"
 	select MTD_RAM
diff --git a/drivers/mtd/maps/Makefile b/drivers/mtd/maps/Makefile
index fbed278157f68..01745eca1f730 100644
--- a/drivers/mtd/maps/Makefile
+++ b/drivers/mtd/maps/Makefile
@@ -30,6 +30,7 @@ obj-$(CONFIG_MTD_SUN_UFLASH)	+= sun_uflash.o
 obj-$(CONFIG_MTD_SCx200_DOCFLASH)+= scx200_docflash.o
 obj-$(CONFIG_MTD_SOLUTIONENGINE)+= solutionengine.o
 obj-$(CONFIG_MTD_PCI)		+= pci.o
+obj-$(CONFIG_MTD_UCLINUX)	+= uclinux.o
 obj-$(CONFIG_MTD_SCB2_FLASH)	+= scb2_flash.o
 obj-$(CONFIG_MTD_PLATRAM)	+= plat-ram.o
 obj-$(CONFIG_MTD_VMU)		+= vmu-flash.o
diff --git a/drivers/mtd/maps/map_funcs.c b/drivers/mtd/maps/map_funcs.c
index 1a4add9e119ac..b0054933cae0c 100644
--- a/drivers/mtd/maps/map_funcs.c
+++ b/drivers/mtd/maps/map_funcs.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Out-of-line map I/O functions for simple maps when CONFIG_COMPLEX_MAPPINGS
+ * Out-of-line map I/O functions for simple maps when CONFIG_MTD_COMPLEX_MAPPINGS
  * is enabled.
  */
 
diff --git a/drivers/mtd/maps/uclinux.c b/drivers/mtd/maps/uclinux.c
new file mode 100644
index 0000000000000..de4c46318abb8
--- /dev/null
+++ b/drivers/mtd/maps/uclinux.c
@@ -0,0 +1,118 @@
+/****************************************************************************/
+
+/*
+ *	uclinux.c -- generic memory mapped MTD driver for uclinux
+ *
+ *	(C) Copyright 2002, Greg Ungerer (gerg@snapgear.com)
+ *
+ *      License: GPL
+ */
+
+/****************************************************************************/
+
+#include <linux/moduleparam.h>
+#include <linux/types.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/major.h>
+#include <linux/mtd/mtd.h>
+#include <linux/mtd/map.h>
+#include <linux/mtd/partitions.h>
+#include <asm/io.h>
+#include <asm/sections.h>
+
+/****************************************************************************/
+
+#ifdef CONFIG_MTD_ROM
+#define MAP_NAME "rom"
+#else
+#define MAP_NAME "ram"
+#endif
+
+static struct map_info uclinux_ram_map = {
+	.name = MAP_NAME,
+	.size = 0,
+};
+
+static unsigned long physaddr = -1;
+module_param(physaddr, ulong, S_IRUGO);
+
+static struct mtd_info *uclinux_ram_mtdinfo;
+
+/****************************************************************************/
+
+static const struct mtd_partition uclinux_romfs[] = {
+	{ .name = "ROMfs" }
+};
+
+#define	NUM_PARTITIONS	ARRAY_SIZE(uclinux_romfs)
+
+/****************************************************************************/
+
+static int uclinux_point(struct mtd_info *mtd, loff_t from, size_t len,
+	size_t *retlen, void **virt, resource_size_t *phys)
+{
+	struct map_info *map = mtd->priv;
+	*virt = map->virt + from;
+	if (phys)
+		*phys = map->phys + from;
+	*retlen = len;
+	return(0);
+}
+
+/****************************************************************************/
+
+static int __init uclinux_mtd_init(void)
+{
+	struct mtd_info *mtd;
+	struct map_info *mapp;
+
+	mapp = &uclinux_ram_map;
+
+	if (physaddr == -1)
+		mapp->phys = (resource_size_t)__bss_stop;
+	else
+		mapp->phys = physaddr;
+
+	if (!mapp->size)
+		mapp->size = PAGE_ALIGN(ntohl(*((unsigned long *)(mapp->phys + 8))));
+	mapp->bankwidth = 4;
+
+	printk("uclinux[mtd]: probe address=0x%x size=0x%x\n",
+	       	(int) mapp->phys, (int) mapp->size);
+
+	/*
+	 * The filesystem is guaranteed to be in direct mapped memory. It is
+	 * directly following the kernels own bss region. Following the same
+	 * mechanism used by architectures setting up traditional initrds we
+	 * use phys_to_virt to get the virtual address of its start.
+	 */
+	mapp->virt = phys_to_virt(mapp->phys);
+
+	if (mapp->virt == 0) {
+		printk("uclinux[mtd]: no virtual mapping?\n");
+		return(-EIO);
+	}
+
+	simple_map_init(mapp);
+
+	mtd = do_map_probe("map_" MAP_NAME, mapp);
+	if (!mtd) {
+		printk("uclinux[mtd]: failed to find a mapping?\n");
+		return(-ENXIO);
+	}
+
+	mtd->owner = THIS_MODULE;
+	mtd->_point = uclinux_point;
+	mtd->priv = mapp;
+
+	uclinux_ram_mtdinfo = mtd;
+	mtd_device_register(mtd, uclinux_romfs, NUM_PARTITIONS);
+
+	return(0);
+}
+device_initcall(uclinux_mtd_init);
+
+/****************************************************************************/
diff --git a/drivers/mtd/mtd_virt_concat.c b/drivers/mtd/mtd_virt_concat.c
index 37075ead0f337..da4277ced4d63 100644
--- a/drivers/mtd/mtd_virt_concat.c
+++ b/drivers/mtd/mtd_virt_concat.c
@@ -75,8 +75,8 @@ void mtd_virt_concat_destroy_joins(void)
 		if (item->concat) {
 			mtd_device_unregister(mtd);
 			kfree(mtd->name);
-			mtd_concat_destroy(mtd);
 			mtd_virt_concat_put_mtd_devices(item->concat);
+			mtd_concat_destroy(mtd);
 		}
 	}
 }
@@ -126,8 +126,8 @@ int mtd_virt_concat_destroy(struct mtd_info *mtd)
 		if (concat->mtd.name) {
 			del_mtd_device(&concat->mtd);
 			kfree(concat->mtd.name);
-			mtd_concat_destroy(&concat->mtd);
 			mtd_virt_concat_put_mtd_devices(item->concat);
+			mtd_concat_destroy(&concat->mtd);
 		}
 
 		for (idx = 0; idx < item->count; idx++)
@@ -321,8 +321,10 @@ int mtd_virt_concat_create_join(void)
 
 			if (concat->mtd.name) {
 				ret = memcmp(concat->mtd.name, name, name_sz);
-				if (ret == 0)
+				if (ret == 0) {
+					kfree(name);
 					continue;
+				}
 			}
 			mtd = mtd_concat_create(concat->subdev, concat->num_subdev, name);
 			if (!mtd) {
diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c
index 5765377746288..16629382a787b 100644
--- a/drivers/mtd/mtdcore.c
+++ b/drivers/mtd/mtdcore.c
@@ -105,6 +105,15 @@ static void mtd_release(struct device *dev)
 	device_destroy(&mtd_class, index + 1);
 }
 
+/*
+ * No-op device release used in add_mtd_device() error paths.
+ * Prevents mtd_release() from being called via device_release(),
+ * which would free the mtd_info that the caller still manages.
+ */
+static void mtd_dev_release_nop(struct device *dev)
+{
+}
+
 static void mtd_device_release(struct kref *kref)
 {
 	struct mtd_info *mtd = container_of(kref, struct mtd_info, refcnt);
@@ -799,10 +808,8 @@ int add_mtd_device(struct mtd_info *mtd)
 	mtd_check_of_node(mtd);
 	of_node_get(mtd_get_of_node(mtd));
 	error = device_register(&mtd->dev);
-	if (error) {
-		put_device(&mtd->dev);
+	if (error)
 		goto fail_added;
-	}
 
 	/* Add the nvmem provider */
 	error = mtd_nvmem_add(mtd);
@@ -840,8 +847,16 @@ int add_mtd_device(struct mtd_info *mtd)
 	return 0;
 
 fail_nvmem_add:
-	device_unregister(&mtd->dev);
+	device_del(&mtd->dev);
 fail_added:
+	/*
+	 * Clear type and set nop release to prevent mtd_release() ->
+	 * release_mtd_partition() -> free_partition() from freeing mtd.
+	 * The caller handles cleanup on failure.
+	 */
+	mtd->dev.type = NULL;
+	mtd->dev.release = mtd_dev_release_nop;
+	put_device(&mtd->dev);
 	of_node_put(mtd_get_of_node(mtd));
 fail_devname:
 	idr_remove(&mtd_idr, i);
diff --git a/drivers/mtd/mtdpart.c b/drivers/mtd/mtdpart.c
index 795a94e6b482f..4b41550fd374e 100644
--- a/drivers/mtd/mtdpart.c
+++ b/drivers/mtd/mtdpart.c
@@ -118,6 +118,9 @@ static struct mtd_info *allocate_partition(struct mtd_info *parent,
 				part->name, parent_size - child->part.offset,
 				child->part.size);
 			/* register to preserve ordering */
+			child->part.offset = 0;
+			child->part.size = 0;
+			child->erasesize = parent->erasesize;
 			goto out_register;
 		}
 	}
@@ -264,6 +267,11 @@ int mtd_add_partition(struct mtd_info *parent, const char *name,
 	if (length <= 0)
 		return -EINVAL;
 
+	if (offset < 0 || offset >= (long long)parent_size)
+		return -EINVAL;
+
+	if ((u64)offset + (u64)length > parent_size)
+		return -EINVAL;
 	memset(&part, 0, sizeof(part));
 	part.name = name;
 	part.size = length;
diff --git a/drivers/mtd/mtdswap.c b/drivers/mtd/mtdswap.c
index 866933fc84265..f33f753f0a9fd 100644
--- a/drivers/mtd/mtdswap.c
+++ b/drivers/mtd/mtdswap.c
@@ -125,6 +125,7 @@ struct mtdswap_dev {
 
 	char *page_buf;
 	char *oob_buf;
+	struct dentry *debugfs_stats;
 };
 
 struct mtdswap_oobdata {
@@ -1262,7 +1263,8 @@ static int mtdswap_add_debugfs(struct mtdswap_dev *d)
 	if (IS_ERR_OR_NULL(root))
 		return -1;
 
-	debugfs_create_file("mtdswap_stats", S_IRUSR, root, d, &mtdswap_fops);
+	d->debugfs_stats = debugfs_create_file("mtdswap_stats", 0400, root,
+					       d, &mtdswap_fops);
 
 	return 0;
 }
@@ -1463,6 +1465,7 @@ static void mtdswap_remove_dev(struct mtd_blktrans_dev *dev)
 {
 	struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
 
+	debugfs_remove(d->debugfs_stats);
 	del_mtd_blktrans_dev(dev);
 	mtdswap_cleanup(d);
 	kfree(d);
diff --git a/drivers/mtd/nand/ecc-mtk.c b/drivers/mtd/nand/ecc-mtk.c
index c75bb8b80cc1e..66f0985ef7cdb 100644
--- a/drivers/mtd/nand/ecc-mtk.c
+++ b/drivers/mtd/nand/ecc-mtk.c
@@ -123,8 +123,8 @@ static int mt7622_ecc_regs[] = {
 	[ECC_DECIRQ_STA] =      0x144,
 };
 
-static inline void mtk_ecc_wait_idle(struct mtk_ecc *ecc,
-				     enum mtk_ecc_operation op)
+static inline int mtk_ecc_wait_idle(struct mtk_ecc *ecc,
+				    enum mtk_ecc_operation op)
 {
 	struct device *dev = ecc->dev;
 	u32 val;
@@ -136,6 +136,8 @@ static inline void mtk_ecc_wait_idle(struct mtk_ecc *ecc,
 	if (ret)
 		dev_warn(dev, "%s NOT idle\n",
 			 op == ECC_ENCODE ? "encoder" : "decoder");
+
+	return ret;
 }
 
 static irqreturn_t mtk_ecc_irq(int irq, void *id)
@@ -265,6 +267,7 @@ static struct mtk_ecc *mtk_ecc_get(struct device_node *np)
 {
 	struct platform_device *pdev;
 	struct mtk_ecc *ecc;
+	int ret;
 
 	pdev = of_find_device_by_node(np);
 	if (!pdev)
@@ -276,7 +279,12 @@ static struct mtk_ecc *mtk_ecc_get(struct device_node *np)
 		return ERR_PTR(-EPROBE_DEFER);
 	}
 
-	clk_prepare_enable(ecc->clk);
+	ret = clk_prepare_enable(ecc->clk);
+	if (ret) {
+		put_device(&pdev->dev);
+		return ERR_PTR(ret);
+	}
+
 	mtk_ecc_hw_init(ecc);
 
 	return ecc;
@@ -312,7 +320,11 @@ int mtk_ecc_enable(struct mtk_ecc *ecc, struct mtk_ecc_config *config)
 		return ret;
 	}
 
-	mtk_ecc_wait_idle(ecc, op);
+	ret = mtk_ecc_wait_idle(ecc, op);
+	if (ret) {
+		mutex_unlock(&ecc->lock);
+		return ret;
+	}
 
 	ret = mtk_ecc_config(ecc, config);
 	if (ret) {
@@ -412,7 +424,9 @@ int mtk_ecc_encode(struct mtk_ecc *ecc, struct mtk_ecc_config *config,
 	if (ret)
 		goto timeout;
 
-	mtk_ecc_wait_idle(ecc, ECC_ENCODE);
+	ret = mtk_ecc_wait_idle(ecc, ECC_ENCODE);
+	if (ret)
+		goto timeout;
 
 	/* Program ECC bytes to OOB: per sector oob = FDM + ECC + SPARE */
 	len = (config->strength * ecc->caps->parity_bits + 7) >> 3;
diff --git a/drivers/mtd/nand/onenand/onenand_samsung.c b/drivers/mtd/nand/onenand/onenand_samsung.c
index 6d6aa709a21f8..b7b7758ce4d86 100644
--- a/drivers/mtd/nand/onenand/onenand_samsung.c
+++ b/drivers/mtd/nand/onenand/onenand_samsung.c
@@ -554,6 +554,9 @@ static int s5pc110_dma_poll(dma_addr_t dst, dma_addr_t src, size_t count, int di
 	} while (!(status & S5PC110_DMA_TRANS_STATUS_TD) &&
 		time_before(jiffies, timeout));
 
+	if (!(status & S5PC110_DMA_TRANS_STATUS_TD))
+		return -ETIMEDOUT;
+
 	writel(S5PC110_DMA_TRANS_CMD_TDC, base + S5PC110_DMA_TRANS_CMD);
 
 	return 0;
@@ -608,7 +611,9 @@ static int s5pc110_dma_irq(dma_addr_t dst, dma_addr_t src, size_t count, int dir
 
 	writel(S5PC110_DMA_TRANS_CMD_TR, base + S5PC110_DMA_TRANS_CMD);
 
-	wait_for_completion_timeout(&onenand->complete, msecs_to_jiffies(20));
+	if (!wait_for_completion_timeout(&onenand->complete,
+					 msecs_to_jiffies(20)))
+		return -ETIMEDOUT;
 
 	return 0;
 }
diff --git a/drivers/mtd/nand/raw/Kconfig b/drivers/mtd/nand/raw/Kconfig
index 64b8b99a3a682..1f4053e531fd8 100644
--- a/drivers/mtd/nand/raw/Kconfig
+++ b/drivers/mtd/nand/raw/Kconfig
@@ -72,6 +72,7 @@ config MTD_NAND_AU1550
 config MTD_NAND_NDFC
 	tristate "IBM/MCC 4xx NAND controller"
 	depends on 44x || COMPILE_TEST
+	depends on OF
 	select MTD_NAND_ECC_SW_HAMMING
 	select MTD_NAND_ECC_SW_HAMMING_SMC
 	help
diff --git a/drivers/mtd/nand/raw/atmel/nand-controller.c b/drivers/mtd/nand/raw/atmel/nand-controller.c
index e7fdf532c5fef..8a19408abb635 100644
--- a/drivers/mtd/nand/raw/atmel/nand-controller.c
+++ b/drivers/mtd/nand/raw/atmel/nand-controller.c
@@ -1799,8 +1799,7 @@ atmel_nand_controller_legacy_add_nands(struct atmel_nand_controller *nc)
 	 * Legacy bindings only allow connecting a single NAND with a unique CS
 	 * line to the controller.
 	 */
-	nand = devm_kzalloc(nc->dev, sizeof(*nand) + sizeof(*nand->cs),
-			    GFP_KERNEL);
+	nand = devm_kzalloc(nc->dev, struct_size(nand, cs, 1), GFP_KERNEL);
 	if (!nand)
 		return -ENOMEM;
 
diff --git a/drivers/mtd/nand/raw/fsl_ifc_nand.c b/drivers/mtd/nand/raw/fsl_ifc_nand.c
index fad0334f759dd..a88ac2cfaccdc 100644
--- a/drivers/mtd/nand/raw/fsl_ifc_nand.c
+++ b/drivers/mtd/nand/raw/fsl_ifc_nand.c
@@ -684,8 +684,15 @@ static int fsl_ifc_read_page(struct nand_chip *chip, uint8_t *buf,
 		return check_erased_page(chip, buf);
 	}
 
-	if (ctrl->nand_stat != IFC_NAND_EVTER_STAT_OPC)
+	if (!ctrl->nand_stat) {
 		mtd->ecc_stats.failed++;
+		return -ETIMEDOUT;
+	}
+
+	if (ctrl->nand_stat != IFC_NAND_EVTER_STAT_OPC) {
+		mtd->ecc_stats.failed++;
+		return -EIO;
+	}
 
 	return nctrl->max_bitflips;
 }
diff --git a/drivers/mtd/nand/raw/ingenic/ingenic_ecc.c b/drivers/mtd/nand/raw/ingenic/ingenic_ecc.c
index 525c34c281b65..beb033705cf3e 100644
--- a/drivers/mtd/nand/raw/ingenic/ingenic_ecc.c
+++ b/drivers/mtd/nand/raw/ingenic/ingenic_ecc.c
@@ -67,6 +67,7 @@ static struct ingenic_ecc *ingenic_ecc_get(struct device_node *np)
 {
 	struct platform_device *pdev;
 	struct ingenic_ecc *ecc;
+	int ret;
 
 	pdev = of_find_device_by_node(np);
 	if (!pdev)
@@ -78,7 +79,11 @@ static struct ingenic_ecc *ingenic_ecc_get(struct device_node *np)
 	}
 
 	ecc = platform_get_drvdata(pdev);
-	clk_prepare_enable(ecc->clk);
+	ret = clk_prepare_enable(ecc->clk);
+	if (ret) {
+		put_device(&pdev->dev);
+		return ERR_PTR(ret);
+	}
 
 	return ecc;
 }
diff --git a/drivers/mtd/nand/raw/lpc32xx_mlc.c b/drivers/mtd/nand/raw/lpc32xx_mlc.c
index 19b13ae536d48..8f6a89d9ba83f 100644
--- a/drivers/mtd/nand/raw/lpc32xx_mlc.c
+++ b/drivers/mtd/nand/raw/lpc32xx_mlc.c
@@ -396,6 +396,7 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, void *mem, int len,
 	struct lpc32xx_nand_host *host = nand_get_controller_data(chip);
 	struct dma_async_tx_descriptor *desc;
 	int flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
+	unsigned long time_left;
 	int res;
 
 	sg_init_one(&host->sgl, mem, len);
@@ -410,6 +411,7 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, void *mem, int len,
 				       flags);
 	if (!desc) {
 		dev_err(mtd->dev.parent, "Failed to prepare slave sg\n");
+		res = -ENXIO;
 		goto out1;
 	}
 
@@ -420,7 +422,13 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, void *mem, int len,
 	dmaengine_submit(desc);
 	dma_async_issue_pending(host->dma_chan);
 
-	wait_for_completion_timeout(&host->comp_dma, msecs_to_jiffies(1000));
+	time_left = wait_for_completion_timeout(&host->comp_dma,
+						msecs_to_jiffies(1000));
+	if (!time_left) {
+		dmaengine_terminate_sync(host->dma_chan);
+		res = -ETIMEDOUT;
+		goto out1;
+	}
 
 	dma_unmap_sg(host->dma_chan->device->dev, &host->sgl, 1,
 		     DMA_BIDIRECTIONAL);
@@ -428,7 +436,7 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, void *mem, int len,
 out1:
 	dma_unmap_sg(host->dma_chan->device->dev, &host->sgl, 1,
 		     DMA_BIDIRECTIONAL);
-	return -ENXIO;
+	return res;
 }
 
 static int lpc32xx_read_page(struct nand_chip *chip, uint8_t *buf,
diff --git a/drivers/mtd/nand/raw/lpc32xx_slc.c b/drivers/mtd/nand/raw/lpc32xx_slc.c
index 3ca30e7dce336..10c8080207f49 100644
--- a/drivers/mtd/nand/raw/lpc32xx_slc.c
+++ b/drivers/mtd/nand/raw/lpc32xx_slc.c
@@ -430,6 +430,7 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, dma_addr_t dma,
 	struct dma_async_tx_descriptor *desc;
 	int flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
 	int res;
+	unsigned long time_left;
 
 	host->dma_slave_config.direction = dir;
 	host->dma_slave_config.src_addr = dma;
@@ -467,12 +468,19 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, dma_addr_t dma,
 	dmaengine_submit(desc);
 	dma_async_issue_pending(host->dma_chan);
 
-	wait_for_completion_timeout(&host->comp, msecs_to_jiffies(1000));
+	time_left = wait_for_completion_timeout(&host->comp,
+						msecs_to_jiffies(1000));
+	if (!time_left) {
+		dmaengine_terminate_sync(host->dma_chan);
+		res = -ETIMEDOUT;
+	} else {
+		res = 0;
+	}
 
 	dma_unmap_sg(host->dma_chan->device->dev, &host->sgl, 1,
 		     DMA_BIDIRECTIONAL);
 
-	return 0;
+	return res;
 out1:
 	dma_unmap_sg(host->dma_chan->device->dev, &host->sgl, 1,
 		     DMA_BIDIRECTIONAL);
diff --git a/drivers/mtd/nand/raw/ndfc.c b/drivers/mtd/nand/raw/ndfc.c
index a937ca3eeff56..a48274297d3b1 100644
--- a/drivers/mtd/nand/raw/ndfc.c
+++ b/drivers/mtd/nand/raw/ndfc.c
@@ -188,7 +188,7 @@ static int ndfc_probe(struct platform_device *ofdev)
 	const __be32 *reg;
 	u32 ccr;
 	u32 cs;
-	int err, len;
+	int err, len = 0;
 
 	/* Read the reg property to get the chip select */
 	reg = of_get_property(ofdev->dev.of_node, "reg", &len);
diff --git a/drivers/mtd/nand/raw/pl35x-nand-controller.c b/drivers/mtd/nand/raw/pl35x-nand-controller.c
index 06f8f1e14b9c2..bd89aaadd1b2f 100644
--- a/drivers/mtd/nand/raw/pl35x-nand-controller.c
+++ b/drivers/mtd/nand/raw/pl35x-nand-controller.c
@@ -862,8 +862,11 @@ static int pl35x_nfc_setup_interface(struct nand_chip *chip, int cs,
 			  PL35X_SMC_NAND_TAR_CYCLES(tmgs.t_ar) |
 			  PL35X_SMC_NAND_TRR_CYCLES(tmgs.t_rr);
 
-	writel(plnand->timings, nfc->conf_regs + PL35X_SMC_CYCLES);
-	pl35x_smc_update_regs(nfc);
+	/*
+	 * Reset nfc->selected_chip so the next command will cause the timing
+	 * registers to be updated in ->*_select_target().
+	 */
+	nfc->selected_chip = NULL;
 
 	return 0;
 }
@@ -914,7 +917,6 @@ static int pl35x_nand_init_hw_ecc_controller(struct pl35x_nandc *nfc,
 	chip->ecc.steps = mtd->writesize / chip->ecc.size;
 	chip->ecc.read_page = pl35x_nand_read_page_hwecc;
 	chip->ecc.write_page = pl35x_nand_write_page_hwecc;
-	chip->ecc.write_page_raw = nand_monolithic_write_page_raw;
 	pl35x_smc_set_ecc_pg_size(nfc, chip, mtd->writesize);
 
 	nfc->ecc_buf = devm_kmalloc(nfc->dev, chip->ecc.bytes * chip->ecc.steps,
@@ -973,18 +975,19 @@ static int pl35x_nand_attach_chip(struct nand_chip *chip)
 
 	switch (chip->ecc.engine_type) {
 	case NAND_ECC_ENGINE_TYPE_ON_DIE:
-		dev_dbg(nfc->dev, "Using on-die ECC\n");
+		dev_dbg(nfc->dev, "Using on-die hardware ECC\n");
 		/* Keep these legacy BBT descriptors for ON_DIE situations */
 		chip->bbt_td = &bbt_main_descr;
 		chip->bbt_md = &bbt_mirror_descr;
 		fallthrough;
 	case NAND_ECC_ENGINE_TYPE_NONE:
+		dev_dbg(nfc->dev, "Using no ECC engine\n");
+		break;
 	case NAND_ECC_ENGINE_TYPE_SOFT:
-		dev_dbg(nfc->dev, "Using software ECC (Hamming 1-bit/512B)\n");
-		chip->ecc.write_page_raw = nand_monolithic_write_page_raw;
+		dev_dbg(nfc->dev, "Using software ECC\n");
 		break;
 	case NAND_ECC_ENGINE_TYPE_ON_HOST:
-		dev_dbg(nfc->dev, "Using hardware ECC\n");
+		dev_dbg(nfc->dev, "Using on-host hardware ECC\n");
 		ret = pl35x_nand_init_hw_ecc_controller(nfc, chip);
 		if (ret)
 			return ret;
@@ -995,6 +998,9 @@ static int pl35x_nand_attach_chip(struct nand_chip *chip)
 		return -EINVAL;
 	}
 
+	chip->ecc.read_page_raw = nand_monolithic_read_page_raw;
+	chip->ecc.write_page_raw = nand_monolithic_write_page_raw;
+
 	return 0;
 }
 
diff --git a/drivers/mtd/nand/spi/Makefile b/drivers/mtd/nand/spi/Makefile
index a47bd22cd309c..b5ccb44860df8 100644
--- a/drivers/mtd/nand/spi/Makefile
+++ b/drivers/mtd/nand/spi/Makefile
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
 spinand-objs := core.o otp.o
-spinand-objs += alliancememory.o ato.o dosilicon.o esmt.o fmsh.o foresee.o gigadevice.o
+spinand-objs += alliancememory.o ato.o dosilicon.o esmt.o fmsh.o foresee.o gigadevice.o heyangtek.o
 spinand-objs += macronix.o micron.o paragon.o skyhigh.o toshiba.o winbond.o xtx.o
 obj-$(CONFIG_MTD_SPI_NAND) += spinand.o
diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c
index f86786344d52b..8bf9301f25e75 100644
--- a/drivers/mtd/nand/spi/core.c
+++ b/drivers/mtd/nand/spi/core.c
@@ -822,7 +822,7 @@ static int spinand_mtd_regular_page_read(struct mtd_info *mtd, loff_t from,
 	bool disable_ecc = false;
 	bool ecc_failed = false;
 	unsigned int retry_mode = 0;
-	int ret;
+	int ret = 0;
 
 	old_stats = mtd->ecc_stats;
 
@@ -1359,6 +1359,7 @@ static const struct spinand_manufacturer *spinand_manufacturers[] = {
 	&fmsh_spinand_manufacturer,
 	&foresee_spinand_manufacturer,
 	&gigadevice_spinand_manufacturer,
+	&heyangtek_spinand_manufacturer,
 	&macronix_spinand_manufacturer,
 	&micron_spinand_manufacturer,
 	&paragon_spinand_manufacturer,
diff --git a/drivers/mtd/nand/spi/fmsh.c b/drivers/mtd/nand/spi/fmsh.c
index f417955f7d1c5..be5db9f9502ba 100644
--- a/drivers/mtd/nand/spi/fmsh.c
+++ b/drivers/mtd/nand/spi/fmsh.c
@@ -9,6 +9,16 @@
 #include <linux/kernel.h>
 #include <linux/mtd/spinand.h>
 
+#define FM25G01B_STATUS_ECC_MASK		(7 << 4)
+	#define FM25G01B_STATUS_ECC_NO_BITFLIPS		(0 << 4)
+	#define FM25G01B_STATUS_ECC_1_3_BITFLIPS	(1 << 4)
+	#define FM25G01B_STATUS_ECC_4_BITFLIPS		(2 << 4)
+	#define FM25G01B_STATUS_ECC_5_BITFLIPS		(3 << 4)
+	#define FM25G01B_STATUS_ECC_6_BITFLIPS		(4 << 4)
+	#define FM25G01B_STATUS_ECC_7_BITFLIPS		(5 << 4)
+	#define FM25G01B_STATUS_ECC_8_BITFLIPS		(6 << 4)
+	#define FM25G01B_STATUS_ECC_UNCOR_ERROR		(7 << 4)
+
 #define FM25S01BI3_STATUS_ECC_MASK		(7 << 4)
 	#define FM25S01BI3_STATUS_ECC_NO_BITFLIPS	(0 << 4)
 	#define FM25S01BI3_STATUS_ECC_1_3_BITFLIPS	(1 << 4)
@@ -34,6 +44,66 @@ static SPINAND_OP_VARIANTS(update_cache_variants,
 		SPINAND_PROG_LOAD_1S_1S_4S_OP(false, 0, NULL, 0),
 		SPINAND_PROG_LOAD_1S_1S_1S_OP(false, 0, NULL, 0));
 
+static int fm25g01b_ooblayout_ecc(struct mtd_info *mtd, int section,
+				  struct mtd_oob_region *region)
+{
+	if (section)
+		return -ERANGE;
+
+	region->offset = 64;
+	region->length = 64;
+
+	return 0;
+}
+
+static int fm25g01b_ooblayout_free(struct mtd_info *mtd, int section,
+				   struct mtd_oob_region *region)
+{
+	if (section)
+		return -ERANGE;
+
+	/* reserve 2 bytes for the BBM */
+	region->offset = 2;
+	region->length = 62;
+
+	return 0;
+}
+
+static int fm25g01b_ecc_get_status(struct spinand_device *spinand,
+				   u8 status)
+{
+	switch (status & FM25G01B_STATUS_ECC_MASK) {
+	case FM25G01B_STATUS_ECC_NO_BITFLIPS:
+		return 0;
+
+	case FM25G01B_STATUS_ECC_1_3_BITFLIPS:
+		return 3;
+
+	case FM25G01B_STATUS_ECC_4_BITFLIPS:
+		return 4;
+
+	case FM25G01B_STATUS_ECC_5_BITFLIPS:
+		return 5;
+
+	case FM25G01B_STATUS_ECC_6_BITFLIPS:
+		return 6;
+
+	case FM25G01B_STATUS_ECC_7_BITFLIPS:
+		return 7;
+
+	case FM25G01B_STATUS_ECC_8_BITFLIPS:
+		return 8;
+
+	case FM25G01B_STATUS_ECC_UNCOR_ERROR:
+		return -EBADMSG;
+
+	default:
+		break;
+	}
+
+	return -EINVAL;
+}
+
 static int fm25s01a_ooblayout_ecc(struct mtd_info *mtd, int section,
 				  struct mtd_oob_region *region)
 {
@@ -102,6 +172,11 @@ static int fm25s01bi3_ooblayout_free(struct mtd_info *mtd, int section,
 	return 0;
 }
 
+static const struct mtd_ooblayout_ops fm25g01b_ooblayout = {
+	.ecc = fm25g01b_ooblayout_ecc,
+	.free = fm25g01b_ooblayout_free,
+};
+
 static const struct mtd_ooblayout_ops fm25s01a_ooblayout = {
 	.ecc = fm25s01a_ooblayout_ecc,
 	.free = fm25s01a_ooblayout_free,
@@ -113,6 +188,26 @@ static const struct mtd_ooblayout_ops fm25s01bi3_ooblayout = {
 };
 
 static const struct spinand_info fmsh_spinand_table[] = {
+	SPINAND_INFO("FM25G01B",
+		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xd1),
+		     NAND_MEMORG(1, 2048, 128, 64, 1024, 21, 1, 1, 1),
+		     NAND_ECCREQ(8, 528),
+		     SPINAND_INFO_OP_VARIANTS(&read_cache_variants,
+					      &write_cache_variants,
+					      &update_cache_variants),
+		     SPINAND_HAS_QE_BIT,
+		     SPINAND_ECCINFO(&fm25g01b_ooblayout,
+				     fm25g01b_ecc_get_status)),
+	SPINAND_INFO("FM25G02B",
+		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xd2),
+		     NAND_MEMORG(1, 2048, 128, 64, 2048, 41, 1, 1, 1),
+		     NAND_ECCREQ(8, 528),
+		     SPINAND_INFO_OP_VARIANTS(&read_cache_variants,
+					      &write_cache_variants,
+					      &update_cache_variants),
+		     SPINAND_HAS_QE_BIT,
+		     SPINAND_ECCINFO(&fm25g01b_ooblayout,
+				     fm25g01b_ecc_get_status)),
 	SPINAND_INFO("FM25S01A",
 		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xE4),
 		     NAND_MEMORG(1, 2048, 64, 64, 1024, 20, 1, 1, 1),
diff --git a/drivers/mtd/nand/spi/heyangtek.c b/drivers/mtd/nand/spi/heyangtek.c
new file mode 100644
index 0000000000000..7fc50fd3de087
--- /dev/null
+++ b/drivers/mtd/nand/spi/heyangtek.c
@@ -0,0 +1,132 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Authors:
+ *	Andrey Zolotarev <andrey.zolotarev@keenetic.com> - the main driver logic
+ *	Aleksei Sviridkin <f@lex.la> - adaptation to the mainline Linux kernel
+ *
+ * Based on:
+ *	https://github.com/keenetic/kernel-49/commit/bacade569fb12bc0ad31ba09bca9b890118fbca7
+ */
+
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/mtd/spinand.h>
+
+#define SPINAND_MFR_HEYANGTEK			0xc9
+
+#define HYF1GQ4_STATUS_ECC_LIMIT_BITFLIPS	(3 << 4)
+
+static SPINAND_OP_VARIANTS(read_cache_variants,
+		SPINAND_PAGE_READ_FROM_CACHE_1S_4S_4S_OP(0, 1, NULL, 0, 0),
+		SPINAND_PAGE_READ_FROM_CACHE_1S_1S_4S_OP(0, 1, NULL, 0, 0),
+		SPINAND_PAGE_READ_FROM_CACHE_1S_2S_2S_OP(0, 1, NULL, 0, 0),
+		SPINAND_PAGE_READ_FROM_CACHE_1S_1S_2S_OP(0, 1, NULL, 0, 0),
+		SPINAND_PAGE_READ_FROM_CACHE_FAST_1S_1S_1S_OP(0, 1, NULL, 0, 0),
+		SPINAND_PAGE_READ_FROM_CACHE_1S_1S_1S_OP(0, 1, NULL, 0, 0));
+
+static SPINAND_OP_VARIANTS(write_cache_variants,
+		SPINAND_PROG_LOAD_1S_1S_4S_OP(true, 0, NULL, 0),
+		SPINAND_PROG_LOAD_1S_1S_1S_OP(true, 0, NULL, 0));
+
+static SPINAND_OP_VARIANTS(update_cache_variants,
+		SPINAND_PROG_LOAD_1S_1S_4S_OP(false, 0, NULL, 0),
+		SPINAND_PROG_LOAD_1S_1S_1S_OP(false, 0, NULL, 0));
+
+/*
+ * HYF1GQ4UDACAE is a GD5F1GQ4-compatible die, so the OOB layout is taken
+ * from gd5fxgq4xa: the on-die ECC parity occupies bytes 8..15 of each
+ * 16-byte section, the bad block marker sits in byte 0 and the remaining
+ * bytes are exposed as free.
+ */
+static int hyf1gq4_ooblayout_ecc(struct mtd_info *mtd, int section,
+				 struct mtd_oob_region *region)
+{
+	if (section > 3)
+		return -ERANGE;
+
+	region->offset = (16 * section) + 8;
+	region->length = 8;
+
+	return 0;
+}
+
+static int hyf1gq4_ooblayout_free(struct mtd_info *mtd, int section,
+				  struct mtd_oob_region *region)
+{
+	if (section > 3)
+		return -ERANGE;
+
+	if (section) {
+		region->offset = 16 * section;
+		region->length = 8;
+	} else {
+		/* section 0 has one byte reserved for the bad block marker */
+		region->offset = 1;
+		region->length = 7;
+	}
+
+	return 0;
+}
+
+static const struct mtd_ooblayout_ops hyf1gq4_ooblayout = {
+	.ecc = hyf1gq4_ooblayout_ecc,
+	.free = hyf1gq4_ooblayout_free,
+};
+
+static int hyf1gq4_ecc_get_status(struct spinand_device *spinand, u8 status)
+{
+	struct nand_device *nand = spinand_to_nand(spinand);
+
+	switch (status & STATUS_ECC_MASK) {
+	case STATUS_ECC_NO_BITFLIPS:
+		return 0;
+
+	case STATUS_ECC_UNCOR_ERROR:
+		return -EBADMSG;
+
+	case STATUS_ECC_HAS_BITFLIPS:
+		/*
+		 * The die exposes only a coarse 2-bit ECC status and has no
+		 * register for the exact bitflip count. This code means
+		 * "corrected, below the refresh threshold", so report half of
+		 * the ECC strength as a representative value.
+		 */
+		return nanddev_get_ecc_conf(nand)->strength / 2;
+
+	case HYF1GQ4_STATUS_ECC_LIMIT_BITFLIPS:
+		/*
+		 * "Corrected, refresh recommended": report the full ECC
+		 * strength so the upper layers relocate the data.
+		 */
+		return nanddev_get_ecc_conf(nand)->strength;
+
+	default:
+		break;
+	}
+
+	return -EINVAL;
+}
+
+static const struct spinand_info heyangtek_spinand_table[] = {
+	SPINAND_INFO("HYF1GQ4UDACAE",
+		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_ADDR, 0x21),
+		     NAND_MEMORG(1, 2048, 64, 64, 1024, 20, 1, 1, 1),
+		     NAND_ECCREQ(4, 512),
+		     SPINAND_INFO_OP_VARIANTS(&read_cache_variants,
+					      &write_cache_variants,
+					      &update_cache_variants),
+		     SPINAND_HAS_QE_BIT,
+		     SPINAND_ECCINFO(&hyf1gq4_ooblayout,
+				     hyf1gq4_ecc_get_status)),
+};
+
+static const struct spinand_manufacturer_ops heyangtek_spinand_manuf_ops = {
+};
+
+const struct spinand_manufacturer heyangtek_spinand_manufacturer = {
+	.id = SPINAND_MFR_HEYANGTEK,
+	.name = "HeYangTek",
+	.chips = heyangtek_spinand_table,
+	.nchips = ARRAY_SIZE(heyangtek_spinand_table),
+	.ops = &heyangtek_spinand_manuf_ops,
+};
diff --git a/drivers/mtd/parsers/redboot.c b/drivers/mtd/parsers/redboot.c
index bf162c44eafe0..120b2eab21fc9 100644
--- a/drivers/mtd/parsers/redboot.c
+++ b/drivers/mtd/parsers/redboot.c
@@ -192,6 +192,7 @@ static int parse_redboot_partitions(struct mtd_info *master,
 
 	for (i = 0; i < numslots; i++) {
 		struct fis_list *new_fl, **prev;
+		size_t name_len;
 
 		if (buf[i].name[0] == 0xff) {
 			if (buf[i].name[1] == 0xff) {
@@ -203,8 +204,14 @@ static int parse_redboot_partitions(struct mtd_info *master,
 		if (!redboot_checksum(&buf[i]))
 			break;
 
+		name_len = strnlen(buf[i].name, sizeof(buf[i].name));
+		if (name_len == sizeof(buf[i].name)) {
+			ret = -EINVAL;
+			goto out;
+		}
+
 		new_fl = kmalloc_obj(struct fis_list);
-		namelen += strlen(buf[i].name) + 1;
+		namelen += name_len + 1;
 		if (!new_fl) {
 			ret = -ENOMEM;
 			goto out;
diff --git a/drivers/mtd/ubi/ubi.h b/drivers/mtd/ubi/ubi.h
index af466cd83ae07..95548d5aa9308 100644
--- a/drivers/mtd/ubi/ubi.h
+++ b/drivers/mtd/ubi/ubi.h
@@ -160,6 +160,7 @@ struct ubi_vid_io_buf {
  * struct ubi_wl_entry - wear-leveling entry.
  * @u.rb: link in the corresponding (free/used) RB-tree
  * @u.list: link in the protection queue
+ * @u: union of rb_node and list_head
  * @ec: erase counter
  * @pnum: physical eraseblock number
  *
@@ -282,6 +283,7 @@ struct ubi_eba_leb_desc {
  * @writers: number of users holding this volume in read-write mode
  * @exclusive: whether somebody holds this volume in exclusive mode
  * @metaonly: whether somebody is altering only meta data of this volume
+ * @is_dead: prevents taking a reference to the volume
  *
  * @reserved_pebs: how many physical eraseblocks are reserved for this volume
  * @vol_type: volume type (%UBI_DYNAMIC_VOLUME or %UBI_STATIC_VOLUME)
@@ -449,6 +451,7 @@ struct ubi_debug_info {
  *                @vol->eba_tbl.
  * @ref_count: count of references on the UBI device
  * @image_seq: image sequence number recorded on EC headers
+ * @is_dead: prevents taking a reference to the volume
  *
  * @rsvd_pebs: count of reserved physical eraseblocks
  * @avail_pebs: count of available physical eraseblocks
@@ -741,7 +744,7 @@ struct ubi_ainf_volume {
  * @highest_vol_id: highest volume ID
  * @is_empty: flag indicating whether the MTD device is empty or not
  * @force_full_scan: flag indicating whether we need to do a full scan and drop
-		     all existing Fastmap data structures
+ *		     all existing Fastmap data structures
  * @min_ec: lowest erase counter value
  * @max_ec: highest erase counter value
  * @max_sqnum: highest sequence number value
@@ -750,7 +753,7 @@ struct ubi_ainf_volume {
  * @ec_count: a temporary variable used when calculating @mean_ec
  * @aeb_slab_cache: slab cache for &struct ubi_ainf_peb objects
  * @ech: temporary EC header. Only available during scan
- * @vidh: temporary VID buffer. Only available during scan
+ * @vidb: temporary VID buffer. Only available during scan
  *
  * This data structure contains the result of attaching an MTD device and may
  * be used by other UBI sub-systems to build final UBI data structures, further
@@ -1090,7 +1093,7 @@ static inline void ubi_init_vid_buf(const struct ubi_device *ubi,
 }
 
 /**
- * ubi_init_vid_buf - Allocate a VID buffer
+ * ubi_alloc_vid_buf - Allocate a VID buffer
  * @ubi: the UBI device
  * @gfp_flags: GFP flags to use for the allocation
  */
diff --git a/drivers/net/amt.c b/drivers/net/amt.c
index 951dd10e192b7..02168862f676d 100644
--- a/drivers/net/amt.c
+++ b/drivers/net/amt.c
@@ -708,11 +708,13 @@ static void amt_send_request(struct amt_dev *amt, bool v6)
 	struct iphdr *iph;
 	struct rtable *rt;
 	struct flowi4 fl4;
+	__be32 remote_ip;
 	struct sock *sk;
 	u32 len;
 	int err;
 
 	rcu_read_lock();
+	remote_ip = READ_ONCE(amt->remote_ip);
 	sk = rcu_dereference(amt->sk);
 	if (!sk)
 		goto out;
@@ -721,7 +723,7 @@ static void amt_send_request(struct amt_dev *amt, bool v6)
 		goto out;
 
 	rt = ip_route_output_ports(amt->net, &fl4, sk,
-				   amt->remote_ip, amt->local_ip,
+				   remote_ip, amt->local_ip,
 				   amt->gw_port, amt->relay_port,
 				   IPPROTO_UDP, 0,
 				   amt->stream_dev->ifindex);
@@ -762,7 +764,7 @@ static void amt_send_request(struct amt_dev *amt, bool v6)
 	udph->check	= 0;
 	offset = skb_transport_offset(skb);
 	skb->csum = skb_checksum(skb, offset, skb->len - offset, 0);
-	udph->check = csum_tcpudp_magic(amt->local_ip, amt->remote_ip,
+	udph->check = csum_tcpudp_magic(amt->local_ip, remote_ip,
 					sizeof(*udph) + sizeof(*amtrh),
 					IPPROTO_UDP, skb->csum);
 
@@ -773,7 +775,7 @@ static void amt_send_request(struct amt_dev *amt, bool v6)
 	iph->tos	= AMT_TOS;
 	iph->frag_off	= 0;
 	iph->ttl	= ip4_dst_hoplimit(&rt->dst);
-	iph->daddr	= amt->remote_ip;
+	iph->daddr	= remote_ip;
 	iph->saddr	= amt->local_ip;
 	iph->protocol	= IPPROTO_UDP;
 	iph->tot_len	= htons(len);
@@ -962,7 +964,7 @@ static void amt_event_send_request(struct amt_dev *amt)
 		amt->qi = AMT_INIT_REQ_TIMEOUT;
 		WRITE_ONCE(amt->ready4, false);
 		WRITE_ONCE(amt->ready6, false);
-		amt->remote_ip = 0;
+		WRITE_ONCE(amt->remote_ip, 0);
 		amt_update_gw_status(amt, AMT_STATUS_INIT, false);
 		amt->req_cnt = 0;
 		amt->nonce = 0;
@@ -999,6 +1001,7 @@ static bool amt_send_membership_update(struct amt_dev *amt,
 				       struct sk_buff *skb,
 				       bool v6)
 {
+	__be32 remote_ip = READ_ONCE(amt->remote_ip);
 	struct amt_header_membership_update *amtmu;
 	struct iphdr *iph;
 	struct flowi4 fl4;
@@ -1018,13 +1021,13 @@ static bool amt_send_membership_update(struct amt_dev *amt,
 	skb_reset_inner_headers(skb);
 	memset(&fl4, 0, sizeof(struct flowi4));
 	fl4.flowi4_oif         = amt->stream_dev->ifindex;
-	fl4.daddr              = amt->remote_ip;
+	fl4.daddr              = remote_ip;
 	fl4.saddr              = amt->local_ip;
 	fl4.flowi4_dscp        = inet_dsfield_to_dscp(AMT_TOS);
 	fl4.flowi4_proto       = IPPROTO_UDP;
 	rt = ip_route_output_key(amt->net, &fl4);
 	if (IS_ERR(rt)) {
-		netdev_dbg(amt->dev, "no route to %pI4\n", &amt->remote_ip);
+		netdev_dbg(amt->dev, "no route to %pI4\n", &remote_ip);
 		return true;
 	}
 
@@ -2272,8 +2275,8 @@ static bool amt_advertisement_handler(struct amt_dev *amt, struct sk_buff *skb)
 	    amt->nonce != amta->nonce)
 		return true;
 
-	amt->remote_ip = amta->ip4;
-	netdev_dbg(amt->dev, "advertised remote ip = %pI4\n", &amt->remote_ip);
+	WRITE_ONCE(amt->remote_ip, amta->ip4);
+	netdev_dbg(amt->dev, "advertised remote ip = %pI4\n", &amta->ip4);
 	mod_delayed_work(amt_wq, &amt->req_wq, 0);
 
 	amt_update_gw_status(amt, AMT_STATUS_RECEIVED_ADVERTISEMENT, true);
@@ -2773,6 +2776,7 @@ static int amt_rcv(struct sock *sk, struct sk_buff *skb)
 {
 	struct amt_dev *amt;
 	struct iphdr *iph;
+	__be32 remote_ip;
 	int type;
 	bool err;
 
@@ -2783,6 +2787,7 @@ static int amt_rcv(struct sock *sk, struct sk_buff *skb)
 		kfree_skb(skb);
 		goto out;
 	}
+	remote_ip = READ_ONCE(amt->remote_ip);
 
 	skb->dev = amt->dev;
 	iph = ip_hdr(skb);
@@ -2807,7 +2812,7 @@ static int amt_rcv(struct sock *sk, struct sk_buff *skb)
 			}
 			goto out;
 		case AMT_MSG_MULTICAST_DATA:
-			if (iph->saddr != amt->remote_ip) {
+			if (iph->saddr != remote_ip) {
 				netdev_dbg(amt->dev, "Invalid Relay IP\n");
 				err = true;
 				goto drop;
@@ -2818,7 +2823,7 @@ static int amt_rcv(struct sock *sk, struct sk_buff *skb)
 			else
 				goto out;
 		case AMT_MSG_MEMBERSHIP_QUERY:
-			if (iph->saddr != amt->remote_ip) {
+			if (iph->saddr != remote_ip) {
 				netdev_dbg(amt->dev, "Invalid Relay IP\n");
 				err = true;
 				goto drop;
@@ -3000,7 +3005,7 @@ static int amt_dev_open(struct net_device *dev)
 		return err;
 
 	amt->req_cnt = 0;
-	amt->remote_ip = 0;
+	WRITE_ONCE(amt->remote_ip, 0);
 	amt->nonce = 0;
 	get_random_bytes(&amt->key, sizeof(siphash_key_t));
 
@@ -3045,7 +3050,7 @@ static int amt_dev_stop(struct net_device *dev)
 	amt->ready4 = false;
 	amt->ready6 = false;
 	amt->req_cnt = 0;
-	amt->remote_ip = 0;
+	WRITE_ONCE(amt->remote_ip, 0);
 
 	list_for_each_entry_safe(tunnel, tmp, &amt->tunnel_list, list) {
 		list_del_rcu(&tunnel->list);
@@ -3244,7 +3249,7 @@ static int amt_newlink(struct net_device *dev,
 					    "gateway port must not be 0");
 			goto err;
 		}
-		amt->remote_ip = 0;
+		WRITE_ONCE(amt->remote_ip, 0);
 		amt->discovery_ip = nla_get_in_addr(data[IFLA_AMT_DISCOVERY_IP]);
 		if (ipv4_is_loopback(amt->discovery_ip) ||
 		    ipv4_is_zeronet(amt->discovery_ip) ||
@@ -3308,8 +3313,10 @@ static size_t amt_get_size(const struct net_device *dev)
 
 static int amt_fill_info(struct sk_buff *skb, const struct net_device *dev)
 {
-	struct amt_dev *amt = netdev_priv(dev);
+	const struct amt_dev *amt = netdev_priv(dev);
+	__be32 remote_ip;
 
+	rcu_read_lock();
 	if (nla_put_u32(skb, IFLA_AMT_MODE, amt->mode))
 		goto nla_put_failure;
 	if (nla_put_be16(skb, IFLA_AMT_RELAY_PORT, amt->relay_port))
@@ -3322,15 +3329,19 @@ static int amt_fill_info(struct sk_buff *skb, const struct net_device *dev)
 		goto nla_put_failure;
 	if (nla_put_in_addr(skb, IFLA_AMT_DISCOVERY_IP, amt->discovery_ip))
 		goto nla_put_failure;
-	if (amt->remote_ip)
-		if (nla_put_in_addr(skb, IFLA_AMT_REMOTE_IP, amt->remote_ip))
+
+	remote_ip = READ_ONCE(amt->remote_ip);
+	if (remote_ip)
+		if (nla_put_in_addr(skb, IFLA_AMT_REMOTE_IP, remote_ip))
 			goto nla_put_failure;
 	if (nla_put_u32(skb, IFLA_AMT_MAX_TUNNELS, amt->max_tunnels))
 		goto nla_put_failure;
 
+	rcu_read_unlock();
 	return 0;
 
 nla_put_failure:
+	rcu_read_unlock();
 	return -EMSGSIZE;
 }
 
diff --git a/drivers/net/bareudp.c b/drivers/net/bareudp.c
index 5ef841c855269..c3b5ed52d8778 100644
--- a/drivers/net/bareudp.c
+++ b/drivers/net/bareudp.c
@@ -36,6 +36,7 @@ static unsigned int bareudp_net_id;
 
 struct bareudp_net {
 	struct list_head        bareudp_list;
+	struct mutex		lock;
 };
 
 struct bareudp_conf {
@@ -636,10 +637,15 @@ static struct bareudp_dev *bareudp_find_dev(struct bareudp_net *bn,
 {
 	struct bareudp_dev *bareudp, *t = NULL;
 
+	mutex_lock(&bn->lock);
+
 	list_for_each_entry(bareudp, &bn->bareudp_list, next) {
 		if (conf->port == bareudp->port)
 			t = bareudp;
 	}
+
+	mutex_unlock(&bn->lock);
+
 	return t;
 }
 
@@ -675,7 +681,10 @@ static int bareudp_configure(struct net *net, struct net_device *dev,
 	if (err)
 		return err;
 
+	mutex_lock(&bn->lock);
 	list_add(&bareudp->next, &bn->bareudp_list);
+	mutex_unlock(&bn->lock);
+
 	return 0;
 }
 
@@ -692,12 +701,26 @@ static int bareudp_link_config(struct net_device *dev,
 	return 0;
 }
 
+static void __bareudp_dellink(struct net *net, struct net_device *dev,
+			      struct list_head *head)
+{
+	struct bareudp_dev *bareudp = netdev_priv(dev);
+
+	list_del_init(&bareudp->next);
+	unregister_netdevice_queue_net(net, dev, head);
+}
+
 static void bareudp_dellink(struct net_device *dev, struct list_head *head)
 {
 	struct bareudp_dev *bareudp = netdev_priv(dev);
+	struct bareudp_net *bn;
 
-	list_del(&bareudp->next);
-	unregister_netdevice_queue(dev, head);
+	bn = net_generic(bareudp->net, bareudp_net_id);
+
+	mutex_lock(&bn->lock);
+	if (!list_empty(&bareudp->next))
+		__bareudp_dellink(dev_net(dev), dev, head);
+	mutex_unlock(&bn->lock);
 }
 
 static int bareudp_newlink(struct net_device *dev,
@@ -776,6 +799,8 @@ static __net_init int bareudp_init_net(struct net *net)
 	struct bareudp_net *bn = net_generic(net, bareudp_net_id);
 
 	INIT_LIST_HEAD(&bn->bareudp_list);
+	mutex_init(&bn->lock);
+
 	return 0;
 }
 
@@ -785,13 +810,25 @@ static void __net_exit bareudp_exit_rtnl_net(struct net *net,
 	struct bareudp_net *bn = net_generic(net, bareudp_net_id);
 	struct bareudp_dev *bareudp, *next;
 
+	mutex_lock(&bn->lock);
+
 	list_for_each_entry_safe(bareudp, next, &bn->bareudp_list, next)
-		bareudp_dellink(bareudp->dev, dev_kill_list);
+		__bareudp_dellink(net, bareudp->dev, dev_kill_list);
+
+	mutex_unlock(&bn->lock);
+}
+
+static void __net_exit bareudp_exit_net(struct net *net)
+{
+	struct bareudp_net *bn = net_generic(net, bareudp_net_id);
+
+	WARN_ON_ONCE(!list_empty(&bn->bareudp_list));
 }
 
 static struct pernet_operations bareudp_net_ops = {
 	.init = bareudp_init_net,
 	.exit_rtnl = bareudp_exit_rtnl_net,
+	.exit = bareudp_exit_net,
 	.id   = &bareudp_net_id,
 	.size = sizeof(struct bareudp_net),
 };
diff --git a/drivers/net/bonding/bond_3ad.c b/drivers/net/bonding/bond_3ad.c
index acbba08dbdfad..b8e4b4d68dd65 100644
--- a/drivers/net/bonding/bond_3ad.c
+++ b/drivers/net/bonding/bond_3ad.c
@@ -760,14 +760,14 @@ static int __agg_usable_ports(struct aggregator *agg)
 	return valid;
 }
 
-static int __agg_active_ports(struct aggregator *agg)
+static int __agg_active_ports(const struct aggregator *agg)
 {
-	struct port *port;
+	const struct port *port;
 	int active = 0;
 
 	for (port = agg->lag_ports; port;
 	     port = port->next_port_in_aggregator) {
-		if (port->is_enabled)
+		if (READ_ONCE(port->is_enabled))
 			active++;
 	}
 
@@ -2801,11 +2801,11 @@ void bond_3ad_handle_link_change(struct slave *slave, char link)
 	 * some of he adaptors(ce1000.lan) report.
 	 */
 	if (link == BOND_LINK_UP) {
-		port->is_enabled = true;
+		WRITE_ONCE(port->is_enabled, true);
 		ad_update_actor_keys(port, false);
 	} else {
 		/* link has failed */
-		port->is_enabled = false;
+		WRITE_ONCE(port->is_enabled, false);
 		ad_update_actor_keys(port, true);
 	}
 	agg = __get_first_agg(port);
@@ -2878,16 +2878,20 @@ int bond_3ad_set_carrier(struct bonding *bond)
  * Returns:   0 on success
  *          < 0 on error
  */
-int __bond_3ad_get_active_agg_info(struct bonding *bond,
+int __bond_3ad_get_active_agg_info(const struct bonding *bond,
 				   struct ad_info *ad_info)
 {
-	struct aggregator *aggregator = NULL, *tmp;
+	const struct aggregator *aggregator = NULL, *tmp;
+	struct ad_slave_info *ad_slave_info;
+	const struct port *port;
 	struct list_head *iter;
 	struct slave *slave;
-	struct port *port;
 
 	bond_for_each_slave_rcu(bond, slave, iter) {
-		port = &(SLAVE_AD_INFO(slave)->port);
+		ad_slave_info = SLAVE_AD_INFO(slave);
+		if (!ad_slave_info)
+			continue;
+		port = &ad_slave_info->port;
 		tmp = rcu_dereference(port->aggregator);
 		if (tmp && tmp->is_active) {
 			aggregator = tmp;
@@ -2907,7 +2911,7 @@ int __bond_3ad_get_active_agg_info(struct bonding *bond,
 	return 0;
 }
 
-int bond_3ad_get_active_agg_info(struct bonding *bond, struct ad_info *ad_info)
+int bond_3ad_get_active_agg_info(const struct bonding *bond, struct ad_info *ad_info)
 {
 	int ret;
 
diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
index e044fc733b8c0..e1bec81642217 100644
--- a/drivers/net/bonding/bond_main.c
+++ b/drivers/net/bonding/bond_main.c
@@ -4857,12 +4857,6 @@ static int bond_set_mac_address(struct net_device *bond_dev, void *addr)
 			  __func__, slave);
 		res = dev_set_mac_address(slave->dev, addr, NULL);
 		if (res) {
-			/* TODO: consider downing the slave
-			 * and retry ?
-			 * User should expect communications
-			 * breakage anyway until ARP finish
-			 * updating, so...
-			 */
 			slave_dbg(bond_dev, slave->dev, "%s: err %d\n",
 				  __func__, res);
 			goto unwind;
diff --git a/drivers/net/bonding/bond_netlink.c b/drivers/net/bonding/bond_netlink.c
index 4a11572f663d3..55d2f8a539d46 100644
--- a/drivers/net/bonding/bond_netlink.c
+++ b/drivers/net/bonding/bond_netlink.c
@@ -686,53 +686,58 @@ static size_t bond_get_size(const struct net_device *bond_dev)
 		0;
 }
 
-static int bond_option_active_slave_get_ifindex(struct bonding *bond)
+static int bond_option_active_slave_get_ifindex_rcu(const struct bonding *bond)
 {
-	const struct net_device *slave;
-	int ifindex;
+	const struct net_device *dev = NULL;
+	const struct slave *slave;
 
-	rcu_read_lock();
-	slave = bond_option_active_slave_get_rcu(bond);
-	ifindex = slave ? slave->ifindex : 0;
-	rcu_read_unlock();
-	return ifindex;
+	slave = rcu_dereference(bond->curr_active_slave);
+	if (slave)
+		dev = slave->dev;
+	return dev ? dev->ifindex : 0;
 }
 
 static int bond_fill_info(struct sk_buff *skb,
 			  const struct net_device *bond_dev)
 {
-	struct bonding *bond = netdev_priv(bond_dev);
-	unsigned int packets_per_slave;
-	int ifindex, i, targets_added;
+	const struct bonding *bond = netdev_priv(bond_dev);
+	int i, targets_added, miimon, mode;
+	const struct slave *primary;
 	struct nlattr *targets;
-	struct slave *primary;
 
-	if (nla_put_u8(skb, IFLA_BOND_MODE, BOND_MODE(bond)))
+	rcu_read_lock();
+	mode = READ_ONCE(bond->params.mode);
+	if (nla_put_u8(skb, IFLA_BOND_MODE, mode))
 		goto nla_put_failure;
 
-	ifindex = bond_option_active_slave_get_ifindex(bond);
-	if (ifindex && nla_put_u32(skb, IFLA_BOND_ACTIVE_SLAVE, ifindex))
-		goto nla_put_failure;
+	if (bond_mode_uses_primary(mode)) {
+		int ifindex = bond_option_active_slave_get_ifindex_rcu(bond);
+
+		if (ifindex && nla_put_u32(skb, IFLA_BOND_ACTIVE_SLAVE, ifindex))
+			goto nla_put_failure;
+	}
 
-	if (nla_put_u32(skb, IFLA_BOND_MIIMON, bond->params.miimon))
+	miimon = READ_ONCE(bond->params.miimon);
+	if (nla_put_u32(skb, IFLA_BOND_MIIMON, miimon))
 		goto nla_put_failure;
 
 	if (nla_put_u32(skb, IFLA_BOND_UPDELAY,
-			bond->params.updelay * bond->params.miimon))
+			READ_ONCE(bond->params.updelay) * miimon))
 		goto nla_put_failure;
 
 	if (nla_put_u32(skb, IFLA_BOND_DOWNDELAY,
-			bond->params.downdelay * bond->params.miimon))
+			READ_ONCE(bond->params.downdelay) * miimon))
 		goto nla_put_failure;
 
 	if (nla_put_u32(skb, IFLA_BOND_PEER_NOTIF_DELAY,
-			bond->params.peer_notif_delay * bond->params.miimon))
+			READ_ONCE(bond->params.peer_notif_delay) * miimon))
 		goto nla_put_failure;
 
 	if (nla_put_u8(skb, IFLA_BOND_USE_CARRIER, 1))
 		goto nla_put_failure;
 
-	if (nla_put_u32(skb, IFLA_BOND_ARP_INTERVAL, bond->params.arp_interval))
+	if (nla_put_u32(skb, IFLA_BOND_ARP_INTERVAL,
+			READ_ONCE(bond->params.arp_interval)))
 		goto nla_put_failure;
 
 	targets = nla_nest_start_noflag(skb, IFLA_BOND_ARP_IP_TARGET);
@@ -741,8 +746,10 @@ static int bond_fill_info(struct sk_buff *skb,
 
 	targets_added = 0;
 	for (i = 0; i < BOND_MAX_ARP_TARGETS; i++) {
-		if (bond->params.arp_targets[i]) {
-			if (nla_put_be32(skb, i, bond->params.arp_targets[i]))
+		__be32 t = READ_ONCE(bond->params.arp_targets[i]);
+
+		if (t) {
+			if (nla_put_be32(skb, i, t))
 				goto nla_put_failure;
 			targets_added = 1;
 		}
@@ -753,11 +760,12 @@ static int bond_fill_info(struct sk_buff *skb,
 	else
 		nla_nest_cancel(skb, targets);
 
-	if (nla_put_u32(skb, IFLA_BOND_ARP_VALIDATE, bond->params.arp_validate))
+	if (nla_put_u32(skb, IFLA_BOND_ARP_VALIDATE,
+			READ_ONCE(bond->params.arp_validate)))
 		goto nla_put_failure;
 
 	if (nla_put_u32(skb, IFLA_BOND_ARP_ALL_TARGETS,
-			bond->params.arp_all_targets))
+			READ_ONCE(bond->params.arp_all_targets)))
 		goto nla_put_failure;
 
 #if IS_ENABLED(CONFIG_IPV6)
@@ -767,6 +775,9 @@ static int bond_fill_info(struct sk_buff *skb,
 
 	targets_added = 0;
 	for (i = 0; i < BOND_MAX_NS_TARGETS; i++) {
+		/* Note: IPv6 addresses can not be read in an atomic READ_ONCE() yet.
+		 * We accept this minor race for the moment.
+		 */
 		if (!ipv6_addr_any(&bond->params.ns_targets[i])) {
 			if (nla_put_in6_addr(skb, i, &bond->params.ns_targets[i]))
 				goto nla_put_failure;
@@ -780,97 +791,97 @@ static int bond_fill_info(struct sk_buff *skb,
 		nla_nest_cancel(skb, targets);
 #endif
 
-	primary = rtnl_dereference(bond->primary_slave);
+	primary = rcu_dereference(bond->primary_slave);
 	if (primary &&
 	    nla_put_u32(skb, IFLA_BOND_PRIMARY, primary->dev->ifindex))
 		goto nla_put_failure;
 
 	if (nla_put_u8(skb, IFLA_BOND_PRIMARY_RESELECT,
-		       bond->params.primary_reselect))
+		       READ_ONCE(bond->params.primary_reselect)))
 		goto nla_put_failure;
 
 	if (nla_put_u8(skb, IFLA_BOND_FAIL_OVER_MAC,
-		       bond->params.fail_over_mac))
+		       READ_ONCE(bond->params.fail_over_mac)))
 		goto nla_put_failure;
 
 	if (nla_put_u8(skb, IFLA_BOND_XMIT_HASH_POLICY,
-		       bond->params.xmit_policy))
+		       READ_ONCE(bond->params.xmit_policy)))
 		goto nla_put_failure;
 
 	if (nla_put_u32(skb, IFLA_BOND_RESEND_IGMP,
-			bond->params.resend_igmp))
+			READ_ONCE(bond->params.resend_igmp)))
 		goto nla_put_failure;
 
 	if (nla_put_u8(skb, IFLA_BOND_NUM_PEER_NOTIF,
-		       bond->params.num_peer_notif))
+		       READ_ONCE(bond->params.num_peer_notif)))
 		goto nla_put_failure;
 
 	if (nla_put_u8(skb, IFLA_BOND_ALL_SLAVES_ACTIVE,
-		       bond->params.all_slaves_active))
+		       READ_ONCE(bond->params.all_slaves_active)))
 		goto nla_put_failure;
 
 	if (nla_put_u32(skb, IFLA_BOND_MIN_LINKS,
-			bond->params.min_links))
+			READ_ONCE(bond->params.min_links)))
 		goto nla_put_failure;
 
 	if (nla_put_u32(skb, IFLA_BOND_LP_INTERVAL,
-			bond->params.lp_interval))
+			READ_ONCE(bond->params.lp_interval)))
 		goto nla_put_failure;
 
-	packets_per_slave = bond->params.packets_per_slave;
 	if (nla_put_u32(skb, IFLA_BOND_PACKETS_PER_SLAVE,
-			packets_per_slave))
+			READ_ONCE(bond->params.packets_per_slave)))
 		goto nla_put_failure;
 
 	if (nla_put_u8(skb, IFLA_BOND_AD_LACP_ACTIVE,
-		       bond->params.lacp_active))
+		       READ_ONCE(bond->params.lacp_active)))
 		goto nla_put_failure;
 
 	if (nla_put_u8(skb, IFLA_BOND_AD_LACP_RATE,
-		       bond->params.lacp_fast))
+		       READ_ONCE(bond->params.lacp_fast)))
 		goto nla_put_failure;
 
 	if (nla_put_u8(skb, IFLA_BOND_AD_SELECT,
-		       bond->params.ad_select))
+		       READ_ONCE(bond->params.ad_select)))
 		goto nla_put_failure;
 
 	if (nla_put_u8(skb, IFLA_BOND_TLB_DYNAMIC_LB,
-		       bond->params.tlb_dynamic_lb))
+		       READ_ONCE(bond->params.tlb_dynamic_lb)))
 		goto nla_put_failure;
 
 	if (nla_put_u8(skb, IFLA_BOND_MISSED_MAX,
-		       bond->params.missed_max))
+		       READ_ONCE(bond->params.missed_max)))
 		goto nla_put_failure;
 
 	if (nla_put_u8(skb, IFLA_BOND_COUPLED_CONTROL,
-		       bond->params.coupled_control))
+		       READ_ONCE(bond->params.coupled_control)))
 		goto nla_put_failure;
 
 	if (nla_put_u8(skb, IFLA_BOND_BROADCAST_NEIGH,
-		       bond->params.broadcast_neighbor))
+		       READ_ONCE(bond->params.broadcast_neighbor)))
 		goto nla_put_failure;
 
 	if (nla_put_u8(skb, IFLA_BOND_LACP_STRICT,
-		       bond->params.lacp_strict))
+		       READ_ONCE(bond->params.lacp_strict)))
 		goto nla_put_failure;
 
-	if (BOND_MODE(bond) == BOND_MODE_8023AD) {
+	if (mode == BOND_MODE_8023AD) {
 		struct ad_info info;
 
 		if (capable(CAP_NET_ADMIN)) {
 			if (nla_put_u16(skb, IFLA_BOND_AD_ACTOR_SYS_PRIO,
-					bond->params.ad_actor_sys_prio))
+					READ_ONCE(bond->params.ad_actor_sys_prio)))
 				goto nla_put_failure;
 
 			if (nla_put_u16(skb, IFLA_BOND_AD_USER_PORT_KEY,
-					bond->params.ad_user_port_key))
+					READ_ONCE(bond->params.ad_user_port_key)))
 				goto nla_put_failure;
 
+			/* Small race here, this is a minor trade off. */
 			if (nla_put(skb, IFLA_BOND_AD_ACTOR_SYSTEM,
 				    ETH_ALEN, &bond->params.ad_actor_system))
 				goto nla_put_failure;
 		}
-		if (!bond_3ad_get_active_agg_info(bond, &info)) {
+		if (!__bond_3ad_get_active_agg_info(bond, &info)) {
 			struct nlattr *nest;
 
 			nest = nla_nest_start_noflag(skb, IFLA_BOND_AD_INFO);
@@ -898,9 +909,11 @@ static int bond_fill_info(struct sk_buff *skb,
 		}
 	}
 
+	rcu_read_unlock();
 	return 0;
 
 nla_put_failure:
+	rcu_read_unlock();
 	return -EMSGSIZE;
 }
 
diff --git a/drivers/net/bonding/bond_options.c b/drivers/net/bonding/bond_options.c
index e590c8dee86e1..36b8d89387ee5 100644
--- a/drivers/net/bonding/bond_options.c
+++ b/drivers/net/bonding/bond_options.c
@@ -934,14 +934,14 @@ static int bond_option_mode_set(struct bonding *bond,
 
 	/* don't cache arp_validate between modes */
 	WRITE_ONCE(bond->params.arp_validate, BOND_ARP_VALIDATE_NONE);
-	bond->params.mode = newval->value;
+	WRITE_ONCE(bond->params.mode, newval->value);
 
 	/* When changing mode, the bond device is down, we may reduce
 	 * the bond_bcast_neigh_enabled in bond_close() if broadcast_neighbor
 	 * enabled in 8023ad mode. Therefore, only clear broadcast_neighbor
 	 * to 0.
 	 */
-	bond->params.broadcast_neighbor = 0;
+	WRITE_ONCE(bond->params.broadcast_neighbor, 0);
 
 	if (bond->dev->reg_state == NETREG_REGISTERED) {
 		bool update = false;
@@ -1706,7 +1706,7 @@ static int bond_option_lacp_strict_set(struct bonding *bond,
 {
 	netdev_dbg(bond->dev, "Setting LACP fallback to %s (%llu)\n",
 		   newval->string, newval->value);
-	bond->params.lacp_strict = newval->value;
+	WRITE_ONCE(bond->params.lacp_strict, newval->value);
 	bond_3ad_set_carrier(bond);
 
 	return 0;
@@ -1927,7 +1927,7 @@ static int bond_option_broadcast_neigh_set(struct bonding *bond,
 	if (bond->params.broadcast_neighbor == newval->value)
 		return 0;
 
-	bond->params.broadcast_neighbor = newval->value;
+	WRITE_ONCE(bond->params.broadcast_neighbor, newval->value);
 	if (bond->dev->flags & IFF_UP) {
 		if (bond->params.broadcast_neighbor)
 			static_branch_inc(&bond_bcast_neigh_enabled);
diff --git a/drivers/net/can/Kconfig b/drivers/net/can/Kconfig
index e4058708ae683..a8fad6fe53020 100644
--- a/drivers/net/can/Kconfig
+++ b/drivers/net/can/Kconfig
@@ -40,11 +40,8 @@ config CAN_VXCAN
 	  When one end receives the packet it appears on its pair and vice
 	  versa. The vxcan can be used for cross namespace communication.
 
-	  In opposite to vcan loopback devices the vxcan only forwards CAN
-	  frames to its pair and does *not* provide a local echo of sent
-	  CAN frames. To disable a potential echo in af_can.c the vxcan driver
-	  announces IFF_ECHO in the interface flags. To have a clean start
-	  in each namespace the CAN GW hop counter is set to zero.
+	  To have a clean start in each namespace the CAN GW hop counter is
+	  set to zero.
 
 	  This driver can also be built as a module.  If so, the module
 	  will be called vxcan.
diff --git a/drivers/net/can/peak_canfd/peak_canfd.c b/drivers/net/can/peak_canfd/peak_canfd.c
index 06cb2629f66ab..4fd1aefb780fd 100644
--- a/drivers/net/can/peak_canfd/peak_canfd.c
+++ b/drivers/net/can/peak_canfd/peak_canfd.c
@@ -2,7 +2,7 @@
 /* Copyright (C) 2007, 2011 Wolfgang Grandegger <wg@grandegger.com>
  *
  * Copyright (C) 2016-2025 PEAK System-Technik GmbH
- * Author: Stéphane Grosjean <stephane.grosjean@hms-networks.com>
+ * Author: Stéphane Grosjean <s.grosjean@peak-system.fr>
  */
 
 #include <linux/can.h>
diff --git a/drivers/net/can/peak_canfd/peak_canfd_user.h b/drivers/net/can/peak_canfd/peak_canfd_user.h
index 60c6542028cf4..dc0ecb566a85a 100644
--- a/drivers/net/can/peak_canfd/peak_canfd_user.h
+++ b/drivers/net/can/peak_canfd/peak_canfd_user.h
@@ -2,7 +2,7 @@
 /* CAN driver for PEAK System micro-CAN based adapters
  *
  * Copyright (C) 2003-2025 PEAK System-Technik GmbH
- * Author: Stéphane Grosjean <stephane.grosjean@hms-networks.com>
+ * Author: Stéphane Grosjean <s.grosjean@peak-system.fr>
  */
 #ifndef PEAK_CANFD_USER_H
 #define PEAK_CANFD_USER_H
diff --git a/drivers/net/can/peak_canfd/peak_pciefd_main.c b/drivers/net/can/peak_canfd/peak_pciefd_main.c
index 93558e33bc02b..7c749301ea84c 100644
--- a/drivers/net/can/peak_canfd/peak_pciefd_main.c
+++ b/drivers/net/can/peak_canfd/peak_pciefd_main.c
@@ -4,7 +4,7 @@
  * Derived from the PCAN project file driver/src/pcan_pci.c:
  *
  * Copyright (C) 2001-2025 PEAK System-Technik GmbH
- * Author: Stéphane Grosjean <stephane.grosjean@hms-networks.com>
+ * Author: Stéphane Grosjean <s.grosjean@peak-system.fr>
  */
 
 #include <linux/kernel.h>
@@ -19,7 +19,7 @@
 
 #include "peak_canfd_user.h"
 
-MODULE_AUTHOR("Stéphane Grosjean <stephane.grosjean@hms-networks.com>");
+MODULE_AUTHOR("Stéphane Grosjean <s.grosjean@peak-system.fr>");
 MODULE_DESCRIPTION("Socket-CAN driver for PEAK PCAN PCIe/M.2 FD family cards");
 MODULE_LICENSE("GPL v2");
 
diff --git a/drivers/net/can/sja1000/peak_pci.c b/drivers/net/can/sja1000/peak_pci.c
index 4cc4a1581dd11..69c61ccf621df 100644
--- a/drivers/net/can/sja1000/peak_pci.c
+++ b/drivers/net/can/sja1000/peak_pci.c
@@ -5,7 +5,7 @@
  * Derived from the PCAN project file driver/src/pcan_pci.c:
  *
  * Copyright (C) 2001-2025 PEAK System-Technik GmbH
- * Author: Stéphane Grosjean <stephane.grosjean@hms-networks.com>
+ * Author: Stéphane Grosjean <s.grosjean@peak-system.fr>
  */
 
 #include <linux/kernel.h>
@@ -22,7 +22,7 @@
 
 #include "sja1000.h"
 
-MODULE_AUTHOR("Stéphane Grosjean <stephane.grosjean@hms-networks.com>");
+MODULE_AUTHOR("Stéphane Grosjean <s.grosjean@peak-system.fr>");
 MODULE_DESCRIPTION("Socket-CAN driver for PEAK PCAN PCI family cards");
 MODULE_LICENSE("GPL v2");
 
diff --git a/drivers/net/can/sja1000/peak_pcmcia.c b/drivers/net/can/sja1000/peak_pcmcia.c
index 42a77d435b392..c3c2aa21da47b 100644
--- a/drivers/net/can/sja1000/peak_pcmcia.c
+++ b/drivers/net/can/sja1000/peak_pcmcia.c
@@ -4,7 +4,7 @@
  * Derived from the PCAN project file driver/src/pcan_pccard.c
  *
  * Copyright (C) 2006-2025 PEAK System-Technik GmbH
- * Author: Stéphane Grosjean <stephane.grosjean@hms-networks.com>
+ * Author: Stéphane Grosjean <s.grosjean@peak-system.fr>
  */
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -19,7 +19,7 @@
 #include <linux/can/dev.h>
 #include "sja1000.h"
 
-MODULE_AUTHOR("Stéphane Grosjean <stephane.grosjean@hms-networks.com>");
+MODULE_AUTHOR("Stéphane Grosjean <s.grosjean@peak-system.fr>");
 MODULE_DESCRIPTION("CAN driver for PEAK-System PCAN-PC Cards");
 MODULE_LICENSE("GPL v2");
 
diff --git a/drivers/net/can/usb/esd_usb.c b/drivers/net/can/usb/esd_usb.c
index d257440fa01ff..f41d4a0d140f7 100644
--- a/drivers/net/can/usb/esd_usb.c
+++ b/drivers/net/can/usb/esd_usb.c
@@ -1390,10 +1390,13 @@ static void esd_usb_disconnect(struct usb_interface *intf)
 				netdev = dev->nets[i]->netdev;
 				netdev_info(netdev, "unregister\n");
 				unregister_netdev(netdev);
-				free_candev(netdev);
 			}
 		}
 		unlink_all_urbs(dev);
+		for (i = 0; i < dev->net_count; i++) {
+			if (dev->nets[i])
+				free_candev(dev->nets[i]->netdev);
+		}
 		kfree(dev);
 	}
 }
diff --git a/drivers/net/can/usb/peak_usb/pcan_usb.c b/drivers/net/can/usb/peak_usb/pcan_usb.c
index 9278a1522aae5..8fd058c328564 100644
--- a/drivers/net/can/usb/peak_usb/pcan_usb.c
+++ b/drivers/net/can/usb/peak_usb/pcan_usb.c
@@ -4,7 +4,7 @@
  * Derived from the PCAN project file driver/src/pcan_usb.c
  *
  * Copyright (C) 2003-2025 PEAK System-Technik GmbH
- * Author: Stéphane Grosjean <stephane.grosjean@hms-networks.com>
+ * Author: Stéphane Grosjean <s.grosjean@peak-system.fr>
  *
  * Many thanks to Klaus Hitschler <klaus.hitschler@gmx.de>
  */
diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.c b/drivers/net/can/usb/peak_usb/pcan_usb_core.c
index cf48bb26d46d2..c7933d1acc99a 100644
--- a/drivers/net/can/usb/peak_usb/pcan_usb_core.c
+++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.c
@@ -4,7 +4,7 @@
  * Derived from the PCAN project file driver/src/pcan_usb_core.c
  *
  * Copyright (C) 2003-2025 PEAK System-Technik GmbH
- * Author: Stéphane Grosjean <stephane.grosjean@hms-networks.com>
+ * Author: Stéphane Grosjean <s.grosjean@peak-system.fr>
  *
  * Many thanks to Klaus Hitschler <klaus.hitschler@gmx.de>
  */
@@ -24,7 +24,7 @@
 
 #include "pcan_usb_core.h"
 
-MODULE_AUTHOR("Stéphane Grosjean <stephane.grosjean@hms-networks.com>");
+MODULE_AUTHOR("Stéphane Grosjean <s.grosjean@peak-system.fr>");
 MODULE_DESCRIPTION("CAN driver for PEAK-System USB adapters");
 MODULE_LICENSE("GPL v2");
 
diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.h b/drivers/net/can/usb/peak_usb/pcan_usb_core.h
index d1c1897d47b9c..65999f04f4b76 100644
--- a/drivers/net/can/usb/peak_usb/pcan_usb_core.h
+++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.h
@@ -4,7 +4,7 @@
  * Derived from the PCAN project file driver/src/pcan_usb_core.c
  *
  * Copyright (C) 2003-2025 PEAK System-Technik GmbH
- * Author: Stéphane Grosjean <stephane.grosjean@hms-networks.com>
+ * Author: Stéphane Grosjean <s.grosjean@peak-system.fr>
  *
  * Many thanks to Klaus Hitschler <klaus.hitschler@gmx.de>
  */
diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_fd.c b/drivers/net/can/usb/peak_usb/pcan_usb_fd.c
index eb4f5884ad735..ef9fd693e9bd3 100644
--- a/drivers/net/can/usb/peak_usb/pcan_usb_fd.c
+++ b/drivers/net/can/usb/peak_usb/pcan_usb_fd.c
@@ -3,7 +3,7 @@
  * CAN driver for PEAK System PCAN-USB FD / PCAN-USB Pro FD adapter
  *
  * Copyright (C) 2013-2025 PEAK System-Technik GmbH
- * Author: Stéphane Grosjean <stephane.grosjean@hms-networks.com>
+ * Author: Stéphane Grosjean <s.grosjean@peak-system.fr>
  */
 #include <linux/ethtool.h>
 #include <linux/module.h>
diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_pro.c b/drivers/net/can/usb/peak_usb/pcan_usb_pro.c
index 4bfa8d0fbb32f..aefcded8e12a8 100644
--- a/drivers/net/can/usb/peak_usb/pcan_usb_pro.c
+++ b/drivers/net/can/usb/peak_usb/pcan_usb_pro.c
@@ -4,7 +4,7 @@
  * Derived from the PCAN project file driver/src/pcan_usbpro.c
  *
  * Copyright (C) 2003-2025 PEAK System-Technik GmbH
- * Author: Stéphane Grosjean <stephane.grosjean@hms-networks.com>
+ * Author: Stéphane Grosjean <s.grosjean@peak-system.fr>
  */
 #include <linux/ethtool.h>
 #include <linux/module.h>
diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_pro.h b/drivers/net/can/usb/peak_usb/pcan_usb_pro.h
index 162c7546d3a83..d669c9e610c7e 100644
--- a/drivers/net/can/usb/peak_usb/pcan_usb_pro.h
+++ b/drivers/net/can/usb/peak_usb/pcan_usb_pro.h
@@ -4,7 +4,7 @@
  * Derived from the PCAN project file driver/src/pcan_usbpro_fw.h
  *
  * Copyright (C) 2003-2025 PEAK System-Technik GmbH
- * Author: Stéphane Grosjean <stephane.grosjean@hms-networks.com>
+ * Author: Stéphane Grosjean <s.grosjean@peak-system.fr>
  */
 #ifndef PCAN_USB_PRO_H
 #define PCAN_USB_PRO_H
diff --git a/drivers/net/dsa/b53/b53_priv.h b/drivers/net/dsa/b53/b53_priv.h
index cd27a7344e894..29cca7945df71 100644
--- a/drivers/net/dsa/b53/b53_priv.h
+++ b/drivers/net/dsa/b53/b53_priv.h
@@ -23,6 +23,7 @@
 #include <linux/mutex.h>
 #include <linux/phylink.h>
 #include <linux/etherdevice.h>
+#include <linux/gpio/consumer.h>
 #include <net/dsa.h>
 
 #include "b53_regs.h"
@@ -467,7 +468,7 @@ static inline void b53_arl_search_read(struct b53_device *dev, u8 idx,
 #ifdef CONFIG_BCM47XX
 
 #include <linux/bcm47xx_nvram.h>
-#include <linux/gpio.h>
+#include <linux/gpio/legacy.h>
 #include <bcm47xx_board.h>
 static inline struct gpio_desc *b53_switch_get_reset_gpio(struct b53_device *dev)
 {
diff --git a/drivers/net/dsa/microchip/ksz8.c b/drivers/net/dsa/microchip/ksz8.c
index 138f2ab0774eb..c4c769028a20c 100644
--- a/drivers/net/dsa/microchip/ksz8.c
+++ b/drivers/net/dsa/microchip/ksz8.c
@@ -18,7 +18,7 @@
 #include <linux/delay.h>
 #include <linux/dsa/ksz_common.h>
 #include <linux/export.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/if_vlan.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -36,6 +36,52 @@
 #include "ksz8_reg.h"
 #include "ksz8.h"
 
+/* ksz88x3_drive_strengths - Drive strength mapping for KSZ8863, KSZ8873, ..
+ *			     variants.
+ * This values are documented in KSZ8873 and KSZ8863 datasheets.
+ */
+static const struct ksz_drive_strength ksz88x3_drive_strengths[] = {
+	{ 0,  8000 },
+	{ KSZ8873_DRIVE_STRENGTH_16MA, 16000 },
+};
+
+struct ksz88xx_stats_raw {
+	u64 rx;
+	u64 rx_hi;
+	u64 rx_undersize;
+	u64 rx_fragments;
+	u64 rx_oversize;
+	u64 rx_jabbers;
+	u64 rx_symbol_err;
+	u64 rx_crc_err;
+	u64 rx_align_err;
+	u64 rx_mac_ctrl;
+	u64 rx_pause;
+	u64 rx_bcast;
+	u64 rx_mcast;
+	u64 rx_ucast;
+	u64 rx_64_or_less;
+	u64 rx_65_127;
+	u64 rx_128_255;
+	u64 rx_256_511;
+	u64 rx_512_1023;
+	u64 rx_1024_1522;
+	u64 tx;
+	u64 tx_hi;
+	u64 tx_late_col;
+	u64 tx_pause;
+	u64 tx_bcast;
+	u64 tx_mcast;
+	u64 tx_ucast;
+	u64 tx_deferred;
+	u64 tx_total_col;
+	u64 tx_exc_col;
+	u64 tx_single_col;
+	u64 tx_mult_col;
+	u64 rx_discards;
+	u64 tx_discards;
+};
+
 static void ksz_cfg(struct ksz_device *dev, u32 addr, u8 bits, bool set)
 {
 	ksz_rmw8(dev, addr, bits, set ? bits : 0);
@@ -158,10 +204,17 @@ static int ksz8_reset_switch(struct ksz_device *dev)
 	return 0;
 }
 
-static int ksz8863_change_mtu(struct ksz_device *dev, int frame_size)
+static int ksz88xx_change_mtu(struct dsa_switch *ds, int port, int mtu)
 {
+	struct ksz_device *dev = ds->priv;
+	int frame_size;
 	u8 ctrl2 = 0;
 
+	if (!dsa_is_cpu_port(dev->ds, port))
+		return 0;
+
+	frame_size = mtu + VLAN_ETH_HLEN + ETH_FCS_LEN;
+
 	if (frame_size <= KSZ8_LEGAL_PACKET_SIZE)
 		ctrl2 |= KSZ8863_LEGAL_PACKET_ENABLE;
 	else if (frame_size > KSZ8863_NORMAL_PACKET_SIZE)
@@ -171,11 +224,18 @@ static int ksz8863_change_mtu(struct ksz_device *dev, int frame_size)
 			KSZ8863_HUGE_PACKET_ENABLE, ctrl2);
 }
 
-static int ksz8795_change_mtu(struct ksz_device *dev, int frame_size)
+static int ksz87xx_change_mtu(struct dsa_switch *ds, int port, int mtu)
 {
+	struct ksz_device *dev = ds->priv;
 	u8 ctrl1 = 0, ctrl2 = 0;
+	u16 frame_size;
 	int ret;
 
+	if (!dsa_is_cpu_port(dev->ds, port))
+		return 0;
+
+	frame_size = mtu + VLAN_ETH_HLEN + ETH_FCS_LEN;
+
 	if (frame_size > KSZ8_LEGAL_PACKET_SIZE)
 		ctrl2 |= SW_LEGAL_PACKET_DISABLE;
 	if (frame_size > KSZ8863_NORMAL_PACKET_SIZE)
@@ -188,29 +248,14 @@ static int ksz8795_change_mtu(struct ksz_device *dev, int frame_size)
 	return ksz_rmw8(dev, REG_SW_CTRL_2, SW_LEGAL_PACKET_DISABLE, ctrl2);
 }
 
-static int ksz8_change_mtu(struct dsa_switch *ds, int port, int mtu)
+static int ksz87xx_max_mtu(struct dsa_switch *ds, int port)
 {
-	struct ksz_device *dev = ds->priv;
-	u16 frame_size;
-
-	if (!dsa_is_cpu_port(dev->ds, port))
-		return 0;
-
-	frame_size = mtu + VLAN_ETH_HLEN + ETH_FCS_LEN;
-
-	switch (dev->chip_id) {
-	case KSZ8795_CHIP_ID:
-	case KSZ8794_CHIP_ID:
-	case KSZ8765_CHIP_ID:
-		return ksz8795_change_mtu(dev, frame_size);
-	case KSZ8463_CHIP_ID:
-	case KSZ88X3_CHIP_ID:
-	case KSZ8864_CHIP_ID:
-	case KSZ8895_CHIP_ID:
-		return ksz8863_change_mtu(dev, frame_size);
-	}
+	return KSZ8795_HUGE_PACKET_SIZE - VLAN_ETH_HLEN - ETH_FCS_LEN;
+}
 
-	return -EOPNOTSUPP;
+static int ksz88xx_max_mtu(struct dsa_switch *ds, int port)
+{
+	return KSZ8863_HUGE_PACKET_SIZE - VLAN_ETH_HLEN - ETH_FCS_LEN;
 }
 
 static int ksz8_port_queue_split(struct ksz_device *dev, int port, int queues)
@@ -1746,6 +1791,159 @@ static void ksz8_port_mirror_del(struct dsa_switch *ds, int port,
 			     PORT_MIRROR_SNIFFER, false);
 }
 
+static u8 ksz8463_tc_ctrl(int port, int queue)
+{
+	u8 reg;
+
+	reg = 0xC8 + port * 4;
+	reg += ((3 - queue) / 2) * 2;
+	reg++;
+	reg -= (queue & 1);
+	return reg;
+}
+
+/**
+ * ksz88x3_tc_ets_add - Configure ETS (Enhanced Transmission Selection)
+ *                      for a port on KSZ88x3 switch
+ * @dev: Pointer to the KSZ switch device structure
+ * @port: Port number to configure
+ * @p: Pointer to offload replace parameters describing ETS bands and mapping
+ *
+ * The KSZ88x3 supports two scheduling modes: Strict Priority and
+ * Weighted Fair Queuing (WFQ). Both modes have fixed behavior:
+ *   - No configurable queue-to-priority mapping
+ *   - No weight adjustment in WFQ mode
+ *
+ * This function configures the switch to use strict priority mode by
+ * clearing the WFQ enable bit for all queues associated with ETS bands.
+ * If strict priority is not explicitly requested, the switch will default
+ * to WFQ mode.
+ *
+ * Return: 0 on success, or a negative error code on failure
+ */
+static int ksz88x3_tc_ets_add(struct ksz_device *dev, int port,
+			      struct tc_ets_qopt_offload_replace_params *p)
+{
+	int ret, band;
+
+	/* Only strict priority mode is supported for now.
+	 * WFQ is implicitly enabled when strict mode is disabled.
+	 */
+	for (band = 0; band < p->bands; band++) {
+		int queue = ksz_ets_band_to_queue(p, band);
+		u8 reg;
+
+		/* Calculate TXQ Split Control register address for this
+		 * port/queue
+		 */
+		reg = KSZ8873_TXQ_SPLIT_CTRL_REG(port, queue);
+		if (ksz_is_ksz8463(dev))
+			reg = ksz8463_tc_ctrl(port, queue);
+
+		/* Clear WFQ enable bit to select strict priority scheduling */
+		ret = ksz_rmw8(dev, reg, KSZ8873_TXQ_WFQ_ENABLE, 0);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * ksz88x3_tc_ets_del - Reset ETS (Enhanced Transmission Selection) config
+ *                      for a port on KSZ88x3 switch
+ * @dev: Pointer to the KSZ switch device structure
+ * @port: Port number to reset
+ *
+ * The KSZ88x3 supports only fixed scheduling modes: Strict Priority or
+ * Weighted Fair Queuing (WFQ), with no reconfiguration of weights or
+ * queue mapping. This function resets the port’s scheduling mode to
+ * the default, which is WFQ, by enabling the WFQ bit for all queues.
+ *
+ * Return: 0 on success, or a negative error code on failure
+ */
+static int ksz88x3_tc_ets_del(struct ksz_device *dev, int port)
+{
+	int ret, queue;
+
+	/* Iterate over all transmit queues for this port */
+	for (queue = 0; queue < dev->info->num_tx_queues; queue++) {
+		u8 reg;
+
+		/* Calculate TXQ Split Control register address for this
+		 * port/queue
+		 */
+		reg = KSZ8873_TXQ_SPLIT_CTRL_REG(port, queue);
+		if (ksz_is_ksz8463(dev))
+			reg = ksz8463_tc_ctrl(port, queue);
+
+		/* Set WFQ enable bit to revert back to default scheduling
+		 * mode
+		 */
+		ret = ksz_rmw8(dev, reg, KSZ8873_TXQ_WFQ_ENABLE,
+			       KSZ8873_TXQ_WFQ_ENABLE);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int ksz8_tc_setup_qdisc_ets(struct dsa_switch *ds, int port,
+				   struct tc_ets_qopt_offload *qopt)
+{
+	struct ksz_device *dev = ds->priv;
+	int ret;
+
+	if (!(ksz_is_ksz88x3(dev) || ksz_is_ksz8463(dev)))
+		return -EOPNOTSUPP;
+
+	if (qopt->parent != TC_H_ROOT) {
+		dev_err(dev->dev, "Parent should be \"root\"\n");
+		return -EOPNOTSUPP;
+	}
+
+	switch (qopt->command) {
+	case TC_ETS_REPLACE:
+		ret = ksz_tc_ets_validate(dev, port, &qopt->replace_params);
+		if (ret)
+			return ret;
+
+		return ksz88x3_tc_ets_add(dev, port, &qopt->replace_params);
+	case TC_ETS_DESTROY:
+		return ksz88x3_tc_ets_del(dev, port);
+	case TC_ETS_STATS:
+	case TC_ETS_GRAFT:
+		return -EOPNOTSUPP;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static int ksz87xx_setup_tc(struct dsa_switch *ds, int port,
+			    enum tc_setup_type type, void *type_data)
+{
+	switch (type) {
+	case TC_SETUP_QDISC_CBS:
+		return ksz_setup_tc_cbs(ds, port, type_data);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int ksz8_setup_tc(struct dsa_switch *ds, int port,
+			 enum tc_setup_type type, void *type_data)
+{
+	switch (type) {
+	case TC_SETUP_QDISC_CBS:
+		return ksz_setup_tc_cbs(ds, port, type_data);
+	case TC_SETUP_QDISC_ETS:
+		return ksz8_tc_setup_qdisc_ets(ds, port, type_data);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
 static void ksz8795_cpu_interface_select(struct ksz_device *dev, int port)
 {
 	struct ksz_port *p = &dev->ports[port];
@@ -2053,6 +2251,171 @@ static int ksz8_enable_stp_addr(struct ksz_device *dev)
 	return ksz8_w_sta_mac_table(dev, 0, &alu);
 }
 
+static void ksz88xx_r_mib_stats64(struct ksz_device *dev, int port)
+{
+	struct ethtool_pause_stats *pstats;
+	struct rtnl_link_stats64 *stats;
+	struct ksz88xx_stats_raw *raw;
+	struct ksz_port_mib *mib;
+
+	mib = &dev->ports[port].mib;
+	stats = &mib->stats64;
+	pstats = &mib->pause_stats;
+	raw = (struct ksz88xx_stats_raw *)mib->counters;
+
+	spin_lock(&mib->stats64_lock);
+
+	stats->rx_packets = raw->rx_bcast + raw->rx_mcast + raw->rx_ucast +
+		raw->rx_pause;
+	stats->tx_packets = raw->tx_bcast + raw->tx_mcast + raw->tx_ucast +
+		raw->tx_pause;
+
+	/* HW counters are counting bytes + FCS which is not acceptable
+	 * for rtnl_link_stats64 interface
+	 */
+	stats->rx_bytes = raw->rx + raw->rx_hi - stats->rx_packets * ETH_FCS_LEN;
+	stats->tx_bytes = raw->tx + raw->tx_hi - stats->tx_packets * ETH_FCS_LEN;
+
+	stats->rx_length_errors = raw->rx_undersize + raw->rx_fragments +
+		raw->rx_oversize;
+
+	stats->rx_crc_errors = raw->rx_crc_err;
+	stats->rx_frame_errors = raw->rx_align_err;
+	stats->rx_dropped = raw->rx_discards;
+	stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors +
+		stats->rx_frame_errors  + stats->rx_dropped;
+
+	stats->tx_window_errors = raw->tx_late_col;
+	stats->tx_fifo_errors = raw->tx_discards;
+	stats->tx_aborted_errors = raw->tx_exc_col;
+	stats->tx_errors = stats->tx_window_errors + stats->tx_fifo_errors +
+		stats->tx_aborted_errors;
+
+	stats->multicast = raw->rx_mcast;
+	stats->collisions = raw->tx_total_col;
+
+	pstats->tx_pause_frames = raw->tx_pause;
+	pstats->rx_pause_frames = raw->rx_pause;
+
+	spin_unlock(&mib->stats64_lock);
+}
+
+/**
+ * ksz88x3_drive_strength_write() - Set the drive strength configuration for
+ *				    KSZ8863 compatible chip variants.
+ * @dev:       ksz device
+ * @props:     Array of drive strength properties to be set
+ * @num_props: Number of properties in the array
+ *
+ * This function applies the specified drive strength settings to KSZ88X3 chip
+ * variants (KSZ8873, KSZ8863).
+ * It ensures the configurations align with what the chip variant supports and
+ * warns or errors out on unsupported settings.
+ *
+ * Return: 0 on success, error code otherwise
+ */
+static int ksz88x3_drive_strength_write(struct ksz_device *dev,
+					struct ksz_driver_strength_prop *props,
+					int num_props)
+{
+	size_t array_size = ARRAY_SIZE(ksz88x3_drive_strengths);
+	int microamp;
+	int i, ret;
+
+	for (i = 0; i < num_props; i++) {
+		if (props[i].value == -1 || i == KSZ_DRIVER_STRENGTH_IO)
+			continue;
+
+		dev_warn(dev->dev, "%s is not supported by this chip variant\n",
+			 props[i].name);
+	}
+
+	microamp = props[KSZ_DRIVER_STRENGTH_IO].value;
+	ret = ksz_drive_strength_to_reg(ksz88x3_drive_strengths, array_size,
+					microamp);
+	if (ret < 0) {
+		ksz_drive_strength_error(dev, ksz88x3_drive_strengths,
+					 array_size, microamp);
+		return ret;
+	}
+
+	return ksz_rmw8(dev, KSZ8873_REG_GLOBAL_CTRL_12,
+			KSZ8873_DRIVE_STRENGTH_16MA, ret);
+}
+
+/**
+ * ksz8_parse_drive_strength() - Extract and apply drive strength configurations
+ *				 from device tree properties.
+ * @dev:	ksz device
+ *
+ * This function reads the specified drive strength properties from the
+ * device tree, validates against the supported chip variants, and sets
+ * them accordingly. An error should be critical here, as the drive strength
+ * settings are crucial for EMI compliance.
+ *
+ * Return: 0 on success, error code otherwise
+ */
+static int ksz8_parse_drive_strength(struct ksz_device *dev)
+{
+	struct ksz_driver_strength_prop of_props[] = {
+		[KSZ_DRIVER_STRENGTH_HI] = {
+			.name = "microchip,hi-drive-strength-microamp",
+			.offset = SW_HI_SPEED_DRIVE_STRENGTH_S,
+			.value = -1,
+		},
+		[KSZ_DRIVER_STRENGTH_LO] = {
+			.name = "microchip,lo-drive-strength-microamp",
+			.offset = SW_LO_SPEED_DRIVE_STRENGTH_S,
+			.value = -1,
+		},
+		[KSZ_DRIVER_STRENGTH_IO] = {
+			.name = "microchip,io-drive-strength-microamp",
+			.offset = 0, /* don't care */
+			.value = -1,
+		},
+	};
+	struct device_node *np = dev->dev->of_node;
+	bool have_any_prop = false;
+	int i, ret;
+
+	for (i = 0; i < ARRAY_SIZE(of_props); i++) {
+		ret = of_property_read_u32(np, of_props[i].name,
+					   &of_props[i].value);
+		if (ret && ret != -EINVAL)
+			dev_warn(dev->dev, "Failed to read %s\n",
+				 of_props[i].name);
+		if (ret)
+			continue;
+
+		have_any_prop = true;
+	}
+
+	if (!have_any_prop)
+		return 0;
+
+	switch (dev->chip_id) {
+	case KSZ88X3_CHIP_ID:
+		return ksz88x3_drive_strength_write(dev, of_props,
+						    ARRAY_SIZE(of_props));
+	case KSZ8795_CHIP_ID:
+	case KSZ8794_CHIP_ID:
+	case KSZ8765_CHIP_ID:
+		return ksz_drive_strength_write(dev, of_props,
+						ARRAY_SIZE(of_props));
+	default:
+		/* KSZ8864, KSZ8895 */
+		for (i = 0; i < ARRAY_SIZE(of_props); i++) {
+			if (of_props[i].value == -1)
+				continue;
+
+			dev_warn(dev->dev, "%s is not supported by this chip variant\n",
+				 of_props[i].name);
+		}
+	}
+
+	return 0;
+}
+
 static int ksz8_setup(struct dsa_switch *ds)
 {
 	struct ksz_device *dev = ds->priv;
@@ -2075,7 +2438,7 @@ static int ksz8_setup(struct dsa_switch *ds)
 		return ret;
 	}
 
-	ret = ksz_parse_drive_strength(dev);
+	ret = ksz8_parse_drive_strength(dev);
 	if (ret)
 		return ret;
 
@@ -2551,8 +2914,8 @@ const struct dsa_switch_ops ksz8463_switch_ops = {
 	.port_mirror_del	= ksz8_port_mirror_del,
 	.get_stats64		= ksz_get_stats64,
 	.get_pause_stats	= ksz_get_pause_stats,
-	.port_change_mtu	= ksz8_change_mtu,
-	.port_max_mtu		= ksz_max_mtu,
+	.port_change_mtu	= ksz88xx_change_mtu,
+	.port_max_mtu		= ksz88xx_max_mtu,
 	.suspend		= ksz_suspend,
 	.resume			= ksz_resume,
 	.get_ts_info		= ksz_get_ts_info,
@@ -2560,7 +2923,7 @@ const struct dsa_switch_ops ksz8463_switch_ops = {
 	.port_hwtstamp_set	= ksz_hwtstamp_set,
 	.port_txtstamp		= ksz_port_txtstamp,
 	.port_rxtstamp		= ksz_port_rxtstamp,
-	.port_setup_tc		= ksz_setup_tc,
+	.port_setup_tc		= ksz8_setup_tc,
 	.port_get_default_prio	= ksz_port_get_default_prio,
 	.port_set_default_prio	= ksz_port_set_default_prio,
 	.port_get_dscp_prio	= ksz_port_get_dscp_prio,
@@ -2601,8 +2964,8 @@ const struct dsa_switch_ops ksz87xx_switch_ops = {
 	.port_mirror_del	= ksz8_port_mirror_del,
 	.get_stats64		= ksz_get_stats64,
 	.get_pause_stats	= ksz_get_pause_stats,
-	.port_change_mtu	= ksz8_change_mtu,
-	.port_max_mtu		= ksz_max_mtu,
+	.port_change_mtu	= ksz87xx_change_mtu,
+	.port_max_mtu		= ksz87xx_max_mtu,
 	.suspend		= ksz_suspend,
 	.resume			= ksz_resume,
 	.get_ts_info		= ksz_get_ts_info,
@@ -2610,7 +2973,7 @@ const struct dsa_switch_ops ksz87xx_switch_ops = {
 	.port_hwtstamp_set	= ksz_hwtstamp_set,
 	.port_txtstamp		= ksz_port_txtstamp,
 	.port_rxtstamp		= ksz_port_rxtstamp,
-	.port_setup_tc		= ksz_setup_tc,
+	.port_setup_tc		= ksz87xx_setup_tc,
 	.port_get_default_prio	= ksz_port_get_default_prio,
 	.port_set_default_prio	= ksz_port_set_default_prio,
 	.port_get_dscp_prio	= ksz_port_get_dscp_prio,
@@ -2652,8 +3015,8 @@ const struct dsa_switch_ops ksz88xx_switch_ops = {
 	.port_mirror_del	= ksz8_port_mirror_del,
 	.get_stats64		= ksz_get_stats64,
 	.get_pause_stats	= ksz_get_pause_stats,
-	.port_change_mtu	= ksz8_change_mtu,
-	.port_max_mtu		= ksz_max_mtu,
+	.port_change_mtu	= ksz88xx_change_mtu,
+	.port_max_mtu		= ksz88xx_max_mtu,
 	.get_wol		= ksz_get_wol,
 	.set_wol		= ksz_set_wol,
 	.suspend		= ksz_suspend,
@@ -2663,7 +3026,7 @@ const struct dsa_switch_ops ksz88xx_switch_ops = {
 	.port_hwtstamp_set	= ksz_hwtstamp_set,
 	.port_txtstamp		= ksz_port_txtstamp,
 	.port_rxtstamp		= ksz_port_rxtstamp,
-	.port_setup_tc		= ksz_setup_tc,
+	.port_setup_tc		= ksz8_setup_tc,
 	.port_get_default_prio	= ksz_port_get_default_prio,
 	.port_set_default_prio	= ksz_port_set_default_prio,
 	.port_get_dscp_prio	= ksz_port_get_dscp_prio,
diff --git a/drivers/net/dsa/microchip/ksz9477.c b/drivers/net/dsa/microchip/ksz9477.c
index f3f0c98dfb5a7..3ee995545c579 100644
--- a/drivers/net/dsa/microchip/ksz9477.c
+++ b/drivers/net/dsa/microchip/ksz9477.c
@@ -15,6 +15,7 @@
 #include <linux/if_hsr.h>
 #include <linux/if_vlan.h>
 #include <net/dsa.h>
+#include <net/ieee8021q.h>
 #include <net/switchdev.h>
 
 #include "ksz9477_reg.h"
@@ -60,6 +61,11 @@ static int ksz9477_change_mtu(struct dsa_switch *ds, int port, int mtu)
 				  REG_SW_MTU_MASK, frame_size);
 }
 
+int ksz9477_max_mtu(struct dsa_switch *ds, int port)
+{
+	return KSZ9477_MAX_FRAME_SIZE - VLAN_ETH_HLEN - ETH_FCS_LEN;
+}
+
 static int ksz9477_wait_vlan_ctrl_ready(struct ksz_device *dev)
 {
 	unsigned int val;
@@ -478,8 +484,8 @@ static int ksz9477_half_duplex_monitor(struct ksz_device *dev, int port,
 	return ret;
 }
 
-int ksz9477_errata_monitor(struct ksz_device *dev, int port,
-			   u64 tx_late_col)
+static int ksz9477_errata_monitor(struct ksz_device *dev, int port,
+				  u64 tx_late_col)
 {
 	u8 status;
 	int ret;
@@ -497,6 +503,24 @@ int ksz9477_errata_monitor(struct ksz_device *dev, int port,
 	return ret;
 }
 
+static void ksz9477_r_mib_stats64(struct ksz_device *dev, int port)
+{
+	struct ksz_stats_raw *raw;
+	struct ksz_port_mib *mib;
+	int ret;
+
+	ksz_r_mib_stats64(dev, port);
+
+	if (dev->info->phy_errata_9477 && !ksz_is_sgmii_port(dev, port)) {
+		mib = &dev->ports[port].mib;
+		raw = (struct ksz_stats_raw *)mib->counters;
+
+		ret = ksz9477_errata_monitor(dev, port, raw->tx_late_col);
+		if (ret)
+			dev_err(dev->dev, "Failed to monitor transmission halt\n");
+	}
+};
+
 void ksz9477_port_init_cnt(struct ksz_device *dev, int port)
 {
 	struct ksz_port_mib *mib = &dev->ports[port].mib;
@@ -1176,6 +1200,58 @@ void ksz9477_port_mirror_del(struct dsa_switch *ds, int port,
 			     PORT_MIRROR_SNIFFER, false);
 }
 
+static bool ksz9477_get_gbit(struct ksz_device *dev, int port)
+{
+	const u8 *bitval = dev->info->xmii_ctrl1;
+	const u16 *regs = dev->info->regs;
+	bool gbit = false;
+	u8 data8;
+	bool val;
+
+	ksz_pread8(dev, port, regs[P_XMII_CTRL_1], &data8);
+
+	val = FIELD_GET(P_GMII_1GBIT_M, data8);
+
+	if (val == bitval[P_GMII_1GBIT])
+		gbit = true;
+
+	return gbit;
+}
+
+static phy_interface_t ksz9477_get_xmii(struct ksz_device *dev, int port,
+					bool gbit)
+{
+	const u8 *bitval = dev->info->xmii_ctrl1;
+	const u16 *regs = dev->info->regs;
+	phy_interface_t interface;
+	u8 data8;
+	u8 val;
+
+	ksz_pread8(dev, port, regs[P_XMII_CTRL_1], &data8);
+
+	val = FIELD_GET(P_MII_SEL_M, data8);
+
+	if (val == bitval[P_MII_SEL]) {
+		if (gbit)
+			interface = PHY_INTERFACE_MODE_GMII;
+		else
+			interface = PHY_INTERFACE_MODE_MII;
+	} else if (val == bitval[P_RMII_SEL]) {
+		interface = PHY_INTERFACE_MODE_RMII;
+	} else {
+		interface = PHY_INTERFACE_MODE_RGMII;
+		if (data8 & P_RGMII_ID_EG_ENABLE)
+			interface = PHY_INTERFACE_MODE_RGMII_TXID;
+		if (data8 & P_RGMII_ID_IG_ENABLE) {
+			interface = PHY_INTERFACE_MODE_RGMII_RXID;
+			if (data8 & P_RGMII_ID_EG_ENABLE)
+				interface = PHY_INTERFACE_MODE_RGMII_ID;
+		}
+	}
+
+	return interface;
+}
+
 static phy_interface_t ksz9477_get_interface(struct ksz_device *dev, int port)
 {
 	phy_interface_t interface;
@@ -1184,9 +1260,9 @@ static phy_interface_t ksz9477_get_interface(struct ksz_device *dev, int port)
 	if (dev->info->internal_phy[port])
 		return PHY_INTERFACE_MODE_NA;
 
-	gbit = ksz_get_gbit(dev, port);
+	gbit = ksz9477_get_gbit(dev, port);
 
-	interface = ksz_get_xmii(dev, port, gbit);
+	interface = ksz9477_get_xmii(dev, port, gbit);
 
 	return interface;
 }
@@ -1335,6 +1411,28 @@ static void ksz9477_port_setup(struct ksz_device *dev, int port, bool cpu_port)
 	ksz_pwrite8(dev, port, regs[REG_PORT_PME_CTRL], 0);
 }
 
+int ksz9477_set_default_prio_queue_mapping(struct ksz_device *dev, int port)
+{
+	u32 queue_map = 0;
+	int ipm;
+
+	for (ipm = 0; ipm < dev->info->num_ipms; ipm++) {
+		int queue;
+
+		/* Traffic Type (TT) is corresponding to the Internal Priority
+		 * Map (IPM) in the switch. Traffic Class (TC) is
+		 * corresponding to the queue in the switch.
+		 */
+		queue = ieee8021q_tt_to_tc(ipm, dev->info->num_tx_queues);
+		if (queue < 0)
+			return queue;
+
+		queue_map |= queue << (ipm * KSZ9477_PORT_TC_MAP_S);
+	}
+
+	return ksz_pwrite32(dev, port, KSZ9477_PORT_MRI_TC_MAP__4, queue_map);
+}
+
 static int ksz9477_dsa_port_setup(struct dsa_switch *ds, int port)
 {
 	struct ksz_device *dev = ds->priv;
@@ -1517,6 +1615,58 @@ int ksz9477_enable_stp_addr(struct ksz_device *dev)
 	return 0;
 }
 
+/**
+ * ksz9477_parse_drive_strength() - Extract and apply drive strength
+ *				    configurations from device tree properties.
+ * @dev:	ksz device
+ *
+ * This function reads the specified drive strength properties from the
+ * device tree, validates against the supported chip variants, and sets
+ * them accordingly. An error should be critical here, as the drive strength
+ * settings are crucial for EMI compliance.
+ *
+ * Return: 0 on success, error code otherwise
+ */
+static int ksz9477_parse_drive_strength(struct ksz_device *dev)
+{
+	struct ksz_driver_strength_prop of_props[] = {
+		[KSZ_DRIVER_STRENGTH_HI] = {
+			.name = "microchip,hi-drive-strength-microamp",
+			.offset = SW_HI_SPEED_DRIVE_STRENGTH_S,
+			.value = -1,
+		},
+		[KSZ_DRIVER_STRENGTH_LO] = {
+			.name = "microchip,lo-drive-strength-microamp",
+			.offset = SW_LO_SPEED_DRIVE_STRENGTH_S,
+			.value = -1,
+		},
+		[KSZ_DRIVER_STRENGTH_IO] = {
+			.name = "microchip,io-drive-strength-microamp",
+			.offset = 0, /* don't care */
+			.value = -1,
+		},
+	};
+	struct device_node *np = dev->dev->of_node;
+	bool have_any_prop = false;
+	int i, ret;
+
+	for (i = 0; i < ARRAY_SIZE(of_props); i++) {
+		ret = of_property_read_u32(np, of_props[i].name,
+					   &of_props[i].value);
+		if (ret && ret != -EINVAL)
+			dev_warn(dev->dev, "Failed to read %s\n",
+				 of_props[i].name);
+		if (ret)
+			continue;
+
+		have_any_prop = true;
+	}
+
+	if (!have_any_prop)
+		return 0;
+
+	return ksz_drive_strength_write(dev, of_props, ARRAY_SIZE(of_props));
+}
 static int ksz9477_setup(struct dsa_switch *ds)
 {
 	struct ksz_device *dev = ds->priv;
@@ -1539,7 +1689,7 @@ static int ksz9477_setup(struct dsa_switch *ds)
 		return ret;
 	}
 
-	ret = ksz_parse_drive_strength(dev);
+	ret = ksz9477_parse_drive_strength(dev);
 	if (ret)
 		return ret;
 
@@ -2052,7 +2202,7 @@ const struct ksz_dev_ops ksz9477_dev_ops = {
 	.cfg_port_member = ksz9477_cfg_port_member,
 	.r_mib_cnt = ksz9477_r_mib_cnt,
 	.r_mib_pkt = ksz9477_r_mib_pkt,
-	.r_mib_stat64 = ksz_r_mib_stats64,
+	.r_mib_stat64 = ksz9477_r_mib_stats64,
 	.freeze_mib = ksz9477_freeze_mib,
 	.port_init_cnt = ksz9477_port_init_cnt,
 	.pme_write8 = ksz_write8,
@@ -2098,7 +2248,7 @@ const struct dsa_switch_ops ksz9477_switch_ops = {
 	.get_stats64		= ksz_get_stats64,
 	.get_pause_stats	= ksz_get_pause_stats,
 	.port_change_mtu	= ksz9477_change_mtu,
-	.port_max_mtu		= ksz_max_mtu,
+	.port_max_mtu		= ksz9477_max_mtu,
 	.get_wol		= ksz_get_wol,
 	.set_wol		= ksz_set_wol,
 	.suspend		= ksz_suspend,
diff --git a/drivers/net/dsa/microchip/ksz9477.h b/drivers/net/dsa/microchip/ksz9477.h
index 92a1d889224d3..a84c000000e66 100644
--- a/drivers/net/dsa/microchip/ksz9477.h
+++ b/drivers/net/dsa/microchip/ksz9477.h
@@ -21,6 +21,7 @@ void ksz9477_freeze_mib(struct ksz_device *dev, int port, bool freeze);
 void ksz9477_port_init_cnt(struct ksz_device *dev, int port);
 int ksz9477_port_vlan_filtering(struct dsa_switch *ds, int port,
 				bool flag, struct netlink_ext_ack *extack);
+int ksz9477_max_mtu(struct dsa_switch *ds, int port);
 int ksz9477_port_vlan_add(struct dsa_switch *ds, int port,
 			  const struct switchdev_obj_port_vlan *vlan,
 			  struct netlink_ext_ack *extack);
@@ -31,8 +32,6 @@ int ksz9477_port_mirror_add(struct dsa_switch *ds, int port,
 			    bool ingress, struct netlink_ext_ack *extack);
 void ksz9477_port_mirror_del(struct dsa_switch *ds, int port,
 			     struct dsa_mall_mirror_tc_entry *mirror);
-int ksz9477_errata_monitor(struct ksz_device *dev, int port,
-			   u64 tx_late_col);
 int ksz9477_fdb_dump(struct dsa_switch *ds, int port,
 		     dsa_fdb_dump_cb_t *cb, void *data);
 int ksz9477_fdb_add(struct dsa_switch *ds, int port,
@@ -45,6 +44,7 @@ int ksz9477_mdb_del(struct dsa_switch *ds, int port,
 		    const struct switchdev_obj_port_mdb *mdb, struct dsa_db db);
 int ksz9477_enable_stp_addr(struct ksz_device *dev);
 void ksz9477_port_queue_split(struct ksz_device *dev, int port);
+int ksz9477_set_default_prio_queue_mapping(struct ksz_device *dev, int port);
 
 int ksz9477_port_acl_init(struct ksz_device *dev, int port);
 void ksz9477_port_acl_free(struct ksz_device *dev, int port);
diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c
index d1726778bb488..67ab6ddb9e53f 100644
--- a/drivers/net/dsa/microchip/ksz_common.c
+++ b/drivers/net/dsa/microchip/ksz_common.c
@@ -37,82 +37,6 @@
 
 #define MIB_COUNTER_NUM 0x20
 
-struct ksz_stats_raw {
-	u64 rx_hi;
-	u64 rx_undersize;
-	u64 rx_fragments;
-	u64 rx_oversize;
-	u64 rx_jabbers;
-	u64 rx_symbol_err;
-	u64 rx_crc_err;
-	u64 rx_align_err;
-	u64 rx_mac_ctrl;
-	u64 rx_pause;
-	u64 rx_bcast;
-	u64 rx_mcast;
-	u64 rx_ucast;
-	u64 rx_64_or_less;
-	u64 rx_65_127;
-	u64 rx_128_255;
-	u64 rx_256_511;
-	u64 rx_512_1023;
-	u64 rx_1024_1522;
-	u64 rx_1523_2000;
-	u64 rx_2001;
-	u64 tx_hi;
-	u64 tx_late_col;
-	u64 tx_pause;
-	u64 tx_bcast;
-	u64 tx_mcast;
-	u64 tx_ucast;
-	u64 tx_deferred;
-	u64 tx_total_col;
-	u64 tx_exc_col;
-	u64 tx_single_col;
-	u64 tx_mult_col;
-	u64 rx_total;
-	u64 tx_total;
-	u64 rx_discards;
-	u64 tx_discards;
-};
-
-struct ksz88xx_stats_raw {
-	u64 rx;
-	u64 rx_hi;
-	u64 rx_undersize;
-	u64 rx_fragments;
-	u64 rx_oversize;
-	u64 rx_jabbers;
-	u64 rx_symbol_err;
-	u64 rx_crc_err;
-	u64 rx_align_err;
-	u64 rx_mac_ctrl;
-	u64 rx_pause;
-	u64 rx_bcast;
-	u64 rx_mcast;
-	u64 rx_ucast;
-	u64 rx_64_or_less;
-	u64 rx_65_127;
-	u64 rx_128_255;
-	u64 rx_256_511;
-	u64 rx_512_1023;
-	u64 rx_1024_1522;
-	u64 tx;
-	u64 tx_hi;
-	u64 tx_late_col;
-	u64 tx_pause;
-	u64 tx_bcast;
-	u64 tx_mcast;
-	u64 tx_ucast;
-	u64 tx_deferred;
-	u64 tx_total_col;
-	u64 tx_exc_col;
-	u64 tx_single_col;
-	u64 tx_mult_col;
-	u64 rx_discards;
-	u64 tx_discards;
-};
-
 static const struct ksz_mib_names ksz88xx_mib_names[] = {
 	{ 0x00, "rx" },
 	{ 0x01, "rx_hi" },
@@ -189,28 +113,6 @@ static const struct ksz_mib_names ksz9477_mib_names[] = {
 	{ 0x83, "tx_discards" },
 };
 
-struct ksz_driver_strength_prop {
-	const char *name;
-	int offset;
-	int value;
-};
-
-enum ksz_driver_strength_type {
-	KSZ_DRIVER_STRENGTH_HI,
-	KSZ_DRIVER_STRENGTH_LO,
-	KSZ_DRIVER_STRENGTH_IO,
-};
-
-/**
- * struct ksz_drive_strength - drive strength mapping
- * @reg_val:	register value
- * @microamp:	microamp value
- */
-struct ksz_drive_strength {
-	u32 reg_val;
-	u32 microamp;
-};
-
 /* ksz9477_drive_strengths - Drive strength mapping for KSZ9477 variants
  *
  * This values are not documented in KSZ9477 variants but confirmed by
@@ -246,15 +148,6 @@ static const struct ksz_drive_strength ksz9477_drive_strengths[] = {
 	{ SW_DRIVE_STRENGTH_28MA, 28000 },
 };
 
-/* ksz88x3_drive_strengths - Drive strength mapping for KSZ8863, KSZ8873, ..
- *			     variants.
- * This values are documented in KSZ8873 and KSZ8863 datasheets.
- */
-static const struct ksz_drive_strength ksz88x3_drive_strengths[] = {
-	{ 0,  8000 },
-	{ KSZ8873_DRIVE_STRENGTH_16MA, 16000 },
-};
-
 /**
  * ksz_phylink_mac_disable_tx_lpi() - Callback to signal LPI support (Dummy)
  * @config: phylink config structure
@@ -2013,7 +1906,6 @@ void ksz_r_mib_stats64(struct ksz_device *dev, int port)
 	struct rtnl_link_stats64 *stats;
 	struct ksz_stats_raw *raw;
 	struct ksz_port_mib *mib;
-	int ret;
 
 	mib = &dev->ports[port].mib;
 	stats = &mib->stats64;
@@ -2055,61 +1947,6 @@ void ksz_r_mib_stats64(struct ksz_device *dev, int port)
 	pstats->rx_pause_frames = raw->rx_pause;
 
 	spin_unlock(&mib->stats64_lock);
-
-	if (dev->info->phy_errata_9477 && !ksz_is_sgmii_port(dev, port)) {
-		ret = ksz9477_errata_monitor(dev, port, raw->tx_late_col);
-		if (ret)
-			dev_err(dev->dev, "Failed to monitor transmission halt\n");
-	}
-}
-
-void ksz88xx_r_mib_stats64(struct ksz_device *dev, int port)
-{
-	struct ethtool_pause_stats *pstats;
-	struct rtnl_link_stats64 *stats;
-	struct ksz88xx_stats_raw *raw;
-	struct ksz_port_mib *mib;
-
-	mib = &dev->ports[port].mib;
-	stats = &mib->stats64;
-	pstats = &mib->pause_stats;
-	raw = (struct ksz88xx_stats_raw *)mib->counters;
-
-	spin_lock(&mib->stats64_lock);
-
-	stats->rx_packets = raw->rx_bcast + raw->rx_mcast + raw->rx_ucast +
-		raw->rx_pause;
-	stats->tx_packets = raw->tx_bcast + raw->tx_mcast + raw->tx_ucast +
-		raw->tx_pause;
-
-	/* HW counters are counting bytes + FCS which is not acceptable
-	 * for rtnl_link_stats64 interface
-	 */
-	stats->rx_bytes = raw->rx + raw->rx_hi - stats->rx_packets * ETH_FCS_LEN;
-	stats->tx_bytes = raw->tx + raw->tx_hi - stats->tx_packets * ETH_FCS_LEN;
-
-	stats->rx_length_errors = raw->rx_undersize + raw->rx_fragments +
-		raw->rx_oversize;
-
-	stats->rx_crc_errors = raw->rx_crc_err;
-	stats->rx_frame_errors = raw->rx_align_err;
-	stats->rx_dropped = raw->rx_discards;
-	stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors +
-		stats->rx_frame_errors  + stats->rx_dropped;
-
-	stats->tx_window_errors = raw->tx_late_col;
-	stats->tx_fifo_errors = raw->tx_discards;
-	stats->tx_aborted_errors = raw->tx_exc_col;
-	stats->tx_errors = stats->tx_window_errors + stats->tx_fifo_errors +
-		stats->tx_aborted_errors;
-
-	stats->multicast = raw->rx_mcast;
-	stats->collisions = raw->tx_total_col;
-
-	pstats->tx_pause_frames = raw->tx_pause;
-	pstats->rx_pause_frames = raw->rx_pause;
-
-	spin_unlock(&mib->stats64_lock);
 }
 
 void ksz_get_stats64(struct dsa_switch *ds, int port,
@@ -2885,28 +2722,6 @@ void ksz_port_bridge_leave(struct dsa_switch *ds, int port,
 	 */
 }
 
-int ksz9477_set_default_prio_queue_mapping(struct ksz_device *dev, int port)
-{
-	u32 queue_map = 0;
-	int ipm;
-
-	for (ipm = 0; ipm < dev->info->num_ipms; ipm++) {
-		int queue;
-
-		/* Traffic Type (TT) is corresponding to the Internal Priority
-		 * Map (IPM) in the switch. Traffic Class (TC) is
-		 * corresponding to the queue in the switch.
-		 */
-		queue = ieee8021q_tt_to_tc(ipm, dev->info->num_tx_queues);
-		if (queue < 0)
-			return queue;
-
-		queue_map |= queue << (ipm * KSZ9477_PORT_TC_MAP_S);
-	}
-
-	return ksz_pwrite32(dev, port, KSZ9477_PORT_MRI_TC_MAP__4, queue_map);
-}
-
 void ksz_port_stp_state_set(struct dsa_switch *ds, int port, u8 state)
 {
 	struct ksz_device *dev = ds->priv;
@@ -2984,40 +2799,6 @@ int ksz_port_bridge_flags(struct dsa_switch *ds, int port,
 	return 0;
 }
 
-int ksz_max_mtu(struct dsa_switch *ds, int port)
-{
-	struct ksz_device *dev = ds->priv;
-
-	switch (dev->chip_id) {
-	case KSZ8795_CHIP_ID:
-	case KSZ8794_CHIP_ID:
-	case KSZ8765_CHIP_ID:
-		return KSZ8795_HUGE_PACKET_SIZE - VLAN_ETH_HLEN - ETH_FCS_LEN;
-	case KSZ8463_CHIP_ID:
-	case KSZ88X3_CHIP_ID:
-	case KSZ8864_CHIP_ID:
-	case KSZ8895_CHIP_ID:
-		return KSZ8863_HUGE_PACKET_SIZE - VLAN_ETH_HLEN - ETH_FCS_LEN;
-	case KSZ8563_CHIP_ID:
-	case KSZ8567_CHIP_ID:
-	case KSZ9477_CHIP_ID:
-	case KSZ9563_CHIP_ID:
-	case KSZ9567_CHIP_ID:
-	case KSZ9893_CHIP_ID:
-	case KSZ9896_CHIP_ID:
-	case KSZ9897_CHIP_ID:
-	case LAN9370_CHIP_ID:
-	case LAN9371_CHIP_ID:
-	case LAN9372_CHIP_ID:
-	case LAN9373_CHIP_ID:
-	case LAN9374_CHIP_ID:
-	case LAN9646_CHIP_ID:
-		return KSZ9477_MAX_FRAME_SIZE - VLAN_ETH_HLEN - ETH_FCS_LEN;
-	}
-
-	return -EOPNOTSUPP;
-}
-
 int ksz_set_mac_eee(struct dsa_switch *ds, int port,
 		    struct ethtool_keee *e)
 {
@@ -3086,39 +2867,6 @@ void ksz_set_xmii(struct ksz_device *dev, int port, phy_interface_t interface)
 	ksz_pwrite8(dev, port, regs[P_XMII_CTRL_1], data8);
 }
 
-phy_interface_t ksz_get_xmii(struct ksz_device *dev, int port, bool gbit)
-{
-	const u8 *bitval = dev->info->xmii_ctrl1;
-	const u16 *regs = dev->info->regs;
-	phy_interface_t interface;
-	u8 data8;
-	u8 val;
-
-	ksz_pread8(dev, port, regs[P_XMII_CTRL_1], &data8);
-
-	val = FIELD_GET(P_MII_SEL_M, data8);
-
-	if (val == bitval[P_MII_SEL]) {
-		if (gbit)
-			interface = PHY_INTERFACE_MODE_GMII;
-		else
-			interface = PHY_INTERFACE_MODE_MII;
-	} else if (val == bitval[P_RMII_SEL]) {
-		interface = PHY_INTERFACE_MODE_RMII;
-	} else {
-		interface = PHY_INTERFACE_MODE_RGMII;
-		if (data8 & P_RGMII_ID_EG_ENABLE)
-			interface = PHY_INTERFACE_MODE_RGMII_TXID;
-		if (data8 & P_RGMII_ID_IG_ENABLE) {
-			interface = PHY_INTERFACE_MODE_RGMII_RXID;
-			if (data8 & P_RGMII_ID_EG_ENABLE)
-				interface = PHY_INTERFACE_MODE_RGMII_ID;
-		}
-	}
-
-	return interface;
-}
-
 bool ksz_phylink_need_config(struct phylink_config *config,
 			     unsigned int mode)
 {
@@ -3154,24 +2902,6 @@ void ksz_phylink_mac_config(struct phylink_config *config,
 		ksz_set_xmii(dev, port, state->interface);
 }
 
-bool ksz_get_gbit(struct ksz_device *dev, int port)
-{
-	const u8 *bitval = dev->info->xmii_ctrl1;
-	const u16 *regs = dev->info->regs;
-	bool gbit = false;
-	u8 data8;
-	bool val;
-
-	ksz_pread8(dev, port, regs[P_XMII_CTRL_1], &data8);
-
-	val = FIELD_GET(P_GMII_1GBIT_M, data8);
-
-	if (val == bitval[P_GMII_1GBIT])
-		gbit = true;
-
-	return gbit;
-}
-
 static int ksz_switch_detect(struct ksz_device *dev)
 {
 	u8 id1, id2, id4;
@@ -3314,8 +3044,8 @@ static int ksz_setup_tc_mode(struct ksz_device *dev, int port, u8 scheduler,
 			   FIELD_PREP(MTI_SHAPING_M, shaper));
 }
 
-static int ksz_setup_tc_cbs(struct dsa_switch *ds, int port,
-			    struct tc_cbs_qopt_offload *qopt)
+int ksz_setup_tc_cbs(struct dsa_switch *ds, int port,
+		     struct tc_cbs_qopt_offload *qopt)
 {
 	struct ksz_device *dev = ds->priv;
 	int ret;
@@ -3380,8 +3110,8 @@ static int ksz_disable_egress_rate_limit(struct ksz_device *dev, int port)
 	return 0;
 }
 
-static int ksz_ets_band_to_queue(struct tc_ets_qopt_offload_replace_params *p,
-				 int band)
+int ksz_ets_band_to_queue(struct tc_ets_qopt_offload_replace_params *p,
+			  int band)
 {
 	/* Compared to queues, bands prioritize packets differently. In strict
 	 * priority mode, the lowest priority is assigned to Queue 0 while the
@@ -3390,104 +3120,6 @@ static int ksz_ets_band_to_queue(struct tc_ets_qopt_offload_replace_params *p,
 	return p->bands - 1 - band;
 }
 
-static u8 ksz8463_tc_ctrl(int port, int queue)
-{
-	u8 reg;
-
-	reg = 0xC8 + port * 4;
-	reg += ((3 - queue) / 2) * 2;
-	reg++;
-	reg -= (queue & 1);
-	return reg;
-}
-
-/**
- * ksz88x3_tc_ets_add - Configure ETS (Enhanced Transmission Selection)
- *                      for a port on KSZ88x3 switch
- * @dev: Pointer to the KSZ switch device structure
- * @port: Port number to configure
- * @p: Pointer to offload replace parameters describing ETS bands and mapping
- *
- * The KSZ88x3 supports two scheduling modes: Strict Priority and
- * Weighted Fair Queuing (WFQ). Both modes have fixed behavior:
- *   - No configurable queue-to-priority mapping
- *   - No weight adjustment in WFQ mode
- *
- * This function configures the switch to use strict priority mode by
- * clearing the WFQ enable bit for all queues associated with ETS bands.
- * If strict priority is not explicitly requested, the switch will default
- * to WFQ mode.
- *
- * Return: 0 on success, or a negative error code on failure
- */
-static int ksz88x3_tc_ets_add(struct ksz_device *dev, int port,
-			      struct tc_ets_qopt_offload_replace_params *p)
-{
-	int ret, band;
-
-	/* Only strict priority mode is supported for now.
-	 * WFQ is implicitly enabled when strict mode is disabled.
-	 */
-	for (band = 0; band < p->bands; band++) {
-		int queue = ksz_ets_band_to_queue(p, band);
-		u8 reg;
-
-		/* Calculate TXQ Split Control register address for this
-		 * port/queue
-		 */
-		reg = KSZ8873_TXQ_SPLIT_CTRL_REG(port, queue);
-		if (ksz_is_ksz8463(dev))
-			reg = ksz8463_tc_ctrl(port, queue);
-
-		/* Clear WFQ enable bit to select strict priority scheduling */
-		ret = ksz_rmw8(dev, reg, KSZ8873_TXQ_WFQ_ENABLE, 0);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-/**
- * ksz88x3_tc_ets_del - Reset ETS (Enhanced Transmission Selection) config
- *                      for a port on KSZ88x3 switch
- * @dev: Pointer to the KSZ switch device structure
- * @port: Port number to reset
- *
- * The KSZ88x3 supports only fixed scheduling modes: Strict Priority or
- * Weighted Fair Queuing (WFQ), with no reconfiguration of weights or
- * queue mapping. This function resets the port’s scheduling mode to
- * the default, which is WFQ, by enabling the WFQ bit for all queues.
- *
- * Return: 0 on success, or a negative error code on failure
- */
-static int ksz88x3_tc_ets_del(struct ksz_device *dev, int port)
-{
-	int ret, queue;
-
-	/* Iterate over all transmit queues for this port */
-	for (queue = 0; queue < dev->info->num_tx_queues; queue++) {
-		u8 reg;
-
-		/* Calculate TXQ Split Control register address for this
-		 * port/queue
-		 */
-		reg = KSZ8873_TXQ_SPLIT_CTRL_REG(port, queue);
-		if (ksz_is_ksz8463(dev))
-			reg = ksz8463_tc_ctrl(port, queue);
-
-		/* Set WFQ enable bit to revert back to default scheduling
-		 * mode
-		 */
-		ret = ksz_rmw8(dev, reg, KSZ8873_TXQ_WFQ_ENABLE,
-			       KSZ8873_TXQ_WFQ_ENABLE);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
 static int ksz_queue_set_strict(struct ksz_device *dev, int port, int queue)
 {
 	int ret;
@@ -3580,8 +3212,8 @@ static int ksz_tc_ets_del(struct ksz_device *dev, int port)
 	return ksz9477_set_default_prio_queue_mapping(dev, port);
 }
 
-static int ksz_tc_ets_validate(struct ksz_device *dev, int port,
-			       struct tc_ets_qopt_offload_replace_params *p)
+int ksz_tc_ets_validate(struct ksz_device *dev, int port,
+			struct tc_ets_qopt_offload_replace_params *p)
 {
 	int band;
 
@@ -3622,9 +3254,6 @@ static int ksz_tc_setup_qdisc_ets(struct dsa_switch *ds, int port,
 	struct ksz_device *dev = ds->priv;
 	int ret;
 
-	if (is_ksz8(dev) && !(ksz_is_ksz88x3(dev) || ksz_is_ksz8463(dev)))
-		return -EOPNOTSUPP;
-
 	if (qopt->parent != TC_H_ROOT) {
 		dev_err(dev->dev, "Parent should be \"root\"\n");
 		return -EOPNOTSUPP;
@@ -3636,16 +3265,9 @@ static int ksz_tc_setup_qdisc_ets(struct dsa_switch *ds, int port,
 		if (ret)
 			return ret;
 
-		if (ksz_is_ksz88x3(dev) || ksz_is_ksz8463(dev))
-			return ksz88x3_tc_ets_add(dev, port,
-						  &qopt->replace_params);
-		else
-			return ksz_tc_ets_add(dev, port, &qopt->replace_params);
+		return ksz_tc_ets_add(dev, port, &qopt->replace_params);
 	case TC_ETS_DESTROY:
-		if (ksz_is_ksz88x3(dev) || ksz_is_ksz8463(dev))
-			return ksz88x3_tc_ets_del(dev, port);
-		else
-			return ksz_tc_ets_del(dev, port);
+		return ksz_tc_ets_del(dev, port);
 	case TC_ETS_STATS:
 	case TC_ETS_GRAFT:
 		return -EOPNOTSUPP;
@@ -3704,6 +3326,34 @@ int ksz_handle_wake_reason(struct ksz_device *dev, int port)
 				pme_status);
 }
 
+/**
+ * ksz_is_port_mac_global_usable - Check if the MAC address on a given port
+ *                                 can be used as a global address.
+ * @ds: Pointer to the DSA switch structure.
+ * @port: The port number on which the MAC address is to be checked.
+ *
+ * This function examines the MAC address set on the specified port and
+ * determines if it can be used as a global address for the switch.
+ *
+ * Return: true if the port's MAC address can be used as a global address, false
+ * otherwise.
+ */
+static bool ksz_is_port_mac_global_usable(struct dsa_switch *ds, int port)
+{
+	struct net_device *user = dsa_to_port(ds, port)->user;
+	const unsigned char *addr = user->dev_addr;
+	struct ksz_switch_macaddr *switch_macaddr;
+	struct ksz_device *dev = ds->priv;
+
+	ASSERT_RTNL();
+
+	switch_macaddr = dev->switch_macaddr;
+	if (switch_macaddr && !ether_addr_equal(switch_macaddr->addr, addr))
+		return false;
+
+	return true;
+}
+
 /**
  * ksz_get_wol - Get Wake-on-LAN settings for a specified port.
  * @ds: The dsa_switch structure.
@@ -3897,34 +3547,6 @@ int ksz_port_set_mac_address(struct dsa_switch *ds, int port,
 	return 0;
 }
 
-/**
- * ksz_is_port_mac_global_usable - Check if the MAC address on a given port
- *                                 can be used as a global address.
- * @ds: Pointer to the DSA switch structure.
- * @port: The port number on which the MAC address is to be checked.
- *
- * This function examines the MAC address set on the specified port and
- * determines if it can be used as a global address for the switch.
- *
- * Return: true if the port's MAC address can be used as a global address, false
- * otherwise.
- */
-bool ksz_is_port_mac_global_usable(struct dsa_switch *ds, int port)
-{
-	struct net_device *user = dsa_to_port(ds, port)->user;
-	const unsigned char *addr = user->dev_addr;
-	struct ksz_switch_macaddr *switch_macaddr;
-	struct ksz_device *dev = ds->priv;
-
-	ASSERT_RTNL();
-
-	switch_macaddr = dev->switch_macaddr;
-	if (switch_macaddr && !ether_addr_equal(switch_macaddr->addr, addr))
-		return false;
-
-	return true;
-}
-
 /**
  * ksz_switch_macaddr_get - Program the switch's MAC address register.
  * @ds: DSA switch instance.
@@ -4132,8 +3754,8 @@ static void ksz_parse_rgmii_delay(struct ksz_device *dev, int port_num,
  * Returns: If found, the corresponding register value for that drive strength
  * is returned. Otherwise, -EINVAL is returned indicating an invalid value.
  */
-static int ksz_drive_strength_to_reg(const struct ksz_drive_strength *array,
-				     size_t array_size, int microamp)
+int ksz_drive_strength_to_reg(const struct ksz_drive_strength *array,
+			      size_t array_size, int microamp)
 {
 	int i;
 
@@ -4156,9 +3778,9 @@ static int ksz_drive_strength_to_reg(const struct ksz_drive_strength *array,
  * is detected. It lists out all the supported drive strength values for
  * reference in the error message.
  */
-static void ksz_drive_strength_error(struct ksz_device *dev,
-				     const struct ksz_drive_strength *array,
-				     size_t array_size, int microamp)
+void ksz_drive_strength_error(struct ksz_device *dev,
+			      const struct ksz_drive_strength *array,
+			      size_t array_size, int microamp)
 {
 	char supported_values[100];
 	size_t remaining_size;
@@ -4185,8 +3807,8 @@ static void ksz_drive_strength_error(struct ksz_device *dev,
 }
 
 /**
- * ksz9477_drive_strength_write() - Set the drive strength for specific KSZ9477
- *				    chip variants.
+ * ksz_drive_strength_write() - Set the drive strength for specific KSZ9477
+ *				and the KSZ87xx chip variants.
  * @dev:       ksz device
  * @props:     Array of drive strength properties to be applied
  * @num_props: Number of properties in the array
@@ -4197,9 +3819,9 @@ static void ksz_drive_strength_error(struct ksz_device *dev,
  *
  * Return: 0 on successful configuration, a negative error code on failure.
  */
-static int ksz9477_drive_strength_write(struct ksz_device *dev,
-					struct ksz_driver_strength_prop *props,
-					int num_props)
+int ksz_drive_strength_write(struct ksz_device *dev,
+			     struct ksz_driver_strength_prop *props,
+			     int num_props)
 {
 	size_t array_size = ARRAY_SIZE(ksz9477_drive_strengths);
 	int i, ret, reg;
@@ -4236,130 +3858,6 @@ static int ksz9477_drive_strength_write(struct ksz_device *dev,
 	return ksz_rmw8(dev, reg, mask, val);
 }
 
-/**
- * ksz88x3_drive_strength_write() - Set the drive strength configuration for
- *				    KSZ8863 compatible chip variants.
- * @dev:       ksz device
- * @props:     Array of drive strength properties to be set
- * @num_props: Number of properties in the array
- *
- * This function applies the specified drive strength settings to KSZ88X3 chip
- * variants (KSZ8873, KSZ8863).
- * It ensures the configurations align with what the chip variant supports and
- * warns or errors out on unsupported settings.
- *
- * Return: 0 on success, error code otherwise
- */
-static int ksz88x3_drive_strength_write(struct ksz_device *dev,
-					struct ksz_driver_strength_prop *props,
-					int num_props)
-{
-	size_t array_size = ARRAY_SIZE(ksz88x3_drive_strengths);
-	int microamp;
-	int i, ret;
-
-	for (i = 0; i < num_props; i++) {
-		if (props[i].value == -1 || i == KSZ_DRIVER_STRENGTH_IO)
-			continue;
-
-		dev_warn(dev->dev, "%s is not supported by this chip variant\n",
-			 props[i].name);
-	}
-
-	microamp = props[KSZ_DRIVER_STRENGTH_IO].value;
-	ret = ksz_drive_strength_to_reg(ksz88x3_drive_strengths, array_size,
-					microamp);
-	if (ret < 0) {
-		ksz_drive_strength_error(dev, ksz88x3_drive_strengths,
-					 array_size, microamp);
-		return ret;
-	}
-
-	return ksz_rmw8(dev, KSZ8873_REG_GLOBAL_CTRL_12,
-			KSZ8873_DRIVE_STRENGTH_16MA, ret);
-}
-
-/**
- * ksz_parse_drive_strength() - Extract and apply drive strength configurations
- *				from device tree properties.
- * @dev:	ksz device
- *
- * This function reads the specified drive strength properties from the
- * device tree, validates against the supported chip variants, and sets
- * them accordingly. An error should be critical here, as the drive strength
- * settings are crucial for EMI compliance.
- *
- * Return: 0 on success, error code otherwise
- */
-int ksz_parse_drive_strength(struct ksz_device *dev)
-{
-	struct ksz_driver_strength_prop of_props[] = {
-		[KSZ_DRIVER_STRENGTH_HI] = {
-			.name = "microchip,hi-drive-strength-microamp",
-			.offset = SW_HI_SPEED_DRIVE_STRENGTH_S,
-			.value = -1,
-		},
-		[KSZ_DRIVER_STRENGTH_LO] = {
-			.name = "microchip,lo-drive-strength-microamp",
-			.offset = SW_LO_SPEED_DRIVE_STRENGTH_S,
-			.value = -1,
-		},
-		[KSZ_DRIVER_STRENGTH_IO] = {
-			.name = "microchip,io-drive-strength-microamp",
-			.offset = 0, /* don't care */
-			.value = -1,
-		},
-	};
-	struct device_node *np = dev->dev->of_node;
-	bool have_any_prop = false;
-	int i, ret;
-
-	for (i = 0; i < ARRAY_SIZE(of_props); i++) {
-		ret = of_property_read_u32(np, of_props[i].name,
-					   &of_props[i].value);
-		if (ret && ret != -EINVAL)
-			dev_warn(dev->dev, "Failed to read %s\n",
-				 of_props[i].name);
-		if (ret)
-			continue;
-
-		have_any_prop = true;
-	}
-
-	if (!have_any_prop)
-		return 0;
-
-	switch (dev->chip_id) {
-	case KSZ88X3_CHIP_ID:
-		return ksz88x3_drive_strength_write(dev, of_props,
-						    ARRAY_SIZE(of_props));
-	case KSZ8795_CHIP_ID:
-	case KSZ8794_CHIP_ID:
-	case KSZ8765_CHIP_ID:
-	case KSZ8563_CHIP_ID:
-	case KSZ8567_CHIP_ID:
-	case KSZ9477_CHIP_ID:
-	case KSZ9563_CHIP_ID:
-	case KSZ9567_CHIP_ID:
-	case KSZ9893_CHIP_ID:
-	case KSZ9896_CHIP_ID:
-	case KSZ9897_CHIP_ID:
-	case LAN9646_CHIP_ID:
-		return ksz9477_drive_strength_write(dev, of_props,
-						    ARRAY_SIZE(of_props));
-	default:
-		for (i = 0; i < ARRAY_SIZE(of_props); i++) {
-			if (of_props[i].value == -1)
-				continue;
-
-			dev_warn(dev->dev, "%s is not supported by this chip variant\n",
-				 of_props[i].name);
-		}
-	}
-
-	return 0;
-}
-
 static int ksz8463_configure_straps_spi(struct ksz_device *dev)
 {
 	struct pinctrl *pinctrl;
diff --git a/drivers/net/dsa/microchip/ksz_common.h b/drivers/net/dsa/microchip/ksz_common.h
index b4a5673ba365c..acaf70e6f393c 100644
--- a/drivers/net/dsa/microchip/ksz_common.h
+++ b/drivers/net/dsa/microchip/ksz_common.h
@@ -14,6 +14,7 @@
 #include <linux/phy.h>
 #include <linux/regmap.h>
 #include <net/dsa.h>
+#include <net/pkt_cls.h>
 #include <linux/irq.h>
 #include <linux/platform_data/microchip-ksz.h>
 
@@ -40,6 +41,45 @@ struct vlan_table {
 	u32 table[3];
 };
 
+struct ksz_stats_raw {
+	u64 rx_hi;
+	u64 rx_undersize;
+	u64 rx_fragments;
+	u64 rx_oversize;
+	u64 rx_jabbers;
+	u64 rx_symbol_err;
+	u64 rx_crc_err;
+	u64 rx_align_err;
+	u64 rx_mac_ctrl;
+	u64 rx_pause;
+	u64 rx_bcast;
+	u64 rx_mcast;
+	u64 rx_ucast;
+	u64 rx_64_or_less;
+	u64 rx_65_127;
+	u64 rx_128_255;
+	u64 rx_256_511;
+	u64 rx_512_1023;
+	u64 rx_1024_1522;
+	u64 rx_1523_2000;
+	u64 rx_2001;
+	u64 tx_hi;
+	u64 tx_late_col;
+	u64 tx_pause;
+	u64 tx_bcast;
+	u64 tx_mcast;
+	u64 tx_ucast;
+	u64 tx_deferred;
+	u64 tx_total_col;
+	u64 tx_exc_col;
+	u64 tx_single_col;
+	u64 tx_mult_col;
+	u64 rx_total;
+	u64 tx_total;
+	u64 rx_discards;
+	u64 tx_discards;
+};
+
 struct ksz_port_mib {
 	struct mutex cnt_mutex;		/* structure access */
 	u8 cnt_ptr;
@@ -393,12 +433,8 @@ int ksz_switch_resume(struct device *dev);
 void ksz_teardown(struct dsa_switch *ds);
 
 void ksz_init_mib_timer(struct ksz_device *dev);
-bool ksz_is_port_mac_global_usable(struct dsa_switch *ds, int port);
 void ksz_r_mib_stats64(struct ksz_device *dev, int port);
-void ksz88xx_r_mib_stats64(struct ksz_device *dev, int port);
 void ksz_port_stp_state_set(struct dsa_switch *ds, int port, u8 state);
-bool ksz_get_gbit(struct ksz_device *dev, int port);
-phy_interface_t ksz_get_xmii(struct ksz_device *dev, int port, bool gbit);
 extern const struct ksz_chip_data ksz_switch_chips[];
 int ksz_switch_macaddr_get(struct dsa_switch *ds, int port,
 			   struct netlink_ext_ack *extack);
@@ -443,11 +479,15 @@ void ksz_phylink_mac_link_down(struct phylink_config *config,
 			       unsigned int mode,
 			       phy_interface_t interface);
 
-int ksz_max_mtu(struct dsa_switch *ds, int port);
-
 int ksz_set_mac_eee(struct dsa_switch *ds, int port,
 		    struct ethtool_keee *e);
 
+int ksz_ets_band_to_queue(struct tc_ets_qopt_offload_replace_params *p,
+			  int band);
+int ksz_setup_tc_cbs(struct dsa_switch *ds, int port,
+		     struct tc_cbs_qopt_offload *qopt);
+int ksz_tc_ets_validate(struct ksz_device *dev, int port,
+			struct tc_ets_qopt_offload_replace_params *p);
 int ksz_setup_tc(struct dsa_switch *ds, int port,
 		 enum tc_setup_type type, void *type_data);
 
@@ -471,8 +511,37 @@ int ksz_mdio_register(struct ksz_device *dev);
 int ksz_pirq_setup(struct ksz_device *dev, u8 p);
 int ksz_girq_setup(struct ksz_device *dev);
 void ksz_irq_free(struct ksz_irq *kirq);
-int ksz_parse_drive_strength(struct ksz_device *dev);
-int ksz9477_set_default_prio_queue_mapping(struct ksz_device *dev, int port);
+
+struct ksz_driver_strength_prop {
+	const char *name;
+	int offset;
+	int value;
+};
+
+enum ksz_driver_strength_type {
+	KSZ_DRIVER_STRENGTH_HI,
+	KSZ_DRIVER_STRENGTH_LO,
+	KSZ_DRIVER_STRENGTH_IO,
+};
+
+/**
+ * struct ksz_drive_strength - drive strength mapping
+ * @reg_val:	register value
+ * @microamp:	microamp value
+ */
+struct ksz_drive_strength {
+	u32 reg_val;
+	u32 microamp;
+};
+
+void ksz_drive_strength_error(struct ksz_device *dev,
+			      const struct ksz_drive_strength *array,
+			      size_t array_size, int microamp);
+int ksz_drive_strength_to_reg(const struct ksz_drive_strength *array,
+			      size_t array_size, int microamp);
+int ksz_drive_strength_write(struct ksz_device *dev,
+			     struct ksz_driver_strength_prop *props,
+			     int num_props);
 
 /* Common register access functions */
 static inline struct regmap *ksz_regmap_8(struct ksz_device *dev)
diff --git a/drivers/net/dsa/microchip/lan937x_main.c b/drivers/net/dsa/microchip/lan937x_main.c
index 8eb5337b0c102..86ce3a86705f9 100644
--- a/drivers/net/dsa/microchip/lan937x_main.c
+++ b/drivers/net/dsa/microchip/lan937x_main.c
@@ -787,10 +787,6 @@ static int lan937x_setup(struct dsa_switch *ds)
 		return ret;
 	}
 
-	ret = ksz_parse_drive_strength(dev);
-	if (ret)
-		return ret;
-
 	/* set broadcast storm protection 10% rate */
 	storm_mask = BROADCAST_STORM_RATE;
 	storm_rate = (BROADCAST_STORM_VALUE * BROADCAST_STORM_PROT_RATE) / 100;
@@ -983,7 +979,7 @@ const struct dsa_switch_ops lan937x_switch_ops = {
 	.get_stats64		= ksz_get_stats64,
 	.get_pause_stats	= ksz_get_pause_stats,
 	.port_change_mtu	= lan937x_change_mtu,
-	.port_max_mtu		= ksz_max_mtu,
+	.port_max_mtu		= ksz9477_max_mtu,
 	.suspend		= ksz_suspend,
 	.resume			= ksz_resume,
 	.get_ts_info		= ksz_get_ts_info,
diff --git a/drivers/net/dsa/qca/qca8k-leds.c b/drivers/net/dsa/qca/qca8k-leds.c
index ef496e345a4e7..0ada28377f461 100644
--- a/drivers/net/dsa/qca/qca8k-leds.c
+++ b/drivers/net/dsa/qca/qca8k-leds.c
@@ -457,6 +457,9 @@ qca8k_setup_led_ctrl(struct qca8k_priv *priv)
 	int ret;
 
 	ports = device_get_named_child_node(priv->dev, "ports");
+	if (!ports)
+		ports = device_get_named_child_node(priv->dev, "ethernet-ports");
+
 	if (!ports) {
 		dev_info(priv->dev, "No ports node specified in device tree!");
 		return 0;
diff --git a/drivers/net/dsa/realtek/rtl8366rb.c b/drivers/net/dsa/realtek/rtl8366rb.c
index 103039fe30864..d2fa8ff6a5d08 100644
--- a/drivers/net/dsa/realtek/rtl8366rb.c
+++ b/drivers/net/dsa/realtek/rtl8366rb.c
@@ -791,11 +791,87 @@ static int rtl8366rb_setup_all_leds_off(struct realtek_priv *priv)
 	return ret;
 }
 
+static int rtl8366rb_port_set_isolation(struct realtek_priv *priv, int port,
+					u32 mask)
+{
+	/* Bit 0 enables isolation so set this if we enable isolation
+	 * any of the ports an clear it if we disable on all of them.
+	 */
+	if (mask)
+		mask = RTL8366RB_PORT_ISO_PORTS(mask) | RTL8366RB_PORT_ISO_EN;
+
+	return regmap_write(priv->map, RTL8366RB_PORT_ISO(port),
+			    mask);
+}
+
+static int rtl8366rb_port_add_isolation(struct realtek_priv *priv, int port,
+					u32 mask)
+{
+	/* We assume isolation bit is on */
+	return regmap_update_bits(priv->map, RTL8366RB_PORT_ISO(port),
+				  RTL8366RB_PORT_ISO_PORTS(mask),
+				  RTL8366RB_PORT_ISO_PORTS(mask));
+}
+
+static int rtl8366rb_port_remove_isolation(struct realtek_priv *priv, int port,
+					   u32 mask)
+{
+	return regmap_update_bits(priv->map, RTL8366RB_PORT_ISO(port),
+				  RTL8366RB_PORT_ISO_PORTS(mask), 0);
+}
+
+static void
+rtl8366rb_port_stp_state_set(struct dsa_switch *ds, int port, u8 state)
+{
+	struct realtek_priv *priv = ds->priv;
+	u32 val;
+	int i;
+
+	switch (state) {
+	case BR_STATE_DISABLED:
+		val = RTL8366RB_STP_STATE_DISABLED;
+		break;
+	case BR_STATE_BLOCKING:
+	case BR_STATE_LISTENING:
+		val = RTL8366RB_STP_STATE_BLOCKING;
+		break;
+	case BR_STATE_LEARNING:
+		val = RTL8366RB_STP_STATE_LEARNING;
+		break;
+	case BR_STATE_FORWARDING:
+		val = RTL8366RB_STP_STATE_FORWARDING;
+		break;
+	default:
+		dev_err(priv->dev, "unknown bridge state requested\n");
+		return;
+	}
+
+	/* Set the same status for the port on all the FIDs */
+	for (i = 0; i < RTL8366RB_NUM_FIDS; i++) {
+		regmap_update_bits(priv->map, RTL8366RB_STP_STATE_BASE + i,
+				   RTL8366RB_STP_STATE_MASK(port),
+				   RTL8366RB_STP_STATE(port, val));
+	}
+}
+
+static int rtl8366rb_port_set_learning(struct realtek_priv *priv, int port,
+				       bool enable)
+{
+	/* Notice inverted semantics in this register: setting a bit disables
+	 * learning instead of enabling it.
+	 */
+	return regmap_update_bits(priv->map, RTL8366RB_PORT_LEARNDIS_CTRL,
+				  BIT(port), enable ? 0 : BIT(port));
+}
+
 static int rtl8366rb_setup(struct dsa_switch *ds)
 {
 	struct realtek_priv *priv = ds->priv;
 	const struct rtl8366rb_jam_tbl_entry *jam_table;
+	u32 downports_mask = 0;
 	struct rtl8366rb *rb;
+	u32 upports_mask = 0;
+	struct dsa_port *dp;
 	u32 chip_ver = 0;
 	u32 chip_id = 0;
 	int jam_size;
@@ -866,20 +942,58 @@ static int rtl8366rb_setup(struct dsa_switch *ds)
 	if (ret)
 		return ret;
 
-	/* Isolate all user ports so they can only send packets to itself and the CPU port */
-	for (i = 0; i < RTL8366RB_PORT_NUM_CPU; i++) {
-		ret = regmap_write(priv->map, RTL8366RB_PORT_ISO(i),
-				   RTL8366RB_PORT_ISO_PORTS(BIT(RTL8366RB_PORT_NUM_CPU)) |
-				   RTL8366RB_PORT_ISO_EN);
+	/* Start with all ports blocked, including unused ports */
+	dsa_switch_for_each_port(dp, ds) {
+		/* Set the initial STP state of all ports to DISABLED, otherwise
+		 * ports will still forward frames to the CPU despite being
+		 * administratively down by default.
+		 */
+		rtl8366rb_port_stp_state_set(ds, dp->index, BR_STATE_DISABLED);
+
+		/* Start with all ports completely isolated */
+		ret = rtl8366rb_port_set_isolation(priv, dp->index, 0);
+		if (ret)
+			return ret;
+
+		/* Disable learning */
+		ret = rtl8366rb_port_set_learning(priv, dp->index, false);
+		if (ret)
+			return ret;
+
+		/* Collect CPU ports. If we support cascade switches, it should
+		 * also include the upstream DSA ports.
+		 */
+		if (!dsa_port_is_cpu(dp))
+			continue;
+
+		upports_mask |= BIT(dp->index);
+	}
+
+	/* Configure user ports */
+	dsa_switch_for_each_port(dp, ds) {
+		if (!dsa_port_is_user(dp))
+			continue;
+
+		/* Forward only to the CPU */
+		ret = rtl8366rb_port_set_isolation(priv, dp->index, upports_mask);
+		if (ret)
+			return ret;
+
+		/* If we support cascade switches, it should also include the
+		 * downstream DSA ports.
+		 */
+		downports_mask |= BIT(dp->index);
+	}
+
+	/* Configure CPU ports. If we support cascade switches, this will also
+	 * include DSA ports.
+	 */
+	dsa_switch_for_each_cpu_port(dp, ds) {
+		/* Forward to all user ports */
+		ret = rtl8366rb_port_set_isolation(priv, dp->index, downports_mask);
 		if (ret)
 			return ret;
 	}
-	/* CPU port can send packets to all ports */
-	ret = regmap_write(priv->map, RTL8366RB_PORT_ISO(RTL8366RB_PORT_NUM_CPU),
-			   RTL8366RB_PORT_ISO_PORTS(dsa_user_ports(ds)) |
-			   RTL8366RB_PORT_ISO_EN);
-	if (ret)
-		return ret;
 
 	/* Set up the "green ethernet" feature */
 	ret = rtl8366rb_jam_table(rtl8366rb_green_jam,
@@ -938,12 +1052,6 @@ static int rtl8366rb_setup(struct dsa_switch *ds)
 			rb->max_mtu[i] = ETH_DATA_LEN;
 	}
 
-	/* Disable learning for all ports */
-	ret = regmap_write(priv->map, RTL8366RB_PORT_LEARNDIS_CTRL,
-			   RTL8366RB_PORT_ALL);
-	if (ret)
-		return ret;
-
 	/* Enable auto ageing for all ports */
 	ret = regmap_write(priv->map, RTL8366RB_SECURITY_CTRL, 0);
 	if (ret)
@@ -1184,70 +1292,6 @@ rtl8366rb_port_disable(struct dsa_switch *ds, int port)
 		return;
 }
 
-static int
-rtl8366rb_port_bridge_join(struct dsa_switch *ds, int port,
-			   struct dsa_bridge bridge,
-			   bool *tx_fwd_offload,
-			   struct netlink_ext_ack *extack)
-{
-	struct realtek_priv *priv = ds->priv;
-	unsigned int port_bitmap = 0;
-	int ret, i;
-
-	/* Loop over all other ports than the current one */
-	for (i = 0; i < RTL8366RB_PORT_NUM_CPU; i++) {
-		/* Current port handled last */
-		if (i == port)
-			continue;
-		/* Not on this bridge */
-		if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge))
-			continue;
-		/* Join this port to each other port on the bridge */
-		ret = regmap_update_bits(priv->map, RTL8366RB_PORT_ISO(i),
-					 RTL8366RB_PORT_ISO_PORTS(BIT(port)),
-					 RTL8366RB_PORT_ISO_PORTS(BIT(port)));
-		if (ret)
-			dev_err(priv->dev, "failed to join port %d\n", port);
-
-		port_bitmap |= BIT(i);
-	}
-
-	/* Set the bits for the ports we can access */
-	return regmap_update_bits(priv->map, RTL8366RB_PORT_ISO(port),
-				  RTL8366RB_PORT_ISO_PORTS(port_bitmap),
-				  RTL8366RB_PORT_ISO_PORTS(port_bitmap));
-}
-
-static void
-rtl8366rb_port_bridge_leave(struct dsa_switch *ds, int port,
-			    struct dsa_bridge bridge)
-{
-	struct realtek_priv *priv = ds->priv;
-	unsigned int port_bitmap = 0;
-	int ret, i;
-
-	/* Loop over all other ports than this one */
-	for (i = 0; i < RTL8366RB_PORT_NUM_CPU; i++) {
-		/* Current port handled last */
-		if (i == port)
-			continue;
-		/* Not on this bridge */
-		if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge))
-			continue;
-		/* Remove this port from any other port on the bridge */
-		ret = regmap_update_bits(priv->map, RTL8366RB_PORT_ISO(i),
-					 RTL8366RB_PORT_ISO_PORTS(BIT(port)), 0);
-		if (ret)
-			dev_err(priv->dev, "failed to leave port %d\n", port);
-
-		port_bitmap |= BIT(i);
-	}
-
-	/* Clear the bits for the ports we can not access, leave ourselves */
-	regmap_update_bits(priv->map, RTL8366RB_PORT_ISO(port),
-			   RTL8366RB_PORT_ISO_PORTS(port_bitmap), 0);
-}
-
 /**
  * rtl8366rb_drop_untagged() - make the switch drop untagged and C-tagged frames
  * @priv: SMI state container
@@ -1306,59 +1350,6 @@ rtl8366rb_port_pre_bridge_flags(struct dsa_switch *ds, int port,
 	return 0;
 }
 
-static int
-rtl8366rb_port_bridge_flags(struct dsa_switch *ds, int port,
-			    struct switchdev_brport_flags flags,
-			    struct netlink_ext_ack *extack)
-{
-	struct realtek_priv *priv = ds->priv;
-	int ret;
-
-	if (flags.mask & BR_LEARNING) {
-		ret = regmap_update_bits(priv->map, RTL8366RB_PORT_LEARNDIS_CTRL,
-					 BIT(port),
-					 (flags.val & BR_LEARNING) ? 0 : BIT(port));
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static void
-rtl8366rb_port_stp_state_set(struct dsa_switch *ds, int port, u8 state)
-{
-	struct realtek_priv *priv = ds->priv;
-	u32 val;
-	int i;
-
-	switch (state) {
-	case BR_STATE_DISABLED:
-		val = RTL8366RB_STP_STATE_DISABLED;
-		break;
-	case BR_STATE_BLOCKING:
-	case BR_STATE_LISTENING:
-		val = RTL8366RB_STP_STATE_BLOCKING;
-		break;
-	case BR_STATE_LEARNING:
-		val = RTL8366RB_STP_STATE_LEARNING;
-		break;
-	case BR_STATE_FORWARDING:
-		val = RTL8366RB_STP_STATE_FORWARDING;
-		break;
-	default:
-		dev_err(priv->dev, "unknown bridge state requested\n");
-		return;
-	}
-
-	/* Set the same status for the port on all the FIDs */
-	for (i = 0; i < RTL8366RB_NUM_FIDS; i++) {
-		regmap_update_bits(priv->map, RTL8366RB_STP_STATE_BASE + i,
-				   RTL8366RB_STP_STATE_MASK(port),
-				   RTL8366RB_STP_STATE(port, val));
-	}
-}
-
 static void
 rtl8366rb_port_fast_age(struct dsa_switch *ds, int port)
 {
@@ -1801,15 +1792,15 @@ static const struct dsa_switch_ops rtl8366rb_switch_ops = {
 	.get_strings = rtl8366_get_strings,
 	.get_ethtool_stats = rtl8366_get_ethtool_stats,
 	.get_sset_count = rtl8366_get_sset_count,
-	.port_bridge_join = rtl8366rb_port_bridge_join,
-	.port_bridge_leave = rtl8366rb_port_bridge_leave,
+	.port_bridge_join = rtl83xx_port_bridge_join,
+	.port_bridge_leave = rtl83xx_port_bridge_leave,
 	.port_vlan_filtering = rtl8366rb_vlan_filtering,
 	.port_vlan_add = rtl8366_vlan_add,
 	.port_vlan_del = rtl8366_vlan_del,
 	.port_enable = rtl8366rb_port_enable,
 	.port_disable = rtl8366rb_port_disable,
 	.port_pre_bridge_flags = rtl8366rb_port_pre_bridge_flags,
-	.port_bridge_flags = rtl8366rb_port_bridge_flags,
+	.port_bridge_flags = rtl83xx_port_bridge_flags,
 	.port_stp_state_set = rtl8366rb_port_stp_state_set,
 	.port_fast_age = rtl8366rb_port_fast_age,
 	.port_change_mtu = rtl8366rb_change_mtu,
@@ -1830,6 +1821,9 @@ static const struct realtek_ops rtl8366rb_ops = {
 	.is_vlan_valid	= rtl8366rb_is_vlan_valid,
 	.enable_vlan	= rtl8366rb_enable_vlan,
 	.enable_vlan4k	= rtl8366rb_enable_vlan4k,
+	.port_add_isolation = rtl8366rb_port_add_isolation,
+	.port_remove_isolation = rtl8366rb_port_remove_isolation,
+	.port_set_learning = rtl8366rb_port_set_learning,
 	.phy_read	= rtl8366rb_phy_read,
 	.phy_write	= rtl8366rb_phy_write,
 };
diff --git a/drivers/net/dsa/realtek/rtl83xx.c b/drivers/net/dsa/realtek/rtl83xx.c
index 71124ecca92f2..90843d52c5a85 100644
--- a/drivers/net/dsa/realtek/rtl83xx.c
+++ b/drivers/net/dsa/realtek/rtl83xx.c
@@ -356,9 +356,6 @@ int rtl83xx_port_bridge_join(struct dsa_switch *ds, int port,
 	if (!priv->ops->port_add_isolation)
 		return -EOPNOTSUPP;
 
-	if (!priv->ops->port_set_learning)
-		return -EOPNOTSUPP;
-
 	dev_dbg(priv->dev, "bridge %d join port %d\n", bridge.num, port);
 
 	/* Add this port to the isolation group of every other port
@@ -396,9 +393,11 @@ int rtl83xx_port_bridge_join(struct dsa_switch *ds, int port,
 			goto undo_self_isolation;
 	}
 
-	ret = priv->ops->port_set_learning(priv, port, true);
-	if (ret)
-		goto undo_efid;
+	if (priv->ops->port_set_learning) {
+		ret = priv->ops->port_set_learning(priv, port, true);
+		if (ret)
+			goto undo_efid;
+	}
 
 	return 0;
 
@@ -443,9 +442,6 @@ void rtl83xx_port_bridge_leave(struct dsa_switch *ds, int port,
 	if (!priv->ops->port_remove_isolation)
 		return;
 
-	if (!priv->ops->port_set_learning)
-		return;
-
 	dev_dbg(priv->dev, "bridge %d leave port %d\n", bridge.num, port);
 
 	/* Remove this port from the isolation group of every other
@@ -474,11 +470,13 @@ void rtl83xx_port_bridge_leave(struct dsa_switch *ds, int port,
 	 * downstream DSA ports from the isolation group.
 	 */
 
-	ret = priv->ops->port_set_learning(priv, port, false);
-	if (ret)
-		dev_err(priv->dev,
-			"failed to disable learning on port %d: %pe\n",
-			port, ERR_PTR(ret));
+	if (priv->ops->port_set_learning) {
+		ret = priv->ops->port_set_learning(priv, port, false);
+		if (ret)
+			dev_err(priv->dev,
+				"failed to disable learning on port %d: %pe\n",
+				port, ERR_PTR(ret));
+	}
 
 	/* Remove those ports from the isolation group of this port */
 	ret = priv->ops->port_remove_isolation(priv, port, mask);
diff --git a/drivers/net/ethernet/airoha/airoha_ppe.c b/drivers/net/ethernet/airoha/airoha_ppe.c
index e7c78293002ab..fdb973fc779c8 100644
--- a/drivers/net/ethernet/airoha/airoha_ppe.c
+++ b/drivers/net/ethernet/airoha/airoha_ppe.c
@@ -331,7 +331,7 @@ static int airoha_ppe_foe_entry_prepare(struct airoha_eth *eth,
 					struct airoha_foe_entry *hwe,
 					struct net_device *netdev, int type,
 					struct airoha_flow_data *data,
-					int l4proto)
+					int l4proto, u8 priority)
 {
 	u32 qdata = FIELD_PREP(AIROHA_FOE_SHAPER_ID, 0x7f), ports_pad, val;
 	int wlan_etype = -EINVAL, dsa_port = airoha_get_dsa_port(&netdev);
@@ -386,7 +386,9 @@ static int airoha_ppe_foe_entry_prepare(struct airoha_eth *eth,
 			 */
 			channel = dsa_port >= 0 ? dsa_port : port->id;
 			channel = channel % AIROHA_NUM_QOS_CHANNELS;
-			qdata |= FIELD_PREP(AIROHA_FOE_CHANNEL, channel);
+			priority = priority % AIROHA_NUM_QOS_QUEUES;
+			qdata |= FIELD_PREP(AIROHA_FOE_CHANNEL, channel) |
+				 FIELD_PREP(AIROHA_FOE_QID, priority);
 
 			val |= FIELD_PREP(AIROHA_FOE_IB2_PSE_PORT, pse_port) |
 			       AIROHA_FOE_IB2_PSE_QOS;
@@ -1079,10 +1081,10 @@ static int airoha_ppe_flow_offload_replace(struct airoha_eth *eth,
 	struct airoha_flow_data data = {};
 	struct net_device *odev = NULL;
 	struct flow_action_entry *act;
+	u8 l4proto = 0, priority = 0;
 	struct airoha_foe_entry hwe;
 	int err, i, offload_type;
 	u16 addr_type = 0;
-	u8 l4proto = 0;
 
 	if (rhashtable_lookup(&eth->flow_table, &f->cookie,
 			      airoha_flow_table_params))
@@ -1177,7 +1179,7 @@ static int airoha_ppe_flow_offload_replace(struct airoha_eth *eth,
 		return -EINVAL;
 
 	err = airoha_ppe_foe_entry_prepare(eth, &hwe, odev, offload_type,
-					   &data, l4proto);
+					   &data, l4proto, priority);
 	if (err)
 		return err;
 
diff --git a/drivers/net/ethernet/allwinner/sun4i-emac.c b/drivers/net/ethernet/allwinner/sun4i-emac.c
index fc7341a5cbb7d..42174249ef61d 100644
--- a/drivers/net/ethernet/allwinner/sun4i-emac.c
+++ b/drivers/net/ethernet/allwinner/sun4i-emac.c
@@ -15,7 +15,7 @@
 #include <linux/clk.h>
 #include <linux/etherdevice.h>
 #include <linux/ethtool.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/mii.h>
diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c
index 3b2951030a389..507db46daf2b8 100644
--- a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c
+++ b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c
@@ -7,7 +7,7 @@
  *	    Keyur Chudgar <kchudgar@apm.com>
  */
 
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include "xgene_enet_main.h"
 #include "xgene_enet_hw.h"
 #include "xgene_enet_sgmac.h"
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.c
index 07a908a2c72f2..d560524d317d9 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.c
@@ -26,6 +26,7 @@
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
 #include <linux/crc32c.h>
+#include <linux/slab.h>
 #include "bnx2x.h"
 #include "bnx2x_cmn.h"
 #include "bnx2x_sp.h"
@@ -2664,7 +2665,7 @@ static void bnx2x_free_groups(struct list_head *mcast_group_list)
 				      struct bnx2x_mcast_elem_group,
 				      mcast_group_link);
 		list_del(&current_mcast_group->mcast_group_link);
-		free_page((unsigned long)current_mcast_group);
+		kfree(current_mcast_group);
 	}
 }
 
@@ -2713,8 +2714,7 @@ static int bnx2x_mcast_enqueue_cmd(struct bnx2x *bp,
 				total_elems = BNX2X_MCAST_BINS_NUM;
 		}
 		while (total_elems > 0) {
-			elem_group = (struct bnx2x_mcast_elem_group *)
-				     __get_free_page(GFP_ATOMIC | __GFP_ZERO);
+			elem_group = kzalloc(PAGE_SIZE, GFP_ATOMIC);
 			if (!elem_group) {
 				bnx2x_free_groups(&new_cmd->group_head);
 				kfree(new_cmd);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
index 5c751933da6a9..a515c368bac01 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
@@ -566,6 +566,18 @@ void bnxt_aux_devices_init(struct bnxt *bp)
 		if (!aux_priv)
 			goto next_auxdev;
 
+		edev = kzalloc_obj(*edev);
+		if (!edev)
+			goto aux_priv_free;
+		aux_priv->edev = edev;
+		bnxt_set_edev_info(edev, bp);
+
+		ulp = kzalloc_obj(*ulp);
+		if (!ulp)
+			goto edev_free;
+		edev->ulp_tbl = ulp;
+		aux_priv->id = idx;
+
 		aux_dev = &aux_priv->aux_dev;
 		aux_dev->id = bp->auxdev_id;
 		aux_dev->name = bnxt_aux_devices[idx].name;
@@ -573,37 +585,26 @@ void bnxt_aux_devices_init(struct bnxt *bp)
 		aux_dev->dev.release = bnxt_aux_dev_release;
 
 		rc = auxiliary_device_init(aux_dev);
-		if (rc) {
-			kfree(aux_priv);
-			goto next_auxdev;
-		}
+		if (rc)
+			goto ulp_free;
 		bp->aux_priv[idx] = aux_priv;
 
 		/* From this point, all cleanup will happen via the .release
 		 * callback & any error unwinding will need to include a call
 		 * to auxiliary_device_uninit.
 		 */
-		edev = kzalloc_obj(*edev);
-		if (!edev)
-			goto aux_dev_uninit;
-
-		aux_priv->edev = edev;
-		bnxt_set_edev_info(edev, bp);
-
-		ulp = kzalloc_obj(*ulp);
-		if (!ulp)
-			goto aux_dev_uninit;
-
-		edev->ulp_tbl = ulp;
 		bp->edev[idx] = edev;
 		if (idx == BNXT_AUXDEV_RDMA)
 			bp->ulp_num_msix_want = bnxt_set_dflt_ulp_msix(bp);
-		aux_priv->id = idx;
 		bnxt_auxdev_set_state(bp, idx, BNXT_ADEV_STATE_INIT);
 
 		continue;
-aux_dev_uninit:
-		auxiliary_device_uninit(aux_dev);
+ulp_free:
+		kfree(ulp);
+edev_free:
+		kfree(edev);
+aux_priv_free:
+		kfree(aux_priv);
 next_auxdev:
 		if (idx == BNXT_AUXDEV_RDMA)
 			bp->flags &= ~BNXT_FLAG_ROCE_CAP;
diff --git a/drivers/net/ethernet/broadcom/genet/bcmmii.c b/drivers/net/ethernet/broadcom/genet/bcmmii.c
index a4e0d5a682687..0f0dbabfaabbc 100644
--- a/drivers/net/ethernet/broadcom/genet/bcmmii.c
+++ b/drivers/net/ethernet/broadcom/genet/bcmmii.c
@@ -490,8 +490,9 @@ static int bcmgenet_mii_register(struct bcmgenet_priv *priv)
 	/* Retain this platform_device pointer for later cleanup */
 	priv->mii_pdev = ppdev;
 	ppdev->dev.parent = &pdev->dev;
+
 	if (dn)
-		ppdev->dev.of_node = bcmgenet_mii_of_find_mdio(priv);
+		platform_device_set_of_node(ppdev, bcmgenet_mii_of_find_mdio(priv));
 	else
 		ppd.phy_mask = ~0;
 
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
index 6871127427fad..29d7b786e51eb 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
@@ -33,6 +33,7 @@
  */
 
 #include <linux/delay.h>
+#include <linux/string_choices.h>
 #include "cxgb4.h"
 #include "t4_regs.h"
 #include "t4_values.h"
@@ -4867,7 +4868,7 @@ static void mem_intr_handler(struct adapter *adapter, int idx)
 		if (printk_ratelimit())
 			dev_warn(adapter->pdev_dev,
 				 "%u %s correctable ECC data error%s\n",
-				 cnt, name[idx], cnt > 1 ? "s" : "");
+				 cnt, name[idx], str_plural(cnt));
 	}
 	if (v & ECC_UE_INT_CAUSE_F)
 		dev_alert(adapter->pdev_dev,
diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c
index 858ba844ac516..8320b26c3f726 100644
--- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c
+++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c
@@ -25,6 +25,9 @@
 
 #define DEFAULT_VLAN_ID			1
 
+static struct notifier_block dpaa2_switch_port_switchdev_nb;
+static struct notifier_block dpaa2_switch_port_switchdev_blocking_nb;
+
 static u16 dpaa2_switch_port_get_fdb_id(struct ethsw_port_priv *port_priv)
 {
 	return port_priv->fdb->fdb_id;
@@ -51,6 +54,17 @@ dpaa2_switch_filter_block_get_unused(struct ethsw_core *ethsw)
 	return NULL;
 }
 
+static struct dpaa2_switch_lag *
+dpaa2_switch_lag_get_unused(struct ethsw_core *ethsw)
+{
+	int i;
+
+	for (i = 0; i < ethsw->sw_attr.num_ifs; i++)
+		if (!ethsw->lags[i].in_use)
+			return &ethsw->lags[i];
+	return NULL;
+}
+
 static bool dpaa2_switch_fdb_in_use_by_others(struct ethsw_core *ethsw,
 					      struct dpaa2_switch_fdb *fdb,
 					      struct ethsw_port_priv *except)
@@ -71,9 +85,9 @@ static struct dpaa2_switch_fdb *
 dpaa2_switch_fdb_for_join(struct ethsw_port_priv *port_priv,
 			  struct net_device *upper_dev)
 {
-	struct ethsw_port_priv *other_port_priv;
-	struct net_device *other_dev;
-	struct list_head *iter;
+	struct ethsw_port_priv *other_port_priv = NULL;
+	struct net_device *other_dev, *other_dev2;
+	struct list_head *iter, *iter2;
 
 	/* The below call to netdev_for_each_lower_dev() demands the RTNL lock
 	 * being held. Assert on it so that it's easier to catch new code
@@ -82,17 +96,32 @@ dpaa2_switch_fdb_for_join(struct ethsw_port_priv *port_priv,
 	ASSERT_RTNL();
 
 	/* If part of a bridge, use the FDB of the first dpaa2 switch interface
-	 * to be present in that bridge
+	 * to be present in that bridge. The search descends one level through
+	 * a bridged bond's lowers as well.
 	 */
 	netdev_for_each_lower_dev(upper_dev, other_dev, iter) {
-		if (!dpaa2_switch_port_dev_check(other_dev))
-			continue;
+		if (netif_is_lag_master(other_dev)) {
+			netdev_for_each_lower_dev(other_dev, other_dev2, iter2) {
+				if (!dpaa2_switch_port_dev_check(other_dev2))
+					continue;
 
-		if (other_dev == port_priv->netdev)
-			continue;
+				if (other_dev2 == port_priv->netdev)
+					continue;
 
-		other_port_priv = netdev_priv(other_dev);
-		return other_port_priv->fdb;
+				other_port_priv = netdev_priv(other_dev2);
+				break;
+			}
+		} else {
+			if (!dpaa2_switch_port_dev_check(other_dev))
+				continue;
+
+			if (other_dev == port_priv->netdev)
+				continue;
+
+			other_port_priv = netdev_priv(other_dev);
+		}
+		if (other_port_priv)
+			return other_port_priv->fdb;
 	}
 
 	return port_priv->fdb;
@@ -537,6 +566,103 @@ static int dpaa2_switch_port_fdb_del_mc(struct ethsw_port_priv *port_priv,
 	return err;
 }
 
+static int dpaa2_switch_port_fdb_add(struct ethsw_port_priv *port_priv,
+				     const unsigned char *addr)
+{
+	int err;
+
+	if (is_unicast_ether_addr(addr))
+		err = dpaa2_switch_port_fdb_add_uc(port_priv, addr);
+	else
+		err = dpaa2_switch_port_fdb_add_mc(port_priv, addr);
+
+	return err;
+}
+
+static int dpaa2_switch_port_fdb_del(struct ethsw_port_priv *port_priv,
+				     const unsigned char *addr)
+{
+	if (is_unicast_ether_addr(addr))
+		return dpaa2_switch_port_fdb_del_uc(port_priv, addr);
+	else
+		return dpaa2_switch_port_fdb_del_mc(port_priv, addr);
+}
+
+static struct dpaa2_mac_addr *
+dpaa2_switch_mac_addr_find(struct list_head *addr_list,
+			   const unsigned char *addr, u16 vid)
+{
+	struct dpaa2_mac_addr *a;
+
+	list_for_each_entry(a, addr_list, list)
+		if (ether_addr_equal(a->addr, addr) && a->vid == vid)
+			return a;
+
+	return NULL;
+}
+
+static int dpaa2_switch_lag_fdb_add(struct dpaa2_switch_lag *lag,
+				    const unsigned char *addr, u16 vid)
+{
+	struct ethsw_port_priv *port_priv = lag->primary;
+	struct dpaa2_mac_addr *a;
+	int err = 0;
+
+	mutex_lock(&lag->fdb_lock);
+
+	a = dpaa2_switch_mac_addr_find(&lag->fdbs, addr, vid);
+	if (a) {
+		refcount_inc(&a->refcount);
+		goto out;
+	}
+
+	a = kzalloc(sizeof(*a), GFP_KERNEL);
+	if (!a) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	err = dpaa2_switch_port_fdb_add(port_priv, addr);
+	if (err) {
+		kfree(a);
+		goto out;
+	}
+
+	ether_addr_copy(a->addr, addr);
+	a->vid = vid;
+	refcount_set(&a->refcount, 1);
+	list_add_tail(&a->list, &lag->fdbs);
+
+out:
+	mutex_unlock(&lag->fdb_lock);
+
+	return err;
+}
+
+static void dpaa2_switch_lag_fdb_del(struct dpaa2_switch_lag *lag,
+				     const unsigned char *addr, u16 vid)
+{
+	struct ethsw_port_priv *port_priv = lag->primary;
+	struct dpaa2_mac_addr *a;
+
+	mutex_lock(&lag->fdb_lock);
+
+	a = dpaa2_switch_mac_addr_find(&lag->fdbs, addr, vid);
+	if (!a)
+		goto out;
+
+	if (!refcount_dec_and_test(&a->refcount))
+		goto out;
+
+	list_del(&a->list);
+	kfree(a);
+
+	dpaa2_switch_port_fdb_del(port_priv, addr);
+
+out:
+	mutex_unlock(&lag->fdb_lock);
+}
+
 static void dpaa2_switch_port_get_stats(struct net_device *netdev,
 					struct rtnl_link_stats64 *stats)
 {
@@ -1485,6 +1611,33 @@ bool dpaa2_switch_port_dev_check(const struct net_device *netdev)
 	return netdev->netdev_ops == &dpaa2_switch_port_ops;
 }
 
+static bool dpaa2_switch_foreign_dev_check(const struct net_device *dev,
+					   const struct net_device *foreign_dev)
+{
+	struct ethsw_port_priv *port_priv = netdev_priv(dev);
+	struct ethsw_core *ethsw = port_priv->ethsw_data;
+	struct ethsw_port_priv *other_port;
+	int i;
+
+	if (netif_is_bridge_master(foreign_dev))
+		if (port_priv->fdb->bridge_dev == foreign_dev)
+			return false;
+
+	if (netif_is_bridge_port(foreign_dev)) {
+		for (i = 0; i < ethsw->sw_attr.num_ifs; i++) {
+			other_port = ethsw->ports[i];
+
+			if (!other_port)
+				continue;
+			if (dpaa2_switch_port_offloads_bridge_port(other_port,
+								   foreign_dev))
+				return false;
+		}
+	}
+
+	return true;
+}
+
 static int dpaa2_switch_port_connect_mac(struct ethsw_port_priv *port_priv)
 {
 	struct fsl_mc_device *dpsw_port_dev, *dpmac_dev;
@@ -1860,51 +2013,32 @@ int dpaa2_switch_port_vlans_add(struct net_device *netdev,
 					  vlan->changed);
 }
 
-static int dpaa2_switch_port_lookup_address(struct net_device *netdev, int is_uc,
-					    const unsigned char *addr)
-{
-	struct netdev_hw_addr_list *list = (is_uc) ? &netdev->uc : &netdev->mc;
-	struct netdev_hw_addr *ha;
-
-	netif_addr_lock_bh(netdev);
-	list_for_each_entry(ha, &list->list, list) {
-		if (ether_addr_equal(ha->addr, addr)) {
-			netif_addr_unlock_bh(netdev);
-			return 1;
-		}
-	}
-	netif_addr_unlock_bh(netdev);
-	return 0;
-}
-
 static int dpaa2_switch_port_mdb_add(struct net_device *netdev,
 				     const struct switchdev_obj_port_mdb *mdb)
 {
 	struct ethsw_port_priv *port_priv = netdev_priv(netdev);
-	int err;
+	struct dpaa2_switch_lag *lag;
 
-	/* Check if address is already set on this port */
-	if (dpaa2_switch_port_lookup_address(netdev, 0, mdb->addr))
-		return -EEXIST;
-
-	err = dpaa2_switch_port_fdb_add_mc(port_priv, mdb->addr);
-	if (err)
-		return err;
-
-	err = dev_mc_add(netdev, mdb->addr);
-	if (err) {
-		netdev_err(netdev, "dev_mc_add err %d\n", err);
-		dpaa2_switch_port_fdb_del_mc(port_priv, mdb->addr);
-	}
-
-	return err;
+	lag = rtnl_dereference(port_priv->lag);
+	if (lag)
+		return dpaa2_switch_lag_fdb_add(lag, mdb->addr, mdb->vid);
+	else
+		return dpaa2_switch_port_fdb_add(port_priv, mdb->addr);
 }
 
-static int dpaa2_switch_port_obj_add(struct net_device *netdev,
-				     const struct switchdev_obj *obj)
+static int dpaa2_switch_port_obj_add(struct net_device *netdev, const void *ctx,
+				     const struct switchdev_obj *obj,
+				     struct netlink_ext_ack *extack)
 {
+	struct ethsw_port_priv *port_priv = netdev_priv(netdev);
 	int err;
 
+	if (ctx && ctx != port_priv)
+		return 0;
+
+	if (!dpaa2_switch_port_offloads_bridge_port(port_priv, obj->orig_dev))
+		return -EOPNOTSUPP;
+
 	switch (obj->id) {
 	case SWITCHDEV_OBJ_ID_PORT_VLAN:
 		err = dpaa2_switch_port_vlans_add(netdev,
@@ -2000,29 +2134,29 @@ static int dpaa2_switch_port_mdb_del(struct net_device *netdev,
 				     const struct switchdev_obj_port_mdb *mdb)
 {
 	struct ethsw_port_priv *port_priv = netdev_priv(netdev);
-	int err;
-
-	if (!dpaa2_switch_port_lookup_address(netdev, 0, mdb->addr))
-		return -ENOENT;
-
-	err = dpaa2_switch_port_fdb_del_mc(port_priv, mdb->addr);
-	if (err)
-		return err;
+	struct dpaa2_switch_lag *lag;
 
-	err = dev_mc_del(netdev, mdb->addr);
-	if (err) {
-		netdev_err(netdev, "dev_mc_del err %d\n", err);
-		return err;
-	}
+	lag = rtnl_dereference(port_priv->lag);
+	if (lag)
+		dpaa2_switch_lag_fdb_del(lag, mdb->addr, mdb->vid);
+	else
+		dpaa2_switch_port_fdb_del(port_priv, mdb->addr);
 
-	return err;
+	return 0;
 }
 
-static int dpaa2_switch_port_obj_del(struct net_device *netdev,
+static int dpaa2_switch_port_obj_del(struct net_device *netdev, const void *ctx,
 				     const struct switchdev_obj *obj)
 {
+	struct ethsw_port_priv *port_priv = netdev_priv(netdev);
 	int err;
 
+	if (ctx && ctx != port_priv)
+		return 0;
+
+	if (!dpaa2_switch_port_offloads_bridge_port(port_priv, obj->orig_dev))
+		return -EOPNOTSUPP;
+
 	switch (obj->id) {
 	case SWITCHDEV_OBJ_ID_PORT_VLAN:
 		err = dpaa2_switch_port_vlans_del(netdev, SWITCHDEV_OBJ_PORT_VLAN(obj));
@@ -2048,6 +2182,21 @@ static int dpaa2_switch_port_attr_set_event(struct net_device *netdev,
 	return notifier_from_errno(err);
 }
 
+static struct net_device *
+dpaa2_switch_port_to_bridge_port(struct ethsw_port_priv *port_priv)
+{
+	struct dpaa2_switch_lag *lag;
+
+	if (!port_priv->fdb->bridge_dev)
+		return NULL;
+
+	lag = rtnl_dereference(port_priv->lag);
+	if (lag)
+		return lag->bond_dev;
+
+	return port_priv->netdev;
+}
+
 static int dpaa2_switch_port_bridge_join(struct net_device *netdev,
 					 struct net_device *upper_dev,
 					 struct netlink_ext_ack *extack)
@@ -2055,6 +2204,7 @@ static int dpaa2_switch_port_bridge_join(struct net_device *netdev,
 	struct ethsw_port_priv *port_priv = netdev_priv(netdev);
 	struct dpaa2_switch_fdb *old_fdb = port_priv->fdb;
 	struct ethsw_core *ethsw = port_priv->ethsw_data;
+	struct net_device *brport_dev;
 	bool learn_ena;
 	int err;
 
@@ -2066,7 +2216,8 @@ static int dpaa2_switch_port_bridge_join(struct net_device *netdev,
 	dpaa2_switch_port_set_fdb(port_priv, upper_dev, true);
 
 	/* Inherit the initial bridge port learning state */
-	learn_ena = br_port_flag_is_set(netdev, BR_LEARNING);
+	brport_dev = dpaa2_switch_port_to_bridge_port(port_priv);
+	learn_ena = br_port_flag_is_set(brport_dev, BR_LEARNING);
 	err = dpaa2_switch_port_set_learning(port_priv, learn_ena);
 	port_priv->learn_ena = learn_ena;
 
@@ -2080,8 +2231,11 @@ static int dpaa2_switch_port_bridge_join(struct net_device *netdev,
 	if (err)
 		goto err_egress_flood;
 
-	err = switchdev_bridge_port_offload(netdev, netdev, NULL,
-					    NULL, NULL, false, extack);
+	brport_dev = dpaa2_switch_port_to_bridge_port(port_priv);
+	err = switchdev_bridge_port_offload(brport_dev, netdev, port_priv,
+					    &dpaa2_switch_port_switchdev_nb,
+					    &dpaa2_switch_port_switchdev_blocking_nb,
+					    false, extack);
 	if (err)
 		goto err_switchdev_offload;
 
@@ -2115,7 +2269,22 @@ static int dpaa2_switch_port_restore_rxvlan(struct net_device *vdev, int vid, vo
 
 static void dpaa2_switch_port_pre_bridge_leave(struct net_device *netdev)
 {
-	switchdev_bridge_port_unoffload(netdev, NULL, NULL, NULL);
+	struct ethsw_port_priv *port_priv = netdev_priv(netdev);
+	struct ethsw_core *ethsw = port_priv->ethsw_data;
+	struct net_device *brport_dev;
+
+	brport_dev = dpaa2_switch_port_to_bridge_port(port_priv);
+	if (!brport_dev)
+		return;
+
+	switchdev_bridge_port_unoffload(brport_dev, port_priv,
+					&dpaa2_switch_port_switchdev_nb,
+					&dpaa2_switch_port_switchdev_blocking_nb);
+
+	/* Make sure that any FDB add/del operations are completed before the
+	 * bridge layout changes
+	 */
+	flush_workqueue(ethsw->workqueue);
 }
 
 static int dpaa2_switch_port_bridge_leave(struct net_device *netdev)
@@ -2177,30 +2346,53 @@ static int dpaa2_switch_port_bridge_leave(struct net_device *netdev)
 					  false);
 }
 
+static int
+dpaa2_switch_have_vlan_upper(struct net_device *upper_dev,
+			     __always_unused struct netdev_nested_priv *priv)
+{
+	return is_vlan_dev(upper_dev);
+}
+
 static int dpaa2_switch_prevent_bridging_with_8021q_upper(struct net_device *netdev)
 {
-	struct net_device *upper_dev;
-	struct list_head *iter;
+	struct netdev_nested_priv priv = {};
 
 	/* RCU read lock not necessary because we have write-side protection
-	 * (rtnl_mutex), however a non-rcu iterator does not exist.
+	 * (rtnl_mutex), however a non-rcu iterator does not exist. Walk the
+	 * entire upper chain so that a VLAN device stacked on a intermediate
+	 * bond is caught too.
 	 */
-	netdev_for_each_upper_dev_rcu(netdev, upper_dev, iter)
-		if (is_vlan_dev(upper_dev))
-			return -EOPNOTSUPP;
+	if (netdev_walk_all_upper_dev_rcu(netdev, dpaa2_switch_have_vlan_upper,
+					  &priv))
+		return -EOPNOTSUPP;
 
 	return 0;
 }
 
+static int dpaa2_switch_check_dpsw_instance(struct net_device *dev,
+					    struct netdev_nested_priv *priv)
+{
+	struct ethsw_port_priv *port_priv = (struct ethsw_port_priv *)priv->data;
+	struct ethsw_port_priv *other_priv = netdev_priv(dev);
+
+	if (!dpaa2_switch_port_dev_check(dev))
+		return 0;
+
+	if (other_priv->ethsw_data == port_priv->ethsw_data)
+		return 0;
+
+	return 1;
+}
+
 static int
 dpaa2_switch_prechangeupper_sanity_checks(struct net_device *netdev,
 					  struct net_device *upper_dev,
 					  struct netlink_ext_ack *extack)
 {
 	struct ethsw_port_priv *port_priv = netdev_priv(netdev);
-	struct ethsw_port_priv *other_port_priv;
-	struct net_device *other_dev;
-	struct list_head *iter;
+	struct netdev_nested_priv data = {
+		.data = (void *)port_priv,
+	};
 	int err;
 
 	if (!br_vlan_enabled(upper_dev)) {
@@ -2215,6 +2407,70 @@ dpaa2_switch_prechangeupper_sanity_checks(struct net_device *netdev,
 		return err;
 	}
 
+	err = netdev_walk_all_lower_dev(upper_dev,
+					dpaa2_switch_check_dpsw_instance,
+					&data);
+	if (err) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Interface from a different DPSW is in the bridge already");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int dpaa2_switch_pre_lag_join(struct net_device *netdev,
+				     struct net_device *upper_dev,
+				     struct netdev_lag_upper_info *info,
+				     struct netlink_ext_ack *extack)
+{
+	struct ethsw_port_priv *port_priv = netdev_priv(netdev);
+	struct ethsw_core *ethsw = port_priv->ethsw_data;
+	struct ethsw_port_priv *other_port_priv;
+	struct dpaa2_switch_lag *lag = NULL;
+	struct dpsw_lag_cfg cfg = {0};
+	struct net_device *other_dev;
+	int i, num_ifs = 0, err;
+	struct list_head *iter;
+
+	if (!(ethsw->features & ETHSW_FEATURE_LAG_OFFLOAD)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "LAG offload is supported only for DPSW >= v8.13");
+		return -EOPNOTSUPP;
+	}
+
+	if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Can only offload LAG using hash TX type");
+		return -EOPNOTSUPP;
+	}
+
+	if (info->hash_type != NETDEV_LAG_HASH_L23) {
+		NL_SET_ERR_MSG_MOD(extack, "Can only offload L2+L3 Tx hash");
+		return -EOPNOTSUPP;
+	}
+
+	if (!dpaa2_switch_port_has_mac(port_priv)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Only switch interfaces connected to MACs can be under a LAG");
+		return -EINVAL;
+	}
+
+	if (vlan_uses_dev(upper_dev)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Cannot join a LAG upper that has a VLAN");
+		return -EOPNOTSUPP;
+	}
+
+	for (i = 0; i < ethsw->sw_attr.num_ifs; i++) {
+		if (!ethsw->lags[i].in_use)
+			continue;
+		if (ethsw->lags[i].bond_dev != upper_dev)
+			continue;
+		lag = &ethsw->lags[i];
+		break;
+	}
+
 	netdev_for_each_lower_dev(upper_dev, other_dev, iter) {
 		if (!dpaa2_switch_port_dev_check(other_dev))
 			continue;
@@ -2222,9 +2478,248 @@ dpaa2_switch_prechangeupper_sanity_checks(struct net_device *netdev,
 		other_port_priv = netdev_priv(other_dev);
 		if (other_port_priv->ethsw_data != port_priv->ethsw_data) {
 			NL_SET_ERR_MSG_MOD(extack,
-					   "Interface from a different DPSW is in the bridge already");
+					   "Interface from a different DPSW is in the bond already");
 			return -EINVAL;
 		}
+
+		cfg.if_id[num_ifs++] = other_port_priv->idx;
+
+		if (num_ifs >= DPSW_MAX_LAG_IFS) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Cannot add more than 8 DPAA2 switch ports under the same bond");
+			return -EINVAL;
+		}
+	}
+
+	if (lag) {
+		cfg.group_id = lag->id;
+		cfg.if_id[num_ifs++] = port_priv->idx;
+		cfg.num_ifs = num_ifs;
+		cfg.phase = DPSW_LAG_SET_PHASE_CHECK;
+
+		err = dpsw_lag_set(ethsw->mc_io, 0, ethsw->dpsw_handle, &cfg);
+		if (err) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Cannot offload LAG configuration");
+			return -EOPNOTSUPP;
+		}
+	}
+
+	return 0;
+}
+
+static void dpaa2_switch_port_set_lag_group(struct ethsw_port_priv *port_priv,
+					    struct net_device *bond_dev)
+{
+	struct ethsw_core *ethsw = port_priv->ethsw_data;
+	struct ethsw_port_priv *other_port_priv = NULL;
+	struct dpaa2_switch_lag *lag = NULL;
+	struct dpaa2_switch_lag *other_lag;
+	struct net_device *other_dev;
+	struct list_head *iter;
+
+	netdev_for_each_lower_dev(bond_dev, other_dev, iter) {
+		if (!dpaa2_switch_port_dev_check(other_dev))
+			continue;
+
+		other_port_priv = netdev_priv(other_dev);
+		other_lag = rtnl_dereference(other_port_priv->lag);
+		if (!other_lag)
+			continue;
+
+		if (other_lag->bond_dev == bond_dev) {
+			rcu_assign_pointer(port_priv->lag, other_lag);
+			return;
+		}
+	}
+
+	/* This is the first interface to be added under a bond device. Find an
+	 * unused LAG group. No need to check for NULL since there are the same
+	 * amount of DPSW ports as LAG groups, meaning that each port can have
+	 * its own LAG group.
+	 */
+	lag = dpaa2_switch_lag_get_unused(ethsw);
+	lag->in_use = true;
+	lag->bond_dev = bond_dev;
+	lag->primary = port_priv;
+	rcu_assign_pointer(port_priv->lag, lag);
+}
+
+static bool dpaa2_switch_port_in_lag(struct ethsw_port_priv *port_priv,
+				     struct net_device *bond_dev)
+{
+	struct dpaa2_switch_lag *lag;
+
+	if (!port_priv)
+		return false;
+
+	lag = rtnl_dereference(port_priv->lag);
+	return lag && lag->bond_dev == bond_dev;
+}
+
+static int dpaa2_switch_set_lag_cfg(struct net_device *bond_dev, u8 lag_id,
+				    struct ethsw_core *ethsw)
+{
+	struct dpaa2_switch_lag *lag = &ethsw->lags[lag_id - 1];
+	struct ethsw_port_priv *primary, *port_priv;
+	struct ethsw_port_priv *new_primary = NULL;
+	struct dpsw_lag_cfg cfg = {0};
+	struct dpaa2_mac_addr *a;
+	u8 num_ifs = 0;
+	int err, i;
+
+	cfg.group_id = lag_id;
+
+	/* Determine the primary port. The caller clears ->lag on the port that
+	 * is leaving, so a NULL ->lag on the current primary means it is the
+	 * one leaving: elect the first remaining member as the new primary.
+	 * Otherwise keep the current primary.
+	 */
+	if (rtnl_dereference(lag->primary->lag)) {
+		primary = lag->primary;
+	} else {
+		primary = NULL;
+		for (i = 0; i < ethsw->sw_attr.num_ifs; i++) {
+			if (dpaa2_switch_port_in_lag(ethsw->ports[i], bond_dev)) {
+				new_primary = ethsw->ports[i];
+				primary = new_primary;
+				break;
+			}
+		}
+	}
+
+	/* Build the interface list, always placing the primary first */
+	if (primary)
+		cfg.if_id[num_ifs++] = primary->idx;
+	for (i = 0; i < ethsw->sw_attr.num_ifs; i++) {
+		port_priv = ethsw->ports[i];
+		if (port_priv == primary)
+			continue;
+		if (!dpaa2_switch_port_in_lag(port_priv, bond_dev))
+			continue;
+
+		cfg.if_id[num_ifs++] = port_priv->idx;
+	}
+	cfg.num_ifs = num_ifs;
+
+	/* No more interfaces under this LAG group, mark it as not in use. Wait
+	 * for a grace period so that any readers of the lag structure finished.
+	 */
+	if (!num_ifs) {
+		synchronize_net();
+
+		lag->bond_dev = NULL;
+		lag->primary = NULL;
+		lag->in_use = false;
+	}
+
+	/* When the primary changes, migrate the FDB entries from the old
+	 * primary to the new one: remove them before reconfiguring the LAG in
+	 * hardware and re-add them on the new primary afterwards. We do not
+	 * touch any refcounting since the intention is to change the HW entry,
+	 * not the parallel software tracking.
+	 */
+	if (new_primary) {
+		mutex_lock(&lag->fdb_lock);
+		list_for_each_entry(a, &lag->fdbs, list)
+			dpaa2_switch_port_fdb_del(lag->primary, a->addr);
+		mutex_unlock(&lag->fdb_lock);
+	}
+
+	err = dpsw_lag_set(ethsw->mc_io, 0, ethsw->dpsw_handle, &cfg);
+	if (err)
+		return err;
+
+	if (new_primary) {
+		mutex_lock(&lag->fdb_lock);
+		list_for_each_entry(a, &lag->fdbs, list) {
+			err = dpaa2_switch_port_fdb_add(new_primary, a->addr);
+			if (err)
+				netdev_err(new_primary->netdev, "Unable to migrate FDB\n");
+		}
+		mutex_unlock(&lag->fdb_lock);
+
+		synchronize_net();
+		lag->primary = new_primary;
+	}
+
+	return 0;
+}
+
+static int dpaa2_switch_port_bond_join(struct net_device *netdev,
+				       struct net_device *bond_dev,
+				       struct netdev_lag_upper_info *info,
+				       struct netlink_ext_ack *extack)
+{
+	struct ethsw_port_priv *port_priv = netdev_priv(netdev);
+	struct ethsw_core *ethsw = port_priv->ethsw_data;
+	struct net_device *bridge_dev;
+	struct dpaa2_switch_lag *lag;
+	int err = 0;
+	u8 lag_id;
+
+	/* Setup the port_priv->lag pointer for this switch port */
+	dpaa2_switch_port_set_lag_group(port_priv, bond_dev);
+
+	/* Create the LAG configuration and apply it in MC */
+	lag = rtnl_dereference(port_priv->lag);
+	lag_id = lag->id;
+	err = dpaa2_switch_set_lag_cfg(bond_dev, lag_id, ethsw);
+	if (err)
+		goto err_lag_cfg;
+
+	/* If the bond device is a switch port, join the bridge as well */
+	bridge_dev = netdev_master_upper_dev_get(bond_dev);
+	if (!bridge_dev || !netif_is_bridge_master(bridge_dev))
+		return 0;
+
+	err = dpaa2_switch_port_bridge_join(netdev, bridge_dev, extack);
+	if (err)
+		goto err_lag_cfg;
+
+	return err;
+
+err_lag_cfg:
+	rcu_assign_pointer(port_priv->lag, NULL);
+	dpaa2_switch_set_lag_cfg(bond_dev, lag_id, ethsw);
+
+	return err;
+}
+
+static int dpaa2_switch_port_bond_leave(struct net_device *netdev,
+					struct net_device *bond_dev)
+{
+	struct net_device *bridge_dev = netdev_master_upper_dev_get(bond_dev);
+	struct ethsw_port_priv *port_priv = netdev_priv(netdev);
+	struct dpaa2_switch_lag *lag = rtnl_dereference(port_priv->lag);
+	struct ethsw_core *ethsw = port_priv->ethsw_data;
+	struct net_device *brpdev;
+	bool learn_ena;
+	int err;
+
+	if (!lag)
+		return 0;
+
+	/* Recreate the LAG configuration for the LAG group that we left. */
+	rcu_assign_pointer(port_priv->lag, NULL);
+	dpaa2_switch_set_lag_cfg(bond_dev, lag->id, ethsw);
+
+	if (bridge_dev && netif_is_bridge_master(bridge_dev)) {
+		/* Make sure that the new primary inherits the learning state */
+		if (lag->primary) {
+			brpdev = dpaa2_switch_port_to_bridge_port(lag->primary);
+			learn_ena = br_port_flag_is_set(brpdev, BR_LEARNING);
+			err = dpaa2_switch_port_set_learning(lag->primary,
+							     learn_ena);
+			if (err)
+				return err;
+			lag->primary->learn_ena = learn_ena;
+		}
+
+		/* In case the bond is a bridge port, leave the upper bridge as
+		 * well.
+		 */
+		return dpaa2_switch_port_bridge_leave(netdev);
 	}
 
 	return 0;
@@ -2234,8 +2729,8 @@ static int dpaa2_switch_port_prechangeupper(struct net_device *netdev,
 					    struct netdev_notifier_changeupper_info *info)
 {
 	struct ethsw_port_priv *port_priv;
+	struct net_device *upper_dev, *br;
 	struct netlink_ext_ack *extack;
-	struct net_device *upper_dev;
 	int err;
 
 	if (!dpaa2_switch_port_dev_check(netdev))
@@ -2252,6 +2747,24 @@ static int dpaa2_switch_port_prechangeupper(struct net_device *netdev,
 
 		if (!info->linking)
 			dpaa2_switch_port_pre_bridge_leave(netdev);
+	} else if (netif_is_lag_master(upper_dev)) {
+		if (!info->linking) {
+			if (netif_is_bridge_port(upper_dev))
+				dpaa2_switch_port_pre_bridge_leave(netdev);
+			return 0;
+		}
+
+		if (netif_is_bridge_port(upper_dev)) {
+			br = netdev_master_upper_dev_get(upper_dev);
+			err = dpaa2_switch_prechangeupper_sanity_checks(netdev,
+									br,
+									extack);
+			if (err)
+				return err;
+		}
+
+		return dpaa2_switch_pre_lag_join(netdev, upper_dev,
+						 info->upper_info, extack);
 	} else if (is_vlan_dev(upper_dev)) {
 		port_priv = netdev_priv(netdev);
 		if (port_priv->fdb->bridge_dev) {
@@ -2283,6 +2796,80 @@ static int dpaa2_switch_port_changeupper(struct net_device *netdev,
 							     extack);
 		else
 			return dpaa2_switch_port_bridge_leave(netdev);
+	} else if (netif_is_lag_master(upper_dev)) {
+		if (info->linking)
+			return dpaa2_switch_port_bond_join(netdev, upper_dev,
+							   info->upper_info,
+							   extack);
+		else
+			return dpaa2_switch_port_bond_leave(netdev, upper_dev);
+	}
+
+	return 0;
+}
+
+static int
+dpaa2_switch_lag_prechangeupper(struct net_device *netdev,
+				struct netdev_notifier_changeupper_info *info)
+{
+	struct net_device *lower;
+	struct list_head *iter;
+	int err = 0;
+
+	if (!netif_is_lag_master(netdev))
+		return 0;
+
+	netdev_for_each_lower_dev(netdev, lower, iter) {
+		if (!dpaa2_switch_port_dev_check(lower))
+			continue;
+
+		err = dpaa2_switch_port_prechangeupper(lower, info);
+		if (err)
+			return err;
+	}
+
+	return err;
+}
+
+static int
+dpaa2_switch_lag_changeupper(struct net_device *netdev,
+			     struct netdev_notifier_changeupper_info *info)
+{
+	struct net_device *lower;
+	struct list_head *iter;
+	int err = 0;
+
+	if (!netif_is_lag_master(netdev))
+		return 0;
+
+	netdev_for_each_lower_dev(netdev, lower, iter) {
+		if (!dpaa2_switch_port_dev_check(lower))
+			continue;
+
+		err = dpaa2_switch_port_changeupper(lower, info);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int
+dpaa2_switch_port_changelowerstate(struct net_device *netdev,
+				   struct netdev_lag_lower_state_info *linfo)
+{
+	struct ethsw_port_priv *port_priv = netdev_priv(netdev);
+	struct ethsw_core *ethsw = port_priv->ethsw_data;
+	int err;
+
+	if (!rtnl_dereference(port_priv->lag))
+		return 0;
+
+	err = dpsw_if_set_lag_state(ethsw->mc_io, 0, ethsw->dpsw_handle,
+				    port_priv->idx, linfo->tx_enabled ? 1 : 0);
+	if (err) {
+		netdev_err(netdev, "dpsw_if_set_lag_state() = %d\n", err);
+		return err;
 	}
 
 	return 0;
@@ -2292,6 +2879,7 @@ static int dpaa2_switch_port_netdevice_event(struct notifier_block *nb,
 					     unsigned long event, void *ptr)
 {
 	struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
+	struct netdev_notifier_changelowerstate_info *info;
 	int err = 0;
 
 	switch (event) {
@@ -2300,13 +2888,29 @@ static int dpaa2_switch_port_netdevice_event(struct notifier_block *nb,
 		if (err)
 			return notifier_from_errno(err);
 
+		err = dpaa2_switch_lag_prechangeupper(netdev, ptr);
+		if (err)
+			return notifier_from_errno(err);
+
 		break;
 	case NETDEV_CHANGEUPPER:
 		err = dpaa2_switch_port_changeupper(netdev, ptr);
 		if (err)
 			return notifier_from_errno(err);
 
+		err = dpaa2_switch_lag_changeupper(netdev, ptr);
+		if (err)
+			return notifier_from_errno(err);
+
 		break;
+	case NETDEV_CHANGELOWERSTATE:
+		info = ptr;
+		if (!dpaa2_switch_port_dev_check(netdev))
+			break;
+
+		err = dpaa2_switch_port_changelowerstate(netdev,
+							 info->lower_state_info);
+		return notifier_from_errno(err);
 	}
 
 	return NOTIFY_DONE;
@@ -2316,80 +2920,97 @@ struct ethsw_switchdev_event_work {
 	struct work_struct work;
 	struct switchdev_notifier_fdb_info fdb_info;
 	struct net_device *dev;
+	struct net_device *orig_dev;
 	unsigned long event;
+	u16 vid;
 };
 
 static void dpaa2_switch_event_work(struct work_struct *work)
 {
 	struct ethsw_switchdev_event_work *switchdev_work =
 		container_of(work, struct ethsw_switchdev_event_work, work);
+	struct net_device *orig_dev = switchdev_work->orig_dev;
 	struct net_device *dev = switchdev_work->dev;
+	struct ethsw_port_priv *port_priv = netdev_priv(dev);
 	struct switchdev_notifier_fdb_info *fdb_info;
+	struct dpaa2_switch_lag *lag;
 	int err;
 
-	rtnl_lock();
 	fdb_info = &switchdev_work->fdb_info;
 
+	/* The lag structures are freed only from dpaa2_switch_remove(), which
+	 * first flushes this workqueue, so the pointer stays valid for the
+	 * lifetime of the work item. Only the dereference needs the RCU
+	 * read-side lock; the FDB helpers below can sleep and must run outside
+	 * of it.
+	 */
+	rcu_read_lock();
+	lag = rcu_dereference(port_priv->lag);
+	rcu_read_unlock();
+
 	switch (switchdev_work->event) {
 	case SWITCHDEV_FDB_ADD_TO_DEVICE:
-		if (!fdb_info->added_by_user || fdb_info->is_local)
-			break;
-		if (is_unicast_ether_addr(fdb_info->addr))
-			err = dpaa2_switch_port_fdb_add_uc(netdev_priv(dev),
-							   fdb_info->addr);
+		if (lag)
+			err = dpaa2_switch_lag_fdb_add(lag, fdb_info->addr,
+						       switchdev_work->vid);
 		else
-			err = dpaa2_switch_port_fdb_add_mc(netdev_priv(dev),
-							   fdb_info->addr);
+			err = dpaa2_switch_port_fdb_add(port_priv,
+							fdb_info->addr);
 		if (err)
 			break;
 		fdb_info->offloaded = true;
-		call_switchdev_notifiers(SWITCHDEV_FDB_OFFLOADED, dev,
+		call_switchdev_notifiers(SWITCHDEV_FDB_OFFLOADED, orig_dev,
 					 &fdb_info->info, NULL);
 		break;
 	case SWITCHDEV_FDB_DEL_TO_DEVICE:
-		if (!fdb_info->added_by_user || fdb_info->is_local)
-			break;
-		if (is_unicast_ether_addr(fdb_info->addr))
-			dpaa2_switch_port_fdb_del_uc(netdev_priv(dev), fdb_info->addr);
+		if (lag)
+			dpaa2_switch_lag_fdb_del(lag, fdb_info->addr,
+						 switchdev_work->vid);
 		else
-			dpaa2_switch_port_fdb_del_mc(netdev_priv(dev), fdb_info->addr);
+			dpaa2_switch_port_fdb_del(port_priv, fdb_info->addr);
 		break;
 	}
 
-	rtnl_unlock();
 	kfree(switchdev_work->fdb_info.addr);
 	kfree(switchdev_work);
 	dev_put(dev);
+	dev_put(orig_dev);
 }
 
-/* Called under rcu_read_lock() */
-static int dpaa2_switch_port_event(struct notifier_block *nb,
-				   unsigned long event, void *ptr)
+static int
+dpaa2_switch_port_fdb_event(struct net_device *dev,
+			    struct net_device *orig_dev,
+			    unsigned long event, const void *ctx,
+			    const struct switchdev_notifier_fdb_info *fdb_info)
 {
-	struct net_device *dev = switchdev_notifier_info_to_dev(ptr);
 	struct ethsw_port_priv *port_priv = netdev_priv(dev);
 	struct ethsw_switchdev_event_work *switchdev_work;
-	struct switchdev_notifier_fdb_info *fdb_info = ptr;
 	struct ethsw_core *ethsw = port_priv->ethsw_data;
 
-	if (event == SWITCHDEV_PORT_ATTR_SET)
-		return dpaa2_switch_port_attr_set_event(dev, ptr);
+	if (ctx && ctx != port_priv)
+		return 0;
 
-	if (!dpaa2_switch_port_dev_check(dev))
-		return NOTIFY_DONE;
+	/* For the moment, do nothing with entries towards foreign devices */
+	if (dpaa2_switch_foreign_dev_check(dev, orig_dev))
+		return 0;
+
+	if (!fdb_info->added_by_user || fdb_info->is_local)
+		return 0;
 
 	switchdev_work = kzalloc_obj(*switchdev_work, GFP_ATOMIC);
 	if (!switchdev_work)
-		return NOTIFY_BAD;
+		return -ENOMEM;
 
 	INIT_WORK(&switchdev_work->work, dpaa2_switch_event_work);
 	switchdev_work->dev = dev;
 	switchdev_work->event = event;
+	switchdev_work->orig_dev = orig_dev;
+	switchdev_work->vid = fdb_info->vid;
 
 	switch (event) {
 	case SWITCHDEV_FDB_ADD_TO_DEVICE:
 	case SWITCHDEV_FDB_DEL_TO_DEVICE:
-		memcpy(&switchdev_work->fdb_info, ptr,
+		memcpy(&switchdev_work->fdb_info, fdb_info,
 		       sizeof(switchdev_work->fdb_info));
 		switchdev_work->fdb_info.addr = kzalloc(ETH_ALEN, GFP_ATOMIC);
 		if (!switchdev_work->fdb_info.addr)
@@ -2400,52 +3021,61 @@ static int dpaa2_switch_port_event(struct notifier_block *nb,
 
 		/* Take a reference on the device to avoid being freed. */
 		dev_hold(dev);
+		dev_hold(orig_dev);
 		break;
 	default:
 		kfree(switchdev_work);
-		return NOTIFY_DONE;
+		return 0;
 	}
 
 	queue_work(ethsw->workqueue, &switchdev_work->work);
 
-	return NOTIFY_DONE;
+	return 0;
 
 err_addr_alloc:
 	kfree(switchdev_work);
-	return NOTIFY_BAD;
+	return -ENOMEM;
 }
 
-static int dpaa2_switch_port_obj_event(unsigned long event,
-				       struct net_device *netdev,
-				       struct switchdev_notifier_port_obj_info *port_obj_info)
+/* Called under rcu_read_lock() */
+static int dpaa2_switch_port_event(struct notifier_block *nb,
+				   unsigned long event, void *ptr)
 {
-	int err = -EOPNOTSUPP;
-
-	if (!dpaa2_switch_port_dev_check(netdev))
-		return NOTIFY_DONE;
+	struct net_device *dev = switchdev_notifier_info_to_dev(ptr);
+	int err;
 
 	switch (event) {
-	case SWITCHDEV_PORT_OBJ_ADD:
-		err = dpaa2_switch_port_obj_add(netdev, port_obj_info->obj);
-		break;
-	case SWITCHDEV_PORT_OBJ_DEL:
-		err = dpaa2_switch_port_obj_del(netdev, port_obj_info->obj);
-		break;
+	case SWITCHDEV_PORT_ATTR_SET:
+		return dpaa2_switch_port_attr_set_event(dev, ptr);
+	case SWITCHDEV_FDB_ADD_TO_DEVICE:
+	case SWITCHDEV_FDB_DEL_TO_DEVICE:
+		err = switchdev_handle_fdb_event_to_device(dev, event, ptr,
+							   dpaa2_switch_port_dev_check,
+							   dpaa2_switch_foreign_dev_check,
+							   dpaa2_switch_port_fdb_event);
+		return notifier_from_errno(err);
+	default:
+		return NOTIFY_DONE;
 	}
-
-	port_obj_info->handled = true;
-	return notifier_from_errno(err);
 }
 
 static int dpaa2_switch_port_blocking_event(struct notifier_block *nb,
 					    unsigned long event, void *ptr)
 {
 	struct net_device *dev = switchdev_notifier_info_to_dev(ptr);
+	int err;
 
 	switch (event) {
 	case SWITCHDEV_PORT_OBJ_ADD:
+		err = switchdev_handle_port_obj_add(dev, ptr,
+						    dpaa2_switch_port_dev_check,
+						    dpaa2_switch_port_obj_add);
+		return notifier_from_errno(err);
 	case SWITCHDEV_PORT_OBJ_DEL:
-		return dpaa2_switch_port_obj_event(event, dev, ptr);
+		err = switchdev_handle_port_obj_del(dev, ptr,
+						    dpaa2_switch_port_dev_check,
+						    dpaa2_switch_port_obj_del);
+		return notifier_from_errno(err);
 	case SWITCHDEV_PORT_ATTR_SET:
 		return dpaa2_switch_port_attr_set_event(dev, ptr);
 	}
@@ -2490,19 +3120,22 @@ static void dpaa2_switch_rx(struct dpaa2_switch_fq *fq,
 	dma_addr_t addr = dpaa2_fd_get_addr(fd);
 	struct ethsw_core *ethsw = fq->ethsw;
 	struct ethsw_port_priv *port_priv;
+	struct dpaa2_switch_lag *lag;
 	struct net_device *netdev;
 	struct vlan_ethhdr *hdr;
 	struct sk_buff *skb;
 	u16 vlan_tci, vid;
 	int if_id, err;
 	void *vaddr;
+	u64 flc;
 
 	vaddr = dpaa2_iova_to_virt(ethsw->iommu_domain, addr);
 	dma_unmap_page(ethsw->dev, addr, DPAA2_SWITCH_RX_BUF_SIZE,
 		       DMA_FROM_DEVICE);
 
 	/* get switch ingress interface ID */
-	if_id = upper_32_bits(dpaa2_fd_get_flc(fd)) & 0x0000FFFF;
+	flc = dpaa2_fd_get_flc(fd);
+	if_id = DPAA2_ETHSW_FLC_IF_ID(flc);
 	if (if_id >= ethsw->sw_attr.num_ifs) {
 		dev_err(ethsw->dev, "Frame received from unknown interface!\n");
 		goto err_free_fd;
@@ -2541,12 +3174,20 @@ static void dpaa2_switch_rx(struct dpaa2_switch_fq *fq,
 		}
 	}
 
-	skb->dev = netdev;
+	rcu_read_lock();
+
+	lag = rcu_dereference(port_priv->lag);
+	if (DPAA2_ETHSW_FLC_IMPRECISE_IF_ID(flc) && lag)
+		skb->dev = lag->bond_dev;
+	else
+		skb->dev = netdev;
 	skb->protocol = eth_type_trans(skb, skb->dev);
 
 	/* Setup the offload_fwd_mark only if the port is under a bridge */
 	skb->offload_fwd_mark = !!(port_priv->fdb->bridge_dev);
 
+	rcu_read_unlock();
+
 	netif_receive_skb(skb);
 
 	return;
@@ -2561,6 +3202,9 @@ static void dpaa2_switch_detect_features(struct ethsw_core *ethsw)
 
 	if (ethsw->major > 8 || (ethsw->major == 8 && ethsw->minor >= 6))
 		ethsw->features |= ETHSW_FEATURE_MAC_ADDR;
+
+	if (ethsw->major > 8 || (ethsw->major == 8 && ethsw->minor >= 13))
+		ethsw->features |= ETHSW_FEATURE_LAG_OFFLOAD;
 }
 
 static int dpaa2_switch_setup_fqs(struct ethsw_core *ethsw)
@@ -3195,17 +3839,15 @@ static int dpaa2_switch_init(struct fsl_mc_device *sw_dev)
 	return err;
 }
 
-/* Add an ACL to redirect frames with specific destination MAC address to
- * control interface
- */
+/* Add an ACL to redirect frames to control interface based on the dst MAC */
 static int dpaa2_switch_port_trap_mac_addr(struct ethsw_port_priv *port_priv,
-					   const char *mac)
+					   const u8 *mac, const u8 *mask)
 {
 	struct dpaa2_switch_acl_entry acl_entry = {0};
 
 	/* Match on the destination MAC address */
 	ether_addr_copy(acl_entry.key.match.l2_dest_mac, mac);
-	eth_broadcast_addr(acl_entry.key.mask.l2_dest_mac);
+	ether_addr_copy(acl_entry.key.mask.l2_dest_mac, mask);
 
 	/* Trap to CPU */
 	acl_entry.cfg.precedence = 0;
@@ -3216,7 +3858,8 @@ static int dpaa2_switch_port_trap_mac_addr(struct ethsw_port_priv *port_priv,
 
 static int dpaa2_switch_port_init(struct ethsw_port_priv *port_priv, u16 port)
 {
-	const char stpa[ETH_ALEN] = {0x01, 0x80, 0xc2, 0x00, 0x00, 0x00};
+	const u8 ll_mac[ETH_ALEN] = {0x01, 0x80, 0xc2, 0x00, 0x00, 0x00};
+	const u8 ll_mask[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xf0};
 	struct switchdev_obj_port_vlan vlan = {
 		.obj.id = SWITCHDEV_OBJ_ID_PORT_VLAN,
 		.vid = DEFAULT_VLAN_ID,
@@ -3291,7 +3934,7 @@ static int dpaa2_switch_port_init(struct ethsw_port_priv *port_priv, u16 port)
 	if (err)
 		return err;
 
-	err = dpaa2_switch_port_trap_mac_addr(port_priv, stpa);
+	err = dpaa2_switch_port_trap_mac_addr(port_priv, ll_mac, ll_mask);
 	if (err)
 		return err;
 
@@ -3331,6 +3974,9 @@ static void dpaa2_switch_remove(struct fsl_mc_device *sw_dev)
 	dev = &sw_dev->dev;
 	ethsw = dev_get_drvdata(dev);
 
+	/* Make sure that all events were handled before we kfree anything */
+	flush_workqueue(ethsw->workqueue);
+
 	dpaa2_switch_teardown_irqs(sw_dev);
 
 	dpsw_disable(ethsw->mc_io, 0, ethsw->dpsw_handle);
@@ -3344,12 +3990,15 @@ static void dpaa2_switch_remove(struct fsl_mc_device *sw_dev)
 	for (i = 0; i < DPAA2_SWITCH_RX_NUM_FQS; i++)
 		netif_napi_del(&ethsw->fq[i].napi);
 
-	for (i = 0; i < ethsw->sw_attr.num_ifs; i++)
+	for (i = 0; i < ethsw->sw_attr.num_ifs; i++) {
 		dpaa2_switch_remove_port(ethsw, i);
+		mutex_destroy(&ethsw->lags[i].fdb_lock);
+	}
 
 	kfree(ethsw->fdbs);
 	kfree(ethsw->filter_blocks);
 	kfree(ethsw->ports);
+	kfree(ethsw->lags);
 
 	dpaa2_switch_teardown(sw_dev);
 
@@ -3377,6 +4026,7 @@ static int dpaa2_switch_probe_port(struct ethsw_core *ethsw,
 	port_priv = netdev_priv(port_netdev);
 	port_priv->netdev = port_netdev;
 	port_priv->ethsw_data = ethsw;
+	rcu_assign_pointer(port_priv->lag, NULL);
 
 	mutex_init(&port_priv->mac_lock);
 
@@ -3484,6 +4134,21 @@ static int dpaa2_switch_probe(struct fsl_mc_device *sw_dev)
 		goto err_free_fdbs;
 	}
 
+	ethsw->lags = kcalloc(ethsw->sw_attr.num_ifs, sizeof(*ethsw->lags),
+			      GFP_KERNEL);
+	if (!ethsw->lags) {
+		err = -ENOMEM;
+		goto err_free_filter;
+	}
+	for (i = 0; i < ethsw->sw_attr.num_ifs; i++) {
+		ethsw->lags[i].bond_dev = NULL;
+		ethsw->lags[i].ethsw = ethsw;
+		ethsw->lags[i].id = i + 1;
+		ethsw->lags[i].in_use = 0;
+		mutex_init(&ethsw->lags[i].fdb_lock);
+		INIT_LIST_HEAD(&ethsw->lags[i].fdbs);
+	}
+
 	for (i = 0; i < ethsw->sw_attr.num_ifs; i++) {
 		err = dpaa2_switch_probe_port(ethsw, i);
 		if (err)
@@ -3530,6 +4195,10 @@ static int dpaa2_switch_probe(struct fsl_mc_device *sw_dev)
 err_free_netdev:
 	for (i--; i >= 0; i--)
 		dpaa2_switch_remove_port(ethsw, i);
+	for (i = 0; i < ethsw->sw_attr.num_ifs; i++)
+		mutex_destroy(&ethsw->lags[i].fdb_lock);
+	kfree(ethsw->lags);
+err_free_filter:
 	kfree(ethsw->filter_blocks);
 err_free_fdbs:
 	kfree(ethsw->fdbs);
diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.h b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.h
index 42b3ca73f55d5..63b702b0000c2 100644
--- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.h
+++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.h
@@ -41,7 +41,8 @@
 #define ETHSW_MAX_FRAME_LENGTH	(DPAA2_MFL - VLAN_ETH_HLEN - ETH_FCS_LEN)
 #define ETHSW_L2_MAX_FRM(mtu)	((mtu) + VLAN_ETH_HLEN + ETH_FCS_LEN)
 
-#define ETHSW_FEATURE_MAC_ADDR	BIT(0)
+#define ETHSW_FEATURE_MAC_ADDR		BIT(0)
+#define ETHSW_FEATURE_LAG_OFFLOAD	BIT(1)
 
 /* Number of receive queues (one RX and one TX_CONF) */
 #define DPAA2_SWITCH_RX_NUM_FQS	2
@@ -86,6 +87,9 @@
 
 #define DPAA2_ETHSW_PORT_ACL_CMD_BUF_SIZE	256
 
+#define DPAA2_ETHSW_FLC_IF_ID(flc)		(((flc) >> 32) & GENMASK(15, 0))
+#define DPAA2_ETHSW_FLC_IMPRECISE_IF_ID(flc)	((flc) & BIT_ULL(63))
+
 extern const struct ethtool_ops dpaa2_switch_port_ethtool_ops;
 
 struct ethsw_core;
@@ -99,12 +103,30 @@ struct dpaa2_switch_fq {
 	u32 fqid;
 };
 
+struct dpaa2_mac_addr {
+	unsigned char addr[ETH_ALEN];
+	u16 vid;
+	refcount_t refcount;
+	struct list_head list;
+};
+
 struct dpaa2_switch_fdb {
 	struct net_device	*bridge_dev;
 	u16			fdb_id;
 	bool			in_use;
 };
 
+struct dpaa2_switch_lag {
+	struct ethsw_core	*ethsw;
+	struct net_device	*bond_dev;
+	bool			in_use;
+	u8			id;
+	struct ethsw_port_priv	*primary;
+	/* Protects the list of fdbs installed on this LAG */
+	struct mutex		fdb_lock;
+	struct list_head	fdbs;
+};
+
 struct dpaa2_switch_acl_entry {
 	struct list_head	list;
 	u16			prio;
@@ -163,6 +185,8 @@ struct ethsw_port_priv {
 	struct dpaa2_mac	*mac;
 	/* Protects against changes to port_priv->mac */
 	struct mutex		mac_lock;
+
+	struct dpaa2_switch_lag __rcu *lag;
 };
 
 /* Switch data */
@@ -190,6 +214,8 @@ struct ethsw_core {
 	struct dpaa2_switch_fdb		*fdbs;
 	struct dpaa2_switch_filter_block *filter_blocks;
 	u16				mirror_port;
+
+	struct dpaa2_switch_lag		*lags;
 };
 
 static inline int dpaa2_switch_get_index(struct ethsw_core *ethsw,
@@ -274,4 +300,18 @@ int dpaa2_switch_block_offload_mirror(struct dpaa2_switch_filter_block *block,
 
 int dpaa2_switch_block_unoffload_mirror(struct dpaa2_switch_filter_block *block,
 					struct ethsw_port_priv *port_priv);
+
+static inline bool
+dpaa2_switch_port_offloads_bridge_port(struct ethsw_port_priv *port_priv,
+				       const struct net_device *dev)
+{
+	struct dpaa2_switch_lag *lag = rcu_dereference_rtnl(port_priv->lag);
+
+	if (lag && lag->bond_dev == dev)
+		return true;
+	if (port_priv->netdev == dev)
+		return true;
+	return false;
+}
+
 #endif	/* __ETHSW_H */
diff --git a/drivers/net/ethernet/freescale/dpaa2/dpsw-cmd.h b/drivers/net/ethernet/freescale/dpaa2/dpsw-cmd.h
index 397d55f2bd995..9a2055c64983c 100644
--- a/drivers/net/ethernet/freescale/dpaa2/dpsw-cmd.h
+++ b/drivers/net/ethernet/freescale/dpaa2/dpsw-cmd.h
@@ -12,7 +12,7 @@
 
 /* DPSW Version */
 #define DPSW_VER_MAJOR		8
-#define DPSW_VER_MINOR		9
+#define DPSW_VER_MINOR		13
 
 #define DPSW_CMD_BASE_VERSION	1
 #define DPSW_CMD_VERSION_2	2
@@ -92,11 +92,14 @@
 #define DPSW_CMDID_CTRL_IF_SET_POOLS        DPSW_CMD_ID(0x0A1)
 #define DPSW_CMDID_CTRL_IF_ENABLE           DPSW_CMD_ID(0x0A2)
 #define DPSW_CMDID_CTRL_IF_DISABLE          DPSW_CMD_ID(0x0A3)
+#define DPSW_CMDID_SET_LAG                  DPSW_CMD_V2(0x0A4)
 #define DPSW_CMDID_CTRL_IF_SET_QUEUE        DPSW_CMD_ID(0x0A6)
 
 #define DPSW_CMDID_SET_EGRESS_FLOOD         DPSW_CMD_ID(0x0AC)
 #define DPSW_CMDID_IF_SET_LEARNING_MODE     DPSW_CMD_ID(0x0AD)
 
+#define DPSW_CMDID_IF_SET_LAG_STATE         DPSW_CMD_ID(0x0B0)
+
 /* Macros for accessing command fields smaller than 1byte */
 #define DPSW_MASK(field)        \
 	GENMASK(DPSW_##field##_SHIFT + DPSW_##field##_SIZE - 1, \
@@ -552,5 +555,18 @@ struct dpsw_cmd_if_reflection {
 	/* only 2 bits from the LSB */
 	u8 filter;
 };
+
+struct dpsw_cmd_lag {
+	u8 group_id;
+	u8 num_ifs;
+	u8 pad[6];
+	u8 if_id[DPSW_MAX_LAG_IFS];
+	u8 phase;
+};
+
+struct dpsw_cmd_if_set_lag_state {
+	__le16 if_id;
+	u8 tx_enabled;
+};
 #pragma pack(pop)
 #endif /* __FSL_DPSW_CMD_H */
diff --git a/drivers/net/ethernet/freescale/dpaa2/dpsw.c b/drivers/net/ethernet/freescale/dpaa2/dpsw.c
index ab921d75deb2e..f75cbdce42ba2 100644
--- a/drivers/net/ethernet/freescale/dpaa2/dpsw.c
+++ b/drivers/net/ethernet/freescale/dpaa2/dpsw.c
@@ -1659,3 +1659,63 @@ int dpsw_if_remove_reflection(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
 
 	return mc_send_command(mc_io, &cmd);
 }
+
+/**
+ * dpsw_lag_set() - Set LAG configuration
+ * @mc_io:	Pointer to MC portal's I/O object
+ * @cmd_flags:	Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:	Token of DPSW object
+ * @cfg:	pointer to LAG configuration
+ *
+ * Return:   '0' on Success; Error code otherwise.
+ */
+int dpsw_lag_set(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
+		 const struct dpsw_lag_cfg *cfg)
+{
+	struct fsl_mc_command cmd = { 0 };
+	struct dpsw_cmd_lag *cmd_params;
+	int i = 0;
+
+	cmd.header = mc_encode_cmd_header(DPSW_CMDID_SET_LAG, cmd_flags, token);
+
+	if (cfg->num_ifs > DPSW_MAX_LAG_IFS)
+		return -EOPNOTSUPP;
+
+	cmd_params = (struct dpsw_cmd_lag *)cmd.params;
+	cmd_params->group_id = cfg->group_id;
+	cmd_params->num_ifs = cfg->num_ifs;
+	cmd_params->phase = cfg->phase;
+
+	for (i = 0; i < cfg->num_ifs; i++)
+		cmd_params->if_id[i] = cfg->if_id[i];
+
+	return mc_send_command(mc_io, &cmd);
+}
+
+/**
+ * dpsw_if_set_lag_state() - Change per port LAG state
+ * @mc_io:      Pointer to MC portal's I/O object
+ * @cmd_flags:  Command flags; one or more of 'MC_CMD_FLAG_'
+ * @token:      Token of DPSW object
+ * @if_id:      ID of the switch interface
+ * @tx_enabled: Value of the per port LAG state
+ *     - 0 if the interface will not be active as part of the LAG group
+ *     - 1 if the interface will be active in the LAG group
+ *
+ * Return:   '0' on Success; Error code otherwise.
+ */
+int dpsw_if_set_lag_state(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
+			  u16 if_id, u8 tx_enabled)
+{
+	struct dpsw_cmd_if_set_lag_state *cmd_params;
+	struct fsl_mc_command cmd = { 0 };
+
+	cmd.header = mc_encode_cmd_header(DPSW_CMDID_IF_SET_LAG_STATE,
+					  cmd_flags, token);
+
+	cmd_params = (struct dpsw_cmd_if_set_lag_state *)cmd.params;
+	cmd_params->if_id = cpu_to_le16(if_id);
+	cmd_params->tx_enabled = tx_enabled;
+
+	return mc_send_command(mc_io, &cmd);
+}
diff --git a/drivers/net/ethernet/freescale/dpaa2/dpsw.h b/drivers/net/ethernet/freescale/dpaa2/dpsw.h
index b90bd363f47a1..89f0267de8e9b 100644
--- a/drivers/net/ethernet/freescale/dpaa2/dpsw.h
+++ b/drivers/net/ethernet/freescale/dpaa2/dpsw.h
@@ -20,6 +20,8 @@ struct fsl_mc_io;
 
 #define DPSW_MAX_IF		64
 
+#define DPSW_MAX_LAG_IFS	8
+
 int dpsw_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpsw_id, u16 *token);
 
 int dpsw_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token);
@@ -788,4 +790,32 @@ int dpsw_if_add_reflection(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
 
 int dpsw_if_remove_reflection(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
 			      u16 if_id, const struct dpsw_reflection_cfg *cfg);
+
+/* Link Aggregation Group configuration */
+
+#define DPSW_LAG_SET_PHASE_APPLY 0
+#define DPSW_LAG_SET_PHASE_CHECK 1
+
+/**
+ * struct dpsw_lag_cfg - Configuration structure for a LAG group
+ * @group_id: Link aggregation group ID. Valid values are in the
+ * [1, DPSW_MAX_LAG_IFS] range.
+ * @num_ifs: Number of interfaces in this LAG group, valid range is
+ * [0, DPSW_MAX_LAG_IFS].
+ * @if_id: Array containing the interface IDs of the ports part of a LAG group
+ * @phase: Use DPSW_LAG_SET_PHASE_APPLY for LAG configuration processing or
+ * DPSW_LAG_SET_PHASE_CHECK for LAG configuration validation.
+ */
+struct dpsw_lag_cfg {
+	u8 group_id;
+	u8 num_ifs;
+	u8 if_id[DPSW_MAX_LAG_IFS];
+	u8 phase;
+};
+
+int dpsw_lag_set(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
+		 const struct dpsw_lag_cfg *cfg);
+
+int dpsw_if_set_lag_state(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token,
+			  u16 if_id, u8 tx_enabled);
 #endif /* __FSL_DPSW_H */
diff --git a/drivers/net/ethernet/fungible/funeth/funeth_rx.c b/drivers/net/ethernet/fungible/funeth/funeth_rx.c
index 7e2584895de39..d7000017ac2bd 100644
--- a/drivers/net/ethernet/fungible/funeth/funeth_rx.c
+++ b/drivers/net/ethernet/fungible/funeth/funeth_rx.c
@@ -103,7 +103,7 @@ static int funeth_alloc_page(struct funeth_rxq *q, struct funeth_rxbuf *rb,
 	if (cache_get(q, rb))
 		return 0;
 
-	p = __alloc_pages_node(node, gfp | __GFP_NOWARN, 0);
+	p = alloc_pages_node(node, gfp | __GFP_NOWARN, 0);
 	if (unlikely(!p))
 		return -ENOMEM;
 
diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h
index a8798eecd9fbd..4286af9239b01 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h
@@ -602,6 +602,10 @@ typedef int (*read_func)(struct seq_file *s, void *data);
  *   Config wake on lan
  * dbg_get_read_func
  *   Return the read func for debugfs seq file
+ * set_pfc_prevention_tout
+ *   Set PFC storm prevention timeout
+ * get_pfc_prevention_tout
+ *   Get PFC storm prevention timeout
  */
 struct hnae3_ae_ops {
 	int (*init_ae_dev)(struct hnae3_ae_dev *ae_dev);
@@ -810,6 +814,8 @@ struct hnae3_ae_ops {
 	int (*hwtstamp_set)(struct hnae3_handle *handle,
 			    struct kernel_hwtstamp_config *config,
 			    struct netlink_ext_ack *extack);
+	int (*set_pfc_prevention_tout)(struct hnae3_handle *handle, u16 times);
+	int (*get_pfc_prevention_tout)(struct hnae3_handle *handle, u16 *times);
 };
 
 struct hnae3_dcb_ops {
@@ -891,6 +897,14 @@ struct hnae3_roce_private_info {
 	unsigned long state;
 };
 
+struct hnae3_pfc_storm_para {
+	u32 dir;
+	u32 enable;
+	u32 period_ms;
+	u32 times;
+	u32 recovery_period_ms;
+};
+
 #define HNAE3_SUPPORT_APP_LOOPBACK    BIT(0)
 #define HNAE3_SUPPORT_PHY_LOOPBACK    BIT(1)
 #define HNAE3_SUPPORT_SERDES_SERIAL_LOOPBACK	BIT(2)
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h
index 2c2a2f1e0d7a6..6dde07dde1e8e 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h
@@ -314,6 +314,9 @@ enum hclge_opcode_type {
 
 	/* Query link diagnosis info command */
 	HCLGE_OPC_QUERY_LINK_DIAGNOSIS	= 0x702A,
+
+	/* Config pause storm param command */
+	HCLGE_OPC_CFG_PAUSE_STORM_PARA = 0x7019,
 };
 
 enum hclge_comm_cmd_return_status {
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
index 442f15476af32..e7318f2363157 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
@@ -1895,6 +1895,12 @@ static int hns3_get_tunable(struct net_device *netdev,
 	case ETHTOOL_TX_COPYBREAK_BUF_SIZE:
 		*(u32 *)data = h->kinfo.tx_spare_buf_size;
 		break;
+	case ETHTOOL_PFC_PREVENTION_TOUT:
+		if (!h->ae_algo->ops->get_pfc_prevention_tout)
+			return -EOPNOTSUPP;
+
+		ret = h->ae_algo->ops->get_pfc_prevention_tout(h, (u16 *)data);
+		break;
 	default:
 		ret = -EOPNOTSUPP;
 		break;
@@ -2020,6 +2026,12 @@ static int hns3_set_tunable(struct net_device *netdev,
 			netdev_info(netdev, "the active tx spare buf size is %u, due to page order\n",
 				    priv->ring->tx_spare->len);
 
+		break;
+	case ETHTOOL_PFC_PREVENTION_TOUT:
+		if (!h->ae_algo->ops->set_pfc_prevention_tout)
+			return -EOPNOTSUPP;
+
+		ret = h->ae_algo->ops->set_pfc_prevention_tout(h, *(u16 *)data);
 		break;
 	default:
 		ret = -EOPNOTSUPP;
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h
index 4ce92ddefcded..1c029dc32ab8f 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h
@@ -890,6 +890,15 @@ struct hclge_query_wol_supported_cmd {
 	u8 rsv[20];
 };
 
+struct hclge_pfc_storm_para_cmd {
+	__le32 dir;
+	__le32 enable;
+	__le32 period_ms;
+	__le32 times;
+	__le32 recovery_period_ms;
+	__le32 rsv;
+};
+
 struct hclge_hw;
 int hclge_cmd_send(struct hclge_hw *hw, struct hclge_desc *desc, int num);
 #endif
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
index fc8587c808137..a08d8a35aef98 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
@@ -3524,6 +3524,122 @@ static int hclge_set_vf_link_state(struct hnae3_handle *handle, int vf,
 	return ret;
 }
 
+static int hclge_set_pfc_storm_para(struct hclge_dev *hdev,
+				    struct hnae3_pfc_storm_para *para)
+{
+	struct hclge_pfc_storm_para_cmd *para_cmd;
+	struct hclge_desc desc;
+	int ret;
+
+	if (hdev->ae_dev->dev_version < HNAE3_DEVICE_VERSION_V3)
+		return -EOPNOTSUPP;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CFG_PAUSE_STORM_PARA,
+				   false);
+	para_cmd = (struct hclge_pfc_storm_para_cmd *)desc.data;
+	para_cmd->dir = cpu_to_le32(para->dir);
+	para_cmd->enable = cpu_to_le32(para->enable);
+	para_cmd->period_ms = cpu_to_le32(para->period_ms);
+	para_cmd->times = cpu_to_le32(para->times);
+	para_cmd->recovery_period_ms = cpu_to_le32(para->recovery_period_ms);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret)
+		dev_err(&hdev->pdev->dev,
+			"failed to set pfc storm para, ret = %d\n", ret);
+	return ret;
+}
+
+static int hclge_get_pfc_storm_para(struct hclge_dev *hdev,
+				    struct hnae3_pfc_storm_para *para)
+{
+	struct hclge_pfc_storm_para_cmd *para_cmd;
+	struct hclge_desc desc;
+	int ret;
+
+	if (hdev->ae_dev->dev_version < HNAE3_DEVICE_VERSION_V3)
+		return -EOPNOTSUPP;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CFG_PAUSE_STORM_PARA, true);
+	para_cmd = (struct hclge_pfc_storm_para_cmd *)desc.data;
+	para_cmd->dir = cpu_to_le32(para->dir);
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"failed to get pfc storm para, ret = %d\n", ret);
+		return ret;
+	}
+
+	para->enable = le32_to_cpu(para_cmd->enable);
+	para->period_ms = le32_to_cpu(para_cmd->period_ms);
+	para->times = le32_to_cpu(para_cmd->times);
+	para->recovery_period_ms = le32_to_cpu(para_cmd->recovery_period_ms);
+
+	return 0;
+}
+
+static int hclge_enable_pfc_storm_prevent(struct hclge_dev *hdev,
+					  int dir, bool enable)
+{
+	struct hnae3_pfc_storm_para para = {0};
+	int ret;
+
+	para.dir = dir;
+	ret = hclge_get_pfc_storm_para(hdev, &para);
+	if (ret)
+		return ret;
+
+	para.enable = enable;
+	return hclge_set_pfc_storm_para(hdev, &para);
+}
+
+static int hclge_set_pfc_prevention_tout(struct hnae3_handle *h, u16 times)
+{
+	struct hclge_vport *vport = hclge_get_vport(h);
+	struct hclge_dev *hdev = vport->back;
+	struct hnae3_pfc_storm_para para;
+	int ret;
+
+	if (times > HCLGE_MAX_PFC_PREVENTION_TOUT) {
+		dev_err(&hdev->pdev->dev,
+			"times %u should be no more than %u!\n",
+			times, HCLGE_MAX_PFC_PREVENTION_TOUT);
+		return -EINVAL;
+	}
+
+	para.dir = HCLGE_DIR_TX;
+	ret = hclge_get_pfc_storm_para(hdev, &para);
+	if (ret)
+		return ret;
+
+	para.enable = times ? 1 : 0;
+	para.times = (u32)times;
+	ret = hclge_set_pfc_storm_para(hdev, &para);
+	if (ret)
+		return ret;
+
+	hdev->pfc_prevention_tout = times;
+
+	return 0;
+}
+
+static int hclge_get_pfc_prevention_tout(struct hnae3_handle *h, u16 *times)
+{
+	struct hclge_vport *vport = hclge_get_vport(h);
+	struct hclge_dev *hdev = vport->back;
+	struct hnae3_pfc_storm_para para;
+	int ret;
+
+	para.dir = HCLGE_DIR_TX;
+	ret = hclge_get_pfc_storm_para(hdev, &para);
+	if (ret)
+		return ret;
+
+	*times = para.enable ? (u16)para.times : 0;
+
+	return 0;
+}
+
 static void hclge_set_reset_pending(struct hclge_dev *hdev,
 				    enum hnae3_reset_type reset_type)
 {
@@ -4317,6 +4433,26 @@ static int hclge_reset_prepare(struct hclge_dev *hdev)
 	return hclge_reset_prepare_wait(hdev);
 }
 
+static void hclge_restore_pfc_storm_prevention_tout(struct hclge_dev *hdev)
+{
+	struct hnae3_handle *handle = &hdev->vport[0].nic;
+	int ret;
+
+	ret = hclge_enable_pfc_storm_prevent(hdev, HCLGE_DIR_RX, false);
+	if (ret == -EOPNOTSUPP)
+		return;
+	else if (ret)
+		dev_warn(&hdev->pdev->dev,
+			 "failed to disable rx pfc storm prevent, ret = %d\n",
+			 ret);
+
+	ret = hclge_set_pfc_prevention_tout(handle, hdev->pfc_prevention_tout);
+	if (ret)
+		dev_warn(&hdev->pdev->dev,
+			 "failed to set tx pfc storm prevent, ret = %d\n",
+			 ret);
+}
+
 static int hclge_reset_rebuild(struct hclge_dev *hdev)
 {
 	int ret;
@@ -9278,6 +9414,32 @@ static int hclge_init_wol(struct hclge_dev *hdev)
 	return hclge_update_wol(hdev);
 }
 
+static void hclge_init_pfc_prevention_tout(struct hclge_dev *hdev)
+{
+	struct hnae3_handle *handle = &hdev->vport[0].nic;
+	u16 times;
+	int ret;
+
+	ret = hclge_enable_pfc_storm_prevent(hdev, HCLGE_DIR_RX, false);
+	if (ret == -EOPNOTSUPP)
+		return;
+	else if (ret)
+		dev_warn(&hdev->pdev->dev,
+			 "failed to disable rx pfc storm prevent, ret = %d\n",
+			 ret);
+
+	ret = hclge_get_pfc_prevention_tout(handle, &times);
+	if (ret) {
+		dev_warn(&hdev->pdev->dev,
+			 "failed to get tx pfc prevention timeout, ret = %d\n",
+			 ret);
+		times = HCLGE_DEFAULT_PFC_PREVENTION_TOUT;
+	}
+
+	hdev->pfc_prevention_tout = times;
+	hdev->pfc_prevention_tout_default = times;
+}
+
 static void hclge_get_wol(struct hnae3_handle *handle,
 			  struct ethtool_wolinfo *wol)
 {
@@ -9547,6 +9709,8 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev)
 		dev_warn(&pdev->dev,
 			 "failed to wake on lan init, ret = %d\n", ret);
 
+	hclge_init_pfc_prevention_tout(hdev);
+
 	ret = hclge_devlink_init(hdev);
 	if (ret)
 		goto err_ptp_uninit;
@@ -9946,6 +10110,8 @@ static int hclge_reset_ae_dev(struct hnae3_ae_dev *ae_dev)
 		dev_warn(&pdev->dev,
 			 "failed to update wol config, ret = %d\n", ret);
 
+	hclge_restore_pfc_storm_prevention_tout(hdev);
+
 	dev_info(&pdev->dev, "Reset done, %s driver initialization finished.\n",
 		 HCLGE_DRIVER_NAME);
 
@@ -9977,6 +10143,10 @@ static void hclge_uninit_ae_dev(struct hnae3_ae_dev *ae_dev)
 	hclge_config_nic_hw_error(hdev, false);
 	hclge_config_rocee_ras_interrupt(hdev, false);
 
+	/* Restore hw default values for the next initialization */
+	hclge_set_pfc_prevention_tout(&hdev->vport->nic,
+				      hdev->pfc_prevention_tout_default);
+
 	hclge_comm_cmd_uninit(hdev->ae_dev, &hdev->hw.hw);
 	hclge_misc_irq_uninit(hdev);
 	hclge_devlink_uninit(hdev);
@@ -10539,6 +10709,8 @@ static const struct hnae3_ae_ops hclge_ops = {
 	.set_wol = hclge_set_wol,
 	.hwtstamp_get = hclge_ptp_get_cfg,
 	.hwtstamp_set = hclge_ptp_set_cfg,
+	.set_pfc_prevention_tout = hclge_set_pfc_prevention_tout,
+	.get_pfc_prevention_tout = hclge_get_pfc_prevention_tout,
 };
 
 static struct hnae3_ae_algo ae_algo = {
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
index 7419481422c31..0cee8947f6b44 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
@@ -346,6 +346,11 @@ enum hclge_link_fail_code {
 #define HCLGE_LINK_STATUS_DOWN 0
 #define HCLGE_LINK_STATUS_UP   1
 
+#define HCLGE_DIR_RX 0
+#define HCLGE_DIR_TX 1
+#define HCLGE_MAX_PFC_PREVENTION_TOUT		2000
+#define HCLGE_DEFAULT_PFC_PREVENTION_TOUT	1000
+
 #define HCLGE_PG_NUM		4
 #define HCLGE_SCH_MODE_SP	0
 #define HCLGE_SCH_MODE_DWRR	1
@@ -898,6 +903,8 @@ struct hclge_dev {
 	u16 vf_rss_size_max;		/* HW defined VF max RSS task queue */
 	u16 pf_rss_size_max;		/* HW defined PF max RSS task queue */
 	u32 tx_spare_buf_size;		/* HW defined TX spare buffer size */
+	u16 pfc_prevention_tout;	/* User config, restored after reset */
+	u16 pfc_prevention_tout_default; /* HW default, to avoid stale state */
 
 	u16 fdir_pf_filter_count; /* Num of guaranteed filters for this PF */
 	u16 num_alloc_vport;		/* Num vports this driver supports */
diff --git a/drivers/net/ethernet/ibm/Kconfig b/drivers/net/ethernet/ibm/Kconfig
index 4f4b23465c478..8e55faac2035e 100644
--- a/drivers/net/ethernet/ibm/Kconfig
+++ b/drivers/net/ethernet/ibm/Kconfig
@@ -42,15 +42,6 @@ config IBMVETH_KUNIT_TEST
 
 source "drivers/net/ethernet/ibm/emac/Kconfig"
 
-config EHEA
-	tristate "eHEA Ethernet support"
-	depends on IBMEBUS && SPARSEMEM
-	help
-	  This driver supports the IBM pSeries eHEA ethernet adapter.
-
-	  To compile the driver as a module, choose M here. The module
-	  will be called ehea.
-
 config IBMVNIC
 	tristate "IBM Virtual NIC support"
 	depends on PPC_PSERIES
diff --git a/drivers/net/ethernet/ibm/Makefile b/drivers/net/ethernet/ibm/Makefile
index 1d17d0c33d4d3..c7e5d891c9461 100644
--- a/drivers/net/ethernet/ibm/Makefile
+++ b/drivers/net/ethernet/ibm/Makefile
@@ -6,4 +6,3 @@
 obj-$(CONFIG_IBMVETH) += ibmveth.o
 obj-$(CONFIG_IBMVNIC) += ibmvnic.o
 obj-$(CONFIG_IBM_EMAC) += emac/
-obj-$(CONFIG_EHEA) += ehea/
diff --git a/drivers/net/ethernet/ibm/ehea/Makefile b/drivers/net/ethernet/ibm/ehea/Makefile
deleted file mode 100644
index 9e1e5c7aafe20..0000000000000
--- a/drivers/net/ethernet/ibm/ehea/Makefile
+++ /dev/null
@@ -1,7 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# Makefile for the eHEA ethernet device driver for IBM eServer System p
-#
-ehea-y = ehea_main.o ehea_phyp.o ehea_qmr.o ehea_ethtool.o
-obj-$(CONFIG_EHEA) += ehea.o
-
diff --git a/drivers/net/ethernet/ibm/ehea/ehea.h b/drivers/net/ethernet/ibm/ehea/ehea.h
deleted file mode 100644
index 208c440a602ba..0000000000000
--- a/drivers/net/ethernet/ibm/ehea/ehea.h
+++ /dev/null
@@ -1,477 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- *  linux/drivers/net/ethernet/ibm/ehea/ehea.h
- *
- *  eHEA ethernet device driver for IBM eServer System p
- *
- *  (C) Copyright IBM Corp. 2006
- *
- *  Authors:
- *       Christoph Raisch <raisch@de.ibm.com>
- *       Jan-Bernd Themann <themann@de.ibm.com>
- *       Thomas Klein <tklein@de.ibm.com>
- */
-
-#ifndef __EHEA_H__
-#define __EHEA_H__
-
-#include <linux/module.h>
-#include <linux/ethtool.h>
-#include <linux/vmalloc.h>
-#include <linux/if_vlan.h>
-#include <linux/platform_device.h>
-
-#include <asm/ibmebus.h>
-#include <asm/io.h>
-
-#define DRV_NAME	"ehea"
-#define DRV_VERSION	"EHEA_0107"
-
-/* eHEA capability flags */
-#define DLPAR_PORT_ADD_REM 1
-#define DLPAR_MEM_ADD      2
-#define DLPAR_MEM_REM      4
-#define EHEA_CAPABILITIES  (DLPAR_PORT_ADD_REM | DLPAR_MEM_ADD | DLPAR_MEM_REM)
-
-#define EHEA_MSG_DEFAULT (NETIF_MSG_LINK | NETIF_MSG_TIMER \
-	| NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
-
-#define EHEA_MAX_ENTRIES_RQ1 32767
-#define EHEA_MAX_ENTRIES_RQ2 16383
-#define EHEA_MAX_ENTRIES_RQ3 16383
-#define EHEA_MAX_ENTRIES_SQ  32767
-#define EHEA_MIN_ENTRIES_QP  127
-
-#define EHEA_SMALL_QUEUES
-
-#ifdef EHEA_SMALL_QUEUES
-#define EHEA_MAX_CQE_COUNT      1023
-#define EHEA_DEF_ENTRIES_SQ     1023
-#define EHEA_DEF_ENTRIES_RQ1    1023
-#define EHEA_DEF_ENTRIES_RQ2    1023
-#define EHEA_DEF_ENTRIES_RQ3    511
-#else
-#define EHEA_MAX_CQE_COUNT      4080
-#define EHEA_DEF_ENTRIES_SQ     4080
-#define EHEA_DEF_ENTRIES_RQ1    8160
-#define EHEA_DEF_ENTRIES_RQ2    2040
-#define EHEA_DEF_ENTRIES_RQ3    2040
-#endif
-
-#define EHEA_MAX_ENTRIES_EQ 20
-
-#define EHEA_SG_SQ  2
-#define EHEA_SG_RQ1 1
-#define EHEA_SG_RQ2 0
-#define EHEA_SG_RQ3 0
-
-#define EHEA_MAX_PACKET_SIZE    9022	/* for jumbo frames */
-#define EHEA_RQ2_PKT_SIZE       2048
-#define EHEA_L_PKT_SIZE         256	/* low latency */
-
-/* Send completion signaling */
-
-/* Protection Domain Identifier */
-#define EHEA_PD_ID        0xaabcdeff
-
-#define EHEA_RQ2_THRESHOLD 	   1
-#define EHEA_RQ3_THRESHOLD	   4	/* use RQ3 threshold of 2048 bytes */
-
-#define EHEA_SPEED_10G         10000
-#define EHEA_SPEED_1G           1000
-#define EHEA_SPEED_100M          100
-#define EHEA_SPEED_10M            10
-#define EHEA_SPEED_AUTONEG         0
-
-/* Broadcast/Multicast registration types */
-#define EHEA_BCMC_SCOPE_ALL	0x08
-#define EHEA_BCMC_SCOPE_SINGLE	0x00
-#define EHEA_BCMC_MULTICAST	0x04
-#define EHEA_BCMC_BROADCAST	0x00
-#define EHEA_BCMC_UNTAGGED	0x02
-#define EHEA_BCMC_TAGGED	0x00
-#define EHEA_BCMC_VLANID_ALL	0x01
-#define EHEA_BCMC_VLANID_SINGLE	0x00
-
-#define EHEA_CACHE_LINE          128
-
-/* Memory Regions */
-#define EHEA_MR_ACC_CTRL       0x00800000
-
-#define EHEA_BUSMAP_START      0x8000000000000000ULL
-#define EHEA_INVAL_ADDR        0xFFFFFFFFFFFFFFFFULL
-#define EHEA_DIR_INDEX_SHIFT 13                   /* 8k Entries in 64k block */
-#define EHEA_TOP_INDEX_SHIFT (EHEA_DIR_INDEX_SHIFT * 2)
-#define EHEA_MAP_ENTRIES (1 << EHEA_DIR_INDEX_SHIFT)
-#define EHEA_MAP_SIZE (0x10000)                   /* currently fixed map size */
-#define EHEA_INDEX_MASK (EHEA_MAP_ENTRIES - 1)
-
-
-#define EHEA_WATCH_DOG_TIMEOUT 10*HZ
-
-/* utility functions */
-
-void ehea_dump(void *adr, int len, char *msg);
-
-#define EHEA_BMASK(pos, length) (((pos) << 16) + (length))
-
-#define EHEA_BMASK_IBM(from, to) (((63 - to) << 16) + ((to) - (from) + 1))
-
-#define EHEA_BMASK_SHIFTPOS(mask) (((mask) >> 16) & 0xffff)
-
-#define EHEA_BMASK_MASK(mask) \
-	(0xffffffffffffffffULL >> ((64 - (mask)) & 0xffff))
-
-#define EHEA_BMASK_SET(mask, value) \
-	((EHEA_BMASK_MASK(mask) & ((u64)(value))) << EHEA_BMASK_SHIFTPOS(mask))
-
-#define EHEA_BMASK_GET(mask, value) \
-	(EHEA_BMASK_MASK(mask) & (((u64)(value)) >> EHEA_BMASK_SHIFTPOS(mask)))
-
-/*
- * Generic ehea page
- */
-struct ehea_page {
-	u8 entries[PAGE_SIZE];
-};
-
-/*
- * Generic queue in linux kernel virtual memory
- */
-struct hw_queue {
-	u64 current_q_offset;		/* current queue entry */
-	struct ehea_page **queue_pages;	/* array of pages belonging to queue */
-	u32 qe_size;			/* queue entry size */
-	u32 queue_length;      		/* queue length allocated in bytes */
-	u32 pagesize;
-	u32 toggle_state;		/* toggle flag - per page */
-	u32 reserved;			/* 64 bit alignment */
-};
-
-/*
- * For pSeries this is a 64bit memory address where
- * I/O memory is mapped into CPU address space
- */
-struct h_epa {
-	void __iomem *addr;
-};
-
-struct h_epa_user {
-	u64 addr;
-};
-
-struct h_epas {
-	struct h_epa kernel;	/* kernel space accessible resource,
-				   set to 0 if unused */
-	struct h_epa_user user;	/* user space accessible resource
-				   set to 0 if unused */
-};
-
-/*
- * Memory map data structures
- */
-struct ehea_dir_bmap
-{
-	u64 ent[EHEA_MAP_ENTRIES];
-};
-struct ehea_top_bmap
-{
-	struct ehea_dir_bmap *dir[EHEA_MAP_ENTRIES];
-};
-struct ehea_bmap
-{
-	struct ehea_top_bmap *top[EHEA_MAP_ENTRIES];
-};
-
-struct ehea_qp;
-struct ehea_cq;
-struct ehea_eq;
-struct ehea_port;
-struct ehea_av;
-
-/*
- * Queue attributes passed to ehea_create_qp()
- */
-struct ehea_qp_init_attr {
-	/* input parameter */
-	u32 qp_token;           /* queue token */
-	u8 low_lat_rq1;
-	u8 signalingtype;       /* cqe generation flag */
-	u8 rq_count;            /* num of receive queues */
-	u8 eqe_gen;             /* eqe generation flag */
-	u16 max_nr_send_wqes;   /* max number of send wqes */
-	u16 max_nr_rwqes_rq1;   /* max number of receive wqes */
-	u16 max_nr_rwqes_rq2;
-	u16 max_nr_rwqes_rq3;
-	u8 wqe_size_enc_sq;
-	u8 wqe_size_enc_rq1;
-	u8 wqe_size_enc_rq2;
-	u8 wqe_size_enc_rq3;
-	u8 swqe_imm_data_len;   /* immediate data length for swqes */
-	u16 port_nr;
-	u16 rq2_threshold;
-	u16 rq3_threshold;
-	u64 send_cq_handle;
-	u64 recv_cq_handle;
-	u64 aff_eq_handle;
-
-	/* output parameter */
-	u32 qp_nr;
-	u16 act_nr_send_wqes;
-	u16 act_nr_rwqes_rq1;
-	u16 act_nr_rwqes_rq2;
-	u16 act_nr_rwqes_rq3;
-	u8 act_wqe_size_enc_sq;
-	u8 act_wqe_size_enc_rq1;
-	u8 act_wqe_size_enc_rq2;
-	u8 act_wqe_size_enc_rq3;
-	u32 nr_sq_pages;
-	u32 nr_rq1_pages;
-	u32 nr_rq2_pages;
-	u32 nr_rq3_pages;
-	u32 liobn_sq;
-	u32 liobn_rq1;
-	u32 liobn_rq2;
-	u32 liobn_rq3;
-};
-
-/*
- * Event Queue attributes, passed as parameter
- */
-struct ehea_eq_attr {
-	u32 type;
-	u32 max_nr_of_eqes;
-	u8 eqe_gen;        /* generate eqe flag */
-	u64 eq_handle;
-	u32 act_nr_of_eqes;
-	u32 nr_pages;
-	u32 ist1;          /* Interrupt service token */
-	u32 ist2;
-	u32 ist3;
-	u32 ist4;
-};
-
-
-/*
- * Event Queue
- */
-struct ehea_eq {
-	struct ehea_adapter *adapter;
-	struct hw_queue hw_queue;
-	u64 fw_handle;
-	struct h_epas epas;
-	spinlock_t spinlock;
-	struct ehea_eq_attr attr;
-};
-
-/*
- * HEA Queues
- */
-struct ehea_qp {
-	struct ehea_adapter *adapter;
-	u64 fw_handle;			/* QP handle for firmware calls */
-	struct hw_queue hw_squeue;
-	struct hw_queue hw_rqueue1;
-	struct hw_queue hw_rqueue2;
-	struct hw_queue hw_rqueue3;
-	struct h_epas epas;
-	struct ehea_qp_init_attr init_attr;
-};
-
-/*
- * Completion Queue attributes
- */
-struct ehea_cq_attr {
-	/* input parameter */
-	u32 max_nr_of_cqes;
-	u32 cq_token;
-	u64 eq_handle;
-
-	/* output parameter */
-	u32 act_nr_of_cqes;
-	u32 nr_pages;
-};
-
-/*
- * Completion Queue
- */
-struct ehea_cq {
-	struct ehea_adapter *adapter;
-	u64 fw_handle;
-	struct hw_queue hw_queue;
-	struct h_epas epas;
-	struct ehea_cq_attr attr;
-};
-
-/*
- * Memory Region
- */
-struct ehea_mr {
-	struct ehea_adapter *adapter;
-	u64 handle;
-	u64 vaddr;
-	u32 lkey;
-};
-
-/*
- * Port state information
- */
-struct port_stats {
-	int poll_receive_errors;
-	int queue_stopped;
-	int err_tcp_cksum;
-	int err_ip_cksum;
-	int err_frame_crc;
-};
-
-#define EHEA_IRQ_NAME_SIZE 20
-
-/*
- * Queue SKB Array
- */
-struct ehea_q_skb_arr {
-	struct sk_buff **arr;		/* skb array for queue */
-	int len;                	/* array length */
-	int index;			/* array index */
-	int os_skbs;			/* rq2/rq3 only: outstanding skbs */
-};
-
-/*
- * Port resources
- */
-struct ehea_port_res {
-	struct napi_struct napi;
-	struct port_stats p_stats;
-	struct ehea_mr send_mr;       	/* send memory region */
-	struct ehea_mr recv_mr;       	/* receive memory region */
-	struct ehea_port *port;
-	char int_recv_name[EHEA_IRQ_NAME_SIZE];
-	char int_send_name[EHEA_IRQ_NAME_SIZE];
-	struct ehea_qp *qp;
-	struct ehea_cq *send_cq;
-	struct ehea_cq *recv_cq;
-	struct ehea_eq *eq;
-	struct ehea_q_skb_arr rq1_skba;
-	struct ehea_q_skb_arr rq2_skba;
-	struct ehea_q_skb_arr rq3_skba;
-	struct ehea_q_skb_arr sq_skba;
-	int sq_skba_size;
-	int swqe_refill_th;
-	atomic_t swqe_avail;
-	int swqe_ll_count;
-	u32 swqe_id_counter;
-	u64 tx_packets;
-	u64 tx_bytes;
-	u64 rx_packets;
-	u64 rx_bytes;
-	int sq_restart_flag;
-};
-
-
-#define EHEA_MAX_PORTS 16
-
-#define EHEA_NUM_PORTRES_FW_HANDLES    6  /* QP handle, SendCQ handle,
-					     RecvCQ handle, EQ handle,
-					     SendMR handle, RecvMR handle */
-#define EHEA_NUM_PORT_FW_HANDLES       1  /* EQ handle */
-#define EHEA_NUM_ADAPTER_FW_HANDLES    2  /* MR handle, NEQ handle */
-
-struct ehea_adapter {
-	u64 handle;
-	struct platform_device *ofdev;
-	struct ehea_port *port[EHEA_MAX_PORTS];
-	struct ehea_eq *neq;       /* notification event queue */
-	struct tasklet_struct neq_tasklet;
-	struct ehea_mr mr;
-	u32 pd;                    /* protection domain */
-	u64 max_mc_mac;            /* max number of multicast mac addresses */
-	int active_ports;
-	struct list_head list;
-};
-
-
-struct ehea_mc_list {
-	struct list_head list;
-	u64 macaddr;
-};
-
-/* kdump support */
-struct ehea_fw_handle_entry {
-	u64 adh;               /* Adapter Handle */
-	u64 fwh;               /* Firmware Handle */
-};
-
-struct ehea_fw_handle_array {
-	struct ehea_fw_handle_entry *arr;
-	int num_entries;
-	struct mutex lock;
-};
-
-struct ehea_bcmc_reg_entry {
-	u64 adh;               /* Adapter Handle */
-	u32 port_id;           /* Logical Port Id */
-	u8 reg_type;           /* Registration Type */
-	u64 macaddr;
-};
-
-struct ehea_bcmc_reg_array {
-	struct ehea_bcmc_reg_entry *arr;
-	int num_entries;
-	spinlock_t lock;
-};
-
-#define EHEA_PORT_UP 1
-#define EHEA_PORT_DOWN 0
-#define EHEA_PHY_LINK_UP 1
-#define EHEA_PHY_LINK_DOWN 0
-#define EHEA_MAX_PORT_RES 16
-struct ehea_port {
-	struct ehea_adapter *adapter;	 /* adapter that owns this port */
-	struct net_device *netdev;
-	struct rtnl_link_stats64 stats;
-	struct ehea_port_res port_res[EHEA_MAX_PORT_RES];
-	struct platform_device  ofdev; /* Open Firmware Device */
-	struct ehea_mc_list *mc_list;	 /* Multicast MAC addresses */
-	struct ehea_eq *qp_eq;
-	struct work_struct reset_task;
-	struct delayed_work stats_work;
-	struct mutex port_lock;
-	char int_aff_name[EHEA_IRQ_NAME_SIZE];
-	int allmulti;			 /* Indicates IFF_ALLMULTI state */
-	int promisc;		 	 /* Indicates IFF_PROMISC state */
-	int num_mcs;
-	int resets;
-	unsigned long flags;
-	u64 mac_addr;
-	u32 logical_port_id;
-	u32 port_speed;
-	u32 msg_enable;
-	u32 sig_comp_iv;
-	u32 state;
-	u8 phy_link;
-	u8 full_duplex;
-	u8 autoneg;
-	u8 num_def_qps;
-	wait_queue_head_t swqe_avail_wq;
-	wait_queue_head_t restart_wq;
-};
-
-struct port_res_cfg {
-	int max_entries_rcq;
-	int max_entries_scq;
-	int max_entries_sq;
-	int max_entries_rq1;
-	int max_entries_rq2;
-	int max_entries_rq3;
-};
-
-enum ehea_flag_bits {
-	__EHEA_STOP_XFER,
-	__EHEA_DISABLE_PORT_RESET
-};
-
-void ehea_set_ethtool_ops(struct net_device *netdev);
-int ehea_sense_port_attr(struct ehea_port *port);
-int ehea_set_portspeed(struct ehea_port *port, u32 port_speed);
-
-#endif	/* __EHEA_H__ */
diff --git a/drivers/net/ethernet/ibm/ehea/ehea_ethtool.c b/drivers/net/ethernet/ibm/ehea/ehea_ethtool.c
deleted file mode 100644
index 1db5b6790a41a..0000000000000
--- a/drivers/net/ethernet/ibm/ehea/ehea_ethtool.c
+++ /dev/null
@@ -1,277 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- *  linux/drivers/net/ethernet/ibm/ehea/ehea_ethtool.c
- *
- *  eHEA ethernet device driver for IBM eServer System p
- *
- *  (C) Copyright IBM Corp. 2006
- *
- *  Authors:
- *       Christoph Raisch <raisch@de.ibm.com>
- *       Jan-Bernd Themann <themann@de.ibm.com>
- *       Thomas Klein <tklein@de.ibm.com>
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include "ehea.h"
-#include "ehea_phyp.h"
-
-static int ehea_get_link_ksettings(struct net_device *dev,
-				   struct ethtool_link_ksettings *cmd)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	u32 supported, advertising;
-	u32 speed;
-	int ret;
-
-	ret = ehea_sense_port_attr(port);
-
-	if (ret)
-		return ret;
-
-	if (netif_carrier_ok(dev)) {
-		switch (port->port_speed) {
-		case EHEA_SPEED_10M:
-			speed = SPEED_10;
-			break;
-		case EHEA_SPEED_100M:
-			speed = SPEED_100;
-			break;
-		case EHEA_SPEED_1G:
-			speed = SPEED_1000;
-			break;
-		case EHEA_SPEED_10G:
-			speed = SPEED_10000;
-			break;
-		default:
-			speed = -1;
-			break; /* BUG */
-		}
-		cmd->base.duplex = port->full_duplex == 1 ?
-						     DUPLEX_FULL : DUPLEX_HALF;
-	} else {
-		speed = SPEED_UNKNOWN;
-		cmd->base.duplex = DUPLEX_UNKNOWN;
-	}
-	cmd->base.speed = speed;
-
-	if (cmd->base.speed == SPEED_10000) {
-		supported = (SUPPORTED_10000baseT_Full | SUPPORTED_FIBRE);
-		advertising = (ADVERTISED_10000baseT_Full | ADVERTISED_FIBRE);
-		cmd->base.port = PORT_FIBRE;
-	} else {
-		supported = (SUPPORTED_1000baseT_Full | SUPPORTED_100baseT_Full
-			       | SUPPORTED_100baseT_Half | SUPPORTED_10baseT_Full
-			       | SUPPORTED_10baseT_Half | SUPPORTED_Autoneg
-			       | SUPPORTED_TP);
-		advertising = (ADVERTISED_1000baseT_Full | ADVERTISED_Autoneg
-				 | ADVERTISED_TP);
-		cmd->base.port = PORT_TP;
-	}
-
-	cmd->base.autoneg = port->autoneg == 1 ?
-		AUTONEG_ENABLE : AUTONEG_DISABLE;
-
-	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
-						supported);
-	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising,
-						advertising);
-
-	return 0;
-}
-
-static int ehea_set_link_ksettings(struct net_device *dev,
-				   const struct ethtool_link_ksettings *cmd)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	int ret = 0;
-	u32 sp;
-
-	if (cmd->base.autoneg == AUTONEG_ENABLE) {
-		sp = EHEA_SPEED_AUTONEG;
-		goto doit;
-	}
-
-	switch (cmd->base.speed) {
-	case SPEED_10:
-		if (cmd->base.duplex == DUPLEX_FULL)
-			sp = H_SPEED_10M_F;
-		else
-			sp = H_SPEED_10M_H;
-		break;
-
-	case SPEED_100:
-		if (cmd->base.duplex == DUPLEX_FULL)
-			sp = H_SPEED_100M_F;
-		else
-			sp = H_SPEED_100M_H;
-		break;
-
-	case SPEED_1000:
-		if (cmd->base.duplex == DUPLEX_FULL)
-			sp = H_SPEED_1G_F;
-		else
-			ret = -EINVAL;
-		break;
-
-	case SPEED_10000:
-		if (cmd->base.duplex == DUPLEX_FULL)
-			sp = H_SPEED_10G_F;
-		else
-			ret = -EINVAL;
-		break;
-
-	default:
-			ret = -EINVAL;
-		break;
-	}
-
-	if (ret)
-		goto out;
-doit:
-	ret = ehea_set_portspeed(port, sp);
-
-	if (!ret)
-		netdev_info(dev,
-			    "Port speed successfully set: %dMbps %s Duplex\n",
-			    port->port_speed,
-			    port->full_duplex == 1 ? "Full" : "Half");
-out:
-	return ret;
-}
-
-static int ehea_nway_reset(struct net_device *dev)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	int ret;
-
-	ret = ehea_set_portspeed(port, EHEA_SPEED_AUTONEG);
-
-	if (!ret)
-		netdev_info(port->netdev,
-			    "Port speed successfully set: %dMbps %s Duplex\n",
-			    port->port_speed,
-			    port->full_duplex == 1 ? "Full" : "Half");
-	return ret;
-}
-
-static void ehea_get_drvinfo(struct net_device *dev,
-			       struct ethtool_drvinfo *info)
-{
-	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strscpy(info->version, DRV_VERSION, sizeof(info->version));
-}
-
-static u32 ehea_get_msglevel(struct net_device *dev)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	return port->msg_enable;
-}
-
-static void ehea_set_msglevel(struct net_device *dev, u32 value)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	port->msg_enable = value;
-}
-
-static const char ehea_ethtool_stats_keys[][ETH_GSTRING_LEN] = {
-	{"sig_comp_iv"},
-	{"swqe_refill_th"},
-	{"port resets"},
-	{"Receive errors"},
-	{"TCP cksum errors"},
-	{"IP cksum errors"},
-	{"Frame cksum errors"},
-	{"num SQ stopped"},
-	{"PR0 free_swqes"},
-	{"PR1 free_swqes"},
-	{"PR2 free_swqes"},
-	{"PR3 free_swqes"},
-	{"PR4 free_swqes"},
-	{"PR5 free_swqes"},
-	{"PR6 free_swqes"},
-	{"PR7 free_swqes"},
-	{"PR8 free_swqes"},
-	{"PR9 free_swqes"},
-	{"PR10 free_swqes"},
-	{"PR11 free_swqes"},
-	{"PR12 free_swqes"},
-	{"PR13 free_swqes"},
-	{"PR14 free_swqes"},
-	{"PR15 free_swqes"},
-};
-
-static void ehea_get_strings(struct net_device *dev, u32 stringset, u8 *data)
-{
-	if (stringset == ETH_SS_STATS) {
-		memcpy(data, &ehea_ethtool_stats_keys,
-		       sizeof(ehea_ethtool_stats_keys));
-	}
-}
-
-static int ehea_get_sset_count(struct net_device *dev, int sset)
-{
-	switch (sset) {
-	case ETH_SS_STATS:
-		return ARRAY_SIZE(ehea_ethtool_stats_keys);
-	default:
-		return -EOPNOTSUPP;
-	}
-}
-
-static void ehea_get_ethtool_stats(struct net_device *dev,
-				     struct ethtool_stats *stats, u64 *data)
-{
-	int i, k, tmp;
-	struct ehea_port *port = netdev_priv(dev);
-
-	for (i = 0; i < ehea_get_sset_count(dev, ETH_SS_STATS); i++)
-		data[i] = 0;
-	i = 0;
-
-	data[i++] = port->sig_comp_iv;
-	data[i++] = port->port_res[0].swqe_refill_th;
-	data[i++] = port->resets;
-
-	for (k = 0, tmp = 0; k < EHEA_MAX_PORT_RES; k++)
-		tmp += port->port_res[k].p_stats.poll_receive_errors;
-	data[i++] = tmp;
-
-	for (k = 0, tmp = 0; k < EHEA_MAX_PORT_RES; k++)
-		tmp += port->port_res[k].p_stats.err_tcp_cksum;
-	data[i++] = tmp;
-
-	for (k = 0, tmp = 0; k < EHEA_MAX_PORT_RES; k++)
-		tmp += port->port_res[k].p_stats.err_ip_cksum;
-	data[i++] = tmp;
-
-	for (k = 0, tmp = 0; k < EHEA_MAX_PORT_RES; k++)
-		tmp += port->port_res[k].p_stats.err_frame_crc;
-	data[i++] = tmp;
-
-	for (k = 0, tmp = 0; k < EHEA_MAX_PORT_RES; k++)
-		tmp += port->port_res[k].p_stats.queue_stopped;
-	data[i++] = tmp;
-
-	for (k = 0; k < 16; k++)
-		data[i++] = atomic_read(&port->port_res[k].swqe_avail);
-}
-
-static const struct ethtool_ops ehea_ethtool_ops = {
-	.get_drvinfo = ehea_get_drvinfo,
-	.get_msglevel = ehea_get_msglevel,
-	.set_msglevel = ehea_set_msglevel,
-	.get_link = ethtool_op_get_link,
-	.get_strings = ehea_get_strings,
-	.get_sset_count = ehea_get_sset_count,
-	.get_ethtool_stats = ehea_get_ethtool_stats,
-	.nway_reset = ehea_nway_reset,		/* Restart autonegotiation */
-	.get_link_ksettings = ehea_get_link_ksettings,
-	.set_link_ksettings = ehea_set_link_ksettings,
-};
-
-void ehea_set_ethtool_ops(struct net_device *netdev)
-{
-	netdev->ethtool_ops = &ehea_ethtool_ops;
-}
diff --git a/drivers/net/ethernet/ibm/ehea/ehea_hw.h b/drivers/net/ethernet/ibm/ehea/ehea_hw.h
deleted file mode 100644
index 590933a45d65b..0000000000000
--- a/drivers/net/ethernet/ibm/ehea/ehea_hw.h
+++ /dev/null
@@ -1,253 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- *  linux/drivers/net/ethernet/ibm/ehea/ehea_hw.h
- *
- *  eHEA ethernet device driver for IBM eServer System p
- *
- *  (C) Copyright IBM Corp. 2006
- *
- *  Authors:
- *       Christoph Raisch <raisch@de.ibm.com>
- *       Jan-Bernd Themann <themann@de.ibm.com>
- *       Thomas Klein <tklein@de.ibm.com>
- */
-
-#ifndef __EHEA_HW_H__
-#define __EHEA_HW_H__
-
-#define QPX_SQA_VALUE   EHEA_BMASK_IBM(48, 63)
-#define QPX_RQ1A_VALUE  EHEA_BMASK_IBM(48, 63)
-#define QPX_RQ2A_VALUE  EHEA_BMASK_IBM(48, 63)
-#define QPX_RQ3A_VALUE  EHEA_BMASK_IBM(48, 63)
-
-#define QPTEMM_OFFSET(x) offsetof(struct ehea_qptemm, x)
-
-struct ehea_qptemm {
-	u64 qpx_hcr;
-	u64 qpx_c;
-	u64 qpx_herr;
-	u64 qpx_aer;
-	u64 qpx_sqa;
-	u64 qpx_sqc;
-	u64 qpx_rq1a;
-	u64 qpx_rq1c;
-	u64 qpx_st;
-	u64 qpx_aerr;
-	u64 qpx_tenure;
-	u64 qpx_reserved1[(0x098 - 0x058) / 8];
-	u64 qpx_portp;
-	u64 qpx_reserved2[(0x100 - 0x0A0) / 8];
-	u64 qpx_t;
-	u64 qpx_sqhp;
-	u64 qpx_sqptp;
-	u64 qpx_reserved3[(0x140 - 0x118) / 8];
-	u64 qpx_sqwsize;
-	u64 qpx_reserved4[(0x170 - 0x148) / 8];
-	u64 qpx_sqsize;
-	u64 qpx_reserved5[(0x1B0 - 0x178) / 8];
-	u64 qpx_sigt;
-	u64 qpx_wqecnt;
-	u64 qpx_rq1hp;
-	u64 qpx_rq1ptp;
-	u64 qpx_rq1size;
-	u64 qpx_reserved6[(0x220 - 0x1D8) / 8];
-	u64 qpx_rq1wsize;
-	u64 qpx_reserved7[(0x240 - 0x228) / 8];
-	u64 qpx_pd;
-	u64 qpx_scqn;
-	u64 qpx_rcqn;
-	u64 qpx_aeqn;
-	u64 reserved49;
-	u64 qpx_ram;
-	u64 qpx_reserved8[(0x300 - 0x270) / 8];
-	u64 qpx_rq2a;
-	u64 qpx_rq2c;
-	u64 qpx_rq2hp;
-	u64 qpx_rq2ptp;
-	u64 qpx_rq2size;
-	u64 qpx_rq2wsize;
-	u64 qpx_rq2th;
-	u64 qpx_rq3a;
-	u64 qpx_rq3c;
-	u64 qpx_rq3hp;
-	u64 qpx_rq3ptp;
-	u64 qpx_rq3size;
-	u64 qpx_rq3wsize;
-	u64 qpx_rq3th;
-	u64 qpx_lpn;
-	u64 qpx_reserved9[(0x400 - 0x378) / 8];
-	u64 reserved_ext[(0x500 - 0x400) / 8];
-	u64 reserved2[(0x1000 - 0x500) / 8];
-};
-
-#define MRx_HCR_LPARID_VALID EHEA_BMASK_IBM(0, 0)
-
-#define MRMWMM_OFFSET(x) offsetof(struct ehea_mrmwmm, x)
-
-struct ehea_mrmwmm {
-	u64 mrx_hcr;
-	u64 mrx_c;
-	u64 mrx_herr;
-	u64 mrx_aer;
-	u64 mrx_pp;
-	u64 reserved1;
-	u64 reserved2;
-	u64 reserved3;
-	u64 reserved4[(0x200 - 0x40) / 8];
-	u64 mrx_ctl[64];
-};
-
-#define QPEDMM_OFFSET(x) offsetof(struct ehea_qpedmm, x)
-
-struct ehea_qpedmm {
-
-	u64 reserved0[(0x400) / 8];
-	u64 qpedx_phh;
-	u64 qpedx_ppsgp;
-	u64 qpedx_ppsgu;
-	u64 qpedx_ppdgp;
-	u64 qpedx_ppdgu;
-	u64 qpedx_aph;
-	u64 qpedx_apsgp;
-	u64 qpedx_apsgu;
-	u64 qpedx_apdgp;
-	u64 qpedx_apdgu;
-	u64 qpedx_apav;
-	u64 qpedx_apsav;
-	u64 qpedx_hcr;
-	u64 reserved1[4];
-	u64 qpedx_rrl0;
-	u64 qpedx_rrrkey0;
-	u64 qpedx_rrva0;
-	u64 reserved2;
-	u64 qpedx_rrl1;
-	u64 qpedx_rrrkey1;
-	u64 qpedx_rrva1;
-	u64 reserved3;
-	u64 qpedx_rrl2;
-	u64 qpedx_rrrkey2;
-	u64 qpedx_rrva2;
-	u64 reserved4;
-	u64 qpedx_rrl3;
-	u64 qpedx_rrrkey3;
-	u64 qpedx_rrva3;
-};
-
-#define CQX_FECADDER EHEA_BMASK_IBM(32, 63)
-#define CQX_FEC_CQE_CNT EHEA_BMASK_IBM(32, 63)
-#define CQX_N1_GENERATE_COMP_EVENT EHEA_BMASK_IBM(0, 0)
-#define CQX_EP_EVENT_PENDING EHEA_BMASK_IBM(0, 0)
-
-#define CQTEMM_OFFSET(x) offsetof(struct ehea_cqtemm, x)
-
-struct ehea_cqtemm {
-	u64 cqx_hcr;
-	u64 cqx_c;
-	u64 cqx_herr;
-	u64 cqx_aer;
-	u64 cqx_ptp;
-	u64 cqx_tp;
-	u64 cqx_fec;
-	u64 cqx_feca;
-	u64 cqx_ep;
-	u64 cqx_eq;
-	u64 reserved1;
-	u64 cqx_n0;
-	u64 cqx_n1;
-	u64 reserved2[(0x1000 - 0x60) / 8];
-};
-
-#define EQTEMM_OFFSET(x) offsetof(struct ehea_eqtemm, x)
-
-struct ehea_eqtemm {
-	u64 eqx_hcr;
-	u64 eqx_c;
-	u64 eqx_herr;
-	u64 eqx_aer;
-	u64 eqx_ptp;
-	u64 eqx_tp;
-	u64 eqx_ssba;
-	u64 eqx_psba;
-	u64 eqx_cec;
-	u64 eqx_meql;
-	u64 eqx_xisbi;
-	u64 eqx_xisc;
-	u64 eqx_it;
-};
-
-/*
- * These access functions will be changed when the dissuccsion about
- * the new access methods for POWER has settled.
- */
-
-static inline u64 epa_load(struct h_epa epa, u32 offset)
-{
-	return __raw_readq((void __iomem *)(epa.addr + offset));
-}
-
-static inline void epa_store(struct h_epa epa, u32 offset, u64 value)
-{
-	__raw_writeq(value, (void __iomem *)(epa.addr + offset));
-	epa_load(epa, offset);	/* synchronize explicitly to eHEA */
-}
-
-static inline void epa_store_acc(struct h_epa epa, u32 offset, u64 value)
-{
-	__raw_writeq(value, (void __iomem *)(epa.addr + offset));
-}
-
-#define epa_store_cq(epa, offset, value)\
-	epa_store(epa, CQTEMM_OFFSET(offset), value)
-#define epa_load_cq(epa, offset)\
-	epa_load(epa, CQTEMM_OFFSET(offset))
-
-static inline void ehea_update_sqa(struct ehea_qp *qp, u16 nr_wqes)
-{
-	struct h_epa epa = qp->epas.kernel;
-	epa_store_acc(epa, QPTEMM_OFFSET(qpx_sqa),
-		      EHEA_BMASK_SET(QPX_SQA_VALUE, nr_wqes));
-}
-
-static inline void ehea_update_rq3a(struct ehea_qp *qp, u16 nr_wqes)
-{
-	struct h_epa epa = qp->epas.kernel;
-	epa_store_acc(epa, QPTEMM_OFFSET(qpx_rq3a),
-		      EHEA_BMASK_SET(QPX_RQ1A_VALUE, nr_wqes));
-}
-
-static inline void ehea_update_rq2a(struct ehea_qp *qp, u16 nr_wqes)
-{
-	struct h_epa epa = qp->epas.kernel;
-	epa_store_acc(epa, QPTEMM_OFFSET(qpx_rq2a),
-		      EHEA_BMASK_SET(QPX_RQ2A_VALUE, nr_wqes));
-}
-
-static inline void ehea_update_rq1a(struct ehea_qp *qp, u16 nr_wqes)
-{
-	struct h_epa epa = qp->epas.kernel;
-	epa_store_acc(epa, QPTEMM_OFFSET(qpx_rq1a),
-		      EHEA_BMASK_SET(QPX_RQ3A_VALUE, nr_wqes));
-}
-
-static inline void ehea_update_feca(struct ehea_cq *cq, u32 nr_cqes)
-{
-	struct h_epa epa = cq->epas.kernel;
-	epa_store_acc(epa, CQTEMM_OFFSET(cqx_feca),
-		      EHEA_BMASK_SET(CQX_FECADDER, nr_cqes));
-}
-
-static inline void ehea_reset_cq_n1(struct ehea_cq *cq)
-{
-	struct h_epa epa = cq->epas.kernel;
-	epa_store_cq(epa, cqx_n1,
-		     EHEA_BMASK_SET(CQX_N1_GENERATE_COMP_EVENT, 1));
-}
-
-static inline void ehea_reset_cq_ep(struct ehea_cq *my_cq)
-{
-	struct h_epa epa = my_cq->epas.kernel;
-	epa_store_acc(epa, CQTEMM_OFFSET(cqx_ep),
-		      EHEA_BMASK_SET(CQX_EP_EVENT_PENDING, 0));
-}
-
-#endif	/* __EHEA_HW_H__ */
diff --git a/drivers/net/ethernet/ibm/ehea/ehea_main.c b/drivers/net/ethernet/ibm/ehea/ehea_main.c
deleted file mode 100644
index bfc8699a05b94..0000000000000
--- a/drivers/net/ethernet/ibm/ehea/ehea_main.c
+++ /dev/null
@@ -1,3581 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- *  linux/drivers/net/ethernet/ibm/ehea/ehea_main.c
- *
- *  eHEA ethernet device driver for IBM eServer System p
- *
- *  (C) Copyright IBM Corp. 2006
- *
- *  Authors:
- *	 Christoph Raisch <raisch@de.ibm.com>
- *	 Jan-Bernd Themann <themann@de.ibm.com>
- *	 Thomas Klein <tklein@de.ibm.com>
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/device.h>
-#include <linux/in.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <linux/udp.h>
-#include <linux/if.h>
-#include <linux/list.h>
-#include <linux/slab.h>
-#include <linux/if_ether.h>
-#include <linux/notifier.h>
-#include <linux/reboot.h>
-#include <linux/memory.h>
-#include <asm/kexec.h>
-#include <linux/mutex.h>
-#include <linux/prefetch.h>
-#include <linux/of.h>
-#include <linux/of_device.h>
-#include <linux/platform_device.h>
-
-#include <net/ip.h>
-
-#include "ehea.h"
-#include "ehea_qmr.h"
-#include "ehea_phyp.h"
-
-
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Christoph Raisch <raisch@de.ibm.com>");
-MODULE_DESCRIPTION("IBM eServer HEA Driver");
-MODULE_VERSION(DRV_VERSION);
-
-
-static int msg_level = -1;
-static int rq1_entries = EHEA_DEF_ENTRIES_RQ1;
-static int rq2_entries = EHEA_DEF_ENTRIES_RQ2;
-static int rq3_entries = EHEA_DEF_ENTRIES_RQ3;
-static int sq_entries = EHEA_DEF_ENTRIES_SQ;
-static int use_mcs = 1;
-static int prop_carrier_state;
-
-module_param(msg_level, int, 0);
-module_param(rq1_entries, int, 0);
-module_param(rq2_entries, int, 0);
-module_param(rq3_entries, int, 0);
-module_param(sq_entries, int, 0);
-module_param(prop_carrier_state, int, 0);
-module_param(use_mcs, int, 0);
-
-MODULE_PARM_DESC(msg_level, "msg_level");
-MODULE_PARM_DESC(prop_carrier_state, "Propagate carrier state of physical "
-		 "port to stack. 1:yes, 0:no.  Default = 0 ");
-MODULE_PARM_DESC(rq3_entries, "Number of entries for Receive Queue 3 "
-		 "[2^x - 1], x = [7..14]. Default = "
-		 __MODULE_STRING(EHEA_DEF_ENTRIES_RQ3) ")");
-MODULE_PARM_DESC(rq2_entries, "Number of entries for Receive Queue 2 "
-		 "[2^x - 1], x = [7..14]. Default = "
-		 __MODULE_STRING(EHEA_DEF_ENTRIES_RQ2) ")");
-MODULE_PARM_DESC(rq1_entries, "Number of entries for Receive Queue 1 "
-		 "[2^x - 1], x = [7..14]. Default = "
-		 __MODULE_STRING(EHEA_DEF_ENTRIES_RQ1) ")");
-MODULE_PARM_DESC(sq_entries, " Number of entries for the Send Queue  "
-		 "[2^x - 1], x = [7..14]. Default = "
-		 __MODULE_STRING(EHEA_DEF_ENTRIES_SQ) ")");
-MODULE_PARM_DESC(use_mcs, " Multiple receive queues, 1: enable, 0: disable, "
-		 "Default = 1");
-
-static int port_name_cnt;
-static LIST_HEAD(adapter_list);
-static unsigned long ehea_driver_flags;
-static DEFINE_MUTEX(dlpar_mem_lock);
-static struct ehea_fw_handle_array ehea_fw_handles;
-static struct ehea_bcmc_reg_array ehea_bcmc_regs;
-
-
-static int ehea_probe_adapter(struct platform_device *dev);
-
-static void ehea_remove(struct platform_device *dev);
-
-static const struct of_device_id ehea_module_device_table[] = {
-	{
-		.name = "lhea",
-		.compatible = "IBM,lhea",
-	},
-	{
-		.type = "network",
-		.compatible = "IBM,lhea-ethernet",
-	},
-	{},
-};
-MODULE_DEVICE_TABLE(of, ehea_module_device_table);
-
-static const struct of_device_id ehea_device_table[] = {
-	{
-		.name = "lhea",
-		.compatible = "IBM,lhea",
-	},
-	{},
-};
-MODULE_DEVICE_TABLE(of, ehea_device_table);
-
-static struct platform_driver ehea_driver = {
-	.driver = {
-		.name = "ehea",
-		.owner = THIS_MODULE,
-		.of_match_table = ehea_device_table,
-	},
-	.probe = ehea_probe_adapter,
-	.remove = ehea_remove,
-};
-
-void ehea_dump(void *adr, int len, char *msg)
-{
-	int x;
-	unsigned char *deb = adr;
-	for (x = 0; x < len; x += 16) {
-		pr_info("%s adr=%p ofs=%04x %016llx %016llx\n",
-			msg, deb, x, *((u64 *)&deb[0]), *((u64 *)&deb[8]));
-		deb += 16;
-	}
-}
-
-static void ehea_schedule_port_reset(struct ehea_port *port)
-{
-	if (!test_bit(__EHEA_DISABLE_PORT_RESET, &port->flags))
-		schedule_work(&port->reset_task);
-}
-
-static void ehea_update_firmware_handles(void)
-{
-	struct ehea_fw_handle_entry *arr = NULL;
-	struct ehea_adapter *adapter;
-	int num_adapters = 0;
-	int num_ports = 0;
-	int num_portres = 0;
-	int i = 0;
-	int num_fw_handles, k, l;
-
-	/* Determine number of handles */
-	mutex_lock(&ehea_fw_handles.lock);
-
-	list_for_each_entry(adapter, &adapter_list, list) {
-		num_adapters++;
-
-		for (k = 0; k < EHEA_MAX_PORTS; k++) {
-			struct ehea_port *port = adapter->port[k];
-
-			if (!port || (port->state != EHEA_PORT_UP))
-				continue;
-
-			num_ports++;
-			num_portres += port->num_def_qps;
-		}
-	}
-
-	num_fw_handles = num_adapters * EHEA_NUM_ADAPTER_FW_HANDLES +
-			 num_ports * EHEA_NUM_PORT_FW_HANDLES +
-			 num_portres * EHEA_NUM_PORTRES_FW_HANDLES;
-
-	if (num_fw_handles) {
-		arr = kzalloc_objs(*arr, num_fw_handles);
-		if (!arr)
-			goto out;  /* Keep the existing array */
-	} else
-		goto out_update;
-
-	list_for_each_entry(adapter, &adapter_list, list) {
-		if (num_adapters == 0)
-			break;
-
-		for (k = 0; k < EHEA_MAX_PORTS; k++) {
-			struct ehea_port *port = adapter->port[k];
-
-			if (!port || (port->state != EHEA_PORT_UP) ||
-			    (num_ports == 0))
-				continue;
-
-			for (l = 0; l < port->num_def_qps; l++) {
-				struct ehea_port_res *pr = &port->port_res[l];
-
-				arr[i].adh = adapter->handle;
-				arr[i++].fwh = pr->qp->fw_handle;
-				arr[i].adh = adapter->handle;
-				arr[i++].fwh = pr->send_cq->fw_handle;
-				arr[i].adh = adapter->handle;
-				arr[i++].fwh = pr->recv_cq->fw_handle;
-				arr[i].adh = adapter->handle;
-				arr[i++].fwh = pr->eq->fw_handle;
-				arr[i].adh = adapter->handle;
-				arr[i++].fwh = pr->send_mr.handle;
-				arr[i].adh = adapter->handle;
-				arr[i++].fwh = pr->recv_mr.handle;
-			}
-			arr[i].adh = adapter->handle;
-			arr[i++].fwh = port->qp_eq->fw_handle;
-			num_ports--;
-		}
-
-		arr[i].adh = adapter->handle;
-		arr[i++].fwh = adapter->neq->fw_handle;
-
-		if (adapter->mr.handle) {
-			arr[i].adh = adapter->handle;
-			arr[i++].fwh = adapter->mr.handle;
-		}
-		num_adapters--;
-	}
-
-out_update:
-	kfree(ehea_fw_handles.arr);
-	ehea_fw_handles.arr = arr;
-	ehea_fw_handles.num_entries = i;
-out:
-	mutex_unlock(&ehea_fw_handles.lock);
-}
-
-static void ehea_update_bcmc_registrations(void)
-{
-	unsigned long flags;
-	struct ehea_bcmc_reg_entry *arr = NULL;
-	struct ehea_adapter *adapter;
-	struct ehea_mc_list *mc_entry;
-	int num_registrations = 0;
-	int i = 0;
-	int k;
-
-	spin_lock_irqsave(&ehea_bcmc_regs.lock, flags);
-
-	/* Determine number of registrations */
-	list_for_each_entry(adapter, &adapter_list, list)
-		for (k = 0; k < EHEA_MAX_PORTS; k++) {
-			struct ehea_port *port = adapter->port[k];
-
-			if (!port || (port->state != EHEA_PORT_UP))
-				continue;
-
-			num_registrations += 2;	/* Broadcast registrations */
-
-			list_for_each_entry(mc_entry, &port->mc_list->list,list)
-				num_registrations += 2;
-		}
-
-	if (num_registrations) {
-		arr = kzalloc_objs(*arr, num_registrations, GFP_ATOMIC);
-		if (!arr)
-			goto out;  /* Keep the existing array */
-	} else
-		goto out_update;
-
-	list_for_each_entry(adapter, &adapter_list, list) {
-		for (k = 0; k < EHEA_MAX_PORTS; k++) {
-			struct ehea_port *port = adapter->port[k];
-
-			if (!port || (port->state != EHEA_PORT_UP))
-				continue;
-
-			if (num_registrations == 0)
-				goto out_update;
-
-			arr[i].adh = adapter->handle;
-			arr[i].port_id = port->logical_port_id;
-			arr[i].reg_type = EHEA_BCMC_BROADCAST |
-					  EHEA_BCMC_UNTAGGED;
-			arr[i++].macaddr = port->mac_addr;
-
-			arr[i].adh = adapter->handle;
-			arr[i].port_id = port->logical_port_id;
-			arr[i].reg_type = EHEA_BCMC_BROADCAST |
-					  EHEA_BCMC_VLANID_ALL;
-			arr[i++].macaddr = port->mac_addr;
-			num_registrations -= 2;
-
-			list_for_each_entry(mc_entry,
-					    &port->mc_list->list, list) {
-				if (num_registrations == 0)
-					goto out_update;
-
-				arr[i].adh = adapter->handle;
-				arr[i].port_id = port->logical_port_id;
-				arr[i].reg_type = EHEA_BCMC_MULTICAST |
-						  EHEA_BCMC_UNTAGGED;
-				if (mc_entry->macaddr == 0)
-					arr[i].reg_type |= EHEA_BCMC_SCOPE_ALL;
-				arr[i++].macaddr = mc_entry->macaddr;
-
-				arr[i].adh = adapter->handle;
-				arr[i].port_id = port->logical_port_id;
-				arr[i].reg_type = EHEA_BCMC_MULTICAST |
-						  EHEA_BCMC_VLANID_ALL;
-				if (mc_entry->macaddr == 0)
-					arr[i].reg_type |= EHEA_BCMC_SCOPE_ALL;
-				arr[i++].macaddr = mc_entry->macaddr;
-				num_registrations -= 2;
-			}
-		}
-	}
-
-out_update:
-	kfree(ehea_bcmc_regs.arr);
-	ehea_bcmc_regs.arr = arr;
-	ehea_bcmc_regs.num_entries = i;
-out:
-	spin_unlock_irqrestore(&ehea_bcmc_regs.lock, flags);
-}
-
-static void ehea_get_stats64(struct net_device *dev,
-			     struct rtnl_link_stats64 *stats)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	u64 rx_packets = 0, tx_packets = 0, rx_bytes = 0, tx_bytes = 0;
-	int i;
-
-	for (i = 0; i < port->num_def_qps; i++) {
-		rx_packets += port->port_res[i].rx_packets;
-		rx_bytes   += port->port_res[i].rx_bytes;
-	}
-
-	for (i = 0; i < port->num_def_qps; i++) {
-		tx_packets += port->port_res[i].tx_packets;
-		tx_bytes   += port->port_res[i].tx_bytes;
-	}
-
-	stats->tx_packets = tx_packets;
-	stats->rx_bytes = rx_bytes;
-	stats->tx_bytes = tx_bytes;
-	stats->rx_packets = rx_packets;
-
-	stats->multicast = port->stats.multicast;
-	stats->rx_errors = port->stats.rx_errors;
-}
-
-static void ehea_update_stats(struct work_struct *work)
-{
-	struct ehea_port *port =
-		container_of(work, struct ehea_port, stats_work.work);
-	struct net_device *dev = port->netdev;
-	struct rtnl_link_stats64 *stats = &port->stats;
-	struct hcp_ehea_port_cb2 *cb2;
-	u64 hret;
-
-	cb2 = (void *)get_zeroed_page(GFP_KERNEL);
-	if (!cb2) {
-		netdev_err(dev, "No mem for cb2. Some interface statistics were not updated\n");
-		goto resched;
-	}
-
-	hret = ehea_h_query_ehea_port(port->adapter->handle,
-				      port->logical_port_id,
-				      H_PORT_CB2, H_PORT_CB2_ALL, cb2);
-	if (hret != H_SUCCESS) {
-		netdev_err(dev, "query_ehea_port failed\n");
-		goto out_herr;
-	}
-
-	if (netif_msg_hw(port))
-		ehea_dump(cb2, sizeof(*cb2), "net_device_stats");
-
-	stats->multicast = cb2->rxmcp;
-	stats->rx_errors = cb2->rxuerr;
-
-out_herr:
-	free_page((unsigned long)cb2);
-resched:
-	schedule_delayed_work(&port->stats_work,
-			      round_jiffies_relative(msecs_to_jiffies(1000)));
-}
-
-static void ehea_refill_rq1(struct ehea_port_res *pr, int index, int nr_of_wqes)
-{
-	struct sk_buff **skb_arr_rq1 = pr->rq1_skba.arr;
-	struct net_device *dev = pr->port->netdev;
-	int max_index_mask = pr->rq1_skba.len - 1;
-	int fill_wqes = pr->rq1_skba.os_skbs + nr_of_wqes;
-	int adder = 0;
-	int i;
-
-	pr->rq1_skba.os_skbs = 0;
-
-	if (unlikely(test_bit(__EHEA_STOP_XFER, &ehea_driver_flags))) {
-		if (nr_of_wqes > 0)
-			pr->rq1_skba.index = index;
-		pr->rq1_skba.os_skbs = fill_wqes;
-		return;
-	}
-
-	for (i = 0; i < fill_wqes; i++) {
-		if (!skb_arr_rq1[index]) {
-			skb_arr_rq1[index] = netdev_alloc_skb(dev,
-							      EHEA_L_PKT_SIZE);
-			if (!skb_arr_rq1[index]) {
-				pr->rq1_skba.os_skbs = fill_wqes - i;
-				break;
-			}
-		}
-		index--;
-		index &= max_index_mask;
-		adder++;
-	}
-
-	if (adder == 0)
-		return;
-
-	/* Ring doorbell */
-	ehea_update_rq1a(pr->qp, adder);
-}
-
-static void ehea_init_fill_rq1(struct ehea_port_res *pr, int nr_rq1a)
-{
-	struct sk_buff **skb_arr_rq1 = pr->rq1_skba.arr;
-	struct net_device *dev = pr->port->netdev;
-	int i;
-
-	if (nr_rq1a > pr->rq1_skba.len) {
-		netdev_err(dev, "NR_RQ1A bigger than skb array len\n");
-		return;
-	}
-
-	for (i = 0; i < nr_rq1a; i++) {
-		skb_arr_rq1[i] = netdev_alloc_skb(dev, EHEA_L_PKT_SIZE);
-		if (!skb_arr_rq1[i])
-			break;
-	}
-	/* Ring doorbell */
-	ehea_update_rq1a(pr->qp, i - 1);
-}
-
-static int ehea_refill_rq_def(struct ehea_port_res *pr,
-			      struct ehea_q_skb_arr *q_skba, int rq_nr,
-			      int num_wqes, int wqe_type, int packet_size)
-{
-	struct net_device *dev = pr->port->netdev;
-	struct ehea_qp *qp = pr->qp;
-	struct sk_buff **skb_arr = q_skba->arr;
-	struct ehea_rwqe *rwqe;
-	int i, index, max_index_mask, fill_wqes;
-	int adder = 0;
-	int ret = 0;
-
-	fill_wqes = q_skba->os_skbs + num_wqes;
-	q_skba->os_skbs = 0;
-
-	if (unlikely(test_bit(__EHEA_STOP_XFER, &ehea_driver_flags))) {
-		q_skba->os_skbs = fill_wqes;
-		return ret;
-	}
-
-	index = q_skba->index;
-	max_index_mask = q_skba->len - 1;
-	for (i = 0; i < fill_wqes; i++) {
-		u64 tmp_addr;
-		struct sk_buff *skb;
-
-		skb = netdev_alloc_skb_ip_align(dev, packet_size);
-		if (!skb) {
-			q_skba->os_skbs = fill_wqes - i;
-			if (q_skba->os_skbs == q_skba->len - 2) {
-				netdev_info(pr->port->netdev,
-					    "rq%i ran dry - no mem for skb\n",
-					    rq_nr);
-				ret = -ENOMEM;
-			}
-			break;
-		}
-
-		skb_arr[index] = skb;
-		tmp_addr = ehea_map_vaddr(skb->data);
-		if (tmp_addr == -1) {
-			dev_consume_skb_any(skb);
-			q_skba->os_skbs = fill_wqes - i;
-			ret = 0;
-			break;
-		}
-
-		rwqe = ehea_get_next_rwqe(qp, rq_nr);
-		rwqe->wr_id = EHEA_BMASK_SET(EHEA_WR_ID_TYPE, wqe_type)
-			    | EHEA_BMASK_SET(EHEA_WR_ID_INDEX, index);
-		rwqe->sg_list[0].l_key = pr->recv_mr.lkey;
-		rwqe->sg_list[0].vaddr = tmp_addr;
-		rwqe->sg_list[0].len = packet_size;
-		rwqe->data_segments = 1;
-
-		index++;
-		index &= max_index_mask;
-		adder++;
-	}
-
-	q_skba->index = index;
-	if (adder == 0)
-		goto out;
-
-	/* Ring doorbell */
-	iosync();
-	if (rq_nr == 2)
-		ehea_update_rq2a(pr->qp, adder);
-	else
-		ehea_update_rq3a(pr->qp, adder);
-out:
-	return ret;
-}
-
-
-static int ehea_refill_rq2(struct ehea_port_res *pr, int nr_of_wqes)
-{
-	return ehea_refill_rq_def(pr, &pr->rq2_skba, 2,
-				  nr_of_wqes, EHEA_RWQE2_TYPE,
-				  EHEA_RQ2_PKT_SIZE);
-}
-
-
-static int ehea_refill_rq3(struct ehea_port_res *pr, int nr_of_wqes)
-{
-	return ehea_refill_rq_def(pr, &pr->rq3_skba, 3,
-				  nr_of_wqes, EHEA_RWQE3_TYPE,
-				  EHEA_MAX_PACKET_SIZE);
-}
-
-static inline int ehea_check_cqe(struct ehea_cqe *cqe, int *rq_num)
-{
-	*rq_num = (cqe->type & EHEA_CQE_TYPE_RQ) >> 5;
-	if ((cqe->status & EHEA_CQE_STAT_ERR_MASK) == 0)
-		return 0;
-	if (((cqe->status & EHEA_CQE_STAT_ERR_TCP) != 0) &&
-	    (cqe->header_length == 0))
-		return 0;
-	return -EINVAL;
-}
-
-static inline void ehea_fill_skb(struct net_device *dev,
-				 struct sk_buff *skb, struct ehea_cqe *cqe,
-				 struct ehea_port_res *pr)
-{
-	int length = cqe->num_bytes_transfered - 4;	/*remove CRC */
-
-	skb_put(skb, length);
-	skb->protocol = eth_type_trans(skb, dev);
-
-	/* The packet was not an IPV4 packet so a complemented checksum was
-	   calculated. The value is found in the Internet Checksum field. */
-	if (cqe->status & EHEA_CQE_BLIND_CKSUM) {
-		skb->ip_summed = CHECKSUM_COMPLETE;
-		skb->csum = csum_unfold(~cqe->inet_checksum_value);
-	} else
-		skb->ip_summed = CHECKSUM_UNNECESSARY;
-
-	skb_record_rx_queue(skb, pr - &pr->port->port_res[0]);
-}
-
-static inline struct sk_buff *get_skb_by_index(struct sk_buff **skb_array,
-					       int arr_len,
-					       struct ehea_cqe *cqe)
-{
-	int skb_index = EHEA_BMASK_GET(EHEA_WR_ID_INDEX, cqe->wr_id);
-	struct sk_buff *skb;
-	void *pref;
-	int x;
-
-	x = skb_index + 1;
-	x &= (arr_len - 1);
-
-	pref = skb_array[x];
-	if (pref) {
-		prefetchw(pref);
-		prefetchw(pref + EHEA_CACHE_LINE);
-
-		pref = (skb_array[x]->data);
-		prefetch(pref);
-		prefetch(pref + EHEA_CACHE_LINE);
-		prefetch(pref + EHEA_CACHE_LINE * 2);
-		prefetch(pref + EHEA_CACHE_LINE * 3);
-	}
-
-	skb = skb_array[skb_index];
-	skb_array[skb_index] = NULL;
-	return skb;
-}
-
-static inline struct sk_buff *get_skb_by_index_ll(struct sk_buff **skb_array,
-						  int arr_len, int wqe_index)
-{
-	struct sk_buff *skb;
-	void *pref;
-	int x;
-
-	x = wqe_index + 1;
-	x &= (arr_len - 1);
-
-	pref = skb_array[x];
-	if (pref) {
-		prefetchw(pref);
-		prefetchw(pref + EHEA_CACHE_LINE);
-
-		pref = (skb_array[x]->data);
-		prefetchw(pref);
-		prefetchw(pref + EHEA_CACHE_LINE);
-	}
-
-	skb = skb_array[wqe_index];
-	skb_array[wqe_index] = NULL;
-	return skb;
-}
-
-static int ehea_treat_poll_error(struct ehea_port_res *pr, int rq,
-				 struct ehea_cqe *cqe, int *processed_rq2,
-				 int *processed_rq3)
-{
-	struct sk_buff *skb;
-
-	if (cqe->status & EHEA_CQE_STAT_ERR_TCP)
-		pr->p_stats.err_tcp_cksum++;
-	if (cqe->status & EHEA_CQE_STAT_ERR_IP)
-		pr->p_stats.err_ip_cksum++;
-	if (cqe->status & EHEA_CQE_STAT_ERR_CRC)
-		pr->p_stats.err_frame_crc++;
-
-	if (rq == 2) {
-		*processed_rq2 += 1;
-		skb = get_skb_by_index(pr->rq2_skba.arr, pr->rq2_skba.len, cqe);
-		dev_kfree_skb(skb);
-	} else if (rq == 3) {
-		*processed_rq3 += 1;
-		skb = get_skb_by_index(pr->rq3_skba.arr, pr->rq3_skba.len, cqe);
-		dev_kfree_skb(skb);
-	}
-
-	if (cqe->status & EHEA_CQE_STAT_FAT_ERR_MASK) {
-		if (netif_msg_rx_err(pr->port)) {
-			pr_err("Critical receive error for QP %d. Resetting port.\n",
-			       pr->qp->init_attr.qp_nr);
-			ehea_dump(cqe, sizeof(*cqe), "CQE");
-		}
-		ehea_schedule_port_reset(pr->port);
-		return 1;
-	}
-
-	return 0;
-}
-
-static int ehea_proc_rwqes(struct net_device *dev,
-			   struct ehea_port_res *pr,
-			   int budget)
-{
-	struct ehea_port *port = pr->port;
-	struct ehea_qp *qp = pr->qp;
-	struct ehea_cqe *cqe;
-	struct sk_buff *skb;
-	struct sk_buff **skb_arr_rq1 = pr->rq1_skba.arr;
-	struct sk_buff **skb_arr_rq2 = pr->rq2_skba.arr;
-	struct sk_buff **skb_arr_rq3 = pr->rq3_skba.arr;
-	int skb_arr_rq1_len = pr->rq1_skba.len;
-	int skb_arr_rq2_len = pr->rq2_skba.len;
-	int skb_arr_rq3_len = pr->rq3_skba.len;
-	int processed, processed_rq1, processed_rq2, processed_rq3;
-	u64 processed_bytes = 0;
-	int wqe_index, last_wqe_index, rq, port_reset;
-
-	processed = processed_rq1 = processed_rq2 = processed_rq3 = 0;
-	last_wqe_index = 0;
-
-	cqe = ehea_poll_rq1(qp, &wqe_index);
-	while ((processed < budget) && cqe) {
-		ehea_inc_rq1(qp);
-		processed_rq1++;
-		processed++;
-		if (netif_msg_rx_status(port))
-			ehea_dump(cqe, sizeof(*cqe), "CQE");
-
-		last_wqe_index = wqe_index;
-		rmb();
-		if (!ehea_check_cqe(cqe, &rq)) {
-			if (rq == 1) {
-				/* LL RQ1 */
-				skb = get_skb_by_index_ll(skb_arr_rq1,
-							  skb_arr_rq1_len,
-							  wqe_index);
-				if (unlikely(!skb)) {
-					netif_info(port, rx_err, dev,
-						  "LL rq1: skb=NULL\n");
-
-					skb = netdev_alloc_skb(dev,
-							       EHEA_L_PKT_SIZE);
-					if (!skb)
-						break;
-				}
-				skb_copy_to_linear_data(skb, ((char *)cqe) + 64,
-						 cqe->num_bytes_transfered - 4);
-				ehea_fill_skb(dev, skb, cqe, pr);
-			} else if (rq == 2) {
-				/* RQ2 */
-				skb = get_skb_by_index(skb_arr_rq2,
-						       skb_arr_rq2_len, cqe);
-				if (unlikely(!skb)) {
-					netif_err(port, rx_err, dev,
-						  "rq2: skb=NULL\n");
-					break;
-				}
-				ehea_fill_skb(dev, skb, cqe, pr);
-				processed_rq2++;
-			} else {
-				/* RQ3 */
-				skb = get_skb_by_index(skb_arr_rq3,
-						       skb_arr_rq3_len, cqe);
-				if (unlikely(!skb)) {
-					netif_err(port, rx_err, dev,
-						  "rq3: skb=NULL\n");
-					break;
-				}
-				ehea_fill_skb(dev, skb, cqe, pr);
-				processed_rq3++;
-			}
-
-			processed_bytes += skb->len;
-
-			if (cqe->status & EHEA_CQE_VLAN_TAG_XTRACT)
-				__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
-						       cqe->vlan_tag);
-
-			napi_gro_receive(&pr->napi, skb);
-		} else {
-			pr->p_stats.poll_receive_errors++;
-			port_reset = ehea_treat_poll_error(pr, rq, cqe,
-							   &processed_rq2,
-							   &processed_rq3);
-			if (port_reset)
-				break;
-		}
-		cqe = ehea_poll_rq1(qp, &wqe_index);
-	}
-
-	pr->rx_packets += processed;
-	pr->rx_bytes += processed_bytes;
-
-	ehea_refill_rq1(pr, last_wqe_index, processed_rq1);
-	ehea_refill_rq2(pr, processed_rq2);
-	ehea_refill_rq3(pr, processed_rq3);
-
-	return processed;
-}
-
-#define SWQE_RESTART_CHECK 0xdeadbeaff00d0000ull
-
-static void reset_sq_restart_flag(struct ehea_port *port)
-{
-	int i;
-
-	for (i = 0; i < port->num_def_qps; i++) {
-		struct ehea_port_res *pr = &port->port_res[i];
-		pr->sq_restart_flag = 0;
-	}
-	wake_up(&port->restart_wq);
-}
-
-static void check_sqs(struct ehea_port *port)
-{
-	struct ehea_swqe *swqe;
-	int swqe_index;
-	int i;
-
-	for (i = 0; i < port->num_def_qps; i++) {
-		struct ehea_port_res *pr = &port->port_res[i];
-		int ret;
-		swqe = ehea_get_swqe(pr->qp, &swqe_index);
-		memset(swqe, 0, SWQE_HEADER_SIZE);
-		atomic_dec(&pr->swqe_avail);
-
-		swqe->tx_control |= EHEA_SWQE_PURGE;
-		swqe->wr_id = SWQE_RESTART_CHECK;
-		swqe->tx_control |= EHEA_SWQE_SIGNALLED_COMPLETION;
-		swqe->tx_control |= EHEA_SWQE_IMM_DATA_PRESENT;
-		swqe->immediate_data_length = 80;
-
-		ehea_post_swqe(pr->qp, swqe);
-
-		ret = wait_event_timeout(port->restart_wq,
-					 pr->sq_restart_flag == 0,
-					 msecs_to_jiffies(100));
-
-		if (!ret) {
-			pr_err("HW/SW queues out of sync\n");
-			ehea_schedule_port_reset(pr->port);
-			return;
-		}
-	}
-}
-
-
-static struct ehea_cqe *ehea_proc_cqes(struct ehea_port_res *pr, int my_quota)
-{
-	struct sk_buff *skb;
-	struct ehea_cq *send_cq = pr->send_cq;
-	struct ehea_cqe *cqe;
-	int quota = my_quota;
-	int cqe_counter = 0;
-	int swqe_av = 0;
-	int index;
-	struct netdev_queue *txq = netdev_get_tx_queue(pr->port->netdev,
-						pr - &pr->port->port_res[0]);
-
-	cqe = ehea_poll_cq(send_cq);
-	while (cqe && (quota > 0)) {
-		ehea_inc_cq(send_cq);
-
-		cqe_counter++;
-		rmb();
-
-		if (cqe->wr_id == SWQE_RESTART_CHECK) {
-			pr->sq_restart_flag = 1;
-			swqe_av++;
-			break;
-		}
-
-		if (cqe->status & EHEA_CQE_STAT_ERR_MASK) {
-			pr_err("Bad send completion status=0x%04X\n",
-			       cqe->status);
-
-			if (netif_msg_tx_err(pr->port))
-				ehea_dump(cqe, sizeof(*cqe), "Send CQE");
-
-			if (cqe->status & EHEA_CQE_STAT_RESET_MASK) {
-				pr_err("Resetting port\n");
-				ehea_schedule_port_reset(pr->port);
-				break;
-			}
-		}
-
-		if (netif_msg_tx_done(pr->port))
-			ehea_dump(cqe, sizeof(*cqe), "CQE");
-
-		if (likely(EHEA_BMASK_GET(EHEA_WR_ID_TYPE, cqe->wr_id)
-			   == EHEA_SWQE2_TYPE)) {
-
-			index = EHEA_BMASK_GET(EHEA_WR_ID_INDEX, cqe->wr_id);
-			skb = pr->sq_skba.arr[index];
-			dev_consume_skb_any(skb);
-			pr->sq_skba.arr[index] = NULL;
-		}
-
-		swqe_av += EHEA_BMASK_GET(EHEA_WR_ID_REFILL, cqe->wr_id);
-		quota--;
-
-		cqe = ehea_poll_cq(send_cq);
-	}
-
-	ehea_update_feca(send_cq, cqe_counter);
-	atomic_add(swqe_av, &pr->swqe_avail);
-
-	if (unlikely(netif_tx_queue_stopped(txq) &&
-		     (atomic_read(&pr->swqe_avail) >= pr->swqe_refill_th))) {
-		__netif_tx_lock(txq, smp_processor_id());
-		if (netif_tx_queue_stopped(txq) &&
-		    (atomic_read(&pr->swqe_avail) >= pr->swqe_refill_th))
-			netif_tx_wake_queue(txq);
-		__netif_tx_unlock(txq);
-	}
-
-	wake_up(&pr->port->swqe_avail_wq);
-
-	return cqe;
-}
-
-#define EHEA_POLL_MAX_CQES 65535
-
-static int ehea_poll(struct napi_struct *napi, int budget)
-{
-	struct ehea_port_res *pr = container_of(napi, struct ehea_port_res,
-						napi);
-	struct net_device *dev = pr->port->netdev;
-	struct ehea_cqe *cqe;
-	struct ehea_cqe *cqe_skb = NULL;
-	int wqe_index;
-	int rx = 0;
-
-	cqe_skb = ehea_proc_cqes(pr, EHEA_POLL_MAX_CQES);
-	rx += ehea_proc_rwqes(dev, pr, budget - rx);
-
-	while (rx != budget) {
-		napi_complete(napi);
-		ehea_reset_cq_ep(pr->recv_cq);
-		ehea_reset_cq_ep(pr->send_cq);
-		ehea_reset_cq_n1(pr->recv_cq);
-		ehea_reset_cq_n1(pr->send_cq);
-		rmb();
-		cqe = ehea_poll_rq1(pr->qp, &wqe_index);
-		cqe_skb = ehea_poll_cq(pr->send_cq);
-
-		if (!cqe && !cqe_skb)
-			return rx;
-
-		if (!napi_schedule(napi))
-			return rx;
-
-		cqe_skb = ehea_proc_cqes(pr, EHEA_POLL_MAX_CQES);
-		rx += ehea_proc_rwqes(dev, pr, budget - rx);
-	}
-
-	return rx;
-}
-
-static irqreturn_t ehea_recv_irq_handler(int irq, void *param)
-{
-	struct ehea_port_res *pr = param;
-
-	napi_schedule(&pr->napi);
-
-	return IRQ_HANDLED;
-}
-
-static irqreturn_t ehea_qp_aff_irq_handler(int irq, void *param)
-{
-	struct ehea_port *port = param;
-	struct ehea_eqe *eqe;
-	struct ehea_qp *qp;
-	u32 qp_token;
-	u64 resource_type, aer, aerr;
-	int reset_port = 0;
-
-	eqe = ehea_poll_eq(port->qp_eq);
-
-	while (eqe) {
-		qp_token = EHEA_BMASK_GET(EHEA_EQE_QP_TOKEN, eqe->entry);
-		pr_err("QP aff_err: entry=0x%llx, token=0x%x\n",
-		       eqe->entry, qp_token);
-
-		qp = port->port_res[qp_token].qp;
-
-		resource_type = ehea_error_data(port->adapter, qp->fw_handle,
-						&aer, &aerr);
-
-		if (resource_type == EHEA_AER_RESTYPE_QP) {
-			if ((aer & EHEA_AER_RESET_MASK) ||
-			    (aerr & EHEA_AERR_RESET_MASK))
-				 reset_port = 1;
-		} else
-			reset_port = 1;   /* Reset in case of CQ or EQ error */
-
-		eqe = ehea_poll_eq(port->qp_eq);
-	}
-
-	if (reset_port) {
-		pr_err("Resetting port\n");
-		ehea_schedule_port_reset(port);
-	}
-
-	return IRQ_HANDLED;
-}
-
-static struct ehea_port *ehea_get_port(struct ehea_adapter *adapter,
-				       int logical_port)
-{
-	int i;
-
-	for (i = 0; i < EHEA_MAX_PORTS; i++)
-		if (adapter->port[i])
-			if (adapter->port[i]->logical_port_id == logical_port)
-				return adapter->port[i];
-	return NULL;
-}
-
-int ehea_sense_port_attr(struct ehea_port *port)
-{
-	int ret;
-	u64 hret;
-	struct hcp_ehea_port_cb0 *cb0;
-
-	/* may be called via ehea_neq_tasklet() */
-	cb0 = (void *)get_zeroed_page(GFP_ATOMIC);
-	if (!cb0) {
-		pr_err("no mem for cb0\n");
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	hret = ehea_h_query_ehea_port(port->adapter->handle,
-				      port->logical_port_id, H_PORT_CB0,
-				      EHEA_BMASK_SET(H_PORT_CB0_ALL, 0xFFFF),
-				      cb0);
-	if (hret != H_SUCCESS) {
-		ret = -EIO;
-		goto out_free;
-	}
-
-	/* MAC address */
-	port->mac_addr = cb0->port_mac_addr << 16;
-
-	if (!is_valid_ether_addr((u8 *)&port->mac_addr)) {
-		ret = -EADDRNOTAVAIL;
-		goto out_free;
-	}
-
-	/* Port speed */
-	switch (cb0->port_speed) {
-	case H_SPEED_10M_H:
-		port->port_speed = EHEA_SPEED_10M;
-		port->full_duplex = 0;
-		break;
-	case H_SPEED_10M_F:
-		port->port_speed = EHEA_SPEED_10M;
-		port->full_duplex = 1;
-		break;
-	case H_SPEED_100M_H:
-		port->port_speed = EHEA_SPEED_100M;
-		port->full_duplex = 0;
-		break;
-	case H_SPEED_100M_F:
-		port->port_speed = EHEA_SPEED_100M;
-		port->full_duplex = 1;
-		break;
-	case H_SPEED_1G_F:
-		port->port_speed = EHEA_SPEED_1G;
-		port->full_duplex = 1;
-		break;
-	case H_SPEED_10G_F:
-		port->port_speed = EHEA_SPEED_10G;
-		port->full_duplex = 1;
-		break;
-	default:
-		port->port_speed = 0;
-		port->full_duplex = 0;
-		break;
-	}
-
-	port->autoneg = 1;
-	port->num_mcs = cb0->num_default_qps;
-
-	/* Number of default QPs */
-	if (use_mcs)
-		port->num_def_qps = cb0->num_default_qps;
-	else
-		port->num_def_qps = 1;
-
-	if (!port->num_def_qps) {
-		ret = -EINVAL;
-		goto out_free;
-	}
-
-	ret = 0;
-out_free:
-	if (ret || netif_msg_probe(port))
-		ehea_dump(cb0, sizeof(*cb0), "ehea_sense_port_attr");
-	free_page((unsigned long)cb0);
-out:
-	return ret;
-}
-
-int ehea_set_portspeed(struct ehea_port *port, u32 port_speed)
-{
-	struct hcp_ehea_port_cb4 *cb4;
-	u64 hret;
-	int ret = 0;
-
-	cb4 = (void *)get_zeroed_page(GFP_KERNEL);
-	if (!cb4) {
-		pr_err("no mem for cb4\n");
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	cb4->port_speed = port_speed;
-
-	netif_carrier_off(port->netdev);
-
-	hret = ehea_h_modify_ehea_port(port->adapter->handle,
-				       port->logical_port_id,
-				       H_PORT_CB4, H_PORT_CB4_SPEED, cb4);
-	if (hret == H_SUCCESS) {
-		port->autoneg = port_speed == EHEA_SPEED_AUTONEG ? 1 : 0;
-
-		hret = ehea_h_query_ehea_port(port->adapter->handle,
-					      port->logical_port_id,
-					      H_PORT_CB4, H_PORT_CB4_SPEED,
-					      cb4);
-		if (hret == H_SUCCESS) {
-			switch (cb4->port_speed) {
-			case H_SPEED_10M_H:
-				port->port_speed = EHEA_SPEED_10M;
-				port->full_duplex = 0;
-				break;
-			case H_SPEED_10M_F:
-				port->port_speed = EHEA_SPEED_10M;
-				port->full_duplex = 1;
-				break;
-			case H_SPEED_100M_H:
-				port->port_speed = EHEA_SPEED_100M;
-				port->full_duplex = 0;
-				break;
-			case H_SPEED_100M_F:
-				port->port_speed = EHEA_SPEED_100M;
-				port->full_duplex = 1;
-				break;
-			case H_SPEED_1G_F:
-				port->port_speed = EHEA_SPEED_1G;
-				port->full_duplex = 1;
-				break;
-			case H_SPEED_10G_F:
-				port->port_speed = EHEA_SPEED_10G;
-				port->full_duplex = 1;
-				break;
-			default:
-				port->port_speed = 0;
-				port->full_duplex = 0;
-				break;
-			}
-		} else {
-			pr_err("Failed sensing port speed\n");
-			ret = -EIO;
-		}
-	} else {
-		if (hret == H_AUTHORITY) {
-			pr_info("Hypervisor denied setting port speed\n");
-			ret = -EPERM;
-		} else {
-			ret = -EIO;
-			pr_err("Failed setting port speed\n");
-		}
-	}
-	if (!prop_carrier_state || (port->phy_link == EHEA_PHY_LINK_UP))
-		netif_carrier_on(port->netdev);
-
-	free_page((unsigned long)cb4);
-out:
-	return ret;
-}
-
-static void ehea_parse_eqe(struct ehea_adapter *adapter, u64 eqe)
-{
-	int ret;
-	u8 ec;
-	u8 portnum;
-	struct ehea_port *port;
-	struct net_device *dev;
-
-	ec = EHEA_BMASK_GET(NEQE_EVENT_CODE, eqe);
-	portnum = EHEA_BMASK_GET(NEQE_PORTNUM, eqe);
-	port = ehea_get_port(adapter, portnum);
-	if (!port) {
-		netdev_err(NULL, "unknown portnum %x\n", portnum);
-		return;
-	}
-	dev = port->netdev;
-
-	switch (ec) {
-	case EHEA_EC_PORTSTATE_CHG:	/* port state change */
-
-		if (EHEA_BMASK_GET(NEQE_PORT_UP, eqe)) {
-			if (!netif_carrier_ok(dev)) {
-				ret = ehea_sense_port_attr(port);
-				if (ret) {
-					netdev_err(dev, "failed resensing port attributes\n");
-					break;
-				}
-
-				netif_info(port, link, dev,
-					   "Logical port up: %dMbps %s Duplex\n",
-					   port->port_speed,
-					   port->full_duplex == 1 ?
-					   "Full" : "Half");
-
-				netif_carrier_on(dev);
-				netif_wake_queue(dev);
-			}
-		} else
-			if (netif_carrier_ok(dev)) {
-				netif_info(port, link, dev,
-					   "Logical port down\n");
-				netif_carrier_off(dev);
-				netif_tx_disable(dev);
-			}
-
-		if (EHEA_BMASK_GET(NEQE_EXTSWITCH_PORT_UP, eqe)) {
-			port->phy_link = EHEA_PHY_LINK_UP;
-			netif_info(port, link, dev,
-				   "Physical port up\n");
-			if (prop_carrier_state)
-				netif_carrier_on(dev);
-		} else {
-			port->phy_link = EHEA_PHY_LINK_DOWN;
-			netif_info(port, link, dev,
-				   "Physical port down\n");
-			if (prop_carrier_state)
-				netif_carrier_off(dev);
-		}
-
-		if (EHEA_BMASK_GET(NEQE_EXTSWITCH_PRIMARY, eqe))
-			netdev_info(dev,
-				    "External switch port is primary port\n");
-		else
-			netdev_info(dev,
-				    "External switch port is backup port\n");
-
-		break;
-	case EHEA_EC_ADAPTER_MALFUNC:
-		netdev_err(dev, "Adapter malfunction\n");
-		break;
-	case EHEA_EC_PORT_MALFUNC:
-		netdev_info(dev, "Port malfunction\n");
-		netif_carrier_off(dev);
-		netif_tx_disable(dev);
-		break;
-	default:
-		netdev_err(dev, "unknown event code %x, eqe=0x%llX\n", ec, eqe);
-		break;
-	}
-}
-
-static void ehea_neq_tasklet(struct tasklet_struct *t)
-{
-	struct ehea_adapter *adapter = from_tasklet(adapter, t, neq_tasklet);
-	struct ehea_eqe *eqe;
-	u64 event_mask;
-
-	eqe = ehea_poll_eq(adapter->neq);
-	pr_debug("eqe=%p\n", eqe);
-
-	while (eqe) {
-		pr_debug("*eqe=%lx\n", (unsigned long) eqe->entry);
-		ehea_parse_eqe(adapter, eqe->entry);
-		eqe = ehea_poll_eq(adapter->neq);
-		pr_debug("next eqe=%p\n", eqe);
-	}
-
-	event_mask = EHEA_BMASK_SET(NELR_PORTSTATE_CHG, 1)
-		   | EHEA_BMASK_SET(NELR_ADAPTER_MALFUNC, 1)
-		   | EHEA_BMASK_SET(NELR_PORT_MALFUNC, 1);
-
-	ehea_h_reset_events(adapter->handle,
-			    adapter->neq->fw_handle, event_mask);
-}
-
-static irqreturn_t ehea_interrupt_neq(int irq, void *param)
-{
-	struct ehea_adapter *adapter = param;
-	tasklet_hi_schedule(&adapter->neq_tasklet);
-	return IRQ_HANDLED;
-}
-
-
-static int ehea_fill_port_res(struct ehea_port_res *pr)
-{
-	int ret;
-	struct ehea_qp_init_attr *init_attr = &pr->qp->init_attr;
-
-	ehea_init_fill_rq1(pr, pr->rq1_skba.len);
-
-	ret = ehea_refill_rq2(pr, init_attr->act_nr_rwqes_rq2 - 1);
-
-	ret |= ehea_refill_rq3(pr, init_attr->act_nr_rwqes_rq3 - 1);
-
-	return ret;
-}
-
-static int ehea_reg_interrupts(struct net_device *dev)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	struct ehea_port_res *pr;
-	int i, ret;
-
-
-	snprintf(port->int_aff_name, EHEA_IRQ_NAME_SIZE - 1, "%s-aff",
-		 dev->name);
-
-	ret = ibmebus_request_irq(port->qp_eq->attr.ist1,
-				  ehea_qp_aff_irq_handler,
-				  0, port->int_aff_name, port);
-	if (ret) {
-		netdev_err(dev, "failed registering irq for qp_aff_irq_handler:ist=%X\n",
-			   port->qp_eq->attr.ist1);
-		goto out_free_qpeq;
-	}
-
-	netif_info(port, ifup, dev,
-		   "irq_handle 0x%X for function qp_aff_irq_handler registered\n",
-		   port->qp_eq->attr.ist1);
-
-
-	for (i = 0; i < port->num_def_qps; i++) {
-		pr = &port->port_res[i];
-		snprintf(pr->int_send_name, EHEA_IRQ_NAME_SIZE - 1,
-			 "%s-queue%d", dev->name, i);
-		ret = ibmebus_request_irq(pr->eq->attr.ist1,
-					  ehea_recv_irq_handler,
-					  0, pr->int_send_name, pr);
-		if (ret) {
-			netdev_err(dev, "failed registering irq for ehea_queue port_res_nr:%d, ist=%X\n",
-				   i, pr->eq->attr.ist1);
-			goto out_free_req;
-		}
-		netif_info(port, ifup, dev,
-			   "irq_handle 0x%X for function ehea_queue_int %d registered\n",
-			   pr->eq->attr.ist1, i);
-	}
-out:
-	return ret;
-
-
-out_free_req:
-	while (--i >= 0) {
-		u32 ist = port->port_res[i].eq->attr.ist1;
-		ibmebus_free_irq(ist, &port->port_res[i]);
-	}
-
-out_free_qpeq:
-	ibmebus_free_irq(port->qp_eq->attr.ist1, port);
-	i = port->num_def_qps;
-
-	goto out;
-
-}
-
-static void ehea_free_interrupts(struct net_device *dev)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	struct ehea_port_res *pr;
-	int i;
-
-	/* send */
-
-	for (i = 0; i < port->num_def_qps; i++) {
-		pr = &port->port_res[i];
-		ibmebus_free_irq(pr->eq->attr.ist1, pr);
-		netif_info(port, intr, dev,
-			   "free send irq for res %d with handle 0x%X\n",
-			   i, pr->eq->attr.ist1);
-	}
-
-	/* associated events */
-	ibmebus_free_irq(port->qp_eq->attr.ist1, port);
-	netif_info(port, intr, dev,
-		   "associated event interrupt for handle 0x%X freed\n",
-		   port->qp_eq->attr.ist1);
-}
-
-static int ehea_configure_port(struct ehea_port *port)
-{
-	int ret, i;
-	u64 hret, mask;
-	struct hcp_ehea_port_cb0 *cb0;
-
-	ret = -ENOMEM;
-	cb0 = (void *)get_zeroed_page(GFP_KERNEL);
-	if (!cb0)
-		goto out;
-
-	cb0->port_rc = EHEA_BMASK_SET(PXLY_RC_VALID, 1)
-		     | EHEA_BMASK_SET(PXLY_RC_IP_CHKSUM, 1)
-		     | EHEA_BMASK_SET(PXLY_RC_TCP_UDP_CHKSUM, 1)
-		     | EHEA_BMASK_SET(PXLY_RC_VLAN_XTRACT, 1)
-		     | EHEA_BMASK_SET(PXLY_RC_VLAN_TAG_FILTER,
-				      PXLY_RC_VLAN_FILTER)
-		     | EHEA_BMASK_SET(PXLY_RC_JUMBO_FRAME, 1);
-
-	for (i = 0; i < port->num_mcs; i++)
-		if (use_mcs)
-			cb0->default_qpn_arr[i] =
-				port->port_res[i].qp->init_attr.qp_nr;
-		else
-			cb0->default_qpn_arr[i] =
-				port->port_res[0].qp->init_attr.qp_nr;
-
-	if (netif_msg_ifup(port))
-		ehea_dump(cb0, sizeof(*cb0), "ehea_configure_port");
-
-	mask = EHEA_BMASK_SET(H_PORT_CB0_PRC, 1)
-	     | EHEA_BMASK_SET(H_PORT_CB0_DEFQPNARRAY, 1);
-
-	hret = ehea_h_modify_ehea_port(port->adapter->handle,
-				       port->logical_port_id,
-				       H_PORT_CB0, mask, cb0);
-	ret = -EIO;
-	if (hret != H_SUCCESS)
-		goto out_free;
-
-	ret = 0;
-
-out_free:
-	free_page((unsigned long)cb0);
-out:
-	return ret;
-}
-
-static int ehea_gen_smrs(struct ehea_port_res *pr)
-{
-	int ret;
-	struct ehea_adapter *adapter = pr->port->adapter;
-
-	ret = ehea_gen_smr(adapter, &adapter->mr, &pr->send_mr);
-	if (ret)
-		goto out;
-
-	ret = ehea_gen_smr(adapter, &adapter->mr, &pr->recv_mr);
-	if (ret)
-		goto out_free;
-
-	return 0;
-
-out_free:
-	ehea_rem_mr(&pr->send_mr);
-out:
-	pr_err("Generating SMRS failed\n");
-	return -EIO;
-}
-
-static int ehea_rem_smrs(struct ehea_port_res *pr)
-{
-	if ((ehea_rem_mr(&pr->send_mr)) ||
-	    (ehea_rem_mr(&pr->recv_mr)))
-		return -EIO;
-	else
-		return 0;
-}
-
-static int ehea_init_q_skba(struct ehea_q_skb_arr *q_skba, int max_q_entries)
-{
-	int arr_size = sizeof(void *) * max_q_entries;
-
-	q_skba->arr = vzalloc(arr_size);
-	if (!q_skba->arr)
-		return -ENOMEM;
-
-	q_skba->len = max_q_entries;
-	q_skba->index = 0;
-	q_skba->os_skbs = 0;
-
-	return 0;
-}
-
-static int ehea_init_port_res(struct ehea_port *port, struct ehea_port_res *pr,
-			      struct port_res_cfg *pr_cfg, int queue_token)
-{
-	struct ehea_adapter *adapter = port->adapter;
-	enum ehea_eq_type eq_type = EHEA_EQ;
-	struct ehea_qp_init_attr *init_attr = NULL;
-	int ret = -EIO;
-	u64 tx_bytes, rx_bytes, tx_packets, rx_packets;
-
-	tx_bytes = pr->tx_bytes;
-	tx_packets = pr->tx_packets;
-	rx_bytes = pr->rx_bytes;
-	rx_packets = pr->rx_packets;
-
-	memset(pr, 0, sizeof(struct ehea_port_res));
-
-	pr->tx_bytes = tx_bytes;
-	pr->tx_packets = tx_packets;
-	pr->rx_bytes = rx_bytes;
-	pr->rx_packets = rx_packets;
-
-	pr->port = port;
-
-	pr->eq = ehea_create_eq(adapter, eq_type, EHEA_MAX_ENTRIES_EQ, 0);
-	if (!pr->eq) {
-		pr_err("create_eq failed (eq)\n");
-		goto out_free;
-	}
-
-	pr->recv_cq = ehea_create_cq(adapter, pr_cfg->max_entries_rcq,
-				     pr->eq->fw_handle,
-				     port->logical_port_id);
-	if (!pr->recv_cq) {
-		pr_err("create_cq failed (cq_recv)\n");
-		goto out_free;
-	}
-
-	pr->send_cq = ehea_create_cq(adapter, pr_cfg->max_entries_scq,
-				     pr->eq->fw_handle,
-				     port->logical_port_id);
-	if (!pr->send_cq) {
-		pr_err("create_cq failed (cq_send)\n");
-		goto out_free;
-	}
-
-	if (netif_msg_ifup(port))
-		pr_info("Send CQ: act_nr_cqes=%d, Recv CQ: act_nr_cqes=%d\n",
-			pr->send_cq->attr.act_nr_of_cqes,
-			pr->recv_cq->attr.act_nr_of_cqes);
-
-	init_attr = kzalloc_obj(*init_attr);
-	if (!init_attr) {
-		ret = -ENOMEM;
-		pr_err("no mem for ehea_qp_init_attr\n");
-		goto out_free;
-	}
-
-	init_attr->low_lat_rq1 = 1;
-	init_attr->signalingtype = 1;	/* generate CQE if specified in WQE */
-	init_attr->rq_count = 3;
-	init_attr->qp_token = queue_token;
-	init_attr->max_nr_send_wqes = pr_cfg->max_entries_sq;
-	init_attr->max_nr_rwqes_rq1 = pr_cfg->max_entries_rq1;
-	init_attr->max_nr_rwqes_rq2 = pr_cfg->max_entries_rq2;
-	init_attr->max_nr_rwqes_rq3 = pr_cfg->max_entries_rq3;
-	init_attr->wqe_size_enc_sq = EHEA_SG_SQ;
-	init_attr->wqe_size_enc_rq1 = EHEA_SG_RQ1;
-	init_attr->wqe_size_enc_rq2 = EHEA_SG_RQ2;
-	init_attr->wqe_size_enc_rq3 = EHEA_SG_RQ3;
-	init_attr->rq2_threshold = EHEA_RQ2_THRESHOLD;
-	init_attr->rq3_threshold = EHEA_RQ3_THRESHOLD;
-	init_attr->port_nr = port->logical_port_id;
-	init_attr->send_cq_handle = pr->send_cq->fw_handle;
-	init_attr->recv_cq_handle = pr->recv_cq->fw_handle;
-	init_attr->aff_eq_handle = port->qp_eq->fw_handle;
-
-	pr->qp = ehea_create_qp(adapter, adapter->pd, init_attr);
-	if (!pr->qp) {
-		pr_err("create_qp failed\n");
-		ret = -EIO;
-		goto out_free;
-	}
-
-	if (netif_msg_ifup(port))
-		pr_info("QP: qp_nr=%d\n act_nr_snd_wqe=%d\n nr_rwqe_rq1=%d\n nr_rwqe_rq2=%d\n nr_rwqe_rq3=%d\n",
-			init_attr->qp_nr,
-			init_attr->act_nr_send_wqes,
-			init_attr->act_nr_rwqes_rq1,
-			init_attr->act_nr_rwqes_rq2,
-			init_attr->act_nr_rwqes_rq3);
-
-	pr->sq_skba_size = init_attr->act_nr_send_wqes + 1;
-
-	ret = ehea_init_q_skba(&pr->sq_skba, pr->sq_skba_size);
-	ret |= ehea_init_q_skba(&pr->rq1_skba, init_attr->act_nr_rwqes_rq1 + 1);
-	ret |= ehea_init_q_skba(&pr->rq2_skba, init_attr->act_nr_rwqes_rq2 + 1);
-	ret |= ehea_init_q_skba(&pr->rq3_skba, init_attr->act_nr_rwqes_rq3 + 1);
-	if (ret)
-		goto out_free;
-
-	pr->swqe_refill_th = init_attr->act_nr_send_wqes / 10;
-	if (ehea_gen_smrs(pr) != 0) {
-		ret = -EIO;
-		goto out_free;
-	}
-
-	atomic_set(&pr->swqe_avail, init_attr->act_nr_send_wqes - 1);
-
-	kfree(init_attr);
-
-	netif_napi_add(pr->port->netdev, &pr->napi, ehea_poll);
-
-	ret = 0;
-	goto out;
-
-out_free:
-	kfree(init_attr);
-	vfree(pr->sq_skba.arr);
-	vfree(pr->rq1_skba.arr);
-	vfree(pr->rq2_skba.arr);
-	vfree(pr->rq3_skba.arr);
-	ehea_destroy_qp(pr->qp);
-	ehea_destroy_cq(pr->send_cq);
-	ehea_destroy_cq(pr->recv_cq);
-	ehea_destroy_eq(pr->eq);
-out:
-	return ret;
-}
-
-static int ehea_clean_portres(struct ehea_port *port, struct ehea_port_res *pr)
-{
-	int ret, i;
-
-	if (pr->qp)
-		netif_napi_del(&pr->napi);
-
-	ret = ehea_destroy_qp(pr->qp);
-
-	if (!ret) {
-		ehea_destroy_cq(pr->send_cq);
-		ehea_destroy_cq(pr->recv_cq);
-		ehea_destroy_eq(pr->eq);
-
-		for (i = 0; i < pr->rq1_skba.len; i++)
-			dev_kfree_skb(pr->rq1_skba.arr[i]);
-
-		for (i = 0; i < pr->rq2_skba.len; i++)
-			dev_kfree_skb(pr->rq2_skba.arr[i]);
-
-		for (i = 0; i < pr->rq3_skba.len; i++)
-			dev_kfree_skb(pr->rq3_skba.arr[i]);
-
-		for (i = 0; i < pr->sq_skba.len; i++)
-			dev_kfree_skb(pr->sq_skba.arr[i]);
-
-		vfree(pr->rq1_skba.arr);
-		vfree(pr->rq2_skba.arr);
-		vfree(pr->rq3_skba.arr);
-		vfree(pr->sq_skba.arr);
-		ret = ehea_rem_smrs(pr);
-	}
-	return ret;
-}
-
-static void write_swqe2_immediate(struct sk_buff *skb, struct ehea_swqe *swqe,
-				  u32 lkey)
-{
-	int skb_data_size = skb_headlen(skb);
-	u8 *imm_data = &swqe->u.immdata_desc.immediate_data[0];
-	struct ehea_vsgentry *sg1entry = &swqe->u.immdata_desc.sg_entry;
-	unsigned int immediate_len = SWQE2_MAX_IMM;
-
-	swqe->descriptors = 0;
-
-	if (skb_is_gso(skb)) {
-		swqe->tx_control |= EHEA_SWQE_TSO;
-		swqe->mss = skb_shinfo(skb)->gso_size;
-		/*
-		 * For TSO packets we only copy the headers into the
-		 * immediate area.
-		 */
-		immediate_len = skb_tcp_all_headers(skb);
-	}
-
-	if (skb_is_gso(skb) || skb_data_size >= SWQE2_MAX_IMM) {
-		skb_copy_from_linear_data(skb, imm_data, immediate_len);
-		swqe->immediate_data_length = immediate_len;
-
-		if (skb_data_size > immediate_len) {
-			sg1entry->l_key = lkey;
-			sg1entry->len = skb_data_size - immediate_len;
-			sg1entry->vaddr =
-				ehea_map_vaddr(skb->data + immediate_len);
-			swqe->descriptors++;
-		}
-	} else {
-		skb_copy_from_linear_data(skb, imm_data, skb_data_size);
-		swqe->immediate_data_length = skb_data_size;
-	}
-}
-
-static inline void write_swqe2_data(struct sk_buff *skb, struct net_device *dev,
-				    struct ehea_swqe *swqe, u32 lkey)
-{
-	struct ehea_vsgentry *sg_list, *sg1entry, *sgentry;
-	skb_frag_t *frag;
-	int nfrags, sg1entry_contains_frag_data, i;
-
-	nfrags = skb_shinfo(skb)->nr_frags;
-	sg1entry = &swqe->u.immdata_desc.sg_entry;
-	sg_list = (struct ehea_vsgentry *)&swqe->u.immdata_desc.sg_list;
-	sg1entry_contains_frag_data = 0;
-
-	write_swqe2_immediate(skb, swqe, lkey);
-
-	/* write descriptors */
-	if (nfrags > 0) {
-		if (swqe->descriptors == 0) {
-			/* sg1entry not yet used */
-			frag = &skb_shinfo(skb)->frags[0];
-
-			/* copy sg1entry data */
-			sg1entry->l_key = lkey;
-			sg1entry->len = skb_frag_size(frag);
-			sg1entry->vaddr =
-				ehea_map_vaddr(skb_frag_address(frag));
-			swqe->descriptors++;
-			sg1entry_contains_frag_data = 1;
-		}
-
-		for (i = sg1entry_contains_frag_data; i < nfrags; i++) {
-
-			frag = &skb_shinfo(skb)->frags[i];
-			sgentry = &sg_list[i - sg1entry_contains_frag_data];
-
-			sgentry->l_key = lkey;
-			sgentry->len = skb_frag_size(frag);
-			sgentry->vaddr = ehea_map_vaddr(skb_frag_address(frag));
-			swqe->descriptors++;
-		}
-	}
-}
-
-static int ehea_broadcast_reg_helper(struct ehea_port *port, u32 hcallid)
-{
-	int ret = 0;
-	u64 hret;
-	u8 reg_type;
-
-	/* De/Register untagged packets */
-	reg_type = EHEA_BCMC_BROADCAST | EHEA_BCMC_UNTAGGED;
-	hret = ehea_h_reg_dereg_bcmc(port->adapter->handle,
-				     port->logical_port_id,
-				     reg_type, port->mac_addr, 0, hcallid);
-	if (hret != H_SUCCESS) {
-		pr_err("%sregistering bc address failed (tagged)\n",
-		       hcallid == H_REG_BCMC ? "" : "de");
-		ret = -EIO;
-		goto out_herr;
-	}
-
-	/* De/Register VLAN packets */
-	reg_type = EHEA_BCMC_BROADCAST | EHEA_BCMC_VLANID_ALL;
-	hret = ehea_h_reg_dereg_bcmc(port->adapter->handle,
-				     port->logical_port_id,
-				     reg_type, port->mac_addr, 0, hcallid);
-	if (hret != H_SUCCESS) {
-		pr_err("%sregistering bc address failed (vlan)\n",
-		       hcallid == H_REG_BCMC ? "" : "de");
-		ret = -EIO;
-	}
-out_herr:
-	return ret;
-}
-
-static int ehea_set_mac_addr(struct net_device *dev, void *sa)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	struct sockaddr *mac_addr = sa;
-	struct hcp_ehea_port_cb0 *cb0;
-	int ret;
-	u64 hret;
-
-	if (!is_valid_ether_addr(mac_addr->sa_data)) {
-		ret = -EADDRNOTAVAIL;
-		goto out;
-	}
-
-	cb0 = (void *)get_zeroed_page(GFP_KERNEL);
-	if (!cb0) {
-		pr_err("no mem for cb0\n");
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	memcpy(&(cb0->port_mac_addr), &(mac_addr->sa_data[0]), ETH_ALEN);
-
-	cb0->port_mac_addr = cb0->port_mac_addr >> 16;
-
-	hret = ehea_h_modify_ehea_port(port->adapter->handle,
-				       port->logical_port_id, H_PORT_CB0,
-				       EHEA_BMASK_SET(H_PORT_CB0_MAC, 1), cb0);
-	if (hret != H_SUCCESS) {
-		ret = -EIO;
-		goto out_free;
-	}
-
-	eth_hw_addr_set(dev, mac_addr->sa_data);
-
-	/* Deregister old MAC in pHYP */
-	if (port->state == EHEA_PORT_UP) {
-		ret = ehea_broadcast_reg_helper(port, H_DEREG_BCMC);
-		if (ret)
-			goto out_upregs;
-	}
-
-	port->mac_addr = cb0->port_mac_addr << 16;
-
-	/* Register new MAC in pHYP */
-	if (port->state == EHEA_PORT_UP) {
-		ret = ehea_broadcast_reg_helper(port, H_REG_BCMC);
-		if (ret)
-			goto out_upregs;
-	}
-
-	ret = 0;
-
-out_upregs:
-	ehea_update_bcmc_registrations();
-out_free:
-	free_page((unsigned long)cb0);
-out:
-	return ret;
-}
-
-static void ehea_promiscuous_error(u64 hret, int enable)
-{
-	if (hret == H_AUTHORITY)
-		pr_info("Hypervisor denied %sabling promiscuous mode\n",
-			enable == 1 ? "en" : "dis");
-	else
-		pr_err("failed %sabling promiscuous mode\n",
-		       enable == 1 ? "en" : "dis");
-}
-
-static void ehea_promiscuous(struct net_device *dev, int enable)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	struct hcp_ehea_port_cb7 *cb7;
-	u64 hret;
-
-	if (enable == port->promisc)
-		return;
-
-	cb7 = (void *)get_zeroed_page(GFP_ATOMIC);
-	if (!cb7) {
-		pr_err("no mem for cb7\n");
-		goto out;
-	}
-
-	/* Modify Pxs_DUCQPN in CB7 */
-	cb7->def_uc_qpn = enable == 1 ? port->port_res[0].qp->fw_handle : 0;
-
-	hret = ehea_h_modify_ehea_port(port->adapter->handle,
-				       port->logical_port_id,
-				       H_PORT_CB7, H_PORT_CB7_DUCQPN, cb7);
-	if (hret) {
-		ehea_promiscuous_error(hret, enable);
-		goto out;
-	}
-
-	port->promisc = enable;
-out:
-	free_page((unsigned long)cb7);
-}
-
-static u64 ehea_multicast_reg_helper(struct ehea_port *port, u64 mc_mac_addr,
-				     u32 hcallid)
-{
-	u64 hret;
-	u8 reg_type;
-
-	reg_type = EHEA_BCMC_MULTICAST | EHEA_BCMC_UNTAGGED;
-	if (mc_mac_addr == 0)
-		reg_type |= EHEA_BCMC_SCOPE_ALL;
-
-	hret = ehea_h_reg_dereg_bcmc(port->adapter->handle,
-				     port->logical_port_id,
-				     reg_type, mc_mac_addr, 0, hcallid);
-	if (hret)
-		goto out;
-
-	reg_type = EHEA_BCMC_MULTICAST | EHEA_BCMC_VLANID_ALL;
-	if (mc_mac_addr == 0)
-		reg_type |= EHEA_BCMC_SCOPE_ALL;
-
-	hret = ehea_h_reg_dereg_bcmc(port->adapter->handle,
-				     port->logical_port_id,
-				     reg_type, mc_mac_addr, 0, hcallid);
-out:
-	return hret;
-}
-
-static int ehea_drop_multicast_list(struct net_device *dev)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	struct ehea_mc_list *mc_entry = port->mc_list;
-	struct list_head *pos;
-	struct list_head *temp;
-	int ret = 0;
-	u64 hret;
-
-	list_for_each_safe(pos, temp, &(port->mc_list->list)) {
-		mc_entry = list_entry(pos, struct ehea_mc_list, list);
-
-		hret = ehea_multicast_reg_helper(port, mc_entry->macaddr,
-						 H_DEREG_BCMC);
-		if (hret) {
-			pr_err("failed deregistering mcast MAC\n");
-			ret = -EIO;
-		}
-
-		list_del(pos);
-		kfree(mc_entry);
-	}
-	return ret;
-}
-
-static void ehea_allmulti(struct net_device *dev, int enable)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	u64 hret;
-
-	if (!port->allmulti) {
-		if (enable) {
-			/* Enable ALLMULTI */
-			ehea_drop_multicast_list(dev);
-			hret = ehea_multicast_reg_helper(port, 0, H_REG_BCMC);
-			if (!hret)
-				port->allmulti = 1;
-			else
-				netdev_err(dev,
-					   "failed enabling IFF_ALLMULTI\n");
-		}
-	} else {
-		if (!enable) {
-			/* Disable ALLMULTI */
-			hret = ehea_multicast_reg_helper(port, 0, H_DEREG_BCMC);
-			if (!hret)
-				port->allmulti = 0;
-			else
-				netdev_err(dev,
-					   "failed disabling IFF_ALLMULTI\n");
-		}
-	}
-}
-
-static void ehea_add_multicast_entry(struct ehea_port *port, u8 *mc_mac_addr)
-{
-	struct ehea_mc_list *ehea_mcl_entry;
-	u64 hret;
-
-	ehea_mcl_entry = kzalloc_obj(*ehea_mcl_entry, GFP_ATOMIC);
-	if (!ehea_mcl_entry)
-		return;
-
-	INIT_LIST_HEAD(&ehea_mcl_entry->list);
-
-	memcpy(&ehea_mcl_entry->macaddr, mc_mac_addr, ETH_ALEN);
-
-	hret = ehea_multicast_reg_helper(port, ehea_mcl_entry->macaddr,
-					 H_REG_BCMC);
-	if (!hret)
-		list_add(&ehea_mcl_entry->list, &port->mc_list->list);
-	else {
-		pr_err("failed registering mcast MAC\n");
-		kfree(ehea_mcl_entry);
-	}
-}
-
-static void ehea_set_multicast_list(struct net_device *dev)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	struct netdev_hw_addr *ha;
-	int ret;
-
-	ehea_promiscuous(dev, !!(dev->flags & IFF_PROMISC));
-
-	if (dev->flags & IFF_ALLMULTI) {
-		ehea_allmulti(dev, 1);
-		goto out;
-	}
-	ehea_allmulti(dev, 0);
-
-	if (!netdev_mc_empty(dev)) {
-		ret = ehea_drop_multicast_list(dev);
-		if (ret) {
-			/* Dropping the current multicast list failed.
-			 * Enabling ALL_MULTI is the best we can do.
-			 */
-			ehea_allmulti(dev, 1);
-		}
-
-		if (netdev_mc_count(dev) > port->adapter->max_mc_mac) {
-			pr_info("Mcast registration limit reached (0x%llx). Use ALLMULTI!\n",
-				port->adapter->max_mc_mac);
-			goto out;
-		}
-
-		netdev_for_each_mc_addr(ha, dev)
-			ehea_add_multicast_entry(port, ha->addr);
-
-	}
-out:
-	ehea_update_bcmc_registrations();
-}
-
-static void xmit_common(struct sk_buff *skb, struct ehea_swqe *swqe)
-{
-	swqe->tx_control |= EHEA_SWQE_IMM_DATA_PRESENT | EHEA_SWQE_CRC;
-
-	if (vlan_get_protocol(skb) != htons(ETH_P_IP))
-		return;
-
-	if (skb->ip_summed == CHECKSUM_PARTIAL)
-		swqe->tx_control |= EHEA_SWQE_IP_CHECKSUM;
-
-	swqe->ip_start = skb_network_offset(skb);
-	swqe->ip_end = swqe->ip_start + ip_hdrlen(skb) - 1;
-
-	switch (ip_hdr(skb)->protocol) {
-	case IPPROTO_UDP:
-		if (skb->ip_summed == CHECKSUM_PARTIAL)
-			swqe->tx_control |= EHEA_SWQE_TCP_CHECKSUM;
-
-		swqe->tcp_offset = swqe->ip_end + 1 +
-				   offsetof(struct udphdr, check);
-		break;
-
-	case IPPROTO_TCP:
-		if (skb->ip_summed == CHECKSUM_PARTIAL)
-			swqe->tx_control |= EHEA_SWQE_TCP_CHECKSUM;
-
-		swqe->tcp_offset = swqe->ip_end + 1 +
-				   offsetof(struct tcphdr, check);
-		break;
-	}
-}
-
-static void ehea_xmit2(struct sk_buff *skb, struct net_device *dev,
-		       struct ehea_swqe *swqe, u32 lkey)
-{
-	swqe->tx_control |= EHEA_SWQE_DESCRIPTORS_PRESENT;
-
-	xmit_common(skb, swqe);
-
-	write_swqe2_data(skb, dev, swqe, lkey);
-}
-
-static void ehea_xmit3(struct sk_buff *skb, struct net_device *dev,
-		       struct ehea_swqe *swqe)
-{
-	u8 *imm_data = &swqe->u.immdata_nodesc.immediate_data[0];
-
-	xmit_common(skb, swqe);
-
-	if (!skb->data_len)
-		skb_copy_from_linear_data(skb, imm_data, skb->len);
-	else
-		skb_copy_bits(skb, 0, imm_data, skb->len);
-
-	swqe->immediate_data_length = skb->len;
-	dev_consume_skb_any(skb);
-}
-
-static netdev_tx_t ehea_start_xmit(struct sk_buff *skb, struct net_device *dev)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	struct ehea_swqe *swqe;
-	u32 lkey;
-	int swqe_index;
-	struct ehea_port_res *pr;
-	struct netdev_queue *txq;
-
-	pr = &port->port_res[skb_get_queue_mapping(skb)];
-	txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
-
-	swqe = ehea_get_swqe(pr->qp, &swqe_index);
-	memset(swqe, 0, SWQE_HEADER_SIZE);
-	atomic_dec(&pr->swqe_avail);
-
-	if (skb_vlan_tag_present(skb)) {
-		swqe->tx_control |= EHEA_SWQE_VLAN_INSERT;
-		swqe->vlan_tag = skb_vlan_tag_get(skb);
-	}
-
-	pr->tx_packets++;
-	pr->tx_bytes += skb->len;
-
-	if (skb->len <= SWQE3_MAX_IMM) {
-		u32 sig_iv = port->sig_comp_iv;
-		u32 swqe_num = pr->swqe_id_counter;
-		ehea_xmit3(skb, dev, swqe);
-		swqe->wr_id = EHEA_BMASK_SET(EHEA_WR_ID_TYPE, EHEA_SWQE3_TYPE)
-			| EHEA_BMASK_SET(EHEA_WR_ID_COUNT, swqe_num);
-		if (pr->swqe_ll_count >= (sig_iv - 1)) {
-			swqe->wr_id |= EHEA_BMASK_SET(EHEA_WR_ID_REFILL,
-						      sig_iv);
-			swqe->tx_control |= EHEA_SWQE_SIGNALLED_COMPLETION;
-			pr->swqe_ll_count = 0;
-		} else
-			pr->swqe_ll_count += 1;
-	} else {
-		swqe->wr_id =
-			EHEA_BMASK_SET(EHEA_WR_ID_TYPE, EHEA_SWQE2_TYPE)
-		      | EHEA_BMASK_SET(EHEA_WR_ID_COUNT, pr->swqe_id_counter)
-		      | EHEA_BMASK_SET(EHEA_WR_ID_REFILL, 1)
-		      | EHEA_BMASK_SET(EHEA_WR_ID_INDEX, pr->sq_skba.index);
-		pr->sq_skba.arr[pr->sq_skba.index] = skb;
-
-		pr->sq_skba.index++;
-		pr->sq_skba.index &= (pr->sq_skba.len - 1);
-
-		lkey = pr->send_mr.lkey;
-		ehea_xmit2(skb, dev, swqe, lkey);
-		swqe->tx_control |= EHEA_SWQE_SIGNALLED_COMPLETION;
-	}
-	pr->swqe_id_counter += 1;
-
-	netif_info(port, tx_queued, dev,
-		   "post swqe on QP %d\n", pr->qp->init_attr.qp_nr);
-	if (netif_msg_tx_queued(port))
-		ehea_dump(swqe, 512, "swqe");
-
-	if (unlikely(test_bit(__EHEA_STOP_XFER, &ehea_driver_flags))) {
-		netif_tx_stop_queue(txq);
-		swqe->tx_control |= EHEA_SWQE_PURGE;
-	}
-
-	ehea_post_swqe(pr->qp, swqe);
-
-	if (unlikely(atomic_read(&pr->swqe_avail) <= 1)) {
-		pr->p_stats.queue_stopped++;
-		netif_tx_stop_queue(txq);
-	}
-
-	return NETDEV_TX_OK;
-}
-
-static int ehea_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	struct ehea_adapter *adapter = port->adapter;
-	struct hcp_ehea_port_cb1 *cb1;
-	int index;
-	u64 hret;
-	int err = 0;
-
-	cb1 = (void *)get_zeroed_page(GFP_KERNEL);
-	if (!cb1) {
-		pr_err("no mem for cb1\n");
-		err = -ENOMEM;
-		goto out;
-	}
-
-	hret = ehea_h_query_ehea_port(adapter->handle, port->logical_port_id,
-				      H_PORT_CB1, H_PORT_CB1_ALL, cb1);
-	if (hret != H_SUCCESS) {
-		pr_err("query_ehea_port failed\n");
-		err = -EINVAL;
-		goto out;
-	}
-
-	index = (vid / 64);
-	cb1->vlan_filter[index] |= ((u64)(0x8000000000000000 >> (vid & 0x3F)));
-
-	hret = ehea_h_modify_ehea_port(adapter->handle, port->logical_port_id,
-				       H_PORT_CB1, H_PORT_CB1_ALL, cb1);
-	if (hret != H_SUCCESS) {
-		pr_err("modify_ehea_port failed\n");
-		err = -EINVAL;
-	}
-out:
-	free_page((unsigned long)cb1);
-	return err;
-}
-
-static int ehea_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	struct ehea_adapter *adapter = port->adapter;
-	struct hcp_ehea_port_cb1 *cb1;
-	int index;
-	u64 hret;
-	int err = 0;
-
-	cb1 = (void *)get_zeroed_page(GFP_KERNEL);
-	if (!cb1) {
-		pr_err("no mem for cb1\n");
-		err = -ENOMEM;
-		goto out;
-	}
-
-	hret = ehea_h_query_ehea_port(adapter->handle, port->logical_port_id,
-				      H_PORT_CB1, H_PORT_CB1_ALL, cb1);
-	if (hret != H_SUCCESS) {
-		pr_err("query_ehea_port failed\n");
-		err = -EINVAL;
-		goto out;
-	}
-
-	index = (vid / 64);
-	cb1->vlan_filter[index] &= ~((u64)(0x8000000000000000 >> (vid & 0x3F)));
-
-	hret = ehea_h_modify_ehea_port(adapter->handle, port->logical_port_id,
-				       H_PORT_CB1, H_PORT_CB1_ALL, cb1);
-	if (hret != H_SUCCESS) {
-		pr_err("modify_ehea_port failed\n");
-		err = -EINVAL;
-	}
-out:
-	free_page((unsigned long)cb1);
-	return err;
-}
-
-static int ehea_activate_qp(struct ehea_adapter *adapter, struct ehea_qp *qp)
-{
-	int ret = -EIO;
-	u64 hret;
-	u16 dummy16 = 0;
-	u64 dummy64 = 0;
-	struct hcp_modify_qp_cb0 *cb0;
-
-	cb0 = (void *)get_zeroed_page(GFP_KERNEL);
-	if (!cb0) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	hret = ehea_h_query_ehea_qp(adapter->handle, 0, qp->fw_handle,
-				    EHEA_BMASK_SET(H_QPCB0_ALL, 0xFFFF), cb0);
-	if (hret != H_SUCCESS) {
-		pr_err("query_ehea_qp failed (1)\n");
-		goto out;
-	}
-
-	cb0->qp_ctl_reg = H_QP_CR_STATE_INITIALIZED;
-	hret = ehea_h_modify_ehea_qp(adapter->handle, 0, qp->fw_handle,
-				     EHEA_BMASK_SET(H_QPCB0_QP_CTL_REG, 1), cb0,
-				     &dummy64, &dummy64, &dummy16, &dummy16);
-	if (hret != H_SUCCESS) {
-		pr_err("modify_ehea_qp failed (1)\n");
-		goto out;
-	}
-
-	hret = ehea_h_query_ehea_qp(adapter->handle, 0, qp->fw_handle,
-				    EHEA_BMASK_SET(H_QPCB0_ALL, 0xFFFF), cb0);
-	if (hret != H_SUCCESS) {
-		pr_err("query_ehea_qp failed (2)\n");
-		goto out;
-	}
-
-	cb0->qp_ctl_reg = H_QP_CR_ENABLED | H_QP_CR_STATE_INITIALIZED;
-	hret = ehea_h_modify_ehea_qp(adapter->handle, 0, qp->fw_handle,
-				     EHEA_BMASK_SET(H_QPCB0_QP_CTL_REG, 1), cb0,
-				     &dummy64, &dummy64, &dummy16, &dummy16);
-	if (hret != H_SUCCESS) {
-		pr_err("modify_ehea_qp failed (2)\n");
-		goto out;
-	}
-
-	hret = ehea_h_query_ehea_qp(adapter->handle, 0, qp->fw_handle,
-				    EHEA_BMASK_SET(H_QPCB0_ALL, 0xFFFF), cb0);
-	if (hret != H_SUCCESS) {
-		pr_err("query_ehea_qp failed (3)\n");
-		goto out;
-	}
-
-	cb0->qp_ctl_reg = H_QP_CR_ENABLED | H_QP_CR_STATE_RDY2SND;
-	hret = ehea_h_modify_ehea_qp(adapter->handle, 0, qp->fw_handle,
-				     EHEA_BMASK_SET(H_QPCB0_QP_CTL_REG, 1), cb0,
-				     &dummy64, &dummy64, &dummy16, &dummy16);
-	if (hret != H_SUCCESS) {
-		pr_err("modify_ehea_qp failed (3)\n");
-		goto out;
-	}
-
-	hret = ehea_h_query_ehea_qp(adapter->handle, 0, qp->fw_handle,
-				    EHEA_BMASK_SET(H_QPCB0_ALL, 0xFFFF), cb0);
-	if (hret != H_SUCCESS) {
-		pr_err("query_ehea_qp failed (4)\n");
-		goto out;
-	}
-
-	ret = 0;
-out:
-	free_page((unsigned long)cb0);
-	return ret;
-}
-
-static int ehea_port_res_setup(struct ehea_port *port, int def_qps)
-{
-	int ret, i;
-	struct port_res_cfg pr_cfg, pr_cfg_small_rx;
-	enum ehea_eq_type eq_type = EHEA_EQ;
-
-	port->qp_eq = ehea_create_eq(port->adapter, eq_type,
-				   EHEA_MAX_ENTRIES_EQ, 1);
-	if (!port->qp_eq) {
-		ret = -EINVAL;
-		pr_err("ehea_create_eq failed (qp_eq)\n");
-		goto out_kill_eq;
-	}
-
-	pr_cfg.max_entries_rcq = rq1_entries + rq2_entries + rq3_entries;
-	pr_cfg.max_entries_scq = sq_entries * 2;
-	pr_cfg.max_entries_sq = sq_entries;
-	pr_cfg.max_entries_rq1 = rq1_entries;
-	pr_cfg.max_entries_rq2 = rq2_entries;
-	pr_cfg.max_entries_rq3 = rq3_entries;
-
-	pr_cfg_small_rx.max_entries_rcq = 1;
-	pr_cfg_small_rx.max_entries_scq = sq_entries;
-	pr_cfg_small_rx.max_entries_sq = sq_entries;
-	pr_cfg_small_rx.max_entries_rq1 = 1;
-	pr_cfg_small_rx.max_entries_rq2 = 1;
-	pr_cfg_small_rx.max_entries_rq3 = 1;
-
-	for (i = 0; i < def_qps; i++) {
-		ret = ehea_init_port_res(port, &port->port_res[i], &pr_cfg, i);
-		if (ret)
-			goto out_clean_pr;
-	}
-	for (i = def_qps; i < def_qps; i++) {
-		ret = ehea_init_port_res(port, &port->port_res[i],
-					 &pr_cfg_small_rx, i);
-		if (ret)
-			goto out_clean_pr;
-	}
-
-	return 0;
-
-out_clean_pr:
-	while (--i >= 0)
-		ehea_clean_portres(port, &port->port_res[i]);
-
-out_kill_eq:
-	ehea_destroy_eq(port->qp_eq);
-	return ret;
-}
-
-static int ehea_clean_all_portres(struct ehea_port *port)
-{
-	int ret = 0;
-	int i;
-
-	for (i = 0; i < port->num_def_qps; i++)
-		ret |= ehea_clean_portres(port, &port->port_res[i]);
-
-	ret |= ehea_destroy_eq(port->qp_eq);
-
-	return ret;
-}
-
-static void ehea_remove_adapter_mr(struct ehea_adapter *adapter)
-{
-	if (adapter->active_ports)
-		return;
-
-	ehea_rem_mr(&adapter->mr);
-}
-
-static int ehea_add_adapter_mr(struct ehea_adapter *adapter)
-{
-	if (adapter->active_ports)
-		return 0;
-
-	return ehea_reg_kernel_mr(adapter, &adapter->mr);
-}
-
-static int ehea_up(struct net_device *dev)
-{
-	int ret, i;
-	struct ehea_port *port = netdev_priv(dev);
-
-	if (port->state == EHEA_PORT_UP)
-		return 0;
-
-	ret = ehea_port_res_setup(port, port->num_def_qps);
-	if (ret) {
-		netdev_err(dev, "port_res_failed\n");
-		goto out;
-	}
-
-	/* Set default QP for this port */
-	ret = ehea_configure_port(port);
-	if (ret) {
-		netdev_err(dev, "ehea_configure_port failed. ret:%d\n", ret);
-		goto out_clean_pr;
-	}
-
-	ret = ehea_reg_interrupts(dev);
-	if (ret) {
-		netdev_err(dev, "reg_interrupts failed. ret:%d\n", ret);
-		goto out_clean_pr;
-	}
-
-	for (i = 0; i < port->num_def_qps; i++) {
-		ret = ehea_activate_qp(port->adapter, port->port_res[i].qp);
-		if (ret) {
-			netdev_err(dev, "activate_qp failed\n");
-			goto out_free_irqs;
-		}
-	}
-
-	for (i = 0; i < port->num_def_qps; i++) {
-		ret = ehea_fill_port_res(&port->port_res[i]);
-		if (ret) {
-			netdev_err(dev, "out_free_irqs\n");
-			goto out_free_irqs;
-		}
-	}
-
-	ret = ehea_broadcast_reg_helper(port, H_REG_BCMC);
-	if (ret) {
-		ret = -EIO;
-		goto out_free_irqs;
-	}
-
-	port->state = EHEA_PORT_UP;
-
-	ret = 0;
-	goto out;
-
-out_free_irqs:
-	ehea_free_interrupts(dev);
-
-out_clean_pr:
-	ehea_clean_all_portres(port);
-out:
-	if (ret)
-		netdev_info(dev, "Failed starting. ret=%i\n", ret);
-
-	ehea_update_bcmc_registrations();
-	ehea_update_firmware_handles();
-
-	return ret;
-}
-
-static void port_napi_disable(struct ehea_port *port)
-{
-	int i;
-
-	for (i = 0; i < port->num_def_qps; i++)
-		napi_disable(&port->port_res[i].napi);
-}
-
-static void port_napi_enable(struct ehea_port *port)
-{
-	int i;
-
-	for (i = 0; i < port->num_def_qps; i++)
-		napi_enable(&port->port_res[i].napi);
-}
-
-static int ehea_open(struct net_device *dev)
-{
-	int ret;
-	struct ehea_port *port = netdev_priv(dev);
-
-	mutex_lock(&port->port_lock);
-
-	netif_info(port, ifup, dev, "enabling port\n");
-
-	netif_carrier_off(dev);
-
-	ret = ehea_up(dev);
-	if (!ret) {
-		port_napi_enable(port);
-		netif_tx_start_all_queues(dev);
-	}
-
-	mutex_unlock(&port->port_lock);
-	schedule_delayed_work(&port->stats_work,
-			      round_jiffies_relative(msecs_to_jiffies(1000)));
-
-	return ret;
-}
-
-static int ehea_down(struct net_device *dev)
-{
-	int ret;
-	struct ehea_port *port = netdev_priv(dev);
-
-	if (port->state == EHEA_PORT_DOWN)
-		return 0;
-
-	ehea_drop_multicast_list(dev);
-	ehea_allmulti(dev, 0);
-	ehea_broadcast_reg_helper(port, H_DEREG_BCMC);
-
-	ehea_free_interrupts(dev);
-
-	port->state = EHEA_PORT_DOWN;
-
-	ehea_update_bcmc_registrations();
-
-	ret = ehea_clean_all_portres(port);
-	if (ret)
-		netdev_info(dev, "Failed freeing resources. ret=%i\n", ret);
-
-	ehea_update_firmware_handles();
-
-	return ret;
-}
-
-static int ehea_stop(struct net_device *dev)
-{
-	int ret;
-	struct ehea_port *port = netdev_priv(dev);
-
-	netif_info(port, ifdown, dev, "disabling port\n");
-
-	set_bit(__EHEA_DISABLE_PORT_RESET, &port->flags);
-	cancel_work_sync(&port->reset_task);
-	cancel_delayed_work_sync(&port->stats_work);
-	mutex_lock(&port->port_lock);
-	netif_tx_stop_all_queues(dev);
-	port_napi_disable(port);
-	ret = ehea_down(dev);
-	mutex_unlock(&port->port_lock);
-	clear_bit(__EHEA_DISABLE_PORT_RESET, &port->flags);
-	return ret;
-}
-
-static void ehea_purge_sq(struct ehea_qp *orig_qp)
-{
-	struct ehea_qp qp = *orig_qp;
-	struct ehea_qp_init_attr *init_attr = &qp.init_attr;
-	struct ehea_swqe *swqe;
-	int wqe_index;
-	int i;
-
-	for (i = 0; i < init_attr->act_nr_send_wqes; i++) {
-		swqe = ehea_get_swqe(&qp, &wqe_index);
-		swqe->tx_control |= EHEA_SWQE_PURGE;
-	}
-}
-
-static void ehea_flush_sq(struct ehea_port *port)
-{
-	int i;
-
-	for (i = 0; i < port->num_def_qps; i++) {
-		struct ehea_port_res *pr = &port->port_res[i];
-		int swqe_max = pr->sq_skba_size - 2 - pr->swqe_ll_count;
-		int ret;
-
-		ret = wait_event_timeout(port->swqe_avail_wq,
-			 atomic_read(&pr->swqe_avail) >= swqe_max,
-			 msecs_to_jiffies(100));
-
-		if (!ret) {
-			pr_err("WARNING: sq not flushed completely\n");
-			break;
-		}
-	}
-}
-
-static int ehea_stop_qps(struct net_device *dev)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	struct ehea_adapter *adapter = port->adapter;
-	struct hcp_modify_qp_cb0 *cb0;
-	int ret = -EIO;
-	int dret;
-	int i;
-	u64 hret;
-	u64 dummy64 = 0;
-	u16 dummy16 = 0;
-
-	cb0 = (void *)get_zeroed_page(GFP_KERNEL);
-	if (!cb0) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	for (i = 0; i < (port->num_def_qps); i++) {
-		struct ehea_port_res *pr =  &port->port_res[i];
-		struct ehea_qp *qp = pr->qp;
-
-		/* Purge send queue */
-		ehea_purge_sq(qp);
-
-		/* Disable queue pair */
-		hret = ehea_h_query_ehea_qp(adapter->handle, 0, qp->fw_handle,
-					    EHEA_BMASK_SET(H_QPCB0_ALL, 0xFFFF),
-					    cb0);
-		if (hret != H_SUCCESS) {
-			pr_err("query_ehea_qp failed (1)\n");
-			goto out;
-		}
-
-		cb0->qp_ctl_reg = (cb0->qp_ctl_reg & H_QP_CR_RES_STATE) << 8;
-		cb0->qp_ctl_reg &= ~H_QP_CR_ENABLED;
-
-		hret = ehea_h_modify_ehea_qp(adapter->handle, 0, qp->fw_handle,
-					     EHEA_BMASK_SET(H_QPCB0_QP_CTL_REG,
-							    1), cb0, &dummy64,
-					     &dummy64, &dummy16, &dummy16);
-		if (hret != H_SUCCESS) {
-			pr_err("modify_ehea_qp failed (1)\n");
-			goto out;
-		}
-
-		hret = ehea_h_query_ehea_qp(adapter->handle, 0, qp->fw_handle,
-					    EHEA_BMASK_SET(H_QPCB0_ALL, 0xFFFF),
-					    cb0);
-		if (hret != H_SUCCESS) {
-			pr_err("query_ehea_qp failed (2)\n");
-			goto out;
-		}
-
-		/* deregister shared memory regions */
-		dret = ehea_rem_smrs(pr);
-		if (dret) {
-			pr_err("unreg shared memory region failed\n");
-			goto out;
-		}
-	}
-
-	ret = 0;
-out:
-	free_page((unsigned long)cb0);
-
-	return ret;
-}
-
-static void ehea_update_rqs(struct ehea_qp *orig_qp, struct ehea_port_res *pr)
-{
-	struct ehea_qp qp = *orig_qp;
-	struct ehea_qp_init_attr *init_attr = &qp.init_attr;
-	struct ehea_rwqe *rwqe;
-	struct sk_buff **skba_rq2 = pr->rq2_skba.arr;
-	struct sk_buff **skba_rq3 = pr->rq3_skba.arr;
-	struct sk_buff *skb;
-	u32 lkey = pr->recv_mr.lkey;
-
-
-	int i;
-	int index;
-
-	for (i = 0; i < init_attr->act_nr_rwqes_rq2 + 1; i++) {
-		rwqe = ehea_get_next_rwqe(&qp, 2);
-		rwqe->sg_list[0].l_key = lkey;
-		index = EHEA_BMASK_GET(EHEA_WR_ID_INDEX, rwqe->wr_id);
-		skb = skba_rq2[index];
-		if (skb)
-			rwqe->sg_list[0].vaddr = ehea_map_vaddr(skb->data);
-	}
-
-	for (i = 0; i < init_attr->act_nr_rwqes_rq3 + 1; i++) {
-		rwqe = ehea_get_next_rwqe(&qp, 3);
-		rwqe->sg_list[0].l_key = lkey;
-		index = EHEA_BMASK_GET(EHEA_WR_ID_INDEX, rwqe->wr_id);
-		skb = skba_rq3[index];
-		if (skb)
-			rwqe->sg_list[0].vaddr = ehea_map_vaddr(skb->data);
-	}
-}
-
-static int ehea_restart_qps(struct net_device *dev)
-{
-	struct ehea_port *port = netdev_priv(dev);
-	struct ehea_adapter *adapter = port->adapter;
-	int ret = 0;
-	int i;
-
-	struct hcp_modify_qp_cb0 *cb0;
-	u64 hret;
-	u64 dummy64 = 0;
-	u16 dummy16 = 0;
-
-	cb0 = (void *)get_zeroed_page(GFP_KERNEL);
-	if (!cb0)
-		return -ENOMEM;
-
-	for (i = 0; i < (port->num_def_qps); i++) {
-		struct ehea_port_res *pr =  &port->port_res[i];
-		struct ehea_qp *qp = pr->qp;
-
-		ret = ehea_gen_smrs(pr);
-		if (ret) {
-			netdev_err(dev, "creation of shared memory regions failed\n");
-			goto out;
-		}
-
-		ehea_update_rqs(qp, pr);
-
-		/* Enable queue pair */
-		hret = ehea_h_query_ehea_qp(adapter->handle, 0, qp->fw_handle,
-					    EHEA_BMASK_SET(H_QPCB0_ALL, 0xFFFF),
-					    cb0);
-		if (hret != H_SUCCESS) {
-			netdev_err(dev, "query_ehea_qp failed (1)\n");
-			ret = -EFAULT;
-			goto out;
-		}
-
-		cb0->qp_ctl_reg = (cb0->qp_ctl_reg & H_QP_CR_RES_STATE) << 8;
-		cb0->qp_ctl_reg |= H_QP_CR_ENABLED;
-
-		hret = ehea_h_modify_ehea_qp(adapter->handle, 0, qp->fw_handle,
-					     EHEA_BMASK_SET(H_QPCB0_QP_CTL_REG,
-							    1), cb0, &dummy64,
-					     &dummy64, &dummy16, &dummy16);
-		if (hret != H_SUCCESS) {
-			netdev_err(dev, "modify_ehea_qp failed (1)\n");
-			ret = -EFAULT;
-			goto out;
-		}
-
-		hret = ehea_h_query_ehea_qp(adapter->handle, 0, qp->fw_handle,
-					    EHEA_BMASK_SET(H_QPCB0_ALL, 0xFFFF),
-					    cb0);
-		if (hret != H_SUCCESS) {
-			netdev_err(dev, "query_ehea_qp failed (2)\n");
-			ret = -EFAULT;
-			goto out;
-		}
-
-		/* refill entire queue */
-		ehea_refill_rq1(pr, pr->rq1_skba.index, 0);
-		ehea_refill_rq2(pr, 0);
-		ehea_refill_rq3(pr, 0);
-	}
-out:
-	free_page((unsigned long)cb0);
-
-	return ret;
-}
-
-static void ehea_reset_port(struct work_struct *work)
-{
-	int ret;
-	struct ehea_port *port =
-		container_of(work, struct ehea_port, reset_task);
-	struct net_device *dev = port->netdev;
-
-	mutex_lock(&dlpar_mem_lock);
-	port->resets++;
-	mutex_lock(&port->port_lock);
-	netif_tx_disable(dev);
-
-	port_napi_disable(port);
-
-	ehea_down(dev);
-
-	ret = ehea_up(dev);
-	if (ret)
-		goto out;
-
-	ehea_set_multicast_list(dev);
-
-	netif_info(port, timer, dev, "reset successful\n");
-
-	port_napi_enable(port);
-
-	netif_tx_wake_all_queues(dev);
-out:
-	mutex_unlock(&port->port_lock);
-	mutex_unlock(&dlpar_mem_lock);
-}
-
-static void ehea_rereg_mrs(void)
-{
-	int ret, i;
-	struct ehea_adapter *adapter;
-
-	pr_info("LPAR memory changed - re-initializing driver\n");
-
-	list_for_each_entry(adapter, &adapter_list, list)
-		if (adapter->active_ports) {
-			/* Shutdown all ports */
-			for (i = 0; i < EHEA_MAX_PORTS; i++) {
-				struct ehea_port *port = adapter->port[i];
-				struct net_device *dev;
-
-				if (!port)
-					continue;
-
-				dev = port->netdev;
-
-				if (dev->flags & IFF_UP) {
-					mutex_lock(&port->port_lock);
-					netif_tx_disable(dev);
-					ehea_flush_sq(port);
-					ret = ehea_stop_qps(dev);
-					if (ret) {
-						mutex_unlock(&port->port_lock);
-						goto out;
-					}
-					port_napi_disable(port);
-					mutex_unlock(&port->port_lock);
-				}
-				reset_sq_restart_flag(port);
-			}
-
-			/* Unregister old memory region */
-			ret = ehea_rem_mr(&adapter->mr);
-			if (ret) {
-				pr_err("unregister MR failed - driver inoperable!\n");
-				goto out;
-			}
-		}
-
-	clear_bit(__EHEA_STOP_XFER, &ehea_driver_flags);
-
-	list_for_each_entry(adapter, &adapter_list, list)
-		if (adapter->active_ports) {
-			/* Register new memory region */
-			ret = ehea_reg_kernel_mr(adapter, &adapter->mr);
-			if (ret) {
-				pr_err("register MR failed - driver inoperable!\n");
-				goto out;
-			}
-
-			/* Restart all ports */
-			for (i = 0; i < EHEA_MAX_PORTS; i++) {
-				struct ehea_port *port = adapter->port[i];
-
-				if (port) {
-					struct net_device *dev = port->netdev;
-
-					if (dev->flags & IFF_UP) {
-						mutex_lock(&port->port_lock);
-						ret = ehea_restart_qps(dev);
-						if (!ret) {
-							check_sqs(port);
-							port_napi_enable(port);
-							netif_tx_wake_all_queues(dev);
-						} else {
-							netdev_err(dev, "Unable to restart QPS\n");
-						}
-						mutex_unlock(&port->port_lock);
-					}
-				}
-			}
-		}
-	pr_info("re-initializing driver complete\n");
-out:
-	return;
-}
-
-static void ehea_tx_watchdog(struct net_device *dev, unsigned int txqueue)
-{
-	struct ehea_port *port = netdev_priv(dev);
-
-	if (netif_carrier_ok(dev) &&
-	    !test_bit(__EHEA_STOP_XFER, &ehea_driver_flags))
-		ehea_schedule_port_reset(port);
-}
-
-static int ehea_sense_adapter_attr(struct ehea_adapter *adapter)
-{
-	struct hcp_query_ehea *cb;
-	u64 hret;
-	int ret;
-
-	cb = (void *)get_zeroed_page(GFP_KERNEL);
-	if (!cb) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	hret = ehea_h_query_ehea(adapter->handle, cb);
-
-	if (hret != H_SUCCESS) {
-		ret = -EIO;
-		goto out_herr;
-	}
-
-	adapter->max_mc_mac = cb->max_mc_mac - 1;
-	ret = 0;
-
-out_herr:
-	free_page((unsigned long)cb);
-out:
-	return ret;
-}
-
-static int ehea_get_jumboframe_status(struct ehea_port *port, int *jumbo)
-{
-	struct hcp_ehea_port_cb4 *cb4;
-	u64 hret;
-	int ret = 0;
-
-	*jumbo = 0;
-
-	/* (Try to) enable *jumbo frames */
-	cb4 = (void *)get_zeroed_page(GFP_KERNEL);
-	if (!cb4) {
-		pr_err("no mem for cb4\n");
-		ret = -ENOMEM;
-		goto out;
-	} else {
-		hret = ehea_h_query_ehea_port(port->adapter->handle,
-					      port->logical_port_id,
-					      H_PORT_CB4,
-					      H_PORT_CB4_JUMBO, cb4);
-		if (hret == H_SUCCESS) {
-			if (cb4->jumbo_frame)
-				*jumbo = 1;
-			else {
-				cb4->jumbo_frame = 1;
-				hret = ehea_h_modify_ehea_port(port->adapter->
-							       handle,
-							       port->
-							       logical_port_id,
-							       H_PORT_CB4,
-							       H_PORT_CB4_JUMBO,
-							       cb4);
-				if (hret == H_SUCCESS)
-					*jumbo = 1;
-			}
-		} else
-			ret = -EINVAL;
-
-		free_page((unsigned long)cb4);
-	}
-out:
-	return ret;
-}
-
-static ssize_t log_port_id_show(struct device *dev,
-				struct device_attribute *attr, char *buf)
-{
-	struct ehea_port *port = container_of(dev, struct ehea_port, ofdev.dev);
-	return sprintf(buf, "%d", port->logical_port_id);
-}
-
-static DEVICE_ATTR_RO(log_port_id);
-
-static void logical_port_release(struct device *dev)
-{
-	struct ehea_port *port = container_of(dev, struct ehea_port, ofdev.dev);
-	of_node_put(port->ofdev.dev.of_node);
-}
-
-static struct device *ehea_register_port(struct ehea_port *port,
-					 struct device_node *dn)
-{
-	int ret;
-
-	port->ofdev.dev.of_node = of_node_get(dn);
-	port->ofdev.dev.parent = &port->adapter->ofdev->dev;
-	port->ofdev.dev.bus = &ibmebus_bus_type;
-
-	dev_set_name(&port->ofdev.dev, "port%d", port_name_cnt++);
-	port->ofdev.dev.release = logical_port_release;
-
-	ret = of_device_register(&port->ofdev);
-	if (ret) {
-		pr_err("failed to register device. ret=%d\n", ret);
-		put_device(&port->ofdev.dev);
-		goto out;
-	}
-
-	ret = device_create_file(&port->ofdev.dev, &dev_attr_log_port_id);
-	if (ret) {
-		pr_err("failed to register attributes, ret=%d\n", ret);
-		goto out_unreg_of_dev;
-	}
-
-	return &port->ofdev.dev;
-
-out_unreg_of_dev:
-	of_device_unregister(&port->ofdev);
-out:
-	return NULL;
-}
-
-static void ehea_unregister_port(struct ehea_port *port)
-{
-	device_remove_file(&port->ofdev.dev, &dev_attr_log_port_id);
-	of_device_unregister(&port->ofdev);
-}
-
-static const struct net_device_ops ehea_netdev_ops = {
-	.ndo_open		= ehea_open,
-	.ndo_stop		= ehea_stop,
-	.ndo_start_xmit		= ehea_start_xmit,
-	.ndo_get_stats64	= ehea_get_stats64,
-	.ndo_set_mac_address	= ehea_set_mac_addr,
-	.ndo_validate_addr	= eth_validate_addr,
-	.ndo_set_rx_mode	= ehea_set_multicast_list,
-	.ndo_vlan_rx_add_vid	= ehea_vlan_rx_add_vid,
-	.ndo_vlan_rx_kill_vid	= ehea_vlan_rx_kill_vid,
-	.ndo_tx_timeout		= ehea_tx_watchdog,
-};
-
-static struct ehea_port *ehea_setup_single_port(struct ehea_adapter *adapter,
-					 u32 logical_port_id,
-					 struct device_node *dn)
-{
-	int ret;
-	struct net_device *dev;
-	struct ehea_port *port;
-	struct device *port_dev;
-	int jumbo;
-
-	/* allocate memory for the port structures */
-	dev = alloc_etherdev_mq(sizeof(struct ehea_port), EHEA_MAX_PORT_RES);
-
-	if (!dev) {
-		ret = -ENOMEM;
-		goto out_err;
-	}
-
-	port = netdev_priv(dev);
-
-	mutex_init(&port->port_lock);
-	port->state = EHEA_PORT_DOWN;
-	port->sig_comp_iv = sq_entries / 10;
-
-	port->adapter = adapter;
-	port->netdev = dev;
-	port->logical_port_id = logical_port_id;
-
-	port->msg_enable = netif_msg_init(msg_level, EHEA_MSG_DEFAULT);
-
-	port->mc_list = kzalloc_obj(struct ehea_mc_list);
-	if (!port->mc_list) {
-		ret = -ENOMEM;
-		goto out_free_ethdev;
-	}
-
-	INIT_LIST_HEAD(&port->mc_list->list);
-
-	ret = ehea_sense_port_attr(port);
-	if (ret)
-		goto out_free_mc_list;
-
-	netif_set_real_num_rx_queues(dev, port->num_def_qps);
-	netif_set_real_num_tx_queues(dev, port->num_def_qps);
-
-	port_dev = ehea_register_port(port, dn);
-	if (!port_dev)
-		goto out_free_mc_list;
-
-	SET_NETDEV_DEV(dev, port_dev);
-
-	/* initialize net_device structure */
-	eth_hw_addr_set(dev, (u8 *)&port->mac_addr);
-
-	dev->netdev_ops = &ehea_netdev_ops;
-	ehea_set_ethtool_ops(dev);
-
-	dev->hw_features = NETIF_F_SG | NETIF_F_TSO |
-		      NETIF_F_IP_CSUM | NETIF_F_HW_VLAN_CTAG_TX;
-	dev->features = NETIF_F_SG | NETIF_F_TSO |
-		      NETIF_F_HIGHDMA | NETIF_F_IP_CSUM |
-		      NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
-		      NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_RXCSUM;
-	dev->vlan_features = NETIF_F_SG | NETIF_F_TSO | NETIF_F_HIGHDMA |
-			NETIF_F_IP_CSUM;
-	dev->watchdog_timeo = EHEA_WATCH_DOG_TIMEOUT;
-
-	/* MTU range: 68 - 9022 */
-	dev->min_mtu = ETH_MIN_MTU;
-	dev->max_mtu = EHEA_MAX_PACKET_SIZE;
-
-	INIT_WORK(&port->reset_task, ehea_reset_port);
-	INIT_DELAYED_WORK(&port->stats_work, ehea_update_stats);
-
-	init_waitqueue_head(&port->swqe_avail_wq);
-	init_waitqueue_head(&port->restart_wq);
-
-	ret = register_netdev(dev);
-	if (ret) {
-		pr_err("register_netdev failed. ret=%d\n", ret);
-		goto out_unreg_port;
-	}
-
-	ret = ehea_get_jumboframe_status(port, &jumbo);
-	if (ret)
-		netdev_err(dev, "failed determining jumbo frame status\n");
-
-	netdev_info(dev, "Jumbo frames are %sabled\n",
-		    jumbo == 1 ? "en" : "dis");
-
-	adapter->active_ports++;
-
-	return port;
-
-out_unreg_port:
-	ehea_unregister_port(port);
-
-out_free_mc_list:
-	kfree(port->mc_list);
-
-out_free_ethdev:
-	free_netdev(dev);
-
-out_err:
-	pr_err("setting up logical port with id=%d failed, ret=%d\n",
-	       logical_port_id, ret);
-	return NULL;
-}
-
-static void ehea_shutdown_single_port(struct ehea_port *port)
-{
-	struct ehea_adapter *adapter = port->adapter;
-
-	cancel_work_sync(&port->reset_task);
-	cancel_delayed_work_sync(&port->stats_work);
-	unregister_netdev(port->netdev);
-	ehea_unregister_port(port);
-	kfree(port->mc_list);
-	free_netdev(port->netdev);
-	adapter->active_ports--;
-}
-
-static int ehea_setup_ports(struct ehea_adapter *adapter)
-{
-	struct device_node *lhea_dn;
-	struct device_node *eth_dn;
-
-	const u32 *dn_log_port_id;
-	int i = 0;
-
-	lhea_dn = adapter->ofdev->dev.of_node;
-	for_each_child_of_node(lhea_dn, eth_dn) {
-		dn_log_port_id = of_get_property(eth_dn, "ibm,hea-port-no",
-						 NULL);
-		if (!dn_log_port_id) {
-			pr_err("bad device node: eth_dn name=%pOF\n", eth_dn);
-			continue;
-		}
-
-		if (ehea_add_adapter_mr(adapter)) {
-			pr_err("creating MR failed\n");
-			of_node_put(eth_dn);
-			return -EIO;
-		}
-
-		adapter->port[i] = ehea_setup_single_port(adapter,
-							  *dn_log_port_id,
-							  eth_dn);
-		if (adapter->port[i])
-			netdev_info(adapter->port[i]->netdev,
-				    "logical port id #%d\n", *dn_log_port_id);
-		else
-			ehea_remove_adapter_mr(adapter);
-
-		i++;
-	}
-	return 0;
-}
-
-static struct device_node *ehea_get_eth_dn(struct ehea_adapter *adapter,
-					   u32 logical_port_id)
-{
-	struct device_node *lhea_dn;
-	struct device_node *eth_dn;
-	const u32 *dn_log_port_id;
-
-	lhea_dn = adapter->ofdev->dev.of_node;
-	for_each_child_of_node(lhea_dn, eth_dn) {
-		dn_log_port_id = of_get_property(eth_dn, "ibm,hea-port-no",
-						 NULL);
-		if (dn_log_port_id)
-			if (*dn_log_port_id == logical_port_id)
-				return eth_dn;
-	}
-
-	return NULL;
-}
-
-static ssize_t probe_port_store(struct device *dev,
-			       struct device_attribute *attr,
-			       const char *buf, size_t count)
-{
-	struct ehea_adapter *adapter = dev_get_drvdata(dev);
-	struct ehea_port *port;
-	struct device_node *eth_dn = NULL;
-	int i;
-
-	u32 logical_port_id;
-
-	sscanf(buf, "%d", &logical_port_id);
-
-	port = ehea_get_port(adapter, logical_port_id);
-
-	if (port) {
-		netdev_info(port->netdev, "adding port with logical port id=%d failed: port already configured\n",
-			    logical_port_id);
-		return -EINVAL;
-	}
-
-	eth_dn = ehea_get_eth_dn(adapter, logical_port_id);
-
-	if (!eth_dn) {
-		pr_info("no logical port with id %d found\n", logical_port_id);
-		return -EINVAL;
-	}
-
-	if (ehea_add_adapter_mr(adapter)) {
-		pr_err("creating MR failed\n");
-		of_node_put(eth_dn);
-		return -EIO;
-	}
-
-	port = ehea_setup_single_port(adapter, logical_port_id, eth_dn);
-
-	of_node_put(eth_dn);
-
-	if (port) {
-		for (i = 0; i < EHEA_MAX_PORTS; i++)
-			if (!adapter->port[i]) {
-				adapter->port[i] = port;
-				break;
-			}
-
-		netdev_info(port->netdev, "added: (logical port id=%d)\n",
-			    logical_port_id);
-	} else {
-		ehea_remove_adapter_mr(adapter);
-		return -EIO;
-	}
-
-	return (ssize_t) count;
-}
-
-static ssize_t remove_port_store(struct device *dev,
-				 struct device_attribute *attr,
-				 const char *buf, size_t count)
-{
-	struct ehea_adapter *adapter = dev_get_drvdata(dev);
-	struct ehea_port *port;
-	int i;
-	u32 logical_port_id;
-
-	sscanf(buf, "%d", &logical_port_id);
-
-	port = ehea_get_port(adapter, logical_port_id);
-
-	if (port) {
-		netdev_info(port->netdev, "removed: (logical port id=%d)\n",
-			    logical_port_id);
-
-		ehea_shutdown_single_port(port);
-
-		for (i = 0; i < EHEA_MAX_PORTS; i++)
-			if (adapter->port[i] == port) {
-				adapter->port[i] = NULL;
-				break;
-			}
-	} else {
-		pr_err("removing port with logical port id=%d failed. port not configured.\n",
-		       logical_port_id);
-		return -EINVAL;
-	}
-
-	ehea_remove_adapter_mr(adapter);
-
-	return (ssize_t) count;
-}
-
-static DEVICE_ATTR_WO(probe_port);
-static DEVICE_ATTR_WO(remove_port);
-
-static int ehea_create_device_sysfs(struct platform_device *dev)
-{
-	int ret = device_create_file(&dev->dev, &dev_attr_probe_port);
-	if (ret)
-		goto out;
-
-	ret = device_create_file(&dev->dev, &dev_attr_remove_port);
-	if (ret)
-		device_remove_file(&dev->dev, &dev_attr_probe_port);
-out:
-	return ret;
-}
-
-static void ehea_remove_device_sysfs(struct platform_device *dev)
-{
-	device_remove_file(&dev->dev, &dev_attr_probe_port);
-	device_remove_file(&dev->dev, &dev_attr_remove_port);
-}
-
-static int ehea_reboot_notifier(struct notifier_block *nb,
-				unsigned long action, void *unused)
-{
-	if (action == SYS_RESTART) {
-		pr_info("Reboot: freeing all eHEA resources\n");
-		ibmebus_unregister_driver(&ehea_driver);
-	}
-	return NOTIFY_DONE;
-}
-
-static struct notifier_block ehea_reboot_nb = {
-	.notifier_call = ehea_reboot_notifier,
-};
-
-static int ehea_mem_notifier(struct notifier_block *nb,
-			     unsigned long action, void *data)
-{
-	int ret = NOTIFY_BAD;
-	struct memory_notify *arg = data;
-
-	mutex_lock(&dlpar_mem_lock);
-
-	switch (action) {
-	case MEM_CANCEL_OFFLINE:
-		pr_info("memory offlining canceled");
-		fallthrough;	/* re-add canceled memory block */
-
-	case MEM_ONLINE:
-		pr_info("memory is going online");
-		set_bit(__EHEA_STOP_XFER, &ehea_driver_flags);
-		if (ehea_add_sect_bmap(arg->start_pfn, arg->nr_pages))
-			goto out_unlock;
-		ehea_rereg_mrs();
-		break;
-
-	case MEM_GOING_OFFLINE:
-		pr_info("memory is going offline");
-		set_bit(__EHEA_STOP_XFER, &ehea_driver_flags);
-		if (ehea_rem_sect_bmap(arg->start_pfn, arg->nr_pages))
-			goto out_unlock;
-		ehea_rereg_mrs();
-		break;
-
-	default:
-		break;
-	}
-
-	ehea_update_firmware_handles();
-	ret = NOTIFY_OK;
-
-out_unlock:
-	mutex_unlock(&dlpar_mem_lock);
-	return ret;
-}
-
-static struct notifier_block ehea_mem_nb = {
-	.notifier_call = ehea_mem_notifier,
-};
-
-static void ehea_crash_handler(void)
-{
-	int i;
-
-	if (ehea_fw_handles.arr)
-		for (i = 0; i < ehea_fw_handles.num_entries; i++)
-			ehea_h_free_resource(ehea_fw_handles.arr[i].adh,
-					     ehea_fw_handles.arr[i].fwh,
-					     FORCE_FREE);
-
-	if (ehea_bcmc_regs.arr)
-		for (i = 0; i < ehea_bcmc_regs.num_entries; i++)
-			ehea_h_reg_dereg_bcmc(ehea_bcmc_regs.arr[i].adh,
-					      ehea_bcmc_regs.arr[i].port_id,
-					      ehea_bcmc_regs.arr[i].reg_type,
-					      ehea_bcmc_regs.arr[i].macaddr,
-					      0, H_DEREG_BCMC);
-}
-
-static atomic_t ehea_memory_hooks_registered;
-
-/* Register memory hooks on probe of first adapter */
-static int ehea_register_memory_hooks(void)
-{
-	int ret = 0;
-
-	if (atomic_inc_return(&ehea_memory_hooks_registered) > 1)
-		return 0;
-
-	ret = ehea_create_busmap();
-	if (ret) {
-		pr_info("ehea_create_busmap failed\n");
-		goto out;
-	}
-
-	ret = register_reboot_notifier(&ehea_reboot_nb);
-	if (ret) {
-		pr_info("register_reboot_notifier failed\n");
-		goto out;
-	}
-
-	ret = register_memory_notifier(&ehea_mem_nb);
-	if (ret) {
-		pr_info("register_memory_notifier failed\n");
-		goto out2;
-	}
-
-	ret = crash_shutdown_register(ehea_crash_handler);
-	if (ret) {
-		pr_info("crash_shutdown_register failed\n");
-		goto out3;
-	}
-
-	return 0;
-
-out3:
-	unregister_memory_notifier(&ehea_mem_nb);
-out2:
-	unregister_reboot_notifier(&ehea_reboot_nb);
-out:
-	atomic_dec(&ehea_memory_hooks_registered);
-	return ret;
-}
-
-static void ehea_unregister_memory_hooks(void)
-{
-	/* Only remove the hooks if we've registered them */
-	if (atomic_read(&ehea_memory_hooks_registered) == 0)
-		return;
-
-	unregister_reboot_notifier(&ehea_reboot_nb);
-	if (crash_shutdown_unregister(ehea_crash_handler))
-		pr_info("failed unregistering crash handler\n");
-	unregister_memory_notifier(&ehea_mem_nb);
-}
-
-static int ehea_probe_adapter(struct platform_device *dev)
-{
-	struct ehea_adapter *adapter;
-	const u64 *adapter_handle;
-	int ret;
-	int i;
-
-	ret = ehea_register_memory_hooks();
-	if (ret)
-		return ret;
-
-	if (!dev || !dev->dev.of_node) {
-		pr_err("Invalid ibmebus device probed\n");
-		return -EINVAL;
-	}
-
-	adapter = devm_kzalloc(&dev->dev, sizeof(*adapter), GFP_KERNEL);
-	if (!adapter) {
-		ret = -ENOMEM;
-		dev_err(&dev->dev, "no mem for ehea_adapter\n");
-		goto out;
-	}
-
-	list_add(&adapter->list, &adapter_list);
-
-	adapter->ofdev = dev;
-
-	adapter_handle = of_get_property(dev->dev.of_node, "ibm,hea-handle",
-					 NULL);
-	if (adapter_handle)
-		adapter->handle = *adapter_handle;
-
-	if (!adapter->handle) {
-		dev_err(&dev->dev, "failed getting handle for adapter"
-			" '%pOF'\n", dev->dev.of_node);
-		ret = -ENODEV;
-		goto out_free_ad;
-	}
-
-	adapter->pd = EHEA_PD_ID;
-
-	platform_set_drvdata(dev, adapter);
-
-
-	/* initialize adapter and ports */
-	/* get adapter properties */
-	ret = ehea_sense_adapter_attr(adapter);
-	if (ret) {
-		dev_err(&dev->dev, "sense_adapter_attr failed: %d\n", ret);
-		goto out_free_ad;
-	}
-
-	adapter->neq = ehea_create_eq(adapter,
-				      EHEA_NEQ, EHEA_MAX_ENTRIES_EQ, 1);
-	if (!adapter->neq) {
-		ret = -EIO;
-		dev_err(&dev->dev, "NEQ creation failed\n");
-		goto out_free_ad;
-	}
-
-	tasklet_setup(&adapter->neq_tasklet, ehea_neq_tasklet);
-
-	ret = ehea_create_device_sysfs(dev);
-	if (ret)
-		goto out_kill_eq;
-
-	ret = ehea_setup_ports(adapter);
-	if (ret) {
-		dev_err(&dev->dev, "setup_ports failed\n");
-		goto out_rem_dev_sysfs;
-	}
-
-	ret = ibmebus_request_irq(adapter->neq->attr.ist1,
-				  ehea_interrupt_neq, 0,
-				  "ehea_neq", adapter);
-	if (ret) {
-		dev_err(&dev->dev, "requesting NEQ IRQ failed\n");
-		goto out_shutdown_ports;
-	}
-
-	/* Handle any events that might be pending. */
-	tasklet_hi_schedule(&adapter->neq_tasklet);
-
-	ret = 0;
-	goto out;
-
-out_shutdown_ports:
-	for (i = 0; i < EHEA_MAX_PORTS; i++)
-		if (adapter->port[i]) {
-			ehea_shutdown_single_port(adapter->port[i]);
-			adapter->port[i] = NULL;
-		}
-
-out_rem_dev_sysfs:
-	ehea_remove_device_sysfs(dev);
-
-out_kill_eq:
-	ehea_destroy_eq(adapter->neq);
-
-out_free_ad:
-	list_del(&adapter->list);
-
-out:
-	ehea_update_firmware_handles();
-
-	return ret;
-}
-
-static void ehea_remove(struct platform_device *dev)
-{
-	struct ehea_adapter *adapter = platform_get_drvdata(dev);
-	int i;
-
-	for (i = 0; i < EHEA_MAX_PORTS; i++)
-		if (adapter->port[i]) {
-			ehea_shutdown_single_port(adapter->port[i]);
-			adapter->port[i] = NULL;
-		}
-
-	ehea_remove_device_sysfs(dev);
-
-	ibmebus_free_irq(adapter->neq->attr.ist1, adapter);
-	tasklet_kill(&adapter->neq_tasklet);
-
-	ehea_destroy_eq(adapter->neq);
-	ehea_remove_adapter_mr(adapter);
-	list_del(&adapter->list);
-
-	ehea_update_firmware_handles();
-}
-
-static int check_module_parm(void)
-{
-	int ret = 0;
-
-	if ((rq1_entries < EHEA_MIN_ENTRIES_QP) ||
-	    (rq1_entries > EHEA_MAX_ENTRIES_RQ1)) {
-		pr_info("Bad parameter: rq1_entries\n");
-		ret = -EINVAL;
-	}
-	if ((rq2_entries < EHEA_MIN_ENTRIES_QP) ||
-	    (rq2_entries > EHEA_MAX_ENTRIES_RQ2)) {
-		pr_info("Bad parameter: rq2_entries\n");
-		ret = -EINVAL;
-	}
-	if ((rq3_entries < EHEA_MIN_ENTRIES_QP) ||
-	    (rq3_entries > EHEA_MAX_ENTRIES_RQ3)) {
-		pr_info("Bad parameter: rq3_entries\n");
-		ret = -EINVAL;
-	}
-	if ((sq_entries < EHEA_MIN_ENTRIES_QP) ||
-	    (sq_entries > EHEA_MAX_ENTRIES_SQ)) {
-		pr_info("Bad parameter: sq_entries\n");
-		ret = -EINVAL;
-	}
-
-	return ret;
-}
-
-static ssize_t capabilities_show(struct device_driver *drv, char *buf)
-{
-	return sprintf(buf, "%d", EHEA_CAPABILITIES);
-}
-
-static DRIVER_ATTR_RO(capabilities);
-
-static int __init ehea_module_init(void)
-{
-	int ret;
-
-	pr_info("IBM eHEA ethernet device driver (Release %s)\n", DRV_VERSION);
-
-	memset(&ehea_fw_handles, 0, sizeof(ehea_fw_handles));
-	memset(&ehea_bcmc_regs, 0, sizeof(ehea_bcmc_regs));
-
-	mutex_init(&ehea_fw_handles.lock);
-	spin_lock_init(&ehea_bcmc_regs.lock);
-
-	ret = check_module_parm();
-	if (ret)
-		goto out;
-
-	ret = ibmebus_register_driver(&ehea_driver);
-	if (ret) {
-		pr_err("failed registering eHEA device driver on ebus\n");
-		goto out;
-	}
-
-	ret = driver_create_file(&ehea_driver.driver,
-				 &driver_attr_capabilities);
-	if (ret) {
-		pr_err("failed to register capabilities attribute, ret=%d\n",
-		       ret);
-		goto out2;
-	}
-
-	return ret;
-
-out2:
-	ibmebus_unregister_driver(&ehea_driver);
-out:
-	return ret;
-}
-
-static void __exit ehea_module_exit(void)
-{
-	driver_remove_file(&ehea_driver.driver, &driver_attr_capabilities);
-	ibmebus_unregister_driver(&ehea_driver);
-	ehea_unregister_memory_hooks();
-	kfree(ehea_fw_handles.arr);
-	kfree(ehea_bcmc_regs.arr);
-	ehea_destroy_busmap();
-}
-
-module_init(ehea_module_init);
-module_exit(ehea_module_exit);
diff --git a/drivers/net/ethernet/ibm/ehea/ehea_phyp.c b/drivers/net/ethernet/ibm/ehea/ehea_phyp.c
deleted file mode 100644
index e63716e139f58..0000000000000
--- a/drivers/net/ethernet/ibm/ehea/ehea_phyp.c
+++ /dev/null
@@ -1,612 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- *  linux/drivers/net/ethernet/ibm/ehea/ehea_phyp.c
- *
- *  eHEA ethernet device driver for IBM eServer System p
- *
- *  (C) Copyright IBM Corp. 2006
- *
- *  Authors:
- *	 Christoph Raisch <raisch@de.ibm.com>
- *	 Jan-Bernd Themann <themann@de.ibm.com>
- *	 Thomas Klein <tklein@de.ibm.com>
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include "ehea_phyp.h"
-
-
-static inline u16 get_order_of_qentries(u16 queue_entries)
-{
-	u8 ld = 1;		/*  logarithmus dualis */
-	while (((1U << ld) - 1) < queue_entries)
-		ld++;
-	return ld - 1;
-}
-
-/* Defines for H_CALL H_ALLOC_RESOURCE */
-#define H_ALL_RES_TYPE_QP	 1
-#define H_ALL_RES_TYPE_CQ	 2
-#define H_ALL_RES_TYPE_EQ	 3
-#define H_ALL_RES_TYPE_MR	 5
-#define H_ALL_RES_TYPE_MW	 6
-
-static long ehea_plpar_hcall_norets(unsigned long opcode,
-				    unsigned long arg1,
-				    unsigned long arg2,
-				    unsigned long arg3,
-				    unsigned long arg4,
-				    unsigned long arg5,
-				    unsigned long arg6,
-				    unsigned long arg7)
-{
-	long ret;
-	int i, sleep_msecs;
-
-	for (i = 0; i < 5; i++) {
-		ret = plpar_hcall_norets(opcode, arg1, arg2, arg3, arg4,
-					 arg5, arg6, arg7);
-
-		if (H_IS_LONG_BUSY(ret)) {
-			sleep_msecs = get_longbusy_msecs(ret);
-			msleep_interruptible(sleep_msecs);
-			continue;
-		}
-
-		if (ret < H_SUCCESS)
-			pr_err("opcode=%lx ret=%lx"
-			       " arg1=%lx arg2=%lx arg3=%lx arg4=%lx"
-			       " arg5=%lx arg6=%lx arg7=%lx\n",
-			       opcode, ret,
-			       arg1, arg2, arg3, arg4, arg5, arg6, arg7);
-
-		return ret;
-	}
-
-	return H_BUSY;
-}
-
-static long ehea_plpar_hcall9(unsigned long opcode,
-			      unsigned long *outs, /* array of 9 outputs */
-			      unsigned long arg1,
-			      unsigned long arg2,
-			      unsigned long arg3,
-			      unsigned long arg4,
-			      unsigned long arg5,
-			      unsigned long arg6,
-			      unsigned long arg7,
-			      unsigned long arg8,
-			      unsigned long arg9)
-{
-	long ret;
-	int i, sleep_msecs;
-	u8 cb_cat;
-
-	for (i = 0; i < 5; i++) {
-		ret = plpar_hcall9(opcode, outs,
-				   arg1, arg2, arg3, arg4, arg5,
-				   arg6, arg7, arg8, arg9);
-
-		if (H_IS_LONG_BUSY(ret)) {
-			sleep_msecs = get_longbusy_msecs(ret);
-			msleep_interruptible(sleep_msecs);
-			continue;
-		}
-
-		cb_cat = EHEA_BMASK_GET(H_MEHEAPORT_CAT, arg2);
-
-		if ((ret < H_SUCCESS) && !(((ret == H_AUTHORITY)
-		    && (opcode == H_MODIFY_HEA_PORT))
-		    && (((cb_cat == H_PORT_CB4) && ((arg3 == H_PORT_CB4_JUMBO)
-		    || (arg3 == H_PORT_CB4_SPEED))) || ((cb_cat == H_PORT_CB7)
-		    && (arg3 == H_PORT_CB7_DUCQPN)))))
-			pr_err("opcode=%lx ret=%lx"
-			       " arg1=%lx arg2=%lx arg3=%lx arg4=%lx"
-			       " arg5=%lx arg6=%lx arg7=%lx arg8=%lx"
-			       " arg9=%lx"
-			       " out1=%lx out2=%lx out3=%lx out4=%lx"
-			       " out5=%lx out6=%lx out7=%lx out8=%lx"
-			       " out9=%lx\n",
-			       opcode, ret,
-			       arg1, arg2, arg3, arg4, arg5,
-			       arg6, arg7, arg8, arg9,
-			       outs[0], outs[1], outs[2], outs[3], outs[4],
-			       outs[5], outs[6], outs[7], outs[8]);
-		return ret;
-	}
-
-	return H_BUSY;
-}
-
-u64 ehea_h_query_ehea_qp(const u64 adapter_handle, const u8 qp_category,
-			 const u64 qp_handle, const u64 sel_mask, void *cb_addr)
-{
-	return ehea_plpar_hcall_norets(H_QUERY_HEA_QP,
-				       adapter_handle,		/* R4 */
-				       qp_category,		/* R5 */
-				       qp_handle,		/* R6 */
-				       sel_mask,		/* R7 */
-				       __pa(cb_addr),		/* R8 */
-				       0, 0);
-}
-
-/* input param R5 */
-#define H_ALL_RES_QP_EQPO	  EHEA_BMASK_IBM(9, 11)
-#define H_ALL_RES_QP_QPP	  EHEA_BMASK_IBM(12, 12)
-#define H_ALL_RES_QP_RQR	  EHEA_BMASK_IBM(13, 15)
-#define H_ALL_RES_QP_EQEG	  EHEA_BMASK_IBM(16, 16)
-#define H_ALL_RES_QP_LL_QP	  EHEA_BMASK_IBM(17, 17)
-#define H_ALL_RES_QP_DMA128	  EHEA_BMASK_IBM(19, 19)
-#define H_ALL_RES_QP_HSM	  EHEA_BMASK_IBM(20, 21)
-#define H_ALL_RES_QP_SIGT	  EHEA_BMASK_IBM(22, 23)
-#define H_ALL_RES_QP_TENURE	  EHEA_BMASK_IBM(48, 55)
-#define H_ALL_RES_QP_RES_TYP	  EHEA_BMASK_IBM(56, 63)
-
-/* input param R9  */
-#define H_ALL_RES_QP_TOKEN	  EHEA_BMASK_IBM(0, 31)
-#define H_ALL_RES_QP_PD		  EHEA_BMASK_IBM(32, 63)
-
-/* input param R10 */
-#define H_ALL_RES_QP_MAX_SWQE	  EHEA_BMASK_IBM(4, 7)
-#define H_ALL_RES_QP_MAX_R1WQE	  EHEA_BMASK_IBM(12, 15)
-#define H_ALL_RES_QP_MAX_R2WQE	  EHEA_BMASK_IBM(20, 23)
-#define H_ALL_RES_QP_MAX_R3WQE	  EHEA_BMASK_IBM(28, 31)
-/* Max Send Scatter Gather Elements */
-#define H_ALL_RES_QP_MAX_SSGE	  EHEA_BMASK_IBM(37, 39)
-#define H_ALL_RES_QP_MAX_R1SGE	  EHEA_BMASK_IBM(45, 47)
-/* Max Receive SG Elements RQ1 */
-#define H_ALL_RES_QP_MAX_R2SGE	  EHEA_BMASK_IBM(53, 55)
-#define H_ALL_RES_QP_MAX_R3SGE	  EHEA_BMASK_IBM(61, 63)
-
-/* input param R11 */
-#define H_ALL_RES_QP_SWQE_IDL	  EHEA_BMASK_IBM(0, 7)
-/* max swqe immediate data length */
-#define H_ALL_RES_QP_PORT_NUM	  EHEA_BMASK_IBM(48, 63)
-
-/* input param R12 */
-#define H_ALL_RES_QP_TH_RQ2	  EHEA_BMASK_IBM(0, 15)
-/* Threshold RQ2 */
-#define H_ALL_RES_QP_TH_RQ3	  EHEA_BMASK_IBM(16, 31)
-/* Threshold RQ3 */
-
-/* output param R6 */
-#define H_ALL_RES_QP_ACT_SWQE	  EHEA_BMASK_IBM(0, 15)
-#define H_ALL_RES_QP_ACT_R1WQE	  EHEA_BMASK_IBM(16, 31)
-#define H_ALL_RES_QP_ACT_R2WQE	  EHEA_BMASK_IBM(32, 47)
-#define H_ALL_RES_QP_ACT_R3WQE	  EHEA_BMASK_IBM(48, 63)
-
-/* output param, R7 */
-#define H_ALL_RES_QP_ACT_SSGE	  EHEA_BMASK_IBM(0, 7)
-#define H_ALL_RES_QP_ACT_R1SGE	  EHEA_BMASK_IBM(8, 15)
-#define H_ALL_RES_QP_ACT_R2SGE	  EHEA_BMASK_IBM(16, 23)
-#define H_ALL_RES_QP_ACT_R3SGE	  EHEA_BMASK_IBM(24, 31)
-#define H_ALL_RES_QP_ACT_SWQE_IDL EHEA_BMASK_IBM(32, 39)
-
-/* output param R8,R9 */
-#define H_ALL_RES_QP_SIZE_SQ	  EHEA_BMASK_IBM(0, 31)
-#define H_ALL_RES_QP_SIZE_RQ1	  EHEA_BMASK_IBM(32, 63)
-#define H_ALL_RES_QP_SIZE_RQ2	  EHEA_BMASK_IBM(0, 31)
-#define H_ALL_RES_QP_SIZE_RQ3	  EHEA_BMASK_IBM(32, 63)
-
-/* output param R11,R12 */
-#define H_ALL_RES_QP_LIOBN_SQ	  EHEA_BMASK_IBM(0, 31)
-#define H_ALL_RES_QP_LIOBN_RQ1	  EHEA_BMASK_IBM(32, 63)
-#define H_ALL_RES_QP_LIOBN_RQ2	  EHEA_BMASK_IBM(0, 31)
-#define H_ALL_RES_QP_LIOBN_RQ3	  EHEA_BMASK_IBM(32, 63)
-
-u64 ehea_h_alloc_resource_qp(const u64 adapter_handle,
-			     struct ehea_qp_init_attr *init_attr, const u32 pd,
-			     u64 *qp_handle, struct h_epas *h_epas)
-{
-	u64 hret;
-	unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
-	u64 allocate_controls =
-	    EHEA_BMASK_SET(H_ALL_RES_QP_EQPO, init_attr->low_lat_rq1 ? 1 : 0)
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_QPP, 0)
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_RQR, 6)	/* rq1 & rq2 & rq3 */
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_EQEG, 0)	/* EQE gen. disabled */
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_LL_QP, init_attr->low_lat_rq1)
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_DMA128, 0)
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_HSM, 0)
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_SIGT, init_attr->signalingtype)
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_RES_TYP, H_ALL_RES_TYPE_QP);
-
-	u64 r9_reg = EHEA_BMASK_SET(H_ALL_RES_QP_PD, pd)
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_TOKEN, init_attr->qp_token);
-
-	u64 max_r10_reg =
-	    EHEA_BMASK_SET(H_ALL_RES_QP_MAX_SWQE,
-			   get_order_of_qentries(init_attr->max_nr_send_wqes))
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_MAX_R1WQE,
-			     get_order_of_qentries(init_attr->max_nr_rwqes_rq1))
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_MAX_R2WQE,
-			     get_order_of_qentries(init_attr->max_nr_rwqes_rq2))
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_MAX_R3WQE,
-			     get_order_of_qentries(init_attr->max_nr_rwqes_rq3))
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_MAX_SSGE, init_attr->wqe_size_enc_sq)
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_MAX_R1SGE,
-			     init_attr->wqe_size_enc_rq1)
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_MAX_R2SGE,
-			     init_attr->wqe_size_enc_rq2)
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_MAX_R3SGE,
-			     init_attr->wqe_size_enc_rq3);
-
-	u64 r11_in =
-	    EHEA_BMASK_SET(H_ALL_RES_QP_SWQE_IDL, init_attr->swqe_imm_data_len)
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_PORT_NUM, init_attr->port_nr);
-	u64 threshold =
-	    EHEA_BMASK_SET(H_ALL_RES_QP_TH_RQ2, init_attr->rq2_threshold)
-	    | EHEA_BMASK_SET(H_ALL_RES_QP_TH_RQ3, init_attr->rq3_threshold);
-
-	hret = ehea_plpar_hcall9(H_ALLOC_HEA_RESOURCE,
-				 outs,
-				 adapter_handle,		/* R4 */
-				 allocate_controls,		/* R5 */
-				 init_attr->send_cq_handle,	/* R6 */
-				 init_attr->recv_cq_handle,	/* R7 */
-				 init_attr->aff_eq_handle,	/* R8 */
-				 r9_reg,			/* R9 */
-				 max_r10_reg,			/* R10 */
-				 r11_in,			/* R11 */
-				 threshold);			/* R12 */
-
-	*qp_handle = outs[0];
-	init_attr->qp_nr = (u32)outs[1];
-
-	init_attr->act_nr_send_wqes =
-	    (u16)EHEA_BMASK_GET(H_ALL_RES_QP_ACT_SWQE, outs[2]);
-	init_attr->act_nr_rwqes_rq1 =
-	    (u16)EHEA_BMASK_GET(H_ALL_RES_QP_ACT_R1WQE, outs[2]);
-	init_attr->act_nr_rwqes_rq2 =
-	    (u16)EHEA_BMASK_GET(H_ALL_RES_QP_ACT_R2WQE, outs[2]);
-	init_attr->act_nr_rwqes_rq3 =
-	    (u16)EHEA_BMASK_GET(H_ALL_RES_QP_ACT_R3WQE, outs[2]);
-
-	init_attr->act_wqe_size_enc_sq = init_attr->wqe_size_enc_sq;
-	init_attr->act_wqe_size_enc_rq1 = init_attr->wqe_size_enc_rq1;
-	init_attr->act_wqe_size_enc_rq2 = init_attr->wqe_size_enc_rq2;
-	init_attr->act_wqe_size_enc_rq3 = init_attr->wqe_size_enc_rq3;
-
-	init_attr->nr_sq_pages =
-	    (u32)EHEA_BMASK_GET(H_ALL_RES_QP_SIZE_SQ, outs[4]);
-	init_attr->nr_rq1_pages =
-	    (u32)EHEA_BMASK_GET(H_ALL_RES_QP_SIZE_RQ1, outs[4]);
-	init_attr->nr_rq2_pages =
-	    (u32)EHEA_BMASK_GET(H_ALL_RES_QP_SIZE_RQ2, outs[5]);
-	init_attr->nr_rq3_pages =
-	    (u32)EHEA_BMASK_GET(H_ALL_RES_QP_SIZE_RQ3, outs[5]);
-
-	init_attr->liobn_sq =
-	    (u32)EHEA_BMASK_GET(H_ALL_RES_QP_LIOBN_SQ, outs[7]);
-	init_attr->liobn_rq1 =
-	    (u32)EHEA_BMASK_GET(H_ALL_RES_QP_LIOBN_RQ1, outs[7]);
-	init_attr->liobn_rq2 =
-	    (u32)EHEA_BMASK_GET(H_ALL_RES_QP_LIOBN_RQ2, outs[8]);
-	init_attr->liobn_rq3 =
-	    (u32)EHEA_BMASK_GET(H_ALL_RES_QP_LIOBN_RQ3, outs[8]);
-
-	if (!hret)
-		hcp_epas_ctor(h_epas, outs[6], outs[6]);
-
-	return hret;
-}
-
-u64 ehea_h_alloc_resource_cq(const u64 adapter_handle,
-			     struct ehea_cq_attr *cq_attr,
-			     u64 *cq_handle, struct h_epas *epas)
-{
-	u64 hret;
-	unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
-	hret = ehea_plpar_hcall9(H_ALLOC_HEA_RESOURCE,
-				 outs,
-				 adapter_handle,		/* R4 */
-				 H_ALL_RES_TYPE_CQ,		/* R5 */
-				 cq_attr->eq_handle,		/* R6 */
-				 cq_attr->cq_token,		/* R7 */
-				 cq_attr->max_nr_of_cqes,	/* R8 */
-				 0, 0, 0, 0);			/* R9-R12 */
-
-	*cq_handle = outs[0];
-	cq_attr->act_nr_of_cqes = outs[3];
-	cq_attr->nr_pages = outs[4];
-
-	if (!hret)
-		hcp_epas_ctor(epas, outs[5], outs[6]);
-
-	return hret;
-}
-
-/* Defines for H_CALL H_ALLOC_RESOURCE */
-#define H_ALL_RES_TYPE_QP	 1
-#define H_ALL_RES_TYPE_CQ	 2
-#define H_ALL_RES_TYPE_EQ	 3
-#define H_ALL_RES_TYPE_MR	 5
-#define H_ALL_RES_TYPE_MW	 6
-
-/*  input param R5 */
-#define H_ALL_RES_EQ_NEQ	     EHEA_BMASK_IBM(0, 0)
-#define H_ALL_RES_EQ_NON_NEQ_ISN     EHEA_BMASK_IBM(6, 7)
-#define H_ALL_RES_EQ_INH_EQE_GEN     EHEA_BMASK_IBM(16, 16)
-#define H_ALL_RES_EQ_RES_TYPE	     EHEA_BMASK_IBM(56, 63)
-/*  input param R6 */
-#define H_ALL_RES_EQ_MAX_EQE	     EHEA_BMASK_IBM(32, 63)
-
-/*  output param R6 */
-#define H_ALL_RES_EQ_LIOBN	     EHEA_BMASK_IBM(32, 63)
-
-/*  output param R7 */
-#define H_ALL_RES_EQ_ACT_EQE	     EHEA_BMASK_IBM(32, 63)
-
-/*  output param R8 */
-#define H_ALL_RES_EQ_ACT_PS	     EHEA_BMASK_IBM(32, 63)
-
-/*  output param R9 */
-#define H_ALL_RES_EQ_ACT_EQ_IST_C    EHEA_BMASK_IBM(30, 31)
-#define H_ALL_RES_EQ_ACT_EQ_IST_1    EHEA_BMASK_IBM(40, 63)
-
-/*  output param R10 */
-#define H_ALL_RES_EQ_ACT_EQ_IST_2    EHEA_BMASK_IBM(40, 63)
-
-/*  output param R11 */
-#define H_ALL_RES_EQ_ACT_EQ_IST_3    EHEA_BMASK_IBM(40, 63)
-
-/*  output param R12 */
-#define H_ALL_RES_EQ_ACT_EQ_IST_4    EHEA_BMASK_IBM(40, 63)
-
-u64 ehea_h_alloc_resource_eq(const u64 adapter_handle,
-			     struct ehea_eq_attr *eq_attr, u64 *eq_handle)
-{
-	u64 hret, allocate_controls;
-	unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
-	/* resource type */
-	allocate_controls =
-	    EHEA_BMASK_SET(H_ALL_RES_EQ_RES_TYPE, H_ALL_RES_TYPE_EQ)
-	    | EHEA_BMASK_SET(H_ALL_RES_EQ_NEQ, eq_attr->type ? 1 : 0)
-	    | EHEA_BMASK_SET(H_ALL_RES_EQ_INH_EQE_GEN, !eq_attr->eqe_gen)
-	    | EHEA_BMASK_SET(H_ALL_RES_EQ_NON_NEQ_ISN, 1);
-
-	hret = ehea_plpar_hcall9(H_ALLOC_HEA_RESOURCE,
-				 outs,
-				 adapter_handle,		/* R4 */
-				 allocate_controls,		/* R5 */
-				 eq_attr->max_nr_of_eqes,	/* R6 */
-				 0, 0, 0, 0, 0, 0);		/* R7-R10 */
-
-	*eq_handle = outs[0];
-	eq_attr->act_nr_of_eqes = outs[3];
-	eq_attr->nr_pages = outs[4];
-	eq_attr->ist1 = outs[5];
-	eq_attr->ist2 = outs[6];
-	eq_attr->ist3 = outs[7];
-	eq_attr->ist4 = outs[8];
-
-	return hret;
-}
-
-u64 ehea_h_modify_ehea_qp(const u64 adapter_handle, const u8 cat,
-			  const u64 qp_handle, const u64 sel_mask,
-			  void *cb_addr, u64 *inv_attr_id, u64 *proc_mask,
-			  u16 *out_swr, u16 *out_rwr)
-{
-	u64 hret;
-	unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
-	hret = ehea_plpar_hcall9(H_MODIFY_HEA_QP,
-				 outs,
-				 adapter_handle,		/* R4 */
-				 (u64) cat,			/* R5 */
-				 qp_handle,			/* R6 */
-				 sel_mask,			/* R7 */
-				 __pa(cb_addr),			/* R8 */
-				 0, 0, 0, 0);			/* R9-R12 */
-
-	*inv_attr_id = outs[0];
-	*out_swr = outs[3];
-	*out_rwr = outs[4];
-	*proc_mask = outs[5];
-
-	return hret;
-}
-
-u64 ehea_h_register_rpage(const u64 adapter_handle, const u8 pagesize,
-			  const u8 queue_type, const u64 resource_handle,
-			  const u64 log_pageaddr, u64 count)
-{
-	u64  reg_control;
-
-	reg_control = EHEA_BMASK_SET(H_REG_RPAGE_PAGE_SIZE, pagesize)
-		    | EHEA_BMASK_SET(H_REG_RPAGE_QT, queue_type);
-
-	return ehea_plpar_hcall_norets(H_REGISTER_HEA_RPAGES,
-				       adapter_handle,		/* R4 */
-				       reg_control,		/* R5 */
-				       resource_handle,		/* R6 */
-				       log_pageaddr,		/* R7 */
-				       count,			/* R8 */
-				       0, 0);			/* R9-R10 */
-}
-
-u64 ehea_h_register_smr(const u64 adapter_handle, const u64 orig_mr_handle,
-			const u64 vaddr_in, const u32 access_ctrl, const u32 pd,
-			struct ehea_mr *mr)
-{
-	u64 hret;
-	unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
-	hret = ehea_plpar_hcall9(H_REGISTER_SMR,
-				 outs,
-				 adapter_handle	      ,		 /* R4 */
-				 orig_mr_handle,		 /* R5 */
-				 vaddr_in,			 /* R6 */
-				 (((u64)access_ctrl) << 32ULL),	 /* R7 */
-				 pd,				 /* R8 */
-				 0, 0, 0, 0);			 /* R9-R12 */
-
-	mr->handle = outs[0];
-	mr->lkey = (u32)outs[2];
-
-	return hret;
-}
-
-u64 ehea_h_disable_and_get_hea(const u64 adapter_handle, const u64 qp_handle)
-{
-	unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
-	return ehea_plpar_hcall9(H_DISABLE_AND_GET_HEA,
-				 outs,
-				 adapter_handle,		/* R4 */
-				 H_DISABLE_GET_EHEA_WQE_P,	/* R5 */
-				 qp_handle,			/* R6 */
-				 0, 0, 0, 0, 0, 0);		/* R7-R12 */
-}
-
-u64 ehea_h_free_resource(const u64 adapter_handle, const u64 res_handle,
-			 u64 force_bit)
-{
-	return ehea_plpar_hcall_norets(H_FREE_RESOURCE,
-				       adapter_handle,	   /* R4 */
-				       res_handle,	   /* R5 */
-				       force_bit,
-				       0, 0, 0, 0);	   /* R7-R10 */
-}
-
-u64 ehea_h_alloc_resource_mr(const u64 adapter_handle, const u64 vaddr,
-			     const u64 length, const u32 access_ctrl,
-			     const u32 pd, u64 *mr_handle, u32 *lkey)
-{
-	u64 hret;
-	unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
-	hret = ehea_plpar_hcall9(H_ALLOC_HEA_RESOURCE,
-				 outs,
-				 adapter_handle,		   /* R4 */
-				 5,				   /* R5 */
-				 vaddr,				   /* R6 */
-				 length,			   /* R7 */
-				 (((u64) access_ctrl) << 32ULL),   /* R8 */
-				 pd,				   /* R9 */
-				 0, 0, 0);			   /* R10-R12 */
-
-	*mr_handle = outs[0];
-	*lkey = (u32)outs[2];
-	return hret;
-}
-
-u64 ehea_h_register_rpage_mr(const u64 adapter_handle, const u64 mr_handle,
-			     const u8 pagesize, const u8 queue_type,
-			     const u64 log_pageaddr, const u64 count)
-{
-	if ((count > 1) && (log_pageaddr & ~PAGE_MASK)) {
-		pr_err("not on pageboundary\n");
-		return H_PARAMETER;
-	}
-
-	return ehea_h_register_rpage(adapter_handle, pagesize,
-				     queue_type, mr_handle,
-				     log_pageaddr, count);
-}
-
-u64 ehea_h_query_ehea(const u64 adapter_handle, void *cb_addr)
-{
-	u64 hret, cb_logaddr;
-
-	cb_logaddr = __pa(cb_addr);
-
-	hret = ehea_plpar_hcall_norets(H_QUERY_HEA,
-				       adapter_handle,		/* R4 */
-				       cb_logaddr,		/* R5 */
-				       0, 0, 0, 0, 0);		/* R6-R10 */
-#ifdef DEBUG
-	ehea_dump(cb_addr, sizeof(struct hcp_query_ehea), "hcp_query_ehea");
-#endif
-	return hret;
-}
-
-u64 ehea_h_query_ehea_port(const u64 adapter_handle, const u16 port_num,
-			   const u8 cb_cat, const u64 select_mask,
-			   void *cb_addr)
-{
-	u64 port_info;
-	u64 cb_logaddr = __pa(cb_addr);
-	u64 arr_index = 0;
-
-	port_info = EHEA_BMASK_SET(H_MEHEAPORT_CAT, cb_cat)
-		  | EHEA_BMASK_SET(H_MEHEAPORT_PN, port_num);
-
-	return ehea_plpar_hcall_norets(H_QUERY_HEA_PORT,
-				       adapter_handle,		/* R4 */
-				       port_info,		/* R5 */
-				       select_mask,		/* R6 */
-				       arr_index,		/* R7 */
-				       cb_logaddr,		/* R8 */
-				       0, 0);			/* R9-R10 */
-}
-
-u64 ehea_h_modify_ehea_port(const u64 adapter_handle, const u16 port_num,
-			    const u8 cb_cat, const u64 select_mask,
-			    void *cb_addr)
-{
-	unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-	u64 port_info;
-	u64 arr_index = 0;
-	u64 cb_logaddr = __pa(cb_addr);
-
-	port_info = EHEA_BMASK_SET(H_MEHEAPORT_CAT, cb_cat)
-		  | EHEA_BMASK_SET(H_MEHEAPORT_PN, port_num);
-#ifdef DEBUG
-	ehea_dump(cb_addr, sizeof(struct hcp_ehea_port_cb0), "Before HCALL");
-#endif
-	return ehea_plpar_hcall9(H_MODIFY_HEA_PORT,
-				 outs,
-				 adapter_handle,		/* R4 */
-				 port_info,			/* R5 */
-				 select_mask,			/* R6 */
-				 arr_index,			/* R7 */
-				 cb_logaddr,			/* R8 */
-				 0, 0, 0, 0);			/* R9-R12 */
-}
-
-u64 ehea_h_reg_dereg_bcmc(const u64 adapter_handle, const u16 port_num,
-			  const u8 reg_type, const u64 mc_mac_addr,
-			  const u16 vlan_id, const u32 hcall_id)
-{
-	u64 r5_port_num, r6_reg_type, r7_mc_mac_addr, r8_vlan_id;
-	u64 mac_addr = mc_mac_addr >> 16;
-
-	r5_port_num = EHEA_BMASK_SET(H_REGBCMC_PN, port_num);
-	r6_reg_type = EHEA_BMASK_SET(H_REGBCMC_REGTYPE, reg_type);
-	r7_mc_mac_addr = EHEA_BMASK_SET(H_REGBCMC_MACADDR, mac_addr);
-	r8_vlan_id = EHEA_BMASK_SET(H_REGBCMC_VLANID, vlan_id);
-
-	return ehea_plpar_hcall_norets(hcall_id,
-				       adapter_handle,		/* R4 */
-				       r5_port_num,		/* R5 */
-				       r6_reg_type,		/* R6 */
-				       r7_mc_mac_addr,		/* R7 */
-				       r8_vlan_id,		/* R8 */
-				       0, 0);			/* R9-R12 */
-}
-
-u64 ehea_h_reset_events(const u64 adapter_handle, const u64 neq_handle,
-			const u64 event_mask)
-{
-	return ehea_plpar_hcall_norets(H_RESET_EVENTS,
-				       adapter_handle,		/* R4 */
-				       neq_handle,		/* R5 */
-				       event_mask,		/* R6 */
-				       0, 0, 0, 0);		/* R7-R12 */
-}
-
-u64 ehea_h_error_data(const u64 adapter_handle, const u64 ressource_handle,
-		      void *rblock)
-{
-	return ehea_plpar_hcall_norets(H_ERROR_DATA,
-				       adapter_handle,		/* R4 */
-				       ressource_handle,	/* R5 */
-				       __pa(rblock),		/* R6 */
-				       0, 0, 0, 0);		/* R7-R12 */
-}
diff --git a/drivers/net/ethernet/ibm/ehea/ehea_phyp.h b/drivers/net/ethernet/ibm/ehea/ehea_phyp.h
deleted file mode 100644
index e8b56c1034102..0000000000000
--- a/drivers/net/ethernet/ibm/ehea/ehea_phyp.h
+++ /dev/null
@@ -1,433 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- *  linux/drivers/net/ethernet/ibm/ehea/ehea_phyp.h
- *
- *  eHEA ethernet device driver for IBM eServer System p
- *
- *  (C) Copyright IBM Corp. 2006
- *
- *  Authors:
- *       Christoph Raisch <raisch@de.ibm.com>
- *       Jan-Bernd Themann <themann@de.ibm.com>
- *       Thomas Klein <tklein@de.ibm.com>
- */
-
-#ifndef __EHEA_PHYP_H__
-#define __EHEA_PHYP_H__
-
-#include <linux/delay.h>
-#include <asm/hvcall.h>
-#include "ehea.h"
-#include "ehea_hw.h"
-
-/* Some abbreviations used here:
- *
- * hcp_*  - structures, variables and functions releated to Hypervisor Calls
- */
-
-/* Number of pages which can be registered at once by H_REGISTER_HEA_RPAGES */
-#define EHEA_MAX_RPAGE 512
-
-/* Notification Event Queue (NEQ) Entry bit masks */
-#define NEQE_EVENT_CODE		EHEA_BMASK_IBM(2, 7)
-#define NEQE_PORTNUM  		EHEA_BMASK_IBM(32, 47)
-#define NEQE_PORT_UP		EHEA_BMASK_IBM(16, 16)
-#define NEQE_EXTSWITCH_PORT_UP	EHEA_BMASK_IBM(17, 17)
-#define NEQE_EXTSWITCH_PRIMARY	EHEA_BMASK_IBM(18, 18)
-#define NEQE_PLID		EHEA_BMASK_IBM(16, 47)
-
-/* Notification Event Codes */
-#define EHEA_EC_PORTSTATE_CHG	0x30
-#define EHEA_EC_ADAPTER_MALFUNC	0x32
-#define EHEA_EC_PORT_MALFUNC	0x33
-
-/* Notification Event Log Register (NELR) bit masks */
-#define NELR_PORT_MALFUNC	EHEA_BMASK_IBM(61, 61)
-#define NELR_ADAPTER_MALFUNC	EHEA_BMASK_IBM(62, 62)
-#define NELR_PORTSTATE_CHG	EHEA_BMASK_IBM(63, 63)
-
-static inline void hcp_epas_ctor(struct h_epas *epas, u64 paddr_kernel,
-				 u64 paddr_user)
-{
-	/* To support 64k pages we must round to 64k page boundary */
-	epas->kernel.addr = ioremap((paddr_kernel & PAGE_MASK), PAGE_SIZE) +
-			    (paddr_kernel & ~PAGE_MASK);
-	epas->user.addr = paddr_user;
-}
-
-static inline void hcp_epas_dtor(struct h_epas *epas)
-{
-	if (epas->kernel.addr)
-		iounmap((void __iomem *)((u64)epas->kernel.addr & PAGE_MASK));
-
-	epas->user.addr = 0;
-	epas->kernel.addr = 0;
-}
-
-struct hcp_modify_qp_cb0 {
-	u64 qp_ctl_reg;		/* 00 */
-	u32 max_swqe;		/* 02 */
-	u32 max_rwqe;		/* 03 */
-	u32 port_nb;		/* 04 */
-	u32 reserved0;		/* 05 */
-	u64 qp_aer;		/* 06 */
-	u64 qp_tenure;		/* 08 */
-};
-
-/* Hcall Query/Modify Queue Pair Control Block 0 Selection Mask Bits */
-#define H_QPCB0_ALL             EHEA_BMASK_IBM(0, 5)
-#define H_QPCB0_QP_CTL_REG      EHEA_BMASK_IBM(0, 0)
-#define H_QPCB0_MAX_SWQE        EHEA_BMASK_IBM(1, 1)
-#define H_QPCB0_MAX_RWQE        EHEA_BMASK_IBM(2, 2)
-#define H_QPCB0_PORT_NB         EHEA_BMASK_IBM(3, 3)
-#define H_QPCB0_QP_AER          EHEA_BMASK_IBM(4, 4)
-#define H_QPCB0_QP_TENURE       EHEA_BMASK_IBM(5, 5)
-
-/* Queue Pair Control Register Status Bits */
-#define H_QP_CR_ENABLED		    0x8000000000000000ULL /* QP enabled */
-							  /* QP States: */
-#define H_QP_CR_STATE_RESET	    0x0000010000000000ULL /*  Reset */
-#define H_QP_CR_STATE_INITIALIZED   0x0000020000000000ULL /*  Initialized */
-#define H_QP_CR_STATE_RDY2RCV	    0x0000030000000000ULL /*  Ready to recv */
-#define H_QP_CR_STATE_RDY2SND	    0x0000050000000000ULL /*  Ready to send */
-#define H_QP_CR_STATE_ERROR	    0x0000800000000000ULL /*  Error */
-#define H_QP_CR_RES_STATE 	    0x0000007F00000000ULL /* Resultant state */
-
-struct hcp_modify_qp_cb1 {
-	u32 qpn;		/* 00 */
-	u32 qp_asyn_ev_eq_nb;	/* 01 */
-	u64 sq_cq_handle;	/* 02 */
-	u64 rq_cq_handle;	/* 04 */
-	/* sgel = scatter gather element */
-	u32 sgel_nb_sq;		/* 06 */
-	u32 sgel_nb_rq1;	/* 07 */
-	u32 sgel_nb_rq2;	/* 08 */
-	u32 sgel_nb_rq3;	/* 09 */
-};
-
-/* Hcall Query/Modify Queue Pair Control Block 1 Selection Mask Bits */
-#define H_QPCB1_ALL             EHEA_BMASK_IBM(0, 7)
-#define H_QPCB1_QPN             EHEA_BMASK_IBM(0, 0)
-#define H_QPCB1_ASYN_EV_EQ_NB   EHEA_BMASK_IBM(1, 1)
-#define H_QPCB1_SQ_CQ_HANDLE    EHEA_BMASK_IBM(2, 2)
-#define H_QPCB1_RQ_CQ_HANDLE    EHEA_BMASK_IBM(3, 3)
-#define H_QPCB1_SGEL_NB_SQ      EHEA_BMASK_IBM(4, 4)
-#define H_QPCB1_SGEL_NB_RQ1     EHEA_BMASK_IBM(5, 5)
-#define H_QPCB1_SGEL_NB_RQ2     EHEA_BMASK_IBM(6, 6)
-#define H_QPCB1_SGEL_NB_RQ3     EHEA_BMASK_IBM(7, 7)
-
-struct hcp_query_ehea {
-	u32 cur_num_qps;		/* 00 */
-	u32 cur_num_cqs;		/* 01 */
-	u32 cur_num_eqs;		/* 02 */
-	u32 cur_num_mrs;		/* 03 */
-	u32 auth_level;			/* 04 */
-	u32 max_num_qps;		/* 05 */
-	u32 max_num_cqs;		/* 06 */
-	u32 max_num_eqs;		/* 07 */
-	u32 max_num_mrs;		/* 08 */
-	u32 reserved0;			/* 09 */
-	u32 int_clock_freq;		/* 10 */
-	u32 max_num_pds;		/* 11 */
-	u32 max_num_addr_handles;	/* 12 */
-	u32 max_num_cqes;		/* 13 */
-	u32 max_num_wqes;		/* 14 */
-	u32 max_num_sgel_rq1wqe;	/* 15 */
-	u32 max_num_sgel_rq2wqe;	/* 16 */
-	u32 max_num_sgel_rq3wqe;	/* 17 */
-	u32 mr_page_size;		/* 18 */
-	u32 reserved1;			/* 19 */
-	u64 max_mr_size;		/* 20 */
-	u64 reserved2;			/* 22 */
-	u32 num_ports;			/* 24 */
-	u32 reserved3;			/* 25 */
-	u32 reserved4;			/* 26 */
-	u32 reserved5;			/* 27 */
-	u64 max_mc_mac;			/* 28 */
-	u64 ehea_cap;			/* 30 */
-	u32 max_isn_per_eq;		/* 32 */
-	u32 max_num_neq;		/* 33 */
-	u64 max_num_vlan_ids;		/* 34 */
-	u32 max_num_port_group;		/* 36 */
-	u32 max_num_phys_port;		/* 37 */
-
-};
-
-/* Hcall Query/Modify Port Control Block defines */
-#define H_PORT_CB0	 0
-#define H_PORT_CB1	 1
-#define H_PORT_CB2	 2
-#define H_PORT_CB3	 3
-#define H_PORT_CB4	 4
-#define H_PORT_CB5	 5
-#define H_PORT_CB6	 6
-#define H_PORT_CB7	 7
-
-struct hcp_ehea_port_cb0 {
-	u64 port_mac_addr;
-	u64 port_rc;
-	u64 reserved0;
-	u32 port_op_state;
-	u32 port_speed;
-	u32 ext_swport_op_state;
-	u32 neg_tpf_prpf;
-	u32 num_default_qps;
-	u32 reserved1;
-	u64 default_qpn_arr[16];
-};
-
-/* Hcall Query/Modify Port Control Block 0 Selection Mask Bits */
-#define H_PORT_CB0_ALL		EHEA_BMASK_IBM(0, 7)    /* Set all bits */
-#define H_PORT_CB0_MAC		EHEA_BMASK_IBM(0, 0)    /* MAC address */
-#define H_PORT_CB0_PRC		EHEA_BMASK_IBM(1, 1)    /* Port Recv Control */
-#define H_PORT_CB0_DEFQPNARRAY	EHEA_BMASK_IBM(7, 7)    /* Default QPN Array */
-
-/*  Hcall Query Port: Returned port speed values */
-#define H_SPEED_10M_H	1	/*  10 Mbps, Half Duplex */
-#define H_SPEED_10M_F	2	/*  10 Mbps, Full Duplex */
-#define H_SPEED_100M_H	3	/* 100 Mbps, Half Duplex */
-#define H_SPEED_100M_F	4	/* 100 Mbps, Full Duplex */
-#define H_SPEED_1G_F	6	/*   1 Gbps, Full Duplex */
-#define H_SPEED_10G_F	8	/*  10 Gbps, Full Duplex */
-
-/* Port Receive Control Status Bits */
-#define PXLY_RC_VALID           EHEA_BMASK_IBM(49, 49)
-#define PXLY_RC_VLAN_XTRACT     EHEA_BMASK_IBM(50, 50)
-#define PXLY_RC_TCP_6_TUPLE     EHEA_BMASK_IBM(51, 51)
-#define PXLY_RC_UDP_6_TUPLE     EHEA_BMASK_IBM(52, 52)
-#define PXLY_RC_TCP_3_TUPLE     EHEA_BMASK_IBM(53, 53)
-#define PXLY_RC_TCP_2_TUPLE     EHEA_BMASK_IBM(54, 54)
-#define PXLY_RC_LLC_SNAP        EHEA_BMASK_IBM(55, 55)
-#define PXLY_RC_JUMBO_FRAME     EHEA_BMASK_IBM(56, 56)
-#define PXLY_RC_FRAG_IP_PKT     EHEA_BMASK_IBM(57, 57)
-#define PXLY_RC_TCP_UDP_CHKSUM  EHEA_BMASK_IBM(58, 58)
-#define PXLY_RC_IP_CHKSUM       EHEA_BMASK_IBM(59, 59)
-#define PXLY_RC_MAC_FILTER      EHEA_BMASK_IBM(60, 60)
-#define PXLY_RC_UNTAG_FILTER    EHEA_BMASK_IBM(61, 61)
-#define PXLY_RC_VLAN_TAG_FILTER EHEA_BMASK_IBM(62, 63)
-
-#define PXLY_RC_VLAN_FILTER     2
-#define PXLY_RC_VLAN_PERM       0
-
-
-#define H_PORT_CB1_ALL          0x8000000000000000ULL
-
-struct hcp_ehea_port_cb1 {
-	u64 vlan_filter[64];
-};
-
-#define H_PORT_CB2_ALL          0xFFE0000000000000ULL
-
-struct hcp_ehea_port_cb2 {
-	u64 rxo;
-	u64 rxucp;
-	u64 rxufd;
-	u64 rxuerr;
-	u64 rxftl;
-	u64 rxmcp;
-	u64 rxbcp;
-	u64 txo;
-	u64 txucp;
-	u64 txmcp;
-	u64 txbcp;
-};
-
-struct hcp_ehea_port_cb3 {
-	u64 vlan_bc_filter[64];
-	u64 vlan_mc_filter[64];
-	u64 vlan_un_filter[64];
-	u64 port_mac_hash_array[64];
-};
-
-#define H_PORT_CB4_ALL          0xF000000000000000ULL
-#define H_PORT_CB4_JUMBO        0x1000000000000000ULL
-#define H_PORT_CB4_SPEED        0x8000000000000000ULL
-
-struct hcp_ehea_port_cb4 {
-	u32 port_speed;
-	u32 pause_frame;
-	u32 ens_port_op_state;
-	u32 jumbo_frame;
-	u32 ens_port_wrap;
-};
-
-/* Hcall Query/Modify Port Control Block 5 Selection Mask Bits */
-#define H_PORT_CB5_RCU		0x0001000000000000ULL
-#define PXS_RCU			EHEA_BMASK_IBM(61, 63)
-
-struct hcp_ehea_port_cb5 {
-	u64 prc;	        /* 00 */
-	u64 uaa;		/* 01 */
-	u64 macvc;		/* 02 */
-	u64 xpcsc;		/* 03 */
-	u64 xpcsp;		/* 04 */
-	u64 pcsid;		/* 05 */
-	u64 xpcsst;		/* 06 */
-	u64 pthlb;		/* 07 */
-	u64 pthrb;		/* 08 */
-	u64 pqu;		/* 09 */
-	u64 pqd;		/* 10 */
-	u64 prt;		/* 11 */
-	u64 wsth;		/* 12 */
-	u64 rcb;		/* 13 */
-	u64 rcm;		/* 14 */
-	u64 rcu;		/* 15 */
-	u64 macc;		/* 16 */
-	u64 pc;			/* 17 */
-	u64 pst;		/* 18 */
-	u64 ducqpn;		/* 19 */
-	u64 mcqpn;		/* 20 */
-	u64 mma;		/* 21 */
-	u64 pmc0h;		/* 22 */
-	u64 pmc0l;		/* 23 */
-	u64 lbc;		/* 24 */
-};
-
-#define H_PORT_CB6_ALL  0xFFFFFE7FFFFF8000ULL
-
-struct hcp_ehea_port_cb6 {
-	u64 rxo;		/* 00 */
-	u64 rx64;		/* 01 */
-	u64 rx65;		/* 02 */
-	u64 rx128;		/* 03 */
-	u64 rx256;		/* 04 */
-	u64 rx512;		/* 05 */
-	u64 rx1024;		/* 06 */
-	u64 rxbfcs;		/* 07 */
-	u64 rxime;		/* 08 */
-	u64 rxrle;		/* 09 */
-	u64 rxorle;		/* 10 */
-	u64 rxftl;		/* 11 */
-	u64 rxjab;		/* 12 */
-	u64 rxse;		/* 13 */
-	u64 rxce;		/* 14 */
-	u64 rxrf;		/* 15 */
-	u64 rxfrag;		/* 16 */
-	u64 rxuoc;		/* 17 */
-	u64 rxcpf;		/* 18 */
-	u64 rxsb;		/* 19 */
-	u64 rxfd;		/* 20 */
-	u64 rxoerr;		/* 21 */
-	u64 rxaln;		/* 22 */
-	u64 ducqpn;		/* 23 */
-	u64 reserved0;		/* 24 */
-	u64 rxmcp;		/* 25 */
-	u64 rxbcp;		/* 26 */
-	u64 txmcp;		/* 27 */
-	u64 txbcp;		/* 28 */
-	u64 txo;		/* 29 */
-	u64 tx64;		/* 30 */
-	u64 tx65;		/* 31 */
-	u64 tx128;		/* 32 */
-	u64 tx256;		/* 33 */
-	u64 tx512;		/* 34 */
-	u64 tx1024;		/* 35 */
-	u64 txbfcs;		/* 36 */
-	u64 txcpf;		/* 37 */
-	u64 txlf;		/* 38 */
-	u64 txrf;		/* 39 */
-	u64 txime;		/* 40 */
-	u64 txsc;		/* 41 */
-	u64 txmc;		/* 42 */
-	u64 txsqe;		/* 43 */
-	u64 txdef;		/* 44 */
-	u64 txlcol;		/* 45 */
-	u64 txexcol;		/* 46 */
-	u64 txcse;		/* 47 */
-	u64 txbor;		/* 48 */
-};
-
-#define H_PORT_CB7_DUCQPN 0x8000000000000000ULL
-
-struct hcp_ehea_port_cb7 {
-	u64 def_uc_qpn;
-};
-
-u64 ehea_h_query_ehea_qp(const u64 adapter_handle,
-			 const u8 qp_category,
-			 const u64 qp_handle, const u64 sel_mask,
-			 void *cb_addr);
-
-u64 ehea_h_modify_ehea_qp(const u64 adapter_handle,
-			  const u8 cat,
-			  const u64 qp_handle,
-			  const u64 sel_mask,
-			  void *cb_addr,
-			  u64 *inv_attr_id,
-			  u64 *proc_mask, u16 *out_swr, u16 *out_rwr);
-
-u64 ehea_h_alloc_resource_eq(const u64 adapter_handle,
-			     struct ehea_eq_attr *eq_attr, u64 *eq_handle);
-
-u64 ehea_h_alloc_resource_cq(const u64 adapter_handle,
-			     struct ehea_cq_attr *cq_attr,
-			     u64 *cq_handle, struct h_epas *epas);
-
-u64 ehea_h_alloc_resource_qp(const u64 adapter_handle,
-			     struct ehea_qp_init_attr *init_attr,
-			     const u32 pd,
-			     u64 *qp_handle, struct h_epas *h_epas);
-
-#define H_REG_RPAGE_PAGE_SIZE          EHEA_BMASK_IBM(48, 55)
-#define H_REG_RPAGE_QT                 EHEA_BMASK_IBM(62, 63)
-
-u64 ehea_h_register_rpage(const u64 adapter_handle,
-			  const u8 pagesize,
-			  const u8 queue_type,
-			  const u64 resource_handle,
-			  const u64 log_pageaddr, u64 count);
-
-#define H_DISABLE_GET_EHEA_WQE_P  1
-#define H_DISABLE_GET_SQ_WQE_P    2
-#define H_DISABLE_GET_RQC         3
-
-u64 ehea_h_disable_and_get_hea(const u64 adapter_handle, const u64 qp_handle);
-
-#define FORCE_FREE 1
-#define NORMAL_FREE 0
-
-u64 ehea_h_free_resource(const u64 adapter_handle, const u64 res_handle,
-			 u64 force_bit);
-
-u64 ehea_h_alloc_resource_mr(const u64 adapter_handle, const u64 vaddr,
-			     const u64 length, const u32 access_ctrl,
-			     const u32 pd, u64 *mr_handle, u32 *lkey);
-
-u64 ehea_h_register_rpage_mr(const u64 adapter_handle, const u64 mr_handle,
-			     const u8 pagesize, const u8 queue_type,
-			     const u64 log_pageaddr, const u64 count);
-
-u64 ehea_h_register_smr(const u64 adapter_handle, const u64 orig_mr_handle,
-			const u64 vaddr_in, const u32 access_ctrl, const u32 pd,
-			struct ehea_mr *mr);
-
-u64 ehea_h_query_ehea(const u64 adapter_handle, void *cb_addr);
-
-/* output param R5 */
-#define H_MEHEAPORT_CAT		EHEA_BMASK_IBM(40, 47)
-#define H_MEHEAPORT_PN		EHEA_BMASK_IBM(48, 63)
-
-u64 ehea_h_query_ehea_port(const u64 adapter_handle, const u16 port_num,
-			   const u8 cb_cat, const u64 select_mask,
-			   void *cb_addr);
-
-u64 ehea_h_modify_ehea_port(const u64 adapter_handle, const u16 port_num,
-			    const u8 cb_cat, const u64 select_mask,
-			    void *cb_addr);
-
-#define H_REGBCMC_PN            EHEA_BMASK_IBM(48, 63)
-#define H_REGBCMC_REGTYPE       EHEA_BMASK_IBM(60, 63)
-#define H_REGBCMC_MACADDR       EHEA_BMASK_IBM(16, 63)
-#define H_REGBCMC_VLANID        EHEA_BMASK_IBM(52, 63)
-
-u64 ehea_h_reg_dereg_bcmc(const u64 adapter_handle, const u16 port_num,
-			  const u8 reg_type, const u64 mc_mac_addr,
-			  const u16 vlan_id, const u32 hcall_id);
-
-u64 ehea_h_reset_events(const u64 adapter_handle, const u64 neq_handle,
-			const u64 event_mask);
-
-u64 ehea_h_error_data(const u64 adapter_handle, const u64 ressource_handle,
-		      void *rblock);
-
-#endif	/* __EHEA_PHYP_H__ */
diff --git a/drivers/net/ethernet/ibm/ehea/ehea_qmr.c b/drivers/net/ethernet/ibm/ehea/ehea_qmr.c
deleted file mode 100644
index 60629a0032b24..0000000000000
--- a/drivers/net/ethernet/ibm/ehea/ehea_qmr.c
+++ /dev/null
@@ -1,999 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- *  linux/drivers/net/ethernet/ibm/ehea/ehea_qmr.c
- *
- *  eHEA ethernet device driver for IBM eServer System p
- *
- *  (C) Copyright IBM Corp. 2006
- *
- *  Authors:
- *       Christoph Raisch <raisch@de.ibm.com>
- *       Jan-Bernd Themann <themann@de.ibm.com>
- *       Thomas Klein <tklein@de.ibm.com>
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/mm.h>
-#include <linux/slab.h>
-#include "ehea.h"
-#include "ehea_phyp.h"
-#include "ehea_qmr.h"
-
-static struct ehea_bmap *ehea_bmap;
-
-static void *hw_qpageit_get_inc(struct hw_queue *queue)
-{
-	void *retvalue = hw_qeit_get(queue);
-
-	queue->current_q_offset += queue->pagesize;
-	if (queue->current_q_offset > queue->queue_length) {
-		queue->current_q_offset -= queue->pagesize;
-		retvalue = NULL;
-	} else if (((u64) retvalue) & (EHEA_PAGESIZE-1)) {
-		pr_err("not on pageboundary\n");
-		retvalue = NULL;
-	}
-	return retvalue;
-}
-
-static int hw_queue_ctor(struct hw_queue *queue, const u32 nr_of_pages,
-			  const u32 pagesize, const u32 qe_size)
-{
-	int pages_per_kpage = PAGE_SIZE / pagesize;
-	int i, k;
-
-	if ((pagesize > PAGE_SIZE) || (!pages_per_kpage)) {
-		pr_err("pagesize conflict! kernel pagesize=%d, ehea pagesize=%d\n",
-		       (int)PAGE_SIZE, (int)pagesize);
-		return -EINVAL;
-	}
-
-	queue->queue_length = nr_of_pages * pagesize;
-	queue->queue_pages = kmalloc_array(nr_of_pages, sizeof(void *),
-					   GFP_KERNEL);
-	if (!queue->queue_pages)
-		return -ENOMEM;
-
-	/*
-	 * allocate pages for queue:
-	 * outer loop allocates whole kernel pages (page aligned) and
-	 * inner loop divides a kernel page into smaller hea queue pages
-	 */
-	i = 0;
-	while (i < nr_of_pages) {
-		u8 *kpage = (u8 *)get_zeroed_page(GFP_KERNEL);
-		if (!kpage)
-			goto out_nomem;
-		for (k = 0; k < pages_per_kpage && i < nr_of_pages; k++) {
-			(queue->queue_pages)[i] = (struct ehea_page *)kpage;
-			kpage += pagesize;
-			i++;
-		}
-	}
-
-	queue->current_q_offset = 0;
-	queue->qe_size = qe_size;
-	queue->pagesize = pagesize;
-	queue->toggle_state = 1;
-
-	return 0;
-out_nomem:
-	for (i = 0; i < nr_of_pages; i += pages_per_kpage) {
-		if (!(queue->queue_pages)[i])
-			break;
-		free_page((unsigned long)(queue->queue_pages)[i]);
-	}
-	return -ENOMEM;
-}
-
-static void hw_queue_dtor(struct hw_queue *queue)
-{
-	int pages_per_kpage;
-	int i, nr_pages;
-
-	if (!queue || !queue->queue_pages)
-		return;
-
-	pages_per_kpage = PAGE_SIZE / queue->pagesize;
-
-	nr_pages = queue->queue_length / queue->pagesize;
-
-	for (i = 0; i < nr_pages; i += pages_per_kpage)
-		free_page((unsigned long)(queue->queue_pages)[i]);
-
-	kfree(queue->queue_pages);
-}
-
-struct ehea_cq *ehea_create_cq(struct ehea_adapter *adapter,
-			       int nr_of_cqe, u64 eq_handle, u32 cq_token)
-{
-	struct ehea_cq *cq;
-	u64 hret, rpage;
-	u32 counter;
-	int ret;
-	void *vpage;
-
-	cq = kzalloc_obj(*cq);
-	if (!cq)
-		goto out_nomem;
-
-	cq->attr.max_nr_of_cqes = nr_of_cqe;
-	cq->attr.cq_token = cq_token;
-	cq->attr.eq_handle = eq_handle;
-
-	cq->adapter = adapter;
-
-	hret = ehea_h_alloc_resource_cq(adapter->handle, &cq->attr,
-					&cq->fw_handle, &cq->epas);
-	if (hret != H_SUCCESS) {
-		pr_err("alloc_resource_cq failed\n");
-		goto out_freemem;
-	}
-
-	ret = hw_queue_ctor(&cq->hw_queue, cq->attr.nr_pages,
-			    EHEA_PAGESIZE, sizeof(struct ehea_cqe));
-	if (ret)
-		goto out_freeres;
-
-	for (counter = 0; counter < cq->attr.nr_pages; counter++) {
-		vpage = hw_qpageit_get_inc(&cq->hw_queue);
-		if (!vpage) {
-			pr_err("hw_qpageit_get_inc failed\n");
-			goto out_kill_hwq;
-		}
-
-		rpage = __pa(vpage);
-		hret = ehea_h_register_rpage(adapter->handle,
-					     0, EHEA_CQ_REGISTER_ORIG,
-					     cq->fw_handle, rpage, 1);
-		if (hret < H_SUCCESS) {
-			pr_err("register_rpage_cq failed ehea_cq=%p hret=%llx counter=%i act_pages=%i\n",
-			       cq, hret, counter, cq->attr.nr_pages);
-			goto out_kill_hwq;
-		}
-
-		if (counter == (cq->attr.nr_pages - 1)) {
-			vpage = hw_qpageit_get_inc(&cq->hw_queue);
-
-			if ((hret != H_SUCCESS) || (vpage)) {
-				pr_err("registration of pages not complete hret=%llx\n",
-				       hret);
-				goto out_kill_hwq;
-			}
-		} else {
-			if (hret != H_PAGE_REGISTERED) {
-				pr_err("CQ: registration of page failed hret=%llx\n",
-				       hret);
-				goto out_kill_hwq;
-			}
-		}
-	}
-
-	hw_qeit_reset(&cq->hw_queue);
-	ehea_reset_cq_ep(cq);
-	ehea_reset_cq_n1(cq);
-
-	return cq;
-
-out_kill_hwq:
-	hw_queue_dtor(&cq->hw_queue);
-
-out_freeres:
-	ehea_h_free_resource(adapter->handle, cq->fw_handle, FORCE_FREE);
-
-out_freemem:
-	kfree(cq);
-
-out_nomem:
-	return NULL;
-}
-
-static u64 ehea_destroy_cq_res(struct ehea_cq *cq, u64 force)
-{
-	u64 hret;
-	u64 adapter_handle = cq->adapter->handle;
-
-	/* deregister all previous registered pages */
-	hret = ehea_h_free_resource(adapter_handle, cq->fw_handle, force);
-	if (hret != H_SUCCESS)
-		return hret;
-
-	hw_queue_dtor(&cq->hw_queue);
-	kfree(cq);
-
-	return hret;
-}
-
-int ehea_destroy_cq(struct ehea_cq *cq)
-{
-	u64 hret, aer, aerr;
-	if (!cq)
-		return 0;
-
-	hcp_epas_dtor(&cq->epas);
-	hret = ehea_destroy_cq_res(cq, NORMAL_FREE);
-	if (hret == H_R_STATE) {
-		ehea_error_data(cq->adapter, cq->fw_handle, &aer, &aerr);
-		hret = ehea_destroy_cq_res(cq, FORCE_FREE);
-	}
-
-	if (hret != H_SUCCESS) {
-		pr_err("destroy CQ failed\n");
-		return -EIO;
-	}
-
-	return 0;
-}
-
-struct ehea_eq *ehea_create_eq(struct ehea_adapter *adapter,
-			       const enum ehea_eq_type type,
-			       const u32 max_nr_of_eqes, const u8 eqe_gen)
-{
-	int ret, i;
-	u64 hret, rpage;
-	void *vpage;
-	struct ehea_eq *eq;
-
-	eq = kzalloc_obj(*eq);
-	if (!eq)
-		return NULL;
-
-	eq->adapter = adapter;
-	eq->attr.type = type;
-	eq->attr.max_nr_of_eqes = max_nr_of_eqes;
-	eq->attr.eqe_gen = eqe_gen;
-	spin_lock_init(&eq->spinlock);
-
-	hret = ehea_h_alloc_resource_eq(adapter->handle,
-					&eq->attr, &eq->fw_handle);
-	if (hret != H_SUCCESS) {
-		pr_err("alloc_resource_eq failed\n");
-		goto out_freemem;
-	}
-
-	ret = hw_queue_ctor(&eq->hw_queue, eq->attr.nr_pages,
-			    EHEA_PAGESIZE, sizeof(struct ehea_eqe));
-	if (ret) {
-		pr_err("can't allocate eq pages\n");
-		goto out_freeres;
-	}
-
-	for (i = 0; i < eq->attr.nr_pages; i++) {
-		vpage = hw_qpageit_get_inc(&eq->hw_queue);
-		if (!vpage) {
-			pr_err("hw_qpageit_get_inc failed\n");
-			hret = H_RESOURCE;
-			goto out_kill_hwq;
-		}
-
-		rpage = __pa(vpage);
-
-		hret = ehea_h_register_rpage(adapter->handle, 0,
-					     EHEA_EQ_REGISTER_ORIG,
-					     eq->fw_handle, rpage, 1);
-
-		if (i == (eq->attr.nr_pages - 1)) {
-			/* last page */
-			vpage = hw_qpageit_get_inc(&eq->hw_queue);
-			if ((hret != H_SUCCESS) || (vpage))
-				goto out_kill_hwq;
-
-		} else {
-			if (hret != H_PAGE_REGISTERED)
-				goto out_kill_hwq;
-
-		}
-	}
-
-	hw_qeit_reset(&eq->hw_queue);
-	return eq;
-
-out_kill_hwq:
-	hw_queue_dtor(&eq->hw_queue);
-
-out_freeres:
-	ehea_h_free_resource(adapter->handle, eq->fw_handle, FORCE_FREE);
-
-out_freemem:
-	kfree(eq);
-	return NULL;
-}
-
-struct ehea_eqe *ehea_poll_eq(struct ehea_eq *eq)
-{
-	struct ehea_eqe *eqe;
-	unsigned long flags;
-
-	spin_lock_irqsave(&eq->spinlock, flags);
-	eqe = hw_eqit_eq_get_inc_valid(&eq->hw_queue);
-	spin_unlock_irqrestore(&eq->spinlock, flags);
-
-	return eqe;
-}
-
-static u64 ehea_destroy_eq_res(struct ehea_eq *eq, u64 force)
-{
-	u64 hret;
-	unsigned long flags;
-
-	spin_lock_irqsave(&eq->spinlock, flags);
-
-	hret = ehea_h_free_resource(eq->adapter->handle, eq->fw_handle, force);
-	spin_unlock_irqrestore(&eq->spinlock, flags);
-
-	if (hret != H_SUCCESS)
-		return hret;
-
-	hw_queue_dtor(&eq->hw_queue);
-	kfree(eq);
-
-	return hret;
-}
-
-int ehea_destroy_eq(struct ehea_eq *eq)
-{
-	u64 hret, aer, aerr;
-	if (!eq)
-		return 0;
-
-	hcp_epas_dtor(&eq->epas);
-
-	hret = ehea_destroy_eq_res(eq, NORMAL_FREE);
-	if (hret == H_R_STATE) {
-		ehea_error_data(eq->adapter, eq->fw_handle, &aer, &aerr);
-		hret = ehea_destroy_eq_res(eq, FORCE_FREE);
-	}
-
-	if (hret != H_SUCCESS) {
-		pr_err("destroy EQ failed\n");
-		return -EIO;
-	}
-
-	return 0;
-}
-
-/* allocates memory for a queue and registers pages in phyp */
-static int ehea_qp_alloc_register(struct ehea_qp *qp, struct hw_queue *hw_queue,
-			   int nr_pages, int wqe_size, int act_nr_sges,
-			   struct ehea_adapter *adapter, int h_call_q_selector)
-{
-	u64 hret, rpage;
-	int ret, cnt;
-	void *vpage;
-
-	ret = hw_queue_ctor(hw_queue, nr_pages, EHEA_PAGESIZE, wqe_size);
-	if (ret)
-		return ret;
-
-	for (cnt = 0; cnt < nr_pages; cnt++) {
-		vpage = hw_qpageit_get_inc(hw_queue);
-		if (!vpage) {
-			pr_err("hw_qpageit_get_inc failed\n");
-			goto out_kill_hwq;
-		}
-		rpage = __pa(vpage);
-		hret = ehea_h_register_rpage(adapter->handle,
-					     0, h_call_q_selector,
-					     qp->fw_handle, rpage, 1);
-		if (hret < H_SUCCESS) {
-			pr_err("register_rpage_qp failed\n");
-			goto out_kill_hwq;
-		}
-	}
-	hw_qeit_reset(hw_queue);
-	return 0;
-
-out_kill_hwq:
-	hw_queue_dtor(hw_queue);
-	return -EIO;
-}
-
-static inline u32 map_wqe_size(u8 wqe_enc_size)
-{
-	return 128 << wqe_enc_size;
-}
-
-struct ehea_qp *ehea_create_qp(struct ehea_adapter *adapter,
-			       u32 pd, struct ehea_qp_init_attr *init_attr)
-{
-	int ret;
-	u64 hret;
-	struct ehea_qp *qp;
-	u32 wqe_size_in_bytes_sq, wqe_size_in_bytes_rq1;
-	u32 wqe_size_in_bytes_rq2, wqe_size_in_bytes_rq3;
-
-
-	qp = kzalloc_obj(*qp);
-	if (!qp)
-		return NULL;
-
-	qp->adapter = adapter;
-
-	hret = ehea_h_alloc_resource_qp(adapter->handle, init_attr, pd,
-					&qp->fw_handle, &qp->epas);
-	if (hret != H_SUCCESS) {
-		pr_err("ehea_h_alloc_resource_qp failed\n");
-		goto out_freemem;
-	}
-
-	wqe_size_in_bytes_sq = map_wqe_size(init_attr->act_wqe_size_enc_sq);
-	wqe_size_in_bytes_rq1 = map_wqe_size(init_attr->act_wqe_size_enc_rq1);
-	wqe_size_in_bytes_rq2 = map_wqe_size(init_attr->act_wqe_size_enc_rq2);
-	wqe_size_in_bytes_rq3 = map_wqe_size(init_attr->act_wqe_size_enc_rq3);
-
-	ret = ehea_qp_alloc_register(qp, &qp->hw_squeue, init_attr->nr_sq_pages,
-				     wqe_size_in_bytes_sq,
-				     init_attr->act_wqe_size_enc_sq, adapter,
-				     0);
-	if (ret) {
-		pr_err("can't register for sq ret=%x\n", ret);
-		goto out_freeres;
-	}
-
-	ret = ehea_qp_alloc_register(qp, &qp->hw_rqueue1,
-				     init_attr->nr_rq1_pages,
-				     wqe_size_in_bytes_rq1,
-				     init_attr->act_wqe_size_enc_rq1,
-				     adapter, 1);
-	if (ret) {
-		pr_err("can't register for rq1 ret=%x\n", ret);
-		goto out_kill_hwsq;
-	}
-
-	if (init_attr->rq_count > 1) {
-		ret = ehea_qp_alloc_register(qp, &qp->hw_rqueue2,
-					     init_attr->nr_rq2_pages,
-					     wqe_size_in_bytes_rq2,
-					     init_attr->act_wqe_size_enc_rq2,
-					     adapter, 2);
-		if (ret) {
-			pr_err("can't register for rq2 ret=%x\n", ret);
-			goto out_kill_hwr1q;
-		}
-	}
-
-	if (init_attr->rq_count > 2) {
-		ret = ehea_qp_alloc_register(qp, &qp->hw_rqueue3,
-					     init_attr->nr_rq3_pages,
-					     wqe_size_in_bytes_rq3,
-					     init_attr->act_wqe_size_enc_rq3,
-					     adapter, 3);
-		if (ret) {
-			pr_err("can't register for rq3 ret=%x\n", ret);
-			goto out_kill_hwr2q;
-		}
-	}
-
-	qp->init_attr = *init_attr;
-
-	return qp;
-
-out_kill_hwr2q:
-	hw_queue_dtor(&qp->hw_rqueue2);
-
-out_kill_hwr1q:
-	hw_queue_dtor(&qp->hw_rqueue1);
-
-out_kill_hwsq:
-	hw_queue_dtor(&qp->hw_squeue);
-
-out_freeres:
-	ehea_h_disable_and_get_hea(adapter->handle, qp->fw_handle);
-	ehea_h_free_resource(adapter->handle, qp->fw_handle, FORCE_FREE);
-
-out_freemem:
-	kfree(qp);
-	return NULL;
-}
-
-static u64 ehea_destroy_qp_res(struct ehea_qp *qp, u64 force)
-{
-	u64 hret;
-	struct ehea_qp_init_attr *qp_attr = &qp->init_attr;
-
-
-	ehea_h_disable_and_get_hea(qp->adapter->handle, qp->fw_handle);
-	hret = ehea_h_free_resource(qp->adapter->handle, qp->fw_handle, force);
-	if (hret != H_SUCCESS)
-		return hret;
-
-	hw_queue_dtor(&qp->hw_squeue);
-	hw_queue_dtor(&qp->hw_rqueue1);
-
-	if (qp_attr->rq_count > 1)
-		hw_queue_dtor(&qp->hw_rqueue2);
-	if (qp_attr->rq_count > 2)
-		hw_queue_dtor(&qp->hw_rqueue3);
-	kfree(qp);
-
-	return hret;
-}
-
-int ehea_destroy_qp(struct ehea_qp *qp)
-{
-	u64 hret, aer, aerr;
-	if (!qp)
-		return 0;
-
-	hcp_epas_dtor(&qp->epas);
-
-	hret = ehea_destroy_qp_res(qp, NORMAL_FREE);
-	if (hret == H_R_STATE) {
-		ehea_error_data(qp->adapter, qp->fw_handle, &aer, &aerr);
-		hret = ehea_destroy_qp_res(qp, FORCE_FREE);
-	}
-
-	if (hret != H_SUCCESS) {
-		pr_err("destroy QP failed\n");
-		return -EIO;
-	}
-
-	return 0;
-}
-
-static inline int ehea_calc_index(unsigned long i, unsigned long s)
-{
-	return (i >> s) & EHEA_INDEX_MASK;
-}
-
-static inline int ehea_init_top_bmap(struct ehea_top_bmap *ehea_top_bmap,
-				     int dir)
-{
-	if (!ehea_top_bmap->dir[dir]) {
-		ehea_top_bmap->dir[dir] =
-			kzalloc_obj(struct ehea_dir_bmap);
-		if (!ehea_top_bmap->dir[dir])
-			return -ENOMEM;
-	}
-	return 0;
-}
-
-static inline int ehea_init_bmap(struct ehea_bmap *ehea_bmap, int top, int dir)
-{
-	if (!ehea_bmap->top[top]) {
-		ehea_bmap->top[top] =
-			kzalloc_obj(struct ehea_top_bmap);
-		if (!ehea_bmap->top[top])
-			return -ENOMEM;
-	}
-	return ehea_init_top_bmap(ehea_bmap->top[top], dir);
-}
-
-static DEFINE_MUTEX(ehea_busmap_mutex);
-static unsigned long ehea_mr_len;
-
-#define EHEA_BUSMAP_ADD_SECT 1
-#define EHEA_BUSMAP_REM_SECT 0
-
-static void ehea_rebuild_busmap(void)
-{
-	u64 vaddr = EHEA_BUSMAP_START;
-	int top, dir, idx;
-
-	for (top = 0; top < EHEA_MAP_ENTRIES; top++) {
-		struct ehea_top_bmap *ehea_top;
-		int valid_dir_entries = 0;
-
-		if (!ehea_bmap->top[top])
-			continue;
-		ehea_top = ehea_bmap->top[top];
-		for (dir = 0; dir < EHEA_MAP_ENTRIES; dir++) {
-			struct ehea_dir_bmap *ehea_dir;
-			int valid_entries = 0;
-
-			if (!ehea_top->dir[dir])
-				continue;
-			valid_dir_entries++;
-			ehea_dir = ehea_top->dir[dir];
-			for (idx = 0; idx < EHEA_MAP_ENTRIES; idx++) {
-				if (!ehea_dir->ent[idx])
-					continue;
-				valid_entries++;
-				ehea_dir->ent[idx] = vaddr;
-				vaddr += EHEA_SECTSIZE;
-			}
-			if (!valid_entries) {
-				ehea_top->dir[dir] = NULL;
-				kfree(ehea_dir);
-			}
-		}
-		if (!valid_dir_entries) {
-			ehea_bmap->top[top] = NULL;
-			kfree(ehea_top);
-		}
-	}
-}
-
-static int ehea_update_busmap(unsigned long pfn, unsigned long nr_pages, int add)
-{
-	unsigned long i, start_section, end_section;
-
-	if (!nr_pages)
-		return 0;
-
-	if (!ehea_bmap) {
-		ehea_bmap = kzalloc_obj(struct ehea_bmap);
-		if (!ehea_bmap)
-			return -ENOMEM;
-	}
-
-	start_section = (pfn * PAGE_SIZE) / EHEA_SECTSIZE;
-	end_section = start_section + ((nr_pages * PAGE_SIZE) / EHEA_SECTSIZE);
-	/* Mark entries as valid or invalid only; address is assigned later */
-	for (i = start_section; i < end_section; i++) {
-		u64 flag;
-		int top = ehea_calc_index(i, EHEA_TOP_INDEX_SHIFT);
-		int dir = ehea_calc_index(i, EHEA_DIR_INDEX_SHIFT);
-		int idx = i & EHEA_INDEX_MASK;
-
-		if (add) {
-			int ret = ehea_init_bmap(ehea_bmap, top, dir);
-			if (ret)
-				return ret;
-			flag = 1; /* valid */
-			ehea_mr_len += EHEA_SECTSIZE;
-		} else {
-			if (!ehea_bmap->top[top])
-				continue;
-			if (!ehea_bmap->top[top]->dir[dir])
-				continue;
-			flag = 0; /* invalid */
-			ehea_mr_len -= EHEA_SECTSIZE;
-		}
-
-		ehea_bmap->top[top]->dir[dir]->ent[idx] = flag;
-	}
-	ehea_rebuild_busmap(); /* Assign contiguous addresses for mr */
-	return 0;
-}
-
-int ehea_add_sect_bmap(unsigned long pfn, unsigned long nr_pages)
-{
-	int ret;
-
-	mutex_lock(&ehea_busmap_mutex);
-	ret = ehea_update_busmap(pfn, nr_pages, EHEA_BUSMAP_ADD_SECT);
-	mutex_unlock(&ehea_busmap_mutex);
-	return ret;
-}
-
-int ehea_rem_sect_bmap(unsigned long pfn, unsigned long nr_pages)
-{
-	int ret;
-
-	mutex_lock(&ehea_busmap_mutex);
-	ret = ehea_update_busmap(pfn, nr_pages, EHEA_BUSMAP_REM_SECT);
-	mutex_unlock(&ehea_busmap_mutex);
-	return ret;
-}
-
-static int ehea_is_hugepage(unsigned long pfn)
-{
-	if (pfn & EHEA_HUGEPAGE_PFN_MASK)
-		return 0;
-
-	if (page_shift(pfn_to_page(pfn)) != EHEA_HUGEPAGESHIFT)
-		return 0;
-
-	return 1;
-}
-
-static int ehea_create_busmap_callback(unsigned long initial_pfn,
-				       unsigned long total_nr_pages, void *arg)
-{
-	int ret;
-	unsigned long pfn, start_pfn, end_pfn, nr_pages;
-
-	if ((total_nr_pages * PAGE_SIZE) < EHEA_HUGEPAGE_SIZE)
-		return ehea_update_busmap(initial_pfn, total_nr_pages,
-					  EHEA_BUSMAP_ADD_SECT);
-
-	/* Given chunk is >= 16GB -> check for hugepages */
-	start_pfn = initial_pfn;
-	end_pfn = initial_pfn + total_nr_pages;
-	pfn = start_pfn;
-
-	while (pfn < end_pfn) {
-		if (ehea_is_hugepage(pfn)) {
-			/* Add mem found in front of the hugepage */
-			nr_pages = pfn - start_pfn;
-			ret = ehea_update_busmap(start_pfn, nr_pages,
-						 EHEA_BUSMAP_ADD_SECT);
-			if (ret)
-				return ret;
-
-			/* Skip the hugepage */
-			pfn += (EHEA_HUGEPAGE_SIZE / PAGE_SIZE);
-			start_pfn = pfn;
-		} else
-			pfn += (EHEA_SECTSIZE / PAGE_SIZE);
-	}
-
-	/* Add mem found behind the hugepage(s)  */
-	nr_pages = pfn - start_pfn;
-	return ehea_update_busmap(start_pfn, nr_pages, EHEA_BUSMAP_ADD_SECT);
-}
-
-int ehea_create_busmap(void)
-{
-	int ret;
-
-	mutex_lock(&ehea_busmap_mutex);
-	ehea_mr_len = 0;
-	ret = walk_system_ram_range(0, 1ULL << MAX_PHYSMEM_BITS, NULL,
-				   ehea_create_busmap_callback);
-	mutex_unlock(&ehea_busmap_mutex);
-	return ret;
-}
-
-void ehea_destroy_busmap(void)
-{
-	int top, dir;
-	mutex_lock(&ehea_busmap_mutex);
-	if (!ehea_bmap)
-		goto out_destroy;
-
-	for (top = 0; top < EHEA_MAP_ENTRIES; top++) {
-		if (!ehea_bmap->top[top])
-			continue;
-
-		for (dir = 0; dir < EHEA_MAP_ENTRIES; dir++) {
-			if (!ehea_bmap->top[top]->dir[dir])
-				continue;
-
-			kfree(ehea_bmap->top[top]->dir[dir]);
-		}
-
-		kfree(ehea_bmap->top[top]);
-	}
-
-	kfree(ehea_bmap);
-	ehea_bmap = NULL;
-out_destroy:
-	mutex_unlock(&ehea_busmap_mutex);
-}
-
-u64 ehea_map_vaddr(void *caddr)
-{
-	int top, dir, idx;
-	unsigned long index, offset;
-
-	if (!ehea_bmap)
-		return EHEA_INVAL_ADDR;
-
-	index = __pa(caddr) >> SECTION_SIZE_BITS;
-	top = (index >> EHEA_TOP_INDEX_SHIFT) & EHEA_INDEX_MASK;
-	if (!ehea_bmap->top[top])
-		return EHEA_INVAL_ADDR;
-
-	dir = (index >> EHEA_DIR_INDEX_SHIFT) & EHEA_INDEX_MASK;
-	if (!ehea_bmap->top[top]->dir[dir])
-		return EHEA_INVAL_ADDR;
-
-	idx = index & EHEA_INDEX_MASK;
-	if (!ehea_bmap->top[top]->dir[dir]->ent[idx])
-		return EHEA_INVAL_ADDR;
-
-	offset = (unsigned long)caddr & (EHEA_SECTSIZE - 1);
-	return ehea_bmap->top[top]->dir[dir]->ent[idx] | offset;
-}
-
-static inline void *ehea_calc_sectbase(int top, int dir, int idx)
-{
-	unsigned long ret = idx;
-	ret |= dir << EHEA_DIR_INDEX_SHIFT;
-	ret |= top << EHEA_TOP_INDEX_SHIFT;
-	return __va(ret << SECTION_SIZE_BITS);
-}
-
-static u64 ehea_reg_mr_section(int top, int dir, int idx, u64 *pt,
-			       struct ehea_adapter *adapter,
-			       struct ehea_mr *mr)
-{
-	void *pg;
-	u64 j, m, hret;
-	unsigned long k = 0;
-	u64 pt_abs = __pa(pt);
-
-	void *sectbase = ehea_calc_sectbase(top, dir, idx);
-
-	for (j = 0; j < (EHEA_PAGES_PER_SECTION / EHEA_MAX_RPAGE); j++) {
-
-		for (m = 0; m < EHEA_MAX_RPAGE; m++) {
-			pg = sectbase + ((k++) * EHEA_PAGESIZE);
-			pt[m] = __pa(pg);
-		}
-		hret = ehea_h_register_rpage_mr(adapter->handle, mr->handle, 0,
-						0, pt_abs, EHEA_MAX_RPAGE);
-
-		if ((hret != H_SUCCESS) &&
-		    (hret != H_PAGE_REGISTERED)) {
-			ehea_h_free_resource(adapter->handle, mr->handle,
-					     FORCE_FREE);
-			pr_err("register_rpage_mr failed\n");
-			return hret;
-		}
-	}
-	return hret;
-}
-
-static u64 ehea_reg_mr_sections(int top, int dir, u64 *pt,
-				struct ehea_adapter *adapter,
-				struct ehea_mr *mr)
-{
-	u64 hret = H_SUCCESS;
-	int idx;
-
-	for (idx = 0; idx < EHEA_MAP_ENTRIES; idx++) {
-		if (!ehea_bmap->top[top]->dir[dir]->ent[idx])
-			continue;
-
-		hret = ehea_reg_mr_section(top, dir, idx, pt, adapter, mr);
-		if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED))
-			return hret;
-	}
-	return hret;
-}
-
-static u64 ehea_reg_mr_dir_sections(int top, u64 *pt,
-				    struct ehea_adapter *adapter,
-				    struct ehea_mr *mr)
-{
-	u64 hret = H_SUCCESS;
-	int dir;
-
-	for (dir = 0; dir < EHEA_MAP_ENTRIES; dir++) {
-		if (!ehea_bmap->top[top]->dir[dir])
-			continue;
-
-		hret = ehea_reg_mr_sections(top, dir, pt, adapter, mr);
-		if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED))
-			return hret;
-	}
-	return hret;
-}
-
-int ehea_reg_kernel_mr(struct ehea_adapter *adapter, struct ehea_mr *mr)
-{
-	int ret;
-	u64 *pt;
-	u64 hret;
-	u32 acc_ctrl = EHEA_MR_ACC_CTRL;
-
-	unsigned long top;
-
-	pt = (void *)get_zeroed_page(GFP_KERNEL);
-	if (!pt) {
-		pr_err("no mem\n");
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	hret = ehea_h_alloc_resource_mr(adapter->handle, EHEA_BUSMAP_START,
-					ehea_mr_len, acc_ctrl, adapter->pd,
-					&mr->handle, &mr->lkey);
-
-	if (hret != H_SUCCESS) {
-		pr_err("alloc_resource_mr failed\n");
-		ret = -EIO;
-		goto out;
-	}
-
-	if (!ehea_bmap) {
-		ehea_h_free_resource(adapter->handle, mr->handle, FORCE_FREE);
-		pr_err("no busmap available\n");
-		ret = -EIO;
-		goto out;
-	}
-
-	for (top = 0; top < EHEA_MAP_ENTRIES; top++) {
-		if (!ehea_bmap->top[top])
-			continue;
-
-		hret = ehea_reg_mr_dir_sections(top, pt, adapter, mr);
-		if((hret != H_PAGE_REGISTERED) && (hret != H_SUCCESS))
-			break;
-	}
-
-	if (hret != H_SUCCESS) {
-		ehea_h_free_resource(adapter->handle, mr->handle, FORCE_FREE);
-		pr_err("registering mr failed\n");
-		ret = -EIO;
-		goto out;
-	}
-
-	mr->vaddr = EHEA_BUSMAP_START;
-	mr->adapter = adapter;
-	ret = 0;
-out:
-	free_page((unsigned long)pt);
-	return ret;
-}
-
-int ehea_rem_mr(struct ehea_mr *mr)
-{
-	u64 hret;
-
-	if (!mr || !mr->adapter)
-		return -EINVAL;
-
-	hret = ehea_h_free_resource(mr->adapter->handle, mr->handle,
-				    FORCE_FREE);
-	if (hret != H_SUCCESS) {
-		pr_err("destroy MR failed\n");
-		return -EIO;
-	}
-
-	return 0;
-}
-
-int ehea_gen_smr(struct ehea_adapter *adapter, struct ehea_mr *old_mr,
-		 struct ehea_mr *shared_mr)
-{
-	u64 hret;
-
-	hret = ehea_h_register_smr(adapter->handle, old_mr->handle,
-				   old_mr->vaddr, EHEA_MR_ACC_CTRL,
-				   adapter->pd, shared_mr);
-	if (hret != H_SUCCESS)
-		return -EIO;
-
-	shared_mr->adapter = adapter;
-
-	return 0;
-}
-
-static void print_error_data(u64 *data)
-{
-	int length;
-	u64 type = EHEA_BMASK_GET(ERROR_DATA_TYPE, data[2]);
-	u64 resource = data[1];
-
-	length = EHEA_BMASK_GET(ERROR_DATA_LENGTH, data[0]);
-
-	if (length > EHEA_PAGESIZE)
-		length = EHEA_PAGESIZE;
-
-	if (type == EHEA_AER_RESTYPE_QP)
-		pr_err("QP (resource=%llX) state: AER=0x%llX, AERR=0x%llX, port=%llX\n",
-		       resource, data[6], data[12], data[22]);
-	else if (type == EHEA_AER_RESTYPE_CQ)
-		pr_err("CQ (resource=%llX) state: AER=0x%llX\n",
-		       resource, data[6]);
-	else if (type == EHEA_AER_RESTYPE_EQ)
-		pr_err("EQ (resource=%llX) state: AER=0x%llX\n",
-		       resource, data[6]);
-
-	ehea_dump(data, length, "error data");
-}
-
-u64 ehea_error_data(struct ehea_adapter *adapter, u64 res_handle,
-		    u64 *aer, u64 *aerr)
-{
-	unsigned long ret;
-	u64 *rblock;
-	u64 type = 0;
-
-	rblock = (void *)get_zeroed_page(GFP_KERNEL);
-	if (!rblock) {
-		pr_err("Cannot allocate rblock memory\n");
-		goto out;
-	}
-
-	ret = ehea_h_error_data(adapter->handle, res_handle, rblock);
-
-	if (ret == H_SUCCESS) {
-		type = EHEA_BMASK_GET(ERROR_DATA_TYPE, rblock[2]);
-		*aer = rblock[6];
-		*aerr = rblock[12];
-		print_error_data(rblock);
-	} else if (ret == H_R_STATE) {
-		pr_err("No error data available: %llX\n", res_handle);
-	} else
-		pr_err("Error data could not be fetched: %llX\n", res_handle);
-
-	free_page((unsigned long)rblock);
-out:
-	return type;
-}
diff --git a/drivers/net/ethernet/ibm/ehea/ehea_qmr.h b/drivers/net/ethernet/ibm/ehea/ehea_qmr.h
deleted file mode 100644
index 7c7cccd820f79..0000000000000
--- a/drivers/net/ethernet/ibm/ehea/ehea_qmr.h
+++ /dev/null
@@ -1,390 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- *  linux/drivers/net/ethernet/ibm/ehea/ehea_qmr.h
- *
- *  eHEA ethernet device driver for IBM eServer System p
- *
- *  (C) Copyright IBM Corp. 2006
- *
- *  Authors:
- *       Christoph Raisch <raisch@de.ibm.com>
- *       Jan-Bernd Themann <themann@de.ibm.com>
- *       Thomas Klein <tklein@de.ibm.com>
- */
-
-#ifndef __EHEA_QMR_H__
-#define __EHEA_QMR_H__
-
-#include <linux/prefetch.h>
-#include "ehea.h"
-#include "ehea_hw.h"
-
-/*
- * page size of ehea hardware queues
- */
-
-#define EHEA_PAGESHIFT         12
-#define EHEA_PAGESIZE          (1UL << EHEA_PAGESHIFT)
-#define EHEA_SECTSIZE          (1UL << 24)
-#define EHEA_PAGES_PER_SECTION (EHEA_SECTSIZE >> EHEA_PAGESHIFT)
-#define EHEA_HUGEPAGESHIFT     34
-#define EHEA_HUGEPAGE_SIZE     (1UL << EHEA_HUGEPAGESHIFT)
-#define EHEA_HUGEPAGE_PFN_MASK ((EHEA_HUGEPAGE_SIZE - 1) >> PAGE_SHIFT)
-
-#if ((1UL << SECTION_SIZE_BITS) < EHEA_SECTSIZE)
-#error eHEA module cannot work if kernel sectionsize < ehea sectionsize
-#endif
-
-/* Some abbreviations used here:
- *
- * WQE  - Work Queue Entry
- * SWQE - Send Work Queue Entry
- * RWQE - Receive Work Queue Entry
- * CQE  - Completion Queue Entry
- * EQE  - Event Queue Entry
- * MR   - Memory Region
- */
-
-/* Use of WR_ID field for EHEA */
-#define EHEA_WR_ID_COUNT   EHEA_BMASK_IBM(0, 19)
-#define EHEA_WR_ID_TYPE    EHEA_BMASK_IBM(20, 23)
-#define EHEA_SWQE2_TYPE    0x1
-#define EHEA_SWQE3_TYPE    0x2
-#define EHEA_RWQE2_TYPE    0x3
-#define EHEA_RWQE3_TYPE    0x4
-#define EHEA_WR_ID_INDEX   EHEA_BMASK_IBM(24, 47)
-#define EHEA_WR_ID_REFILL  EHEA_BMASK_IBM(48, 63)
-
-struct ehea_vsgentry {
-	u64 vaddr;
-	u32 l_key;
-	u32 len;
-};
-
-/* maximum number of sg entries allowed in a WQE */
-#define EHEA_MAX_WQE_SG_ENTRIES  	252
-#define SWQE2_MAX_IMM            	(0xD0 - 0x30)
-#define SWQE3_MAX_IMM            	224
-
-/* tx control flags for swqe */
-#define EHEA_SWQE_CRC                   0x8000
-#define EHEA_SWQE_IP_CHECKSUM           0x4000
-#define EHEA_SWQE_TCP_CHECKSUM          0x2000
-#define EHEA_SWQE_TSO                   0x1000
-#define EHEA_SWQE_SIGNALLED_COMPLETION  0x0800
-#define EHEA_SWQE_VLAN_INSERT           0x0400
-#define EHEA_SWQE_IMM_DATA_PRESENT      0x0200
-#define EHEA_SWQE_DESCRIPTORS_PRESENT   0x0100
-#define EHEA_SWQE_WRAP_CTL_REC          0x0080
-#define EHEA_SWQE_WRAP_CTL_FORCE        0x0040
-#define EHEA_SWQE_BIND                  0x0020
-#define EHEA_SWQE_PURGE                 0x0010
-
-/* sizeof(struct ehea_swqe) less the union */
-#define SWQE_HEADER_SIZE		32
-
-struct ehea_swqe {
-	u64 wr_id;
-	u16 tx_control;
-	u16 vlan_tag;
-	u8 reserved1;
-	u8 ip_start;
-	u8 ip_end;
-	u8 immediate_data_length;
-	u8 tcp_offset;
-	u8 reserved2;
-	u16 reserved2b;
-	u8 wrap_tag;
-	u8 descriptors;		/* number of valid descriptors in WQE */
-	u16 reserved3;
-	u16 reserved4;
-	u16 mss;
-	u32 reserved5;
-	union {
-		/*  Send WQE Format 1 */
-		struct {
-			struct ehea_vsgentry sg_list[EHEA_MAX_WQE_SG_ENTRIES];
-		} no_immediate_data;
-
-		/*  Send WQE Format 2 */
-		struct {
-			struct ehea_vsgentry sg_entry;
-			/* 0x30 */
-			u8 immediate_data[SWQE2_MAX_IMM];
-			/* 0xd0 */
-			struct ehea_vsgentry sg_list[EHEA_MAX_WQE_SG_ENTRIES-1];
-		} immdata_desc __packed;
-
-		/*  Send WQE Format 3 */
-		struct {
-			u8 immediate_data[SWQE3_MAX_IMM];
-		} immdata_nodesc;
-	} u;
-};
-
-struct ehea_rwqe {
-	u64 wr_id;		/* work request ID */
-	u8 reserved1[5];
-	u8 data_segments;
-	u16 reserved2;
-	u64 reserved3;
-	u64 reserved4;
-	struct ehea_vsgentry sg_list[EHEA_MAX_WQE_SG_ENTRIES];
-};
-
-#define EHEA_CQE_VLAN_TAG_XTRACT   0x0400
-
-#define EHEA_CQE_TYPE_RQ           0x60
-#define EHEA_CQE_STAT_ERR_MASK     0x700F
-#define EHEA_CQE_STAT_FAT_ERR_MASK 0xF
-#define EHEA_CQE_BLIND_CKSUM       0x8000
-#define EHEA_CQE_STAT_ERR_TCP      0x4000
-#define EHEA_CQE_STAT_ERR_IP       0x2000
-#define EHEA_CQE_STAT_ERR_CRC      0x1000
-
-/* Defines which bad send cqe stati lead to a port reset */
-#define EHEA_CQE_STAT_RESET_MASK   0x0002
-
-struct ehea_cqe {
-	u64 wr_id;		/* work request ID from WQE */
-	u8 type;
-	u8 valid;
-	u16 status;
-	u16 reserved1;
-	u16 num_bytes_transfered;
-	u16 vlan_tag;
-	u16 inet_checksum_value;
-	u8 reserved2;
-	u8 header_length;
-	u16 reserved3;
-	u16 page_offset;
-	u16 wqe_count;
-	u32 qp_token;
-	u32 timestamp;
-	u32 reserved4;
-	u64 reserved5[3];
-};
-
-#define EHEA_EQE_VALID           EHEA_BMASK_IBM(0, 0)
-#define EHEA_EQE_IS_CQE          EHEA_BMASK_IBM(1, 1)
-#define EHEA_EQE_IDENTIFIER      EHEA_BMASK_IBM(2, 7)
-#define EHEA_EQE_QP_CQ_NUMBER    EHEA_BMASK_IBM(8, 31)
-#define EHEA_EQE_QP_TOKEN        EHEA_BMASK_IBM(32, 63)
-#define EHEA_EQE_CQ_TOKEN        EHEA_BMASK_IBM(32, 63)
-#define EHEA_EQE_KEY             EHEA_BMASK_IBM(32, 63)
-#define EHEA_EQE_PORT_NUMBER     EHEA_BMASK_IBM(56, 63)
-#define EHEA_EQE_EQ_NUMBER       EHEA_BMASK_IBM(48, 63)
-#define EHEA_EQE_SM_ID           EHEA_BMASK_IBM(48, 63)
-#define EHEA_EQE_SM_MECH_NUMBER  EHEA_BMASK_IBM(48, 55)
-#define EHEA_EQE_SM_PORT_NUMBER  EHEA_BMASK_IBM(56, 63)
-
-#define EHEA_AER_RESTYPE_QP  0x8
-#define EHEA_AER_RESTYPE_CQ  0x4
-#define EHEA_AER_RESTYPE_EQ  0x3
-
-/* Defines which affiliated errors lead to a port reset */
-#define EHEA_AER_RESET_MASK   0xFFFFFFFFFEFFFFFFULL
-#define EHEA_AERR_RESET_MASK  0xFFFFFFFFFFFFFFFFULL
-
-struct ehea_eqe {
-	u64 entry;
-};
-
-#define ERROR_DATA_LENGTH  EHEA_BMASK_IBM(52, 63)
-#define ERROR_DATA_TYPE    EHEA_BMASK_IBM(0, 7)
-
-static inline void *hw_qeit_calc(struct hw_queue *queue, u64 q_offset)
-{
-	struct ehea_page *current_page;
-
-	if (q_offset >= queue->queue_length)
-		q_offset -= queue->queue_length;
-	current_page = (queue->queue_pages)[q_offset >> EHEA_PAGESHIFT];
-	return &current_page->entries[q_offset & (EHEA_PAGESIZE - 1)];
-}
-
-static inline void *hw_qeit_get(struct hw_queue *queue)
-{
-	return hw_qeit_calc(queue, queue->current_q_offset);
-}
-
-static inline void hw_qeit_inc(struct hw_queue *queue)
-{
-	queue->current_q_offset += queue->qe_size;
-	if (queue->current_q_offset >= queue->queue_length) {
-		queue->current_q_offset = 0;
-		/* toggle the valid flag */
-		queue->toggle_state = (~queue->toggle_state) & 1;
-	}
-}
-
-static inline void *hw_qeit_get_inc(struct hw_queue *queue)
-{
-	void *retvalue = hw_qeit_get(queue);
-	hw_qeit_inc(queue);
-	return retvalue;
-}
-
-static inline void *hw_qeit_get_inc_valid(struct hw_queue *queue)
-{
-	struct ehea_cqe *retvalue = hw_qeit_get(queue);
-	u8 valid = retvalue->valid;
-	void *pref;
-
-	if ((valid >> 7) == (queue->toggle_state & 1)) {
-		/* this is a good one */
-		hw_qeit_inc(queue);
-		pref = hw_qeit_calc(queue, queue->current_q_offset);
-		prefetch(pref);
-		prefetch(pref + 128);
-	} else
-		retvalue = NULL;
-	return retvalue;
-}
-
-static inline void *hw_qeit_get_valid(struct hw_queue *queue)
-{
-	struct ehea_cqe *retvalue = hw_qeit_get(queue);
-	void *pref;
-	u8 valid;
-
-	pref = hw_qeit_calc(queue, queue->current_q_offset);
-	prefetch(pref);
-	prefetch(pref + 128);
-	prefetch(pref + 256);
-	valid = retvalue->valid;
-	if (!((valid >> 7) == (queue->toggle_state & 1)))
-		retvalue = NULL;
-	return retvalue;
-}
-
-static inline void *hw_qeit_reset(struct hw_queue *queue)
-{
-	queue->current_q_offset = 0;
-	return hw_qeit_get(queue);
-}
-
-static inline void *hw_qeit_eq_get_inc(struct hw_queue *queue)
-{
-	u64 last_entry_in_q = queue->queue_length - queue->qe_size;
-	void *retvalue;
-
-	retvalue = hw_qeit_get(queue);
-	queue->current_q_offset += queue->qe_size;
-	if (queue->current_q_offset > last_entry_in_q) {
-		queue->current_q_offset = 0;
-		queue->toggle_state = (~queue->toggle_state) & 1;
-	}
-	return retvalue;
-}
-
-static inline void *hw_eqit_eq_get_inc_valid(struct hw_queue *queue)
-{
-	void *retvalue = hw_qeit_get(queue);
-	u32 qe = *(u8 *)retvalue;
-	if ((qe >> 7) == (queue->toggle_state & 1))
-		hw_qeit_eq_get_inc(queue);
-	else
-		retvalue = NULL;
-	return retvalue;
-}
-
-static inline struct ehea_rwqe *ehea_get_next_rwqe(struct ehea_qp *qp,
-						   int rq_nr)
-{
-	struct hw_queue *queue;
-
-	if (rq_nr == 1)
-		queue = &qp->hw_rqueue1;
-	else if (rq_nr == 2)
-		queue = &qp->hw_rqueue2;
-	else
-		queue = &qp->hw_rqueue3;
-
-	return hw_qeit_get_inc(queue);
-}
-
-static inline struct ehea_swqe *ehea_get_swqe(struct ehea_qp *my_qp,
-					      int *wqe_index)
-{
-	struct hw_queue *queue = &my_qp->hw_squeue;
-	struct ehea_swqe *wqe_p;
-
-	*wqe_index = (queue->current_q_offset) >> (7 + EHEA_SG_SQ);
-	wqe_p = hw_qeit_get_inc(&my_qp->hw_squeue);
-
-	return wqe_p;
-}
-
-static inline void ehea_post_swqe(struct ehea_qp *my_qp, struct ehea_swqe *swqe)
-{
-	iosync();
-	ehea_update_sqa(my_qp, 1);
-}
-
-static inline struct ehea_cqe *ehea_poll_rq1(struct ehea_qp *qp, int *wqe_index)
-{
-	struct hw_queue *queue = &qp->hw_rqueue1;
-
-	*wqe_index = (queue->current_q_offset) >> (7 + EHEA_SG_RQ1);
-	return hw_qeit_get_valid(queue);
-}
-
-static inline void ehea_inc_cq(struct ehea_cq *cq)
-{
-	hw_qeit_inc(&cq->hw_queue);
-}
-
-static inline void ehea_inc_rq1(struct ehea_qp *qp)
-{
-	hw_qeit_inc(&qp->hw_rqueue1);
-}
-
-static inline struct ehea_cqe *ehea_poll_cq(struct ehea_cq *my_cq)
-{
-	return hw_qeit_get_valid(&my_cq->hw_queue);
-}
-
-#define EHEA_CQ_REGISTER_ORIG 0
-#define EHEA_EQ_REGISTER_ORIG 0
-
-enum ehea_eq_type {
-	EHEA_EQ = 0,		/* event queue              */
-	EHEA_NEQ		/* notification event queue */
-};
-
-struct ehea_eq *ehea_create_eq(struct ehea_adapter *adapter,
-			       enum ehea_eq_type type,
-			       const u32 length, const u8 eqe_gen);
-
-int ehea_destroy_eq(struct ehea_eq *eq);
-
-struct ehea_eqe *ehea_poll_eq(struct ehea_eq *eq);
-
-struct ehea_cq *ehea_create_cq(struct ehea_adapter *adapter, int cqe,
-			       u64 eq_handle, u32 cq_token);
-
-int ehea_destroy_cq(struct ehea_cq *cq);
-
-struct ehea_qp *ehea_create_qp(struct ehea_adapter *adapter, u32 pd,
-			       struct ehea_qp_init_attr *init_attr);
-
-int ehea_destroy_qp(struct ehea_qp *qp);
-
-int ehea_reg_kernel_mr(struct ehea_adapter *adapter, struct ehea_mr *mr);
-
-int ehea_gen_smr(struct ehea_adapter *adapter, struct ehea_mr *old_mr,
-		 struct ehea_mr *shared_mr);
-
-int ehea_rem_mr(struct ehea_mr *mr);
-
-u64 ehea_error_data(struct ehea_adapter *adapter, u64 res_handle,
-		    u64 *aer, u64 *aerr);
-
-int ehea_add_sect_bmap(unsigned long pfn, unsigned long nr_pages);
-int ehea_rem_sect_bmap(unsigned long pfn, unsigned long nr_pages);
-int ehea_create_busmap(void);
-void ehea_destroy_busmap(void);
-u64 ehea_map_vaddr(void *caddr);
-
-#endif	/* __EHEA_QMR_H__ */
diff --git a/drivers/net/ethernet/intel/ice/ice_gnss.c b/drivers/net/ethernet/intel/ice/ice_gnss.c
index 8fd954f1ebd67..7d21c3417b0b2 100644
--- a/drivers/net/ethernet/intel/ice/ice_gnss.c
+++ b/drivers/net/ethernet/intel/ice/ice_gnss.c
@@ -2,6 +2,7 @@
 /* Copyright (C) 2021-2022, Intel Corporation. */
 
 #include "ice.h"
+#include <linux/slab.h>
 #include "ice_lib.h"
 
 /**
@@ -124,7 +125,7 @@ static void ice_gnss_read(struct kthread_work *work)
 
 	data_len = min_t(typeof(data_len), data_len, PAGE_SIZE);
 
-	buf = (char *)get_zeroed_page(GFP_KERNEL);
+	buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!buf) {
 		err = -ENOMEM;
 		goto requeue;
@@ -151,7 +152,7 @@ static void ice_gnss_read(struct kthread_work *work)
 			 count, i);
 	delay = ICE_GNSS_TIMER_DELAY_TIME;
 free_buf:
-	free_page((unsigned long)buf);
+	kfree(buf);
 requeue:
 	kthread_queue_delayed_work(gnss->kworker, &gnss->read_work, delay);
 	if (err)
diff --git a/drivers/net/ethernet/intel/igb/e1000_82575.h b/drivers/net/ethernet/intel/igb/e1000_82575.h
index 63ec253ac7885..9e696d55e5128 100644
--- a/drivers/net/ethernet/intel/igb/e1000_82575.h
+++ b/drivers/net/ethernet/intel/igb/e1000_82575.h
@@ -87,6 +87,27 @@ union e1000_adv_rx_desc {
 	} wb;  /* writeback */
 };
 
+#define E1000_RSS_TYPE_NO_HASH		 0
+#define E1000_RSS_TYPE_HASH_TCP_IPV4	 1
+#define E1000_RSS_TYPE_HASH_IPV4	 2
+#define E1000_RSS_TYPE_HASH_TCP_IPV6	 3
+#define E1000_RSS_TYPE_HASH_IPV6_EX	 4
+#define E1000_RSS_TYPE_HASH_IPV6	 5
+#define E1000_RSS_TYPE_HASH_TCP_IPV6_EX	 6
+#define E1000_RSS_TYPE_HASH_UDP_IPV4	 7
+#define E1000_RSS_TYPE_HASH_UDP_IPV6	 8
+#define E1000_RSS_TYPE_HASH_UDP_IPV6_EX	 9
+
+#define E1000_RSS_TYPE_MASK		GENMASK(3, 0)
+
+#define E1000_RSS_L4_TYPES_MASK	\
+	(BIT(E1000_RSS_TYPE_HASH_TCP_IPV4)	| \
+	 BIT(E1000_RSS_TYPE_HASH_TCP_IPV6)	| \
+	 BIT(E1000_RSS_TYPE_HASH_TCP_IPV6_EX)	| \
+	 BIT(E1000_RSS_TYPE_HASH_UDP_IPV4)	| \
+	 BIT(E1000_RSS_TYPE_HASH_UDP_IPV6)	| \
+	 BIT(E1000_RSS_TYPE_HASH_UDP_IPV6_EX))
+
 #define E1000_RXDADV_HDRBUFLEN_MASK      0x7FE0
 #define E1000_RXDADV_HDRBUFLEN_SHIFT     5
 #define E1000_RXDADV_STAT_TS             0x10000 /* Pkt was time stamped */
diff --git a/drivers/net/ethernet/intel/igb/igb.h b/drivers/net/ethernet/intel/igb/igb.h
index 0fff1df81b7bd..8c9b02058cecb 100644
--- a/drivers/net/ethernet/intel/igb/igb.h
+++ b/drivers/net/ethernet/intel/igb/igb.h
@@ -495,6 +495,7 @@ struct hwmon_buff {
 #define IGB_N_PEROUT	2
 #define IGB_N_SDP	4
 #define IGB_RETA_SIZE	128
+#define IGB_RSS_KEY_SIZE	40
 
 enum igb_filter_match_flags {
 	IGB_FILTER_FLAG_ETHER_TYPE = 0x1,
@@ -655,6 +656,7 @@ struct igb_adapter {
 	struct i2c_client *i2c_client;
 	u32 rss_indir_tbl_init;
 	u8 rss_indir_tbl[IGB_RETA_SIZE];
+	u8 rss_key[IGB_RSS_KEY_SIZE];
 
 	unsigned long link_check_timeout;
 	int copper_tries;
@@ -735,6 +737,7 @@ void igb_down(struct igb_adapter *);
 void igb_reinit_locked(struct igb_adapter *);
 void igb_reset(struct igb_adapter *);
 int igb_reinit_queues(struct igb_adapter *);
+void igb_write_rss_key(struct igb_adapter *adapter);
 void igb_write_rss_indir_tbl(struct igb_adapter *);
 int igb_set_spd_dplx(struct igb_adapter *, u32, u8);
 int igb_setup_tx_resources(struct igb_ring *);
diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c
index f7938c1da835f..65014a54a6d11 100644
--- a/drivers/net/ethernet/intel/igb/igb_ethtool.c
+++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c
@@ -3019,6 +3019,27 @@ static int igb_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd)
 	return ret;
 }
 
+/**
+ * igb_write_rss_key - Program the RSS key into device registers
+ * @adapter: board private structure
+ *
+ * Write the RSS key stored in adapter->rss_key to the E1000 hardware registers.
+ * Each 32-bit chunk of the key is read using get_unaligned_le32() and written
+ * to the appropriate register.
+ */
+void igb_write_rss_key(struct igb_adapter *adapter)
+{
+	struct e1000_hw *hw = &adapter->hw;
+
+	ASSERT_RTNL();
+
+	for (int i = 0; i < IGB_RSS_KEY_SIZE / 4; i++) {
+		u32 val = get_unaligned_le32(&adapter->rss_key[i * 4]);
+
+		wr32(E1000_RSSRK(i), val);
+	}
+}
+
 static int igb_get_eee(struct net_device *netdev, struct ethtool_keee *edata)
 {
 	struct igb_adapter *adapter = netdev_priv(netdev);
@@ -3276,10 +3297,12 @@ static int igb_get_rxfh(struct net_device *netdev,
 	int i;
 
 	rxfh->hfunc = ETH_RSS_HASH_TOP;
-	if (!rxfh->indir)
-		return 0;
-	for (i = 0; i < IGB_RETA_SIZE; i++)
-		rxfh->indir[i] = adapter->rss_indir_tbl[i];
+	if (rxfh->indir)
+		for (i = 0; i < IGB_RETA_SIZE; i++)
+			rxfh->indir[i] = adapter->rss_indir_tbl[i];
+
+	if (rxfh->key)
+		memcpy(rxfh->key, adapter->rss_key, sizeof(adapter->rss_key));
 
 	return 0;
 }
@@ -3319,6 +3342,11 @@ void igb_write_rss_indir_tbl(struct igb_adapter *adapter)
 	}
 }
 
+static u32 igb_get_rxfh_key_size(struct net_device *netdev)
+{
+	return IGB_RSS_KEY_SIZE;
+}
+
 static int igb_set_rxfh(struct net_device *netdev,
 			struct ethtool_rxfh_param *rxfh,
 			struct netlink_ext_ack *extack)
@@ -3329,35 +3357,39 @@ static int igb_set_rxfh(struct net_device *netdev,
 	u32 num_queues;
 
 	/* We do not allow change in unsupported parameters */
-	if (rxfh->key ||
-	    (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE &&
-	     rxfh->hfunc != ETH_RSS_HASH_TOP))
+	if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE &&
+	    rxfh->hfunc != ETH_RSS_HASH_TOP)
 		return -EOPNOTSUPP;
-	if (!rxfh->indir)
-		return 0;
 
-	num_queues = adapter->rss_queues;
+	if (rxfh->indir) {
+		num_queues = adapter->rss_queues;
 
-	switch (hw->mac.type) {
-	case e1000_82576:
-		/* 82576 supports 2 RSS queues for SR-IOV */
-		if (adapter->vfs_allocated_count)
-			num_queues = 2;
-		break;
-	default:
-		break;
-	}
+		switch (hw->mac.type) {
+		case e1000_82576:
+			/* 82576 supports 2 RSS queues for SR-IOV */
+			if (adapter->vfs_allocated_count)
+				num_queues = 2;
+			break;
+		default:
+			break;
+		}
+
+		/* Verify user input. */
+		for (i = 0; i < IGB_RETA_SIZE; i++)
+			if (rxfh->indir[i] >= num_queues)
+				return -EINVAL;
 
-	/* Verify user input. */
-	for (i = 0; i < IGB_RETA_SIZE; i++)
-		if (rxfh->indir[i] >= num_queues)
-			return -EINVAL;
 
+		for (i = 0; i < IGB_RETA_SIZE; i++)
+			adapter->rss_indir_tbl[i] = rxfh->indir[i];
 
-	for (i = 0; i < IGB_RETA_SIZE; i++)
-		adapter->rss_indir_tbl[i] = rxfh->indir[i];
+		igb_write_rss_indir_tbl(adapter);
+	}
 
-	igb_write_rss_indir_tbl(adapter);
+	if (rxfh->key) {
+		memcpy(adapter->rss_key, rxfh->key, sizeof(adapter->rss_key));
+		igb_write_rss_key(adapter);
+	}
 
 	return 0;
 }
@@ -3483,6 +3515,7 @@ static const struct ethtool_ops igb_ethtool_ops = {
 	.get_module_eeprom	= igb_get_module_eeprom,
 	.get_rxfh_indir_size	= igb_get_rxfh_indir_size,
 	.get_rxfh		= igb_get_rxfh,
+	.get_rxfh_key_size	= igb_get_rxfh_key_size,
 	.set_rxfh		= igb_set_rxfh,
 	.get_rxfh_fields	= igb_get_rxfh_fields,
 	.set_rxfh_fields	= igb_set_rxfh_fields,
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index a1e89a375744c..d4a897a8c82c6 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -4048,6 +4048,9 @@ static int igb_sw_init(struct igb_adapter *adapter)
 
 	pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
 
+	/* init RSS key */
+	netdev_rss_key_fill(adapter->rss_key, sizeof(adapter->rss_key));
+
 	/* set default ring sizes */
 	adapter->tx_ring_count = IGB_DEFAULT_TXD;
 	adapter->rx_ring_count = IGB_DEFAULT_RXD;
@@ -4522,11 +4525,8 @@ static void igb_setup_mrqc(struct igb_adapter *adapter)
 	struct e1000_hw *hw = &adapter->hw;
 	u32 mrqc, rxcsum;
 	u32 j, num_rx_queues;
-	u32 rss_key[10];
 
-	netdev_rss_key_fill(rss_key, sizeof(rss_key));
-	for (j = 0; j < 10; j++)
-		wr32(E1000_RSSRK(j), rss_key[j]);
+	igb_write_rss_key(adapter);
 
 	num_rx_queues = adapter->rss_queues;
 
@@ -8820,10 +8820,19 @@ static inline void igb_rx_hash(struct igb_ring *ring,
 			       union e1000_adv_rx_desc *rx_desc,
 			       struct sk_buff *skb)
 {
-	if (ring->netdev->features & NETIF_F_RXHASH)
-		skb_set_hash(skb,
-			     le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
-			     PKT_HASH_TYPE_L3);
+	u16 rss_type;
+
+	if (!(ring->netdev->features & NETIF_F_RXHASH))
+		return;
+
+	rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.pkt_info) &
+		   E1000_RSS_TYPE_MASK;
+	if (!rss_type)
+		return;
+
+	skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
+		     (E1000_RSS_L4_TYPES_MASK & BIT(rss_type)) ?
+		     PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
 }
 
 /**
diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h
index 46d625b15f448..17f213cc93e45 100644
--- a/drivers/net/ethernet/intel/igc/igc.h
+++ b/drivers/net/ethernet/intel/igc/igc.h
@@ -30,6 +30,7 @@ void igc_ethtool_set_ops(struct net_device *);
 
 #define MAX_ETYPE_FILTER		8
 #define IGC_RETA_SIZE			128
+#define IGC_RSS_KEY_SIZE		40
 
 /* SDP support */
 #define IGC_N_EXTTS	2
@@ -302,6 +303,7 @@ struct igc_adapter {
 	unsigned int nfc_rule_count;
 
 	u8 rss_indir_tbl[IGC_RETA_SIZE];
+	u8 rss_key[IGC_RSS_KEY_SIZE];
 
 	unsigned long link_check_timeout;
 	struct igc_info ei;
@@ -361,6 +363,7 @@ unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter);
 void igc_set_flag_queue_pairs(struct igc_adapter *adapter,
 			      const u32 max_rss_queues);
 int igc_reinit_queues(struct igc_adapter *adapter);
+void igc_write_rss_key(struct igc_adapter *adapter);
 void igc_write_rss_indir_tbl(struct igc_adapter *adapter);
 bool igc_has_link(struct igc_adapter *adapter);
 void igc_reset(struct igc_adapter *adapter);
diff --git a/drivers/net/ethernet/intel/igc/igc_base.c b/drivers/net/ethernet/intel/igc/igc_base.c
index 1613b562d17c5..ab9120a3127f2 100644
--- a/drivers/net/ethernet/intel/igc/igc_base.c
+++ b/drivers/net/ethernet/intel/igc/igc_base.c
@@ -114,11 +114,35 @@ static s32 igc_setup_copper_link_base(struct igc_hw *hw)
 	u32 ctrl;
 
 	ctrl = rd32(IGC_CTRL);
-	ctrl |= IGC_CTRL_SLU;
-	ctrl &= ~(IGC_CTRL_FRCSPD | IGC_CTRL_FRCDPX);
-	wr32(IGC_CTRL, ctrl);
-
-	ret_val = igc_setup_copper_link(hw);
+	ctrl &= ~(IGC_CTRL_FRCSPD | IGC_CTRL_FRCDPX |
+		  IGC_CTRL_SPEED_MASK | IGC_CTRL_FD);
+
+	if (hw->mac.autoneg_enabled) {
+		ctrl |= IGC_CTRL_SLU;
+		wr32(IGC_CTRL, ctrl);
+		ret_val = igc_setup_copper_link(hw);
+	} else {
+		ctrl |= IGC_CTRL_SLU | IGC_CTRL_FRCSPD | IGC_CTRL_FRCDPX;
+
+		switch (hw->mac.forced_speed_duplex) {
+		case IGC_FORCED_10H:
+			ctrl |= IGC_CTRL_SPEED_10;
+			break;
+		case IGC_FORCED_10F:
+			ctrl |= IGC_CTRL_SPEED_10 | IGC_CTRL_FD;
+			break;
+		case IGC_FORCED_100H:
+			ctrl |= IGC_CTRL_SPEED_100;
+			break;
+		case IGC_FORCED_100F:
+			ctrl |= IGC_CTRL_SPEED_100 | IGC_CTRL_FD;
+			break;
+		default:
+			return -IGC_ERR_CONFIG;
+		}
+		wr32(IGC_CTRL, ctrl);
+		ret_val = igc_setup_copper_link(hw);
+	}
 
 	return ret_val;
 }
@@ -443,6 +467,7 @@ static const struct igc_phy_operations igc_phy_ops_base = {
 	.reset			= igc_phy_hw_reset,
 	.read_reg		= igc_read_phy_reg_gpy,
 	.write_reg		= igc_write_phy_reg_gpy,
+	.force_speed_duplex	= igc_force_speed_duplex,
 };
 
 const struct igc_info igc_base_info = {
diff --git a/drivers/net/ethernet/intel/igc/igc_defines.h b/drivers/net/ethernet/intel/igc/igc_defines.h
index 9482ab11f050f..3f504751c2d92 100644
--- a/drivers/net/ethernet/intel/igc/igc_defines.h
+++ b/drivers/net/ethernet/intel/igc/igc_defines.h
@@ -129,10 +129,13 @@
 #define IGC_ERR_SWFW_SYNC		13
 
 /* Device Control */
+#define IGC_CTRL_FD		BIT(0)  /* Full Duplex */
 #define IGC_CTRL_RST		0x04000000  /* Global reset */
-
 #define IGC_CTRL_PHY_RST	0x80000000  /* PHY Reset */
 #define IGC_CTRL_SLU		0x00000040  /* Set link up (Force Link) */
+#define IGC_CTRL_SPEED_MASK	GENMASK(10, 8)
+#define IGC_CTRL_SPEED_10	FIELD_PREP(IGC_CTRL_SPEED_MASK, 0)
+#define IGC_CTRL_SPEED_100	FIELD_PREP(IGC_CTRL_SPEED_MASK, 1)
 #define IGC_CTRL_FRCSPD		0x00000800  /* Force Speed */
 #define IGC_CTRL_FRCDPX		0x00001000  /* Force Duplex */
 #define IGC_CTRL_VME		0x40000000  /* IEEE VLAN mode enable */
@@ -673,6 +676,10 @@
 #define IGC_GEN_POLL_TIMEOUT	1920
 
 /* PHY Control Register */
+#define MII_CR_SPEED_MASK	(BIT(6) | BIT(13))
+#define MII_CR_SPEED_10		0x0000	/* SSM=0, SSL=0: 10 Mb/s */
+#define MII_CR_SPEED_100	BIT(13)	/* SSM=0, SSL=1: 100 Mb/s */
+#define MII_CR_DUPLEX_EN	BIT(8)	/* 0 = Half Duplex, 1 = Full Duplex */
 #define MII_CR_RESTART_AUTO_NEG	0x0200  /* Restart auto negotiation */
 #define MII_CR_POWER_DOWN	0x0800  /* Power down */
 #define MII_CR_AUTO_NEG_EN	0x1000  /* Auto Neg Enable */
diff --git a/drivers/net/ethernet/intel/igc/igc_ethtool.c b/drivers/net/ethernet/intel/igc/igc_ethtool.c
index 0122009bedd03..89fe2788a565e 100644
--- a/drivers/net/ethernet/intel/igc/igc_ethtool.c
+++ b/drivers/net/ethernet/intel/igc/igc_ethtool.c
@@ -1460,6 +1460,26 @@ static int igc_ethtool_set_rxnfc(struct net_device *dev,
 	}
 }
 
+/**
+ * igc_write_rss_key - Program the RSS key into device registers
+ * @adapter: board private structure
+ *
+ * Write the RSS key stored in adapter->rss_key to the IGC_RSSRK registers.
+ * Each 32-bit chunk of the key is read using get_unaligned_le32() and written
+ * to the appropriate register.
+ */
+void igc_write_rss_key(struct igc_adapter *adapter)
+{
+	struct igc_hw *hw = &adapter->hw;
+	u32 val;
+	int i;
+
+	for (i = 0; i < IGC_RSS_KEY_SIZE / 4; i++) {
+		val = get_unaligned_le32(&adapter->rss_key[i * 4]);
+		wr32(IGC_RSSRK(i), val);
+	}
+}
+
 void igc_write_rss_indir_tbl(struct igc_adapter *adapter)
 {
 	struct igc_hw *hw = &adapter->hw;
@@ -1482,6 +1502,11 @@ void igc_write_rss_indir_tbl(struct igc_adapter *adapter)
 	}
 }
 
+static u32 igc_ethtool_get_rxfh_key_size(struct net_device *netdev)
+{
+	return IGC_RSS_KEY_SIZE;
+}
+
 static u32 igc_ethtool_get_rxfh_indir_size(struct net_device *netdev)
 {
 	return IGC_RETA_SIZE;
@@ -1494,10 +1519,13 @@ static int igc_ethtool_get_rxfh(struct net_device *netdev,
 	int i;
 
 	rxfh->hfunc = ETH_RSS_HASH_TOP;
-	if (!rxfh->indir)
-		return 0;
-	for (i = 0; i < IGC_RETA_SIZE; i++)
-		rxfh->indir[i] = adapter->rss_indir_tbl[i];
+
+	if (rxfh->indir)
+		for (i = 0; i < IGC_RETA_SIZE; i++)
+			rxfh->indir[i] = adapter->rss_indir_tbl[i];
+
+	if (rxfh->key)
+		memcpy(rxfh->key, adapter->rss_key, sizeof(adapter->rss_key));
 
 	return 0;
 }
@@ -1511,24 +1539,28 @@ static int igc_ethtool_set_rxfh(struct net_device *netdev,
 	int i;
 
 	/* We do not allow change in unsupported parameters */
-	if (rxfh->key ||
-	    (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE &&
-	     rxfh->hfunc != ETH_RSS_HASH_TOP))
+	if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE &&
+	    rxfh->hfunc != ETH_RSS_HASH_TOP)
 		return -EOPNOTSUPP;
-	if (!rxfh->indir)
-		return 0;
 
-	num_queues = adapter->rss_queues;
+	if (rxfh->indir) {
+		num_queues = adapter->rss_queues;
 
-	/* Verify user input. */
-	for (i = 0; i < IGC_RETA_SIZE; i++)
-		if (rxfh->indir[i] >= num_queues)
-			return -EINVAL;
+		/* Verify user input. */
+		for (i = 0; i < IGC_RETA_SIZE; i++)
+			if (rxfh->indir[i] >= num_queues)
+				return -EINVAL;
 
-	for (i = 0; i < IGC_RETA_SIZE; i++)
-		adapter->rss_indir_tbl[i] = rxfh->indir[i];
+		for (i = 0; i < IGC_RETA_SIZE; i++)
+			adapter->rss_indir_tbl[i] = rxfh->indir[i];
+
+		igc_write_rss_indir_tbl(adapter);
+	}
 
-	igc_write_rss_indir_tbl(adapter);
+	if (rxfh->key) {
+		memcpy(adapter->rss_key, rxfh->key, sizeof(adapter->rss_key));
+		igc_write_rss_key(adapter);
+	}
 
 	return 0;
 }
@@ -1914,44 +1946,58 @@ static int igc_ethtool_get_link_ksettings(struct net_device *netdev,
 	ethtool_link_ksettings_add_link_mode(cmd, supported, TP);
 	ethtool_link_ksettings_add_link_mode(cmd, advertising, TP);
 
-	/* advertising link modes */
-	if (hw->phy.autoneg_advertised & ADVERTISE_10_HALF)
-		ethtool_link_ksettings_add_link_mode(cmd, advertising, 10baseT_Half);
-	if (hw->phy.autoneg_advertised & ADVERTISE_10_FULL)
-		ethtool_link_ksettings_add_link_mode(cmd, advertising, 10baseT_Full);
-	if (hw->phy.autoneg_advertised & ADVERTISE_100_HALF)
-		ethtool_link_ksettings_add_link_mode(cmd, advertising, 100baseT_Half);
-	if (hw->phy.autoneg_advertised & ADVERTISE_100_FULL)
-		ethtool_link_ksettings_add_link_mode(cmd, advertising, 100baseT_Full);
-	if (hw->phy.autoneg_advertised & ADVERTISE_1000_FULL)
-		ethtool_link_ksettings_add_link_mode(cmd, advertising, 1000baseT_Full);
-	if (hw->phy.autoneg_advertised & ADVERTISE_2500_FULL)
-		ethtool_link_ksettings_add_link_mode(cmd, advertising, 2500baseT_Full);
-
 	/* set autoneg settings */
 	ethtool_link_ksettings_add_link_mode(cmd, supported, Autoneg);
-	ethtool_link_ksettings_add_link_mode(cmd, advertising, Autoneg);
+	if (hw->mac.autoneg_enabled) {
+		ethtool_link_ksettings_add_link_mode(cmd, advertising, Autoneg);
+		cmd->base.autoneg = AUTONEG_ENABLE;
+
+		/* advertising link modes only apply when autoneg is on */
+		if (hw->phy.autoneg_advertised & ADVERTISE_10_HALF)
+			ethtool_link_ksettings_add_link_mode(cmd, advertising,
+							     10baseT_Half);
+		if (hw->phy.autoneg_advertised & ADVERTISE_10_FULL)
+			ethtool_link_ksettings_add_link_mode(cmd, advertising,
+							     10baseT_Full);
+		if (hw->phy.autoneg_advertised & ADVERTISE_100_HALF)
+			ethtool_link_ksettings_add_link_mode(cmd, advertising,
+							     100baseT_Half);
+		if (hw->phy.autoneg_advertised & ADVERTISE_100_FULL)
+			ethtool_link_ksettings_add_link_mode(cmd, advertising,
+							     100baseT_Full);
+		if (hw->phy.autoneg_advertised & ADVERTISE_1000_FULL)
+			ethtool_link_ksettings_add_link_mode(cmd, advertising,
+							     1000baseT_Full);
+		if (hw->phy.autoneg_advertised & ADVERTISE_2500_FULL)
+			ethtool_link_ksettings_add_link_mode(cmd, advertising,
+							     2500baseT_Full);
+
+		/* Set pause flow control advertising */
+		switch (hw->fc.requested_mode) {
+		case igc_fc_full:
+			ethtool_link_ksettings_add_link_mode(cmd, advertising,
+							     Pause);
+			break;
+		case igc_fc_rx_pause:
+			ethtool_link_ksettings_add_link_mode(cmd, advertising,
+							     Pause);
+			ethtool_link_ksettings_add_link_mode(cmd, advertising,
+							     Asym_Pause);
+			break;
+		case igc_fc_tx_pause:
+			ethtool_link_ksettings_add_link_mode(cmd, advertising,
+							     Asym_Pause);
+			break;
+		default:
+			break;
+		}
+	} else {
+		cmd->base.autoneg = AUTONEG_DISABLE;
+	}
 
-	/* Set pause flow control settings */
+	/* Pause is always supported */
 	ethtool_link_ksettings_add_link_mode(cmd, supported, Pause);
 
-	switch (hw->fc.requested_mode) {
-	case igc_fc_full:
-		ethtool_link_ksettings_add_link_mode(cmd, advertising, Pause);
-		break;
-	case igc_fc_rx_pause:
-		ethtool_link_ksettings_add_link_mode(cmd, advertising, Pause);
-		ethtool_link_ksettings_add_link_mode(cmd, advertising,
-						     Asym_Pause);
-		break;
-	case igc_fc_tx_pause:
-		ethtool_link_ksettings_add_link_mode(cmd, advertising,
-						     Asym_Pause);
-		break;
-	default:
-		break;
-	}
-
 	status = pm_runtime_suspended(&adapter->pdev->dev) ?
 		 0 : rd32(IGC_STATUS);
 
@@ -1983,7 +2029,6 @@ static int igc_ethtool_get_link_ksettings(struct net_device *netdev,
 		cmd->base.duplex = DUPLEX_UNKNOWN;
 	}
 	cmd->base.speed = speed;
-	cmd->base.autoneg = AUTONEG_ENABLE;
 
 	/* MDI-X => 2; MDI =>1; Invalid =>0 */
 	if (hw->phy.media_type == igc_media_type_copper)
@@ -2000,37 +2045,50 @@ static int igc_ethtool_get_link_ksettings(struct net_device *netdev,
 	return 0;
 }
 
-static int
-igc_ethtool_set_link_ksettings(struct net_device *netdev,
-			       const struct ethtool_link_ksettings *cmd)
+/**
+ * igc_handle_autoneg_disabled - Configure forced speed/duplex settings
+ * @adapter: private driver structure
+ * @speed: requested speed (must be SPEED_10 or SPEED_100)
+ * @duplex: requested duplex
+ *
+ * Records forced speed/duplex when autoneg is disabled.
+ * Caller must validate speed before calling this function.
+ */
+static void igc_handle_autoneg_disabled(struct igc_adapter *adapter, u32 speed,
+					u8 duplex)
 {
-	struct igc_adapter *adapter = netdev_priv(netdev);
-	struct net_device *dev = adapter->netdev;
-	struct igc_hw *hw = &adapter->hw;
-	u16 advertised = 0;
+	struct igc_mac_info *mac = &adapter->hw.mac;
 
-	/* When adapter in resetting mode, autoneg/speed/duplex
-	 * cannot be changed
-	 */
-	if (igc_check_reset_block(hw)) {
-		netdev_err(dev, "Cannot change link characteristics when reset is active\n");
-		return -EINVAL;
+	switch (speed) {
+	case SPEED_10:
+		mac->forced_speed_duplex = (duplex == DUPLEX_FULL) ?
+			IGC_FORCED_10F : IGC_FORCED_10H;
+		break;
+	case SPEED_100:
+		mac->forced_speed_duplex = (duplex == DUPLEX_FULL) ?
+			IGC_FORCED_100F : IGC_FORCED_100H;
+		break;
+	default:
+		WARN_ONCE(1, "Unsupported speed %u\n", speed);
+		return;
 	}
 
-	/* MDI setting is only allowed when autoneg enabled because
-	 * some hardware doesn't allow MDI setting when speed or
-	 * duplex is forced.
-	 */
-	if (cmd->base.eth_tp_mdix_ctrl) {
-		if (cmd->base.eth_tp_mdix_ctrl != ETH_TP_MDI_AUTO &&
-		    cmd->base.autoneg != AUTONEG_ENABLE) {
-			netdev_err(dev, "Forcing MDI/MDI-X state is not supported when link speed and/or duplex are forced\n");
-			return -EINVAL;
-		}
-	}
+	mac->autoneg_enabled = false;
+}
 
-	while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
-		usleep_range(1000, 2000);
+/**
+ * igc_handle_autoneg_enabled - Configure autonegotiation advertisement
+ * @adapter: private driver structure
+ * @cmd: ethtool link ksettings from user
+ *
+ * Records advertised speeds and flow control settings when autoneg
+ * is enabled.
+ */
+static void igc_handle_autoneg_enabled(struct igc_adapter *adapter,
+				       const struct ethtool_link_ksettings *cmd)
+{
+	struct igc_hw *hw = &adapter->hw;
+	u16 advertised = 0;
 
 	if (ethtool_link_ksettings_test_link_mode(cmd, advertising,
 						  2500baseT_Full))
@@ -2056,14 +2114,66 @@ igc_ethtool_set_link_ksettings(struct net_device *netdev,
 						  10baseT_Half))
 		advertised |= ADVERTISE_10_HALF;
 
-	if (cmd->base.autoneg == AUTONEG_ENABLE) {
-		hw->phy.autoneg_advertised = advertised;
-		if (adapter->fc_autoneg)
-			hw->fc.requested_mode = igc_fc_default;
-	} else {
-		netdev_info(dev, "Force mode currently not supported\n");
+	hw->mac.autoneg_enabled = true;
+	hw->phy.autoneg_advertised = advertised;
+	if (adapter->fc_autoneg)
+		hw->fc.requested_mode = igc_fc_default;
+}
+
+static int
+igc_ethtool_set_link_ksettings(struct net_device *netdev,
+			       const struct ethtool_link_ksettings *cmd)
+{
+	struct igc_adapter *adapter = netdev_priv(netdev);
+	struct net_device *dev = adapter->netdev;
+	struct igc_hw *hw = &adapter->hw;
+
+	/* When adapter in resetting mode, autoneg/speed/duplex
+	 * cannot be changed
+	 */
+	if (igc_check_reset_block(hw)) {
+		netdev_err(dev, "Cannot change link characteristics when reset is active\n");
+		return -EINVAL;
+	}
+
+	if (cmd->base.autoneg != AUTONEG_ENABLE &&
+	    cmd->base.autoneg != AUTONEG_DISABLE) {
+		netdev_info(dev, "Unsupported autoneg setting\n");
+		return -EINVAL;
+	}
+
+	/* MDI setting is only allowed when autoneg enabled because
+	 * some hardware doesn't allow MDI setting when speed or
+	 * duplex is forced.
+	 */
+	if (cmd->base.eth_tp_mdix_ctrl) {
+		if (cmd->base.eth_tp_mdix_ctrl != ETH_TP_MDI_AUTO &&
+		    cmd->base.autoneg != AUTONEG_ENABLE) {
+			netdev_err(dev, "Forcing MDI/MDI-X state is not supported when link speed and/or duplex are forced\n");
+			return -EINVAL;
+		}
 	}
 
+	if (cmd->base.autoneg == AUTONEG_DISABLE) {
+		if (cmd->base.speed != SPEED_10 && cmd->base.speed != SPEED_100) {
+			netdev_info(dev, "Unsupported speed for forced link\n");
+			return -EINVAL;
+		}
+		if (cmd->base.duplex != DUPLEX_HALF && cmd->base.duplex != DUPLEX_FULL) {
+			netdev_info(dev, "Duplex must be half or full for forced link\n");
+			return -EINVAL;
+		}
+	}
+
+	while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
+		usleep_range(1000, 2000);
+
+	if (cmd->base.autoneg == AUTONEG_ENABLE)
+		igc_handle_autoneg_enabled(adapter, cmd);
+	else
+		igc_handle_autoneg_disabled(adapter, cmd->base.speed,
+					    cmd->base.duplex);
+
 	/* MDI-X => 2; MDI => 1; Auto => 3 */
 	if (cmd->base.eth_tp_mdix_ctrl) {
 		/* fix up the value for auto (3 => 0) as zero is mapped
@@ -2175,6 +2285,7 @@ static const struct ethtool_ops igc_ethtool_ops = {
 	.get_rxnfc		= igc_ethtool_get_rxnfc,
 	.set_rxnfc		= igc_ethtool_set_rxnfc,
 	.get_rx_ring_count	= igc_ethtool_get_rx_ring_count,
+	.get_rxfh_key_size	= igc_ethtool_get_rxfh_key_size,
 	.get_rxfh_indir_size	= igc_ethtool_get_rxfh_indir_size,
 	.get_rxfh		= igc_ethtool_get_rxfh,
 	.set_rxfh		= igc_ethtool_set_rxfh,
diff --git a/drivers/net/ethernet/intel/igc/igc_hw.h b/drivers/net/ethernet/intel/igc/igc_hw.h
index be8a49a86d09f..62aaee55668a5 100644
--- a/drivers/net/ethernet/intel/igc/igc_hw.h
+++ b/drivers/net/ethernet/intel/igc/igc_hw.h
@@ -73,6 +73,13 @@ struct igc_info {
 
 extern const struct igc_info igc_base_info;
 
+enum igc_forced_speed_duplex {
+	IGC_FORCED_10H,
+	IGC_FORCED_10F,
+	IGC_FORCED_100H,
+	IGC_FORCED_100F,
+};
+
 struct igc_mac_info {
 	struct igc_mac_operations ops;
 
@@ -92,8 +99,9 @@ struct igc_mac_info {
 	bool asf_firmware_present;
 	bool arc_subsystem_valid;
 
-	bool autoneg_failed;
 	bool get_link_status;
+	bool autoneg_enabled;
+	enum igc_forced_speed_duplex forced_speed_duplex;
 };
 
 struct igc_nvm_operations {
diff --git a/drivers/net/ethernet/intel/igc/igc_mac.c b/drivers/net/ethernet/intel/igc/igc_mac.c
index 7ac6637f8db73..d6f3f6618469d 100644
--- a/drivers/net/ethernet/intel/igc/igc_mac.c
+++ b/drivers/net/ethernet/intel/igc/igc_mac.c
@@ -438,26 +438,23 @@ void igc_config_collision_dist(struct igc_hw *hw)
  * Checks the status of auto-negotiation after link up to ensure that the
  * speed and duplex were not forced.  If the link needed to be forced, then
  * flow control needs to be forced also.  If auto-negotiation is enabled
- * and did not fail, then we configure flow control based on our link
- * partner.
+ * then we configure flow control based on our link partner.
  */
 s32 igc_config_fc_after_link_up(struct igc_hw *hw)
 {
 	u16 mii_status_reg, mii_nway_adv_reg, mii_nway_lp_ability_reg;
-	struct igc_mac_info *mac = &hw->mac;
 	u16 speed, duplex;
 	s32 ret_val = 0;
 
-	/* Check for the case where we have fiber media and auto-neg failed
-	 * so we had to force link.  In this case, we need to force the
-	 * configuration of the MAC to match the "fc" parameter.
+	/* Without autoneg, flow control capability is not exchanged with the
+	 * link partner. IEEE 802.3 prohibits flow control in half-duplex mode.
 	 */
-	if (mac->autoneg_failed)
-		ret_val = igc_force_mac_fc(hw);
+	if (!hw->mac.autoneg_enabled) {
+		if (hw->mac.forced_speed_duplex == IGC_FORCED_10H ||
+		    hw->mac.forced_speed_duplex == IGC_FORCED_100H)
+			hw->fc.current_mode = igc_fc_none;
 
-	if (ret_val) {
-		hw_dbg("Error forcing flow control settings\n");
-		goto out;
+		goto force_fc;
 	}
 
 	/* In auto-neg, we need to check and see if Auto-Neg has completed,
@@ -472,15 +469,15 @@ s32 igc_config_fc_after_link_up(struct igc_hw *hw)
 	ret_val = hw->phy.ops.read_reg(hw, PHY_STATUS,
 				       &mii_status_reg);
 	if (ret_val)
-		goto out;
+		return ret_val;
 	ret_val = hw->phy.ops.read_reg(hw, PHY_STATUS,
 				       &mii_status_reg);
 	if (ret_val)
-		goto out;
+		return ret_val;
 
 	if (!(mii_status_reg & MII_SR_AUTONEG_COMPLETE)) {
 		hw_dbg("Copper PHY and Auto Neg has not completed.\n");
-		goto out;
+		return ret_val;
 	}
 
 	/* The AutoNeg process has completed, so we now need to
@@ -492,11 +489,11 @@ s32 igc_config_fc_after_link_up(struct igc_hw *hw)
 	ret_val = hw->phy.ops.read_reg(hw, PHY_AUTONEG_ADV,
 				       &mii_nway_adv_reg);
 	if (ret_val)
-		goto out;
+		return ret_val;
 	ret_val = hw->phy.ops.read_reg(hw, PHY_LP_ABILITY,
 				       &mii_nway_lp_ability_reg);
 	if (ret_val)
-		goto out;
+		return ret_val;
 	/* Two bits in the Auto Negotiation Advertisement Register
 	 * (Address 4) and two bits in the Auto Negotiation Base
 	 * Page Ability Register (Address 5) determine flow control
@@ -612,7 +609,7 @@ s32 igc_config_fc_after_link_up(struct igc_hw *hw)
 	ret_val = hw->mac.ops.get_speed_and_duplex(hw, &speed, &duplex);
 	if (ret_val) {
 		hw_dbg("Error getting link speed and duplex\n");
-		goto out;
+		return ret_val;
 	}
 
 	if (duplex == HALF_DUPLEX)
@@ -621,13 +618,13 @@ s32 igc_config_fc_after_link_up(struct igc_hw *hw)
 	/* Now we call a subroutine to actually force the MAC
 	 * controller to use the correct flow control settings.
 	 */
+force_fc:
 	ret_val = igc_force_mac_fc(hw);
 	if (ret_val) {
 		hw_dbg("Error forcing flow control settings\n");
-		goto out;
+		return ret_val;
 	}
 
-out:
 	return ret_val;
 }
 
diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c
index 2c9e2dfd84991..e6e9441fc3d4e 100644
--- a/drivers/net/ethernet/intel/igc/igc_main.c
+++ b/drivers/net/ethernet/intel/igc/igc_main.c
@@ -785,11 +785,8 @@ static void igc_setup_mrqc(struct igc_adapter *adapter)
 	struct igc_hw *hw = &adapter->hw;
 	u32 j, num_rx_queues;
 	u32 mrqc, rxcsum;
-	u32 rss_key[10];
 
-	netdev_rss_key_fill(rss_key, sizeof(rss_key));
-	for (j = 0; j < 10; j++)
-		wr32(IGC_RSSRK(j), rss_key[j]);
+	igc_write_rss_key(adapter);
 
 	num_rx_queues = adapter->rss_queues;
 
@@ -5048,6 +5045,9 @@ static int igc_sw_init(struct igc_adapter *adapter)
 
 	pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
 
+	/* init RSS key */
+	netdev_rss_key_fill(adapter->rss_key, sizeof(adapter->rss_key));
+
 	/* set default ring sizes */
 	adapter->tx_ring_count = IGC_DEFAULT_TXD;
 	adapter->rx_ring_count = IGC_DEFAULT_RXD;
@@ -7298,7 +7298,7 @@ static int igc_probe(struct pci_dev *pdev,
 	/* Initialize link properties that are user-changeable */
 	adapter->fc_autoneg = true;
 	hw->phy.autoneg_advertised = 0xaf;
-
+	hw->mac.autoneg_enabled = true;
 	hw->fc.requested_mode = igc_fc_default;
 	hw->fc.current_mode = igc_fc_default;
 
diff --git a/drivers/net/ethernet/intel/igc/igc_phy.c b/drivers/net/ethernet/intel/igc/igc_phy.c
index 6c4d204aecfae..4cf737fb3b21e 100644
--- a/drivers/net/ethernet/intel/igc/igc_phy.c
+++ b/drivers/net/ethernet/intel/igc/igc_phy.c
@@ -494,12 +494,20 @@ s32 igc_setup_copper_link(struct igc_hw *hw)
 	s32 ret_val = 0;
 	bool link;
 
-	/* Setup autoneg and flow control advertisement and perform
-	 * autonegotiation.
-	 */
-	ret_val = igc_copper_link_autoneg(hw);
-	if (ret_val)
-		goto out;
+	if (hw->mac.autoneg_enabled) {
+		/* Setup autoneg and flow control advertisement and perform
+		 * autonegotiation.
+		 */
+		ret_val = igc_copper_link_autoneg(hw);
+		if (ret_val)
+			goto out;
+	} else {
+		ret_val = hw->phy.ops.force_speed_duplex(hw);
+		if (ret_val) {
+			hw_dbg("Error Forcing Speed/Duplex\n");
+			goto out;
+		}
+	}
 
 	/* Check link status. Wait up to 100 microseconds for link to become
 	 * valid.
@@ -778,3 +786,48 @@ u16 igc_read_phy_fw_version(struct igc_hw *hw)
 
 	return gphy_version;
 }
+
+/**
+ * igc_force_speed_duplex - Force PHY speed and duplex settings
+ * @hw: pointer to the HW structure
+ *
+ * Programs the GPY PHY control register to disable autonegotiation
+ * and force the speed/duplex indicated by hw->mac.forced_speed_duplex.
+ */
+s32 igc_force_speed_duplex(struct igc_hw *hw)
+{
+	struct igc_phy_info *phy = &hw->phy;
+	u16 phy_ctrl;
+	s32 ret_val;
+
+	ret_val = phy->ops.read_reg(hw, PHY_CONTROL, &phy_ctrl);
+	if (ret_val)
+		return ret_val;
+
+	phy_ctrl &= ~(MII_CR_SPEED_MASK | MII_CR_DUPLEX_EN |
+		      MII_CR_AUTO_NEG_EN | MII_CR_RESTART_AUTO_NEG);
+
+	switch (hw->mac.forced_speed_duplex) {
+	case IGC_FORCED_10H:
+		phy_ctrl |= MII_CR_SPEED_10;
+		break;
+	case IGC_FORCED_10F:
+		phy_ctrl |= MII_CR_SPEED_10 | MII_CR_DUPLEX_EN;
+		break;
+	case IGC_FORCED_100H:
+		phy_ctrl |= MII_CR_SPEED_100;
+		break;
+	case IGC_FORCED_100F:
+		phy_ctrl |= MII_CR_SPEED_100 | MII_CR_DUPLEX_EN;
+		break;
+	default:
+		return -IGC_ERR_CONFIG;
+	}
+
+	ret_val = phy->ops.write_reg(hw, PHY_CONTROL, phy_ctrl);
+	if (ret_val)
+		return ret_val;
+
+	hw->mac.get_link_status = true;
+	return 0;
+}
diff --git a/drivers/net/ethernet/intel/igc/igc_phy.h b/drivers/net/ethernet/intel/igc/igc_phy.h
index 832a7e359f186..d37a89174826f 100644
--- a/drivers/net/ethernet/intel/igc/igc_phy.h
+++ b/drivers/net/ethernet/intel/igc/igc_phy.h
@@ -18,5 +18,6 @@ void igc_power_down_phy_copper(struct igc_hw *hw);
 s32 igc_write_phy_reg_gpy(struct igc_hw *hw, u32 offset, u16 data);
 s32 igc_read_phy_reg_gpy(struct igc_hw *hw, u32 offset, u16 *data);
 u16 igc_read_phy_fw_version(struct igc_hw *hw);
+s32 igc_force_speed_duplex(struct igc_hw *hw);
 
 #endif
diff --git a/drivers/net/ethernet/marvell/mv643xx_eth.c b/drivers/net/ethernet/marvell/mv643xx_eth.c
index 1881583be5ce2..9caa1e47c174c 100644
--- a/drivers/net/ethernet/marvell/mv643xx_eth.c
+++ b/drivers/net/ethernet/marvell/mv643xx_eth.c
@@ -2780,7 +2780,7 @@ static int mv643xx_eth_shared_of_add_port(struct platform_device *pdev,
 		goto put_err;
 	}
 	ppdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
-	ppdev->dev.of_node = of_node_get(pnp);
+	platform_device_set_of_node(ppdev, pnp);
 
 	ret = platform_device_add_resources(ppdev, &res, 1);
 	if (ret)
diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c
index 744d6585a949d..543e566425c1e 100644
--- a/drivers/net/ethernet/marvell/mvneta.c
+++ b/drivers/net/ethernet/marvell/mvneta.c
@@ -5678,6 +5678,24 @@ static int mvneta_probe(struct platform_device *pdev)
 					 "use SW buffer management\n");
 				mvneta_bm_put(pp->bm_priv);
 				pp->bm_priv = NULL;
+			} else if (!device_link_add(&pdev->dev,
+						    &pp->bm_priv->pdev->dev,
+						    DL_FLAG_AUTOREMOVE_CONSUMER)) {
+				/*
+				 * Link guarantees BM resumes before mvneta.
+				 * Without it, BM may not be ready when
+				 * mvneta_bm_port_init() runs on resume,
+				 * causing stale buffer addresses and a crash.
+				 * Fall back to SW management to be safe.
+				 */
+				dev_warn(&pdev->dev,
+					 "failed to link to BM, use SW buffer management\n");
+				mvneta_bm_pool_destroy(pp->bm_priv,
+						       pp->pool_long, 1 << pp->id);
+				mvneta_bm_pool_destroy(pp->bm_priv,
+						       pp->pool_short, 1 << pp->id);
+				mvneta_bm_put(pp->bm_priv);
+				pp->bm_priv = NULL;
 			}
 		}
 		/* Set RX packet offset correction for platforms, whose
diff --git a/drivers/net/ethernet/marvell/mvneta_bm.c b/drivers/net/ethernet/marvell/mvneta_bm.c
index 6bb3804949190..e0c693c0a9106 100644
--- a/drivers/net/ethernet/marvell/mvneta_bm.c
+++ b/drivers/net/ethernet/marvell/mvneta_bm.c
@@ -129,6 +129,7 @@ static int mvneta_bm_pool_create(struct mvneta_bm *priv,
 	if (!IS_ALIGNED((u32)bm_pool->virt_addr, MVNETA_BM_POOL_PTR_ALIGN)) {
 		dma_free_coherent(&pdev->dev, size_bytes, bm_pool->virt_addr,
 				  bm_pool->phys_addr);
+		bm_pool->virt_addr = NULL;
 		dev_err(&pdev->dev, "BM pool %d is not %d bytes aligned\n",
 			bm_pool->id, MVNETA_BM_POOL_PTR_ALIGN);
 		return -ENOMEM;
@@ -139,6 +140,7 @@ static int mvneta_bm_pool_create(struct mvneta_bm *priv,
 	if (err < 0) {
 		dma_free_coherent(&pdev->dev, size_bytes, bm_pool->virt_addr,
 				  bm_pool->phys_addr);
+		bm_pool->virt_addr = NULL;
 		return err;
 	}
 
@@ -477,6 +479,75 @@ static void mvneta_bm_remove(struct platform_device *pdev)
 	clk_disable_unprepare(priv->clk);
 }
 
+static int mvneta_bm_suspend(struct device *dev)
+{
+	struct mvneta_bm *priv = dev_get_drvdata(dev);
+	int i;
+
+	/* Drain buffers and free pool resources while BM is still clocked */
+	for (i = 0; i < MVNETA_BM_POOLS_NUM; i++) {
+		struct mvneta_bm_pool *bm_pool = &priv->bm_pools[i];
+		int size_bytes;
+
+		if (bm_pool->type == MVNETA_BM_FREE)
+			continue;
+
+		mvneta_bm_bufs_free(priv, bm_pool, bm_pool->port_map);
+		if (bm_pool->hwbm_pool.buf_num)
+			dev_warn(&priv->pdev->dev,
+				 "pool %d: %d buffers not freed\n",
+				 bm_pool->id, bm_pool->hwbm_pool.buf_num);
+
+		mvneta_bm_pool_disable(priv, bm_pool->id);
+
+		if (bm_pool->virt_addr) {
+			size_bytes = sizeof(u32) * bm_pool->hwbm_pool.size;
+			dma_free_coherent(&priv->pdev->dev, size_bytes,
+					  bm_pool->virt_addr,
+					  bm_pool->phys_addr);
+			bm_pool->virt_addr = NULL;
+		}
+		/*
+		 * Safe to destroy: device_link guarantees all mvneta ports
+		 * have already suspended, so no hwbm_pool_add() can be in
+		 * progress holding buf_lock.  Pairs with mutex_init() in
+		 * mvneta_bm_pool_use() on resume.
+		 */
+		mutex_destroy(&bm_pool->hwbm_pool.buf_lock);
+		bm_pool->type = MVNETA_BM_FREE;
+	}
+
+	mvneta_bm_write(priv, MVNETA_BM_COMMAND_REG, MVNETA_BM_STOP_MASK);
+	clk_disable_unprepare(priv->clk);
+	return 0;
+}
+
+static int mvneta_bm_resume(struct device *dev)
+{
+	struct mvneta_bm *priv = dev_get_drvdata(dev);
+	int i, err;
+
+	err = clk_prepare_enable(priv->clk);
+	if (err)
+		return err;
+
+	/* Reinitialize BM hardware; pools are refilled by mvneta_resume() */
+	mvneta_bm_default_set(priv);
+
+	/* Restore pool registers lost during clock gating */
+	for (i = 0; i < MVNETA_BM_POOLS_NUM; i++) {
+		mvneta_bm_write(priv, MVNETA_BM_POOL_READ_PTR_REG(i), 0);
+		mvneta_bm_write(priv, MVNETA_BM_POOL_WRITE_PTR_REG(i), 0);
+		mvneta_bm_write(priv, MVNETA_BM_POOL_SIZE_REG(i),
+				priv->bm_pools[i].hwbm_pool.size);
+	}
+
+	mvneta_bm_write(priv, MVNETA_BM_COMMAND_REG, MVNETA_BM_START_MASK);
+	return 0;
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(mvneta_bm_pm_ops, mvneta_bm_suspend, mvneta_bm_resume);
+
 static const struct of_device_id mvneta_bm_match[] = {
 	{ .compatible = "marvell,armada-380-neta-bm" },
 	{ }
@@ -489,6 +560,7 @@ static struct platform_driver mvneta_bm_driver = {
 	.driver = {
 		.name = MVNETA_BM_DRIVER_NAME,
 		.of_match_table = mvneta_bm_match,
+		.pm = pm_sleep_ptr(&mvneta_bm_pm_ops),
 	},
 };
 
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h
index 714e47f68d932..f87cdf1b971db 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h
@@ -803,6 +803,7 @@ struct npc_set_pkind {
 			 */
 	u8 var_len_off_mask; /* Mask for length with in offset */
 	u8 shift_dir; /* shift direction to get length of the header at var_len_off */
+	u8 skip_size; /* l2 size to skip */
 };
 
 /* NPA mbox message formats */
@@ -1267,6 +1268,7 @@ struct nix_rss_flowkey_cfg {
 #define NIX_FLOW_KEY_TYPE_IPV4_PROTO	BIT(21)
 #define NIX_FLOW_KEY_TYPE_AH		BIT(22)
 #define NIX_FLOW_KEY_TYPE_ESP		BIT(23)
+#define NIX_FLOW_KEY_TYPE_ROCEV2	BIT(24)
 #define NIX_FLOW_KEY_TYPE_L4_DST_ONLY BIT(28)
 #define NIX_FLOW_KEY_TYPE_L4_SRC_ONLY BIT(29)
 #define NIX_FLOW_KEY_TYPE_L3_DST_ONLY BIT(30)
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/npc.h b/drivers/net/ethernet/marvell/octeontx2/af/npc.h
index eaed172f1606f..719b3618eeb52 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/npc.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/npc.h
@@ -161,10 +161,12 @@ enum npc_kpu_lh_ltype {
  * Software assigns pkind for each incoming port such as CGX
  * Ethernet interfaces, LBK interfaces, etc.
  */
-#define NPC_UNRESERVED_PKIND_COUNT NPC_RX_CPT_HDR_PTP_PKIND
+#define NPC_UNRESERVED_PKIND_COUNT NPC_RX_SKIP_SIZE_PKIND
 
 enum npc_pkind_type {
 	NPC_RX_LBK_PKIND = 0ULL,
+	NPC_RX_SKIP_SIZE_PKIND = 46ULL,
+	NPC_RX_CPT_SKIP_SIZE_PKIND = 50ULL,
 	NPC_RX_CPT_HDR_PTP_PKIND = 54ULL,
 	NPC_RX_CUSTOM_PRE_L2_PKIND = 55ULL,
 	NPC_RX_VLAN_EXDSA_PKIND = 56ULL,
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
index 7f3505ae68604..c5610f2426879 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
@@ -1181,7 +1181,7 @@ void rvu_switch_enable_lbk_link(struct rvu *rvu, u16 pcifunc, bool ena);
 
 int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir,
 			   u64 pkind, u8 var_len_off, u8 var_len_off_mask,
-			   u8 shift_dir);
+			   u8 shift_dir, u8 skip_size);
 int rvu_get_hwvf(struct rvu *rvu, int pcifunc);
 
 /* CN10K MCS */
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
index 6a0ce2665031d..78667a0977c01 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
@@ -4305,6 +4305,13 @@ static int set_flowkey_fields(struct nix_rx_flowkey_alg *alg, u32 flow_cfg)
 				keyoff_marker = false;
 			}
 			break;
+		case NIX_FLOW_KEY_TYPE_ROCEV2:
+			field->hdr_offset = 5;
+			field->bytesm1 = 2; /* Destination QP */
+			field->ltype_mask = 0xF;
+			field->lid = NPC_LID_LE;
+			field->ltype_match = NPC_LT_LE_ROCEV2;
+			break;
 		}
 		field->ena = 1;
 
@@ -5392,7 +5399,7 @@ void rvu_nix_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int nixlf)
 
 	/* reset HW config done for Switch headers */
 	rvu_npc_set_parse_mode(rvu, pcifunc, OTX2_PRIV_FLAGS_DEFAULT,
-			       (PKIND_TX | PKIND_RX), 0, 0, 0, 0);
+			       (PKIND_TX | PKIND_RX), 0, 0, 0, 0, 0);
 
 	/* Disabling CGX and NPC config done for PTP */
 	if (pfvf->hw_rx_tstamp_en) {
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c
index c7bc0b3a29b98..08b83de9beb46 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c
@@ -4194,10 +4194,40 @@ npc_set_var_len_offset_pkind(struct rvu *rvu, u16 pcifunc, u64 pkind,
 	return 0;
 }
 
+static int npc_set_skip_size_pkind(struct rvu *rvu, u16 pcifunc, u64 pkind,
+				   u8 skip_size)
+{
+	struct npc_kpu_action0 *act0;
+	int blkaddr;
+	u64 val;
+
+	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc);
+	if (blkaddr < 0) {
+		dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);
+		return -EINVAL;
+	}
+
+	val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind));
+	act0 = (struct npc_kpu_action0 *)&val;
+	act0->ptr_advance = skip_size;
+	rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val);
+
+	/* Update CPT_HR new PKIND */
+	val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind + 4));
+	act0 = (struct npc_kpu_action0 *)&val;
+	act0->ptr_advance = (skip_size + 40);
+	act0->next_state = NPC_S_KPU1_CPT_HDR;
+	act0->var_len_offset = (skip_size + 6);
+	act0->var_len_mask = 0xe0;
+	act0->var_len_shift = 0x5;
+	act0->var_len_right = 0x1;
+	rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind + 4), val);
+	return 0;
+}
+
 int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir,
 			   u64 pkind, u8 var_len_off, u8 var_len_off_mask,
-			   u8 shift_dir)
-
+			   u8 shift_dir, u8 skip_size)
 {
 	struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
 	int blkaddr, nixlf, rc, intf_mode;
@@ -4218,6 +4248,12 @@ int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir,
 							  shift_dir);
 			if (rc)
 				return rc;
+		} else if (pkind >= NPC_RX_SKIP_SIZE_PKIND &&
+			   pkind <= NPC_RX_SKIP_SIZE_PKIND + 3) {
+			rc = npc_set_skip_size_pkind(rvu, pcifunc, pkind,
+						     skip_size);
+			if (rc)
+				return rc;
 		}
 		rxpkind = pkind;
 		txpkind = pkind;
@@ -4254,7 +4290,8 @@ int rvu_mbox_handler_npc_set_pkind(struct rvu *rvu, struct npc_set_pkind *req,
 {
 	return rvu_npc_set_parse_mode(rvu, req->hdr.pcifunc, req->mode,
 				      req->dir, req->pkind, req->var_len_off,
-				      req->var_len_off_mask, req->shift_dir);
+				      req->var_len_off_mask, req->shift_dir,
+				      req->skip_size);
 }
 
 int rvu_mbox_handler_npc_read_base_steer_rule(struct rvu *rvu,
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c
index dbf1731966082..8e41431c7f9c7 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c
@@ -656,6 +656,7 @@ static int cn20k_pool_aq_init(struct otx2_nic *pfvf, u16 pool_id,
 	pp_params.nid = NUMA_NO_NODE;
 	pp_params.dev = pfvf->dev;
 	pp_params.dma_dir = DMA_FROM_DEVICE;
+	pp_params.netdev = pfvf->netdev;
 	pool->page_pool = page_pool_create(&pp_params);
 	if (IS_ERR(pool->page_pool)) {
 		netdev_err(pfvf->netdev, "Creation of page pool failed\n");
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
index 3d253132a17f0..ca73a94db7943 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
@@ -1035,7 +1035,6 @@ int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura)
 	if (qidx > pfvf->hw.xdp_queues)
 		otx2_attach_xsk_buff(pfvf, sq, (qidx - pfvf->hw.xdp_queues));
 
-
 	chan_offset = qidx % pfvf->hw.tx_chan_cnt;
 	err = pfvf->hw_ops->sq_aq_init(pfvf, qidx, chan_offset, sqb_aura);
 	if (err) {
@@ -1517,6 +1516,7 @@ int otx2_pool_aq_init(struct otx2_nic *pfvf, u16 pool_id,
 		pp_params.nid = NUMA_NO_NODE;
 		pp_params.dev = pfvf->dev;
 		pp_params.dma_dir = DMA_FROM_DEVICE;
+		pp_params.netdev = pfvf->netdev;
 		pool->page_pool = page_pool_create(&pp_params);
 		if (IS_ERR(pool->page_pool)) {
 			netdev_err(pfvf->netdev, "Creation of page pool failed\n");
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
index 5d291e50a47bb..fe7610c42e5d3 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
@@ -26,6 +26,7 @@
 #include <linux/bitfield.h>
 #include <net/dsa.h>
 #include <net/dst_metadata.h>
+#include <net/netdev_lock.h>
 #include <net/page_pool/helpers.h>
 #include <linux/genalloc.h>
 
@@ -5030,10 +5031,14 @@ void mtk_eth_set_dma_device(struct mtk_eth *eth, struct device *dma_dev)
 			continue;
 
 		list_add_tail(&dev->close_list, &dev_list);
+		netdev_lock_ops(dev);
 	}
 
 	netif_close_many(&dev_list, false);
 
+	list_for_each_entry(dev, &dev_list, close_list)
+		netdev_unlock_ops(dev);
+
 	eth->dma_dev = dma_dev;
 
 	list_for_each_entry_safe(dev, tmp, &dev_list, close_list) {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
index c31e05529fc40..b9026cc643835 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
@@ -383,6 +383,7 @@ static const struct devlink_ops mlx5_devlink_ops = {
 	.rate_node_del = mlx5_esw_devlink_rate_node_del,
 	.rate_leaf_parent_set = mlx5_esw_devlink_rate_leaf_parent_set,
 	.rate_node_parent_set = mlx5_esw_devlink_rate_node_parent_set,
+	.supported_cross_device_rate_nodes = true,
 #endif
 #ifdef CONFIG_MLX5_SF_MANAGER
 	.port_new = mlx5_devlink_sf_port_new,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c
index 6e50311faa27f..36b00a856bc21 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c
@@ -74,7 +74,7 @@ static void mlx5_esw_offloads_pf_vf_devlink_port_attrs_set(struct mlx5_eswitch *
 		memcpy(dl_port->attrs.switch_id.id, ppid.id, ppid.id_len);
 		dl_port->attrs.switch_id.id_len = ppid.id_len;
 		devlink_port_attrs_pci_pf_set(dl_port, controller_num, pfnum,
-					      true);
+					      false);
 	}
 }
 
@@ -134,13 +134,16 @@ static void mlx5_esw_offloads_sf_devlink_port_attrs_set(struct mlx5_eswitch *esw
 {
 	struct mlx5_core_dev *dev = esw->dev;
 	struct netdev_phys_item_id ppid = {};
+	u32 hpf_ctrl;
 	u16 pfnum;
 
 	pfnum = mlx5_esw_sf_controller_to_pfnum(dev, controller);
+	hpf_ctrl = mlx5_esw_get_hpf_host_number(dev) + 1;
 	mlx5_esw_get_port_parent_id(dev, &ppid);
 	memcpy(dl_port->attrs.switch_id.id, &ppid.id[0], ppid.id_len);
 	dl_port->attrs.switch_id.id_len = ppid.id_len;
-	devlink_port_attrs_pci_sf_set(dl_port, controller, pfnum, sfnum, !!controller);
+	devlink_port_attrs_pci_sf_set(dl_port, controller, pfnum, sfnum,
+				      controller == hpf_ctrl);
 }
 
 int mlx5_esw_offloads_sf_devlink_port_init(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
@@ -268,7 +271,6 @@ void mlx5_esw_offloads_devlink_port_unregister(struct mlx5_vport *vport)
 	dl_port = vport->dl_port;
 	mlx5_esw_devlink_port_res_unregister(&dl_port->dl_port);
 
-	mlx5_esw_qos_vport_update_parent(vport, NULL, NULL);
 	devl_rate_leaf_destroy(&dl_port->dl_port);
 
 	devl_port_unregister(&dl_port->dl_port);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
index faccc60fc93a8..0d20f51b97021 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
@@ -11,57 +11,8 @@
 /* Minimum supported BW share value by the HW is 1 Mbit/sec */
 #define MLX5_MIN_BW_SHARE 1
 
-/* Holds rate nodes associated with an E-Switch. */
-struct mlx5_qos_domain {
-	/* Serializes access to all qos changes in the qos domain. */
-	struct mutex lock;
-	/* List of all mlx5_esw_sched_nodes. */
-	struct list_head nodes;
-};
-
-static void esw_qos_lock(struct mlx5_eswitch *esw)
-{
-	mutex_lock(&esw->qos.domain->lock);
-}
-
-static void esw_qos_unlock(struct mlx5_eswitch *esw)
-{
-	mutex_unlock(&esw->qos.domain->lock);
-}
-
-static void esw_assert_qos_lock_held(struct mlx5_eswitch *esw)
-{
-	lockdep_assert_held(&esw->qos.domain->lock);
-}
-
-static struct mlx5_qos_domain *esw_qos_domain_alloc(void)
-{
-	struct mlx5_qos_domain *qos_domain;
-
-	qos_domain = kzalloc_obj(*qos_domain);
-	if (!qos_domain)
-		return NULL;
-
-	mutex_init(&qos_domain->lock);
-	INIT_LIST_HEAD(&qos_domain->nodes);
-
-	return qos_domain;
-}
-
-static int esw_qos_domain_init(struct mlx5_eswitch *esw)
-{
-	esw->qos.domain = esw_qos_domain_alloc();
-
-	return esw->qos.domain ? 0 : -ENOMEM;
-}
-
-static void esw_qos_domain_release(struct mlx5_eswitch *esw)
-{
-	kfree(esw->qos.domain);
-	esw->qos.domain = NULL;
-}
-
 enum sched_node_type {
+	SCHED_NODE_TYPE_ROOT,
 	SCHED_NODE_TYPE_VPORTS_TSAR,
 	SCHED_NODE_TYPE_VPORT,
 	SCHED_NODE_TYPE_TC_ARBITER_TSAR,
@@ -94,7 +45,9 @@ struct mlx5_esw_sched_node {
 	enum sched_node_type type;
 	/* The eswitch this node belongs to. */
 	struct mlx5_eswitch *esw;
-	/* The children nodes of this node, empty list for leaf nodes. */
+	/* The children nodes of this node, empty list for leaf nodes.
+	 * Can be from multiple E-Switches.
+	 */
 	struct list_head children;
 	/* Valid only if this node is associated with a vport. */
 	struct mlx5_vport *vport;
@@ -106,16 +59,63 @@ struct mlx5_esw_sched_node {
 	u32 tc_bw[DEVLINK_RATE_TCS_MAX];
 };
 
-static void esw_qos_node_attach_to_parent(struct mlx5_esw_sched_node *node)
+/* Locking notes:
+ * QoS changes are normally protected by the shd lock. But on older HW shd
+ * might not be created at all, so there needs to be a fallback serialization
+ * mechanism. This is esw->state_lock.
+ * Callers into QoS hold a combination of RTNL, devlink instance lock and
+ * esw->state_lock. Devlink rate ops additionally hold the shd lock if it
+ * exists.
+ * - VF rate ops use esw_qos_lock/esw_qos_unlock.
+ * - callers with esw->state_lock held use esw_qos_shd_lock/esw_qos_shd_unlock.
+ * - devlink callers use esw_qos_devlink_lock/esw_qos_devlink_unlock.
+ */
+static void esw_assert_qos_lock_held(struct mlx5_core_dev *dev)
 {
-	if (!node->parent) {
-		/* Root children are assigned a depth level of 2. */
-		node->level = 2;
-		list_add_tail(&node->entry, &node->esw->qos.domain->nodes);
-	} else {
-		node->level = node->parent->level + 1;
-		list_add_tail(&node->entry, &node->parent->children);
-	}
+	if (dev->shd)
+		devl_assert_locked(dev->shd);
+	else
+		lockdep_assert_held(&dev->priv.eswitch->state_lock);
+}
+
+static void esw_qos_lock(struct mlx5_core_dev *dev)
+{
+	if (dev->shd)
+		devl_lock(dev->shd);
+	else
+		mutex_lock(&dev->priv.eswitch->state_lock);
+}
+
+static void esw_qos_unlock(struct mlx5_core_dev *dev)
+{
+	if (dev->shd)
+		devl_unlock(dev->shd);
+	else
+		mutex_unlock(&dev->priv.eswitch->state_lock);
+}
+
+static void esw_qos_shd_lock(struct mlx5_core_dev *dev)
+{
+	if (dev->shd)
+		devl_lock(dev->shd);
+}
+
+static void esw_qos_shd_unlock(struct mlx5_core_dev *dev)
+{
+	if (dev->shd)
+		devl_unlock(dev->shd);
+}
+
+static void esw_qos_devlink_lock(struct mlx5_core_dev *dev)
+{
+	if (!dev->shd)
+		mutex_lock(&dev->priv.eswitch->state_lock);
+}
+
+static void esw_qos_devlink_unlock(struct mlx5_core_dev *dev)
+{
+	if (!dev->shd)
+		mutex_unlock(&dev->priv.eswitch->state_lock);
 }
 
 static int esw_qos_num_tcs(struct mlx5_core_dev *dev)
@@ -125,14 +125,14 @@ static int esw_qos_num_tcs(struct mlx5_core_dev *dev)
 	return num_tcs < DEVLINK_RATE_TCS_MAX ? num_tcs : DEVLINK_RATE_TCS_MAX;
 }
 
-static void
-esw_qos_node_set_parent(struct mlx5_esw_sched_node *node, struct mlx5_esw_sched_node *parent)
+static void esw_qos_node_set_parent(struct mlx5_esw_sched_node *node,
+				    struct mlx5_esw_sched_node *parent)
 {
-	list_del_init(&node->entry);
 	node->parent = parent;
-	if (parent)
-		node->esw = parent->esw;
-	esw_qos_node_attach_to_parent(node);
+	node->esw = parent->esw;
+	node->level = parent->level + 1;
+	list_del(&node->entry);
+	list_add_tail(&node->entry, &parent->children);
 }
 
 static void esw_qos_nodes_set_parent(struct list_head *nodes,
@@ -321,22 +321,19 @@ static int esw_qos_create_rate_limit_element(struct mlx5_esw_sched_node *node,
 	return esw_qos_node_create_sched_element(node, sched_ctx, extack);
 }
 
-static u32 esw_qos_calculate_min_rate_divider(struct mlx5_eswitch *esw,
-					      struct mlx5_esw_sched_node *parent)
+static u32
+esw_qos_calculate_min_rate_divider(struct mlx5_esw_sched_node *parent)
 {
-	struct list_head *nodes = parent ? &parent->children : &esw->qos.domain->nodes;
-	u32 fw_max_bw_share = MLX5_CAP_QOS(esw->dev, max_tsar_bw_share);
+	u32 fw_max_bw_share = MLX5_CAP_QOS(parent->esw->dev, max_tsar_bw_share);
 	struct mlx5_esw_sched_node *node;
 	u32 max_guarantee = 0;
 
 	/* Find max min_rate across all nodes.
 	 * This will correspond to fw_max_bw_share in the final bw_share calculation.
 	 */
-	list_for_each_entry(node, nodes, entry) {
-		if (node->esw == esw && node->ix != esw->qos.root_tsar_ix &&
-		    node->min_rate > max_guarantee)
+	list_for_each_entry(node, &parent->children, entry)
+		if (node->min_rate > max_guarantee)
 			max_guarantee = node->min_rate;
-	}
 
 	if (max_guarantee)
 		return max_t(u32, max_guarantee / fw_max_bw_share, 1);
@@ -368,18 +365,13 @@ static void esw_qos_update_sched_node_bw_share(struct mlx5_esw_sched_node *node,
 	esw_qos_sched_elem_config(node, node->max_rate, bw_share, extack);
 }
 
-static void esw_qos_normalize_min_rate(struct mlx5_eswitch *esw,
-				       struct mlx5_esw_sched_node *parent,
+static void esw_qos_normalize_min_rate(struct mlx5_esw_sched_node *parent,
 				       struct netlink_ext_ack *extack)
 {
-	struct list_head *nodes = parent ? &parent->children : &esw->qos.domain->nodes;
-	u32 divider = esw_qos_calculate_min_rate_divider(esw, parent);
+	u32 divider = esw_qos_calculate_min_rate_divider(parent);
 	struct mlx5_esw_sched_node *node;
 
-	list_for_each_entry(node, nodes, entry) {
-		if (node->esw != esw || node->ix == esw->qos.root_tsar_ix)
-			continue;
-
+	list_for_each_entry(node, &parent->children, entry) {
 		/* Vports TC TSARs don't have a minimum rate configured,
 		 * so there's no need to update the bw_share on them.
 		 */
@@ -391,7 +383,7 @@ static void esw_qos_normalize_min_rate(struct mlx5_eswitch *esw,
 		if (list_empty(&node->children))
 			continue;
 
-		esw_qos_normalize_min_rate(node->esw, node, extack);
+		esw_qos_normalize_min_rate(node, extack);
 	}
 }
 
@@ -412,14 +404,11 @@ static u32 esw_qos_calculate_tc_bw_divider(u32 *tc_bw)
 static int esw_qos_set_node_min_rate(struct mlx5_esw_sched_node *node,
 				     u32 min_rate, struct netlink_ext_ack *extack)
 {
-	struct mlx5_eswitch *esw = node->esw;
-
 	if (min_rate == node->min_rate)
 		return 0;
 
 	node->min_rate = min_rate;
-	esw_qos_normalize_min_rate(esw, node->parent, extack);
-
+	esw_qos_normalize_min_rate(node->parent, extack);
 	return 0;
 }
 
@@ -460,6 +449,7 @@ esw_qos_vport_create_sched_element(struct mlx5_esw_sched_node *vport_node,
 	struct mlx5_esw_sched_node *parent = vport_node->parent;
 	u32 sched_ctx[MLX5_ST_SZ_DW(scheduling_context)] = {};
 	struct mlx5_core_dev *dev = vport_node->esw->dev;
+	struct mlx5_vport *vport = vport_node->vport;
 	void *attr;
 
 	if (!mlx5_qos_element_type_supported(
@@ -471,11 +461,17 @@ esw_qos_vport_create_sched_element(struct mlx5_esw_sched_node *vport_node,
 	MLX5_SET(scheduling_context, sched_ctx, element_type,
 		 SCHEDULING_CONTEXT_ELEMENT_TYPE_VPORT);
 	attr = MLX5_ADDR_OF(scheduling_context, sched_ctx, element_attributes);
-	MLX5_SET(vport_element, attr, vport_number, vport_node->vport->vport);
-	MLX5_SET(scheduling_context, sched_ctx, parent_element_id,
-		 parent ? parent->ix : vport_node->esw->qos.root_tsar_ix);
+	MLX5_SET(vport_element, attr, vport_number, vport->vport);
+	MLX5_SET(scheduling_context, sched_ctx, parent_element_id, parent->ix);
 	MLX5_SET(scheduling_context, sched_ctx, max_average_bw,
 		 vport_node->max_rate);
+	if (vport->dev != dev) {
+		/* The port is assigned to a node on another eswitch. */
+		MLX5_SET(vport_element, attr, eswitch_owner_vhca_id_valid,
+			 true);
+		MLX5_SET(vport_element, attr, eswitch_owner_vhca_id,
+			 MLX5_CAP_GEN(vport->dev, vhca_id));
+	}
 
 	return esw_qos_node_create_sched_element(vport_node, sched_ctx, extack);
 }
@@ -487,6 +483,7 @@ esw_qos_vport_tc_create_sched_element(struct mlx5_esw_sched_node *vport_tc_node,
 {
 	u32 sched_ctx[MLX5_ST_SZ_DW(scheduling_context)] = {};
 	struct mlx5_core_dev *dev = vport_tc_node->esw->dev;
+	struct mlx5_vport *vport = vport_tc_node->vport;
 	void *attr;
 
 	if (!mlx5_qos_element_type_supported(
@@ -498,8 +495,7 @@ esw_qos_vport_tc_create_sched_element(struct mlx5_esw_sched_node *vport_tc_node,
 	MLX5_SET(scheduling_context, sched_ctx, element_type,
 		 SCHEDULING_CONTEXT_ELEMENT_TYPE_VPORT_TC);
 	attr = MLX5_ADDR_OF(scheduling_context, sched_ctx, element_attributes);
-	MLX5_SET(vport_tc_element, attr, vport_number,
-		 vport_tc_node->vport->vport);
+	MLX5_SET(vport_tc_element, attr, vport_number, vport->vport);
 	MLX5_SET(vport_tc_element, attr, traffic_class, vport_tc_node->tc);
 	MLX5_SET(scheduling_context, sched_ctx, max_bw_obj_id,
 		 rate_limit_elem_ix);
@@ -507,13 +503,20 @@ esw_qos_vport_tc_create_sched_element(struct mlx5_esw_sched_node *vport_tc_node,
 		 vport_tc_node->parent->ix);
 	MLX5_SET(scheduling_context, sched_ctx, bw_share,
 		 vport_tc_node->bw_share);
+	if (vport->dev != dev) {
+		/* The port is assigned to a node on another eswitch. */
+		MLX5_SET(vport_tc_element, attr, eswitch_owner_vhca_id_valid,
+			 true);
+		MLX5_SET(vport_tc_element, attr, eswitch_owner_vhca_id,
+			 MLX5_CAP_GEN(vport->dev, vhca_id));
+	}
 
 	return esw_qos_node_create_sched_element(vport_tc_node, sched_ctx,
 						 extack);
 }
 
 static struct mlx5_esw_sched_node *
-__esw_qos_alloc_node(struct mlx5_eswitch *esw, u32 tsar_ix, enum sched_node_type type,
+__esw_qos_alloc_node(u32 tsar_ix, enum sched_node_type type,
 		     struct mlx5_esw_sched_node *parent)
 {
 	struct mlx5_esw_sched_node *node;
@@ -522,20 +525,12 @@ __esw_qos_alloc_node(struct mlx5_eswitch *esw, u32 tsar_ix, enum sched_node_type
 	if (!node)
 		return NULL;
 
-	node->esw = esw;
 	node->ix = tsar_ix;
 	node->type = type;
-	node->parent = parent;
 	INIT_LIST_HEAD(&node->children);
-	esw_qos_node_attach_to_parent(node);
-	if (!parent) {
-		/* The caller is responsible for inserting the node into the
-		 * parent list if necessary. This function can also be used with
-		 * a NULL parent, which doesn't necessarily indicate that it
-		 * refers to the root scheduling element.
-		 */
-		list_del_init(&node->entry);
-	}
+	INIT_LIST_HEAD(&node->entry);
+	if (parent)
+		esw_qos_node_set_parent(node, parent);
 
 	return node;
 }
@@ -570,7 +565,7 @@ static int esw_qos_create_vports_tc_node(struct mlx5_esw_sched_node *parent,
 					  SCHEDULING_HIERARCHY_E_SWITCH))
 		return -EOPNOTSUPP;
 
-	vports_tc_node = __esw_qos_alloc_node(parent->esw, 0,
+	vports_tc_node = __esw_qos_alloc_node(0,
 					      SCHED_NODE_TYPE_VPORTS_TC_TSAR,
 					      parent);
 	if (!vports_tc_node) {
@@ -665,7 +660,6 @@ static int esw_qos_create_tc_arbiter_sched_elem(
 		struct netlink_ext_ack *extack)
 {
 	u32 tsar_ctx[MLX5_ST_SZ_DW(scheduling_context)] = {};
-	u32 tsar_parent_ix;
 	void *attr;
 
 	if (!mlx5_qos_tsar_type_supported(tc_arbiter_node->esw->dev,
@@ -678,10 +672,8 @@ static int esw_qos_create_tc_arbiter_sched_elem(
 
 	attr = MLX5_ADDR_OF(scheduling_context, tsar_ctx, element_attributes);
 	MLX5_SET(tsar_element, attr, tsar_type, TSAR_ELEMENT_TSAR_TYPE_TC_ARB);
-	tsar_parent_ix = tc_arbiter_node->parent ? tc_arbiter_node->parent->ix :
-			 tc_arbiter_node->esw->qos.root_tsar_ix;
 	MLX5_SET(scheduling_context, tsar_ctx, parent_element_id,
-		 tsar_parent_ix);
+		 tc_arbiter_node->parent->ix);
 	MLX5_SET(scheduling_context, tsar_ctx, element_type,
 		 SCHEDULING_CONTEXT_ELEMENT_TYPE_TSAR);
 	MLX5_SET(scheduling_context, tsar_ctx, max_average_bw,
@@ -694,37 +686,36 @@ static int esw_qos_create_tc_arbiter_sched_elem(
 }
 
 static struct mlx5_esw_sched_node *
-__esw_qos_create_vports_sched_node(struct mlx5_eswitch *esw, struct mlx5_esw_sched_node *parent,
+__esw_qos_create_vports_sched_node(struct mlx5_esw_sched_node *parent,
 				   struct netlink_ext_ack *extack)
 {
 	struct mlx5_esw_sched_node *node;
-	u32 tsar_ix;
 	int err;
+	u32 ix;
 
-	err = esw_qos_create_node_sched_elem(esw->dev, esw->qos.root_tsar_ix, 0,
-					     0, &tsar_ix);
+	err = esw_qos_create_node_sched_elem(parent->esw->dev, parent->ix, 0, 0,
+					     &ix);
 	if (err) {
 		NL_SET_ERR_MSG_MOD(extack, "E-Switch create TSAR for node failed");
 		return ERR_PTR(err);
 	}
 
-	node = __esw_qos_alloc_node(esw, tsar_ix, SCHED_NODE_TYPE_VPORTS_TSAR, parent);
+	node = __esw_qos_alloc_node(ix, SCHED_NODE_TYPE_VPORTS_TSAR, parent);
 	if (!node) {
 		NL_SET_ERR_MSG_MOD(extack, "E-Switch alloc node failed");
 		err = -ENOMEM;
 		goto err_alloc_node;
 	}
 
-	list_add_tail(&node->entry, &esw->qos.domain->nodes);
-	esw_qos_normalize_min_rate(esw, NULL, extack);
-	trace_mlx5_esw_node_qos_create(esw->dev, node, node->ix);
+	esw_qos_normalize_min_rate(parent, extack);
+	trace_mlx5_esw_node_qos_create(parent->esw->dev, node, node->ix);
 
 	return node;
 
 err_alloc_node:
-	if (mlx5_destroy_scheduling_element_cmd(esw->dev,
+	if (mlx5_destroy_scheduling_element_cmd(parent->esw->dev,
 						SCHEDULING_HIERARCHY_E_SWITCH,
-						tsar_ix))
+						ix))
 		NL_SET_ERR_MSG_MOD(extack, "E-Switch destroy TSAR for node failed");
 	return ERR_PTR(err);
 }
@@ -738,7 +729,7 @@ esw_qos_create_vports_sched_node(struct mlx5_eswitch *esw, struct netlink_ext_ac
 	struct mlx5_esw_sched_node *node;
 	int err;
 
-	esw_assert_qos_lock_held(esw);
+	esw_assert_qos_lock_held(esw->dev);
 	if (!MLX5_CAP_QOS(esw->dev, log_esw_max_sched_depth))
 		return ERR_PTR(-EOPNOTSUPP);
 
@@ -746,7 +737,7 @@ esw_qos_create_vports_sched_node(struct mlx5_eswitch *esw, struct netlink_ext_ac
 	if (err)
 		return ERR_PTR(err);
 
-	node = __esw_qos_create_vports_sched_node(esw, NULL, extack);
+	node = __esw_qos_create_vports_sched_node(esw->qos.root, extack);
 	if (IS_ERR(node))
 		esw_qos_put(esw);
 
@@ -762,45 +753,54 @@ static void __esw_qos_destroy_node(struct mlx5_esw_sched_node *node, struct netl
 
 	trace_mlx5_esw_node_qos_destroy(esw->dev, node, node->ix);
 	esw_qos_destroy_node(node, extack);
-	esw_qos_normalize_min_rate(esw, NULL, extack);
+	esw_qos_normalize_min_rate(esw->qos.root, extack);
 }
 
 static int esw_qos_create(struct mlx5_eswitch *esw, struct netlink_ext_ack *extack)
 {
 	struct mlx5_core_dev *dev = esw->dev;
+	struct mlx5_esw_sched_node *root;
+	u32 root_ix;
 	int err;
 
 	if (!MLX5_CAP_GEN(dev, qos) || !MLX5_CAP_QOS(dev, esw_scheduling))
 		return -EOPNOTSUPP;
 
-	err = esw_qos_create_node_sched_elem(esw->dev, 0, 0, 0,
-					     &esw->qos.root_tsar_ix);
+	err = esw_qos_create_node_sched_elem(esw->dev, 0, 0, 0, &root_ix);
 	if (err) {
 		esw_warn(dev, "E-Switch create root TSAR failed (%d)\n", err);
 		return err;
 	}
 
+	root = __esw_qos_alloc_node(root_ix, SCHED_NODE_TYPE_ROOT, NULL);
+	if (!root) {
+		esw_warn(dev, "E-Switch create root node failed\n");
+		err = -ENOMEM;
+		goto out_err;
+	}
+	root->esw = esw;
+	root->level = 1;
+	esw->qos.root = root;
 	refcount_set(&esw->qos.refcnt, 1);
 
 	return 0;
+out_err:
+	mlx5_destroy_scheduling_element_cmd(dev, SCHEDULING_HIERARCHY_E_SWITCH,
+					    root_ix);
+	return err;
 }
 
 static void esw_qos_destroy(struct mlx5_eswitch *esw)
 {
-	int err;
-
-	err = mlx5_destroy_scheduling_element_cmd(esw->dev,
-						  SCHEDULING_HIERARCHY_E_SWITCH,
-						  esw->qos.root_tsar_ix);
-	if (err)
-		esw_warn(esw->dev, "E-Switch destroy root TSAR failed (%d)\n", err);
+	esw_qos_destroy_node(esw->qos.root, NULL);
+	esw->qos.root = NULL;
 }
 
 static int esw_qos_get(struct mlx5_eswitch *esw, struct netlink_ext_ack *extack)
 {
 	int err = 0;
 
-	esw_assert_qos_lock_held(esw);
+	esw_assert_qos_lock_held(esw->dev);
 	if (!refcount_inc_not_zero(&esw->qos.refcnt)) {
 		/* esw_qos_create() set refcount to 1 only on success.
 		 * No need to decrement on failure.
@@ -813,7 +813,7 @@ static int esw_qos_get(struct mlx5_eswitch *esw, struct netlink_ext_ack *extack)
 
 static void esw_qos_put(struct mlx5_eswitch *esw)
 {
-	esw_assert_qos_lock_held(esw);
+	esw_assert_qos_lock_held(esw->dev);
 	if (refcount_dec_and_test(&esw->qos.refcnt))
 		esw_qos_destroy(esw);
 }
@@ -866,8 +866,7 @@ esw_qos_create_vport_tc_sched_node(struct mlx5_vport *vport,
 	u8 tc = vports_tc_node->tc;
 	int err;
 
-	vport_tc_node = __esw_qos_alloc_node(vport_node->esw, 0,
-					     SCHED_NODE_TYPE_VPORT_TC,
+	vport_tc_node = __esw_qos_alloc_node(0, SCHED_NODE_TYPE_VPORT_TC,
 					     vports_tc_node);
 	if (!vport_tc_node)
 		return -ENOMEM;
@@ -959,7 +958,7 @@ esw_qos_vport_tc_enable(struct mlx5_vport *vport, enum sched_node_type type,
 		/* Increase the parent's level by 2 to account for both the
 		 * TC arbiter and the vports TC scheduling element.
 		 */
-		new_level = (parent ? parent->level : 2) + 2;
+		new_level = parent->level + 2;
 		max_level = 1 << MLX5_CAP_QOS(vport_node->esw->dev,
 					      log_esw_max_sched_depth);
 		if (new_level > max_level) {
@@ -970,7 +969,7 @@ esw_qos_vport_tc_enable(struct mlx5_vport *vport, enum sched_node_type type,
 		}
 	}
 
-	esw_assert_qos_lock_held(vport->dev->priv.eswitch);
+	esw_assert_qos_lock_held(vport->dev);
 
 	if (type == SCHED_NODE_TYPE_RATE_LIMITER)
 		err = esw_qos_create_rate_limit_element(vport_node, extack);
@@ -997,7 +996,7 @@ esw_qos_vport_tc_enable(struct mlx5_vport *vport, enum sched_node_type type,
 err_sched_nodes:
 	if (type == SCHED_NODE_TYPE_RATE_LIMITER) {
 		esw_qos_node_destroy_sched_element(vport_node, NULL);
-		esw_qos_node_attach_to_parent(vport_node);
+		esw_qos_node_set_parent(vport_node, vport_node->parent);
 	} else {
 		esw_qos_tc_arbiter_scheduling_teardown(vport_node, NULL);
 	}
@@ -1055,7 +1054,7 @@ static void esw_qos_vport_disable(struct mlx5_vport *vport, struct netlink_ext_a
 	vport_node->bw_share = 0;
 	memset(vport_node->tc_bw, 0, sizeof(vport_node->tc_bw));
 	list_del_init(&vport_node->entry);
-	esw_qos_normalize_min_rate(vport_node->esw, vport_node->parent, extack);
+	esw_qos_normalize_min_rate(vport_node->parent, extack);
 
 	trace_mlx5_esw_vport_qos_destroy(vport_node->esw->dev, vport);
 }
@@ -1068,7 +1067,7 @@ static int esw_qos_vport_enable(struct mlx5_vport *vport,
 	struct mlx5_esw_sched_node *vport_node = vport->qos.sched_node;
 	int err;
 
-	esw_assert_qos_lock_held(vport->dev->priv.eswitch);
+	esw_assert_qos_lock_held(vport->dev);
 
 	esw_qos_node_set_parent(vport_node, parent);
 	if (type == SCHED_NODE_TYPE_VPORT)
@@ -1079,9 +1078,10 @@ static int esw_qos_vport_enable(struct mlx5_vport *vport,
 		return err;
 
 	vport_node->type = type;
-	esw_qos_normalize_min_rate(vport_node->esw, parent, extack);
-	trace_mlx5_esw_vport_qos_create(vport->dev, vport, vport_node->max_rate,
-					vport_node->bw_share);
+	esw_qos_normalize_min_rate(parent, extack);
+	trace_mlx5_esw_vport_qos_create(vport_node->esw->dev, vport,
+					vport_node->bw_share,
+					vport_node->max_rate);
 
 	return 0;
 }
@@ -1092,22 +1092,20 @@ static int mlx5_esw_qos_vport_enable(struct mlx5_vport *vport, enum sched_node_t
 {
 	struct mlx5_eswitch *esw = vport->dev->priv.eswitch;
 	struct mlx5_esw_sched_node *sched_node;
-	struct mlx5_eswitch *parent_esw;
 	int err;
 
-	esw_assert_qos_lock_held(esw);
+	esw_assert_qos_lock_held(vport->dev);
 	err = esw_qos_get(esw, extack);
 	if (err)
 		return err;
 
-	parent_esw = parent ? parent->esw : esw;
-	sched_node = __esw_qos_alloc_node(parent_esw, 0, type, parent);
+	if (!parent)
+		parent = esw->qos.root;
+	sched_node = __esw_qos_alloc_node(0, type, parent);
 	if (!sched_node) {
 		esw_qos_put(esw);
 		return -ENOMEM;
 	}
-	if (!parent)
-		list_add_tail(&sched_node->entry, &esw->qos.domain->nodes);
 
 	sched_node->max_rate = max_rate;
 	sched_node->min_rate = min_rate;
@@ -1125,33 +1123,23 @@ static int mlx5_esw_qos_vport_enable(struct mlx5_vport *vport, enum sched_node_t
 
 static void mlx5_esw_qos_vport_disable_locked(struct mlx5_vport *vport)
 {
-	struct mlx5_eswitch *esw = vport->dev->priv.eswitch;
-
-	esw_assert_qos_lock_held(esw);
+	esw_assert_qos_lock_held(vport->dev);
 	if (!vport->qos.sched_node)
 		return;
 
 	esw_qos_vport_disable(vport, NULL);
 	mlx5_esw_qos_vport_qos_free(vport);
-	esw_qos_put(esw);
+	esw_qos_put(vport->dev->priv.eswitch);
 }
 
 void mlx5_esw_qos_vport_disable(struct mlx5_vport *vport)
 {
 	struct mlx5_eswitch *esw = vport->dev->priv.eswitch;
-	struct mlx5_esw_sched_node *parent;
 
 	lockdep_assert_held(&esw->state_lock);
-	esw_qos_lock(esw);
-	if (!vport->qos.sched_node)
-		goto unlock;
-
-	parent = vport->qos.sched_node->parent;
-	WARN(parent, "Disabling QoS on port before detaching it from node");
-
+	esw_qos_shd_lock(vport->dev);
 	mlx5_esw_qos_vport_disable_locked(vport);
-unlock:
-	esw_qos_unlock(esw);
+	esw_qos_shd_unlock(vport->dev);
 }
 
 static int mlx5_esw_qos_set_vport_max_rate(struct mlx5_vport *vport, u32 max_rate,
@@ -1159,7 +1147,7 @@ static int mlx5_esw_qos_set_vport_max_rate(struct mlx5_vport *vport, u32 max_rat
 {
 	struct mlx5_esw_sched_node *vport_node = vport->qos.sched_node;
 
-	esw_assert_qos_lock_held(vport->dev->priv.eswitch);
+	esw_assert_qos_lock_held(vport->dev);
 
 	if (!vport_node)
 		return mlx5_esw_qos_vport_enable(vport, SCHED_NODE_TYPE_VPORT, NULL, max_rate, 0,
@@ -1174,7 +1162,7 @@ static int mlx5_esw_qos_set_vport_min_rate(struct mlx5_vport *vport, u32 min_rat
 {
 	struct mlx5_esw_sched_node *vport_node = vport->qos.sched_node;
 
-	esw_assert_qos_lock_held(vport->dev->priv.eswitch);
+	esw_assert_qos_lock_held(vport->dev);
 
 	if (!vport_node)
 		return mlx5_esw_qos_vport_enable(vport, SCHED_NODE_TYPE_VPORT, NULL, 0, min_rate,
@@ -1187,29 +1175,27 @@ static int mlx5_esw_qos_set_vport_min_rate(struct mlx5_vport *vport, u32 min_rat
 
 int mlx5_esw_qos_set_vport_rate(struct mlx5_vport *vport, u32 max_rate, u32 min_rate)
 {
-	struct mlx5_eswitch *esw = vport->dev->priv.eswitch;
 	int err;
 
-	esw_qos_lock(esw);
+	esw_qos_lock(vport->dev);
 	err = mlx5_esw_qos_set_vport_min_rate(vport, min_rate, NULL);
 	if (!err)
 		err = mlx5_esw_qos_set_vport_max_rate(vport, max_rate, NULL);
-	esw_qos_unlock(esw);
+	esw_qos_unlock(vport->dev);
 	return err;
 }
 
 bool mlx5_esw_qos_get_vport_rate(struct mlx5_vport *vport, u32 *max_rate, u32 *min_rate)
 {
-	struct mlx5_eswitch *esw = vport->dev->priv.eswitch;
 	bool enabled;
 
-	esw_qos_lock(esw);
+	esw_qos_shd_lock(vport->dev);
 	enabled = !!vport->qos.sched_node;
 	if (enabled) {
 		*max_rate = vport->qos.sched_node->max_rate;
 		*min_rate = vport->qos.sched_node->min_rate;
 	}
-	esw_qos_unlock(esw);
+	esw_qos_shd_unlock(vport->dev);
 	return enabled;
 }
 
@@ -1234,6 +1220,28 @@ static int esw_qos_vport_tc_check_type(enum sched_node_type curr_type,
 	return 0;
 }
 
+static bool esw_qos_validate_unsupported_tc_bw(struct mlx5_eswitch *esw,
+					       u32 *tc_bw)
+{
+	int i, num_tcs = esw_qos_num_tcs(esw->dev);
+
+	for (i = num_tcs; i < DEVLINK_RATE_TCS_MAX; i++)
+		if (tc_bw[i])
+			return false;
+
+	return true;
+}
+
+static bool esw_qos_vport_validate_unsupported_tc_bw(struct mlx5_vport *vport,
+						     u32 *tc_bw)
+{
+	struct mlx5_esw_sched_node *node = vport->qos.sched_node;
+	struct mlx5_eswitch *esw = node ?
+		node->parent->esw : vport->dev->priv.eswitch;
+
+	return esw_qos_validate_unsupported_tc_bw(esw, tc_bw);
+}
+
 static int esw_qos_vport_update(struct mlx5_vport *vport,
 				enum sched_node_type type,
 				struct mlx5_esw_sched_node *parent,
@@ -1245,7 +1253,7 @@ static int esw_qos_vport_update(struct mlx5_vport *vport,
 	u32 curr_tc_bw[DEVLINK_RATE_TCS_MAX] = {0};
 	int err;
 
-	esw_assert_qos_lock_held(vport->dev->priv.eswitch);
+	esw_assert_qos_lock_held(vport->dev);
 	if (curr_type == type && curr_parent == parent)
 		return 0;
 
@@ -1253,8 +1261,15 @@ static int esw_qos_vport_update(struct mlx5_vport *vport,
 	if (err)
 		return err;
 
-	if (curr_type == SCHED_NODE_TYPE_TC_ARBITER_TSAR && curr_type == type)
+	if (curr_type == SCHED_NODE_TYPE_TC_ARBITER_TSAR && curr_type == type) {
 		esw_qos_tc_arbiter_get_bw_shares(vport_node, curr_tc_bw);
+		if (!esw_qos_validate_unsupported_tc_bw(parent->esw,
+							curr_tc_bw)) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Unsupported traffic classes on the new device");
+			return -EOPNOTSUPP;
+		}
+	}
 
 	esw_qos_vport_disable(vport, extack);
 
@@ -1275,11 +1290,10 @@ static int esw_qos_vport_update(struct mlx5_vport *vport,
 static int esw_qos_vport_update_parent(struct mlx5_vport *vport, struct mlx5_esw_sched_node *parent,
 				       struct netlink_ext_ack *extack)
 {
-	struct mlx5_eswitch *esw = vport->dev->priv.eswitch;
 	struct mlx5_esw_sched_node *curr_parent;
 	enum sched_node_type type;
 
-	esw_assert_qos_lock_held(esw);
+	esw_assert_qos_lock_held(vport->dev);
 	curr_parent = vport->qos.sched_node->parent;
 	if (curr_parent == parent)
 		return 0;
@@ -1287,10 +1301,9 @@ static int esw_qos_vport_update_parent(struct mlx5_vport *vport, struct mlx5_esw
 	/* Set vport QoS type based on parent node type if different from
 	 * default QoS; otherwise, use the vport's current QoS type.
 	 */
-	if (parent && parent->type == SCHED_NODE_TYPE_TC_ARBITER_TSAR)
+	if (parent->type == SCHED_NODE_TYPE_TC_ARBITER_TSAR)
 		type = SCHED_NODE_TYPE_RATE_LIMITER;
-	else if (curr_parent &&
-		 curr_parent->type == SCHED_NODE_TYPE_TC_ARBITER_TSAR)
+	else if (curr_parent->type == SCHED_NODE_TYPE_TC_ARBITER_TSAR)
 		type = SCHED_NODE_TYPE_VPORT;
 	else
 		type = vport->qos.sched_node->type;
@@ -1319,11 +1332,9 @@ static int esw_qos_switch_tc_arbiter_node_to_vports(
 	struct mlx5_esw_sched_node *node,
 	struct netlink_ext_ack *extack)
 {
-	u32 parent_tsar_ix = node->parent ?
-			     node->parent->ix : node->esw->qos.root_tsar_ix;
 	int err;
 
-	err = esw_qos_create_node_sched_elem(node->esw->dev, parent_tsar_ix,
+	err = esw_qos_create_node_sched_elem(node->esw->dev, node->parent->ix,
 					     node->max_rate, node->bw_share,
 					     &node->ix);
 	if (err) {
@@ -1378,8 +1389,8 @@ esw_qos_move_node(struct mlx5_esw_sched_node *curr_node)
 {
 	struct mlx5_esw_sched_node *new_node;
 
-	new_node = __esw_qos_alloc_node(curr_node->esw, curr_node->ix,
-					curr_node->type, NULL);
+	new_node = __esw_qos_alloc_node(curr_node->ix, curr_node->type,
+					curr_node->parent);
 	if (!new_node)
 		return ERR_PTR(-ENOMEM);
 
@@ -1489,41 +1500,16 @@ static int esw_qos_node_enable_tc_arbitration(struct mlx5_esw_sched_node *node,
 	return err;
 }
 
-static u32 mlx5_esw_qos_lag_link_speed_get(struct mlx5_core_dev *mdev,
-					   bool take_rtnl)
-{
-	struct ethtool_link_ksettings lksettings;
-	struct net_device *slave, *master;
-	u32 speed = SPEED_UNKNOWN;
-
-	slave = mlx5_uplink_netdev_get(mdev);
-	if (!slave)
-		goto out;
-
-	if (take_rtnl)
-		rtnl_lock();
-	master = netdev_master_upper_dev_get(slave);
-	if (master && !__ethtool_get_link_ksettings(master, &lksettings))
-		speed = lksettings.base.speed;
-	if (take_rtnl)
-		rtnl_unlock();
-
-out:
-	mlx5_uplink_netdev_put(mdev, slave);
-	return speed;
-}
-
 static int mlx5_esw_qos_max_link_speed_get(struct mlx5_core_dev *mdev, u32 *link_speed_max,
-					   bool take_rtnl,
 					   struct netlink_ext_ack *extack)
 {
 	int err;
 
-	if (!mlx5_lag_is_active(mdev))
+	if (!mlx5_lag_is_active(mdev) ||
+	    mlx5_lag_query_bond_speed(mdev, link_speed_max) < 0 ||
+	    *link_speed_max == 0)
 		goto skip_lag;
 
-	*link_speed_max = mlx5_esw_qos_lag_link_speed_get(mdev, take_rtnl);
-
 	if (*link_speed_max != (u32)SPEED_UNKNOWN)
 		return 0;
 
@@ -1560,7 +1546,8 @@ int mlx5_esw_qos_modify_vport_rate(struct mlx5_eswitch *esw, u16 vport_num, u32
 		return PTR_ERR(vport);
 
 	if (rate_mbps) {
-		err = mlx5_esw_qos_max_link_speed_get(esw->dev, &link_speed_max, false, NULL);
+		err = mlx5_esw_qos_max_link_speed_get(esw->dev, &link_speed_max,
+						      NULL);
 		if (err)
 			return err;
 
@@ -1570,9 +1557,9 @@ int mlx5_esw_qos_modify_vport_rate(struct mlx5_eswitch *esw, u16 vport_num, u32
 			return err;
 	}
 
-	esw_qos_lock(esw);
+	esw_qos_lock(vport->dev);
 	err = mlx5_esw_qos_set_vport_max_rate(vport, rate_mbps, NULL);
-	esw_qos_unlock(esw);
+	esw_qos_unlock(vport->dev);
 
 	return err;
 }
@@ -1598,7 +1585,7 @@ static int esw_qos_devlink_rate_to_mbps(struct mlx5_core_dev *mdev, const char *
 		return -EINVAL;
 	}
 
-	err = mlx5_esw_qos_max_link_speed_get(mdev, &link_speed_max, true, extack);
+	err = mlx5_esw_qos_max_link_speed_get(mdev, &link_speed_max, extack);
 	if (err)
 		return err;
 
@@ -1610,30 +1597,6 @@ static int esw_qos_devlink_rate_to_mbps(struct mlx5_core_dev *mdev, const char *
 	return 0;
 }
 
-static bool esw_qos_validate_unsupported_tc_bw(struct mlx5_eswitch *esw,
-					       u32 *tc_bw)
-{
-	int i, num_tcs = esw_qos_num_tcs(esw->dev);
-
-	for (i = num_tcs; i < DEVLINK_RATE_TCS_MAX; i++) {
-		if (tc_bw[i])
-			return false;
-	}
-
-	return true;
-}
-
-static bool esw_qos_vport_validate_unsupported_tc_bw(struct mlx5_vport *vport,
-						     u32 *tc_bw)
-{
-	struct mlx5_esw_sched_node *node = vport->qos.sched_node;
-	struct mlx5_eswitch *esw = vport->dev->priv.eswitch;
-
-	esw = (node && node->parent) ? node->parent->esw : esw;
-
-	return esw_qos_validate_unsupported_tc_bw(esw, tc_bw);
-}
-
 static bool esw_qos_tc_bw_disabled(u32 *tc_bw)
 {
 	int i;
@@ -1650,55 +1613,38 @@ static void esw_vport_qos_prune_empty(struct mlx5_vport *vport)
 {
 	struct mlx5_esw_sched_node *vport_node = vport->qos.sched_node;
 
-	esw_assert_qos_lock_held(vport->dev->priv.eswitch);
+	esw_assert_qos_lock_held(vport->dev);
 	if (!vport_node)
 		return;
 
-	if (vport_node->parent || vport_node->max_rate ||
-	    vport_node->min_rate || !esw_qos_tc_bw_disabled(vport_node->tc_bw))
+	if (vport_node->parent != vport_node->esw->qos.root ||
+	    vport_node->max_rate || vport_node->min_rate ||
+	    !esw_qos_tc_bw_disabled(vport_node->tc_bw))
 		return;
 
 	mlx5_esw_qos_vport_disable_locked(vport);
 }
 
-int mlx5_esw_qos_init(struct mlx5_eswitch *esw)
-{
-	if (esw->qos.domain)
-		return 0;  /* Nothing to change. */
-
-	return esw_qos_domain_init(esw);
-}
-
-void mlx5_esw_qos_cleanup(struct mlx5_eswitch *esw)
-{
-	if (esw->qos.domain)
-		esw_qos_domain_release(esw);
-}
-
 /* Eswitch devlink rate API */
 
 int mlx5_esw_devlink_rate_leaf_tx_share_set(struct devlink_rate *rate_leaf, void *priv,
 					    u64 tx_share, struct netlink_ext_ack *extack)
 {
 	struct mlx5_vport *vport = priv;
-	struct mlx5_eswitch *esw;
 	int err;
 
-	esw = vport->dev->priv.eswitch;
-	if (!mlx5_esw_allowed(esw))
+	if (!mlx5_esw_allowed(vport->dev->priv.eswitch))
 		return -EPERM;
 
 	err = esw_qos_devlink_rate_to_mbps(vport->dev, "tx_share", &tx_share, extack);
 	if (err)
 		return err;
 
-	esw_qos_lock(esw);
+	esw_qos_devlink_lock(vport->dev);
 	err = mlx5_esw_qos_set_vport_min_rate(vport, tx_share, extack);
-	if (err)
-		goto out;
-	esw_vport_qos_prune_empty(vport);
-out:
-	esw_qos_unlock(esw);
+	if (!err)
+		esw_vport_qos_prune_empty(vport);
+	esw_qos_devlink_unlock(vport->dev);
 	return err;
 }
 
@@ -1706,24 +1652,20 @@ int mlx5_esw_devlink_rate_leaf_tx_max_set(struct devlink_rate *rate_leaf, void *
 					  u64 tx_max, struct netlink_ext_ack *extack)
 {
 	struct mlx5_vport *vport = priv;
-	struct mlx5_eswitch *esw;
 	int err;
 
-	esw = vport->dev->priv.eswitch;
-	if (!mlx5_esw_allowed(esw))
+	if (!mlx5_esw_allowed(vport->dev->priv.eswitch))
 		return -EPERM;
 
 	err = esw_qos_devlink_rate_to_mbps(vport->dev, "tx_max", &tx_max, extack);
 	if (err)
 		return err;
 
-	esw_qos_lock(esw);
+	esw_qos_devlink_lock(vport->dev);
 	err = mlx5_esw_qos_set_vport_max_rate(vport, tx_max, extack);
-	if (err)
-		goto out;
-	esw_vport_qos_prune_empty(vport);
-out:
-	esw_qos_unlock(esw);
+	if (!err)
+		esw_vport_qos_prune_empty(vport);
+	esw_qos_devlink_unlock(vport->dev);
 	return err;
 }
 
@@ -1734,16 +1676,14 @@ int mlx5_esw_devlink_rate_leaf_tc_bw_set(struct devlink_rate *rate_leaf,
 {
 	struct mlx5_esw_sched_node *vport_node;
 	struct mlx5_vport *vport = priv;
-	struct mlx5_eswitch *esw;
 	bool disable;
 	int err = 0;
 
-	esw = vport->dev->priv.eswitch;
-	if (!mlx5_esw_allowed(esw))
+	if (!mlx5_esw_allowed(vport->dev->priv.eswitch))
 		return -EPERM;
 
 	disable = esw_qos_tc_bw_disabled(tc_bw);
-	esw_qos_lock(esw);
+	esw_qos_devlink_lock(vport->dev);
 
 	if (!esw_qos_vport_validate_unsupported_tc_bw(vport, tc_bw)) {
 		NL_SET_ERR_MSG_MOD(extack,
@@ -1777,7 +1717,7 @@ int mlx5_esw_devlink_rate_leaf_tc_bw_set(struct devlink_rate *rate_leaf,
 	if (!err)
 		esw_qos_set_tc_arbiter_bw_shares(vport_node, tc_bw, extack);
 unlock:
-	esw_qos_unlock(esw);
+	esw_qos_devlink_unlock(vport->dev);
 	return err;
 }
 
@@ -1787,18 +1727,17 @@ int mlx5_esw_devlink_rate_node_tc_bw_set(struct devlink_rate *rate_node,
 					 struct netlink_ext_ack *extack)
 {
 	struct mlx5_esw_sched_node *node = priv;
-	struct mlx5_eswitch *esw = node->esw;
 	bool disable;
 	int err;
 
-	if (!esw_qos_validate_unsupported_tc_bw(esw, tc_bw)) {
+	if (!esw_qos_validate_unsupported_tc_bw(node->esw, tc_bw)) {
 		NL_SET_ERR_MSG_MOD(extack,
 				   "E-Switch traffic classes number is not supported");
 		return -EOPNOTSUPP;
 	}
 
 	disable = esw_qos_tc_bw_disabled(tc_bw);
-	esw_qos_lock(esw);
+	esw_qos_devlink_lock(node->esw->dev);
 	if (disable) {
 		err = esw_qos_node_disable_tc_arbitration(node, extack);
 		goto unlock;
@@ -1808,7 +1747,7 @@ int mlx5_esw_devlink_rate_node_tc_bw_set(struct devlink_rate *rate_node,
 	if (!err)
 		esw_qos_set_tc_arbiter_bw_shares(node, tc_bw, extack);
 unlock:
-	esw_qos_unlock(esw);
+	esw_qos_devlink_unlock(node->esw->dev);
 	return err;
 }
 
@@ -1823,9 +1762,9 @@ int mlx5_esw_devlink_rate_node_tx_share_set(struct devlink_rate *rate_node, void
 	if (err)
 		return err;
 
-	esw_qos_lock(esw);
+	esw_qos_devlink_lock(esw->dev);
 	err = esw_qos_set_node_min_rate(node, tx_share, extack);
-	esw_qos_unlock(esw);
+	esw_qos_devlink_unlock(esw->dev);
 	return err;
 }
 
@@ -1840,9 +1779,9 @@ int mlx5_esw_devlink_rate_node_tx_max_set(struct devlink_rate *rate_node, void *
 	if (err)
 		return err;
 
-	esw_qos_lock(esw);
+	esw_qos_devlink_lock(esw->dev);
 	err = esw_qos_sched_elem_config(node, tx_max, node->bw_share, extack);
-	esw_qos_unlock(esw);
+	esw_qos_devlink_unlock(esw->dev);
 	return err;
 }
 
@@ -1857,7 +1796,7 @@ int mlx5_esw_devlink_rate_node_new(struct devlink_rate *rate_node, void **priv,
 	if (IS_ERR(esw))
 		return PTR_ERR(esw);
 
-	esw_qos_lock(esw);
+	esw_qos_devlink_lock(esw->dev);
 	if (esw->mode != MLX5_ESWITCH_OFFLOADS) {
 		NL_SET_ERR_MSG_MOD(extack,
 				   "Rate node creation supported only in switchdev mode");
@@ -1870,10 +1809,9 @@ int mlx5_esw_devlink_rate_node_new(struct devlink_rate *rate_node, void **priv,
 		err = PTR_ERR(node);
 		goto unlock;
 	}
-
 	*priv = node;
 unlock:
-	esw_qos_unlock(esw);
+	esw_qos_devlink_unlock(esw->dev);
 	return err;
 }
 
@@ -1883,25 +1821,53 @@ int mlx5_esw_devlink_rate_node_del(struct devlink_rate *rate_node, void *priv,
 	struct mlx5_esw_sched_node *node = priv;
 	struct mlx5_eswitch *esw = node->esw;
 
-	esw_qos_lock(esw);
+	esw_qos_devlink_lock(esw->dev);
 	__esw_qos_destroy_node(node, extack);
 	esw_qos_put(esw);
-	esw_qos_unlock(esw);
+	esw_qos_devlink_unlock(esw->dev);
+
 	return 0;
 }
 
-int mlx5_esw_qos_vport_update_parent(struct mlx5_vport *vport, struct mlx5_esw_sched_node *parent,
-				     struct netlink_ext_ack *extack)
+static int
+mlx5_esw_validate_cross_esw_scheduling(struct mlx5_eswitch *esw,
+				       struct mlx5_esw_sched_node *parent,
+				       struct netlink_ext_ack *extack)
 {
-	struct mlx5_eswitch *esw = vport->dev->priv.eswitch;
-	int err = 0;
+	if (!parent || esw == parent->esw)
+		return 0;
 
-	if (parent && parent->esw != esw) {
-		NL_SET_ERR_MSG_MOD(extack, "Cross E-Switch scheduling is not supported");
+	if (!MLX5_CAP_QOS(esw->dev, esw_cross_esw_sched)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Cross E-Switch scheduling is not supported");
+		return -EOPNOTSUPP;
+	}
+	if (!esw->dev->shd || esw->dev->shd != parent->esw->dev->shd) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Cannot add vport to a parent belonging to a different device");
 		return -EOPNOTSUPP;
 	}
+	if (!mlx5_lag_is_active(esw->dev)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Cross E-Switch scheduling requires LAG to be activated");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int
+mlx5_esw_qos_vport_update_parent(struct mlx5_vport *vport,
+				 struct mlx5_esw_sched_node *parent,
+				 struct netlink_ext_ack *extack)
+{
+	struct mlx5_eswitch *esw = vport->dev->priv.eswitch;
+	int err;
+
+	err = mlx5_esw_validate_cross_esw_scheduling(esw, parent, extack);
+	if (err)
+		return err;
 
-	esw_qos_lock(esw);
 	if (!vport->qos.sched_node && parent) {
 		enum sched_node_type type;
 
@@ -1910,9 +1876,11 @@ int mlx5_esw_qos_vport_update_parent(struct mlx5_vport *vport, struct mlx5_esw_s
 		err = mlx5_esw_qos_vport_enable(vport, type, parent, 0, 0,
 						extack);
 	} else if (vport->qos.sched_node) {
-		err = esw_qos_vport_update_parent(vport, parent, extack);
+		err = esw_qos_vport_update_parent(vport,
+						  parent ? : esw->qos.root,
+						  extack);
 	}
-	esw_qos_unlock(esw);
+
 	return err;
 }
 
@@ -1925,14 +1893,11 @@ int mlx5_esw_devlink_rate_leaf_parent_set(struct devlink_rate *devlink_rate,
 	struct mlx5_vport *vport = priv;
 	int err;
 
+	esw_qos_devlink_lock(vport->dev);
 	err = mlx5_esw_qos_vport_update_parent(vport, node, extack);
-	if (!err) {
-		struct mlx5_eswitch *esw = vport->dev->priv.eswitch;
-
-		esw_qos_lock(esw);
+	if (!err)
 		esw_vport_qos_prune_empty(vport);
-		esw_qos_unlock(esw);
-	}
+	esw_qos_devlink_unlock(vport->dev);
 
 	return err;
 }
@@ -1958,7 +1923,7 @@ mlx5_esw_qos_node_validate_set_parent(struct mlx5_esw_sched_node *node,
 {
 	u8 new_level, max_level;
 
-	if (parent && parent->esw != node->esw) {
+	if (parent->esw != node->esw) {
 		NL_SET_ERR_MSG_MOD(extack,
 				   "Cannot assign node to another E-Switch");
 		return -EOPNOTSUPP;
@@ -1970,13 +1935,13 @@ mlx5_esw_qos_node_validate_set_parent(struct mlx5_esw_sched_node *node,
 		return -EOPNOTSUPP;
 	}
 
-	if (parent && parent->type == SCHED_NODE_TYPE_TC_ARBITER_TSAR) {
+	if (parent->type == SCHED_NODE_TYPE_TC_ARBITER_TSAR) {
 		NL_SET_ERR_MSG_MOD(extack,
 				   "Cannot attach a node to a parent with TC bandwidth configured");
 		return -EOPNOTSUPP;
 	}
 
-	new_level = parent ? parent->level + 1 : 2;
+	new_level = parent->level + 1;
 	if (node->type == SCHED_NODE_TYPE_TC_ARBITER_TSAR) {
 		/* Increase by one to account for the vports TC scheduling
 		 * element.
@@ -2027,14 +1992,12 @@ static int esw_qos_vports_node_update_parent(struct mlx5_esw_sched_node *node,
 {
 	struct mlx5_esw_sched_node *curr_parent = node->parent;
 	struct mlx5_eswitch *esw = node->esw;
-	u32 parent_ix;
 	int err;
 
-	parent_ix = parent ? parent->ix : node->esw->qos.root_tsar_ix;
 	mlx5_destroy_scheduling_element_cmd(esw->dev,
 					    SCHEDULING_HIERARCHY_E_SWITCH,
 					    node->ix);
-	err = esw_qos_create_node_sched_elem(esw->dev, parent_ix,
+	err = esw_qos_create_node_sched_elem(esw->dev, parent->ix,
 					     node->max_rate, 0, &node->ix);
 	if (err) {
 		NL_SET_ERR_MSG_MOD(extack,
@@ -2061,12 +2024,15 @@ static int mlx5_esw_qos_node_update_parent(struct mlx5_esw_sched_node *node,
 	struct mlx5_eswitch *esw = node->esw;
 	int err;
 
+	esw_qos_devlink_lock(esw->dev);
+	curr_parent = node->parent;
+	if (!parent)
+		parent = esw->qos.root;
+
 	err = mlx5_esw_qos_node_validate_set_parent(node, parent, extack);
 	if (err)
-		return err;
+		goto out;
 
-	esw_qos_lock(esw);
-	curr_parent = node->parent;
 	if (node->type == SCHED_NODE_TYPE_TC_ARBITER_TSAR) {
 		err = esw_qos_tc_arbiter_node_update_parent(node, parent,
 							    extack);
@@ -2077,12 +2043,11 @@ static int mlx5_esw_qos_node_update_parent(struct mlx5_esw_sched_node *node,
 	if (err)
 		goto out;
 
-	esw_qos_normalize_min_rate(esw, curr_parent, extack);
-	esw_qos_normalize_min_rate(esw, parent, extack);
+	esw_qos_normalize_min_rate(curr_parent, extack);
+	esw_qos_normalize_min_rate(parent, extack);
 
 out:
-	esw_qos_unlock(esw);
-
+	esw_qos_devlink_unlock(esw->dev);
 	return err;
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.h b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.h
index 0a50982b0e27c..f275e850d2c9e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.h
@@ -6,9 +6,6 @@
 
 #ifdef CONFIG_MLX5_ESWITCH
 
-int mlx5_esw_qos_init(struct mlx5_eswitch *esw);
-void mlx5_esw_qos_cleanup(struct mlx5_eswitch *esw);
-
 int mlx5_esw_qos_set_vport_rate(struct mlx5_vport *evport, u32 max_rate, u32 min_rate);
 bool mlx5_esw_qos_get_vport_rate(struct mlx5_vport *vport, u32 *max_rate, u32 *min_rate);
 void mlx5_esw_qos_vport_disable(struct mlx5_vport *vport);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
index a0e2ca87b8d8f..b6e2c153b4f7d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
@@ -1885,10 +1885,6 @@ int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int num_vfs)
 	MLX5_NB_INIT(&esw->nb, eswitch_vport_event, NIC_VPORT_CHANGE);
 	mlx5_eq_notifier_register(esw->dev, &esw->nb);
 
-	err = mlx5_esw_qos_init(esw);
-	if (err)
-		goto err_esw_init;
-
 	if (esw->mode == MLX5_ESWITCH_LEGACY) {
 		err = esw_legacy_enable(esw);
 	} else {
@@ -1990,6 +1986,13 @@ void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf)
 		 esw->esw_funcs.num_vfs, esw->esw_funcs.num_ec_vfs, esw->enabled_vports);
 
 	mlx5_eswitch_invalidate_wq(esw);
+
+	if (esw->mode == MLX5_ESWITCH_OFFLOADS) {
+		struct devlink *devlink = priv_to_devlink(esw->dev);
+
+		devl_rate_nodes_destroy(devlink);
+	}
+
 	mlx5_esw_reps_block(esw);
 
 	if (!mlx5_core_is_ecpf(esw->dev)) {
@@ -2003,12 +2006,6 @@ void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf)
 	}
 
 	mlx5_esw_reps_unblock(esw);
-
-	if (esw->mode == MLX5_ESWITCH_OFFLOADS) {
-		struct devlink *devlink = priv_to_devlink(esw->dev);
-
-		devl_rate_nodes_destroy(devlink);
-	}
 	/* Destroy legacy fdb when disabling sriov in legacy mode. */
 	if (esw->mode == MLX5_ESWITCH_LEGACY)
 		mlx5_eswitch_disable_locked(esw);
@@ -2039,6 +2036,9 @@ void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw)
 		 esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS",
 		 esw->esw_funcs.num_vfs, esw->esw_funcs.num_ec_vfs, esw->enabled_vports);
 
+	if (esw->mode == MLX5_ESWITCH_OFFLOADS)
+		devl_rate_nodes_destroy(devlink);
+
 	if (esw->fdb_table.flags & MLX5_ESW_FDB_CREATED) {
 		esw->fdb_table.flags &= ~MLX5_ESW_FDB_CREATED;
 		if (esw->mode == MLX5_ESWITCH_OFFLOADS)
@@ -2047,9 +2047,6 @@ void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw)
 			esw_legacy_disable(esw);
 		mlx5_esw_acls_ns_cleanup(esw);
 	}
-
-	if (esw->mode == MLX5_ESWITCH_OFFLOADS)
-		devl_rate_nodes_destroy(devlink);
 }
 
 void mlx5_eswitch_disable(struct mlx5_eswitch *esw)
@@ -2554,9 +2551,6 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev)
 		goto reps_err;
 
 	esw->mode = MLX5_ESWITCH_LEGACY;
-	err = mlx5_esw_qos_init(esw);
-	if (err)
-		goto reps_err;
 
 	mutex_init(&esw->offloads.encap_tbl_lock);
 	hash_init(esw->offloads.encap_tbl);
@@ -2611,7 +2605,6 @@ void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw)
 
 	mlx5_eswitch_invalidate_wq(esw);
 	destroy_workqueue(esw->work_queue);
-	mlx5_esw_qos_cleanup(esw);
 	WARN_ON(refcount_read(&esw->qos.refcnt));
 	mutex_destroy(&esw->state_lock);
 	WARN_ON(!xa_empty(&esw->offloads.vhca_map));
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
index fea72b1dedab9..c655f6e8da1ce 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
@@ -234,8 +234,10 @@ struct mlx5_vport {
 
 	struct mlx5_vport_info  info;
 
-	/* Protected with the E-Switch qos domain lock. The Vport QoS can
-	 * either be disabled (sched_node is NULL) or in one of three states:
+	/* Protected by either the shared devlink (dev->shd) lock or by
+	 * esw->state_lock. See esw_assert_qos_lock_held() for more details.
+	 * The Vport QoS can either be disabled (sched_node is NULL) or in one
+	 * of three states:
 	 * 1. Regular QoS (sched_node is a vport node).
 	 * 2. TC QoS enabled on the vport (sched_node is a TC arbiter).
 	 * 3. TC QoS enabled on the vport's parent node
@@ -382,7 +384,6 @@ enum {
 };
 
 struct dentry;
-struct mlx5_qos_domain;
 
 struct mlx5_eswitch {
 	struct mlx5_core_dev    *dev;
@@ -411,12 +412,15 @@ struct mlx5_eswitch {
 	atomic64_t user_count;
 	wait_queue_head_t work_queue_wait;
 
-	/* Protected with the E-Switch qos domain lock. */
+	/* QoS changes are serialized by either the shared devlink (dev->shd)
+	 * lock or by esw->state_lock. See esw_assert_qos_lock_held() for more
+	 * details.
+	 */
 	struct {
 		/* Initially 0, meaning no QoS users and QoS is disabled. */
 		refcount_t refcnt;
-		u32 root_tsar_ix;
-		struct mlx5_qos_domain *domain;
+		/* The root node of the hierarchy. */
+		struct mlx5_esw_sched_node *root;
 	} qos;
 
 	struct mlx5_esw_bridge_offloads *br_offloads;
@@ -482,8 +486,6 @@ int mlx5_eswitch_set_vport_trust(struct mlx5_eswitch *esw,
 				 u16 vport_num, bool setting);
 int mlx5_eswitch_set_vport_rate(struct mlx5_eswitch *esw, u16 vport,
 				u32 max_rate, u32 min_rate);
-int mlx5_esw_qos_vport_update_parent(struct mlx5_vport *vport, struct mlx5_esw_sched_node *node,
-				     struct netlink_ext_ack *extack);
 int mlx5_eswitch_set_vepa(struct mlx5_eswitch *esw, u8 setting);
 int mlx5_eswitch_get_vepa(struct mlx5_eswitch *esw, u8 *setting);
 int mlx5_eswitch_get_vport_config(struct mlx5_eswitch *esw,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
index dbaf33b537f71..906584345a022 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
@@ -214,17 +214,7 @@ struct mlx5_ft_underlay_qp {
 	u32 qpn;
 };
 
-#define MLX5_FTE_MATCH_PARAM_RESERVED	reserved_at_e00
-/* Calculate the fte_match_param length and without the reserved length.
- * Make sure the reserved field is the last.
- */
-#define MLX5_ST_SZ_DW_MATCH_PARAM					    \
-	((MLX5_BYTE_OFF(fte_match_param, MLX5_FTE_MATCH_PARAM_RESERVED) / sizeof(u32)) + \
-	 BUILD_BUG_ON_ZERO(MLX5_ST_SZ_BYTES(fte_match_param) !=		     \
-			   MLX5_FLD_SZ_BYTES(fte_match_param,		     \
-					     MLX5_FTE_MATCH_PARAM_RESERVED) +\
-			   MLX5_BYTE_OFF(fte_match_param,		     \
-					 MLX5_FTE_MATCH_PARAM_RESERVED)))
+#define MLX5_ST_SZ_DW_MATCH_PARAM MLX5_ST_SZ_DW(fte_match_param)
 
 struct fs_fte_action {
 	int				modify_mask;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/st.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/st.c
index 997be91f0a135..7cedc348790db 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/st.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/st.c
@@ -175,6 +175,7 @@ int mlx5_st_dealloc_index(struct mlx5_core_dev *dev, u16 st_index)
 
 	if (refcount_dec_and_test(&idx_data->usecount)) {
 		xa_erase(&st->idx_xa, st_index);
+		kfree(idx_data);
 		/* We leave PCI config space as was before, no mkey will refer to it */
 	}
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/cmd.c
index e624f5da96c81..8fae90101653f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/cmd.c
@@ -1172,13 +1172,13 @@ int mlx5hws_cmd_query_caps(struct mlx5_core_dev *mdev,
 		}
 
 		if (MLX5_GET(query_hca_cap_out, out,
-			     capability.esw_cap.esw_manager_vport_number_valid))
+			     capability.e_switch_cap.esw_manager_vport_number_valid))
 			caps->eswitch_manager_vport_number =
 				MLX5_GET(query_hca_cap_out, out,
-					 capability.esw_cap.esw_manager_vport_number);
+					 capability.e_switch_cap.esw_manager_vport_number);
 
 		caps->merged_eswitch = MLX5_GET(query_hca_cap_out, out,
-						capability.esw_cap.merged_eswitch);
+						capability.e_switch_cap.merged_eswitch);
 	}
 
 	ret = mlx5_cmd_exec(mdev, in, sizeof(in), out, out_size);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_erp.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_erp.c
index cbb272a963595..0d0cd093b3c6d 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_erp.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_erp.c
@@ -1118,7 +1118,7 @@ u8 mlxsw_sp_acl_erp_delta_value(const struct mlxsw_sp_acl_erp_delta *delta,
 }
 
 void mlxsw_sp_acl_erp_delta_clear(const struct mlxsw_sp_acl_erp_delta *delta,
-				  const char *enc_key)
+				  char *enc_key)
 {
 	u16 start = delta->start;
 	u8 mask = delta->mask;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.h
index 010204f73ea46..67cc7a5737dd2 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.h
@@ -245,7 +245,7 @@ u8 mlxsw_sp_acl_erp_delta_mask(const struct mlxsw_sp_acl_erp_delta *delta);
 u8 mlxsw_sp_acl_erp_delta_value(const struct mlxsw_sp_acl_erp_delta *delta,
 				const char *enc_key);
 void mlxsw_sp_acl_erp_delta_clear(const struct mlxsw_sp_acl_erp_delta *delta,
-				  const char *enc_key);
+				  char *enc_key);
 
 struct mlxsw_sp_acl_erp_mask;
 
diff --git a/drivers/net/ethernet/microsoft/Kconfig b/drivers/net/ethernet/microsoft/Kconfig
index 3f36ee6a8ecee..e9be18c92ca5f 100644
--- a/drivers/net/ethernet/microsoft/Kconfig
+++ b/drivers/net/ethernet/microsoft/Kconfig
@@ -21,6 +21,7 @@ config MICROSOFT_MANA
 	depends on X86_64 || (ARM64 && !CPU_BIG_ENDIAN)
 	depends on PCI_HYPERV
 	select AUXILIARY_BUS
+	select DIMLIB
 	select PAGE_POOL
 	select NET_SHAPER
 	help
diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c
index e8b7ffb47eb98..a38d4bb74621f 100644
--- a/drivers/net/ethernet/microsoft/mana/gdma_main.c
+++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /* Copyright (c) 2021, Microsoft Corporation. */
 
+#include <linux/bitfield.h>
 #include <linux/debugfs.h>
 #include <linux/module.h>
 #include <linux/pci.h>
@@ -466,6 +467,7 @@ static int mana_gd_disable_queue(struct gdma_queue *queue)
 #define DOORBELL_OFFSET_RQ	0x400
 #define DOORBELL_OFFSET_CQ	0x800
 #define DOORBELL_OFFSET_EQ	0xFF8
+#define DOORBELL_OFFSET_DIM	0x820
 
 static void mana_gd_ring_doorbell(struct gdma_context *gc, u32 db_index,
 				  enum gdma_queue_type q_type, u32 qid,
@@ -506,6 +508,16 @@ static void mana_gd_ring_doorbell(struct gdma_context *gc, u32 db_index,
 		addr += DOORBELL_OFFSET_SQ;
 		break;
 
+	case GDMA_DIM:
+		e.dim.id = qid;
+		e.dim.mod_usec = FIELD_GET(MANA_INTR_MODR_USEC_MAX, tail_ptr);
+		e.dim.mod_usec_vld = !!(tail_ptr & MANA_INTR_MODR_USEC_VLD);
+		e.dim.mod_comps = FIELD_GET(MANA_INTR_MODR_COMP_MASK, tail_ptr);
+		e.dim.mod_comps_vld = num_req;
+
+		addr += DOORBELL_OFFSET_DIM;
+		break;
+
 	default:
 		WARN_ON(1);
 		return;
@@ -540,6 +552,23 @@ void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit)
 }
 EXPORT_SYMBOL_NS(mana_gd_ring_cq, "NET_MANA");
 
+void mana_gd_ring_dim(struct gdma_queue *cq, u32 mod_usec, bool mod_usec_vld,
+		      u32 mod_comps, bool mod_comps_vld)
+{
+	struct gdma_context *gc = cq->gdma_dev->gdma_context;
+	u32 dim_val;
+
+	/* Convert the DIM values to doorbell parameters */
+	dim_val = FIELD_PREP(MANA_INTR_MODR_USEC_MAX, mod_usec) |
+		  FIELD_PREP(MANA_INTR_MODR_COMP_MASK, mod_comps);
+	if (mod_usec_vld)
+		dim_val |= MANA_INTR_MODR_USEC_VLD;
+
+	mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, GDMA_DIM, cq->id,
+			      dim_val, mod_comps_vld);
+}
+EXPORT_SYMBOL_NS(mana_gd_ring_dim, "NET_MANA");
+
 #define MANA_SERVICE_PERIOD 10
 
 static void mana_serv_rescan(struct pci_dev *pdev)
@@ -2427,6 +2456,8 @@ static void mana_gd_remove(struct pci_dev *pdev)
 {
 	struct gdma_context *gc = pci_get_drvdata(pdev);
 
+	pci_disable_sriov(pdev);
+
 	mana_rdma_remove(&gc->mana_ib);
 	mana_remove(&gc->mana, false);
 
@@ -2496,6 +2527,27 @@ static void mana_gd_shutdown(struct pci_dev *pdev)
 	pci_disable_device(pdev);
 }
 
+static int mana_sriov_configure(struct pci_dev *pdev, int numvfs)
+{
+	int err = 0;
+
+	dev_info(&pdev->dev, "Requested num VFs: %d\n", numvfs);
+
+	if (numvfs > 0) {
+		err = pci_enable_sriov(pdev, numvfs);
+	} else {
+		if (pci_vfs_assigned(pdev)) {
+			dev_warn(&pdev->dev,
+				 "Cannot disable SR-IOV while VFs are assigned\n");
+			return -EPERM;
+		}
+
+		pci_disable_sriov(pdev);
+	}
+
+	return err ? err : numvfs;
+}
+
 static const struct pci_device_id mana_id_table[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF_DEVICE_ID) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF2_DEVICE_ID) },
@@ -2511,6 +2563,7 @@ static struct pci_driver mana_driver = {
 	.suspend	= mana_gd_suspend,
 	.resume		= mana_gd_resume,
 	.shutdown	= mana_gd_shutdown,
+	.sriov_configure = mana_sriov_configure,
 };
 
 static int __init mana_driver_init(void)
diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c
index 9d9bfd116dabc..a8c329bdbacf8 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_en.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_en.c
@@ -892,6 +892,17 @@ static void mana_tx_timeout(struct net_device *netdev, unsigned int txqueue)
 	struct mana_context *ac = apc->ac;
 	struct gdma_context *gc = ac->gdma_dev->gdma_context;
 
+	/* Debug knob for bringup/qualification: when set, log the timeout and
+	 * skip the reset so the failing state is preserved for telemetry.
+	 * Disabled by default; production behaviour is unchanged.
+	 */
+	if (READ_ONCE(apc->tx_timeout_skip_reset)) {
+		netdev_warn(netdev,
+			    "TX timeout on queue %u: reset skipped (tx_timeout_skip_reset enabled)\n",
+			    txqueue);
+		return;
+	}
+
 	/* Already in service, hence tx queue reset is not required.*/
 	if (test_bit(GC_IN_SERVICE, &gc->flags))
 		return;
@@ -1591,6 +1602,15 @@ int mana_create_wq_obj(struct mana_port_context *apc,
 
 	mana_gd_init_req_hdr(&req.hdr, MANA_CREATE_WQ_OBJ,
 			     sizeof(req), sizeof(resp));
+
+	/* Our driver uses different message versions for request and
+	 * response in this case.
+	 * Our firmware is forward compatible with newer message versions, so
+	 * the old firmware still properly handles this message, just the new
+	 * feature fields are ignored, and queue creation will be successful.
+	 */
+	req.hdr.req.msg_version = GDMA_MESSAGE_V3;
+	req.hdr.resp.msg_version = GDMA_MESSAGE_V2;
 	req.vport = vport;
 	req.wq_type = wq_type;
 	req.wq_gdma_region = wq_spec->gdma_region;
@@ -1599,6 +1619,9 @@ int mana_create_wq_obj(struct mana_port_context *apc,
 	req.cq_size = cq_spec->queue_size;
 	req.cq_moderation_ctx_id = cq_spec->modr_ctx_id;
 	req.cq_parent_qid = cq_spec->attached_eq;
+	req.req_cq_moderation = cq_spec->req_cq_moderation;
+	req.cq_moderation_comp = cq_spec->cq_moderation_comp;
+	req.cq_moderation_usec = cq_spec->cq_moderation_usec;
 
 	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
 				sizeof(resp));
@@ -1856,6 +1879,7 @@ static void mana_poll_tx_cq(struct mana_cq *cq)
 	struct gdma_posted_wqe_info *wqe_info;
 	unsigned int pkt_transmitted = 0;
 	unsigned int wqe_unit_cnt = 0;
+	unsigned int tx_bytes = 0;
 	struct mana_txq *txq = cq->txq;
 	struct mana_port_context *apc;
 	struct netdev_queue *net_txq;
@@ -1937,6 +1961,8 @@ static void mana_poll_tx_cq(struct mana_cq *cq)
 
 		mana_unmap_skb(skb, apc);
 
+		tx_bytes += skb->len;
+
 		napi_consume_skb(skb, cq->budget);
 
 		pkt_transmitted++;
@@ -1967,6 +1993,10 @@ static void mana_poll_tx_cq(struct mana_cq *cq)
 	if (atomic_sub_return(pkt_transmitted, &txq->pending_sends) < 0)
 		WARN_ON_ONCE(1);
 
+	/* Feed DIM with the completion rate observed here, in NAPI context. */
+	cq->tx_dim_pkts += pkt_transmitted;
+	cq->tx_dim_bytes += tx_bytes;
+
 	cq->work_done = pkt_transmitted;
 }
 
@@ -2352,6 +2382,117 @@ static void mana_poll_rx_cq(struct mana_cq *cq)
 		xdp_do_flush();
 }
 
+static void mana_rx_dim_work(struct work_struct *work)
+{
+	struct dim *dim = container_of(work, struct dim, work);
+	struct dim_cq_moder cur_moder;
+	struct mana_cq *cq;
+
+	cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
+	cq = container_of(dim, struct mana_cq, dim);
+
+	cur_moder.usec = min_t(u16, cur_moder.usec, MANA_INTR_MODR_USEC_MAX);
+	cur_moder.pkts = min_t(u16, cur_moder.pkts, MANA_INTR_MODR_COMP_MAX);
+
+	mana_gd_ring_dim(cq->gdma_cq, cur_moder.usec, true,
+			 cur_moder.pkts, true);
+
+	dim->state = DIM_START_MEASURE;
+}
+
+static void mana_tx_dim_work(struct work_struct *work)
+{
+	struct dim *dim = container_of(work, struct dim, work);
+	struct dim_cq_moder cur_moder;
+	struct mana_cq *cq;
+
+	cur_moder = net_dim_get_tx_moderation(dim->mode, dim->profile_ix);
+	cq = container_of(dim, struct mana_cq, dim);
+
+	cur_moder.usec = min_t(u16, cur_moder.usec, MANA_INTR_MODR_USEC_MAX);
+	cur_moder.pkts = min_t(u16, cur_moder.pkts, MANA_INTR_MODR_COMP_MAX);
+
+	mana_gd_ring_dim(cq->gdma_cq, cur_moder.usec, true,
+			 cur_moder.pkts, true);
+
+	dim->state = DIM_START_MEASURE;
+}
+
+/* The caller must update apc->rx/tx_dim_enabled before disabling and
+ * after enabling. And synchronize_net() before draining the DIM work,
+ * so that NAPI cannot observe a stale flag.
+ */
+void mana_dim_change(struct mana_cq *cq, bool enable)
+{
+	bool is_rx = cq->type == MANA_CQ_TYPE_RX;
+	struct mana_port_context *apc;
+	work_func_t work_func;
+	u32 usec, comp;
+
+	if (is_rx) {
+		apc = netdev_priv(cq->rxq->ndev);
+		usec = apc->intr_modr_rx_usec;
+		comp = apc->intr_modr_rx_comp;
+		work_func = mana_rx_dim_work;
+	} else {
+		apc = netdev_priv(cq->txq->ndev);
+		usec = apc->intr_modr_tx_usec;
+		comp = apc->intr_modr_tx_comp;
+		work_func = mana_tx_dim_work;
+	}
+
+	/* On enable, zero the DIM state so net_dim() starts measuring from
+	 * scratch.
+	 * On disable, drain any pending DIM work and restore the static
+	 * moderation values.
+	 */
+	if (enable) {
+		memset(&cq->dim, 0, sizeof(cq->dim));
+		cq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
+		INIT_WORK(&cq->dim.work, work_func);
+	} else {
+		cancel_work_sync(&cq->dim.work);
+		mana_gd_ring_dim(cq->gdma_cq, usec, true, comp, true);
+	}
+}
+
+static void mana_update_rx_dim(struct mana_cq *cq)
+{
+	struct mana_port_context *apc = netdev_priv(cq->rxq->ndev);
+	struct dim_sample dim_sample = {};
+	struct mana_rxq *rxq = cq->rxq;
+
+	/* Pairs with smp_store_release() in mana_set_coalesce(): observing the
+	 * enable flag set guarantees the DIM (re)initialization is visible.
+	 */
+	if (!smp_load_acquire(&apc->rx_dim_enabled))
+		return;
+
+	dim_update_sample(READ_ONCE(cq->dim_event_ctr), rxq->stats.packets,
+			  rxq->stats.bytes, &dim_sample);
+	net_dim(&cq->dim, &dim_sample);
+}
+
+static void mana_update_tx_dim(struct mana_cq *cq)
+{
+	struct mana_port_context *apc = netdev_priv(cq->txq->ndev);
+	struct dim_sample dim_sample = {};
+
+	/* Pairs with smp_store_release() in mana_set_coalesce(): observing the
+	 * enable flag set guarantees the DIM (re)initialization is visible.
+	 */
+	if (!smp_load_acquire(&apc->tx_dim_enabled))
+		return;
+
+	/* cq->tx_dim_pkts/bytes are accumulated in mana_poll_tx_cq(), in the
+	 * same NAPI context as this read, so they track the hardware
+	 * completion rate and need no u64_stats_sync protection.
+	 */
+	dim_update_sample(READ_ONCE(cq->dim_event_ctr), cq->tx_dim_pkts,
+			  cq->tx_dim_bytes, &dim_sample);
+	net_dim(&cq->dim, &dim_sample);
+}
+
 static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue)
 {
 	struct mana_cq *cq = context;
@@ -2370,6 +2511,15 @@ static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue)
 	if (w < cq->budget) {
 		mana_gd_ring_cq(gdma_queue, SET_ARM_BIT);
 		cq->work_done_since_doorbell = 0;
+
+		/* Update DIM before napi_complete_done() to prevent running
+		 * net_dim() concurrently.
+		 */
+		if (cq->type == MANA_CQ_TYPE_RX)
+			mana_update_rx_dim(cq);
+		else
+			mana_update_tx_dim(cq);
+
 		napi_complete_done(&cq->napi, w);
 	} else if (cq->work_done_since_doorbell >=
 		   (cq->gdma_cq->queue_size / COMP_ENTRY_SIZE) * 4) {
@@ -2402,6 +2552,7 @@ static void mana_schedule_napi(void *context, struct gdma_queue *gdma_queue)
 {
 	struct mana_cq *cq = context;
 
+	WRITE_ONCE(cq->dim_event_ctr, cq->dim_event_ctr + 1);
 	napi_schedule_irqoff(&cq->napi);
 }
 
@@ -2444,6 +2595,7 @@ static void mana_destroy_txq(struct mana_port_context *apc)
 		if (apc->tx_qp[i]->txq.napi_initialized) {
 			napi_synchronize(napi);
 			napi_disable_locked(napi);
+			cancel_work_sync(&apc->tx_qp[i]->tx_cq.dim.work);
 			netif_napi_del_locked(napi);
 			apc->tx_qp[i]->txq.napi_initialized = false;
 		}
@@ -2577,6 +2729,11 @@ static int mana_create_txq(struct mana_port_context *apc,
 		cq_spec.modr_ctx_id = 0;
 		cq_spec.attached_eq = cq->gdma_cq->cq.parent->id;
 
+		/* DIM setting can be changed at runtime */
+		cq_spec.req_cq_moderation = true;
+		cq_spec.cq_moderation_usec = apc->intr_modr_tx_usec;
+		cq_spec.cq_moderation_comp = apc->intr_modr_tx_comp;
+
 		err = mana_create_wq_obj(apc, apc->port_handle, GDMA_SQ,
 					 &wq_spec, &cq_spec,
 					 &apc->tx_qp[i]->tx_object);
@@ -2607,6 +2764,13 @@ static int mana_create_txq(struct mana_port_context *apc,
 
 		set_bit(NAPI_STATE_NO_BUSY_POLL, &cq->napi.state);
 		netif_napi_add_locked(net, &cq->napi, mana_poll);
+
+		/* Initialize the DIM work before enabling NAPI, so that a poll
+		 * cannot reach net_dim() with an uninitialized cq->dim.work.
+		 */
+		INIT_WORK(&cq->dim.work, mana_tx_dim_work);
+		cq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
+
 		napi_enable_locked(&cq->napi);
 		txq->napi_initialized = true;
 
@@ -2644,6 +2808,7 @@ static void mana_destroy_rxq(struct mana_port_context *apc,
 		napi_synchronize(napi);
 
 		napi_disable_locked(napi);
+		cancel_work_sync(&rxq->rx_cq.dim.work);
 		netif_napi_del_locked(napi);
 	}
 
@@ -2887,6 +3052,11 @@ static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc,
 	cq_spec.modr_ctx_id = 0;
 	cq_spec.attached_eq = cq->gdma_cq->cq.parent->id;
 
+	/* DIM setting can be changed at runtime */
+	cq_spec.req_cq_moderation = true;
+	cq_spec.cq_moderation_usec = apc->intr_modr_rx_usec;
+	cq_spec.cq_moderation_comp = apc->intr_modr_rx_comp;
+
 	err = mana_create_wq_obj(apc, apc->port_handle, GDMA_RQ,
 				 &wq_spec, &cq_spec, &rxq->rxobj);
 	if (err)
@@ -2919,6 +3089,12 @@ static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc,
 	WARN_ON(xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, MEM_TYPE_PAGE_POOL,
 					   rxq->page_pool));
 
+	/* Initialize the DIM work before enabling NAPI, so that a poll
+	 * cannot reach net_dim() with an uninitialized cq->dim.work.
+	 */
+	INIT_WORK(&cq->dim.work, mana_rx_dim_work);
+	cq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
+
 	napi_enable_locked(&cq->napi);
 
 	mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT);
@@ -3321,6 +3497,9 @@ static int mana_init_port(struct net_device *ndev)
 			   &apc->steer_cqe_coalescing);
 	debugfs_create_u32("current_speed", 0400, apc->mana_port_debugfs,
 			   &apc->speed);
+	debugfs_create_bool("tx_timeout_skip_reset", 0600,
+			    apc->mana_port_debugfs,
+			    &apc->tx_timeout_skip_reset);
 	return 0;
 
 reset_apc:
@@ -3585,6 +3764,16 @@ static int mana_probe_port(struct mana_context *ac, int port_idx,
 	apc->link_cfg_error = 1;
 	apc->cqe_coalescing_enable = 0;
 
+	/* Initialize interrupt moderation settings if supported by HW */
+	if (gc->pf_cap_flags1 & GDMA_PF_CAP_FLAG_1_DYN_INTERRUPT_MODERATION) {
+		apc->intr_modr_rx_usec = MANA_INTR_MODR_USEC_DEF;
+		apc->intr_modr_rx_comp = MANA_INTR_MODR_COMP_DEF;
+		apc->intr_modr_tx_usec = MANA_INTR_MODR_USEC_DEF;
+		apc->intr_modr_tx_comp = MANA_INTR_MODR_COMP_DEF;
+		apc->rx_dim_enabled = MANA_ADAPTIVE_RX_DEF;
+		apc->tx_dim_enabled = MANA_ADAPTIVE_TX_DEF;
+	}
+
 	mutex_init(&apc->vport_mutex);
 	apc->vport_use_count = 0;
 
diff --git a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
index 881df597d7f9c..9e31e2595ae32 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
@@ -419,6 +419,15 @@ static int mana_get_coalesce(struct net_device *ndev,
 	    !kernel_coal->rx_cqe_nsecs)
 		kernel_coal->rx_cqe_nsecs = MANA_RX_CQE_NSEC_DEF;
 
+	ec->rx_coalesce_usecs = apc->intr_modr_rx_usec;
+	ec->rx_max_coalesced_frames = apc->intr_modr_rx_comp;
+
+	ec->tx_coalesce_usecs = apc->intr_modr_tx_usec;
+	ec->tx_max_coalesced_frames = apc->intr_modr_tx_comp;
+
+	ec->use_adaptive_rx_coalesce = apc->rx_dim_enabled;
+	ec->use_adaptive_tx_coalesce = apc->tx_dim_enabled;
+
 	return 0;
 }
 
@@ -428,9 +437,34 @@ static int mana_set_coalesce(struct net_device *ndev,
 			     struct netlink_ext_ack *extack)
 {
 	struct mana_port_context *apc = netdev_priv(ndev);
-	u8 saved_cqe_coalescing_enable;
+	struct {
+		u16 intr_modr_rx_usec;
+		u16 intr_modr_rx_comp;
+		u16 intr_modr_tx_usec;
+		u16 intr_modr_tx_comp;
+		u8 cqe_coalescing_enable;
+		bool rx_dim_enabled;
+		bool tx_dim_enabled;
+	} saved;
+	bool modr_changed = false;
+	bool dim_changed = false;
+	struct gdma_context *gc;
 	int err;
 
+	gc = apc->ac->gdma_dev->gdma_context;
+
+	/* Both static and dynamic interrupt moderation (DIM) rely on the
+	 * same HW capability advertised by the PF.
+	 */
+	if ((ec->use_adaptive_rx_coalesce || ec->use_adaptive_tx_coalesce ||
+	     ec->rx_coalesce_usecs || ec->tx_coalesce_usecs ||
+	     ec->rx_max_coalesced_frames || ec->tx_max_coalesced_frames) &&
+	    !(gc->pf_cap_flags1 & GDMA_PF_CAP_FLAG_1_DYN_INTERRUPT_MODERATION)) {
+		NL_SET_ERR_MSG(extack,
+			       "Interrupt Moderation is not supported by HW");
+		return -EOPNOTSUPP;
+	}
+
 	if (kernel_coal->rx_cqe_frames != 1 &&
 	    kernel_coal->rx_cqe_frames != MANA_RXCOMP_OOB_NUM_PPI) {
 		NL_SET_ERR_MSG_FMT(extack,
@@ -440,18 +474,129 @@ static int mana_set_coalesce(struct net_device *ndev,
 		return -EINVAL;
 	}
 
-	saved_cqe_coalescing_enable = apc->cqe_coalescing_enable;
+	if (ec->rx_coalesce_usecs > MANA_INTR_MODR_USEC_MAX ||
+	    ec->tx_coalesce_usecs > MANA_INTR_MODR_USEC_MAX) {
+		NL_SET_ERR_MSG_FMT(extack,
+				   "coalesce usecs must be <= %lu",
+				   MANA_INTR_MODR_USEC_MAX);
+		return -EINVAL;
+	}
+
+	if (ec->rx_max_coalesced_frames > MANA_INTR_MODR_COMP_MAX ||
+	    ec->tx_max_coalesced_frames > MANA_INTR_MODR_COMP_MAX) {
+		NL_SET_ERR_MSG_FMT(extack,
+				   "coalesce frames must be <= %lu",
+				   MANA_INTR_MODR_COMP_MAX);
+		return -EINVAL;
+	}
+
+	if (ec->rx_coalesce_usecs != apc->intr_modr_rx_usec ||
+	    ec->rx_max_coalesced_frames != apc->intr_modr_rx_comp ||
+	    ec->tx_coalesce_usecs != apc->intr_modr_tx_usec ||
+	    ec->tx_max_coalesced_frames != apc->intr_modr_tx_comp)
+		modr_changed = true;
+
+	saved.intr_modr_rx_usec = apc->intr_modr_rx_usec;
+	saved.intr_modr_rx_comp = apc->intr_modr_rx_comp;
+	saved.intr_modr_tx_usec = apc->intr_modr_tx_usec;
+	saved.intr_modr_tx_comp = apc->intr_modr_tx_comp;
+
+	apc->intr_modr_rx_usec = ec->rx_coalesce_usecs;
+	apc->intr_modr_rx_comp = ec->rx_max_coalesced_frames;
+	apc->intr_modr_tx_usec = ec->tx_coalesce_usecs;
+	apc->intr_modr_tx_comp = ec->tx_max_coalesced_frames;
+
+	if (!!ec->use_adaptive_rx_coalesce != apc->rx_dim_enabled ||
+	    !!ec->use_adaptive_tx_coalesce != apc->tx_dim_enabled)
+		dim_changed = true;
+
+	saved.rx_dim_enabled = apc->rx_dim_enabled;
+	saved.tx_dim_enabled = apc->tx_dim_enabled;
+
+	saved.cqe_coalescing_enable = apc->cqe_coalescing_enable;
 	apc->cqe_coalescing_enable =
 		kernel_coal->rx_cqe_frames == MANA_RXCOMP_OOB_NUM_PPI;
 
-	if (!apc->port_is_up)
+	if (!apc->port_is_up) {
+		WRITE_ONCE(apc->rx_dim_enabled, !!ec->use_adaptive_rx_coalesce);
+		WRITE_ONCE(apc->tx_dim_enabled, !!ec->use_adaptive_tx_coalesce);
 		return 0;
+	}
 
-	err = mana_config_rss(apc, TRI_STATE_TRUE, false, false);
-	if (err)
-		apc->cqe_coalescing_enable = saved_cqe_coalescing_enable;
+	if (apc->cqe_coalescing_enable != saved.cqe_coalescing_enable) {
+		/* CQE coalescing setting is applied via RSS configuration. */
+		err = mana_config_rss(apc, TRI_STATE_TRUE, false, false);
+		if (err) {
+			netdev_err(ndev, "Change CQE coalescing failed: %d\n",
+				   err);
+			apc->cqe_coalescing_enable =
+				saved.cqe_coalescing_enable;
+			apc->intr_modr_rx_usec = saved.intr_modr_rx_usec;
+			apc->intr_modr_rx_comp = saved.intr_modr_rx_comp;
+			apc->intr_modr_tx_usec = saved.intr_modr_tx_usec;
+			apc->intr_modr_tx_comp = saved.intr_modr_tx_comp;
+			return err;
+		}
+	}
 
-	return err;
+	if (modr_changed || dim_changed) {
+		bool new_rx_dim = !!ec->use_adaptive_rx_coalesce;
+		bool new_tx_dim = !!ec->use_adaptive_tx_coalesce;
+		bool disable_rx_dim = saved.rx_dim_enabled && !new_rx_dim;
+		bool disable_tx_dim = saved.tx_dim_enabled && !new_tx_dim;
+		bool enable_rx_dim = !saved.rx_dim_enabled && new_rx_dim;
+		bool enable_tx_dim = !saved.tx_dim_enabled && new_tx_dim;
+		int q;
+
+		/* On disable: clear the per-port flag first and
+		 * synchronize_net() so any in-flight NAPI poll observes
+		 * the new value and will not schedule further DIM work;
+		 * then drain pending work and restore the static
+		 * moderation values.
+		 */
+		if (disable_rx_dim)
+			WRITE_ONCE(apc->rx_dim_enabled, false);
+		if (disable_tx_dim)
+			WRITE_ONCE(apc->tx_dim_enabled, false);
+		if (disable_rx_dim || disable_tx_dim)
+			synchronize_net();
+
+		for (q = 0; q < apc->num_queues; q++) {
+			struct mana_cq *rx_cq = &apc->rxqs[q]->rx_cq;
+			struct mana_cq *tx_cq = &apc->tx_qp[q]->tx_cq;
+
+			if (disable_rx_dim)
+				mana_dim_change(rx_cq, false);
+			else if (enable_rx_dim)
+				mana_dim_change(rx_cq, true);
+			else if (!new_rx_dim && modr_changed)
+				mana_gd_ring_dim(rx_cq->gdma_cq,
+						 apc->intr_modr_rx_usec, true,
+						 apc->intr_modr_rx_comp, true);
+
+			if (disable_tx_dim)
+				mana_dim_change(tx_cq, false);
+			else if (enable_tx_dim)
+				mana_dim_change(tx_cq, true);
+			else if (!new_tx_dim && modr_changed)
+				mana_gd_ring_dim(tx_cq->gdma_cq,
+						 apc->intr_modr_tx_usec, true,
+						 apc->intr_modr_tx_comp, true);
+		}
+
+		/* Publish the enable flag with release semantics so a
+		 * concurrent NAPI poll that observes it set also sees the DIM
+		 * (re)init done by mana_dim_change() above.
+		 */
+		if (enable_rx_dim)
+			/* pairs with smp_load_acquire() in mana_update_rx_dim() */
+			smp_store_release(&apc->rx_dim_enabled, true);
+		if (enable_tx_dim)
+			/* pairs with smp_load_acquire() in mana_update_tx_dim() */
+			smp_store_release(&apc->tx_dim_enabled, true);
+	}
+
+	return 0;
 }
 
 /* mana_set_channels - change the number of queues on a port
@@ -595,7 +740,13 @@ static int mana_get_link_ksettings(struct net_device *ndev,
 }
 
 const struct ethtool_ops mana_ethtool_ops = {
-	.supported_coalesce_params = ETHTOOL_COALESCE_RX_CQE_FRAMES,
+	.supported_coalesce_params = ETHTOOL_COALESCE_RX_CQE_FRAMES |
+				     ETHTOOL_COALESCE_RX_USECS |
+				     ETHTOOL_COALESCE_RX_MAX_FRAMES |
+				     ETHTOOL_COALESCE_TX_USECS |
+				     ETHTOOL_COALESCE_TX_MAX_FRAMES |
+				     ETHTOOL_COALESCE_USE_ADAPTIVE_RX |
+				     ETHTOOL_COALESCE_USE_ADAPTIVE_TX,
 	.op_needs_rtnl		= ETHTOOL_OP_NEEDS_RTNL_SCHANNELS |
 				  ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM |
 				  ETHTOOL_OP_NEEDS_RTNL_GLINK,
diff --git a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c
index 48b94ce774906..88c5c52e0e381 100644
--- a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c
+++ b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c
@@ -16,7 +16,7 @@
 #include <linux/net_tstamp.h>
 #include <linux/ptp_classify.h>
 #include <linux/ptp_pch.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 
 #define PCH_GBE_MAR_ENTRIES		16
 #define PCH_GBE_SHORT_PKT		64
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c b/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c
index 5281148776774..c999754afb5f3 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c
@@ -558,18 +558,15 @@ struct sync_item {
 void ionic_rx_filter_sync(struct ionic_lif *lif)
 {
 	struct device *dev = lif->ionic->dev;
-	struct list_head sync_add_list;
-	struct list_head sync_del_list;
 	struct sync_item *sync_item;
 	struct ionic_rx_filter *f;
+	LIST_HEAD(sync_add_list);
+	LIST_HEAD(sync_del_list);
 	struct hlist_head *head;
 	struct hlist_node *tmp;
 	struct sync_item *spos;
 	unsigned int i;
 
-	INIT_LIST_HEAD(&sync_add_list);
-	INIT_LIST_HEAD(&sync_del_list);
-
 	clear_bit(IONIC_LIF_F_FILTER_SYNC_NEEDED, lif->state);
 
 	/* Copy the filters to be added and deleted
diff --git a/drivers/net/ethernet/qlogic/qed/qed_dcbx.c b/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
index 3a5c25026858b..19c2b870feedf 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
@@ -639,7 +639,7 @@ qed_dcbx_get_operational_params(struct qed_hwfn *p_hwfn,
 	flags = p_hwfn->p_dcbx_info->operational.flags;
 
 	/* If DCBx version is non zero, then negotiation
-	 * was successfuly performed
+	 * was successfully performed
 	 */
 	p_operational = &params->operational;
 	enabled = !!(QED_MFW_GET_FIELD(flags, DCBX_CONFIG_VERSION) !=
diff --git a/drivers/net/ethernet/qualcomm/ppe/ppe.c b/drivers/net/ethernet/qualcomm/ppe/ppe.c
index be747510d9477..3c301e609d3e3 100644
--- a/drivers/net/ethernet/qualcomm/ppe/ppe.c
+++ b/drivers/net/ethernet/qualcomm/ppe/ppe.c
@@ -106,7 +106,6 @@ static const struct regmap_config regmap_config_ipq9574 = {
 	.rd_table = &ppe_reg_table,
 	.wr_table = &ppe_reg_table,
 	.max_register = 0xbef800,
-	.fast_io = true,
 };
 
 static int ppe_clock_init_and_reset(struct ppe_device *ppe_dev)
diff --git a/drivers/net/ethernet/qualcomm/ppe/ppe_config.c b/drivers/net/ethernet/qualcomm/ppe/ppe_config.c
index e9a0e22907a68..125b73be92b13 100644
--- a/drivers/net/ethernet/qualcomm/ppe/ppe_config.c
+++ b/drivers/net/ethernet/qualcomm/ppe/ppe_config.c
@@ -1367,7 +1367,8 @@ int ppe_rss_hash_config_set(struct ppe_device *ppe_dev, int mode,
  *
  * Return: 0 on success, negative error code on failure.
  */
-int ppe_ring_queue_map_set(struct ppe_device *ppe_dev, int ring_id, u32 *queue_map)
+int ppe_ring_queue_map_set(struct ppe_device *ppe_dev, int ring_id,
+			   const u32 *queue_map)
 {
 	u32 reg, queue_bitmap_val[PPE_RING_TO_QUEUE_BITMAP_WORD_CNT];
 
@@ -1380,7 +1381,7 @@ int ppe_ring_queue_map_set(struct ppe_device *ppe_dev, int ring_id, u32 *queue_m
 }
 
 static int ppe_config_bm_threshold(struct ppe_device *ppe_dev, int bm_port_id,
-				   const struct ppe_bm_port_config port_cfg)
+				   struct ppe_bm_port_config port_cfg)
 {
 	u32 reg, val, bm_fc_val[2];
 	int ret;
@@ -1586,7 +1587,7 @@ static int ppe_config_qm(struct ppe_device *ppe_dev)
 }
 
 static int ppe_node_scheduler_config(struct ppe_device *ppe_dev,
-				     const struct ppe_scheduler_port_config config)
+				     struct ppe_scheduler_port_config config)
 {
 	struct ppe_scheduler_cfg sch_cfg;
 	int ret, i;
diff --git a/drivers/net/ethernet/qualcomm/ppe/ppe_config.h b/drivers/net/ethernet/qualcomm/ppe/ppe_config.h
index 4bb45ca40144f..60493e51e0a43 100644
--- a/drivers/net/ethernet/qualcomm/ppe/ppe_config.h
+++ b/drivers/net/ethernet/qualcomm/ppe/ppe_config.h
@@ -313,5 +313,5 @@ int ppe_rss_hash_config_set(struct ppe_device *ppe_dev, int mode,
 			    struct ppe_rss_hash_cfg hash_cfg);
 int ppe_ring_queue_map_set(struct ppe_device *ppe_dev,
 			   int ring_id,
-			   u32 *queue_map);
+			   const u32 *queue_map);
 #endif
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
index 78d4df55740a1..bed6f63facf25 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
@@ -423,7 +423,8 @@ struct rmnet_endpoint *rmnet_get_endpoint(struct rmnet_port *port, u8 mux_id)
 {
 	struct rmnet_endpoint *ep;
 
-	hlist_for_each_entry_rcu(ep, &port->muxed_ep[mux_id], hlnode) {
+	hlist_for_each_entry_rcu(ep, &port->muxed_ep[mux_id], hlnode,
+				 lockdep_rtnl_is_held()) {
 		if (ep->mux_id == mux_id)
 			return ep;
 	}
diff --git a/drivers/net/ethernet/sfc/Kconfig b/drivers/net/ethernet/sfc/Kconfig
index c4c43434f3143..979f2801e2a8e 100644
--- a/drivers/net/ethernet/sfc/Kconfig
+++ b/drivers/net/ethernet/sfc/Kconfig
@@ -66,6 +66,15 @@ config SFC_MCDI_LOGGING
 	  Driver-Interface) commands and responses, allowing debugging of
 	  driver/firmware interaction.  The tracing is actually enabled by
 	  a sysfs file 'mcdi_logging' under the PCI device.
+config SFC_CXL
+	bool "Solarflare SFC9100-family CXL support"
+	depends on SFC && CXL_BUS >= SFC
+	default SFC
+	help
+	  This enables SFC CXL support if the kernel is configuring CXL for
+	  using CTPIO with CXL.mem. The SFC device with CXL support and
+	  with a CXL-aware firmware can be used for minimizing latencies
+	  when sending through CTPIO.
 
 source "drivers/net/ethernet/sfc/falcon/Kconfig"
 source "drivers/net/ethernet/sfc/siena/Kconfig"
diff --git a/drivers/net/ethernet/sfc/Makefile b/drivers/net/ethernet/sfc/Makefile
index d99039ec468d6..bb0f1891cde65 100644
--- a/drivers/net/ethernet/sfc/Makefile
+++ b/drivers/net/ethernet/sfc/Makefile
@@ -13,6 +13,7 @@ sfc-$(CONFIG_SFC_SRIOV)	+= sriov.o ef10_sriov.o ef100_sriov.o ef100_rep.o \
                            mae.o tc.o tc_bindings.o tc_counters.o \
                            tc_encap_actions.o tc_conntrack.o
 
+sfc-$(CONFIG_SFC_CXL)	+= efx_cxl.o
 obj-$(CONFIG_SFC)	+= sfc.o
 
 obj-$(CONFIG_SFC_FALCON) += falcon/
diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c
index 7e04f115bbaaa..73bc064929f63 100644
--- a/drivers/net/ethernet/sfc/ef10.c
+++ b/drivers/net/ethernet/sfc/ef10.c
@@ -24,6 +24,7 @@
 #include <linux/wait.h>
 #include <linux/workqueue.h>
 #include <net/udp_tunnel.h>
+#include "efx_cxl.h"
 
 /* Hardware control for EF10 architecture including 'Huntington'. */
 
@@ -106,7 +107,7 @@ static int efx_ef10_get_vf_index(struct efx_nic *efx)
 
 static int efx_ef10_init_datapath_caps(struct efx_nic *efx)
 {
-	MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_CAPABILITIES_V4_OUT_LEN);
+	MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_CAPABILITIES_V7_OUT_LEN);
 	struct efx_ef10_nic_data *nic_data = efx->nic_data;
 	size_t outlen;
 	int rc;
@@ -177,6 +178,12 @@ static int efx_ef10_init_datapath_caps(struct efx_nic *efx)
 			  efx->num_mac_stats);
 	}
 
+	if (outlen < MC_CMD_GET_CAPABILITIES_V7_OUT_LEN)
+		nic_data->datapath_caps3 = 0;
+	else
+		nic_data->datapath_caps3 = MCDI_DWORD(outbuf,
+						      GET_CAPABILITIES_V7_OUT_FLAGS3);
+
 	return 0;
 }
 
@@ -1140,6 +1147,9 @@ static int efx_ef10_dimension_resources(struct efx_nic *efx)
 	unsigned int channel_vis, pio_write_vi_base, max_vis;
 	struct efx_ef10_nic_data *nic_data = efx->nic_data;
 	unsigned int uc_mem_map_size, wc_mem_map_size;
+#ifdef CONFIG_SFC_CXL
+	struct efx_probe_data *probe_data;
+#endif
 	void __iomem *membase;
 	int rc;
 
@@ -1263,8 +1273,23 @@ static int efx_ef10_dimension_resources(struct efx_nic *efx)
 	iounmap(efx->membase);
 	efx->membase = membase;
 
-	/* Set up the WC mapping if needed */
-	if (wc_mem_map_size) {
+	if (!wc_mem_map_size)
+		goto skip_pio;
+
+	/* Set up the WC mapping */
+
+#ifdef CONFIG_SFC_CXL
+	probe_data = container_of(efx, struct efx_probe_data, efx);
+	if ((nic_data->datapath_caps3 &
+	    (1 << MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN)) &&
+	    probe_data->cxl_pio_initialised) {
+		/* Using PIO through CXL mapping */
+		nic_data->pio_write_base = probe_data->cxl->ctpio_cxl;
+		nic_data->pio_write_vi_base = pio_write_vi_base;
+	} else
+#endif
+	{
+		/* Using legacy PIO BAR mapping */
 		nic_data->wc_membase = ioremap_wc(efx->membase_phys +
 						  uc_mem_map_size,
 						  wc_mem_map_size);
@@ -1279,12 +1304,14 @@ static int efx_ef10_dimension_resources(struct efx_nic *efx)
 			nic_data->wc_membase +
 			(pio_write_vi_base * efx->vi_stride + ER_DZ_TX_PIOBUF -
 			 uc_mem_map_size);
-
-		rc = efx_ef10_link_piobufs(efx);
-		if (rc)
-			efx_ef10_free_piobufs(efx);
 	}
 
+	rc = efx_ef10_link_piobufs(efx);
+	if (rc)
+		efx_ef10_free_piobufs(efx);
+
+skip_pio:
+
 	netif_dbg(efx, probe, efx->net_dev,
 		  "memory BAR at %pa (virtual %p+%x UC, %p+%x WC)\n",
 		  &efx->membase_phys, efx->membase, uc_mem_map_size,
diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
index 8f136a11d3968..3806cd3dd7f48 100644
--- a/drivers/net/ethernet/sfc/efx.c
+++ b/drivers/net/ethernet/sfc/efx.c
@@ -34,6 +34,7 @@
 #include "selftest.h"
 #include "sriov.h"
 #include "efx_devlink.h"
+#include "efx_cxl.h"
 
 #include "mcdi_port_common.h"
 #include "mcdi_pcol.h"
@@ -981,12 +982,15 @@ static void efx_pci_remove(struct pci_dev *pci_dev)
 	efx_pci_remove_main(efx);
 
 	efx_fini_io(efx);
+
+	probe_data = container_of(efx, struct efx_probe_data, efx);
+	efx_cxl_exit(probe_data);
+
 	pci_dbg(efx->pci_dev, "shutdown successful\n");
 
 	efx_fini_devlink_and_unlock(efx);
 	efx_fini_struct(efx);
 	free_netdev(efx->net_dev);
-	probe_data = container_of(efx, struct efx_probe_data, efx);
 	kfree(probe_data);
 };
 
@@ -1190,6 +1194,17 @@ static int efx_pci_probe(struct pci_dev *pci_dev,
 	if (rc)
 		goto fail2;
 
+	/* A successful cxl initialization implies a CXL region created to be
+	 * used for PIO buffers. If there is no CXL support legacy PIO buffers
+	 * defined at specific PCI BAR regions will be used. If there is CXL
+	 * support and the cxl initialization fails, the driver probe fails.
+	 */
+	rc = efx_cxl_init(probe_data);
+	if (rc) {
+		pci_err(pci_dev, "CXL initialization failed with error %d\n", rc);
+		goto fail3;
+	}
+
 	rc = efx_pci_probe_post_io(efx);
 	if (rc) {
 		/* On failure, retry once immediately.
@@ -1228,6 +1243,7 @@ static int efx_pci_probe(struct pci_dev *pci_dev,
 	return 0;
 
  fail3:
+	efx_cxl_exit(probe_data);
 	efx_fini_io(efx);
  fail2:
 	efx_fini_struct(efx);
diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
new file mode 100644
index 0000000000000..348d7404cd7aa
--- /dev/null
+++ b/drivers/net/ethernet/sfc/efx_cxl.c
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/****************************************************************************
+ *
+ * Driver for AMD network controllers and boards
+ * Copyright (C) 2025, Advanced Micro Devices, Inc.
+ */
+
+#include <linux/pci.h>
+
+#include <cxl/cxl.h>
+#include <cxl/pci.h>
+#include "net_driver.h"
+#include "efx_cxl.h"
+
+#define EFX_CTPIO_BUFFER_SIZE	SZ_256M
+
+int efx_cxl_init(struct efx_probe_data *probe_data)
+{
+	struct efx_nic *efx = &probe_data->efx;
+	struct pci_dev *pci_dev = efx->pci_dev;
+	struct range cxl_pio_range;
+	struct efx_cxl *cxl;
+	u16 dvsec;
+	int rc;
+
+	/* Is the device configured with and using CXL? */
+	if (!pcie_is_cxl(pci_dev))
+		return 0;
+
+	dvsec = pci_find_dvsec_capability(pci_dev, PCI_VENDOR_ID_CXL,
+					  PCI_DVSEC_CXL_DEVICE);
+	if (!dvsec) {
+		pci_info(pci_dev, "CXL_DVSEC_PCIE_DEVICE capability not found\n");
+		return 0;
+	}
+
+	pci_dbg(pci_dev, "CXL_DVSEC_PCIE_DEVICE capability found\n");
+
+	/* Create a cxl_dev_state embedded in the cxl struct using cxl core api
+	 * specifying no mbox available.
+	 */
+	cxl = devm_cxl_dev_state_create(&pci_dev->dev, CXL_DEVTYPE_DEVMEM,
+					pci_get_dsn(pci_dev), dvsec,
+					struct efx_cxl, cxlds, false);
+
+	if (!cxl)
+		return -ENOMEM;
+
+	rc = cxl_pci_setup_regs(pci_dev, CXL_REGLOC_RBI_COMPONENT,
+				&cxl->cxlds.reg_map);
+	if (rc) {
+		pci_err(pci_dev, "No component registers\n");
+		return rc;
+	}
+
+	if (!cxl->cxlds.reg_map.component_map.hdm_decoder.valid) {
+		pci_err(pci_dev, "Expected HDM component register not found\n");
+		return -ENODEV;
+	}
+
+	if (!cxl->cxlds.reg_map.component_map.ras.valid) {
+		pci_err(pci_dev, "Expected RAS component register not found\n");
+		return -ENODEV;
+	}
+
+	/* Set media ready explicitly as there are neither mailbox for checking
+	 * this state nor the CXL register involved, both not mandatory for
+	 * type2.
+	 */
+	cxl->cxlds.media_ready = true;
+
+	if (cxl_set_capacity(&cxl->cxlds, EFX_CTPIO_BUFFER_SIZE)) {
+		pci_err(pci_dev, "dpa capacity setup failed\n");
+		return -ENODEV;
+	}
+
+	cxl->cxlmd = devm_cxl_probe_mem(&cxl->cxlds, &cxl_pio_range);
+	if (IS_ERR(cxl->cxlmd)) {
+		pci_err(pci_dev, "CXL accel memdev creation failed\n");
+		return PTR_ERR(cxl->cxlmd);
+	}
+
+	cxl->ctpio_cxl = ioremap_wc(cxl_pio_range.start,
+				    range_len(&cxl_pio_range));
+	if (!cxl->ctpio_cxl) {
+		pci_err(pci_dev, "CXL ioremap region (%pra) failed\n",
+			&cxl_pio_range);
+		return -ENOMEM;
+	}
+
+	probe_data->cxl_pio_initialised = true;
+	probe_data->cxl = cxl;
+
+	return 0;
+}
+
+void efx_cxl_exit(struct efx_probe_data *probe_data)
+{
+	if (!probe_data->cxl)
+		return;
+
+	iounmap(probe_data->cxl->ctpio_cxl);
+}
+
+MODULE_IMPORT_NS("CXL");
diff --git a/drivers/net/ethernet/sfc/efx_cxl.h b/drivers/net/ethernet/sfc/efx_cxl.h
new file mode 100644
index 0000000000000..3e2705cb063f6
--- /dev/null
+++ b/drivers/net/ethernet/sfc/efx_cxl.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/****************************************************************************
+ * Driver for AMD network controllers and boards
+ * Copyright (C) 2025, Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, incorporated herein by reference.
+ */
+
+#ifndef EFX_CXL_H
+#define EFX_CXL_H
+
+#ifdef CONFIG_SFC_CXL
+
+#include <cxl/cxl.h>
+
+struct efx_probe_data;
+
+struct efx_cxl {
+	struct cxl_dev_state cxlds;
+	struct cxl_memdev *cxlmd;
+	void __iomem *ctpio_cxl;
+};
+
+int efx_cxl_init(struct efx_probe_data *probe_data);
+void efx_cxl_exit(struct efx_probe_data *probe_data);
+#else
+static inline int efx_cxl_init(struct efx_probe_data *probe_data) { return 0; }
+static inline void efx_cxl_exit(struct efx_probe_data *probe_data) {}
+#endif
+#endif
diff --git a/drivers/net/ethernet/sfc/mcdi.c b/drivers/net/ethernet/sfc/mcdi.c
index e65db9b707244..b806d3d90c42e 100644
--- a/drivers/net/ethernet/sfc/mcdi.c
+++ b/drivers/net/ethernet/sfc/mcdi.c
@@ -7,6 +7,7 @@
 #include <linux/delay.h>
 #include <linux/moduleparam.h>
 #include <linux/atomic.h>
+#include <linux/slab.h>
 #include "net_driver.h"
 #include "nic.h"
 #include "io.h"
@@ -71,7 +72,7 @@ int efx_mcdi_init(struct efx_nic *efx)
 	mcdi->efx = efx;
 #ifdef CONFIG_SFC_MCDI_LOGGING
 	/* consuming code assumes buffer is page-sized */
-	mcdi->logging_buffer = (char *)__get_free_page(GFP_KERNEL);
+	mcdi->logging_buffer = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!mcdi->logging_buffer)
 		goto fail1;
 	mcdi->logging_enabled = mcdi_logging_default;
@@ -112,7 +113,7 @@ int efx_mcdi_init(struct efx_nic *efx)
 	return 0;
 fail2:
 #ifdef CONFIG_SFC_MCDI_LOGGING
-	free_page((unsigned long)mcdi->logging_buffer);
+	kfree(mcdi->logging_buffer);
 fail1:
 #endif
 	kfree(efx->mcdi);
@@ -138,7 +139,7 @@ void efx_mcdi_fini(struct efx_nic *efx)
 		return;
 
 #ifdef CONFIG_SFC_MCDI_LOGGING
-	free_page((unsigned long)efx->mcdi->iface.logging_buffer);
+	kfree(efx->mcdi->iface.logging_buffer);
 #endif
 
 	kfree(efx->mcdi);
diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
index b98c259f672db..3964b2c56609c 100644
--- a/drivers/net/ethernet/sfc/net_driver.h
+++ b/drivers/net/ethernet/sfc/net_driver.h
@@ -1197,14 +1197,24 @@ struct efx_nic {
 	atomic_t n_rx_noskb_drops;
 };
 
+#ifdef CONFIG_SFC_CXL
+struct efx_cxl;
+#endif
+
 /**
  * struct efx_probe_data - State after hardware probe
  * @pci_dev: The PCI device
  * @efx: Efx NIC details
+ * @cxl: details of related cxl objects
+ * @cxl_pio_initialised: cxl initialization outcome.
  */
 struct efx_probe_data {
 	struct pci_dev *pci_dev;
 	struct efx_nic efx;
+#ifdef CONFIG_SFC_CXL
+	struct efx_cxl *cxl;
+	bool cxl_pio_initialised;
+#endif
 };
 
 static inline struct efx_nic *efx_netdev_priv(struct net_device *dev)
diff --git a/drivers/net/ethernet/sfc/nic.h b/drivers/net/ethernet/sfc/nic.h
index ec3b2df43b68d..7480f9995dfb8 100644
--- a/drivers/net/ethernet/sfc/nic.h
+++ b/drivers/net/ethernet/sfc/nic.h
@@ -152,6 +152,8 @@ enum {
  *	%MC_CMD_GET_CAPABILITIES response)
  * @datapath_caps2: Further Capabilities of datapath firmware (FLAGS2 field of
  * %MC_CMD_GET_CAPABILITIES response)
+ * @datapath_caps3: Further Capabilities of datapath firmware (FLAGS3 field of
+ * %MC_CMD_GET_CAPABILITIES response)
  * @rx_dpcpu_fw_id: Firmware ID of the RxDPCPU
  * @tx_dpcpu_fw_id: Firmware ID of the TxDPCPU
  * @must_probe_vswitching: Flag: vswitching has yet to be setup after MC reboot
@@ -187,6 +189,7 @@ struct efx_ef10_nic_data {
 	bool must_check_datapath_caps;
 	u32 datapath_caps;
 	u32 datapath_caps2;
+	u32 datapath_caps3;
 	unsigned int rx_dpcpu_fw_id;
 	unsigned int tx_dpcpu_fw_id;
 	bool must_probe_vswitching;
diff --git a/drivers/net/ethernet/sfc/siena/mcdi.c b/drivers/net/ethernet/sfc/siena/mcdi.c
index 4d0d6bd5d3d11..048c1e6017c0a 100644
--- a/drivers/net/ethernet/sfc/siena/mcdi.c
+++ b/drivers/net/ethernet/sfc/siena/mcdi.c
@@ -7,6 +7,7 @@
 #include <linux/delay.h>
 #include <linux/moduleparam.h>
 #include <linux/atomic.h>
+#include <linux/slab.h>
 #include "net_driver.h"
 #include "nic.h"
 #include "io.h"
@@ -73,7 +74,7 @@ int efx_siena_mcdi_init(struct efx_nic *efx)
 	mcdi->efx = efx;
 #ifdef CONFIG_SFC_SIENA_MCDI_LOGGING
 	/* consuming code assumes buffer is page-sized */
-	mcdi->logging_buffer = (char *)__get_free_page(GFP_KERNEL);
+	mcdi->logging_buffer = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!mcdi->logging_buffer)
 		goto fail1;
 	mcdi->logging_enabled = efx_siena_mcdi_logging_default;
@@ -116,7 +117,7 @@ int efx_siena_mcdi_init(struct efx_nic *efx)
 	return 0;
 fail2:
 #ifdef CONFIG_SFC_SIENA_MCDI_LOGGING
-	free_page((unsigned long)mcdi->logging_buffer);
+	kfree(mcdi->logging_buffer);
 fail1:
 #endif
 	kfree(efx->mcdi);
@@ -142,7 +143,7 @@ void efx_siena_mcdi_fini(struct efx_nic *efx)
 		return;
 
 #ifdef CONFIG_SFC_SIENA_MCDI_LOGGING
-	free_page((unsigned long)efx->mcdi->iface.logging_buffer);
+	kfree(efx->mcdi->iface.logging_buffer);
 #endif
 
 	kfree(efx->mcdi);
diff --git a/drivers/net/ethernet/sgi/ioc3-eth.c b/drivers/net/ethernet/sgi/ioc3-eth.c
index 39731069d99e0..009f37105eafd 100644
--- a/drivers/net/ethernet/sgi/ioc3-eth.c
+++ b/drivers/net/ethernet/sgi/ioc3-eth.c
@@ -967,11 +967,12 @@ static void ioc3eth_remove(struct platform_device *pdev)
 	struct net_device *dev = platform_get_drvdata(pdev);
 	struct ioc3_private *ip = netdev_priv(dev);
 
+	unregister_netdev(dev);
+	timer_delete_sync(&ip->ioc3_timer);
+
 	dma_free_coherent(ip->dma_dev, RX_RING_SIZE, ip->rxr, ip->rxr_dma);
 	dma_free_coherent(ip->dma_dev, TX_RING_SIZE + SZ_16K - 1, ip->tx_ring, ip->txr_dma);
 
-	unregister_netdev(dev);
-	timer_delete_sync(&ip->ioc3_timer);
 	free_netdev(dev);
 }
 
@@ -1061,9 +1062,9 @@ static netdev_tx_t ioc3_start_xmit(struct sk_buff *skb, struct net_device *dev)
 		d1 = dma_map_single(ip->dma_dev, skb->data, s1, DMA_TO_DEVICE);
 		if (dma_mapping_error(ip->dma_dev, d1))
 			goto drop_packet;
-		d2 = dma_map_single(ip->dma_dev, (void *)b2, s1, DMA_TO_DEVICE);
+		d2 = dma_map_single(ip->dma_dev, (void *)b2, s2, DMA_TO_DEVICE);
 		if (dma_mapping_error(ip->dma_dev, d2)) {
-			dma_unmap_single(ip->dma_dev, d1, len, DMA_TO_DEVICE);
+			dma_unmap_single(ip->dma_dev, d1, s1, DMA_TO_DEVICE);
 			goto drop_packet;
 		}
 		desc->p1     = cpu_to_be64(ioc3_map(d1, PCI64_ATTR_PREF));
diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c
index a60ef32b35b82..4ad9528322c45 100644
--- a/drivers/net/gtp.c
+++ b/drivers/net/gtp.c
@@ -151,7 +151,8 @@ static struct pdp_ctx *gtp0_pdp_find(struct gtp_dev *gtp, u64 tid, u16 family)
 
 	head = &gtp->tid_hash[gtp0_hashfn(tid) % gtp->hash_size];
 
-	hlist_for_each_entry_rcu(pdp, head, hlist_tid) {
+	hlist_for_each_entry_rcu(pdp, head, hlist_tid,
+				 lockdep_rtnl_is_held()) {
 		if (pdp->af == family &&
 		    pdp->gtp_version == GTP_V0 &&
 		    pdp->u.v0.tid == tid)
@@ -168,7 +169,8 @@ static struct pdp_ctx *gtp1_pdp_find(struct gtp_dev *gtp, u32 tid, u16 family)
 
 	head = &gtp->tid_hash[gtp1u_hashfn(tid) % gtp->hash_size];
 
-	hlist_for_each_entry_rcu(pdp, head, hlist_tid) {
+	hlist_for_each_entry_rcu(pdp, head, hlist_tid,
+				 lockdep_rtnl_is_held()) {
 		if (pdp->af == family &&
 		    pdp->gtp_version == GTP_V1 &&
 		    pdp->u.v1.i_tei == tid)
@@ -185,7 +187,8 @@ static struct pdp_ctx *ipv4_pdp_find(struct gtp_dev *gtp, __be32 ms_addr)
 
 	head = &gtp->addr_hash[ipv4_hashfn(ms_addr) % gtp->hash_size];
 
-	hlist_for_each_entry_rcu(pdp, head, hlist_addr) {
+	hlist_for_each_entry_rcu(pdp, head, hlist_addr,
+				 lockdep_rtnl_is_held()) {
 		if (pdp->af == AF_INET &&
 		    pdp->ms.addr.s_addr == ms_addr)
 			return pdp;
@@ -220,7 +223,8 @@ static struct pdp_ctx *ipv6_pdp_find(struct gtp_dev *gtp,
 
 	head = &gtp->addr_hash[ipv6_hashfn(ms_addr) % gtp->hash_size];
 
-	hlist_for_each_entry_rcu(pdp, head, hlist_addr) {
+	hlist_for_each_entry_rcu(pdp, head, hlist_addr,
+				 lockdep_rtnl_is_held()) {
 		if (pdp->af == AF_INET6 &&
 		    ipv6_pdp_addr_equal(&pdp->ms.addr6, ms_addr))
 			return pdp;
diff --git a/drivers/net/ipvlan/ipvlan.h b/drivers/net/ipvlan/ipvlan.h
index 80f84fc87008b..8d05ad480438f 100644
--- a/drivers/net/ipvlan/ipvlan.h
+++ b/drivers/net/ipvlan/ipvlan.h
@@ -69,6 +69,7 @@ struct ipvl_dev {
 	DECLARE_BITMAP(mac_filters, IPVLAN_MAC_FILTER_SIZE);
 	netdev_features_t	sfeatures;
 	u32			msg_enable;
+	bool			dying;
 };
 
 struct ipvl_addr {
@@ -91,12 +92,13 @@ struct ipvl_port {
 	struct hlist_head	hlhead[IPVLAN_HASH_SIZE];
 	spinlock_t		addrs_lock; /* guards hash-table and addrs */
 	struct list_head	ipvlans;
+	struct mutex		pnodes_lock;
 	u16			mode;
 	u16			flags;
 	u16			dev_id_start;
 	struct work_struct	wq;
 	struct sk_buff_head	backlog;
-	int			count;
+	refcount_t		count;
 	struct ida		ida;
 	netdevice_tracker	dev_tracker;
 };
@@ -168,7 +170,8 @@ void ipvlan_count_rx(const struct ipvl_dev *ipvlan,
 		     unsigned int len, bool success, bool mcast);
 int ipvlan_link_new(struct net_device *dev, struct rtnl_newlink_params *params,
 		    struct netlink_ext_ack *extack);
-void ipvlan_link_delete(struct net_device *dev, struct list_head *head);
+void __ipvlan_link_delete(struct net *net, struct net_device *dev,
+			  struct list_head *head);
 void ipvlan_link_setup(struct net_device *dev);
 int ipvlan_link_register(struct rtnl_link_ops *ops);
 #ifdef CONFIG_IPVLAN_L3S
@@ -207,4 +210,9 @@ static inline bool netif_is_ipvlan_port(const struct net_device *dev)
 	return rcu_access_pointer(dev->rx_handler) == ipvlan_handle_frame;
 }
 
+#if IS_ENABLED(CONFIG_IPVTAP)
+extern void (*__ipvtap_dellink_ptr)(struct net *net, struct net_device *dev,
+				    struct list_head *head);
+#endif
+
 #endif /* __IPVLAN_H */
diff --git a/drivers/net/ipvlan/ipvlan_main.c b/drivers/net/ipvlan/ipvlan_main.c
index ed46439a9f4eb..ee46a55f73d14 100644
--- a/drivers/net/ipvlan/ipvlan_main.c
+++ b/drivers/net/ipvlan/ipvlan_main.c
@@ -7,6 +7,12 @@
 
 #include "ipvlan.h"
 
+#if IS_ENABLED(CONFIG_IPVTAP)
+void (*__ipvtap_dellink_ptr)(struct net *net, struct net_device *dev,
+			     struct list_head *head);
+EXPORT_SYMBOL(__ipvtap_dellink_ptr);
+#endif
+
 static int ipvlan_set_port_mode(struct ipvl_port *port, u16 nval,
 				struct netlink_ext_ack *extack)
 {
@@ -16,6 +22,8 @@ static int ipvlan_set_port_mode(struct ipvl_port *port, u16 nval,
 
 	ASSERT_RTNL();
 	if (port->mode != nval) {
+		mutex_lock(&port->pnodes_lock);
+
 		list_for_each_entry(ipvlan, &port->ipvlans, pnode) {
 			flags = ipvlan->dev->flags;
 			if (nval == IPVLAN_MODE_L3 || nval == IPVLAN_MODE_L3S) {
@@ -40,6 +48,8 @@ static int ipvlan_set_port_mode(struct ipvl_port *port, u16 nval,
 			ipvlan_l3s_unregister(port);
 		}
 		port->mode = nval;
+
+		mutex_unlock(&port->pnodes_lock);
 	}
 	return 0;
 
@@ -56,6 +66,8 @@ static int ipvlan_set_port_mode(struct ipvl_port *port, u16 nval,
 					 NULL);
 	}
 
+	mutex_unlock(&port->pnodes_lock);
+
 	return err;
 }
 
@@ -76,6 +88,7 @@ static int ipvlan_port_create(struct net_device *dev)
 		INIT_HLIST_HEAD(&port->hlhead[idx]);
 
 	spin_lock_init(&port->addrs_lock);
+	mutex_init(&port->pnodes_lock);
 	skb_queue_head_init(&port->backlog);
 	INIT_WORK(&port->wq, ipvlan_process_multicast);
 	ida_init(&port->ida);
@@ -86,6 +99,7 @@ static int ipvlan_port_create(struct net_device *dev)
 		goto err;
 
 	netdev_hold(dev, &port->dev_tracker, GFP_KERNEL);
+
 	return 0;
 
 err:
@@ -93,16 +107,18 @@ static int ipvlan_port_create(struct net_device *dev)
 	return err;
 }
 
-static void ipvlan_port_destroy(struct net_device *dev)
+static void ipvlan_port_destroy(struct ipvl_port *port)
 {
-	struct ipvl_port *port = ipvlan_port_get_rtnl(dev);
+	struct net_device *dev = port->dev;
 	struct sk_buff *skb;
 
-	netdev_put(dev, &port->dev_tracker);
 	if (port->mode == IPVLAN_MODE_L3S)
 		ipvlan_l3s_unregister(port);
+
 	netdev_rx_handler_unregister(dev);
 	cancel_work_sync(&port->wq);
+	netdev_put(dev, &port->dev_tracker);
+
 	while ((skb = __skb_dequeue(&port->backlog)) != NULL) {
 		dev_put(skb->dev);
 		kfree_skb(skb);
@@ -111,6 +127,27 @@ static void ipvlan_port_destroy(struct net_device *dev)
 	kfree(port);
 }
 
+static void ipvlan_port_put(struct ipvl_port *port)
+{
+	if (refcount_dec_and_test(&port->count))
+		ipvlan_port_destroy(port);
+}
+
+static struct ipvl_port *ipvlan_port_get(struct net_device *dev)
+{
+	struct ipvl_port *port = NULL;
+
+	rcu_read_lock();
+	if (netif_is_ipvlan_port(dev)) {
+		port = ipvlan_port_get_rcu(dev);
+		if (!refcount_inc_not_zero(&port->count))
+			port = NULL;
+	}
+	rcu_read_unlock();
+
+	return port;
+}
+
 #define IPVLAN_ALWAYS_ON_OFLOADS \
 	(NETIF_F_SG | NETIF_F_HW_CSUM | \
 	 NETIF_F_GSO_ROBUST | NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL)
@@ -153,30 +190,45 @@ static int ipvlan_init(struct net_device *dev)
 	if (!ipvlan->pcpu_stats)
 		return -ENOMEM;
 
+	netdev_lock(phy_dev);
+
 	if (!netif_is_ipvlan_port(phy_dev)) {
 		err = ipvlan_port_create(phy_dev);
 		if (err < 0) {
+			netdev_unlock(phy_dev);
 			free_percpu(ipvlan->pcpu_stats);
 			return err;
 		}
+		port = ipvlan_port_get_rtnl(phy_dev);
+		refcount_set(&port->count, 1);
+	} else {
+		port = ipvlan_port_get_rtnl(phy_dev);
+		refcount_inc(&port->count);
 	}
-	port = ipvlan_port_get_rtnl(phy_dev);
-	port->count += 1;
+
+	netdev_unlock(phy_dev);
+
+	ipvlan->port = port;
+
 	return 0;
 }
 
 static void ipvlan_uninit(struct net_device *dev)
 {
 	struct ipvl_dev *ipvlan = netdev_priv(dev);
-	struct net_device *phy_dev = ipvlan->phy_dev;
-	struct ipvl_port *port;
+	netdevice_tracker dev_tracker;
+	struct net_device *phy_dev;
 
 	free_percpu(ipvlan->pcpu_stats);
 
-	port = ipvlan_port_get_rtnl(phy_dev);
-	port->count -= 1;
-	if (!port->count)
-		ipvlan_port_destroy(port->dev);
+	phy_dev = ipvlan->phy_dev;
+	netdev_hold(phy_dev, &dev_tracker, GFP_KERNEL);
+	netdev_lock(phy_dev);
+
+	ipvlan_port_put(ipvlan->port);
+
+	netdev_unlock(phy_dev);
+	netdev_put(phy_dev, &dev_tracker);
 }
 
 static int ipvlan_open(struct net_device *dev)
@@ -594,9 +646,7 @@ int ipvlan_link_new(struct net_device *dev, struct rtnl_newlink_params *params,
 	if (err < 0)
 		return err;
 
-	/* ipvlan_init() would have created the port, if required */
-	port = ipvlan_port_get_rtnl(phy_dev);
-	ipvlan->port = port;
+	port = ipvlan->port;
 
 	/* If the port-id base is at the MAX value, then wrap it around and
 	 * begin from 0x1 again. This may be due to a busy system where lots
@@ -639,7 +689,10 @@ int ipvlan_link_new(struct net_device *dev, struct rtnl_newlink_params *params,
 	if (err)
 		goto unlink_netdev;
 
+	mutex_lock(&port->pnodes_lock);
 	list_add_tail_rcu(&ipvlan->pnode, &port->ipvlans);
+	mutex_unlock(&port->pnodes_lock);
+
 	netif_stacked_transfer_operstate(phy_dev, dev);
 	return 0;
 
@@ -653,7 +706,8 @@ int ipvlan_link_new(struct net_device *dev, struct rtnl_newlink_params *params,
 }
 EXPORT_SYMBOL_GPL(ipvlan_link_new);
 
-void ipvlan_link_delete(struct net_device *dev, struct list_head *head)
+void __ipvlan_link_delete(struct net *net, struct net_device *dev,
+			  struct list_head *head)
 {
 	struct ipvl_dev *ipvlan = netdev_priv(dev);
 	struct ipvl_addr *addr, *next;
@@ -668,10 +722,20 @@ void ipvlan_link_delete(struct net_device *dev, struct list_head *head)
 
 	ida_free(&ipvlan->port->ida, dev->dev_id);
 	list_del_rcu(&ipvlan->pnode);
-	unregister_netdevice_queue(dev, head);
+	unregister_netdevice_queue_net(net, dev, head);
 	netdev_upper_dev_unlink(ipvlan->phy_dev, dev);
 }
-EXPORT_SYMBOL_GPL(ipvlan_link_delete);
+EXPORT_SYMBOL(__ipvlan_link_delete);
+
+static void ipvlan_link_delete(struct net_device *dev, struct list_head *head)
+{
+	struct ipvl_dev *ipvlan = netdev_priv(dev);
+
+	mutex_lock(&ipvlan->port->pnodes_lock);
+	if (!ipvlan->dying)
+		__ipvlan_link_delete(dev_net(dev), dev, head);
+	mutex_unlock(&ipvlan->port->pnodes_lock);
+}
 
 void ipvlan_link_setup(struct net_device *dev)
 {
@@ -729,14 +793,19 @@ static int ipvlan_device_event(struct notifier_block *unused,
 	struct netdev_notifier_pre_changeaddr_info *prechaddr_info;
 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
 	struct ipvl_dev *ipvlan, *next;
+	int err, ret = NOTIFY_DONE;
 	struct ipvl_port *port;
 	LIST_HEAD(lst_kill);
-	int err;
 
-	if (!netif_is_ipvlan_port(dev))
-		return NOTIFY_DONE;
+	if (event == NETDEV_PRECHANGEUPPER ||
+	    event == NETDEV_CHANGEUPPER)
+		return ret;
 
-	port = ipvlan_port_get_rtnl(dev);
+	port = ipvlan_port_get(dev);
+	if (!port)
+		return ret;
+
+	mutex_lock(&port->pnodes_lock);
 
 	switch (event) {
 	case NETDEV_UP:
@@ -760,16 +829,26 @@ static int ipvlan_device_event(struct notifier_block *unused,
 			ipvlan_migrate_l3s_hook(oldnet, newnet);
 		break;
 	}
-	case NETDEV_UNREGISTER:
+	case NETDEV_UNREGISTER: {
+		struct net *net = dev_net(dev);
+
 		if (dev->reg_state != NETREG_UNREGISTERING)
 			break;
 
-		list_for_each_entry_safe(ipvlan, next, &port->ipvlans, pnode)
-			ipvlan->dev->rtnl_link_ops->dellink(ipvlan->dev,
-							    &lst_kill);
+		list_for_each_entry_safe(ipvlan, next, &port->ipvlans, pnode) {
+			ipvlan->dying = true;
+
+#if IS_ENABLED(CONFIG_IPVTAP)
+			if (ipvlan->dev->rtnl_link_ops != &ipvlan_link_ops)
+				__ipvtap_dellink_ptr(net, ipvlan->dev, &lst_kill);
+			else
+#endif
+				__ipvlan_link_delete(net, ipvlan->dev, &lst_kill);
+		}
+
 		unregister_netdevice_many(&lst_kill);
 		break;
-
+	}
 	case NETDEV_FEAT_CHANGE:
 		list_for_each_entry(ipvlan, &port->ipvlans, pnode) {
 			netif_inherit_tso_max(ipvlan->dev, dev);
@@ -788,8 +867,10 @@ static int ipvlan_device_event(struct notifier_block *unused,
 			err = netif_pre_changeaddr_notify(ipvlan->dev,
 							  prechaddr_info->dev_addr,
 							  extack);
-			if (err)
-				return notifier_from_errno(err);
+			if (err) {
+				ret = notifier_from_errno(err);
+				break;
+			}
 		}
 		break;
 
@@ -802,7 +883,8 @@ static int ipvlan_device_event(struct notifier_block *unused,
 
 	case NETDEV_PRE_TYPE_CHANGE:
 		/* Forbid underlying device to change its type. */
-		return NOTIFY_BAD;
+		ret = NOTIFY_BAD;
+		break;
 
 	case NETDEV_NOTIFY_PEERS:
 	case NETDEV_BONDING_FAILOVER:
@@ -810,7 +892,12 @@ static int ipvlan_device_event(struct notifier_block *unused,
 		list_for_each_entry(ipvlan, &port->ipvlans, pnode)
 			call_netdevice_notifiers(event, ipvlan->dev);
 	}
-	return NOTIFY_DONE;
+
+	mutex_unlock(&port->pnodes_lock);
+
+	ipvlan_port_put(port);
+
+	return ret;
 }
 
 /* the caller must held the addrs lock */
diff --git a/drivers/net/ipvlan/ipvtap.c b/drivers/net/ipvlan/ipvtap.c
index 2d6bbddd1edd5..66c949d942610 100644
--- a/drivers/net/ipvlan/ipvtap.c
+++ b/drivers/net/ipvlan/ipvtap.c
@@ -109,14 +109,26 @@ static int ipvtap_newlink(struct net_device *dev,
 	return err;
 }
 
+static void __ipvtap_dellink(struct net *net, struct net_device *dev,
+			     struct list_head *head)
+{
+	struct ipvtap_dev *vlantap = netdev_priv(dev);
+
+	netdev_rx_handler_unregister(dev);
+	tap_del_queues(&vlantap->tap);
+	__ipvlan_link_delete(net, dev, head);
+}
+
 static void ipvtap_dellink(struct net_device *dev,
 			   struct list_head *head)
 {
-	struct ipvtap_dev *vlan = netdev_priv(dev);
+	struct ipvtap_dev *vlantap = netdev_priv(dev);
+	struct ipvl_port *port = vlantap->vlan.port;
 
-	netdev_rx_handler_unregister(dev);
-	tap_del_queues(&vlan->tap);
-	ipvlan_link_delete(dev, head);
+	mutex_lock(&port->pnodes_lock);
+	if (!vlantap->vlan.dying)
+		__ipvtap_dellink(dev_net(dev), dev, head);
+	mutex_unlock(&port->pnodes_lock);
 }
 
 static void ipvtap_setup(struct net_device *dev)
@@ -198,6 +210,8 @@ static int __init ipvtap_init(void)
 {
 	int err;
 
+	__ipvtap_dellink_ptr = __ipvtap_dellink;
+
 	err = tap_create_cdev(&ipvtap_cdev, &ipvtap_major, "ipvtap",
 			      THIS_MODULE);
 	if (err)
@@ -224,6 +238,8 @@ static int __init ipvtap_init(void)
 out2:
 	tap_destroy_cdev(ipvtap_major, &ipvtap_cdev);
 out1:
+	__ipvtap_dellink_ptr = NULL;
+
 	return err;
 }
 module_init(ipvtap_init);
@@ -234,6 +250,7 @@ static void __exit ipvtap_exit(void)
 	unregister_netdevice_notifier(&ipvtap_notifier_block);
 	class_unregister(&ipvtap_class);
 	tap_destroy_cdev(ipvtap_major, &ipvtap_cdev);
+	__ipvtap_dellink_ptr = NULL;
 }
 module_exit(ipvtap_exit);
 MODULE_ALIAS_RTNL_LINK("ipvtap");
diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c
index dd89282f01791..6f9f3aceffaad 100644
--- a/drivers/net/macsec.c
+++ b/drivers/net/macsec.c
@@ -2636,7 +2636,7 @@ static int macsec_update_offload(struct net_device *dev,
 		vlan_drop_rx_ctag_filter_info(dev);
 		vlan_drop_rx_stag_filter_info(dev);
 	}
-	macsec->offload = offload;
+	WRITE_ONCE(macsec->offload, offload);
 	/* Add VLAN filters when enabling offload. */
 	if (prev_offload == MACSEC_OFFLOAD_OFF) {
 		ret = vlan_get_rx_ctag_filter_info(dev);
@@ -2666,7 +2666,7 @@ static int macsec_update_offload(struct net_device *dev,
 	return 0;
 
 rollback_offload:
-	macsec->offload = prev_offload;
+	WRITE_ONCE(macsec->offload, prev_offload);
 	macsec_offload(ops->mdo_del_secy, &ctx);
 
 	return ret;
@@ -3615,19 +3615,22 @@ static int macsec_dev_open(struct net_device *dev)
 		ops = macsec_get_ops(netdev_priv(dev), &ctx);
 		if (!ops) {
 			err = -EOPNOTSUPP;
-			goto clear_allmulti;
+			goto clear_promisc;
 		}
 
 		ctx.secy = &macsec->secy;
 		err = macsec_offload(ops->mdo_dev_open, &ctx);
 		if (err)
-			goto clear_allmulti;
+			goto clear_promisc;
 	}
 
 	if (netif_carrier_ok(real_dev))
 		netif_carrier_on(dev);
 
 	return 0;
+clear_promisc:
+	if (dev->flags & IFF_PROMISC)
+		dev_set_promiscuity(real_dev, -1);
 clear_allmulti:
 	if (dev->flags & IFF_ALLMULTI)
 		dev_set_allmulti(real_dev, -1);
@@ -3875,52 +3878,53 @@ static int macsec_changelink_common(struct net_device *dev,
 
 	if (data[IFLA_MACSEC_ENCODING_SA]) {
 		struct macsec_tx_sa *tx_sa;
+		u8 encoding_sa = nla_get_u8(data[IFLA_MACSEC_ENCODING_SA]);
 
-		tx_sc->encoding_sa = nla_get_u8(data[IFLA_MACSEC_ENCODING_SA]);
-		tx_sa = rtnl_dereference(tx_sc->sa[tx_sc->encoding_sa]);
+		WRITE_ONCE(tx_sc->encoding_sa, encoding_sa);
+		tx_sa = rtnl_dereference(tx_sc->sa[encoding_sa]);
 
 		secy->operational = tx_sa && tx_sa->active;
 	}
 
 	if (data[IFLA_MACSEC_ENCRYPT])
-		tx_sc->encrypt = !!nla_get_u8(data[IFLA_MACSEC_ENCRYPT]);
+		WRITE_ONCE(tx_sc->encrypt, !!nla_get_u8(data[IFLA_MACSEC_ENCRYPT]));
 
 	if (data[IFLA_MACSEC_PROTECT])
-		secy->protect_frames = !!nla_get_u8(data[IFLA_MACSEC_PROTECT]);
+		WRITE_ONCE(secy->protect_frames, !!nla_get_u8(data[IFLA_MACSEC_PROTECT]));
 
 	if (data[IFLA_MACSEC_INC_SCI])
-		tx_sc->send_sci = !!nla_get_u8(data[IFLA_MACSEC_INC_SCI]);
+		WRITE_ONCE(tx_sc->send_sci, !!nla_get_u8(data[IFLA_MACSEC_INC_SCI]));
 
 	if (data[IFLA_MACSEC_ES])
-		tx_sc->end_station = !!nla_get_u8(data[IFLA_MACSEC_ES]);
+		WRITE_ONCE(tx_sc->end_station, !!nla_get_u8(data[IFLA_MACSEC_ES]));
 
 	if (data[IFLA_MACSEC_SCB])
-		tx_sc->scb = !!nla_get_u8(data[IFLA_MACSEC_SCB]);
+		WRITE_ONCE(tx_sc->scb, !!nla_get_u8(data[IFLA_MACSEC_SCB]));
 
 	if (data[IFLA_MACSEC_REPLAY_PROTECT])
-		secy->replay_protect = !!nla_get_u8(data[IFLA_MACSEC_REPLAY_PROTECT]);
+		WRITE_ONCE(secy->replay_protect, !!nla_get_u8(data[IFLA_MACSEC_REPLAY_PROTECT]));
 
 	if (data[IFLA_MACSEC_VALIDATION])
-		secy->validate_frames = nla_get_u8(data[IFLA_MACSEC_VALIDATION]);
+		WRITE_ONCE(secy->validate_frames, nla_get_u8(data[IFLA_MACSEC_VALIDATION]));
 
 	if (data[IFLA_MACSEC_CIPHER_SUITE]) {
 		switch (nla_get_u64(data[IFLA_MACSEC_CIPHER_SUITE])) {
 		case MACSEC_CIPHER_ID_GCM_AES_128:
 		case MACSEC_DEFAULT_CIPHER_ID:
-			secy->key_len = MACSEC_GCM_AES_128_SAK_LEN;
-			secy->xpn = false;
+			WRITE_ONCE(secy->key_len, MACSEC_GCM_AES_128_SAK_LEN);
+			WRITE_ONCE(secy->xpn, false);
 			break;
 		case MACSEC_CIPHER_ID_GCM_AES_256:
-			secy->key_len = MACSEC_GCM_AES_256_SAK_LEN;
-			secy->xpn = false;
+			WRITE_ONCE(secy->key_len, MACSEC_GCM_AES_256_SAK_LEN);
+			WRITE_ONCE(secy->xpn, false);
 			break;
 		case MACSEC_CIPHER_ID_GCM_AES_XPN_128:
-			secy->key_len = MACSEC_GCM_AES_128_SAK_LEN;
-			secy->xpn = true;
+			WRITE_ONCE(secy->key_len, MACSEC_GCM_AES_128_SAK_LEN);
+			WRITE_ONCE(secy->xpn, true);
 			break;
 		case MACSEC_CIPHER_ID_GCM_AES_XPN_256:
-			secy->key_len = MACSEC_GCM_AES_256_SAK_LEN;
-			secy->xpn = true;
+			WRITE_ONCE(secy->key_len, MACSEC_GCM_AES_256_SAK_LEN);
+			WRITE_ONCE(secy->xpn, true);
 			break;
 		default:
 			return -EINVAL;
@@ -3928,13 +3932,14 @@ static int macsec_changelink_common(struct net_device *dev,
 	}
 
 	if (data[IFLA_MACSEC_WINDOW]) {
-		secy->replay_window = nla_get_u32(data[IFLA_MACSEC_WINDOW]);
+		u32 replay_window = nla_get_u32(data[IFLA_MACSEC_WINDOW]);
 
 		/* IEEE 802.1AEbw-2013 10.7.8 - maximum replay window
 		 * for XPN cipher suites */
 		if (secy->xpn &&
-		    secy->replay_window > MACSEC_XPN_MAX_REPLAY_WINDOW)
+		    replay_window > MACSEC_XPN_MAX_REPLAY_WINDOW)
 			return -EINVAL;
+		WRITE_ONCE(secy->replay_window, replay_window);
 	}
 
 	return 0;
@@ -4382,21 +4387,21 @@ static size_t macsec_get_size(const struct net_device *dev)
 static int macsec_fill_info(struct sk_buff *skb,
 			    const struct net_device *dev)
 {
-	struct macsec_tx_sc *tx_sc;
-	struct macsec_dev *macsec;
-	struct macsec_secy *secy;
+	const struct macsec_tx_sc *tx_sc;
+	const struct macsec_dev *macsec;
+	const struct macsec_secy *secy;
 	u64 csid;
 
 	macsec = macsec_priv(dev);
 	secy = &macsec->secy;
 	tx_sc = &secy->tx_sc;
 
-	switch (secy->key_len) {
+	switch (READ_ONCE(secy->key_len)) {
 	case MACSEC_GCM_AES_128_SAK_LEN:
-		csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_128 : MACSEC_DEFAULT_CIPHER_ID;
+		csid = READ_ONCE(secy->xpn) ? MACSEC_CIPHER_ID_GCM_AES_XPN_128 : MACSEC_DEFAULT_CIPHER_ID;
 		break;
 	case MACSEC_GCM_AES_256_SAK_LEN:
-		csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_256 : MACSEC_CIPHER_ID_GCM_AES_256;
+		csid = READ_ONCE(secy->xpn) ? MACSEC_CIPHER_ID_GCM_AES_XPN_256 : MACSEC_CIPHER_ID_GCM_AES_256;
 		break;
 	default:
 		goto nla_put_failure;
@@ -4407,20 +4412,20 @@ static int macsec_fill_info(struct sk_buff *skb,
 	    nla_put_u8(skb, IFLA_MACSEC_ICV_LEN, secy->icv_len) ||
 	    nla_put_u64_64bit(skb, IFLA_MACSEC_CIPHER_SUITE,
 			      csid, IFLA_MACSEC_PAD) ||
-	    nla_put_u8(skb, IFLA_MACSEC_ENCODING_SA, tx_sc->encoding_sa) ||
-	    nla_put_u8(skb, IFLA_MACSEC_ENCRYPT, tx_sc->encrypt) ||
-	    nla_put_u8(skb, IFLA_MACSEC_PROTECT, secy->protect_frames) ||
-	    nla_put_u8(skb, IFLA_MACSEC_INC_SCI, tx_sc->send_sci) ||
-	    nla_put_u8(skb, IFLA_MACSEC_ES, tx_sc->end_station) ||
-	    nla_put_u8(skb, IFLA_MACSEC_SCB, tx_sc->scb) ||
-	    nla_put_u8(skb, IFLA_MACSEC_REPLAY_PROTECT, secy->replay_protect) ||
-	    nla_put_u8(skb, IFLA_MACSEC_VALIDATION, secy->validate_frames) ||
-	    nla_put_u8(skb, IFLA_MACSEC_OFFLOAD, macsec->offload) ||
+	    nla_put_u8(skb, IFLA_MACSEC_ENCODING_SA, READ_ONCE(tx_sc->encoding_sa)) ||
+	    nla_put_u8(skb, IFLA_MACSEC_ENCRYPT, READ_ONCE(tx_sc->encrypt)) ||
+	    nla_put_u8(skb, IFLA_MACSEC_PROTECT, READ_ONCE(secy->protect_frames)) ||
+	    nla_put_u8(skb, IFLA_MACSEC_INC_SCI, READ_ONCE(tx_sc->send_sci)) ||
+	    nla_put_u8(skb, IFLA_MACSEC_ES, READ_ONCE(tx_sc->end_station)) ||
+	    nla_put_u8(skb, IFLA_MACSEC_SCB, READ_ONCE(tx_sc->scb)) ||
+	    nla_put_u8(skb, IFLA_MACSEC_REPLAY_PROTECT, READ_ONCE(secy->replay_protect)) ||
+	    nla_put_u8(skb, IFLA_MACSEC_VALIDATION, READ_ONCE(secy->validate_frames)) ||
+	    nla_put_u8(skb, IFLA_MACSEC_OFFLOAD, READ_ONCE(macsec->offload)) ||
 	    0)
 		goto nla_put_failure;
 
-	if (secy->replay_protect) {
-		if (nla_put_u32(skb, IFLA_MACSEC_WINDOW, secy->replay_window))
+	if (READ_ONCE(secy->replay_protect)) {
+		if (nla_put_u32(skb, IFLA_MACSEC_WINDOW, READ_ONCE(secy->replay_window)))
 			goto nla_put_failure;
 	}
 
diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
index c40fa331836bb..9a4bc99dbf53b 100644
--- a/drivers/net/macvlan.c
+++ b/drivers/net/macvlan.c
@@ -171,7 +171,7 @@ static int macvlan_hash_add_source(struct macvlan_dev *vlan,
 	RCU_INIT_POINTER(entry->vlan, vlan);
 	h = &port->vlan_source_hash[macvlan_eth_hash(addr)];
 	hlist_add_head_rcu(&entry->hlist, h);
-	vlan->macaddr_count++;
+	WRITE_ONCE(vlan->macaddr_count, vlan->macaddr_count + 1);
 
 	return 0;
 }
@@ -277,7 +277,7 @@ static void macvlan_broadcast(struct sk_buff *skb,
 		return;
 
 	hash_for_each_rcu(port->vlan_hash, i, vlan, hlist) {
-		if (vlan->dev == src || !(vlan->mode & mode))
+		if (vlan->dev == src || !(READ_ONCE(vlan->mode) & mode))
 			continue;
 
 		hash = mc_hash(vlan, eth->h_dest);
@@ -306,7 +306,7 @@ static void macvlan_multicast_rx(const struct macvlan_port *port,
 				  MACVLAN_MODE_VEPA    |
 				  MACVLAN_MODE_PASSTHRU|
 				  MACVLAN_MODE_BRIDGE);
-	else if (src->mode == MACVLAN_MODE_VEPA)
+	else if (READ_ONCE(src->mode) == MACVLAN_MODE_VEPA)
 		/* flood to everyone except source */
 		macvlan_broadcast(skb, port, src->dev,
 				  MACVLAN_MODE_VEPA |
@@ -402,7 +402,7 @@ static void macvlan_flush_sources(struct macvlan_port *port,
 		if (rcu_access_pointer(entry->vlan) == vlan)
 			macvlan_hash_del_source(entry);
 
-	vlan->macaddr_count = 0;
+	WRITE_ONCE(vlan->macaddr_count, 0);
 }
 
 static void macvlan_forward_source_one(struct sk_buff *skb,
@@ -447,7 +447,7 @@ static bool macvlan_forward_source(struct sk_buff *skb,
 			if (!vlan)
 				continue;
 
-			if (vlan->flags & MACVLAN_FLAG_NODST)
+			if (READ_ONCE(vlan->flags) & MACVLAN_FLAG_NODST)
 				consume = true;
 			macvlan_forward_source_one(skb, vlan);
 		}
@@ -487,14 +487,18 @@ static rx_handler_result_t macvlan_handle_frame(struct sk_buff **pskb)
 			return RX_HANDLER_CONSUMED;
 		}
 		src = macvlan_hash_lookup(port, eth->h_source);
-		if (src && src->mode != MACVLAN_MODE_VEPA &&
-		    src->mode != MACVLAN_MODE_BRIDGE) {
-			/* forward to original port. */
-			vlan = src;
-			ret = macvlan_broadcast_one(skb, vlan, eth, 0) ?:
-			      __netif_rx(skb);
-			handle_res = RX_HANDLER_CONSUMED;
-			goto out;
+		if (src) {
+			enum macvlan_mode mode = READ_ONCE(src->mode);
+
+			if (mode != MACVLAN_MODE_VEPA &&
+			    mode != MACVLAN_MODE_BRIDGE) {
+				/* forward to original port. */
+				vlan = src;
+				ret = macvlan_broadcast_one(skb, vlan, eth, 0) ?:
+				      __netif_rx(skb);
+				handle_res = RX_HANDLER_CONSUMED;
+				goto out;
+			}
 		}
 
 		hash = mc_hash(NULL, eth->h_dest);
@@ -515,7 +519,7 @@ static rx_handler_result_t macvlan_handle_frame(struct sk_buff **pskb)
 					      struct macvlan_dev, list);
 	else
 		vlan = macvlan_hash_lookup(port, eth->h_dest);
-	if (!vlan || vlan->mode == MACVLAN_MODE_SOURCE)
+	if (!vlan || READ_ONCE(vlan->mode) == MACVLAN_MODE_SOURCE)
 		return RX_HANDLER_PASS;
 
 	dev = vlan->dev;
@@ -548,7 +552,7 @@ static int macvlan_queue_xmit(struct sk_buff *skb, struct net_device *dev)
 	const struct macvlan_port *port = vlan->port;
 	const struct macvlan_dev *dest;
 
-	if (vlan->mode == MACVLAN_MODE_BRIDGE) {
+	if (READ_ONCE(vlan->mode) == MACVLAN_MODE_BRIDGE) {
 		const struct ethhdr *eth = skb_eth_hdr(skb);
 
 		/* send to other bridge ports directly */
@@ -559,7 +563,7 @@ static int macvlan_queue_xmit(struct sk_buff *skb, struct net_device *dev)
 		}
 
 		dest = macvlan_hash_lookup(port, eth->h_dest);
-		if (dest && dest->mode == MACVLAN_MODE_BRIDGE) {
+		if (dest && READ_ONCE(dest->mode) == MACVLAN_MODE_BRIDGE) {
 			/* send to lowerdev first for its network taps */
 			dev_forward_skb(vlan->lowerdev, skb);
 
@@ -777,7 +781,7 @@ static int macvlan_set_mac_address(struct net_device *dev, void *p)
 	if (ether_addr_equal(dev->dev_addr, addr->__data))
 		return 0;
 
-	if (vlan->mode == MACVLAN_MODE_PASSTHRU) {
+	if (READ_ONCE(vlan->mode) == MACVLAN_MODE_PASSTHRU) {
 		macvlan_set_addr_change(vlan->port);
 		return dev_set_mac_address(vlan->lowerdev, addr, NULL);
 	}
@@ -870,7 +874,7 @@ static void update_port_bc_cutoff(struct macvlan_dev *vlan, int cutoff)
 	if (vlan->port->bc_cutoff == cutoff)
 		return;
 
-	vlan->port->bc_cutoff = cutoff;
+	WRITE_ONCE(vlan->port->bc_cutoff, cutoff);
 	macvlan_recompute_bc_filter(vlan);
 }
 
@@ -1423,7 +1427,7 @@ static int macvlan_changelink_sources(struct macvlan_dev *vlan, u32 mode,
 		entry = macvlan_hash_lookup_source(vlan, addr);
 		if (entry) {
 			macvlan_hash_del_source(entry);
-			vlan->macaddr_count--;
+			WRITE_ONCE(vlan->macaddr_count, vlan->macaddr_count - 1);
 		}
 	} else if (mode == MACVLAN_MACADDR_FLUSH) {
 		macvlan_flush_sources(vlan->port, vlan);
@@ -1645,11 +1649,12 @@ static int macvlan_changelink(struct net_device *dev,
 			if (err < 0)
 				return err;
 		}
-		vlan->flags = flags;
+		WRITE_ONCE(vlan->flags, flags);
 	}
 
 	if (data && data[IFLA_MACVLAN_BC_QUEUE_LEN]) {
-		vlan->bc_queue_len_req = nla_get_u32(data[IFLA_MACVLAN_BC_QUEUE_LEN]);
+		WRITE_ONCE(vlan->bc_queue_len_req,
+			   nla_get_u32(data[IFLA_MACVLAN_BC_QUEUE_LEN]));
 		update_port_bc_queue_len(vlan->port);
 	}
 
@@ -1658,7 +1663,7 @@ static int macvlan_changelink(struct net_device *dev,
 			vlan, nla_get_s32(data[IFLA_MACVLAN_BC_CUTOFF]));
 
 	if (set_mode)
-		vlan->mode = mode;
+		WRITE_ONCE(vlan->mode, mode);
 	if (data && data[IFLA_MACVLAN_MACADDR_MODE]) {
 		if (vlan->mode != MACVLAN_MODE_SOURCE)
 			return -EINVAL;
@@ -1672,10 +1677,12 @@ static int macvlan_changelink(struct net_device *dev,
 
 static size_t macvlan_get_size_mac(const struct macvlan_dev *vlan)
 {
-	if (vlan->macaddr_count == 0)
+	unsigned int macaddr_count = READ_ONCE(vlan->macaddr_count);
+
+	if (!macaddr_count)
 		return 0;
 	return nla_total_size(0) /* IFLA_MACVLAN_MACADDR_DATA */
-		+ vlan->macaddr_count * nla_total_size(sizeof(u8) * ETH_ALEN);
+		+ macaddr_count * nla_total_size(sizeof(u8) * ETH_ALEN);
 }
 
 static size_t macvlan_get_size(const struct net_device *dev)
@@ -1698,53 +1705,75 @@ static int macvlan_fill_info_macaddr(struct sk_buff *skb,
 				     const int i)
 {
 	struct hlist_head *h = &vlan->port->vlan_source_hash[i];
-	struct macvlan_source_entry *entry;
+	const struct macvlan_source_entry *entry;
+	int cnt = 0;
 
-	hlist_for_each_entry_rcu(entry, h, hlist, lockdep_rtnl_is_held()) {
+	hlist_for_each_entry_rcu(entry, h, hlist) {
 		if (rcu_access_pointer(entry->vlan) != vlan)
 			continue;
 		if (nla_put(skb, IFLA_MACVLAN_MACADDR, ETH_ALEN, entry->addr))
-			return 1;
+			return -EMSGSIZE;
+		cnt++;
 	}
-	return 0;
+	return cnt;
 }
 
 static int macvlan_fill_info(struct sk_buff *skb,
 				const struct net_device *dev)
 {
-	struct macvlan_dev *vlan = netdev_priv(dev);
+	const struct macvlan_dev *vlan = netdev_priv(dev);
 	struct macvlan_port *port = vlan->port;
-	int i;
-	struct nlattr *nest;
+	unsigned int macaddr_count = 0;
+	struct nlattr *nest, *attr;
+	int bc_cutoff, cnt, i;
 
-	if (nla_put_u32(skb, IFLA_MACVLAN_MODE, vlan->mode))
+	rcu_read_lock();
+	if (nla_put_u32(skb, IFLA_MACVLAN_MODE, READ_ONCE(vlan->mode)))
 		goto nla_put_failure;
-	if (nla_put_u16(skb, IFLA_MACVLAN_FLAGS, vlan->flags))
+
+	if (nla_put_u16(skb, IFLA_MACVLAN_FLAGS, READ_ONCE(vlan->flags)))
 		goto nla_put_failure;
-	if (nla_put_u32(skb, IFLA_MACVLAN_MACADDR_COUNT, vlan->macaddr_count))
+
+	attr = nla_reserve(skb, IFLA_MACVLAN_MACADDR_COUNT, sizeof(u32));
+	if (!attr)
 		goto nla_put_failure;
-	if (vlan->macaddr_count > 0) {
+
+	if (READ_ONCE(vlan->macaddr_count) > 0) {
 		nest = nla_nest_start_noflag(skb, IFLA_MACVLAN_MACADDR_DATA);
 		if (nest == NULL)
 			goto nla_put_failure;
 
 		for (i = 0; i < MACVLAN_HASH_SIZE; i++) {
-			if (macvlan_fill_info_macaddr(skb, vlan, i))
+			cnt = macvlan_fill_info_macaddr(skb, vlan, i);
+			if (cnt < 0)
 				goto nla_put_failure;
+			macaddr_count += cnt;
 		}
-		nla_nest_end(skb, nest);
+		if (!macaddr_count)
+			nla_nest_cancel(skb, nest);
+		else if (nla_nest_end_safe(skb, nest) < 0)
+			goto nla_put_failure;
 	}
-	if (nla_put_u32(skb, IFLA_MACVLAN_BC_QUEUE_LEN, vlan->bc_queue_len_req))
+	*(u32 *)nla_data(attr) = macaddr_count;
+
+	if (nla_put_u32(skb, IFLA_MACVLAN_BC_QUEUE_LEN,
+			READ_ONCE(vlan->bc_queue_len_req)))
 		goto nla_put_failure;
+
 	if (nla_put_u32(skb, IFLA_MACVLAN_BC_QUEUE_LEN_USED,
 			READ_ONCE(port->bc_queue_len_used)))
 		goto nla_put_failure;
-	if (port->bc_cutoff != 1 &&
-	    nla_put_s32(skb, IFLA_MACVLAN_BC_CUTOFF, port->bc_cutoff))
+
+	bc_cutoff = READ_ONCE(port->bc_cutoff);
+	if (bc_cutoff != 1 &&
+	    nla_put_s32(skb, IFLA_MACVLAN_BC_CUTOFF, bc_cutoff))
 		goto nla_put_failure;
+
+	rcu_read_unlock();
 	return 0;
 
 nla_put_failure:
+	rcu_read_unlock();
 	return -EMSGSIZE;
 }
 
diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c
index 862001d09aa84..49b2243a20b2d 100644
--- a/drivers/net/netconsole.c
+++ b/drivers/net/netconsole.c
@@ -61,6 +61,19 @@ MODULE_IMPORT_NS("NETDEV_INTERNAL");
 #define MAX_USERDATA_ITEMS		256
 #define MAX_PRINT_CHUNK			1000
 
+/*
+ * Sizing for the per-target fallback skb pool consulted by find_skb()
+ * when its GFP_ATOMIC allocation fails so messages still get out under
+ * memory pressure.
+ */
+#define MAX_UDP_CHUNK			1460
+#define MAX_SKBS			32
+#define MAX_SKB_SIZE							\
+	(sizeof(struct ethhdr) +					\
+	 sizeof(struct iphdr) +						\
+	 sizeof(struct udphdr) +					\
+	 MAX_UDP_CHUNK)
+
 static char config[MAX_PARAM_LENGTH];
 module_param_string(netconsole, config, MAX_PARAM_LENGTH, 0);
 MODULE_PARM_DESC(netconsole, " netconsole=[src-port]@[src-ip]/[dev],[tgt-port]@<tgt-ip>/[tgt-macaddr]");
@@ -162,14 +175,19 @@ enum target_state {
  * @np:		The netpoll structure for this target.
  *		Contains the other userspace visible parameters:
  *		dev_name	(read-write)
- *		local_port	(read-write)
- *		remote_port	(read-write)
  *		local_ip	(read-write)
  *		remote_ip	(read-write)
  *		local_mac	(read-only)
- *		remote_mac	(read-write)
+ * @local_port:	Source UDP port of the target (read-write).
+ * @remote_port: Destination UDP port of the target (read-write).
+ * @remote_mac:	Destination ethernet address of the target (read-write).
  * @buf:	The buffer used to send the full msg to the network stack
  * @resume_wq:	Workqueue to resume deactivated target
+ * @skb_pool:	Per-target fallback skb pool consulted by find_skb() when
+ *		its GFP_ATOMIC allocation fails. Lifetime brackets a
+ *		successful netpoll_setup() / netpoll_cleanup() pair on @np.
+ * @refill_wq:	Work item that asynchronously tops @skb_pool back up to
+ *		MAX_SKBS after find_skb() drains an entry.
  */
 struct netconsole_target {
 	struct list_head	list;
@@ -190,11 +208,15 @@ struct netconsole_target {
 	bool			extended;
 	bool			release;
 	struct netpoll		np;
+	u16			local_port, remote_port;
+	u8			remote_mac[ETH_ALEN];
 	/* protected by target_list_lock; +1 gives scnprintf() room for its
 	 * NUL terminator so a full MAX_PRINT_CHUNK payload is not truncated
 	 */
 	char			buf[MAX_PRINT_CHUNK + 1];
 	struct work_struct	resume_wq;
+	struct sk_buff_head	skb_pool;
+	struct work_struct	refill_wq;
 };
 
 #ifdef	CONFIG_NETCONSOLE_DYNAMIC
@@ -292,11 +314,56 @@ static void netcons_release_dev(struct netconsole_target *nt)
 		memset(&nt->np.dev_name, 0, IFNAMSIZ);
 }
 
+static void refill_skbs(struct netconsole_target *nt)
+{
+	struct sk_buff_head *skb_pool = &nt->skb_pool;
+	struct sk_buff *skb;
+
+	while (READ_ONCE(skb_pool->qlen) < MAX_SKBS) {
+		skb = alloc_skb(MAX_SKB_SIZE, GFP_ATOMIC | __GFP_NOWARN);
+		if (!skb)
+			break;
+
+		skb_queue_tail(skb_pool, skb);
+	}
+}
+
+static void refill_skbs_work_handler(struct work_struct *work)
+{
+	struct netconsole_target *nt =
+		container_of(work, struct netconsole_target, refill_wq);
+
+	refill_skbs(nt);
+}
+
+/* Seed the per-target skb pool that find_skb() falls back to. The queue
+ * head and refill work are set up once in alloc_and_init(); this only
+ * (re)fills the pool. Pair with netconsole_skb_pool_flush().
+ */
+static void netconsole_skb_pool_init(struct netconsole_target *nt)
+{
+	refill_skbs(nt);
+}
+
+static void netconsole_skb_pool_flush(struct netconsole_target *nt)
+{
+	cancel_work_sync(&nt->refill_wq);
+	skb_queue_purge_reason(&nt->skb_pool, SKB_CONSUMED);
+}
+
 /* Attempts to resume logging to a deactivated target. */
 static void resume_target(struct netconsole_target *nt)
 {
+	/* Initialise the skb pool before netpoll_setup() makes nt->np.dev
+	 * visible to target_list walkers (e.g. netconsole_netdev_event),
+	 * which otherwise may move the target to the cleanup list and
+	 * call netconsole_skb_pool_flush() on uninitialised state.
+	 */
+	netconsole_skb_pool_init(nt);
+
 	if (netpoll_setup(&nt->np)) {
 		/* netpoll fails setup once, do not try again. */
+		netconsole_skb_pool_flush(nt);
 		nt->state = STATE_DISABLED;
 		return;
 	}
@@ -358,6 +425,7 @@ static void process_resume_target(struct work_struct *work)
 	rtnl_lock();
 	if (nt->state == STATE_ENABLED && nt->np.dev &&
 	    nt->np.dev->reg_state != NETREG_REGISTERED) {
+		netconsole_skb_pool_flush(nt);
 		netcons_release_dev(nt);
 		nt->state = STATE_DISABLED;
 	}
@@ -393,11 +461,14 @@ static struct netconsole_target *alloc_and_init(void)
 
 	nt->np.name = "netconsole";
 	strscpy(nt->np.dev_name, "eth0", IFNAMSIZ);
-	nt->np.local_port = 6665;
-	nt->np.remote_port = 6666;
-	eth_broadcast_addr(nt->np.remote_mac);
+	nt->local_port = 6665;
+	nt->remote_port = 6666;
+	eth_broadcast_addr(nt->remote_mac);
 	nt->state = STATE_DISABLED;
 	INIT_WORK(&nt->resume_wq, process_resume_target);
+	/* Set up the skb pool primitives once; enabling only refills it. */
+	skb_queue_head_init(&nt->skb_pool);
+	INIT_WORK(&nt->refill_wq, refill_skbs_work_handler);
 
 	return nt;
 }
@@ -417,6 +488,7 @@ static void netconsole_process_cleanups_core(void)
 	list_for_each_entry_safe(nt, tmp, &target_cleanup_list, list) {
 		/* all entries in the cleanup_list needs to be disabled */
 		WARN_ON_ONCE(nt->state == STATE_ENABLED);
+		netconsole_skb_pool_flush(nt);
 		netcons_release_dev(nt);
 		/* moved the cleaned target to target_list. Need to hold both
 		 * locks
@@ -429,21 +501,23 @@ static void netconsole_process_cleanups_core(void)
 	mutex_unlock(&target_cleanup_list_lock);
 }
 
-static void netconsole_print_banner(struct netpoll *np)
+static void netconsole_print_banner(struct netconsole_target *nt)
 {
-	np_info(np, "local port %d\n", np->local_port);
+	struct netpoll *np = &nt->np;
+
+	np_info(np, "local port %d\n", nt->local_port);
 	if (np->ipv6)
 		np_info(np, "local IPv6 address %pI6c\n", &np->local_ip.in6);
 	else
 		np_info(np, "local IPv4 address %pI4\n", &np->local_ip.ip);
 	np_info(np, "interface name '%s'\n", np->dev_name);
 	np_info(np, "local ethernet address '%pM'\n", np->dev_mac);
-	np_info(np, "remote port %d\n", np->remote_port);
+	np_info(np, "remote port %d\n", nt->remote_port);
 	if (np->ipv6)
 		np_info(np, "remote IPv6 address %pI6c\n", &np->remote_ip.in6);
 	else
 		np_info(np, "remote IPv4 address %pI4\n", &np->remote_ip.ip);
-	np_info(np, "remote ethernet address %pM\n", np->remote_mac);
+	np_info(np, "remote ethernet address %pM\n", nt->remote_mac);
 }
 
 /* Parse the string and populate the `inet_addr` union. Return 0 if IPv4 is
@@ -561,12 +635,12 @@ static ssize_t dev_name_show(struct config_item *item, char *buf)
 
 static ssize_t local_port_show(struct config_item *item, char *buf)
 {
-	return sysfs_emit(buf, "%d\n", to_target(item)->np.local_port);
+	return sysfs_emit(buf, "%d\n", to_target(item)->local_port);
 }
 
 static ssize_t remote_port_show(struct config_item *item, char *buf)
 {
-	return sysfs_emit(buf, "%d\n", to_target(item)->np.remote_port);
+	return sysfs_emit(buf, "%d\n", to_target(item)->remote_port);
 }
 
 static ssize_t local_ip_show(struct config_item *item, char *buf)
@@ -599,7 +673,7 @@ static ssize_t local_mac_show(struct config_item *item, char *buf)
 
 static ssize_t remote_mac_show(struct config_item *item, char *buf)
 {
-	return sysfs_emit(buf, "%pM\n", to_target(item)->np.remote_mac);
+	return sysfs_emit(buf, "%pM\n", to_target(item)->remote_mac);
 }
 
 static ssize_t transmit_errors_show(struct config_item *item, char *buf)
@@ -756,11 +830,21 @@ static ssize_t enabled_store(struct config_item *item,
 		 * Skip netconsole_parser_cmdline() -- all the attributes are
 		 * already configured via configfs. Just print them out.
 		 */
-		netconsole_print_banner(&nt->np);
+		netconsole_print_banner(nt);
+
+		/* Initialise the skb pool before netpoll_setup() so the pool
+		 * is valid as soon as nt->np.dev becomes visible to
+		 * target_list walkers (netconsole_netdev_event), which would
+		 * otherwise call netconsole_skb_pool_flush() on uninitialised
+		 * state.
+		 */
+		netconsole_skb_pool_init(nt);
 
 		ret = netpoll_setup(&nt->np);
-		if (ret)
+		if (ret) {
+			netconsole_skb_pool_flush(nt);
 			goto out_unlock;
+		}
 
 		nt->state = STATE_ENABLED;
 		pr_info("network logging started\n");
@@ -885,7 +969,7 @@ static ssize_t local_port_store(struct config_item *item, const char *buf,
 		goto out_unlock;
 	}
 
-	ret = kstrtou16(buf, 10, &nt->np.local_port);
+	ret = kstrtou16(buf, 10, &nt->local_port);
 	if (ret < 0)
 		goto out_unlock;
 	ret = count;
@@ -907,7 +991,7 @@ static ssize_t remote_port_store(struct config_item *item,
 		goto out_unlock;
 	}
 
-	ret = kstrtou16(buf, 10, &nt->np.remote_port);
+	ret = kstrtou16(buf, 10, &nt->remote_port);
 	if (ret < 0)
 		goto out_unlock;
 	ret = count;
@@ -994,7 +1078,7 @@ static ssize_t remote_mac_store(struct config_item *item, const char *buf,
 		goto out_unlock;
 	if (buf[MAC_ADDR_STR_LEN] && buf[MAC_ADDR_STR_LEN] != '\n')
 		goto out_unlock;
-	memcpy(nt->np.remote_mac, remote_mac, ETH_ALEN);
+	memcpy(nt->remote_mac, remote_mac, ETH_ALEN);
 
 	ret = count;
 out_unlock:
@@ -1481,15 +1565,15 @@ static void drop_netconsole_target(struct config_group *group,
 
 	mutex_lock(&target_cleanup_list_lock);
 	spin_lock_irqsave(&target_list_lock, flags);
-	/* A STATE_DEACTIVATED target may have been moved to
-	 * target_cleanup_list by netconsole_netdev_event() but not yet
-	 * processed by netconsole_process_cleanups_core(). Unlinking it below
-	 * hides it from the cleanup worker, so this path has to clean it up
-	 * itself. Record that the target still owns a netpoll before the
-	 * state is downgraded.
+	/* A target moved to target_cleanup_list by netconsole_netdev_event()
+	 * but not yet processed still owns a netpoll; unlinking it below hides
+	 * it from the cleanup worker, so this path must tear it down itself.
+	 * This covers NETDEV_UNREGISTER (STATE_DEACTIVATED) and
+	 * NETDEV_RELEASE / NETDEV_JOIN (STATE_DISABLED); key off nt->np.dev,
+	 * which stays set until the netpoll is cleaned up.
 	 */
 	needs_cleanup = nt->state == STATE_ENABLED ||
-			nt->state == STATE_DEACTIVATED;
+			nt->state == STATE_DEACTIVATED || nt->np.dev;
 	/* Disable deactivated target to prevent races between resume attempt
 	 * and target removal.
 	 */
@@ -1514,8 +1598,10 @@ static void drop_netconsole_target(struct config_group *group,
 	 * netpoll_cleanup() is idempotent (it skips when np->dev is NULL), so
 	 * it is safe even if the cleanup worker already tore the netpoll down.
 	 */
-	if (needs_cleanup)
+	if (needs_cleanup) {
+		netconsole_skb_pool_flush(nt);
 		netpoll_cleanup(&nt->np);
+	}
 
 	config_item_put(&nt->group.cg_item);
 }
@@ -1706,7 +1792,7 @@ static struct notifier_block netconsole_netdev_notifier = {
  * pool locks and is therefore not NMI-safe. Skip the refill when called
  * from NMI context; the next non-NMI caller will top the pool back up.
  */
-static struct sk_buff *netcons_skb_pop(struct netpoll *np, int len)
+static struct sk_buff *netcons_skb_pop(struct netconsole_target *nt, int len)
 {
 	struct sk_buff *skb;
 
@@ -1718,28 +1804,30 @@ static struct sk_buff *netcons_skb_pop(struct netpoll *np, int len)
 		if (!in_nmi())
 			net_warn_ratelimited("netconsole: dropping message, requested skb len %d exceeds pool buffer size %zu on %s\n",
 					     len, (size_t)MAX_SKB_SIZE,
-					     np->dev->name);
+					     nt->np.dev->name);
 		return NULL;
 	}
 
-	skb = skb_dequeue(&np->skb_pool);
+	skb = skb_dequeue(&nt->skb_pool);
 	if (!in_nmi())
-		schedule_work(&np->refill_wq);
+		schedule_work(&nt->refill_wq);
 
 	return skb;
 }
 
-static struct sk_buff *find_skb(struct netpoll *np, int len, int reserve)
+static struct sk_buff *find_skb(struct netconsole_target *nt, int len,
+				int reserve)
 {
+	struct netpoll *np = &nt->np;
 	int count = 0;
 	struct sk_buff *skb;
 
 	netpoll_zap_completion_queue();
 repeat:
 
-	skb = alloc_skb(len, GFP_ATOMIC);
+	skb = alloc_skb(len, GFP_ATOMIC | __GFP_NOWARN);
 	if (!skb)
-		skb = netcons_skb_pop(np, len);
+		skb = netcons_skb_pop(nt, len);
 
 	if (!skb) {
 		if (++count < 10) {
@@ -1779,8 +1867,9 @@ static void netpoll_udp_checksum(struct netpoll *np, struct sk_buff *skb,
 		udph->check = CSUM_MANGLED_0;
 }
 
-static void push_udp(struct netpoll *np, struct sk_buff *skb, int len)
+static void push_udp(struct netconsole_target *nt, struct sk_buff *skb, int len)
 {
+	struct netpoll *np = &nt->np;
 	struct udphdr *udph;
 	int udp_len;
 
@@ -1790,21 +1879,22 @@ static void push_udp(struct netpoll *np, struct sk_buff *skb, int len)
 	skb_reset_transport_header(skb);
 
 	udph = udp_hdr(skb);
-	udph->source = htons(np->local_port);
-	udph->dest = htons(np->remote_port);
+	udph->source = htons(nt->local_port);
+	udph->dest = htons(nt->remote_port);
 	udph->len = htons(udp_len);
 
 	netpoll_udp_checksum(np, skb, len);
 }
 
-static void push_eth(struct netpoll *np, struct sk_buff *skb)
+static void push_eth(struct netconsole_target *nt, struct sk_buff *skb)
 {
+	struct netpoll *np = &nt->np;
 	struct ethhdr *eth;
 
 	eth = skb_push(skb, ETH_HLEN);
 	skb_reset_mac_header(skb);
 	ether_addr_copy(eth->h_source, np->dev->dev_addr);
-	ether_addr_copy(eth->h_dest, np->remote_mac);
+	ether_addr_copy(eth->h_dest, nt->remote_mac);
 	if (np->ipv6)
 		eth->h_proto = htons(ETH_P_IPV6);
 	else
@@ -1861,8 +1951,10 @@ static void push_ipv6(struct netpoll *np, struct sk_buff *skb, int len)
 	skb->protocol = htons(ETH_P_IPV6);
 }
 
-static int netpoll_send_udp(struct netpoll *np, const char *msg, int len)
+static int netpoll_send_udp(struct netconsole_target *nt, const char *msg,
+			    int len)
 {
+	struct netpoll *np = &nt->np;
 	int total_len, ip_len, udp_len;
 	struct sk_buff *skb;
 
@@ -1877,7 +1969,7 @@ static int netpoll_send_udp(struct netpoll *np, const char *msg, int len)
 
 	total_len = ip_len + LL_RESERVED_SPACE(np->dev);
 
-	skb = find_skb(np, total_len + np->dev->needed_tailroom,
+	skb = find_skb(nt, total_len + np->dev->needed_tailroom,
 		       total_len - len);
 	if (!skb)
 		return -ENOMEM;
@@ -1885,12 +1977,12 @@ static int netpoll_send_udp(struct netpoll *np, const char *msg, int len)
 	skb_copy_to_linear_data(skb, msg, len);
 	skb_put(skb, len);
 
-	push_udp(np, skb, len);
+	push_udp(nt, skb, len);
 	if (np->ipv6)
 		push_ipv6(np, skb, len);
 	else
 		push_ipv4(np, skb, len);
-	push_eth(np, skb);
+	push_eth(nt, skb);
 	skb->dev = np->dev;
 
 	return (int)netpoll_send_skb(np, skb);
@@ -1908,7 +2000,7 @@ static int netpoll_send_udp(struct netpoll *np, const char *msg, int len)
  */
 static void send_udp(struct netconsole_target *nt, const char *msg, int len)
 {
-	int result = netpoll_send_udp(&nt->np, msg, len);
+	int result = netpoll_send_udp(nt, msg, len);
 
 	if (IS_ENABLED(CONFIG_NETCONSOLE_DYNAMIC)) {
 		if (result == NET_XMIT_DROP) {
@@ -2203,8 +2295,9 @@ __releases(&target_list_lock)
 	spin_unlock_irqrestore(&target_list_lock, flags);
 }
 
-static int netconsole_parser_cmdline(struct netpoll *np, char *opt)
+static int netconsole_parser_cmdline(struct netconsole_target *nt, char *opt)
 {
+	struct netpoll *np = &nt->np;
 	bool ipversion_set = false;
 	char *cur = opt;
 	char *delim;
@@ -2215,7 +2308,7 @@ static int netconsole_parser_cmdline(struct netpoll *np, char *opt)
 		if (!delim)
 			goto parse_failed;
 		*delim = 0;
-		if (kstrtou16(cur, 10, &np->local_port))
+		if (kstrtou16(cur, 10, &nt->local_port))
 			goto parse_failed;
 		cur = delim;
 	}
@@ -2262,7 +2355,7 @@ static int netconsole_parser_cmdline(struct netpoll *np, char *opt)
 		*delim = 0;
 		if (*cur == ' ' || *cur == '\t')
 			np_info(np, "warning: whitespace is not allowed\n");
-		if (kstrtou16(cur, 10, &np->remote_port))
+		if (kstrtou16(cur, 10, &nt->remote_port))
 			goto parse_failed;
 		cur = delim;
 	}
@@ -2284,11 +2377,11 @@ static int netconsole_parser_cmdline(struct netpoll *np, char *opt)
 
 	if (*cur != 0) {
 		/* MAC address */
-		if (!mac_pton(cur, np->remote_mac))
+		if (!mac_pton(cur, nt->remote_mac))
 			goto parse_failed;
 	}
 
-	netconsole_print_banner(np);
+	netconsole_print_banner(nt);
 
 	return 0;
 
@@ -2326,14 +2419,22 @@ static struct netconsole_target *alloc_param_target(char *target_config,
 	}
 
 	/* Parse parameters and setup netpoll */
-	err = netconsole_parser_cmdline(&nt->np, target_config);
+	err = netconsole_parser_cmdline(nt, target_config);
 	if (err)
 		goto fail;
 
+	/* Initialise the skb pool before netpoll_setup() so the pool is
+	 * valid as soon as nt->np.dev becomes visible. The target is not
+	 * yet on target_list, so a netdev event cannot reach it here, but
+	 * mirror the configfs path for symmetry.
+	 */
+	netconsole_skb_pool_init(nt);
+
 	err = netpoll_setup(&nt->np);
 	if (err) {
 		pr_err("Not enabling netconsole for %s%d. Netpoll setup failed\n",
 		       NETCONSOLE_PARAM_TARGET_PREFIX, cmdline_count);
+		netconsole_skb_pool_flush(nt);
 		if (!IS_ENABLED(CONFIG_NETCONSOLE_DYNAMIC))
 			/* only fail if dynamic reconfiguration is set,
 			 * otherwise, keep the target in the list, but disabled.
@@ -2355,6 +2456,8 @@ static struct netconsole_target *alloc_param_target(char *target_config,
 static void free_param_target(struct netconsole_target *nt)
 {
 	cancel_work_sync(&nt->resume_wq);
+	if (nt->state == STATE_ENABLED)
+		netconsole_skb_pool_flush(nt);
 	netpoll_cleanup(&nt->np);
 #ifdef	CONFIG_NETCONSOLE_DYNAMIC
 	kfree(nt->userdata);
diff --git a/drivers/net/phy/mdio_device.c b/drivers/net/phy/mdio_device.c
index 56080d3d2d25e..a18263d5bb029 100644
--- a/drivers/net/phy/mdio_device.c
+++ b/drivers/net/phy/mdio_device.c
@@ -8,7 +8,7 @@
 
 #include <linux/delay.h>
 #include <linux/errno.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/gpio/consumer.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c
index 087ac63f9193d..59dfe35afa54f 100644
--- a/drivers/net/phy/phylink.c
+++ b/drivers/net/phy/phylink.c
@@ -153,8 +153,7 @@ static DECLARE_PHY_INTERFACE_MASK(phylink_sfp_interfaces);
  * phylink_set_port_modes() - set the port type modes in the ethtool mask
  * @mask: ethtool link mode mask
  *
- * Sets all the port type modes in the ethtool mask.  MAC drivers should
- * use this in their 'validate' callback.
+ * Sets all the port type modes in the ethtool mask.
  */
 void phylink_set_port_modes(unsigned long *mask)
 {
@@ -2095,9 +2094,9 @@ static int phylink_bringup_phy(struct phylink *pl, struct phy_device *phy,
 	/*
 	 * This is the new way of dealing with flow control for PHYs,
 	 * as described by Timur Tabi in commit 529ed1275263 ("net: phy:
-	 * phy drivers should not set SUPPORTED_[Asym_]Pause") except
-	 * using our validate call to the MAC, we rely upon the MAC
-	 * clearing the bits from both supported and advertising fields.
+	 * phy drivers should not set SUPPORTED_[Asym_]Pause"). MAC drivers
+	 * set their support using the MAC_SYM_PAUSE and MAC_ASYM_PAUSE
+	 * capabilities and must NOT change the phy's pause settings directly.
 	 */
 	phy_support_asym_pause(phy);
 
@@ -3931,9 +3930,9 @@ static int phylink_sfp_connect_phy(void *upstream, struct phy_device *phy)
 	/*
 	 * This is the new way of dealing with flow control for PHYs,
 	 * as described by Timur Tabi in commit 529ed1275263 ("net: phy:
-	 * phy drivers should not set SUPPORTED_[Asym_]Pause") except
-	 * using our validate call to the MAC, we rely upon the MAC
-	 * clearing the bits from both supported and advertising fields.
+	 * phy drivers should not set SUPPORTED_[Asym_]Pause"). MAC drivers
+	 * set their support using the MAC_SYM_PAUSE and MAC_ASYM_PAUSE
+	 * capabilities and must NOT change the phy's pause settings directly.
 	 */
 	phy_support_asym_pause(phy);
 
diff --git a/drivers/net/ppp/ppp_async.c b/drivers/net/ppp/ppp_async.c
index 93a7b0f6c4e7e..583426d063814 100644
--- a/drivers/net/ppp/ppp_async.c
+++ b/drivers/net/ppp/ppp_async.c
@@ -49,8 +49,6 @@ struct asyncppp {
 	unsigned long	xmit_flags;
 	u32		xaccm[8];
 	u32		raccm;
-	unsigned int	bytes_sent;
-	unsigned int	bytes_rcvd;
 
 	struct sk_buff	*tpkt;
 	int		tpkt_pos;
diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c
index 57c68efa5ff81..717c1d3aa9538 100644
--- a/drivers/net/ppp/ppp_generic.c
+++ b/drivers/net/ppp/ppp_generic.c
@@ -184,6 +184,7 @@ struct channel {
 	struct list_head clist;		/* link in list of channels per unit */
 	spinlock_t	upl;		/* protects `ppp' and 'bridge' */
 	struct channel __rcu *bridge;	/* "bridged" ppp channel */
+	struct rcu_head rcu;		/* for RCU-deferred free of the channel */
 #ifdef CONFIG_PPP_MULTILINK
 	u8		avail;		/* flag used in multilink stuff */
 	u8		had_frag;	/* >= 1 fragments have been sent */
@@ -3562,6 +3563,18 @@ ppp_disconnect_channel(struct channel *pch)
 	return err;
 }
 
+/* Purge after the grace period: a late ppp_input() may still queue an
+ * skb on pch->file.rq before the last RCU reader drains.
+ */
+static void ppp_release_channel_free(struct rcu_head *rcu)
+{
+	struct channel *pch = container_of(rcu, struct channel, rcu);
+
+	skb_queue_purge(&pch->file.xq);
+	skb_queue_purge(&pch->file.rq);
+	kfree(pch);
+}
+
 /*
  * Drop a reference to a ppp channel and free its memory if the refcount reaches
  * zero.
@@ -3581,9 +3594,7 @@ static void ppp_release_channel(struct channel *pch)
 		pr_err("ppp: destroying undead channel %p !\n", pch);
 		return;
 	}
-	skb_queue_purge(&pch->file.xq);
-	skb_queue_purge(&pch->file.rq);
-	kfree(pch);
+	call_rcu(&pch->rcu, ppp_release_channel_free);
 }
 
 static void __exit ppp_cleanup(void)
@@ -3596,6 +3607,7 @@ static void __exit ppp_cleanup(void)
 	device_destroy(&ppp_class, MKDEV(PPP_MAJOR, 0));
 	class_unregister(&ppp_class);
 	unregister_pernet_device(&ppp_net_ops);
+	rcu_barrier(); /* wait for RCU callbacks before module unload */
 }
 
 /*
diff --git a/drivers/net/ppp/ppp_synctty.c b/drivers/net/ppp/ppp_synctty.c
index b7f243b416f82..0b1bd1635c39a 100644
--- a/drivers/net/ppp/ppp_synctty.c
+++ b/drivers/net/ppp/ppp_synctty.c
@@ -59,8 +59,6 @@ struct syncppp {
 	unsigned long	xmit_flags;
 	u32		xaccm[8];
 	u32		raccm;
-	unsigned int	bytes_sent;
-	unsigned int	bytes_rcvd;
 
 	struct sk_buff	*tpkt;
 	unsigned long	last_xmit;
diff --git a/drivers/net/thunderbolt/main.c b/drivers/net/thunderbolt/main.c
index 02a91650561ab..eab443ffe0fdd 100644
--- a/drivers/net/thunderbolt/main.c
+++ b/drivers/net/thunderbolt/main.c
@@ -1339,7 +1339,7 @@ static void tbnet_generate_mac(struct net_device *dev)
 	dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
 }
 
-static int tbnet_probe(struct tb_service *svc, const struct tb_service_id *id)
+static int tbnet_probe(struct tb_service *svc)
 {
 	struct tb_xdomain *xd = tb_service_parent(svc);
 	struct net_device *dev;
@@ -1459,7 +1459,7 @@ static DEFINE_SIMPLE_DEV_PM_OPS(tbnet_pm_ops, tbnet_suspend, tbnet_resume);
 
 static const struct tb_service_id tbnet_ids[] = {
 	{ TB_SERVICE("network", 1) },
-	{ },
+	{ }
 };
 MODULE_DEVICE_TABLE(tbsvc, tbnet_ids);
 
diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index ffbe6f13fb1fa..9e27618878969 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -532,7 +532,7 @@ static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile)
 {
 	tfile->detached = tun;
 	list_add_tail(&tfile->next, &tun->disabled);
-	++tun->numdisabled;
+	WRITE_ONCE(tun->numdisabled, tun->numdisabled + 1);
 }
 
 static struct tun_struct *tun_enable_queue(struct tun_file *tfile)
@@ -541,7 +541,7 @@ static struct tun_struct *tun_enable_queue(struct tun_file *tfile)
 
 	tfile->detached = NULL;
 	list_del_init(&tfile->next);
-	--tun->numdisabled;
+	WRITE_ONCE(tun->numdisabled, tun->numdisabled - 1);
 	return tun;
 }
 
@@ -600,7 +600,7 @@ static void __tun_detach(struct tun_file *tfile, bool clean)
 		rcu_assign_pointer(tun->tfiles[tun->numqueues - 1],
 				   NULL);
 
-		--tun->numqueues;
+		WRITE_ONCE(tun->numqueues, tun->numqueues - 1);
 		if (clean) {
 			RCU_INIT_POINTER(tfile->tun, NULL);
 			sock_put(&tfile->sk);
@@ -663,7 +663,7 @@ static void tun_detach_all(struct net_device *dev)
 		tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
 		tfile->socket.sk->sk_data_ready(tfile->socket.sk);
 		RCU_INIT_POINTER(tfile->tun, NULL);
-		--tun->numqueues;
+		WRITE_ONCE(tun->numqueues, tun->numqueues - 1);
 	}
 	list_for_each_entry(tfile, &tun->disabled, next) {
 		tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
@@ -786,7 +786,7 @@ static int tun_attach(struct tun_struct *tun, struct file *file,
 	if (publish_tun)
 		rcu_assign_pointer(tfile->tun, tun);
 	rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile);
-	tun->numqueues++;
+	WRITE_ONCE(tun->numqueues, tun->numqueues + 1);
 	tun_set_real_num_queues(tun);
 out:
 	return err;
@@ -2370,32 +2370,36 @@ static size_t tun_get_size(const struct net_device *dev)
 
 static int tun_fill_info(struct sk_buff *skb, const struct net_device *dev)
 {
-	struct tun_struct *tun = netdev_priv(dev);
+	const struct tun_struct *tun = netdev_priv(dev);
+	unsigned int flags = READ_ONCE(tun->flags);
+	kuid_t owner = READ_ONCE(tun->owner);
+	kgid_t group = READ_ONCE(tun->group);
 
-	if (nla_put_u8(skb, IFLA_TUN_TYPE, tun->flags & TUN_TYPE_MASK))
+	if (nla_put_u8(skb, IFLA_TUN_TYPE, flags & TUN_TYPE_MASK))
 		goto nla_put_failure;
-	if (uid_valid(tun->owner) &&
+	if (uid_valid(owner) &&
 	    nla_put_u32(skb, IFLA_TUN_OWNER,
-			from_kuid_munged(current_user_ns(), tun->owner)))
+			from_kuid_munged(current_user_ns(), owner)))
 		goto nla_put_failure;
-	if (gid_valid(tun->group) &&
+	if (gid_valid(group) &&
 	    nla_put_u32(skb, IFLA_TUN_GROUP,
-			from_kgid_munged(current_user_ns(), tun->group)))
+			from_kgid_munged(current_user_ns(), group)))
 		goto nla_put_failure;
-	if (nla_put_u8(skb, IFLA_TUN_PI, !(tun->flags & IFF_NO_PI)))
+	if (nla_put_u8(skb, IFLA_TUN_PI, !(flags & IFF_NO_PI)))
 		goto nla_put_failure;
-	if (nla_put_u8(skb, IFLA_TUN_VNET_HDR, !!(tun->flags & IFF_VNET_HDR)))
+	if (nla_put_u8(skb, IFLA_TUN_VNET_HDR, !!(flags & IFF_VNET_HDR)))
 		goto nla_put_failure;
-	if (nla_put_u8(skb, IFLA_TUN_PERSIST, !!(tun->flags & IFF_PERSIST)))
+	if (nla_put_u8(skb, IFLA_TUN_PERSIST, !!(flags & IFF_PERSIST)))
 		goto nla_put_failure;
 	if (nla_put_u8(skb, IFLA_TUN_MULTI_QUEUE,
-		       !!(tun->flags & IFF_MULTI_QUEUE)))
+		       !!(flags & IFF_MULTI_QUEUE)))
 		goto nla_put_failure;
-	if (tun->flags & IFF_MULTI_QUEUE) {
-		if (nla_put_u32(skb, IFLA_TUN_NUM_QUEUES, tun->numqueues))
+	if (flags & IFF_MULTI_QUEUE) {
+		if (nla_put_u32(skb, IFLA_TUN_NUM_QUEUES,
+				READ_ONCE(tun->numqueues)))
 			goto nla_put_failure;
 		if (nla_put_u32(skb, IFLA_TUN_NUM_DISABLED_QUEUES,
-				tun->numdisabled))
+				READ_ONCE(tun->numdisabled)))
 			goto nla_put_failure;
 	}
 
@@ -2814,8 +2818,8 @@ static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr)
 			return 0;
 		}
 
-		tun->flags = (tun->flags & ~TUN_FEATURES) |
-			      (ifr->ifr_flags & TUN_FEATURES);
+		WRITE_ONCE(tun->flags, (tun->flags & ~TUN_FEATURES) |
+				       (ifr->ifr_flags & TUN_FEATURES));
 
 		netdev_state_change(dev);
 	} else {
@@ -3213,13 +3217,13 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
 		/* Disable/Enable persist mode. Keep an extra reference to the
 		 * module to prevent the module being unprobed.
 		 */
-		if (arg && !(tun->flags & IFF_PERSIST)) {
-			tun->flags |= IFF_PERSIST;
+		if (arg && !(READ_ONCE(tun->flags) & IFF_PERSIST)) {
+			WRITE_ONCE(tun->flags, READ_ONCE(tun->flags) | IFF_PERSIST);
 			__module_get(THIS_MODULE);
 			do_notify = true;
 		}
-		if (!arg && (tun->flags & IFF_PERSIST)) {
-			tun->flags &= ~IFF_PERSIST;
+		if (!arg && (READ_ONCE(tun->flags) & IFF_PERSIST)) {
+			WRITE_ONCE(tun->flags, READ_ONCE(tun->flags) & ~IFF_PERSIST);
 			module_put(THIS_MODULE);
 			do_notify = true;
 		}
@@ -3235,10 +3239,10 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
 			ret = -EINVAL;
 			break;
 		}
-		tun->owner = owner;
+		WRITE_ONCE(tun->owner, owner);
 		do_notify = true;
 		netif_info(tun, drv, tun->dev, "owner set to %u\n",
-			   from_kuid(&init_user_ns, tun->owner));
+			   from_kuid(&init_user_ns, owner));
 		break;
 
 	case TUNSETGROUP:
@@ -3248,10 +3252,10 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
 			ret = -EINVAL;
 			break;
 		}
-		tun->group = group;
+		WRITE_ONCE(tun->group, group);
 		do_notify = true;
 		netif_info(tun, drv, tun->dev, "group set to %u\n",
-			   from_kgid(&init_user_ns, tun->group));
+			   from_kgid(&init_user_ns, group));
 		break;
 
 	case TUNSETLINK:
diff --git a/drivers/net/tun_vnet.h b/drivers/net/tun_vnet.h
index fa5cab9d3e55c..f4c652b1fa44d 100644
--- a/drivers/net/tun_vnet.h
+++ b/drivers/net/tun_vnet.h
@@ -40,9 +40,9 @@ static inline long tun_set_vnet_be(unsigned int *flags, int __user *argp)
 		return -EFAULT;
 
 	if (be)
-		*flags |= TUN_VNET_BE;
+		WRITE_ONCE(*flags, *flags | TUN_VNET_BE);
 	else
-		*flags &= ~TUN_VNET_BE;
+		WRITE_ONCE(*flags, *flags & ~TUN_VNET_BE);
 
 	return 0;
 }
@@ -93,9 +93,9 @@ static inline long tun_vnet_ioctl(int *vnet_hdr_sz, unsigned int *flags,
 		if (get_user(s, sp))
 			return -EFAULT;
 		if (s)
-			*flags |= TUN_VNET_LE;
+			WRITE_ONCE(*flags, *flags | TUN_VNET_LE);
 		else
-			*flags &= ~TUN_VNET_LE;
+			WRITE_ONCE(*flags, *flags & ~TUN_VNET_LE);
 		return 0;
 
 	case TUNGETVNETBE:
diff --git a/drivers/net/usb/pegasus.c b/drivers/net/usb/pegasus.c
index 8700eeb8e22d0..aba1a640fc268 100644
--- a/drivers/net/usb/pegasus.c
+++ b/drivers/net/usb/pegasus.c
@@ -43,21 +43,12 @@ static bool loopback;
 static bool mii_mode;
 static char *devid;
 
-static struct usb_eth_dev usb_dev_id[] = {
-#define	PEGASUS_DEV(pn, vid, pid, flags)	\
-	{.name = pn, .vendor = vid, .device = pid, .private = flags},
-#define PEGASUS_DEV_CLASS(pn, vid, pid, dclass, flags) \
-	PEGASUS_DEV(pn, vid, pid, flags)
-#include "pegasus.h"
-#undef	PEGASUS_DEV
-#undef	PEGASUS_DEV_CLASS
-	{NULL, 0, 0, 0},
-	{NULL, 0, 0, 0}
-};
+static struct usb_eth_dev dynamic_id_info = {};
 
 static struct usb_device_id pegasus_ids[] = {
 #define	PEGASUS_DEV(pn, vid, pid, flags) \
-	{.match_flags = USB_DEVICE_ID_MATCH_DEVICE, .idVendor = vid, .idProduct = pid},
+	{.match_flags = USB_DEVICE_ID_MATCH_DEVICE, .idVendor = vid, .idProduct = pid, \
+	 .driver_info = (kernel_ulong_t)&(const struct usb_eth_dev) {.name = pn, .private = flags}},
 /*
  * The Belkin F8T012xx1 bluetooth adaptor has the same vendor and product
  * IDs as the Belkin F5D5050, so we need to teach the pegasus driver to
@@ -66,7 +57,8 @@ static struct usb_device_id pegasus_ids[] = {
  */
 #define PEGASUS_DEV_CLASS(pn, vid, pid, dclass, flags) \
 	{.match_flags = (USB_DEVICE_ID_MATCH_DEVICE | USB_DEVICE_ID_MATCH_DEV_CLASS), \
-	.idVendor = vid, .idProduct = pid, .bDeviceClass = dclass},
+	.idVendor = vid, .idProduct = pid, .bDeviceClass = dclass, \
+	.driver_info = (kernel_ulong_t)&(const struct usb_eth_dev) {.name = pn, .private = flags}},
 #include "pegasus.h"
 #undef	PEGASUS_DEV
 #undef	PEGASUS_DEV_CLASS
@@ -402,12 +394,12 @@ static inline int reset_mac(pegasus_t *pegasus)
 	if (i == REG_TIMEOUT)
 		return -ETIMEDOUT;
 
-	if (usb_dev_id[pegasus->dev_index].vendor == VENDOR_LINKSYS ||
-	    usb_dev_id[pegasus->dev_index].vendor == VENDOR_DLINK) {
+	if (le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_LINKSYS ||
+	    le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_DLINK) {
 		set_register(pegasus, Gpio0, 0x24);
 		set_register(pegasus, Gpio0, 0x26);
 	}
-	if (usb_dev_id[pegasus->dev_index].vendor == VENDOR_ELCON) {
+	if (le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_ELCON) {
 		__u16 auxmode;
 		ret = read_mii_word(pegasus, 3, 0x1b, &auxmode);
 		if (ret < 0)
@@ -445,9 +437,9 @@ static int enable_net_traffic(struct net_device *dev, struct usb_device *usb)
 	memcpy(pegasus->eth_regs, data, sizeof(data));
 	ret = set_registers(pegasus, EthCtrl0, 3, data);
 
-	if (usb_dev_id[pegasus->dev_index].vendor == VENDOR_LINKSYS ||
-	    usb_dev_id[pegasus->dev_index].vendor == VENDOR_LINKSYS2 ||
-	    usb_dev_id[pegasus->dev_index].vendor == VENDOR_DLINK) {
+	if (le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_LINKSYS ||
+	    le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_LINKSYS2 ||
+	    le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_DLINK) {
 		u16 auxmode;
 		ret = read_mii_word(pegasus, 0, 0x1b, &auxmode);
 		if (ret < 0)
@@ -1153,7 +1145,7 @@ static int pegasus_probe(struct usb_interface *intf,
 	struct usb_device *dev = interface_to_usbdev(intf);
 	struct net_device *net;
 	pegasus_t *pegasus;
-	int dev_index = id - pegasus_ids;
+	const struct usb_eth_dev *info = (const struct usb_eth_dev *)id->driver_info;
 	int res = -ENOMEM;
 	static const u8 bulk_ep_addr[] = {
 		PEGASUS_USB_EP_BULK_IN | USB_DIR_IN,
@@ -1178,7 +1170,6 @@ static int pegasus_probe(struct usb_interface *intf,
 		goto out;
 
 	pegasus = netdev_priv(net);
-	pegasus->dev_index = dev_index;
 	pegasus->intf = intf;
 
 	res = alloc_urbs(pegasus);
@@ -1206,7 +1197,7 @@ static int pegasus_probe(struct usb_interface *intf,
 	pegasus->msg_enable = netif_msg_init(msg_level, NETIF_MSG_DRV
 				| NETIF_MSG_PROBE | NETIF_MSG_LINK);
 
-	pegasus->features = usb_dev_id[dev_index].private;
+	pegasus->features = info ? info->private : DEFAULT_GPIO_RESET;
 	res = get_interrupt_interval(pegasus);
 	if (res)
 		goto out2;
@@ -1235,7 +1226,7 @@ static int pegasus_probe(struct usb_interface *intf,
 	queue_delayed_work(system_long_wq, &pegasus->carrier_check,
 			   CARRIER_CHECK_DELAY);
 	dev_info(&intf->dev, "%s, %s, %pM\n", net->name,
-		 usb_dev_id[dev_index].name, net->dev_addr);
+		 info ? info->name : "(unknown)", net->dev_addr);
 	return 0;
 
 out3:
@@ -1325,8 +1316,9 @@ static struct usb_driver pegasus_driver = {
 
 static void __init parse_id(char *id)
 {
-	unsigned int vendor_id = 0, device_id = 0, flags = 0, i = 0;
+	unsigned int vendor_id = 0, device_id = 0, flags = 0;
 	char *token, *name = NULL;
+	int dyn_id_index = ARRAY_SIZE(pegasus_ids) - 2;
 
 	token = strsep(&id, ":");
 	if (token)
@@ -1348,14 +1340,12 @@ static void __init parse_id(char *id)
 	if (device_id > 0x10000 || device_id == 0)
 		return;
 
-	for (i = 0; usb_dev_id[i].name; i++);
-	usb_dev_id[i].name = name;
-	usb_dev_id[i].vendor = vendor_id;
-	usb_dev_id[i].device = device_id;
-	usb_dev_id[i].private = flags;
-	pegasus_ids[i].match_flags = USB_DEVICE_ID_MATCH_DEVICE;
-	pegasus_ids[i].idVendor = vendor_id;
-	pegasus_ids[i].idProduct = device_id;
+	dynamic_id_info.name = name;
+	dynamic_id_info.private = flags;
+	pegasus_ids[dyn_id_index].match_flags = USB_DEVICE_ID_MATCH_DEVICE;
+	pegasus_ids[dyn_id_index].idVendor = vendor_id;
+	pegasus_ids[dyn_id_index].idProduct = device_id;
+	pegasus_ids[dyn_id_index].driver_info = (kernel_ulong_t)&dynamic_id_info;
 }
 
 static int __init pegasus_init(void)
diff --git a/drivers/net/usb/pegasus.h b/drivers/net/usb/pegasus.h
index a05b143155ba8..ccdedcef52e76 100644
--- a/drivers/net/usb/pegasus.h
+++ b/drivers/net/usb/pegasus.h
@@ -85,7 +85,6 @@ typedef struct pegasus {
 	unsigned		features;
 	u32			msg_enable;
 	u32			wolopts;
-	int			dev_index;
 	int			intr_interval;
 	struct tasklet_struct	rx_tl;
 	struct delayed_work	carrier_check;
@@ -102,8 +101,6 @@ typedef struct pegasus {
 
 struct usb_eth_dev {
 	char	*name;
-	__u16	vendor;
-	__u16	device;
 	__u32	private; /* LSB is gpio reset value */
 };
 
diff --git a/drivers/net/usb/rtl8150.c b/drivers/net/usb/rtl8150.c
index c880c95c41a5e..d51e43170e033 100644
--- a/drivers/net/usb/rtl8150.c
+++ b/drivers/net/usb/rtl8150.c
@@ -732,7 +732,9 @@ static void set_carrier(struct net_device *netdev)
 	rtl8150_t *dev = netdev_priv(netdev);
 	short tmp;
 
-	get_registers(dev, CSCR, 2, &tmp);
+	if (get_registers(dev, CSCR, 2, &tmp))
+		return;
+
 	if (tmp & CSCR_LINK_STATUS)
 		netif_carrier_on(netdev);
 	else
diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index 1c51421491753..8170bf33ccf95 100644
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -77,6 +77,7 @@ struct veth_priv {
 	struct bpf_prog		*_xdp_prog;
 	struct veth_rq		*rq;
 	unsigned int		requested_headroom;
+	netdevice_tracker	peer_tracker;
 };
 
 struct veth_xdp_tx_bq {
@@ -1901,15 +1902,17 @@ static int veth_newlink(struct net_device *dev,
 
 	priv = netdev_priv(dev);
 	rcu_assign_pointer(priv->peer, peer);
+	netdev_hold(peer, &priv->peer_tracker, GFP_KERNEL);
 	err = veth_init_queues(dev, tb);
 	if (err)
 		goto err_queues;
 
 	priv = netdev_priv(peer);
 	rcu_assign_pointer(priv->peer, dev);
+	netdev_hold(dev, &priv->peer_tracker, GFP_KERNEL);
 	err = veth_init_queues(peer, tb);
 	if (err)
-		goto err_queues;
+		goto err_peer_queues;
 
 	veth_disable_gro(dev);
 	/* update XDP supported features */
@@ -1918,7 +1921,11 @@ static int veth_newlink(struct net_device *dev,
 
 	return 0;
 
+err_peer_queues:
+	netdev_put(dev, &priv->peer_tracker);
+	priv = netdev_priv(dev);
 err_queues:
+	netdev_put(peer, &priv->peer_tracker);
 	unregister_netdevice(dev);
 err_register_dev:
 	/* nothing to do */
@@ -1933,24 +1940,25 @@ static int veth_newlink(struct net_device *dev,
 
 static void veth_dellink(struct net_device *dev, struct list_head *head)
 {
-	struct veth_priv *priv;
+	netdevice_tracker *peer_tracker;
 	struct net_device *peer;
+	struct veth_priv *priv;
 
 	priv = netdev_priv(dev);
-	peer = rtnl_dereference(priv->peer);
+	peer_tracker = &priv->peer_tracker;
+	peer = unrcu_pointer(xchg(&priv->peer, NULL));
+	if (!peer)
+		return;
 
-	/* Note : dellink() is called from default_device_exit_batch(),
-	 * before a rcu_synchronize() point. The devices are guaranteed
-	 * not being freed before one RCU grace period.
-	 */
-	RCU_INIT_POINTER(priv->peer, NULL);
 	unregister_netdevice_queue(dev, head);
 
-	if (peer) {
-		priv = netdev_priv(peer);
-		RCU_INIT_POINTER(priv->peer, NULL);
-		unregister_netdevice_queue(peer, head);
-	}
+	priv = netdev_priv(peer);
+	dev = unrcu_pointer(xchg(&priv->peer, NULL));
+	if (dev)
+		unregister_netdevice_queue_net(dev_net(dev), peer, head);
+
+	netdev_put(peer, peer_tracker);
+	netdev_put(dev, &priv->peer_tracker);
 }
 
 static const struct nla_policy veth_policy[VETH_INFO_MAX + 1] = {
diff --git a/drivers/net/wireless/ath/ath10k/ahb.c b/drivers/net/wireless/ath/ath10k/ahb.c
index eb8b35b6224db..7456f885d2b56 100644
--- a/drivers/net/wireless/ath/ath10k/ahb.c
+++ b/drivers/net/wireless/ath/ath10k/ahb.c
@@ -87,24 +87,24 @@ static int ath10k_ahb_clock_init(struct ath10k *ar)
 	dev = &ar_ahb->pdev->dev;
 
 	ar_ahb->cmd_clk = devm_clk_get(dev, "wifi_wcss_cmd");
-	if (IS_ERR_OR_NULL(ar_ahb->cmd_clk)) {
+	if (IS_ERR(ar_ahb->cmd_clk)) {
 		ath10k_err(ar, "failed to get cmd clk: %ld\n",
 			   PTR_ERR(ar_ahb->cmd_clk));
-		return ar_ahb->cmd_clk ? PTR_ERR(ar_ahb->cmd_clk) : -ENODEV;
+		return PTR_ERR(ar_ahb->cmd_clk);
 	}
 
 	ar_ahb->ref_clk = devm_clk_get(dev, "wifi_wcss_ref");
-	if (IS_ERR_OR_NULL(ar_ahb->ref_clk)) {
+	if (IS_ERR(ar_ahb->ref_clk)) {
 		ath10k_err(ar, "failed to get ref clk: %ld\n",
 			   PTR_ERR(ar_ahb->ref_clk));
-		return ar_ahb->ref_clk ? PTR_ERR(ar_ahb->ref_clk) : -ENODEV;
+		return PTR_ERR(ar_ahb->ref_clk);
 	}
 
 	ar_ahb->rtc_clk = devm_clk_get(dev, "wifi_wcss_rtc");
-	if (IS_ERR_OR_NULL(ar_ahb->rtc_clk)) {
+	if (IS_ERR(ar_ahb->rtc_clk)) {
 		ath10k_err(ar, "failed to get rtc clk: %ld\n",
 			   PTR_ERR(ar_ahb->rtc_clk));
-		return ar_ahb->rtc_clk ? PTR_ERR(ar_ahb->rtc_clk) : -ENODEV;
+		return PTR_ERR(ar_ahb->rtc_clk);
 	}
 
 	return 0;
diff --git a/drivers/net/wireless/ath/ath10k/htt_rx.c b/drivers/net/wireless/ath/ath10k/htt_rx.c
index faac359aa9acb..ab2d373b4750d 100644
--- a/drivers/net/wireless/ath/ath10k/htt_rx.c
+++ b/drivers/net/wireless/ath/ath10k/htt_rx.c
@@ -706,6 +706,7 @@ static int ath10k_htt_rx_pop_paddr32_list(struct ath10k_htt *htt,
 			if (!(__le32_to_cpu(rxd_attention->flags) &
 			      RX_ATTENTION_FLAGS_MSDU_DONE)) {
 				ath10k_warn(htt->ar, "tried to pop an incomplete frame, oops!\n");
+				__skb_queue_purge(list);
 				return -EIO;
 			}
 		}
@@ -770,6 +771,7 @@ static int ath10k_htt_rx_pop_paddr64_list(struct ath10k_htt *htt,
 			if (!(__le32_to_cpu(rxd_attention->flags) &
 			      RX_ATTENTION_FLAGS_MSDU_DONE)) {
 				ath10k_warn(htt->ar, "tried to pop an incomplete frame, oops!\n");
+				__skb_queue_purge(list);
 				return -EIO;
 			}
 		}
@@ -2343,10 +2345,8 @@ static int ath10k_htt_rx_handle_amsdu(struct ath10k_htt *htt)
 	if (ret < 0) {
 		ath10k_warn(ar, "rx ring became corrupted: %d\n", ret);
 		__skb_queue_purge(&amsdu);
-		/* FIXME: It's probably a good idea to reboot the
-		 * device instead of leaving it inoperable.
-		 */
 		htt->rx_confused = true;
+		ath10k_core_start_recovery(ar);
 		return ret;
 	}
 
@@ -3311,6 +3311,7 @@ static int ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
 	if (ret < 0) {
 		ath10k_warn(ar, "failed to pop paddr list: %d\n", ret);
 		htt->rx_confused = true;
+		ath10k_core_start_recovery(ar);
 		return -EIO;
 	}
 
@@ -3344,6 +3345,7 @@ static int ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
 			ath10k_warn(ar, "failed to extract amsdu: %d\n", ret);
 			htt->rx_confused = true;
 			__skb_queue_purge(&list);
+			ath10k_core_start_recovery(ar);
 			return -EIO;
 		}
 	}
diff --git a/drivers/net/wireless/ath/ath11k/ahb.c b/drivers/net/wireless/ath/ath11k/ahb.c
index f566d699d074a..1e1dea485760b 100644
--- a/drivers/net/wireless/ath/ath11k/ahb.c
+++ b/drivers/net/wireless/ath/ath11k/ahb.c
@@ -996,6 +996,7 @@ static int ath11k_ahb_fw_resources_init(struct ath11k_base *ab)
 	ret = ath11k_ahb_setup_msa_resources(ab);
 	if (ret) {
 		ath11k_err(ab, "failed to setup msa resources\n");
+		of_node_put(node);
 		return ret;
 	}
 
diff --git a/drivers/net/wireless/ath/ath11k/dp_rx.c b/drivers/net/wireless/ath/ath11k/dp_rx.c
index 9e90d8e3f1557..33425707c0842 100644
--- a/drivers/net/wireless/ath/ath11k/dp_rx.c
+++ b/drivers/net/wireless/ath/ath11k/dp_rx.c
@@ -2334,10 +2334,10 @@ static void ath11k_dp_rx_h_rate(struct ath11k *ar, struct hal_rx_desc *rx_desc,
 	case RX_MSDU_START_PKT_TYPE_11N:
 		rx_status->encoding = RX_ENC_HT;
 		if (rate_mcs > ATH11K_HT_MCS_MAX) {
-			ath11k_warn(ar->ab,
-				    "Received with invalid mcs in HT mode %d\n",
-				     rate_mcs);
-			break;
+			ath11k_dbg(ar->ab, ATH11K_DBG_DP_RX,
+				   "Received HT frame with out-of-range mcs %d, capping to %d\n",
+				   rate_mcs, ATH11K_HT_MCS_MAX);
+			rate_mcs = ATH11K_HT_MCS_MAX;
 		}
 		rx_status->rate_idx = rate_mcs + (8 * (nss - 1));
 		if (sgi)
@@ -2346,13 +2346,13 @@ static void ath11k_dp_rx_h_rate(struct ath11k *ar, struct hal_rx_desc *rx_desc,
 		break;
 	case RX_MSDU_START_PKT_TYPE_11AC:
 		rx_status->encoding = RX_ENC_VHT;
-		rx_status->rate_idx = rate_mcs;
 		if (rate_mcs > ATH11K_VHT_MCS_MAX) {
-			ath11k_warn(ar->ab,
-				    "Received with invalid mcs in VHT mode %d\n",
-				     rate_mcs);
-			break;
+			ath11k_dbg(ar->ab, ATH11K_DBG_DP_RX,
+				   "Received VHT frame with out-of-range mcs %d, capping to %d\n",
+				   rate_mcs, ATH11K_VHT_MCS_MAX);
+			rate_mcs = ATH11K_VHT_MCS_MAX;
 		}
+		rx_status->rate_idx = rate_mcs;
 		rx_status->nss = nss;
 		if (sgi)
 			rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI;
@@ -2362,14 +2362,14 @@ static void ath11k_dp_rx_h_rate(struct ath11k *ar, struct hal_rx_desc *rx_desc,
 			rx_status->enc_flags |= RX_ENC_FLAG_LDPC;
 		break;
 	case RX_MSDU_START_PKT_TYPE_11AX:
-		rx_status->rate_idx = rate_mcs;
+		rx_status->encoding = RX_ENC_HE;
 		if (rate_mcs > ATH11K_HE_MCS_MAX) {
-			ath11k_warn(ar->ab,
-				    "Received with invalid mcs in HE mode %d\n",
-				    rate_mcs);
-			break;
+			ath11k_dbg(ar->ab, ATH11K_DBG_DP_RX,
+				   "Received HE frame with out-of-range mcs %d, capping to %d\n",
+				   rate_mcs, ATH11K_HE_MCS_MAX);
+			rate_mcs = ATH11K_HE_MCS_MAX;
 		}
-		rx_status->encoding = RX_ENC_HE;
+		rx_status->rate_idx = rate_mcs;
 		rx_status->nss = nss;
 		rx_status->he_gi = ath11k_mac_he_gi_to_nl80211_he_gi(sgi);
 		rx_status->bw = ath11k_mac_bw_to_mac80211_bw(bw);
@@ -4618,6 +4618,9 @@ static void ath11k_hal_rx_msdu_list_get(struct ath11k *ar,
 	msdu_details = &msdu_link->msdu_link[0];
 
 	for (i = 0; i < HAL_RX_NUM_MSDU_DESC; i++) {
+		if (!i && FIELD_GET(BUFFER_ADDR_INFO0_ADDR,
+				    msdu_details[i].buf_addr_info.info0) == 0)
+			break;
 		if (FIELD_GET(BUFFER_ADDR_INFO0_ADDR,
 			      msdu_details[i].buf_addr_info.info0) == 0) {
 			msdu_desc_info = &msdu_details[i - 1].rx_msdu_info;
diff --git a/drivers/net/wireless/ath/ath11k/pci.c b/drivers/net/wireless/ath/ath11k/pci.c
index 35bb9e7a63a20..a163168f36176 100644
--- a/drivers/net/wireless/ath/ath11k/pci.c
+++ b/drivers/net/wireless/ath/ath11k/pci.c
@@ -199,6 +199,8 @@ static void ath11k_pci_soc_global_reset(struct ath11k_base *ab)
 	val |= PCIE_SOC_GLOBAL_RESET_V;
 
 	ath11k_pcic_write32(ab, PCIE_SOC_GLOBAL_RESET, val);
+	/* Flush the posted write to the device */
+	ath11k_pcic_read32(ab, PCIE_SOC_GLOBAL_RESET);
 
 	/* TODO: exact time to sleep is uncertain */
 	delay = 10;
@@ -208,6 +210,8 @@ static void ath11k_pci_soc_global_reset(struct ath11k_base *ab)
 	val &= ~PCIE_SOC_GLOBAL_RESET_V;
 
 	ath11k_pcic_write32(ab, PCIE_SOC_GLOBAL_RESET, val);
+	/* Flush the posted write to the device */
+	ath11k_pcic_read32(ab, PCIE_SOC_GLOBAL_RESET);
 
 	mdelay(delay);
 
diff --git a/drivers/net/wireless/ath/ath11k/qmi.c b/drivers/net/wireless/ath/ath11k/qmi.c
index 410a7ee076a02..7dc07339b9579 100644
--- a/drivers/net/wireless/ath/ath11k/qmi.c
+++ b/drivers/net/wireless/ath/ath11k/qmi.c
@@ -3294,9 +3294,14 @@ static void ath11k_qmi_driver_event_work(struct work_struct *work)
 			clear_bit(ATH11K_FLAG_CRASH_FLUSH,
 				  &ab->dev_flags);
 			clear_bit(ATH11K_FLAG_RECOVERY, &ab->dev_flags);
-			ath11k_core_qmi_firmware_ready(ab);
-			set_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags);
-
+			if (!test_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags)) {
+				ret = ath11k_core_qmi_firmware_ready(ab);
+				if (ret) {
+					set_bit(ATH11K_FLAG_QMI_FAIL, &ab->dev_flags);
+					break;
+				}
+				set_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags);
+			}
 			break;
 		case ATH11K_QMI_EVENT_COLD_BOOT_CAL_DONE:
 			break;
diff --git a/drivers/net/wireless/ath/ath11k/wmi.c b/drivers/net/wireless/ath/ath11k/wmi.c
index dca6e011cc40e..28acfc6829df2 100644
--- a/drivers/net/wireless/ath/ath11k/wmi.c
+++ b/drivers/net/wireless/ath/ath11k/wmi.c
@@ -4858,6 +4858,12 @@ static int ath11k_wmi_tlv_ext_hal_reg_caps(struct ath11k_base *soc,
 			return ret;
 		}
 
+		if (reg_cap.phy_id >= ARRAY_SIZE(soc->hal_reg_cap)) {
+			ath11k_warn(soc, "invalid reg cap phy_id %u\n",
+				    reg_cap.phy_id);
+			return -EINVAL;
+		}
+
 		memcpy(&soc->hal_reg_cap[reg_cap.phy_id],
 		       &reg_cap, sizeof(reg_cap));
 	}
diff --git a/drivers/net/wireless/ath/ath12k/Kconfig b/drivers/net/wireless/ath/ath12k/Kconfig
index 4a2b240f967af..0d5d1c55bfc13 100644
--- a/drivers/net/wireless/ath/ath12k/Kconfig
+++ b/drivers/net/wireless/ath/ath12k/Kconfig
@@ -18,7 +18,7 @@ config ATH12K_AHB
 	bool "Qualcomm ath12k AHB support"
 	depends on ATH12K && REMOTEPROC
 	select QCOM_MDT_LOADER
-	select QCOM_SCM
+	select QCOM_PAS
 	help
 	  Enable support for Ath12k AHB bus chipsets, example IPQ5332.
 
diff --git a/drivers/net/wireless/ath/ath12k/ahb.c b/drivers/net/wireless/ath/ath12k/ahb.c
index 30733a244454e..4944ea7855cad 100644
--- a/drivers/net/wireless/ath/ath12k/ahb.c
+++ b/drivers/net/wireless/ath/ath12k/ahb.c
@@ -5,7 +5,7 @@
  */
 
 #include <linux/dma-mapping.h>
-#include <linux/firmware/qcom/qcom_scm.h>
+#include <linux/firmware/qcom/qcom_pas.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
@@ -17,7 +17,6 @@
 #include "hif.h"
 
 #define ATH12K_IRQ_CE0_OFFSET 4
-#define ATH12K_MAX_UPDS 1
 #define ATH12K_UPD_IRQ_WRD_LEN  18
 
 static struct ath12k_ahb_driver *ath12k_ahb_family_drivers[ATH12K_DEVICE_FAMILY_MAX];
@@ -420,7 +419,7 @@ static int ath12k_ahb_power_up(struct ath12k_base *ab)
 
 	if (ab_ahb->scm_auth_enabled) {
 		/* Authenticate FW image using peripheral ID */
-		ret = qcom_scm_pas_auth_and_reset(pasid);
+		ret = qcom_pas_auth_and_reset(pasid);
 		if (ret) {
 			ath12k_err(ab, "failed to boot the remote processor %d\n", ret);
 			goto err_fw2;
@@ -485,10 +484,10 @@ static void ath12k_ahb_power_down(struct ath12k_base *ab, bool is_suspend)
 		pasid = (u32_encode_bits(ab_ahb->userpd_id, ATH12K_USERPD_ID_MASK)) |
 			 ATH12K_AHB_UPD_SWID;
 		/* Release the firmware */
-		ret = qcom_scm_pas_shutdown(pasid);
+		ret = qcom_pas_shutdown(pasid);
 		if (ret)
-			ath12k_err(ab, "scm pas shutdown failed for userPD%d\n",
-				   ab_ahb->userpd_id);
+			ath12k_err(ab, "PAS shutdown failed for userPD%d: %d\n",
+				   ab_ahb->userpd_id, ret);
 	}
 }
 
diff --git a/drivers/net/wireless/ath/ath12k/ahb.h b/drivers/net/wireless/ath/ath12k/ahb.h
index 0fa15daaa3e6c..a153db6cf1d3d 100644
--- a/drivers/net/wireless/ath/ath12k/ahb.h
+++ b/drivers/net/wireless/ath/ath12k/ahb.h
@@ -27,7 +27,7 @@
 #define ATH12K_USERPD_SPAWN_TIMEOUT		(5 * HZ)
 #define ATH12K_USERPD_READY_TIMEOUT		(10 * HZ)
 #define ATH12K_USERPD_STOP_TIMEOUT		(5 * HZ)
-#define ATH12K_USERPD_ID_MASK			GENMASK(9, 8)
+#define ATH12K_USERPD_ID_MASK			GENMASK(10, 8)
 #define ATH12K_USERPD_FW_NAME_LEN		35
 
 enum ath12k_ahb_smp2p_msg_id {
diff --git a/drivers/net/wireless/ath/ath12k/core.c b/drivers/net/wireless/ath/ath12k/core.c
index 742d4fd1b598c..fb599caa3aab8 100644
--- a/drivers/net/wireless/ath/ath12k/core.c
+++ b/drivers/net/wireless/ath/ath12k/core.c
@@ -49,7 +49,7 @@ ath12k_mem_profile_based_param ath12k_mem_profile_based_param[] = {
 		.dp_params = {
 			.tx_comp_ring_size = 32768,
 			.rxdma_monitor_buf_ring_size = 4096,
-			.rxdma_monitor_dst_ring_size = 8092,
+			.rxdma_monitor_dst_ring_size = 8192,
 			.num_pool_tx_desc = 32768,
 			.rx_desc_count = 12288,
 		},
@@ -708,8 +708,10 @@ static void ath12k_core_stop(struct ath12k_base *ab)
 
 	ath12k_core_to_group_ref_put(ab);
 
-	if (!test_bit(ATH12K_FLAG_CRASH_FLUSH, &ab->dev_flags))
+	if (!test_bit(ATH12K_FLAG_CRASH_FLUSH, &ab->dev_flags)) {
+		ath12k_dp_reoq_lut_addr_reset(ath12k_ab_to_dp(ab));
 		ath12k_qmi_firmware_stop(ab);
+	}
 
 	ath12k_acpi_stop(ab);
 
@@ -1371,6 +1373,7 @@ int ath12k_core_qmi_firmware_ready(struct ath12k_base *ab)
 	goto exit;
 
 err_deinit:
+	ath12k_dp_reoq_lut_addr_reset(ath12k_ab_to_dp(ab));
 	ath12k_dp_cmn_device_deinit(ath12k_ab_to_dp(ab));
 	mutex_unlock(&ab->core_lock);
 	mutex_unlock(&ag->mutex);
@@ -1524,7 +1527,7 @@ static void ath12k_core_pre_reconfigure_recovery(struct ath12k_base *ab)
 			complete_all(&ar->scan.completed);
 			complete(&ar->scan.on_channel);
 			complete(&ar->peer_assoc_done);
-			complete(&ar->peer_delete_done);
+			ath12k_peer_delete_wait_flush(ar);
 			complete(&ar->install_key_done);
 			complete(&ar->vdev_setup_done);
 			complete(&ar->vdev_delete_done);
diff --git a/drivers/net/wireless/ath/ath12k/core.h b/drivers/net/wireless/ath/ath12k/core.h
index fc5127b5c1a37..1436ff4316e73 100644
--- a/drivers/net/wireless/ath/ath12k/core.h
+++ b/drivers/net/wireless/ath/ath12k/core.h
@@ -665,7 +665,8 @@ struct ath12k {
 
 	/* protects the radio specific data like debug stats, ppdu_stats_info stats,
 	 * vdev_stop_status info, scan data, ath12k_sta info, ath12k_link_vif info,
-	 * channel context data, survey info, test mode data, regd_channel_update_queue.
+	 * channel context data, survey info, test mode data, regd_channel_update_queue,
+	 * peer_delete_waits.
 	 */
 	spinlock_t data_lock;
 
@@ -687,7 +688,7 @@ struct ath12k {
 	u8 radio_idx;
 
 	struct completion peer_assoc_done;
-	struct completion peer_delete_done;
+	struct list_head peer_delete_waits;
 
 	int install_key_status;
 	struct completion install_key_done;
diff --git a/drivers/net/wireless/ath/ath12k/debugfs.c b/drivers/net/wireless/ath/ath12k/debugfs.c
index d17d4a8f1cb7d..d54995b7adb27 100644
--- a/drivers/net/wireless/ath/ath12k/debugfs.c
+++ b/drivers/net/wireless/ath/ath12k/debugfs.c
@@ -1031,6 +1031,7 @@ static ssize_t ath12k_debugfs_dump_device_dp_stats(struct file *file,
 	struct ath12k_device_dp_stats *device_stats = &dp->device_stats;
 	int len = 0, i, j, ret;
 	struct ath12k *ar;
+	u32 center_freq;
 	const int size = 4096;
 	static const char *rxdma_err[HAL_REO_ENTR_RING_RXDMA_ECODE_MAX] = {
 		[HAL_REO_ENTR_RING_RXDMA_ECODE_OVERFLOW_ERR] = "Overflow",
@@ -1082,6 +1083,9 @@ static ssize_t ath12k_debugfs_dump_device_dp_stats(struct file *file,
 	if (!buf)
 		return -ENOMEM;
 
+	len += scnprintf(buf + len, size - len,
+			 "DEVICE DP STATS (timestamp: %lldms):\n\n",
+			 ktime_to_ms(ktime_get()));
 	len += scnprintf(buf + len, size - len, "DEVICE RX STATS:\n\n");
 	len += scnprintf(buf + len, size - len, "err ring pkts: %u\n",
 			 device_stats->err_ring_pkts);
@@ -1161,6 +1165,12 @@ static ssize_t ath12k_debugfs_dump_device_dp_stats(struct file *file,
 	for (i = 0; i < ab->num_radios; i++) {
 		ar = ath12k_mac_get_ar_by_pdev_id(ab, DP_SW2HW_MACID(i));
 		if (ar) {
+			spin_lock_bh(&ar->data_lock);
+			center_freq = ar->rx_channel ? ar->rx_channel->center_freq : 0;
+			spin_unlock_bh(&ar->data_lock);
+			len += scnprintf(buf + len, size - len,
+					 "\nradio%d center_freq: %u\n",
+					 i, center_freq);
 			len += scnprintf(buf + len, size - len,
 					"\nradio%d tx_pending: %u\n", i,
 					atomic_read(&ar->dp.num_tx_pending));
@@ -1173,7 +1183,7 @@ static ssize_t ath12k_debugfs_dump_device_dp_stats(struct file *file,
 	for (i = 0; i < DP_REO_DST_RING_MAX; i++) {
 		len += scnprintf(buf + len, size - len, "Ring%d:", i + 1);
 
-		for (j = 0; j < ATH12K_MAX_DEVICES; j++) {
+		for (j = 0; j < ab->ag->num_devices; j++) {
 			len += scnprintf(buf + len, size - len,
 					"\t%d:%u", j,
 					 device_stats->reo_rx[i][j]);
@@ -1190,7 +1200,7 @@ static ssize_t ath12k_debugfs_dump_device_dp_stats(struct file *file,
 	for (i = 0; i < HAL_WBM_REL_SRC_MODULE_MAX; i++) {
 		len += scnprintf(buf + len, size - len, "%s:", wbm_rel_src[i]);
 
-		for (j = 0; j < ATH12K_MAX_DEVICES; j++) {
+		for (j = 0; j < ab->ag->num_devices; j++) {
 			len += scnprintf(buf + len,
 					 size - len,
 					 "\t%d:%u", j,
diff --git a/drivers/net/wireless/ath/ath12k/dp.c b/drivers/net/wireless/ath/ath12k/dp.c
index af5f11fc1d84a..fbc0788b37a06 100644
--- a/drivers/net/wireless/ath/ath12k/dp.c
+++ b/drivers/net/wireless/ath/ath12k/dp.c
@@ -1097,7 +1097,6 @@ static void ath12k_dp_reoq_lut_cleanup(struct ath12k_base *ab)
 		return;
 
 	if (dp->reoq_lut.vaddr_unaligned) {
-		ath12k_hal_write_reoq_lut_addr(ab, 0);
 		dma_free_coherent(ab->dev, dp->reoq_lut.size,
 				  dp->reoq_lut.vaddr_unaligned,
 				  dp->reoq_lut.paddr_unaligned);
@@ -1105,7 +1104,6 @@ static void ath12k_dp_reoq_lut_cleanup(struct ath12k_base *ab)
 	}
 
 	if (dp->ml_reoq_lut.vaddr_unaligned) {
-		ath12k_hal_write_ml_reoq_lut_addr(ab, 0);
 		dma_free_coherent(ab->dev, dp->ml_reoq_lut.size,
 				  dp->ml_reoq_lut.vaddr_unaligned,
 				  dp->ml_reoq_lut.paddr_unaligned);
@@ -1568,6 +1566,7 @@ static int ath12k_dp_setup(struct ath12k_base *ab)
 	ath12k_dp_rx_free(ab);
 
 fail_cmn_reoq_cleanup:
+	ath12k_dp_reoq_lut_addr_reset(dp);
 	ath12k_dp_reoq_lut_cleanup(ab);
 
 fail_cmn_srng_cleanup:
@@ -1627,3 +1626,14 @@ void ath12k_dp_cmn_hw_group_assign(struct ath12k_dp *dp,
 	dp->device_id = ab->device_id;
 	dp_hw_grp->dp[dp->device_id] = dp;
 }
+
+void ath12k_dp_reoq_lut_addr_reset(struct ath12k_dp *dp)
+{
+	struct ath12k_base *ab = dp->ab;
+
+	if (dp->reoq_lut.vaddr_unaligned)
+		ath12k_hal_write_reoq_lut_addr(ab, 0);
+
+	if (dp->ml_reoq_lut.vaddr_unaligned)
+		ath12k_hal_write_ml_reoq_lut_addr(ab, 0);
+}
diff --git a/drivers/net/wireless/ath/ath12k/dp.h b/drivers/net/wireless/ath/ath12k/dp.h
index f8cfc7bb29dd7..64f79e43341ea 100644
--- a/drivers/net/wireless/ath/ath12k/dp.h
+++ b/drivers/net/wireless/ath/ath12k/dp.h
@@ -205,7 +205,7 @@ struct ath12k_pdev_dp {
 #define DP_REO_CMD_RING_SIZE		256
 #define DP_REO_STATUS_RING_SIZE		2048
 #define DP_RXDMA_BUF_RING_SIZE		4096
-#define DP_RX_MAC_BUF_RING_SIZE		2048
+#define DP_RX_MAC_BUF_RING_SIZE		4096
 #define DP_RXDMA_REFILL_RING_SIZE	2048
 #define DP_RXDMA_ERR_DST_RING_SIZE	1024
 #define DP_RXDMA_MON_STATUS_RING_SIZE	1024
@@ -701,4 +701,5 @@ struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_dp *dp,
 						  u32 cookie);
 struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_dp *dp,
 						  u32 desc_id);
+void ath12k_dp_reoq_lut_addr_reset(struct ath12k_dp *dp);
 #endif
diff --git a/drivers/net/wireless/ath/ath12k/dp_mon.c b/drivers/net/wireless/ath/ath12k/dp_mon.c
index 44c5cff75f169..7d5be77b081f8 100644
--- a/drivers/net/wireless/ath/ath12k/dp_mon.c
+++ b/drivers/net/wireless/ath/ath12k/dp_mon.c
@@ -493,12 +493,8 @@ EXPORT_SYMBOL(ath12k_dp_mon_update_radiotap);
 void ath12k_dp_mon_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev,
 				   struct napi_struct *napi,
 				   struct sk_buff *msdu,
-				   const struct hal_rx_mon_ppdu_info *ppduinfo,
-				   struct ieee80211_rx_status *status,
-				   u8 decap)
+				   struct ieee80211_rx_status *status)
 {
-	struct ath12k_dp *dp = dp_pdev->dp;
-	struct ath12k_base *ab = dp->ab;
 	static const struct ieee80211_radiotap_he known = {
 		.data1 = cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_DATA_MCS_KNOWN |
 				     IEEE80211_RADIOTAP_HE_DATA1_BW_RU_ALLOC_KNOWN),
@@ -506,14 +502,6 @@ void ath12k_dp_mon_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev,
 	};
 	struct ieee80211_rx_status *rx_status;
 	struct ieee80211_radiotap_he *he = NULL;
-	struct ieee80211_sta *pubsta = NULL;
-	struct ath12k_dp_link_peer *peer;
-	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
-	struct hal_rx_desc_data rx_info;
-	bool is_mcbc = rxcb->is_mcbc;
-	bool is_eapol_tkip = rxcb->is_eapol;
-	struct hal_rx_desc *rx_desc = (struct hal_rx_desc *)msdu->data;
-	u8 addr[ETH_ALEN] = {};
 
 	status->link_valid = 0;
 
@@ -524,64 +512,10 @@ void ath12k_dp_mon_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev,
 		status->flag |= RX_FLAG_RADIOTAP_HE;
 	}
 
-	ath12k_dp_extract_rx_desc_data(dp->hal, &rx_info, rx_desc, rx_desc);
-
-	rcu_read_lock();
-	spin_lock_bh(&dp->dp_lock);
-	peer = ath12k_dp_rx_h_find_link_peer(dp_pdev, msdu, &rx_info);
-	if (peer && peer->sta) {
-		pubsta = peer->sta;
-		memcpy(addr, peer->addr, ETH_ALEN);
-		if (pubsta->valid_links) {
-			status->link_valid = 1;
-			status->link_id = peer->link_id;
-		}
-	}
-
-	spin_unlock_bh(&dp->dp_lock);
-	rcu_read_unlock();
-
-	ath12k_dbg(ab, ATH12K_DBG_DATA,
-		   "rx skb %p len %u peer %pM %u %s %s%s%s%s%s%s%s%s %srate_idx %u vht_nss %u freq %u band %u flag 0x%x fcs-err %i mic-err %i amsdu-more %i\n",
-		   msdu,
-		   msdu->len,
-		   addr,
-		   rxcb->tid,
-		   (is_mcbc) ? "mcast" : "ucast",
-		   (status->encoding == RX_ENC_LEGACY) ? "legacy" : "",
-		   (status->encoding == RX_ENC_HT) ? "ht" : "",
-		   (status->encoding == RX_ENC_VHT) ? "vht" : "",
-		   (status->encoding == RX_ENC_HE) ? "he" : "",
-		   (status->bw == RATE_INFO_BW_40) ? "40" : "",
-		   (status->bw == RATE_INFO_BW_80) ? "80" : "",
-		   (status->bw == RATE_INFO_BW_160) ? "160" : "",
-		   (status->bw == RATE_INFO_BW_320) ? "320" : "",
-		   status->enc_flags & RX_ENC_FLAG_SHORT_GI ? "sgi " : "",
-		   status->rate_idx,
-		   status->nss,
-		   status->freq,
-		   status->band, status->flag,
-		   !!(status->flag & RX_FLAG_FAILED_FCS_CRC),
-		   !!(status->flag & RX_FLAG_MMIC_ERROR),
-		   !!(status->flag & RX_FLAG_AMSDU_MORE));
-
-	ath12k_dbg_dump(ab, ATH12K_DBG_DP_RX, NULL, "dp rx msdu: ",
-			msdu->data, msdu->len);
 	rx_status = IEEE80211_SKB_RXCB(msdu);
 	*rx_status = *status;
 
-	/* TODO: trace rx packet */
-
-	/* PN for multicast packets are not validate in HW,
-	 * so skip 802.3 rx path
-	 * Also, fast_rx expects the STA to be authorized, hence
-	 * eapol packets are sent in slow path.
-	 */
-	if (decap == DP_RX_DECAP_TYPE_ETHERNET2_DIX && !is_eapol_tkip &&
-	    !(is_mcbc && rx_status->flag & RX_FLAG_DECRYPTED))
-		rx_status->flag |= RX_FLAG_8023;
-
-	ieee80211_rx_napi(ath12k_pdev_dp_to_hw(dp_pdev), pubsta, msdu, napi);
+	ieee80211_rx_napi(ath12k_pdev_dp_to_hw(dp_pdev), NULL, msdu, napi);
 }
 EXPORT_SYMBOL(ath12k_dp_mon_rx_deliver_msdu);
 
diff --git a/drivers/net/wireless/ath/ath12k/dp_mon.h b/drivers/net/wireless/ath/ath12k/dp_mon.h
index 167028d27513b..162cdcaa57a77 100644
--- a/drivers/net/wireless/ath/ath12k/dp_mon.h
+++ b/drivers/net/wireless/ath/ath12k/dp_mon.h
@@ -112,9 +112,7 @@ void ath12k_dp_mon_update_radiotap(struct ath12k_pdev_dp *dp_pdev,
 void ath12k_dp_mon_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev,
 				   struct napi_struct *napi,
 				   struct sk_buff *msdu,
-				   const struct hal_rx_mon_ppdu_info *ppduinfo,
-				   struct ieee80211_rx_status *status,
-				   u8 decap);
+				   struct ieee80211_rx_status *status);
 struct sk_buff *
 ath12k_dp_mon_rx_merg_msdus(struct ath12k_pdev_dp *dp_pdev,
 			    struct dp_mon_mpdu *mon_mpdu,
diff --git a/drivers/net/wireless/ath/ath12k/dp_peer.c b/drivers/net/wireless/ath/ath12k/dp_peer.c
index 47d009a0d61fe..a12073afc307c 100644
--- a/drivers/net/wireless/ath/ath12k/dp_peer.c
+++ b/drivers/net/wireless/ath/ath12k/dp_peer.c
@@ -274,11 +274,14 @@ int ath12k_dp_link_peer_rhash_tbl_init(struct ath12k_dp *dp)
 
 void ath12k_dp_link_peer_rhash_tbl_destroy(struct ath12k_dp *dp)
 {
-	mutex_lock(&dp->link_peer_rhash_tbl_lock);
+	guard(mutex)(&dp->link_peer_rhash_tbl_lock);
+
+	if (!dp->rhead_peer_addr)
+		return;
+
 	rhashtable_destroy(dp->rhead_peer_addr);
 	kfree(dp->rhead_peer_addr);
 	dp->rhead_peer_addr = NULL;
-	mutex_unlock(&dp->link_peer_rhash_tbl_lock);
 }
 
 static int ath12k_dp_link_peer_rhash_insert(struct ath12k_dp *dp,
@@ -570,6 +573,8 @@ int ath12k_dp_link_peer_assign(struct ath12k_dp *dp, struct ath12k_dp_hw *dp_hw,
 	peerid_index = ath12k_dp_peer_get_peerid_index(dp, peer->peer_id);
 
 	rcu_assign_pointer(dp_peer->link_peers[peer->link_id], peer);
+	WRITE_ONCE(dp_peer->link_peers_map,
+		   READ_ONCE(dp_peer->link_peers_map) | BIT(peer->link_id));
 
 	rcu_assign_pointer(dp_hw->dp_peers[peerid_index], dp_peer);
 
@@ -632,6 +637,8 @@ void ath12k_dp_link_peer_unassign(struct ath12k_dp *dp, struct ath12k_dp_hw *dp_
 	peerid_index = ath12k_dp_peer_get_peerid_index(dp, peer->peer_id);
 
 	rcu_assign_pointer(dp_peer->link_peers[peer->link_id], NULL);
+	WRITE_ONCE(dp_peer->link_peers_map,
+		   READ_ONCE(dp_peer->link_peers_map) & ~BIT(peer->link_id));
 
 	rcu_assign_pointer(dp_hw->dp_peers[peerid_index], NULL);
 
diff --git a/drivers/net/wireless/ath/ath12k/dp_peer.h b/drivers/net/wireless/ath/ath12k/dp_peer.h
index f5067e66f1e18..7c9709bf717b4 100644
--- a/drivers/net/wireless/ath/ath12k/dp_peer.h
+++ b/drivers/net/wireless/ath/ath12k/dp_peer.h
@@ -140,6 +140,7 @@ struct ath12k_dp_peer {
 
 	/* Info used in MMIC verification of * RX fragments */
 	struct ieee80211_key_conf *keys[WMI_MAX_KEY_INDEX + 1];
+	unsigned long link_peers_map;
 	struct ath12k_dp_link_peer __rcu *link_peers[ATH12K_NUM_MAX_LINKS];
 	struct ath12k_reoq_buf reoq_bufs[IEEE80211_NUM_TIDS + 1];
 	struct ath12k_dp_rx_tid rx_tid[IEEE80211_NUM_TIDS + 1];
diff --git a/drivers/net/wireless/ath/ath12k/dp_rx.c b/drivers/net/wireless/ath/ath12k/dp_rx.c
index 06e74124e57eb..8fa0e90b45316 100644
--- a/drivers/net/wireless/ath/ath12k/dp_rx.c
+++ b/drivers/net/wireless/ath/ath12k/dp_rx.c
@@ -1384,10 +1384,9 @@ void ath12k_dp_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev, struct napi_struc
 
 	pubsta = peer ? peer->sta : NULL;
 
-	if (pubsta && pubsta->valid_links) {
-		status->link_valid = 1;
-		status->link_id = peer->hw_links[rxcb->hw_link_id];
-	}
+	status->link_valid = 0;
+	if (pubsta && pubsta->valid_links)
+		ath12k_hw_set_rx_link_id(dp->hw_params, peer, rxcb, status);
 
 	ath12k_dbg(dp->ab, ATH12K_DBG_DATA,
 		   "rx skb %p len %u peer %pM %d %s sn %u %s%s%s%s%s%s%s%s%s%s rate_idx %u vht_nss %u freq %u band %u flag 0x%x fcs-err %i mic-err %i amsdu-more %i\n",
diff --git a/drivers/net/wireless/ath/ath12k/hal.c b/drivers/net/wireless/ath/ath12k/hal.c
index a164563fff289..c0c3d2f047ef0 100644
--- a/drivers/net/wireless/ath/ath12k/hal.c
+++ b/drivers/net/wireless/ath/ath12k/hal.c
@@ -828,8 +828,8 @@ void *ath12k_hal_encode_tlv64_hdr(void *tlv, u64 tag, u64 len)
 {
 	struct hal_tlv_64_hdr *tlv64 = tlv;
 
-	tlv64->tl = le64_encode_bits(tag, HAL_TLV_HDR_TAG) |
-		    le64_encode_bits(len, HAL_TLV_HDR_LEN);
+	tlv64->tl = le64_encode_bits(tag, HAL_TLV_64_HDR_TAG) |
+		    le64_encode_bits(len, HAL_TLV_64_HDR_LEN);
 
 	return tlv64->value;
 }
@@ -846,26 +846,44 @@ void *ath12k_hal_encode_tlv32_hdr(void *tlv, u64 tag, u64 len)
 }
 EXPORT_SYMBOL(ath12k_hal_encode_tlv32_hdr);
 
-u16 ath12k_hal_decode_tlv64_hdr(void *tlv, void **desc)
+void *ath12k_hal_decode_tlv64_hdr(void *tlv, u16 *tag, u16 *len, u16 *usrid)
 {
 	struct hal_tlv_64_hdr *tlv64 = tlv;
-	u16 tag;
 
-	tag = le64_get_bits(tlv64->tl, HAL_SRNG_TLV_HDR_TAG);
-	*desc = tlv64->value;
+	if (tag)
+		*tag = le64_get_bits(tlv64->tl, HAL_TLV_64_HDR_TAG);
+	if (len)
+		*len = le64_get_bits(tlv64->tl, HAL_TLV_64_HDR_LEN);
+	if (usrid)
+		*usrid = le64_get_bits(tlv64->tl, HAL_TLV_64_USR_ID);
 
-	return tag;
+	return tlv64->value;
 }
 EXPORT_SYMBOL(ath12k_hal_decode_tlv64_hdr);
 
-u16 ath12k_hal_decode_tlv32_hdr(void *tlv, void **desc)
+void *ath12k_hal_decode_tlv32_hdr(void *tlv, u16 *tag, u16 *len, u16 *usrid)
 {
 	struct hal_tlv_hdr *tlv32 = tlv;
-	u16 tag;
 
-	tag = le32_get_bits(tlv32->tl, HAL_SRNG_TLV_HDR_TAG);
-	*desc = tlv32->value;
+	if (tag)
+		*tag = le32_get_bits(tlv32->tl, HAL_TLV_HDR_TAG);
+	if (len)
+		*len = le32_get_bits(tlv32->tl, HAL_TLV_HDR_LEN);
+	if (usrid)
+		*usrid = le32_get_bits(tlv32->tl, HAL_TLV_USR_ID);
 
-	return tag;
+	return tlv32->value;
 }
 EXPORT_SYMBOL(ath12k_hal_decode_tlv32_hdr);
+
+u32 ath12k_hal_get_tlv64_hdr_align(void)
+{
+	return HAL_TLV_64_ALIGN;
+}
+EXPORT_SYMBOL(ath12k_hal_get_tlv64_hdr_align);
+
+u32 ath12k_hal_get_tlv32_hdr_align(void)
+{
+	return HAL_TLV_ALIGN;
+}
+EXPORT_SYMBOL(ath12k_hal_get_tlv32_hdr_align);
diff --git a/drivers/net/wireless/ath/ath12k/hal.h b/drivers/net/wireless/ath/ath12k/hal.h
index 21c551d8b2481..3a874db7968e7 100644
--- a/drivers/net/wireless/ath/ath12k/hal.h
+++ b/drivers/net/wireless/ath/ath12k/hal.h
@@ -1024,7 +1024,7 @@ enum hal_wbm_rel_desc_type {
 
 /* Interrupt mitigation - timer threshold in us */
 #define HAL_SRNG_INT_TIMER_THRESHOLD_TX 1000
-#define HAL_SRNG_INT_TIMER_THRESHOLD_RX 500
+#define HAL_SRNG_INT_TIMER_THRESHOLD_RX 200
 #define HAL_SRNG_INT_TIMER_THRESHOLD_OTHER 256
 
 enum hal_srng_mac_type {
@@ -1441,10 +1441,12 @@ struct hal_ops {
 					 u8 *rbm, u32 *msdu_cnt);
 	void *(*reo_cmd_enc_tlv_hdr)(void *tlv, u64 tag, u64 len);
 	u16 (*reo_status_dec_tlv_hdr)(void *tlv, void **desc);
+	void *(*mon_rx_status_dec_tlv_hdr)(void *tlv, u16 *tag, u16 *len, u16 *usrid);
+	u32 (*get_tlv_hdr_align)(void);
 };
 
 #define HAL_TLV_HDR_TAG		GENMASK(9, 1)
-#define HAL_TLV_HDR_LEN		GENMASK(25, 10)
+#define HAL_TLV_HDR_LEN		GENMASK(21, 10)
 #define HAL_TLV_USR_ID		GENMASK(31, 26)
 
 #define HAL_TLV_ALIGN	4
@@ -1464,9 +1466,6 @@ struct hal_tlv_64_hdr {
 	u8 value[];
 } __packed;
 
-#define HAL_SRNG_TLV_HDR_TAG		GENMASK(9, 1)
-#define HAL_SRNG_TLV_HDR_LEN		GENMASK(25, 10)
-
 dma_addr_t ath12k_hal_srng_get_tp_addr(struct ath12k_base *ab,
 				       struct hal_srng *srng);
 dma_addr_t ath12k_hal_srng_get_hp_addr(struct ath12k_base *ab,
@@ -1556,6 +1555,8 @@ void ath12k_hal_rx_reo_ent_buf_paddr_get(struct ath12k_hal *hal, void *rx_desc,
 					 u8 *rbm, u32 *msdu_cnt);
 void *ath12k_hal_encode_tlv64_hdr(void *tlv, u64 tag, u64 len);
 void *ath12k_hal_encode_tlv32_hdr(void *tlv, u64 tag, u64 len);
-u16 ath12k_hal_decode_tlv64_hdr(void *tlv, void **desc);
-u16 ath12k_hal_decode_tlv32_hdr(void *tlv, void **desc);
+void *ath12k_hal_decode_tlv64_hdr(void *tlv, u16 *tag, u16 *len, u16 *usrid);
+void *ath12k_hal_decode_tlv32_hdr(void *tlv, u16 *tag, u16 *len, u16 *usrid);
+u32 ath12k_hal_get_tlv64_hdr_align(void);
+u32 ath12k_hal_get_tlv32_hdr_align(void);
 #endif
diff --git a/drivers/net/wireless/ath/ath12k/hw.h b/drivers/net/wireless/ath/ath12k/hw.h
index d135b29363788..86fb8b7196135 100644
--- a/drivers/net/wireless/ath/ath12k/hw.h
+++ b/drivers/net/wireless/ath/ath12k/hw.h
@@ -13,6 +13,10 @@
 #include "wmi.h"
 #include "hal.h"
 
+struct ath12k_dp_peer;
+struct ath12k_skb_rxcb;
+struct ieee80211_rx_status;
+
 /* Target configuration defines */
 
 /* Num VDEVS per radio */
@@ -243,6 +247,9 @@ struct ath12k_hw_ops {
 	bool (*dp_srng_is_tx_comp_ring)(int ring_num);
 	bool (*is_frame_link_agnostic)(struct ath12k_link_vif *arvif,
 				       struct ieee80211_mgmt *mgmt);
+	void (*set_rx_link_id)(struct ath12k_dp_peer *dp_peer,
+			       struct ath12k_skb_rxcb *rxcb,
+			       struct ieee80211_rx_status *status);
 };
 
 static inline
@@ -273,6 +280,15 @@ static inline int ath12k_hw_mac_id_to_srng_id(const struct ath12k_hw_params *hw,
 	return 0;
 }
 
+static inline void ath12k_hw_set_rx_link_id(const struct ath12k_hw_params *hw,
+					    struct ath12k_dp_peer *dp_peer,
+					    struct ath12k_skb_rxcb *rxcb,
+					    struct ieee80211_rx_status *status)
+{
+	if (hw->hw_ops->set_rx_link_id)
+		hw->hw_ops->set_rx_link_id(dp_peer, rxcb, status);
+}
+
 struct ath12k_fw_ie {
 	__le32 id;
 	__le32 len;
diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c
index af354bef5c0d7..b9add70097943 100644
--- a/drivers/net/wireless/ath/ath12k/mac.c
+++ b/drivers/net/wireless/ath/ath12k/mac.c
@@ -1238,9 +1238,13 @@ void ath12k_mac_peer_cleanup_all(struct ath12k *ar)
 		/* cleanup dp peer */
 		spin_lock_bh(&dp_hw->peer_lock);
 		dp_peer = peer->dp_peer;
-		peerid_index = ath12k_dp_peer_get_peerid_index(dp, peer->peer_id);
-		rcu_assign_pointer(dp_peer->link_peers[peer->link_id], NULL);
-		rcu_assign_pointer(dp_hw->dp_peers[peerid_index], NULL);
+		if (dp_peer) {
+			peerid_index = ath12k_dp_peer_get_peerid_index(dp, peer->peer_id);
+			rcu_assign_pointer(dp_peer->link_peers[peer->link_id], NULL);
+			WRITE_ONCE(dp_peer->link_peers_map,
+				   READ_ONCE(dp_peer->link_peers_map) & ~BIT(peer->link_id));
+			rcu_assign_pointer(dp_hw->dp_peers[peerid_index], NULL);
+		}
 		spin_unlock_bh(&dp_hw->peer_lock);
 
 		ath12k_dp_link_peer_rhash_delete(dp, peer);
@@ -10117,16 +10121,16 @@ static void ath12k_mac_update_vif_offload(struct ath12k_link_vif *arvif)
 	if (vif->type != NL80211_IFTYPE_STATION &&
 	    vif->type != NL80211_IFTYPE_AP)
 		vif->offload_flags &= ~(IEEE80211_OFFLOAD_ENCAP_ENABLED |
-					IEEE80211_OFFLOAD_DECAP_ENABLED);
+					IEEE80211_OFFLOAD_DECAP_ENABLED |
+					IEEE80211_OFFLOAD_ENCAP_MCAST |
+					IEEE80211_OFFLOAD_ENCAP_4ADDR);
 
-	if (vif->offload_flags & IEEE80211_OFFLOAD_ENCAP_ENABLED) {
+	if (vif->offload_flags & IEEE80211_OFFLOAD_ENCAP_ENABLED)
 		ahvif->dp_vif.tx_encap_type = ATH12K_HW_TXRX_ETHERNET;
-		vif->offload_flags |= IEEE80211_OFFLOAD_ENCAP_4ADDR;
-	} else if (test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags)) {
+	else if (test_bit(ATH12K_FLAG_RAW_MODE, &ab->dev_flags))
 		ahvif->dp_vif.tx_encap_type = ATH12K_HW_TXRX_RAW;
-	} else {
+	else
 		ahvif->dp_vif.tx_encap_type = ATH12K_HW_TXRX_NATIVE_WIFI;
-	}
 
 	ret = ath12k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id,
 					    param_id, ahvif->dp_vif.tx_encap_type);
@@ -10136,6 +10140,10 @@ static void ath12k_mac_update_vif_offload(struct ath12k_link_vif *arvif)
 		vif->offload_flags &= ~IEEE80211_OFFLOAD_ENCAP_ENABLED;
 	}
 
+	if (vif->offload_flags & IEEE80211_OFFLOAD_ENCAP_ENABLED)
+		vif->offload_flags |= (IEEE80211_OFFLOAD_ENCAP_MCAST |
+				       IEEE80211_OFFLOAD_ENCAP_4ADDR);
+
 	param_id = WMI_VDEV_PARAM_RX_DECAP_TYPE;
 	if (vif->offload_flags & IEEE80211_OFFLOAD_DECAP_ENABLED)
 		param_value = ATH12K_HW_TXRX_ETHERNET;
@@ -10564,22 +10572,8 @@ int ath12k_mac_vdev_create(struct ath12k *ar, struct ath12k_link_vif *arvif)
 
 err_peer_del:
 	if (ahvif->vdev_type == WMI_VDEV_TYPE_AP) {
-		reinit_completion(&ar->peer_delete_done);
-
-		ret = ath12k_wmi_send_peer_delete_cmd(ar, arvif->bssid,
-						      arvif->vdev_id);
-		if (ret) {
-			ath12k_warn(ar->ab, "failed to delete peer vdev_id %d addr %pM\n",
-				    arvif->vdev_id, arvif->bssid);
-			goto err_dp_peer_del;
-		}
-
-		ret = ath12k_wait_for_peer_delete_done(ar, arvif->vdev_id,
-						       arvif->bssid);
-		if (ret)
-			goto err_dp_peer_del;
-
-		ar->num_peers--;
+		/* ignore return value: propagate the original error */
+		ath12k_peer_delete(ar, arvif->vdev_id, arvif->bssid);
 	}
 
 err_dp_peer_del:
@@ -11253,6 +11247,8 @@ ath12k_mac_mlo_get_vdev_args(struct ath12k_link_vif *arvif,
 
 	ml_arg->assoc_link = arvif->is_sta_assoc_link;
 
+	ml_arg->ieee_link_id = arvif->link_id;
+
 	partner_info = ml_arg->partner_info;
 
 	links = ahvif->links_map;
@@ -11276,6 +11272,7 @@ ath12k_mac_mlo_get_vdev_args(struct ath12k_link_vif *arvif,
 
 		partner_info->vdev_id = arvif_p->vdev_id;
 		partner_info->hw_link_id = arvif_p->ar->pdev->hw_link_id;
+		partner_info->ieee_link_id = arvif_p->link_id;
 		ether_addr_copy(partner_info->addr, link_conf->addr);
 		ml_arg->num_partner_links++;
 		partner_info++;
@@ -14871,12 +14868,6 @@ static int ath12k_mac_hw_register(struct ath12k_hw *ah)
 
 	wiphy->features |= NL80211_FEATURE_TX_POWER_INSERTION;
 
-	/* MLO is not yet supported so disable Wireless Extensions for now
-	 * to make sure ath12k users don't use it. This flag can be removed
-	 * once WIPHY_FLAG_SUPPORTS_MLO is enabled.
-	 */
-	wiphy->flags |= WIPHY_FLAG_DISABLE_WEXT;
-
 	/* Copy over MLO related capabilities received from
 	 * WMI_SERVICE_READY_EXT2_EVENT if single_chip_mlo_supp is set.
 	 */
@@ -15054,11 +15045,11 @@ static void ath12k_mac_setup(struct ath12k *ar)
 	spin_lock_init(&ar->dp.ppdu_list_lock);
 	INIT_LIST_HEAD(&ar->arvifs);
 	INIT_LIST_HEAD(&ar->dp.ppdu_stats_info);
+	INIT_LIST_HEAD(&ar->peer_delete_waits);
 
 	init_completion(&ar->vdev_setup_done);
 	init_completion(&ar->vdev_delete_done);
 	init_completion(&ar->peer_assoc_done);
-	init_completion(&ar->peer_delete_done);
 	init_completion(&ar->install_key_done);
 	init_completion(&ar->bss_survey_done);
 	init_completion(&ar->scan.started);
diff --git a/drivers/net/wireless/ath/ath12k/pci.c b/drivers/net/wireless/ath/ath12k/pci.c
index d9a22d6afbb02..ad74140e0fa50 100644
--- a/drivers/net/wireless/ath/ath12k/pci.c
+++ b/drivers/net/wireless/ath/ath12k/pci.c
@@ -5,6 +5,7 @@
  */
 
 #include <linux/module.h>
+#include <linux/interrupt.h>
 #include <linux/msi.h>
 #include <linux/pci.h>
 #include <linux/time.h>
@@ -188,6 +189,8 @@ static void ath12k_pci_soc_global_reset(struct ath12k_base *ab)
 	val |= PCIE_SOC_GLOBAL_RESET_V;
 
 	ath12k_pci_write32(ab, PCIE_SOC_GLOBAL_RESET, val);
+	/* Flush the posted write to the device */
+	ath12k_pci_read32(ab, PCIE_SOC_GLOBAL_RESET);
 
 	/* TODO: exact time to sleep is uncertain */
 	delay = 10;
@@ -197,6 +200,8 @@ static void ath12k_pci_soc_global_reset(struct ath12k_base *ab)
 	val &= ~PCIE_SOC_GLOBAL_RESET_V;
 
 	ath12k_pci_write32(ab, PCIE_SOC_GLOBAL_RESET, val);
+	/* Flush the posted write to the device */
+	ath12k_pci_read32(ab, PCIE_SOC_GLOBAL_RESET);
 
 	mdelay(delay);
 
@@ -537,6 +542,8 @@ static int ath12k_pci_ext_irq_config(struct ath12k_base *ab)
 	int i, j, n, ret, num_vectors = 0;
 	u32 user_base_data = 0, base_vector = 0, base_idx;
 	struct ath12k_ext_irq_grp *irq_grp;
+	bool threaded_napi = false;
+	int irq;
 
 	base_idx = ATH12K_PCI_IRQ_CE0_OFFSET + CE_COUNT_MAX;
 	ret = ath12k_pci_get_user_msi_assignment(ab, "DP",
@@ -546,6 +553,10 @@ static int ath12k_pci_ext_irq_config(struct ath12k_base *ab)
 	if (ret < 0)
 		return ret;
 
+	irq = ath12k_pci_get_msi_irq(ab->dev, base_vector);
+	if (irq >= 0)
+		threaded_napi = !irq_can_set_affinity(irq);
+
 	for (i = 0; i < ATH12K_EXT_IRQ_GRP_NUM_MAX; i++) {
 		irq_grp = &ab->ext_irq_grp[i];
 		u32 num_irq = 0;
@@ -560,6 +571,8 @@ static int ath12k_pci_ext_irq_config(struct ath12k_base *ab)
 
 		netif_napi_add(irq_grp->napi_ndev, &irq_grp->napi,
 			       ath12k_pci_ext_grp_napi_poll);
+		if (threaded_napi)
+			netif_threaded_enable(irq_grp->napi_ndev);
 
 		if (ab->hw_params->ring_mask->tx[i] ||
 		    ab->hw_params->ring_mask->rx[i] ||
@@ -578,7 +591,8 @@ static int ath12k_pci_ext_irq_config(struct ath12k_base *ab)
 		for (j = 0; j < irq_grp->num_irq; j++) {
 			int irq_idx = irq_grp->irqs[j];
 			int vector = (i % num_vectors) + base_vector;
-			int irq = ath12k_pci_get_msi_irq(ab->dev, vector);
+
+			irq = ath12k_pci_get_msi_irq(ab->dev, vector);
 
 			ab->irq_num[irq_idx] = irq;
 
diff --git a/drivers/net/wireless/ath/ath12k/peer.c b/drivers/net/wireless/ath/ath12k/peer.c
index c222bdaa333c5..b2fea15b97c83 100644
--- a/drivers/net/wireless/ath/ath12k/peer.c
+++ b/drivers/net/wireless/ath/ath12k/peer.c
@@ -9,6 +9,55 @@
 #include "debug.h"
 #include "debugfs.h"
 
+static void ath12k_peer_delete_wait_register(struct ath12k *ar,
+					     struct ath12k_peer_delete_wait *wait,
+					     u32 vdev_id, const u8 *addr)
+{
+	wait->vdev_id = vdev_id;
+	ether_addr_copy(wait->addr, addr);
+	init_completion(&wait->done);
+
+	spin_lock_bh(&ar->data_lock);
+	list_add(&wait->list, &ar->peer_delete_waits);
+	spin_unlock_bh(&ar->data_lock);
+}
+
+static void ath12k_peer_delete_wait_unregister(struct ath12k *ar,
+					       struct ath12k_peer_delete_wait *wait)
+{
+	spin_lock_bh(&ar->data_lock);
+	list_del(&wait->list);
+	spin_unlock_bh(&ar->data_lock);
+}
+
+void ath12k_peer_delete_resp_signal(struct ath12k *ar, u32 vdev_id, const u8 *addr)
+{
+	struct ath12k_peer_delete_wait *wait;
+
+	guard(spinlock_bh)(&ar->data_lock);
+
+	list_for_each_entry(wait, &ar->peer_delete_waits, list) {
+		if (wait->vdev_id == vdev_id &&
+		    ether_addr_equal(wait->addr, addr)) {
+			complete(&wait->done);
+			return;
+		}
+	}
+
+	ath12k_warn(ar->ab, "failed to find link peer with vdev id %u addr %pM\n",
+		    vdev_id, addr);
+}
+
+void ath12k_peer_delete_wait_flush(struct ath12k *ar)
+{
+	struct ath12k_peer_delete_wait *wait;
+
+	spin_lock_bh(&ar->data_lock);
+	list_for_each_entry(wait, &ar->peer_delete_waits, list)
+		complete(&wait->done);
+	spin_unlock_bh(&ar->data_lock);
+}
+
 static int ath12k_wait_for_dp_link_peer_common(struct ath12k_base *ab, int vdev_id,
 					       const u8 *addr, bool expect_mapped)
 {
@@ -62,20 +111,19 @@ static int ath12k_wait_for_peer_deleted(struct ath12k *ar, int vdev_id, const u8
 	return ath12k_wait_for_dp_link_peer_common(ar->ab, vdev_id, addr, false);
 }
 
-int ath12k_wait_for_peer_delete_done(struct ath12k *ar, u32 vdev_id,
-				     const u8 *addr)
+int ath12k_wait_for_peer_delete_done(struct ath12k *ar,
+				     struct ath12k_peer_delete_wait *wait)
 {
-	int ret;
 	unsigned long time_left;
+	int ret;
 
-	ret = ath12k_wait_for_peer_deleted(ar, vdev_id, addr);
+	ret = ath12k_wait_for_peer_deleted(ar, wait->vdev_id, wait->addr);
 	if (ret) {
-		ath12k_warn(ar->ab, "failed wait for peer deleted");
+		ath12k_warn(ar->ab, "failed wait for peer deleted\n");
 		return ret;
 	}
 
-	time_left = wait_for_completion_timeout(&ar->peer_delete_done,
-						3 * HZ);
+	time_left = wait_for_completion_timeout(&wait->done, 3 * HZ);
 	if (time_left == 0) {
 		ath12k_warn(ar->ab, "Timeout in receiving peer delete response\n");
 		return -ETIMEDOUT;
@@ -91,8 +139,6 @@ static int ath12k_peer_delete_send(struct ath12k *ar, u32 vdev_id, const u8 *add
 
 	lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy);
 
-	reinit_completion(&ar->peer_delete_done);
-
 	ret = ath12k_wmi_send_peer_delete_cmd(ar, addr, vdev_id);
 	if (ret) {
 		ath12k_warn(ab,
@@ -106,6 +152,7 @@ static int ath12k_peer_delete_send(struct ath12k *ar, u32 vdev_id, const u8 *add
 
 int ath12k_peer_delete(struct ath12k *ar, u32 vdev_id, u8 *addr)
 {
+	struct ath12k_peer_delete_wait wait;
 	int ret;
 
 	lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy);
@@ -114,17 +161,25 @@ int ath12k_peer_delete(struct ath12k *ar, u32 vdev_id, u8 *addr)
 				     &(ath12k_ar_to_ah(ar)->dp_hw), vdev_id,
 				     addr, ar->hw_link_id);
 
+	/*
+	 * Register the stack waiter before sending so the resp_event for
+	 * this peer cannot arrive while no waiter is queued.
+	 */
+	ath12k_peer_delete_wait_register(ar, &wait, vdev_id, addr);
+
 	ret = ath12k_peer_delete_send(ar, vdev_id, addr);
 	if (ret)
-		return ret;
+		goto out;
 
-	ret = ath12k_wait_for_peer_delete_done(ar, vdev_id, addr);
+	ret = ath12k_wait_for_peer_delete_done(ar, &wait);
 	if (ret)
-		return ret;
+		goto out;
 
 	ar->num_peers--;
 
-	return 0;
+out:
+	ath12k_peer_delete_wait_unregister(ar, &wait);
+	return ret;
 }
 
 static int ath12k_wait_for_peer_created(struct ath12k *ar, int vdev_id, const u8 *addr)
@@ -184,22 +239,26 @@ int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif,
 	peer = ath12k_dp_link_peer_find_by_vdev_and_addr(dp, arg->vdev_id,
 							 arg->peer_addr);
 	if (!peer) {
+		struct ath12k_peer_delete_wait wait;
+
 		spin_unlock_bh(&dp->dp_lock);
 		ath12k_warn(ar->ab, "failed to find peer %pM on vdev %i after creation\n",
 			    arg->peer_addr, arg->vdev_id);
 
-		reinit_completion(&ar->peer_delete_done);
+		ath12k_peer_delete_wait_register(ar, &wait, arg->vdev_id,
+						 arg->peer_addr);
 
 		ret = ath12k_wmi_send_peer_delete_cmd(ar, arg->peer_addr,
 						      arg->vdev_id);
 		if (ret) {
 			ath12k_warn(ar->ab, "failed to delete peer vdev_id %d addr %pM\n",
 				    arg->vdev_id, arg->peer_addr);
+			ath12k_peer_delete_wait_unregister(ar, &wait);
 			return ret;
 		}
 
-		ret = ath12k_wait_for_peer_delete_done(ar, arg->vdev_id,
-						       arg->peer_addr);
+		ret = ath12k_wait_for_peer_delete_done(ar, &wait);
+		ath12k_peer_delete_wait_unregister(ar, &wait);
 		if (ret)
 			return ret;
 
@@ -283,13 +342,14 @@ u16 ath12k_peer_ml_alloc(struct ath12k_hw *ah)
 
 int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_sta *ahsta)
 {
+	DECLARE_BITMAP(registered, IEEE80211_MLD_MAX_NUM_LINKS);
 	struct ieee80211_sta *sta = ath12k_ahsta_to_sta(ahsta);
 	struct ath12k_hw *ah = ahvif->ah;
 	struct ath12k_link_vif *arvif;
 	struct ath12k_link_sta *arsta;
+	int ret, err_ret = 0;
 	unsigned long links;
 	struct ath12k *ar;
-	int ret, err_ret = 0;
 	u8 link_id;
 
 	lockdep_assert_wiphy(ah->hw->wiphy);
@@ -297,8 +357,19 @@ int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_st
 	if (!sta->mlo)
 		return -EINVAL;
 
-	/* FW expects delete of all link peers at once before waiting for reception
-	 * of peer unmap or delete responses
+	struct ath12k_peer_delete_wait *waits __free(kfree) =
+				kzalloc_objs(*waits, IEEE80211_MLD_MAX_NUM_LINKS);
+	if (!waits)
+		return -ENOMEM;
+
+	bitmap_zero(registered, IEEE80211_MLD_MAX_NUM_LINKS);
+
+	/*
+	 * Firmware expects delete of all link peers at once before waiting
+	 * for reception of peer unmap or delete responses. Phase 1 registers
+	 * a per-link stack waiter and sends WMI peer delete for every
+	 * link; the resp_event handler matches each response to its
+	 * (vdev_id, addr) waiter on ar->peer_delete_waits.
 	 */
 	links = ahsta->links_map;
 	for_each_set_bit(link_id, &links, IEEE80211_MLD_MAX_NUM_LINKS) {
@@ -318,29 +389,36 @@ int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_st
 					     arvif->vdev_id, arsta->addr,
 					     ar->hw_link_id);
 
+		ath12k_peer_delete_wait_register(ar, &waits[link_id],
+						 arvif->vdev_id, arsta->addr);
+
 		ret = ath12k_peer_delete_send(ar, arvif->vdev_id, arsta->addr);
 		if (ret) {
 			ath12k_warn(ar->ab,
 				    "failed to delete peer vdev_id %d addr %pM ret %d\n",
 				    arvif->vdev_id, arsta->addr, ret);
 			err_ret = ret;
+			ath12k_peer_delete_wait_unregister(ar, &waits[link_id]);
 			continue;
 		}
+
+		set_bit(link_id, registered);
 	}
 
-	/* Ensure all link peers are deleted and unmapped */
+	/*
+	 * Phase 2: wait for unmap + delete_resp on each registered link
+	 * and tear down the waiter.
+	 */
 	links = ahsta->links_map;
 	for_each_set_bit(link_id, &links, IEEE80211_MLD_MAX_NUM_LINKS) {
-		arvif = wiphy_dereference(ah->hw->wiphy, ahvif->link[link_id]);
-		arsta = wiphy_dereference(ah->hw->wiphy, ahsta->link[link_id]);
-		if (!arvif || !arsta)
+		if (!test_bit(link_id, registered))
 			continue;
 
+		arvif = wiphy_dereference(ah->hw->wiphy, ahvif->link[link_id]);
 		ar = arvif->ar;
-		if (!ar)
-			continue;
 
-		ret = ath12k_wait_for_peer_delete_done(ar, arvif->vdev_id, arsta->addr);
+		ret = ath12k_wait_for_peer_delete_done(ar, &waits[link_id]);
+		ath12k_peer_delete_wait_unregister(ar, &waits[link_id]);
 		if (ret) {
 			err_ret = ret;
 			continue;
@@ -453,6 +531,9 @@ int ath12k_link_sta_rhash_tbl_init(struct ath12k_base *ab)
 
 void ath12k_link_sta_rhash_tbl_destroy(struct ath12k_base *ab)
 {
+	if (!ab->rhead_sta_addr)
+		return;
+
 	rhashtable_destroy(ab->rhead_sta_addr);
 	kfree(ab->rhead_sta_addr);
 	ab->rhead_sta_addr = NULL;
diff --git a/drivers/net/wireless/ath/ath12k/peer.h b/drivers/net/wireless/ath/ath12k/peer.h
index 49d89796bc46e..9343944c2b8eb 100644
--- a/drivers/net/wireless/ath/ath12k/peer.h
+++ b/drivers/net/wireless/ath/ath12k/peer.h
@@ -9,13 +9,23 @@
 
 #include "dp_peer.h"
 
+struct ath12k_peer_delete_wait {
+	struct list_head list;
+	u32 vdev_id;
+	u8 addr[ETH_ALEN];
+	struct completion done;
+};
+
+void ath12k_peer_delete_resp_signal(struct ath12k *ar, u32 vdev_id, const u8 *addr);
+void ath12k_peer_delete_wait_flush(struct ath12k *ar);
+
 void ath12k_peer_cleanup(struct ath12k *ar, u32 vdev_id);
 int ath12k_peer_delete(struct ath12k *ar, u32 vdev_id, u8 *addr);
 int ath12k_peer_create(struct ath12k *ar, struct ath12k_link_vif *arvif,
 		       struct ieee80211_sta *sta,
 		       struct ath12k_wmi_peer_create_arg *arg);
-int ath12k_wait_for_peer_delete_done(struct ath12k *ar, u32 vdev_id,
-				     const u8 *addr);
+int ath12k_wait_for_peer_delete_done(struct ath12k *ar,
+				     struct ath12k_peer_delete_wait *wait);
 int ath12k_peer_mlo_link_peers_delete(struct ath12k_vif *ahvif, struct ath12k_sta *ahsta);
 struct ath12k_ml_peer *ath12k_peer_ml_find(struct ath12k_hw *ah,
 					   const u8 *addr);
diff --git a/drivers/net/wireless/ath/ath12k/qmi.c b/drivers/net/wireless/ath/ath12k/qmi.c
index fd762b5d7bb59..cabdd544bbc73 100644
--- a/drivers/net/wireless/ath/ath12k/qmi.c
+++ b/drivers/net/wireless/ath/ath12k/qmi.c
@@ -21,45 +21,45 @@
 
 static const struct qmi_elem_info wlfw_host_mlo_chip_info_s_v01_ei[] = {
 	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
+		.data_type	= QMI_UNSIGNED_1_BYTE,
+		.elem_len	= 1,
+		.elem_size	= sizeof(u8),
 		.array_type	= NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct wlfw_host_mlo_chip_info_s_v01,
+		.tlv_type	= 0,
+		.offset		= offsetof(struct wlfw_host_mlo_chip_info_s_v01,
 					   chip_id),
 	},
 	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
+		.data_type	= QMI_UNSIGNED_1_BYTE,
+		.elem_len	= 1,
+		.elem_size	= sizeof(u8),
 		.array_type	= NO_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct wlfw_host_mlo_chip_info_s_v01,
+		.tlv_type	= 0,
+		.offset		= offsetof(struct wlfw_host_mlo_chip_info_s_v01,
 					   num_local_links),
 	},
 	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = QMI_WLFW_MAX_NUM_MLO_LINKS_PER_CHIP_V01,
-		.elem_size      = sizeof(u8),
-		.array_type     = STATIC_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct wlfw_host_mlo_chip_info_s_v01,
+		.data_type	= QMI_UNSIGNED_1_BYTE,
+		.elem_len	= QMI_WLFW_MAX_NUM_MLO_LINKS_PER_CHIP_V01,
+		.elem_size	= sizeof(u8),
+		.array_type	= STATIC_ARRAY,
+		.tlv_type	= 0,
+		.offset		= offsetof(struct wlfw_host_mlo_chip_info_s_v01,
 					   hw_link_id),
 	},
 	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = QMI_WLFW_MAX_NUM_MLO_LINKS_PER_CHIP_V01,
-		.elem_size      = sizeof(u8),
-		.array_type     = STATIC_ARRAY,
-		.tlv_type       = 0,
-		.offset         = offsetof(struct wlfw_host_mlo_chip_info_s_v01,
+		.data_type	= QMI_UNSIGNED_1_BYTE,
+		.elem_len	= QMI_WLFW_MAX_NUM_MLO_LINKS_PER_CHIP_V01,
+		.elem_size	= sizeof(u8),
+		.array_type	= STATIC_ARRAY,
+		.tlv_type	= 0,
+		.offset		= offsetof(struct wlfw_host_mlo_chip_info_s_v01,
 					   valid_mlo_link_id),
 	},
 	{
-		.data_type      = QMI_EOTI,
+		.data_type	= QMI_EOTI,
 		.array_type	= NO_ARRAY,
-		.tlv_type       = QMI_COMMON_TLV_TYPE,
+		.tlv_type	= QMI_COMMON_TLV_TYPE,
 	},
 };
 
@@ -506,6 +506,24 @@ static const struct qmi_elem_info qmi_wlanfw_host_cap_req_msg_v01_ei[] = {
 		.offset		= offsetof(struct qmi_wlanfw_host_cap_req_msg_v01,
 					   feature_list),
 	},
+	{
+		.data_type	= QMI_OPT_FLAG,
+		.elem_len	= 1,
+		.elem_size	= sizeof(u8),
+		.array_type	= NO_ARRAY,
+		.tlv_type	= 0x33,
+		.offset		= offsetof(struct qmi_wlanfw_host_cap_req_msg_v01,
+					   dynamic_mem_support_valid),
+	},
+	{
+		.data_type	= QMI_UNSIGNED_1_BYTE,
+		.elem_len	= 1,
+		.elem_size	= sizeof(u8),
+		.array_type	= NO_ARRAY,
+		.tlv_type	= 0x33,
+		.offset		= offsetof(struct qmi_wlanfw_host_cap_req_msg_v01,
+					   dynamic_mem_support),
+	},
 	{
 		.data_type	= QMI_EOTI,
 		.array_type	= NO_ARRAY,
@@ -585,23 +603,41 @@ static const struct qmi_elem_info qmi_wlanfw_phy_cap_resp_msg_v01_ei[] = {
 					   board_id),
 	},
 	{
-		.data_type      = QMI_OPT_FLAG,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type     = NO_ARRAY,
-		.tlv_type       = 0x13,
-		.offset         = offsetof(struct qmi_wlanfw_phy_cap_resp_msg_v01,
+		.data_type	= QMI_OPT_FLAG,
+		.elem_len	= 1,
+		.elem_size	= sizeof(u8),
+		.array_type	= NO_ARRAY,
+		.tlv_type	= 0x13,
+		.offset		= offsetof(struct qmi_wlanfw_phy_cap_resp_msg_v01,
 					   single_chip_mlo_support_valid),
 	},
 	{
-		.data_type      = QMI_UNSIGNED_1_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u8),
-		.array_type     = NO_ARRAY,
-		.tlv_type       = 0x13,
-		.offset         = offsetof(struct qmi_wlanfw_phy_cap_resp_msg_v01,
+		.data_type	= QMI_UNSIGNED_1_BYTE,
+		.elem_len	= 1,
+		.elem_size	= sizeof(u8),
+		.array_type	= NO_ARRAY,
+		.tlv_type	= 0x13,
+		.offset		= offsetof(struct qmi_wlanfw_phy_cap_resp_msg_v01,
 					   single_chip_mlo_support),
 	},
+	{
+		.data_type	= QMI_OPT_FLAG,
+		.elem_len	= 1,
+		.elem_size	= sizeof(u8),
+		.array_type	= NO_ARRAY,
+		.tlv_type	= 0x17,
+		.offset		= offsetof(struct qmi_wlanfw_phy_cap_resp_msg_v01,
+					   dynamic_ddr_support_valid),
+	},
+	{
+		.data_type	= QMI_UNSIGNED_1_BYTE,
+		.elem_len	= 1,
+		.elem_size	= sizeof(u8),
+		.array_type	= NO_ARRAY,
+		.tlv_type	= 0x17,
+		.offset		= offsetof(struct qmi_wlanfw_phy_cap_resp_msg_v01,
+					   dynamic_ddr_support),
+	},
 	{
 		.data_type	= QMI_EOTI,
 		.array_type	= NO_ARRAY,
@@ -1625,42 +1661,45 @@ static const struct qmi_elem_info qmi_wlanfw_m3_info_resp_msg_v01_ei[] = {
 
 static const struct qmi_elem_info qmi_wlanfw_aux_uc_info_req_msg_v01_ei[] = {
 	{
-		.data_type      = QMI_UNSIGNED_8_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u64),
-		.array_type     = NO_ARRAY,
-		.tlv_type       = 0x01,
-		.offset         = offsetof(struct qmi_wlanfw_aux_uc_info_req_msg_v01, addr),
+		.data_type	= QMI_UNSIGNED_8_BYTE,
+		.elem_len	= 1,
+		.elem_size	= sizeof(u64),
+		.array_type	= NO_ARRAY,
+		.tlv_type	= 0x01,
+		.offset		= offsetof(struct qmi_wlanfw_aux_uc_info_req_msg_v01,
+					   addr),
 	},
 	{
-		.data_type      = QMI_UNSIGNED_4_BYTE,
-		.elem_len       = 1,
-		.elem_size      = sizeof(u32),
-		.array_type     = NO_ARRAY,
-		.tlv_type       = 0x02,
-		.offset         = offsetof(struct qmi_wlanfw_aux_uc_info_req_msg_v01, size),
+		.data_type	= QMI_UNSIGNED_4_BYTE,
+		.elem_len	= 1,
+		.elem_size	= sizeof(u32),
+		.array_type	= NO_ARRAY,
+		.tlv_type	= 0x02,
+		.offset		= offsetof(struct qmi_wlanfw_aux_uc_info_req_msg_v01,
+					   size),
 	},
 	{
-		.data_type      = QMI_EOTI,
-		.array_type     = NO_ARRAY,
-		.tlv_type       = QMI_COMMON_TLV_TYPE,
+		.data_type	= QMI_EOTI,
+		.array_type	= NO_ARRAY,
+		.tlv_type	= QMI_COMMON_TLV_TYPE,
 	},
 };
 
 static const struct qmi_elem_info qmi_wlanfw_aux_uc_info_resp_msg_v01_ei[] = {
 	{
-		.data_type      = QMI_STRUCT,
-		.elem_len       = 1,
-		.elem_size      = sizeof(struct qmi_response_type_v01),
-		.array_type     = NO_ARRAY,
-		.tlv_type       = 0x02,
-		.offset         = offsetof(struct qmi_wlanfw_aux_uc_info_resp_msg_v01, resp),
-		.ei_array       = qmi_response_type_v01_ei,
+		.data_type	= QMI_STRUCT,
+		.elem_len	= 1,
+		.elem_size	= sizeof(struct qmi_response_type_v01),
+		.array_type	= NO_ARRAY,
+		.tlv_type	= 0x02,
+		.offset		= offsetof(struct qmi_wlanfw_aux_uc_info_resp_msg_v01,
+					   resp),
+		.ei_array	= qmi_response_type_v01_ei,
 	},
 	{
-		.data_type      = QMI_EOTI,
-		.array_type     = NO_ARRAY,
-		.tlv_type       = QMI_COMMON_TLV_TYPE,
+		.data_type	= QMI_EOTI,
+		.array_type	= NO_ARRAY,
+		.tlv_type	= QMI_COMMON_TLV_TYPE,
 	},
 };
 
@@ -1772,7 +1811,8 @@ static const struct qmi_elem_info qmi_wlanfw_shadow_reg_cfg_s_v01_ei[] = {
 	},
 	{
 		.data_type	= QMI_EOTI,
-		.array_type	= QMI_COMMON_TLV_TYPE,
+		.array_type	= NO_ARRAY,
+		.tlv_type	= QMI_COMMON_TLV_TYPE,
 	},
 };
 
@@ -1925,7 +1965,7 @@ static const struct qmi_elem_info qmi_wlanfw_wlan_cfg_req_msg_v01_ei[] = {
 		.data_type	= QMI_OPT_FLAG,
 		.elem_len	= 1,
 		.elem_size	= sizeof(u8),
-		.array_type = NO_ARRAY,
+		.array_type	= NO_ARRAY,
 		.tlv_type	= 0x13,
 		.offset		= offsetof(struct qmi_wlanfw_wlan_cfg_req_msg_v01,
 					   shadow_reg_valid),
@@ -1934,7 +1974,7 @@ static const struct qmi_elem_info qmi_wlanfw_wlan_cfg_req_msg_v01_ei[] = {
 		.data_type	= QMI_DATA_LEN,
 		.elem_len	= 1,
 		.elem_size	= sizeof(u8),
-		.array_type = NO_ARRAY,
+		.array_type	= NO_ARRAY,
 		.tlv_type	= 0x13,
 		.offset		= offsetof(struct qmi_wlanfw_wlan_cfg_req_msg_v01,
 					   shadow_reg_len),
@@ -1943,7 +1983,7 @@ static const struct qmi_elem_info qmi_wlanfw_wlan_cfg_req_msg_v01_ei[] = {
 		.data_type	= QMI_STRUCT,
 		.elem_len	= QMI_WLANFW_MAX_NUM_SHADOW_REG_V01,
 		.elem_size	= sizeof(struct qmi_wlanfw_shadow_reg_cfg_s_v01),
-		.array_type = VAR_LEN_ARRAY,
+		.array_type	= VAR_LEN_ARRAY,
 		.tlv_type	= 0x13,
 		.offset		= offsetof(struct qmi_wlanfw_wlan_cfg_req_msg_v01,
 					   shadow_reg),
@@ -2003,15 +2043,17 @@ static const struct qmi_elem_info qmi_wlanfw_wlan_cfg_resp_msg_v01_ei[] = {
 
 static const struct qmi_elem_info qmi_wlanfw_mem_ready_ind_msg_v01_ei[] = {
 	{
-		.data_type = QMI_EOTI,
-		.array_type = NO_ARRAY,
+		.data_type	= QMI_EOTI,
+		.array_type	= NO_ARRAY,
+		.tlv_type	= QMI_COMMON_TLV_TYPE,
 	},
 };
 
 static const struct qmi_elem_info qmi_wlanfw_fw_ready_ind_msg_v01_ei[] = {
 	{
-		.data_type = QMI_EOTI,
-		.array_type = NO_ARRAY,
+		.data_type	= QMI_EOTI,
+		.array_type	= NO_ARRAY,
+		.tlv_type	= QMI_COMMON_TLV_TYPE,
 	},
 };
 
@@ -2094,14 +2136,14 @@ static int ath12k_host_cap_parse_mlo(struct ath12k_base *ab,
 
 	if (!ag->mlo_capable) {
 		ath12k_dbg(ab, ATH12K_DBG_QMI,
-			   "MLO is disabled hence skip QMI MLO cap");
+			   "MLO is disabled hence skip QMI MLO cap\n");
 		return 0;
 	}
 
 	if (!ab->qmi.num_radios || ab->qmi.num_radios == U8_MAX) {
 		ag->mlo_capable = false;
 		ath12k_dbg(ab, ATH12K_DBG_QMI,
-			   "skip QMI MLO cap due to invalid num_radio %d\n",
+			   "skip QMI MLO cap due to invalid num_radio %u\n",
 			   ab->qmi.num_radios);
 		return 0;
 	}
@@ -2125,7 +2167,7 @@ static int ath12k_host_cap_parse_mlo(struct ath12k_base *ab,
 	req->mlo_num_chips_valid = 1;
 	req->mlo_num_chips = ag->num_devices;
 
-	ath12k_dbg(ab, ATH12K_DBG_QMI, "mlo capability advertisement device_id %d group_id %d num_devices %d",
+	ath12k_dbg(ab, ATH12K_DBG_QMI, "mlo capability advertisement device_id %u group_id %u num_devices %u\n",
 		   req->mlo_chip_id, req->mlo_group_id, req->mlo_num_chips);
 
 	mutex_lock(&ag->mutex);
@@ -2146,14 +2188,14 @@ static int ath12k_host_cap_parse_mlo(struct ath12k_base *ab,
 		info->chip_id = partner_ab->device_id;
 		info->num_local_links = partner_ab->qmi.num_radios;
 
-		ath12k_dbg(ab, ATH12K_DBG_QMI, "mlo device id %d num_link %d\n",
+		ath12k_dbg(ab, ATH12K_DBG_QMI, "mlo device id %u num_link %u\n",
 			   info->chip_id, info->num_local_links);
 
 		for (j = 0; j < info->num_local_links; j++) {
 			info->hw_link_id[j] = partner_ab->wsi_info.hw_link_id_base + j;
 			info->valid_mlo_link_id[j] = 1;
 
-			ath12k_dbg(ab, ATH12K_DBG_QMI, "mlo hw_link_id %d\n",
+			ath12k_dbg(ab, ATH12K_DBG_QMI, "mlo hw_link_id %u\n",
 				   info->hw_link_id[j]);
 
 			hw_link_id++;
@@ -2248,6 +2290,11 @@ int ath12k_qmi_host_cap_send(struct ath12k_base *ab)
 	if (ret < 0)
 		goto out;
 
+	if (ab->qmi.dynamic_ddr_support) {
+		req.dynamic_mem_support_valid = 1;
+		req.dynamic_mem_support = 1;
+	}
+
 	ret = qmi_txn_init(&ab->qmi.handle, &txn,
 			   qmi_wlanfw_host_cap_resp_msg_v01_ei, &resp);
 	if (ret < 0)
@@ -2268,7 +2315,7 @@ int ath12k_qmi_host_cap_send(struct ath12k_base *ab)
 		goto out;
 
 	if (resp.resp.result != QMI_RESULT_SUCCESS_V01) {
-		ath12k_warn(ab, "Host capability request failed, result: %d, err: %d\n",
+		ath12k_warn(ab, "Host capability request failed, result: %u, err: %u\n",
 			    resp.resp.result, resp.resp.error);
 		ret = -EINVAL;
 		goto out;
@@ -2319,11 +2366,15 @@ static void ath12k_qmi_phy_cap_send(struct ath12k_base *ab)
 
 	ab->qmi.num_radios = resp.num_phy;
 
+	if (resp.dynamic_ddr_support_valid)
+		ab->qmi.dynamic_ddr_support = resp.dynamic_ddr_support;
+
 	ath12k_dbg(ab, ATH12K_DBG_QMI,
-		   "phy capability resp valid %d single_chip_mlo_support %d valid %d num_phy %d valid %d board_id %d\n",
+		   "phy capability resp valid %u single_chip_mlo_support %u valid %u num_phy %u valid %u board_id %u dynamic_ddr_valid %u dynamic_ddr_support %u\n",
 		   resp.single_chip_mlo_support_valid, resp.single_chip_mlo_support,
 		   resp.num_phy_valid, resp.num_phy,
-		   resp.board_id_valid, resp.board_id);
+		   resp.board_id_valid, resp.board_id, resp.dynamic_ddr_support_valid,
+		   resp.dynamic_ddr_support);
 
 	return;
 
@@ -2332,7 +2383,7 @@ static void ath12k_qmi_phy_cap_send(struct ath12k_base *ab)
 	ab->qmi.num_radios = ab->hw_params->def_num_link;
 
 	ath12k_dbg(ab, ATH12K_DBG_QMI,
-		   "no valid response from PHY capability, choose default num_phy %d\n",
+		   "no valid response from PHY capability, choose default num_phy %u\n",
 		   ab->qmi.num_radios);
 }
 
@@ -2393,7 +2444,7 @@ static int ath12k_qmi_fw_ind_register_send(struct ath12k_base *ab)
 	}
 
 	if (resp->resp.result != QMI_RESULT_SUCCESS_V01) {
-		ath12k_warn(ab, "FW Ind register request failed, result: %d, err: %d\n",
+		ath12k_warn(ab, "FW Ind register request failed, result: %u, err: %u\n",
 			    resp->resp.result, resp->resp.error);
 		ret = -EINVAL;
 		goto out;
@@ -2428,7 +2479,7 @@ int ath12k_qmi_respond_fw_mem_request(struct ath12k_base *ab)
 	if (!test_bit(ATH12K_FLAG_FIXED_MEM_REGION, &ab->dev_flags) &&
 	    ab->qmi.target_mem_delayed) {
 		delayed = true;
-		ath12k_dbg(ab, ATH12K_DBG_QMI, "qmi delays mem_request %d\n",
+		ath12k_dbg(ab, ATH12K_DBG_QMI, "qmi delays mem_request %u\n",
 			   ab->qmi.mem_seg_count);
 	} else {
 		delayed = false;
@@ -2474,7 +2525,7 @@ int ath12k_qmi_respond_fw_mem_request(struct ath12k_base *ab)
 		if (delayed && resp.resp.error == 0)
 			goto out;
 
-		ath12k_warn(ab, "Respond mem req failed, result: %d, err: %d\n",
+		ath12k_warn(ab, "Respond mem req failed, result: %u, err: %u\n",
 			    resp.resp.result, resp.resp.error);
 		ret = -EINVAL;
 		goto out;
@@ -2606,13 +2657,13 @@ static int ath12k_qmi_alloc_chunk(struct ath12k_base *ab,
 		if (chunk->size > ATH12K_QMI_MAX_CHUNK_SIZE) {
 			ab->qmi.target_mem_delayed = true;
 			ath12k_warn(ab,
-				    "qmi dma allocation failed (%d B type %u), will try later with small size\n",
+				    "qmi dma allocation failed (%u B type %u), will try later with small size\n",
 				    chunk->size,
 				    chunk->type);
 			ath12k_qmi_free_target_mem_chunk(ab);
 			return -EAGAIN;
 		}
-		ath12k_warn(ab, "memory allocation failure for %u size: %d\n",
+		ath12k_warn(ab, "memory allocation failure for %u size: %u\n",
 			    chunk->type, chunk->size);
 		return -ENOMEM;
 	}
@@ -2659,7 +2710,7 @@ static int ath12k_qmi_alloc_target_mem_chunk(struct ath12k_base *ab)
 			mlo_size += chunk->size;
 			if (ag->mlo_mem.mlo_mem_size &&
 			    mlo_size > ag->mlo_mem.mlo_mem_size) {
-				ath12k_err(ab, "QMI MLO memory allocation failure, requested size %d is more than allocated size %d",
+				ath12k_err(ab, "QMI MLO memory allocation failure, requested size %d is more than allocated size %d\n",
 					   mlo_size, ag->mlo_mem.mlo_mem_size);
 				ret = -EINVAL;
 				goto err;
@@ -2668,7 +2719,7 @@ static int ath12k_qmi_alloc_target_mem_chunk(struct ath12k_base *ab)
 			mlo_chunk = &ag->mlo_mem.chunk[mlo_idx];
 			if (mlo_chunk->paddr) {
 				if (chunk->size != mlo_chunk->size) {
-					ath12k_err(ab, "QMI MLO chunk memory allocation failure for index %d, requested size %d is more than allocated size %d",
+					ath12k_err(ab, "QMI MLO chunk memory allocation failure for index %d, requested size %u is more than allocated size %u\n",
 						   mlo_idx, chunk->size, mlo_chunk->size);
 					ret = -EINVAL;
 					goto err;
@@ -2699,7 +2750,7 @@ static int ath12k_qmi_alloc_target_mem_chunk(struct ath12k_base *ab)
 	if (!ag->mlo_mem.mlo_mem_size) {
 		ag->mlo_mem.mlo_mem_size = mlo_size;
 	} else if (ag->mlo_mem.mlo_mem_size != mlo_size) {
-		ath12k_err(ab, "QMI MLO memory size error, expected size is %d but requested size is %d",
+		ath12k_err(ab, "QMI MLO memory size error, expected size is %d but requested size is %d\n",
 			   ag->mlo_mem.mlo_mem_size, mlo_size);
 		ret = -EINVAL;
 		goto err;
@@ -2884,7 +2935,7 @@ int ath12k_qmi_request_target_cap(struct ath12k_base *ab)
 	}
 
 	if (resp.resp.result != QMI_RESULT_SUCCESS_V01) {
-		ath12k_warn(ab, "qmi targetcap req failed, result: %d, err: %d\n",
+		ath12k_warn(ab, "qmi targetcap req failed, result: %u, err: %u\n",
 			    resp.resp.result, resp.resp.error);
 		ret = -EINVAL;
 		goto out;
@@ -2936,7 +2987,7 @@ int ath12k_qmi_request_target_cap(struct ath12k_base *ab)
 		    ab->qmi.target.chip_id, ab->qmi.target.chip_family,
 		    ab->qmi.target.board_id, ab->qmi.target.soc_id);
 
-	ath12k_info(ab, "fw_version 0x%x fw_build_timestamp %s fw_build_id %s",
+	ath12k_info(ab, "fw_version 0x%x fw_build_timestamp %s fw_build_id %s\n",
 		    ab->qmi.target.fw_version,
 		    ab->qmi.target.fw_build_timestamp,
 		    ab->qmi.target.fw_build_id);
@@ -3006,7 +3057,7 @@ static int ath12k_qmi_load_file_target_mem(struct ath12k_base *ab,
 		if (ret < 0)
 			goto out;
 
-		ath12k_dbg(ab, ATH12K_DBG_QMI, "qmi bdf download req fixed addr type %d\n",
+		ath12k_dbg(ab, ATH12K_DBG_QMI, "qmi bdf download req fixed addr type %u\n",
 			   type);
 
 		ret = qmi_send_request(&ab->qmi.handle, NULL, &txn,
@@ -3023,7 +3074,7 @@ static int ath12k_qmi_load_file_target_mem(struct ath12k_base *ab,
 			goto out;
 
 		if (resp.resp.result != QMI_RESULT_SUCCESS_V01) {
-			ath12k_warn(ab, "qmi BDF download failed, result: %d, err: %d\n",
+			ath12k_warn(ab, "qmi BDF download failed, result: %u, err: %u\n",
 				    resp.resp.result, resp.resp.error);
 			ret = -EINVAL;
 			goto out;
@@ -3036,7 +3087,7 @@ static int ath12k_qmi_load_file_target_mem(struct ath12k_base *ab,
 			temp += req->data_len;
 			req->seg_id++;
 			ath12k_dbg(ab, ATH12K_DBG_QMI,
-				   "qmi bdf download request remaining %i\n",
+				   "qmi bdf download request remaining %u\n",
 				   remaining);
 		}
 	}
@@ -3127,7 +3178,7 @@ int ath12k_qmi_load_bdf_qmi(struct ath12k_base *ab,
 			release_firmware(fw_entry);
 		return ret;
 	default:
-		ath12k_warn(ab, "unknown file type for load %d", type);
+		ath12k_warn(ab, "unknown file type for load %d\n", type);
 		goto out;
 	}
 
@@ -3239,7 +3290,7 @@ int ath12k_qmi_wlanfw_m3_info_send(struct ath12k_base *ab)
 	if (ab->hw_params->fw.m3_loader == ath12k_m3_fw_loader_driver) {
 		ret = ath12k_qmi_m3_load(ab);
 		if (ret) {
-			ath12k_err(ab, "failed to load m3 firmware: %d", ret);
+			ath12k_err(ab, "failed to load m3 firmware: %d\n", ret);
 			return ret;
 		}
 		req.addr = m3_mem->paddr;
@@ -3269,7 +3320,7 @@ int ath12k_qmi_wlanfw_m3_info_send(struct ath12k_base *ab)
 	}
 
 	if (resp.resp.result != QMI_RESULT_SUCCESS_V01) {
-		ath12k_warn(ab, "qmi M3 info request failed, result: %d, err: %d\n",
+		ath12k_warn(ab, "qmi M3 info request failed, result: %u, err: %u\n",
 			    resp.resp.result, resp.resp.error);
 		ret = -EINVAL;
 		goto out;
@@ -3364,7 +3415,7 @@ int ath12k_qmi_wlanfw_aux_uc_info_send(struct ath12k_base *ab)
 
 	ret = ath12k_qmi_aux_uc_load(ab);
 	if (ret) {
-		ath12k_err(ab, "failed to load aux_uc firmware: %d", ret);
+		ath12k_err(ab, "failed to load aux_uc firmware: %d\n", ret);
 		return ret;
 	}
 
@@ -3394,7 +3445,7 @@ int ath12k_qmi_wlanfw_aux_uc_info_send(struct ath12k_base *ab)
 	}
 
 	if (resp.resp.result != QMI_RESULT_SUCCESS_V01) {
-		ath12k_warn(ab, "qmi AUX_UC info request failed, result: %d, err: %d\n",
+		ath12k_warn(ab, "qmi AUX_UC info request failed, result: %u, err: %u\n",
 			    resp.resp.result, resp.resp.error);
 		ret = -EINVAL;
 		goto out;
@@ -3426,7 +3477,7 @@ static int ath12k_qmi_wlanfw_mode_send(struct ath12k_base *ab,
 			       qmi_wlanfw_wlan_mode_req_msg_v01_ei, &req);
 	if (ret < 0) {
 		qmi_txn_cancel(&txn);
-		ath12k_warn(ab, "qmi failed to send mode request, mode: %d, err = %d\n",
+		ath12k_warn(ab, "qmi failed to send mode request, mode: %u, err = %d\n",
 			    mode, ret);
 		goto out;
 	}
@@ -3437,13 +3488,13 @@ static int ath12k_qmi_wlanfw_mode_send(struct ath12k_base *ab,
 			ath12k_warn(ab, "WLFW service is dis-connected\n");
 			return 0;
 		}
-		ath12k_warn(ab, "qmi failed set mode request, mode: %d, err = %d\n",
+		ath12k_warn(ab, "qmi failed set mode request, mode: %u, err = %d\n",
 			    mode, ret);
 		goto out;
 	}
 
 	if (resp.resp.result != QMI_RESULT_SUCCESS_V01) {
-		ath12k_warn(ab, "Mode request failed, mode: %d, result: %d err: %d\n",
+		ath12k_warn(ab, "Mode request failed, mode: %u, result: %u err: %u\n",
 			    mode, resp.resp.result, resp.resp.error);
 		ret = -EINVAL;
 		goto out;
@@ -3536,7 +3587,7 @@ static int ath12k_qmi_wlanfw_wlan_cfg_send(struct ath12k_base *ab)
 	}
 
 	if (resp.resp.result != QMI_RESULT_SUCCESS_V01) {
-		ath12k_warn(ab, "qmi wlan config request failed, result: %d, err: %d\n",
+		ath12k_warn(ab, "qmi wlan config request failed, result: %u, err: %u\n",
 			    resp.resp.result, resp.resp.error);
 		ret = -EINVAL;
 		goto out;
@@ -3580,7 +3631,7 @@ static int ath12k_qmi_wlanfw_wlan_ini_send(struct ath12k_base *ab)
 	}
 
 	if (resp.resp.result != QMI_RESULT_SUCCESS_V01) {
-		ath12k_warn(ab, "QMI wlan ini response failure: %d %d\n",
+		ath12k_warn(ab, "QMI wlan ini response failure: %u %u\n",
 			    resp.resp.result, resp.resp.error);
 		ret = -EINVAL;
 		goto out;
@@ -3663,7 +3714,7 @@ void ath12k_qmi_trigger_host_cap(struct ath12k_base *ab)
 
 	spin_unlock(&qmi->event_lock);
 
-	ath12k_dbg(ab, ATH12K_DBG_QMI, "trigger host cap for device id %d\n",
+	ath12k_dbg(ab, ATH12K_DBG_QMI, "trigger host cap for device id %u\n",
 		   ab->device_id);
 
 	ath12k_qmi_driver_event_post(qmi, ATH12K_QMI_EVENT_HOST_CAP, NULL);
@@ -3833,7 +3884,7 @@ static void ath12k_qmi_msg_mem_request_cb(struct qmi_handle *qmi_hdl,
 	for (i = 0; i < qmi->mem_seg_count ; i++) {
 		ab->qmi.target_mem[i].type = msg->mem_seg[i].type;
 		ab->qmi.target_mem[i].size = msg->mem_seg[i].size;
-		ath12k_dbg(ab, ATH12K_DBG_QMI, "qmi mem seg type %d size %d\n",
+		ath12k_dbg(ab, ATH12K_DBG_QMI, "qmi mem seg type %d size %u\n",
 			   msg->mem_seg[i].type, msg->mem_seg[i].size);
 	}
 
@@ -3954,7 +4005,7 @@ static int ath12k_qmi_event_host_cap(struct ath12k_qmi *qmi)
 
 	ret = ath12k_qmi_host_cap_send(ab);
 	if (ret < 0) {
-		ath12k_warn(ab, "failed to send qmi host cap for device id %d: %d\n",
+		ath12k_warn(ab, "failed to send qmi host cap for device id %u: %d\n",
 			    ab->device_id, ret);
 		return ret;
 	}
@@ -4024,7 +4075,7 @@ static void ath12k_qmi_driver_event_work(struct work_struct *work)
 				set_bit(ATH12K_FLAG_QMI_FAIL, &ab->dev_flags);
 			break;
 		default:
-			ath12k_warn(ab, "invalid event type: %d", event->type);
+			ath12k_warn(ab, "invalid event type: %d\n", event->type);
 			break;
 		}
 
diff --git a/drivers/net/wireless/ath/ath12k/qmi.h b/drivers/net/wireless/ath/ath12k/qmi.h
index 2a63e214eb42c..cbe5be30053a0 100644
--- a/drivers/net/wireless/ath/ath12k/qmi.h
+++ b/drivers/net/wireless/ath/ath12k/qmi.h
@@ -13,7 +13,6 @@
 #define ATH12K_HOST_VERSION_STRING		"WIN"
 #define ATH12K_QMI_WLANFW_TIMEOUT_MS		10000
 #define ATH12K_QMI_MAX_BDF_FILE_NAME_SIZE	64
-#define ATH12K_QMI_CALDB_ADDRESS		0x4BA00000
 #define ATH12K_QMI_WLANFW_MAX_BUILD_ID_LEN_V01	128
 #define ATH12K_QMI_WLFW_SERVICE_VERS_V01	0x01
 #define ATH12K_QMI_WLFW_SERVICE_INS_ID_V01	0x02
@@ -24,9 +23,7 @@
 #define ATH12K_QMI_WLANFW_MAX_TIMESTAMP_LEN_V01	32
 #define ATH12K_QMI_RESP_LEN_MAX			8192
 #define ATH12K_QMI_WLANFW_MAX_NUM_MEM_SEG_V01	52
-#define ATH12K_QMI_CALDB_SIZE			0x480000
 #define ATH12K_QMI_BDF_EXT_STR_LENGTH		0x20
-#define ATH12K_QMI_FW_MEM_REQ_SEGMENT_CNT	3
 #define ATH12K_QMI_WLFW_MAX_DEV_MEM_NUM_V01 4
 #define ATH12K_QMI_DEVMEM_CMEM_INDEX	0
 
@@ -156,12 +153,11 @@ struct ath12k_qmi {
 	struct m3_mem_region aux_uc_mem;
 	unsigned int service_ins_id;
 	struct dev_mem_info dev_mem[ATH12K_QMI_WLFW_MAX_DEV_MEM_NUM_V01];
+	u8 dynamic_ddr_support;
 };
 
-#define QMI_WLANFW_HOST_CAP_REQ_MSG_V01_MAX_LEN		261
+#define QMI_WLANFW_HOST_CAP_REQ_MSG_V01_MAX_LEN		265
 #define QMI_WLANFW_HOST_CAP_REQ_V01			0x0034
-#define QMI_WLANFW_HOST_CAP_RESP_MSG_V01_MAX_LEN	7
-#define QMI_WLFW_HOST_CAP_RESP_V01			0x0034
 #define QMI_WLFW_MAX_NUM_GPIO_V01			32
 #define QMI_WLANFW_MAX_PLATFORM_NAME_LEN_V01		64
 #define QMI_WLANFW_MAX_HOST_DDR_RANGE_SIZE_V01		3
@@ -258,7 +254,8 @@ struct qmi_wlanfw_host_cap_req_msg_v01 {
 	struct wlfw_host_mlo_chip_info_s_v01 mlo_chip_info[QMI_WLFW_MAX_NUM_MLO_CHIPS_V01];
 	u8 feature_list_valid;
 	u64 feature_list;
-
+	u8 dynamic_mem_support_valid;
+	u8 dynamic_mem_support;
 };
 
 struct qmi_wlanfw_host_cap_resp_msg_v01 {
@@ -267,8 +264,6 @@ struct qmi_wlanfw_host_cap_resp_msg_v01 {
 
 #define QMI_WLANFW_PHY_CAP_REQ_MSG_V01_MAX_LEN		0
 #define QMI_WLANFW_PHY_CAP_REQ_V01			0x0057
-#define QMI_WLANFW_PHY_CAP_RESP_MSG_V01_MAX_LEN		18
-#define QMI_WLANFW_PHY_CAP_RESP_V01			0x0057
 
 struct qmi_wlanfw_phy_cap_req_msg_v01 {
 };
@@ -281,12 +276,12 @@ struct qmi_wlanfw_phy_cap_resp_msg_v01 {
 	u32 board_id;
 	u8 single_chip_mlo_support_valid;
 	u8 single_chip_mlo_support;
+	u8 dynamic_ddr_support_valid;
+	u8 dynamic_ddr_support;
 };
 
 #define QMI_WLANFW_IND_REGISTER_REQ_MSG_V01_MAX_LEN		54
 #define QMI_WLANFW_IND_REGISTER_REQ_V01				0x0020
-#define QMI_WLANFW_IND_REGISTER_RESP_MSG_V01_MAX_LEN		18
-#define QMI_WLANFW_IND_REGISTER_RESP_V01			0x0020
 #define QMI_WLANFW_CLIENT_ID					0x4b4e454c
 
 struct qmi_wlanfw_ind_register_req_msg_v01 {
@@ -322,12 +317,8 @@ struct qmi_wlanfw_ind_register_resp_msg_v01 {
 	u64 fw_status;
 };
 
-#define QMI_WLANFW_REQUEST_MEM_IND_MSG_V01_MAX_LEN	1824
 #define QMI_WLANFW_RESPOND_MEM_REQ_MSG_V01_MAX_LEN	888
-#define QMI_WLANFW_RESPOND_MEM_RESP_MSG_V01_MAX_LEN	7
-#define QMI_WLANFW_REQUEST_MEM_IND_V01			0x0035
 #define QMI_WLANFW_RESPOND_MEM_REQ_V01			0x0036
-#define QMI_WLANFW_RESPOND_MEM_RESP_V01			0x0036
 #define QMI_WLANFW_MAX_NUM_MEM_CFG_V01			2
 #define QMI_WLANFW_MAX_STR_LEN_V01                      16
 
@@ -385,9 +376,7 @@ struct qmi_wlanfw_fw_ready_ind_msg_v01 {
 };
 
 #define QMI_WLANFW_CAP_REQ_MSG_V01_MAX_LEN	0
-#define QMI_WLANFW_CAP_RESP_MSG_V01_MAX_LEN	207
 #define QMI_WLANFW_CAP_REQ_V01			0x0024
-#define QMI_WLANFW_CAP_RESP_V01			0x0024
 
 enum qmi_wlanfw_pipedir_enum_v01 {
 	QMI_WLFW_PIPEDIR_NONE_V01 = 0,
@@ -500,8 +489,6 @@ struct qmi_wlanfw_cap_req_msg_v01 {
 };
 
 #define QMI_WLANFW_BDF_DOWNLOAD_REQ_MSG_V01_MAX_LEN	6182
-#define QMI_WLANFW_BDF_DOWNLOAD_RESP_MSG_V01_MAX_LEN	7
-#define QMI_WLANFW_BDF_DOWNLOAD_RESP_V01		0x0025
 #define QMI_WLANFW_BDF_DOWNLOAD_REQ_V01			0x0025
 /* TODO: Need to check with MCL and FW team that data can be pointer and
  * can be last element in structure
@@ -529,8 +516,6 @@ struct qmi_wlanfw_bdf_download_resp_msg_v01 {
 };
 
 #define QMI_WLANFW_M3_INFO_REQ_MSG_V01_MAX_MSG_LEN	18
-#define QMI_WLANFW_M3_INFO_RESP_MSG_V01_MAX_MSG_LEN	7
-#define QMI_WLANFW_M3_INFO_RESP_V01		0x003C
 #define QMI_WLANFW_M3_INFO_REQ_V01		0x003C
 
 struct qmi_wlanfw_m3_info_req_msg_v01 {
@@ -543,7 +528,6 @@ struct qmi_wlanfw_m3_info_resp_msg_v01 {
 };
 
 #define QMI_WLANFW_AUX_UC_INFO_REQ_MSG_V01_MAX_MSG_LEN	18
-#define QMI_WLANFW_AUX_UC_INFO_RESP_MSG_V01_MAX_MSG_LEN	7
 #define QMI_WLANFW_AUX_UC_INFO_REQ_V01	0x005A
 
 struct qmi_wlanfw_aux_uc_info_req_msg_v01 {
@@ -556,13 +540,9 @@ struct qmi_wlanfw_aux_uc_info_resp_msg_v01 {
 };
 
 #define QMI_WLANFW_WLAN_MODE_REQ_MSG_V01_MAX_LEN	11
-#define QMI_WLANFW_WLAN_MODE_RESP_MSG_V01_MAX_LEN	7
 #define QMI_WLANFW_WLAN_CFG_REQ_MSG_V01_MAX_LEN		803
-#define QMI_WLANFW_WLAN_CFG_RESP_MSG_V01_MAX_LEN	7
 #define QMI_WLANFW_WLAN_MODE_REQ_V01			0x0022
-#define QMI_WLANFW_WLAN_MODE_RESP_V01			0x0022
 #define QMI_WLANFW_WLAN_CFG_REQ_V01			0x0023
-#define QMI_WLANFW_WLAN_CFG_RESP_V01			0x0023
 #define QMI_WLANFW_MAX_STR_LEN_V01			16
 #define QMI_WLANFW_MAX_NUM_CE_V01			12
 #define QMI_WLANFW_MAX_NUM_SVC_V01			24
@@ -605,9 +585,7 @@ struct qmi_wlanfw_wlan_cfg_resp_msg_v01 {
 };
 
 #define ATH12K_QMI_WLANFW_WLAN_INI_REQ_V01	0x002F
-#define ATH12K_QMI_WLANFW_WLAN_INI_RESP_V01	0x002F
 #define QMI_WLANFW_WLAN_INI_REQ_MSG_V01_MAX_LEN		7
-#define QMI_WLANFW_WLAN_INI_RESP_MSG_V01_MAX_LEN	7
 
 struct qmi_wlanfw_wlan_ini_req_msg_v01 {
 	/* Must be set to true if enable_fwlog is being passed */
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c b/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c
index 7dd4a49d64d5f..016b0c38e51e2 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c
+++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c
@@ -1565,16 +1565,17 @@ ath12k_wifi7_dp_mon_parse_status_msdu_end(struct ath12k_mon_data *pmon,
 static enum hal_rx_mon_status
 ath12k_wifi7_dp_mon_rx_parse_status_tlv(struct ath12k_pdev_dp *dp_pdev,
 					struct ath12k_mon_data *pmon,
-					const struct hal_tlv_64_hdr *tlv)
+					const void *tlv)
 {
 	struct hal_rx_mon_ppdu_info *ppdu_info = &pmon->mon_ppdu_info;
-	const void *tlv_data = tlv->value;
-	u32 info[7], userid;
-	u16 tlv_tag, tlv_len;
+	struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev);
+	struct ath12k_hal *hal = &ar->ab->hal;
+	u16 tlv_tag, tlv_len, userid;
+	void *tlv_data;
+	u32 info[7];
 
-	tlv_tag = le64_get_bits(tlv->tl, HAL_TLV_64_HDR_TAG);
-	tlv_len = le64_get_bits(tlv->tl, HAL_TLV_64_HDR_LEN);
-	userid = le64_get_bits(tlv->tl, HAL_TLV_64_USR_ID);
+	tlv_data = hal->ops->mon_rx_status_dec_tlv_hdr((void *)tlv, &tlv_tag,
+						       &tlv_len, &userid);
 
 	if (ppdu_info->tlv_aggr.in_progress && ppdu_info->tlv_aggr.tlv_tag != tlv_tag) {
 		ath12k_wifi7_dp_mon_parse_eht_sig_hdr(ppdu_info,
@@ -2480,7 +2481,6 @@ ath12k_wifi7_dp_mon_rx_deliver(struct ath12k_pdev_dp *dp_pdev,
 {
 	struct sk_buff *mon_skb, *skb_next, *header;
 	struct ieee80211_rx_status *rxs = &dp_pdev->rx_status;
-	u8 decap = DP_RX_DECAP_TYPE_RAW;
 
 	mon_skb = ath12k_dp_mon_rx_merg_msdus(dp_pdev, mon_mpdu, ppduinfo, rxs);
 	if (!mon_skb)
@@ -2507,12 +2507,8 @@ ath12k_wifi7_dp_mon_rx_deliver(struct ath12k_pdev_dp *dp_pdev,
 		}
 		rxs->flag |= RX_FLAG_ONLY_MONITOR;
 
-		if (!(rxs->flag & RX_FLAG_ONLY_MONITOR))
-			decap = mon_mpdu->decap_format;
-
 		ath12k_dp_mon_update_radiotap(dp_pdev, ppduinfo, mon_skb, rxs);
-		ath12k_dp_mon_rx_deliver_msdu(dp_pdev, napi, mon_skb, ppduinfo,
-					      rxs, decap);
+		ath12k_dp_mon_rx_deliver_msdu(dp_pdev, napi, mon_skb, rxs);
 		mon_skb = skb_next;
 	} while (mon_skb);
 	rxs->flag = 0;
@@ -2930,11 +2926,12 @@ static enum dp_mon_status_buf_state
 ath12k_wifi7_dp_rx_mon_buf_done(struct ath12k_base *ab, struct hal_srng *srng,
 				struct dp_rxdma_mon_ring *rx_ring)
 {
+	struct ath12k_hal *hal = &ab->hal;
 	struct ath12k_skb_rxcb *rxcb;
-	struct hal_tlv_64_hdr *tlv;
 	struct sk_buff *skb;
 	void *status_desc;
 	dma_addr_t paddr;
+	u16 tlv_tag;
 	u32 cookie;
 	int buf_id;
 	u8 rbm;
@@ -2959,8 +2956,8 @@ ath12k_wifi7_dp_rx_mon_buf_done(struct ath12k_base *ab, struct hal_srng *srng,
 				skb->len + skb_tailroom(skb),
 				DMA_FROM_DEVICE);
 
-	tlv = (struct hal_tlv_64_hdr *)skb->data;
-	if (le64_get_bits(tlv->tl, HAL_TLV_HDR_TAG) != HAL_RX_STATUS_BUFFER_DONE)
+	hal->ops->mon_rx_status_dec_tlv_hdr(skb->data, &tlv_tag, NULL, NULL);
+	if (tlv_tag != HAL_RX_STATUS_BUFFER_DONE)
 		return DP_MON_STATUS_NO_DMA;
 
 	return DP_MON_STATUS_REPLINISH;
@@ -2972,41 +2969,40 @@ ath12k_wifi7_dp_mon_parse_rx_dest(struct ath12k_pdev_dp *dp_pdev,
 				  struct sk_buff *skb)
 {
 	struct ath12k *ar = ath12k_pdev_dp_to_ar(dp_pdev);
-	struct hal_tlv_64_hdr *tlv;
+	struct ath12k_hal *hal = &ar->ab->hal;
+	u8 *tlv_value, *tlv = skb->data;
 	struct ath12k_skb_rxcb *rxcb;
 	enum hal_rx_mon_status hal_status;
 	u16 tlv_tag, tlv_len;
-	u8 *ptr = skb->data;
+	u32 tlv_hdr_len;
+
+	tlv_hdr_len = hal->ops->get_tlv_hdr_align();
 
 	do {
-		tlv = (struct hal_tlv_64_hdr *)ptr;
-		tlv_tag = le64_get_bits(tlv->tl, HAL_TLV_64_HDR_TAG);
+		tlv_value = hal->ops->mon_rx_status_dec_tlv_hdr(tlv, &tlv_tag,
+								&tlv_len, NULL);
 
 		/* The actual length of PPDU_END is the combined length of many PHY
 		 * TLVs that follow. Skip the TLV header and
 		 * rx_rxpcu_classification_overview that follows the header to get to
 		 * next TLV.
 		 */
-
 		if (tlv_tag == HAL_RX_PPDU_END)
 			tlv_len = sizeof(struct hal_rx_rxpcu_classification_overview);
-		else
-			tlv_len = le64_get_bits(tlv->tl, HAL_TLV_64_HDR_LEN);
 
 		hal_status = ath12k_wifi7_dp_mon_rx_parse_status_tlv(dp_pdev, pmon,
 								     tlv);
 
 		if (ar->monitor_started && ar->ab->hw_params->rxdma1_enable &&
 		    ath12k_wifi7_dp_mon_parse_rx_dest_tlv(dp_pdev, pmon, hal_status,
-							  tlv->value))
+							  tlv_value))
 			return HAL_RX_MON_STATUS_PPDU_DONE;
 
-		ptr += sizeof(*tlv) + tlv_len;
-		ptr = PTR_ALIGN(ptr, HAL_TLV_64_ALIGN);
+		tlv = PTR_ALIGN(tlv + tlv_len + tlv_hdr_len, tlv_hdr_len);
 
-		if ((ptr - skb->data) > skb->len)
+		if (unlikely(tlv - skb->data > skb->len ||
+			     skb->len - (tlv - skb->data) < tlv_hdr_len))
 			break;
-
 	} while ((hal_status == HAL_RX_MON_STATUS_PPDU_NOT_DONE) ||
 		 (hal_status == HAL_RX_MON_STATUS_BUF_ADDR) ||
 		 (hal_status == HAL_RX_MON_STATUS_MPDU_START) ||
@@ -3056,15 +3052,16 @@ ath12k_wifi7_dp_rx_reap_mon_status_ring(struct ath12k_base *ab, int mac_id,
 	int buf_id, srng_id, num_buffs_reaped = 0;
 	enum dp_mon_status_buf_state reap_status;
 	struct dp_rxdma_mon_ring *rx_ring;
+	struct ath12k_hal *hal = &ab->hal;
 	struct ath12k_mon_data *pmon;
 	struct ath12k_skb_rxcb *rxcb;
-	struct hal_tlv_64_hdr *tlv;
 	void *rx_mon_status_desc;
 	struct hal_srng *srng;
 	struct ath12k_dp *dp;
 	struct sk_buff *skb;
 	struct ath12k *ar;
 	dma_addr_t paddr;
+	u16 tlv_tag;
 	u32 cookie;
 	u8 rbm;
 
@@ -3109,14 +3106,13 @@ ath12k_wifi7_dp_rx_reap_mon_status_ring(struct ath12k_base *ab, int mac_id,
 						skb->len + skb_tailroom(skb),
 						DMA_FROM_DEVICE);
 
-			tlv = (struct hal_tlv_64_hdr *)skb->data;
-			if (le64_get_bits(tlv->tl, HAL_TLV_HDR_TAG) !=
-					HAL_RX_STATUS_BUFFER_DONE) {
+			hal->ops->mon_rx_status_dec_tlv_hdr(skb->data, &tlv_tag,
+							    NULL, NULL);
+			if (tlv_tag != HAL_RX_STATUS_BUFFER_DONE) {
 				pmon->buf_state = DP_MON_STATUS_NO_DMA;
 				ath12k_warn(ab,
-					    "mon status DONE not set %llx, buf_id %d\n",
-					    le64_get_bits(tlv->tl, HAL_TLV_HDR_TAG),
-					    buf_id);
+					    "mon status DONE not set %x, buf_id %d\n",
+					    tlv_tag, buf_id);
 				/* RxDMA status done bit might not be set even
 				 * though tp is moved by HW.
 				 */
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c
index cb9dd8cb28b67..95d87dd67872f 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c
+++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c
@@ -5,6 +5,7 @@
  */
 
 #include "dp_rx.h"
+#include "../dp_peer.h"
 #include "../dp_tx.h"
 #include "../peer.h"
 #include "hal_qcn9274.h"
@@ -2315,3 +2316,35 @@ ath12k_wifi7_dp_rxdesc_mpdu_valid(struct ath12k_base *ab,
 
 	return tlv_tag == HAL_RX_MPDU_START;
 }
+
+void
+ath12k_wifi7_dp_rx_set_link_id_qcn9274(struct ath12k_dp_peer *dp_peer,
+				       struct ath12k_skb_rxcb *rxcb,
+				       struct ieee80211_rx_status *status)
+{
+	status->link_valid = 1;
+	status->link_id = dp_peer->hw_links[rxcb->hw_link_id];
+}
+
+void
+ath12k_wifi7_dp_rx_set_link_id_wcn7850(struct ath12k_dp_peer *dp_peer,
+				       struct ath12k_skb_rxcb *rxcb,
+				       struct ieee80211_rx_status *status)
+{
+	struct ath12k_dp_link_peer *link_peer;
+	unsigned long links_map;
+	int i;
+
+	RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
+			 "ath12k set rx link id called without rcu lock");
+
+	links_map = READ_ONCE(dp_peer->link_peers_map);
+	for_each_set_bit(i, &links_map, ATH12K_NUM_MAX_LINKS) {
+		link_peer = rcu_dereference(dp_peer->link_peers[i]);
+		if (link_peer && link_peer->peer_id == rxcb->peer_id) {
+			status->link_valid = 1;
+			status->link_id = link_peer->link_id;
+			return;
+		}
+	}
+}
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h
index 8aa79faf567fd..1d3a4788a2dd9 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h
+++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h
@@ -57,4 +57,10 @@ ath12k_wifi7_dp_rxdesc_mpdu_valid(struct ath12k_base *ab,
 				  struct hal_rx_desc *rx_desc);
 int ath12k_wifi7_dp_rx_tid_delete_handler(struct ath12k_base *ab,
 					  struct ath12k_dp_rx_tid_rxq *rx_tid);
+void ath12k_wifi7_dp_rx_set_link_id_qcn9274(struct ath12k_dp_peer *dp_peer,
+					    struct ath12k_skb_rxcb *rxcb,
+					    struct ieee80211_rx_status *status);
+void ath12k_wifi7_dp_rx_set_link_id_wcn7850(struct ath12k_dp_peer *dp_peer,
+					    struct ath12k_skb_rxcb *rxcb,
+					    struct ieee80211_rx_status *status);
 #endif
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_qcc2072.c b/drivers/net/wireless/ath/ath12k/wifi7/hal_qcc2072.c
index 8cebb229ebed8..7cf00fda996f3 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/hal_qcc2072.c
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_qcc2072.c
@@ -455,7 +455,7 @@ static u16 ath12k_hal_reo_status_dec_tlv_hdr_qcc2072(void *tlv, void **desc)
 	struct hal_reo_get_queue_stats_status_qcc2072 *status_tlv;
 	u16 tag;
 
-	tag = ath12k_hal_decode_tlv32_hdr(tlv, (void **)&status_tlv);
+	status_tlv = ath12k_hal_decode_tlv32_hdr(tlv, &tag, NULL, NULL);
 	/*
 	 * actual desc of REO status entry starts after tlv32_padding,
 	 * see hal_reo_get_queue_stats_status_qcc2072
@@ -506,6 +506,8 @@ const struct hal_ops hal_qcc2072_ops = {
 	.rx_reo_ent_buf_paddr_get = ath12k_wifi7_hal_rx_reo_ent_buf_paddr_get,
 	.reo_cmd_enc_tlv_hdr = ath12k_hal_encode_tlv32_hdr,
 	.reo_status_dec_tlv_hdr = ath12k_hal_reo_status_dec_tlv_hdr_qcc2072,
+	.mon_rx_status_dec_tlv_hdr = ath12k_hal_decode_tlv32_hdr,
+	.get_tlv_hdr_align = ath12k_hal_get_tlv32_hdr_align,
 };
 
 u32 ath12k_hal_rx_desc_get_mpdu_start_offset_qcc2072(void)
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.c b/drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.c
index 9d5180ef83b48..052b59265af85 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.c
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_qcn9274.c
@@ -950,6 +950,15 @@ void ath12k_hal_extract_rx_desc_data_qcn9274(struct hal_rx_desc_data *rx_desc_da
 	rx_desc_data->err_bitmap = ath12k_hal_rx_h_mpdu_err_qcn9274(rx_desc);
 }
 
+static u16 ath12k_hal_reo_status_dec_tlv_hdr_qcn9274(void *tlv, void **desc)
+{
+	u16 tag;
+
+	*desc = ath12k_hal_decode_tlv64_hdr(tlv, &tag, NULL, NULL);
+
+	return tag;
+}
+
 const struct ath12k_hw_hal_params ath12k_hw_hal_params_qcn9274 = {
 	.rx_buf_rbm = HAL_RX_BUF_RBM_SW3_BM,
 	.wbm2sw_cc_enable = HAL_WBM_SW_COOKIE_CONV_CFG_WBM2SW0_EN |
@@ -1138,5 +1147,7 @@ const struct hal_ops hal_qcn9274_ops = {
 	.rx_msdu_list_get = ath12k_wifi7_hal_rx_msdu_list_get,
 	.rx_reo_ent_buf_paddr_get = ath12k_wifi7_hal_rx_reo_ent_buf_paddr_get,
 	.reo_cmd_enc_tlv_hdr = ath12k_hal_encode_tlv64_hdr,
-	.reo_status_dec_tlv_hdr = ath12k_hal_decode_tlv64_hdr,
+	.reo_status_dec_tlv_hdr = ath12k_hal_reo_status_dec_tlv_hdr_qcn9274,
+	.mon_rx_status_dec_tlv_hdr = ath12k_hal_decode_tlv64_hdr,
+	.get_tlv_hdr_align = ath12k_hal_get_tlv64_hdr_align,
 };
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h
index 0d19a9cbb68ce..6d69851e529d8 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx_desc.h
@@ -140,6 +140,38 @@ struct rx_mpdu_start_qcn9274 {
 	__le32 res1;
 } __packed;
 
+struct rx_mpdu_start_qcc2072 {
+	__le32 info0;
+	__le32 info2;
+	__le32 reo_queue_desc_lo;
+	__le32 info1;
+	__le32 pn[4];
+	__le32 info4;
+	__le32 peer_meta_data;
+	__le16 ast_index;
+	__le16 sw_peer_id;
+	__le16 info3;
+	__le16 phy_ppdu_id;
+	__le32 info5;
+	__le32 info6;
+	__le16 frame_ctrl;
+	__le16 duration;
+	u8 addr1[ETH_ALEN];
+	u8 addr2[ETH_ALEN];
+	u8 addr3[ETH_ALEN];
+	__le16 seq_ctrl;
+	u8 addr4[ETH_ALEN];
+	__le16 qos_ctrl;
+	__le32 ht_ctrl;
+	__le32 info7;
+	__le32 res0;
+	__le32 res1;
+	__le32 res2;
+	__le32 info8;
+	__le32 res3;
+	__le32 res4;
+} __packed;
+
 #define QCN9274_MPDU_START_SELECT_MPDU_START_TAG			BIT(0)
 #define QCN9274_MPDU_START_SELECT_INFO0_REO_QUEUE_DESC_LO		BIT(1)
 #define QCN9274_MPDU_START_SELECT_INFO1_PN_31_0				BIT(2)
@@ -1492,7 +1524,7 @@ struct hal_rx_desc_qcc2072 {
 	struct rx_msdu_end_qcn9274 msdu_end;
 	u8 rx_padding0[RX_BE_PADDING0_BYTES];
 	__le32 mpdu_start_tag;
-	struct rx_mpdu_start_qcn9274 mpdu_start;
+	struct rx_mpdu_start_qcc2072 mpdu_start;
 	struct rx_pkt_hdr_tlv_qcc2072 pkt_hdr_tlv;
 	u8 msdu_payload[];
 };
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.c b/drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.c
index efbbc1cbd3e42..61be8443e46ee 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.c
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_wcn7850.c
@@ -756,6 +756,15 @@ int ath12k_hal_srng_create_config_wcn7850(struct ath12k_hal *hal)
 	return 0;
 }
 
+static u16 ath12k_hal_reo_status_dec_tlv_hdr_wcn7850(void *tlv, void **desc)
+{
+	u16 tag;
+
+	*desc = ath12k_hal_decode_tlv64_hdr(tlv, &tag, NULL, NULL);
+
+	return tag;
+}
+
 const struct ath12k_hal_tcl_to_wbm_rbm_map
 ath12k_hal_tcl_to_wbm_rbm_map_wcn7850[DP_TCL_NUM_RING_MAX] = {
 	{
@@ -821,5 +830,7 @@ const struct hal_ops hal_wcn7850_ops = {
 	.rx_msdu_list_get = ath12k_wifi7_hal_rx_msdu_list_get,
 	.rx_reo_ent_buf_paddr_get = ath12k_wifi7_hal_rx_reo_ent_buf_paddr_get,
 	.reo_cmd_enc_tlv_hdr = ath12k_hal_encode_tlv64_hdr,
-	.reo_status_dec_tlv_hdr = ath12k_hal_decode_tlv64_hdr,
+	.reo_status_dec_tlv_hdr = ath12k_hal_reo_status_dec_tlv_hdr_wcn7850,
+	.mon_rx_status_dec_tlv_hdr = ath12k_hal_decode_tlv64_hdr,
+	.get_tlv_hdr_align = ath12k_hal_get_tlv64_hdr_align,
 };
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hw.c b/drivers/net/wireless/ath/ath12k/wifi7/hw.c
index 3d59fa452ec03..685bb62e6cbcf 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/hw.c
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hw.c
@@ -158,6 +158,7 @@ static const struct ath12k_hw_ops qcn9274_ops = {
 	.get_ring_selector = ath12k_wifi7_hw_get_ring_selector_qcn9274,
 	.dp_srng_is_tx_comp_ring = ath12k_wifi7_dp_srng_is_comp_ring_qcn9274,
 	.is_frame_link_agnostic = ath12k_wifi7_is_frame_link_agnostic_qcn9274,
+	.set_rx_link_id = ath12k_wifi7_dp_rx_set_link_id_qcn9274,
 };
 
 static const struct ath12k_hw_ops wcn7850_ops = {
@@ -168,6 +169,7 @@ static const struct ath12k_hw_ops wcn7850_ops = {
 	.get_ring_selector = ath12k_wifi7_hw_get_ring_selector_wcn7850,
 	.dp_srng_is_tx_comp_ring = ath12k_wifi7_dp_srng_is_comp_ring_wcn7850,
 	.is_frame_link_agnostic = ath12k_wifi7_is_frame_link_agnostic_wcn7850,
+	.set_rx_link_id = ath12k_wifi7_dp_rx_set_link_id_wcn7850,
 };
 
 static const struct ath12k_hw_ops qcc2072_ops = {
@@ -178,6 +180,7 @@ static const struct ath12k_hw_ops qcc2072_ops = {
 	.get_ring_selector = ath12k_wifi7_hw_get_ring_selector_wcn7850,
 	.dp_srng_is_tx_comp_ring = ath12k_wifi7_dp_srng_is_comp_ring_wcn7850,
 	.is_frame_link_agnostic = ath12k_wifi7_is_frame_link_agnostic_wcn7850,
+	.set_rx_link_id = ath12k_wifi7_dp_rx_set_link_id_wcn7850,
 };
 
 #define ATH12K_TX_RING_MASK_0 0x1
@@ -689,7 +692,7 @@ static const struct ath12k_hw_params ath12k_wifi7_hw_params[] = {
 
 		.ce_ie_addr = &ath12k_wifi7_ce_ie_addr_ipq5332,
 		.ce_remap = &ath12k_wifi7_ce_remap_ipq5332,
-		.bdf_addr_offset = 0xC00000,
+		.bdf_addr_offset = 0x1A00000,
 
 		.dp_primary_link_only = true,
 		.client = {
@@ -903,6 +906,7 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw,
 	struct ethhdr *eth;
 	bool is_prb_rsp;
 	u16 mcbc_gsn;
+	u8 cb_flags;
 	u8 link_id;
 	int ret;
 	struct ath12k_dp *tmp_dp;
@@ -996,8 +1000,13 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw,
 	    ieee80211_has_protected(hdr->frame_control))
 		is_dvlan = true;
 
+	/*
+	 * Add a sta pointer check to differentiate multicast encapsulation
+	 * offload packets, as the ATH12K_SKB_HW_80211_ENCAP flag is also set
+	 * for such packets.
+	 */
 	if (!vif->valid_links || !is_mcast || is_dvlan ||
-	    (skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) ||
+	    ((skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP) && sta) ||
 	    test_bit(ATH12K_FLAG_RAW_MODE, &ar->ab->dev_flags)) {
 		ret = ath12k_wifi7_dp_tx(dp_pdev, arvif, arsta, skb, false, 0, is_mcast);
 		if (unlikely(ret)) {
@@ -1009,6 +1018,7 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw,
 		mcbc_gsn = atomic_inc_return(&ahvif->dp_vif.mcbc_gsn) & 0xfff;
 
 		links_map = ahvif->links_map;
+		cb_flags = skb_cb->flags;
 		for_each_set_bit(link_id, &links_map,
 				 IEEE80211_MLD_MAX_NUM_LINKS) {
 			tmp_arvif = rcu_dereference(ahvif->link[link_id]);
@@ -1016,21 +1026,49 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw,
 				continue;
 
 			tmp_ar = tmp_arvif->ar;
-			tmp_dp_pdev = ath12k_dp_to_pdev_dp(tmp_ar->ab->dp,
+			tmp_dp = ath12k_ab_to_dp(tmp_ar->ab);
+			tmp_dp_pdev = ath12k_dp_to_pdev_dp(tmp_dp,
 							   tmp_ar->pdev_idx);
 			if (!tmp_dp_pdev)
 				continue;
-			msdu_copied = skb_copy(skb, GFP_ATOMIC);
-			if (!msdu_copied) {
-				ath12k_err(ar->ab,
-					   "skb copy failure link_id 0x%X vdevid 0x%X\n",
-					   link_id, tmp_arvif->vdev_id);
-				continue;
-			}
 
-			ath12k_mlo_mcast_update_tx_link_address(vif, link_id,
-								msdu_copied,
-								info_flags);
+			if (cb_flags & ATH12K_SKB_HW_80211_ENCAP) {
+				/*
+				 * skb->data may be modified for the
+				 * iova_mask devices. It is better to
+				 * use skb_copy() for such devices to
+				 * avoid any potential skb corruption
+				 * related issues.
+				 */
+				if (tmp_dp->hw_params->iova_mask) {
+					msdu_copied = skb_copy(skb, GFP_ATOMIC);
+				} else {
+					/*
+					 * ath12k_wifi7_dp_tx() should
+					 * treat cloned HW-encap Ethernet
+					 * multicast frames as read-only.
+					 */
+					msdu_copied = skb_clone(skb, GFP_ATOMIC);
+				}
+				if (!msdu_copied) {
+					ath12k_err(ar->ab,
+						   "skb copy/clone failure link_id 0x%X vdevid 0x%X\n",
+						   link_id, tmp_arvif->vdev_id);
+					continue;
+				}
+			} else {
+				msdu_copied = skb_copy(skb, GFP_ATOMIC);
+				if (!msdu_copied) {
+					ath12k_err(ar->ab,
+						   "skb copy failure link_id 0x%X vdevid 0x%X\n",
+						   link_id, tmp_arvif->vdev_id);
+					continue;
+				}
+
+				ath12k_mlo_mcast_update_tx_link_address(vif, link_id,
+									msdu_copied,
+									info_flags);
+			}
 
 			skb_cb = ATH12K_SKB_CB(msdu_copied);
 			skb_cb->link_id = link_id;
@@ -1046,7 +1084,6 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw,
 			if (unlikely(!ahvif->dp_vif.key_cipher))
 				goto skip_peer_find;
 
-			tmp_dp = ath12k_ab_to_dp(tmp_ar->ab);
 			spin_lock_bh(&tmp_dp->dp_lock);
 			peer = ath12k_dp_link_peer_find_by_addr(tmp_dp,
 								tmp_arvif->bssid);
@@ -1065,11 +1102,16 @@ static void ath12k_wifi7_mac_op_tx(struct ieee80211_hw *hw,
 				skb_cb->cipher = key->cipher;
 				skb_cb->flags |= ATH12K_SKB_CIPHER_SET;
 
+				if (skb_cb->flags & ATH12K_SKB_HW_80211_ENCAP)
+					goto skip_fctl_protected_check;
+
 				hdr = (struct ieee80211_hdr *)msdu_copied->data;
 				if (!ieee80211_has_protected(hdr->frame_control))
 					hdr->frame_control |=
 						cpu_to_le16(IEEE80211_FCTL_PROTECTED);
 			}
+
+skip_fctl_protected_check:
 			spin_unlock_bh(&tmp_dp->dp_lock);
 
 skip_peer_find:
diff --git a/drivers/net/wireless/ath/ath12k/wmi.c b/drivers/net/wireless/ath/ath12k/wmi.c
index 84a31b953db81..614e02dbb6f92 100644
--- a/drivers/net/wireless/ath/ath12k/wmi.c
+++ b/drivers/net/wireless/ath/ath12k/wmi.c
@@ -1228,10 +1228,16 @@ int ath12k_wmi_vdev_start(struct ath12k *ar, struct wmi_vdev_start_req_arg *arg,
 				   le32_encode_bits(arg->ml.mcast_link,
 						    ATH12K_WMI_FLAG_MLO_MCAST_VDEV) |
 				   le32_encode_bits(arg->ml.link_add,
-						    ATH12K_WMI_FLAG_MLO_LINK_ADD);
+						    ATH12K_WMI_FLAG_MLO_LINK_ADD) |
+				   le32_encode_bits(arg->ml.assoc_link,
+						    ATH12K_WMI_FLAG_MLO_START_AS_ACTIVE) |
+				   cpu_to_le32(ATH12K_WMI_FLAG_MLO_IEEE_LINK_IDX_VALID);
+
+		ml_params->ieee_link_id = cpu_to_le32(arg->ml.ieee_link_id);
 
-		ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "vdev %d start ml flags 0x%x\n",
-			   arg->vdev_id, ml_params->flags);
+		ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "vdev %u start link_id %u ml flags 0x%x\n",
+			   arg->vdev_id, arg->ml.ieee_link_id,
+			   le32_to_cpu(ml_params->flags));
 
 		ptr += sizeof(*ml_params);
 
@@ -1244,19 +1250,23 @@ int ath12k_wmi_vdev_start(struct ath12k *ar, struct wmi_vdev_start_req_arg *arg,
 		partner_info = ptr;
 
 		for (i = 0; i < arg->ml.num_partner_links; i++) {
+			struct wmi_ml_partner_info *pinfo = &arg->ml.partner_info[i];
+
 			partner_info->tlv_header =
 				ath12k_wmi_tlv_cmd_hdr(WMI_TAG_MLO_PARTNER_LINK_PARAMS,
 						       sizeof(*partner_info));
-			partner_info->vdev_id =
-				cpu_to_le32(arg->ml.partner_info[i].vdev_id);
-			partner_info->hw_link_id =
-				cpu_to_le32(arg->ml.partner_info[i].hw_link_id);
+			partner_info->vdev_id = cpu_to_le32(pinfo->vdev_id);
+			partner_info->hw_link_id = cpu_to_le32(pinfo->hw_link_id);
 			ether_addr_copy(partner_info->vdev_addr.addr,
-					arg->ml.partner_info[i].addr);
+					pinfo->addr);
+			partner_info->flags =
+				cpu_to_le32(ATH12K_WMI_FLAG_MLO_IEEE_LINK_IDX_VALID_PARTNER);
+			partner_info->ieee_link_id = cpu_to_le32(pinfo->ieee_link_id);
 
-			ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "partner vdev %d hw_link_id %d macaddr%pM\n",
-				   partner_info->vdev_id, partner_info->hw_link_id,
-				   partner_info->vdev_addr.addr);
+			ath12k_dbg(ar->ab, ATH12K_DBG_WMI, "partner vdev %u hw_link_id %u macaddr %pM link_id %u ml flags 0x%x\n",
+				   pinfo->vdev_id, pinfo->hw_link_id,
+				   pinfo->addr, pinfo->ieee_link_id,
+				   le32_to_cpu(partner_info->flags));
 
 			partner_info++;
 		}
@@ -5154,6 +5164,7 @@ static void ath12k_wmi_eht_caps_parse(struct ath12k_pdev *pdev, u32 band,
 				       __le32 cap_info_internal)
 {
 	struct ath12k_band_cap *cap_band = &pdev->cap.band[band];
+	u8 *phy_cap = (u8 *)&cap_band->eht_cap_phy_info[0];
 	u32 support_320mhz;
 	u8 i;
 
@@ -5167,8 +5178,22 @@ static void ath12k_wmi_eht_caps_parse(struct ath12k_pdev *pdev, u32 band,
 	for (i = 0; i < WMI_MAX_EHTCAP_PHY_SIZE; i++)
 		cap_band->eht_cap_phy_info[i] = le32_to_cpu(cap_phy_info[i]);
 
-	if (band == NL80211_BAND_6GHZ)
+	if (band == NL80211_BAND_6GHZ) {
 		cap_band->eht_cap_phy_info[0] |= support_320mhz;
+	} else {
+		/*
+		 * Firmware may report 6 GHz/320 MHz specific capabilities for
+		 * non-6 GHz bands, so explicitly clear them.
+		 */
+		phy_cap[0] &= ~IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ;
+		phy_cap[1] &= ~IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK;
+		phy_cap[2] &= ~IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_320MHZ_MASK;
+		phy_cap[3] &= ~IEEE80211_EHT_PHY_CAP3_SOUNDING_DIM_320MHZ_MASK;
+		phy_cap[6] &= ~IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_320MHZ;
+		phy_cap[6] &= ~IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP;
+		phy_cap[7] &= ~IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_320MHZ;
+		phy_cap[7] &= ~IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_320MHZ;
+	}
 
 	cap_band->eht_mcs_20_only = le32_to_cpu(supp_mcs[0]);
 	cap_band->eht_mcs_80 = le32_to_cpu(supp_mcs[1]);
@@ -7072,25 +7097,29 @@ static void ath12k_peer_delete_resp_event(struct ath12k_base *ab, struct sk_buff
 {
 	struct wmi_peer_delete_resp_event peer_del_resp;
 	struct ath12k *ar;
+	u32 vdev_id;
 
 	if (ath12k_pull_peer_del_resp_ev(ab, skb, &peer_del_resp) != 0) {
-		ath12k_warn(ab, "failed to extract peer delete resp");
+		ath12k_warn(ab, "failed to extract peer delete resp\n");
 		return;
 	}
 
+	vdev_id = le32_to_cpu(peer_del_resp.vdev_id);
+
 	rcu_read_lock();
-	ar = ath12k_mac_get_ar_by_vdev_id(ab, le32_to_cpu(peer_del_resp.vdev_id));
+	ar = ath12k_mac_get_ar_by_vdev_id(ab, vdev_id);
 	if (!ar) {
-		ath12k_warn(ab, "invalid vdev id in peer delete resp ev %d",
-			    peer_del_resp.vdev_id);
+		ath12k_warn(ab, "invalid vdev id in peer delete resp ev %d\n",
+			    vdev_id);
 		rcu_read_unlock();
 		return;
 	}
 
-	complete(&ar->peer_delete_done);
+	ath12k_peer_delete_resp_signal(ar, vdev_id,
+				       peer_del_resp.peer_macaddr.addr);
 	rcu_read_unlock();
 	ath12k_dbg(ab, ATH12K_DBG_WMI, "peer delete resp for vdev id %d addr %pM\n",
-		   peer_del_resp.vdev_id, peer_del_resp.peer_macaddr.addr);
+		   vdev_id, peer_del_resp.peer_macaddr.addr);
 }
 
 static void ath12k_vdev_delete_resp_event(struct ath12k_base *ab,
diff --git a/drivers/net/wireless/ath/ath12k/wmi.h b/drivers/net/wireless/ath/ath12k/wmi.h
index c452e3d57a29a..20e3939e8820b 100644
--- a/drivers/net/wireless/ath/ath12k/wmi.h
+++ b/drivers/net/wireless/ath/ath12k/wmi.h
@@ -2954,10 +2954,14 @@ struct wmi_vdev_create_mlo_params {
 #define ATH12K_WMI_FLAG_MLO_EMLSR_SUPPORT		BIT(6)
 #define ATH12K_WMI_FLAG_MLO_FORCED_INACTIVE		BIT(7)
 #define ATH12K_WMI_FLAG_MLO_LINK_ADD			BIT(8)
+#define ATH12K_WMI_FLAG_MLO_START_AS_ACTIVE		BIT(17)
+#define ATH12K_WMI_FLAG_MLO_IEEE_LINK_IDX_VALID		BIT(18)
+#define ATH12K_WMI_FLAG_MLO_IEEE_LINK_IDX_VALID_PARTNER	BIT(19)
 
 struct wmi_vdev_start_mlo_params {
 	__le32 tlv_header;
 	__le32 flags;
+	__le32 ieee_link_id;
 } __packed;
 
 struct wmi_partner_link_info {
@@ -2965,6 +2969,8 @@ struct wmi_partner_link_info {
 	__le32 vdev_id;
 	__le32 hw_link_id;
 	struct ath12k_wmi_mac_addr_params vdev_addr;
+	__le32 flags;
+	__le32 ieee_link_id;
 } __packed;
 
 struct wmi_vdev_delete_cmd {
@@ -3120,6 +3126,7 @@ struct wmi_ml_partner_info {
 	bool primary_umac;
 	bool logical_link_idx_valid;
 	u32 logical_link_idx;
+	u32 ieee_link_id;
 };
 
 struct wmi_ml_arg {
@@ -3127,6 +3134,7 @@ struct wmi_ml_arg {
 	bool assoc_link;
 	bool mcast_link;
 	bool link_add;
+	u32 ieee_link_id;
 	u8 num_partner_links;
 	struct wmi_ml_partner_info partner_info[ATH12K_WMI_MLO_MAX_LINKS];
 };
diff --git a/drivers/net/wireless/ath/ath6kl/txrx.c b/drivers/net/wireless/ath/ath6kl/txrx.c
index 97fdac7237e26..d818254139061 100644
--- a/drivers/net/wireless/ath/ath6kl/txrx.c
+++ b/drivers/net/wireless/ath/ath6kl/txrx.c
@@ -1723,13 +1723,15 @@ void aggr_recv_addba_req_evt(struct ath6kl_vif *vif, u8 tid_mux, u16 seq_no,
 
 	rxtid = &aggr_conn->rx_tid[tid];
 
-	if (win_sz < AGGR_WIN_SZ_MIN || win_sz > AGGR_WIN_SZ_MAX)
-		ath6kl_dbg(ATH6KL_DBG_WLAN_RX, "%s: win_sz %d, tid %d\n",
-			   __func__, win_sz, tid);
-
 	if (rxtid->aggr)
 		aggr_delete_tid_state(aggr_conn, tid);
 
+	if (win_sz < AGGR_WIN_SZ_MIN || win_sz > AGGR_WIN_SZ_MAX) {
+		ath6kl_dbg(ATH6KL_DBG_WLAN_RX, "%s: win_sz %d, tid %d\n",
+			   __func__, win_sz, tid);
+		return;
+	}
+
 	rxtid->seq_next = seq_no;
 	hold_q_size = TID_WINDOW_SZ(win_sz) * sizeof(struct skb_hold_q);
 	rxtid->hold_q = kzalloc(hold_q_size, GFP_KERNEL);
@@ -1828,7 +1830,7 @@ void aggr_reset_state(struct aggr_info_conn *aggr_conn)
 		return;
 
 	if (aggr_conn->timer_scheduled) {
-		timer_delete(&aggr_conn->timer);
+		timer_delete_sync(&aggr_conn->timer);
 		aggr_conn->timer_scheduled = false;
 	}
 
diff --git a/drivers/net/wireless/ath/ath6kl/wmi.c b/drivers/net/wireless/ath/ath6kl/wmi.c
index 72611a2ceb9d8..6c29f0bcec9f5 100644
--- a/drivers/net/wireless/ath/ath6kl/wmi.c
+++ b/drivers/net/wireless/ath/ath6kl/wmi.c
@@ -484,6 +484,18 @@ static int ath6kl_wmi_tx_complete_event_rx(u8 *datap, int len)
 
 	evt = (struct wmi_tx_complete_event *) datap;
 
+	if (len < sizeof(*evt)) {
+		ath6kl_dbg(ATH6KL_DBG_WMI, "tx complete: invalid len %d\n",
+			   len);
+		return -EINVAL;
+	}
+
+	if (len < sizeof(*evt) + evt->num_msg * sizeof(struct tx_complete_msg_v1)) {
+		ath6kl_dbg(ATH6KL_DBG_WMI, "tx complete: invalid len %d for %u msgs\n",
+			   len, evt->num_msg);
+		return -EINVAL;
+	}
+
 	ath6kl_dbg(ATH6KL_DBG_WMI, "comp: %d %d %d\n",
 		   evt->num_msg, evt->msg_len, evt->msg_type);
 
@@ -862,6 +874,14 @@ static int ath6kl_wmi_connect_event_rx(struct wmi *wmi, u8 *datap, int len,
 
 	ev = (struct wmi_connect_event *) datap;
 
+	if (len < sizeof(*ev) + ev->beacon_ie_len +
+	    ev->assoc_req_len + ev->assoc_resp_len) {
+		ath6kl_dbg(ATH6KL_DBG_WMI,
+			   "connect event: IE lengths %u+%u+%u exceed buffer %d\n",
+			   ev->beacon_ie_len, ev->assoc_req_len,
+			   ev->assoc_resp_len, len);
+		return -EINVAL;
+	}
 	if (vif->nw_type == AP_NETWORK) {
 		/* AP mode start/STA connected event */
 		struct net_device *dev = vif->ndev;
@@ -1276,6 +1296,9 @@ static int ath6kl_wmi_scan_complete_rx(struct wmi *wmi, u8 *datap, int len,
 {
 	struct wmi_scan_complete_event *ev;
 
+	if (len < sizeof(*ev))
+		return -EINVAL;
+
 	ev = (struct wmi_scan_complete_event *) datap;
 
 	ath6kl_scan_complete_evt(vif, a_sle32_to_cpu(ev->status));
@@ -3352,7 +3375,12 @@ static int ath6kl_wmi_get_pmkid_list_event_rx(struct wmi *wmi, u8 *datap,
 static int ath6kl_wmi_addba_req_event_rx(struct wmi *wmi, u8 *datap, int len,
 					 struct ath6kl_vif *vif)
 {
-	struct wmi_addba_req_event *cmd = (struct wmi_addba_req_event *) datap;
+	struct wmi_addba_req_event *cmd;
+
+	if (len < sizeof(*cmd))
+		return -EINVAL;
+
+	cmd = (struct wmi_addba_req_event *)datap;
 
 	aggr_recv_addba_req_evt(vif, cmd->tid,
 				le16_to_cpu(cmd->st_seq_no), cmd->win_sz);
@@ -3363,7 +3391,12 @@ static int ath6kl_wmi_addba_req_event_rx(struct wmi *wmi, u8 *datap, int len,
 static int ath6kl_wmi_delba_req_event_rx(struct wmi *wmi, u8 *datap, int len,
 					 struct ath6kl_vif *vif)
 {
-	struct wmi_delba_event *cmd = (struct wmi_delba_event *) datap;
+	struct wmi_delba_event *cmd;
+
+	if (len < sizeof(*cmd))
+		return -EINVAL;
+
+	cmd = (struct wmi_delba_event *)datap;
 
 	aggr_recv_delba_req_evt(vif, cmd->tid);
 
diff --git a/drivers/net/wireless/ath/ath9k/hif_usb.c b/drivers/net/wireless/ath/ath9k/hif_usb.c
index 515267f48d803..d3491ff08e6ef 100644
--- a/drivers/net/wireless/ath/ath9k/hif_usb.c
+++ b/drivers/net/wireless/ath/ath9k/hif_usb.c
@@ -1087,7 +1087,7 @@ static int ath9k_hif_usb_download_fw(struct hif_device_usb *hif_dev)
 	}
 	kfree(buf);
 
-	if (IS_AR7010_DEVICE(hif_dev->usb_device_id->driver_info))
+	if (IS_AR7010_DEVICE(hif_dev->id_info))
 		firm_offset = AR7010_FIRMWARE_TEXT;
 	else
 		firm_offset = AR9271_FIRMWARE_TEXT;
@@ -1182,7 +1182,7 @@ static int ath9k_hif_request_firmware(struct hif_device_usb *hif_dev,
 	if (MAJOR_VERSION_REQ == 1 && hif_dev->fw_minor_index == 3) {
 		const char *filename;
 
-		if (IS_AR7010_DEVICE(hif_dev->usb_device_id->driver_info))
+		if (IS_AR7010_DEVICE(hif_dev->id_info))
 			filename = FIRMWARE_AR7010_1_1;
 		else
 			filename = FIRMWARE_AR9271;
@@ -1198,7 +1198,7 @@ static int ath9k_hif_request_firmware(struct hif_device_usb *hif_dev,
 
 		return -ENOENT;
 	} else {
-		if (IS_AR7010_DEVICE(hif_dev->usb_device_id->driver_info))
+		if (IS_AR7010_DEVICE(hif_dev->id_info))
 			chip = "7010";
 		else
 			chip = "9271";
@@ -1215,15 +1215,10 @@ static int ath9k_hif_request_firmware(struct hif_device_usb *hif_dev,
 	ret = request_firmware_nowait(THIS_MODULE, true, hif_dev->fw_name,
 				      &hif_dev->udev->dev, GFP_KERNEL,
 				      hif_dev, ath9k_hif_usb_firmware_cb);
-	if (ret) {
+	if (ret)
 		dev_err(&hif_dev->udev->dev,
 			"ath9k_htc: Async request for firmware %s failed\n",
 			hif_dev->fw_name);
-		return ret;
-	}
-
-	dev_info(&hif_dev->udev->dev, "ath9k_htc: Firmware %s requested\n",
-		 hif_dev->fw_name);
 
 	return ret;
 }
@@ -1260,9 +1255,9 @@ static void ath9k_hif_usb_firmware_cb(const struct firmware *fw, void *context)
 
 	ret = ath9k_htc_hw_init(hif_dev->htc_handle,
 				&hif_dev->interface->dev,
-				hif_dev->usb_device_id->idProduct,
+				le16_to_cpu(hif_dev->udev->descriptor.idProduct),
 				hif_dev->udev->product,
-				hif_dev->usb_device_id->driver_info);
+				hif_dev->id_info);
 	if (ret) {
 		ret = -EINVAL;
 		goto err_htc_hw_init;
@@ -1374,7 +1369,7 @@ static int ath9k_hif_usb_probe(struct usb_interface *interface,
 
 	hif_dev->udev = udev;
 	hif_dev->interface = interface;
-	hif_dev->usb_device_id = id;
+	hif_dev->id_info = id->driver_info;
 #ifdef CONFIG_PM
 	udev->reset_resume = 1;
 #endif
diff --git a/drivers/net/wireless/ath/ath9k/hif_usb.h b/drivers/net/wireless/ath/ath9k/hif_usb.h
index dc0b0fa5c3257..b3e7b0fb54b84 100644
--- a/drivers/net/wireless/ath/ath9k/hif_usb.h
+++ b/drivers/net/wireless/ath/ath9k/hif_usb.h
@@ -115,7 +115,7 @@ struct cmd_buf {
 struct hif_device_usb {
 	struct usb_device *udev;
 	struct usb_interface *interface;
-	const struct usb_device_id *usb_device_id;
+	int id_info;
 	const void *fw_data;
 	size_t fw_size;
 	struct completion fw_done;
diff --git a/drivers/net/wireless/ath/carl9170/rx.c b/drivers/net/wireless/ath/carl9170/rx.c
index 6833430130f4c..0383d5c9698bf 100644
--- a/drivers/net/wireless/ath/carl9170/rx.c
+++ b/drivers/net/wireless/ath/carl9170/rx.c
@@ -150,7 +150,8 @@ static void carl9170_cmd_callback(struct ar9170 *ar, u32 len, void *buffer)
 	spin_lock(&ar->cmd_lock);
 	if (ar->readbuf) {
 		if (len >= 4)
-			memcpy(ar->readbuf, buffer + 4, len - 4);
+			memcpy(ar->readbuf, buffer + 4,
+			       min_t(u32, len - 4, ar->readlen));
 
 		ar->readbuf = NULL;
 	}
@@ -917,7 +918,9 @@ static void carl9170_rx_stream(struct ar9170 *ar, void *buf, unsigned int len)
 				}
 			}
 
-			skb_put_data(ar->rx_failover, tbuf, tlen);
+			skb_put_data(ar->rx_failover, tbuf,
+				     min_t(unsigned int, tlen,
+					   ar->rx_failover_missing));
 			ar->rx_failover_missing -= tlen;
 
 			if (ar->rx_failover_missing <= 0) {
diff --git a/drivers/net/wireless/ath/carl9170/tx.c b/drivers/net/wireless/ath/carl9170/tx.c
index 59caf1e4b1589..06aaf281655b1 100644
--- a/drivers/net/wireless/ath/carl9170/tx.c
+++ b/drivers/net/wireless/ath/carl9170/tx.c
@@ -692,7 +692,7 @@ void carl9170_tx_process_status(struct ar9170 *ar,
 	unsigned int i;
 
 	for (i = 0;  i < cmd->hdr.ext; i++) {
-		if (WARN_ON(i > ((cmd->hdr.len / 2) + 1))) {
+		if (WARN_ON(i >= (cmd->hdr.len / 2))) {
 			print_hex_dump_bytes("UU:", DUMP_PREFIX_NONE,
 					     (void *) cmd, cmd->hdr.len + 4);
 			break;
diff --git a/drivers/net/wireless/ath/wil6210/cfg80211.c b/drivers/net/wireless/ath/wil6210/cfg80211.c
index d6ef92cfcbaff..5f2bd9a31faf6 100644
--- a/drivers/net/wireless/ath/wil6210/cfg80211.c
+++ b/drivers/net/wireless/ath/wil6210/cfg80211.c
@@ -2326,7 +2326,7 @@ static void wil_probe_client_handle(struct wil6210_priv *wil,
 	 */
 	bool alive = (sta->status == wil_sta_connected);
 
-	cfg80211_probe_status(ndev, sta->addr, req->cookie, alive,
+	cfg80211_probe_status(ndev, sta->addr, req->cookie, -1, alive,
 			      0, false, GFP_KERNEL);
 }
 
@@ -2379,9 +2379,9 @@ void wil_probe_client_flush(struct wil6210_vif *vif)
 	mutex_unlock(&vif->probe_client_mutex);
 }
 
-static int wil_cfg80211_probe_client(struct wiphy *wiphy,
-				     struct net_device *dev,
-				     const u8 *peer, u64 *cookie)
+static int wil_cfg80211_probe_peer(struct wiphy *wiphy,
+				   struct net_device *dev,
+				   const u8 *peer, u64 *cookie)
 {
 	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
 	struct wil6210_vif *vif = ndev_to_vif(dev);
@@ -2660,7 +2660,7 @@ static const struct cfg80211_ops wil_cfg80211_ops = {
 	.add_station = wil_cfg80211_add_station,
 	.del_station = wil_cfg80211_del_station,
 	.change_station = wil_cfg80211_change_station,
-	.probe_client = wil_cfg80211_probe_client,
+	.probe_peer = wil_cfg80211_probe_peer,
 	.change_bss = wil_cfg80211_change_bss,
 	/* P2P device */
 	.start_p2p_device = wil_cfg80211_start_p2p_device,
diff --git a/drivers/net/wireless/broadcom/b43/debugfs.c b/drivers/net/wireless/broadcom/b43/debugfs.c
index acddae68947aa..31a1ff00c1a45 100644
--- a/drivers/net/wireless/broadcom/b43/debugfs.c
+++ b/drivers/net/wireless/broadcom/b43/debugfs.c
@@ -495,7 +495,6 @@ static ssize_t b43_debugfs_read(struct file *file, char __user *userbuf,
 	ssize_t ret;
 	char *buf;
 	const size_t bufsize = 1024 * 16; /* 16 kiB buffer */
-	const size_t buforder = get_order(bufsize);
 	int err = 0;
 
 	if (!count)
@@ -518,15 +517,14 @@ static ssize_t b43_debugfs_read(struct file *file, char __user *userbuf,
 	dfile = fops_to_dfs_file(dev, dfops);
 
 	if (!dfile->buffer) {
-		buf = (char *)__get_free_pages(GFP_KERNEL, buforder);
+		buf = kzalloc(bufsize, GFP_KERNEL);
 		if (!buf) {
 			err = -ENOMEM;
 			goto out_unlock;
 		}
-		memset(buf, 0, bufsize);
 		ret = dfops->read(dev, buf, bufsize);
 		if (ret <= 0) {
-			free_pages((unsigned long)buf, buforder);
+			kfree(buf);
 			err = ret;
 			goto out_unlock;
 		}
@@ -538,7 +536,7 @@ static ssize_t b43_debugfs_read(struct file *file, char __user *userbuf,
 				      dfile->buffer,
 				      dfile->data_len);
 	if (*ppos >= dfile->data_len) {
-		free_pages((unsigned long)dfile->buffer, buforder);
+		kfree(dfile->buffer);
 		dfile->buffer = NULL;
 		dfile->data_len = 0;
 	}
@@ -577,7 +575,7 @@ static ssize_t b43_debugfs_write(struct file *file,
 		goto out_unlock;
 	}
 
-	buf = (char *)get_zeroed_page(GFP_KERNEL);
+	buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!buf) {
 		err = -ENOMEM;
 		goto out_unlock;
@@ -591,7 +589,7 @@ static ssize_t b43_debugfs_write(struct file *file,
 		goto out_freepage;
 
 out_freepage:
-	free_page((unsigned long)buf);
+	kfree(buf);
 out_unlock:
 	mutex_unlock(&dev->wl->mutex);
 
diff --git a/drivers/net/wireless/broadcom/b43legacy/debugfs.c b/drivers/net/wireless/broadcom/b43legacy/debugfs.c
index 3ad99124d5227..a04d90d7307cf 100644
--- a/drivers/net/wireless/broadcom/b43legacy/debugfs.c
+++ b/drivers/net/wireless/broadcom/b43legacy/debugfs.c
@@ -192,7 +192,6 @@ static ssize_t b43legacy_debugfs_read(struct file *file, char __user *userbuf,
 	ssize_t ret;
 	char *buf;
 	const size_t bufsize = 1024 * 16; /* 16 KiB buffer */
-	const size_t buforder = get_order(bufsize);
 	int err = 0;
 
 	if (!count)
@@ -215,12 +214,11 @@ static ssize_t b43legacy_debugfs_read(struct file *file, char __user *userbuf,
 	dfile = fops_to_dfs_file(dev, dfops);
 
 	if (!dfile->buffer) {
-		buf = (char *)__get_free_pages(GFP_KERNEL, buforder);
+		buf = kzalloc(bufsize, GFP_KERNEL);
 		if (!buf) {
 			err = -ENOMEM;
 			goto out_unlock;
 		}
-		memset(buf, 0, bufsize);
 		if (dfops->take_irqlock) {
 			spin_lock_irq(&dev->wl->irq_lock);
 			ret = dfops->read(dev, buf, bufsize);
@@ -228,7 +226,7 @@ static ssize_t b43legacy_debugfs_read(struct file *file, char __user *userbuf,
 		} else
 			ret = dfops->read(dev, buf, bufsize);
 		if (ret <= 0) {
-			free_pages((unsigned long)buf, buforder);
+			kfree(buf);
 			err = ret;
 			goto out_unlock;
 		}
@@ -240,7 +238,7 @@ static ssize_t b43legacy_debugfs_read(struct file *file, char __user *userbuf,
 				      dfile->buffer,
 				      dfile->data_len);
 	if (*ppos >= dfile->data_len) {
-		free_pages((unsigned long)dfile->buffer, buforder);
+		kfree(dfile->buffer);
 		dfile->buffer = NULL;
 		dfile->data_len = 0;
 	}
@@ -279,7 +277,7 @@ static ssize_t b43legacy_debugfs_write(struct file *file,
 		goto out_unlock;
 	}
 
-	buf = (char *)get_zeroed_page(GFP_KERNEL);
+	buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!buf) {
 		err = -ENOMEM;
 		goto out_unlock;
@@ -298,7 +296,7 @@ static ssize_t b43legacy_debugfs_write(struct file *file,
 		goto out_freepage;
 
 out_freepage:
-	free_page((unsigned long)buf);
+	kfree(buf);
 out_unlock:
 	mutex_unlock(&dev->wl->mutex);
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c
index ce09d44fa73cf..873754be5174b 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cyw/core.c
@@ -293,6 +293,12 @@ brcmf_notify_auth_frame_rx(struct brcmf_if *ifp,
 		return -EINVAL;
 	}
 
+	if (mgmt_frame_len < offsetof(struct ieee80211_mgmt, u)) {
+		bphy_err(drvr, "Event %s (%d) frame too small. Ignore\n",
+			 brcmf_fweh_event_name(e->event_code), e->event_code);
+		return -EINVAL;
+	}
+
 	wdev = &ifp->vif->wdev;
 	WARN_ON(!wdev);
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
index 8fb595733b9c3..b725c64e5b5c6 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
@@ -4465,6 +4465,7 @@ int brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev)
 	bus->sdiodev = sdiodev;
 	sdiodev->bus = bus;
 	skb_queue_head_init(&bus->glom);
+	INIT_WORK(&bus->datawork, brcmf_sdio_dataworker);
 	bus->txbound = BRCMF_TXBOUND;
 	bus->rxbound = BRCMF_RXBOUND;
 	bus->txminmax = BRCMF_TXMINMAX;
@@ -4479,7 +4480,6 @@ int brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev)
 		goto fail;
 	}
 	brcmf_sdiod_freezer_count(sdiodev);
-	INIT_WORK(&bus->datawork, brcmf_sdio_dataworker);
 	bus->brcmf_wq = wq;
 
 	/* attempt to attach to the dongle */
diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2100.c b/drivers/net/wireless/intel/ipw2x00/ipw2100.c
index c11428485dccf..2b8a23865bfb2 100644
--- a/drivers/net/wireless/intel/ipw2x00/ipw2100.c
+++ b/drivers/net/wireless/intel/ipw2x00/ipw2100.c
@@ -6157,6 +6157,8 @@ static int ipw2100_pci_init_one(struct pci_dev *pci_dev,
 	if (err) {
 		printk(KERN_WARNING DRV_NAME
 		       "Error calling pci_enable_device.\n");
+		free_libipw(dev, 0);
+		pci_iounmap(pci_dev, ioaddr);
 		return err;
 	}
 
@@ -6169,16 +6171,14 @@ static int ipw2100_pci_init_one(struct pci_dev *pci_dev,
 	if (err) {
 		printk(KERN_WARNING DRV_NAME
 		       "Error calling pci_set_dma_mask.\n");
-		pci_disable_device(pci_dev);
-		return err;
+		goto fail;
 	}
 
 	err = pci_request_regions(pci_dev, DRV_NAME);
 	if (err) {
 		printk(KERN_WARNING DRV_NAME
 		       "Error calling pci_request_regions.\n");
-		pci_disable_device(pci_dev);
-		return err;
+		goto fail;
 	}
 
 	/* We disable the RETRY_TIMEOUT register (0x41) to keep
diff --git a/drivers/net/wireless/intel/ipw2x00/libipw_rx.c b/drivers/net/wireless/intel/ipw2x00/libipw_rx.c
index b7bc94f7abd8a..c8841f9b9ad91 100644
--- a/drivers/net/wireless/intel/ipw2x00/libipw_rx.c
+++ b/drivers/net/wireless/intel/ipw2x00/libipw_rx.c
@@ -414,7 +414,7 @@ int libipw_rx(struct libipw_device *ieee, struct sk_buff *skb,
 	    ieee->host_mc_decrypt : ieee->host_decrypt;
 
 	if (can_be_decrypted) {
-		if (skb->len >= hdrlen + 3) {
+		if (skb->len >= hdrlen + 4) {
 			/* Top two-bits of byte 3 are the key index */
 			keyidx = skb->data[hdrlen + 3] >> 6;
 		}
@@ -660,7 +660,7 @@ int libipw_rx(struct libipw_device *ieee, struct sk_buff *skb,
 		int trimlen = 0;
 
 		/* Top two-bits of byte 3 are the key index */
-		if (skb->len >= hdrlen + 3)
+		if (skb->len >= hdrlen + 4)
 			keyidx = skb->data[hdrlen + 3] >> 6;
 
 		/* To strip off any security data which appears before the
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/rf-pe.c b/drivers/net/wireless/intel/iwlwifi/cfg/rf-pe.c
index 7a04cb120b1b2..6080f5f23e698 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/rf-pe.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/rf-pe.c
@@ -15,6 +15,7 @@
 	.non_shared_ant = ANT_B,					\
 	.vht_mu_mimo_supported = true,					\
 	.uhb_supported = true,						\
+	.unii9_supported = true,					\
 	.eht_supported = true,						\
 	.uhr_supported = true,						\
 	.num_rbds = IWL_NUM_RBDS_EHT,					\
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/main.c b/drivers/net/wireless/intel/iwlwifi/dvm/main.c
index ca5a8140908af..6bd5b6d84b2a9 100644
--- a/drivers/net/wireless/intel/iwlwifi/dvm/main.c
+++ b/drivers/net/wireless/intel/iwlwifi/dvm/main.c
@@ -1511,10 +1511,10 @@ static struct iwl_op_mode *iwl_op_mode_dvm_start(struct iwl_trans *trans,
 	priv->workqueue = NULL;
 out_uninit_drv:
 	iwl_uninit_drv(priv);
-out_free_eeprom_blob:
-	kfree(priv->eeprom_blob);
 out_free_eeprom:
 	kfree(priv->nvm_data);
+out_free_eeprom_blob:
+	kfree(priv->eeprom_blob);
 out_leave_trans:
 	iwl_trans_op_mode_leave(priv->trans);
 out_free_hw:
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/tx.c b/drivers/net/wireless/intel/iwlwifi/dvm/tx.c
index a7806776a51eb..1a0167f67c7bd 100644
--- a/drivers/net/wireless/intel/iwlwifi/dvm/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/dvm/tx.c
@@ -604,8 +604,10 @@ int iwlagn_tx_agg_start(struct iwl_priv *priv, struct ieee80211_vif *vif,
 	}
 
 	ret = iwl_sta_tx_modify_enable_tid(priv, sta_id, tid);
-	if (ret)
+	if (ret) {
+		iwlagn_dealloc_agg_txq(priv, txq_id);
 		return ret;
+	}
 
 	spin_lock_bh(&priv->sta_lock);
 	tid_data = &priv->tid_data[sta_id][tid];
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/acpi.c b/drivers/net/wireless/intel/iwlwifi/fw/acpi.c
index bf0f851a90755..9f2f4a6af1ca2 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/acpi.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/acpi.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2017 Intel Deutschland GmbH
- * Copyright (C) 2019-2025 Intel Corporation
+ * Copyright (C) 2019-2026 Intel Corporation
  */
 #include <linux/uuid.h>
 #include "iwl-drv.h"
@@ -1188,6 +1188,8 @@ int iwl_acpi_get_wbem(struct iwl_fw_runtime *fwrt, u32 *value)
 
 	*value = wifi_pkg->package.elements[1].integer.value &
 		 IWL_ACPI_WBEM_REV0_MASK;
+	fwrt->wbem_source = BIOS_SOURCE_ACPI;
+	fwrt->wbem_revision = tbl_rev;
 	IWL_DEBUG_RADIO(fwrt, "Loaded WBEM config from ACPI\n");
 	ret = 0;
 out_free:
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/nvm-reg.h b/drivers/net/wireless/intel/iwlwifi/fw/api/nvm-reg.h
index 443a9a4163252..590c8510899c7 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/nvm-reg.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/nvm-reg.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -46,6 +46,11 @@ enum iwl_regulatory_and_nvm_subcmd_ids {
 	 */
 	MCC_ALLOWED_AP_TYPE_CMD = 0x5,
 
+	/**
+	 * @LARI_CONFIG_EXTENSION: &struct iwl_lari_config_extension_cmd
+	 */
+	LARI_CONFIG_EXTENSION = 0x8,
+
 	/**
 	 * @PNVM_INIT_COMPLETE_NTFY: &struct iwl_pnvm_init_complete_ntfy
 	 */
@@ -205,6 +210,7 @@ struct iwl_nvm_get_info_phy {
 
 #define IWL_NUM_CHANNELS_V1	51
 #define IWL_NUM_CHANNELS_V2	110
+#define IWL_NUM_CHANNELS_V3	115
 
 /**
  * struct iwl_nvm_get_info_regulatory_v1 - regulatory information
@@ -219,12 +225,12 @@ struct iwl_nvm_get_info_regulatory_v1 {
 } __packed; /* REGULATORY_NVM_GET_INFO_REGULATORY_S_VER_1 */
 
 /**
- * struct iwl_nvm_get_info_regulatory - regulatory information
+ * struct iwl_nvm_get_info_regulatory_v2 - regulatory information
  * @lar_enabled: is LAR enabled
  * @n_channels: number of valid channels in the array
  * @channel_profile: regulatory data of this channel
  */
-struct iwl_nvm_get_info_regulatory {
+struct iwl_nvm_get_info_regulatory_v2 {
 	__le32 lar_enabled;
 	__le32 n_channels;
 	__le32 channel_profile[IWL_NUM_CHANNELS_V2];
@@ -244,6 +250,32 @@ struct iwl_nvm_get_info_rsp_v3 {
 	struct iwl_nvm_get_info_regulatory_v1 regulatory;
 } __packed; /* REGULATORY_NVM_GET_INFO_RSP_API_S_VER_3 */
 
+/**
+ * struct iwl_nvm_get_info_rsp_v4 - response to get NVM data
+ * @general: general NVM data
+ * @mac_sku: data relating to MAC sku
+ * @phy_sku: data relating to PHY sku
+ * @regulatory: regulatory data
+ */
+struct iwl_nvm_get_info_rsp_v4 {
+	struct iwl_nvm_get_info_general general;
+	struct iwl_nvm_get_info_sku mac_sku;
+	struct iwl_nvm_get_info_phy phy_sku;
+	struct iwl_nvm_get_info_regulatory_v2 regulatory;
+} __packed; /* REGULATORY_NVM_GET_INFO_RSP_API_S_VER_4 */
+
+/**
+ * struct iwl_nvm_get_info_regulatory - regulatory information
+ * @lar_enabled: is LAR enabled
+ * @n_channels: number of valid channels in the array
+ * @channel_profile: regulatory data of this channel
+ */
+struct iwl_nvm_get_info_regulatory {
+	__le32 lar_enabled;
+	__le32 n_channels;
+	__le32 channel_profile[IWL_NUM_CHANNELS_V3];
+} __packed; /* REGULATORY_NVM_GET_INFO_REGULATORY_S_VER_3 */
+
 /**
  * struct iwl_nvm_get_info_rsp - response to get NVM data
  * @general: general NVM data
@@ -256,7 +288,7 @@ struct iwl_nvm_get_info_rsp {
 	struct iwl_nvm_get_info_sku mac_sku;
 	struct iwl_nvm_get_info_phy phy_sku;
 	struct iwl_nvm_get_info_regulatory regulatory;
-} __packed; /* REGULATORY_NVM_GET_INFO_RSP_API_S_VER_4 */
+} __packed; /* REGULATORY_NVM_GET_INFO_RSP_API_S_VER_5 */
 
 /**
  * struct iwl_nvm_access_complete_cmd - NVM_ACCESS commands are completed
@@ -514,6 +546,27 @@ struct iwl_bios_config_hdr {
 	u8 reserved[2];
 } __packed; /* BIOS_CONFIG_HDR_API_S_VER_1 */
 
+/**
+ * struct iwl_lari_config_extension_cmd - extend LARI configuration
+ *
+ * LARI_CONFIG_CHANGE's version must remain stable for frozen firmware.
+ * Because the driver might not know this version but still load that
+ * frozen FW and then send some default old version of LARI, causing the FW to
+ * assert about the bad size of it. To handle this, we do the following:
+ * 1. For newer firmware: increase the LARI_CONFIG_CHANGE version to support the
+ *	new firmware API with extra UHB bits.
+ * 2. For frozen firmware: add a special alternative API that doesn't require
+ *	modifying the frozen LARI_CONFIG_CHANGE's version.
+ * @dsm_table_hdr: BIOS DSM table source and revision
+ * @oem_uhb_allow_extension_bitmap: extension bitmap for OEM UHB config
+ * @reserved: reserved
+ */
+struct iwl_lari_config_extension_cmd {
+	struct iwl_bios_config_hdr dsm_table_hdr;
+	__le32 oem_uhb_allow_extension_bitmap;
+	__le32 reserved[10];
+} __packed; /* LARI_CONFIG_EXTENSION_CMD_API_S_VER_1 */
+
 /**
  * struct bios_value_u32 - BIOS configuration.
  * @hdr: bios config header
@@ -662,6 +715,12 @@ struct iwl_lari_config_change_cmd_v8 {
  *	get the data from the BIOS.
  * @oem_unii9_enable: UNII-9 enablement as read from the BIOS
  * @bios_hdr: bios config header
+ * @oem_uhb_allow_extension_bitmap: DSM Function 4 data as an extension of UHB
+ *	enabled MCC sets
+ * @bios_wcpe_hdr: puncturing config header
+ * @wcpe_bitmap: bitmap of puncturing enablement per MCC
+ * @bios_wbem_hdr: 320 MHz per-MCC WBEM config header
+ * @reserved: reserved
  */
 struct iwl_lari_config_change_cmd {
 	__le32 config_bitmap;
@@ -679,9 +738,16 @@ struct iwl_lari_config_change_cmd {
 	__le32 oem_unii9_enable;
 	/* since version 13 */
 	struct iwl_bios_config_hdr bios_hdr;
+	/* All the below are since version 14 */
+	__le32 oem_uhb_allow_extension_bitmap;
+	struct iwl_bios_config_hdr bios_wcpe_hdr;
+	__le32 wcpe_bitmap;
+	struct iwl_bios_config_hdr bios_wbem_hdr;
+	__le32 reserved[10];
 } __packed;
 /* LARI_CHANGE_CONF_CMD_S_VER_12
  * LARI_CHANGE_CONF_CMD_S_VER_13
+ * LARI_CHANGE_CONF_CMD_S_VER_14
  */
 
 /* Activate UNII-1 (5.2GHz) for World Wide */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h
index 08f4cc3ea1c30..ee78371082b09 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -1319,7 +1319,7 @@ struct iwl_scan_offload_match_info {
 	u8 resume_while_scanning;
 	u8 self_recovery;
 	__le16 reserved;
-	struct iwl_scan_offload_profile_match matches[];
+	struct iwl_scan_offload_profile_match matches[IWL_SCAN_MAX_PROFILES_V2];
 } __packed; /* SCAN_OFFLOAD_PROFILES_QUERY_RSP_S_VER_3 and
 	     * SCAN_OFFLOAD_MATCH_INFO_NOTIFICATION_S_VER_1
 	     */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dump.c b/drivers/net/wireless/intel/iwlwifi/fw/dump.c
index c2af66899a780..bbbf3669a555d 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/dump.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/dump.c
@@ -369,7 +369,7 @@ static void iwl_fwrt_dump_fseq_regs(struct iwl_fw_runtime *fwrt)
 void iwl_fwrt_dump_error_logs(struct iwl_fw_runtime *fwrt)
 {
 	struct iwl_pc_data *pc_data;
-	u8 count;
+	u32 count;
 
 	if (!iwl_trans_device_enabled(fwrt->trans)) {
 		IWL_ERR(fwrt,
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h b/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h
index 07f1240df8667..692bb17dfc331 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h
@@ -352,7 +352,7 @@ struct iwl_fw_ini_error_dump_register {
  * struct iwl_fw_ini_dump_cfg_name - configuration name
  * @image_type: image type the configuration is related to
  * @cfg_name_len: length of the configuration name
- * @cfg_name: name of the configuraiton
+ * @cfg_name: name of the configuration
  */
 struct iwl_fw_ini_dump_cfg_name {
 	__le32 image_type;
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/file.h b/drivers/net/wireless/intel/iwlwifi/fw/file.h
index a26ed82a8106f..664ac25754e9f 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/file.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/file.h
@@ -448,8 +448,9 @@ typedef unsigned int __bitwise iwl_ucode_tlv_capa_t;
  * @IWL_UCODE_TLV_CAPA_EXT_FSEQ_IMAGE_SUPPORT: external FSEQ image support
  * @IWL_UCODE_TLV_CAPA_RESET_DURING_ASSERT: FW reset handshake is needed
  *	during assert handling even if the dump isn't split
- * @IWL_UCODE_TLV_CAPA_FW_ACCEPTS_RAW_DSM_TABLE: Firmware has capability of
- *	handling raw DSM table data.
+ * @IWL_UCODE_TLV_CAPA_FW_ACCEPTS_RAW_DSM_TABLE: Firmware can handle raw DSM
+ *	table data. For LARI_CONFIG_CHANGE command version 13 and above, this
+ *	capability is obsolete since raw DSM values are accepted by default.
  * @IWL_UCODE_TLV_CAPA_NAN_SYNC_SUPPORT: Supports NAN synchronization
  *
  * @NUM_IWL_UCODE_TLV_CAPA: number of bits used
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/pnvm.c b/drivers/net/wireless/intel/iwlwifi/fw/pnvm.c
index afff8d51ca950..545a213df71d2 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/pnvm.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/pnvm.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright(c) 2020-2025 Intel Corporation
+ * Copyright(c) 2020-2026 Intel Corporation
  */
 
 #include "iwl-drv.h"
@@ -12,6 +12,7 @@
 #include "fw/api/alive.h"
 #include "fw/uefi.h"
 #include "fw/img.h"
+#include "fw/dbg.h"
 
 #define IWL_PNVM_REDUCED_CAP_BIT BIT(25)
 
@@ -26,6 +27,12 @@ static bool iwl_pnvm_complete_fn(struct iwl_notif_wait_data *notif_wait,
 	struct iwl_trans *trans = (struct iwl_trans *)data;
 	struct iwl_pnvm_init_complete_ntfy *pnvm_ntf = (void *)pkt->data;
 
+	if (IWL_FW_CHECK(trans,
+			 iwl_rx_packet_payload_len(pkt) < sizeof(*pnvm_ntf),
+			 "Bad notif len: %d\n",
+			 iwl_rx_packet_payload_len(pkt)))
+		return true;
+
 	IWL_DEBUG_FW(trans,
 		     "PNVM complete notification received with status 0x%0x\n",
 		     le32_to_cpu(pnvm_ntf->status));
@@ -198,6 +205,12 @@ static int iwl_pnvm_parse(struct iwl_trans *trans, const u8 *data,
 			IWL_DEBUG_FW(trans,
 				     "Got IWL_UCODE_TLV_PNVM_SKU len %d\n",
 				     tlv_len);
+			if (tlv_len < sizeof(*tlv_sku_id)) {
+				IWL_ERR(trans, "invalid PNVM SKU TLV len: %u\n",
+					tlv_len);
+				return -EINVAL;
+			}
+
 			IWL_DEBUG_FW(trans, "sku_id 0x%0x 0x%0x 0x%0x\n",
 				     le32_to_cpu(tlv_sku_id->data[0]),
 				     le32_to_cpu(tlv_sku_id->data[1]),
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c b/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c
index 8d9ff36e30f5e..1d6d38ee55b49 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c
@@ -389,7 +389,7 @@ bool iwl_add_mcc_to_tas_block_list(u16 *list, u8 *size, u16 mcc)
 	if (*size >= IWL_WTAS_BLACK_LIST_MAX)
 		return false;
 
-	list[*size++] = mcc;
+	list[(*size)++] = mcc;
 	return true;
 }
 IWL_EXPORT_SYMBOL(iwl_add_mcc_to_tas_block_list);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/runtime.h b/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
index d80ae610e56c3..ac01ac0092eea 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
  * Copyright (C) 2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2025 Intel Corporation
+ * Copyright (C) 2018-2026 Intel Corporation
  */
 #ifndef __iwl_fw_runtime_h__
 #define __iwl_fw_runtime_h__
@@ -144,6 +144,11 @@ struct iwl_txf_iter_data {
  * @tpc_enabled: TPC enabled
  * @dsm_source: one of &enum bios_source. UEFI, ACPI or NONE
  * @dsm_revision: the revision of the DSM table
+ * @wbem_source: one of &enum bios_source for the WBEM table
+ * @wbem_revision: the revision of the WBEM table
+ * @puncturing_source: one of &enum bios_source for the puncturing table
+ * @puncturing_revision: the revision of the puncturing table
+ * @bios_puncturing: per-country puncturing enablement bitmap from BIOS
  */
 struct iwl_fw_runtime {
 	struct iwl_trans *trans;
@@ -226,6 +231,12 @@ struct iwl_fw_runtime {
 	u32 dsm_funcs_valid;
 	u32 dsm_values[DSM_FUNC_NUM_FUNCS];
 #endif
+
+	enum bios_source wbem_source;
+	u8 wbem_revision;
+	enum bios_source puncturing_source;
+	u8 puncturing_revision;
+	u32 bios_puncturing;
 };
 
 void iwl_fw_runtime_init(struct iwl_fw_runtime *fwrt, struct iwl_trans *trans,
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/smem.c b/drivers/net/wireless/intel/iwlwifi/fw/smem.c
index 344ddde85b189..20ed26a1bb00e 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/smem.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/smem.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2021, 2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2021, 2025-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -14,9 +14,17 @@ static void iwl_parse_shared_mem_22000(struct iwl_fw_runtime *fwrt,
 {
 	struct iwl_shared_mem_cfg *mem_cfg = (void *)pkt->data;
 	int i, lmac;
-	int lmac_num = le32_to_cpu(mem_cfg->lmac_num);
-	u8 api_ver = iwl_fw_lookup_notif_ver(fwrt->fw, SYSTEM_GROUP,
-					     SHARED_MEM_CFG_CMD, 0);
+	int lmac_num;
+	u8 api_ver;
+
+	if (IWL_FW_CHECK(fwrt, iwl_rx_packet_payload_len(pkt) <
+			 offsetofend(struct iwl_shared_mem_cfg, lmac_smem[1]),
+			 "bad shared mem notification size\n"))
+		return;
+
+	lmac_num = le32_to_cpu(mem_cfg->lmac_num);
+	api_ver = iwl_fw_lookup_notif_ver(fwrt->fw, SYSTEM_GROUP,
+					  SHARED_MEM_CFG_CMD, 0);
 
 	/* Note: notification has 3 entries, but we only expect 2 */
 	if (IWL_FW_CHECK(fwrt, lmac_num > ARRAY_SIZE(fwrt->smem_cfg.lmac),
@@ -30,7 +38,7 @@ static void iwl_parse_shared_mem_22000(struct iwl_fw_runtime *fwrt,
 
 	if (api_ver >= 4 &&
 	    !IWL_FW_CHECK(fwrt, iwl_rx_packet_payload_len(pkt) < sizeof(*mem_cfg),
-			  "bad shared mem notification size\n")) {
+			  "bad shared mem notification size (v4)\n")) {
 		fwrt->smem_cfg.rxfifo2_control_size =
 			le32_to_cpu(mem_cfg->rxfifo2_control_size);
 	}
@@ -53,6 +61,11 @@ static void iwl_parse_shared_mem(struct iwl_fw_runtime *fwrt,
 	struct iwl_shared_mem_cfg_v2 *mem_cfg = (void *)pkt->data;
 	int i;
 
+	if (IWL_FW_CHECK(fwrt, iwl_rx_packet_payload_len(pkt) <
+			 offsetof(struct iwl_shared_mem_cfg_v2, rxfifo_addr),
+			 "bad shared mem notification size\n"))
+		return;
+
 	fwrt->smem_cfg.num_lmacs = 1;
 
 	fwrt->smem_cfg.num_txfifo_entries = ARRAY_SIZE(mem_cfg->txfifo_size);
@@ -67,6 +80,11 @@ static void iwl_parse_shared_mem(struct iwl_fw_runtime *fwrt,
 	/* new API has more data, from rxfifo_addr field and on */
 	if (fw_has_capa(&fwrt->fw->ucode_capa,
 			IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG)) {
+		if (IWL_FW_CHECK(fwrt, iwl_rx_packet_payload_len(pkt) <
+				 sizeof(*mem_cfg),
+				 "bad shared mem notification size (extend)\n"))
+			return;
+
 		BUILD_BUG_ON(sizeof(fwrt->smem_cfg.internal_txfifo_size) !=
 			     sizeof(mem_cfg->internal_txfifo_size));
 
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/uefi.c b/drivers/net/wireless/intel/iwlwifi/fw/uefi.c
index 2ef0a7a920adb..01e495e24ebcc 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/uefi.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/uefi.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright(c) 2021-2025 Intel Corporation
+ * Copyright(c) 2021-2026 Intel Corporation
  */
 
 #include "iwl-drv.h"
@@ -248,6 +248,12 @@ int iwl_uefi_reduce_power_parse(struct iwl_trans *trans,
 			IWL_DEBUG_FW(trans,
 				     "Got IWL_UCODE_TLV_PNVM_SKU len %d\n",
 				     tlv_len);
+			if (tlv_len < sizeof(*tlv_sku_id)) {
+				IWL_ERR(trans, "invalid PNVM SKU TLV len: %u\n",
+					tlv_len);
+				return -EINVAL;
+			}
+
 			IWL_DEBUG_FW(trans, "sku_id 0x%0x 0x%0x 0x%0x\n",
 				     le32_to_cpu(tlv_sku_id->data[0]),
 				     le32_to_cpu(tlv_sku_id->data[1]),
@@ -884,6 +890,8 @@ int iwl_uefi_get_wbem(struct iwl_fw_runtime *fwrt, u32 *value)
 		goto out;
 	}
 	*value = data->wbem_320mhz_per_mcc & IWL_UEFI_WBEM_REV0_MASK;
+	fwrt->wbem_source = BIOS_SOURCE_UEFI;
+	fwrt->wbem_revision = data->revision;
 	IWL_DEBUG_RADIO(fwrt, "Loaded WBEM config from UEFI\n");
 out:
 	kfree(data);
@@ -970,29 +978,29 @@ int iwl_uefi_get_dsm(struct iwl_fw_runtime *fwrt, enum iwl_dsm_funcs func,
 int iwl_uefi_get_puncturing(struct iwl_fw_runtime *fwrt)
 {
 	struct uefi_cnv_var_puncturing_data *data;
-	/* default value is not enabled if there is any issue in reading
-	 * uefi variable or revision is not supported
-	 */
-	int puncturing = 0;
+	int ret = 0;
 
 	data = iwl_uefi_get_verified_variable(fwrt->trans,
 					      IWL_UEFI_PUNCTURING_NAME,
 					      "UefiCnvWlanPuncturing",
 					      sizeof(*data), NULL);
 	if (IS_ERR(data))
-		return puncturing;
+		return -EINVAL;
 
 	if (data->revision != IWL_UEFI_PUNCTURING_REVISION) {
 		IWL_DEBUG_RADIO(fwrt, "Unsupported UEFI PUNCTURING rev:%d\n",
 				data->revision);
+		ret = -EINVAL;
 	} else {
-		puncturing = data->puncturing & IWL_UEFI_PUNCTURING_REV0_MASK;
+		fwrt->puncturing_source = BIOS_SOURCE_UEFI;
+		fwrt->puncturing_revision = data->revision;
+		fwrt->bios_puncturing = data->puncturing;
 		IWL_DEBUG_RADIO(fwrt, "Loaded puncturing bits from UEFI: %d\n",
-				puncturing);
+				fwrt->bios_puncturing);
 	}
 
 	kfree(data);
-	return puncturing;
+	return ret;
 }
 IWL_EXPORT_SYMBOL(iwl_uefi_get_puncturing);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/uefi.h b/drivers/net/wireless/intel/iwlwifi/fw/uefi.h
index 474f06db4d430..386ca37444089 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/uefi.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/uefi.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright(c) 2021-2025 Intel Corporation
+ * Copyright(c) 2021-2026 Intel Corporation
  */
 #ifndef __iwl_fw_uefi__
 #define __iwl_fw_uefi__
@@ -275,8 +275,6 @@ enum iwl_uefi_cnv_puncturing_flags {
 	IWL_UEFI_CNV_PUNCTURING_CANADA_EN_MSK	= BIT(1),
 };
 
-#define IWL_UEFI_PUNCTURING_REV0_MASK (IWL_UEFI_CNV_PUNCTURING_USA_EN_MSK | \
-				       IWL_UEFI_CNV_PUNCTURING_CANADA_EN_MSK)
 /**
  * struct uefi_cnv_var_puncturing_data - controlling channel
  *	puncturing for few countries.
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-config.h b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
index 6a3539ad73317..3ac7f000ede47 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-config.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
@@ -415,6 +415,7 @@ struct iwl_mac_cfg {
  * @vht_mu_mimo_supported: VHT MU-MIMO support
  * @nvm_type: see &enum iwl_nvm_type
  * @uhb_supported: ultra high band channels supported
+ * @unii9_supported: UNII-9 channels supported
  * @eht_supported: EHT supported
  * @uhr_supported: UHR supported
  * @num_rbds: number of receive buffer descriptors to use
@@ -450,6 +451,7 @@ struct iwl_rf_cfg {
 	    lp_xtal_workaround:1,
 	    vht_mu_mimo_supported:1,
 	    uhb_supported:1,
+	    unii9_supported:1,
 	    eht_supported:1,
 	    uhr_supported:1;
 	u8 valid_tx_ant;
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c
index d021b24d04d6b..8b0f091ff4c1b 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2018-2025 Intel Corporation
+ * Copyright (C) 2018-2026 Intel Corporation
  */
 #include <linux/firmware.h>
 #include "iwl-drv.h"
@@ -602,6 +602,9 @@ static int iwl_dbg_tlv_alloc_fragments(struct iwl_fw_runtime *fwrt,
 	    cpu_to_le32(IWL_FW_INI_LOCATION_DRAM_PATH))
 		return 0;
 
+	if (!fw_mon_cfg->req_size)
+		return -EIO;
+
 	num_frags = le32_to_cpu(fw_mon_cfg->max_frags_num);
 	if (fwrt->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210) {
 		if (alloc_id != IWL_FW_INI_ALLOCATION_ID_DBGC1)
@@ -612,6 +615,9 @@ static int iwl_dbg_tlv_alloc_fragments(struct iwl_fw_runtime *fwrt,
 		return -EIO;
 	}
 
+	if (!num_frags)
+		return -EIO;
+
 	remain_pages = DIV_ROUND_UP(le32_to_cpu(fw_mon_cfg->req_size),
 				    PAGE_SIZE);
 	num_frags = min_t(u32, num_frags, BUF_ALLOC_MAX_NUM_FRAGS);
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
index d47b4ae2f4863..863d5e358152d 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
@@ -85,16 +85,7 @@ static const u16 iwl_nvm_channels[] = {
 	149, 153, 157, 161, 165
 };
 
-static const u16 iwl_ext_nvm_channels[] = {
-	/* 2.4 GHz */
-	1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
-	/* 5 GHz */
-	36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92,
-	96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144,
-	149, 153, 157, 161, 165, 169, 173, 177, 181
-};
-
-static const u16 iwl_uhb_nvm_channels[] = {
+static const u16 iwl_unii9_nvm_channels[] = {
 	/* 2.4 GHz */
 	1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
 	/* 5 GHz */
@@ -105,12 +96,16 @@ static const u16 iwl_uhb_nvm_channels[] = {
 	1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69,
 	73, 77, 81, 85, 89, 93, 97, 101, 105, 109, 113, 117, 121, 125, 129,
 	133, 137, 141, 145, 149, 153, 157, 161, 165, 169, 173, 177, 181, 185,
-	189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233
+	189, 193, 197, 201, 205, 209, 213, 217, 221, 225, 229, 233,
+
+	/* UNII-9 */
+	237, 241, 245, 249, 253
 };
 
 #define IWL_NVM_NUM_CHANNELS		ARRAY_SIZE(iwl_nvm_channels)
-#define IWL_NVM_NUM_CHANNELS_EXT	ARRAY_SIZE(iwl_ext_nvm_channels)
-#define IWL_NVM_NUM_CHANNELS_UHB	ARRAY_SIZE(iwl_uhb_nvm_channels)
+#define IWL_NVM_NUM_CHANNELS_EXT	51
+#define IWL_NVM_NUM_CHANNELS_UHB	110
+#define IWL_NVM_NUM_CHANNELS_UNII9	ARRAY_SIZE(iwl_unii9_nvm_channels)
 #define NUM_2GHZ_CHANNELS		14
 #define NUM_5GHZ_CHANNELS		37
 #define FIRST_2GHZ_HT_MINUS		5
@@ -230,6 +225,18 @@ enum iwl_reg_capa_flags_v5 {
 	REG_CAPA_V5_11BN_DISABLED		= BIT(17),
 }; /* GEO_CHANNEL_CAPABILITIES_API_S_VER_4, 5 */
 
+/**
+ * enum iwl_reg_capa_flags_v6 - global capability flags,
+ *	applicable from MCC response version 10 onwards.
+ * Response v6 includes all members of iwl_reg_capa_flags_v5; only v6-specific
+ * additions are listed here.
+ * @REG_CAPA_V6_EHT_PUNCTURING_ENABLED: EHT puncturing is enabled for this
+ *	regulatory domain.
+ */
+enum iwl_reg_capa_flags_v6 {
+	REG_CAPA_V6_EHT_PUNCTURING_ENABLED	= BIT(18),
+}; /* GEO_CHANNEL_CAPABILITIES_API_S_VER_6 */
+
 /*
 * API v2 for reg_capa_flags is relevant from version 6 and onwards of the
 * MCC update command response.
@@ -241,6 +248,11 @@ enum iwl_reg_capa_flags_v5 {
  */
 #define REG_CAPA_V4_RESP_VER	8
 
+/* API v6 for reg_capa_flags is relevant from version 10 and onwards of the
+ * MCC update command response.
+ */
+#define REG_CAPA_V6_RESP_VER	10
+
 static inline void iwl_nvm_print_channel_flags(struct device *dev, u32 level,
 					       int chan, u32 flags)
 {
@@ -351,12 +363,15 @@ static int iwl_init_channel_map(struct iwl_trans *trans,
 	int num_of_ch;
 	const u16 *nvm_chan;
 
-	if (cfg->uhb_supported) {
+	if (cfg->unii9_supported) {
+		num_of_ch = IWL_NVM_NUM_CHANNELS_UNII9;
+		nvm_chan = iwl_unii9_nvm_channels;
+	} else if (cfg->uhb_supported) {
 		num_of_ch = IWL_NVM_NUM_CHANNELS_UHB;
-		nvm_chan = iwl_uhb_nvm_channels;
+		nvm_chan = iwl_unii9_nvm_channels;
 	} else if (cfg->nvm_type == IWL_NVM_EXT) {
 		num_of_ch = IWL_NVM_NUM_CHANNELS_EXT;
-		nvm_chan = iwl_ext_nvm_channels;
+		nvm_chan = iwl_unii9_nvm_channels;
 	} else {
 		num_of_ch = IWL_NVM_NUM_CHANNELS;
 		nvm_chan = iwl_nvm_channels;
@@ -699,7 +714,8 @@ static const struct ieee80211_sband_iftype_data iwl_iftype_cap[] = {
 			.mac.mac_cap = {
 				[0] = IEEE80211_UHR_MAC_CAP0_NPCA_SUPP |
 				      IEEE80211_UHR_MAC_CAP0_DPS_SUPP,
-				[1] = IEEE80211_UHR_MAC_CAP1_DUO_SUPP,
+				[1] = IEEE80211_UHR_MAC_CAP1_DUO_SUPP |
+				      IEEE80211_UHR_MAC_CAP1_DBE_SUPP,
 			},
 		},
 	},
@@ -1441,7 +1457,9 @@ iwl_parse_mei_nvm_data(struct iwl_trans *trans, const struct iwl_rf_cfg *cfg,
 	u8 rx_chains = fw->valid_rx_ant;
 	u8 tx_chains = fw->valid_rx_ant;
 
-	if (cfg->uhb_supported)
+	if (cfg->unii9_supported)
+		data = kzalloc_flex(*data, channels, IWL_NVM_NUM_CHANNELS_UNII9);
+	else if (cfg->uhb_supported)
 		data = kzalloc_flex(*data, channels, IWL_NVM_NUM_CHANNELS_UHB);
 	else
 		data = kzalloc_flex(*data, channels, IWL_NVM_NUM_CHANNELS_EXT);
@@ -1506,7 +1524,9 @@ iwl_parse_nvm_data(struct iwl_trans *trans, const struct iwl_rf_cfg *cfg,
 	u16 lar_config;
 	const __le16 *ch_section;
 
-	if (cfg->uhb_supported)
+	if (cfg->unii9_supported)
+		data = kzalloc_flex(*data, channels, IWL_NVM_NUM_CHANNELS_UNII9);
+	else if (cfg->uhb_supported)
 		data = kzalloc_flex(*data, channels, IWL_NVM_NUM_CHANNELS_UHB);
 	else if (cfg->nvm_type != IWL_NVM_EXT)
 		data = kzalloc_flex(*data, channels, IWL_NVM_NUM_CHANNELS);
@@ -1685,6 +1705,13 @@ static struct iwl_reg_capa iwl_get_reg_capa(u32 flags, u8 resp_ver)
 {
 	struct iwl_reg_capa reg_capa = {};
 
+	if (resp_ver >= REG_CAPA_V6_RESP_VER) {
+		if (flags & REG_CAPA_V6_EHT_PUNCTURING_ENABLED)
+			reg_capa.puncturing_status = IWL_PUNCTURING_STATUS_ENABLED;
+		else
+			reg_capa.puncturing_status = IWL_PUNCTURING_STATUS_DISABLED;
+	}
+
 	if (resp_ver >= REG_CAPA_V4_RESP_VER) {
 		reg_capa.allow_40mhz = true;
 		reg_capa.allow_80mhz = flags & REG_CAPA_V5_80MHZ_ALLOWED;
@@ -1711,7 +1738,8 @@ static struct iwl_reg_capa iwl_get_reg_capa(u32 flags, u8 resp_ver)
 struct ieee80211_regdomain *
 iwl_parse_nvm_mcc_info(struct iwl_trans *trans,
 		       int num_of_ch, __le32 *channels, u16 fw_mcc,
-		       u16 geo_info, u32 cap, u8 resp_ver)
+		       u16 geo_info, u32 cap, u8 resp_ver,
+		       enum iwl_puncturing_status *puncturing_status)
 {
 	const struct iwl_rf_cfg *cfg = trans->cfg;
 	struct device *dev = trans->dev;
@@ -1727,12 +1755,15 @@ iwl_parse_nvm_mcc_info(struct iwl_trans *trans,
 	int max_num_ch;
 	struct iwl_reg_capa reg_capa;
 
-	if (cfg->uhb_supported) {
+	if (cfg->unii9_supported) {
+		max_num_ch = IWL_NVM_NUM_CHANNELS_UNII9;
+		nvm_chan = iwl_unii9_nvm_channels;
+	} else if (cfg->uhb_supported) {
 		max_num_ch = IWL_NVM_NUM_CHANNELS_UHB;
-		nvm_chan = iwl_uhb_nvm_channels;
+		nvm_chan = iwl_unii9_nvm_channels;
 	} else if (cfg->nvm_type == IWL_NVM_EXT) {
 		max_num_ch = IWL_NVM_NUM_CHANNELS_EXT;
-		nvm_chan = iwl_ext_nvm_channels;
+		nvm_chan = iwl_unii9_nvm_channels;
 	} else {
 		max_num_ch = IWL_NVM_NUM_CHANNELS;
 		nvm_chan = iwl_nvm_channels;
@@ -1771,6 +1802,9 @@ iwl_parse_nvm_mcc_info(struct iwl_trans *trans,
 	/* parse regulatory capability flags */
 	reg_capa = iwl_get_reg_capa(cap, resp_ver);
 
+	if (puncturing_status)
+		*puncturing_status = reg_capa.puncturing_status;
+
 	for (ch_idx = 0; ch_idx < num_of_ch; ch_idx++) {
 		enum nl80211_band band =
 			iwl_nl80211_band_from_channel_idx(ch_idx);
@@ -2087,13 +2121,26 @@ struct iwl_nvm_data *iwl_get_nvm(struct iwl_trans *trans,
 	struct iwl_nvm_get_info_rsp_v3 *rsp_v3;
 	bool v4 = fw_has_api(&fw->ucode_capa,
 			     IWL_UCODE_TLV_API_REGULATORY_NVM_INFO);
-	size_t rsp_size = v4 ? sizeof(*rsp) : sizeof(*rsp_v3);
+	size_t rsp_size;
 	void *channel_profile;
 
 	ret = iwl_trans_send_cmd(trans, &hcmd);
 	if (ret)
 		return ERR_PTR(ret);
 
+	switch (iwl_fw_lookup_notif_ver(fw, REGULATORY_AND_NVM_GROUP,
+					NVM_GET_INFO, 0)) {
+	case 5:
+		rsp_size = sizeof(struct iwl_nvm_get_info_rsp);
+		break;
+	case 4:
+		rsp_size = sizeof(struct iwl_nvm_get_info_rsp_v4);
+		break;
+	default:
+		rsp_size = sizeof(struct iwl_nvm_get_info_rsp_v3);
+		break;
+	}
+
 	if (WARN(iwl_rx_packet_payload_len(hcmd.resp_pkt) != rsp_size,
 		 "Invalid payload len in NVM response from FW %d",
 		 iwl_rx_packet_payload_len(hcmd.resp_pkt))) {
@@ -2107,7 +2154,7 @@ struct iwl_nvm_data *iwl_get_nvm(struct iwl_trans *trans,
 	if (empty_otp)
 		IWL_INFO(trans, "OTP is empty\n");
 
-	nvm = kzalloc_flex(*nvm, channels, IWL_NUM_CHANNELS_V2);
+	nvm = kzalloc_flex(*nvm, channels, IWL_NUM_CHANNELS_V3);
 	if (!nvm) {
 		ret = -ENOMEM;
 		goto out;
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h
index e676d7c2d6cc0..9ebb72d3726a5 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2005-2015, 2018-2025 Intel Corporation
+ * Copyright (C) 2005-2015, 2018-2026 Intel Corporation
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
 #ifndef __iwl_nvm_parse_h__
@@ -21,6 +21,19 @@ enum iwl_nvm_sbands_flags {
 	IWL_NVM_SBANDS_FLAGS_NO_WIDE_IN_5GHZ	= BIT(1),
 };
 
+/**
+ * enum iwl_puncturing_status - EHT puncturing status from MCC capabilities
+ * @IWL_PUNCTURING_STATUS_UNKNOWN: puncturing status is not provided by FW
+ *	or is not initialized yet
+ * @IWL_PUNCTURING_STATUS_ENABLED: puncturing is enabled for the current MCC
+ * @IWL_PUNCTURING_STATUS_DISABLED: puncturing is disabled for the current MCC
+ */
+enum iwl_puncturing_status {
+	IWL_PUNCTURING_STATUS_UNKNOWN,
+	IWL_PUNCTURING_STATUS_ENABLED,
+	IWL_PUNCTURING_STATUS_DISABLED,
+};
+
 /**
  * struct iwl_reg_capa - struct for global regulatory capabilities, Used for
  * handling the different APIs of reg_capa_flags.
@@ -36,6 +49,8 @@ enum iwl_nvm_sbands_flags {
  * @disable_11ax: 11ax is forbidden for this regulatory domain.
  * @disable_11be: 11be is forbidden for this regulatory domain.
  * @disable_11bn: UHR/11bn is not allowed for this regulatory domain
+ * @puncturing_status: EHT puncturing status for the current MCC.
+ *	See &enum iwl_puncturing_status.
  */
 struct iwl_reg_capa {
 	bool allow_40mhz;
@@ -45,6 +60,7 @@ struct iwl_reg_capa {
 	bool disable_11ax;
 	bool disable_11be;
 	bool disable_11bn;
+	enum iwl_puncturing_status puncturing_status;
 };
 
 /**
@@ -131,11 +147,13 @@ iwl_parse_nvm_data(struct iwl_trans *trans, const struct iwl_rf_cfg *cfg,
  * @geo_info: geo info value
  * @cap: capability
  * @resp_ver: FW response version
+ * @puncturing_status: FW puncturing status for current MCC, when available
  */
 struct ieee80211_regdomain *
 iwl_parse_nvm_mcc_info(struct iwl_trans *trans,
 		       int num_of_ch, __le32 *channels, u16 fw_mcc,
-		       u16 geo_info, u32 cap, u8 resp_ver);
+		       u16 geo_info, u32 cap, u8 resp_ver,
+		       enum iwl_puncturing_status *puncturing_status);
 
 /**
  * struct iwl_nvm_section - describes an NVM section in memory.
diff --git a/drivers/net/wireless/intel/iwlwifi/mei/main.c b/drivers/net/wireless/intel/iwlwifi/mei/main.c
index c5ff1b1b720f3..ce92e26c42d7f 100644
--- a/drivers/net/wireless/intel/iwlwifi/mei/main.c
+++ b/drivers/net/wireless/intel/iwlwifi/mei/main.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2021-2024 Intel Corporation
+ * Copyright (C) 2026 Intel Corporation
  */
 
 #include <linux/etherdevice.h>
@@ -457,7 +458,7 @@ static int iwl_mei_send_sap_msg_payload(struct mei_cl_device *cldev,
 	notif_q = &dir->q_ctrl_blk[SAP_QUEUE_IDX_NOTIF];
 	q_head = mei->shared_mem.q_head[SAP_DIRECTION_HOST_TO_ME][SAP_QUEUE_IDX_NOTIF];
 	q_sz = mei->shared_mem.q_size[SAP_DIRECTION_HOST_TO_ME][SAP_QUEUE_IDX_NOTIF];
-	ret = iwl_mei_write_cyclic_buf(q_head, notif_q, q_head, hdr, q_sz);
+	ret = iwl_mei_write_cyclic_buf(cldev, notif_q, q_head, hdr, q_sz);
 
 	if (ret < 0)
 		return ret;
@@ -1040,11 +1041,14 @@ static void iwl_mei_read_from_q(const u8 *q_head, u32 q_sz,
 	u32 rd = *_rd;
 
 	if (rd + len <= q_sz) {
-		memcpy(buf, q_head + rd, len);
+		if (buf)
+			memcpy(buf, q_head + rd, len);
 		rd += len;
 	} else {
-		memcpy(buf, q_head + rd, q_sz - rd);
-		memcpy(buf + q_sz - rd, q_head, len - (q_sz - rd));
+		if (buf) {
+			memcpy(buf, q_head + rd, q_sz - rd);
+			memcpy(buf + q_sz - rd, q_head, len - (q_sz - rd));
+		}
 		rd = len - (q_sz - rd);
 	}
 
@@ -1085,24 +1089,29 @@ static void iwl_mei_handle_sap_data(struct mei_cl_device *cldev,
 			break;
 		}
 
+		valid_rx_sz -= len;
+
 		if (len < sizeof(*ethhdr)) {
 			dev_err(&cldev->dev,
 				"Data len is smaller than an ethernet header? len = %d\n",
 				len);
+			iwl_mei_read_from_q(q_head, q_sz, &rd, wr, NULL, len);
+			continue;
 		}
 
-		valid_rx_sz -= len;
-
 		if (le16_to_cpu(hdr.type) != SAP_MSG_DATA_PACKET) {
 			dev_err(&cldev->dev, "Unsupported Rx data: type %d, len %d\n",
 				le16_to_cpu(hdr.type), len);
+			iwl_mei_read_from_q(q_head, q_sz, &rd, wr, NULL, len);
 			continue;
 		}
 
 		/* We need enough room for the WiFi header + SNAP + IV */
 		skb = netdev_alloc_skb(netdev, len + QOS_HDR_IV_SNAP_LEN);
-		if (!skb)
+		if (!skb) {
+			iwl_mei_read_from_q(q_head, q_sz, &rd, wr, NULL, len);
 			continue;
+		}
 
 		skb_reserve(skb, QOS_HDR_IV_SNAP_LEN);
 		ethhdr = skb_push(skb, sizeof(*ethhdr));
@@ -1147,6 +1156,11 @@ static void iwl_mei_handle_sap_rx_cmd(struct mei_cl_device *cldev,
 		iwl_mei_read_from_q(q_head, q_sz, &rd, wr, hdr, sizeof(*hdr));
 		valid_rx_sz -= sizeof(*hdr);
 		len = le16_to_cpu(hdr->len);
+		if (len + sizeof(*hdr) > PAGE_SIZE) {
+			dev_err(&cldev->dev,
+				"SAP message is too big: %u\n", len);
+			break;
+		}
 
 		if (valid_rx_sz < len)
 			break;
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/agg.c b/drivers/net/wireless/intel/iwlwifi/mld/agg.c
index e3627ad0321c8..96102d2af2cf9 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/agg.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/agg.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #include "agg.h"
 #include "sta.h"
@@ -200,11 +200,11 @@ iwl_mld_reorder(struct iwl_mld *mld, struct napi_struct *napi,
 	struct iwl_mld_baid_data *baid_data;
 	struct iwl_mld_reorder_buffer *buffer;
 	struct iwl_mld_reorder_buf_entry *entries;
-	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
 	struct iwl_mld_link_sta *mld_link_sta;
 	u32 reorder = le32_to_cpu(desc->reorder_data);
 	bool amsdu, last_subframe, is_old_sn, is_dup;
 	u8 tid = ieee80211_get_tid(hdr);
+	struct iwl_mld_sta *mld_sta;
 	u8 baid;
 	u16 nssn, sn;
 	u32 sta_mask = 0;
@@ -222,11 +222,19 @@ iwl_mld_reorder(struct iwl_mld *mld, struct napi_struct *napi,
 	if (baid == IWL_RX_REORDER_DATA_INVALID_BAID)
 		return IWL_MLD_PASS_SKB;
 
+	if (IWL_FW_CHECK(mld, baid >= ARRAY_SIZE(mld->fw_id_to_ba),
+			 "Got out-of-range BAID %u in reorder_data=0x%x\n",
+			 baid, reorder))
+		return IWL_MLD_PASS_SKB;
+
 	/* no sta yet */
-	if (WARN_ONCE(!sta,
-		      "Got valid BAID without a valid station assigned\n"))
+	if (IWL_FW_CHECK(mld, !sta,
+			 "Got valid BAID without a valid station assigned - %d\n",
+			 baid))
 		return IWL_MLD_PASS_SKB;
 
+	mld_sta = iwl_mld_sta_from_mac80211(sta);
+
 	/* not a data packet */
 	if (!ieee80211_is_data_qos(hdr->frame_control) ||
 	    is_multicast_ether_addr(hdr->addr1))
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/d3.c b/drivers/net/wireless/intel/iwlwifi/mld/d3.c
index 458a668ba9168..3b785c53948fb 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/d3.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/d3.c
@@ -573,18 +573,42 @@ iwl_mld_convert_wowlan_notif_v5(const struct iwl_wowlan_info_notif_v5 *notif_v5,
 	}
 }
 
-static bool iwl_mld_validate_wowlan_notif_size(struct iwl_mld *mld,
-					       u32 len,
-					       u32 expected_len,
-					       u8 num_mlo_keys,
+static bool iwl_mld_validate_wowlan_notif_size(struct iwl_mld *mld, u32 len,
+					       const void *notif_data,
 					       int version)
 {
 	u32 len_with_mlo_keys;
+	u32 expected_len;
+	u8 num_mlo_keys;
 
-	if (IWL_FW_CHECK(mld, len < expected_len,
-			 "Invalid wowlan_info_notif v%d (expected=%u got=%u)\n",
-			 version, expected_len, len))
+	/* Extract num_mlo_keys from the void pointer based on version */
+	if (version == 5) {
+		const struct iwl_wowlan_info_notif_v5 *notif_v5 = notif_data;
+
+		expected_len = sizeof(*notif_v5);
+
+		if (IWL_FW_CHECK(mld, len < expected_len,
+				 "Invalid wowlan_info_notif v5 (expected=%u got=%u)\n",
+				 expected_len, len))
+			return false;
+
+		num_mlo_keys = notif_v5->num_mlo_link_keys;
+	} else if (version == 6) {
+		const struct iwl_wowlan_info_notif *notif = notif_data;
+
+		expected_len = sizeof(*notif);
+
+		if (IWL_FW_CHECK(mld, len < expected_len,
+				 "Invalid wowlan_info_notif v6 (expected=%u got=%u)\n",
+				 expected_len, len))
+			return false;
+
+		num_mlo_keys = notif->num_mlo_link_keys;
+	} else {
+		IWL_WARN(mld, "Unsupported wowlan_info_notif version %d\n",
+			 version);
 		return false;
+	}
 
 	len_with_mlo_keys = expected_len +
 		(num_mlo_keys * sizeof(struct iwl_wowlan_mlo_gtk));
@@ -616,16 +640,14 @@ iwl_mld_handle_wowlan_info_notif(struct iwl_mld *mld,
 
 	if (wowlan_info_ver == 5) {
 		/* v5 format - validate before conversion */
-		const struct iwl_wowlan_info_notif_v5 *notif_v5 = (void *)pkt->data;
+		const struct iwl_wowlan_info_notif_v5 *_notif =
+			(void *)pkt->data;
 
-		if (!iwl_mld_validate_wowlan_notif_size(mld, len,
-							sizeof(*notif_v5),
-							notif_v5->num_mlo_link_keys,
-							5))
+		if (!iwl_mld_validate_wowlan_notif_size(mld, len, _notif, 5))
 			return true;
 
 		converted_notif = kzalloc_flex(*converted_notif, mlo_gtks,
-					       notif_v5->num_mlo_link_keys,
+					       _notif->num_mlo_link_keys,
 					       GFP_ATOMIC);
 		if (!converted_notif) {
 			IWL_ERR(mld,
@@ -633,15 +655,12 @@ iwl_mld_handle_wowlan_info_notif(struct iwl_mld *mld,
 			return true;
 		}
 
-		iwl_mld_convert_wowlan_notif_v5(notif_v5,
-						converted_notif);
+		iwl_mld_convert_wowlan_notif_v5(_notif, converted_notif);
 		notif = converted_notif;
 	} else if (wowlan_info_ver == 6) {
 		notif = (void *)pkt->data;
-		if (!iwl_mld_validate_wowlan_notif_size(mld, len,
-							sizeof(*notif),
-							notif->num_mlo_link_keys,
-							6))
+
+		if (!iwl_mld_validate_wowlan_notif_size(mld, len, notif, 6))
 			return true;
 	} else {
 		/* smaller versions are not supported */
@@ -686,7 +705,7 @@ iwl_mld_handle_wake_pkt_notif(struct iwl_mld *mld,
 {
 	const struct iwl_wowlan_wake_pkt_notif *notif = (void *)pkt->data;
 	u32 actual_size, len = iwl_rx_packet_payload_len(pkt);
-	u32 expected_size = le32_to_cpu(notif->wake_packet_length);
+	u32 expected_size;
 
 	if (IWL_FW_CHECK(mld, len < sizeof(*notif),
 			 "Invalid WoWLAN wake packet notification (expected size=%zu got=%u)\n",
@@ -699,6 +718,7 @@ iwl_mld_handle_wake_pkt_notif(struct iwl_mld *mld,
 			 wowlan_status->wakeup_reasons))
 		return true;
 
+	expected_size = le32_to_cpu(notif->wake_packet_length);
 	actual_size = len - offsetof(struct iwl_wowlan_wake_pkt_notif,
 				     wake_packet);
 
@@ -1449,8 +1469,16 @@ static bool iwl_mld_handle_d3_notif(struct iwl_notif_wait_data *notif_wait,
 	}
 	case WIDE_ID(PROT_OFFLOAD_GROUP, D3_END_NOTIFICATION): {
 		struct iwl_d3_end_notif *notif = (void *)pkt->data;
+		u32 len = iwl_rx_packet_payload_len(pkt);
+
+		if (IWL_FW_CHECK(mld, len < sizeof(*notif),
+				 "Invalid D3_END notification (expected=%zu got=%u)\n",
+				 sizeof(*notif), len)) {
+			resume_data->notif_handling_err = true;
+		} else {
+			resume_data->d3_end_flags = le32_to_cpu(notif->flags);
+		}
 
-		resume_data->d3_end_flags = le32_to_cpu(notif->flags);
 		resume_data->notifs_received |= IWL_D3_NOTIF_D3_END_NOTIF;
 		break;
 	}
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/ftm-initiator.c b/drivers/net/wireless/intel/iwlwifi/mld/ftm-initiator.c
index 81df3fdfcbf56..a4b8037b5c765 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/ftm-initiator.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/ftm-initiator.c
@@ -459,3 +459,24 @@ void iwl_mld_ftm_restart_cleanup(struct iwl_mld *mld)
 			       mld->ftm_initiator.req, GFP_KERNEL);
 	iwl_mld_ftm_reset(mld);
 }
+
+void iwl_mld_ftm_abort(struct iwl_mld *mld, struct cfg80211_pmsr_request *req)
+{
+	struct iwl_tof_range_abort_cmd cmd = {
+		.request_id = req->cookie,
+	};
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (req != mld->ftm_initiator.req)
+		return;
+
+	if (iwl_mld_send_cmd_pdu(mld, WIDE_ID(LOCATION_GROUP,
+					      TOF_RANGE_ABORT_CMD),
+				 &cmd))
+		IWL_ERR(mld, "failed to abort FTM process\n");
+
+	iwl_mld_cancel_notifications_of_object(mld, IWL_MLD_OBJECT_TYPE_FTM_REQ,
+					       mld->ftm_initiator.req->cookie);
+	iwl_mld_ftm_reset(mld);
+}
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/ftm-initiator.h b/drivers/net/wireless/intel/iwlwifi/mld/ftm-initiator.h
index 3fab25a52508a..e82237b314673 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/ftm-initiator.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/ftm-initiator.h
@@ -25,5 +25,6 @@ int iwl_mld_ftm_start(struct iwl_mld *mld, struct ieee80211_vif *vif,
 void iwl_mld_handle_ftm_resp_notif(struct iwl_mld *mld,
 				   struct iwl_rx_packet *pkt);
 void iwl_mld_ftm_restart_cleanup(struct iwl_mld *mld);
+void iwl_mld_ftm_abort(struct iwl_mld *mld, struct cfg80211_pmsr_request *req);
 
 #endif /* __iwl_mld_ftm_initiator_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/hcmd.h b/drivers/net/wireless/intel/iwlwifi/mld/hcmd.h
index 64a8d42483241..19e1c9e509cfc 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/hcmd.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/hcmd.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #ifndef __iwl_mld_hcmd_h__
 #define __iwl_mld_hcmd_h__
@@ -42,7 +42,12 @@ __iwl_mld_send_cmd_with_flags_pdu(struct iwl_mld *mld, u32 id,
 
 #define _iwl_mld_send_cmd_with_flags_pdu(mld, id, flags, data, len,	\
 					 ignored...)			\
-	__iwl_mld_send_cmd_with_flags_pdu(mld, id, flags, data, len)
+	({								\
+		BUILD_BUG_ON(__builtin_constant_p(len) &&		\
+			     (u16)(len) > IWL_MAX_CMD_PAYLOAD_SIZE);	\
+		__iwl_mld_send_cmd_with_flags_pdu(mld, id, flags,	\
+						  data, len);		\
+	})
 #define iwl_mld_send_cmd_with_flags_pdu(mld, id, flags, data, len...)	\
 	_iwl_mld_send_cmd_with_flags_pdu(mld, id, flags, data, ##len,	\
 					 sizeof(*(data)))
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
index 17286b3341c02..3a4c8fda68d0a 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
@@ -543,8 +543,7 @@ iwl_mld_mac80211_tx(struct ieee80211_hw *hw,
 	iwl_mld_tx_skb(mld, skb, NULL);
 }
 
-static void
-iwl_mld_restart_cleanup(struct iwl_mld *mld)
+void iwl_mld_restart_cleanup(struct iwl_mld *mld)
 {
 	iwl_cleanup_mld(mld);
 
@@ -1281,11 +1280,13 @@ void iwl_mld_unassign_vif_chanctx(struct ieee80211_hw *hw,
 	iwl_mld_tlc_update_phy(mld, vif, link);
 
 	/* in the non-MLO case, remove/re-add the link to clean up FW state.
-	 * In MLO, it'll be done in drv_change_vif_link
+	 * In MLO, it'll be done in drv_change_vif_link.
+	 * Do not do so during restart or in case the device is dead.
 	 */
 	if (!ieee80211_vif_is_mld(vif) && !mld_vif->ap_sta &&
 	    !WARN_ON_ONCE(vif->cfg.assoc) &&
-	    vif->type != NL80211_IFTYPE_AP && !mld->fw_status.in_hw_restart) {
+	    vif->type != NL80211_IFTYPE_AP && !mld->fw_status.in_hw_restart &&
+	    !iwl_trans_is_dead(mld->trans)) {
 		iwl_mld_remove_link(mld, link);
 		iwl_mld_add_link(mld, link);
 	}
@@ -1708,7 +1709,7 @@ static void iwl_mld_check_he_obss_narrow_bw_ru_iter(struct wiphy *wiphy,
 	elem = cfg80211_find_elem(WLAN_EID_EXT_CAPABILITY, ies->data,
 				  ies->len);
 
-	if (!elem || elem->datalen < 10 ||
+	if (!elem || elem->datalen < 11 ||
 	    !(elem->data[10] &
 	      WLAN_EXT_CAPA10_OBSS_NARROW_BW_RU_TOLERANCE_SUPPORT)) {
 		*tolerated = false;
@@ -2853,6 +2854,15 @@ static int iwl_mld_start_pmsr(struct ieee80211_hw *hw,
 	return iwl_mld_ftm_start(mld, vif, request);
 }
 
+static void iwl_mld_abort_pmsr(struct ieee80211_hw *hw,
+			       struct ieee80211_vif *vif,
+			       struct cfg80211_pmsr_request *request)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+
+	iwl_mld_ftm_abort(mld, request);
+}
+
 static enum ieee80211_neg_ttlm_res
 iwl_mld_can_neg_ttlm(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		     struct ieee80211_neg_ttlm *neg_ttlm)
@@ -2974,6 +2984,7 @@ const struct ieee80211_ops iwl_mld_hw_ops = {
 	.prep_add_interface = iwl_mld_prep_add_interface,
 	.set_hw_timestamp = iwl_mld_set_hw_timestamp,
 	.start_pmsr = iwl_mld_start_pmsr,
+	.abort_pmsr = iwl_mld_abort_pmsr,
 	.can_neg_ttlm = iwl_mld_can_neg_ttlm,
 	.start_nan = iwl_mld_start_nan,
 	.stop_nan = iwl_mld_stop_nan,
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.h b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.h
index aad04d7b2617d..f3302997b28f2 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.h
@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
  * Copyright (C) 2024 Intel Corporation
+ * Copyright (C) 2026 Intel Corporation
  */
 #ifndef __iwl_mld_mac80211_h__
 #define __iwl_mld_mac80211_h__
@@ -9,5 +10,6 @@
 
 int iwl_mld_register_hw(struct iwl_mld *mld);
 void iwl_mld_recalc_multicast_filter(struct iwl_mld *mld);
+void iwl_mld_restart_cleanup(struct iwl_mld *mld);
 
 #endif /* __iwl_mld_mac80211_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mcc.c b/drivers/net/wireless/intel/iwlwifi/mld/mcc.c
index 8502129abe493..b4657e82468cb 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mcc.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mcc.c
@@ -18,9 +18,15 @@ static struct iwl_mcc_update_resp_v8 *
 iwl_mld_copy_mcc_resp(const struct iwl_rx_packet *pkt)
 {
 	const struct iwl_mcc_update_resp_v8 *mcc_resp_v8 = (const void *)pkt->data;
-	int n_channels = __le32_to_cpu(mcc_resp_v8->n_channels);
 	struct iwl_mcc_update_resp_v8 *resp_cp;
-	int notif_len = struct_size(resp_cp, channels, n_channels);
+	int n_channels;
+	int notif_len;
+
+	if (iwl_rx_packet_payload_len(pkt) < sizeof(*mcc_resp_v8))
+		return ERR_PTR(-EINVAL);
+
+	n_channels = __le32_to_cpu(mcc_resp_v8->n_channels);
+	notif_len = struct_size(resp_cp, channels, n_channels);
 
 	if (iwl_rx_packet_payload_len(pkt) != notif_len)
 		return ERR_PTR(-EINVAL);
@@ -89,6 +95,8 @@ iwl_mld_get_regdomain(struct iwl_mld *mld,
 	struct iwl_mcc_update_resp_v8 *resp;
 	u8 resp_ver = iwl_fw_lookup_notif_ver(mld->fw, IWL_ALWAYS_LONG_GROUP,
 					      MCC_UPDATE_CMD, 0);
+	enum iwl_puncturing_status puncturing_status;
+	u16 mcc;
 
 	IWL_DEBUG_LAR(mld, "Getting regdomain data for %s from FW\n", alpha2);
 
@@ -110,12 +118,13 @@ iwl_mld_get_regdomain(struct iwl_mld *mld,
 	}
 	IWL_DEBUG_LAR(mld, "MCC update response version: %d\n", resp_ver);
 
+	mcc = le16_to_cpu(resp->mcc);
 	regd = iwl_parse_nvm_mcc_info(mld->trans,
 				      __le32_to_cpu(resp->n_channels),
-				      resp->channels,
-				      __le16_to_cpu(resp->mcc),
+				      resp->channels, mcc,
 				      __le16_to_cpu(resp->geo_info),
-				      le32_to_cpu(resp->cap), resp_ver);
+				      le32_to_cpu(resp->cap), resp_ver,
+				      &puncturing_status);
 
 	if (IS_ERR(regd)) {
 		IWL_DEBUG_LAR(mld, "Could not get parse update from FW %ld\n",
@@ -129,18 +138,25 @@ iwl_mld_get_regdomain(struct iwl_mld *mld,
 
 	mld->mcc_src = resp->source_id;
 
+	if (puncturing_status == IWL_PUNCTURING_STATUS_ENABLED)
+		__clear_bit(IEEE80211_HW_DISALLOW_PUNCTURING,
+			    mld->hw->flags);
+	else if (puncturing_status == IWL_PUNCTURING_STATUS_DISABLED)
+		ieee80211_hw_set(mld->hw, DISALLOW_PUNCTURING);
+
+	if (resp_ver >= 10)
+		goto out;
+
 	/* FM follows BIOS/MCC policy, WH disallows puncturing only in US/CA. */
 	if (CSR_HW_RFID_TYPE(mld->trans->info.hw_rf_id) == IWL_CFG_RF_TYPE_FM) {
-		if (!iwl_puncturing_is_allowed_in_bios(mld->bios_enable_puncturing,
-						       le16_to_cpu(resp->mcc)))
+		if (!iwl_puncturing_is_allowed_in_bios(mld->fwrt.bios_puncturing,
+						       mcc))
 			ieee80211_hw_set(mld->hw, DISALLOW_PUNCTURING);
 		else
 			__clear_bit(IEEE80211_HW_DISALLOW_PUNCTURING,
 				    mld->hw->flags);
 	} else if (CSR_HW_RFID_TYPE(mld->trans->info.hw_rf_id) ==
-			IWL_CFG_RF_TYPE_WH) {
-		u16 mcc = le16_to_cpu(resp->mcc);
-
+		   IWL_CFG_RF_TYPE_WH) {
 		if (mcc == IWL_MCC_US || mcc == IWL_MCC_CANADA)
 			ieee80211_hw_set(mld->hw, DISALLOW_PUNCTURING);
 		else
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mld.c b/drivers/net/wireless/intel/iwlwifi/mld/mld.c
index 78c78cf891cda..7c49d0441fdf5 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mld.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mld.c
@@ -212,6 +212,7 @@ static const struct iwl_hcmd_names iwl_mld_reg_and_nvm_names[] = {
 	HCMD_NAME(TAS_CONFIG),
 	HCMD_NAME(SAR_OFFSET_MAPPING_TABLE_CMD),
 	HCMD_NAME(MCC_ALLOWED_AP_TYPE_CMD),
+	HCMD_NAME(PNVM_INIT_COMPLETE_NTFY),
 };
 
 /* Please keep this array *SORTED* by hex value.
@@ -419,7 +420,7 @@ iwl_op_mode_mld_start(struct iwl_trans *trans, const struct iwl_rf_cfg *cfg,
 
 	iwl_mld_get_bios_tables(mld);
 	iwl_uefi_get_sgom_table(trans, &mld->fwrt);
-	mld->bios_enable_puncturing = iwl_uefi_get_puncturing(&mld->fwrt);
+	iwl_uefi_get_puncturing(&mld->fwrt);
 
 	iwl_mld_hw_set_regulatory(mld);
 
@@ -446,6 +447,8 @@ iwl_op_mode_mld_start(struct iwl_trans *trans, const struct iwl_rf_cfg *cfg,
 	}
 
 	if (ret) {
+		/* wiphy memory is about to be freed, we should cancel any pending work */
+		wiphy_work_cancel(mld->wiphy, &mld->async_handlers_wk);
 		wiphy_unlock(mld->wiphy);
 		rtnl_unlock();
 		goto err;
@@ -511,6 +514,7 @@ iwl_op_mode_mld_stop(struct iwl_op_mode *op_mode)
 	iwl_mld_thermal_exit(mld);
 
 	wiphy_lock(mld->wiphy);
+	wiphy_work_cancel(mld->wiphy, &mld->async_handlers_wk);
 	iwl_mld_low_latency_stop(mld);
 	iwl_mld_deinit_time_sync(mld);
 	wiphy_unlock(mld->wiphy);
@@ -745,6 +749,7 @@ static void iwl_mld_device_powered_off(struct iwl_op_mode *op_mode)
 
 	wiphy_lock(mld->wiphy);
 	iwl_mld_stop_fw(mld);
+	iwl_mld_restart_cleanup(mld);
 	mld->fw_status.in_d3 = false;
 	wiphy_unlock(mld->wiphy);
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mld.h b/drivers/net/wireless/intel/iwlwifi/mld/mld.h
index 922aa3dbff543..37f41b504b54c 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mld.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mld.h
@@ -172,7 +172,6 @@
  * @debug_max_sleep: maximum sleep time in D3 (for debug purposes)
  * @led: the led device
  * @mcc_src: the source id of the MCC, comes from the firmware
- * @bios_enable_puncturing: is puncturing enabled by bios
  * @fw_id_to_ba: maps a fw (BA) id to a corresponding Block Ack session data.
  * @num_rx_ba_sessions: tracks the number of active Rx Block Ack (BA) sessions.
  *	the driver ensures that new BA sessions are blocked once the maximum
@@ -279,7 +278,6 @@ struct iwl_mld {
 	struct led_classdev led;
 #endif
 	enum iwl_mcc_source mcc_src;
-	bool bios_enable_puncturing;
 
 	struct iwl_mld_baid_data __rcu *fw_id_to_ba[IWL_MAX_BAID];
 	u8 num_rx_ba_sessions;
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/notif.c b/drivers/net/wireless/intel/iwlwifi/mld/notif.c
index 7574689e40883..b3a899828db91 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/notif.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/notif.c
@@ -517,7 +517,7 @@ iwl_mld_notif_is_valid(struct iwl_mld *mld, struct iwl_rx_packet *pkt,
 			    handler->cmd_id, notif_ver,
 			    handler->sizes[handler->n_sizes - 1].ver);
 
-	return size < handler->sizes[handler->n_sizes - 1].size;
+	return false;
 }
 
 struct iwl_async_handler_entry {
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/regulatory.c b/drivers/net/wireless/intel/iwlwifi/mld/regulatory.c
index 659243ada86c6..4a93ccbef495d 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/regulatory.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/regulatory.c
@@ -13,6 +13,23 @@
 #include "mld.h"
 #include "hcmd.h"
 
+static ssize_t iwl_mld_get_lari_config_cmd_size(u8 cmd_ver)
+{
+	switch (cmd_ver) {
+	case 14:
+		return sizeof(struct iwl_lari_config_change_cmd);
+	case 13:
+		return offsetof(struct iwl_lari_config_change_cmd,
+				oem_uhb_allow_extension_bitmap);
+	case 12:
+		return offsetof(struct iwl_lari_config_change_cmd,
+				oem_11bn_allow_bitmap);
+	default:
+		WARN(true, "unsupported version: %d", cmd_ver);
+		return -EINVAL;
+	}
+}
+
 void iwl_mld_get_bios_tables(struct iwl_mld *mld)
 {
 	int ret;
@@ -325,11 +342,23 @@ void iwl_mld_configure_lari(struct iwl_mld *mld)
 	struct iwl_lari_config_change_cmd cmd = {
 		.config_bitmap = iwl_mld_get_lari_config_bitmap(fwrt),
 	};
-	bool has_raw_dsm_capa = fw_has_capa(&fwrt->fw->ucode_capa,
-					    IWL_UCODE_TLV_CAPA_FW_ACCEPTS_RAW_DSM_TABLE);
+	u8 cmd_ver = iwl_fw_lookup_cmd_ver(mld->fw,
+					   WIDE_ID(REGULATORY_AND_NVM_GROUP,
+						   LARI_CONFIG_CHANGE), 12);
+	/*
+	 * For LARI_CONFIG_CHANGE command version 13 and above, firmware accepts
+	 * raw DSM values by default and this TLV is no longer needed.
+	 */
+	bool has_raw_dsm_capa = cmd_ver >= 13 ||
+		fw_has_capa(&fwrt->fw->ucode_capa,
+			    IWL_UCODE_TLV_CAPA_FW_ACCEPTS_RAW_DSM_TABLE);
+	ssize_t cmd_len = iwl_mld_get_lari_config_cmd_size(cmd_ver);
 	int ret;
 	u32 value;
 
+	if (cmd_len < 0)
+		return;
+
 	ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_11AX_ENABLEMENT, &value);
 	if (!ret) {
 		if (!has_raw_dsm_capa)
@@ -371,8 +400,11 @@ void iwl_mld_configure_lari(struct iwl_mld *mld)
 	}
 
 	ret = iwl_bios_get_wbem(fwrt, &value);
-	if (!ret)
+	if (!ret) {
 		cmd.oem_320mhz_allow_bitmap = cpu_to_le32(value);
+		cmd.bios_wbem_hdr.table_source = fwrt->wbem_source;
+		cmd.bios_wbem_hdr.table_revision = fwrt->wbem_revision;
+	}
 
 	ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_ENABLE_11BE, &value);
 	if (!ret)
@@ -386,8 +418,17 @@ void iwl_mld_configure_lari(struct iwl_mld *mld)
 	if (!ret)
 		cmd.oem_unii9_enable = cpu_to_le32(value);
 
+	ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_REGULATORY_CONFIG, &value);
+	if (!ret)
+		cmd.oem_uhb_allow_extension_bitmap = cpu_to_le32(value);
+
+	cmd.bios_wcpe_hdr.table_source = fwrt->puncturing_source;
+	cmd.bios_wcpe_hdr.table_revision = fwrt->puncturing_revision;
+	cmd.wcpe_bitmap = cpu_to_le32(fwrt->bios_puncturing);
+
 	if (!cmd.config_bitmap &&
 	    !cmd.oem_uhb_allow_bitmap &&
+	    !cmd.oem_uhb_allow_extension_bitmap &&
 	    !cmd.oem_11ax_allow_bitmap &&
 	    !cmd.oem_unii4_allow_bitmap &&
 	    !cmd.chan_state_active_bitmap &&
@@ -396,7 +437,8 @@ void iwl_mld_configure_lari(struct iwl_mld *mld)
 	    !cmd.oem_320mhz_allow_bitmap &&
 	    !cmd.oem_11be_allow_bitmap &&
 	    !cmd.oem_11bn_allow_bitmap &&
-	    !cmd.oem_unii9_enable)
+	    !cmd.oem_unii9_enable &&
+	    !cmd.wcpe_bitmap)
 		return;
 
 	cmd.bios_hdr.table_source = fwrt->dsm_source;
@@ -414,6 +456,9 @@ void iwl_mld_configure_lari(struct iwl_mld *mld)
 			"sending LARI_CONFIG_CHANGE, oem_uhb_allow_bitmap=0x%x, force_disable_channels_bitmap=0x%x\n",
 			le32_to_cpu(cmd.oem_uhb_allow_bitmap),
 			le32_to_cpu(cmd.force_disable_channels_bitmap));
+	IWL_DEBUG_RADIO(mld,
+			"sending LARI_CONFIG_CHANGE, oem_uhb_allow_extension_bitmap=0x%x\n",
+			le32_to_cpu(cmd.oem_uhb_allow_extension_bitmap));
 	IWL_DEBUG_RADIO(mld,
 			"sending LARI_CONFIG_CHANGE, edt_bitmap=0x%x, oem_320mhz_allow_bitmap=0x%x\n",
 			le32_to_cpu(cmd.edt_bitmap),
@@ -427,22 +472,13 @@ void iwl_mld_configure_lari(struct iwl_mld *mld)
 	IWL_DEBUG_RADIO(mld,
 			"sending LARI_CONFIG_CHANGE, oem_unii9_enable=0x%x\n",
 			le32_to_cpu(cmd.oem_unii9_enable));
+	IWL_DEBUG_RADIO(mld,
+			"sending LARI_CONFIG_CHANGE, wcpe_bitmap=0x%x\n",
+			le32_to_cpu(cmd.wcpe_bitmap));
 
-	if (iwl_fw_lookup_cmd_ver(mld->fw,
-				  WIDE_ID(REGULATORY_AND_NVM_GROUP,
-					  LARI_CONFIG_CHANGE), 12) == 12) {
-		int cmd_size = offsetof(typeof(cmd), oem_11bn_allow_bitmap);
-
-		ret = iwl_mld_send_cmd_pdu(mld,
-					   WIDE_ID(REGULATORY_AND_NVM_GROUP,
-						   LARI_CONFIG_CHANGE),
-					   &cmd, cmd_size);
-	} else {
-		ret = iwl_mld_send_cmd_pdu(mld,
-					   WIDE_ID(REGULATORY_AND_NVM_GROUP,
-						   LARI_CONFIG_CHANGE),
-					   &cmd);
-	}
+	ret = iwl_mld_send_cmd_pdu(mld,
+				   WIDE_ID(REGULATORY_AND_NVM_GROUP,
+					   LARI_CONFIG_CHANGE), &cmd, cmd_len);
 	if (ret)
 		IWL_DEBUG_RADIO(mld,
 				"Failed to send LARI_CONFIG_CHANGE (%d)\n",
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/sta.c b/drivers/net/wireless/intel/iwlwifi/mld/sta.c
index e18d86f021dcc..7957ac11b0cd5 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/sta.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/sta.c
@@ -355,10 +355,9 @@ static void iwl_mld_fill_pkt_ext(struct iwl_mld *mld,
 
 	for (int i = 0; i < MAX_HE_SUPP_NSS; i++) {
 		for (int bw = 0;
-		     bw < ARRAY_SIZE(*pkt_ext->pkt_ext_qam_th[i]);
+		     bw < ARRAY_SIZE(pkt_ext->pkt_ext_qam_th[i]);
 		     bw++) {
-			u8 *qam_th =
-				&pkt_ext->pkt_ext_qam_th[i][bw][0];
+			u8 *qam_th = pkt_ext->pkt_ext_qam_th[i][bw];
 
 			IWL_DEBUG_HT(mld,
 				     "PPE table: nss[%d] bw[%d] PPET8 = %d, PPET16 = %d\n",
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/thermal.c b/drivers/net/wireless/intel/iwlwifi/mld/thermal.c
index f8a8c35066beb..e445b1d7d4b00 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/thermal.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/thermal.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #ifdef CONFIG_THERMAL
 #include <linux/sort.h>
@@ -272,6 +272,7 @@ static void iwl_mld_thermal_zone_register(struct iwl_mld *mld)
 	if (ret) {
 		IWL_DEBUG_TEMP(mld, "Failed to enable thermal zone\n");
 		thermal_zone_device_unregister(mld->tzone);
+		mld->tzone = NULL;
 	}
 }
 
@@ -385,10 +386,8 @@ static void iwl_mld_thermal_zone_unregister(struct iwl_mld *mld)
 		return;
 
 	IWL_DEBUG_TEMP(mld, "Thermal zone device unregister\n");
-	if (mld->tzone) {
-		thermal_zone_device_unregister(mld->tzone);
-		mld->tzone = NULL;
-	}
+	thermal_zone_device_unregister(mld->tzone);
+	mld->tzone = NULL;
 }
 
 static void iwl_mld_cooling_device_unregister(struct iwl_mld *mld)
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/tx.c b/drivers/net/wireless/intel/iwlwifi/mld/tx.c
index 2185dade95f8c..2df6643a5aa80 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/tx.c
@@ -1120,6 +1120,10 @@ void iwl_mld_handle_tx_resp_notif(struct iwl_mld *mld,
 	ssn = le32_to_cpup((__le32 *)agg_status +
 			   tx_resp->frame_count) & 0xFFFF;
 
+	if (IWL_FW_CHECK(mld, txq_id >= ARRAY_SIZE(mld->fw_id_to_txq),
+			 "Invalid txq id %d\n", txq_id))
+		return;
+
 	__skb_queue_head_init(&skbs);
 
 	/* we can free until ssn % q.n_bd not inclusive */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
index 9a74f60c91850..6b11fa32ea5c6 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
@@ -1509,6 +1509,10 @@ static void iwl_mvm_report_wakeup_reasons(struct iwl_mvm *mvm,
 		if (WARN_ON_ONCE(truncated < 0))
 			truncated = 0;
 
+		/* this would be a firmware bug */
+		if (WARN_ON_ONCE(pktsize < sizeof(*hdr)))
+			return;
+
 		if (ieee80211_is_data(hdr->frame_control)) {
 			int hdrlen = ieee80211_hdrlen(hdr->frame_control);
 			int ivlen = 0, icvlen = 4; /* also FCS */
@@ -1517,10 +1521,6 @@ static void iwl_mvm_report_wakeup_reasons(struct iwl_mvm *mvm,
 			if (!pkt)
 				goto report;
 
-			skb_put_data(pkt, pktdata, hdrlen);
-			pktdata += hdrlen;
-			pktsize -= hdrlen;
-
 			if (ieee80211_has_protected(hdr->frame_control)) {
 				/*
 				 * This is unlocked and using gtk_i(c)vlen,
@@ -1539,13 +1539,24 @@ static void iwl_mvm_report_wakeup_reasons(struct iwl_mvm *mvm,
 
 			/* if truncated, FCS/ICV is (partially) gone */
 			if (truncated >= icvlen) {
-				icvlen = 0;
 				truncated -= icvlen;
+				icvlen = 0;
 			} else {
 				icvlen -= truncated;
 				truncated = 0;
 			}
 
+			if (IWL_FW_CHECK(mvm,
+					 pktsize <= hdrlen + ivlen + icvlen,
+					 "pktsize is too small %d\n",
+					 pktsize)) {
+				kfree_skb(pkt);
+				return;
+			}
+
+			skb_put_data(pkt, pktdata, hdrlen);
+			pktdata += hdrlen;
+			pktsize -= hdrlen;
 			pktsize -= ivlen + icvlen;
 			pktdata += ivlen;
 
@@ -2123,16 +2134,16 @@ static void iwl_mvm_parse_wowlan_info_notif(struct iwl_mvm *mvm,
 					    struct iwl_wowlan_status_data *status,
 					    u32 len)
 {
-	if (IWL_FW_CHECK(mvm, data->num_mlo_link_keys,
-			 "MLO is not supported, shouldn't receive MLO keys\n"))
-		return;
-
 	if (len < sizeof(*data)) {
 		IWL_ERR(mvm, "Invalid WoWLAN info notification!\n");
 		status = NULL;
 		return;
 	}
 
+	if (IWL_FW_CHECK(mvm, data->num_mlo_link_keys,
+			 "MLO is not supported, shouldn't receive MLO keys\n"))
+		return;
+
 	if (mvm->fast_resume)
 		return;
 
@@ -2448,13 +2459,15 @@ iwl_mvm_netdetect_query_results(struct iwl_mvm *mvm,
 
 	if (fw_has_api(&mvm->fw->ucode_capa,
 		       IWL_UCODE_TLV_API_SCAN_OFFLOAD_CHANS)) {
-		query_len = sizeof(struct iwl_scan_offload_match_info);
 		matches_len = sizeof(struct iwl_scan_offload_profile_match) *
 			max_profiles;
+		query_len = offsetof(struct iwl_scan_offload_match_info,
+				     matches) + matches_len;
 	} else {
-		query_len = sizeof(struct iwl_scan_offload_profiles_query_v1);
 		matches_len = sizeof(struct iwl_scan_offload_profile_match_v1) *
 			max_profiles;
+		query_len = sizeof(struct iwl_scan_offload_profiles_query_v1) +
+			matches_len;
 	}
 
 	len = iwl_rx_packet_payload_len(cmd.resp_pkt);
@@ -2756,7 +2769,7 @@ static int iwl_mvm_wowlan_store_wake_pkt(struct iwl_mvm *mvm,
 					 struct iwl_wowlan_status_data *status,
 					 u32 len)
 {
-	u32 data_size, packet_len = le32_to_cpu(notif->wake_packet_length);
+	u32 data_size, packet_len;
 
 	if (len < sizeof(*notif)) {
 		IWL_ERR(mvm, "Invalid WoWLAN wake packet notification!\n");
@@ -2775,6 +2788,7 @@ static int iwl_mvm_wowlan_store_wake_pkt(struct iwl_mvm *mvm,
 		return -EIO;
 	}
 
+	packet_len = le32_to_cpu(notif->wake_packet_length);
 	data_size = len - offsetof(struct iwl_wowlan_wake_pkt_notif, wake_packet);
 
 	/* data_size got the padding from the notification, remove it. */
@@ -2807,7 +2821,8 @@ static void iwl_mvm_nd_match_info_handler(struct iwl_mvm *mvm,
 	if (IS_ERR_OR_NULL(vif))
 		return;
 
-	if (len < sizeof(struct iwl_scan_offload_match_info) + matches_len) {
+	if (len < offsetof(struct iwl_scan_offload_match_info, matches) +
+		    matches_len) {
 		IWL_ERR(mvm, "Invalid scan match info notification\n");
 		return;
 	}
@@ -2942,6 +2957,11 @@ static bool iwl_mvm_wait_d3_notif(struct iwl_notif_wait_data *notif_wait,
 	case WIDE_ID(PROT_OFFLOAD_GROUP, D3_END_NOTIFICATION): {
 		struct iwl_d3_end_notif *notif = (void *)pkt->data;
 
+		if (len < sizeof(*notif)) {
+			IWL_ERR(mvm, "Invalid D3 end notification size\n");
+			break;
+		}
+
 		d3_data->d3_end_flags = __le32_to_cpu(notif->flags);
 		d3_data->notif_received |= IWL_D3_NOTIF_D3_END_NOTIF;
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c b/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c
index 3a14ca5e512ae..8d7f60dcd0270 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c
@@ -646,7 +646,8 @@ static void iter(struct ieee80211_hw *hw,
 	WARN_ON(*target->cipher == IWL_LOCATION_CIPHER_INVALID);
 }
 
-static void
+/* The tk buffer is assumed to be exactly TK_11AZ_LEN bytes */
+static int
 iwl_mvm_ftm_set_secured_ranging(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 				u8 *bssid, u8 *cipher, u8 *hltk, u8 *tk,
 				u8 *rx_pn, u8 *tx_pn, __le32 *flags)
@@ -656,12 +657,12 @@ iwl_mvm_ftm_set_secured_ranging(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 
 	if (mvmvif->ftm_unprotected)
-		return;
+		return 0;
 #endif
 
 	if (!(le32_to_cpu(*flags) & (IWL_INITIATOR_AP_FLAGS_NON_TB |
 		       IWL_INITIATOR_AP_FLAGS_TB)))
-		return;
+		return 0;
 
 	lockdep_assert_held(&mvm->mutex);
 
@@ -679,14 +680,31 @@ iwl_mvm_ftm_set_secured_ranging(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		if (vif->cfg.assoc &&
 		    !memcmp(vif->bss_conf.bssid, bssid, ETH_ALEN)) {
 			struct iwl_mvm_ftm_iter_data target;
+			u32 key_len;
 
 			target.bssid = bssid;
 			target.cipher = cipher;
 			target.tk = NULL;
 			ieee80211_iter_keys(mvm->hw, vif, iter, &target);
 
-			if (!WARN_ON(!target.tk))
-				memcpy(tk, target.tk, TK_11AZ_LEN);
+			if (WARN_ON(!target.tk))
+				return -EINVAL;
+
+			switch (*target.cipher) {
+			case IWL_LOCATION_CIPHER_CCMP_128:
+			case IWL_LOCATION_CIPHER_GCMP_128:
+				key_len = WLAN_KEY_LEN_CCMP;
+				break;
+			case IWL_LOCATION_CIPHER_GCMP_256:
+				key_len = WLAN_KEY_LEN_GCMP_256;
+				break;
+			default:
+				WARN_ON(1);
+				return -EINVAL;
+			}
+
+			memset(tk, 0, TK_11AZ_LEN);
+			memcpy(tk, target.tk, key_len);
 		} else {
 			memcpy(tk, entry->tk, sizeof(entry->tk));
 		}
@@ -695,8 +713,10 @@ iwl_mvm_ftm_set_secured_ranging(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		memcpy(tx_pn, entry->tx_pn, sizeof(entry->tx_pn));
 
 		FTM_SET_FLAG(SECURED);
-		return;
+		return 0;
 	}
+
+	return 0;
 }
 
 static int
@@ -708,12 +728,11 @@ iwl_mvm_ftm_put_target_v7(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	if (err)
 		return err;
 
-	iwl_mvm_ftm_set_secured_ranging(mvm, vif, target->bssid,
-					&target->cipher, target->hltk,
-					target->tk, target->rx_pn,
-					target->tx_pn,
-					&target->initiator_ap_flags);
-	return err;
+	return iwl_mvm_ftm_set_secured_ranging(mvm, vif, target->bssid,
+					      &target->cipher, target->hltk,
+					      target->tk, target->rx_pn,
+					      target->tx_pn,
+					      &target->initiator_ap_flags);
 }
 
 static int iwl_mvm_ftm_start_v11(struct iwl_mvm *mvm,
@@ -881,11 +900,13 @@ iwl_mvm_ftm_put_target_v10(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	iwl_mvm_ftm_set_target_flags(mvm, peer, &target->initiator_ap_flags);
 	iwl_mvm_ftm_set_sta(mvm, vif, peer, &target->sta_id,
 			    &target->initiator_ap_flags);
-	iwl_mvm_ftm_set_secured_ranging(mvm, vif, target->bssid,
-					&target->cipher, target->hltk,
-					target->tk, target->rx_pn,
-					target->tx_pn,
-					&target->initiator_ap_flags);
+	ret = iwl_mvm_ftm_set_secured_ranging(mvm, vif, target->bssid,
+					      &target->cipher, target->hltk,
+					      target->tk, target->rx_pn,
+					      target->tx_pn,
+					      &target->initiator_ap_flags);
+	if (ret)
+		return ret;
 
 	i2r_max_sts = IWL_MVM_FTM_I2R_MAX_STS > 1 ? 1 :
 		IWL_MVM_FTM_I2R_MAX_STS;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
index 6e507d6dcdd2a..43a7230f1ab40 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
@@ -964,12 +964,22 @@ int iwl_mvm_get_sar_geo_profile(struct iwl_mvm *mvm)
 		return ret;
 	}
 
+	if (IWL_FW_CHECK(mvm,
+			 iwl_rx_packet_payload_len(cmd.resp_pkt) !=
+			 sizeof(*resp),
+			 "Wrong size for iwl_geo_tx_power_profiles_resp: %d\n",
+			 iwl_rx_packet_payload_len(cmd.resp_pkt))) {
+		ret = -EIO;
+		goto out;
+	}
+
 	resp = (void *)cmd.resp_pkt->data;
 	ret = le32_to_cpu(resp->profile_idx);
 
 	if (WARN_ON(ret > BIOS_GEO_MAX_PROFILE_NUM))
 		ret = -EIO;
 
+out:
 	iwl_free_resp(&cmd);
 	return ret;
 }
@@ -1507,12 +1517,48 @@ static int iwl_mvm_fill_lari_config(struct iwl_fw_runtime *fwrt,
 	return 0;
 }
 
+static void iwl_mvm_send_lari_cfg_extension(struct iwl_mvm *mvm)
+{
+	struct iwl_fw_runtime *fwrt = &mvm->fwrt;
+	struct iwl_lari_config_extension_cmd cmd = {};
+	u32 cmd_id = WIDE_ID(REGULATORY_AND_NVM_GROUP,
+			     LARI_CONFIG_EXTENSION);
+	u32 value;
+	int ret;
+
+	if (iwl_fw_lookup_cmd_ver(mvm->fw, cmd_id, 0) < 1)
+		return;
+
+	ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_REGULATORY_CONFIG, &value);
+	if (ret)
+		return;
+
+	cmd.dsm_table_hdr.table_source = fwrt->dsm_source;
+	cmd.dsm_table_hdr.table_revision = fwrt->dsm_revision;
+	cmd.oem_uhb_allow_extension_bitmap = cpu_to_le32(value);
+
+	IWL_DEBUG_RADIO(mvm,
+			"sending LARI_CONFIG_EXTENSION, oem_uhb_allow_extension_bitmap=0x%x\n",
+			le32_to_cpu(cmd.oem_uhb_allow_extension_bitmap));
+	ret = iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, sizeof(cmd), &cmd);
+	if (ret < 0)
+		IWL_DEBUG_RADIO(mvm,
+				"Failed to send LARI_CONFIG_EXTENSION (%d)\n",
+				ret);
+}
+
 static void iwl_mvm_lari_cfg(struct iwl_mvm *mvm)
 {
 	struct iwl_lari_config_change_cmd cmd;
 	size_t cmd_size;
 	int ret;
 
+	/*
+	 * LARI_CONFIG_CHANGE triggers a profile update, so send
+	 * LARI_CONFIG_EXTENSION first to make sure its data is applied
+	 * in the same update.
+	 */
+	iwl_mvm_send_lari_cfg_extension(mvm);
 	ret = iwl_mvm_fill_lari_config(&mvm->fwrt, &cmd, &cmd_size);
 	if (!ret) {
 		ret = iwl_mvm_send_cmd_pdu(mvm,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
index d6a8624b1ae5c..b2c5be22c293c 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
@@ -1493,49 +1493,54 @@ void iwl_mvm_rx_beacon_notif(struct iwl_mvm *mvm,
 {
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
 	unsigned int pkt_len = iwl_rx_packet_payload_len(pkt);
-	struct iwl_extended_beacon_notif *beacon = (void *)pkt->data;
-	struct iwl_extended_beacon_notif_v5 *beacon_v5 = (void *)pkt->data;
 	struct ieee80211_vif *csa_vif;
 	struct ieee80211_vif *tx_blocked_vif;
 	struct agg_tx_status *agg_status;
+	u32 beacon_gp2;
 	u16 status;
 
 	lockdep_assert_held(&mvm->mutex);
 
-	mvm->ap_last_beacon_gp2 = le32_to_cpu(beacon->gp2);
-
 	if (!iwl_mvm_is_short_beacon_notif_supported(mvm)) {
+		struct iwl_extended_beacon_notif_v5 *beacon = (void *)pkt->data;
 		struct iwl_tx_resp *beacon_notify_hdr =
-			&beacon_v5->beacon_notify_hdr;
+			&beacon->beacon_notify_hdr;
 
-		if (unlikely(pkt_len < sizeof(*beacon_v5)))
+		if (unlikely(pkt_len < sizeof(*beacon)))
 			return;
 
-		mvm->ibss_manager = beacon_v5->ibss_mgr_status != 0;
+		beacon_gp2 = le32_to_cpu(beacon->gp2);
+
+		mvm->ibss_manager = beacon->ibss_mgr_status != 0;
 		agg_status = iwl_mvm_get_agg_status(mvm, beacon_notify_hdr);
 		status = le16_to_cpu(agg_status->status) & TX_STATUS_MSK;
 		IWL_DEBUG_RX(mvm,
 			     "beacon status %#x retries:%d tsf:0x%016llX gp2:0x%X rate:%d\n",
 			     status, beacon_notify_hdr->failure_frame,
 			     le64_to_cpu(beacon->tsf),
-			     mvm->ap_last_beacon_gp2,
+			     beacon_gp2,
 			     le32_to_cpu(beacon_notify_hdr->initial_rate));
 	} else {
+		const struct iwl_extended_beacon_notif *beacon =
+			(void *)pkt->data;
+
 		if (unlikely(pkt_len < sizeof(*beacon)))
 			return;
 
+		beacon_gp2 = le32_to_cpu(beacon->gp2);
+
 		mvm->ibss_manager = beacon->ibss_mgr_status != 0;
 		status = le32_to_cpu(beacon->status) & TX_STATUS_MSK;
 		IWL_DEBUG_RX(mvm,
 			     "beacon status %#x tsf:0x%016llX gp2:0x%X\n",
 			     status, le64_to_cpu(beacon->tsf),
-			     mvm->ap_last_beacon_gp2);
+			     beacon_gp2);
 	}
 
 	csa_vif = rcu_dereference_protected(mvm->csa_vif,
 					    lockdep_is_held(&mvm->mutex));
 	if (unlikely(csa_vif && csa_vif->bss_conf.csa_active))
-		iwl_mvm_csa_count_down(mvm, csa_vif, mvm->ap_last_beacon_gp2,
+		iwl_mvm_csa_count_down(mvm, csa_vif, beacon_gp2,
 				       (status == TX_STATUS_SUCCESS));
 
 	tx_blocked_vif = rcu_dereference_protected(mvm->csa_tx_blocked_vif,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
index 74bd4038fd560..5bd246e379432 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
@@ -152,7 +152,7 @@ struct ieee80211_regdomain *iwl_mvm_get_regdomain(struct wiphy *wiphy,
 				      resp->channels,
 				      __le16_to_cpu(resp->mcc),
 				      __le16_to_cpu(resp->geo_info),
-				      le32_to_cpu(resp->cap), resp_ver);
+				      le32_to_cpu(resp->cap), resp_ver, NULL);
 	/* Store the return source id */
 	src_id = resp->source_id;
 	if (IS_ERR_OR_NULL(regd)) {
@@ -625,9 +625,7 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 	hw->wiphy->max_sched_scan_reqs = 1;
 	hw->wiphy->max_sched_scan_ssids = PROBE_OPTION_MAX;
 	hw->wiphy->max_match_sets = iwl_umac_scan_get_max_profiles(mvm->fw);
-	/* we create the 802.11 header and zero length SSID IE. */
-	hw->wiphy->max_sched_scan_ie_len =
-		SCAN_OFFLOAD_PROBE_REQ_SIZE - 24 - 2;
+	hw->wiphy->max_sched_scan_ie_len = iwl_mvm_max_scan_ie_len(mvm);
 	hw->wiphy->max_sched_scan_plans = IWL_MAX_SCHED_SCAN_PLANS;
 	hw->wiphy->max_sched_scan_plan_interval = U16_MAX;
 
@@ -1129,7 +1127,7 @@ static void iwl_mvm_cleanup_iterator(void *data, u8 *mac,
 	RCU_INIT_POINTER(mvmvif->deflink.probe_resp_data, NULL);
 }
 
-static void iwl_mvm_restart_cleanup(struct iwl_mvm *mvm)
+void iwl_mvm_restart_cleanup(struct iwl_mvm *mvm)
 {
 	iwl_mvm_stop_device(mvm);
 
@@ -2352,10 +2350,9 @@ int iwl_mvm_set_sta_pkt_ext(struct iwl_mvm *mvm,
 		int bw;
 
 		for (bw = 0;
-		     bw < ARRAY_SIZE(*pkt_ext->pkt_ext_qam_th[i]);
+		     bw < ARRAY_SIZE(pkt_ext->pkt_ext_qam_th[i]);
 		     bw++) {
-			u8 *qam_th =
-				&pkt_ext->pkt_ext_qam_th[i][bw][0];
+			u8 *qam_th = pkt_ext->pkt_ext_qam_th[i][bw];
 
 			IWL_DEBUG_HT(mvm,
 				     "PPE table: nss[%d] bw[%d] PPET8 = %d, PPET16 = %d\n",
@@ -3061,7 +3058,6 @@ void iwl_mvm_stop_ap_ibss_common(struct iwl_mvm *mvm,
 	}
 
 	mvmvif->ap_ibss_active = false;
-	mvm->ap_last_beacon_gp2 = 0;
 
 	if (vif->type == NL80211_IFTYPE_AP && !vif->p2p) {
 		iwl_mvm_vif_set_low_latency(mvmvif, false,
@@ -3490,7 +3486,7 @@ static void iwl_mvm_check_he_obss_narrow_bw_ru_iter(struct wiphy *wiphy,
 	elem = cfg80211_find_elem(WLAN_EID_EXT_CAPABILITY, ies->data,
 				  ies->len);
 
-	if (!elem || elem->datalen < 10 ||
+	if (!elem || elem->datalen < 11 ||
 	    !(elem->data[10] &
 	      WLAN_EXT_CAPA10_OBSS_NARROW_BW_RU_TOLERANCE_SUPPORT)) {
 		data->tolerated = false;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
index 683cac56822c8..c64e4ed00986a 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
@@ -1168,9 +1168,6 @@ struct iwl_mvm {
 	struct ieee80211_vif __rcu *csa_tx_blocked_vif;
 	u8 csa_tx_block_bcn_timeout;
 
-	/* system time of last beacon (for AP/GO interface) */
-	u32 ap_last_beacon_gp2;
-
 	/* indicates that we transmitted the last beacon */
 	bool ibss_manager;
 
@@ -1660,6 +1657,7 @@ void iwl_mvm_hwrate_to_tx_rate(u32 rate_n_flags,
 u8 iwl_mvm_rate_idx_to_fw_idx(const struct iwl_fw *fw, int rate_idx);
 u8 iwl_mvm_mac80211_ac_to_ucode_ac(enum ieee80211_ac_numbers ac);
 bool iwl_mvm_is_nic_ack_enabled(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
+void iwl_mvm_restart_cleanup(struct iwl_mvm *mvm);
 
 static inline void iwl_mvm_dump_nic_error_log(struct iwl_mvm *mvm)
 {
@@ -1675,14 +1673,24 @@ u32 iwl_mvm_get_systime(struct iwl_mvm *mvm);
 /* Tx / Host Commands */
 int iwl_mvm_send_cmd(struct iwl_mvm *mvm,
 		     struct iwl_host_cmd *cmd);
-int iwl_mvm_send_cmd_pdu(struct iwl_mvm *mvm, u32 id,
-			 u32 flags, u16 len, const void *data);
+int _iwl_mvm_send_cmd_pdu(struct iwl_mvm *mvm, u32 id,
+			  u32 flags, u16 len, const void *data);
+#define iwl_mvm_send_cmd_pdu(mvm, id, flags, len, data) ({		\
+	BUILD_BUG_ON(__builtin_constant_p(len) &&			\
+		     (u16)(len) > IWL_MAX_CMD_PAYLOAD_SIZE);		\
+	_iwl_mvm_send_cmd_pdu(mvm, id, flags, len, data);		\
+})
 int iwl_mvm_send_cmd_status(struct iwl_mvm *mvm,
 			    struct iwl_host_cmd *cmd,
 			    u32 *status);
-int iwl_mvm_send_cmd_pdu_status(struct iwl_mvm *mvm, u32 id,
-				u16 len, const void *data,
-				u32 *status);
+int _iwl_mvm_send_cmd_pdu_status(struct iwl_mvm *mvm, u32 id,
+				 u16 len, const void *data,
+				 u32 *status);
+#define iwl_mvm_send_cmd_pdu_status(mvm, id, len, data, status) ({	\
+	BUILD_BUG_ON(__builtin_constant_p(len) &&			\
+		     (u16)(len) > IWL_MAX_CMD_PAYLOAD_SIZE);		\
+	_iwl_mvm_send_cmd_pdu_status(mvm, id, len, data, status);	\
+})
 int iwl_mvm_tx_skb_sta(struct iwl_mvm *mvm, struct sk_buff *skb,
 		       struct ieee80211_sta *sta);
 int iwl_mvm_tx_skb_non_sta(struct iwl_mvm *mvm, struct sk_buff *skb);
@@ -2395,14 +2403,12 @@ void iwl_mvm_sync_rx_queues_internal(struct iwl_mvm *mvm,
 				     bool sync,
 				     const void *data, u32 size);
 struct ieee80211_vif *iwl_mvm_get_bss_vif(struct iwl_mvm *mvm);
-struct ieee80211_vif *iwl_mvm_get_vif_by_macid(struct iwl_mvm *mvm, u32 macid);
 bool iwl_mvm_is_vif_assoc(struct iwl_mvm *mvm);
 
 #define MVM_TCM_PERIOD_MSEC 500
 #define MVM_TCM_PERIOD (HZ * MVM_TCM_PERIOD_MSEC / 1000)
 #define MVM_LL_PERIOD (10 * HZ)
 void iwl_mvm_tcm_work(struct work_struct *work);
-void iwl_mvm_recalc_tcm(struct iwl_mvm *mvm);
 void iwl_mvm_pause_tcm(struct iwl_mvm *mvm, bool with_cancel);
 void iwl_mvm_resume_tcm(struct iwl_mvm *mvm);
 void iwl_mvm_tcm_add_vif(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c b/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c
index 953218f1e025a..f76e57399c1f3 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2019, 2021-2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2019, 2021-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -416,6 +416,7 @@ iwl_mvm_update_mcc(struct iwl_mvm *mvm, const char *alpha2,
 	int ret, resp_ver;
 	u32 status;
 	int resp_len, n_channels;
+	unsigned int pkt_len;
 	u16 mcc;
 
 	if (WARN_ON_ONCE(!iwl_mvm_is_lar_supported(mvm)))
@@ -431,6 +432,7 @@ iwl_mvm_update_mcc(struct iwl_mvm *mvm, const char *alpha2,
 		return ERR_PTR(ret);
 
 	pkt = cmd.resp_pkt;
+	pkt_len = iwl_rx_packet_payload_len(pkt);
 
 	resp_ver = iwl_fw_lookup_notif_ver(mvm->fw, IWL_ALWAYS_LONG_GROUP,
 					   MCC_UPDATE_CMD, 0);
@@ -439,9 +441,18 @@ iwl_mvm_update_mcc(struct iwl_mvm *mvm, const char *alpha2,
 	if (resp_ver >= 8) {
 		struct iwl_mcc_update_resp_v8 *mcc_resp_v8 = (void *)pkt->data;
 
+		if (IWL_FW_CHECK(mvm, pkt_len < sizeof(*mcc_resp_v8),
+				 "MCC v8 response too short: %u\n", pkt_len)) {
+			resp_cp = ERR_PTR(-EINVAL);
+			goto exit;
+		}
+
 		n_channels =  __le32_to_cpu(mcc_resp_v8->n_channels);
-		if (iwl_rx_packet_payload_len(pkt) !=
-		    struct_size(mcc_resp_v8, channels, n_channels)) {
+		if (IWL_FW_CHECK(mvm,
+				 pkt_len !=
+				 struct_size(mcc_resp_v8, channels, n_channels),
+				 "invalid MCC v8 response size: %u (n_channels=%d)\n",
+				 pkt_len, n_channels)) {
 			resp_cp = ERR_PTR(-EINVAL);
 			goto exit;
 		}
@@ -464,9 +475,18 @@ iwl_mvm_update_mcc(struct iwl_mvm *mvm, const char *alpha2,
 			       IWL_UCODE_TLV_CAPA_MCC_UPDATE_11AX_SUPPORT)) {
 		struct iwl_mcc_update_resp_v4 *mcc_resp_v4 = (void *)pkt->data;
 
+		if (IWL_FW_CHECK(mvm, pkt_len < sizeof(*mcc_resp_v4),
+				 "MCC v4 response too short: %u\n", pkt_len)) {
+			resp_cp = ERR_PTR(-EINVAL);
+			goto exit;
+		}
+
 		n_channels =  __le32_to_cpu(mcc_resp_v4->n_channels);
-		if (iwl_rx_packet_payload_len(pkt) !=
-		    struct_size(mcc_resp_v4, channels, n_channels)) {
+		if (IWL_FW_CHECK(mvm,
+				 pkt_len !=
+				 struct_size(mcc_resp_v4, channels, n_channels),
+				 "invalid MCC v4 response size: %u (n_channels=%d)\n",
+				 pkt_len, n_channels)) {
 			resp_cp = ERR_PTR(-EINVAL);
 			goto exit;
 		}
@@ -489,9 +509,18 @@ iwl_mvm_update_mcc(struct iwl_mvm *mvm, const char *alpha2,
 	} else {
 		struct iwl_mcc_update_resp_v3 *mcc_resp_v3 = (void *)pkt->data;
 
+		if (IWL_FW_CHECK(mvm, pkt_len < sizeof(*mcc_resp_v3),
+				 "MCC v3 response too short: %u\n", pkt_len)) {
+			resp_cp = ERR_PTR(-EINVAL);
+			goto exit;
+		}
+
 		n_channels =  __le32_to_cpu(mcc_resp_v3->n_channels);
-		if (iwl_rx_packet_payload_len(pkt) !=
-		    struct_size(mcc_resp_v3, channels, n_channels)) {
+		if (IWL_FW_CHECK(mvm,
+				 pkt_len !=
+				 struct_size(mcc_resp_v3, channels, n_channels),
+				 "invalid MCC v3 response size: %u (n_channels=%d)\n",
+				 pkt_len, n_channels)) {
 			resp_cp = ERR_PTR(-EINVAL);
 			goto exit;
 		}
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
index 2297392db9558..1dd27292af9b8 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
@@ -155,8 +155,8 @@ static void iwl_mvm_rx_monitor_notif(struct iwl_mvm *mvm,
 	if (notif->type != cpu_to_le32(IWL_DP_MON_NOTIF_TYPE_EXT_CCA))
 		return;
 
-	/* FIXME: should fetch the link and not the vif */
-	vif = iwl_mvm_get_vif_by_macid(mvm, notif->link_id);
+	/* mac_id = link_id since we don't support MLO */
+	vif = iwl_mvm_rcu_dereference_vif_id(mvm, notif->link_id, false);
 	if (!vif || vif->type != NL80211_IFTYPE_STATION)
 		return;
 
@@ -707,6 +707,7 @@ static const struct iwl_hcmd_names iwl_mvm_regulatory_and_nvm_names[] = {
 	HCMD_NAME(NVM_ACCESS_COMPLETE),
 	HCMD_NAME(NVM_GET_INFO),
 	HCMD_NAME(TAS_CONFIG),
+	HCMD_NAME(LARI_CONFIG_EXTENSION),
 };
 
 /* Please keep this array *SORTED* by hex value.
@@ -954,7 +955,7 @@ static void iwl_mvm_frob_txf_key_iter(struct ieee80211_hw *hw,
 		}
 		match++;
 		if (match == keylen) {
-			memset(txf->buf + i - keylen, 0xAA, keylen);
+			memset(txf->buf + i + 1 - keylen, 0xAA, keylen);
 			match = 0;
 		}
 	}
@@ -1535,6 +1536,7 @@ void iwl_mvm_stop_device(struct iwl_mvm *mvm)
 	iwl_fw_cancel_timestamp(&mvm->fwrt);
 
 	clear_bit(IWL_MVM_STATUS_FIRMWARE_RUNNING, &mvm->status);
+	mvm->sf_state = SF_UNINIT;
 
 	iwl_mvm_pause_tcm(mvm, false);
 
@@ -2090,7 +2092,7 @@ static void iwl_op_mode_mvm_device_powered_off(struct iwl_op_mode *op_mode)
 
 	mutex_lock(&mvm->mutex);
 	clear_bit(IWL_MVM_STATUS_IN_D3, &mvm->status);
-	iwl_mvm_stop_device(mvm);
+	iwl_mvm_restart_cleanup(mvm);
 	mvm->fast_resume = false;
 	mutex_unlock(&mvm->mutex);
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c b/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c
index f39eb48864eb4..49dcb1388007b 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2021 - 2023, 2025 Intel Corporation
+ * Copyright (C) 2021 - 2023, 2025-2026 Intel Corporation
  */
 
 #include "mvm.h"
@@ -121,6 +121,8 @@ iwl_mvm_get_crosstimestamp_fw(struct iwl_mvm *mvm, u32 *gp2, u64 *sys_time)
 	*sys_time = (u64)le32_to_cpu(resp->platform_timestamp_hi) << 32 |
 		le32_to_cpu(resp->platform_timestamp_lo);
 
+	iwl_free_resp(&cmd);
+
 	return ret;
 }
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c
index 89ac4c6b3e543..e2382be8edd78 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2025 Intel Corporation
+ * Copyright (C) 2018-2026 Intel Corporation
  */
 #include "rs.h"
 #include "fw-api.h"
@@ -423,9 +423,14 @@ void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm,
 	struct iwl_lq_sta_rs_fw *lq_sta;
 	u32 flags;
 
+	notif = (void *)pkt->data;
+	if (IWL_FW_CHECK(mvm, notif->sta_id >= mvm->fw->ucode_capa.num_stations,
+			 "Invalid sta id (%d) in TLC notification\n",
+			 notif->sta_id))
+		return;
+
 	rcu_read_lock();
 
-	notif = (void *)pkt->data;
 	link_sta = rcu_dereference(mvm->fw_id_to_link_sta[notif->sta_id]);
 	sta = rcu_dereference(mvm->fw_id_to_mac_id[notif->sta_id]);
 	if (IS_ERR_OR_NULL(sta) || !link_sta) {
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rx.c b/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
index 269c4b45de807..ab1eb2eb0c3cd 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
@@ -1227,6 +1227,11 @@ void iwl_mvm_window_status_notif(struct iwl_mvm *mvm,
 		/* get the station */
 		sta_id = (ratid & BA_WINDOW_STATUS_STA_ID_MSK)
 			 >> BA_WINDOW_STATUS_STA_ID_POS;
+		if (IWL_FW_CHECK(mvm,
+				 sta_id >= mvm->fw->ucode_capa.num_stations,
+				 "Invalid sta id (%d) in BA window status notification\n",
+				 sta_id))
+			continue;
 		sta = rcu_dereference(mvm->fw_id_to_mac_id[sta_id]);
 		if (IS_ERR_OR_NULL(sta))
 			continue;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
index 79829f775c89b..3831b3c27e0f3 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -3144,8 +3144,6 @@ int iwl_mvm_sched_scan_start(struct iwl_mvm *mvm,
 	if (ret)
 		return ret;
 
-	iwl_mvm_build_scan_probe(mvm, vif, ies, &params);
-
 	/* for 6 GHZ band only PSC channels need to be added */
 	for (i = 0; i < params.n_channels; i++) {
 		struct ieee80211_channel *channel = params.channels[i];
@@ -3179,6 +3177,8 @@ int iwl_mvm_sched_scan_start(struct iwl_mvm *mvm,
 		goto out;
 	}
 
+	iwl_mvm_build_scan_probe(mvm, vif, ies, &params);
+
 	uid = iwl_mvm_build_scan_cmd(mvm, vif, &hcmd, &params, type);
 	if (uid < 0) {
 		ret = uid;
@@ -3217,6 +3217,10 @@ void iwl_mvm_rx_umac_scan_complete_notif(struct iwl_mvm *mvm,
 
 	mvm->mei_scan_filter.is_mei_limited_scan = false;
 
+	if (IWL_FW_CHECK(mvm, uid >= ARRAY_SIZE(mvm->scan_uid_status),
+			 "FW reports out-of-range scan UID %d\n", uid))
+		return;
+
 	IWL_DEBUG_SCAN(mvm,
 		       "Scan completed: uid=%u type=%u, status=%s, EBS=%s\n",
 		       uid, mvm->scan_uid_status[uid],
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c b/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c
index 1692b6e75f579..93b46c9e2333d 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c
@@ -951,6 +951,11 @@ void iwl_mvm_rx_session_protect_notif(struct iwl_mvm *mvm,
 	struct ieee80211_vif *vif;
 	struct iwl_mvm_vif *mvmvif;
 
+	if (IWL_FW_CHECK(mvm, id >= ARRAY_SIZE(mvm->vif_id_to_mac),
+			 "Invalid mac_link_id (%d) in session protect notif\n",
+			 id))
+		return;
+
 	rcu_read_lock();
 
 	/* note we use link ID == MAC ID */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
index dc69c71faa76c..d3c2fe8304777 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
@@ -1544,6 +1544,17 @@ static inline u32 iwl_mvm_get_scd_ssn(struct iwl_mvm *mvm,
 	return val & 0xFFF;
 }
 
+static inline size_t iwl_mvm_tx_resp_min_len(struct iwl_mvm *mvm,
+					     struct iwl_tx_resp *tx_resp)
+{
+	struct agg_tx_status *agg_status =
+		iwl_mvm_get_agg_status(mvm, tx_resp);
+
+	/* The aggregate response ends with a trailing SCD SSN __le32 word. */
+	return (u8 *)(agg_status + tx_resp->frame_count) - (u8 *)tx_resp +
+		sizeof(__le32);
+}
+
 static void iwl_mvm_rx_tx_cmd_single(struct iwl_mvm *mvm,
 				     struct iwl_rx_packet *pkt)
 {
@@ -1847,6 +1858,9 @@ static void iwl_mvm_rx_tx_cmd_agg(struct iwl_mvm *mvm,
 	int queue = SEQ_TO_QUEUE(sequence);
 	struct ieee80211_sta *sta;
 
+	if (WARN_ON_ONCE(iwl_mvm_has_new_tx_api(mvm)))
+		return;
+
 	if (WARN_ON_ONCE(queue < IWL_MVM_DQA_MIN_DATA_QUEUE &&
 			 (queue != IWL_MVM_DQA_BSS_CLIENT_QUEUE)))
 		return;
@@ -1881,6 +1895,26 @@ void iwl_mvm_rx_tx_cmd(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 {
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
 	struct iwl_tx_resp *tx_resp = (void *)pkt->data;
+	size_t min_len;
+
+	if (IWL_FW_CHECK(mvm, !tx_resp->frame_count,
+			 "invalid TX_CMD frame_count %u\n",
+			 tx_resp->frame_count))
+		return;
+
+	if (IWL_FW_CHECK(mvm,
+			 iwl_mvm_has_new_tx_api(mvm) &&
+			 tx_resp->frame_count != 1,
+			 "invalid TX_CMD frame_count %u for new TX API\n",
+			 tx_resp->frame_count))
+		return;
+
+	min_len = iwl_mvm_tx_resp_min_len(mvm, tx_resp);
+	if (IWL_FW_CHECK(mvm, iwl_rx_packet_payload_len(pkt) < min_len,
+			 "invalid TX_CMD len %u (frame_count %u, min %zu)\n",
+			 iwl_rx_packet_payload_len(pkt), tx_resp->frame_count,
+			 min_len))
+		return;
 
 	if (tx_resp->frame_count == 1)
 		iwl_mvm_rx_tx_cmd_single(mvm, pkt);
@@ -2100,8 +2134,14 @@ void iwl_mvm_rx_ba_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 			if (tid == IWL_MGMT_TID)
 				tid = IWL_MAX_TID_COUNT;
 
+			if (IWL_FW_CHECK(mvm, tid >=
+					 ARRAY_SIZE(mvmsta->tid_data),
+					 "invalid TID %d in compressed BA\n",
+					 tid))
+				continue;
+
 			if (mvmsta)
-				mvmsta->tid_data[i].lq_color = lq_color;
+				mvmsta->tid_data[tid].lq_color = lq_color;
 
 			iwl_mvm_tx_reclaim(mvm, sta_id, tid,
 					   (int)(le16_to_cpu(ba_tfd->q_num)),
@@ -2121,6 +2161,9 @@ void iwl_mvm_rx_ba_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 	ba_notif = (void *)pkt->data;
 	sta_id = ba_notif->sta_id;
 	tid = ba_notif->tid;
+	if (IWL_FW_CHECK(mvm, tid >= ARRAY_SIZE(mvmsta->tid_data),
+			 "invalid TID %d in BA notif\n", tid))
+		return;
 	/* "flow" corresponds to Tx queue */
 	txq = le16_to_cpu(ba_notif->scd_flow);
 	/* "ssn" is start of block-ack Tx window, corresponds to index
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
index 2e12f93ad32b4..2fd4a99611450 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
@@ -73,8 +73,8 @@ int iwl_mvm_send_cmd(struct iwl_mvm *mvm, struct iwl_host_cmd *cmd)
 	return ret;
 }
 
-int iwl_mvm_send_cmd_pdu(struct iwl_mvm *mvm, u32 id,
-			 u32 flags, u16 len, const void *data)
+int _iwl_mvm_send_cmd_pdu(struct iwl_mvm *mvm, u32 id,
+			  u32 flags, u16 len, const void *data)
 {
 	struct iwl_host_cmd cmd = {
 		.id = id,
@@ -137,8 +137,8 @@ int iwl_mvm_send_cmd_status(struct iwl_mvm *mvm, struct iwl_host_cmd *cmd,
 /*
  * We assume that the caller set the status to the sucess value
  */
-int iwl_mvm_send_cmd_pdu_status(struct iwl_mvm *mvm, u32 id, u16 len,
-				const void *data, u32 *status)
+int _iwl_mvm_send_cmd_pdu_status(struct iwl_mvm *mvm, u32 id, u16 len,
+				 const void *data, u32 *status)
 {
 	struct iwl_host_cmd cmd = {
 		.id = id,
@@ -689,36 +689,6 @@ struct ieee80211_vif *iwl_mvm_get_bss_vif(struct iwl_mvm *mvm)
 	return bss_iter_data.vif;
 }
 
-struct iwl_bss_find_iter_data {
-	struct ieee80211_vif *vif;
-	u32 macid;
-};
-
-static void iwl_mvm_bss_find_iface_iterator(void *_data, u8 *mac,
-					    struct ieee80211_vif *vif)
-{
-	struct iwl_bss_find_iter_data *data = _data;
-	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-
-	if (mvmvif->id == data->macid)
-		data->vif = vif;
-}
-
-struct ieee80211_vif *iwl_mvm_get_vif_by_macid(struct iwl_mvm *mvm, u32 macid)
-{
-	struct iwl_bss_find_iter_data data = {
-		.macid = macid,
-	};
-
-	lockdep_assert_held(&mvm->mutex);
-
-	ieee80211_iterate_active_interfaces_atomic(
-		mvm->hw, IEEE80211_IFACE_ITER_NORMAL,
-		iwl_mvm_bss_find_iface_iterator, &data);
-
-	return data.vif;
-}
-
 struct iwl_sta_iter_data {
 	bool assoc;
 };
@@ -1055,7 +1025,7 @@ static unsigned long iwl_mvm_calc_tcm_stats(struct iwl_mvm *mvm,
 	/*
 	 * If the current load isn't low we need to force re-evaluation
 	 * in the TCM period, so that we can return to low load if there
-	 * was no traffic at all (and thus iwl_mvm_recalc_tcm didn't get
+	 * was no traffic at all (and thus iwl_mvm_tcm_work() didn't get
 	 * triggered by traffic).
 	 */
 	if (load != IWL_MVM_TRAFFIC_LOW)
@@ -1081,8 +1051,11 @@ static unsigned long iwl_mvm_calc_tcm_stats(struct iwl_mvm *mvm,
 	return 0;
 }
 
-void iwl_mvm_recalc_tcm(struct iwl_mvm *mvm)
+void iwl_mvm_tcm_work(struct work_struct *work)
 {
+	struct delayed_work *delayed_work = to_delayed_work(work);
+	struct iwl_mvm *mvm = container_of(delayed_work, struct iwl_mvm,
+					   tcm.work);
 	unsigned long ts = jiffies;
 	bool handle_uapsd =
 		time_after(ts, mvm->tcm.uapsd_nonagg_ts +
@@ -1119,15 +1092,6 @@ void iwl_mvm_recalc_tcm(struct iwl_mvm *mvm)
 	iwl_mvm_tcm_results(mvm);
 }
 
-void iwl_mvm_tcm_work(struct work_struct *work)
-{
-	struct delayed_work *delayed_work = to_delayed_work(work);
-	struct iwl_mvm *mvm = container_of(delayed_work, struct iwl_mvm,
-					   tcm.work);
-
-	iwl_mvm_recalc_tcm(mvm);
-}
-
 void iwl_mvm_pause_tcm(struct iwl_mvm *mvm, bool with_cancel)
 {
 	spin_lock_bh(&mvm->tcm.lock);
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info.c b/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info.c
index d5eb895144efc..7f886200d6933 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2025 Intel Corporation
+ * Copyright (C) 2018-2026 Intel Corporation
  */
 #include "iwl-trans.h"
 #include "iwl-fh.h"
@@ -99,6 +99,11 @@ int iwl_pcie_init_fw_sec(struct iwl_trans *trans,
 	/* add 2 due to separators */
 	paging_cnt = iwl_pcie_get_num_sections(fw, lmac_cnt + umac_cnt + 2);
 
+	if (WARN_ON(lmac_cnt > ARRAY_SIZE(ctxt_dram->lmac_img) ||
+		    umac_cnt > ARRAY_SIZE(ctxt_dram->umac_img) ||
+		    paging_cnt > ARRAY_SIZE(ctxt_dram->virtual_img)))
+		return -EINVAL;
+
 	dram->fw = kzalloc_objs(*dram->fw, umac_cnt + lmac_cnt);
 	if (!dram->fw)
 		return -ENOMEM;
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
index 7a7b10152e5ae..a3e6c9e09a3be 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
@@ -548,6 +548,7 @@ VISIBLE_IF_IWLWIFI_KUNIT const struct pci_device_id iwl_hw_card_ids[] = {
 	{IWL_PCI_DEVICE(0xD340, PCI_ANY_ID, iwl_sc_mac_cfg)},
 	{IWL_PCI_DEVICE(0x6E70, PCI_ANY_ID, iwl_sc_mac_cfg)},
 	{IWL_PCI_DEVICE(0xD240, PCI_ANY_ID, iwl_sc_mac_cfg)},
+	{IWL_PCI_DEVICE(0x9327, PCI_ANY_ID, iwl_sc_mac_cfg)},
 #endif /* CONFIG_IWLMVM || CONFIG_IWLMLD */
 
 	{0}
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/rx.c b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/rx.c
index fe263cdc2e4f4..4631e11f2a964 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/rx.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2003-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2003-2014, 2018-2024, 2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -1238,11 +1238,16 @@ void iwl_pcie_rx_free(struct iwl_trans *trans)
 		}
 	}
 	kfree(trans_pcie->rx_pool);
+	trans_pcie->rx_pool = NULL;
 	kfree(trans_pcie->global_table);
+	trans_pcie->global_table = NULL;
 	kfree(trans_pcie->rxq);
+	trans_pcie->rxq = NULL;
 
-	if (trans_pcie->alloc_page)
+	if (trans_pcie->alloc_page) {
 		__free_pages(trans_pcie->alloc_page, trans_pcie->rx_page_order);
+		trans_pcie->alloc_page = NULL;
+	}
 }
 
 static void iwl_pcie_rx_move_to_allocator(struct iwl_rxq *rxq,
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/tx-gen2.c b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/tx-gen2.c
index bda9f807321ec..27f4973791d00 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/tx-gen2.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/tx-gen2.c
@@ -773,6 +773,8 @@ int iwl_txq_gen2_tx(struct iwl_trans *trans, struct sk_buff *skb,
 
 	tfd = iwl_txq_gen2_build_tfd(trans, txq, dev_cmd, skb, out_meta);
 	if (!tfd) {
+		txq->entries[idx].skb = NULL;
+		txq->entries[idx].cmd = NULL;
 		spin_unlock(&txq->lock);
 		return -1;
 	}
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/tx.c b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/tx.c
index fdb3ba4f63c01..dc2fe0c4f68c8 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/tx.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2003-2014, 2018-2021, 2023-2025 Intel Corporation
+ * Copyright (C) 2003-2014, 2018-2021, 2023-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -1149,10 +1149,17 @@ bool iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int txq_id, u16 ssn,
 			       unsigned int wdg_timeout)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	struct iwl_txq *txq = trans_pcie->txqs.txq[txq_id];
+	struct iwl_txq *txq;
 	int fifo = -1;
 	bool scd_bug = false;
 
+	if (WARN_ONCE(txq_id < 0 ||
+		      txq_id >= trans->mac_cfg->base->num_of_queues,
+		      "queue %d out of range", txq_id))
+		return false;
+
+	txq = trans_pcie->txqs.txq[txq_id];
+
 	if (test_and_set_bit(txq_id, trans_pcie->txqs.queue_used))
 		WARN_ONCE(1, "queue %d already used - expect issues", txq_id);
 
diff --git a/drivers/net/wireless/intersil/p54/Kconfig b/drivers/net/wireless/intersil/p54/Kconfig
index 003c378ed131d..44b0f1a724aad 100644
--- a/drivers/net/wireless/intersil/p54/Kconfig
+++ b/drivers/net/wireless/intersil/p54/Kconfig
@@ -10,7 +10,7 @@ config P54_COMMON
 	  also need to be enabled in order to support any devices.
 
 	  These devices require softmac firmware which can be found at
-	  <http://wireless.wiki.kernel.org/en/users/Drivers/p54>
+	  <https://wireless.docs.kernel.org/en/latest/en/users/drivers/p54.html>
 
 	  If you choose to build a module, it'll be called p54common.
 
@@ -22,7 +22,7 @@ config P54_USB
 	  This driver is for USB isl38xx based wireless cards.
 
 	  These devices require softmac firmware which can be found at
-	  <http://wireless.wiki.kernel.org/en/users/Drivers/p54>
+	  <https://wireless.docs.kernel.org/en/latest/en/users/drivers/p54.html>
 
 	  If you choose to build a module, it'll be called p54usb.
 
@@ -36,7 +36,7 @@ config P54_PCI
 	  supported by the fullmac driver/firmware.
 
 	  This driver requires softmac firmware which can be found at
-	  <http://wireless.wiki.kernel.org/en/users/Drivers/p54>
+	  <https://wireless.docs.kernel.org/en/latest/en/users/drivers/p54.html>
 
 	  If you choose to build a module, it'll be called p54pci.
 
diff --git a/drivers/net/wireless/intersil/p54/fwio.c b/drivers/net/wireless/intersil/p54/fwio.c
index 3baf8ab01e22b..a3d9053f043c6 100644
--- a/drivers/net/wireless/intersil/p54/fwio.c
+++ b/drivers/net/wireless/intersil/p54/fwio.c
@@ -131,9 +131,7 @@ int p54_parse_firmware(struct ieee80211_hw *dev, const struct firmware *fw)
 
 	if (priv->fw_var < 0x500)
 		wiphy_info(priv->hw->wiphy,
-			   "you are using an obsolete firmware. "
-			   "visit http://wireless.wiki.kernel.org/en/users/Drivers/p54 "
-			   "and grab one for \"kernel >= 2.6.28\"!\n");
+			   "you are using an obsolete firmware. visit https://wireless.docs.kernel.org/en/latest/en/users/drivers/p54.html and grab one for \"kernel >= 2.6.28\"!\n");
 
 	if (priv->fw_var >= 0x300) {
 		/* Firmware supports QoS, use it! */
diff --git a/drivers/net/wireless/intersil/p54/p54usb.c b/drivers/net/wireless/intersil/p54/p54usb.c
index c0d3b5329f4eb..b88a3dadddc09 100644
--- a/drivers/net/wireless/intersil/p54/p54usb.c
+++ b/drivers/net/wireless/intersil/p54/p54usb.c
@@ -36,7 +36,7 @@ static struct usb_driver p54u_driver;
  * Note:
  *
  * Always update our wiki's device list (located at:
- * http://wireless.wiki.kernel.org/en/users/Drivers/p54/devices ),
+ * https://wireless.docs.kernel.org/en/latest/en/users/drivers/p54/devices.html ),
  * whenever you add a new device.
  */
 
diff --git a/drivers/net/wireless/intersil/p54/txrx.c b/drivers/net/wireless/intersil/p54/txrx.c
index 1294a1d6528e2..9f491334c8d04 100644
--- a/drivers/net/wireless/intersil/p54/txrx.c
+++ b/drivers/net/wireless/intersil/p54/txrx.c
@@ -499,11 +499,19 @@ static void p54_rx_eeprom_readback(struct p54_common *priv,
 		if (le16_to_cpu(eeprom->v2.len) != priv->eeprom_slice_size)
 			return;
 
+		if (eeprom->v2.data + priv->eeprom_slice_size >
+		    skb_tail_pointer(skb))
+			return;
+
 		memcpy(priv->eeprom, eeprom->v2.data, priv->eeprom_slice_size);
 	} else {
 		if (le16_to_cpu(eeprom->v1.len) != priv->eeprom_slice_size)
 			return;
 
+		if (eeprom->v1.data + priv->eeprom_slice_size >
+		    skb_tail_pointer(skb))
+			return;
+
 		memcpy(priv->eeprom, eeprom->v1.data, priv->eeprom_slice_size);
 	}
 
diff --git a/drivers/net/wireless/marvell/libertas/debugfs.c b/drivers/net/wireless/marvell/libertas/debugfs.c
index 9ebd691349400..9428f954837a1 100644
--- a/drivers/net/wireless/marvell/libertas/debugfs.c
+++ b/drivers/net/wireless/marvell/libertas/debugfs.c
@@ -35,8 +35,7 @@ static ssize_t lbs_dev_info(struct file *file, char __user *userbuf,
 {
 	struct lbs_private *priv = file->private_data;
 	size_t pos = 0;
-	unsigned long addr = get_zeroed_page(GFP_KERNEL);
-	char *buf = (char *)addr;
+	char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	ssize_t res;
 	if (!buf)
 		return -ENOMEM;
@@ -48,7 +47,7 @@ static ssize_t lbs_dev_info(struct file *file, char __user *userbuf,
 
 	res = simple_read_from_buffer(userbuf, count, ppos, buf, pos);
 
-	free_page(addr);
+	kfree(buf);
 	return res;
 }
 
@@ -96,8 +95,7 @@ static ssize_t lbs_sleepparams_read(struct file *file, char __user *userbuf,
 	ssize_t ret;
 	size_t pos = 0;
 	struct sleep_params sp;
-	unsigned long addr = get_zeroed_page(GFP_KERNEL);
-	char *buf = (char *)addr;
+	char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 
@@ -113,7 +111,7 @@ static ssize_t lbs_sleepparams_read(struct file *file, char __user *userbuf,
 	ret = simple_read_from_buffer(userbuf, count, ppos, buf, pos);
 
 out_unlock:
-	free_page(addr);
+	kfree(buf);
 	return ret;
 }
 
@@ -165,8 +163,7 @@ static ssize_t lbs_host_sleep_read(struct file *file, char __user *userbuf,
 	struct lbs_private *priv = file->private_data;
 	ssize_t ret;
 	size_t pos = 0;
-	unsigned long addr = get_zeroed_page(GFP_KERNEL);
-	char *buf = (char *)addr;
+	char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 
@@ -174,7 +171,7 @@ static ssize_t lbs_host_sleep_read(struct file *file, char __user *userbuf,
 
 	ret = simple_read_from_buffer(userbuf, count, ppos, buf, pos);
 
-	free_page(addr);
+	kfree(buf);
 	return ret;
 }
 
@@ -228,7 +225,7 @@ static ssize_t lbs_threshold_read(uint16_t tlv_type, uint16_t event_mask,
 	u8 freq;
 	int events = 0;
 
-	buf = (char *)get_zeroed_page(GFP_KERNEL);
+	buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 
@@ -261,7 +258,7 @@ static ssize_t lbs_threshold_read(uint16_t tlv_type, uint16_t event_mask,
 	kfree(subscribed);
 
  out_page:
-	free_page((unsigned long)buf);
+	kfree(buf);
 	return ret;
 }
 
@@ -436,8 +433,7 @@ static ssize_t lbs_rdmac_read(struct file *file, char __user *userbuf,
 	struct lbs_private *priv = file->private_data;
 	ssize_t pos = 0;
 	int ret;
-	unsigned long addr = get_zeroed_page(GFP_KERNEL);
-	char *buf = (char *)addr;
+	char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	u32 val = 0;
 
 	if (!buf)
@@ -450,7 +446,7 @@ static ssize_t lbs_rdmac_read(struct file *file, char __user *userbuf,
 				priv->mac_offset, val);
 		ret = simple_read_from_buffer(userbuf, count, ppos, buf, pos);
 	}
-	free_page(addr);
+	kfree(buf);
 	return ret;
 }
 
@@ -506,8 +502,7 @@ static ssize_t lbs_rdbbp_read(struct file *file, char __user *userbuf,
 	struct lbs_private *priv = file->private_data;
 	ssize_t pos = 0;
 	int ret;
-	unsigned long addr = get_zeroed_page(GFP_KERNEL);
-	char *buf = (char *)addr;
+	char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	u32 val;
 
 	if (!buf)
@@ -520,7 +515,7 @@ static ssize_t lbs_rdbbp_read(struct file *file, char __user *userbuf,
 				priv->bbp_offset, val);
 		ret = simple_read_from_buffer(userbuf, count, ppos, buf, pos);
 	}
-	free_page(addr);
+	kfree(buf);
 
 	return ret;
 }
@@ -578,8 +573,7 @@ static ssize_t lbs_rdrf_read(struct file *file, char __user *userbuf,
 	struct lbs_private *priv = file->private_data;
 	ssize_t pos = 0;
 	int ret;
-	unsigned long addr = get_zeroed_page(GFP_KERNEL);
-	char *buf = (char *)addr;
+	char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	u32 val;
 
 	if (!buf)
@@ -592,7 +586,7 @@ static ssize_t lbs_rdrf_read(struct file *file, char __user *userbuf,
 				priv->rf_offset, val);
 		ret = simple_read_from_buffer(userbuf, count, ppos, buf, pos);
 	}
-	free_page(addr);
+	kfree(buf);
 
 	return ret;
 }
@@ -812,8 +806,7 @@ static ssize_t lbs_debugfs_read(struct file *file, char __user *userbuf,
 	char *p;
 	int i;
 	struct debug_data *d;
-	unsigned long addr = get_zeroed_page(GFP_KERNEL);
-	char *buf = (char *)addr;
+	char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 
@@ -836,7 +829,7 @@ static ssize_t lbs_debugfs_read(struct file *file, char __user *userbuf,
 
 	res = simple_read_from_buffer(userbuf, count, ppos, p, pos);
 
-	free_page(addr);
+	kfree(buf);
 	return res;
 }
 
diff --git a/drivers/net/wireless/marvell/libertas/firmware.c b/drivers/net/wireless/marvell/libertas/firmware.c
index f124110944b7e..9bf7d4c207b9e 100644
--- a/drivers/net/wireless/marvell/libertas/firmware.c
+++ b/drivers/net/wireless/marvell/libertas/firmware.c
@@ -78,6 +78,7 @@ static void helper_firmware_cb(const struct firmware *firmware, void *context)
 	} else {
 		/* No main firmware needed for this helper --> success! */
 		lbs_fw_loaded(priv, 0, firmware, NULL);
+		release_firmware(firmware);
 	}
 }
 
diff --git a/drivers/net/wireless/marvell/libertas/tx.c b/drivers/net/wireless/marvell/libertas/tx.c
index 27304a98787d6..13d08022e4141 100644
--- a/drivers/net/wireless/marvell/libertas/tx.c
+++ b/drivers/net/wireless/marvell/libertas/tx.c
@@ -117,6 +117,13 @@ netdev_tx_t lbs_hard_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	if (priv->wdev->iftype == NL80211_IFTYPE_MONITOR) {
 		struct tx_radiotap_hdr *rtap_hdr = (void *)skb->data;
 
+		if (skb->len < sizeof(*rtap_hdr) + 4 + ETH_ALEN) {
+			lbs_deb_tx("tx err: short monitor frame %u\n", skb->len);
+			dev->stats.tx_dropped++;
+			dev->stats.tx_errors++;
+			goto free;
+		}
+
 		/* set txpd fields from the radiotap header */
 		txpd->tx_control = cpu_to_le32(convert_radiotap_rate_to_mv(rtap_hdr->rate));
 
diff --git a/drivers/net/wireless/marvell/libertas_tf/main.c b/drivers/net/wireless/marvell/libertas_tf/main.c
index fb20fe31cd363..42be6fa22f9c6 100644
--- a/drivers/net/wireless/marvell/libertas_tf/main.c
+++ b/drivers/net/wireless/marvell/libertas_tf/main.c
@@ -174,7 +174,7 @@ static void lbtf_free_adapter(struct lbtf_private *priv)
 {
 	lbtf_deb_enter(LBTF_DEB_MAIN);
 	lbtf_free_cmd_buffer(priv);
-	timer_delete(&priv->command_timer);
+	timer_delete_sync(&priv->command_timer);
 	lbtf_deb_leave(LBTF_DEB_MAIN);
 }
 
diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
index c9daf893472fe..8ec2d22a8c33a 100644
--- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c
+++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
@@ -4334,7 +4334,7 @@ mwifiex_cfg80211_authenticate(struct wiphy *wiphy,
 		return -EOPNOTSUPP;
 	}
 
-	if (!priv->auth_flag) {
+	if (!(priv->auth_flag & HOST_MLME_AUTH_PENDING)) {
 		ret = mwifiex_remain_on_chan_cfg(priv, HostCmd_ACT_GEN_SET,
 						 req->bss->channel,
 						 AUTH_TX_DEFAULT_WAIT_TIME);
@@ -4558,9 +4558,9 @@ mwifiex_cfg80211_disassociate(struct wiphy *wiphy,
 }
 
 static int
-mwifiex_cfg80211_probe_client(struct wiphy *wiphy,
-			      struct net_device *dev, const u8 *peer,
-			      u64 *cookie)
+mwifiex_cfg80211_probe_peer(struct wiphy *wiphy,
+			    struct net_device *dev, const u8 *peer,
+			    u64 *cookie)
 {
 	/* hostapd looks for NL80211_CMD_PROBE_CLIENT support; otherwise,
 	 * it requires monitor-mode support (which mwifiex doesn't support).
@@ -4726,7 +4726,7 @@ int mwifiex_register_cfg80211(struct mwifiex_adapter *adapter)
 		ops->disassoc = mwifiex_cfg80211_disassociate;
 		ops->disconnect = NULL;
 		ops->connect = NULL;
-		ops->probe_client = mwifiex_cfg80211_probe_client;
+		ops->probe_peer = mwifiex_cfg80211_probe_peer;
 	}
 	wiphy->max_scan_ssids = MWIFIEX_MAX_SSID_LIST_LENGTH;
 	wiphy->max_scan_ie_len = MWIFIEX_MAX_VSIE_LEN;
diff --git a/drivers/net/wireless/marvell/mwifiex/debugfs.c b/drivers/net/wireless/marvell/mwifiex/debugfs.c
index 9deaf59dcb625..573768b6ad912 100644
--- a/drivers/net/wireless/marvell/mwifiex/debugfs.c
+++ b/drivers/net/wireless/marvell/mwifiex/debugfs.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/debugfs.h>
+#include <linux/slab.h>
 
 #include "main.h"
 #include "11n.h"
@@ -67,8 +68,8 @@ mwifiex_info_read(struct file *file, char __user *ubuf,
 	struct net_device *netdev = priv->netdev;
 	struct netdev_hw_addr *ha;
 	struct netdev_queue *txq;
-	unsigned long page = get_zeroed_page(GFP_KERNEL);
-	char *p = (char *) page, fmt[64];
+	char *page = kzalloc(PAGE_SIZE, GFP_KERNEL);
+	char *p = page, fmt[64];
 	struct mwifiex_bss_info info;
 	ssize_t ret;
 	int i = 0;
@@ -133,11 +134,10 @@ mwifiex_info_read(struct file *file, char __user *ubuf,
 	}
 	p += sprintf(p, "\n");
 
-	ret = simple_read_from_buffer(ubuf, count, ppos, (char *) page,
-				      (unsigned long) p - page);
+	ret = simple_read_from_buffer(ubuf, count, ppos, page, p - page);
 
 free_and_exit:
-	free_page(page);
+	kfree(page);
 	return ret;
 }
 
@@ -168,8 +168,8 @@ mwifiex_getlog_read(struct file *file, char __user *ubuf,
 {
 	struct mwifiex_private *priv =
 		(struct mwifiex_private *) file->private_data;
-	unsigned long page = get_zeroed_page(GFP_KERNEL);
-	char *p = (char *) page;
+	char *page = kzalloc(PAGE_SIZE, GFP_KERNEL);
+	char *p = page;
 	ssize_t ret;
 	struct mwifiex_ds_get_stats stats;
 
@@ -220,11 +220,10 @@ mwifiex_getlog_read(struct file *file, char __user *ubuf,
 		     stats.bcn_miss_cnt);
 
 
-	ret = simple_read_from_buffer(ubuf, count, ppos, (char *) page,
-				      (unsigned long) p - page);
+	ret = simple_read_from_buffer(ubuf, count, ppos, page, p - page);
 
 free_and_exit:
-	free_page(page);
+	kfree(page);
 	return ret;
 }
 
@@ -247,8 +246,8 @@ mwifiex_histogram_read(struct file *file, char __user *ubuf,
 	ssize_t ret;
 	struct mwifiex_histogram_data *phist_data;
 	int i, value;
-	unsigned long page = get_zeroed_page(GFP_KERNEL);
-	char *p = (char *)page;
+	char *page = kzalloc(PAGE_SIZE, GFP_KERNEL);
+	char *p = page;
 
 	if (!p)
 		return -ENOMEM;
@@ -309,11 +308,10 @@ mwifiex_histogram_read(struct file *file, char __user *ubuf,
 				i, value);
 	}
 
-	ret = simple_read_from_buffer(ubuf, count, ppos, (char *)page,
-				      (unsigned long)p - page);
+	ret = simple_read_from_buffer(ubuf, count, ppos, page, p - page);
 
 free_and_exit:
-	free_page(page);
+	kfree(page);
 	return ret;
 }
 
@@ -383,8 +381,8 @@ mwifiex_debug_read(struct file *file, char __user *ubuf,
 {
 	struct mwifiex_private *priv =
 		(struct mwifiex_private *) file->private_data;
-	unsigned long page = get_zeroed_page(GFP_KERNEL);
-	char *p = (char *) page;
+	char *page = kzalloc(PAGE_SIZE, GFP_KERNEL);
+	char *p = page;
 	ssize_t ret;
 
 	if (!p)
@@ -396,11 +394,10 @@ mwifiex_debug_read(struct file *file, char __user *ubuf,
 
 	p += mwifiex_debug_info_to_buffer(priv, p, &info);
 
-	ret = simple_read_from_buffer(ubuf, count, ppos, (char *) page,
-				      (unsigned long) p - page);
+	ret = simple_read_from_buffer(ubuf, count, ppos, page, p - page);
 
 free_and_exit:
-	free_page(page);
+	kfree(page);
 	return ret;
 }
 
@@ -457,8 +454,7 @@ mwifiex_regrdwr_read(struct file *file, char __user *ubuf,
 {
 	struct mwifiex_private *priv =
 		(struct mwifiex_private *) file->private_data;
-	unsigned long addr = get_zeroed_page(GFP_KERNEL);
-	char *buf = (char *) addr;
+	char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	int pos = 0, ret = 0;
 	u32 reg_value;
 
@@ -497,7 +493,7 @@ mwifiex_regrdwr_read(struct file *file, char __user *ubuf,
 	ret = simple_read_from_buffer(ubuf, count, ppos, buf, pos);
 
 done:
-	free_page(addr);
+	kfree(buf);
 	return ret;
 }
 
@@ -511,8 +507,7 @@ mwifiex_debug_mask_read(struct file *file, char __user *ubuf,
 {
 	struct mwifiex_private *priv =
 		(struct mwifiex_private *)file->private_data;
-	unsigned long page = get_zeroed_page(GFP_KERNEL);
-	char *buf = (char *)page;
+	char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	size_t ret = 0;
 	int pos = 0;
 
@@ -523,7 +518,7 @@ mwifiex_debug_mask_read(struct file *file, char __user *ubuf,
 			priv->adapter->debug_mask);
 	ret = simple_read_from_buffer(ubuf, count, ppos, buf, pos);
 
-	free_page(page);
+	kfree(buf);
 	return ret;
 }
 
@@ -652,8 +647,7 @@ mwifiex_memrw_read(struct file *file, char __user *ubuf,
 		   size_t count, loff_t *ppos)
 {
 	struct mwifiex_private *priv = (void *)file->private_data;
-	unsigned long addr = get_zeroed_page(GFP_KERNEL);
-	char *buf = (char *)addr;
+	char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	int ret, pos = 0;
 
 	if (!buf)
@@ -663,7 +657,7 @@ mwifiex_memrw_read(struct file *file, char __user *ubuf,
 			priv->mem_rw.value);
 	ret = simple_read_from_buffer(ubuf, count, ppos, buf, pos);
 
-	free_page(addr);
+	kfree(buf);
 	return ret;
 }
 
@@ -719,8 +713,7 @@ mwifiex_rdeeprom_read(struct file *file, char __user *ubuf,
 {
 	struct mwifiex_private *priv =
 		(struct mwifiex_private *) file->private_data;
-	unsigned long addr = get_zeroed_page(GFP_KERNEL);
-	char *buf = (char *) addr;
+	char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	int pos, ret, i;
 	u8 value[MAX_EEPROM_DATA];
 
@@ -749,7 +742,7 @@ mwifiex_rdeeprom_read(struct file *file, char __user *ubuf,
 done:
 	ret = simple_read_from_buffer(ubuf, count, ppos, buf, pos);
 out_free:
-	free_page(addr);
+	kfree(buf);
 	return ret;
 }
 
@@ -820,8 +813,7 @@ mwifiex_hscfg_read(struct file *file, char __user *ubuf,
 		   size_t count, loff_t *ppos)
 {
 	struct mwifiex_private *priv = (void *)file->private_data;
-	unsigned long addr = get_zeroed_page(GFP_KERNEL);
-	char *buf = (char *)addr;
+	char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	int pos, ret;
 	struct mwifiex_ds_hs_cfg hscfg;
 
@@ -836,7 +828,7 @@ mwifiex_hscfg_read(struct file *file, char __user *ubuf,
 
 	ret = simple_read_from_buffer(ubuf, count, ppos, buf, pos);
 
-	free_page(addr);
+	kfree(buf);
 	return ret;
 }
 
diff --git a/drivers/net/wireless/marvell/mwifiex/join.c b/drivers/net/wireless/marvell/mwifiex/join.c
index 5a1a0287c1d58..b48f7febaf03f 100644
--- a/drivers/net/wireless/marvell/mwifiex/join.c
+++ b/drivers/net/wireless/marvell/mwifiex/join.c
@@ -736,7 +736,6 @@ int mwifiex_ret_802_11_associate(struct mwifiex_private *priv,
 	/* Send a Media Connected event, according to the Spec */
 	priv->media_connected = true;
 
-	priv->adapter->ps_state = PS_STATE_AWAKE;
 	priv->adapter->pps_uapsd_mode = false;
 	priv->adapter->tx_lock_flag = false;
 
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2400pci.c b/drivers/net/wireless/ralink/rt2x00/rt2400pci.c
index cac191304bf5f..b846fe589c2ba 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2400pci.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2400pci.c
@@ -1429,7 +1429,7 @@ static irqreturn_t rt2400pci_interrupt(int irq, void *dev_instance)
  */
 static int rt2400pci_validate_eeprom(struct rt2x00_dev *rt2x00dev)
 {
-	struct eeprom_93cx6 eeprom;
+	struct eeprom_93cx6 eeprom = {};
 	u32 reg;
 	u16 word;
 	u8 *mac;
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2500pci.c b/drivers/net/wireless/ralink/rt2x00/rt2500pci.c
index fc35b60e422c0..be9df35acc33b 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2500pci.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2500pci.c
@@ -1555,7 +1555,7 @@ static irqreturn_t rt2500pci_interrupt(int irq, void *dev_instance)
  */
 static int rt2500pci_validate_eeprom(struct rt2x00_dev *rt2x00dev)
 {
-	struct eeprom_93cx6 eeprom;
+	struct eeprom_93cx6 eeprom = {};
 	u32 reg;
 	u16 word;
 	u8 *mac;
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800pci.c b/drivers/net/wireless/ralink/rt2x00/rt2800pci.c
index 4fa14bb573add..2596b9fcc7dd3 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800pci.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800pci.c
@@ -108,7 +108,7 @@ static void rt2800pci_eepromregister_write(struct eeprom_93cx6 *eeprom)
 
 static int rt2800pci_read_eeprom_pci(struct rt2x00_dev *rt2x00dev)
 {
-	struct eeprom_93cx6 eeprom;
+	struct eeprom_93cx6 eeprom = {};
 	u32 reg;
 
 	reg = rt2x00mmio_register_read(rt2x00dev, E2PROM_CSR);
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c b/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c
index 82fb230a73bb7..4d94b7062f440 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c
@@ -1388,7 +1388,7 @@ int rt2x00lib_probe_dev(struct rt2x00_dev *rt2x00dev)
 			                      GFP_KERNEL);
 		if (!rt2x00dev->drv_data) {
 			retval = -ENOMEM;
-			goto exit;
+			return retval;
 		}
 	}
 
@@ -1422,7 +1422,7 @@ int rt2x00lib_probe_dev(struct rt2x00_dev *rt2x00dev)
 	    alloc_ordered_workqueue("%s", 0, wiphy_name(rt2x00dev->hw->wiphy));
 	if (!rt2x00dev->workqueue) {
 		retval = -ENOMEM;
-		goto exit;
+		goto exit_free_drv_data;
 	}
 
 	INIT_WORK(&rt2x00dev->intf_work, rt2x00lib_intf_scheduled);
@@ -1494,6 +1494,14 @@ int rt2x00lib_probe_dev(struct rt2x00_dev *rt2x00dev)
 exit:
 	rt2x00lib_remove_dev(rt2x00dev);
 
+	return retval;
+
+exit_free_drv_data:
+	clear_bit(DEVICE_STATE_PRESENT, &rt2x00dev->flags);
+
+	kfree(rt2x00dev->drv_data);
+	rt2x00dev->drv_data = NULL;
+
 	return retval;
 }
 EXPORT_SYMBOL_GPL(rt2x00lib_probe_dev);
diff --git a/drivers/net/wireless/ralink/rt2x00/rt61pci.c b/drivers/net/wireless/ralink/rt2x00/rt61pci.c
index 79e1fd0a1fbdd..d4783658b2c5d 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt61pci.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt61pci.c
@@ -2298,7 +2298,7 @@ static irqreturn_t rt61pci_interrupt(int irq, void *dev_instance)
  */
 static int rt61pci_validate_eeprom(struct rt2x00_dev *rt2x00dev)
 {
-	struct eeprom_93cx6 eeprom;
+	struct eeprom_93cx6 eeprom = {};
 	u32 reg;
 	u16 word;
 	u8 *mac;
diff --git a/drivers/net/wireless/realtek/rtw89/core.c b/drivers/net/wireless/realtek/rtw89/core.c
index 68dad6090f87e..0f0e46cb42605 100644
--- a/drivers/net/wireless/realtek/rtw89/core.c
+++ b/drivers/net/wireless/realtek/rtw89/core.c
@@ -7432,9 +7432,6 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev)
 	if (!chip->support_rnr)
 		hw->wiphy->flags |= WIPHY_FLAG_SPLIT_SCAN_6GHZ;
 
-	if (chip->chip_gen == RTW89_CHIP_BE)
-		hw->wiphy->flags |= WIPHY_FLAG_DISABLE_WEXT;
-
 	if (rtwdev->support_mlo) {
 		hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_MLO;
 		hw->wiphy->iftype_ext_capab = rtw89_iftypes_ext_capa;
diff --git a/drivers/net/wireless/rsi/rsi_91x_hal.c b/drivers/net/wireless/rsi/rsi_91x_hal.c
index a0c36144eb0b2..501071104a9fe 100644
--- a/drivers/net/wireless/rsi/rsi_91x_hal.c
+++ b/drivers/net/wireless/rsi/rsi_91x_hal.c
@@ -431,6 +431,7 @@ int rsi_prepare_beacon(struct rsi_common *common, struct sk_buff *skb)
 	struct ieee80211_vif *vif;
 	struct sk_buff *mac_bcn;
 	u8 vap_id = 0, i;
+	unsigned int tailroom;
 	u16 tim_offset = 0;
 
 	for (i = 0; i < RSI_MAX_VIFS; i++) {
@@ -480,6 +481,13 @@ int rsi_prepare_beacon(struct rsi_common *common, struct sk_buff *skb)
 	if (mac_bcn->data[tim_offset + 2] == 0)
 		bcn_frm->frame_info |= cpu_to_le16(RSI_DATA_DESC_DTIM_BEACON);
 
+	tailroom = skb_tailroom(skb);
+	if (tailroom < FRAME_DESC_SZ ||
+	    mac_bcn->len > tailroom - FRAME_DESC_SZ) {
+		dev_kfree_skb(mac_bcn);
+		return -EMSGSIZE;
+	}
+
 	memcpy(&skb->data[FRAME_DESC_SZ], mac_bcn->data, mac_bcn->len);
 	skb_put(skb, mac_bcn->len + FRAME_DESC_SZ);
 
diff --git a/drivers/net/wireless/rsi/rsi_91x_mgmt.c b/drivers/net/wireless/rsi/rsi_91x_mgmt.c
index 7f2c1608f2ce3..bb167f03367bf 100644
--- a/drivers/net/wireless/rsi/rsi_91x_mgmt.c
+++ b/drivers/net/wireless/rsi/rsi_91x_mgmt.c
@@ -848,8 +848,10 @@ int rsi_hal_load_key(struct rsi_common *common,
 		} else {
 			memcpy(&set_key->key[0][0], data, key_len);
 		}
-		memcpy(set_key->tx_mic_key, &data[16], 8);
-		memcpy(set_key->rx_mic_key, &data[24], 8);
+		if (cipher == WLAN_CIPHER_SUITE_TKIP) {
+			memcpy(set_key->tx_mic_key, &data[16], 8);
+			memcpy(set_key->rx_mic_key, &data[24], 8);
+		}
 	} else {
 		memset(&set_key[FRAME_DESC_SZ], 0, frame_len - FRAME_DESC_SZ);
 	}
@@ -1911,6 +1913,12 @@ int rsi_send_bgscan_probe_req(struct rsi_common *common,
 		return -ENOMEM;
 	}
 
+	if (probereq_skb->len > MAX_BGSCAN_PROBE_REQ_LEN) {
+		dev_kfree_skb(probereq_skb);
+		dev_kfree_skb(skb);
+		return -EINVAL;
+	}
+
 	memcpy(&skb->data[frame_len], probereq_skb->data, probereq_skb->len);
 
 	bgscan->probe_req_length = cpu_to_le16(probereq_skb->len);
diff --git a/drivers/net/wireless/ti/wlcore/main.c b/drivers/net/wireless/ti/wlcore/main.c
index be583ae331c03..5595f7a1fc0c3 100644
--- a/drivers/net/wireless/ti/wlcore/main.c
+++ b/drivers/net/wireless/ti/wlcore/main.c
@@ -6354,7 +6354,6 @@ struct ieee80211_hw *wlcore_alloc_hw(size_t priv_size, u32 aggr_buf_size,
 	struct ieee80211_hw *hw;
 	struct wl1271 *wl;
 	int i, j, ret;
-	unsigned int order;
 
 	hw = ieee80211_alloc_hw(sizeof(*wl), &wl1271_ops);
 	if (!hw) {
@@ -6434,8 +6433,7 @@ struct ieee80211_hw *wlcore_alloc_hw(size_t priv_size, u32 aggr_buf_size,
 	mutex_init(&wl->flush_mutex);
 	init_completion(&wl->nvs_loading_complete);
 
-	order = get_order(aggr_buf_size);
-	wl->aggr_buf = (u8 *)__get_free_pages(GFP_KERNEL, order);
+	wl->aggr_buf = kmalloc(round_up(aggr_buf_size, PAGE_SIZE), GFP_KERNEL);
 	if (!wl->aggr_buf) {
 		ret = -ENOMEM;
 		goto err_wq;
@@ -6449,7 +6447,7 @@ struct ieee80211_hw *wlcore_alloc_hw(size_t priv_size, u32 aggr_buf_size,
 	}
 
 	/* Allocate one page for the FW log */
-	wl->fwlog = (u8 *)get_zeroed_page(GFP_KERNEL);
+	wl->fwlog = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!wl->fwlog) {
 		ret = -ENOMEM;
 		goto err_dummy_packet;
@@ -6474,13 +6472,13 @@ struct ieee80211_hw *wlcore_alloc_hw(size_t priv_size, u32 aggr_buf_size,
 	kfree(wl->mbox);
 
 err_fwlog:
-	free_page((unsigned long)wl->fwlog);
+	kfree(wl->fwlog);
 
 err_dummy_packet:
 	dev_kfree_skb(wl->dummy_packet);
 
 err_aggr:
-	free_pages((unsigned long)wl->aggr_buf, order);
+	kfree(wl->aggr_buf);
 
 err_wq:
 	destroy_workqueue(wl->freezable_wq);
@@ -6509,9 +6507,9 @@ int wlcore_free_hw(struct wl1271 *wl)
 
 	kfree(wl->buffer_32);
 	kfree(wl->mbox);
-	free_page((unsigned long)wl->fwlog);
+	kfree(wl->fwlog);
 	dev_kfree_skb(wl->dummy_packet);
-	free_pages((unsigned long)wl->aggr_buf, get_order(wl->aggr_buf_size));
+	kfree(wl->aggr_buf);
 
 	wl1271_debugfs_exit(wl);
 
diff --git a/drivers/net/wireless/virtual/mac80211_hwsim_main.c b/drivers/net/wireless/virtual/mac80211_hwsim_main.c
index 0dd8a6c859534..e1094b48c315e 100644
--- a/drivers/net/wireless/virtual/mac80211_hwsim_main.c
+++ b/drivers/net/wireless/virtual/mac80211_hwsim_main.c
@@ -1324,6 +1324,17 @@ static void mac80211_hwsim_set_tsf(struct ieee80211_hw *hw,
 	}
 }
 
+static struct ieee80211_rate *
+mac80211_hwsim_get_tx_rate(struct ieee80211_hw *hw,
+			   struct ieee80211_tx_info *info)
+{
+	if (info->control.rates[0].flags &
+	    (IEEE80211_TX_RC_MCS | IEEE80211_TX_RC_VHT_MCS))
+		return NULL;
+
+	return ieee80211_get_tx_rate(hw, info);
+}
+
 static void mac80211_hwsim_monitor_rx(struct ieee80211_hw *hw,
 				      struct sk_buff *tx_skb,
 				      struct ieee80211_channel *chan)
@@ -1333,7 +1344,7 @@ static void mac80211_hwsim_monitor_rx(struct ieee80211_hw *hw,
 	struct hwsim_radiotap_hdr *hdr;
 	u16 flags, bitrate;
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx_skb);
-	struct ieee80211_rate *txrate = ieee80211_get_tx_rate(hw, info);
+	struct ieee80211_rate *txrate = mac80211_hwsim_get_tx_rate(hw, info);
 
 	if (!txrate)
 		bitrate = 0;
@@ -1603,7 +1614,7 @@ static void mac80211_hwsim_write_tsf(struct mac80211_hwsim_data *data,
 
 	spin_lock_bh(&data->tsf_offset_lock);
 
-	txrate = ieee80211_get_tx_rate(data->hw, info);
+	txrate = mac80211_hwsim_get_tx_rate(data->hw, info);
 	if (txrate)
 		bitrate = txrate->bitrate;
 
@@ -2303,6 +2314,7 @@ static int mac80211_hwsim_start(struct ieee80211_hw *hw)
 static void mac80211_hwsim_stop(struct ieee80211_hw *hw, bool suspend)
 {
 	struct mac80211_hwsim_data *data = hw->priv;
+	struct sk_buff *skb;
 	int i;
 
 	data->started = false;
@@ -2310,8 +2322,8 @@ static void mac80211_hwsim_stop(struct ieee80211_hw *hw, bool suspend)
 	for (i = 0; i < ARRAY_SIZE(data->link_data); i++)
 		hrtimer_cancel(&data->link_data[i].beacon_timer);
 
-	while (!skb_queue_empty(&data->pending))
-		ieee80211_free_txskb(hw, skb_dequeue(&data->pending));
+	while ((skb = skb_dequeue(&data->pending)))
+		ieee80211_free_txskb(hw, skb);
 
 	wiphy_dbg(hw->wiphy, "%s\n", __func__);
 }
@@ -6274,6 +6286,8 @@ static void mac80211_hwsim_free(void)
 						struct mac80211_hwsim_data,
 						list))) {
 		list_del(&data->list);
+		rhashtable_remove_fast(&hwsim_radios_rht, &data->rht,
+				       hwsim_rht_params);
 		spin_unlock_bh(&hwsim_radio_lock);
 		mac80211_hwsim_del_radio(data, wiphy_name(data->hw->wiphy),
 					 NULL);
@@ -7289,6 +7303,7 @@ static void hwsim_virtio_rx_work(struct work_struct *work)
 
 	skb->data = skb->head;
 	skb_reset_tail_pointer(skb);
+	len = min(len, skb_end_offset(skb));
 	skb_put(skb, len);
 	hwsim_virtio_handle_cmd(skb);
 
diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c
index 7e1112960d7f8..380e352dc3ccd 100644
--- a/drivers/nvdimm/btt.c
+++ b/drivers/nvdimm/btt.c
@@ -883,6 +883,14 @@ static int discover_arenas(struct btt *btt)
 		arena->external_lba_start = cur_nlba;
 		parse_arena_meta(arena, super, cur_off);
 
+		if (arena->nfree < btt->nd_region->num_lanes) {
+			dev_err(to_dev(arena),
+				"nfree %u smaller than lane count %d\n",
+				arena->nfree, btt->nd_region->num_lanes);
+			ret = -ENODEV;
+			goto out;
+		}
+
 		ret = log_set_indices(arena);
 		if (ret) {
 			dev_err(to_dev(arena),
diff --git a/drivers/nvdimm/label.c b/drivers/nvdimm/label.c
index 4218e3ac4a2ab..dea2eee86d132 100644
--- a/drivers/nvdimm/label.c
+++ b/drivers/nvdimm/label.c
@@ -145,10 +145,21 @@ static int __nd_label_validate(struct nvdimm_drvdata *ndd)
 		/* label sizes larger than 128 arrived with v1.2 */
 		version = __le16_to_cpu(nsindex[i]->major) * 100
 			+ __le16_to_cpu(nsindex[i]->minor);
-		if (version >= 102)
+		if (version >= 102) {
+			/*
+			 * labelsize feeds the shift below; only 0 (128-byte)
+			 * and 1 (256-byte) are valid -- a larger value would
+			 * overflow or exceed the width of int.
+			 */
+			if (nsindex[i]->labelsize > 1) {
+				dev_dbg(dev, "nsindex%d labelsize: %d invalid\n",
+					i, nsindex[i]->labelsize);
+				continue;
+			}
 			labelsize = 1 << (7 + nsindex[i]->labelsize);
-		else
+		} else {
 			labelsize = 128;
+		}
 
 		if (labelsize != sizeof_namespace_label(ndd)) {
 			dev_dbg(dev, "nsindex%d labelsize %d invalid\n",
@@ -202,7 +213,7 @@ static int __nd_label_validate(struct nvdimm_drvdata *ndd)
 		}
 
 		nslot = __le32_to_cpu(nsindex[i]->nslot);
-		if (nslot * sizeof_namespace_label(ndd)
+		if ((u64)nslot * sizeof_namespace_label(ndd)
 				+ 2 * sizeof_namespace_index(ndd)
 				> ndd->nsarea.config_size) {
 			dev_dbg(dev, "nsindex%d nslot: %u invalid, config_size: %#x\n",
diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index 6909e35427942..56cd228af1d57 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -394,8 +394,10 @@ nvme_rdma_find_get_device(struct rdma_cm_id *cm_id)
 		goto out_free_pd;
 	}
 
-	ndev->num_inline_segments = min(NVME_RDMA_MAX_INLINE_SEGMENTS,
-					ndev->dev->attrs.max_send_sge - 1);
+	ndev->num_inline_segments = ndev->dev->attrs.max_send_sge;
+	if (ndev->num_inline_segments)
+		ndev->num_inline_segments--;
+	ndev->num_inline_segments = min(ndev->num_inline_segments, NVME_RDMA_MAX_INLINE_SEGMENTS);
 	list_add(&ndev->entry, &device_list);
 out_unlock:
 	mutex_unlock(&device_list_mutex);
@@ -1847,7 +1849,7 @@ static int nvme_rdma_route_resolved(struct nvme_rdma_queue *queue)
 	param.qp_num = queue->qp->qp_num;
 	param.flow_control = 1;
 
-	param.responder_resources = queue->device->dev->attrs.max_qp_rd_atom;
+	param.responder_resources = min(queue->device->dev->attrs.max_qp_rd_atom, U8_MAX);
 	/* maximum retry count */
 	param.retry_count = 7;
 	param.rnr_retry_count = 7;
diff --git a/drivers/nvme/target/pci-epf.c b/drivers/nvme/target/pci-epf.c
index 4e9db96ebfecd..b1ba2d0bea6d9 100644
--- a/drivers/nvme/target/pci-epf.c
+++ b/drivers/nvme/target/pci-epf.c
@@ -210,9 +210,7 @@ struct nvmet_pci_epf {
 
 	bool				dma_enabled;
 	struct dma_chan			*dma_tx_chan;
-	struct mutex			dma_tx_lock;
 	struct dma_chan			*dma_rx_chan;
-	struct mutex			dma_rx_lock;
 
 	struct mutex			mmio_lock;
 
@@ -295,9 +293,6 @@ static void nvmet_pci_epf_init_dma(struct nvmet_pci_epf *nvme_epf)
 	struct dma_chan *chan;
 	dma_cap_mask_t mask;
 
-	mutex_init(&nvme_epf->dma_rx_lock);
-	mutex_init(&nvme_epf->dma_tx_lock);
-
 	dma_cap_zero(mask);
 	dma_cap_set(DMA_SLAVE, mask);
 
@@ -336,8 +331,6 @@ static void nvmet_pci_epf_init_dma(struct nvmet_pci_epf *nvme_epf)
 	nvme_epf->dma_rx_chan = NULL;
 
 out_dma_no_rx:
-	mutex_destroy(&nvme_epf->dma_rx_lock);
-	mutex_destroy(&nvme_epf->dma_tx_lock);
 	nvme_epf->dma_enabled = false;
 
 	dev_info(&epf->dev, "DMA not supported, falling back to MMIO\n");
@@ -352,8 +345,6 @@ static void nvmet_pci_epf_deinit_dma(struct nvmet_pci_epf *nvme_epf)
 	nvme_epf->dma_tx_chan = NULL;
 	dma_release_channel(nvme_epf->dma_rx_chan);
 	nvme_epf->dma_rx_chan = NULL;
-	mutex_destroy(&nvme_epf->dma_rx_lock);
-	mutex_destroy(&nvme_epf->dma_tx_lock);
 	nvme_epf->dma_enabled = false;
 }
 
@@ -368,18 +359,15 @@ static int nvmet_pci_epf_dma_transfer(struct nvmet_pci_epf *nvme_epf,
 	struct dma_chan *chan;
 	dma_cookie_t cookie;
 	dma_addr_t dma_addr;
-	struct mutex *lock;
 	int ret;
 
 	switch (dir) {
 	case DMA_FROM_DEVICE:
-		lock = &nvme_epf->dma_rx_lock;
 		chan = nvme_epf->dma_rx_chan;
 		sconf.direction = DMA_DEV_TO_MEM;
 		sconf.src_addr = seg->pci_addr;
 		break;
 	case DMA_TO_DEVICE:
-		lock = &nvme_epf->dma_tx_lock;
 		chan = nvme_epf->dma_tx_chan;
 		sconf.direction = DMA_MEM_TO_DEV;
 		sconf.dst_addr = seg->pci_addr;
@@ -388,22 +376,15 @@ static int nvmet_pci_epf_dma_transfer(struct nvmet_pci_epf *nvme_epf,
 		return -EINVAL;
 	}
 
-	mutex_lock(lock);
-
 	dma_dev = dmaengine_get_dma_device(chan);
 	dma_addr = dma_map_single(dma_dev, seg->buf, seg->length, dir);
 	ret = dma_mapping_error(dma_dev, dma_addr);
 	if (ret)
-		goto unlock;
-
-	ret = dmaengine_slave_config(chan, &sconf);
-	if (ret) {
-		dev_err(dev, "Failed to configure DMA channel\n");
-		goto unmap;
-	}
+		return ret;
 
-	desc = dmaengine_prep_slave_single(chan, dma_addr, seg->length,
-					   sconf.direction, DMA_CTRL_ACK);
+	desc = dmaengine_prep_config_single_safe(chan, dma_addr, seg->length,
+						 sconf.direction,
+						 DMA_CTRL_ACK, &sconf);
 	if (!desc) {
 		dev_err(dev, "Failed to prepare DMA\n");
 		ret = -EIO;
@@ -420,16 +401,12 @@ static int nvmet_pci_epf_dma_transfer(struct nvmet_pci_epf *nvme_epf,
 	if (dma_sync_wait(chan, cookie) != DMA_COMPLETE) {
 		dev_err(dev, "DMA transfer failed\n");
 		ret = -EIO;
+		dmaengine_terminate_sync(chan);
 	}
 
-	dmaengine_terminate_sync(chan);
-
 unmap:
 	dma_unmap_single(dma_dev, dma_addr, seg->length, dir);
 
-unlock:
-	mutex_unlock(lock);
-
 	return ret;
 }
 
diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c
index ea1185b8267ef..bdd28ce99faac 100644
--- a/drivers/nvme/target/rdma.c
+++ b/drivers/nvme/target/rdma.c
@@ -149,10 +149,10 @@ MODULE_PARM_DESC(use_srq, "Use shared receive queue.");
 static int srq_size_set(const char *val, const struct kernel_param *kp);
 static const struct kernel_param_ops srq_size_ops = {
 	.set = srq_size_set,
-	.get = param_get_int,
+	.get = param_get_uint,
 };
 
-static int nvmet_rdma_srq_size = 1024;
+static unsigned int nvmet_rdma_srq_size = 1024;
 module_param_cb(srq_size, &srq_size_ops, &nvmet_rdma_srq_size, 0644);
 MODULE_PARM_DESC(srq_size, "set Shared Receive Queue (SRQ) size, should >= 256 (default: 1024)");
 
@@ -180,13 +180,14 @@ static const struct nvmet_fabrics_ops nvmet_rdma_ops;
 
 static int srq_size_set(const char *val, const struct kernel_param *kp)
 {
-	int n = 0, ret;
+	unsigned int n;
+	int ret;
 
-	ret = kstrtoint(val, 10, &n);
+	ret = kstrtouint(val, 10, &n);
 	if (ret != 0 || n < 256)
 		return -EINVAL;
 
-	return param_set_int(val, kp);
+	return param_set_uint(val, kp);
 }
 
 static int num_pages(int len)
@@ -1153,8 +1154,8 @@ static int nvmet_rdma_init_srqs(struct nvmet_rdma_device *ndev)
 
 	ndev->srq_size = min(ndev->device->attrs.max_srq_wr,
 			     nvmet_rdma_srq_size);
-	ndev->srq_count = min(ndev->device->num_comp_vectors,
-			      ndev->device->attrs.max_srq);
+	ndev->srq_count = min_t(u32, ndev->device->num_comp_vectors,
+				ndev->device->attrs.max_srq);
 
 	ndev->srqs = kzalloc_objs(*ndev->srqs, ndev->srq_count);
 	if (!ndev->srqs)
@@ -1199,7 +1200,7 @@ nvmet_rdma_find_get_device(struct rdma_cm_id *cm_id)
 	struct nvmet_port *nport = port->nport;
 	struct nvmet_rdma_device *ndev;
 	int inline_page_count;
-	int inline_sge_count;
+	u32 inline_sge_count;
 	int ret;
 
 	mutex_lock(&device_list_mutex);
@@ -1215,7 +1216,9 @@ nvmet_rdma_find_get_device(struct rdma_cm_id *cm_id)
 
 	inline_page_count = num_pages(nport->inline_data_size);
 	inline_sge_count = max(cm_id->device->attrs.max_sge_rd,
-				cm_id->device->attrs.max_recv_sge) - 1;
+				cm_id->device->attrs.max_recv_sge);
+	if (inline_sge_count)
+		inline_sge_count--;
 	if (inline_page_count > inline_sge_count) {
 		pr_warn("inline_data_size %d cannot be supported by device %s. Reducing to %lu.\n",
 			nport->inline_data_size, cm_id->device->name,
@@ -1555,8 +1558,9 @@ static int nvmet_rdma_cm_accept(struct rdma_cm_id *cm_id,
 
 	param.rnr_retry_count = 7;
 	param.flow_control = 1;
-	param.initiator_depth = min_t(u8, p->initiator_depth,
-		queue->dev->device->attrs.max_qp_init_rd_atom);
+	param.initiator_depth = min3(p->initiator_depth,
+				     queue->dev->device->attrs.max_qp_init_rd_atom,
+				     U8_MAX);
 	param.private_data = &priv;
 	param.private_data_len = sizeof(priv);
 	priv.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0);
diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig
index 74ddbd0f79b0e..a452045bcf0cb 100644
--- a/drivers/nvmem/Kconfig
+++ b/drivers/nvmem/Kconfig
@@ -28,6 +28,107 @@ source "drivers/nvmem/layouts/Kconfig"
 
 # Devices
 
+if NVMEM_SYSFS
+
+config EEPROM_AT24
+	tristate "I2C EEPROMs / RAMs / ROMs from most vendors"
+	depends on I2C && SYSFS
+	select REGMAP
+	select REGMAP_I2C
+	help
+	  Enable this driver to get read/write support to most I2C EEPROMs
+	  and compatible devices like FRAMs, SRAMs, ROMs etc. After you
+	  configure the driver to know about each chip on your target
+	  board.  Use these generic chip names, instead of vendor-specific
+	  ones like at24c64, 24lc02 or fm24c04:
+
+	     24c00, 24c01, 24c02, spd (readonly 24c02), 24c04, 24c08,
+	     24c16, 24c32, 24c64, 24c128, 24c256, 24c512, 24c1024, 24c2048
+
+	  Unless you like data loss puzzles, always be sure that any chip
+	  you configure as a 24c32 (32 kbit) or larger is NOT really a
+	  24c16 (16 kbit) or smaller, and vice versa. Marking the chip
+	  as read-only won't help recover from this. Also, if your chip
+	  has any software write-protect mechanism you may want to review the
+	  code to make sure this driver won't turn it on by accident.
+
+	  If you use this with an SMBus adapter instead of an I2C adapter,
+	  full functionality is not available.  Only smaller devices are
+	  supported (24c16 and below, max 4 kByte).
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called at24.
+
+config EEPROM_AT25
+	tristate "SPI EEPROMs (FRAMs) from most vendors"
+	depends on SPI && SYSFS
+	select SPI_MEM
+	help
+	  Enable this driver to get read/write support to most SPI EEPROMs
+	  and Cypress FRAMs,
+	  after you configure the board init code to know about each eeprom
+	  on your target board.
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called at25.
+
+config EEPROM_93XX46
+	tristate "Microwire EEPROM 93XX46 support"
+	depends on SPI && SYSFS
+	select REGMAP
+	help
+	  Driver for the microwire EEPROM chipsets 93xx46x. The driver
+	  supports both read and write commands and also the command to
+	  erase the whole EEPROM.
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called eeprom_93xx46.
+
+	  If unsure, say N.
+
+config EEPROM_EE1004
+	tristate "SPD EEPROMs on DDR4 memory modules"
+	depends on I2C && SYSFS
+	help
+	  Enable this driver to get read support to SPD EEPROMs following
+	  the JEDEC EE1004 standard. These are typically found on DDR4
+	  SDRAM memory modules.
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called ee1004.
+
+config EEPROM_M24LR
+	tristate "STMicroelectronics M24LR RFID/NFC EEPROM support"
+	depends on I2C && SYSFS
+	select REGMAP_I2C
+	help
+	  This enables support for STMicroelectronics M24LR RFID/NFC EEPROM
+	  chips. These dual-interface devices expose two I2C addresses:
+	  one for EEPROM memory access and another for control and system
+	  configuration (e.g. UID, password handling).
+
+	  This driver provides a sysfs interface for control functions and
+	  integrates with the nvmem subsystem for EEPROM access.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called m24lr.
+
+endif # NVMEM_SYSFS
+
+config NVMEM_AIROHA_SMC_EFUSES
+	tristate "Airoha SMC eFuse support"
+	depends on ARM64
+	depends on ARCH_AIROHA || COMPILE_TEST
+	depends on HAVE_ARM_SMCCC
+	default ARCH_AIROHA
+	help
+	  Say y here to enable support for reading eFuses on Airoha AN7581
+	  SoCs. These are e.g. used to store factory programmed
+	  calibration data required for the PCIe or the USB-C PHY or Thermal.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called nvmem-airoha-smc-efuses.
+
 config NVMEM_AN8855_EFUSE
 	tristate "Airoha AN8855 eFuse support"
 	depends on COMPILE_TEST
@@ -138,7 +239,7 @@ config NVMEM_JZ4780_EFUSE
 
 config NVMEM_LAN9662_OTPC
 	tristate "Microchip LAN9662 OTP controller support"
-	depends on SOC_LAN966 || COMPILE_TEST
+	depends on SOC_LAN966 || ARCH_LAN969X || COMPILE_TEST
 	depends on HAS_IOMEM
 	help
 	  This driver enables the OTP controller available on Microchip LAN9662
@@ -227,6 +328,17 @@ config NVMEM_MTK_EFUSE
 	  This driver can also be built as a module. If so, the module
 	  will be called efuse-mtk.
 
+config NVMEM_MT6323_EFUSE
+	tristate "Mediatek MT6323 PMIC EFUSE support"
+	depends on ARCH_MEDIATEK || COMPILE_TEST
+	depends on MFD_MT6397
+	help
+	  This is a driver to access hardware related data like sensor
+	  calibration, etc.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called efuse-mt6323.
+
 config NVMEM_MXS_OCOTP
 	tristate "Freescale MXS On-Chip OTP Memory Support"
 	depends on ARCH_MXS || COMPILE_TEST
@@ -262,7 +374,7 @@ config NVMEM_S32G_OCOTP
 	  Programmable memory pages.
 
 config NVMEM_QCOM_QFPROM
-	tristate "QCOM QFPROM Support"
+	tristate "Qualcomm QFPROM Support"
 	depends on ARCH_QCOM || COMPILE_TEST
 	depends on HAS_IOMEM
 	help
@@ -273,7 +385,7 @@ config NVMEM_QCOM_QFPROM
 	  will be called nvmem_qfprom.
 
 config NVMEM_QCOM_SEC_QFPROM
-        tristate "QCOM SECURE QFPROM Support"
+        tristate "Qualcomm SECURE QFPROM Support"
         depends on ARCH_QCOM || COMPILE_TEST
         depends on HAS_IOMEM
         depends on OF
@@ -303,7 +415,6 @@ config NVMEM_RAVE_SP_EEPROM
 config NVMEM_RCAR_EFUSE
 	tristate "Renesas R-Car Gen4 E-FUSE support"
 	depends on (ARCH_RENESAS && ARM64) || COMPILE_TEST
-	depends on NVMEM
 	help
 	  Enable support for reading the fuses in the E-FUSE or OTP
 	  non-volatile memory block on Renesas R-Car Gen4 SoCs.
@@ -483,4 +594,4 @@ config NVMEM_QORIQ_EFUSE
 	  This driver can also be built as a module. If so, the module
 	  will be called nvmem_qoriq_efuse.
 
-endif
+endif # NVMEM
diff --git a/drivers/nvmem/Makefile b/drivers/nvmem/Makefile
index 7252b8ec88d46..626936e4dfdd8 100644
--- a/drivers/nvmem/Makefile
+++ b/drivers/nvmem/Makefile
@@ -10,6 +10,13 @@ nvmem_layouts-y			:= layouts.o
 obj-y				+= layouts/
 
 # Devices
+obj-$(CONFIG_EEPROM_AT24)		+= at24.o
+obj-$(CONFIG_EEPROM_AT25)		+= at25.o
+obj-$(CONFIG_EEPROM_93XX46)		+= eeprom_93xx46.o
+obj-$(CONFIG_EEPROM_EE1004)		+= ee1004.o
+obj-$(CONFIG_EEPROM_M24LR)		+= m24lr.o
+obj-$(CONFIG_NVMEM_AIROHA_SMC_EFUSES)	+= nvmem-airoha-smc-efuses.o
+nvmem-airoha-smc-efuses-y 		:= airoha-smc-efuses.o
 obj-$(CONFIG_NVMEM_AN8855_EFUSE)	+= nvmem-an8855-efuse.o
 nvmem-an8855-efuse-y 			:= an8855-efuse.o
 obj-$(CONFIG_NVMEM_APPLE_EFUSES)	+= nvmem-apple-efuses.o
@@ -48,6 +55,8 @@ obj-$(CONFIG_NVMEM_MICROCHIP_OTPC)	+= nvmem-microchip-otpc.o
 nvmem-microchip-otpc-y			:= microchip-otpc.o
 obj-$(CONFIG_NVMEM_MTK_EFUSE)		+= nvmem_mtk-efuse.o
 nvmem_mtk-efuse-y			:= mtk-efuse.o
+obj-$(CONFIG_NVMEM_MT6323_EFUSE)		+= nvmem_mt6323-efuse.o
+nvmem_mt6323-efuse-y			:= mt6323-efuse.o
 obj-$(CONFIG_NVMEM_MXS_OCOTP)		+= nvmem-mxs-ocotp.o
 nvmem-mxs-ocotp-y			:= mxs-ocotp.o
 obj-$(CONFIG_NVMEM_NINTENDO_OTP)	+= nvmem-nintendo-otp.o
diff --git a/drivers/nvmem/airoha-smc-efuses.c b/drivers/nvmem/airoha-smc-efuses.c
new file mode 100644
index 0000000000000..e56a99f4aa1f9
--- /dev/null
+++ b/drivers/nvmem/airoha-smc-efuses.c
@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  Author: Christian Marangi <ansuelsmth@gmail.com>
+ */
+
+#include <linux/arm-smccc.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/nvmem-provider.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+#include <linux/regmap.h>
+
+#define AIROHA_SMC_EFUSE_FID		0x82000001
+#define AIROHA_SMC_EFUSE_SUB_ID_READ	0x44414552
+
+#define AIROHA_EFUSE_CELLS		64
+
+struct airoha_efuse_bank_priv {
+	u32 bank_index;
+};
+
+static int airoha_efuse_read(void *context, unsigned int offset,
+			     void *val, size_t bytes)
+{
+	struct regmap *regmap = context;
+
+	return regmap_bulk_read(regmap, offset,
+				val, bytes / sizeof(u32));
+}
+
+static int airoha_efuse_reg_read(void *context, unsigned int offset,
+				 unsigned int *val)
+{
+	struct airoha_efuse_bank_priv *priv = context;
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(AIROHA_SMC_EFUSE_FID,
+			     AIROHA_SMC_EFUSE_SUB_ID_READ,
+			     priv->bank_index, offset, 0, 0, 0, 0, &res);
+
+	/* check if SMC reported an error */
+	if (res.a0)
+		return -EIO;
+
+	*val = res.a1;
+	return 0;
+}
+
+static int airoha_efuse_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	for_each_child_of_node_scoped(dev->of_node, child) {
+		struct nvmem_config nvmem_config = {
+			.size = AIROHA_EFUSE_CELLS * sizeof(u32),
+			.stride = sizeof(u32),
+			.word_size = sizeof(u32),
+			.reg_read = airoha_efuse_read,
+		};
+		struct regmap_config regmap_config = {
+			.reg_read = airoha_efuse_reg_read,
+			.reg_bits = 32,
+			.val_bits = 32,
+			.reg_stride = 4,
+		};
+		struct airoha_efuse_bank_priv *priv;
+		struct nvmem_device *nvmem;
+		struct regmap *regmap;
+		const char *name;
+		u32 bank;
+
+		ret = of_property_read_u32(child, "reg", &bank);
+		if (ret)
+			return ret;
+
+		priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+		if (!priv)
+			return -ENOMEM;
+
+		name = devm_kasprintf(dev, GFP_KERNEL, "airoha-efuse-%u",
+				      bank);
+		if (!name)
+			return -ENOMEM;
+
+		priv->bank_index = bank;
+
+		regmap_config.name = name;
+		regmap = devm_regmap_init(dev, NULL, priv,
+					  &regmap_config);
+		if (IS_ERR(regmap))
+			return PTR_ERR(regmap);
+
+		nvmem_config.name = name;
+		nvmem_config.priv = regmap;
+		nvmem_config.dev = dev;
+		nvmem_config.id = bank;
+		nvmem_config.of_node = child;
+		nvmem = devm_nvmem_register(dev, &nvmem_config);
+		if (IS_ERR(nvmem))
+			return PTR_ERR(nvmem);
+	}
+
+	return 0;
+}
+
+static const struct of_device_id airoha_efuse_of_match[] = {
+	{ .compatible = "airoha,an7581-efuses", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, airoha_efuse_of_match);
+
+static struct platform_driver airoha_efuse_driver = {
+	.probe = airoha_efuse_probe,
+	.driver = {
+		.name = "airoha-efuse",
+		.of_match_table = airoha_efuse_of_match,
+	},
+};
+module_platform_driver(airoha_efuse_driver);
+
+MODULE_AUTHOR("Christian Marangi <ansuelsmth@gmail.com>");
+MODULE_DESCRIPTION("Driver for Airoha SMC eFUSEs");
+MODULE_LICENSE("GPL");
diff --git a/drivers/nvmem/apple-spmi-nvmem.c b/drivers/nvmem/apple-spmi-nvmem.c
index 88614005d5ce1..7acb0c07d6abe 100644
--- a/drivers/nvmem/apple-spmi-nvmem.c
+++ b/drivers/nvmem/apple-spmi-nvmem.c
@@ -18,6 +18,22 @@ static const struct regmap_config apple_spmi_regmap_config = {
 	.max_register	= 0xffff,
 };
 
+static int apple_spmi_nvmem_read(void *priv, unsigned int offset, void *val,
+				 size_t bytes)
+{
+	struct regmap *map = priv;
+
+	return regmap_bulk_read(map, offset, val, bytes);
+}
+
+static int apple_spmi_nvmem_write(void *priv, unsigned int offset, void *val,
+				  size_t bytes)
+{
+	struct regmap *map = priv;
+
+	return regmap_bulk_write(map, offset, val, bytes);
+}
+
 static int apple_spmi_nvmem_probe(struct spmi_device *sdev)
 {
 	struct regmap *regmap;
@@ -28,8 +44,8 @@ static int apple_spmi_nvmem_probe(struct spmi_device *sdev)
 		.word_size = 1,
 		.stride = 1,
 		.size = 0xffff,
-		.reg_read = (void *)regmap_bulk_read,
-		.reg_write = (void *)regmap_bulk_write,
+		.reg_read = apple_spmi_nvmem_read,
+		.reg_write = apple_spmi_nvmem_write,
 	};
 
 	regmap = devm_regmap_init_spmi_ext(sdev, &apple_spmi_regmap_config);
diff --git a/drivers/nvmem/at24.c b/drivers/nvmem/at24.c
new file mode 100644
index 0000000000000..772c4d9fa6512
--- /dev/null
+++ b/drivers/nvmem/at24.c
@@ -0,0 +1,871 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * at24.c - handle most I2C EEPROMs
+ *
+ * Copyright (C) 2005-2007 David Brownell
+ * Copyright (C) 2008 Wolfram Sang, Pengutronix
+ */
+
+#include <linux/acpi.h>
+#include <linux/bitops.h>
+#include <linux/capability.h>
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/jiffies.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/nvmem-provider.h>
+#include <linux/pm_runtime.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/slab.h>
+
+/* Address pointer is 16 bit. */
+#define AT24_FLAG_ADDR16	BIT(7)
+/* sysfs-entry will be read-only. */
+#define AT24_FLAG_READONLY	BIT(6)
+/* sysfs-entry will be world-readable. */
+#define AT24_FLAG_IRUGO		BIT(5)
+/* Take always 8 addresses (24c00). */
+#define AT24_FLAG_TAKE8ADDR	BIT(4)
+/* Factory-programmed serial number. */
+#define AT24_FLAG_SERIAL	BIT(3)
+/* Factory-programmed mac address. */
+#define AT24_FLAG_MAC		BIT(2)
+/* Does not auto-rollover reads to the next slave address. */
+#define AT24_FLAG_NO_RDROL	BIT(1)
+
+/*
+ * I2C EEPROMs from most vendors are inexpensive and mostly interchangeable.
+ * Differences between different vendor product lines (like Atmel AT24C or
+ * MicroChip 24LC, etc) won't much matter for typical read/write access.
+ * There are also I2C RAM chips, likewise interchangeable. One example
+ * would be the PCF8570, which acts like a 24c02 EEPROM (256 bytes).
+ *
+ * However, misconfiguration can lose data. "Set 16-bit memory address"
+ * to a part with 8-bit addressing will overwrite data. Writing with too
+ * big a page size also loses data. And it's not safe to assume that the
+ * conventional addresses 0x50..0x57 only hold eeproms; a PCF8563 RTC
+ * uses 0x51, for just one example.
+ *
+ * Accordingly, explicit board-specific configuration data should be used
+ * in almost all cases. (One partial exception is an SMBus used to access
+ * "SPD" data for DRAM sticks. Those only use 24c02 EEPROMs.)
+ *
+ * So this driver uses "new style" I2C driver binding, expecting to be
+ * told what devices exist. That may be in arch/X/mach-Y/board-Z.c or
+ * similar kernel-resident tables; or, configuration data coming from
+ * a bootloader.
+ *
+ * Other than binding model, current differences from "eeprom" driver are
+ * that this one handles write access and isn't restricted to 24c02 devices.
+ * It also handles larger devices (32 kbit and up) with two-byte addresses,
+ * which won't work on pure SMBus systems.
+ */
+
+struct at24_data {
+	/*
+	 * Lock protects against activities from other Linux tasks,
+	 * but not from changes by other I2C masters.
+	 */
+	struct mutex lock;
+
+	unsigned int write_max;
+	unsigned int num_addresses;
+	unsigned int offset_adj;
+
+	u32 byte_len;
+	u16 page_size;
+	u8 flags;
+
+	struct nvmem_device *nvmem;
+	struct regulator *vcc_reg;
+	void (*read_post)(unsigned int off, char *buf, size_t count);
+
+	/*
+	 * Some chips tie up multiple I2C addresses; dummy devices reserve
+	 * them for us.
+	 */
+	u8 bank_addr_shift;
+	struct regmap *client_regmaps[] __counted_by(num_addresses);
+};
+
+/*
+ * This parameter is to help this driver avoid blocking other drivers out
+ * of I2C for potentially troublesome amounts of time. With a 100 kHz I2C
+ * clock, one 256 byte read takes about 1/43 second which is excessive;
+ * but the 1/170 second it takes at 400 kHz may be quite reasonable; and
+ * at 1 MHz (Fm+) a 1/430 second delay could easily be invisible.
+ *
+ * This value is forced to be a power of two so that writes align on pages.
+ */
+static unsigned int at24_io_limit = 128;
+module_param_named(io_limit, at24_io_limit, uint, 0);
+MODULE_PARM_DESC(at24_io_limit, "Maximum bytes per I/O (default 128)");
+
+/*
+ * Specs often allow 5 msec for a page write, sometimes 20 msec;
+ * it's important to recover from write timeouts.
+ */
+static unsigned int at24_write_timeout = 25;
+module_param_named(write_timeout, at24_write_timeout, uint, 0);
+MODULE_PARM_DESC(at24_write_timeout, "Time (in ms) to try writes (default 25)");
+
+struct at24_chip_data {
+	u32 byte_len;
+	u8 flags;
+	u8 bank_addr_shift;
+	void (*read_post)(unsigned int off, char *buf, size_t count);
+};
+
+#define AT24_CHIP_DATA(_name, _len, _flags)				\
+	static const struct at24_chip_data _name = {			\
+		.byte_len = _len, .flags = _flags,			\
+	}
+
+#define AT24_CHIP_DATA_CB(_name, _len, _flags, _read_post)		\
+	static const struct at24_chip_data _name = {			\
+		.byte_len = _len, .flags = _flags,			\
+		.read_post = _read_post,				\
+	}
+
+#define AT24_CHIP_DATA_BS(_name, _len, _flags, _bank_addr_shift)	\
+	static const struct at24_chip_data _name = {			\
+		.byte_len = _len, .flags = _flags,			\
+		.bank_addr_shift = _bank_addr_shift			\
+	}
+
+static void at24_read_post_vaio(unsigned int off, char *buf, size_t count)
+{
+	int i;
+
+	if (capable(CAP_SYS_ADMIN))
+		return;
+
+	/*
+	 * Hide VAIO private settings to regular users:
+	 * - BIOS passwords: bytes 0x00 to 0x0f
+	 * - UUID: bytes 0x10 to 0x1f
+	 * - Serial number: 0xc0 to 0xdf
+	 */
+	for (i = 0; i < count; i++) {
+		if ((off + i <= 0x1f) ||
+		    (off + i >= 0xc0 && off + i <= 0xdf))
+			buf[i] = 0;
+	}
+}
+
+/* needs 8 addresses as A0-A2 are ignored */
+AT24_CHIP_DATA(at24_data_24c00, 128 / 8, AT24_FLAG_TAKE8ADDR);
+/* old variants can't be handled with this generic entry! */
+AT24_CHIP_DATA(at24_data_24c01, 1024 / 8, 0);
+AT24_CHIP_DATA(at24_data_24cs01, 16,
+	AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
+AT24_CHIP_DATA(at24_data_24c02, 2048 / 8, 0);
+AT24_CHIP_DATA(at24_data_24cs02, 16,
+	AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
+AT24_CHIP_DATA(at24_data_24mac402, 48 / 8,
+	AT24_FLAG_MAC | AT24_FLAG_READONLY);
+AT24_CHIP_DATA(at24_data_24mac602, 64 / 8,
+	AT24_FLAG_MAC | AT24_FLAG_READONLY);
+AT24_CHIP_DATA(at24_data_24aa025e48, 48 / 8,
+	AT24_FLAG_READONLY);
+AT24_CHIP_DATA(at24_data_24aa025e64, 64 / 8,
+	AT24_FLAG_READONLY);
+/* spd is a 24c02 in memory DIMMs */
+AT24_CHIP_DATA(at24_data_spd, 2048 / 8,
+	AT24_FLAG_READONLY | AT24_FLAG_IRUGO);
+/* 24c02_vaio is a 24c02 on some Sony laptops */
+AT24_CHIP_DATA_CB(at24_data_24c02_vaio, 2048 / 8,
+	AT24_FLAG_READONLY | AT24_FLAG_IRUGO,
+	at24_read_post_vaio);
+AT24_CHIP_DATA(at24_data_24c04, 4096 / 8, 0);
+AT24_CHIP_DATA(at24_data_24cs04, 16,
+	AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
+/* 24rf08 quirk is handled at i2c-core */
+AT24_CHIP_DATA(at24_data_24c08, 8192 / 8, 0);
+AT24_CHIP_DATA(at24_data_24cs08, 16,
+	AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
+AT24_CHIP_DATA(at24_data_24c16, 16384 / 8, 0);
+AT24_CHIP_DATA(at24_data_24cs16, 16,
+	AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
+AT24_CHIP_DATA(at24_data_24c32, 32768 / 8, AT24_FLAG_ADDR16);
+/* M24C32-D Additional Write lockable page (M24C32-D order codes) */
+AT24_CHIP_DATA(at24_data_24c32d_wlp, 32, AT24_FLAG_ADDR16);
+AT24_CHIP_DATA(at24_data_24cs32, 16,
+	AT24_FLAG_ADDR16 | AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
+AT24_CHIP_DATA(at24_data_24c64, 65536 / 8, AT24_FLAG_ADDR16);
+/* M24C64-D Additional Write lockable page (M24C64-D order codes) */
+AT24_CHIP_DATA(at24_data_24c64d_wlp, 32, AT24_FLAG_ADDR16);
+AT24_CHIP_DATA(at24_data_24cs64, 16,
+	AT24_FLAG_ADDR16 | AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
+AT24_CHIP_DATA(at24_data_24c128, 131072 / 8, AT24_FLAG_ADDR16);
+AT24_CHIP_DATA(at24_data_24c256, 262144 / 8, AT24_FLAG_ADDR16);
+/* M24256E Additional Write lockable page (M24256E-F order codes) */
+AT24_CHIP_DATA(at24_data_24256e_wlp, 64, AT24_FLAG_ADDR16);
+AT24_CHIP_DATA(at24_data_24c512, 524288 / 8, AT24_FLAG_ADDR16);
+AT24_CHIP_DATA(at24_data_24c1024, 1048576 / 8, AT24_FLAG_ADDR16);
+AT24_CHIP_DATA_BS(at24_data_24c1025, 1048576 / 8, AT24_FLAG_ADDR16, 2);
+AT24_CHIP_DATA(at24_data_24c2048, 2097152 / 8, AT24_FLAG_ADDR16);
+/* identical to 24c08 ? */
+AT24_CHIP_DATA(at24_data_INT3499, 8192 / 8, 0);
+
+static const struct i2c_device_id at24_ids[] = {
+	{ .name = "24c00",	.driver_data = (kernel_ulong_t)&at24_data_24c00 },
+	{ .name = "24c01",	.driver_data = (kernel_ulong_t)&at24_data_24c01 },
+	{ .name = "24cs01",	.driver_data = (kernel_ulong_t)&at24_data_24cs01 },
+	{ .name = "24c02",	.driver_data = (kernel_ulong_t)&at24_data_24c02 },
+	{ .name = "24cs02",	.driver_data = (kernel_ulong_t)&at24_data_24cs02 },
+	{ .name = "24mac402",	.driver_data = (kernel_ulong_t)&at24_data_24mac402 },
+	{ .name = "24mac602",	.driver_data = (kernel_ulong_t)&at24_data_24mac602 },
+	{ .name = "24aa025e48",	.driver_data = (kernel_ulong_t)&at24_data_24aa025e48 },
+	{ .name = "24aa025e64",	.driver_data = (kernel_ulong_t)&at24_data_24aa025e64 },
+	{ .name = "spd",	.driver_data = (kernel_ulong_t)&at24_data_spd },
+	{ .name = "24c02-vaio",	.driver_data = (kernel_ulong_t)&at24_data_24c02_vaio },
+	{ .name = "24c04",	.driver_data = (kernel_ulong_t)&at24_data_24c04 },
+	{ .name = "24cs04",	.driver_data = (kernel_ulong_t)&at24_data_24cs04 },
+	{ .name = "24c08",	.driver_data = (kernel_ulong_t)&at24_data_24c08 },
+	{ .name = "24cs08",	.driver_data = (kernel_ulong_t)&at24_data_24cs08 },
+	{ .name = "24c16",	.driver_data = (kernel_ulong_t)&at24_data_24c16 },
+	{ .name = "24cs16",	.driver_data = (kernel_ulong_t)&at24_data_24cs16 },
+	{ .name = "24c32",	.driver_data = (kernel_ulong_t)&at24_data_24c32 },
+	{ .name = "24c32d-wl",	.driver_data = (kernel_ulong_t)&at24_data_24c32d_wlp },
+	{ .name = "24cs32",	.driver_data = (kernel_ulong_t)&at24_data_24cs32 },
+	{ .name = "24c64",	.driver_data = (kernel_ulong_t)&at24_data_24c64 },
+	{ .name = "24c64-wl",	.driver_data = (kernel_ulong_t)&at24_data_24c64d_wlp },
+	{ .name = "24cs64",	.driver_data = (kernel_ulong_t)&at24_data_24cs64 },
+	{ .name = "24c128",	.driver_data = (kernel_ulong_t)&at24_data_24c128 },
+	{ .name = "24c256",	.driver_data = (kernel_ulong_t)&at24_data_24c256 },
+	{ .name = "24256e-wl",	.driver_data = (kernel_ulong_t)&at24_data_24256e_wlp },
+	{ .name = "24c512",	.driver_data = (kernel_ulong_t)&at24_data_24c512 },
+	{ .name = "24c1024",	.driver_data = (kernel_ulong_t)&at24_data_24c1024 },
+	{ .name = "24c1025",	.driver_data = (kernel_ulong_t)&at24_data_24c1025 },
+	{ .name = "24c2048",	.driver_data = (kernel_ulong_t)&at24_data_24c2048 },
+	{ .name = "at24",	.driver_data = 0 },
+	{ /* END OF LIST */ }
+};
+MODULE_DEVICE_TABLE(i2c, at24_ids);
+
+static const struct of_device_id at24_of_match[] = {
+	{ .compatible = "atmel,24c00",		.data = &at24_data_24c00 },
+	{ .compatible = "atmel,24c01",		.data = &at24_data_24c01 },
+	{ .compatible = "atmel,24cs01",		.data = &at24_data_24cs01 },
+	{ .compatible = "atmel,24c02",		.data = &at24_data_24c02 },
+	{ .compatible = "atmel,24cs02",		.data = &at24_data_24cs02 },
+	{ .compatible = "atmel,24mac402",	.data = &at24_data_24mac402 },
+	{ .compatible = "atmel,24mac602",	.data = &at24_data_24mac602 },
+	{ .compatible = "atmel,spd",		.data = &at24_data_spd },
+	{ .compatible = "atmel,24c04",		.data = &at24_data_24c04 },
+	{ .compatible = "atmel,24cs04",		.data = &at24_data_24cs04 },
+	{ .compatible = "atmel,24c08",		.data = &at24_data_24c08 },
+	{ .compatible = "atmel,24cs08",		.data = &at24_data_24cs08 },
+	{ .compatible = "atmel,24c16",		.data = &at24_data_24c16 },
+	{ .compatible = "atmel,24cs16",		.data = &at24_data_24cs16 },
+	{ .compatible = "atmel,24c32",		.data = &at24_data_24c32 },
+	{ .compatible = "atmel,24c32d-wl",	.data = &at24_data_24c32d_wlp },
+	{ .compatible = "atmel,24cs32",		.data = &at24_data_24cs32 },
+	{ .compatible = "atmel,24c64",		.data = &at24_data_24c64 },
+	{ .compatible = "atmel,24c64d-wl",	.data = &at24_data_24c64d_wlp },
+	{ .compatible = "atmel,24cs64",		.data = &at24_data_24cs64 },
+	{ .compatible = "atmel,24c128",		.data = &at24_data_24c128 },
+	{ .compatible = "atmel,24c256",		.data = &at24_data_24c256 },
+	{ .compatible = "atmel,24c512",		.data = &at24_data_24c512 },
+	{ .compatible = "atmel,24c1024",	.data = &at24_data_24c1024 },
+	{ .compatible = "atmel,24c1025",	.data = &at24_data_24c1025 },
+	{ .compatible = "atmel,24c2048",	.data = &at24_data_24c2048 },
+	{ .compatible = "microchip,24aa025e48",	.data = &at24_data_24aa025e48 },
+	{ .compatible = "microchip,24aa025e64",	.data = &at24_data_24aa025e64 },
+	{ .compatible = "st,24256e-wl",		.data = &at24_data_24256e_wlp },
+	{ /* END OF LIST */ },
+};
+MODULE_DEVICE_TABLE(of, at24_of_match);
+
+static const struct acpi_device_id at24_acpi_ids[] = {
+	{ "INT3499",	(kernel_ulong_t)&at24_data_INT3499 },
+	{ "TPF0001",	(kernel_ulong_t)&at24_data_24c1024 },
+	{ /* END OF LIST */ }
+};
+MODULE_DEVICE_TABLE(acpi, at24_acpi_ids);
+
+/*
+ * This routine supports chips which consume multiple I2C addresses. It
+ * computes the addressing information to be used for a given r/w request.
+ * Assumes that sanity checks for offset happened at sysfs-layer.
+ *
+ * Slave address and byte offset derive from the offset. Always
+ * set the byte address; on a multi-master board, another master
+ * may have changed the chip's "current" address pointer.
+ */
+static struct regmap *at24_translate_offset(struct at24_data *at24,
+					    unsigned int *offset)
+{
+	unsigned int i;
+
+	if (at24->flags & AT24_FLAG_ADDR16) {
+		i = *offset >> 16;
+		*offset &= 0xffff;
+	} else {
+		i = *offset >> 8;
+		*offset &= 0xff;
+	}
+
+	return at24->client_regmaps[i];
+}
+
+static struct device *at24_base_client_dev(struct at24_data *at24)
+{
+	return regmap_get_device(at24->client_regmaps[0]);
+}
+
+static size_t at24_adjust_read_count(struct at24_data *at24,
+				      unsigned int offset, size_t count)
+{
+	unsigned int bits;
+	size_t remainder;
+
+	/*
+	 * In case of multi-address chips that don't rollover reads to
+	 * the next slave address: truncate the count to the slave boundary,
+	 * so that the read never straddles slaves.
+	 */
+	if (at24->flags & AT24_FLAG_NO_RDROL) {
+		bits = (at24->flags & AT24_FLAG_ADDR16) ? 16 : 8;
+		remainder = BIT(bits) - offset;
+		if (count > remainder)
+			count = remainder;
+	}
+
+	if (count > at24_io_limit)
+		count = at24_io_limit;
+
+	return count;
+}
+
+static ssize_t at24_regmap_read(struct at24_data *at24, char *buf,
+				unsigned int offset, size_t count)
+{
+	unsigned long timeout, read_time;
+	struct regmap *regmap;
+	int ret;
+
+	regmap = at24_translate_offset(at24, &offset);
+	count = at24_adjust_read_count(at24, offset, count);
+
+	/* adjust offset for mac and serial read ops */
+	offset += at24->offset_adj;
+
+	timeout = jiffies + msecs_to_jiffies(at24_write_timeout);
+	do {
+		/*
+		 * The timestamp shall be taken before the actual operation
+		 * to avoid a premature timeout in case of high CPU load.
+		 */
+		read_time = jiffies;
+
+		ret = regmap_bulk_read(regmap, offset, buf, count);
+		dev_dbg(regmap_get_device(regmap), "read %zu@%d --> %d (%ld)\n",
+			count, offset, ret, jiffies);
+		if (!ret)
+			return count;
+
+		usleep_range(1000, 1500);
+	} while (time_before(read_time, timeout));
+
+	return -ETIMEDOUT;
+}
+
+/*
+ * Note that if the hardware write-protect pin is pulled high, the whole
+ * chip is normally write protected. But there are plenty of product
+ * variants here, including OTP fuses and partial chip protect.
+ *
+ * We only use page mode writes; the alternative is sloooow. These routines
+ * write at most one page.
+ */
+
+static size_t at24_adjust_write_count(struct at24_data *at24,
+				      unsigned int offset, size_t count)
+{
+	unsigned int next_page;
+
+	/* write_max is at most a page */
+	if (count > at24->write_max)
+		count = at24->write_max;
+
+	/* Never roll over backwards, to the start of this page */
+	next_page = roundup(offset + 1, at24->page_size);
+	if (offset + count > next_page)
+		count = next_page - offset;
+
+	return count;
+}
+
+static ssize_t at24_regmap_write(struct at24_data *at24, const char *buf,
+				 unsigned int offset, size_t count)
+{
+	unsigned long timeout, write_time;
+	struct regmap *regmap;
+	int ret;
+
+	regmap = at24_translate_offset(at24, &offset);
+	count = at24_adjust_write_count(at24, offset, count);
+	timeout = jiffies + msecs_to_jiffies(at24_write_timeout);
+
+	do {
+		/*
+		 * The timestamp shall be taken before the actual operation
+		 * to avoid a premature timeout in case of high CPU load.
+		 */
+		write_time = jiffies;
+
+		ret = regmap_bulk_write(regmap, offset, buf, count);
+		dev_dbg(regmap_get_device(regmap), "write %zu@%d --> %d (%ld)\n",
+			count, offset, ret, jiffies);
+		if (!ret)
+			return count;
+
+		usleep_range(1000, 1500);
+	} while (time_before(write_time, timeout));
+
+	return -ETIMEDOUT;
+}
+
+static int at24_read(void *priv, unsigned int off, void *val, size_t count)
+{
+	struct at24_data *at24;
+	struct device *dev;
+	char *buf = val;
+	int i, ret;
+
+	at24 = priv;
+	dev = at24_base_client_dev(at24);
+
+	if (unlikely(!count))
+		return count;
+
+	if (off + count > at24->byte_len)
+		return -EINVAL;
+
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret)
+		return ret;
+	/*
+	 * Read data from chip, protecting against concurrent updates
+	 * from this host, but not from other I2C masters.
+	 */
+	mutex_lock(&at24->lock);
+
+	for (i = 0; count; i += ret, count -= ret) {
+		ret = at24_regmap_read(at24, buf + i, off + i, count);
+		if (ret < 0) {
+			mutex_unlock(&at24->lock);
+			pm_runtime_put(dev);
+			return ret;
+		}
+	}
+
+	mutex_unlock(&at24->lock);
+
+	pm_runtime_put(dev);
+
+	if (unlikely(at24->read_post))
+		at24->read_post(off, buf, i);
+
+	return 0;
+}
+
+static int at24_write(void *priv, unsigned int off, void *val, size_t count)
+{
+	struct at24_data *at24;
+	struct device *dev;
+	char *buf = val;
+	int ret;
+
+	at24 = priv;
+	dev = at24_base_client_dev(at24);
+
+	if (unlikely(!count))
+		return -EINVAL;
+
+	if (off + count > at24->byte_len)
+		return -EINVAL;
+
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret)
+		return ret;
+	/*
+	 * Write data to chip, protecting against concurrent updates
+	 * from this host, but not from other I2C masters.
+	 */
+	mutex_lock(&at24->lock);
+
+	while (count) {
+		ret = at24_regmap_write(at24, buf, off, count);
+		if (ret < 0) {
+			mutex_unlock(&at24->lock);
+			pm_runtime_put(dev);
+			return ret;
+		}
+		buf += ret;
+		off += ret;
+		count -= ret;
+	}
+
+	mutex_unlock(&at24->lock);
+
+	pm_runtime_put(dev);
+
+	return 0;
+}
+
+static int at24_make_dummy_client(struct at24_data *at24, unsigned int index,
+				  struct i2c_client *base_client,
+				  struct regmap_config *regmap_config)
+{
+	struct i2c_client *dummy_client;
+	struct regmap *regmap;
+
+	dummy_client = devm_i2c_new_dummy_device(&base_client->dev,
+						 base_client->adapter,
+						 base_client->addr +
+						 (index << at24->bank_addr_shift));
+	if (IS_ERR(dummy_client))
+		return PTR_ERR(dummy_client);
+
+	regmap = devm_regmap_init_i2c(dummy_client, regmap_config);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	at24->client_regmaps[index] = regmap;
+
+	return 0;
+}
+
+static unsigned int at24_get_offset_adj(u8 flags, unsigned int byte_len)
+{
+	if (flags & AT24_FLAG_MAC) {
+		/* EUI-48 starts from 0x9a, EUI-64 from 0x98 */
+		return 0xa0 - byte_len;
+	} else if (flags & AT24_FLAG_SERIAL && flags & AT24_FLAG_ADDR16) {
+		/*
+		 * For 16 bit address pointers, the word address must contain
+		 * a '10' sequence in bits 11 and 10 regardless of the
+		 * intended position of the address pointer.
+		 */
+		return 0x0800;
+	} else if (flags & AT24_FLAG_SERIAL) {
+		/*
+		 * Otherwise the word address must begin with a '10' sequence,
+		 * regardless of the intended address.
+		 */
+		return 0x0080;
+	} else {
+		return 0;
+	}
+}
+
+static void at24_probe_temp_sensor(struct i2c_client *client)
+{
+	struct at24_data *at24 = i2c_get_clientdata(client);
+	struct i2c_board_info info = { .type = "jc42" };
+	int ret;
+	u8 val;
+
+	/*
+	 * Byte 2 has value 11 for DDR3, earlier versions don't
+	 * support the thermal sensor present flag
+	 */
+	ret = at24_read(at24, 2, &val, 1);
+	if (ret || val != 11)
+		return;
+
+	/* Byte 32, bit 7 is set if temp sensor is present */
+	ret = at24_read(at24, 32, &val, 1);
+	if (ret || !(val & BIT(7)))
+		return;
+
+	info.addr = 0x18 | (client->addr & 7);
+
+	i2c_new_client_device(client->adapter, &info);
+}
+
+static int at24_probe(struct i2c_client *client)
+{
+	struct regmap_config regmap_config = { };
+	struct nvmem_config nvmem_config = { };
+	u32 byte_len, page_size, flags, addrw;
+	const struct at24_chip_data *cdata;
+	struct device *dev = &client->dev;
+	bool i2c_fn_i2c, i2c_fn_block;
+	unsigned int i, num_addresses;
+	struct at24_data *at24;
+	bool full_power;
+	struct regmap *regmap;
+	bool writable;
+	u8 test_byte;
+	int err;
+
+	i2c_fn_i2c = i2c_check_functionality(client->adapter, I2C_FUNC_I2C);
+	i2c_fn_block = i2c_check_functionality(client->adapter,
+					       I2C_FUNC_SMBUS_WRITE_I2C_BLOCK);
+
+	cdata = i2c_get_match_data(client);
+	if (!cdata)
+		return -ENODEV;
+
+	err = device_property_read_u32(dev, "pagesize", &page_size);
+	if (err)
+		/*
+		 * This is slow, but we can't know all eeproms, so we better
+		 * play safe. Specifying custom eeprom-types via device tree
+		 * or properties is recommended anyhow.
+		 */
+		page_size = 1;
+
+	flags = cdata->flags;
+	if (device_property_present(dev, "read-only"))
+		flags |= AT24_FLAG_READONLY;
+	if (device_property_present(dev, "no-read-rollover"))
+		flags |= AT24_FLAG_NO_RDROL;
+
+	err = device_property_read_u32(dev, "address-width", &addrw);
+	if (!err) {
+		switch (addrw) {
+		case 8:
+			if (flags & AT24_FLAG_ADDR16)
+				dev_warn(dev,
+					 "Override address width to be 8, while default is 16\n");
+			flags &= ~AT24_FLAG_ADDR16;
+			break;
+		case 16:
+			flags |= AT24_FLAG_ADDR16;
+			break;
+		default:
+			dev_warn(dev, "Bad \"address-width\" property: %u\n",
+				 addrw);
+		}
+	}
+
+	err = device_property_read_u32(dev, "size", &byte_len);
+	if (err)
+		byte_len = cdata->byte_len;
+
+	if (!i2c_fn_i2c && !i2c_fn_block)
+		page_size = 1;
+
+	if (!page_size)
+		return dev_err_probe(dev, -EINVAL, "page_size must not be 0!\n");
+
+	if (!is_power_of_2(page_size))
+		dev_warn(dev, "page_size looks suspicious (no power of 2)!\n");
+
+	err = device_property_read_u32(dev, "num-addresses", &num_addresses);
+	if (err) {
+		if (flags & AT24_FLAG_TAKE8ADDR)
+			num_addresses = 8;
+		else
+			num_addresses =	DIV_ROUND_UP(byte_len,
+				(flags & AT24_FLAG_ADDR16) ? 65536 : 256);
+	}
+
+	if ((flags & AT24_FLAG_SERIAL) && (flags & AT24_FLAG_MAC))
+		return dev_err_probe(dev, -EINVAL,
+				     "invalid device data - cannot have both AT24_FLAG_SERIAL & AT24_FLAG_MAC.");
+
+	regmap_config.val_bits = 8;
+	regmap_config.reg_bits = (flags & AT24_FLAG_ADDR16) ? 16 : 8;
+	regmap_config.disable_locking = true;
+
+	regmap = devm_regmap_init_i2c(client, &regmap_config);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	at24 = devm_kzalloc(dev, struct_size(at24, client_regmaps, num_addresses),
+			    GFP_KERNEL);
+	if (!at24)
+		return -ENOMEM;
+
+	mutex_init(&at24->lock);
+	at24->byte_len = byte_len;
+	at24->page_size = page_size;
+	at24->flags = flags;
+	at24->read_post = cdata->read_post;
+	at24->bank_addr_shift = cdata->bank_addr_shift;
+	at24->num_addresses = num_addresses;
+	at24->offset_adj = at24_get_offset_adj(flags, byte_len);
+	at24->client_regmaps[0] = regmap;
+
+	at24->vcc_reg = devm_regulator_get(dev, "vcc");
+	if (IS_ERR(at24->vcc_reg))
+		return PTR_ERR(at24->vcc_reg);
+
+	writable = !(flags & AT24_FLAG_READONLY);
+	if (writable) {
+		at24->write_max = min_t(unsigned int,
+					page_size, at24_io_limit);
+		if (!i2c_fn_i2c && at24->write_max > I2C_SMBUS_BLOCK_MAX)
+			at24->write_max = I2C_SMBUS_BLOCK_MAX;
+	}
+
+	/* use dummy devices for multiple-address chips */
+	for (i = 1; i < num_addresses; i++) {
+		err = at24_make_dummy_client(at24, i, client, &regmap_config);
+		if (err)
+			return err;
+	}
+
+	/*
+	 * We initialize nvmem_config.id to NVMEM_DEVID_AUTO even if the
+	 * label property is set as some platform can have multiple eeproms
+	 * with same label and we can not register each of those with same
+	 * label. Failing to register those eeproms trigger cascade failure
+	 * on such platform.
+	 */
+	nvmem_config.id = NVMEM_DEVID_AUTO;
+
+	if (device_property_present(dev, "label")) {
+		err = device_property_read_string(dev, "label",
+						  &nvmem_config.name);
+		if (err)
+			return err;
+	} else {
+		nvmem_config.name = dev_name(dev);
+	}
+
+	nvmem_config.type = NVMEM_TYPE_EEPROM;
+	nvmem_config.dev = dev;
+	nvmem_config.read_only = !writable;
+	nvmem_config.root_only = !(flags & AT24_FLAG_IRUGO);
+	nvmem_config.owner = THIS_MODULE;
+	nvmem_config.compat = true;
+	nvmem_config.base_dev = dev;
+	nvmem_config.reg_read = at24_read;
+	nvmem_config.reg_write = at24_write;
+	nvmem_config.priv = at24;
+	nvmem_config.stride = 1;
+	nvmem_config.word_size = 1;
+	nvmem_config.size = byte_len;
+
+	i2c_set_clientdata(client, at24);
+
+	full_power = acpi_dev_state_d0(&client->dev);
+	if (full_power) {
+		err = regulator_enable(at24->vcc_reg);
+		if (err)
+			return dev_err_probe(dev, err, "Failed to enable vcc regulator\n");
+
+		pm_runtime_set_active(dev);
+	}
+	pm_runtime_enable(dev);
+
+	/*
+	 * Perform a one-byte test read to verify that the chip is functional,
+	 * unless powering on the device is to be avoided during probe (i.e.
+	 * it's powered off right now).
+	 */
+	if (full_power) {
+		err = at24_read(at24, 0, &test_byte, 1);
+		if (err) {
+			pm_runtime_disable(dev);
+			if (!pm_runtime_status_suspended(dev))
+				regulator_disable(at24->vcc_reg);
+			return -ENODEV;
+		}
+	}
+
+	at24->nvmem = devm_nvmem_register(dev, &nvmem_config);
+	if (IS_ERR(at24->nvmem)) {
+		pm_runtime_disable(dev);
+		if (!pm_runtime_status_suspended(dev))
+			regulator_disable(at24->vcc_reg);
+		return dev_err_probe(dev, PTR_ERR(at24->nvmem),
+				     "failed to register nvmem\n");
+	}
+
+	/* If this a SPD EEPROM, probe for DDR3 thermal sensor */
+	if (cdata == &at24_data_spd)
+		at24_probe_temp_sensor(client);
+
+	pm_runtime_idle(dev);
+
+	if (writable)
+		dev_info(dev, "%u byte %s EEPROM, writable, %u bytes/write\n",
+			 byte_len, client->name, at24->write_max);
+	else
+		dev_info(dev, "%u byte %s EEPROM, read-only\n",
+			 byte_len, client->name);
+
+	return 0;
+}
+
+static void at24_remove(struct i2c_client *client)
+{
+	struct at24_data *at24 = i2c_get_clientdata(client);
+
+	pm_runtime_disable(&client->dev);
+	if (acpi_dev_state_d0(&client->dev)) {
+		if (!pm_runtime_status_suspended(&client->dev))
+			regulator_disable(at24->vcc_reg);
+		pm_runtime_set_suspended(&client->dev);
+	}
+}
+
+static int __maybe_unused at24_suspend(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct at24_data *at24 = i2c_get_clientdata(client);
+
+	return regulator_disable(at24->vcc_reg);
+}
+
+static int __maybe_unused at24_resume(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct at24_data *at24 = i2c_get_clientdata(client);
+
+	return regulator_enable(at24->vcc_reg);
+}
+
+static const struct dev_pm_ops at24_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(at24_suspend, at24_resume, NULL)
+};
+
+static struct i2c_driver at24_driver = {
+	.driver = {
+		.name = "at24",
+		.pm = &at24_pm_ops,
+		.of_match_table = at24_of_match,
+		.acpi_match_table = at24_acpi_ids,
+	},
+	.probe = at24_probe,
+	.remove = at24_remove,
+	.id_table = at24_ids,
+	.flags = I2C_DRV_ACPI_WAIVE_D0_PROBE,
+};
+
+static int __init at24_init(void)
+{
+	if (!at24_io_limit) {
+		pr_err("at24: at24_io_limit must not be 0!\n");
+		return -EINVAL;
+	}
+
+	at24_io_limit = rounddown_pow_of_two(at24_io_limit);
+	return i2c_add_driver(&at24_driver);
+}
+module_init(at24_init);
+
+static void __exit at24_exit(void)
+{
+	i2c_del_driver(&at24_driver);
+}
+module_exit(at24_exit);
+
+MODULE_DESCRIPTION("Driver for most I2C EEPROMs");
+MODULE_AUTHOR("David Brownell and Wolfram Sang");
+MODULE_LICENSE("GPL");
diff --git a/drivers/nvmem/at25.c b/drivers/nvmem/at25.c
new file mode 100644
index 0000000000000..bc2cfb75d9bb4
--- /dev/null
+++ b/drivers/nvmem/at25.c
@@ -0,0 +1,585 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for most of the SPI EEPROMs, such as Atmel AT25 models
+ * and Cypress FRAMs FM25 models.
+ *
+ * Copyright (C) 2006 David Brownell
+ */
+
+#include <linux/bits.h>
+#include <linux/cleanup.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+
+#include <linux/spi/eeprom.h>
+#include <linux/spi/spi.h>
+#include <linux/spi/spi-mem.h>
+
+#include <linux/nvmem-provider.h>
+
+/*
+ * NOTE: this is an *EEPROM* driver. The vagaries of product naming
+ * mean that some AT25 products are EEPROMs, and others are FLASH.
+ * Handle FLASH chips with the drivers/mtd/devices/m25p80.c driver,
+ * not this one!
+ *
+ * EEPROMs that can be used with this driver include, for example:
+ *   AT25M02, AT25128B
+ */
+
+#define	FM25_SN_LEN	8		/* serial number length */
+#define	FM25_MAX_ID_LEN	9		/* ID length */
+#define EE_MAXADDRLEN	3		/* 24 bit addresses, up to 2 MBytes */
+
+struct at25_data {
+	struct spi_eeprom	chip;
+	struct spi_mem		*spimem;
+	struct mutex		lock;
+	unsigned		addrlen;
+	struct nvmem_config	nvmem_config;
+	struct nvmem_device	*nvmem;
+	u8 sernum[FM25_SN_LEN];
+	u8 id[FM25_MAX_ID_LEN];
+	u8 id_len;
+};
+
+#define	AT25_WREN	0x06		/* latch the write enable */
+#define	AT25_WRDI	0x04		/* reset the write enable */
+#define	AT25_RDSR	0x05		/* read status register */
+#define	AT25_WRSR	0x01		/* write status register */
+#define	AT25_READ	0x03		/* read byte(s) */
+#define	AT25_WRITE	0x02		/* write byte(s)/sector */
+#define	FM25_SLEEP	0xb9		/* enter sleep mode */
+#define	FM25_RDID	0x9f		/* read device ID */
+#define	FM25_RDSN	0xc3		/* read S/N */
+
+#define	AT25_SR_nRDY	0x01		/* nRDY = write-in-progress */
+#define	AT25_SR_WEN	0x02		/* write enable (latched) */
+#define	AT25_SR_BP0	0x04		/* BP for software writeprotect */
+#define	AT25_SR_BP1	0x08
+#define	AT25_SR_WPEN	0x80		/* writeprotect enable */
+
+#define	AT25_INSTR_BIT3	0x08		/* additional address bit in instr */
+
+/*
+ * Specs often allow 5ms for a page write, sometimes 20ms;
+ * it's important to recover from write timeouts.
+ */
+#define	EE_TIMEOUT	25
+
+/*-------------------------------------------------------------------------*/
+
+#define	io_limit	PAGE_SIZE	/* bytes */
+
+/* Handle the address MSB as part of instruction byte */
+static u8 at25_instr(struct at25_data *at25, u8 instr, unsigned int off)
+{
+	if (!(at25->chip.flags & EE_INSTR_BIT3_IS_ADDR))
+		return instr;
+	if (off < BIT(at25->addrlen * 8))
+		return instr;
+	return instr | AT25_INSTR_BIT3;
+}
+
+static int at25_ee_read(void *priv, unsigned int offset,
+			void *val, size_t count)
+{
+	u8 *bounce __free(kfree) = kmalloc(min(count, io_limit), GFP_KERNEL);
+	struct at25_data *at25 = priv;
+	char *buf = val;
+	unsigned int msg_offset = offset;
+	size_t bytes_left = count;
+	size_t segment;
+	int status;
+
+	if (!bounce)
+		return -ENOMEM;
+
+	if (unlikely(offset >= at25->chip.byte_len))
+		return -EINVAL;
+	if ((offset + count) > at25->chip.byte_len)
+		count = at25->chip.byte_len - offset;
+	if (unlikely(!count))
+		return -EINVAL;
+
+	do {
+		struct spi_mem_op op;
+
+		segment = min(bytes_left, io_limit);
+
+		op = (struct spi_mem_op)SPI_MEM_OP(SPI_MEM_OP_CMD(at25_instr(at25, AT25_READ,
+									     msg_offset), 1),
+						   SPI_MEM_OP_ADDR(at25->addrlen, msg_offset, 1),
+						   SPI_MEM_OP_NO_DUMMY,
+						   SPI_MEM_OP_DATA_IN(segment, bounce, 1));
+
+		status = spi_mem_adjust_op_size(at25->spimem, &op);
+		if (status)
+			return status;
+		segment = op.data.nbytes;
+
+		mutex_lock(&at25->lock);
+		status = spi_mem_exec_op(at25->spimem, &op);
+		mutex_unlock(&at25->lock);
+		if (status)
+			return status;
+		memcpy(buf, bounce, segment);
+
+		msg_offset += segment;
+		buf += segment;
+		bytes_left -= segment;
+	} while (bytes_left > 0);
+
+	dev_dbg(&at25->spimem->spi->dev, "read %zu bytes at %d\n",
+		count, offset);
+	return 0;
+}
+
+/*
+ * Read extra registers as ID or serial number
+ *
+ * Allow for the callers to provide @buf on stack (not necessary DMA-capable)
+ * by allocating a bounce buffer internally.
+ */
+static int fm25_aux_read(struct at25_data *at25, u8 *buf, uint8_t command,
+			 int len)
+{
+	u8 *bounce __free(kfree) = kmalloc(len, GFP_KERNEL);
+	struct spi_mem_op op;
+	int status;
+
+	if (!bounce)
+		return -ENOMEM;
+
+	op = (struct spi_mem_op)SPI_MEM_OP(SPI_MEM_OP_CMD(command, 1),
+					   SPI_MEM_OP_NO_ADDR,
+					   SPI_MEM_OP_NO_DUMMY,
+					   SPI_MEM_OP_DATA_IN(len, bounce, 1));
+
+	status = spi_mem_exec_op(at25->spimem, &op);
+	dev_dbg(&at25->spimem->spi->dev, "read %d aux bytes --> %d\n", len, status);
+	if (status)
+		return status;
+
+	memcpy(buf, bounce, len);
+
+	return 0;
+}
+
+static ssize_t sernum_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct at25_data *at25;
+
+	at25 = dev_get_drvdata(dev);
+	return sysfs_emit(buf, "%*ph\n", (int)sizeof(at25->sernum), at25->sernum);
+}
+static DEVICE_ATTR_RO(sernum);
+
+static ssize_t jedec_id_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct at25_data *at25;
+
+	at25 = dev_get_drvdata(dev);
+
+	if (!at25->id_len)
+		return -EOPNOTSUPP;
+
+	return sysfs_emit(buf, "%*phN\n", at25->id_len, at25->id);
+}
+static DEVICE_ATTR_RO(jedec_id);
+
+static struct attribute *at25_attrs[] = {
+	&dev_attr_sernum.attr,
+	&dev_attr_jedec_id.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(at25);
+
+/*
+ * Poll Read Status Register with timeout
+ *
+ * Return:
+ * 0, if the chip is ready
+ * [positive] Status Register value as-is, if the chip is busy
+ * [negative] error code in case of read failure
+ */
+static int at25_wait_ready(struct at25_data *at25)
+{
+	u8 *bounce __free(kfree) = kmalloc(1, GFP_KERNEL);
+	struct spi_mem_op op;
+	int status;
+
+	if (!bounce)
+		return -ENOMEM;
+
+	op = (struct spi_mem_op)SPI_MEM_OP(SPI_MEM_OP_CMD(AT25_RDSR, 1),
+					   SPI_MEM_OP_NO_ADDR,
+					   SPI_MEM_OP_NO_DUMMY,
+					   SPI_MEM_OP_DATA_IN(1, bounce, 1));
+
+	read_poll_timeout(spi_mem_exec_op, status,
+			  status || !(bounce[0] & AT25_SR_nRDY), false,
+			  USEC_PER_MSEC, USEC_PER_MSEC * EE_TIMEOUT,
+			  at25->spimem, &op);
+	if (status < 0)
+		return status;
+	if (!(bounce[0] & AT25_SR_nRDY))
+		return 0;
+
+	return bounce[0];
+}
+
+static int at25_ee_write(void *priv, unsigned int off, void *val, size_t count)
+{
+	u8 *bounce __free(kfree) = kmalloc(min(count, io_limit), GFP_KERNEL);
+	struct at25_data *at25 = priv;
+	const char *buf = val;
+	unsigned int buf_size;
+	int status;
+
+	if (unlikely(off >= at25->chip.byte_len))
+		return -EFBIG;
+	if ((off + count) > at25->chip.byte_len)
+		count = at25->chip.byte_len - off;
+	if (unlikely(!count))
+		return -EINVAL;
+
+	buf_size = at25->chip.page_size;
+
+	if (!bounce)
+		return -ENOMEM;
+
+	/*
+	 * For write, rollover is within the page ... so we write at
+	 * most one page, then manually roll over to the next page.
+	 */
+	guard(mutex)(&at25->lock);
+	do {
+		struct spi_mem_op op = SPI_MEM_OP(SPI_MEM_OP_CMD(AT25_WREN, 1),
+						  SPI_MEM_OP_NO_ADDR,
+						  SPI_MEM_OP_NO_DUMMY,
+						  SPI_MEM_OP_NO_DATA);
+		unsigned int segment;
+
+		status = spi_mem_exec_op(at25->spimem, &op);
+		if (status < 0) {
+			dev_dbg(&at25->spimem->spi->dev, "WREN --> %d\n", status);
+			return status;
+		}
+
+		/* Write as much of a page as we can */
+		segment = buf_size - (off % buf_size);
+		if (segment > count)
+			segment = count;
+		if (segment > io_limit)
+			segment = io_limit;
+
+		op = (struct spi_mem_op)SPI_MEM_OP(SPI_MEM_OP_CMD(at25_instr(at25, AT25_WRITE, off),
+								  1),
+						   SPI_MEM_OP_ADDR(at25->addrlen, off, 1),
+						   SPI_MEM_OP_NO_DUMMY,
+						   SPI_MEM_OP_DATA_OUT(segment, bounce, 1));
+
+		status = spi_mem_adjust_op_size(at25->spimem, &op);
+		if (status)
+			return status;
+		segment = op.data.nbytes;
+
+		memcpy(bounce, buf, segment);
+
+		status = spi_mem_exec_op(at25->spimem, &op);
+		dev_dbg(&at25->spimem->spi->dev, "write %u bytes at %u --> %d\n",
+			segment, off, status);
+		if (status)
+			return status;
+
+		/*
+		 * REVISIT this should detect (or prevent) failed writes
+		 * to read-only sections of the EEPROM...
+		 */
+
+		status = at25_wait_ready(at25);
+		if (status < 0) {
+			dev_err_probe(&at25->spimem->spi->dev, status,
+				      "Read Status Redister command failed\n");
+			return status;
+		}
+		if (status) {
+			dev_dbg(&at25->spimem->spi->dev,
+				"Status %02x\n", status);
+			dev_err(&at25->spimem->spi->dev,
+				"write %u bytes offset %u, timeout after %u msecs\n",
+				segment, off, EE_TIMEOUT);
+			return -ETIMEDOUT;
+		}
+
+		off += segment;
+		buf += segment;
+		count -= segment;
+
+	} while (count > 0);
+
+	return status;
+}
+
+/*-------------------------------------------------------------------------*/
+
+static int at25_fw_to_chip(struct device *dev, struct spi_eeprom *chip)
+{
+	u32 val;
+	int err;
+
+	strscpy(chip->name, "at25", sizeof(chip->name));
+
+	err = device_property_read_u32(dev, "size", &val);
+	if (err)
+		err = device_property_read_u32(dev, "at25,byte-len", &val);
+	if (err) {
+		dev_err(dev, "Error: missing \"size\" property\n");
+		return err;
+	}
+	chip->byte_len = val;
+
+	err = device_property_read_u32(dev, "pagesize", &val);
+	if (err)
+		err = device_property_read_u32(dev, "at25,page-size", &val);
+	if (err) {
+		dev_err(dev, "Error: missing \"pagesize\" property\n");
+		return err;
+	}
+	chip->page_size = val;
+
+	err = device_property_read_u32(dev, "address-width", &val);
+	if (err) {
+		err = device_property_read_u32(dev, "at25,addr-mode", &val);
+		if (err) {
+			dev_err(dev, "Error: missing \"address-width\" property\n");
+			return err;
+		}
+		chip->flags = (u16)val;
+	} else {
+		switch (val) {
+		case 9:
+			chip->flags |= EE_INSTR_BIT3_IS_ADDR;
+			fallthrough;
+		case 8:
+			chip->flags |= EE_ADDR1;
+			break;
+		case 16:
+			chip->flags |= EE_ADDR2;
+			break;
+		case 24:
+			chip->flags |= EE_ADDR3;
+			break;
+		default:
+			dev_err(dev,
+				"Error: bad \"address-width\" property: %u\n",
+				val);
+			return -ENODEV;
+		}
+		if (device_property_present(dev, "read-only"))
+			chip->flags |= EE_READONLY;
+	}
+	return 0;
+}
+
+static int at25_fram_to_chip(struct device *dev, struct spi_eeprom *chip)
+{
+	struct at25_data *at25 = container_of(chip, struct at25_data, chip);
+	u8 sernum[FM25_SN_LEN];
+	u8 id[FM25_MAX_ID_LEN];
+	u32 val;
+	int i;
+
+	strscpy(chip->name, "fm25", sizeof(chip->name));
+
+	if (!device_property_read_u32(dev, "size", &val)) {
+		chip->byte_len = val;
+	} else {
+		/* Get ID of chip */
+		fm25_aux_read(at25, id, FM25_RDID, FM25_MAX_ID_LEN);
+
+		/* Store the unprocessed ID for exposing via sysfs */
+		memcpy(at25->id, id, FM25_MAX_ID_LEN);
+		at25->id_len = FM25_MAX_ID_LEN;
+
+		/* There are inside-out FRAM variations, detect them and reverse the ID bytes */
+		if (id[6] == 0x7f && id[2] == 0xc2)
+			for (i = 0; i < ARRAY_SIZE(id) / 2; i++) {
+				u8 tmp = id[i];
+				int j = ARRAY_SIZE(id) - i - 1;
+
+				id[i] = id[j];
+				id[j] = tmp;
+			}
+
+		if (id[6] == 0xc2) {
+			at25->id_len = 9;
+			switch (id[7]) {
+			case 0x21 ... 0x26:
+				chip->byte_len = BIT(id[7] - 0x21 + 4) * 1024;
+				break;
+			case 0x2a ... 0x30:
+				/* CY15B102QN ... CY15B116QN */
+				chip->byte_len = BIT(((id[7] >> 1) & 0xf) + 13);
+				break;
+			default:
+				dev_err(dev, "Error: unsupported size (id %02x)\n", id[7]);
+				return -ENODEV;
+			}
+		} else if (id[2] == 0x82 && id[3] == 0x06) {
+			at25->id_len = 8;
+			switch (id[1]) {
+			case 0x51 ... 0x54:
+				/* CY15B102QSN ... CY15B204QSN */
+				chip->byte_len = BIT(((id[0] >> 3) & 0x1F) + 9);
+				break;
+			default:
+				dev_err(dev, "Error: unsupported product id %02x\n", id[1]);
+				return -ENODEV;
+			}
+		} else {
+			dev_err(dev, "Error: unrecognized JEDEC ID format: %*ph\n",
+				FM25_MAX_ID_LEN, id);
+			return -ENODEV;
+		}
+
+		fm25_aux_read(at25, sernum, FM25_RDSN, FM25_SN_LEN);
+		/* Swap byte order */
+		for (i = 0; i < FM25_SN_LEN; i++)
+			at25->sernum[i] = sernum[FM25_SN_LEN - 1 - i];
+	}
+
+	if (chip->byte_len > 64 * 1024)
+		chip->flags |= EE_ADDR3;
+	else
+		chip->flags |= EE_ADDR2;
+
+	chip->page_size = PAGE_SIZE;
+	return 0;
+}
+
+static const struct of_device_id at25_of_match[] = {
+	{ .compatible = "atmel,at25" },
+	{ .compatible = "cypress,fm25" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, at25_of_match);
+
+static const struct spi_device_id at25_spi_ids[] = {
+	{ .name = "at25" },
+	{ .name = "fm25" },
+	{ }
+};
+MODULE_DEVICE_TABLE(spi, at25_spi_ids);
+
+static int at25_probe(struct spi_mem *mem)
+{
+	struct spi_device *spi = mem->spi;
+	struct spi_eeprom *pdata;
+	struct at25_data *at25;
+	bool is_fram;
+	int err;
+
+	at25 = devm_kzalloc(&spi->dev, sizeof(*at25), GFP_KERNEL);
+	if (!at25)
+		return -ENOMEM;
+
+	at25->spimem = mem;
+
+	/*
+	 * Ping the chip ... the status register is pretty portable,
+	 * unlike probing manufacturer IDs.
+	 */
+	err = at25_wait_ready(at25);
+	if (err < 0)
+		return dev_err_probe(&spi->dev, err, "Read Status Register command failed\n");
+	if (err) {
+		dev_err(&spi->dev, "Not ready (%02x)\n", err);
+		return -ENXIO;
+	}
+
+	mutex_init(&at25->lock);
+	spi_set_drvdata(spi, at25);
+
+	is_fram = fwnode_device_is_compatible(dev_fwnode(&spi->dev), "cypress,fm25");
+
+	/* Chip description */
+	pdata = dev_get_platdata(&spi->dev);
+	if (pdata) {
+		at25->chip = *pdata;
+	} else {
+		if (is_fram)
+			err = at25_fram_to_chip(&spi->dev, &at25->chip);
+		else
+			err = at25_fw_to_chip(&spi->dev, &at25->chip);
+		if (err)
+			return err;
+	}
+
+	/* For now we only support 8/16/24 bit addressing */
+	if (at25->chip.flags & EE_ADDR1)
+		at25->addrlen = 1;
+	else if (at25->chip.flags & EE_ADDR2)
+		at25->addrlen = 2;
+	else if (at25->chip.flags & EE_ADDR3)
+		at25->addrlen = 3;
+	else {
+		dev_dbg(&spi->dev, "unsupported address type\n");
+		return -EINVAL;
+	}
+
+	at25->nvmem_config.type = is_fram ? NVMEM_TYPE_FRAM : NVMEM_TYPE_EEPROM;
+	at25->nvmem_config.name = dev_name(&spi->dev);
+	at25->nvmem_config.dev = &spi->dev;
+	at25->nvmem_config.read_only = at25->chip.flags & EE_READONLY;
+	at25->nvmem_config.root_only = true;
+	at25->nvmem_config.owner = THIS_MODULE;
+	at25->nvmem_config.compat = true;
+	at25->nvmem_config.base_dev = &spi->dev;
+	at25->nvmem_config.reg_read = at25_ee_read;
+	at25->nvmem_config.reg_write = at25_ee_write;
+	at25->nvmem_config.priv = at25;
+	at25->nvmem_config.stride = 1;
+	at25->nvmem_config.word_size = 1;
+	at25->nvmem_config.size = at25->chip.byte_len;
+
+	at25->nvmem = devm_nvmem_register(&spi->dev, &at25->nvmem_config);
+	if (IS_ERR(at25->nvmem))
+		return PTR_ERR(at25->nvmem);
+
+	dev_info(&spi->dev, "%d %s %s %s%s, pagesize %u\n",
+		 (at25->chip.byte_len < 1024) ?
+			at25->chip.byte_len : (at25->chip.byte_len / 1024),
+		 (at25->chip.byte_len < 1024) ? "Byte" : "KByte",
+		 at25->chip.name, is_fram ? "fram" : "eeprom",
+		 (at25->chip.flags & EE_READONLY) ? " (readonly)" : "",
+		 at25->chip.page_size);
+	return 0;
+}
+
+/*-------------------------------------------------------------------------*/
+
+static struct spi_mem_driver at25_driver = {
+	.spidrv = {
+		.driver = {
+			.name		= "at25",
+			.of_match_table = at25_of_match,
+			.dev_groups	= at25_groups,
+		},
+		.id_table	= at25_spi_ids,
+	},
+	.probe		= at25_probe,
+};
+
+module_spi_mem_driver(at25_driver);
+
+MODULE_DESCRIPTION("Driver for most SPI EEPROMs");
+MODULE_AUTHOR("David Brownell");
+MODULE_LICENSE("GPL");
diff --git a/drivers/nvmem/brcm_nvram.c b/drivers/nvmem/brcm_nvram.c
index aaa6537798bf9..9f77aee37121f 100644
--- a/drivers/nvmem/brcm_nvram.c
+++ b/drivers/nvmem/brcm_nvram.c
@@ -84,6 +84,11 @@ static int brcm_nvram_copy_data(struct brcm_nvram *priv, struct platform_device
 	}
 	WARN(priv->data_len > SZ_128K, "Unexpected (big) NVRAM size: %zu B\n", priv->data_len);
 
+	if (priv->data_len < sizeof(struct brcm_nvram_header)) {
+		dev_err(priv->dev, "NVRAM data too small (%zu)\n", priv->data_len);
+		return -EINVAL;
+	}
+
 	priv->data = devm_kzalloc(priv->dev, priv->data_len, GFP_KERNEL);
 	if (!priv->data)
 		return -ENOMEM;
@@ -187,9 +192,13 @@ static int brcm_nvram_parse(struct brcm_nvram *priv)
 	}
 
 	len = le32_to_cpu(header->len);
-	if (len > priv->nvmem_size) {
-		dev_err(dev, "NVRAM length (%zd) exceeds mapped size (%zd)\n", len,
-			priv->nvmem_size);
+	if (len < sizeof(*header)) {
+		dev_err(dev, "NVRAM length (%zd) too small\n", len);
+		return -EINVAL;
+	}
+	if (len > priv->data_len) {
+		dev_err(dev, "NVRAM length (%zd) exceeds data size (%zd)\n", len,
+			priv->data_len);
 		return -EINVAL;
 	}
 
diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c
index e871181751f3c..f65aebe7837b3 100644
--- a/drivers/nvmem/core.c
+++ b/drivers/nvmem/core.c
@@ -6,6 +6,7 @@
  * Copyright (C) 2013 Maxime Ripard <maxime.ripard@free-electrons.com>
  */
 
+#include <linux/cleanup.h>
 #include <linux/device.h>
 #include <linux/export.h>
 #include <linux/fs.h>
@@ -13,6 +14,7 @@
 #include <linux/init.h>
 #include <linux/kref.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/nvmem-consumer.h>
 #include <linux/nvmem-provider.h>
 #include <linux/gpio/consumer.h>
@@ -55,25 +57,44 @@ static BLOCKING_NOTIFIER_HEAD(nvmem_notifier);
 static int __nvmem_reg_read(struct nvmem_device *nvmem, unsigned int offset,
 			    void *val, size_t bytes)
 {
-	if (nvmem->reg_read)
-		return nvmem->reg_read(nvmem->priv, offset, val, bytes);
+	struct nvmem_operations *ops;
 
-	return -EINVAL;
+	guard(srcu)(&nvmem->srcu);
+	ops = srcu_dereference(nvmem->ops, &nvmem->srcu);
+	if (!ops)
+		return -ENODEV;
+
+	if (!ops->reg_read)
+		return -EOPNOTSUPP;
+
+	return ops->reg_read(nvmem->priv, offset, val, bytes);
 }
 
 static int __nvmem_reg_write(struct nvmem_device *nvmem, unsigned int offset,
 			     void *val, size_t bytes)
 {
-	int ret;
+	struct nvmem_operations *ops;
+	int ret, wr_ok;
+
+	guard(srcu)(&nvmem->srcu);
+	ops = srcu_dereference(nvmem->ops, &nvmem->srcu);
+	if (!ops)
+		return -ENODEV;
+
+	if (!ops->reg_write)
+		return -EOPNOTSUPP;
 
-	if (nvmem->reg_write) {
-		gpiod_set_value_cansleep(nvmem->wp_gpio, 0);
-		ret = nvmem->reg_write(nvmem->priv, offset, val, bytes);
-		gpiod_set_value_cansleep(nvmem->wp_gpio, 1);
+	ret = gpiod_set_value_cansleep(nvmem->wp_gpio, 0);
+	if (ret)
+		return ret;
+
+	wr_ok = ops->reg_write(nvmem->priv, offset, val, bytes);
+
+	ret = gpiod_set_value_cansleep(nvmem->wp_gpio, 1);
+	if (ret)
 		return ret;
-	}
 
-	return -EINVAL;
+	return wr_ok;
 }
 
 static int nvmem_access_with_keepouts(struct nvmem_device *nvmem,
@@ -231,13 +252,12 @@ static ssize_t bin_attr_nvmem_read(struct file *filp, struct kobject *kobj,
 
 	count = round_down(count, nvmem->word_size);
 
-	if (!nvmem->reg_read)
-		return -EPERM;
-
 	rc = nvmem_reg_read(nvmem, pos, buf, count);
-
-	if (rc)
+	if (rc) {
+		if (rc == -EOPNOTSUPP)
+			return -EPERM;
 		return rc;
+	}
 
 	return count;
 }
@@ -264,19 +284,23 @@ static ssize_t bin_attr_nvmem_write(struct file *filp, struct kobject *kobj,
 
 	count = round_down(count, nvmem->word_size);
 
-	if (!nvmem->reg_write || nvmem->read_only)
+	if (nvmem->read_only)
 		return -EPERM;
 
 	rc = nvmem_reg_write(nvmem, pos, buf, count);
-
-	if (rc)
+	if (rc) {
+		if (rc == -EOPNOTSUPP)
+			return -EPERM;
 		return rc;
+	}
 
 	return count;
 }
 
 static umode_t nvmem_bin_attr_get_umode(struct nvmem_device *nvmem)
 {
+	struct nvmem_operations *ops = rcu_dereference_raw(nvmem->ops);
+
 	umode_t mode = 0400;
 
 	if (!nvmem->root_only)
@@ -285,10 +309,10 @@ static umode_t nvmem_bin_attr_get_umode(struct nvmem_device *nvmem)
 	if (!nvmem->read_only)
 		mode |= 0200;
 
-	if (!nvmem->reg_write)
+	if (!ops->reg_write)
 		mode &= ~0200;
 
-	if (!nvmem->reg_read)
+	if (!ops->reg_read)
 		mode &= ~0444;
 
 	return mode;
@@ -319,6 +343,7 @@ static umode_t nvmem_attr_is_visible(struct kobject *kobj,
 {
 	struct device *dev = kobj_to_dev(kobj);
 	struct nvmem_device *nvmem = to_nvmem_device(dev);
+	struct nvmem_operations *ops = rcu_dereference_raw(nvmem->ops);
 
 	/*
 	 * If the device has no .reg_write operation, do not allow
@@ -327,7 +352,7 @@ static umode_t nvmem_attr_is_visible(struct kobject *kobj,
 	 * can be forced into read-write mode using the 'force_ro'
 	 * attribute.
 	 */
-	if (attr == &dev_attr_force_ro.attr && !nvmem->reg_write)
+	if (attr == &dev_attr_force_ro.attr && !ops->reg_write)
 		return 0;	/* Attribute not visible */
 
 	return attr->mode;
@@ -445,10 +470,9 @@ static int nvmem_sysfs_setup_compat(struct nvmem_device *nvmem,
 	return 0;
 }
 
-static void nvmem_sysfs_remove_compat(struct nvmem_device *nvmem,
-			      const struct nvmem_config *config)
+static void nvmem_sysfs_remove_compat(struct nvmem_device *nvmem)
 {
-	if (config->compat)
+	if (nvmem->flags & FLAG_COMPAT)
 		device_remove_bin_file(nvmem->base_dev, &nvmem->eeprom);
 }
 
@@ -461,27 +485,23 @@ static int nvmem_populate_sysfs_cells(struct nvmem_device *nvmem)
 	const struct bin_attribute **pattrs;
 	struct bin_attribute *attrs;
 	unsigned int ncells = 0, i = 0;
-	int ret = 0;
+	int ret;
 
-	mutex_lock(&nvmem_mutex);
+	guard(mutex)(&nvmem_mutex);
 
 	if (list_empty(&nvmem->cells) || nvmem->sysfs_cells_populated)
-		goto unlock_mutex;
+		return 0;
 
 	/* Allocate an array of attributes with a sentinel */
 	ncells = list_count_nodes(&nvmem->cells);
 	pattrs = devm_kcalloc(&nvmem->dev, ncells + 1,
 			      sizeof(struct bin_attribute *), GFP_KERNEL);
-	if (!pattrs) {
-		ret = -ENOMEM;
-		goto unlock_mutex;
-	}
+	if (!pattrs)
+		return -ENOMEM;
 
 	attrs = devm_kcalloc(&nvmem->dev, ncells, sizeof(struct bin_attribute), GFP_KERNEL);
-	if (!attrs) {
-		ret = -ENOMEM;
-		goto unlock_mutex;
-	}
+	if (!attrs)
+		return -ENOMEM;
 
 	/* Initialize each attribute to take the name and size of the cell */
 	list_for_each_entry(entry, &nvmem->cells, node) {
@@ -494,10 +514,8 @@ static int nvmem_populate_sysfs_cells(struct nvmem_device *nvmem)
 		attrs[i].size = entry->bytes;
 		attrs[i].read = &nvmem_cell_attr_read;
 		attrs[i].private = entry;
-		if (!attrs[i].attr.name) {
-			ret = -ENOMEM;
-			goto unlock_mutex;
-		}
+		if (!attrs[i].attr.name)
+			return -ENOMEM;
 
 		pattrs[i] = &attrs[i];
 		i++;
@@ -507,13 +525,10 @@ static int nvmem_populate_sysfs_cells(struct nvmem_device *nvmem)
 
 	ret = device_add_group(&nvmem->dev, &group);
 	if (ret)
-		goto unlock_mutex;
+		return ret;
 
 	nvmem->sysfs_cells_populated = true;
 
-unlock_mutex:
-	mutex_unlock(&nvmem_mutex);
-
 	return ret;
 }
 
@@ -524,36 +539,17 @@ static int nvmem_sysfs_setup_compat(struct nvmem_device *nvmem,
 {
 	return -ENOSYS;
 }
-static void nvmem_sysfs_remove_compat(struct nvmem_device *nvmem,
-				      const struct nvmem_config *config)
+static void nvmem_sysfs_remove_compat(struct nvmem_device *nvmem)
 {
 }
 
 #endif /* CONFIG_NVMEM_SYSFS */
 
-static void nvmem_release(struct device *dev)
-{
-	struct nvmem_device *nvmem = to_nvmem_device(dev);
-
-	ida_free(&nvmem_ida, nvmem->id);
-	gpiod_put(nvmem->wp_gpio);
-	kfree(nvmem);
-}
-
-static const struct device_type nvmem_provider_type = {
-	.release	= nvmem_release,
-};
-
-static const struct bus_type nvmem_bus_type = {
-	.name		= "nvmem",
-};
-
 static void nvmem_cell_entry_drop(struct nvmem_cell_entry *cell)
 {
 	blocking_notifier_call_chain(&nvmem_notifier, NVMEM_CELL_REMOVE, cell);
-	mutex_lock(&nvmem_mutex);
-	list_del(&cell->node);
-	mutex_unlock(&nvmem_mutex);
+	scoped_guard(mutex, &nvmem_mutex)
+		list_del(&cell->node);
 	of_node_put(cell->np);
 	kfree_const(cell->name);
 	kfree(cell);
@@ -567,11 +563,29 @@ static void nvmem_device_remove_all_cells(const struct nvmem_device *nvmem)
 		nvmem_cell_entry_drop(cell);
 }
 
+static void nvmem_release(struct device *dev)
+{
+	struct nvmem_device *nvmem = to_nvmem_device(dev);
+
+	gpiod_put(nvmem->wp_gpio);
+	nvmem_device_remove_all_cells(nvmem);
+	ida_free(&nvmem_ida, nvmem->id);
+	cleanup_srcu_struct(&nvmem->srcu);
+	kfree(nvmem);
+}
+
+static const struct device_type nvmem_provider_type = {
+	.release	= nvmem_release,
+};
+
+static const struct bus_type nvmem_bus_type = {
+	.name		= "nvmem",
+};
+
 static void nvmem_cell_entry_add(struct nvmem_cell_entry *cell)
 {
-	mutex_lock(&nvmem_mutex);
-	list_add_tail(&cell->node, &cell->nvmem->cells);
-	mutex_unlock(&nvmem_mutex);
+	scoped_guard(mutex, &nvmem_mutex)
+		list_add_tail(&cell->node, &cell->nvmem->cells);
 	blocking_notifier_call_chain(&nvmem_notifier, NVMEM_CELL_ADD, cell);
 }
 
@@ -721,14 +735,14 @@ nvmem_find_cell_entry_by_name(struct nvmem_device *nvmem, const char *cell_id)
 {
 	struct nvmem_cell_entry *iter, *cell = NULL;
 
-	mutex_lock(&nvmem_mutex);
+	guard(mutex)(&nvmem_mutex);
+
 	list_for_each_entry(iter, &nvmem->cells, node) {
 		if (strcmp(cell_id, iter->name) == 0) {
 			cell = iter;
 			break;
 		}
 	}
-	mutex_unlock(&nvmem_mutex);
 
 	return cell;
 }
@@ -786,7 +800,7 @@ static int nvmem_validate_keepouts(struct nvmem_device *nvmem)
 	return 0;
 }
 
-static int nvmem_add_cells_from_dt(struct nvmem_device *nvmem, struct device_node *np)
+int nvmem_add_cells_from_dt(struct nvmem_device *nvmem, struct device_node *np)
 {
 	struct device *dev = &nvmem->dev;
 	const __be32 *addr;
@@ -834,29 +848,13 @@ static int nvmem_add_cells_from_dt(struct nvmem_device *nvmem, struct device_nod
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(nvmem_add_cells_from_dt);
 
 static int nvmem_add_cells_from_legacy_of(struct nvmem_device *nvmem)
 {
 	return nvmem_add_cells_from_dt(nvmem, nvmem->dev.of_node);
 }
 
-static int nvmem_add_cells_from_fixed_layout(struct nvmem_device *nvmem)
-{
-	struct device_node *layout_np;
-	int err = 0;
-
-	layout_np = of_nvmem_layout_get_container(nvmem);
-	if (!layout_np)
-		return 0;
-
-	if (of_device_is_compatible(layout_np, "fixed-layout"))
-		err = nvmem_add_cells_from_dt(nvmem, layout_np);
-
-	of_node_put(layout_np);
-
-	return err;
-}
-
 int nvmem_layout_register(struct nvmem_layout *layout)
 {
 	int ret;
@@ -899,6 +897,7 @@ EXPORT_SYMBOL_GPL(nvmem_layout_unregister);
 
 struct nvmem_device *nvmem_register(const struct nvmem_config *config)
 {
+	struct nvmem_operations *ops;
 	struct nvmem_device *nvmem;
 	int rval;
 
@@ -912,8 +911,15 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config)
 	if (!nvmem)
 		return ERR_PTR(-ENOMEM);
 
+	ops = kzalloc_obj(*ops);
+	if (!ops) {
+		kfree(nvmem);
+		return ERR_PTR(-ENOMEM);
+	}
+
 	rval = ida_alloc(&nvmem_ida, GFP_KERNEL);
 	if (rval < 0) {
+		kfree(ops);
 		kfree(nvmem);
 		return ERR_PTR(rval);
 	}
@@ -923,6 +929,21 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config)
 	nvmem->dev.type = &nvmem_provider_type;
 	nvmem->dev.bus = &nvmem_bus_type;
 	nvmem->dev.parent = config->dev;
+	INIT_LIST_HEAD(&nvmem->cells);
+
+	/*
+	 * Must happen before we assign the release() callback in
+	 * device_initialize().
+	 */
+	rval = init_srcu_struct(&nvmem->srcu);
+	if (rval) {
+		ida_free(&nvmem_ida, nvmem->id);
+		kfree(ops);
+		kfree(nvmem);
+		return ERR_PTR(rval);
+	}
+
+	rcu_assign_pointer(nvmem->ops, ops);
 
 	device_initialize(&nvmem->dev);
 
@@ -935,10 +956,11 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config)
 		goto err_put_device;
 	}
 
-	kref_init(&nvmem->refcnt);
-	INIT_LIST_HEAD(&nvmem->cells);
 	nvmem->fixup_dt_cell_info = config->fixup_dt_cell_info;
 
+	ops->reg_read = config->reg_read;
+	ops->reg_write = config->reg_write;
+
 	nvmem->owner = config->owner;
 	if (!nvmem->owner && config->dev->driver)
 		nvmem->owner = config->dev->driver->owner;
@@ -948,8 +970,6 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config)
 	nvmem->root_only = config->root_only;
 	nvmem->priv = config->priv;
 	nvmem->type = config->type;
-	nvmem->reg_read = config->reg_read;
-	nvmem->reg_write = config->reg_write;
 	nvmem->keepout = config->keepout;
 	nvmem->nkeepout = config->nkeepout;
 	if (config->of_node)
@@ -975,7 +995,7 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config)
 		goto err_put_device;
 
 	nvmem->read_only = device_property_present(config->dev, "read-only") ||
-			   config->read_only || !nvmem->reg_write;
+			   config->read_only || !ops->reg_write;
 
 #ifdef CONFIG_NVMEM_SYSFS
 	nvmem->dev.groups = nvmem_dev_groups;
@@ -996,29 +1016,29 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config)
 	if (config->cells) {
 		rval = nvmem_add_cells(nvmem, config->cells, config->ncells);
 		if (rval)
-			goto err_remove_cells;
+			goto err_remove_compat;
 	}
 
 	if (config->add_legacy_fixed_of_cells) {
 		rval = nvmem_add_cells_from_legacy_of(nvmem);
 		if (rval)
-			goto err_remove_cells;
+			goto err_remove_compat;
 	}
 
-	rval = nvmem_add_cells_from_fixed_layout(nvmem);
-	if (rval)
-		goto err_remove_cells;
-
 	dev_dbg(&nvmem->dev, "Registering nvmem device %s\n", config->name);
 
 	rval = device_add(&nvmem->dev);
 	if (rval)
-		goto err_remove_cells;
+		goto err_remove_compat;
 
 	rval = nvmem_populate_layout(nvmem);
 	if (rval)
 		goto err_remove_dev;
 
+	/* If the device has WP GPIO, default to read-only */
+	if (nvmem->wp_gpio)
+		nvmem->read_only = true;
+
 #ifdef CONFIG_NVMEM_SYSFS
 	rval = nvmem_populate_sysfs_cells(nvmem);
 	if (rval)
@@ -1035,42 +1055,40 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config)
 #endif
 err_remove_dev:
 	device_del(&nvmem->dev);
-err_remove_cells:
-	nvmem_device_remove_all_cells(nvmem);
-	if (config->compat)
-		nvmem_sysfs_remove_compat(nvmem, config);
+err_remove_compat:
+	nvmem_sysfs_remove_compat(nvmem);
 err_put_device:
+	ops = rcu_replace_pointer(nvmem->ops, NULL, true);
+	synchronize_srcu(&nvmem->srcu);
 	put_device(&nvmem->dev);
+	kfree(ops);
 
 	return ERR_PTR(rval);
 }
 EXPORT_SYMBOL_GPL(nvmem_register);
 
-static void nvmem_device_release(struct kref *kref)
+/**
+ * nvmem_unregister() - Unregister previously registered nvmem device
+ *
+ * @nvmem: Pointer to previously registered nvmem device.
+ */
+void nvmem_unregister(struct nvmem_device *nvmem)
 {
-	struct nvmem_device *nvmem;
+	struct nvmem_operations *ops;
 
-	nvmem = container_of(kref, struct nvmem_device, refcnt);
+	if (!nvmem)
+		return;
 
 	blocking_notifier_call_chain(&nvmem_notifier, NVMEM_REMOVE, nvmem);
 
-	if (nvmem->flags & FLAG_COMPAT)
-		device_remove_bin_file(nvmem->base_dev, &nvmem->eeprom);
+	ops = rcu_replace_pointer(nvmem->ops, NULL, true);
+	synchronize_srcu(&nvmem->srcu);
 
-	nvmem_device_remove_all_cells(nvmem);
+	nvmem_sysfs_remove_compat(nvmem);
 	nvmem_destroy_layout(nvmem);
-	device_unregister(&nvmem->dev);
-}
+	kfree(ops);
 
-/**
- * nvmem_unregister() - Unregister previously registered nvmem device
- *
- * @nvmem: Pointer to previously registered nvmem device.
- */
-void nvmem_unregister(struct nvmem_device *nvmem)
-{
-	if (nvmem)
-		kref_put(&nvmem->refcnt, nvmem_device_release);
+	device_unregister(&nvmem->dev);
 }
 EXPORT_SYMBOL_GPL(nvmem_unregister);
 
@@ -1108,17 +1126,17 @@ struct nvmem_device *devm_nvmem_register(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(devm_nvmem_register);
 
-static struct nvmem_device *__nvmem_device_get(void *data,
+static struct nvmem_device *nvmem_device_match(void *data,
 			int (*match)(struct device *dev, const void *data))
 {
 	struct nvmem_device *nvmem = NULL;
 	struct device *dev;
 
-	mutex_lock(&nvmem_mutex);
-	dev = bus_find_device(&nvmem_bus_type, NULL, data, match);
-	if (dev)
-		nvmem = to_nvmem_device(dev);
-	mutex_unlock(&nvmem_mutex);
+	scoped_guard(mutex, &nvmem_mutex) {
+		dev = bus_find_device(&nvmem_bus_type, NULL, data, match);
+		if (dev)
+			nvmem = to_nvmem_device(dev);
+	}
 	if (!nvmem)
 		return ERR_PTR(-EPROBE_DEFER);
 
@@ -1131,18 +1149,9 @@ static struct nvmem_device *__nvmem_device_get(void *data,
 		return ERR_PTR(-EINVAL);
 	}
 
-	kref_get(&nvmem->refcnt);
-
 	return nvmem;
 }
 
-static void __nvmem_device_put(struct nvmem_device *nvmem)
-{
-	put_device(&nvmem->dev);
-	module_put(nvmem->owner);
-	kref_put(&nvmem->refcnt, nvmem_device_release);
-}
-
 #if IS_ENABLED(CONFIG_OF)
 /**
  * of_nvmem_device_get() - Get nvmem device from a given id
@@ -1167,7 +1176,7 @@ struct nvmem_device *of_nvmem_device_get(struct device_node *np, const char *id)
 	if (!nvmem_np)
 		return ERR_PTR(-ENOENT);
 
-	nvmem = __nvmem_device_get(nvmem_np, device_match_of_node);
+	nvmem = nvmem_device_match(nvmem_np, device_match_of_node);
 	of_node_put(nvmem_np);
 	return nvmem;
 }
@@ -1195,7 +1204,7 @@ struct nvmem_device *nvmem_device_get(struct device *dev, const char *dev_name)
 
 	}
 
-	return __nvmem_device_get((void *)dev_name, device_match_name);
+	return nvmem_device_match((void *)dev_name, device_match_name);
 }
 EXPORT_SYMBOL_GPL(nvmem_device_get);
 
@@ -1211,7 +1220,7 @@ EXPORT_SYMBOL_GPL(nvmem_device_get);
 struct nvmem_device *nvmem_device_find(void *data,
 			int (*match)(struct device *dev, const void *data))
 {
-	return __nvmem_device_get(data, match);
+	return nvmem_device_match(data, match);
 }
 EXPORT_SYMBOL_GPL(nvmem_device_find);
 
@@ -1255,7 +1264,8 @@ EXPORT_SYMBOL_GPL(devm_nvmem_device_put);
  */
 void nvmem_device_put(struct nvmem_device *nvmem)
 {
-	__nvmem_device_put(nvmem);
+	module_put(nvmem->owner);
+	put_device(&nvmem->dev);
 }
 EXPORT_SYMBOL_GPL(nvmem_device_put);
 
@@ -1328,35 +1338,32 @@ nvmem_cell_get_from_lookup(struct device *dev, const char *con_id)
 
 	dev_id = dev_name(dev);
 
-	mutex_lock(&nvmem_lookup_mutex);
+	guard(mutex)(&nvmem_lookup_mutex);
 
 	list_for_each_entry(lookup, &nvmem_lookup_list, node) {
 		if ((strcmp(lookup->dev_id, dev_id) == 0) &&
 		    (strcmp(lookup->con_id, con_id) == 0)) {
 			/* This is the right entry. */
-			nvmem = __nvmem_device_get((void *)lookup->nvmem_name,
+			nvmem = nvmem_device_match((void *)lookup->nvmem_name,
 						   device_match_name);
-			if (IS_ERR(nvmem)) {
+			if (IS_ERR(nvmem))
 				/* Provider may not be registered yet. */
-				cell = ERR_CAST(nvmem);
-				break;
-			}
+				return ERR_CAST(nvmem);
 
 			cell_entry = nvmem_find_cell_entry_by_name(nvmem,
 								   lookup->cell_name);
 			if (!cell_entry) {
-				__nvmem_device_put(nvmem);
+				nvmem_device_put(nvmem);
 				cell = ERR_PTR(-ENOENT);
 			} else {
 				cell = nvmem_create_cell(cell_entry, con_id, 0);
 				if (IS_ERR(cell))
-					__nvmem_device_put(nvmem);
+					nvmem_device_put(nvmem);
 			}
 			break;
 		}
 	}
 
-	mutex_unlock(&nvmem_lookup_mutex);
 	return cell;
 }
 
@@ -1370,18 +1377,16 @@ static void nvmem_layout_module_put(struct nvmem_device *nvmem)
 static struct nvmem_cell_entry *
 nvmem_find_cell_entry_by_node(struct nvmem_device *nvmem, struct device_node *np)
 {
-	struct nvmem_cell_entry *iter, *cell = NULL;
+	struct nvmem_cell_entry *cell;
 
-	mutex_lock(&nvmem_mutex);
-	list_for_each_entry(iter, &nvmem->cells, node) {
-		if (np == iter->np) {
-			cell = iter;
-			break;
-		}
+	guard(mutex)(&nvmem_mutex);
+
+	list_for_each_entry(cell, &nvmem->cells, node) {
+		if (np == cell->np)
+			return cell;
 	}
-	mutex_unlock(&nvmem_mutex);
 
-	return cell;
+	return NULL;
 }
 
 static int nvmem_layout_module_get_optional(struct nvmem_device *nvmem)
@@ -1451,7 +1456,7 @@ struct nvmem_cell *of_nvmem_cell_get(struct device_node *np, const char *id)
 		}
 	}
 
-	nvmem = __nvmem_device_get(nvmem_np, device_match_of_node);
+	nvmem = nvmem_device_match(nvmem_np, device_match_of_node);
 	of_node_put(nvmem_np);
 	if (IS_ERR(nvmem)) {
 		of_node_put(cell_np);
@@ -1461,7 +1466,7 @@ struct nvmem_cell *of_nvmem_cell_get(struct device_node *np, const char *id)
 	ret = nvmem_layout_module_get_optional(nvmem);
 	if (ret) {
 		of_node_put(cell_np);
-		__nvmem_device_put(nvmem);
+		nvmem_device_put(nvmem);
 		return ERR_PTR(ret);
 	}
 
@@ -1470,14 +1475,14 @@ struct nvmem_cell *of_nvmem_cell_get(struct device_node *np, const char *id)
 	if (!cell_entry) {
 		nvmem_layout_module_put(nvmem);
 		ret = nvmem->layout ? -EPROBE_DEFER : -ENOENT;
-		__nvmem_device_put(nvmem);
+		nvmem_device_put(nvmem);
 		return ERR_PTR(ret);
 	}
 
 	cell = nvmem_create_cell(cell_entry, id, cell_index);
 	if (IS_ERR(cell)) {
 		nvmem_layout_module_put(nvmem);
-		__nvmem_device_put(nvmem);
+		nvmem_device_put(nvmem);
 	}
 
 	return cell;
@@ -1592,7 +1597,7 @@ void nvmem_cell_put(struct nvmem_cell *cell)
 
 	kfree(cell);
 	nvmem_layout_module_put(nvmem);
-	__nvmem_device_put(nvmem);
+	nvmem_device_put(nvmem);
 }
 EXPORT_SYMBOL_GPL(nvmem_cell_put);
 
@@ -2114,10 +2119,10 @@ void nvmem_add_cell_lookups(struct nvmem_cell_lookup *entries, size_t nentries)
 {
 	int i;
 
-	mutex_lock(&nvmem_lookup_mutex);
+	guard(mutex)(&nvmem_lookup_mutex);
+
 	for (i = 0; i < nentries; i++)
 		list_add_tail(&entries[i].node, &nvmem_lookup_list);
-	mutex_unlock(&nvmem_lookup_mutex);
 }
 EXPORT_SYMBOL_GPL(nvmem_add_cell_lookups);
 
@@ -2132,10 +2137,10 @@ void nvmem_del_cell_lookups(struct nvmem_cell_lookup *entries, size_t nentries)
 {
 	int i;
 
-	mutex_lock(&nvmem_lookup_mutex);
+	guard(mutex)(&nvmem_lookup_mutex);
+
 	for (i = 0; i < nentries; i++)
 		list_del(&entries[i].node);
-	mutex_unlock(&nvmem_lookup_mutex);
 }
 EXPORT_SYMBOL_GPL(nvmem_del_cell_lookups);
 
diff --git a/drivers/nvmem/ee1004.c b/drivers/nvmem/ee1004.c
new file mode 100644
index 0000000000000..923f404a44c03
--- /dev/null
+++ b/drivers/nvmem/ee1004.c
@@ -0,0 +1,351 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * ee1004 - driver for DDR4 SPD EEPROMs
+ *
+ * Copyright (C) 2017-2019 Jean Delvare
+ *
+ * Based on the at24 driver:
+ * Copyright (C) 2005-2007 David Brownell
+ * Copyright (C) 2008 Wolfram Sang, Pengutronix
+ */
+
+#include <linux/device.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/nvmem-provider.h>
+
+/*
+ * DDR4 memory modules use special EEPROMs following the Jedec EE1004
+ * specification. These are 512-byte EEPROMs using a single I2C address
+ * in the 0x50-0x57 range for data. One of two 256-byte page is selected
+ * by writing a command to I2C address 0x36 or 0x37 on the same I2C bus.
+ *
+ * Therefore we need to request these 2 additional addresses, and serialize
+ * access to all such EEPROMs with a single mutex.
+ *
+ * We assume it is safe to read up to 32 bytes at once from these EEPROMs.
+ * We use SMBus access even if I2C is available, these EEPROMs are small
+ * enough, and reading from them infrequent enough, that we favor simplicity
+ * over performance.
+ */
+
+#define EE1004_MAX_BUSSES		8
+#define EE1004_ADDR_SET_PAGE		0x36
+#define EE1004_NUM_PAGES		2
+#define EE1004_PAGE_SIZE		256
+#define EE1004_PAGE_SHIFT		8
+#define EE1004_EEPROM_SIZE		(EE1004_PAGE_SIZE * EE1004_NUM_PAGES)
+
+/*
+ * Mutex protects ee1004_set_page and ee1004_dev_count, and must be held
+ * from page selection to end of read.
+ */
+static DEFINE_MUTEX(ee1004_bus_lock);
+
+static struct ee1004_bus_data {
+	struct i2c_adapter *adap;
+	struct i2c_client *set_page[EE1004_NUM_PAGES];
+	unsigned int dev_count;
+	int current_page;
+} ee1004_bus_data[EE1004_MAX_BUSSES];
+
+static const struct i2c_device_id ee1004_ids[] = {
+	{ "ee1004" },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, ee1004_ids);
+
+/*-------------------------------------------------------------------------*/
+
+static struct ee1004_bus_data *ee1004_get_bus_data(struct i2c_adapter *adap)
+{
+	int i;
+
+	for (i = 0; i < EE1004_MAX_BUSSES; i++)
+		if (ee1004_bus_data[i].adap == adap)
+			return ee1004_bus_data + i;
+
+	/* If not existent yet, create new entry */
+	for (i = 0; i < EE1004_MAX_BUSSES; i++)
+		if (!ee1004_bus_data[i].adap) {
+			ee1004_bus_data[i].adap = adap;
+			return ee1004_bus_data + i;
+		}
+
+	return NULL;
+}
+
+static int ee1004_get_current_page(struct ee1004_bus_data *bd)
+{
+	int err;
+
+	err = i2c_smbus_read_byte(bd->set_page[0]);
+	if (err == -ENXIO) {
+		/* Nack means page 1 is selected */
+		return 1;
+	}
+	if (err < 0) {
+		/* Anything else is a real error, bail out */
+		return err;
+	}
+
+	/* Ack means page 0 is selected, returned value meaningless */
+	return 0;
+}
+
+static int ee1004_set_current_page(struct i2c_client *client, int page)
+{
+	struct ee1004_bus_data *bd = i2c_get_clientdata(client);
+	int ret;
+
+	if (page == bd->current_page)
+		return 0;
+
+	/* Data is ignored */
+	ret = i2c_smbus_write_byte(bd->set_page[page], 0x00);
+	/*
+	 * Don't give up just yet. Some memory modules will select the page
+	 * but not ack the command. Check which page is selected now.
+	 */
+	if (ret == -ENXIO && ee1004_get_current_page(bd) == page)
+		ret = 0;
+	if (ret < 0) {
+		dev_err(&client->dev, "Failed to select page %d (%d)\n", page, ret);
+		return ret;
+	}
+
+	dev_dbg(&client->dev, "Selected page %d\n", page);
+	bd->current_page = page;
+
+	return 0;
+}
+
+static ssize_t ee1004_eeprom_read(struct i2c_client *client, char *buf,
+				  unsigned int offset, size_t count)
+{
+	int status, page;
+
+	page = offset >> EE1004_PAGE_SHIFT;
+	offset &= (1 << EE1004_PAGE_SHIFT) - 1;
+
+	status = ee1004_set_current_page(client, page);
+	if (status)
+		return status;
+
+	/* Can't cross page boundaries */
+	if (offset + count > EE1004_PAGE_SIZE)
+		count = EE1004_PAGE_SIZE - offset;
+
+	if (count > I2C_SMBUS_BLOCK_MAX)
+		count = I2C_SMBUS_BLOCK_MAX;
+
+	return i2c_smbus_read_i2c_block_data_or_emulated(client, offset, count, buf);
+}
+
+static int ee1004_read(void *priv, unsigned int off, void *val, size_t count)
+{
+	struct i2c_client *client = priv;
+	char *buf = val;
+	int ret;
+
+	if (unlikely(!count))
+		return count;
+
+	if (off + count > EE1004_EEPROM_SIZE)
+		return -EINVAL;
+
+	/*
+	 * Read data from chip, protecting against concurrent access to
+	 * other EE1004 SPD EEPROMs on the same adapter.
+	 */
+	mutex_lock(&ee1004_bus_lock);
+
+	while (count) {
+		ret = ee1004_eeprom_read(client, buf, off, count);
+		if (ret < 0) {
+			mutex_unlock(&ee1004_bus_lock);
+			return ret;
+		}
+
+		buf += ret;
+		off += ret;
+		count -= ret;
+	}
+
+	mutex_unlock(&ee1004_bus_lock);
+
+	return 0;
+}
+
+static void ee1004_probe_temp_sensor(struct i2c_client *client)
+{
+	struct i2c_board_info info = { .type = "jc42" };
+	unsigned short addr = 0x18 | (client->addr & 7);
+	unsigned short addr_list[] = { addr, I2C_CLIENT_END };
+	u8 data[2];
+	int ret;
+
+	/* byte 14, bit 7 is set if temp sensor is present */
+	ret = ee1004_eeprom_read(client, data, 14, 1);
+	if (ret != 1)
+		return;
+
+	if (!(data[0] & BIT(7))) {
+		/*
+		 * If the SPD data suggests that there is no temperature
+		 * sensor, it may still be there for SPD revision 1.0.
+		 * See SPD Annex L, Revision 1 and 2, for details.
+		 * Check DIMM type and SPD revision; if it is a DDR4
+		 * with SPD revision 1.0, check the thermal sensor address
+		 * and instantiate the jc42 driver if a chip is found at
+		 * that address.
+		 * It is not necessary to check if there is a chip at the
+		 * temperature sensor address since i2c_new_scanned_device()
+		 * will do that and return silently if no chip is found.
+		 */
+		ret = ee1004_eeprom_read(client, data, 1, 2);
+		if (ret != 2 || data[0] != 0x10 || data[1] != 0x0c)
+			return;
+	}
+	i2c_new_scanned_device(client->adapter, &info, addr_list, NULL);
+}
+
+static void ee1004_cleanup(int idx, struct ee1004_bus_data *bd)
+{
+	if (--bd->dev_count == 0) {
+		while (--idx >= 0)
+			i2c_unregister_device(bd->set_page[idx]);
+		memset(bd, 0, sizeof(struct ee1004_bus_data));
+	}
+}
+
+static void ee1004_cleanup_bus_data(void *data)
+{
+	struct ee1004_bus_data *bd = data;
+
+	/* Remove page select clients if this is the last device */
+	mutex_lock(&ee1004_bus_lock);
+	ee1004_cleanup(EE1004_NUM_PAGES, bd);
+	mutex_unlock(&ee1004_bus_lock);
+}
+
+static int ee1004_init_bus_data(struct i2c_client *client)
+{
+	struct ee1004_bus_data *bd;
+	int err, cnr = 0;
+
+	bd = ee1004_get_bus_data(client->adapter);
+	if (!bd)
+		return dev_err_probe(&client->dev, -ENOSPC, "Only %d busses supported",
+				     EE1004_MAX_BUSSES);
+
+	i2c_set_clientdata(client, bd);
+
+	if (++bd->dev_count == 1) {
+		/* Use 2 dummy devices for page select command */
+		for (cnr = 0; cnr < EE1004_NUM_PAGES; cnr++) {
+			struct i2c_client *cl;
+
+			cl = i2c_new_dummy_device(client->adapter, EE1004_ADDR_SET_PAGE + cnr);
+			if (IS_ERR(cl)) {
+				err = PTR_ERR(cl);
+				goto err_out;
+			}
+
+			bd->set_page[cnr] = cl;
+		}
+
+		/* Remember current page to avoid unneeded page select */
+		err = ee1004_get_current_page(bd);
+		if (err < 0)
+			goto err_out;
+
+		dev_dbg(&client->dev, "Currently selected page: %d\n", err);
+		bd->current_page = err;
+	}
+
+	return 0;
+
+err_out:
+	ee1004_cleanup(cnr, bd);
+
+	return err;
+}
+
+static int ee1004_probe(struct i2c_client *client)
+{
+	struct nvmem_config config = {
+		.dev = &client->dev,
+		.name = dev_name(&client->dev),
+		.id = NVMEM_DEVID_NONE,
+		.owner = THIS_MODULE,
+		.type = NVMEM_TYPE_EEPROM,
+		.read_only = true,
+		.root_only = false,
+		.reg_read = ee1004_read,
+		.size = EE1004_EEPROM_SIZE,
+		.word_size = 1,
+		.stride = 1,
+		.priv = client,
+		.compat = true,
+		.base_dev = &client->dev,
+	};
+	struct nvmem_device *ndev;
+	int err;
+
+	/* Make sure we can operate on this adapter */
+	if (!i2c_check_functionality(client->adapter,
+				     I2C_FUNC_SMBUS_BYTE | I2C_FUNC_SMBUS_READ_I2C_BLOCK) &&
+	    !i2c_check_functionality(client->adapter,
+				     I2C_FUNC_SMBUS_BYTE | I2C_FUNC_SMBUS_READ_BYTE_DATA))
+		return -EPFNOSUPPORT;
+
+	err = i2c_smbus_read_byte(client);
+	if (err < 0)
+		return -ENODEV;
+
+	mutex_lock(&ee1004_bus_lock);
+
+	err = ee1004_init_bus_data(client);
+	if (err < 0) {
+		mutex_unlock(&ee1004_bus_lock);
+		return err;
+	}
+
+	ee1004_probe_temp_sensor(client);
+
+	mutex_unlock(&ee1004_bus_lock);
+
+	err = devm_add_action_or_reset(&client->dev, ee1004_cleanup_bus_data,
+				       i2c_get_clientdata(client));
+	if (err < 0)
+		return err;
+
+	ndev = devm_nvmem_register(&client->dev, &config);
+	if (IS_ERR(ndev))
+		return PTR_ERR(ndev);
+
+	dev_info(&client->dev,
+		 "%u byte EE1004-compliant SPD EEPROM, read-only\n",
+		 EE1004_EEPROM_SIZE);
+
+	return 0;
+}
+
+/*-------------------------------------------------------------------------*/
+
+static struct i2c_driver ee1004_driver = {
+	.driver = {
+		.name = "ee1004",
+	},
+	.probe = ee1004_probe,
+	.id_table = ee1004_ids,
+};
+module_i2c_driver(ee1004_driver);
+
+MODULE_DESCRIPTION("Driver for EE1004-compliant DDR4 SPD EEPROMs");
+MODULE_AUTHOR("Jean Delvare");
+MODULE_LICENSE("GPL");
diff --git a/drivers/nvmem/eeprom_93xx46.c b/drivers/nvmem/eeprom_93xx46.c
new file mode 100644
index 0000000000000..f9c3ab52c2f92
--- /dev/null
+++ b/drivers/nvmem/eeprom_93xx46.c
@@ -0,0 +1,560 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Driver for 93xx46 EEPROMs
+ *
+ * (C) 2011 DENX Software Engineering, Anatolij Gustschin <agust@denx.de>
+ */
+
+#include <linux/array_size.h>
+#include <linux/bits.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/gpio/consumer.h>
+#include <linux/kstrtox.h>
+#include <linux/log2.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/property.h>
+#include <linux/slab.h>
+#include <linux/spi/spi.h>
+#include <linux/string_choices.h>
+
+#include <linux/nvmem-provider.h>
+
+struct eeprom_93xx46_platform_data {
+	unsigned char	flags;
+#define EE_ADDR8	0x01		/*  8 bit addr. cfg */
+#define EE_ADDR16	0x02		/* 16 bit addr. cfg */
+#define EE_READONLY	0x08		/* forbid writing */
+#define EE_SIZE1K	0x10		/* 1 kb of data, that is a 93xx46 */
+#define EE_SIZE2K	0x20		/* 2 kb of data, that is a 93xx56 */
+#define EE_SIZE4K	0x40		/* 4 kb of data, that is a 93xx66 */
+
+	unsigned int	quirks;
+/* Single word read transfers only; no sequential read. */
+#define EEPROM_93XX46_QUIRK_SINGLE_WORD_READ		(1 << 0)
+/* Instructions such as EWEN are (addrlen + 2) in length. */
+#define EEPROM_93XX46_QUIRK_INSTRUCTION_LENGTH		(1 << 1)
+/* Add extra cycle after address during a read */
+#define EEPROM_93XX46_QUIRK_EXTRA_READ_CYCLE		BIT(2)
+
+	struct gpio_desc *select;
+};
+
+#define OP_START	0x4
+#define OP_WRITE	(OP_START | 0x1)
+#define OP_READ		(OP_START | 0x2)
+/* The following addresses are offset for the 1K EEPROM variant in 16-bit mode */
+#define ADDR_EWDS	0x00
+#define ADDR_ERAL	0x20
+#define ADDR_EWEN	0x30
+
+struct eeprom_93xx46_devtype_data {
+	unsigned int quirks;
+	unsigned char flags;
+};
+
+static const struct eeprom_93xx46_devtype_data at93c46_data = {
+	.flags = EE_SIZE1K,
+};
+
+static const struct eeprom_93xx46_devtype_data at93c56_data = {
+	.flags = EE_SIZE2K,
+};
+
+static const struct eeprom_93xx46_devtype_data at93c66_data = {
+	.flags = EE_SIZE4K,
+};
+
+static const struct eeprom_93xx46_devtype_data atmel_at93c46d_data = {
+	.flags = EE_SIZE1K,
+	.quirks = EEPROM_93XX46_QUIRK_SINGLE_WORD_READ |
+		  EEPROM_93XX46_QUIRK_INSTRUCTION_LENGTH,
+};
+
+static const struct eeprom_93xx46_devtype_data microchip_93lc46b_data = {
+	.flags = EE_SIZE1K,
+	.quirks = EEPROM_93XX46_QUIRK_EXTRA_READ_CYCLE,
+};
+
+struct eeprom_93xx46_dev {
+	struct spi_device *spi;
+	struct eeprom_93xx46_platform_data *pdata;
+	struct mutex lock;
+	struct nvmem_config nvmem_config;
+	struct nvmem_device *nvmem;
+	int addrlen;
+	int size;
+};
+
+static inline bool has_quirk_single_word_read(struct eeprom_93xx46_dev *edev)
+{
+	return edev->pdata->quirks & EEPROM_93XX46_QUIRK_SINGLE_WORD_READ;
+}
+
+static inline bool has_quirk_instruction_length(struct eeprom_93xx46_dev *edev)
+{
+	return edev->pdata->quirks & EEPROM_93XX46_QUIRK_INSTRUCTION_LENGTH;
+}
+
+static inline bool has_quirk_extra_read_cycle(struct eeprom_93xx46_dev *edev)
+{
+	return edev->pdata->quirks & EEPROM_93XX46_QUIRK_EXTRA_READ_CYCLE;
+}
+
+static int eeprom_93xx46_read(void *priv, unsigned int off,
+			      void *val, size_t count)
+{
+	struct eeprom_93xx46_dev *edev = priv;
+	char *buf = val;
+	int err = 0;
+	int bits;
+
+	if (unlikely(off >= edev->size))
+		return 0;
+	if ((off + count) > edev->size)
+		count = edev->size - off;
+	if (unlikely(!count))
+		return count;
+
+	mutex_lock(&edev->lock);
+
+	gpiod_set_value_cansleep(edev->pdata->select, 1);
+
+	/* The opcode in front of the address is three bits. */
+	bits = edev->addrlen + 3;
+
+	while (count) {
+		struct spi_message m;
+		struct spi_transfer t[2] = {};
+		u16 cmd_addr = OP_READ << edev->addrlen;
+		size_t nbytes = count;
+
+		if (edev->pdata->flags & EE_ADDR8) {
+			cmd_addr |= off;
+			if (has_quirk_single_word_read(edev))
+				nbytes = 1;
+		} else {
+			cmd_addr |= (off >> 1);
+			if (has_quirk_single_word_read(edev))
+				nbytes = 2;
+		}
+
+		dev_dbg(&edev->spi->dev, "read cmd 0x%x, %d Hz\n",
+			cmd_addr, edev->spi->max_speed_hz);
+
+		if (has_quirk_extra_read_cycle(edev)) {
+			cmd_addr <<= 1;
+			bits += 1;
+		}
+
+		t[0].tx_buf = (char *)&cmd_addr;
+		t[0].len = 2;
+		t[0].bits_per_word = bits;
+
+		t[1].rx_buf = buf;
+		t[1].len = count;
+		t[1].bits_per_word = 8;
+
+		spi_message_init_with_transfers(&m, t, ARRAY_SIZE(t));
+
+		err = spi_sync(edev->spi, &m);
+		/* have to wait at least Tcsl ns */
+		ndelay(250);
+
+		if (err) {
+			dev_err(&edev->spi->dev, "read %zu bytes at %u: err. %d\n",
+				nbytes, off, err);
+			break;
+		}
+
+		buf += nbytes;
+		off += nbytes;
+		count -= nbytes;
+	}
+
+	gpiod_set_value_cansleep(edev->pdata->select, 0);
+
+	mutex_unlock(&edev->lock);
+
+	return err;
+}
+
+static int eeprom_93xx46_ew(struct eeprom_93xx46_dev *edev, int is_on)
+{
+	struct spi_message m;
+	struct spi_transfer t = {};
+	int bits, ret;
+	u16 cmd_addr;
+
+	/* The opcode in front of the address is three bits. */
+	bits = edev->addrlen + 3;
+
+	cmd_addr = OP_START << edev->addrlen;
+	cmd_addr |= (is_on ? ADDR_EWEN : ADDR_EWDS) << (edev->addrlen - 6);
+
+	if (has_quirk_instruction_length(edev)) {
+		cmd_addr <<= 2;
+		bits += 2;
+	}
+
+	dev_dbg(&edev->spi->dev, "ew %s cmd 0x%04x, %d bits\n",
+		str_enable_disable(is_on), cmd_addr, bits);
+
+	t.tx_buf = &cmd_addr;
+	t.len = 2;
+	t.bits_per_word = bits;
+
+	spi_message_init_with_transfers(&m, &t, 1);
+
+	mutex_lock(&edev->lock);
+
+	gpiod_set_value_cansleep(edev->pdata->select, 1);
+
+	ret = spi_sync(edev->spi, &m);
+	/* have to wait at least Tcsl ns */
+	ndelay(250);
+	if (ret)
+		dev_err(&edev->spi->dev, "erase/write %s error %d\n",
+			str_enable_disable(is_on), ret);
+
+	gpiod_set_value_cansleep(edev->pdata->select, 0);
+
+	mutex_unlock(&edev->lock);
+	return ret;
+}
+
+static ssize_t
+eeprom_93xx46_write_word(struct eeprom_93xx46_dev *edev,
+			 const char *buf, unsigned int off)
+{
+	struct spi_message m;
+	struct spi_transfer t[2] = {};
+	int bits, data_len, ret;
+	u16 cmd_addr;
+
+	if (unlikely(off >= edev->size))
+		return -EINVAL;
+
+	/* The opcode in front of the address is three bits. */
+	bits = edev->addrlen + 3;
+
+	cmd_addr = OP_WRITE << edev->addrlen;
+
+	if (edev->pdata->flags & EE_ADDR8) {
+		cmd_addr |= off;
+		data_len = 1;
+	} else {
+		cmd_addr |= (off >> 1);
+		data_len = 2;
+	}
+
+	dev_dbg(&edev->spi->dev, "write cmd 0x%x\n", cmd_addr);
+
+	t[0].tx_buf = (char *)&cmd_addr;
+	t[0].len = 2;
+	t[0].bits_per_word = bits;
+
+	t[1].tx_buf = buf;
+	t[1].len = data_len;
+	t[1].bits_per_word = 8;
+
+	spi_message_init_with_transfers(&m, t, ARRAY_SIZE(t));
+
+	ret = spi_sync(edev->spi, &m);
+	/* have to wait program cycle time Twc ms */
+	mdelay(6);
+	return ret;
+}
+
+static int eeprom_93xx46_write(void *priv, unsigned int off,
+				   void *val, size_t count)
+{
+	struct eeprom_93xx46_dev *edev = priv;
+	char *buf = val;
+	int ret, step = 1;
+	unsigned int i;
+
+	if (unlikely(off >= edev->size))
+		return -EFBIG;
+	if ((off + count) > edev->size)
+		count = edev->size - off;
+	if (unlikely(!count))
+		return count;
+
+	/* only write even number of bytes on 16-bit devices */
+	if (edev->pdata->flags & EE_ADDR16) {
+		step = 2;
+		count &= ~1;
+	}
+
+	/* erase/write enable */
+	ret = eeprom_93xx46_ew(edev, 1);
+	if (ret)
+		return ret;
+
+	mutex_lock(&edev->lock);
+
+	gpiod_set_value_cansleep(edev->pdata->select, 1);
+
+	for (i = 0; i < count; i += step) {
+		ret = eeprom_93xx46_write_word(edev, &buf[i], off + i);
+		if (ret) {
+			dev_err(&edev->spi->dev, "write failed at %u: %d\n", off + i, ret);
+			break;
+		}
+	}
+
+	gpiod_set_value_cansleep(edev->pdata->select, 0);
+
+	mutex_unlock(&edev->lock);
+
+	/* erase/write disable */
+	eeprom_93xx46_ew(edev, 0);
+	return ret;
+}
+
+static int eeprom_93xx46_eral(struct eeprom_93xx46_dev *edev)
+{
+	struct spi_message m;
+	struct spi_transfer t = {};
+	int bits, ret;
+	u16 cmd_addr;
+
+	/* The opcode in front of the address is three bits. */
+	bits = edev->addrlen + 3;
+
+	cmd_addr = OP_START << edev->addrlen;
+	cmd_addr |= ADDR_ERAL << (edev->addrlen - 6);
+
+	if (has_quirk_instruction_length(edev)) {
+		cmd_addr <<= 2;
+		bits += 2;
+	}
+
+	dev_dbg(&edev->spi->dev, "eral cmd 0x%04x, %d bits\n", cmd_addr, bits);
+
+	t.tx_buf = &cmd_addr;
+	t.len = 2;
+	t.bits_per_word = bits;
+
+	spi_message_init_with_transfers(&m, &t, 1);
+
+	mutex_lock(&edev->lock);
+
+	gpiod_set_value_cansleep(edev->pdata->select, 1);
+
+	ret = spi_sync(edev->spi, &m);
+	if (ret)
+		dev_err(&edev->spi->dev, "erase error %d\n", ret);
+	/* have to wait erase cycle time Tec ms */
+	mdelay(6);
+
+	gpiod_set_value_cansleep(edev->pdata->select, 0);
+
+	mutex_unlock(&edev->lock);
+	return ret;
+}
+
+static ssize_t erase_store(struct device *dev, struct device_attribute *attr,
+			   const char *buf, size_t count)
+{
+	struct eeprom_93xx46_dev *edev = dev_get_drvdata(dev);
+	bool erase;
+	int ret;
+
+	ret = kstrtobool(buf, &erase);
+	if (ret)
+		return ret;
+
+	if (erase) {
+		ret = eeprom_93xx46_ew(edev, 1);
+		if (ret)
+			return ret;
+		ret = eeprom_93xx46_eral(edev);
+		if (ret)
+			return ret;
+		ret = eeprom_93xx46_ew(edev, 0);
+		if (ret)
+			return ret;
+	}
+	return count;
+}
+static DEVICE_ATTR_WO(erase);
+
+static const struct of_device_id eeprom_93xx46_of_table[] = {
+	{ .compatible = "eeprom-93xx46", .data = &at93c46_data, },
+	{ .compatible = "atmel,at93c46", .data = &at93c46_data, },
+	{ .compatible = "atmel,at93c46d", .data = &atmel_at93c46d_data, },
+	{ .compatible = "atmel,at93c56", .data = &at93c56_data, },
+	{ .compatible = "atmel,at93c66", .data = &at93c66_data, },
+	{ .compatible = "microchip,93lc46b", .data = &microchip_93lc46b_data, },
+	{}
+};
+MODULE_DEVICE_TABLE(of, eeprom_93xx46_of_table);
+
+static const struct spi_device_id eeprom_93xx46_spi_ids[] = {
+	{ .name = "eeprom-93xx46",
+	  .driver_data = (kernel_ulong_t)&at93c46_data, },
+	{ .name = "at93c46",
+	  .driver_data = (kernel_ulong_t)&at93c46_data, },
+	{ .name = "at93c46d",
+	  .driver_data = (kernel_ulong_t)&atmel_at93c46d_data, },
+	{ .name = "at93c56",
+	  .driver_data = (kernel_ulong_t)&at93c56_data, },
+	{ .name = "at93c66",
+	  .driver_data = (kernel_ulong_t)&at93c66_data, },
+	{ .name = "93lc46b",
+	  .driver_data = (kernel_ulong_t)&microchip_93lc46b_data, },
+	{}
+};
+MODULE_DEVICE_TABLE(spi, eeprom_93xx46_spi_ids);
+
+static int eeprom_93xx46_probe_fw(struct device *dev)
+{
+	const struct eeprom_93xx46_devtype_data *data;
+	struct eeprom_93xx46_platform_data *pd;
+	u32 tmp;
+	int ret;
+
+	pd = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL);
+	if (!pd)
+		return -ENOMEM;
+
+	ret = device_property_read_u32(dev, "data-size", &tmp);
+	if (ret < 0) {
+		dev_err(dev, "data-size property not found\n");
+		return ret;
+	}
+
+	if (tmp == 8) {
+		pd->flags |= EE_ADDR8;
+	} else if (tmp == 16) {
+		pd->flags |= EE_ADDR16;
+	} else {
+		dev_err(dev, "invalid data-size (%d)\n", tmp);
+		return -EINVAL;
+	}
+
+	if (device_property_read_bool(dev, "read-only"))
+		pd->flags |= EE_READONLY;
+
+	pd->select = devm_gpiod_get_optional(dev, "select", GPIOD_OUT_LOW);
+	if (IS_ERR(pd->select))
+		return PTR_ERR(pd->select);
+	gpiod_set_consumer_name(pd->select, "93xx46 EEPROMs OE");
+
+	data = spi_get_device_match_data(to_spi_device(dev));
+	if (data) {
+		pd->quirks = data->quirks;
+		pd->flags |= data->flags;
+	}
+
+	dev->platform_data = pd;
+
+	return 0;
+}
+
+static int eeprom_93xx46_probe(struct spi_device *spi)
+{
+	struct eeprom_93xx46_platform_data *pd;
+	struct eeprom_93xx46_dev *edev;
+	struct device *dev = &spi->dev;
+	int err;
+
+	err = eeprom_93xx46_probe_fw(dev);
+	if (err < 0)
+		return err;
+
+	pd = spi->dev.platform_data;
+	if (!pd) {
+		dev_err(&spi->dev, "missing platform data\n");
+		return -ENODEV;
+	}
+
+	edev = devm_kzalloc(&spi->dev, sizeof(*edev), GFP_KERNEL);
+	if (!edev)
+		return -ENOMEM;
+
+	if (pd->flags & EE_SIZE1K)
+		edev->size = 128;
+	else if (pd->flags & EE_SIZE2K)
+		edev->size = 256;
+	else if (pd->flags & EE_SIZE4K)
+		edev->size = 512;
+	else {
+		dev_err(&spi->dev, "unspecified size\n");
+		return -EINVAL;
+	}
+
+	if (pd->flags & EE_ADDR8)
+		edev->addrlen = ilog2(edev->size);
+	else if (pd->flags & EE_ADDR16)
+		edev->addrlen = ilog2(edev->size) - 1;
+	else {
+		dev_err(&spi->dev, "unspecified address type\n");
+		return -EINVAL;
+	}
+
+	mutex_init(&edev->lock);
+
+	edev->spi = spi;
+	edev->pdata = pd;
+
+	edev->nvmem_config.type = NVMEM_TYPE_EEPROM;
+	edev->nvmem_config.name = dev_name(&spi->dev);
+	edev->nvmem_config.dev = &spi->dev;
+	edev->nvmem_config.read_only = pd->flags & EE_READONLY;
+	edev->nvmem_config.root_only = true;
+	edev->nvmem_config.owner = THIS_MODULE;
+	edev->nvmem_config.compat = true;
+	edev->nvmem_config.base_dev = &spi->dev;
+	edev->nvmem_config.reg_read = eeprom_93xx46_read;
+	edev->nvmem_config.reg_write = eeprom_93xx46_write;
+	edev->nvmem_config.priv = edev;
+	edev->nvmem_config.stride = 4;
+	edev->nvmem_config.word_size = 1;
+	edev->nvmem_config.size = edev->size;
+
+	edev->nvmem = devm_nvmem_register(&spi->dev, &edev->nvmem_config);
+	if (IS_ERR(edev->nvmem))
+		return PTR_ERR(edev->nvmem);
+
+	dev_info(&spi->dev, "%d-bit eeprom containing %d bytes %s\n",
+		(pd->flags & EE_ADDR8) ? 8 : 16,
+		edev->size,
+		(pd->flags & EE_READONLY) ? "(readonly)" : "");
+
+	if (!(pd->flags & EE_READONLY)) {
+		if (device_create_file(&spi->dev, &dev_attr_erase))
+			dev_err(&spi->dev, "can't create erase interface\n");
+	}
+
+	spi_set_drvdata(spi, edev);
+	return 0;
+}
+
+static void eeprom_93xx46_remove(struct spi_device *spi)
+{
+	struct eeprom_93xx46_dev *edev = spi_get_drvdata(spi);
+
+	if (!(edev->pdata->flags & EE_READONLY))
+		device_remove_file(&spi->dev, &dev_attr_erase);
+}
+
+static struct spi_driver eeprom_93xx46_driver = {
+	.driver = {
+		.name	= "93xx46",
+		.of_match_table = eeprom_93xx46_of_table,
+	},
+	.probe		= eeprom_93xx46_probe,
+	.remove		= eeprom_93xx46_remove,
+	.id_table	= eeprom_93xx46_spi_ids,
+};
+
+module_spi_driver(eeprom_93xx46_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Driver for 93xx46 EEPROMs");
+MODULE_AUTHOR("Anatolij Gustschin <agust@denx.de>");
+MODULE_ALIAS("spi:93xx46");
diff --git a/drivers/nvmem/internals.h b/drivers/nvmem/internals.h
index 18fed57270e5e..bc7a99f5aefbf 100644
--- a/drivers/nvmem/internals.h
+++ b/drivers/nvmem/internals.h
@@ -6,15 +6,21 @@
 #include <linux/device.h>
 #include <linux/nvmem-consumer.h>
 #include <linux/nvmem-provider.h>
+#include <linux/srcu.h>
+
+/* Hold pointers to callbacks owned by the nvmem provider module. */
+struct nvmem_operations {
+	nvmem_reg_read_t	reg_read;
+	nvmem_reg_write_t	reg_write;
+};
 
 struct nvmem_device {
 	struct module		*owner;
 	struct device		dev;
-	struct list_head	node;
+	struct srcu_struct	srcu;
 	int			stride;
 	int			word_size;
 	int			id;
-	struct kref		refcnt;
 	size_t			size;
 	bool			read_only;
 	bool			root_only;
@@ -27,14 +33,15 @@ struct nvmem_device {
 				   struct nvmem_cell_info *cell);
 	const struct nvmem_keepout *keepout;
 	unsigned int		nkeepout;
-	nvmem_reg_read_t	reg_read;
-	nvmem_reg_write_t	reg_write;
 	struct gpio_desc	*wp_gpio;
 	struct nvmem_layout	*layout;
+	struct nvmem_operations __rcu *ops;
 	void *priv;
 	bool			sysfs_cells_populated;
 };
 
+int nvmem_add_cells_from_dt(struct nvmem_device *nvmem, struct device_node *np);
+
 #if IS_ENABLED(CONFIG_OF)
 int nvmem_layout_bus_register(void);
 void nvmem_layout_bus_unregister(void);
diff --git a/drivers/nvmem/lan9662-otpc.c b/drivers/nvmem/lan9662-otpc.c
index 56fc19f092a7f..62d1d6381bf88 100644
--- a/drivers/nvmem/lan9662-otpc.c
+++ b/drivers/nvmem/lan9662-otpc.c
@@ -27,7 +27,6 @@
 #define OTP_OTP_STATUS_OTP_CPUMPEN		BIT(1)
 #define OTP_OTP_STATUS_OTP_BUSY			BIT(0)
 
-#define OTP_MEM_SIZE 8192
 #define OTP_SLEEP_US 10
 #define OTP_TIMEOUT_US 500000
 
@@ -176,7 +175,6 @@ static struct nvmem_config otp_config = {
 	.word_size = 1,
 	.reg_read = lan9662_otp_read,
 	.reg_write = lan9662_otp_write,
-	.size = OTP_MEM_SIZE,
 };
 
 static int lan9662_otp_probe(struct platform_device *pdev)
@@ -196,6 +194,7 @@ static int lan9662_otp_probe(struct platform_device *pdev)
 
 	otp_config.priv = otp;
 	otp_config.dev = dev;
+	otp_config.size = (uintptr_t) device_get_match_data(dev);
 
 	nvmem = devm_nvmem_register(dev, &otp_config);
 
@@ -203,7 +202,14 @@ static int lan9662_otp_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id lan9662_otp_match[] = {
-	{ .compatible = "microchip,lan9662-otpc", },
+	{
+		.compatible = "microchip,lan9662-otpc",
+		.data = (const void *) SZ_8K,
+	},
+	{
+		.compatible = "microchip,lan9691-otpc",
+		.data = (const void *) SZ_16K,
+	},
 	{ },
 };
 MODULE_DEVICE_TABLE(of, lan9662_otp_match);
diff --git a/drivers/nvmem/layouts.c b/drivers/nvmem/layouts.c
index b90584e1b99ea..07a34be9669ca 100644
--- a/drivers/nvmem/layouts.c
+++ b/drivers/nvmem/layouts.c
@@ -125,11 +125,6 @@ static int nvmem_layout_create_device(struct nvmem_device *nvmem,
 	return 0;
 }
 
-static const struct of_device_id of_nvmem_layout_skip_table[] = {
-	{ .compatible = "fixed-layout", },
-	{}
-};
-
 static int nvmem_layout_bus_populate(struct nvmem_device *nvmem,
 				     struct device_node *layout_dn)
 {
@@ -142,12 +137,6 @@ static int nvmem_layout_bus_populate(struct nvmem_device *nvmem,
 		return 0;
 	}
 
-	/* Fixed layouts are parsed manually somewhere else for now */
-	if (of_match_node(of_nvmem_layout_skip_table, layout_dn)) {
-		pr_debug("%s() - skipping %pOF node\n", __func__, layout_dn);
-		return 0;
-	}
-
 	if (of_node_check_flag(layout_dn, OF_POPULATED_BUS)) {
 		pr_debug("%s() - skipping %pOF, already populated\n",
 			 __func__, layout_dn);
diff --git a/drivers/nvmem/layouts/Kconfig b/drivers/nvmem/layouts/Kconfig
index 5e586dfebe47b..973d3147f109e 100644
--- a/drivers/nvmem/layouts/Kconfig
+++ b/drivers/nvmem/layouts/Kconfig
@@ -8,6 +8,15 @@ if NVMEM_LAYOUTS
 
 menu "Layout Types"
 
+config NVMEM_LAYOUT_FIXED_LAYOUT
+	tristate "Fixed layout support"
+	default y
+	help
+	  Say Y here to enable support for NVMEM fixed layout, which provides a
+	  way to describe memory cells with fixed offsets and sizes.
+
+	  If unsure, say Y.
+
 config NVMEM_LAYOUT_SL28_VPD
 	tristate "Kontron sl28 VPD layout support"
 	select CRC8
diff --git a/drivers/nvmem/layouts/Makefile b/drivers/nvmem/layouts/Makefile
index 4940c9db06656..9da790a9dde9e 100644
--- a/drivers/nvmem/layouts/Makefile
+++ b/drivers/nvmem/layouts/Makefile
@@ -3,6 +3,7 @@
 # Makefile for nvmem layouts.
 #
 
+obj-$(CONFIG_NVMEM_LAYOUT_FIXED_LAYOUT) += fixed-layout.o
 obj-$(CONFIG_NVMEM_LAYOUT_SL28_VPD) += sl28vpd.o
 obj-$(CONFIG_NVMEM_LAYOUT_ONIE_TLV) += onie-tlv.o
 obj-$(CONFIG_NVMEM_LAYOUT_U_BOOT_ENV) += u-boot-env.o
diff --git a/drivers/nvmem/layouts/fixed-layout.c b/drivers/nvmem/layouts/fixed-layout.c
new file mode 100644
index 0000000000000..635d448b3dd25
--- /dev/null
+++ b/drivers/nvmem/layouts/fixed-layout.c
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2026 Bootlin
+ *
+ * Authors: Mathieu Dubois-Briand <mathieu.dubois-briand@bootlin.com>
+ */
+
+#include <linux/nvmem-provider.h>
+#include <linux/of.h>
+
+#include "../internals.h"
+
+static int fixed_layout_add_cells(struct nvmem_layout *layout)
+{
+	struct device_node *np;
+	int ret;
+
+	np = of_nvmem_layout_get_container(layout->nvmem);
+	if (!np)
+		return -ENOENT;
+
+	ret = nvmem_add_cells_from_dt(layout->nvmem, np);
+	of_node_put(np);
+
+	return ret;
+}
+
+static int fixed_layout_probe(struct nvmem_layout *layout)
+{
+	layout->add_cells = fixed_layout_add_cells;
+
+	return nvmem_layout_register(layout);
+}
+
+static void fixed_layout_remove(struct nvmem_layout *layout)
+{
+	nvmem_layout_unregister(layout);
+}
+
+static const struct of_device_id fixed_layout_of_match_table[] = {
+	{ .compatible = "fixed-layout", },
+	{},
+};
+
+static struct nvmem_layout_driver fixed_layout_layout = {
+	.driver = {
+		.name = "fixed-layout",
+		.of_match_table = fixed_layout_of_match_table,
+	},
+	.probe = fixed_layout_probe,
+	.remove = fixed_layout_remove,
+};
+module_nvmem_layout_driver(fixed_layout_layout);
+
+MODULE_AUTHOR("Mathieu Dubois-Briand");
+MODULE_LICENSE("GPL");
+MODULE_DEVICE_TABLE(of, fixed_layout_of_match_table);
+MODULE_DESCRIPTION("NVMEM fixed-layout driver");
diff --git a/drivers/nvmem/layouts/u-boot-env.c b/drivers/nvmem/layouts/u-boot-env.c
index f27f387bb52af..33ec2350386f5 100644
--- a/drivers/nvmem/layouts/u-boot-env.c
+++ b/drivers/nvmem/layouts/u-boot-env.c
@@ -38,9 +38,6 @@ static int u_boot_env_read_post_process_ethaddr(void *context, const char *id, i
 {
 	u8 mac[ETH_ALEN];
 
-	if (bytes != MAC_ADDR_STR_LEN)
-		return -EINVAL;
-
 	if (!mac_pton(buf, mac))
 		return -EINVAL;
 
@@ -75,7 +72,7 @@ static int u_boot_env_parse_cells(struct device *dev, struct nvmem_device *nvmem
 		info.offset = data_offset + value - data;
 		info.bytes = strlen(value);
 		info.np = of_get_child_by_name(dev->of_node, info.name);
-		if (!strcmp(var, "ethaddr")) {
+		if (!strcmp(var, "ethaddr") && info.bytes == MAC_ADDR_STR_LEN) {
 			info.raw_len = strlen(value);
 			info.bytes = ETH_ALEN;
 			info.read_post_process = u_boot_env_read_post_process_ethaddr;
diff --git a/drivers/nvmem/m24lr.c b/drivers/nvmem/m24lr.c
new file mode 100644
index 0000000000000..7a9fd45a8e460
--- /dev/null
+++ b/drivers/nvmem/m24lr.c
@@ -0,0 +1,606 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * m24lr.c - Sysfs control interface for ST M24LR series RFID/NFC chips
+ *
+ * Copyright (c) 2025 Abd-Alrhman Masalkhi <abd.masalkhi@gmail.com>
+ *
+ * This driver implements both the sysfs-based control interface and EEPROM
+ * access for STMicroelectronics M24LR series chips (e.g., M24LR04E-R).
+ * It provides access to control registers for features such as password
+ * authentication, memory protection, and device configuration. In addition,
+ * it manages read and write operations to the EEPROM region of the chip.
+ */
+
+#include <linux/device.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/nvmem-provider.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/regmap.h>
+
+#define M24LR_WRITE_TIMEOUT	  25u
+#define M24LR_READ_TIMEOUT	  (M24LR_WRITE_TIMEOUT)
+
+/**
+ * struct m24lr_chip - describes chip-specific sysfs layout
+ * @sss_len:       the length of the sss region
+ * @page_size:	   chip-specific limit on the maximum number of bytes allowed
+ *		   in a single write operation.
+ * @eeprom_size:   size of the EEPROM in byte
+ *
+ * Supports multiple M24LR chip variants (e.g., M24LRxx) by allowing each
+ * to define its own set of sysfs attributes, depending on its available
+ * registers and features.
+ */
+struct m24lr_chip {
+	unsigned int sss_len;
+	unsigned int page_size;
+	unsigned int eeprom_size;
+};
+
+/**
+ * struct m24lr - core driver data for M24LR chip control
+ * @uid:           64 bits unique identifier stored in the device
+ * @sss_len:       the length of the sss region
+ * @page_size:	   chip-specific limit on the maximum number of bytes allowed
+ *		   in a single write operation.
+ * @eeprom_size:   size of the EEPROM in byte
+ * @ctl_regmap:	   regmap interface for accessing the system parameter sector
+ * @eeprom_regmap: regmap interface for accessing the EEPROM
+ * @lock:	   mutex to synchronize operations to the device
+ *
+ * Central data structure holding the state and resources used by the
+ * M24LR device driver.
+ */
+struct m24lr {
+	u64 uid;
+	unsigned int sss_len;
+	unsigned int page_size;
+	unsigned int eeprom_size;
+	struct regmap *ctl_regmap;
+	struct regmap *eeprom_regmap;
+	struct mutex lock;	 /* synchronize operations to the device */
+};
+
+static const struct regmap_range m24lr_ctl_vo_ranges[] = {
+	regmap_reg_range(0, 63),
+};
+
+static const struct regmap_access_table m24lr_ctl_vo_table = {
+	.yes_ranges = m24lr_ctl_vo_ranges,
+	.n_yes_ranges = ARRAY_SIZE(m24lr_ctl_vo_ranges),
+};
+
+static const struct regmap_config m24lr_ctl_regmap_conf = {
+	.name = "m24lr_ctl",
+	.reg_stride = 1,
+	.reg_bits = 16,
+	.val_bits = 8,
+	.disable_locking = false,
+	.cache_type = REGCACHE_RBTREE,/* Flat can't be used, there's huge gap */
+	.volatile_table = &m24lr_ctl_vo_table,
+};
+
+/* Chip descriptor for M24LR04E-R variant */
+static const struct m24lr_chip m24lr04e_r_chip = {
+	.page_size = 4,
+	.eeprom_size = 512,
+	.sss_len = 4,
+};
+
+/* Chip descriptor for M24LR16E-R variant */
+static const struct m24lr_chip m24lr16e_r_chip = {
+	.page_size = 4,
+	.eeprom_size = 2048,
+	.sss_len = 16,
+};
+
+/* Chip descriptor for M24LR64E-R variant */
+static const struct m24lr_chip m24lr64e_r_chip = {
+	.page_size = 4,
+	.eeprom_size = 8192,
+	.sss_len = 64,
+};
+
+static const struct i2c_device_id m24lr_ids[] = {
+	{ "m24lr04e-r", (kernel_ulong_t)&m24lr04e_r_chip},
+	{ "m24lr16e-r", (kernel_ulong_t)&m24lr16e_r_chip},
+	{ "m24lr64e-r", (kernel_ulong_t)&m24lr64e_r_chip},
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, m24lr_ids);
+
+static const struct of_device_id m24lr_of_match[] = {
+	{ .compatible = "st,m24lr04e-r", .data = &m24lr04e_r_chip},
+	{ .compatible = "st,m24lr16e-r", .data = &m24lr16e_r_chip},
+	{ .compatible = "st,m24lr64e-r", .data = &m24lr64e_r_chip},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, m24lr_of_match);
+
+/**
+ * m24lr_regmap_read - read data using regmap with retry on failure
+ * @regmap:  regmap instance for the device
+ * @buf:     buffer to store the read data
+ * @size:    number of bytes to read
+ * @offset:  starting register address
+ *
+ * Attempts to read a block of data from the device with retries and timeout.
+ * Some M24LR chips may transiently NACK reads (e.g., during internal write
+ * cycles), so this function retries with a short sleep until the timeout
+ * expires.
+ *
+ * Returns:
+ *	 Number of bytes read on success,
+ *	 -ETIMEDOUT if the read fails within the timeout window.
+ */
+static ssize_t m24lr_regmap_read(struct regmap *regmap, u8 *buf,
+				 size_t size, unsigned int offset)
+{
+	int err;
+	unsigned long timeout, read_time;
+	ssize_t ret = -ETIMEDOUT;
+
+	timeout = jiffies + msecs_to_jiffies(M24LR_READ_TIMEOUT);
+	do {
+		read_time = jiffies;
+
+		err = regmap_bulk_read(regmap, offset, buf, size);
+		if (!err) {
+			ret = size;
+			break;
+		}
+
+		usleep_range(1000, 2000);
+	} while (time_before(read_time, timeout));
+
+	return ret;
+}
+
+/**
+ * m24lr_regmap_write - write data using regmap with retry on failure
+ * @regmap: regmap instance for the device
+ * @buf:    buffer containing the data to write
+ * @size:   number of bytes to write
+ * @offset: starting register address
+ *
+ * Attempts to write a block of data to the device with retries and a timeout.
+ * Some M24LR devices may NACK I2C writes while an internal write operation
+ * is in progress. This function retries the write operation with a short delay
+ * until it succeeds or the timeout is reached.
+ *
+ * Returns:
+ *	 Number of bytes written on success,
+ *	 -ETIMEDOUT if the write fails within the timeout window.
+ */
+static ssize_t m24lr_regmap_write(struct regmap *regmap, const u8 *buf,
+				  size_t size, unsigned int offset)
+{
+	int err;
+	unsigned long timeout, write_time;
+	ssize_t ret = -ETIMEDOUT;
+
+	timeout = jiffies + msecs_to_jiffies(M24LR_WRITE_TIMEOUT);
+
+	do {
+		write_time = jiffies;
+
+		err = regmap_bulk_write(regmap, offset, buf, size);
+		if (!err) {
+			ret = size;
+			break;
+		}
+
+		usleep_range(1000, 2000);
+	} while (time_before(write_time, timeout));
+
+	return ret;
+}
+
+static ssize_t m24lr_read(struct m24lr *m24lr, u8 *buf, size_t size,
+			  unsigned int offset, bool is_eeprom)
+{
+	struct regmap *regmap;
+	ssize_t ret;
+
+	if (is_eeprom)
+		regmap = m24lr->eeprom_regmap;
+	else
+		regmap = m24lr->ctl_regmap;
+
+	mutex_lock(&m24lr->lock);
+	ret = m24lr_regmap_read(regmap, buf, size, offset);
+	mutex_unlock(&m24lr->lock);
+
+	return ret;
+}
+
+/**
+ * m24lr_write - write buffer to M24LR device with page alignment handling
+ * @m24lr:     pointer to driver context
+ * @buf:       data buffer to write
+ * @size:      number of bytes to write
+ * @offset:    target register address in the device
+ * @is_eeprom: true if the write should target the EEPROM,
+ *             false if it should target the system parameters sector.
+ *
+ * Writes data to the M24LR device using regmap, split into chunks no larger
+ * than page_size to respect device-specific write limitations (e.g., page
+ * size or I2C hold-time concerns). Each chunk is aligned to the page boundary
+ * defined by page_size.
+ *
+ * Returns:
+ *	 Total number of bytes written on success,
+ *	 A negative error code if any write fails.
+ */
+static ssize_t m24lr_write(struct m24lr *m24lr, const u8 *buf, size_t size,
+			   unsigned int offset, bool is_eeprom)
+{
+	unsigned int n, next_sector;
+	struct regmap *regmap;
+	ssize_t ret = 0;
+	ssize_t err;
+
+	if (is_eeprom)
+		regmap = m24lr->eeprom_regmap;
+	else
+		regmap = m24lr->ctl_regmap;
+
+	n = min_t(unsigned int, size, m24lr->page_size);
+	next_sector = roundup(offset + 1, m24lr->page_size);
+	if (offset + n > next_sector)
+		n = next_sector - offset;
+
+	mutex_lock(&m24lr->lock);
+	while (n) {
+		err = m24lr_regmap_write(regmap, buf + offset, n, offset);
+		if (IS_ERR_VALUE(err)) {
+			if (!ret)
+				ret = err;
+
+			break;
+		}
+
+		offset += n;
+		size -= n;
+		ret += n;
+		n = min_t(unsigned int, size, m24lr->page_size);
+	}
+	mutex_unlock(&m24lr->lock);
+
+	return ret;
+}
+
+/**
+ * m24lr_write_pass - Write password to M24LR043-R using secure format
+ * @m24lr: Pointer to device control structure
+ * @buf:   Input buffer containing hex-encoded password
+ * @count: Number of bytes in @buf
+ * @code:  Operation code to embed between password copies
+ *
+ * This function parses a 4-byte password, encodes it in  big-endian format,
+ * and constructs a 9-byte sequence of the form:
+ *
+ *	  [BE(password), code, BE(password)]
+ *
+ * The result is written to register 0x0900 (2304), which is the password
+ * register in M24LR04E-R chip.
+ *
+ * Return: Number of bytes written on success, or negative error code on failure
+ */
+static ssize_t m24lr_write_pass(struct m24lr *m24lr, const char *buf,
+				size_t count, u8 code)
+{
+	__be32 be_pass;
+	u8 output[9];
+	ssize_t ret;
+	u32 pass;
+	int err;
+
+	if (!count)
+		return -EINVAL;
+
+	if (count > 8)
+		return -EINVAL;
+
+	err = kstrtou32(buf, 16, &pass);
+	if (err)
+		return err;
+
+	be_pass = cpu_to_be32(pass);
+
+	memcpy(output, &be_pass, sizeof(be_pass));
+	output[4] = code;
+	memcpy(output + 5, &be_pass, sizeof(be_pass));
+
+	mutex_lock(&m24lr->lock);
+	ret = m24lr_regmap_write(m24lr->ctl_regmap, output, 9, 2304);
+	mutex_unlock(&m24lr->lock);
+
+	return ret;
+}
+
+static ssize_t m24lr_read_reg_le(struct m24lr *m24lr, u64 *val,
+				 unsigned int reg_addr,
+				 unsigned int reg_size)
+{
+	ssize_t ret;
+	__le64 input = 0;
+
+	ret = m24lr_read(m24lr, (u8 *)&input, reg_size, reg_addr, false);
+	if (IS_ERR_VALUE(ret))
+		return ret;
+
+	if (ret != reg_size)
+		return -EINVAL;
+
+	switch (reg_size) {
+	case 1:
+		*val = *(u8 *)&input;
+		break;
+	case 2:
+		*val = le16_to_cpu((__le16)input);
+		break;
+	case 4:
+		*val = le32_to_cpu((__le32)input);
+		break;
+	case 8:
+		*val = le64_to_cpu((__le64)input);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int m24lr_nvmem_read(void *priv, unsigned int offset, void *val,
+			    size_t bytes)
+{
+	ssize_t err;
+	struct m24lr *m24lr = priv;
+
+	if (!bytes)
+		return bytes;
+
+	if (offset + bytes > m24lr->eeprom_size)
+		return -EINVAL;
+
+	err = m24lr_read(m24lr, val, bytes, offset, true);
+	if (IS_ERR_VALUE(err))
+		return err;
+
+	return 0;
+}
+
+static int m24lr_nvmem_write(void *priv, unsigned int offset, void *val,
+			     size_t bytes)
+{
+	ssize_t err;
+	struct m24lr *m24lr = priv;
+
+	if (!bytes)
+		return -EINVAL;
+
+	if (offset + bytes > m24lr->eeprom_size)
+		return -EINVAL;
+
+	err = m24lr_write(m24lr, val, bytes, offset, true);
+	if (IS_ERR_VALUE(err))
+		return err;
+
+	return 0;
+}
+
+static ssize_t m24lr_ctl_sss_read(struct file *filep, struct kobject *kobj,
+				  const struct bin_attribute *attr, char *buf,
+				  loff_t offset, size_t count)
+{
+	struct m24lr *m24lr = attr->private;
+
+	if (!count)
+		return count;
+
+	if (size_add(offset, count) > m24lr->sss_len)
+		return -EINVAL;
+
+	return m24lr_read(m24lr, buf, count, offset, false);
+}
+
+static ssize_t m24lr_ctl_sss_write(struct file *filep, struct kobject *kobj,
+				   const struct bin_attribute *attr, char *buf,
+				   loff_t offset, size_t count)
+{
+	struct m24lr *m24lr = attr->private;
+
+	if (!count)
+		return -EINVAL;
+
+	if (size_add(offset, count) > m24lr->sss_len)
+		return -EINVAL;
+
+	return m24lr_write(m24lr, buf, count, offset, false);
+}
+static BIN_ATTR(sss, 0600, m24lr_ctl_sss_read, m24lr_ctl_sss_write, 0);
+
+static ssize_t new_pass_store(struct device *dev, struct device_attribute *attr,
+			      const char *buf, size_t count)
+{
+	struct m24lr *m24lr = i2c_get_clientdata(to_i2c_client(dev));
+
+	return m24lr_write_pass(m24lr, buf, count, 7);
+}
+static DEVICE_ATTR_WO(new_pass);
+
+static ssize_t unlock_store(struct device *dev, struct device_attribute *attr,
+			    const char *buf, size_t count)
+{
+	struct m24lr *m24lr = i2c_get_clientdata(to_i2c_client(dev));
+
+	return m24lr_write_pass(m24lr, buf, count, 9);
+}
+static DEVICE_ATTR_WO(unlock);
+
+static ssize_t uid_show(struct device *dev, struct device_attribute *attr,
+			char *buf)
+{
+	struct m24lr *m24lr = i2c_get_clientdata(to_i2c_client(dev));
+
+	return sysfs_emit(buf, "%llx\n", m24lr->uid);
+}
+static DEVICE_ATTR_RO(uid);
+
+static ssize_t total_sectors_show(struct device *dev,
+				  struct device_attribute *attr, char *buf)
+{
+	struct m24lr *m24lr = i2c_get_clientdata(to_i2c_client(dev));
+
+	return sysfs_emit(buf, "%x\n", m24lr->sss_len);
+}
+static DEVICE_ATTR_RO(total_sectors);
+
+static struct attribute *m24lr_ctl_dev_attrs[] = {
+	&dev_attr_unlock.attr,
+	&dev_attr_new_pass.attr,
+	&dev_attr_uid.attr,
+	&dev_attr_total_sectors.attr,
+	NULL,
+};
+
+static const struct m24lr_chip *m24lr_get_chip(struct device *dev)
+{
+	const struct m24lr_chip *ret;
+	const struct i2c_device_id *id;
+
+	id = i2c_match_id(m24lr_ids, to_i2c_client(dev));
+
+	if (dev->of_node && of_match_device(m24lr_of_match, dev))
+		ret = of_device_get_match_data(dev);
+	else if (id)
+		ret = (void *)id->driver_data;
+	else
+		ret = acpi_device_get_match_data(dev);
+
+	return ret;
+}
+
+static int m24lr_probe(struct i2c_client *client)
+{
+	struct regmap_config eeprom_regmap_conf = {0};
+	struct nvmem_config nvmem_conf = {0};
+	struct device *dev = &client->dev;
+	struct i2c_client *eeprom_client;
+	const struct m24lr_chip *chip;
+	struct regmap *eeprom_regmap;
+	struct nvmem_device *nvmem;
+	struct regmap *ctl_regmap;
+	struct m24lr *m24lr;
+	u32 regs[2];
+	long err;
+
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C))
+		return -EOPNOTSUPP;
+
+	chip = m24lr_get_chip(dev);
+	if (!chip)
+		return -ENODEV;
+
+	m24lr = devm_kzalloc(dev, sizeof(struct m24lr), GFP_KERNEL);
+	if (!m24lr)
+		return -ENOMEM;
+
+	err = device_property_read_u32_array(dev, "reg", regs, ARRAY_SIZE(regs));
+	if (err)
+		return dev_err_probe(dev, err, "Failed to read 'reg' property\n");
+
+	/* Create a second I2C client for the eeprom interface */
+	eeprom_client = devm_i2c_new_dummy_device(dev, client->adapter, regs[1]);
+	if (IS_ERR(eeprom_client))
+		return dev_err_probe(dev, PTR_ERR(eeprom_client),
+				     "Failed to create dummy I2C client for the EEPROM\n");
+
+	ctl_regmap = devm_regmap_init_i2c(client, &m24lr_ctl_regmap_conf);
+	if (IS_ERR(ctl_regmap))
+		return dev_err_probe(dev, PTR_ERR(ctl_regmap),
+				      "Failed to init regmap\n");
+
+	eeprom_regmap_conf.name = "m24lr_eeprom";
+	eeprom_regmap_conf.reg_bits = 16;
+	eeprom_regmap_conf.val_bits = 8;
+	eeprom_regmap_conf.disable_locking = true;
+	eeprom_regmap_conf.max_register = chip->eeprom_size - 1;
+
+	eeprom_regmap = devm_regmap_init_i2c(eeprom_client,
+					     &eeprom_regmap_conf);
+	if (IS_ERR(eeprom_regmap))
+		return dev_err_probe(dev, PTR_ERR(eeprom_regmap),
+				     "Failed to init regmap\n");
+
+	mutex_init(&m24lr->lock);
+	m24lr->sss_len = chip->sss_len;
+	m24lr->page_size = chip->page_size;
+	m24lr->eeprom_size = chip->eeprom_size;
+	m24lr->eeprom_regmap = eeprom_regmap;
+	m24lr->ctl_regmap = ctl_regmap;
+
+	nvmem_conf.dev = &eeprom_client->dev;
+	nvmem_conf.owner = THIS_MODULE;
+	nvmem_conf.type = NVMEM_TYPE_EEPROM;
+	nvmem_conf.reg_read = m24lr_nvmem_read;
+	nvmem_conf.reg_write = m24lr_nvmem_write;
+	nvmem_conf.size = chip->eeprom_size;
+	nvmem_conf.word_size = 1;
+	nvmem_conf.stride = 1;
+	nvmem_conf.priv = m24lr;
+
+	nvmem = devm_nvmem_register(dev, &nvmem_conf);
+	if (IS_ERR(nvmem))
+		return dev_err_probe(dev, PTR_ERR(nvmem),
+				     "Failed to register nvmem\n");
+
+	i2c_set_clientdata(client, m24lr);
+	i2c_set_clientdata(eeprom_client, m24lr);
+
+	bin_attr_sss.size = chip->sss_len;
+	bin_attr_sss.private = m24lr;
+	err = sysfs_create_bin_file(&dev->kobj, &bin_attr_sss);
+	if (err)
+		return dev_err_probe(dev, err,
+				     "Failed to create sss bin file\n");
+
+	/* test by reading the uid, if success store it */
+	err = m24lr_read_reg_le(m24lr, &m24lr->uid, 2324, sizeof(m24lr->uid));
+	if (IS_ERR_VALUE(err))
+		goto remove_bin_file;
+
+	return 0;
+
+remove_bin_file:
+	sysfs_remove_bin_file(&dev->kobj, &bin_attr_sss);
+
+	return err;
+}
+
+static void m24lr_remove(struct i2c_client *client)
+{
+	sysfs_remove_bin_file(&client->dev.kobj, &bin_attr_sss);
+}
+
+ATTRIBUTE_GROUPS(m24lr_ctl_dev);
+
+static struct i2c_driver m24lr_driver = {
+	.driver = {
+		.name = "m24lr",
+		.of_match_table = m24lr_of_match,
+		.dev_groups = m24lr_ctl_dev_groups,
+	},
+	.probe	  = m24lr_probe,
+	.remove = m24lr_remove,
+	.id_table = m24lr_ids,
+};
+module_i2c_driver(m24lr_driver);
+
+MODULE_AUTHOR("Abd-Alrhman Masalkhi");
+MODULE_DESCRIPTION("st m24lr control driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/nvmem/mt6323-efuse.c b/drivers/nvmem/mt6323-efuse.c
new file mode 100644
index 0000000000000..41678915d5b62
--- /dev/null
+++ b/drivers/nvmem/mt6323-efuse.c
@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (c) 2026 Roman Vivchar <rva333@protonmail.com>
+ */
+
+#include <linux/device-id/of.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/nvmem-provider.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/types.h>
+
+#include <linux/mfd/mt6323/registers.h>
+
+#define MT6323_EFUSE_DOUT_BASE	MT6323_EFUSE_DOUT_0_15
+#define MT6323_EFUSE_SIZE	24
+
+static int mt6323_efuse_read(void *context, unsigned int offset, void *val,
+			     size_t bytes)
+{
+	struct regmap *map = context;
+	u16 *buf = val;
+	u32 tmp;
+	int ret;
+
+	/*
+	 * A manual loop using regmap_read is required because PWRAP is not
+	 * a continuous MMIO space, but rather a FSM that doesn't implement the
+	 * necessary read callback for the regmap_read_raw and regmap_read_bulk
+	 * functions.
+	 */
+	for (size_t i = 0; i < bytes; i += sizeof(*buf)) {
+		ret = regmap_read(map, MT6323_EFUSE_DOUT_BASE + offset + i, &tmp);
+		if (ret)
+			return ret;
+
+		*buf++ = tmp;
+	}
+
+	return 0;
+}
+
+static int mt6323_efuse_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct nvmem_config config = {
+		.name = "mt6323-efuse",
+		.stride = 2,
+		.word_size = 2,
+		.size = MT6323_EFUSE_SIZE,
+		.reg_read = mt6323_efuse_read,
+	};
+	struct nvmem_device *nvmem;
+	struct regmap *regmap;
+
+	/* efuse -> mfd -> pwrap */
+	regmap = dev_get_regmap(dev->parent->parent, NULL);
+	if (!regmap)
+		return dev_err_probe(dev, -ENODEV, "failed to get regmap\n");
+
+	config.dev = dev;
+	config.priv = regmap;
+
+	nvmem = devm_nvmem_register(dev, &config);
+	return PTR_ERR_OR_ZERO(nvmem);
+}
+
+static const struct of_device_id mt6323_efuse_of_match[] = {
+	{ .compatible = "mediatek,mt6323-efuse" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, mt6323_efuse_of_match);
+
+static struct platform_driver mt6323_efuse_driver = {
+	.probe = mt6323_efuse_probe,
+	.driver = {
+		.name = "mt6323-efuse",
+		.of_match_table = mt6323_efuse_of_match,
+	},
+};
+module_platform_driver(mt6323_efuse_driver);
+
+MODULE_DESCRIPTION("MediaTek MT6323 PMIC EFUSE driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/nvmem/nintendo-otp.c b/drivers/nvmem/nintendo-otp.c
index 4440d4e5fb836..a4d986e4588ea 100644
--- a/drivers/nvmem/nintendo-otp.c
+++ b/drivers/nvmem/nintendo-otp.c
@@ -16,7 +16,7 @@
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/nvmem-provider.h>
-#include <linux/of_device.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 
 #define HW_OTPCMD  0
@@ -73,8 +73,7 @@ MODULE_DEVICE_TABLE(of, nintendo_otp_of_table);
 static int nintendo_otp_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	const struct of_device_id *of_id =
-		of_match_device(nintendo_otp_of_table, dev);
+	const struct nintendo_otp_devtype_data *data;
 	struct nvmem_device *nvmem;
 	struct nintendo_otp_priv *priv;
 
@@ -94,8 +93,8 @@ static int nintendo_otp_probe(struct platform_device *pdev)
 	if (IS_ERR(priv->regs))
 		return PTR_ERR(priv->regs);
 
-	if (of_id->data) {
-		const struct nintendo_otp_devtype_data *data = of_id->data;
+	data = of_device_get_match_data(dev);
+	if (data) {
 		config.name = data->name;
 		config.size = data->num_banks * BANK_SIZE;
 	}
diff --git a/drivers/nvmem/rockchip-otp.c b/drivers/nvmem/rockchip-otp.c
index 0ec78b5e19e7d..2c0feb036f3fd 100644
--- a/drivers/nvmem/rockchip-otp.c
+++ b/drivers/nvmem/rockchip-otp.c
@@ -78,9 +78,9 @@ struct rockchip_data {
 struct rockchip_otp {
 	struct device *dev;
 	void __iomem *base;
-	struct clk_bulk_data *clks;
 	struct reset_control *rst;
 	const struct rockchip_data *data;
+	struct clk_bulk_data clks[];
 };
 
 static int rockchip_otp_reset(struct rockchip_otp *otp)
@@ -424,7 +424,7 @@ static int rockchip_otp_probe(struct platform_device *pdev)
 	if (!data)
 		return dev_err_probe(dev, -EINVAL, "failed to get match data\n");
 
-	otp = devm_kzalloc(&pdev->dev, sizeof(struct rockchip_otp),
+	otp = devm_kzalloc(&pdev->dev, struct_size(otp, clks, data->num_clks),
 			   GFP_KERNEL);
 	if (!otp)
 		return -ENOMEM;
@@ -436,11 +436,6 @@ static int rockchip_otp_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, PTR_ERR(otp->base),
 				     "failed to ioremap resource\n");
 
-	otp->clks = devm_kcalloc(dev, data->num_clks, sizeof(*otp->clks),
-				 GFP_KERNEL);
-	if (!otp->clks)
-		return -ENOMEM;
-
 	for (i = 0; i < data->num_clks; ++i)
 		otp->clks[i].id = data->clks[i];
 
diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c
index 26f66046cc323..a64afc3ded3d4 100644
--- a/drivers/of/fdt.c
+++ b/drivers/of/fdt.c
@@ -878,11 +878,12 @@ static unsigned long chosen_node_offset = -FDT_ERR_NOTFOUND;
 /*
  * The main usage of linux,usable-memory-range is for crash dump kernel.
  * Originally, the number of usable-memory regions is one. Now there may
- * be two regions, low region and high region.
- * To make compatibility with existing user-space and older kdump, the low
- * region is always the last range of linux,usable-memory-range if exist.
+ * be 2 + CRASHK_CMA_RANGES_MAX regions, low region, high region and cma
+ * regions. To make compatibility with existing user-space and older kdump,
+ * the high and low region are always the first two ranges of
+ * linux,usable-memory-range if exist.
  */
-#define MAX_USABLE_RANGES		2
+#define MAX_USABLE_RANGES		(2 + CRASHK_CMA_RANGES_MAX)
 
 /**
  * early_init_dt_check_for_usable_mem_range - Decode usable memory range
diff --git a/drivers/of/kexec.c b/drivers/of/kexec.c
index b6837e299e7fe..029903b986cbd 100644
--- a/drivers/of/kexec.c
+++ b/drivers/of/kexec.c
@@ -458,6 +458,15 @@ void *of_kexec_alloc_and_setup_fdt(const struct kimage *image,
 			if (ret)
 				goto out;
 		}
+
+		for (int i = 0; i < crashk_cma_cnt; i++) {
+			ret = fdt_appendprop_addrrange(fdt, 0, chosen_node,
+					"linux,usable-memory-range",
+					crashk_cma_ranges[i].start,
+					crashk_cma_ranges[i].end - crashk_cma_ranges[i].start + 1);
+			if (ret)
+				goto out;
+		}
 #endif
 	}
 
diff --git a/drivers/of/of_numa.c b/drivers/of/of_numa.c
index cd2dc8e825c92..230d5f628c1b4 100644
--- a/drivers/of/of_numa.c
+++ b/drivers/of/of_numa.c
@@ -59,11 +59,8 @@ static int __init of_numa_parse_memory_nodes(void)
 			r = -EINVAL;
 		}
 
-		for (i = 0; !r && !of_address_to_resource(np, i, &rsrc); i++) {
+		for (i = 0; !r && !of_address_to_resource(np, i, &rsrc); i++)
 			r = numa_add_memblk(nid, rsrc.start, rsrc.end + 1);
-			if (!r)
-				node_set(nid, numa_nodes_parsed);
-		}
 
 		if (!i || r) {
 			of_node_put(np);
diff --git a/drivers/of/platform.c b/drivers/of/platform.c
index 53bca8c6f7810..8b1e764077825 100644
--- a/drivers/of/platform.c
+++ b/drivers/of/platform.c
@@ -126,7 +126,7 @@ struct platform_device *of_device_alloc(struct device_node *np,
 	}
 
 	/* setup generic device info */
-	device_set_node(&dev->dev, of_fwnode_handle(of_node_get(np)));
+	platform_device_set_of_node(dev, np);
 	dev->dev.parent = parent ? : &platform_bus;
 
 	if (bus_id)
diff --git a/drivers/opp/core.c b/drivers/opp/core.c
index ab0b0a2f85a17..b6966e509f7dd 100644
--- a/drivers/opp/core.c
+++ b/drivers/opp/core.c
@@ -1412,13 +1412,12 @@ static int _set_opp(struct device *dev, struct opp_table *opp_table,
  */
 int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq)
 {
+	struct opp_table *opp_table __free(put_opp_table) =
+		_find_opp_table(dev);
 	struct dev_pm_opp *opp __free(put_opp) = NULL;
 	unsigned long freq = 0, temp_freq;
 	bool forced = false;
 
-	struct opp_table *opp_table __free(put_opp_table) =
-		_find_opp_table(dev);
-
 	if (IS_ERR(opp_table)) {
 		dev_err(dev, "%s: device's opp table doesn't exist\n", __func__);
 		return PTR_ERR(opp_table);
@@ -2870,11 +2869,10 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_add_dynamic);
 static int _opp_set_availability(struct device *dev, unsigned long freq,
 				 bool availability_req)
 {
-	struct dev_pm_opp *opp __free(put_opp) = ERR_PTR(-ENODEV), *tmp_opp;
-
 	/* Find the opp_table */
 	struct opp_table *opp_table __free(put_opp_table) =
 		_find_opp_table(dev);
+	struct dev_pm_opp *opp __free(put_opp) = ERR_PTR(-ENODEV), *tmp_opp;
 
 	if (IS_ERR(opp_table)) {
 		dev_warn(dev, "%s: Device OPP not found (%ld)\n", __func__,
@@ -2932,12 +2930,11 @@ int dev_pm_opp_adjust_voltage(struct device *dev, unsigned long freq,
 			      unsigned long u_volt_max)
 
 {
-	struct dev_pm_opp *opp __free(put_opp) = ERR_PTR(-ENODEV), *tmp_opp;
-	int r;
-
 	/* Find the opp_table */
 	struct opp_table *opp_table __free(put_opp_table) =
 		_find_opp_table(dev);
+	struct dev_pm_opp *opp __free(put_opp) = ERR_PTR(-ENODEV), *tmp_opp;
+	int r;
 
 	if (IS_ERR(opp_table)) {
 		r = PTR_ERR(opp_table);
diff --git a/drivers/parisc/superio.c b/drivers/parisc/superio.c
index e973c68932038..3aaab40df86ec 100644
--- a/drivers/parisc/superio.c
+++ b/drivers/parisc/superio.c
@@ -15,7 +15,7 @@
  *	(C) Copyright 2005 Kyle McMartin <kyle@parisc-linux.org>
  *	(C) Copyright 2006 Helge Deller <deller@gmx.de>
  *
- *	The initial version of this is by Martin Peterson.  Alex deVries
+ *	The initial version of this is by Martin Petersen.  Alex deVries
  *	has spent a bit of time trying to coax it into working.
  *
  *      Major changes to get basic interrupt infrastructure working to
diff --git a/drivers/pci/ats.c b/drivers/pci/ats.c
index 96efa00d97433..9cb23780093dd 100644
--- a/drivers/pci/ats.c
+++ b/drivers/pci/ats.c
@@ -40,10 +40,13 @@ void pci_ats_init(struct pci_dev *dev)
  */
 bool pci_ats_supported(struct pci_dev *dev)
 {
-	if (!dev->ats_cap)
+	if (!dev->ats_cap || dev->untrusted)
 		return false;
 
-	return (dev->untrusted == 0);
+	if (dev->is_virtfn)
+		return pci_ats_supported(pci_physfn(dev));
+
+	return true;
 }
 EXPORT_SYMBOL_GPL(pci_ats_supported);
 
@@ -70,8 +73,12 @@ int pci_prepare_ats(struct pci_dev *dev, int ps)
 	if (ps < PCI_ATS_MIN_STU)
 		return -EINVAL;
 
-	if (dev->is_virtfn)
+	if (dev->is_virtfn) {
+		if (pci_physfn(dev)->ats_stu != ps)
+			return -EINVAL;
+
 		return 0;
+	}
 
 	dev->ats_stu = ps;
 	ctrl = PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
index 2247709ef6d69..8a3a31b2bc12c 100644
--- a/drivers/pci/controller/Kconfig
+++ b/drivers/pci/controller/Kconfig
@@ -209,7 +209,7 @@ config PCI_MVEBU
 
 config PCIE_MEDIATEK
 	tristate "MediaTek PCIe controller"
-	depends on ARCH_AIROHA || ARCH_MEDIATEK || COMPILE_TEST
+	depends on ARCH_AIROHA || ARCH_MEDIATEK || ECONET || COMPILE_TEST
 	depends on OF
 	depends on PCI_MSI
 	select IRQ_MSI_LIB
diff --git a/drivers/pci/controller/cadence/pcie-cadence-plat.c b/drivers/pci/controller/cadence/pcie-cadence-plat.c
index b067a3296dd37..a1ea24fc3b63f 100644
--- a/drivers/pci/controller/cadence/pcie-cadence-plat.c
+++ b/drivers/pci/controller/cadence/pcie-cadence-plat.c
@@ -163,6 +163,7 @@ static const struct of_device_id cdns_plat_pcie_of_match[] = {
 	},
 	{},
 };
+MODULE_DEVICE_TABLE(of, cdns_plat_pcie_of_match);
 
 static struct platform_driver cdns_plat_pcie_driver = {
 	.driver = {
diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig
index aa0b784c85b47..3260d916110b2 100644
--- a/drivers/pci/controller/dwc/Kconfig
+++ b/drivers/pci/controller/dwc/Kconfig
@@ -126,7 +126,9 @@ config PCI_IMX6_EP
 	depends on ARCH_MXC || COMPILE_TEST
 	depends on PCI_ENDPOINT
 	select PCIE_DW_EP
+	select PCI_HOST_COMMON
 	select PCI_IMX6
+	select PCI_PWRCTRL_GENERIC
 	help
 	  Enables support for the PCIe controller in the i.MX SoCs to
 	  work in endpoint mode. The PCI controller on i.MX is based
@@ -344,7 +346,7 @@ config PCIE_RCAR_GEN4
 
 config PCIE_RCAR_GEN4_HOST
 	tristate "Renesas R-Car Gen4 PCIe controller (host mode)"
-	depends on ARCH_RENESAS || COMPILE_TEST
+	depends on ARM64 && (ARCH_RENESAS || COMPILE_TEST)
 	depends on PCI_MSI
 	select PCIE_DW_HOST
 	select PCIE_RCAR_GEN4
@@ -355,7 +357,7 @@ config PCIE_RCAR_GEN4_HOST
 
 config PCIE_RCAR_GEN4_EP
 	tristate "Renesas R-Car Gen4 PCIe controller (endpoint mode)"
-	depends on ARCH_RENESAS || COMPILE_TEST
+	depends on ARM64 && (ARCH_RENESAS || COMPILE_TEST)
 	depends on PCI_ENDPOINT
 	select PCIE_DW_EP
 	select PCIE_RCAR_GEN4
diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c
index 98e1db751132a..afcf3b6bf3cd7 100644
--- a/drivers/pci/controller/dwc/pci-imx6.c
+++ b/drivers/pci/controller/dwc/pci-imx6.c
@@ -1318,6 +1318,18 @@ static void imx_pcie_assert_perst(struct imx_pcie *imx_pcie, bool assert)
 	}
 }
 
+static bool imx_pcie_perst_found(struct pci_host_bridge *bridge)
+{
+	struct pci_host_port *port;
+
+	list_for_each_entry(port, &bridge->ports, list) {
+		if (!list_empty(&port->perst))
+			return true;
+	}
+
+	return false;
+}
+
 static int imx_pcie_host_init(struct dw_pcie_rp *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
@@ -1330,15 +1342,12 @@ static int imx_pcie_host_init(struct dw_pcie_rp *pp)
 		/* Parse Root Port nodes if present */
 		ret = pci_host_common_parse_ports(dev, bridge);
 		if (ret) {
-			if (ret != -ENODEV) {
-				dev_err(dev, "Failed to parse Root Port nodes: %d\n", ret);
-				return ret;
-			}
+			dev_err(dev, "Failed to parse Root Port nodes: %d\n", ret);
+			return ret;
+		}
 
-			/*
-			 * Fall back to legacy binding for DT backwards
-			 * compatibility
-			 */
+		/* Fall back to legacy binding for DT backwards compatibility */
+		if (!imx_pcie_perst_found(bridge)) {
 			ret = imx_pcie_parse_legacy_binding(imx_pcie);
 			if (ret)
 				return ret;
@@ -1373,16 +1382,12 @@ static int imx_pcie_host_init(struct dw_pcie_rp *pp)
 		}
 	}
 
-	ret = pci_pwrctrl_create_devices(dev);
-	if (ret) {
-		dev_err(dev, "failed to create pwrctrl devices\n");
-		goto err_reg_disable;
-	}
-
-	ret = pci_pwrctrl_power_on_devices(dev);
-	if (ret) {
-		dev_err(dev, "failed to power on pwrctrl devices\n");
-		goto err_pwrctrl_destroy;
+	if (!pp->skip_pwrctrl_off) {
+		ret = pci_pwrctrl_power_on_devices(dev);
+		if (ret) {
+			dev_err(dev, "failed to power on pwrctrl devices\n");
+			goto err_reg_disable;
+		}
 	}
 
 	ret = imx_pcie_clk_enable(imx_pcie);
@@ -1451,10 +1456,8 @@ static int imx_pcie_host_init(struct dw_pcie_rp *pp)
 err_clk_disable:
 	imx_pcie_clk_disable(imx_pcie);
 err_pwrctrl_power_off:
-	pci_pwrctrl_power_off_devices(dev);
-err_pwrctrl_destroy:
-	if (ret != -EPROBE_DEFER)
-		pci_pwrctrl_destroy_devices(dev);
+	if (!pp->skip_pwrctrl_off)
+		pci_pwrctrl_power_off_devices(dev);
 err_reg_disable:
 	if (imx_pcie->vpcie)
 		regulator_disable(imx_pcie->vpcie);
@@ -1473,7 +1476,8 @@ static void imx_pcie_host_exit(struct dw_pcie_rp *pp)
 	}
 	imx_pcie_clk_disable(imx_pcie);
 
-	pci_pwrctrl_power_off_devices(pci->dev);
+	if (!pci->pp.skip_pwrctrl_off)
+		pci_pwrctrl_power_off_devices(pci->dev);
 	if (imx_pcie->vpcie)
 		regulator_disable(imx_pcie->vpcie);
 }
@@ -1945,11 +1949,15 @@ static int imx_pcie_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	ret = pci_pwrctrl_create_devices(dev);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to create pwrctrl devices\n");
+
 	pci->use_parent_dt_ranges = true;
 	if (imx_pcie->drvdata->mode == DW_PCIE_EP_TYPE) {
 		ret = imx_add_pcie_ep(imx_pcie, pdev);
 		if (ret < 0)
-			return ret;
+			goto err_pwrctrl_destroy;
 
 		/*
 		 * FIXME: Only single Device (EPF) is supported due to the
@@ -1964,7 +1972,7 @@ static int imx_pcie_probe(struct platform_device *pdev)
 		pci->pp.use_atu_msg = true;
 		ret = dw_pcie_host_init(&pci->pp);
 		if (ret < 0)
-			return ret;
+			goto err_pwrctrl_destroy;
 
 		if (pci_msi_enabled()) {
 			u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_MSI);
@@ -1976,6 +1984,11 @@ static int imx_pcie_probe(struct platform_device *pdev)
 	}
 
 	return 0;
+
+err_pwrctrl_destroy:
+	if (ret != -EPROBE_DEFER)
+		pci_pwrctrl_destroy_devices(dev);
+	return ret;
 }
 
 static void imx_pcie_shutdown(struct platform_device *pdev)
diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c
index 278d2dba1db08..f1b27aed488de 100644
--- a/drivers/pci/controller/dwc/pci-keystone.c
+++ b/drivers/pci/controller/dwc/pci-keystone.c
@@ -1389,13 +1389,17 @@ static int ks_pcie_fault(unsigned long addr, unsigned int fsr,
 
 static int __init ks_pcie_init(void)
 {
+	struct device_node *np;
 	/*
 	 * PCIe access errors that result into OCP errors are caught by ARM as
 	 * "External aborts"
 	 */
-	if (of_find_matching_node(NULL, ks_pcie_of_match))
+	np = of_find_matching_node(NULL, ks_pcie_of_match);
+	if (np) {
+		of_node_put(np);
 		hook_fault_code(17, ks_pcie_fault, SIGBUS, 0,
 				"Asynchronous external abort");
+	}
 
 	return platform_driver_register(&ks_pcie_driver);
 }
diff --git a/drivers/pci/controller/dwc/pci-meson.c b/drivers/pci/controller/dwc/pci-meson.c
index 7a4da9fae1eaa..71b0ebc0e4727 100644
--- a/drivers/pci/controller/dwc/pci-meson.c
+++ b/drivers/pci/controller/dwc/pci-meson.c
@@ -401,7 +401,7 @@ static int meson_pcie_probe(struct platform_device *pdev)
 		return PTR_ERR(mp->phy);
 	}
 
-	mp->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);
+	mp->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
 	if (IS_ERR(mp->reset_gpio)) {
 		dev_err(dev, "get reset gpio failed\n");
 		return PTR_ERR(mp->reset_gpio);
diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
index 06722259d2e37..f5a38e6fd8d79 100644
--- a/drivers/pci/controller/dwc/pcie-designware-host.c
+++ b/drivers/pci/controller/dwc/pcie-designware-host.c
@@ -587,6 +587,12 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
 	if (ret)
 		return ret;
 
+	if (pci_msi_enabled()) {
+		pp->use_imsi_rx = !(pp->ops->msi_init ||
+				    of_property_present(np, "msi-parent") ||
+				    of_property_present(np, "msi-map"));
+	}
+
 	if (pp->ops->init) {
 		ret = pp->ops->init(pp);
 		if (ret)
@@ -594,10 +600,6 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
 	}
 
 	if (pci_msi_enabled()) {
-		pp->use_imsi_rx = !(pp->ops->msi_init ||
-				     of_property_present(np, "msi-parent") ||
-				     of_property_present(np, "msi-map"));
-
 		/*
 		 * For the use_imsi_rx case the default assignment is handled
 		 * in the dw_pcie_msi_host_init().
diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
index d8eb52857f69c..b193c989b2b80 100644
--- a/drivers/pci/controller/dwc/pcie-qcom.c
+++ b/drivers/pci/controller/dwc/pcie-qcom.c
@@ -1820,6 +1820,23 @@ static const struct pci_ecam_ops pci_qcom_ecam_ops = {
 	}
 };
 
+/* Check if @node is a child of @dev in DT */
+static bool qcom_pcie_is_child_node(struct device *dev,
+				    struct device_node *node)
+{
+	struct device_node *parent;
+
+	for (parent = of_get_parent(node); parent;
+	     parent = of_get_next_parent(parent)) {
+		if (parent == dev->of_node) {
+			of_node_put(parent);
+			return true;
+		}
+	}
+
+	return false;
+}
+
 /* Parse PERST# from all nodes in depth first manner starting from @np */
 static int qcom_pcie_parse_perst(struct qcom_pcie *pcie,
 				 struct qcom_pcie_port *port,
@@ -1827,6 +1844,7 @@ static int qcom_pcie_parse_perst(struct qcom_pcie *pcie,
 {
 	struct device *dev = pcie->pci->dev;
 	struct qcom_pcie_perst *perst;
+	struct device_node *gpio_np;
 	struct gpio_desc *reset;
 	int ret;
 
@@ -1840,6 +1858,25 @@ static int qcom_pcie_parse_perst(struct qcom_pcie *pcie,
 	if (!of_find_property(np, "reset-gpios", NULL))
 		goto parse_child_node;
 
+	/*
+	 * Skip GPIOs provided by a PCIe device which is a child of the Root
+	 * Complex (e.g., a PCIe switch with GPIO controller capability). Such
+	 * controllers won't be available at RC probe time and their PERST#
+	 * should be controlled by the respective PCI client driver
+	 * implementation.
+	 */
+	gpio_np = of_parse_phandle(np, "reset-gpios", 0);
+	if (!gpio_np) {
+		dev_err(dev, "Failed to parse GPIO provider\n");
+		return -EINVAL;
+	}
+
+	if (qcom_pcie_is_child_node(dev, gpio_np)) {
+		of_node_put(gpio_np);
+		goto parse_child_node;
+	}
+	of_node_put(gpio_np);
+
 	reset = devm_fwnode_gpiod_get(dev, of_fwnode_handle(np), "reset",
 				      GPIOD_OUT_HIGH, "PERST#");
 	if (IS_ERR(reset)) {
@@ -2282,6 +2319,7 @@ static int qcom_pcie_resume_noirq(struct device *dev)
 }
 
 static const struct of_device_id qcom_pcie_match[] = {
+	{ .compatible = "qcom,hawi-pcie", .data = &cfg_1_9_0 },
 	{ .compatible = "qcom,pcie-apq8064", .data = &cfg_2_1_0 },
 	{ .compatible = "qcom,pcie-apq8084", .data = &cfg_1_0_0 },
 	{ .compatible = "qcom,pcie-ipq4019", .data = &cfg_2_4_0 },
diff --git a/drivers/pci/controller/dwc/pcie-rcar-gen4.c b/drivers/pci/controller/dwc/pcie-rcar-gen4.c
index 8b03c42f8c84c..5f7211b91ee5b 100644
--- a/drivers/pci/controller/dwc/pcie-rcar-gen4.c
+++ b/drivers/pci/controller/dwc/pcie-rcar-gen4.c
@@ -13,8 +13,11 @@
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/iopoll.h>
+#include <linux/irqchip/arm-gic-v3.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
 #include <linux/pci.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
@@ -31,6 +34,10 @@
 #define DEVICE_TYPE_RC		BIT(4)
 #define BIFUR_MOD_SET_ON	BIT(0)
 
+/* MSI Capability */
+#define MSICAP0			0x0050
+#define MSICAP0_MSIE		BIT(16)
+
 /* PCIe Interrupt Status 0 */
 #define PCIEINTSTS0		0x0084
 
@@ -55,6 +62,14 @@
 #define APP_HOLD_PHY_RST	BIT(16)
 #define APP_LTSSM_ENABLE	BIT(0)
 
+/* INTC address */
+#define AXIINTCADDR		0x0a00
+
+/* INTC control & mask */
+#define AXIINTCCONT		0x0a04
+#define INTC_EN			BIT(31)
+#define INTC_MASK		GENMASK(11, 3)
+
 /* PCIe Power Management Control */
 #define PCIEPWRMNGCTRL		0x0070
 #define APP_CLK_REQ_N		BIT(11)
@@ -305,13 +320,103 @@ static struct rcar_gen4_pcie *rcar_gen4_pcie_alloc(struct platform_device *pdev)
 	return rcar;
 }
 
+static int rcar_gen4_pcie_host_msi_addr(struct dw_pcie_rp *pp, u32 *msi_addr)
+{
+	struct dw_pcie *dw = to_dw_pcie_from_pp(pp);
+	struct device_node *msi_node = NULL;
+	struct device *dev = dw->dev;
+	struct resource res;
+	u64 addr;
+	int ret;
+
+	/*
+	 * Either the "msi-parent" or the "msi-map" phandle needs to exist
+	 * to obtain the MSI node.
+	 */
+	of_msi_xlate(dev, &msi_node, 0);
+	if (!msi_node)
+		return -ENODEV;
+
+	/* Check if "msi-parent" or the "msi-map" points to ARM GICv3 ITS. */
+	if (!of_device_is_compatible(msi_node, "arm,gic-v3-its"))
+		return dev_err_probe(dev, -ENODEV, "Compatible MSI controller not found\n");
+
+	/* Derive GITS_TRANSLATER address from GICv3 */
+	ret = of_address_to_resource(msi_node, 0, &res);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "MSI controller resources not obtained\n");
+
+	addr = res.start + GITS_TRANSLATER;
+	if (addr >= SZ_4G)
+		return dev_err_probe(dev, -EINVAL, "MSI controller address above 32bit range\n");
+
+	*msi_addr = addr;
+	return 0;
+}
+
+static int rcar_gen4_pcie_host_msi_init(struct dw_pcie_rp *pp)
+{
+	struct dw_pcie *dw = to_dw_pcie_from_pp(pp);
+	struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw);
+	u32 val;
+	int ret;
+
+	/* Make sure MSICAP0 MSIE is configured. */
+	val = dw_pcie_readl_dbi(dw, MSICAP0);
+	if (pci_msi_enabled())
+		val |= MSICAP0_MSIE;
+	else
+		val &= ~MSICAP0_MSIE;
+	dw_pcie_writel_dbi(dw, MSICAP0, val);
+
+	if (!pci_msi_enabled() || pp->use_imsi_rx) {
+		/* Clear AXIINTC mapping. */
+		writel(0, rcar->base + AXIINTCADDR);
+		writel(0, rcar->base + AXIINTCCONT);
+	} else {
+		ret = rcar_gen4_pcie_host_msi_addr(pp, &val);
+		if (ret)
+			goto err;
+
+		/* Point AXIINTC to GIC ITS and enable. */
+		writel(val, rcar->base + AXIINTCADDR);
+		writel(INTC_EN | INTC_MASK, rcar->base + AXIINTCCONT);
+	}
+
+	/* Configure MSI interrupt signal */
+	val = readl(rcar->base + PCIEINTSTS0EN);
+	if (pci_msi_enabled())
+		val |= MSI_CTRL_INT;
+	else
+		val &= ~MSI_CTRL_INT;
+	writel(val, rcar->base + PCIEINTSTS0EN);
+
+	return 0;
+
+err:
+	/* Deconfigure MSICAP0 MSIE. */
+	val = dw_pcie_readl_dbi(dw, MSICAP0);
+	val &= ~MSICAP0_MSIE;
+	dw_pcie_writel_dbi(dw, MSICAP0, val);
+
+	/* Clear AXIINTC mapping. */
+	writel(0, rcar->base + AXIINTCADDR);
+	writel(0, rcar->base + AXIINTCCONT);
+
+	/* Deconfigure MSI interrupt signal */
+	val = readl(rcar->base + PCIEINTSTS0EN);
+	val &= ~MSI_CTRL_INT;
+	writel(val, rcar->base + PCIEINTSTS0EN);
+
+	return ret;
+}
+
 /* Host mode */
 static int rcar_gen4_pcie_host_init(struct dw_pcie_rp *pp)
 {
 	struct dw_pcie *dw = to_dw_pcie_from_pp(pp);
 	struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw);
 	int ret;
-	u32 val;
 
 	gpiod_set_value_cansleep(dw->pe_rst, 1);
 
@@ -328,16 +433,19 @@ static int rcar_gen4_pcie_host_init(struct dw_pcie_rp *pp)
 	dw_pcie_writel_dbi2(dw, PCI_BASE_ADDRESS_0, 0x0);
 	dw_pcie_writel_dbi2(dw, PCI_BASE_ADDRESS_1, 0x0);
 
-	/* Enable MSI interrupt signal */
-	val = readl(rcar->base + PCIEINTSTS0EN);
-	val |= MSI_CTRL_INT;
-	writel(val, rcar->base + PCIEINTSTS0EN);
+	ret = rcar_gen4_pcie_host_msi_init(pp);
+	if (ret)
+		goto err;
 
 	msleep(PCIE_T_PVPERL_MS);	/* pe_rst requires 100msec delay */
 
 	gpiod_set_value_cansleep(dw->pe_rst, 0);
 
 	return 0;
+
+err:
+	rcar_gen4_pcie_common_deinit(rcar);
+	return ret;
 }
 
 static void rcar_gen4_pcie_host_deinit(struct dw_pcie_rp *pp)
diff --git a/drivers/pci/controller/dwc/pcie-spacemit-k1.c b/drivers/pci/controller/dwc/pcie-spacemit-k1.c
index 04241df8fd595..0b187efe85510 100644
--- a/drivers/pci/controller/dwc/pcie-spacemit-k1.c
+++ b/drivers/pci/controller/dwc/pcie-spacemit-k1.c
@@ -341,6 +341,7 @@ static const struct of_device_id k1_pcie_of_match_table[] = {
 	{ .compatible = "spacemit,k1-pcie", },
 	{ }
 };
+MODULE_DEVICE_TABLE(of, k1_pcie_of_match_table);
 
 static struct platform_driver k1_pcie_driver = {
 	.probe	= k1_pcie_probe,
diff --git a/drivers/pci/controller/pci-host-common.c b/drivers/pci/controller/pci-host-common.c
index 2ce6f4b661334..da6636505c74d 100644
--- a/drivers/pci/controller/pci-host-common.c
+++ b/drivers/pci/controller/pci-host-common.c
@@ -108,8 +108,7 @@ static int pci_host_common_parse_perst(struct device *dev,
  * dependencies and the driver may fail to operate if required resources
  * are missing.
  *
- * Return: 0 on success, -ENODEV if PERST# found in RC node (legacy binding
- * should be used), Other negative error codes on failure.
+ * Return: 0 on success, negative error codes on failure.
  */
 static int pci_host_common_parse_port(struct device *dev,
 				      struct pci_host_bridge *bridge,
@@ -128,22 +127,6 @@ static int pci_host_common_parse_port(struct device *dev,
 	if (ret)
 		return ret;
 
-	/*
-	 * 1. PERST# found in RP or its child nodes - list is not empty,
-	 *    continue
-	 *
-	 * 2. PERST# not found in RP/children, but found in RC node -
-	 *    return -ENODEV to fallback legacy binding
-	 *
-	 * 3. PERST# not found anywhere - list is empty, continue (optional
-	 *    PERST#)
-	 */
-	if (list_empty(&port->perst)) {
-		if (of_property_present(dev->of_node, "reset-gpios") ||
-		    of_property_present(dev->of_node, "reset-gpio"))
-			return -ENODEV;
-	}
-
 	INIT_LIST_HEAD(&port->list);
 	list_add_tail(&port->list, &bridge->ports);
 
@@ -158,13 +141,11 @@ static int pci_host_common_parse_port(struct device *dev,
  * Iterate through child nodes of the host bridge and parse Root Port
  * properties (currently only reset GPIOs).
  *
- * Return: 0 on success, -ENODEV if no ports found or PERST# found in RC
- * node (legacy binding should be used), Other negative error codes on
- * failure.
+ * Return: 0 on success or ports not found, negative error codes on failure.
  */
 int pci_host_common_parse_ports(struct device *dev, struct pci_host_bridge *bridge)
 {
-	int ret = -ENODEV;
+	int ret = 0;
 
 	for_each_available_child_of_node_scoped(dev->of_node, of_port) {
 		if (!of_node_is_type(of_port, "pci"))
@@ -174,8 +155,8 @@ int pci_host_common_parse_ports(struct device *dev, struct pci_host_bridge *brid
 			goto err_cleanup;
 	}
 
-	if (ret)
-		return ret;
+	if (list_empty(&bridge->ports))
+		return 0;
 
 	return devm_add_action_or_reset(dev, pci_host_common_delete_ports,
 					&bridge->ports);
diff --git a/drivers/pci/controller/pci-xgene.c b/drivers/pci/controller/pci-xgene.c
index b95afa35201d0..83c9a2930eeca 100644
--- a/drivers/pci/controller/pci-xgene.c
+++ b/drivers/pci/controller/pci-xgene.c
@@ -58,7 +58,6 @@
 #define XGENE_PCIE_IP_VER_2		2
 
 struct xgene_pcie {
-	struct device_node	*node;
 	struct device		*dev;
 	struct clk		*clk;
 	void __iomem		*csr_base;
@@ -526,7 +525,7 @@ static void xgene_pcie_setup_ib_reg(struct xgene_pcie *port,
 
 static int xgene_pcie_parse_map_dma_ranges(struct xgene_pcie *port)
 {
-	struct device_node *np = port->node;
+	struct device_node *np = port->dev->of_node;
 	struct of_pci_range range;
 	struct of_pci_range_parser parser;
 	struct device *dev = port->dev;
@@ -612,7 +611,6 @@ static bool xgene_check_pcie_msi_ready(void)
 static int xgene_pcie_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct device_node *dn = dev->of_node;
 	struct xgene_pcie *port;
 	struct pci_host_bridge *bridge;
 	int ret;
@@ -627,7 +625,6 @@ static int xgene_pcie_probe(struct platform_device *pdev)
 
 	port = pci_host_bridge_priv(bridge);
 
-	port->node = of_node_get(dn);
 	port->dev = dev;
 	port->version = XGENE_PCIE_IP_VER_1;
 
diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c
index 8b57c3d75b942..a60d1ae076f80 100644
--- a/drivers/pci/controller/pcie-mediatek.c
+++ b/drivers/pci/controller/pcie-mediatek.c
@@ -10,11 +10,13 @@
 #include <linux/bitfield.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
+#include <linux/errno.h>
 #include <linux/iopoll.h>
 #include <linux/irq.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/irqchip/irq-msi-lib.h>
 #include <linux/irqdomain.h>
+#include <linux/kconfig.h>
 #include <linux/kernel.h>
 #include <linux/mfd/syscon.h>
 #include <linux/msi.h>
@@ -78,6 +80,7 @@
 
 #define PCIE_CONF_VEND_ID	0x100
 #define PCIE_CONF_DEVICE_ID	0x102
+#define PCIE_CONF_REV_CLASS	0x104
 #define PCIE_CONF_CLASS_ID	0x106
 
 #define PCIE_INT_MASK		0x420
@@ -90,6 +93,11 @@
 #define MSI_MASK		BIT(23)
 #define MTK_MSI_IRQS_NUM	32
 
+#define EN7528_HOST_MODE	0x00804201
+#define EN7528_LINKUP_REG	0x50
+#define EN7528_RC0_LINKUP	BIT(1)
+#define EN7528_RC1_LINKUP	BIT(2)
+
 #define PCIE_AHB_TRANS_BASE0_L	0x438
 #define PCIE_AHB_TRANS_BASE0_H	0x43c
 #define AHB2PCIE_SIZE(x)	((x) & GENMASK(4, 0))
@@ -145,12 +153,15 @@ struct mtk_pcie_port;
  * @MTK_PCIE_FIX_DEVICE_ID: host's device ID needed to be fixed
  * @MTK_PCIE_NO_MSI: Bridge has no MSI support, and relies on an external block
  * @MTK_PCIE_SKIP_RSTB: Skip calling RSTB bits on PCIe probe
+ * @MTK_PCIE_RETRAIN: Retrain link to bridge after startup because some
+ *                    Gen2-capable devices start as Gen1.
  */
 enum mtk_pcie_quirks {
 	MTK_PCIE_FIX_CLASS_ID = BIT(0),
 	MTK_PCIE_FIX_DEVICE_ID = BIT(1),
 	MTK_PCIE_NO_MSI = BIT(2),
 	MTK_PCIE_SKIP_RSTB = BIT(3),
+	MTK_PCIE_RETRAIN = BIT(4),
 };
 
 /**
@@ -756,6 +767,134 @@ static int mtk_pcie_startup_port_v2(struct mtk_pcie_port *port)
 	return 0;
 }
 
+static int mtk_pcie_startup_port_en7528(struct mtk_pcie_port *port)
+{
+	struct mtk_pcie *pcie = port->pcie;
+	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
+	struct resource *mem = NULL;
+	struct resource_entry *entry;
+	u32 val, link_mask;
+	int err;
+
+	entry = resource_list_first_type(&host->windows, IORESOURCE_MEM);
+	if (entry)
+		mem = entry->res;
+	if (!mem)
+		return -EINVAL;
+
+	if (!pcie->cfg) {
+		dev_err(pcie->dev, "EN7528: pciecfg syscon not available\n");
+		return -EINVAL;
+	}
+
+	/* Assert all reset signals */
+	writel(0, port->base + PCIE_RST_CTRL);
+
+	/*
+	 * Enable PCIe link down reset, if link status changed from link up to
+	 * link down, this will reset MAC control registers and configuration
+	 * space.
+	 */
+	writel(PCIE_LINKDOWN_RST_EN, port->base + PCIE_RST_CTRL);
+
+	msleep(PCIE_T_PVPERL_MS);
+
+	/* De-assert PHY, PE, PIPE, MAC and configuration reset */
+	val = readl(port->base + PCIE_RST_CTRL);
+	val |= PCIE_PHY_RSTB | PCIE_PERSTB | PCIE_PIPE_SRSTB |
+	       PCIE_MAC_SRSTB | PCIE_CRSTB;
+	writel(val, port->base + PCIE_RST_CTRL);
+
+	writel(PCIE_CLASS_CODE | PCIE_REVISION_ID,
+	       port->base + PCIE_CONF_REV_CLASS);
+	writel(EN7528_HOST_MODE, port->base);
+
+	link_mask = (port->slot == 0) ? EN7528_RC0_LINKUP : EN7528_RC1_LINKUP;
+
+	/* 100ms timeout value should be enough for Gen1/2 training */
+	err = regmap_read_poll_timeout(pcie->cfg, EN7528_LINKUP_REG, val,
+				       !!(val & link_mask), 20,
+				       PCI_PM_D3COLD_WAIT * USEC_PER_MSEC);
+	if (err) {
+		dev_err(pcie->dev, "EN7528: port%d link timeout\n", port->slot);
+		return -ETIMEDOUT;
+	}
+
+	/* Activate INTx interrupts */
+	val = readl(port->base + PCIE_INT_MASK);
+	val &= ~INTX_MASK;
+	writel(val, port->base + PCIE_INT_MASK);
+
+	if (IS_ENABLED(CONFIG_PCI_MSI))
+		mtk_pcie_enable_msi(port);
+
+	/* Set AHB to PCIe translation windows */
+	val = lower_32_bits(mem->start) |
+	      AHB2PCIE_SIZE(fls(resource_size(mem)));
+	writel(val, port->base + PCIE_AHB_TRANS_BASE0_L);
+
+	val = upper_32_bits(mem->start);
+	writel(val, port->base + PCIE_AHB_TRANS_BASE0_H);
+
+	writel(WIN_ENABLE, port->base + PCIE_AXI_WINDOW0);
+
+	if (!IS_BUILTIN(CONFIG_PCIE_MEDIATEK))
+		dev_info(pcie->dev,
+			 "module not built-in, Gen2 unavailable even if supported\n");
+
+	return 0;
+}
+
+/**
+ * mtk_pcie_retrain - retrain the root bridge link if needed
+ * @dev: The device, for use in logging
+ * @host: The host bridge which contains the link
+ *
+ * Due to what is likely a hardware bug, some devices (notably EcoNet) start up
+ * as Gen1, and must be retrained once after initial configuration in order to
+ * reach Gen2.
+ *
+ * These devices always self-identify as Gen2 capable, but sometimes the PHY is
+ * only capable of Gen1 operation, and sometimes the PCIe card (e.g. wifi) is
+ * only Gen1 capable. Therefore it is most convenient to retrain every port
+ * after startup.
+ */
+static int mtk_pcie_retrain(struct device *dev, struct pci_host_bridge *host)
+{
+	struct pci_dev *rp;
+	int ret = -ENOENT;
+	u16 lnksta = 0;
+	u32 speed;
+
+	/* Should already have been warned about during startup_port */
+	if (!IS_BUILTIN(CONFIG_PCIE_MEDIATEK))
+		return 0;
+
+	guard(rwsem_read)(&pci_bus_sem);
+
+	for_each_pci_bridge(rp, host->bus) {
+		if (pci_pcie_type(rp) != PCI_EXP_TYPE_ROOT_PORT)
+			continue;
+
+#if IS_BUILTIN(CONFIG_PCIE_MEDIATEK)
+		ret = pcie_retrain_link(rp, true);
+#endif
+
+		if (ret)
+			return dev_err_probe(&rp->dev, ret,
+					     "failed to retrain port\n");
+
+		pcie_capability_read_word(rp, PCI_EXP_LNKSTA, &lnksta);
+		speed = lnksta & PCI_EXP_LNKSTA_CLS;
+
+		pci_info(rp, "link retrained, speed %s\n",
+			 pci_speed_string(pcie_link_speed[speed]));
+
+	}
+
+	return 0;
+}
+
 static void __iomem *mtk_pcie_map_bus(struct pci_bus *bus,
 				      unsigned int devfn, int where)
 {
@@ -1169,6 +1308,13 @@ static int mtk_pcie_probe(struct platform_device *pdev)
 	if (err)
 		goto put_resources;
 
+	/*
+	 * Ignore error because pci_host_probe() was already called, and in any
+	 * case it is possible that the port will still work as Gen1.
+	 */
+	if (pcie->soc->quirks & MTK_PCIE_RETRAIN)
+		mtk_pcie_retrain(dev, host);
+
 	return 0;
 
 put_resources:
@@ -1290,8 +1436,16 @@ static const struct mtk_pcie_soc mtk_pcie_soc_mt7629 = {
 	.quirks = MTK_PCIE_FIX_CLASS_ID | MTK_PCIE_FIX_DEVICE_ID,
 };
 
+static const struct mtk_pcie_soc mtk_pcie_soc_en7528 = {
+	.ops = &mtk_pcie_ops_v2,
+	.startup = mtk_pcie_startup_port_en7528,
+	.setup_irq = mtk_pcie_setup_irq,
+	.quirks = MTK_PCIE_RETRAIN,
+};
+
 static const struct of_device_id mtk_pcie_ids[] = {
 	{ .compatible = "airoha,an7583-pcie", .data = &mtk_pcie_soc_an7583 },
+	{ .compatible = "econet,en7528-pcie", .data = &mtk_pcie_soc_en7528 },
 	{ .compatible = "mediatek,mt2701-pcie", .data = &mtk_pcie_soc_v1 },
 	{ .compatible = "mediatek,mt7623-pcie", .data = &mtk_pcie_soc_v1 },
 	{ .compatible = "mediatek,mt2712-pcie", .data = &mtk_pcie_soc_mt2712 },
diff --git a/drivers/pci/controller/pcie-rzg3s-host.c b/drivers/pci/controller/pcie-rzg3s-host.c
index 66f687304c1cb..a1a78bdb7af11 100644
--- a/drivers/pci/controller/pcie-rzg3s-host.c
+++ b/drivers/pci/controller/pcie-rzg3s-host.c
@@ -180,6 +180,16 @@
 /* Timeouts experimentally determined */
 #define RZG3S_REQ_ISSUE_TIMEOUT_US		2500
 
+/**
+ * enum rzg3s_sysc_link_mode - PCIe link configuration modes
+ * @RZG3S_SYSC_LINK_MODE_SINGLE_X4: Single port with x4 lanes
+ * @RZG3S_SYSC_LINK_MODE_DUAL_X2: Dual ports with x2 lanes each
+ */
+enum rzg3s_sysc_link_mode {
+	RZG3S_SYSC_LINK_MODE_SINGLE_X4 = 1,
+	RZG3S_SYSC_LINK_MODE_DUAL_X2 = 3,
+};
+
 /**
  * struct rzg3s_sysc_function - System Controller function descriptor
  * @offset: Register offset from the System Controller base address
@@ -195,12 +205,14 @@ struct rzg3s_sysc_function {
  * @RZG3S_SYSC_FUNC_ID_RST_RSM_B: RST_RSM_B SYSC function ID
  * @RZG3S_SYSC_FUNC_ID_L1_ALLOW: L1 allow SYSC function ID
  * @RZG3S_SYSC_FUNC_ID_MODE: Mode SYSC function ID
+ * @RZG3S_SYSC_FUNC_ID_LINK_MASTER: Link master SYSC function ID
  * @RZG3S_SYSC_FUNC_ID_MAX: Max SYSC function ID
  */
 enum rzg3s_sysc_func_id {
 	RZG3S_SYSC_FUNC_ID_RST_RSM_B,
 	RZG3S_SYSC_FUNC_ID_L1_ALLOW,
 	RZG3S_SYSC_FUNC_ID_MODE,
+	RZG3S_SYSC_FUNC_ID_LINK_MASTER,
 	RZG3S_SYSC_FUNC_ID_MAX,
 };
 
@@ -242,6 +254,18 @@ struct rzg3s_pcie_msi {
 	int irq;
 };
 
+/**
+ * enum rzg3s_pcie_controller_id - RZ/G3S PCIe controller IDs
+ * @RZG3S_PCIE_CONTROLLER_ID_0: PCIe controller 0
+ * @RZG3S_PCIE_CONTROLLER_ID_1: PCIe controller 1
+ * @RZG3S_PCIE_CONTROLLER_ID_MAX: Max PCIe controllers
+ */
+enum rzg3s_pcie_controller_id {
+	RZG3S_PCIE_CONTROLLER_ID_0,
+	RZG3S_PCIE_CONTROLLER_ID_1,
+	RZG3S_PCIE_CONTROLLER_ID_MAX,
+};
+
 struct rzg3s_pcie_host;
 
 /**
@@ -250,24 +274,28 @@ struct rzg3s_pcie_host;
  * @config_pre_init: Optional callback for SoC-specific pre-configuration
  * @config_post_init: Callback for SoC-specific post-configuration
  * @config_deinit: Callback for SoC-specific de-initialization
+ * @setup_lanes: Callback for setting up the number of lanes
  * @power_resets: array with the resets that need to be de-asserted after
  *                power-on
  * @cfg_resets: array with the resets that need to be de-asserted after
  *              configuration
- * @sysc_info: SYSC info
+ * @sysc_info: System Controller info for each controller
  * @num_power_resets: number of power resets
  * @num_cfg_resets: number of configuration resets
+ * @num_pcie_controllers: number of PCIe controllers
  */
 struct rzg3s_pcie_soc_data {
 	int (*init_phy)(struct rzg3s_pcie_host *host);
 	void (*config_pre_init)(struct rzg3s_pcie_host *host);
 	int (*config_post_init)(struct rzg3s_pcie_host *host);
 	int (*config_deinit)(struct rzg3s_pcie_host *host);
+	int (*setup_lanes)(struct rzg3s_pcie_host *host);
 	const char * const *power_resets;
 	const char * const *cfg_resets;
-	struct rzg3s_sysc_info sysc_info;
+	struct rzg3s_sysc_info sysc_info[RZG3S_PCIE_CONTROLLER_ID_MAX];
 	u8 num_power_resets;
 	u8 num_cfg_resets;
+	u8 num_pcie_controllers;
 };
 
 /**
@@ -297,6 +325,8 @@ struct rzg3s_pcie_port {
  * @hw_lock: lock for access to the HW resources
  * @intx_irqs: INTx interrupts
  * @max_link_speed: maximum supported link speed
+ * @controller_id: PCIe controller identifier, used for System Controller access
+ * @num_lanes: The number of lanes
  */
 struct rzg3s_pcie_host {
 	void __iomem *axi;
@@ -312,10 +342,24 @@ struct rzg3s_pcie_host {
 	raw_spinlock_t hw_lock;
 	int intx_irqs[PCI_NUM_INTX];
 	int max_link_speed;
+	enum rzg3s_pcie_controller_id controller_id;
+	u8 num_lanes;
 };
 
 #define rzg3s_msi_to_host(_msi)	container_of(_msi, struct rzg3s_pcie_host, msi)
 
+/*
+ * RZ/V2H(P) supports a total of 4 lanes shared across two controllers.
+ * rzv2h_lane_lock serialises both the counter update and the SYSC
+ * register write so that concurrent async probes cannot race on the
+ * shared LINK_MASTER register (offset 0x1060).
+ * rzv2h_num_total_lanes tracks global lane usage to prevent
+ * over-allocation or invalid bifurcation modes.
+ */
+#define RZV2H_PCIE_MAX_LANES		4
+static DEFINE_SPINLOCK(rzv2h_lane_lock);
+static u8 rzv2h_num_total_lanes;
+
 static int rzg3s_sysc_config_func(struct rzg3s_sysc *sysc,
 				  enum rzg3s_sysc_func_id fid, u32 val)
 {
@@ -1142,6 +1186,13 @@ static int rzg3s_pcie_config_init(struct rzg3s_pcie_host *host)
 	rzg3s_pcie_update_bits(host->pcie, PCI_CLASS_REVISION, mask,
 			       field_prep(mask, PCI_CLASS_BRIDGE_PCI_NORMAL));
 
+	if (host->num_lanes) {
+		rzg3s_pcie_update_bits(host->pcie + RZG3S_PCI_CFG_PCIEC,
+				       PCI_EXP_LNKCAP, PCI_EXP_LNKCAP_MLW,
+				       FIELD_PREP(PCI_EXP_LNKCAP_MLW,
+						  host->num_lanes));
+	}
+
 	/* Disable access control to the CFGU */
 	writel_relaxed(0, host->axi + RZG3S_PCI_PERM);
 
@@ -1277,9 +1328,9 @@ static int rzg3s_pcie_resets_prepare_and_get(struct rzg3s_pcie_host *host)
 	for (i = 0; i < data->num_cfg_resets; i++)
 		host->cfg_resets[i].id = data->cfg_resets[i];
 
-	ret = devm_reset_control_bulk_get_exclusive(host->dev,
-						    data->num_power_resets,
-						    host->power_resets);
+	ret = devm_reset_control_bulk_get_shared(host->dev,
+						 data->num_power_resets,
+						 host->power_resets);
 	if (ret)
 		return ret;
 
@@ -1674,6 +1725,76 @@ rzg3s_pcie_host_setup(struct rzg3s_pcie_host *host,
 	return ret;
 }
 
+static int rzg3s_pcie_get_controller_id(struct rzg3s_pcie_host *host)
+{
+	struct device_node *np = host->dev->of_node;
+	struct of_phandle_args sysc_args;
+	int ret;
+
+	if (host->data->num_pcie_controllers == 1)
+		return 0;
+
+	ret = of_parse_phandle_with_fixed_args(np, "renesas,sysc", 1, 0, &sysc_args);
+	if (ret)
+		return ret;
+
+	of_node_put(sysc_args.np);
+
+	if (sysc_args.args[0] >= host->data->num_pcie_controllers ||
+	    sysc_args.args[0] >= RZG3S_PCIE_CONTROLLER_ID_MAX)
+		return -EINVAL;
+
+	host->controller_id = sysc_args.args[0];
+
+	return 0;
+}
+
+static int rzv2h_pcie_setup_lanes(struct rzg3s_pcie_host *host)
+{
+	struct device_node *np = host->dev->of_node;
+	u32 num_lanes;
+	int ret;
+
+	ret = of_property_read_u32(np, "num-lanes", &num_lanes);
+	if (ret)
+		return ret;
+
+	/*
+	 * RZ/V2H(P) supports up to 4 lanes, but only in single x4 mode
+	 * for the first controller. Dual x2 mode is supported with 2
+	 * lanes for both controllers.
+	 */
+	if (num_lanes != 4 && num_lanes != 2)
+		return -EINVAL;
+
+	if (host->controller_id == RZG3S_PCIE_CONTROLLER_ID_1 && num_lanes > 2)
+		return -EINVAL;
+
+	guard(spinlock)(&rzv2h_lane_lock);
+	if (rzv2h_num_total_lanes + num_lanes > RZV2H_PCIE_MAX_LANES)
+		return -EINVAL;
+
+	ret = rzg3s_sysc_config_func(host->sysc, RZG3S_SYSC_FUNC_ID_LINK_MASTER,
+				     num_lanes == 2 ?
+				     RZG3S_SYSC_LINK_MODE_DUAL_X2 :
+				     RZG3S_SYSC_LINK_MODE_SINGLE_X4);
+	if (ret)
+		return ret;
+
+	rzv2h_num_total_lanes += num_lanes;
+	host->num_lanes = num_lanes;
+
+	return 0;
+}
+
+static void rzv2h_pcie_release_lanes(void *data)
+{
+	struct rzg3s_pcie_host *host = data;
+
+	guard(spinlock)(&rzv2h_lane_lock);
+	rzv2h_num_total_lanes -= host->num_lanes;
+}
+
 static int rzg3s_pcie_probe(struct platform_device *pdev)
 {
 	struct pci_host_bridge *bridge;
@@ -1698,8 +1819,12 @@ static int rzg3s_pcie_probe(struct platform_device *pdev)
 	if (!host->sysc)
 		return -ENOMEM;
 
+	ret = rzg3s_pcie_get_controller_id(host);
+	if (ret)
+		return ret;
+
 	sysc = host->sysc;
-	sysc->info = &host->data->sysc_info;
+	sysc->info = &host->data->sysc_info[host->controller_id];
 
 	host->axi = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(host->axi))
@@ -1727,6 +1852,16 @@ static int rzg3s_pcie_probe(struct platform_device *pdev)
 	if (ret)
 		goto port_refclk_put;
 
+	if (host->data->setup_lanes) {
+		ret = host->data->setup_lanes(host);
+		if (ret)
+			goto sysc_signal_restore;
+
+		ret = devm_add_action_or_reset(dev, rzv2h_pcie_release_lanes, host);
+		if (ret)
+			goto sysc_signal_restore;
+	}
+
 	ret = rzg3s_pcie_resets_prepare_and_get(host);
 	if (ret)
 		goto sysc_signal_restore;
@@ -1841,6 +1976,16 @@ static int rzg3s_pcie_resume_noirq(struct device *dev)
 	if (ret)
 		return ret;
 
+	if (host->num_lanes) {
+		ret = rzg3s_sysc_config_func(host->sysc,
+					     RZG3S_SYSC_FUNC_ID_LINK_MASTER,
+					     host->num_lanes == 2  ?
+					     RZG3S_SYSC_LINK_MODE_DUAL_X2 :
+					     RZG3S_SYSC_LINK_MODE_SINGLE_X4);
+		if (ret)
+			goto assert_rst_rsm_b;
+	}
+
 	ret = rzg3s_pcie_power_resets_deassert(host);
 	if (ret)
 		goto assert_rst_rsm_b;
@@ -1888,14 +2033,17 @@ static const struct rzg3s_pcie_soc_data rzg3s_soc_data = {
 	.num_power_resets = ARRAY_SIZE(rzg3s_soc_power_resets),
 	.cfg_resets = rzg3s_soc_cfg_resets,
 	.num_cfg_resets = ARRAY_SIZE(rzg3s_soc_cfg_resets),
+	.num_pcie_controllers = 1,
 	.config_post_init = rzg3s_pcie_config_post_init,
 	.config_deinit = rzg3s_pcie_config_deinit,
 	.init_phy = rzg3s_soc_pcie_init_phy,
 	.sysc_info = {
-		.functions = {
-			[RZG3S_SYSC_FUNC_ID_RST_RSM_B] = {
-				.offset = 0xd74,
-				.mask = BIT(0),
+		[RZG3S_PCIE_CONTROLLER_ID_0] = {
+			.functions = {
+				[RZG3S_SYSC_FUNC_ID_RST_RSM_B] = {
+					.offset = 0xd74,
+					.mask = BIT(0),
+				},
 			},
 		},
 	},
@@ -1906,18 +2054,65 @@ static const char * const rzg3e_soc_power_resets[] = { "aresetn" };
 static const struct rzg3s_pcie_soc_data rzg3e_soc_data = {
 	.power_resets = rzg3e_soc_power_resets,
 	.num_power_resets = ARRAY_SIZE(rzg3e_soc_power_resets),
+	.num_pcie_controllers = 1,
 	.config_pre_init = rzg3e_pcie_config_pre_init,
 	.config_post_init = rzg3e_pcie_config_post_init,
 	.config_deinit = rzg3e_pcie_config_deinit,
 	.sysc_info = {
-		.functions = {
-			[RZG3S_SYSC_FUNC_ID_L1_ALLOW] = {
-				.offset = 0x1020,
-				.mask = BIT(0),
+		[RZG3S_PCIE_CONTROLLER_ID_0] = {
+			.functions = {
+				[RZG3S_SYSC_FUNC_ID_L1_ALLOW] = {
+					.offset = 0x1020,
+					.mask = BIT(0),
+				},
+				[RZG3S_SYSC_FUNC_ID_MODE] = {
+					.offset = 0x1024,
+					.mask = BIT(0),
+				},
 			},
-			[RZG3S_SYSC_FUNC_ID_MODE] = {
-				.offset = 0x1024,
-				.mask = BIT(0),
+		},
+	},
+};
+
+static const struct rzg3s_pcie_soc_data rzv2h_soc_data = {
+	.power_resets = rzg3e_soc_power_resets,
+	.num_power_resets = ARRAY_SIZE(rzg3e_soc_power_resets),
+	.num_pcie_controllers = 2,
+	.config_pre_init = rzg3e_pcie_config_pre_init,
+	.config_post_init = rzg3e_pcie_config_post_init,
+	.config_deinit = rzg3e_pcie_config_deinit,
+	.setup_lanes = rzv2h_pcie_setup_lanes,
+	.sysc_info = {
+		[RZG3S_PCIE_CONTROLLER_ID_0] = {
+			.functions = {
+				[RZG3S_SYSC_FUNC_ID_L1_ALLOW] = {
+					.offset = 0x1020,
+					.mask = BIT(0),
+				},
+				[RZG3S_SYSC_FUNC_ID_MODE] = {
+					.offset = 0x1024,
+					.mask = BIT(0),
+				},
+				[RZG3S_SYSC_FUNC_ID_LINK_MASTER] = {
+					.offset = 0x1060,
+					.mask = GENMASK(9, 8),
+				},
+			},
+		},
+		[RZG3S_PCIE_CONTROLLER_ID_1] = {
+			.functions = {
+				[RZG3S_SYSC_FUNC_ID_L1_ALLOW] = {
+					.offset = 0x1050,
+					.mask = BIT(0),
+				},
+				[RZG3S_SYSC_FUNC_ID_MODE] = {
+					.offset = 0x1054,
+					.mask = BIT(0),
+				},
+				[RZG3S_SYSC_FUNC_ID_LINK_MASTER] = {
+					.offset = 0x1060,
+					.mask = GENMASK(9, 8),
+				},
 			},
 		},
 	},
@@ -1932,6 +2127,10 @@ static const struct of_device_id rzg3s_pcie_of_match[] = {
 		.compatible = "renesas,r9a09g047-pcie",
 		.data = &rzg3e_soc_data,
 	},
+	{
+		.compatible = "renesas,r9a09g057-pcie",
+		.data = &rzv2h_soc_data,
+	},
 	{}
 };
 
diff --git a/drivers/pci/endpoint/functions/pci-epf-mhi.c b/drivers/pci/endpoint/functions/pci-epf-mhi.c
index 7f5326925ed54..c3e3b58fb86cd 100644
--- a/drivers/pci/endpoint/functions/pci-epf-mhi.c
+++ b/drivers/pci/endpoint/functions/pci-epf-mhi.c
@@ -328,12 +328,6 @@ static int pci_epf_mhi_edma_read(struct mhi_ep_cntrl *mhi_cntrl,
 	config.direction = DMA_DEV_TO_MEM;
 	config.src_addr = buf_info->host_addr;
 
-	ret = dmaengine_slave_config(chan, &config);
-	if (ret) {
-		dev_err(dev, "Failed to configure DMA channel\n");
-		goto err_unlock;
-	}
-
 	dst_addr = dma_map_single(dma_dev, buf_info->dev_addr, buf_info->size,
 				  DMA_FROM_DEVICE);
 	ret = dma_mapping_error(dma_dev, dst_addr);
@@ -342,9 +336,10 @@ static int pci_epf_mhi_edma_read(struct mhi_ep_cntrl *mhi_cntrl,
 		goto err_unlock;
 	}
 
-	desc = dmaengine_prep_slave_single(chan, dst_addr, buf_info->size,
-					   DMA_DEV_TO_MEM,
-					   DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
+	desc = dmaengine_prep_config_single(chan, dst_addr, buf_info->size,
+					    DMA_DEV_TO_MEM,
+					    DMA_CTRL_ACK | DMA_PREP_INTERRUPT,
+					    &config);
 	if (!desc) {
 		dev_err(dev, "Failed to prepare DMA\n");
 		ret = -EIO;
@@ -401,12 +396,6 @@ static int pci_epf_mhi_edma_write(struct mhi_ep_cntrl *mhi_cntrl,
 	config.direction = DMA_MEM_TO_DEV;
 	config.dst_addr = buf_info->host_addr;
 
-	ret = dmaengine_slave_config(chan, &config);
-	if (ret) {
-		dev_err(dev, "Failed to configure DMA channel\n");
-		goto err_unlock;
-	}
-
 	src_addr = dma_map_single(dma_dev, buf_info->dev_addr, buf_info->size,
 				  DMA_TO_DEVICE);
 	ret = dma_mapping_error(dma_dev, src_addr);
@@ -415,9 +404,10 @@ static int pci_epf_mhi_edma_write(struct mhi_ep_cntrl *mhi_cntrl,
 		goto err_unlock;
 	}
 
-	desc = dmaengine_prep_slave_single(chan, src_addr, buf_info->size,
-					   DMA_MEM_TO_DEV,
-					   DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
+	desc = dmaengine_prep_config_single(chan, src_addr, buf_info->size,
+					    DMA_MEM_TO_DEV,
+					    DMA_CTRL_ACK | DMA_PREP_INTERRUPT,
+					    &config);
 	if (!desc) {
 		dev_err(dev, "Failed to prepare DMA\n");
 		ret = -EIO;
@@ -506,12 +496,6 @@ static int pci_epf_mhi_edma_read_async(struct mhi_ep_cntrl *mhi_cntrl,
 	config.direction = DMA_DEV_TO_MEM;
 	config.src_addr = buf_info->host_addr;
 
-	ret = dmaengine_slave_config(chan, &config);
-	if (ret) {
-		dev_err(dev, "Failed to configure DMA channel\n");
-		goto err_unlock;
-	}
-
 	dst_addr = dma_map_single(dma_dev, buf_info->dev_addr, buf_info->size,
 				  DMA_FROM_DEVICE);
 	ret = dma_mapping_error(dma_dev, dst_addr);
@@ -520,9 +504,10 @@ static int pci_epf_mhi_edma_read_async(struct mhi_ep_cntrl *mhi_cntrl,
 		goto err_unlock;
 	}
 
-	desc = dmaengine_prep_slave_single(chan, dst_addr, buf_info->size,
-					   DMA_DEV_TO_MEM,
-					   DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
+	desc = dmaengine_prep_config_single(chan, dst_addr, buf_info->size,
+					    DMA_DEV_TO_MEM,
+					    DMA_CTRL_ACK | DMA_PREP_INTERRUPT,
+					    &config);
 	if (!desc) {
 		dev_err(dev, "Failed to prepare DMA\n");
 		ret = -EIO;
@@ -585,12 +570,6 @@ static int pci_epf_mhi_edma_write_async(struct mhi_ep_cntrl *mhi_cntrl,
 	config.direction = DMA_MEM_TO_DEV;
 	config.dst_addr = buf_info->host_addr;
 
-	ret = dmaengine_slave_config(chan, &config);
-	if (ret) {
-		dev_err(dev, "Failed to configure DMA channel\n");
-		goto err_unlock;
-	}
-
 	src_addr = dma_map_single(dma_dev, buf_info->dev_addr, buf_info->size,
 				  DMA_TO_DEVICE);
 	ret = dma_mapping_error(dma_dev, src_addr);
@@ -599,9 +578,10 @@ static int pci_epf_mhi_edma_write_async(struct mhi_ep_cntrl *mhi_cntrl,
 		goto err_unlock;
 	}
 
-	desc = dmaengine_prep_slave_single(chan, src_addr, buf_info->size,
-					   DMA_MEM_TO_DEV,
-					   DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
+	desc = dmaengine_prep_config_single(chan, src_addr, buf_info->size,
+					    DMA_MEM_TO_DEV,
+					    DMA_CTRL_ACK | DMA_PREP_INTERRUPT,
+					    &config);
 	if (!desc) {
 		dev_err(dev, "Failed to prepare DMA\n");
 		ret = -EIO;
diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c
index 4802d4f80f780..d4905aa8e4c01 100644
--- a/drivers/pci/endpoint/functions/pci-epf-test.c
+++ b/drivers/pci/endpoint/functions/pci-epf-test.c
@@ -183,12 +183,8 @@ static int pci_epf_test_data_transfer(struct pci_epf_test *epf_test,
 		else
 			sconf.src_addr = dma_remote;
 
-		if (dmaengine_slave_config(chan, &sconf)) {
-			dev_err(dev, "DMA slave config fail\n");
-			return -EIO;
-		}
-		tx = dmaengine_prep_slave_single(chan, dma_local, len, dir,
-						 flags);
+		tx = dmaengine_prep_config_single(chan, dma_local, len,
+						  dir, flags, &sconf);
 	} else {
 		tx = dmaengine_prep_dma_memcpy(chan, dma_dst, dma_src, len,
 					       flags);
diff --git a/drivers/pci/hotplug/acpiphp_ibm.c b/drivers/pci/hotplug/acpiphp_ibm.c
index f8016ed245239..7337bd1ac6df7 100644
--- a/drivers/pci/hotplug/acpiphp_ibm.c
+++ b/drivers/pci/hotplug/acpiphp_ibm.c
@@ -258,8 +258,7 @@ static void ibm_handle_events(acpi_handle handle, u32 event, void *context)
 
 	if (subevent == 0x80) {
 		pr_debug("%s: generating bus event\n", __func__);
-		acpi_bus_generate_netlink_event(note->device->pnp.device_class,
-						  dev_name(&note->device->dev),
+		acpi_bus_generate_netlink_event("", dev_name(&note->device->dev),
 						  note->event, detail);
 	} else
 		note->event = event;
diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
index 5ec0b245a69bd..4d5dc62a7c860 100644
--- a/drivers/pci/pci-sysfs.c
+++ b/drivers/pci/pci-sysfs.c
@@ -112,7 +112,8 @@ static ssize_t pci_dev_show_local_cpu(struct device *dev, bool list,
 #else
 	mask = cpumask_of_pcibus(to_pci_dev(dev)->bus);
 #endif
-	return cpumap_print_to_pagebuf(list, buf, mask);
+	return sysfs_emit(buf, list ? "%*pbl\n" : "%*pb\n",
+			  cpumask_pr_args(mask));
 }
 
 static ssize_t local_cpus_show(struct device *dev,
@@ -137,7 +138,7 @@ static ssize_t cpuaffinity_show(struct device *dev,
 {
 	const struct cpumask *cpumask = cpumask_of_pcibus(to_pci_bus(dev));
 
-	return cpumap_print_to_pagebuf(false, buf, cpumask);
+	return sysfs_emit(buf, "%*pb\n", cpumask_pr_args(cpumask));
 }
 static DEVICE_ATTR_RO(cpuaffinity);
 
@@ -146,7 +147,7 @@ static ssize_t cpulistaffinity_show(struct device *dev,
 {
 	const struct cpumask *cpumask = cpumask_of_pcibus(to_pci_bus(dev));
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask));
 }
 static DEVICE_ATTR_RO(cpulistaffinity);
 
@@ -888,12 +889,30 @@ static ssize_t pci_read_legacy_io(struct file *filp, struct kobject *kobj,
 				  char *buf, loff_t off, size_t count)
 {
 	struct pci_bus *bus = to_pci_bus(kobj_to_dev(kobj));
+	u32 val = 0;
+	int ret;
 
 	/* Only support 1, 2 or 4 byte accesses */
 	if (count != 1 && count != 2 && count != 4)
 		return -EINVAL;
 
-	return pci_legacy_read(bus, off, (u32 *)buf, count);
+	ret = pci_legacy_read(bus, off, &val, count);
+	if (ret < 0)
+		return ret;
+
+	switch (count) {
+	case 1:
+		buf[0] = *(u8 *)&val;
+		break;
+	case 2:
+		put_unaligned_le16(*(u16 *)&val, buf);
+		break;
+	case 4:
+		put_unaligned_le32(val, buf);
+		break;
+	}
+
+	return ret;
 }
 
 /**
@@ -913,12 +932,24 @@ static ssize_t pci_write_legacy_io(struct file *filp, struct kobject *kobj,
 				   char *buf, loff_t off, size_t count)
 {
 	struct pci_bus *bus = to_pci_bus(kobj_to_dev(kobj));
+	u32 val;
 
-	/* Only support 1, 2 or 4 byte accesses */
-	if (count != 1 && count != 2 && count != 4)
+	/* Only support 1, 2 or 4 byte accesses. */
+	switch (count) {
+	case 1:
+		val = *(u8 *)buf;
+		break;
+	case 2:
+		val = get_unaligned_le16(buf);
+		break;
+	case 4:
+		val = get_unaligned_le32(buf);
+		break;
+	default:
 		return -EINVAL;
+	}
 
-	return pci_legacy_write(bus, off, *(u32 *)buf, count);
+	return pci_legacy_write(bus, off, val, count);
 }
 
 /**
diff --git a/drivers/pci/pcie/portdrv.c b/drivers/pci/pcie/portdrv.c
index 2d6aa488fe7b6..65f502602dee3 100644
--- a/drivers/pci/pcie/portdrv.c
+++ b/drivers/pci/pcie/portdrv.c
@@ -264,7 +264,7 @@ static int get_port_device_capability(struct pci_dev *dev)
 	 */
 	if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DPC) &&
 	    pci_aer_available() &&
-	    (pcie_ports_dpc_native || (services & PCIE_PORT_SERVICE_AER)))
+	    (pcie_ports_dpc_native || host->native_aer))
 		services |= PCIE_PORT_SERVICE_DPC;
 
 	/* Enable bandwidth control if more than one speed is supported. */
diff --git a/drivers/perf/alibaba_uncore_drw_pmu.c b/drivers/perf/alibaba_uncore_drw_pmu.c
index ac49d3b2dad6a..74786a5dd6a23 100644
--- a/drivers/perf/alibaba_uncore_drw_pmu.c
+++ b/drivers/perf/alibaba_uncore_drw_pmu.c
@@ -221,7 +221,7 @@ static ssize_t ali_drw_pmu_cpumask_show(struct device *dev,
 {
 	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(dev_get_drvdata(dev));
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(drw_pmu->cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(drw_pmu->cpu)));
 }
 
 static struct device_attribute ali_drw_pmu_cpumask_attr =
diff --git a/drivers/perf/amlogic/meson_ddr_pmu_core.c b/drivers/perf/amlogic/meson_ddr_pmu_core.c
index c1e755c356a33..f614aa3434a51 100644
--- a/drivers/perf/amlogic/meson_ddr_pmu_core.c
+++ b/drivers/perf/amlogic/meson_ddr_pmu_core.c
@@ -191,7 +191,7 @@ static ssize_t meson_ddr_perf_cpumask_show(struct device *dev,
 {
 	struct ddr_pmu *pmu = dev_get_drvdata(dev);
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(pmu->cpu)));
 }
 
 static struct device_attribute meson_ddr_perf_cpumask_attr =
diff --git a/drivers/perf/arm-cci.c b/drivers/perf/arm-cci.c
index 1cc3214d6b6da..f0ef0a679e746 100644
--- a/drivers/perf/arm-cci.c
+++ b/drivers/perf/arm-cci.c
@@ -1351,7 +1351,7 @@ static ssize_t pmu_cpumask_attr_show(struct device *dev,
 	struct pmu *pmu = dev_get_drvdata(dev);
 	struct cci_pmu *cci_pmu = to_cci_pmu(pmu);
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(cci_pmu->cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(cci_pmu->cpu)));
 }
 
 static struct device_attribute pmu_cpumask_attr =
diff --git a/drivers/perf/arm-ccn.c b/drivers/perf/arm-ccn.c
index c18a0e3205abf..c40da3450c8fd 100644
--- a/drivers/perf/arm-ccn.c
+++ b/drivers/perf/arm-ccn.c
@@ -537,7 +537,7 @@ static ssize_t arm_ccn_pmu_cpumask_show(struct device *dev,
 {
 	struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev));
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(ccn->dt.cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(ccn->dt.cpu)));
 }
 
 static struct device_attribute arm_ccn_pmu_cpumask_attr =
diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c
index 6e5cc4086a9e2..ff8bbf07c94e9 100644
--- a/drivers/perf/arm-cmn.c
+++ b/drivers/perf/arm-cmn.c
@@ -1327,7 +1327,7 @@ static ssize_t arm_cmn_cpumask_show(struct device *dev,
 {
 	struct arm_cmn *cmn = to_cmn(dev_get_drvdata(dev));
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(cmn->cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(cmn->cpu)));
 }
 
 static struct device_attribute arm_cmn_cpumask_attr =
diff --git a/drivers/perf/arm-ni.c b/drivers/perf/arm-ni.c
index 66858c65215d2..03a1c6bf9223a 100644
--- a/drivers/perf/arm-ni.c
+++ b/drivers/perf/arm-ni.c
@@ -239,7 +239,7 @@ static ssize_t arm_ni_cpumask_show(struct device *dev,
 {
 	struct arm_ni *ni = cd_to_ni(pmu_to_cd(dev_get_drvdata(dev)));
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(ni->cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(ni->cpu)));
 }
 
 static struct device_attribute arm_ni_cpumask_attr =
diff --git a/drivers/perf/arm_cspmu/arm_cspmu.c b/drivers/perf/arm_cspmu/arm_cspmu.c
index 80fb314d51359..e6292021f653e 100644
--- a/drivers/perf/arm_cspmu/arm_cspmu.c
+++ b/drivers/perf/arm_cspmu/arm_cspmu.c
@@ -305,7 +305,7 @@ static ssize_t arm_cspmu_cpumask_show(struct device *dev,
 	default:
 		return 0;
 	}
-	return cpumap_print_to_pagebuf(true, buf, cpumask);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask));
 }
 
 static struct attribute *arm_cspmu_cpumask_attrs[] = {
diff --git a/drivers/perf/arm_dmc620_pmu.c b/drivers/perf/arm_dmc620_pmu.c
index 4f6b196160f81..467147a05eec3 100644
--- a/drivers/perf/arm_dmc620_pmu.c
+++ b/drivers/perf/arm_dmc620_pmu.c
@@ -237,8 +237,8 @@ static ssize_t dmc620_pmu_cpumask_show(struct device *dev,
 {
 	struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(dev_get_drvdata(dev));
 
-	return cpumap_print_to_pagebuf(true, buf,
-				       cpumask_of(dmc620_pmu->irq->cpu));
+	return sysfs_emit(buf, "%*pbl\n",
+			  cpumask_pr_args(cpumask_of(dmc620_pmu->irq->cpu)));
 }
 
 static struct device_attribute dmc620_pmu_cpumask_attr =
diff --git a/drivers/perf/arm_dsu_pmu.c b/drivers/perf/arm_dsu_pmu.c
index 32b0dd7c693b3..bcbd19e075a57 100644
--- a/drivers/perf/arm_dsu_pmu.c
+++ b/drivers/perf/arm_dsu_pmu.c
@@ -157,7 +157,7 @@ static ssize_t dsu_pmu_cpumask_show(struct device *dev,
 	default:
 		return 0;
 	}
-	return cpumap_print_to_pagebuf(true, buf, cpumask);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask));
 }
 
 static struct attribute *dsu_pmu_format_attrs[] = {
diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c
index 939bcbd433aab..51ab6cc52ca02 100644
--- a/drivers/perf/arm_pmu.c
+++ b/drivers/perf/arm_pmu.c
@@ -570,7 +570,7 @@ static ssize_t cpus_show(struct device *dev,
 			 struct device_attribute *attr, char *buf)
 {
 	struct arm_pmu *armpmu = to_arm_pmu(dev_get_drvdata(dev));
-	return cpumap_print_to_pagebuf(true, buf, &armpmu->supported_cpus);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&armpmu->supported_cpus));
 }
 
 static DEVICE_ATTR_RO(cpus);
diff --git a/drivers/perf/arm_smmuv3_pmu.c b/drivers/perf/arm_smmuv3_pmu.c
index 621f02a7f43be..8ce34e6bb82b1 100644
--- a/drivers/perf/arm_smmuv3_pmu.c
+++ b/drivers/perf/arm_smmuv3_pmu.c
@@ -537,7 +537,7 @@ static ssize_t smmu_pmu_cpumask_show(struct device *dev,
 {
 	struct smmu_pmu *smmu_pmu = to_smmu_pmu(dev_get_drvdata(dev));
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(smmu_pmu->on_cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(smmu_pmu->on_cpu)));
 }
 
 static struct device_attribute smmu_pmu_cpumask_attr =
diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c
index dbd0da1116390..9f786fd48cdd9 100644
--- a/drivers/perf/arm_spe_pmu.c
+++ b/drivers/perf/arm_spe_pmu.c
@@ -343,7 +343,7 @@ static ssize_t cpumask_show(struct device *dev,
 {
 	struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev);
 
-	return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&spe_pmu->supported_cpus));
 }
 static DEVICE_ATTR_RO(cpumask);
 
diff --git a/drivers/perf/cxl_pmu.c b/drivers/perf/cxl_pmu.c
index 68a54d97d2a8a..0735eb33f5f36 100644
--- a/drivers/perf/cxl_pmu.c
+++ b/drivers/perf/cxl_pmu.c
@@ -493,7 +493,7 @@ static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
 {
 	struct cxl_pmu_info *info = dev_get_drvdata(dev);
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(info->on_cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(info->on_cpu)));
 }
 static DEVICE_ATTR_RO(cpumask);
 
diff --git a/drivers/perf/dwc_pcie_pmu.c b/drivers/perf/dwc_pcie_pmu.c
index 5385401fa9cf6..291e776d6f6a5 100644
--- a/drivers/perf/dwc_pcie_pmu.c
+++ b/drivers/perf/dwc_pcie_pmu.c
@@ -117,7 +117,7 @@ static ssize_t cpumask_show(struct device *dev,
 {
 	struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(dev_get_drvdata(dev));
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(pcie_pmu->on_cpu)));
 }
 static DEVICE_ATTR_RO(cpumask);
 
diff --git a/drivers/perf/fsl_imx8_ddr_perf.c b/drivers/perf/fsl_imx8_ddr_perf.c
index bcdf5575d71ca..3760ebe026749 100644
--- a/drivers/perf/fsl_imx8_ddr_perf.c
+++ b/drivers/perf/fsl_imx8_ddr_perf.c
@@ -237,7 +237,7 @@ static ssize_t ddr_perf_cpumask_show(struct device *dev,
 {
 	struct ddr_pmu *pmu = dev_get_drvdata(dev);
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(pmu->cpu)));
 }
 
 static struct device_attribute ddr_perf_cpumask_attr =
diff --git a/drivers/perf/fsl_imx9_ddr_perf.c b/drivers/perf/fsl_imx9_ddr_perf.c
index 7050b48c04671..6fee5eb5087aa 100644
--- a/drivers/perf/fsl_imx9_ddr_perf.c
+++ b/drivers/perf/fsl_imx9_ddr_perf.c
@@ -159,7 +159,7 @@ static ssize_t ddr_perf_cpumask_show(struct device *dev,
 {
 	struct ddr_pmu *pmu = dev_get_drvdata(dev);
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(pmu->cpu)));
 }
 
 static struct device_attribute ddr_perf_cpumask_attr =
diff --git a/drivers/perf/fujitsu_uncore_pmu.c b/drivers/perf/fujitsu_uncore_pmu.c
index aeeb68c66e1e6..ede652f757da0 100644
--- a/drivers/perf/fujitsu_uncore_pmu.c
+++ b/drivers/perf/fujitsu_uncore_pmu.c
@@ -373,7 +373,7 @@ static ssize_t cpumask_show(struct device *dev,
 {
 	struct uncore_pmu *uncorepmu = to_uncore_pmu(dev_get_drvdata(dev));
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(uncorepmu->cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(uncorepmu->cpu)));
 }
 static DEVICE_ATTR_RO(cpumask);
 
diff --git a/drivers/perf/hisilicon/hisi_pcie_pmu.c b/drivers/perf/hisilicon/hisi_pcie_pmu.c
index c5394d007b61b..0f55d871c67e7 100644
--- a/drivers/perf/hisilicon/hisi_pcie_pmu.c
+++ b/drivers/perf/hisilicon/hisi_pcie_pmu.c
@@ -121,7 +121,7 @@ static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, c
 {
 	struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev));
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(pcie_pmu->on_cpu)));
 }
 static DEVICE_ATTR_RO(cpumask);
 
diff --git a/drivers/perf/hisilicon/hisi_uncore_pmu.c b/drivers/perf/hisilicon/hisi_uncore_pmu.c
index de71dcf116538..0ff2fdf4b3e2a 100644
--- a/drivers/perf/hisilicon/hisi_uncore_pmu.c
+++ b/drivers/perf/hisilicon/hisi_uncore_pmu.c
@@ -56,7 +56,7 @@ static ssize_t hisi_associated_cpus_sysfs_show(struct device *dev,
 {
 	struct hisi_pmu *hisi_pmu = to_hisi_pmu(dev_get_drvdata(dev));
 
-	return cpumap_print_to_pagebuf(true, buf, &hisi_pmu->associated_cpus);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&hisi_pmu->associated_cpus));
 }
 static DEVICE_ATTR(associated_cpus, 0444, hisi_associated_cpus_sysfs_show, NULL);
 
diff --git a/drivers/perf/marvell_cn10k_ddr_pmu.c b/drivers/perf/marvell_cn10k_ddr_pmu.c
index 72ac17efd846a..8681e8715cb37 100644
--- a/drivers/perf/marvell_cn10k_ddr_pmu.c
+++ b/drivers/perf/marvell_cn10k_ddr_pmu.c
@@ -364,7 +364,7 @@ static ssize_t cn10k_ddr_perf_cpumask_show(struct device *dev,
 {
 	struct cn10k_ddr_pmu *pmu = dev_get_drvdata(dev);
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(pmu->cpu)));
 }
 
 static struct device_attribute cn10k_ddr_perf_cpumask_attr =
diff --git a/drivers/perf/marvell_cn10k_tad_pmu.c b/drivers/perf/marvell_cn10k_tad_pmu.c
index 51ccb0befa05d..54909d0031b77 100644
--- a/drivers/perf/marvell_cn10k_tad_pmu.c
+++ b/drivers/perf/marvell_cn10k_tad_pmu.c
@@ -258,7 +258,7 @@ static ssize_t tad_pmu_cpumask_show(struct device *dev,
 {
 	struct tad_pmu *tad_pmu = to_tad_pmu(dev_get_drvdata(dev));
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(tad_pmu->cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(tad_pmu->cpu)));
 }
 
 static DEVICE_ATTR(cpumask, 0444, tad_pmu_cpumask_show, NULL);
diff --git a/drivers/perf/marvell_pem_pmu.c b/drivers/perf/marvell_pem_pmu.c
index 29fbcd1848e44..cf1d8cdb13186 100644
--- a/drivers/perf/marvell_pem_pmu.c
+++ b/drivers/perf/marvell_pem_pmu.c
@@ -164,7 +164,7 @@ static ssize_t pem_perf_cpumask_show(struct device *dev,
 {
 	struct pem_pmu *pmu = dev_get_drvdata(dev);
 
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(pmu->cpu)));
 }
 
 static struct device_attribute pem_perf_cpumask_attr =
diff --git a/drivers/perf/nvidia_t410_c2c_pmu.c b/drivers/perf/nvidia_t410_c2c_pmu.c
index 411987153ff3f..bff875f4f625d 100644
--- a/drivers/perf/nvidia_t410_c2c_pmu.c
+++ b/drivers/perf/nvidia_t410_c2c_pmu.c
@@ -658,7 +658,7 @@ static ssize_t nv_c2c_pmu_cpumask_show(struct device *dev,
 	default:
 		return 0;
 	}
-	return cpumap_print_to_pagebuf(true, buf, cpumask);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask));
 }
 
 #define NV_C2C_PMU_CPUMASK_ATTR(_name, _config)			\
diff --git a/drivers/perf/nvidia_t410_cmem_latency_pmu.c b/drivers/perf/nvidia_t410_cmem_latency_pmu.c
index acb8f5571522c..6c8e41598ec1d 100644
--- a/drivers/perf/nvidia_t410_cmem_latency_pmu.c
+++ b/drivers/perf/nvidia_t410_cmem_latency_pmu.c
@@ -501,7 +501,7 @@ static ssize_t cmem_lat_pmu_cpumask_show(struct device *dev,
 	default:
 		return 0;
 	}
-	return cpumap_print_to_pagebuf(true, buf, cpumask);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask));
 }
 
 #define NV_PMU_CPUMASK_ATTR(_name, _config)			\
diff --git a/drivers/perf/qcom_l2_pmu.c b/drivers/perf/qcom_l2_pmu.c
index ea8c857299373..c0c522b10b72e 100644
--- a/drivers/perf/qcom_l2_pmu.c
+++ b/drivers/perf/qcom_l2_pmu.c
@@ -638,7 +638,7 @@ static ssize_t l2_cache_pmu_cpumask_show(struct device *dev,
 {
 	struct l2cache_pmu *l2cache_pmu = to_l2cache_pmu(dev_get_drvdata(dev));
 
-	return cpumap_print_to_pagebuf(true, buf, &l2cache_pmu->cpumask);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&l2cache_pmu->cpumask));
 }
 
 static struct device_attribute l2_cache_pmu_cpumask_attr =
diff --git a/drivers/perf/qcom_l3_pmu.c b/drivers/perf/qcom_l3_pmu.c
index 66e6cabd6fff1..c8d259dd1f807 100644
--- a/drivers/perf/qcom_l3_pmu.c
+++ b/drivers/perf/qcom_l3_pmu.c
@@ -663,7 +663,7 @@ static ssize_t cpumask_show(struct device *dev,
 {
 	struct l3cache_pmu *l3pmu = to_l3cache_pmu(dev_get_drvdata(dev));
 
-	return cpumap_print_to_pagebuf(true, buf, &l3pmu->cpumask);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&l3pmu->cpumask));
 }
 
 static DEVICE_ATTR_RO(cpumask);
diff --git a/drivers/perf/starfive_starlink_pmu.c b/drivers/perf/starfive_starlink_pmu.c
index b1c7dc4869bd9..dc4e653761fcc 100644
--- a/drivers/perf/starfive_starlink_pmu.c
+++ b/drivers/perf/starfive_starlink_pmu.c
@@ -130,7 +130,7 @@ cpumask_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct starlink_pmu *starlink_pmu = to_starlink_pmu(dev_get_drvdata(dev));
 
-	return cpumap_print_to_pagebuf(true, buf, &starlink_pmu->cpumask);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&starlink_pmu->cpumask));
 }
 
 static DEVICE_ATTR_RO(cpumask);
diff --git a/drivers/perf/thunderx2_pmu.c b/drivers/perf/thunderx2_pmu.c
index 6ed4707bd6bb5..a69c02d2d874f 100644
--- a/drivers/perf/thunderx2_pmu.c
+++ b/drivers/perf/thunderx2_pmu.c
@@ -254,7 +254,7 @@ static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
 	struct tx2_uncore_pmu *tx2_pmu;
 
 	tx2_pmu = pmu_to_tx2_pmu(dev_get_drvdata(dev));
-	return cpumap_print_to_pagebuf(true, buf, cpumask_of(tx2_pmu->cpu));
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(tx2_pmu->cpu)));
 }
 static DEVICE_ATTR_RO(cpumask);
 
diff --git a/drivers/perf/xgene_pmu.c b/drivers/perf/xgene_pmu.c
index 33b5497bdc06e..e9e4871db08d4 100644
--- a/drivers/perf/xgene_pmu.c
+++ b/drivers/perf/xgene_pmu.c
@@ -595,7 +595,7 @@ static ssize_t cpumask_show(struct device *dev,
 {
 	struct xgene_pmu_dev *pmu_dev = to_pmu_dev(dev_get_drvdata(dev));
 
-	return cpumap_print_to_pagebuf(true, buf, &pmu_dev->parent->cpu);
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&pmu_dev->parent->cpu));
 }
 
 static DEVICE_ATTR_RO(cpumask);
diff --git a/drivers/phy/motorola/phy-cpcap-usb.c b/drivers/phy/motorola/phy-cpcap-usb.c
index 7cb020dd3423c..9591672b0511c 100644
--- a/drivers/phy/motorola/phy-cpcap-usb.c
+++ b/drivers/phy/motorola/phy-cpcap-usb.c
@@ -717,3 +717,4 @@ MODULE_ALIAS("platform:cpcap_usb");
 MODULE_AUTHOR("Tony Lindgren <tony@atomide.com>");
 MODULE_DESCRIPTION("CPCAP usb phy driver");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/pinctrl/aspeed/pinctrl-aspeed-g7-soc1.c b/drivers/pinctrl/aspeed/pinctrl-aspeed-g7-soc1.c
index 50027d69c3428..f8b4066699ce5 100644
--- a/drivers/pinctrl/aspeed/pinctrl-aspeed-g7-soc1.c
+++ b/drivers/pinctrl/aspeed/pinctrl-aspeed-g7-soc1.c
@@ -1018,7 +1018,8 @@ PIN_GROUP(I3C6, AA22, AB20);
 PIN_GROUP(I3C7, AF18, AE19);
 PIN_GROUP(I3C8, AD20, AC20);
 PIN_GROUP(I3C9, AA21, AB21);
-PIN_GROUP(JTAGM1, D12, F10, E11, F11, F13);
+PIN_GROUP(JTAGM1, F10, E11, F11, F13);
+PIN_GROUP(JTAGM1TRST, D12);
 PIN_GROUP(LPC0, AF26, AF25, B16, D14, B15, B14, C17, B13, E14, C15);
 PIN_GROUP(LPC1, C16, C14, C11, D9, F14, D10, C12, C13, AE16, AE17);
 PIN_GROUP(LTPI, U25, U26, Y26, AA24);
@@ -1263,6 +1264,7 @@ static const struct pingroup aspeed_g7_soc1_groups[] = {
 	GROUP(I3C8),
 	GROUP(I3C9),
 	GROUP(JTAGM1),
+	GROUP(JTAGM1TRST),
 	GROUP(LPC0),
 	GROUP(LPC1),
 	GROUP(LTPI),
@@ -1528,7 +1530,7 @@ static const struct aspeed_g7_soc1_function aspeed_g7_soc1_functions[] = {
 	FUNC(I3C7, (1), "I3C7"),
 	FUNC(I3C8, (1), "I3C8"),
 	FUNC(I3C9, (1), "I3C9"),
-	FUNC(JTAGM1, (1), "JTAGM1"),
+	FUNC(JTAGM1, (1, 1), "JTAGM1", "JTAGM1TRST"),
 	FUNC(LPC0, (2), "LPC0"),
 	FUNC(LPC1, (2), "LPC1"),
 	FUNC(LTPI, (2), "LTPI"),
diff --git a/drivers/pinctrl/bcm/pinctrl-bcm2835.c b/drivers/pinctrl/bcm/pinctrl-bcm2835.c
index e7b35019a5a7d..725e880ae086f 100644
--- a/drivers/pinctrl/bcm/pinctrl-bcm2835.c
+++ b/drivers/pinctrl/bcm/pinctrl-bcm2835.c
@@ -1350,7 +1350,6 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev)
 	pc->pctl_desc = *pdata->pctl_desc;
 	pc->pctl_dev = devm_pinctrl_register(dev, &pc->pctl_desc, pc);
 	if (IS_ERR(pc->pctl_dev)) {
-		gpiochip_remove(&pc->gpio_chip);
 		return PTR_ERR(pc->pctl_dev);
 	}
 
diff --git a/drivers/pinctrl/freescale/pinctrl-imx1-core.c b/drivers/pinctrl/freescale/pinctrl-imx1-core.c
index b7bd4ef9c0db5..4a6bdaefa42f2 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx1-core.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx1-core.c
@@ -547,14 +547,11 @@ static int imx1_pinctrl_parse_functions(struct device_node *np,
  */
 static bool imx1_pinctrl_dt_is_flat_functions(struct device_node *np)
 {
-	struct device_node *function_np;
-	struct device_node *pinctrl_np;
-
-	for_each_child_of_node(np, function_np) {
+	for_each_child_of_node_scoped(np, function_np) {
 		if (of_property_present(function_np, "fsl,pins"))
 			return true;
 
-		for_each_child_of_node(function_np, pinctrl_np) {
+		for_each_child_of_node_scoped(function_np, pinctrl_np) {
 			if (of_property_present(pinctrl_np, "fsl,pins"))
 				return false;
 		}
diff --git a/drivers/pinctrl/intel/pinctrl-tigerlake.c b/drivers/pinctrl/intel/pinctrl-tigerlake.c
index 3ac35a153e8ca..93234f4a9bd07 100644
--- a/drivers/pinctrl/intel/pinctrl-tigerlake.c
+++ b/drivers/pinctrl/intel/pinctrl-tigerlake.c
@@ -329,6 +329,34 @@ static const struct pinctrl_pin_desc tgllp_pins[] = {
 	PINCTRL_PIN(276, "SPI0_CLK_LOOPBK"),
 };
 
+static const unsigned int tgllp_i2c0_pins[] = { 5, 6 };
+static const unsigned int tgllp_i2c1_pins[] = { 7, 8 };
+static const unsigned int tgllp_pwm0_pins[] = { 99 };
+static const unsigned int tgllp_uart1_pins[] = { 85, 86, 87, 88 };
+static const unsigned int tgllp_ssp2_pins[] = { 108, 109, 110, 111 };
+
+static const struct intel_pingroup tgllp_groups[] = {
+	PIN_GROUP("i2c0_grp", tgllp_i2c0_pins, 2),
+	PIN_GROUP("i2c1_grp", tgllp_i2c1_pins, 2),
+	PIN_GROUP("pwm0_grp", tgllp_pwm0_pins, 2),
+	PIN_GROUP("uart1_grp", tgllp_uart1_pins, 2),
+	PIN_GROUP("ssp2_grp", tgllp_ssp2_pins, 7),
+};
+
+static const char * const tgllp_i2c0_groups[] = { "i2c0_grp" };
+static const char * const tgllp_i2c1_groups[] = { "i2c1_grp" };
+static const char * const tgllp_pwm0_groups[] = { "pwm0_grp" };
+static const char * const tgllp_uart1_groups[] = { "uart1_grp" };
+static const char * const tgllp_ssp2_groups[] = { "ssp2_grp" };
+
+static const struct intel_function tgllp_functions[] = {
+	FUNCTION("i2c0", tgllp_i2c0_groups),
+	FUNCTION("i2c1", tgllp_i2c1_groups),
+	FUNCTION("pwm0", tgllp_pwm0_groups),
+	FUNCTION("uart1", tgllp_uart1_groups),
+	FUNCTION("ssp2", tgllp_ssp2_groups),
+};
+
 static const struct intel_padgroup tgllp_community0_gpps[] = {
 	INTEL_GPP(0, 0, 25, 0),				/* GPP_B */
 	INTEL_GPP(1, 26, 41, 32),			/* GPP_T */
@@ -366,6 +394,10 @@ static const struct intel_community tgllp_communities[] = {
 static const struct intel_pinctrl_soc_data tgllp_soc_data = {
 	.pins = tgllp_pins,
 	.npins = ARRAY_SIZE(tgllp_pins),
+	.groups = tgllp_groups,
+	.ngroups = ARRAY_SIZE(tgllp_groups),
+	.functions = tgllp_functions,
+	.nfunctions = ARRAY_SIZE(tgllp_functions),
 	.communities = tgllp_communities,
 	.ncommunities = ARRAY_SIZE(tgllp_communities),
 };
diff --git a/drivers/pinctrl/nxp/Kconfig b/drivers/pinctrl/nxp/Kconfig
index abca7ef97003d..711c0fe11565d 100644
--- a/drivers/pinctrl/nxp/Kconfig
+++ b/drivers/pinctrl/nxp/Kconfig
@@ -1,10 +1,11 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config PINCTRL_S32CC
 	bool
-	depends on ARCH_S32 && OF
+	depends on ARCH_S32 && OF && GPIOLIB
 	select GENERIC_PINCTRL_GROUPS
 	select GENERIC_PINMUX_FUNCTIONS
 	select GENERIC_PINCONF
+	select GPIO_REGMAP
 	select REGMAP_MMIO
 
 config PINCTRL_S32G2
diff --git a/drivers/pinctrl/nxp/pinctrl-s32.h b/drivers/pinctrl/nxp/pinctrl-s32.h
index 8715befd5f055..028578a090e43 100644
--- a/drivers/pinctrl/nxp/pinctrl-s32.h
+++ b/drivers/pinctrl/nxp/pinctrl-s32.h
@@ -2,7 +2,7 @@
  *
  * S32 pinmux core definitions
  *
- * Copyright 2016-2020, 2022 NXP
+ * Copyright 2016-2020, 2022, 2026 NXP
  * Copyright (C) 2022 SUSE LLC
  * Copyright 2015-2016 Freescale Semiconductor, Inc.
  * Copyright (C) 2012 Linaro Ltd.
@@ -34,11 +34,42 @@ struct s32_pin_range {
 	unsigned int end;
 };
 
+/**
+ * struct s32_gpio_range - contiguous GPIO pin range within a SIUL2 module
+ * @gpio_base: first GPIO line offset in the GPIO range
+ * @pin_base: first pinctrl pin number mapped by this GPIO range
+ * @gpio_num: number of consecutive GPIO pins in the range
+ * @sparse: true if the PGPD layout is non-linear (resolved via pad map only);
+ *          pins not found in the pad map are invalid for this range
+ */
+struct s32_gpio_range {
+	unsigned int gpio_base;
+	unsigned int pin_base;
+	unsigned int gpio_num;
+	bool sparse;
+};
+
+/**
+ * struct s32_gpio_pad_map - mapping between GPIO ranges and PGPD pads
+ * @gpio_start: first GPIO line offset in the range
+ * @gpio_end: last GPIO line offset in the range
+ * @pad: PGPD pad number serving the range
+ */
+struct s32_gpio_pad_map {
+	unsigned int gpio_start;
+	unsigned int gpio_end;
+	unsigned int pad;
+};
+
 struct s32_pinctrl_soc_data {
 	const struct pinctrl_pin_desc *pins;
 	unsigned int npins;
 	const struct s32_pin_range *mem_pin_ranges;
 	unsigned int mem_regions;
+	const struct s32_gpio_range *gpio_ranges;
+	unsigned int num_gpio_ranges;
+	const struct s32_gpio_pad_map *gpio_pad_maps;
+	unsigned int num_gpio_pad_maps;
 };
 
 struct s32_pinctrl_soc_info {
@@ -53,6 +84,8 @@ struct s32_pinctrl_soc_info {
 
 #define S32_PINCTRL_PIN(pin)	PINCTRL_PIN(pin, #pin)
 #define S32_PIN_RANGE(_start, _end) { .start = _start, .end = _end }
+#define S32_GPIO_RANGE(gpio, pin, num) \
+	{ .gpio_base = gpio, .pin_base = pin, .gpio_num = num }
 
 int s32_pinctrl_probe(struct platform_device *pdev,
 		      const struct s32_pinctrl_soc_data *soc_data);
diff --git a/drivers/pinctrl/nxp/pinctrl-s32cc.c b/drivers/pinctrl/nxp/pinctrl-s32cc.c
index 56be6e8d624e0..0025add684950 100644
--- a/drivers/pinctrl/nxp/pinctrl-s32cc.c
+++ b/drivers/pinctrl/nxp/pinctrl-s32cc.c
@@ -2,7 +2,7 @@
 /*
  * Core driver for the S32 CC (Common Chassis) pin controller
  *
- * Copyright 2017-2022,2024-2025 NXP
+ * Copyright 2017-2022,2024-2026 NXP
  * Copyright (C) 2022 SUSE LLC
  * Copyright 2015-2016 Freescale Semiconductor, Inc.
  */
@@ -10,6 +10,7 @@
 #include <linux/bitops.h>
 #include <linux/err.h>
 #include <linux/gpio/driver.h>
+#include <linux/gpio/regmap.h>
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/module.h>
@@ -39,6 +40,40 @@
 #define S32_MSCR_ODE		BIT(20)
 #define S32_MSCR_OBE		BIT(21)
 
+#define S32_GPIO_OP_SHIFT	16
+#define S32_GPIO_OP_MASK	GENMASK(19, 16)
+
+#define S32_GPIO_OP_DIR		0 /* MSCR direction */
+#define S32_GPIO_OP_DAT		BIT(S32_GPIO_OP_SHIFT) /* PGPDI read */
+#define S32_GPIO_OP_SET		BIT(S32_GPIO_OP_SHIFT + 1) /* PGPDO write */
+
+/*
+ * [15:12] = GPIO bank / gpio range index
+ * [11:0]  = real register offset or pin id
+ */
+#define S32_GPIO_BANK_SHIFT    12
+#define S32_GPIO_BANK_MASK    GENMASK(15, 12)
+#define S32_GPIO_REG_MASK    GENMASK(11, 0)
+
+#define S32_GPIO_ENCODE(bank, off) \
+	((((bank) << S32_GPIO_BANK_SHIFT) & S32_GPIO_BANK_MASK) | \
+		((off) & S32_GPIO_REG_MASK))
+
+#define S32_GPIO_DECODE_BANK(reg) \
+	(((reg) & S32_GPIO_BANK_MASK) >> S32_GPIO_BANK_SHIFT)
+
+#define S32_GPIO_DECODE_OFF(reg) \
+	((reg) & S32_GPIO_REG_MASK)
+
+/*
+ * PGPDOs are 16bit registers that come in big endian
+ * order if they are grouped in pairs of two.
+ *
+ * For example, the order is PGPDO1, PGPDO0, PGPDO3, PGPDO2...
+ */
+#define S32_PGPD(N)		(((N) ^ 1) * 2)
+#define S32_PGPD_SIZE		16
+
 enum s32_write_type {
 	S32_PINCONF_UPDATE_ONLY,
 	S32_PINCONF_OVERWRITE,
@@ -60,6 +95,12 @@ static u32 get_pin_func(u32 pinmux)
 	return pinmux & GENMASK(3, 0);
 }
 
+/*
+ * struct s32_pinctrl_mem_region - memory region for a set of SIUL2 registers
+ * @map: regmap used for this range
+ * @pin_range: the pins controlled by these registers
+ * @name: name of the current range
+ */
 struct s32_pinctrl_mem_region {
 	struct regmap *map;
 	const struct s32_pin_range *pin_range;
@@ -67,7 +108,19 @@ struct s32_pinctrl_mem_region {
 };
 
 /*
- * Holds pin configuration for GPIO's.
+ * struct s32_gpio_regmaps - GPIO register maps for a SIUL2 instance
+ * @pgpdo: regmap for Parallel GPIO Pad Data Out registers
+ * @pgpdi: regmap for Parallel GPIO Pad Data In registers
+ * @range: GPIO range info
+ */
+struct s32_gpio_regmaps {
+	struct regmap *pgpdo;
+	struct regmap *pgpdi;
+	const struct s32_gpio_range *range;
+};
+
+/*
+ * struct gpio_pin_config - holds pin configuration for GPIO's
  * @pin_id: Pin ID for this GPIO
  * @config: Pin settings
  * @list: Linked list entry for each gpio pin
@@ -79,26 +132,40 @@ struct gpio_pin_config {
 };
 
 /*
- * Pad config save/restore for power suspend/resume.
+ * struct s32_pinctrl_context - pad config save/restore for suspend/resume
+ * @pads: saved values for the pads
  */
 struct s32_pinctrl_context {
 	unsigned int *pads;
 };
 
 /*
+ * struct s32_pinctrl - private driver data
  * @dev: a pointer back to containing device
  * @pctl: a pointer to the pinctrl device structure
  * @regions: reserved memory regions with start/end pin
  * @info: structure containing information about the pin
- * @gpio_configs: Saved configurations for GPIO pins
- * @gpiop_configs_lock: lock for the `gpio_configs` list
- * @s32_pinctrl_context: Configuration saved over system sleep
+ * @gpio_regmaps: PGPDO/PGPDI regmaps for each SIUL2 module
+ * @num_gpio_regmaps: number of GPIO regmap entries
+ * @gpio_regmap: regmap bridging gpio-regmap to SIUL2 registers
+ * @gpio_rgm: gpio-regmap instance registered for this controller
+ * @ngpio: total number of GPIO line offsets
+ * @gpio_names: GPIO line names array passed to gpio-regmap
+ * @gpio_configs: saved configurations for GPIO pins
+ * @gpio_configs_lock: lock for the `gpio_configs` list
+ * @saved_context: configuration saved over system sleep
  */
 struct s32_pinctrl {
 	struct device *dev;
 	struct pinctrl_dev *pctl;
 	struct s32_pinctrl_mem_region *regions;
 	struct s32_pinctrl_soc_info *info;
+	struct s32_gpio_regmaps *gpio_regmaps;
+	unsigned int num_gpio_regmaps;
+	struct regmap *gpio_regmap;
+	struct gpio_regmap *gpio_rgm;
+	unsigned int ngpio;
+	const char *const *gpio_names;
 	struct list_head gpio_configs;
 	spinlock_t gpio_configs_lock;
 #ifdef CONFIG_PM_SLEEP
@@ -123,13 +190,13 @@ s32_get_region(struct pinctrl_dev *pctldev, unsigned int pin)
 	return NULL;
 }
 
-static inline int s32_check_pin(struct pinctrl_dev *pctldev,
-				unsigned int pin)
+static int s32_check_pin(struct pinctrl_dev *pctldev,
+			 unsigned int pin)
 {
 	return s32_get_region(pctldev, pin) ? 0 : -EINVAL;
 }
 
-static inline int s32_regmap_read(struct pinctrl_dev *pctldev,
+static int s32_regmap_read(struct pinctrl_dev *pctldev,
 			   unsigned int pin, unsigned int *val)
 {
 	struct s32_pinctrl_mem_region *region;
@@ -145,7 +212,7 @@ static inline int s32_regmap_read(struct pinctrl_dev *pctldev,
 	return regmap_read(region->map, offset, val);
 }
 
-static inline int s32_regmap_write(struct pinctrl_dev *pctldev,
+static int s32_regmap_write(struct pinctrl_dev *pctldev,
 			    unsigned int pin,
 			    unsigned int val)
 {
@@ -163,7 +230,7 @@ static inline int s32_regmap_write(struct pinctrl_dev *pctldev,
 
 }
 
-static inline int s32_regmap_update(struct pinctrl_dev *pctldev, unsigned int pin,
+static int s32_regmap_update(struct pinctrl_dev *pctldev, unsigned int pin,
 			     unsigned int mask, unsigned int val)
 {
 	struct s32_pinctrl_mem_region *region;
@@ -348,88 +415,84 @@ static int s32_pmx_get_funcs_count(struct pinctrl_dev *pctldev)
 	return info->nfunctions;
 }
 
-static const char *s32_pmx_get_func_name(struct pinctrl_dev *pctldev,
-					 unsigned int selector)
-{
-	struct s32_pinctrl *ipctl = pinctrl_dev_get_drvdata(pctldev);
-	const struct s32_pinctrl_soc_info *info = ipctl->info;
-
-	return info->functions[selector].name;
-}
-
-static int s32_pmx_get_groups(struct pinctrl_dev *pctldev,
-			      unsigned int selector,
-			      const char * const **groups,
-			      unsigned int * const num_groups)
-{
-	struct s32_pinctrl *ipctl = pinctrl_dev_get_drvdata(pctldev);
-	const struct s32_pinctrl_soc_info *info = ipctl->info;
-
-	*groups = info->functions[selector].groups;
-	*num_groups = info->functions[selector].ngroups;
-
-	return 0;
-}
-
 static int s32_pmx_gpio_request_enable(struct pinctrl_dev *pctldev,
 				       struct pinctrl_gpio_range *range,
-				       unsigned int offset)
+				       unsigned int pin)
 {
 	struct s32_pinctrl *ipctl = pinctrl_dev_get_drvdata(pctldev);
-	struct gpio_pin_config *gpio_pin;
+	struct gpio_pin_config *gpio_pin __free(kfree) = NULL;
 	unsigned int config;
-	unsigned long flags;
 	int ret;
 
-	ret = s32_regmap_read(pctldev, offset, &config);
+	ret = s32_regmap_read(pctldev, pin, &config);
 	if (ret)
 		return ret;
 
-	/* Save current configuration */
-	gpio_pin = kmalloc_obj(*gpio_pin);
+	gpio_pin = kmalloc_obj(*gpio_pin, GFP_KERNEL);
 	if (!gpio_pin)
 		return -ENOMEM;
 
-	gpio_pin->pin_id = offset;
+	gpio_pin->pin_id = pin;
 	gpio_pin->config = config;
-	INIT_LIST_HEAD(&gpio_pin->list);
-
-	spin_lock_irqsave(&ipctl->gpio_configs_lock, flags);
-	list_add(&gpio_pin->list, &ipctl->gpio_configs);
-	spin_unlock_irqrestore(&ipctl->gpio_configs_lock, flags);
 
 	/* GPIO pin means SSS = 0 */
-	config &= ~S32_MSCR_SSS_MASK;
+	ret = s32_regmap_update(pctldev, pin,
+				S32_MSCR_SSS_MASK | S32_MSCR_IBE,
+				S32_MSCR_IBE);
+	if (ret)
+		return ret;
 
-	return s32_regmap_write(pctldev, offset, config);
+	scoped_guard(spinlock_irqsave, &ipctl->gpio_configs_lock)
+		list_add(&no_free_ptr(gpio_pin)->list, &ipctl->gpio_configs);
+
+	return 0;
 }
 
 static void s32_pmx_gpio_disable_free(struct pinctrl_dev *pctldev,
 				      struct pinctrl_gpio_range *range,
-				      unsigned int offset)
+				      unsigned int pin)
 {
 	struct s32_pinctrl *ipctl = pinctrl_dev_get_drvdata(pctldev);
-	struct gpio_pin_config *gpio_pin, *tmp;
+	struct gpio_pin_config *gpio_pin, *found = NULL;
 	unsigned long flags;
-	int ret;
 
 	spin_lock_irqsave(&ipctl->gpio_configs_lock, flags);
-
-	list_for_each_entry_safe(gpio_pin, tmp, &ipctl->gpio_configs, list) {
-		if (gpio_pin->pin_id == offset) {
-			ret = s32_regmap_write(pctldev, gpio_pin->pin_id,
-						 gpio_pin->config);
-			if (ret != 0)
-				goto unlock;
-
+	list_for_each_entry(gpio_pin, &ipctl->gpio_configs, list) {
+		if (gpio_pin->pin_id == pin) {
 			list_del(&gpio_pin->list);
-			kfree(gpio_pin);
+			found = gpio_pin;
 			break;
 		}
 	}
-
-unlock:
 	spin_unlock_irqrestore(&ipctl->gpio_configs_lock, flags);
+
+	if (found) {
+		s32_regmap_write(pctldev, found->pin_id, found->config);
+		kfree(found);
+	}
+}
+
+static const char *s32_pmx_get_func_name(struct pinctrl_dev *pctldev,
+					 unsigned int selector)
+{
+	struct s32_pinctrl *ipctl = pinctrl_dev_get_drvdata(pctldev);
+	const struct s32_pinctrl_soc_info *info = ipctl->info;
+
+	return info->functions[selector].name;
+}
+
+static int s32_pmx_get_groups(struct pinctrl_dev *pctldev,
+			      unsigned int selector,
+			      const char * const **groups,
+			      unsigned int * const num_groups)
+{
+	struct s32_pinctrl *ipctl = pinctrl_dev_get_drvdata(pctldev);
+	const struct s32_pinctrl_soc_info *info = ipctl->info;
+
+	*groups = info->functions[selector].groups;
+	*num_groups = info->functions[selector].ngroups;
+
+	return 0;
 }
 
 static int s32_pmx_gpio_set_direction(struct pinctrl_dev *pctldev,
@@ -476,8 +539,8 @@ static int s32_get_slew_regval(int arg)
 	return -EINVAL;
 }
 
-static inline void s32_pin_set_pull(enum pin_config_param param,
-				   unsigned int *mask, unsigned int *config)
+static void s32_pin_set_pull(enum pin_config_param param,
+			     unsigned int *mask, unsigned int *config)
 {
 	switch (param) {
 	case PIN_CONFIG_BIAS_DISABLE:
@@ -641,9 +704,9 @@ static void s32_pinconf_dbg_show(struct pinctrl_dev *pctldev,
 
 	ret = s32_regmap_read(pctldev, pin_id, &config);
 	if (ret)
-		return;
-
-	seq_printf(s, "0x%x", config);
+		seq_printf(s, "error %d", ret);
+	else
+		seq_printf(s, "0x%x", config);
 }
 
 static void s32_pinconf_group_dbg_show(struct pinctrl_dev *pctldev,
@@ -661,8 +724,11 @@ static void s32_pinconf_group_dbg_show(struct pinctrl_dev *pctldev,
 	for (i = 0; i < grp->data.npins; i++) {
 		name = pin_get_name(pctldev, grp->data.pins[i]);
 		ret = s32_regmap_read(pctldev, grp->data.pins[i], &config);
-		if (ret)
-			return;
+		if (ret) {
+			seq_printf(s, "%s: error %d\n", name, ret);
+			continue;
+		}
+
 		seq_printf(s, "%s: 0x%x\n", name, config);
 	}
 }
@@ -675,6 +741,477 @@ static const struct pinconf_ops s32_pinconf_ops = {
 	.pin_config_group_dbg_show = s32_pinconf_group_dbg_show,
 };
 
+static void s32_gpio_free_saved_configs(void *data)
+{
+	struct s32_pinctrl *ipctl = data;
+	struct gpio_pin_config *gpio_pin, *tmp;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ipctl->gpio_configs_lock, flags);
+	list_for_each_entry_safe(gpio_pin, tmp, &ipctl->gpio_configs, list) {
+		list_del(&gpio_pin->list);
+		kfree(gpio_pin);
+	}
+	spin_unlock_irqrestore(&ipctl->gpio_configs_lock, flags);
+}
+
+static unsigned int s32_pin2pad(unsigned int pin)
+{
+	return pin / S32_PGPD_SIZE;
+}
+
+static u16 s32_pin2mask(unsigned int pin)
+{
+	/*
+	 * From Reference manual :
+	 * PGPDOx[PPDOy] = GPDO(x × 16) + (15 - y)[PDO_(x × 16) + (15 - y)]
+	 */
+	return BIT(S32_PGPD_SIZE - 1 - pin % S32_PGPD_SIZE);
+}
+
+static int s32_gpio_get_range(struct s32_pinctrl *ipctl,
+			      unsigned int gpio,
+			      unsigned int *pin,
+			      unsigned int *bank)
+{
+	const struct s32_pinctrl_soc_data *soc_data = ipctl->info->soc_data;
+	const struct s32_gpio_range *range;
+	int i;
+
+	for (i = 0; i < soc_data->num_gpio_ranges; i++) {
+		range = &soc_data->gpio_ranges[i];
+
+		if (gpio < range->gpio_base ||
+		    gpio >= range->gpio_base + range->gpio_num)
+			continue;
+
+		if (pin)
+			*pin = range->pin_base + gpio - range->gpio_base;
+
+		if (bank)
+			*bank = i;
+
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static int s32_gpio_pad_map_xlate(struct s32_pinctrl *ipctl,
+				  unsigned int gpio,
+				  unsigned int *reg_offset,
+				  u16 *mask)
+{
+	const struct s32_pinctrl_soc_data *soc_data = ipctl->info->soc_data;
+	const struct s32_gpio_pad_map *map;
+	unsigned int bit;
+	int i;
+
+	if (!soc_data->gpio_pad_maps || !soc_data->num_gpio_pad_maps)
+		return -EINVAL;
+
+	for (i = 0; i < soc_data->num_gpio_pad_maps; i++) {
+		map = &soc_data->gpio_pad_maps[i];
+
+		if (gpio < map->gpio_start || gpio > map->gpio_end)
+			continue;
+
+		bit = gpio - map->gpio_start;
+		*mask = BIT(S32_PGPD_SIZE - 1 - bit);
+		*reg_offset = S32_PGPD(map->pad);
+
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static bool s32_gpio_pin_is_sparse(struct s32_pinctrl *ipctl, unsigned int pin)
+{
+	const struct s32_pinctrl_soc_data *soc_data = ipctl->info->soc_data;
+	const struct s32_gpio_range *range;
+	int i;
+
+	for (i = 0; i < soc_data->num_gpio_ranges; i++) {
+		range = &soc_data->gpio_ranges[i];
+		if (pin >= range->pin_base &&
+		    pin < range->pin_base + range->gpio_num)
+			return range->sparse;
+	}
+
+	return false;
+}
+
+static int s32_gpio_xlate_pgpd(struct s32_pinctrl *ipctl,
+			       unsigned int pin,
+			       unsigned int *reg_offset,
+			       u16 *mask)
+{
+	int ret;
+
+	/*
+	 * Try the pad map first. For sparse ranges (SIUL2_1), only pins
+	 * listed in the pad map are valid, return the error directly without
+	 * falling back to the linear layout.
+	 * For linear ranges (SIUL2_0), fall back to the linear pad-to-PGPD
+	 * formula if no pad map entry matches.
+	 */
+	ret = s32_gpio_pad_map_xlate(ipctl, pin, reg_offset, mask);
+	if (ret != -EINVAL)
+		return ret;
+
+	if (s32_gpio_pin_is_sparse(ipctl, pin))
+		return -EINVAL;
+
+	/* Linear layout fallback for non-sparse ranges. */
+	*mask = s32_pin2mask(pin);
+	*reg_offset = S32_PGPD(s32_pin2pad(pin));
+
+	return 0;
+}
+
+static int s32_gpio_reg_mask_xlate(struct gpio_regmap *gpio,
+				   unsigned int base, unsigned int offset,
+				   unsigned int *reg, unsigned int *mask)
+{
+	struct s32_pinctrl *ipctl = gpio_regmap_get_drvdata(gpio);
+	unsigned int pgpd_reg, pin, bank;
+	u16 pgpd_mask;
+	int ret;
+
+	ret = s32_gpio_get_range(ipctl, offset, &pin, &bank);
+	if (ret)
+		return ret;
+
+	switch (base) {
+	case S32_GPIO_OP_DIR:
+		/*
+		 * Direction is controlled through MSCR OBE.
+		 * Encode the real pin id in the virtual register.
+		 */
+		*reg = S32_GPIO_OP_DIR | pin;
+		*mask = S32_MSCR_OBE;
+		return 0;
+
+	case S32_GPIO_OP_DAT:
+	case S32_GPIO_OP_SET:
+		ret = s32_gpio_xlate_pgpd(ipctl, pin, &pgpd_reg, &pgpd_mask);
+		if (ret)
+			return ret;
+		/*
+		 * Encode both the GPIO bank and the real PGPD register offset.
+		 */
+		*reg = base | S32_GPIO_ENCODE(bank, pgpd_reg);
+		*mask = pgpd_mask;
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int s32_gpio_reg_read(void *context, unsigned int reg,
+			     unsigned int *val)
+{
+	struct s32_pinctrl *ipctl = context;
+	unsigned int op = reg & S32_GPIO_OP_MASK;
+	unsigned int vreg = reg & ~S32_GPIO_OP_MASK;
+	unsigned int bank;
+	unsigned int offset;
+	struct regmap *map;
+
+	switch (op) {
+	case S32_GPIO_OP_DIR:
+		/*
+		 * Lower bits contain the real MSCR pin id.
+		 */
+		offset = S32_GPIO_DECODE_OFF(vreg);
+
+		return s32_regmap_read(ipctl->pctl, offset, val);
+
+	case S32_GPIO_OP_DAT:
+		bank = S32_GPIO_DECODE_BANK(vreg);
+		offset = S32_GPIO_DECODE_OFF(vreg);
+
+		if (bank >= ipctl->num_gpio_regmaps)
+			return -EINVAL;
+
+		map = ipctl->gpio_regmaps[bank].pgpdi;
+		if (!map)
+			return -ENODEV;
+
+		return regmap_read(map, offset, val);
+
+	case S32_GPIO_OP_SET:
+		/*
+		 * gpio-regmap uses update_bits() for set, so it needs to read
+		 * the output register before writing the updated value.
+		 */
+		bank = S32_GPIO_DECODE_BANK(vreg);
+		offset = S32_GPIO_DECODE_OFF(vreg);
+
+		if (bank >= ipctl->num_gpio_regmaps)
+			return -EINVAL;
+
+		map = ipctl->gpio_regmaps[bank].pgpdo;
+		if (!map)
+			return -ENODEV;
+
+		return regmap_read(map, offset, val);
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int s32_gpio_reg_write(void *context, unsigned int reg,
+			      unsigned int val)
+{
+	struct s32_pinctrl *ipctl = context;
+	unsigned int op = reg & S32_GPIO_OP_MASK;
+	unsigned int vreg = reg & ~S32_GPIO_OP_MASK;
+	unsigned int bank, offset, config;
+	struct regmap *map;
+
+	switch (op) {
+	case S32_GPIO_OP_DIR:
+		/*
+		 * gpio-regmap sets S32_MSCR_OBE for output and clears it for
+		 * input. Keep IBE enabled for GPIOs in both cases.
+		 */
+		offset = S32_GPIO_DECODE_OFF(vreg);
+
+		config = S32_MSCR_IBE;
+		if (val & S32_MSCR_OBE)
+			config |= S32_MSCR_OBE;
+
+		return s32_regmap_update(ipctl->pctl, offset,
+					 S32_MSCR_OBE | S32_MSCR_IBE,
+					 config);
+
+	case S32_GPIO_OP_SET:
+		bank = S32_GPIO_DECODE_BANK(vreg);
+		offset = S32_GPIO_DECODE_OFF(vreg);
+
+		if (bank >= ipctl->num_gpio_regmaps)
+			return -EINVAL;
+
+		map = ipctl->gpio_regmaps[bank].pgpdo;
+		if (!map)
+			return -ENODEV;
+
+		return regmap_write(map, offset, val);
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct regmap_bus s32_gpio_regmap_bus = {
+	.reg_read = s32_gpio_reg_read,
+	.reg_write = s32_gpio_reg_write,
+};
+
+static const struct regmap_config s32_gpio_regmap_config = {
+	.name = "s32-gpio",
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 1,
+	.max_register = S32_GPIO_OP_SET | S32_GPIO_BANK_MASK | S32_GPIO_REG_MASK,
+	.cache_type = REGCACHE_NONE,
+	.fast_io = true,
+};
+
+static int s32_gpio_get_ngpio(const struct s32_pinctrl_soc_data *soc_data,
+			      unsigned int *ngpio)
+{
+	const struct s32_gpio_range *range;
+	unsigned int end, max = 0;
+	int i;
+
+	if (!soc_data->gpio_ranges || !soc_data->num_gpio_ranges)
+		return -EINVAL;
+
+	for (i = 0; i < soc_data->num_gpio_ranges; i++) {
+		range = &soc_data->gpio_ranges[i];
+
+		if (!range->gpio_num)
+			return -EINVAL;
+
+		end = range->gpio_base + range->gpio_num;
+
+		/*
+		 * gpio_ranges must be ordered by gpio_base and must not overlap.
+		 * The GPIO line space size is derived from the highest range end.
+		 */
+		if (i > 0 && range->gpio_base < max)
+			return -EINVAL;
+
+		if (end > max)
+			max = end;
+	}
+
+	*ngpio = max;
+
+	return 0;
+}
+
+static int s32_init_gpio_regmap(struct platform_device *pdev,
+				struct s32_pinctrl *ipctl)
+{
+	ipctl->gpio_regmap =
+		devm_regmap_init(&pdev->dev, &s32_gpio_regmap_bus,
+				 ipctl, &s32_gpio_regmap_config);
+	if (IS_ERR(ipctl->gpio_regmap))
+		return dev_err_probe(&pdev->dev,
+				     PTR_ERR(ipctl->gpio_regmap),
+				     "Failed to init GPIO regmap\n");
+
+	return 0;
+}
+
+static int s32_init_valid_mask(struct gpio_chip *chip, unsigned long *mask,
+			       unsigned int ngpios)
+{
+	struct gpio_regmap *gpio = gpiochip_get_data(chip);
+	struct s32_pinctrl *ipctl = gpio_regmap_get_drvdata(gpio);
+	unsigned int gpio_num, pin, reg_offset;
+	u16 pgpd_mask;
+	int ret;
+
+	bitmap_zero(mask, ngpios);
+
+	for (gpio_num = 0; gpio_num < ngpios; gpio_num++) {
+		ret = s32_gpio_get_range(ipctl, gpio_num, &pin, NULL);
+		if (ret)
+			continue;
+
+		ret = s32_gpio_xlate_pgpd(ipctl, pin, &reg_offset, &pgpd_mask);
+		if (ret)
+			continue;
+
+		bitmap_set(mask, gpio_num, 1);
+	}
+
+	return 0;
+}
+
+static int s32_gpio_populate_names(struct s32_pinctrl *ipctl)
+{
+	char **names;
+	unsigned int gpio;
+	unsigned int pin;
+	char port;
+	int ret;
+
+	names = devm_kcalloc(ipctl->dev, ipctl->ngpio, sizeof(*names),
+			     GFP_KERNEL);
+	if (!names)
+		return -ENOMEM;
+
+	for (gpio = 0; gpio < ipctl->ngpio; gpio++) {
+		ret = s32_gpio_get_range(ipctl, gpio, &pin, NULL);
+		if (ret)
+			continue;
+
+		port = 'A' + pin / 16;
+
+		names[gpio] = devm_kasprintf(ipctl->dev, GFP_KERNEL,
+					     "P%c_%02u", port, pin & 0xf);
+		if (!names[gpio])
+			return -ENOMEM;
+	}
+
+	ipctl->gpio_names = (const char *const *)names;
+
+	return 0;
+}
+
+static int s32_pinctrl_init_gpio_regmaps(struct platform_device *pdev,
+					 struct s32_pinctrl *ipctl)
+{
+	const struct s32_pinctrl_soc_data *soc_data = ipctl->info->soc_data;
+	static const struct regmap_config pgpd_config = {
+		.reg_bits = 32,
+		.val_bits = 16,
+		.reg_stride = 2,
+	};
+	struct regmap_config cfg;
+	struct resource *res;
+	void __iomem *base;
+	unsigned int pgpdo_idx, pgpdi_idx;
+	unsigned int i;
+
+	if (!soc_data->gpio_ranges || !soc_data->num_gpio_ranges)
+		return 0;
+
+	ipctl->num_gpio_regmaps = soc_data->num_gpio_ranges;
+	ipctl->gpio_regmaps = devm_kcalloc(&pdev->dev, ipctl->num_gpio_regmaps,
+					   sizeof(*ipctl->gpio_regmaps),
+					   GFP_KERNEL);
+	if (!ipctl->gpio_regmaps)
+		return -ENOMEM;
+
+	for (i = 0; i < ipctl->num_gpio_regmaps; i++) {
+		ipctl->gpio_regmaps[i].range = &soc_data->gpio_ranges[i];
+
+		/*
+		 * GPIO resources are placed after the pinctrl regions
+		 */
+		pgpdo_idx = soc_data->mem_regions + i * 2;
+		pgpdi_idx = soc_data->mem_regions + i * 2 + 1;
+
+		/* PGPDO */
+		res = platform_get_resource(pdev, IORESOURCE_MEM, pgpdo_idx);
+		if (!res)
+			return dev_err_probe(&pdev->dev, -ENOENT,
+						 "Missing PGPDO resource %u\n", i);
+
+		base = devm_ioremap_resource(&pdev->dev, res);
+		if (IS_ERR(base))
+			return PTR_ERR(base);
+
+		cfg = pgpd_config;
+		cfg.name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "pgpdo%u", i);
+		if (!cfg.name)
+			return -ENOMEM;
+
+		cfg.max_register = resource_size(res) - cfg.reg_stride;
+
+		ipctl->gpio_regmaps[i].pgpdo =
+			devm_regmap_init_mmio(&pdev->dev, base, &cfg);
+		if (IS_ERR(ipctl->gpio_regmaps[i].pgpdo))
+			return dev_err_probe(&pdev->dev,
+						 PTR_ERR(ipctl->gpio_regmaps[i].pgpdo),
+						 "Failed to init PGPDO regmap %u\n", i);
+
+		/* PGPDI */
+		res = platform_get_resource(pdev, IORESOURCE_MEM, pgpdi_idx);
+		if (!res)
+			return dev_err_probe(&pdev->dev, -ENOENT,
+						 "Missing PGPDI resource %u\n", i);
+
+		base = devm_ioremap_resource(&pdev->dev, res);
+		if (IS_ERR(base))
+			return PTR_ERR(base);
+
+		cfg = pgpd_config;
+		cfg.name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "pgpdi%u", i);
+		if (!cfg.name)
+			return -ENOMEM;
+
+		cfg.max_register = resource_size(res) - cfg.reg_stride;
+
+		ipctl->gpio_regmaps[i].pgpdi =
+			devm_regmap_init_mmio(&pdev->dev, base, &cfg);
+		if (IS_ERR(ipctl->gpio_regmaps[i].pgpdi))
+			return dev_err_probe(&pdev->dev,
+						 PTR_ERR(ipctl->gpio_regmaps[i].pgpdi),
+						 "Failed to init PGPDI regmap %u\n", i);
+	}
+
+	return 0;
+}
+
 #ifdef CONFIG_PM_SLEEP
 static bool s32_pinctrl_should_save(struct s32_pinctrl *ipctl,
 				    unsigned int pin)
@@ -701,8 +1238,7 @@ int s32_pinctrl_suspend(struct device *dev)
 	const struct pinctrl_pin_desc *pin;
 	const struct s32_pinctrl_soc_info *info = ipctl->info;
 	struct s32_pinctrl_context *saved_context = &ipctl->saved_context;
-	int i;
-	int ret;
+	int i, ret;
 	unsigned int config;
 
 	for (i = 0; i < info->soc_data->npins; i++) {
@@ -713,7 +1249,7 @@ int s32_pinctrl_suspend(struct device *dev)
 
 		ret = s32_regmap_read(ipctl->pctl, pin->number, &config);
 		if (ret)
-			return -EINVAL;
+			return ret;
 
 		saved_context->pads[i] = config;
 	}
@@ -728,7 +1264,7 @@ int s32_pinctrl_resume(struct device *dev)
 	const struct s32_pinctrl_soc_info *info = ipctl->info;
 	const struct pinctrl_pin_desc *pin;
 	struct s32_pinctrl_context *saved_context = &ipctl->saved_context;
-	int ret, i;
+	int i, ret;
 
 	for (i = 0; i < info->soc_data->npins; i++) {
 		pin = &info->soc_data->pins[i];
@@ -737,7 +1273,7 @@ int s32_pinctrl_resume(struct device *dev)
 			continue;
 
 		ret = s32_regmap_write(ipctl->pctl, pin->number,
-					 saved_context->pads[i]);
+				 saved_context->pads[i]);
 		if (ret)
 			return ret;
 	}
@@ -920,9 +1456,11 @@ int s32_pinctrl_probe(struct platform_device *pdev,
 #ifdef CONFIG_PM_SLEEP
 	struct s32_pinctrl_context *saved_context;
 #endif
+	struct gpio_regmap_config gpio_cfg = {};
 	struct pinctrl_desc *s32_pinctrl_desc;
 	struct s32_pinctrl_soc_info *info;
 	struct s32_pinctrl *ipctl;
+	unsigned int ngpio;
 	int ret;
 
 	if (!soc_data || !soc_data->pins || !soc_data->npins)
@@ -948,6 +1486,11 @@ int s32_pinctrl_probe(struct platform_device *pdev,
 	INIT_LIST_HEAD(&ipctl->gpio_configs);
 	spin_lock_init(&ipctl->gpio_configs_lock);
 
+	ret = devm_add_action_or_reset(&pdev->dev,
+				       s32_gpio_free_saved_configs, ipctl);
+	if (ret)
+		return ret;
+
 	s32_pinctrl_desc =
 		devm_kzalloc(&pdev->dev, sizeof(*s32_pinctrl_desc), GFP_KERNEL);
 	if (!s32_pinctrl_desc)
@@ -966,11 +1509,16 @@ int s32_pinctrl_probe(struct platform_device *pdev,
 		return dev_err_probe(&pdev->dev, ret,
 				     "Fail to probe dt properties\n");
 
-	ipctl->pctl = devm_pinctrl_register(&pdev->dev, s32_pinctrl_desc,
-					    ipctl);
-	if (IS_ERR(ipctl->pctl))
-		return dev_err_probe(&pdev->dev, PTR_ERR(ipctl->pctl),
-				     "could not register s32 pinctrl driver\n");
+	ret = s32_pinctrl_init_gpio_regmaps(pdev, ipctl);
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret,
+				     "Failed to init GPIO regmaps\n");
+
+	ret = devm_pinctrl_register_and_init(&pdev->dev, s32_pinctrl_desc,
+					     ipctl, &ipctl->pctl);
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret,
+				     "Could not register s32 pinctrl driver\n");
 
 #ifdef CONFIG_PM_SLEEP
 	saved_context = &ipctl->saved_context;
@@ -982,7 +1530,47 @@ int s32_pinctrl_probe(struct platform_device *pdev,
 		return -ENOMEM;
 #endif
 
-	dev_info(&pdev->dev, "initialized s32 pinctrl driver\n");
+	ret = pinctrl_enable(ipctl->pctl);
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret,
+				     "Failed to enable pinctrl\n");
+
+	/* Setup GPIO if GPIO ranges are defined */
+	if (!soc_data->gpio_ranges || !soc_data->num_gpio_ranges)
+		return 0;
+
+	ret = s32_gpio_get_ngpio(soc_data, &ngpio);
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret, "Invalid GPIO ranges\n");
+
+	ipctl->ngpio = ngpio;
+
+	ret = s32_gpio_populate_names(ipctl);
+	if (ret)
+		return ret;
+
+	ret = s32_init_gpio_regmap(pdev, ipctl);
+	if (ret)
+		return ret;
+
+	gpio_cfg.parent = &pdev->dev;
+	gpio_cfg.fwnode = dev_fwnode(&pdev->dev);
+	gpio_cfg.label = dev_name(&pdev->dev);
+	gpio_cfg.regmap = ipctl->gpio_regmap;
+	gpio_cfg.ngpio = ngpio;
+	gpio_cfg.names = ipctl->gpio_names;
+	gpio_cfg.reg_dir_out_base = GPIO_REGMAP_ADDR(S32_GPIO_OP_DIR);
+	gpio_cfg.reg_dat_base = GPIO_REGMAP_ADDR(S32_GPIO_OP_DAT);
+	gpio_cfg.reg_set_base = GPIO_REGMAP_ADDR(S32_GPIO_OP_SET);
+	gpio_cfg.reg_mask_xlate = s32_gpio_reg_mask_xlate;
+	gpio_cfg.init_valid_mask = s32_init_valid_mask;
+	gpio_cfg.drvdata = ipctl;
+
+	ipctl->gpio_rgm = devm_gpio_regmap_register(&pdev->dev, &gpio_cfg);
+	if (IS_ERR(ipctl->gpio_rgm))
+		return dev_err_probe(&pdev->dev,
+				     PTR_ERR(ipctl->gpio_rgm),
+				     "Unable to add gpio_regmap chip\n");
 
 	return 0;
 }
diff --git a/drivers/pinctrl/nxp/pinctrl-s32g2.c b/drivers/pinctrl/nxp/pinctrl-s32g2.c
index c49d28793b696..f9546c67a2693 100644
--- a/drivers/pinctrl/nxp/pinctrl-s32g2.c
+++ b/drivers/pinctrl/nxp/pinctrl-s32g2.c
@@ -3,7 +3,7 @@
  * NXP S32G pinctrl driver
  *
  * Copyright 2015-2016 Freescale Semiconductor, Inc.
- * Copyright 2017-2018, 2020-2022 NXP
+ * Copyright 2017-2018, 2020-2022, 2025-2026 NXP
  * Copyright (C) 2022 SUSE LLC
  */
 
@@ -773,17 +773,48 @@ static const struct s32_pin_range s32_pin_ranges_siul2[] = {
 	S32_PIN_RANGE(942, 1007),
 };
 
-static const struct s32_pinctrl_soc_data s32_pinctrl_data = {
+static const struct s32_gpio_range s32_gpio_ranges_siul2[] = {
+	S32_GPIO_RANGE(0, 0, 102),
+	/* SIUL2_1: sparse layout, PGPD mapping required for all pins */
+	{ .gpio_base = 112, .pin_base = 112, .gpio_num = 79, .sparse = true },
+};
+
+/*
+ * SIUL2_1 GPIO ranges mapped to sparse PGPD pads.
+ *
+ * SIUL2_1 does not expose GPIO data registers as a linear pad
+ * sequence. Each entry describes a contiguous GPIO offset range
+ * and the PGPD pad servicing that range.
+ */
+static const struct s32_gpio_pad_map s32g_gpio_pad_maps[] = {
+	{ 112, 122, 7  }, /* PH_00 .. PH_10 -> PGPD7  */
+	{ 144, 159, 9  }, /* PJ_00 .. PJ_15 -> PGPD9  */
+	{ 160, 175, 10 }, /* PK_00 .. PK_15 -> PGPD10 */
+	{ 176, 190, 11 }, /* PL_00 .. PL_14 -> PGPD11 */
+};
+
+/* Legacy data for old DT bindings without GPIO support */
+static const struct s32_pinctrl_soc_data legacy_s32g_pinctrl_data = {
+	.pins = s32_pinctrl_pads_siul2,
+	.npins = ARRAY_SIZE(s32_pinctrl_pads_siul2),
+	.mem_pin_ranges = s32_pin_ranges_siul2,
+	.mem_regions = ARRAY_SIZE(s32_pin_ranges_siul2),
+};
+
+static const struct s32_pinctrl_soc_data s32g_pinctrl_data = {
 	.pins = s32_pinctrl_pads_siul2,
 	.npins = ARRAY_SIZE(s32_pinctrl_pads_siul2),
 	.mem_pin_ranges = s32_pin_ranges_siul2,
 	.mem_regions = ARRAY_SIZE(s32_pin_ranges_siul2),
+	.gpio_ranges = s32_gpio_ranges_siul2,
+	.num_gpio_ranges = ARRAY_SIZE(s32_gpio_ranges_siul2),
+	.gpio_pad_maps = s32g_gpio_pad_maps,
+	.num_gpio_pad_maps = ARRAY_SIZE(s32g_gpio_pad_maps),
 };
 
 static const struct of_device_id s32_pinctrl_of_match[] = {
 	{
 		.compatible = "nxp,s32g2-siul2-pinctrl",
-		.data = &s32_pinctrl_data,
 	},
 	{ /* sentinel */ }
 };
@@ -792,8 +823,16 @@ MODULE_DEVICE_TABLE(of, s32_pinctrl_of_match);
 static int s32g_pinctrl_probe(struct platform_device *pdev)
 {
 	const struct s32_pinctrl_soc_data *soc_data;
+	struct device_node *np = pdev->dev.of_node;
 
-	soc_data = of_device_get_match_data(&pdev->dev);
+	/*
+	 * Legacy DTs only describe the pinctrl resources.
+	 * New DT changes extend the same node with GPIO resources.
+	 */
+	if (of_property_present(np, "gpio-controller"))
+		soc_data = &s32g_pinctrl_data;
+	else
+		soc_data = &legacy_s32g_pinctrl_data;
 
 	return s32_pinctrl_probe(pdev, soc_data);
 }
diff --git a/drivers/pinctrl/pinctrl-rp1.c b/drivers/pinctrl/pinctrl-rp1.c
index fc4ed68ea5ac0..dd0e85c2ca33a 100644
--- a/drivers/pinctrl/pinctrl-rp1.c
+++ b/drivers/pinctrl/pinctrl-rp1.c
@@ -1085,7 +1085,7 @@ static void rp1_pctl_pin_dbg_show(struct pinctrl_dev *pctldev,
 
 	seq_printf(s, "function %s (%s) in %s; irq %d (%s)",
 		   rp1_func_names[fsel].name, rp1_func_names[func].name,
-		   value ? "hi" : "lo",
+		   str_hi_lo(value),
 		   irq, irq_type_names[pin->irq_type]);
 }
 
diff --git a/drivers/pinctrl/pinctrl-sx150x.c b/drivers/pinctrl/pinctrl-sx150x.c
index b59d913513d59..89fdfa96843fd 100644
--- a/drivers/pinctrl/pinctrl-sx150x.c
+++ b/drivers/pinctrl/pinctrl-sx150x.c
@@ -850,6 +850,7 @@ static const struct i2c_device_id sx150x_id[] = {
 	{ .name = "sx1509q", .driver_data = (kernel_ulong_t)&sx1509q_device_data },
 	{ }
 };
+MODULE_DEVICE_TABLE(i2c, sx150x_id);
 
 static const struct of_device_id sx150x_of_match[] = {
 	{ .compatible = "semtech,sx1501q", .data = &sx1501q_device_data },
diff --git a/drivers/pinctrl/pinctrl-tb10x.c b/drivers/pinctrl/pinctrl-tb10x.c
index 3f581404a9b9a..b7cbd69dd877d 100644
--- a/drivers/pinctrl/pinctrl-tb10x.c
+++ b/drivers/pinctrl/pinctrl-tb10x.c
@@ -479,7 +479,7 @@ struct tb10x_port {
  */
 struct tb10x_pinctrl {
 	struct pinctrl_dev *pctl;
-	void *base;
+	void __iomem *base;
 	const struct tb10x_pinfuncgrp *pingroups;
 	unsigned int pinfuncgrpcnt;
 	unsigned int pinfuncnt;
diff --git a/drivers/pinctrl/pinctrl-upboard.c b/drivers/pinctrl/pinctrl-upboard.c
index f8c8b9d849900..de1920a3387d5 100644
--- a/drivers/pinctrl/pinctrl-upboard.c
+++ b/drivers/pinctrl/pinctrl-upboard.c
@@ -912,6 +912,19 @@ static const struct upboard_pinctrl_map upboard_pinctrl_map_apl01 = {
 	.nmaps = ARRAY_SIZE(pinctrl_map_apl01),
 };
 
+static const struct pinctrl_map pinctrl_map_adl[] = {
+	PIN_MAP_MUX_GROUP_DEFAULT("upboard-pinctrl", "INTC1055:00", "i2c0_grp", "i2c0"),
+	PIN_MAP_MUX_GROUP_DEFAULT("upboard-pinctrl", "INTC1055:00", "i2c1_grp", "i2c1"),
+	PIN_MAP_MUX_GROUP_DEFAULT("upboard-pinctrl", "INTC1055:00", "pwm0_grp", "pwm0"),
+	PIN_MAP_MUX_GROUP_DEFAULT("upboard-pinctrl", "INTC1055:00", "uart1_grp", "uart1"),
+	PIN_MAP_MUX_GROUP_DEFAULT("upboard-pinctrl", "INTC1055:00", "ssp2_grp", "ssp2"),
+};
+
+static const struct upboard_pinctrl_map upboard_pinctrl_map_adl = {
+	.maps = &pinctrl_map_adl[0],
+	.nmaps = ARRAY_SIZE(pinctrl_map_adl),
+};
+
 static const struct dmi_system_id dmi_platform_info[] = {
 	{
 		/* UP Squared */
@@ -921,6 +934,14 @@ static const struct dmi_system_id dmi_platform_info[] = {
 		},
 		.driver_data = (void *)&upboard_pinctrl_map_apl01,
 	},
+	{
+		/* UP Xtreme i12 */
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "AAEON"),
+			DMI_EXACT_MATCH(DMI_BOARD_NAME, "UPX-ADLP01"),
+		},
+		.driver_data = (void *)&upboard_pinctrl_map_adl,
+	},
 	{ }
 };
 
diff --git a/drivers/pinctrl/qcom/Kconfig.msm b/drivers/pinctrl/qcom/Kconfig.msm
index 9409e678ec6dc..42875457b5fc1 100644
--- a/drivers/pinctrl/qcom/Kconfig.msm
+++ b/drivers/pinctrl/qcom/Kconfig.msm
@@ -153,6 +153,16 @@ config PINCTRL_KAANAPALI
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	  Technologies Inc Kaanapali platform.
 
+config PINCTRL_MAILI
+	tristate "Qualcomm Technologies Inc Maili pin controller driver"
+	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
+	help
+	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
+	  Qualcomm Technologies Inc TLMM block found on the Maili platform.
+	  Say Y here to compile statically, or M here to compile it as a module.
+	  If unsure, say N.
+
 config PINCTRL_MSM8226
 	tristate "Qualcomm 8226 pin controller driver"
 	depends on ARM || COMPILE_TEST
diff --git a/drivers/pinctrl/qcom/Makefile b/drivers/pinctrl/qcom/Makefile
index 93cc4e7965cab..43ecd246afe88 100644
--- a/drivers/pinctrl/qcom/Makefile
+++ b/drivers/pinctrl/qcom/Makefile
@@ -17,6 +17,7 @@ obj-$(CONFIG_PINCTRL_IPQ6018)	+= pinctrl-ipq6018.o
 obj-$(CONFIG_PINCTRL_IPQ9574)	+= pinctrl-ipq9574.o
 obj-$(CONFIG_PINCTRL_IPQ9650)	+= pinctrl-ipq9650.o
 obj-$(CONFIG_PINCTRL_KAANAPALI) += pinctrl-kaanapali.o
+obj-$(CONFIG_PINCTRL_MAILI)	+= pinctrl-maili.o
 obj-$(CONFIG_PINCTRL_MSM8226)	+= pinctrl-msm8226.o
 obj-$(CONFIG_PINCTRL_MSM8660)	+= pinctrl-msm8660.o
 obj-$(CONFIG_PINCTRL_MSM8960)	+= pinctrl-msm8960.o
diff --git a/drivers/pinctrl/qcom/pinctrl-maili.c b/drivers/pinctrl/qcom/pinctrl-maili.c
new file mode 100644
index 0000000000000..ffa084cfad648
--- /dev/null
+++ b/drivers/pinctrl/qcom/pinctrl-maili.c
@@ -0,0 +1,1625 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+
+#include "pinctrl-msm.h"
+
+#define REG_SIZE 0x1000
+#define PINGROUP(id, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11)	\
+	{						\
+		.grp = PINCTRL_PINGROUP("gpio" #id,	\
+			gpio##id##_pins,		\
+			ARRAY_SIZE(gpio##id##_pins)),	\
+		.funcs = (int[]){			\
+			msm_mux_gpio, /* gpio mode */	\
+			msm_mux_##f1,			\
+			msm_mux_##f2,			\
+			msm_mux_##f3,			\
+			msm_mux_##f4,			\
+			msm_mux_##f5,			\
+			msm_mux_##f6,			\
+			msm_mux_##f7,			\
+			msm_mux_##f8,			\
+			msm_mux_##f9,			\
+			msm_mux_##f10,			\
+			msm_mux_##f11 /* egpio mode */	\
+		},					\
+		.nfuncs = 12,				\
+		.ctl_reg = REG_SIZE * id,		\
+		.io_reg = 0x4 + REG_SIZE * id,		\
+		.intr_cfg_reg = 0x8 + REG_SIZE * id,	\
+		.intr_status_reg = 0xc + REG_SIZE * id,	\
+		.mux_bit = 2,				\
+		.pull_bit = 0,				\
+		.drv_bit = 6,				\
+		.egpio_enable = 12,			\
+		.egpio_present = 11,			\
+		.oe_bit = 9,				\
+		.in_bit = 0,				\
+		.out_bit = 1,				\
+		.intr_enable_bit = 0,			\
+		.intr_status_bit = 0,			\
+		.intr_wakeup_present_bit = 6,		\
+		.intr_wakeup_enable_bit = 7,		\
+		.intr_target_bit = 8,			\
+		.intr_target_kpss_val = 3,		\
+		.intr_raw_status_bit = 4,		\
+		.intr_polarity_bit = 1,			\
+		.intr_detection_bit = 2,		\
+		.intr_detection_width = 2,		\
+	}
+
+#define SDC_QDSD_PINGROUP(pg_name, ctl, pull, drv)	\
+	{					        \
+		.grp = PINCTRL_PINGROUP(#pg_name,	\
+			pg_name##_pins,			\
+			ARRAY_SIZE(pg_name##_pins)),	\
+		.ctl_reg = ctl,				\
+		.io_reg = 0,				\
+		.intr_cfg_reg = 0,			\
+		.intr_status_reg = 0,			\
+		.mux_bit = -1,				\
+		.pull_bit = pull,			\
+		.drv_bit = drv,				\
+		.oe_bit = -1,				\
+		.in_bit = -1,				\
+		.out_bit = -1,				\
+		.intr_enable_bit = -1,			\
+		.intr_status_bit = -1,			\
+		.intr_target_bit = -1,			\
+		.intr_raw_status_bit = -1,		\
+		.intr_polarity_bit = -1,		\
+		.intr_detection_bit = -1,		\
+		.intr_detection_width = -1,		\
+	}
+
+#define UFS_RESET(pg_name, ctl, io)			\
+	{						\
+		.grp = PINCTRL_PINGROUP(#pg_name,	\
+			pg_name##_pins,			\
+			ARRAY_SIZE(pg_name##_pins)),	\
+		.ctl_reg = ctl,				\
+		.io_reg = io,				\
+		.intr_cfg_reg = 0,			\
+		.intr_status_reg = 0,			\
+		.mux_bit = -1,				\
+		.pull_bit = 3,				\
+		.drv_bit = 0,				\
+		.oe_bit = -1,				\
+		.in_bit = -1,				\
+		.out_bit = 0,				\
+		.intr_enable_bit = -1,			\
+		.intr_status_bit = -1,			\
+		.intr_target_bit = -1,			\
+		.intr_raw_status_bit = -1,		\
+		.intr_polarity_bit = -1,		\
+		.intr_detection_bit = -1,		\
+		.intr_detection_width = -1,		\
+	}
+
+static const struct pinctrl_pin_desc maili_pins[] = {
+	PINCTRL_PIN(0, "GPIO_0"),
+	PINCTRL_PIN(1, "GPIO_1"),
+	PINCTRL_PIN(2, "GPIO_2"),
+	PINCTRL_PIN(3, "GPIO_3"),
+	PINCTRL_PIN(4, "GPIO_4"),
+	PINCTRL_PIN(5, "GPIO_5"),
+	PINCTRL_PIN(6, "GPIO_6"),
+	PINCTRL_PIN(7, "GPIO_7"),
+	PINCTRL_PIN(8, "GPIO_8"),
+	PINCTRL_PIN(9, "GPIO_9"),
+	PINCTRL_PIN(10, "GPIO_10"),
+	PINCTRL_PIN(11, "GPIO_11"),
+	PINCTRL_PIN(12, "GPIO_12"),
+	PINCTRL_PIN(13, "GPIO_13"),
+	PINCTRL_PIN(14, "GPIO_14"),
+	PINCTRL_PIN(15, "GPIO_15"),
+	PINCTRL_PIN(16, "GPIO_16"),
+	PINCTRL_PIN(17, "GPIO_17"),
+	PINCTRL_PIN(18, "GPIO_18"),
+	PINCTRL_PIN(19, "GPIO_19"),
+	PINCTRL_PIN(20, "GPIO_20"),
+	PINCTRL_PIN(21, "GPIO_21"),
+	PINCTRL_PIN(22, "GPIO_22"),
+	PINCTRL_PIN(23, "GPIO_23"),
+	PINCTRL_PIN(24, "GPIO_24"),
+	PINCTRL_PIN(25, "GPIO_25"),
+	PINCTRL_PIN(26, "GPIO_26"),
+	PINCTRL_PIN(27, "GPIO_27"),
+	PINCTRL_PIN(28, "GPIO_28"),
+	PINCTRL_PIN(29, "GPIO_29"),
+	PINCTRL_PIN(30, "GPIO_30"),
+	PINCTRL_PIN(31, "GPIO_31"),
+	PINCTRL_PIN(32, "GPIO_32"),
+	PINCTRL_PIN(33, "GPIO_33"),
+	PINCTRL_PIN(34, "GPIO_34"),
+	PINCTRL_PIN(35, "GPIO_35"),
+	PINCTRL_PIN(36, "GPIO_36"),
+	PINCTRL_PIN(37, "GPIO_37"),
+	PINCTRL_PIN(38, "GPIO_38"),
+	PINCTRL_PIN(39, "GPIO_39"),
+	PINCTRL_PIN(40, "GPIO_40"),
+	PINCTRL_PIN(41, "GPIO_41"),
+	PINCTRL_PIN(42, "GPIO_42"),
+	PINCTRL_PIN(43, "GPIO_43"),
+	PINCTRL_PIN(44, "GPIO_44"),
+	PINCTRL_PIN(45, "GPIO_45"),
+	PINCTRL_PIN(46, "GPIO_46"),
+	PINCTRL_PIN(47, "GPIO_47"),
+	PINCTRL_PIN(48, "GPIO_48"),
+	PINCTRL_PIN(49, "GPIO_49"),
+	PINCTRL_PIN(50, "GPIO_50"),
+	PINCTRL_PIN(51, "GPIO_51"),
+	PINCTRL_PIN(52, "GPIO_52"),
+	PINCTRL_PIN(53, "GPIO_53"),
+	PINCTRL_PIN(54, "GPIO_54"),
+	PINCTRL_PIN(55, "GPIO_55"),
+	PINCTRL_PIN(56, "GPIO_56"),
+	PINCTRL_PIN(57, "GPIO_57"),
+	PINCTRL_PIN(58, "GPIO_58"),
+	PINCTRL_PIN(59, "GPIO_59"),
+	PINCTRL_PIN(60, "GPIO_60"),
+	PINCTRL_PIN(61, "GPIO_61"),
+	PINCTRL_PIN(62, "GPIO_62"),
+	PINCTRL_PIN(63, "GPIO_63"),
+	PINCTRL_PIN(64, "GPIO_64"),
+	PINCTRL_PIN(65, "GPIO_65"),
+	PINCTRL_PIN(66, "GPIO_66"),
+	PINCTRL_PIN(67, "GPIO_67"),
+	PINCTRL_PIN(68, "GPIO_68"),
+	PINCTRL_PIN(69, "GPIO_69"),
+	PINCTRL_PIN(70, "GPIO_70"),
+	PINCTRL_PIN(71, "GPIO_71"),
+	PINCTRL_PIN(72, "GPIO_72"),
+	PINCTRL_PIN(73, "GPIO_73"),
+	PINCTRL_PIN(74, "GPIO_74"),
+	PINCTRL_PIN(75, "GPIO_75"),
+	PINCTRL_PIN(76, "GPIO_76"),
+	PINCTRL_PIN(77, "GPIO_77"),
+	PINCTRL_PIN(78, "GPIO_78"),
+	PINCTRL_PIN(79, "GPIO_79"),
+	PINCTRL_PIN(80, "GPIO_80"),
+	PINCTRL_PIN(81, "GPIO_81"),
+	PINCTRL_PIN(82, "GPIO_82"),
+	PINCTRL_PIN(83, "GPIO_83"),
+	PINCTRL_PIN(84, "GPIO_84"),
+	PINCTRL_PIN(85, "GPIO_85"),
+	PINCTRL_PIN(86, "GPIO_86"),
+	PINCTRL_PIN(87, "GPIO_87"),
+	PINCTRL_PIN(88, "GPIO_88"),
+	PINCTRL_PIN(89, "GPIO_89"),
+	PINCTRL_PIN(90, "GPIO_90"),
+	PINCTRL_PIN(91, "GPIO_91"),
+	PINCTRL_PIN(92, "GPIO_92"),
+	PINCTRL_PIN(93, "GPIO_93"),
+	PINCTRL_PIN(94, "GPIO_94"),
+	PINCTRL_PIN(95, "GPIO_95"),
+	PINCTRL_PIN(96, "GPIO_96"),
+	PINCTRL_PIN(97, "GPIO_97"),
+	PINCTRL_PIN(98, "GPIO_98"),
+	PINCTRL_PIN(99, "GPIO_99"),
+	PINCTRL_PIN(100, "GPIO_100"),
+	PINCTRL_PIN(101, "GPIO_101"),
+	PINCTRL_PIN(102, "GPIO_102"),
+	PINCTRL_PIN(103, "GPIO_103"),
+	PINCTRL_PIN(104, "GPIO_104"),
+	PINCTRL_PIN(105, "GPIO_105"),
+	PINCTRL_PIN(106, "GPIO_106"),
+	PINCTRL_PIN(107, "GPIO_107"),
+	PINCTRL_PIN(108, "GPIO_108"),
+	PINCTRL_PIN(109, "GPIO_109"),
+	PINCTRL_PIN(110, "GPIO_110"),
+	PINCTRL_PIN(111, "GPIO_111"),
+	PINCTRL_PIN(112, "GPIO_112"),
+	PINCTRL_PIN(113, "GPIO_113"),
+	PINCTRL_PIN(114, "GPIO_114"),
+	PINCTRL_PIN(115, "GPIO_115"),
+	PINCTRL_PIN(116, "GPIO_116"),
+	PINCTRL_PIN(117, "GPIO_117"),
+	PINCTRL_PIN(118, "GPIO_118"),
+	PINCTRL_PIN(119, "GPIO_119"),
+	PINCTRL_PIN(120, "GPIO_120"),
+	PINCTRL_PIN(121, "GPIO_121"),
+	PINCTRL_PIN(122, "GPIO_122"),
+	PINCTRL_PIN(123, "GPIO_123"),
+	PINCTRL_PIN(124, "GPIO_124"),
+	PINCTRL_PIN(125, "GPIO_125"),
+	PINCTRL_PIN(126, "GPIO_126"),
+	PINCTRL_PIN(127, "GPIO_127"),
+	PINCTRL_PIN(128, "GPIO_128"),
+	PINCTRL_PIN(129, "GPIO_129"),
+	PINCTRL_PIN(130, "GPIO_130"),
+	PINCTRL_PIN(131, "GPIO_131"),
+	PINCTRL_PIN(132, "GPIO_132"),
+	PINCTRL_PIN(133, "GPIO_133"),
+	PINCTRL_PIN(134, "GPIO_134"),
+	PINCTRL_PIN(135, "GPIO_135"),
+	PINCTRL_PIN(136, "GPIO_136"),
+	PINCTRL_PIN(137, "GPIO_137"),
+	PINCTRL_PIN(138, "GPIO_138"),
+	PINCTRL_PIN(139, "GPIO_139"),
+	PINCTRL_PIN(140, "GPIO_140"),
+	PINCTRL_PIN(141, "GPIO_141"),
+	PINCTRL_PIN(142, "GPIO_142"),
+	PINCTRL_PIN(143, "GPIO_143"),
+	PINCTRL_PIN(144, "GPIO_144"),
+	PINCTRL_PIN(145, "GPIO_145"),
+	PINCTRL_PIN(146, "GPIO_146"),
+	PINCTRL_PIN(147, "GPIO_147"),
+	PINCTRL_PIN(148, "GPIO_148"),
+	PINCTRL_PIN(149, "GPIO_149"),
+	PINCTRL_PIN(150, "GPIO_150"),
+	PINCTRL_PIN(151, "GPIO_151"),
+	PINCTRL_PIN(152, "GPIO_152"),
+	PINCTRL_PIN(153, "GPIO_153"),
+	PINCTRL_PIN(154, "GPIO_154"),
+	PINCTRL_PIN(155, "GPIO_155"),
+	PINCTRL_PIN(156, "GPIO_156"),
+	PINCTRL_PIN(157, "GPIO_157"),
+	PINCTRL_PIN(158, "GPIO_158"),
+	PINCTRL_PIN(159, "GPIO_159"),
+	PINCTRL_PIN(160, "GPIO_160"),
+	PINCTRL_PIN(161, "GPIO_161"),
+	PINCTRL_PIN(162, "GPIO_162"),
+	PINCTRL_PIN(163, "GPIO_163"),
+	PINCTRL_PIN(164, "GPIO_164"),
+	PINCTRL_PIN(165, "GPIO_165"),
+	PINCTRL_PIN(166, "GPIO_166"),
+	PINCTRL_PIN(167, "GPIO_167"),
+	PINCTRL_PIN(168, "GPIO_168"),
+	PINCTRL_PIN(169, "GPIO_169"),
+	PINCTRL_PIN(170, "GPIO_170"),
+	PINCTRL_PIN(171, "GPIO_171"),
+	PINCTRL_PIN(172, "GPIO_172"),
+	PINCTRL_PIN(173, "GPIO_173"),
+	PINCTRL_PIN(174, "GPIO_174"),
+	PINCTRL_PIN(175, "GPIO_175"),
+	PINCTRL_PIN(176, "GPIO_176"),
+	PINCTRL_PIN(177, "GPIO_177"),
+	PINCTRL_PIN(178, "GPIO_178"),
+	PINCTRL_PIN(179, "GPIO_179"),
+	PINCTRL_PIN(180, "GPIO_180"),
+	PINCTRL_PIN(181, "GPIO_181"),
+	PINCTRL_PIN(182, "GPIO_182"),
+	PINCTRL_PIN(183, "GPIO_183"),
+	PINCTRL_PIN(184, "GPIO_184"),
+	PINCTRL_PIN(185, "GPIO_185"),
+	PINCTRL_PIN(186, "GPIO_186"),
+	PINCTRL_PIN(187, "GPIO_187"),
+	PINCTRL_PIN(188, "GPIO_188"),
+	PINCTRL_PIN(189, "GPIO_189"),
+	PINCTRL_PIN(190, "GPIO_190"),
+	PINCTRL_PIN(191, "GPIO_191"),
+	PINCTRL_PIN(192, "GPIO_192"),
+	PINCTRL_PIN(193, "GPIO_193"),
+	PINCTRL_PIN(194, "GPIO_194"),
+	PINCTRL_PIN(195, "GPIO_195"),
+	PINCTRL_PIN(196, "GPIO_196"),
+	PINCTRL_PIN(197, "GPIO_197"),
+	PINCTRL_PIN(198, "GPIO_198"),
+	PINCTRL_PIN(199, "GPIO_199"),
+	PINCTRL_PIN(200, "GPIO_200"),
+	PINCTRL_PIN(201, "GPIO_201"),
+	PINCTRL_PIN(202, "GPIO_202"),
+	PINCTRL_PIN(203, "GPIO_203"),
+	PINCTRL_PIN(204, "GPIO_204"),
+	PINCTRL_PIN(205, "GPIO_205"),
+	PINCTRL_PIN(206, "GPIO_206"),
+	PINCTRL_PIN(207, "GPIO_207"),
+	PINCTRL_PIN(208, "GPIO_208"),
+	PINCTRL_PIN(209, "GPIO_209"),
+	PINCTRL_PIN(210, "GPIO_210"),
+	PINCTRL_PIN(211, "GPIO_211"),
+	PINCTRL_PIN(212, "GPIO_212"),
+	PINCTRL_PIN(213, "GPIO_213"),
+	PINCTRL_PIN(214, "GPIO_214"),
+	PINCTRL_PIN(215, "GPIO_215"),
+	PINCTRL_PIN(216, "GPIO_216"),
+	PINCTRL_PIN(217, "GPIO_217"),
+	PINCTRL_PIN(218, "GPIO_218"),
+	PINCTRL_PIN(219, "GPIO_219"),
+	PINCTRL_PIN(220, "GPIO_220"),
+	PINCTRL_PIN(221, "GPIO_221"),
+	PINCTRL_PIN(222, "GPIO_222"),
+	PINCTRL_PIN(223, "GPIO_223"),
+	PINCTRL_PIN(224, "GPIO_224"),
+	PINCTRL_PIN(225, "GPIO_225"),
+	PINCTRL_PIN(226, "UFS_RESET"),
+	PINCTRL_PIN(227, "SDC2_CLK"),
+	PINCTRL_PIN(228, "SDC2_CMD"),
+	PINCTRL_PIN(229, "SDC2_DATA"),
+};
+
+#define DECLARE_MSM_GPIO_PINS(pin) \
+	static const unsigned int gpio##pin##_pins[] = { pin }
+DECLARE_MSM_GPIO_PINS(0);
+DECLARE_MSM_GPIO_PINS(1);
+DECLARE_MSM_GPIO_PINS(2);
+DECLARE_MSM_GPIO_PINS(3);
+DECLARE_MSM_GPIO_PINS(4);
+DECLARE_MSM_GPIO_PINS(5);
+DECLARE_MSM_GPIO_PINS(6);
+DECLARE_MSM_GPIO_PINS(7);
+DECLARE_MSM_GPIO_PINS(8);
+DECLARE_MSM_GPIO_PINS(9);
+DECLARE_MSM_GPIO_PINS(10);
+DECLARE_MSM_GPIO_PINS(11);
+DECLARE_MSM_GPIO_PINS(12);
+DECLARE_MSM_GPIO_PINS(13);
+DECLARE_MSM_GPIO_PINS(14);
+DECLARE_MSM_GPIO_PINS(15);
+DECLARE_MSM_GPIO_PINS(16);
+DECLARE_MSM_GPIO_PINS(17);
+DECLARE_MSM_GPIO_PINS(18);
+DECLARE_MSM_GPIO_PINS(19);
+DECLARE_MSM_GPIO_PINS(20);
+DECLARE_MSM_GPIO_PINS(21);
+DECLARE_MSM_GPIO_PINS(22);
+DECLARE_MSM_GPIO_PINS(23);
+DECLARE_MSM_GPIO_PINS(24);
+DECLARE_MSM_GPIO_PINS(25);
+DECLARE_MSM_GPIO_PINS(26);
+DECLARE_MSM_GPIO_PINS(27);
+DECLARE_MSM_GPIO_PINS(28);
+DECLARE_MSM_GPIO_PINS(29);
+DECLARE_MSM_GPIO_PINS(30);
+DECLARE_MSM_GPIO_PINS(31);
+DECLARE_MSM_GPIO_PINS(32);
+DECLARE_MSM_GPIO_PINS(33);
+DECLARE_MSM_GPIO_PINS(34);
+DECLARE_MSM_GPIO_PINS(35);
+DECLARE_MSM_GPIO_PINS(36);
+DECLARE_MSM_GPIO_PINS(37);
+DECLARE_MSM_GPIO_PINS(38);
+DECLARE_MSM_GPIO_PINS(39);
+DECLARE_MSM_GPIO_PINS(40);
+DECLARE_MSM_GPIO_PINS(41);
+DECLARE_MSM_GPIO_PINS(42);
+DECLARE_MSM_GPIO_PINS(43);
+DECLARE_MSM_GPIO_PINS(44);
+DECLARE_MSM_GPIO_PINS(45);
+DECLARE_MSM_GPIO_PINS(46);
+DECLARE_MSM_GPIO_PINS(47);
+DECLARE_MSM_GPIO_PINS(48);
+DECLARE_MSM_GPIO_PINS(49);
+DECLARE_MSM_GPIO_PINS(50);
+DECLARE_MSM_GPIO_PINS(51);
+DECLARE_MSM_GPIO_PINS(52);
+DECLARE_MSM_GPIO_PINS(53);
+DECLARE_MSM_GPIO_PINS(54);
+DECLARE_MSM_GPIO_PINS(55);
+DECLARE_MSM_GPIO_PINS(56);
+DECLARE_MSM_GPIO_PINS(57);
+DECLARE_MSM_GPIO_PINS(58);
+DECLARE_MSM_GPIO_PINS(59);
+DECLARE_MSM_GPIO_PINS(60);
+DECLARE_MSM_GPIO_PINS(61);
+DECLARE_MSM_GPIO_PINS(62);
+DECLARE_MSM_GPIO_PINS(63);
+DECLARE_MSM_GPIO_PINS(64);
+DECLARE_MSM_GPIO_PINS(65);
+DECLARE_MSM_GPIO_PINS(66);
+DECLARE_MSM_GPIO_PINS(67);
+DECLARE_MSM_GPIO_PINS(68);
+DECLARE_MSM_GPIO_PINS(69);
+DECLARE_MSM_GPIO_PINS(70);
+DECLARE_MSM_GPIO_PINS(71);
+DECLARE_MSM_GPIO_PINS(72);
+DECLARE_MSM_GPIO_PINS(73);
+DECLARE_MSM_GPIO_PINS(74);
+DECLARE_MSM_GPIO_PINS(75);
+DECLARE_MSM_GPIO_PINS(76);
+DECLARE_MSM_GPIO_PINS(77);
+DECLARE_MSM_GPIO_PINS(78);
+DECLARE_MSM_GPIO_PINS(79);
+DECLARE_MSM_GPIO_PINS(80);
+DECLARE_MSM_GPIO_PINS(81);
+DECLARE_MSM_GPIO_PINS(82);
+DECLARE_MSM_GPIO_PINS(83);
+DECLARE_MSM_GPIO_PINS(84);
+DECLARE_MSM_GPIO_PINS(85);
+DECLARE_MSM_GPIO_PINS(86);
+DECLARE_MSM_GPIO_PINS(87);
+DECLARE_MSM_GPIO_PINS(88);
+DECLARE_MSM_GPIO_PINS(89);
+DECLARE_MSM_GPIO_PINS(90);
+DECLARE_MSM_GPIO_PINS(91);
+DECLARE_MSM_GPIO_PINS(92);
+DECLARE_MSM_GPIO_PINS(93);
+DECLARE_MSM_GPIO_PINS(94);
+DECLARE_MSM_GPIO_PINS(95);
+DECLARE_MSM_GPIO_PINS(96);
+DECLARE_MSM_GPIO_PINS(97);
+DECLARE_MSM_GPIO_PINS(98);
+DECLARE_MSM_GPIO_PINS(99);
+DECLARE_MSM_GPIO_PINS(100);
+DECLARE_MSM_GPIO_PINS(101);
+DECLARE_MSM_GPIO_PINS(102);
+DECLARE_MSM_GPIO_PINS(103);
+DECLARE_MSM_GPIO_PINS(104);
+DECLARE_MSM_GPIO_PINS(105);
+DECLARE_MSM_GPIO_PINS(106);
+DECLARE_MSM_GPIO_PINS(107);
+DECLARE_MSM_GPIO_PINS(108);
+DECLARE_MSM_GPIO_PINS(109);
+DECLARE_MSM_GPIO_PINS(110);
+DECLARE_MSM_GPIO_PINS(111);
+DECLARE_MSM_GPIO_PINS(112);
+DECLARE_MSM_GPIO_PINS(113);
+DECLARE_MSM_GPIO_PINS(114);
+DECLARE_MSM_GPIO_PINS(115);
+DECLARE_MSM_GPIO_PINS(116);
+DECLARE_MSM_GPIO_PINS(117);
+DECLARE_MSM_GPIO_PINS(118);
+DECLARE_MSM_GPIO_PINS(119);
+DECLARE_MSM_GPIO_PINS(120);
+DECLARE_MSM_GPIO_PINS(121);
+DECLARE_MSM_GPIO_PINS(122);
+DECLARE_MSM_GPIO_PINS(123);
+DECLARE_MSM_GPIO_PINS(124);
+DECLARE_MSM_GPIO_PINS(125);
+DECLARE_MSM_GPIO_PINS(126);
+DECLARE_MSM_GPIO_PINS(127);
+DECLARE_MSM_GPIO_PINS(128);
+DECLARE_MSM_GPIO_PINS(129);
+DECLARE_MSM_GPIO_PINS(130);
+DECLARE_MSM_GPIO_PINS(131);
+DECLARE_MSM_GPIO_PINS(132);
+DECLARE_MSM_GPIO_PINS(133);
+DECLARE_MSM_GPIO_PINS(134);
+DECLARE_MSM_GPIO_PINS(135);
+DECLARE_MSM_GPIO_PINS(136);
+DECLARE_MSM_GPIO_PINS(137);
+DECLARE_MSM_GPIO_PINS(138);
+DECLARE_MSM_GPIO_PINS(139);
+DECLARE_MSM_GPIO_PINS(140);
+DECLARE_MSM_GPIO_PINS(141);
+DECLARE_MSM_GPIO_PINS(142);
+DECLARE_MSM_GPIO_PINS(143);
+DECLARE_MSM_GPIO_PINS(144);
+DECLARE_MSM_GPIO_PINS(145);
+DECLARE_MSM_GPIO_PINS(146);
+DECLARE_MSM_GPIO_PINS(147);
+DECLARE_MSM_GPIO_PINS(148);
+DECLARE_MSM_GPIO_PINS(149);
+DECLARE_MSM_GPIO_PINS(150);
+DECLARE_MSM_GPIO_PINS(151);
+DECLARE_MSM_GPIO_PINS(152);
+DECLARE_MSM_GPIO_PINS(153);
+DECLARE_MSM_GPIO_PINS(154);
+DECLARE_MSM_GPIO_PINS(155);
+DECLARE_MSM_GPIO_PINS(156);
+DECLARE_MSM_GPIO_PINS(157);
+DECLARE_MSM_GPIO_PINS(158);
+DECLARE_MSM_GPIO_PINS(159);
+DECLARE_MSM_GPIO_PINS(160);
+DECLARE_MSM_GPIO_PINS(161);
+DECLARE_MSM_GPIO_PINS(162);
+DECLARE_MSM_GPIO_PINS(163);
+DECLARE_MSM_GPIO_PINS(164);
+DECLARE_MSM_GPIO_PINS(165);
+DECLARE_MSM_GPIO_PINS(166);
+DECLARE_MSM_GPIO_PINS(167);
+DECLARE_MSM_GPIO_PINS(168);
+DECLARE_MSM_GPIO_PINS(169);
+DECLARE_MSM_GPIO_PINS(170);
+DECLARE_MSM_GPIO_PINS(171);
+DECLARE_MSM_GPIO_PINS(172);
+DECLARE_MSM_GPIO_PINS(173);
+DECLARE_MSM_GPIO_PINS(174);
+DECLARE_MSM_GPIO_PINS(175);
+DECLARE_MSM_GPIO_PINS(176);
+DECLARE_MSM_GPIO_PINS(177);
+DECLARE_MSM_GPIO_PINS(178);
+DECLARE_MSM_GPIO_PINS(179);
+DECLARE_MSM_GPIO_PINS(180);
+DECLARE_MSM_GPIO_PINS(181);
+DECLARE_MSM_GPIO_PINS(182);
+DECLARE_MSM_GPIO_PINS(183);
+DECLARE_MSM_GPIO_PINS(184);
+DECLARE_MSM_GPIO_PINS(185);
+DECLARE_MSM_GPIO_PINS(186);
+DECLARE_MSM_GPIO_PINS(187);
+DECLARE_MSM_GPIO_PINS(188);
+DECLARE_MSM_GPIO_PINS(189);
+DECLARE_MSM_GPIO_PINS(190);
+DECLARE_MSM_GPIO_PINS(191);
+DECLARE_MSM_GPIO_PINS(192);
+DECLARE_MSM_GPIO_PINS(193);
+DECLARE_MSM_GPIO_PINS(194);
+DECLARE_MSM_GPIO_PINS(195);
+DECLARE_MSM_GPIO_PINS(196);
+DECLARE_MSM_GPIO_PINS(197);
+DECLARE_MSM_GPIO_PINS(198);
+DECLARE_MSM_GPIO_PINS(199);
+DECLARE_MSM_GPIO_PINS(200);
+DECLARE_MSM_GPIO_PINS(201);
+DECLARE_MSM_GPIO_PINS(202);
+DECLARE_MSM_GPIO_PINS(203);
+DECLARE_MSM_GPIO_PINS(204);
+DECLARE_MSM_GPIO_PINS(205);
+DECLARE_MSM_GPIO_PINS(206);
+DECLARE_MSM_GPIO_PINS(207);
+DECLARE_MSM_GPIO_PINS(208);
+DECLARE_MSM_GPIO_PINS(209);
+DECLARE_MSM_GPIO_PINS(210);
+DECLARE_MSM_GPIO_PINS(211);
+DECLARE_MSM_GPIO_PINS(212);
+DECLARE_MSM_GPIO_PINS(213);
+DECLARE_MSM_GPIO_PINS(214);
+DECLARE_MSM_GPIO_PINS(215);
+DECLARE_MSM_GPIO_PINS(216);
+DECLARE_MSM_GPIO_PINS(217);
+DECLARE_MSM_GPIO_PINS(218);
+DECLARE_MSM_GPIO_PINS(219);
+DECLARE_MSM_GPIO_PINS(220);
+DECLARE_MSM_GPIO_PINS(221);
+DECLARE_MSM_GPIO_PINS(222);
+DECLARE_MSM_GPIO_PINS(223);
+DECLARE_MSM_GPIO_PINS(224);
+DECLARE_MSM_GPIO_PINS(225);
+
+static const unsigned int ufs_reset_pins[] = { 226 };
+static const unsigned int sdc2_clk_pins[] = { 227 };
+static const unsigned int sdc2_cmd_pins[] = { 228 };
+static const unsigned int sdc2_data_pins[] = { 229 };
+
+enum maili_functions {
+	msm_mux_gpio,
+	msm_mux_aoss_cti,
+	msm_mux_atest_char,
+	msm_mux_atest_usb,
+	msm_mux_audio_ext_mclk,
+	msm_mux_audio_ref_clk,
+	msm_mux_cam_mclk,
+	msm_mux_cci_async_in,
+	msm_mux_cci_i2c0,
+	msm_mux_cci_i2c1,
+	msm_mux_cci_i2c2,
+	msm_mux_cci_i2c3,
+	msm_mux_cci_timer,
+	msm_mux_coex_espmi,
+	msm_mux_coex_uart1_rx,
+	msm_mux_coex_uart1_tx,
+	msm_mux_dbg_out_clk,
+	msm_mux_ddr_bist,
+	msm_mux_ddr_pxi,
+	msm_mux_dp_hot,
+	msm_mux_egpio,
+	msm_mux_gcc_gp,
+	msm_mux_gnss_adc,
+	msm_mux_host2wlan_sol,
+	msm_mux_host_rst,
+	msm_mux_i2chub0_se0,
+	msm_mux_i2chub0_se1,
+	msm_mux_i2chub0_se2,
+	msm_mux_i2chub0_se3,
+	msm_mux_i2chub0_se4,
+	msm_mux_i2s0,
+	msm_mux_i2s1,
+	msm_mux_ibi_i3c,
+	msm_mux_ibi_i3c_qup5_se0,
+	msm_mux_jitter_bist,
+	msm_mux_mdp_esync0,
+	msm_mux_mdp_esync1,
+	msm_mux_mdp_esync2,
+	msm_mux_mdp_vsync,
+	msm_mux_mdp_vsync_e,
+	msm_mux_mdp_vsync_p,
+	msm_mux_mdp_vsync0_out,
+	msm_mux_mdp_vsync1_out,
+	msm_mux_mdp_vsync2_out,
+	msm_mux_mdp_vsync3_out,
+	msm_mux_mdp_vsync5_out,
+	msm_mux_modem_pps_in,
+	msm_mux_modem_pps_out,
+	msm_mux_nav_gpio,
+	msm_mux_nav_gpio0,
+	msm_mux_nav_gpio3,
+	msm_mux_nav_rffe,
+	msm_mux_pcie0_clk_req_n,
+	msm_mux_pcie1_clk_req_n,
+	msm_mux_pcie1_rst_n,
+	msm_mux_phase_flag,
+	msm_mux_pll_bist_sync,
+	msm_mux_pll_clk_aux,
+	msm_mux_qdss_cti,
+	msm_mux_qlink,
+	msm_mux_qspi,
+	msm_mux_qspi_clk,
+	msm_mux_qspi_cs,
+	msm_mux_qup1_se0,
+	msm_mux_qup1_se1,
+	msm_mux_qup1_se2,
+	msm_mux_qup1_se3,
+	msm_mux_qup1_se4,
+	msm_mux_qup1_se5,
+	msm_mux_qup1_se6,
+	msm_mux_qup1_se7,
+	msm_mux_qup2_se0,
+	msm_mux_qup2_se1,
+	msm_mux_qup2_se2,
+	msm_mux_qup2_se3,
+	msm_mux_qup2_se4_01,
+	msm_mux_qup2_se4_23,
+	msm_mux_qup3_se0,
+	msm_mux_qup3_se1,
+	msm_mux_qup3_se2,
+	msm_mux_qup3_se3,
+	msm_mux_qup3_se4,
+	msm_mux_qup3_se5,
+	msm_mux_qup4_se0,
+	msm_mux_qup4_se1,
+	msm_mux_qup4_se2,
+	msm_mux_qup4_se3_01,
+	msm_mux_qup4_se3_23,
+	msm_mux_qup4_se3_l3,
+	msm_mux_qup4_se4_01,
+	msm_mux_qup4_se4_23,
+	msm_mux_qup4_se4_l3,
+	msm_mux_qup5_se0,
+	msm_mux_rng_rosc,
+	msm_mux_sd_write_protect,
+	msm_mux_sdc2_clk,
+	msm_mux_sdc2_cmd,
+	msm_mux_sdc2_data,
+	msm_mux_sdc2_rclk,
+	msm_mux_sdc4_clk,
+	msm_mux_sdc4_cmd,
+	msm_mux_sdc4_data,
+	msm_mux_sys_throttle,
+	msm_mux_tb_trig_sdc,
+	msm_mux_tmess_rng,
+	msm_mux_tsense_clm,
+	msm_mux_tsense_pwm,
+	msm_mux_uim0,
+	msm_mux_uim1,
+	msm_mux_usb0_hs,
+	msm_mux_usb_phy,
+	msm_mux_vfr,
+	msm_mux_vsense_trigger_mirnat,
+	msm_mux_wcn_sw,
+	msm_mux__,
+};
+
+static const char *const gpio_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio5",
+	"gpio6", "gpio7", "gpio8", "gpio9", "gpio10", "gpio11",
+	"gpio12", "gpio13", "gpio14", "gpio15", "gpio16", "gpio17",
+	"gpio18", "gpio19", "gpio20", "gpio21", "gpio22", "gpio23",
+	"gpio24", "gpio25", "gpio26", "gpio27", "gpio28", "gpio29",
+	"gpio30", "gpio31", "gpio32", "gpio33", "gpio34", "gpio35",
+	"gpio36", "gpio37", "gpio38", "gpio39", "gpio40", "gpio41",
+	"gpio42", "gpio43", "gpio44", "gpio45", "gpio46", "gpio47",
+	"gpio48", "gpio49", "gpio50", "gpio51", "gpio52", "gpio53",
+	"gpio54", "gpio55", "gpio56", "gpio57", "gpio58", "gpio59",
+	"gpio60", "gpio61", "gpio62", "gpio63", "gpio64", "gpio65",
+	"gpio66", "gpio67", "gpio68", "gpio69", "gpio70", "gpio71",
+	"gpio72", "gpio73", "gpio74", "gpio75", "gpio76", "gpio77",
+	"gpio78", "gpio79", "gpio80", "gpio81", "gpio82", "gpio83",
+	"gpio84", "gpio85", "gpio86", "gpio87", "gpio88", "gpio89",
+	"gpio90", "gpio91", "gpio92", "gpio93", "gpio94", "gpio95",
+	"gpio96", "gpio97", "gpio98", "gpio99", "gpio100", "gpio101",
+	"gpio102", "gpio103", "gpio104", "gpio105", "gpio106", "gpio107",
+	"gpio108", "gpio109", "gpio110", "gpio111", "gpio112", "gpio113",
+	"gpio114", "gpio115", "gpio116", "gpio117", "gpio118", "gpio119",
+	"gpio120", "gpio121", "gpio122", "gpio123", "gpio124", "gpio125",
+	"gpio126", "gpio127", "gpio128", "gpio129", "gpio130", "gpio131",
+	"gpio132", "gpio133", "gpio134", "gpio135", "gpio136", "gpio137",
+	"gpio138", "gpio139", "gpio140", "gpio141", "gpio142", "gpio143",
+	"gpio144", "gpio145", "gpio146", "gpio147", "gpio148", "gpio149",
+	"gpio150", "gpio151", "gpio152", "gpio153", "gpio154", "gpio155",
+	"gpio156", "gpio157", "gpio158", "gpio159", "gpio160", "gpio161",
+	"gpio162", "gpio163", "gpio164", "gpio165", "gpio166", "gpio167",
+	"gpio168", "gpio169", "gpio170", "gpio171", "gpio172", "gpio173",
+	"gpio174", "gpio175", "gpio176", "gpio177", "gpio178", "gpio179",
+	"gpio180", "gpio181", "gpio182", "gpio183", "gpio184", "gpio185",
+	"gpio186", "gpio187", "gpio188", "gpio189", "gpio190", "gpio191",
+	"gpio192", "gpio193", "gpio194", "gpio195", "gpio196", "gpio197",
+	"gpio198", "gpio199", "gpio200", "gpio201", "gpio202", "gpio203",
+	"gpio204", "gpio205", "gpio206", "gpio207", "gpio208", "gpio209",
+	"gpio210", "gpio211", "gpio212", "gpio213", "gpio214", "gpio215",
+	"gpio216", "gpio217", "gpio218", "gpio219", "gpio220", "gpio221",
+	"gpio222", "gpio223", "gpio224", "gpio225",
+};
+
+static const char *const aoss_cti_groups[] = {
+	"gpio74", "gpio75", "gpio76", "gpio77",
+};
+
+static const char *const atest_char_groups[] = {
+	"gpio126", "gpio127", "gpio128", "gpio129", "gpio133",
+};
+
+static const char *const atest_usb_groups[] = {
+	"gpio78", "gpio79", "gpio102", "gpio103", "gpio104",
+};
+
+static const char *const audio_ext_mclk_groups[] = {
+	"gpio120", "gpio121",
+};
+
+static const char *const audio_ref_clk_groups[] = {
+	"gpio120",
+};
+
+static const char *const cam_mclk_groups[] = {
+	"gpio90", "gpio91", "gpio92", "gpio93", "gpio94", "gpio95",
+};
+
+static const char *const cci_async_in_groups[] = {
+	"gpio15", "gpio109", "gpio110",
+};
+
+static const char *const cci_i2c0_groups[] = {
+	"gpio109", "gpio110",
+};
+
+static const char *const cci_i2c1_groups[] = {
+	"gpio111", "gpio112",
+};
+
+static const char *const cci_i2c2_groups[] = {
+	"gpio113", "gpio114",
+};
+
+static const char *const cci_i2c3_groups[] = {
+	"gpio107", "gpio160",
+};
+
+static const char *const cci_timer_groups[] = {
+	"gpio105", "gpio106", "gpio107", "gpio159", "gpio160",
+};
+
+static const char *const coex_espmi_groups[] = {
+	"gpio144", "gpio145",
+};
+
+static const char *const coex_uart1_rx_groups[] = {
+	"gpio144",
+};
+
+static const char *const coex_uart1_tx_groups[] = {
+	"gpio145",
+};
+
+static const char *const dbg_out_clk_groups[] = {
+	"gpio82",
+};
+
+static const char *const ddr_bist_groups[] = {
+	"gpio40", "gpio41", "gpio44", "gpio45",
+};
+
+static const char *const ddr_pxi_groups[] = {
+	"gpio66", "gpio67", "gpio68", "gpio69", "gpio70", "gpio71", "gpio72", "gpio73",
+};
+
+static const char *const dp_hot_groups[] = {
+	"gpio47",
+};
+
+static const char *const egpio_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio5",
+	"gpio6", "gpio7", "gpio28", "gpio29", "gpio30", "gpio31",
+	"gpio48", "gpio49", "gpio50", "gpio51", "gpio163", "gpio164",
+	"gpio165", "gpio166", "gpio167", "gpio168", "gpio169", "gpio170",
+	"gpio171", "gpio172", "gpio173", "gpio174", "gpio175", "gpio176",
+	"gpio177", "gpio178", "gpio179", "gpio180", "gpio181", "gpio182",
+	"gpio183", "gpio184", "gpio185", "gpio186", "gpio187", "gpio188",
+	"gpio189", "gpio190", "gpio191", "gpio192", "gpio193", "gpio194",
+	"gpio195", "gpio196", "gpio197", "gpio198", "gpio199", "gpio200",
+	"gpio201", "gpio202", "gpio203", "gpio204", "gpio205", "gpio206",
+	"gpio207", "gpio208", "gpio209", "gpio212", "gpio213", "gpio214",
+	"gpio215", "gpio216", "gpio217", "gpio218",
+};
+
+static const char *const gcc_gp_groups[] = {
+	"gpio17", "gpio86", "gpio87", "gpio97", "gpio155", "gpio156",
+};
+
+static const char *const gnss_adc_groups[] = {
+	"gpio18", "gpio19", "gpio20", "gpio23",
+};
+
+static const char *const host2wlan_sol_groups[] = {
+	"gpio204",
+};
+
+static const char *const host_rst_groups[] = {
+	"gpio106",
+};
+
+static const char *const i2chub0_se0_groups[] = {
+	"gpio66", "gpio67",
+};
+
+static const char *const i2chub0_se1_groups[] = {
+	"gpio78", "gpio79",
+};
+
+static const char *const i2chub0_se2_groups[] = {
+	"gpio68", "gpio69",
+};
+
+static const char *const i2chub0_se3_groups[] = {
+	"gpio70", "gpio71",
+};
+
+static const char *const i2chub0_se4_groups[] = {
+	"gpio72", "gpio73",
+};
+
+static const char *const i2s0_groups[] = {
+	"gpio46", "gpio84", "gpio161", "gpio162",
+};
+
+static const char *const i2s1_groups[] = {
+	"gpio222", "gpio223", "gpio224", "gpio225",
+};
+
+static const char *const ibi_i3c_groups[] = {
+	"gpio0", "gpio1", "gpio4", "gpio5", "gpio8", "gpio9",
+	"gpio12", "gpio13", "gpio24", "gpio25", "gpio28", "gpio29",
+	"gpio32", "gpio33", "gpio36", "gpio37", "gpio48", "gpio49",
+	"gpio60", "gpio61", "gpio64", "gpio65", "gpio85", "gpio89",
+	"gpio117", "gpio118",
+};
+
+static const char *const ibi_i3c_qup5_se0_groups[] = {
+	"gpio8", "gpio9",
+};
+
+static const char *const jitter_bist_groups[] = {
+	"gpio73",
+};
+
+static const char *const mdp_esync0_groups[] = {
+	"gpio88", "gpio100",
+};
+
+static const char *const mdp_esync1_groups[] = {
+	"gpio86", "gpio100",
+};
+
+static const char *const mdp_esync2_groups[] = {
+	"gpio87", "gpio97",
+};
+
+static const char *const mdp_vsync_groups[] = {
+	"gpio86", "gpio87", "gpio88", "gpio97",
+};
+
+static const char *const mdp_vsync_e_groups[] = {
+	"gpio98",
+};
+
+static const char *const mdp_vsync_p_groups[] = {
+	"gpio98",
+};
+
+static const char *const mdp_vsync0_out_groups[] = {
+	"gpio86",
+};
+
+static const char *const mdp_vsync1_out_groups[] = {
+	"gpio86",
+};
+
+static const char *const mdp_vsync2_out_groups[] = {
+	"gpio87",
+};
+
+static const char *const mdp_vsync3_out_groups[] = {
+	"gpio87",
+};
+
+static const char *const mdp_vsync5_out_groups[] = {
+	"gpio87",
+};
+
+static const char *const modem_pps_in_groups[] = {
+	"gpio151",
+};
+
+static const char *const modem_pps_out_groups[] = {
+	"gpio151",
+};
+
+static const char *const nav_gpio_groups[] = {
+	"gpio146", "gpio147", "gpio148", "gpio151",
+};
+
+static const char *const nav_gpio0_groups[] = {
+	"gpio150",
+};
+
+static const char *const nav_gpio3_groups[] = {
+	"gpio150",
+};
+
+static const char *const nav_rffe_groups[] = {
+	"gpio134", "gpio135", "gpio138", "gpio139",
+};
+
+static const char *const pcie0_clk_req_n_groups[] = {
+	"gpio103",
+};
+
+static const char *const pcie1_clk_req_n_groups[] = {
+	"gpio221",
+};
+
+static const char *const pcie1_rst_n_groups[] = {
+	"gpio220",
+};
+
+static const char *const phase_flag_groups[] = {
+	"gpio40", "gpio41", "gpio44", "gpio45", "gpio46", "gpio47", "gpio126",
+	"gpio127", "gpio128", "gpio129", "gpio130", "gpio131", "gpio132",
+	"gpio133", "gpio161", "gpio162", "gpio169", "gpio170", "gpio171",
+	"gpio174", "gpio175", "gpio178", "gpio179", "gpio180", "gpio181",
+	"gpio182", "gpio183", "gpio205", "gpio222", "gpio223", "gpio224",
+	"gpio225",
+};
+
+static const char *const pll_bist_sync_groups[] = {
+	"gpio104",
+};
+
+static const char *const pll_clk_aux_groups[] = {
+	"gpio94",
+};
+
+static const char *const qdss_cti_groups[] = {
+	"gpio72", "gpio73", "gpio82", "gpio83", "gpio222", "gpio223",
+	"gpio224", "gpio225",
+};
+
+static const char *const qlink_groups[] = {
+	"gpio152", "gpio153", "gpio154",
+};
+
+static const char *const qspi_groups[] = {
+	"gpio80", "gpio81", "gpio82", "gpio147",
+};
+
+static const char *const qspi_clk_groups[] = {
+	"gpio83",
+};
+
+static const char *const qspi_cs_groups[] = {
+	"gpio146", "gpio148",
+};
+
+static const char *const qup1_se0_groups[] = {
+	"gpio222", "gpio223", "gpio224", "gpio225",
+};
+
+static const char *const qup1_se1_groups[] = {
+	"gpio74", "gpio75", "gpio76", "gpio77", "gpio188", "gpio189", "gpio192", "gpio193",
+};
+
+static const char *const qup1_se2_groups[] = {
+	"gpio40", "gpio41", "gpio42", "gpio43", "gpio130", "gpio131", "gpio132",
+};
+
+static const char *const qup1_se3_groups[] = {
+	"gpio44", "gpio45", "gpio46", "gpio47",
+};
+
+static const char *const qup1_se4_groups[] = {
+	"gpio36", "gpio37", "gpio38", "gpio39",
+};
+
+static const char *const qup1_se5_groups[] = {
+	"gpio52", "gpio53", "gpio54", "gpio55",
+};
+
+static const char *const qup1_se6_groups[] = {
+	"gpio56", "gpio57", "gpio58", "gpio59",
+};
+
+static const char *const qup1_se7_groups[] = {
+	"gpio60", "gpio61", "gpio62", "gpio63",
+};
+
+static const char *const qup2_se0_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio3",
+};
+
+static const char *const qup2_se1_groups[] = {
+	"gpio4", "gpio5", "gpio6", "gpio7",
+};
+
+static const char *const qup2_se2_groups[] = {
+	"gpio117", "gpio118", "gpio119", "gpio120",
+};
+
+static const char *const qup2_se3_groups[] = {
+	"gpio97", "gpio122", "gpio123", "gpio124", "gpio125",
+};
+
+static const char *const qup2_se4_01_groups[] = {
+	"gpio208", "gpio209",
+};
+
+static const char *const qup2_se4_23_groups[] = {
+	"gpio208", "gpio209",
+};
+
+static const char *const qup3_se0_groups[] = {
+	"gpio64", "gpio65", "gpio66", "gpio67", "gpio82", "gpio83",
+};
+
+static const char *const qup3_se1_groups[] = {
+	"gpio8", "gpio9", "gpio10", "gpio11", "gpio12", "gpio13", "gpio15",
+};
+
+static const char *const qup3_se2_groups[] = {
+	"gpio12", "gpio13", "gpio14", "gpio15",
+};
+
+static const char *const qup3_se3_groups[] = {
+	"gpio16", "gpio17", "gpio18", "gpio19",
+};
+
+static const char *const qup3_se4_groups[] = {
+	"gpio20", "gpio21", "gpio22", "gpio23",
+};
+
+static const char *const qup3_se5_groups[] = {
+	"gpio24", "gpio25", "gpio26", "gpio27",
+};
+
+static const char *const qup4_se0_groups[] = {
+	"gpio48", "gpio49", "gpio50", "gpio51",
+};
+
+static const char *const qup4_se1_groups[] = {
+	"gpio28", "gpio29", "gpio30", "gpio31",
+};
+
+static const char *const qup4_se2_groups[] = {
+	"gpio32", "gpio33", "gpio34", "gpio35",
+};
+
+static const char *const qup4_se3_01_groups[] = {
+	"gpio84", "gpio121",
+};
+
+static const char *const qup4_se3_23_groups[] = {
+	"gpio84", "gpio121",
+};
+
+static const char *const qup4_se3_l3_groups[] = {
+	"gpio98",
+};
+
+static const char *const qup4_se4_01_groups[] = {
+	"gpio161", "gpio162",
+};
+
+static const char *const qup4_se4_23_groups[] = {
+	"gpio161", "gpio162",
+};
+
+static const char *const qup4_se4_l3_groups[] = {
+	"gpio88",
+};
+
+static const char *const qup5_se0_groups[] = {
+	"gpio8", "gpio9", "gpio10", "gpio11", "gpio12", "gpio13", "gpio14",
+	"gpio16", "gpio17", "gpio85", "gpio89", "gpio100", "gpio214", "gpio215",
+};
+
+static const char *const rng_rosc_groups[] = {
+	"gpio64", "gpio65", "gpio66", "gpio84",
+};
+
+static const char *const sd_write_protect_groups[] = {
+	"gpio85",
+};
+
+static const char *const sdc2_clk_groups[] = {
+	"gpio56",
+};
+
+static const char *const sdc2_cmd_groups[] = {
+	"gpio57",
+};
+
+static const char *const sdc2_data_groups[] = {
+	"gpio74", "gpio75", "gpio76", "gpio77",
+};
+
+static const char *const sdc2_rclk_groups[] = {
+	"gpio54",
+};
+
+static const char *const sdc4_clk_groups[] = {
+	"gpio83",
+};
+
+static const char *const sdc4_cmd_groups[] = {
+	"gpio148",
+};
+
+static const char *const sdc4_data_groups[] = {
+	"gpio80", "gpio81", "gpio82", "gpio147",
+};
+
+static const char *const sys_throttle_groups[] = {
+	"gpio99",
+};
+
+static const char *const tb_trig_sdc_groups[] = {
+	"gpio88", "gpio146",
+};
+
+static const char *const tmess_rng_groups[] = {
+	"gpio64", "gpio65", "gpio66", "gpio84",
+};
+
+static const char *const tsense_clm_groups[] = {
+	"gpio87", "gpio97", "gpio98", "gpio99", "gpio105", "gpio106",
+	"gpio159",
+};
+
+static const char *const tsense_pwm_groups[] = {
+	"gpio35", "gpio38", "gpio58", "gpio87", "gpio97", "gpio98", "gpio99",
+};
+
+static const char *const uim0_groups[] = {
+	"gpio126", "gpio127", "gpio128", "gpio129",
+};
+
+static const char *const uim1_groups[] = {
+	"gpio36", "gpio37", "gpio39", "gpio54", "gpio55", "gpio56", "gpio70",
+	"gpio71", "gpio72", "gpio130", "gpio131", "gpio132", "gpio133",
+};
+
+static const char *const usb0_hs_groups[] = {
+	"gpio79",
+};
+
+static const char *const usb_phy_groups[] = {
+	"gpio59", "gpio60",
+};
+
+static const char *const vfr_groups[] = {
+	"gpio146", "gpio151",
+};
+
+static const char *const vsense_trigger_mirnat_groups[] = {
+	"gpio16",
+};
+
+static const char *const wcn_sw_groups[] = {
+	"gpio18", "gpio19", "gpio155", "gpio156",
+};
+
+static const struct pinfunction maili_functions[] = {
+	MSM_GPIO_PIN_FUNCTION(gpio),
+	MSM_PIN_FUNCTION(aoss_cti),
+	MSM_PIN_FUNCTION(atest_char),
+	MSM_PIN_FUNCTION(atest_usb),
+	MSM_PIN_FUNCTION(audio_ext_mclk),
+	MSM_PIN_FUNCTION(audio_ref_clk),
+	MSM_PIN_FUNCTION(cam_mclk),
+	MSM_PIN_FUNCTION(cci_async_in),
+	MSM_PIN_FUNCTION(cci_i2c0),
+	MSM_PIN_FUNCTION(cci_i2c1),
+	MSM_PIN_FUNCTION(cci_i2c2),
+	MSM_PIN_FUNCTION(cci_i2c3),
+	MSM_PIN_FUNCTION(cci_timer),
+	MSM_PIN_FUNCTION(coex_espmi),
+	MSM_PIN_FUNCTION(coex_uart1_rx),
+	MSM_PIN_FUNCTION(coex_uart1_tx),
+	MSM_PIN_FUNCTION(dbg_out_clk),
+	MSM_PIN_FUNCTION(ddr_bist),
+	MSM_PIN_FUNCTION(ddr_pxi),
+	MSM_PIN_FUNCTION(dp_hot),
+	MSM_PIN_FUNCTION(egpio),
+	MSM_PIN_FUNCTION(gcc_gp),
+	MSM_PIN_FUNCTION(gnss_adc),
+	MSM_PIN_FUNCTION(host2wlan_sol),
+	MSM_PIN_FUNCTION(host_rst),
+	MSM_PIN_FUNCTION(i2chub0_se0),
+	MSM_PIN_FUNCTION(i2chub0_se1),
+	MSM_PIN_FUNCTION(i2chub0_se2),
+	MSM_PIN_FUNCTION(i2chub0_se3),
+	MSM_PIN_FUNCTION(i2chub0_se4),
+	MSM_PIN_FUNCTION(i2s0),
+	MSM_PIN_FUNCTION(i2s1),
+	MSM_PIN_FUNCTION(ibi_i3c),
+	MSM_PIN_FUNCTION(ibi_i3c_qup5_se0),
+	MSM_PIN_FUNCTION(jitter_bist),
+	MSM_PIN_FUNCTION(mdp_esync0),
+	MSM_PIN_FUNCTION(mdp_esync1),
+	MSM_PIN_FUNCTION(mdp_esync2),
+	MSM_PIN_FUNCTION(mdp_vsync),
+	MSM_PIN_FUNCTION(mdp_vsync_e),
+	MSM_PIN_FUNCTION(mdp_vsync_p),
+	MSM_PIN_FUNCTION(mdp_vsync0_out),
+	MSM_PIN_FUNCTION(mdp_vsync1_out),
+	MSM_PIN_FUNCTION(mdp_vsync2_out),
+	MSM_PIN_FUNCTION(mdp_vsync3_out),
+	MSM_PIN_FUNCTION(mdp_vsync5_out),
+	MSM_PIN_FUNCTION(modem_pps_in),
+	MSM_PIN_FUNCTION(modem_pps_out),
+	MSM_PIN_FUNCTION(nav_gpio),
+	MSM_PIN_FUNCTION(nav_gpio0),
+	MSM_PIN_FUNCTION(nav_gpio3),
+	MSM_PIN_FUNCTION(nav_rffe),
+	MSM_PIN_FUNCTION(pcie0_clk_req_n),
+	MSM_PIN_FUNCTION(pcie1_clk_req_n),
+	MSM_PIN_FUNCTION(pcie1_rst_n),
+	MSM_PIN_FUNCTION(phase_flag),
+	MSM_PIN_FUNCTION(pll_bist_sync),
+	MSM_PIN_FUNCTION(pll_clk_aux),
+	MSM_PIN_FUNCTION(qdss_cti),
+	MSM_PIN_FUNCTION(qlink),
+	MSM_PIN_FUNCTION(qspi),
+	MSM_PIN_FUNCTION(qspi_clk),
+	MSM_PIN_FUNCTION(qspi_cs),
+	MSM_PIN_FUNCTION(qup1_se0),
+	MSM_PIN_FUNCTION(qup1_se1),
+	MSM_PIN_FUNCTION(qup1_se2),
+	MSM_PIN_FUNCTION(qup1_se3),
+	MSM_PIN_FUNCTION(qup1_se4),
+	MSM_PIN_FUNCTION(qup1_se5),
+	MSM_PIN_FUNCTION(qup1_se6),
+	MSM_PIN_FUNCTION(qup1_se7),
+	MSM_PIN_FUNCTION(qup2_se0),
+	MSM_PIN_FUNCTION(qup2_se1),
+	MSM_PIN_FUNCTION(qup2_se2),
+	MSM_PIN_FUNCTION(qup2_se3),
+	MSM_PIN_FUNCTION(qup2_se4_01),
+	MSM_PIN_FUNCTION(qup2_se4_23),
+	MSM_PIN_FUNCTION(qup3_se0),
+	MSM_PIN_FUNCTION(qup3_se1),
+	MSM_PIN_FUNCTION(qup3_se2),
+	MSM_PIN_FUNCTION(qup3_se3),
+	MSM_PIN_FUNCTION(qup3_se4),
+	MSM_PIN_FUNCTION(qup3_se5),
+	MSM_PIN_FUNCTION(qup4_se0),
+	MSM_PIN_FUNCTION(qup4_se1),
+	MSM_PIN_FUNCTION(qup4_se2),
+	MSM_PIN_FUNCTION(qup4_se3_01),
+	MSM_PIN_FUNCTION(qup4_se3_23),
+	MSM_PIN_FUNCTION(qup4_se3_l3),
+	MSM_PIN_FUNCTION(qup4_se4_01),
+	MSM_PIN_FUNCTION(qup4_se4_23),
+	MSM_PIN_FUNCTION(qup4_se4_l3),
+	MSM_PIN_FUNCTION(qup5_se0),
+	MSM_PIN_FUNCTION(rng_rosc),
+	MSM_PIN_FUNCTION(sd_write_protect),
+	MSM_PIN_FUNCTION(sdc2_clk),
+	MSM_PIN_FUNCTION(sdc2_cmd),
+	MSM_PIN_FUNCTION(sdc2_data),
+	MSM_PIN_FUNCTION(sdc2_rclk),
+	MSM_PIN_FUNCTION(sdc4_clk),
+	MSM_PIN_FUNCTION(sdc4_cmd),
+	MSM_PIN_FUNCTION(sdc4_data),
+	MSM_PIN_FUNCTION(sys_throttle),
+	MSM_PIN_FUNCTION(tb_trig_sdc),
+	MSM_PIN_FUNCTION(tmess_rng),
+	MSM_PIN_FUNCTION(tsense_clm),
+	MSM_PIN_FUNCTION(tsense_pwm),
+	MSM_PIN_FUNCTION(uim0),
+	MSM_PIN_FUNCTION(uim1),
+	MSM_PIN_FUNCTION(usb0_hs),
+	MSM_PIN_FUNCTION(usb_phy),
+	MSM_PIN_FUNCTION(vfr),
+	MSM_PIN_FUNCTION(vsense_trigger_mirnat),
+	MSM_PIN_FUNCTION(wcn_sw),
+};
+
+static const struct msm_pingroup maili_groups[] = {
+	[0] = PINGROUP(0, qup2_se0, ibi_i3c, _, _, _, _, _, _, _, _, egpio),
+	[1] = PINGROUP(1, qup2_se0, ibi_i3c, _, _, _, _, _, _, _, _, egpio),
+	[2] = PINGROUP(2, qup2_se0, _, _, _, _, _, _, _, _, _, egpio),
+	[3] = PINGROUP(3, qup2_se0, _, _, _, _, _, _, _, _, _, egpio),
+	[4] = PINGROUP(4, qup2_se1, ibi_i3c, _, _, _, _, _, _, _, _, egpio),
+	[5] = PINGROUP(5, qup2_se1, ibi_i3c, _, _, _, _, _, _, _, _, egpio),
+	[6] = PINGROUP(6, qup2_se1, _, _, _, _, _, _, _, _, _, egpio),
+	[7] = PINGROUP(7, qup2_se1, _, _, _, _, _, _, _, _, _, egpio),
+	[8] = PINGROUP(8, qup3_se1, ibi_i3c, qup5_se0, ibi_i3c_qup5_se0, _, _, _, _, _, _, _),
+	[9] = PINGROUP(9, qup3_se1, ibi_i3c, qup5_se0, ibi_i3c_qup5_se0, _, _, _, _, _, _, _),
+	[10] = PINGROUP(10, qup3_se1, qup5_se0, _, _, _, _, _, _, _, _, _),
+	[11] = PINGROUP(11, qup3_se1, qup5_se0, _, _, _, _, _, _, _, _, _),
+	[12] = PINGROUP(12, qup3_se2, ibi_i3c, qup3_se1, qup5_se0, _, _, _, _, _, _, _),
+	[13] = PINGROUP(13, qup3_se2, ibi_i3c, qup3_se1, qup5_se0, _, _, _, _, _, _, _),
+	[14] = PINGROUP(14, qup3_se2, qup5_se0, _, _, _, _, _, _, _, _, _),
+	[15] = PINGROUP(15, qup3_se2, cci_async_in, qup3_se1, _, _, _, _, _, _, _, _),
+	[16] = PINGROUP(16, qup3_se3, qup5_se0, _, vsense_trigger_mirnat, _, _, _, _, _, _, _),
+	[17] = PINGROUP(17, qup3_se3, qup5_se0, gcc_gp, _, _, _, _, _, _, _, _),
+	[18] = PINGROUP(18, wcn_sw, qup3_se3, _, gnss_adc, _, _, _, _, _, _, _),
+	[19] = PINGROUP(19, wcn_sw, qup3_se3, _, gnss_adc, _, _, _, _, _, _, _),
+	[20] = PINGROUP(20, qup3_se4, _, gnss_adc, _, _, _, _, _, _, _, _),
+	[21] = PINGROUP(21, qup3_se4, _, _, _, _, _, _, _, _, _, _),
+	[22] = PINGROUP(22, qup3_se4, _, _, _, _, _, _, _, _, _, _),
+	[23] = PINGROUP(23, qup3_se4, _, gnss_adc, _, _, _, _, _, _, _, _),
+	[24] = PINGROUP(24, qup3_se5, ibi_i3c, _, _, _, _, _, _, _, _, _),
+	[25] = PINGROUP(25, qup3_se5, ibi_i3c, _, _, _, _, _, _, _, _, _),
+	[26] = PINGROUP(26, qup3_se5, _, _, _, _, _, _, _, _, _, _),
+	[27] = PINGROUP(27, qup3_se5, _, _, _, _, _, _, _, _, _, _),
+	[28] = PINGROUP(28, qup4_se1, ibi_i3c, _, _, _, _, _, _, _, _, egpio),
+	[29] = PINGROUP(29, qup4_se1, ibi_i3c, _, _, _, _, _, _, _, _, egpio),
+	[30] = PINGROUP(30, qup4_se1, _, _, _, _, _, _, _, _, _, egpio),
+	[31] = PINGROUP(31, qup4_se1, _, _, _, _, _, _, _, _, _, egpio),
+	[32] = PINGROUP(32, qup4_se2, ibi_i3c, _, _, _, _, _, _, _, _, _),
+	[33] = PINGROUP(33, qup4_se2, ibi_i3c, _, _, _, _, _, _, _, _, _),
+	[34] = PINGROUP(34, qup4_se2, _, _, _, _, _, _, _, _, _, _),
+	[35] = PINGROUP(35, qup4_se2, tsense_pwm, _, _, _, _, _, _, _, _, _),
+	[36] = PINGROUP(36, qup1_se4, uim1, ibi_i3c, _, _, _, _, _, _, _, _),
+	[37] = PINGROUP(37, qup1_se4, uim1, ibi_i3c, _, _, _, _, _, _, _, _),
+	[38] = PINGROUP(38, qup1_se4, tsense_pwm, _, _, _, _, _, _, _, _, _),
+	[39] = PINGROUP(39, qup1_se4, uim1, _, _, _, _, _, _, _, _, _),
+	[40] = PINGROUP(40, qup1_se2, ddr_bist, phase_flag, _, _, _, _, _, _, _, _),
+	[41] = PINGROUP(41, qup1_se2, ddr_bist, phase_flag, _, _, _, _, _, _, _, _),
+	[42] = PINGROUP(42, qup1_se2, _, _, _, _, _, _, _, _, _, _),
+	[43] = PINGROUP(43, qup1_se2, _, _, _, _, _, _, _, _, _, _),
+	[44] = PINGROUP(44, qup1_se3, ddr_bist, phase_flag, _, _, _, _, _, _, _, _),
+	[45] = PINGROUP(45, qup1_se3, ddr_bist, phase_flag, _, _, _, _, _, _, _, _),
+	[46] = PINGROUP(46, qup1_se3, i2s0, phase_flag, _, _, _, _, _, _, _, _),
+	[47] = PINGROUP(47, qup1_se3, dp_hot, phase_flag, _, _, _, _, _, _, _, _),
+	[48] = PINGROUP(48, qup4_se0, ibi_i3c, _, _, _, _, _, _, _, _, egpio),
+	[49] = PINGROUP(49, qup4_se0, ibi_i3c, _, _, _, _, _, _, _, _, egpio),
+	[50] = PINGROUP(50, qup4_se0, _, _, _, _, _, _, _, _, _, egpio),
+	[51] = PINGROUP(51, qup4_se0, _, _, _, _, _, _, _, _, _, egpio),
+	[52] = PINGROUP(52, qup1_se5, _, _, _, _, _, _, _, _, _, _),
+	[53] = PINGROUP(53, qup1_se5, _, _, _, _, _, _, _, _, _, _),
+	[54] = PINGROUP(54, qup1_se5, uim1, sdc2_rclk, _, _, _, _, _, _, _, _),
+	[55] = PINGROUP(55, qup1_se5, uim1, _, _, _, _, _, _, _, _, _),
+	[56] = PINGROUP(56, qup1_se6, uim1, sdc2_clk, _, _, _, _, _, _, _, _),
+	[57] = PINGROUP(57, qup1_se6, sdc2_cmd, _, _, _, _, _, _, _, _, _),
+	[58] = PINGROUP(58, qup1_se6, tsense_pwm, _, _, _, _, _, _, _, _, _),
+	[59] = PINGROUP(59, qup1_se6, usb_phy, _, _, _, _, _, _, _, _, _),
+	[60] = PINGROUP(60, qup1_se7, usb_phy, ibi_i3c, _, _, _, _, _, _, _, _),
+	[61] = PINGROUP(61, qup1_se7, ibi_i3c, _, _, _, _, _, _, _, _, _),
+	[62] = PINGROUP(62, qup1_se7, _, _, _, _, _, _, _, _, _, _),
+	[63] = PINGROUP(63, qup1_se7, _, _, _, _, _, _, _, _, _, _),
+	[64] = PINGROUP(64, qup3_se0, rng_rosc, tmess_rng, ibi_i3c, _, _, _, _, _, _, _),
+	[65] = PINGROUP(65, qup3_se0, rng_rosc, tmess_rng, ibi_i3c, _, _, _, _, _, _, _),
+	[66] = PINGROUP(66, i2chub0_se0, qup3_se0, rng_rosc, tmess_rng, _, ddr_pxi, _, _, _, _, _),
+	[67] = PINGROUP(67, i2chub0_se0, qup3_se0, _, ddr_pxi, _, _, _, _, _, _, _),
+	[68] = PINGROUP(68, i2chub0_se2, _, ddr_pxi, _, _, _, _, _, _, _, _),
+	[69] = PINGROUP(69, i2chub0_se2, _, ddr_pxi, _, _, _, _, _, _, _, _),
+	[70] = PINGROUP(70, i2chub0_se3, uim1, _, _, ddr_pxi, _, _, _, _, _, _),
+	[71] = PINGROUP(71, i2chub0_se3, uim1, _, _, ddr_pxi, _, _, _, _, _, _),
+	[72] = PINGROUP(72, i2chub0_se4, uim1, qdss_cti, _, ddr_pxi, _, _, _, _, _, _),
+	[73] = PINGROUP(73, i2chub0_se4, qdss_cti, jitter_bist, ddr_pxi, _, _, _, _, _, _, _),
+	[74] = PINGROUP(74, qup1_se1, aoss_cti, sdc2_data, _, _, _, _, _, _, _, _),
+	[75] = PINGROUP(75, qup1_se1, aoss_cti, sdc2_data, _, _, _, _, _, _, _, _),
+	[76] = PINGROUP(76, qup1_se1, aoss_cti, sdc2_data, _, _, _, _, _, _, _, _),
+	[77] = PINGROUP(77, qup1_se1, aoss_cti, sdc2_data, _, _, _, _, _, _, _, _),
+	[78] = PINGROUP(78, i2chub0_se1, _, atest_usb, _, _, _, _, _, _, _, _),
+	[79] = PINGROUP(79, i2chub0_se1, usb0_hs, _, atest_usb, _, _, _, _, _, _, _),
+	[80] = PINGROUP(80, sdc4_data, qspi, _, _, _, _, _, _, _, _, _),
+	[81] = PINGROUP(81, sdc4_data, qspi, _, _, _, _, _, _, _, _, _),
+	[82] = PINGROUP(82, sdc4_data, qdss_cti, qspi, qup3_se0, dbg_out_clk, _, _, _, _, _, _),
+	[83] = PINGROUP(83, sdc4_clk, qdss_cti, qspi_clk, qup3_se0, _, _, _, _, _, _, _),
+	[84] = PINGROUP(84, qup4_se3_01, qup4_se3_23, rng_rosc, tmess_rng, i2s0, _, _, _, _, _, _),
+	[85] = PINGROUP(85, sd_write_protect, qup5_se0, ibi_i3c, _, _, _, _, _, _, _, _),
+	[86] = PINGROUP(86, mdp_vsync, mdp_vsync0_out, mdp_vsync1_out, mdp_esync1, gcc_gp,
+			_, _, _, _, _, _),
+	[87] = PINGROUP(87, mdp_vsync, mdp_vsync2_out, mdp_vsync3_out, mdp_vsync5_out, mdp_esync2,
+			gcc_gp, tsense_clm, tsense_pwm, _, _, _),
+	[88] = PINGROUP(88, mdp_esync0, mdp_vsync, qup4_se4_l3, tb_trig_sdc, _, _, _, _, _, _, _),
+	[89] = PINGROUP(89, qup5_se0, ibi_i3c, _, _, _, _, _, _, _, _, _),
+	[90] = PINGROUP(90, cam_mclk, _, _, _, _, _, _, _, _, _, _),
+	[91] = PINGROUP(91, cam_mclk, _, _, _, _, _, _, _, _, _, _),
+	[92] = PINGROUP(92, cam_mclk, _, _, _, _, _, _, _, _, _, _),
+	[93] = PINGROUP(93, cam_mclk, _, _, _, _, _, _, _, _, _, _),
+	[94] = PINGROUP(94, cam_mclk, pll_clk_aux, _, _, _, _, _, _, _, _, _),
+	[95] = PINGROUP(95, cam_mclk, _, _, _, _, _, _, _, _, _, _),
+	[96] = PINGROUP(96, _, _, _, _, _, _, _, _, _, _, _),
+	[97] = PINGROUP(97, mdp_esync2, qup2_se3, mdp_vsync, gcc_gp, tsense_clm, tsense_pwm,
+			_, _, _, _, _),
+	[98] = PINGROUP(98, mdp_vsync_e, qup4_se3_l3, mdp_vsync_p, tsense_clm, tsense_pwm,
+			_, _, _, _, _, _),
+	[99] = PINGROUP(99, sys_throttle, tsense_clm, tsense_pwm, _, _, _, _, _, _, _, _),
+	[100] = PINGROUP(100, mdp_esync1, mdp_esync0, qup5_se0, _, _, _, _, _, _, _, _),
+	[101] = PINGROUP(101, _, _, _, _, _, _, _, _, _, _, _),
+	[102] = PINGROUP(102, atest_usb, _, _, _, _, _, _, _, _, _, _),
+	[103] = PINGROUP(103, pcie0_clk_req_n, atest_usb, _, _, _, _, _, _, _, _, _),
+	[104] = PINGROUP(104, pll_bist_sync, atest_usb, _, _, _, _, _, _, _, _, _),
+	[105] = PINGROUP(105, cci_timer, tsense_clm, _, _, _, _, _, _, _, _, _),
+	[106] = PINGROUP(106, host_rst, cci_timer, tsense_clm, _, _, _, _, _, _, _, _),
+	[107] = PINGROUP(107, cci_i2c3, cci_timer, _, _, _, _, _, _, _, _, _),
+	[108] = PINGROUP(108, _, _, _, _, _, _, _, _, _, _, _),
+	[109] = PINGROUP(109, cci_i2c0, cci_async_in, _, _, _, _, _, _, _, _, _),
+	[110] = PINGROUP(110, cci_i2c0, cci_async_in, _, _, _, _, _, _, _, _, _),
+	[111] = PINGROUP(111, cci_i2c1, _, _, _, _, _, _, _, _, _, _),
+	[112] = PINGROUP(112, cci_i2c1, _, _, _, _, _, _, _, _, _, _),
+	[113] = PINGROUP(113, cci_i2c2, _, _, _, _, _, _, _, _, _, _),
+	[114] = PINGROUP(114, cci_i2c2, _, _, _, _, _, _, _, _, _, _),
+	[115] = PINGROUP(115, _, _, _, _, _, _, _, _, _, _, _),
+	[116] = PINGROUP(116, _, _, _, _, _, _, _, _, _, _, _),
+	[117] = PINGROUP(117, qup2_se2, ibi_i3c, _, _, _, _, _, _, _, _, _),
+	[118] = PINGROUP(118, qup2_se2, ibi_i3c, _, _, _, _, _, _, _, _, _),
+	[119] = PINGROUP(119, qup2_se2, _, _, _, _, _, _, _, _, _, _),
+	[120] = PINGROUP(120, qup2_se2, audio_ext_mclk, audio_ref_clk, _, _, _, _, _, _, _, _),
+	[121] = PINGROUP(121, audio_ext_mclk, qup4_se3_01, qup4_se3_23, _, _, _, _, _, _, _, _),
+	[122] = PINGROUP(122, qup2_se3, _, _, _, _, _, _, _, _, _, _),
+	[123] = PINGROUP(123, qup2_se3, _, _, _, _, _, _, _, _, _, _),
+	[124] = PINGROUP(124, qup2_se3, _, _, _, _, _, _, _, _, _, _),
+	[125] = PINGROUP(125, qup2_se3, _, _, _, _, _, _, _, _, _, _),
+	[126] = PINGROUP(126, uim0, phase_flag, _, atest_char, _, _, _, _, _, _, _),
+	[127] = PINGROUP(127, uim0, phase_flag, _, atest_char, _, _, _, _, _, _, _),
+	[128] = PINGROUP(128, uim0, phase_flag, _, atest_char, _, _, _, _, _, _, _),
+	[129] = PINGROUP(129, uim0, phase_flag, _, atest_char, _, _, _, _, _, _, _),
+	[130] = PINGROUP(130, uim1, qup1_se2, _, phase_flag, _, _, _, _, _, _, _),
+	[131] = PINGROUP(131, uim1, qup1_se2, _, phase_flag, _, _, _, _, _, _, _),
+	[132] = PINGROUP(132, uim1, qup1_se2, _, phase_flag, _, _, _, _, _, _, _),
+	[133] = PINGROUP(133, uim1, phase_flag, _, atest_char, _, _, _, _, _, _, _),
+	[134] = PINGROUP(134, _, _, nav_rffe, _, _, _, _, _, _, _, _),
+	[135] = PINGROUP(135, _, _, nav_rffe, _, _, _, _, _, _, _, _),
+	[136] = PINGROUP(136, _, _, _, _, _, _, _, _, _, _, _),
+	[137] = PINGROUP(137, _, _, _, _, _, _, _, _, _, _, _),
+	[138] = PINGROUP(138, _, _, nav_rffe, _, _, _, _, _, _, _, _),
+	[139] = PINGROUP(139, _, _, nav_rffe, _, _, _, _, _, _, _, _),
+	[140] = PINGROUP(140, _, _, _, _, _, _, _, _, _, _, _),
+	[141] = PINGROUP(141, _, _, _, _, _, _, _, _, _, _, _),
+	[142] = PINGROUP(142, _, _, _, _, _, _, _, _, _, _, _),
+	[143] = PINGROUP(143, _, _, _, _, _, _, _, _, _, _, _),
+	[144] = PINGROUP(144, coex_uart1_rx, coex_espmi, _, _, _, _, _, _, _, _, _),
+	[145] = PINGROUP(145, coex_uart1_tx, coex_espmi, _, _, _, _, _, _, _, _, _),
+	[146] = PINGROUP(146, _, vfr, nav_gpio, tb_trig_sdc, qspi_cs, _, _, _, _, _, _),
+	[147] = PINGROUP(147, _, nav_gpio, sdc4_data, qspi, _, _, _, _, _, _, _),
+	[148] = PINGROUP(148, nav_gpio, _, sdc4_cmd, qspi_cs, _, _, _, _, _, _, _),
+	[149] = PINGROUP(149, _, _, _, _, _, _, _, _, _, _, _),
+	[150] = PINGROUP(150, nav_gpio0, nav_gpio3, _, _, _, _, _, _, _, _, _),
+	[151] = PINGROUP(151, nav_gpio, vfr, modem_pps_in, modem_pps_out, _, _, _, _, _, _, _),
+	[152] = PINGROUP(152, qlink, _, _, _, _, _, _, _, _, _, _),
+	[153] = PINGROUP(153, qlink, _, _, _, _, _, _, _, _, _, _),
+	[154] = PINGROUP(154, qlink, _, _, _, _, _, _, _, _, _, _),
+	[155] = PINGROUP(155, wcn_sw, gcc_gp, _, _, _, _, _, _, _, _, _),
+	[156] = PINGROUP(156, wcn_sw, gcc_gp, _, _, _, _, _, _, _, _, _),
+	[157] = PINGROUP(157, _, _, _, _, _, _, _, _, _, _, _),
+	[158] = PINGROUP(158, _, _, _, _, _, _, _, _, _, _, _),
+	[159] = PINGROUP(159, cci_timer, tsense_clm, _, _, _, _, _, _, _, _, _),
+	[160] = PINGROUP(160, cci_timer, cci_i2c3, _, _, _, _, _, _, _, _, _),
+	[161] = PINGROUP(161, qup4_se4_01, qup4_se4_23, i2s0, phase_flag, _, _, _, _, _, _, _),
+	[162] = PINGROUP(162, qup4_se4_01, qup4_se4_23, i2s0, phase_flag, _, _, _, _, _, _, _),
+	[163] = PINGROUP(163, _, _, _, _, _, _, _, _, _, _, egpio),
+	[164] = PINGROUP(164, _, _, _, _, _, _, _, _, _, _, egpio),
+	[165] = PINGROUP(165, _, _, _, _, _, _, _, _, _, _, egpio),
+	[166] = PINGROUP(166, _, _, _, _, _, _, _, _, _, _, egpio),
+	[167] = PINGROUP(167, _, _, _, _, _, _, _, _, _, _, egpio),
+	[168] = PINGROUP(168, _, _, _, _, _, _, _, _, _, _, egpio),
+	[169] = PINGROUP(169, phase_flag, _, _, _, _, _, _, _, _, _, egpio),
+	[170] = PINGROUP(170, phase_flag, _, _, _, _, _, _, _, _, _, egpio),
+	[171] = PINGROUP(171, phase_flag, _, _, _, _, _, _, _, _, _, egpio),
+	[172] = PINGROUP(172, _, _, _, _, _, _, _, _, _, _, egpio),
+	[173] = PINGROUP(173, _, _, _, _, _, _, _, _, _, _, egpio),
+	[174] = PINGROUP(174, phase_flag, _, _, _, _, _, _, _, _, _, egpio),
+	[175] = PINGROUP(175, phase_flag, _, _, _, _, _, _, _, _, _, egpio),
+	[176] = PINGROUP(176, _, _, _, _, _, _, _, _, _, _, egpio),
+	[177] = PINGROUP(177, _, _, _, _, _, _, _, _, _, _, egpio),
+	[178] = PINGROUP(178, phase_flag, _, _, _, _, _, _, _, _, _, egpio),
+	[179] = PINGROUP(179, phase_flag, _, _, _, _, _, _, _, _, _, egpio),
+	[180] = PINGROUP(180, phase_flag, _, _, _, _, _, _, _, _, _, egpio),
+	[181] = PINGROUP(181, phase_flag, _, _, _, _, _, _, _, _, _, egpio),
+	[182] = PINGROUP(182, phase_flag, _, _, _, _, _, _, _, _, _, egpio),
+	[183] = PINGROUP(183, phase_flag, _, _, _, _, _, _, _, _, _, egpio),
+	[184] = PINGROUP(184, _, _, _, _, _, _, _, _, _, _, egpio),
+	[185] = PINGROUP(185, _, _, _, _, _, _, _, _, _, _, egpio),
+	[186] = PINGROUP(186, _, _, _, _, _, _, _, _, _, _, egpio),
+	[187] = PINGROUP(187, _, _, _, _, _, _, _, _, _, _, egpio),
+	[188] = PINGROUP(188, qup1_se1, _, _, _, _, _, _, _, _, _, egpio),
+	[189] = PINGROUP(189, qup1_se1, _, _, _, _, _, _, _, _, _, egpio),
+	[190] = PINGROUP(190, _, _, _, _, _, _, _, _, _, _, egpio),
+	[191] = PINGROUP(191, _, _, _, _, _, _, _, _, _, _, egpio),
+	[192] = PINGROUP(192, qup1_se1, _, _, _, _, _, _, _, _, _, egpio),
+	[193] = PINGROUP(193, qup1_se1, _, _, _, _, _, _, _, _, _, egpio),
+	[194] = PINGROUP(194, _, _, _, _, _, _, _, _, _, _, egpio),
+	[195] = PINGROUP(195, _, _, _, _, _, _, _, _, _, _, egpio),
+	[196] = PINGROUP(196, _, _, _, _, _, _, _, _, _, _, egpio),
+	[197] = PINGROUP(197, _, _, _, _, _, _, _, _, _, _, egpio),
+	[198] = PINGROUP(198, _, _, _, _, _, _, _, _, _, _, egpio),
+	[199] = PINGROUP(199, _, _, _, _, _, _, _, _, _, _, egpio),
+	[200] = PINGROUP(200, _, _, _, _, _, _, _, _, _, _, egpio),
+	[201] = PINGROUP(201, _, _, _, _, _, _, _, _, _, _, egpio),
+	[202] = PINGROUP(202, _, _, _, _, _, _, _, _, _, _, egpio),
+	[203] = PINGROUP(203, _, _, _, _, _, _, _, _, _, _, egpio),
+	[204] = PINGROUP(204, host2wlan_sol, _, _, _, _, _, _, _, _, _, egpio),
+	[205] = PINGROUP(205, phase_flag, _, _, _, _, _, _, _, _, _, egpio),
+	[206] = PINGROUP(206, _, _, _, _, _, _, _, _, _, _, egpio),
+	[207] = PINGROUP(207, _, _, _, _, _, _, _, _, _, _, egpio),
+	[208] = PINGROUP(208, qup2_se4_01, qup2_se4_23, _, _, _, _, _, _, _, _, egpio),
+	[209] = PINGROUP(209, qup2_se4_01, qup2_se4_23, _, _, _, _, _, _, _, _, egpio),
+	[210] = PINGROUP(210, _, _, _, _, _, _, _, _, _, _, _),
+	[211] = PINGROUP(211, _, _, _, _, _, _, _, _, _, _, _),
+	[212] = PINGROUP(212, _, _, _, _, _, _, _, _, _, _, egpio),
+	[213] = PINGROUP(213, _, _, _, _, _, _, _, _, _, _, egpio),
+	[214] = PINGROUP(214, qup5_se0, _, _, _, _, _, _, _, _, _, egpio),
+	[215] = PINGROUP(215, qup5_se0, _, _, _, _, _, _, _, _, _, egpio),
+	[216] = PINGROUP(216, _, _, _, _, _, _, _, _, _, _, egpio),
+	[217] = PINGROUP(217, _, _, _, _, _, _, _, _, _, _, egpio),
+	[218] = PINGROUP(218, _, _, _, _, _, _, _, _, _, _, egpio),
+	[219] = PINGROUP(219, _, _, _, _, _, _, _, _, _, _, _),
+	[220] = PINGROUP(220, pcie1_rst_n, _, _, _, _, _, _, _, _, _, _),
+	[221] = PINGROUP(221, pcie1_clk_req_n, _, _, _, _, _, _, _, _, _, _),
+	[222] = PINGROUP(222, qup1_se0, i2s1, qdss_cti, phase_flag, _, _, _, _, _, _, _),
+	[223] = PINGROUP(223, qup1_se0, i2s1, qdss_cti, phase_flag, _, _, _, _, _, _, _),
+	[224] = PINGROUP(224, qup1_se0, i2s1, qdss_cti, phase_flag, _, _, _, _, _, _, _),
+	[225] = PINGROUP(225, qup1_se0, i2s1, qdss_cti, phase_flag, _, _, _, _, _, _, _),
+	[226] = UFS_RESET(ufs_reset, 0xf1004, 0xf2000),
+	[227] = SDC_QDSD_PINGROUP(sdc2_clk, 0xe6000, 14, 6),
+	[228] = SDC_QDSD_PINGROUP(sdc2_cmd, 0xe6000, 11, 3),
+	[229] = SDC_QDSD_PINGROUP(sdc2_data, 0xe6000, 9, 0),
+};
+
+static const struct msm_gpio_wakeirq_map maili_pdc_map[] = {
+	{ 0, 111 }, { 3, 119 }, { 4, 112 }, { 7, 113 }, { 8, 114 }, { 11, 115 },
+	{ 12, 121 }, { 15, 137 }, { 16, 122 }, { 17, 147 }, { 18, 149 }, { 19, 118 },
+	{ 23, 123 }, { 24, 124 }, { 27, 125 }, { 28, 131 }, { 31, 132 }, { 32, 133 },
+	{ 35, 107 }, { 36, 134 }, { 39, 135 }, { 43, 136 }, { 47, 160 }, { 48, 141 },
+	{ 51, 120 }, { 55, 110 }, { 57, 142 }, { 58, 143 }, { 59, 144 }, { 60, 145 },
+	{ 61, 151 }, { 63, 130 }, { 64, 116 }, { 65, 129 }, { 67, 138 }, { 68, 152 },
+	{ 69, 153 }, { 75, 157 }, { 77, 154 }, { 78, 155 }, { 79, 161 }, { 80, 162 },
+	{ 81, 163 }, { 82, 164 }, { 83, 171 }, { 84, 140 }, { 85, 165 }, { 86, 166 },
+	{ 87, 167 }, { 88, 168 }, { 95, 169 }, { 96, 170 }, { 97, 139 }, { 98, 156 },
+	{ 99, 117 }, { 100, 199 }, { 103, 173 }, { 104, 174 }, { 117, 201 }, { 120, 175 },
+	{ 123, 176 }, { 125, 177 }, { 129, 159 }, { 133, 106 }, { 144, 178 }, { 146, 179 },
+	{ 151, 180 }, { 152, 181 }, { 155, 128 }, { 158, 126 }, { 162, 148 }, { 164, 182 },
+	{ 165, 183 }, { 167, 184 }, { 168, 185 }, { 174, 186 }, { 177, 187 }, { 179, 188 },
+	{ 183, 189 }, { 184, 190 }, { 185, 191 }, { 186, 158 }, { 188, 150 }, { 202, 108 },
+	{ 203, 109 }, { 205, 146 }, { 209, 192 }, { 213, 127 }, { 215, 200 }, { 216, 193 },
+	{ 220, 172 }, { 221, 194 }, { 222, 195 }, { 223, 196 }, { 224, 197 }, { 225, 198 },
+};
+
+static const struct msm_pinctrl_soc_data maili_tlmm = {
+	.pins = maili_pins,
+	.npins = ARRAY_SIZE(maili_pins),
+	.functions = maili_functions,
+	.nfunctions = ARRAY_SIZE(maili_functions),
+	.groups = maili_groups,
+	.ngroups = ARRAY_SIZE(maili_groups),
+	.ngpios = 227,
+	.wakeirq_map = maili_pdc_map,
+	.nwakeirq_map = ARRAY_SIZE(maili_pdc_map),
+	.egpio_func = 11,
+};
+
+static const struct of_device_id maili_tlmm_of_match[] = {
+	{ .compatible = "qcom,maili-tlmm", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, maili_tlmm_of_match);
+
+static int maili_tlmm_probe(struct platform_device *pdev)
+{
+	return msm_pinctrl_probe(pdev, &maili_tlmm);
+}
+
+static struct platform_driver maili_tlmm_driver = {
+	.driver = {
+		.name = "maili-tlmm",
+		.of_match_table = maili_tlmm_of_match,
+	},
+	.probe = maili_tlmm_probe,
+};
+
+static int __init maili_tlmm_init(void)
+{
+	return platform_driver_register(&maili_tlmm_driver);
+}
+arch_initcall(maili_tlmm_init);
+
+static void __exit maili_tlmm_exit(void)
+{
+	platform_driver_unregister(&maili_tlmm_driver);
+}
+module_exit(maili_tlmm_exit);
+
+MODULE_DESCRIPTION("Qualcomm Maili TLMM driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/qcom/pinctrl-milos-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-milos-lpass-lpi.c
index 72b8ffd97860b..cb4934cd6f757 100644
--- a/drivers/pinctrl/qcom/pinctrl-milos-lpass-lpi.c
+++ b/drivers/pinctrl/qcom/pinctrl-milos-lpass-lpi.c
@@ -148,6 +148,33 @@ static const struct lpi_pingroup milos_groups[] = {
 	LPI_PINGROUP(22, LPI_NO_SLEW, i2s3_data, dmic4_data, ext_mclk1_e, _),
 };
 
+static const struct lpi_pingroup eliza_groups[] = {
+	LPI_PINGROUP(0, 11, swr_tx_clk, i2s0_clk, _, _),
+	LPI_PINGROUP(1, 11, swr_tx_data, i2s0_ws, _, _),
+	LPI_PINGROUP(2, 11, swr_tx_data, i2s0_data, _, _),
+	LPI_PINGROUP(3, 11, swr_rx_clk, i2s0_data, _, _),
+	LPI_PINGROUP(4, 11, swr_rx_data, i2s0_data, _, _),
+	LPI_PINGROUP(5, 11, swr_rx_data, ext_mclk1_c, i2s0_data, _),
+	LPI_PINGROUP(6, LPI_NO_SLEW, dmic1_clk, i2s1_clk, _, _),
+	LPI_PINGROUP(7, LPI_NO_SLEW, dmic1_data, i2s1_ws, _, _),
+	LPI_PINGROUP(8, LPI_NO_SLEW, dmic2_clk, i2s1_data, _, _),
+	LPI_PINGROUP(9, LPI_NO_SLEW, dmic2_data, i2s1_data, ext_mclk1_b, _),
+	LPI_PINGROUP(10, 11, wsa_swr_clk, i2s2_clk, _, _),
+	LPI_PINGROUP(11, 11, wsa_swr_data, i2s2_ws, _, _),
+	LPI_PINGROUP(12, LPI_NO_SLEW, dmic3_clk, i2s2_data, _, _),
+	LPI_PINGROUP(13, LPI_NO_SLEW, dmic3_data, i2s2_data, ext_mclk1_a, _),
+	LPI_PINGROUP(14, 11, swr_tx_data, ext_mclk1_d, _, _),
+	/* gpio15 - gpio18 do not really exist */
+	LPI_PINGROUP(15, 11, _, _, _, _),
+	LPI_PINGROUP(16, 11, _, _, _, _),
+	LPI_PINGROUP(17, LPI_NO_SLEW, _, _, _, _),
+	LPI_PINGROUP(18, LPI_NO_SLEW, _, _, _, _),
+	LPI_PINGROUP(19, LPI_NO_SLEW, i2s3_clk, slimbus_clk, qca_swr_clk, _),
+	LPI_PINGROUP(20, LPI_NO_SLEW, i2s3_ws, slimbus_data, qca_swr_data, _),
+	LPI_PINGROUP(21, LPI_NO_SLEW, i2s3_data, dmic4_clk, _, _),
+	LPI_PINGROUP(22, LPI_NO_SLEW, i2s3_data, dmic4_data, ext_mclk1_e, _),
+};
+
 static const struct lpi_function milos_functions[] = {
 	LPI_FUNCTION(gpio),
 	LPI_FUNCTION(dmic1_clk),
@@ -196,8 +223,21 @@ static const struct lpi_pinctrl_variant_data milos_lpi_data = {
 	.nfunctions = ARRAY_SIZE(milos_functions),
 };
 
+static const struct lpi_pinctrl_variant_data eliza_lpi_data = {
+	.pins = milos_lpi_pins,
+	.npins = ARRAY_SIZE(milos_lpi_pins),
+	.groups = eliza_groups,
+	.ngroups = ARRAY_SIZE(eliza_groups),
+	.functions = milos_functions,
+	.nfunctions = ARRAY_SIZE(milos_functions),
+	.flags = LPI_FLAG_SLEW_RATE_SAME_REG,
+};
+
 static const struct of_device_id lpi_pinctrl_of_match[] = {
 	{
+		.compatible = "qcom,eliza-lpass-lpi-pinctrl",
+		.data = &eliza_lpi_data,
+	}, {
 		.compatible = "qcom,milos-lpass-lpi-pinctrl",
 		.data = &milos_lpi_data,
 	},
diff --git a/drivers/pinctrl/qcom/pinctrl-msm.c b/drivers/pinctrl/qcom/pinctrl-msm.c
index 11db6564c44df..9c720f49465bd 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm.c
@@ -810,8 +810,7 @@ static const struct gpio_chip msm_gpio_template = {
  * Algorithm comes from Google's msmgpio driver.
  */
 static void msm_gpio_update_dual_edge_pos(struct msm_pinctrl *pctrl,
-					  const struct msm_pingroup *g,
-					  struct irq_data *d)
+					  const struct msm_pingroup *g)
 {
 	int loop_limit = 100;
 	unsigned val, val2, intstat;
@@ -995,6 +994,16 @@ static void msm_gpio_irq_ack(struct irq_data *d)
 	if (test_bit(d->hwirq, pctrl->skip_wake_irqs)) {
 		if (test_bit(d->hwirq, pctrl->dual_edge_irqs))
 			msm_gpio_update_dual_edge_parent(d);
+
+		/*
+		 * During early initialization of the IRQ hierarchy,
+		 * irq_ack() is called by __irq_set_handler() before
+		 * the parent IRQ chip has been set up. This is why
+		 * we additionally need to check for d->parent_data->chip.
+		 */
+
+		if (d->parent_data->chip && d->parent_data->chip->irq_ack)
+			irq_chip_ack_parent(d);
 		return;
 	}
 
@@ -1005,7 +1014,7 @@ static void msm_gpio_irq_ack(struct irq_data *d)
 	msm_ack_intr_status(pctrl, g);
 
 	if (test_bit(d->hwirq, pctrl->dual_edge_irqs))
-		msm_gpio_update_dual_edge_pos(pctrl, g, d);
+		msm_gpio_update_dual_edge_pos(pctrl, g);
 
 	raw_spin_unlock_irqrestore(&pctrl->lock, flags);
 }
@@ -1035,8 +1044,6 @@ static void msm_gpio_irq_init_valid_mask(struct gpio_chip *gc,
 	const struct msm_pingroup *g;
 	int i;
 
-	bitmap_fill(valid_mask, ngpios);
-
 	for (i = 0; i < ngpios; i++) {
 		g = &pctrl->soc->groups[i];
 
@@ -1067,7 +1074,10 @@ static int msm_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 
 	if (test_bit(d->hwirq, pctrl->skip_wake_irqs)) {
 		clear_bit(d->hwirq, pctrl->dual_edge_irqs);
-		irq_set_handler_locked(d, handle_fasteoi_irq);
+		if (type & IRQ_TYPE_LEVEL_MASK)
+			irq_set_handler_locked(d, handle_fasteoi_irq);
+		else
+			irq_set_handler_locked(d, handle_fasteoi_ack_irq);
 		return 0;
 	}
 
@@ -1177,7 +1187,7 @@ static int msm_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 		msm_ack_intr_status(pctrl, g);
 
 	if (test_bit(d->hwirq, pctrl->dual_edge_irqs))
-		msm_gpio_update_dual_edge_pos(pctrl, g, d);
+		msm_gpio_update_dual_edge_pos(pctrl, g);
 
 	raw_spin_unlock_irqrestore(&pctrl->lock, flags);
 
@@ -1240,12 +1250,12 @@ static int msm_gpio_irq_reqres(struct irq_data *d)
 	/*
 	 * If the wakeup_enable bit is present and marked as available for the
 	 * requested GPIO, it should be enabled when the GPIO is marked as
-	 * wake irq in order to allow the interrupt event to be transfered to
-	 * the PDC HW.
+	 * wake irq in order to allow the interrupt event to be transferred to
+	 * the PDC/MPM HW.
 	 * While the name implies only the wakeup event, it's also required for
 	 * the interrupt event.
 	 */
-	if (test_bit(d->hwirq, pctrl->skip_wake_irqs) && g->intr_wakeup_present_bit) {
+	if (g->intr_wakeup_present_bit) {
 		u32 intr_cfg;
 
 		raw_spin_lock_irqsave(&pctrl->lock, flags);
@@ -1273,7 +1283,7 @@ static void msm_gpio_irq_relres(struct irq_data *d)
 	unsigned long flags;
 
 	/* Disable the wakeup_enable bit if it has been set in msm_gpio_irq_reqres() */
-	if (test_bit(d->hwirq, pctrl->skip_wake_irqs) && g->intr_wakeup_present_bit) {
+	if (g->intr_wakeup_present_bit) {
 		u32 intr_cfg;
 
 		raw_spin_lock_irqsave(&pctrl->lock, flags);
diff --git a/drivers/pinctrl/qcom/pinctrl-sc8280xp.c b/drivers/pinctrl/qcom/pinctrl-sc8280xp.c
index 4056b9fa32f8c..e018bd11626ca 100644
--- a/drivers/pinctrl/qcom/pinctrl-sc8280xp.c
+++ b/drivers/pinctrl/qcom/pinctrl-sc8280xp.c
@@ -1881,16 +1881,17 @@ static const struct msm_gpio_wakeirq_map sc8280xp_pdc_map[] = {
 	{ 126, 200 }, { 127, 225 }, { 128, 262 }, { 129, 201 },
 	{ 130, 209 }, { 131, 173 }, { 132, 202 }, { 136, 210 },
 	{ 138, 171 }, { 139, 226 }, { 140, 227 }, { 142, 228 },
-	{ 144, 229 }, { 145, 230 }, { 146, 231 }, { 148, 232 },
-	{ 149, 233 }, { 150, 234 }, { 152, 235 }, { 154, 212 },
-	{ 157, 213 }, { 161, 219 }, { 170, 236 }, { 171, 221 },
-	{ 174, 222 }, { 175, 237 }, { 176, 223 }, { 177, 170 },
-	{ 180, 238 }, { 181, 239 }, { 182, 240 }, { 183, 241 },
-	{ 184, 242 }, { 185, 243 }, { 190, 178 }, { 193, 184 },
-	{ 196, 185 }, { 198, 186 }, { 200, 174 }, { 201, 175 },
-	{ 205, 176 }, { 206, 177 }, { 208, 187 }, { 210, 198 },
-	{ 211, 199 }, { 212, 204 }, { 215, 205 }, { 220, 188 },
-	{ 221, 194 }, { 223, 195 }, { 225, 196 }, { 227, 197 },
+	{ 143, 261 }, { 144, 229 }, { 145, 230 }, { 146, 231 },
+	{ 148, 232 }, { 149, 233 }, { 150, 234 }, { 151, 264 },
+	{ 152, 235 }, { 154, 212 }, { 157, 213 }, { 161, 219 },
+	{ 170, 236 }, { 171, 221 }, { 174, 222 }, { 175, 237 },
+	{ 176, 223 }, { 177, 170 }, { 180, 238 }, { 181, 239 },
+	{ 182, 240 }, { 183, 241 }, { 184, 242 }, { 185, 243 },
+	{ 190, 178 }, { 193, 184 }, { 196, 185 }, { 198, 186 },
+	{ 200, 174 }, { 201, 175 }, { 205, 176 }, { 206, 177 },
+	{ 208, 187 }, { 210, 198 }, { 211, 199 }, { 212, 204 },
+	{ 215, 205 }, { 220, 188 }, { 221, 194 }, { 223, 195 },
+	{ 225, 196 }, { 227, 197 },
 };
 
 static struct msm_pinctrl_soc_data sc8280xp_pinctrl = {
diff --git a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
index cdd61dae74cf9..922c27177b5ac 100644
--- a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
+++ b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
@@ -1262,6 +1262,7 @@ static const struct of_device_id pmic_gpio_of_match[] = {
 	{ .compatible = "qcom,pmc8380-gpio", .data = (void *) 10 },
 	{ .compatible = "qcom,pmcx0102-gpio", .data = (void *)14 },
 	{ .compatible = "qcom,pmd8028-gpio", .data = (void *) 4 },
+	{ .compatible = "qcom,pmg1110-gpio", .data = (void *) 4 },
 	{ .compatible = "qcom,pmh0101-gpio", .data = (void *)18 },
 	{ .compatible = "qcom,pmh0104-gpio", .data = (void *)8 },
 	{ .compatible = "qcom,pmh0110-gpio", .data = (void *)14 },
diff --git a/drivers/pinctrl/qcom/pinctrl-x1e80100.c b/drivers/pinctrl/qcom/pinctrl-x1e80100.c
index 8d2b8246170b4..e4c0abcd95b9f 100644
--- a/drivers/pinctrl/qcom/pinctrl-x1e80100.c
+++ b/drivers/pinctrl/qcom/pinctrl-x1e80100.c
@@ -1836,9 +1836,7 @@ static const struct msm_pinctrl_soc_data x1e80100_pinctrl = {
 	.ngroups = ARRAY_SIZE(x1e80100_groups),
 	.ngpios = 239,
 	.wakeirq_map = x1e80100_pdc_map,
-	/* TODO: Enabling PDC currently breaks GPIO interrupts */
-	.nwakeirq_map = 0,
-	/* .nwakeirq_map = ARRAY_SIZE(x1e80100_pdc_map), */
+	.nwakeirq_map = ARRAY_SIZE(x1e80100_pdc_map),
 	.egpio_func = 9,
 };
 
diff --git a/drivers/pinctrl/qcom/tlmm-test.c b/drivers/pinctrl/qcom/tlmm-test.c
index 007d6539ceced..13c417731be7a 100644
--- a/drivers/pinctrl/qcom/tlmm-test.c
+++ b/drivers/pinctrl/qcom/tlmm-test.c
@@ -273,7 +273,6 @@ static void tlmm_test_low(struct kunit *test)
 	int i;
 
 	priv->intr_op = TLMM_TEST_COUNT | TLMM_TEST_OUTPUT_HIGH;
-	atomic_set(&priv->intr_op_remain, 9);
 
 	tlmm_output_high();
 
@@ -298,7 +297,6 @@ static void tlmm_test_high(struct kunit *test)
 	int i;
 
 	priv->intr_op = TLMM_TEST_COUNT | TLMM_TEST_OUTPUT_LOW;
-	atomic_set(&priv->intr_op_remain, 9);
 
 	tlmm_output_low();
 
@@ -521,7 +519,6 @@ static void tlmm_test_rising_while_disabled(struct kunit *test)
 	unsigned int before_edge;
 
 	priv->intr_op = TLMM_TEST_COUNT;
-	atomic_set(&priv->thread_op_remain, 10);
 
 	tlmm_output_low();
 
diff --git a/drivers/pinctrl/renesas/gpio.c b/drivers/pinctrl/renesas/gpio.c
index 2293af6428493..b49a3e14da913 100644
--- a/drivers/pinctrl/renesas/gpio.c
+++ b/drivers/pinctrl/renesas/gpio.c
@@ -271,13 +271,41 @@ static int gpio_function_request(struct gpio_chip *gc, unsigned offset)
 	return ret;
 }
 
+static void sh_pfc_fwnode_put(void *data)
+{
+	fwnode_handle_put(data);
+}
+
 static int gpio_function_setup(struct sh_pfc_chip *chip)
 {
 	struct sh_pfc *pfc = chip->pfc;
 	struct gpio_chip *gc = &chip->gpio_chip;
+	struct fwnode_handle *fwnode = dev_fwnode(pfc->dev);
 
 	gc->request = gpio_function_request;
 
+	if (is_software_node(fwnode)) {
+		fwnode = fwnode_get_named_child_node(fwnode, "functions");
+		if (fwnode) {
+			int ret;
+
+			ret = devm_add_action_or_reset(pfc->dev,
+						       sh_pfc_fwnode_put,
+						       fwnode);
+			if (ret)
+				return ret;
+
+			gc->fwnode = fwnode;
+		}
+	}
+
+	/*
+	 * If we did not find 'functions' node, explicitly mask the parent's
+	 * fwnode to prevent gpiolib from reusing it for function GPIOs.
+	 */
+	if (!gc->fwnode)
+		gc->fwnode = ERR_PTR(-ENODEV);
+
 	gc->label = pfc->info->name;
 	gc->owner = THIS_MODULE;
 	gc->base = pfc->nr_gpio_pins;
diff --git a/drivers/pinctrl/renesas/pinctrl-rza2.c b/drivers/pinctrl/renesas/pinctrl-rza2.c
index 8618f32ed26a4..4cab641c33509 100644
--- a/drivers/pinctrl/renesas/pinctrl-rza2.c
+++ b/drivers/pinctrl/renesas/pinctrl-rza2.c
@@ -44,12 +44,12 @@ struct rza2_pinctrl_priv {
 	struct device *dev;
 	void __iomem *base;
 
-	struct pinctrl_pin_desc *pins;
 	struct pinctrl_desc desc;
 	struct pinctrl_dev *pctl;
 	struct pinctrl_gpio_range gpio_range;
-	int npins;
+	unsigned int npins;
 	struct mutex mutex; /* serialize adding groups and functions */
+	struct pinctrl_pin_desc pins[] __counted_by(npins);
 };
 
 #define RZA2_PDR(port)		(0x0000 + (port) * 2)	/* Direction 16-bit */
@@ -289,21 +289,15 @@ static int rza2_gpio_register(struct rza2_pinctrl_priv *priv)
 
 static int rza2_pinctrl_register(struct rza2_pinctrl_priv *priv)
 {
-	struct pinctrl_pin_desc *pins;
 	unsigned int i;
 	int ret;
 
-	pins = devm_kcalloc(priv->dev, priv->npins, sizeof(*pins), GFP_KERNEL);
-	if (!pins)
-		return -ENOMEM;
-
-	priv->pins = pins;
-	priv->desc.pins = pins;
+	priv->desc.pins = priv->pins;
 	priv->desc.npins = priv->npins;
 
 	for (i = 0; i < priv->npins; i++) {
-		pins[i].number = i;
-		pins[i].name = rza2_gpio_names[i];
+		priv->pins[i].number = i;
+		priv->pins[i].name = rza2_gpio_names[i];
 	}
 
 	ret = devm_pinctrl_register_and_init(priv->dev, &priv->desc, priv,
@@ -482,12 +476,17 @@ static const struct pinmux_ops rza2_pinmux_ops = {
 static int rza2_pinctrl_probe(struct platform_device *pdev)
 {
 	struct rza2_pinctrl_priv *priv;
+	unsigned int npins;
 	int ret;
 
-	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	npins = (uintptr_t)of_device_get_match_data(&pdev->dev) *
+		      RZA2_PINS_PER_PORT;
+
+	priv = devm_kzalloc(&pdev->dev, struct_size(priv, pins, npins), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
+	priv->npins = npins;
 	priv->dev = &pdev->dev;
 
 	priv->base = devm_platform_ioremap_resource(pdev, 0);
@@ -498,9 +497,6 @@ static int rza2_pinctrl_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, priv);
 
-	priv->npins = (int)(uintptr_t)of_device_get_match_data(&pdev->dev) *
-		      RZA2_PINS_PER_PORT;
-
 	priv->desc.name		= DRIVER_NAME;
 	priv->desc.pctlops	= &rza2_pinctrl_ops;
 	priv->desc.pmxops	= &rza2_pinmux_ops;
diff --git a/drivers/pinctrl/samsung/pinctrl-exynos-arm64.c b/drivers/pinctrl/samsung/pinctrl-exynos-arm64.c
index fe9f92cb037ea..1aa977d2a1f80 100644
--- a/drivers/pinctrl/samsung/pinctrl-exynos-arm64.c
+++ b/drivers/pinctrl/samsung/pinctrl-exynos-arm64.c
@@ -943,6 +943,138 @@ const struct samsung_pinctrl_of_match_data exynos850_of_data __initconst = {
 	.num_ctrl	= ARRAY_SIZE(exynos850_pin_ctrl),
 };
 
+/* pin banks of exynos8855 pin-controller 0 (ALIVE) */
+static const struct samsung_pin_bank_data exynos8855_pin_banks0[] __initconst = {
+	GS101_PIN_BANK_EINTW(8, 0x000, "gpa0", 0x00, 0x00),
+	GS101_PIN_BANK_EINTW(4, 0x020, "gpa1", 0x04, 0x08),
+	EXYNOS850_PIN_BANK_EINTN(3, 0x040, "gpq0"),
+	EXYNOS850_PIN_BANK_EINTN(2, 0x060, "gpq1"),
+	GS101_PIN_BANK_EINTW(1, 0x080, "gpc0", 0x08, 0x10),
+	GS101_PIN_BANK_EINTW(1, 0x0a0, "gpc1", 0x0c, 0x14),
+	GS101_PIN_BANK_EINTW(1, 0x0c0, "gpc2", 0x10, 0x18),
+	GS101_PIN_BANK_EINTW(1, 0x0e0, "gpc3", 0x14, 0x1c),
+	GS101_PIN_BANK_EINTW(1, 0x100, "gpc4", 0x18, 0x20),
+	GS101_PIN_BANK_EINTW(1, 0x120, "gpc5", 0x1c, 0x24),
+	GS101_PIN_BANK_EINTW(1, 0x140, "gpc6", 0x20, 0x28),
+	GS101_PIN_BANK_EINTW(1, 0x160, "gpc7", 0x24, 0x2c),
+	GS101_PIN_BANK_EINTW(1, 0x180, "gpc8", 0x28, 0x30),
+	GS101_PIN_BANK_EINTW(1, 0x1a0, "gpc9", 0x2c, 0x34),
+	GS101_PIN_BANK_EINTW(1, 0x1c0, "gpc10", 0x30, 0x38),
+	GS101_PIN_BANK_EINTW(1, 0x1e0, "gpc11", 0x34, 0x3c),
+	GS101_PIN_BANK_EINTW(1, 0x200, "gpc12", 0x38, 0x40),
+	GS101_PIN_BANK_EINTW(1, 0x220, "gpc13", 0x3c, 0x44),
+	GS101_PIN_BANK_EINTW(1, 0x240, "gpc14", 0x40, 0x48),
+	GS101_PIN_BANK_EINTW(1, 0x260, "gpj0", 0x44, 0x4c),
+	GS101_PIN_BANK_EINTW(1, 0x280, "gpj1", 0x48, 0x50),
+	GS101_PIN_BANK_EINTW(1, 0x2a0, "gpj2", 0x4c, 0x54),
+};
+
+/* pin banks of exynos8855 pin-controller 1 (CMGP) */
+static const struct samsung_pin_bank_data exynos8855_pin_banks1[] __initconst = {
+	GS101_PIN_BANK_EINTW(1, 0x00,  "gpm0",  0x00, 0x00),
+	GS101_PIN_BANK_EINTW(1, 0x20,  "gpm1",  0x04, 0x04),
+	GS101_PIN_BANK_EINTW(1, 0x40,  "gpm2",  0x08, 0x08),
+	GS101_PIN_BANK_EINTW(1, 0x60,  "gpm3",  0x0c, 0x0c),
+	GS101_PIN_BANK_EINTW(1, 0x80,  "gpm4",  0x10, 0x10),
+	GS101_PIN_BANK_EINTW(1, 0xa0,  "gpm5",  0x14, 0x14),
+	GS101_PIN_BANK_EINTW(1, 0xc0,  "gpm6",  0x18, 0x18),
+	GS101_PIN_BANK_EINTW(1, 0xe0,  "gpm7",  0x1c, 0x1c),
+	GS101_PIN_BANK_EINTW(1, 0x100, "gpm8",  0x20, 0x20),
+	GS101_PIN_BANK_EINTW(1, 0x120, "gpm9",  0x24, 0x24),
+	GS101_PIN_BANK_EINTW(1, 0x140, "gpm10", 0x28, 0x28),
+	GS101_PIN_BANK_EINTW(1, 0x160, "gpm11", 0x2c, 0x2c),
+	GS101_PIN_BANK_EINTW(1, 0x180, "gpm12", 0x30, 0x30),
+	GS101_PIN_BANK_EINTW(1, 0x1a0, "gpm13", 0x34, 0x34),
+	GS101_PIN_BANK_EINTW(1, 0x1c0, "gpm14", 0x38, 0x38),
+	GS101_PIN_BANK_EINTW(1, 0x1e0, "gpm15", 0x3c, 0x3c),
+	GS101_PIN_BANK_EINTW(1, 0x200, "gpm16", 0x40, 0x40),
+	GS101_PIN_BANK_EINTW(1, 0x220, "gpm17", 0x44, 0x44),
+	GS101_PIN_BANK_EINTW(1, 0x240, "gpm18", 0x48, 0x48),
+	GS101_PIN_BANK_EINTW(1, 0x260, "gpm19", 0x4c, 0x4c),
+	GS101_PIN_BANK_EINTW(1, 0x280, "gpm20", 0x50, 0x50),
+	GS101_PIN_BANK_EINTW(1, 0x2a0, "gpm21", 0x54, 0x54),
+};
+
+/* pin banks of exynos8855 pin-controller 2 (HSI UFS) */
+static const struct samsung_pin_bank_data exynos8855_pin_banks2[] __initconst = {
+	GS101_PIN_BANK_EINTG(2, 0x0, "gpf3", 0x00, 0x00),
+};
+
+/* pin banks of exynos8855 pin-controller 3 (PERIC) */
+static const struct samsung_pin_bank_data exynos8855_pin_banks3[] __initconst = {
+	GS101_PIN_BANK_EINTG(8, 0x0,   "gpp0", 0x00, 0x00),
+	GS101_PIN_BANK_EINTG(8, 0x20,  "gpp1", 0x04, 0x08),
+	GS101_PIN_BANK_EINTG(6, 0x40,  "gpp2", 0x08, 0x10),
+	GS101_PIN_BANK_EINTG(4, 0x60,  "gpg0", 0x0c, 0x18),
+	GS101_PIN_BANK_EINTG(3, 0x80,  "gpg1", 0x10, 0x1c),
+	GS101_PIN_BANK_EINTG(6, 0xa0,  "gpb0", 0x14, 0x20),
+	GS101_PIN_BANK_EINTG(4, 0xc0,  "gpb1", 0x18, 0x28),
+};
+
+/* pin banks of exynos8855 pin-controller 4 (PERICMMC) */
+static const struct samsung_pin_bank_data exynos8855_pin_banks4[] __initconst = {
+	GS101_PIN_BANK_EINTG(7, 0x0, "gpf2", 0x00, 0x00),
+};
+
+/* pin banks of exynos8855 pin-controller 5 (USI) */
+static const struct samsung_pin_bank_data exynos8855_pin_banks5[] __initconst = {
+	GS101_PIN_BANK_EINTG(8, 0x00, "gpp3", 0x00, 0x00),
+	GS101_PIN_BANK_EINTG(2, 0x20, "gpp4", 0x04, 0x08),
+	GS101_PIN_BANK_EINTG(2, 0x40, "gpg2", 0x08, 0x0c),
+	GS101_PIN_BANK_EINTG(1, 0x60, "gpg3", 0x0c, 0x10),
+};
+
+static const struct samsung_pin_ctrl exynos8855_pin_ctrl[] __initconst = {
+	{
+		/* pin-controller instance 0 ALIVE data */
+		.pin_banks	= exynos8855_pin_banks0,
+		.nr_banks	= ARRAY_SIZE(exynos8855_pin_banks0),
+		.eint_wkup_init = exynos_eint_wkup_init,
+		.suspend	= gs101_pinctrl_suspend,
+		.resume		= gs101_pinctrl_resume,
+	}, {
+		/* pin-controller instance 1 CMGP data */
+		.pin_banks	= exynos8855_pin_banks1,
+		.nr_banks	= ARRAY_SIZE(exynos8855_pin_banks1),
+		.eint_gpio_init = exynos_eint_gpio_init,
+		.suspend	= gs101_pinctrl_suspend,
+		.resume		= gs101_pinctrl_resume,
+	}, {
+		/* pin-controller instance 2 HSI UFS data */
+		.pin_banks	= exynos8855_pin_banks2,
+		.nr_banks	= ARRAY_SIZE(exynos8855_pin_banks2),
+		.eint_gpio_init = exynos_eint_gpio_init,
+		.suspend	= gs101_pinctrl_suspend,
+		.resume		= gs101_pinctrl_resume,
+	}, {
+		/* pin-controller instance 3 PERIC data */
+		.pin_banks	= exynos8855_pin_banks3,
+		.nr_banks	= ARRAY_SIZE(exynos8855_pin_banks3),
+		.eint_gpio_init = exynos_eint_gpio_init,
+		.suspend	= gs101_pinctrl_suspend,
+		.resume		= gs101_pinctrl_resume,
+	}, {
+		/* pin-controller instance 4 PERICMMC data */
+		.pin_banks	= exynos8855_pin_banks4,
+		.nr_banks	= ARRAY_SIZE(exynos8855_pin_banks4),
+		.eint_gpio_init = exynos_eint_gpio_init,
+		.suspend	= gs101_pinctrl_suspend,
+		.resume		= gs101_pinctrl_resume,
+	}, {
+		/* pin-controller instance 5 USI data */
+		.pin_banks	= exynos8855_pin_banks5,
+		.nr_banks	= ARRAY_SIZE(exynos8855_pin_banks5),
+		.eint_gpio_init = exynos_eint_gpio_init,
+		.suspend	= gs101_pinctrl_suspend,
+		.resume		= gs101_pinctrl_resume,
+	},
+};
+
+const struct samsung_pinctrl_of_match_data exynos8855_of_data __initconst = {
+	.ctrl		= exynos8855_pin_ctrl,
+	.num_ctrl	= ARRAY_SIZE(exynos8855_pin_ctrl),
+};
+
 /* pin banks of exynos990 pin-controller 0 (ALIVE) */
 static struct samsung_pin_bank_data exynos990_pin_banks0[] = {
 	/* Must start with EINTG banks, ordered by EINT group number. */
diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.c b/drivers/pinctrl/samsung/pinctrl-samsung.c
index 5ac6f6b023272..5ecc9ed4c44db 100644
--- a/drivers/pinctrl/samsung/pinctrl-samsung.c
+++ b/drivers/pinctrl/samsung/pinctrl-samsung.c
@@ -1500,6 +1500,8 @@ static const struct of_device_id samsung_pinctrl_dt_match[] = {
 		.data = &exynos7885_of_data },
 	{ .compatible = "samsung,exynos850-pinctrl",
 		.data = &exynos850_of_data },
+	{ .compatible = "samsung,exynos8855-pinctrl",
+		.data = &exynos8855_of_data },
 	{ .compatible = "samsung,exynos8890-pinctrl",
 		.data = &exynos8890_of_data },
 	{ .compatible = "samsung,exynos8895-pinctrl",
diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.h b/drivers/pinctrl/samsung/pinctrl-samsung.h
index 937600430a6ef..bb02fb49b2afd 100644
--- a/drivers/pinctrl/samsung/pinctrl-samsung.h
+++ b/drivers/pinctrl/samsung/pinctrl-samsung.h
@@ -396,6 +396,7 @@ extern const struct samsung_pinctrl_of_match_data exynos7_of_data;
 extern const struct samsung_pinctrl_of_match_data exynos7870_of_data;
 extern const struct samsung_pinctrl_of_match_data exynos7885_of_data;
 extern const struct samsung_pinctrl_of_match_data exynos850_of_data;
+extern const struct samsung_pinctrl_of_match_data exynos8855_of_data;
 extern const struct samsung_pinctrl_of_match_data exynos8890_of_data;
 extern const struct samsung_pinctrl_of_match_data exynos8895_of_data;
 extern const struct samsung_pinctrl_of_match_data exynos9610_of_data;
diff --git a/drivers/platform/chrome/chromeos_of_hw_prober.c b/drivers/platform/chrome/chromeos_of_hw_prober.c
index f3cd612e55849..8562a0e89dc69 100644
--- a/drivers/platform/chrome/chromeos_of_hw_prober.c
+++ b/drivers/platform/chrome/chromeos_of_hw_prober.c
@@ -100,7 +100,7 @@ static const struct hw_prober_entry hw_prober_platforms[] = {
 	}, {
 		.compatible = "google,spherion",
 		.prober = chromeos_i2c_component_prober,
-		.data = &chromeos_i2c_probe_hana_trackpad,
+		.data = &chromeos_i2c_probe_dumb_trackpad,
 	}, {
 		.compatible = "google,squirtle",
 		.prober = chromeos_i2c_component_prober,
diff --git a/drivers/platform/chrome/cros_ec_chardev.c b/drivers/platform/chrome/cros_ec_chardev.c
index 399ab85b61917..18763947aaa74 100644
--- a/drivers/platform/chrome/cros_ec_chardev.c
+++ b/drivers/platform/chrome/cros_ec_chardev.c
@@ -484,8 +484,8 @@ static void cros_ec_chardev_remove(struct platform_device *pdev)
 }
 
 static const struct platform_device_id cros_ec_chardev_id[] = {
-	{ DRV_NAME, 0 },
-	{}
+	{ .name = DRV_NAME },
+	{ }
 };
 MODULE_DEVICE_TABLE(platform, cros_ec_chardev_id);
 
diff --git a/drivers/platform/chrome/cros_ec_debugfs.c b/drivers/platform/chrome/cros_ec_debugfs.c
index 139cab6fcba17..a1c76e6552f08 100644
--- a/drivers/platform/chrome/cros_ec_debugfs.c
+++ b/drivers/platform/chrome/cros_ec_debugfs.c
@@ -557,8 +557,8 @@ static SIMPLE_DEV_PM_OPS(cros_ec_debugfs_pm_ops,
 			 cros_ec_debugfs_suspend, cros_ec_debugfs_resume);
 
 static const struct platform_device_id cros_ec_debugfs_id[] = {
-	{ DRV_NAME, 0 },
-	{}
+	{ .name = DRV_NAME },
+	{ }
 };
 MODULE_DEVICE_TABLE(platform, cros_ec_debugfs_id);
 
diff --git a/drivers/platform/chrome/cros_ec_lightbar.c b/drivers/platform/chrome/cros_ec_lightbar.c
index ac919c14c631e..b0a31c6ef5fd3 100644
--- a/drivers/platform/chrome/cros_ec_lightbar.c
+++ b/drivers/platform/chrome/cros_ec_lightbar.c
@@ -683,8 +683,8 @@ static SIMPLE_DEV_PM_OPS(cros_ec_lightbar_pm_ops,
 			 cros_ec_lightbar_suspend, cros_ec_lightbar_resume);
 
 static const struct platform_device_id cros_ec_lightbar_id[] = {
-	{ DRV_NAME, 0 },
-	{}
+	{ .name = DRV_NAME },
+	{ }
 };
 MODULE_DEVICE_TABLE(platform, cros_ec_lightbar_id);
 
diff --git a/drivers/platform/chrome/cros_ec_sensorhub.c b/drivers/platform/chrome/cros_ec_sensorhub.c
index f7019fb80a76d..a8395ab0a12a0 100644
--- a/drivers/platform/chrome/cros_ec_sensorhub.c
+++ b/drivers/platform/chrome/cros_ec_sensorhub.c
@@ -267,8 +267,8 @@ static SIMPLE_DEV_PM_OPS(cros_ec_sensorhub_pm_ops,
 		cros_ec_sensorhub_resume);
 
 static const struct platform_device_id cros_ec_sensorhub_id[] = {
-	{ DRV_NAME, 0 },
-	{}
+	{ .name = DRV_NAME },
+	{ }
 };
 MODULE_DEVICE_TABLE(platform, cros_ec_sensorhub_id);
 
diff --git a/drivers/platform/chrome/cros_ec_sensorhub_ring.c b/drivers/platform/chrome/cros_ec_sensorhub_ring.c
index a10579144c34b..d92b602137207 100644
--- a/drivers/platform/chrome/cros_ec_sensorhub_ring.c
+++ b/drivers/platform/chrome/cros_ec_sensorhub_ring.c
@@ -475,6 +475,21 @@ cros_ec_sensor_ring_process_event(struct cros_ec_sensorhub *sensorhub,
 						  fifo_timestamp,
 						  *current_timestamp,
 						  now);
+
+		/*
+		 * A standalone timestamp event typically has a sensor_num of
+		 * 0xff.  Return early here to prevent it from hitting the
+		 * bounds check below and spamming the logs.
+		 */
+		return false;
+	}
+
+	/* Skip event if sensor_num from EC is out of bounds. */
+	if (in->sensor_num >= sensorhub->sensor_num) {
+		dev_warn_ratelimited(sensorhub->dev,
+				     "Invalid sensor number %u from EC\n",
+				     in->sensor_num);
+		return false;
 	}
 
 	if (in->flags & MOTIONSENSE_SENSOR_FLAG_ODR) {
@@ -502,10 +517,6 @@ cros_ec_sensor_ring_process_event(struct cros_ec_sensorhub *sensorhub,
 		return true;
 	}
 
-	if (in->flags & MOTIONSENSE_SENSOR_FLAG_TIMESTAMP)
-		/* If we just have a timestamp, skip this entry. */
-		return false;
-
 	/* Regular sample */
 	out->sensor_id = in->sensor_num;
 	trace_cros_ec_sensorhub_data(in->sensor_num,
@@ -825,8 +836,15 @@ static void cros_ec_sensorhub_ring_handler(struct cros_ec_sensorhub *sensorhub)
 		sensorhub->msg->outsize = 1;
 		sensorhub->msg->insize = fifo_info_length;
 
-		if (cros_ec_cmd_xfer_status(ec->ec_dev, sensorhub->msg) < 0)
+		ret = cros_ec_cmd_xfer_status(ec->ec_dev, sensorhub->msg);
+		if (ret < 0)
 			goto error;
+		if (ret != fifo_info_length) {
+			dev_warn_ratelimited(sensorhub->dev,
+					     "Mismatch read length: size %d - expected %d\n",
+					     ret, fifo_info_length);
+			goto error;
+		}
 
 		memcpy(fifo_info, &sensorhub->resp->fifo_info,
 		       fifo_info_length);
diff --git a/drivers/platform/chrome/cros_ec_sysfs.c b/drivers/platform/chrome/cros_ec_sysfs.c
index b668a3cc118e7..fad593df3b584 100644
--- a/drivers/platform/chrome/cros_ec_sysfs.c
+++ b/drivers/platform/chrome/cros_ec_sysfs.c
@@ -420,8 +420,8 @@ static void cros_ec_sysfs_remove(struct platform_device *pd)
 }
 
 static const struct platform_device_id cros_ec_sysfs_id[] = {
-	{ DRV_NAME, 0 },
-	{}
+	{ .name = DRV_NAME },
+	{ }
 };
 MODULE_DEVICE_TABLE(platform, cros_ec_sysfs_id);
 
diff --git a/drivers/platform/chrome/cros_ec_typec.c b/drivers/platform/chrome/cros_ec_typec.c
index c0806c562bb93..50a68819ceb7b 100644
--- a/drivers/platform/chrome/cros_ec_typec.c
+++ b/drivers/platform/chrome/cros_ec_typec.c
@@ -1119,6 +1119,12 @@ static void cros_typec_register_partner_pdos(struct cros_typec_data *typec,
 	if (!resp->source_cap_count && !resp->sink_cap_count)
 		return;
 
+	if (resp->source_cap_count > PDO_MAX_OBJECTS ||
+	    resp->sink_cap_count > PDO_MAX_OBJECTS) {
+		dev_warn(typec->dev, "Invalid PDO count from EC, port: %d\n", port_num);
+		return;
+	}
+
 	port->partner_pd = typec_partner_usb_power_delivery_register(port->partner, &desc);
 	if (IS_ERR(port->partner_pd)) {
 		dev_warn(typec->dev, "Failed to register partner PD device, port: %d\n", port_num);
diff --git a/drivers/platform/chrome/cros_ec_vbc.c b/drivers/platform/chrome/cros_ec_vbc.c
index 061e84b32b042..c21e1f85f2fb4 100644
--- a/drivers/platform/chrome/cros_ec_vbc.c
+++ b/drivers/platform/chrome/cros_ec_vbc.c
@@ -134,8 +134,8 @@ static void cros_ec_vbc_remove(struct platform_device *pd)
 }
 
 static const struct platform_device_id cros_ec_vbc_id[] = {
-	{ DRV_NAME, 0 },
-	{}
+	{ .name = DRV_NAME },
+	{ }
 };
 MODULE_DEVICE_TABLE(platform, cros_ec_vbc_id);
 
diff --git a/drivers/platform/chrome/cros_kbd_led_backlight.c b/drivers/platform/chrome/cros_kbd_led_backlight.c
index 906eb490e506e..94888db1c00e8 100644
--- a/drivers/platform/chrome/cros_kbd_led_backlight.c
+++ b/drivers/platform/chrome/cros_kbd_led_backlight.c
@@ -220,8 +220,8 @@ MODULE_DEVICE_TABLE(acpi, keyboard_led_acpi_match);
 #endif
 
 static const struct platform_device_id keyboard_led_id[] = {
-	{ "cros-keyboard-leds", 0 },
-	{}
+	{ .name = "cros-keyboard-leds" },
+	{ }
 };
 MODULE_DEVICE_TABLE(platform, keyboard_led_id);
 
diff --git a/drivers/platform/chrome/cros_usbpd_logger.c b/drivers/platform/chrome/cros_usbpd_logger.c
index 060a49f2b9628..0eb714d679c85 100644
--- a/drivers/platform/chrome/cros_usbpd_logger.c
+++ b/drivers/platform/chrome/cros_usbpd_logger.c
@@ -248,8 +248,8 @@ static SIMPLE_DEV_PM_OPS(cros_usbpd_logger_pm_ops, cros_usbpd_logger_suspend,
 			 cros_usbpd_logger_resume);
 
 static const struct platform_device_id cros_usbpd_logger_id[] = {
-	{ DRV_NAME, 0 },
-	{}
+	{ .name = DRV_NAME },
+	{ }
 };
 MODULE_DEVICE_TABLE(platform, cros_usbpd_logger_id);
 
diff --git a/drivers/platform/chrome/cros_usbpd_notify.c b/drivers/platform/chrome/cros_usbpd_notify.c
index 828f00a3191bd..d842c05f4db00 100644
--- a/drivers/platform/chrome/cros_usbpd_notify.c
+++ b/drivers/platform/chrome/cros_usbpd_notify.c
@@ -232,8 +232,8 @@ static void cros_usbpd_notify_remove_plat(struct platform_device *pdev)
 }
 
 static const struct platform_device_id cros_usbpd_notify_id[] = {
-	{ DRV_NAME, 0 },
-	{}
+	{ .name = DRV_NAME },
+	{ }
 };
 MODULE_DEVICE_TABLE(platform, cros_usbpd_notify_id);
 
diff --git a/drivers/platform/chrome/wilco_ec/core.c b/drivers/platform/chrome/wilco_ec/core.c
index fd2a9bc8327c0..9d4521889d40d 100644
--- a/drivers/platform/chrome/wilco_ec/core.c
+++ b/drivers/platform/chrome/wilco_ec/core.c
@@ -151,8 +151,8 @@ static const struct acpi_device_id wilco_ec_acpi_device_ids[] = {
 MODULE_DEVICE_TABLE(acpi, wilco_ec_acpi_device_ids);
 
 static const struct platform_device_id wilco_ec_id[] = {
-	{ DRV_NAME, 0 },
-	{}
+	{ .name = DRV_NAME },
+	{ }
 };
 MODULE_DEVICE_TABLE(platform, wilco_ec_id);
 
diff --git a/drivers/platform/chrome/wilco_ec/debugfs.c b/drivers/platform/chrome/wilco_ec/debugfs.c
index e43ad67b9bcbc..32fb11939b9be 100644
--- a/drivers/platform/chrome/wilco_ec/debugfs.c
+++ b/drivers/platform/chrome/wilco_ec/debugfs.c
@@ -265,8 +265,8 @@ static void wilco_ec_debugfs_remove(struct platform_device *pdev)
 }
 
 static const struct platform_device_id wilco_ec_debugfs_id[] = {
-	{ DRV_NAME, 0 },
-	{}
+	{ .name = DRV_NAME },
+	{ }
 };
 MODULE_DEVICE_TABLE(platform, wilco_ec_debugfs_id);
 
diff --git a/drivers/platform/chrome/wilco_ec/telemetry.c b/drivers/platform/chrome/wilco_ec/telemetry.c
index 05e35d1b16a62..45be318b05f61 100644
--- a/drivers/platform/chrome/wilco_ec/telemetry.c
+++ b/drivers/platform/chrome/wilco_ec/telemetry.c
@@ -409,8 +409,8 @@ static void telem_device_remove(struct platform_device *pdev)
 }
 
 static const struct platform_device_id telem_id[] = {
-	{ DRV_NAME, 0 },
-	{}
+	{ .name = DRV_NAME },
+	{ }
 };
 MODULE_DEVICE_TABLE(platform, telem_id);
 
diff --git a/drivers/platform/loongarch/loongson-laptop.c b/drivers/platform/loongarch/loongson-laptop.c
index 61b18ac206c9e..f2dcc6d78051a 100644
--- a/drivers/platform/loongarch/loongson-laptop.c
+++ b/drivers/platform/loongarch/loongson-laptop.c
@@ -30,7 +30,6 @@
 #define LOONGSON_ACPI_HKEY_HID	"LOON0000"
 
 #define ACPI_LAPTOP_NAME "loongson-laptop"
-#define ACPI_LAPTOP_ACPI_EVENT_PREFIX "loongson"
 
 #define MAX_ACPI_ARGS			3
 #define GENERIC_HOTKEY_MAP_MAX		64
@@ -167,8 +166,6 @@ static int __init setup_acpi_notify(struct generic_sub_driver *sub_driver)
 	}
 
 	sub_driver->device->driver_data = sub_driver;
-	sprintf(acpi_device_class(sub_driver->device), "%s/%s",
-		ACPI_LAPTOP_ACPI_EVENT_PREFIX, sub_driver->name);
 
 	status = acpi_install_notify_handler(*sub_driver->handle,
 			sub_driver->type, dispatch_acpi_notify, sub_driver);
diff --git a/drivers/platform/mellanox/mlxbf-pmc.c b/drivers/platform/mellanox/mlxbf-pmc.c
index 5ec1ad4716967..2ad9e2b0493c4 100644
--- a/drivers/platform/mellanox/mlxbf-pmc.c
+++ b/drivers/platform/mellanox/mlxbf-pmc.c
@@ -2262,13 +2262,19 @@ static int mlxbf_pmc_map_counters(struct device *dev)
 
 static int mlxbf_pmc_probe(struct platform_device *pdev)
 {
-	struct acpi_device *acpi_dev = ACPI_COMPANION(&pdev->dev);
-	const char *hid = acpi_device_hid(acpi_dev);
 	struct device *dev = &pdev->dev;
+	struct acpi_device *acpi_dev;
 	struct arm_smccc_res res;
+	const char *hid;
 	guid_t guid;
 	int ret;
 
+	acpi_dev = ACPI_COMPANION(&pdev->dev);
+	if (!acpi_dev)
+		return -ENODEV;
+
+	hid = acpi_device_hid(acpi_dev);
+
 	/* Ensure we have the UUID we expect for this service. */
 	arm_smccc_smc(MLXBF_PMC_SIP_SVC_UID, 0, 0, 0, 0, 0, 0, 0, &res);
 	guid_parse(mlxbf_pmc_svc_uuid_str, &guid);
diff --git a/drivers/platform/surface/surface_acpi_notify.c b/drivers/platform/surface/surface_acpi_notify.c
index a9dcb0bbe90ee..593a7aba62432 100644
--- a/drivers/platform/surface/surface_acpi_notify.c
+++ b/drivers/platform/surface/surface_acpi_notify.c
@@ -777,12 +777,16 @@ static int san_consumer_links_setup(struct platform_device *pdev)
 
 static int san_probe(struct platform_device *pdev)
 {
-	struct acpi_device *san = ACPI_COMPANION(&pdev->dev);
 	struct ssam_controller *ctrl;
+	struct acpi_device *san;
 	struct san_data *data;
 	acpi_status astatus;
 	int status;
 
+	san = ACPI_COMPANION(&pdev->dev);
+	if (!san)
+		return -ENODEV;
+
 	ctrl = ssam_client_bind(&pdev->dev);
 	if (IS_ERR(ctrl))
 		return PTR_ERR(ctrl) == -ENODEV ? -EPROBE_DEFER : PTR_ERR(ctrl);
diff --git a/drivers/platform/surface/surface_aggregator_hub.c b/drivers/platform/surface/surface_aggregator_hub.c
index 8b8b80228c147..541e9180f9760 100644
--- a/drivers/platform/surface/surface_aggregator_hub.c
+++ b/drivers/platform/surface/surface_aggregator_hub.c
@@ -348,8 +348,13 @@ static const struct ssam_hub_desc kip_hub = {
 /* -- Driver registration. -------------------------------------------------- */
 
 static const struct ssam_device_id ssam_hub_match[] = {
-	{ SSAM_VDEV(HUB, SAM, SSAM_SSH_TC_KIP, 0x00), (unsigned long)&kip_hub  },
-	{ SSAM_VDEV(HUB, SAM, SSAM_SSH_TC_BAS, 0x00), (unsigned long)&base_hub },
+	{
+		SSAM_VDEV(HUB, SAM, SSAM_SSH_TC_KIP, 0x00),
+		.driver_data = (unsigned long)&kip_hub,
+	}, {
+		SSAM_VDEV(HUB, SAM, SSAM_SSH_TC_BAS, 0x00),
+		.driver_data = (unsigned long)&base_hub,
+	},
 	{ }
 };
 MODULE_DEVICE_TABLE(ssam, ssam_hub_match);
diff --git a/drivers/platform/surface/surface_aggregator_tabletsw.c b/drivers/platform/surface/surface_aggregator_tabletsw.c
index ffa36ed928970..13031c329553d 100644
--- a/drivers/platform/surface/surface_aggregator_tabletsw.c
+++ b/drivers/platform/surface/surface_aggregator_tabletsw.c
@@ -622,8 +622,13 @@ static const struct ssam_tablet_sw_desc ssam_pos_sw_desc = {
 /* -- Driver registration. -------------------------------------------------- */
 
 static const struct ssam_device_id ssam_tablet_sw_match[] = {
-	{ SSAM_SDEV(KIP, SAM, 0x00, 0x01), (unsigned long)&ssam_kip_sw_desc },
-	{ SSAM_SDEV(POS, SAM, 0x00, 0x01), (unsigned long)&ssam_pos_sw_desc },
+	{
+		SSAM_SDEV(KIP, SAM, 0x00, 0x01),
+		.driver_data = (unsigned long)&ssam_kip_sw_desc,
+	}, {
+		SSAM_SDEV(POS, SAM, 0x00, 0x01),
+		.driver_data = (unsigned long)&ssam_pos_sw_desc,
+	},
 	{ },
 };
 MODULE_DEVICE_TABLE(ssam, ssam_tablet_sw_match);
diff --git a/drivers/platform/surface/surface_gpe.c b/drivers/platform/surface/surface_gpe.c
index b359413903b13..40896a8544b0a 100644
--- a/drivers/platform/surface/surface_gpe.c
+++ b/drivers/platform/surface/surface_gpe.c
@@ -317,7 +317,7 @@ static int __init surface_gpe_init(void)
 		goto err_alloc;
 	}
 
-	pdev->dev.fwnode = fwnode;
+	platform_device_set_fwnode(pdev, fwnode);
 
 	status = platform_device_add(pdev);
 	if (status)
diff --git a/drivers/platform/surface/surfacepro3_button.c b/drivers/platform/surface/surfacepro3_button.c
index 388a3e1a488cb..b38aa010053e5 100644
--- a/drivers/platform/surface/surfacepro3_button.c
+++ b/drivers/platform/surface/surfacepro3_button.c
@@ -20,7 +20,6 @@
 #define SURFACE_PRO3_BUTTON_HID		"MSHW0028"
 #define SURFACE_PRO4_BUTTON_HID		"MSHW0040"
 #define SURFACE_BUTTON_OBJ_NAME		"VGBI"
-#define SURFACE_BUTTON_DEVICE_NAME	"Surface Pro 3/4 Buttons"
 
 #define MSHW0040_DSM_REVISION		0x01
 #define MSHW0040_DSM_GET_OMPR		0x02	// get OEM Platform Revision
@@ -212,11 +211,10 @@ static int surface_button_probe(struct platform_device *pdev)
 		goto err_free_button;
 	}
 
-	strscpy(acpi_device_name(device), SURFACE_BUTTON_DEVICE_NAME);
 	snprintf(button->phys, sizeof(button->phys), "%s/buttons",
 		 acpi_device_hid(device));
 
-	input->name = acpi_device_name(device);
+	input->name = "Surface Pro 3/4 Buttons";
 	input->phys = button->phys;
 	input->id.bustype = BUS_HOST;
 	input->dev.parent = &pdev->dev;
@@ -239,8 +237,8 @@ static int surface_button_probe(struct platform_device *pdev)
 		goto err_free_button;
 	}
 
-	dev_info(&pdev->dev, "%s [%s]\n", acpi_device_name(device),
-		 acpi_device_bid(device));
+	dev_info(&pdev->dev, "%s [%s]\n", input->name, acpi_device_bid(device));
+
 	return 0;
 
  err_free_input:
diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig
index b54b5212b2047..957034f39e4e7 100644
--- a/drivers/platform/x86/Kconfig
+++ b/drivers/platform/x86/Kconfig
@@ -802,7 +802,6 @@ config LG_LAPTOP
 	tristate "LG Laptop Extras"
 	depends on ACPI
 	depends on ACPI_BATTERY
-	depends on ACPI_WMI
 	depends on INPUT
 	select INPUT_SPARSEKMAP
 	select NEW_LEDS
diff --git a/drivers/platform/x86/amd/Kconfig b/drivers/platform/x86/amd/Kconfig
index b813f92653686..a1a74ef6c8593 100644
--- a/drivers/platform/x86/amd/Kconfig
+++ b/drivers/platform/x86/amd/Kconfig
@@ -34,6 +34,17 @@ config AMD_WBRF
 	  This mechanism will only be activated on platforms that advertise a
 	  need for it.
 
+config AMD_HALO_LED
+	tristate "AMD Halo Box RGB LED Driver"
+	depends on ACPI_WMI && LEDS_CLASS_MULTICOLOR
+	help
+	  This driver provides RGB LED control for AMD Halo Box devices
+	  through the LED multicolor subsystem. The Halo Box light bar can
+	  be controlled via sysfs to display any RGB color combination.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called amd_halo_led.
+
 config AMD_ISP_PLATFORM
 	tristate "AMD ISP4 platform driver"
 	depends on I2C && X86_64 && ACPI
diff --git a/drivers/platform/x86/amd/Makefile b/drivers/platform/x86/amd/Makefile
index f6ff0c837f345..2f467dbbfc8a3 100644
--- a/drivers/platform/x86/amd/Makefile
+++ b/drivers/platform/x86/amd/Makefile
@@ -10,5 +10,6 @@ obj-$(CONFIG_AMD_PMC)		+= pmc/
 obj-$(CONFIG_AMD_HSMP)		+= hsmp/
 obj-$(CONFIG_AMD_PMF)		+= pmf/
 obj-$(CONFIG_AMD_WBRF)		+= wbrf.o
+obj-$(CONFIG_AMD_HALO_LED)	+= amd_halo_led.o
 obj-$(CONFIG_AMD_ISP_PLATFORM)	+= amd_isp4.o
 obj-$(CONFIG_AMD_HFI)		+= hfi/
diff --git a/drivers/platform/x86/amd/amd_halo_led.c b/drivers/platform/x86/amd/amd_halo_led.c
new file mode 100644
index 0000000000000..b21d956c7d96c
--- /dev/null
+++ b/drivers/platform/x86/amd/amd_halo_led.c
@@ -0,0 +1,315 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * AMD Halo Box RGB LED Driver
+ *
+ * Copyright (C) 2026 Advanced Micro Devices, Inc.
+ *
+ * This driver provides RGB LED control for AMD Halo Box devices through
+ * the LED multicolor subsystem. The Halo Box light bar can be controlled
+ * via sysfs to display any RGB color combination.
+ */
+
+#include <linux/array_size.h>
+#include <linux/cleanup.h>
+#include <linux/compiler_attributes.h>
+#include <linux/container_of.h>
+#include <linux/dev_printk.h>
+#include <linux/device.h>
+#include <linux/device/devres.h>
+#include <linux/led-class-multicolor.h>
+#include <linux/leds.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/wmi.h>
+
+#include <linux/byteorder/generic.h>
+
+#define AMD_HALO_GUID "081E747B-E028-4232-AF24-EAAEAB2B1E86"
+
+/* WMI method IDs from MOF */
+enum {
+	AMD_HALO_WMI_TURN_OFF	= 0x04,
+	AMD_HALO_WMI_RGB	= 0x07,
+};
+
+/* Arg0 of the AMD_HALO_WMI_RGB Method */
+enum {
+	AMD_HALO_RGB_CMD_GET	= 0x0,
+	AMD_HALO_RGB_CMD_SET	= 0x1,
+};
+
+/* Status codes from spec */
+#define AMD_HALO_STATUS_SUCCESS		0x0000
+#define AMD_HALO_STATUS_INVALID_PARAM	0xFFFD
+
+/* Brightness uses 0-100 range */
+#define AMD_HALO_MAX_HW_BRIGHTNESS	100
+
+/**
+ * struct amd_halo_led_data - Driver private data
+ * @wdev: WMI device pointer
+ * @led_mc: LED multicolor class device
+ * @subled_info: RGB channel information
+ * @lock: Mutex to protect WMI calls
+ */
+struct amd_halo_led_data {
+	struct wmi_device *wdev;
+	struct led_classdev_mc led_mc;
+	struct mc_subled subled_info[3];
+	struct mutex lock;
+};
+
+struct amd_halo_wmi_args {
+	__le32 arg0;
+	__le32 arg1;
+};
+
+struct amd_halo_wmi_args_rgb {
+	__le32 cmd;
+	__le32 red;
+	__le32 green;
+	__le32 blue;
+};
+
+struct amd_halo_wmi_output_rgb {
+	__le16 status;
+	u8 red;
+	u8 green;
+	u8 blue;
+} __packed;
+
+static inline int wmi_status_to_err(u16 status)
+{
+	switch (status) {
+	case AMD_HALO_STATUS_SUCCESS:
+		return 0;
+	case AMD_HALO_STATUS_INVALID_PARAM:
+		return -EINVAL;
+	default:
+		return -EIO;
+	}
+}
+
+static int __amd_halo_wmi_call(struct wmi_device *wdev,
+			       u32 method_id, void *data, size_t length)
+{
+	struct wmi_buffer input = {
+		.length = length,
+		.data = data,
+	};
+	struct wmi_buffer output = { };
+	int ret;
+
+	/* Return buffer per spec: Bytes[0:1] = Status (little-endian) */
+	ret = wmidev_invoke_method(wdev, 0, method_id,
+				   &input, &output, sizeof(__le16));
+	if (ret)
+		return ret;
+
+	__le16 *result_status __free(kfree) = output.data;
+
+	return wmi_status_to_err(le16_to_cpu(*result_status));
+}
+
+/**
+ * amd_halo_wmi_turn_off - Turn off all LED channels
+ * @wdev: WMI device pointer
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int amd_halo_wmi_turn_off(struct wmi_device *wdev)
+{
+	struct amd_halo_wmi_args args = { };
+
+	return __amd_halo_wmi_call(wdev, AMD_HALO_WMI_TURN_OFF, &args, sizeof(args));
+}
+
+/**
+ * amd_halo_wmi_set_rgb - Set all RGB channels atomically
+ * @wdev: WMI device pointer
+ * @r: brightness for red channel (0 - 100)
+ * @g: brightness for green channel (0 - 100)
+ * @b: brightness for blue channel (0 - 100)
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int amd_halo_wmi_set_rgb(struct wmi_device *wdev, u32 r, u32 g, u32 b)
+{
+	struct amd_halo_wmi_args_rgb args = {
+		.cmd = cpu_to_le32(AMD_HALO_RGB_CMD_SET),
+		.red = cpu_to_le32(r),
+		.green = cpu_to_le32(g),
+		.blue = cpu_to_le32(b),
+	};
+
+	if (r > AMD_HALO_MAX_HW_BRIGHTNESS ||
+	    g > AMD_HALO_MAX_HW_BRIGHTNESS ||
+	    b > AMD_HALO_MAX_HW_BRIGHTNESS) {
+		return -EINVAL;
+	}
+
+	return __amd_halo_wmi_call(wdev, AMD_HALO_WMI_RGB, &args, sizeof(args));
+}
+
+/**
+ * amd_halo_wmi_get_rgb - Get RGB values
+ * @wdev: WMI device pointer
+ * @r: output buffer for red value
+ * @g: output buffer for green value
+ * @b: output buffer for blue value
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int amd_halo_wmi_get_rgb(struct wmi_device *wdev, u8 *r, u8 *g, u8 *b)
+{
+	struct amd_halo_wmi_args_rgb args = {
+		.cmd = cpu_to_le32(AMD_HALO_RGB_CMD_GET),
+	};
+	struct wmi_buffer input = {
+		.length = sizeof(args),
+		.data = &args,
+	};
+	struct wmi_buffer output = { };
+	int ret;
+
+	ret = wmidev_invoke_method(wdev, 0, AMD_HALO_WMI_RGB,
+				   &input, &output, sizeof(struct amd_halo_wmi_output_rgb));
+	if (ret)
+		return ret;
+
+	struct amd_halo_wmi_output_rgb *data __free(kfree) = output.data;
+
+	ret = wmi_status_to_err(le16_to_cpu(data->status));
+	if (ret)
+		return ret;
+
+	if (data->red > AMD_HALO_MAX_HW_BRIGHTNESS ||
+	    data->green > AMD_HALO_MAX_HW_BRIGHTNESS ||
+	    data->blue > AMD_HALO_MAX_HW_BRIGHTNESS) {
+		return -EPROTO;
+	}
+
+	*r = data->red;
+	*g = data->green;
+	*b = data->blue;
+
+	return 0;
+}
+
+/**
+ * amd_halo_brightness_set - Set LED brightness
+ * @cdev: LED class device
+ * @brightness: Brightness value
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int amd_halo_brightness_set(struct led_classdev *cdev,
+				   enum led_brightness brightness)
+{
+	struct led_classdev_mc *mc_cdev = lcdev_to_mccdev(cdev);
+	struct amd_halo_led_data *data = container_of(mc_cdev,
+						      struct amd_halo_led_data,
+						      led_mc);
+	u32 red_hw, green_hw, blue_hw;
+	int ret;
+
+	guard(mutex)(&data->lock);
+
+	led_mc_calc_color_components(mc_cdev, brightness);
+
+	if (brightness == 0)
+		return amd_halo_wmi_turn_off(data->wdev);
+
+	red_hw = mc_cdev->subled_info[0].brightness;
+	green_hw = mc_cdev->subled_info[1].brightness;
+	blue_hw = mc_cdev->subled_info[2].brightness;
+
+	ret = amd_halo_wmi_set_rgb(data->wdev, red_hw, green_hw, blue_hw);
+	if (ret)
+		goto out;
+
+	return 0;
+
+out:
+	/*
+	 * Consider the light bar non-functional if AMD_HALO_WMI_RGB failed.
+	 * Attempt to turn the LED off completely as clean-up.
+	 */
+	if (amd_halo_wmi_turn_off(data->wdev))
+		dev_warn_ratelimited(&data->wdev->dev, "Failed to turn LED off on cleanup\n");
+
+	return ret;
+}
+
+static int amd_halo_probe(struct wmi_device *wdev, const void *context)
+{
+	struct led_init_data led_init_data = {
+		.devicename = "amd_halo",
+		.default_label = "multicolor:" LED_FUNCTION_STATUS,
+		.devname_mandatory = true,
+	};
+	struct amd_halo_led_data *data;
+	u8 r, g, b;
+	int ret;
+
+	data = devm_kzalloc(&wdev->dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	ret = devm_mutex_init(&wdev->dev, &data->lock);
+	if (ret)
+		return ret;
+
+	data->wdev = wdev;
+	dev_set_drvdata(&wdev->dev, data);
+
+	data->subled_info[0].color_index = LED_COLOR_ID_RED;
+	data->subled_info[1].color_index = LED_COLOR_ID_GREEN;
+	data->subled_info[2].color_index = LED_COLOR_ID_BLUE;
+
+	data->led_mc.led_cdev.brightness = AMD_HALO_MAX_HW_BRIGHTNESS;
+	data->led_mc.led_cdev.max_brightness = AMD_HALO_MAX_HW_BRIGHTNESS;
+	data->led_mc.led_cdev.brightness_set_blocking = amd_halo_brightness_set;
+	data->led_mc.led_cdev.flags = LED_CORE_SUSPENDRESUME | LED_RETAIN_AT_SHUTDOWN;
+	data->led_mc.num_colors = ARRAY_SIZE(data->subled_info);
+	data->led_mc.subled_info = data->subled_info;
+
+	ret = amd_halo_wmi_get_rgb(wdev, &r, &g, &b);
+	if (ret)
+		return ret;
+
+	data->subled_info[0].intensity = r;
+	data->subled_info[1].intensity = g;
+	data->subled_info[2].intensity = b;
+
+	ret = devm_led_classdev_multicolor_register_ext(&wdev->dev, &data->led_mc,
+							&led_init_data);
+	if (ret)
+		return dev_err_probe(&wdev->dev, ret,
+				     "Failed to register multicolor LED\n");
+	return 0;
+}
+
+static const struct wmi_device_id amd_halo_id_table[] = {
+	{ .guid_string = AMD_HALO_GUID },
+	{ }
+};
+MODULE_DEVICE_TABLE(wmi, amd_halo_id_table);
+
+static struct wmi_driver amd_halo_driver = {
+	.driver = {
+		.name = "amd_halo_led",
+	},
+	.id_table = amd_halo_id_table,
+	.probe = amd_halo_probe,
+	.no_singleton = true,
+};
+
+module_wmi_driver(amd_halo_driver);
+
+MODULE_AUTHOR("Mario Limonciello (AMD) <superm1@kernel.org>");
+MODULE_AUTHOR("Yo-Jung Leo Lin (AMD) <Leo.Lin@amd.com>");
+MODULE_DESCRIPTION("AMD Halo Box RGB LED Control Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/platform/x86/amd/hsmp/acpi.c b/drivers/platform/x86/amd/hsmp/acpi.c
index 97ed71593bdf3..72f68cef1297f 100644
--- a/drivers/platform/x86/amd/hsmp/acpi.c
+++ b/drivers/platform/x86/amd/hsmp/acpi.c
@@ -21,6 +21,7 @@
 #include <linux/kstrtox.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/string.h>
 #include <linux/sysfs.h>
 #include <linux/topology.h>
 #include <linux/uuid.h>
@@ -77,6 +78,8 @@ static inline int hsmp_get_uid(struct device *dev, u16 *sock_ind)
 	 * bytes to integer.
 	 */
 	uid = acpi_device_uid(ACPI_COMPANION(dev));
+	if (!uid || strlen(uid) < 3)
+		return -EINVAL;
 
 	return kstrtou16(uid + 2, 10, sock_ind);
 }
@@ -104,7 +107,7 @@ static acpi_status hsmp_resource(struct acpi_resource *res, void *data)
 	return AE_OK;
 }
 
-static int hsmp_read_acpi_dsd(struct hsmp_socket *sock)
+static int hsmp_read_acpi_dsd(struct device *dev, struct hsmp_socket *sock)
 {
 	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
 	union acpi_object *guid, *mailbox_package;
@@ -113,10 +116,10 @@ static int hsmp_read_acpi_dsd(struct hsmp_socket *sock)
 	int ret = 0;
 	int j;
 
-	status = acpi_evaluate_object_typed(ACPI_HANDLE(sock->dev), "_DSD", NULL,
+	status = acpi_evaluate_object_typed(ACPI_HANDLE(dev), "_DSD", NULL,
 					    &buf, ACPI_TYPE_PACKAGE);
 	if (ACPI_FAILURE(status)) {
-		dev_err(sock->dev, "Failed to read mailbox reg offsets from DSD table, err: %s\n",
+		dev_err(dev, "Failed to read mailbox reg offsets from DSD table, err: %s\n",
 			acpi_format_exception(status));
 		return -ENODEV;
 	}
@@ -139,7 +142,7 @@ static int hsmp_read_acpi_dsd(struct hsmp_socket *sock)
 	guid = &dsd->package.elements[0];
 	mailbox_package = &dsd->package.elements[1];
 	if (!is_acpi_hsmp_uuid(guid) || mailbox_package->type != ACPI_TYPE_PACKAGE) {
-		dev_err(sock->dev, "Invalid hsmp _DSD table data\n");
+		dev_err(dev, "Invalid hsmp _DSD table data\n");
 		ret = -EINVAL;
 		goto free_buf;
 	}
@@ -148,12 +151,18 @@ static int hsmp_read_acpi_dsd(struct hsmp_socket *sock)
 		union acpi_object *msgobj, *msgstr, *msgint;
 
 		msgobj	= &mailbox_package->package.elements[j];
-		msgstr	= &msgobj->package.elements[0];
-		msgint	= &msgobj->package.elements[1];
 
 		/* package should have 1 string and 1 integer object */
 		if (msgobj->type != ACPI_TYPE_PACKAGE ||
-		    msgstr->type != ACPI_TYPE_STRING ||
+		    msgobj->package.count < 2) {
+			ret = -EINVAL;
+			goto free_buf;
+		}
+
+		msgstr	= &msgobj->package.elements[0];
+		msgint	= &msgobj->package.elements[1];
+
+		if (msgstr->type != ACPI_TYPE_STRING ||
 		    msgint->type != ACPI_TYPE_INTEGER) {
 			ret = -EINVAL;
 			goto free_buf;
@@ -183,14 +192,14 @@ static int hsmp_read_acpi_dsd(struct hsmp_socket *sock)
 	return ret;
 }
 
-static int hsmp_read_acpi_crs(struct hsmp_socket *sock)
+static int hsmp_read_acpi_crs(struct device *dev, struct hsmp_socket *sock)
 {
 	acpi_status status;
 
-	status = acpi_walk_resources(ACPI_HANDLE(sock->dev), METHOD_NAME__CRS,
+	status = acpi_walk_resources(ACPI_HANDLE(dev), METHOD_NAME__CRS,
 				     hsmp_resource, sock);
 	if (ACPI_FAILURE(status)) {
-		dev_err(sock->dev, "Failed to look up MP1 base address from CRS method, err: %s\n",
+		dev_err(dev, "Failed to look up MP1 base address from CRS method, err: %s\n",
 			acpi_format_exception(status));
 		return -EINVAL;
 	}
@@ -198,10 +207,10 @@ static int hsmp_read_acpi_crs(struct hsmp_socket *sock)
 		return -EINVAL;
 
 	/* The mapped region should be un-cached */
-	sock->virt_base_addr = devm_ioremap_uc(sock->dev, sock->mbinfo.base_addr,
+	sock->virt_base_addr = devm_ioremap_uc(dev, sock->mbinfo.base_addr,
 					       sock->mbinfo.size);
 	if (!sock->virt_base_addr) {
-		dev_err(sock->dev, "Failed to ioremap MP1 base address\n");
+		dev_err(dev, "Failed to ioremap MP1 base address\n");
 		return -ENOMEM;
 	}
 
@@ -215,7 +224,6 @@ static int hsmp_parse_acpi_table(struct device *dev, u16 sock_ind)
 	int ret;
 
 	sock->sock_ind		= sock_ind;
-	sock->dev		= dev;
 	sock->amd_hsmp_rdwr	= amd_hsmp_acpi_rdwr;
 
 	sema_init(&sock->hsmp_sem, 1);
@@ -223,12 +231,27 @@ static int hsmp_parse_acpi_table(struct device *dev, u16 sock_ind)
 	dev_set_drvdata(dev, sock);
 
 	/* Read MP1 base address from CRS method */
-	ret = hsmp_read_acpi_crs(sock);
+	ret = hsmp_read_acpi_crs(dev, sock);
 	if (ret)
 		return ret;
 
 	/* Read mailbox offsets from DSD table */
-	return hsmp_read_acpi_dsd(sock);
+	ret = hsmp_read_acpi_dsd(dev, sock);
+	if (ret)
+		return ret;
+
+	/*
+	 * Publish sock->dev last.  hsmp_send_message() uses it (via
+	 * smp_load_acquire()) as the readiness gate for the lock-free data
+	 * plane, so it must become visible only after virt_base_addr, the
+	 * mailbox offsets and the semaphore are fully initialized.  On a
+	 * multi-socket system socket 0 exposes /dev/hsmp before later sockets
+	 * finish probing, so without this an ioctl aimed at a socket still in
+	 * bring-up could pass the gate and dereference a NULL virt_base_addr.
+	 */
+	smp_store_release(&sock->dev, dev);
+
+	return 0;
 }
 
 static ssize_t hsmp_metric_tbl_acpi_read(struct file *filp, struct kobject *kobj,
diff --git a/drivers/platform/x86/amd/hsmp/hsmp.c b/drivers/platform/x86/amd/hsmp/hsmp.c
index 6a26937fc2b58..1a87931136fd4 100644
--- a/drivers/platform/x86/amd/hsmp/hsmp.c
+++ b/drivers/platform/x86/amd/hsmp/hsmp.c
@@ -223,6 +223,19 @@ int hsmp_send_message(struct hsmp_message *msg)
 	sock_ind = array_index_nospec(msg->sock_ind, hsmp_pdev.num_sockets);
 	sock = &hsmp_pdev.sock[sock_ind];
 
+	/*
+	 * A slot exists for every possible socket, but it is only usable once
+	 * that socket has actually been probed.  Reject messages aimed at a
+	 * socket that was never brought up or is still in bring-up, so we never
+	 * operate on a zero-initialized semaphore or an unmapped mailbox.  A
+	 * non-NULL dev also guarantees virt_base_addr, the mailbox offsets and
+	 * the semaphore are visible.
+	 *
+	 * Pairs with smp_store_release(&sock->dev) in hsmp_parse_acpi_table().
+	 */
+	if (!smp_load_acquire(&sock->dev))
+		return -ENODEV;
+
 	ret = down_interruptible(&sock->hsmp_sem);
 	if (ret < 0)
 		return ret;
diff --git a/drivers/platform/x86/amd/pmf/Kconfig b/drivers/platform/x86/amd/pmf/Kconfig
index 25b8f7ae3abde..ad4faf18de47e 100644
--- a/drivers/platform/x86/amd/pmf/Kconfig
+++ b/drivers/platform/x86/amd/pmf/Kconfig
@@ -30,3 +30,13 @@ config AMD_PMF_DEBUG
 	 in the PMF config store.
 
 	 Say Y here to enable more debug logs and Say N here if you are not sure.
+
+config AMD_PMF_UTIL_SUPPORT
+	bool "AMD PMF Util layer support"
+	depends on AMD_PMF
+	help
+	  Enabling this option provides a character device for userspace to capture
+	  PMF features (Smart PC Builder, Auto Mode, Static Power Slider, Dynamic
+	  Power Slider AC/DC) along with PMF metrics from the AMD PMF driver.
+
+	  Say Y here to enable it and Say N here if you are not sure.
diff --git a/drivers/platform/x86/amd/pmf/Makefile b/drivers/platform/x86/amd/pmf/Makefile
index 5978464e0eb7c..bf7aad80b9e9b 100644
--- a/drivers/platform/x86/amd/pmf/Makefile
+++ b/drivers/platform/x86/amd/pmf/Makefile
@@ -8,3 +8,5 @@ obj-$(CONFIG_AMD_PMF)		+= amd-pmf.o
 amd-pmf-y 			:= core.o acpi.o sps.o \
 				   auto-mode.o cnqf.o \
 				   tee-if.o spc.o
+# Build util.c only when AMD_PMF_UTIL_SUPPORT is enabled
+amd-pmf-$(CONFIG_AMD_PMF_UTIL_SUPPORT) += util.o
diff --git a/drivers/platform/x86/amd/pmf/core.c b/drivers/platform/x86/amd/pmf/core.c
index b9e5a2cf3aae8..58d86b4c2828e 100644
--- a/drivers/platform/x86/amd/pmf/core.c
+++ b/drivers/platform/x86/amd/pmf/core.c
@@ -634,6 +634,10 @@ static int amd_pmf_probe(struct platform_device *pdev)
 
 	pmf_device = dev->dev;
 
+	err = amd_pmf_cdev_register(dev);
+	if (err)
+		dev_warn(dev->dev, "failed to register util interface: %d\n", err);
+
 	dev_info(dev->dev, "registered PMF device successfully\n");
 
 	return 0;
@@ -643,6 +647,7 @@ static void amd_pmf_remove(struct platform_device *pdev)
 {
 	struct amd_pmf_dev *dev = platform_get_drvdata(pdev);
 
+	amd_pmf_cdev_unregister();
 	amd_pmf_deinit_features(dev);
 	if (is_apmf_func_supported(dev, APMF_FUNC_SBIOS_HEARTBEAT_V2))
 		amd_pmf_notify_sbios_heartbeat_event_v2(dev, ON_UNLOAD);
diff --git a/drivers/platform/x86/amd/pmf/pmf.h b/drivers/platform/x86/amd/pmf/pmf.h
index 69fef7448744f..752fa5dd22677 100644
--- a/drivers/platform/x86/amd/pmf/pmf.h
+++ b/drivers/platform/x86/amd/pmf/pmf.h
@@ -130,6 +130,7 @@ struct cookie_header {
 #define GET_CMD		true
 
 #define METRICS_TABLE_ID	7
+#define BIOS_OUTPUT_MAX		10
 
 typedef void (*apmf_event_handler_t)(acpi_handle handle, u32 event, void *data);
 
@@ -442,6 +443,7 @@ struct amd_pmf_dev {
 	struct pmf_cbi_ring_buffer cbi_buf;
 	struct mutex cbi_mutex;		     /* Protects ring buffer access */
 	struct mutex metrics_mutex;
+	u32 bios_output[BIOS_OUTPUT_MAX];
 };
 
 struct apmf_sps_prop_granular_v2 {
@@ -680,14 +682,6 @@ enum system_state {
 	SYSTEM_STATE_MAX,
 };
 
-enum ta_slider {
-	TA_BEST_BATTERY,
-	TA_BETTER_BATTERY,
-	TA_BETTER_PERFORMANCE,
-	TA_BEST_PERFORMANCE,
-	TA_MAX,
-};
-
 struct amd_pmf_pb_bitmap {
 	const char *name;
 	u32 bit_mask;
@@ -719,20 +713,6 @@ static const struct amd_pmf_pb_bitmap custom_bios_inputs_v1[] __used = {
 	{"NOTIFY_CUSTOM_BIOS_INPUT10",    BIT(16)},
 };
 
-enum platform_type {
-	PTYPE_UNKNOWN = 0,
-	LID_CLOSE,
-	CLAMSHELL,
-	FLAT,
-	TENT,
-	STAND,
-	TABLET,
-	BOOK,
-	PRESENTATION,
-	PULL_FWD,
-	PTYPE_INVALID = 0xf,
-};
-
 /* Command ids for TA communication */
 enum ta_pmf_command {
 	TA_PMF_COMMAND_POLICY_BUILDER_INITIALIZE,
@@ -923,9 +903,19 @@ int amd_pmf_smartpc_apply_bios_output(struct amd_pmf_dev *dev, u32 val, u32 preq
 void amd_pmf_populate_ta_inputs(struct amd_pmf_dev *dev, struct ta_pmf_enact_table *in);
 void amd_pmf_dump_ta_inputs(struct amd_pmf_dev *dev, struct ta_pmf_enact_table *in);
 int amd_pmf_invoke_cmd_enact(struct amd_pmf_dev *dev);
+u32 amd_pmf_get_ta_custom_bios_inputs(struct ta_pmf_enact_table *in, int index);
 
 int amd_pmf_tee_init(struct amd_pmf_dev *dev, const uuid_t *uuid);
 void amd_pmf_tee_deinit(struct amd_pmf_dev *dev);
 int amd_pmf_start_policy_engine(struct amd_pmf_dev *dev);
 
+/* Util Layer */
+#if IS_ENABLED(CONFIG_AMD_PMF_UTIL_SUPPORT)
+int amd_pmf_cdev_register(struct amd_pmf_dev *dev);
+void amd_pmf_cdev_unregister(void);
+#else
+static inline int amd_pmf_cdev_register(struct amd_pmf_dev *dev) { return 0; }
+static inline void amd_pmf_cdev_unregister(void) {}
+#endif
+
 #endif /* PMF_H */
diff --git a/drivers/platform/x86/amd/pmf/spc.c b/drivers/platform/x86/amd/pmf/spc.c
index f48678a23cc71..94355b435a66e 100644
--- a/drivers/platform/x86/amd/pmf/spc.c
+++ b/drivers/platform/x86/amd/pmf/spc.c
@@ -10,6 +10,7 @@
  */
 
 #include <acpi/button.h>
+#include <linux/amd-pmf.h>
 #include <linux/amd-pmf-io.h>
 #include <linux/cleanup.h>
 #include <linux/power_supply.h>
@@ -17,61 +18,7 @@
 #include "pmf.h"
 
 #ifdef CONFIG_AMD_PMF_DEBUG
-static const char *platform_type_as_str(u16 platform_type)
-{
-	switch (platform_type) {
-	case CLAMSHELL:
-		return "CLAMSHELL";
-	case FLAT:
-		return "FLAT";
-	case TENT:
-		return "TENT";
-	case STAND:
-		return "STAND";
-	case TABLET:
-		return "TABLET";
-	case BOOK:
-		return "BOOK";
-	case PRESENTATION:
-		return "PRESENTATION";
-	case PULL_FWD:
-		return "PULL_FWD";
-	default:
-		return "UNKNOWN";
-	}
-}
-
-static const char *laptop_placement_as_str(u16 device_state)
-{
-	switch (device_state) {
-	case ON_TABLE:
-		return "ON_TABLE";
-	case ON_LAP_MOTION:
-		return "ON_LAP_MOTION";
-	case IN_BAG:
-		return "IN_BAG";
-	case OUT_OF_BAG:
-		return "OUT_OF_BAG";
-	default:
-		return "UNKNOWN";
-	}
-}
-
-static const char *ta_slider_as_str(unsigned int state)
-{
-	switch (state) {
-	case TA_BEST_PERFORMANCE:
-		return "PERFORMANCE";
-	case TA_BETTER_PERFORMANCE:
-		return "BALANCED";
-	case TA_BEST_BATTERY:
-		return "POWER_SAVER";
-	default:
-		return "Unknown TA Slider State";
-	}
-}
-
-static u32 amd_pmf_get_ta_custom_bios_inputs(struct ta_pmf_enact_table *in, int index)
+u32 amd_pmf_get_ta_custom_bios_inputs(struct ta_pmf_enact_table *in, int index)
 {
 	switch (index) {
 	case 0 ... 1:
@@ -82,13 +29,15 @@ static u32 amd_pmf_get_ta_custom_bios_inputs(struct ta_pmf_enact_table *in, int
 		return 0;
 	}
 }
+EXPORT_SYMBOL(amd_pmf_get_ta_custom_bios_inputs);
 
 void amd_pmf_dump_ta_inputs(struct amd_pmf_dev *dev, struct ta_pmf_enact_table *in)
 {
 	int i;
 
 	dev_dbg(dev->dev, "==== TA inputs START ====\n");
-	dev_dbg(dev->dev, "Slider State: %s\n", ta_slider_as_str(in->ev_info.power_slider));
+	dev_dbg(dev->dev, "Slider State: %s\n",
+		amd_pmf_get_slider_position(in->ev_info.power_slider));
 	dev_dbg(dev->dev, "Power Source: %s\n", amd_pmf_source_as_str(in->ev_info.power_source));
 	dev_dbg(dev->dev, "Battery Percentage: %u\n", in->ev_info.bat_percentage);
 	dev_dbg(dev->dev, "Designed Battery Capacity: %u\n", in->ev_info.bat_design);
@@ -102,9 +51,10 @@ void amd_pmf_dump_ta_inputs(struct amd_pmf_dev *dev, struct ta_pmf_enact_table *
 	dev_dbg(dev->dev, "LID State: %s\n", in->ev_info.lid_state ? "close" : "open");
 	dev_dbg(dev->dev, "User Presence: %s\n", in->ev_info.user_present ? "Present" : "Away");
 	dev_dbg(dev->dev, "Ambient Light: %d\n", in->ev_info.ambient_light);
-	dev_dbg(dev->dev, "Platform type: %s\n", platform_type_as_str(in->ev_info.platform_type));
+	dev_dbg(dev->dev, "Platform type: %s\n",
+		amd_pmf_get_platform_type(in->ev_info.platform_type));
 	dev_dbg(dev->dev, "Laptop placement: %s\n",
-		laptop_placement_as_str(in->ev_info.device_state));
+		amd_pmf_get_laptop_placement(in->ev_info.device_state));
 	for (i = 0; i < ARRAY_SIZE(custom_bios_inputs); i++)
 		dev_dbg(dev->dev, "Custom BIOS input%d: %u\n", i + 1,
 			amd_pmf_get_ta_custom_bios_inputs(in, i));
@@ -287,14 +237,14 @@ static int amd_pmf_get_slider_info(struct amd_pmf_dev *dev, struct ta_pmf_enact_
 	switch (dev->current_profile) {
 	case PLATFORM_PROFILE_PERFORMANCE:
 	case PLATFORM_PROFILE_BALANCED_PERFORMANCE:
-		val = TA_BEST_PERFORMANCE;
+		val = AMD_PMF_TA_BEST_PERFORMANCE;
 		break;
 	case PLATFORM_PROFILE_BALANCED:
-		val = TA_BETTER_PERFORMANCE;
+		val = AMD_PMF_TA_BETTER_PERFORMANCE;
 		break;
 	case PLATFORM_PROFILE_LOW_POWER:
 	case PLATFORM_PROFILE_QUIET:
-		val = TA_BEST_BATTERY;
+		val = AMD_PMF_TA_BEST_BATTERY;
 		break;
 	default:
 		dev_err(dev->dev, "Unknown Platform Profile.\n");
diff --git a/drivers/platform/x86/amd/pmf/tee-if.c b/drivers/platform/x86/amd/pmf/tee-if.c
index 7ccd93f506b25..3fe79f0f37c56 100644
--- a/drivers/platform/x86/amd/pmf/tee-if.c
+++ b/drivers/platform/x86/amd/pmf/tee-if.c
@@ -8,7 +8,9 @@
  * Author: Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
  */
 
+#include <linux/array_size.h>
 #include <linux/debugfs.h>
+#include <linux/dev_printk.h>
 #include <linux/tee_drv.h>
 #include <linux/uuid.h>
 #include "pmf.h"
@@ -97,11 +99,18 @@ static int amd_pmf_get_bios_output_idx(u32 action_idx)
 
 static void amd_pmf_update_bios_output(struct amd_pmf_dev *pdev, struct ta_pmf_action *action)
 {
-	u32 bios_idx;
+	int bios_idx;
+	int ret;
 
 	bios_idx = amd_pmf_get_bios_output_idx(action->action_index);
+	if (bios_idx < 0 || bios_idx >= ARRAY_SIZE(pdev->bios_output)) {
+		dev_warn(pdev->dev, "BIOS output index %d out of bounds\n", bios_idx);
+		return;
+	}
 
-	amd_pmf_smartpc_apply_bios_output(pdev, action->value, BIT(bios_idx), bios_idx);
+	ret = amd_pmf_smartpc_apply_bios_output(pdev, action->value, BIT(bios_idx), bios_idx);
+	if (!ret)
+		pdev->bios_output[bios_idx] = action->value;
 }
 
 static void amd_pmf_apply_policies(struct amd_pmf_dev *dev, struct ta_pmf_enact_result *out)
diff --git a/drivers/platform/x86/amd/pmf/util.c b/drivers/platform/x86/amd/pmf/util.c
new file mode 100644
index 0000000000000..4d1a61e4f1b90
--- /dev/null
+++ b/drivers/platform/x86/amd/pmf/util.c
@@ -0,0 +1,153 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * AMD Platform Management Framework Util Layer
+ *
+ * Copyright (c) 2026, Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Authors: Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
+ *	    Sanket Goswami <Sanket.Goswami@amd.com>
+ */
+
+#include <linux/amd-pmf.h>
+#include <linux/minmax.h>
+#include <linux/miscdevice.h>
+#include <linux/mutex.h>
+#include <linux/uaccess.h>
+
+#include "pmf.h"
+
+static struct amd_pmf_dev *pmf_dev_handle;
+static DEFINE_MUTEX(pmf_util_lock);
+
+static int amd_pmf_populate_data(struct amd_pmf_dev *pdev, struct amd_pmf_info *info)
+{
+	struct ta_pmf_shared_memory *ta_sm = NULL;
+	struct ta_pmf_enact_table *in = NULL;
+	int idx;
+
+	if (!pdev || !info)
+		return -EINVAL;
+
+	if (!pdev->shbuf)
+		return -EINVAL;
+
+	ta_sm = pdev->shbuf;
+	in = &ta_sm->pmf_input.enact_table;
+
+	/* Set size */
+	info->size = sizeof(*info);
+
+	/* PMF Feature support flags */
+	if (is_apmf_func_supported(pdev, APMF_FUNC_AUTO_MODE))
+		info->features_supported |= AMD_PMF_FEAT_AUTO_MODE;
+	if (is_apmf_func_supported(pdev, APMF_FUNC_STATIC_SLIDER_GRANULAR))
+		info->features_supported |= AMD_PMF_FEAT_STATIC_POWER_SLIDER;
+	if (pdev->smart_pc_enabled)
+		info->features_supported |= AMD_PMF_FEAT_POLICY_BUILDER;
+	if (is_apmf_func_supported(pdev, APMF_FUNC_DYN_SLIDER_AC))
+		info->features_supported |= AMD_PMF_FEAT_DYNAMIC_POWER_SLIDER_AC;
+	if (is_apmf_func_supported(pdev, APMF_FUNC_DYN_SLIDER_DC))
+		info->features_supported |= AMD_PMF_FEAT_DYNAMIC_POWER_SLIDER_DC;
+
+	/* Device States */
+	info->platform_type = in->ev_info.platform_type;
+	info->laptop_placement = in->ev_info.device_state;
+	info->lid_state = in->ev_info.lid_state;
+	info->user_presence = in->ev_info.user_present;
+	info->slider_position = in->ev_info.power_slider;
+
+	/* Thermal and Power Metrics */
+	info->power_source = in->ev_info.power_source;
+	info->skin_temp = in->ev_info.skin_temperature;
+	info->gfx_busy = in->ev_info.gfx_busy;
+	info->ambient_light = in->ev_info.ambient_light;
+	info->avg_c0_residency = in->ev_info.avg_c0residency;
+	info->max_c0_residency = in->ev_info.max_c0residency;
+	info->socket_power = in->ev_info.socket_power;
+
+	/* Custom BIOS input parameters */
+	for (idx = 0; idx < AMD_PMF_BIOS_PARAMS_MAX; idx++)
+		info->bios_input[idx] = amd_pmf_get_ta_custom_bios_inputs(in, idx);
+
+	/* BIOS output parameters */
+	for (idx = 0; idx < AMD_PMF_BIOS_PARAMS_MAX; idx++)
+		info->bios_output[idx] = pdev->bios_output[idx];
+
+	return 0;
+}
+
+static long amd_pmf_set_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+	struct amd_pmf_dev *pdev = filp->private_data;
+	void __user *argp = (void __user *)arg;
+	struct amd_pmf_info info = {};
+	size_t copy_size;
+	__u64 user_size;
+	int ret;
+
+	if (cmd != IOCTL_AMD_PMF_POPULATE_DATA)
+		return -ENOTTY;
+
+	/* First read just the size field from userspace */
+	if (copy_from_user(&user_size, argp, sizeof(user_size)))
+		return -EFAULT;
+
+	guard(mutex)(&pmf_util_lock);
+	ret = amd_pmf_populate_data(pdev, &info);
+	if (ret)
+		return ret;
+
+	copy_size = min_t(size_t, user_size, sizeof(info));
+
+	/* Set actual size being copied */
+	info.size = copy_size;
+
+	if (copy_to_user(argp, &info, copy_size))
+		return -EFAULT;
+
+	return 0;
+}
+
+static int amd_pmf_open(struct inode *inode, struct file *filp)
+{
+	guard(mutex)(&pmf_util_lock);
+	if (!pmf_dev_handle)
+		return -ENODEV;
+
+	filp->private_data = pmf_dev_handle;
+	return 0;
+}
+
+static const struct file_operations pmf_if_ops = {
+	.owner          = THIS_MODULE,
+	.open           = amd_pmf_open,
+	.unlocked_ioctl = amd_pmf_set_ioctl,
+};
+
+static struct miscdevice amd_pmf_util_if = {
+	.minor		= MISC_DYNAMIC_MINOR,
+	.name		= "amdpmf_interface",
+	.fops		= &pmf_if_ops,
+};
+
+int amd_pmf_cdev_register(struct amd_pmf_dev *dev)
+{
+	int ret;
+
+	guard(mutex)(&pmf_util_lock);
+	pmf_dev_handle = dev;
+	ret = misc_register(&amd_pmf_util_if);
+	if (ret)
+		pmf_dev_handle = NULL;
+
+	return ret;
+}
+
+void amd_pmf_cdev_unregister(void)
+{
+	guard(mutex)(&pmf_util_lock);
+	if (pmf_dev_handle)
+		misc_deregister(&amd_pmf_util_if);
+	pmf_dev_handle = NULL;
+}
diff --git a/drivers/platform/x86/asus-armoury.c b/drivers/platform/x86/asus-armoury.c
index 495dc1e31d40e..93d9665717afa 100644
--- a/drivers/platform/x86/asus-armoury.c
+++ b/drivers/platform/x86/asus-armoury.c
@@ -16,6 +16,7 @@
 #include <linux/acpi.h>
 #include <linux/array_size.h>
 #include <linux/bitfield.h>
+#include <linux/cleanup.h>
 #include <linux/device.h>
 #include <linux/dmi.h>
 #include <linux/err.h>
@@ -93,6 +94,8 @@ struct asus_armoury_priv {
 
 	u32 mini_led_dev_id;
 	u32 gpu_mux_dev_id;
+
+	bool requires_fan_curve;
 };
 
 static struct asus_armoury_priv asus_armoury = {
@@ -216,6 +219,22 @@ static int armoury_set_devstate(struct kobj_attribute *attr,
 	u32 result;
 	int err;
 
+	/* On some models, PPT changes require an active fan curve */
+	if (asus_armoury.requires_fan_curve) {
+		switch (dev_id) {
+		case ASUS_WMI_DEVID_PPT_PL1_SPL:
+		case ASUS_WMI_DEVID_PPT_PL2_SPPT:
+		case ASUS_WMI_DEVID_PPT_PL3_FPPT:
+		case ASUS_WMI_DEVID_PPT_APU_SPPT:
+		case ASUS_WMI_DEVID_PPT_PLAT_SPPT:
+			if (!asus_wmi_custom_fan_curve_is_enabled()) {
+				pr_warn_once("PPT change requires an active fan curve on this model. Enable a custom fan curve first.\n");
+				return -EBUSY;
+			}
+			break;
+		}
+	}
+
 	/*
 	 * Prevent developers from bricking devices or issuing dangerous
 	 * commands that can be difficult or impossible to recover from.
@@ -989,10 +1008,11 @@ static int asus_fw_attr_add(void)
 /* Init / exit ****************************************************************/
 
 /* Set up the min/max and defaults for ROG tunables */
-static void init_rog_tunables(void)
+static int init_rog_tunables(void)
 {
 	const struct power_limits *ac_limits, *dc_limits;
-	struct rog_tunables *ac_rog_tunables = NULL, *dc_rog_tunables = NULL;
+	struct rog_tunables *ac_rog_tunables __free(kfree) = NULL;
+	struct rog_tunables *dc_rog_tunables __free(kfree) = NULL;
 	const struct power_data *power_data;
 	const struct dmi_system_id *dmi_id;
 
@@ -1000,24 +1020,25 @@ static void init_rog_tunables(void)
 	dmi_id = dmi_first_match(power_limits);
 	if (!dmi_id) {
 		pr_warn("No matching power limits found for this system\n");
-		return;
+		return 0;
 	}
 
 	/* Get the power data for this system */
 	power_data = dmi_id->driver_data;
 	if (!power_data) {
 		pr_info("No power data available for this system\n");
-		return;
+		return 0;
 	}
 
+	asus_armoury.requires_fan_curve = power_data->requires_fan_curve;
+
 	/* Initialize AC power tunables */
 	ac_limits = power_data->ac_data;
 	if (ac_limits) {
-		ac_rog_tunables = kzalloc_obj(*asus_armoury.rog_tunables[ASUS_ROG_TUNABLE_AC]);
+		ac_rog_tunables = kzalloc_obj(*ac_rog_tunables);
 		if (!ac_rog_tunables)
-			goto err_nomem;
+			return -ENOMEM;
 
-		asus_armoury.rog_tunables[ASUS_ROG_TUNABLE_AC] = ac_rog_tunables;
 		ac_rog_tunables->power_limits = ac_limits;
 
 		/* Set initial AC values */
@@ -1060,13 +1081,10 @@ static void init_rog_tunables(void)
 	/* Initialize DC power tunables */
 	dc_limits = power_data->dc_data;
 	if (dc_limits) {
-		dc_rog_tunables = kzalloc_obj(*asus_armoury.rog_tunables[ASUS_ROG_TUNABLE_DC]);
-		if (!dc_rog_tunables) {
-			kfree(ac_rog_tunables);
-			goto err_nomem;
-		}
+		dc_rog_tunables = kzalloc_obj(*dc_rog_tunables);
+		if (!dc_rog_tunables)
+			return -ENOMEM;
 
-		asus_armoury.rog_tunables[ASUS_ROG_TUNABLE_DC] = dc_rog_tunables;
 		dc_rog_tunables->power_limits = dc_limits;
 
 		/* Set initial DC values */
@@ -1106,15 +1124,16 @@ static void init_rog_tunables(void)
 		pr_debug("No DC PPT limits defined\n");
 	}
 
-	return;
+	asus_armoury.rog_tunables[ASUS_ROG_TUNABLE_AC] = no_free_ptr(ac_rog_tunables);
+	asus_armoury.rog_tunables[ASUS_ROG_TUNABLE_DC] = no_free_ptr(dc_rog_tunables);
 
-err_nomem:
-	pr_err("Failed to allocate memory for tunables\n");
+	return 0;
 }
 
 static int __init asus_fw_init(void)
 {
 	char *wmi_uid;
+	int err;
 
 	wmi_uid = wmi_get_acpi_device_uid(ASUS_WMI_MGMT_GUID);
 	if (!wmi_uid)
@@ -1127,10 +1146,21 @@ static int __init asus_fw_init(void)
 	if (!strcmp(wmi_uid, ASUS_ACPI_UID_ASUSWMI))
 		return -ENODEV;
 
-	init_rog_tunables();
+	err = init_rog_tunables();
+	if (err)
+		return err;
 
 	/* Must always be last step to ensure data is available */
-	return asus_fw_attr_add();
+	err = asus_fw_attr_add();
+	if (err)
+		goto err_free_tunables;
+
+	return 0;
+
+err_free_tunables:
+	kfree(asus_armoury.rog_tunables[ASUS_ROG_TUNABLE_AC]);
+	kfree(asus_armoury.rog_tunables[ASUS_ROG_TUNABLE_DC]);
+	return err;
 }
 
 static void __exit asus_fw_exit(void)
diff --git a/drivers/platform/x86/asus-armoury.h b/drivers/platform/x86/asus-armoury.h
index 322bfa647c2dc..afcb517dc5bf6 100644
--- a/drivers/platform/x86/asus-armoury.h
+++ b/drivers/platform/x86/asus-armoury.h
@@ -369,6 +369,38 @@ static const struct dmi_system_id power_limits[] = {
 			},
 		},
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_BOARD_NAME, "FA401KM"),
+		},
+		.driver_data = &(struct power_data) {
+			.ac_data = &(struct power_limits) {
+				.nv_dynamic_boost_max = 15,
+				.nv_dynamic_boost_min = 5,
+				.nv_temp_target_max = 87,
+				.nv_temp_target_min = 75,
+				.nv_tgp_max = 95,
+				.nv_tgp_min = 55,
+				.ppt_pl1_spl_max = 80,
+				.ppt_pl1_spl_min = 15,
+				.ppt_pl2_sppt_max = 80,
+				.ppt_pl2_sppt_min = 35,
+				.ppt_pl3_fppt_max = 80,
+				.ppt_pl3_fppt_min = 35,
+			},
+			.dc_data = &(struct power_limits) {
+				.nv_temp_target_max = 87,
+				.nv_temp_target_min = 75,
+				.ppt_pl1_spl_max = 35,
+				.ppt_pl1_spl_min = 25,
+				.ppt_pl2_sppt_max = 44,
+				.ppt_pl2_sppt_min = 31,
+				.ppt_pl3_fppt_max = 65,
+				.ppt_pl3_fppt_min = 45,
+			},
+			.requires_fan_curve = true,
+		},
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_BOARD_NAME, "FA401UM"),
@@ -754,6 +786,40 @@ static const struct dmi_system_id power_limits[] = {
 			},
 		},
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_BOARD_NAME, "FA608WV"),
+		},
+		.driver_data = &(struct power_data) {
+			.ac_data = &(struct power_limits) {
+				.nv_dynamic_boost_max = 25,
+				.nv_dynamic_boost_min = 5,
+				.nv_temp_target_max = 87,
+				.nv_temp_target_min = 75,
+				.nv_tgp_max = 115,
+				.nv_tgp_min = 55,
+				.ppt_pl1_spl_max = 90,
+				.ppt_pl1_spl_min = 15,
+				.ppt_pl2_sppt_max = 90,
+				.ppt_pl2_sppt_min = 35,
+				.ppt_pl3_fppt_max = 90,
+				.ppt_pl3_fppt_min = 35,
+			},
+			.dc_data = &(struct power_limits) {
+				.nv_temp_target_max = 87,
+				.nv_temp_target_min = 75,
+				.ppt_pl1_spl_def = 45,
+				.ppt_pl1_spl_max = 65,
+				.ppt_pl1_spl_min = 15,
+				.ppt_pl2_sppt_def = 54,
+				.ppt_pl2_sppt_max = 65,
+				.ppt_pl2_sppt_min = 35,
+				.ppt_pl3_fppt_max = 65,
+				.ppt_pl3_fppt_min = 35,
+			},
+			.requires_fan_curve = true,
+		},
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_BOARD_NAME, "FA617NS"),
@@ -2079,6 +2145,35 @@ static const struct dmi_system_id power_limits[] = {
 			.requires_fan_curve = true,
 		},
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_BOARD_NAME, "G635LX"),
+		},
+		.driver_data = &(struct power_data) {
+			.ac_data = &(struct power_limits) {
+				.ppt_pl1_spl_min = 28,
+				.ppt_pl1_spl_def = 140,
+				.ppt_pl1_spl_max = 175,
+				.ppt_pl2_sppt_min = 28,
+				.ppt_pl2_sppt_max = 175,
+				.nv_dynamic_boost_min = 5,
+				.nv_dynamic_boost_max = 25,
+				.nv_temp_target_min = 75,
+				.nv_temp_target_max = 87,
+				.nv_tgp_min = 80,
+				.nv_tgp_max = 150,
+			},
+			.dc_data = &(struct power_limits) {
+				.ppt_pl1_spl_min = 25,
+				.ppt_pl1_spl_max = 55,
+				.ppt_pl2_sppt_min = 25,
+				.ppt_pl2_sppt_max = 70,
+				.nv_temp_target_min = 75,
+				.nv_temp_target_max = 87,
+			},
+			.requires_fan_curve = true,
+		},
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_BOARD_NAME, "G713PV"),
@@ -2322,6 +2417,35 @@ static const struct dmi_system_id power_limits[] = {
 			},
 		},
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_BOARD_NAME, "HN7306EA"),
+		},
+		.driver_data = &(struct power_data) {
+			.ac_data = &(struct power_limits) {
+				.ppt_pl1_spl_min = 35,
+				.ppt_pl1_spl_def = 60,
+				.ppt_pl1_spl_max = 85,
+				.ppt_pl2_sppt_min = 40,
+				.ppt_pl2_sppt_def = 70,
+				.ppt_pl2_sppt_max = 95,
+				.ppt_pl3_fppt_min = 50,
+				.ppt_pl3_fppt_def = 85,
+				.ppt_pl3_fppt_max = 115,
+			},
+			.dc_data = &(struct power_limits) {
+				.ppt_pl1_spl_min = 35,
+				.ppt_pl1_spl_def = 45,
+				.ppt_pl1_spl_max = 60,
+				.ppt_pl2_sppt_min = 40,
+				.ppt_pl2_sppt_def = 55,
+				.ppt_pl2_sppt_max = 70,
+				.ppt_pl3_fppt_min = 50,
+				.ppt_pl3_fppt_def = 65,
+				.ppt_pl3_fppt_max = 85,
+			},
+		},
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_BOARD_NAME, "RC71"),
diff --git a/drivers/platform/x86/asus-laptop.c b/drivers/platform/x86/asus-laptop.c
index 140ac8a105372..449addd1ac7ab 100644
--- a/drivers/platform/x86/asus-laptop.c
+++ b/drivers/platform/x86/asus-laptop.c
@@ -42,8 +42,6 @@
 #define ASUS_LAPTOP_VERSION	"0.42"
 
 #define ASUS_LAPTOP_NAME	"Asus Laptop Support"
-#define ASUS_LAPTOP_CLASS	"hotkey"
-#define ASUS_LAPTOP_DEVICE_NAME	"Hotkey"
 #define ASUS_LAPTOP_FILE	KBUILD_MODNAME
 #define ASUS_LAPTOP_PREFIX	"\\_SB.ATKD."
 
@@ -1524,9 +1522,8 @@ static void asus_acpi_notify(acpi_handle handle, u32 event, void *data)
 
 	/* TODO Find a better way to handle events count. */
 	count = asus->event_count[event % 128]++;
-	acpi_bus_generate_netlink_event(asus->device->pnp.device_class,
-					dev_name(&asus->device->dev), event,
-					count);
+	acpi_bus_generate_netlink_event("hotkey", dev_name(&asus->device->dev),
+					event, count);
 
 	if (event >= ATKD_BRNUP_MIN && event <= ATKD_BRNUP_MAX)
 		event = ATKD_BRNUP;
@@ -1840,8 +1837,6 @@ static int asus_acpi_probe(struct platform_device *pdev)
 	if (!asus)
 		return -ENOMEM;
 	asus->handle = device->handle;
-	strscpy(acpi_device_name(device), ASUS_LAPTOP_DEVICE_NAME);
-	strscpy(acpi_device_class(device), ASUS_LAPTOP_CLASS);
 	asus->device = device;
 
 	asus_dmi_check();
diff --git a/drivers/platform/x86/asus-wireless.c b/drivers/platform/x86/asus-wireless.c
index 2b494bf3cba8c..aab45f0442c5b 100644
--- a/drivers/platform/x86/asus-wireless.c
+++ b/drivers/platform/x86/asus-wireless.c
@@ -132,6 +132,10 @@ static int asus_wireless_probe(struct platform_device *pdev)
 	const struct acpi_device_id *id;
 	int err;
 
+	id = acpi_match_acpi_device(device_ids, adev);
+	if (!id)
+		return -ENODEV;
+
 	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
@@ -139,6 +143,7 @@ static int asus_wireless_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, data);
 
 	data->adev = adev;
+	data->hswc_params = (const struct hswc_params *)id->driver_data;
 
 	data->idev = devm_input_allocate_device(&pdev->dev);
 	if (!data->idev)
@@ -153,12 +158,6 @@ static int asus_wireless_probe(struct platform_device *pdev)
 	if (err)
 		return err;
 
-	id = acpi_match_acpi_device(device_ids, adev);
-	if (!id)
-		return 0;
-
-	data->hswc_params = (const struct hswc_params *)id->driver_data;
-
 	data->wq = create_singlethread_workqueue("asus_wireless_workqueue");
 	if (!data->wq)
 		return -ENOMEM;
diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c
index 3c9ef826551d3..a65090429ca70 100644
--- a/drivers/platform/x86/asus-wmi.c
+++ b/drivers/platform/x86/asus-wmi.c
@@ -1618,6 +1618,8 @@ static DEVICE_ATTR_RW(charge_control_end_threshold);
 
 static int asus_wmi_battery_add(struct power_supply *battery, struct acpi_battery_hook *hook)
 {
+	int ret, rv;
+
 	/* The WMI method does not provide a way to specific a battery, so we
 	 * just assume it is the first battery.
 	 * Note: On some newer ASUS laptops (Zenbook UM431DA), the primary/first
@@ -1635,12 +1637,30 @@ static int asus_wmi_battery_add(struct power_supply *battery, struct acpi_batter
 
 	/* The charge threshold is only reset when the system is power cycled,
 	 * and we can't read the current threshold, however the majority of
-	 * platforms retains it, therefore signal the threshold as unknown
-	 * until user explicitly sets it to a new value.
+	 * platforms retains it.
+	 *
+	 * Setting a negative value would signal the threshold as unknown
+	 * until user explicitly sets it to a new value, however to avoid
+	 * regressing userspace, we initialize it to a value of 100.
 	 */
-	charge_end_threshold = -1;
+	charge_end_threshold = 100;
+	ret = asus_wmi_set_devstate(ASUS_WMI_DEVID_RSOC, charge_end_threshold, &rv);
+	if (ret) {
+		pr_err("Failed to reset battery charge threshold\n");
+		goto asus_wmi_battery_add_err;
+	}
+
+	if (rv != 1) {
+		pr_err("Error in battery charge threshold reset\n");
+		ret = -EIO;
+		goto asus_wmi_battery_add_err;
+	}
 
 	return 0;
+asus_wmi_battery_add_err:
+	device_remove_file(&battery->dev,
+			   &dev_attr_charge_control_end_threshold);
+	return ret;
 }
 
 static int asus_wmi_battery_remove(struct power_supply *battery, struct acpi_battery_hook *hook)
@@ -4053,6 +4073,30 @@ static int asus_wmi_custom_fan_curve_init(struct asus_wmi *asus)
 	return 0;
 }
 
+/*
+ * Returns true if at least one custom fan curve is active
+ *
+ * Used by asus-armoury to check if PPT writes will be accepted by the BIOS
+ * on models that require an active fan curve for TDP changes.
+ */
+bool asus_wmi_custom_fan_curve_is_enabled(void)
+{
+	struct fan_curve_data *curves;
+	struct asus_wmi *asus;
+
+	guard(spinlock_irqsave)(&asus_ref.lock);
+	asus = asus_ref.asus;
+	if (!asus)
+		return false;
+
+	curves = asus->custom_fan_curves;
+
+	return (asus->cpu_fan_curve_available && curves[FAN_CURVE_DEV_CPU].enabled) ||
+	       (asus->gpu_fan_curve_available && curves[FAN_CURVE_DEV_GPU].enabled) ||
+	       (asus->mid_fan_curve_available && curves[FAN_CURVE_DEV_MID].enabled);
+}
+EXPORT_SYMBOL_NS_GPL(asus_wmi_custom_fan_curve_is_enabled, "ASUS_WMI");
+
 /* Throttle thermal policy ****************************************************/
 static int throttle_thermal_policy_write(struct asus_wmi *asus)
 {
@@ -5224,20 +5268,20 @@ static int asus_wmi_add(struct platform_device *pdev)
 	return 0;
 
 fail_wmi_handler:
+	asus_screenpad_exit(asus);
+fail_screenpad:
 	asus_wmi_backlight_exit(asus);
 fail_backlight:
 	asus_wmi_rfkill_exit(asus);
-fail_screenpad:
-	asus_screenpad_exit(asus);
 fail_rfkill:
 	asus_wmi_led_exit(asus);
 fail_leds:
+fail_custom_fan_curve:
 fail_hwmon:
 	asus_wmi_input_exit(asus);
 fail_input:
 	asus_wmi_sysfs_exit(asus->platform_device);
 fail_sysfs:
-fail_custom_fan_curve:
 fail_platform_profile_setup:
 fail_fan_boost_mode:
 fail_platform:
diff --git a/drivers/platform/x86/dell/dell-wmi-base.c b/drivers/platform/x86/dell/dell-wmi-base.c
index 997383ba18465..38a6b3ae2f75c 100644
--- a/drivers/platform/x86/dell/dell-wmi-base.c
+++ b/drivers/platform/x86/dell/dell-wmi-base.c
@@ -456,7 +456,7 @@ static int dell_wmi_process_key(struct wmi_device *wdev, int type, int code, __l
 			key++;
 		used = 1;
 	} else if (type == 0x0012 && code == 0x000d && remaining > 0) {
-		value = (le16_to_cpu(buffer[2]) == 2);
+		value = (le16_to_cpu(buffer[0]) == 2);
 		used = 1;
 	}
 
@@ -843,9 +843,22 @@ static int __init dell_wmi_init(void)
 
 	err = dell_privacy_register_driver();
 	if (err)
-		return err;
+		goto out_smbios;
 
-	return wmi_driver_register(&dell_wmi_driver);
+	err = wmi_driver_register(&dell_wmi_driver);
+	if (err)
+		goto out_privacy;
+
+	return 0;
+
+out_privacy:
+	dell_privacy_unregister_driver();
+
+out_smbios:
+	if (wmi_requires_smbios_request)
+		dell_wmi_events_set_enabled(false);
+
+	return err;
 }
 late_initcall(dell_wmi_init);
 
diff --git a/drivers/platform/x86/dell/dell-wmi-ddv.c b/drivers/platform/x86/dell/dell-wmi-ddv.c
index 736d9b1fdcfbb..f8903ced461b1 100644
--- a/drivers/platform/x86/dell/dell-wmi-ddv.c
+++ b/drivers/platform/x86/dell/dell-wmi-ddv.c
@@ -196,40 +196,30 @@ static int dell_wmi_ddv_query_integer(struct wmi_device *wdev, enum dell_ddv_met
 static int dell_wmi_ddv_query_buffer(struct wmi_device *wdev, enum dell_ddv_method method,
 				     u32 arg, struct dell_wmi_buffer **result)
 {
-	struct dell_wmi_buffer *buffer;
 	struct wmi_buffer output;
 	size_t buffer_size;
 	int ret;
 
-	ret = dell_wmi_ddv_query(wdev, method, arg, &output, sizeof(*buffer));
+	ret = dell_wmi_ddv_query(wdev, method, arg, &output, sizeof(struct dell_wmi_buffer));
 	if (ret < 0)
 		return ret;
 
-	buffer = output.data;
-	if (!le32_to_cpu(buffer->raw_size)) {
-		ret = -ENODATA;
+	struct dell_wmi_buffer *buffer __free(kfree) = output.data;
 
-		goto err_free;
-	}
+	if (!le32_to_cpu(buffer->raw_size))
+		return -ENODATA;
 
 	buffer_size = struct_size(buffer, raw_data, le32_to_cpu(buffer->raw_size));
 	if (buffer_size > output.length) {
 		dev_warn(&wdev->dev,
 			 FW_WARN "Dell WMI buffer size (%zu) exceeds WMI buffer size (%zu)\n",
 			 buffer_size, output.length);
-		ret = -EMSGSIZE;
-
-		goto err_free;
+		return -EMSGSIZE;
 	}
 
-	*result = buffer;
+	*result = no_free_ptr(buffer);
 
 	return 0;
-
-err_free:
-	kfree(output.data);
-
-	return ret;
 }
 
 static ssize_t dell_wmi_ddv_query_string(struct wmi_device *wdev, enum dell_ddv_method method,
diff --git a/drivers/platform/x86/dell/dell-wmi-privacy.c b/drivers/platform/x86/dell/dell-wmi-privacy.c
index f9d275b2f900c..366e5b8dc868f 100644
--- a/drivers/platform/x86/dell/dell-wmi-privacy.c
+++ b/drivers/platform/x86/dell/dell-wmi-privacy.c
@@ -92,11 +92,11 @@ bool dell_privacy_has_mic_mute(void)
 {
 	struct privacy_wmi_data *priv;
 
-	mutex_lock(&list_mutex);
+	guard(mutex)(&list_mutex);
+
 	priv = list_first_entry_or_null(&wmi_list,
 			struct privacy_wmi_data,
 			list);
-	mutex_unlock(&list_mutex);
 
 	return priv && (priv->features_present & BIT(DELL_PRIVACY_TYPE_AUDIO));
 }
diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/biosattr-interface.c b/drivers/platform/x86/dell/dell-wmi-sysman/biosattr-interface.c
index db278ff4cc4d4..154e115af4b4c 100644
--- a/drivers/platform/x86/dell/dell-wmi-sysman/biosattr-interface.c
+++ b/drivers/platform/x86/dell/dell-wmi-sysman/biosattr-interface.c
@@ -84,7 +84,6 @@ int set_attribute(const char *a_name, const char *a_value)
 	if (ret < 0)
 		goto out;
 
-	print_hex_dump_bytes("set attribute data: ", DUMP_PREFIX_NONE, buffer, buffer_size);
 	ret = call_biosattributes_interface(wmi_priv.bios_attr_wdev,
 					    buffer, buffer_size,
 					    SETATTRIBUTE_METHOD_ID);
diff --git a/drivers/platform/x86/eeepc-laptop.c b/drivers/platform/x86/eeepc-laptop.c
index d18a80907611e..8a640c25830ad 100644
--- a/drivers/platform/x86/eeepc-laptop.c
+++ b/drivers/platform/x86/eeepc-laptop.c
@@ -34,8 +34,6 @@
 #define EEEPC_LAPTOP_NAME	"Eee PC Hotkey Driver"
 #define EEEPC_LAPTOP_FILE	"eeepc"
 
-#define EEEPC_ACPI_CLASS	"hotkey"
-#define EEEPC_ACPI_DEVICE_NAME	"Hotkey"
 #define EEEPC_ACPI_HID		"ASUS010"
 
 MODULE_AUTHOR("Corentin Chary, Eric Cooper");
@@ -1214,8 +1212,7 @@ static void eeepc_acpi_notify(acpi_handle handle, u32 event, void *data)
 	if (event > ACPI_MAX_SYS_NOTIFY)
 		return;
 	count = eeepc->event_count[event % 128]++;
-	acpi_bus_generate_netlink_event(device->pnp.device_class,
-					dev_name(&device->dev), event,
+	acpi_bus_generate_netlink_event("hotkey", dev_name(&device->dev), event,
 					count);
 
 	/* Brightness events are special */
@@ -1376,8 +1373,6 @@ static int eeepc_acpi_probe(struct platform_device *pdev)
 	if (!eeepc)
 		return -ENOMEM;
 	eeepc->handle = device->handle;
-	strscpy(acpi_device_name(device), EEEPC_ACPI_DEVICE_NAME);
-	strscpy(acpi_device_class(device), EEEPC_ACPI_CLASS);
 	eeepc->device = device;
 
 	platform_set_drvdata(pdev, eeepc);
diff --git a/drivers/platform/x86/fujitsu-laptop.c b/drivers/platform/x86/fujitsu-laptop.c
index 54d0b9cec4d3c..ea543deef68f6 100644
--- a/drivers/platform/x86/fujitsu-laptop.c
+++ b/drivers/platform/x86/fujitsu-laptop.c
@@ -56,7 +56,6 @@
 
 #define FUJITSU_LCD_N_LEVELS		8
 
-#define ACPI_FUJITSU_CLASS		"fujitsu"
 #define ACPI_FUJITSU_BL_HID		"FUJ02B1"
 #define ACPI_FUJITSU_BL_DRIVER_NAME	"Fujitsu laptop FUJ02B1 ACPI brightness driver"
 #define ACPI_FUJITSU_BL_DEVICE_NAME	"Fujitsu FUJ02B1"
@@ -466,7 +465,7 @@ static int acpi_fujitsu_bl_input_setup(struct device *dev)
 	snprintf(priv->phys, sizeof(priv->phys), "%s/video/input0",
 		 acpi_device_hid(device));
 
-	priv->input->name = acpi_device_name(device);
+	priv->input->name = ACPI_FUJITSU_BL_DEVICE_NAME;
 	priv->input->phys = priv->phys;
 	priv->input->id.bustype = BUS_HOST;
 	priv->input->id.product = 0x06;
@@ -546,13 +545,11 @@ static int acpi_fujitsu_bl_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	fujitsu_bl = priv;
-	strscpy(acpi_device_name(device), ACPI_FUJITSU_BL_DEVICE_NAME);
-	strscpy(acpi_device_class(device), ACPI_FUJITSU_CLASS);
 
 	platform_set_drvdata(pdev, priv);
 
-	pr_info("ACPI: %s [%s]\n",
-		acpi_device_name(device), acpi_device_bid(device));
+	pr_info("ACPI: %s [%s]\n", ACPI_FUJITSU_BL_DEVICE_NAME,
+		acpi_device_bid(device));
 
 	if (get_max_brightness(&pdev->dev) <= 0)
 		priv->max_brightness = FUJITSU_LCD_N_LEVELS;
@@ -681,7 +678,7 @@ static int acpi_fujitsu_laptop_input_setup(struct device *dev)
 	snprintf(priv->phys, sizeof(priv->phys), "%s/input0",
 		 acpi_device_hid(device));
 
-	priv->input->name = acpi_device_name(device);
+	priv->input->name = ACPI_FUJITSU_LAPTOP_DEVICE_NAME;
 	priv->input->phys = priv->phys;
 	priv->input->id.bustype = BUS_HOST;
 
@@ -1012,9 +1009,6 @@ static int acpi_fujitsu_laptop_probe(struct platform_device *pdev)
 	WARN_ONCE(fext, "More than one FUJ02E3 ACPI device was found.  Driver may not work as intended.");
 	fext = &pdev->dev;
 
-	strscpy(acpi_device_name(device), ACPI_FUJITSU_LAPTOP_DEVICE_NAME);
-	strscpy(acpi_device_class(device), ACPI_FUJITSU_CLASS);
-
 	platform_set_drvdata(pdev, priv);
 
 	/* kfifo */
@@ -1024,8 +1018,8 @@ static int acpi_fujitsu_laptop_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	pr_info("ACPI: %s [%s]\n",
-		acpi_device_name(device), acpi_device_bid(device));
+	pr_info("ACPI: %s [%s]\n", ACPI_FUJITSU_LAPTOP_DEVICE_NAME,
+		acpi_device_bid(device));
 
 	while (call_fext_func(fext, FUNC_BUTTONS, 0x1, 0x0, 0x0) != 0 &&
 	       i++ < MAX_HOTKEY_RINGBUFFER_SIZE)
diff --git a/drivers/platform/x86/fujitsu-tablet.c b/drivers/platform/x86/fujitsu-tablet.c
index 2f8c1b89cbca1..e1f5dc86dc4d7 100644
--- a/drivers/platform/x86/fujitsu-tablet.c
+++ b/drivers/platform/x86/fujitsu-tablet.c
@@ -22,8 +22,6 @@
 
 #define MODULENAME "fujitsu-tablet"
 
-#define ACPI_FUJITSU_CLASS "fujitsu"
-
 #define INVERT_TABLET_MODE_BIT      0x01
 #define INVERT_DOCK_STATE_BIT       0x02
 #define FORCE_TABLET_MODE_IF_UNDOCK 0x04
@@ -160,6 +158,7 @@ static struct {
 	struct fujitsu_config config;
 	unsigned long prev_keymask;
 
+	char name[17];
 	char phys[21];
 
 	int irq;
@@ -458,14 +457,10 @@ static int acpi_fujitsu_probe(struct platform_device *pdev)
 	if (ACPI_FAILURE(status) || !fujitsu.irq || !fujitsu.io_base)
 		return -ENODEV;
 
-	sprintf(acpi_device_name(adev), "Fujitsu %s", acpi_device_hid(adev));
-	sprintf(acpi_device_class(adev), "%s", ACPI_FUJITSU_CLASS);
-
-	snprintf(fujitsu.phys, sizeof(fujitsu.phys),
-			"%s/input0", acpi_device_hid(adev));
+	scnprintf(fujitsu.name, sizeof(fujitsu.name), "Fujitsu %s", acpi_device_hid(adev));
+	scnprintf(fujitsu.phys, sizeof(fujitsu.phys), "%s/input0", acpi_device_hid(adev));
 
-	error = input_fujitsu_setup(&pdev->dev,
-		acpi_device_name(adev), fujitsu.phys);
+	error = input_fujitsu_setup(&pdev->dev, fujitsu.name, fujitsu.phys);
 	if (error)
 		return error;
 
diff --git a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c
index 27fd6cd215290..78019644ec358 100644
--- a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c
+++ b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c
@@ -661,12 +661,17 @@ static int hp_init_bios_package_attribute(enum hp_wmi_data_type attr_type,
 	int ret = 0;
 
 	/* Take action appropriate to each ACPI TYPE */
-	if (obj->package.count < min_elements) {
-		pr_err("ACPI-package does not have enough elements: %d < %d\n",
-		       obj->package.count, min_elements);
+	if (obj->package.count < COMMON_ELEM_CNT) {
+		pr_err("ACPI-package is missing common elements: %d < %d\n",
+		       obj->package.count, COMMON_ELEM_CNT);
 		goto pack_attr_exit;
 	}
 
+	if (obj->package.count < min_elements) {
+		pr_warn("ACPI-package has fewer elements than expected: %d < %d, parsing available elements\n",
+			obj->package.count, min_elements);
+	}
+
 	elements = obj->package.elements;
 
 	/* sanity checking */
@@ -731,26 +736,31 @@ static int hp_init_bios_package_attribute(enum hp_wmi_data_type attr_type,
 	switch (attr_type) {
 	case HPWMI_STRING_TYPE:
 		ret = hp_populate_string_package_data(elements,
+						      obj->package.count,
 						      instance_id,
 						      attr_name_kobj);
 		break;
 	case HPWMI_INTEGER_TYPE:
 		ret = hp_populate_integer_package_data(elements,
+						       obj->package.count,
 						       instance_id,
 						       attr_name_kobj);
 		break;
 	case HPWMI_ENUMERATION_TYPE:
 		ret = hp_populate_enumeration_package_data(elements,
+							   obj->package.count,
 							   instance_id,
 							   attr_name_kobj);
 		break;
 	case HPWMI_ORDERED_LIST_TYPE:
 		ret = hp_populate_ordered_list_package_data(elements,
+							    obj->package.count,
 							    instance_id,
 							    attr_name_kobj);
 		break;
 	case HPWMI_PASSWORD_TYPE:
 		ret = hp_populate_password_package_data(elements,
+							obj->package.count,
 							instance_id,
 							attr_name_kobj);
 		break;
diff --git a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.h b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.h
index f1eec0e4ba075..ac57d6eab4c35 100644
--- a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.h
+++ b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.h
@@ -279,6 +279,9 @@ enum hp_wmi_data_elements {
 	PSWD_ENCODINGS = 13,
 	PSWD_IS_SET = 14,
 	PSWD_ELEM_CNT = 15,
+
+	/* Minimum elements shared by all attribute types (NAME..SECURITY_LEVEL) */
+	COMMON_ELEM_CNT = SECURITY_LEVEL + 1,
 };
 
 #define GET_INSTANCE_ID(type)						\
@@ -401,6 +404,7 @@ int hp_populate_string_buffer_data(u8 *buffer_ptr, u32 *buffer_size,
 int hp_alloc_string_data(void);
 void hp_exit_string_attributes(void);
 int hp_populate_string_package_data(union acpi_object *str_obj,
+				    int str_obj_count,
 				    int instance_id,
 				    struct kobject *attr_name_kobj);
 
@@ -411,6 +415,7 @@ int hp_populate_integer_buffer_data(u8 *buffer_ptr, u32 *buffer_size,
 int hp_alloc_integer_data(void);
 void hp_exit_integer_attributes(void);
 int hp_populate_integer_package_data(union acpi_object *integer_obj,
+				     int integer_obj_count,
 				     int instance_id,
 				     struct kobject *attr_name_kobj);
 
@@ -421,6 +426,7 @@ int hp_populate_enumeration_buffer_data(u8 *buffer_ptr, u32 *buffer_size,
 int hp_alloc_enumeration_data(void);
 void hp_exit_enumeration_attributes(void);
 int hp_populate_enumeration_package_data(union acpi_object *enum_obj,
+					 int enum_obj_count,
 					 int instance_id,
 					 struct kobject *attr_name_kobj);
 
@@ -432,6 +438,7 @@ int hp_populate_ordered_list_buffer_data(u8 *buffer_ptr,
 int hp_alloc_ordered_list_data(void);
 void hp_exit_ordered_list_attributes(void);
 int hp_populate_ordered_list_package_data(union acpi_object *order_obj,
+					  int order_obj_count,
 					  int instance_id,
 					  struct kobject *attr_name_kobj);
 
@@ -440,6 +447,7 @@ int hp_populate_password_buffer_data(u8 *buffer_ptr, u32 *buffer_size,
 				     int instance_id,
 				     struct kobject *attr_name_kobj);
 int hp_populate_password_package_data(union acpi_object *password_obj,
+				      int password_obj_count,
 				      int instance_id,
 				      struct kobject *attr_name_kobj);
 int hp_alloc_password_data(void);
diff --git a/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c
index af4d1920d4880..21077d17113b9 100644
--- a/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c
+++ b/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c
@@ -163,10 +163,11 @@ static int hp_populate_enumeration_elements_from_package(union acpi_object *enum
 
 		/* Check that both expected and read object type match */
 		if (expected_enum_types[eloc] != enum_obj[elem].type) {
-			pr_err("Error expected type %d for elem %d, but got type %d instead\n",
-			       expected_enum_types[eloc], elem, enum_obj[elem].type);
+			pr_warn("Unexpected element type at elem %d: expected %d, got %d, skipping\n",
+				elem, expected_enum_types[eloc], enum_obj[elem].type);
 			kfree(str_value);
-			return -EIO;
+			str_value = NULL;
+			continue;
 		}
 
 		/* Assign appropriate element value to corresponding field */
@@ -300,10 +301,12 @@ static int hp_populate_enumeration_elements_from_package(union acpi_object *enum
  * Populate all properties of an instance under enumeration attribute
  *
  * @enum_obj: ACPI object with enumeration data
+ * @enum_obj_count: Number of elements in @enum_obj
  * @instance_id: The instance to enumerate
  * @attr_name_kobj: The parent kernel object
  */
 int hp_populate_enumeration_package_data(union acpi_object *enum_obj,
+					 int enum_obj_count,
 					 int instance_id,
 					 struct kobject *attr_name_kobj)
 {
@@ -312,7 +315,7 @@ int hp_populate_enumeration_package_data(union acpi_object *enum_obj,
 	enum_data->attr_name_kobj = attr_name_kobj;
 
 	hp_populate_enumeration_elements_from_package(enum_obj,
-						      enum_obj->package.count,
+						      enum_obj_count,
 						      instance_id);
 	hp_update_attribute_permissions(enum_data->common.is_readonly,
 					&enumeration_current_val);
diff --git a/drivers/platform/x86/hp/hp-bioscfg/int-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/int-attributes.c
index d96e160953e39..f2fd966c9ca4c 100644
--- a/drivers/platform/x86/hp/hp-bioscfg/int-attributes.c
+++ b/drivers/platform/x86/hp/hp-bioscfg/int-attributes.c
@@ -275,10 +275,12 @@ static int hp_populate_integer_elements_from_package(union acpi_object *integer_
  * Populate all properties of an instance under integer attribute
  *
  * @integer_obj: ACPI object with integer data
+ * @integer_obj_count: Number of elements in @integer_obj
  * @instance_id: The instance to enumerate
  * @attr_name_kobj: The parent kernel object
  */
 int hp_populate_integer_package_data(union acpi_object *integer_obj,
+				     int integer_obj_count,
 				     int instance_id,
 				     struct kobject *attr_name_kobj)
 {
@@ -286,7 +288,7 @@ int hp_populate_integer_package_data(union acpi_object *integer_obj,
 
 	integer_data->attr_name_kobj = attr_name_kobj;
 	hp_populate_integer_elements_from_package(integer_obj,
-						  integer_obj->package.count,
+						  integer_obj_count,
 						  instance_id);
 	hp_update_attribute_permissions(integer_data->common.is_readonly,
 					&integer_current_val);
diff --git a/drivers/platform/x86/hp/hp-bioscfg/order-list-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/order-list-attributes.c
index f09489a085c86..863e486474ad0 100644
--- a/drivers/platform/x86/hp/hp-bioscfg/order-list-attributes.c
+++ b/drivers/platform/x86/hp/hp-bioscfg/order-list-attributes.c
@@ -145,7 +145,7 @@ static int hp_populate_ordered_list_elements_from_package(union acpi_object *ord
 	if (!order_obj)
 		return -EINVAL;
 
-	for (elem = 1, eloc = 1; eloc < ORD_ELEM_CNT; elem++, eloc++) {
+	for (elem = 1, eloc = 1; eloc < ORD_ELEM_CNT && elem < order_obj_count; elem++, eloc++) {
 
 		switch (order_obj[elem].type) {
 		case ACPI_TYPE_STRING:
@@ -298,10 +298,12 @@ static int hp_populate_ordered_list_elements_from_package(union acpi_object *ord
  * Populate all properties of an instance under ordered_list attribute
  *
  * @order_obj: ACPI object with ordered_list data
+ * @order_obj_count: Number of elements in @order_obj
  * @instance_id: The instance to enumerate
  * @attr_name_kobj: The parent kernel object
  */
-int hp_populate_ordered_list_package_data(union acpi_object *order_obj, int instance_id,
+int hp_populate_ordered_list_package_data(union acpi_object *order_obj, int order_obj_count,
+					  int instance_id,
 					  struct kobject *attr_name_kobj)
 {
 	struct ordered_list_data *ordered_list_data = &bioscfg_drv.ordered_list_data[instance_id];
@@ -309,7 +311,7 @@ int hp_populate_ordered_list_package_data(union acpi_object *order_obj, int inst
 	ordered_list_data->attr_name_kobj = attr_name_kobj;
 
 	hp_populate_ordered_list_elements_from_package(order_obj,
-						       order_obj->package.count,
+						       order_obj_count,
 						       instance_id);
 	hp_update_attribute_permissions(ordered_list_data->common.is_readonly,
 					&ordered_list_current_val);
diff --git a/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c
index 4d79eb8056a5d..ed5e2080f22bd 100644
--- a/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c
+++ b/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c
@@ -385,10 +385,12 @@ static int hp_populate_password_elements_from_package(union acpi_object *passwor
  *	Populate all properties for an instance under password attribute
  *
  * @password_obj: ACPI object with password data
+ * @password_obj_count: Number of elements in @password_obj
  * @instance_id: The instance to enumerate
  * @attr_name_kobj: The parent kernel object
  */
-int hp_populate_password_package_data(union acpi_object *password_obj, int instance_id,
+int hp_populate_password_package_data(union acpi_object *password_obj, int password_obj_count,
+				      int instance_id,
 				      struct kobject *attr_name_kobj)
 {
 	struct password_data *password_data = &bioscfg_drv.password_data[instance_id];
@@ -396,7 +398,7 @@ int hp_populate_password_package_data(union acpi_object *password_obj, int insta
 	password_data->attr_name_kobj = attr_name_kobj;
 
 	hp_populate_password_elements_from_package(password_obj,
-						   password_obj->package.count,
+						   password_obj_count,
 						   instance_id);
 
 	hp_friendly_user_name_update(password_data->common.path,
diff --git a/drivers/platform/x86/hp/hp-bioscfg/string-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/string-attributes.c
index fe5a9a3a4ef17..f98c32dacbc74 100644
--- a/drivers/platform/x86/hp/hp-bioscfg/string-attributes.c
+++ b/drivers/platform/x86/hp/hp-bioscfg/string-attributes.c
@@ -263,10 +263,12 @@ static int hp_populate_string_elements_from_package(union acpi_object *string_ob
  * Populate all properties of an instance under string attribute
  *
  * @string_obj: ACPI object with string data
+ * @string_obj_count: Number of elements in @string_obj
  * @instance_id: The instance to enumerate
  * @attr_name_kobj: The parent kernel object
  */
 int hp_populate_string_package_data(union acpi_object *string_obj,
+				    int string_obj_count,
 				    int instance_id,
 				    struct kobject *attr_name_kobj)
 {
@@ -275,7 +277,7 @@ int hp_populate_string_package_data(union acpi_object *string_obj,
 	string_data->attr_name_kobj = attr_name_kobj;
 
 	hp_populate_string_elements_from_package(string_obj,
-						 string_obj->package.count,
+						 string_obj_count,
 						 instance_id);
 
 	hp_update_attribute_permissions(string_data->common.is_readonly,
diff --git a/drivers/platform/x86/hp/hp-wmi.c b/drivers/platform/x86/hp/hp-wmi.c
index 8ba286ed87216..5353d997d272a 100644
--- a/drivers/platform/x86/hp/hp-wmi.c
+++ b/drivers/platform/x86/hp/hp-wmi.c
@@ -133,11 +133,52 @@ static const struct thermal_profile_params omen_v1_no_ec_thermal_params = {
 	.ec_tp_offset	= HP_NO_THERMAL_PROFILE_OFFSET,
 };
 
-/*
- * A generic pointer for the currently-active board's thermal profile
- * parameters.
- */
-static struct thermal_profile_params *active_thermal_profile_params;
+struct hp_wmi_fan_profile_params {
+	int (*get_fan_speed)(int fan);
+	bool fan_table;
+};
+
+struct hp_wmi_board_params {
+	const struct thermal_profile_params *thermal_profile;
+	const struct hp_wmi_fan_profile_params *fan_profile;
+};
+
+static int hp_wmi_get_fan_speed_victus_s(int fan);
+
+static const struct hp_wmi_fan_profile_params victus_s_fan_profile_params = {
+	.get_fan_speed	= hp_wmi_get_fan_speed_victus_s,
+	.fan_table	= true,
+};
+
+static const struct hp_wmi_board_params victus_s_board_params = {
+	.thermal_profile	= &victus_s_thermal_params,
+	.fan_profile		= &victus_s_fan_profile_params,
+};
+
+static const struct hp_wmi_board_params omen_v1_board_params = {
+	.thermal_profile	= &omen_v1_thermal_params,
+	.fan_profile		= &victus_s_fan_profile_params,
+};
+
+static const struct hp_wmi_board_params omen_v1_legacy_board_params = {
+	.thermal_profile	= &omen_v1_legacy_thermal_params,
+	.fan_profile		= &victus_s_fan_profile_params,
+};
+
+static const struct hp_wmi_board_params omen_v1_no_ec_board_params = {
+	.thermal_profile	= &omen_v1_no_ec_thermal_params,
+	.fan_profile		= &victus_s_fan_profile_params,
+};
+
+static const struct hp_wmi_board_params *active_board_params;
+
+static const struct thermal_profile_params *hp_wmi_thermal_profile(void)
+{
+	if (!active_board_params)
+		return NULL;
+
+	return active_board_params->thermal_profile;
+}
 
 /* DMI board names of devices that should use the omen specific path for
  * thermal profiles.
@@ -187,87 +228,95 @@ static const char * const victus_thermal_profile_boards[] = {
 	"8A25",
 };
 
-/* DMI Board names of Victus 16-r and Victus 16-s laptops */
-static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst = {
+/* DMI board-specific feature data for Omen and Victus laptops. */
+static const struct dmi_system_id hp_wmi_feature_boards[] __initconst = {
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8902") },
-		.driver_data = (void *)&omen_v1_legacy_thermal_params,
+		.driver_data = (void *)&omen_v1_legacy_board_params,
+	},
+	{
+		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8A3D") },
+		.driver_data = (void *)&victus_s_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8A44") },
-		.driver_data = (void *)&omen_v1_legacy_thermal_params,
+		.driver_data = (void *)&omen_v1_legacy_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8A4D") },
-		.driver_data = (void *)&omen_v1_legacy_thermal_params,
+		.driver_data = (void *)&omen_v1_legacy_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8BAB") },
-		.driver_data = (void *)&omen_v1_thermal_params,
+		.driver_data = (void *)&omen_v1_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8B2F") },
-		.driver_data = (void *)&victus_s_thermal_params,
+		.driver_data = (void *)&victus_s_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8BBE") },
-		.driver_data = (void *)&victus_s_thermal_params,
+		.driver_data = (void *)&victus_s_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8BC2") },
-		.driver_data = (void *)&omen_v1_thermal_params,
+		.driver_data = (void *)&omen_v1_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8BCA") },
-		.driver_data = (void *)&omen_v1_thermal_params,
+		.driver_data = (void *)&omen_v1_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8BCD") },
-		.driver_data = (void *)&omen_v1_thermal_params,
+		.driver_data = (void *)&omen_v1_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8BD4") },
-		.driver_data = (void *)&victus_s_thermal_params,
+		.driver_data = (void *)&victus_s_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8BD5") },
-		.driver_data = (void *)&victus_s_thermal_params,
+		.driver_data = (void *)&victus_s_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8C76") },
-		.driver_data = (void *)&omen_v1_thermal_params,
+		.driver_data = (void *)&omen_v1_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8C77") },
-		.driver_data = (void *)&omen_v1_thermal_params,
+		.driver_data = (void *)&omen_v1_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8C78") },
-		.driver_data = (void *)&omen_v1_thermal_params,
+		.driver_data = (void *)&omen_v1_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8C99") },
-		.driver_data = (void *)&victus_s_thermal_params,
+		.driver_data = (void *)&victus_s_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8C9C") },
-		.driver_data = (void *)&victus_s_thermal_params,
+		.driver_data = (void *)&victus_s_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8D26") },
-		.driver_data = (void *)&omen_v1_legacy_thermal_params,
+		.driver_data = (void *)&omen_v1_legacy_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8D41") },
-		.driver_data = (void *)&omen_v1_no_ec_thermal_params,
+		.driver_data = (void *)&omen_v1_no_ec_board_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8D87") },
+		.driver_data = (void *)&omen_v1_no_ec_board_params,
+	},
+	{
+		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8DD6") },
 		.driver_data = (void *)&omen_v1_no_ec_thermal_params,
 	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8E35") },
-		.driver_data = (void *)&omen_v1_legacy_thermal_params,
+		.driver_data = (void *)&omen_v1_legacy_board_params,
 	},
 	{},
 };
@@ -500,9 +549,9 @@ struct hp_wmi_hwmon_priv {
 	struct mutex lock;	/* protects mode, pwm */
 	u8 min_rpm;
 	u8 max_rpm;
-	int gpu_delta;
 	u8 mode;
-	u8 pwm;
+	u8 cpu_pwm;
+	u8 gpu_pwm;
 	struct delayed_work keep_alive_dwork;
 };
 
@@ -829,24 +878,20 @@ static int hp_wmi_fan_speed_max_set(int enabled)
 	return enabled;
 }
 
-static int hp_wmi_fan_speed_set(struct hp_wmi_hwmon_priv *priv, u8 speed)
+static int hp_wmi_fan_speed_set(struct hp_wmi_hwmon_priv *priv)
 {
 	u8 fan_speed[2];
-	int gpu_speed, ret;
+	int ret;
 
-	fan_speed[CPU_FAN] = speed;
-	fan_speed[GPU_FAN] = speed;
+	if (priv->cpu_pwm == HP_FAN_SPEED_AUTOMATIC)
+		fan_speed[CPU_FAN] = HP_FAN_SPEED_AUTOMATIC;
+	else
+		fan_speed[CPU_FAN] = pwm_to_rpm(priv->cpu_pwm, priv);
 
-	/*
-	 * GPU fan speed is always a little higher than CPU fan speed, we fetch
-	 * this delta value from the fan table during hwmon init.
-	 * Exception: Speed is set to HP_FAN_SPEED_AUTOMATIC, to revert to
-	 * automatic mode.
-	 */
-	if (speed != HP_FAN_SPEED_AUTOMATIC) {
-		gpu_speed = speed + priv->gpu_delta;
-		fan_speed[GPU_FAN] = clamp_val(gpu_speed, 0, U8_MAX);
-	}
+	if (priv->gpu_pwm == HP_FAN_SPEED_AUTOMATIC)
+		fan_speed[GPU_FAN] = HP_FAN_SPEED_AUTOMATIC;
+	else
+		fan_speed[GPU_FAN] = pwm_to_rpm(priv->gpu_pwm, priv);
 
 	ret = hp_wmi_get_fan_count_userdefine_trigger();
 	if (ret < 0)
@@ -863,7 +908,9 @@ static int hp_wmi_fan_speed_set(struct hp_wmi_hwmon_priv *priv, u8 speed)
 
 static int hp_wmi_fan_speed_reset(struct hp_wmi_hwmon_priv *priv)
 {
-	return hp_wmi_fan_speed_set(priv, HP_FAN_SPEED_AUTOMATIC);
+	priv->cpu_pwm = HP_FAN_SPEED_AUTOMATIC;
+	priv->gpu_pwm = HP_FAN_SPEED_AUTOMATIC;
+	return hp_wmi_fan_speed_set(priv);
 }
 
 static int hp_wmi_fan_speed_max_reset(struct hp_wmi_hwmon_priv *priv)
@@ -1789,6 +1836,38 @@ static bool is_victus_s_thermal_profile(void)
 	return is_victus_s_board;
 }
 
+static const struct hp_wmi_fan_profile_params *hp_wmi_fan_profile(void)
+{
+	if (!active_board_params)
+		return NULL;
+
+	return active_board_params->fan_profile;
+}
+
+static bool hp_wmi_fan_control_supported(void)
+{
+	const struct hp_wmi_fan_profile_params *params = hp_wmi_fan_profile();
+
+	return params && params->get_fan_speed;
+}
+
+static bool hp_wmi_fan_table_supported(void)
+{
+	const struct hp_wmi_fan_profile_params *params = hp_wmi_fan_profile();
+
+	return params && params->fan_table;
+}
+
+static int hp_wmi_get_active_fan_speed(int fan)
+{
+	const struct hp_wmi_fan_profile_params *params = hp_wmi_fan_profile();
+
+	if (!params || !params->get_fan_speed)
+		return -EOPNOTSUPP;
+
+	return params->get_fan_speed(fan);
+}
+
 static int victus_s_gpu_thermal_profile_get(bool *ctgp_enable,
 					    bool *ppab_enable,
 					    u8 *dstate,
@@ -1874,7 +1953,10 @@ static int platform_profile_victus_s_get_ec(enum platform_profile_option *profil
 	u8 current_dstate, current_gpu_slowdown_temp, tp;
 	const struct thermal_profile_params *params;
 
-	params = active_thermal_profile_params;
+	params = hp_wmi_thermal_profile();
+	if (!params)
+		return -ENODEV;
+
 	if (params->ec_tp_offset == HP_EC_OFFSET_UNKNOWN ||
 	    params->ec_tp_offset == HP_NO_THERMAL_PROFILE_OFFSET) {
 		*profile = active_platform_profile;
@@ -1886,10 +1968,10 @@ static int platform_profile_victus_s_get_ec(enum platform_profile_option *profil
 		return ret;
 
 	/*
-	 * We cannot use active_thermal_profile_params here, because boards
-	 * like 8C78 have tp == 0x0 || tp == 0x1 after cold boot, but logically
-	 * it should have tp == 0x30 || tp == 0x31, as corrected by the Omen
-	 * Gaming Hub on windows. Hence accept both of these values.
+	 * Boards like 8C78 have tp == 0x0 || tp == 0x1 after cold boot,
+	 * but logically it should have tp == 0x30 || tp == 0x31, as
+	 * corrected by the Omen Gaming Hub on windows. Hence accept both
+	 * of these values.
 	 */
 	if (tp == victus_s_thermal_params.performance ||
 	    tp == omen_v1_thermal_params.performance) {
@@ -1924,12 +2006,12 @@ static int platform_profile_victus_s_get_ec(enum platform_profile_option *profil
 
 static int platform_profile_victus_s_set_ec(enum platform_profile_option profile)
 {
-	struct thermal_profile_params *params;
+	const struct thermal_profile_params *params;
 	bool gpu_ctgp_enable, gpu_ppab_enable;
 	u8 gpu_dstate; /* Test shows 1 = 100%, 2 = 50%, 3 = 25%, 4 = 12.5% */
 	int err, tp;
 
-	params = active_thermal_profile_params;
+	params = hp_wmi_thermal_profile();
 	if (!params)
 		return -ENODEV;
 
@@ -2195,6 +2277,7 @@ static const struct platform_profile_ops hp_wmi_platform_profile_ops = {
 static int thermal_profile_setup(struct platform_device *device)
 {
 	const struct platform_profile_ops *ops;
+	const struct thermal_profile_params *params;
 	int err, tp;
 
 	if (is_omen_thermal_profile()) {
@@ -2226,13 +2309,17 @@ static int thermal_profile_setup(struct platform_device *device)
 
 		ops = &platform_profile_victus_ops;
 	} else if (is_victus_s_thermal_profile()) {
+		params = hp_wmi_thermal_profile();
+		if (!params)
+			return -ENODEV;
+
 		/*
 		 * For an unknown EC layout board, platform_profile_victus_s_get_ec(),
 		 * behaves like a wrapper around active_platform_profile, to avoid using
 		 * uninitialized data, we default to PLATFORM_PROFILE_BALANCED.
 		 */
-		if (active_thermal_profile_params->ec_tp_offset == HP_EC_OFFSET_UNKNOWN ||
-		    active_thermal_profile_params->ec_tp_offset == HP_NO_THERMAL_PROFILE_OFFSET) {
+		if (params->ec_tp_offset == HP_EC_OFFSET_UNKNOWN ||
+		    params->ec_tp_offset == HP_NO_THERMAL_PROFILE_OFFSET) {
 			active_platform_profile = PLATFORM_PROFILE_BALANCED;
 		} else {
 			err = platform_profile_victus_s_get_ec(&active_platform_profile);
@@ -2400,7 +2487,7 @@ static int hp_wmi_apply_fan_settings(struct hp_wmi_hwmon_priv *priv)
 
 	switch (priv->mode) {
 	case PWM_MODE_MAX:
-		if (is_victus_s_thermal_profile()) {
+		if (hp_wmi_fan_control_supported()) {
 			ret = hp_wmi_get_fan_count_userdefine_trigger();
 			if (ret < 0)
 				return ret;
@@ -2412,16 +2499,16 @@ static int hp_wmi_apply_fan_settings(struct hp_wmi_hwmon_priv *priv)
 				 secs_to_jiffies(KEEP_ALIVE_DELAY_SECS));
 		return 0;
 	case PWM_MODE_MANUAL:
-		if (!is_victus_s_thermal_profile())
+		if (!hp_wmi_fan_control_supported())
 			return -EOPNOTSUPP;
-		ret = hp_wmi_fan_speed_set(priv, pwm_to_rpm(priv->pwm, priv));
+		ret = hp_wmi_fan_speed_set(priv);
 		if (ret < 0)
 			return ret;
 		mod_delayed_work(system_dfl_wq, &priv->keep_alive_dwork,
 				 secs_to_jiffies(KEEP_ALIVE_DELAY_SECS));
 		return 0;
 	case PWM_MODE_AUTO:
-		if (is_victus_s_thermal_profile()) {
+		if (hp_wmi_fan_control_supported()) {
 			ret = hp_wmi_get_fan_count_userdefine_trigger();
 			if (ret < 0)
 				return ret;
@@ -2445,12 +2532,12 @@ static umode_t hp_wmi_hwmon_is_visible(const void *data,
 {
 	switch (type) {
 	case hwmon_pwm:
-		if (attr == hwmon_pwm_input && !is_victus_s_thermal_profile())
+		if (attr == hwmon_pwm_input && !hp_wmi_fan_control_supported())
 			return 0;
 		return 0644;
 	case hwmon_fan:
-		if (is_victus_s_thermal_profile()) {
-			if (hp_wmi_get_fan_speed_victus_s(channel) >= 0)
+		if (hp_wmi_fan_control_supported()) {
+			if (hp_wmi_get_active_fan_speed(channel) >= 0)
 				return 0444;
 		} else {
 			if (hp_wmi_get_fan_speed(channel) >= 0)
@@ -2474,8 +2561,8 @@ static int hp_wmi_hwmon_read(struct device *dev, enum hwmon_sensor_types type,
 	priv = dev_get_drvdata(dev);
 	switch (type) {
 	case hwmon_fan:
-		if (is_victus_s_thermal_profile())
-			ret = hp_wmi_get_fan_speed_victus_s(channel);
+		if (hp_wmi_fan_control_supported())
+			ret = hp_wmi_get_active_fan_speed(channel);
 		else
 			ret = hp_wmi_get_fan_speed(channel);
 		if (ret < 0)
@@ -2484,10 +2571,10 @@ static int hp_wmi_hwmon_read(struct device *dev, enum hwmon_sensor_types type,
 		return 0;
 	case hwmon_pwm:
 		if (attr == hwmon_pwm_input) {
-			if (!is_victus_s_thermal_profile())
+			if (!hp_wmi_fan_control_supported())
 				return -EOPNOTSUPP;
 
-			rpm = hp_wmi_get_fan_speed_victus_s(channel);
+			rpm = hp_wmi_get_active_fan_speed(channel);
 			if (rpm < 0)
 				return rpm;
 			*val = rpm_to_pwm(rpm / 100, priv);
@@ -2514,14 +2601,15 @@ static int hp_wmi_hwmon_write(struct device *dev, enum hwmon_sensor_types type,
 			      u32 attr, int channel, long val)
 {
 	struct hp_wmi_hwmon_priv *priv;
-	int rpm;
+	int cpu_rpm, gpu_rpm;
 
 	priv = dev_get_drvdata(dev);
 	guard(mutex)(&priv->lock);
 	switch (type) {
 	case hwmon_pwm:
 		if (attr == hwmon_pwm_input) {
-			if (!is_victus_s_thermal_profile())
+			int rpm;
+			if (!hp_wmi_fan_control_supported())
 				return -EOPNOTSUPP;
 			/* PWM input is invalid when not in manual mode */
 			if (priv->mode != PWM_MODE_MANUAL)
@@ -2530,7 +2618,10 @@ static int hp_wmi_hwmon_write(struct device *dev, enum hwmon_sensor_types type,
 			/* ensure PWM input is within valid fan speeds */
 			rpm = pwm_to_rpm(val, priv);
 			rpm = clamp_val(rpm, priv->min_rpm, priv->max_rpm);
-			priv->pwm = rpm_to_pwm(rpm, priv);
+			if (channel == CPU_FAN)
+				priv->cpu_pwm = rpm_to_pwm(rpm, priv);
+			else if (channel == GPU_FAN)
+				priv->gpu_pwm = rpm_to_pwm(rpm, priv);
 			return hp_wmi_apply_fan_settings(priv);
 		}
 		switch (val) {
@@ -2538,16 +2629,20 @@ static int hp_wmi_hwmon_write(struct device *dev, enum hwmon_sensor_types type,
 			priv->mode = PWM_MODE_MAX;
 			return hp_wmi_apply_fan_settings(priv);
 		case PWM_MODE_MANUAL:
-			if (!is_victus_s_thermal_profile())
+			if (!hp_wmi_fan_control_supported())
 				return -EOPNOTSUPP;
 			/*
 			 * When switching to manual mode, set fan speed to
 			 * current RPM values to ensure a smooth transition.
 			 */
-			rpm = hp_wmi_get_fan_speed_victus_s(channel);
-			if (rpm < 0)
-				return rpm;
-			priv->pwm = rpm_to_pwm(rpm / 100, priv);
+			cpu_rpm = hp_wmi_get_active_fan_speed(CPU_FAN);
+			if (cpu_rpm < 0)
+				return cpu_rpm;
+			gpu_rpm = hp_wmi_get_active_fan_speed(GPU_FAN);
+			if (gpu_rpm < 0)
+				return gpu_rpm;
+			priv->cpu_pwm = rpm_to_pwm(cpu_rpm / 100, priv);
+			priv->gpu_pwm = rpm_to_pwm(gpu_rpm / 100, priv);
 			priv->mode = PWM_MODE_MANUAL;
 			return hp_wmi_apply_fan_settings(priv);
 		case PWM_MODE_AUTO:
@@ -2563,7 +2658,7 @@ static int hp_wmi_hwmon_write(struct device *dev, enum hwmon_sensor_types type,
 
 static const struct hwmon_channel_info * const info[] = {
 	HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT, HWMON_F_INPUT),
-	HWMON_CHANNEL_INFO(pwm, HWMON_PWM_ENABLE | HWMON_PWM_INPUT),
+	HWMON_CHANNEL_INFO(pwm, HWMON_PWM_ENABLE | HWMON_PWM_INPUT, HWMON_PWM_INPUT),
 	NULL
 };
 
@@ -2604,14 +2699,14 @@ static int hp_wmi_setup_fan_settings(struct hp_wmi_hwmon_priv *priv)
 	struct victus_s_fan_table *fan_table;
 	u8 min_rpm, max_rpm;
 	u8 cpu_rpm, gpu_rpm, noise_db;
-	int gpu_delta, i, num_entries, ret;
+	int i, num_entries, ret;
 	size_t header_size, entry_size;
 
 	/* Default behaviour on hwmon init is automatic mode */
 	priv->mode = PWM_MODE_AUTO;
 
-	/* Bypass all non-Victus S devices */
-	if (!is_victus_s_thermal_profile())
+	/* Bypass devices without fan control support. */
+	if (!hp_wmi_fan_table_supported())
 		return 0;
 
 	ret = hp_wmi_perform_query(HPWMI_VICTUS_S_GET_FAN_TABLE_QUERY,
@@ -2650,10 +2745,8 @@ static int hp_wmi_setup_fan_settings(struct hp_wmi_hwmon_priv *priv)
 	if (min_rpm == U8_MAX || max_rpm == 0)
 		return -EINVAL;
 
-	gpu_delta = fan_table->entries[0].gpu_rpm - fan_table->entries[0].cpu_rpm;
 	priv->min_rpm = min_rpm;
 	priv->max_rpm = max_rpm;
-	priv->gpu_delta = gpu_delta;
 
 	return 0;
 }
@@ -2693,24 +2786,25 @@ static int hp_wmi_hwmon_init(void)
 	return 0;
 }
 
-static void __init setup_active_thermal_profile_params(void)
+static void __init setup_active_board_params(void)
 {
 	const struct dmi_system_id *id;
+	const struct thermal_profile_params *params;
 
-	/*
-	 * Currently only victus_s devices use the
-	 * active_thermal_profile_params
-	 */
-	id = dmi_first_match(victus_s_thermal_profile_boards);
+	id = dmi_first_match(hp_wmi_feature_boards);
 	if (id) {
+		active_board_params = id->driver_data;
+		params = hp_wmi_thermal_profile();
+		if (!params)
+			return;
+
 		/*
 		 * Marking this boolean is required to ensure that
 		 * is_victus_s_thermal_profile() behaves like a valid
 		 * wrapper.
 		 */
 		is_victus_s_board = true;
-		active_thermal_profile_params = id->driver_data;
-		if (active_thermal_profile_params->ec_tp_offset == HP_EC_OFFSET_UNKNOWN) {
+		if (params->ec_tp_offset == HP_EC_OFFSET_UNKNOWN) {
 			pr_warn("Unknown EC layout for board %s. Thermal profile readback will be disabled. Please report this to platform-driver-x86@vger.kernel.org\n",
 				dmi_get_system_info(DMI_BOARD_NAME));
 		}
@@ -2745,10 +2839,10 @@ static int __init hp_wmi_init(void)
 		}
 
 		/*
-		 * Setup active board's thermal profile parameters before
-		 * starting platform driver probe.
+		 * Setup active board feature data before starting platform
+		 * driver probe.
 		 */
-		setup_active_thermal_profile_params();
+		setup_active_board_params();
 		err = platform_driver_probe(&hp_wmi_driver, hp_wmi_bios_setup);
 		if (err)
 			goto err_unregister_device;
diff --git a/drivers/platform/x86/huawei-wmi.c b/drivers/platform/x86/huawei-wmi.c
index 93cca17fdf586..d6aaf14d66a5d 100644
--- a/drivers/platform/x86/huawei-wmi.c
+++ b/drivers/platform/x86/huawei-wmi.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/acpi.h>
+#include <linux/cleanup.h>
 #include <linux/debugfs.h>
 #include <linux/delay.h>
 #include <linux/dmi.h>
@@ -527,11 +528,104 @@ static void huawei_wmi_battery_exit(struct device *dev)
 
 /* Fn lock */
 
+/* GFRS byte[1] / SFRS byte[2] (FRSR) fn-lock state values */
+#define FN_LOCK_ACPI_OFF	1
+#define FN_LOCK_ACPI_ON		2
+#define FN_LOCK_ACPI_STAT_OK	0
+
+/*
+ * Newer Huawei models (e.g. HUAWEI FLMH-XX / MateBook 14 2024) use direct
+ * ACPI methods \GFRS / \SFRS (Get/Set Fn key Reversal Status) to control
+ * Fn-lock via EC registers 0x6B (read) and 0x6C (write).
+ *
+ * GFRS response buffer layout:
+ *   byte[0] = STAT (FN_LOCK_ACPI_STAT_OK = success)
+ *   byte[1] = FN_LOCK_ACPI_OFF (fn-lock off) or FN_LOCK_ACPI_ON (fn-lock on)
+ *
+ * SFRS argument layout (CreateByteField(Arg0, 0x02, FRSR)):
+ *   Value is read from byte[2] of the integer argument, so it must be
+ *   passed as (value << 16):
+ *   (FN_LOCK_ACPI_OFF << 16) = fn-lock off (writes 0x55 to EC 0x6C)
+ *   (FN_LOCK_ACPI_ON  << 16) = fn-lock on  (writes 0x5A to EC 0x6C)
+ */
+
+static int huawei_acpi_fn_lock_get(int *on)
+{
+	union acpi_object acpi_arg;
+	struct acpi_object_list arg_list = { .count = 1, .pointer = &acpi_arg };
+	struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
+	acpi_status status;
+
+	acpi_arg.type = ACPI_TYPE_INTEGER;
+	acpi_arg.integer.value = 0;
+
+	status = acpi_evaluate_object(NULL, "\\GFRS", &arg_list, &output);
+	if (ACPI_FAILURE(status))
+		return -EIO;
+
+	union acpi_object *obj __free(kfree) = output.pointer;
+	if (!obj || obj->type != ACPI_TYPE_BUFFER || obj->buffer.length < 2)
+		return -ENODATA;
+
+	/* byte[0] = STAT, byte[1] = fn-lock state */
+	if (obj->buffer.pointer[0] != FN_LOCK_ACPI_STAT_OK)
+		return -EIO;
+
+	switch (obj->buffer.pointer[1]) {
+	case FN_LOCK_ACPI_OFF:
+		if (on)
+			*on = 0;
+		break;
+	case FN_LOCK_ACPI_ON:
+		if (on)
+			*on = 1;
+		break;
+	default:
+		return -ENODATA;
+	}
+
+	return 0;
+}
+
+static int huawei_acpi_fn_lock_set(int on)
+{
+	union acpi_object acpi_arg;
+	struct acpi_object_list arg_list = { .count = 1, .pointer = &acpi_arg };
+	struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
+	acpi_status status;
+
+	/*
+	 * SFRS reads byte[2] of its argument via CreateByteField(Arg0, 0x02).
+	 * on=0 → FRSR=FN_LOCK_ACPI_OFF → EC gets 0x55 (fn-lock off)
+	 * on=1 → FRSR=FN_LOCK_ACPI_ON  → EC gets 0x5A (fn-lock on)
+	 */
+	acpi_arg.type = ACPI_TYPE_INTEGER;
+	acpi_arg.integer.value = (on ? FN_LOCK_ACPI_ON : FN_LOCK_ACPI_OFF) << 16;
+
+	status = acpi_evaluate_object(NULL, "\\SFRS", &arg_list, &output);
+	if (ACPI_FAILURE(status))
+		return -EIO;
+
+	union acpi_object *obj __free(kfree) = output.pointer;
+	if (!obj || obj->type != ACPI_TYPE_BUFFER || obj->buffer.length < 1)
+		return -ENODATA;
+
+	if (obj->buffer.pointer[0] != FN_LOCK_ACPI_STAT_OK)
+		return -EIO;
+
+	return 0;
+}
+
 static int huawei_wmi_fn_lock_get(int *on)
 {
 	u8 ret[0x100] = { 0 };
 	int err, i;
 
+	/* Newer models: use direct ACPI \GFRS method */
+	if (acpi_has_method(NULL, "\\GFRS"))
+		return huawei_acpi_fn_lock_get(on);
+
+	/* WMI fallback */
 	err = huawei_wmi_cmd(FN_LOCK_GET, ret, 0x100);
 	if (err)
 		return err;
@@ -550,6 +644,11 @@ static int huawei_wmi_fn_lock_set(int on)
 {
 	union hwmi_arg arg;
 
+	/* Newer models: use direct ACPI \SFRS method */
+	if (acpi_has_method(NULL, "\\SFRS"))
+		return huawei_acpi_fn_lock_set(on);
+
+	/* WMI fallback */
 	arg.cmd = FN_LOCK_SET;
 	arg.args[2] = on + 1; // 0 undefined, 1 off, 2 on.
 
diff --git a/drivers/platform/x86/intel/int3472/discrete.c b/drivers/platform/x86/intel/int3472/discrete.c
index 115bb37577a18..adff564bf3fd6 100644
--- a/drivers/platform/x86/intel/int3472/discrete.c
+++ b/drivers/platform/x86/intel/int3472/discrete.c
@@ -164,6 +164,24 @@ static const struct int3472_gpio_map int3472_gpio_map[] = {
 		.con_id = "dvdd",
 		.enable_time_us = 45 * USEC_PER_MSEC,
 	},
+	{	/* imx471 expects "vana" as con_id for power enable */
+		.hid = "SONY471A",
+		.type_from = INT3472_GPIO_TYPE_POWER_ENABLE,
+		.type_to = INT3472_GPIO_TYPE_POWER_ENABLE,
+		.con_id = "vana",
+		.enable_time_us = GPIO_REGULATOR_ENABLE_TIME,
+	},
+	{
+		/*
+		 * imx471 (on Lenovo ThinkPads X1 G14) expects "vana" as con_id
+		 * for power enable
+		 */
+		.hid = "TBE20A0",
+		.type_from = INT3472_GPIO_TYPE_POWER_ENABLE,
+		.type_to = INT3472_GPIO_TYPE_POWER_ENABLE,
+		.con_id = "vana",
+		.enable_time_us = GPIO_REGULATOR_ENABLE_TIME,
+	},
 };
 
 static void int3472_get_con_id_and_polarity(struct int3472_discrete_device *int3472, u8 *type,
diff --git a/drivers/platform/x86/intel/ishtp_eclite.c b/drivers/platform/x86/intel/ishtp_eclite.c
index 93ac8b2dbf387..bca7e217878b4 100644
--- a/drivers/platform/x86/intel/ishtp_eclite.c
+++ b/drivers/platform/x86/intel/ishtp_eclite.c
@@ -600,13 +600,16 @@ static int ecl_ishtp_cl_probe(struct ishtp_cl_device *cl_device)
 	rv = acpi_opregion_init(opr_dev);
 	if (rv) {
 		dev_err(cl_data_to_dev(opr_dev), "ACPI opregion init failed\n");
-		goto err_exit;
+		goto err_put;
 	}
 
 	/* Reprobe devices depending on ECLite - battery, fan, etc. */
 	acpi_dev_clear_dependencies(opr_dev->adev);
 
 	return 0;
+
+err_put:
+	acpi_dev_put(opr_dev->adev);
 err_exit:
 	ishtp_set_connection_state(ecl_ishtp_cl, ISHTP_CL_DISCONNECTING);
 	ishtp_cl_disconnect(ecl_ishtp_cl);
diff --git a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c
index 24334ae70d825..5ded0e25b6c75 100644
--- a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c
+++ b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c
@@ -558,7 +558,7 @@ static bool disable_dynamic_sst_features(void)
 {
 	u64 value;
 
-	if (!static_cpu_has(X86_FEATURE_HWP))
+	if (!cpu_feature_enabled(X86_FEATURE_HWP))
 		return true;
 
 	rdmsrq(MSR_PM_ENABLE, value);
diff --git a/drivers/platform/x86/intel/vsec.c b/drivers/platform/x86/intel/vsec.c
index 3ae4557b32b47..5ab2215fdd7fa 100644
--- a/drivers/platform/x86/intel/vsec.c
+++ b/drivers/platform/x86/intel/vsec.c
@@ -103,6 +103,12 @@ static void intel_vsec_remove_aux(void *data)
 	auxiliary_device_uninit(data);
 }
 
+static void intel_vsec_dev_free(struct intel_vsec_device *intel_vsec_dev)
+{
+	kfree(intel_vsec_dev->acpi_disc);
+	kfree(intel_vsec_dev);
+}
+
 static void intel_vsec_dev_release(struct device *dev)
 {
 	struct intel_vsec_device *intel_vsec_dev = dev_to_ivdev(dev);
@@ -111,8 +117,7 @@ static void intel_vsec_dev_release(struct device *dev)
 
 	ida_free(intel_vsec_dev->ida, intel_vsec_dev->auxdev.id);
 
-	kfree(intel_vsec_dev->acpi_disc);
-	kfree(intel_vsec_dev);
+	intel_vsec_dev_free(intel_vsec_dev);
 }
 
 static const struct vsec_feature_dependency *
@@ -218,20 +223,22 @@ int intel_vsec_add_aux(struct device *parent,
 	struct auxiliary_device *auxdev = &intel_vsec_dev->auxdev;
 	int ret, id;
 
-	if (!parent)
+	if (!parent) {
+		intel_vsec_dev_free(intel_vsec_dev);
 		return -EINVAL;
+	}
 
 	ret = xa_alloc(&auxdev_array, &intel_vsec_dev->id, intel_vsec_dev,
 		       PMT_XA_LIMIT, GFP_KERNEL);
 	if (ret < 0) {
-		kfree(intel_vsec_dev);
+		intel_vsec_dev_free(intel_vsec_dev);
 		return ret;
 	}
 
 	id = ida_alloc(intel_vsec_dev->ida, GFP_KERNEL);
 	if (id < 0) {
 		xa_erase(&auxdev_array, intel_vsec_dev->id);
-		kfree(intel_vsec_dev);
+		intel_vsec_dev_free(intel_vsec_dev);
 		return id;
 	}
 
diff --git a/drivers/platform/x86/lenovo/thinkpad_acpi.c b/drivers/platform/x86/lenovo/thinkpad_acpi.c
index 445e1403308ee..6dd7c28fc0dbf 100644
--- a/drivers/platform/x86/lenovo/thinkpad_acpi.c
+++ b/drivers/platform/x86/lenovo/thinkpad_acpi.c
@@ -38,6 +38,7 @@
 #include <linux/backlight.h>
 #include <linux/bitfield.h>
 #include <linux/bitops.h>
+#include <linux/cleanup.h>
 #include <linux/debugfs.h>
 #include <linux/delay.h>
 #include <linux/dmi.h>
@@ -67,6 +68,7 @@
 #include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/string.h>
+#include <linux/string_choices.h>
 #include <linux/string_helpers.h>
 #include <linux/sysfs.h>
 #include <linux/types.h>
@@ -186,6 +188,7 @@ enum tpacpi_hkey_event_t {
 	TP_HKEY_EV_AMT_TOGGLE		= 0x131a, /* Toggle AMT on/off */
 	TP_HKEY_EV_CAMERASHUTTER_TOGGLE = 0x131b, /* Toggle Camera Shutter */
 	TP_HKEY_EV_DOUBLETAP_TOGGLE	= 0x131c, /* Toggle trackpoint doubletap on/off */
+	TP_HKEY_EV_USB_C_SECURITY	= 0x131e, /* USB C Security (Fn+U, Fn+S) */
 	TP_HKEY_EV_PROFILE_TOGGLE	= 0x131f, /* Toggle platform profile in 2024 systems */
 	TP_HKEY_EV_PROFILE_TOGGLE2	= 0x1401, /* Toggle platform profile in 2025 + systems */
 
@@ -309,6 +312,7 @@ struct tp_acpi_drv_struct {
 
 struct ibm_struct {
 	char *name;
+	acpi_device_class device_class;
 
 	int (*read) (struct seq_file *);
 	int (*write) (char *);
@@ -374,6 +378,8 @@ static struct {
 	u32 has_adaptive_kbd:1;
 	u32 kbd_lang:1;
 	u32 trackpoint_doubletap_enable:1;
+	u32 usbc_security_supported:1;
+	bool usbc_security_enabled;
 	struct quirk_entry *quirks;
 } tp_features;
 
@@ -838,9 +844,8 @@ static int __init setup_acpi_notify(struct ibm_struct *ibm)
 	}
 
 	ibm->acpi->device->driver_data = ibm;
-	scnprintf(acpi_device_class(ibm->acpi->device),
-		  sizeof(acpi_device_class(ibm->acpi->device)),
-		  "%s/%s", TPACPI_ACPI_EVENT_PREFIX, ibm->name);
+	scnprintf(ibm->device_class, sizeof(ibm->device_class), "%s/%s",
+		  TPACPI_ACPI_EVENT_PREFIX, ibm->name);
 
 	status = acpi_install_notify_handler(*ibm->acpi->handle,
 			ibm->acpi->type, dispatch_acpi_notify, ibm);
@@ -3869,7 +3874,7 @@ static void hotkey_notify(struct ibm_struct *ibm, u32 event)
 		pr_err("unknown HKEY notification event %d\n", event);
 		/* forward it to userspace, maybe it knows how to handle it */
 		acpi_bus_generate_netlink_event(
-					ibm->acpi->device->pnp.device_class,
+					ibm->device_class,
 					dev_name(&ibm->acpi->device->dev),
 					event, 0);
 		return;
@@ -3949,7 +3954,7 @@ static void hotkey_notify(struct ibm_struct *ibm, u32 event)
 		/* netlink events */
 		if (send_acpi_ev) {
 			acpi_bus_generate_netlink_event(
-					ibm->acpi->device->pnp.device_class,
+					ibm->device_class,
 					dev_name(&ibm->acpi->device->dev),
 					event, hkey);
 		}
@@ -11285,6 +11290,111 @@ static struct ibm_struct hwdd_driver_data = {
 	.name = "hwdd",
 };
 
+/*************************************************************************
+ * USB-C Security subdriver
+ *
+ * HKEY.USCS(0) is a read-only ACPI method; its argument is ignored.
+ * It always returns:
+ *   bit 16 - USB-C security capability present on this SKU or not
+ *   bit  0 - USB-C Security state (enable or disable)
+ *
+ * Hotkey
+ * ------
+ * 0x131e (Fn+U, Fn+S): firmware toggles USBS before firing the event.
+ * The driver reads back the new state and notifies the sysfs attribute.
+ */
+
+/* USCS() return word bit layout */
+#define USCS_CAP_BIT		BIT(16)	/* capability: feature present on SKU */
+#define USCS_STATUS_BIT		BIT(0)	/* current security state */
+
+/* Protects USCS() ACPI method calls in usbc_security_query() */
+static DEFINE_MUTEX(usbc_security_mutex);
+
+/**
+ * usbc_security_query - read current USB-C security state via USCS()
+ * @enabled: out - true when security is ON (data connections blocked)
+ *
+ * Returns:
+ *   0        success, @enabled contains the current state
+ *  -EIO      ACPI evaluation failed
+ *  -ENODEV   capability bit absent; feature not present on this SKU*
+ */
+static int usbc_security_query(bool *enabled)
+{
+	int status;
+
+	guard(mutex)(&usbc_security_mutex);
+	if (!acpi_evalf(hkey_handle, &status, "USCS", "dd", 0))
+		return -EIO;
+
+	if (!(status & USCS_CAP_BIT)) {
+		pr_debug("USCS cap bit absent (raw=0x%x)\n", status);
+		return -ENODEV;
+	}
+
+	*enabled = status & USCS_STATUS_BIT;
+	return 0;
+}
+
+/* sysfs: /sys/devices/platform/thinkpad_acpi/usb_c_security ---------- */
+static ssize_t usb_c_security_show(struct device *dev,
+				   struct device_attribute *attr,
+				   char *buf)
+{
+	return sysfs_emit(buf, "%s\n",
+			  str_enabled_disabled(tp_features.usbc_security_enabled));
+}
+
+static DEVICE_ATTR_RO(usb_c_security);
+
+static struct attribute *usbc_security_attributes[] = {
+	&dev_attr_usb_c_security.attr,
+	NULL,
+};
+
+static umode_t usbc_security_attr_is_visible(struct kobject *kobj,
+					     struct attribute *attr, int n)
+{
+	return tp_features.usbc_security_supported ? attr->mode : 0;
+}
+
+static const struct attribute_group usbc_security_attr_group = {
+	.is_visible = usbc_security_attr_is_visible,
+	.attrs = usbc_security_attributes,
+};
+
+static int tpacpi_usbc_security_init(struct ibm_init_struct *iibm)
+{
+	int err;
+
+	err = usbc_security_query(&tp_features.usbc_security_enabled);
+	if (err == -ENODEV)
+		return 0;
+	if (err)
+		return err;
+
+	tp_features.usbc_security_supported = true;
+	return 0;
+}
+
+/* tpacpi_usbc_security_hotkey - handle Fn+U Fn+S hotkey (0x131e) */
+static bool tpacpi_usbc_security_hotkey(void)
+{
+	if (!tp_features.usbc_security_supported)
+		return false;
+
+	if (usbc_security_query(&tp_features.usbc_security_enabled))
+		return false;
+
+	sysfs_notify(&tpacpi_pdev->dev.kobj, NULL, "usb_c_security");
+	return true;
+}
+
+static struct ibm_struct usbc_security_driver_data = {
+	.name = "usbc_security",
+};
+
 /* --------------------------------------------------------------------- */
 
 static struct attribute *tpacpi_driver_attributes[] = {
@@ -11345,6 +11455,7 @@ static const struct attribute_group *tpacpi_groups[] = {
 	&dprc_attr_group,
 	&auxmac_attr_group,
 	&hwdd_attr_group,
+	&usbc_security_attr_group,
 	NULL,
 };
 
@@ -11499,6 +11610,8 @@ static bool tpacpi_driver_event(const unsigned int hkey_event)
 	case TP_HKEY_EV_PROFILE_TOGGLE2:
 		platform_profile_cycle();
 		return true;
+	case TP_HKEY_EV_USB_C_SECURITY:
+		return tpacpi_usbc_security_hotkey();
 	}
 
 	return false;
@@ -11964,6 +12077,10 @@ static struct ibm_init_struct ibms_init[] __initdata = {
 		.init = tpacpi_hwdd_init,
 		.data = &hwdd_driver_data,
 	},
+	{
+		.init = tpacpi_usbc_security_init,
+		.data = &usbc_security_driver_data,
+	},
 };
 
 static int __init set_ibm_param(const char *val, const struct kernel_param *kp)
diff --git a/drivers/platform/x86/lenovo/ymc.c b/drivers/platform/x86/lenovo/ymc.c
index 1b73a55f1b89c..015e046b0fcea 100644
--- a/drivers/platform/x86/lenovo/ymc.c
+++ b/drivers/platform/x86/lenovo/ymc.c
@@ -8,6 +8,8 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/acpi.h>
+#include <linux/bitfield.h>
+#include <linux/bits.h>
 #include <linux/dmi.h>
 #include <linux/input.h>
 #include <linux/input/sparse-keymap.h>
@@ -20,6 +22,8 @@
 #define LENOVO_YMC_QUERY_INSTANCE 0
 #define LENOVO_YMC_QUERY_METHOD 0x01
 
+#define LENOVO_YMC_STATE_MASK GENMASK(7, 0)
+
 static bool force;
 module_param(force, bool, 0444);
 MODULE_PARM_DESC(force, "Force loading on boards without a convertible DMI chassis-type");
@@ -85,7 +89,9 @@ static void lenovo_ymc_notify(struct wmi_device *wdev, union acpi_object *data)
 			"WMI event data is not an integer\n");
 		goto free_obj;
 	}
-	code = obj->integer.value;
+
+	/* strip upper bits (e.g. 0x50000) on newer devices */
+	code = FIELD_GET(LENOVO_YMC_STATE_MASK, obj->integer.value);
 
 	if (!sparse_keymap_report_event(priv->input_dev, code, 1, true))
 		dev_warn(&wdev->dev, "Unknown key %d pressed\n", code);
diff --git a/drivers/platform/x86/lg-laptop.c b/drivers/platform/x86/lg-laptop.c
index a8f2f465ef3f9..36ad1e2694433 100644
--- a/drivers/platform/x86/lg-laptop.c
+++ b/drivers/platform/x86/lg-laptop.c
@@ -8,8 +8,11 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/acpi.h>
+#include <linux/array_size.h>
 #include <linux/bitfield.h>
 #include <linux/bits.h>
+#include <linux/compiler_attributes.h>
+#include <linux/cleanup.h>
 #include <linux/device.h>
 #include <linux/dev_printk.h>
 #include <linux/dmi.h>
@@ -17,10 +20,12 @@
 #include <linux/input/sparse-keymap.h>
 #include <linux/kernel.h>
 #include <linux/leds.h>
+#include <linux/limits.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/string_choices.h>
 #include <linux/types.h>
+#include <linux/unaligned.h>
 
 #include <acpi/battery.h>
 
@@ -57,13 +62,18 @@ MODULE_PARM_DESC(fw_debug, "Enable printing of firmware debug messages");
 #define LG_ADDRESS_SPACE_DEBUG_MSG_START_ADR	0x3E8
 #define LG_ADDRESS_SPACE_DEBUG_MSG_END_ADR	0x5E8
 
-#define WMI_EVENT_GUID0	"E4FB94F9-7F2B-4173-AD1A-CD1D95086248"
-#define WMI_EVENT_GUID1	"023B133E-49D1-4E10-B313-698220140DC2"
-#define WMI_EVENT_GUID2	"37BE1AC0-C3F2-4B1F-BFBE-8FDEAF2814D6"
-#define WMI_EVENT_GUID3	"911BAD44-7DF8-4FBB-9319-BABA1C4B293B"
-#define WMI_METHOD_WMAB "C3A72B38-D3EF-42D3-8CBB-D5A57049F66D"
-#define WMI_METHOD_WMBB "2B4F501A-BD3C-4394-8DCF-00A7D2BC8210"
-#define WMI_EVENT_GUID  WMI_EVENT_GUID0
+#define LG_NOTIFY_TABLET_MODE_OFF	0x50
+#define LG_NOTIFY_TABLET_MODE_ON	0x51
+#define LG_NOTIFY_HOTKEY		0x80
+#define LG_NOTIFY_THERMAL		0x81
+#define LG_NOTIFY_MISC			0x82
+
+#define LG_OREP_READ_EC			0
+#define LG_OREP_WRITE_EC		1
+#define LG_OREP_DEBUG			2
+#define LG_OREP_UPDATE_SYSTEM_STATE	3
+#define LG_OREP_INTERCEPT_WMI_EVENTS	4
+#define LG_OREP_WAKE_ON_LAN		6
 
 #define SB_GGOV_METHOD  "\\_SB.GGOV"
 #define GOV_TLED        0x2020008
@@ -78,42 +88,95 @@ MODULE_PARM_DESC(fw_debug, "Enable printing of firmware debug messages");
 #define WMBB_USB_CHARGE 0x10B
 #define WMBB_BATT_LIMIT 0x10C
 
+#define KBD_LED_BRIGHTNESS_MASK GENMASK(4, 0)
+#define KBD_LED_BRIGHTNESS_OFF	0x0
+#define KBD_LED_BRIGHTNESS_HALF 0x2
+#define KBD_LED_BRIGHTNESS_FULL 0x4
+#define KBD_LED_MODE_MASK	GENMASK(6, 5)
+#define KBD_LED_MODE_OFF	0x0
+#define KBD_LED_MODE_ON		0x1
+#define KBD_LED_STATUS		BIT(7)
+#define KBD_LED_MAGIC_MASK	GENMASK(15, 8)
+/* Exact purpose is unknown, maybe some sort of brightness limit? */
+#define KBD_LED_MAGIC		0x05
+
 #define FAN_MODE_LOWER GENMASK(1, 0)
 #define FAN_MODE_UPPER GENMASK(5, 4)
 
 #define PLATFORM_NAME   "lg-laptop"
 
-MODULE_ALIAS("wmi:" WMI_EVENT_GUID0);
-MODULE_ALIAS("wmi:" WMI_EVENT_GUID1);
-MODULE_ALIAS("wmi:" WMI_EVENT_GUID2);
-MODULE_ALIAS("wmi:" WMI_EVENT_GUID3);
-MODULE_ALIAS("wmi:" WMI_METHOD_WMAB);
-MODULE_ALIAS("wmi:" WMI_METHOD_WMBB);
+struct lg_wmab_buffer_result {
+	__le32 value;
+	__le32 status;
+} __packed;
 
 static struct platform_device *pf_device;
-static struct input_dev *wmi_input_dev;
-
-static u32 inited;
-#define INIT_INPUT_WMI_0        0x01
-#define INIT_INPUT_WMI_2        0x02
-#define INIT_INPUT_ACPI         0x04
-#define INIT_SPARSE_KEYMAP      0x80
 
 static int battery_limit_use_wmbb;
+static bool kbd_backlight_available;
 static struct led_classdev kbd_backlight;
 static enum led_brightness get_kbd_backlight_level(struct device *dev);
 
 static const struct key_entry wmi_keymap[] = {
-	{KE_KEY, 0x70, {KEY_F15} },	 /* LG control panel (F1) */
-	{KE_KEY, 0x74, {KEY_F21} },	 /* Touchpad toggle (F5) */
-	{KE_KEY, 0xf020000, {KEY_F14} }, /* Read mode (F9) */
-	{KE_KEY, 0x10000000, {KEY_F16} },/* Keyboard backlight (F8) - pressing
-					  * this key both sends an event and
-					  * changes backlight level.
-					  */
+	/* Placeholder value send by multiple hotkeys handled by ACPI */
+	{ KE_IGNORE,	0x0,		{ KEY_UNKNOWN }},
+	/* LG control panel */
+	{ KE_KEY,	0x70,		{ KEY_F15 }},
+	/* Touchpad toggle */
+	{ KE_KEY,	0x74,		{ KEY_F21 }},
+	/* Mute Audio, already handled by ACPI */
+	{ KE_IGNORE,	0x78,		{ KEY_MUTE }},
+	/* Read mode */
+	{ KE_KEY,	0xf020000,	{ KEY_F14 }},
+	/* Open settings */
+	{ KE_KEY,	0xf070002,	{ KEY_CONFIG }},
+	/* Keyboard backlight - pressing this key both sends an event and changes backlight level */
+	{ KE_KEY,	0x10000000,	{ KEY_F16 }},
+	/* Hotkey combination pressed */
+	{ KE_IGNORE,	0x30010000,	{ KEY_UNKNOWN }},
+	/* Hotkey combination released */
+	{ KE_IGNORE,	0x30010001,	{ KEY_UNKNOWN }},
+	/* Change fan mode */
+	{ KE_KEY,	0x30010051,	{ KEY_PERFORMANCE }},
+	/* Disable camera */
+	{ KE_KEY,	0x40020000,	{ KEY_CAMERA_ACCESS_TOGGLE }},
+	/* Mute microphone */
+	{ KE_KEY,	0x40020001,	{ KEY_MICMUTE }},
+	/* Fn-Lock */
+	{ KE_KEY,	0x40030001,	{ KEY_FN_ESC }},
 	{KE_END, 0}
 };
 
+static int lg_laptop_execute_orep(acpi_handle handle, u64 command, u64 value,
+				  unsigned long long *result)
+{
+	union acpi_object objs[] = {
+		{
+			.integer = {
+				.type = ACPI_TYPE_INTEGER,
+				.value = command,
+			},
+		},
+		{
+			.integer = {
+				.type = ACPI_TYPE_INTEGER,
+				.value = value,
+			},
+		}
+	};
+	struct acpi_object_list args = {
+		.count = ARRAY_SIZE(objs),
+		.pointer = objs,
+	};
+	acpi_status status;
+
+	status = acpi_evaluate_integer(handle, "OREP", &args, result);
+	if (ACPI_FAILURE(status))
+		return -EIO;
+
+	return 0;
+}
+
 static int ggov(u32 arg0)
 {
 	union acpi_object args[1];
@@ -154,30 +217,97 @@ static int ggov(u32 arg0)
 	return res;
 }
 
-static union acpi_object *lg_wmab(struct device *dev, u32 method, u32 arg1, u32 arg2)
+static int lg_wmab_get(struct device *dev, u32 method, u32 *result)
 {
-	union acpi_object args[3];
-	acpi_status status;
-	struct acpi_object_list arg;
 	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object in[] = {
+		{
+			.integer = {
+				.type = ACPI_TYPE_INTEGER,
+				.value = method,
+			},
+		},
+		{
+			.integer = {
+				.type = ACPI_TYPE_INTEGER,
+				.value = WM_GET,
+			},
+		},
+		{
+			.integer = {
+				.type = ACPI_TYPE_INTEGER,
+				.value = 0,
+			},
+		},
+	};
+	struct lg_wmab_buffer_result *buffer_result;
+	struct acpi_object_list input = {
+		.count = ARRAY_SIZE(in),
+		.pointer = in,
+	};
+	acpi_status status;
 
-	args[0].type = ACPI_TYPE_INTEGER;
-	args[0].integer.value = method;
-	args[1].type = ACPI_TYPE_INTEGER;
-	args[1].integer.value = arg1;
-	args[2].type = ACPI_TYPE_INTEGER;
-	args[2].integer.value = arg2;
+	status = acpi_evaluate_object(ACPI_HANDLE(dev), "WMAB", &input, &buffer);
+	if (ACPI_FAILURE(status))
+		return -EIO;
 
-	arg.count = 3;
-	arg.pointer = args;
+	union acpi_object *obj __free(kfree) = buffer.pointer;
 
-	status = acpi_evaluate_object(ACPI_HANDLE(dev), "WMAB", &arg, &buffer);
-	if (ACPI_FAILURE(status)) {
-		dev_err(dev, "WMAB: call failed.\n");
-		return NULL;
+	if (!obj)
+		return -ENODATA;
+
+	switch (obj->type) {
+	case ACPI_TYPE_INTEGER:
+		*result = obj->integer.value;
+		return 0;
+	case ACPI_TYPE_BUFFER:
+		if (obj->buffer.length != sizeof(*buffer_result))
+			return -EPROTO;
+
+		buffer_result = (struct lg_wmab_buffer_result *)obj->buffer.pointer;
+		if (get_unaligned_le32(&buffer_result->status))
+			return -EIO;
+
+		*result = get_unaligned_le32(&buffer_result->value);
+		return 0;
+	default:
+		return -EPROTO;
 	}
+}
+
+static int lg_wmab_set(struct device *dev, u32 method, u32 arg)
+{
+	union acpi_object in[] = {
+		{
+			.integer = {
+				.type = ACPI_TYPE_INTEGER,
+				.value = method,
+			},
+		},
+		{
+			.integer = {
+				.type = ACPI_TYPE_INTEGER,
+				.value = WM_SET,
+			},
+		},
+		{
+			.integer = {
+				.type = ACPI_TYPE_INTEGER,
+				.value = arg,
+			},
+		},
+	};
+	struct acpi_object_list input = {
+		.count = ARRAY_SIZE(in),
+		.pointer = in,
+	};
+	acpi_status status;
+
+	status = acpi_evaluate_object(ACPI_HANDLE(dev), "WMAB", &input, NULL);
+	if (ACPI_FAILURE(status))
+		return -EIO;
 
-	return buffer.pointer;
+	return 0;
 }
 
 static union acpi_object *lg_wmbb(struct device *dev, u32 method_id, u32 arg1, u32 arg2)
@@ -211,72 +341,11 @@ static union acpi_object *lg_wmbb(struct device *dev, u32 method_id, u32 arg1, u
 	return (union acpi_object *)buffer.pointer;
 }
 
-static void wmi_notify(union acpi_object *obj, void *context)
-{
-	long data = (long)context;
-
-	pr_debug("event guid %li\n", data);
-	if (!obj)
-		return;
-
-	if (obj->type == ACPI_TYPE_INTEGER) {
-		int eventcode = obj->integer.value;
-		struct key_entry *key;
-
-		if (eventcode == 0x10000000) {
-			led_classdev_notify_brightness_hw_changed(
-				&kbd_backlight, get_kbd_backlight_level(kbd_backlight.dev->parent));
-		} else {
-			key = sparse_keymap_entry_from_scancode(
-				wmi_input_dev, eventcode);
-			if (key && key->type == KE_KEY)
-				sparse_keymap_report_entry(wmi_input_dev,
-							   key, 1, true);
-		}
-	}
-
-	pr_debug("Type: %i    Eventcode: 0x%llx\n", obj->type,
-		 obj->integer.value);
-}
-
-static void wmi_input_setup(void)
-{
-	acpi_status status;
-
-	wmi_input_dev = input_allocate_device();
-	if (wmi_input_dev) {
-		wmi_input_dev->name = "LG WMI hotkeys";
-		wmi_input_dev->phys = "wmi/input0";
-		wmi_input_dev->id.bustype = BUS_HOST;
-
-		if (sparse_keymap_setup(wmi_input_dev, wmi_keymap, NULL) ||
-		    input_register_device(wmi_input_dev)) {
-			pr_info("Cannot initialize input device");
-			input_free_device(wmi_input_dev);
-			return;
-		}
-
-		inited |= INIT_SPARSE_KEYMAP;
-		status = wmi_install_notify_handler(WMI_EVENT_GUID0, wmi_notify,
-						    (void *)0);
-		if (ACPI_SUCCESS(status))
-			inited |= INIT_INPUT_WMI_0;
-
-		status = wmi_install_notify_handler(WMI_EVENT_GUID2, wmi_notify,
-						    (void *)2);
-		if (ACPI_SUCCESS(status))
-			inited |= INIT_INPUT_WMI_2;
-	} else {
-		pr_info("Cannot allocate input device");
-	}
-}
-
 static ssize_t fan_mode_store(struct device *dev,
 			      struct device_attribute *attr,
 			      const char *buffer, size_t count)
 {
 	unsigned long value;
-	union acpi_object *r;
 	int ret;
 
 	ret = kstrtoul(buffer, 10, &value);
@@ -285,10 +354,10 @@ static ssize_t fan_mode_store(struct device *dev,
 	if (value >= 3)
 		return -EINVAL;
 
-	r = lg_wmab(dev, WM_FAN_MODE, WM_SET,
-		FIELD_PREP(FAN_MODE_LOWER, value) |
-		FIELD_PREP(FAN_MODE_UPPER, value));
-	kfree(r);
+	ret = lg_wmab_set(dev, WM_FAN_MODE, FIELD_PREP(FAN_MODE_LOWER, value) |
+			  FIELD_PREP(FAN_MODE_UPPER, value));
+	if (ret < 0)
+		return ret;
 
 	return count;
 }
@@ -296,22 +365,14 @@ static ssize_t fan_mode_store(struct device *dev,
 static ssize_t fan_mode_show(struct device *dev,
 			     struct device_attribute *attr, char *buffer)
 {
-	unsigned int mode;
-	union acpi_object *r;
-
-	r = lg_wmab(dev, WM_FAN_MODE, WM_GET, 0);
-	if (!r)
-		return -EIO;
-
-	if (r->type != ACPI_TYPE_INTEGER) {
-		kfree(r);
-		return -EIO;
-	}
+	u32 mode;
+	int ret;
 
-	mode = FIELD_GET(FAN_MODE_LOWER, r->integer.value);
-	kfree(r);
+	ret = lg_wmab_get(dev, WM_FAN_MODE, &mode);
+	if (ret < 0)
+		return ret;
 
-	return sysfs_emit(buffer, "%d\n", mode);
+	return sysfs_emit(buffer, "%lu\n", FIELD_GET(FAN_MODE_LOWER, mode));
 }
 
 static ssize_t usb_charge_store(struct device *dev,
@@ -361,41 +422,30 @@ static ssize_t reader_mode_store(struct device *dev,
 				 const char *buffer, size_t count)
 {
 	bool value;
-	union acpi_object *r;
 	int ret;
 
 	ret = kstrtobool(buffer, &value);
 	if (ret)
 		return ret;
 
-	r = lg_wmab(dev, WM_READER_MODE, WM_SET, value);
-	if (!r)
-		return -EIO;
+	ret = lg_wmab_set(dev, WM_READER_MODE, value);
+	if (ret < 0)
+		return ret;
 
-	kfree(r);
 	return count;
 }
 
 static ssize_t reader_mode_show(struct device *dev,
 				struct device_attribute *attr, char *buffer)
 {
-	unsigned int status;
-	union acpi_object *r;
-
-	r = lg_wmab(dev, WM_READER_MODE, WM_GET, 0);
-	if (!r)
-		return -EIO;
-
-	if (r->type != ACPI_TYPE_INTEGER) {
-		kfree(r);
-		return -EIO;
-	}
-
-	status = !!r->integer.value;
+	u32 status;
+	int ret;
 
-	kfree(r);
+	ret = lg_wmab_get(dev, WM_READER_MODE, &status);
+	if (ret < 0)
+		return ret;
 
-	return sysfs_emit(buffer, "%d\n", status);
+	return sysfs_emit(buffer, "%d\n", !!status);
 }
 
 static ssize_t fn_lock_store(struct device *dev,
@@ -403,40 +453,30 @@ static ssize_t fn_lock_store(struct device *dev,
 			     const char *buffer, size_t count)
 {
 	bool value;
-	union acpi_object *r;
 	int ret;
 
 	ret = kstrtobool(buffer, &value);
 	if (ret)
 		return ret;
 
-	r = lg_wmab(dev, WM_FN_LOCK, WM_SET, value);
-	if (!r)
-		return -EIO;
+	ret = lg_wmab_set(dev, WM_FN_LOCK, value);
+	if (ret < 0)
+		return ret;
 
-	kfree(r);
 	return count;
 }
 
 static ssize_t fn_lock_show(struct device *dev,
 			    struct device_attribute *attr, char *buffer)
 {
-	unsigned int status;
-	union acpi_object *r;
-
-	r = lg_wmab(dev, WM_FN_LOCK, WM_GET, 0);
-	if (!r)
-		return -EIO;
-
-	if (r->type != ACPI_TYPE_BUFFER) {
-		kfree(r);
-		return -EIO;
-	}
+	u32 status;
+	int ret;
 
-	status = !!r->buffer.pointer[0];
-	kfree(r);
+	ret = lg_wmab_get(dev, WM_FN_LOCK, &status);
+	if (ret < 0)
+		return ret;
 
-	return sysfs_emit(buffer, "%d\n", status);
+	return sysfs_emit(buffer, "%d\n", !!status);
 }
 
 static ssize_t charge_control_end_threshold_store(struct device *dev,
@@ -453,14 +493,18 @@ static ssize_t charge_control_end_threshold_store(struct device *dev,
 	if (value == 100 || value == 80) {
 		union acpi_object *r;
 
-		if (battery_limit_use_wmbb)
+		if (battery_limit_use_wmbb) {
 			r = lg_wmbb(&pf_device->dev, WMBB_BATT_LIMIT, WM_SET, value);
-		else
-			r = lg_wmab(&pf_device->dev, WM_BATT_LIMIT, WM_SET, value);
-		if (!r)
-			return -EIO;
+			if (!r)
+				return -EIO;
+
+			kfree(r);
+		} else {
+			ret = lg_wmab_set(&pf_device->dev, WM_BATT_LIMIT, value);
+			if (ret < 0)
+				return ret;
+		}
 
-		kfree(r);
 		return count;
 	}
 
@@ -471,8 +515,9 @@ static ssize_t charge_control_end_threshold_show(struct device *device,
 						 struct device_attribute *attr,
 						 char *buf)
 {
-	unsigned int status;
 	union acpi_object *r;
+	u32 status;
+	int ret;
 
 	if (battery_limit_use_wmbb) {
 		r = lg_wmbb(&pf_device->dev, WMBB_BATT_LIMIT, WM_GET, 0);
@@ -485,19 +530,13 @@ static ssize_t charge_control_end_threshold_show(struct device *device,
 		}
 
 		status = r->buffer.pointer[0x10];
+		kfree(r);
 	} else {
-		r = lg_wmab(&pf_device->dev, WM_BATT_LIMIT, WM_GET, 0);
-		if (!r)
-			return -EIO;
-
-		if (r->type != ACPI_TYPE_INTEGER) {
-			kfree(r);
-			return -EIO;
-		}
-
-		status = r->integer.value;
+		ret = lg_wmab_get(&pf_device->dev, WM_BATT_LIMIT, &status);
+		if (ret < 0)
+			return ret;
 	}
-	kfree(r);
+
 	if (status != 80 && status != 100)
 		status = 0;
 
@@ -563,10 +602,7 @@ static const struct attribute_group dev_attribute_group = {
 static void tpad_led_set(struct led_classdev *cdev,
 			 enum led_brightness brightness)
 {
-	union acpi_object *r;
-
-	r = lg_wmab(cdev->dev->parent, WM_TLED, WM_SET, brightness > LED_OFF);
-	kfree(r);
+	lg_wmab_set(cdev->dev->parent, WM_TLED, brightness > LED_OFF);
 }
 
 static enum led_brightness tpad_led_get(struct led_classdev *cdev)
@@ -579,47 +615,50 @@ static LED_DEVICE(tpad_led, 1, 0);
 static void kbd_backlight_set(struct led_classdev *cdev,
 			      enum led_brightness brightness)
 {
-	u32 val;
-	union acpi_object *r;
+	/* Must always be written */
+	u32 value = KBD_LED_STATUS;
+	u32 mode, bright;
 
-	val = 0x22;
-	if (brightness <= LED_OFF)
-		val = 0;
-	if (brightness >= LED_FULL)
-		val = 0x24;
-	r = lg_wmab(cdev->dev->parent, WM_KEY_LIGHT, WM_SET, val);
-	kfree(r);
+	if (brightness <= LED_OFF) {
+		mode = KBD_LED_MODE_OFF;
+		bright = KBD_LED_BRIGHTNESS_OFF;
+	} else {
+		mode = KBD_LED_MODE_ON;
+		if (brightness >= LED_FULL)
+			bright = KBD_LED_BRIGHTNESS_FULL;
+		else
+			bright = KBD_LED_BRIGHTNESS_HALF;
+	}
+
+	value |= FIELD_PREP(KBD_LED_BRIGHTNESS_MASK, bright);
+	value |= FIELD_PREP(KBD_LED_MODE_MASK, mode);
+
+	lg_wmab_set(cdev->dev->parent, WM_KEY_LIGHT, value);
 }
 
 static enum led_brightness get_kbd_backlight_level(struct device *dev)
 {
-	union acpi_object *r;
-	int val;
+	u32 value;
+	int ret;
 
-	r = lg_wmab(dev, WM_KEY_LIGHT, WM_GET, 0);
+	ret = lg_wmab_get(dev, WM_KEY_LIGHT, &value);
+	if (ret < 0)
+		return LED_OFF;
 
-	if (!r)
+	if (FIELD_GET(KBD_LED_MAGIC_MASK, value) != KBD_LED_MAGIC)
 		return LED_OFF;
 
-	if (r->type != ACPI_TYPE_BUFFER || r->buffer.pointer[1] != 0x05) {
-		kfree(r);
+	if (FIELD_GET(KBD_LED_MODE_MASK, value) == KBD_LED_MODE_OFF)
 		return LED_OFF;
-	}
 
-	switch (r->buffer.pointer[0] & 0x27) {
-	case 0x24:
-		val = LED_FULL;
-		break;
-	case 0x22:
-		val = LED_HALF;
-		break;
+	switch (FIELD_GET(KBD_LED_BRIGHTNESS_MASK, value)) {
+	case KBD_LED_BRIGHTNESS_FULL:
+		return LED_FULL;
+	case KBD_LED_BRIGHTNESS_HALF:
+		return LED_HALF;
 	default:
-		val = LED_OFF;
+		return LED_OFF;
 	}
-
-	kfree(r);
-
-	return val;
 }
 
 static enum led_brightness kbd_backlight_get(struct led_classdev *cdev)
@@ -629,26 +668,163 @@ static enum led_brightness kbd_backlight_get(struct led_classdev *cdev)
 
 static LED_DEVICE(kbd_backlight, 255, LED_BRIGHT_HW_CHANGED);
 
-static void wmi_input_destroy(void)
+static struct platform_driver pf_driver = {
+	.driver = {
+		   .name = PLATFORM_NAME,
+	}
+};
+
+static int lg_laptop_get_event_data(acpi_handle handle, u32 value, u32 *data)
 {
-	if (inited & INIT_INPUT_WMI_2)
-		wmi_remove_notify_handler(WMI_EVENT_GUID2);
+	union acpi_object objs[] = {
+		{
+			.integer = {
+				.type = ACPI_TYPE_INTEGER,
+				.value = value,
+			},
+		}
+	};
+	struct acpi_object_list args = {
+		.count = ARRAY_SIZE(objs),
+		.pointer = objs,
+	};
+	unsigned long long result;
+	acpi_status status;
+
+	status = acpi_evaluate_integer(handle, "_WED", &args, &result);
+	if (ACPI_FAILURE(status))
+		return -EIO;
 
-	if (inited & INIT_INPUT_WMI_0)
-		wmi_remove_notify_handler(WMI_EVENT_GUID0);
+	if (result > U32_MAX)
+		return -EPROTO;
 
-	if (inited & INIT_SPARSE_KEYMAP)
-		input_unregister_device(wmi_input_dev);
+	*data = result;
 
-	inited &= ~(INIT_INPUT_WMI_0 | INIT_INPUT_WMI_2 | INIT_SPARSE_KEYMAP);
+	return 0;
 }
 
-static struct platform_driver pf_driver = {
-	.driver = {
-		   .name = PLATFORM_NAME,
+static void lg_laptop_handle_input_event(struct input_dev *input_dev, u32 value, u32 data)
+{
+	unsigned int kbd_brightness;
+
+	switch (value) {
+	case LG_NOTIFY_HOTKEY:
+		sparse_keymap_report_event(input_dev, data, 1, true);
+		break;
+	case LG_NOTIFY_THERMAL:
+		/* Currently not supported */
+		break;
+	case LG_NOTIFY_MISC:
+		switch (data) {
+		case 0x10000000:
+			if (!kbd_backlight_available)
+				break;
+
+			kbd_brightness = get_kbd_backlight_level(kbd_backlight.dev->parent);
+			led_classdev_notify_brightness_hw_changed(&kbd_backlight, kbd_brightness);
+			break;
+		default:
+			sparse_keymap_report_event(input_dev, data, 1, true);
+		}
+		break;
+	default:
+		break;
+	}
+}
+
+static void lg_laptop_notify_handler(acpi_handle handle, u32 value, void *context)
+{
+	struct input_dev *input_dev = context;
+	u32 data;
+	int ret;
+
+	switch (value) {
+	case LG_NOTIFY_TABLET_MODE_OFF:
+	case LG_NOTIFY_TABLET_MODE_ON:
+		/* Already handled by intel-hid */
+		return;
+	case LG_NOTIFY_HOTKEY:
+	case LG_NOTIFY_THERMAL:
+	case LG_NOTIFY_MISC:
+		ret = lg_laptop_get_event_data(handle, value, &data);
+		if (ret < 0) {
+			dev_notice(input_dev->dev.parent, "Failed to get event data: %d\n", ret);
+			return;
+		}
+
+		dev_dbg(input_dev->dev.parent, "Received event %u (%u)\n", value, data);
+
+		lg_laptop_handle_input_event(input_dev, value, data);
+		return;
+	default:
+		dev_notice(input_dev->dev.parent, "Received unknown event %u\n", value);
 	}
 };
 
+static void lg_laptop_remove_notify_handler(void *context)
+{
+	acpi_handle handle = context;
+
+	acpi_remove_notify_handler(handle, ACPI_ALL_NOTIFY, lg_laptop_notify_handler);
+}
+
+static void lg_laptop_reenable_wmi_events(void *context)
+{
+	acpi_handle handle = context;
+	unsigned long long dummy;
+
+	lg_laptop_execute_orep(handle, LG_OREP_INTERCEPT_WMI_EVENTS, 0, &dummy);
+}
+
+static int lg_laptop_input_init(struct device *dev, acpi_handle handle)
+{
+	struct input_dev *input_dev;
+	unsigned long long result;
+	acpi_status status;
+	int ret;
+
+	if (!acpi_has_method(handle, "_WED"))
+		return 0;
+
+	input_dev = devm_input_allocate_device(dev);
+	if (!input_dev)
+		return -ENOMEM;
+
+	input_dev->name = "LG WMI hotkeys";
+	input_dev->phys = "wmi/input0";
+	input_dev->id.bustype = BUS_HOST;
+	ret = sparse_keymap_setup(input_dev, wmi_keymap, NULL);
+	if (ret < 0)
+		return ret;
+
+	ret = input_register_device(input_dev);
+	if (ret < 0)
+		return ret;
+
+	status = acpi_install_notify_handler(handle, ACPI_ALL_NOTIFY, lg_laptop_notify_handler,
+					     input_dev);
+	if (ACPI_FAILURE(status))
+		return -EIO;
+
+	ret = devm_add_action_or_reset(dev, lg_laptop_remove_notify_handler, handle);
+	if (ret < 0)
+		return ret;
+
+	if (acpi_has_method(handle, "OREP")) {
+		ret = lg_laptop_execute_orep(handle, LG_OREP_INTERCEPT_WMI_EVENTS, 1, &result);
+		if (ret < 0)
+			return ret;
+		if (result)
+			return -EIO;
+
+		ret = devm_add_action_or_reset(dev, lg_laptop_reenable_wmi_events, handle);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
 static acpi_status lg_laptop_address_space_write(struct device *dev, acpi_physical_address address,
 						 size_t size, u64 value)
 {
@@ -860,15 +1036,23 @@ static int acpi_probe(struct platform_device *pdev)
 	if (year >= 2019)
 		battery_limit_use_wmbb = 1;
 
+	/* LEDs are optional */
+	ret = devm_led_classdev_register(&pdev->dev, &kbd_backlight);
+	if (ret < 0)
+		kbd_backlight_available = false;
+	else
+		kbd_backlight_available = true;
+
+	devm_led_classdev_register(&pdev->dev, &tpad_led);
+
+	ret = lg_laptop_input_init(&pdev->dev, device->handle);
+	if (ret < 0)
+		goto out_platform_device;
+
 	ret = sysfs_create_group(&pf_device->dev.kobj, &dev_attribute_group);
 	if (ret)
 		goto out_platform_device;
 
-	/* LEDs are optional */
-	led_classdev_register(&pf_device->dev, &kbd_backlight);
-	led_classdev_register(&pf_device->dev, &tpad_led);
-
-	wmi_input_setup();
 	battery_hook_register(&battery_hook);
 
 	return 0;
@@ -884,11 +1068,7 @@ static void acpi_remove(struct platform_device *pdev)
 {
 	sysfs_remove_group(&pf_device->dev.kobj, &dev_attribute_group);
 
-	led_classdev_unregister(&tpad_led);
-	led_classdev_unregister(&kbd_backlight);
-
 	battery_hook_unregister(&battery_hook);
-	wmi_input_destroy();
 	platform_device_unregister(pf_device);
 	pf_device = NULL;
 	platform_driver_unregister(&pf_driver);
diff --git a/drivers/platform/x86/msi-ec.c b/drivers/platform/x86/msi-ec.c
index dfe4532ebe561..667df9509798a 100644
--- a/drivers/platform/x86/msi-ec.c
+++ b/drivers/platform/x86/msi-ec.c
@@ -1130,6 +1130,86 @@ static struct msi_ec_conf CONF13 __initdata = {
 	},
 };
 
+static const char * const ALLOWED_FW_14[] __initconst = {
+	"1824EMS1.108", // Raider 18 HX AI A2XWJG (MS-1824)
+	"182LIMS1.108", // Vector A18 HX A9WHG
+	"182LIMS1.111", // MSI Raider A18 HX A9WJG / Vector A18 HX A9WHG
+	"182KIMS1.113", // Raider A18 HX A7VIG
+	NULL
+};
+
+static struct msi_ec_conf CONF14 __initdata = {
+	.allowed_fw = ALLOWED_FW_14,
+	.charge_control = {
+		.address      = 0xd7,
+		.offset_start = 0x8a,
+		.offset_end   = 0x80,
+		.range_min    = 0x8a,
+		.range_max    = 0xe4,
+	},
+	.webcam = {
+		.address       = 0x2e,
+		.block_address = 0x2f,
+		.bit           = 1,
+	},
+	.fn_win_swap = {
+		.address = 0xe8,
+		.bit     = 4,
+	},
+	.cooler_boost = {
+		.address = 0x98,
+		.bit     = 7,
+	},
+	.shift_mode = {
+		.address = 0xd2,
+		.modes = {
+			{ SM_ECO_NAME,     0xc2 },
+			{ SM_COMFORT_NAME, 0xc1 },
+			{ SM_TURBO_NAME,   0xc4 },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.super_battery = {
+		.address = 0xeb,
+		.mask    = 0x0f,
+	},
+	.fan_mode = {
+		.address = 0xd4,
+		.modes = {
+			{ FM_AUTO_NAME,     0x0d },
+			{ FM_SILENT_NAME,   0x1d },
+			{ FM_ADVANCED_NAME, 0x8d },
+			MSI_EC_MODE_NULL
+		},
+	},
+	.cpu = {
+		.rt_temp_address       = 0x68,
+		.rt_fan_speed_address  = 0x71,
+		.rt_fan_speed_base_min = 0x00,
+		.rt_fan_speed_base_max = 0x96,
+		.bs_fan_speed_address  = MSI_EC_ADDR_UNSUPP,
+		.bs_fan_speed_base_min = 0x00,
+		.bs_fan_speed_base_max = 0x0f,
+	},
+	.gpu = {
+		.rt_temp_address      = 0x80,
+		.rt_fan_speed_address = 0x89,
+	},
+	.leds = {
+		.micmute_led_address = 0x2c,
+		.mute_led_address    = 0x2d,
+		.bit                 = 1,
+	},
+	.kbd_bl = {
+		.bl_mode_address  = MSI_EC_ADDR_UNSUPP,
+		.bl_modes         = { 0x00, 0x08 },
+		.max_mode         = 1,
+		.bl_state_address = MSI_EC_ADDR_UNSUPP,
+		.state_base_value = 0x80,
+		.max_state        = 3,
+	},
+};
+
 static struct msi_ec_conf *CONFIGS[] __initdata = {
 	&CONF0,
 	&CONF1,
@@ -1145,6 +1225,7 @@ static struct msi_ec_conf *CONFIGS[] __initdata = {
 	&CONF11,
 	&CONF12,
 	&CONF13,
+	&CONF14,
 	NULL
 };
 
diff --git a/drivers/platform/x86/msi-wmi.c b/drivers/platform/x86/msi-wmi.c
index 4a7ac85c4db48..c9db750fe5ae6 100644
--- a/drivers/platform/x86/msi-wmi.c
+++ b/drivers/platform/x86/msi-wmi.c
@@ -46,6 +46,12 @@ enum msi_scancodes {
 	WIND_KEY_WLAN		= 0x5f,	/* Fn+F11 Wi-Fi toggle */
 	WIND_KEY_TURBO,			/* Fn+F10 turbo mode toggle */
 	WIND_KEY_ECO		= 0x69,	/* Fn+F10 ECO mode toggle */
+	/* MSI Claw keys */
+	CLAW_KEY_VOLUMEDOWN	= 0x21,
+	CLAW_KEY_CENTER		= 0x29,	/* MSI M-Center main menu */
+	CLAW_KEY_QUICK_LONG	= 0x2a,	/* MSI M-Center quick access long hold */
+	CLAW_KEY_VOLUMEUP	= 0x32,
+	CLAW_KEY_QUICK_SHORT	= 0x58,	/* MSI M-Center quick access short press */
 };
 static struct key_entry msi_wmi_keymap[] = {
 	{ KE_KEY, MSI_KEY_BRIGHTNESSUP,		{KEY_BRIGHTNESSUP} },
@@ -69,6 +75,15 @@ static struct key_entry msi_wmi_keymap[] = {
 	{ KE_KEY, WIND_KEY_TURBO,		{KEY_PROG1} },
 	{ KE_KEY, WIND_KEY_ECO,			{KEY_PROG2} },
 
+	/* These are MSI Claw keys, used for MSI M-Center in Windows */
+	{ KE_KEY, CLAW_KEY_CENTER,		{KEY_F15} },
+
+	/* These MSI Claw keys work without WMI. Ignore them to avoid double keycodes */
+	{ KE_IGNORE, CLAW_KEY_QUICK_SHORT },
+	{ KE_IGNORE, CLAW_KEY_QUICK_LONG },
+	{ KE_IGNORE, CLAW_KEY_VOLUMEUP },
+	{ KE_IGNORE, CLAW_KEY_VOLUMEDOWN },
+
 	{ KE_END, 0 }
 };
 
@@ -172,44 +187,62 @@ static const struct backlight_ops msi_backlight_ops = {
 
 static void msi_wmi_notify(union acpi_object *obj, void *context)
 {
-	struct key_entry *key;
+	struct key_entry *key = NULL;
+	int eventcode = 0;
+
+	if (!obj)
+		return;
 
-	if (obj && obj->type == ACPI_TYPE_INTEGER) {
-		int eventcode = obj->integer.value;
+	switch (obj->type) {
+	case ACPI_TYPE_INTEGER:
+		eventcode = obj->integer.value;
 		pr_debug("Eventcode: 0x%x\n", eventcode);
-		key = sparse_keymap_entry_from_scancode(msi_wmi_input_dev,
-				eventcode);
-		if (!key) {
-			pr_info("Unknown key pressed - %x\n", eventcode);
+		break;
+	case ACPI_TYPE_BUFFER:
+		/* Field returns u8[2] here, but is u32 by spec. Allow "oversized" buffers. */
+		if (obj->buffer.length < 2)
 			return;
-		}
 
-		if (event_wmi->quirk_last_pressed) {
-			ktime_t cur = ktime_get_real();
-			ktime_t diff = ktime_sub(cur, last_pressed);
-			/* Ignore event if any event happened in a 50 ms
-			   timeframe -> Key press may result in 10-20 GPEs */
-			if (ktime_to_us(diff) < 1000 * 50) {
-				pr_debug("Suppressed key event 0x%X - "
-					 "Last press was %lld us ago\n",
-					 key->code, ktime_to_us(diff));
-				return;
-			}
-			last_pressed = cur;
-		}
+		/* pointer[0] is key ID, pointer[1] is active state. We don't get release
+		 * events, so ignore the active state and treat as autorelease.
+		 */
+		eventcode = obj->buffer.pointer[0];
+		pr_debug("Eventcode: 0x%x\n", eventcode);
+		break;
+	default:
+		pr_info("Unknown event received\n");
+		return;
+	}
+
+	key = sparse_keymap_entry_from_scancode(msi_wmi_input_dev, eventcode);
 
-		if (key->type == KE_KEY &&
-		/* Brightness is served via acpi video driver */
-		(backlight ||
-		(key->code != MSI_KEY_BRIGHTNESSUP &&
-		key->code != MSI_KEY_BRIGHTNESSDOWN))) {
-			pr_debug("Send key: 0x%X - Input layer keycode: %d\n",
-				 key->code, key->keycode);
-			sparse_keymap_report_entry(msi_wmi_input_dev, key, 1,
-						   true);
+	if (!key) {
+		pr_info("Unknown key pressed - 0x%x\n", eventcode);
+		return;
+	}
+
+	if (event_wmi->quirk_last_pressed) {
+		ktime_t cur = ktime_get_real();
+		ktime_t diff = ktime_sub(cur, last_pressed);
+		/* Ignore event if any event happened in a 50 ms
+		 * timeframe -> Key press may result in 10-20 GPEs
+		 */
+		if (ktime_to_us(diff) < 1000 * 50) {
+			pr_debug("Suppressed key event 0x%X - Last press was %lld us ago\n",
+				 key->code, ktime_to_us(diff));
+			return;
 		}
-	} else
-		pr_info("Unknown event received\n");
+		last_pressed = cur;
+	}
+
+	/* Brightness is served via acpi video driver */
+	if (key->type == KE_KEY &&
+	    (backlight || (key->code == MSI_KEY_BRIGHTNESSUP ||
+			   key->code == MSI_KEY_BRIGHTNESSDOWN)))
+		return;
+
+	pr_debug("Send key: 0x%X - Input layer keycode: %d\n", key->code, key->keycode);
+	sparse_keymap_report_entry(msi_wmi_input_dev, key, 1, true);
 }
 
 static int __init msi_wmi_backlight_setup(void)
diff --git a/drivers/platform/x86/panasonic-laptop.c b/drivers/platform/x86/panasonic-laptop.c
index 719add753cb3d..3effb11c41535 100644
--- a/drivers/platform/x86/panasonic-laptop.c
+++ b/drivers/platform/x86/panasonic-laptop.c
@@ -159,8 +159,6 @@ MODULE_LICENSE("GPL");
 #define ECO_MODE_ON		0x80
 
 #define ACPI_PCC_DRIVER_NAME	"Panasonic Laptop Support"
-#define ACPI_PCC_DEVICE_NAME	"Hotkey"
-#define ACPI_PCC_CLASS		"pcc"
 
 #define ACPI_PCC_INPUT_PHYS	"panasonic/hkey0"
 
@@ -1017,8 +1015,6 @@ static int acpi_pcc_hotkey_probe(struct platform_device *pdev)
 	pcc->device = device;
 	pcc->handle = device->handle;
 	device->driver_data = pcc;
-	strscpy(acpi_device_name(device), ACPI_PCC_DEVICE_NAME);
-	strscpy(acpi_device_class(device), ACPI_PCC_CLASS);
 
 	result = acpi_pcc_init_input(pcc);
 	if (result) {
diff --git a/drivers/platform/x86/sony-laptop.c b/drivers/platform/x86/sony-laptop.c
index 67370967df6f5..0bf4d3054e374 100644
--- a/drivers/platform/x86/sony-laptop.c
+++ b/drivers/platform/x86/sony-laptop.c
@@ -1264,7 +1264,7 @@ static void sony_nc_notify(acpi_handle ah, u32 event, void *data)
 		ev_type = HOTKEY;
 		sony_laptop_report_input_event(real_ev);
 	}
-	acpi_bus_generate_netlink_event(sony_nc_acpi_device->pnp.device_class,
+	acpi_bus_generate_netlink_event("sony/hotkey",
 			dev_name(&sony_nc_acpi_device->dev), ev_type, real_ev);
 }
 
@@ -3157,8 +3157,6 @@ static int sony_nc_probe(struct platform_device *pdev)
 		return -ENODEV;
 
 	sony_nc_acpi_device = device;
-	strscpy(acpi_device_class(device), "sony/hotkey");
-
 	sony_nc_acpi_handle = device->handle;
 
 	/* read device status */
@@ -4523,7 +4521,6 @@ static int sony_pic_probe(struct platform_device *pdev)
 		return -ENODEV;
 
 	spic_dev.acpi_dev = device;
-	strscpy(acpi_device_class(device), "sony/hotkey");
 	sony_pic_detect_device_type(&spic_dev);
 	mutex_init(&spic_dev.lock);
 
diff --git a/drivers/platform/x86/topstar-laptop.c b/drivers/platform/x86/topstar-laptop.c
index e09d7f8ce45f9..0e842c55dcc52 100644
--- a/drivers/platform/x86/topstar-laptop.c
+++ b/drivers/platform/x86/topstar-laptop.c
@@ -299,8 +299,6 @@ static int topstar_acpi_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, topstar);
 
-	strscpy(acpi_device_name(device), "Topstar TPSACPI");
-	strscpy(acpi_device_class(device), TOPSTAR_LAPTOP_CLASS);
 	topstar->device = device;
 
 	err = topstar_acpi_init(topstar);
diff --git a/drivers/platform/x86/toshiba_acpi.c b/drivers/platform/x86/toshiba_acpi.c
index 7cecb3a70b9c3..a0b8060836d01 100644
--- a/drivers/platform/x86/toshiba_acpi.c
+++ b/drivers/platform/x86/toshiba_acpi.c
@@ -2505,8 +2505,7 @@ static void toshiba_acpi_kbd_bl_work(struct work_struct *work)
 				LED_FULL : LED_OFF);
 
 	/* Emulate the keyboard backlight event */
-	acpi_bus_generate_netlink_event(toshiba_acpi->acpi_dev->pnp.device_class,
-					dev_name(&toshiba_acpi->acpi_dev->dev),
+	acpi_bus_generate_netlink_event("", dev_name(&toshiba_acpi->acpi_dev->dev),
 					0x92, 0);
 }
 
@@ -3250,8 +3249,7 @@ static void toshiba_acpi_notify(acpi_handle handle, u32 event, void *data)
 		break;
 	}
 
-	acpi_bus_generate_netlink_event(acpi_dev->pnp.device_class,
-					dev_name(&acpi_dev->dev),
+	acpi_bus_generate_netlink_event("", dev_name(&acpi_dev->dev),
 					event, (event == 0x80) ?
 					dev->last_key_event : 0);
 }
diff --git a/drivers/platform/x86/toshiba_bluetooth.c b/drivers/platform/x86/toshiba_bluetooth.c
index e00abba58c7c0..bf199fba49992 100644
--- a/drivers/platform/x86/toshiba_bluetooth.c
+++ b/drivers/platform/x86/toshiba_bluetooth.c
@@ -252,10 +252,8 @@ static int toshiba_bt_rfkill_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, bt_dev);
 
 	result = toshiba_bluetooth_sync_status(bt_dev);
-	if (result) {
-		kfree(bt_dev);
-		return result;
-	}
+	if (result)
+		goto err_free_bt_dev;
 
 	bt_dev->rfk = rfkill_alloc("Toshiba Bluetooth",
 				   &pdev->dev,
diff --git a/drivers/platform/x86/toshiba_haps.c b/drivers/platform/x86/toshiba_haps.c
index 8d12241924df3..c6633b74029f0 100644
--- a/drivers/platform/x86/toshiba_haps.c
+++ b/drivers/platform/x86/toshiba_haps.c
@@ -136,9 +136,7 @@ static void toshiba_haps_notify(acpi_handle handle, u32 event, void *data)
 
 	pr_debug("Received event: 0x%x\n", event);
 
-	acpi_bus_generate_netlink_event(device->pnp.device_class,
-					dev_name(&device->dev),
-					event, 0);
+	acpi_bus_generate_netlink_event("", dev_name(&device->dev), event, 0);
 }
 
 static void toshiba_haps_remove(struct platform_device *pdev)
diff --git a/drivers/platform/x86/uniwill/uniwill-acpi.c b/drivers/platform/x86/uniwill/uniwill-acpi.c
index ab063ead45b90..f55b239bd4d18 100644
--- a/drivers/platform/x86/uniwill/uniwill-acpi.c
+++ b/drivers/platform/x86/uniwill/uniwill-acpi.c
@@ -95,6 +95,9 @@
 
 #define EC_ADDR_MAIN_FAN_RPM_2		0x0465
 
+#define EC_ADDR_SCREEN_STATUS		0x0466
+#define SCREEN_SUSPENDED		BIT(6)
+
 #define EC_ADDR_SECOND_FAN_RPM_1	0x046C
 
 #define EC_ADDR_SECOND_FAN_RPM_2	0x046D
@@ -109,9 +112,35 @@
 
 #define EC_ADDR_BAT_CYCLE_COUNT_2	0x04A7
 
+#define EC_ADDR_OEM_9			0x0726
+#define AC_AUTO_BOOT_ENABLE		BIT(3)
+
 #define EC_ADDR_PROJECT_ID		0x0740
+#define PROJECT_ID_NONE			0x00
+#define PROJECT_ID_GI			0x01
+#define PROJECT_ID_GJ			0x02
+#define PROJECT_ID_GK			0x03
+#define PROJECT_ID_GICN			0x04
+#define PROJECT_ID_GJCN			0x05
+#define PROJECT_ID_GK5CN_X		0x06
+#define PROJECT_ID_GK7CN_S		0x07
+#define PROJECT_ID_GK7CPCS_GK5CQ7Z	0x08
+#define PROJECT_ID_PF			0x09
+#define PROJECT_ID_GK5CP_4X_5X_6X	0x0A
+#define PROJECT_ID_IDP			0x0B
+#define PROJECT_ID_IDY_6Y		0x0C
+#define PROJECT_ID_IDY_7Y		0x0D
+#define PROJECT_ID_PF4MU_PF4MN_PF5MU	0x0E
+#define PROJECT_ID_CML_GAMING		0x0F
+#define PROJECT_ID_GK7NXXR		0x10
+#define PROJECT_ID_GM5MU1Y		0x11
 #define PROJECT_ID_PH4TRX1		0x12
+#define PROJECT_ID_PH4TUX1		0x13
+#define PROJECT_ID_PH4TQX1		0x14
 #define PROJECT_ID_PH6TRX1		0x15
+#define PROJECT_ID_PH6TQXX		0x16
+#define PROJECT_ID_PHXAXXX		0x17
+#define PROJECT_ID_PHXPXXX		0x18
 
 #define EC_ADDR_AP_OEM			0x0741
 #define	ENABLE_MANUAL_CTRL		BIT(0)
@@ -214,6 +243,7 @@
 #define FAN_TABLE_OFFICE_MODE		BIT(2)
 #define FAN_V3				BIT(3)
 #define DEFAULT_MODE			BIT(4)
+#define ENABLE_CHINA_MODE		BIT(6)
 
 #define EC_ADDR_PL1_SETTING		0x0783
 
@@ -225,11 +255,11 @@
 #define FAN_CURVE_LENGTH		5
 
 #define EC_ADDR_KBD_STATUS		0x078C
-#define KBD_WHITE_ONLY			BIT(0)	// ~single color
-#define KBD_SINGLE_COLOR_OFF		BIT(1)
+#define KBD_WHITE_ONLY			BIT(0)
+#define KBD_POWER_OFF			BIT(1)
 #define KBD_TURBO_LEVEL_MASK		GENMASK(3, 2)
 #define KBD_APPLY			BIT(4)
-#define KBD_BRIGHTNESS			GENMASK(7, 5)
+#define KBD_BRIGHTNESS_MASK		GENMASK(7, 5)
 
 #define EC_ADDR_FAN_CTRL		0x078E
 #define FAN3P5				BIT(1)
@@ -318,7 +348,9 @@
 #define FAN_TABLE_LENGTH	16
 
 #define LED_CHANNELS		3
-#define LED_MAX_BRIGHTNESS	200
+
+#define KBD_LED_CHANNELS	3
+#define KBD_LED_MAX_INTENSITY	50
 
 #define UNIWILL_FEATURE_FN_LOCK			BIT(0)
 #define UNIWILL_FEATURE_SUPER_KEY		BIT(1)
@@ -333,6 +365,9 @@
 #define UNIWILL_FEATURE_SECONDARY_FAN		BIT(9)
 #define UNIWILL_FEATURE_NVIDIA_CTGP_CONTROL	BIT(10)
 #define UNIWILL_FEATURE_USB_C_POWER_PRIORITY	BIT(11)
+#define UNIWILL_FEATURE_KEYBOARD_BACKLIGHT	BIT(12)
+#define UNIWILL_FEATURE_AC_AUTO_BOOT		BIT(13)
+#define UNIWILL_FEATURE_USB_POWERSHARE		BIT(14)
 
 enum usb_c_power_priority_options {
 	USB_C_POWER_PRIORITY_CHARGING = 0,
@@ -344,6 +379,7 @@ struct uniwill_data {
 	acpi_handle handle;
 	struct regmap *regmap;
 	unsigned int features;
+	u8 project_id;
 	struct acpi_battery_hook hook;
 	struct mutex battery_lock;	/* Protects the list of currently registered batteries */
 	union {
@@ -357,11 +393,25 @@ struct uniwill_data {
 	bool last_fn_lock_state;
 	bool last_super_key_enable_state;
 	bool last_touchpad_toggle_enable_state;
+	bool last_usb_powershare_high_state;
 	struct mutex super_key_lock;	/* Protects the toggling of the super key lock state */
 	struct list_head batteries;
 	struct mutex led_lock;		/* Protects writes to the lightbar registers */
+	u8 lightbar_max_brightness;
 	struct led_classdev_mc led_mc_cdev;
 	struct mc_subled led_mc_subled_info[LED_CHANNELS];
+	bool single_color_kbd;
+	u8 kbd_led_max_brightness;
+	unsigned int last_kbd_status;
+	union {
+		struct {
+			/* Protects writes to the RGB keyboard backlight registers */
+			struct mutex kbd_rgb_led_lock;
+			struct led_classdev_mc kbd_led_mc_cdev;
+			struct mc_subled kbd_led_mc_subled_info[KBD_LED_CHANNELS];
+		};
+		struct led_classdev kbd_led_cdev;
+	};
 	struct mutex input_lock;	/* Protects input sequence during notify */
 	struct input_dev *input_device;
 	struct notifier_block nb;
@@ -376,6 +426,8 @@ struct uniwill_battery_entry {
 
 struct uniwill_device_descriptor {
 	unsigned int features;
+	u8 kbd_led_max_brightness;
+	u8 lightbar_max_brightness;
 	/* Executed during driver probing */
 	int (*probe)(struct uniwill_data *data);
 };
@@ -427,6 +479,9 @@ static const struct key_entry uniwill_keymap[] = {
 	{ KE_KEY,       UNIWILL_OSD_KBDILLUMDOWN,               { KEY_KBDILLUMDOWN }},
 	{ KE_KEY,       UNIWILL_OSD_KBDILLUMUP,                 { KEY_KBDILLUMUP }},
 
+	/* Reported when the EC changed the keyboard backlight brightness */
+	{ KE_IGNORE,	UNIWILL_OSD_BACKLIGHT_LEVEL_CHANGE,	{ KEY_UNKNOWN }},
+
 	/* Reported when the user wants to toggle the microphone mute status */
 	{ KE_KEY,       UNIWILL_OSD_MIC_MUTE,                   { KEY_MICMUTE }},
 
@@ -435,11 +490,6 @@ static const struct key_entry uniwill_keymap[] = {
 
 	/* Reported when the user wants to toggle the brightness of the keyboard */
 	{ KE_KEY,       UNIWILL_OSD_KBDILLUMTOGGLE,             { KEY_KBDILLUMTOGGLE }},
-	{ KE_KEY,       UNIWILL_OSD_KB_LED_LEVEL0,              { KEY_KBDILLUMTOGGLE }},
-	{ KE_KEY,       UNIWILL_OSD_KB_LED_LEVEL1,              { KEY_KBDILLUMTOGGLE }},
-	{ KE_KEY,       UNIWILL_OSD_KB_LED_LEVEL2,              { KEY_KBDILLUMTOGGLE }},
-	{ KE_KEY,       UNIWILL_OSD_KB_LED_LEVEL3,              { KEY_KBDILLUMTOGGLE }},
-	{ KE_KEY,       UNIWILL_OSD_KB_LED_LEVEL4,              { KEY_KBDILLUMTOGGLE }},
 
 	/* FIXME: find out the exact meaning of those events */
 	{ KE_IGNORE,    UNIWILL_OSD_BAT_CHARGE_FULL_24_H,       { KEY_UNKNOWN }},
@@ -448,6 +498,9 @@ static const struct key_entry uniwill_keymap[] = {
 	/* Reported when the user wants to toggle the benchmark mode status */
 	{ KE_IGNORE,    UNIWILL_OSD_BENCHMARK_MODE_TOGGLE,      { KEY_UNKNOWN }},
 
+	/* Reported when the screen is enabled/disabled during resume/suspend */
+	{ KE_IGNORE,	UNIWILL_OSD_SCREEN_STATE_CHANGED,	{ KEY_UNKNOWN }},
+
 	/* Reported when the user wants to toggle the webcam */
 	{ KE_IGNORE,    UNIWILL_OSD_WEBCAM_TOGGLE,              { KEY_UNKNOWN }},
 
@@ -540,6 +593,7 @@ static const struct regmap_bus uniwill_ec_bus = {
 static bool uniwill_writeable_reg(struct device *dev, unsigned int reg)
 {
 	switch (reg) {
+	case EC_ADDR_OEM_9:
 	case EC_ADDR_AP_OEM:
 	case EC_ADDR_LIGHTBAR_AC_CTRL:
 	case EC_ADDR_LIGHTBAR_AC_RED:
@@ -547,6 +601,11 @@ static bool uniwill_writeable_reg(struct device *dev, unsigned int reg)
 	case EC_ADDR_LIGHTBAR_AC_BLUE:
 	case EC_ADDR_BIOS_OEM:
 	case EC_ADDR_TRIGGER:
+	case EC_ADDR_RGB_RED:
+	case EC_ADDR_RGB_GREEN:
+	case EC_ADDR_RGB_BLUE:
+	case EC_ADDR_BIOS_OEM_2:
+	case EC_ADDR_KBD_STATUS:
 	case EC_ADDR_OEM_4:
 	case EC_ADDR_CHARGE_CTRL:
 	case EC_ADDR_LIGHTBAR_BAT_CTRL:
@@ -574,6 +633,7 @@ static bool uniwill_readable_reg(struct device *dev, unsigned int reg)
 	case EC_ADDR_SECOND_FAN_RPM_1:
 	case EC_ADDR_SECOND_FAN_RPM_2:
 	case EC_ADDR_BAT_ALERT:
+	case EC_ADDR_OEM_9:
 	case EC_ADDR_PROJECT_ID:
 	case EC_ADDR_AP_OEM:
 	case EC_ADDR_LIGHTBAR_AC_CTRL:
@@ -583,8 +643,14 @@ static bool uniwill_readable_reg(struct device *dev, unsigned int reg)
 	case EC_ADDR_BIOS_OEM:
 	case EC_ADDR_PWM_1:
 	case EC_ADDR_PWM_2:
+	case EC_ADDR_SUPPORT_2:
 	case EC_ADDR_TRIGGER:
 	case EC_ADDR_SWITCH_STATUS:
+	case EC_ADDR_RGB_RED:
+	case EC_ADDR_RGB_GREEN:
+	case EC_ADDR_RGB_BLUE:
+	case EC_ADDR_BIOS_OEM_2:
+	case EC_ADDR_KBD_STATUS:
 	case EC_ADDR_OEM_4:
 	case EC_ADDR_CHARGE_CTRL:
 	case EC_ADDR_LIGHTBAR_BAT_CTRL:
@@ -616,8 +682,10 @@ static bool uniwill_volatile_reg(struct device *dev, unsigned int reg)
 	case EC_ADDR_BIOS_OEM:
 	case EC_ADDR_PWM_1:
 	case EC_ADDR_PWM_2:
+	case EC_ADDR_SUPPORT_2:
 	case EC_ADDR_TRIGGER:
 	case EC_ADDR_SWITCH_STATUS:
+	case EC_ADDR_KBD_STATUS:
 	case EC_ADDR_OEM_4:
 	case EC_ADDR_CHARGE_CTRL:
 	case EC_ADDR_USB_C_POWER_PRIORITY:
@@ -1077,6 +1145,109 @@ static int usb_c_power_priority_init(struct uniwill_data *data)
 	return 0;
 }
 
+static ssize_t ac_auto_boot_store(struct device *dev, struct device_attribute *attr,
+				  const char *buf, size_t count)
+{
+	struct uniwill_data *data = dev_get_drvdata(dev);
+	unsigned int regval;
+	bool enable;
+	int ret;
+
+	ret = kstrtobool(buf, &enable);
+	if (ret < 0)
+		return ret;
+
+	if (enable)
+		regval = AC_AUTO_BOOT_ENABLE;
+	else
+		regval = 0;
+
+	ret = regmap_update_bits(data->regmap, EC_ADDR_OEM_9, AC_AUTO_BOOT_ENABLE, regval);
+	if (ret < 0)
+		return ret;
+
+	return count;
+}
+
+static ssize_t ac_auto_boot_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct uniwill_data *data = dev_get_drvdata(dev);
+	unsigned int regval;
+	int ret;
+
+	ret = regmap_read(data->regmap, EC_ADDR_OEM_9, &regval);
+	if (ret < 0)
+		return ret;
+
+	return sysfs_emit(buf, "%d\n", !!(regval & AC_AUTO_BOOT_ENABLE));
+}
+
+static DEVICE_ATTR_RW(ac_auto_boot);
+
+static int uniwill_write_usb_powershare_high(struct uniwill_data *data, bool status)
+{
+	unsigned int value;
+
+	if (status)
+		value = TRIGGER_USB_CHARGING;
+	else
+		value = 0;
+
+	/*
+	 * Normaly this RMW-sequence could also trigger the super key toggle,
+	 * but the EC seems to take care that those bits are always read as 0.
+	 */
+	return regmap_update_bits(data->regmap, EC_ADDR_TRIGGER, TRIGGER_USB_CHARGING, value);
+}
+
+static ssize_t usb_powershare_high_store(struct device *dev, struct device_attribute *attr,
+					 const char *buf, size_t count)
+{
+	struct uniwill_data *data = dev_get_drvdata(dev);
+	bool enable;
+	int ret;
+
+	ret = kstrtobool(buf, &enable);
+	if (ret < 0)
+		return ret;
+
+	ret = uniwill_write_usb_powershare_high(data, enable);
+	if (ret < 0)
+		return ret;
+
+	return count;
+}
+
+static int uniwill_read_usb_powershare_high(struct uniwill_data *data, bool *status)
+{
+	unsigned int value;
+	int ret;
+
+	ret = regmap_read(data->regmap, EC_ADDR_TRIGGER, &value);
+	if (ret < 0)
+		return ret;
+
+	*status = !!(value & TRIGGER_USB_CHARGING);
+
+	return 0;
+}
+
+static ssize_t usb_powershare_high_show(struct device *dev, struct device_attribute *attr,
+					char *buf)
+{
+	struct uniwill_data *data = dev_get_drvdata(dev);
+	bool status;
+	int ret;
+
+	ret = uniwill_read_usb_powershare_high(data, &status);
+	if (ret < 0)
+		return ret;
+
+	return sysfs_emit(buf, "%d\n", status);
+}
+
+static DEVICE_ATTR_RW(usb_powershare_high);
+
 static struct attribute *uniwill_attrs[] = {
 	/* Keyboard-related */
 	&dev_attr_fn_lock.attr,
@@ -1088,6 +1259,8 @@ static struct attribute *uniwill_attrs[] = {
 	/* Power-management-related */
 	&dev_attr_ctgp_offset.attr,
 	&dev_attr_usb_c_power_priority.attr,
+	&dev_attr_ac_auto_boot.attr,
+	&dev_attr_usb_powershare_high.attr,
 	NULL
 };
 
@@ -1127,6 +1300,16 @@ static umode_t uniwill_attr_is_visible(struct kobject *kobj, struct attribute *a
 			return attr->mode;
 	}
 
+	if (attr == &dev_attr_ac_auto_boot.attr) {
+		if (uniwill_device_supports(data, UNIWILL_FEATURE_AC_AUTO_BOOT))
+			return attr->mode;
+	}
+
+	if (attr == &dev_attr_usb_powershare_high.attr) {
+		if (uniwill_device_supports(data, UNIWILL_FEATURE_USB_POWERSHARE))
+			return attr->mode;
+	}
+
 	return 0;
 }
 
@@ -1332,7 +1515,7 @@ static int uniwill_led_brightness_set(struct led_classdev *led_cdev, enum led_br
 
 	for (int i = 0; i < LED_CHANNELS; i++) {
 		/* Prevent the brightness values from overflowing */
-		value = min(LED_MAX_BRIGHTNESS, data->led_mc_subled_info[i].brightness);
+		value = min(data->lightbar_max_brightness, data->led_mc_subled_info[i].brightness);
 		ret = regmap_write(data->regmap, uniwill_led_channel_to_ac_reg[i], value);
 		if (ret < 0)
 			return ret;
@@ -1401,14 +1584,14 @@ static int uniwill_led_init(struct uniwill_data *data)
 		return ret;
 
 	data->led_mc_cdev.led_cdev.color = LED_COLOR_ID_MULTI;
-	data->led_mc_cdev.led_cdev.max_brightness = LED_MAX_BRIGHTNESS;
+	data->led_mc_cdev.led_cdev.max_brightness = data->lightbar_max_brightness;
 	data->led_mc_cdev.led_cdev.flags = LED_REJECT_NAME_CONFLICT;
 	data->led_mc_cdev.led_cdev.brightness_set_blocking = uniwill_led_brightness_set;
 
 	if (value & LIGHTBAR_S0_OFF)
 		data->led_mc_cdev.led_cdev.brightness = 0;
 	else
-		data->led_mc_cdev.led_cdev.brightness = LED_MAX_BRIGHTNESS;
+		data->led_mc_cdev.led_cdev.brightness = data->lightbar_max_brightness;
 
 	for (int i = 0; i < LED_CHANNELS; i++) {
 		data->led_mc_subled_info[i].color_index = color_indices[i];
@@ -1421,7 +1604,7 @@ static int uniwill_led_init(struct uniwill_data *data)
 		 * Make sure that the initial intensity value is not greater than
 		 * the maximum brightness.
 		 */
-		value = min(LED_MAX_BRIGHTNESS, value);
+		value = min(data->lightbar_max_brightness, value);
 		ret = regmap_write(data->regmap, uniwill_led_channel_to_ac_reg[i], value);
 		if (ret < 0)
 			return ret;
@@ -1441,6 +1624,246 @@ static int uniwill_led_init(struct uniwill_data *data)
 							 &init_data);
 }
 
+static int uniwill_notify_kbd_led(struct uniwill_data *data, int brightness)
+{
+	struct led_classdev *led_cdev;
+	int ret;
+
+	if (data->single_color_kbd)
+		led_cdev = &data->kbd_led_cdev;
+	else
+		led_cdev = &data->kbd_led_mc_cdev.led_cdev;
+
+	guard(mutex)(&led_cdev->led_access);
+
+	/* Sync the LED brightness with the actual hardware state */
+	ret = led_update_brightness(led_cdev);
+	if (ret < 0)
+		return ret;
+
+	led_classdev_notify_brightness_hw_changed(led_cdev, brightness);
+
+	return 0;
+}
+
+#define KBD_LED_MASK	(KBD_BRIGHTNESS_MASK | KBD_APPLY | KBD_POWER_OFF)
+
+static int uniwill_kbd_led_write_brightness(struct uniwill_data *data, int brightness)
+{
+	/* KBD_POWER_OFF is always implicitly cleared */
+	unsigned int regval = FIELD_PREP(KBD_BRIGHTNESS_MASK, brightness) | KBD_APPLY;
+
+	/* We must ensure that the "apply" bit is always written */
+	return regmap_write_bits(data->regmap, EC_ADDR_KBD_STATUS, KBD_LED_MASK, regval);
+}
+
+static int uniwill_kbd_led_read_brightness(struct uniwill_data *data)
+{
+	unsigned int regval;
+	int ret;
+
+	ret = regmap_read(data->regmap, EC_ADDR_KBD_STATUS, &regval);
+	if (ret < 0)
+		return ret;
+
+	return min(FIELD_GET(KBD_BRIGHTNESS_MASK, regval), data->kbd_led_max_brightness);
+}
+
+static int uniwill_kbd_led_brightness_set(struct led_classdev *led_cdev,
+					  enum led_brightness brightness)
+{
+	struct uniwill_data *data = container_of(led_cdev, struct uniwill_data, kbd_led_cdev);
+
+	return uniwill_kbd_led_write_brightness(data, brightness);
+}
+
+static enum led_brightness uniwill_kbd_led_brightness_get(struct led_classdev *led_cdev)
+{
+	struct uniwill_data *data = container_of(led_cdev, struct uniwill_data, kbd_led_cdev);
+
+	return uniwill_kbd_led_read_brightness(data);
+}
+
+static const unsigned int uniwill_kbd_led_channel_to_reg[KBD_LED_CHANNELS] = {
+	EC_ADDR_RGB_RED,
+	EC_ADDR_RGB_GREEN,
+	EC_ADDR_RGB_BLUE,
+};
+
+static int uniwill_kbd_led_mc_brightness_set(struct led_classdev *led_cdev,
+					     enum led_brightness brightness)
+{
+	struct led_classdev_mc *led_mc_cdev = lcdev_to_mccdev(led_cdev);
+	struct uniwill_data *data = container_of(led_mc_cdev, struct uniwill_data, kbd_led_mc_cdev);
+	unsigned int min_intensity = 0;
+	unsigned int regval;
+	int ret;
+
+	guard(mutex)(&data->kbd_rgb_led_lock);
+
+	/*
+	 * The EC interprets a RGB value of 0x000000 as a command to restore
+	 * the device-specfic default RGB value. Work around this by writing
+	 * a RGB value of 0x010101 (faint white) instead.
+	 */
+	if (data->kbd_led_mc_subled_info[0].intensity == 0 &&
+	    data->kbd_led_mc_subled_info[1].intensity == 0 &&
+	    data->kbd_led_mc_subled_info[2].intensity == 0)
+		min_intensity = 1;
+
+	for (int i = 0; i < KBD_LED_CHANNELS; i++) {
+		regval = max(data->kbd_led_mc_subled_info[i].intensity, min_intensity);
+		ret = regmap_write(data->regmap, uniwill_kbd_led_channel_to_reg[i], regval);
+		if (ret < 0)
+			return ret;
+	}
+
+	ret = regmap_write_bits(data->regmap, EC_ADDR_TRIGGER, RGB_APPLY_COLOR, RGB_APPLY_COLOR);
+	if (ret < 0)
+		return ret;
+
+	return uniwill_kbd_led_write_brightness(data, brightness);
+}
+
+static enum led_brightness uniwill_kbd_led_mc_brightness_get(struct led_classdev *led_cdev)
+{
+	struct led_classdev_mc *led_mc_cdev = lcdev_to_mccdev(led_cdev);
+	struct uniwill_data *data = container_of(led_mc_cdev, struct uniwill_data, kbd_led_mc_cdev);
+
+	return uniwill_kbd_led_read_brightness(data);
+}
+
+static int uniwill_kbd_led_init(struct uniwill_data *data)
+{
+	unsigned int color_indices[KBD_LED_CHANNELS] = {
+		LED_COLOR_ID_RED,
+		LED_COLOR_ID_GREEN,
+		LED_COLOR_ID_BLUE,
+	};
+	struct led_init_data init_data = {
+		.devicename = DRIVER_NAME,
+		.devname_mandatory = true,
+	};
+	bool intensity_all_zeros = true;
+	bool needs_trigger = false;
+	unsigned int regval;
+	int ret;
+
+	if (!uniwill_device_supports(data, UNIWILL_FEATURE_KEYBOARD_BACKLIGHT))
+		return 0;
+
+	ret = regmap_read(data->regmap, EC_ADDR_SUPPORT_2, &regval);
+	if (ret < 0)
+		return ret;
+
+	if (!(regval & CHINA_MODE)) {
+		ret = regmap_set_bits(data->regmap, EC_ADDR_BIOS_OEM_2, ENABLE_CHINA_MODE);
+		if (ret < 0)
+			return ret;
+	}
+
+	ret = regmap_read(data->regmap, EC_ADDR_KBD_STATUS, &regval);
+	if (ret < 0)
+		return ret;
+
+	regval |= KBD_APPLY;
+	regval &= ~KBD_POWER_OFF;
+	ret = regmap_write(data->regmap, EC_ADDR_KBD_STATUS, regval);
+	if (ret < 0)
+		return ret;
+
+	switch (data->project_id) {
+	case PROJECT_ID_PF:
+	case PROJECT_ID_PF4MU_PF4MN_PF5MU:
+	case PROJECT_ID_PH4TRX1:
+	case PROJECT_ID_PH4TUX1:
+	case PROJECT_ID_PH4TQX1:
+	case PROJECT_ID_PH6TRX1:
+	case PROJECT_ID_PH6TQXX:
+	case PROJECT_ID_PHXAXXX:
+	case PROJECT_ID_PHXPXXX:
+		data->single_color_kbd = true;
+		break;
+	default:
+		data->single_color_kbd = regval & KBD_WHITE_ONLY;
+		break;
+	}
+
+	if (data->single_color_kbd) {
+		init_data.default_label = "white:" LED_FUNCTION_KBD_BACKLIGHT;
+		data->kbd_led_cdev.max_brightness = data->kbd_led_max_brightness;
+		data->kbd_led_cdev.color = LED_COLOR_ID_WHITE;
+		data->kbd_led_cdev.flags = LED_BRIGHT_HW_CHANGED | LED_REJECT_NAME_CONFLICT;
+		data->kbd_led_cdev.brightness_set_blocking = uniwill_kbd_led_brightness_set;
+		data->kbd_led_cdev.brightness_get = uniwill_kbd_led_brightness_get;
+
+		return devm_led_classdev_register_ext(data->dev, &data->kbd_led_cdev, &init_data);
+	}
+
+	for (int i = 0; i < KBD_LED_CHANNELS; i++) {
+		data->kbd_led_mc_subled_info[i].color_index = color_indices[i];
+
+		ret = regmap_read(data->regmap, uniwill_kbd_led_channel_to_reg[i], &regval);
+		if (ret < 0)
+			return ret;
+
+		/*
+		 * Make sure that the initial intensity value is not greater than
+		 * the maximum intensity.
+		 */
+		if (regval > KBD_LED_MAX_INTENSITY) {
+			regval = KBD_LED_MAX_INTENSITY;
+			ret = regmap_write(data->regmap, uniwill_kbd_led_channel_to_reg[i], regval);
+			if (ret < 0)
+				return ret;
+
+			needs_trigger = true;
+		}
+
+		if (regval)
+			intensity_all_zeros = false;
+
+		data->kbd_led_mc_subled_info[i].intensity = regval;
+		data->kbd_led_mc_subled_info[i].max_intensity = KBD_LED_MAX_INTENSITY;
+		data->kbd_led_mc_subled_info[i].channel = i;
+	}
+
+	/* See uniwill_kbd_led_mc_brightness_set() for an explaination. */
+	if (intensity_all_zeros) {
+		for (int i = 0; i < KBD_LED_CHANNELS; i++) {
+			data->kbd_led_mc_subled_info[i].intensity = 1;
+			ret = regmap_write(data->regmap, uniwill_kbd_led_channel_to_reg[i], 1);
+			if (ret < 0)
+				return ret;
+		}
+
+		needs_trigger = true;
+	}
+
+	if (needs_trigger) {
+		ret = regmap_write_bits(data->regmap, EC_ADDR_TRIGGER, RGB_APPLY_COLOR,
+					RGB_APPLY_COLOR);
+		if (ret < 0)
+			return ret;
+	}
+
+	ret = devm_mutex_init(data->dev, &data->kbd_rgb_led_lock);
+	if (ret < 0)
+		return ret;
+
+	init_data.default_label = "multicolor:" LED_FUNCTION_KBD_BACKLIGHT;
+	data->kbd_led_mc_cdev.led_cdev.max_brightness = data->kbd_led_max_brightness;
+	data->kbd_led_mc_cdev.led_cdev.color = LED_COLOR_ID_MULTI;
+	data->kbd_led_mc_cdev.led_cdev.flags = LED_BRIGHT_HW_CHANGED | LED_REJECT_NAME_CONFLICT;
+	data->kbd_led_mc_cdev.led_cdev.brightness_set_blocking = uniwill_kbd_led_mc_brightness_set;
+	data->kbd_led_mc_cdev.led_cdev.brightness_get = uniwill_kbd_led_mc_brightness_get;
+	data->kbd_led_mc_cdev.subled_info = data->kbd_led_mc_subled_info;
+	data->kbd_led_mc_cdev.num_colors = KBD_LED_CHANNELS;
+
+	return devm_led_classdev_multicolor_register_ext(data->dev, &data->kbd_led_mc_cdev,
+							 &init_data);
+}
+
 static unsigned int uniwill_sanitize_battery_threshold(unsigned int value)
 {
 	/* 0 means "charging threshold not active" */
@@ -1789,6 +2212,31 @@ static int uniwill_notifier_call(struct notifier_block *nb, unsigned long action
 		sysfs_notify(&data->dev->kobj, NULL, "fn_lock");
 
 		return NOTIFY_OK;
+	case UNIWILL_OSD_KB_LED_LEVEL0:
+		if (!uniwill_device_supports(data, UNIWILL_FEATURE_KEYBOARD_BACKLIGHT))
+			return NOTIFY_DONE;
+
+		return notifier_from_errno(uniwill_notify_kbd_led(data, 0));
+	case UNIWILL_OSD_KB_LED_LEVEL1:
+		if (!uniwill_device_supports(data, UNIWILL_FEATURE_KEYBOARD_BACKLIGHT))
+			return NOTIFY_DONE;
+
+		return notifier_from_errno(uniwill_notify_kbd_led(data, 1));
+	case UNIWILL_OSD_KB_LED_LEVEL2:
+		if (!uniwill_device_supports(data, UNIWILL_FEATURE_KEYBOARD_BACKLIGHT))
+			return NOTIFY_DONE;
+
+		return notifier_from_errno(uniwill_notify_kbd_led(data, 2));
+	case UNIWILL_OSD_KB_LED_LEVEL3:
+		if (!uniwill_device_supports(data, UNIWILL_FEATURE_KEYBOARD_BACKLIGHT))
+			return NOTIFY_DONE;
+
+		return notifier_from_errno(uniwill_notify_kbd_led(data, 3));
+	case UNIWILL_OSD_KB_LED_LEVEL4:
+		if (!uniwill_device_supports(data, UNIWILL_FEATURE_KEYBOARD_BACKLIGHT))
+			return NOTIFY_DONE;
+
+		return notifier_from_errno(uniwill_notify_kbd_led(data, 4));
 	default:
 		mutex_lock(&data->input_lock);
 		sparse_keymap_report_event(data->input_device, action, 1, true);
@@ -1842,6 +2290,7 @@ static int uniwill_ec_init(struct uniwill_data *data)
 	if (ret < 0)
 		return ret;
 
+	data->project_id = value;
 	dev_dbg(data->dev, "Project ID: %u\n", value);
 
 	ret = regmap_set_bits(data->regmap, EC_ADDR_AP_OEM, ENABLE_MANUAL_CTRL);
@@ -1885,6 +2334,8 @@ static int uniwill_probe(struct platform_device *pdev)
 		return ret;
 
 	data->features = device_descriptor.features;
+	data->kbd_led_max_brightness = device_descriptor.kbd_led_max_brightness;
+	data->lightbar_max_brightness = device_descriptor.lightbar_max_brightness;
 
 	/*
 	 * Some devices might need to perform some device-specific initialization steps
@@ -1905,6 +2356,10 @@ static int uniwill_probe(struct platform_device *pdev)
 	if (ret < 0)
 		return ret;
 
+	ret = uniwill_kbd_led_init(data);
+	if (ret < 0)
+		return ret;
+
 	ret = uniwill_hwmon_init(data);
 	if (ret < 0)
 		return ret;
@@ -1976,6 +2431,43 @@ static int uniwill_suspend_battery(struct uniwill_data *data)
 	return regmap_read(data->regmap, EC_ADDR_CHARGE_CTRL, &data->last_charge_ctrl);
 }
 
+static int uniwill_suspend_kbd_led(struct uniwill_data *data)
+{
+	unsigned int regval;
+	int ret;
+
+	if (!uniwill_device_supports(data, UNIWILL_FEATURE_KEYBOARD_BACKLIGHT))
+		return 0;
+
+	ret = regmap_read(data->regmap, EC_ADDR_KBD_STATUS, &regval);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * Save the current keyboard backlight settings in order to restore them
+	 * during resume. We cannot use the regmap code for that since this register
+	 * needs to be declared as volatile because the brightness can be changed
+	 * by the EC.
+	 */
+	data->last_kbd_status = regval;
+	FIELD_MODIFY(KBD_BRIGHTNESS_MASK, &regval, 0);
+	regval |= KBD_APPLY | KBD_POWER_OFF;
+
+	return regmap_write(data->regmap, EC_ADDR_KBD_STATUS, regval);
+}
+
+static int uniwill_suspend_usb_powershare(struct uniwill_data *data)
+{
+	if (!uniwill_device_supports(data, UNIWILL_FEATURE_USB_POWERSHARE))
+		return 0;
+
+	/*
+	 * EC_ADDR_TRIGGER is marked as volatile, so we have to restore it
+	 * ourselves.
+	 */
+	return uniwill_read_usb_powershare_high(data, &data->last_usb_powershare_high_state);
+}
+
 static int uniwill_suspend_nvidia_ctgp(struct uniwill_data *data)
 {
 	if (!uniwill_device_supports(data, UNIWILL_FEATURE_NVIDIA_CTGP_CONTROL))
@@ -2006,6 +2498,14 @@ static int uniwill_suspend(struct device *dev)
 	if (ret < 0)
 		return ret;
 
+	ret = uniwill_suspend_kbd_led(data);
+	if (ret < 0)
+		return ret;
+
+	ret = uniwill_suspend_usb_powershare(data);
+	if (ret < 0)
+		return ret;
+
 	ret = uniwill_suspend_nvidia_ctgp(data);
 	if (ret < 0)
 		return ret;
@@ -2052,6 +2552,31 @@ static int uniwill_resume_battery(struct uniwill_data *data)
 	return 0;
 }
 
+static int uniwill_resume_kbd_led(struct uniwill_data *data)
+{
+	int ret;
+
+	if (!uniwill_device_supports(data, UNIWILL_FEATURE_KEYBOARD_BACKLIGHT))
+		return 0;
+
+	ret = regmap_write(data->regmap, EC_ADDR_KBD_STATUS, data->last_kbd_status | KBD_APPLY);
+	if (ret < 0)
+		return ret;
+
+	if (data->single_color_kbd)
+		return 0;
+
+	return regmap_write_bits(data->regmap, EC_ADDR_TRIGGER, RGB_APPLY_COLOR, RGB_APPLY_COLOR);
+}
+
+static int uniwill_resume_usb_powershare(struct uniwill_data *data)
+{
+	if (!uniwill_device_supports(data, UNIWILL_FEATURE_USB_POWERSHARE))
+		return 0;
+
+	return uniwill_write_usb_powershare_high(data, data->last_usb_powershare_high_state);
+}
+
 static int uniwill_resume_nvidia_ctgp(struct uniwill_data *data)
 {
 	if (!uniwill_device_supports(data, UNIWILL_FEATURE_NVIDIA_CTGP_CONTROL))
@@ -2096,6 +2621,14 @@ static int uniwill_resume(struct device *dev)
 	if (ret < 0)
 		return ret;
 
+	ret = uniwill_resume_kbd_led(data);
+	if (ret < 0)
+		return ret;
+
+	ret = uniwill_resume_usb_powershare(data);
+	if (ret < 0)
+		return ret;
+
 	ret = uniwill_resume_nvidia_ctgp(data);
 	if (ret < 0)
 		return ret;
@@ -2126,13 +2659,29 @@ static struct platform_driver uniwill_driver = {
 	.shutdown = uniwill_shutdown,
 };
 
+static struct uniwill_device_descriptor machenike_l16p_descriptor __initdata = {
+	.features = UNIWILL_FEATURE_FN_LOCK |
+		    UNIWILL_FEATURE_SUPER_KEY |
+		    UNIWILL_FEATURE_CPU_TEMP |
+		    UNIWILL_FEATURE_GPU_TEMP |
+		    UNIWILL_FEATURE_PRIMARY_FAN |
+		    UNIWILL_FEATURE_SECONDARY_FAN |
+		    UNIWILL_FEATURE_NVIDIA_CTGP_CONTROL |
+		    UNIWILL_FEATURE_KEYBOARD_BACKLIGHT |
+		    UNIWILL_FEATURE_AC_AUTO_BOOT |
+		    UNIWILL_FEATURE_USB_POWERSHARE,
+	.kbd_led_max_brightness = 4,
+};
+
 static struct uniwill_device_descriptor lapqc71a_lapqc71b_descriptor __initdata = {
 	.features = UNIWILL_FEATURE_SUPER_KEY |
+		    UNIWILL_FEATURE_LIGHTBAR |
 		    UNIWILL_FEATURE_BATTERY_CHARGE_LIMIT |
 		    UNIWILL_FEATURE_CPU_TEMP |
 		    UNIWILL_FEATURE_GPU_TEMP |
 		    UNIWILL_FEATURE_PRIMARY_FAN |
 		    UNIWILL_FEATURE_SECONDARY_FAN,
+	.lightbar_max_brightness = 36,
 };
 
 static struct uniwill_device_descriptor lapac71h_descriptor __initdata = {
@@ -2156,6 +2705,7 @@ static struct uniwill_device_descriptor lapkc71f_descriptor __initdata = {
 		    UNIWILL_FEATURE_GPU_TEMP |
 		    UNIWILL_FEATURE_PRIMARY_FAN |
 		    UNIWILL_FEATURE_SECONDARY_FAN,
+	.lightbar_max_brightness = 200,
 };
 
 /*
@@ -2277,7 +2827,36 @@ static struct uniwill_device_descriptor pf5pu1g_descriptor __initdata = {
 		    UNIWILL_FEATURE_PRIMARY_FAN,
 };
 
+static struct uniwill_device_descriptor x4sp4nal_descriptor __initdata = {
+	.features = UNIWILL_FEATURE_FN_LOCK |
+		    UNIWILL_FEATURE_SUPER_KEY |
+		    UNIWILL_FEATURE_BATTERY_CHARGE_MODES |
+		    UNIWILL_FEATURE_CPU_TEMP |
+		    UNIWILL_FEATURE_PRIMARY_FAN |
+		    UNIWILL_FEATURE_SECONDARY_FAN |
+		    UNIWILL_FEATURE_KEYBOARD_BACKLIGHT |
+		    UNIWILL_FEATURE_AC_AUTO_BOOT |
+		    UNIWILL_FEATURE_USB_POWERSHARE,
+	.kbd_led_max_brightness = 2,
+};
+
 static const struct dmi_system_id uniwill_dmi_table[] __initconst = {
+	{
+		.ident = "AiStone X4SP4NAL",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "AiStone"),
+			DMI_EXACT_MATCH(DMI_BOARD_NAME, "X4SP4NAL"),
+		},
+		.driver_data = &x4sp4nal_descriptor,
+	},
+	{
+		.ident = "MACHENIKE L16 Pro",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "MACHENIKE"),
+			DMI_EXACT_MATCH(DMI_BOARD_NAME, "L16P"),
+		},
+		.driver_data = &machenike_l16p_descriptor,
+	},
 	{
 		.ident = "XMG FUSION 15 (L19)",
 		.matches = {
@@ -2745,6 +3324,10 @@ static int __init uniwill_init(void)
 	if (force) {
 		/* Assume that the device supports all features except the charge limit */
 		device_descriptor.features = UINT_MAX & ~UNIWILL_FEATURE_BATTERY_CHARGE_LIMIT;
+		/* Some models only support 3 brightness levels */
+		device_descriptor.kbd_led_max_brightness = 4;
+		/* Some models only support 36 brightness levels per color component */
+		device_descriptor.lightbar_max_brightness = 200;
 		pr_warn("Enabling potentially unsupported features\n");
 	}
 
diff --git a/drivers/platform/x86/uniwill/uniwill-wmi.h b/drivers/platform/x86/uniwill/uniwill-wmi.h
index fb1910c0f7418..b25b2f31211cf 100644
--- a/drivers/platform/x86/uniwill/uniwill-wmi.h
+++ b/drivers/platform/x86/uniwill/uniwill-wmi.h
@@ -113,6 +113,8 @@
 
 #define UNIWILL_OSD_BENCHMARK_MODE_TOGGLE	0xC0
 
+#define UNIWILL_OSD_SCREEN_STATE_CHANGED	0xCC
+
 #define UNIWILL_OSD_WEBCAM_TOGGLE		0xCF
 
 #define UNIWILL_OSD_KBD_BACKLIGHT_CHANGED	0xF0
diff --git a/drivers/platform/x86/xo15-ebook.c b/drivers/platform/x86/xo15-ebook.c
index e40e385c52bd4..4c3185e2ceec2 100644
--- a/drivers/platform/x86/xo15-ebook.c
+++ b/drivers/platform/x86/xo15-ebook.c
@@ -19,13 +19,10 @@
 
 #define MODULE_NAME "xo15-ebook"
 
-#define XO15_EBOOK_CLASS		MODULE_NAME
 #define XO15_EBOOK_TYPE_UNKNOWN	0x00
 #define XO15_EBOOK_NOTIFY_STATUS	0x80
 
-#define XO15_EBOOK_SUBCLASS		"ebook"
 #define XO15_EBOOK_HID			"XO15EBK"
-#define XO15_EBOOK_DEVICE_NAME		"EBook Switch"
 
 MODULE_DESCRIPTION("OLPC XO-1.5 ebook switch driver");
 MODULE_LICENSE("GPL");
@@ -105,12 +102,9 @@ static int ebook_switch_probe(struct platform_device *pdev)
 	if (!id)
 		return dev_err_probe(dev, -ENODEV, "Unsupported hid\n");
 
-	strscpy(acpi_device_name(device), XO15_EBOOK_DEVICE_NAME);
-	strscpy(acpi_device_class(device), XO15_EBOOK_CLASS "/" XO15_EBOOK_SUBCLASS);
-
 	snprintf(button->phys, sizeof(button->phys), "%s/button/input0", id->id);
 
-	input->name = acpi_device_name(device);
+	input->name = "EBook Switch";
 	input->phys = button->phys;
 	input->id.bustype = BUS_HOST;
 
diff --git a/drivers/pmdomain/arm/Kconfig b/drivers/pmdomain/arm/Kconfig
index afed10d382ad7..ba0b190a63d6b 100644
--- a/drivers/pmdomain/arm/Kconfig
+++ b/drivers/pmdomain/arm/Kconfig
@@ -21,7 +21,7 @@ config ARM_SCMI_POWER_DOMAIN
 	  enabled or disabled via the SCP firmware
 
 	  This driver can also be built as a module. If so, the module
-	  will be called scmi_pm_domain. Note this may needed early in boot
+	  will be called scmi_pm_domain. Note this may be needed early in boot
 	  before rootfs may be available.
 
 config ARM_SCPI_POWER_DOMAIN
diff --git a/drivers/pmdomain/bcm/bcm2835-power.c b/drivers/pmdomain/bcm/bcm2835-power.c
index b76d74e3849be..fad3113093d53 100644
--- a/drivers/pmdomain/bcm/bcm2835-power.c
+++ b/drivers/pmdomain/bcm/bcm2835-power.c
@@ -175,7 +175,7 @@ static int bcm2835_asb_control(struct bcm2835_power *power, u32 reg, bool enable
 	writel(PM_PASSWORD | val, base + reg);
 
 	if (readl_poll_timeout_atomic(base + reg, val,
-				      !!(val & ASB_ACK) != enable, 0, 5))
+				      !!(val & ASB_ACK) != enable, 0, 100))
 		return -ETIMEDOUT;
 
 	return 0;
@@ -677,7 +677,12 @@ static int bcm2835_power_probe(struct platform_device *pdev)
 	if (ret)
 		goto fail;
 
-	of_genpd_add_provider_onecell(dev->parent->of_node, &power->pd_xlate);
+	ret = of_genpd_add_provider_onecell(dev->parent->of_node,
+					    &power->pd_xlate);
+	if (ret) {
+		dev_err_probe(dev, ret, "failed to add genpd provider\n");
+		goto fail;
+	}
 
 	dev_info(dev, "Broadcom BCM2835 power domains driver");
 	return 0;
diff --git a/drivers/pmdomain/imx/Kconfig b/drivers/pmdomain/imx/Kconfig
index 00203615c65e9..9168d183b0c5d 100644
--- a/drivers/pmdomain/imx/Kconfig
+++ b/drivers/pmdomain/imx/Kconfig
@@ -10,15 +10,18 @@ config IMX_GPCV2_PM_DOMAINS
 	default y if SOC_IMX7D
 
 config IMX8M_BLK_CTRL
-	bool
-	default SOC_IMX8M && IMX_GPCV2_PM_DOMAINS
+	tristate "i.MX8M BLK CTRL driver"
+	depends on SOC_IMX8M
+	depends on IMX_GPCV2_PM_DOMAINS
 	depends on PM_GENERIC_DOMAINS
 	depends on COMMON_CLK
+	default y
 
 config IMX9_BLK_CTRL
-	bool
-	default SOC_IMX9 && IMX_GPCV2_PM_DOMAINS
+	tristate "i.MX93 BLK CTRL driver"
+	depends on SOC_IMX9
 	depends on PM_GENERIC_DOMAINS
+	default y
 
 config IMX_SCU_PD
 	bool "IMX SCU Power Domain driver"
diff --git a/drivers/pmdomain/imx/gpc.c b/drivers/pmdomain/imx/gpc.c
index 42e50c9b4fb9f..abca5f449a226 100644
--- a/drivers/pmdomain/imx/gpc.c
+++ b/drivers/pmdomain/imx/gpc.c
@@ -487,8 +487,7 @@ static int imx_gpc_probe(struct platform_device *pdev)
 			domain->ipg_rate_mhz = ipg_rate_mhz;
 
 			pd_pdev->dev.parent = &pdev->dev;
-			pd_pdev->dev.of_node = of_node_get(np);
-			pd_pdev->dev.fwnode = of_fwnode_handle(np);
+			platform_device_set_of_node(pd_pdev, np);
 
 			ret = platform_device_add(pd_pdev);
 			if (ret) {
diff --git a/drivers/pmdomain/imx/imx8m-blk-ctrl.c b/drivers/pmdomain/imx/imx8m-blk-ctrl.c
index 19e992d2ee3b8..99d100e1d923f 100644
--- a/drivers/pmdomain/imx/imx8m-blk-ctrl.c
+++ b/drivers/pmdomain/imx/imx8m-blk-ctrl.c
@@ -54,6 +54,15 @@ struct imx8m_blk_ctrl_domain_data {
 	 * register.
 	 */
 	u32 mipi_phy_rst_mask;
+
+	/*
+	 * VC8000E reset de-assertion edge and AXI clock may have a timing issue.
+	 * Workaround: Set bit2 (vc8000e_clk_en) of BLK_CLK_EN_CSR to 0 to gate off
+	 * both AXI clock and VC8000E clock sent to VC8000E and AXI clock sent to
+	 * VPU_NOC m_v_2 interface during VC8000E power up(VC8000E reset is
+	 * de-asserted by HW)
+	 */
+	bool is_errata_err050531;
 };
 
 #define DOMAIN_MAX_CLKS 4
@@ -108,7 +117,11 @@ static int imx8m_blk_ctrl_power_on(struct generic_pm_domain *genpd)
 		dev_err(bc->dev, "failed to enable clocks\n");
 		goto bus_put;
 	}
-	regmap_set_bits(bc->regmap, BLK_CLK_EN, data->clk_mask);
+
+	if (data->is_errata_err050531)
+		regmap_clear_bits(bc->regmap, BLK_CLK_EN, data->clk_mask);
+	else
+		regmap_set_bits(bc->regmap, BLK_CLK_EN, data->clk_mask);
 
 	/* power up upstream GPC domain */
 	ret = pm_runtime_get_sync(domain->power_dev);
@@ -117,6 +130,9 @@ static int imx8m_blk_ctrl_power_on(struct generic_pm_domain *genpd)
 		goto clk_disable;
 	}
 
+	if (data->is_errata_err050531)
+		regmap_set_bits(bc->regmap, BLK_CLK_EN, data->clk_mask);
+
 	/* wait for reset to propagate */
 	udelay(5);
 
@@ -511,12 +527,38 @@ static const struct imx8m_blk_ctrl_domain_data imx8mp_vpu_blk_ctl_domain_data[]
 		.clk_mask = BIT(2),
 		.path_names = (const char *[]){"vc8000e"},
 		.num_paths = 1,
+		.is_errata_err050531 = true,
 	},
 };
 
+static int imx8mp_vpu_power_notifier(struct notifier_block *nb,
+				     unsigned long action, void *data)
+{
+	struct imx8m_blk_ctrl *bc = container_of(nb, struct imx8m_blk_ctrl,
+						 power_nb);
+
+	if (action == GENPD_NOTIFY_ON) {
+		/*
+		 * On power up we have no software backchannel to the GPC to
+		 * wait for the ADB handshake to happen, so we just delay for a
+		 * bit. On power down the GPC driver waits for the handshake.
+		 */
+
+		udelay(5);
+
+		/* set "fuse" bits to enable the VPUs */
+		regmap_set_bits(bc->regmap, 0x8, 0xffffffff);
+		regmap_set_bits(bc->regmap, 0xc, 0xffffffff);
+		regmap_set_bits(bc->regmap, 0x10, 0xffffffff);
+		regmap_set_bits(bc->regmap, 0x14, 0xffffffff);
+	}
+
+	return NOTIFY_OK;
+}
+
 static const struct imx8m_blk_ctrl_data imx8mp_vpu_blk_ctl_dev_data = {
 	.max_reg = 0x18,
-	.power_notifier_fn = imx8mm_vpu_power_notifier,
+	.power_notifier_fn = imx8mp_vpu_power_notifier,
 	.domains = imx8mp_vpu_blk_ctl_domain_data,
 	.num_domains = ARRAY_SIZE(imx8mp_vpu_blk_ctl_domain_data),
 };
diff --git a/drivers/pmdomain/imx/imx93-blk-ctrl.c b/drivers/pmdomain/imx/imx93-blk-ctrl.c
index 1afc78b034fad..243ce939ba680 100644
--- a/drivers/pmdomain/imx/imx93-blk-ctrl.c
+++ b/drivers/pmdomain/imx/imx93-blk-ctrl.c
@@ -48,6 +48,8 @@
 
 #define PRIO(X)			(X)
 
+#define BLK_CTRL_NO_PARENT	UINT_MAX
+
 struct imx93_blk_ctrl_domain;
 
 struct imx93_blk_ctrl {
@@ -68,12 +70,18 @@ struct imx93_blk_ctrl_qos {
 	u32 cfg_prio;
 };
 
+struct imx93_blk_ctrl_subdomain_link {
+	struct generic_pm_domain *parent;
+	struct generic_pm_domain *subdomain;
+};
+
 struct imx93_blk_ctrl_domain_data {
 	const char *name;
 	const char * const *clk_names;
 	int num_clks;
 	u32 rst_mask;
 	u32 clk_mask;
+	u32 parent;
 	int num_qos;
 	struct imx93_blk_ctrl_qos qos[DOMAIN_MAX_QOS];
 };
@@ -203,6 +211,13 @@ static void imx93_release_pm_genpd(void *data)
 	pm_genpd_remove(genpd);
 }
 
+static void imx93_release_subdomain(void *data)
+{
+	struct imx93_blk_ctrl_subdomain_link *link = data;
+
+	pm_genpd_remove_subdomain(link->parent, link->subdomain);
+}
+
 static struct lock_class_key blk_ctrl_genpd_lock_class;
 
 static int imx93_blk_ctrl_probe(struct platform_device *pdev)
@@ -302,6 +317,34 @@ static int imx93_blk_ctrl_probe(struct platform_device *pdev)
 		bc->onecell_data.domains[i] = &domain->genpd;
 	}
 
+	for (i = 0; i < bc_data->num_domains; i++) {
+		struct imx93_blk_ctrl_domain *domain = &bc->domains[i];
+		const struct imx93_blk_ctrl_domain_data *data = domain->data;
+		struct imx93_blk_ctrl_subdomain_link *link;
+
+		if (bc_data->skip_mask & BIT(i) ||
+		    data->parent == BLK_CTRL_NO_PARENT)
+			continue;
+
+		link = devm_kzalloc(dev, sizeof(*link), GFP_KERNEL);
+		if (!link)
+			return -ENOMEM;
+
+		link->parent = &bc->domains[data->parent].genpd;
+		link->subdomain = &domain->genpd;
+
+		ret = pm_genpd_add_subdomain(&bc->domains[data->parent].genpd,
+					     &domain->genpd);
+		if (ret)
+			return dev_err_probe(dev, ret, "failed to add subdomain %s\n",
+					     domain->genpd.name);
+
+		ret = devm_add_action_or_reset(dev, imx93_release_subdomain, link);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "failed to add subdomain release callback\n");
+	}
+
 	ret = devm_pm_runtime_enable(dev);
 	if (ret)
 		return dev_err_probe(dev, ret, "failed to enable pm-runtime\n");
@@ -326,8 +369,9 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[]
 		.name = "mediablk-mipi-dsi",
 		.clk_names = (const char *[]){ "dsi" },
 		.num_clks = 1,
-		.rst_mask = BIT(11) | BIT(12),
-		.clk_mask = BIT(11) | BIT(12),
+		.rst_mask = BIT(11),
+		.clk_mask = BIT(11),
+		.parent = IMX93_MEDIABLK_PD_MIPI_PHY,
 	},
 	[IMX93_MEDIABLK_PD_MIPI_CSI] = {
 		.name = "mediablk-mipi-csi",
@@ -335,6 +379,7 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[]
 		.num_clks = 2,
 		.rst_mask = BIT(9) | BIT(10),
 		.clk_mask = BIT(9) | BIT(10),
+		.parent = IMX93_MEDIABLK_PD_MIPI_PHY,
 	},
 	[IMX93_MEDIABLK_PD_PXP] = {
 		.name = "mediablk-pxp",
@@ -342,6 +387,7 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[]
 		.num_clks = 1,
 		.rst_mask = BIT(7) | BIT(8),
 		.clk_mask = BIT(7) | BIT(8),
+		.parent = BLK_CTRL_NO_PARENT,
 		.num_qos = 2,
 		.qos = {
 			{
@@ -363,6 +409,7 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[]
 		.num_clks = 2,
 		.rst_mask = BIT(4) | BIT(5) | BIT(6),
 		.clk_mask = BIT(4) | BIT(5) | BIT(6),
+		.parent = BLK_CTRL_NO_PARENT,
 		.num_qos = 1,
 		.qos = {
 			{
@@ -379,6 +426,7 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[]
 		.num_clks = 1,
 		.rst_mask = BIT(2) | BIT(3),
 		.clk_mask = BIT(2) | BIT(3),
+		.parent = BLK_CTRL_NO_PARENT,
 		.num_qos = 4,
 		.qos = {
 			{
@@ -404,6 +452,14 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[]
 			}
 		}
 	},
+	[IMX93_MEDIABLK_PD_MIPI_PHY] = {
+		.name = "mediablk-mipi-phy",
+		.clk_names = NULL,
+		.num_clks = 0,
+		.rst_mask = BIT(12),
+		.clk_mask = BIT(12),
+		.parent = BLK_CTRL_NO_PARENT,
+	},
 };
 
 static const struct regmap_range imx93_media_blk_ctl_yes_ranges[] = {
diff --git a/drivers/pmdomain/mediatek/mt8196-pm-domains.h b/drivers/pmdomain/mediatek/mt8196-pm-domains.h
index 2e4b287206596..d704c9fa93374 100644
--- a/drivers/pmdomain/mediatek/mt8196-pm-domains.h
+++ b/drivers/pmdomain/mediatek/mt8196-pm-domains.h
@@ -602,6 +602,27 @@ static const struct scpsys_hwv_domain_data hfrpsys_hwv_domain_data_mt8196[] = {
 	},
 };
 
+static const struct scpsys_domain_data hfrpsys_domain_data_mt8196[] = {
+	[MT8196_POWER_DOMAIN_EDPTX] = {
+		.name = "edp-tx",
+		.sta_mask = MT8196_PWR_ACK,
+		.sta2nd_mask = MT8196_PWR_ACK_2ND,
+		.ctl_offs = 0x74,
+		.pwr_sta_offs = 0x74,
+		.pwr_sta2nd_offs = 0x74,
+		.caps = MTK_SCPD_SIMPLE_PWRSEQ,
+	},
+	[MT8196_POWER_DOMAIN_DPTX] = {
+		.name = "dp-tx",
+		.sta_mask = MT8196_PWR_ACK,
+		.sta2nd_mask = MT8196_PWR_ACK_2ND,
+		.ctl_offs = 0x78,
+		.pwr_sta_offs = 0x78,
+		.pwr_sta2nd_offs = 0x78,
+		.caps = MTK_SCPD_SIMPLE_PWRSEQ,
+	},
+};
+
 static const struct scpsys_soc_data mt8196_scpsys_data = {
 	.domains_data = scpsys_domain_data_mt8196,
 	.num_domains = ARRAY_SIZE(scpsys_domain_data_mt8196),
@@ -616,6 +637,12 @@ static const struct scpsys_soc_data mt8196_scpsys_hwv_data = {
 	.type = SCPSYS_MTCMOS_TYPE_HW_VOTER,
 };
 
+static const struct scpsys_soc_data mt8196_hfrpsys_data = {
+	.domains_data = hfrpsys_domain_data_mt8196,
+	.num_domains = ARRAY_SIZE(hfrpsys_domain_data_mt8196),
+	.type = SCPSYS_MTCMOS_TYPE_DIRECT_CTL,
+};
+
 static const struct scpsys_soc_data mt8196_hfrpsys_hwv_data = {
 	.hwv_domains_data = hfrpsys_hwv_domain_data_mt8196,
 	.num_hwv_domains = ARRAY_SIZE(hfrpsys_hwv_domain_data_mt8196),
diff --git a/drivers/pmdomain/mediatek/mtk-pm-domains.c b/drivers/pmdomain/mediatek/mtk-pm-domains.c
index e1cfd42234734..8309a4b46afb0 100644
--- a/drivers/pmdomain/mediatek/mtk-pm-domains.c
+++ b/drivers/pmdomain/mediatek/mtk-pm-domains.c
@@ -393,9 +393,8 @@ static int scpsys_hwv_power_on(struct generic_pm_domain *genpd)
 	return ret;
 };
 
-static int scpsys_hwv_power_off(struct generic_pm_domain *genpd)
+static int scpsys_hwv_power_off_internal(struct scpsys_domain *pd)
 {
-	struct scpsys_domain *pd = container_of(genpd, struct scpsys_domain, genpd);
 	const struct scpsys_hwv_domain_data *hwv = pd->hwv_data;
 	struct scpsys *scpsys = pd->scpsys;
 	u32 val;
@@ -464,6 +463,13 @@ static int scpsys_hwv_power_off(struct generic_pm_domain *genpd)
 	return ret;
 };
 
+static int scpsys_hwv_power_off(struct generic_pm_domain *genpd)
+{
+	struct scpsys_domain *pd = container_of(genpd, struct scpsys_domain, genpd);
+
+	return scpsys_hwv_power_off_internal(pd);
+}
+
 static int scpsys_ctl_pwrseq_on(struct scpsys_domain *pd)
 {
 	struct scpsys *scpsys = pd->scpsys;
@@ -549,9 +555,11 @@ static int scpsys_ctl_pwrseq_on(struct scpsys_domain *pd)
 	return 0;
 }
 
-static void scpsys_ctl_pwrseq_off(struct scpsys_domain *pd)
+static int scpsys_ctl_pwrseq_off(struct scpsys_domain *pd)
 {
 	struct scpsys *scpsys = pd->scpsys;
+	bool tmp;
+	int ret;
 
 	switch (pd->data->rtff_type) {
 	case SCPSYS_RTFF_TYPE_GENERIC:
@@ -583,6 +591,41 @@ static void scpsys_ctl_pwrseq_off(struct scpsys_domain *pd)
 	regmap_clear_bits(scpsys->base, pd->data->ctl_offs, PWR_RST_B_BIT);
 	regmap_clear_bits(scpsys->base, pd->data->ctl_offs, PWR_ON_2ND_BIT);
 	regmap_clear_bits(scpsys->base, pd->data->ctl_offs, PWR_ON_BIT);
+
+	/* wait until PWR_ACK = 0 */
+	ret = readx_poll_timeout(scpsys_domain_is_on, pd, tmp, !tmp, MTK_POLL_DELAY_US,
+				 MTK_POLL_TIMEOUT);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int scpsys_simple_pwrseq_on(struct scpsys_domain *pd)
+{
+	struct scpsys *scpsys = pd->scpsys;
+
+	/* Enable subsys clock input and trigger power domain reset state */
+	regmap_clear_bits(scpsys->base, pd->data->ctl_offs, PWR_CLK_DIS_BIT);
+	regmap_clear_bits(scpsys->base, pd->data->ctl_offs, PWR_RST_B_BIT);
+
+	/* Wait for the hardware to stabilize */
+	udelay(1);
+
+	/* Get out of reset: set power on */
+	regmap_set_bits(scpsys->base, pd->data->ctl_offs, PWR_RST_B_BIT);
+
+	return 0;
+}
+
+static int scpsys_simple_pwrseq_off(struct scpsys_domain *pd)
+{
+	struct scpsys *scpsys = pd->scpsys;
+
+	regmap_clear_bits(scpsys->base, pd->data->ctl_offs, PWR_RST_B_BIT);
+	regmap_set_bits(scpsys->base, pd->data->ctl_offs, PWR_CLK_DIS_BIT);
+
+	return 0;
 }
 
 static int scpsys_modem_pwrseq_on(struct scpsys_domain *pd)
@@ -605,14 +648,24 @@ static int scpsys_modem_pwrseq_on(struct scpsys_domain *pd)
 	return 0;
 }
 
-static void scpsys_modem_pwrseq_off(struct scpsys_domain *pd)
+static int scpsys_modem_pwrseq_off(struct scpsys_domain *pd)
 {
 	struct scpsys *scpsys = pd->scpsys;
+	bool tmp;
+	int ret;
 
 	regmap_clear_bits(scpsys->base, pd->data->ctl_offs, PWR_ON_BIT);
 
 	if (!MTK_SCPD_CAPS(pd, MTK_SCPD_SKIP_RESET_B))
 		regmap_clear_bits(scpsys->base, pd->data->ctl_offs, PWR_RST_B_BIT);
+
+	/* wait until PWR_ACK = 0 */
+	ret = readx_poll_timeout(scpsys_domain_is_on, pd, tmp, !tmp, MTK_POLL_DELAY_US,
+				 MTK_POLL_TIMEOUT);
+	if (ret < 0)
+		return ret;
+
+	return 0;
 }
 
 static int scpsys_power_on(struct generic_pm_domain *genpd)
@@ -635,6 +688,8 @@ static int scpsys_power_on(struct generic_pm_domain *genpd)
 
 	if (MTK_SCPD_CAPS(pd, MTK_SCPD_MODEM_PWRSEQ))
 		ret = scpsys_modem_pwrseq_on(pd);
+	else if (MTK_SCPD_CAPS(pd, MTK_SCPD_SIMPLE_PWRSEQ))
+		ret = scpsys_simple_pwrseq_on(pd);
 	else
 		ret = scpsys_ctl_pwrseq_on(pd);
 
@@ -662,9 +717,11 @@ static int scpsys_power_on(struct generic_pm_domain *genpd)
 			goto err_pwr_ack;
 	}
 
-	ret = scpsys_sram_enable(pd);
-	if (ret < 0)
-		goto err_disable_subsys_clks;
+	if (!MTK_SCPD_CAPS(pd, MTK_SCPD_SIMPLE_PWRSEQ)) {
+		ret = scpsys_sram_enable(pd);
+		if (ret < 0)
+			goto err_disable_subsys_clks;
+	}
 
 	ret = scpsys_bus_protect_disable(pd, 0);
 	if (ret < 0)
@@ -682,7 +739,8 @@ static int scpsys_power_on(struct generic_pm_domain *genpd)
 err_enable_bus_protect:
 	scpsys_bus_protect_enable(pd, 0);
 err_disable_sram:
-	scpsys_sram_disable(pd);
+	if (!MTK_SCPD_CAPS(pd, MTK_SCPD_SIMPLE_PWRSEQ))
+		scpsys_sram_disable(pd);
 err_disable_subsys_clks:
 	if (!MTK_SCPD_CAPS(pd, MTK_SCPD_STRICT_BUS_PROTECTION))
 		clk_bulk_disable_unprepare(pd->num_subsys_clks,
@@ -694,20 +752,20 @@ static int scpsys_power_on(struct generic_pm_domain *genpd)
 	return ret;
 }
 
-static int scpsys_power_off(struct generic_pm_domain *genpd)
+static int scpsys_power_off_internal(struct scpsys_domain *pd)
 {
-	struct scpsys_domain *pd = container_of(genpd, struct scpsys_domain, genpd);
 	struct scpsys *scpsys = pd->scpsys;
-	bool tmp;
 	int ret;
 
 	ret = scpsys_bus_protect_enable(pd, 0);
 	if (ret < 0)
 		return ret;
 
-	ret = scpsys_sram_disable(pd);
-	if (ret < 0)
-		return ret;
+	if (!MTK_SCPD_CAPS(pd, MTK_SCPD_SIMPLE_PWRSEQ)) {
+		ret = scpsys_sram_disable(pd);
+		if (ret < 0)
+			return ret;
+	}
 
 	if (pd->data->ext_buck_iso_offs && MTK_SCPD_CAPS(pd, MTK_SCPD_EXT_BUCK_ISO))
 		regmap_set_bits(scpsys->base, pd->data->ext_buck_iso_offs,
@@ -720,15 +778,21 @@ static int scpsys_power_off(struct generic_pm_domain *genpd)
 		return ret;
 
 	if (MTK_SCPD_CAPS(pd, MTK_SCPD_MODEM_PWRSEQ))
-		scpsys_modem_pwrseq_off(pd);
+		ret = scpsys_modem_pwrseq_off(pd);
+	else if (MTK_SCPD_CAPS(pd, MTK_SCPD_SIMPLE_PWRSEQ))
+		ret = scpsys_simple_pwrseq_off(pd);
 	else
-		scpsys_ctl_pwrseq_off(pd);
+		ret = scpsys_ctl_pwrseq_off(pd);
+
+	if (ret < 0) {
+		/* Re-enable clocks so that next power off doesn't break the refcount */
+		int r = clk_bulk_prepare_enable(pd->num_subsys_clks, pd->subsys_clks);
+
+		if (r)
+			dev_warn(scpsys->dev, "Could not re-enable clocks: %d\n", r);
 
-	/* wait until PWR_ACK = 0 */
-	ret = readx_poll_timeout(scpsys_domain_is_on, pd, tmp, !tmp, MTK_POLL_DELAY_US,
-				 MTK_POLL_TIMEOUT);
-	if (ret < 0)
 		return ret;
+	}
 
 	clk_bulk_disable_unprepare(pd->num_clks, pd->clks);
 
@@ -737,8 +801,16 @@ static int scpsys_power_off(struct generic_pm_domain *genpd)
 	return 0;
 }
 
+static int scpsys_power_off(struct generic_pm_domain *genpd)
+{
+	struct scpsys_domain *pd = container_of(genpd, struct scpsys_domain, genpd);
+
+	return scpsys_power_off_internal(pd);
+}
+
 static struct
-generic_pm_domain *scpsys_add_one_domain(struct scpsys *scpsys, struct device_node *node)
+generic_pm_domain *scpsys_add_one_domain(struct scpsys *scpsys, struct device_node *node,
+					 u8 *domains_idx, u8 *num_domains)
 {
 	const struct scpsys_domain_data *domain_data;
 	const struct scpsys_hwv_domain_data *hwv_domain_data;
@@ -884,7 +956,14 @@ generic_pm_domain *scpsys_add_one_domain(struct scpsys *scpsys, struct device_no
 	 * late_init time.
 	 */
 	if (MTK_SCPD_CAPS(pd, MTK_SCPD_KEEP_DEFAULT_OFF)) {
-		if (scpsys_domain_is_on(pd))
+		bool domain_is_on;
+
+		if (scpsys->soc_data->type == SCPSYS_MTCMOS_TYPE_HW_VOTER)
+			domain_is_on = scpsys_hwv_domain_is_enable_done(pd);
+		else
+			domain_is_on = scpsys_domain_is_on(pd);
+
+		if (domain_is_on)
 			dev_warn(scpsys->dev,
 				 "%pOF: A default off power domain has been ON\n", node);
 	} else {
@@ -906,6 +985,7 @@ generic_pm_domain *scpsys_add_one_domain(struct scpsys *scpsys, struct device_no
 	else
 		pm_genpd_init(&pd->genpd, NULL, false);
 
+	domains_idx[(*num_domains)++] = (u8) id;
 	scpsys->domains[id] = &pd->genpd;
 
 	return scpsys->pd_data.domains[id];
@@ -917,7 +997,8 @@ generic_pm_domain *scpsys_add_one_domain(struct scpsys *scpsys, struct device_no
 	return ERR_PTR(ret);
 }
 
-static int scpsys_add_subdomain(struct scpsys *scpsys, struct device_node *parent)
+static int scpsys_add_subdomain(struct scpsys *scpsys, struct device_node *parent,
+				u8 *domains_idx, u8 *num_domains)
 {
 	struct generic_pm_domain *child_pd, *parent_pd;
 	struct device_node *child;
@@ -940,7 +1021,7 @@ static int scpsys_add_subdomain(struct scpsys *scpsys, struct device_node *paren
 
 		parent_pd = scpsys->pd_data.domains[id];
 
-		child_pd = scpsys_add_one_domain(scpsys, child);
+		child_pd = scpsys_add_one_domain(scpsys, child, domains_idx, num_domains);
 		if (IS_ERR(child_pd)) {
 			ret = PTR_ERR(child_pd);
 			dev_err_probe(scpsys->dev, ret, "%pOF: failed to get child domain id\n",
@@ -949,7 +1030,7 @@ static int scpsys_add_subdomain(struct scpsys *scpsys, struct device_node *paren
 		}
 
 		/* recursive call to add all subdomains */
-		ret = scpsys_add_subdomain(scpsys, child);
+		ret = scpsys_add_subdomain(scpsys, child, domains_idx, num_domains);
 		if (ret)
 			goto err_put_node;
 
@@ -973,6 +1054,7 @@ static int scpsys_add_subdomain(struct scpsys *scpsys, struct device_node *paren
 
 static void scpsys_remove_one_domain(struct scpsys_domain *pd)
 {
+	struct scpsys *scpsys = pd->scpsys;
 	int ret;
 
 	/*
@@ -984,21 +1066,29 @@ static void scpsys_remove_one_domain(struct scpsys_domain *pd)
 		dev_err(pd->scpsys->dev,
 			"failed to remove domain '%s' : %d - state may be inconsistent\n",
 			pd->genpd.name, ret);
-	if (scpsys_domain_is_on(pd))
-		scpsys_power_off(&pd->genpd);
+
+	if (scpsys->soc_data->type == SCPSYS_MTCMOS_TYPE_HW_VOTER) {
+		if (scpsys_hwv_domain_is_enable_done(pd))
+			scpsys_hwv_power_off_internal(pd);
+	} else {
+		if (scpsys_domain_is_on(pd) || MTK_SCPD_CAPS(pd, MTK_SCPD_SIMPLE_PWRSEQ))
+			scpsys_power_off_internal(pd);
+	}
 
 	clk_bulk_put(pd->num_clks, pd->clks);
 	clk_bulk_put(pd->num_subsys_clks, pd->subsys_clks);
 }
 
-static void scpsys_domain_cleanup(struct scpsys *scpsys)
+static void scpsys_domain_cleanup(struct scpsys *scpsys, u8 *domains_idx, u8 num_probed)
 {
 	struct generic_pm_domain *genpd;
 	struct scpsys_domain *pd;
 	int i;
 
-	for (i = scpsys->pd_data.num_domains - 1; i >= 0; i--) {
-		genpd = scpsys->pd_data.domains[i];
+	for (i = num_probed - 1; i >= 0; i--) {
+		u8 pd_idx = domains_idx[i];
+
+		genpd = scpsys->pd_data.domains[pd_idx];
 		if (genpd) {
 			pd = to_scpsys_domain(genpd);
 			scpsys_remove_one_domain(pd);
@@ -1078,6 +1168,12 @@ static int scpsys_get_bus_protection_legacy(struct device *dev, struct scpsys *s
 		regmap[2] = NULL;
 	}
 
+	/* If no access controllers are needed, don't allocate and don't fail */
+	if (num_regmaps == 0) {
+		scpsys->bus_prot = NULL;
+		return 0;
+	}
+
 	scpsys->bus_prot = devm_kmalloc_array(dev, num_regmaps,
 					      sizeof(*scpsys->bus_prot), GFP_KERNEL);
 	if (!scpsys->bus_prot)
@@ -1192,6 +1288,10 @@ static const struct of_device_id scpsys_of_match[] = {
 		.compatible = "mediatek,mt8196-power-controller",
 		.data = &mt8196_scpsys_data,
 	},
+	{
+		.compatible = "mediatek,mt8196-hfrp-power-controller",
+		.data = &mt8196_hfrpsys_data,
+	},
 	{
 		.compatible = "mediatek,mt8196-hwv-hfrp-power-controller",
 		.data = &mt8196_hfrpsys_hwv_data,
@@ -1215,6 +1315,8 @@ static int scpsys_probe(struct platform_device *pdev)
 	struct device *parent;
 	struct scpsys *scpsys;
 	int num_domains, ret;
+	u8 num_added_pds = 0;
+	u8 *added_pds_idx;
 
 	soc = of_device_get_match_data(&pdev->dev);
 	if (!soc) {
@@ -1228,6 +1330,19 @@ static int scpsys_probe(struct platform_device *pdev)
 	if (!scpsys)
 		return -ENOMEM;
 
+	/*
+	 * Temporarily store the IDs of the power domains that are added as in
+	 * case of a probe deferral this can be used to correctly cleanup all
+	 * of what was added before.
+	 *
+	 * Note that this array is used only in the probe function and must be
+	 * freed at the end, regardless of whether all of the power domains were
+	 * probed successfully or any failure happened.
+	 */
+	added_pds_idx = devm_kmalloc_array(dev, num_domains, sizeof(*added_pds_idx), GFP_KERNEL);
+	if (!added_pds_idx)
+		return -ENOMEM;
+
 	scpsys->dev = dev;
 	scpsys->soc_data = soc;
 
@@ -1258,13 +1373,15 @@ static int scpsys_probe(struct platform_device *pdev)
 	for_each_available_child_of_node_scoped(np, node) {
 		struct generic_pm_domain *domain;
 
-		domain = scpsys_add_one_domain(scpsys, node);
+		domain = scpsys_add_one_domain(scpsys, node,
+					       added_pds_idx, &num_added_pds);
 		if (IS_ERR(domain)) {
 			ret = PTR_ERR(domain);
 			goto err_cleanup_domains;
 		}
 
-		ret = scpsys_add_subdomain(scpsys, node);
+		ret = scpsys_add_subdomain(scpsys, node,
+					   added_pds_idx, &num_added_pds);
 		if (ret)
 			goto err_cleanup_domains;
 	}
@@ -1280,10 +1397,11 @@ static int scpsys_probe(struct platform_device *pdev)
 		goto err_cleanup_domains;
 	}
 
+	devm_kfree(dev, added_pds_idx);
 	return 0;
 
 err_cleanup_domains:
-	scpsys_domain_cleanup(scpsys);
+	scpsys_domain_cleanup(scpsys, added_pds_idx, num_added_pds);
 	return ret;
 }
 
diff --git a/drivers/pmdomain/mediatek/mtk-pm-domains.h b/drivers/pmdomain/mediatek/mtk-pm-domains.h
index a5dca24cbc2f5..092403de66fa0 100644
--- a/drivers/pmdomain/mediatek/mtk-pm-domains.h
+++ b/drivers/pmdomain/mediatek/mtk-pm-domains.h
@@ -17,6 +17,7 @@
 #define MTK_SCPD_MODEM_PWRSEQ		BIT(10)
 #define MTK_SCPD_SKIP_RESET_B		BIT(11)
 #define MTK_SCPD_INFRA_PWR_CTL		BIT(12)
+#define MTK_SCPD_SIMPLE_PWRSEQ		BIT(13)
 #define MTK_SCPD_CAPS(_scpd, _x)	((_scpd)->data ?		\
 					 (_scpd)->data->caps & (_x) :	\
 					 (_scpd)->hwv_data->caps & (_x))
diff --git a/drivers/pmdomain/qcom/rpmhpd.c b/drivers/pmdomain/qcom/rpmhpd.c
index 63120e703923f..90743275942db 100644
--- a/drivers/pmdomain/qcom/rpmhpd.c
+++ b/drivers/pmdomain/qcom/rpmhpd.c
@@ -41,7 +41,6 @@
  * @addr:		Resource address as looped up using resource name from
  *			cmd-db
  * @state_synced:	Indicator that sync_state has been invoked for the rpmhpd resource
- * @skip_retention_level: Indicate that retention level should not be used for the power domain
  */
 struct rpmhpd {
 	struct device	*dev;
@@ -58,7 +57,6 @@ struct rpmhpd {
 	const char	*res_name;
 	u32		addr;
 	bool		state_synced;
-	bool            skip_retention_level;
 };
 
 struct rpmhpd_desc {
@@ -191,7 +189,6 @@ static struct rpmhpd mxc = {
 	.pd = { .name = "mxc", },
 	.peer = &mxc_ao,
 	.res_name = "mxc.lvl",
-	.skip_retention_level = true,
 };
 
 static struct rpmhpd mxc_ao = {
@@ -199,7 +196,6 @@ static struct rpmhpd mxc_ao = {
 	.active_only = true,
 	.peer = &mxc,
 	.res_name = "mxc.lvl",
-	.skip_retention_level = true,
 };
 
 static struct rpmhpd nsp = {
@@ -305,10 +301,10 @@ static struct rpmhpd *sa8775p_rpmhpds[] = {
 	[SA8775P_LMX] = &lmx,
 	[SA8775P_MMCX] = &mmcx,
 	[SA8775P_MMCX_AO] = &mmcx_ao,
-	[SA8775P_MXC] = &mxc,
-	[SA8775P_MXC_AO] = &mxc_ao,
 	[SA8775P_MX] = &mx,
 	[SA8775P_MX_AO] = &mx_ao,
+	[SA8775P_MXC] = &mxc,
+	[SA8775P_MXC_AO] = &mxc_ao,
 	[SA8775P_NSP0] = &nsp0,
 	[SA8775P_NSP1] = &nsp1,
 };
@@ -325,10 +321,10 @@ static struct rpmhpd *nord_rpmhpds[] = {
 	[RPMHPD_EBI] = &ebi,
 	[RPMHPD_GFX] = &gfx,
 	[RPMHPD_GFX1] = &gfx1,
-	[RPMHPD_MX] = &mx,
-	[RPMHPD_MX_AO] = &mx_ao,
 	[RPMHPD_MMCX] = &mmcx,
 	[RPMHPD_MMCX_AO] = &mmcx_ao,
+	[RPMHPD_MX] = &mx,
+	[RPMHPD_MX_AO] = &mx_ao,
 	[RPMHPD_MXC] = &mxc,
 	[RPMHPD_MXC_AO] = &mxc_ao,
 	[RPMHPD_NSP0] = &nsp0,
@@ -480,9 +476,9 @@ static struct rpmhpd *sm7150_rpmhpds[] = {
 	[RPMHPD_GFX] = &gfx,
 	[RPMHPD_LCX] = &lcx,
 	[RPMHPD_LMX] = &lmx,
+	[RPMHPD_MSS] = &mss,
 	[RPMHPD_MX] = &mx,
 	[RPMHPD_MX_AO] = &mx_ao,
-	[RPMHPD_MSS] = &mss,
 };
 
 static const struct rpmhpd_desc sm7150_desc = {
@@ -698,11 +694,11 @@ static struct rpmhpd *hawi_rpmhpds[] = {
 	[RPMHPD_LMX] = &lmx,
 	[RPMHPD_MMCX] = &mmcx,
 	[RPMHPD_MMCX_AO] = &mmcx_ao,
+	[RPMHPD_MSS] = &mss,
 	[RPMHPD_MX] = &mx,
 	[RPMHPD_MX_AO] = &mx_ao,
 	[RPMHPD_MXC] = &mxc,
 	[RPMHPD_MXC_AO] = &mxc_ao,
-	[RPMHPD_MSS] = &mss,
 	[RPMHPD_NSP] = &nsp,
 	[RPMHPD_NSP2] = &nsp2,
 };
@@ -809,18 +805,18 @@ static struct rpmhpd *glymur_rpmhpds[] = {
 	[RPMHPD_CX_AO] = &cx_ao,
 	[RPMHPD_EBI] = &ebi,
 	[RPMHPD_GFX] = &gfx,
+	[RPMHPD_GMXC] = &gmxc,
 	[RPMHPD_LCX] = &lcx,
 	[RPMHPD_LMX] = &lmx,
 	[RPMHPD_MMCX] = &mmcx,
 	[RPMHPD_MMCX_AO] = &mmcx_ao,
+	[RPMHPD_MSS] = &mss,
 	[RPMHPD_MX] = &mx,
 	[RPMHPD_MX_AO] = &mx_ao,
 	[RPMHPD_MXC] = &mxc,
 	[RPMHPD_MXC_AO] = &mxc_ao,
-	[RPMHPD_MSS] = &mss,
 	[RPMHPD_NSP] = &nsp,
 	[RPMHPD_NSP2] = &nsp2,
-	[RPMHPD_GMXC] = &gmxc,
 };
 
 static const struct rpmhpd_desc glymur_desc = {
@@ -834,15 +830,15 @@ static struct rpmhpd *x1e80100_rpmhpds[] = {
 	[RPMHPD_CX_AO] = &cx_ao,
 	[RPMHPD_EBI] = &ebi,
 	[RPMHPD_GFX] = &gfx,
+	[RPMHPD_GMXC] = &gmxc,
 	[RPMHPD_LCX] = &lcx,
 	[RPMHPD_LMX] = &lmx,
 	[RPMHPD_MMCX] = &mmcx,
 	[RPMHPD_MMCX_AO] = &mmcx_ao,
 	[RPMHPD_MX] = &mx,
 	[RPMHPD_MX_AO] = &mx_ao,
-	[RPMHPD_NSP] = &nsp,
 	[RPMHPD_MXC] = &mxc,
-	[RPMHPD_GMXC] = &gmxc,
+	[RPMHPD_NSP] = &nsp,
 };
 
 static const struct rpmhpd_desc x1e80100_desc = {
@@ -860,10 +856,10 @@ static struct rpmhpd *qcs8300_rpmhpds[] = {
 	[RPMHPD_LMX] = &lmx,
 	[RPMHPD_MMCX] = &mmcx_w_cx_parent,
 	[RPMHPD_MMCX_AO] = &mmcx_ao_w_cx_parent,
-	[RPMHPD_MXC] = &mxc,
-	[RPMHPD_MXC_AO] = &mxc_ao,
 	[RPMHPD_MX] = &mx,
 	[RPMHPD_MX_AO] = &mx_ao,
+	[RPMHPD_MXC] = &mxc,
+	[RPMHPD_MXC_AO] = &mxc_ao,
 	[RPMHPD_NSP0] = &nsp0,
 	[RPMHPD_NSP1] = &nsp1,
 };
@@ -1093,7 +1089,15 @@ static int rpmhpd_update_level_mapping(struct rpmhpd *rpmhpd)
 		return -EINVAL;
 
 	for (i = 0; i < rpmhpd->level_count; i++) {
-		if (rpmhpd->skip_retention_level && buf[i] == RPMH_REGULATOR_LEVEL_RETENTION)
+		/*
+		 * Most HW won't function properly at Retention. The minimum
+		 * operational level is the first level above Retention. The
+		 * small subset of HW that can operate at Retention isn't
+		 * controlled by HLOS. Skip the Retention level to avoid HW
+		 * failures when the PD is enabled without first having an
+		 * explicit OPP level set.
+		 */
+		if (buf[i] == RPMH_REGULATOR_LEVEL_RETENTION)
 			continue;
 
 		rpmhpd->level[i] = buf[i];
diff --git a/drivers/pmdomain/tegra/powergate-bpmp.c b/drivers/pmdomain/tegra/powergate-bpmp.c
index 8cde4f384846a..e6a7e025b0122 100644
--- a/drivers/pmdomain/tegra/powergate-bpmp.c
+++ b/drivers/pmdomain/tegra/powergate-bpmp.c
@@ -137,6 +137,13 @@ static char *tegra_bpmp_powergate_get_name(struct tegra_bpmp *bpmp,
 	if (err < 0 || msg.rx.ret < 0)
 		return NULL;
 
+	if (response.get_name.name[0] == '\0')
+		return NULL;
+
+	if (dev_to_node(bpmp->dev) != NUMA_NO_NODE)
+		return kasprintf(GFP_KERNEL, "%d-%s", dev_to_node(bpmp->dev),
+				 response.get_name.name);
+
 	return kstrdup(response.get_name.name, GFP_KERNEL);
 }
 
@@ -234,7 +241,7 @@ tegra_bpmp_probe_powergates(struct tegra_bpmp *bpmp,
 		struct tegra_powergate_info *info = &powergates[count];
 
 		info->name = tegra_bpmp_powergate_get_name(bpmp, id);
-		if (!info->name || info->name[0] == '\0') {
+		if (!info->name) {
 			num_holes++;
 			continue;
 		}
diff --git a/drivers/pnp/card.c b/drivers/pnp/card.c
index 87f5af454751f..df5e3d3cbf6c0 100644
--- a/drivers/pnp/card.c
+++ b/drivers/pnp/card.c
@@ -254,9 +254,16 @@ int pnp_add_card(struct pnp_card *card)
 	/* we wait until now to add devices in order to ensure the drivers
 	 * will be able to use all of the related devices on the card
 	 * without waiting an unreasonable length of time */
-	list_for_each(pos, &card->devices) {
+	list_for_each_safe(pos, temp, &card->devices) {
 		struct pnp_dev *dev = card_to_pnp_dev(pos);
-		__pnp_add_device(dev);
+		error = __pnp_add_device(dev);
+		if (error) {
+			mutex_lock(&pnp_lock);
+			list_del(&dev->card_list);
+			dev->card = NULL;
+			mutex_unlock(&pnp_lock);
+			put_device(&dev->dev);
+		}
 	}
 
 	/* match with card drivers */
diff --git a/drivers/pnp/pnpacpi/core.c b/drivers/pnp/pnpacpi/core.c
index fbf03ff007ebe..d8ec0ad2ab4e7 100644
--- a/drivers/pnp/pnpacpi/core.c
+++ b/drivers/pnp/pnpacpi/core.c
@@ -248,10 +248,7 @@ static int __init pnpacpi_add_device(struct acpi_device *device)
 	if (acpi_has_method(device->handle, "_DIS"))
 		dev->capabilities |= PNP_DISABLE;
 
-	if (strlen(acpi_device_name(device)))
-		strscpy(dev->name, acpi_device_name(device), sizeof(dev->name));
-	else
-		strscpy(dev->name, acpi_device_bid(device), sizeof(dev->name));
+	strscpy(dev->name, acpi_device_bid(device), sizeof(dev->name));
 
 	if (dev->active)
 		pnpacpi_parse_allocated_resource(dev);
diff --git a/drivers/pnp/system.c b/drivers/pnp/system.c
index 835113b2cb041..bd939b5643799 100644
--- a/drivers/pnp/system.c
+++ b/drivers/pnp/system.c
@@ -17,10 +17,10 @@
 
 static const struct pnp_device_id pnp_dev_table[] = {
 	/* General ID for reserving resources */
-	{"PNP0c02", 0},
+	{ .id = "PNP0c02" },
 	/* memory controller */
-	{"PNP0c01", 0},
-	{"", 0}
+	{ .id = "PNP0c01" },
+	{ }
 };
 
 static void reserve_range(struct pnp_dev *dev, struct resource *r, int port)
diff --git a/drivers/power/sequencing/pwrseq-pcie-m2.c b/drivers/power/sequencing/pwrseq-pcie-m2.c
index ddfe6ca824945..83fe6a1396bc1 100644
--- a/drivers/power/sequencing/pwrseq-pcie-m2.c
+++ b/drivers/power/sequencing/pwrseq-pcie-m2.c
@@ -185,10 +185,18 @@ static int pwrseq_pcie_m2_match(struct pwrseq_device *pwrseq,
 }
 
 static const struct pci_device_id pwrseq_m2_pci_ids[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x2b43),
+	  .driver_data = (kernel_ulong_t)"nxp,88w8987-bt" },
+	{ PCI_DEVICE(PCI_VENDOR_ID_PHILIPS, 0x3003),
+	  .driver_data = (kernel_ulong_t)"nxp,88w8987-bt" },
+	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x1103, PCI_VENDOR_ID_QCOM, 0x0108),
+	  .driver_data = (kernel_ulong_t)"qcom,qca2066-bt" },
 	{ PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x1103),
 	  .driver_data = (kernel_ulong_t)"qcom,wcn6855-bt" },
 	{ PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x1107),
 	  .driver_data = (kernel_ulong_t)"qcom,wcn7850-bt" },
+	{ PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x1112),
+	  .driver_data = (kernel_ulong_t)"qcom,qcc2072-bt" },
 	{ } /* Sentinel */
 };
 
diff --git a/drivers/power/supply/ab8500_btemp.c b/drivers/power/supply/ab8500_btemp.c
index e5202a7b6209b..36b0c52a4b8b3 100644
--- a/drivers/power/supply/ab8500_btemp.c
+++ b/drivers/power/supply/ab8500_btemp.c
@@ -829,3 +829,4 @@ MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Johan Palsson, Karl Komierowski, Arun R Murthy");
 MODULE_ALIAS("platform:ab8500-btemp");
 MODULE_DESCRIPTION("AB8500 battery temperature driver");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/ab8500_charger.c b/drivers/power/supply/ab8500_charger.c
index 1813fbdfa1c1f..5c3af35710139 100644
--- a/drivers/power/supply/ab8500_charger.c
+++ b/drivers/power/supply/ab8500_charger.c
@@ -3751,3 +3751,4 @@ MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Johan Palsson, Karl Komierowski, Arun R Murthy");
 MODULE_ALIAS("platform:ab8500-charger");
 MODULE_DESCRIPTION("AB8500 charger management driver");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/ab8500_fg.c b/drivers/power/supply/ab8500_fg.c
index eb5c1ae68e440..3ab33cf465c6e 100644
--- a/drivers/power/supply/ab8500_fg.c
+++ b/drivers/power/supply/ab8500_fg.c
@@ -3252,3 +3252,4 @@ MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Johan Palsson, Karl Komierowski");
 MODULE_ALIAS("platform:ab8500-fg");
 MODULE_DESCRIPTION("AB8500 Fuel Gauge driver");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/axp20x_ac_power.c b/drivers/power/supply/axp20x_ac_power.c
index 5f6ea416fa30a..e9049d6229df9 100644
--- a/drivers/power/supply/axp20x_ac_power.c
+++ b/drivers/power/supply/axp20x_ac_power.c
@@ -421,3 +421,4 @@ module_platform_driver(axp20x_ac_power_driver);
 MODULE_AUTHOR("Quentin Schulz <quentin.schulz@free-electrons.com>");
 MODULE_DESCRIPTION("AXP20X and AXP22X PMICs' AC power supply driver");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/axp20x_battery.c b/drivers/power/supply/axp20x_battery.c
index 50ca8e110085b..ee8701a6e9077 100644
--- a/drivers/power/supply/axp20x_battery.c
+++ b/drivers/power/supply/axp20x_battery.c
@@ -1155,3 +1155,4 @@ module_platform_driver(axp20x_batt_driver);
 MODULE_DESCRIPTION("Battery power supply driver for AXP20X and AXP22X PMICs");
 MODULE_AUTHOR("Quentin Schulz <quentin.schulz@free-electrons.com>");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/axp20x_usb_power.c b/drivers/power/supply/axp20x_usb_power.c
index e75d1e377ac12..599adcf84968f 100644
--- a/drivers/power/supply/axp20x_usb_power.c
+++ b/drivers/power/supply/axp20x_usb_power.c
@@ -1080,3 +1080,4 @@ module_platform_driver(axp20x_usb_power_driver);
 MODULE_AUTHOR("Hans de Goede <hdegoede@redhat.com>");
 MODULE_DESCRIPTION("AXP20x PMIC USB power supply status driver");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/axp288_fuel_gauge.c b/drivers/power/supply/axp288_fuel_gauge.c
index 5af334c0a980c..05a850ec01317 100644
--- a/drivers/power/supply/axp288_fuel_gauge.c
+++ b/drivers/power/supply/axp288_fuel_gauge.c
@@ -817,3 +817,4 @@ MODULE_AUTHOR("Ramakrishna Pallala <ramakrishna.pallala@intel.com>");
 MODULE_AUTHOR("Todd Brandt <todd.e.brandt@linux.intel.com>");
 MODULE_DESCRIPTION("Xpower AXP288 Fuel Gauge Driver");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/cpcap-battery.c b/drivers/power/supply/cpcap-battery.c
index 59c741993ef8d..4d7e77d7375bb 100644
--- a/drivers/power/supply/cpcap-battery.c
+++ b/drivers/power/supply/cpcap-battery.c
@@ -1206,3 +1206,4 @@ module_platform_driver(cpcap_battery_driver);
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Tony Lindgren <tony@atomide.com>");
 MODULE_DESCRIPTION("CPCAP PMIC Battery Driver");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/cpcap-charger.c b/drivers/power/supply/cpcap-charger.c
index ec8d2a9245d94..744ee12edd1e4 100644
--- a/drivers/power/supply/cpcap-charger.c
+++ b/drivers/power/supply/cpcap-charger.c
@@ -977,3 +977,4 @@ MODULE_AUTHOR("Tony Lindgren <tony@atomide.com>");
 MODULE_DESCRIPTION("CPCAP Battery Charger Interface driver");
 MODULE_LICENSE("GPL v2");
 MODULE_ALIAS("platform:cpcap-charger");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/da9150-charger.c b/drivers/power/supply/da9150-charger.c
index 27f36ef5b88dc..58449df6068c5 100644
--- a/drivers/power/supply/da9150-charger.c
+++ b/drivers/power/supply/da9150-charger.c
@@ -644,3 +644,4 @@ module_platform_driver(da9150_charger_driver);
 MODULE_DESCRIPTION("Charger Driver for DA9150");
 MODULE_AUTHOR("Adam Thomson <Adam.Thomson.Opensource@diasemi.com>");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/generic-adc-battery.c b/drivers/power/supply/generic-adc-battery.c
index f5f2566b3a32d..d18c8ee404056 100644
--- a/drivers/power/supply/generic-adc-battery.c
+++ b/drivers/power/supply/generic-adc-battery.c
@@ -298,3 +298,4 @@ module_platform_driver(gab_driver);
 MODULE_AUTHOR("anish kumar <yesanishhere@gmail.com>");
 MODULE_DESCRIPTION("generic battery driver using IIO");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/ingenic-battery.c b/drivers/power/supply/ingenic-battery.c
index b111c7ce2be30..5be269f17bffc 100644
--- a/drivers/power/supply/ingenic-battery.c
+++ b/drivers/power/supply/ingenic-battery.c
@@ -190,3 +190,4 @@ module_platform_driver(ingenic_battery_driver);
 MODULE_DESCRIPTION("Battery driver for Ingenic JZ47xx SoCs");
 MODULE_AUTHOR("Artur Rojek <contact@artur-rojek.eu>");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/intel_dc_ti_battery.c b/drivers/power/supply/intel_dc_ti_battery.c
index 67a75281b0acb..cb6fa8d88b433 100644
--- a/drivers/power/supply/intel_dc_ti_battery.c
+++ b/drivers/power/supply/intel_dc_ti_battery.c
@@ -389,3 +389,4 @@ MODULE_ALIAS("platform:" DEV_NAME);
 MODULE_AUTHOR("Hans de Goede <hansg@kernel.org>");
 MODULE_DESCRIPTION("Intel Dollar Cove (TI) battery driver");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/lego_ev3_battery.c b/drivers/power/supply/lego_ev3_battery.c
index 582644f4e8b3b..1e53f11a8e2b2 100644
--- a/drivers/power/supply/lego_ev3_battery.c
+++ b/drivers/power/supply/lego_ev3_battery.c
@@ -230,3 +230,4 @@ module_platform_driver(lego_ev3_battery_driver);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("David Lechner <david@lechnology.com>");
 MODULE_DESCRIPTION("LEGO MINDSTORMS EV3 Battery Driver");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/lp8788-charger.c b/drivers/power/supply/lp8788-charger.c
index f0a680c155c48..8c6ec98362d0f 100644
--- a/drivers/power/supply/lp8788-charger.c
+++ b/drivers/power/supply/lp8788-charger.c
@@ -727,3 +727,4 @@ MODULE_DESCRIPTION("TI LP8788 Charger Driver");
 MODULE_AUTHOR("Milo Kim");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("platform:lp8788-charger");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/max17040_battery.c b/drivers/power/supply/max17040_battery.c
index e94d53b36aa49..19a9b1bed29ee 100644
--- a/drivers/power/supply/max17040_battery.c
+++ b/drivers/power/supply/max17040_battery.c
@@ -639,3 +639,4 @@ module_i2c_driver(max17040_i2c_driver);
 MODULE_AUTHOR("Minkyu Kang <mk7.kang@samsung.com>");
 MODULE_DESCRIPTION("MAX17040 Fuel Gauge");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/mp2629_charger.c b/drivers/power/supply/mp2629_charger.c
index f758d6a7bc8c1..c06743d2abffd 100644
--- a/drivers/power/supply/mp2629_charger.c
+++ b/drivers/power/supply/mp2629_charger.c
@@ -659,3 +659,4 @@ module_platform_driver(mp2629_charger_driver);
 MODULE_AUTHOR("Saravanan Sekar <sravanhome@gmail.com>");
 MODULE_DESCRIPTION("MP2629 Charger driver");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/mt6370-charger.c b/drivers/power/supply/mt6370-charger.c
index 916556baa854a..8ac5f4d59f39d 100644
--- a/drivers/power/supply/mt6370-charger.c
+++ b/drivers/power/supply/mt6370-charger.c
@@ -930,3 +930,4 @@ module_platform_driver(mt6370_chg_driver);
 MODULE_AUTHOR("ChiaEn Wu <chiaen_wu@richtek.com>");
 MODULE_DESCRIPTION("MediaTek MT6370 Charger Driver");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/qcom_smbx.c b/drivers/power/supply/qcom_smbx.c
index bf2e2ccc454a6..7010374b302af 100644
--- a/drivers/power/supply/qcom_smbx.c
+++ b/drivers/power/supply/qcom_smbx.c
@@ -1057,3 +1057,4 @@ module_platform_driver(qcom_spmi_smb);
 MODULE_AUTHOR("Casey Connolly <casey.connolly@linaro.org>");
 MODULE_DESCRIPTION("Qualcomm SMB2 Charger Driver");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/rn5t618_power.c b/drivers/power/supply/rn5t618_power.c
index 40dec55a9f736..a3f30e390c11f 100644
--- a/drivers/power/supply/rn5t618_power.c
+++ b/drivers/power/supply/rn5t618_power.c
@@ -821,3 +821,4 @@ module_platform_driver(rn5t618_power_driver);
 MODULE_ALIAS("platform:rn5t618-power");
 MODULE_DESCRIPTION("Power supply driver for RICOH RN5T618");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/rx51_battery.c b/drivers/power/supply/rx51_battery.c
index b0220ec2d9266..57266921dc8e2 100644
--- a/drivers/power/supply/rx51_battery.c
+++ b/drivers/power/supply/rx51_battery.c
@@ -246,3 +246,4 @@ MODULE_ALIAS("platform:rx51-battery");
 MODULE_AUTHOR("Pali Rohár <pali@kernel.org>");
 MODULE_DESCRIPTION("Nokia RX-51 battery driver");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/sc27xx_fuel_gauge.c b/drivers/power/supply/sc27xx_fuel_gauge.c
index a7ed9de8a2896..1719ec4173e6a 100644
--- a/drivers/power/supply/sc27xx_fuel_gauge.c
+++ b/drivers/power/supply/sc27xx_fuel_gauge.c
@@ -1350,3 +1350,4 @@ module_platform_driver(sc27xx_fgu_driver);
 
 MODULE_DESCRIPTION("Spreadtrum SC27XX PMICs Fual Gauge Unit Driver");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/surface_battery.c b/drivers/power/supply/surface_battery.c
index c759add4df49d..1273b6082311d 100644
--- a/drivers/power/supply/surface_battery.c
+++ b/drivers/power/supply/surface_battery.c
@@ -852,9 +852,14 @@ static const struct spwr_psy_properties spwr_psy_props_bat2_sb3 = {
 };
 
 static const struct ssam_device_id surface_battery_match[] = {
-	{ SSAM_SDEV(BAT, SAM, 0x01, 0x00), (unsigned long)&spwr_psy_props_bat1     },
-	{ SSAM_SDEV(BAT, KIP, 0x01, 0x00), (unsigned long)&spwr_psy_props_bat2_sb3 },
-	{ },
+	{
+		SSAM_SDEV(BAT, SAM, 0x01, 0x00),
+		.driver_data = (unsigned long)&spwr_psy_props_bat1,
+	}, {
+		SSAM_SDEV(BAT, KIP, 0x01, 0x00),
+		.driver_data = (unsigned long)&spwr_psy_props_bat2_sb3,
+	},
+	{ }
 };
 MODULE_DEVICE_TABLE(ssam, surface_battery_match);
 
diff --git a/drivers/power/supply/surface_charger.c b/drivers/power/supply/surface_charger.c
index 90b823848c998..d4bba6b417947 100644
--- a/drivers/power/supply/surface_charger.c
+++ b/drivers/power/supply/surface_charger.c
@@ -260,8 +260,11 @@ static const struct spwr_psy_properties spwr_psy_props_adp1 = {
 };
 
 static const struct ssam_device_id surface_ac_match[] = {
-	{ SSAM_SDEV(BAT, SAM, 0x01, 0x01), (unsigned long)&spwr_psy_props_adp1 },
-	{ },
+	{
+		SSAM_SDEV(BAT, SAM, 0x01, 0x01),
+		.driver_data = (unsigned long)&spwr_psy_props_adp1,
+	},
+	{ }
 };
 MODULE_DEVICE_TABLE(ssam, surface_ac_match);
 
diff --git a/drivers/power/supply/twl4030_charger.c b/drivers/power/supply/twl4030_charger.c
index 04216b2bfb6c3..151f7b24e9b9b 100644
--- a/drivers/power/supply/twl4030_charger.c
+++ b/drivers/power/supply/twl4030_charger.c
@@ -1144,3 +1144,4 @@ MODULE_AUTHOR("Gražvydas Ignotas");
 MODULE_DESCRIPTION("TWL4030 Battery Charger Interface driver");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("platform:twl4030_bci");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/twl4030_madc_battery.c b/drivers/power/supply/twl4030_madc_battery.c
index a99b3ff26929d..5e04b4f0a1350 100644
--- a/drivers/power/supply/twl4030_madc_battery.c
+++ b/drivers/power/supply/twl4030_madc_battery.c
@@ -235,3 +235,4 @@ MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Lukas Märdian <lukas@goldelico.com>");
 MODULE_DESCRIPTION("twl4030_madc battery driver");
 MODULE_ALIAS("platform:twl4030_madc_battery");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/power/supply/twl6030_charger.c b/drivers/power/supply/twl6030_charger.c
index b4ec26ff257cc..82911a811f4ec 100644
--- a/drivers/power/supply/twl6030_charger.c
+++ b/drivers/power/supply/twl6030_charger.c
@@ -579,3 +579,4 @@ module_platform_driver(twl6030_charger_driver);
 
 MODULE_DESCRIPTION("TWL6030 Battery Charger Interface driver");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c
index 1006d183d5081..3cee4ccb79976 100644
--- a/drivers/powercap/intel_rapl_common.c
+++ b/drivers/powercap/intel_rapl_common.c
@@ -752,17 +752,17 @@ EXPORT_SYMBOL_NS_GPL(rapl_default_check_unit, "INTEL_RAPL");
 
 static void power_limit_irq_save_cpu(void *info)
 {
-	u32 l, h = 0;
+	struct msr val;
 	struct rapl_package *rp = (struct rapl_package *)info;
 
 	/* save the state of PLN irq mask bit before disabling it */
-	rdmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h);
+	rdmsrq_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &val.q);
 	if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) {
-		rp->power_limit_irq = l & PACKAGE_THERM_INT_PLN_ENABLE;
+		rp->power_limit_irq = val.l & PACKAGE_THERM_INT_PLN_ENABLE;
 		rp->power_limit_irq |= PACKAGE_PLN_INT_SAVED;
 	}
-	l &= ~PACKAGE_THERM_INT_PLN_ENABLE;
-	wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
+	val.l &= ~PACKAGE_THERM_INT_PLN_ENABLE;
+	wrmsrq_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, val.q);
 }
 
 /* REVISIT:
@@ -792,7 +792,7 @@ static void package_power_limit_irq_save(struct rapl_package *rp)
  */
 static void package_power_limit_irq_restore(struct rapl_package *rp)
 {
-	u32 l, h;
+	struct msr val;
 
 	if (rp->lead_cpu < 0)
 		return;
@@ -804,14 +804,14 @@ static void package_power_limit_irq_restore(struct rapl_package *rp)
 	if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED))
 		return;
 
-	rdmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h);
+	rdmsrq_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &val.q);
 
 	if (rp->power_limit_irq & PACKAGE_THERM_INT_PLN_ENABLE)
-		l |= PACKAGE_THERM_INT_PLN_ENABLE;
+		val.l |= PACKAGE_THERM_INT_PLN_ENABLE;
 	else
-		l &= ~PACKAGE_THERM_INT_PLN_ENABLE;
+		val.l &= ~PACKAGE_THERM_INT_PLN_ENABLE;
 
-	wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
+	wrmsrq_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, val.q);
 }
 
 void rapl_default_set_floor_freq(struct rapl_domain *rd, bool mode)
diff --git a/drivers/pps/clients/pps-gpio.c b/drivers/pps/clients/pps-gpio.c
index 402f910f3e25d..73ec2c7335e51 100644
--- a/drivers/pps/clients/pps-gpio.c
+++ b/drivers/pps/clients/pps-gpio.c
@@ -31,38 +31,37 @@ struct pps_gpio_device_data {
 	struct gpio_desc *echo_pin;
 	struct timer_list echo_timer;	/* timer to reset echo active state */
 	bool assert_falling_edge;
-	bool capture_clear;
 	unsigned int echo_active_ms;	/* PPS echo active duration */
 	unsigned long echo_timeout;	/* timer timeout value in jiffies */
+	struct pps_event_time ts;	/* timestamp captured in hardirq */
 };
 
 /*
  * Report the PPS event
  */
 
-static irqreturn_t pps_gpio_irq_handler(int irq, void *data)
+/*
+ * Primary hardirq handler -- runs in hardirq context even on PREEMPT_RT.
+ * Only captures the timestamp; all other work is deferred to the thread.
+ */
+static irqreturn_t pps_gpio_irq_hardirq(int irq, void *data)
 {
-	const struct pps_gpio_device_data *info;
-	struct pps_event_time ts;
-	int rising_edge;
-
-	/* Get the time stamp first */
-	pps_get_ts(&ts);
-
-	info = data;
-
-	/* Small trick to bypass the check on edge's direction when capture_clear is unset */
-	rising_edge = info->capture_clear ?
-		      gpiod_get_value(info->gpio_pin) : !info->assert_falling_edge;
-	if ((rising_edge && !info->assert_falling_edge) ||
-			(!rising_edge && info->assert_falling_edge))
-		pps_event(info->pps, &ts, PPS_CAPTUREASSERT, data);
-	else if (info->capture_clear &&
-			((rising_edge && info->assert_falling_edge) ||
-			(!rising_edge && !info->assert_falling_edge)))
-		pps_event(info->pps, &ts, PPS_CAPTURECLEAR, data);
-	else
-		dev_warn_ratelimited(&info->pps->dev, "IRQ did not trigger any PPS event\n");
+	struct pps_gpio_device_data *info = data;
+
+	pps_get_ts(&info->ts);
+
+	return IRQ_WAKE_THREAD;
+}
+
+/*
+ * Threaded handler -- processes the PPS event using the timestamp
+ * captured in hardirq context above.
+ */
+static irqreturn_t pps_gpio_irq_thread(int irq, void *data)
+{
+	struct pps_gpio_device_data *info = data;
+
+	pps_event(info->pps, &info->ts, PPS_CAPTUREASSERT, data);
 
 	return IRQ_HANDLED;
 }
@@ -78,11 +77,6 @@ static void pps_gpio_echo(struct pps_device *pps, int event, void *data)
 		if (pps->params.mode & PPS_ECHOASSERT)
 			gpiod_set_value(info->echo_pin, 1);
 		break;
-
-	case PPS_CAPTURECLEAR:
-		if (pps->params.mode & PPS_ECHOCLEAR)
-			gpiod_set_value(info->echo_pin, 1);
-		break;
 	}
 
 	/* fire the timer */
@@ -144,15 +138,8 @@ static int pps_gpio_setup(struct device *dev)
 static unsigned long
 get_irqf_trigger_flags(const struct pps_gpio_device_data *data)
 {
-	unsigned long flags = data->assert_falling_edge ?
-		IRQF_TRIGGER_FALLING : IRQF_TRIGGER_RISING;
-
-	if (data->capture_clear) {
-		flags |= ((flags & IRQF_TRIGGER_RISING) ?
-				IRQF_TRIGGER_FALLING : IRQF_TRIGGER_RISING);
-	}
-
-	return flags;
+	return data->assert_falling_edge ? IRQF_TRIGGER_FALLING :
+					   IRQF_TRIGGER_RISING;
 }
 
 static int pps_gpio_probe(struct platform_device *pdev)
@@ -185,9 +172,6 @@ static int pps_gpio_probe(struct platform_device *pdev)
 	/* initialize PPS specific parts of the bookkeeping data structure. */
 	data->info.mode = PPS_CAPTUREASSERT | PPS_OFFSETASSERT |
 		PPS_ECHOASSERT | PPS_CANWAIT | PPS_TSFMT_TSPEC;
-	if (data->capture_clear)
-		data->info.mode |= PPS_CAPTURECLEAR | PPS_OFFSETCLEAR |
-			PPS_ECHOCLEAR;
 	data->info.owner = THIS_MODULE;
 	snprintf(data->info.name, PPS_MAX_NAME_LEN - 1, "%s.%d",
 		 pdev->name, pdev->id);
@@ -199,8 +183,6 @@ static int pps_gpio_probe(struct platform_device *pdev)
 
 	/* register PPS source */
 	pps_default_params = PPS_CAPTUREASSERT | PPS_OFFSETASSERT;
-	if (data->capture_clear)
-		pps_default_params |= PPS_CAPTURECLEAR | PPS_OFFSETCLEAR;
 	data->pps = pps_register_source(&data->info, pps_default_params);
 	if (IS_ERR(data->pps)) {
 		dev_err(dev, "failed to register IRQ %d as PPS source\n",
@@ -209,8 +191,10 @@ static int pps_gpio_probe(struct platform_device *pdev)
 	}
 
 	/* register IRQ interrupt handler */
-	ret = request_irq(data->irq, pps_gpio_irq_handler,
-			  get_irqf_trigger_flags(data), data->info.name, data);
+	ret = request_threaded_irq(data->irq,
+			  pps_gpio_irq_hardirq, pps_gpio_irq_thread,
+			  get_irqf_trigger_flags(data) | IRQF_ONESHOT,
+			  data->info.name, data);
 	if (ret) {
 		pps_unregister_source(data->pps);
 		dev_err(dev, "failed to acquire IRQ %d\n", data->irq);
diff --git a/drivers/pps/kc.c b/drivers/pps/kc.c
index fbd23295afd7d..4f8fffa7edd64 100644
--- a/drivers/pps/kc.c
+++ b/drivers/pps/kc.c
@@ -38,6 +38,14 @@ int pps_kc_bind(struct pps_device *pps, struct pps_bind_args *bind_args)
 	/* Check if another consumer is already bound */
 	spin_lock_irq(&pps_kc_hardpps_lock);
 
+	/*
+	 * Don't allow PPS_KC_BIND on a removed device.
+	 */
+	if (pps->kc_removed) {
+		spin_unlock_irq(&pps_kc_hardpps_lock);
+		return -ENODEV;
+	}
+
 	if (bind_args->edge == 0)
 		if (pps_kc_hardpps_dev == pps) {
 			pps_kc_hardpps_mode = 0;
@@ -79,6 +87,8 @@ int pps_kc_bind(struct pps_device *pps, struct pps_bind_args *bind_args)
 void pps_kc_remove(struct pps_device *pps)
 {
 	spin_lock_irq(&pps_kc_hardpps_lock);
+
+	pps->kc_removed = true;
 	if (pps == pps_kc_hardpps_dev) {
 		pps_kc_hardpps_mode = 0;
 		pps_kc_hardpps_dev = NULL;
diff --git a/drivers/pps/pps.c b/drivers/pps/pps.c
index de1122bb69ea6..7ecdd774a44b9 100644
--- a/drivers/pps/pps.c
+++ b/drivers/pps/pps.c
@@ -66,13 +66,19 @@ static int pps_cdev_pps_fetch(struct pps_device *pps, struct pps_fdata *fdata)
 		err = wait_event_interruptible(pps->queue,
 				ev != pps->last_ev);
 	else {
+		struct timespec64 ts;
 		unsigned long ticks;
 
 		dev_dbg(&pps->dev, "timeout %lld.%09d\n",
 				(long long) fdata->timeout.sec,
 				fdata->timeout.nsec);
-		ticks = fdata->timeout.sec * HZ;
-		ticks += fdata->timeout.nsec / (NSEC_PER_SEC / HZ);
+
+		if (fdata->timeout.sec < 0)
+			return -ETIMEDOUT;
+
+		ts.tv_sec = fdata->timeout.sec;
+		ts.tv_nsec = fdata->timeout.nsec;
+		ticks = timespec64_to_jiffies(&ts);
 
 		if (ticks != 0) {
 			err = wait_event_interruptible_timeout(
diff --git a/drivers/pwm/Kconfig b/drivers/pwm/Kconfig
index e8886a9b64d96..7297760868790 100644
--- a/drivers/pwm/Kconfig
+++ b/drivers/pwm/Kconfig
@@ -569,7 +569,7 @@ config PWM_PCA9685
 config PWM_PXA
 	tristate "PXA PWM support"
 	depends on ARCH_PXA || ARCH_MMP || ARCH_SPACEMIT || COMPILE_TEST
-	depends on HAS_IOMEM
+	depends on HAS_IOMEM && OF
 	help
 	  Generic PWM framework driver for PXA.
 
diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c
index be4885da85b67..fa0f8f1f6ed02 100644
--- a/drivers/pwm/core.c
+++ b/drivers/pwm/core.c
@@ -2654,7 +2654,7 @@ static void pwm_dbg_show(struct pwm_chip *chip, struct seq_file *s)
 			   state.polarity ? "inverse" : "normal");
 		if (state.usage_power)
 			seq_puts(s, ", usage_power");
-		seq_puts(s, "\n");
+		seq_putc(s, '\n');
 
 		if (pwmchip_supports_waveform(chip)) {
 			struct pwm_waveform wf;
@@ -2677,7 +2677,7 @@ static void pwm_dbg_show(struct pwm_chip *chip, struct seq_file *s)
 				seq_printf(s, "  actual configuration: read out error: %pe", ERR_PTR(err));
 		}
 
-		seq_puts(s, "\n");
+		seq_putc(s, '\n');
 	}
 }
 
diff --git a/drivers/pwm/pwm-adp5585.c b/drivers/pwm/pwm-adp5585.c
index 0644ff40f0fba..9a1dbba75bc14 100644
--- a/drivers/pwm/pwm-adp5585.c
+++ b/drivers/pwm/pwm-adp5585.c
@@ -202,8 +202,8 @@ static const struct adp5585_pwm_chip adp5589_pwm_chip_info = {
 };
 
 static const struct platform_device_id adp5585_pwm_id_table[] = {
-	{ "adp5585-pwm", (kernel_ulong_t)&adp5585_pwm_chip_info },
-	{ "adp5589-pwm", (kernel_ulong_t)&adp5589_pwm_chip_info },
+	{ .name = "adp5585-pwm", .driver_data = (kernel_ulong_t)&adp5585_pwm_chip_info },
+	{ .name = "adp5589-pwm", .driver_data = (kernel_ulong_t)&adp5589_pwm_chip_info },
 	{ /* Sentinel */ }
 };
 MODULE_DEVICE_TABLE(platform, adp5585_pwm_id_table);
diff --git a/drivers/pwm/pwm-apple.c b/drivers/pwm/pwm-apple.c
index aa49ca1c30e8c..dac8c9a76b2d1 100644
--- a/drivers/pwm/pwm-apple.c
+++ b/drivers/pwm/pwm-apple.c
@@ -141,7 +141,7 @@ static int apple_pwm_probe(struct platform_device *pdev)
 
 static const struct of_device_id apple_pwm_of_match[] = {
 	{ .compatible = "apple,s5l-fpwm" },
-	{}
+	{ }
 };
 MODULE_DEVICE_TABLE(of, apple_pwm_of_match);
 
diff --git a/drivers/pwm/pwm-argon-fan-hat.c b/drivers/pwm/pwm-argon-fan-hat.c
index 2c59bd142d40c..b480663e87d59 100644
--- a/drivers/pwm/pwm-argon-fan-hat.c
+++ b/drivers/pwm/pwm-argon-fan-hat.c
@@ -89,7 +89,7 @@ static int argon_fan_hat_i2c_probe(struct i2c_client *i2c)
 
 static const struct of_device_id argon_fan_hat_dt_ids[] = {
 	{ .compatible = "argon40,fan-hat" },
-	{ },
+	{ }
 };
 MODULE_DEVICE_TABLE(of, argon_fan_hat_dt_ids);
 
diff --git a/drivers/pwm/pwm-atmel-hlcdc.c b/drivers/pwm/pwm-atmel-hlcdc.c
index 387a0d1fa4f2d..1ba2c9a049b95 100644
--- a/drivers/pwm/pwm-atmel-hlcdc.c
+++ b/drivers/pwm/pwm-atmel-hlcdc.c
@@ -217,23 +217,20 @@ static const struct of_device_id atmel_hlcdc_dt_ids[] = {
 		.compatible = "atmel,at91sam9n12-hlcdc",
 		/* 9n12 has same errata as 9x5 HLCDC PWM */
 		.data = &atmel_hlcdc_pwm_at91sam9x5_errata,
-	},
-	{
+	}, {
 		.compatible = "atmel,at91sam9x5-hlcdc",
 		.data = &atmel_hlcdc_pwm_at91sam9x5_errata,
-	},
-	{
+	}, {
 		.compatible = "atmel,sama5d2-hlcdc",
-	},
-	{
+	}, {
 		.compatible = "atmel,sama5d3-hlcdc",
 		.data = &atmel_hlcdc_pwm_sama5d3_errata,
-	},
-	{
+	}, {
 		.compatible = "atmel,sama5d4-hlcdc",
 		.data = &atmel_hlcdc_pwm_sama5d3_errata,
+	}, {
+		.compatible = "microchip,sam9x60-hlcdc",
 	},
-	{	.compatible = "microchip,sam9x60-hlcdc", },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, atmel_hlcdc_dt_ids);
diff --git a/drivers/pwm/pwm-atmel-tcb.c b/drivers/pwm/pwm-atmel-tcb.c
index a765ef279b512..f74a42d75b81c 100644
--- a/drivers/pwm/pwm-atmel-tcb.c
+++ b/drivers/pwm/pwm-atmel-tcb.c
@@ -374,9 +374,9 @@ static struct atmel_tcb_config tcb_sama5d2_config = {
 };
 
 static const struct of_device_id atmel_tcb_of_match[] = {
-	{ .compatible = "atmel,at91rm9200-tcb", .data = &tcb_rm9200_config, },
-	{ .compatible = "atmel,at91sam9x5-tcb", .data = &tcb_sam9x5_config, },
-	{ .compatible = "atmel,sama5d2-tcb", .data = &tcb_sama5d2_config, },
+	{ .compatible = "atmel,at91rm9200-tcb", .data = &tcb_rm9200_config },
+	{ .compatible = "atmel,at91sam9x5-tcb", .data = &tcb_sam9x5_config },
+	{ .compatible = "atmel,sama5d2-tcb", .data = &tcb_sama5d2_config },
 	{ /* sentinel */ }
 };
 
diff --git a/drivers/pwm/pwm-atmel.c b/drivers/pwm/pwm-atmel.c
index 06d22d0f7b261..86918523d8213 100644
--- a/drivers/pwm/pwm-atmel.c
+++ b/drivers/pwm/pwm-atmel.c
@@ -441,9 +441,8 @@ static const struct of_device_id atmel_pwm_dt_ids[] = {
 	}, {
 		.compatible = "microchip,sam9x60-pwm",
 		.data = &mchp_sam9x60_pwm_data,
-	}, {
-		/* sentinel */
 	},
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, atmel_pwm_dt_ids);
 
diff --git a/drivers/pwm/pwm-bcm-iproc.c b/drivers/pwm/pwm-bcm-iproc.c
index f4c9f10e490ee..dc1e792b1e08e 100644
--- a/drivers/pwm/pwm-bcm-iproc.c
+++ b/drivers/pwm/pwm-bcm-iproc.c
@@ -230,7 +230,7 @@ static int iproc_pwmc_probe(struct platform_device *pdev)
 
 static const struct of_device_id bcm_iproc_pwmc_dt[] = {
 	{ .compatible = "brcm,iproc-pwm" },
-	{ },
+	{ }
 };
 MODULE_DEVICE_TABLE(of, bcm_iproc_pwmc_dt);
 
diff --git a/drivers/pwm/pwm-bcm-kona.c b/drivers/pwm/pwm-bcm-kona.c
index 022c078aae842..ecfa3e3707f4b 100644
--- a/drivers/pwm/pwm-bcm-kona.c
+++ b/drivers/pwm/pwm-bcm-kona.c
@@ -319,7 +319,7 @@ static int kona_pwmc_probe(struct platform_device *pdev)
 
 static const struct of_device_id bcm_kona_pwmc_dt[] = {
 	{ .compatible = "brcm,kona-pwm" },
-	{ },
+	{ }
 };
 MODULE_DEVICE_TABLE(of, bcm_kona_pwmc_dt);
 
diff --git a/drivers/pwm/pwm-bcm2835.c b/drivers/pwm/pwm-bcm2835.c
index 532903da521fd..a4fbf9cfc6f73 100644
--- a/drivers/pwm/pwm-bcm2835.c
+++ b/drivers/pwm/pwm-bcm2835.c
@@ -165,7 +165,7 @@ static DEFINE_SIMPLE_DEV_PM_OPS(bcm2835_pwm_pm_ops, bcm2835_pwm_suspend,
 				bcm2835_pwm_resume);
 
 static const struct of_device_id bcm2835_pwm_of_match[] = {
-	{ .compatible = "brcm,bcm2835-pwm", },
+	{ .compatible = "brcm,bcm2835-pwm" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, bcm2835_pwm_of_match);
diff --git a/drivers/pwm/pwm-berlin.c b/drivers/pwm/pwm-berlin.c
index da9954818302d..d36a245b6b4f0 100644
--- a/drivers/pwm/pwm-berlin.c
+++ b/drivers/pwm/pwm-berlin.c
@@ -190,7 +190,7 @@ static const struct pwm_ops berlin_pwm_ops = {
 
 static const struct of_device_id berlin_pwm_match[] = {
 	{ .compatible = "marvell,berlin-pwm" },
-	{ },
+	{ }
 };
 MODULE_DEVICE_TABLE(of, berlin_pwm_match);
 
diff --git a/drivers/pwm/pwm-brcmstb.c b/drivers/pwm/pwm-brcmstb.c
index 82d27d07ba91b..790ef7ffbbe57 100644
--- a/drivers/pwm/pwm-brcmstb.c
+++ b/drivers/pwm/pwm-brcmstb.c
@@ -222,7 +222,7 @@ static const struct pwm_ops brcmstb_pwm_ops = {
 };
 
 static const struct of_device_id brcmstb_pwm_of_match[] = {
-	{ .compatible = "brcm,bcm7038-pwm", },
+	{ .compatible = "brcm,bcm7038-pwm" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, brcmstb_pwm_of_match);
diff --git a/drivers/pwm/pwm-clk.c b/drivers/pwm/pwm-clk.c
index f8f5af57acba2..1907f2f10e5c7 100644
--- a/drivers/pwm/pwm-clk.c
+++ b/drivers/pwm/pwm-clk.c
@@ -119,7 +119,7 @@ static void pwm_clk_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id pwm_clk_dt_ids[] = {
-	{ .compatible = "clk-pwm", },
+	{ .compatible = "clk-pwm" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, pwm_clk_dt_ids);
diff --git a/drivers/pwm/pwm-clps711x.c b/drivers/pwm/pwm-clps711x.c
index 2c92ce7548723..1ce12ee0329bc 100644
--- a/drivers/pwm/pwm-clps711x.c
+++ b/drivers/pwm/pwm-clps711x.c
@@ -91,7 +91,7 @@ static int clps711x_pwm_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id clps711x_pwm_dt_ids[] = {
-	{ .compatible = "cirrus,ep7209-pwm", },
+	{ .compatible = "cirrus,ep7209-pwm" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, clps711x_pwm_dt_ids);
diff --git a/drivers/pwm/pwm-cros-ec.c b/drivers/pwm/pwm-cros-ec.c
index 67cfa17f58e0d..08a24a45c76d5 100644
--- a/drivers/pwm/pwm-cros-ec.c
+++ b/drivers/pwm/pwm-cros-ec.c
@@ -267,7 +267,7 @@ static int cros_ec_pwm_probe(struct platform_device *pdev)
 static const struct of_device_id cros_ec_pwm_of_match[] = {
 	{ .compatible = "google,cros-ec-pwm" },
 	{ .compatible = "google,cros-ec-pwm-type" },
-	{},
+	{ }
 };
 MODULE_DEVICE_TABLE(of, cros_ec_pwm_of_match);
 #endif
diff --git a/drivers/pwm/pwm-hibvt.c b/drivers/pwm/pwm-hibvt.c
index e02ee6383dbce..29ad8d0b5536b 100644
--- a/drivers/pwm/pwm-hibvt.c
+++ b/drivers/pwm/pwm-hibvt.c
@@ -266,7 +266,7 @@ static const struct of_device_id hibvt_pwm_of_match[] = {
 	  .data = &hi3559v100_shub_soc_info },
 	{ .compatible = "hisilicon,hi3559v100-pwm",
 	  .data = &hi3559v100_soc_info },
-	{  }
+	{ }
 };
 MODULE_DEVICE_TABLE(of, hibvt_pwm_of_match);
 
diff --git a/drivers/pwm/pwm-imx-tpm.c b/drivers/pwm/pwm-imx-tpm.c
index 80fdb3303400f..c20753a36d6d1 100644
--- a/drivers/pwm/pwm-imx-tpm.c
+++ b/drivers/pwm/pwm-imx-tpm.c
@@ -442,7 +442,7 @@ static DEFINE_SIMPLE_DEV_PM_OPS(imx_tpm_pwm_pm,
 				pwm_imx_tpm_suspend, pwm_imx_tpm_resume);
 
 static const struct of_device_id imx_tpm_pwm_dt_ids[] = {
-	{ .compatible = "fsl,imx7ulp-pwm", },
+	{ .compatible = "fsl,imx7ulp-pwm" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, imx_tpm_pwm_dt_ids);
diff --git a/drivers/pwm/pwm-imx1.c b/drivers/pwm/pwm-imx1.c
index d5535d2080059..075d46180dfff 100644
--- a/drivers/pwm/pwm-imx1.c
+++ b/drivers/pwm/pwm-imx1.c
@@ -151,7 +151,7 @@ static const struct pwm_ops pwm_imx1_ops = {
 };
 
 static const struct of_device_id pwm_imx1_dt_ids[] = {
-	{ .compatible = "fsl,imx1-pwm", },
+	{ .compatible = "fsl,imx1-pwm" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, pwm_imx1_dt_ids);
diff --git a/drivers/pwm/pwm-imx27.c b/drivers/pwm/pwm-imx27.c
index c8b801fcb5251..7c1ad15008d28 100644
--- a/drivers/pwm/pwm-imx27.c
+++ b/drivers/pwm/pwm-imx27.c
@@ -378,7 +378,7 @@ static const struct pwm_ops pwm_imx27_ops = {
 };
 
 static const struct of_device_id pwm_imx27_dt_ids[] = {
-	{ .compatible = "fsl,imx27-pwm", },
+	{ .compatible = "fsl,imx27-pwm" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, pwm_imx27_dt_ids);
diff --git a/drivers/pwm/pwm-ipq.c b/drivers/pwm/pwm-ipq.c
index c533739481361..6b7aec566f7eb 100644
--- a/drivers/pwm/pwm-ipq.c
+++ b/drivers/pwm/pwm-ipq.c
@@ -245,8 +245,8 @@ static int ipq_pwm_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id pwm_ipq_dt_match[] = {
-	{ .compatible = "qcom,ipq6018-pwm", },
-	{}
+	{ .compatible = "qcom,ipq6018-pwm" },
+	{ }
 };
 MODULE_DEVICE_TABLE(of, pwm_ipq_dt_match);
 
diff --git a/drivers/pwm/pwm-jz4740.c b/drivers/pwm/pwm-jz4740.c
index e0b5966fc7fea..0db1474c7c8c2 100644
--- a/drivers/pwm/pwm-jz4740.c
+++ b/drivers/pwm/pwm-jz4740.c
@@ -261,7 +261,7 @@ static const struct of_device_id jz4740_pwm_dt_ids[] = {
 	{ .compatible = "ingenic,jz4740-pwm", .data = &jz4740_soc_info },
 	{ .compatible = "ingenic,jz4725b-pwm", .data = &jz4725b_soc_info },
 	{ .compatible = "ingenic,x1000-pwm", .data = &x1000_soc_info },
-	{},
+	{ }
 };
 MODULE_DEVICE_TABLE(of, jz4740_pwm_dt_ids);
 
diff --git a/drivers/pwm/pwm-loongson.c b/drivers/pwm/pwm-loongson.c
index 31a57edecfd0b..f2fb35b7af2b3 100644
--- a/drivers/pwm/pwm-loongson.c
+++ b/drivers/pwm/pwm-loongson.c
@@ -269,7 +269,7 @@ static const struct of_device_id pwm_loongson_of_ids[] = {
 MODULE_DEVICE_TABLE(of, pwm_loongson_of_ids);
 
 static const struct acpi_device_id pwm_loongson_acpi_ids[] = {
-	{ "LOON0006" },
+	{ .id = "LOON0006" },
 	{ }
 };
 MODULE_DEVICE_TABLE(acpi, pwm_loongson_acpi_ids);
diff --git a/drivers/pwm/pwm-lp3943.c b/drivers/pwm/pwm-lp3943.c
index 90b0733c00c1b..10537e74be288 100644
--- a/drivers/pwm/pwm-lp3943.c
+++ b/drivers/pwm/pwm-lp3943.c
@@ -296,7 +296,7 @@ static int lp3943_pwm_probe(struct platform_device *pdev)
 
 #ifdef CONFIG_OF
 static const struct of_device_id lp3943_pwm_of_match[] = {
-	{ .compatible = "ti,lp3943-pwm", },
+	{ .compatible = "ti,lp3943-pwm" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, lp3943_pwm_of_match);
diff --git a/drivers/pwm/pwm-lpc18xx-sct.c b/drivers/pwm/pwm-lpc18xx-sct.c
index 01d4717251060..59e9b58a31bc2 100644
--- a/drivers/pwm/pwm-lpc18xx-sct.c
+++ b/drivers/pwm/pwm-lpc18xx-sct.c
@@ -329,7 +329,7 @@ static const struct pwm_ops lpc18xx_pwm_ops = {
 
 static const struct of_device_id lpc18xx_pwm_of_match[] = {
 	{ .compatible = "nxp,lpc1850-sct-pwm" },
-	{}
+	{ }
 };
 MODULE_DEVICE_TABLE(of, lpc18xx_pwm_of_match);
 
diff --git a/drivers/pwm/pwm-lpc32xx.c b/drivers/pwm/pwm-lpc32xx.c
index c748537e57d1a..19ec53648f406 100644
--- a/drivers/pwm/pwm-lpc32xx.c
+++ b/drivers/pwm/pwm-lpc32xx.c
@@ -155,7 +155,7 @@ static int lpc32xx_pwm_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id lpc32xx_pwm_dt_ids[] = {
-	{ .compatible = "nxp,lpc3220-pwm", },
+	{ .compatible = "nxp,lpc3220-pwm" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, lpc32xx_pwm_dt_ids);
diff --git a/drivers/pwm/pwm-lpss-pci.c b/drivers/pwm/pwm-lpss-pci.c
index 3a0fd65935201..bcbfda1a6624f 100644
--- a/drivers/pwm/pwm-lpss-pci.c
+++ b/drivers/pwm/pwm-lpss-pci.c
@@ -57,7 +57,7 @@ static const struct pci_device_id pwm_lpss_pci_ids[] = {
 	{ PCI_VDEVICE(INTEL, 0x2289), .driver_data = (unsigned long)&pwm_lpss_bsw_info },
 	{ PCI_VDEVICE(INTEL, 0x31c8), .driver_data = (unsigned long)&pwm_lpss_bxt_info },
 	{ PCI_VDEVICE(INTEL, 0x5ac8), .driver_data = (unsigned long)&pwm_lpss_bxt_info },
-	{ },
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, pwm_lpss_pci_ids);
 
diff --git a/drivers/pwm/pwm-lpss-platform.c b/drivers/pwm/pwm-lpss-platform.c
index 8b95064b87031..489ca0c09c041 100644
--- a/drivers/pwm/pwm-lpss-platform.c
+++ b/drivers/pwm/pwm-lpss-platform.c
@@ -58,11 +58,11 @@ static int pwm_lpss_probe_platform(struct platform_device *pdev)
 }
 
 static const struct acpi_device_id pwm_lpss_acpi_match[] = {
-	{ "80860F09", (unsigned long)&pwm_lpss_byt_info },
-	{ "80862288", (unsigned long)&pwm_lpss_bsw_info },
-	{ "80862289", (unsigned long)&pwm_lpss_bsw_info },
-	{ "80865AC8", (unsigned long)&pwm_lpss_bxt_info },
-	{ },
+	{ .id = "80860F09", .driver_data = (unsigned long)&pwm_lpss_byt_info },
+	{ .id = "80862288", .driver_data = (unsigned long)&pwm_lpss_bsw_info },
+	{ .id = "80862289", .driver_data = (unsigned long)&pwm_lpss_bsw_info },
+	{ .id = "80865AC8", .driver_data = (unsigned long)&pwm_lpss_bxt_info },
+	{ }
 };
 MODULE_DEVICE_TABLE(acpi, pwm_lpss_acpi_match);
 
diff --git a/drivers/pwm/pwm-mc33xs2410.c b/drivers/pwm/pwm-mc33xs2410.c
index 6d99e3ff72398..9f89b60dda785 100644
--- a/drivers/pwm/pwm-mc33xs2410.c
+++ b/drivers/pwm/pwm-mc33xs2410.c
@@ -381,7 +381,7 @@ static int mc33xs2410_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id mc33xs2410_spi_id[] = {
-	{ "mc33xs2410" },
+	{ .name = "mc33xs2410" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, mc33xs2410_spi_id);
diff --git a/drivers/pwm/pwm-mediatek.c b/drivers/pwm/pwm-mediatek.c
index 992137a277501..0bb49be657b7f 100644
--- a/drivers/pwm/pwm-mediatek.c
+++ b/drivers/pwm/pwm-mediatek.c
@@ -627,7 +627,7 @@ static const struct of_device_id pwm_mediatek_of_match[] = {
 	{ .compatible = "mediatek,mt8183-pwm", .data = &mt8183_pwm_data },
 	{ .compatible = "mediatek,mt8365-pwm", .data = &mt8365_pwm_data },
 	{ .compatible = "mediatek,mt8516-pwm", .data = &mt8516_pwm_data },
-	{ },
+	{ }
 };
 MODULE_DEVICE_TABLE(of, pwm_mediatek_of_match);
 
diff --git a/drivers/pwm/pwm-meson.c b/drivers/pwm/pwm-meson.c
index 8c6bf3d49753d..20e6cd07127c4 100644
--- a/drivers/pwm/pwm-meson.c
+++ b/drivers/pwm/pwm-meson.c
@@ -596,15 +596,14 @@ static const struct of_device_id meson_pwm_matches[] = {
 	{
 		.compatible = "amlogic,meson8-pwm-v2",
 		.data = &pwm_meson8_v2_data
-	},
-	{
+	}, {
 		.compatible = "amlogic,meson-axg-pwm-v2",
 		.data = &pwm_meson_axg_v2_data
-	},
-	{
+	}, {
 		.compatible = "amlogic,meson-g12-pwm-v2",
 		.data = &pwm_meson_axg_v2_data
 	},
+
 	/* The following compatibles are obsolete */
 	{
 		.compatible = "amlogic,meson8b-pwm",
@@ -642,7 +641,7 @@ static const struct of_device_id meson_pwm_matches[] = {
 		.compatible = "amlogic,meson-s4-pwm",
 		.data = &pwm_s4_data
 	},
-	{},
+	{ }
 };
 MODULE_DEVICE_TABLE(of, meson_pwm_matches);
 
diff --git a/drivers/pwm/pwm-mtk-disp.c b/drivers/pwm/pwm-mtk-disp.c
index bafd6b6195f66..b5409e15dac86 100644
--- a/drivers/pwm/pwm-mtk-disp.c
+++ b/drivers/pwm/pwm-mtk-disp.c
@@ -295,10 +295,10 @@ static const struct mtk_pwm_data mt8183_pwm_data = {
 };
 
 static const struct of_device_id mtk_disp_pwm_of_match[] = {
-	{ .compatible = "mediatek,mt2701-disp-pwm", .data = &mt2701_pwm_data},
-	{ .compatible = "mediatek,mt6595-disp-pwm", .data = &mt8173_pwm_data},
-	{ .compatible = "mediatek,mt8173-disp-pwm", .data = &mt8173_pwm_data},
-	{ .compatible = "mediatek,mt8183-disp-pwm", .data = &mt8183_pwm_data},
+	{ .compatible = "mediatek,mt2701-disp-pwm", .data = &mt2701_pwm_data },
+	{ .compatible = "mediatek,mt6595-disp-pwm", .data = &mt8173_pwm_data },
+	{ .compatible = "mediatek,mt8173-disp-pwm", .data = &mt8173_pwm_data },
+	{ .compatible = "mediatek,mt8183-disp-pwm", .data = &mt8183_pwm_data },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, mtk_disp_pwm_of_match);
diff --git a/drivers/pwm/pwm-mxs.c b/drivers/pwm/pwm-mxs.c
index 8cad214b1c299..e1a3df6549fc9 100644
--- a/drivers/pwm/pwm-mxs.c
+++ b/drivers/pwm/pwm-mxs.c
@@ -163,7 +163,7 @@ static int mxs_pwm_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id mxs_pwm_dt_ids[] = {
-	{ .compatible = "fsl,imx23-pwm", },
+	{ .compatible = "fsl,imx23-pwm" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, mxs_pwm_dt_ids);
diff --git a/drivers/pwm/pwm-omap-dmtimer.c b/drivers/pwm/pwm-omap-dmtimer.c
index 1858a77401f80..51830a87275cf 100644
--- a/drivers/pwm/pwm-omap-dmtimer.c
+++ b/drivers/pwm/pwm-omap-dmtimer.c
@@ -444,8 +444,8 @@ static void pwm_omap_dmtimer_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id pwm_omap_dmtimer_of_match[] = {
-	{.compatible = "ti,omap-dmtimer-pwm"},
-	{}
+	{ .compatible = "ti,omap-dmtimer-pwm" },
+	{ }
 };
 MODULE_DEVICE_TABLE(of, pwm_omap_dmtimer_of_match);
 
diff --git a/drivers/pwm/pwm-pca9685.c b/drivers/pwm/pwm-pca9685.c
index a02255a64ea81..a2792b8036437 100644
--- a/drivers/pwm/pwm-pca9685.c
+++ b/drivers/pwm/pwm-pca9685.c
@@ -544,13 +544,13 @@ static const struct i2c_device_id pca9685_id[] = {
 MODULE_DEVICE_TABLE(i2c, pca9685_id);
 
 static const struct acpi_device_id pca9685_acpi_ids[] = {
-	{ "INT3492", 0 },
-	{ /* sentinel */ },
+	{ .id = "INT3492" },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(acpi, pca9685_acpi_ids);
 
 static const struct of_device_id pca9685_dt_ids[] = {
-	{ .compatible = "nxp,pca9685-pwm", },
+	{ .compatible = "nxp,pca9685-pwm" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, pca9685_dt_ids);
diff --git a/drivers/pwm/pwm-pxa.c b/drivers/pwm/pwm-pxa.c
index 156c0c74cd801..5f06088f82d46 100644
--- a/drivers/pwm/pwm-pxa.c
+++ b/drivers/pwm/pwm-pxa.c
@@ -31,12 +31,12 @@
 #define HAS_SECONDARY_PWM	0x10
 
 static const struct platform_device_id pwm_id_table[] = {
-	/*   PWM    has_secondary_pwm? */
-	{ "pxa25x-pwm", 0 },
-	{ "pxa27x-pwm", HAS_SECONDARY_PWM },
-	{ "pxa168-pwm", 0 },
-	{ "pxa910-pwm", 0 },
-	{ },
+	/*             PWM            has_secondary_pwm? */
+	{ .name = "pxa25x-pwm", .driver_data = 0 },
+	{ .name = "pxa27x-pwm", .driver_data = HAS_SECONDARY_PWM },
+	{ .name = "pxa168-pwm", .driver_data = 0 },
+	{ .name = "pxa910-pwm", .driver_data = 0 },
+	{ }
 };
 MODULE_DEVICE_TABLE(platform, pwm_id_table);
 
@@ -136,7 +136,6 @@ static const struct pwm_ops pxa_pwm_ops = {
 	.apply = pxa_pwm_apply,
 };
 
-#ifdef CONFIG_OF
 /*
  * Device tree users must create one device instance for each PWM channel.
  * Hence we dispense with the HAS_SECONDARY_PWM and "tell" the original driver
@@ -144,16 +143,13 @@ static const struct pwm_ops pxa_pwm_ops = {
  * supported identically.
  */
 static const struct of_device_id pwm_of_match[] = {
-	{ .compatible = "marvell,pxa250-pwm", .data = &pwm_id_table[0]},
-	{ .compatible = "marvell,pxa270-pwm", .data = &pwm_id_table[0]},
-	{ .compatible = "marvell,pxa168-pwm", .data = &pwm_id_table[0]},
-	{ .compatible = "marvell,pxa910-pwm", .data = &pwm_id_table[0]},
+	{ .compatible = "marvell,pxa250-pwm", .data = &pwm_id_table[0] },
+	{ .compatible = "marvell,pxa270-pwm", .data = &pwm_id_table[0] },
+	{ .compatible = "marvell,pxa168-pwm", .data = &pwm_id_table[0] },
+	{ .compatible = "marvell,pxa910-pwm", .data = &pwm_id_table[0] },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, pwm_of_match);
-#else
-#define pwm_of_match NULL
-#endif
 
 static int pwm_probe(struct platform_device *pdev)
 {
@@ -165,7 +161,7 @@ static int pwm_probe(struct platform_device *pdev)
 	struct reset_control *rst;
 	int ret = 0;
 
-	if (IS_ENABLED(CONFIG_OF) && id == NULL)
+	if (id == NULL)
 		id = of_device_get_match_data(dev);
 
 	if (id == NULL)
@@ -191,9 +187,7 @@ static int pwm_probe(struct platform_device *pdev)
 		return PTR_ERR(rst);
 
 	chip->ops = &pxa_pwm_ops;
-
-	if (IS_ENABLED(CONFIG_OF))
-		chip->of_xlate = of_pwm_single_xlate;
+	chip->of_xlate = of_pwm_single_xlate;
 
 	pc->mmio_base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(pc->mmio_base))
diff --git a/drivers/pwm/pwm-raspberrypi-poe.c b/drivers/pwm/pwm-raspberrypi-poe.c
index 8921e7ea2ceaa..7cea6561c7305 100644
--- a/drivers/pwm/pwm-raspberrypi-poe.c
+++ b/drivers/pwm/pwm-raspberrypi-poe.c
@@ -177,7 +177,7 @@ static int raspberrypi_pwm_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id raspberrypi_pwm_of_match[] = {
-	{ .compatible = "raspberrypi,firmware-poe-pwm", },
+	{ .compatible = "raspberrypi,firmware-poe-pwm" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, raspberrypi_pwm_of_match);
diff --git a/drivers/pwm/pwm-rcar.c b/drivers/pwm/pwm-rcar.c
index 578dbdd2d5a72..1733cd7006937 100644
--- a/drivers/pwm/pwm-rcar.c
+++ b/drivers/pwm/pwm-rcar.c
@@ -248,8 +248,8 @@ static void rcar_pwm_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id rcar_pwm_of_table[] = {
-	{ .compatible = "renesas,pwm-rcar", },
-	{ },
+	{ .compatible = "renesas,pwm-rcar" },
+	{ }
 };
 MODULE_DEVICE_TABLE(of, rcar_pwm_of_table);
 
diff --git a/drivers/pwm/pwm-renesas-tpu.c b/drivers/pwm/pwm-renesas-tpu.c
index 2196080b41770..140fd0bcb93f3 100644
--- a/drivers/pwm/pwm-renesas-tpu.c
+++ b/drivers/pwm/pwm-renesas-tpu.c
@@ -479,11 +479,11 @@ static int tpu_probe(struct platform_device *pdev)
 
 #ifdef CONFIG_OF
 static const struct of_device_id tpu_of_table[] = {
-	{ .compatible = "renesas,tpu-r8a73a4", },
-	{ .compatible = "renesas,tpu-r8a7740", },
-	{ .compatible = "renesas,tpu-r8a7790", },
-	{ .compatible = "renesas,tpu", },
-	{ },
+	{ .compatible = "renesas,tpu-r8a73a4" },
+	{ .compatible = "renesas,tpu-r8a7740" },
+	{ .compatible = "renesas,tpu-r8a7790" },
+	{ .compatible = "renesas,tpu" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(of, tpu_of_table);
diff --git a/drivers/pwm/pwm-rockchip.c b/drivers/pwm/pwm-rockchip.c
index 67b85bdb491b1..050ca8aad0cad 100644
--- a/drivers/pwm/pwm-rockchip.c
+++ b/drivers/pwm/pwm-rockchip.c
@@ -292,10 +292,10 @@ static const struct rockchip_pwm_data pwm_data_v3 = {
 };
 
 static const struct of_device_id rockchip_pwm_dt_ids[] = {
-	{ .compatible = "rockchip,rk2928-pwm", .data = &pwm_data_v1},
-	{ .compatible = "rockchip,rk3288-pwm", .data = &pwm_data_v2},
-	{ .compatible = "rockchip,vop-pwm", .data = &pwm_data_vop},
-	{ .compatible = "rockchip,rk3328-pwm", .data = &pwm_data_v3},
+	{ .compatible = "rockchip,rk2928-pwm", .data = &pwm_data_v1 },
+	{ .compatible = "rockchip,rk3288-pwm", .data = &pwm_data_v2 },
+	{ .compatible = "rockchip,vop-pwm", .data = &pwm_data_vop },
+	{ .compatible = "rockchip,rk3328-pwm", .data = &pwm_data_v3 },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, rockchip_pwm_dt_ids);
diff --git a/drivers/pwm/pwm-rzg2l-gpt.c b/drivers/pwm/pwm-rzg2l-gpt.c
index dfa1d11a48a82..1569f3ba8c537 100644
--- a/drivers/pwm/pwm-rzg2l-gpt.c
+++ b/drivers/pwm/pwm-rzg2l-gpt.c
@@ -118,8 +118,7 @@ static void rzg2l_gpt_modify(struct rzg2l_gpt_chip *rzg2l_gpt, u32 reg, u32 clr,
 			(rzg2l_gpt_read(rzg2l_gpt, reg) & ~clr) | set);
 }
 
-static u8 rzg2l_gpt_calculate_prescale(struct rzg2l_gpt_chip *rzg2l_gpt,
-				       u64 period_ticks)
+static u8 rzg2l_gpt_calculate_prescale(u64 period_ticks)
 {
 	u32 prescaled_period_ticks;
 	u8 prescale;
@@ -286,7 +285,7 @@ static int rzg2l_gpt_config(struct pwm_chip *chip, struct pwm_device *pwm,
 		}
 	}
 
-	prescale = rzg2l_gpt_calculate_prescale(rzg2l_gpt, period_ticks);
+	prescale = rzg2l_gpt_calculate_prescale(period_ticks);
 	pv = rzg2l_gpt_calculate_pv_or_dc(period_ticks, prescale);
 
 	duty_ticks = mul_u64_u64_div_u64(state->duty_cycle, rzg2l_gpt->rate_khz, USEC_PER_SEC);
@@ -437,7 +436,7 @@ static int rzg2l_gpt_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id rzg2l_gpt_of_table[] = {
-	{ .compatible = "renesas,rzg2l-gpt", },
+	{ .compatible = "renesas,rzg2l-gpt" },
 	{ /* Sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, rzg2l_gpt_of_table);
diff --git a/drivers/pwm/pwm-samsung.c b/drivers/pwm/pwm-samsung.c
index 951b38ff5f8ec..331e81f1594a4 100644
--- a/drivers/pwm/pwm-samsung.c
+++ b/drivers/pwm/pwm-samsung.c
@@ -501,7 +501,7 @@ static const struct of_device_id samsung_pwm_matches[] = {
 	{ .compatible = "samsung,s5p6440-pwm", .data = &s5p64x0_variant },
 	{ .compatible = "samsung,s5pc100-pwm", .data = &s5pc100_variant },
 	{ .compatible = "samsung,exynos4210-pwm", .data = &s5p64x0_variant },
-	{},
+	{ }
 };
 MODULE_DEVICE_TABLE(of, samsung_pwm_matches);
 
diff --git a/drivers/pwm/pwm-sifive.c b/drivers/pwm/pwm-sifive.c
index e11ecf1fa0f92..4d0e4512d8a57 100644
--- a/drivers/pwm/pwm-sifive.c
+++ b/drivers/pwm/pwm-sifive.c
@@ -355,7 +355,7 @@ static void pwm_sifive_remove(struct platform_device *dev)
 
 static const struct of_device_id pwm_sifive_of_match[] = {
 	{ .compatible = "sifive,pwm0" },
-	{},
+	{ }
 };
 MODULE_DEVICE_TABLE(of, pwm_sifive_of_match);
 
diff --git a/drivers/pwm/pwm-sl28cpld.c b/drivers/pwm/pwm-sl28cpld.c
index 0dc2e3f809c34..0e25a05ea5386 100644
--- a/drivers/pwm/pwm-sl28cpld.c
+++ b/drivers/pwm/pwm-sl28cpld.c
@@ -244,7 +244,7 @@ static int sl28cpld_pwm_probe(struct platform_device *pdev)
 
 static const struct of_device_id sl28cpld_pwm_of_match[] = {
 	{ .compatible = "kontron,sl28cpld-pwm" },
-	{}
+	{ }
 };
 MODULE_DEVICE_TABLE(of, sl28cpld_pwm_of_match);
 
diff --git a/drivers/pwm/pwm-sophgo-sg2042.c b/drivers/pwm/pwm-sophgo-sg2042.c
index 7d07b0ca7d298..0b6461172e6ab 100644
--- a/drivers/pwm/pwm-sophgo-sg2042.c
+++ b/drivers/pwm/pwm-sophgo-sg2042.c
@@ -225,11 +225,10 @@ static const struct sg2042_chip_data sg2044_chip_data = {
 static const struct of_device_id sg2042_pwm_ids[] = {
 	{
 		.compatible = "sophgo,sg2042-pwm",
-		.data = &sg2042_chip_data
-	},
-	{
+		.data = &sg2042_chip_data,
+	}, {
 		.compatible = "sophgo,sg2044-pwm",
-		.data = &sg2044_chip_data
+		.data = &sg2044_chip_data,
 	},
 	{ }
 };
diff --git a/drivers/pwm/pwm-sprd.c b/drivers/pwm/pwm-sprd.c
index 438dbaa3a98f5..51cab6a54fe2e 100644
--- a/drivers/pwm/pwm-sprd.c
+++ b/drivers/pwm/pwm-sprd.c
@@ -276,8 +276,8 @@ static int sprd_pwm_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id sprd_pwm_of_match[] = {
-	{ .compatible = "sprd,ums512-pwm", },
-	{ },
+	{ .compatible = "sprd,ums512-pwm" },
+	{ }
 };
 MODULE_DEVICE_TABLE(of, sprd_pwm_of_match);
 
diff --git a/drivers/pwm/pwm-sti.c b/drivers/pwm/pwm-sti.c
index 3b702b8f0c7f0..e5c3cf594571d 100644
--- a/drivers/pwm/pwm-sti.c
+++ b/drivers/pwm/pwm-sti.c
@@ -625,7 +625,7 @@ static int sti_pwm_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id sti_pwm_of_match[] = {
-	{ .compatible = "st,sti-pwm", },
+	{ .compatible = "st,sti-pwm" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, sti_pwm_of_match);
diff --git a/drivers/pwm/pwm-stm32-lp.c b/drivers/pwm/pwm-stm32-lp.c
index 4789eafb8bac7..2aa88f4985f51 100644
--- a/drivers/pwm/pwm-stm32-lp.c
+++ b/drivers/pwm/pwm-stm32-lp.c
@@ -412,8 +412,8 @@ static DEFINE_SIMPLE_DEV_PM_OPS(stm32_pwm_lp_pm_ops, stm32_pwm_lp_suspend,
 				stm32_pwm_lp_resume);
 
 static const struct of_device_id stm32_pwm_lp_of_match[] = {
-	{ .compatible = "st,stm32-pwm-lp", },
-	{},
+	{ .compatible = "st,stm32-pwm-lp" },
+	{ }
 };
 MODULE_DEVICE_TABLE(of, stm32_pwm_lp_of_match);
 
diff --git a/drivers/pwm/pwm-stm32.c b/drivers/pwm/pwm-stm32.c
index c708e4a7ad709..ea5a51a2800fc 100644
--- a/drivers/pwm/pwm-stm32.c
+++ b/drivers/pwm/pwm-stm32.c
@@ -918,9 +918,9 @@ static int stm32_pwm_resume(struct device *dev)
 static DEFINE_SIMPLE_DEV_PM_OPS(stm32_pwm_pm_ops, stm32_pwm_suspend, stm32_pwm_resume);
 
 static const struct of_device_id stm32_pwm_of_match[] = {
-	{ .compatible = "st,stm32-pwm",	},
-	{ .compatible = "st,stm32mp25-pwm", },
-	{ /* end node */ },
+	{ .compatible = "st,stm32-pwm" },
+	{ .compatible = "st,stm32mp25-pwm" },
+	{ /* end node */ }
 };
 MODULE_DEVICE_TABLE(of, stm32_pwm_of_match);
 
diff --git a/drivers/pwm/pwm-sun4i.c b/drivers/pwm/pwm-sun4i.c
index 6c5591ca868b4..0752152b14706 100644
--- a/drivers/pwm/pwm-sun4i.c
+++ b/drivers/pwm/pwm-sun4i.c
@@ -369,7 +369,7 @@ static const struct of_device_id sun4i_pwm_dt_ids[] = {
 		.data = &sun50i_h6_pwm_data,
 	}, {
 		/* sentinel */
-	},
+	}
 };
 MODULE_DEVICE_TABLE(of, sun4i_pwm_dt_ids);
 
diff --git a/drivers/pwm/pwm-sunplus.c b/drivers/pwm/pwm-sunplus.c
index cc8137f108df7..4c4a4b0f5f183 100644
--- a/drivers/pwm/pwm-sunplus.c
+++ b/drivers/pwm/pwm-sunplus.c
@@ -213,8 +213,8 @@ static int sunplus_pwm_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id sunplus_pwm_of_match[] = {
-	{ .compatible = "sunplus,sp7021-pwm", },
-	{}
+	{ .compatible = "sunplus,sp7021-pwm" },
+	{ }
 };
 MODULE_DEVICE_TABLE(of, sunplus_pwm_of_match);
 
diff --git a/drivers/pwm/pwm-tegra.c b/drivers/pwm/pwm-tegra.c
index 172063b51d442..5cdbe120ba2d8 100644
--- a/drivers/pwm/pwm-tegra.c
+++ b/drivers/pwm/pwm-tegra.c
@@ -7,22 +7,61 @@
  * Copyright (c) 2010-2020, NVIDIA Corporation.
  * Based on arch/arm/plat-mxc/pwm.c by Sascha Hauer <s.hauer@pengutronix.de>
  *
- * Overview of Tegra Pulse Width Modulator Register:
- * 1. 13-bit: Frequency division (SCALE)
- * 2. 8-bit : Pulse division (DUTY)
- * 3. 1-bit : Enable bit
+ * Overview of Tegra Pulse Width Modulator Register
+ * CSR_0 of Tegra20, Tegra186, and Tegra194:
+ * +-------+-------+-----------------------------------------------------------+
+ * | Bit   | Field | Description                                               |
+ * +-------+-------+-----------------------------------------------------------+
+ * | 31    | ENB   | Enable Pulse width modulator.                             |
+ * |       |       | 0 = DISABLE, 1 = ENABLE.                                  |
+ * +-------+-------+-----------------------------------------------------------+
+ * | 30:16 | PWM_0 | Pulse width that needs to be programmed.                  |
+ * |       |       | 0 = Always low.                                           |
+ * |       |       | 1 = 1 / 256 pulse high.                                   |
+ * |       |       | 2 = 2 / 256 pulse high.                                   |
+ * |       |       | N = N / 256 pulse high.                                   |
+ * |       |       | Only 8 bits are usable [23:16].                           |
+ * |       |       | Bit[24] can be programmed to 1 to achieve 100% duty       |
+ * |       |       | cycle. In this case the other bits [23:16] are set to     |
+ * |       |       | don’t care.                                               |
+ * +-------+-------+-----------------------------------------------------------+
+ * | 12:0  | PFM_0 | Frequency divider that needs to be programmed, also known |
+ * |       |       | as SCALE. Division by (1 + PFM_0).                        |
+ * +-------+-------+-----------------------------------------------------------+
  *
- * The PWM clock frequency is divided by 256 before subdividing it based
- * on the programmable frequency division value to generate the required
- * frequency for PWM output. The maximum output frequency that can be
- * achieved is (max rate of source clock) / 256.
- * e.g. if source clock rate is 408 MHz, maximum output frequency can be:
- * 408 MHz/256 = 1.6 MHz.
- * This 1.6 MHz frequency can further be divided using SCALE value in PWM.
+ * CSR_0 of Tegra264:
+ * +-------+-------+-----------------------------------------------------------+
+ * | Bit   | Field | Description                                               |
+ * +-------+-------+-----------------------------------------------------------+
+ * | 31:16 | PWM_0 | Pulse width that needs to be programmed.                  |
+ * |       |       | 0 = Always low.                                           |
+ * |       |       | 1 = 1 / (1 + CSR_1.DEPTH) pulse high.                     |
+ * |       |       | 2 = 2 / (1 + CSR_1.DEPTH) pulse high.                     |
+ * |       |       | N = N / (1 + CSR_1.DEPTH) pulse high.                     |
+ * +-------+-------+-----------------------------------------------------------+
+ * | 15:0  | PFM_0 | Frequency divider that needs to be programmed, also known |
+ * |       |       | as SCALE. Division by (1 + PFM_0).                        |
+ * +-------+-------+-----------------------------------------------------------+
+ *
+ * CSR_1 of Tegra264:
+ * +-------+-------+-----------------------------------------------------------+
+ * | Bit   | Field | Description                                               |
+ * +-------+-------+-----------------------------------------------------------+
+ * | 31    | ENB   | Enable Pulse width modulator.                             |
+ * |       |       | 0 = DISABLE, 1 = ENABLE.                                  |
+ * +-------+-------+-----------------------------------------------------------+
+ * | 30:15 | DEPTH | Depth for pulse width modulator. This controls the pulse  |
+ * |       |       | time generated. Division by (1 + CSR_1.DEPTH).            |
+ * +-------+-------+-----------------------------------------------------------+
  *
- * PWM pulse width: 8 bits are usable [23:16] for varying pulse width.
- * To achieve 100% duty cycle, program Bit [24] of this register to
- * 1’b1. In which case the other bits [23:16] are set to don't care.
+ * The PWM clock frequency is divided by DEPTH = (1 + CSR_1.DEPTH) before
+ * subdividing it based on the programmable frequency division value to
+ * generate the required frequency for PWM output. DEPTH is fixed to 256
+ * before Tegra264. The maximum output frequency that can be achieved is
+ * (max rate of source clock) / DEPTH.
+ * e.g. if source clock rate is 408 MHz, and DEPTH = 256, maximum output
+ * frequency can be: 408 MHz / 256 ~= 1.6 MHz.
+ * This 1.6 MHz frequency can further be divided using SCALE value in PWM.
  *
  * Limitations:
  * -	When PWM is disabled, the output is driven to inactive.
@@ -51,17 +90,20 @@
 
 #include <soc/tegra/common.h>
 
-#define PWM_ENABLE	(1 << 31)
-#define PWM_DUTY_WIDTH	8
-#define PWM_DUTY_SHIFT	16
-#define PWM_SCALE_WIDTH	13
-#define PWM_SCALE_SHIFT	0
+#define TEGRA_PWM_ENABLE	(1 << 31)
+#define TEGRA_PWM_DUTY_SHIFT	16
+#define TEGRA_PWM_SCALE_SHIFT	0
+
+#define TEGRA_PWM_CSR_0	0
+#define TEGRA_PWM_CSR_1	4
+
+#define TEGRA_PWM_DEPTH	256
 
 struct tegra_pwm_soc {
 	unsigned int num_channels;
+	unsigned int enable_reg;
 
-	/* Maximum IP frequency for given SoCs */
-	unsigned long max_frequency;
+	unsigned int scale_width;
 };
 
 struct tegra_pwm_chip {
@@ -81,14 +123,18 @@ static inline struct tegra_pwm_chip *to_tegra_pwm_chip(struct pwm_chip *chip)
 	return pwmchip_get_drvdata(chip);
 }
 
-static inline u32 pwm_readl(struct tegra_pwm_chip *pc, unsigned int offset)
+static inline u32 tegra_pwm_readl(struct pwm_device *pwm, unsigned int offset)
 {
-	return readl(pc->regs + (offset << 4));
+	struct tegra_pwm_chip *chip = to_tegra_pwm_chip(pwm->chip);
+
+	return readl(chip->regs + (pwm->hwpwm * 16) + offset);
 }
 
-static inline void pwm_writel(struct tegra_pwm_chip *pc, unsigned int offset, u32 value)
+static inline void tegra_pwm_writel(struct pwm_device *pwm, unsigned int offset, u32 value)
 {
-	writel(value, pc->regs + (offset << 4));
+	struct tegra_pwm_chip *chip = to_tegra_pwm_chip(pwm->chip);
+
+	writel(value, chip->regs + (pwm->hwpwm * 16) + offset);
 }
 
 static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
@@ -102,22 +148,22 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 
 	/*
 	 * Convert from duty_ns / period_ns to a fixed number of duty ticks
-	 * per (1 << PWM_DUTY_WIDTH) cycles and make sure to round to the
+	 * per TEGRA_PWM_DEPTH cycles and make sure to round to the
 	 * nearest integer during division.
 	 */
-	c *= (1 << PWM_DUTY_WIDTH);
+	c *= TEGRA_PWM_DEPTH;
 	c = DIV_ROUND_CLOSEST_ULL(c, period_ns);
 
-	val = (u32)c << PWM_DUTY_SHIFT;
+	val = (u32)c << TEGRA_PWM_DUTY_SHIFT;
 
 	/*
-	 *  min period = max clock limit >> PWM_DUTY_WIDTH
+	 *  min period = max clock limit / TEGRA_PWM_DEPTH
 	 */
 	if (period_ns < pc->min_period_ns)
 		return -EINVAL;
 
 	/*
-	 * Compute the prescaler value for which (1 << PWM_DUTY_WIDTH)
+	 * Compute the prescaler value for which TEGRA_PWM_DEPTH
 	 * cycles at the PWM clock rate will take period_ns nanoseconds.
 	 *
 	 * num_channels: If single instance of PWM controller has multiple
@@ -131,7 +177,7 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 	 */
 	if (pc->soc->num_channels == 1) {
 		/*
-		 * Rate is multiplied with 2^PWM_DUTY_WIDTH so that it matches
+		 * Rate is multiplied with TEGRA_PWM_DEPTH so that it matches
 		 * with the maximum possible rate that the controller can
 		 * provide. Any further lower value can be derived by setting
 		 * PFM bits[0:12].
@@ -141,7 +187,7 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 		 * source clock rate as required_clk_rate, PWM controller will
 		 * be able to configure the requested period.
 		 */
-		required_clk_rate = DIV_ROUND_UP_ULL((u64)NSEC_PER_SEC << PWM_DUTY_WIDTH,
+		required_clk_rate = DIV_ROUND_UP_ULL((u64)NSEC_PER_SEC * TEGRA_PWM_DEPTH,
 						     period_ns);
 
 		if (required_clk_rate > clk_round_rate(pc->clk, required_clk_rate))
@@ -163,9 +209,9 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 		pc->clk_rate = clk_get_rate(pc->clk);
 	}
 
-	/* Consider precision in PWM_SCALE_WIDTH rate calculation */
+	/* Consider precision in scale_width rate calculation */
 	rate = mul_u64_u64_div_u64(pc->clk_rate, period_ns,
-				   (u64)NSEC_PER_SEC << PWM_DUTY_WIDTH);
+				   (u64)NSEC_PER_SEC * TEGRA_PWM_DEPTH);
 
 	/*
 	 * Since the actual PWM divider is the register's frequency divider
@@ -181,10 +227,10 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 	 * Make sure that the rate will fit in the register's frequency
 	 * divider field.
 	 */
-	if (rate >> PWM_SCALE_WIDTH)
+	if (rate >> pc->soc->scale_width)
 		return -EINVAL;
 
-	val |= rate << PWM_SCALE_SHIFT;
+	val |= rate << TEGRA_PWM_SCALE_SHIFT;
 
 	/*
 	 * If the PWM channel is disabled, make sure to turn on the clock
@@ -194,10 +240,11 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 		err = pm_runtime_resume_and_get(pwmchip_parent(chip));
 		if (err)
 			return err;
-	} else
-		val |= PWM_ENABLE;
+	} else if (pc->soc->enable_reg == TEGRA_PWM_CSR_0) {
+		val |= TEGRA_PWM_ENABLE;
+	}
 
-	pwm_writel(pc, pwm->hwpwm, val);
+	tegra_pwm_writel(pwm, TEGRA_PWM_CSR_0, val);
 
 	/*
 	 * If the PWM is not enabled, turn the clock off again to save power.
@@ -218,9 +265,9 @@ static int tegra_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm)
 	if (rc)
 		return rc;
 
-	val = pwm_readl(pc, pwm->hwpwm);
-	val |= PWM_ENABLE;
-	pwm_writel(pc, pwm->hwpwm, val);
+	val = tegra_pwm_readl(pwm, pc->soc->enable_reg);
+	val |= TEGRA_PWM_ENABLE;
+	tegra_pwm_writel(pwm, pc->soc->enable_reg, val);
 
 	return 0;
 }
@@ -230,9 +277,9 @@ static void tegra_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm)
 	struct tegra_pwm_chip *pc = to_tegra_pwm_chip(chip);
 	u32 val;
 
-	val = pwm_readl(pc, pwm->hwpwm);
-	val &= ~PWM_ENABLE;
-	pwm_writel(pc, pwm->hwpwm, val);
+	val = tegra_pwm_readl(pwm, pc->soc->enable_reg);
+	val &= ~TEGRA_PWM_ENABLE;
+	tegra_pwm_writel(pwm, pc->soc->enable_reg, val);
 
 	pm_runtime_put_sync(pwmchip_parent(chip));
 }
@@ -303,7 +350,7 @@ static int tegra_pwm_probe(struct platform_device *pdev)
 		return ret;
 
 	/* Set maximum frequency of the IP */
-	ret = dev_pm_opp_set_rate(&pdev->dev, pc->soc->max_frequency);
+	ret = dev_pm_opp_set_rate(&pdev->dev, ULONG_MAX);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to set max frequency: %d\n", ret);
 		goto put_pm;
@@ -315,10 +362,15 @@ static int tegra_pwm_probe(struct platform_device *pdev)
 	 * so that PWM period can be calculated more accurately.
 	 */
 	pc->clk_rate = clk_get_rate(pc->clk);
+	if (pc->clk_rate < TEGRA_PWM_DEPTH) {
+		dev_err(&pdev->dev, "clock maximum frequency out of range\n");
+		ret = -ERANGE;
+		goto put_pm;
+	}
 
 	/* Set minimum limit of PWM period for the IP */
 	pc->min_period_ns =
-	    (NSEC_PER_SEC / (pc->soc->max_frequency >> PWM_DUTY_WIDTH)) + 1;
+	    (NSEC_PER_SEC / (pc->clk_rate / TEGRA_PWM_DEPTH)) + 1;
 
 	pc->rst = devm_reset_control_get_exclusive(&pdev->dev, "pwm");
 	if (IS_ERR(pc->rst)) {
@@ -397,23 +449,27 @@ static int __maybe_unused tegra_pwm_runtime_resume(struct device *dev)
 
 static const struct tegra_pwm_soc tegra20_pwm_soc = {
 	.num_channels = 4,
-	.max_frequency = 48000000UL,
+	.enable_reg = TEGRA_PWM_CSR_0,
+	.scale_width = 13,
 };
 
 static const struct tegra_pwm_soc tegra186_pwm_soc = {
 	.num_channels = 1,
-	.max_frequency = 102000000UL,
+	.enable_reg = TEGRA_PWM_CSR_0,
+	.scale_width = 13,
 };
 
-static const struct tegra_pwm_soc tegra194_pwm_soc = {
+static const struct tegra_pwm_soc tegra264_pwm_soc = {
 	.num_channels = 1,
-	.max_frequency = 408000000UL,
+	.enable_reg = TEGRA_PWM_CSR_1,
+	.scale_width = 16,
 };
 
 static const struct of_device_id tegra_pwm_of_match[] = {
 	{ .compatible = "nvidia,tegra20-pwm", .data = &tegra20_pwm_soc },
 	{ .compatible = "nvidia,tegra186-pwm", .data = &tegra186_pwm_soc },
-	{ .compatible = "nvidia,tegra194-pwm", .data = &tegra194_pwm_soc },
+	{ .compatible = "nvidia,tegra194-pwm", .data = &tegra186_pwm_soc },
+	{ .compatible = "nvidia,tegra264-pwm", .data = &tegra264_pwm_soc },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, tegra_pwm_of_match);
diff --git a/drivers/pwm/pwm-tiecap.c b/drivers/pwm/pwm-tiecap.c
index 67cc5e8bdb0ef..3751737ead43d 100644
--- a/drivers/pwm/pwm-tiecap.c
+++ b/drivers/pwm/pwm-tiecap.c
@@ -211,9 +211,9 @@ static const struct pwm_ops ecap_pwm_ops = {
 };
 
 static const struct of_device_id ecap_of_match[] = {
-	{ .compatible	= "ti,am3352-ecap" },
-	{ .compatible	= "ti,am33xx-ecap" },
-	{},
+	{ .compatible = "ti,am3352-ecap" },
+	{ .compatible = "ti,am33xx-ecap" },
+	{ }
 };
 MODULE_DEVICE_TABLE(of, ecap_of_match);
 
diff --git a/drivers/pwm/pwm-tiehrpwm.c b/drivers/pwm/pwm-tiehrpwm.c
index 2533c95b0ba9d..3a53489f0b982 100644
--- a/drivers/pwm/pwm-tiehrpwm.c
+++ b/drivers/pwm/pwm-tiehrpwm.c
@@ -407,7 +407,7 @@ static const struct pwm_ops ehrpwm_pwm_ops = {
 static const struct of_device_id ehrpwm_of_match[] = {
 	{ .compatible = "ti,am3352-ehrpwm" },
 	{ .compatible = "ti,am33xx-ehrpwm" },
-	{},
+	{ }
 };
 MODULE_DEVICE_TABLE(of, ehrpwm_of_match);
 
diff --git a/drivers/pwm/pwm-twl-led.c b/drivers/pwm/pwm-twl-led.c
index a555cc3be4b31..e1b6871af81ee 100644
--- a/drivers/pwm/pwm-twl-led.c
+++ b/drivers/pwm/pwm-twl-led.c
@@ -338,7 +338,7 @@ static int twl_pwmled_probe(struct platform_device *pdev)
 static const struct of_device_id twl_pwmled_of_match[] = {
 	{ .compatible = "ti,twl4030-pwmled" },
 	{ .compatible = "ti,twl6030-pwmled" },
-	{ },
+	{ }
 };
 MODULE_DEVICE_TABLE(of, twl_pwmled_of_match);
 #endif
diff --git a/drivers/pwm/pwm-twl.c b/drivers/pwm/pwm-twl.c
index 8f981ffff4b4e..99c1caf4f19c8 100644
--- a/drivers/pwm/pwm-twl.c
+++ b/drivers/pwm/pwm-twl.c
@@ -362,7 +362,7 @@ static int twl_pwm_probe(struct platform_device *pdev)
 static const struct of_device_id twl_pwm_of_match[] = {
 	{ .compatible = "ti,twl4030-pwm" },
 	{ .compatible = "ti,twl6030-pwm" },
-	{ },
+	{ }
 };
 MODULE_DEVICE_TABLE(of, twl_pwm_of_match);
 #endif
diff --git a/drivers/pwm/pwm-visconti.c b/drivers/pwm/pwm-visconti.c
index 28fae4979e3ff..830aef483bd03 100644
--- a/drivers/pwm/pwm-visconti.c
+++ b/drivers/pwm/pwm-visconti.c
@@ -156,7 +156,7 @@ static int visconti_pwm_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id visconti_pwm_of_match[] = {
-	{ .compatible = "toshiba,visconti-pwm", },
+	{ .compatible = "toshiba,visconti-pwm" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, visconti_pwm_of_match);
diff --git a/drivers/pwm/pwm-vt8500.c b/drivers/pwm/pwm-vt8500.c
index 149d9e35b78cf..8013e31b11470 100644
--- a/drivers/pwm/pwm-vt8500.c
+++ b/drivers/pwm/pwm-vt8500.c
@@ -226,7 +226,7 @@ static const struct pwm_ops vt8500_pwm_ops = {
 };
 
 static const struct of_device_id vt8500_pwm_dt_ids[] = {
-	{ .compatible = "via,vt8500-pwm", },
+	{ .compatible = "via,vt8500-pwm" },
 	{ /* Sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, vt8500_pwm_dt_ids);
diff --git a/drivers/pwm/pwm-xilinx.c b/drivers/pwm/pwm-xilinx.c
index 52c2419828073..fa2d10e658ddb 100644
--- a/drivers/pwm/pwm-xilinx.c
+++ b/drivers/pwm/pwm-xilinx.c
@@ -281,8 +281,8 @@ static int xilinx_pwm_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id xilinx_pwm_of_match[] = {
-	{ .compatible = "xlnx,xps-timer-1.00.a", },
-	{},
+	{ .compatible = "xlnx,xps-timer-1.00.a" },
+	{ }
 };
 MODULE_DEVICE_TABLE(of, xilinx_pwm_of_match);
 
diff --git a/drivers/pwm/pwm_th1520.rs b/drivers/pwm/pwm_th1520.rs
index 3e3fa51ccef91..9372cd3e6a6dd 100644
--- a/drivers/pwm/pwm_th1520.rs
+++ b/drivers/pwm/pwm_th1520.rs
@@ -20,13 +20,11 @@
 //!   this method is not used in this driver.
 //!
 
-use core::ops::Deref;
 use kernel::{
     clk::Clk,
     device::{Bound, Core, Device},
-    devres,
     io::{
-        mem::IoMem,
+        mem::DevresIoMem,
         Io, //
     },
     of, platform,
@@ -86,7 +84,7 @@ struct Th1520WfHw {
 #[pin_data(PinnedDrop)]
 struct Th1520PwmDriverData {
     #[pin]
-    iomem: devres::Devres<IoMem<'static, TH1520_PWM_REG_SIZE>>,
+    iomem: DevresIoMem<TH1520_PWM_REG_SIZE>,
     clk: Clk,
 }
 
@@ -213,8 +211,7 @@ fn read_waveform(
     ) -> Result<Self::WfHw> {
         let data = chip.drvdata();
         let hwpwm = pwm.hwpwm();
-        let iomem_accessor = data.iomem.access(parent_dev)?;
-        let iomap = iomem_accessor.deref();
+        let iomap = data.iomem.access(parent_dev)?;
 
         let ctrl = iomap.try_read32(th1520_pwm_ctrl(hwpwm))?;
         let period_cycles = iomap.try_read32(th1520_pwm_per(hwpwm))?;
@@ -248,8 +245,7 @@ fn write_waveform(
     ) -> Result {
         let data = chip.drvdata();
         let hwpwm = pwm.hwpwm();
-        let iomem_accessor = data.iomem.access(parent_dev)?;
-        let iomap = iomem_accessor.deref();
+        let iomap = data.iomem.access(parent_dev)?;
         let duty_cycles = iomap.try_read32(th1520_pwm_fp(hwpwm))?;
         let was_enabled = duty_cycles != 0;
 
diff --git a/drivers/rapidio/rio-scan.c b/drivers/rapidio/rio-scan.c
index dcd6619a4b027..3cc25d0534513 100644
--- a/drivers/rapidio/rio-scan.c
+++ b/drivers/rapidio/rio-scan.c
@@ -874,6 +874,7 @@ static struct rio_net *rio_scan_alloc_net(struct rio_mport *mport,
 		net->dev.release = rio_scan_release_dev;
 		if (rio_add_net(net)) {
 			put_device(&net->dev);
+			mport->net = NULL;
 			net = NULL;
 		}
 	}
diff --git a/drivers/ras/Kconfig b/drivers/ras/Kconfig
index fc4f4bb94a4c6..070e17639faa3 100644
--- a/drivers/ras/Kconfig
+++ b/drivers/ras/Kconfig
@@ -31,7 +31,29 @@ menuconfig RAS
 
 if RAS
 
-source "arch/x86/ras/Kconfig"
+config RAS_CEC
+	bool "Correctable Errors Collector"
+	depends on X86_MCE && MEMORY_FAILURE && DEBUG_FS
+	help
+	  This is a small cache which collects correctable memory errors per 4K
+	  page PFN and counts their repeated occurrence. Once the counter for a
+	  PFN overflows, we try to soft-offline that page as we take it to mean
+	  that it has reached a relatively high error count and would probably
+	  be best if we don't use it anymore.
+
+	  Bear in mind that this is absolutely useless if your platform doesn't
+	  have ECC DIMMs and doesn't have DRAM ECC checking enabled in the BIOS.
+
+config RAS_CEC_DEBUG
+	bool "CEC debugging machinery"
+	default n
+	depends on RAS_CEC
+	help
+	  Add extra files to (debugfs)/ras/cec to test the correctable error
+	  collector feature. "pfn" is a writable file that allows user to
+	  simulate an error in a particular page frame. "array" is a read-only
+	  file that dumps out the current state of all pages logged so far.
+
 source "drivers/ras/amd/atl/Kconfig"
 
 config RAS_FMPM
diff --git a/drivers/ras/amd/atl/core.c b/drivers/ras/amd/atl/core.c
index 0f7cd6dab0b0e..d77dacdd4f569 100644
--- a/drivers/ras/amd/atl/core.c
+++ b/drivers/ras/amd/atl/core.c
@@ -190,7 +190,6 @@ static const struct x86_cpu_id amd_atl_cpuids[] = {
 	X86_MATCH_FEATURE(X86_FEATURE_ZEN, NULL),
 	{ }
 };
-MODULE_DEVICE_TABLE(x86cpu, amd_atl_cpuids);
 
 static int __init amd_atl_init(void)
 {
diff --git a/drivers/regulator/88pg86x.c b/drivers/regulator/88pg86x.c
index e6598e74ec94e..8c25a1db412f8 100644
--- a/drivers/regulator/88pg86x.c
+++ b/drivers/regulator/88pg86x.c
@@ -92,8 +92,8 @@ static const struct of_device_id __maybe_unused pg86x_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, pg86x_dt_ids);
 
 static const struct i2c_device_id pg86x_i2c_id[] = {
-	{ "88pg867", },
-	{ "88pg868", },
+	{ .name = "88pg867" },
+	{ .name = "88pg868" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pg86x_i2c_id);
diff --git a/drivers/regulator/Kconfig b/drivers/regulator/Kconfig
index a54a549196fe3..89789ac7a786c 100644
--- a/drivers/regulator/Kconfig
+++ b/drivers/regulator/Kconfig
@@ -1477,6 +1477,13 @@ config REGULATOR_S5M8767
 	 via I2C bus. S5M8767A have 9 Bucks and 28 LDOs output and
 	 supports DVS mode with 8bits of output voltage control.
 
+config REGULATOR_SC2730
+	tristate "Spreadtrum SC2730 power regulator driver"
+	depends on MFD_SC27XX_PMIC || COMPILE_TEST
+	help
+	  This driver provides support for the voltage regulators on the
+	  SC2730 PMIC.
+
 config REGULATOR_SC2731
 	tristate "Spreadtrum SC2731 power regulator driver"
 	depends on MFD_SC27XX_PMIC || COMPILE_TEST
diff --git a/drivers/regulator/Makefile b/drivers/regulator/Makefile
index 134eee274dbfb..5a764cec8df83 100644
--- a/drivers/regulator/Makefile
+++ b/drivers/regulator/Makefile
@@ -170,6 +170,7 @@ obj-$(CONFIG_REGULATOR_S2DOS05) += s2dos05-regulator.o
 obj-$(CONFIG_REGULATOR_S2MPA01) += s2mpa01.o
 obj-$(CONFIG_REGULATOR_S2MPS11) += s2mps11.o
 obj-$(CONFIG_REGULATOR_S5M8767) += s5m8767.o
+obj-$(CONFIG_REGULATOR_SC2730) += sc2730-regulator.o
 obj-$(CONFIG_REGULATOR_SC2731) += sc2731-regulator.o
 obj-$(CONFIG_REGULATOR_SGM3804) += sgm3804-regulator.o
 obj-$(CONFIG_REGULATOR_SKY81452) += sky81452-regulator.o
diff --git a/drivers/regulator/act8865-regulator.c b/drivers/regulator/act8865-regulator.c
index b2a6ddc6f56d3..882b6e58e9110 100644
--- a/drivers/regulator/act8865-regulator.c
+++ b/drivers/regulator/act8865-regulator.c
@@ -780,7 +780,7 @@ static const struct i2c_device_id act8865_ids[] = {
 	{ .name = "act8600", .driver_data = ACT8600 },
 	{ .name = "act8846", .driver_data = ACT8846 },
 	{ .name = "act8865", .driver_data = ACT8865 },
-	{ },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, act8865_ids);
 
diff --git a/drivers/regulator/ad5398.c b/drivers/regulator/ad5398.c
index eb2a666a45cba..0123ca8157a88 100644
--- a/drivers/regulator/ad5398.c
+++ b/drivers/regulator/ad5398.c
@@ -207,8 +207,8 @@ struct ad5398_current_data_format {
 static const struct ad5398_current_data_format df_10_4_120 = {10, 4, 0, 120000};
 
 static const struct i2c_device_id ad5398_id[] = {
-	{ "ad5398", (kernel_ulong_t)&df_10_4_120 },
-	{ "ad5821", (kernel_ulong_t)&df_10_4_120 },
+	{ .name = "ad5398", .driver_data = (kernel_ulong_t)&df_10_4_120 },
+	{ .name = "ad5821", .driver_data = (kernel_ulong_t)&df_10_4_120 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ad5398_id);
diff --git a/drivers/regulator/adp5055-regulator.c b/drivers/regulator/adp5055-regulator.c
index 9ebd52b392357..194dcfe4f5279 100644
--- a/drivers/regulator/adp5055-regulator.c
+++ b/drivers/regulator/adp5055-regulator.c
@@ -224,7 +224,7 @@ static int adp5055_of_parse_cb(struct device_node *np,
 		adp5055->dvs_limit_upper[id] = pval;
 
 	if (adp5055->dvs_limit_upper[id] > 192000 || adp5055->dvs_limit_upper[id] < 12000)
-		return dev_err_probe(config->dev, adp5055->dvs_limit_upper[id],
+		return dev_err_probe(config->dev, -EINVAL,
 			"Out of range - dvs-limit-upper-microvolt value.");
 
 	ret = of_property_read_u32(np, "adi,dvs-limit-lower-microvolt", &pval);
@@ -403,8 +403,8 @@ static const struct of_device_id adp5055_of_match[] = {
 MODULE_DEVICE_TABLE(of, adp5055_of_match);
 
 static const struct i2c_device_id adp5055_ids[] = {
-	{ .name = "adp5055"},
-	{ },
+	{ .name = "adp5055" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adp5055_ids);
 
diff --git a/drivers/regulator/as3722-regulator.c b/drivers/regulator/as3722-regulator.c
index da378bfdba406..85e2aaa1a06bd 100644
--- a/drivers/regulator/as3722-regulator.c
+++ b/drivers/regulator/as3722-regulator.c
@@ -600,7 +600,6 @@ static int as3722_get_regulator_dt_data(struct platform_device *pdev,
 
 	ret = of_regulator_match(&pdev->dev, np, as3722_regulator_matches,
 			ARRAY_SIZE(as3722_regulator_matches));
-	of_node_put(np);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Parsing of regulator node failed: %d\n",
 			ret);
diff --git a/drivers/regulator/aw37503-regulator.c b/drivers/regulator/aw37503-regulator.c
index a5ff6dfd29b5f..ca5f2a14c28c0 100644
--- a/drivers/regulator/aw37503-regulator.c
+++ b/drivers/regulator/aw37503-regulator.c
@@ -212,8 +212,8 @@ static int aw37503_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id aw37503_id[] = {
-	{.name = "aw37503",},
-	{},
+	{ .name = "aw37503" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, aw37503_id);
 
diff --git a/drivers/regulator/da9121-regulator.c b/drivers/regulator/da9121-regulator.c
index 36a52f707602e..f501cf8152c23 100644
--- a/drivers/regulator/da9121-regulator.c
+++ b/drivers/regulator/da9121-regulator.c
@@ -1195,16 +1195,16 @@ static void da9121_i2c_remove(struct i2c_client *i2c)
 }
 
 static const struct i2c_device_id da9121_i2c_id[] = {
-	{"da9121", DA9121_SUBTYPE_DA9121},
-	{"da9130", DA9121_SUBTYPE_DA9130},
-	{"da9217", DA9121_SUBTYPE_DA9217},
-	{"da9122", DA9121_SUBTYPE_DA9122},
-	{"da9131", DA9121_SUBTYPE_DA9131},
-	{"da9220", DA9121_SUBTYPE_DA9220},
-	{"da9132", DA9121_SUBTYPE_DA9132},
-	{"da9141", DA9121_SUBTYPE_DA9141},
-	{"da9142", DA9121_SUBTYPE_DA9142},
-	{},
+	{ .name = "da9121", .driver_data = DA9121_SUBTYPE_DA9121 },
+	{ .name = "da9130", .driver_data = DA9121_SUBTYPE_DA9130 },
+	{ .name = "da9217", .driver_data = DA9121_SUBTYPE_DA9217 },
+	{ .name = "da9122", .driver_data = DA9121_SUBTYPE_DA9122 },
+	{ .name = "da9131", .driver_data = DA9121_SUBTYPE_DA9131 },
+	{ .name = "da9220", .driver_data = DA9121_SUBTYPE_DA9220 },
+	{ .name = "da9132", .driver_data = DA9121_SUBTYPE_DA9132 },
+	{ .name = "da9141", .driver_data = DA9121_SUBTYPE_DA9141 },
+	{ .name = "da9142", .driver_data = DA9121_SUBTYPE_DA9142 },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, da9121_i2c_id);
 
diff --git a/drivers/regulator/da9210-regulator.c b/drivers/regulator/da9210-regulator.c
index 39ade0dba40f0..9154e32bd7453 100644
--- a/drivers/regulator/da9210-regulator.c
+++ b/drivers/regulator/da9210-regulator.c
@@ -202,8 +202,8 @@ static int da9210_i2c_probe(struct i2c_client *i2c)
 }
 
 static const struct i2c_device_id da9210_i2c_id[] = {
-	{ "da9210" },
-	{}
+	{ .name = "da9210" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, da9210_i2c_id);
diff --git a/drivers/regulator/da9211-regulator.c b/drivers/regulator/da9211-regulator.c
index d4f14d7ea8cfa..9cf7137556367 100644
--- a/drivers/regulator/da9211-regulator.c
+++ b/drivers/regulator/da9211-regulator.c
@@ -522,15 +522,15 @@ static int da9211_i2c_probe(struct i2c_client *i2c)
 }
 
 static const struct i2c_device_id da9211_i2c_id[] = {
-	{"da9211", DA9211},
-	{"da9212", DA9212},
-	{"da9213", DA9213},
-	{"da9223", DA9223},
-	{"da9214", DA9214},
-	{"da9224", DA9224},
-	{"da9215", DA9215},
-	{"da9225", DA9225},
-	{},
+	{ .name = "da9211", .driver_data = DA9211 },
+	{ .name = "da9212", .driver_data = DA9212 },
+	{ .name = "da9213", .driver_data = DA9213 },
+	{ .name = "da9223", .driver_data = DA9223 },
+	{ .name = "da9214", .driver_data = DA9214 },
+	{ .name = "da9224", .driver_data = DA9224 },
+	{ .name = "da9215", .driver_data = DA9215 },
+	{ .name = "da9225", .driver_data = DA9225 },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, da9211_i2c_id);
 
diff --git a/drivers/regulator/db8500-prcmu.c b/drivers/regulator/db8500-prcmu.c
index 1ec2e1348891d..751fe36580faf 100644
--- a/drivers/regulator/db8500-prcmu.c
+++ b/drivers/regulator/db8500-prcmu.c
@@ -13,7 +13,7 @@
 #include <linux/err.h>
 #include <linux/spinlock.h>
 #include <linux/platform_device.h>
-#include <linux/mfd/dbx500-prcmu.h>
+#include <linux/mfd/db8500-prcmu.h>
 #include <linux/regulator/driver.h>
 #include <linux/regulator/machine.h>
 #include <linux/regulator/db8500-prcmu.h>
@@ -93,13 +93,13 @@ static int enable_epod(u16 epod_id, bool ramret)
 
 	if (ramret) {
 		if (!epod_on[epod_id]) {
-			ret = prcmu_set_epod(epod_id, EPOD_STATE_RAMRET);
+			ret = db8500_prcmu_set_epod(epod_id, EPOD_STATE_RAMRET);
 			if (ret < 0)
 				return ret;
 		}
 		epod_ramret[epod_id] = true;
 	} else {
-		ret = prcmu_set_epod(epod_id, EPOD_STATE_ON);
+		ret = db8500_prcmu_set_epod(epod_id, EPOD_STATE_ON);
 		if (ret < 0)
 			return ret;
 		epod_on[epod_id] = true;
@@ -114,18 +114,18 @@ static int disable_epod(u16 epod_id, bool ramret)
 
 	if (ramret) {
 		if (!epod_on[epod_id]) {
-			ret = prcmu_set_epod(epod_id, EPOD_STATE_OFF);
+			ret = db8500_prcmu_set_epod(epod_id, EPOD_STATE_OFF);
 			if (ret < 0)
 				return ret;
 		}
 		epod_ramret[epod_id] = false;
 	} else {
 		if (epod_ramret[epod_id]) {
-			ret = prcmu_set_epod(epod_id, EPOD_STATE_RAMRET);
+			ret = db8500_prcmu_set_epod(epod_id, EPOD_STATE_RAMRET);
 			if (ret < 0)
 				return ret;
 		} else {
-			ret = prcmu_set_epod(epod_id, EPOD_STATE_OFF);
+			ret = db8500_prcmu_set_epod(epod_id, EPOD_STATE_OFF);
 			if (ret < 0)
 				return ret;
 		}
diff --git a/drivers/regulator/fan53880.c b/drivers/regulator/fan53880.c
index 6cb5656845f9e..79ba705ec324f 100644
--- a/drivers/regulator/fan53880.c
+++ b/drivers/regulator/fan53880.c
@@ -164,8 +164,8 @@ static const struct of_device_id fan53880_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, fan53880_dt_ids);
 
 static const struct i2c_device_id fan53880_i2c_id[] = {
-	{ "fan53880", },
-	{}
+	{ .name = "fan53880" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, fan53880_i2c_id);
 
diff --git a/drivers/regulator/isl6271a-regulator.c b/drivers/regulator/isl6271a-regulator.c
index 7883cd160727d..868f2b823a107 100644
--- a/drivers/regulator/isl6271a-regulator.c
+++ b/drivers/regulator/isl6271a-regulator.c
@@ -138,7 +138,7 @@ static int isl6271a_probe(struct i2c_client *i2c)
 }
 
 static const struct i2c_device_id isl6271a_id[] = {
-	{ .name = "isl6271a", },
+	{ .name = "isl6271a" },
 	{ }
 };
 
diff --git a/drivers/regulator/isl9305.c b/drivers/regulator/isl9305.c
index 5a234f25e6bb0..ec6bd6bb9721c 100644
--- a/drivers/regulator/isl9305.c
+++ b/drivers/regulator/isl9305.c
@@ -186,8 +186,8 @@ MODULE_DEVICE_TABLE(of, isl9305_dt_ids);
 #endif
 
 static const struct i2c_device_id isl9305_i2c_id[] = {
-	{ "isl9305", },
-	{ "isl9305h", },
+	{ .name = "isl9305" },
+	{ .name = "isl9305h" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, isl9305_i2c_id);
diff --git a/drivers/regulator/lp3971.c b/drivers/regulator/lp3971.c
index d4dab86fe3854..6f830ae1bb613 100644
--- a/drivers/regulator/lp3971.c
+++ b/drivers/regulator/lp3971.c
@@ -439,7 +439,7 @@ static int lp3971_i2c_probe(struct i2c_client *i2c)
 }
 
 static const struct i2c_device_id lp3971_i2c_id[] = {
-	{ "lp3971" },
+	{ .name = "lp3971" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lp3971_i2c_id);
diff --git a/drivers/regulator/lp3972.c b/drivers/regulator/lp3972.c
index 1b918fb721346..235c640ba57f5 100644
--- a/drivers/regulator/lp3972.c
+++ b/drivers/regulator/lp3972.c
@@ -537,7 +537,7 @@ static int lp3972_i2c_probe(struct i2c_client *i2c)
 }
 
 static const struct i2c_device_id lp3972_i2c_id[] = {
-	{ "lp3972" },
+	{ .name = "lp3972" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lp3972_i2c_id);
diff --git a/drivers/regulator/lp872x.c b/drivers/regulator/lp872x.c
index 942f37082cb13..5b2faddd8110f 100644
--- a/drivers/regulator/lp872x.c
+++ b/drivers/regulator/lp872x.c
@@ -935,8 +935,8 @@ static const struct of_device_id lp872x_dt_ids[] __maybe_unused = {
 MODULE_DEVICE_TABLE(of, lp872x_dt_ids);
 
 static const struct i2c_device_id lp872x_ids[] = {
-	{"lp8720", LP8720},
-	{"lp8725", LP8725},
+	{ .name = "lp8720", .driver_data = LP8720 },
+	{ .name = "lp8725", .driver_data = LP8725 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lp872x_ids);
diff --git a/drivers/regulator/lp8755.c b/drivers/regulator/lp8755.c
index 5509bee49bda4..632320ba18004 100644
--- a/drivers/regulator/lp8755.c
+++ b/drivers/regulator/lp8755.c
@@ -430,8 +430,8 @@ static void lp8755_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id lp8755_id[] = {
-	{ LP8755_NAME },
-	{}
+	{ .name = LP8755_NAME },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, lp8755_id);
diff --git a/drivers/regulator/ltc3589.c b/drivers/regulator/ltc3589.c
index 3f70c2225dba0..8bae5d8aeaf4e 100644
--- a/drivers/regulator/ltc3589.c
+++ b/drivers/regulator/ltc3589.c
@@ -445,9 +445,9 @@ static const struct ltc3589_info ltc3589_12_info = {
 };
 
 static const struct i2c_device_id ltc3589_i2c_id[] = {
-	{ "ltc3589",   (kernel_ulong_t)&ltc3589_info },
-	{ "ltc3589-1", (kernel_ulong_t)&ltc3589_12_info },
-	{ "ltc3589-2", (kernel_ulong_t)&ltc3589_12_info },
+	{ .name = "ltc3589", .driver_data = (kernel_ulong_t)&ltc3589_info },
+	{ .name = "ltc3589-1", .driver_data = (kernel_ulong_t)&ltc3589_12_info },
+	{ .name = "ltc3589-2", .driver_data = (kernel_ulong_t)&ltc3589_12_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ltc3589_i2c_id);
diff --git a/drivers/regulator/ltc3676.c b/drivers/regulator/ltc3676.c
index 73d511eb1c1dc..307c6f7a155d9 100644
--- a/drivers/regulator/ltc3676.c
+++ b/drivers/regulator/ltc3676.c
@@ -45,11 +45,11 @@
 #define LTC3676_DVBxA_REF_SELECT	BIT(5)
 #define LTC3676_DVBxB_PGOOD_MASK	BIT(5)
 
-#define LTC3676_IRQSTAT_PGOOD_TIMEOUT	BIT(3)
-#define LTC3676_IRQSTAT_UNDERVOLT_WARN	BIT(4)
-#define LTC3676_IRQSTAT_UNDERVOLT_FAULT	BIT(5)
-#define LTC3676_IRQSTAT_THERMAL_WARN	BIT(6)
-#define LTC3676_IRQSTAT_THERMAL_FAULT	BIT(7)
+#define LTC3676_IRQSTAT_PGOOD_TIMEOUT	BIT(2)
+#define LTC3676_IRQSTAT_UNDERVOLT_WARN	BIT(3)
+#define LTC3676_IRQSTAT_UNDERVOLT_FAULT	BIT(4)
+#define LTC3676_IRQSTAT_THERMAL_WARN	BIT(5)
+#define LTC3676_IRQSTAT_THERMAL_FAULT	BIT(6)
 
 enum ltc3676_reg {
 	LTC3676_SW1,
@@ -357,7 +357,7 @@ static int ltc3676_regulator_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ltc3676_i2c_id[] = {
-	{ "ltc3676" },
+	{ .name = "ltc3676" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ltc3676_i2c_id);
diff --git a/drivers/regulator/max14577-regulator.c b/drivers/regulator/max14577-regulator.c
index c9d8d5e31cbd8..cd592c5de1483 100644
--- a/drivers/regulator/max14577-regulator.c
+++ b/drivers/regulator/max14577-regulator.c
@@ -123,15 +123,88 @@ static const struct regulator_desc max14577_supported_regulators[] = {
 	[MAX14577_CHARGER] = MAX14577_CHARGER_REG,
 };
 
+struct max77836_ldo {
+	struct max14577	*max14577;
+	unsigned int	mode;
+};
+
+static int max77836_ldo_enable(struct regulator_dev *rdev)
+{
+	struct max77836_ldo *ldo = rdev_get_drvdata(rdev);
+
+	return regmap_update_bits(rdev->regmap, rdev->desc->enable_reg,
+			MAX77836_CNFG1_LDO_PWRMD_MASK, ldo->mode);
+}
+
+static int max77836_ldo_disable(struct regulator_dev *rdev)
+{
+	return regmap_update_bits(rdev->regmap, rdev->desc->enable_reg,
+			MAX77836_CNFG1_LDO_PWRMD_MASK,
+			MAX77836_CNFG1_LDO_PWRMD_OFF);
+}
+
+static unsigned int max77836_ldo_get_mode(struct regulator_dev *rdev)
+{
+	struct max77836_ldo *ldo = rdev_get_drvdata(rdev);
+
+	switch (ldo->mode) {
+	case MAX77836_CNFG1_LDO_PWRMD_LPM:
+		return REGULATOR_MODE_IDLE;
+	case MAX77836_CNFG1_LDO_PWRMD_NORMAL:
+		return REGULATOR_MODE_NORMAL;
+	default:
+		return REGULATOR_MODE_INVALID;
+	}
+}
+
+static int max77836_ldo_set_mode(struct regulator_dev *rdev,
+				 unsigned int mode)
+{
+	struct max77836_ldo *ldo = rdev_get_drvdata(rdev);
+	unsigned int val;
+
+	switch (mode) {
+	case REGULATOR_MODE_NORMAL:
+		val = MAX77836_CNFG1_LDO_PWRMD_NORMAL;
+		break;
+	case REGULATOR_MODE_IDLE:
+		val = MAX77836_CNFG1_LDO_PWRMD_LPM;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ldo->mode = val;
+
+	/* Only touch hardware if the regulator is already on */
+	if (regulator_is_enabled_regmap(rdev))
+		return regmap_update_bits(rdev->regmap, rdev->desc->enable_reg,
+				MAX77836_CNFG1_LDO_PWRMD_MASK, val);
+
+	return 0;
+}
+
+static unsigned int max77836_ldo_of_map_mode(unsigned int mode)
+{
+	switch (mode) {
+	case REGULATOR_MODE_NORMAL:
+	case REGULATOR_MODE_IDLE:
+		return mode;
+	default:
+		return REGULATOR_MODE_INVALID;
+	}
+}
+
 static const struct regulator_ops max77836_ldo_ops = {
 	.is_enabled		= regulator_is_enabled_regmap,
-	.enable			= regulator_enable_regmap,
-	.disable		= regulator_disable_regmap,
+	.enable			= max77836_ldo_enable,
+	.disable		= max77836_ldo_disable,
 	.list_voltage		= regulator_list_voltage_linear,
 	.map_voltage		= regulator_map_voltage_linear,
 	.get_voltage_sel	= regulator_get_voltage_sel_regmap,
 	.set_voltage_sel	= regulator_set_voltage_sel_regmap,
-	/* TODO: add .set_suspend_mode */
+	.get_mode		= max77836_ldo_get_mode,
+	.set_mode		= max77836_ldo_set_mode,
 };
 
 #define MAX77836_LDO_REG(num)	{ \
@@ -147,6 +220,7 @@ static const struct regulator_ops max77836_ldo_ops = {
 	.uV_step	= MAX77836_REGULATOR_LDO_VOLTAGE_STEP, \
 	.enable_reg	= MAX77836_LDO_REG_CNFG1_LDO ## num, \
 	.enable_mask	= MAX77836_CNFG1_LDO_PWRMD_MASK, \
+	.of_map_mode	= max77836_ldo_of_map_mode, \
 	.vsel_reg	= MAX77836_LDO_REG_CNFG1_LDO ## num, \
 	.vsel_mask	= MAX77836_CNFG1_LDO_TV_MASK, \
 }
@@ -205,7 +279,6 @@ static int max14577_regulator_probe(struct platform_device *pdev)
 	}
 
 	config.dev = max14577->dev;
-	config.driver_data = max14577;
 
 	for (i = 0; i < supported_regulators_size; i++) {
 		struct regulator_dev *regulator;
@@ -217,6 +290,28 @@ static int max14577_regulator_probe(struct platform_device *pdev)
 			config.init_data = pdata->regulators[i].initdata;
 			config.of_node = pdata->regulators[i].of_node;
 		}
+
+		/*
+		 * LDOs need per-regulator driver data to store their mode.
+		 * The charger and safeout share the core MFD struct.
+		 */
+		if (dev_type == MAXIM_DEVICE_TYPE_MAX77836 &&
+		    (supported_regulators[i].id == MAX77836_LDO1 ||
+		     supported_regulators[i].id == MAX77836_LDO2)) {
+			struct max77836_ldo *ldo;
+
+			ldo = devm_kzalloc(&pdev->dev, sizeof(*ldo),
+					   GFP_KERNEL);
+			if (!ldo)
+				return -ENOMEM;
+
+			ldo->max14577 = max14577;
+			ldo->mode = MAX77836_CNFG1_LDO_PWRMD_NORMAL;
+			config.driver_data = ldo;
+		} else {
+			config.driver_data = max14577;
+		}
+
 		config.regmap = max14577_get_regmap(max14577,
 				supported_regulators[i].id);
 
diff --git a/drivers/regulator/max1586.c b/drivers/regulator/max1586.c
index 4242fbb7b1476..e5cbc09c2d396 100644
--- a/drivers/regulator/max1586.c
+++ b/drivers/regulator/max1586.c
@@ -276,7 +276,7 @@ static int max1586_pmic_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max1586_id[] = {
-	{ "max1586" },
+	{ .name = "max1586" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max1586_id);
diff --git a/drivers/regulator/max20086-regulator.c b/drivers/regulator/max20086-regulator.c
index fcdd2d0317a57..92594b2915f3d 100644
--- a/drivers/regulator/max20086-regulator.c
+++ b/drivers/regulator/max20086-regulator.c
@@ -301,10 +301,10 @@ static const struct max20086_chip_info max20089_chip_info = {
 };
 
 static const struct i2c_device_id max20086_i2c_id[] = {
-	{ "max20086", (kernel_ulong_t)&max20086_chip_info },
-	{ "max20087", (kernel_ulong_t)&max20087_chip_info },
-	{ "max20088", (kernel_ulong_t)&max20088_chip_info },
-	{ "max20089", (kernel_ulong_t)&max20089_chip_info },
+	{ .name = "max20086", .driver_data = (kernel_ulong_t)&max20086_chip_info },
+	{ .name = "max20087", .driver_data = (kernel_ulong_t)&max20087_chip_info },
+	{ .name = "max20088", .driver_data = (kernel_ulong_t)&max20088_chip_info },
+	{ .name = "max20089", .driver_data = (kernel_ulong_t)&max20089_chip_info },
 	{ /* Sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, max20086_i2c_id);
diff --git a/drivers/regulator/max20411-regulator.c b/drivers/regulator/max20411-regulator.c
index 6c0ebb970e902..ac7a9aa014aae 100644
--- a/drivers/regulator/max20411-regulator.c
+++ b/drivers/regulator/max20411-regulator.c
@@ -145,7 +145,7 @@ static const struct of_device_id of_max20411_match_tbl[] = {
 MODULE_DEVICE_TABLE(of, of_max20411_match_tbl);
 
 static const struct i2c_device_id max20411_id[] = {
-	{ "max20411" },
+	{ .name = "max20411" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max20411_id);
diff --git a/drivers/regulator/max77503-regulator.c b/drivers/regulator/max77503-regulator.c
index c7c94e868fc13..1cae846f96d08 100644
--- a/drivers/regulator/max77503-regulator.c
+++ b/drivers/regulator/max77503-regulator.c
@@ -107,7 +107,7 @@ static const struct of_device_id of_max77503_match_tbl[] = {
 MODULE_DEVICE_TABLE(of, of_max77503_match_tbl);
 
 static const struct i2c_device_id max77503_regulator_id[] = {
-	{"max77503"},
+	{ .name = "max77503" },
 	{ }
 };
 
diff --git a/drivers/regulator/max77675-regulator.c b/drivers/regulator/max77675-regulator.c
index fad0844f364e7..23b54b44f8b7d 100644
--- a/drivers/regulator/max77675-regulator.c
+++ b/drivers/regulator/max77675-regulator.c
@@ -1028,7 +1028,7 @@ static int max77675_regulator_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max77675_i2c_id[] = {
-	{ "max77675", 0 },
+	{ .name = "max77675" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max77675_i2c_id);
diff --git a/drivers/regulator/max77826-regulator.c b/drivers/regulator/max77826-regulator.c
index 310bc8ee7af84..8b60a9fcab44d 100644
--- a/drivers/regulator/max77826-regulator.c
+++ b/drivers/regulator/max77826-regulator.c
@@ -278,7 +278,7 @@ static const struct of_device_id __maybe_unused max77826_of_match[] = {
 MODULE_DEVICE_TABLE(of, max77826_of_match);
 
 static const struct i2c_device_id max77826_id[] = {
-	{ "max77826-regulator" },
+	{ .name = "max77826-regulator" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, max77826_id);
diff --git a/drivers/regulator/max77838-regulator.c b/drivers/regulator/max77838-regulator.c
index 9faddbfd25fd8..765756fdcf6e2 100644
--- a/drivers/regulator/max77838-regulator.c
+++ b/drivers/regulator/max77838-regulator.c
@@ -200,7 +200,7 @@ static const struct of_device_id __maybe_unused max77838_of_match[] = {
 MODULE_DEVICE_TABLE(of, max77838_of_match);
 
 static const struct i2c_device_id max77838_id[] = {
-	{ "max77838-regulator" },
+	{ .name = "max77838-regulator" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, max77838_id);
diff --git a/drivers/regulator/max77857-regulator.c b/drivers/regulator/max77857-regulator.c
index 1216cc3a6f72b..f1410f8456539 100644
--- a/drivers/regulator/max77857-regulator.c
+++ b/drivers/regulator/max77857-regulator.c
@@ -428,10 +428,10 @@ static int max77857_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max77857_id[] = {
-	{ "max77831", ID_MAX77831 },
-	{ "max77857", ID_MAX77857 },
-	{ "max77859", ID_MAX77859 },
-	{ "max77859a", ID_MAX77859A },
+	{ .name = "max77831", .driver_data = ID_MAX77831 },
+	{ .name = "max77857", .driver_data = ID_MAX77857 },
+	{ .name = "max77859", .driver_data = ID_MAX77859 },
+	{ .name = "max77859a", .driver_data = ID_MAX77859A },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max77857_id);
diff --git a/drivers/regulator/max8649.c b/drivers/regulator/max8649.c
index f57c588bcf28b..2d17405242e70 100644
--- a/drivers/regulator/max8649.c
+++ b/drivers/regulator/max8649.c
@@ -240,7 +240,7 @@ static int max8649_regulator_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max8649_id[] = {
-	{ "max8649" },
+	{ .name = "max8649" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max8649_id);
diff --git a/drivers/regulator/max8893.c b/drivers/regulator/max8893.c
index 5a90633d8536b..7a0e44a16d49c 100644
--- a/drivers/regulator/max8893.c
+++ b/drivers/regulator/max8893.c
@@ -162,7 +162,7 @@ MODULE_DEVICE_TABLE(of, max8893_dt_match);
 #endif
 
 static const struct i2c_device_id max8893_ids[] = {
-	{ "max8893" },
+	{ .name = "max8893" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max8893_ids);
diff --git a/drivers/regulator/max8952.c b/drivers/regulator/max8952.c
index 1f94315bfb02b..f8b91a5701f3e 100644
--- a/drivers/regulator/max8952.c
+++ b/drivers/regulator/max8952.c
@@ -307,7 +307,7 @@ static int max8952_pmic_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max8952_ids[] = {
-	{ "max8952" },
+	{ .name = "max8952" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max8952_ids);
diff --git a/drivers/regulator/max8973-regulator.c b/drivers/regulator/max8973-regulator.c
index f68caa07f546f..95062032dc7e3 100644
--- a/drivers/regulator/max8973-regulator.c
+++ b/drivers/regulator/max8973-regulator.c
@@ -780,9 +780,9 @@ static int max8973_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max8973_id[] = {
-	{.name = "max8973", .driver_data = MAX8973},
-	{.name = "max77621", .driver_data = MAX77621},
-	{},
+	{ .name = "max8973", .driver_data = MAX8973 },
+	{ .name = "max77621", .driver_data = MAX77621 },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max8973_id);
 
diff --git a/drivers/regulator/max8998.c b/drivers/regulator/max8998.c
index cc85fbe8b77c3..90651f4d18281 100644
--- a/drivers/regulator/max8998.c
+++ b/drivers/regulator/max8998.c
@@ -582,7 +582,6 @@ static int max8998_pmic_dt_parse_pdata(struct max8998_dev *iodev,
 	}
 	pdata->num_regulators = rdata - pdata->regulators;
 
-	of_node_put(reg_np);
 	of_node_put(regulators_np);
 
 	pdata->buck_voltage_lock = of_property_read_bool(pmic_np, "max8998,pmic-buck-voltage-lock");
diff --git a/drivers/regulator/mcp16502.c b/drivers/regulator/mcp16502.c
index b34ae0bbba6f7..07c67ad1ab7f2 100644
--- a/drivers/regulator/mcp16502.c
+++ b/drivers/regulator/mcp16502.c
@@ -508,7 +508,7 @@ static int mcp16502_probe(struct i2c_client *client)
 	struct device *dev;
 	struct mcp16502 *mcp;
 	struct regmap *rmap;
-	int i, ret;
+	int i;
 
 	dev = &client->dev;
 	config.dev = dev;
@@ -518,30 +518,23 @@ static int mcp16502_probe(struct i2c_client *client)
 		return -ENOMEM;
 
 	rmap = devm_regmap_init_i2c(client, &mcp16502_regmap_config);
-	if (IS_ERR(rmap)) {
-		ret = PTR_ERR(rmap);
-		dev_err(dev, "regmap init failed: %d\n", ret);
-		return ret;
-	}
+	if (IS_ERR(rmap))
+		return dev_err_probe(dev, PTR_ERR(rmap), "regmap init failed\n");
 
 	i2c_set_clientdata(client, mcp);
 	config.regmap = rmap;
 	config.driver_data = mcp;
 
 	mcp->lpm = devm_gpiod_get_optional(dev, "lpm", GPIOD_OUT_LOW);
-	if (IS_ERR(mcp->lpm)) {
-		dev_err(dev, "failed to get lpm pin: %ld\n", PTR_ERR(mcp->lpm));
-		return PTR_ERR(mcp->lpm);
-	}
+	if (IS_ERR(mcp->lpm))
+		return dev_err_probe(dev, PTR_ERR(mcp->lpm), "failed to get lpm pin\n");
 
 	for (i = 0; i < NUM_REGULATORS; i++) {
 		rdev = devm_regulator_register(dev, &mcp16502_desc[i], &config);
-		if (IS_ERR(rdev)) {
-			dev_err(dev,
-				"failed to register %s regulator %ld\n",
-				mcp16502_desc[i].name, PTR_ERR(rdev));
-			return PTR_ERR(rdev);
-		}
+		if (IS_ERR(rdev))
+			return dev_err_probe(dev, PTR_ERR(rdev),
+					     "failed to register %s regulator\n",
+					     mcp16502_desc[i].name);
 	}
 
 	mcp16502_gpio_set_mode(mcp, MCP16502_OPMODE_ACTIVE);
@@ -578,7 +571,7 @@ static const struct dev_pm_ops mcp16502_pm_ops = {
 };
 #endif
 static const struct i2c_device_id mcp16502_i2c_id[] = {
-	{ "mcp16502" },
+	{ .name = "mcp16502" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mcp16502_i2c_id);
diff --git a/drivers/regulator/mp5416.c b/drivers/regulator/mp5416.c
index e6794190cb68e..2948635b1b9ff 100644
--- a/drivers/regulator/mp5416.c
+++ b/drivers/regulator/mp5416.c
@@ -228,9 +228,9 @@ static const struct of_device_id mp5416_of_match[] = {
 MODULE_DEVICE_TABLE(of, mp5416_of_match);
 
 static const struct i2c_device_id mp5416_id[] = {
-	{ "mp5416", (kernel_ulong_t)&mp5416_regulators_desc },
-	{ "mp5496", (kernel_ulong_t)&mp5496_regulators_desc },
-	{}
+	{ .name = "mp5416", .driver_data = (kernel_ulong_t)&mp5416_regulators_desc },
+	{ .name = "mp5496", .driver_data = (kernel_ulong_t)&mp5496_regulators_desc },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mp5416_id);
 
diff --git a/drivers/regulator/mp8859.c b/drivers/regulator/mp8859.c
index ab105ffd6a2e0..9a708e826d931 100644
--- a/drivers/regulator/mp8859.c
+++ b/drivers/regulator/mp8859.c
@@ -386,8 +386,8 @@ static const struct of_device_id mp8859_dt_id[] __maybe_unused = {
 MODULE_DEVICE_TABLE(of, mp8859_dt_id);
 
 static const struct i2c_device_id mp8859_i2c_id[] = {
-	{ "mp8859", },
-	{  },
+	{ .name = "mp8859" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mp8859_i2c_id);
 
diff --git a/drivers/regulator/mp886x.c b/drivers/regulator/mp886x.c
index 9ad16b04c913c..e0b62bc02a1e1 100644
--- a/drivers/regulator/mp886x.c
+++ b/drivers/regulator/mp886x.c
@@ -348,9 +348,9 @@ static const struct of_device_id mp886x_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, mp886x_dt_ids);
 
 static const struct i2c_device_id mp886x_id[] = {
-	{ "mp8867", (kernel_ulong_t)&mp8867_ci },
-	{ "mp8869", (kernel_ulong_t)&mp8869_ci },
-	{ },
+	{ .name = "mp8867", .driver_data = (kernel_ulong_t)&mp8867_ci },
+	{ .name = "mp8869", .driver_data = (kernel_ulong_t)&mp8869_ci },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mp886x_id);
 
diff --git a/drivers/regulator/mpq7920.c b/drivers/regulator/mpq7920.c
index a670e09891e71..0cbc17deb1d16 100644
--- a/drivers/regulator/mpq7920.c
+++ b/drivers/regulator/mpq7920.c
@@ -309,8 +309,8 @@ static const struct of_device_id mpq7920_of_match[] = {
 MODULE_DEVICE_TABLE(of, mpq7920_of_match);
 
 static const struct i2c_device_id mpq7920_id[] = {
-	{ "mpq7920", },
-	{ },
+	{ .name = "mpq7920" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mpq7920_id);
 
diff --git a/drivers/regulator/mt6311-regulator.c b/drivers/regulator/mt6311-regulator.c
index 2ebc1c0b5e6f6..1d457d1fdf23e 100644
--- a/drivers/regulator/mt6311-regulator.c
+++ b/drivers/regulator/mt6311-regulator.c
@@ -133,8 +133,8 @@ static int mt6311_i2c_probe(struct i2c_client *i2c)
 }
 
 static const struct i2c_device_id mt6311_i2c_id[] = {
-	{ "mt6311" },
-	{}
+	{ .name = "mt6311" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mt6311_i2c_id);
 
diff --git a/drivers/regulator/pf530x-regulator.c b/drivers/regulator/pf530x-regulator.c
index f789c4b6a499e..8ad1cbbd7a8c4 100644
--- a/drivers/regulator/pf530x-regulator.c
+++ b/drivers/regulator/pf530x-regulator.c
@@ -353,10 +353,10 @@ static const struct of_device_id pf530x_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, pf530x_dt_ids);
 
 static const struct i2c_device_id pf530x_i2c_id[] = {
-	{ "pf5300", 0 },
-	{ "pf5301", 0 },
-	{ "pf5302", 0 },
-	{},
+	{ .name = "pf5300" },
+	{ .name = "pf5301" },
+	{ .name = "pf5302" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pf530x_i2c_id);
 
diff --git a/drivers/regulator/pf8x00-regulator.c b/drivers/regulator/pf8x00-regulator.c
index ea3611de42b4a..c938b4632ef16 100644
--- a/drivers/regulator/pf8x00-regulator.c
+++ b/drivers/regulator/pf8x00-regulator.c
@@ -596,10 +596,10 @@ static const struct of_device_id pf8x00_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, pf8x00_dt_ids);
 
 static const struct i2c_device_id pf8x00_i2c_id[] = {
-	{ "pf8100" },
-	{ "pf8121a" },
-	{ "pf8200" },
-	{}
+	{ .name = "pf8100" },
+	{ .name = "pf8121a" },
+	{ .name = "pf8200" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pf8x00_i2c_id);
 
diff --git a/drivers/regulator/pv88060-regulator.c b/drivers/regulator/pv88060-regulator.c
index ae1c4b9daaa1e..375d9e759c475 100644
--- a/drivers/regulator/pv88060-regulator.c
+++ b/drivers/regulator/pv88060-regulator.c
@@ -360,8 +360,8 @@ static int pv88060_i2c_probe(struct i2c_client *i2c)
 }
 
 static const struct i2c_device_id pv88060_i2c_id[] = {
-	{ "pv88060" },
-	{}
+	{ .name = "pv88060" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pv88060_i2c_id);
 
diff --git a/drivers/regulator/pv88080-regulator.c b/drivers/regulator/pv88080-regulator.c
index 112eb1c73647e..b96afbf11ff34 100644
--- a/drivers/regulator/pv88080-regulator.c
+++ b/drivers/regulator/pv88080-regulator.c
@@ -522,10 +522,10 @@ static const struct of_device_id pv88080_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, pv88080_dt_ids);
 
 static const struct i2c_device_id pv88080_i2c_id[] = {
-	{ "pv88080",    (kernel_ulong_t)&pv88080_aa_regs },
-	{ "pv88080-aa", (kernel_ulong_t)&pv88080_aa_regs },
-	{ "pv88080-ba", (kernel_ulong_t)&pv88080_ba_regs },
-	{}
+	{ .name = "pv88080", .driver_data = (kernel_ulong_t)&pv88080_aa_regs },
+	{ .name = "pv88080-aa", .driver_data = (kernel_ulong_t)&pv88080_aa_regs },
+	{ .name = "pv88080-ba", .driver_data = (kernel_ulong_t)&pv88080_ba_regs },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pv88080_i2c_id);
 
diff --git a/drivers/regulator/pv88090-regulator.c b/drivers/regulator/pv88090-regulator.c
index 3c48757bbbdad..ca5eeb5dfe621 100644
--- a/drivers/regulator/pv88090-regulator.c
+++ b/drivers/regulator/pv88090-regulator.c
@@ -381,8 +381,8 @@ static int pv88090_i2c_probe(struct i2c_client *i2c)
 }
 
 static const struct i2c_device_id pv88090_i2c_id[] = {
-	{ "pv88090" },
-	{}
+	{ .name = "pv88090" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pv88090_i2c_id);
 
diff --git a/drivers/regulator/qcom-refgen-regulator.c b/drivers/regulator/qcom-refgen-regulator.c
index 299ac3c8c3bc3..cc72e5a43ba62 100644
--- a/drivers/regulator/qcom-refgen-regulator.c
+++ b/drivers/regulator/qcom-refgen-regulator.c
@@ -3,6 +3,7 @@
 // Copyright (c) 2023, Linaro Limited
 
 #include <linux/bitfield.h>
+#include <linux/clk.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
@@ -16,6 +17,9 @@
  #define REFGEN_BIAS_EN_ENABLE		0x7
  #define REFGEN_BIAS_EN_DISABLE		0x6
 
+#define REFGEN_REG_REFGEN_STATUS	0xc
+#define REFGEN_STATUS_OUT_MASK		BIT(3)
+
 #define REFGEN_REG_BG_CTRL		0x14
 #define REFGEN_BG_CTRL_MASK		GENMASK(2, 1)
  #define REFGEN_BG_CTRL_ENABLE		0x3
@@ -25,6 +29,17 @@
 #define REFGEN_PWRDWN_CTRL5_MASK	BIT(0)
  #define REFGEN_PWRDWN_CTRL5_ENABLE	0x1
 
+struct qcom_refgen_regulator_data {
+	const struct regulator_desc *rdesc;
+	bool has_clocks;
+};
+
+struct qcom_refgen_drvdata {
+	struct clk_bulk_data *clks;
+	int num_clks;
+	bool is_enabled;
+};
+
 static int qcom_sdm845_refgen_enable(struct regulator_dev *rdev)
 {
 	regmap_update_bits(rdev->regmap, REFGEN_REG_BG_CTRL, REFGEN_BG_CTRL_MASK,
@@ -62,11 +77,67 @@ static int qcom_sdm845_refgen_is_enabled(struct regulator_dev *rdev)
 	return 1;
 }
 
+static int qcom_ipq9650_refgen_enable(struct regulator_dev *rdev)
+{
+	struct qcom_refgen_drvdata *drvdata = rdev_get_drvdata(rdev);
+	int ret;
+
+	ret = clk_bulk_prepare_enable(drvdata->num_clks, drvdata->clks);
+	if (ret)
+		return ret;
+
+	drvdata->is_enabled = true;
+
+	return 0;
+}
+
+static int qcom_ipq9650_refgen_disable(struct regulator_dev *rdev)
+{
+	struct qcom_refgen_drvdata *drvdata = rdev_get_drvdata(rdev);
+
+	clk_bulk_disable_unprepare(drvdata->num_clks, drvdata->clks);
+
+	drvdata->is_enabled = false;
+
+	return 0;
+}
+
+static int qcom_ipq9650_refgen_is_enabled(struct regulator_dev *rdev)
+{
+	struct qcom_refgen_drvdata *drvdata = rdev_get_drvdata(rdev);
+
+	return drvdata->is_enabled;
+}
+
+static int qcom_ipq9650_refgen_get_status(struct regulator_dev *rdev)
+{
+	u32 val;
+
+	regmap_read(rdev->regmap, REFGEN_REG_REFGEN_STATUS, &val);
+	if (FIELD_GET(REFGEN_STATUS_OUT_MASK, val))
+		return REGULATOR_STATUS_ON;
+
+	return REGULATOR_STATUS_OFF;
+}
+
+static const struct regulator_desc ipq9650_refgen_desc = {
+	.enable_time = 5,
+	.name = "refgen",
+	.owner = THIS_MODULE,
+	.type = REGULATOR_CURRENT,
+	.ops = &(const struct regulator_ops) {
+		.enable		= qcom_ipq9650_refgen_enable,
+		.disable	= qcom_ipq9650_refgen_disable,
+		.is_enabled	= qcom_ipq9650_refgen_is_enabled,
+		.get_status	= qcom_ipq9650_refgen_get_status,
+	},
+};
+
 static const struct regulator_desc sdm845_refgen_desc = {
 	.enable_time = 5,
 	.name = "refgen",
 	.owner = THIS_MODULE,
-	.type = REGULATOR_VOLTAGE,
+	.type = REGULATOR_CURRENT,
 	.ops = &(const struct regulator_ops) {
 		.enable		= qcom_sdm845_refgen_enable,
 		.disable	= qcom_sdm845_refgen_disable,
@@ -82,7 +153,7 @@ static const struct regulator_desc sm8250_refgen_desc = {
 	.enable_time = 5,
 	.name = "refgen",
 	.owner = THIS_MODULE,
-	.type = REGULATOR_VOLTAGE,
+	.type = REGULATOR_CURRENT,
 	.ops = &(const struct regulator_ops) {
 		.enable		= regulator_enable_regmap,
 		.disable	= regulator_disable_regmap,
@@ -90,6 +161,19 @@ static const struct regulator_desc sm8250_refgen_desc = {
 	},
 };
 
+static const struct qcom_refgen_regulator_data ipq9650_data = {
+	.rdesc = &ipq9650_refgen_desc,
+	.has_clocks = true,
+};
+
+static const struct qcom_refgen_regulator_data sdm845_data = {
+	.rdesc = &sdm845_refgen_desc,
+};
+
+static const struct qcom_refgen_regulator_data sm8250_data = {
+	.rdesc = &sm8250_refgen_desc,
+};
+
 static const struct regmap_config qcom_refgen_regmap_config = {
 	.reg_bits = 32,
 	.reg_stride = 4,
@@ -98,6 +182,8 @@ static const struct regmap_config qcom_refgen_regmap_config = {
 
 static int qcom_refgen_probe(struct platform_device *pdev)
 {
+	const struct qcom_refgen_regulator_data *data;
+	struct qcom_refgen_drvdata *drvdata = NULL;
 	struct regulator_init_data *init_data;
 	struct regulator_config config = {};
 	const struct regulator_desc *rdesc;
@@ -106,10 +192,23 @@ static int qcom_refgen_probe(struct platform_device *pdev)
 	struct regmap *regmap;
 	void __iomem *base;
 
-	rdesc = of_device_get_match_data(dev);
-	if (!rdesc)
+	data = of_device_get_match_data(dev);
+	if (!data)
 		return -ENODATA;
 
+	if (data->has_clocks) {
+		drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
+		if (!drvdata)
+			return -ENOMEM;
+
+		drvdata->num_clks = devm_clk_bulk_get_all(dev, &drvdata->clks);
+		if (drvdata->num_clks < 0)
+			return dev_err_probe(dev, drvdata->num_clks,
+					     "failed to get clocks\n");
+	}
+
+	rdesc = data->rdesc;
+
 	base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(base))
 		return PTR_ERR(base);
@@ -126,6 +225,7 @@ static int qcom_refgen_probe(struct platform_device *pdev)
 	config.init_data = init_data;
 	config.of_node = dev->of_node;
 	config.regmap = regmap;
+	config.driver_data = drvdata;
 
 	rdev = devm_regulator_register(dev, rdesc, &config);
 	if (IS_ERR(rdev))
@@ -135,8 +235,9 @@ static int qcom_refgen_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id qcom_refgen_match_table[] = {
-	{ .compatible = "qcom,sdm845-refgen-regulator", .data = &sdm845_refgen_desc },
-	{ .compatible = "qcom,sm8250-refgen-regulator", .data = &sm8250_refgen_desc },
+	{ .compatible = "qcom,ipq9650-refgen-regulator", .data = &ipq9650_data },
+	{ .compatible = "qcom,sdm845-refgen-regulator", .data = &sdm845_data },
+	{ .compatible = "qcom,sm8250-refgen-regulator", .data = &sm8250_data },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, qcom_refgen_match_table);
diff --git a/drivers/regulator/qcom_usb_vbus-regulator.c b/drivers/regulator/qcom_usb_vbus-regulator.c
index cd94ed67621fe..9aea68876284e 100644
--- a/drivers/regulator/qcom_usb_vbus-regulator.c
+++ b/drivers/regulator/qcom_usb_vbus-regulator.c
@@ -20,10 +20,35 @@
 #define OTG_CFG				0x53
 #define OTG_EN_SRC_CFG			BIT(1)
 
+#define PM4125_VBOOST_EN		0x50
+#define PM4125_VBOOST_SEL		0x52
+#define PM4125_VBOOST_CFG_MASK		GENMASK(1, 0)
+#define PM4125_VBOOST_CFG		0x56
+#define PM4125_VBOOST_EN_SRC_CFG	BIT(0)
+
+struct qcom_usb_vbus_reg_data {
+	u16 cmd_otg;
+	u16 otg_cfg;
+	u8  otg_en_src_cfg;
+	u16 csel_reg;
+	u8 csel_mask;
+	const unsigned int *curr_table;
+	unsigned int n_current_limits;
+	u16 vsel_reg;
+	u8 vsel_mask;
+	const unsigned int *volt_table;
+	unsigned int n_voltages;
+	const struct regulator_ops *ops;
+};
+
 static const unsigned int curr_table[] = {
 	500000, 1000000, 1500000, 2000000, 2500000, 3000000,
 };
 
+static const unsigned int pm4125_vboost_table[] = {
+	4250000, 4500000, 4750000, 5000000,
+};
+
 static const struct regulator_ops qcom_usb_vbus_reg_ops = {
 	.enable = regulator_enable_regmap,
 	.disable = regulator_disable_regmap,
@@ -32,19 +57,43 @@ static const struct regulator_ops qcom_usb_vbus_reg_ops = {
 	.set_current_limit = regulator_set_current_limit_regmap,
 };
 
-static struct regulator_desc qcom_usb_vbus_rdesc = {
-	.name = "usb_vbus",
-	.ops = &qcom_usb_vbus_reg_ops,
-	.owner = THIS_MODULE,
-	.type = REGULATOR_VOLTAGE,
+static const struct qcom_usb_vbus_reg_data pm8150b_data = {
+	.cmd_otg = CMD_OTG,
+	.otg_cfg = OTG_CFG,
+	.otg_en_src_cfg = OTG_EN_SRC_CFG,
+	.csel_reg = OTG_CURRENT_LIMIT_CFG,
+	.csel_mask = OTG_CURRENT_LIMIT_MASK,
 	.curr_table = curr_table,
 	.n_current_limits = ARRAY_SIZE(curr_table),
+	.ops = &qcom_usb_vbus_reg_ops,
+};
+
+static const struct regulator_ops qcom_usb_vbus_pm4125_reg_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.list_voltage = regulator_list_voltage_table,
+};
+
+static const struct qcom_usb_vbus_reg_data pm4125_data = {
+	.cmd_otg = PM4125_VBOOST_EN,
+	.otg_cfg = PM4125_VBOOST_CFG,
+	.otg_en_src_cfg = PM4125_VBOOST_EN_SRC_CFG,
+	.vsel_reg = PM4125_VBOOST_SEL,
+	.vsel_mask = PM4125_VBOOST_CFG_MASK,
+	.volt_table = pm4125_vboost_table,
+	.n_voltages = ARRAY_SIZE(pm4125_vboost_table),
+	.ops = &qcom_usb_vbus_pm4125_reg_ops,
 };
 
 static int qcom_usb_vbus_regulator_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
+	const struct qcom_usb_vbus_reg_data *data;
 	struct regulator_dev *rdev;
+	struct regulator_desc *rdesc;
 	struct regmap *regmap;
 	struct regulator_config config = { };
 	struct regulator_init_data *init_data;
@@ -57,27 +106,51 @@ static int qcom_usb_vbus_regulator_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	data = of_device_get_match_data(dev);
+	if (!data)
+		return -EINVAL;
+
 	regmap = dev_get_regmap(dev->parent, NULL);
 	if (!regmap) {
 		dev_err(dev, "Failed to get regmap\n");
 		return -ENOENT;
 	}
 
-	init_data = of_get_regulator_init_data(dev, dev->of_node,
-					       &qcom_usb_vbus_rdesc);
+	rdesc = devm_kzalloc(dev, sizeof(*rdesc), GFP_KERNEL);
+	if (!rdesc)
+		return -ENOMEM;
+
+	rdesc->name = "usb_vbus";
+	rdesc->ops = data->ops;
+	rdesc->owner = THIS_MODULE;
+	rdesc->type = REGULATOR_VOLTAGE;
+	rdesc->enable_reg = base + data->cmd_otg;
+	rdesc->enable_mask = OTG_EN;
+
+	if (data->curr_table) {
+		rdesc->curr_table = data->curr_table;
+		rdesc->n_current_limits = data->n_current_limits;
+		rdesc->csel_reg = base + data->csel_reg;
+		rdesc->csel_mask = data->csel_mask;
+	}
+
+	if (data->volt_table) {
+		rdesc->volt_table = data->volt_table;
+		rdesc->n_voltages = data->n_voltages;
+		rdesc->vsel_reg = base + data->vsel_reg;
+		rdesc->vsel_mask = data->vsel_mask;
+	}
+
+	init_data = of_get_regulator_init_data(dev, dev->of_node, rdesc);
 	if (!init_data)
 		return -ENOMEM;
 
-	qcom_usb_vbus_rdesc.enable_reg = base + CMD_OTG;
-	qcom_usb_vbus_rdesc.enable_mask = OTG_EN;
-	qcom_usb_vbus_rdesc.csel_reg = base + OTG_CURRENT_LIMIT_CFG;
-	qcom_usb_vbus_rdesc.csel_mask = OTG_CURRENT_LIMIT_MASK;
 	config.dev = dev;
 	config.init_data = init_data;
 	config.of_node = dev->of_node;
 	config.regmap = regmap;
 
-	rdev = devm_regulator_register(dev, &qcom_usb_vbus_rdesc, &config);
+	rdev = devm_regulator_register(dev, rdesc, &config);
 	if (IS_ERR(rdev)) {
 		ret = PTR_ERR(rdev);
 		dev_err(dev, "not able to register vbus reg %d\n", ret);
@@ -85,13 +158,14 @@ static int qcom_usb_vbus_regulator_probe(struct platform_device *pdev)
 	}
 
 	/* Disable HW logic for VBUS enable */
-	regmap_update_bits(regmap, base + OTG_CFG, OTG_EN_SRC_CFG, 0);
+	regmap_update_bits(regmap, base + data->otg_cfg, data->otg_en_src_cfg, 0);
 
 	return 0;
 }
 
 static const struct of_device_id qcom_usb_vbus_regulator_match[] = {
-	{ .compatible = "qcom,pm8150b-vbus-reg" },
+	{ .compatible = "qcom,pm8150b-vbus-reg", .data = &pm8150b_data },
+	{ .compatible = "qcom,pm4125-vbus-reg",  .data = &pm4125_data },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, qcom_usb_vbus_regulator_match);
diff --git a/drivers/regulator/rtq2208-regulator.c b/drivers/regulator/rtq2208-regulator.c
index 4fe111aa018fa..b4eb5d3eb5bde 100644
--- a/drivers/regulator/rtq2208-regulator.c
+++ b/drivers/regulator/rtq2208-regulator.c
@@ -11,6 +11,7 @@
 #include <linux/regulator/of_regulator.h>
 
 /* Register */
+#define RTQ2208_REG_FSOUTB_CNTL			0x11
 #define RTQ2208_REG_GLOBAL_INT1			0x12
 #define RTQ2208_REG_FLT_RECORDBUCK_CB		0x18
 #define RTQ2208_REG_GLOBAL_INT1_MASK		0x1D
@@ -33,6 +34,7 @@
 #define RTQ2208_REG_HIDDEN1			0xFF
 
 /* Mask */
+#define RTQ2208_MTP_SEL_RO_MASK			BIT(7)
 #define RTQ2208_BUCK_NR_MTP_SEL_MASK		GENMASK(7, 0)
 #define RTQ2208_BUCK_EN_NR_MTP_SEL0_MASK	BIT(0)
 #define RTQ2208_BUCK_EN_NR_MTP_SEL1_MASK	BIT(1)
@@ -464,10 +466,13 @@ static int rtq2208_parse_regulator_dt_data(int n_regulator, const unsigned int *
 		struct rtq2208_regulator_desc *rdesc[RTQ2208_LDO_MAX], struct device *dev,
 		unsigned int ldo1_fixed, unsigned int ldo2_fixed)
 {
+	struct regmap *regmap = dev_get_regmap(dev, NULL);
 	int mtp_sel, i, idx;
 
 	/* get mtp_sel0 or mtp_sel1 */
-	mtp_sel = device_property_read_bool(dev, "richtek,mtp-sel-high");
+	mtp_sel = regmap_test_bits(regmap, RTQ2208_REG_FSOUTB_CNTL, RTQ2208_MTP_SEL_RO_MASK);
+	if (mtp_sel < 0)
+		return dev_err_probe(dev, mtp_sel, "Failed to init mtp_sel state\n");
 
 	for (i = 0; i < n_regulator; i++) {
 		idx = regulator_idx_table[i];
diff --git a/drivers/regulator/sc2730-regulator.c b/drivers/regulator/sc2730-regulator.c
new file mode 100644
index 0000000000000..166b19ed539b2
--- /dev/null
+++ b/drivers/regulator/sc2730-regulator.c
@@ -0,0 +1,375 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018-2021 Unisoc Inc.
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/of_regulator.h>
+
+/*
+ * SC2730 regulator base address
+ */
+#define SC2730_REGULATOR_BASE		0x1800
+
+/*
+ * SC2730 regulator lock register
+ */
+#define SC2730_WR_UNLOCK_VALUE		0x6e7f
+#define SC2730_PWR_WR_PROT		(SC2730_REGULATOR_BASE + 0x3d0)
+
+/*
+ * SC2730 enable register
+ */
+#define SC2730_POWER_PD_SW		(SC2730_REGULATOR_BASE + 0x01c)
+#define SC2730_LDO_VDDRF18_PD		(SC2730_REGULATOR_BASE + 0x10c)
+#define SC2730_LDO_VDDCAMIO_PD		(SC2730_REGULATOR_BASE + 0x118)
+#define SC2730_LDO_VDDWCN_PD		(SC2730_REGULATOR_BASE + 0x11c)
+#define SC2730_LDO_VDDCAMD1_PD		(SC2730_REGULATOR_BASE + 0x128)
+#define SC2730_LDO_VDDCAMD0_PD		(SC2730_REGULATOR_BASE + 0x134)
+#define SC2730_LDO_VDDRF1V25_PD		(SC2730_REGULATOR_BASE + 0x140)
+#define SC2730_LDO_AVDD12_PD		(SC2730_REGULATOR_BASE + 0x14c)
+#define SC2730_LDO_VDDCAMA0_PD		(SC2730_REGULATOR_BASE + 0x158)
+#define SC2730_LDO_VDDCAMA1_PD		(SC2730_REGULATOR_BASE + 0x164)
+#define SC2730_LDO_VDDCAMMOT_PD		(SC2730_REGULATOR_BASE + 0x170)
+#define SC2730_LDO_VDDSIM2_PD		(SC2730_REGULATOR_BASE + 0x194)
+#define SC2730_LDO_VDDEMMCCORE_PD	(SC2730_REGULATOR_BASE + 0x1a0)
+#define SC2730_LDO_VDDSDCORE_PD		(SC2730_REGULATOR_BASE + 0x1ac)
+#define SC2730_LDO_VDDSDIO_PD		(SC2730_REGULATOR_BASE + 0x1b8)
+#define SC2730_LDO_VDDWIFIPA_PD		(SC2730_REGULATOR_BASE + 0x1d0)
+#define SC2730_LDO_VDDUSB33_PD		(SC2730_REGULATOR_BASE + 0x1e8)
+#define SC2730_LDO_VDDLDO0_PD		(SC2730_REGULATOR_BASE + 0x1f4)
+#define SC2730_LDO_VDDLDO1_PD		(SC2730_REGULATOR_BASE + 0x200)
+#define SC2730_LDO_VDDLDO2_PD		(SC2730_REGULATOR_BASE + 0x20c)
+#define SC2730_LDO_VDDKPLED_PD		(SC2730_REGULATOR_BASE + 0x38c)
+
+/*
+ * SC2730 enable mask
+ */
+#define SC2730_DCDC_CPU_PD_MASK		BIT(4)
+#define SC2730_DCDC_GPU_PD_MASK		BIT(3)
+#define SC2730_DCDC_CORE_PD_MASK	BIT(5)
+#define SC2730_DCDC_MODEM_PD_MASK	BIT(11)
+#define SC2730_DCDC_MEM_PD_MASK		BIT(6)
+#define SC2730_DCDC_MEMQ_PD_MASK	BIT(12)
+#define SC2730_DCDC_GEN0_PD_MASK	BIT(8)
+#define SC2730_DCDC_GEN1_PD_MASK	BIT(7)
+#define SC2730_DCDC_SRAM_PD_MASK	BIT(13)
+#define SC2730_LDO_AVDD18_PD_MASK	BIT(2)
+#define SC2730_LDO_VDDRF18_PD_MASK	BIT(0)
+#define SC2730_LDO_VDDCAMIO_PD_MASK	BIT(0)
+#define SC2730_LDO_VDDWCN_PD_MASK	BIT(0)
+#define SC2730_LDO_VDDCAMD1_PD_MASK	BIT(0)
+#define SC2730_LDO_VDDCAMD0_PD_MASK	BIT(0)
+#define SC2730_LDO_VDDRF1V25_PD_MASK	BIT(0)
+#define SC2730_LDO_AVDD12_PD_MASK	BIT(0)
+#define SC2730_LDO_VDDCAMA0_PD_MASK	BIT(0)
+#define SC2730_LDO_VDDCAMA1_PD_MASK	BIT(0)
+#define SC2730_LDO_VDDCAMMOT_PD_MASK	BIT(0)
+#define SC2730_LDO_VDDSIM2_PD_MASK	BIT(0)
+#define SC2730_LDO_VDDEMMCCORE_PD_MASK	BIT(0)
+#define SC2730_LDO_VDDSDCORE_PD_MASK	BIT(0)
+#define SC2730_LDO_VDDSDIO_PD_MASK	BIT(0)
+#define SC2730_LDO_VDD28_PD_MASK	BIT(1)
+#define SC2730_LDO_VDDWIFIPA_PD_MASK	BIT(0)
+#define SC2730_LDO_VDD18_DCXO_PD_MASK	BIT(10)
+#define SC2730_LDO_VDDUSB33_PD_MASK	BIT(0)
+#define SC2730_LDO_VDDLDO0_PD_MASK	BIT(0)
+#define SC2730_LDO_VDDLDO1_PD_MASK	BIT(0)
+#define SC2730_LDO_VDDLDO2_PD_MASK	BIT(0)
+#define SC2730_LDO_VDDKPLED_PD_MASK	BIT(15)
+
+/*
+ * SC2730 vsel register
+ */
+#define SC2730_DCDC_CPU_VOL		(SC2730_REGULATOR_BASE + 0x44)
+#define SC2730_DCDC_GPU_VOL		(SC2730_REGULATOR_BASE + 0x54)
+#define SC2730_DCDC_CORE_VOL		(SC2730_REGULATOR_BASE + 0x64)
+#define SC2730_DCDC_MODEM_VOL		(SC2730_REGULATOR_BASE + 0x74)
+#define SC2730_DCDC_MEM_VOL		(SC2730_REGULATOR_BASE + 0x84)
+#define SC2730_DCDC_MEMQ_VOL		(SC2730_REGULATOR_BASE + 0x94)
+#define SC2730_DCDC_GEN0_VOL		(SC2730_REGULATOR_BASE + 0xa4)
+#define SC2730_DCDC_GEN1_VOL		(SC2730_REGULATOR_BASE + 0xb4)
+#define SC2730_DCDC_SRAM_VOL		(SC2730_REGULATOR_BASE + 0xdc)
+#define SC2730_LDO_AVDD18_VOL		(SC2730_REGULATOR_BASE + 0x104)
+#define SC2730_LDO_VDDRF18_VOL		(SC2730_REGULATOR_BASE + 0x110)
+#define SC2730_LDO_VDDCAMIO_VOL		(SC2730_REGULATOR_BASE + 0x28)
+#define SC2730_LDO_VDDWCN_VOL		(SC2730_REGULATOR_BASE + 0x120)
+#define SC2730_LDO_VDDCAMD1_VOL		(SC2730_REGULATOR_BASE + 0x12c)
+#define SC2730_LDO_VDDCAMD0_VOL		(SC2730_REGULATOR_BASE + 0x138)
+#define SC2730_LDO_VDDRF1V25_VOL	(SC2730_REGULATOR_BASE + 0x144)
+#define SC2730_LDO_AVDD12_VOL		(SC2730_REGULATOR_BASE + 0x150)
+#define SC2730_LDO_VDDCAMA0_VOL		(SC2730_REGULATOR_BASE + 0x15c)
+#define SC2730_LDO_VDDCAMA1_VOL		(SC2730_REGULATOR_BASE + 0x168)
+#define SC2730_LDO_VDDCAMMOT_VOL	(SC2730_REGULATOR_BASE + 0x174)
+#define SC2730_LDO_VDDSIM2_VOL		(SC2730_REGULATOR_BASE + 0x198)
+#define SC2730_LDO_VDDEMMCCORE_VOL	(SC2730_REGULATOR_BASE + 0x1a4)
+#define SC2730_LDO_VDDSDCORE_VOL	(SC2730_REGULATOR_BASE + 0x1b0)
+#define SC2730_LDO_VDDSDIO_VOL		(SC2730_REGULATOR_BASE + 0x1bc)
+#define SC2730_LDO_VDD28_VOL		(SC2730_REGULATOR_BASE + 0x1c8)
+#define SC2730_LDO_VDDWIFIPA_VOL	(SC2730_REGULATOR_BASE + 0x1d4)
+#define SC2730_LDO_VDD18_DCXO_VOL	(SC2730_REGULATOR_BASE + 0x1e0)
+#define SC2730_LDO_VDDUSB33_VOL		(SC2730_REGULATOR_BASE + 0x1ec)
+#define SC2730_LDO_VDDLDO0_VOL		(SC2730_REGULATOR_BASE + 0x1f8)
+#define SC2730_LDO_VDDLDO1_VOL		(SC2730_REGULATOR_BASE + 0x204)
+#define SC2730_LDO_VDDLDO2_VOL		(SC2730_REGULATOR_BASE + 0x210)
+#define SC2730_LDO_VDDKPLED_VOL		(SC2730_REGULATOR_BASE + 0x38c)
+
+/*
+ * SC2730 vsel register mask
+ */
+#define SC2730_DCDC_CPU_VOL_MASK	GENMASK(8, 0)
+#define SC2730_DCDC_GPU_VOL_MASK	GENMASK(8, 0)
+#define SC2730_DCDC_CORE_VOL_MASK	GENMASK(8, 0)
+#define SC2730_DCDC_MODEM_VOL_MASK	GENMASK(8, 0)
+#define SC2730_DCDC_MEM_VOL_MASK	GENMASK(7, 0)
+#define SC2730_DCDC_MEMQ_VOL_MASK	GENMASK(8, 0)
+#define SC2730_DCDC_GEN0_VOL_MASK	GENMASK(7, 0)
+#define SC2730_DCDC_GEN1_VOL_MASK	GENMASK(7, 0)
+#define SC2730_DCDC_SRAM_VOL_MASK	GENMASK(8, 0)
+#define SC2730_LDO_AVDD18_VOL_MASK	GENMASK(5, 0)
+#define SC2730_LDO_VDDRF18_VOL_MASK	GENMASK(5, 0)
+#define SC2730_LDO_VDDCAMIO_VOL_MASK	GENMASK(5, 0)
+#define SC2730_LDO_VDDWCN_VOL_MASK	GENMASK(5, 0)
+#define SC2730_LDO_VDDCAMD1_VOL_MASK	GENMASK(4, 0)
+#define SC2730_LDO_VDDCAMD0_VOL_MASK	GENMASK(4, 0)
+#define SC2730_LDO_VDDRF1V25_VOL_MASK	GENMASK(4, 0)
+#define SC2730_LDO_AVDD12_VOL_MASK	GENMASK(4, 0)
+#define SC2730_LDO_VDDCAMA0_VOL_MASK	GENMASK(7, 0)
+#define SC2730_LDO_VDDCAMA1_VOL_MASK	GENMASK(7, 0)
+#define SC2730_LDO_VDDCAMMOT_VOL_MASK	GENMASK(7, 0)
+#define SC2730_LDO_VDDSIM2_VOL_MASK	GENMASK(7, 0)
+#define SC2730_LDO_VDDEMMCCORE_VOL_MASK	GENMASK(7, 0)
+#define SC2730_LDO_VDDSDCORE_VOL_MASK	GENMASK(7, 0)
+#define SC2730_LDO_VDDSDIO_VOL_MASK	GENMASK(7, 0)
+#define SC2730_LDO_VDD28_VOL_MASK	GENMASK(7, 0)
+#define SC2730_LDO_VDDWIFIPA_VOL_MASK	GENMASK(7, 0)
+#define SC2730_LDO_VDD18_DCXO_VOL_MASK	GENMASK(7, 0)
+#define SC2730_LDO_VDDUSB33_VOL_MASK	GENMASK(7, 0)
+#define SC2730_LDO_VDDLDO0_VOL_MASK	GENMASK(7, 0)
+#define SC2730_LDO_VDDLDO1_VOL_MASK	GENMASK(7, 0)
+#define SC2730_LDO_VDDLDO2_VOL_MASK	GENMASK(7, 0)
+#define SC2730_LDO_VDDKPLED_VOL_MASK	GENMASK(14, 7)
+
+enum sc2730_regulator_id {
+	SC2730_DCDC_CPU,
+	SC2730_DCDC_GPU,
+	SC2730_DCDC_CORE,
+	SC2730_DCDC_MODEM,
+	SC2730_DCDC_MEM,
+	SC2730_DCDC_MEMQ,
+	SC2730_DCDC_GEN0,
+	SC2730_DCDC_GEN1,
+	SC2730_DCDC_SRAM,
+	SC2730_LDO_AVDD18,
+	SC2730_LDO_VDDRF18,
+	SC2730_LDO_VDDCAMIO,
+	SC2730_LDO_VDDWCN,
+	SC2730_LDO_VDDCAMD1,
+	SC2730_LDO_VDDCAMD0,
+	SC2730_LDO_VDDRF1V25,
+	SC2730_LDO_AVDD12,
+	SC2730_LDO_VDDCAMA0,
+	SC2730_LDO_VDDCAMA1,
+	SC2730_LDO_VDDCAMMOT,
+	SC2730_LDO_VDDSIM2,
+	SC2730_LDO_VDDEMMCCORE,
+	SC2730_LDO_VDDSDCORE,
+	SC2730_LDO_VDDSDIO,
+	SC2730_LDO_VDD28,
+	SC2730_LDO_VDDWIFIPA,
+	SC2730_LDO_VDD18_DCXO,
+	SC2730_LDO_VDDUSB33,
+	SC2730_LDO_VDDLDO0,
+	SC2730_LDO_VDDLDO1,
+	SC2730_LDO_VDDLDO2,
+	SC2730_LDO_VDDKPLED,
+};
+
+static const struct regulator_ops sc2730_regu_linear_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.list_voltage = regulator_list_voltage_linear,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+};
+
+#define SC2730_REGU_LINEAR(_id, of_name, en_reg, en_mask, vreg, vmask,	\
+			   vstep, vmin, vmax) {				\
+	.name			= #_id,					\
+	.of_match		= of_name,				\
+	.regulators_node	= "regulators",				\
+	.ops			= &sc2730_regu_linear_ops,		\
+	.type			= REGULATOR_VOLTAGE,			\
+	.id			= SC2730_##_id,				\
+	.owner			= THIS_MODULE,				\
+	.min_uV			= vmin,					\
+	.n_voltages		= ((vmax) - (vmin)) / (vstep) + 1,	\
+	.uV_step		= vstep,				\
+	.enable_is_inverted	= true,					\
+	.enable_val		= 0,					\
+	.enable_reg		= en_reg,				\
+	.enable_mask		= en_mask,				\
+	.vsel_reg		= vreg,					\
+	.vsel_mask		= vmask,				\
+	.linear_min_sel		= 0,					\
+}
+
+static const struct regulator_desc regulators[] = {
+	SC2730_REGU_LINEAR(DCDC_CPU, "dcdc-cpu", SC2730_POWER_PD_SW,
+			   SC2730_DCDC_CPU_PD_MASK, SC2730_DCDC_CPU_VOL,
+			   SC2730_DCDC_CPU_VOL_MASK, 3125, 0, 1596875),
+	SC2730_REGU_LINEAR(DCDC_GPU, "dcdc-gpu", SC2730_POWER_PD_SW,
+			   SC2730_DCDC_GPU_PD_MASK, SC2730_DCDC_GPU_VOL,
+			   SC2730_DCDC_GPU_VOL_MASK, 3125, 0, 1596875),
+	SC2730_REGU_LINEAR(DCDC_CORE, "dcdc-core", SC2730_POWER_PD_SW,
+			   SC2730_DCDC_CORE_PD_MASK, SC2730_DCDC_CORE_VOL,
+			   SC2730_DCDC_CORE_VOL_MASK, 3125, 0, 1596875),
+	SC2730_REGU_LINEAR(DCDC_MODEM, "dcdc-modem", SC2730_POWER_PD_SW,
+			   SC2730_DCDC_MODEM_PD_MASK, SC2730_DCDC_MODEM_VOL,
+			   SC2730_DCDC_MODEM_VOL_MASK, 3125, 0, 1596875),
+	SC2730_REGU_LINEAR(DCDC_MEM, "dcdc-mem", SC2730_POWER_PD_SW,
+			   SC2730_DCDC_MEM_PD_MASK, SC2730_DCDC_MEM_VOL,
+			   SC2730_DCDC_MEM_VOL_MASK, 6250, 0, 1593750),
+	SC2730_REGU_LINEAR(DCDC_MEMQ, "dcdc-memq", SC2730_POWER_PD_SW,
+			   SC2730_DCDC_MEMQ_PD_MASK, SC2730_DCDC_MEMQ_VOL,
+			   SC2730_DCDC_MEMQ_VOL_MASK, 3125, 0, 1596875),
+	SC2730_REGU_LINEAR(DCDC_GEN0, "dcdc-gen0", SC2730_POWER_PD_SW,
+			   SC2730_DCDC_GEN0_PD_MASK, SC2730_DCDC_GEN0_VOL,
+			   SC2730_DCDC_GEN0_VOL_MASK, 9375, 20000, 2410625),
+	SC2730_REGU_LINEAR(DCDC_GEN1, "dcdc-gen1", SC2730_POWER_PD_SW,
+			   SC2730_DCDC_GEN1_PD_MASK, SC2730_DCDC_GEN1_VOL,
+			   SC2730_DCDC_GEN1_VOL_MASK, 6250, 50000, 1643750),
+	SC2730_REGU_LINEAR(DCDC_SRAM, "dcdc-sram", SC2730_POWER_PD_SW,
+			   SC2730_DCDC_SRAM_PD_MASK, SC2730_DCDC_SRAM_VOL,
+			   SC2730_DCDC_SRAM_VOL_MASK, 3125, 0, 1596875),
+	SC2730_REGU_LINEAR(LDO_AVDD18, "ldo-avdd18", SC2730_POWER_PD_SW,
+			   SC2730_LDO_AVDD18_PD_MASK, SC2730_LDO_AVDD18_VOL,
+			   SC2730_LDO_AVDD18_VOL_MASK, 10000, 1175000, 1805000),
+	SC2730_REGU_LINEAR(LDO_VDDRF18, "ldo-vddrf18", SC2730_LDO_VDDRF18_PD,
+			   SC2730_LDO_VDDRF18_PD_MASK, SC2730_LDO_VDDRF18_VOL,
+			   SC2730_LDO_VDDRF18_VOL_MASK, 10000, 1175000, 1805000),
+	SC2730_REGU_LINEAR(LDO_VDDCAMIO, "ldo-vddcamio", SC2730_LDO_VDDCAMIO_PD,
+			   SC2730_LDO_VDDCAMIO_PD_MASK, SC2730_LDO_VDDCAMIO_VOL,
+			   SC2730_LDO_VDDCAMIO_VOL_MASK, 10000, 1200000, 1830000),
+	SC2730_REGU_LINEAR(LDO_VDDWCN, "ldo-vddwcn", SC2730_LDO_VDDWCN_PD,
+			   SC2730_LDO_VDDWCN_PD_MASK, SC2730_LDO_VDDWCN_VOL,
+			   SC2730_LDO_VDDWCN_VOL_MASK, 15000, 900000, 1845000),
+	SC2730_REGU_LINEAR(LDO_VDDCAMD1, "ldo-vddcamd1", SC2730_LDO_VDDCAMD1_PD,
+			   SC2730_LDO_VDDCAMD1_PD_MASK, SC2730_LDO_VDDCAMD1_VOL,
+			   SC2730_LDO_VDDCAMD1_VOL_MASK, 15000, 900000, 1365000),
+	SC2730_REGU_LINEAR(LDO_VDDCAMD0, "ldo-vddcamd0", SC2730_LDO_VDDCAMD0_PD,
+			   SC2730_LDO_VDDCAMD0_PD_MASK, SC2730_LDO_VDDCAMD0_VOL,
+			   SC2730_LDO_VDDCAMD0_VOL_MASK, 15000, 900000, 1365000),
+	SC2730_REGU_LINEAR(LDO_VDDRF1V25, "ldo-vddrf1v25", SC2730_LDO_VDDRF1V25_PD,
+			   SC2730_LDO_VDDRF1V25_PD_MASK, SC2730_LDO_VDDRF1V25_VOL,
+			   SC2730_LDO_VDDRF1V25_VOL_MASK, 15000, 900000, 1365000),
+	SC2730_REGU_LINEAR(LDO_AVDD12, "ldo-avdd12", SC2730_LDO_AVDD12_PD,
+			   SC2730_LDO_AVDD12_PD_MASK, SC2730_LDO_AVDD12_VOL,
+			   SC2730_LDO_AVDD12_VOL_MASK, 15000, 900000, 1365000),
+	SC2730_REGU_LINEAR(LDO_VDDCAMA0, "ldo-vddcama0", SC2730_LDO_VDDCAMA0_PD,
+			   SC2730_LDO_VDDCAMA0_PD_MASK, SC2730_LDO_VDDCAMA0_VOL,
+			   SC2730_LDO_VDDCAMA0_VOL_MASK, 10000, 1200000, 3750000),
+	SC2730_REGU_LINEAR(LDO_VDDCAMA1, "ldo-vddcama1", SC2730_LDO_VDDCAMA1_PD,
+			   SC2730_LDO_VDDCAMA1_PD_MASK, SC2730_LDO_VDDCAMA1_VOL,
+			   SC2730_LDO_VDDCAMA1_VOL_MASK, 10000, 1200000, 3750000),
+	SC2730_REGU_LINEAR(LDO_VDDCAMMOT, "ldo-vddcammot", SC2730_LDO_VDDCAMMOT_PD,
+			   SC2730_LDO_VDDCAMMOT_PD_MASK, SC2730_LDO_VDDCAMMOT_VOL,
+			   SC2730_LDO_VDDCAMMOT_VOL_MASK, 10000, 1200000, 3750000),
+	SC2730_REGU_LINEAR(LDO_VDDSIM2, "ldo-vddsim2", SC2730_LDO_VDDSIM2_PD,
+			   SC2730_LDO_VDDSIM2_PD_MASK, SC2730_LDO_VDDSIM2_VOL,
+			   SC2730_LDO_VDDSIM2_VOL_MASK, 10000, 1200000, 3750000),
+	SC2730_REGU_LINEAR(LDO_VDDEMMCCORE, "ldo-vddemmccore", SC2730_LDO_VDDEMMCCORE_PD,
+			   SC2730_LDO_VDDEMMCCORE_PD_MASK, SC2730_LDO_VDDEMMCCORE_VOL,
+			   SC2730_LDO_VDDEMMCCORE_VOL_MASK, 10000, 1200000, 3750000),
+	SC2730_REGU_LINEAR(LDO_VDDSDCORE, "ldo-vddsdcore", SC2730_LDO_VDDSDCORE_PD,
+			   SC2730_LDO_VDDSDCORE_PD_MASK, SC2730_LDO_VDDSDCORE_VOL,
+			   SC2730_LDO_VDDSDCORE_VOL_MASK, 10000, 1200000, 3750000),
+	SC2730_REGU_LINEAR(LDO_VDDSDIO, "ldo-vddsdio", SC2730_LDO_VDDSDIO_PD,
+			   SC2730_LDO_VDDSDIO_PD_MASK, SC2730_LDO_VDDSDIO_VOL,
+			   SC2730_LDO_VDDSDIO_VOL_MASK, 10000, 1200000, 3750000),
+	SC2730_REGU_LINEAR(LDO_VDD28, "ldo-vdd28", SC2730_POWER_PD_SW,
+			   SC2730_LDO_VDD28_PD_MASK, SC2730_LDO_VDD28_VOL,
+			   SC2730_LDO_VDD28_VOL_MASK, 10000, 1200000, 3750000),
+	SC2730_REGU_LINEAR(LDO_VDDWIFIPA, "ldo-vddwifipa", SC2730_LDO_VDDWIFIPA_PD,
+			   SC2730_LDO_VDDWIFIPA_PD_MASK, SC2730_LDO_VDDWIFIPA_VOL,
+			   SC2730_LDO_VDDWIFIPA_VOL_MASK, 10000, 1200000, 3750000),
+	SC2730_REGU_LINEAR(LDO_VDD18_DCXO, "ldo-vdd18-dcxo", SC2730_POWER_PD_SW,
+			   SC2730_LDO_VDD18_DCXO_PD_MASK, SC2730_LDO_VDD18_DCXO_VOL,
+			   SC2730_LDO_VDD18_DCXO_VOL_MASK, 10000, 1200000, 3750000),
+	SC2730_REGU_LINEAR(LDO_VDDUSB33, "ldo-vddusb33", SC2730_LDO_VDDUSB33_PD,
+			   SC2730_LDO_VDDUSB33_PD_MASK, SC2730_LDO_VDDUSB33_VOL,
+			   SC2730_LDO_VDDUSB33_VOL_MASK, 10000, 1200000, 3750000),
+	SC2730_REGU_LINEAR(LDO_VDDLDO0, "ldo-vddldo0", SC2730_LDO_VDDLDO0_PD,
+			   SC2730_LDO_VDDLDO0_PD_MASK, SC2730_LDO_VDDLDO0_VOL,
+			   SC2730_LDO_VDDLDO0_VOL_MASK, 10000, 1200000, 3750000),
+	SC2730_REGU_LINEAR(LDO_VDDLDO1, "ldo-vddldo1", SC2730_LDO_VDDLDO1_PD,
+			   SC2730_LDO_VDDLDO1_PD_MASK, SC2730_LDO_VDDLDO1_VOL,
+			   SC2730_LDO_VDDLDO1_VOL_MASK, 10000, 1200000, 3750000),
+	SC2730_REGU_LINEAR(LDO_VDDLDO2, "ldo-vddldo2", SC2730_LDO_VDDLDO2_PD,
+			   SC2730_LDO_VDDLDO2_PD_MASK, SC2730_LDO_VDDLDO2_VOL,
+			   SC2730_LDO_VDDLDO2_VOL_MASK, 10000, 1200000, 3750000),
+	SC2730_REGU_LINEAR(LDO_VDDKPLED, "ldo-vddkpled", SC2730_LDO_VDDKPLED_PD,
+			   SC2730_LDO_VDDKPLED_PD_MASK, SC2730_LDO_VDDKPLED_VOL,
+			   SC2730_LDO_VDDKPLED_VOL_MASK, 10000, 1200000, 3750000),
+};
+
+static int sc2730_regulator_unlock(struct regmap *regmap)
+{
+	return regmap_write(regmap, SC2730_PWR_WR_PROT, SC2730_WR_UNLOCK_VALUE);
+}
+
+static int sc2730_regulator_probe(struct platform_device *pdev)
+{
+	int i, ret;
+	struct regmap *regmap;
+	struct regulator_config config = { };
+	struct regulator_dev *rdev;
+
+	regmap = dev_get_regmap(pdev->dev.parent, NULL);
+	if (!regmap)
+		return dev_err_probe(&pdev->dev, -ENODEV, "failed to get regmap\n");
+
+	ret = sc2730_regulator_unlock(regmap);
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret, "failed to release regulator lock\n");
+
+	config.dev = pdev->dev.parent;
+	config.regmap = regmap;
+
+	for (i = 0; i < ARRAY_SIZE(regulators); i++) {
+		rdev = devm_regulator_register(&pdev->dev, &regulators[i], &config);
+		if (IS_ERR(rdev)) {
+			return dev_err_probe(&pdev->dev, PTR_ERR(rdev),
+					     "failed to register regulator %s\n",
+					     regulators[i].name);
+		}
+	}
+
+	return 0;
+}
+
+static const struct platform_device_id sc2730_regulator_id_table[] = {
+	{ .name = "sc2730-regulator" },
+	{ }
+};
+MODULE_DEVICE_TABLE(platform, sc2730_regulator_id_table);
+
+static struct platform_driver sc2730_regulator_driver = {
+	.driver = {
+		.name = "sc2730-regulator",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+	},
+	.probe = sc2730_regulator_probe,
+	.id_table = sc2730_regulator_id_table,
+};
+
+module_platform_driver(sc2730_regulator_driver);
+
+MODULE_AUTHOR("Zhongfa Wang <zhongfa.wang@unisoc.com>");
+MODULE_DESCRIPTION("Spreadtrum SC2730 regulator driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/regulator/sgm3804-regulator.c b/drivers/regulator/sgm3804-regulator.c
index c3406cfb73d03..a34c9418b2758 100644
--- a/drivers/regulator/sgm3804-regulator.c
+++ b/drivers/regulator/sgm3804-regulator.c
@@ -285,7 +285,7 @@ static int sgm3804_probe(struct i2c_client *i2c)
 }
 
 static const struct i2c_device_id sgm3804_id[] = {
-	{ "sgm3804" },
+	{ .name = "sgm3804" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, sgm3804_id);
diff --git a/drivers/regulator/slg51000-regulator.c b/drivers/regulator/slg51000-regulator.c
index 3bbd4a29e6d32..d682764cdbf8f 100644
--- a/drivers/regulator/slg51000-regulator.c
+++ b/drivers/regulator/slg51000-regulator.c
@@ -497,8 +497,8 @@ static int slg51000_i2c_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id slg51000_i2c_id[] = {
-	{ "slg51000" },
-	{}
+	{ .name = "slg51000" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, slg51000_i2c_id);
 
diff --git a/drivers/regulator/sy8106a-regulator.c b/drivers/regulator/sy8106a-regulator.c
index d79a4cc25a0da..b2b835c602629 100644
--- a/drivers/regulator/sy8106a-regulator.c
+++ b/drivers/regulator/sy8106a-regulator.c
@@ -130,7 +130,7 @@ static const struct of_device_id sy8106a_i2c_of_match[] = {
 MODULE_DEVICE_TABLE(of, sy8106a_i2c_of_match);
 
 static const struct i2c_device_id sy8106a_i2c_id[] = {
-	{ "sy8106a" },
+	{ .name = "sy8106a" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, sy8106a_i2c_id);
diff --git a/drivers/regulator/sy8824x.c b/drivers/regulator/sy8824x.c
index 5bec84db25f13..3f07e7da90cbd 100644
--- a/drivers/regulator/sy8824x.c
+++ b/drivers/regulator/sy8824x.c
@@ -213,10 +213,10 @@ static const struct of_device_id sy8824_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, sy8824_dt_ids);
 
 static const struct i2c_device_id sy8824_id[] = {
-	{ "sy8824c", (kernel_ulong_t)&sy8824c_cfg },
-	{ "sy8824e", (kernel_ulong_t)&sy8824e_cfg },
-	{ "sy20276", (kernel_ulong_t)&sy20276_cfg },
-	{ "sy20278", (kernel_ulong_t)&sy20278_cfg },
+	{ .name = "sy8824c", .driver_data = (kernel_ulong_t)&sy8824c_cfg },
+	{ .name = "sy8824e", .driver_data = (kernel_ulong_t)&sy8824e_cfg },
+	{ .name = "sy20276", .driver_data = (kernel_ulong_t)&sy20276_cfg },
+	{ .name = "sy20278", .driver_data = (kernel_ulong_t)&sy20278_cfg },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, sy8824_id);
diff --git a/drivers/regulator/sy8827n.c b/drivers/regulator/sy8827n.c
index 0b811514782f5..a1cac8cc3d96b 100644
--- a/drivers/regulator/sy8827n.c
+++ b/drivers/regulator/sy8827n.c
@@ -180,8 +180,8 @@ static const struct of_device_id sy8827n_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, sy8827n_dt_ids);
 
 static const struct i2c_device_id sy8827n_id[] = {
-	{ "sy8827n", },
-	{ },
+	{ .name = "sy8827n" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, sy8827n_id);
 
diff --git a/drivers/regulator/tps51632-regulator.c b/drivers/regulator/tps51632-regulator.c
index 3a384bf9d2c59..8459b49ec26e6 100644
--- a/drivers/regulator/tps51632-regulator.c
+++ b/drivers/regulator/tps51632-regulator.c
@@ -331,8 +331,8 @@ static int tps51632_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tps51632_id[] = {
-	{.name = "tps51632",},
-	{},
+	{ .name = "tps51632" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, tps51632_id);
diff --git a/drivers/regulator/tps62360-regulator.c b/drivers/regulator/tps62360-regulator.c
index be6a6702cbfa9..00f19082c1daf 100644
--- a/drivers/regulator/tps62360-regulator.c
+++ b/drivers/regulator/tps62360-regulator.c
@@ -476,11 +476,11 @@ static void tps62360_shutdown(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tps62360_id[] = {
-	{.name = "tps62360", .driver_data = TPS62360},
-	{.name = "tps62361", .driver_data = TPS62361},
-	{.name = "tps62362", .driver_data = TPS62362},
-	{.name = "tps62363", .driver_data = TPS62363},
-	{},
+	{ .name = "tps62360", .driver_data = TPS62360 },
+	{ .name = "tps62361", .driver_data = TPS62361 },
+	{ .name = "tps62362", .driver_data = TPS62362 },
+	{ .name = "tps62363", .driver_data = TPS62363 },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, tps62360_id);
diff --git a/drivers/regulator/tps6286x-regulator.c b/drivers/regulator/tps6286x-regulator.c
index e29aab06bf79e..1ab53bee9f6e2 100644
--- a/drivers/regulator/tps6286x-regulator.c
+++ b/drivers/regulator/tps6286x-regulator.c
@@ -145,11 +145,11 @@ static int tps6286x_i2c_probe(struct i2c_client *i2c)
 }
 
 static const struct i2c_device_id tps6286x_i2c_id[] = {
-	{ "tps62864" },
-	{ "tps62866" },
-	{ "tps62868" },
-	{ "tps62869" },
-	{}
+	{ .name = "tps62864" },
+	{ .name = "tps62866" },
+	{ .name = "tps62868" },
+	{ .name = "tps62869" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tps6286x_i2c_id);
 
diff --git a/drivers/regulator/tps6287x-regulator.c b/drivers/regulator/tps6287x-regulator.c
index 9df104cb6bd2e..e6403398facc4 100644
--- a/drivers/regulator/tps6287x-regulator.c
+++ b/drivers/regulator/tps6287x-regulator.c
@@ -228,11 +228,11 @@ static const struct of_device_id tps6287x_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, tps6287x_dt_ids);
 
 static const struct i2c_device_id tps6287x_i2c_id[] = {
-	{ "tps62870" },
-	{ "tps62871" },
-	{ "tps62872" },
-	{ "tps62873" },
-	{}
+	{ .name = "tps62870" },
+	{ .name = "tps62871" },
+	{ .name = "tps62872" },
+	{ .name = "tps62873" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, tps6287x_i2c_id);
diff --git a/drivers/regulator/tps65023-regulator.c b/drivers/regulator/tps65023-regulator.c
index 3334b5b7d907f..5a2fc57cddf6b 100644
--- a/drivers/regulator/tps65023-regulator.c
+++ b/drivers/regulator/tps65023-regulator.c
@@ -319,13 +319,13 @@ MODULE_DEVICE_TABLE(of, tps65023_of_match);
 static const struct i2c_device_id tps_65023_id[] = {
 	{
 		.name = "tps65023",
-		.driver_data = (kernel_ulong_t)&tps65023_drv_data
+		.driver_data = (kernel_ulong_t)&tps65023_drv_data,
 	}, {
 		.name = "tps65021",
-		.driver_data = (kernel_ulong_t)&tps65021_drv_data
+		.driver_data = (kernel_ulong_t)&tps65021_drv_data,
 	}, {
 		.name = "tps65020",
-		.driver_data = (kernel_ulong_t)&tps65020_drv_data
+		.driver_data = (kernel_ulong_t)&tps65020_drv_data,
 	},
 	{ },
 };
diff --git a/drivers/regulator/tps65132-regulator.c b/drivers/regulator/tps65132-regulator.c
index 9c2f0dd42613d..fecda3fd8ebc4 100644
--- a/drivers/regulator/tps65132-regulator.c
+++ b/drivers/regulator/tps65132-regulator.c
@@ -262,8 +262,8 @@ static int tps65132_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tps65132_id[] = {
-	{.name = "tps65132",},
-	{},
+	{ .name = "tps65132" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tps65132_id);
 
diff --git a/drivers/regulator/tps6594-regulator.c b/drivers/regulator/tps6594-regulator.c
index 645e83462c645..31a5218d55105 100644
--- a/drivers/regulator/tps6594-regulator.c
+++ b/drivers/regulator/tps6594-regulator.c
@@ -669,13 +669,20 @@ static int tps6594_regulator_probe(struct platform_device *pdev)
 	 * buck_configured to avoid creating bucks for every buck in multiphase
 	 */
 	for (multi = 0; multi < desc->num_multi_phase_regs; multi++) {
+		struct device_node *np_parent;
+
 		multi_regs = &desc->multi_phase_regs[multi];
 		np = of_find_node_by_name(tps->dev->of_node, multi_regs->supply_name);
-		npname = of_node_full_name(np);
-		np_pmic_parent = of_get_parent(of_get_parent(np));
-		if (of_node_cmp(of_node_full_name(np_pmic_parent), tps->dev->of_node->full_name))
+		if (!np)
 			continue;
-		if (strcmp(npname, multi_regs->supply_name) == 0) {
+
+		npname = of_node_full_name(np);
+		np_parent = of_get_parent(np);
+		np_pmic_parent = of_get_parent(np_parent);
+
+		if (np_pmic_parent &&
+		    !of_node_cmp(of_node_full_name(np_pmic_parent), tps->dev->of_node->full_name) &&
+		    strcmp(npname, multi_regs->supply_name) == 0) {
 			switch (multi) {
 			case MULTI_BUCK12:
 				buck_multi[0] = true;
@@ -706,6 +713,10 @@ static int tps6594_regulator_probe(struct platform_device *pdev)
 				break;
 			}
 		}
+
+		of_node_put(np_pmic_parent);
+		of_node_put(np_parent);
+		of_node_put(np);
 	}
 
 	reg_irq_nb = desc->num_irq_types * (desc->num_buck_regs + desc->num_ldo_regs);
diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig
index c521c744e7dbe..65befdbfa5f7d 100644
--- a/drivers/remoteproc/Kconfig
+++ b/drivers/remoteproc/Kconfig
@@ -210,6 +210,7 @@ config QCOM_Q6V5_MSS
 	select QCOM_Q6V5_COMMON
 	select QCOM_RPROC_COMMON
 	select QCOM_SCM
+	select QCOM_PAS
 	help
 	  Say y here to support the Qualcomm self-authenticating modem
 	  subsystem based on Hexagon V5. The TrustZone based system is
@@ -230,6 +231,7 @@ config QCOM_Q6V5_PAS
 	select QCOM_Q6V5_COMMON
 	select QCOM_RPROC_COMMON
 	select QCOM_SCM
+	select QCOM_PAS
 	help
 	  Say y here to support the TrustZone based Peripheral Image Loader for
 	  the Qualcomm remote processors. This is commonly used to control
@@ -282,7 +284,7 @@ config QCOM_WCNSS_PIL
 	select QCOM_MDT_LOADER
 	select QCOM_PIL_INFO
 	select QCOM_RPROC_COMMON
-	select QCOM_SCM
+	select QCOM_PAS
 	help
 	  Say y here to support the Peripheral Image Loader for loading WCNSS
 	  firmware and boot the core on e.g. MSM8974, MSM8916. The firmware is
diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c
index 008741af9f11c..fd60c67ba8a9f 100644
--- a/drivers/remoteproc/imx_dsp_rproc.c
+++ b/drivers/remoteproc/imx_dsp_rproc.c
@@ -644,32 +644,6 @@ static void imx_dsp_rproc_free_mbox(struct imx_dsp_rproc *priv)
 	mbox_free_channel(priv->rxdb_ch);
 }
 
-static int imx_dsp_rproc_mem_alloc(struct rproc *rproc,
-				   struct rproc_mem_entry *mem)
-{
-	struct device *dev = rproc->dev.parent;
-	void *va;
-
-	va = ioremap_wc(mem->dma, mem->len);
-	if (!va) {
-		dev_err(dev, "Unable to map memory region: %pa+%zx\n",
-			&mem->dma, mem->len);
-		return -ENOMEM;
-	}
-
-	mem->va = va;
-
-	return 0;
-}
-
-static int imx_dsp_rproc_mem_release(struct rproc *rproc,
-				     struct rproc_mem_entry *mem)
-{
-	iounmap(mem->va);
-
-	return 0;
-}
-
 /**
  * imx_dsp_rproc_add_carveout() - request mailbox channels
  * @priv: private data pointer
@@ -700,8 +674,10 @@ static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv)
 
 		/* Register memory region */
 		mem = rproc_mem_entry_init(dev, NULL, (dma_addr_t)att->sa,
-					   att->size, da, imx_dsp_rproc_mem_alloc,
-					   imx_dsp_rproc_mem_release, "dsp_mem");
+					   att->size, da,
+					   rproc_mem_entry_ioremap_wc,
+					   rproc_mem_entry_iounmap,
+					   "dsp_mem");
 
 		if (mem)
 			rproc_coredump_add_segment(rproc, da, att->size);
@@ -732,8 +708,8 @@ static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv)
 		/* Register memory region */
 		mem = rproc_mem_entry_init(dev, NULL, (dma_addr_t)res.start,
 					   resource_size(&res), da,
-					    imx_dsp_rproc_mem_alloc,
-					    imx_dsp_rproc_mem_release,
+					   rproc_mem_entry_ioremap_wc,
+					   rproc_mem_entry_iounmap,
 					   "%.*s", strchrnul(res.name, '@') - res.name, res.name);
 		if (!mem)
 			return -ENOMEM;
@@ -980,9 +956,8 @@ static int imx_dsp_rproc_elf_load_segments(struct rproc *rproc, const struct fir
 
 static int imx_dsp_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw)
 {
-	if (rproc_elf_load_rsc_table(rproc, fw))
-		dev_warn(&rproc->dev, "no resource table found for this firmware\n");
-
+	rproc_elf_load_rsc_table_optional(rproc, fw, dev_warn,
+					  "no resource table found for this firmware\n");
 	return 0;
 }
 
diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c
index 7662ebd9d2f49..745ce52cd8226 100644
--- a/drivers/remoteproc/imx_rproc.c
+++ b/drivers/remoteproc/imx_rproc.c
@@ -600,35 +600,6 @@ static void *imx_rproc_da_to_va(struct rproc *rproc, u64 da, size_t len, bool *i
 	return va;
 }
 
-static int imx_rproc_mem_alloc(struct rproc *rproc,
-			       struct rproc_mem_entry *mem)
-{
-	struct device *dev = rproc->dev.parent;
-	void *va;
-
-	dev_dbg(dev, "map memory: %p+%zx\n", &mem->dma, mem->len);
-	va = ioremap_wc(mem->dma, mem->len);
-	if (IS_ERR_OR_NULL(va)) {
-		dev_err(dev, "Unable to map memory region: %p+%zx\n",
-			&mem->dma, mem->len);
-		return -ENOMEM;
-	}
-
-	/* Update memory entry va */
-	mem->va = va;
-
-	return 0;
-}
-
-static int imx_rproc_mem_release(struct rproc *rproc,
-				 struct rproc_mem_entry *mem)
-{
-	dev_dbg(rproc->dev.parent, "unmap memory: %pa\n", &mem->dma);
-	iounmap(mem->va);
-
-	return 0;
-}
-
 static int imx_rproc_sm_lmm_prepare(struct rproc *rproc)
 {
 	struct imx_rproc *priv = rproc->priv;
@@ -692,7 +663,8 @@ static int imx_rproc_prepare(struct rproc *rproc)
 		/* Register memory region */
 		mem = rproc_mem_entry_init(priv->dev, NULL, (dma_addr_t)res.start,
 					   resource_size(&res), da,
-					   imx_rproc_mem_alloc, imx_rproc_mem_release,
+					   rproc_mem_entry_ioremap_wc,
+					   rproc_mem_entry_iounmap,
 					   "%.*s", strchrnul(res.name, '@') - res.name,
 					   res.name);
 		if (!mem)
@@ -710,12 +682,8 @@ static int imx_rproc_prepare(struct rproc *rproc)
 
 static int imx_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw)
 {
-	int ret;
-
-	ret = rproc_elf_load_rsc_table(rproc, fw);
-	if (ret)
-		dev_info(&rproc->dev, "No resource table in elf\n");
-
+	rproc_elf_load_rsc_table_optional(rproc, fw, dev_info,
+					  "No resource table in elf\n");
 	return 0;
 }
 
diff --git a/drivers/remoteproc/keystone_remoteproc.c b/drivers/remoteproc/keystone_remoteproc.c
index e7fde55097866..83763d640c4e4 100644
--- a/drivers/remoteproc/keystone_remoteproc.c
+++ b/drivers/remoteproc/keystone_remoteproc.c
@@ -317,7 +317,7 @@ static int keystone_rproc_of_get_dev_syscon(struct platform_device *pdev,
 	struct device_node *np = pdev->dev.of_node;
 	struct device *dev = &pdev->dev;
 
-	if (!of_property_read_bool(np, "ti,syscon-dev")) {
+	if (!of_property_present(np, "ti,syscon-dev")) {
 		dev_err(dev, "ti,syscon-dev property is absent\n");
 		return -EINVAL;
 	}
diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c
index 85a74c9ec5216..436656bdfa8be 100644
--- a/drivers/remoteproc/mtk_scp.c
+++ b/drivers/remoteproc/mtk_scp.c
@@ -36,6 +36,7 @@ struct mtk_scp *scp_get(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct device_node *scp_node;
 	struct platform_device *scp_pdev;
+	struct mtk_scp *scp;
 
 	scp_node = of_parse_phandle(dev->of_node, "mediatek,scp", 0);
 	if (!scp_node) {
@@ -51,7 +52,13 @@ struct mtk_scp *scp_get(struct platform_device *pdev)
 		return NULL;
 	}
 
-	return platform_get_drvdata(scp_pdev);
+	scp = platform_get_drvdata(scp_pdev);
+	if (!scp) {
+		put_device(&scp_pdev->dev);
+		return NULL;
+	}
+
+	return scp;
 }
 EXPORT_SYMBOL_GPL(scp_get);
 
diff --git a/drivers/remoteproc/omap_remoteproc.c b/drivers/remoteproc/omap_remoteproc.c
index cb01354248afc..6ed0f28edac9c 100644
--- a/drivers/remoteproc/omap_remoteproc.c
+++ b/drivers/remoteproc/omap_remoteproc.c
@@ -1140,7 +1140,7 @@ static int omap_rproc_get_boot_data(struct platform_device *pdev,
 	if (!data)
 		return -ENODEV;
 
-	if (!of_property_read_bool(np, "ti,bootreg"))
+	if (!of_property_present(np, "ti,bootreg"))
 		return 0;
 
 	oproc->boot_data = devm_kzalloc(&pdev->dev, sizeof(*oproc->boot_data),
diff --git a/drivers/remoteproc/qcom_common.c b/drivers/remoteproc/qcom_common.c
index e1a955476c9b3..5294e327f1587 100644
--- a/drivers/remoteproc/qcom_common.c
+++ b/drivers/remoteproc/qcom_common.c
@@ -253,8 +253,10 @@ void qcom_add_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink,
 		return;
 
 	glink->ssr_name = kstrdup_const(ssr_name, GFP_KERNEL);
-	if (!glink->ssr_name)
+	if (!glink->ssr_name) {
+		of_node_put(glink->node);
 		return;
+	}
 
 	glink->dev = dev;
 	glink->subdev.start = glink_subdev_start;
diff --git a/drivers/remoteproc/qcom_common.h b/drivers/remoteproc/qcom_common.h
index b07fbaa091a06..b0e7e336d363e 100644
--- a/drivers/remoteproc/qcom_common.h
+++ b/drivers/remoteproc/qcom_common.h
@@ -68,6 +68,7 @@ struct qcom_sysmon *qcom_add_sysmon_subdev(struct rproc *rproc,
 					   int ssctl_instance);
 void qcom_remove_sysmon_subdev(struct qcom_sysmon *sysmon);
 bool qcom_sysmon_shutdown_acked(struct qcom_sysmon *sysmon);
+bool qcom_sysmon_shutdown_irq_state(struct qcom_sysmon *sysmon);
 #else
 static inline struct qcom_sysmon *qcom_add_sysmon_subdev(struct rproc *rproc,
 							 const char *name,
@@ -84,6 +85,11 @@ static inline bool qcom_sysmon_shutdown_acked(struct qcom_sysmon *sysmon)
 {
 	return false;
 }
+
+static inline bool qcom_sysmon_shutdown_irq_state(struct qcom_sysmon *sysmon)
+{
+	return false;
+}
 #endif
 
 #endif
diff --git a/drivers/remoteproc/qcom_q6v5.c b/drivers/remoteproc/qcom_q6v5.c
index 58d5b85e58cda..0206a4a19254d 100644
--- a/drivers/remoteproc/qcom_q6v5.c
+++ b/drivers/remoteproc/qcom_q6v5.c
@@ -36,6 +36,40 @@ static int q6v5_load_state_toggle(struct qcom_q6v5 *q6v5, bool enable)
 	return ret;
 }
 
+static void q6v5_handover_irq_enable(struct qcom_q6v5 *q6v5)
+{
+	unsigned long flags;
+	bool enable = false;
+
+	spin_lock_irqsave(&q6v5->handover_lock, flags);
+	if (!q6v5->handover_irq_enabled) {
+		q6v5->handover_irq_enabled = true;
+		enable = true;
+	}
+	spin_unlock_irqrestore(&q6v5->handover_lock, flags);
+
+	if (enable)
+		enable_irq(q6v5->handover_irq);
+}
+
+static void q6v5_handover_irq_disable(struct qcom_q6v5 *q6v5, bool sync)
+{
+	unsigned long flags;
+	bool disable = false;
+
+	spin_lock_irqsave(&q6v5->handover_lock, flags);
+	if (q6v5->handover_irq_enabled) {
+		q6v5->handover_irq_enabled = false;
+		disable = true;
+	}
+	spin_unlock_irqrestore(&q6v5->handover_lock, flags);
+
+	if (disable)
+		disable_irq_nosync(q6v5->handover_irq);
+	if (sync)
+		synchronize_irq(q6v5->handover_irq);
+}
+
 /**
  * qcom_q6v5_prepare() - reinitialize the qcom_q6v5 context before start
  * @q6v5:	reference to qcom_q6v5 context to be reinitialized
@@ -64,7 +98,7 @@ int qcom_q6v5_prepare(struct qcom_q6v5 *q6v5)
 	q6v5->running = true;
 	q6v5->handover_issued = false;
 
-	enable_irq(q6v5->handover_irq);
+	q6v5_handover_irq_enable(q6v5);
 
 	return 0;
 }
@@ -78,7 +112,8 @@ EXPORT_SYMBOL_GPL(qcom_q6v5_prepare);
  */
 int qcom_q6v5_unprepare(struct qcom_q6v5 *q6v5)
 {
-	disable_irq(q6v5->handover_irq);
+	q6v5_handover_irq_disable(q6v5, true);
+
 	q6v5_load_state_toggle(q6v5, false);
 
 	/* Disable interconnect vote, in case handover never happened */
@@ -164,18 +199,15 @@ static irqreturn_t q6v5_handover_interrupt(int irq, void *data)
 {
 	struct qcom_q6v5 *q6v5 = data;
 
-	if (q6v5->handover_issued) {
-		dev_err(q6v5->dev, "Handover signaled, but it already happened\n");
-		return IRQ_HANDLED;
-	}
+	q6v5->handover_issued = true;
+
+	q6v5_handover_irq_disable(q6v5, false);
 
 	if (q6v5->handover)
 		q6v5->handover(q6v5);
 
 	icc_set_bw(q6v5->path, 0, 0);
 
-	q6v5->handover_issued = true;
-
 	return IRQ_HANDLED;
 }
 
@@ -202,7 +234,8 @@ int qcom_q6v5_request_stop(struct qcom_q6v5 *q6v5, struct qcom_sysmon *sysmon)
 	q6v5->running = false;
 
 	/* Don't perform SMP2P dance if remote isn't running */
-	if (q6v5->rproc->state != RPROC_RUNNING || qcom_sysmon_shutdown_acked(sysmon))
+	if ((q6v5->rproc->state != RPROC_RUNNING && q6v5->rproc->state != RPROC_ATTACHED) ||
+	    qcom_sysmon_shutdown_acked(sysmon))
 		return 0;
 
 	qcom_smem_state_update_bits(q6v5->state,
@@ -256,6 +289,8 @@ int qcom_q6v5_init(struct qcom_q6v5 *q6v5, struct platform_device *pdev,
 	q6v5->crash_reason = crash_reason;
 	q6v5->handover = handover;
 
+	spin_lock_init(&q6v5->handover_lock);
+
 	init_completion(&q6v5->start_done);
 	init_completion(&q6v5->stop_done);
 
@@ -304,13 +339,13 @@ int qcom_q6v5_init(struct qcom_q6v5 *q6v5, struct platform_device *pdev,
 
 	ret = devm_request_threaded_irq(&pdev->dev, q6v5->handover_irq,
 					NULL, q6v5_handover_interrupt,
-					IRQF_TRIGGER_RISING | IRQF_ONESHOT,
+					IRQF_TRIGGER_RISING | IRQF_ONESHOT |
+					IRQF_NO_AUTOEN,
 					"q6v5 handover", q6v5);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to acquire handover IRQ\n");
 		return ret;
 	}
-	disable_irq(q6v5->handover_irq);
 
 	q6v5->stop_irq = platform_get_irq_byname(pdev, "stop-ack");
 	if (q6v5->stop_irq < 0)
diff --git a/drivers/remoteproc/qcom_q6v5.h b/drivers/remoteproc/qcom_q6v5.h
index 5a859c41896e9..8991ff0905792 100644
--- a/drivers/remoteproc/qcom_q6v5.h
+++ b/drivers/remoteproc/qcom_q6v5.h
@@ -5,6 +5,7 @@
 
 #include <linux/kernel.h>
 #include <linux/completion.h>
+#include <linux/spinlock.h>
 #include <linux/soc/qcom/qcom_aoss.h>
 
 struct icc_path;
@@ -29,6 +30,9 @@ struct qcom_q6v5 {
 	int handover_irq;
 	int stop_irq;
 
+	/* Protects handover_irq_enabled against stop/handover races. */
+	spinlock_t handover_lock;
+	bool handover_irq_enabled;
 	bool handover_issued;
 
 	struct completion start_done;
diff --git a/drivers/remoteproc/qcom_q6v5_adsp.c b/drivers/remoteproc/qcom_q6v5_adsp.c
index b5c8d6d38c9cb..c81e6c33c7479 100644
--- a/drivers/remoteproc/qcom_q6v5_adsp.c
+++ b/drivers/remoteproc/qcom_q6v5_adsp.c
@@ -355,6 +355,7 @@ static int adsp_map_carveout(struct rproc *rproc)
 		return ret;
 
 	sid = args.args[0] & SID_MASK_DEFAULT;
+	of_node_put(args.np);
 
 	/* Add SID configuration for ADSP Firmware to SMMU */
 	iova =  adsp->mem_phys | (sid << 32);
diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c
index ae78f5c7c1b69..bef198b9ee636 100644
--- a/drivers/remoteproc/qcom_q6v5_mss.c
+++ b/drivers/remoteproc/qcom_q6v5_mss.c
@@ -34,6 +34,7 @@
 #include "qcom_pil_info.h"
 #include "qcom_q6v5.h"
 
+#include <linux/firmware/qcom/qcom_pas.h>
 #include <linux/firmware/qcom/qcom_scm.h>
 
 #define MPSS_CRASH_REASON_SMEM		421
@@ -162,6 +163,7 @@ struct rproc_hexagon_res {
 	char **active_clk_names;
 	char **proxy_pd_names;
 	int version;
+	int ssctl_id;
 	bool need_mem_protection;
 	bool need_pas_mem_setup;
 	bool has_alt_reset;
@@ -1480,7 +1482,7 @@ static int q6v5_mpss_load(struct q6v5 *qproc)
 	}
 
 	if (qproc->need_pas_mem_setup) {
-		ret = qcom_scm_pas_mem_setup(MPSS_PAS_ID, qproc->mpss_phys, qproc->mpss_size);
+		ret = qcom_pas_mem_setup(MPSS_PAS_ID, qproc->mpss_phys, qproc->mpss_size);
 		if (ret) {
 			dev_err(qproc->dev,
 				"setting up mpss memory failed: %d\n", ret);
@@ -2077,7 +2079,7 @@ static int q6v5_probe(struct platform_device *pdev)
 	if (!desc)
 		return -EINVAL;
 
-	if (desc->need_mem_protection && !qcom_scm_is_available())
+	if (desc->need_mem_protection && !qcom_pas_is_available())
 		return -EPROBE_DEFER;
 
 	mba_image = desc->hexagon_mba_image;
@@ -2191,7 +2193,7 @@ static int q6v5_probe(struct platform_device *pdev)
 	qcom_add_smd_subdev(rproc, &qproc->smd_subdev);
 	qcom_add_pdm_subdev(rproc, &qproc->pdm_subdev);
 	qcom_add_ssr_subdev(rproc, &qproc->ssr_subdev, "mpss");
-	qproc->sysmon = qcom_add_sysmon_subdev(rproc, "modem", 0x12);
+	qproc->sysmon = qcom_add_sysmon_subdev(rproc, "modem", desc->ssctl_id);
 	if (IS_ERR(qproc->sysmon)) {
 		ret = PTR_ERR(qproc->sysmon);
 		goto remove_subdevs;
@@ -2271,6 +2273,7 @@ static const struct rproc_hexagon_res sc7180_mss = {
 	.has_ext_cntl_regs = false,
 	.has_vq6 = false,
 	.version = MSS_SC7180,
+	.ssctl_id = 0x12,
 };
 
 static const struct rproc_hexagon_res sc7280_mss = {
@@ -2301,6 +2304,7 @@ static const struct rproc_hexagon_res sc7280_mss = {
 	.has_ext_cntl_regs = true,
 	.has_vq6 = true,
 	.version = MSS_SC7280,
+	.ssctl_id = 0x12,
 };
 
 static const struct rproc_hexagon_res sdm660_mss = {
@@ -2334,6 +2338,7 @@ static const struct rproc_hexagon_res sdm660_mss = {
 	.has_ext_cntl_regs = false,
 	.has_vq6 = false,
 	.version = MSS_SDM660,
+	.ssctl_id = 0x12,
 };
 
 static const struct rproc_hexagon_res sdm845_mss = {
@@ -2371,6 +2376,7 @@ static const struct rproc_hexagon_res sdm845_mss = {
 	.has_ext_cntl_regs = false,
 	.has_vq6 = false,
 	.version = MSS_SDM845,
+	.ssctl_id = 0x12,
 };
 
 static const struct rproc_hexagon_res msm8998_mss = {
@@ -2404,6 +2410,7 @@ static const struct rproc_hexagon_res msm8998_mss = {
 	.has_ext_cntl_regs = false,
 	.has_vq6 = false,
 	.version = MSS_MSM8998,
+	.ssctl_id = 0x12,
 };
 
 static const struct rproc_hexagon_res msm8996_mss = {
@@ -2444,6 +2451,7 @@ static const struct rproc_hexagon_res msm8996_mss = {
 	.has_ext_cntl_regs = false,
 	.has_vq6 = false,
 	.version = MSS_MSM8996,
+	.ssctl_id = 0x12,
 };
 
 static const struct rproc_hexagon_res mdm9607_mss = {
@@ -2479,6 +2487,7 @@ static const struct rproc_hexagon_res mdm9607_mss = {
 	.has_ext_cntl_regs = false,
 	.has_vq6 = false,
 	.version = MSS_MDM9607,
+	.ssctl_id = 0x22,
 };
 
 static const struct rproc_hexagon_res msm8909_mss = {
@@ -2515,6 +2524,7 @@ static const struct rproc_hexagon_res msm8909_mss = {
 	.has_ext_cntl_regs = false,
 	.has_vq6 = false,
 	.version = MSS_MSM8909,
+	.ssctl_id = 0x12,
 };
 
 static const struct rproc_hexagon_res msm8916_mss = {
@@ -2562,6 +2572,7 @@ static const struct rproc_hexagon_res msm8916_mss = {
 	.has_ext_cntl_regs = false,
 	.has_vq6 = false,
 	.version = MSS_MSM8916,
+	.ssctl_id = 0x12,
 };
 
 static const struct rproc_hexagon_res msm8917_mss = {
@@ -2606,6 +2617,7 @@ static const struct rproc_hexagon_res msm8917_mss = {
 	.has_ext_cntl_regs = false,
 	.has_vq6 = false,
 	.version = MSS_MSM8917,
+	.ssctl_id = 0x12,
 };
 
 static const struct rproc_hexagon_res msm8937_mss = {
@@ -2650,6 +2662,7 @@ static const struct rproc_hexagon_res msm8937_mss = {
 	.has_ext_cntl_regs = false,
 	.has_vq6 = false,
 	.version = MSS_MSM8937,
+	.ssctl_id = 0x12,
 };
 
 static const struct rproc_hexagon_res msm8940_mss = {
@@ -2694,6 +2707,7 @@ static const struct rproc_hexagon_res msm8940_mss = {
 	.has_ext_cntl_regs = false,
 	.has_vq6 = false,
 	.version = MSS_MSM8940,
+	.ssctl_id = 0x12,
 };
 
 static const struct rproc_hexagon_res msm8953_mss = {
@@ -2731,6 +2745,7 @@ static const struct rproc_hexagon_res msm8953_mss = {
 	.has_ext_cntl_regs = false,
 	.has_vq6 = false,
 	.version = MSS_MSM8953,
+	.ssctl_id = 0x12,
 };
 
 static const struct rproc_hexagon_res msm8974_mss = {
@@ -2785,6 +2800,7 @@ static const struct rproc_hexagon_res msm8974_mss = {
 	.has_ext_cntl_regs = false,
 	.has_vq6 = false,
 	.version = MSS_MSM8974,
+	.ssctl_id = 0x12,
 };
 
 static const struct rproc_hexagon_res msm8226_mss = {
@@ -2824,6 +2840,7 @@ static const struct rproc_hexagon_res msm8226_mss = {
 	.has_ext_cntl_regs = false,
 	.has_vq6 = false,
 	.version = MSS_MSM8226,
+	.ssctl_id = 0x12,
 };
 
 static const struct rproc_hexagon_res msm8926_mss = {
@@ -2871,6 +2888,7 @@ static const struct rproc_hexagon_res msm8926_mss = {
 	.has_ext_cntl_regs = false,
 	.has_vq6 = false,
 	.version = MSS_MSM8926,
+	.ssctl_id = 0x12,
 };
 
 static const struct of_device_id q6v5_of_match[] = {
diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c
index 808e9609988d3..25599d7282083 100644
--- a/drivers/remoteproc/qcom_q6v5_pas.c
+++ b/drivers/remoteproc/qcom_q6v5_pas.c
@@ -16,10 +16,12 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
+#include <linux/of_platform.h>
 #include <linux/of_reserved_mem.h>
 #include <linux/platform_device.h>
 #include <linux/pm_domain.h>
 #include <linux/pm_runtime.h>
+#include <linux/firmware/qcom/qcom_pas.h>
 #include <linux/firmware/qcom/qcom_scm.h>
 #include <linux/regulator/consumer.h>
 #include <linux/remoteproc.h>
@@ -60,6 +62,8 @@ struct qcom_pas_data {
 	int region_assign_count;
 	bool region_assign_shared;
 	int region_assign_vmid;
+	bool early_boot;
+	bool needs_tzmem;
 };
 
 struct qcom_pas {
@@ -114,9 +118,10 @@ struct qcom_pas {
 	struct qcom_rproc_pdm pdm_subdev;
 	struct qcom_rproc_ssr ssr_subdev;
 	struct qcom_sysmon *sysmon;
+	struct platform_device *bam_dmux;
 
-	struct qcom_scm_pas_context *pas_ctx;
-	struct qcom_scm_pas_context *dtb_pas_ctx;
+	struct qcom_pas_context *pas_ctx;
+	struct qcom_pas_context *dtb_pas_ctx;
 };
 
 static void qcom_pas_segment_dump(struct rproc *rproc,
@@ -193,7 +198,7 @@ static int qcom_pas_shutdown_poll_decrypt(struct qcom_pas *pas)
 
 	do {
 		msleep(QCOM_PAS_DECRYPT_SHUTDOWN_DELAY_MS);
-		ret = qcom_scm_pas_shutdown(pas->pas_id);
+		ret = qcom_pas_shutdown(pas->pas_id);
 	} while (ret == -EINVAL && --retry_num);
 
 	return ret;
@@ -209,9 +214,9 @@ static int qcom_pas_unprepare(struct rproc *rproc)
 	 * auth_and_reset() was successful, but in other cases clean it up
 	 * here.
 	 */
-	qcom_scm_pas_metadata_release(pas->pas_ctx);
+	qcom_pas_metadata_release(pas->pas_ctx);
 	if (pas->dtb_pas_id)
-		qcom_scm_pas_metadata_release(pas->dtb_pas_ctx);
+		qcom_pas_metadata_release(pas->dtb_pas_ctx);
 
 	return 0;
 }
@@ -225,9 +230,9 @@ static int qcom_pas_load(struct rproc *rproc, const struct firmware *fw)
 	pas->firmware = fw;
 
 	if (pas->lite_pas_id)
-		qcom_scm_pas_shutdown(pas->lite_pas_id);
+		qcom_pas_shutdown(pas->lite_pas_id);
 	if (pas->lite_dtb_pas_id)
-		qcom_scm_pas_shutdown(pas->lite_dtb_pas_id);
+		qcom_pas_shutdown(pas->lite_dtb_pas_id);
 
 	if (pas->dtb_pas_id) {
 		ret = request_firmware(&pas->dtb_firmware, pas->dtb_firmware_name, pas->dev);
@@ -247,7 +252,7 @@ static int qcom_pas_load(struct rproc *rproc, const struct firmware *fw)
 	return 0;
 
 release_dtb_metadata:
-	qcom_scm_pas_metadata_release(pas->dtb_pas_ctx);
+	qcom_pas_metadata_release(pas->dtb_pas_ctx);
 	release_firmware(pas->dtb_firmware);
 
 	return ret;
@@ -307,7 +312,7 @@ static int qcom_pas_start(struct rproc *rproc)
 		if (ret)
 			goto disable_px_supply;
 
-		ret = qcom_scm_pas_prepare_and_auth_reset(pas->dtb_pas_ctx);
+		ret = qcom_pas_prepare_and_auth_reset(pas->dtb_pas_ctx);
 		if (ret) {
 			dev_err(pas->dev,
 				"failed to authenticate dtb image and release reset\n");
@@ -326,7 +331,7 @@ static int qcom_pas_start(struct rproc *rproc)
 	if (ret)
 		goto release_pas_metadata;
 
-	ret = qcom_scm_pas_prepare_and_auth_reset(pas->pas_ctx);
+	ret = qcom_pas_prepare_and_auth_reset(pas->pas_ctx);
 	if (ret) {
 		dev_err(pas->dev,
 			"failed to authenticate image and release reset\n");
@@ -336,13 +341,13 @@ static int qcom_pas_start(struct rproc *rproc)
 	ret = qcom_q6v5_wait_for_start(&pas->q6v5, msecs_to_jiffies(5000));
 	if (ret == -ETIMEDOUT) {
 		dev_err(pas->dev, "start timed out\n");
-		qcom_scm_pas_shutdown(pas->pas_id);
+		qcom_pas_shutdown(pas->pas_id);
 		goto unmap_carveout;
 	}
 
-	qcom_scm_pas_metadata_release(pas->pas_ctx);
+	qcom_pas_metadata_release(pas->pas_ctx);
 	if (pas->dtb_pas_id)
-		qcom_scm_pas_metadata_release(pas->dtb_pas_ctx);
+		qcom_pas_metadata_release(pas->dtb_pas_ctx);
 
 	/* firmware is used to pass reference from qcom_pas_start(), drop it now */
 	pas->firmware = NULL;
@@ -352,9 +357,9 @@ static int qcom_pas_start(struct rproc *rproc)
 unmap_carveout:
 	qcom_pas_unmap_carveout(rproc, pas->mem_phys, pas->mem_size);
 release_pas_metadata:
-	qcom_scm_pas_metadata_release(pas->pas_ctx);
+	qcom_pas_metadata_release(pas->pas_ctx);
 	if (pas->dtb_pas_id)
-		qcom_scm_pas_metadata_release(pas->dtb_pas_ctx);
+		qcom_pas_metadata_release(pas->dtb_pas_ctx);
 
 unmap_dtb_carveout:
 	if (pas->dtb_pas_id)
@@ -403,7 +408,7 @@ static int qcom_pas_stop(struct rproc *rproc)
 	if (ret == -ETIMEDOUT)
 		dev_err(pas->dev, "timed out on wait\n");
 
-	ret = qcom_scm_pas_shutdown(pas->pas_id);
+	ret = qcom_pas_shutdown(pas->pas_id);
 	if (ret && pas->decrypt_shutdown)
 		ret = qcom_pas_shutdown_poll_decrypt(pas);
 
@@ -411,7 +416,7 @@ static int qcom_pas_stop(struct rproc *rproc)
 		dev_err(pas->dev, "failed to shutdown: %d\n", ret);
 
 	if (pas->dtb_pas_id) {
-		ret = qcom_scm_pas_shutdown(pas->dtb_pas_id);
+		ret = qcom_pas_shutdown(pas->dtb_pas_id);
 		if (ret)
 			dev_err(pas->dev, "failed to shutdown dtb: %d\n", ret);
 
@@ -481,11 +486,11 @@ static int qcom_pas_parse_firmware(struct rproc *rproc, const struct firmware *f
 	 *
 	 * Here, we call rproc_elf_load_rsc_table() to check firmware binary has resources
 	 * or not and if it is not having then we pass NULL and zero as input resource
-	 * table pointer and size respectively to the argument of qcom_scm_pas_get_rsc_table()
+	 * table pointer and size respectively to the argument of qcom_pas_get_rsc_table()
 	 * and this is even true for Qualcomm remote processor who does follow remoteproc
 	 * framework.
 	 */
-	output_rt = qcom_scm_pas_get_rsc_table(pas->pas_ctx, table, table_sz, &output_rt_size);
+	output_rt = qcom_pas_get_rsc_table(pas->pas_ctx, table, table_sz, &output_rt_size);
 	ret = IS_ERR(output_rt) ? PTR_ERR(output_rt) : 0;
 	if (ret) {
 		dev_err(pas->dev, "Error in getting resource table: %d\n", ret);
@@ -507,6 +512,67 @@ static unsigned long qcom_pas_panic(struct rproc *rproc)
 	return qcom_q6v5_panic(&pas->q6v5);
 }
 
+static int qcom_pas_attach(struct rproc *rproc)
+{
+	struct qcom_pas *pas = rproc->priv;
+	bool ready_state;
+	bool crash_state;
+	bool stop_state;
+	int ret;
+
+	pas->q6v5.handover_issued = true;
+	enable_irq(pas->q6v5.handover_irq);
+
+	pas->q6v5.running = true;
+	ret = irq_get_irqchip_state(pas->q6v5.fatal_irq,
+				    IRQCHIP_STATE_LINE_LEVEL, &crash_state);
+	if (ret)
+		goto disable_running;
+
+	if (crash_state) {
+		dev_err(pas->dev, "Subsystem has crashed before driver probe\n");
+		rproc_report_crash(rproc, RPROC_FATAL_ERROR);
+		ret = -EINVAL;
+		goto disable_running;
+	}
+
+	ret = irq_get_irqchip_state(pas->q6v5.stop_irq,
+				    IRQCHIP_STATE_LINE_LEVEL, &stop_state);
+	if (ret)
+		goto disable_running;
+
+	if (stop_state || qcom_sysmon_shutdown_irq_state(pas->sysmon)) {
+		dev_info(pas->dev, "Subsystem found stop state set. Falling back to start.\n");
+		goto unroll_attach;
+	}
+
+	ret = irq_get_irqchip_state(pas->q6v5.ready_irq,
+				    IRQCHIP_STATE_LINE_LEVEL, &ready_state);
+	if (ret)
+		goto disable_running;
+
+	if (unlikely(!ready_state)) {
+		/*
+		 * The bootloader may not support early boot, mark the state as
+		 * RPROC_OFFLINE so that the PAS driver can load the firmware and
+		 * start the remoteproc.
+		 */
+		dev_err(pas->dev, "Failed to get subsystem ready interrupt\n");
+		goto unroll_attach;
+	}
+
+	return 0;
+
+unroll_attach:
+	pas->rproc->state = RPROC_OFFLINE;
+	ret = -EINVAL;
+disable_running:
+	disable_irq(pas->q6v5.handover_irq);
+	pas->q6v5.running = false;
+
+	return ret;
+}
+
 static const struct rproc_ops qcom_pas_ops = {
 	.unprepare = qcom_pas_unprepare,
 	.start = qcom_pas_start,
@@ -515,6 +581,7 @@ static const struct rproc_ops qcom_pas_ops = {
 	.parse_fw = qcom_pas_parse_firmware,
 	.load = qcom_pas_load,
 	.panic = qcom_pas_panic,
+	.attach = qcom_pas_attach,
 };
 
 static const struct rproc_ops qcom_pas_minidump_ops = {
@@ -526,6 +593,7 @@ static const struct rproc_ops qcom_pas_minidump_ops = {
 	.load = qcom_pas_load,
 	.panic = qcom_pas_panic,
 	.coredump = qcom_pas_minidump,
+	.attach = qcom_pas_attach,
 };
 
 static int qcom_pas_init_clock(struct qcom_pas *pas)
@@ -735,6 +803,7 @@ static int qcom_pas_probe(struct platform_device *pdev)
 	const struct qcom_pas_data *desc;
 	struct qcom_pas *pas;
 	struct rproc *rproc;
+	struct device_node *node;
 	const char *fw_name, *dtb_fw_name = NULL;
 	const struct rproc_ops *ops = &qcom_pas_ops;
 	int ret;
@@ -743,7 +812,7 @@ static int qcom_pas_probe(struct platform_device *pdev)
 	if (!desc)
 		return -EINVAL;
 
-	if (!qcom_scm_is_available())
+	if (!qcom_pas_is_available())
 		return -EPROBE_DEFER;
 
 	fw_name = desc->firmware_name;
@@ -835,27 +904,35 @@ static int qcom_pas_probe(struct platform_device *pdev)
 
 	qcom_add_ssr_subdev(rproc, &pas->ssr_subdev, desc->ssr_name);
 
-	pas->pas_ctx = devm_qcom_scm_pas_context_alloc(pas->dev, pas->pas_id,
-						       pas->mem_phys, pas->mem_size);
+	pas->pas_ctx = devm_qcom_pas_context_alloc(pas->dev, pas->pas_id,
+						   pas->mem_phys, pas->mem_size);
 	if (IS_ERR(pas->pas_ctx)) {
 		ret = PTR_ERR(pas->pas_ctx);
 		goto remove_ssr_sysmon;
 	}
 
-	pas->dtb_pas_ctx = devm_qcom_scm_pas_context_alloc(pas->dev, pas->dtb_pas_id,
-							   pas->dtb_mem_phys,
-							   pas->dtb_mem_size);
+	pas->dtb_pas_ctx = devm_qcom_pas_context_alloc(pas->dev, pas->dtb_pas_id,
+						       pas->dtb_mem_phys,
+						       pas->dtb_mem_size);
 	if (IS_ERR(pas->dtb_pas_ctx)) {
 		ret = PTR_ERR(pas->dtb_pas_ctx);
 		goto remove_ssr_sysmon;
 	}
 
-	pas->pas_ctx->use_tzmem = rproc->has_iommu;
-	pas->dtb_pas_ctx->use_tzmem = rproc->has_iommu;
+	pas->pas_ctx->use_tzmem = desc->needs_tzmem || rproc->has_iommu;
+	pas->dtb_pas_ctx->use_tzmem = desc->needs_tzmem || rproc->has_iommu;
+
+	if (desc->early_boot)
+		pas->rproc->state = RPROC_DETACHED;
+
 	ret = rproc_add(rproc);
 	if (ret)
 		goto remove_ssr_sysmon;
 
+	node = of_get_compatible_child(pdev->dev.of_node, "qcom,bam-dmux");
+	pas->bam_dmux = of_platform_device_create(node, NULL, &pdev->dev);
+	of_node_put(node);
+
 	return 0;
 
 remove_ssr_sysmon:
@@ -880,6 +957,9 @@ static void qcom_pas_remove(struct platform_device *pdev)
 {
 	struct qcom_pas *pas = platform_get_drvdata(pdev);
 
+	if (pas->bam_dmux)
+		of_platform_device_destroy(&pas->bam_dmux->dev, NULL);
+
 	rproc_del(pas->rproc);
 
 	qcom_q6v5_deinit(&pas->q6v5);
@@ -1572,8 +1652,45 @@ static const struct qcom_pas_data sm8750_mpss_resource = {
 	.region_assign_vmid = QCOM_SCM_VMID_MSS_MSA,
 };
 
+static const struct qcom_pas_data kaanapali_soccp_resource = {
+	.crash_reason_smem = 656,
+	.firmware_name = "soccp.mbn",
+	.dtb_firmware_name = "soccp_dtb.mbn",
+	.pas_id = 51,
+	.dtb_pas_id = 0x41,
+	.proxy_pd_names = (char*[]){
+		"cx",
+		"mx",
+		NULL
+	},
+	.ssr_name = "soccp",
+	.sysmon_name = "soccp",
+	.auto_boot = true,
+	.early_boot = true,
+};
+
+static const struct qcom_pas_data glymur_soccp_resource = {
+	.crash_reason_smem = 656,
+	.firmware_name = "soccp.mbn",
+	.dtb_firmware_name = "soccp_dtb.mbn",
+	.pas_id = 51,
+	.dtb_pas_id = 0x41,
+	.proxy_pd_names = (char*[]){
+		"cx",
+		"mx",
+		NULL
+	},
+	.ssr_name = "soccp",
+	.sysmon_name = "soccp",
+	.auto_boot = true,
+	.early_boot = true,
+	.needs_tzmem = true,
+};
+
 static const struct of_device_id qcom_pas_of_match[] = {
 	{ .compatible = "qcom,eliza-adsp-pas", .data = &sm8550_adsp_resource },
+	{ .compatible = "qcom,glymur-soccp-pas", .data = &glymur_soccp_resource },
+	{ .compatible = "qcom,kaanapali-soccp-pas", .data = &kaanapali_soccp_resource },
 	{ .compatible = "qcom,milos-adsp-pas", .data = &sm8550_adsp_resource },
 	{ .compatible = "qcom,milos-cdsp-pas", .data = &milos_cdsp_resource },
 	{ .compatible = "qcom,milos-mpss-pas", .data = &sm8450_mpss_resource },
diff --git a/drivers/remoteproc/qcom_sysmon.c b/drivers/remoteproc/qcom_sysmon.c
index 913e3b750a869..a0830a48b1f40 100644
--- a/drivers/remoteproc/qcom_sysmon.c
+++ b/drivers/remoteproc/qcom_sysmon.c
@@ -736,6 +736,25 @@ bool qcom_sysmon_shutdown_acked(struct qcom_sysmon *sysmon)
 }
 EXPORT_SYMBOL_GPL(qcom_sysmon_shutdown_acked);
 
+bool qcom_sysmon_shutdown_irq_state(struct qcom_sysmon *sysmon)
+{
+	bool shutdown_state;
+	int ret;
+
+	if (!sysmon)
+		return false;
+
+	ret = irq_get_irqchip_state(sysmon->shutdown_irq,
+				    IRQCHIP_STATE_LINE_LEVEL, &shutdown_state);
+	if (ret) {
+		dev_warn(sysmon->dev, "failed to get shutdown_state: %d\n", ret);
+		return false;
+	}
+
+	return shutdown_state;
+}
+EXPORT_SYMBOL_GPL(qcom_sysmon_shutdown_irq_state);
+
 /**
  * sysmon_probe() - probe sys_mon channel
  * @rpdev:	rpmsg device handle
diff --git a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c
index 4add9037dbd5a..3392c9380202d 100644
--- a/drivers/remoteproc/qcom_wcnss.c
+++ b/drivers/remoteproc/qcom_wcnss.c
@@ -19,7 +19,7 @@
 #include <linux/platform_device.h>
 #include <linux/pm_domain.h>
 #include <linux/pm_runtime.h>
-#include <linux/firmware/qcom/qcom_scm.h>
+#include <linux/firmware/qcom/qcom_pas.h>
 #include <linux/regulator/consumer.h>
 #include <linux/remoteproc.h>
 #include <linux/soc/qcom/mdt_loader.h>
@@ -257,7 +257,7 @@ static int wcnss_start(struct rproc *rproc)
 	wcnss_indicate_nv_download(wcnss);
 	wcnss_configure_iris(wcnss);
 
-	ret = qcom_scm_pas_auth_and_reset(WCNSS_PAS_ID);
+	ret = qcom_pas_auth_and_reset(WCNSS_PAS_ID);
 	if (ret) {
 		dev_err(wcnss->dev,
 			"failed to authenticate image and release reset\n");
@@ -269,7 +269,7 @@ static int wcnss_start(struct rproc *rproc)
 	if (wcnss->ready_irq > 0 && ret == 0) {
 		/* We have a ready_irq, but it didn't fire in time. */
 		dev_err(wcnss->dev, "start timed out\n");
-		qcom_scm_pas_shutdown(WCNSS_PAS_ID);
+		qcom_pas_shutdown(WCNSS_PAS_ID);
 		ret = -ETIMEDOUT;
 		goto disable_iris;
 	}
@@ -311,7 +311,7 @@ static int wcnss_stop(struct rproc *rproc)
 					    0);
 	}
 
-	ret = qcom_scm_pas_shutdown(WCNSS_PAS_ID);
+	ret = qcom_pas_shutdown(WCNSS_PAS_ID);
 	if (ret)
 		dev_err(wcnss->dev, "failed to shutdown: %d\n", ret);
 
@@ -441,25 +441,31 @@ static void wcnss_release_pds(struct qcom_wcnss *wcnss)
 }
 
 static int wcnss_init_regulators(struct qcom_wcnss *wcnss,
-				 const struct wcnss_vreg_info *info,
-				 int num_vregs, int num_pd_vregs)
+				 const struct wcnss_data *data)
 {
+	const struct wcnss_vreg_info *info = data->vregs;
 	struct regulator_bulk_data *bulk;
+	size_t i, possible_pds = 0, num_vregs = data->num_vregs;
 	int ret;
-	int i;
+
+	for (i = 0; i < WCNSS_MAX_PDS; i++)
+		if (data->pd_names[i])
+			possible_pds++;
 
 	/*
 	 * If attaching the power domains suceeded we can skip requesting
 	 * the regulators for the power domains. For old device trees we need to
 	 * reserve extra space to manage them through the regulator interface.
 	 */
-	if (wcnss->num_pds) {
+	if (possible_pds >= num_vregs) {
+		/* Do nothing if vregs do not include PD regulators (pronto-v3) */
+	} else if (wcnss->num_pds) {
 		info += wcnss->num_pds;
 		/* Handle single power domain case */
-		if (wcnss->num_pds < num_pd_vregs)
-			num_vregs += num_pd_vregs - wcnss->num_pds;
+		if (wcnss->num_pds < data->num_pd_vregs)
+			num_vregs += data->num_pd_vregs - wcnss->num_pds;
 	} else {
-		num_vregs += num_pd_vregs;
+		num_vregs += data->num_pd_vregs;
 	}
 
 	bulk = devm_kcalloc(wcnss->dev,
@@ -557,10 +563,10 @@ static int wcnss_probe(struct platform_device *pdev)
 
 	data = of_device_get_match_data(&pdev->dev);
 
-	if (!qcom_scm_is_available())
+	if (!qcom_pas_is_available())
 		return -EPROBE_DEFER;
 
-	if (!qcom_scm_pas_supported(WCNSS_PAS_ID)) {
+	if (!qcom_pas_supported(WCNSS_PAS_ID)) {
 		dev_err(&pdev->dev, "PAS is not available for WCNSS\n");
 		return -ENXIO;
 	}
@@ -607,8 +613,7 @@ static int wcnss_probe(struct platform_device *pdev)
 	if (ret && (ret != -ENODATA || !data->num_pd_vregs))
 		return ret;
 
-	ret = wcnss_init_regulators(wcnss, data->vregs, data->num_vregs,
-				    data->num_pd_vregs);
+	ret = wcnss_init_regulators(wcnss, data);
 	if (ret)
 		goto detach_pds;
 
diff --git a/drivers/remoteproc/rcar_rproc.c b/drivers/remoteproc/rcar_rproc.c
index 3c25625f966dc..1fe6c01bde406 100644
--- a/drivers/remoteproc/rcar_rproc.c
+++ b/drivers/remoteproc/rcar_rproc.c
@@ -19,35 +19,6 @@ struct rcar_rproc {
 	struct reset_control *rst;
 };
 
-static int rcar_rproc_mem_alloc(struct rproc *rproc,
-				 struct rproc_mem_entry *mem)
-{
-	struct device *dev = &rproc->dev;
-	void *va;
-
-	dev_dbg(dev, "map memory: %pa+%zx\n", &mem->dma, mem->len);
-	va = ioremap_wc(mem->dma, mem->len);
-	if (!va) {
-		dev_err(dev, "Unable to map memory region: %pa+%zx\n",
-			&mem->dma, mem->len);
-		return -ENOMEM;
-	}
-
-	/* Update memory entry va */
-	mem->va = va;
-
-	return 0;
-}
-
-static int rcar_rproc_mem_release(struct rproc *rproc,
-				   struct rproc_mem_entry *mem)
-{
-	dev_dbg(&rproc->dev, "unmap memory: %pa\n", &mem->dma);
-	iounmap(mem->va);
-
-	return 0;
-}
-
 static int rcar_rproc_prepare(struct rproc *rproc)
 {
 	struct device *dev = rproc->dev.parent;
@@ -73,8 +44,8 @@ static int rcar_rproc_prepare(struct rproc *rproc)
 		mem = rproc_mem_entry_init(dev, NULL,
 					   res.start,
 					   resource_size(&res), da,
-					   rcar_rproc_mem_alloc,
-					   rcar_rproc_mem_release,
+					   rproc_mem_entry_ioremap_wc,
+					   rproc_mem_entry_iounmap,
 					   res.name);
 
 		if (!mem)
@@ -86,12 +57,8 @@ static int rcar_rproc_prepare(struct rproc *rproc)
 
 static int rcar_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw)
 {
-	int ret;
-
-	ret = rproc_elf_load_rsc_table(rproc, fw);
-	if (ret)
-		dev_info(&rproc->dev, "No resource table in elf\n");
-
+	rproc_elf_load_rsc_table_optional(rproc, fw, dev_info,
+					  "No resource table in elf\n");
 	return 0;
 }
 
diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c
index f003be006b1bf..0583d3a56212c 100644
--- a/drivers/remoteproc/remoteproc_core.c
+++ b/drivers/remoteproc/remoteproc_core.c
@@ -1099,6 +1099,8 @@ static int rproc_start_subdevices(struct rproc *rproc)
 		}
 	}
 
+	rproc->subdevs_started = true;
+
 	return 0;
 
 unroll_registration:
@@ -1114,10 +1116,15 @@ static void rproc_stop_subdevices(struct rproc *rproc, bool crashed)
 {
 	struct rproc_subdev *subdev;
 
+	if (!rproc->subdevs_started)
+		return;
+
 	list_for_each_entry_reverse(subdev, &rproc->subdevs, node) {
 		if (subdev->stop)
 			subdev->stop(subdev, crashed);
 	}
+
+	rproc->subdevs_started = false;
 }
 
 static void rproc_unprepare_subdevices(struct rproc *rproc)
@@ -1668,18 +1675,21 @@ static void rproc_auto_boot_callback(const struct firmware *fw, void *context)
 	release_firmware(fw);
 }
 
+static void rproc_attach_work(struct work_struct *work)
+{
+	struct rproc *rproc = container_of(work, struct rproc, attach_work);
+
+	rproc_boot(rproc);
+}
+
 static int rproc_trigger_auto_boot(struct rproc *rproc)
 {
 	int ret;
 
-	/*
-	 * Since the remote processor is in a detached state, it has already
-	 * been booted by another entity.  As such there is no point in waiting
-	 * for a firmware image to be loaded, we can simply initiate the process
-	 * of attaching to it immediately.
-	 */
-	if (rproc->state == RPROC_DETACHED)
-		return rproc_boot(rproc);
+	if (rproc->state == RPROC_DETACHED) {
+		schedule_work(&rproc->attach_work);
+		return 0;
+	}
 
 	/*
 	 * We're initiating an asynchronous firmware loading, so we can
@@ -1776,7 +1786,20 @@ static int rproc_attach_recovery(struct rproc *rproc)
 	if (ret)
 		return ret;
 
-	return __rproc_attach(rproc);
+	/* clean up all acquired resources */
+	rproc_resource_cleanup(rproc);
+
+	/* release HW resources if needed */
+	rproc_unprepare_device(rproc);
+
+	rproc_disable_iommu(rproc);
+
+	/* Free the copy of the resource table */
+	kfree(rproc->cached_table);
+	rproc->cached_table = NULL;
+	rproc->table_ptr = NULL;
+
+	return rproc_attach(rproc);
 }
 
 static int rproc_boot_recovery(struct rproc *rproc)
@@ -2317,6 +2340,7 @@ int rproc_add(struct rproc *rproc)
 	return 0;
 
 rproc_remove_dev:
+	cancel_work_sync(&rproc->crash_handler);
 	rproc_delete_debug_dir(rproc);
 	device_del(dev);
 rproc_remove_cdev:
@@ -2507,6 +2531,7 @@ struct rproc *rproc_alloc(struct device *dev, const char *name,
 	INIT_LIST_HEAD(&rproc->dump_segments);
 
 	INIT_WORK(&rproc->crash_handler, rproc_crash_handler_work);
+	INIT_WORK(&rproc->attach_work, rproc_attach_work);
 
 	rproc->state = RPROC_OFFLINE;
 
diff --git a/drivers/remoteproc/remoteproc_internal.h b/drivers/remoteproc/remoteproc_internal.h
index 0a5e15744b1d9..3a742ef6ef606 100644
--- a/drivers/remoteproc/remoteproc_internal.h
+++ b/drivers/remoteproc/remoteproc_internal.h
@@ -14,6 +14,9 @@
 
 #include <linux/irqreturn.h>
 #include <linux/firmware.h>
+#ifdef CONFIG_HAS_IOMEM
+#include <linux/io.h>
+#endif
 
 struct rproc;
 
@@ -122,6 +125,57 @@ rproc_find_carveout_by_name(struct rproc *rproc, const char *name, ...);
 void rproc_add_rvdev(struct rproc *rproc, struct rproc_vdev *rvdev);
 void rproc_remove_rvdev(struct rproc_vdev *rvdev);
 
+#ifdef CONFIG_HAS_IOMEM
+static inline int rproc_mem_entry_ioremap_wc(struct rproc *rproc,
+					     struct rproc_mem_entry *mem)
+{
+	void __iomem *va;
+
+	va = ioremap_wc(mem->dma, mem->len);
+	if (!va) {
+		dev_err(&rproc->dev, "Unable to map memory region: %pa+%zx\n",
+			&mem->dma, mem->len);
+		return -ENOMEM;
+	}
+
+	mem->va = (__force void *)va;
+	mem->is_iomem = true;
+
+	return 0;
+}
+
+static inline int rproc_mem_entry_iounmap(struct rproc *rproc,
+					  struct rproc_mem_entry *mem)
+{
+	iounmap((__force __iomem void *)mem->va);
+
+	return 0;
+}
+#else
+static inline int rproc_mem_entry_ioremap_wc(struct rproc *rproc,
+					     struct rproc_mem_entry *mem)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int rproc_mem_entry_iounmap(struct rproc *rproc,
+					  struct rproc_mem_entry *mem)
+{
+	return 0;
+}
+#endif
+
+#define rproc_elf_load_rsc_table_optional(rproc, fw, dev_func, fmt, ...)	\
+	({									\
+		int ret = rproc_elf_load_rsc_table(rproc, fw);			\
+		if (ret == -EINVAL) {						\
+			dev_func(&rproc->dev, fmt, ##__VA_ARGS__);		\
+			return 0;						\
+		} else {							\
+			return ret;						\
+		}								\
+	})
+
 static inline int rproc_prepare_device(struct rproc *rproc)
 {
 	if (rproc->ops->prepare)
diff --git a/drivers/remoteproc/st_remoteproc.c b/drivers/remoteproc/st_remoteproc.c
index a07edf7217d21..486180cdccb40 100644
--- a/drivers/remoteproc/st_remoteproc.c
+++ b/drivers/remoteproc/st_remoteproc.c
@@ -88,33 +88,6 @@ static void st_rproc_kick(struct rproc *rproc, int vqid)
 		dev_err(dev, "failed to send message via mbox: %d\n", ret);
 }
 
-static int st_rproc_mem_alloc(struct rproc *rproc,
-			      struct rproc_mem_entry *mem)
-{
-	struct device *dev = rproc->dev.parent;
-	void *va;
-
-	va = ioremap_wc(mem->dma, mem->len);
-	if (!va) {
-		dev_err(dev, "Unable to map memory region: %pa+%zx\n",
-			&mem->dma, mem->len);
-		return -ENOMEM;
-	}
-
-	/* Update memory entry va */
-	mem->va = va;
-
-	return 0;
-}
-
-static int st_rproc_mem_release(struct rproc *rproc,
-				struct rproc_mem_entry *mem)
-{
-	iounmap(mem->va);
-
-	return 0;
-}
-
 static int st_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw)
 {
 	struct device *dev = rproc->dev.parent;
@@ -138,8 +111,8 @@ static int st_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw)
 			mem = rproc_mem_entry_init(dev, NULL,
 						   (dma_addr_t)res.start,
 						   resource_size(&res), res.start,
-						   st_rproc_mem_alloc,
-						   st_rproc_mem_release,
+						   rproc_mem_entry_ioremap_wc,
+						   rproc_mem_entry_iounmap,
 						   "%.*s",
 						   strchrnul(res.name, '@') - res.name,
 						   res.name);
diff --git a/drivers/remoteproc/stm32_rproc.c b/drivers/remoteproc/stm32_rproc.c
index 632614013dc65..0e5d64fbe52c1 100644
--- a/drivers/remoteproc/stm32_rproc.c
+++ b/drivers/remoteproc/stm32_rproc.c
@@ -113,35 +113,6 @@ static int stm32_rproc_pa_to_da(struct rproc *rproc, phys_addr_t pa, u64 *da)
 	return -EINVAL;
 }
 
-static int stm32_rproc_mem_alloc(struct rproc *rproc,
-				 struct rproc_mem_entry *mem)
-{
-	struct device *dev = rproc->dev.parent;
-	void *va;
-
-	dev_dbg(dev, "map memory: %pad+%zx\n", &mem->dma, mem->len);
-	va = (__force void *)ioremap_wc(mem->dma, mem->len);
-	if (IS_ERR_OR_NULL(va)) {
-		dev_err(dev, "Unable to map memory region: %pad+0x%zx\n",
-			&mem->dma, mem->len);
-		return -ENOMEM;
-	}
-
-	/* Update memory entry va */
-	mem->va = va;
-
-	return 0;
-}
-
-static int stm32_rproc_mem_release(struct rproc *rproc,
-				   struct rproc_mem_entry *mem)
-{
-	dev_dbg(rproc->dev.parent, "unmap memory: %pa\n", &mem->dma);
-	iounmap((__force __iomem void *)mem->va);
-
-	return 0;
-}
-
 static int stm32_rproc_of_memory_translations(struct platform_device *pdev,
 					      struct stm32_rproc *ddata)
 {
@@ -237,8 +208,8 @@ static int stm32_rproc_prepare(struct rproc *rproc)
 			mem = rproc_mem_entry_init(dev, NULL,
 						   (dma_addr_t)res.start,
 						   resource_size(&res), da,
-						   stm32_rproc_mem_alloc,
-						   stm32_rproc_mem_release,
+						   rproc_mem_entry_ioremap_wc,
+						   rproc_mem_entry_iounmap,
 						   "%.*s", strchrnul(res.name, '@') - res.name,
 						   res.name);
 			if (mem)
@@ -263,9 +234,8 @@ static int stm32_rproc_prepare(struct rproc *rproc)
 
 static int stm32_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw)
 {
-	if (rproc_elf_load_rsc_table(rproc, fw))
-		dev_warn(&rproc->dev, "no resource table found for this firmware\n");
-
+	rproc_elf_load_rsc_table_optional(rproc, fw, dev_warn,
+					  "no resource table found for this firmware\n");
 	return 0;
 }
 
@@ -957,4 +927,3 @@ MODULE_DESCRIPTION("STM32 Remote Processor Control Driver");
 MODULE_AUTHOR("Ludovic Barre <ludovic.barre@st.com>");
 MODULE_AUTHOR("Fabien Dessenne <fabien.dessenne@st.com>");
 MODULE_LICENSE("GPL v2");
-
diff --git a/drivers/remoteproc/ti_k3_r5_remoteproc.c b/drivers/remoteproc/ti_k3_r5_remoteproc.c
index 04f23295ffc10..b1d04d082e44c 100644
--- a/drivers/remoteproc/ti_k3_r5_remoteproc.c
+++ b/drivers/remoteproc/ti_k3_r5_remoteproc.c
@@ -1074,11 +1074,9 @@ static int k3_r5_cluster_rproc_init(struct platform_device *pdev)
 		}
 
 		kproc->reset = devm_reset_control_get_exclusive(cdev, NULL);
-		if (IS_ERR_OR_NULL(kproc->reset)) {
-			ret = PTR_ERR_OR_ZERO(kproc->reset);
-			if (!ret)
-				ret = -ENODEV;
-			dev_err_probe(cdev, ret, "failed to get reset handle\n");
+		if (IS_ERR(kproc->reset)) {
+			ret = dev_err_probe(cdev, PTR_ERR(kproc->reset),
+					    "failed to get reset handle\n");
 			goto out;
 		}
 
diff --git a/drivers/remoteproc/xlnx_r5_remoteproc.c b/drivers/remoteproc/xlnx_r5_remoteproc.c
index 3349d18777510..c685bb5fa62c3 100644
--- a/drivers/remoteproc/xlnx_r5_remoteproc.c
+++ b/drivers/remoteproc/xlnx_r5_remoteproc.c
@@ -112,6 +112,10 @@ struct rsc_tbl_data {
 	const uintptr_t rsc_tbl;
 } __packed;
 
+enum xlnx_rproc_fw_rsc {
+	XLNX_RPROC_FW_CRASH_REPORT = RSC_VENDOR_START,
+};
+
 /*
  * Hardcoded TCM bank values. This will stay in driver to maintain backward
  * compatibility with device-tree that does not have TCM information.
@@ -131,9 +135,27 @@ static const struct mem_bank_data zynqmp_tcm_banks_lockstep[] = {
 	{0xffe30000UL, 0x30000, 0x10000UL, PD_R5_1_BTCM, "btcm1"},
 };
 
+#define CRASH_REASON_STR_LEN 16
+
+/**
+ * struct xlnx_rproc_crash_report - resource to know crash status and reason
+ *
+ * @version: version of this resource
+ * @crashed: if true, the rproc is notifying crash, time to recover
+ * @crash_reason: number to describe reason of crash
+ * @crash_reason_str: short string description of crash reason
+ */
+struct xlnx_rproc_crash_report {
+	u8 version;
+	u8 crashed;
+	u8 crash_reason;
+	char crash_reason_str[CRASH_REASON_STR_LEN];
+} __packed;
+
 /**
  * struct zynqmp_r5_core - remoteproc core's internal data
  *
+ * @crash_report: rproc crash state and reason
  * @rsc_tbl_va: resource table virtual address
  * @sram: Array of sram memories assigned to this core
  * @num_sram: number of sram for this core
@@ -147,6 +169,7 @@ static const struct mem_bank_data zynqmp_tcm_banks_lockstep[] = {
  * @ipi: pointer to mailbox information
  */
 struct zynqmp_r5_core {
+	struct xlnx_rproc_crash_report *crash_report;
 	void __iomem *rsc_tbl_va;
 	struct zynqmp_sram_bank *sram;
 	int num_sram;
@@ -204,11 +227,27 @@ static int event_notified_idr_cb(int id, void *ptr, void *data)
  */
 static void handle_event_notified(struct work_struct *work)
 {
+	struct xlnx_rproc_crash_report *report;
+	struct zynqmp_r5_core *r5_core;
 	struct mbox_info *ipi;
 	struct rproc *rproc;
 
 	ipi = container_of(work, struct mbox_info, mbox_work);
 	rproc = ipi->r5_core->rproc;
+	r5_core = ipi->r5_core;
+	report = r5_core->crash_report;
+
+	/* report crash only if expected */
+	if (report && report->crashed) {
+		if (rproc->state == RPROC_ATTACHED || rproc->state == RPROC_RUNNING) {
+			report->crash_reason_str[CRASH_REASON_STR_LEN - 1] = '\0';
+			dev_warn(&rproc->dev, "crash reason id: %d %s\n",
+				 report->crash_reason, report->crash_reason_str);
+			rproc_report_crash(rproc, RPROC_FATAL_ERROR);
+			report->crashed = false;
+			return;
+		}
+	}
 
 	/*
 	 * We only use IPI for interrupt. The RPU firmware side may or may
@@ -364,49 +403,12 @@ static void zynqmp_r5_rproc_kick(struct rproc *rproc, int vqid)
 static int zynqmp_r5_rproc_start(struct rproc *rproc)
 {
 	struct zynqmp_r5_core *r5_core = rproc->priv;
-	enum rpu_boot_mem bootmem;
 	int ret;
 
-	/*
-	 * The exception vector pointers (EVP) refer to the base-address of
-	 * exception vectors (for reset, IRQ, FIQ, etc). The reset-vector
-	 * starts at the base-address and subsequent vectors are on 4-byte
-	 * boundaries.
-	 *
-	 * Exception vectors can start either from 0x0000_0000 (LOVEC) or
-	 * from 0xFFFF_0000 (HIVEC) which is mapped in the OCM (On-Chip Memory)
-	 *
-	 * Usually firmware will put Exception vectors at LOVEC.
-	 *
-	 * It is not recommend that you change the exception vector.
-	 * Changing the EVP to HIVEC will result in increased interrupt latency
-	 * and jitter. Also, if the OCM is secured and the Cortex-R5F processor
-	 * is non-secured, then the Cortex-R5F processor cannot access the
-	 * HIVEC exception vectors in the OCM.
-	 */
-	bootmem = (rproc->bootaddr >= 0xFFFC0000) ?
-		   PM_RPU_BOOTMEM_HIVEC : PM_RPU_BOOTMEM_LOVEC;
-
-	dev_dbg(r5_core->dev, "RPU boot addr 0x%llx from %s.", rproc->bootaddr,
-		bootmem == PM_RPU_BOOTMEM_HIVEC ? "OCM" : "TCM");
-
-	/* Request node before starting RPU core if new version of API is supported */
-	if (zynqmp_pm_feature(PM_REQUEST_NODE) > 1) {
-		ret = zynqmp_pm_request_node(r5_core->pm_domain_id,
-					     ZYNQMP_PM_CAPABILITY_ACCESS, 0,
-					     ZYNQMP_PM_REQUEST_ACK_BLOCKING);
-		if (ret < 0) {
-			dev_err(r5_core->dev, "failed to request 0x%x",
-				r5_core->pm_domain_id);
-			return ret;
-		}
-	}
-
-	ret = zynqmp_pm_request_wake(r5_core->pm_domain_id, 1,
-				     bootmem, ZYNQMP_PM_REQUEST_ACK_NO);
+	ret = zynqmp_pm_start_rpu(r5_core->pm_domain_id, rproc->bootaddr);
 	if (ret)
-		dev_err(r5_core->dev,
-			"failed to start RPU = 0x%x\n", r5_core->pm_domain_id);
+		dev_err(&rproc->dev, "failed to start RPU\n");
+
 	return ret;
 }
 
@@ -423,73 +425,20 @@ static int zynqmp_r5_rproc_stop(struct rproc *rproc)
 	struct zynqmp_r5_core *r5_core = rproc->priv;
 	int ret;
 
-	/* Use release node API to stop core if new version of API is supported */
-	if (zynqmp_pm_feature(PM_RELEASE_NODE) > 1) {
-		ret = zynqmp_pm_release_node(r5_core->pm_domain_id);
-		if (ret)
-			dev_err(r5_core->dev, "failed to stop remoteproc RPU %d\n", ret);
-		return ret;
-	}
+	ret = zynqmp_pm_stop_rpu(r5_core->pm_domain_id);
+	if (ret)
+		dev_err(&rproc->dev, "failed to stop RPU\n");
 
 	/*
-	 * Check expected version of EEMI call before calling it. This avoids
-	 * any error or warning prints from firmware as it is expected that fw
-	 * doesn't support it.
+	 * Clear attach on recovery flag during stop operation. The next state
+	 * of the remote processor is expected to be in "Running" state. In this
+	 * state, boot recovery method must take place over attach on recovery.
 	 */
-	if (zynqmp_pm_feature(PM_FORCE_POWERDOWN) != 1) {
-		dev_dbg(r5_core->dev, "EEMI interface %d ver 1 not supported\n",
-			PM_FORCE_POWERDOWN);
-		return -EOPNOTSUPP;
-	}
-
-	/* maintain force pwr down for backward compatibility */
-	ret = zynqmp_pm_force_pwrdwn(r5_core->pm_domain_id,
-				     ZYNQMP_PM_REQUEST_ACK_BLOCKING);
-	if (ret)
-		dev_err(r5_core->dev, "core force power down failed\n");
+	test_and_clear_bit(RPROC_FEAT_ATTACH_ON_RECOVERY, rproc->features);
 
 	return ret;
 }
 
-/*
- * zynqmp_r5_mem_region_map()
- * @rproc: single R5 core's corresponding rproc instance
- * @mem: mem descriptor to map reserved memory-regions
- *
- * Callback to map va for memory-region's carveout.
- *
- * return 0 on success, otherwise non-zero value on failure
- */
-static int zynqmp_r5_mem_region_map(struct rproc *rproc,
-				    struct rproc_mem_entry *mem)
-{
-	void __iomem *va;
-
-	va = ioremap_wc(mem->dma, mem->len);
-	if (IS_ERR_OR_NULL(va))
-		return -ENOMEM;
-
-	mem->va = (void *)va;
-
-	return 0;
-}
-
-/*
- * zynqmp_r5_rproc_mem_unmap
- * @rproc: single R5 core's corresponding rproc instance
- * @mem: mem entry to unmap
- *
- * Unmap memory-region carveout
- *
- * return: always returns 0
- */
-static int zynqmp_r5_mem_region_unmap(struct rproc *rproc,
-				      struct rproc_mem_entry *mem)
-{
-	iounmap((void __iomem *)mem->va);
-	return 0;
-}
-
 /*
  * add_mem_regions_carveout()
  * @rproc: single R5 core's corresponding rproc instance
@@ -526,8 +475,8 @@ static int add_mem_regions_carveout(struct rproc *rproc)
 			rproc_mem = rproc_mem_entry_init(&rproc->dev, NULL,
 							 (dma_addr_t)res.start,
 							 resource_size(&res), res.start,
-							 zynqmp_r5_mem_region_map,
-							 zynqmp_r5_mem_region_unmap,
+							 rproc_mem_entry_ioremap_wc,
+							 rproc_mem_entry_iounmap,
 							 "%.*s",
 							 strchrnul(res.name, '@') - res.name,
 							 res.name);
@@ -564,8 +513,8 @@ static int add_sram_carveouts(struct rproc *rproc)
 		rproc_mem = rproc_mem_entry_init(&rproc->dev, NULL,
 						 dma_addr,
 						 len, da,
-						 zynqmp_r5_mem_region_map,
-						 zynqmp_r5_mem_region_unmap,
+						 rproc_mem_entry_ioremap_wc,
+						 rproc_mem_entry_iounmap,
 						 sram->sram_res.name);
 		if (!rproc_mem) {
 			dev_err(&rproc->dev, "failed to add sram %s da=0x%x, size=0x%lx",
@@ -718,20 +667,15 @@ static int add_tcm_banks(struct rproc *rproc)
  */
 static int zynqmp_r5_parse_fw(struct rproc *rproc, const struct firmware *fw)
 {
-	int ret;
-
-	ret = rproc_elf_load_rsc_table(rproc, fw);
-	if (ret == -EINVAL) {
-		/*
-		 * resource table only required for IPC.
-		 * if not present, this is not necessarily an error;
-		 * for example, loading r5 hello world application
-		 * so simply inform user and keep going.
-		 */
-		dev_info(&rproc->dev, "no resource table found.\n");
-		ret = 0;
-	}
-	return ret;
+	/*
+	 * resource table only required for IPC.
+	 * if not present, this is not necessarily an error;
+	 * for example, loading r5 hello world application
+	 * so simply inform user and keep going.
+	 */
+	rproc_elf_load_rsc_table_optional(rproc, fw, dev_info,
+					  "no resource table found.\n");
+	return 0;
 }
 
 /**
@@ -886,6 +830,24 @@ static int zynqmp_r5_detach(struct rproc *rproc)
 	return 0;
 }
 
+static int zynqmp_r5_handle_rsc(struct rproc *rproc, u32 rsc_type, void *rsc,
+				int offset, int avail)
+{
+	struct zynqmp_r5_core *r5_core = rproc->priv;
+	void *rsc_offset = (r5_core->rsc_tbl_va + offset);
+
+	if (rsc_type != XLNX_RPROC_FW_CRASH_REPORT)
+		return RSC_IGNORED;
+
+	r5_core->crash_report = rsc_offset;
+	/* reset all values */
+	r5_core->crash_report->crashed = false;
+	r5_core->crash_report->crash_reason = 0;
+	r5_core->crash_report->crash_reason_str[0] = '\0';
+
+	return RSC_HANDLED;
+}
+
 static const struct rproc_ops zynqmp_r5_rproc_ops = {
 	.prepare	= zynqmp_r5_rproc_prepare,
 	.unprepare	= zynqmp_r5_rproc_unprepare,
@@ -900,6 +862,7 @@ static const struct rproc_ops zynqmp_r5_rproc_ops = {
 	.get_loaded_rsc_table = zynqmp_r5_get_loaded_rsc_table,
 	.attach		= zynqmp_r5_attach,
 	.detach		= zynqmp_r5_detach,
+	.handle_rsc	= zynqmp_r5_handle_rsc,
 };
 
 /**
@@ -939,7 +902,7 @@ static struct zynqmp_r5_core *zynqmp_r5_alloc_rproc_core(struct device *cdev)
 
 	rproc_coredump_set_elf_info(r5_rproc, ELFCLASS32, EM_ARM);
 
-	r5_rproc->recovery_disabled = true;
+	r5_rproc->recovery_disabled = false;
 	r5_rproc->has_iommu = false;
 	r5_rproc->auto_boot = false;
 
@@ -1288,6 +1251,9 @@ static int zynqmp_r5_core_init(struct zynqmp_r5_cluster *cluster,
 			if (zynqmp_r5_get_rsc_table_va(r5_core))
 				dev_dbg(r5_core->dev, "rsc tbl not found\n");
 			r5_core->rproc->state = RPROC_DETACHED;
+			/* Enable attach on recovery method. Clear it during rproc stop. */
+			rproc_set_feature(r5_core->rproc,
+					  RPROC_FEAT_ATTACH_ON_RECOVERY);
 			r5_core->rproc->auto_boot = true;
 		}
 	}
diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c
index b69f994881111..2f09f4b78bd3b 100644
--- a/drivers/resctrl/mpam_devices.c
+++ b/drivers/resctrl/mpam_devices.c
@@ -1552,12 +1552,9 @@ static u16 mpam_wa_t241_calc_min_from_max(struct mpam_props *props,
 static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid,
 				      struct mpam_config *cfg)
 {
-	u32 pri_val = 0;
 	u16 cmax = MPAMCFG_CMAX_CMAX;
 	struct mpam_msc *msc = ris->vmsc->msc;
 	struct mpam_props *rprops = &ris->props;
-	u16 dspri = GENMASK(rprops->dspri_wd, 0);
-	u16 intpri = GENMASK(rprops->intpri_wd, 0);
 
 	mutex_lock(&msc->part_sel_lock);
 	__mpam_part_sel(ris->ris_idx, partid, msc);
@@ -1583,9 +1580,9 @@ static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid,
 
 	if (mpam_has_feature(mpam_feat_mbw_part, rprops)) {
 		if (mpam_has_feature(mpam_feat_mbw_part, cfg))
-			mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM, rprops->mbw_pbm_bits);
-		else
 			mpam_write_partsel_reg(msc, MBW_PBM, cfg->mbw_pbm);
+		else
+			mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM, rprops->mbw_pbm_bits);
 	}
 
 	if (mpam_has_feature(mpam_feat_mbw_min, rprops)) {
@@ -1622,16 +1619,25 @@ static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid,
 
 	if (mpam_has_feature(mpam_feat_intpri_part, rprops) ||
 	    mpam_has_feature(mpam_feat_dspri_part, rprops)) {
-		/* aces high? */
-		if (!mpam_has_feature(mpam_feat_intpri_part_0_low, rprops))
-			intpri = 0;
-		if (!mpam_has_feature(mpam_feat_dspri_part_0_low, rprops))
-			dspri = 0;
+		u32 pri_val = 0;
+
+		if (mpam_has_feature(mpam_feat_intpri_part, rprops)) {
+			u16 intpri = GENMASK(rprops->intpri_wd - 1, 0);
+
+			/* aces high? */
+			if (!mpam_has_feature(mpam_feat_intpri_part_0_low, rprops))
+				intpri = 0;
 
-		if (mpam_has_feature(mpam_feat_intpri_part, rprops))
 			pri_val |= FIELD_PREP(MPAMCFG_PRI_INTPRI, intpri);
-		if (mpam_has_feature(mpam_feat_dspri_part, rprops))
+		}
+		if (mpam_has_feature(mpam_feat_dspri_part, rprops)) {
+			u16 dspri = GENMASK(rprops->dspri_wd - 1, 0);
+
+			if (!mpam_has_feature(mpam_feat_dspri_part_0_low, rprops))
+				dspri = 0;
+
 			pri_val |= FIELD_PREP(MPAMCFG_PRI_DSPRI, dspri);
+		}
 
 		mpam_write_partsel_reg(msc, PRI, pri_val);
 	}
@@ -2606,8 +2612,10 @@ static void __destroy_component_cfg(struct mpam_component *comp)
 		msc = vmsc->msc;
 
 		if (mpam_mon_sel_lock(msc)) {
-			list_for_each_entry(ris, &vmsc->ris, vmsc_list)
-				add_to_garbage(ris->mbwu_state);
+			list_for_each_entry(ris, &vmsc->ris, vmsc_list) {
+				if (ris->mbwu_state)
+					add_to_garbage(ris->mbwu_state);
+			}
 			mpam_mon_sel_unlock(msc);
 		}
 	}
diff --git a/drivers/reset/reset-imx7.c b/drivers/reset/reset-imx7.c
index dd01fe11c5cb3..a3cb8244d76a6 100644
--- a/drivers/reset/reset-imx7.c
+++ b/drivers/reset/reset-imx7.c
@@ -236,6 +236,12 @@ static int imx8mq_reset_set(struct reset_controller_dev *rcdev,
 
 	case IMX8MQ_RESET_PCIE_CTRL_APPS_EN:
 	case IMX8MQ_RESET_PCIE2_CTRL_APPS_EN:
+	case IMX8MQ_RESET_MIPI_CSI1_CORE_RESET:
+	case IMX8MQ_RESET_MIPI_CSI1_PHY_REF_RESET:
+	case IMX8MQ_RESET_MIPI_CSI1_ESC_RESET:
+	case IMX8MQ_RESET_MIPI_CSI2_CORE_RESET:
+	case IMX8MQ_RESET_MIPI_CSI2_PHY_REF_RESET:
+	case IMX8MQ_RESET_MIPI_CSI2_ESC_RESET:
 	case IMX8MQ_RESET_MIPI_DSI_PCLK_RESET_N:
 	case IMX8MQ_RESET_MIPI_DSI_ESC_RESET_N:
 	case IMX8MQ_RESET_MIPI_DSI_DPI_RESET_N:
diff --git a/drivers/reset/reset-rzg2l-usbphy-ctrl.c b/drivers/reset/reset-rzg2l-usbphy-ctrl.c
index fd75d9601a3bf..f003b360629c9 100644
--- a/drivers/reset/reset-rzg2l-usbphy-ctrl.c
+++ b/drivers/reset/reset-rzg2l-usbphy-ctrl.c
@@ -249,7 +249,7 @@ static int rzg2l_usbphy_ctrl_probe(struct platform_device *pdev)
 	vdev->dev.parent = dev;
 	priv->vdev = vdev;
 
-	device_set_of_node_from_dev(&vdev->dev, dev);
+	platform_device_set_of_node_from_dev(vdev, dev);
 	error = platform_device_add(vdev);
 	if (error)
 		goto err_device_put;
diff --git a/drivers/reset/reset-sunxi.c b/drivers/reset/reset-sunxi.c
index 2544de6576e44..2f6df7707cad1 100644
--- a/drivers/reset/reset-sunxi.c
+++ b/drivers/reset/reset-sunxi.c
@@ -44,7 +44,7 @@ static int sunxi_reset_init(struct device_node *np)
 	data->membase = ioremap(res.start, size);
 	if (!data->membase) {
 		ret = -ENOMEM;
-		goto err_alloc;
+		goto err_mem_region;
 	}
 
 	spin_lock_init(&data->lock);
@@ -57,6 +57,8 @@ static int sunxi_reset_init(struct device_node *np)
 
 	return reset_controller_register(&data->rcdev);
 
+err_mem_region:
+	release_mem_region(res.start, size);
 err_alloc:
 	kfree(data);
 	return ret;
diff --git a/drivers/reset/spacemit/reset-spacemit-k3.c b/drivers/reset/spacemit/reset-spacemit-k3.c
index 9841f5e057b2a..2e87f320cf118 100644
--- a/drivers/reset/spacemit/reset-spacemit-k3.c
+++ b/drivers/reset/spacemit/reset-spacemit-k3.c
@@ -112,7 +112,7 @@ static const struct ccu_reset_data k3_apmu_resets[] = {
 	[RESET_APMU_SDH0]	= RESET_DATA(APMU_SDH0_CLK_RES_CTRL,	0, BIT(1)),
 	[RESET_APMU_SDH1]	= RESET_DATA(APMU_SDH1_CLK_RES_CTRL,	0, BIT(1)),
 	[RESET_APMU_SDH2]	= RESET_DATA(APMU_SDH2_CLK_RES_CTRL,	0, BIT(1)),
-	[RESET_APMU_USB2_AHB]	= RESET_DATA(APMU_USB_CLK_RES_CTRL,	0, BIT(1)),
+	[RESET_APMU_USB2_AHB]	= RESET_DATA(APMU_USB_CLK_RES_CTRL,	0, BIT(0)),
 	[RESET_APMU_USB2_VCC]	= RESET_DATA(APMU_USB_CLK_RES_CTRL,	0, BIT(2)),
 	[RESET_APMU_USB2_PHY]	= RESET_DATA(APMU_USB_CLK_RES_CTRL,	0, BIT(3)),
 	[RESET_APMU_USB3_A_AHB]	= RESET_DATA(APMU_USB_CLK_RES_CTRL,	0, BIT(5)),
diff --git a/drivers/rpmsg/qcom_glink_native.c b/drivers/rpmsg/qcom_glink_native.c
index d9d4468e4cbdf..55793fc18293b 100644
--- a/drivers/rpmsg/qcom_glink_native.c
+++ b/drivers/rpmsg/qcom_glink_native.c
@@ -501,7 +501,7 @@ static int qcom_glink_send_open_req(struct qcom_glink *glink,
 	req->cmd = cpu_to_le16(GLINK_CMD_OPEN);
 	req->param1 = cpu_to_le16(channel->lcid);
 	req->param2 = cpu_to_le32(name_len);
-	strcpy(req->data, channel->name);
+	strscpy(req->data, channel->name, GLINK_NAME_SIZE);
 
 	trace_qcom_glink_cmd_open_tx(glink->label, channel->name,
 				     channel->lcid, channel->rcid);
@@ -1418,9 +1418,6 @@ static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept)
 	channel->ept.cb = NULL;
 	spin_unlock_irqrestore(&channel->recv_lock, flags);
 
-	/* Decouple the potential rpdev from the channel */
-	qcom_glink_remove_rpmsg_device(glink, channel);
-
 	qcom_glink_send_close_req(glink, channel);
 }
 
diff --git a/drivers/rpmsg/qcom_glink_smem.c b/drivers/rpmsg/qcom_glink_smem.c
index 62adc4db23170..35bb03e67ae8a 100644
--- a/drivers/rpmsg/qcom_glink_smem.c
+++ b/drivers/rpmsg/qcom_glink_smem.c
@@ -103,6 +103,13 @@ static void glink_smem_rx_peek(struct qcom_glink_pipe *np,
 	if (tail >= pipe->native.length)
 		tail -= pipe->native.length;
 
+	/*
+	 * Order the availability (head) read in glink_smem_rx_avail()
+	 * against the FIFO payload read below, so APPS never consumes
+	 * stale data the remote has not yet published.
+	 */
+	rmb();
+
 	len = min_t(size_t, count, pipe->native.length - tail);
 	if (len)
 		memcpy_fromio(data, pipe->fifo + tail, len);
diff --git a/drivers/rpmsg/qcom_smd.c b/drivers/rpmsg/qcom_smd.c
index 3ac863f400ec6..7dbe1c6efe41d 100644
--- a/drivers/rpmsg/qcom_smd.c
+++ b/drivers/rpmsg/qcom_smd.c
@@ -1460,7 +1460,7 @@ static ssize_t rpmsg_name_show(struct device *dev,
 {
 	struct qcom_smd_edge *edge = to_smd_edge(dev);
 
-	return sprintf(buf, "%s\n", edge->name);
+	return sysfs_emit(buf, "%s\n", edge->name);
 }
 static DEVICE_ATTR_RO(rpmsg_name);
 
diff --git a/drivers/rpmsg/rpmsg_char.c b/drivers/rpmsg/rpmsg_char.c
index bff5aefee2126..550fda217da45 100644
--- a/drivers/rpmsg/rpmsg_char.c
+++ b/drivers/rpmsg/rpmsg_char.c
@@ -79,6 +79,11 @@ int rpmsg_chrdev_eptdev_destroy(struct device *dev, void *data)
 	struct rpmsg_eptdev *eptdev = dev_to_eptdev(dev);
 
 	mutex_lock(&eptdev->ept_lock);
+	if (!eptdev->rpdev) {
+		mutex_unlock(&eptdev->ept_lock);
+		return 0;
+	}
+
 	eptdev->rpdev = NULL;
 	if (eptdev->ept) {
 		/* The default endpoint is released by the rpmsg core */
@@ -371,7 +376,7 @@ static ssize_t name_show(struct device *dev, struct device_attribute *attr,
 {
 	struct rpmsg_eptdev *eptdev = dev_get_drvdata(dev);
 
-	return sprintf(buf, "%s\n", eptdev->chinfo.name);
+	return sysfs_emit(buf, "%s\n", eptdev->chinfo.name);
 }
 static DEVICE_ATTR_RO(name);
 
@@ -380,7 +385,7 @@ static ssize_t src_show(struct device *dev, struct device_attribute *attr,
 {
 	struct rpmsg_eptdev *eptdev = dev_get_drvdata(dev);
 
-	return sprintf(buf, "%d\n", eptdev->chinfo.src);
+	return sysfs_emit(buf, "%d\n", eptdev->chinfo.src);
 }
 static DEVICE_ATTR_RO(src);
 
@@ -389,7 +394,7 @@ static ssize_t dst_show(struct device *dev, struct device_attribute *attr,
 {
 	struct rpmsg_eptdev *eptdev = dev_get_drvdata(dev);
 
-	return sprintf(buf, "%d\n", eptdev->chinfo.dst);
+	return sysfs_emit(buf, "%d\n", eptdev->chinfo.dst);
 }
 static DEVICE_ATTR_RO(dst);
 
diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c
index c56f69c22e420..04bfcc75f05b4 100644
--- a/drivers/rpmsg/rpmsg_core.c
+++ b/drivers/rpmsg/rpmsg_core.c
@@ -348,7 +348,7 @@ field##_show(struct device *dev,					\
 {									\
 	struct rpmsg_device *rpdev = to_rpmsg_device(dev);		\
 									\
-	return sprintf(buf, format_string, rpdev->path);		\
+	return sysfs_emit(buf, format_string, rpdev->path);		\
 }									\
 static DEVICE_ATTR_RO(field);
 
@@ -368,7 +368,7 @@ static ssize_t modalias_show(struct device *dev,
 	if (len != -ENODEV)
 		return len;
 
-	return sprintf(buf, RPMSG_DEVICE_MODALIAS_FMT "\n", rpdev->id.name);
+	return sysfs_emit(buf, RPMSG_DEVICE_MODALIAS_FMT "\n", rpdev->id.name);
 }
 static DEVICE_ATTR_RO(modalias);
 
@@ -598,8 +598,6 @@ EXPORT_SYMBOL(__register_rpmsg_driver);
 /**
  * unregister_rpmsg_driver() - unregister an rpmsg driver from the rpmsg bus
  * @rpdrv: pointer to a struct rpmsg_driver
- *
- * Return: 0 on success, and an appropriate error value on failure.
  */
 void unregister_rpmsg_driver(struct rpmsg_driver *rpdrv)
 {
diff --git a/drivers/scsi/bfa/bfa_fcs_lport.c b/drivers/scsi/bfa/bfa_fcs_lport.c
index 2df399c537c13..8c9d423129c07 100644
--- a/drivers/scsi/bfa/bfa_fcs_lport.c
+++ b/drivers/scsi/bfa/bfa_fcs_lport.c
@@ -2627,7 +2627,7 @@ bfa_fcs_fdmi_get_hbaattr(struct bfa_fcs_lport_fdmi_s *fdmi,
 
 }
 
-static void
+static noinline_for_stack void
 bfa_fcs_fdmi_get_portattr(struct bfa_fcs_lport_fdmi_s *fdmi,
 			  struct bfa_fcs_fdmi_port_attr_s *port_attr)
 {
diff --git a/drivers/scsi/elx/efct/efct_hw.c b/drivers/scsi/elx/efct/efct_hw.c
index 1838032f64862..b79c6a7ea7912 100644
--- a/drivers/scsi/elx/efct/efct_hw.c
+++ b/drivers/scsi/elx/efct/efct_hw.c
@@ -1997,6 +1997,8 @@ efct_hw_io_abort(struct efct_hw *hw, struct efct_hw_io *io_to_abort,
 	wqcb = efct_hw_reqtag_alloc(hw, efct_hw_wq_process_abort, io_to_abort);
 	if (!wqcb) {
 		efc_log_err(hw->os, "can't allocate request tag\n");
+		io_to_abort->abort_in_progress = false;
+		kref_put(&io_to_abort->ref, io_to_abort->release);
 		return -ENOSPC;
 	}
 
diff --git a/drivers/scsi/elx/efct/efct_unsol.c b/drivers/scsi/elx/efct/efct_unsol.c
index e6addab66a603..6a871a59c9096 100644
--- a/drivers/scsi/elx/efct/efct_unsol.c
+++ b/drivers/scsi/elx/efct/efct_unsol.c
@@ -385,6 +385,7 @@ efct_dispatch_fcp_cmd(struct efct_node *node, struct efc_hw_sequence *seq)
 
 		if (cmnd->fc_flags & FCP_CFL_LEN_MASK) {
 			efc_log_err(efct, "Additional CDB not supported\n");
+			efct_scsi_io_free(io);
 			return -EIO;
 		}
 		/*
diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c
index e047747d4ecf8..46cc8e3c79a26 100644
--- a/drivers/scsi/hosts.c
+++ b/drivers/scsi/hosts.c
@@ -357,6 +357,7 @@ static void scsi_host_dev_release(struct device *dev)
 	/* Wait for functions invoked through call_rcu(&scmd->rcu, ...) */
 	rcu_barrier();
 
+	cancel_work_sync(&shost->eh_work);
 	if (shost->tmf_work_q)
 		destroy_workqueue(shost->tmf_work_q);
 	if (shost->ehandler)
@@ -422,6 +423,7 @@ struct Scsi_Host *scsi_host_alloc(const struct scsi_host_template *sht, int priv
 	INIT_LIST_HEAD(&shost->starved_list);
 	init_waitqueue_head(&shost->host_wait);
 	mutex_init(&shost->scan_mutex);
+	INIT_WORK(&shost->eh_work, scsi_rcu_eh_wakeup);
 
 	index = ida_alloc(&host_index_ida, GFP_KERNEL);
 	if (index < 0) {
diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c
index a1b116cd4723d..8edad1830abe4 100644
--- a/drivers/scsi/hpsa.c
+++ b/drivers/scsi/hpsa.c
@@ -5017,6 +5017,10 @@ static int hpsa_scsi_ioaccel2_queue_command(struct ctlr_info *h,
 
 	if (phys_disk->in_reset) {
 		cmd->result = DID_RESET << 16;
+		atomic_dec(&phys_disk->ioaccel_cmds_out);
+		scsi_dma_unmap(cmd);
+		if (use_sg > h->ioaccel_maxsg)
+			hpsa_unmap_ioaccel2_sg_chain_block(h, cp);
 		return -1;
 	}
 
diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
index 82af59c913e93..23355f12fbffa 100644
--- a/drivers/scsi/lpfc/lpfc_init.c
+++ b/drivers/scsi/lpfc/lpfc_init.c
@@ -8189,6 +8189,7 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)
 		mempool_free(mboxq, phba->mbox_mem_pool);
 		goto out_free_bsmbx;
 	}
+	mempool_free(mboxq, phba->mbox_mem_pool);
 
 	/*
 	 * 1 for cmd, 1 for rsp, NVME adds an extra one
@@ -8311,8 +8312,6 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)
 		goto out_free_sg_dma_buf;
 	}
 
-	mempool_free(mboxq, phba->mbox_mem_pool);
-
 	/* Verify OAS is supported */
 	lpfc_sli4_oas_verify(phba);
 
diff --git a/drivers/scsi/mpt3sas/Makefile b/drivers/scsi/mpt3sas/Makefile
index e76d994dbed31..18e2d87eb4a2d 100644
--- a/drivers/scsi/mpt3sas/Makefile
+++ b/drivers/scsi/mpt3sas/Makefile
@@ -9,3 +9,5 @@ mpt3sas-y +=  mpt3sas_base.o     \
 		mpt3sas_trigger_diag.o \
 		mpt3sas_warpdrive.o \
 		mpt3sas_debugfs.o \
+
+mpt3sas-$(CONFIG_HWMON) += mpt3sas_hwmon.o
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.h b/drivers/scsi/mpt3sas/mpt3sas_base.h
index d4597d058705d..47255bf9cdda1 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.h
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.h
@@ -1629,6 +1629,9 @@ struct MPT3SAS_ADAPTER {
 	u8		is_aero_ioc;
 	struct dentry	*debugfs_root;
 	struct dentry	*ioc_dump;
+#if IS_REACHABLE(CONFIG_HWMON)
+	struct mpt3sas_hwmon *hwmon;
+#endif
 	PUT_SMID_IO_FP_HIP put_smid_scsi_io;
 	PUT_SMID_IO_FP_HIP put_smid_fast_path;
 	PUT_SMID_IO_FP_HIP put_smid_hi_priority;
@@ -1904,6 +1907,9 @@ int mpt3sas_config_get_iounit_pg3(struct MPT3SAS_ADAPTER *ioc,
 	Mpi2ConfigReply_t *mpi_reply, Mpi2IOUnitPage3_t *config_page, u16 sz);
 int mpt3sas_config_set_iounit_pg1(struct MPT3SAS_ADAPTER *ioc, Mpi2ConfigReply_t
 	*mpi_reply, Mpi2IOUnitPage1_t *config_page);
+int mpt3sas_config_get_iounit_pg7(struct MPT3SAS_ADAPTER *ioc,
+				  Mpi2ConfigReply_t *mpi_reply,
+				  Mpi2IOUnitPage7_t *config_page);
 int mpt3sas_config_get_iounit_pg8(struct MPT3SAS_ADAPTER *ioc, Mpi2ConfigReply_t
 	*mpi_reply, Mpi2IOUnitPage8_t *config_page);
 int mpt3sas_config_get_sas_iounit_pg1(struct MPT3SAS_ADAPTER *ioc,
@@ -2047,6 +2053,20 @@ void mpt3sas_destroy_debugfs(struct MPT3SAS_ADAPTER *ioc);
 void mpt3sas_init_debugfs(void);
 void mpt3sas_exit_debugfs(void);
 
+#if IS_REACHABLE(CONFIG_HWMON)
+int mpt3sas_hwmon_register(struct MPT3SAS_ADAPTER *ioc);
+void mpt3sas_hwmon_unregister(struct MPT3SAS_ADAPTER *ioc);
+#else
+static inline int mpt3sas_hwmon_register(struct MPT3SAS_ADAPTER *ioc)
+{
+	return 0;
+}
+
+static inline void mpt3sas_hwmon_unregister(struct MPT3SAS_ADAPTER *ioc)
+{
+}
+#endif
+
 /**
  * _scsih_is_pcie_scsi_device - determines if device is an pcie scsi device
  * @device_info: bitfield providing information about the device.
diff --git a/drivers/scsi/mpt3sas/mpt3sas_config.c b/drivers/scsi/mpt3sas/mpt3sas_config.c
index 45ac853e12892..b0d5ef8936008 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_config.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_config.c
@@ -991,6 +991,42 @@ mpt3sas_config_get_iounit_pg3(struct MPT3SAS_ADAPTER *ioc,
 	return r;
 }
 
+/**
+ * mpt3sas_config_get_iounit_pg7 - obtain iounit page 7
+ * @ioc: per adapter object
+ * @mpi_reply: reply mf payload returned from firmware
+ * @config_page: contents of the config page
+ * Context: sleep.
+ *
+ * Return: 0 for success, non-zero for failure.
+ */
+int mpt3sas_config_get_iounit_pg7(struct MPT3SAS_ADAPTER *ioc,
+				  Mpi2ConfigReply_t *mpi_reply,
+				  Mpi2IOUnitPage7_t *config_page)
+{
+	Mpi2ConfigRequest_t mpi_request;
+	int r;
+
+	memset(&mpi_request, 0, sizeof(Mpi2ConfigRequest_t));
+	mpi_request.Function = MPI2_FUNCTION_CONFIG;
+	mpi_request.Action = MPI2_CONFIG_ACTION_PAGE_HEADER;
+	mpi_request.Header.PageType = MPI2_CONFIG_PAGETYPE_IO_UNIT;
+	mpi_request.Header.PageNumber = 7;
+	mpi_request.Header.PageVersion = MPI2_IOUNITPAGE7_PAGEVERSION;
+	ioc->build_zero_len_sge_mpi(ioc, &mpi_request.PageBufferSGE);
+	r = _config_request(ioc, &mpi_request, mpi_reply,
+			    MPT3_CONFIG_PAGE_DEFAULT_TIMEOUT, NULL, 0);
+	if (r)
+		goto out;
+
+	mpi_request.Action = MPI2_CONFIG_ACTION_PAGE_READ_CURRENT;
+	r = _config_request(ioc, &mpi_request, mpi_reply,
+			    MPT3_CONFIG_PAGE_DEFAULT_TIMEOUT,
+			    config_page, sizeof(*config_page));
+ out:
+	return r;
+}
+
 /**
  * mpt3sas_config_get_iounit_pg8 - obtain iounit page 8
  * @ioc: per adapter object
diff --git a/drivers/scsi/mpt3sas/mpt3sas_hwmon.c b/drivers/scsi/mpt3sas/mpt3sas_hwmon.c
new file mode 100644
index 0000000000000..297a8efecadbd
--- /dev/null
+++ b/drivers/scsi/mpt3sas/mpt3sas_hwmon.c
@@ -0,0 +1,191 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Hardware monitoring (hwmon) support for the LSI / Broadcom mpt3sas
+ * SAS HBA driver. Exposes the IOC and board temperature sensors by
+ * reading MPI IO Unit Page 7.
+ */
+
+#include <linux/err.h>
+#include <linux/hwmon.h>
+#include <linux/slab.h>
+
+#include "mpt3sas_base.h"
+
+struct mpt3sas_hwmon {
+	struct MPT3SAS_ADAPTER *ioc;
+	struct device *hwmon_dev;
+	bool ioc_present;
+	bool board_present;
+};
+
+/*
+ * Convert a (raw, units) reading to millidegrees Celsius.
+ * Returns -ENODATA when the sensor reports "not present" or
+ * unknown units. Temperature values are interpreted as signed
+ * two's-complement integers.
+ *
+ * The MPI2_IOUNITPAGE7_IOC_TEMP_* and MPI2_IOUNITPAGE7_BOARD_TEMP_*
+ * defines in mpi2_cnfg.h share the same values; the IOC ones are
+ * used for both channels.
+ */
+static int _hwmon_to_mdegc(s16 raw, u8 units, long *out)
+{
+	switch (units) {
+	case MPI2_IOUNITPAGE7_IOC_TEMP_CELSIUS:
+		*out = (long)raw * 1000;
+		return 0;
+	case MPI2_IOUNITPAGE7_IOC_TEMP_FAHRENHEIT:
+		/* (F - 32) * 5 / 9, expressed in milli-units */
+		*out = ((long)raw - 32) * 5000 / 9;
+		return 0;
+	default:
+		return -ENODATA;
+	}
+}
+
+static umode_t _hwmon_is_visible(const void *drvdata,
+				 enum hwmon_sensor_types type,
+				 u32 attr, int channel)
+{
+	const struct mpt3sas_hwmon *h = drvdata;
+
+	if (type != hwmon_temp)
+		return 0;
+	if (attr != hwmon_temp_input && attr != hwmon_temp_label)
+		return 0;
+	if (channel == 0 && h->ioc_present)
+		return 0444;
+	if (channel == 1 && h->board_present)
+		return 0444;
+	return 0;
+}
+
+static int _hwmon_read(struct device *dev, enum hwmon_sensor_types type,
+		       u32 attr, int channel, long *val)
+{
+	struct mpt3sas_hwmon *h = dev_get_drvdata(dev);
+	Mpi2ConfigReply_t mpi_reply;
+	Mpi2IOUnitPage7_t page;
+	int r;
+
+	if (type != hwmon_temp || attr != hwmon_temp_input)
+		return -EOPNOTSUPP;
+
+	r = mpt3sas_config_get_iounit_pg7(h->ioc, &mpi_reply, &page);
+	if (r)
+		return r;
+
+	if (channel == 0)
+		return _hwmon_to_mdegc((s16)le16_to_cpu(page.IOCTemperature),
+				       page.IOCTemperatureUnits, val);
+	if (channel == 1)
+		return _hwmon_to_mdegc((s16)le16_to_cpu(page.BoardTemperature),
+				       page.BoardTemperatureUnits, val);
+	return -EOPNOTSUPP;
+}
+
+static const char * const mpt3sas_hwmon_temp_labels[] = {
+	"IOC",
+	"Board",
+};
+
+static int _hwmon_read_string(struct device *dev,
+			      enum hwmon_sensor_types type,
+			      u32 attr, int channel, const char **str)
+{
+	if (type != hwmon_temp || attr != hwmon_temp_label)
+		return -EOPNOTSUPP;
+	*str = mpt3sas_hwmon_temp_labels[channel];
+	return 0;
+}
+
+static const struct hwmon_channel_info * const mpt3sas_hwmon_info[] = {
+	HWMON_CHANNEL_INFO(temp,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL),
+	NULL,
+};
+
+static const struct hwmon_ops mpt3sas_hwmon_ops = {
+	.is_visible	= _hwmon_is_visible,
+	.read		= _hwmon_read,
+	.read_string	= _hwmon_read_string,
+};
+
+static const struct hwmon_chip_info mpt3sas_hwmon_chip_info = {
+	.ops	= &mpt3sas_hwmon_ops,
+	.info	= mpt3sas_hwmon_info,
+};
+
+/**
+ * mpt3sas_hwmon_register - register an hwmon device for the IOC
+ * @ioc: per adapter object
+ * Context: sleep.
+ *
+ * Succeeds without registering when no temperature sensors are present,
+ * so cards without thermal monitoring do not expose an empty hwmon node.
+ * Paired with mpt3sas_hwmon_unregister() from the driver's remove path.
+ *
+ * Return: 0 for success, non-zero for failure.
+ */
+int mpt3sas_hwmon_register(struct MPT3SAS_ADAPTER *ioc)
+{
+	struct device *parent = &ioc->pdev->dev;
+	struct mpt3sas_hwmon *h;
+	struct device *hwdev;
+	Mpi2ConfigReply_t mpi_reply;
+	Mpi2IOUnitPage7_t page;
+	int r;
+
+	r = mpt3sas_config_get_iounit_pg7(ioc, &mpi_reply, &page);
+	if (r)
+		return r;
+
+	/*
+	 * A page where both *TemperatureUnits are NOT_PRESENT covers
+	 * two cases: cards that genuinely lack sensors, and firmware
+	 * errors that left the page zero-filled (the accessor mirrors
+	 * _config_request() behaviour). Either way: skip registration.
+	 */
+	if (page.IOCTemperatureUnits == MPI2_IOUNITPAGE7_IOC_TEMP_NOT_PRESENT &&
+	    page.BoardTemperatureUnits == MPI2_IOUNITPAGE7_BOARD_TEMP_NOT_PRESENT)
+		return 0;
+
+	h = kzalloc_obj(*h);
+	if (!h)
+		return -ENOMEM;
+
+	h->ioc = ioc;
+	h->ioc_present = page.IOCTemperatureUnits != MPI2_IOUNITPAGE7_IOC_TEMP_NOT_PRESENT;
+	h->board_present = page.BoardTemperatureUnits != MPI2_IOUNITPAGE7_BOARD_TEMP_NOT_PRESENT;
+
+	hwdev = hwmon_device_register_with_info(parent, "mpt3sas", h,
+						&mpt3sas_hwmon_chip_info,
+						NULL);
+	if (IS_ERR(hwdev)) {
+		kfree(h);
+		return PTR_ERR(hwdev);
+	}
+
+	h->hwmon_dev = hwdev;
+	ioc->hwmon = h;
+	return 0;
+}
+
+/**
+ * mpt3sas_hwmon_unregister - tear down the hwmon device, if any
+ * @ioc: per adapter object
+ *
+ * Safe to call when registration was skipped (no sensors) or
+ * failed; in those cases ioc->hwmon is NULL and this is a no-op.
+ */
+void mpt3sas_hwmon_unregister(struct MPT3SAS_ADAPTER *ioc)
+{
+	struct mpt3sas_hwmon *h = ioc->hwmon;
+
+	if (!h)
+		return;
+	hwmon_device_unregister(h->hwmon_dev);
+	kfree(h);
+	ioc->hwmon = NULL;
+}
diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
index 12caffeed3a0d..dea78688cc9b5 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
@@ -12562,6 +12562,7 @@ static void scsih_remove(struct pci_dev *pdev)
 	/* release all the volumes */
 	_scsih_ir_shutdown(ioc);
 	mpt3sas_destroy_debugfs(ioc);
+	mpt3sas_hwmon_unregister(ioc);
 	sas_remove_host(shost);
 	list_for_each_entry_safe(raid_device, next, &ioc->raid_device_list,
 	    list) {
@@ -13651,6 +13652,11 @@ _scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	}
 
 	scsi_scan_host(shost);
+
+	if (mpt3sas_hwmon_register(ioc))
+		ioc_warn(ioc,
+			 "hwmon registration failed; temperatures not exposed\n");
+
 	mpt3sas_setup_debugfs(ioc);
 	return 0;
 out_add_shost_fail:
diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
index 72b1c28e4dae0..39ee33398df2d 100644
--- a/drivers/scsi/qla2xxx/qla_os.c
+++ b/drivers/scsi/qla2xxx/qla_os.c
@@ -2803,11 +2803,16 @@ qla2x00_set_isp_flags(struct qla_hw_data *ha)
 	else {
 		/* Get adapter physical port no from interrupt pin register. */
 		pci_read_config_byte(ha->pdev, PCI_INTERRUPT_PIN, &ha->port_no);
-		if (IS_QLA25XX(ha) || IS_QLA2031(ha) ||
-		    IS_QLA27XX(ha) || IS_QLA28XX(ha))
-			ha->port_no--;
-		else
-			ha->port_no = !(ha->port_no & 1);
+		if (ha->port_no == 0) {
+			/* None of INT[A|B|C|D] may be virtualized by vfio */
+			ha->port_no = PCI_FUNC(ha->pdev->devfn);
+		} else {
+			if (IS_QLA25XX(ha) || IS_QLA2031(ha) ||
+			    IS_QLA27XX(ha) || IS_QLA28XX(ha))
+				ha->port_no--;
+			else
+				ha->port_no = !(ha->port_no & 1);
+		}
 	}
 
 	ql_dbg_pci(ql_dbg_init, ha->pdev, 0x000b,
diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c
index 9d1c9c41d0f99..4a95e6bae38b0 100644
--- a/drivers/scsi/scsi_debug.c
+++ b/drivers/scsi/scsi_debug.c
@@ -67,52 +67,6 @@ static const char *sdebug_version_date = "20210520";
 
 #define MY_NAME "scsi_debug"
 
-/* Additional Sense Code (ASC) */
-#define NO_ADDITIONAL_SENSE 0x0
-#define OVERLAP_ATOMIC_COMMAND_ASC 0x0
-#define OVERLAP_ATOMIC_COMMAND_ASCQ 0x23
-#define FILEMARK_DETECTED_ASCQ 0x1
-#define EOP_EOM_DETECTED_ASCQ 0x2
-#define BEGINNING_OF_P_M_DETECTED_ASCQ 0x4
-#define EOD_DETECTED_ASCQ 0x5
-#define LOGICAL_UNIT_NOT_READY 0x4
-#define LOGICAL_UNIT_COMMUNICATION_FAILURE 0x8
-#define UNRECOVERED_READ_ERR 0x11
-#define PARAMETER_LIST_LENGTH_ERR 0x1a
-#define INVALID_OPCODE 0x20
-#define LBA_OUT_OF_RANGE 0x21
-#define INVALID_FIELD_IN_CDB 0x24
-#define INVALID_FIELD_IN_PARAM_LIST 0x26
-#define WRITE_PROTECTED 0x27
-#define UA_READY_ASC 0x28
-#define UA_RESET_ASC 0x29
-#define UA_CHANGED_ASC 0x2a
-#define TOO_MANY_IN_PARTITION_ASC 0x3b
-#define TARGET_CHANGED_ASC 0x3f
-#define LUNS_CHANGED_ASCQ 0x0e
-#define INSUFF_RES_ASC 0x55
-#define INSUFF_RES_ASCQ 0x3
-#define POWER_ON_RESET_ASCQ 0x0
-#define POWER_ON_OCCURRED_ASCQ 0x1
-#define BUS_RESET_ASCQ 0x2	/* scsi bus reset occurred */
-#define MODE_CHANGED_ASCQ 0x1	/* mode parameters changed */
-#define CAPACITY_CHANGED_ASCQ 0x9
-#define SAVING_PARAMS_UNSUP 0x39
-#define TRANSPORT_PROBLEM 0x4b
-#define THRESHOLD_EXCEEDED 0x5d
-#define LOW_POWER_COND_ON 0x5e
-#define MISCOMPARE_VERIFY_ASC 0x1d
-#define MICROCODE_CHANGED_ASCQ 0x1	/* with TARGET_CHANGED_ASC */
-#define MICROCODE_CHANGED_WO_RESET_ASCQ 0x16
-#define WRITE_ERROR_ASC 0xc
-#define UNALIGNED_WRITE_ASCQ 0x4
-#define WRITE_BOUNDARY_ASCQ 0x5
-#define READ_INVDATA_ASCQ 0x6
-#define READ_BOUNDARY_ASCQ 0x7
-#define ATTEMPT_ACCESS_GAP 0x9
-#define INSUFF_ZONE_ASCQ 0xe
-/* see drivers/scsi/sense_codes.h */
-
 /* Additional Sense Code Qualifier (ASCQ) */
 #define ACK_NAK_TO 0x3
 
diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
index 147127fb4db9c..453a2232452db 100644
--- a/drivers/scsi/scsi_error.c
+++ b/drivers/scsi/scsi_error.c
@@ -73,6 +73,26 @@ void scsi_eh_wakeup(struct Scsi_Host *shost, unsigned int busy)
 	}
 }
 
+void scsi_rcu_eh_wakeup(struct work_struct *work)
+{
+	struct Scsi_Host *shost = container_of(work, struct Scsi_Host, eh_work);
+	unsigned long flags;
+	unsigned int busy;
+
+	/*
+	 * Ensure any running scsi_dec_host_busy has completed its rcu section
+	 * so changes to host state and host_eh_scheduled are visible to all
+	 * future calls of scsi_dec_host_busy
+	 */
+	synchronize_rcu();
+
+	busy = scsi_host_busy(shost);
+
+	spin_lock_irqsave(shost->host_lock, flags);
+	scsi_eh_wakeup(shost, busy);
+	spin_unlock_irqrestore(shost->host_lock, flags);
+}
+
 /**
  * scsi_schedule_eh - schedule EH for SCSI host
  * @shost:	SCSI host to invoke error handling on.
@@ -88,7 +108,7 @@ void scsi_schedule_eh(struct Scsi_Host *shost)
 	if (scsi_host_set_state(shost, SHOST_RECOVERY) == 0 ||
 	    scsi_host_set_state(shost, SHOST_CANCEL_RECOVERY) == 0) {
 		shost->host_eh_scheduled++;
-		scsi_eh_wakeup(shost, scsi_host_busy(shost));
+		queue_work(shost->tmf_work_q, &shost->eh_work);
 	}
 
 	spin_unlock_irqrestore(shost->host_lock, flags);
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 22e2e3223440d..daeb3693fe559 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -2224,14 +2224,6 @@ struct scsi_device *scsi_device_from_queue(struct request_queue *q)
 
 	return sdev;
 }
-/*
- * pktcdvd should have been integrated into the SCSI layers, but for historical
- * reasons like the old IDE driver it isn't.  This export allows it to safely
- * probe if a given device is a SCSI one and only attach to that.
- */
-#ifdef CONFIG_CDROM_PKTCDVD_MODULE
-EXPORT_SYMBOL_GPL(scsi_device_from_queue);
-#endif
 
 /**
  * scsi_block_requests - Utility function used by low-level drivers to prevent
diff --git a/drivers/scsi/scsi_priv.h b/drivers/scsi/scsi_priv.h
index 7a193cc04e5b6..3dbf2ca59536c 100644
--- a/drivers/scsi/scsi_priv.h
+++ b/drivers/scsi/scsi_priv.h
@@ -91,6 +91,7 @@ extern enum blk_eh_timer_return scsi_timeout(struct request *req);
 extern int scsi_error_handler(void *host);
 extern enum scsi_disposition scsi_decide_disposition(struct scsi_cmnd *cmd);
 extern void scsi_eh_wakeup(struct Scsi_Host *shost, unsigned int busy);
+extern void scsi_rcu_eh_wakeup(struct work_struct *work);
 extern void scsi_eh_scmd_add(struct scsi_cmnd *);
 void scsi_eh_ready_devs(struct Scsi_Host *shost,
 			struct list_head *work_q,
@@ -102,6 +103,7 @@ void scsi_eh_done(struct scsi_cmnd *scmd);
 
 /* scsi_lib.c */
 extern void scsi_device_unbusy(struct scsi_device *sdev, struct scsi_cmnd *cmd);
+extern struct scsi_device *scsi_device_from_queue(struct request_queue *q);
 extern void scsi_queue_insert(struct scsi_cmnd *cmd,
 			      enum scsi_qc_status reason);
 extern void scsi_io_completion(struct scsi_cmnd *, unsigned int);
diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c
index 74cd4e8a61c2a..5408f002e6c01 100644
--- a/drivers/scsi/sg.c
+++ b/drivers/scsi/sg.c
@@ -863,10 +863,9 @@ sg_fill_request_table(Sg_fd *sfp, sg_req_info_t *rinfo)
 		if (val >= SG_MAX_QUEUE)
 			break;
 		rinfo[val].req_state = srp->done + 1;
-		rinfo[val].problem =
-			srp->header.masked_status &
-			srp->header.host_status &
-			srp->header.driver_status;
+		rinfo[val].problem = srp->header.masked_status ||
+					     srp->header.host_status ||
+					     srp->header.driver_status;
 		if (srp->done)
 			rinfo[val].duration =
 				srp->header.duration;
diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c
index 5fdaa71f06528..35731b18c519e 100644
--- a/drivers/scsi/virtio_scsi.c
+++ b/drivers/scsi/virtio_scsi.c
@@ -122,10 +122,11 @@ static void virtscsi_complete_cmd(struct virtio_scsi *vscsi, void *buf)
 	struct virtio_scsi_cmd *cmd = buf;
 	struct scsi_cmnd *sc = cmd->sc;
 	struct virtio_scsi_cmd_resp *resp = &cmd->resp.cmd;
+	unsigned sense_len = virtio32_to_cpu(vscsi->vdev, resp->sense_len);
 
 	dev_dbg(&sc->device->sdev_gendev,
 		"cmd %p response %u status %#02x sense_len %u\n",
-		sc, resp->response, resp->status, resp->sense_len);
+		sc, resp->response, resp->status, sense_len);
 
 	sc->result = resp->status;
 	virtscsi_compute_resid(sc, virtio32_to_cpu(vscsi->vdev, resp->resid));
@@ -166,13 +167,10 @@ static void virtscsi_complete_cmd(struct virtio_scsi *vscsi, void *buf)
 		break;
 	}
 
-	WARN_ON(virtio32_to_cpu(vscsi->vdev, resp->sense_len) >
-		VIRTIO_SCSI_SENSE_SIZE);
+	WARN_ON(sense_len > VIRTIO_SCSI_SENSE_SIZE);
 	if (resp->sense_len) {
 		memcpy(sc->sense_buffer, resp->sense,
-		       min_t(u32,
-			     virtio32_to_cpu(vscsi->vdev, resp->sense_len),
-			     VIRTIO_SCSI_SENSE_SIZE));
+		       min_t(u32, sense_len, VIRTIO_SCSI_SENSE_SIZE));
 	}
 
 	scsi_done(sc);
@@ -288,8 +286,9 @@ static void virtscsi_handle_transport_reset(struct virtio_scsi *vscsi,
 	struct Scsi_Host *shost = virtio_scsi_host(vscsi->vdev);
 	unsigned int target = event->lun[1];
 	unsigned int lun = (event->lun[2] << 8) | event->lun[3];
+	unsigned int reason = virtio32_to_cpu(vscsi->vdev, event->reason);
 
-	switch (virtio32_to_cpu(vscsi->vdev, event->reason)) {
+	switch (reason) {
 	case VIRTIO_SCSI_EVT_RESET_RESCAN:
 		if (lun == 0) {
 			scsi_scan_target(&shost->shost_gendev, 0, target,
@@ -309,7 +308,7 @@ static void virtscsi_handle_transport_reset(struct virtio_scsi *vscsi,
 		}
 		break;
 	default:
-		pr_info("Unsupported virtio scsi event reason %x\n", event->reason);
+		pr_info("Unsupported virtio scsi event reason %x\n", reason);
 	}
 }
 
@@ -409,7 +408,8 @@ static void virtscsi_handle_event(struct work_struct *work)
 		virtscsi_handle_param_change(vscsi, event);
 		break;
 	default:
-		pr_err("Unsupported virtio scsi event %x\n", event->event);
+		pr_err("Unsupported virtio scsi event %x\n",
+		       virtio32_to_cpu(vscsi->vdev, event->event));
 	}
 	virtscsi_kick_event(vscsi, event_node);
 }
diff --git a/drivers/slimbus/qcom-ngd-ctrl.c b/drivers/slimbus/qcom-ngd-ctrl.c
index 3071e46d03bea..934c44c5bc1a9 100644
--- a/drivers/slimbus/qcom-ngd-ctrl.c
+++ b/drivers/slimbus/qcom-ngd-ctrl.c
@@ -48,7 +48,6 @@
 				NGD_INT_RX_MSG_RCVD)
 
 /* Slimbus QMI service */
-#define SLIMBUS_QMI_SVC_ID	0x0301
 #define SLIMBUS_QMI_SVC_V1	1
 #define SLIMBUS_QMI_INS_ID	0
 #define SLIMBUS_QMI_SELECT_INSTANCE_REQ_V01	0x0020
@@ -1408,8 +1407,8 @@ static int qcom_slim_ngd_qmi_svc_event_init(struct qcom_slim_ngd_ctrl *ctrl)
 		return ret;
 	}
 
-	ret = qmi_add_lookup(&qmi->svc_event_hdl, SLIMBUS_QMI_SVC_ID,
-			SLIMBUS_QMI_SVC_V1, SLIMBUS_QMI_INS_ID);
+	ret = qmi_add_lookup(&qmi->svc_event_hdl, QMI_SERVICE_ID_SLIMBUS,
+			     SLIMBUS_QMI_SVC_V1, SLIMBUS_QMI_INS_ID);
 	if (ret < 0) {
 		dev_err(ctrl->dev, "qmi_add_lookup failed: %d\n", ret);
 		qmi_handle_release(&qmi->svc_event_hdl);
@@ -1539,7 +1538,7 @@ static int of_qcom_slim_ngd_register(struct device *parent,
 			kfree(ngd);
 			return ret;
 		}
-		ngd->pdev->dev.of_node = of_node_get(node);
+		platform_device_set_of_node(ngd->pdev, node);
 		ctrl->ngd = ngd;
 
 		ret = platform_device_add(ngd->pdev);
diff --git a/drivers/soc/amlogic/meson-clk-measure.c b/drivers/soc/amlogic/meson-clk-measure.c
index d862e30a244ed..8c4f3cc8c8abf 100644
--- a/drivers/soc/amlogic/meson-clk-measure.c
+++ b/drivers/soc/amlogic/meson-clk-measure.c
@@ -787,6 +787,258 @@ static const struct meson_msr_id clk_msr_s4[] = {
 
 };
 
+static const struct meson_msr_id clk_msr_a1[] = {
+	CLK_MSR_ID(0, "tdmout_b_sclk"),
+	CLK_MSR_ID(1, "tdmout_a_sclk"),
+	CLK_MSR_ID(2, "tdmin_lb_sclk"),
+	CLK_MSR_ID(3, "tdmin_b_sclk"),
+	CLK_MSR_ID(4, "tdmin_a_sclk"),
+	CLK_MSR_ID(5, "vad"),
+	CLK_MSR_ID(6, "resamplea"),
+	CLK_MSR_ID(7, "pdm_sysclk"),
+	CLK_MSR_ID(8, "pdm_dclk"),
+	CLK_MSR_ID(9, "locker_out"),
+	CLK_MSR_ID(10, "locker_in"),
+	CLK_MSR_ID(11, "spdifin"),
+	CLK_MSR_ID(12, "tdmin_vad"),
+	CLK_MSR_ID(13, "au_adc"),
+	CLK_MSR_ID(14, "au_dac"),
+	CLK_MSR_ID(16, "spicc_a"),
+	CLK_MSR_ID(17, "spifc"),
+	CLK_MSR_ID(18, "sd_emmc_a"),
+	CLK_MSR_ID(19, "dmcx4"),
+	CLK_MSR_ID(20, "dmc"),
+	CLK_MSR_ID(21, "psram"),
+	CLK_MSR_ID(22, "cecb"),
+	CLK_MSR_ID(23, "ceca"),
+	CLK_MSR_ID(24, "ts"),
+	CLK_MSR_ID(25, "pwm_f"),
+	CLK_MSR_ID(26, "pwm_e"),
+	CLK_MSR_ID(27, "pwm_d"),
+	CLK_MSR_ID(28, "pwm_c"),
+	CLK_MSR_ID(29, "pwm_b"),
+	CLK_MSR_ID(30, "pwm_a"),
+	CLK_MSR_ID(31, "saradc"),
+	CLK_MSR_ID(32, "usb_bus"),
+	CLK_MSR_ID(33, "dsp_b"),
+	CLK_MSR_ID(34, "dsp_a"),
+	CLK_MSR_ID(35, "axi"),
+	CLK_MSR_ID(36, "sys"),
+	CLK_MSR_ID(40, "rng_ring_osc0"),
+	CLK_MSR_ID(41, "rng_ring_osc1"),
+	CLK_MSR_ID(42, "rng_ring_osc2"),
+	CLK_MSR_ID(43, "rng_ring_osc3"),
+	CLK_MSR_ID(44, "dds_out"),
+	CLK_MSR_ID(45, "cpu_clk_div16"),
+	CLK_MSR_ID(46, "gpio_msr"),
+	CLK_MSR_ID(50, "osc_ring_cpu0"),
+	CLK_MSR_ID(51, "osc_ring_cpu1"),
+	CLK_MSR_ID(54, "osc_ring_top0"),
+	CLK_MSR_ID(55, "osc_ring_top1"),
+	CLK_MSR_ID(56, "osc_ring_ddr"),
+	CLK_MSR_ID(57, "osc_ring_dmc"),
+	CLK_MSR_ID(58, "osc_ring_dspa"),
+	CLK_MSR_ID(59, "osc_ring_dspb"),
+	CLK_MSR_ID(60, "osc_ring_rama"),
+	CLK_MSR_ID(61, "osc_ring_ramb"),
+};
+
+static const struct meson_msr_id clk_msr_t7[] = {
+	CLK_MSR_ID(0, "sys"),
+	CLK_MSR_ID(1, "axi"),
+	CLK_MSR_ID(2, "rtc"),
+	CLK_MSR_ID(3, "dspa"),
+	CLK_MSR_ID(4, "dspb"),
+	CLK_MSR_ID(5, "mali"),
+	CLK_MSR_ID(6, "sys_cpu_clk_div16"),
+	CLK_MSR_ID(7, "ceca"),
+	CLK_MSR_ID(8, "cecb"),
+	CLK_MSR_ID(10, "fclk_div5"),
+	CLK_MSR_ID(11, "mpll0"),
+	CLK_MSR_ID(12, "mpll1"),
+	CLK_MSR_ID(13, "mpll2"),
+	CLK_MSR_ID(14, "mpll3"),
+	CLK_MSR_ID(15, "mpll_50m"),
+	CLK_MSR_ID(16, "pcie_inp"),
+	CLK_MSR_ID(17, "pcie_inn"),
+	CLK_MSR_ID(18, "mpll_test_out"),
+	CLK_MSR_ID(19, "hifi_pll"),
+	CLK_MSR_ID(20, "gp0_pll"),
+	CLK_MSR_ID(21, "gp1_pll"),
+	CLK_MSR_ID(22, "eth_mppll_50m"),
+	CLK_MSR_ID(23, "sys_pll_div16"),
+	CLK_MSR_ID(24, "ddr_dpll_pt"),
+	CLK_MSR_ID(25, "earcrx_pll"),
+	CLK_MSR_ID(26, "paie1_clk_inp"),
+	CLK_MSR_ID(27, "paie1_clk_inn"),
+	CLK_MSR_ID(28, "amlgdc"),
+	CLK_MSR_ID(29, "gdc"),
+	CLK_MSR_ID(30, "mod_eth_phy_ref"),
+	CLK_MSR_ID(31, "mod_eth_tx"),
+	CLK_MSR_ID(32, "eth_clk125Mhz"),
+	CLK_MSR_ID(33, "eth_clk_rmii"),
+	CLK_MSR_ID(34, "co_clkin_to_mac"),
+	CLK_MSR_ID(35, "mod_eth_rx_clk_rmii"),
+	CLK_MSR_ID(36, "co_rx"),
+	CLK_MSR_ID(37, "co_tx"),
+	CLK_MSR_ID(38, "eth_phy_rxclk"),
+	CLK_MSR_ID(39, "eth_phy_plltxclk"),
+	CLK_MSR_ID(40, "ephy_test"),
+	CLK_MSR_ID(41, "dsi_b_meas"),
+	CLK_MSR_ID(42, "hdmirx_apl"),
+	CLK_MSR_ID(43, "hdmirx_tmds"),
+	CLK_MSR_ID(44, "hdmirx_cable"),
+	CLK_MSR_ID(45, "hdmirx_apll_clk_audio"),
+	CLK_MSR_ID(46, "hdmirx_5m"),
+	CLK_MSR_ID(47, "hdmirx_2m"),
+	CLK_MSR_ID(48, "hdmirx_cfg"),
+	CLK_MSR_ID(49, "hdmirx_hdcp2x_eclk"),
+	CLK_MSR_ID(50, "vid_pll0_div"),
+	CLK_MSR_ID(51, "hdmi_vid_pll"),
+	CLK_MSR_ID(54, "vdac_clk"),
+	CLK_MSR_ID(55, "vpu_clk_buf"),
+	CLK_MSR_ID(56, "mod_tcon_clko"),
+	CLK_MSR_ID(57, "lcd_an_clk_ph2"),
+	CLK_MSR_ID(58, "lcd_an_clk_ph3"),
+	CLK_MSR_ID(59, "hdmi_tx_pixel"),
+	CLK_MSR_ID(60, "vdin_meas"),
+	CLK_MSR_ID(61, "vpu_clk"),
+	CLK_MSR_ID(62, "vpu_clkb"),
+	CLK_MSR_ID(63, "vpu_clkb_tmp"),
+	CLK_MSR_ID(64, "vpu_clkc"),
+	CLK_MSR_ID(65, "vid_lock"),
+	CLK_MSR_ID(66, "vapbclk"),
+	CLK_MSR_ID(67, "ge2d"),
+	CLK_MSR_ID(68, "aud_pll"),
+	CLK_MSR_ID(69, "aud_sck"),
+	CLK_MSR_ID(70, "dsi_a_meas"),
+	CLK_MSR_ID(72, "mipi_csi_phy"),
+	CLK_MSR_ID(73, "mipi_isp"),
+	CLK_MSR_ID(76, "hdmitx_tmds"),
+	CLK_MSR_ID(77, "hdmitx_sys"),
+	CLK_MSR_ID(78, "hdmitx_fe"),
+	CLK_MSR_ID(80, "hdmitx_prif"),
+	CLK_MSR_ID(81, "hdmitx_200m"),
+	CLK_MSR_ID(82, "hdmitx_aud"),
+	CLK_MSR_ID(83, "hdmitx_pnx"),
+	CLK_MSR_ID(84, "spicc5"),
+	CLK_MSR_ID(85, "spicc4"),
+	CLK_MSR_ID(86, "spicc3"),
+	CLK_MSR_ID(87, "spicc2"),
+	CLK_MSR_ID(93, "vdec"),
+	CLK_MSR_ID(94, "wave521_aclk"),
+	CLK_MSR_ID(95, "wave521_cclk"),
+	CLK_MSR_ID(96, "wave521_bclk"),
+	CLK_MSR_ID(97, "hcodec"),
+	CLK_MSR_ID(98, "hevcb"),
+	CLK_MSR_ID(99, "hevcf"),
+	CLK_MSR_ID(100, "hdmi_aud_pll"),
+	CLK_MSR_ID(101, "hdmi_acr_ref"),
+	CLK_MSR_ID(102, "hdmi_meter"),
+	CLK_MSR_ID(103, "hdmi_vid"),
+	CLK_MSR_ID(104, "hdmi_aud"),
+	CLK_MSR_ID(105, "hdmi_dsd"),
+	CLK_MSR_ID(108, "dsi1_phy"),
+	CLK_MSR_ID(109, "dsi0_phy"),
+	CLK_MSR_ID(110, "smartcard"),
+	CLK_MSR_ID(111, "sar_adc"),
+	CLK_MSR_ID(113, "sd_emmc_c"),
+	CLK_MSR_ID(114, "sd_emmc_b"),
+	CLK_MSR_ID(115, "sd_emmc_a"),
+	CLK_MSR_ID(116, "gpio_msr"),
+	CLK_MSR_ID(117, "spicc1"),
+	CLK_MSR_ID(118, "spicc0"),
+	CLK_MSR_ID(119, "anakin"),
+	CLK_MSR_ID(121, "ts_clk(temp sensor)"),
+	CLK_MSR_ID(122, "ts_a73"),
+	CLK_MSR_ID(123, "ts_a53"),
+	CLK_MSR_ID(124, "ts_nna"),
+	CLK_MSR_ID(130, "audio_vad"),
+	CLK_MSR_ID(131, "acodec_dac_clk_x128"),
+	CLK_MSR_ID(132, "audio_locker_in"),
+	CLK_MSR_ID(133, "audio_locker_out"),
+	CLK_MSR_ID(134, "audio_tdmout_c_sclk"),
+	CLK_MSR_ID(135, "audio_tdmout_b_sclk"),
+	CLK_MSR_ID(136, "audio_tdmout_a_sclk"),
+	CLK_MSR_ID(137, "audio_tdmin_lb_sclk"),
+	CLK_MSR_ID(138, "audio_tdmin_c_sclk"),
+	CLK_MSR_ID(139, "audio_tdmin_b_sclk"),
+	CLK_MSR_ID(140, "audio_tdmin_a_sclk"),
+	CLK_MSR_ID(141, "audio_resamplea"),
+	CLK_MSR_ID(142, "audio_pdm_sysclk"),
+	CLK_MSR_ID(143, "audio_spdifoutb_mst"),
+	CLK_MSR_ID(144, "audio_spdifout_mst"),
+	CLK_MSR_ID(145, "audio_spdifin_mst"),
+	CLK_MSR_ID(146, "audio_pdm_dclk"),
+	CLK_MSR_ID(147, "audio_resampleb"),
+	CLK_MSR_ID(148, "earcrx_pll_dmac"),
+	CLK_MSR_ID(156, "pwm_ao_h"),
+	CLK_MSR_ID(157, "pwm_ao_g"),
+	CLK_MSR_ID(158, "pwm_ao_f"),
+	CLK_MSR_ID(159, "pwm_ao_e"),
+	CLK_MSR_ID(160, "pwm_ao_d"),
+	CLK_MSR_ID(161, "pwm_ao_c"),
+	CLK_MSR_ID(162, "pwm_ao_b"),
+	CLK_MSR_ID(163, "pwm_ao_a"),
+	CLK_MSR_ID(164, "pwm_f"),
+	CLK_MSR_ID(165, "pwm_e"),
+	CLK_MSR_ID(166, "pwm_d"),
+	CLK_MSR_ID(167, "pwm_c"),
+	CLK_MSR_ID(168, "pwm_b"),
+	CLK_MSR_ID(169, "pwm_a"),
+	CLK_MSR_ID(170, "aclkm"),
+	CLK_MSR_ID(171, "mclk_pll"),
+	CLK_MSR_ID(172, "a73_sys_pll_div16"),
+	CLK_MSR_ID(173, "a73_cpu_clk_div16"),
+	CLK_MSR_ID(176, "rng_ring_0"),
+	CLK_MSR_ID(177, "rng_ring_1"),
+	CLK_MSR_ID(178, "rng_ring_2"),
+	CLK_MSR_ID(179, "rng_ring_3"),
+	CLK_MSR_ID(180, "am_ring_out0"),
+	CLK_MSR_ID(181, "am_ring_out1"),
+	CLK_MSR_ID(182, "am_ring_out2"),
+	CLK_MSR_ID(183, "am_ring_out3"),
+	CLK_MSR_ID(184, "am_ring_out4"),
+	CLK_MSR_ID(185, "am_ring_out5"),
+	CLK_MSR_ID(186, "am_ring_out6"),
+	CLK_MSR_ID(187, "am_ring_out7"),
+	CLK_MSR_ID(188, "am_ring_out8"),
+	CLK_MSR_ID(189, "am_ring_out9"),
+	CLK_MSR_ID(190, "am_ring_out10"),
+	CLK_MSR_ID(191, "am_ring_out11"),
+	CLK_MSR_ID(192, "am_ring_out12"),
+	CLK_MSR_ID(193, "am_ring_out13"),
+	CLK_MSR_ID(194, "am_ring_out14"),
+	CLK_MSR_ID(195, "am_ring_out15"),
+	CLK_MSR_ID(196, "am_ring_out16"),
+	CLK_MSR_ID(197, "am_ring_out17"),
+	CLK_MSR_ID(198, "am_ring_out18"),
+	CLK_MSR_ID(199, "am_ring_out19"),
+	CLK_MSR_ID(200, "mipi_csi_phy0"),
+	CLK_MSR_ID(201, "mipi_csi_phy1"),
+	CLK_MSR_ID(202, "mipi_csi_phy2"),
+	CLK_MSR_ID(203, "mipi_csi_phy3"),
+	CLK_MSR_ID(204, "vid_pll1_div"),
+	CLK_MSR_ID(205, "vid_pll2_div"),
+	CLK_MSR_ID(206, "am_ring_out20"),
+	CLK_MSR_ID(207, "am_ring_out21"),
+	CLK_MSR_ID(208, "am_ring_out22"),
+	CLK_MSR_ID(209, "am_ring_out23"),
+	CLK_MSR_ID(210, "am_ring_out24"),
+	CLK_MSR_ID(211, "am_ring_out25"),
+	CLK_MSR_ID(212, "am_ring_out26"),
+	CLK_MSR_ID(213, "am_ring_out27"),
+	CLK_MSR_ID(214, "am_ring_out28"),
+	CLK_MSR_ID(215, "am_ring_out29"),
+	CLK_MSR_ID(216, "am_ring_out30"),
+	CLK_MSR_ID(217, "am_ring_out31"),
+	CLK_MSR_ID(218, "am_ring_out32"),
+	CLK_MSR_ID(219, "enc0_if"),
+	CLK_MSR_ID(220, "enc2"),
+	CLK_MSR_ID(221, "enc1"),
+	CLK_MSR_ID(222, "enc0")
+};
+
 static int meson_measure_id(struct meson_msr_id *clk_msr_id,
 			    unsigned int duration)
 {
@@ -1026,6 +1278,18 @@ static const struct meson_msr_data clk_msr_s4_data = {
 	.reg = &msr_reg_offset_v2,
 };
 
+static const struct meson_msr_data clk_msr_a1_data = {
+	.msr_table = (void *)clk_msr_a1,
+	.msr_count = ARRAY_SIZE(clk_msr_a1),
+	.reg = &msr_reg_offset_v2,
+};
+
+static const struct meson_msr_data clk_msr_t7_data = {
+	.msr_table = (void *)clk_msr_t7,
+	.msr_count = ARRAY_SIZE(clk_msr_t7),
+	.reg = &msr_reg_offset_v2,
+};
+
 static const struct of_device_id meson_msr_match_table[] = {
 	{
 		.compatible = "amlogic,meson-gx-clk-measure",
@@ -1059,6 +1323,14 @@ static const struct of_device_id meson_msr_match_table[] = {
 		.compatible = "amlogic,s4-clk-measure",
 		.data = &clk_msr_s4_data,
 	},
+	{
+		.compatible = "amlogic,a1-clk-measure",
+		.data = &clk_msr_a1_data,
+	},
+	{
+		.compatible = "amlogic,t7-clk-measure",
+		.data = &clk_msr_t7_data,
+	},
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, meson_msr_match_table);
diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig
index d0ff32182a2b4..db30c0ade948c 100644
--- a/drivers/soc/apple/Kconfig
+++ b/drivers/soc/apple/Kconfig
@@ -16,6 +16,16 @@ config APPLE_MAILBOX
 
 	  Say Y here if you have an Apple SoC.
 
+config APPLE_PMGR_MISC
+	tristate "Apple SoC PMGR miscellaneous support"
+	depends on PM
+	help
+	  The PMGR block in Apple SoCs provides high-level power state
+	  controls for SoC devices. This driver manages miscellaneous
+	  power controls.
+
+	  Say 'y' here if you have a Pro/Max/Ultra Apple SoC.
+
 config APPLE_RTKIT
 	tristate "Apple RTKit co-processor IPC protocol"
 	depends on APPLE_MAILBOX
diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile
index 0b85ab61aefe1..0481f7f43b063 100644
--- a/drivers/soc/apple/Makefile
+++ b/drivers/soc/apple/Makefile
@@ -3,6 +3,8 @@
 obj-$(CONFIG_APPLE_MAILBOX) += apple-mailbox.o
 apple-mailbox-y = mailbox.o
 
+obj-$(CONFIG_APPLE_PMGR_MISC)	+= apple-pmgr-misc.o
+
 obj-$(CONFIG_APPLE_RTKIT) += apple-rtkit.o
 apple-rtkit-y = rtkit.o rtkit-crashlog.o
 
diff --git a/drivers/soc/apple/apple-pmgr-misc.c b/drivers/soc/apple/apple-pmgr-misc.c
new file mode 100644
index 0000000000000..7db29f7de3294
--- /dev/null
+++ b/drivers/soc/apple/apple-pmgr-misc.c
@@ -0,0 +1,181 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Apple SoC PMGR device power state driver
+ *
+ * Copyright The Asahi Linux Contributors
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+
+#define APPLE_CLKGEN_PSTATE 0
+#define APPLE_CLKGEN_PSTATE_DESIRED GENMASK(3, 0)
+
+#define DCS_DEV_PSTATE_MIN_T600X 7
+#define SYS_DEV_PSTATE_SUSPEND 1
+
+enum sys_device {
+	DEV_FABRIC,
+	DEV_DCS,
+	DEV_MAX,
+};
+
+struct apple_pmgr_sys_device {
+	void __iomem *base;
+	u32 active_state;
+	u32 suspend_state;
+};
+
+struct apple_pmgr_misc_hw {
+	u32 dev_min_ps[DEV_MAX];
+};
+
+struct apple_pmgr_misc {
+	struct device *dev;
+	struct apple_pmgr_sys_device devices[DEV_MAX];
+};
+
+static void apple_pmgr_sys_dev_set_pstate(struct apple_pmgr_misc *misc,
+					  enum sys_device dev, bool active)
+{
+	u32 pstate;
+	u32 val;
+
+	if (!misc->devices[dev].base)
+		return;
+
+	if (active)
+		pstate = misc->devices[dev].active_state;
+	else
+		pstate = misc->devices[dev].suspend_state;
+
+	dev_dbg(misc->dev, "set %d ps to pstate %d\n", dev, pstate);
+
+	val = readl_relaxed(misc->devices[dev].base + APPLE_CLKGEN_PSTATE);
+	FIELD_MODIFY(APPLE_CLKGEN_PSTATE_DESIRED, &val, pstate);
+	writel_relaxed(val, misc->devices[dev].base + APPLE_CLKGEN_PSTATE);
+}
+
+static int __maybe_unused apple_pmgr_misc_suspend_noirq(struct device *dev)
+{
+	struct apple_pmgr_misc *misc = dev_get_drvdata(dev);
+	int i;
+
+	for (i = 0; i < DEV_MAX; i++)
+		apple_pmgr_sys_dev_set_pstate(misc, i, false);
+
+	return 0;
+}
+
+static int __maybe_unused apple_pmgr_misc_resume_noirq(struct device *dev)
+{
+	struct apple_pmgr_misc *misc = dev_get_drvdata(dev);
+	int i;
+
+	for (i = 0; i < DEV_MAX; i++)
+		apple_pmgr_sys_dev_set_pstate(misc, i, true);
+
+	return 0;
+}
+
+static int apple_pmgr_init_device(struct apple_pmgr_misc *misc,
+				  const struct apple_pmgr_misc_hw *hw,
+				  enum sys_device dev,
+				  const char *device_name)
+{
+	void __iomem *base;
+	char name[32];
+	u32 val;
+
+	snprintf(name, sizeof(name), "%s-ps", device_name);
+
+	base = devm_platform_ioremap_resource_byname(
+		to_platform_device(misc->dev), name);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	val = readl_relaxed(base + APPLE_CLKGEN_PSTATE);
+
+	misc->devices[dev].base = base;
+	misc->devices[dev].active_state =
+		FIELD_GET(APPLE_CLKGEN_PSTATE_DESIRED, val);
+	misc->devices[dev].suspend_state = hw->dev_min_ps[dev];
+
+	return 0;
+}
+
+static int apple_pmgr_misc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	const struct apple_pmgr_misc_hw *hw;
+	struct apple_pmgr_misc *misc;
+	int ret;
+
+	misc = devm_kzalloc(dev, sizeof(*misc), GFP_KERNEL);
+	if (!misc)
+		return -ENOMEM;
+
+	misc->dev = dev;
+	hw = of_device_get_match_data(dev);
+	if (!hw)
+		return -EINVAL;
+
+	ret = apple_pmgr_init_device(misc, hw, DEV_FABRIC, "fabric");
+	if (ret)
+		return ret;
+
+	ret = apple_pmgr_init_device(misc, hw, DEV_DCS, "dcs");
+	if (ret)
+		return ret;
+
+	platform_set_drvdata(pdev, misc);
+
+	return 0;
+}
+
+static const struct apple_pmgr_misc_hw apple_pmgr_misc_hw_t600x = {
+	.dev_min_ps = {
+		[DEV_FABRIC] = SYS_DEV_PSTATE_SUSPEND,
+		[DEV_DCS] = DCS_DEV_PSTATE_MIN_T600X,
+	},
+};
+
+static const struct apple_pmgr_misc_hw apple_pmgr_misc_hw_t602x = {
+	.dev_min_ps = {
+		[DEV_FABRIC] = SYS_DEV_PSTATE_SUSPEND,
+		[DEV_DCS] = SYS_DEV_PSTATE_SUSPEND,
+	},
+};
+
+static const struct of_device_id apple_pmgr_misc_of_match[] = {
+	{ .compatible = "apple,t6000-pmgr-misc", .data = &apple_pmgr_misc_hw_t600x },
+	{ .compatible = "apple,t6020-pmgr-misc", .data = &apple_pmgr_misc_hw_t602x },
+	{}
+};
+
+MODULE_DEVICE_TABLE(of, apple_pmgr_misc_of_match);
+
+static const struct dev_pm_ops apple_pmgr_misc_pm_ops = {
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_pmgr_misc_suspend_noirq,
+				      apple_pmgr_misc_resume_noirq)
+};
+
+static struct platform_driver apple_pmgr_misc_driver = {
+	.probe = apple_pmgr_misc_probe,
+	.driver = {
+		.name = "apple-pmgr-misc",
+		.of_match_table = apple_pmgr_misc_of_match,
+		.pm = pm_ptr(&apple_pmgr_misc_pm_ops),
+	},
+};
+
+MODULE_AUTHOR("Hector Martin <marcan@marcan.st>");
+MODULE_DESCRIPTION("PMGR misc driver for Apple SoCs");
+MODULE_LICENSE("GPL");
+
+module_platform_driver(apple_pmgr_misc_driver);
diff --git a/drivers/soc/apple/sart.c b/drivers/soc/apple/sart.c
index 9eaf3febb3826..66b99955b395b 100644
--- a/drivers/soc/apple/sart.c
+++ b/drivers/soc/apple/sart.c
@@ -218,6 +218,7 @@ struct apple_sart *devm_apple_sart_get(struct device *dev)
 {
 	struct device_node *sart_node;
 	struct platform_device *sart_pdev;
+	struct device_link *link;
 	struct apple_sart *sart;
 
 	sart_node = of_parse_phandle(dev->of_node, "apple,sart", 0);
@@ -236,8 +237,12 @@ struct apple_sart *devm_apple_sart_get(struct device *dev)
 		return ERR_PTR(-EPROBE_DEFER);
 	}
 
-	device_link_add(dev, &sart_pdev->dev,
-			DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
+	link = device_link_add(dev, &sart_pdev->dev,
+			       DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
+	if (!link) {
+		put_device(&sart_pdev->dev);
+		return ERR_PTR(-ENODEV);
+	}
 
 	put_device(&sart_pdev->dev);
 
diff --git a/drivers/soc/fsl/qe/qe_ports_ic.c b/drivers/soc/fsl/qe/qe_ports_ic.c
index 9b0bba64e91e4..33ca1ddafe18c 100644
--- a/drivers/soc/fsl/qe/qe_ports_ic.c
+++ b/drivers/soc/fsl/qe/qe_ports_ic.c
@@ -23,36 +23,39 @@ struct qepic_data {
 static void qepic_mask(struct irq_data *d)
 {
 	struct qepic_data *data = irq_data_get_irq_chip_data(d);
+	u32 val = ioread32be(data->reg + CEPIMR);
 
-	clrbits32(data->reg + CEPIMR, 1 << (31 - irqd_to_hwirq(d)));
+	iowrite32be(val & ~(1 << (31 - irqd_to_hwirq(d))), data->reg + CEPIMR);
 }
 
 static void qepic_unmask(struct irq_data *d)
 {
 	struct qepic_data *data = irq_data_get_irq_chip_data(d);
+	u32 val = ioread32be(data->reg + CEPIMR);
 
-	setbits32(data->reg + CEPIMR, 1 << (31 - irqd_to_hwirq(d)));
+	iowrite32be(val | 1 << (31 - irqd_to_hwirq(d)), data->reg + CEPIMR);
 }
 
 static void qepic_end(struct irq_data *d)
 {
 	struct qepic_data *data = irq_data_get_irq_chip_data(d);
 
-	out_be32(data->reg + CEPIER, 1 << (31 - irqd_to_hwirq(d)));
+	iowrite32be(1 << (31 - irqd_to_hwirq(d)), data->reg + CEPIER);
 }
 
 static int qepic_set_type(struct irq_data *d, unsigned int flow_type)
 {
 	struct qepic_data *data = irq_data_get_irq_chip_data(d);
 	unsigned int vec = (unsigned int)irqd_to_hwirq(d);
+	u32 val = ioread32be(data->reg + CEPICR);
 
 	switch (flow_type & IRQ_TYPE_SENSE_MASK) {
 	case IRQ_TYPE_EDGE_FALLING:
-		setbits32(data->reg + CEPICR, 1 << (31 - vec));
+		iowrite32be(val | 1 << (31 - vec), data->reg + CEPICR);
 		return 0;
 	case IRQ_TYPE_EDGE_BOTH:
 	case IRQ_TYPE_NONE:
-		clrbits32(data->reg + CEPICR, 1 << (31 - vec));
+		iowrite32be(val & ~(1 << (31 - vec)), data->reg + CEPICR);
 		return 0;
 	}
 	return -EINVAL;
@@ -69,7 +72,7 @@ static struct irq_chip qepic = {
 static int qepic_get_irq(struct irq_desc *desc)
 {
 	struct qepic_data *data = irq_desc_get_handler_data(desc);
-	u32 event = in_be32(data->reg + CEPIER);
+	u32 event = ioread32be(data->reg + CEPIER);
 
 	if (!event)
 		return -1;
diff --git a/drivers/soc/imx/soc-imx9.c b/drivers/soc/imx/soc-imx9.c
index 58eef7d4f9089..23b54b7d75027 100644
--- a/drivers/soc/imx/soc-imx9.c
+++ b/drivers/soc/imx/soc-imx9.c
@@ -35,6 +35,8 @@ static int imx9_soc_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, err, "%s: missing model property\n", __func__);
 
 	attr->family = devm_kasprintf(dev, GFP_KERNEL, "Freescale i.MX");
+	if (!attr->family)
+		return -ENOMEM;
 
 	/*
 	 * Retrieve the soc id, rev & uid info:
@@ -52,11 +54,18 @@ static int imx9_soc_probe(struct platform_device *pdev)
 	rev_minor = SOC_REV_MINOR(res.a1);
 
 	attr->soc_id = devm_kasprintf(dev, GFP_KERNEL, "i.MX%2x", soc_id);
+	if (!attr->soc_id)
+		return -ENOMEM;
+
 	attr->revision = devm_kasprintf(dev, GFP_KERNEL, "%d.%d", rev_major, rev_minor);
+	if (!attr->revision)
+		return -ENOMEM;
 
 	uid127_64 = res.a2;
 	uid63_0 = res.a3;
 	attr->serial_number = devm_kasprintf(dev, GFP_KERNEL, "%016llx%016llx", uid127_64, uid63_0);
+	if (!attr->serial_number)
+		return -ENOMEM;
 
 	sdev = soc_device_register(attr);
 	if (IS_ERR(sdev))
diff --git a/drivers/soc/mediatek/mt8167-mmsys.h b/drivers/soc/mediatek/mt8167-mmsys.h
index eef14083c47b5..d579feee4212f 100644
--- a/drivers/soc/mediatek/mt8167-mmsys.h
+++ b/drivers/soc/mediatek/mt8167-mmsys.h
@@ -10,29 +10,24 @@
 #define MT8167_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN	0x06c
 
 #define MT8167_DITHER_MOUT_EN_RDMA0			0x1
-#define MT8167_DITHER_MOUT_EN_MASK			0x7
-
 #define MT8167_RDMA0_SOUT_DSI0				0x2
-#define MT8167_RDMA0_SOUT_MASK				0x3
-
 #define MT8167_DSI0_SEL_IN_RDMA0			0x1
-#define MT8167_DSI0_SEL_IN_MASK				0x3
 
 static const struct mtk_mmsys_routes mt8167_mmsys_routing_table[] = {
-	MMSYS_ROUTE(OVL0, COLOR0,
+	MMSYS_ROUTE(OVL, 0, COLOR, 0,
 		    MT8167_DISP_REG_CONFIG_DISP_OVL0_MOUT_EN, OVL0_MOUT_EN_COLOR0,
 		    OVL0_MOUT_EN_COLOR0),
-	MMSYS_ROUTE(DITHER0, RDMA0,
-		    MT8167_DISP_REG_CONFIG_DISP_DITHER_MOUT_EN, MT8167_DITHER_MOUT_EN_MASK,
+	MMSYS_ROUTE(DITHER, 0, RDMA, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_DITHER_MOUT_EN, MT8167_DITHER_MOUT_EN_RDMA0,
 		    MT8167_DITHER_MOUT_EN_RDMA0),
-	MMSYS_ROUTE(OVL0, COLOR0,
+	MMSYS_ROUTE(OVL, 0, COLOR, 0,
 		    MT8167_DISP_REG_CONFIG_DISP_COLOR0_SEL_IN, COLOR0_SEL_IN_OVL0,
 		    COLOR0_SEL_IN_OVL0),
-	MMSYS_ROUTE(RDMA0, DSI0,
-		    MT8167_DISP_REG_CONFIG_DISP_DSI0_SEL_IN, MT8167_DSI0_SEL_IN_MASK,
+	MMSYS_ROUTE(RDMA, 0, DSI, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_DSI0_SEL_IN, MT8167_DSI0_SEL_IN_RDMA0,
 		    MT8167_DSI0_SEL_IN_RDMA0),
-	MMSYS_ROUTE(RDMA0, DSI0,
-		    MT8167_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN, MT8167_RDMA0_SOUT_MASK,
+	MMSYS_ROUTE(RDMA, 0, DSI, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN, MT8167_RDMA0_SOUT_DSI0,
 		    MT8167_RDMA0_SOUT_DSI0),
 };
 
diff --git a/drivers/soc/mediatek/mt8173-mmsys.h b/drivers/soc/mediatek/mt8173-mmsys.h
index 957876d7c1666..af67879ff8b44 100644
--- a/drivers/soc/mediatek/mt8173-mmsys.h
+++ b/drivers/soc/mediatek/mt8173-mmsys.h
@@ -33,46 +33,46 @@
 #define MT8173_RDMA0_SOUT_COLOR0			BIT(0)
 
 static const struct mtk_mmsys_routes mt8173_mmsys_routing_table[] = {
-	MMSYS_ROUTE(OVL0, COLOR0,
+	MMSYS_ROUTE(OVL, 0, COLOR, 0,
 		    MT8173_DISP_REG_CONFIG_DISP_OVL0_MOUT_EN, MT8173_OVL0_MOUT_EN_COLOR0,
 		    MT8173_OVL0_MOUT_EN_COLOR0),
-	MMSYS_ROUTE(OD0, RDMA0,
+	MMSYS_ROUTE(OD, 0, RDMA, 0,
 		    MT8173_DISP_REG_CONFIG_DISP_OD_MOUT_EN, MT8173_OD0_MOUT_EN_RDMA0,
 		    MT8173_OD0_MOUT_EN_RDMA0),
-	MMSYS_ROUTE(UFOE, DSI0,
+	MMSYS_ROUTE(UFOE, 0, DSI, 0,
 		    MT8173_DISP_REG_CONFIG_DISP_UFOE_MOUT_EN, MT8173_UFOE_MOUT_EN_DSI0,
 		    MT8173_UFOE_MOUT_EN_DSI0),
-	MMSYS_ROUTE(COLOR0, AAL0,
+	MMSYS_ROUTE(COLOR, 0, AAL, 0,
 		    MT8173_DISP_REG_CONFIG_DISP_COLOR0_SOUT_SEL_IN, MT8173_COLOR0_SOUT_MERGE,
 		    0 /* SOUT to AAL */),
-	MMSYS_ROUTE(RDMA0, UFOE,
+	MMSYS_ROUTE(RDMA, 0, UFOE, 0,
 		    MT8173_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN, MT8173_RDMA0_SOUT_COLOR0,
 		    0 /* SOUT to UFOE */),
-	MMSYS_ROUTE(OVL0, COLOR0,
+	MMSYS_ROUTE(OVL, 0, COLOR, 0,
 		    MT8173_DISP_REG_CONFIG_DISP_COLOR0_SEL_IN, MT8173_COLOR0_SEL_IN_OVL0,
 		    MT8173_COLOR0_SEL_IN_OVL0),
-	MMSYS_ROUTE(AAL0, COLOR0,
+	MMSYS_ROUTE(AAL, 0, COLOR, 0,
 		    MT8173_DISP_REG_CONFIG_DISP_AAL_SEL_IN, MT8173_AAL_SEL_IN_MERGE,
 		    0 /* SEL_IN from COLOR0 */),
-	MMSYS_ROUTE(RDMA0, UFOE,
+	MMSYS_ROUTE(RDMA, 0, UFOE, 0,
 		    MT8173_DISP_REG_CONFIG_DISP_UFOE_SEL_IN, MT8173_UFOE_SEL_IN_RDMA0,
 		    0 /* SEL_IN from RDMA0 */),
-	MMSYS_ROUTE(UFOE, DSI0,
+	MMSYS_ROUTE(UFOE, 0, DSI, 0,
 		    MT8173_DISP_REG_CONFIG_DSI0_SEL_IN, MT8173_DSI0_SEL_IN_UFOE,
 		    0 /* SEL_IN from UFOE */),
-	MMSYS_ROUTE(OVL1, COLOR1,
+	MMSYS_ROUTE(OVL, 1, COLOR, 1,
 		    MT8173_DISP_REG_CONFIG_DISP_OVL1_MOUT_EN, MT8173_OVL1_MOUT_EN_COLOR1,
 		    MT8173_OVL1_MOUT_EN_COLOR1),
-	MMSYS_ROUTE(GAMMA, RDMA1,
+	MMSYS_ROUTE(GAMMA, 0, RDMA, 1,
 		    MT8173_DISP_REG_CONFIG_DISP_GAMMA_MOUT_EN, MT8173_GAMMA_MOUT_EN_RDMA1,
 		    MT8173_GAMMA_MOUT_EN_RDMA1),
-	MMSYS_ROUTE(RDMA1, DPI0,
+	MMSYS_ROUTE(RDMA, 1, DPI, 0,
 		    MT8173_DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN, RDMA1_SOUT_MASK,
 		    RDMA1_SOUT_DPI0),
-	MMSYS_ROUTE(OVL1, COLOR1,
+	MMSYS_ROUTE(OVL, 1, COLOR, 1,
 		    MT8173_DISP_REG_CONFIG_DISP_COLOR1_SEL_IN, COLOR1_SEL_IN_OVL1,
 		    COLOR1_SEL_IN_OVL1),
-	MMSYS_ROUTE(RDMA1, DPI0,
+	MMSYS_ROUTE(RDMA, 1, DPI, 0,
 		    MT8173_DISP_REG_CONFIG_DPI_SEL_IN, MT8173_DPI0_SEL_IN_MASK,
 		    MT8173_DPI0_SEL_IN_RDMA1),
 };
diff --git a/drivers/soc/mediatek/mt8183-mmsys.h b/drivers/soc/mediatek/mt8183-mmsys.h
index 123384958c4b7..cf221ef203d22 100644
--- a/drivers/soc/mediatek/mt8183-mmsys.h
+++ b/drivers/soc/mediatek/mt8183-mmsys.h
@@ -28,25 +28,25 @@
 #define MT8183_MMSYS_SW0_RST_B			0x140
 
 static const struct mtk_mmsys_routes mmsys_mt8183_routing_table[] = {
-	MMSYS_ROUTE(OVL0, OVL_2L0,
+	MMSYS_ROUTE(OVL, 0, OVL_2L, 0,
 		    MT8183_DISP_OVL0_MOUT_EN, MT8183_OVL0_MOUT_EN_OVL0_2L,
 		    MT8183_OVL0_MOUT_EN_OVL0_2L),
-	MMSYS_ROUTE(OVL_2L0, RDMA0,
+	MMSYS_ROUTE(OVL_2L, 0, RDMA, 0,
 		    MT8183_DISP_OVL0_2L_MOUT_EN, MT8183_OVL0_2L_MOUT_EN_DISP_PATH0,
 		    MT8183_OVL0_2L_MOUT_EN_DISP_PATH0),
-	MMSYS_ROUTE(OVL_2L1, RDMA1,
+	MMSYS_ROUTE(OVL_2L, 1, RDMA, 1,
 		    MT8183_DISP_OVL1_2L_MOUT_EN, MT8183_OVL1_2L_MOUT_EN_RDMA1,
 		    MT8183_OVL1_2L_MOUT_EN_RDMA1),
-	MMSYS_ROUTE(DITHER0, DSI0,
+	MMSYS_ROUTE(DITHER, 0, DSI, 0,
 		    MT8183_DISP_DITHER0_MOUT_EN, MT8183_DITHER0_MOUT_IN_DSI0,
 		    MT8183_DITHER0_MOUT_IN_DSI0),
-	MMSYS_ROUTE(OVL_2L0, RDMA0,
+	MMSYS_ROUTE(OVL_2L, 0, RDMA, 0,
 		    MT8183_DISP_PATH0_SEL_IN, MT8183_DISP_PATH0_SEL_IN_OVL0_2L,
 		    MT8183_DISP_PATH0_SEL_IN_OVL0_2L),
-	MMSYS_ROUTE(RDMA1, DPI0,
+	MMSYS_ROUTE(RDMA, 1, DPI, 0,
 		    MT8183_DISP_DPI0_SEL_IN, MT8183_DPI0_SEL_IN_RDMA1,
 		    MT8183_DPI0_SEL_IN_RDMA1),
-	MMSYS_ROUTE(RDMA0, COLOR0,
+	MMSYS_ROUTE(RDMA, 0, COLOR, 0,
 		    MT8183_DISP_RDMA0_SOUT_SEL_IN, MT8183_RDMA0_SOUT_COLOR0,
 		    MT8183_RDMA0_SOUT_COLOR0),
 };
diff --git a/drivers/soc/mediatek/mt8186-mmsys.h b/drivers/soc/mediatek/mt8186-mmsys.h
index 354664be72bd7..0c6941be6fa56 100644
--- a/drivers/soc/mediatek/mt8186-mmsys.h
+++ b/drivers/soc/mediatek/mt8186-mmsys.h
@@ -63,37 +63,37 @@
 #define MT8186_MMSYS_SW0_RST_B				0x160
 
 static const struct mtk_mmsys_routes mmsys_mt8186_routing_table[] = {
-	MMSYS_ROUTE(OVL0, RDMA0,
+	MMSYS_ROUTE(OVL, 0, RDMA, 0,
 		    MT8186_DISP_OVL0_MOUT_EN, MT8186_OVL0_MOUT_EN_MASK,
 		    MT8186_OVL0_MOUT_TO_RDMA0),
-	MMSYS_ROUTE(OVL0, RDMA0,
+	MMSYS_ROUTE(OVL, 0, RDMA, 0,
 		    MT8186_DISP_RDMA0_SEL_IN, MT8186_RDMA0_SEL_IN_MASK,
 		    MT8186_RDMA0_FROM_OVL0),
-	MMSYS_ROUTE(OVL0, RDMA0,
+	MMSYS_ROUTE(OVL, 0, RDMA, 0,
 		    MT8186_MMSYS_OVL_CON, MT8186_MMSYS_OVL0_CON_MASK,
 		    MT8186_OVL0_GO_BLEND),
-	MMSYS_ROUTE(RDMA0, COLOR0,
+	MMSYS_ROUTE(RDMA, 0, COLOR, 0,
 		    MT8186_DISP_RDMA0_SOUT_SEL, MT8186_RDMA0_SOUT_SEL_MASK,
 		    MT8186_RDMA0_SOUT_TO_COLOR0),
-	MMSYS_ROUTE(DITHER0, DSI0,
+	MMSYS_ROUTE(DITHER, 0, DSI, 0,
 		    MT8186_DISP_DITHER0_MOUT_EN, MT8186_DITHER0_MOUT_EN_MASK,
 		    MT8186_DITHER0_MOUT_TO_DSI0),
-	MMSYS_ROUTE(DITHER0, DSI0,
+	MMSYS_ROUTE(DITHER, 0, DSI, 0,
 		    MT8186_DISP_DSI0_SEL_IN, MT8186_DSI0_SEL_IN_MASK,
 		    MT8186_DSI0_FROM_DITHER0),
-	MMSYS_ROUTE(OVL_2L0, RDMA1,
+	MMSYS_ROUTE(OVL_2L, 0, RDMA, 1,
 		    MT8186_DISP_OVL0_2L_MOUT_EN, MT8186_OVL0_2L_MOUT_EN_MASK,
 		    MT8186_OVL0_2L_MOUT_TO_RDMA1),
-	MMSYS_ROUTE(OVL_2L0, RDMA1,
+	MMSYS_ROUTE(OVL_2L, 0, RDMA, 1,
 		    MT8186_DISP_RDMA1_SEL_IN, MT8186_RDMA1_SEL_IN_MASK,
 		    MT8186_RDMA1_FROM_OVL0_2L),
-	MMSYS_ROUTE(OVL_2L0, RDMA1,
+	MMSYS_ROUTE(OVL_2L, 0, RDMA, 1,
 		    MT8186_MMSYS_OVL_CON, MT8186_MMSYS_OVL0_2L_CON_MASK,
 		    MT8186_OVL0_2L_GO_BLEND),
-	MMSYS_ROUTE(RDMA1, DPI0,
+	MMSYS_ROUTE(RDMA, 1, DPI, 0,
 		    MT8186_DISP_RDMA1_MOUT_EN, MT8186_RDMA1_MOUT_EN_MASK,
 		    MT8186_RDMA1_MOUT_TO_DPI0_SEL),
-	MMSYS_ROUTE(RDMA1, DPI0,
+	MMSYS_ROUTE(RDMA, 1, DPI, 0,
 		    MT8186_DISP_DPI0_SEL_IN, MT8186_DPI0_SEL_IN_MASK,
 		    MT8186_DPI0_FROM_RDMA1),
 };
diff --git a/drivers/soc/mediatek/mt8188-mmsys.h b/drivers/soc/mediatek/mt8188-mmsys.h
index 99080afead7e7..c70c4b4623812 100644
--- a/drivers/soc/mediatek/mt8188-mmsys.h
+++ b/drivers/soc/mediatek/mt8188-mmsys.h
@@ -202,124 +202,124 @@ static const u8 mmsys_mt8188_vdo1_rst_tb[] = {
 };
 
 static const struct mtk_mmsys_routes mmsys_mt8188_routing_table[] = {
-	MMSYS_ROUTE(OVL0, RDMA0,
+	MMSYS_ROUTE(OVL, 0, RDMA, 0,
 		    MT8188_VDO0_OVL_MOUT_EN, MT8188_MOUT_DISP_OVL0_TO_DISP_RDMA0,
 		    MT8188_MOUT_DISP_OVL0_TO_DISP_RDMA0),
-	MMSYS_ROUTE(OVL0, WDMA0,
+	MMSYS_ROUTE(OVL, 0, WDMA, 0,
 		    MT8188_VDO0_OVL_MOUT_EN, MT8188_MOUT_DISP_OVL0_TO_DISP_WDMA0,
 		    MT8188_MOUT_DISP_OVL0_TO_DISP_WDMA0),
-	MMSYS_ROUTE(OVL0, RDMA0,
+	MMSYS_ROUTE(OVL, 0, RDMA, 0,
 		    MT8188_VDO0_DISP_RDMA_SEL, MT8188_SEL_IN_DISP_RDMA0_FROM_MASK,
 		    MT8188_SEL_IN_DISP_RDMA0_FROM_DISP_OVL0),
-	MMSYS_ROUTE(DITHER0, DSI0,
+	MMSYS_ROUTE(DITHER, 0, DSI, 0,
 		    MT8188_VDO0_DSI0_SEL_IN, MT8188_SEL_IN_DSI0_FROM_MASK,
 		    MT8188_SEL_IN_DSI0_FROM_DISP_DITHER0),
-	MMSYS_ROUTE(DITHER0, MERGE0,
+	MMSYS_ROUTE(DITHER, 0, MERGE, 0,
 		    MT8188_VDO0_VPP_MERGE_SEL, MT8188_SEL_IN_VPP_MERGE_FROM_MASK,
 		    MT8188_SEL_IN_DP_INTF0_FROM_DISP_DITHER0),
-	MMSYS_ROUTE(DITHER0, DSC0,
+	MMSYS_ROUTE(DITHER, 0, DSC, 0,
 		    MT8188_VDO0_DSC_WARP_SEL, MT8188_SEL_IN_DSC_WRAP0C0_IN_FROM_MASK,
 		    MT8188_SEL_IN_DSC_WRAP0C0_IN_FROM_DISP_DITHER0),
-	MMSYS_ROUTE(DITHER0, DP_INTF0,
+	MMSYS_ROUTE(DITHER, 0, DP_INTF, 0,
 		    MT8188_VDO0_DP_INTF0_SEL_IN, MT8188_SEL_IN_DP_INTF0_FROM_MASK,
 		    MT8188_SEL_IN_DP_INTF0_FROM_DISP_DITHER0),
-	MMSYS_ROUTE(DSC0, MERGE0,
+	MMSYS_ROUTE(DSC, 0, MERGE, 0,
 		    MT8188_VDO0_VPP_MERGE_SEL, MT8188_SEL_IN_VPP_MERGE_FROM_MASK,
 		    MT8188_SEL_IN_VPP_MERGE_FROM_DSC_WRAP0_OUT),
-	MMSYS_ROUTE(MERGE0, DP_INTF0,
+	MMSYS_ROUTE(MERGE, 0, DP_INTF, 0,
 		    MT8188_VDO0_DP_INTF0_SEL_IN, MT8188_SEL_IN_DP_INTF0_FROM_MASK,
 		    MT8188_SEL_IN_DP_INTF0_FROM_VPP_MERGE),
-	MMSYS_ROUTE(DSC0, DSI0,
+	MMSYS_ROUTE(DSC, 0, DSI, 0,
 		    MT8188_VDO0_DSI0_SEL_IN, MT8188_SEL_IN_DSI0_FROM_MASK,
 		    MT8188_SEL_IN_DSI0_FROM_DSC_WRAP0_OUT),
-	MMSYS_ROUTE(RDMA0, COLOR0,
+	MMSYS_ROUTE(RDMA, 0, COLOR, 0,
 		    MT8188_VDO0_DISP_RDMA_SEL, GENMASK(1, 0),
 		    MT8188_SOUT_DISP_RDMA0_TO_DISP_COLOR0),
-	MMSYS_ROUTE(DITHER0, DSC0,
+	MMSYS_ROUTE(DITHER, 0, DSC, 0,
 		    MT8188_VDO0_DISP_DITHER0_SEL_OUT, MT8188_SOUT_DISP_DITHER0_TO_MASK,
 		    MT8188_SOUT_DISP_DITHER0_TO_DSC_WRAP0_IN),
-	MMSYS_ROUTE(DITHER0, DSI0,
+	MMSYS_ROUTE(DITHER, 0, DSI, 0,
 		    MT8188_VDO0_DISP_DITHER0_SEL_OUT, MT8188_SOUT_DISP_DITHER0_TO_MASK,
 		    MT8188_SOUT_DISP_DITHER0_TO_DSI0),
-	MMSYS_ROUTE(DITHER0, MERGE0,
+	MMSYS_ROUTE(DITHER, 0, MERGE, 0,
 		    MT8188_VDO0_DISP_DITHER0_SEL_OUT, MT8188_SOUT_DISP_DITHER0_TO_MASK,
 		    MT8188_SOUT_DISP_DITHER0_TO_VPP_MERGE0),
-	MMSYS_ROUTE(DITHER0, DP_INTF0,
+	MMSYS_ROUTE(DITHER, 0, DP_INTF, 0,
 		    MT8188_VDO0_DISP_DITHER0_SEL_OUT, MT8188_SOUT_DISP_DITHER0_TO_MASK,
 		    MT8188_SOUT_DISP_DITHER0_TO_DP_INTF0),
-	MMSYS_ROUTE(MERGE0, DP_INTF0,
+	MMSYS_ROUTE(MERGE, 0, DP_INTF, 0,
 		    MT8188_VDO0_VPP_MERGE_SEL, MT8188_SOUT_VPP_MERGE_TO_MASK,
 		    MT8188_SOUT_VPP_MERGE_TO_DP_INTF0),
-	MMSYS_ROUTE(MERGE0, DPI0,
+	MMSYS_ROUTE(MERGE, 0, DPI, 0,
 		    MT8188_VDO0_VPP_MERGE_SEL, MT8188_SOUT_VPP_MERGE_TO_MASK,
 		    MT8188_SOUT_VPP_MERGE_TO_SINA_VIRTUAL0),
-	MMSYS_ROUTE(MERGE0, WDMA0,
+	MMSYS_ROUTE(MERGE, 0, WDMA, 0,
 		    MT8188_VDO0_VPP_MERGE_SEL, MT8188_SOUT_VPP_MERGE_TO_MASK,
 		    MT8188_SOUT_VPP_MERGE_TO_DISP_WDMA0),
-	MMSYS_ROUTE(MERGE0, DSC0,
+	MMSYS_ROUTE(MERGE, 0, DSC, 0,
 		    MT8188_VDO0_VPP_MERGE_SEL, MT8188_SOUT_VPP_MERGE_TO_MASK,
 		    MT8188_SOUT_VPP_MERGE_TO_DSC_WRAP0_IN),
-	MMSYS_ROUTE(DSC0, DSI0,
+	MMSYS_ROUTE(DSC, 0, DSI, 0,
 		    MT8188_VDO0_DSC_WARP_SEL, MT8188_SOUT_DSC_WRAP0_OUT_TO_MASK,
 		    MT8188_SOUT_DSC_WRAP0_OUT_TO_DSI0),
-	MMSYS_ROUTE(DSC0, MERGE0,
+	MMSYS_ROUTE(DSC, 0, MERGE, 0,
 		    MT8188_VDO0_DSC_WARP_SEL, MT8188_SOUT_DSC_WRAP0_OUT_TO_MASK,
 		    MT8188_SOUT_DSC_WRAP0_OUT_TO_VPP_MERGE),
 };
 
 static const struct mtk_mmsys_routes mmsys_mt8188_vdo1_routing_table[] = {
-	MMSYS_ROUTE(MDP_RDMA0, MERGE1,
+	MMSYS_ROUTE(MDP_RDMA, 0, MERGE, 1,
 		    MT8188_VDO1_VPP_MERGE0_P0_SEL_IN, GENMASK(0, 0),
 		    MT8188_VPP_MERGE0_P0_SEL_IN_FROM_MDP_RDMA0),
-	MMSYS_ROUTE(MDP_RDMA1, MERGE1,
+	MMSYS_ROUTE(MDP_RDMA, 1, MERGE, 1,
 		    MT8188_VDO1_VPP_MERGE0_P1_SEL_IN, GENMASK(0, 0),
 		    MT8188_VPP_MERGE0_P1_SEL_IN_FROM_MDP_RDMA1),
-	MMSYS_ROUTE(MDP_RDMA2, MERGE2,
+	MMSYS_ROUTE(MDP_RDMA, 2, MERGE, 2,
 		    MT8188_VDO1_VPP_MERGE1_P0_SEL_IN, GENMASK(0, 0),
 		    MT8188_VPP_MERGE1_P0_SEL_IN_FROM_MDP_RDMA2),
-	MMSYS_ROUTE(MERGE1, ETHDR_MIXER,
+	MMSYS_ROUTE(MERGE, 1, ETHDR_MIXER, 0,
 		    MT8188_VDO1_MERGE0_ASYNC_SOUT_SEL, GENMASK(1, 0),
 		    MT8188_SOUT_TO_MIXER_IN1_SEL),
-	MMSYS_ROUTE(MERGE2, ETHDR_MIXER,
+	MMSYS_ROUTE(MERGE, 2, ETHDR_MIXER, 0,
 		    MT8188_VDO1_MERGE1_ASYNC_SOUT_SEL, GENMASK(1, 0),
 		    MT8188_SOUT_TO_MIXER_IN2_SEL),
-	MMSYS_ROUTE(MERGE3, ETHDR_MIXER,
+	MMSYS_ROUTE(MERGE, 3, ETHDR_MIXER, 0,
 		    MT8188_VDO1_MERGE2_ASYNC_SOUT_SEL, GENMASK(1, 0),
 		    MT8188_SOUT_TO_MIXER_IN3_SEL),
-	MMSYS_ROUTE(MERGE4, ETHDR_MIXER,
+	MMSYS_ROUTE(MERGE, 4, ETHDR_MIXER, 0,
 		    MT8188_VDO1_MERGE3_ASYNC_SOUT_SEL, GENMASK(1, 0),
 		    MT8188_SOUT_TO_MIXER_IN4_SEL),
-	MMSYS_ROUTE(ETHDR_MIXER, MERGE5,
+	MMSYS_ROUTE(ETHDR_MIXER, 0, MERGE, 5,
 		    MT8188_VDO1_MIXER_OUT_SOUT_SEL, GENMASK(0, 0),
 		    MT8188_MIXER_SOUT_TO_MERGE4_ASYNC_SEL),
-	MMSYS_ROUTE(MERGE1, ETHDR_MIXER,
+	MMSYS_ROUTE(MERGE, 1, ETHDR_MIXER, 0,
 		    MT8188_VDO1_MIXER_IN1_SEL_IN, GENMASK(0, 0),
 		    MT8188_MIXER_IN1_SEL_IN_FROM_MERGE0_ASYNC_SOUT),
-	MMSYS_ROUTE(MERGE2, ETHDR_MIXER,
+	MMSYS_ROUTE(MERGE, 2, ETHDR_MIXER, 0,
 		    MT8188_VDO1_MIXER_IN2_SEL_IN, GENMASK(0, 0),
 		    MT8188_MIXER_IN2_SEL_IN_FROM_MERGE1_ASYNC_SOUT),
-	MMSYS_ROUTE(MERGE3, ETHDR_MIXER,
+	MMSYS_ROUTE(MERGE, 3, ETHDR_MIXER, 0,
 		    MT8188_VDO1_MIXER_IN3_SEL_IN, GENMASK(0, 0),
 		    MT8188_MIXER_IN3_SEL_IN_FROM_MERGE2_ASYNC_SOUT),
-	MMSYS_ROUTE(MERGE4, ETHDR_MIXER,
+	MMSYS_ROUTE(MERGE, 4, ETHDR_MIXER, 0,
 		    MT8188_VDO1_MIXER_IN4_SEL_IN, GENMASK(0, 0),
 		    MT8188_MIXER_IN4_SEL_IN_FROM_MERGE3_ASYNC_SOUT),
-	MMSYS_ROUTE(ETHDR_MIXER, MERGE5,
+	MMSYS_ROUTE(ETHDR_MIXER, 0, MERGE, 5,
 		    MT8188_VDO1_MIXER_SOUT_SEL_IN, GENMASK(2, 0),
 		    MT8188_MIXER_SOUT_SEL_IN_FROM_DISP_MIXER),
-	MMSYS_ROUTE(ETHDR_MIXER, MERGE5,
+	MMSYS_ROUTE(ETHDR_MIXER, 0, MERGE, 5,
 		    MT8188_VDO1_MERGE4_ASYNC_SEL_IN, GENMASK(2, 0),
 		    MT8188_MERGE4_ASYNC_SEL_IN_FROM_MIXER_OUT_SOUT),
-	MMSYS_ROUTE(MERGE5, DPI1,
+	MMSYS_ROUTE(MERGE, 5, DPI, 1,
 		    MT8188_VDO1_DISP_DPI1_SEL_IN, GENMASK(1, 0),
 		    MT8188_DISP_DPI1_SEL_IN_FROM_VPP_MERGE4_MOUT),
-	MMSYS_ROUTE(MERGE5, DPI1,
+	MMSYS_ROUTE(MERGE, 5, DPI, 1,
 		    MT8188_VDO1_MERGE4_SOUT_SEL, GENMASK(3, 0),
 		    MT8188_MERGE4_SOUT_TO_DPI1_SEL),
-	MMSYS_ROUTE(MERGE5, DP_INTF1,
+	MMSYS_ROUTE(MERGE, 5, DP_INTF, 1,
 		    MT8188_VDO1_DISP_DP_INTF0_SEL_IN, GENMASK(1, 0),
 		    MT8188_DISP_DP_INTF0_SEL_IN_FROM_VPP_MERGE4_MOUT),
-	MMSYS_ROUTE(MERGE5, DP_INTF1,
+	MMSYS_ROUTE(MERGE, 5, DP_INTF, 1,
 		    MT8188_VDO1_MERGE4_SOUT_SEL, GENMASK(3, 0),
 		    MT8188_MERGE4_SOUT_TO_DP_INTF0_SEL),
 };
diff --git a/drivers/soc/mediatek/mt8192-mmsys.h b/drivers/soc/mediatek/mt8192-mmsys.h
index 7cafa2455fd09..37ced5152ba7f 100644
--- a/drivers/soc/mediatek/mt8192-mmsys.h
+++ b/drivers/soc/mediatek/mt8192-mmsys.h
@@ -31,34 +31,34 @@
 #define MT8192_DSI0_SEL_IN_DITHER0			0x1
 
 static const struct mtk_mmsys_routes mmsys_mt8192_routing_table[] = {
-	MMSYS_ROUTE(OVL_2L0, RDMA0,
+	MMSYS_ROUTE(OVL_2L, 0, RDMA, 0,
 		    MT8192_DISP_OVL0_2L_MOUT_EN, MT8192_OVL0_MOUT_EN_DISP_RDMA0,
 		    MT8192_OVL0_MOUT_EN_DISP_RDMA0),
-	MMSYS_ROUTE(OVL_2L2, RDMA4,
+	MMSYS_ROUTE(OVL_2L, 2, RDMA, 4,
 		    MT8192_DISP_OVL2_2L_MOUT_EN, MT8192_OVL2_2L_MOUT_EN_RDMA4,
 		    MT8192_OVL2_2L_MOUT_EN_RDMA4),
-	MMSYS_ROUTE(DITHER0, DSI0,
+	MMSYS_ROUTE(DITHER, 0, DSI, 0,
 		    MT8192_DISP_DITHER0_MOUT_EN, MT8192_DITHER0_MOUT_IN_DSI0,
 		    MT8192_DITHER0_MOUT_IN_DSI0),
-	MMSYS_ROUTE(OVL_2L0, RDMA0,
+	MMSYS_ROUTE(OVL_2L, 0, RDMA, 0,
 		    MT8192_DISP_RDMA0_SEL_IN, MT8192_RDMA0_SEL_IN_OVL0_2L,
 		    MT8192_RDMA0_SEL_IN_OVL0_2L),
-	MMSYS_ROUTE(CCORR, AAL0,
+	MMSYS_ROUTE(CCORR, 0, AAL, 0,
 		    MT8192_DISP_AAL0_SEL_IN, MT8192_AAL0_SEL_IN_CCORR0,
 		    MT8192_AAL0_SEL_IN_CCORR0),
-	MMSYS_ROUTE(DITHER0, DSI0,
+	MMSYS_ROUTE(DITHER, 0, DSI, 0,
 		    MT8192_DISP_DSI0_SEL_IN, MT8192_DSI0_SEL_IN_DITHER0,
 		    MT8192_DSI0_SEL_IN_DITHER0),
-	MMSYS_ROUTE(RDMA0, COLOR0,
+	MMSYS_ROUTE(RDMA, 0, COLOR, 0,
 		    MT8192_DISP_RDMA0_SOUT_SEL, MT8192_RDMA0_SOUT_COLOR0,
 		    MT8192_RDMA0_SOUT_COLOR0),
-	MMSYS_ROUTE(CCORR, AAL0,
+	MMSYS_ROUTE(CCORR, 0, AAL, 0,
 		    MT8192_DISP_CCORR0_SOUT_SEL, MT8192_CCORR0_SOUT_AAL0,
 		    MT8192_CCORR0_SOUT_AAL0),
-	MMSYS_ROUTE(OVL0, OVL_2L0,
+	MMSYS_ROUTE(OVL, 0, OVL_2L, 0,
 		    MT8192_MMSYS_OVL_MOUT_EN, MT8192_DISP_OVL0_GO_BG,
 		    MT8192_DISP_OVL0_GO_BG),
-	MMSYS_ROUTE(OVL_2L0, RDMA0,
+	MMSYS_ROUTE(OVL_2L, 0, RDMA, 0,
 		    MT8192_MMSYS_OVL_MOUT_EN, MT8192_DISP_OVL0_2L_GO_BLEND,
 		    MT8192_DISP_OVL0_2L_GO_BLEND),
 };
diff --git a/drivers/soc/mediatek/mt8195-mmsys.h b/drivers/soc/mediatek/mt8195-mmsys.h
index f69929a2a4d4d..3a58b9b74282c 100644
--- a/drivers/soc/mediatek/mt8195-mmsys.h
+++ b/drivers/soc/mediatek/mt8195-mmsys.h
@@ -160,278 +160,279 @@
 #define MT8195_SVPP3_MDP_RSZ					BIT(5)
 
 static const struct mtk_mmsys_routes mmsys_mt8195_routing_table[] = {
-	MMSYS_ROUTE(OVL0, RDMA0,
+	MMSYS_ROUTE(OVL, 0, RDMA, 0,
 		    MT8195_VDO0_OVL_MOUT_EN, MT8195_MOUT_DISP_OVL0_TO_DISP_RDMA0,
 		    MT8195_MOUT_DISP_OVL0_TO_DISP_RDMA0),
-	MMSYS_ROUTE(OVL0, WDMA0,
+	MMSYS_ROUTE(OVL, 0, WDMA, 0,
 		    MT8195_VDO0_OVL_MOUT_EN, MT8195_MOUT_DISP_OVL0_TO_DISP_WDMA0,
 		    MT8195_MOUT_DISP_OVL0_TO_DISP_WDMA0),
-	MMSYS_ROUTE(OVL0, OVL1,
+	MMSYS_ROUTE(OVL, 0, OVL, 1,
 		    MT8195_VDO0_OVL_MOUT_EN, MT8195_MOUT_DISP_OVL0_TO_DISP_OVL1,
 		    MT8195_MOUT_DISP_OVL0_TO_DISP_OVL1),
-	MMSYS_ROUTE(OVL1, RDMA1,
+	MMSYS_ROUTE(OVL, 1, RDMA, 1,
 		    MT8195_VDO0_OVL_MOUT_EN, MT8195_MOUT_DISP_OVL1_TO_DISP_RDMA1,
 		    MT8195_MOUT_DISP_OVL1_TO_DISP_RDMA1),
-	MMSYS_ROUTE(OVL1, WDMA1,
+	MMSYS_ROUTE(OVL, 1, WDMA, 1,
 		    MT8195_VDO0_OVL_MOUT_EN, MT8195_MOUT_DISP_OVL1_TO_DISP_WDMA1,
 		    MT8195_MOUT_DISP_OVL1_TO_DISP_WDMA1),
-	MMSYS_ROUTE(OVL1, OVL0,
+	MMSYS_ROUTE(OVL, 1, OVL, 0,
 		    MT8195_VDO0_OVL_MOUT_EN, MT8195_MOUT_DISP_OVL1_TO_DISP_OVL0,
 		    MT8195_MOUT_DISP_OVL1_TO_DISP_OVL0),
-	MMSYS_ROUTE(DSC0, MERGE0,
+	MMSYS_ROUTE(DSC, 0, MERGE, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_VPP_MERGE_FROM_MASK,
 		    MT8195_SEL_IN_VPP_MERGE_FROM_DSC_WRAP0_OUT),
-	MMSYS_ROUTE(DITHER1, MERGE0,
+	MMSYS_ROUTE(DITHER, 1, MERGE, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_VPP_MERGE_FROM_MASK,
 		    MT8195_SEL_IN_VPP_MERGE_FROM_DISP_DITHER1),
-	MMSYS_ROUTE(MERGE5, MERGE0,
+	MMSYS_ROUTE(MERGE, 5, MERGE, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_VPP_MERGE_FROM_MASK,
 		    MT8195_SEL_IN_VPP_MERGE_FROM_VDO1_VIRTUAL0),
-	MMSYS_ROUTE(DITHER0, DSC0,
+	MMSYS_ROUTE(DITHER, 0, DSC, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSC_WRAP0_IN_FROM_MASK,
 		    MT8195_SEL_IN_DSC_WRAP0_IN_FROM_DISP_DITHER0),
-	MMSYS_ROUTE(MERGE0, DSC0,
+	MMSYS_ROUTE(MERGE, 0, DSC, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSC_WRAP0_IN_FROM_MASK,
 		    MT8195_SEL_IN_DSC_WRAP0_IN_FROM_VPP_MERGE),
-	MMSYS_ROUTE(DITHER1, DSC1,
+	MMSYS_ROUTE(DITHER, 1, DSC, 1,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSC_WRAP1_IN_FROM_MASK,
 		    MT8195_SEL_IN_DSC_WRAP1_IN_FROM_DISP_DITHER1),
-	MMSYS_ROUTE(MERGE0, DSC1,
+	MMSYS_ROUTE(MERGE, 0, DSC, 1,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSC_WRAP1_IN_FROM_MASK,
 		    MT8195_SEL_IN_DSC_WRAP1_IN_FROM_VPP_MERGE),
-	MMSYS_ROUTE(MERGE0, DP_INTF1,
+	MMSYS_ROUTE(MERGE, 0, DP_INTF, 1,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_SINA_VIRTUAL0_FROM_MASK,
 		    MT8195_SEL_IN_SINA_VIRTUAL0_FROM_VPP_MERGE),
-	MMSYS_ROUTE(MERGE0, DPI0,
+	MMSYS_ROUTE(MERGE, 0, DPI, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_SINA_VIRTUAL0_FROM_MASK,
 		    MT8195_SEL_IN_SINA_VIRTUAL0_FROM_VPP_MERGE),
-	MMSYS_ROUTE(MERGE0, DPI1,
+	MMSYS_ROUTE(MERGE, 0, DPI, 1,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_SINA_VIRTUAL0_FROM_MASK,
 		    MT8195_SEL_IN_SINA_VIRTUAL0_FROM_VPP_MERGE),
-	MMSYS_ROUTE(DSC1, DP_INTF1,
+	MMSYS_ROUTE(DSC, 1, DP_INTF, 1,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_SINA_VIRTUAL0_FROM_MASK,
 		    MT8195_SEL_IN_SINA_VIRTUAL0_FROM_DSC_WRAP1_OUT),
-	MMSYS_ROUTE(DSC1, DPI0,
+	MMSYS_ROUTE(DSC, 1, DPI, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_SINA_VIRTUAL0_FROM_MASK,
 		    MT8195_SEL_IN_SINA_VIRTUAL0_FROM_DSC_WRAP1_OUT),
-	MMSYS_ROUTE(DSC1, DPI1,
+	MMSYS_ROUTE(DSC, 1, DPI, 1,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_SINA_VIRTUAL0_FROM_MASK,
 		    MT8195_SEL_IN_SINA_VIRTUAL0_FROM_DSC_WRAP1_OUT),
-	MMSYS_ROUTE(DSC0, DP_INTF1,
+	MMSYS_ROUTE(DSC, 0, DP_INTF, 1,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_SINB_VIRTUAL0_FROM_MASK,
 		    MT8195_SEL_IN_SINB_VIRTUAL0_FROM_DSC_WRAP0_OUT),
-	MMSYS_ROUTE(DSC0, DPI0,
+	MMSYS_ROUTE(DSC, 0, DPI, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_SINB_VIRTUAL0_FROM_MASK,
 		    MT8195_SEL_IN_SINB_VIRTUAL0_FROM_DSC_WRAP0_OUT),
-	MMSYS_ROUTE(DSC0, DPI1,
+	MMSYS_ROUTE(DSC, 0, DPI, 1,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_SINB_VIRTUAL0_FROM_MASK,
 		    MT8195_SEL_IN_SINB_VIRTUAL0_FROM_DSC_WRAP0_OUT),
-	MMSYS_ROUTE(DSC1, DP_INTF0,
+	MMSYS_ROUTE(DSC, 1, DP_INTF, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DP_INTF0_FROM_MASK,
 		    MT8195_SEL_IN_DP_INTF0_FROM_DSC_WRAP1_OUT),
-	MMSYS_ROUTE(MERGE0, DP_INTF0,
+	MMSYS_ROUTE(MERGE, 0, DP_INTF, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DP_INTF0_FROM_MASK,
 		    MT8195_SEL_IN_DP_INTF0_FROM_VPP_MERGE),
-	MMSYS_ROUTE(MERGE5, DP_INTF0,
+	MMSYS_ROUTE(MERGE, 5, DP_INTF, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DP_INTF0_FROM_MASK,
 		    MT8195_SEL_IN_DP_INTF0_FROM_VDO1_VIRTUAL0),
-	MMSYS_ROUTE(DSC0, DSI0,
+	MMSYS_ROUTE(DSC, 0, DSI, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSI0_FROM_MASK,
 		    MT8195_SEL_IN_DSI0_FROM_DSC_WRAP0_OUT),
-	MMSYS_ROUTE(DITHER0, DSI0,
+	MMSYS_ROUTE(DITHER, 0, DSI, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSI0_FROM_MASK,
 		    MT8195_SEL_IN_DSI0_FROM_DISP_DITHER0),
-	MMSYS_ROUTE(DSC1, DSI1,
+	MMSYS_ROUTE(DSC, 1, DSI, 1,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSI1_FROM_MASK,
 		    MT8195_SEL_IN_DSI1_FROM_DSC_WRAP1_OUT),
-	MMSYS_ROUTE(MERGE0, DSI1,
+	MMSYS_ROUTE(MERGE, 0, DSI, 1,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSI1_FROM_MASK,
 		    MT8195_SEL_IN_DSI1_FROM_VPP_MERGE),
-	MMSYS_ROUTE(OVL1, WDMA1,
+	MMSYS_ROUTE(OVL, 1, WDMA, 1,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DISP_WDMA1_FROM_MASK,
 		    MT8195_SEL_IN_DISP_WDMA1_FROM_DISP_OVL1),
-	MMSYS_ROUTE(MERGE0, WDMA1,
+	MMSYS_ROUTE(MERGE, 0, WDMA, 1,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DISP_WDMA1_FROM_MASK,
 		    MT8195_SEL_IN_DISP_WDMA1_FROM_VPP_MERGE),
-	MMSYS_ROUTE(DSC1, DSI1,
+	MMSYS_ROUTE(DSC, 1, DSI, 1,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSC_WRAP1_FROM_MASK,
 		    MT8195_SEL_IN_DSC_WRAP1_OUT_FROM_DSC_WRAP1_IN),
-	MMSYS_ROUTE(DSC1, DP_INTF0,
+	MMSYS_ROUTE(DSC, 1, DP_INTF, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSC_WRAP1_FROM_MASK,
 		    MT8195_SEL_IN_DSC_WRAP1_OUT_FROM_DSC_WRAP1_IN),
-	MMSYS_ROUTE(DSC1, DP_INTF1,
+	MMSYS_ROUTE(DSC, 1, DP_INTF, 1,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSC_WRAP1_FROM_MASK,
 		    MT8195_SEL_IN_DSC_WRAP1_OUT_FROM_DSC_WRAP1_IN),
-	MMSYS_ROUTE(DSC1, DPI0,
+	MMSYS_ROUTE(DSC, 1, DPI, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSC_WRAP1_FROM_MASK,
 		    MT8195_SEL_IN_DSC_WRAP1_OUT_FROM_DSC_WRAP1_IN),
-	MMSYS_ROUTE(DSC1, DPI1,
+	MMSYS_ROUTE(DSC, 1, DPI, 1,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSC_WRAP1_FROM_MASK,
 		    MT8195_SEL_IN_DSC_WRAP1_OUT_FROM_DSC_WRAP1_IN),
-	MMSYS_ROUTE(DSC1, MERGE0,
+	MMSYS_ROUTE(DSC, 1, MERGE, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSC_WRAP1_FROM_MASK,
 		    MT8195_SEL_IN_DSC_WRAP1_OUT_FROM_DSC_WRAP1_IN),
-	MMSYS_ROUTE(DITHER1, DSI1,
+	MMSYS_ROUTE(DITHER, 1, DSI, 1,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSC_WRAP1_FROM_MASK,
 		    MT8195_SEL_IN_DSC_WRAP1_OUT_FROM_DISP_DITHER1),
-	MMSYS_ROUTE(DITHER1, DP_INTF0,
+	MMSYS_ROUTE(DITHER, 1, DP_INTF, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSC_WRAP1_FROM_MASK,
 		    MT8195_SEL_IN_DSC_WRAP1_OUT_FROM_DISP_DITHER1),
-	MMSYS_ROUTE(DITHER1, DPI0,
+	MMSYS_ROUTE(DITHER, 1, DPI, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSC_WRAP1_FROM_MASK,
 		    MT8195_SEL_IN_DSC_WRAP1_OUT_FROM_DISP_DITHER1),
-	MMSYS_ROUTE(DITHER1, DPI1,
+	MMSYS_ROUTE(DITHER, 1, DPI, 1,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DSC_WRAP1_FROM_MASK,
 		    MT8195_SEL_IN_DSC_WRAP1_OUT_FROM_DISP_DITHER1),
-	MMSYS_ROUTE(OVL0, WDMA0,
+	MMSYS_ROUTE(OVL, 0, WDMA, 0,
 		    MT8195_VDO0_SEL_IN, MT8195_SEL_IN_DISP_WDMA0_FROM_MASK,
 		    MT8195_SEL_IN_DISP_WDMA0_FROM_DISP_OVL0),
-	MMSYS_ROUTE(DITHER0, DSC0,
+	MMSYS_ROUTE(DITHER, 0, DSC, 0,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DISP_DITHER0_TO_MASK,
 		    MT8195_SOUT_DISP_DITHER0_TO_DSC_WRAP0_IN),
-	MMSYS_ROUTE(DITHER0, DSI0,
+	MMSYS_ROUTE(DITHER, 0, DSI, 0,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DISP_DITHER0_TO_MASK,
 		    MT8195_SOUT_DISP_DITHER0_TO_DSI0),
-	MMSYS_ROUTE(DITHER1, DSC1,
+	MMSYS_ROUTE(DITHER, 1, DSC, 1,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DISP_DITHER1_TO_MASK,
 		    MT8195_SOUT_DISP_DITHER1_TO_DSC_WRAP1_IN),
-	MMSYS_ROUTE(DITHER1, MERGE0,
+	MMSYS_ROUTE(DITHER, 1, MERGE, 0,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DISP_DITHER1_TO_MASK,
 		    MT8195_SOUT_DISP_DITHER1_TO_VPP_MERGE),
-	MMSYS_ROUTE(DITHER1, DSI1,
+	MMSYS_ROUTE(DITHER, 1, DSI, 1,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DISP_DITHER1_TO_MASK,
 		    MT8195_SOUT_DISP_DITHER1_TO_DSC_WRAP1_OUT),
-	MMSYS_ROUTE(DITHER1, DP_INTF0,
+	MMSYS_ROUTE(DITHER, 1, DP_INTF, 0,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DISP_DITHER1_TO_MASK,
 		    MT8195_SOUT_DISP_DITHER1_TO_DSC_WRAP1_OUT),
-	MMSYS_ROUTE(DITHER1, DP_INTF1,
+	MMSYS_ROUTE(DITHER, 1, DP_INTF, 1,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DISP_DITHER1_TO_MASK,
 		    MT8195_SOUT_DISP_DITHER1_TO_DSC_WRAP1_OUT),
-	MMSYS_ROUTE(DITHER1, DPI0,
+	MMSYS_ROUTE(DITHER, 1, DPI, 0,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DISP_DITHER1_TO_MASK,
 		    MT8195_SOUT_DISP_DITHER1_TO_DSC_WRAP1_OUT),
-	MMSYS_ROUTE(DITHER1, DPI1,
+	MMSYS_ROUTE(DITHER, 1, DPI, 1,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DISP_DITHER1_TO_MASK,
 		    MT8195_SOUT_DISP_DITHER1_TO_DSC_WRAP1_OUT),
-	MMSYS_ROUTE(MERGE5, MERGE0,
+	MMSYS_ROUTE(MERGE, 5, MERGE, 0,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_VDO1_VIRTUAL0_TO_MASK,
 		    MT8195_SOUT_VDO1_VIRTUAL0_TO_VPP_MERGE),
-	MMSYS_ROUTE(MERGE5, DP_INTF0,
+	MMSYS_ROUTE(MERGE, 5, DP_INTF, 0,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_VDO1_VIRTUAL0_TO_MASK,
 		    MT8195_SOUT_VDO1_VIRTUAL0_TO_DP_INTF0),
-	MMSYS_ROUTE(MERGE0, DSI1,
+	MMSYS_ROUTE(MERGE, 0, DSI, 1,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_VPP_MERGE_TO_MASK,
 		    MT8195_SOUT_VPP_MERGE_TO_DSI1),
-	MMSYS_ROUTE(MERGE0, DP_INTF0,
+	MMSYS_ROUTE(MERGE, 0, DP_INTF, 0,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_VPP_MERGE_TO_MASK,
 		    MT8195_SOUT_VPP_MERGE_TO_DP_INTF0),
-	MMSYS_ROUTE(MERGE0, DP_INTF1,
+	MMSYS_ROUTE(MERGE, 0, DP_INTF, 1,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_VPP_MERGE_TO_MASK,
 		    MT8195_SOUT_VPP_MERGE_TO_SINA_VIRTUAL0),
-	MMSYS_ROUTE(MERGE0, DPI0,
+	MMSYS_ROUTE(MERGE, 0, DPI, 0,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_VPP_MERGE_TO_MASK,
 		    MT8195_SOUT_VPP_MERGE_TO_SINA_VIRTUAL0),
-	MMSYS_ROUTE(MERGE0, DPI1,
+	MMSYS_ROUTE(MERGE, 0, DPI, 1,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_VPP_MERGE_TO_MASK,
 		    MT8195_SOUT_VPP_MERGE_TO_SINA_VIRTUAL0),
-	MMSYS_ROUTE(MERGE0, WDMA1,
+	MMSYS_ROUTE(MERGE, 0, WDMA, 1,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_VPP_MERGE_TO_MASK,
 		    MT8195_SOUT_VPP_MERGE_TO_DISP_WDMA1),
-	MMSYS_ROUTE(MERGE0, DSC0,
+	MMSYS_ROUTE(MERGE, 0, DSC, 0,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_VPP_MERGE_TO_MASK,
 		    MT8195_SOUT_VPP_MERGE_TO_DSC_WRAP0_IN),
-	MMSYS_ROUTE(MERGE0, DSC1,
+	MMSYS_ROUTE(MERGE, 0, DSC, 1,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_VPP_MERGE_TO_DSC_WRAP1_IN_MASK,
 		    MT8195_SOUT_VPP_MERGE_TO_DSC_WRAP1_IN),
-	MMSYS_ROUTE(DSC0, DSI0,
+	MMSYS_ROUTE(DSC, 0, DSI, 0,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DSC_WRAP0_OUT_TO_MASK,
 		    MT8195_SOUT_DSC_WRAP0_OUT_TO_DSI0),
-	MMSYS_ROUTE(DSC0, DP_INTF1,
+	MMSYS_ROUTE(DSC, 0, DP_INTF, 1,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DSC_WRAP0_OUT_TO_MASK,
 		    MT8195_SOUT_DSC_WRAP0_OUT_TO_SINB_VIRTUAL0),
-	MMSYS_ROUTE(DSC0, DPI0,
+	MMSYS_ROUTE(DSC, 0, DPI, 0,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DSC_WRAP0_OUT_TO_MASK,
 		    MT8195_SOUT_DSC_WRAP0_OUT_TO_SINB_VIRTUAL0),
-	MMSYS_ROUTE(DSC0, DPI1,
+	MMSYS_ROUTE(DSC, 0, DPI, 1,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DSC_WRAP0_OUT_TO_MASK,
 		    MT8195_SOUT_DSC_WRAP0_OUT_TO_SINB_VIRTUAL0),
-	MMSYS_ROUTE(DSC0, MERGE0,
+	MMSYS_ROUTE(DSC, 0, MERGE, 0,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DSC_WRAP0_OUT_TO_MASK,
 		    MT8195_SOUT_DSC_WRAP0_OUT_TO_VPP_MERGE),
-	MMSYS_ROUTE(DSC1, DSI1,
+	MMSYS_ROUTE(DSC, 1, DSI, 1,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DSC_WRAP1_OUT_TO_MASK,
 		    MT8195_SOUT_DSC_WRAP1_OUT_TO_DSI1),
-	MMSYS_ROUTE(DSC1, DP_INTF0,
+	MMSYS_ROUTE(DSC, 1, DP_INTF, 0,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DSC_WRAP1_OUT_TO_MASK,
 		    MT8195_SOUT_DSC_WRAP1_OUT_TO_DP_INTF0),
-	MMSYS_ROUTE(DSC1, DP_INTF1,
+	MMSYS_ROUTE(DSC, 1, DP_INTF, 1,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DSC_WRAP1_OUT_TO_MASK,
 		    MT8195_SOUT_DSC_WRAP1_OUT_TO_SINA_VIRTUAL0),
-	MMSYS_ROUTE(DSC1, DPI0,
+	MMSYS_ROUTE(DSC, 1, DPI, 0,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DSC_WRAP1_OUT_TO_MASK,
 		    MT8195_SOUT_DSC_WRAP1_OUT_TO_SINA_VIRTUAL0),
-	MMSYS_ROUTE(DSC1, DPI1,
+	MMSYS_ROUTE(DSC, 1, DPI, 1,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DSC_WRAP1_OUT_TO_MASK,
 		    MT8195_SOUT_DSC_WRAP1_OUT_TO_SINA_VIRTUAL0),
-	MMSYS_ROUTE(DSC1, MERGE0,
+	MMSYS_ROUTE(DSC, 1, MERGE, 0,
 		    MT8195_VDO0_SEL_OUT, MT8195_SOUT_DSC_WRAP1_OUT_TO_MASK,
 		    MT8195_SOUT_DSC_WRAP1_OUT_TO_VPP_MERGE),
 };
 
 static const struct mtk_mmsys_routes mmsys_mt8195_vdo1_routing_table[] = {
-	MMSYS_ROUTE(MDP_RDMA0, MERGE1,
+	MMSYS_ROUTE(MDP_RDMA, 0, MERGE, 1,
 		    MT8195_VDO1_VPP_MERGE0_P0_SEL_IN, GENMASK(0, 0),
 		    MT8195_VPP_MERGE0_P0_SEL_IN_FROM_MDP_RDMA0),
-	MMSYS_ROUTE(MDP_RDMA1, MERGE1,
+	MMSYS_ROUTE(MDP_RDMA, 1, MERGE, 1,
 		    MT8195_VDO1_VPP_MERGE0_P1_SEL_IN, GENMASK(0, 0),
 		    MT8195_VPP_MERGE0_P1_SEL_IN_FROM_MDP_RDMA1),
-	MMSYS_ROUTE(MDP_RDMA2, MERGE2,
+	MMSYS_ROUTE(MDP_RDMA, 2, MERGE, 2,
 		    MT8195_VDO1_VPP_MERGE1_P0_SEL_IN, GENMASK(0, 0),
 		    MT8195_VPP_MERGE1_P0_SEL_IN_FROM_MDP_RDMA2),
-	MMSYS_ROUTE(MERGE1, ETHDR_MIXER,
+	MMSYS_ROUTE(MERGE, 1, ETHDR_MIXER, 0,
 		    MT8195_VDO1_MERGE0_ASYNC_SOUT_SEL, GENMASK(1, 0),
 		    MT8195_SOUT_TO_MIXER_IN1_SEL),
-	MMSYS_ROUTE(MERGE2, ETHDR_MIXER,
+	MMSYS_ROUTE(MERGE, 2, ETHDR_MIXER, 0,
 		    MT8195_VDO1_MERGE1_ASYNC_SOUT_SEL, GENMASK(1, 0),
 		    MT8195_SOUT_TO_MIXER_IN2_SEL),
-	MMSYS_ROUTE(MERGE3, ETHDR_MIXER,
+	MMSYS_ROUTE(MERGE, 3, ETHDR_MIXER, 0,
 		    MT8195_VDO1_MERGE2_ASYNC_SOUT_SEL, GENMASK(1, 0),
 		    MT8195_SOUT_TO_MIXER_IN3_SEL),
-	MMSYS_ROUTE(MERGE4, ETHDR_MIXER,
+	MMSYS_ROUTE(MERGE, 4, ETHDR_MIXER, 0,
 		    MT8195_VDO1_MERGE3_ASYNC_SOUT_SEL, GENMASK(1, 0),
 		    MT8195_SOUT_TO_MIXER_IN4_SEL),
-	MMSYS_ROUTE(ETHDR_MIXER, MERGE5,
+	MMSYS_ROUTE(ETHDR_MIXER, 0, MERGE, 5,
 		    MT8195_VDO1_MIXER_OUT_SOUT_SEL, GENMASK(0, 0),
 		    MT8195_MIXER_SOUT_TO_MERGE4_ASYNC_SEL),
-	MMSYS_ROUTE(MERGE1, ETHDR_MIXER,
+	MMSYS_ROUTE(MERGE, 1, ETHDR_MIXER, 0,
 		    MT8195_VDO1_MIXER_IN1_SEL_IN, GENMASK(0, 0),
 		    MT8195_MIXER_IN1_SEL_IN_FROM_MERGE0_ASYNC_SOUT),
-	MMSYS_ROUTE(MERGE2, ETHDR_MIXER,
+	MMSYS_ROUTE(MERGE, 2, ETHDR_MIXER, 0,
 		    MT8195_VDO1_MIXER_IN2_SEL_IN, GENMASK(0, 0),
 		    MT8195_MIXER_IN2_SEL_IN_FROM_MERGE1_ASYNC_SOUT),
-	MMSYS_ROUTE(MERGE3, ETHDR_MIXER,
+	MMSYS_ROUTE(MERGE, 3, ETHDR_MIXER, 0,
 		    MT8195_VDO1_MIXER_IN3_SEL_IN, GENMASK(0, 0),
 		    MT8195_MIXER_IN3_SEL_IN_FROM_MERGE2_ASYNC_SOUT),
-	MMSYS_ROUTE(MERGE4, ETHDR_MIXER,
+	MMSYS_ROUTE(MERGE, 4, ETHDR_MIXER, 0,
 		    MT8195_VDO1_MIXER_IN4_SEL_IN, GENMASK(0, 0),
 		    MT8195_MIXER_IN4_SEL_IN_FROM_MERGE3_ASYNC_SOUT),
-	MMSYS_ROUTE(ETHDR_MIXER, MERGE5,
+	MMSYS_ROUTE(ETHDR_MIXER, 0, MERGE, 5,
 		    MT8195_VDO1_MIXER_SOUT_SEL_IN, GENMASK(2, 0),
 		    MT8195_MIXER_SOUT_SEL_IN_FROM_DISP_MIXER),
-	MMSYS_ROUTE(ETHDR_MIXER, MERGE5,
+	MMSYS_ROUTE(ETHDR_MIXER, 0, MERGE, 5,
 		    MT8195_VDO1_MERGE4_ASYNC_SEL_IN, GENMASK(2, 0),
 		    MT8195_MERGE4_ASYNC_SEL_IN_FROM_MIXER_OUT_SOUT),
-	MMSYS_ROUTE(MERGE5, DPI1,
+	MMSYS_ROUTE(MERGE, 5, DPI, 1,
 		    MT8195_VDO1_DISP_DPI1_SEL_IN, GENMASK(1, 0),
 		    MT8195_DISP_DPI1_SEL_IN_FROM_VPP_MERGE4_MOUT),
-	MMSYS_ROUTE(MERGE5, DPI1,
+	MMSYS_ROUTE(MERGE, 5, DPI, 1,
 		    MT8195_VDO1_MERGE4_SOUT_SEL, GENMASK(1, 0),
 		    MT8195_MERGE4_SOUT_TO_DPI1_SEL),
-	MMSYS_ROUTE(MERGE5, DP_INTF1,
+	MMSYS_ROUTE(MERGE, 5, DP_INTF, 1,
 		    MT8195_VDO1_DISP_DP_INTF0_SEL_IN, GENMASK(1, 0),
 		    MT8195_DISP_DP_INTF0_SEL_IN_FROM_VPP_MERGE4_MOUT),
-	MMSYS_ROUTE(MERGE5, DP_INTF1,
+	MMSYS_ROUTE(MERGE, 5, DP_INTF, 1,
 		    MT8195_VDO1_MERGE4_SOUT_SEL, GENMASK(1, 0),
 		    MT8195_MERGE4_SOUT_TO_DP_INTF0_SEL),
 };
+
 #endif /* __SOC_MEDIATEK_MT8195_MMSYS_H */
diff --git a/drivers/soc/mediatek/mt8365-mmsys.h b/drivers/soc/mediatek/mt8365-mmsys.h
index 533a3fd0923b6..b438ab7ae00bc 100644
--- a/drivers/soc/mediatek/mt8365-mmsys.h
+++ b/drivers/soc/mediatek/mt8365-mmsys.h
@@ -28,35 +28,35 @@
 #define MT8365_DPI0_SEL_IN_RDMA1			0x0
 
 static const struct mtk_mmsys_routes mt8365_mmsys_routing_table[] = {
-	MMSYS_ROUTE(OVL0, RDMA0,
+	MMSYS_ROUTE(OVL, 0, RDMA, 0,
 		    MT8365_DISP_REG_CONFIG_DISP_OVL0_MOUT_EN,
 		    MT8365_DISP_MS_IN_OUT_MASK, MT8365_OVL0_MOUT_PATH0_SEL),
-	MMSYS_ROUTE(OVL0, RDMA0,
+	MMSYS_ROUTE(OVL, 0, RDMA, 0,
 		    MT8365_DISP_REG_CONFIG_DISP_RDMA0_SEL_IN,
 		    MT8365_DISP_MS_IN_OUT_MASK, MT8365_RDMA0_SEL_IN_OVL0),
-	MMSYS_ROUTE(RDMA0, COLOR0,
+	MMSYS_ROUTE(RDMA, 0, COLOR, 0,
 		    MT8365_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL,
 		    MT8365_DISP_MS_IN_OUT_MASK, MT8365_RDMA0_SOUT_COLOR0),
-	MMSYS_ROUTE(COLOR0, CCORR,
+	MMSYS_ROUTE(COLOR, 0, CCORR, 0,
 		    MT8365_DISP_REG_CONFIG_DISP_COLOR0_SEL_IN,
 		    MT8365_DISP_MS_IN_OUT_MASK, MT8365_DISP_COLOR_SEL_IN_COLOR0),
-	MMSYS_ROUTE(DITHER0, DSI0,
+	MMSYS_ROUTE(DITHER, 0, DSI, 0,
 		    MT8365_DISP_REG_CONFIG_DISP_DITHER0_MOUT_EN,
 		    MT8365_DISP_MS_IN_OUT_MASK, MT8365_DITHER_MOUT_EN_DSI0),
-	MMSYS_ROUTE(DITHER0, DSI0,
+	MMSYS_ROUTE(DITHER, 0, DSI, 0,
 		    MT8365_DISP_REG_CONFIG_DISP_DSI0_SEL_IN,
 		    MT8365_DISP_MS_IN_OUT_MASK, MT8365_DSI0_SEL_IN_DITHER),
-	MMSYS_ROUTE(RDMA0, COLOR0,
+	MMSYS_ROUTE(RDMA, 0, COLOR, 0,
 		    MT8365_DISP_REG_CONFIG_DISP_RDMA0_RSZ0_SEL_IN,
 		    MT8365_DISP_MS_IN_OUT_MASK, MT8365_RDMA0_RSZ0_SEL_IN_RDMA0),
-	MMSYS_ROUTE(RDMA1, DPI0,
+	MMSYS_ROUTE(RDMA, 1, DPI, 0,
 		    MT8365_DISP_REG_CONFIG_DISP_LVDS_SYS_CFG_00,
 		    MT8365_LVDS_SYS_CFG_00_SEL_LVDS_PXL_CLK,
 		    MT8365_LVDS_SYS_CFG_00_SEL_LVDS_PXL_CLK),
-	MMSYS_ROUTE(RDMA1, DPI0,
+	MMSYS_ROUTE(RDMA, 1, DPI, 0,
 		    MT8365_DISP_REG_CONFIG_DISP_DPI0_SEL_IN,
 		    MT8365_DISP_MS_IN_OUT_MASK, MT8365_DPI0_SEL_IN_RDMA1),
-	MMSYS_ROUTE(RDMA1, DPI0,
+	MMSYS_ROUTE(RDMA, 1, DPI, 0,
 		    MT8365_DISP_REG_CONFIG_DISP_RDMA1_SOUT_SEL,
 		    MT8365_DISP_MS_IN_OUT_MASK, MT8365_RDMA1_SOUT_DPI0),
 };
diff --git a/drivers/soc/mediatek/mtk-dvfsrc.c b/drivers/soc/mediatek/mtk-dvfsrc.c
index 548a28f502426..e8cdee8aed353 100644
--- a/drivers/soc/mediatek/mtk-dvfsrc.c
+++ b/drivers/soc/mediatek/mtk-dvfsrc.c
@@ -862,6 +862,7 @@ static const struct of_device_id mtk_dvfsrc_of_match[] = {
 	{ .compatible = "mediatek,mt8196-dvfsrc", .data = &mt8196_data },
 	{ /* sentinel */ }
 };
+MODULE_DEVICE_TABLE(of, mtk_dvfsrc_of_match);
 
 static struct platform_driver mtk_dvfsrc_driver = {
 	.probe	= mtk_dvfsrc_probe,
diff --git a/drivers/soc/mediatek/mtk-mmsys.h b/drivers/soc/mediatek/mtk-mmsys.h
index fe628d5f51986..c51a5e334e291 100644
--- a/drivers/soc/mediatek/mtk-mmsys.h
+++ b/drivers/soc/mediatek/mtk-mmsys.h
@@ -80,18 +80,25 @@
 
 #define MMSYS_RST_NR(bank, bit) (((bank) * 32) + (bit))
 
+/* Temporary compatibility definitions */
+#define DDP_COMPONENT_BLS0		DDP_COMPONENT_BLS
+#define DDP_COMPONENT_CCORR0		DDP_COMPONENT_CCORR
+#define DDP_COMPONENT_UFOE0		DDP_COMPONENT_UFOE
+#define DDP_COMPONENT_GAMMA0		DDP_COMPONENT_GAMMA
+#define DDP_COMPONENT_ETHDR_MIXER0	DDP_COMPONENT_ETHDR_MIXER
+
 /*
  * This macro adds a compile time check to make sure that the in/out
  * selection bit(s) fit in the register mask, similar to bitfield
  * macros, but this does not transform the value.
  */
-#define MMSYS_ROUTE(from, to, reg_addr, reg_mask, selection)		\
-	{ DDP_COMPONENT_##from, DDP_COMPONENT_##to, reg_addr, reg_mask,	\
-	  (__BUILD_BUG_ON_ZERO_MSG((reg_mask) == 0, "Invalid mask") +	\
-	   __BUILD_BUG_ON_ZERO_MSG(~(reg_mask) & (selection),		\
-				   #selection " does not fit in "	\
-				   #reg_mask) +				\
-	   (selection))							\
+#define MMSYS_ROUTE(from, fsid, to, tsid, reg_addr, reg_mask, selection)	\
+	{ DDP_COMPONENT_##from##fsid, DDP_COMPONENT_##to##tsid, reg_addr, reg_mask,	\
+	  (__BUILD_BUG_ON_ZERO_MSG((reg_mask) == 0, "Invalid mask") +		\
+	   __BUILD_BUG_ON_ZERO_MSG(~(reg_mask) & (selection),			\
+				   #selection " does not fit in "		\
+				   #reg_mask) +					\
+	   (selection))								\
 	}
 
 struct mtk_mmsys_routes {
@@ -151,171 +158,120 @@ struct mtk_mmsys_driver_data {
  * to an independent table.
  */
 static const struct mtk_mmsys_routes mmsys_default_routing_table[] = {
-	{
-		DDP_COMPONENT_BLS, DDP_COMPONENT_DSI0,
-		DISP_REG_CONFIG_OUT_SEL, BLS_RDMA1_DSI_DPI_MASK,
-		BLS_TO_DSI_RDMA1_TO_DPI1
-	}, {
-		DDP_COMPONENT_BLS, DDP_COMPONENT_DSI0,
-		DISP_REG_CONFIG_DSI_SEL, DSI_SEL_IN_MASK,
-		DSI_SEL_IN_BLS
-	}, {
-		DDP_COMPONENT_BLS, DDP_COMPONENT_DPI0,
-		DISP_REG_CONFIG_OUT_SEL, BLS_RDMA1_DSI_DPI_MASK,
-		BLS_TO_DPI_RDMA1_TO_DSI
-	}, {
-		DDP_COMPONENT_BLS, DDP_COMPONENT_DPI0,
-		DISP_REG_CONFIG_DSI_SEL, DSI_SEL_IN_MASK,
-		DSI_SEL_IN_RDMA
-	}, {
-		DDP_COMPONENT_BLS, DDP_COMPONENT_DPI0,
-		DISP_REG_CONFIG_DPI_SEL, DPI_SEL_IN_MASK,
-		DPI_SEL_IN_BLS
-	}, {
-		DDP_COMPONENT_GAMMA, DDP_COMPONENT_RDMA1,
-		DISP_REG_CONFIG_DISP_GAMMA_MOUT_EN, GAMMA_MOUT_EN_RDMA1,
-		GAMMA_MOUT_EN_RDMA1
-	}, {
-		DDP_COMPONENT_OD0, DDP_COMPONENT_RDMA0,
-		DISP_REG_CONFIG_DISP_OD_MOUT_EN, OD_MOUT_EN_RDMA0,
-		OD_MOUT_EN_RDMA0
-	}, {
-		DDP_COMPONENT_OD1, DDP_COMPONENT_RDMA1,
-		DISP_REG_CONFIG_DISP_OD_MOUT_EN, OD1_MOUT_EN_RDMA1,
-		OD1_MOUT_EN_RDMA1
-	}, {
-		DDP_COMPONENT_OVL0, DDP_COMPONENT_COLOR0,
-		DISP_REG_CONFIG_DISP_OVL0_MOUT_EN, OVL0_MOUT_EN_COLOR0,
-		OVL0_MOUT_EN_COLOR0
-	}, {
-		DDP_COMPONENT_OVL0, DDP_COMPONENT_COLOR0,
-		DISP_REG_CONFIG_DISP_COLOR0_SEL_IN, COLOR0_SEL_IN_OVL0,
-		COLOR0_SEL_IN_OVL0
-	}, {
-		DDP_COMPONENT_OVL0, DDP_COMPONENT_RDMA0,
-		DISP_REG_CONFIG_DISP_OVL_MOUT_EN, OVL_MOUT_EN_RDMA,
-		OVL_MOUT_EN_RDMA
-	}, {
-		DDP_COMPONENT_OVL1, DDP_COMPONENT_COLOR1,
-		DISP_REG_CONFIG_DISP_OVL1_MOUT_EN, OVL1_MOUT_EN_COLOR1,
-		OVL1_MOUT_EN_COLOR1
-	}, {
-		DDP_COMPONENT_OVL1, DDP_COMPONENT_COLOR1,
-		DISP_REG_CONFIG_DISP_COLOR1_SEL_IN, COLOR1_SEL_IN_OVL1,
-		COLOR1_SEL_IN_OVL1
-	}, {
-		DDP_COMPONENT_RDMA0, DDP_COMPONENT_DPI0,
-		DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN, RDMA0_SOUT_MASK,
-		RDMA0_SOUT_DPI0
-	}, {
-		DDP_COMPONENT_RDMA0, DDP_COMPONENT_DPI1,
-		DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN, RDMA0_SOUT_MASK,
-		RDMA0_SOUT_DPI1
-	}, {
-		DDP_COMPONENT_RDMA0, DDP_COMPONENT_DSI1,
-		DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN, RDMA0_SOUT_MASK,
-		RDMA0_SOUT_DSI1
-	}, {
-		DDP_COMPONENT_RDMA0, DDP_COMPONENT_DSI2,
-		DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN, RDMA0_SOUT_MASK,
-		RDMA0_SOUT_DSI2
-	}, {
-		DDP_COMPONENT_RDMA0, DDP_COMPONENT_DSI3,
-		DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN, RDMA0_SOUT_MASK,
-		RDMA0_SOUT_DSI3
-	}, {
-		DDP_COMPONENT_RDMA1, DDP_COMPONENT_DPI0,
-		DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN, RDMA1_SOUT_MASK,
-		RDMA1_SOUT_DPI0
-	}, {
-		DDP_COMPONENT_RDMA1, DDP_COMPONENT_DPI0,
-		DISP_REG_CONFIG_DPI_SEL_IN, DPI0_SEL_IN_MASK,
-		DPI0_SEL_IN_RDMA1
-	}, {
-		DDP_COMPONENT_RDMA1, DDP_COMPONENT_DPI1,
-		DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN, RDMA1_SOUT_MASK,
-		RDMA1_SOUT_DPI1
-	}, {
-		DDP_COMPONENT_RDMA1, DDP_COMPONENT_DPI1,
-		DISP_REG_CONFIG_DPI_SEL_IN, DPI1_SEL_IN_MASK,
-		DPI1_SEL_IN_RDMA1
-	}, {
-		DDP_COMPONENT_RDMA1, DDP_COMPONENT_DSI0,
-		DISP_REG_CONFIG_DSIE_SEL_IN, DSI0_SEL_IN_MASK,
-		DSI0_SEL_IN_RDMA1
-	}, {
-		DDP_COMPONENT_RDMA1, DDP_COMPONENT_DSI1,
-		DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN, RDMA1_SOUT_MASK,
-		RDMA1_SOUT_DSI1
-	}, {
-		DDP_COMPONENT_RDMA1, DDP_COMPONENT_DSI1,
-		DISP_REG_CONFIG_DSIO_SEL_IN, DSI1_SEL_IN_MASK,
-		DSI1_SEL_IN_RDMA1
-	}, {
-		DDP_COMPONENT_RDMA1, DDP_COMPONENT_DSI2,
-		DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN, RDMA1_SOUT_MASK,
-		RDMA1_SOUT_DSI2
-	}, {
-		DDP_COMPONENT_RDMA1, DDP_COMPONENT_DSI2,
-		DISP_REG_CONFIG_DSIE_SEL_IN, DSI2_SEL_IN_MASK,
-		DSI2_SEL_IN_RDMA1
-	}, {
-		DDP_COMPONENT_RDMA1, DDP_COMPONENT_DSI3,
-		DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN, RDMA1_SOUT_MASK,
-		RDMA1_SOUT_DSI3
-	}, {
-		DDP_COMPONENT_RDMA1, DDP_COMPONENT_DSI3,
-		DISP_REG_CONFIG_DSIO_SEL_IN, DSI3_SEL_IN_MASK,
-		DSI3_SEL_IN_RDMA1
-	}, {
-		DDP_COMPONENT_RDMA2, DDP_COMPONENT_DPI0,
-		DISP_REG_CONFIG_DISP_RDMA2_SOUT, RDMA2_SOUT_MASK,
-		RDMA2_SOUT_DPI0
-	}, {
-		DDP_COMPONENT_RDMA2, DDP_COMPONENT_DPI0,
-		DISP_REG_CONFIG_DPI_SEL_IN, DPI0_SEL_IN_MASK,
-		DPI0_SEL_IN_RDMA2
-	}, {
-		DDP_COMPONENT_RDMA2, DDP_COMPONENT_DPI1,
-		DISP_REG_CONFIG_DISP_RDMA2_SOUT, RDMA2_SOUT_MASK,
-		RDMA2_SOUT_DPI1
-	}, {
-		DDP_COMPONENT_RDMA2, DDP_COMPONENT_DPI1,
-		DISP_REG_CONFIG_DPI_SEL_IN, DPI1_SEL_IN_MASK,
-		DPI1_SEL_IN_RDMA2
-	}, {
-		DDP_COMPONENT_RDMA2, DDP_COMPONENT_DSI0,
-		DISP_REG_CONFIG_DSIE_SEL_IN, DSI0_SEL_IN_MASK,
-		DSI0_SEL_IN_RDMA2
-	}, {
-		DDP_COMPONENT_RDMA2, DDP_COMPONENT_DSI1,
-		DISP_REG_CONFIG_DISP_RDMA2_SOUT, RDMA2_SOUT_MASK,
-		RDMA2_SOUT_DSI1
-	}, {
-		DDP_COMPONENT_RDMA2, DDP_COMPONENT_DSI1,
-		DISP_REG_CONFIG_DSIO_SEL_IN, DSI1_SEL_IN_MASK,
-		DSI1_SEL_IN_RDMA2
-	}, {
-		DDP_COMPONENT_RDMA2, DDP_COMPONENT_DSI2,
-		DISP_REG_CONFIG_DISP_RDMA2_SOUT, RDMA2_SOUT_MASK,
-		RDMA2_SOUT_DSI2
-	}, {
-		DDP_COMPONENT_RDMA2, DDP_COMPONENT_DSI2,
-		DISP_REG_CONFIG_DSIE_SEL_IN, DSI2_SEL_IN_MASK,
-		DSI2_SEL_IN_RDMA2
-	}, {
-		DDP_COMPONENT_RDMA2, DDP_COMPONENT_DSI3,
-		DISP_REG_CONFIG_DISP_RDMA2_SOUT, RDMA2_SOUT_MASK,
-		RDMA2_SOUT_DSI3
-	}, {
-		DDP_COMPONENT_RDMA2, DDP_COMPONENT_DSI3,
-		DISP_REG_CONFIG_DSIO_SEL_IN, DSI3_SEL_IN_MASK,
-		DSI3_SEL_IN_RDMA2
-	}, {
-		DDP_COMPONENT_UFOE, DDP_COMPONENT_DSI0,
-		DISP_REG_CONFIG_DISP_UFOE_MOUT_EN, UFOE_MOUT_EN_DSI0,
-		UFOE_MOUT_EN_DSI0
-	}
+	MMSYS_ROUTE(BLS, 0, DSI, 0,
+		    DISP_REG_CONFIG_OUT_SEL, BLS_RDMA1_DSI_DPI_MASK,
+		    BLS_TO_DSI_RDMA1_TO_DPI1),
+	MMSYS_ROUTE(BLS, 0, DSI, 0,
+		    DISP_REG_CONFIG_DSI_SEL, DSI_SEL_IN_MASK,
+		    DSI_SEL_IN_BLS),
+	MMSYS_ROUTE(BLS, 0, DPI, 0,
+		    DISP_REG_CONFIG_OUT_SEL, BLS_RDMA1_DSI_DPI_MASK,
+		    BLS_TO_DPI_RDMA1_TO_DSI),
+	MMSYS_ROUTE(BLS, 0, DPI, 0,
+		    DISP_REG_CONFIG_DSI_SEL, DSI_SEL_IN_MASK,
+		    DSI_SEL_IN_RDMA),
+	MMSYS_ROUTE(BLS, 0, DPI, 0,
+		    DISP_REG_CONFIG_DPI_SEL, DPI_SEL_IN_MASK,
+		    DPI_SEL_IN_BLS),
+	MMSYS_ROUTE(GAMMA, 0, RDMA, 1,
+		    DISP_REG_CONFIG_DISP_GAMMA_MOUT_EN, GAMMA_MOUT_EN_RDMA1,
+		    GAMMA_MOUT_EN_RDMA1),
+	MMSYS_ROUTE(OD, 0, RDMA, 0,
+		    DISP_REG_CONFIG_DISP_OD_MOUT_EN, OD_MOUT_EN_RDMA0,
+		    OD_MOUT_EN_RDMA0),
+	MMSYS_ROUTE(OD, 1, RDMA, 1,
+		    DISP_REG_CONFIG_DISP_OD_MOUT_EN, OD1_MOUT_EN_RDMA1,
+		    OD1_MOUT_EN_RDMA1),
+	MMSYS_ROUTE(OVL, 0, COLOR, 0,
+		    DISP_REG_CONFIG_DISP_OVL0_MOUT_EN, OVL0_MOUT_EN_COLOR0,
+		    OVL0_MOUT_EN_COLOR0),
+	MMSYS_ROUTE(OVL, 0, COLOR, 0,
+		    DISP_REG_CONFIG_DISP_COLOR0_SEL_IN, COLOR0_SEL_IN_OVL0,
+		    COLOR0_SEL_IN_OVL0),
+	MMSYS_ROUTE(OVL, 0, RDMA, 0,
+		    DISP_REG_CONFIG_DISP_OVL_MOUT_EN, OVL_MOUT_EN_RDMA,
+		    OVL_MOUT_EN_RDMA),
+	MMSYS_ROUTE(OVL, 1, COLOR, 1,
+		    DISP_REG_CONFIG_DISP_OVL1_MOUT_EN, OVL1_MOUT_EN_COLOR1,
+		    OVL1_MOUT_EN_COLOR1),
+	MMSYS_ROUTE(OVL, 1, COLOR, 1,
+		    DISP_REG_CONFIG_DISP_COLOR1_SEL_IN, COLOR1_SEL_IN_OVL1,
+		    COLOR1_SEL_IN_OVL1),
+	MMSYS_ROUTE(RDMA, 0, DPI, 0,
+		    DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN, RDMA0_SOUT_MASK,
+		    RDMA0_SOUT_DPI0),
+	MMSYS_ROUTE(RDMA, 0, DPI, 1,
+		    DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN, RDMA0_SOUT_MASK,
+		    RDMA0_SOUT_DPI1),
+	MMSYS_ROUTE(RDMA, 0, DSI, 1,
+		    DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN, RDMA0_SOUT_MASK,
+		    RDMA0_SOUT_DSI1),
+	MMSYS_ROUTE(RDMA, 0, DSI, 2,
+		    DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN, RDMA0_SOUT_MASK,
+		    RDMA0_SOUT_DSI2),
+	MMSYS_ROUTE(RDMA, 0, DSI, 3,
+		    DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN, RDMA0_SOUT_MASK,
+		    RDMA0_SOUT_DSI3),
+	MMSYS_ROUTE(RDMA, 1, DPI, 0,
+		    DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN, RDMA1_SOUT_MASK,
+		    RDMA1_SOUT_DPI0),
+	MMSYS_ROUTE(RDMA, 1, DPI, 0,
+		    DISP_REG_CONFIG_DPI_SEL_IN, DPI0_SEL_IN_MASK,
+		    DPI0_SEL_IN_RDMA1),
+	MMSYS_ROUTE(RDMA, 1, DPI, 1,
+		    DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN, RDMA1_SOUT_MASK,
+		    RDMA1_SOUT_DPI1),
+	MMSYS_ROUTE(RDMA, 1, DPI, 1,
+		    DISP_REG_CONFIG_DPI_SEL_IN, DPI1_SEL_IN_MASK,
+		    DPI1_SEL_IN_RDMA1),
+	MMSYS_ROUTE(RDMA, 1, DSI, 0,
+		    DISP_REG_CONFIG_DSIE_SEL_IN, DSI0_SEL_IN_MASK,
+		    DSI0_SEL_IN_RDMA1),
+	MMSYS_ROUTE(RDMA, 1, DSI, 1,
+		    DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN, RDMA1_SOUT_MASK,
+		    RDMA1_SOUT_DSI1),
+	MMSYS_ROUTE(RDMA, 1, DSI, 1,
+		    DISP_REG_CONFIG_DSIO_SEL_IN, DSI1_SEL_IN_MASK,
+		    DSI1_SEL_IN_RDMA1),
+	MMSYS_ROUTE(RDMA, 1, DSI, 2,
+		    DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN, RDMA1_SOUT_MASK,
+		    RDMA1_SOUT_DSI2),
+	MMSYS_ROUTE(RDMA, 1, DSI, 2,
+		    DISP_REG_CONFIG_DSIE_SEL_IN, DSI2_SEL_IN_MASK,
+		    DSI2_SEL_IN_RDMA1),
+	MMSYS_ROUTE(RDMA, 1, DSI, 3,
+		    DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN, RDMA1_SOUT_MASK,
+		    RDMA1_SOUT_DSI3),
+	MMSYS_ROUTE(RDMA, 1, DSI, 3,
+		    DISP_REG_CONFIG_DSIO_SEL_IN, DSI3_SEL_IN_MASK,
+		    DSI3_SEL_IN_RDMA1),
+	MMSYS_ROUTE(RDMA, 2, DPI, 0,
+		    DISP_REG_CONFIG_DISP_RDMA2_SOUT, RDMA2_SOUT_MASK,
+		    RDMA2_SOUT_DPI0),
+	MMSYS_ROUTE(RDMA, 2, DPI, 0,
+		    DISP_REG_CONFIG_DPI_SEL_IN, DPI0_SEL_IN_MASK,
+		    DPI0_SEL_IN_RDMA2),
+	MMSYS_ROUTE(RDMA, 2, DPI, 1,
+		    DISP_REG_CONFIG_DISP_RDMA2_SOUT, RDMA2_SOUT_MASK,
+		    RDMA2_SOUT_DPI1),
+	MMSYS_ROUTE(RDMA, 2, DPI, 1,
+		    DISP_REG_CONFIG_DPI_SEL_IN, DPI1_SEL_IN_MASK,
+		    DPI1_SEL_IN_RDMA2),
+	MMSYS_ROUTE(RDMA, 2, DSI, 0,
+		    DISP_REG_CONFIG_DSIE_SEL_IN, DSI0_SEL_IN_MASK,
+		    DSI0_SEL_IN_RDMA2),
+	MMSYS_ROUTE(RDMA, 2, DSI, 1,
+		    DISP_REG_CONFIG_DISP_RDMA2_SOUT, RDMA2_SOUT_MASK,
+		    RDMA2_SOUT_DSI1),
+	MMSYS_ROUTE(RDMA, 2, DSI, 1,
+		    DISP_REG_CONFIG_DSIO_SEL_IN, DSI1_SEL_IN_MASK,
+		    DSI1_SEL_IN_RDMA2),
+	MMSYS_ROUTE(RDMA, 2, DSI, 2,
+		    DISP_REG_CONFIG_DISP_RDMA2_SOUT, RDMA2_SOUT_MASK,
+		    RDMA2_SOUT_DSI2),
+	MMSYS_ROUTE(RDMA, 2, DSI, 2,
+		    DISP_REG_CONFIG_DSIE_SEL_IN, DSI2_SEL_IN_MASK,
+		    DSI2_SEL_IN_RDMA2),
 };
 
 #endif /* __SOC_MEDIATEK_MTK_MMSYS_H */
diff --git a/drivers/soc/mediatek/mtk-socinfo.c b/drivers/soc/mediatek/mtk-socinfo.c
index 424a1eb82c20f..e294730ba7094 100644
--- a/drivers/soc/mediatek/mtk-socinfo.c
+++ b/drivers/soc/mediatek/mtk-socinfo.c
@@ -76,11 +76,14 @@ static int mtk_socinfo_create_socinfo_node(struct mtk_socinfo *mtk_socinfop)
 	if (!attrs)
 		return -ENOMEM;
 
-	if (data->marketing_name != NULL && data->marketing_name[0] != '\0')
+	if (data->marketing_name != NULL && data->marketing_name[0] != '\0') {
 		attrs->family = devm_kasprintf(mtk_socinfop->dev, GFP_KERNEL, "MediaTek %s",
 					       data->marketing_name);
-	else
+		if (!attrs->family)
+			return -ENOMEM;
+	} else {
 		attrs->family = soc_manufacturer;
+	}
 
 	attrs->soc_id = data->soc_name;
 	/*
diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig
index 2caadbbcf8307..fd4d4ecd2df0f 100644
--- a/drivers/soc/qcom/Kconfig
+++ b/drivers/soc/qcom/Kconfig
@@ -2,14 +2,25 @@
 #
 # QCOM Soc drivers
 #
-menu "Qualcomm SoC drivers"
+menuconfig QCOM_SOC
+	bool "Qualcomm SoC drivers"
+	depends on ARCH_QCOM || COMPILE_TEST
+	default ARCH_QCOM
+	help
+	  This collection of drivers is specific to Qualcomm System-on-Chips
+	  and most of them are necessary for a fully functional boot of the
+	  Linux kernel (plus a few debugging drivers).
+	  Drivers can be skipped when building Linux kernel not intended to run
+	  said processors.
+
+if QCOM_SOC
 
 config QCOM_AOSS_QMP
 	tristate "Qualcomm AOSS Driver"
-	depends on ARCH_QCOM || COMPILE_TEST
 	depends on MAILBOX
 	depends on COMMON_CLK && PM
 	select PM_GENERIC_DOMAINS
+	default ARCH_QCOM
 	help
 	  This driver provides the means of communicating with and controlling
 	  the low-power state for resources related to the remoteproc
@@ -18,8 +29,8 @@ config QCOM_AOSS_QMP
 
 config QCOM_COMMAND_DB
 	tristate "Qualcomm Command DB"
-	depends on ARCH_QCOM || COMPILE_TEST
 	depends on OF_RESERVED_MEM
+	default ARCH_QCOM
 	help
 	  Command DB queries shared memory by key string for shared system
 	  resources. Platform drivers that require to set state of a shared
@@ -27,8 +38,9 @@ config QCOM_COMMAND_DB
 	  SoC specific identifier and information for the shared resources.
 
 config QCOM_GENI_SE
-	tristate "QCOM GENI Serial Engine Driver"
-	depends on ARCH_QCOM || COMPILE_TEST
+	tristate "Qualcomm GENI Serial Engine Driver"
+	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This driver is used to manage Generic Interface (GENI) firmware based
 	  Qualcomm Technologies, Inc. Universal Peripheral (QUP) Wrapper. This
@@ -36,8 +48,9 @@ config QCOM_GENI_SE
 	  Engines present in the QUP.
 
 config QCOM_GSBI
-	tristate "QCOM General Serial Bus Interface"
-	depends on ARCH_QCOM || COMPILE_TEST
+	tristate "Qualcomm General Serial Bus Interface"
+	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	select MFD_SYSCON
 	help
 	  Say y here to enable GSBI support.  The GSBI provides control
@@ -45,27 +58,19 @@ config QCOM_GSBI
 	  devices to the output pins.
 
 config QCOM_LLCC
-	tristate "Qualcomm Technologies, Inc. LLCC driver"
-	depends on ARCH_QCOM || COMPILE_TEST
+	tristate "Qualcomm LLCC driver"
 	select REGMAP_MMIO
+	default m if ARCH_QCOM
 	help
 	  Qualcomm Technologies, Inc. platform specific
 	  Last Level Cache Controller(LLCC) driver for platforms such as,
 	  SDM845. This provides interfaces to clients that use the LLCC.
 	  Say yes here to enable LLCC slice driver.
 
-config QCOM_KRYO_L2_ACCESSORS
-	bool
-	depends on (ARCH_QCOM || COMPILE_TEST) && ARM64
-
-config QCOM_MDT_LOADER
-	tristate
-	select QCOM_SCM
-
 config QCOM_OCMEM
 	tristate "Qualcomm On Chip Memory (OCMEM) driver"
-	depends on ARCH_QCOM
 	select QCOM_SCM
+	default m if ARCH_QCOM
 	help
 	  The On Chip Memory (OCMEM) allocator allows various clients to
 	  allocate memory from OCMEM based on performance, latency and power
@@ -77,7 +82,7 @@ config QCOM_PD_MAPPER
 	select QCOM_QMI_HELPERS
 	select QCOM_PDR_MSG
 	select AUXILIARY_BUS
-	depends on NET && QRTR && (ARCH_QCOM || COMPILE_TEST)
+	depends on NET && QRTR
 	default QCOM_RPROC_COMMON
 	help
 	  The Protection Domain Mapper maps registered services to the domains
@@ -85,12 +90,6 @@ config QCOM_PD_MAPPER
 	  implementation of the service. It is a simpler alternative to the
 	  userspace daemon.
 
-config QCOM_PDR_HELPERS
-	tristate
-	select QCOM_QMI_HELPERS
-	select QCOM_PDR_MSG
-	depends on NET
-
 config QCOM_PDR_MSG
 	tristate
 
@@ -113,9 +112,11 @@ config QCOM_PMIC_GLINK
 	depends on DRM
 	depends on NET
 	depends on OF
+	depends on ARM64 || COMPILE_TEST
 	select AUXILIARY_BUS
 	select QCOM_PDR_HELPERS
 	select DRM_AUX_HPD_BRIDGE
+	default m if ARCH_QCOM
 	help
 	  The Qualcomm PMIC GLINK driver provides access, over GLINK, to the
 	  USB and battery firmware running on one of the coprocessors in
@@ -124,13 +125,8 @@ config QCOM_PMIC_GLINK
 	  Say yes here to support USB-C and battery status on modern Qualcomm
 	  platforms.
 
-config QCOM_QMI_HELPERS
-	tristate
-	depends on NET
-
 config QCOM_RAMP_CTRL
 	tristate "Qualcomm Ramp Controller driver"
-	depends on ARCH_QCOM || COMPILE_TEST
 	help
 	  The Ramp Controller is used to program the sequence ID for pulse
 	  swallowing, enable sequence and link sequence IDs for the CPU
@@ -139,8 +135,8 @@ config QCOM_RAMP_CTRL
 
 config QCOM_RMTFS_MEM
 	tristate "Qualcomm Remote Filesystem memory driver"
-	depends on ARCH_QCOM || COMPILE_TEST
 	select QCOM_SCM
+	default m if ARCH_QCOM
 	help
 	  The Qualcomm remote filesystem memory driver is used for allocating
 	  and exposing regions of shared memory with remote processors for the
@@ -151,7 +147,6 @@ config QCOM_RMTFS_MEM
 
 config QCOM_RPM_MASTER_STATS
 	tristate "Qualcomm RPM Master stats"
-	depends on ARCH_QCOM || COMPILE_TEST
 	help
 	  The RPM Master sleep stats driver provides detailed per-subsystem
 	  sleep/wake data, read from the RPM message RAM. It can be used to
@@ -162,8 +157,8 @@ config QCOM_RPM_MASTER_STATS
 
 config QCOM_RPMH
 	tristate "Qualcomm RPM-Hardened (RPMH) Communication"
-	depends on ARCH_QCOM || COMPILE_TEST
 	depends on (QCOM_COMMAND_DB || !QCOM_COMMAND_DB)
+	default ARCH_QCOM
 	help
 	  Support for communication with the hardened-RPM blocks in
 	  Qualcomm Technologies Inc (QTI) SoCs. RPMH communication uses an
@@ -173,8 +168,8 @@ config QCOM_RPMH
 
 config QCOM_SMEM
 	tristate "Qualcomm Shared Memory Manager (SMEM)"
-	depends on ARCH_QCOM || COMPILE_TEST
 	depends on HWSPINLOCK
+	default ARCH_QCOM
 	help
 	  Say y here to enable support for the Qualcomm Shared Memory Manager.
 	  The driver provides an interface to items in a heap shared among all
@@ -182,9 +177,9 @@ config QCOM_SMEM
 
 config QCOM_SMD_RPM
 	tristate "Qualcomm Resource Power Manager (RPM) over SMD"
-	depends on ARCH_QCOM || COMPILE_TEST
 	depends on RPMSG
 	depends on RPMSG_QCOM_SMD || RPMSG_QCOM_SMD=n
+	default ARCH_QCOM
 	help
 	  If you say yes to this option, support will be included for the
 	  Resource Power Manager system found in the Qualcomm 8974 based
@@ -205,6 +200,7 @@ config QCOM_SMP2P
 	depends on QCOM_SMEM
 	select QCOM_SMEM_STATE
 	select IRQ_DOMAIN
+	default ARCH_QCOM
 	help
 	  Say yes here to support the Qualcomm Shared Memory Point to Point
 	  protocol.
@@ -215,6 +211,7 @@ config QCOM_SMSM
 	depends on QCOM_SMEM
 	select QCOM_SMEM_STATE
 	select IRQ_DOMAIN
+	default ARCH_QCOM
 	help
 	  Say yes here to support the Qualcomm Shared Memory State Machine.
 	  The state machine is represented by bits in shared memory.
@@ -223,24 +220,26 @@ config QCOM_SOCINFO
 	tristate "Qualcomm socinfo driver"
 	depends on QCOM_SMEM
 	select SOC_BUS
+	default m if ARCH_QCOM
 	help
 	 Say yes here to support the Qualcomm socinfo driver, providing
 	 information about the SoC to user space.
 
 config QCOM_SPM
 	tristate "Qualcomm Subsystem Power Manager (SPM)"
-	depends on ARCH_QCOM || COMPILE_TEST
-	select QCOM_SCM
+	depends on QCOM_SCM
+	default ARCH_QCOM if ARM
 	help
 	  Enable the support for the Qualcomm Subsystem Power Manager, used
 	  to manage cores, L2 low power modes and to configure the internal
 	  Adaptive Voltage Scaler parameters, where supported.
 
 config QCOM_STATS
-	tristate "Qualcomm Technologies, Inc. (QTI) Sleep stats driver"
-	depends on (ARCH_QCOM && DEBUG_FS) || COMPILE_TEST
+	tristate "Qualcomm Sleep stats driver"
+	depends on DEBUG_FS || COMPILE_TEST
 	depends on QCOM_SMEM
 	depends on QCOM_AOSS_QMP || QCOM_AOSS_QMP=n
+	default m if ARCH_QCOM
 	help
 	  Qualcomm Technologies, Inc. (QTI) Sleep stats driver to read
 	  the shared memory exported by the remote processor related to
@@ -249,18 +248,18 @@ config QCOM_STATS
 
 config QCOM_WCNSS_CTRL
 	tristate "Qualcomm WCNSS control driver"
-	depends on ARCH_QCOM || COMPILE_TEST
 	depends on RPMSG
+	default m if ARCH_QCOM
 	help
 	  Client driver for the WCNSS_CTRL SMD channel, used to download nv
 	  firmware to a newly booted WCNSS chip.
 
 config QCOM_APR
 	tristate "Qualcomm APR/GPR Bus (Asynchronous/Generic Packet Router)"
-	depends on ARCH_QCOM || COMPILE_TEST
 	depends on RPMSG
 	depends on NET
 	select QCOM_PDR_HELPERS
+	default m if ARCH_QCOM
 	help
 	  Enable APR IPC protocol support between
 	  application processor and QDSP6. APR is
@@ -268,10 +267,11 @@ config QCOM_APR
 	  ASM, ADM and AFE modules.
 
 config QCOM_ICC_BWMON
-	tristate "QCOM Interconnect Bandwidth Monitor driver"
-	depends on ARCH_QCOM || COMPILE_TEST
+	tristate "Qualcomm Interconnect Bandwidth Monitor driver"
+	depends on ARM64 || COMPILE_TEST
 	select PM_OPP
 	select REGMAP_MMIO
+	default m if ARCH_QCOM
 	help
 	  Sets up driver monitoring bandwidth on various interconnects and
 	  based on that voting for interconnect bandwidth, adjusting their
@@ -282,20 +282,42 @@ config QCOM_ICC_BWMON
 	  the fixed bandwidth votes from cpufreq (CPU nodes) thus achieve high
 	  memory throughput even with lower CPU frequencies.
 
-config QCOM_INLINE_CRYPTO_ENGINE
-	tristate
-	select QCOM_SCM
-
 config QCOM_PBS
-	tristate "PBS trigger support for Qualcomm Technologies, Inc. PMICS"
+	tristate "PBS trigger support for Qualcomm PMICs"
 	depends on SPMI
+	default m if ARCH_QCOM
 	help
 	  This driver supports configuring software programmable boot sequencer (PBS)
 	  trigger event through PBS RAM on Qualcomm Technologies, Inc. PMICs.
 	  This module provides the APIs to the client drivers that wants to send the
 	  PBS trigger event to the PBS RAM.
 
-endmenu
+endif
+
+# Options selected by other drivers from different subsystems must be outside
+# of the menuconfig if-block:
+
+config QCOM_INLINE_CRYPTO_ENGINE
+	tristate
+	select QCOM_SCM
+
+config QCOM_KRYO_L2_ACCESSORS
+	bool
+	depends on ARM64
+
+config QCOM_MDT_LOADER
+	tristate
+	select QCOM_SCM
+
+config QCOM_PDR_HELPERS
+	tristate
+	select QCOM_QMI_HELPERS
+	select QCOM_PDR_MSG
+	depends on NET
+
+config QCOM_QMI_HELPERS
+	tristate
+	depends on NET
 
 config QCOM_UBWC_CONFIG
 	tristate
diff --git a/drivers/soc/qcom/cmd-db.c b/drivers/soc/qcom/cmd-db.c
index 84a75d8c4b702..3f1eee7843876 100644
--- a/drivers/soc/qcom/cmd-db.c
+++ b/drivers/soc/qcom/cmd-db.c
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2016-2018, 2020, The Linux Foundation. All rights reserved.
  * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
diff --git a/drivers/soc/qcom/ice.c b/drivers/soc/qcom/ice.c
index 5f20108aa03eb..e256595b9826f 100644
--- a/drivers/soc/qcom/ice.c
+++ b/drivers/soc/qcom/ice.c
@@ -126,8 +126,8 @@ static bool qcom_ice_check_supported(struct qcom_ice *ice)
 	int minor = FIELD_GET(GENMASK(23, 16), regval);
 	int step = FIELD_GET(GENMASK(15, 0), regval);
 
-	/* For now this driver only supports ICE version 3 and 4. */
-	if (major != 3 && major != 4) {
+	/* Support ICE version 3 and above. */
+	if (major < 3) {
 		dev_warn(dev, "Unsupported ICE version: v%d.%d.%d\n",
 			 major, minor, step);
 		return false;
diff --git a/drivers/soc/qcom/llcc-qcom.c b/drivers/soc/qcom/llcc-qcom.c
index 8948b5fd42d2a..22c8099cf6bbb 100644
--- a/drivers/soc/qcom/llcc-qcom.c
+++ b/drivers/soc/qcom/llcc-qcom.c
@@ -2305,6 +2305,20 @@ static const struct llcc_slice_config sdm845_data[] =  {{
 	},
 };
 
+static const struct llcc_slice_config shikra_data[] = {
+	{
+		.usecase_id = LLCC_ECC,
+		.slice_id = 26,
+		.max_cap = 256,
+		.priority = 3,
+		.fixed_size = true,
+		.bonus_ways = 0x3,
+		.cache_mode = 0,
+		.activate_on_init = true,
+		.vict_prio = true,
+	},
+};
+
 static const struct llcc_slice_config sm6350_data[] =  {
 	{
 		.usecase_id = LLCC_CPUSS,
@@ -4575,6 +4589,15 @@ static const struct qcom_llcc_config sdm845_cfg[] = {
 	},
 };
 
+static const struct qcom_llcc_config shikra_cfg[] = {
+	{
+		.sct_data	= shikra_data,
+		.size		= ARRAY_SIZE(shikra_data),
+		.reg_offset	= llcc_v2_1_reg_offset,
+		.edac_reg_offset = &llcc_v2_1_edac_reg_offset,
+	},
+};
+
 static const struct qcom_llcc_config sm6350_cfg[] = {
 	{
 		.sct_data	= sm6350_data,
@@ -4752,6 +4775,11 @@ static const struct qcom_sct_config sdm845_cfgs = {
 	.num_config	= ARRAY_SIZE(sdm845_cfg),
 };
 
+static const struct qcom_sct_config shikra_cfgs = {
+	.llcc_config	= shikra_cfg,
+	.num_config	= ARRAY_SIZE(shikra_cfg),
+};
+
 static const struct qcom_sct_config sm6350_cfgs = {
 	.llcc_config	= sm6350_cfg,
 	.num_config	= ARRAY_SIZE(sm6350_cfg),
@@ -5632,6 +5660,7 @@ static const struct of_device_id qcom_llcc_of_match[] = {
 	{ .compatible = "qcom,sc8280xp-llcc", .data = &sc8280xp_cfgs },
 	{ .compatible = "qcom,sdm670-llcc", .data = &sdm670_cfgs },
 	{ .compatible = "qcom,sdm845-llcc", .data = &sdm845_cfgs },
+	{ .compatible = "qcom,shikra-llcc", .data = &shikra_cfgs },
 	{ .compatible = "qcom,sm6350-llcc", .data = &sm6350_cfgs },
 	{ .compatible = "qcom,sm7150-llcc", .data = &sm7150_cfgs },
 	{ .compatible = "qcom,sm8150-llcc", .data = &sm8150_cfgs },
diff --git a/drivers/soc/qcom/mdt_loader.c b/drivers/soc/qcom/mdt_loader.c
index c004d444d6985..137992456b712 100644
--- a/drivers/soc/qcom/mdt_loader.c
+++ b/drivers/soc/qcom/mdt_loader.c
@@ -13,7 +13,7 @@
 #include <linux/firmware.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/firmware/qcom/qcom_scm.h>
+#include <linux/firmware/qcom/qcom_pas.h>
 #include <linux/sizes.h>
 #include <linux/slab.h>
 #include <linux/soc/qcom/mdt_loader.h>
@@ -229,7 +229,7 @@ EXPORT_SYMBOL_GPL(qcom_mdt_read_metadata);
 
 static int __qcom_mdt_pas_init(struct device *dev, const struct firmware *fw,
 			       const char *fw_name, int pas_id, phys_addr_t mem_phys,
-			       struct qcom_scm_pas_context *ctx)
+			       struct qcom_pas_context *ctx)
 {
 	const struct elf32_phdr *phdrs;
 	const struct elf32_phdr *phdr;
@@ -271,7 +271,7 @@ static int __qcom_mdt_pas_init(struct device *dev, const struct firmware *fw,
 		goto out;
 	}
 
-	ret = qcom_scm_pas_init_image(pas_id, metadata, metadata_len, ctx);
+	ret = qcom_pas_init_image(pas_id, metadata, metadata_len, ctx);
 	kfree(metadata);
 	if (ret) {
 		/* Invalid firmware metadata */
@@ -280,7 +280,7 @@ static int __qcom_mdt_pas_init(struct device *dev, const struct firmware *fw,
 	}
 
 	if (relocate) {
-		ret = qcom_scm_pas_mem_setup(pas_id, mem_phys, max_addr - min_addr);
+		ret = qcom_pas_mem_setup(pas_id, mem_phys, max_addr - min_addr);
 		if (ret) {
 			/* Unable to set up relocation */
 			dev_err(dev, "error %d setting up firmware %s\n", ret, fw_name);
@@ -472,7 +472,7 @@ EXPORT_SYMBOL_GPL(qcom_mdt_load);
  * firmware segments (e.g., .bXX files). Authentication of the segments done
  * by a separate call.
  *
- * The PAS context must be initialized using qcom_scm_pas_context_init()
+ * The PAS context must be initialized using devm_qcom_pas_context_alloc()
  * prior to invoking this function.
  *
  * @ctx:        Pointer to the PAS (Peripheral Authentication Service) context
@@ -483,7 +483,7 @@ EXPORT_SYMBOL_GPL(qcom_mdt_load);
  *
  * Return: 0 on success or a negative error code on failure.
  */
-int qcom_mdt_pas_load(struct qcom_scm_pas_context *ctx, const struct firmware *fw,
+int qcom_mdt_pas_load(struct qcom_pas_context *ctx, const struct firmware *fw,
 		      const char *firmware, void *mem_region, phys_addr_t *reloc_base)
 {
 	int ret;
diff --git a/drivers/soc/qcom/qcom_pd_mapper.c b/drivers/soc/qcom/qcom_pd_mapper.c
index 0dc1a7946050c..64ca99d7fad92 100644
--- a/drivers/soc/qcom/qcom_pd_mapper.c
+++ b/drivers/soc/qcom/qcom_pd_mapper.c
@@ -310,6 +310,24 @@ static const struct qcom_pdm_domain_data cdsp_root_pd = {
 	.services = { NULL },
 };
 
+static const struct qcom_pdm_domain_data cdsp1_root_pd = {
+	.domain = "msm/cdsp1/root_pd",
+	.instance_id = 125,
+	.services = { NULL },
+};
+
+static const struct qcom_pdm_domain_data gpdsp_root_pd = {
+	.domain = "msm/gpdsp/root_pd",
+	.instance_id = 192,
+	.services = { NULL },
+};
+
+static const struct qcom_pdm_domain_data gpdsp1_root_pd = {
+	.domain = "msm/gpdsp1/root_pd",
+	.instance_id = 241,
+	.services = { NULL },
+};
+
 static const struct qcom_pdm_domain_data slpi_root_pd = {
 	.domain = "msm/slpi/root_pd",
 	.instance_id = 90,
@@ -425,6 +443,24 @@ static const struct qcom_pdm_domain_data *qcs615_domains[] = {
 	NULL,
 };
 
+static const struct qcom_pdm_domain_data *qcs8300_domains[] = {
+	&adsp_audio_pd,
+	&adsp_root_pd,
+	&cdsp_root_pd,
+	&gpdsp_root_pd,
+	NULL,
+};
+
+static const struct qcom_pdm_domain_data *sa8775p_domains[] = {
+	&adsp_audio_pd,
+	&adsp_root_pd,
+	&cdsp_root_pd,
+	&cdsp1_root_pd,
+	&gpdsp_root_pd,
+	&gpdsp1_root_pd,
+	NULL,
+};
+
 static const struct qcom_pdm_domain_data *sc7180_domains[] = {
 	&adsp_audio_pd,
 	&adsp_root_pd_pdr,
@@ -602,6 +638,8 @@ static const struct of_device_id qcom_pdm_domains[] __maybe_unused = {
 	{ .compatible = "qcom,qcm6490", .data = sc7280_domains, },
 	{ .compatible = "qcom,qcs404", .data = qcs404_domains, },
 	{ .compatible = "qcom,qcs615", .data = qcs615_domains, },
+	{ .compatible = "qcom,qcs8300", .data = qcs8300_domains, },
+	{ .compatible = "qcom,sa8775p", .data = sa8775p_domains, },
 	{ .compatible = "qcom,sc7180", .data = sc7180_domains, },
 	{ .compatible = "qcom,sc7280", .data = sc7280_domains, },
 	{ .compatible = "qcom,sc8180x", .data = sc8180x_domains, },
diff --git a/drivers/soc/qcom/qcom_stats.c b/drivers/soc/qcom/qcom_stats.c
index 2e380faf90803..e7122444f12a3 100644
--- a/drivers/soc/qcom/qcom_stats.c
+++ b/drivers/soc/qcom/qcom_stats.c
@@ -54,6 +54,8 @@ static const struct subsystem_data subsystems[] = {
 	{ "cdsp1", 607, 12 },
 	{ "gpdsp0", 607, 17 },
 	{ "gpdsp1", 607, 18 },
+	{ "soccp", 607, 19 },
+	{ "dcp", 607, 22 },
 	{ "slpi", 608, 3 },
 	{ "gpu", 609, 0 },
 	{ "display", 610, 0 },
diff --git a/drivers/soc/qcom/rpmh-rsc.c b/drivers/soc/qcom/rpmh-rsc.c
index c6f7d5c9c493d..66928ca40b9aa 100644
--- a/drivers/soc/qcom/rpmh-rsc.c
+++ b/drivers/soc/qcom/rpmh-rsc.c
@@ -944,17 +944,30 @@ static int rpmh_rsc_pd_callback(struct notifier_block *nfb,
 	return NOTIFY_OK;
 }
 
+static void rpmh_rsc_pd_detach(void *data)
+{
+	dev_pm_genpd_remove_notifier(data);
+}
+
 static int rpmh_rsc_pd_attach(struct rsc_drv *drv, struct device *dev)
 {
 	int ret;
 
-	pm_runtime_enable(dev);
+	ret = devm_pm_runtime_enable(dev);
+	if (ret)
+		return ret;
+
 	drv->genpd_nb.notifier_call = rpmh_rsc_pd_callback;
 	ret = dev_pm_genpd_add_notifier(dev, &drv->genpd_nb);
 	if (ret)
-		pm_runtime_disable(dev);
+		return ret;
 
-	return ret;
+	return devm_add_action_or_reset(dev, rpmh_rsc_pd_detach, dev);
+}
+
+static void rpmh_rsc_cpu_pm_unregister(void *data)
+{
+	cpu_pm_unregister_notifier(data);
 }
 
 static int rpmh_probe_tcs_config(struct platform_device *pdev, struct rsc_drv *drv)
@@ -1107,7 +1120,15 @@ static int rpmh_rsc_probe(struct platform_device *pdev)
 				return ret;
 		} else {
 			drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback;
-			cpu_pm_register_notifier(&drv->rsc_pm);
+			ret = cpu_pm_register_notifier(&drv->rsc_pm);
+			if (ret)
+				return ret;
+
+			ret = devm_add_action_or_reset(&pdev->dev,
+						       rpmh_rsc_cpu_pm_unregister,
+						       &drv->rsc_pm);
+			if (ret)
+				return ret;
 		}
 	}
 
@@ -1122,13 +1143,7 @@ static int rpmh_rsc_probe(struct platform_device *pdev)
 	dev_set_drvdata(&pdev->dev, drv);
 	drv->dev = &pdev->dev;
 
-	ret = devm_of_platform_populate(&pdev->dev);
-	if (ret && pdev->dev.pm_domain) {
-		dev_pm_genpd_remove_notifier(&pdev->dev);
-		pm_runtime_disable(&pdev->dev);
-	}
-
-	return ret;
+	return devm_of_platform_populate(&pdev->dev);
 }
 
 static const struct of_device_id rpmh_drv_match[] = {
diff --git a/drivers/soc/qcom/socinfo.c b/drivers/soc/qcom/socinfo.c
index af418adad7aaf..1836e226a2945 100644
--- a/drivers/soc/qcom/socinfo.c
+++ b/drivers/soc/qcom/socinfo.c
@@ -425,6 +425,7 @@ static const struct soc_id soc_id[] = {
 	{ qcom_board_id(IPQ8078) },
 	{ qcom_board_id(SDM636) },
 	{ qcom_board_id(SDA636) },
+	{ qcom_board_id(SDM850) },
 	{ qcom_board_id(SDM632) },
 	{ qcom_board_id(SDA632) },
 	{ qcom_board_id(SDA450) },
diff --git a/drivers/soc/qcom/ubwc_config.c b/drivers/soc/qcom/ubwc_config.c
index 3fe47d8f0f638..0209a02c390f0 100644
--- a/drivers/soc/qcom/ubwc_config.c
+++ b/drivers/soc/qcom/ubwc_config.c
@@ -16,287 +16,155 @@ static const struct qcom_ubwc_cfg_data no_ubwc_data = {
 	/* no UBWC, no HBB */
 };
 
-static const struct qcom_ubwc_cfg_data eliza_data = {
-	.ubwc_enc_version = UBWC_5_0,
-	.ubwc_dec_version = UBWC_5_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
-	.ubwc_bank_spread = true,
-	/* TODO: highest_bank_bit = 14 for LP_DDR4 */
+static const struct qcom_ubwc_cfg_data ubwc_0_0_hbb15 = {
+	/* no UBWC */
 	.highest_bank_bit = 15,
-	.macrotile_mode = true,
 };
 
-static const struct qcom_ubwc_cfg_data kaanapali_data = {
-	.ubwc_enc_version = UBWC_6_0,
-	.ubwc_dec_version = UBWC_6_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
-	.ubwc_bank_spread = true,
-	.highest_bank_bit = 16,
-	.macrotile_mode = true,
-};
-
-static const struct qcom_ubwc_cfg_data msm8937_data = {
+static const struct qcom_ubwc_cfg_data ubwc_1_0_hbb14 = {
 	.ubwc_enc_version = UBWC_1_0,
-	.ubwc_dec_version = UBWC_1_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL1 |
-			UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
 	.highest_bank_bit = 14,
 };
 
-static const struct qcom_ubwc_cfg_data msm8998_data = {
+static const struct qcom_ubwc_cfg_data ubwc_1_0_hbb15 = {
 	.ubwc_enc_version = UBWC_1_0,
-	.ubwc_dec_version = UBWC_1_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL1 |
-			UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
-	.highest_bank_bit = 15,
-};
-
-static const struct qcom_ubwc_cfg_data qcm2290_data = {
-	/* no UBWC */
 	.highest_bank_bit = 15,
 };
 
-static const struct qcom_ubwc_cfg_data sa8775p_data = {
-	.ubwc_enc_version = UBWC_4_0,
-	.ubwc_dec_version = UBWC_4_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL3,
-	.ubwc_bank_spread = true,
-	.highest_bank_bit = 13,
-	.macrotile_mode = true,
-};
-
-static const struct qcom_ubwc_cfg_data sar2130p_data = {
-	.ubwc_enc_version = UBWC_3_0, /* 4.0.2 in hw */
-	.ubwc_dec_version = UBWC_4_3,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
-	.ubwc_bank_spread = true,
-	.highest_bank_bit = 13,
-	.macrotile_mode = true,
-};
-
-static const struct qcom_ubwc_cfg_data sc7180_data = {
-	.ubwc_enc_version = UBWC_2_0,
-	.ubwc_dec_version = UBWC_2_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
-	.ubwc_bank_spread = true,
-	.highest_bank_bit = 14,
-};
-
-static const struct qcom_ubwc_cfg_data sc7280_data = {
-	.ubwc_enc_version = UBWC_3_0,
-	.ubwc_dec_version = UBWC_4_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
-	.ubwc_bank_spread = true,
-	.highest_bank_bit = 14,
-	.macrotile_mode = true,
-};
-
-static const struct qcom_ubwc_cfg_data sc8180x_data = {
-	.ubwc_enc_version = UBWC_3_0,
-	.ubwc_dec_version = UBWC_3_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
-	.highest_bank_bit = 16,
-	.macrotile_mode = true,
-};
-
-static const struct qcom_ubwc_cfg_data sc8280xp_data = {
-	.ubwc_enc_version = UBWC_4_0,
-	.ubwc_dec_version = UBWC_4_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
-	.ubwc_bank_spread = true,
-	.highest_bank_bit = 16,
-	.macrotile_mode = true,
-};
-
-static const struct qcom_ubwc_cfg_data sdm670_data = {
+static const struct qcom_ubwc_cfg_data ubwc_2_0_hbb14 = {
 	.ubwc_enc_version = UBWC_2_0,
-	.ubwc_dec_version = UBWC_2_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
 	.highest_bank_bit = 14,
 };
 
-static const struct qcom_ubwc_cfg_data sdm845_data = {
+static const struct qcom_ubwc_cfg_data ubwc_2_0_hbb15 = {
 	.ubwc_enc_version = UBWC_2_0,
-	.ubwc_dec_version = UBWC_2_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
 	.highest_bank_bit = 15,
 };
 
-static const struct qcom_ubwc_cfg_data sm6115_data = {
-	.ubwc_enc_version = UBWC_1_0,
-	.ubwc_dec_version = UBWC_2_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL1 |
-			UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
-	.ubwc_bank_spread = true,
-	.highest_bank_bit = 14,
+static const struct qcom_ubwc_cfg_data ubwc_3_0_hbb15 = {
+	.ubwc_enc_version = UBWC_3_0,
+	.highest_bank_bit = 15,
 };
 
-static const struct qcom_ubwc_cfg_data sm6125_data = {
-	.ubwc_enc_version = UBWC_1_0,
-	.ubwc_dec_version = UBWC_3_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL1 |
-			UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
-	.highest_bank_bit = 14,
+static const struct qcom_ubwc_cfg_data ubwc_3_1_hbb13 = {
+	.ubwc_enc_version = UBWC_3_1,
+	.highest_bank_bit = 13,
 };
 
-static const struct qcom_ubwc_cfg_data sm6150_data = {
-	.ubwc_enc_version = UBWC_2_0,
-	.ubwc_dec_version = UBWC_2_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
+static const struct qcom_ubwc_cfg_data ubwc_3_1_hbb14  = {
+	.ubwc_enc_version = UBWC_3_1,
 	.highest_bank_bit = 14,
 };
 
-static const struct qcom_ubwc_cfg_data sm6350_data = {
-	.ubwc_enc_version = UBWC_2_0,
-	.ubwc_dec_version = UBWC_2_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
-	.ubwc_bank_spread = true,
-	.highest_bank_bit = 14,
+static const struct qcom_ubwc_cfg_data ubwc_3_1_hbb16  = {
+	.ubwc_enc_version = UBWC_3_1,
+	.highest_bank_bit = 16,
 };
 
-static const struct qcom_ubwc_cfg_data sm7150_data = {
-	.ubwc_enc_version = UBWC_2_0,
-	.ubwc_dec_version = UBWC_2_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
-	.highest_bank_bit = 14,
+static const struct qcom_ubwc_cfg_data ubwc_4_0_hbb16 = {
+	.ubwc_enc_version = UBWC_4_0,
+	.highest_bank_bit = 16,
 };
 
-static const struct qcom_ubwc_cfg_data sm8150_data = {
-	.ubwc_enc_version = UBWC_3_0,
-	.ubwc_dec_version = UBWC_3_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
+static const struct qcom_ubwc_cfg_data ubwc_5_0_hbb15 = {
+	.ubwc_enc_version = UBWC_5_0,
+	/* TODO: highest_bank_bit = 14 for LP_DDR4 */
 	.highest_bank_bit = 15,
 };
 
-static const struct qcom_ubwc_cfg_data sm8250_data = {
-	.ubwc_enc_version = UBWC_4_0,
-	.ubwc_dec_version = UBWC_4_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
-	.ubwc_bank_spread = true,
-	/* TODO: highest_bank_bit = 15 for LP_DDR4 */
+static const struct qcom_ubwc_cfg_data ubwc_5_0_hbb16 = {
+	.ubwc_enc_version = UBWC_5_0,
 	.highest_bank_bit = 16,
-	.macrotile_mode = true,
 };
 
-static const struct qcom_ubwc_cfg_data sm8350_data = {
-	.ubwc_enc_version = UBWC_4_0,
-	.ubwc_dec_version = UBWC_4_0,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
-	.ubwc_bank_spread = true,
-	/* TODO: highest_bank_bit = 15 for LP_DDR4 */
+static const struct qcom_ubwc_cfg_data ubwc_6_0_hbb16 = {
+	.ubwc_enc_version = UBWC_6_0,
 	.highest_bank_bit = 16,
-	.macrotile_mode = true,
 };
 
-static const struct qcom_ubwc_cfg_data sm8550_data = {
+static const struct qcom_ubwc_cfg_data sa8775p_data = {
 	.ubwc_enc_version = UBWC_4_0,
-	.ubwc_dec_version = UBWC_4_3,
-	.ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 |
-			UBWC_SWIZZLE_ENABLE_LVL3,
-	.ubwc_bank_spread = true,
-	/* TODO: highest_bank_bit = 15 for LP_DDR4 */
-	.highest_bank_bit = 16,
-	.macrotile_mode = true,
+	.flags = UBWC_FLAG_DISABLE_SWIZZLE_LVL2,
+	.highest_bank_bit = 13,
 };
 
-static const struct qcom_ubwc_cfg_data sm8750_data = {
+static const struct qcom_ubwc_cfg_data glymur_data = {
 	.ubwc_enc_version = UBWC_5_0,
-	.ubwc_dec_version = UBWC_5_0,
-	.ubwc_swizzle = 6,
-	.ubwc_bank_spread = true,
+	.flags = UBWC_FLAG_DISABLE_SWIZZLE_LVL2 |
+		 UBWC_FLAG_DISABLE_SWIZZLE_LVL3,
 	/* TODO: highest_bank_bit = 15 for LP_DDR4 */
 	.highest_bank_bit = 16,
-	.macrotile_mode = true,
 };
 
-static const struct qcom_ubwc_cfg_data glymur_data = {
-	.ubwc_enc_version = UBWC_5_0,
-	.ubwc_dec_version = UBWC_5_0,
-	.ubwc_swizzle = 0,
-	.ubwc_bank_spread = true,
-	/* TODO: highest_bank_bit = 15 for LP_DDR4 */
-	.highest_bank_bit = 16,
-	.macrotile_mode = true,
+static const struct qcom_ubwc_cfg_data milos_data = {
+	.ubwc_enc_version = UBWC_4_0,
+	.flags = UBWC_SWIZZLE_ENABLE_LVL2 |
+		 UBWC_SWIZZLE_ENABLE_LVL3,
+	/* TODO: highest_bank_bit = 14 for LP_DDR4 */
+	.highest_bank_bit = 15,
 };
 
 static const struct of_device_id qcom_ubwc_configs[] __maybe_unused = {
 	{ .compatible = "qcom,apq8016", .data = &no_ubwc_data },
 	{ .compatible = "qcom,apq8026", .data = &no_ubwc_data },
 	{ .compatible = "qcom,apq8074", .data = &no_ubwc_data },
-	{ .compatible = "qcom,apq8096", .data = &msm8998_data },
-	{ .compatible = "qcom,eliza", .data = &eliza_data, },
-	{ .compatible = "qcom,kaanapali", .data = &kaanapali_data, },
+	{ .compatible = "qcom,apq8096", .data = &ubwc_1_0_hbb15 },
+	{ .compatible = "qcom,eliza", .data = &ubwc_5_0_hbb15 },
 	{ .compatible = "qcom,glymur", .data = &glymur_data},
+	{ .compatible = "qcom,kaanapali", .data = &ubwc_6_0_hbb16 },
 	{ .compatible = "qcom,mahua", .data = &glymur_data },
+	{ .compatible = "qcom,milos", .data = &milos_data },
 	{ .compatible = "qcom,msm8226", .data = &no_ubwc_data },
 	{ .compatible = "qcom,msm8916", .data = &no_ubwc_data },
 	{ .compatible = "qcom,msm8917", .data = &no_ubwc_data },
-	{ .compatible = "qcom,msm8937", .data = &msm8937_data },
+	{ .compatible = "qcom,msm8937", .data = &ubwc_1_0_hbb14 },
 	{ .compatible = "qcom,msm8929", .data = &no_ubwc_data },
 	{ .compatible = "qcom,msm8939", .data = &no_ubwc_data },
-	{ .compatible = "qcom,msm8953", .data = &msm8937_data },
+	{ .compatible = "qcom,msm8953", .data = &ubwc_1_0_hbb14 },
 	{ .compatible = "qcom,msm8956", .data = &no_ubwc_data },
 	{ .compatible = "qcom,msm8974", .data = &no_ubwc_data },
 	{ .compatible = "qcom,msm8976", .data = &no_ubwc_data },
-	{ .compatible = "qcom,msm8996", .data = &msm8998_data },
-	{ .compatible = "qcom,msm8998", .data = &msm8998_data },
-	{ .compatible = "qcom,qcm2290", .data = &qcm2290_data, },
-	{ .compatible = "qcom,qcm6490", .data = &sc7280_data, },
-	{ .compatible = "qcom,qcs8300", .data = &sc8280xp_data, },
-	{ .compatible = "qcom,sa8155p", .data = &sm8150_data, },
-	{ .compatible = "qcom,sa8540p", .data = &sc8280xp_data, },
+	{ .compatible = "qcom,msm8996", .data = &ubwc_1_0_hbb15 },
+	{ .compatible = "qcom,msm8998", .data = &ubwc_1_0_hbb15 },
+	{ .compatible = "qcom,qcm2290", .data = &ubwc_0_0_hbb15, },
+	{ .compatible = "qcom,qcm6490", .data = &ubwc_3_1_hbb14, },
+	{ .compatible = "qcom,qcs8300", .data = &ubwc_4_0_hbb16, },
+	{ .compatible = "qcom,sa8155p", .data = &ubwc_3_0_hbb15, },
+	{ .compatible = "qcom,sa8540p", .data = &ubwc_4_0_hbb16, },
 	{ .compatible = "qcom,sa8775p", .data = &sa8775p_data, },
-	{ .compatible = "qcom,sar2130p", .data = &sar2130p_data },
-	{ .compatible = "qcom,sc7180", .data = &sc7180_data },
-	{ .compatible = "qcom,sc7280", .data = &sc7280_data, },
-	{ .compatible = "qcom,sc8180x", .data = &sc8180x_data, },
-	{ .compatible = "qcom,sc8280xp", .data = &sc8280xp_data, },
-	{ .compatible = "qcom,sda660", .data = &msm8937_data },
-	{ .compatible = "qcom,sdm450", .data = &msm8937_data },
-	{ .compatible = "qcom,sdm630", .data = &msm8937_data },
-	{ .compatible = "qcom,sdm632", .data = &msm8937_data },
-	{ .compatible = "qcom,sdm636", .data = &msm8937_data },
-	{ .compatible = "qcom,sdm660", .data = &msm8937_data },
-	{ .compatible = "qcom,sdm670", .data = &sdm670_data, },
-	{ .compatible = "qcom,sdm845", .data = &sdm845_data, },
-	{ .compatible = "qcom,sm4250", .data = &sm6115_data, },
-	{ .compatible = "qcom,sm6115", .data = &sm6115_data, },
-	{ .compatible = "qcom,sm6125", .data = &sm6125_data, },
-	{ .compatible = "qcom,sm6150", .data = &sm6150_data, },
-	{ .compatible = "qcom,sm6350", .data = &sm6350_data, },
-	{ .compatible = "qcom,sm6375", .data = &sm6350_data, },
-	{ .compatible = "qcom,sm7125", .data = &sc7180_data },
-	{ .compatible = "qcom,sm7150", .data = &sm7150_data, },
-	{ .compatible = "qcom,sm7225", .data = &sm6350_data, },
-	{ .compatible = "qcom,sm7325", .data = &sc7280_data, },
-	{ .compatible = "qcom,sm8150", .data = &sm8150_data, },
-	{ .compatible = "qcom,sm8250", .data = &sm8250_data, },
-	{ .compatible = "qcom,sm8350", .data = &sm8350_data, },
-	{ .compatible = "qcom,sm8450", .data = &sm8350_data, },
-	{ .compatible = "qcom,sm8550", .data = &sm8550_data, },
-	{ .compatible = "qcom,sm8650", .data = &sm8550_data, },
-	{ .compatible = "qcom,sm8750", .data = &sm8750_data, },
-	{ .compatible = "qcom,x1e80100", .data = &sm8550_data, },
-	{ .compatible = "qcom,x1p42100", .data = &sm8550_data, },
+	{ .compatible = "qcom,sar2130p", .data = &ubwc_3_1_hbb13 },
+	{ .compatible = "qcom,sc7180", .data = &ubwc_2_0_hbb14, },
+	{ .compatible = "qcom,sc7280", .data = &ubwc_3_1_hbb14, },
+	{ .compatible = "qcom,sc8180x", .data = &ubwc_3_1_hbb16, },
+	{ .compatible = "qcom,sc8280xp", .data = &ubwc_4_0_hbb16, },
+	{ .compatible = "qcom,sda660", .data = &ubwc_1_0_hbb14 },
+	{ .compatible = "qcom,sdm450", .data = &ubwc_1_0_hbb14 },
+	{ .compatible = "qcom,sdm630", .data = &ubwc_1_0_hbb14 },
+	{ .compatible = "qcom,sdm632", .data = &ubwc_1_0_hbb14 },
+	{ .compatible = "qcom,sdm636", .data = &ubwc_1_0_hbb14 },
+	{ .compatible = "qcom,sdm660", .data = &ubwc_1_0_hbb14 },
+	{ .compatible = "qcom,sdm670", .data = &ubwc_2_0_hbb14, },
+	{ .compatible = "qcom,sdm845", .data = &ubwc_2_0_hbb15, },
+	{ .compatible = "qcom,shikra", .data = &ubwc_0_0_hbb15, },
+	{ .compatible = "qcom,sm4250", .data = &ubwc_1_0_hbb14, },
+	{ .compatible = "qcom,sm6115", .data = &ubwc_1_0_hbb14, },
+	{ .compatible = "qcom,sm6125", .data = &ubwc_1_0_hbb14, },
+	{ .compatible = "qcom,sm6150", .data = &ubwc_2_0_hbb14, },
+	{ .compatible = "qcom,sm6350", .data = &ubwc_2_0_hbb14, },
+	{ .compatible = "qcom,sm6375", .data = &ubwc_2_0_hbb14, },
+	{ .compatible = "qcom,sm7125", .data = &ubwc_2_0_hbb14, },
+	{ .compatible = "qcom,sm7150", .data = &ubwc_2_0_hbb14, },
+	{ .compatible = "qcom,sm7225", .data = &ubwc_2_0_hbb14, },
+	{ .compatible = "qcom,sm7325", .data = &ubwc_3_1_hbb14, },
+	{ .compatible = "qcom,sm8150", .data = &ubwc_3_0_hbb15, },
+	{ .compatible = "qcom,sm8250", .data = &ubwc_4_0_hbb16, },
+	{ .compatible = "qcom,sm8350", .data = &ubwc_4_0_hbb16, },
+	{ .compatible = "qcom,sm8450", .data = &ubwc_4_0_hbb16, },
+	{ .compatible = "qcom,sm8550", .data = &ubwc_4_0_hbb16, },
+	{ .compatible = "qcom,sm8650", .data = &ubwc_4_0_hbb16, },
+	{ .compatible = "qcom,sm8750", .data = &ubwc_5_0_hbb16, },
+	{ .compatible = "qcom,x1e80100", .data = &ubwc_4_0_hbb16, },
+	{ .compatible = "qcom,x1p42100", .data = &ubwc_4_0_hbb16, },
 	{ }
 };
 
diff --git a/drivers/soc/renesas/Kconfig b/drivers/soc/renesas/Kconfig
index 2ab150d04bb1f..fdf18ed2dfc21 100644
--- a/drivers/soc/renesas/Kconfig
+++ b/drivers/soc/renesas/Kconfig
@@ -356,7 +356,6 @@ config ARCH_R8A779H0
 config ARCH_R8A78000
 	bool "ARM64 Platform support for R8A78000 (R-Car X5H)"
 	default y if ARCH_RENESAS
-	default ARCH_RENESAS
 	select ARCH_RCAR_GEN5
 	help
 	  This enables support for the Renesas R-Car X5H SoC.
diff --git a/drivers/soc/samsung/exynos-pmu.c b/drivers/soc/samsung/exynos-pmu.c
index d58376c38179b..f5fcdde9750e2 100644
--- a/drivers/soc/samsung/exynos-pmu.c
+++ b/drivers/soc/samsung/exynos-pmu.c
@@ -167,8 +167,8 @@ static const struct mfd_cell exynos_pmu_devs[] = {
  */
 struct regmap *exynos_get_pmu_regmap(void)
 {
-	struct device_node *np = of_find_matching_node(NULL,
-						      exynos_pmu_of_device_ids);
+	struct device_node *np __free(device_node) =
+		of_find_matching_node(NULL, exynos_pmu_of_device_ids);
 	if (np)
 		return exynos_get_pmu_regmap_by_phandle(np, NULL);
 	return ERR_PTR(-ENODEV);
diff --git a/drivers/soc/tegra/fuse/tegra-apbmisc.c b/drivers/soc/tegra/fuse/tegra-apbmisc.c
index 5b0e9dae231fb..6af69ccaa295c 100644
--- a/drivers/soc/tegra/fuse/tegra-apbmisc.c
+++ b/drivers/soc/tegra/fuse/tegra-apbmisc.c
@@ -116,7 +116,7 @@ bool tegra_is_silicon(void)
 
 u32 tegra_read_straps(void)
 {
-	WARN(!chipid, "Tegra ABP MISC not yet available\n");
+	WARN(!apbmisc_base, "Tegra ABP MISC not yet available\n");
 
 	return strapping;
 }
diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c
index f419a5395545f..41ed716d58573 100644
--- a/drivers/soc/tegra/pmc.c
+++ b/drivers/soc/tegra/pmc.c
@@ -3074,15 +3074,18 @@ static int tegra_pmc_probe(struct platform_device *pdev)
 		}
 	}
 
-	err = devm_register_sys_off_handler(&pdev->dev,
-					    SYS_OFF_MODE_RESTART,
-					    SYS_OFF_PRIO_LOW,
-					    tegra_pmc_restart_handler,
-					    pmc);
-	if (err) {
-		dev_err(&pdev->dev, "failed to register sys-off handler: %d\n",
-			err);
-		return err;
+	if (dev_to_node(&pdev->dev) == NUMA_NO_NODE) {
+		err = devm_register_sys_off_handler(&pdev->dev,
+						    SYS_OFF_MODE_RESTART,
+						    SYS_OFF_PRIO_LOW,
+						    tegra_pmc_restart_handler,
+						    pmc);
+		if (err) {
+			dev_err(&pdev->dev,
+				"failed to register sys-off handler: %d\n",
+				err);
+			return err;
+		}
 	}
 
 	/*
diff --git a/drivers/soundwire/bus.h b/drivers/soundwire/bus.h
index 8115c64dd48e2..44e4f51939176 100644
--- a/drivers/soundwire/bus.h
+++ b/drivers/soundwire/bus.h
@@ -73,7 +73,7 @@ struct sdw_msg {
 };
 
 /**
- * struct sdw_btp_section - Message section structure
+ * struct sdw_bpt_section - Message section structure
  * @addr: Start Register address accessed in the Slave
  * @len: number of bytes to transfer. More than 64Kb can be transferred
  * but a practical limit of SDW_BPT_MSG_MAX_BYTES is enforced.
@@ -87,7 +87,7 @@ struct sdw_bpt_section {
 };
 
 /**
- * struct sdw_btp_msg - Message structure
+ * struct sdw_bpt_msg - Message structure
  * @sec: Pointer to array of sections
  * @sections: Number of sections in the array
  * @dev_num: Slave device number
@@ -110,7 +110,7 @@ int sdw_find_row_index(int row);
 int sdw_find_col_index(int col);
 
 /**
- * sdw_port_runtime: Runtime port parameters for Master or Slave
+ * struct sdw_port_runtime - Runtime port parameters for Master or Slave
  *
  * @num: Port number. For audio streams, valid port number ranges from
  * [1,14]
@@ -133,7 +133,7 @@ struct sdw_port_runtime {
 };
 
 /**
- * sdw_slave_runtime: Runtime Stream parameters for Slave
+ * struct sdw_slave_runtime - Runtime Stream parameters for Slave
  *
  * @slave: Slave handle
  * @direction: Data direction for Slave
@@ -151,7 +151,7 @@ struct sdw_slave_runtime {
 };
 
 /**
- * sdw_master_runtime: Runtime stream parameters for Master
+ * struct sdw_master_runtime - Runtime stream parameters for Master
  *
  * @bus: Bus handle
  * @stream: Stream runtime handle
diff --git a/drivers/soundwire/dmi-quirks.c b/drivers/soundwire/dmi-quirks.c
index 32a46a2d90f7c..62cee6e2179da 100644
--- a/drivers/soundwire/dmi-quirks.c
+++ b/drivers/soundwire/dmi-quirks.c
@@ -15,6 +15,14 @@ struct adr_remap {
 	u64 remapped_adr;
 };
 
+static const struct adr_remap global_ghost_adr[] = {
+	{
+		0x000000D010010500ull,
+		0x0000000000000000ull
+	},
+	{}
+};
+
 /*
  * Some TigerLake devices based on an initial Intel BIOS do not expose
  * the correct _ADR in the DSDT.
@@ -178,6 +186,13 @@ static const struct dmi_system_id adr_remap_quirk_table[] = {
 		.driver_data = (void *)hp_omen_16,
 	},
 	/* PTL devices */
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUS"),
+			DMI_MATCH(DMI_BOARD_NAME, "B9406CAA"),
+		},
+		.driver_data = (void *)ghost_realtek,
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "ASUS"),
@@ -205,6 +220,7 @@ static const struct dmi_system_id adr_remap_quirk_table[] = {
 u64 sdw_dmi_override_adr(struct sdw_bus *bus, u64 addr)
 {
 	const struct dmi_system_id *dmi_id;
+	int i;
 
 	/* check if any address remap quirk applies */
 	dmi_id = dmi_first_match(adr_remap_quirk_table);
@@ -216,10 +232,20 @@ u64 sdw_dmi_override_adr(struct sdw_bus *bus, u64 addr)
 				dev_dbg(bus->dev, "remapped _ADR 0x%llx as 0x%llx\n",
 					addr, map->remapped_adr);
 				addr = map->remapped_adr;
-				break;
+				goto out;
 			}
 		}
 	}
 
+	/* remap the ghost ADRs */
+	for (i = 0; i < ARRAY_SIZE(global_ghost_adr); i++) {
+		if (global_ghost_adr[i].adr == addr) {
+			dev_dbg(bus->dev, "remapped _ADR 0x%llx as 0x%llx\n",
+				addr, global_ghost_adr[i].remapped_adr);
+			addr = global_ghost_adr[i].remapped_adr;
+			break;
+		}
+	}
+out:
 	return addr;
 }
diff --git a/drivers/soundwire/intel_auxdevice.c b/drivers/soundwire/intel_auxdevice.c
index 0b8107bec9abe..a8407560bf4fd 100644
--- a/drivers/soundwire/intel_auxdevice.c
+++ b/drivers/soundwire/intel_auxdevice.c
@@ -54,6 +54,7 @@ static struct wake_capable_part wake_capable_list[] = {
 	{0x01fa, 0x2A30},
 	{0x01fa, 0x2A3B},
 	{0x01fa, 0x4243},
+	{0x01fa, 0x4244},
 	{0x01fa, 0x4245},
 	{0x01fa, 0x4249},
 	{0x01fa, 0x4747},
diff --git a/drivers/soundwire/qcom.c b/drivers/soundwire/qcom.c
index 3d8f5a81eff19..3562802f42045 100644
--- a/drivers/soundwire/qcom.c
+++ b/drivers/soundwire/qcom.c
@@ -134,7 +134,6 @@
 #define TIMEOUT_MS						100
 #define QCOM_SWRM_MAX_RD_LEN					0x1
 #define DEFAULT_CLK_FREQ					9600000
-#define SWRM_MAX_DAIS						0xF
 #define SWR_INVALID_PARAM					0xFF
 #define SWR_HSTOP_MAX_VAL					0xF
 #define SWR_HSTART_MIN_VAL					0x0
@@ -215,7 +214,7 @@ struct qcom_swrm_ctrl {
 	u8 wcmd_id;
 	/* Port numbers are 1 - 14 */
 	struct qcom_swrm_port_config *pconfig;
-	struct sdw_stream_runtime *sruntime[SWRM_MAX_DAIS];
+	struct sdw_stream_runtime **sruntime;
 	enum sdw_slave_status status[SDW_MAX_DEVICES + 1];
 	int (*reg_read)(struct qcom_swrm_ctrl *ctrl, int reg, u32 *val);
 	int (*reg_write)(struct qcom_swrm_ctrl *ctrl, int reg, int val);
@@ -976,6 +975,20 @@ static enum sdw_command_response qcom_swrm_xfer_msg(struct sdw_bus *bus,
 	struct qcom_swrm_ctrl *ctrl = to_qcom_sdw(bus);
 	int ret, i, len;
 
+	if (msg->page) {
+		ret = qcom_swrm_cmd_fifo_wr_cmd(ctrl, msg->addr_page1,
+						msg->dev_num,
+						SDW_SCP_ADDRPAGE1);
+		if (ret)
+			return ret;
+
+		ret = qcom_swrm_cmd_fifo_wr_cmd(ctrl, msg->addr_page2,
+						msg->dev_num,
+						SDW_SCP_ADDRPAGE2);
+		if (ret)
+			return ret;
+	}
+
 	if (msg->flags == SDW_MSG_FLAG_READ) {
 		for (i = 0; i < msg->len;) {
 			len = min(msg->len - i, QCOM_SWRM_MAX_RD_LEN);
@@ -1271,7 +1284,7 @@ static int qcom_swrm_stream_alloc_ports(struct qcom_swrm_ctrl *ctrl,
 				else
 					pn = find_first_zero_bit(port_mask, maxport);
 
-				if (pn > maxport) {
+				if (pn >= maxport) {
 					dev_err(ctrl->dev, "All ports busy\n");
 					return -EBUSY;
 				}
@@ -1384,6 +1397,10 @@ static int qcom_swrm_register_dais(struct qcom_swrm_ctrl *ctrl)
 	struct device *dev = ctrl->dev;
 	int i;
 
+	ctrl->sruntime = devm_kcalloc(dev, num_dais, sizeof(*ctrl->sruntime), GFP_KERNEL);
+	if (!ctrl->sruntime)
+		return -ENOMEM;
+
 	/* PDM dais are only tested for now */
 	dais = devm_kcalloc(dev, num_dais, sizeof(*dais), GFP_KERNEL);
 	if (!dais)
diff --git a/drivers/spi/atmel-quadspi.c b/drivers/spi/atmel-quadspi.c
index aaf7f4c46b22f..e3a4697fabce7 100644
--- a/drivers/spi/atmel-quadspi.c
+++ b/drivers/spi/atmel-quadspi.c
@@ -1347,20 +1347,21 @@ static const struct atmel_qspi_ops atmel_qspi_sama7g5_ops = {
 
 static int atmel_qspi_probe(struct platform_device *pdev)
 {
+	struct device *dev = &pdev->dev;
 	struct spi_controller *ctrl;
 	struct atmel_qspi *aq;
 	struct resource *res;
 	int irq, err = 0;
 
-	ctrl = devm_spi_alloc_host(&pdev->dev, sizeof(*aq));
+	ctrl = devm_spi_alloc_host(dev, sizeof(*aq));
 	if (!ctrl)
 		return -ENOMEM;
 
 	aq = spi_controller_get_devdata(ctrl);
 
-	aq->caps = of_device_get_match_data(&pdev->dev);
+	aq->caps = of_device_get_match_data(dev);
 	if (!aq->caps) {
-		dev_err(&pdev->dev, "Could not retrieve QSPI caps\n");
+		dev_err(dev, "Could not retrieve QSPI caps\n");
 		return -EINVAL;
 	}
 
@@ -1387,45 +1388,40 @@ static int atmel_qspi_probe(struct platform_device *pdev)
 	/* Map the registers */
 	aq->regs = devm_platform_ioremap_resource_byname(pdev, "qspi_base");
 	if (IS_ERR(aq->regs))
-		return dev_err_probe(&pdev->dev, PTR_ERR(aq->regs),
+		return dev_err_probe(dev, PTR_ERR(aq->regs),
 				     "missing registers\n");
 
 	/* Map the AHB memory */
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "qspi_mmap");
-	aq->mem = devm_ioremap_resource(&pdev->dev, res);
+	aq->mem = devm_ioremap_resource(dev, res);
 	if (IS_ERR(aq->mem))
-		return dev_err_probe(&pdev->dev, PTR_ERR(aq->mem),
+		return dev_err_probe(dev, PTR_ERR(aq->mem),
 				     "missing AHB memory\n");
 
 	aq->mmap_size = resource_size(res);
 	aq->mmap_phys_base = (dma_addr_t)res->start;
 
 	/* Get the peripheral clock */
-	aq->pclk = devm_clk_get_enabled(&pdev->dev, "pclk");
+	aq->pclk = devm_clk_get_enabled(dev, "pclk");
 	if (IS_ERR(aq->pclk))
-		aq->pclk = devm_clk_get_enabled(&pdev->dev, NULL);
+		aq->pclk = devm_clk_get_enabled(dev, NULL);
 
 	if (IS_ERR(aq->pclk))
-		return dev_err_probe(&pdev->dev, PTR_ERR(aq->pclk),
+		return dev_err_probe(dev, PTR_ERR(aq->pclk),
 				     "missing peripheral clock\n");
 
 	if (aq->caps->has_qspick) {
 		/* Get the QSPI system clock */
-		aq->qspick = devm_clk_get_enabled(&pdev->dev, "qspick");
-		if (IS_ERR(aq->qspick)) {
-			dev_err(&pdev->dev, "missing system clock\n");
-			err = PTR_ERR(aq->qspick);
-			return err;
-		}
-
+		aq->qspick = devm_clk_get_enabled(dev, "qspick");
+		if (IS_ERR(aq->qspick))
+			return dev_err_probe(dev, PTR_ERR(aq->qspick),
+					     "missing system clock\n");
 	} else if (aq->caps->has_gclk) {
 		/* Get the QSPI generic clock */
-		aq->gclk = devm_clk_get(&pdev->dev, "gclk");
-		if (IS_ERR(aq->gclk)) {
-			dev_err(&pdev->dev, "missing Generic clock\n");
-			err = PTR_ERR(aq->gclk);
-			return err;
-		}
+		aq->gclk = devm_clk_get(dev, "gclk");
+		if (IS_ERR(aq->gclk))
+			return dev_err_probe(dev, PTR_ERR(aq->gclk),
+					     "missing Generic clock\n");
 	}
 
 	if (aq->caps->has_dma) {
@@ -1439,15 +1435,15 @@ static int atmel_qspi_probe(struct platform_device *pdev)
 	if (irq < 0)
 		return irq;
 
-	err = devm_request_irq(&pdev->dev, irq, atmel_qspi_interrupt,
-			       0, dev_name(&pdev->dev), aq);
+	err = devm_request_irq(dev, irq, atmel_qspi_interrupt,
+			       0, dev_name(dev), aq);
 	if (err)
 		return err;
 
-	pm_runtime_set_autosuspend_delay(&pdev->dev, 500);
-	pm_runtime_use_autosuspend(&pdev->dev);
-	devm_pm_runtime_set_active_enabled(&pdev->dev);
-	devm_pm_runtime_get_noresume(&pdev->dev);
+	pm_runtime_set_autosuspend_delay(dev, 500);
+	pm_runtime_use_autosuspend(dev);
+	devm_pm_runtime_set_active_enabled(dev);
+	devm_pm_runtime_get_noresume(dev);
 
 	err = atmel_qspi_init(aq);
 	if (err)
@@ -1457,7 +1453,7 @@ static int atmel_qspi_probe(struct platform_device *pdev)
 	if (err)
 		return err;
 
-	pm_runtime_put_autosuspend(&pdev->dev);
+	pm_runtime_put_autosuspend(dev);
 
 	return 0;
 }
diff --git a/drivers/spi/spi-altera-platform.c b/drivers/spi/spi-altera-platform.c
index 3de7df73f2165..4fe69ce9da16b 100644
--- a/drivers/spi/spi-altera-platform.c
+++ b/drivers/spi/spi-altera-platform.c
@@ -77,10 +77,8 @@ static int altera_spi_probe(struct platform_device *pdev)
 		struct resource *regoff;
 
 		hw->regmap = dev_get_regmap(pdev->dev.parent, NULL);
-		if (!hw->regmap) {
-			dev_err(&pdev->dev, "get regmap failed\n");
-			return -ENODEV;
-		}
+		if (!hw->regmap)
+			return dev_err_probe(&pdev->dev, -ENODEV, "get regmap failed\n");
 
 		regoff = platform_get_resource(pdev, IORESOURCE_REG, 0);
 		if (regoff)
@@ -94,10 +92,9 @@ static int altera_spi_probe(struct platform_device *pdev)
 
 		hw->regmap = devm_regmap_init_mmio(&pdev->dev, res,
 						   &spi_altera_config);
-		if (IS_ERR(hw->regmap)) {
-			dev_err(&pdev->dev, "regmap mmio init failed\n");
-			return PTR_ERR(hw->regmap);
-		}
+		if (IS_ERR(hw->regmap))
+			return dev_err_probe(&pdev->dev, PTR_ERR(hw->regmap),
+					     "regmap mmio init failed\n");
 	}
 
 	altera_spi_init_host(host);
diff --git a/drivers/spi/spi-atcspi200.c b/drivers/spi/spi-atcspi200.c
index 3c5098421ba38..b6ea8e5563359 100644
--- a/drivers/spi/spi-atcspi200.c
+++ b/drivers/spi/spi-atcspi200.c
@@ -598,8 +598,11 @@ static int atcspi_suspend(struct device *dev)
 {
 	struct spi_controller *host = dev_get_drvdata(dev);
 	struct atcspi_dev *spi = spi_controller_get_devdata(host);
+	int ret;
 
-	spi_controller_suspend(host);
+	ret = spi_controller_suspend(host);
+	if (ret)
+		return ret;
 
 	clk_disable_unprepare(spi->clk);
 
diff --git a/drivers/spi/spi-bcm-qspi.c b/drivers/spi/spi-bcm-qspi.c
index dcade7b99cef1..ae1ffe13fa467 100644
--- a/drivers/spi/spi-bcm-qspi.c
+++ b/drivers/spi/spi-bcm-qspi.c
@@ -1692,13 +1692,17 @@ EXPORT_SYMBOL_GPL(bcm_qspi_remove);
 static int __maybe_unused bcm_qspi_suspend(struct device *dev)
 {
 	struct bcm_qspi *qspi = dev_get_drvdata(dev);
+	int ret;
 
 	/* store the override strap value */
 	if (!bcm_qspi_bspi_ver_three(qspi))
 		qspi->s3_strap_override_ctrl =
 			bcm_qspi_read(qspi, BSPI, BSPI_STRAP_OVERRIDE_CTRL);
 
-	spi_controller_suspend(qspi->host);
+	ret = spi_controller_suspend(qspi->host);
+	if (ret)
+		return ret;
+
 	clk_disable_unprepare(qspi->clk);
 	bcm_qspi_hw_uninit(qspi);
 
diff --git a/drivers/spi/spi-bcm2835.c b/drivers/spi/spi-bcm2835.c
index ff94482fefcc4..8f8715809c7ce 100644
--- a/drivers/spi/spi-bcm2835.c
+++ b/drivers/spi/spi-bcm2835.c
@@ -1226,7 +1226,7 @@ static int bcm2835_spi_setup(struct spi_device *spi)
 	struct bcm2835_spi *bs = spi_controller_get_devdata(ctlr);
 	struct bcm2835_spidev *target = spi_get_ctldata(spi);
 	struct gpiod_lookup_table *lookup __free(kfree) = NULL;
-	const char *pinctrl_compats[] = {
+	static const char * const pinctrl_compats[] = {
 		"brcm,bcm2835-gpio",
 		"brcm,bcm2711-gpio",
 		"brcm,bcm7211-gpio",
diff --git a/drivers/spi/spi-bcm63xx-hsspi.c b/drivers/spi/spi-bcm63xx-hsspi.c
index 58012e1b5ae74..37b3534d0b0a6 100644
--- a/drivers/spi/spi-bcm63xx-hsspi.c
+++ b/drivers/spi/spi-bcm63xx-hsspi.c
@@ -889,8 +889,12 @@ static int bcm63xx_hsspi_suspend(struct device *dev)
 {
 	struct spi_controller *host = dev_get_drvdata(dev);
 	struct bcm63xx_hsspi *bs = spi_controller_get_devdata(host);
+	int ret;
+
+	ret = spi_controller_suspend(host);
+	if (ret)
+		return ret;
 
-	spi_controller_suspend(host);
 	clk_disable_unprepare(bs->pll_clk);
 	clk_disable_unprepare(bs->clk);
 
diff --git a/drivers/spi/spi-bcm63xx.c b/drivers/spi/spi-bcm63xx.c
index 43d7b54e3ae85..8cef725aab267 100644
--- a/drivers/spi/spi-bcm63xx.c
+++ b/drivers/spi/spi-bcm63xx.c
@@ -632,8 +632,11 @@ static int bcm63xx_spi_suspend(struct device *dev)
 {
 	struct spi_controller *host = dev_get_drvdata(dev);
 	struct bcm63xx_spi *bs = spi_controller_get_devdata(host);
+	int ret;
 
-	spi_controller_suspend(host);
+	ret = spi_controller_suspend(host);
+	if (ret)
+		return ret;
 
 	clk_disable_unprepare(bs->clk);
 
diff --git a/drivers/spi/spi-bcmbca-hsspi.c b/drivers/spi/spi-bcmbca-hsspi.c
index 09c1472ae4fa1..af88ce04948b7 100644
--- a/drivers/spi/spi-bcmbca-hsspi.c
+++ b/drivers/spi/spi-bcmbca-hsspi.c
@@ -568,8 +568,12 @@ static int bcmbca_hsspi_suspend(struct device *dev)
 {
 	struct spi_controller *host = dev_get_drvdata(dev);
 	struct bcmbca_hsspi *bs = spi_controller_get_devdata(host);
+	int ret;
+
+	ret = spi_controller_suspend(host);
+	if (ret)
+		return ret;
 
-	spi_controller_suspend(host);
 	clk_disable_unprepare(bs->pll_clk);
 	clk_disable_unprepare(bs->clk);
 
diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c
index aaba1a3ad5778..ecb0be3946965 100644
--- a/drivers/spi/spi-cadence-quadspi.c
+++ b/drivers/spi/spi-cadence-quadspi.c
@@ -382,12 +382,16 @@ static irqreturn_t cqspi_irq_handler(int this_irq, void *dev)
 	/* Clear interrupt */
 	writel(irq_status, cqspi->iobase + CQSPI_REG_IRQSTATUS);
 
-	if (cqspi->use_dma_read && ddata && ddata->get_dma_status)
-		irq_status = ddata->get_dma_status(cqspi);
-	else if (cqspi->slow_sram)
+	if (cqspi->use_dma_read && ddata && ddata->get_dma_status) {
+		if (ddata->get_dma_status(cqspi)) {
+			complete(&cqspi->transfer_complete);
+			return IRQ_HANDLED;
+		}
+	} else if (cqspi->slow_sram) {
 		irq_status &= CQSPI_IRQ_MASK_RD_SLOW_SRAM | CQSPI_IRQ_MASK_WR;
-	else
+	} else {
 		irq_status &= CQSPI_IRQ_MASK_RD | CQSPI_IRQ_MASK_WR;
+	}
 
 	if (irq_status)
 		complete(&cqspi->transfer_complete);
diff --git a/drivers/spi/spi-dw-dma.c b/drivers/spi/spi-dw-dma.c
index bd70a7ed8067d..f7d848fec9ab6 100644
--- a/drivers/spi/spi-dw-dma.c
+++ b/drivers/spi/spi-dw-dma.c
@@ -282,7 +282,8 @@ static int dw_spi_dma_wait(struct dw_spi *dws, unsigned int len, u32 speed)
 
 static inline bool dw_spi_dma_tx_busy(struct dw_spi *dws)
 {
-	return !(dw_readl(dws, DW_SPI_SR) & DW_SPI_SR_TF_EMPT);
+	return (dw_readl(dws, DW_SPI_SR) &
+		(DW_SPI_SR_BUSY | DW_SPI_SR_TF_EMPT)) != DW_SPI_SR_TF_EMPT;
 }
 
 static int dw_spi_dma_wait_tx_done(struct dw_spi *dws,
diff --git a/drivers/spi/spi-fsl-dspi.c b/drivers/spi/spi-fsl-dspi.c
index 019d05cdefe69..c2d283876ef80 100644
--- a/drivers/spi/spi-fsl-dspi.c
+++ b/drivers/spi/spi-fsl-dspi.c
@@ -1464,10 +1464,18 @@ static int dspi_init(struct fsl_dspi *dspi)
 static int dspi_suspend(struct device *dev)
 {
 	struct fsl_dspi *dspi = dev_get_drvdata(dev);
+	int ret;
 
 	if (dspi->irq)
 		disable_irq(dspi->irq);
-	spi_controller_suspend(dspi->ctlr);
+
+	ret = spi_controller_suspend(dspi->ctlr);
+	if (ret) {
+		if (dspi->irq)
+			enable_irq(dspi->irq);
+		return ret;
+	}
+
 	clk_disable_unprepare(dspi->clk);
 
 	pinctrl_pm_select_sleep_state(dev);
@@ -1485,12 +1493,15 @@ static int dspi_resume(struct device *dev)
 	ret = clk_prepare_enable(dspi->clk);
 	if (ret)
 		return ret;
-	spi_controller_resume(dspi->ctlr);
+
+	ret = spi_controller_resume(dspi->ctlr);
+	if (ret)
+		goto disable_clk;
 
 	ret = dspi_init(dspi);
 	if (ret) {
 		dev_err(dev, "failed to initialize dspi during resume\n");
-		return ret;
+		goto disable_clk;
 	}
 
 	dspi_set_mtf(dspi);
@@ -1499,6 +1510,10 @@ static int dspi_resume(struct device *dev)
 		enable_irq(dspi->irq);
 
 	return 0;
+
+disable_clk:
+	clk_disable_unprepare(dspi->clk);
+	return ret;
 }
 #endif /* CONFIG_PM_SLEEP */
 
diff --git a/drivers/spi/spi-geni-qcom.c b/drivers/spi/spi-geni-qcom.c
index 26e723cfea61e..d262206ed6632 100644
--- a/drivers/spi/spi-geni-qcom.c
+++ b/drivers/spi/spi-geni-qcom.c
@@ -78,6 +78,13 @@
 #define GSI_CPHA		BIT(4)
 #define GSI_CPOL		BIT(5)
 
+struct geni_spi_desc {
+	int (*resources_init)(struct geni_se *se);
+	int (*set_rate)(struct geni_se *se, unsigned long clk_freq);
+	int (*power_on)(struct geni_se *se);
+	int (*power_off)(struct geni_se *se);
+};
+
 struct spi_geni_master {
 	struct geni_se se;
 	struct device *dev;
@@ -105,6 +112,7 @@ struct spi_geni_master {
 	struct dma_chan *tx;
 	struct dma_chan *rx;
 	int cur_xfer_mode;
+	const struct geni_spi_desc *dev_data;
 };
 
 static void spi_slv_setup(struct spi_geni_master *mas)
@@ -305,11 +313,12 @@ static void spi_setup_word_len(struct spi_geni_master *mas, u16 mode,
 	writel(word_len, se->base + SE_SPI_WORD_LEN);
 }
 
-static int geni_spi_set_clock_and_bw(struct spi_geni_master *mas,
-					unsigned long clk_hz)
+static int geni_spi_set_clock_and_bw(struct geni_se *se,
+				     unsigned long clk_hz)
 {
+	struct spi_controller *spi = dev_get_drvdata(se->dev);
+	struct spi_geni_master *mas = spi_controller_get_devdata(spi);
 	u32 clk_sel, m_clk_cfg, idx, div;
-	struct geni_se *se = &mas->se;
 	int ret;
 
 	if (clk_hz == mas->cur_speed_hz)
@@ -613,25 +622,30 @@ static int spi_geni_init(struct spi_geni_master *mas)
 	u32 spi_tx_cfg, fifo_disable;
 	int ret = -ENXIO;
 
-	pm_runtime_get_sync(mas->dev);
+	PM_RUNTIME_ACQUIRE_IF_ENABLED(mas->dev, pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret < 0) {
+		dev_err(mas->dev, "Failed to resume and get %d\n", ret);
+		return ret;
+	}
 
 	proto = geni_se_read_proto(se);
 
 	if (spi->target) {
 		if (proto != GENI_SE_SPI_SLAVE) {
 			dev_err(mas->dev, "Invalid proto %d\n", proto);
-			goto out_pm;
+			return -EINVAL;
 		}
 		spi_slv_setup(mas);
 	} else if (proto == GENI_SE_INVALID_PROTO) {
 		ret = geni_load_se_firmware(se, GENI_SE_SPI);
 		if (ret) {
 			dev_err(mas->dev, "spi master firmware load failed ret: %d\n", ret);
-			goto out_pm;
+			return ret;
 		}
 	} else if (proto != GENI_SE_SPI) {
 		dev_err(mas->dev, "Invalid proto %d\n", proto);
-		goto out_pm;
+		return -EINVAL;
 	}
 	mas->tx_fifo_depth = geni_se_get_tx_fifo_depth(se);
 
@@ -664,7 +678,7 @@ static int spi_geni_init(struct spi_geni_master *mas)
 			dev_dbg(mas->dev, "Using GPI DMA mode for SPI\n");
 			break;
 		} else if (ret == -EPROBE_DEFER) {
-			goto out_pm;
+			return ret;
 		}
 		/*
 		 * in case of failure to get gpi dma channel, we can still do the
@@ -693,8 +707,6 @@ static int spi_geni_init(struct spi_geni_master *mas)
 		writel(spi_tx_cfg, se->base + SE_SPI_TRANS_CFG);
 	}
 
-out_pm:
-	pm_runtime_put(mas->dev);
 	return ret;
 }
 
@@ -825,7 +837,7 @@ static int setup_se_xfer(struct spi_transfer *xfer,
 	}
 
 	/* Speed and bits per word can be overridden per transfer */
-	ret = geni_spi_set_clock_and_bw(mas, xfer->speed_hz);
+	ret = mas->dev_data->set_rate(&mas->se, xfer->speed_hz);
 	if (ret)
 		return ret;
 
@@ -1045,7 +1057,6 @@ static int spi_geni_probe(struct platform_device *pdev)
 	struct spi_controller *spi;
 	struct spi_geni_master *mas;
 	void __iomem *base;
-	struct clk *clk;
 	struct device *dev = &pdev->dev;
 
 	irq = platform_get_irq(pdev, 0);
@@ -1060,10 +1071,6 @@ static int spi_geni_probe(struct platform_device *pdev)
 	if (IS_ERR(base))
 		return PTR_ERR(base);
 
-	clk = devm_clk_get(dev, "se");
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
-
 	if (device_property_read_bool(dev, "spi-slave"))
 		spi = devm_spi_alloc_target(dev, sizeof(*mas));
 	else
@@ -1079,17 +1086,14 @@ static int spi_geni_probe(struct platform_device *pdev)
 	mas->se.dev = dev;
 	mas->se.wrapper = dev_get_drvdata(dev->parent);
 	mas->se.base = base;
-	mas->se.clk = clk;
 
-	ret = devm_pm_opp_set_clkname(&pdev->dev, "se");
+	mas->dev_data = device_get_match_data(&pdev->dev);
+	if (!mas->dev_data)
+		return -EINVAL;
+
+	ret = mas->dev_data->resources_init(&mas->se);
 	if (ret)
 		return ret;
-	/* OPP table is optional */
-	ret = devm_pm_opp_of_add_table(&pdev->dev);
-	if (ret && ret != -ENODEV) {
-		dev_err(&pdev->dev, "invalid OPP table in device tree\n");
-		return ret;
-	}
 
 	spi->bus_num = -1;
 	spi->mode_bits = SPI_CPOL | SPI_CPHA | SPI_LOOP | SPI_CS_HIGH;
@@ -1115,24 +1119,12 @@ static int spi_geni_probe(struct platform_device *pdev)
 	if (spi->target)
 		spi->target_abort = spi_geni_target_abort;
 
-	ret = geni_icc_get(&mas->se, NULL);
-	if (ret)
-		return ret;
-
 	pm_runtime_use_autosuspend(&pdev->dev);
 	pm_runtime_set_autosuspend_delay(&pdev->dev, 250);
 	ret = devm_pm_runtime_enable(dev);
 	if (ret)
 		return ret;
 
-	/* Set the bus quota to a reasonable value for register access */
-	mas->se.icc_paths[GENI_TO_CORE].avg_bw = Bps_to_icc(CORE_2X_50_MHZ);
-	mas->se.icc_paths[CPU_TO_GENI].avg_bw = GENI_DEFAULT_BW;
-
-	ret = geni_icc_set_bw(&mas->se);
-	if (ret)
-		return ret;
-
 	ret = spi_geni_init(mas);
 	if (ret)
 		return ret;
@@ -1154,16 +1146,9 @@ static int __maybe_unused spi_geni_runtime_suspend(struct device *dev)
 {
 	struct spi_controller *spi = dev_get_drvdata(dev);
 	struct spi_geni_master *mas = spi_controller_get_devdata(spi);
-	int ret;
-
-	/* Drop the performance state vote */
-	dev_pm_opp_set_rate(dev, 0);
-
-	ret = geni_se_resources_off(&mas->se);
-	if (ret)
-		return ret;
 
-	return geni_icc_disable(&mas->se);
+	return mas->dev_data->power_off ?
+	       mas->dev_data->power_off(&mas->se) : 0;
 }
 
 static int __maybe_unused spi_geni_runtime_resume(struct device *dev)
@@ -1172,15 +1157,16 @@ static int __maybe_unused spi_geni_runtime_resume(struct device *dev)
 	struct spi_geni_master *mas = spi_controller_get_devdata(spi);
 	int ret;
 
-	ret = geni_icc_enable(&mas->se);
-	if (ret)
-		return ret;
+	if (mas->dev_data->power_on) {
+		ret = mas->dev_data->power_on(&mas->se);
+		if (ret)
+			return ret;
+	}
 
-	ret = geni_se_resources_on(&mas->se);
-	if (ret)
-		return ret;
+	if (mas->se.has_opp)
+		return dev_pm_opp_set_rate(mas->dev, mas->cur_sclk_hz);
 
-	return dev_pm_opp_set_rate(mas->dev, mas->cur_sclk_hz);
+	return 0;
 }
 
 static int __maybe_unused spi_geni_suspend(struct device *dev)
@@ -1221,8 +1207,21 @@ static const struct dev_pm_ops spi_geni_pm_ops = {
 	SET_SYSTEM_SLEEP_PM_OPS(spi_geni_suspend, spi_geni_resume)
 };
 
+static const struct geni_spi_desc geni_spi = {
+	.resources_init = geni_se_resources_init,
+	.set_rate = geni_spi_set_clock_and_bw,
+	.power_on = geni_se_resources_activate,
+	.power_off = geni_se_resources_deactivate,
+};
+
+static const struct geni_spi_desc sa8255p_geni_spi = {
+	.resources_init = geni_se_domain_attach,
+	.set_rate = geni_se_set_perf_opp,
+};
+
 static const struct of_device_id spi_geni_dt_match[] = {
-	{ .compatible = "qcom,geni-spi" },
+	{ .compatible = "qcom,geni-spi", .data = &geni_spi },
+	{ .compatible = "qcom,sa8255p-geni-spi", .data = &sa8255p_geni_spi },
 	{}
 };
 MODULE_DEVICE_TABLE(of, spi_geni_dt_match);
diff --git a/drivers/spi/spi-npcm-fiu.c b/drivers/spi/spi-npcm-fiu.c
index 4b825044038b2..96dfef4d90674 100644
--- a/drivers/spi/spi-npcm-fiu.c
+++ b/drivers/spi/spi-npcm-fiu.c
@@ -393,7 +393,7 @@ static int npcm_fiu_uma_write(struct spi_mem *mem,
 {
 	struct npcm_fiu_spi *fiu =
 		spi_controller_get_devdata(mem->spi->controller);
-	u32 uma_cfg = BIT(10);
+	u32 uma_cfg = cmd ? BIT(10) : 0;
 	u32 data_reg[4] = {0};
 	u32 val;
 	u32 i;
@@ -403,8 +403,11 @@ static int npcm_fiu_uma_write(struct spi_mem *mem,
 			   (spi_get_chipselect(mem->spi, 0) <<
 			    NPCM_FIU_UMA_CTS_DEV_NUM_SHIFT));
 
-	regmap_update_bits(fiu->regmap, NPCM_FIU_UMA_CMD,
-			   NPCM_FIU_UMA_CMD_CMD, cmd);
+	if (cmd)
+		regmap_update_bits(fiu->regmap, NPCM_FIU_UMA_CMD,
+				   NPCM_FIU_UMA_CMD_CMD, cmd);
+	else
+		uma_cfg |= ilog2(op->data.buswidth) << NPCM_FIU_UMA_CFG_WDBPCK_SHIFT;
 
 	if (data_size) {
 		memcpy(data_reg, data, data_size);
@@ -464,8 +467,7 @@ static int npcm_fiu_manualwrite(struct spi_mem *mem,
 
 	/* Starting the data writing loop in multiples of 8 */
 	for (idx = 0; idx < num_data_chunks; ++idx) {
-		ret = npcm_fiu_uma_write(mem, op, data[0], false,
-					 &data[1], CHUNK_SIZE - 1);
+		ret = npcm_fiu_uma_write(mem, op, 0, false, &data[0], CHUNK_SIZE);
 		if (ret)
 			return ret;
 
@@ -474,8 +476,7 @@ static int npcm_fiu_manualwrite(struct spi_mem *mem,
 
 	/* Handling chunk remains */
 	if (remain_data > 0) {
-		ret = npcm_fiu_uma_write(mem, op, data[0], false,
-					 &data[1], remain_data - 1);
+		ret = npcm_fiu_uma_write(mem, op, 0, false, &data[0], remain_data);
 		if (ret)
 			return ret;
 	}
diff --git a/drivers/spi/spi-nxp-fspi.c b/drivers/spi/spi-nxp-fspi.c
index 1e36ae084dd86..d94a2a7b98d44 100644
--- a/drivers/spi/spi-nxp-fspi.c
+++ b/drivers/spi/spi-nxp-fspi.c
@@ -1350,9 +1350,11 @@ static int nxp_fspi_probe(struct platform_device *pdev)
 	pm_runtime_use_autosuspend(dev);
 
 	/* enable clock */
-	ret = pm_runtime_get_sync(f->dev);
-	if (ret < 0)
-		return dev_err_probe(dev, ret, "Failed to enable clock");
+	ret = pm_runtime_resume_and_get(f->dev);
+	if (ret < 0) {
+		ret = dev_err_probe(dev, ret, "Failed to enable clock");
+		goto err_disable_pm;
+	}
 
 	/* Clear potential interrupts */
 	reg = fspi_readl(f, f->iobase + FSPI_INTR);
@@ -1362,18 +1364,24 @@ static int nxp_fspi_probe(struct platform_device *pdev)
 	nxp_fspi_default_setup(f);
 
 	ret = pm_runtime_put_sync(dev);
-	if (ret < 0)
-		return dev_err_probe(dev, ret, "Failed to disable clock");
+	if (ret < 0) {
+		ret = dev_err_probe(dev, ret, "Failed to disable clock");
+		goto err_disable_pm;
+	}
 
 	init_completion(&f->c);
 	ret = devm_request_irq(dev, irq,
 			nxp_fspi_irq_handler, 0, pdev->name, f);
-	if (ret)
-		return dev_err_probe(dev, ret, "Failed to request irq\n");
+	if (ret) {
+		ret = dev_err_probe(dev, ret, "Failed to request irq\n");
+		goto err_disable_pm;
+	}
 
 	ret = devm_mutex_init(dev, &f->lock);
-	if (ret)
-		return dev_err_probe(dev, ret, "Failed to initialize lock\n");
+	if (ret) {
+		ret = dev_err_probe(dev, ret, "Failed to initialize lock\n");
+		goto err_disable_pm;
+	}
 
 	ctlr->bus_num = -1;
 	ctlr->num_chipselect = NXP_FSPI_MAX_CHIPSELECT;
@@ -1389,6 +1397,11 @@ static int nxp_fspi_probe(struct platform_device *pdev)
 		return ret;
 
 	return devm_spi_register_controller(&pdev->dev, ctlr);
+
+err_disable_pm:
+	pm_runtime_dont_use_autosuspend(dev);
+	pm_runtime_disable(dev);
+	return ret;
 }
 
 static int nxp_fspi_runtime_suspend(struct device *dev)
diff --git a/drivers/spi/spi-qpic-snand.c b/drivers/spi/spi-qpic-snand.c
index 66f2d1b78ade8..6cc53586b8a8f 100644
--- a/drivers/spi/spi-qpic-snand.c
+++ b/drivers/spi/spi-qpic-snand.c
@@ -394,14 +394,19 @@ static int qcom_spi_ecc_init_ctx_pipelined(struct nand_device *nand)
 	return 0;
 
 err_free_ecc_cfg:
+	kfree(snandc->qspi->oob_buf);
+	snandc->qspi->oob_buf = NULL;
 	kfree(ecc_cfg);
 	return ret;
 }
 
 static void qcom_spi_ecc_cleanup_ctx_pipelined(struct nand_device *nand)
 {
+	struct qcom_nand_controller *snandc = nand_to_qcom_snand(nand);
 	struct qpic_ecc *ecc_cfg = nand_to_ecc_ctx(nand);
 
+	kfree(snandc->qspi->oob_buf);
+	snandc->qspi->oob_buf = NULL;
 	kfree(ecc_cfg);
 }
 
@@ -1645,4 +1650,3 @@ module_platform_driver(qcom_spi_driver);
 MODULE_DESCRIPTION("SPI driver for QPIC QSPI cores");
 MODULE_AUTHOR("Md Sadre Alam <quic_mdalam@quicinc.com>");
 MODULE_LICENSE("GPL");
-
diff --git a/drivers/spi/spi-realtek-rtl.c b/drivers/spi/spi-realtek-rtl.c
index 73065c8934dc7..e1c40ff2e49d3 100644
--- a/drivers/spi/spi-realtek-rtl.c
+++ b/drivers/spi/spi-realtek-rtl.c
@@ -35,58 +35,58 @@ static void rt_set_cs(struct spi_device *spi, bool active)
 	u32 value;
 
 	/* CS0 bit is active low */
-	value = readl(REG(RTL_SPI_SFCSR));
+	value = __raw_readl(REG(RTL_SPI_SFCSR));
 	if (active)
 		value |= RTL_SPI_SFCSR_CSB0;
 	else
 		value &= ~RTL_SPI_SFCSR_CSB0;
-	writel(value, REG(RTL_SPI_SFCSR));
+	__raw_writel(value, REG(RTL_SPI_SFCSR));
 }
 
 static void set_size(struct rtspi *rtspi, int size)
 {
 	u32 value;
 
-	value = readl(REG(RTL_SPI_SFCSR));
+	value = __raw_readl(REG(RTL_SPI_SFCSR));
 	value &= RTL_SPI_SFCSR_LEN_MASK;
 	if (size == 4)
 		value |= RTL_SPI_SFCSR_LEN4;
 	else if (size == 1)
 		value |= RTL_SPI_SFCSR_LEN1;
-	writel(value, REG(RTL_SPI_SFCSR));
+	__raw_writel(value, REG(RTL_SPI_SFCSR));
 }
 
 static inline void wait_ready(struct rtspi *rtspi)
 {
-	while (!(readl(REG(RTL_SPI_SFCSR)) & RTL_SPI_SFCSR_RDY))
+	while (!(__raw_readl(REG(RTL_SPI_SFCSR)) & RTL_SPI_SFCSR_RDY))
 		cpu_relax();
 }
 static void send4(struct rtspi *rtspi, const u32 *buf)
 {
 	wait_ready(rtspi);
 	set_size(rtspi, 4);
-	writel(*buf, REG(RTL_SPI_SFDR));
+	__raw_writel(*buf, REG(RTL_SPI_SFDR));
 }
 
 static void send1(struct rtspi *rtspi, const u8 *buf)
 {
 	wait_ready(rtspi);
 	set_size(rtspi, 1);
-	writel(buf[0] << 24, REG(RTL_SPI_SFDR));
+	__raw_writel(buf[0] << 24, REG(RTL_SPI_SFDR));
 }
 
 static void rcv4(struct rtspi *rtspi, u32 *buf)
 {
 	wait_ready(rtspi);
 	set_size(rtspi, 4);
-	*buf = readl(REG(RTL_SPI_SFDR));
+	*buf = __raw_readl(REG(RTL_SPI_SFDR));
 }
 
 static void rcv1(struct rtspi *rtspi, u8 *buf)
 {
 	wait_ready(rtspi);
 	set_size(rtspi, 1);
-	*buf = readl(REG(RTL_SPI_SFDR)) >> 24;
+	*buf = __raw_readl(REG(RTL_SPI_SFDR)) >> 24;
 }
 
 static int transfer_one(struct spi_controller *ctrl, struct spi_device *spi,
@@ -134,16 +134,16 @@ static void init_hw(struct rtspi *rtspi)
 	u32 value;
 
 	/* Turn on big-endian byte ordering */
-	value = readl(REG(RTL_SPI_SFCR));
+	value = __raw_readl(REG(RTL_SPI_SFCR));
 	value |= RTL_SPI_SFCR_RBO | RTL_SPI_SFCR_WBO;
-	writel(value, REG(RTL_SPI_SFCR));
+	__raw_writel(value, REG(RTL_SPI_SFCR));
 
-	value = readl(REG(RTL_SPI_SFCSR));
+	value = __raw_readl(REG(RTL_SPI_SFCSR));
 	/* Permanently disable CS1, since it's never used */
 	value |= RTL_SPI_SFCSR_CSB1;
 	/* Select CS0 for use */
 	value &= RTL_SPI_SFCSR_CS;
-	writel(value, REG(RTL_SPI_SFCSR));
+	__raw_writel(value, REG(RTL_SPI_SFCSR));
 }
 
 static int realtek_rtl_spi_probe(struct platform_device *pdev)
diff --git a/drivers/ssb/driver_gpio.c b/drivers/ssb/driver_gpio.c
index 905657c925bce..87922479946cb 100644
--- a/drivers/ssb/driver_gpio.c
+++ b/drivers/ssb/driver_gpio.c
@@ -15,8 +15,14 @@
 #include <linux/interrupt.h>
 #include <linux/irqdomain.h>
 #include <linux/export.h>
+#include <linux/property.h>
 #include <linux/ssb/ssb.h>
 
+const struct software_node ssb_gpio_swnode = {
+	.name = "ssb-gpio",
+};
+EXPORT_SYMBOL_GPL(ssb_gpio_swnode);
+
 
 /**************************************************
  * Shared
@@ -232,6 +238,8 @@ static int ssb_gpio_chipco_init(struct ssb_bus *bus)
 	chip->to_irq		= ssb_gpio_to_irq;
 #endif
 	chip->ngpio		= 16;
+	if (bus->bustype == SSB_BUSTYPE_SSB)
+		chip->fwnode	= software_node_fwnode(&ssb_gpio_swnode);
 	/* There is just one SoC in one device and its GPIO addresses should be
 	 * deterministic to address them more easily. The other buses could get
 	 * a random base number.
@@ -433,10 +441,12 @@ static int ssb_gpio_extif_init(struct ssb_bus *bus)
 	 * deterministic to address them more easily. The other buses could get
 	 * a random base number.
 	 */
-	if (bus->bustype == SSB_BUSTYPE_SSB)
-		chip->base		= 0;
-	else
-		chip->base		= -1;
+	if (bus->bustype == SSB_BUSTYPE_SSB) {
+		chip->base	= 0;
+		chip->fwnode	= software_node_fwnode(&ssb_gpio_swnode);
+	} else {
+		chip->base	= -1;
+	}
 
 	err = ssb_gpio_irq_extif_domain_init(bus);
 	if (err)
@@ -464,11 +474,33 @@ static int ssb_gpio_extif_init(struct ssb_bus *bus)
 
 int ssb_gpio_init(struct ssb_bus *bus)
 {
+	int err = 0;
+
+	/*
+	 * Register software node only for the host SoC bus. There is only
+	 * one SoC instance in the system, so there are no concerns with
+	 * registration conflicts.
+	 */
+	if (bus->bustype == SSB_BUSTYPE_SSB) {
+		err = software_node_register(&ssb_gpio_swnode);
+		if (err)
+			return err;
+	}
+
 	if (ssb_chipco_available(&bus->chipco))
-		return ssb_gpio_chipco_init(bus);
+		err = ssb_gpio_chipco_init(bus);
 	else if (ssb_extif_available(&bus->extif))
-		return ssb_gpio_extif_init(bus);
-	return -1;
+		err = ssb_gpio_extif_init(bus);
+	else
+		err = -ENODEV;
+
+	if (err) {
+		if (bus->bustype == SSB_BUSTYPE_SSB)
+			software_node_unregister(&ssb_gpio_swnode);
+		return err;
+	}
+
+	return 0;
 }
 
 int ssb_gpio_unregister(struct ssb_bus *bus)
@@ -476,6 +508,8 @@ int ssb_gpio_unregister(struct ssb_bus *bus)
 	if (ssb_chipco_available(&bus->chipco) ||
 	    ssb_extif_available(&bus->extif)) {
 		gpiochip_remove(&bus->gpio);
+		if (bus->bustype == SSB_BUSTYPE_SSB)
+			software_node_unregister(&ssb_gpio_swnode);
 		return 0;
 	}
 	return -1;
diff --git a/drivers/staging/axis-fifo/axis-fifo.c b/drivers/staging/axis-fifo/axis-fifo.c
index 3aa2aa870ea91..1c34de020cf85 100644
--- a/drivers/staging/axis-fifo/axis-fifo.c
+++ b/drivers/staging/axis-fifo/axis-fifo.c
@@ -246,7 +246,8 @@ static ssize_t axis_fifo_write(struct file *f, const char __user *buf,
 		mutex_lock(&fifo->write_lock);
 
 		ret = wait_event_interruptible(fifo->write_queue,
-			ioread32(fifo->base_addr + XLLF_TDFV_OFFSET) >= words_to_write);
+					       ioread32(fifo->base_addr + XLLF_TDFV_OFFSET) >=
+					       words_to_write);
 		if (ret)
 			goto end_unlock;
 	}
diff --git a/drivers/staging/fbtft/fb_upd161704.c b/drivers/staging/fbtft/fb_upd161704.c
index c680160d63807..7288140e7c3e1 100644
--- a/drivers/staging/fbtft/fb_upd161704.c
+++ b/drivers/staging/fbtft/fb_upd161704.c
@@ -67,7 +67,7 @@ static int init_display(struct fbtft_par *par)
 	write_reg(par, 0x001D, 0x0000);	/* Regulator current setting */
 	write_reg(par, 0x001E, 0x0009);	/* VCOM output setting */
 	write_reg(par, 0x001F, 0x0035);	/* VCOM amplitude setting */
-	write_reg(par, 0x0020, 0x0015);	/* VCOMM cencter setting */
+	write_reg(par, 0x0020, 0x0015);	/* VCOMM center setting */
 	write_reg(par, 0x0018, 0x1E7B);	/* DC/DC operation setting */
 
 	/* windows setting */
diff --git a/drivers/staging/fbtft/fbtft-sysfs.c b/drivers/staging/fbtft/fbtft-sysfs.c
index d05599d80011a..343545e83a377 100644
--- a/drivers/staging/fbtft/fbtft-sysfs.c
+++ b/drivers/staging/fbtft/fbtft-sysfs.c
@@ -98,7 +98,7 @@ sprintf_gamma(struct fbtft_par *par, u32 *curves, char *buf)
 	mutex_lock(&par->gamma.lock);
 	for (i = 0; i < par->gamma.num_curves; i++) {
 		for (j = 0; j < par->gamma.num_values; j++)
-			len += scnprintf(&buf[len], PAGE_SIZE,
+			len += sysfs_emit_at(buf, len,
 			     "%04x ", curves[i * par->gamma.num_values + j]);
 		buf[len - 1] = '\n';
 	}
diff --git a/drivers/staging/greybus/audio_codec.c b/drivers/staging/greybus/audio_codec.c
index 720aa752e17e6..6daa4e706792c 100644
--- a/drivers/staging/greybus/audio_codec.c
+++ b/drivers/staging/greybus/audio_codec.c
@@ -311,8 +311,7 @@ int gbaudio_module_update(struct gbaudio_codec_info *codec,
 	}
 
 	/* parse dai_id from AIF widget's stream_name */
-	ret = sscanf(w->sname, "%s %d %s", intf_name, &dai_id, dir);
-	if (ret < 3) {
+	if (sscanf(w->sname, "%s %d %s", intf_name, &dai_id, dir) != 3) {
 		dev_err(codec->dev, "Error while parsing dai_id for %s\n", w->name);
 		return -EINVAL;
 	}
@@ -323,7 +322,7 @@ int gbaudio_module_update(struct gbaudio_codec_info *codec,
 			ret = gbaudio_module_enable_tx(codec, module, dai_id);
 		else
 			ret = gbaudio_module_disable_tx(module, dai_id);
-	} else if (w->id == snd_soc_dapm_aif_out) {
+	} else {
 		if (enable)
 			ret = gbaudio_module_enable_rx(codec, module, dai_id);
 		else
diff --git a/drivers/staging/greybus/audio_gb.c b/drivers/staging/greybus/audio_gb.c
index 9d8994fdb41a2..144591f1a5128 100644
--- a/drivers/staging/greybus/audio_gb.c
+++ b/drivers/staging/greybus/audio_gb.c
@@ -37,6 +37,19 @@ int gb_audio_gb_get_topology(struct gb_connection *connection,
 		return ret;
 	}
 
+	/*
+	 * The size_* fields are supplied by the module and are used by
+	 * gbaudio_tplg_parse_data() to compute offsets into the blob; make
+	 * sure the sections fit within the fetched topology, so walking it
+	 * cannot read out of bounds.
+	 */
+	if ((u64)le32_to_cpu(topo->size_dais) + le32_to_cpu(topo->size_controls) +
+	    le32_to_cpu(topo->size_widgets) + le32_to_cpu(topo->size_routes) >
+	    size - sizeof(*topo)) {
+		kfree(topo);
+		return -EINVAL;
+	}
+
 	*topology = topo;
 
 	return 0;
diff --git a/drivers/staging/greybus/audio_manager_sysfs.c b/drivers/staging/greybus/audio_manager_sysfs.c
index fcd518f9540cd..ff323ca8154f3 100644
--- a/drivers/staging/greybus/audio_manager_sysfs.c
+++ b/drivers/staging/greybus/audio_manager_sysfs.c
@@ -23,7 +23,7 @@ static ssize_t manager_sysfs_add_store(struct kobject *kobj,
 			desc.name, &desc.vid, &desc.pid, &desc.intf_id,
 			&desc.ip_devices, &desc.op_devices);
 
-	if (num != 7)
+	if (num != 6)
 		return -EINVAL;
 
 	num = gb_audio_manager_add(&desc);
diff --git a/drivers/staging/greybus/authentication.c b/drivers/staging/greybus/authentication.c
index 97b9937bbdc1a..d8f2cd43201f8 100644
--- a/drivers/staging/greybus/authentication.c
+++ b/drivers/staging/greybus/authentication.c
@@ -15,8 +15,6 @@
 #include "greybus_authentication.h"
 #include "firmware.h"
 
-#define CAP_TIMEOUT_MS		1000
-
 /*
  * Number of minor devices this driver supports.
  * There will be exactly one required per Interface.
diff --git a/drivers/staging/greybus/hid.c b/drivers/staging/greybus/hid.c
index f1f9f6fbc00e5..1d7186eecd23e 100644
--- a/drivers/staging/greybus/hid.c
+++ b/drivers/staging/greybus/hid.c
@@ -256,7 +256,7 @@ static int __gb_hid_output_raw_report(struct hid_device *hid, __u8 *buf,
 	if (report_id && ret >= 0)
 		ret++; /* add report_id to the number of transferred bytes */
 
-	return 0;
+	return ret;
 }
 
 static int gb_hid_raw_request(struct hid_device *hid, unsigned char reportnum,
diff --git a/drivers/staging/greybus/loopback.c b/drivers/staging/greybus/loopback.c
index ea57b1f5d1562..2a623fc2e99b1 100644
--- a/drivers/staging/greybus/loopback.c
+++ b/drivers/staging/greybus/loopback.c
@@ -167,32 +167,6 @@ static DEVICE_ATTR_RO(name##_avg)
 	gb_loopback_ro_stats_attr(field, max, u);		\
 	gb_loopback_ro_avg_attr(field)
 
-#define gb_loopback_attr(field, type)					\
-static ssize_t field##_show(struct device *dev,				\
-			    struct device_attribute *attr,		\
-			    char *buf)					\
-{									\
-	struct gb_loopback *gb = dev_get_drvdata(dev);			\
-	return sysfs_emit(buf, "%" #type "\n", gb->field);			\
-}									\
-static ssize_t field##_store(struct device *dev,			\
-			    struct device_attribute *attr,		\
-			    const char *buf,				\
-			    size_t len)					\
-{									\
-	int ret;							\
-	struct gb_loopback *gb = dev_get_drvdata(dev);			\
-	mutex_lock(&gb->mutex);						\
-	ret = sscanf(buf, "%"#type, &gb->field);			\
-	if (ret != 1)							\
-		len = -EINVAL;						\
-	else								\
-		gb_loopback_check_attr(gb, bundle);			\
-	mutex_unlock(&gb->mutex);					\
-	return len;							\
-}									\
-static DEVICE_ATTR_RW(field)
-
 #define gb_dev_loopback_ro_attr(field)				\
 static ssize_t field##_show(struct device *dev,		\
 			    struct device_attribute *attr,		\
diff --git a/drivers/staging/greybus/power_supply.c b/drivers/staging/greybus/power_supply.c
index 33b7a63979d66..44bd8a72fa508 100644
--- a/drivers/staging/greybus/power_supply.c
+++ b/drivers/staging/greybus/power_supply.c
@@ -336,7 +336,7 @@ static int is_psy_prop_writeable(struct gb_power_supply *gbpsy,
 
 static int is_prop_valint(enum power_supply_property psp)
 {
-	return ((psp < POWER_SUPPLY_PROP_MODEL_NAME) ? 1 : 0);
+	return (psp < POWER_SUPPLY_PROP_MODEL_NAME) ? 1 : 0;
 }
 
 static void next_interval(struct gb_power_supply *gbpsy)
@@ -423,7 +423,7 @@ static void check_changed(struct gb_power_supply *gbpsy,
 static int total_props(struct gb_power_supply *gbpsy)
 {
 	/* this return the intval plus the strval properties */
-	return (gbpsy->properties_count + gbpsy->properties_count_str);
+	return gbpsy->properties_count + gbpsy->properties_count_str;
 }
 
 static void prop_append(struct gb_power_supply *gbpsy,
diff --git a/drivers/staging/greybus/vibrator.c b/drivers/staging/greybus/vibrator.c
index 0ec4d317c9db8..763c234fbc038 100644
--- a/drivers/staging/greybus/vibrator.c
+++ b/drivers/staging/greybus/vibrator.c
@@ -161,7 +161,7 @@ static int gb_vibrator_probe(struct gb_bundle *bundle,
 	dev = device_create(&vibrator_class, &bundle->dev,
 			    MKDEV(0, 0), vib, "vibrator%d", vib->minor);
 	if (IS_ERR(dev)) {
-		retval = -EINVAL;
+		retval = PTR_ERR(dev);
 		goto err_ida_remove;
 	}
 	vib->dev = dev;
diff --git a/drivers/staging/iio/adc/ad7816.c b/drivers/staging/iio/adc/ad7816.c
index 0e32a2295990d..30644d2d7c549 100644
--- a/drivers/staging/iio/adc/ad7816.c
+++ b/drivers/staging/iio/adc/ad7816.c
@@ -426,9 +426,9 @@ static const struct of_device_id ad7816_of_match[] = {
 MODULE_DEVICE_TABLE(of, ad7816_of_match);
 
 static const struct spi_device_id ad7816_id[] = {
-	{ "ad7816", ID_AD7816 },
-	{ "ad7817", ID_AD7817 },
-	{ "ad7818", ID_AD7818 },
+	{ .name = "ad7816", .driver_data = ID_AD7816 },
+	{ .name = "ad7817", .driver_data = ID_AD7817 },
+	{ .name = "ad7818", .driver_data = ID_AD7818 },
 	{ }
 };
 
diff --git a/drivers/staging/iio/addac/adt7316-spi.c b/drivers/staging/iio/addac/adt7316-spi.c
index 1debcc36c1afb..459dd7416b1d5 100644
--- a/drivers/staging/iio/addac/adt7316-spi.c
+++ b/drivers/staging/iio/addac/adt7316-spi.c
@@ -126,15 +126,14 @@ static int adt7316_spi_probe(struct spi_device *spi_dev)
 }
 
 static const struct spi_device_id adt7316_spi_id[] = {
-	{ "adt7316", 0 },
-	{ "adt7317", 0 },
-	{ "adt7318", 0 },
-	{ "adt7516", 0 },
-	{ "adt7517", 0 },
-	{ "adt7519", 0 },
+	{ .name = "adt7316" },
+	{ .name = "adt7317" },
+	{ .name = "adt7318" },
+	{ .name = "adt7516" },
+	{ .name = "adt7517" },
+	{ .name = "adt7519" },
 	{ }
 };
-
 MODULE_DEVICE_TABLE(spi, adt7316_spi_id);
 
 static const struct of_device_id adt7316_of_spi_match[] = {
diff --git a/drivers/staging/iio/frequency/ad9832.c b/drivers/staging/iio/frequency/ad9832.c
index 60c33e10c46f4..a78249812b262 100644
--- a/drivers/staging/iio/frequency/ad9832.c
+++ b/drivers/staging/iio/frequency/ad9832.c
@@ -252,22 +252,22 @@ static ssize_t ad9832_write(struct device *dev, struct device_attribute *attr,
  * see dds.h for further information
  */
 
-static IIO_DEV_ATTR_FREQ(0, 0, 0200, NULL, ad9832_write, AD9832_FREQ0HM);
-static IIO_DEV_ATTR_FREQ(0, 1, 0200, NULL, ad9832_write, AD9832_FREQ1HM);
-static IIO_DEV_ATTR_FREQSYMBOL(0, 0200, NULL, ad9832_write, AD9832_FREQ_SYM);
+static IIO_DEV_ATTR_FREQ(0200, 0, 0, NULL, ad9832_write, AD9832_FREQ0HM);
+static IIO_DEV_ATTR_FREQ(0200, 0, 1, NULL, ad9832_write, AD9832_FREQ1HM);
+static IIO_DEV_ATTR_FREQSYMBOL(0200, 0, NULL, ad9832_write, AD9832_FREQ_SYM);
 static IIO_CONST_ATTR_FREQ_SCALE(0, "1"); /* 1Hz */
 
-static IIO_DEV_ATTR_PHASE(0, 0, 0200, NULL, ad9832_write, AD9832_PHASE0H);
-static IIO_DEV_ATTR_PHASE(0, 1, 0200, NULL, ad9832_write, AD9832_PHASE1H);
-static IIO_DEV_ATTR_PHASE(0, 2, 0200, NULL, ad9832_write, AD9832_PHASE2H);
-static IIO_DEV_ATTR_PHASE(0, 3, 0200, NULL, ad9832_write, AD9832_PHASE3H);
-static IIO_DEV_ATTR_PHASESYMBOL(0, 0200, NULL,
+static IIO_DEV_ATTR_PHASE(0200, 0, 0, NULL, ad9832_write, AD9832_PHASE0H);
+static IIO_DEV_ATTR_PHASE(0200, 0, 1, NULL, ad9832_write, AD9832_PHASE1H);
+static IIO_DEV_ATTR_PHASE(0200, 0, 2, NULL, ad9832_write, AD9832_PHASE2H);
+static IIO_DEV_ATTR_PHASE(0200, 0, 3, NULL, ad9832_write, AD9832_PHASE3H);
+static IIO_DEV_ATTR_PHASESYMBOL(0200, 0, NULL,
 				ad9832_write, AD9832_PHASE_SYM);
 static IIO_CONST_ATTR_PHASE_SCALE(0, "0.0015339808"); /* 2PI/2^12 rad*/
 
-static IIO_DEV_ATTR_PINCONTROL_EN(0, 0200, NULL,
+static IIO_DEV_ATTR_PINCONTROL_EN(0200, 0, NULL,
 				ad9832_write, AD9832_PINCTRL_EN);
-static IIO_DEV_ATTR_OUT_ENABLE(0, 0200, NULL,
+static IIO_DEV_ATTR_OUT_ENABLE(0200, 0, NULL,
 				ad9832_write, AD9832_OUTPUT_EN);
 
 static struct attribute *ad9832_attributes[] = {
@@ -380,8 +380,8 @@ static const struct of_device_id ad9832_of_match[] = {
 MODULE_DEVICE_TABLE(of, ad9832_of_match);
 
 static const struct spi_device_id ad9832_id[] = {
-	{"ad9832", 0},
-	{"ad9835", 0},
+	{ .name = "ad9832" },
+	{ .name = "ad9835" },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad9832_id);
diff --git a/drivers/staging/iio/frequency/ad9834.c b/drivers/staging/iio/frequency/ad9834.c
index 33dfd723923cb..0d20f796e0f09 100644
--- a/drivers/staging/iio/frequency/ad9834.c
+++ b/drivers/staging/iio/frequency/ad9834.c
@@ -314,21 +314,21 @@ static IIO_DEVICE_ATTR(out_altvoltage0_out1_wavetype_available, 0444,
  * see dds.h for further information
  */
 
-static IIO_DEV_ATTR_FREQ(0, 0, 0200, NULL, ad9834_write, AD9834_REG_FREQ0);
-static IIO_DEV_ATTR_FREQ(0, 1, 0200, NULL, ad9834_write, AD9834_REG_FREQ1);
-static IIO_DEV_ATTR_FREQSYMBOL(0, 0200, NULL, ad9834_write, AD9834_FSEL);
+static IIO_DEV_ATTR_FREQ(0200, 0, 0, NULL, ad9834_write, AD9834_REG_FREQ0);
+static IIO_DEV_ATTR_FREQ(0200, 0, 1, NULL, ad9834_write, AD9834_REG_FREQ1);
+static IIO_DEV_ATTR_FREQSYMBOL(0200, 0, NULL, ad9834_write, AD9834_FSEL);
 static IIO_CONST_ATTR_FREQ_SCALE(0, "1"); /* 1Hz */
 
-static IIO_DEV_ATTR_PHASE(0, 0, 0200, NULL, ad9834_write, AD9834_REG_PHASE0);
-static IIO_DEV_ATTR_PHASE(0, 1, 0200, NULL, ad9834_write, AD9834_REG_PHASE1);
-static IIO_DEV_ATTR_PHASESYMBOL(0, 0200, NULL, ad9834_write, AD9834_PSEL);
+static IIO_DEV_ATTR_PHASE(0200, 0, 0, NULL, ad9834_write, AD9834_REG_PHASE0);
+static IIO_DEV_ATTR_PHASE(0200, 0, 1, NULL, ad9834_write, AD9834_REG_PHASE1);
+static IIO_DEV_ATTR_PHASESYMBOL(0200, 0, NULL, ad9834_write, AD9834_PSEL);
 static IIO_CONST_ATTR_PHASE_SCALE(0, "0.0015339808"); /* 2PI/2^12 rad*/
 
-static IIO_DEV_ATTR_PINCONTROL_EN(0, 0200, NULL, ad9834_write, AD9834_PIN_SW);
-static IIO_DEV_ATTR_OUT_ENABLE(0, 0200, NULL, ad9834_write, AD9834_RESET);
-static IIO_DEV_ATTR_OUTY_ENABLE(0, 1, 0200, NULL, ad9834_write, AD9834_OPBITEN);
-static IIO_DEV_ATTR_OUT_WAVETYPE(0, 0, ad9834_store_wavetype, 0);
-static IIO_DEV_ATTR_OUT_WAVETYPE(0, 1, ad9834_store_wavetype, 1);
+static IIO_DEV_ATTR_PINCONTROL_EN(0200, 0, NULL, ad9834_write, AD9834_PIN_SW);
+static IIO_DEV_ATTR_OUT_ENABLE(0200, 0, NULL, ad9834_write, AD9834_RESET);
+static IIO_DEV_ATTR_OUTY_ENABLE(0200, 0, 1, NULL, ad9834_write, AD9834_OPBITEN);
+static IIO_DEV_ATTR_OUT_WAVETYPE(0200, 0, 0, ad9834_store_wavetype, 0);
+static IIO_DEV_ATTR_OUT_WAVETYPE(0200, 0, 1, ad9834_store_wavetype, 1);
 
 static struct attribute *ad9834_attributes[] = {
 	&iio_dev_attr_out_altvoltage0_frequency0.dev_attr.attr,
@@ -464,10 +464,10 @@ static int ad9834_probe(struct spi_device *spi)
 }
 
 static const struct spi_device_id ad9834_id[] = {
-	{"ad9833", ID_AD9833},
-	{"ad9834", ID_AD9834},
-	{"ad9837", ID_AD9837},
-	{"ad9838", ID_AD9838},
+	{ .name = "ad9833", .driver_data = ID_AD9833 },
+	{ .name = "ad9834", .driver_data = ID_AD9834 },
+	{ .name = "ad9837", .driver_data = ID_AD9837 },
+	{ .name = "ad9838", .driver_data = ID_AD9838 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad9834_id);
diff --git a/drivers/staging/iio/frequency/dds.h b/drivers/staging/iio/frequency/dds.h
index 2ebe68eb73986..b2ca8bb971e81 100644
--- a/drivers/staging/iio/frequency/dds.h
+++ b/drivers/staging/iio/frequency/dds.h
@@ -11,7 +11,7 @@
  * /sys/bus/iio/devices/.../out_altvoltageX_frequencyY
  */
 
-#define IIO_DEV_ATTR_FREQ(_channel, _num, _mode, _show, _store, _addr)	\
+#define IIO_DEV_ATTR_FREQ(_mode, _channel, _num, _show, _store, _addr)	\
 	IIO_DEVICE_ATTR(out_altvoltage##_channel##_frequency##_num,	\
 			_mode, _show, _store, _addr)
 
@@ -26,7 +26,7 @@
  * /sys/bus/iio/devices/.../out_altvoltageX_frequencysymbol
  */
 
-#define IIO_DEV_ATTR_FREQSYMBOL(_channel, _mode, _show, _store, _addr)	\
+#define IIO_DEV_ATTR_FREQSYMBOL(_mode, _channel, _show, _store, _addr)	\
 	IIO_DEVICE_ATTR(out_altvoltage##_channel##_frequencysymbol,	\
 			_mode, _show, _store, _addr)
 
@@ -34,7 +34,7 @@
  * /sys/bus/iio/devices/.../out_altvoltageX_phaseY
  */
 
-#define IIO_DEV_ATTR_PHASE(_channel, _num, _mode, _show, _store, _addr)	\
+#define IIO_DEV_ATTR_PHASE(_mode, _channel, _num, _show, _store, _addr)	\
 	IIO_DEVICE_ATTR(out_altvoltage##_channel##_phase##_num,		\
 			_mode, _show, _store, _addr)
 
@@ -49,7 +49,7 @@
  * /sys/bus/iio/devices/.../out_altvoltageX_phasesymbol
  */
 
-#define IIO_DEV_ATTR_PHASESYMBOL(_channel, _mode, _show, _store, _addr)	\
+#define IIO_DEV_ATTR_PHASESYMBOL(_mode, _channel, _show, _store, _addr)	\
 	IIO_DEVICE_ATTR(out_altvoltage##_channel##_phasesymbol,		\
 			_mode, _show, _store, _addr)
 
@@ -57,7 +57,7 @@
  * /sys/bus/iio/devices/.../out_altvoltageX_pincontrol_en
  */
 
-#define IIO_DEV_ATTR_PINCONTROL_EN(_channel, _mode, _show, _store, _addr)\
+#define IIO_DEV_ATTR_PINCONTROL_EN(_mode, _channel, _show, _store, _addr)\
 	IIO_DEVICE_ATTR(out_altvoltage##_channel##_pincontrol_en,	\
 			_mode, _show, _store, _addr)
 
@@ -65,7 +65,7 @@
  * /sys/bus/iio/devices/.../out_altvoltageX_pincontrol_frequency_en
  */
 
-#define IIO_DEV_ATTR_PINCONTROL_FREQ_EN(_channel, _mode, _show, _store, _addr)\
+#define IIO_DEV_ATTR_PINCONTROL_FREQ_EN(_mode, _channel, _show, _store, _addr)\
 	IIO_DEVICE_ATTR(out_altvoltage##_channel##_pincontrol_frequency_en,\
 			_mode, _show, _store, _addr)
 
@@ -73,7 +73,7 @@
  * /sys/bus/iio/devices/.../out_altvoltageX_pincontrol_phase_en
  */
 
-#define IIO_DEV_ATTR_PINCONTROL_PHASE_EN(_channel, _mode, _show, _store, _addr)\
+#define IIO_DEV_ATTR_PINCONTROL_PHASE_EN(_mode, _channel, _show, _store, _addr)\
 	IIO_DEVICE_ATTR(out_altvoltage##_channel##_pincontrol_phase_en,	\
 			_mode, _show, _store, _addr)
 
@@ -81,7 +81,7 @@
  * /sys/bus/iio/devices/.../out_altvoltageX_out_enable
  */
 
-#define IIO_DEV_ATTR_OUT_ENABLE(_channel, _mode, _show, _store, _addr)	\
+#define IIO_DEV_ATTR_OUT_ENABLE(_mode, _channel, _show, _store, _addr)	\
 	IIO_DEVICE_ATTR(out_altvoltage##_channel##_out_enable,		\
 			_mode, _show, _store, _addr)
 
@@ -89,8 +89,8 @@
  * /sys/bus/iio/devices/.../out_altvoltageX_outY_enable
  */
 
-#define IIO_DEV_ATTR_OUTY_ENABLE(_channel, _output,			\
-			_mode, _show, _store, _addr)			\
+#define IIO_DEV_ATTR_OUTY_ENABLE(_mode, _channel, _output,		\
+			_show, _store, _addr)				\
 	IIO_DEVICE_ATTR(out_altvoltage##_channel##_out##_output##_enable,\
 			_mode, _show, _store, _addr)
 
@@ -98,9 +98,9 @@
  * /sys/bus/iio/devices/.../out_altvoltageX_outY_wavetype
  */
 
-#define IIO_DEV_ATTR_OUT_WAVETYPE(_channel, _output, _store, _addr)	\
+#define IIO_DEV_ATTR_OUT_WAVETYPE(_mode, _channel, _output, _store, _addr)\
 	IIO_DEVICE_ATTR(out_altvoltage##_channel##_out##_output##_wavetype,\
-			0200, NULL, _store, _addr)
+			_mode, NULL, _store, _addr)
 
 /**
  * /sys/bus/iio/devices/.../out_altvoltageX_outY_wavetype_available
diff --git a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf.h b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf.h
index 4732b45b25eeb..fbcd73b711baf 100644
--- a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf.h
+++ b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf.h
@@ -20,7 +20,7 @@
  * Data structures.
  *
  ****************************************************************/
-/**
+/*
  * @brief Data structure for the circular buffer.
  */
 typedef struct ia_css_circbuf_s ia_css_circbuf_t;
@@ -29,7 +29,7 @@ struct ia_css_circbuf_s {
 	ia_css_circbuf_elem_t *elems;	/* an array of elements    */
 };
 
-/**
+/*
  * @brief Create the circular buffer.
  *
  * @param cb	The pointer to the circular buffer.
@@ -41,7 +41,7 @@ void ia_css_circbuf_create(
     ia_css_circbuf_elem_t *elems,
     ia_css_circbuf_desc_t *desc);
 
-/**
+/*
  * @brief Destroy the circular buffer.
  *
  * @param cb The pointer to the circular buffer.
@@ -49,7 +49,7 @@ void ia_css_circbuf_create(
 void ia_css_circbuf_destroy(
     ia_css_circbuf_t *cb);
 
-/**
+/*
  * @brief Pop a value out of the circular buffer.
  * Get a value at the head of the circular buffer.
  * The user should call "ia_css_circbuf_is_empty()"
@@ -62,7 +62,7 @@ void ia_css_circbuf_destroy(
 uint32_t ia_css_circbuf_pop(
     ia_css_circbuf_t *cb);
 
-/**
+/*
  * @brief Extract a value out of the circular buffer.
  * Get a value at an arbitrary position in the circular
  * buffer. The user should call "ia_css_circbuf_is_empty()"
@@ -82,7 +82,7 @@ uint32_t ia_css_circbuf_extract(
  * Inline functions.
  *
  ****************************************************************/
-/**
+/*
  * @brief Set the "val" field in the element.
  *
  * @param elem The pointer to the element.
@@ -97,7 +97,7 @@ static inline void ia_css_circbuf_elem_set_val(
 	elem->val = val;
 }
 
-/**
+/*
  * @brief Initialize the element.
  *
  * @param elem The pointer to the element.
@@ -109,7 +109,7 @@ static inline void ia_css_circbuf_elem_init(
 	ia_css_circbuf_elem_set_val(elem, 0);
 }
 
-/**
+/*
  * @brief Copy an element.
  *
  * @param src  The element as the copy source.
@@ -125,7 +125,7 @@ static inline void ia_css_circbuf_elem_cpy(
 	ia_css_circbuf_elem_set_val(dest, src->val);
 }
 
-/**
+/*
  * @brief Get position in the circular buffer.
  *
  * @param cb		The pointer to the circular buffer.
@@ -151,7 +151,7 @@ static inline uint8_t ia_css_circbuf_get_pos_at_offset(
 	return (base + offset) % cb->desc->size;
 }
 
-/**
+/*
  * @brief Get the offset between two positions in the circular buffer.
  * Get the offset from the source position to the terminal position,
  * along the direction in which the new elements come in.
@@ -178,7 +178,7 @@ static inline int ia_css_circbuf_get_offset(
 	return offset;
 }
 
-/**
+/*
  * @brief Get the maximum number of elements.
  *
  * @param cb The pointer to the circular buffer.
@@ -196,7 +196,7 @@ static inline uint32_t ia_css_circbuf_get_size(
 	return cb->desc->size;
 }
 
-/**
+/*
  * @brief Get the number of available elements.
  *
  * @param cb The pointer to the circular buffer.
@@ -216,7 +216,7 @@ static inline uint32_t ia_css_circbuf_get_num_elems(
 	return (uint32_t)num;
 }
 
-/**
+/*
  * @brief Test if the circular buffer is empty.
  *
  * @param cb	The pointer to the circular buffer.
@@ -234,7 +234,7 @@ static inline bool ia_css_circbuf_is_empty(
 	return ia_css_circbuf_desc_is_empty(cb->desc);
 }
 
-/**
+/*
  * @brief Test if the circular buffer is full.
  *
  * @param cb	The pointer to the circular buffer.
@@ -251,7 +251,7 @@ static inline bool ia_css_circbuf_is_full(ia_css_circbuf_t *cb)
 	return ia_css_circbuf_desc_is_full(cb->desc);
 }
 
-/**
+/*
  * @brief Write a new element into the circular buffer.
  * Write a new element WITHOUT checking whether the
  * circular buffer is full or not. So it also overwrites
@@ -275,7 +275,7 @@ static inline void ia_css_circbuf_write(
 	cb->desc->end = ia_css_circbuf_get_pos_at_offset(cb, cb->desc->end, 1);
 }
 
-/**
+/*
  * @brief Push a value in the circular buffer.
  * Put a new value at the tail of the circular buffer.
  * The user should call "ia_css_circbuf_is_full()"
@@ -300,7 +300,7 @@ static inline void ia_css_circbuf_push(
 	ia_css_circbuf_write(cb, elem);
 }
 
-/**
+/*
  * @brief Get the number of free elements.
  *
  * @param cb The pointer to the circular buffer.
@@ -316,7 +316,7 @@ static inline uint32_t ia_css_circbuf_get_free_elems(
 	return ia_css_circbuf_desc_get_free_elems(cb->desc);
 }
 
-/**
+/*
  * @brief Peek an element in Circular Buffer.
  *
  * @param cb	 The pointer to the circular buffer.
@@ -328,7 +328,7 @@ uint32_t ia_css_circbuf_peek(
     ia_css_circbuf_t *cb,
     int offset);
 
-/**
+/*
  * @brief Get an element in Circular Buffer.
  *
  * @param cb	 The pointer to the circular buffer.
@@ -340,7 +340,7 @@ uint32_t ia_css_circbuf_peek_from_start(
     ia_css_circbuf_t *cb,
     int offset);
 
-/**
+/*
  * @brief Increase Size of a Circular Buffer.
  * Use 'CAUTION' before using this function, This was added to
  * support / fix issue with increasing size for tagger only
diff --git a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_comm.h b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_comm.h
index 971e07f2acc53..090e2e7640421 100644
--- a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_comm.h
+++ b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_comm.h
@@ -24,7 +24,7 @@
  * Portable Data structures
  *
  ****************************************************************/
-/**
+/*
  * @brief Data structure for the circular descriptor.
  */
 typedef struct ia_css_circbuf_desc_s ia_css_circbuf_desc_t;
@@ -40,7 +40,7 @@ struct ia_css_circbuf_desc_s {
 
 static_assert(sizeof(struct ia_css_circbuf_desc_s) == SIZE_OF_IA_CSS_CIRCBUF_DESC_S_STRUCT);
 
-/**
+/*
  * @brief Data structure for the circular buffer element.
  */
 typedef struct ia_css_circbuf_elem_s ia_css_circbuf_elem_t;
diff --git a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_desc.h b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_desc.h
index 64f754f1d49b3..3306a7a0fa5eb 100644
--- a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_desc.h
+++ b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_desc.h
@@ -17,7 +17,7 @@
  * Inline functions.
  *
  ****************************************************************/
-/**
+/*
  * @brief Test if the circular buffer is empty.
  *
  * @param cb_desc The pointer to the circular buffer descriptor.
@@ -33,7 +33,7 @@ static inline bool ia_css_circbuf_desc_is_empty(
 	return (cb_desc->end == cb_desc->start);
 }
 
-/**
+/*
  * @brief Test if the circular buffer descriptor is full.
  *
  * @param cb_desc	The pointer to the circular buffer
@@ -50,7 +50,7 @@ static inline bool ia_css_circbuf_desc_is_full(
 	return ((cb_desc->end + 1) % cb_desc->size) == cb_desc->start;
 }
 
-/**
+/*
  * @brief Initialize the circular buffer descriptor
  *
  * @param cb_desc	The pointer circular buffer descriptor
@@ -64,7 +64,7 @@ static inline void ia_css_circbuf_desc_init(
 	cb_desc->size = size;
 }
 
-/**
+/*
  * @brief Get a position in the circular buffer descriptor.
  *
  * @param cb     The pointer to the circular buffer descriptor.
@@ -89,7 +89,7 @@ static inline uint8_t ia_css_circbuf_desc_get_pos_at_offset(
 	return (base + offset) % cb_desc->size;
 }
 
-/**
+/*
  * @brief Get the offset between two positions in the circular buffer
  * descriptor.
  * Get the offset from the source position to the terminal position,
@@ -116,7 +116,7 @@ static inline int ia_css_circbuf_desc_get_offset(
 	return offset;
 }
 
-/**
+/*
  * @brief Get the number of available elements.
  *
  * @param cb_desc The pointer to the circular buffer.
@@ -137,7 +137,7 @@ static inline uint32_t ia_css_circbuf_desc_get_num_elems(
 	return (uint32_t)num;
 }
 
-/**
+/*
  * @brief Get the number of free elements.
  *
  * @param cb_desc The pointer to the circular buffer descriptor.
diff --git a/drivers/staging/media/atomisp/pci/camera/util/interface/ia_css_util.h b/drivers/staging/media/atomisp/pci/camera/util/interface/ia_css_util.h
index 24cd99a659cab..dce8dd23b47c2 100644
--- a/drivers/staging/media/atomisp/pci/camera/util/interface/ia_css_util.h
+++ b/drivers/staging/media/atomisp/pci/camera/util/interface/ia_css_util.h
@@ -81,7 +81,7 @@ bool ia_css_util_res_leq(
     struct ia_css_resolution b);
 
 /* ISP2401 */
-/**
+/*
  * @brief Check if resolution is zero
  *
  * @param[in] resolution The resolution to check
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_private.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_private.h
index 989f55bec5190..43bb352a8e05a 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_private.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_private.h
@@ -21,7 +21,7 @@
  * Device level interface (DLI).
  *
  *****************************************************/
-/**
+/*
  * @brief Load the register value.
  * Refer to "csi_rx_public.h" for details.
  */
@@ -35,7 +35,7 @@ static inline hrt_data csi_rx_fe_ctrl_reg_load(
 					     hrt_data));
 }
 
-/**
+/*
  * @brief Store a value to the register.
  * Refer to "ibuf_ctrl_public.h" for details.
  */
@@ -51,7 +51,7 @@ static inline void csi_rx_fe_ctrl_reg_store(
 				   value);
 }
 
-/**
+/*
  * @brief Load the register value.
  * Refer to "csi_rx_public.h" for details.
  */
@@ -65,7 +65,7 @@ static inline hrt_data csi_rx_be_ctrl_reg_load(
 					     hrt_data));
 }
 
-/**
+/*
  * @brief Store a value to the register.
  * Refer to "ibuf_ctrl_public.h" for details.
  */
@@ -88,7 +88,7 @@ static inline void csi_rx_be_ctrl_reg_store(
  * Native command interface (NCI).
  *
  *****************************************************/
-/**
+/*
  * @brief Get the state of the csi rx fe dlane process.
  * Refer to "csi_rx_public.h" for details.
  */
@@ -103,7 +103,7 @@ static inline void csi_rx_fe_ctrl_get_dlane_state(
 	    csi_rx_fe_ctrl_reg_load(ID, _HRT_CSI_RX_DLY_CNT_SETTLE_DLANE_REG_IDX(lane));
 }
 
-/**
+/*
  * @brief Get the csi rx fe state.
  * Refer to "csi_rx_public.h" for details.
  */
@@ -142,7 +142,7 @@ static inline void csi_rx_fe_ctrl_get_state(
 	}
 }
 
-/**
+/*
  * @brief dump the csi rx fe state.
  * Refer to "csi_rx_public.h" for details.
  */
@@ -180,7 +180,7 @@ static inline void csi_rx_fe_ctrl_dump_state(
 	}
 }
 
-/**
+/*
  * @brief Get the csi rx be state.
  * Refer to "csi_rx_public.h" for details.
  */
@@ -243,7 +243,7 @@ static inline void csi_rx_be_ctrl_get_state(
 	}
 }
 
-/**
+/*
  * @brief Dump the csi rx be state.
  * Refer to "csi_rx_public.h" for details.
  */
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_private.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_private.h
index 4bd8209aaa019..85439d561ba3c 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_private.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_private.h
@@ -15,7 +15,7 @@
  |             Native command interface (NCI)             |
  + -------------------------------------------------------*/
 
-/**
+/*
 * @brief Get the isys irq status.
 * Refer to "isys_irq.h" for details.
 */
@@ -34,7 +34,7 @@ void isys_irqc_state_get(
 	*/
 }
 
-/**
+/*
 * @brief Dump the isys irq status.
 * Refer to "isys_irq.h" for details.
 */
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_private.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_private.h
index 8e295cd781299..88a7633656310 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_private.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_private.h
@@ -27,7 +27,7 @@
  * Native command interface (NCI).
  *
  *****************************************************/
-/**
+/*
  * @brief Get the stream2mmio-controller state.
  * Refer to "stream2mmio_public.h" for details.
  */
@@ -45,7 +45,7 @@ STORAGE_CLASS_STREAM2MMIO_C void stream2mmio_get_state(
 		stream2mmio_get_sid_state(ID, i, &state->sid_state[i]);
 }
 
-/**
+/*
  * @brief Get the state of the stream2mmio-controller sidess.
  * Refer to "stream2mmio_public.h" for details.
  */
@@ -76,7 +76,7 @@ STORAGE_CLASS_STREAM2MMIO_C void stream2mmio_get_sid_state(
 	    stream2mmio_reg_load(ID, sid_id, STREAM2MMIO_BLOCK_WHEN_NO_CMD_REG_ID);
 }
 
-/**
+/*
  * @brief Dump the state of the stream2mmio-controller sidess.
  * Refer to "stream2mmio_public.h" for details.
  */
@@ -92,7 +92,7 @@ STORAGE_CLASS_STREAM2MMIO_C void stream2mmio_print_sid_state(
 	ia_css_print("\t \t block when no cmd 0x%x\n", state->block_when_no_cmd);
 }
 
-/**
+/*
  * @brief Dump the ibuf-controller state.
  * Refer to "stream2mmio_public.h" for details.
  */
@@ -119,7 +119,7 @@ STORAGE_CLASS_STREAM2MMIO_C void stream2mmio_dump_state(
  * Device level interface (DLI).
  *
  *****************************************************/
-/**
+/*
  * @brief Load the register value.
  * Refer to "stream2mmio_public.h" for details.
  */
@@ -137,7 +137,7 @@ STORAGE_CLASS_STREAM2MMIO_C hrt_data stream2mmio_reg_load(
 					 (reg_bank_offset + reg_idx) * sizeof(hrt_data));
 }
 
-/**
+/*
  * @brief Store a value to the register.
  * Refer to "stream2mmio_public.h" for details.
  */
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_private.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_private.h
index b8b98106bd318..0faf6ffdf55da 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_private.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_private.h
@@ -16,7 +16,7 @@
  * Device level interface (DLI).
  *
  *****************************************************/
-/**
+/*
  * @brief Load the register value.
  * Refer to "pixelgen_public.h" for details.
  */
@@ -30,7 +30,7 @@ STORAGE_CLASS_PIXELGEN_C hrt_data pixelgen_ctrl_reg_load(
 					     hrt_data));
 }
 
-/**
+/*
  * @brief Store a value to the register.
  * Refer to "pixelgen_ctrl_public.h" for details.
  */
@@ -53,7 +53,7 @@ STORAGE_CLASS_PIXELGEN_C void pixelgen_ctrl_reg_store(
  * Native command interface (NCI).
  *
  *****************************************************/
-/**
+/*
  * @brief Get the pixelgen state.
  * Refer to "pixelgen_public.h" for details.
  */
@@ -117,7 +117,7 @@ STORAGE_CLASS_PIXELGEN_C void pixelgen_ctrl_get_state(
 	    pixelgen_ctrl_reg_load(ID, _PXG_TPG_B2_REG_IDX);
 }
 
-/**
+/*
  * @brief Dump the pixelgen state.
  * Refer to "pixelgen_public.h" for details.
  */
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/pixelgen_global.h b/drivers/staging/media/atomisp/pci/css_2401_system/pixelgen_global.h
index 59e0b44bfdc3d..b5a6a8f0c3d2f 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/pixelgen_global.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/pixelgen_global.h
@@ -9,7 +9,7 @@
 
 #include <type_support.h>
 
-/**
+/*
  * Pixel-generator. ("pixelgen_global.h")
  */
 /*
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/timed_ctrl_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/timed_ctrl_global.h
index 2dc0fb88399fa..e4c136660b1e4 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/timed_ctrl_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/timed_ctrl_global.h
@@ -11,7 +11,7 @@
 
 #include "timed_controller_defs.h"
 
-/**
+/*
  * Order of the input bits for the timed controller taken from
  * ISP_CSS_2401 System Architecture Description valid for
  * 2400, 2401.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/device_access/device_access.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/device_access/device_access.h
index ca33a1e03d8d6..f5b0ff4143955 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/device_access/device_access.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/device_access/device_access.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/**
+/*
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
 
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/csi_rx_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/csi_rx_public.h
index 2002960f078e7..e8ce556bfd6a0 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/csi_rx_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/csi_rx_public.h
@@ -12,7 +12,7 @@
  * Native command interface (NCI).
  *
  *****************************************************/
-/**
+/*
  * @brief Get the csi rx frontend state.
  * Get the state of the csi rx frontend regiester-set.
  *
@@ -22,7 +22,7 @@
 void csi_rx_fe_ctrl_get_state(
     const csi_rx_frontend_ID_t ID,
     csi_rx_fe_ctrl_state_t *state);
-/**
+/*
  * @brief Dump the csi rx frontend state.
  * Dump the state of the csi rx frontend regiester-set.
  *
@@ -32,7 +32,7 @@ void csi_rx_fe_ctrl_get_state(
 void csi_rx_fe_ctrl_dump_state(
     const csi_rx_frontend_ID_t ID,
     csi_rx_fe_ctrl_state_t *state);
-/**
+/*
  * @brief Get the state of the csi rx fe dlane.
  * Get the state of the register set per dlane process.
  *
@@ -44,7 +44,7 @@ void csi_rx_fe_ctrl_get_dlane_state(
     const csi_rx_frontend_ID_t ID,
     const u32 lane,
     csi_rx_fe_ctrl_lane_t *dlane_state);
-/**
+/*
  * @brief Get the csi rx backend state.
  * Get the state of the csi rx backend regiester-set.
  *
@@ -54,7 +54,7 @@ void csi_rx_fe_ctrl_get_dlane_state(
 void csi_rx_be_ctrl_get_state(
     const csi_rx_backend_ID_t ID,
     csi_rx_be_ctrl_state_t *state);
-/**
+/*
  * @brief Dump the csi rx backend state.
  * Dump the state of the csi rx backend regiester-set.
  *
@@ -71,7 +71,7 @@ void csi_rx_be_ctrl_dump_state(
  * Device level interface (DLI).
  *
  *****************************************************/
-/**
+/*
  * @brief Load the register value.
  * Load the value of the register of the csi rx fe.
  *
@@ -83,7 +83,7 @@ void csi_rx_be_ctrl_dump_state(
 hrt_data csi_rx_fe_ctrl_reg_load(
     const csi_rx_frontend_ID_t ID,
     const hrt_address reg);
-/**
+/*
  * @brief Store a value to the register.
  * Store a value to the register of the csi rx fe.
  *
@@ -96,7 +96,7 @@ void csi_rx_fe_ctrl_reg_store(
     const csi_rx_frontend_ID_t ID,
     const hrt_address reg,
     const hrt_data value);
-/**
+/*
  * @brief Load the register value.
  * Load the value of the register of the csirx be.
  *
@@ -108,7 +108,7 @@ void csi_rx_fe_ctrl_reg_store(
 hrt_data csi_rx_be_ctrl_reg_load(
     const csi_rx_backend_ID_t ID,
     const hrt_address reg);
-/**
+/*
  * @brief Store a value to the register.
  * Store a value to the register of the csi rx be.
  *
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_stream2mmio_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_stream2mmio_public.h
index 7ed55d427cf63..e4f77e975f4e1 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_stream2mmio_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_stream2mmio_public.h
@@ -12,7 +12,7 @@
  * Native command interface (NCI).
  *
  *****************************************************/
-/**
+/*
  * @brief Get the stream2mmio-controller state.
  * Get the state of the stream2mmio-controller regiester-set.
  *
@@ -23,7 +23,7 @@ STORAGE_CLASS_STREAM2MMIO_H void stream2mmio_get_state(
     const stream2mmio_ID_t ID,
     stream2mmio_state_t *state);
 
-/**
+/*
  * @brief Get the state of the stream2mmio-controller sidess.
  * Get the state of the register set per buf-controller sidess.
  *
@@ -42,7 +42,7 @@ STORAGE_CLASS_STREAM2MMIO_H void stream2mmio_get_sid_state(
  * Device level interface (DLI).
  *
  *****************************************************/
-/**
+/*
  * @brief Load the register value.
  * Load the value of the register of the stream2mmio-controller.
  *
@@ -57,7 +57,7 @@ STORAGE_CLASS_STREAM2MMIO_H hrt_data stream2mmio_reg_load(
     const stream2mmio_sid_ID_t sid_id,
     const uint32_t reg_idx);
 
-/**
+/*
  * @brief Dump the SID processor state.
  * Dump the state of the sid regiester-set.
  *
@@ -65,7 +65,7 @@ STORAGE_CLASS_STREAM2MMIO_H hrt_data stream2mmio_reg_load(
  */
 STORAGE_CLASS_STREAM2MMIO_H void stream2mmio_print_sid_state(
     stream2mmio_sid_state_t	*state);
-/**
+/*
  * @brief Dump the stream2mmio state.
  * Dump the state of the ibuf-controller regiester-set.
  *
@@ -75,7 +75,7 @@ STORAGE_CLASS_STREAM2MMIO_H void stream2mmio_print_sid_state(
 STORAGE_CLASS_STREAM2MMIO_H void stream2mmio_dump_state(
     const stream2mmio_ID_t ID,
     stream2mmio_state_t *state);
-/**
+/*
  * @brief Store a value to the register.
  * Store a value to the registe of the stream2mmio-controller.
  *
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/pixelgen_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/pixelgen_public.h
index dc31ce3cd741e..6aabf0b1995f0 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/pixelgen_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/pixelgen_public.h
@@ -12,7 +12,7 @@
  * Native command interface (NCI).
  *
  *****************************************************/
-/**
+/*
  * @brief Get the pixelgen state.
  * Get the state of the pixelgen regiester-set.
  *
@@ -22,7 +22,7 @@
 STORAGE_CLASS_PIXELGEN_H void pixelgen_ctrl_get_state(
     const pixelgen_ID_t ID,
     pixelgen_ctrl_state_t *state);
-/**
+/*
  * @brief Dump the pixelgen state.
  * Dump the state of the pixelgen regiester-set.
  *
@@ -39,7 +39,7 @@ STORAGE_CLASS_PIXELGEN_H void pixelgen_ctrl_dump_state(
  * Device level interface (DLI).
  *
  *****************************************************/
-/**
+/*
  * @brief Load the register value.
  * Load the value of the register of the pixelgen
  *
@@ -51,7 +51,7 @@ STORAGE_CLASS_PIXELGEN_H void pixelgen_ctrl_dump_state(
 STORAGE_CLASS_PIXELGEN_H hrt_data pixelgen_ctrl_reg_load(
     const pixelgen_ID_t ID,
     const hrt_address reg);
-/**
+/*
  * @brief Store a value to the register.
  * Store a value to the registe of the pixelgen
  *
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/tag_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/tag_public.h
index ad83ff97bbd6a..1652c9f793499 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/tag_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/tag_public.h
@@ -7,7 +7,7 @@
 #ifndef __TAG_PUBLIC_H_INCLUDED__
 #define __TAG_PUBLIC_H_INCLUDED__
 
-/**
+/*
  * @brief	Creates the tag description from the given parameters.
  * @param[in]	num_captures
  * @param[in]	skip
@@ -21,7 +21,7 @@ sh_css_create_tag_descr(int num_captures,
 			unsigned int exp_id,
 			struct sh_css_tag_descr *tag_descr);
 
-/**
+/*
  * @brief	Encodes the members of tag description into a 32-bit value.
  * @param[in]	tag		Pointer to the tag description
  * @return	(unsigned int)	Encoded 32-bit tag-info
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/platform_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/platform_support.h
index 473d8d4fb9baa..20a5faad0716b 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/platform_support.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/platform_support.h
@@ -7,7 +7,7 @@
 #ifndef __PLATFORM_SUPPORT_H_INCLUDED__
 #define __PLATFORM_SUPPORT_H_INCLUDED__
 
-/**
+/*
 * @file
 * Platform specific includes and functionality.
 */
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/type_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/type_support.h
index 097be6bd3cb52..cbd290121a284 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/type_support.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/type_support.h
@@ -7,7 +7,7 @@
 #ifndef __TYPE_SUPPORT_H_INCLUDED__
 #define __TYPE_SUPPORT_H_INCLUDED__
 
-/**
+/*
 * @file
 * Platform specific types.
 *
diff --git a/drivers/staging/media/atomisp/pci/ia_css_frame_public.h b/drivers/staging/media/atomisp/pci/ia_css_frame_public.h
index 7acfedb541d86..936b17950b3ad 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_frame_public.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_frame_public.h
@@ -119,7 +119,7 @@ struct ia_css_frame_info {
 	.raw_bayer_order	= IA_CSS_BAYER_ORDER_NUM, \
 }
 
-/**
+/*
  *  Specifies the DVS loop delay in "frame periods"
  */
 enum ia_css_frame_delay {
diff --git a/drivers/staging/media/atomisp/pci/ia_css_host_data.h b/drivers/staging/media/atomisp/pci/ia_css_host_data.h
index 0e45650cc1ab2..547542577902f 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_host_data.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_host_data.h
@@ -10,7 +10,7 @@
 
 #include <ia_css_types.h>	/* ia_css_pipe */
 
-/**
+/*
  * @brief Allocate structure ia_css_host_data.
  *
  * @param[in]	size		Size of the requested host data
@@ -22,7 +22,7 @@
 struct ia_css_host_data *
 ia_css_host_data_allocate(size_t size);
 
-/**
+/*
  * @brief Free structure ia_css_host_data.
  *
  * @param[in]	me	Pointer to structure, if a NULL is passed functions
diff --git a/drivers/staging/media/atomisp/pci/ia_css_pipe_public.h b/drivers/staging/media/atomisp/pci/ia_css_pipe_public.h
index 2bb06b0ff5dba..1d1b704b2bc65 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_pipe_public.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_pipe_public.h
@@ -44,7 +44,7 @@ enum ia_css_pipe_mode {
 /* Temporary define  */
 #define IA_CSS_PIPE_MODE_NUM (IA_CSS_PIPE_MODE_YUVPP + 1)
 
-/**
+/*
  * Enumeration of pipe versions.
  * the order should match with definition in sh_css_defs.h
  */
@@ -55,7 +55,7 @@ enum ia_css_pipe_version {
 	IA_CSS_PIPE_VERSION_2_7 = 4		/** ISP2.7 pipe */
 };
 
-/**
+/*
  * Pipe configuration structure.
  * Resolution properties are filled by Driver, kernel configurations are
  * set by AIC
@@ -123,7 +123,7 @@ struct ia_css_pipe_config {
 		   to retrieve shading gains which correspond to bayer data. */
 };
 
-/**
+/*
  * Default settings for newly created pipe configurations.
  */
 #define DEFAULT_PIPE_CONFIG { \
@@ -175,7 +175,7 @@ struct ia_css_pipe_info {
 	     output at the first valid frame. */
 };
 
-/**
+/*
  * Defaults for ia_css_pipe_info structs.
  */
 #define DEFAULT_PIPE_INFO {\
diff --git a/drivers/staging/media/atomisp/pci/ia_css_prbs.h b/drivers/staging/media/atomisp/pci/ia_css_prbs.h
index abdbcb8fda533..3bb1e5dbb4fc0 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_prbs.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_prbs.h
@@ -19,7 +19,7 @@ enum ia_css_prbs_id {
 	IA_CSS_PRBS_ID2
 };
 
-/**
+/*
  * Maximum number of PRBS IDs.
  *
  * Make sure the value of this define gets changed to reflect the correct
@@ -27,7 +27,7 @@ enum ia_css_prbs_id {
  */
 #define N_CSS_PRBS_IDS (IA_CSS_PRBS_ID2 + 1)
 
-/**
+/*
  * PRBS configuration structure.
  *
  * Seed the for the Pseudo Random Bit Sequence.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_stream.h b/drivers/staging/media/atomisp/pci/ia_css_stream.h
index c8de632a8e125..578dc45785fcb 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_stream.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_stream.h
@@ -13,7 +13,7 @@
 #include "ia_css_types.h"
 #include "ia_css_stream_public.h"
 
-/**
+/*
  * structure to hold all internal stream related information
  */
 struct ia_css_stream {
diff --git a/drivers/staging/media/atomisp/pci/ia_css_timer.h b/drivers/staging/media/atomisp/pci/ia_css_timer.h
index da752834adf45..5c73e8c615884 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_timer.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_timer.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/**
+/*
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
 
@@ -42,7 +42,10 @@ struct ia_css_time_meas {
 	clock_value_t	end_timer_value;	/** measured time in ticks */
 };
 
-/**@brief SIZE_OF_IA_CSS_CLOCK_TICK_STRUCT checks to ensure correct alignment for struct ia_css_clock_tick. */
+/*
+ * @brief SIZE_OF_IA_CSS_CLOCK_TICK_STRUCT checks to ensure correct alignment
+ * for struct ia_css_clock_tick.
+ */
 #define SIZE_OF_IA_CSS_CLOCK_TICK_STRUCT sizeof(clock_value_t)
 /* @brief checks to ensure correct alignment for ia_css_time_meas. */
 #define SIZE_OF_IA_CSS_TIME_MEAS_STRUCT (sizeof(clock_value_t) \
diff --git a/drivers/staging/media/atomisp/pci/ia_css_types.h b/drivers/staging/media/atomisp/pci/ia_css_types.h
index 2b7db9cda23ab..7065eded7a33b 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_types.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_types.h
@@ -53,7 +53,7 @@
 #include "isp/kernels/output/output_1.0/ia_css_output_types.h"
 
 #define IA_CSS_DVS_STAT_GRID_INFO_SUPPORTED
-/** Should be removed after Driver adaptation will be done */
+/* Should be removed after Driver adaptation will be done */
 
 #define IA_CSS_VERSION_MAJOR    2
 #define IA_CSS_VERSION_MINOR    0
@@ -421,7 +421,7 @@ struct ia_css_dvs_6axis_config {
 	u32 *ycoords_uv;
 };
 
-/**
+/*
  * This specifies the coordinates (x,y)
  */
 struct ia_css_point {
@@ -429,7 +429,7 @@ struct ia_css_point {
 	s32 y; /** y coordinate */
 };
 
-/**
+/*
  * Digital zoom:
  * This feature is currently available only for video, but will become
  * available for preview and capture as well.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_types.h
index 374261d25520e..ee2cf2877b7db 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_types.h
@@ -7,7 +7,7 @@
 #ifndef __IA_CSS_CONVERSION_TYPES_H
 #define __IA_CSS_CONVERSION_TYPES_H
 
-/**
+/*
  *  Conversion Kernel parameters.
  *  Deinterleave bayer quad into isys format
  *
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_types.h
index 0b977eb7ad717..bbd97c71a6fd8 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_types.h
@@ -13,7 +13,7 @@
 
 #include "type_support.h"
 
-/**
+/*
  * \brief EED1_8 public parameters.
  * \details Struct with all parameters for the EED1.8 kernel that can be set
  * from the CSS API.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_types.h
index e32290d1c86b0..d6785724fe976 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_types.h
@@ -10,44 +10,46 @@
 
 #define IA_CSS_HDR_MAX_NUM_INPUT_FRAMES         (3)
 
-/**
+/*
  * \brief HDR Irradiance Parameters
  * \detail Currently HDR parameters are used only for testing purposes
  */
 struct ia_css_hdr_irradiance_params {
-	int test_irr;                                          /** Test parameter */
-	int match_shift[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES -
-							1];  /** Histogram matching shift parameter */
-	int match_mul[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES -
-						      1];    /** Histogram matching multiplication parameter */
-	int thr_low[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES -
-						    1];      /** Weight map soft threshold low bound parameter */
-	int thr_high[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES -
-						     1];     /** Weight map soft threshold high bound parameter */
-	int thr_coeff[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES -
-						      1];    /** Soft threshold linear function coefficien */
-	int thr_shift[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES -
-						      1];    /** Soft threshold precision shift parameter */
-	int weight_bpp;                                        /** Weight map bits per pixel */
+	/* Test parameter */
+	int test_irr;
+	/* Histogram matching shift parameter */
+	int match_shift[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES - 1];
+	/* Histogram matching multiplication parameter */
+	int match_mul[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES - 1];
+	/* Weight map soft threshold low bound parameter */
+	int thr_low[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES - 1];
+	/* Weight map soft threshold high bound parameter */
+	int thr_high[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES - 1];
+	/* Soft threshold linear function coefficien */
+	int thr_coeff[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES - 1];
+	/* Soft threshold precision shift parameter */
+	int thr_shift[IA_CSS_HDR_MAX_NUM_INPUT_FRAMES - 1];
+	/* Weight map bits per pixel */
+	int weight_bpp;
 };
 
-/**
+/*
  * \brief HDR Deghosting Parameters
  * \detail Currently HDR parameters are used only for testing purposes
  */
 struct ia_css_hdr_deghost_params {
-	int test_deg; /** Test parameter */
+	int test_deg; /* Test parameter */
 };
 
-/**
+/*
  * \brief HDR Exclusion Parameters
  * \detail Currently HDR parameters are used only for testing purposes
  */
 struct ia_css_hdr_exclusion_params {
-	int test_excl; /** Test parameter */
+	int test_excl; /* Test parameter */
 };
 
-/**
+/*
  * \brief HDR public parameters.
  * \details Struct with all parameters for HDR that can be seet from
  * the CSS API. Currently, only test parameters are defined.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_param.h
index c9a3f7bfaa902..de535503b70ee 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_param.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/**
+/*
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
 
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_types.h
index c1c93b2245c5e..69a40a15264bd 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_types.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/**
+/*
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
 
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.h
index e6ce0cba44b95..72310781cf7e5 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/**
+/*
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
 
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h
index 429fcdb73f60b..9fabb8b2d3b82 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/**
+/*
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
 
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h
index 485ad17cb29ba..1daafd96a0cdb 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/**
+/*
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
 
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h
index 42b760ffac67d..676a633336a29 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/**
+/*
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
 
@@ -12,7 +12,7 @@ Copyright (c) 2010 - 2015, Intel Corporation.
 * CSS-API header file for Temporal Noise Reduction v3 (TNR3) kernel
 */
 
-/**
+/*
  * \brief Number of piecewise linear segments.
  * \details The parameters to TNR3 are specified as a piecewise linear segment.
  * The number of such segments is fixed at 3.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h
index c012c1865d1ec..fc4b9033951c1 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h
@@ -11,25 +11,25 @@
 * CSS-API header file for Extra Noise Reduction (XNR) parameters.
 */
 
-/**
+/*
  * \brief Scale of the XNR sigma parameters.
  * \details The define specifies which fixed-point value represents 1.0.
  */
 #define IA_CSS_XNR3_SIGMA_SCALE  BIT(10)
 
-/**
+/*
  * \brief Scale of the XNR coring parameters.
  * \details The define specifies which fixed-point value represents 1.0.
  */
 #define IA_CSS_XNR3_CORING_SCALE BIT(15)
 
-/**
+/*
  * \brief Scale of the XNR blending parameter.
  * \details The define specifies which fixed-point value represents 1.0.
  */
 #define IA_CSS_XNR3_BLENDING_SCALE BIT(11)
 
-/**
+/*
  * \brief XNR3 Sigma Parameters.
  * \details Sigma parameters define the strength of the XNR filter.
  * A higher number means stronger filtering. There are two values for each of
@@ -46,7 +46,7 @@ struct ia_css_xnr3_sigma_params {
 	int v1;     /** Sigma for V range similarity in bright area */
 };
 
-/**
+/*
  * \brief XNR3 Coring Parameters
  * \details Coring parameters define the "coring" strength, which is a soft
  * thresholding technique to avoid false coloring. There are two values for
@@ -61,7 +61,7 @@ struct ia_css_xnr3_coring_params {
 	int v1;     /** Coring threshold of V channel in bright area */
 };
 
-/**
+/*
  * \brief XNR3 Blending Parameters
  * \details Blending parameters define the blending strength of filtered
  * output pixels with the original chroma pixels from before xnr3. The
@@ -75,7 +75,7 @@ struct ia_css_xnr3_blending_params {
 	int strength;   /** Blending strength */
 };
 
-/**
+/*
  * \brief XNR3 public parameters.
  * \details Struct with all parameters for the XNR3 kernel that can be set
  * from the CSS API.
diff --git a/drivers/staging/media/atomisp/pci/isp/modes/interface/input_buf.isp.h b/drivers/staging/media/atomisp/pci/isp/modes/interface/input_buf.isp.h
index 6a0257359e695..d6ed1fbb0e8e2 100644
--- a/drivers/staging/media/atomisp/pci/isp/modes/interface/input_buf.isp.h
+++ b/drivers/staging/media/atomisp/pci/isp/modes/interface/input_buf.isp.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/**
+/*
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
 
diff --git a/drivers/staging/media/atomisp/pci/runtime/binary/interface/ia_css_binary.h b/drivers/staging/media/atomisp/pci/runtime/binary/interface/ia_css_binary.h
index 9c682f2ecbb2c..5f768af9bc1d6 100644
--- a/drivers/staging/media/atomisp/pci/runtime/binary/interface/ia_css_binary.h
+++ b/drivers/staging/media/atomisp/pci/runtime/binary/interface/ia_css_binary.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/**
+/*
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
 
diff --git a/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq.h b/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq.h
index 3e7dadca6d707..618172f17b636 100644
--- a/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq.h
+++ b/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq.h
@@ -13,7 +13,7 @@
 #include "ia_css_err.h"
 #define BUFQ_EVENT_SIZE 4
 
-/**
+/*
  * @brief Query the internal frame ID.
  *
  * @param[in]	key	The query key.
@@ -29,7 +29,7 @@ bool ia_css_query_internal_queue_id(
     enum sh_css_queue_id *val
 );
 
-/**
+/*
  * @brief  Map buffer type to a internal queue id.
  *
  * @param[in] thread id		Thread in which the buffer type has to be mapped or unmapped
@@ -43,13 +43,13 @@ void ia_css_queue_map(
     bool map
 );
 
-/**
+/*
  * @brief  Initialize buffer type to a queue id mapping
  * @return none
  */
 void ia_css_queue_map_init(void);
 
-/**
+/*
  * @brief initializes bufq module
  * It create instances of
  * -host to SP buffer queue  which is a list with predefined size,
@@ -62,7 +62,7 @@ void ia_css_queue_map_init(void);
  */
 void ia_css_bufq_init(void);
 
-/**
+/*
 * @brief Enqueues an item into host to SP buffer queue
  *
  * @param thread_index[in]	Thread in which the item to be enqueued
@@ -77,7 +77,7 @@ int ia_css_bufq_enqueue_buffer(
     int queue_id,
     uint32_t item);
 
-/**
+/*
 * @brief Dequeues an item from SP to host buffer queue.
  *
  * @param queue_id[in]		Specifies  the index of the queue in the list where
@@ -90,7 +90,7 @@ int ia_css_bufq_dequeue_buffer(
     int queue_id,
     uint32_t *item);
 
-/**
+/*
 * @brief  Enqueue an event item into host to SP communication event queue.
  *
  * @param[in]	evt_id		      The event ID.
@@ -107,7 +107,7 @@ int ia_css_bufq_enqueue_psys_event(
     uint8_t evt_payload_2
 );
 
-/**
+/*
  * @brief   Dequeue an item from  SP to host communication event queue.
  *
  * @param item	Object to be dequeued into this item.
@@ -119,7 +119,7 @@ int ia_css_bufq_dequeue_psys_event(
 
 );
 
-/**
+/*
  * @brief  Enqueue an event item into host to SP EOF event queue.
  *
  * @param[in]	evt_id		      The event ID.
@@ -129,7 +129,7 @@ int ia_css_bufq_dequeue_psys_event(
 int ia_css_bufq_enqueue_isys_event(
     uint8_t evt_id);
 
-/**
+/*
 * @brief   Dequeue an item from  SP to host communication EOF event queue.
 
  *
@@ -140,7 +140,7 @@ int ia_css_bufq_enqueue_isys_event(
 int ia_css_bufq_dequeue_isys_event(
     u8 item[BUFQ_EVENT_SIZE]);
 
-/**
+/*
 * @brief   Enqueue a tagger command item into tagger command queue..
  *
  * @param item	Object to be enqueue.
@@ -150,7 +150,7 @@ int ia_css_bufq_dequeue_isys_event(
 int ia_css_bufq_enqueue_tag_cmd(
     uint32_t item);
 
-/**
+/*
 * @brief  Uninitializes bufq module.
  *
  * @return	0 or error code upon error.
@@ -158,7 +158,7 @@ int ia_css_bufq_enqueue_tag_cmd(
 */
 int ia_css_bufq_deinit(void);
 
-/**
+/*
 * @brief  Dump queue states
  *
  * @return	None
diff --git a/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug.h b/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug.h
index 2d0e906530af4..b7ff809eeeab3 100644
--- a/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug.h
+++ b/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug.h
@@ -312,7 +312,7 @@ void ia_css_debug_dump_stream_config(
     const struct ia_css_stream_config *config,
     int num_pipes);
 
-/**
+/*
  * @brief Initialize the debug mode.
  *
  * WARNING:
@@ -324,7 +324,7 @@ void ia_css_debug_dump_stream_config(
  */
 bool ia_css_debug_mode_init(void);
 
-/**
+/*
  * @brief Disable the DMA channel.
  *
  * @param[in]	dma_ID		The ID of the target DMA.
@@ -344,7 +344,7 @@ bool ia_css_debug_mode_disable_dma_channel(
     int dma_ID,
     int channel_id,
     int request_type);
-/**
+/*
  * @brief Enable the DMA channel.
  *
  * @param[in]	dma_ID		The ID of the target DMA.
@@ -363,7 +363,7 @@ bool ia_css_debug_mode_enable_dma_channel(
     int channel_id,
     int request_type);
 
-/**
+/*
  * @brief Dump tracer data.
  * [Currently support is only for SKC]
  *
@@ -373,7 +373,7 @@ bool ia_css_debug_mode_enable_dma_channel(
 void ia_css_debug_dump_trace(void);
 
 /* ISP2401 */
-/**
+/*
  * @brief Program counter dumping (in loop)
  *
  * @param[in]	id		The ID of the SP
diff --git a/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_pipe.h b/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_pipe.h
index 80c58cb934d5e..6f6cd77ce91d5 100644
--- a/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_pipe.h
+++ b/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_pipe.h
@@ -13,21 +13,21 @@
 #include <ia_css_stream_public.h>
 #include "ia_css_pipeline.h"
 
-/**
+/*
  * @brief Internal debug support for constructing a pipe graph.
  *
  * @return	None
  */
 void ia_css_debug_pipe_graph_dump_prologue(void);
 
-/**
+/*
  * @brief Internal debug support for constructing a pipe graph.
  *
  * @return	None
  */
 void ia_css_debug_pipe_graph_dump_epilogue(void);
 
-/**
+/*
  * @brief Internal debug support for constructing a pipe graph.
  * @param[in]	stage		Pipeline stage.
  * @param[in]	id		Pipe id.
@@ -38,7 +38,7 @@ void ia_css_debug_pipe_graph_dump_stage(
     struct ia_css_pipeline_stage *stage,
     enum ia_css_pipe_id id);
 
-/**
+/*
  * @brief Internal debug support for constructing a pipe graph.
  * @param[in]	out_frame	Output frame of SP raw copy.
  *
@@ -47,7 +47,7 @@ void ia_css_debug_pipe_graph_dump_stage(
 void ia_css_debug_pipe_graph_dump_sp_raw_copy(
     struct ia_css_frame *out_frame);
 
-/**
+/*
  * @brief Internal debug support for constructing a pipe graph.
  * @param[in]	stream_config	info about sensor and input formatter.
  *
diff --git a/drivers/staging/media/atomisp/pci/runtime/eventq/interface/ia_css_eventq.h b/drivers/staging/media/atomisp/pci/runtime/eventq/interface/ia_css_eventq.h
index 9b058e296c93d..f8ce32e9b90d5 100644
--- a/drivers/staging/media/atomisp/pci/runtime/eventq/interface/ia_css_eventq.h
+++ b/drivers/staging/media/atomisp/pci/runtime/eventq/interface/ia_css_eventq.h
@@ -9,7 +9,7 @@
 
 #include "ia_css_queue.h"	/* queue APIs */
 
-/**
+/*
  * @brief HOST receives event from SP.
  *
  * @param[in]	eventq_handle	eventq_handle.
@@ -22,7 +22,7 @@ int ia_css_eventq_recv(
     ia_css_queue_t *eventq_handle,
     uint8_t *payload);
 
-/**
+/*
  * @brief The Host sends the event to SP.
  * The caller of this API will be blocked until the event
  * is sent.
diff --git a/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param_types.h b/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param_types.h
index d6d60508c1bf0..095d1853670ff 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param_types.h
+++ b/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param_types.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/**
+/*
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
 
diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys.h b/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys.h
index 29eebe8f90782..8870ff294c1ef 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys.h
+++ b/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys.h
@@ -15,7 +15,7 @@
 #include <system_global.h>
 #include "ia_css_isys_comm.h"
 
-/**
+/*
  * Virtual Input System. (Input System 2401)
  */
 typedef isp2401_input_system_cfg_t	ia_css_isys_descr_t;
@@ -28,7 +28,7 @@ enum mipi_port_id ia_css_isys_port_to_mipi_port(
     enum mipi_port_id api_port);
 
 
-/**
+/*
  * @brief Register one (virtual) stream. This is used to track when all
  * virtual streams are configured inside the input system. The CSI RX is
  * only started when all registered streams are configured.
@@ -43,7 +43,7 @@ int ia_css_isys_csi_rx_register_stream(
     enum mipi_port_id port,
     uint32_t isys_stream_id);
 
-/**
+/*
  * @brief Unregister one (virtual) stream. This is used to track when all
  * virtual streams are configured inside the input system. The CSI RX is
  * only started when all registered streams are configured.
@@ -97,7 +97,7 @@ int ia_css_isys_convert_stream_format_to_mipi_format(
     mipi_predictor_t compression,
     unsigned int *fmt_type);
 
-/**
+/*
  * Virtual Input System. (Input System 2401)
  */
 ia_css_isys_error_t ia_css_isys_stream_create(
diff --git a/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline.h b/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline.h
index 8b7cbf31a1a2a..04d9f1fdb5b06 100644
--- a/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline.h
+++ b/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline.h
@@ -219,7 +219,7 @@ int ia_css_pipeline_get_output_stage(
  */
 bool ia_css_pipeline_uses_params(struct ia_css_pipeline *pipeline);
 
-/**
+/*
  * @brief get the SP thread ID.
  *
  * @param[in]	key	The query key, typical use is pipe_num.
@@ -231,7 +231,7 @@ bool ia_css_pipeline_uses_params(struct ia_css_pipeline *pipeline);
  */
 bool ia_css_pipeline_get_sp_thread_id(unsigned int key, unsigned int *val);
 
-/**
+/*
  * @brief Get the pipeline io status
  *
  * @param[in] None
@@ -240,7 +240,7 @@ bool ia_css_pipeline_get_sp_thread_id(unsigned int key, unsigned int *val);
  */
 struct sh_css_sp_pipeline_io_status *ia_css_pipeline_get_pipe_io_status(void);
 
-/**
+/*
  * @brief Map an SP thread to this pipeline
  *
  * @param[in]	pipe_num
@@ -249,7 +249,7 @@ struct sh_css_sp_pipeline_io_status *ia_css_pipeline_get_pipe_io_status(void);
  */
 void ia_css_pipeline_map(unsigned int pipe_num, bool map);
 
-/**
+/*
  * @brief Checks whether the pipeline is mapped to SP threads
  *
  * @param[in]	Query key, typical use is pipe_num
@@ -260,7 +260,7 @@ void ia_css_pipeline_map(unsigned int pipe_num, bool map);
  */
 bool ia_css_pipeline_is_mapped(unsigned int key);
 
-/**
+/*
  * @brief Print pipeline thread mapping
  *
  * @param[in]	none
diff --git a/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr.h b/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr.h
index 00b903d950df7..4c86c19811bb7 100644
--- a/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr.h
+++ b/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr.h
@@ -17,12 +17,12 @@
 #define STORAGE_CLASS_RMGR_C static inline
 #endif				/* __INLINE_RMGR__ */
 
-/**
+/*
  * @brief Initialize resource manager (host/common)
  */
 int ia_css_rmgr_init(void);
 
-/**
+/*
  * @brief Uninitialize resource manager (host/common)
  */
 void ia_css_rmgr_uninit(void);
diff --git a/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr_vbuf.h b/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr_vbuf.h
index 6820bfc774322..68285334bc88b 100644
--- a/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr_vbuf.h
+++ b/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr_vbuf.h
@@ -12,7 +12,7 @@
 #include <ia_css_types.h>
 #include <system_local.h>
 
-/**
+/*
  * @brief Data structure for the resource handle (host, vbuf)
  */
 struct ia_css_rmgr_vbuf_handle {
@@ -21,7 +21,7 @@ struct ia_css_rmgr_vbuf_handle {
 	u32 size;
 };
 
-/**
+/*
  * @brief Data structure for the resource pool (host, vbuf)
  */
 struct ia_css_rmgr_vbuf_pool {
@@ -32,14 +32,14 @@ struct ia_css_rmgr_vbuf_pool {
 	struct ia_css_rmgr_vbuf_handle **handles;
 };
 
-/**
+/*
  * @brief VBUF resource pools
  */
 extern struct ia_css_rmgr_vbuf_pool *vbuf_ref;
 extern struct ia_css_rmgr_vbuf_pool *vbuf_write;
 extern struct ia_css_rmgr_vbuf_pool *hmm_buffer_pool;
 
-/**
+/*
  * @brief Initialize the resource pool (host, vbuf)
  *
  * @param pool	The pointer to the pool
@@ -47,7 +47,7 @@ extern struct ia_css_rmgr_vbuf_pool *hmm_buffer_pool;
 STORAGE_CLASS_RMGR_H int ia_css_rmgr_init_vbuf(
     struct ia_css_rmgr_vbuf_pool *pool);
 
-/**
+/*
  * @brief Uninitialize the resource pool (host, vbuf)
  *
  * @param pool	The pointer to the pool
@@ -55,7 +55,7 @@ STORAGE_CLASS_RMGR_H int ia_css_rmgr_init_vbuf(
 STORAGE_CLASS_RMGR_H void ia_css_rmgr_uninit_vbuf(
     struct ia_css_rmgr_vbuf_pool *pool);
 
-/**
+/*
  * @brief Acquire a handle from the pool (host, vbuf)
  *
  * @param pool		The pointer to the pool
@@ -65,7 +65,7 @@ STORAGE_CLASS_RMGR_H void ia_css_rmgr_acq_vbuf(
     struct ia_css_rmgr_vbuf_pool *pool,
     struct ia_css_rmgr_vbuf_handle **handle);
 
-/**
+/*
  * @brief Release a handle to the pool (host, vbuf)
  *
  * @param pool		The pointer to the pool
@@ -75,14 +75,14 @@ STORAGE_CLASS_RMGR_H void ia_css_rmgr_rel_vbuf(
     struct ia_css_rmgr_vbuf_pool *pool,
     struct ia_css_rmgr_vbuf_handle **handle);
 
-/**
+/*
  * @brief Retain the reference count for a handle (host, vbuf)
  *
  * @param handle	The pointer to the handle
  */
 void ia_css_rmgr_refcount_retain_vbuf(struct ia_css_rmgr_vbuf_handle **handle);
 
-/**
+/*
  * @brief Release the reference count for a handle (host, vbuf)
  *
  * @param handle	The pointer to the handle
diff --git a/drivers/staging/media/atomisp/pci/runtime/tagger/interface/ia_css_tagger_common.h b/drivers/staging/media/atomisp/pci/runtime/tagger/interface/ia_css_tagger_common.h
index 79006c325de60..9208c650fde7e 100644
--- a/drivers/staging/media/atomisp/pci/runtime/tagger/interface/ia_css_tagger_common.h
+++ b/drivers/staging/media/atomisp/pci/runtime/tagger/interface/ia_css_tagger_common.h
@@ -10,14 +10,14 @@
 #include <system_local.h>
 #include <type_support.h>
 
-/**
+/*
  * @brief The tagger's circular buffer.
  *
  * Should be one less than NUM_CONTINUOUS_FRAMES in sh_css_internal.h
  */
 #define MAX_CB_ELEMS_FOR_TAGGER 14
 
-/**
+/*
  * @brief Data structure for the tagger buffer element.
  */
 typedef struct {
diff --git a/drivers/staging/media/atomisp/pci/sh_css_internal.h b/drivers/staging/media/atomisp/pci/sh_css_internal.h
index 9155a83fcc03d..ec68c16d42722 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_internal.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_internal.h
@@ -80,7 +80,7 @@
 
 #define SH_CSS_MAX_SP_THREADS		5
 
-/**
+/*
  * The C99 standard does not specify the exact object representation of structs;
  * the representation is compiler dependent.
  *
@@ -638,7 +638,7 @@ struct sh_css_sp_output {
 	unsigned int		sw_interrupt_value[SH_CSS_NUM_SDW_IRQS];
 };
 
-/**
+/*
  * @brief Data structure for the circular buffer.
  * The circular buffer is empty if "start == end". The
  * circular buffer is full if "(end + 1) % size == start".
diff --git a/drivers/staging/media/atomisp/pci/sh_css_sp.h b/drivers/staging/media/atomisp/pci/sh_css_sp.h
index 78aec5b7e8fa7..6985ca200f099 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_sp.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_sp.h
@@ -68,7 +68,7 @@ sh_css_read_host2sp_command(void);
 void
 sh_css_init_host2sp_frame_data(void);
 
-/**
+/*
  * @brief Update the offline frame information in host_sp_communication.
  *
  * @param[in] frame_num The offline frame number.
@@ -80,7 +80,7 @@ sh_css_update_host2sp_offline_frame(
     struct ia_css_frame *frame,
     struct ia_css_metadata *metadata);
 
-/**
+/*
  * @brief Update the mipi frame information in host_sp_communication.
  *
  * @param[in] frame_num The mipi frame number.
@@ -91,7 +91,7 @@ sh_css_update_host2sp_mipi_frame(
     unsigned int frame_num,
     struct ia_css_frame *frame);
 
-/**
+/*
  * @brief Update the mipi metadata information in host_sp_communication.
  *
  * @param[in] frame_num The mipi frame number.
@@ -102,7 +102,7 @@ sh_css_update_host2sp_mipi_metadata(
     unsigned int frame_num,
     struct ia_css_metadata *metadata);
 
-/**
+/*
  * @brief Update the nr of mipi frames to use in host_sp_communication.
  *
  * @param[in] num_frames The number of mipi frames to use.
@@ -110,7 +110,7 @@ sh_css_update_host2sp_mipi_metadata(
 void
 sh_css_update_host2sp_num_mipi_frames(unsigned int num_frames);
 
-/**
+/*
  * @brief Update the nr of offline frames to use in host_sp_communication.
  *
  * @param[in] num_frames The number of raw frames to use.
@@ -170,7 +170,7 @@ sh_css_sp_set_disable_continuous_viewfinder(bool flag);
 void
 sh_css_sp_reset_global_vars(void);
 
-/**
+/*
  * @brief Initialize the DMA software-mask in the debug mode.
  * This API should be ONLY called in the debugging mode.
  * And it should be always called before the first call of
@@ -185,7 +185,7 @@ sh_css_sp_reset_global_vars(void);
 bool
 sh_css_sp_init_dma_sw_reg(int dma_id);
 
-/**
+/*
  * @brief Set the DMA software-mask in the debug mode.
  * This API should be ONLYL called in the debugging mode. Must
  * call "sh_css_set_dma_sw_reg(...)" before this
diff --git a/drivers/staging/media/atomisp/pci/system_global.h b/drivers/staging/media/atomisp/pci/system_global.h
index e8a29f73d67a7..ee1d9b66ccb11 100644
--- a/drivers/staging/media/atomisp/pci/system_global.h
+++ b/drivers/staging/media/atomisp/pci/system_global.h
@@ -293,7 +293,7 @@ typedef enum {
 } stream2mmio_sid_ID_t;
 /* end of Stream2MMIO */
 
-/**
+/*
  * Input System 2401: CSI-MIPI recevier.
  */
 typedef enum {
@@ -324,7 +324,7 @@ typedef enum {
 	N_ISYS2401_DMA_ID
 } isys2401_dma_ID_t;
 
-/**
+/*
  * Pixel-generator. ("system_global.h")
  */
 typedef enum {
diff --git a/drivers/staging/media/av7110/av7110.c b/drivers/staging/media/av7110/av7110.c
index 862aee993889c..06c43b5f8a87a 100644
--- a/drivers/staging/media/av7110/av7110.c
+++ b/drivers/staging/media/av7110/av7110.c
@@ -942,16 +942,16 @@ static int av7110_start_feed(struct dvb_demux_feed *feed)
 		    (feed->pes_type <= DMX_PES_PCR)) {
 			switch (demux->dmx.frontend->source) {
 			case DMX_MEMORY_FE:
-				if (feed->ts_type & TS_DECODER)
-					if (feed->pes_type < 2 &&
-					    !(demux->pids[0] & 0x8000) &&
-					    !(demux->pids[1] & 0x8000)) {
-						dvb_ringbuffer_flush_spinlock_wakeup(&av7110->avout);
-						dvb_ringbuffer_flush_spinlock_wakeup(&av7110->aout);
-						ret = av7110_av_start_play(av7110, RP_AV);
-						if (!ret)
-							demux->playing = 1;
-					}
+				if (feed->pes_type >= 2 ||
+				    (demux->pids[0] & 0x8000) ||
+				    (demux->pids[1] & 0x8000))
+					break;
+
+				dvb_ringbuffer_flush_spinlock_wakeup(&av7110->avout);
+				dvb_ringbuffer_flush_spinlock_wakeup(&av7110->aout);
+				ret = av7110_av_start_play(av7110, RP_AV);
+				if (!ret)
+					demux->playing = 1;
 				break;
 			default:
 				ret = dvb_feed_start_pid(feed);
diff --git a/drivers/staging/media/ipu7/ipu7.c b/drivers/staging/media/ipu7/ipu7.c
index 310e3f24e571f..48a35bda42370 100644
--- a/drivers/staging/media/ipu7/ipu7.c
+++ b/drivers/staging/media/ipu7/ipu7.c
@@ -2343,7 +2343,7 @@ static int ipu7_init_fw_code_region_by_sys(struct ipu7_bus_device *sys,
 		return ret;
 	}
 
-	ret = pm_runtime_get_sync(dev);
+	ret = pm_runtime_resume_and_get(dev);
 	if (ret < 0) {
 		dev_err(dev, "Failed to get runtime PM\n");
 		return ret;
@@ -2702,7 +2702,7 @@ static int ipu7_resume(struct device *dev)
 	if (ret)
 		dev_err(dev, "IPC reset protocol failed!\n");
 
-	ret = pm_runtime_get_sync(&isp->psys->auxdev.dev);
+	ret = pm_runtime_resume_and_get(&isp->psys->auxdev.dev);
 	if (ret < 0) {
 		dev_err(dev, "Failed to get runtime PM\n");
 		return 0;
diff --git a/drivers/staging/media/meson/vdec/vdec.c b/drivers/staging/media/meson/vdec/vdec.c
index a039d925c0fe5..c18bf352b1996 100644
--- a/drivers/staging/media/meson/vdec/vdec.c
+++ b/drivers/staging/media/meson/vdec/vdec.c
@@ -504,8 +504,8 @@ vdec_try_fmt_common(struct amvdec_session *sess, u32 size,
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 		fmt_out = find_format(fmts, size, pixmp->pixelformat);
 		if (!fmt_out) {
-			pixmp->pixelformat = V4L2_PIX_FMT_MPEG2;
-			fmt_out = find_format(fmts, size, pixmp->pixelformat);
+			pixmp->pixelformat = fmts[0].pixfmt;
+			fmt_out = &fmts[0];
 		}
 		break;
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.c b/drivers/staging/media/sunxi/cedrus/cedrus.c
index bbd186b8035b2..96acd52e380c8 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus.c
@@ -285,8 +285,10 @@ static int cedrus_init_ctrls(struct cedrus_dev *dev, struct cedrus_ctx *ctx)
 	ctrl_size = sizeof(ctrl) * CEDRUS_CONTROLS_COUNT + 1;
 
 	ctx->ctrls = kzalloc(ctrl_size, GFP_KERNEL);
-	if (!ctx->ctrls)
+	if (!ctx->ctrls) {
+		v4l2_ctrl_handler_free(hdl);
 		return -ENOMEM;
+	}
 
 	j = 0;
 	for (i = 0; i < CEDRUS_CONTROLS_COUNT; i++) {
diff --git a/drivers/staging/media/tegra-video/vi.c b/drivers/staging/media/tegra-video/vi.c
index 456134a9e8cf2..01622013c1092 100644
--- a/drivers/staging/media/tegra-video/vi.c
+++ b/drivers/staging/media/tegra-video/vi.c
@@ -1257,6 +1257,7 @@ static int tegra_vi_channels_alloc(struct tegra_vi *vi)
 	struct device_node *parent;
 	struct v4l2_fwnode_endpoint v4l2_ep = { .bus_type = 0 };
 	unsigned int lanes;
+	int err;
 	int ret = 0;
 
 	ports = of_get_child_by_name(node, "ports");
@@ -1267,8 +1268,8 @@ static int tegra_vi_channels_alloc(struct tegra_vi *vi)
 		if (!of_node_name_eq(port, "port"))
 			continue;
 
-		ret = of_property_read_u32(port, "reg", &port_num);
-		if (ret < 0)
+		err = of_property_read_u32(port, "reg", &port_num);
+		if (err < 0)
 			continue;
 
 		if (port_num > vi->soc->vi_max_channels) {
@@ -1289,10 +1290,10 @@ static int tegra_vi_channels_alloc(struct tegra_vi *vi)
 
 		ep = of_graph_get_endpoint_by_regs(parent, 0, 0);
 		of_node_put(parent);
-		ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(ep),
+		err = v4l2_fwnode_endpoint_parse(of_fwnode_handle(ep),
 						 &v4l2_ep);
 		of_node_put(ep);
-		if (ret)
+		if (err)
 			continue;
 
 		lanes = v4l2_ep.bus.mipi_csi2.num_data_lanes;
@@ -1468,7 +1469,6 @@ static int tegra_vi_graph_build(struct tegra_vi_channel *chan,
 	struct tegra_vi *vi = chan->vi;
 	struct tegra_vi_graph_entity *ent;
 	struct fwnode_handle *ep = NULL;
-	struct v4l2_fwnode_link link;
 	struct media_entity *local = entity->entity;
 	struct media_entity *remote;
 	struct media_pad *local_pad;
@@ -1478,70 +1478,64 @@ static int tegra_vi_graph_build(struct tegra_vi_channel *chan,
 
 	dev_dbg(vi->dev, "creating links for entity %s\n", local->name);
 
-	while (1) {
-		ep = fwnode_graph_get_next_endpoint(entity->asd.match.fwnode,
-						    ep);
-		if (!ep)
-			break;
+	fwnode_graph_for_each_endpoint(entity->asd.match.fwnode, ep) {
+		struct fwnode_handle *remote_parent __free(fwnode_handle) = NULL;
+		struct fwnode_handle *sink_ep __free(fwnode_handle) = NULL;
+		int src_idx, sink_idx;
 
-		ret = v4l2_fwnode_parse_link(ep, &link);
-		if (ret < 0) {
-			dev_err(vi->dev, "failed to parse link for %pOF: %d\n",
-				to_of_node(ep), ret);
+		src_idx = media_entity_get_fwnode_pad(local, ep,
+						      MEDIA_PAD_FL_SOURCE);
+		if (src_idx < 0) {
+			dev_dbg(vi->dev, "no source pad found for %pfw\n", ep);
 			continue;
 		}
 
-		if (link.local_port >= local->num_pads) {
-			dev_err(vi->dev, "invalid port number %u on %pOF\n",
-				link.local_port, to_of_node(link.local_node));
-			v4l2_fwnode_put_link(&link);
-			ret = -EINVAL;
-			break;
+		remote_parent = fwnode_graph_get_remote_port_parent(ep);
+		if (!remote_parent) {
+			dev_dbg(vi->dev, "no remote parent found for %pfw\n",
+				ep);
+			continue;
 		}
 
-		local_pad = &local->pads[link.local_port];
+		local_pad = &local->pads[src_idx];
 		/* Remote node is vi node. So use channel video entity and pad
 		 * as remote/sink.
 		 */
-		if (link.remote_node == of_fwnode_handle(vi->dev->of_node)) {
+		if (remote_parent == of_fwnode_handle(vi->dev->of_node)) {
 			remote = &chan->video.entity;
 			remote_pad = &chan->pad;
 			goto create_link;
 		}
 
-		/*
-		 * Skip sink ports, they will be processed from the other end
-		 * of the link.
-		 */
-		if (local_pad->flags & MEDIA_PAD_FL_SINK) {
-			dev_dbg(vi->dev, "skipping sink port %pOF:%u\n",
-				to_of_node(link.local_node), link.local_port);
-			v4l2_fwnode_put_link(&link);
-			continue;
-		}
-
 		/* find the remote entity from notifier list */
 		ent = tegra_vi_graph_find_entity(&chan->notifier.done_list,
-						 link.remote_node);
+						 remote_parent);
 		if (!ent) {
-			dev_err(vi->dev, "no entity found for %pOF\n",
-				to_of_node(link.remote_node));
-			v4l2_fwnode_put_link(&link);
+			fwnode_handle_put(ep);
+			dev_err(vi->dev, "no entity found for %pfw\n",
+				remote_parent);
 			ret = -ENODEV;
 			break;
 		}
 
 		remote = ent->entity;
-		if (link.remote_port >= remote->num_pads) {
-			dev_err(vi->dev, "invalid port number %u on %pOF\n",
-				link.remote_port,
-				to_of_node(link.remote_node));
-			v4l2_fwnode_put_link(&link);
-			ret = -EINVAL;
-			break;
+
+		sink_ep = fwnode_graph_get_remote_endpoint(ep);
+		if (!sink_ep) {
+			dev_dbg(vi->dev, "no sink ep found for %pfw\n",
+				ep);
+			continue;
+		}
+
+		sink_idx = media_entity_get_fwnode_pad(remote, sink_ep,
+						       MEDIA_PAD_FL_SINK);
+		if (sink_idx < 0) {
+			dev_dbg(vi->dev, "no sink pad found for %pfw\n",
+				sink_ep);
+			continue;
 		}
 
-		remote_pad = &remote->pads[link.remote_port];
+		remote_pad = &remote->pads[sink_idx];
 
 create_link:
 		dev_dbg(vi->dev, "creating %s:%u -> %s:%u link\n",
@@ -1551,8 +1545,8 @@ static int tegra_vi_graph_build(struct tegra_vi_channel *chan,
 		ret = media_create_pad_link(local, local_pad->index,
 					    remote, remote_pad->index,
 					    link_flags);
-		v4l2_fwnode_put_link(&link);
 		if (ret < 0) {
+			fwnode_handle_put(ep);
 			dev_err(vi->dev,
 				"failed to create %s:%u -> %s:%u link: %d\n",
 				local->name, local_pad->index,
@@ -1561,7 +1555,6 @@ static int tegra_vi_graph_build(struct tegra_vi_channel *chan,
 		}
 	}
 
-	fwnode_handle_put(ep);
 	return ret;
 }
 
diff --git a/drivers/staging/media/tegra-video/vip.c b/drivers/staging/media/tegra-video/vip.c
index 9ff1f1750a154..5fba11e31e1d0 100644
--- a/drivers/staging/media/tegra-video/vip.c
+++ b/drivers/staging/media/tegra-video/vip.c
@@ -126,7 +126,7 @@ static int tegra_vip_channel_of_parse(struct tegra_vip *vip)
 	if (!ep) {
 		err = -EINVAL;
 		dev_err_probe(dev, err, "%pOF: error getting endpoint node\n", np);
-		goto err_node_put;
+		return err;
 	}
 
 	fwh = of_fwnode_handle(ep);
@@ -134,14 +134,14 @@ static int tegra_vip_channel_of_parse(struct tegra_vip *vip)
 	of_node_put(ep);
 	if (err) {
 		dev_err_probe(dev, err, "%pOF: failed to parse v4l2 endpoint\n", np);
-		goto err_node_put;
+		return err;
 	}
 
 	num_pads = of_graph_get_endpoint_count(np);
 	if (num_pads != TEGRA_VIP_PADS_NUM) {
 		err = -EINVAL;
 		dev_err_probe(dev, err, "%pOF: need 2 pads, got %d\n", np, num_pads);
-		goto err_node_put;
+		return err;
 	}
 
 	vip->chan.of_node = of_node_get(np);
@@ -149,10 +149,6 @@ static int tegra_vip_channel_of_parse(struct tegra_vip *vip)
 	vip->chan.pads[TEGRA_VIP_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE;
 
 	return 0;
-
-err_node_put:
-	of_node_put(np);
-	return err;
 }
 
 static int tegra_vip_channel_init(struct tegra_vip *vip)
diff --git a/drivers/staging/most/dim2/dim2.c b/drivers/staging/most/dim2/dim2.c
index ec0e568ec12d1..0e2512d166329 100644
--- a/drivers/staging/most/dim2/dim2.c
+++ b/drivers/staging/most/dim2/dim2.c
@@ -974,10 +974,9 @@ static int rcar_gen2_enable(struct platform_device *pdev)
 	int ret;
 
 	dev->clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(dev->clk)) {
-		dev_err(&pdev->dev, "cannot get clock\n");
-		return PTR_ERR(dev->clk);
-	}
+	if (IS_ERR(dev->clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(dev->clk),
+				     "cannot get clock\n");
 
 	ret = clk_prepare_enable(dev->clk);
 	if (ret) {
@@ -1019,10 +1018,9 @@ static int rcar_gen3_enable(struct platform_device *pdev)
 	int ret;
 
 	dev->clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(dev->clk)) {
-		dev_err(&pdev->dev, "cannot get clock\n");
-		return PTR_ERR(dev->clk);
-	}
+	if (IS_ERR(dev->clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(dev->clk),
+				     "cannot get clock\n");
 
 	ret = clk_prepare_enable(dev->clk);
 	if (ret) {
diff --git a/drivers/staging/nvec/nvec.c b/drivers/staging/nvec/nvec.c
index 88c416ee0381d..41cbdfdc1da69 100644
--- a/drivers/staging/nvec/nvec.c
+++ b/drivers/staging/nvec/nvec.c
@@ -2,7 +2,7 @@
 /*
  * NVEC: NVIDIA compliant embedded controller interface
  *
- * Copyright (C) 2011 The AC100 Kernel Team <ac100@lists.lauchpad.net>
+ * Copyright (C) 2011 The AC100 Kernel Team <ac100@lists.launchpad.net>
  *
  * Authors:  Pierre-Hugues Husson <phhusson@free.fr>
  *           Ilya Petrov <ilya.muromec@gmail.com>
@@ -57,7 +57,7 @@
  * @NVEC_MSG_RX: The message is an incoming message (from EC)
  * @NVEC_MSG_TX: The message is an outgoing message (to EC)
  */
-enum nvec_msg_category  {
+enum nvec_msg_category {
 	NVEC_MSG_RX,
 	NVEC_MSG_TX,
 };
diff --git a/drivers/staging/octeon/ethernet-rx.c b/drivers/staging/octeon/ethernet-rx.c
index cd36b5ba6f6c2..5be43bbf6a1d8 100644
--- a/drivers/staging/octeon/ethernet-rx.c
+++ b/drivers/staging/octeon/ethernet-rx.c
@@ -5,6 +5,8 @@
  * Copyright (c) 2003-2010 Cavium Networks
  */
 
+#include <asm-generic/errno-base.h>
+#include <linux/dev_printk.h>
 #include <linux/platform_device.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
@@ -445,7 +447,7 @@ void cvm_oct_poll_controller(struct net_device *dev)
 }
 #endif
 
-void cvm_oct_rx_initialize(struct platform_device *pdev)
+int cvm_oct_rx_initialize(struct platform_device *pdev)
 {
 	int i;
 	struct net_device *dev_for_napi = NULL;
@@ -459,8 +461,10 @@ void cvm_oct_rx_initialize(struct platform_device *pdev)
 		}
 	}
 
-	if (!dev_for_napi)
-		panic("No net_devices were allocated.");
+	if (!dev_for_napi) {
+		dev_err(&pdev->dev, "No net_devices were allocated.");
+		return -ENODEV;
+	}
 
 	for (i = 0; i < ARRAY_SIZE(plat->rx_group); i++) {
 		int ret;
@@ -479,9 +483,10 @@ void cvm_oct_rx_initialize(struct platform_device *pdev)
 		/* Register an IRQ handler to receive POW interrupts */
 		ret = request_irq(rx_group[i].irq, cvm_oct_do_interrupt, 0,
 				  "Ethernet", &rx_group[i].napi);
-		if (ret)
-			panic("Could not acquire Ethernet IRQ %d\n",
-			      rx_group[i].irq);
+		if (ret) {
+			dev_err(&pdev->dev, "Could not acquire Ethernet IRQ %d\n", rx_group[i].irq);
+			return ret;
+		}
 
 		disable_irq_nosync(rx_group[i].irq);
 
@@ -518,6 +523,7 @@ void cvm_oct_rx_initialize(struct platform_device *pdev)
 		napi_schedule(&rx_group[i].napi);
 	}
 	atomic_inc(&oct_rx_ready);
+	return 0;
 }
 
 void cvm_oct_rx_shutdown(struct platform_device *pdev)
@@ -535,6 +541,8 @@ void cvm_oct_rx_shutdown(struct platform_device *pdev)
 		else
 			cvmx_write_csr(CVMX_POW_WQ_INT_THRX(i), 0);
 
+		napi_disable(&plat->rx_group[i].napi);
+
 		/* Free the interrupt handler */
 		free_irq(plat->rx_group[i].irq, &plat->rx_group[i].napi);
 
diff --git a/drivers/staging/octeon/ethernet-rx.h b/drivers/staging/octeon/ethernet-rx.h
index 6093694326cb6..ff3ed3d784a4b 100644
--- a/drivers/staging/octeon/ethernet-rx.h
+++ b/drivers/staging/octeon/ethernet-rx.h
@@ -8,7 +8,7 @@
 struct platform_device;
 
 void cvm_oct_poll_controller(struct net_device *dev);
-void cvm_oct_rx_initialize(struct platform_device *pdev);
+int cvm_oct_rx_initialize(struct platform_device *pdev);
 void cvm_oct_rx_shutdown(struct platform_device *pdev);
 
 static inline void cvm_oct_rx_refill_pool(struct platform_device *pdev,
diff --git a/drivers/staging/octeon/ethernet-tx.c b/drivers/staging/octeon/ethernet-tx.c
index 14d10659bce71..cc20c1e6791fc 100644
--- a/drivers/staging/octeon/ethernet-tx.c
+++ b/drivers/staging/octeon/ethernet-tx.c
@@ -5,6 +5,7 @@
  * Copyright (c) 2003-2010 Cavium Networks
  */
 
+#include <linux/platform_device.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/netdevice.h>
@@ -649,23 +650,25 @@ static irqreturn_t cvm_oct_tx_cleanup_watchdog(int cpl, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-void cvm_oct_tx_initialize(void)
+int cvm_oct_tx_initialize(struct platform_device *pdev)
 {
-	int i;
+	int ret;
 
 	/* Disable the interrupt.  */
 	cvmx_write_csr(CVMX_CIU_TIMX(1), 0);
 	/* Register an IRQ handler to receive CIU_TIMX(1) interrupts */
-	i = request_irq(OCTEON_IRQ_TIMER1,
-			cvm_oct_tx_cleanup_watchdog, 0,
-			"Ethernet", cvm_oct_device);
-
-	if (i)
-		panic("Could not acquire Ethernet IRQ %d\n", OCTEON_IRQ_TIMER1);
+	ret = request_irq(OCTEON_IRQ_TIMER1, cvm_oct_tx_cleanup_watchdog, 0,
+			  "Ethernet", cvm_oct_device);
+	if (ret)
+		dev_warn(&pdev->dev, "Could not acquire Ethernet IRQ %d\n",
+			 OCTEON_IRQ_TIMER1);
+	return ret;
 }
 
 void cvm_oct_tx_shutdown(void)
 {
 	/* Free the interrupt handler */
 	free_irq(OCTEON_IRQ_TIMER1, cvm_oct_device);
+
+	tasklet_kill(&cvm_oct_tx_cleanup_tasklet);
 }
diff --git a/drivers/staging/octeon/ethernet-tx.h b/drivers/staging/octeon/ethernet-tx.h
index 6c524668f65a5..b03d56b0d2b65 100644
--- a/drivers/staging/octeon/ethernet-tx.h
+++ b/drivers/staging/octeon/ethernet-tx.h
@@ -9,6 +9,6 @@ netdev_tx_t cvm_oct_xmit(struct sk_buff *skb, struct net_device *dev);
 netdev_tx_t cvm_oct_xmit_pow(struct sk_buff *skb, struct net_device *dev);
 int cvm_oct_transmit_qos(struct net_device *dev, void *work_queue_entry,
 			 int do_free, int qos);
-void cvm_oct_tx_initialize(void);
+int cvm_oct_tx_initialize(struct platform_device *pdev);
 void cvm_oct_tx_shutdown(void);
 void cvm_oct_tx_shutdown_dev(struct net_device *dev);
diff --git a/drivers/staging/octeon/ethernet.c b/drivers/staging/octeon/ethernet.c
index f3fa221f452e2..42ec4b8324a83 100644
--- a/drivers/staging/octeon/ethernet.c
+++ b/drivers/staging/octeon/ethernet.c
@@ -104,6 +104,21 @@ struct net_device *cvm_oct_device[TOTAL_NUMBER_OF_PORTS];
 
 u64 cvm_oct_tx_poll_interval;
 
+static void cvm_oct_remove_device(int port)
+{
+	if (cvm_oct_device[port]) {
+		struct net_device *dev = cvm_oct_device[port];
+		struct octeon_ethernet *priv = netdev_priv(dev);
+
+		cancel_delayed_work_sync(&priv->port_periodic_work);
+
+		cvm_oct_tx_shutdown_dev(dev);
+		unregister_netdev(dev);
+		free_netdev(dev);
+		cvm_oct_device[port] = NULL;
+	}
+}
+
 static void cvm_oct_rx_refill_worker(struct work_struct *work)
 {
 	struct octeon_ethernet_platform *plat = container_of(work,
@@ -673,6 +688,7 @@ static int cvm_oct_probe(struct platform_device *pdev)
 {
 	int num_interfaces;
 	int interface;
+	int ret = 0;
 	int fau = FAU_NUM_PACKET_BUFFERS_TO_FREE;
 	int qos;
 	struct device_node *pip;
@@ -919,8 +935,13 @@ static int cvm_oct_probe(struct platform_device *pdev)
 		}
 	}
 
-	cvm_oct_tx_initialize();
-	cvm_oct_rx_initialize(pdev);
+	ret = cvm_oct_tx_initialize(pdev);
+	if (ret)
+		goto err_tx;
+
+	ret = cvm_oct_rx_initialize(pdev);
+	if (ret)
+		goto err_rx;
 
 	/*
 	 * 150 uS: about 10 1500-byte packets at 1GE.
@@ -930,6 +951,32 @@ static int cvm_oct_probe(struct platform_device *pdev)
 	schedule_delayed_work(&plat->rx_refill_work, HZ);
 
 	return 0;
+
+err_rx:
+	cvm_oct_tx_shutdown();
+err_tx:
+	cvmx_ipd_disable();
+
+	atomic_inc_return(&cvm_oct_poll_queue_stopping);
+
+	/* Free the ethernet devices */
+	for (int port = 0; port < TOTAL_NUMBER_OF_PORTS; port++)
+		cvm_oct_remove_device(port);
+
+	cvmx_pko_shutdown();
+
+	cvmx_ipd_free_ptr();
+
+	/* Free the HW pools */
+	cvm_oct_mem_empty_fpa(pdev, CVMX_FPA_PACKET_POOL, CVMX_FPA_PACKET_POOL_SIZE,
+			      num_packet_buffers);
+	cvm_oct_mem_empty_fpa(pdev, CVMX_FPA_WQE_POOL, CVMX_FPA_WQE_POOL_SIZE,
+			      num_packet_buffers);
+	if (CVMX_FPA_OUTPUT_BUFFER_POOL != CVMX_FPA_PACKET_POOL)
+		cvm_oct_mem_empty_fpa(pdev, CVMX_FPA_OUTPUT_BUFFER_POOL,
+				      CVMX_FPA_OUTPUT_BUFFER_POOL_SIZE, 128);
+
+	return ret;
 }
 
 static void cvm_oct_remove(struct platform_device *pdev)
@@ -948,19 +995,8 @@ static void cvm_oct_remove(struct platform_device *pdev)
 	cvmx_pko_disable();
 
 	/* Free the ethernet devices */
-	for (port = 0; port < TOTAL_NUMBER_OF_PORTS; port++) {
-		if (cvm_oct_device[port]) {
-			struct net_device *dev = cvm_oct_device[port];
-			struct octeon_ethernet *priv = netdev_priv(dev);
-
-			cancel_delayed_work_sync(&priv->port_periodic_work);
-
-			cvm_oct_tx_shutdown_dev(dev);
-			unregister_netdev(dev);
-			free_netdev(dev);
-			cvm_oct_device[port] = NULL;
-		}
-	}
+	for (port = 0; port < TOTAL_NUMBER_OF_PORTS; port++)
+		cvm_oct_remove_device(port);
 
 	cvmx_pko_shutdown();
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_ap.c b/drivers/staging/rtl8723bs/core/rtw_ap.c
index 065850a9e8947..7e272701c68c3 100644
--- a/drivers/staging/rtl8723bs/core/rtw_ap.c
+++ b/drivers/staging/rtl8723bs/core/rtw_ap.c
@@ -232,7 +232,7 @@ void expire_timeout_chk(struct adapter *padapter)
 
 					/* to update bcn with tim_bitmap for this station */
 					pstapriv->tim_bitmap |= BIT(psta->aid);
-					update_beacon(padapter, WLAN_EID_TIM, NULL, true);
+					update_beacon(padapter, WLAN_EID_TIM, true);
 
 					if (!pmlmeext->active_keep_alive_check)
 						continue;
@@ -447,7 +447,7 @@ void update_sta_info_apmode(struct adapter *padapter, struct sta_info *psta)
 	/* ap mode */
 	rtw_hal_set_odm_var(padapter, HAL_ODM_STA_INFO, psta, true);
 
-	if (psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_8021X)
+	if (psecuritypriv->dot11_auth_algrthm == dot11AuthAlgrthm_8021X)
 		psta->ieee8021x_blocked = true;
 	else
 		psta->ieee8021x_blocked = false;
@@ -651,8 +651,8 @@ void start_bss_network(struct adapter *padapter)
 	if (!pmlmepriv->cur_network.join_res) { /* setting only at  first time */
 
 		/* WEP Key will be set before this function, do not clear CAM. */
-		if ((psecuritypriv->dot11PrivacyAlgrthm != _WEP40_) &&
-		    (psecuritypriv->dot11PrivacyAlgrthm != _WEP104_))
+		if ((psecuritypriv->dot11_privacy_algrthm != _WEP40_) &&
+		    (psecuritypriv->dot11_privacy_algrthm != _WEP104_))
 			flush_all_cam_entry(padapter);	/* clear CAM */
 	}
 
@@ -673,7 +673,7 @@ void start_bss_network(struct adapter *padapter)
 	rtw_hal_set_hwreg(padapter, HW_VAR_AC_PARAM_BK, (u8 *)(&acparm));
 
 	/* Set Security */
-	val8 = (psecuritypriv->dot11AuthAlgrthm ==
+	val8 = (psecuritypriv->dot11_auth_algrthm ==
 		dot11AuthAlgrthm_8021X) ? 0xcc : 0xcf;
 	rtw_hal_set_hwreg(padapter, HW_VAR_SEC_CFG, (u8 *)(&val8));
 
@@ -739,7 +739,7 @@ void start_bss_network(struct adapter *padapter)
 		       rtw_get_capability((struct wlan_bssid_ex *)pnetwork));
 
 	if (pmlmeext->bstart_bss) {
-		update_beacon(padapter, WLAN_EID_TIM, NULL, true);
+		update_beacon(padapter, WLAN_EID_TIM, true);
 
 		/* issue beacon frame */
 		send_beacon(padapter);
@@ -782,7 +782,7 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 
 	memcpy(ie, pbuf, pbss_network->ie_length);
 
-	if (pbss_network->infrastructure_mode != Ndis802_11APMode)
+	if (pbss_network->infrastructure_mode != NL80211_IFTYPE_AP)
 		return _FAIL;
 
 	pbss_network->rssi = 0;
@@ -872,7 +872,7 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 				      &group_cipher,
 				      &pairwise_cipher,
 				      NULL) == _SUCCESS) {
-			psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+			psecuritypriv->dot11_auth_algrthm = dot11AuthAlgrthm_8021X;
 
 			psecuritypriv->dot8021xalg = 1;/* psk,  todo:802.1x */
 			psecuritypriv->wpa_psk |= BIT(1);
@@ -898,7 +898,7 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 					     &group_cipher,
 					     &pairwise_cipher,
 					     NULL) == _SUCCESS) {
-				psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+				psecuritypriv->dot11_auth_algrthm = dot11AuthAlgrthm_8021X;
 
 				psecuritypriv->dot8021xalg = 1;/* psk,  todo:802.1x */
 
@@ -1009,21 +1009,6 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 	if (p && ie_len > 0)
 		ht_info_ie = p;
 
-	switch (network_type) {
-	case WIRELESS_11B:
-		pbss_network->network_type_in_use = Ndis802_11DS;
-		break;
-	case WIRELESS_11G:
-	case WIRELESS_11BG:
-	case WIRELESS_11G_24N:
-	case WIRELESS_11BG_24N:
-		pbss_network->network_type_in_use = Ndis802_11OFDM24;
-		break;
-	default:
-		pbss_network->network_type_in_use = Ndis802_11OFDM24;
-		break;
-	}
-
 	pmlmepriv->cur_network.network_type = network_type;
 
 	pmlmepriv->htpriv.ht_option = false;
@@ -1180,7 +1165,7 @@ u8 rtw_ap_set_pairwise_key(struct adapter *padapter, struct sta_info *psta)
 		goto exit;
 	}
 
-	init_h2fwcmd_w_parm_no_rsp(ph2c, psetstakey_para, _SetStaKey_CMD_);
+	init_h2fwcmd_w_parm_no_rsp(ph2c, psetstakey_para, SET_STA_KEY_CMD);
 
 	psetstakey_para->algorithm = (u8)psta->dot118021XPrivacy;
 
@@ -1243,7 +1228,7 @@ static int rtw_ap_set_key(struct adapter *padapter,
 
 	memcpy(&psetkeyparm->key[0], key, keylen);
 
-	pcmd->cmdcode = _SetKey_CMD_;
+	pcmd->cmdcode = SET_KEY_CMD;
 	pcmd->parmbuf = (u8 *)psetkeyparm;
 	pcmd->cmdsz =  (sizeof(struct setkey_parm));
 	pcmd->rsp = NULL;
@@ -1307,20 +1292,21 @@ static void update_bcn_erpinfo_ie(struct adapter *padapter)
 		       &len,
 		       (pnetwork->ie_length - _BEACON_IE_OFFSET_));
 	if (p && len > 0) {
-		struct ndis_80211_var_ie *ie = (struct ndis_80211_var_ie *)p;
+		struct ndis_80211_var_ie *erp_ie = (struct ndis_80211_var_ie *)p;
 
 		if (pmlmepriv->num_sta_non_erp == 1)
-			ie->data[0] |= RTW_ERP_INFO_NON_ERP_PRESENT | RTW_ERP_INFO_USE_PROTECTION;
+			erp_ie->data[0] |= RTW_ERP_INFO_NON_ERP_PRESENT |
+					    RTW_ERP_INFO_USE_PROTECTION;
 		else
-			ie->data[0] &= ~(RTW_ERP_INFO_NON_ERP_PRESENT |
-					  RTW_ERP_INFO_USE_PROTECTION);
+			erp_ie->data[0] &= ~(RTW_ERP_INFO_NON_ERP_PRESENT |
+					      RTW_ERP_INFO_USE_PROTECTION);
 
 		if (pmlmepriv->num_sta_no_short_preamble > 0)
-			ie->data[0] |= RTW_ERP_INFO_BARKER_PREAMBLE_MODE;
+			erp_ie->data[0] |= RTW_ERP_INFO_BARKER_PREAMBLE_MODE;
 		else
-			ie->data[0] &= ~(RTW_ERP_INFO_BARKER_PREAMBLE_MODE);
+			erp_ie->data[0] &= ~(RTW_ERP_INFO_BARKER_PREAMBLE_MODE);
 
-		ERP_IE_handler(padapter, ie);
+		ERP_IE_handler(padapter, erp_ie);
 	}
 }
 
@@ -1336,85 +1322,7 @@ static void update_bcn_rsn_ie(struct adapter *padapter)
 {
 }
 
-static void update_bcn_wpa_ie(struct adapter *padapter)
-{
-}
-
-static void update_bcn_wmm_ie(struct adapter *padapter)
-{
-}
-
-static void update_bcn_wps_ie(struct adapter *padapter)
-{
-	u8 *pwps_ie = NULL;
-	u8 *pwps_ie_src;
-	u8 *premainder_ie;
-	u8 *pbackup_remainder_ie = NULL;
-
-	unsigned int wps_ielen = 0, wps_offset, remainder_ielen;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
-	struct wlan_bssid_ex *pnetwork = &pmlmeinfo->network;
-	unsigned char *ie = pnetwork->ies;
-	u32 ielen = pnetwork->ie_length;
-
-	pwps_ie = rtw_get_wps_ie(ie + _FIXED_IE_LENGTH_,
-				 ielen - _FIXED_IE_LENGTH_,
-				 NULL,
-				 &wps_ielen);
-
-	if (!pwps_ie || wps_ielen == 0)
-		return;
-
-	pwps_ie_src = pmlmepriv->wps_beacon_ie;
-	if (!pwps_ie_src)
-		return;
-
-	wps_offset = (unsigned int)(pwps_ie - ie);
-
-	premainder_ie = pwps_ie + wps_ielen;
-
-	remainder_ielen = ielen - wps_offset - wps_ielen;
-
-	if (remainder_ielen)
-		pbackup_remainder_ie = kmemdup(premainder_ie, remainder_ielen, GFP_ATOMIC);
-
-	wps_ielen = (unsigned int)pwps_ie_src[1];/* to get ie data len */
-	if ((wps_offset + wps_ielen + 2 + remainder_ielen) <= MAX_IE_SZ) {
-		memcpy(pwps_ie, pwps_ie_src, wps_ielen + 2);
-		pwps_ie += (wps_ielen + 2);
-
-		if (pbackup_remainder_ie)
-			memcpy(pwps_ie, pbackup_remainder_ie, remainder_ielen);
-
-		/* update ie_length */
-		pnetwork->ie_length = wps_offset + (wps_ielen + 2) + remainder_ielen;
-	}
-
-	kfree(pbackup_remainder_ie);
-}
-
-static void update_bcn_p2p_ie(struct adapter *padapter)
-{
-}
-
-static void update_bcn_vendor_spec_ie(struct adapter *padapter, u8 *oui)
-{
-	if (!memcmp(RTW_WPA_OUI, oui, 4))
-		update_bcn_wpa_ie(padapter);
-
-	else if (!memcmp(WMM_OUI, oui, 4))
-		update_bcn_wmm_ie(padapter);
-
-	else if (!memcmp(WPS_OUI, oui, 4))
-		update_bcn_wps_ie(padapter);
-
-	else if (!memcmp(P2P_OUI, oui, 4))
-		update_bcn_p2p_ie(padapter);
-}
-
-void update_beacon(struct adapter *padapter, u8 ie_id, u8 *oui, u8 tx)
+void update_beacon(struct adapter *padapter, u8 ie_id, u8 tx)
 {
 	struct mlme_priv *pmlmepriv;
 	struct mlme_ext_priv *pmlmeext;
@@ -1467,12 +1375,6 @@ void update_beacon(struct adapter *padapter, u8 ie_id, u8 *oui, u8 tx)
 
 		break;
 
-	case WLAN_EID_VENDOR_SPECIFIC:
-
-		update_bcn_vendor_spec_ie(padapter, oui);
-
-		break;
-
 	default:
 		break;
 	}
@@ -1597,7 +1499,7 @@ void bss_cap_update_on_sta_join(struct adapter *padapter, struct sta_info *psta)
 			if ((pmlmeext->cur_wireless_mode > WIRELESS_11B) &&
 			    (pmlmepriv->num_sta_no_short_preamble == 1)) {
 				beacon_updated = true;
-				update_beacon(padapter, 0xFF, NULL, true);
+				update_beacon(padapter, 0xFF, true);
 			}
 		}
 	} else {
@@ -1609,7 +1511,7 @@ void bss_cap_update_on_sta_join(struct adapter *padapter, struct sta_info *psta)
 			if ((pmlmeext->cur_wireless_mode > WIRELESS_11B) &&
 			    (pmlmepriv->num_sta_no_short_preamble == 0)) {
 				beacon_updated = true;
-				update_beacon(padapter, 0xFF, NULL, true);
+				update_beacon(padapter, 0xFF, true);
 			}
 		}
 	}
@@ -1622,7 +1524,7 @@ void bss_cap_update_on_sta_join(struct adapter *padapter, struct sta_info *psta)
 
 			if (pmlmepriv->num_sta_non_erp == 1) {
 				beacon_updated = true;
-				update_beacon(padapter, WLAN_EID_ERP_INFO, NULL, true);
+				update_beacon(padapter, WLAN_EID_ERP_INFO, true);
 			}
 		}
 	} else {
@@ -1633,7 +1535,7 @@ void bss_cap_update_on_sta_join(struct adapter *padapter, struct sta_info *psta)
 
 			if (pmlmepriv->num_sta_non_erp == 0) {
 				beacon_updated = true;
-				update_beacon(padapter, WLAN_EID_ERP_INFO, NULL, true);
+				update_beacon(padapter, WLAN_EID_ERP_INFO, true);
 			}
 		}
 	}
@@ -1647,7 +1549,7 @@ void bss_cap_update_on_sta_join(struct adapter *padapter, struct sta_info *psta)
 			if ((pmlmeext->cur_wireless_mode > WIRELESS_11B) &&
 			    (pmlmepriv->num_sta_no_short_slot_time == 1)) {
 				beacon_updated = true;
-				update_beacon(padapter, 0xFF, NULL, true);
+				update_beacon(padapter, 0xFF, true);
 			}
 		}
 	} else {
@@ -1659,7 +1561,7 @@ void bss_cap_update_on_sta_join(struct adapter *padapter, struct sta_info *psta)
 			if ((pmlmeext->cur_wireless_mode > WIRELESS_11B) &&
 			    (pmlmepriv->num_sta_no_short_slot_time == 0)) {
 				beacon_updated = true;
-				update_beacon(padapter, 0xFF, NULL, true);
+				update_beacon(padapter, 0xFF, true);
 			}
 		}
 	}
@@ -1694,8 +1596,8 @@ void bss_cap_update_on_sta_join(struct adapter *padapter, struct sta_info *psta)
 	}
 
 	if (rtw_ht_operation_update(padapter) > 0) {
-		update_beacon(padapter, WLAN_EID_HT_CAPABILITY, NULL, false);
-		update_beacon(padapter, WLAN_EID_HT_OPERATION, NULL, true);
+		update_beacon(padapter, WLAN_EID_HT_CAPABILITY, false);
+		update_beacon(padapter, WLAN_EID_HT_OPERATION, true);
 	}
 
 	/* update associated stations cap. */
@@ -1717,7 +1619,7 @@ u8 bss_cap_update_on_sta_leave(struct adapter *padapter, struct sta_info *psta)
 		if (pmlmeext->cur_wireless_mode > WIRELESS_11B &&
 		    pmlmepriv->num_sta_no_short_preamble == 0){
 			beacon_updated = true;
-			update_beacon(padapter, 0xFF, NULL, true);
+			update_beacon(padapter, 0xFF, true);
 		}
 	}
 
@@ -1726,7 +1628,7 @@ u8 bss_cap_update_on_sta_leave(struct adapter *padapter, struct sta_info *psta)
 		pmlmepriv->num_sta_non_erp--;
 		if (pmlmepriv->num_sta_non_erp == 0) {
 			beacon_updated = true;
-			update_beacon(padapter, WLAN_EID_ERP_INFO, NULL, true);
+			update_beacon(padapter, WLAN_EID_ERP_INFO, true);
 		}
 	}
 
@@ -1736,7 +1638,7 @@ u8 bss_cap_update_on_sta_leave(struct adapter *padapter, struct sta_info *psta)
 		if (pmlmeext->cur_wireless_mode > WIRELESS_11B &&
 		    pmlmepriv->num_sta_no_short_slot_time == 0){
 			beacon_updated = true;
-			update_beacon(padapter, 0xFF, NULL, true);
+			update_beacon(padapter, 0xFF, true);
 		}
 	}
 
@@ -1756,8 +1658,8 @@ u8 bss_cap_update_on_sta_leave(struct adapter *padapter, struct sta_info *psta)
 	}
 
 	if (rtw_ht_operation_update(padapter) > 0) {
-		update_beacon(padapter, WLAN_EID_HT_CAPABILITY, NULL, false);
-		update_beacon(padapter, WLAN_EID_HT_OPERATION, NULL, true);
+		update_beacon(padapter, WLAN_EID_HT_CAPABILITY, false);
+		update_beacon(padapter, WLAN_EID_HT_OPERATION, true);
 	}
 
 	return beacon_updated;
@@ -1889,7 +1791,7 @@ void rtw_ap_restore_network(struct adapter *padapter)
 	char chk_alive_list[NUM_STA];
 	int i;
 
-	rtw_setopmode_cmd(padapter, Ndis802_11APMode, false);
+	rtw_setopmode_cmd(padapter, NL80211_IFTYPE_AP, false);
 
 	set_channel_bwmode(padapter,
 			   pmlmeext->cur_channel,
@@ -1898,8 +1800,8 @@ void rtw_ap_restore_network(struct adapter *padapter)
 
 	start_bss_network(padapter);
 
-	if ((padapter->securitypriv.dot11PrivacyAlgrthm == _TKIP_) ||
-	    (padapter->securitypriv.dot11PrivacyAlgrthm == _AES_)) {
+	if ((padapter->securitypriv.dot11_privacy_algrthm == _TKIP_) ||
+	    (padapter->securitypriv.dot11_privacy_algrthm == _AES_)) {
 		/* restore group key, WEP keys is restored in ips_leave() */
 		rtw_set_key(padapter,
 			    psecuritypriv,
@@ -1934,8 +1836,8 @@ void rtw_ap_restore_network(struct adapter *padapter)
 			Update_RA_Entry(padapter, psta);
 			/* pairwise key */
 			/* per sta pairwise key and settings */
-			if ((psecuritypriv->dot11PrivacyAlgrthm == _TKIP_) ||
-			    (psecuritypriv->dot11PrivacyAlgrthm == _AES_)) {
+			if ((psecuritypriv->dot11_privacy_algrthm == _TKIP_) ||
+			    (psecuritypriv->dot11_privacy_algrthm == _AES_)) {
 				rtw_setstakey_cmd(padapter, psta, true, false);
 			}
 		}
diff --git a/drivers/staging/rtl8723bs/core/rtw_cmd.c b/drivers/staging/rtl8723bs/core/rtw_cmd.c
index b932670f5d63a..d8ad1c75195b7 100644
--- a/drivers/staging/rtl8723bs/core/rtw_cmd.c
+++ b/drivers/staging/rtl8723bs/core/rtw_cmd.c
@@ -11,78 +11,78 @@
 #include <linux/delay.h>
 
 static struct _cmd_callback rtw_cmd_callback[] = {
-	{GEN_CMD_CODE(_Read_MACREG), NULL}, /*0*/
-	{GEN_CMD_CODE(_Write_MACREG), NULL},
-	{GEN_CMD_CODE(_Read_BBREG), &rtw_getbbrfreg_cmdrsp_callback},
-	{GEN_CMD_CODE(_Write_BBREG), NULL},
-	{GEN_CMD_CODE(_Read_RFREG), &rtw_getbbrfreg_cmdrsp_callback},
-	{GEN_CMD_CODE(_Write_RFREG), NULL}, /*5*/
-	{GEN_CMD_CODE(_Read_EEPROM), NULL},
-	{GEN_CMD_CODE(_Write_EEPROM), NULL},
-	{GEN_CMD_CODE(_Read_EFUSE), NULL},
-	{GEN_CMD_CODE(_Write_EFUSE), NULL},
-
-	{GEN_CMD_CODE(_Read_CAM),	NULL},	/*10*/
-	{GEN_CMD_CODE(_Write_CAM),	 NULL},
-	{GEN_CMD_CODE(_setBCNITV), NULL},
-	{GEN_CMD_CODE(_setMBIDCFG), NULL},
-	{GEN_CMD_CODE(_JoinBss), &rtw_joinbss_cmd_callback},  /*14*/
-	{GEN_CMD_CODE(_DisConnect), &rtw_disassoc_cmd_callback}, /*15*/
-	{GEN_CMD_CODE(_CreateBss), &rtw_createbss_cmd_callback},
-	{GEN_CMD_CODE(_SetOpMode), NULL},
-	{GEN_CMD_CODE(_SiteSurvey), &rtw_survey_cmd_callback}, /*18*/
-	{GEN_CMD_CODE(_SetAuth), NULL},
-
-	{GEN_CMD_CODE(_SetKey), NULL},	/*20*/
-	{GEN_CMD_CODE(_SetStaKey), &rtw_setstaKey_cmdrsp_callback},
-	{GEN_CMD_CODE(_SetAssocSta), &rtw_setassocsta_cmdrsp_callback},
-	{GEN_CMD_CODE(_DelAssocSta), NULL},
-	{GEN_CMD_CODE(_SetStaPwrState), NULL},
-	{GEN_CMD_CODE(_SetBasicRate), NULL}, /*25*/
-	{GEN_CMD_CODE(_GetBasicRate), NULL},
-	{GEN_CMD_CODE(_SetDataRate), NULL},
-	{GEN_CMD_CODE(_GetDataRate), NULL},
-	{GEN_CMD_CODE(_SetPhyInfo), NULL},
-
-	{GEN_CMD_CODE(_GetPhyInfo), NULL}, /*30*/
-	{GEN_CMD_CODE(_SetPhy), NULL},
-	{GEN_CMD_CODE(_GetPhy), NULL},
-	{GEN_CMD_CODE(_readRssi), NULL},
-	{GEN_CMD_CODE(_readGain), NULL},
-	{GEN_CMD_CODE(_SetAtim), NULL}, /*35*/
-	{GEN_CMD_CODE(_SetPwrMode), NULL},
-	{GEN_CMD_CODE(_JoinbssRpt), NULL},
-	{GEN_CMD_CODE(_SetRaTable), NULL},
-	{GEN_CMD_CODE(_GetRaTable), NULL},
-
-	{GEN_CMD_CODE(_GetCCXReport), NULL}, /*40*/
-	{GEN_CMD_CODE(_GetDTMReport),	NULL},
-	{GEN_CMD_CODE(_GetTXRateStatistics), NULL},
-	{GEN_CMD_CODE(_SetUsbSuspend), NULL},
-	{GEN_CMD_CODE(_SetH2cLbk), NULL},
-	{GEN_CMD_CODE(_AddBAReq), NULL}, /*45*/
-	{GEN_CMD_CODE(_SetChannel), NULL},		/*46*/
-	{GEN_CMD_CODE(_SetTxPower), NULL},
-	{GEN_CMD_CODE(_SwitchAntenna), NULL},
-	{GEN_CMD_CODE(_SetCrystalCap), NULL},
-	{GEN_CMD_CODE(_SetSingleCarrierTx), NULL},	/*50*/
-
-	{GEN_CMD_CODE(_SetSingleToneTx), NULL}, /*51*/
-	{GEN_CMD_CODE(_SetCarrierSuppressionTx), NULL},
-	{GEN_CMD_CODE(_SetContinuousTx), NULL},
-	{GEN_CMD_CODE(_SwitchBandwidth), NULL},		/*54*/
-	{GEN_CMD_CODE(_TX_Beacon), NULL},/*55*/
-
-	{GEN_CMD_CODE(_Set_MLME_EVT), NULL},/*56*/
-	{GEN_CMD_CODE(_Set_Drv_Extra), NULL},/*57*/
-	{GEN_CMD_CODE(_Set_H2C_MSG), NULL},/*58*/
-	{GEN_CMD_CODE(_SetChannelPlan), NULL},/*59*/
-
-	{GEN_CMD_CODE(_SetChannelSwitch), NULL},/*60*/
-	{GEN_CMD_CODE(_TDLS), NULL},/*61*/
-	{GEN_CMD_CODE(_ChkBMCSleepq), NULL}, /*62*/
-
-	{GEN_CMD_CODE(_RunInThreadCMD), NULL},/*63*/
+	{READ_MACREG_CMD, NULL}, /*0*/
+	{WRITE_MACREG_CMD, NULL},
+	{READ_BBREG_CMD, &rtw_getbbrfreg_cmdrsp_callback},
+	{WRITE_BBREG_CMD, NULL},
+	{READ_RFREG_CMD, &rtw_getbbrfreg_cmdrsp_callback},
+	{WRITE_RFREG_CMD, NULL}, /*5*/
+	{READ_EEPROM_CMD, NULL},
+	{WRITE_EEPROM_CMD, NULL},
+	{READ_EFUSE_CMD, NULL},
+	{WRITE_EFUSE_CMD, NULL},
+
+	{READ_CAM_CMD, NULL},	/*10*/
+	{WRITE_CAM_CMD, NULL},
+	{SET_BCNITV_CMD, NULL},
+	{SET_MBIDCFG_CMD, NULL},
+	{JOIN_BSS_CMD, &rtw_joinbss_cmd_callback},  /*14*/
+	{DISCONNECT_CMD, &rtw_disassoc_cmd_callback}, /*15*/
+	{CREATE_BSS_CMD, &rtw_createbss_cmd_callback},
+	{SET_OP_MODE_CMD, NULL},
+	{SITE_SURVEY_CMD, &rtw_survey_cmd_callback}, /*18*/
+	{SET_AUTH_CMD, NULL},
+
+	{SET_KEY_CMD, NULL},	/*20*/
+	{SET_STA_KEY_CMD, &rtw_setstaKey_cmdrsp_callback},
+	{SET_ASSOC_STA_CMD, &rtw_setassocsta_cmdrsp_callback},
+	{DEL_ASSOC_STA_CMD, NULL},
+	{SET_STA_PWR_STATE_CMD, NULL},
+	{SET_BASIC_RATE_CMD, NULL}, /*25*/
+	{GET_BASIC_RATE_CMD, NULL},
+	{SET_DATA_RATE_CMD, NULL},
+	{GET_DATA_RATE_CMD, NULL},
+	{SET_PHY_INFO_CMD, NULL},
+
+	{GET_PHY_INFO_CMD, NULL}, /*30*/
+	{SET_PHY_CMD, NULL},
+	{GET_PHY_CMD, NULL},
+	{READ_RSSI_CMD, NULL},
+	{READ_GAIN_CMD, NULL},
+	{SET_ATIM_CMD, NULL}, /*35*/
+	{SET_PWR_MODE_CMD, NULL},
+	{JOIN_BSS_RPT_CMD, NULL},
+	{SET_RA_TABLE_CMD, NULL},
+	{GET_RA_TABLE_CMD, NULL},
+
+	{GET_CCX_REPORT_CMD, NULL}, /*40*/
+	{GET_DTM_REPORT_CMD,	NULL},
+	{GET_TX_RATE_STATISTICS_CMD, NULL},
+	{SET_USB_SUSPEND_CMD, NULL},
+	{SET_H2C_LBK_CMD, NULL},
+	{ADD_BA_REQ_CMD, NULL}, /*45*/
+	{SET_CHANNEL_CMD, NULL},		/*46*/
+	{SET_TX_POWER_CMD, NULL},
+	{SWITCH_ANTENNA_CMD, NULL},
+	{SET_CRYSTAL_CAP_CMD, NULL},
+	{SET_SINGLE_CARRIER_TX_CMD, NULL},	/*50*/
+
+	{SET_SINGLE_TONE_TX_CMD, NULL}, /*51*/
+	{SET_CARRIER_SUPPRESSION_TX_CMD, NULL},
+	{SET_CONTINUOUS_TX_CMD, NULL},
+	{SWITCH_BANDWIDTH_CMD, NULL},		/*54*/
+	{TX_BEACON_CMD, NULL},/*55*/
+
+	{SET_MLME_EVT_CMD, NULL},/*56*/
+	{SET_DRV_EXTRA_CMD, NULL},/*57*/
+	{SET_H2C_MSG_CMD, NULL},/*58*/
+	{SET_CHANNEL_PLAN_CMD, NULL},/*59*/
+
+	{SET_CHANNEL_SWITCH_CMD, NULL},/*60*/
+	{TDLS_CMD, NULL},/*61*/
+	{CHK_BMC_SLEEPQ_CMD, NULL}, /*62*/
+
+	{RUN_IN_THREAD_CMD, NULL},/*63*/
 };
 
 static struct cmd_hdl wlancmds[] = {
@@ -178,15 +178,12 @@ int rtw_init_cmd_priv(struct	cmd_priv *pcmdpriv)
 
 	pcmdpriv->cmd_buf = PTR_ALIGN(pcmdpriv->cmd_allocated_buf, CMDBUFF_ALIGN_SZ);
 
-	pcmdpriv->rsp_allocated_buf = kzalloc(MAX_RSPSZ + 4, GFP_ATOMIC);
-	if (!pcmdpriv->rsp_allocated_buf) {
+	pcmdpriv->rsp_buf = kzalloc(MAX_RSPSZ, GFP_ATOMIC);
+	if (!pcmdpriv->rsp_buf) {
 		kfree(pcmdpriv->cmd_allocated_buf);
 		return -ENOMEM;
 	}
 
-	pcmdpriv->rsp_buf = pcmdpriv->rsp_allocated_buf + 4 -
-		((SIZE_PTR)(pcmdpriv->rsp_allocated_buf) & 3);
-
 	pcmdpriv->cmd_issued_cnt = 0;
 	pcmdpriv->cmd_done_cnt = 0;
 	pcmdpriv->rsp_cnt = 0;
@@ -232,7 +229,7 @@ void _rtw_free_cmd_priv(struct	cmd_priv *pcmdpriv)
 	if (pcmdpriv) {
 		kfree(pcmdpriv->cmd_allocated_buf);
 
-		kfree(pcmdpriv->rsp_allocated_buf);
+		kfree(pcmdpriv->rsp_buf);
 
 		mutex_destroy(&pcmdpriv->sctx_mutex);
 	}
@@ -299,12 +296,12 @@ void rtw_free_cmd_priv(struct	cmd_priv *pcmdpriv)
 int rtw_cmd_filter(struct cmd_priv *pcmdpriv, struct cmd_obj *cmd_obj);
 int rtw_cmd_filter(struct cmd_priv *pcmdpriv, struct cmd_obj *cmd_obj)
 {
-	u8 bAllow = false; /* set to true to allow enqueuing cmd when hw_init_completed is false */
+	bool allow = false; /* set to true to allow enqueuing cmd when hw_init_completed is false */
 
-	if (cmd_obj->cmdcode == GEN_CMD_CODE(_SetChannelPlan))
-		bAllow = true;
+	if (cmd_obj->cmdcode == SET_CHANNEL_PLAN_CMD)
+		allow = true;
 
-	if ((!pcmdpriv->padapter->hw_init_completed && !bAllow) ||
+	if ((!pcmdpriv->padapter->hw_init_completed && !allow) ||
 	    !atomic_read(&pcmdpriv->cmdthd_running))	/* com_thread not running */
 		return _FAIL;
 
@@ -344,8 +341,8 @@ struct	cmd_obj	*rtw_dequeue_cmd(struct cmd_priv *pcmdpriv)
 
 void rtw_free_cmd_obj(struct cmd_obj *pcmd)
 {
-	if ((pcmd->cmdcode != _JoinBss_CMD_) &&
-	    (pcmd->cmdcode != _CreateBss_CMD_)) {
+	if ((pcmd->cmdcode != JOIN_BSS_CMD) &&
+	    (pcmd->cmdcode != CREATE_BSS_CMD)) {
 		/* free parmbuf in cmd_obj */
 		kfree(pcmd->parmbuf);
 	}
@@ -502,7 +499,7 @@ int rtw_cmd_thread(void *context)
 			break;
 		}
 
-		if (pcmd->cmdcode == GEN_CMD_CODE(_Set_Drv_Extra)) {
+		if (pcmd->cmdcode == SET_DRV_EXTRA_CMD) {
 			extra_parm = (struct drvextra_cmd_parm *)pcmd->parmbuf;
 			if (extra_parm->pbuf && extra_parm->size > 0)
 				kfree(extra_parm->pbuf);
@@ -547,7 +544,7 @@ u8 rtw_sitesurvey_cmd(struct adapter  *padapter, struct ndis_802_11_ssid *ssid,
 
 	rtw_free_network_queue(padapter, false);
 
-	init_h2fwcmd_w_parm_no_rsp(ph2c, psurveyPara, GEN_CMD_CODE(_SiteSurvey));
+	init_h2fwcmd_w_parm_no_rsp(ph2c, psurveyPara, SITE_SURVEY_CMD);
 
 	/* psurveyPara->bsslimit = 48; */
 	psurveyPara->scan_mode = pmlmepriv->scan_mode;
@@ -610,7 +607,7 @@ u8 rtw_createbss_cmd(struct adapter  *padapter)
 	}
 
 	INIT_LIST_HEAD(&pcmd->list);
-	pcmd->cmdcode = _CreateBss_CMD_;
+	pcmd->cmdcode = CREATE_BSS_CMD;
 	pcmd->parmbuf = (unsigned char *)pdev_network;
 	pcmd->cmdsz = get_wlan_bssid_ex_sz((struct wlan_bssid_ex *)pdev_network);
 	pcmd->rsp = NULL;
@@ -643,7 +640,7 @@ int rtw_startbss_cmd(struct adapter  *padapter, int flags)
 		}
 
 		INIT_LIST_HEAD(&pcmd->list);
-		pcmd->cmdcode = GEN_CMD_CODE(_CreateBss);
+		pcmd->cmdcode = CREATE_BSS_CMD;
 		pcmd->parmbuf = NULL;
 		pcmd->cmdsz =  0;
 		pcmd->rsp = NULL;
@@ -682,7 +679,7 @@ u8 rtw_joinbss_cmd(struct adapter  *padapter, struct wlan_network *pnetwork)
 	struct security_priv *psecuritypriv = &padapter->securitypriv;
 	struct registry_priv *pregistrypriv = &padapter->registrypriv;
 	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
-	enum ndis_802_11_network_infrastructure ndis_network_mode = pnetwork->network.infrastructure_mode;
+	enum nl80211_iftype ndis_network_mode = pnetwork->network.infrastructure_mode;
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
 	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 	u32 tmp_len;
@@ -699,17 +696,15 @@ u8 rtw_joinbss_cmd(struct adapter  *padapter, struct wlan_network *pnetwork)
 	/* for hidden ap to set fw_state here */
 	if (!check_fwstate(pmlmepriv, WIFI_STATION_STATE | WIFI_ADHOC_STATE)) {
 		switch (ndis_network_mode) {
-		case Ndis802_11IBSS:
+		case NL80211_IFTYPE_ADHOC:
 			set_fwstate(pmlmepriv, WIFI_ADHOC_STATE);
 			break;
 
-		case Ndis802_11Infrastructure:
+		case NL80211_IFTYPE_STATION:
 			set_fwstate(pmlmepriv, WIFI_STATION_STATE);
 			break;
 
-		case Ndis802_11APMode:
-		case Ndis802_11AutoUnknown:
-		case Ndis802_11InfrastructureMax:
+		default:
 			break;
 		}
 	}
@@ -759,14 +754,16 @@ u8 rtw_joinbss_cmd(struct adapter  *padapter, struct wlan_network *pnetwork)
 	}
 
 	phtpriv->ht_option = false;
-	ptmp = rtw_get_ie(&pnetwork->network.ies[12], WLAN_EID_HT_CAPABILITY, &tmp_len, pnetwork->network.ie_length - 12);
+	ptmp = rtw_get_ie(&pnetwork->network.ies[12], WLAN_EID_HT_CAPABILITY, &tmp_len,
+			pnetwork->network.ie_length - 12);
+
 	if (pregistrypriv->ht_enable && ptmp && tmp_len > 0) {
 		/* Added by Albert 2010/06/23 */
 		/* For the WEP mode, we will use the bg mode to do the connection to avoid some IOT issue. */
 		/* Especially for Realtek 8192u SoftAP. */
-		if ((padapter->securitypriv.dot11PrivacyAlgrthm != _WEP40_) &&
-		    (padapter->securitypriv.dot11PrivacyAlgrthm != _WEP104_) &&
-		    (padapter->securitypriv.dot11PrivacyAlgrthm != _TKIP_)) {
+		if ((padapter->securitypriv.dot11_privacy_algrthm != _WEP40_) &&
+		    (padapter->securitypriv.dot11_privacy_algrthm != _WEP104_) &&
+		    (padapter->securitypriv.dot11_privacy_algrthm != _TKIP_)) {
 			rtw_ht_use_default_setting(padapter);
 
 			rtw_build_wmm_ie_ht(padapter, &psecnetwork->ies[12], &psecnetwork->ie_length);
@@ -785,7 +782,7 @@ u8 rtw_joinbss_cmd(struct adapter  *padapter, struct wlan_network *pnetwork)
 	pcmd->cmdsz = get_wlan_bssid_ex_sz(psecnetwork);/* get cmdsz before endian conversion */
 
 	INIT_LIST_HEAD(&pcmd->list);
-	pcmd->cmdcode = _JoinBss_CMD_;/* GEN_CMD_CODE(_JoinBss) */
+	pcmd->cmdcode = JOIN_BSS_CMD;
 	pcmd->parmbuf = (unsigned char *)psecnetwork;
 	pcmd->rsp = NULL;
 	pcmd->rspsz = 0;
@@ -819,7 +816,7 @@ u8 rtw_disassoc_cmd(struct adapter *padapter, u32 deauth_timeout_ms, bool enqueu
 			kfree(param);
 			goto exit;
 		}
-		init_h2fwcmd_w_parm_no_rsp(cmdobj, param, _DisConnect_CMD_);
+		init_h2fwcmd_w_parm_no_rsp(cmdobj, param, DISCONNECT_CMD);
 		res = rtw_enqueue_cmd(cmdpriv, cmdobj);
 	} else {
 		/* no need to enqueue, do the cmd hdl directly and free cmd parameter */
@@ -832,7 +829,7 @@ u8 rtw_disassoc_cmd(struct adapter *padapter, u32 deauth_timeout_ms, bool enqueu
 	return res;
 }
 
-u8 rtw_setopmode_cmd(struct adapter  *padapter, enum ndis_802_11_network_infrastructure networktype, bool enqueue)
+u8 rtw_setopmode_cmd(struct adapter  *padapter, enum nl80211_iftype networktype, bool enqueue)
 {
 	struct	cmd_obj *ph2c;
 	struct	setopmode_parm *psetop;
@@ -855,7 +852,7 @@ u8 rtw_setopmode_cmd(struct adapter  *padapter, enum ndis_802_11_network_infrast
 			goto exit;
 		}
 
-		init_h2fwcmd_w_parm_no_rsp(ph2c, psetop, _SetOpMode_CMD_);
+		init_h2fwcmd_w_parm_no_rsp(ph2c, psetop, SET_OP_MODE_CMD);
 		res = rtw_enqueue_cmd(pcmdpriv, ph2c);
 	} else {
 		setopmode_hdl(padapter, (u8 *)psetop);
@@ -884,7 +881,7 @@ u8 rtw_setstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 unicast_
 	memcpy(psetstakey_para->addr, sta->hwaddr, ETH_ALEN);
 
 	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE))
-		psetstakey_para->algorithm = (unsigned char)psecuritypriv->dot11PrivacyAlgrthm;
+		psetstakey_para->algorithm = (unsigned char)psecuritypriv->dot11_privacy_algrthm;
 	else
 		GET_ENCRY_ALGO(psecuritypriv, sta, psetstakey_para->algorithm, false);
 
@@ -912,7 +909,7 @@ u8 rtw_setstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 unicast_
 			goto exit;
 		}
 
-		init_h2fwcmd_w_parm_no_rsp(ph2c, psetstakey_para, _SetStaKey_CMD_);
+		init_h2fwcmd_w_parm_no_rsp(ph2c, psetstakey_para, SET_STA_KEY_CMD);
 		ph2c->rsp = (u8 *)psetstakey_rsp;
 		ph2c->rspsz = sizeof(struct set_stakey_rsp);
 		res = rtw_enqueue_cmd(pcmdpriv, ph2c);
@@ -963,7 +960,7 @@ u8 rtw_clearstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 enqueu
 			goto exit;
 		}
 
-		init_h2fwcmd_w_parm_no_rsp(ph2c, psetstakey_para, _SetStaKey_CMD_);
+		init_h2fwcmd_w_parm_no_rsp(ph2c, psetstakey_para, SET_STA_KEY_CMD);
 		ph2c->rsp = (u8 *)psetstakey_rsp;
 		ph2c->rspsz = sizeof(struct set_stakey_rsp);
 
@@ -1001,7 +998,7 @@ u8 rtw_addbareq_cmd(struct adapter *padapter, u8 tid, u8 *addr)
 	paddbareq_parm->tid = tid;
 	memcpy(paddbareq_parm->addr, addr, ETH_ALEN);
 
-	init_h2fwcmd_w_parm_no_rsp(ph2c, paddbareq_parm, GEN_CMD_CODE(_AddBAReq));
+	init_h2fwcmd_w_parm_no_rsp(ph2c, paddbareq_parm, ADD_BA_REQ_CMD);
 
 	/* rtw_enqueue_cmd(pcmdpriv, ph2c); */
 	res = rtw_enqueue_cmd(pcmdpriv, ph2c);
@@ -1010,7 +1007,6 @@ u8 rtw_addbareq_cmd(struct adapter *padapter, u8 tid, u8 *addr)
 	return res;
 }
 
-/* add for CONFIG_IEEE80211W, none 11w can use it */
 u8 rtw_reset_securitypriv_cmd(struct adapter *padapter)
 {
 	struct cmd_obj *ph2c;
@@ -1036,7 +1032,7 @@ u8 rtw_reset_securitypriv_cmd(struct adapter *padapter)
 	pdrvextra_cmd_parm->size = 0;
 	pdrvextra_cmd_parm->pbuf = NULL;
 
-	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, GEN_CMD_CODE(_Set_Drv_Extra));
+	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, SET_DRV_EXTRA_CMD);
 
 	/* rtw_enqueue_cmd(pcmdpriv, ph2c); */
 	res = rtw_enqueue_cmd(pcmdpriv, ph2c);
@@ -1069,7 +1065,7 @@ u8 rtw_free_assoc_resources_cmd(struct adapter *padapter)
 	pdrvextra_cmd_parm->size = 0;
 	pdrvextra_cmd_parm->pbuf = NULL;
 
-	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, GEN_CMD_CODE(_Set_Drv_Extra));
+	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, SET_DRV_EXTRA_CMD);
 
 	/* rtw_enqueue_cmd(pcmdpriv, ph2c); */
 	res = rtw_enqueue_cmd(pcmdpriv, ph2c);
@@ -1102,7 +1098,7 @@ u8 rtw_dynamic_chk_wk_cmd(struct adapter *padapter)
 	pdrvextra_cmd_parm->type = 0;
 	pdrvextra_cmd_parm->size = 0;
 	pdrvextra_cmd_parm->pbuf = NULL;
-	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, GEN_CMD_CODE(_Set_Drv_Extra));
+	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, SET_DRV_EXTRA_CMD);
 
 	/* rtw_enqueue_cmd(pcmdpriv, ph2c); */
 	res = rtw_enqueue_cmd(pcmdpriv, ph2c);
@@ -1351,7 +1347,7 @@ u8 rtw_lps_ctrl_wk_cmd(struct adapter *padapter, u8 lps_ctrl_type, u8 enqueue)
 		pdrvextra_cmd_parm->size = 0;
 		pdrvextra_cmd_parm->pbuf = NULL;
 
-		init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, GEN_CMD_CODE(_Set_Drv_Extra));
+		init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, SET_DRV_EXTRA_CMD);
 
 		res = rtw_enqueue_cmd(pcmdpriv, ph2c);
 	} else {
@@ -1392,7 +1388,7 @@ u8 rtw_dm_in_lps_wk_cmd(struct adapter *padapter)
 	pdrvextra_cmd_parm->size = 0;
 	pdrvextra_cmd_parm->pbuf = NULL;
 
-	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, GEN_CMD_CODE(_Set_Drv_Extra));
+	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, SET_DRV_EXTRA_CMD);
 
 	res = rtw_enqueue_cmd(pcmdpriv, ph2c);
 
@@ -1454,7 +1450,7 @@ u8 rtw_dm_ra_mask_wk_cmd(struct adapter *padapter, u8 *psta)
 	pdrvextra_cmd_parm->size = 0;
 	pdrvextra_cmd_parm->pbuf = psta;
 
-	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, GEN_CMD_CODE(_Set_Drv_Extra));
+	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, SET_DRV_EXTRA_CMD);
 
 	res = rtw_enqueue_cmd(pcmdpriv, ph2c);
 
@@ -1487,7 +1483,7 @@ u8 rtw_ps_cmd(struct adapter *padapter)
 	pdrvextra_cmd_parm->type = 0;
 	pdrvextra_cmd_parm->size = 0;
 	pdrvextra_cmd_parm->pbuf = NULL;
-	init_h2fwcmd_w_parm_no_rsp(ppscmd, pdrvextra_cmd_parm, GEN_CMD_CODE(_Set_Drv_Extra));
+	init_h2fwcmd_w_parm_no_rsp(ppscmd, pdrvextra_cmd_parm, SET_DRV_EXTRA_CMD);
 
 	res = rtw_enqueue_cmd(pcmdpriv, ppscmd);
 
@@ -1526,7 +1522,7 @@ static void rtw_chk_hi_queue_hdl(struct adapter *padapter)
 			pstapriv->sta_dz_bitmap &= ~BIT(0);
 
 			if (update_tim)
-				update_beacon(padapter, WLAN_EID_TIM, NULL, true);
+				update_beacon(padapter, WLAN_EID_TIM, true);
 		} else {/* re check again */
 			rtw_chk_hi_queue_cmd(padapter);
 		}
@@ -1558,7 +1554,7 @@ u8 rtw_chk_hi_queue_cmd(struct adapter *padapter)
 	pdrvextra_cmd_parm->size = 0;
 	pdrvextra_cmd_parm->pbuf = NULL;
 
-	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, GEN_CMD_CODE(_Set_Drv_Extra));
+	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, SET_DRV_EXTRA_CMD);
 
 	res = rtw_enqueue_cmd(pcmdpriv, ph2c);
 
@@ -1650,7 +1646,7 @@ u8 rtw_c2h_packet_wk_cmd(struct adapter *padapter, u8 *pbuf, u16 length)
 	pdrvextra_cmd_parm->size = length;
 	pdrvextra_cmd_parm->pbuf = pbuf;
 
-	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, GEN_CMD_CODE(_Set_Drv_Extra));
+	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, SET_DRV_EXTRA_CMD);
 
 	res = rtw_enqueue_cmd(pcmdpriv, ph2c);
 
@@ -1685,7 +1681,7 @@ u8 rtw_c2h_wk_cmd(struct adapter *padapter, u8 *c2h_evt)
 	pdrvextra_cmd_parm->size =  c2h_evt ? 16 : 0;
 	pdrvextra_cmd_parm->pbuf = c2h_evt;
 
-	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, GEN_CMD_CODE(_Set_Drv_Extra));
+	init_h2fwcmd_w_parm_no_rsp(ph2c, pdrvextra_cmd_parm, SET_DRV_EXTRA_CMD);
 
 	res = rtw_enqueue_cmd(pcmdpriv, ph2c);
 
@@ -1716,6 +1712,8 @@ static void c2h_wk_callback(struct work_struct *work)
 					kfree(c2h_evt);
 					continue;
 				}
+			} else {
+				continue;
 			}
 		}
 
@@ -1769,7 +1767,6 @@ u8 rtw_drvextra_cmd_hdl(struct adapter *padapter, unsigned char *pbuf)
 	case CHECK_HIQ_WK_CID:
 		rtw_chk_hi_queue_hdl(padapter);
 		break;
-	/* add for CONFIG_IEEE80211W, none 11w can use it */
 	case RESET_SECURITYPRIV:
 		rtw_reset_securitypriv(padapter);
 		break;
diff --git a/drivers/staging/rtl8723bs/core/rtw_efuse.c b/drivers/staging/rtl8723bs/core/rtw_efuse.c
index a40d6eab72496..a0f5b4b7ab39b 100644
--- a/drivers/staging/rtl8723bs/core/rtw_efuse.c
+++ b/drivers/staging/rtl8723bs/core/rtw_efuse.c
@@ -82,11 +82,7 @@ u16		Address)
 } /* rtw_efuse_read_1_byte */
 
 /*  11/16/2008 MH Read one byte from real Efuse. */
-u8
-rtw_efuse_one_byte_read(
-struct adapter *padapter,
-u16	addr,
-u8	*data)
+u8 rtw_efuse_one_byte_read(struct adapter *padapter, u16 addr, u8 *data)
 {
 	u32 tmpidx = 0;
 	u8 bResult;
diff --git a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c
index 54f805a6b5ce5..d670540d5809d 100644
--- a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c
+++ b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c
@@ -472,8 +472,9 @@ int rtw_parse_wpa_ie(u8 *wpa_ie, int wpa_ie_len, int *group_cipher, int *pairwis
 		pos += WPA_SELECTOR_LEN;
 		left -= WPA_SELECTOR_LEN;
 
-	} else if (left > 0)
+	} else if (left > 0) {
 		return _FAIL;
+	}
 
 	/* pairwise_cipher */
 	if (left >= 2) {
@@ -492,8 +493,9 @@ int rtw_parse_wpa_ie(u8 *wpa_ie, int wpa_ie_len, int *group_cipher, int *pairwis
 			left -= WPA_SELECTOR_LEN;
 		}
 
-	} else if (left == 1)
+	} else if (left == 1) {
 		return _FAIL;
+	}
 
 	if (is_8021x) {
 		if (left >= 6) {
@@ -532,8 +534,9 @@ int rtw_parse_wpa2_ie(u8 *rsn_ie, int rsn_ie_len, int *group_cipher, int *pairwi
 		pos += RSN_SELECTOR_LEN;
 		left -= RSN_SELECTOR_LEN;
 
-	} else if (left > 0)
+	} else if (left > 0) {
 		return _FAIL;
+	}
 
 	/* pairwise_cipher */
 	if (left >= 2) {
@@ -552,8 +555,9 @@ int rtw_parse_wpa2_ie(u8 *rsn_ie, int rsn_ie_len, int *group_cipher, int *pairwi
 			left -= RSN_SELECTOR_LEN;
 		}
 
-	} else if (left == 1)
+	} else if (left == 1) {
 		return _FAIL;
+	}
 
 	if (is_8021x) {
 		if (left >= 6) {
diff --git a/drivers/staging/rtl8723bs/core/rtw_io.c b/drivers/staging/rtl8723bs/core/rtw_io.c
index c793ae7ea5c46..49c9e4cb8bf6e 100644
--- a/drivers/staging/rtl8723bs/core/rtw_io.c
+++ b/drivers/staging/rtl8723bs/core/rtw_io.c
@@ -13,13 +13,6 @@
  *
  * c. provides the software interface between caller and the hardware interface
  *
- *
- * Compiler Flag Option:
- *
- * 1. CONFIG_SDIO_HCI:
- *  a. USE_SYNC_IRP:  Only sync operations are provided.
- *  b. USE_ASYNC_IRP:Both sync/async operations are provided.
- *
  *  jackson@realtek.com.tw
  */
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c b/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c
index 12bf7780bea59..7d60da02b46d3 100644
--- a/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c
+++ b/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c
@@ -278,15 +278,15 @@ u8 rtw_set_802_11_connect(struct adapter *padapter, u8 *bssid, struct ndis_802_1
 }
 
 u8 rtw_set_802_11_infrastructure_mode(struct adapter *padapter,
-	enum ndis_802_11_network_infrastructure networktype)
+	enum nl80211_iftype networktype)
 {
 	struct	mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct	wlan_network	*cur_network = &pmlmepriv->cur_network;
-	enum ndis_802_11_network_infrastructure *pold_state = &(cur_network->network.infrastructure_mode);
+	enum nl80211_iftype *pold_state = &(cur_network->network.infrastructure_mode);
 
 	if (*pold_state != networktype) {
-		if (*pold_state == Ndis802_11APMode) {
-			/* change to other mode from Ndis802_11APMode */
+		if (*pold_state == NL80211_IFTYPE_AP) {
+			/* change to other mode from AP mode */
 			cur_network->join_res = -1;
 
 			stop_ap_mode(padapter);
@@ -294,14 +294,14 @@ u8 rtw_set_802_11_infrastructure_mode(struct adapter *padapter,
 
 		spin_lock_bh(&pmlmepriv->lock);
 
-		if (check_fwstate(pmlmepriv, _FW_LINKED) || (*pold_state == Ndis802_11IBSS))
+		if (check_fwstate(pmlmepriv, _FW_LINKED) || (*pold_state == NL80211_IFTYPE_ADHOC))
 			rtw_disassoc_cmd(padapter, 0, true);
 
 		if (check_fwstate(pmlmepriv, _FW_LINKED) ||
 		    check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE))
 			rtw_free_assoc_resources(padapter, 1);
 
-		if ((*pold_state == Ndis802_11Infrastructure) || (*pold_state == Ndis802_11IBSS)) {
+		if ((*pold_state == NL80211_IFTYPE_STATION) || (*pold_state == NL80211_IFTYPE_ADHOC)) {
 			if (check_fwstate(pmlmepriv, _FW_LINKED))
 				rtw_indicate_disconnect(padapter); /* will clr Linked_state; before this function, we must have checked whether issue dis-assoc_cmd or not */
 		}
@@ -311,23 +311,22 @@ u8 rtw_set_802_11_infrastructure_mode(struct adapter *padapter,
 		_clr_fwstate_(pmlmepriv, ~WIFI_NULL_STATE);
 
 		switch (networktype) {
-		case Ndis802_11IBSS:
+		case NL80211_IFTYPE_ADHOC:
 			set_fwstate(pmlmepriv, WIFI_ADHOC_STATE);
 			break;
 
-		case Ndis802_11Infrastructure:
+		case NL80211_IFTYPE_STATION:
 			set_fwstate(pmlmepriv, WIFI_STATION_STATE);
 			break;
 
-		case Ndis802_11APMode:
+		case NL80211_IFTYPE_AP:
 			set_fwstate(pmlmepriv, WIFI_AP_STATE);
 			start_ap_mode(padapter);
 			/* rtw_indicate_connect(padapter); */
 
 			break;
 
-		case Ndis802_11AutoUnknown:
-		case Ndis802_11InfrastructureMax:
+		default:
 			break;
 		}
 
@@ -347,7 +346,6 @@ u8 rtw_set_802_11_disassociate(struct adapter *padapter)
 	if (check_fwstate(pmlmepriv, _FW_LINKED)) {
 		rtw_disassoc_cmd(padapter, 0, true);
 		rtw_indicate_disconnect(padapter);
-		/* modify for CONFIG_IEEE80211W, none 11w can use it */
 		rtw_free_assoc_resources_cmd(padapter);
 		rtw_pwr_wakeup(padapter);
 	}
@@ -400,7 +398,7 @@ u8 rtw_set_802_11_authentication_mode(struct adapter *padapter, enum ndis_802_11
 	psecuritypriv->ndisauthtype = authmode;
 
 	if (psecuritypriv->ndisauthtype > 3)
-		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+		psecuritypriv->dot11_auth_algrthm = dot11AuthAlgrthm_8021X;
 
 	res = rtw_set_auth(padapter, psecuritypriv);
 
@@ -427,13 +425,13 @@ u8 rtw_set_802_11_add_wep(struct adapter *padapter, struct ndis_802_11_wep *wep)
 
 	switch (wep->key_length) {
 	case 5:
-		psecuritypriv->dot11PrivacyAlgrthm = _WEP40_;
+		psecuritypriv->dot11_privacy_algrthm = _WEP40_;
 		break;
 	case 13:
-		psecuritypriv->dot11PrivacyAlgrthm = _WEP104_;
+		psecuritypriv->dot11_privacy_algrthm = _WEP104_;
 		break;
 	default:
-		psecuritypriv->dot11PrivacyAlgrthm = _NO_PRIVACY_;
+		psecuritypriv->dot11_privacy_algrthm = _NO_PRIVACY_;
 		break;
 	}
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme.c b/drivers/staging/rtl8723bs/core/rtw_mlme.c
index 1196ec011455b..aaac09100fe3b 100644
--- a/drivers/staging/rtl8723bs/core/rtw_mlme.c
+++ b/drivers/staging/rtl8723bs/core/rtw_mlme.c
@@ -51,8 +51,8 @@ int	rtw_init_mlme_priv(struct adapter *padapter)
 
 	pmlmepriv->pscanned = NULL;
 	pmlmepriv->fw_state = WIFI_STATION_STATE; /*  Must sync with rtw_wdev_alloc() */
-	pmlmepriv->cur_network.network.infrastructure_mode = Ndis802_11AutoUnknown;
-	pmlmepriv->scan_mode = SCAN_ACTIVE;/*  1: active, 0: passive. Maybe someday we should rename this varable to "active_mode" (Jeff) */
+	pmlmepriv->cur_network.network.infrastructure_mode = NL80211_IFTYPE_UNSPECIFIED;
+	pmlmepriv->scan_mode = SCAN_ACTIVE;/*  1: active, 0: passive. Maybe someday we should rename this variable to "active_mode" (Jeff) */
 
 	spin_lock_init(&pmlmepriv->lock);
 	INIT_LIST_HEAD(&pmlmepriv->free_bss_pool.queue);
@@ -265,7 +265,7 @@ void rtw_free_network_queue(struct adapter *padapter, u8 isfreeall)
 bool rtw_if_up(struct adapter *padapter)
 {
 	if (padapter->bDriverStopped || padapter->bSurpriseRemoved ||
-		!check_fwstate(&padapter->mlmepriv, _FW_LINKED))
+	    !check_fwstate(&padapter->mlmepriv, _FW_LINKED))
 		return false;
 
 	return true;
@@ -330,10 +330,10 @@ bool rtw_is_same_ibss(struct adapter *adapter, struct wlan_network *pnetwork)
 {
 	struct security_priv *psecuritypriv = &adapter->securitypriv;
 
-	if ((psecuritypriv->dot11PrivacyAlgrthm != _NO_PRIVACY_) &&
-		    (pnetwork->network.privacy == 0))
+	if ((psecuritypriv->dot11_privacy_algrthm != _NO_PRIVACY_) &&
+	    (pnetwork->network.privacy == 0))
 		return false;
-	else if ((psecuritypriv->dot11PrivacyAlgrthm == _NO_PRIVACY_) &&
+	else if ((psecuritypriv->dot11_privacy_algrthm == _NO_PRIVACY_) &&
 		 (pnetwork->network.privacy == 1))
 		return false;
 
@@ -406,7 +406,7 @@ struct	wlan_network	*rtw_get_oldest_wlan_network(struct __queue *scanned_queue)
 }
 
 void update_network(struct wlan_bssid_ex *dst, struct wlan_bssid_ex *src,
-	struct adapter *padapter, bool update_ie)
+		    struct adapter *padapter, bool update_ie)
 {
 	long rssi_ori = dst->rssi;
 
@@ -633,11 +633,6 @@ static bool rtw_is_desired_network(struct adapter *adapter, struct wlan_network
 	return bselected;
 }
 
-/* TODO: Perry : For Power Management */
-void rtw_atimdone_event_callback(struct adapter	*adapter, u8 *pbuf)
-{
-}
-
 void rtw_survey_event_callback(struct adapter	*adapter, u8 *pbuf)
 {
 	u32 len;
@@ -900,7 +895,6 @@ void rtw_indicate_disconnect(struct adapter *padapter)
 
 		rtw_cfg80211_indicate_disconnect(padapter);
 
-		/* modify for CONFIG_IEEE80211W, none 11w also can use the same command */
 		rtw_reset_securitypriv_cmd(padapter);
 
 		/* set ips_deny_time to avoid enter IPS before LPS leave */
@@ -978,13 +972,13 @@ static struct sta_info *rtw_joinbss_update_stainfo(struct adapter *padapter, str
 		rtw_hal_set_odm_var(padapter, HAL_ODM_STA_INFO, psta, true);
 
 		/* security related */
-		if (padapter->securitypriv.dot11AuthAlgrthm == dot11AuthAlgrthm_8021X) {
+		if (padapter->securitypriv.dot11_auth_algrthm == dot11AuthAlgrthm_8021X) {
 			padapter->securitypriv.binstallGrpkey = false;
 			padapter->securitypriv.busetkipkey = false;
 			padapter->securitypriv.bgrpkey_handshake = false;
 
 			psta->ieee8021x_blocked = true;
-			psta->dot118021XPrivacy = padapter->securitypriv.dot11PrivacyAlgrthm;
+			psta->dot118021XPrivacy = padapter->securitypriv.dot11_privacy_algrthm;
 
 			memset((u8 *)&psta->dot118021x_UncstKey, 0, sizeof(union Keytype));
 
@@ -1056,7 +1050,7 @@ static void rtw_joinbss_update_network(struct adapter *padapter, struct wlan_net
 
 	/* update fw_state will clr _FW_UNDER_LINKING here indirectly */
 	switch (pnetwork->network.infrastructure_mode) {
-	case Ndis802_11Infrastructure:
+	case NL80211_IFTYPE_STATION:
 
 			if (pmlmepriv->fw_state & WIFI_UNDER_WPS)
 				pmlmepriv->fw_state = WIFI_STATION_STATE | WIFI_UNDER_WPS;
@@ -1064,7 +1058,7 @@ static void rtw_joinbss_update_network(struct adapter *padapter, struct wlan_net
 				pmlmepriv->fw_state = WIFI_STATION_STATE;
 
 			break;
-	case Ndis802_11IBSS:
+	case NL80211_IFTYPE_ADHOC:
 			pmlmepriv->fw_state = WIFI_ADHOC_STATE;
 			break;
 	default:
@@ -1087,12 +1081,11 @@ void rtw_reset_securitypriv(struct adapter *adapter)
 	u8 backupPMKIDIndex = 0;
 	u8 backupTKIPCountermeasure = 0x00;
 	u32 backupTKIPcountermeasure_time = 0;
-	/*  add for CONFIG_IEEE80211W, none 11w also can use */
 	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
 
 	spin_lock_bh(&adapter->security_key_mutex);
 
-	if (adapter->securitypriv.dot11AuthAlgrthm == dot11AuthAlgrthm_8021X) {
+	if (adapter->securitypriv.dot11_auth_algrthm == dot11AuthAlgrthm_8021X) {
 		/* 802.1x */
 		/*  Added by Albert 2009/02/18 */
 		/*  We have to backup the PMK information for WiFi PMK Caching test item. */
@@ -1126,8 +1119,8 @@ void rtw_reset_securitypriv(struct adapter *adapter)
 		/*  */
 		struct security_priv *psec_priv = &adapter->securitypriv;
 
-		psec_priv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open;  /* open system */
-		psec_priv->dot11PrivacyAlgrthm = _NO_PRIVACY_;
+		psec_priv->dot11_auth_algrthm = dot11AuthAlgrthm_Open;  /* open system */
+		psec_priv->dot11_privacy_algrthm = _NO_PRIVACY_;
 		psec_priv->dot11PrivacyKeyIndex = 0;
 
 		psec_priv->dot118021XGrpPrivacy = _NO_PRIVACY_;
@@ -1137,7 +1130,6 @@ void rtw_reset_securitypriv(struct adapter *adapter)
 		psec_priv->ndisencryptstatus = Ndis802_11WEPDisabled;
 		/*  */
 	}
-	/*  add for CONFIG_IEEE80211W, none 11w also can use */
 	spin_unlock_bh(&adapter->security_key_mutex);
 }
 
@@ -1342,8 +1334,8 @@ void rtw_stassoc_event_callback(struct adapter *adapter, u8 *pbuf)
 
 	rtw_sta_media_status_rpt(adapter, psta, 1);
 
-	if (adapter->securitypriv.dot11AuthAlgrthm == dot11AuthAlgrthm_8021X)
-		psta->dot118021XPrivacy = adapter->securitypriv.dot11PrivacyAlgrthm;
+	if (adapter->securitypriv.dot11_auth_algrthm == dot11AuthAlgrthm_8021X)
+		psta->dot118021XPrivacy = adapter->securitypriv.dot11_privacy_algrthm;
 
 	psta->ieee8021x_blocked = false;
 
@@ -1444,7 +1436,7 @@ void rtw_stadel_event_callback(struct adapter *adapter, u8 *pbuf)
 	}
 
 	if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) ||
-	      check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) {
+	    check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) {
 		rtw_free_stainfo(adapter,  psta);
 
 		if (adapter->stapriv.asoc_sta_count == 1) {/* a sta + bc/mc_stainfo (not Ibss_stainfo) */
@@ -1865,9 +1857,9 @@ signed int rtw_set_auth(struct adapter *adapter, struct security_priv *psecurity
 		goto exit;
 	}
 
-	psetauthparm->mode = (unsigned char)psecuritypriv->dot11AuthAlgrthm;
+	psetauthparm->mode = (unsigned char)psecuritypriv->dot11_auth_algrthm;
 
-	pcmd->cmdcode = _SetAuth_CMD_;
+	pcmd->cmdcode = SET_AUTH_CMD;
 	pcmd->parmbuf = (unsigned char *)psetauthparm;
 	pcmd->cmdsz =  (sizeof(struct setauth_parm));
 	pcmd->rsp = NULL;
@@ -1895,10 +1887,10 @@ signed int rtw_set_key(struct adapter *adapter, struct security_priv *psecurityp
 		goto exit;
 	}
 
-	if (psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_8021X)
+	if (psecuritypriv->dot11_auth_algrthm == dot11AuthAlgrthm_8021X)
 		psetkeyparm->algorithm = (unsigned char)psecuritypriv->dot118021XGrpPrivacy;
 	else
-		psetkeyparm->algorithm = (u8)psecuritypriv->dot11PrivacyAlgrthm;
+		psetkeyparm->algorithm = (u8)psecuritypriv->dot11_privacy_algrthm;
 
 	psetkeyparm->keyid = (u8)keyid;/* 0~3 */
 	psetkeyparm->set_tx = set_tx;
@@ -1938,7 +1930,7 @@ signed int rtw_set_key(struct adapter *adapter, struct security_priv *psecurityp
 			goto exit;
 		}
 
-		pcmd->cmdcode = _SetKey_CMD_;
+		pcmd->cmdcode = SET_KEY_CMD;
 		pcmd->parmbuf = (u8 *)psetkeyparm;
 		pcmd->cmdsz =  (sizeof(struct setkey_parm));
 		pcmd->rsp = NULL;
@@ -2003,7 +1995,7 @@ static int SecIsInPMKIDList(struct adapter *Adapter, u8 *bssid)
 
 	for (i = 0; i < NUM_PMKID_CACHE; i++)
 		if ((p->PMKIDList[i].bUsed) &&
-				(!memcmp(p->PMKIDList[i].Bssid, bssid, ETH_ALEN)))
+		    (!memcmp(p->PMKIDList[i].Bssid, bssid, ETH_ALEN)))
 			return i;
 	return -1;
 }
@@ -2128,29 +2120,13 @@ void rtw_update_registrypriv_dev_network(struct adapter *adapter)
 	struct	security_priv *psecuritypriv = &adapter->securitypriv;
 	struct	wlan_network	*cur_network = &adapter->mlmepriv.cur_network;
 
-	pdev_network->privacy = (psecuritypriv->dot11PrivacyAlgrthm > 0 ? 1 : 0) ; /*  adhoc no 802.1x */
+	pdev_network->privacy = (psecuritypriv->dot11_privacy_algrthm > 0 ? 1 : 0) ; /*  adhoc no 802.1x */
 
 	pdev_network->rssi = 0;
 
-	switch (pregistrypriv->wireless_mode) {
-	case WIRELESS_11B:
-		pdev_network->network_type_in_use = (Ndis802_11DS);
-		break;
-	case WIRELESS_11G:
-	case WIRELESS_11BG:
-	case WIRELESS_11_24N:
-	case WIRELESS_11G_24N:
-	case WIRELESS_11BG_24N:
-		pdev_network->network_type_in_use = (Ndis802_11OFDM24);
-		break;
-	default:
-		/*  TODO */
-		break;
-	}
-
 	pdev_network->configuration.ds_config = (pregistrypriv->channel);
 
-	if (cur_network->network.infrastructure_mode == Ndis802_11IBSS)
+	if (cur_network->network.infrastructure_mode == NL80211_IFTYPE_ADHOC)
 		pdev_network->configuration.atim_window = (0);
 
 	pdev_network->infrastructure_mode = (cur_network->network.infrastructure_mode);
@@ -2369,7 +2345,7 @@ unsigned int rtw_restructure_ht_ie(struct adapter *padapter, u8 *in_ie, u8 *out_
 
 	ht_capie.ampdu_params_info = (max_rx_ampdu_factor & 0x03);
 
-	if (padapter->securitypriv.dot11PrivacyAlgrthm == _AES_)
+	if (padapter->securitypriv.dot11_privacy_algrthm == _AES_)
 		ht_capie.ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_DENSITY & (0x07 << 2));
 	else
 		ht_capie.ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_DENSITY & 0x00);
diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
index a443b3530fb98..8cf3c14a0dd79 100644
--- a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
+++ b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
@@ -172,12 +172,12 @@ int rtw_ch_set_search_ch(struct rt_channel_info *ch_set, const u32 ch)
 {
 	int i;
 
-	for (i = 0; ch_set[i].ChannelNum != 0; i++) {
-		if (ch == ch_set[i].ChannelNum)
+	for (i = 0; ch_set[i].channel_num != 0; i++) {
+		if (ch == ch_set[i].channel_num)
 			break;
 	}
 
-	if (i >= ch_set[i].ChannelNum)
+	if (i >= ch_set[i].channel_num)
 		return -1;
 	return i;
 }
@@ -262,7 +262,7 @@ static int has_channel(struct rt_channel_info *channel_set,
 	int i;
 
 	for (i = 0; i < chanset_size; i++)
-		if (channel_set[i].ChannelNum == chan)
+		if (channel_set[i].channel_num == chan)
 			return 1;
 
 	return 0;
@@ -324,7 +324,7 @@ static u8 init_channel_set(struct adapter *padapter, u8 ChannelPlan, struct rt_c
 	u8 b2_4GBand = false;
 	u8 Index2G = 0;
 
-	memset(channel_set, 0, sizeof(struct rt_channel_info)*MAX_CHANNEL_NUM);
+	memset(channel_set, 0, sizeof(struct rt_channel_info) * MAX_CHANNEL_NUM);
 
 	if (ChannelPlan >= RT_CHANNEL_DOMAIN_MAX && ChannelPlan != RT_CHANNEL_DOMAIN_REALTEK_DEFINE)
 		return chanset_size;
@@ -339,22 +339,25 @@ static u8 init_channel_set(struct adapter *padapter, u8 ChannelPlan, struct rt_c
 
 	if (b2_4GBand) {
 		for (index = 0; index < RTW_ChannelPlan2G[Index2G].Len; index++) {
-			channel_set[chanset_size].ChannelNum = RTW_ChannelPlan2G[Index2G].Channel[index];
+			channel_set[chanset_size].channel_num =
+				RTW_ChannelPlan2G[Index2G].Channel[index];
 
 			if ((ChannelPlan == RT_CHANNEL_DOMAIN_GLOBAL_DOAMIN) ||/* Channel 1~11 is active, and 12~14 is passive */
 				(ChannelPlan == RT_CHANNEL_DOMAIN_GLOBAL_NULL)) {
-				if (channel_set[chanset_size].ChannelNum >= 1 && channel_set[chanset_size].ChannelNum <= 11)
-					channel_set[chanset_size].ScanType = SCAN_ACTIVE;
-				else if ((channel_set[chanset_size].ChannelNum  >= 12 && channel_set[chanset_size].ChannelNum  <= 14))
-					channel_set[chanset_size].ScanType  = SCAN_PASSIVE;
+				if (channel_set[chanset_size].channel_num >= 1 &&
+				    channel_set[chanset_size].channel_num <= 11)
+					channel_set[chanset_size].scan_type = SCAN_ACTIVE;
+				else if (channel_set[chanset_size].channel_num  >= 12 &&
+					 channel_set[chanset_size].channel_num  <= 14)
+					channel_set[chanset_size].scan_type  = SCAN_PASSIVE;
 			} else if (ChannelPlan == RT_CHANNEL_DOMAIN_WORLD_WIDE_13 ||
 				 Index2G == RT_CHANNEL_DOMAIN_2G_WORLD) { /*  channel 12~13, passive scan */
-				if (channel_set[chanset_size].ChannelNum <= 11)
-					channel_set[chanset_size].ScanType = SCAN_ACTIVE;
+				if (channel_set[chanset_size].channel_num <= 11)
+					channel_set[chanset_size].scan_type = SCAN_ACTIVE;
 				else
-					channel_set[chanset_size].ScanType = SCAN_PASSIVE;
+					channel_set[chanset_size].scan_type = SCAN_PASSIVE;
 			} else
-				channel_set[chanset_size].ScanType = SCAN_ACTIVE;
+				channel_set[chanset_size].scan_type = SCAN_ACTIVE;
 
 			chanset_size++;
 		}
@@ -454,12 +457,12 @@ void mgt_dispatcher(struct adapter *padapter, union recv_frame *precv_frame)
 
 	if (psta) {
 		if (GetRetry(pframe)) {
-			if (precv_frame->u.hdr.attrib.seq_num == psta->RxMgmtFrameSeqNum) {
+			if (precv_frame->u.hdr.attrib.seq_num == psta->rx_mgmt_frame_seq_num) {
 				/* drop the duplicate management frame */
 				return;
 			}
 		}
-		psta->RxMgmtFrameSeqNum = precv_frame->u.hdr.attrib.seq_num;
+		psta->rx_mgmt_frame_seq_num = precv_frame->u.hdr.attrib.seq_num;
 	}
 
 	switch (GetFrameSubType(pframe)) {
@@ -519,7 +522,7 @@ unsigned int OnProbeReq(struct adapter *padapter, union recv_frame *precv_frame)
 		if (is_valid_p2p_probereq)
 			goto _issue_probersp;
 
-		if ((ielen != 0 && false == !memcmp((p+2), cur->ssid.ssid, cur->ssid.ssid_length))
+		if ((ielen != 0 && false == !memcmp((p + 2), cur->ssid.ssid, cur->ssid.ssid_length))
 			|| (ielen == 0 && pmlmeinfo->hidden_ssid_mode)
 		)
 			return _SUCCESS;
@@ -588,7 +591,9 @@ unsigned int OnBeacon(struct adapter *padapter, union recv_frame *precv_frame)
 			}
 
 			/* check the vendor of the assoc AP */
-			pmlmeinfo->assoc_AP_vendor = check_assoc_AP(pframe+sizeof(struct ieee80211_hdr_3addr), len-sizeof(struct ieee80211_hdr_3addr));
+			pmlmeinfo->assoc_AP_vendor =
+				check_assoc_AP(pframe + sizeof(struct ieee80211_hdr_3addr),
+					       len - sizeof(struct ieee80211_hdr_3addr));
 
 			/* update TSF Value */
 			update_TSF(pmlmeext, pframe, len);
@@ -607,7 +612,8 @@ unsigned int OnBeacon(struct adapter *padapter, union recv_frame *precv_frame)
 			return _SUCCESS;
 		}
 
-		if (((pmlmeinfo->state&0x03) == WIFI_FW_STATION_STATE) && (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS)) {
+		if (((pmlmeinfo->state & 0x03) == WIFI_FW_STATION_STATE) &&
+		    (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS)) {
 			psta = rtw_get_stainfo(pstapriv, GetAddr2Ptr(pframe));
 			if (psta) {
 				ret = rtw_check_bcn_info(padapter, pframe, len);
@@ -625,7 +631,7 @@ unsigned int OnBeacon(struct adapter *padapter, union recv_frame *precv_frame)
 
 				adaptive_early_32k(pmlmeext, pframe, len);
 			}
-		} else if ((pmlmeinfo->state&0x03) == WIFI_FW_ADHOC_STATE) {
+		} else if ((pmlmeinfo->state & 0x03) == WIFI_FW_ADHOC_STATE) {
 			psta = rtw_get_stainfo(pstapriv, GetAddr2Ptr(pframe));
 			if (psta) {
 				/* update WMM, ERP in the beacon */
@@ -674,7 +680,7 @@ unsigned int OnAuth(struct adapter *padapter, union recv_frame *precv_frame)
 	uint len = precv_frame->u.hdr.len;
 	u8 offset = 0;
 
-	if ((pmlmeinfo->state&0x03) != WIFI_FW_AP_STATE)
+	if ((pmlmeinfo->state & 0x03) != WIFI_FW_AP_STATE)
 		return _FAIL;
 
 	if (len < WLAN_HDR_A3_LEN)
@@ -682,7 +688,7 @@ unsigned int OnAuth(struct adapter *padapter, union recv_frame *precv_frame)
 
 	sa = GetAddr2Ptr(pframe);
 
-	auth_mode = psecuritypriv->dot11AuthAlgrthm;
+	auth_mode = psecuritypriv->dot11_auth_algrthm;
 
 	if (GetPrivacy(pframe)) {
 		u8 *iv;
@@ -694,8 +700,8 @@ unsigned int OnAuth(struct adapter *padapter, union recv_frame *precv_frame)
 		if (len < WLAN_HDR_A3_LEN + 8)
 			return _FAIL;
 
-		iv = pframe+prxattrib->hdrlen;
-		prxattrib->key_index = ((iv[3]>>6)&0x3);
+		iv = pframe + prxattrib->hdrlen;
+		prxattrib->key_index = ((iv[3] >> 6) & 0x3);
 
 		prxattrib->iv_len = 4;
 		prxattrib->icv_len = 4;
@@ -709,8 +715,8 @@ unsigned int OnAuth(struct adapter *padapter, union recv_frame *precv_frame)
 	seq	= le16_to_cpu(*(__le16 *)((SIZE_PTR)pframe + WLAN_HDR_A3_LEN + offset + 2));
 
 	if (auth_mode == 2 &&
-			psecuritypriv->dot11PrivacyAlgrthm != _WEP40_ &&
-			psecuritypriv->dot11PrivacyAlgrthm != _WEP104_)
+			psecuritypriv->dot11_privacy_algrthm != _WEP40_ &&
+			psecuritypriv->dot11_privacy_algrthm != _WEP104_)
 		auth_mode = 0;
 
 	if ((algorithm > 0 && auth_mode == 0) ||	/*  rx a shared-key auth but shared not enabled */
@@ -936,7 +942,7 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 	u8 *pframe = precv_frame->u.hdr.rx_data;
 	uint pkt_len = precv_frame->u.hdr.len;
 
-	if ((pmlmeinfo->state&0x03) != WIFI_FW_AP_STATE)
+	if ((pmlmeinfo->state & 0x03) != WIFI_FW_AP_STATE)
 		return _FAIL;
 
 	frame_type = GetFrameSubType(pframe);
@@ -994,7 +1000,7 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 		goto OnAssocReqFail;
 	} else {
 		/*  check if ssid match */
-		if (memcmp(p+2, cur->ssid.ssid, cur->ssid.ssid_length))
+		if (memcmp(p + 2, cur->ssid.ssid, cur->ssid.ssid_length))
 			status = WLAN_STATUS_CHALLENGE_FAIL;
 
 		if (ie_len != cur->ssid.ssid_length)
@@ -1017,7 +1023,7 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 		if (ie_len > sizeof(supportRate))
 			ie_len = sizeof(supportRate);
 
-		memcpy(supportRate, p+2, ie_len);
+		memcpy(supportRate, p + 2, ie_len);
 		support_rate_num = ie_len;
 
 		p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + ie_offset, WLAN_EID_EXT_SUPP_RATES, &ie_len,
@@ -1052,12 +1058,14 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 		wpa_ie = elems.rsn_ie;
 		wpa_ie_len = elems.rsn_ie_len;
 
-		if (rtw_parse_wpa2_ie(wpa_ie-2, wpa_ie_len+2, &group_cipher, &pairwise_cipher, NULL) == _SUCCESS) {
+		if (rtw_parse_wpa2_ie(wpa_ie - 2, wpa_ie_len + 2, &group_cipher,
+				      &pairwise_cipher, NULL) == _SUCCESS) {
 			pstat->dot8021xalg = 1;/* psk,  todo:802.1x */
 			pstat->wpa_psk |= BIT(1);
 
-			pstat->wpa2_group_cipher = group_cipher&psecuritypriv->wpa2_group_cipher;
-			pstat->wpa2_pairwise_cipher = pairwise_cipher&psecuritypriv->wpa2_pairwise_cipher;
+			pstat->wpa2_group_cipher = group_cipher & psecuritypriv->wpa2_group_cipher;
+			pstat->wpa2_pairwise_cipher = pairwise_cipher &
+				psecuritypriv->wpa2_pairwise_cipher;
 
 			if (!pstat->wpa2_group_cipher)
 				status = WLAN_STATUS_INVALID_GROUP_CIPHER;
@@ -1074,12 +1082,14 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 		wpa_ie = elems.wpa_ie;
 		wpa_ie_len = elems.wpa_ie_len;
 
-		if (rtw_parse_wpa_ie(wpa_ie-2, wpa_ie_len+2, &group_cipher, &pairwise_cipher, NULL) == _SUCCESS) {
+		if (rtw_parse_wpa_ie(wpa_ie - 2, wpa_ie_len + 2, &group_cipher,
+				     &pairwise_cipher, NULL) == _SUCCESS) {
 			pstat->dot8021xalg = 1;/* psk,  todo:802.1x */
 			pstat->wpa_psk |= BIT(0);
 
-			pstat->wpa_group_cipher = group_cipher&psecuritypriv->wpa_group_cipher;
-			pstat->wpa_pairwise_cipher = pairwise_cipher&psecuritypriv->wpa_pairwise_cipher;
+			pstat->wpa_group_cipher = group_cipher & psecuritypriv->wpa_group_cipher;
+			pstat->wpa_pairwise_cipher = pairwise_cipher &
+				psecuritypriv->wpa_pairwise_cipher;
 
 			if (!pstat->wpa_group_cipher)
 				status = WLAN_STATUS_INVALID_GROUP_CIPHER;
@@ -1108,8 +1118,8 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 
 		/*  AP support WPA/RSN, and sta is going to do WPS, but AP is not ready */
 		/*  that the selected registrar of AP is _FLASE */
-		if ((psecuritypriv->wpa_psk > 0)
-			&& (pstat->flags & (WLAN_STA_WPS|WLAN_STA_MAYBE_WPS))) {
+		if ((psecuritypriv->wpa_psk > 0) &&
+		    (pstat->flags & (WLAN_STA_WPS | WLAN_STA_MAYBE_WPS))) {
 			if (pmlmepriv->wps_beacon_ie) {
 				u8 selected_registrar = 0;
 
@@ -1140,7 +1150,7 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 		}
 
 		if (copy_len > 0)
-			memcpy(pstat->wpa_ie, wpa_ie-2, copy_len);
+			memcpy(pstat->wpa_ie, wpa_ie - 2, copy_len);
 	}
 
 	/*  check if there is WMM IE & support WWM-PS */
@@ -1159,7 +1169,7 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 			if (!p)
 				break;
 
-			if (memcmp(p+2, WMM_IE, 6)) {
+			if (memcmp(p + 2, WMM_IE, 6)) {
 				p = p + ie_len + 2;
 				continue;
 			}
@@ -1167,33 +1177,33 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 			pstat->flags |= WLAN_STA_WME;
 
 			pstat->qos_option = 1;
-			pstat->qos_info = *(p+8);
+			pstat->qos_info = *(p + 8);
 
-			pstat->max_sp_len = (pstat->qos_info>>5)&0x3;
+			pstat->max_sp_len = (pstat->qos_info >> 5) & 0x3;
 
-			if ((pstat->qos_info&0xf) != 0xf)
+			if ((pstat->qos_info & 0xf) != 0xf)
 				pstat->has_legacy_ac = true;
 			else
 				pstat->has_legacy_ac = false;
 
-			if (pstat->qos_info&0xf) {
-				if (pstat->qos_info&BIT(0))
-					pstat->uapsd_vo = BIT(0)|BIT(1);
+			if (pstat->qos_info & 0xf) {
+				if (pstat->qos_info & BIT(0))
+					pstat->uapsd_vo = BIT(0) | BIT(1);
 				else
 					pstat->uapsd_vo = 0;
 
-				if (pstat->qos_info&BIT(1))
-					pstat->uapsd_vi = BIT(0)|BIT(1);
+				if (pstat->qos_info & BIT(1))
+					pstat->uapsd_vi = BIT(0) | BIT(1);
 				else
 					pstat->uapsd_vi = 0;
 
-				if (pstat->qos_info&BIT(2))
-					pstat->uapsd_bk = BIT(0)|BIT(1);
+				if (pstat->qos_info & BIT(2))
+					pstat->uapsd_bk = BIT(0) | BIT(1);
 				else
 					pstat->uapsd_bk = 0;
 
-				if (pstat->qos_info&BIT(3))
-					pstat->uapsd_be = BIT(0)|BIT(1);
+				if (pstat->qos_info & BIT(3))
+					pstat->uapsd_be = BIT(0) | BIT(1);
 				else
 					pstat->uapsd_be = 0;
 			}
@@ -1214,14 +1224,14 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 	} else
 		pstat->flags &= ~WLAN_STA_HT;
 
-	if (!pmlmepriv->htpriv.ht_option && (pstat->flags&WLAN_STA_HT)) {
+	if (!pmlmepriv->htpriv.ht_option && (pstat->flags & WLAN_STA_HT)) {
 		status = WLAN_STATUS_CHALLENGE_FAIL;
 		goto OnAssocReqFail;
 	}
 
 	if ((pstat->flags & WLAN_STA_HT) &&
-		    ((pstat->wpa2_pairwise_cipher&WPA_CIPHER_TKIP) ||
-		      (pstat->wpa_pairwise_cipher&WPA_CIPHER_TKIP))) {
+		    ((pstat->wpa2_pairwise_cipher & WPA_CIPHER_TKIP) ||
+		      (pstat->wpa_pairwise_cipher & WPA_CIPHER_TKIP))) {
 		/* status = WLAN_STATUS_CIPHER_SUITE_REJECTED; */
 		/* goto OnAssocReqFail; */
 	}
@@ -1365,7 +1375,8 @@ unsigned int OnAssocRsp(struct adapter *padapter, union recv_frame *precv_frame)
 	pmlmeinfo->slotTime = (pmlmeinfo->capability & BIT(10)) ? 9 : 20;
 
 	/* AID */
-	res = pmlmeinfo->aid = (int)(le16_to_cpu(*(__le16 *)(pframe + WLAN_HDR_A3_LEN + 4))&0x3fff);
+	pmlmeinfo->aid = (int)(le16_to_cpu(*(__le16 *)(pframe + WLAN_HDR_A3_LEN + 4)) & 0x3fff);
+	res = pmlmeinfo->aid;
 
 	/* following are moved to join event callback function */
 	/* to handle HT, WMM, rate adaptive, update MAC reg */
@@ -1597,7 +1608,7 @@ unsigned int OnAction_back(struct adapter *padapter, union recv_frame *precv_fra
 	if (memcmp(myid(&(padapter->eeprompriv)), GetAddr1Ptr(pframe), ETH_ALEN))/* for if1, sta/ap mode */
 		return _SUCCESS;
 
-	if ((pmlmeinfo->state&0x03) != WIFI_FW_AP_STATE)
+	if ((pmlmeinfo->state & 0x03) != WIFI_FW_AP_STATE)
 		if (!(pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS))
 			return _SUCCESS;
 
@@ -1677,7 +1688,7 @@ static s32 rtw_action_public_decache(union recv_frame *recv_frame, s32 token)
 	struct adapter *adapter = recv_frame->u.hdr.adapter;
 	struct mlme_ext_priv *mlmeext = &(adapter->mlmeextpriv);
 	u8 *frame = recv_frame->u.hdr.rx_data;
-	u16 seq_ctrl = ((recv_frame->u.hdr.attrib.seq_num&0xffff) << 4) |
+	u16 seq_ctrl = ((recv_frame->u.hdr.attrib.seq_num & 0xffff) << 4) |
 		(recv_frame->u.hdr.attrib.frag_num & 0xf);
 
 	if (GetRetry(frame)) {
@@ -1813,9 +1824,9 @@ unsigned int OnAction_sa_query(struct adapter *padapter, union recv_frame *precv
 	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
 	unsigned short tid;
 
-	switch (pframe[WLAN_HDR_A3_LEN+1]) {
+	switch (pframe[WLAN_HDR_A3_LEN + 1]) {
 	case 0: /* SA Query req */
-		memcpy(&tid, &pframe[WLAN_HDR_A3_LEN+2], sizeof(unsigned short));
+		memcpy(&tid, &pframe[WLAN_HDR_A3_LEN + 2], sizeof(unsigned short));
 		issue_action_SA_Query(padapter, GetAddr2Ptr(pframe), 1, tid);
 		break;
 
@@ -2033,10 +2044,10 @@ static int update_hidden_ssid(u8 *ies, u32 ies_len, u8 hidden_ssid_mode)
 			u8 *next_ie = ssid_ie + 2 + ssid_len_ori;
 			u32 remain_len = 0;
 
-			remain_len = ies_len - (next_ie-ies);
+			remain_len = ies_len - (next_ie - ies);
 
 			ssid_ie[1] = 0;
-			memcpy(ssid_ie+2, next_ie, remain_len);
+			memcpy(ssid_ie + 2, next_ie, remain_len);
 			len_diff -= ssid_len_ori;
 
 			break;
@@ -2096,16 +2107,16 @@ void issue_beacon(struct adapter *padapter, int timeout_ms)
 	pframe += sizeof(struct ieee80211_hdr_3addr);
 	pattrib->pktlen = sizeof(struct ieee80211_hdr_3addr);
 
-	if ((pmlmeinfo->state&0x03) == WIFI_FW_AP_STATE) {
+	if ((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE) {
 		{
 			int len_diff;
 
 			memcpy(pframe, cur_network->ies, cur_network->ie_length);
-			len_diff = update_hidden_ssid(pframe+_BEACON_IE_OFFSET_,
-						      cur_network->ie_length-_BEACON_IE_OFFSET_,
+			len_diff = update_hidden_ssid(pframe + _BEACON_IE_OFFSET_,
+						      cur_network->ie_length - _BEACON_IE_OFFSET_,
 						      pmlmeinfo->hidden_ssid_mode);
-			pframe += (cur_network->ie_length+len_diff);
-			pattrib->pktlen += (cur_network->ie_length+len_diff);
+			pframe += (cur_network->ie_length + len_diff);
+			pattrib->pktlen += (cur_network->ie_length + len_diff);
 		}
 
 		{
@@ -2113,8 +2124,12 @@ void issue_beacon(struct adapter *padapter, int timeout_ms)
 			uint wps_ielen;
 			u8 sr = 0;
 
-			wps_ie = rtw_get_wps_ie(pmgntframe->buf_addr+TXDESC_OFFSET+sizeof(struct ieee80211_hdr_3addr)+_BEACON_IE_OFFSET_,
-				pattrib->pktlen-sizeof(struct ieee80211_hdr_3addr)-_BEACON_IE_OFFSET_, NULL, &wps_ielen);
+			wps_ie = rtw_get_wps_ie(pmgntframe->buf_addr + TXDESC_OFFSET +
+						sizeof(struct ieee80211_hdr_3addr) +
+						_BEACON_IE_OFFSET_,
+						pattrib->pktlen -
+						sizeof(struct ieee80211_hdr_3addr) -
+						_BEACON_IE_OFFSET_, NULL, &wps_ielen);
 			if (wps_ie && wps_ielen > 0)
 				rtw_get_wps_attr_content(wps_ie,  wps_ielen, WPS_ATTR_SELECTED_REGISTRAR, (u8 *)(&sr), NULL);
 			if (sr != 0)
@@ -2246,8 +2261,10 @@ void issue_probersp(struct adapter *padapter, unsigned char *da, u8 is_valid_p2p
 	if (cur_network->ie_length > MAX_IE_SZ)
 		return;
 
-	if ((pmlmeinfo->state&0x03) == WIFI_FW_AP_STATE) {
-		pwps_ie = rtw_get_wps_ie(cur_network->ies+_FIXED_IE_LENGTH_, cur_network->ie_length-_FIXED_IE_LENGTH_, NULL, &wps_ielen);
+	if ((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE) {
+		pwps_ie = rtw_get_wps_ie(cur_network->ies + _FIXED_IE_LENGTH_,
+					 cur_network->ie_length - _FIXED_IE_LENGTH_,
+					 NULL, &wps_ielen);
 
 		/* inerset & update wps_probe_resp_ie */
 		if (pmlmepriv->wps_probe_resp_ie && pwps_ie && wps_ielen > 0) {
@@ -2265,13 +2282,13 @@ void issue_probersp(struct adapter *padapter, unsigned char *da, u8 is_valid_p2p
 			pattrib->pktlen += wps_offset;
 
 			wps_ielen = (uint)pmlmepriv->wps_probe_resp_ie[1];/* to get ie data len */
-			if ((wps_offset+wps_ielen+2) <= MAX_IE_SZ) {
-				memcpy(pframe, pmlmepriv->wps_probe_resp_ie, wps_ielen+2);
-				pframe += wps_ielen+2;
-				pattrib->pktlen += wps_ielen+2;
+			if ((wps_offset + wps_ielen + 2) <= MAX_IE_SZ) {
+				memcpy(pframe, pmlmepriv->wps_probe_resp_ie, wps_ielen + 2);
+				pframe += wps_ielen + 2;
+				pattrib->pktlen += wps_ielen + 2;
 			}
 
-			if ((wps_offset+wps_ielen+2+remainder_ielen) <= MAX_IE_SZ) {
+			if ((wps_offset + wps_ielen + 2 + remainder_ielen) <= MAX_IE_SZ) {
 				memcpy(pframe, premainder_ie, remainder_ielen);
 				pframe += remainder_ielen;
 				pattrib->pktlen += remainder_ielen;
@@ -2288,14 +2305,15 @@ void issue_probersp(struct adapter *padapter, unsigned char *da, u8 is_valid_p2p
 			signed int ssid_ielen;
 			signed int ssid_ielen_diff;
 			u8 *buf;
-			u8 *ies = pmgntframe->buf_addr+TXDESC_OFFSET+sizeof(struct ieee80211_hdr_3addr);
+			u8 *ies = pmgntframe->buf_addr + TXDESC_OFFSET +
+				sizeof(struct ieee80211_hdr_3addr);
 
 			buf = kzalloc(MAX_IE_SZ, GFP_ATOMIC);
 			if (!buf)
 				return;
 
-			ssid_ie = rtw_get_ie(ies+_FIXED_IE_LENGTH_, WLAN_EID_SSID, &ssid_ielen,
-				(pframe-ies)-_FIXED_IE_LENGTH_);
+			ssid_ie = rtw_get_ie(ies + _FIXED_IE_LENGTH_, WLAN_EID_SSID, &ssid_ielen,
+					     (pframe - ies) - _FIXED_IE_LENGTH_);
 
 			ssid_ielen_diff = cur_network->ssid.ssid_length - ssid_ielen;
 
@@ -2303,8 +2321,8 @@ void issue_probersp(struct adapter *padapter, unsigned char *da, u8 is_valid_p2p
 				uint remainder_ielen;
 				u8 *remainder_ie;
 
-				remainder_ie = ssid_ie+2;
-				remainder_ielen = (pframe-remainder_ie);
+				remainder_ie = ssid_ie + 2;
+				remainder_ielen = (pframe - remainder_ie);
 
 				if (remainder_ielen > MAX_IE_SZ) {
 					netdev_warn(padapter->pnetdev,
@@ -2314,9 +2332,10 @@ void issue_probersp(struct adapter *padapter, unsigned char *da, u8 is_valid_p2p
 				}
 
 				memcpy(buf, remainder_ie, remainder_ielen);
-				memcpy(remainder_ie+ssid_ielen_diff, buf, remainder_ielen);
-				*(ssid_ie+1) = cur_network->ssid.ssid_length;
-				memcpy(ssid_ie+2, cur_network->ssid.ssid, cur_network->ssid.ssid_length);
+				memcpy(remainder_ie + ssid_ielen_diff, buf, remainder_ielen);
+				*(ssid_ie + 1) = cur_network->ssid.ssid_length;
+				memcpy(ssid_ie + 2, cur_network->ssid.ssid,
+				       cur_network->ssid.ssid_length);
 
 				pframe += ssid_ielen_diff;
 				pattrib->pktlen += ssid_ielen_diff;
@@ -2354,7 +2373,7 @@ void issue_probersp(struct adapter *padapter, unsigned char *da, u8 is_valid_p2p
 		/*  DS parameter set */
 		pframe = rtw_set_ie(pframe, WLAN_EID_DS_PARAMS, 1, (unsigned char *)&(cur_network->configuration.ds_config), &pattrib->pktlen);
 
-		if ((pmlmeinfo->state&0x03) == WIFI_FW_ADHOC_STATE) {
+		if ((pmlmeinfo->state & 0x03) == WIFI_FW_ADHOC_STATE) {
 			u8 erpinfo = 0;
 			u32 ATIMWindow;
 			/*  IBSS Parameter Set... */
@@ -2694,7 +2713,8 @@ void issue_asocrsp(struct adapter *padapter, unsigned short status, struct sta_i
 		pframe = rtw_set_ie(pframe, WLAN_EID_SUPP_RATES, pstat->bssratelen, pstat->bssrateset, &(pattrib->pktlen));
 	} else {
 		pframe = rtw_set_ie(pframe, WLAN_EID_SUPP_RATES, 8, pstat->bssrateset, &(pattrib->pktlen));
-		pframe = rtw_set_ie(pframe, WLAN_EID_EXT_SUPP_RATES, (pstat->bssratelen-8), pstat->bssrateset+8, &(pattrib->pktlen));
+		pframe = rtw_set_ie(pframe, WLAN_EID_EXT_SUPP_RATES, (pstat->bssratelen - 8),
+				    pstat->bssrateset + 8, &pattrib->pktlen);
 	}
 
 	if ((pstat->flags & WLAN_STA_HT) && (pmlmepriv->htpriv.ht_option)) {
@@ -2704,18 +2724,18 @@ void issue_asocrsp(struct adapter *padapter, unsigned short status, struct sta_i
 		/* p = hostapd_eid_ht_capabilities_info(hapd, p); */
 		pbuf = rtw_get_ie(ie + _BEACON_IE_OFFSET_, WLAN_EID_HT_CAPABILITY, &ie_len, (pnetwork->ie_length - _BEACON_IE_OFFSET_));
 		if (pbuf && ie_len > 0) {
-			memcpy(pframe, pbuf, ie_len+2);
-			pframe += (ie_len+2);
-			pattrib->pktlen += (ie_len+2);
+			memcpy(pframe, pbuf, ie_len + 2);
+			pframe += (ie_len + 2);
+			pattrib->pktlen += (ie_len + 2);
 		}
 
 		/* FILL HT ADD INFO IE */
 		/* p = hostapd_eid_ht_operation(hapd, p); */
 		pbuf = rtw_get_ie(ie + _BEACON_IE_OFFSET_, WLAN_EID_HT_OPERATION, &ie_len, (pnetwork->ie_length - _BEACON_IE_OFFSET_));
 		if (pbuf && ie_len > 0) {
-			memcpy(pframe, pbuf, ie_len+2);
-			pframe += (ie_len+2);
-			pattrib->pktlen += (ie_len+2);
+			memcpy(pframe, pbuf, ie_len + 2);
+			pframe += (ie_len + 2);
+			pattrib->pktlen += (ie_len + 2);
 		}
 	}
 
@@ -2726,10 +2746,10 @@ void issue_asocrsp(struct adapter *padapter, unsigned short status, struct sta_i
 
 		for (pbuf = ie + _BEACON_IE_OFFSET_; ; pbuf += (ie_len + 2)) {
 			pbuf = rtw_get_ie(pbuf, WLAN_EID_VENDOR_SPECIFIC, &ie_len, (pnetwork->ie_length - _BEACON_IE_OFFSET_ - (ie_len + 2)));
-			if (pbuf && !memcmp(pbuf+2, WMM_PARA_IE, 6)) {
-				memcpy(pframe, pbuf, ie_len+2);
-				pframe += (ie_len+2);
-				pattrib->pktlen += (ie_len+2);
+			if (pbuf && !memcmp(pbuf + 2, WMM_PARA_IE, 6)) {
+				memcpy(pframe, pbuf, ie_len + 2);
+				pframe += (ie_len + 2);
+				pattrib->pktlen += (ie_len + 2);
 
 				break;
 			}
@@ -2967,9 +2987,9 @@ static int _issue_nulldata(struct adapter *padapter, unsigned char *da,
 	fctrl = &(pwlanhdr->frame_control);
 	*(fctrl) = 0;
 
-	if ((pmlmeinfo->state&0x03) == WIFI_FW_AP_STATE)
+	if ((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE)
 		SetFrDs(fctrl);
-	else if ((pmlmeinfo->state&0x03) == WIFI_FW_STATION_STATE)
+	else if ((pmlmeinfo->state & 0x03) == WIFI_FW_STATION_STATE)
 		SetToDs(fctrl);
 
 	if (power_mode)
@@ -3109,9 +3129,9 @@ static int _issue_qos_nulldata(struct adapter *padapter, unsigned char *da,
 	fctrl = &(pwlanhdr->frame_control);
 	*(fctrl) = 0;
 
-	if ((pmlmeinfo->state&0x03) == WIFI_FW_AP_STATE)
+	if ((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE)
 		SetFrDs(fctrl);
-	else if ((pmlmeinfo->state&0x03) == WIFI_FW_STATION_STATE)
+	else if ((pmlmeinfo->state & 0x03) == WIFI_FW_STATION_STATE)
 		SetToDs(fctrl);
 
 	qc = (unsigned short *)(pframe + pattrib->hdrlen - 2);
@@ -3638,7 +3658,7 @@ unsigned int send_delba(struct adapter *padapter, u8 initiator, u8 *addr)
 	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 	u16 tid;
 
-	if ((pmlmeinfo->state&0x03) != WIFI_FW_AP_STATE)
+	if ((pmlmeinfo->state & 0x03) != WIFI_FW_AP_STATE)
 		if (!(pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS))
 			return _SUCCESS;
 
@@ -3649,7 +3669,8 @@ unsigned int send_delba(struct adapter *padapter, u8 initiator, u8 *addr)
 	if (initiator == 0) {/*  recipient */
 		for (tid = 0; tid < MAXTID; tid++) {
 			if (psta->recvreorder_ctrl[tid].enable) {
-				issue_action_BA(padapter, addr, WLAN_ACTION_DELBA, (((tid << 1) | initiator)&0x1F));
+				issue_action_BA(padapter, addr, WLAN_ACTION_DELBA,
+						(((tid << 1) | initiator) & 0x1F));
 				psta->recvreorder_ctrl[tid].enable = false;
 				psta->recvreorder_ctrl[tid].indicate_seq = 0xffff;
 			}
@@ -3657,7 +3678,8 @@ unsigned int send_delba(struct adapter *padapter, u8 initiator, u8 *addr)
 	} else if (initiator == 1) {/*  originator */
 		for (tid = 0; tid < MAXTID; tid++) {
 			if (psta->htpriv.agg_enable_bitmap & BIT(tid)) {
-				issue_action_BA(padapter, addr, WLAN_ACTION_DELBA, (((tid << 1) | initiator)&0x1F));
+				issue_action_BA(padapter, addr, WLAN_ACTION_DELBA,
+						(((tid << 1) | initiator) & 0x1F));
 				psta->htpriv.agg_enable_bitmap &= ~BIT(tid);
 				psta->htpriv.candidate_tid_bitmap &= ~BIT(tid);
 			}
@@ -3682,7 +3704,9 @@ unsigned int send_beacon(struct adapter *padapter)
 			cond_resched();
 			rtw_hal_get_hwreg(padapter, HW_VAR_BCN_VALID, (u8 *)(&bxmitok));
 			poll++;
-		} while ((poll%10) != 0 && false == bxmitok && !padapter->bSurpriseRemoved && !padapter->bDriverStopped);
+		} while ((poll % 10) != 0 && !bxmitok &&
+			 !padapter->bSurpriseRemoved &&
+			 !padapter->bDriverStopped);
 
 	} while (false == bxmitok && issue < 100 && !padapter->bSurpriseRemoved && !padapter->bDriverStopped);
 
@@ -3700,7 +3724,7 @@ unsigned int send_beacon(struct adapter *padapter)
 void site_survey(struct adapter *padapter)
 {
 	unsigned char survey_channel = 0, val8;
-	enum rt_scan_type	ScanType = SCAN_PASSIVE;
+	enum rt_scan_type	scan_type = SCAN_PASSIVE;
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
 	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 	u32 initialgain = 0;
@@ -3712,7 +3736,8 @@ void site_survey(struct adapter *padapter)
 		if (pmlmeext->sitesurvey_res.channel_idx < pmlmeext->sitesurvey_res.ch_num) {
 			ch = &pmlmeext->sitesurvey_res.ch[pmlmeext->sitesurvey_res.channel_idx];
 			survey_channel = ch->hw_value;
-			ScanType = (ch->flags & RTW_IEEE80211_CHAN_PASSIVE_SCAN) ? SCAN_PASSIVE : SCAN_ACTIVE;
+			scan_type = (ch->flags & RTW_IEEE80211_CHAN_PASSIVE_SCAN) ?
+				SCAN_PASSIVE : SCAN_ACTIVE;
 		}
 	}
 
@@ -3723,7 +3748,7 @@ void site_survey(struct adapter *padapter)
 		else
 			r8723bs_select_channel(padapter, survey_channel);
 
-		if (ScanType == SCAN_ACTIVE) { /* obey the channel plan setting... */
+		if (scan_type == SCAN_ACTIVE) { /* obey the channel plan setting... */
 			{
 				int i;
 
@@ -3877,14 +3902,12 @@ u8 collect_bss_info(struct adapter *padapter, union recv_frame *precv_frame, str
 
 	p = rtw_get_ie(bssid->ies + ie_offset, WLAN_EID_EXT_SUPP_RATES, &len, bssid->ie_length - ie_offset);
 	if (p) {
-		if (len > (NDIS_802_11_LENGTH_RATES_EX-i))
+		if (len > (NDIS_802_11_LENGTH_RATES_EX - i))
 			return _FAIL;
 
 		memcpy(bssid->supported_rates + i, (p + 2), len);
 	}
 
-	bssid->network_type_in_use = Ndis802_11OFDM24;
-
 	if (bssid->ie_length < 12)
 		return _FAIL;
 
@@ -3915,10 +3938,10 @@ u8 collect_bss_info(struct adapter *padapter, union recv_frame *precv_frame, str
 	val16 = rtw_get_capability((struct wlan_bssid_ex *)bssid);
 
 	if (val16 & BIT(0)) {
-		bssid->infrastructure_mode = Ndis802_11Infrastructure;
+		bssid->infrastructure_mode = NL80211_IFTYPE_STATION;
 		memcpy(bssid->mac_address, GetAddr2Ptr(pframe), ETH_ALEN);
 	} else {
-		bssid->infrastructure_mode = Ndis802_11IBSS;
+		bssid->infrastructure_mode = NL80211_IFTYPE_ADHOC;
 		memcpy(bssid->mac_address, GetAddr3Ptr(pframe), ETH_ALEN);
 	}
 
@@ -3970,7 +3993,7 @@ void start_create_ibss(struct adapter *padapter)
 	/* update capability */
 	caps = rtw_get_capability((struct wlan_bssid_ex *)pnetwork);
 	update_capinfo(padapter, caps);
-	if (caps&WLAN_CAPABILITY_IBSS) {/* adhoc master */
+	if (caps & WLAN_CAPABILITY_IBSS) {/* adhoc master */
 		val8 = 0xcf;
 		rtw_hal_set_hwreg(padapter, HW_VAR_SEC_CFG, (u8 *)(&val8));
 
@@ -4021,7 +4044,7 @@ void start_clnt_join(struct adapter *padapter)
 	/* update capability */
 	caps = rtw_get_capability((struct wlan_bssid_ex *)pnetwork);
 	update_capinfo(padapter, caps);
-	if (caps&WLAN_CAPABILITY_ESS) {
+	if (caps & WLAN_CAPABILITY_ESS) {
 		set_msr(padapter, WIFI_FW_STATION_STATE);
 
 		val8 = (pmlmeinfo->auth_algo == dot11AuthAlgrthm_8021X) ? 0xcc : 0xcf;
@@ -4045,10 +4068,10 @@ void start_clnt_join(struct adapter *padapter)
 		beacon_timeout = decide_wait_for_beacon_timeout(pmlmeinfo->bcn_interval);
 		set_link_timer(pmlmeext, beacon_timeout);
 		_set_timer(&padapter->mlmepriv.assoc_timer,
-			(REAUTH_TO * REAUTH_LIMIT) + (REASSOC_TO*REASSOC_LIMIT) + beacon_timeout);
+			(REAUTH_TO * REAUTH_LIMIT) + (REASSOC_TO * REASSOC_LIMIT) + beacon_timeout);
 
 		pmlmeinfo->state = WIFI_FW_AUTH_NULL | WIFI_FW_STATION_STATE;
-	} else if (caps&WLAN_CAPABILITY_IBSS) { /* adhoc client */
+	} else if (caps & WLAN_CAPABILITY_IBSS) { /* adhoc client */
 		set_msr(padapter, WIFI_FW_ADHOC_STATE);
 
 		val8 = 0xcf;
@@ -4110,7 +4133,7 @@ unsigned int receive_disconnect(struct adapter *padapter, unsigned char *MacAddr
 	if (!(!memcmp(MacAddr, get_my_bssid(&pmlmeinfo->network), ETH_ALEN)))
 		return _SUCCESS;
 
-	if ((pmlmeinfo->state&0x03) == WIFI_FW_STATION_STATE) {
+	if ((pmlmeinfo->state & 0x03) == WIFI_FW_STATION_STATE) {
 		if (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS) {
 			pmlmeinfo->state = WIFI_FW_NULL_STATE;
 			report_del_sta_event(padapter, MacAddr, reason);
@@ -4171,7 +4194,7 @@ static void process_80211d(struct adapter *padapter, struct wlan_bssid_ex *bssid
 				if (fcn <= 14)
 					channel = fcn + j; /*  2.4 GHz */
 				else
-					channel = fcn + j*4; /*  5 GHz */
+					channel = fcn + j * 4; /*  5 GHz */
 
 				chplan_ap.Channel[i++] = channel;
 			}
@@ -4187,27 +4210,27 @@ static void process_80211d(struct adapter *padapter, struct wlan_bssid_ex *bssid
 		if (pregistrypriv->wireless_mode & WIRELESS_11G) {
 			do {
 				if ((i == MAX_CHANNEL_NUM) ||
-					(chplan_sta[i].ChannelNum == 0) ||
-					(chplan_sta[i].ChannelNum > 14))
+					(chplan_sta[i].channel_num == 0) ||
+					(chplan_sta[i].channel_num > 14))
 					break;
 
 				if ((j == chplan_ap.Len) || (chplan_ap.Channel[j] > 14))
 					break;
 
-				if (chplan_sta[i].ChannelNum == chplan_ap.Channel[j]) {
-					chplan_new[k].ChannelNum = chplan_ap.Channel[j];
-					chplan_new[k].ScanType = SCAN_ACTIVE;
+				if (chplan_sta[i].channel_num == chplan_ap.Channel[j]) {
+					chplan_new[k].channel_num = chplan_ap.Channel[j];
+					chplan_new[k].scan_type = SCAN_ACTIVE;
 					i++;
 					j++;
 					k++;
-				} else if (chplan_sta[i].ChannelNum < chplan_ap.Channel[j]) {
-					chplan_new[k].ChannelNum = chplan_sta[i].ChannelNum;
-					chplan_new[k].ScanType = SCAN_PASSIVE;
+				} else if (chplan_sta[i].channel_num < chplan_ap.Channel[j]) {
+					chplan_new[k].channel_num = chplan_sta[i].channel_num;
+					chplan_new[k].scan_type = SCAN_PASSIVE;
 					i++;
 					k++;
-				} else if (chplan_sta[i].ChannelNum > chplan_ap.Channel[j]) {
-					chplan_new[k].ChannelNum = chplan_ap.Channel[j];
-					chplan_new[k].ScanType = SCAN_ACTIVE;
+				} else if (chplan_sta[i].channel_num > chplan_ap.Channel[j]) {
+					chplan_new[k].channel_num = chplan_ap.Channel[j];
+					chplan_new[k].scan_type = SCAN_ACTIVE;
 					j++;
 					k++;
 				}
@@ -4215,28 +4238,28 @@ static void process_80211d(struct adapter *padapter, struct wlan_bssid_ex *bssid
 
 			/*  change AP not support channel to Passive scan */
 			while ((i < MAX_CHANNEL_NUM) &&
-				(chplan_sta[i].ChannelNum != 0) &&
-				(chplan_sta[i].ChannelNum <= 14)) {
-				chplan_new[k].ChannelNum = chplan_sta[i].ChannelNum;
-				chplan_new[k].ScanType = SCAN_PASSIVE;
+				(chplan_sta[i].channel_num != 0) &&
+				(chplan_sta[i].channel_num <= 14)) {
+				chplan_new[k].channel_num = chplan_sta[i].channel_num;
+				chplan_new[k].scan_type = SCAN_PASSIVE;
 				i++;
 				k++;
 			}
 
 			/*  add channel AP supported */
 			while ((j < chplan_ap.Len) && (chplan_ap.Channel[j] <= 14)) {
-				chplan_new[k].ChannelNum = chplan_ap.Channel[j];
-				chplan_new[k].ScanType = SCAN_ACTIVE;
+				chplan_new[k].channel_num = chplan_ap.Channel[j];
+				chplan_new[k].scan_type = SCAN_ACTIVE;
 				j++;
 				k++;
 			}
 		} else {
 			/*  keep original STA 2.4G channel plan */
 			while ((i < MAX_CHANNEL_NUM) &&
-				(chplan_sta[i].ChannelNum != 0) &&
-				(chplan_sta[i].ChannelNum <= 14)) {
-				chplan_new[k].ChannelNum = chplan_sta[i].ChannelNum;
-				chplan_new[k].ScanType = chplan_sta[i].ScanType;
+				(chplan_sta[i].channel_num != 0) &&
+				(chplan_sta[i].channel_num <= 14)) {
+				chplan_new[k].channel_num = chplan_sta[i].channel_num;
+				chplan_new[k].scan_type = chplan_sta[i].scan_type;
 				i++;
 				k++;
 			}
@@ -4253,10 +4276,10 @@ static void process_80211d(struct adapter *padapter, struct wlan_bssid_ex *bssid
 	channel = bssid->configuration.ds_config;
 	chplan_new = pmlmeext->channel_set;
 	i = 0;
-	while ((i < MAX_CHANNEL_NUM) && (chplan_new[i].ChannelNum != 0)) {
-		if (chplan_new[i].ChannelNum == channel) {
-			if (chplan_new[i].ScanType == SCAN_PASSIVE)
-				chplan_new[i].ScanType = SCAN_ACTIVE;
+	while ((i < MAX_CHANNEL_NUM) && (chplan_new[i].channel_num != 0)) {
+		if (chplan_new[i].channel_num == channel) {
+			if (chplan_new[i].scan_type == SCAN_PASSIVE)
+				chplan_new[i].scan_type = SCAN_ACTIVE;
 			break;
 		}
 		i++;
@@ -4296,7 +4319,7 @@ void report_survey_event(struct adapter *padapter, union recv_frame *precv_frame
 
 	INIT_LIST_HEAD(&pcmd_obj->list);
 
-	pcmd_obj->cmdcode = GEN_CMD_CODE(_Set_MLME_EVT);
+	pcmd_obj->cmdcode = SET_MLME_EVT_CMD;
 	pcmd_obj->cmdsz = cmdsz;
 	pcmd_obj->parmbuf = pevtcmd;
 
@@ -4305,7 +4328,7 @@ void report_survey_event(struct adapter *padapter, union recv_frame *precv_frame
 
 	pc2h_evt_hdr = (struct C2HEvent_Header *)(pevtcmd);
 	pc2h_evt_hdr->len = sizeof(struct survey_event);
-	pc2h_evt_hdr->ID = GEN_EVT_CODE(_Survey);
+	pc2h_evt_hdr->ID = SURVEY_EVENT;
 	pc2h_evt_hdr->seq = atomic_inc_return(&pmlmeext->event_seq);
 
 	psurvey_evt = (struct survey_event *)(pevtcmd + sizeof(struct C2HEvent_Header));
@@ -4346,7 +4369,7 @@ void report_surveydone_event(struct adapter *padapter)
 
 	INIT_LIST_HEAD(&pcmd_obj->list);
 
-	pcmd_obj->cmdcode = GEN_CMD_CODE(_Set_MLME_EVT);
+	pcmd_obj->cmdcode = SET_MLME_EVT_CMD;
 	pcmd_obj->cmdsz = cmdsz;
 	pcmd_obj->parmbuf = pevtcmd;
 
@@ -4355,7 +4378,7 @@ void report_surveydone_event(struct adapter *padapter)
 
 	pc2h_evt_hdr = (struct C2HEvent_Header *)(pevtcmd);
 	pc2h_evt_hdr->len = sizeof(struct surveydone_event);
-	pc2h_evt_hdr->ID = GEN_EVT_CODE(_SurveyDone);
+	pc2h_evt_hdr->ID = SURVEY_DONE_EVENT;
 	pc2h_evt_hdr->seq = atomic_inc_return(&pmlmeext->event_seq);
 
 	psurveydone_evt = (struct surveydone_event *)(pevtcmd + sizeof(struct C2HEvent_Header));
@@ -4388,7 +4411,7 @@ void report_join_res(struct adapter *padapter, int res)
 
 	INIT_LIST_HEAD(&pcmd_obj->list);
 
-	pcmd_obj->cmdcode = GEN_CMD_CODE(_Set_MLME_EVT);
+	pcmd_obj->cmdcode = SET_MLME_EVT_CMD;
 	pcmd_obj->cmdsz = cmdsz;
 	pcmd_obj->parmbuf = pevtcmd;
 
@@ -4397,7 +4420,7 @@ void report_join_res(struct adapter *padapter, int res)
 
 	pc2h_evt_hdr = (struct C2HEvent_Header *)(pevtcmd);
 	pc2h_evt_hdr->len = sizeof(struct joinbss_event);
-	pc2h_evt_hdr->ID = GEN_EVT_CODE(_JoinBss);
+	pc2h_evt_hdr->ID = JOIN_BSS_EVENT;
 	pc2h_evt_hdr->seq = atomic_inc_return(&pmlmeext->event_seq);
 
 	pjoinbss_evt = (struct joinbss_event *)(pevtcmd + sizeof(struct C2HEvent_Header));
@@ -4432,7 +4455,7 @@ void report_wmm_edca_update(struct adapter *padapter)
 
 	INIT_LIST_HEAD(&pcmd_obj->list);
 
-	pcmd_obj->cmdcode = GEN_CMD_CODE(_Set_MLME_EVT);
+	pcmd_obj->cmdcode = SET_MLME_EVT_CMD;
 	pcmd_obj->cmdsz = cmdsz;
 	pcmd_obj->parmbuf = pevtcmd;
 
@@ -4441,7 +4464,7 @@ void report_wmm_edca_update(struct adapter *padapter)
 
 	pc2h_evt_hdr = (struct C2HEvent_Header *)(pevtcmd);
 	pc2h_evt_hdr->len = sizeof(struct wmm_event);
-	pc2h_evt_hdr->ID = GEN_EVT_CODE(_WMM);
+	pc2h_evt_hdr->ID = WMM_EVENT;
 	pc2h_evt_hdr->seq = atomic_inc_return(&pmlmeext->event_seq);
 
 	pwmm_event = (struct wmm_event *)(pevtcmd + sizeof(struct C2HEvent_Header));
@@ -4475,7 +4498,7 @@ void report_del_sta_event(struct adapter *padapter, unsigned char *MacAddr, unsi
 
 	INIT_LIST_HEAD(&pcmd_obj->list);
 
-	pcmd_obj->cmdcode = GEN_CMD_CODE(_Set_MLME_EVT);
+	pcmd_obj->cmdcode = SET_MLME_EVT_CMD;
 	pcmd_obj->cmdsz = cmdsz;
 	pcmd_obj->parmbuf = pevtcmd;
 
@@ -4484,7 +4507,7 @@ void report_del_sta_event(struct adapter *padapter, unsigned char *MacAddr, unsi
 
 	pc2h_evt_hdr = (struct C2HEvent_Header *)(pevtcmd);
 	pc2h_evt_hdr->len = sizeof(struct stadel_event);
-	pc2h_evt_hdr->ID = GEN_EVT_CODE(_DelSTA);
+	pc2h_evt_hdr->ID = DEL_STA_EVENT;
 	pc2h_evt_hdr->seq = atomic_inc_return(&pmlmeext->event_seq);
 
 	pdel_sta_evt = (struct stadel_event *)(pevtcmd + sizeof(struct C2HEvent_Header));
@@ -4525,7 +4548,7 @@ void report_add_sta_event(struct adapter *padapter, unsigned char *MacAddr, int
 
 	INIT_LIST_HEAD(&pcmd_obj->list);
 
-	pcmd_obj->cmdcode = GEN_CMD_CODE(_Set_MLME_EVT);
+	pcmd_obj->cmdcode = SET_MLME_EVT_CMD;
 	pcmd_obj->cmdsz = cmdsz;
 	pcmd_obj->parmbuf = pevtcmd;
 
@@ -4534,7 +4557,7 @@ void report_add_sta_event(struct adapter *padapter, unsigned char *MacAddr, int
 
 	pc2h_evt_hdr = (struct C2HEvent_Header *)(pevtcmd);
 	pc2h_evt_hdr->len = sizeof(struct stassoc_event);
-	pc2h_evt_hdr->ID = GEN_EVT_CODE(_AddSTA);
+	pc2h_evt_hdr->ID = ADD_STA_EVENT;
 	pc2h_evt_hdr->seq = atomic_inc_return(&pmlmeext->event_seq);
 
 	padd_sta_evt = (struct stassoc_event *)(pevtcmd + sizeof(struct C2HEvent_Header));
@@ -4562,7 +4585,9 @@ void update_sta_info(struct adapter *padapter, struct sta_info *psta)
 
 		psta->htpriv.ampdu_enable = pmlmepriv->htpriv.ampdu_enable;
 
-		psta->htpriv.rx_ampdu_min_spacing = (pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para&IEEE80211_HT_CAP_AMPDU_DENSITY)>>2;
+		psta->htpriv.rx_ampdu_min_spacing =
+			(pmlmeinfo->HT_caps.u.HT_cap_element.AMPDU_para &
+			 IEEE80211_HT_CAP_AMPDU_DENSITY) >> 2;
 
 		if (support_short_GI(padapter, &(pmlmeinfo->HT_caps), CHANNEL_WIDTH_20))
 			psta->htpriv.sgi_20m = true;
@@ -4667,7 +4692,7 @@ void mlmeext_joinbss_event_callback(struct adapter *padapter, int join_res)
 		return;
 	}
 
-	if ((pmlmeinfo->state&0x03) == WIFI_FW_ADHOC_STATE)
+	if ((pmlmeinfo->state & 0x03) == WIFI_FW_ADHOC_STATE)
 		/* update bc/mc sta_info */
 		update_bmc_sta(padapter);
 
@@ -4716,7 +4741,7 @@ void mlmeext_joinbss_event_callback(struct adapter *padapter, int join_res)
 	join_type = 2;
 	rtw_hal_set_hwreg(padapter, HW_VAR_MLME_JOIN, (u8 *)(&join_type));
 
-	if ((pmlmeinfo->state&0x03) == WIFI_FW_STATION_STATE) {
+	if ((pmlmeinfo->state & 0x03) == WIFI_FW_STATION_STATE) {
 		/*  correcting TSF */
 		correct_TSF(padapter, pmlmeext);
 
@@ -4733,7 +4758,7 @@ void mlmeext_sta_add_event_callback(struct adapter *padapter, struct sta_info *p
 	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 	u8 join_type;
 
-	if ((pmlmeinfo->state&0x03) == WIFI_FW_ADHOC_STATE) {
+	if ((pmlmeinfo->state & 0x03) == WIFI_FW_ADHOC_STATE) {
 		if (pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS) { /* adhoc master or sta_count>1 */
 
 			/* nothing to do */
@@ -4795,7 +4820,7 @@ void _linked_info_dump(struct adapter *padapter)
 	struct dvobj_priv *pdvobj = adapter_to_dvobj(padapter);
 
 	if (padapter->bLinkInfoDump) {
-		if ((pmlmeinfo->state&0x03) == WIFI_FW_STATION_STATE)
+		if ((pmlmeinfo->state & 0x03) == WIFI_FW_STATION_STATE)
 			rtw_hal_get_def_var(padapter,
 					    HAL_DEF_UNDERCORATEDSMOOTHEDPWDB,
 					    &UndecoratedSmoothedPWDB);
@@ -4969,7 +4994,7 @@ void survey_timer_hdl(struct timer_list *t)
 			return;
 		}
 
-		init_h2fwcmd_w_parm_no_rsp(ph2c, psurveyPara, GEN_CMD_CODE(_SiteSurvey));
+		init_h2fwcmd_w_parm_no_rsp(ph2c, psurveyPara, SITE_SURVEY_CMD);
 		rtw_enqueue_cmd(pcmdpriv, ph2c);
 	}
 }
@@ -5053,15 +5078,15 @@ u8 setopmode_hdl(struct adapter *padapter, u8 *pbuf)
 	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 	struct setopmode_parm *psetop = (struct setopmode_parm *)pbuf;
 
-	if (psetop->mode == Ndis802_11APMode) {
+	if (psetop->mode == NL80211_IFTYPE_AP) {
 		pmlmeinfo->state = WIFI_FW_AP_STATE;
 		type = _HW_STATE_AP_;
 		/* start_ap_mode(padapter); */
-	} else if (psetop->mode == Ndis802_11Infrastructure) {
+	} else if (psetop->mode == NL80211_IFTYPE_STATION) {
 		pmlmeinfo->state &= ~(BIT(0) | BIT(1));/*  clear state */
 		pmlmeinfo->state |= WIFI_FW_STATION_STATE;/* set to	STATION_STATE */
 		type = _HW_STATE_STATION_;
-	} else if (psetop->mode == Ndis802_11IBSS) {
+	} else if (psetop->mode == NL80211_IFTYPE_ADHOC) {
 		type = _HW_STATE_ADHOC_;
 	} else {
 		type = _HW_STATE_NOLINK_;
@@ -5070,7 +5095,7 @@ u8 setopmode_hdl(struct adapter *padapter, u8 *pbuf)
 	rtw_hal_set_hwreg(padapter, HW_VAR_SET_OPMODE, (u8 *)(&type));
 	/* set_msr(padapter, type); */
 
-	if (psetop->mode == Ndis802_11APMode) {
+	if (psetop->mode == NL80211_IFTYPE_AP) {
 		/*  Do this after port switch to */
 		/*  prevent from downloading rsvd page to wrong port */
 		rtw_btcoex_media_status_notify(padapter, 1); /* connect */
@@ -5093,7 +5118,7 @@ u8 createbss_hdl(struct adapter *padapter, u8 *pbuf)
 	}
 
 	/* below is for ad-hoc master */
-	if (pparm->network.infrastructure_mode == Ndis802_11IBSS) {
+	if (pparm->network.infrastructure_mode == NL80211_IFTYPE_ADHOC) {
 		rtw_joinbss_reset(padapter);
 
 		pmlmeext->cur_bwmode = CHANNEL_WIDTH_20;
@@ -5289,9 +5314,11 @@ u8 disconnect_hdl(struct adapter *padapter, unsigned char *pbuf)
 	u8 val8;
 
 	if (is_client_associated_to_ap(padapter))
-		issue_deauth_ex(padapter, pnetwork->mac_address, WLAN_REASON_DEAUTH_LEAVING, param->deauth_timeout_ms/100, 100);
+		issue_deauth_ex(padapter, pnetwork->mac_address, WLAN_REASON_DEAUTH_LEAVING,
+				param->deauth_timeout_ms / 100, 100);
 
-	if (((pmlmeinfo->state&0x03) == WIFI_FW_ADHOC_STATE) || ((pmlmeinfo->state&0x03) == WIFI_FW_AP_STATE)) {
+	if (((pmlmeinfo->state & 0x03) == WIFI_FW_ADHOC_STATE) ||
+	    ((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE)) {
 		/* Stop BCN */
 		val8 = 0;
 		rtw_hal_set_hwreg(padapter, HW_VAR_BCN_FUNC, (u8 *)(&val8));
@@ -5312,7 +5339,7 @@ static int rtw_scan_ch_decision(struct adapter *padapter, struct rtw_ieee80211_c
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
 
 	/* clear first */
-	memset(out, 0, sizeof(struct rtw_ieee80211_channel)*out_num);
+	memset(out, 0, sizeof(struct rtw_ieee80211_channel) * out_num);
 
 	/* acquire channels from in */
 	j = 0;
@@ -5330,7 +5357,7 @@ static int rtw_scan_ch_decision(struct adapter *padapter, struct rtw_ieee80211_c
 
 			memcpy(&out[j], &in[i], sizeof(struct rtw_ieee80211_channel));
 
-			if (pmlmeext->channel_set[set_idx].ScanType == SCAN_PASSIVE)
+			if (pmlmeext->channel_set[set_idx].scan_type == SCAN_PASSIVE)
 				out[j].flags |= RTW_IEEE80211_CHAN_PASSIVE_SCAN;
 
 			j++;
@@ -5350,9 +5377,9 @@ static int rtw_scan_ch_decision(struct adapter *padapter, struct rtw_ieee80211_c
 				break;
 			}
 
-			out[j].hw_value = pmlmeext->channel_set[i].ChannelNum;
+			out[j].hw_value = pmlmeext->channel_set[i].channel_num;
 
-			if (pmlmeext->channel_set[i].ScanType == SCAN_PASSIVE)
+			if (pmlmeext->channel_set[i].scan_type == SCAN_PASSIVE)
 				out[j].flags |= RTW_IEEE80211_CHAN_PASSIVE_SCAN;
 
 			j++;
@@ -5551,7 +5578,7 @@ u8 add_ba_hdl(struct adapter *padapter, unsigned char *pbuf)
 		return	H2C_SUCCESS;
 
 	if (((pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS) && (pmlmeinfo->HT_enable)) ||
-		((pmlmeinfo->state&0x03) == WIFI_FW_AP_STATE)) {
+		((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE)) {
 		/* pmlmeinfo->ADDBA_retry_count = 0; */
 		/* pmlmeinfo->candidate_tid_bitmap |= (0x1 << pparm->tid); */
 		/* psta->htpriv.candidate_tid_bitmap |= BIT(pparm->tid); */
@@ -5576,7 +5603,7 @@ u8 chk_bmc_sleepq_cmd(struct adapter *padapter)
 		goto exit;
 	}
 
-	init_h2fwcmd_w_parm_no_parm_rsp(ph2c, GEN_CMD_CODE(_ChkBMCSleepq));
+	init_h2fwcmd_w_parm_no_parm_rsp(ph2c, CHK_BMC_SLEEPQ_CMD);
 
 	res = rtw_enqueue_cmd(pcmdpriv, ph2c);
 
@@ -5614,7 +5641,7 @@ u8 set_tx_beacon_cmd(struct adapter *padapter)
 				      pmlmeinfo->hidden_ssid_mode);
 	ptxBeacon_parm->network.ie_length += len_diff;
 
-	init_h2fwcmd_w_parm_no_rsp(ph2c, ptxBeacon_parm, GEN_CMD_CODE(_TX_Beacon));
+	init_h2fwcmd_w_parm_no_rsp(ph2c, ptxBeacon_parm, TX_BEACON_CMD);
 
 	res = rtw_enqueue_cmd(pcmdpriv, ph2c);
 
@@ -5623,34 +5650,32 @@ u8 set_tx_beacon_cmd(struct adapter *padapter)
 }
 
 static struct fwevent wlanevents[] = {
-	{0, rtw_dummy_event_callback},	/*0*/
-	{0, NULL},
-	{0, NULL},
-	{0, NULL},
-	{0, NULL},
-	{0, NULL},
-	{0, NULL},
-	{0, NULL},
-	{0, &rtw_survey_event_callback},		/*8*/
-	{sizeof(struct surveydone_event), &rtw_surveydone_event_callback},	/*9*/
-
-	{0, &rtw_joinbss_event_callback},		/*10*/
-	{sizeof(struct stassoc_event), &rtw_stassoc_event_callback},
-	{sizeof(struct stadel_event), &rtw_stadel_event_callback},
-	{0, &rtw_atimdone_event_callback},
-	{0, rtw_dummy_event_callback},
-	{0, NULL},	/*15*/
-	{0, NULL},
-	{0, NULL},
-	{0, NULL},
-	{0, rtw_fwdbg_event_callback},
-	{0, NULL},	 /*20*/
-	{0, NULL},
-	{0, NULL},
-	{0, &rtw_cpwm_event_callback},
-	{0, NULL},
-	{0, &rtw_wmm_event_callback},
-
+	{0, rtw_dummy_event_callback},						/*  0 */
+	{0, NULL},								/*  1 */
+	{0, NULL},								/*  2 */
+	{0, NULL},								/*  3 */
+	{0, NULL},								/*  4 */
+	{0, NULL},								/*  5 */
+	{0, NULL},								/*  6 */
+	{0, NULL},								/*  7 */
+	{0, rtw_survey_event_callback},						/*  8 */
+	{sizeof(struct surveydone_event), rtw_surveydone_event_callback},	/*  9 */
+	{0, rtw_joinbss_event_callback},					/* 10 */
+	{sizeof(struct stassoc_event), rtw_stassoc_event_callback},		/* 11 */
+	{sizeof(struct stadel_event), rtw_stadel_event_callback},		/* 12 */
+	{0, rtw_dummy_event_callback},						/* 13 */
+	{0, rtw_dummy_event_callback},						/* 14 */
+	{0, NULL},								/* 15 */
+	{0, NULL},								/* 16 */
+	{0, NULL},								/* 17 */
+	{0, NULL},								/* 18 */
+	{0, rtw_fwdbg_event_callback},						/* 19 */
+	{0, NULL},								/* 20 */
+	{0, NULL},								/* 21 */
+	{0, NULL},								/* 22 */
+	{0, rtw_cpwm_event_callback},						/* 23 */
+	{0, NULL},								/* 24 */
+	{0, rtw_wmm_event_callback},						/* 25 */
 };
 
 u8 mlme_evt_hdl(struct adapter *padapter, unsigned char *pbuf)
diff --git a/drivers/staging/rtl8723bs/core/rtw_pwrctrl.c b/drivers/staging/rtl8723bs/core/rtw_pwrctrl.c
index b9f8cf1014ede..ea005d2fae290 100644
--- a/drivers/staging/rtl8723bs/core/rtw_pwrctrl.c
+++ b/drivers/staging/rtl8723bs/core/rtw_pwrctrl.c
@@ -315,7 +315,7 @@ static u8 PS_RDY_CHECK(struct adapter *padapter)
 	)
 		return false;
 
-	if (padapter->securitypriv.dot11AuthAlgrthm == dot11AuthAlgrthm_8021X &&
+	if (padapter->securitypriv.dot11_auth_algrthm == dot11AuthAlgrthm_8021X &&
 	    !padapter->securitypriv.binstallGrpkey)
 		return false;
 
@@ -389,35 +389,29 @@ void rtw_set_ps_mode(struct adapter *padapter, u8 ps_mode,
 
 /*
  * Return:
- *0:	Leave OK
- *-1:	Timeout
- *-2:	Other error
+ * 0:           Leave OK
+ * -ETIMEDOUT:  Timeout
+ * -ENODEV:     Other error
  */
-s32 LPS_RF_ON_check(struct adapter *padapter, u32 delay_ms)
+int LPS_RF_ON_check(struct adapter *padapter, u32 delay_ms)
 {
 	unsigned long start_time;
 	u8 bAwake = false;
-	s32 err = 0;
 
 	start_time = jiffies;
 	while (1) {
 		rtw_hal_get_hwreg(padapter, HW_VAR_FWLPS_RF_ON, &bAwake);
 		if (bAwake)
-			break;
+			return 0;
 
-		if (padapter->bSurpriseRemoved) {
-			err = -2;
-			break;
-		}
+		if (padapter->bSurpriseRemoved)
+			return -ENODEV;
+
+		if (jiffies_to_msecs(jiffies - start_time) > delay_ms)
+			return -ETIMEDOUT;
 
-		if (jiffies_to_msecs(jiffies - start_time) > delay_ms) {
-			err = -1;
-			break;
-		}
 		msleep(1);
 	}
-
-	return err;
 }
 
 /* Description: Enter the leisure power save mode. */
diff --git a/drivers/staging/rtl8723bs/core/rtw_recv.c b/drivers/staging/rtl8723bs/core/rtw_recv.c
index 86c5e2c4e7ddd..7946acd8869b9 100644
--- a/drivers/staging/rtl8723bs/core/rtw_recv.c
+++ b/drivers/staging/rtl8723bs/core/rtw_recv.c
@@ -44,7 +44,8 @@ signed int _rtw_init_recv_priv(struct recv_priv *precvpriv, struct adapter *pada
 
 	precvpriv->free_recvframe_cnt = NR_RECVFRAME;
 
-	precvpriv->pallocated_frame_buf = vzalloc(NR_RECVFRAME * sizeof(union recv_frame) + RXFRAME_ALIGN_SZ);
+	precvpriv->pallocated_frame_buf = vzalloc(NR_RECVFRAME * sizeof(union recv_frame)
+						 + RXFRAME_ALIGN_SZ);
 
 	if (!precvpriv->pallocated_frame_buf) {
 		res = _FAIL;
@@ -55,7 +56,7 @@ signed int _rtw_init_recv_priv(struct recv_priv *precvpriv, struct adapter *pada
 	/* precvpriv->precv_frame_buf = precvpriv->pallocated_frame_buf + RXFRAME_ALIGN_SZ - */
 	/* ((SIZE_PTR) (precvpriv->pallocated_frame_buf) &(RXFRAME_ALIGN_SZ-1)); */
 
-	precvframe = (union recv_frame *) precvpriv->precv_frame_buf;
+	precvframe = (union recv_frame *)precvpriv->precv_frame_buf;
 
 	for (i = 0; i < NR_RECVFRAME; i++) {
 		INIT_LIST_HEAD(&(precvframe->u.list));
@@ -494,7 +495,7 @@ static union recv_frame *portctrl(struct adapter *adapter, union recv_frame *pre
 
 	pstapriv = &adapter->stapriv;
 
-	auth_alg = adapter->securitypriv.dot11AuthAlgrthm;
+	auth_alg = adapter->securitypriv.dot11_auth_algrthm;
 
 	ptr = precv_frame->u.hdr.rx_data;
 	pfhdr = &precv_frame->u.hdr;
@@ -815,7 +816,7 @@ static signed int ap2sta_data_frame(struct adapter *adapter, union recv_frame *p
 		}
 
 		if (GetFrameSubType(ptr) & BIT(6)) {
-			/* No data, will not indicate to upper layer, temporily count it here */
+			/* No data, will not indicate to upper layer, temporarily count it here */
 			count_rx_stats(adapter, precv_frame, *psta);
 			ret = RTW_RX_HANDLED;
 			goto exit;
@@ -895,7 +896,7 @@ static signed int sta2ap_data_frame(struct adapter *adapter, union recv_frame *p
 			process_wmmps_data(adapter, precv_frame);
 
 		if (GetFrameSubType(ptr) & BIT(6)) {
-			/* No data, will not indicate to upper layer, temporily count it here */
+			/* No data, will not indicate to upper layer, temporarily count it here */
 			count_rx_stats(adapter, precv_frame, *psta);
 			ret = RTW_RX_HANDLED;
 			goto exit;
@@ -1009,7 +1010,7 @@ static signed int validate_recv_ctrl_frame(struct adapter *padapter, union recv_
 
 					/* update BCN for TIM IE */
 					/* update_BCNTIM(padapter); */
-					update_beacon(padapter, WLAN_EID_TIM, NULL, true);
+					update_beacon(padapter, WLAN_EID_TIM, true);
 				}
 
 				/* spin_unlock_bh(&psta->sleep_q.lock); */
@@ -1031,7 +1032,7 @@ static signed int validate_recv_ctrl_frame(struct adapter *padapter, union recv_
 
 					/* update BCN for TIM IE */
 					/* update_BCNTIM(padapter); */
-					update_beacon(padapter, WLAN_EID_TIM, NULL, true);
+					update_beacon(padapter, WLAN_EID_TIM, true);
 				}
 			}
 		}
@@ -1447,7 +1448,6 @@ static signed int validate_recv_frame(struct adapter *adapter, union recv_frame
 	u8 type;
 	u8 subtype;
 	signed int retval = _SUCCESS;
-	u8 bDumpRxPkt;
 
 	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
 
@@ -1473,7 +1473,6 @@ static signed int validate_recv_frame(struct adapter *adapter, union recv_frame
 	pattrib->mdata = GetMData(ptr);
 	pattrib->privacy = GetPrivacy(ptr);
 	pattrib->order = GetOrder(ptr);
-	rtw_hal_get_def_var(adapter, HAL_DEF_DBG_DUMP_RXPKT, &(bDumpRxPkt));
 
 	switch (type) {
 	case WIFI_MGT_TYPE: /* mgnt */
@@ -1576,7 +1575,7 @@ static signed int wlanhdr_to_ethhdr(union recv_frame *precvframe)
 	return _SUCCESS;
 }
 
-static struct sk_buff *rtw_alloc_msdu_pkt(union recv_frame *prframe, u16 nSubframe_Length, u8 *pdata)
+static struct sk_buff *rtw_alloc_msdu_pkt(union recv_frame *prframe, u16 subframe_len, u8 *pdata)
 {
 	u16 eth_type;
 	struct sk_buff *sub_skb;
@@ -1584,12 +1583,12 @@ static struct sk_buff *rtw_alloc_msdu_pkt(union recv_frame *prframe, u16 nSubfra
 
 	pattrib = &prframe->u.hdr.attrib;
 
-	sub_skb = __dev_alloc_skb(nSubframe_Length + 12, GFP_ATOMIC);
+	sub_skb = __dev_alloc_skb(subframe_len + 12, GFP_ATOMIC);
 	if (!sub_skb)
 		return NULL;
 
 	skb_reserve(sub_skb, 12);
-	skb_put_data(sub_skb, (pdata + ETH_HLEN), nSubframe_Length);
+	skb_put_data(sub_skb, (pdata + ETH_HLEN), subframe_len);
 
 	eth_type = get_unaligned_be16(&sub_skb->data[6]);
 
@@ -1666,7 +1665,7 @@ static void rtw_recv_indicate_pkt(struct adapter *padapter, struct sk_buff *pkt,
 static int amsdu_to_msdu(struct adapter *padapter, union recv_frame *prframe)
 {
 	int	a_len, padding_len;
-	u16 nSubframe_Length;
+	u16 subframe_len;
 	u8 nr_subframes, i;
 	u8 *pdata;
 	struct sk_buff *sub_pkt, *subframes[MAX_SUBFRAME_COUNT];
@@ -1686,12 +1685,12 @@ static int amsdu_to_msdu(struct adapter *padapter, union recv_frame *prframe)
 
 	while (a_len > ETH_HLEN) {
 		/* Offset 12 denote 2 mac address */
-		nSubframe_Length = get_unaligned_be16(pdata + 12);
+		subframe_len = get_unaligned_be16(pdata + 12);
 
-		if (a_len < ETH_HLEN + nSubframe_Length)
+		if (a_len < ETH_HLEN + subframe_len)
 			break;
 
-		sub_pkt = rtw_alloc_msdu_pkt(prframe, nSubframe_Length, pdata);
+		sub_pkt = rtw_alloc_msdu_pkt(prframe, subframe_len, pdata);
 		if (!sub_pkt)
 			break;
 
@@ -1704,10 +1703,10 @@ static int amsdu_to_msdu(struct adapter *padapter, union recv_frame *prframe)
 		if (nr_subframes >= MAX_SUBFRAME_COUNT)
 			break;
 
-		pdata += nSubframe_Length;
-		a_len -= nSubframe_Length;
+		pdata += subframe_len;
+		a_len -= subframe_len;
 		if (a_len != 0) {
-			padding_len = 4 - ((nSubframe_Length + ETH_HLEN) & (4 - 1));
+			padding_len = 4 - ((subframe_len + ETH_HLEN) & (4 - 1));
 			if (padding_len == 4)
 				padding_len = 0;
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c
index 0cfd7734c1749..29e2a5a771dd9 100644
--- a/drivers/staging/rtl8723bs/core/rtw_security.c
+++ b/drivers/staging/rtl8723bs/core/rtw_security.c
@@ -409,7 +409,6 @@ static void phase2(u8 *rc4key, const u8 *tk, const u16 *p1k, u16 iv16)
 	rc4key[2] = Lo8(iv16);
 	rc4key[3] = Lo8((PPK[5] ^ TK16(0)) >> 1);
 
-
 	/* Copy 96 bits of PPK[0..5] to RC4KEY[4..15]  (little-endian)       */
 	for (i = 0; i < 6; i++) {
 		rc4key[4 + 2 * i] = Lo8(PPK[i]);
@@ -417,7 +416,6 @@ static void phase2(u8 *rc4key, const u8 *tk, const u16 *p1k, u16 iv16)
 	}
 }
 
-
 /* The hlen isn't include the IV */
 u32 rtw_tkip_encrypt(struct adapter *padapter, u8 *pxmitframe)
 {																	/*  exclude ICV */
@@ -491,7 +489,6 @@ u32 rtw_tkip_encrypt(struct adapter *padapter, u8 *pxmitframe)
 	return res;
 }
 
-
 /* The hlen isn't include the IV */
 u32 rtw_tkip_decrypt(struct adapter *padapter, u8 *precvframe)
 {																	/*  exclude ICV */
@@ -593,11 +590,8 @@ u32 rtw_tkip_decrypt(struct adapter *padapter, u8 *precvframe)
 	return res;
 }
 
-
 /* 3			=====AES related ===== */
 
-
-
 #define MAX_MSG_SIZE	2048
 
 /****************************************/
@@ -626,7 +620,7 @@ static void construct_mic_iv(u8 *mic_iv,
 			     u8 *mpdu,
 			     uint payload_length,
 			     u8 *pn_vector,
-			     uint frtype) /* add for CONFIG_IEEE80211W, none 11w also can use */
+			     uint frtype)
 {
 		signed int i;
 
@@ -662,7 +656,7 @@ static void construct_mic_iv(u8 *mic_iv,
 static void construct_mic_header1(u8 *mic_header1,
 				  signed int header_length,
 				  u8 *mpdu,
-				  uint frtype) /* for CONFIG_IEEE80211W, none 11w also can use */
+				  uint frtype)
 {
 		mic_header1[0] = (u8)((header_length - 2) / 256);
 		mic_header1[1] = (u8)((header_length - 2) % 256);
@@ -745,7 +739,7 @@ static void construct_ctr_preload(u8 *ctr_preload,
 				  u8 *mpdu,
 				  u8 *pn_vector,
 				  signed int c,
-				  uint frtype) /* for CONFIG_IEEE80211W, none 11w also can use */
+				  uint frtype)
 {
 	signed int i = 0;
 
@@ -805,7 +799,7 @@ static signed int aes_cipher(u8 *key, uint	hdrlen,
 		if (hdrlen !=  WLAN_HDR_A3_QOS_LEN)
 			hdrlen += 2;
 
-	} else if ((frtype == WIFI_DATA) && /*  add for CONFIG_IEEE80211W, none 11w also can use */
+	} else if ((frtype == WIFI_DATA) &&
 		   ((frsubtype == 0x08) ||
 		   (frsubtype == 0x09) ||
 		   (frsubtype == 0x0a) ||
@@ -831,12 +825,12 @@ static signed int aes_cipher(u8 *key, uint	hdrlen,
 			 pframe,	 /* message, */
 			 plen,
 			 pn_vector,
-			 frtype); /*  add for CONFIG_IEEE80211W, none 11w also can use */
+			 frtype);
 
 	construct_mic_header1(mic_header1,
 			      hdrlen,
 			      pframe,	/* message */
-			      frtype); /*  add for CONFIG_IEEE80211W, none 11w also can use */
+			      frtype);
 
 	construct_mic_header2(mic_header2,
 			      pframe,	/* message, */
@@ -885,7 +879,6 @@ static signed int aes_cipher(u8 *key, uint	hdrlen,
 	for (i = 0; i < num_blocks; i++) {
 		construct_ctr_preload(ctr_preload, a4_exists, qc_exists, pframe, /* message, */
 				      pn_vector, i + 1, frtype);
-		/*  add for CONFIG_IEEE80211W, none 11w also can use */
 		aes128k128d(key, ctr_preload, aes_out);
 		crypto_xor_cpy(chain_buffer, aes_out, &pframe[payload_index], 16);
 		for (j = 0; j < 16; j++)
@@ -897,7 +890,6 @@ static signed int aes_cipher(u8 *key, uint	hdrlen,
 		/* encrypt it and copy the unpadded part back   */
 		construct_ctr_preload(ctr_preload, a4_exists, qc_exists, pframe, /* message, */
 				      pn_vector, num_blocks + 1, frtype);
-		/*  add for CONFIG_IEEE80211W, none 11w also can use */
 
 		for (j = 0; j < 16; j++)
 			padded_buffer[j] = 0x00;
@@ -913,7 +905,6 @@ static signed int aes_cipher(u8 *key, uint	hdrlen,
 	/* Encrypt the MIC */
 	construct_ctr_preload(ctr_preload, a4_exists, qc_exists, pframe, /* message, */
 			      pn_vector, 0, frtype);
-	/*  add for CONFIG_IEEE80211W, none 11w also can use */
 
 	for (j = 0; j < 16; j++)
 		padded_buffer[j] = 0x00;
@@ -1024,7 +1015,7 @@ static signed int aes_decipher(u8 *key, uint	hdrlen,
 		if (hdrlen !=  WLAN_HDR_A3_QOS_LEN)
 			hdrlen += 2;
 
-	} else if ((frtype == WIFI_DATA) && /* only for data packet . add for CONFIG_IEEE80211W, none 11w also can use */
+	} else if ((frtype == WIFI_DATA) &&
 		   ((frsubtype == 0x08) ||
 		   (frsubtype == 0x09) ||
 		   (frsubtype == 0x0a) ||
@@ -1045,7 +1036,7 @@ static signed int aes_decipher(u8 *key, uint	hdrlen,
 		construct_ctr_preload(ctr_preload, a4_exists,
 				      qc_exists, pframe,
 				      pn_vector, i + 1,
-				      frtype); /*  add for CONFIG_IEEE80211W, none 11w also can use */
+				      frtype);
 
 		aes128k128d(key, ctr_preload, aes_out);
 		crypto_xor_cpy(chain_buffer, aes_out, &pframe[payload_index], 16);
@@ -1059,7 +1050,6 @@ static signed int aes_decipher(u8 *key, uint	hdrlen,
 		/* encrypt it and copy the unpadded part back   */
 		construct_ctr_preload(ctr_preload, a4_exists, qc_exists, pframe, pn_vector,
 				      num_blocks + 1, frtype);
-		/*  add for CONFIG_IEEE80211W, none 11w also can use */
 
 		for (j = 0; j < 16; j++)
 			padded_buffer[j] = 0x00;
@@ -1084,10 +1074,8 @@ static signed int aes_decipher(u8 *key, uint	hdrlen,
 	pn_vector[5] = pframe[hdrlen + 7];
 
 	construct_mic_iv(mic_iv, qc_exists, a4_exists, message, plen - 8, pn_vector, frtype);
-	/*  add for CONFIG_IEEE80211W, none 11w also can use */
 
 	construct_mic_header1(mic_header1, hdrlen, message, frtype);
-	/*  add for CONFIG_IEEE80211W, none 11w also can use */
 	construct_mic_header2(mic_header2, message, a4_exists, qc_exists);
 
 	payload_remainder = (plen - 8) % 16;
@@ -1132,7 +1120,6 @@ static signed int aes_decipher(u8 *key, uint	hdrlen,
 	for (i = 0; i < num_blocks; i++) {
 		construct_ctr_preload(ctr_preload, a4_exists, qc_exists, message, pn_vector, i + 1,
 				      frtype);
-		/*  add for CONFIG_IEEE80211W, none 11w also can use */
 		aes128k128d(key, ctr_preload, aes_out);
 		crypto_xor_cpy(chain_buffer, aes_out, &message[payload_index], 16);
 		for (j = 0; j < 16; j++)
@@ -1144,7 +1131,6 @@ static signed int aes_decipher(u8 *key, uint	hdrlen,
 		/* encrypt it and copy the unpadded part back   */
 		construct_ctr_preload(ctr_preload, a4_exists, qc_exists, message, pn_vector,
 				      num_blocks + 1, frtype);
-		/*  add for CONFIG_IEEE80211W, none 11w also can use */
 
 		for (j = 0; j < 16; j++)
 			padded_buffer[j] = 0x00;
@@ -1159,7 +1145,6 @@ static signed int aes_decipher(u8 *key, uint	hdrlen,
 
 	/* Encrypt the MIC */
 	construct_ctr_preload(ctr_preload, a4_exists, qc_exists, message, pn_vector, 0, frtype);
-	/*  add for CONFIG_IEEE80211W, none 11w also can use */
 
 	for (j = 0; j < 16; j++)
 		padded_buffer[j] = 0x00;
@@ -1433,7 +1418,6 @@ static int omac1_aes_128_vector(u8 *key, size_t num_elem,
  * This is a mode for using block cipher (AES in this case) for authentication.
  * OMAC1 was standardized with the name CMAC by NIST in a Special Publication
  * (SP) 800-38B.
- * modify for CONFIG_IEEE80211W
  */
 int omac1_aes_128(u8 *key, u8 *data, size_t data_len, u8 *mac)
 {
@@ -1446,7 +1430,7 @@ void rtw_sec_restore_wep_key(struct adapter *adapter)
 	struct security_priv *securitypriv = &adapter->securitypriv;
 	signed int keyid;
 
-	if ((securitypriv->dot11PrivacyAlgrthm == _WEP40_) || (securitypriv->dot11PrivacyAlgrthm == _WEP104_)) {
+	if ((securitypriv->dot11_privacy_algrthm == _WEP40_) || (securitypriv->dot11_privacy_algrthm == _WEP104_)) {
 		for (keyid = 0; keyid < 4; keyid++) {
 			if (securitypriv->key_mask & BIT(keyid)) {
 				if (keyid == securitypriv->dot11PrivacyKeyIndex)
diff --git a/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c b/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c
index a1b7fe843979e..aea660f279595 100644
--- a/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c
+++ b/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c
@@ -248,7 +248,7 @@ struct	sta_info *rtw_alloc_stainfo(struct	sta_priv *pstapriv, u8 *hwaddr)
 
 		/* init recv timer */
 		timer_setup(&preorder_ctrl->reordering_ctrl_timer,
-				rtw_reordering_ctrl_timeout_handler, 0);
+			    rtw_reordering_ctrl_timeout_handler, 0);
 	}
 
 	/* init for DM */
@@ -256,7 +256,7 @@ struct	sta_info *rtw_alloc_stainfo(struct	sta_priv *pstapriv, u8 *hwaddr)
 	psta->rssi_stat.UndecoratedSmoothedCCK = (-1);
 
 	/* init for the sequence number of received management frame */
-	psta->RxMgmtFrameSeqNum = 0xffff;
+	psta->rx_mgmt_frame_seq_num = 0xffff;
 	spin_unlock_bh(&pstapriv->sta_hash_lock);
 	/* alloc mac id for non-bc/mc station, */
 	rtw_alloc_macid(pstapriv->padapter, psta);
diff --git a/drivers/staging/rtl8723bs/core/rtw_wlan_util.c b/drivers/staging/rtl8723bs/core/rtw_wlan_util.c
index a4de538722b56..fb9bc9507ce52 100644
--- a/drivers/staging/rtl8723bs/core/rtw_wlan_util.c
+++ b/drivers/staging/rtl8723bs/core/rtw_wlan_util.c
@@ -256,9 +256,8 @@ inline void rtw_set_oper_ch(struct adapter *adapter, u8 ch)
 {
 	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
 
-	if (dvobj->oper_channel != ch) {
+	if (dvobj->oper_channel != ch)
 		dvobj->on_oper_ch_time = jiffies;
-	}
 
 	dvobj->oper_channel = ch;
 }
@@ -351,7 +350,7 @@ int is_client_associated_to_ap(struct adapter *padapter)
 		return _FAIL;
 
 	pmlmeext = &padapter->mlmeextpriv;
-	pmlmeinfo = &(pmlmeext->mlmext_info);
+	pmlmeinfo = &pmlmeext->mlmext_info;
 
 	if ((pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS) && ((pmlmeinfo->state & 0x03) == WIFI_FW_STATION_STATE))
 		return true;
@@ -362,7 +361,7 @@ int is_client_associated_to_ap(struct adapter *padapter)
 int is_client_associated_to_ibss(struct adapter *padapter)
 {
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
 	if ((pmlmeinfo->state & WIFI_FW_ASSOC_SUCCESS) && ((pmlmeinfo->state & 0x03) == WIFI_FW_ADHOC_STATE))
 		return true;
@@ -374,7 +373,7 @@ int is_IBSS_empty(struct adapter *padapter)
 {
 	unsigned int i;
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
 	for (i = IBSS_START_MAC_ID; i < NUM_STA; i++) {
 		if (pmlmeinfo->FW_sta_info[i].status == 1)
@@ -477,7 +476,7 @@ void clear_cam_cache(struct adapter *adapter, u8 id)
 
 	spin_lock_bh(&cam_ctl->lock);
 
-	memset(&(dvobj->cam_cache[id]), 0, sizeof(struct cam_entry_cache));
+	memset(&dvobj->cam_cache[id], 0, sizeof(struct cam_entry_cache));
 
 	spin_unlock_bh(&cam_ctl->lock);
 }
@@ -629,7 +628,7 @@ int allocate_fw_sta_entry(struct adapter *padapter)
 {
 	unsigned int mac_id;
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
 	for (mac_id = IBSS_START_MAC_ID; mac_id < NUM_STA; mac_id++) {
 		if (pmlmeinfo->FW_sta_info[mac_id].status == 0) {
@@ -645,7 +644,7 @@ int allocate_fw_sta_entry(struct adapter *padapter)
 void flush_all_cam_entry(struct adapter *padapter)
 {
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
 	invalidate_cam_all(padapter);
 	/* clear default key related key search setting */
@@ -657,19 +656,19 @@ void flush_all_cam_entry(struct adapter *padapter)
 int WMM_param_handler(struct adapter *padapter, struct ndis_80211_var_ie *pIE)
 {
 	/* struct registry_priv *pregpriv = &padapter->registrypriv; */
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
 	if (pmlmepriv->qospriv.qos_option == 0) {
 		pmlmeinfo->WMM_enable = 0;
 		return false;
 	}
 
-	if (!memcmp(&(pmlmeinfo->WMM_param), (pIE->data + 6), sizeof(struct WMM_para_element)))
+	if (!memcmp(&pmlmeinfo->WMM_param, (pIE->data + 6), sizeof(struct WMM_para_element)))
 		return false;
 
-	memcpy(&(pmlmeinfo->WMM_param), (pIE->data + 6), sizeof(struct WMM_para_element));
+	memcpy(&pmlmeinfo->WMM_param, (pIE->data + 6), sizeof(struct WMM_para_element));
 
 	pmlmeinfo->WMM_enable = 1;
 	return true;
@@ -709,7 +708,7 @@ void WMMOnAssocRsp(struct adapter *padapter)
 	u32 acParm, i;
 	u32 edca[4], inx[4];
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
 	struct registry_priv *pregpriv = &padapter->registrypriv;
 
@@ -810,9 +809,9 @@ static void bwmode_update_check(struct adapter *padapter, struct ndis_80211_var_
 	unsigned char  new_bwmode;
 	unsigned char  new_ch_offset;
 	struct HT_info_element	 *pHT_info;
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 	struct registry_priv *pregistrypriv = &padapter->registrypriv;
 	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
 	u8 cbw40_enable = 0;
@@ -867,7 +866,7 @@ static void bwmode_update_check(struct adapter *padapter, struct ndis_80211_var_
 
 	if (true == pmlmeinfo->bwmode_updated) {
 		struct sta_info *psta;
-		struct wlan_bssid_ex	*cur_network = &(pmlmeinfo->network);
+		struct wlan_bssid_ex	*cur_network = &pmlmeinfo->network;
 		struct sta_priv *pstapriv = &padapter->stapriv;
 
 		/* set_channel_bwmode(padapter, pmlmeext->cur_channel, pmlmeext->cur_ch_offset, pmlmeext->cur_bwmode); */
@@ -897,7 +896,7 @@ void HT_caps_handler(struct adapter *padapter, struct ndis_80211_var_ie *pIE)
 	u8 max_AMPDU_len, min_MPDU_spacing;
 	u8 cur_ldpc_cap = 0, cur_stbc_cap = 0;
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
 
@@ -961,7 +960,7 @@ void HT_caps_handler(struct adapter *padapter, struct ndis_80211_var_ie *pIE)
 void HT_info_handler(struct adapter *padapter, struct ndis_80211_var_ie *pIE)
 {
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
 
@@ -975,7 +974,7 @@ void HT_info_handler(struct adapter *padapter, struct ndis_80211_var_ie *pIE)
 		return;
 
 	pmlmeinfo->HT_info_enable = 1;
-	memcpy(&(pmlmeinfo->HT_info), pIE->data, pIE->length);
+	memcpy(&pmlmeinfo->HT_info, pIE->data, pIE->length);
 }
 
 void HTOnAssocRsp(struct adapter *padapter)
@@ -984,7 +983,7 @@ void HTOnAssocRsp(struct adapter *padapter)
 	unsigned char min_MPDU_spacing;
 	/* struct registry_priv  *pregpriv = &padapter->registrypriv; */
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
 	if ((pmlmeinfo->HT_info_enable) && (pmlmeinfo->HT_caps_enable)) {
 		pmlmeinfo->HT_enable = 1;
@@ -1011,20 +1010,20 @@ void HTOnAssocRsp(struct adapter *padapter)
 void ERP_IE_handler(struct adapter *padapter, struct ndis_80211_var_ie *pIE)
 {
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
 	if (pIE->length > 1)
 		return;
 
 	pmlmeinfo->ERP_enable = 1;
-	memcpy(&(pmlmeinfo->ERP_IE), pIE->data, pIE->length);
+	memcpy(&pmlmeinfo->ERP_IE, pIE->data, pIE->length);
 }
 
 void VCS_update(struct adapter *padapter, struct sta_info *psta)
 {
 	struct registry_priv  *pregpriv = &padapter->registrypriv;
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
 	switch (pregpriv->vrtl_carrier_sense) {/* 0:off 1:on 2:auto */
 	case 0: /* off */
@@ -1079,7 +1078,7 @@ int rtw_check_bcn_info(struct adapter *Adapter, u8 *pframe, u32 packet_len)
 	unsigned int len;
 	unsigned char *p;
 	unsigned short	val16, subtype;
-	struct wlan_network *cur_network = &(Adapter->mlmepriv.cur_network);
+	struct wlan_network *cur_network = &Adapter->mlmepriv.cur_network;
 	/* u8 wpa_ie[255], rsn_ie[255]; */
 	u16 wpa_len = 0, rsn_len = 0;
 	u8 encryp_protocol = 0;
@@ -1301,8 +1300,8 @@ unsigned int is_ap_in_tkip(struct adapter *padapter)
 	u32 i;
 	struct ndis_80211_var_ie *pIE;
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
-	struct wlan_bssid_ex		*cur_network = &(pmlmeinfo->network);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	struct wlan_bssid_ex		*cur_network = &pmlmeinfo->network;
 
 	if (rtw_get_capability((struct wlan_bssid_ex *)cur_network) & WLAN_CAPABILITY_PRIVACY) {
 		for (i = sizeof(struct ndis_802_11_fix_ie); i < pmlmeinfo->network.ie_length;) {
@@ -1345,7 +1344,7 @@ int support_short_GI(struct adapter *padapter, struct HT_caps_element *pHT_caps,
 {
 	unsigned char bit_offset;
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
 	if (!(pmlmeinfo->HT_enable))
 		return _FAIL;
@@ -1451,7 +1450,7 @@ unsigned char check_assoc_AP(u8 *pframe, uint len)
 void update_IOT_info(struct adapter *padapter)
 {
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
 	switch (pmlmeinfo->assoc_AP_vendor) {
 	case HT_IOT_PEER_MARVELL:
@@ -1481,7 +1480,7 @@ void update_IOT_info(struct adapter *padapter)
 void update_capinfo(struct adapter *Adapter, u16 updateCap)
 {
 	struct mlme_ext_priv *pmlmeext = &Adapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 	bool		ShortPreamble;
 
 	/*  Check preamble mode, 2005.01.06, by rcnjko. */
@@ -1533,8 +1532,8 @@ void update_wireless_mode(struct adapter *padapter)
 	int network_type = 0;
 	u32 SIFS_Timer;
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
-	struct wlan_bssid_ex *cur_network = &(pmlmeinfo->network);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	struct wlan_bssid_ex *cur_network = &pmlmeinfo->network;
 	unsigned char *rate = cur_network->supported_rates;
 
 	if ((pmlmeinfo->HT_info_enable) && (pmlmeinfo->HT_caps_enable))
@@ -1557,7 +1556,7 @@ void update_wireless_mode(struct adapter *padapter)
 
 	SetHwReg8723BS(padapter, HW_VAR_RESP_SIFS,  (u8 *)&SIFS_Timer);
 
-	SetHwReg8723BS(padapter, HW_VAR_WIRELESS_MODE,  (u8 *)&(pmlmeext->cur_wireless_mode));
+	SetHwReg8723BS(padapter, HW_VAR_WIRELESS_MODE,  (u8 *)&pmlmeext->cur_wireless_mode);
 
 	if (pmlmeext->cur_wireless_mode & WIRELESS_11B)
 		update_mgnt_tx_rate(padapter, IEEE80211_CCK_RATE_1MB);
@@ -1582,8 +1581,8 @@ int update_sta_support_rate(struct adapter *padapter, u8 *pvar_ie, uint var_ie_l
 	unsigned int	ie_len;
 	struct ndis_80211_var_ie *pIE;
 	int	support_rate_num = 0;
-	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
 	pIE = (struct ndis_80211_var_ie *)rtw_get_ie(pvar_ie, WLAN_EID_SUPP_RATES, &ie_len, var_ie_len);
 	if (!pIE)
@@ -1609,7 +1608,7 @@ void process_addba_req(struct adapter *padapter, u8 *paddba_req, u8 *addr)
 	struct sta_priv *pstapriv = &padapter->stapriv;
 	struct ADDBA_request *preq = (struct ADDBA_request *)paddba_req;
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
 	psta = rtw_get_stainfo(pstapriv, addr);
 
@@ -1652,7 +1651,7 @@ void adaptive_early_32k(struct mlme_ext_priv *pmlmeext, u8 *pframe, uint len)
 	__le32 *pbuf;
 	u64 tsf = 0;
 	u32 delay_ms;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
 	pmlmeext->bcn_cnt++;
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_xmit.c b/drivers/staging/rtl8723bs/core/rtw_xmit.c
index 444966c0de7f3..9f11d010eff8c 100644
--- a/drivers/staging/rtl8723bs/core/rtw_xmit.c
+++ b/drivers/staging/rtl8723bs/core/rtw_xmit.c
@@ -419,7 +419,7 @@ static void update_attrib_vcs_info(struct adapter *padapter, struct xmit_frame *
 			/* IOT action */
 			if ((pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_ATHEROS) &&
 			    pattrib->ampdu_en &&
-			    (padapter->securitypriv.dot11PrivacyAlgrthm == _AES_)) {
+			    (padapter->securitypriv.dot11_privacy_algrthm == _AES_)) {
 				pattrib->vcs_mode = CTS_TO_SELF;
 				break;
 			}
@@ -505,9 +505,8 @@ static void update_attrib_phy_info(struct adapter *padapter, struct pkt_attrib *
 	pattrib->retry_ctrl = false;
 }
 
-static s32 update_attrib_sec_info(struct adapter *padapter, struct pkt_attrib *pattrib, struct sta_info *psta)
+static int update_attrib_sec_info(struct adapter *padapter, struct pkt_attrib *pattrib, struct sta_info *psta)
 {
-	signed int res = _SUCCESS;
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct security_priv *psecuritypriv = &padapter->securitypriv;
 	signed int bmcast = is_multicast_ether_addr(pattrib->ra);
@@ -519,14 +518,12 @@ static s32 update_attrib_sec_info(struct adapter *padapter, struct pkt_attrib *p
 	if (psta->ieee8021x_blocked) {
 		pattrib->encrypt = 0;
 
-		if ((pattrib->ether_type != 0x888e) && !check_fwstate(pmlmepriv, WIFI_MP_STATE)) {
-			res = _FAIL;
-			goto exit;
-		}
+		if ((pattrib->ether_type != 0x888e) && !check_fwstate(pmlmepriv, WIFI_MP_STATE))
+			return -EINVAL;
 	} else {
 		GET_ENCRY_ALGO(psecuritypriv, psta, pattrib->encrypt, bmcast);
 
-		switch (psecuritypriv->dot11AuthAlgrthm) {
+		switch (psecuritypriv->dot11_auth_algrthm) {
 		case dot11AuthAlgrthm_Open:
 		case dot11AuthAlgrthm_Shared:
 		case dot11AuthAlgrthm_Auto:
@@ -560,10 +557,8 @@ static s32 update_attrib_sec_info(struct adapter *padapter, struct pkt_attrib *p
 		pattrib->iv_len = 8;
 		pattrib->icv_len = 4;
 
-		if (psecuritypriv->busetkipkey == _FAIL) {
-			res = _FAIL;
-			goto exit;
-		}
+		if (psecuritypriv->busetkipkey == _FAIL)
+			return -EINVAL;
 
 		if (bmcast)
 			TKIP_IV(pattrib->iv, psta->dot11txpn, pattrib->key_idx);
@@ -601,9 +596,7 @@ static s32 update_attrib_sec_info(struct adapter *padapter, struct pkt_attrib *p
 	else
 		pattrib->bswenc = false;
 
-exit:
-
-	return res;
+	return 0;
 }
 
 u8 qos_acm(u8 acm_mask, u8 priority)
@@ -661,7 +654,7 @@ static int set_qos(struct pkt_file *ppktfile, struct pkt_attrib *pattrib)
 	return 0;
 }
 
-static s32 update_attrib(struct adapter *padapter, struct sk_buff *pkt, struct pkt_attrib *pattrib)
+static int update_attrib(struct adapter *padapter, struct sk_buff *pkt, struct pkt_attrib *pattrib)
 {
 	struct pkt_file pktfile;
 	struct sta_info *psta = NULL;
@@ -671,7 +664,6 @@ static s32 update_attrib(struct adapter *padapter, struct sk_buff *pkt, struct p
 	struct sta_priv *pstapriv = &padapter->stapriv;
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct qos_priv *pqospriv = &pmlmepriv->qospriv;
-	signed int res = _SUCCESS;
 	int ret;
 
 	_rtw_open_pktfile(pkt, &pktfile);
@@ -748,29 +740,25 @@ static s32 update_attrib(struct adapter *padapter, struct sk_buff *pkt, struct p
 		psta = rtw_get_bcmc_stainfo(padapter);
 	} else {
 		psta = rtw_get_stainfo(pstapriv, pattrib->ra);
-		if (!psta)	{ /*  if we cannot get psta => drop the pkt */
-			res = _FAIL;
-			goto exit;
-		} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE) && !(psta->state & _FW_LINKED)) {
-			res = _FAIL;
-			goto exit;
-		}
+		if (!psta)	/*  if we cannot get psta => drop the pkt */
+			return -EINVAL;
+		else if (check_fwstate(pmlmepriv, WIFI_AP_STATE) && !(psta->state & _FW_LINKED))
+			return -EINVAL;
 	}
 
 	if (!psta) {
 		/*  if we cannot get psta => drop the pkt */
-		res = _FAIL;
-		goto exit;
+		return -EINVAL;
 	}
 
 	if (!(psta->state & _FW_LINKED))
-		return _FAIL;
+		return -EINVAL;
 
 	spin_lock_bh(&psta->lock);
-	if (update_attrib_sec_info(padapter, pattrib, psta) == _FAIL) {
+	ret = update_attrib_sec_info(padapter, pattrib, psta);
+	if (ret) {
 		spin_unlock_bh(&psta->lock);
-		res = _FAIL;
-		goto exit;
+		return ret;
 	}
 
 	update_attrib_phy_info(padapter, pattrib, psta);
@@ -806,9 +794,7 @@ static s32 update_attrib(struct adapter *padapter, struct sk_buff *pkt, struct p
 	}
 
 	/* pattrib->priority = 5; force to used VI queue, for testing */
-
-exit:
-	return res;
+	return 0;
 }
 
 static s32 xmitframe_addmic(struct adapter *padapter, struct xmit_frame *pxmitframe)
@@ -1969,7 +1955,7 @@ s32 rtw_xmit(struct adapter *padapter, struct sk_buff **ppkt)
 	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
 	struct xmit_frame *pxmitframe = NULL;
 
-	s32 res;
+	int ret;
 
 	if (start == 0)
 		start = jiffies;
@@ -1982,9 +1968,8 @@ s32 rtw_xmit(struct adapter *padapter, struct sk_buff **ppkt)
 	if (!pxmitframe)
 		return -1;
 
-	res = update_attrib(padapter, *ppkt, &pxmitframe->attrib);
-
-	if (res != _SUCCESS) {
+	ret = update_attrib(padapter, *ppkt, &pxmitframe->attrib);
+	if (ret) {
 		rtw_free_xmitframe(pxmitpriv, pxmitframe);
 		return -1;
 	}
@@ -2083,7 +2068,7 @@ signed int xmitframe_enqueue_for_sleeping_sta(struct adapter *padapter, struct x
 			pstapriv->sta_dz_bitmap |= BIT(0);
 
 			if (update_tim)
-				update_beacon(padapter, WLAN_EID_TIM, NULL, true);
+				update_beacon(padapter, WLAN_EID_TIM, true);
 			else
 				chk_bmc_sleepq_cmd(padapter);
 
@@ -2138,7 +2123,7 @@ signed int xmitframe_enqueue_for_sleeping_sta(struct adapter *padapter, struct x
 
 				if (update_tim)
 					/* update BCN for TIM IE */
-					update_beacon(padapter, WLAN_EID_TIM, NULL, true);
+					update_beacon(padapter, WLAN_EID_TIM, true);
 			}
 
 			ret = true;
@@ -2150,7 +2135,9 @@ signed int xmitframe_enqueue_for_sleeping_sta(struct adapter *padapter, struct x
 	return ret;
 }
 
-static void dequeue_xmitframes_to_sleeping_queue(struct adapter *padapter, struct sta_info *psta, struct __queue *pframequeue)
+static void dequeue_xmitframes_to_sleeping_queue(struct adapter *padapter,
+						 struct sta_info *psta,
+						 struct __queue *pframequeue)
 {
 	signed int ret;
 	struct list_head *plist, *phead, *tmp;
@@ -2333,7 +2320,7 @@ void wakeup_sta_to_xmit(struct adapter *padapter, struct sta_info *psta)
 	spin_unlock_bh(&pxmitpriv->lock);
 
 	if (update_mask)
-		update_beacon(padapter, WLAN_EID_TIM, NULL, true);
+		update_beacon(padapter, WLAN_EID_TIM, true);
 }
 
 void xmit_delivery_enabled_frames(struct adapter *padapter, struct sta_info *psta)
@@ -2393,7 +2380,7 @@ void xmit_delivery_enabled_frames(struct adapter *padapter, struct sta_info *pst
 		if ((psta->sleepq_ac_len == 0) && (!psta->has_legacy_ac) && (wmmps_ac)) {
 			pstapriv->tim_bitmap &= ~BIT(psta->aid);
 
-			update_beacon(padapter, WLAN_EID_TIM, NULL, true);
+			update_beacon(padapter, WLAN_EID_TIM, true);
 		}
 	}
 
diff --git a/drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.c b/drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.c
index 6da6940c90439..0220bacee181a 100644
--- a/drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.c
+++ b/drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.c
@@ -168,7 +168,7 @@ static void halbtc8723b2ant_LimitedRx(
 	u8 rxAggSize = aggBufSize;
 
 	/*  */
-	/* 	Rx Aggregation related setting */
+	/*	Rx Aggregation related setting */
 	/*  */
 	pBtCoexist->fBtcSet(pBtCoexist, BTC_SET_BL_TO_REJ_AP_AGG_PKT, &bRejectRxAgg);
 	/*  decide BT control aggregation buf size or not */
@@ -181,7 +181,7 @@ static void halbtc8723b2ant_LimitedRx(
 
 static void halbtc8723b2ant_QueryBtInfo(struct btc_coexist *pBtCoexist)
 {
-	u8 	H2C_Parameter[1] = {0};
+	u8	H2C_Parameter[1] = {0};
 
 	pCoexSta->bC2hBtInfoReqSent = true;
 
@@ -417,7 +417,7 @@ static void halbtc8723b2ant_SetFwDacSwingLevel(
 	struct btc_coexist *pBtCoexist, u8 dacSwingLvl
 )
 {
-	u8 	H2C_Parameter[1] = {0};
+	u8	H2C_Parameter[1] = {0};
 
 	/*  There are several type of dacswing */
 	/*  0x18/ 0x10/ 0xc/ 0x8/ 0x4/ 0x6 */
@@ -430,7 +430,7 @@ static void halbtc8723b2ant_SetFwDecBtPwr(
 	struct btc_coexist *pBtCoexist, u8 decBtPwrLvl
 )
 {
-	u8 	H2C_Parameter[1] = {0};
+	u8	H2C_Parameter[1] = {0};
 
 	H2C_Parameter[0] = decBtPwrLvl;
 
@@ -504,7 +504,7 @@ static void halbtc8723b2ant_SetSwPenaltyTxRateAdaptive(
 	struct btc_coexist *pBtCoexist, bool bLowPenaltyRa
 )
 {
-	u8 	H2C_Parameter[6] = {0};
+	u8	H2C_Parameter[6] = {0};
 
 	H2C_Parameter[0] = 0x6;	/*  opCode, 0x6 = Retry_Penalty */
 
@@ -743,7 +743,7 @@ static void halbtc8723b2ant_SetFwIgnoreWlanAct(
 	struct btc_coexist *pBtCoexist, bool bEnable
 )
 {
-	u8 	H2C_Parameter[1] = {0};
+	u8	H2C_Parameter[1] = {0};
 
 	if (bEnable)
 		H2C_Parameter[0] |= BIT(0);		/*  function enable */
@@ -775,7 +775,7 @@ static void halbtc8723b2ant_SetFwPstdma(
 	u8 byte5
 )
 {
-	u8 	H2C_Parameter[5] = {0};
+	u8	H2C_Parameter[5] = {0};
 
 	H2C_Parameter[0] = byte1;
 	H2C_Parameter[1] = byte2;
@@ -824,7 +824,7 @@ static void halbtc8723b2ant_SetAntPath(
 	u32 fwVer = 0, u4Tmp = 0;
 	bool bPgExtSwitch = false;
 	bool bUseExtSwitch = false;
-	u8 	H2C_Parameter[2] = {0};
+	u8	H2C_Parameter[2] = {0};
 
 	pBtCoexist->fBtcGet(pBtCoexist, BTC_GET_BL_EXT_SWITCH, &bPgExtSwitch);
 	pBtCoexist->fBtcGet(pBtCoexist, BTC_GET_U4_WIFI_FW_VER, &fwVer);	/*  [31:16]=fw ver, [15:0]=fw sub ver */
@@ -2341,9 +2341,9 @@ void EXhalbtc8723b2ant_PowerOnSetting(struct btc_coexist *pBtCoexist)
 	/*  */
 	/*  S0 or S1 setting and Local register setting(By the setting fw can get ant number, S0/S1, ... info) */
 	/*  Local setting bit define */
-	/* 	BIT0: "0" for no antenna inverse; "1" for antenna inverse */
-	/* 	BIT1: "0" for internal switch; "1" for external switch */
-	/* 	BIT2: "0" for one antenna; "1" for two antenna */
+	/*	BIT0: "0" for no antenna inverse; "1" for antenna inverse */
+	/*	BIT1: "0" for internal switch; "1" for external switch */
+	/*	BIT2: "0" for one antenna; "1" for two antenna */
 	/*  NOTE: here default all internal switch and 1-antenna ==> BIT1 = 0 and BIT2 = 0 */
 	if (pBtCoexist->chipInterface == BTC_INTF_USB) {
 		/*  fixed at S0 for USB interface */
@@ -2460,8 +2460,8 @@ void EXhalbtc8723b2ant_BtInfoNotify(
 	struct btc_coexist *pBtCoexist, u8 *tmpBuf, u8 length
 )
 {
-	u8 	btInfo = 0;
-	u8 	i, rspSource = 0;
+	u8	btInfo = 0;
+	u8	i, rspSource = 0;
 	bool bBtBusy = false, bLimitedDig = false;
 	bool bWifiConnected = false;
 
diff --git a/drivers/staging/rtl8723bs/hal/HalPhyRf.c b/drivers/staging/rtl8723bs/hal/HalPhyRf.c
index 9e4d42e3a122f..bf44c1050be57 100644
--- a/drivers/staging/rtl8723bs/hal/HalPhyRf.c
+++ b/drivers/staging/rtl8723bs/hal/HalPhyRf.c
@@ -219,13 +219,13 @@ void ODM_TXPowerTrackingCallback_ThermalMeter(struct adapter *Adapter)
 				pDM_Odm->RFCalibrateInfo.OFDM_index[p];
 
 			/* 4 7.1 Handle boundary conditions of index. */
-			if (pDM_Odm->RFCalibrateInfo.OFDM_index[p] > c.SwingTableSize_OFDM-1)
-				pDM_Odm->RFCalibrateInfo.OFDM_index[p] = c.SwingTableSize_OFDM-1;
+			if (pDM_Odm->RFCalibrateInfo.OFDM_index[p] > c.SwingTableSize_OFDM - 1)
+				pDM_Odm->RFCalibrateInfo.OFDM_index[p] = c.SwingTableSize_OFDM - 1;
 			else if (pDM_Odm->RFCalibrateInfo.OFDM_index[p] < OFDM_min_index)
 				pDM_Odm->RFCalibrateInfo.OFDM_index[p] = OFDM_min_index;
 		}
-		if (pDM_Odm->RFCalibrateInfo.CCK_index > c.SwingTableSize_CCK-1)
-			pDM_Odm->RFCalibrateInfo.CCK_index = c.SwingTableSize_CCK-1;
+		if (pDM_Odm->RFCalibrateInfo.CCK_index > c.SwingTableSize_CCK - 1)
+			pDM_Odm->RFCalibrateInfo.CCK_index = c.SwingTableSize_CCK - 1;
 		/* else if (pDM_Odm->RFCalibrateInfo.CCK_index < 0) */
 			/* pDM_Odm->RFCalibrateInfo.CCK_index = 0; */
 	} else {
@@ -233,10 +233,6 @@ void ODM_TXPowerTrackingCallback_ThermalMeter(struct adapter *Adapter)
 				pDM_Odm->RFCalibrateInfo.PowerIndexOffset[p] = 0;
 	}
 
-	/* Print Swing base & current */
-	for (p = RF_PATH_A; p < c.RfPathCount; p++) {
-	}
-
 	if (
 		(pDM_Odm->RFCalibrateInfo.PowerIndexOffset[RF_PATH_A] != 0 ||
 		 pDM_Odm->RFCalibrateInfo.PowerIndexOffset[RF_PATH_B] != 0) &&
diff --git a/drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c b/drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c
index 70b68f473d588..6c5f56d5a1f49 100644
--- a/drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c
+++ b/drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c
@@ -349,17 +349,17 @@ static u8 phy_PathA_IQK_8723B(
 	struct dm_odm_t *pDM_Odm = &pHalData->odmpriv;
 
 	/*  Save RF Path */
-	Path_SEL_BB = PHY_QueryBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord);
+	Path_SEL_BB = PHY_QueryBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord);
 
 	/* leave IQK mode */
 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_IQK, bMaskH3Bytes, 0x000000);
 
-	/* 	enable path A PA in TXIQK mode */
+	/*	enable path A PA in TXIQK mode */
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_WE_LUT, 0x80000, 0x1);
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_RCK_OS, bRFRegOffsetMask, 0x18000);
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_TXPA_G1, bRFRegOffsetMask, 0x0003f);
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_TXPA_G2, bRFRegOffsetMask, 0xc7f87);
-	/* 	disable path B PA in TXIQK mode */
+	/*	disable path B PA in TXIQK mode */
 /* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xed, bRFRegOffsetMask, 0x00020); */
 /* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0x43, bRFRegOffsetMask, 0x40ec1); */
 
@@ -387,10 +387,10 @@ static u8 phy_PathA_IQK_8723B(
 	/* Ant switch */
 	if (configPathB || (RF_Path == 0))
 		/*  wifi switch to S1 */
-		PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, 0x00000000);
+		PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, 0x00000000);
 	else
 		/*  wifi switch to S0 */
-		PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, 0x00000280);
+		PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, 0x00000280);
 
 	/* GNT_BT = 0 */
 	PHY_SetBBReg(pDM_Odm->Adapter, 0x764, bMaskDWord, 0x00000800);
@@ -404,7 +404,7 @@ static u8 phy_PathA_IQK_8723B(
 	mdelay(IQK_DELAY_TIME_8723B);
 
 	/* restore Ant Path */
-	PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, Path_SEL_BB);
+	PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, Path_SEL_BB);
 	/* GNT_BT = 1 */
 	PHY_SetBBReg(pDM_Odm->Adapter, 0x764, bMaskDWord, 0x00001800);
 
@@ -447,7 +447,7 @@ static u8 phy_PathA_RxIQK8723B(
 	struct dm_odm_t *pDM_Odm = &pHalData->odmpriv;
 
 	/*  Save RF Path */
-	Path_SEL_BB = PHY_QueryBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord);
+	Path_SEL_BB = PHY_QueryBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord);
 
 	/* leave IQK mode */
 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_IQK, bMaskH3Bytes, 0x000000);
@@ -486,10 +486,10 @@ static u8 phy_PathA_RxIQK8723B(
 	/* Ant switch */
 	if (configPathB || (RF_Path == 0))
 		/*  wifi switch to S1 */
-		PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, 0x00000000);
+		PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, 0x00000000);
 	else
 		/*  wifi switch to S0 */
-		PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, 0x00000280);
+		PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, 0x00000280);
 
 	/* GNT_BT = 0 */
 	PHY_SetBBReg(pDM_Odm->Adapter, 0x764, bMaskDWord, 0x00000800);
@@ -503,7 +503,7 @@ static u8 phy_PathA_RxIQK8723B(
 	mdelay(IQK_DELAY_TIME_8723B);
 
 	/* restore Ant Path */
-	PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, Path_SEL_BB);
+	PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, Path_SEL_BB);
 	/* GNT_BT = 1 */
 	PHY_SetBBReg(pDM_Odm->Adapter, 0x764, bMaskDWord, 0x00001800);
 
@@ -572,10 +572,10 @@ static u8 phy_PathA_RxIQK8723B(
 	/* Ant switch */
 	if (configPathB || (RF_Path == 0))
 		/*  wifi switch to S1 */
-		PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, 0x00000000);
+		PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, 0x00000000);
 	else
 		/*  wifi switch to S0 */
-		PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, 0x00000280);
+		PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, 0x00000280);
 
 	/* GNT_BT = 0 */
 	PHY_SetBBReg(pDM_Odm->Adapter, 0x764, bMaskDWord, 0x00000800);
@@ -589,7 +589,7 @@ static u8 phy_PathA_RxIQK8723B(
 	mdelay(IQK_DELAY_TIME_8723B);
 
 	/* restore Ant Path */
-	PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, Path_SEL_BB);
+	PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, Path_SEL_BB);
 	/* GNT_BT = 1 */
 	PHY_SetBBReg(pDM_Odm->Adapter, 0x764, bMaskDWord, 0x00001800);
 
@@ -600,7 +600,7 @@ static u8 phy_PathA_RxIQK8723B(
 	regEAC = PHY_QueryBBReg(pDM_Odm->Adapter, rRx_Power_After_IQK_A_2, bMaskDWord);
 	regEA4 = PHY_QueryBBReg(pDM_Odm->Adapter, rRx_Power_Before_IQK_A_2, bMaskDWord);
 
-	/* 	PA/PAD controlled by 0x0 */
+	/*	PA/PAD controlled by 0x0 */
 	/* leave IQK mode */
 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_IQK, bMaskH3Bytes, 0x000000);
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xdf, bRFRegOffsetMask, 0x780);
@@ -632,17 +632,17 @@ static u8 phy_PathB_IQK_8723B(struct adapter *padapter)
 	struct dm_odm_t *pDM_Odm = &pHalData->odmpriv;
 
 	/*  Save RF Path */
-	Path_SEL_BB = PHY_QueryBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord);
+	Path_SEL_BB = PHY_QueryBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord);
 
     /* leave IQK mode */
 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_IQK, bMaskH3Bytes, 0x000000);
 
-	/* 	in TXIQK mode */
+	/*	in TXIQK mode */
 /* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_WE_LUT, 0x80000, 0x1); */
 /* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_RCK_OS, bRFRegOffsetMask, 0x20000); */
 /* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_TXPA_G1, bRFRegOffsetMask, 0x0003f); */
 /* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_TXPA_G2, bRFRegOffsetMask, 0xc7f87); */
-	/* 	enable path B PA in TXIQK mode */
+	/*	enable path B PA in TXIQK mode */
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xed, 0x20, 0x1);
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0x43, bRFRegOffsetMask, 0x30fc1);
 
@@ -669,7 +669,7 @@ static u8 phy_PathB_IQK_8723B(struct adapter *padapter)
 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_IQK, bMaskH3Bytes, 0x808000);
 
 	/* switch to path B */
-	PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, 0x00000280);
+	PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, 0x00000280);
 /* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xb0, bRFRegOffsetMask, 0xeffe0); */
 
 	/* GNT_BT = 0 */
@@ -684,7 +684,7 @@ static u8 phy_PathB_IQK_8723B(struct adapter *padapter)
 	mdelay(IQK_DELAY_TIME_8723B);
 
 	/* restore Ant Path */
-	PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, Path_SEL_BB);
+	PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, Path_SEL_BB);
 	/* GNT_BT = 1 */
 	PHY_SetBBReg(pDM_Odm->Adapter, 0x764, bMaskDWord, 0x00001800);
 
@@ -723,12 +723,12 @@ static u8 phy_PathB_RxIQK8723B(struct adapter *padapter, bool configPathB)
 	struct dm_odm_t *pDM_Odm = &pHalData->odmpriv;
 
 	/*  Save RF Path */
-	Path_SEL_BB = PHY_QueryBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord);
+	Path_SEL_BB = PHY_QueryBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord);
     /* leave IQK mode */
 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_IQK, bMaskH3Bytes, 0x000000);
 
 	/* switch to path B */
-	PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, 0x00000280);
+	PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, 0x00000280);
 	/* modify RXIQK mode table */
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_WE_LUT, 0x80000, 0x1);
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_RCK_OS, bRFRegOffsetMask, 0x18000);
@@ -761,7 +761,7 @@ static u8 phy_PathB_RxIQK8723B(struct adapter *padapter, bool configPathB)
 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_IQK, bMaskH3Bytes, 0x808000);
 
 	/* switch to path B */
-	PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, 0x00000280);
+	PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, 0x00000280);
 /* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xb0, bRFRegOffsetMask, 0xeffe0); */
 
 	/* GNT_BT = 0 */
@@ -776,7 +776,7 @@ static u8 phy_PathB_RxIQK8723B(struct adapter *padapter, bool configPathB)
 	mdelay(IQK_DELAY_TIME_8723B);
 
 	/* restore Ant Path */
-	PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, Path_SEL_BB);
+	PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, Path_SEL_BB);
 	/* GNT_BT = 1 */
 	PHY_SetBBReg(pDM_Odm->Adapter, 0x764, bMaskDWord, 0x00001800);
 
@@ -847,7 +847,7 @@ static u8 phy_PathB_RxIQK8723B(struct adapter *padapter, bool configPathB)
 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_IQK, bMaskH3Bytes, 0x808000);
 
 	/* switch to path B */
-	PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, 0x00000280);
+	PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, 0x00000280);
 /* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xb0, bRFRegOffsetMask, 0xeffe0); */
 
 	/* GNT_BT = 0 */
@@ -862,7 +862,7 @@ static u8 phy_PathB_RxIQK8723B(struct adapter *padapter, bool configPathB)
 	mdelay(IQK_DELAY_TIME_8723B);
 
 	/* restore Ant Path */
-	PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, Path_SEL_BB);
+	PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, Path_SEL_BB);
 	/* GNT_BT = 1 */
 	PHY_SetBBReg(pDM_Odm->Adapter, 0x764, bMaskDWord, 0x00001800);
 
@@ -873,7 +873,7 @@ static u8 phy_PathB_RxIQK8723B(struct adapter *padapter, bool configPathB)
 	regEAC = PHY_QueryBBReg(pDM_Odm->Adapter, rRx_Power_After_IQK_A_2, bMaskDWord);
 	regEA4 = PHY_QueryBBReg(pDM_Odm->Adapter, rRx_Power_Before_IQK_A_2, bMaskDWord);
 
-	/* 	PA/PAD controlled by 0x0 */
+	/*	PA/PAD controlled by 0x0 */
 	/* leave IQK mode */
 /* 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_IQK, 0xffffff00, 0x00000000); */
 /* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_B, 0xdf, bRFRegOffsetMask, 0x180); */
@@ -1322,7 +1322,7 @@ static void phy_IQCalibrate_8723B(
 	const u32 retryCount = 2;
 
 	/*  Note: IQ calibration must be performed after loading */
-	/* 		PHY_REG.txt , and radio_a, radio_b.txt */
+	/*		PHY_REG.txt , and radio_a, radio_b.txt */
 
 	/* u32 bbvalue; */
 
@@ -1554,7 +1554,7 @@ void PHY_IQCalibrate_8723B(
 		rOFDM0_RxIQExtAnta
 	};
 /* 	u32 		Path_SEL_BB = 0; */
-	u32 		GNT_BT_default;
+	u32		GNT_BT_default;
 
 	if (!ODM_CheckPowerStatus(padapter))
 		return;
@@ -1745,7 +1745,7 @@ void PHY_IQCalibrate_8723B(
 void PHY_LCCalibrate_8723B(struct dm_odm_t *pDM_Odm)
 {
 	bool		bSingleTone = false, bCarrierSuppression = false;
-	u32 		timeout = 2000, timecount = 0;
+	u32		timeout = 2000, timecount = 0;
 
 	if (!(pDM_Odm->SupportAbility & ODM_RF_CALIBRATION))
 		return;
diff --git a/drivers/staging/rtl8723bs/hal/HalPwrSeqCmd.c b/drivers/staging/rtl8723bs/hal/HalPwrSeqCmd.c
index a59505cc4a073..c254e3481cc26 100644
--- a/drivers/staging/rtl8723bs/hal/HalPwrSeqCmd.c
+++ b/drivers/staging/rtl8723bs/hal/HalPwrSeqCmd.c
@@ -4,22 +4,22 @@
  * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
  *
  ******************************************************************************/
-/*++
-Copyright (c) Realtek Semiconductor Corp. All rights reserved.
-
-Module Name:
-	HalPwrSeqCmd.c
-
-Abstract:
-	Implement HW Power sequence configuration CMD handling routine for Realtek devices.
-
-Major Change History:
-	When       Who               What
-	---------- ---------------   -------------------------------
-	2011-10-26 Lucas            Modify to be compatible with SD4-CE driver.
-	2011-07-07 Roger            Create.
-
---*/
+/*
+ * Copyright (c) Realtek Semiconductor Corp. All rights reserved.
+ *
+ * Module Name:
+ *  HalPwrSeqCmd.c
+ *
+ * Abstract:
+ *  Implement HW Power sequence configuration CMD handling routine for Realtek devices.
+ *
+ * Major Change History:
+ *	When       Who               What
+ *	---------- ---------------   -------------------------------
+ *	2011-10-26 Lucas            Modify to be compatible with SD4-CE driver.
+ *	2011-07-07 Roger            Create.
+ *
+ */
 #include <drv_types.h>
 #include <HalPwrSeqCmd.h>
 
@@ -87,7 +87,7 @@ u8 HalPwrSeqCmdParsing(
 					value &= (~(GET_PWR_CFG_MASK(PwrCfgCmd)));
 					value |= (
 						GET_PWR_CFG_VALUE(PwrCfgCmd)
-						&GET_PWR_CFG_MASK(PwrCfgCmd)
+						 & GET_PWR_CFG_MASK(PwrCfgCmd)
 					);
 
 					/*  Write the value back to system register */
@@ -105,7 +105,7 @@ u8 HalPwrSeqCmdParsing(
 					else
 						value = rtw_read8(padapter, offset);
 
-					value = value&GET_PWR_CFG_MASK(PwrCfgCmd);
+					value = value & GET_PWR_CFG_MASK(PwrCfgCmd);
 					if (
 						value == (GET_PWR_CFG_VALUE(PwrCfgCmd) &
 						GET_PWR_CFG_MASK(PwrCfgCmd))
@@ -125,7 +125,7 @@ u8 HalPwrSeqCmdParsing(
 				if (GET_PWR_CFG_VALUE(PwrCfgCmd) == PWRSEQ_DELAY_US)
 					udelay(GET_PWR_CFG_OFFSET(PwrCfgCmd));
 				else
-					udelay(GET_PWR_CFG_OFFSET(PwrCfgCmd)*1000);
+					udelay(GET_PWR_CFG_OFFSET(PwrCfgCmd) * 1000);
 				break;
 
 			case PWR_CMD_END:
diff --git a/drivers/staging/rtl8723bs/hal/hal_btcoex.c b/drivers/staging/rtl8723bs/hal/hal_btcoex.c
index 195ccc4b710f1..61cdc27a64ac8 100644
--- a/drivers/staging/rtl8723bs/hal/hal_btcoex.c
+++ b/drivers/staging/rtl8723bs/hal/hal_btcoex.c
@@ -9,14 +9,14 @@
 #include <hal_btcoex.h>
 #include <Mp_Precomp.h>
 
-/* 		Global variables */
+/*		Global variables */
 
 struct btc_coexist GLBtCoexist;
 static u8 GLBtcWiFiInScanState;
 static u8 GLBtcWiFiInIQKState;
 
 /*  */
-/* 		Debug related function */
+/*		Debug related function */
 /*  */
 static u8 halbtcoutsrc_IsBtCoexistAvailable(struct btc_coexist *pBtCoexist)
 {
@@ -316,7 +316,7 @@ static u8 halbtcoutsrc_Get(void *pBtcContext, u8 getType, void *pOutBuf)
 		break;
 
 	case BTC_GET_BL_WIFI_ENABLE_ENCRYPTION:
-		*pu8 = padapter->securitypriv.dot11PrivacyAlgrthm != 0;
+		*pu8 = padapter->securitypriv.dot11_privacy_algrthm != 0;
 		break;
 
 	case BTC_GET_BL_WIFI_UNDER_B_MODE:
@@ -549,7 +549,7 @@ static u8 halbtcoutsrc_Set(void *pBtcContext, u8 setType, void *pInBuf)
 }
 
 /*  */
-/* 		IO related function */
+/*		IO related function */
 /*  */
 static u8 halbtcoutsrc_Read1Byte(void *pBtcContext, u32 RegAddr)
 {
@@ -747,7 +747,7 @@ static void halbtcoutsrc_FillH2cCmd(void *pBtcContext, u8 elementId, u32 cmdLen,
 }
 
 /*  */
-/* 		Extern functions called by other module */
+/*		Extern functions called by other module */
 /*  */
 static u8 EXhalbtcoutsrc_BindBtCoexWithAdapter(void *padapter)
 {
diff --git a/drivers/staging/rtl8723bs/hal/hal_com.c b/drivers/staging/rtl8723bs/hal/hal_com.c
index 634af4657eeeb..a3e8f695de6b0 100644
--- a/drivers/staging/rtl8723bs/hal/hal_com.c
+++ b/drivers/staging/rtl8723bs/hal/hal_com.c
@@ -319,7 +319,7 @@ static void _TwoOutPipeMapping(struct adapter *padapter, bool bWIFICfg)
 
 	if (bWIFICfg) { /* WMM */
 
-		/* 	BK,	BE,	VI,	VO,	BCN,	CMD, MGT, HIGH, HCCA */
+		/*	BK,	BE,	VI,	VO,	BCN,	CMD, MGT, HIGH, HCCA */
 		/*   0,		1,	0,	1,	0,	0,	0,	0,		0	}; */
 		/* 0:ep_0 num, 1:ep_1 num */
 
@@ -359,7 +359,7 @@ static void _ThreeOutPipeMapping(struct adapter *padapter, bool bWIFICfg)
 
 	if (bWIFICfg) { /* for WMM */
 
-		/* 	BK,	BE,	VI,	VO,	BCN,	CMD, MGT, HIGH, HCCA */
+		/*	BK,	BE,	VI,	VO,	BCN,	CMD, MGT, HIGH, HCCA */
 		/*   1,		2,	1,	0,	0,	0,	0,	0,		0	}; */
 		/* 0:H, 1:N, 2:L */
 
@@ -375,7 +375,7 @@ static void _ThreeOutPipeMapping(struct adapter *padapter, bool bWIFICfg)
 
 	} else { /* typical setting */
 
-		/* 	BK,	BE,	VI,	VO,	BCN,	CMD, MGT, HIGH, HCCA */
+		/*	BK,	BE,	VI,	VO,	BCN,	CMD, MGT, HIGH, HCCA */
 		/*   2,		2,	1,	0,	0,	0,	0,	0,		0	}; */
 		/* 0:H, 1:N, 2:L */
 
@@ -548,7 +548,7 @@ void SetHwReg(struct adapter *adapter, u8 variable, u8 *val)
 
 		if (val) { /* Enable default key related setting */
 			reg_scr |= SCR_TXBCUSEDK;
-			if (sec->dot11AuthAlgrthm != dot11AuthAlgrthm_8021X)
+			if (sec->dot11_auth_algrthm != dot11AuthAlgrthm_8021X)
 				reg_scr |= (SCR_RxUseDK|SCR_TxUseDK);
 		} else /* Disable default key related setting */
 			reg_scr &= ~(SCR_RXBCUSEDK|SCR_TXBCUSEDK|SCR_RxUseDK|SCR_TxUseDK);
diff --git a/drivers/staging/rtl8723bs/hal/hal_com_phycfg.c b/drivers/staging/rtl8723bs/hal/hal_com_phycfg.c
index b16456d5b6bba..f886699104e59 100644
--- a/drivers/staging/rtl8723bs/hal/hal_com_phycfg.c
+++ b/drivers/staging/rtl8723bs/hal/hal_com_phycfg.c
@@ -459,7 +459,7 @@ u8 PHY_GetTxPowerIndexBase(
 {
 	struct hal_com_data *pHalData = GET_HAL_DATA(padapter);
 	u8 txPower = 0;
-	u8 chnlIdx = (Channel-1);
+	u8 chnlIdx = (Channel - 1);
 
 	if (!HAL_IsLegalChannel(padapter, Channel))
 		chnlIdx = 0;
diff --git a/drivers/staging/rtl8723bs/hal/hal_intf.c b/drivers/staging/rtl8723bs/hal/hal_intf.c
index 9a3ebad35efc0..2caade1a8bdd1 100644
--- a/drivers/staging/rtl8723bs/hal/hal_intf.c
+++ b/drivers/staging/rtl8723bs/hal/hal_intf.c
@@ -9,18 +9,18 @@
 
 static void rtw_hal_init_opmode(struct adapter *padapter)
 {
-	enum ndis_802_11_network_infrastructure networkType = Ndis802_11InfrastructureMax;
+	enum nl80211_iftype networkType = NL80211_IFTYPE_UNSPECIFIED;
 	struct  mlme_priv *pmlmepriv = &(padapter->mlmepriv);
 	signed int fw_state;
 
 	fw_state = get_fwstate(pmlmepriv);
 
 	if (fw_state & WIFI_ADHOC_STATE)
-		networkType = Ndis802_11IBSS;
+		networkType = NL80211_IFTYPE_ADHOC;
 	else if (fw_state & WIFI_STATION_STATE)
-		networkType = Ndis802_11Infrastructure;
+		networkType = NL80211_IFTYPE_STATION;
 	else if (fw_state & WIFI_AP_STATE)
-		networkType = Ndis802_11APMode;
+		networkType = NL80211_IFTYPE_AP;
 	else
 		return;
 
@@ -170,9 +170,8 @@ void rtw_hal_update_ra_mask(struct sta_info *psta, u8 rssi_level)
 
 	if (check_fwstate(pmlmepriv, WIFI_AP_STATE))
 		add_ratid(padapter, psta, rssi_level);
-	else {
+	else
 		UpdateHalRAMask8723B(padapter, psta->mac_id, rssi_level);
-	}
 }
 
 void rtw_hal_add_ra_tid(struct adapter *padapter, u32 bitmap, u8 *arg, u8 rssi_level)
@@ -216,9 +215,8 @@ void rtw_hal_dm_watchdog(struct adapter *padapter)
 
 void rtw_hal_dm_watchdog_in_lps(struct adapter *padapter)
 {
-	if (adapter_to_pwrctl(padapter)->fw_current_in_ps_mode) {
+	if (adapter_to_pwrctl(padapter)->fw_current_in_ps_mode)
 		rtl8723b_HalDmWatchDog_in_LPS(padapter); /* this function caller is in interrupt context */
-	}
 }
 
 void beacon_timing_control(struct adapter *padapter)
diff --git a/drivers/staging/rtl8723bs/hal/hal_sdio.c b/drivers/staging/rtl8723bs/hal/hal_sdio.c
index 1a865d4de8ac7..1bd165a4573f6 100644
--- a/drivers/staging/rtl8723bs/hal/hal_sdio.c
+++ b/drivers/staging/rtl8723bs/hal/hal_sdio.c
@@ -24,7 +24,7 @@ u8 rtw_hal_sdio_query_tx_freepage(
 {
 	struct hal_com_data	*pHalData = GET_HAL_DATA(padapter);
 
-	if ((pHalData->SdioTxFIFOFreePage[PageIdx]+pHalData->SdioTxFIFOFreePage[PUBLIC_QUEUE_IDX]) >= (RequiredPageNum))
+	if ((pHalData->SdioTxFIFOFreePage[PageIdx] + pHalData->SdioTxFIFOFreePage[PUBLIC_QUEUE_IDX]) >= (RequiredPageNum))
 		return true;
 	else
 		return false;
diff --git a/drivers/staging/rtl8723bs/hal/odm_DIG.c b/drivers/staging/rtl8723bs/hal/odm_DIG.c
index 4e28b7742bfe3..c862c86b7695e 100644
--- a/drivers/staging/rtl8723bs/hal/odm_DIG.c
+++ b/drivers/staging/rtl8723bs/hal/odm_DIG.c
@@ -590,7 +590,7 @@ void odm_DIGbyRSSI_LPS(void *pDM_VOID)
 }
 
 /* 3 ============================================================ */
-/* 3 FASLE ALARM CHECK */
+/* 3 FALSE ALARM CHECK */
 /* 3 ============================================================ */
 
 void odm_FalseAlarmCounterStatistics(void *pDM_VOID)
diff --git a/drivers/staging/rtl8723bs/hal/odm_EdcaTurboCheck.c b/drivers/staging/rtl8723bs/hal/odm_EdcaTurboCheck.c
index 578d5712645c8..9baca2aea80e2 100644
--- a/drivers/staging/rtl8723bs/hal/odm_EdcaTurboCheck.c
+++ b/drivers/staging/rtl8723bs/hal/odm_EdcaTurboCheck.c
@@ -150,7 +150,7 @@ void odm_EdcaTurboCheckCE(void *pDM_VOID)
 	} else {
 		/*  Turn Off EDCA turbo here. */
 		/*  Restore original EDCA according to the declaration of AP. */
-		 if (pDM_Odm->DM_EDCA_Table.bCurrentTurboEDCA) {
+		if (pDM_Odm->DM_EDCA_Table.bCurrentTurboEDCA) {
 			rtw_write32(Adapter, REG_EDCA_BE_PARAM, pHalData->AcParam_BE);
 			pDM_Odm->DM_EDCA_Table.bCurrentTurboEDCA = false;
 		}
diff --git a/drivers/staging/rtl8723bs/hal/odm_HWConfig.c b/drivers/staging/rtl8723bs/hal/odm_HWConfig.c
index c88d669cb086d..1716252641d3b 100644
--- a/drivers/staging/rtl8723bs/hal/odm_HWConfig.c
+++ b/drivers/staging/rtl8723bs/hal/odm_HWConfig.c
@@ -236,9 +236,8 @@ static void odm_rx_phy_status_parsing(struct dm_odm_t *dm_odm,
 	if (is_cck_rate) {
 		phy_info->signal_strength = (u8)(odm_signal_scale_mapping(dm_odm, pwdb_all));
 	} else {
-		if (rf_rx_num != 0) {
+		if (rf_rx_num != 0)
 			phy_info->signal_strength = (u8)(odm_signal_scale_mapping(dm_odm, total_rssi /= rf_rx_num));
-		}
 	}
 }
 
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_cmd.c b/drivers/staging/rtl8723bs/hal/rtl8723b_cmd.c
index c35c7f1c38efa..21b837772b245 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_cmd.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_cmd.c
@@ -69,24 +69,23 @@ s32 FillH2CCmd8723B(struct adapter *padapter, u8 ElementID, u32 CmdLen, u8 *pCmd
 		if (!_is_fw_read_cmd_down(padapter, h2c_box_num))
 			goto exit;
 
-		if (CmdLen <= 3)
-			memcpy((u8 *)(&h2c_cmd)+1, pCmdBuffer, CmdLen);
-		else {
-			memcpy((u8 *)(&h2c_cmd)+1, pCmdBuffer, 3);
-			memcpy((u8 *)(&h2c_cmd_ex), pCmdBuffer+3, CmdLen-3);
-/* 			*(u8 *)(&h2c_cmd) |= BIT(7); */
+		if (CmdLen <= 3) {
+			memcpy((u8 *)(&h2c_cmd) + 1, pCmdBuffer, CmdLen);
+		} else {
+			memcpy((u8 *)(&h2c_cmd) + 1, pCmdBuffer, 3);
+			memcpy((u8 *)(&h2c_cmd_ex), pCmdBuffer + 3, CmdLen - 3);
 		}
 
 		*(u8 *)(&h2c_cmd) |= ElementID;
 
 		if (CmdLen > 3) {
-			msgbox_ex_addr = REG_HMEBOX_EXT0_8723B + (h2c_box_num*RTL8723B_EX_MESSAGE_BOX_SIZE);
+			msgbox_ex_addr = REG_HMEBOX_EXT0_8723B + (h2c_box_num * RTL8723B_EX_MESSAGE_BOX_SIZE);
 			rtw_write32(padapter, msgbox_ex_addr, h2c_cmd_ex);
 		}
-		msgbox_addr = REG_HMEBOX_0 + (h2c_box_num*MESSAGE_BOX_SIZE);
+		msgbox_addr = REG_HMEBOX_0 + (h2c_box_num * MESSAGE_BOX_SIZE);
 		rtw_write32(padapter, msgbox_addr, h2c_cmd);
 
-		pHalData->LastHMEBoxNum = (h2c_box_num+1) % MAX_H2C_BOX_NUMS;
+		pHalData->LastHMEBoxNum = (h2c_box_num + 1) % MAX_H2C_BOX_NUMS;
 
 	} while (0);
 
@@ -103,21 +102,20 @@ static void ConstructBeacon(struct adapter *padapter, u8 *pframe, u32 *pLength)
 	struct ieee80211_hdr *pwlanhdr;
 	__le16 *fctrl;
 	u32 rate_len, pktlen;
-	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
-	struct wlan_bssid_ex *cur_network = &(pmlmeinfo->network);
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
+	struct wlan_bssid_ex *cur_network = &pmlmeinfo->network;
 
 	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
-	fctrl = &(pwlanhdr->frame_control);
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
 
 	eth_broadcast_addr(pwlanhdr->addr1);
-	ether_addr_copy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)));
+	ether_addr_copy(pwlanhdr->addr2, myid(&padapter->eeprompriv));
 	ether_addr_copy(pwlanhdr->addr3, get_my_bssid(cur_network));
 
-	SetSeqNum(pwlanhdr, 0/*pmlmeext->mgnt_seq*/);
-	/* pmlmeext->mgnt_seq++; */
+	SetSeqNum(pwlanhdr, 0);
 	SetFrameSubType(pframe, WIFI_BEACON);
 
 	pframe += sizeof(struct ieee80211_hdr_3addr);
@@ -139,9 +137,9 @@ static void ConstructBeacon(struct adapter *padapter, u8 *pframe, u32 *pLength)
 	pframe += 2;
 	pktlen += 2;
 
-	if ((pmlmeinfo->state&0x03) == WIFI_FW_AP_STATE) {
+	if ((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE) {
 		pktlen += cur_network->ie_length - sizeof(struct ndis_802_11_fix_ie);
-		memcpy(pframe, cur_network->ies+sizeof(struct ndis_802_11_fix_ie), pktlen);
+		memcpy(pframe, cur_network->ies + sizeof(struct ndis_802_11_fix_ie), pktlen);
 
 		goto _ConstructBeacon;
 	}
@@ -156,12 +154,11 @@ static void ConstructBeacon(struct adapter *padapter, u8 *pframe, u32 *pLength)
 	pframe = rtw_set_ie(pframe, WLAN_EID_SUPP_RATES, ((rate_len > 8) ? 8 : rate_len), cur_network->supported_rates, &pktlen);
 
 	/*  DS parameter set */
-	pframe = rtw_set_ie(pframe, WLAN_EID_DS_PARAMS, 1, (unsigned char *)&(cur_network->configuration.ds_config), &pktlen);
+	pframe = rtw_set_ie(pframe, WLAN_EID_DS_PARAMS, 1, (unsigned char *)&cur_network->configuration.ds_config, &pktlen);
 
-	if ((pmlmeinfo->state&0x03) == WIFI_FW_ADHOC_STATE) {
+	if ((pmlmeinfo->state & 0x03) == WIFI_FW_ADHOC_STATE) {
 		u32 ATIMWindow;
 		/*  IBSS Parameter Set... */
-		/* ATIMWindow = cur->configuration.ATIMWindow; */
 		ATIMWindow = 0;
 		pframe = rtw_set_ie(pframe, WLAN_EID_IBSS_PARAMS, 2, (unsigned char *)(&ATIMWindow), &pktlen);
 	}
@@ -187,13 +184,13 @@ static void ConstructPSPoll(struct adapter *padapter, u8 *pframe, u32 *pLength)
 {
 	struct ieee80211_hdr *pwlanhdr;
 	__le16 *fctrl;
-	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
 	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
 	/*  Frame control. */
-	fctrl = &(pwlanhdr->frame_control);
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
 	SetPwrMgt(fctrl);
 	SetFrameSubType(pframe, WIFI_PSPOLL);
@@ -202,32 +199,30 @@ static void ConstructPSPoll(struct adapter *padapter, u8 *pframe, u32 *pLength)
 	SetDuration(pframe, (pmlmeinfo->aid | 0xc000));
 
 	/*  BSSID. */
-	ether_addr_copy(pwlanhdr->addr1, get_my_bssid(&(pmlmeinfo->network)));
+	ether_addr_copy(pwlanhdr->addr1, get_my_bssid(&pmlmeinfo->network));
 
 	/*  TA. */
-	ether_addr_copy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)));
+	ether_addr_copy(pwlanhdr->addr2, myid(&padapter->eeprompriv));
 
 	*pLength = 16;
 }
 
-static void ConstructNullFunctionData(
-	struct adapter *padapter,
-	u8 *pframe,
-	u32 *pLength,
-	u8 *StaAddr,
-	u8 bQoS,
-	u8 AC,
-	u8 bEosp,
-	u8 bForcePowerSave
-)
+static void ConstructNullFunctionData(struct adapter *padapter,
+				      u8 *pframe,
+				      u32 *pLength,
+				      u8 *StaAddr,
+				      u8 bQoS,
+				      u8 AC,
+				      u8 bEosp,
+				      u8 bForcePowerSave)
 {
 	struct ieee80211_hdr *pwlanhdr;
 	__le16 *fctrl;
 	u32 pktlen;
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct wlan_network *cur_network = &pmlmepriv->cur_network;
-	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
 	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
@@ -237,19 +232,19 @@ static void ConstructNullFunctionData(
 		SetPwrMgt(fctrl);
 
 	switch (cur_network->network.infrastructure_mode) {
-	case Ndis802_11Infrastructure:
+	case NL80211_IFTYPE_STATION:
 		SetToDs(fctrl);
 		ether_addr_copy(pwlanhdr->addr1, get_my_bssid(&(pmlmeinfo->network)));
 		ether_addr_copy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)));
 		ether_addr_copy(pwlanhdr->addr3, StaAddr);
 		break;
-	case Ndis802_11APMode:
+	case NL80211_IFTYPE_AP:
 		SetFrDs(fctrl);
 		ether_addr_copy(pwlanhdr->addr1, StaAddr);
 		ether_addr_copy(pwlanhdr->addr2, get_my_bssid(&(pmlmeinfo->network)));
 		ether_addr_copy(pwlanhdr->addr3, myid(&(padapter->eeprompriv)));
 		break;
-	case Ndis802_11IBSS:
+	case NL80211_IFTYPE_ADHOC:
 	default:
 		ether_addr_copy(pwlanhdr->addr1, StaAddr);
 		ether_addr_copy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)));
@@ -324,7 +319,7 @@ void rtl8723b_set_rssi_cmd(struct adapter *padapter, u8 *param)
 {
 	u8 u1H2CRssiSettingParm[H2C_RSSI_SETTING_LEN] = {0};
 	u8 mac_id = *param;
-	u8 rssi = *(param+2);
+	u8 rssi = *(param + 2);
 	u8 uldl_state = 0;
 
 	SET_8723B_H2CCMD_RSSI_SETTING_MACID(u1H2CRssiSettingParm, mac_id);
@@ -343,9 +338,9 @@ void rtl8723b_set_FwPwrMode_cmd(struct adapter *padapter, u8 psmode)
 	u8 PowerState = 0, awake_intvl = 1, byte5 = 0, rlbm = 0;
 
 	if (pwrpriv->dtim > 0 && pwrpriv->dtim < 16)
-		awake_intvl = pwrpriv->dtim+1;/* DTIM = (awake_intvl - 1) */
+		awake_intvl = pwrpriv->dtim + 1;
 	else
-		awake_intvl = 3;/* DTIM =2 */
+		awake_intvl = 3;
 
 	rlbm = 2;
 
@@ -394,7 +389,7 @@ void rtl8723b_set_FwPwrMode_cmd(struct adapter *padapter, u8 psmode)
 			pmlmeext->DrvBcnTimeOut = 0xff;
 
 			for (i = 0; i < 9; i++) {
-				pmlmeext->bcn_delay_ratio[i] = (pmlmeext->bcn_delay_cnt[i]*100)/pmlmeext->bcn_cnt;
+				pmlmeext->bcn_delay_ratio[i] = (pmlmeext->bcn_delay_cnt[i] * 100) / pmlmeext->bcn_cnt;
 
 				ratio_20_delay += pmlmeext->bcn_delay_ratio[i];
 				ratio_80_delay += pmlmeext->bcn_delay_ratio[i];
@@ -416,14 +411,6 @@ void rtl8723b_set_FwPwrMode_cmd(struct adapter *padapter, u8 psmode)
 
 		}
 
-/* offload to FW if fw version > v15.10
-		pmlmeext->DrvBcnEarly = 0;
-		pmlmeext->DrvBcnTimeOut =7;
-
-		if ((pmlmeext->DrvBcnEarly!= 0Xff) && (pmlmeext->DrvBcnTimeOut!= 0xff))
-			u1H2CPwrModeParm[H2C_PWRMODE_LEN-1] = BIT(0) | ((pmlmeext->DrvBcnEarly<<1)&0x0E) |((pmlmeext->DrvBcnTimeOut<<4)&0xf0) ;
-*/
-
 	}
 
 	hal_btcoex_RecordPwrMode(padapter, u1H2CPwrModeParm, H2C_PWRMODE_LEN);
@@ -468,9 +455,7 @@ void rtl8723b_set_FwPwrModeInIPS_cmd(struct adapter *padapter, u8 cmd_param)
  * to Hw again and set the length in descriptor to the real beacon length.
  */
 /* 2009.10.15 by tynli. */
-static void rtl8723b_set_FwRsvdPagePkt(
-	struct adapter *padapter, bool bDLFinished
-)
+static void rtl8723b_set_FwRsvdPagePkt(struct adapter *padapter, bool bDLFinished)
 {
 	struct xmit_frame *pcmdframe;
 	struct pkt_attrib *pattrib;
@@ -493,7 +478,7 @@ static void rtl8723b_set_FwRsvdPagePkt(
 	pmlmeinfo = &pmlmeext->mlmext_info;
 
 	RsvdPageNum = BCNQ_PAGE_NUM_8723B + WOWLAN_PAGE_NUM_8723B;
-	MaxRsvdPageBufSize = RsvdPageNum*PageSize;
+	MaxRsvdPageBufSize = RsvdPageNum * PageSize;
 
 	pcmdframe = rtw_alloc_cmdxmitframe(pxmitpriv);
 	if (!pcmdframe)
@@ -515,69 +500,66 @@ static void rtl8723b_set_FwRsvdPagePkt(
 
 	TotalPageNum += CurtPktPageNum;
 
-	BufIndex += (CurtPktPageNum*PageSize);
+	BufIndex += (CurtPktPageNum * PageSize);
 
 	/* 3 (2) ps-poll */
 	RsvdPageLoc.LocPsPoll = TotalPageNum;
 	ConstructPSPoll(padapter, &ReservedPagePacket[BufIndex], &PSPollLength);
-	rtl8723b_fill_fake_txdesc(padapter, &ReservedPagePacket[BufIndex-TxDescLen], PSPollLength, true, false, false);
+	rtl8723b_fill_fake_txdesc(padapter, &ReservedPagePacket[BufIndex - TxDescLen], PSPollLength, true, false, false);
 
 	CurtPktPageNum = (u8)PageNum_128(TxDescLen + PSPollLength);
 
 	TotalPageNum += CurtPktPageNum;
 
-	BufIndex += (CurtPktPageNum*PageSize);
+	BufIndex += (CurtPktPageNum * PageSize);
 
 	/* 3 (3) null data */
 	RsvdPageLoc.LocNullData = TotalPageNum;
-	ConstructNullFunctionData(
-		padapter,
-		&ReservedPagePacket[BufIndex],
-		&NullDataLength,
-		get_my_bssid(&pmlmeinfo->network),
-		false, 0, 0, false
-	);
-	rtl8723b_fill_fake_txdesc(padapter, &ReservedPagePacket[BufIndex-TxDescLen], NullDataLength, false, false, false);
+	ConstructNullFunctionData(padapter,
+				  &ReservedPagePacket[BufIndex],
+				  &NullDataLength,
+				  get_my_bssid(&pmlmeinfo->network),
+				  false, 0, 0, false);
+
+	rtl8723b_fill_fake_txdesc(padapter, &ReservedPagePacket[BufIndex - TxDescLen], NullDataLength, false, false, false);
 
 	CurtPktPageNum = (u8)PageNum_128(TxDescLen + NullDataLength);
 
 	TotalPageNum += CurtPktPageNum;
 
-	BufIndex += (CurtPktPageNum*PageSize);
+	BufIndex += (CurtPktPageNum * PageSize);
 
 	/* 3 (5) Qos null data */
 	RsvdPageLoc.LocQosNull = TotalPageNum;
-	ConstructNullFunctionData(
-		padapter,
-		&ReservedPagePacket[BufIndex],
-		&QosNullLength,
-		get_my_bssid(&pmlmeinfo->network),
-		true, 0, 0, false
-	);
-	rtl8723b_fill_fake_txdesc(padapter, &ReservedPagePacket[BufIndex-TxDescLen], QosNullLength, false, false, false);
+	ConstructNullFunctionData(padapter,
+				  &ReservedPagePacket[BufIndex],
+				  &QosNullLength,
+				  get_my_bssid(&pmlmeinfo->network),
+				  true, 0, 0, false);
+
+	rtl8723b_fill_fake_txdesc(padapter, &ReservedPagePacket[BufIndex - TxDescLen], QosNullLength, false, false, false);
 
 	CurtPktPageNum = (u8)PageNum_128(TxDescLen + QosNullLength);
 
 	TotalPageNum += CurtPktPageNum;
 
-	BufIndex += (CurtPktPageNum*PageSize);
+	BufIndex += (CurtPktPageNum * PageSize);
 
 	/* 3 (6) BT Qos null data */
 	RsvdPageLoc.LocBTQosNull = TotalPageNum;
-	ConstructNullFunctionData(
-		padapter,
-		&ReservedPagePacket[BufIndex],
-		&BTQosNullLength,
-		get_my_bssid(&pmlmeinfo->network),
-		true, 0, 0, false
-	);
-	rtl8723b_fill_fake_txdesc(padapter, &ReservedPagePacket[BufIndex-TxDescLen], BTQosNullLength, false, true, false);
+	ConstructNullFunctionData(padapter,
+				  &ReservedPagePacket[BufIndex],
+				  &BTQosNullLength,
+				  get_my_bssid(&pmlmeinfo->network),
+				  true, 0, 0, false);
+
+	rtl8723b_fill_fake_txdesc(padapter, &ReservedPagePacket[BufIndex - TxDescLen], BTQosNullLength, false, true, false);
 
 	CurtPktPageNum = (u8)PageNum_128(TxDescLen + BTQosNullLength);
 
 	TotalPageNum += CurtPktPageNum;
 
-	BufIndex += (CurtPktPageNum*PageSize);
+	BufIndex += (CurtPktPageNum * PageSize);
 
 	TotalPacketLen = BufIndex + BTQosNullLength;
 
@@ -605,8 +587,8 @@ static void rtl8723b_set_FwRsvdPagePkt(
 void rtl8723b_download_rsvd_page(struct adapter *padapter, u8 mstatus)
 {
 	struct hal_com_data	*pHalData = GET_HAL_DATA(padapter);
-	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 	bool bcn_valid = false;
 	u8 DLBcnCount = 0;
 	u32 poll = 0;
@@ -618,12 +600,12 @@ void rtl8723b_download_rsvd_page(struct adapter *padapter, u8 mstatus)
 
 		/*  We should set AID, correct TSF, HW seq enable before set JoinBssReport to Fw in 88/92C. */
 		/*  Suggested by filen. Added by tynli. */
-		rtw_write16(padapter, REG_BCN_PSR_RPT, (0xC000|pmlmeinfo->aid));
+		rtw_write16(padapter, REG_BCN_PSR_RPT, (0xC000 | pmlmeinfo->aid));
 
 		/*  set REG_CR bit 8 */
-		v8 = rtw_read8(padapter, REG_CR+1);
+		v8 = rtw_read8(padapter, REG_CR + 1);
 		v8 |= BIT(0); /*  ENSWBCN */
-		rtw_write8(padapter, REG_CR+1, v8);
+		rtw_write8(padapter, REG_CR + 1, v8);
 
 		/*  Disable Hw protection for a time which revserd for Hw sending beacon. */
 		/*  Fix download reserved page packet fail that access collision with the protection time. */
@@ -638,7 +620,7 @@ void rtl8723b_download_rsvd_page(struct adapter *padapter, u8 mstatus)
 			bRecover = true;
 
 		/*  To tell Hw the packet is not a real beacon frame. */
-		rtw_write8(padapter, REG_FWHW_TXQ_CTRL+2, pHalData->RegFwHwTxQCtrl & ~BIT(6));
+		rtw_write8(padapter, REG_FWHW_TXQ_CTRL + 2, pHalData->RegFwHwTxQCtrl & ~BIT(6));
 		pHalData->RegFwHwTxQCtrl &= ~BIT(6);
 
 		/*  Clear beacon valid check bit. */
@@ -657,7 +639,7 @@ void rtl8723b_download_rsvd_page(struct adapter *padapter, u8 mstatus)
 				/*  check rsvd page download OK. */
 				rtw_hal_get_hwreg(padapter, HW_VAR_BCN_VALID, (u8 *)(&bcn_valid));
 				poll++;
-			} while (!bcn_valid && (poll%10) != 0 && !padapter->bSurpriseRemoved && !padapter->bDriverStopped);
+			} while (!bcn_valid && (poll % 10) != 0 && !padapter->bSurpriseRemoved && !padapter->bDriverStopped);
 
 		} while (!bcn_valid && DLBcnCount <= 100 && !padapter->bSurpriseRemoved && !padapter->bDriverStopped);
 
@@ -680,14 +662,14 @@ void rtl8723b_download_rsvd_page(struct adapter *padapter, u8 mstatus)
 		/*  the beacon cannot be sent by HW. */
 		/*  2010.06.23. Added by tynli. */
 		if (bRecover) {
-			rtw_write8(padapter, REG_FWHW_TXQ_CTRL+2, pHalData->RegFwHwTxQCtrl | BIT(6));
+			rtw_write8(padapter, REG_FWHW_TXQ_CTRL + 2, pHalData->RegFwHwTxQCtrl | BIT(6));
 			pHalData->RegFwHwTxQCtrl |= BIT(6);
 		}
 
 		/*  Clear CR[8] or beacon packet will not be send to TxBuf anymore. */
-		v8 = rtw_read8(padapter, REG_CR+1);
+		v8 = rtw_read8(padapter, REG_CR + 1);
 		v8 &= ~BIT(0); /*  ~ENSWBCN */
-		rtw_write8(padapter, REG_CR+1, v8);
+		rtw_write8(padapter, REG_CR + 1, v8);
 	}
 }
 
@@ -701,22 +683,17 @@ void rtl8723b_set_FwJoinBssRpt_cmd(struct adapter *padapter, u8 mstatus)
 /* arg[1] = raid */
 /* arg[2] = shortGIrate */
 /* arg[3] = init_rate */
-void rtl8723b_Add_RateATid(
-	struct adapter *padapter,
-	u32 bitmap,
-	u8 *arg,
-	u8 rssi_level
-)
+void rtl8723b_Add_RateATid(struct adapter *padapter, u32 bitmap, u8 *arg, u8 rssi_level)
 {
 	struct hal_com_data	*pHalData = GET_HAL_DATA(padapter);
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 	struct sta_info *psta;
 	u8 mac_id = arg[0];
 	u8 raid = arg[1];
 	u8 shortGI = arg[2];
 	u8 bw;
-	u32 mask = bitmap&0x0FFFFFFF;
+	u32 mask = bitmap & 0x0FFFFFFF;
 
 	psta = pmlmeinfo->FW_sta_info[mac_id].psta;
 	if (!psta)
@@ -730,16 +707,14 @@ void rtl8723b_Add_RateATid(
 	rtl8723b_set_FwMacIdConfig_cmd(padapter, mac_id, raid, bw, shortGI, mask);
 }
 
-static void ConstructBtNullFunctionData(
-	struct adapter *padapter,
-	u8 *pframe,
-	u32 *pLength,
-	u8 *StaAddr,
-	u8 bQoS,
-	u8 AC,
-	u8 bEosp,
-	u8 bForcePowerSave
-)
+static void ConstructBtNullFunctionData(struct adapter *padapter,
+					u8 *pframe,
+					u32 *pLength,
+					u8 *StaAddr,
+					u8 bQoS,
+					u8 AC,
+					u8 bEosp,
+					u8 bForcePowerSave)
 {
 	struct ieee80211_hdr *pwlanhdr;
 	__le16 *fctrl;
@@ -805,7 +780,7 @@ static void SetFwRsvdPagePkt_BTCoex(struct adapter *padapter)
 	PageSize = PAGE_SIZE_TX_8723B;
 
 	RsvdPageNum = BCNQ_PAGE_NUM_8723B;
-	MaxRsvdPageBufSize = RsvdPageNum*PageSize;
+	MaxRsvdPageBufSize = RsvdPageNum * PageSize;
 
 	pcmdframe = rtw_alloc_cmdxmitframe(pxmitpriv);
 	if (!pcmdframe)
@@ -826,7 +801,7 @@ static void SetFwRsvdPagePkt_BTCoex(struct adapter *padapter)
 		CurtPktPageNum += 1;
 	TotalPageNum += CurtPktPageNum;
 
-	BufIndex += (CurtPktPageNum*PageSize);
+	BufIndex += (CurtPktPageNum * PageSize);
 
 	/*  Jump to lastest page */
 	if (BufIndex < (MaxRsvdPageBufSize - PageSize)) {
@@ -836,14 +811,13 @@ static void SetFwRsvdPagePkt_BTCoex(struct adapter *padapter)
 
 	/* 3 (6) BT Qos null data */
 	RsvdPageLoc.LocBTQosNull = TotalPageNum;
-	ConstructBtNullFunctionData(
-		padapter,
-		&ReservedPagePacket[BufIndex],
-		&BTQosNullLength,
-		NULL,
-		true, 0, 0, false
-	);
-	rtl8723b_fill_fake_txdesc(padapter, &ReservedPagePacket[BufIndex-TxDescLen], BTQosNullLength, false, true, false);
+	ConstructBtNullFunctionData(padapter,
+				    &ReservedPagePacket[BufIndex],
+				    &BTQosNullLength,
+				    NULL,
+				    true, 0, 0, false);
+
+	rtl8723b_fill_fake_txdesc(padapter, &ReservedPagePacket[BufIndex - TxDescLen], BTQosNullLength, false, true, false);
 
 	CurtPktPageNum = (u8)PageNum_128(TxDescLen + BTQosNullLength);
 
@@ -885,12 +859,12 @@ void rtl8723b_download_BTCoex_AP_mode_rsvd_page(struct adapter *padapter)
 
 	/*  We should set AID, correct TSF, HW seq enable before set JoinBssReport to Fw in 88/92C. */
 	/*  Suggested by filen. Added by tynli. */
-	rtw_write16(padapter, REG_BCN_PSR_RPT, (0xC000|pmlmeinfo->aid));
+	rtw_write16(padapter, REG_BCN_PSR_RPT, (0xC000 | pmlmeinfo->aid));
 
 	/*  set REG_CR bit 8 */
-	val8 = rtw_read8(padapter, REG_CR+1);
+	val8 = rtw_read8(padapter, REG_CR + 1);
 	val8 |= BIT(0); /*  ENSWBCN */
-	rtw_write8(padapter,  REG_CR+1, val8);
+	rtw_write8(padapter,  REG_CR + 1, val8);
 
 	/*  Disable Hw protection for a time which revserd for Hw sending beacon. */
 	/*  Fix download reserved page packet fail that access collision with the protection time. */
@@ -906,7 +880,7 @@ void rtl8723b_download_BTCoex_AP_mode_rsvd_page(struct adapter *padapter)
 
 	/*  To tell Hw the packet is not a real beacon frame. */
 	pHalData->RegFwHwTxQCtrl &= ~BIT(6);
-	rtw_write8(padapter, REG_FWHW_TXQ_CTRL+2, pHalData->RegFwHwTxQCtrl);
+	rtw_write8(padapter, REG_FWHW_TXQ_CTRL + 2, pHalData->RegFwHwTxQCtrl);
 
 	/*  Clear beacon valid check bit. */
 	rtw_hal_set_hwreg(padapter, HW_VAR_BCN_VALID, NULL);
@@ -923,7 +897,7 @@ void rtl8723b_download_BTCoex_AP_mode_rsvd_page(struct adapter *padapter)
 			/*  check rsvd page download OK. */
 			rtw_hal_get_hwreg(padapter, HW_VAR_BCN_VALID, &bcn_valid);
 			poll++;
-		} while (!bcn_valid && (poll%10) != 0 && !padapter->bSurpriseRemoved && !padapter->bDriverStopped);
+		} while (!bcn_valid && (poll % 10) != 0 && !padapter->bSurpriseRemoved && !padapter->bDriverStopped);
 	} while (!bcn_valid && (DLBcnCount <= 100) && !padapter->bSurpriseRemoved && !padapter->bDriverStopped);
 
 	if (bcn_valid) {
@@ -945,11 +919,11 @@ void rtl8723b_download_BTCoex_AP_mode_rsvd_page(struct adapter *padapter)
 	/*  2010.06.23. Added by tynli. */
 	if (bRecover) {
 		pHalData->RegFwHwTxQCtrl |= BIT(6);
-		rtw_write8(padapter, REG_FWHW_TXQ_CTRL+2, pHalData->RegFwHwTxQCtrl);
+		rtw_write8(padapter, REG_FWHW_TXQ_CTRL + 2, pHalData->RegFwHwTxQCtrl);
 	}
 
 	/*  Clear CR[8] or beacon packet will not be send to TxBuf anymore. */
-	val8 = rtw_read8(padapter, REG_CR+1);
+	val8 = rtw_read8(padapter, REG_CR + 1);
 	val8 &= ~BIT(0); /*  ~ENSWBCN */
-	rtw_write8(padapter, REG_CR+1, val8);
+	rtw_write8(padapter, REG_CR + 1, val8);
 }
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_dm.c b/drivers/staging/rtl8723bs/hal/rtl8723b_dm.c
index 2276dc1cb44ac..bd6fbb80ddb71 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_dm.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_dm.c
@@ -42,7 +42,7 @@ static void Init_ODM_ComInfo_8723b(struct adapter *Adapter)
 	ODM_CmnInfoInit(pDM_Odm, ODM_CMNINFO_CUT_VER, cut_ver);
 
 	ODM_CmnInfoInit(pDM_Odm, ODM_CMNINFO_PATCH_ID, pHalData->CustomerID);
-	/* 	ODM_CMNINFO_BINHCT_TEST only for MP Team */
+	/* ODM_CMNINFO_BINHCT_TEST only for MP Team */
 	ODM_CmnInfoInit(pDM_Odm, ODM_CMNINFO_BWIFI_TEST, Adapter->registrypriv.wifi_spec);
 
 	pdmpriv->InitODMFlag = ODM_RF_CALIBRATION|ODM_RF_TX_PWR_TRACK;
@@ -80,7 +80,7 @@ static void Update_ODM_ComInfo_8723b(struct adapter *Adapter)
 	/*  Pointer reference */
 	/*  */
 	/* ODM_CMNINFO_MAC_PHY_MODE pHalData->MacPhyMode92D */
-	/* 	ODM_CmnInfoHook(pDM_Odm, ODM_CMNINFO_MAC_PHY_MODE,&(pDM_Odm->u8_temp)); */
+	/* ODM_CmnInfoHook(pDM_Odm, ODM_CMNINFO_MAC_PHY_MODE,&(pDM_Odm->u8_temp)); */
 
 	ODM_CmnInfoUpdate(pDM_Odm, ODM_CMNINFO_ABILITY, pdmpriv->InitODMFlag);
 
@@ -88,7 +88,7 @@ static void Update_ODM_ComInfo_8723b(struct adapter *Adapter)
 	ODM_CmnInfoHook(pDM_Odm, ODM_CMNINFO_RX_UNI, &(dvobj->traffic_stat.rx_bytes));
 	ODM_CmnInfoHook(pDM_Odm, ODM_CMNINFO_WM_MODE, &(pmlmeext->cur_wireless_mode));
 	ODM_CmnInfoHook(pDM_Odm, ODM_CMNINFO_SEC_CHNL_OFFSET, &(pHalData->nCur40MhzPrimeSC));
-	ODM_CmnInfoHook(pDM_Odm, ODM_CMNINFO_SEC_MODE, &(Adapter->securitypriv.dot11PrivacyAlgrthm));
+	ODM_CmnInfoHook(pDM_Odm, ODM_CMNINFO_SEC_MODE, &(Adapter->securitypriv.dot11_privacy_algrthm));
 	ODM_CmnInfoHook(pDM_Odm, ODM_CMNINFO_BW, &(pHalData->CurrentChannelBW));
 	ODM_CmnInfoHook(pDM_Odm, ODM_CMNINFO_CHNL, &(pHalData->CurrentChannel));
 	ODM_CmnInfoHook(pDM_Odm, ODM_CMNINFO_NET_CLOSED, &(Adapter->net_closed));
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c b/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
index bcaf63b2893cd..ecd66f090a50e 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
@@ -194,9 +194,8 @@ static s32 polling_fwdl_chksum(
 		yield();
 	} while (jiffies_to_msecs(jiffies-start) < timeout_ms || cnt < min_cnt);
 
-	if (!(value32 & FWDL_ChkSum_rpt)) {
+	if (!(value32 & FWDL_ChkSum_rpt))
 		goto exit;
-	}
 
 	if (g_fwdl_chksum_fail) {
 		g_fwdl_chksum_fail--;
@@ -235,9 +234,8 @@ static s32 _FWFreeToGo(struct adapter *adapter, u32 min_cnt, u32 timeout_ms)
 		yield();
 	} while (jiffies_to_msecs(jiffies - start) < timeout_ms || cnt < min_cnt);
 
-	if (!(value32 & WINTINI_RDY)) {
+	if (!(value32 & WINTINI_RDY))
 		goto exit;
-	}
 
 	if (g_fwdl_wintint_rdy_fail) {
 		g_fwdl_wintint_rdy_fail--;
@@ -283,8 +281,8 @@ void rtl8723b_FirmwareSelfReset(struct adapter *padapter)
 }
 
 /*  */
-/* 	Description: */
-/* 		Download 8192C firmware code. */
+/*	Description: */
+/*		Download 8192C firmware code. */
 /*  */
 /*  */
 s32 rtl8723b_FirmwareDownload(struct adapter *padapter, bool  bUsedWoWLANFw)
@@ -437,7 +435,7 @@ void rtl8723b_InitializeFirmwareVars(struct adapter *padapter)
 }
 
 /*  */
-/* 				Efuse related code */
+/*				Efuse related code */
 /*  */
 static u8 hal_EfuseSwitchToBank(
 	struct adapter *padapter, u8 bank
@@ -830,17 +828,6 @@ static void ReadChipVersion8723B(struct adapter *padapter)
 
 	value32 = rtw_read32(padapter, REG_SYS_CFG);
 	pHalData->chip_normal = ((value32 & RTL_ID) ? false : true);
-
-	/*  For regulator mode. by tynli. 2011.01.14 */
-	pHalData->RegulatorMode = ((value32 & SPS_SEL) ? RT_LDO_REGULATOR : RT_SWITCHING_REGULATOR);
-
-	/*  For multi-function consideration. Added by Roger, 2010.10.06. */
-	pHalData->MultiFunc = RT_MULTI_FUNC_NONE;
-	value32 = rtw_read32(padapter, REG_MULTI_FUNC_CTRL);
-	pHalData->MultiFunc |= ((value32 & WL_FUNC_EN) ? RT_MULTI_FUNC_WIFI : 0);
-	pHalData->MultiFunc |= ((value32 & BT_FUNC_EN) ? RT_MULTI_FUNC_BT : 0);
-	pHalData->MultiFunc |= ((value32 & GPS_FUNC_EN) ? RT_MULTI_FUNC_GPS : 0);
-	pHalData->PolarityCtl = ((value32 & WL_HWPDN_SL) ? RT_POLARITY_HIGH_ACT : RT_POLARITY_LOW_ACT);
 }
 
 void rtl8723b_read_chip_version(struct adapter *padapter)
@@ -861,8 +848,7 @@ void rtl8723b_InitBeaconParameters(struct adapter *padapter)
 
 	rtw_write16(padapter, REG_BCN_CTRL, val16);
 
-	/*  TODO: Remove these magic number */
-	rtw_write16(padapter, REG_TBTT_PROHIBIT, 0x6404);/*  ms */
+	rtw_write16(padapter, REG_TBTT_PROHIBIT, TBTT_PROHIBIT_TIME_8723B);
 	/*  Firmware will control REG_DRVERLYINT when power saving is enable, */
 	/*  so don't set this register on STA mode. */
 	if (!check_fwstate(&padapter->mlmepriv, WIFI_STATION_STATE))
@@ -871,7 +857,7 @@ void rtl8723b_InitBeaconParameters(struct adapter *padapter)
 
 	/*  Suggested by designer timchen. Change beacon AIFS to the largest number */
 	/*  because test chip does not contension before sending beacon. by tynli. 2009.11.03 */
-	rtw_write16(padapter, REG_BCNTCFG, 0x660F);
+	rtw_write16(padapter, REG_BCNTCFG, BCNTCFG_8723B);
 
 	pHalData->RegBcnCtrlVal = rtw_read8(padapter, REG_BCN_CTRL);
 	pHalData->RegTxPause = rtw_read8(padapter, REG_TXPAUSE);
@@ -1031,9 +1017,8 @@ void UpdateHalRAMask8723B(struct adapter *padapter, u32 mac_id, u8 rssi_level)
 	rate_bitmap = hal_btcoex_GetRaMask(padapter);
 	mask &= ~rate_bitmap;
 
-	if (pHalData->fw_ractrl) {
+	if (pHalData->fw_ractrl)
 		rtl8723b_set_FwMacIdConfig_cmd(padapter, mac_id, psta->raid, psta->bw_mode, short_gi_rate, mask);
-	}
 
 	/* set correct initial date rate for each mac_id */
 	pdmpriv->INIDATA_RATE[mac_id] = psta->init_rate;
@@ -1310,7 +1295,7 @@ void Hal_EfuseParseTxPowerInfo_8723B(
 {
 	struct hal_com_data	*pHalData = GET_HAL_DATA(padapter);
 	struct TxPowerInfo24G	pwrInfo24G;
-	u8 	rfPath, ch, TxCount = 1;
+	u8	rfPath, ch, TxCount = 1;
 
 	Hal_ReadPowerValueFromPROM_8723B(padapter, &pwrInfo24G, PROMContent, AutoLoadFail);
 	for (rfPath = 0 ; rfPath < MAX_RF_PATH ; rfPath++) {
@@ -1514,10 +1499,8 @@ void Hal_EfuseParseThermalMeter_8723B(
 	else
 		pHalData->EEPROMThermalMeter = EEPROM_Default_ThermalMeter_8723B;
 
-	if ((pHalData->EEPROMThermalMeter == 0xff) || AutoLoadFail) {
-		pHalData->bAPKThermalMeterIgnore = true;
+	if ((pHalData->EEPROMThermalMeter == 0xff) || AutoLoadFail)
 		pHalData->EEPROMThermalMeter = EEPROM_Default_ThermalMeter_8723B;
-	}
 }
 
 void Hal_ReadRFGainOffset(
@@ -1596,9 +1579,8 @@ static void rtl8723b_cal_txdesc_chksum(struct tx_desc *ptxdesc)
 	/*  Thomas, Lucas@SD4, 20130515 */
 	count = 16;
 
-	for (index = 0; index < count; index++) {
+	for (index = 0; index < count; index++)
 		checksum |= le16_to_cpu(*(__le16 *)(usPtr + index));
-	}
 
 	ptxdesc->txdw7 |= cpu_to_le32(checksum & 0x0000ffff);
 }
@@ -1834,8 +1816,8 @@ void rtl8723b_update_txdesc(struct xmit_frame *pxmitframe, u8 *pbuf)
 
 /*  */
 /*  Description: In normal chip, we should send some packet to Hw which will be used by Fw */
-/* 			in FW LPS mode. The function is to fill the Tx descriptor of this packets, then */
-/* 			Fw can tell Hw to send these packet derectly. */
+/*			in FW LPS mode. The function is to fill the Tx descriptor of this packets, then */
+/*			Fw can tell Hw to send these packet derectly. */
 /*  Added by tynli. 2009.10.15. */
 /*  */
 /* type1:pspoll, type2:null */
@@ -1867,9 +1849,8 @@ void rtl8723b_fill_fake_txdesc(
 		SET_TX_DESC_HWSEQ_SEL_8723B(pDesc, 0);
 	}
 
-	if (IsBTQosNull) {
+	if (IsBTQosNull)
 		SET_TX_DESC_BT_INT_8723B(pDesc, 1);
-	}
 
 	SET_TX_DESC_USE_RATE_8723B(pDesc, 1); /*  use data rate which is set by Sw */
 	SET_TX_DESC_OWN_8723B((u8 *)pDesc, 1);
@@ -1880,7 +1861,7 @@ void rtl8723b_fill_fake_txdesc(
 	/*  Encrypt the data frame if under security mode excepct null data. Suggested by CCW. */
 	/*  */
 	if (bDataFrame) {
-		u32 EncAlg = padapter->securitypriv.dot11PrivacyAlgrthm;
+		u32 EncAlg = padapter->securitypriv.dot11_privacy_algrthm;
 
 		switch (EncAlg) {
 		case _NO_PRIVACY_:
@@ -2184,9 +2165,9 @@ void CCX_FwC2HTxRpt_8723b(struct adapter *padapter, u8 *pdata, u8 len)
 #define	GET_8723B_C2H_TX_RPT_LIFE_TIME_OVER(_Header)	LE_BITS_TO_1BYTE((_Header + 0), 6, 1)
 #define	GET_8723B_C2H_TX_RPT_RETRY_OVER(_Header)	LE_BITS_TO_1BYTE((_Header + 0), 7, 1)
 
-	if (GET_8723B_C2H_TX_RPT_RETRY_OVER(pdata) | GET_8723B_C2H_TX_RPT_LIFE_TIME_OVER(pdata)) {
+	if (GET_8723B_C2H_TX_RPT_RETRY_OVER(pdata) | GET_8723B_C2H_TX_RPT_LIFE_TIME_OVER(pdata))
 		rtw_ack_tx_done(&padapter->xmitpriv, RTW_SCTX_DONE_CCX_PKT_FAIL);
-	}
+
 /*
 	else if (seq_no != padapter->xmitpriv.seq_no) {
 		rtw_ack_tx_done(&padapter->xmitpriv, RTW_SCTX_DONE_CCX_PKT_FAIL);
@@ -2246,7 +2227,7 @@ s32 c2h_handler_8723b(struct adapter *padapter, u8 *buf)
 
 	/*  Clear event to notify FW we have read the command. */
 	/*  Note: */
-	/* 	If this field isn't clear, the FW won't update the next command message. */
+	/*	If this field isn't clear, the FW won't update the next command message. */
 /* 	rtw_write8(padapter, REG_C2HEVT_CLEAR, C2H_EVT_HOST_CLOSE); */
 exit:
 	return ret;
@@ -2555,9 +2536,8 @@ void SetHwReg8723B(struct adapter *padapter, u8 variable, u8 *val)
 
 			/*  Forece leave RF low power mode for 1T1R to prevent conficting setting in Fw power */
 			/*  saving sequence. 2010.06.07. Added by tynli. Suggested by SD3 yschang. */
-			if (psmode != PS_MODE_ACTIVE) {
+			if (psmode != PS_MODE_ACTIVE)
 				ODM_RF_Saving(&pHalData->odmpriv, true);
-			}
 
 			/* if (psmode != PS_MODE_ACTIVE)	{ */
 			/* 	rtl8723b_set_lowpwr_lps_cmd(padapter, true); */
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c b/drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c
index d6369eb7b5705..78fa991157515 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c
@@ -376,7 +376,7 @@ int PHY_BBConfig8723B(struct adapter *Adapter)
 	RegVal = rtw_read16(Adapter, REG_SYS_FUNC_EN);
 	rtw_write16(Adapter, REG_SYS_FUNC_EN, (u16)(RegVal | BIT(13) | BIT(0) | BIT(1)));
 
-	rtw_write32(Adapter, 0x948, 0x280);	/*  Others use Antenna S1 */
+	rtw_write32(Adapter, rS0S1_PathSwitch, 0x280);	/*  Others use Antenna S1 */
 
 	rtw_write8(Adapter, REG_RF_CTRL, RF_EN | RF_RSTB | RF_SDMRSTB);
 
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_rf6052.c b/drivers/staging/rtl8723bs/hal/rtl8723b_rf6052.c
index da4cb28276b26..fac1270853de6 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_rf6052.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_rf6052.c
@@ -81,62 +81,38 @@ void PHY_RF6052SetBandwidth8723B(
 static int phy_RF6052_Config_ParaFile(struct adapter *Adapter)
 {
 	u32 u4RegValue = 0;
-	u8 eRFPath;
 	struct bb_register_def *pPhyReg;
 	struct hal_com_data *pHalData = GET_HAL_DATA(Adapter);
 
 	/* 3----------------------------------------------------------------- */
 	/* 3 <2> Initialize RF */
 	/* 3----------------------------------------------------------------- */
-	/* for (eRFPath = RF_PATH_A; eRFPath <pHalData->NumTotalRFPath; eRFPath++) */
-	for (eRFPath = 0; eRFPath < pHalData->NumTotalRFPath; eRFPath++) {
-
-		pPhyReg = &pHalData->PHYRegDef[eRFPath];
-
-		/*----Store original RFENV control type----*/
-		switch (eRFPath) {
-		case RF_PATH_A:
-			u4RegValue = PHY_QueryBBReg(Adapter, pPhyReg->rfintfs, bRFSI_RFENV);
-			break;
-		case RF_PATH_B:
-			u4RegValue = PHY_QueryBBReg(Adapter, pPhyReg->rfintfs, bRFSI_RFENV << 16);
-			break;
-		}
-
-		/*----Set RF_ENV enable----*/
-		PHY_SetBBReg(Adapter, pPhyReg->rfintfe, bRFSI_RFENV << 16, 0x1);
-		udelay(1);/* PlatformStallExecution(1); */
-
-		/*----Set RF_ENV output high----*/
-		PHY_SetBBReg(Adapter, pPhyReg->rfintfo, bRFSI_RFENV, 0x1);
-		udelay(1);/* PlatformStallExecution(1); */
-
-		/* Set bit number of Address and Data for RF register */
-		PHY_SetBBReg(Adapter, pPhyReg->rfHSSIPara2, b3WireAddressLength, 0x0);	/*  Set 1 to 4 bits for 8255 */
-		udelay(1);/* PlatformStallExecution(1); */
-
-		PHY_SetBBReg(Adapter, pPhyReg->rfHSSIPara2, b3WireDataLength, 0x0);	/*  Set 0 to 12  bits for 8255 */
-		udelay(1);/* PlatformStallExecution(1); */
-
-		/*----Initialize RF fom connfiguration file----*/
-		switch (eRFPath) {
-		case RF_PATH_A:
-		case RF_PATH_B:
-			ODM_ConfigRFWithHeaderFile(&pHalData->odmpriv,
-						   CONFIG_RF_RADIO, eRFPath);
-			break;
-		}
-
-		/*----Restore RFENV control type----*/
-		switch (eRFPath) {
-		case RF_PATH_A:
-			PHY_SetBBReg(Adapter, pPhyReg->rfintfs, bRFSI_RFENV, u4RegValue);
-			break;
-		case RF_PATH_B:
-			PHY_SetBBReg(Adapter, pPhyReg->rfintfs, bRFSI_RFENV << 16, u4RegValue);
-			break;
-		}
-	}
+
+	pPhyReg = &pHalData->PHYRegDef[RF_PATH_A];
+
+	/*----Store original RFENV control type----*/
+	u4RegValue = PHY_QueryBBReg(Adapter, pPhyReg->rfintfs, bRFSI_RFENV);
+
+	/*----Set RF_ENV enable----*/
+	PHY_SetBBReg(Adapter, pPhyReg->rfintfe, bRFSI_RFENV << 16, 0x1);
+	udelay(1);/* PlatformStallExecution(1); */
+
+	/*----Set RF_ENV output high----*/
+	PHY_SetBBReg(Adapter, pPhyReg->rfintfo, bRFSI_RFENV, 0x1);
+	udelay(1);/* PlatformStallExecution(1); */
+
+	/* Set bit number of Address and Data for RF register */
+	PHY_SetBBReg(Adapter, pPhyReg->rfHSSIPara2, b3WireAddressLength, 0x0);	/*  Set 1 to 4 bits for 8255 */
+	udelay(1);/* PlatformStallExecution(1); */
+
+	PHY_SetBBReg(Adapter, pPhyReg->rfHSSIPara2, b3WireDataLength, 0x0);	/*  Set 0 to 12  bits for 8255 */
+	udelay(1);/* PlatformStallExecution(1); */
+
+	/*----Initialize RF fom connfiguration file----*/
+	ODM_ConfigRFWithHeaderFile(&pHalData->odmpriv, CONFIG_RF_RADIO, RF_PATH_A);
+
+	/*----Restore RFENV control type----*/
+	PHY_SetBBReg(Adapter, pPhyReg->rfintfs, bRFSI_RFENV, u4RegValue);
 
 	/* 3 ----------------------------------------------------------------- */
 	/* 3 Configuration of Tx Power Tracking */
@@ -149,13 +125,6 @@ static int phy_RF6052_Config_ParaFile(struct adapter *Adapter)
 
 int PHY_RF6052_Config8723B(struct adapter *Adapter)
 {
-	struct hal_com_data *pHalData = GET_HAL_DATA(Adapter);
-
-	/*  */
-	/*  Initialize general global value */
-	/*  */
-	pHalData->NumTotalRFPath = 1;
-
 	/*  */
 	/*  Config BB and RF */
 	/*  */
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c b/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
index 97dce28fd0a37..e943c304950e7 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
@@ -267,9 +267,9 @@ static void rtl8723bs_recv_tasklet(struct tasklet_struct *t)
 				rtw_free_recvframe(precvframe,
 						   &precvpriv->free_recv_queue);
 			} else {
-				/* 	Modified by Albert 20101213 */
-				/* 	For 8 bytes IP header alignment. */
-				if (pattrib->qos)	/* 	Qos data, wireless lan header length is 26 */
+				/*	Modified by Albert 20101213 */
+				/*	For 8 bytes IP header alignment. */
+				if (pattrib->qos)	/*	Qos data, wireless lan header length is 26 */
 					shift_sz = 6;
 				else
 					shift_sz = 0;
@@ -285,8 +285,8 @@ static void rtl8723bs_recv_tasklet(struct tasklet_struct *t)
 						alloc_sz = skb_len + 14;
 				} else {
 					alloc_sz = skb_len;
-					/* 	6 is for IP header 8 bytes alignment in QoS packet case. */
-					/* 	8 is for skb->data 4 bytes alignment. */
+					/*	6 is for IP header 8 bytes alignment in QoS packet case. */
+					/*	8 is for skb->data 4 bytes alignment. */
 					alloc_sz += 14;
 				}
 
diff --git a/drivers/staging/rtl8723bs/hal/sdio_halinit.c b/drivers/staging/rtl8723bs/hal/sdio_halinit.c
index ed9b8fb07fec2..6f2aea984b30b 100644
--- a/drivers/staging/rtl8723bs/hal/sdio_halinit.c
+++ b/drivers/staging/rtl8723bs/hal/sdio_halinit.c
@@ -27,13 +27,18 @@ static u8 CardEnable(struct adapter *padapter)
 		/*  unlock ISO/CLK/Power control register */
 		rtw_write8(padapter, REG_RSV_CTRL, 0x0);
 
-		ret = HalPwrSeqCmdParsing(padapter, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, rtl8723B_card_enable_flow);
+		ret = HalPwrSeqCmdParsing(padapter,
+					  PWR_CUT_ALL_MSK,
+					  PWR_FAB_ALL_MSK,
+					  PWR_INTF_SDIO_MSK,
+					  rtl8723B_card_enable_flow);
 		if (ret == _SUCCESS) {
 			bMacPwrCtrlOn = true;
 			rtw_hal_set_hwreg(padapter, HW_VAR_APFM_ON_MAC, &bMacPwrCtrlOn);
 		}
-	} else
+	} else {
 		ret = _SUCCESS;
+	}
 
 	return ret;
 }
@@ -104,7 +109,11 @@ u8 _InitPowerOn_8723BS(struct adapter *padapter)
 }
 
 /* Tx Page FIFO threshold */
-static void _init_available_page_threshold(struct adapter *padapter, u8 numHQ, u8 numNQ, u8 numLQ, u8 numPubQ)
+static void _init_available_page_threshold(struct adapter *padapter,
+					   u8 numHQ,
+					   u8 numNQ,
+					   u8 numLQ,
+					   u8 numPubQ)
 {
 	u16 HQ_threshold, NQ_threshold, LQ_threshold;
 
@@ -781,7 +790,11 @@ u32 rtl8723bs_hal_init(struct adapter *padapter)
 
 			restore_iqk_rst = pwrpriv->bips_processing;
 			b2Ant = pHalData->EEPROMBluetoothAntNum == Ant_x2;
-			PHY_IQCalibrate_8723B(padapter, false, restore_iqk_rst, b2Ant, pHalData->ant_path);
+			PHY_IQCalibrate_8723B(padapter,
+					      false,
+					      restore_iqk_rst,
+					      b2Ant,
+					      pHalData->ant_path);
 			pHalData->odmpriv.RFCalibrateInfo.bIQKInitialized = true;
 
 			hal_btcoex_IQKNotify(padapter, false);
@@ -812,7 +825,11 @@ static void CardDisableRTL8723BSdio(struct adapter *padapter)
 	u8 bMacPwrCtrlOn;
 
 	/*  Run LPS WL RFOFF flow */
-	HalPwrSeqCmdParsing(padapter, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, rtl8723B_enter_lps_flow);
+	HalPwrSeqCmdParsing(padapter,
+			    PWR_CUT_ALL_MSK,
+			    PWR_FAB_ALL_MSK,
+			    PWR_INTF_SDIO_MSK,
+			    rtl8723B_enter_lps_flow);
 
 	/*	==== Reset digital sequence   ====== */
 
@@ -841,7 +858,11 @@ static void CardDisableRTL8723BSdio(struct adapter *padapter)
 
 	bMacPwrCtrlOn = false;	/*  Disable CMD53 R/W */
 	rtw_hal_set_hwreg(padapter, HW_VAR_APFM_ON_MAC, &bMacPwrCtrlOn);
-	HalPwrSeqCmdParsing(padapter, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, rtl8723B_card_disable_flow);
+	HalPwrSeqCmdParsing(padapter,
+			    PWR_CUT_ALL_MSK,
+			    PWR_FAB_ALL_MSK,
+			    PWR_INTF_SDIO_MSK,
+			    rtl8723B_card_disable_flow);
 }
 
 u32 rtl8723bs_hal_deinit(struct adapter *padapter)
@@ -853,7 +874,10 @@ u32 rtl8723bs_hal_deinit(struct adapter *padapter)
 				u8 val8 = 0;
 
 				rtl8723b_set_FwPwrModeInIPS_cmd(padapter, 0x3);
-				/* poll 0x1cc to make sure H2C command already finished by FW; MAC_0x1cc = 0 means H2C done by FW. */
+				/*
+				 * poll 0x1cc to make sure H2C command already finished by FW;
+				 * MAC_0x1cc = 0 means H2C done by FW.
+				 */
 				do {
 					val8 = rtw_read8(padapter, REG_HMETFR);
 					cnt++;
@@ -862,10 +886,13 @@ u32 rtl8723bs_hal_deinit(struct adapter *padapter)
 				/* H2C done, enter 32k */
 				if (val8 == 0) {
 					/* set rpwm to enter 32k */
-					val8 = rtw_read8(padapter, SDIO_LOCAL_BASE | SDIO_REG_HRPWM1);
+					val8 = rtw_read8(padapter,
+							 SDIO_LOCAL_BASE | SDIO_REG_HRPWM1);
 					val8 += 0x80;
 					val8 |= BIT(0);
-					rtw_write8(padapter, SDIO_LOCAL_BASE | SDIO_REG_HRPWM1, val8);
+					rtw_write8(padapter,
+						   SDIO_LOCAL_BASE | SDIO_REG_HRPWM1,
+						   val8);
 					adapter_to_pwrctl(padapter)->tog = (val8 + 0x80) & 0x80;
 					cnt = val8 = 0;
 					do {
@@ -985,8 +1012,9 @@ static void Hal_EfuseParseBoardType_8723BS(
 		pHalData->BoardType = (hwinfo[EEPROM_RF_BOARD_OPTION_8723B] & 0xE0) >> 5;
 		if (pHalData->BoardType == 0xFF)
 			pHalData->BoardType = (EEPROM_DEFAULT_BOARD_OPTION & 0xE0) >> 5;
-	} else
+	} else {
 		pHalData->BoardType = 0;
+	}
 }
 
 static void _ReadEfuseInfo8723BS(struct adapter *padapter)
@@ -1063,8 +1091,14 @@ static s32 _ReadAdapterInfo8723BS(struct adapter *padapter)
 	_ReadPROMContent(padapter);
 
 	if (!padapter->hw_init_completed) {
-		rtw_write8(padapter, 0x67, 0x00); /*  for BT, Switch Ant control to BT */
-		CardDisableRTL8723BSdio(padapter);/* for the power consumption issue,  wifi ko module is loaded during booting, but wifi GUI is off */
+		/*  for BT, Switch Ant control to BT */
+		rtw_write8(padapter, 0x67, 0x00);
+
+		/*
+		 * for the power consumption issue, wifi ko module is loaded during booting,
+		 * but wifi GUI is off
+		 */
+		CardDisableRTL8723BSdio(padapter);
 	}
 
 	return _SUCCESS;
diff --git a/drivers/staging/rtl8723bs/include/drv_types.h b/drivers/staging/rtl8723bs/include/drv_types.h
index 552d0c5fa47ff..c837157404d0e 100644
--- a/drivers/staging/rtl8723bs/include/drv_types.h
+++ b/drivers/staging/rtl8723bs/include/drv_types.h
@@ -283,7 +283,7 @@ struct adapter {
 	struct	recv_priv recvpriv;
 	struct	sta_priv stapriv;
 	struct	security_priv securitypriv;
-	spinlock_t   security_key_mutex; /*  add for CONFIG_IEEE80211W, none 11w also can use */
+	spinlock_t   security_key_mutex;
 	struct	registry_priv registrypriv;
 	struct	eeprom_priv eeprompriv;
 
diff --git a/drivers/staging/rtl8723bs/include/hal_com_reg.h b/drivers/staging/rtl8723bs/include/hal_com_reg.h
index 479748d856261..4671fdfff916b 100644
--- a/drivers/staging/rtl8723bs/include/hal_com_reg.h
+++ b/drivers/staging/rtl8723bs/include/hal_com_reg.h
@@ -20,7 +20,7 @@
 #define REG_RSV_CTRL					0x001C
 #define REG_RF_CTRL						0x001F
 #define REG_AFE_XTAL_CTRL				0x0024
-#define REG_MAC_PHY_CTRL				0x002c /* for 92d, DMDP, SMSP, DMSP contrl */
+#define REG_MAC_PHY_CTRL				0x002c /* for 92d, DMDP, SMSP, DMSP control */
 #define REG_EFUSE_CTRL					0x0030
 #define REG_EFUSE_TEST					0x0034
 #define REG_PWR_DATA					0x0038
@@ -186,7 +186,7 @@
 
 /*  */
 /*  */
-/* 	Redifine 8192C register definition for compatibility */
+/* 	Redefine 8192C register definition for compatibility */
 /*  */
 /*  */
 #define EFUSE_CTRL				REG_EFUSE_CTRL		/*  E-Fuse Control. */
@@ -273,7 +273,7 @@
 /*  */
 /*        8192C (RCR) Receive Configuration Register	(Offset 0x608, 32 bits) */
 /*  */
-#define RCR_APPFCS				BIT(31)	/*  WMAC append FCS after pauload */
+#define RCR_APPFCS				BIT(31)	/*  WMAC append FCS after payload */
 #define RCR_APP_MIC				BIT(30)	/*  MACRX will retain the MIC at the bottom of the packet. */
 #define RCR_APP_ICV				BIT(29)	/*  MACRX will retain the ICV at the bottom of the packet. */
 #define RCR_APP_PHYST_RXFF		BIT(28)	/*  PHY Status is appended before RX packet in RXFF */
@@ -342,7 +342,6 @@
 #define VENDOR_ID				BIT(19)
 
 #define RTL_ID					BIT(23) /*  TestChip ID, 1:Test(RLE); 0:MP(RL) */
-#define SPS_SEL					BIT(24) /*  1:LDO regulator mode; 0:Switching regulator mode */
 
 
 #define CHIP_VER_RTL_MASK		0xF000	/* Bit 12 ~ 15 */
@@ -590,9 +589,6 @@
 #define C2H_EVT_FW_CLOSE		0xFF	/*  Set by FW indicating that FW had set the C2H command message and it's not yet read by driver. */
 
 /* 2REG_MULTI_FUNC_CTRL(For RTL8723 Only) */
-#define WL_HWPDN_SL			BIT(1)	/*  WiFi HW PDn polarity control */
-#define WL_FUNC_EN				BIT(2)	/*  WiFi function enable */
 #define BT_FUNC_EN				BIT(18)	/*  BT function enable */
-#define GPS_FUNC_EN			BIT(22)	/*  GPS function enable */
 
 #endif /* __HAL_COMMON_H__ */
diff --git a/drivers/staging/rtl8723bs/include/hal_data.h b/drivers/staging/rtl8723bs/include/hal_data.h
index 3a93ac312ded9..c93b21822ea79 100644
--- a/drivers/staging/rtl8723bs/include/hal_data.h
+++ b/drivers/staging/rtl8723bs/include/hal_data.h
@@ -12,29 +12,6 @@
 
 #include <hal_sdio.h>
 
-/*  */
-/*  <Roger_Notes> For RTL8723 WiFi/BT/GPS multi-function configuration. 2010.10.06. */
-/*  */
-enum rt_multi_func {
-	RT_MULTI_FUNC_NONE	= 0x00,
-	RT_MULTI_FUNC_WIFI	= 0x01,
-	RT_MULTI_FUNC_BT		= 0x02,
-	RT_MULTI_FUNC_GPS	= 0x04,
-};
-/*  */
-/*  <Roger_Notes> For RTL8723 WiFi PDn/GPIO polarity control configuration. 2010.10.08. */
-/*  */
-enum rt_polarity_ctl {
-	RT_POLARITY_LOW_ACT	= 0,
-	RT_POLARITY_HIGH_ACT	= 1,
-};
-
-/*  For RTL8723 regulator mode. by tynli. 2011.01.14. */
-enum rt_regulator_mode {
-	RT_SWITCHING_REGULATOR	= 0,
-	RT_LDO_REGULATOR	= 1,
-};
-
 enum rt_ampdu_burst {
 	RT_AMPDU_BURST_NONE	= 0,
 	RT_AMPDU_BURST_92D	= 1,
@@ -162,10 +139,6 @@ struct dm_priv {
 
 
 struct hal_com_data {
-	enum rt_multi_func MultiFunc; /*  For multi-function consideration. */
-	enum rt_polarity_ctl PolarityCtl; /*  For Wifi PDn Polarity control. */
-	enum rt_regulator_mode	RegulatorMode; /*  switching regulator or LDO */
-
 	u16 FirmwareVersion;
 	u16 FirmwareVersionRev;
 	u16 FirmwareSubVersion;
@@ -187,29 +160,16 @@ struct hal_com_data {
 	/* rf_ctrl */
 	u8 rf_chip;
 	u8 PackageType;
-	u8 NumTotalRFPath;
 
-	u8 InterfaceSel;
-	u8 framesync;
-	u32 framesyncC34;
-	u8 framesyncMonitor;
-	u8 DefaultInitialGain[4];
 	/*  EEPROM setting. */
-	u16 EEPROMVID;
-	u16 EEPROMSVID;
-
 	u8 EEPROMCustomerID;
-	u8 EEPROMSubCustomerID;
 	u8 EEPROMVersion;
 	u8 EEPROMRegulatory;
 	u8 EEPROMThermalMeter;
 	u8 EEPROMBluetoothCoexist;
 	u8 EEPROMBluetoothType;
 	u8 EEPROMBluetoothAntNum;
-	u8 EEPROMBluetoothAntIsolation;
-	u8 EEPROMBluetoothRadioShared;
 	u8 bTXPowerDataReadFromEEPORM;
-	u8 bAPKThermalMeterIgnore;
 	u8 bDisableSWChannelPlan; /*  flag of disable software change channel plan */
 
 	bool		EepromOrEfuse;
@@ -228,24 +188,12 @@ struct hal_com_data {
 
 	u8 Regulation2_4G;
 
-	u8 TxPwrInPercentage;
-
-	u8 TxPwrCalibrateRate;
 	/*  TX power by rate table */
 	/*  RF: at most 2 = AB = 0/1 */
 	/*  CCK = 0 OFDM = 1 HT-MCS 0-7 = 2 */
-	u8 TxPwrByRateTable;
-	u8 TxPwrByRateBand;
 	s8 TxPwrByRateOffset[MAX_RF_PATH_NUM][TX_PWR_BY_RATE_NUM_RATE];
 	/*  */
 
-	/* 2 Power Limit Table */
-	u8 TxPwrLevelCck[RF_PATH_MAX_92C_88E][CHANNEL_MAX_NUMBER];
-	u8 TxPwrLevelHT40_1S[RF_PATH_MAX_92C_88E][CHANNEL_MAX_NUMBER];	/*  For HT 40MHZ pwr */
-	u8 TxPwrLevelHT40_2S[RF_PATH_MAX_92C_88E][CHANNEL_MAX_NUMBER];	/*  For HT 40MHZ pwr */
-	s8	TxPwrHt20Diff[RF_PATH_MAX_92C_88E][CHANNEL_MAX_NUMBER];/*  HT 20<->40 Pwr diff */
-	u8 TxPwrLegacyHtDiff[RF_PATH_MAX_92C_88E][CHANNEL_MAX_NUMBER];/*  For HT<->legacy pwr diff */
-
 	/*  Power Limit Table for 2.4G */
 	s8	TxPwrLimit_2_4G[MAX_REGULATION_NUM]
 						[MAX_2_4G_BANDWIDTH_NUM]
@@ -256,52 +204,26 @@ struct hal_com_data {
 	/*  Store the original power by rate value of the base of each rate section of rf path A & B */
 	u8 TxPwrByRateBase2_4G[MAX_RF_PATH_NUM][MAX_RATE_SECTION_NUM];
 
-	/*  For power group */
-	u8 PwrGroupHT20[RF_PATH_MAX_92C_88E][CHANNEL_MAX_NUMBER];
-	u8 PwrGroupHT40[RF_PATH_MAX_92C_88E][CHANNEL_MAX_NUMBER];
-
-
-
-
-	u8 PGMaxGroup;
-	u8 LegacyHTTxPowerDiff;/*  Legacy to HT rate power diff */
-	/*  The current Tx Power Level */
-	u8 CurrentCckTxPwrIdx;
-	u8 CurrentOfdm24GTxPwrIdx;
-	u8 CurrentBW2024GTxPwrIdx;
-	u8 CurrentBW4024GTxPwrIdx;
-
 	/*  Read/write are allow for following hardware information variables */
 	u8 pwrGroupCnt;
 	u32 MCSTxPowerLevelOriginalOffset[MAX_PG_GROUP][16];
-	u32 CCKTxPowerLevelOriginalOffset;
 
 	u8 CrystalCap;
-	u32 AntennaTxPath;					/*  Antenna path Tx */
-	u32 AntennaRxPath;					/*  Antenna path Rx */
 
-	u8 PAType_2G;
-	u8 LNAType_2G;
-	u8 ExternalPA_2G;
-	u8 ExternalLNA_2G;
 	u8 TypeGLNA;
 	u8 TypeGPA;
 	u8 TypeALNA;
 	u8 TypeAPA;
 	u8 RFEType;
 	u8 BoardType;
-	u8 ExternalPA;
 	u8 bIQKInitialized;
 	bool		bLCKInProgress;
 
 	bool		bSwChnl;
 	bool		bSetChnlBW;
-	bool		bChnlBWInitialized;
 	bool		bNeedIQK;
 
-	u8 bLedOpenDrain; /*  Support Open-drain arrangement for controlling the LED. Added by Roger, 2009.10.16. */
 	u8 TxPowerTrackControl; /* for mp mode, turn off txpwrtracking as default */
-	u8 b1x1RecvCombine;	/*  for 1T1R receive combining */
 
 	u32 AcParam_BE; /* Original parameter for BE, use for EDCA turbo. */
 
@@ -309,9 +231,6 @@ struct hal_com_data {
 
 	u32 RfRegChnlVal[2];
 
-	/* RDG enable */
-	bool	 bRDGEnable;
-
 	/* for host message to fw */
 	u8 LastHMEBoxNum;
 
@@ -322,46 +241,30 @@ struct hal_com_data {
 	u8 RegFwHwTxQCtrl;
 	u8 RegReg542;
 	u8 RegCR_1;
-	u8 Reg837;
-	u8 RegRFPathS1;
 	u16 RegRRSR;
 
 	u8 CurAntenna;
 	u8 AntDivCfg;
 	u8 AntDetection;
-	u8 TRxAntDivType;
 	u8 ant_path; /* for 8723B s0/s1 selection */
 
 	u8 u1ForcedIgiLb;			/*  forced IGI lower bound */
 
 	u8 bDumpRxPkt;/* for debug */
 	u8 bDumpTxPkt;/* for debug */
-	u8 FwRsvdPageStartOffset; /* 2010.06.23. Added by tynli. Reserve page start offset except beacon in TxQ. */
 
 	/*  2010/08/09 MH Add CU power down mode. */
 	bool		pwrdown;
 
-	/*  Add for dual MAC  0--Mac0 1--Mac1 */
-	u32 interfaceIndex;
-
 	u8 OutEpQueueSel;
 	u8 OutEpNumber;
 
 	/*  2010/12/10 MH Add for USB aggregation mode dynamic scheme. */
 	bool		UsbRxHighSpeedMode;
 
-	/*  2010/11/22 MH Add for slim combo debug mode selective. */
-	/*  This is used for fix the drawback of CU TSMC-A/UMC-A cut. HW auto suspend ability. Close BT clock. */
-	bool		SlimComboDbg;
-
-	/* u8 AMPDUDensity; */
-
 	/*  Auto FSM to Turn On, include clock, isolation, power control for MAC only */
 	u8 bMacPwrCtrlOn;
 
-	u8 RegIQKFWOffload;
-	struct submit_ctx	iqk_sctx;
-
 	enum rt_ampdu_burst	AMPDUBurstMode; /* 92C maybe not use, but for compile successfully */
 
 	u32 		sdio_himr;
@@ -392,7 +295,5 @@ struct hal_com_data {
 };
 
 #define GET_HAL_DATA(__padapter)	((struct hal_com_data *)((__padapter)->HalData))
-#define GET_HAL_RFPATH_NUM(__padapter) (((struct hal_com_data *)((__padapter)->HalData))->NumTotalRFPath)
-#define RT_GetInterfaceSelection(_Adapter)	(GET_HAL_DATA(_Adapter)->InterfaceSel)
 
 #endif /* __HAL_DATA_H__ */
diff --git a/drivers/staging/rtl8723bs/include/hal_intf.h b/drivers/staging/rtl8723bs/include/hal_intf.h
index ee8ad26325b5d..8a85e8419e472 100644
--- a/drivers/staging/rtl8723bs/include/hal_intf.h
+++ b/drivers/staging/rtl8723bs/include/hal_intf.h
@@ -239,7 +239,7 @@ c2h_id_filter rtw_hal_c2h_id_filter_ccx(struct adapter *adapter);
 s32 rtw_hal_macid_sleep(struct adapter *padapter, u32 macid);
 s32 rtw_hal_macid_wakeup(struct adapter *padapter, u32 macid);
 
-s32 rtw_hal_fill_h2c_cmd(struct adapter *, u8 ElementID, u32 CmdLen, u8 *pCmdBuffer);
+s32 rtw_hal_fill_h2c_cmd(struct adapter *adapter, u8 ElementID, u32 CmdLen, u8 *pCmdBuffer);
 
 void SetHwReg8723BS(struct adapter *padapter, u8 variable, u8 *val);
 void GetHwReg8723BS(struct adapter *padapter, u8 variable, u8 *val);
diff --git a/drivers/staging/rtl8723bs/include/ieee80211.h b/drivers/staging/rtl8723bs/include/ieee80211.h
index 39ee139f1da96..9f421e4875b7a 100644
--- a/drivers/staging/rtl8723bs/include/ieee80211.h
+++ b/drivers/staging/rtl8723bs/include/ieee80211.h
@@ -529,9 +529,9 @@ enum {
 	RTW_WLAN_CATEGORY_FT = 6,
 	RTW_WLAN_CATEGORY_HT = 7,
 	RTW_WLAN_CATEGORY_SA_QUERY = 8,
-	RTW_WLAN_CATEGORY_UNPROTECTED_WNM = 11, /*  add for CONFIG_IEEE80211W, none 11w also can use */
+	RTW_WLAN_CATEGORY_UNPROTECTED_WNM = 11,
 	RTW_WLAN_CATEGORY_TDLS = 12,
-	RTW_WLAN_CATEGORY_SELF_PROTECTED = 15, /*  add for CONFIG_IEEE80211W, none 11w also can use */
+	RTW_WLAN_CATEGORY_SELF_PROTECTED = 15,
 	RTW_WLAN_CATEGORY_WMM = 17,
 	RTW_WLAN_CATEGORY_P2P = 0x7f,/* P2P action frames */
 };
@@ -623,32 +623,6 @@ struct rtw_ieee80211_channel {
 	/* int orig_mpwr; */
 };
 
-#define CHAN_FMT \
-	/*"band:%d, "*/ \
-	/*"center_freq:%u, "*/ \
-	"hw_value:%u, " \
-	"flags:0x%08x" \
-	/*"max_antenna_gain:%d\n"*/ \
-	/*"max_power:%d\n"*/ \
-	/*"max_reg_power:%d\n"*/ \
-	/*"beacon_found:%u\n"*/ \
-	/*"orig_flags:0x%08x\n"*/ \
-	/*"orig_mag:%d\n"*/ \
-	/*"orig_mpwr:%d\n"*/
-
-#define CHAN_ARG(channel) \
-	/*(channel)->band*/ \
-	/*, (channel)->center_freq*/ \
-	(channel)->hw_value \
-	, (channel)->flags \
-	/*, (channel)->max_antenna_gain*/ \
-	/*, (channel)->max_power*/ \
-	/*, (channel)->max_reg_power*/ \
-	/*, (channel)->beacon_found*/ \
-	/*, (channel)->orig_flags*/ \
-	/*, (channel)->orig_mag*/ \
-	/*, (channel)->orig_mpwr*/ \
-
 /* Parsed Information Elements */
 struct rtw_ieee802_11_elems {
 	u8 *ssid;
diff --git a/drivers/staging/rtl8723bs/include/ioctl_cfg80211.h b/drivers/staging/rtl8723bs/include/ioctl_cfg80211.h
index bc62b7285950f..a1fc18cd247b2 100644
--- a/drivers/staging/rtl8723bs/include/ioctl_cfg80211.h
+++ b/drivers/staging/rtl8723bs/include/ioctl_cfg80211.h
@@ -42,10 +42,10 @@ void rtw_cfg80211_indicate_connect(struct adapter *padapter);
 void rtw_cfg80211_indicate_disconnect(struct adapter *padapter);
 void rtw_cfg80211_indicate_scan_done(struct adapter *adapter, bool aborted);
 
-void rtw_cfg80211_indicate_sta_assoc(struct adapter *padapter, u8 *pmgmt_frame, uint frame_len);
+void rtw_cfg80211_indicate_sta_assoc(struct adapter *padapter, u8 *pmgmt_frame, unsigned int frame_len);
 void rtw_cfg80211_indicate_sta_disassoc(struct adapter *padapter, unsigned char *da, unsigned short reason);
 
-void rtw_cfg80211_rx_action(struct adapter *adapter, u8 *frame, uint frame_len, const char *msg);
+void rtw_cfg80211_rx_action(struct adapter *adapter, u8 *frame, unsigned int frame_len, const char *msg);
 
 bool rtw_cfg80211_pwr_mgmt(struct adapter *adapter);
 
diff --git a/drivers/staging/rtl8723bs/include/osdep_service_linux.h b/drivers/staging/rtl8723bs/include/osdep_service_linux.h
index 2ec54f9e180c2..c8274da940ff4 100644
--- a/drivers/staging/rtl8723bs/include/osdep_service_linux.h
+++ b/drivers/staging/rtl8723bs/include/osdep_service_linux.h
@@ -98,10 +98,7 @@ static inline void rtw_netif_stop_queue(struct net_device *pnetdev)
 
 #define rtw_signal_process(pid, sig) kill_pid(find_vpid((pid)), (sig), 1)
 
-#define NDEV_ARG(ndev) ndev->name
 #define ADPT_ARG(adapter) adapter->pnetdev->name
-#define FUNC_NDEV_FMT "%s(%s)"
-#define FUNC_NDEV_ARG(ndev) __func__, ndev->name
 #define FUNC_ADPT_FMT "%s(%s)"
 #define FUNC_ADPT_ARG(adapter) __func__, adapter->pnetdev->name
 
diff --git a/drivers/staging/rtl8723bs/include/rtl8192c_recv.h b/drivers/staging/rtl8723bs/include/rtl8192c_recv.h
index 1f86654f04033..b63625ab4e452 100644
--- a/drivers/staging/rtl8723bs/include/rtl8192c_recv.h
+++ b/drivers/staging/rtl8723bs/include/rtl8192c_recv.h
@@ -30,7 +30,4 @@ struct phy_stat {
 /*  Rx smooth factor */
 #define	Rx_Smooth_Factor (20)
 
-void rtl8192c_translate_rx_signal_stuff(union recv_frame *precvframe, struct phy_stat *pphy_status);
-void rtl8192c_query_rx_desc_status(union recv_frame *precvframe, struct recv_stat *pdesc);
-
 #endif
diff --git a/drivers/staging/rtl8723bs/include/rtl8723b_hal.h b/drivers/staging/rtl8723bs/include/rtl8723b_hal.h
index b6006110ad6b0..259ba63ba4bc8 100644
--- a/drivers/staging/rtl8723bs/include/rtl8723b_hal.h
+++ b/drivers/staging/rtl8723bs/include/rtl8723b_hal.h
@@ -72,6 +72,19 @@ struct rt_firmware_hdr {
 #define DRIVER_EARLY_INT_TIME_8723B  0x05
 #define BCN_DMA_ATIME_INT_TIME_8723B 0x02
 
+/* REG_TBTT_PROHIBIT (0x0540) - TBTT prohibit hold/setup in 32us units */
+#define TBTT_PROHIBIT_SETUP_8723B 0x04
+#define TBTT_PROHIBIT_HOLD_8723B  0x64
+#define TBTT_PROHIBIT_TIME_8723B \
+	((TBTT_PROHIBIT_HOLD_8723B << 8) | TBTT_PROHIBIT_SETUP_8723B)
+
+/* REG_BCNTCFG (0x0510) - beacon AIFS, CWmin, CWmax (EDCA-like layout) */
+#define BCN_AIFS_8723B   0x0F
+#define BCN_CW_MIN_8723B 0x06
+#define BCN_CW_MAX_8723B 0x06
+#define BCNTCFG_8723B \
+	((BCN_CW_MAX_8723B << 12) | (BCN_CW_MIN_8723B << 8) | BCN_AIFS_8723B)
+
 /* for 8723B */
 /* TX 32K, RX 16K, Page size 128B for TX, 8B for RX */
 #define PAGE_SIZE_TX_8723B 128
@@ -180,11 +193,6 @@ enum { /* tag_Package_Definition */
 	PACKAGE_TFBGA79
 };
 
-#define INCLUDE_MULTI_FUNC_BT(_Adapter)  \
-	(GET_HAL_DATA(_Adapter)->MultiFunc & RT_MULTI_FUNC_BT)
-#define INCLUDE_MULTI_FUNC_GPS(_Adapter) \
-	(GET_HAL_DATA(_Adapter)->MultiFunc & RT_MULTI_FUNC_GPS)
-
 /*  rtl8723a_hal_init.c */
 s32 rtl8723b_FirmwareDownload(struct adapter *padapter, bool  bUsedWoWLANFw);
 void rtl8723b_FirmwareSelfReset(struct adapter *padapter);
diff --git a/drivers/staging/rtl8723bs/include/rtw_ap.h b/drivers/staging/rtl8723bs/include/rtw_ap.h
index 83e835eb6513f..866aa83869314 100644
--- a/drivers/staging/rtl8723bs/include/rtw_ap.h
+++ b/drivers/staging/rtl8723bs/include/rtw_ap.h
@@ -10,7 +10,7 @@
 void init_mlme_ap_info(struct adapter *padapter);
 void free_mlme_ap_info(struct adapter *padapter);
 /* void update_BCNTIM(struct adapter *padapter); */
-void update_beacon(struct adapter *padapter, u8 ie_id, u8 *oui, u8 tx);
+void update_beacon(struct adapter *padapter, u8 ie_id, u8 tx);
 void add_ratid(struct adapter *padapter, struct sta_info *psta, u8 rssi_level);
 void expire_timeout_chk(struct adapter *padapter);
 void update_sta_info_apmode(struct adapter *padapter, struct sta_info *psta);
diff --git a/drivers/staging/rtl8723bs/include/rtw_cmd.h b/drivers/staging/rtl8723bs/include/rtw_cmd.h
index f78aa0d7abc5a..2e791da7e8153 100644
--- a/drivers/staging/rtl8723bs/include/rtw_cmd.h
+++ b/drivers/staging/rtl8723bs/include/rtw_cmd.h
@@ -45,7 +45,6 @@
 		u8 *cmd_buf;	/* shall be non-paged, and 4 bytes aligned */
 		u8 *cmd_allocated_buf;
 		u8 *rsp_buf;	/* shall be non-paged, and 4 bytes aligned */
-		u8 *rsp_allocated_buf;
 		u32 cmd_issued_cnt;
 		u32 cmd_done_cnt;
 		u32 rsp_cnt;
@@ -133,8 +132,8 @@ enum {
 	INTEl_WIDI_WK_CID,
 	C2H_WK_CID,
 	RTP_TIMER_CFG_WK_CID,
-	RESET_SECURITYPRIV, /*  add for CONFIG_IEEE80211W, none 11w also can use */
-	FREE_ASSOC_RESOURCES, /*  add for CONFIG_IEEE80211W, none 11w also can use */
+	RESET_SECURITYPRIV,
+	FREE_ASSOC_RESOURCES,
 	DM_IN_LPS_WK_CID,
 	DM_RA_MSK_WK_CID, /* add for STA update RAMask when bandwidth change. */
 	BEAMFORMING_WK_CID,
@@ -551,9 +550,6 @@ struct RunInThread_param {
 };
 
 
-#define GEN_CMD_CODE(cmd)	cmd ## _CMD_
-
-
 /*
 
 Result:
@@ -586,7 +582,7 @@ extern u8 rtw_clearstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8
 
 extern u8 rtw_joinbss_cmd(struct adapter *padapter, struct wlan_network *pnetwork);
 u8 rtw_disassoc_cmd(struct adapter *padapter, u32 deauth_timeout_ms, bool enqueue);
-extern u8 rtw_setopmode_cmd(struct adapter  *padapter, enum ndis_802_11_network_infrastructure networktype, bool enqueue);
+extern u8 rtw_setopmode_cmd(struct adapter  *padapter, enum nl80211_iftype networktype, bool enqueue);
 extern u8 rtw_setrfintfs_cmd(struct adapter  *padapter, u8 mode);
 
 extern u8 rtw_gettssi_cmd(struct adapter  *padapter, u8 offset, u8 *pval);
@@ -594,7 +590,6 @@ extern u8 rtw_setfwdig_cmd(struct adapter *padapter, u8 type);
 extern u8 rtw_setfwra_cmd(struct adapter *padapter, u8 type);
 
 extern u8 rtw_addbareq_cmd(struct adapter *padapter, u8 tid, u8 *addr);
-/*  add for CONFIG_IEEE80211W, none 11w also can use */
 extern u8 rtw_reset_securitypriv_cmd(struct adapter *padapter);
 extern u8 rtw_free_assoc_resources_cmd(struct adapter *padapter);
 extern u8 rtw_dynamic_chk_wk_cmd(struct adapter *adapter);
@@ -630,86 +625,81 @@ struct _cmd_callback {
 };
 
 enum {
-	GEN_CMD_CODE(_Read_MACREG),	/*0*/
-	GEN_CMD_CODE(_Write_MACREG),
-	GEN_CMD_CODE(_Read_BBREG),
-	GEN_CMD_CODE(_Write_BBREG),
-	GEN_CMD_CODE(_Read_RFREG),
-	GEN_CMD_CODE(_Write_RFREG), /*5*/
-	GEN_CMD_CODE(_Read_EEPROM),
-	GEN_CMD_CODE(_Write_EEPROM),
-	GEN_CMD_CODE(_Read_EFUSE),
-	GEN_CMD_CODE(_Write_EFUSE),
-
-	GEN_CMD_CODE(_Read_CAM),	/*10*/
-	GEN_CMD_CODE(_Write_CAM),
-	GEN_CMD_CODE(_setBCNITV),
-	GEN_CMD_CODE(_setMBIDCFG),
-	GEN_CMD_CODE(_JoinBss),   /*14*/
-	GEN_CMD_CODE(_DisConnect), /*15*/
-	GEN_CMD_CODE(_CreateBss),
-	GEN_CMD_CODE(_SetOpMode),
-	GEN_CMD_CODE(_SiteSurvey),  /*18*/
-	GEN_CMD_CODE(_SetAuth),
-
-	GEN_CMD_CODE(_SetKey),	/*20*/
-	GEN_CMD_CODE(_SetStaKey),
-	GEN_CMD_CODE(_SetAssocSta),
-	GEN_CMD_CODE(_DelAssocSta),
-	GEN_CMD_CODE(_SetStaPwrState),
-	GEN_CMD_CODE(_SetBasicRate), /*25*/
-	GEN_CMD_CODE(_GetBasicRate),
-	GEN_CMD_CODE(_SetDataRate),
-	GEN_CMD_CODE(_GetDataRate),
-	GEN_CMD_CODE(_SetPhyInfo),
-
-	GEN_CMD_CODE(_GetPhyInfo),	/*30*/
-	GEN_CMD_CODE(_SetPhy),
-	GEN_CMD_CODE(_GetPhy),
-	GEN_CMD_CODE(_readRssi),
-	GEN_CMD_CODE(_readGain),
-	GEN_CMD_CODE(_SetAtim), /*35*/
-	GEN_CMD_CODE(_SetPwrMode),
-	GEN_CMD_CODE(_JoinbssRpt),
-	GEN_CMD_CODE(_SetRaTable),
-	GEN_CMD_CODE(_GetRaTable),
-
-	GEN_CMD_CODE(_GetCCXReport), /*40*/
-	GEN_CMD_CODE(_GetDTMReport),
-	GEN_CMD_CODE(_GetTXRateStatistics),
-	GEN_CMD_CODE(_SetUsbSuspend),
-	GEN_CMD_CODE(_SetH2cLbk),
-	GEN_CMD_CODE(_AddBAReq), /*45*/
-	GEN_CMD_CODE(_SetChannel), /*46*/
-	GEN_CMD_CODE(_SetTxPower),
-	GEN_CMD_CODE(_SwitchAntenna),
-	GEN_CMD_CODE(_SetCrystalCap),
-	GEN_CMD_CODE(_SetSingleCarrierTx), /*50*/
-
-	GEN_CMD_CODE(_SetSingleToneTx),/*51*/
-	GEN_CMD_CODE(_SetCarrierSuppressionTx),
-	GEN_CMD_CODE(_SetContinuousTx),
-	GEN_CMD_CODE(_SwitchBandwidth), /*54*/
-	GEN_CMD_CODE(_TX_Beacon), /*55*/
-
-	GEN_CMD_CODE(_Set_MLME_EVT), /*56*/
-	GEN_CMD_CODE(_Set_Drv_Extra), /*57*/
-	GEN_CMD_CODE(_Set_H2C_MSG), /*58*/
-
-	GEN_CMD_CODE(_SetChannelPlan), /*59*/
-
-	GEN_CMD_CODE(_SetChannelSwitch), /*60*/
-	GEN_CMD_CODE(_TDLS), /*61*/
-	GEN_CMD_CODE(_ChkBMCSleepq), /*62*/
-
-	GEN_CMD_CODE(_RunInThreadCMD), /*63*/
+	READ_MACREG_CMD,	/*0*/
+	WRITE_MACREG_CMD,
+	READ_BBREG_CMD,
+	WRITE_BBREG_CMD,
+	READ_RFREG_CMD,
+	WRITE_RFREG_CMD, /*5*/
+	READ_EEPROM_CMD,
+	WRITE_EEPROM_CMD,
+	READ_EFUSE_CMD,
+	WRITE_EFUSE_CMD,
+
+	READ_CAM_CMD,	/*10*/
+	WRITE_CAM_CMD,
+	SET_BCNITV_CMD,
+	SET_MBIDCFG_CMD,
+	JOIN_BSS_CMD,   /*14*/
+	DISCONNECT_CMD, /*15*/
+	CREATE_BSS_CMD,
+	SET_OP_MODE_CMD,
+	SITE_SURVEY_CMD,  /*18*/
+	SET_AUTH_CMD,
+
+	SET_KEY_CMD,	/*20*/
+	SET_STA_KEY_CMD,
+	SET_ASSOC_STA_CMD,
+	DEL_ASSOC_STA_CMD,
+	SET_STA_PWR_STATE_CMD,
+	SET_BASIC_RATE_CMD, /*25*/
+	GET_BASIC_RATE_CMD,
+	SET_DATA_RATE_CMD,
+	GET_DATA_RATE_CMD,
+	SET_PHY_INFO_CMD,
+
+	GET_PHY_INFO_CMD,	/*30*/
+	SET_PHY_CMD,
+	GET_PHY_CMD,
+	READ_RSSI_CMD,
+	READ_GAIN_CMD,
+	SET_ATIM_CMD, /*35*/
+	SET_PWR_MODE_CMD,
+	JOIN_BSS_RPT_CMD,
+	SET_RA_TABLE_CMD,
+	GET_RA_TABLE_CMD,
+
+	GET_CCX_REPORT_CMD, /*40*/
+	GET_DTM_REPORT_CMD,
+	GET_TX_RATE_STATISTICS_CMD,
+	SET_USB_SUSPEND_CMD,
+	SET_H2C_LBK_CMD,
+	ADD_BA_REQ_CMD, /*45*/
+	SET_CHANNEL_CMD, /*46*/
+	SET_TX_POWER_CMD,
+	SWITCH_ANTENNA_CMD,
+	SET_CRYSTAL_CAP_CMD,
+	SET_SINGLE_CARRIER_TX_CMD, /*50*/
+
+	SET_SINGLE_TONE_TX_CMD, /*51*/
+	SET_CARRIER_SUPPRESSION_TX_CMD,
+	SET_CONTINUOUS_TX_CMD,
+	SWITCH_BANDWIDTH_CMD, /*54*/
+	TX_BEACON_CMD, /*55*/
+
+	SET_MLME_EVT_CMD, /*56*/
+	SET_DRV_EXTRA_CMD, /*57*/
+	SET_H2C_MSG_CMD, /*58*/
+
+	SET_CHANNEL_PLAN_CMD, /*59*/
+
+	SET_CHANNEL_SWITCH_CMD, /*60*/
+	TDLS_CMD, /*61*/
+	CHK_BMC_SLEEPQ_CMD, /*62*/
+
+	RUN_IN_THREAD_CMD, /*63*/
 
 	MAX_H2CCMD
 };
 
-#define _GetBBReg_CMD_		_Read_BBREG_CMD_
-#define _SetBBReg_CMD_		_Write_BBREG_CMD_
-#define _GetRFReg_CMD_		_Read_RFREG_CMD_
-#define _SetRFReg_CMD_		_Write_RFREG_CMD_
-
 #endif /* __RTW_CMD_H_ */
diff --git a/drivers/staging/rtl8723bs/include/rtw_event.h b/drivers/staging/rtl8723bs/include/rtw_event.h
index 62e0dec249adb..e5cb46c2a7316 100644
--- a/drivers/staging/rtl8723bs/include/rtw_event.h
+++ b/drivers/staging/rtl8723bs/include/rtw_event.h
@@ -64,10 +64,6 @@ struct wmm_event {
 	unsigned char wmm;
 };
 
-#define GEN_EVT_CODE(event)	event ## _EVT_
-
-
-
 struct fwevent {
 	u32 parmsize;
 	void (*event_callback)(struct adapter *dev, u8 *pbuf);
diff --git a/drivers/staging/rtl8723bs/include/rtw_ioctl_set.h b/drivers/staging/rtl8723bs/include/rtw_ioctl_set.h
index c3a1d1a84f789..768e4868ecfd0 100644
--- a/drivers/staging/rtl8723bs/include/rtw_ioctl_set.h
+++ b/drivers/staging/rtl8723bs/include/rtw_ioctl_set.h
@@ -13,7 +13,7 @@ u8 rtw_set_802_11_authentication_mode(struct adapter *pdapter, enum ndis_802_11_
 u8 rtw_set_802_11_add_wep(struct adapter *padapter, struct ndis_802_11_wep *wep);
 u8 rtw_set_802_11_disassociate(struct adapter *padapter);
 u8 rtw_set_802_11_bssid_list_scan(struct adapter *padapter, struct ndis_802_11_ssid *pssid, int ssid_max_num);
-u8 rtw_set_802_11_infrastructure_mode(struct adapter *padapter, enum ndis_802_11_network_infrastructure networktype);
+u8 rtw_set_802_11_infrastructure_mode(struct adapter *padapter, enum nl80211_iftype networktype);
 u8 rtw_set_802_11_ssid(struct adapter *padapter, struct ndis_802_11_ssid *ssid);
 u8 rtw_set_802_11_connect(struct adapter *padapter, u8 *bssid, struct ndis_802_11_ssid *ssid);
 
diff --git a/drivers/staging/rtl8723bs/include/rtw_mlme.h b/drivers/staging/rtl8723bs/include/rtw_mlme.h
index 8cc96164e1cf2..d1db75d50327f 100644
--- a/drivers/staging/rtl8723bs/include/rtw_mlme.h
+++ b/drivers/staging/rtl8723bs/include/rtw_mlme.h
@@ -13,8 +13,8 @@
 /* define   MAX_JOIN_TIMEOUT	2500 */
 #define   MAX_JOIN_TIMEOUT	6500
 
-/* 	Commented by Albert 20101105 */
-/* 	Increase the scanning timeout because of increasing the SURVEY_TO value. */
+/* Commented by Albert 20101105 */
+/* Increase the scanning timeout because of increasing the SURVEY_TO value. */
 
 #define		SCANNING_TIMEOUT	8000
 
@@ -44,15 +44,15 @@
 
 
 enum {
- dot11AuthAlgrthm_Open = 0,
- dot11AuthAlgrthm_Shared,
- dot11AuthAlgrthm_8021X,
- dot11AuthAlgrthm_Auto,
- dot11AuthAlgrthm_WAPI,
- dot11AuthAlgrthm_MaxNum
+	dot11AuthAlgrthm_Open = 0,
+	dot11AuthAlgrthm_Shared,
+	dot11AuthAlgrthm_8021X,
+	dot11AuthAlgrthm_Auto,
+	dot11AuthAlgrthm_WAPI,
+	dot11AuthAlgrthm_MaxNum
 };
 
-/*  Scan type including active and passive scan. */
+/* Scan type including active and passive scan. */
 enum rt_scan_type {
 	SCAN_PASSIVE,
 	SCAN_ACTIVE,
@@ -67,23 +67,22 @@ enum {
 };
 
 /*
-
-there are several "locks" in mlme_priv,
-since mlme_priv is a shared resource between many threads,
-like ISR/Call-Back functions, the OID handlers, and even timer functions.
-
-Each struct __queue has its own locks, already.
-Other items in mlme_priv are protected by mlme_priv.lock, while items in
-xmit_priv are protected by xmit_priv.lock.
-
-To avoid possible dead lock, any thread trying to modifying mlme_priv
-SHALL not lock up more than one locks at a time!
-
-The only exception is that queue functions which take the __queue.lock
-may be called with the xmit_priv.lock held. In this case the order
-MUST always be first lock xmit_priv.lock and then call any queue functions
-which take __queue.lock.
-*/
+ * There are several "locks" in mlme_priv,
+ * since mlme_priv is a shared resource between many threads,
+ * like ISR/Call-Back functions, the OID handlers, and even timer functions.
+ *
+ * Each struct __queue has its own locks, already.
+ * Other items in mlme_priv are protected by mlme_priv.lock, while items in
+ * xmit_priv are protected by xmit_priv.lock.
+ *
+ * To avoid possible dead lock, any thread trying to modifying mlme_priv
+ * SHALL not lock up more than one locks at a time!
+ *
+ * The only exception is that queue functions which take the __queue.lock
+ * may be called with the xmit_priv.lock held. In this case the order
+ * MUST always be first lock xmit_priv.lock and then call any queue functions
+ * which take __queue.lock.
+ */
 
 struct sitesurvey_ctrl {
 	u64	last_tx_pkts;
@@ -184,7 +183,8 @@ struct mlme_priv {
 	u32 wps_probe_req_ie_len;
 
 	/* Number of associated Non-ERP stations (i.e., stations using 802.11b
-	 * in 802.11g BSS) */
+	 * in 802.11g BSS)
+	 */
 	int num_sta_non_erp;
 
 	/* Number of associated stations that do not support Short Slot Time */
@@ -217,12 +217,12 @@ struct mlme_priv {
 	u8 *wps_beacon_ie;
 	/* u8 *wps_probe_req_ie; */
 	u8 *wps_probe_resp_ie;
-	u8 *wps_assoc_resp_ie; /*  for CONFIG_IOCTL_CFG80211, this IE could include p2p ie / wfd ie */
+	u8 *wps_assoc_resp_ie; /* this IE could include p2p ie / wfd ie */
 
 	u32 wps_beacon_ie_len;
 	/* u32 wps_probe_req_ie_len; */
 	u32 wps_probe_resp_ie_len;
-	u32 wps_assoc_resp_ie_len; /*  for CONFIG_IOCTL_CFG80211, this IE len could include p2p ie / wfd ie */
+	u32 wps_assoc_resp_ie_len; /* this IE len could include p2p ie / wfd ie */
 
 	u8 *p2p_beacon_ie;
 	u8 *p2p_probe_req_ie;
@@ -258,7 +258,6 @@ extern void rtw_surveydone_event_callback(struct adapter *adapter, u8 *pbuf);
 extern void rtw_joinbss_event_callback(struct adapter *adapter, u8 *pbuf);
 extern void rtw_stassoc_event_callback(struct adapter *adapter, u8 *pbuf);
 extern void rtw_stadel_event_callback(struct adapter *adapter, u8 *pbuf);
-extern void rtw_atimdone_event_callback(struct adapter *adapter, u8 *pbuf);
 extern void rtw_cpwm_event_callback(struct adapter *adapter, u8 *pbuf);
 extern void rtw_wmm_event_callback(struct adapter *padapter, u8 *pbuf);
 
@@ -279,7 +278,7 @@ extern signed int rtw_set_auth(struct adapter *adapter, struct security_priv *ps
 
 static inline u8 *get_bssid(struct mlme_priv *pmlmepriv)
 {	/* if sta_mode:pmlmepriv->cur_network.network.mac_address => bssid */
-	/*  if adhoc_mode:pmlmepriv->cur_network.network.mac_address => ibss mac address */
+	/* if adhoc_mode:pmlmepriv->cur_network.network.mac_address => ibss mac address */
 	return pmlmepriv->cur_network.network.mac_address;
 }
 
diff --git a/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h b/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h
index cb23c6939bfc5..bf5a7e9dbe0e2 100644
--- a/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h
+++ b/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h
@@ -322,8 +322,8 @@ struct mlme_ext_info {
 
 /*  The channel information about this channel including joining, scanning, and power constraints. */
 struct rt_channel_info {
-	u8 		ChannelNum;		/*  The channel number. */
-	enum rt_scan_type	ScanType;		/*  Scan type such as passive or active scan. */
+	u8 channel_num;		/*  The channel number. */
+	enum rt_scan_type scan_type;		/*  Scan type such as passive or active scan. */
 };
 
 int rtw_ch_set_search_ch(struct rt_channel_info *ch_set, const u32 ch);
@@ -675,33 +675,33 @@ void rtw_dummy_event_callback(struct adapter *adapter, u8 *pbuf);
 void rtw_fwdbg_event_callback(struct adapter *adapter, u8 *pbuf);
 
 enum {
-	GEN_EVT_CODE(_Read_MACREG) = 0, /*0*/
-	GEN_EVT_CODE(_Read_BBREG),
-	GEN_EVT_CODE(_Read_RFREG),
-	GEN_EVT_CODE(_Read_EEPROM),
-	GEN_EVT_CODE(_Read_EFUSE),
-	GEN_EVT_CODE(_Read_CAM),			/*5*/
-	GEN_EVT_CODE(_Get_BasicRate),
-	GEN_EVT_CODE(_Get_DataRate),
-	GEN_EVT_CODE(_Survey),	 /*8*/
-	GEN_EVT_CODE(_SurveyDone),	 /*9*/
-
-	GEN_EVT_CODE(_JoinBss), /*10*/
-	GEN_EVT_CODE(_AddSTA),
-	GEN_EVT_CODE(_DelSTA),
-	GEN_EVT_CODE(_AtimDone),
-	GEN_EVT_CODE(_TX_Report),
-	GEN_EVT_CODE(_CCX_Report),			/*15*/
-	GEN_EVT_CODE(_DTM_Report),
-	GEN_EVT_CODE(_TX_Rate_Statistics),
-	GEN_EVT_CODE(_C2HLBK),
-	GEN_EVT_CODE(_FWDBG),
-	GEN_EVT_CODE(_C2HFEEDBACK),               /*20*/
-	GEN_EVT_CODE(_ADDBA),
-	GEN_EVT_CODE(_C2HBCN),
-	GEN_EVT_CODE(_ReportPwrState),		/* filen: only for PCIE, USB */
-	GEN_EVT_CODE(_CloseRF),				/* filen: only for PCIE, work around ASPM */
-	GEN_EVT_CODE(_WMM),					/*25*/
+	READ_MACREG_EVENT = 0,		/*0*/
+	READ_BBREG_EVENT,
+	READ_RFREG_EVENT,
+	READ_EEPROM_EVENT,
+	READ_EFUSE_EVENT,
+	READ_CAM_EVENT,			/*5*/
+	GET_BASICRATE_EVENT,
+	GET_DATARATE_EVENT,
+	SURVEY_EVENT,			/*8*/
+	SURVEY_DONE_EVENT,		/*9*/
+
+	JOIN_BSS_EVENT,			/*10*/
+	ADD_STA_EVENT,
+	DEL_STA_EVENT,
+	ATIM_DONE_EVENT,
+	TX_REPORT_EVENT,
+	CCX_REPORT_EVENT,		/*15*/
+	DTM_REPORT_EVENT,
+	TX_RATE_STATISTICS_EVENT,
+	C2HLBK_EVENT,
+	FWDBG_EVENT,
+	C2HFEEDBACK_EVENT,		/*20*/
+	ADDBA_EVENT,
+	C2HBCN_EVENT,
+	REPORT_PWR_STATE_EVENT,		/* filen: only for PCIE, USB */
+	CLOSE_RF_EVENT,			/* filen: only for PCIE, work around ASPM */
+	WMM_EVENT,			/*25*/
 	MAX_C2HEVT
 };
 
diff --git a/drivers/staging/rtl8723bs/include/rtw_pwrctrl.h b/drivers/staging/rtl8723bs/include/rtw_pwrctrl.h
index 6977892a57420..182b8c88ccc83 100644
--- a/drivers/staging/rtl8723bs/include/rtw_pwrctrl.h
+++ b/drivers/staging/rtl8723bs/include/rtw_pwrctrl.h
@@ -215,8 +215,8 @@ struct pwrctrl_priv {
 extern void rtw_init_pwrctrl_priv(struct adapter *adapter);
 extern void rtw_free_pwrctrl_priv(struct adapter *adapter);
 
-s32 rtw_register_task_alive(struct adapter *, u32 task);
-void rtw_unregister_task_alive(struct adapter *, u32 task);
+s32 rtw_register_task_alive(struct adapter *adapter, u32 task);
+void rtw_unregister_task_alive(struct adapter *adapter, u32 task);
 extern s32 rtw_register_tx_alive(struct adapter *padapter);
 extern void rtw_unregister_tx_alive(struct adapter *padapter);
 extern s32 rtw_register_cmd_alive(struct adapter *padapter);
@@ -233,7 +233,7 @@ int ips_leave(struct adapter *padapter);
 
 void rtw_ps_processor(struct adapter *padapter);
 
-s32 LPS_RF_ON_check(struct adapter *padapter, u32 delay_ms);
+int LPS_RF_ON_check(struct adapter *padapter, u32 delay_ms);
 void LPS_Enter(struct adapter *padapter, const char *msg);
 void LPS_Leave(struct adapter *padapter, const char *msg);
 void traffic_check_for_leave_lps(struct adapter *padapter, u8 tx, u32 tx_packets);
diff --git a/drivers/staging/rtl8723bs/include/rtw_security.h b/drivers/staging/rtl8723bs/include/rtw_security.h
index 32f6d3a5e3092..cc7e27603690f 100644
--- a/drivers/staging/rtl8723bs/include/rtw_security.h
+++ b/drivers/staging/rtl8723bs/include/rtw_security.h
@@ -91,8 +91,8 @@ struct rt_pmkid_list {
 
 
 struct security_priv {
-	u32   dot11AuthAlgrthm;		/*  802.11 auth, could be open, shared, 8021x and authswitch */
-	u32   dot11PrivacyAlgrthm;	/*  This specify the privacy for shared auth. algorithm. */
+	u32   dot11_auth_algrthm;	/*  802.11 auth, could be open, shared, 8021x and authswitch */
+	u32   dot11_privacy_algrthm;	/*  This specify the privacy for shared auth. algorithm. */
 
 	/* WEP */
 	u32   dot11PrivacyKeyIndex;	/*  this is only valid for legendary wep, 0~3 for key id. (tx key index) */
@@ -170,11 +170,11 @@ struct security_priv {
 
 #define GET_ENCRY_ALGO(psecuritypriv, psta, encry_algo, bmcst)\
 do {\
-	switch (psecuritypriv->dot11AuthAlgrthm) {\
+	switch (psecuritypriv->dot11_auth_algrthm) {\
 	case dot11AuthAlgrthm_Open:\
 	case dot11AuthAlgrthm_Shared:\
 	case dot11AuthAlgrthm_Auto:\
-		encry_algo = (u8)psecuritypriv->dot11PrivacyAlgrthm;\
+		encry_algo = (u8)psecuritypriv->dot11_privacy_algrthm;\
 		break;\
 	case dot11AuthAlgrthm_8021X:\
 		if (bmcst)\
@@ -183,7 +183,7 @@ do {\
 			encry_algo = (u8)psta->dot118021XPrivacy;\
 		break;\
 	case dot11AuthAlgrthm_WAPI:\
-		encry_algo = (u8)psecuritypriv->dot11PrivacyAlgrthm;\
+		encry_algo = (u8)psecuritypriv->dot11_privacy_algrthm;\
 		break;\
 	} \
 } while (0)
diff --git a/drivers/staging/rtl8723bs/include/sta_info.h b/drivers/staging/rtl8723bs/include/sta_info.h
index 63343998266ad..0f53e77c110c5 100644
--- a/drivers/staging/rtl8723bs/include/sta_info.h
+++ b/drivers/staging/rtl8723bs/include/sta_info.h
@@ -214,7 +214,7 @@ struct sta_info {
 	/*  */
 
 	/* To store the sequence number of received management frame */
-	u16 RxMgmtFrameSeqNum;
+	u16 rx_mgmt_frame_seq_num;
 };
 
 #define sta_rx_pkts(sta) \
diff --git a/drivers/staging/rtl8723bs/include/wifi.h b/drivers/staging/rtl8723bs/include/wifi.h
index c8844e2bf7b45..d5132bc4280a8 100644
--- a/drivers/staging/rtl8723bs/include/wifi.h
+++ b/drivers/staging/rtl8723bs/include/wifi.h
@@ -329,21 +329,21 @@ static inline int IsFrameTypeCtrl(unsigned char *pframe)
  */
 struct ieee80211_ht_addt_info {
 	unsigned char control_chan;
-	unsigned char 	ht_param;
-	__le16	operation_mode;
-	__le16	stbc_param;
-	unsigned char 	basic_set[16];
+	unsigned char ht_param;
+	__le16 operation_mode;
+	__le16 stbc_param;
+	unsigned char basic_set[16];
 } __packed;
 
 
 struct HT_caps_element {
 	union {
 		struct {
-			__le16	HT_caps_info;
+			__le16 HT_caps_info;
 			unsigned char AMPDU_para;
 			unsigned char MCS_rate[16];
-			__le16	HT_ext_caps;
-			__le16	Beamforming_caps;
+			__le16 HT_ext_caps;
+			__le16 Beamforming_caps;
 			unsigned char ASEL_caps;
 		} HT_cap_element;
 		unsigned char HT_cap[26];
@@ -357,22 +357,22 @@ struct HT_info_element {
 } __packed;
 
 struct AC_param {
-	unsigned char 	ACI_AIFSN;
-	unsigned char 	CW;
-	__le16	TXOP_limit;
+	unsigned char ACI_AIFSN;
+	unsigned char CW;
+	__le16 TXOP_limit;
 } __packed;
 
 struct WMM_para_element {
-	unsigned char 	QoS_info;
-	unsigned char 	reserved;
-	struct AC_param	ac_param[4];
+	unsigned char QoS_info;
+	unsigned char reserved;
+	struct AC_param ac_param[4];
 } __packed;
 
 struct ADDBA_request {
-	unsigned char 	dialog_token;
-	__le16	BA_para_set;
-	__le16	BA_timeout_value;
-	__le16	BA_starting_seqctrl;
+	unsigned char dialog_token;
+	__le16 BA_para_set;
+	__le16 BA_timeout_value;
+	__le16 BA_starting_seqctrl;
 } __packed;
 
 /* 802.11n HT capabilities masks */
diff --git a/drivers/staging/rtl8723bs/include/wlan_bssdef.h b/drivers/staging/rtl8723bs/include/wlan_bssdef.h
index 812a683942689..6da1579149251 100644
--- a/drivers/staging/rtl8723bs/include/wlan_bssdef.h
+++ b/drivers/staging/rtl8723bs/include/wlan_bssdef.h
@@ -23,18 +23,10 @@ struct ndis_802_11_ssid {
 	u8  ssid[32];
 };
 
-enum ndis_802_11_network_type {
-	Ndis802_11FH,
-	Ndis802_11DS,
-	Ndis802_11OFDM5,
-	Ndis802_11OFDM24,
-	Ndis802_11NetworkTypeMax    /*  not a real type, defined as an upper bound */
-};
-
 /*
-	FW will only save the channel number in DSConfig.
-	ODI Handler will convert the channel number to freq. number.
-*/
+ * FW will only save the channel number in DSConfig.
+ * ODI Handler will convert the channel number to freq. number.
+ */
 struct ndis_802_11_conf {
 	u32 length;             /*  Length of structure */
 	u32 beacon_period;       /*  units are Kusec */
@@ -42,14 +34,6 @@ struct ndis_802_11_conf {
 	u32 ds_config;           /*  Frequency, units are kHz */
 };
 
-enum ndis_802_11_network_infrastructure {
-	Ndis802_11IBSS,
-	Ndis802_11Infrastructure,
-	Ndis802_11AutoUnknown,
-	Ndis802_11InfrastructureMax,     /*  Not a real value, defined as upper bound */
-	Ndis802_11APMode,
-};
-
 struct ndis_802_11_fix_ie {
 	u8  time_stamp[8];
 	u16  beacon_interval;
@@ -65,7 +49,7 @@ struct ndis_80211_var_ie {
 /* Length is the 4 bytes multiples of the sum of
  * ETH_ALEN + 2 +
  * sizeof (struct ndis_802_11_ssid) + sizeof (u32) +
- * sizeof (long) + sizeof (enum ndis_802_11_network_type) +
+ * sizeof (long) +
  * sizeof (struct ndis_802_11_conf) + sizeof (NDIS_802_11_RATES_EX) + ie_length
  *
  * Except for ie_length, all other fields are fixed length. Therefore, we can
@@ -125,10 +109,6 @@ struct ndis_802_11_wep {
 /*  MIC check time, 60 seconds. */
 #define MIC_CHECK_TIME	60000000
 
-#ifndef Ndis802_11APMode
-#define Ndis802_11APMode (Ndis802_11InfrastructureMax + 1)
-#endif
-
 struct wlan_phy_info {
 	u8 signal_strength;/* in percentage) */
 	u8 signal_quality;/* in percentage) */
@@ -138,7 +118,8 @@ struct wlan_phy_info {
 
 struct wlan_bcn_info {
 	/* these infor get from rtw_get_encrypt_info when
-	 * * translate scan to UI */
+	 * translate scan to UI
+	 */
 	u8 encryp_protocol;/* ENCRYP_PROTOCOL_E: OPEN/WEP/WPA/WPA2/WAPI */
 	int group_cipher; /* WPA/WPA2 group cipher */
 	int pairwise_cipher;/* WPA/WPA2/WEP pairwise cipher */
@@ -150,8 +131,8 @@ struct wlan_bcn_info {
 };
 
 /* temporally add #pragma pack for structure alignment issue of
-*   struct wlan_bssid_ex and get_wlan_bssid_ex_sz()
-*/
+ * struct wlan_bssid_ex and get_wlan_bssid_ex_sz()
+ */
 struct wlan_bssid_ex {
 	u32  length;
 	u8 mac_address[ETH_ALEN];
@@ -159,9 +140,8 @@ struct wlan_bssid_ex {
 	struct ndis_802_11_ssid  ssid;
 	u32  privacy;
 	long  rssi;/* in dBM, raw data , get from PHY) */
-	enum ndis_802_11_network_type  network_type_in_use;
 	struct ndis_802_11_conf  configuration;
-	enum ndis_802_11_network_infrastructure  infrastructure_mode;
+	enum nl80211_iftype  infrastructure_mode;
 	NDIS_802_11_RATES_EX  supported_rates;
 	struct wlan_phy_info phy_info;
 	u32  ie_length;
diff --git a/drivers/staging/rtl8723bs/include/xmit_osdep.h b/drivers/staging/rtl8723bs/include/xmit_osdep.h
index 5b351652e31bd..ee6616880d415 100644
--- a/drivers/staging/rtl8723bs/include/xmit_osdep.h
+++ b/drivers/staging/rtl8723bs/include/xmit_osdep.h
@@ -32,7 +32,7 @@ void rtw_os_xmit_schedule(struct adapter *padapter);
 
 void rtw_os_xmit_resource_free(struct adapter *padapter, struct xmit_buf *pxmitbuf, u32 free_sz, u8 flag);
 
-extern uint rtw_remainder_len(struct pkt_file *pfile);
+extern unsigned int rtw_remainder_len(struct pkt_file *pfile);
 extern void _rtw_open_pktfile(struct sk_buff *pkt, struct pkt_file *pfile);
 int _rtw_pktfile_read(struct pkt_file *pfile, u8 *rmem, unsigned int rlen);
 extern signed int rtw_endofpktfile(struct pkt_file *pfile);
diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
index 6a97afd89dc78..f022931e4af92 100644
--- a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
+++ b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
@@ -110,33 +110,26 @@ static struct ieee80211_supported_band *rtw_spt_band_alloc(
 	)
 {
 	struct ieee80211_supported_band *spt_band = NULL;
-	int n_channels, n_bitrates;
-	size_t alloc_sz;
+	size_t channels_sz, bitrate_size;
 
-	if (band == NL80211_BAND_2GHZ) {
-		n_channels = RTW_2G_CHANNELS_NUM;
-		n_bitrates = RTW_G_RATES_NUM;
-	} else {
+	if (band != NL80211_BAND_2GHZ)
 		goto exit;
-	}
 
-	alloc_sz = sizeof(*spt_band);
-	alloc_sz = size_add(alloc_sz, array_size(n_channels, sizeof(struct ieee80211_channel)));
-	alloc_sz = size_add(alloc_sz, array_size(n_bitrates, sizeof(struct ieee80211_rate)));
-	spt_band = kzalloc(alloc_sz, GFP_KERNEL);
+	channels_sz = array_size(RTW_2G_CHANNELS_NUM, sizeof(struct ieee80211_channel));
+	bitrate_size = array_size(RTW_G_RATES_NUM, sizeof(struct ieee80211_rate));
+
+	spt_band = kzalloc(sizeof(*spt_band) + channels_sz + bitrate_size, GFP_KERNEL);
 	if (!spt_band)
 		goto exit;
 
-	spt_band->channels = (struct ieee80211_channel *)(((u8 *)spt_band) + sizeof(struct ieee80211_supported_band));
-	spt_band->bitrates = (struct ieee80211_rate *)(((u8 *)spt_band->channels) + sizeof(struct ieee80211_channel) * n_channels);
+	spt_band->channels = (void *)spt_band + sizeof(*spt_band);
+	spt_band->bitrates = (void *)spt_band + sizeof(*spt_band) + channels_sz;
 	spt_band->band = band;
-	spt_band->n_channels = n_channels;
-	spt_band->n_bitrates = n_bitrates;
+	spt_band->n_channels = RTW_2G_CHANNELS_NUM;
+	spt_band->n_bitrates = RTW_G_RATES_NUM;
 
-	if (band == NL80211_BAND_2GHZ) {
-		rtw_2g_channels_init(spt_band->channels);
-		rtw_2g_rates_init(spt_band->bitrates);
-	}
+	rtw_2g_channels_init(spt_band->channels);
+	rtw_2g_rates_init(spt_band->bitrates);
 
 	/* spt_band.ht_cap */
 
@@ -546,13 +539,13 @@ static int rtw_cfg80211_ap_set_encryption(struct net_device *dev, struct ieee_pa
 		if (psecuritypriv->bWepDefaultKeyIdxSet == 0) {
 			/* wep default key has not been set, so use this key index as default key. */
 
-			psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Auto;
+			psecuritypriv->dot11_auth_algrthm = dot11AuthAlgrthm_Auto;
 			psecuritypriv->ndisencryptstatus = Ndis802_11Encryption1Enabled;
-			psecuritypriv->dot11PrivacyAlgrthm = _WEP40_;
+			psecuritypriv->dot11_privacy_algrthm = _WEP40_;
 			psecuritypriv->dot118021XGrpPrivacy = _WEP40_;
 
 			if (wep_key_len == 13) {
-				psecuritypriv->dot11PrivacyAlgrthm = _WEP104_;
+				psecuritypriv->dot11_privacy_algrthm = _WEP104_;
 				psecuritypriv->dot118021XGrpPrivacy = _WEP104_;
 			}
 
@@ -602,7 +595,7 @@ static int rtw_cfg80211_ap_set_encryption(struct net_device *dev, struct ieee_pa
 
 			psecuritypriv->binstallGrpkey = true;
 
-			psecuritypriv->dot11PrivacyAlgrthm = psecuritypriv->dot118021XGrpPrivacy;/*  */
+			psecuritypriv->dot11_privacy_algrthm = psecuritypriv->dot118021XGrpPrivacy;/*  */
 
 			rtw_ap_set_group_key(padapter, param->u.crypt.key, psecuritypriv->dot118021XGrpPrivacy, param->u.crypt.idx);
 
@@ -616,7 +609,7 @@ static int rtw_cfg80211_ap_set_encryption(struct net_device *dev, struct ieee_pa
 		goto exit;
 	}
 
-	if (psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_8021X && psta) { /*  psk/802_1x */
+	if (psecuritypriv->dot11_auth_algrthm == dot11AuthAlgrthm_8021X && psta) { /*  psk/802_1x */
 		if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
 			if (param->u.crypt.set_tx == 1) { /* pairwise key */
 				memcpy(psta->dot118021x_UncstKey.skey, param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
@@ -676,7 +669,7 @@ static int rtw_cfg80211_ap_set_encryption(struct net_device *dev, struct ieee_pa
 
 				psecuritypriv->binstallGrpkey = true;
 
-				psecuritypriv->dot11PrivacyAlgrthm = psecuritypriv->dot118021XGrpPrivacy;/*  */
+				psecuritypriv->dot11_privacy_algrthm = psecuritypriv->dot118021XGrpPrivacy;/*  */
 
 				rtw_ap_set_group_key(padapter, param->u.crypt.key, psecuritypriv->dot118021XGrpPrivacy, param->u.crypt.idx);
 
@@ -744,11 +737,11 @@ static int rtw_cfg80211_set_encryption(struct net_device *dev, struct ieee_param
 			wep_key_len = wep_key_len <= 5 ? 5 : 13;
 
 			psecuritypriv->ndisencryptstatus = Ndis802_11Encryption1Enabled;
-			psecuritypriv->dot11PrivacyAlgrthm = _WEP40_;
+			psecuritypriv->dot11_privacy_algrthm = _WEP40_;
 			psecuritypriv->dot118021XGrpPrivacy = _WEP40_;
 
 			if (wep_key_len == 13) {
-				psecuritypriv->dot11PrivacyAlgrthm = _WEP104_;
+				psecuritypriv->dot11_privacy_algrthm = _WEP104_;
 				psecuritypriv->dot118021XGrpPrivacy = _WEP104_;
 			}
 
@@ -764,7 +757,7 @@ static int rtw_cfg80211_set_encryption(struct net_device *dev, struct ieee_param
 		goto exit;
 	}
 
-	if (padapter->securitypriv.dot11AuthAlgrthm == dot11AuthAlgrthm_8021X) { /*  802_1x */
+	if (padapter->securitypriv.dot11_auth_algrthm == dot11AuthAlgrthm_8021X) { /*  802_1x */
 		struct sta_info *psta, *pbcmc_sta;
 		struct sta_priv *pstapriv = &padapter->stapriv;
 
@@ -777,7 +770,7 @@ static int rtw_cfg80211_set_encryption(struct net_device *dev, struct ieee_param
 
 				if ((padapter->securitypriv.ndisencryptstatus == Ndis802_11Encryption2Enabled) ||
 				    (padapter->securitypriv.ndisencryptstatus ==  Ndis802_11Encryption3Enabled)) {
-					psta->dot118021XPrivacy = padapter->securitypriv.dot11PrivacyAlgrthm;
+					psta->dot118021XPrivacy = padapter->securitypriv.dot11_privacy_algrthm;
 				}
 
 				if (param->u.crypt.set_tx == 1) { /* pairwise key */
@@ -819,7 +812,7 @@ static int rtw_cfg80211_set_encryption(struct net_device *dev, struct ieee_param
 
 				if ((padapter->securitypriv.ndisencryptstatus == Ndis802_11Encryption2Enabled) ||
 				    (padapter->securitypriv.ndisencryptstatus ==  Ndis802_11Encryption3Enabled)) {
-					pbcmc_sta->dot118021XPrivacy = padapter->securitypriv.dot11PrivacyAlgrthm;
+					pbcmc_sta->dot118021XPrivacy = padapter->securitypriv.dot11_privacy_algrthm;
 				}
 			}
 		} else if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) { /* adhoc mode */
@@ -942,15 +935,15 @@ static int cfg80211_rtw_set_default_key(struct wiphy *wiphy,
 	struct adapter *padapter = rtw_netdev_priv(ndev);
 	struct security_priv *psecuritypriv = &padapter->securitypriv;
 
-	if ((key_index < WEP_KEYS) && ((psecuritypriv->dot11PrivacyAlgrthm == _WEP40_) || (psecuritypriv->dot11PrivacyAlgrthm == _WEP104_))) { /* set wep default key */
+	if ((key_index < WEP_KEYS) && ((psecuritypriv->dot11_privacy_algrthm == _WEP40_) || (psecuritypriv->dot11_privacy_algrthm == _WEP104_))) { /* set wep default key */
 		psecuritypriv->ndisencryptstatus = Ndis802_11Encryption1Enabled;
 
 		psecuritypriv->dot11PrivacyKeyIndex = key_index;
 
-		psecuritypriv->dot11PrivacyAlgrthm = _WEP40_;
+		psecuritypriv->dot11_privacy_algrthm = _WEP40_;
 		psecuritypriv->dot118021XGrpPrivacy = _WEP40_;
 		if (psecuritypriv->dot11DefKeylen[key_index] == 13) {
-			psecuritypriv->dot11PrivacyAlgrthm = _WEP104_;
+			psecuritypriv->dot11_privacy_algrthm = _WEP104_;
 			psecuritypriv->dot118021XGrpPrivacy = _WEP104_;
 		}
 
@@ -1027,7 +1020,6 @@ static int cfg80211_rtw_change_iface(struct wiphy *wiphy,
 				     struct vif_params *params)
 {
 	enum nl80211_iftype old_type;
-	enum ndis_802_11_network_infrastructure networkType;
 	struct adapter *padapter = rtw_netdev_priv(ndev);
 	struct wireless_dev *rtw_wdev = padapter->rtw_wdev;
 	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
@@ -1057,30 +1049,22 @@ static int cfg80211_rtw_change_iface(struct wiphy *wiphy,
 		pmlmeext->action_public_dialog_token = 0xff;
 	}
 
-	switch (type) {
-	case NL80211_IFTYPE_ADHOC:
-		networkType = Ndis802_11IBSS;
-		break;
-	case NL80211_IFTYPE_STATION:
-		networkType = Ndis802_11Infrastructure;
-		break;
-	case NL80211_IFTYPE_AP:
-		networkType = Ndis802_11APMode;
-		break;
-	default:
+	if (type != NL80211_IFTYPE_ADHOC &&
+	    type != NL80211_IFTYPE_STATION &&
+	    type != NL80211_IFTYPE_AP) {
 		ret = -EOPNOTSUPP;
 		goto exit;
 	}
 
 	rtw_wdev->iftype = type;
 
-	if (!rtw_set_802_11_infrastructure_mode(padapter, networkType)) {
+	if (!rtw_set_802_11_infrastructure_mode(padapter, type)) {
 		rtw_wdev->iftype = old_type;
 		ret = -EPERM;
 		goto exit;
 	}
 
-	rtw_setopmode_cmd(padapter, networkType, true);
+	rtw_setopmode_cmd(padapter, type, true);
 
 exit:
 
@@ -1152,7 +1136,7 @@ void rtw_cfg80211_surveydone_event_callback(struct adapter *padapter)
 static int rtw_cfg80211_set_probe_req_wpsp2pie(struct adapter *padapter, char *buf, int len)
 {
 	int ret = 0;
-	uint wps_ielen = 0;
+	unsigned int wps_ielen = 0;
 	u8 *wps_ie;
 	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
 
@@ -1326,26 +1310,26 @@ static int rtw_cfg80211_set_auth_type(struct security_priv *psecuritypriv,
 	switch (sme_auth_type) {
 	case NL80211_AUTHTYPE_AUTOMATIC:
 
-		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Auto;
+		psecuritypriv->dot11_auth_algrthm = dot11AuthAlgrthm_Auto;
 
 		break;
 	case NL80211_AUTHTYPE_OPEN_SYSTEM:
 
-		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open;
+		psecuritypriv->dot11_auth_algrthm = dot11AuthAlgrthm_Open;
 
 		if (psecuritypriv->ndisauthtype > Ndis802_11AuthModeWPA)
-			psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+			psecuritypriv->dot11_auth_algrthm = dot11AuthAlgrthm_8021X;
 
 		break;
 	case NL80211_AUTHTYPE_SHARED_KEY:
 
-		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Shared;
+		psecuritypriv->dot11_auth_algrthm = dot11AuthAlgrthm_Shared;
 
 		psecuritypriv->ndisencryptstatus = Ndis802_11Encryption1Enabled;
 
 		break;
 	default:
-		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open;
+		psecuritypriv->dot11_auth_algrthm = dot11AuthAlgrthm_Open;
 		/* return -ENOTSUPP; */
 	}
 
@@ -1356,7 +1340,7 @@ static int rtw_cfg80211_set_cipher(struct security_priv *psecuritypriv, u32 ciph
 {
 	u32 ndisencryptstatus = Ndis802_11EncryptionDisabled;
 
-	u32 *profile_cipher = ucast ? &psecuritypriv->dot11PrivacyAlgrthm :
+	u32 *profile_cipher = ucast ? &psecuritypriv->dot11_privacy_algrthm :
 		&psecuritypriv->dot118021XGrpPrivacy;
 
 	if (!cipher) {
@@ -1393,7 +1377,7 @@ static int rtw_cfg80211_set_cipher(struct security_priv *psecuritypriv, u32 ciph
 	if (ucast) {
 		psecuritypriv->ndisencryptstatus = ndisencryptstatus;
 
-		/* if (psecuritypriv->dot11PrivacyAlgrthm >= _AES_) */
+		/* if (psecuritypriv->dot11_privacy_algrthm >= _AES_) */
 		/*	psecuritypriv->ndisauthtype = Ndis802_11AuthModeWPA2PSK; */
 	}
 
@@ -1404,9 +1388,9 @@ static int rtw_cfg80211_set_key_mgt(struct security_priv *psecuritypriv, u32 key
 {
 	if (key_mgt == WLAN_AKM_SUITE_8021X)
 		/* auth_type = UMAC_AUTH_TYPE_8021X; */
-		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+		psecuritypriv->dot11_auth_algrthm = dot11AuthAlgrthm_8021X;
 	else if (key_mgt == WLAN_AKM_SUITE_PSK) {
-		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+		psecuritypriv->dot11_auth_algrthm = dot11AuthAlgrthm_8021X;
 	}
 
 	return 0;
@@ -1451,7 +1435,7 @@ static int rtw_cfg80211_set_wpa_ie(struct adapter *padapter, u8 *pie, size_t iel
 			goto exit;
 		}
 		if (rtw_parse_wpa_ie(pwpa, wpa_ielen + 2, &group_cipher, &pairwise_cipher, NULL) == _SUCCESS) {
-			padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+			padapter->securitypriv.dot11_auth_algrthm = dot11AuthAlgrthm_8021X;
 			padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeWPAPSK;
 			memcpy(padapter->securitypriv.supplicant_ie, &pwpa[0], wpa_ielen + 2);
 		}
@@ -1464,7 +1448,7 @@ static int rtw_cfg80211_set_wpa_ie(struct adapter *padapter, u8 *pie, size_t iel
 			goto exit;
 		}
 		if (rtw_parse_wpa2_ie(pwpa2, wpa2_ielen + 2, &group_cipher, &pairwise_cipher, NULL) == _SUCCESS) {
-			padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
+			padapter->securitypriv.dot11_auth_algrthm = dot11AuthAlgrthm_8021X;
 			padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeWPA2PSK;
 			memcpy(padapter->securitypriv.supplicant_ie, &pwpa2[0], wpa2_ielen + 2);
 		}
@@ -1501,34 +1485,34 @@ static int rtw_cfg80211_set_wpa_ie(struct adapter *padapter, u8 *pie, size_t iel
 
 	switch (pairwise_cipher) {
 	case WPA_CIPHER_NONE:
-		padapter->securitypriv.dot11PrivacyAlgrthm = _NO_PRIVACY_;
+		padapter->securitypriv.dot11_privacy_algrthm = _NO_PRIVACY_;
 		padapter->securitypriv.ndisencryptstatus = Ndis802_11EncryptionDisabled;
 		break;
 	case WPA_CIPHER_WEP40:
-		padapter->securitypriv.dot11PrivacyAlgrthm = _WEP40_;
+		padapter->securitypriv.dot11_privacy_algrthm = _WEP40_;
 		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;
 		break;
 	case WPA_CIPHER_TKIP:
-		padapter->securitypriv.dot11PrivacyAlgrthm = _TKIP_;
+		padapter->securitypriv.dot11_privacy_algrthm = _TKIP_;
 		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption2Enabled;
 		break;
 	case WPA_CIPHER_CCMP:
-		padapter->securitypriv.dot11PrivacyAlgrthm = _AES_;
+		padapter->securitypriv.dot11_privacy_algrthm = _AES_;
 		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption3Enabled;
 		break;
 	case WPA_CIPHER_WEP104:
-		padapter->securitypriv.dot11PrivacyAlgrthm = _WEP104_;
+		padapter->securitypriv.dot11_privacy_algrthm = _WEP104_;
 		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;
 		break;
 	}
 
 	{/* handle wps_ie */
-		uint wps_ielen;
+		unsigned int wps_ielen;
 		u8 *wps_ie;
 
 		wps_ie = rtw_get_wps_ie(buf, ielen, NULL, &wps_ielen);
 		if (wps_ie && wps_ielen > 0) {
-			padapter->securitypriv.wps_ie_len = min_t(uint, wps_ielen, MAX_WPS_IE_LEN);
+			padapter->securitypriv.wps_ie_len = min_t(unsigned int, wps_ielen, MAX_WPS_IE_LEN);
 			memcpy(padapter->securitypriv.wps_ie, wps_ie, padapter->securitypriv.wps_ie_len);
 			set_fwstate(&padapter->mlmepriv, WIFI_UNDER_WPS);
 		} else {
@@ -1537,9 +1521,9 @@ static int rtw_cfg80211_set_wpa_ie(struct adapter *padapter, u8 *pie, size_t iel
 	}
 
 	/* TKIP and AES disallow multicast packets until installing group key */
-	if (padapter->securitypriv.dot11PrivacyAlgrthm == _TKIP_
-		|| padapter->securitypriv.dot11PrivacyAlgrthm == _TKIP_WTMIC_
-		|| padapter->securitypriv.dot11PrivacyAlgrthm == _AES_)
+	if (padapter->securitypriv.dot11_privacy_algrthm == _TKIP_
+		|| padapter->securitypriv.dot11_privacy_algrthm == _TKIP_WTMIC_
+		|| padapter->securitypriv.dot11_privacy_algrthm == _AES_)
 		/* WPS open need to enable multicast */
 		/*  check_fwstate(&padapter->mlmepriv, WIFI_UNDER_WPS) == true) */
 		rtw_hal_set_hwreg(padapter, HW_VAR_OFF_RCR_AM, null_addr);
@@ -1585,9 +1569,9 @@ static int cfg80211_rtw_join_ibss(struct wiphy *wiphy, struct net_device *ndev,
 	memcpy(ndis_ssid.ssid, (u8 *)params->ssid, params->ssid_len);
 
 	psecuritypriv->ndisencryptstatus = Ndis802_11EncryptionDisabled;
-	psecuritypriv->dot11PrivacyAlgrthm = _NO_PRIVACY_;
+	psecuritypriv->dot11_privacy_algrthm = _NO_PRIVACY_;
 	psecuritypriv->dot118021XGrpPrivacy = _NO_PRIVACY_;
-	psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open; /* open system */
+	psecuritypriv->dot11_auth_algrthm = dot11AuthAlgrthm_Open; /* open system */
 	psecuritypriv->ndisauthtype = Ndis802_11AuthModeOpen;
 
 	ret = rtw_cfg80211_set_auth_type(psecuritypriv, NL80211_AUTHTYPE_OPEN_SYSTEM);
@@ -1619,12 +1603,12 @@ static int cfg80211_rtw_leave_ibss(struct wiphy *wiphy, struct net_device *ndev)
 
 		rtw_wdev->iftype = NL80211_IFTYPE_STATION;
 
-		if (!rtw_set_802_11_infrastructure_mode(padapter, Ndis802_11Infrastructure)) {
+		if (!rtw_set_802_11_infrastructure_mode(padapter, NL80211_IFTYPE_STATION)) {
 			rtw_wdev->iftype = old_type;
 			ret = -EPERM;
 			goto leave_ibss;
 		}
-		rtw_setopmode_cmd(padapter, Ndis802_11Infrastructure, true);
+		rtw_setopmode_cmd(padapter, NL80211_IFTYPE_STATION, true);
 	}
 
 leave_ibss:
@@ -1681,9 +1665,9 @@ static int cfg80211_rtw_connect(struct wiphy *wiphy, struct net_device *ndev,
 		rtw_scan_abort(padapter);
 
 	psecuritypriv->ndisencryptstatus = Ndis802_11EncryptionDisabled;
-	psecuritypriv->dot11PrivacyAlgrthm = _NO_PRIVACY_;
+	psecuritypriv->dot11_privacy_algrthm = _NO_PRIVACY_;
 	psecuritypriv->dot118021XGrpPrivacy = _NO_PRIVACY_;
-	psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open; /* open system */
+	psecuritypriv->dot11_auth_algrthm = dot11AuthAlgrthm_Open; /* open system */
 	psecuritypriv->ndisauthtype = Ndis802_11AuthModeOpen;
 
 	ret = rtw_cfg80211_set_wpa_version(psecuritypriv, sme->crypto.wpa_versions);
@@ -1706,8 +1690,8 @@ static int cfg80211_rtw_connect(struct wiphy *wiphy, struct net_device *ndev,
 	}
 
 	/* For WEP Shared auth */
-	if ((psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_Shared ||
-	     psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_Auto) && sme->key) {
+	if ((psecuritypriv->dot11_auth_algrthm == dot11AuthAlgrthm_Shared ||
+	     psecuritypriv->dot11_auth_algrthm == dot11AuthAlgrthm_Auto) && sme->key) {
 		u32 wep_key_idx, wep_key_len, wep_total_len;
 		struct ndis_802_11_wep	 *pwep = NULL;
 
@@ -1733,7 +1717,7 @@ static int cfg80211_rtw_connect(struct wiphy *wiphy, struct net_device *ndev,
 			pwep->length = wep_total_len;
 
 			if (wep_key_len == 13) {
-				padapter->securitypriv.dot11PrivacyAlgrthm = _WEP104_;
+				padapter->securitypriv.dot11_privacy_algrthm = _WEP104_;
 				padapter->securitypriv.dot118021XGrpPrivacy = _WEP104_;
 			}
 		} else {
@@ -1919,7 +1903,7 @@ static int cfg80211_rtw_flush_pmksa(struct wiphy *wiphy,
 	return 0;
 }
 
-void rtw_cfg80211_indicate_sta_assoc(struct adapter *padapter, u8 *pmgmt_frame, uint frame_len)
+void rtw_cfg80211_indicate_sta_assoc(struct adapter *padapter, u8 *pmgmt_frame, unsigned int frame_len)
 {
 	struct wireless_dev *wdev = padapter->rtw_wdev;
 
@@ -2255,7 +2239,7 @@ static int rtw_add_beacon(struct adapter *adapter, const u8 *head, size_t head_l
 {
 	int ret = 0;
 	u8 *pbuf = NULL;
-	uint len, wps_ielen = 0;
+	unsigned int len, wps_ielen = 0;
 	struct mlme_priv *pmlmepriv = &(adapter->mlmepriv);
 
 	if (!check_fwstate(pmlmepriv, WIFI_AP_STATE))
@@ -2450,7 +2434,7 @@ static int	cfg80211_rtw_dump_station(struct wiphy *wiphy,
 	return ret;
 }
 
-void rtw_cfg80211_rx_action(struct adapter *adapter, u8 *frame, uint frame_len, const char *msg)
+void rtw_cfg80211_rx_action(struct adapter *adapter, u8 *frame, unsigned int frame_len, const char *msg)
 {
 	s32 freq;
 	int channel;
diff --git a/drivers/staging/rtl8723bs/os_dep/os_intfs.c b/drivers/staging/rtl8723bs/os_dep/os_intfs.c
index f31196f54b3e0..fb6fc603f279d 100644
--- a/drivers/staging/rtl8723bs/os_dep/os_intfs.c
+++ b/drivers/staging/rtl8723bs/os_dep/os_intfs.c
@@ -19,7 +19,7 @@ static int rtw_rfintfs = HWPI;
 static int rtw_lbkmode;/* RTL8712_AIR_TRX; */
 
 
-static int rtw_network_mode = Ndis802_11IBSS;/* Ndis802_11Infrastructure;infra, ad-hoc, auto */
+static int rtw_network_mode = NL80211_IFTYPE_ADHOC;/* infra, ad-hoc, auto */
 /* struct ndis_802_11_ssid	ssid; */
 static int rtw_channel = 1;/* ad-hoc support requirement */
 static int rtw_wireless_mode = WIRELESS_11BG_24N;
@@ -143,7 +143,6 @@ module_param(rtw_wifi_spec, int, 0644);
 module_param(rtw_antdiv_cfg, int, 0644);
 module_param(rtw_antdiv_type, int, 0644);
 
-
 module_param(rtw_hw_wps_pbc, int, 0644);
 
 static uint rtw_max_roaming_times = 2;
@@ -238,7 +237,6 @@ static void loadparam(struct adapter *padapter, struct net_device *pnetdev)
 	registry_par->lowrate_two_xmit = (u8)rtw_lowrate_two_xmit;
 	registry_par->low_power = (u8)rtw_low_power;
 
-
 	registry_par->wifi_spec = (u8)rtw_wifi_spec;
 
 	registry_par->channel_plan = (u8)rtw_channel_plan;
@@ -335,7 +333,6 @@ static unsigned int rtw_classify8021d(struct sk_buff *skb)
 	return dscp >> 5;
 }
 
-
 static u16 rtw_select_queue(struct net_device *dev, struct sk_buff *skb,
 			    struct net_device *sb_dev)
 {
@@ -534,8 +531,8 @@ static void rtw_init_default_value(struct adapter *padapter)
 	psecuritypriv->sw_encrypt = pregistrypriv->software_encrypt;
 	psecuritypriv->sw_decrypt = pregistrypriv->software_decrypt;
 
-	psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open; /* open system */
-	psecuritypriv->dot11PrivacyAlgrthm = _NO_PRIVACY_;
+	psecuritypriv->dot11_auth_algrthm = dot11AuthAlgrthm_Open; /* open system */
+	psecuritypriv->dot11_privacy_algrthm = _NO_PRIVACY_;
 
 	psecuritypriv->dot11PrivacyKeyIndex = 0;
 
@@ -633,7 +630,6 @@ void rtw_reset_drv_sw(struct adapter *padapter)
 	rtw_set_signal_stat_timer(&padapter->recvpriv);
 }
 
-
 u8 rtw_init_drv_sw(struct adapter *padapter)
 {
 	int res;
@@ -661,7 +657,6 @@ u8 rtw_init_drv_sw(struct adapter *padapter)
 
 	if (_rtw_init_recv_priv(&padapter->recvpriv, padapter) == _FAIL)
 		goto free_xmit_priv;
-	/*  add for CONFIG_IEEE80211W, none 11w also can use */
 	spin_lock_init(&padapter->security_key_mutex);
 
 	/*  We don't need to memset padapter->XXX to zero, because adapter is allocated by vzalloc(). */
@@ -791,7 +786,7 @@ int rtw_drv_register_netdev(struct adapter *if1)
 
 static int _netdev_open(struct net_device *pnetdev)
 {
-	uint status;
+	unsigned int status;
 	struct adapter *padapter = rtw_netdev_priv(pnetdev);
 	struct pwrctrl_priv *pwrctrlpriv = adapter_to_pwrctl(padapter);
 
@@ -891,7 +886,6 @@ static int  ips_netdrv_open(struct adapter *padapter)
 	return _FAIL;
 }
 
-
 int rtw_ips_pwr_up(struct adapter *padapter)
 {
 	return ips_netdrv_open(padapter);
@@ -1024,8 +1018,8 @@ static int rtw_suspend_free_assoc_resource(struct adapter *padapter)
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 
 	if (rtw_chk_roam_flags(padapter, RTW_ROAM_ON_RESUME)) {
-		if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)
-			&& check_fwstate(pmlmepriv, _FW_LINKED)) {
+		if (check_fwstate(pmlmepriv, WIFI_STATION_STATE) &&
+		    check_fwstate(pmlmepriv, _FW_LINKED)) {
 			rtw_set_to_roam(padapter, 1);
 		}
 	}
diff --git a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c
index c43a0391a5ca7..7c8d912dd4622 100644
--- a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c
+++ b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c
@@ -215,7 +215,7 @@ static void sd_intf_stop(struct adapter *padapter)
 }
 
 
-static struct adapter *rtw_sdio_if1_init(struct dvobj_priv *dvobj, const struct sdio_device_id  *pdid)
+static struct adapter *rtw_sdio_if1_init(struct dvobj_priv *dvobj)
 {
 	int status = _FAIL;
 	struct net_device *pnetdev;
@@ -285,8 +285,8 @@ static struct adapter *rtw_sdio_if1_init(struct dvobj_priv *dvobj, const struct
 	status = _SUCCESS;
 
 free_hal_data:
-	if (status != _SUCCESS && padapter->HalData)
-		kfree(padapter->HalData);
+	if (status != _SUCCESS)
+		rtw_hal_data_deinit(padapter);
 
 	if (status != _SUCCESS) {
 		rtw_wdev_unregister(padapter->rtw_wdev);
@@ -334,9 +334,8 @@ static void rtw_sdio_if1_deinit(struct adapter *if1)
  * notes: drv_init() is called when the bus driver has located a card for us to support.
  *        We accept the new device by returning 0.
  */
-static int rtw_drv_init(
-	struct sdio_func *func,
-	const struct sdio_device_id *id)
+static int rtw_drv_init(struct sdio_func *func,
+			const struct sdio_device_id *id)
 {
 	int status = _FAIL;
 	struct adapter *if1 = NULL;
@@ -346,7 +345,7 @@ static int rtw_drv_init(
 	if (!dvobj)
 		goto exit;
 
-	if1 = rtw_sdio_if1_init(dvobj, id);
+	if1 = rtw_sdio_if1_init(dvobj);
 	if (!if1)
 		goto free_dvobj;
 
@@ -357,10 +356,13 @@ static int rtw_drv_init(
 
 	status = sdio_alloc_irq(dvobj);
 	if (status != _SUCCESS)
-		goto free_if1;
+		goto free_netdev;
 
 	status = _SUCCESS;
 
+free_netdev:
+	if (status != _SUCCESS)
+		rtw_unregister_netdevs(dvobj);
 free_if1:
 	if (status != _SUCCESS && if1)
 		rtw_sdio_if1_deinit(if1);
diff --git a/drivers/staging/rtl8723bs/os_dep/wifi_regd.c b/drivers/staging/rtl8723bs/os_dep/wifi_regd.c
index c48a1e3ba4acc..d201a18d78b06 100644
--- a/drivers/staging/rtl8723bs/os_dep/wifi_regd.c
+++ b/drivers/staging/rtl8723bs/os_dep/wifi_regd.c
@@ -70,11 +70,11 @@ static void _rtw_reg_apply_flags(struct wiphy *wiphy)
 
 	/* channels apply by channel plans. */
 	for (i = 0; i < max_chan_nums; i++) {
-		channel = channel_set[i].ChannelNum;
+		channel = channel_set[i].channel_num;
 		freq = rtw_ieee80211_channel_to_frequency(channel);
 		ch = ieee80211_get_channel(wiphy, freq);
 		if (ch) {
-			if (channel_set[i].ScanType == SCAN_PASSIVE)
+			if (channel_set[i].scan_type == SCAN_PASSIVE)
 				ch->flags = IEEE80211_CHAN_NO_IR;
 			else
 				ch->flags = 0;
diff --git a/drivers/staging/rtl8723bs/os_dep/xmit_linux.c b/drivers/staging/rtl8723bs/os_dep/xmit_linux.c
index 4260ed5f4e974..e4b069f5cfa3a 100644
--- a/drivers/staging/rtl8723bs/os_dep/xmit_linux.c
+++ b/drivers/staging/rtl8723bs/os_dep/xmit_linux.c
@@ -7,7 +7,7 @@
 #include <drv_types.h>
 
 
-uint rtw_remainder_len(struct pkt_file *pfile)
+unsigned int rtw_remainder_len(struct pkt_file *pfile)
 {
 	return (pfile->buf_len - ((SIZE_PTR)(pfile->cur_addr) - (SIZE_PTR)(pfile->buf_start)));
 }
@@ -48,7 +48,9 @@ signed int rtw_endofpktfile(struct pkt_file *pfile)
 	return false;
 }
 
-void rtw_os_xmit_resource_free(struct adapter *padapter, struct xmit_buf *pxmitbuf, u32 free_sz, u8 flag)
+void rtw_os_xmit_resource_free(struct adapter *padapter,
+			       struct xmit_buf *pxmitbuf,
+			       u32 free_sz, u8 flag)
 {
 	if (free_sz > 0)
 		kfree(pxmitbuf->pallocated_buf);
diff --git a/drivers/staging/sm750fb/Kconfig b/drivers/staging/sm750fb/Kconfig
index 08bcccdd0f1c4..25fe422f55f2c 100644
--- a/drivers/staging/sm750fb/Kconfig
+++ b/drivers/staging/sm750fb/Kconfig
@@ -6,6 +6,7 @@ config FB_SM750
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
+	select FB_IOMEM_FOPS
 	help
 	  Frame buffer driver for the Silicon Motion SM750 chip
 	  with 2D acceleration and dual head support.
diff --git a/drivers/staging/sm750fb/sm750.c b/drivers/staging/sm750fb/sm750.c
index 89c811e0806c4..a2f244878f857 100644
--- a/drivers/staging/sm750fb/sm750.c
+++ b/drivers/staging/sm750fb/sm750.c
@@ -8,15 +8,6 @@
 #include "sm750_accel.h"
 #include "sm750_cursor.h"
 
-/*
- * #ifdef __BIG_ENDIAN
- * ssize_t lynxfb_ops_write(struct fb_info *info, const char __user *buf,
- * size_t count, loff_t *ppos);
- * ssize_t lynxfb_ops_read(struct fb_info *info, char __user *buf,
- * size_t count, loff_t *ppos);
- * #endif
- */
-
 /* common var for all device */
 static int g_hwcursor = 1;
 static int g_noaccel __ro_after_init;
@@ -352,7 +343,6 @@ static int lynxfb_ops_set_par(struct fb_info *info)
 	if (!info)
 		return -EINVAL;
 
-	ret = 0;
 	par = info->par;
 	crtc = &par->crtc;
 	output = &par->output;
@@ -472,7 +462,6 @@ static int lynxfb_ops_check_var(struct fb_var_screeninfo *var,
 	if (!var->pixclock)
 		return -EINVAL;
 
-	ret = 0;
 	par = info->par;
 	crtc = &par->crtc;
 
@@ -591,7 +580,7 @@ static int sm750fb_set_drv(struct lynxfb_par *par)
 	crtc = &par->crtc;
 
 	crtc->vidmem_size = sm750_dev->vidmem_size;
-	if (sm750_dev->fb_count > 1)
+	if (g_dualview)
 		crtc->vidmem_size >>= 1;
 
 	/* setup crtc and output member */
@@ -743,7 +732,7 @@ static int lynxfb_set_fbinfo(struct fb_info *info, int index)
 	 * must be set after crtc member initialized
 	 */
 	crtc->cursor.offset = crtc->o_screen + crtc->vidmem_size - 1024;
-	crtc->cursor.mmio = sm750_dev->pvReg +
+	crtc->cursor.mmio = sm750_dev->mmio +
 		0x800f0 + (int)crtc->channel * 0x140;
 
 	crtc->cursor.max_h = 64;
@@ -844,11 +833,11 @@ static void sm750fb_setup(struct sm750_dev *sm750_dev, char *src)
 
 	swap = 0;
 
-	sm750_dev->init_parm.chip_clk = 0;
-	sm750_dev->init_parm.mem_clk = 0;
-	sm750_dev->init_parm.master_clk = 0;
+	sm750_dev->init_parm.chip_clock = 0;
+	sm750_dev->init_parm.mem_clock = 0;
+	sm750_dev->init_parm.master_clock = 0;
 	sm750_dev->init_parm.power_mode = 0;
-	sm750_dev->init_parm.setAllEngOff = 0;
+	sm750_dev->init_parm.set_all_eng_off = 0;
 	sm750_dev->init_parm.reset_memory = 1;
 
 	/* defaultly turn g_hwcursor on for both view */
@@ -896,7 +885,7 @@ static void sm750fb_setup(struct sm750_dev *sm750_dev, char *src)
 
 NO_PARAM:
 	if (sm750_dev->revid != SM750LE_REVISION_ID) {
-		if (sm750_dev->fb_count > 1) {
+		if (g_dualview) {
 			if (swap)
 				sm750_dev->dataflow = sm750_dual_swap;
 			else
@@ -1047,7 +1036,7 @@ static void lynxfb_pci_remove(struct pci_dev *pdev)
 	sm750fb_framebuffer_release(sm750_dev);
 	arch_phys_wc_del(sm750_dev->mtrr.vram);
 
-	iounmap(sm750_dev->pvReg);
+	iounmap(sm750_dev->mmio);
 	iounmap(sm750_dev->vmem);
 	pci_release_region(pdev, 1);
 	kfree(g_settings);
diff --git a/drivers/staging/sm750fb/sm750.h b/drivers/staging/sm750fb/sm750.h
index d2c522e67f26a..0324778768cbc 100644
--- a/drivers/staging/sm750fb/sm750.h
+++ b/drivers/staging/sm750fb/sm750.h
@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #ifndef LYNXDRV_H_
 #define LYNXDRV_H_
+#include "ddk750_chip.h"
 
 #define FB_ACCEL_SMI 0xab
 
@@ -38,15 +39,6 @@ enum sm750_path {
 	sm750_pnc = 3,	/* panel and crt */
 };
 
-struct init_status {
-	ushort power_mode;
-	/* below three clocks are in unit of MHZ*/
-	ushort chip_clk;
-	ushort mem_clk;
-	ushort master_clk;
-	ushort setAllEngOff;
-	ushort reset_memory;
-};
 
 struct lynx_accel {
 	/* base virtual address of DPR registers */
@@ -97,12 +89,12 @@ struct sm750_dev {
 	unsigned long vidreg_start;
 	__u32 vidmem_size;
 	__u32 vidreg_size;
-	void __iomem *pvReg;
+	void __iomem *mmio;
 	unsigned char __iomem *vmem;
 	/* locks*/
 	spinlock_t slock;
 
-	struct init_status init_parm;
+	struct initchip_param init_parm;
 	enum sm750_pnltype pnltype;
 	enum sm750_dataflow dataflow;
 	int nocrt;
@@ -145,8 +137,6 @@ struct lynxfb_crtc {
 	u16 ypanstep;
 	u16 ywrapstep;
 
-	void *priv;
-
 	/* cursor information */
 	struct lynx_cursor cursor;
 };
@@ -168,7 +158,6 @@ struct lynxfb_output {
 	 * *channel=1 means secondary channel
 	 * output->channel ==> &crtc->channel
 	 */
-	void *priv;
 };
 
 struct lynxfb_par {
diff --git a/drivers/staging/sm750fb/sm750_hw.c b/drivers/staging/sm750fb/sm750_hw.c
index 34a837fb4b649..c4358d3e50918 100644
--- a/drivers/staging/sm750fb/sm750_hw.c
+++ b/drivers/staging/sm750fb/sm750_hw.c
@@ -23,18 +23,18 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 	}
 
 	/* now map mmio and vidmem */
-	sm750_dev->pvReg =
+	sm750_dev->mmio =
 		ioremap(sm750_dev->vidreg_start, sm750_dev->vidreg_size);
-	if (!sm750_dev->pvReg) {
+	if (!sm750_dev->mmio) {
 		dev_err(&pdev->dev, "mmio failed\n");
 		ret = -EFAULT;
 		goto err_release_region;
 	}
 
-	sm750_dev->accel.dpr_base = sm750_dev->pvReg + DE_BASE_ADDR_TYPE1;
-	sm750_dev->accel.dp_port_base = sm750_dev->pvReg + DE_PORT_ADDR_TYPE1;
+	sm750_dev->accel.dpr_base = sm750_dev->mmio + DE_BASE_ADDR_TYPE1;
+	sm750_dev->accel.dp_port_base = sm750_dev->mmio + DE_PORT_ADDR_TYPE1;
 
-	mmio750 = sm750_dev->pvReg;
+	mmio750 = sm750_dev->mmio;
 	sm750_set_chip_type(sm750_dev->devid, sm750_dev->revid);
 
 	sm750_dev->vidmem_start = pci_resource_start(pdev, 0);
@@ -58,7 +58,7 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 	return 0;
 
 err_unmap_reg:
-	iounmap(sm750_dev->pvReg);
+	iounmap(sm750_dev->mmio);
 err_release_region:
 	pci_release_region(pdev, 1);
 	return ret;
@@ -66,20 +66,20 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 
 int hw_sm750_inithw(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 {
-	struct init_status *parm;
+	struct initchip_param *parm;
 
 	parm = &sm750_dev->init_parm;
-	if (parm->chip_clk == 0)
-		parm->chip_clk = (sm750_get_chip_type() == SM750LE) ?
+	if (parm->chip_clock == 0)
+		parm->chip_clock = (sm750_get_chip_type() == SM750LE) ?
 					       DEFAULT_SM750LE_CHIP_CLOCK :
 					       DEFAULT_SM750_CHIP_CLOCK;
 
-	if (parm->mem_clk == 0)
-		parm->mem_clk = parm->chip_clk;
-	if (parm->master_clk == 0)
-		parm->master_clk = parm->chip_clk / 3;
+	if (parm->mem_clock == 0)
+		parm->mem_clock = parm->chip_clock;
+	if (parm->master_clock == 0)
+		parm->master_clock = parm->chip_clock / 3;
 
-	ddk750_init_hw((struct initchip_param *)&sm750_dev->init_parm);
+	ddk750_init_hw(&sm750_dev->init_parm);
 	/* for sm718, open pci burst */
 	if (sm750_dev->devid == 0x718) {
 		poke32(SYSTEM_CTRL,
diff --git a/drivers/target/target_core_alua.c b/drivers/target/target_core_alua.c
index 10250aca5a816..140154d93c430 100644
--- a/drivers/target/target_core_alua.c
+++ b/drivers/target/target_core_alua.c
@@ -18,6 +18,8 @@
 #include <linux/fcntl.h>
 #include <linux/file.h>
 #include <linux/fs.h>
+#include <linux/fs_struct.h>
+#include <linux/kthread.h>
 #include <scsi/scsi_proto.h>
 #include <linux/unaligned.h>
 
@@ -856,10 +858,17 @@ static int core_alua_write_tpg_metadata(
 	unsigned char *md_buf,
 	u32 md_buf_len)
 {
-	struct file *file = filp_open(path, O_RDWR | O_CREAT | O_TRUNC, 0600);
+	struct file *file;
 	loff_t pos = 0;
 	int ret;
 
+	if (tsk_is_kthread(current)) {
+		scoped_with_init_fs()
+			file = filp_open(path, O_RDWR | O_CREAT | O_TRUNC, 0600);
+	} else {
+		file = filp_open(path, O_RDWR | O_CREAT | O_TRUNC, 0600);
+	}
+
 	if (IS_ERR(file)) {
 		pr_err("filp_open(%s) for ALUA metadata failed\n", path);
 		return -ENODEV;
diff --git a/drivers/target/target_core_fabric_lib.c b/drivers/target/target_core_fabric_lib.c
index 87c5d26a50895..2853b95b2c597 100644
--- a/drivers/target/target_core_fabric_lib.c
+++ b/drivers/target/target_core_fabric_lib.c
@@ -290,13 +290,24 @@ static void sbp_parse_pr_out_transport_id(char *buf, char *i_str)
 static bool iscsi_parse_pr_out_transport_id(
 	struct se_portal_group *se_tpg,
 	char *buf,
+	u32 buf_len,
 	u32 *out_tid_len,
 	char **port_nexus_ptr,
 	char *i_str)
 {
 	char *p;
+	u32 tid_len;
 	int i;
-	u8 format_code = (buf[0] & 0xc0);
+	u8 format_code;
+
+	/*
+	 * The 4-byte iSCSI TransportID header (FORMAT CODE + 2-byte ADDITIONAL
+	 * LENGTH) must be present before any of it can be parsed.
+	 */
+	if (buf_len < 4)
+		return false;
+
+	format_code = buf[0] & 0xc0;
 	/*
 	 * Check for FORMAT CODE 00b or 01b from spc4r17, section 7.5.4.6:
 	 *
@@ -316,15 +327,17 @@ static bool iscsi_parse_pr_out_transport_id(
 		return false;
 	}
 	/*
-	 * If the caller wants the TransportID Length, we set that value for the
-	 * entire iSCSI Tarnsport ID now.
+	 * Reconstruct the self-described TransportID length from the ADDITIONAL
+	 * LENGTH field plus the 4-byte header.  Reject it if it is below the
+	 * spc4r17 section 7.5.4.6 minimum (ADDITIONAL LENGTH shall be at least
+	 * 20) or if it runs past the bytes actually received, so that every
+	 * access below stays inside the TransportID.
 	 */
-	if (out_tid_len) {
-		/* The shift works thanks to integer promotion rules */
-		*out_tid_len = get_unaligned_be16(&buf[2]);
-		/* Add four bytes for iSCSI Transport ID header */
-		*out_tid_len += 4;
-	}
+	tid_len = get_unaligned_be16(&buf[2]) + 4;
+	if (tid_len < 24 || tid_len > buf_len)
+		return false;
+	if (out_tid_len)
+		*out_tid_len = tid_len;
 
 	/*
 	 * Check for ',i,0x' separator between iSCSI Name and iSCSI Initiator
@@ -332,16 +345,32 @@ static bool iscsi_parse_pr_out_transport_id(
 	 * format.
 	 */
 	if (format_code == 0x40) {
-		p = strstr(&buf[4], ",i,0x");
+		p = strnstr(&buf[4], ",i,0x", tid_len - 4);
 		if (!p) {
-			pr_err("Unable to locate \",i,0x\" separator"
-				" for Initiator port identifier: %s\n",
-				&buf[4]);
+			pr_err("Unable to locate \",i,0x\" separator in iSCSI TransportID\n");
+			return false;
+		}
+		/*
+		 * The iSCSI name runs from &buf[4] up to the separator; reject it
+		 * if it cannot fit in i_str[TRANSPORT_IQN_LEN].
+		 */
+		if (p - &buf[4] >= TRANSPORT_IQN_LEN) {
+			pr_err("iSCSI Initiator port name too long in TransportID\n");
 			return false;
 		}
 		*p = '\0'; /* Terminate iSCSI Name */
 		p += 5; /* Skip over ",i,0x" separator */
 
+		/*
+		 * The ISID must follow the separator.  A ",i,0x" sitting at the
+		 * very end of the TransportID leaves no ISID and would point the
+		 * port nexus at buf + tid_len, i.e. past the descriptor, which
+		 * the registration code then reads as the ISID string.
+		 */
+		if (p >= buf + tid_len) {
+			pr_err("Missing ISID in iSCSI Initiator port TransportID\n");
+			return false;
+		}
 		*port_nexus_ptr = p;
 		/*
 		 * Go ahead and do the lower case conversion of the received
@@ -349,7 +378,7 @@ static bool iscsi_parse_pr_out_transport_id(
 		 * for comparison against the running iSCSI session's ISID from
 		 * iscsi_target.c:lio_sess_get_initiator_sid()
 		 */
-		for (i = 0; i < 12; i++) {
+		for (i = 0; i < 12 && p < buf + tid_len; i++) {
 			/*
 			 * The first ISCSI INITIATOR SESSION ID field byte
 			 * containing an ASCII null character terminates the
@@ -367,10 +396,22 @@ static bool iscsi_parse_pr_out_transport_id(
 			*p = tolower(*p);
 			p++;
 		}
-	} else
+		strscpy(i_str, &buf[4], TRANSPORT_IQN_LEN);
+	} else {
 		*port_nexus_ptr = NULL;
-
-	strscpy(i_str, &buf[4], TRANSPORT_IQN_LEN);
+		/*
+		 * FORMAT CODE 00b: the name occupies buf[4..tid_len-1].  The
+		 * declared length tid_len - 4 must fit in i_str[TRANSPORT_IQN_LEN].
+		 * (For 01b the same tid_len bound would be over-restrictive: the
+		 * descriptor also carries the separator and ISID, so a legal
+		 * <=223-byte name gives tid_len up to 244.)
+		 */
+		if (tid_len - 4 >= TRANSPORT_IQN_LEN) {
+			pr_err("iSCSI Initiator port name too long in TransportID\n");
+			return false;
+		}
+		strscpy(i_str, &buf[4], tid_len - 4);
+	}
 	return true;
 }
 
@@ -420,8 +461,16 @@ int target_get_pr_transport_id(struct se_node_acl *nacl,
 }
 
 bool target_parse_pr_out_transport_id(struct se_portal_group *tpg,
-		char *buf, u32 *out_tid_len, char **port_nexus_ptr, char *i_str)
+		char *buf, u32 buf_len, u32 *out_tid_len,
+		char **port_nexus_ptr, char *i_str)
 {
+	/*
+	 * The fixed-length SAS/SRP/FCP/SBP TransportIDs are 24 bytes; the iSCSI
+	 * format is variable and bounds itself against buf_len below.
+	 */
+	if (tpg->proto_id != SCSI_PROTOCOL_ISCSI && buf_len < 24)
+		return false;
+
 	switch (tpg->proto_id) {
 	case SCSI_PROTOCOL_SAS:
 		/*
@@ -440,8 +489,8 @@ bool target_parse_pr_out_transport_id(struct se_portal_group *tpg,
 		sbp_parse_pr_out_transport_id(buf, i_str);
 		break;
 	case SCSI_PROTOCOL_ISCSI:
-		return iscsi_parse_pr_out_transport_id(tpg, buf, out_tid_len,
-					port_nexus_ptr, i_str);
+		return iscsi_parse_pr_out_transport_id(tpg, buf, buf_len,
+					out_tid_len, port_nexus_ptr, i_str);
 	default:
 		pr_err("Unknown proto_id: 0x%02x\n", tpg->proto_id);
 		return false;
diff --git a/drivers/target/target_core_internal.h b/drivers/target/target_core_internal.h
index 763e6d26e1877..f0886ea290345 100644
--- a/drivers/target/target_core_internal.h
+++ b/drivers/target/target_core_internal.h
@@ -104,7 +104,8 @@ int	target_get_pr_transport_id(struct se_node_acl *nacl,
 		struct t10_pr_registration *pr_reg, int *format_code,
 		unsigned char *buf);
 bool target_parse_pr_out_transport_id(struct se_portal_group *tpg,
-		char *buf, u32 *out_tid_len, char **port_nexus_ptr, char *i_str);
+		char *buf, u32 buf_len, u32 *out_tid_len,
+		char **port_nexus_ptr, char *i_str);
 
 /* target_core_hba.c */
 struct se_hba *core_alloc_hba(const char *, u32, u32);
diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c
index 11790f2c5d80f..25b1bcacc0c8f 100644
--- a/drivers/target/target_core_pr.c
+++ b/drivers/target/target_core_pr.c
@@ -18,6 +18,7 @@
 #include <linux/file.h>
 #include <linux/fcntl.h>
 #include <linux/fs.h>
+#include <linux/fs_struct.h>
 #include <scsi/scsi_proto.h>
 #include <linux/unaligned.h>
 
@@ -1573,7 +1574,7 @@ core_scsi3_decode_spec_i_port(
 
 			iport_ptr = NULL;
 			tid_found = target_parse_pr_out_transport_id(tmp_tpg,
-					ptr, &tid_len, &iport_ptr, i_str);
+					ptr, tpdl, &tid_len, &iport_ptr, i_str);
 			if (!tid_found)
 				continue;
 			/*
@@ -1969,7 +1970,8 @@ static int __core_scsi3_write_aptpl_to_file(
 	if (!path)
 		return -ENOMEM;
 
-	file = filp_open(path, flags, 0600);
+	scoped_with_init_fs()
+		file = filp_open(path, flags, 0600);
 	if (IS_ERR(file)) {
 		pr_err("filp_open(%s) for APTPL metadata"
 			" failed\n", path);
@@ -3285,7 +3287,7 @@ core_scsi3_emulate_pro_register_and_move(struct se_cmd *cmd, u64 res_key,
 		goto out;
 	}
 	tid_found = target_parse_pr_out_transport_id(dest_se_tpg,
-			&buf[24], &tmp_tid_len, &iport_ptr, initiator_str);
+			&buf[24], tid_len, &tmp_tid_len, &iport_ptr, initiator_str);
 	if (!tid_found) {
 		pr_err("SPC-3 PR REGISTER_AND_MOVE: Unable to locate"
 			" initiator_str from Transport ID\n");
@@ -3293,9 +3295,6 @@ core_scsi3_emulate_pro_register_and_move(struct se_cmd *cmd, u64 res_key,
 		goto out;
 	}
 
-	transport_kunmap_data_sg(cmd);
-	buf = NULL;
-
 	pr_debug("SPC-3 PR [%s] Extracted initiator %s identifier: %s"
 		" %s\n", dest_tf_ops->fabric_name, (iport_ptr != NULL) ?
 		"port" : "device", initiator_str, (iport_ptr != NULL) ?
@@ -3532,6 +3531,11 @@ core_scsi3_emulate_pro_register_and_move(struct se_cmd *cmd, u64 res_key,
 	core_scsi3_update_and_write_aptpl(cmd->se_dev, aptpl);
 
 	core_scsi3_put_pr_reg(dest_pr_reg);
+	/*
+	 * iport_ptr aliases the PR-OUT parameter list mapped above, so the
+	 * buffer is unmapped only here on success (and at out: on error).
+	 */
+	transport_kunmap_data_sg(cmd);
 	return 0;
 out:
 	if (buf)
diff --git a/drivers/tee/optee/ffa_abi.c b/drivers/tee/optee/ffa_abi.c
index b4372fa268d03..633715b98625c 100644
--- a/drivers/tee/optee/ffa_abi.c
+++ b/drivers/tee/optee/ffa_abi.c
@@ -698,6 +698,9 @@ static int optee_ffa_lend_protmem(struct optee *optee, struct tee_shm *protmem,
 	int rc;
 
 	mem_attr = kzalloc_objs(*mem_attr, ma_count);
+	if (!mem_attr)
+		return -ENOMEM;
+
 	for (n = 0; n < ma_count; n++) {
 		mem_attr[n].receiver = mem_attrs[n] & U16_MAX;
 		mem_attr[n].attrs = mem_attrs[n] >> 16;
diff --git a/drivers/tee/qcomtee/call.c b/drivers/tee/qcomtee/call.c
index 0efc5646242af..4a597eeaf174b 100644
--- a/drivers/tee/qcomtee/call.c
+++ b/drivers/tee/qcomtee/call.c
@@ -798,7 +798,12 @@ static void qcomtee_remove(struct platform_device *pdev)
 	kfree(qcomtee);
 }
 
-static const struct platform_device_id qcomtee_ids[] = { { "qcomtee", 0 }, {} };
+static const struct platform_device_id qcomtee_ids[] = {
+	{
+		.name = "qcomtee",
+	},
+	{ }
+};
 MODULE_DEVICE_TABLE(platform, qcomtee_ids);
 
 static struct platform_driver qcomtee_platform_driver = {
diff --git a/drivers/thermal/db8500_thermal.c b/drivers/thermal/db8500_thermal.c
index 576f88b6a1b35..cf1706569e6d7 100644
--- a/drivers/thermal/db8500_thermal.c
+++ b/drivers/thermal/db8500_thermal.c
@@ -10,7 +10,7 @@
 
 #include <linux/cpu_cooling.h>
 #include <linux/interrupt.h>
-#include <linux/mfd/dbx500-prcmu.h>
+#include <linux/mfd/db8500-prcmu.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
@@ -82,7 +82,7 @@ static void db8500_thermal_update_config(struct db8500_thermal_zone *th,
 					 unsigned long next_low,
 					 unsigned long next_high)
 {
-	prcmu_stop_temp_sense();
+	db8500_prcmu_stop_temp_sense();
 
 	th->cur_index = idx;
 	th->interpolated_temp = (next_low + next_high)/2;
@@ -91,8 +91,8 @@ static void db8500_thermal_update_config(struct db8500_thermal_zone *th,
 	 * The PRCMU accept absolute temperatures in celsius so divide
 	 * down the millicelsius with 1000
 	 */
-	prcmu_config_hotmon((u8)(next_low/1000), (u8)(next_high/1000));
-	prcmu_start_temp_sense(PRCMU_DEFAULT_MEASURE_TIME);
+	db8500_prcmu_config_hotmon((u8)(next_low / 1000), (u8)(next_high / 1000));
+	db8500_prcmu_start_temp_sense(PRCMU_DEFAULT_MEASURE_TIME);
 }
 
 static irqreturn_t prcmu_low_irq_handler(int irq, void *irq_data)
@@ -204,7 +204,7 @@ static int db8500_thermal_probe(struct platform_device *pdev)
 static int db8500_thermal_suspend(struct platform_device *pdev,
 		pm_message_t state)
 {
-	prcmu_stop_temp_sense();
+	db8500_prcmu_stop_temp_sense();
 
 	return 0;
 }
diff --git a/drivers/thermal/intel/intel_tcc.c b/drivers/thermal/intel/intel_tcc.c
index 59f70bb5ffa5e..d1fa3c63d5544 100644
--- a/drivers/thermal/intel/intel_tcc.c
+++ b/drivers/thermal/intel/intel_tcc.c
@@ -185,7 +185,7 @@ int intel_tcc_get_tjmax(int cpu)
 	int val, err;
 
 	if (cpu < 0)
-		err = rdmsr_safe(MSR_IA32_TEMPERATURE_TARGET, &msrval.l, &msrval.h);
+		err = rdmsrq_safe(MSR_IA32_TEMPERATURE_TARGET, &msrval.q);
 	else
 		err = rdmsrq_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, &msrval.q);
 	if (err)
@@ -212,7 +212,7 @@ int intel_tcc_get_offset(int cpu)
 	int err;
 
 	if (cpu < 0)
-		err = rdmsr_safe(MSR_IA32_TEMPERATURE_TARGET, &val.l, &val.h);
+		err = rdmsrq_safe(MSR_IA32_TEMPERATURE_TARGET, &val.q);
 	else
 		err = rdmsrq_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, &val.q);
 	if (err)
@@ -245,7 +245,7 @@ int intel_tcc_set_offset(int cpu, int offset)
 		return -EINVAL;
 
 	if (cpu < 0)
-		err = rdmsr_safe(MSR_IA32_TEMPERATURE_TARGET, &val.l, &val.h);
+		err = rdmsrq_safe(MSR_IA32_TEMPERATURE_TARGET, &val.q);
 	else
 		err = rdmsrq_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, &val.q);
 	if (err)
@@ -259,7 +259,7 @@ int intel_tcc_set_offset(int cpu, int offset)
 	val.l |= offset << 24;
 
 	if (cpu < 0)
-		return wrmsr_safe(MSR_IA32_TEMPERATURE_TARGET, val.l, val.h);
+		return wrmsrq_safe(MSR_IA32_TEMPERATURE_TARGET, val.q);
 	else
 		return wrmsrq_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, val.q);
 }
@@ -288,7 +288,7 @@ int intel_tcc_get_temp(int cpu, int *temp, bool pkg)
 		return tjmax;
 
 	if (cpu < 0)
-		err = rdmsr_safe(msr, &val.l, &val.h);
+		err = rdmsrq_safe(msr, &val.q);
 	else
 		err = rdmsrq_safe_on_cpu(cpu, msr, &val.q);
 	if (err)
diff --git a/drivers/thermal/intel/therm_throt.c b/drivers/thermal/intel/therm_throt.c
index 45a8ef4a608b1..4d4e81601d179 100644
--- a/drivers/thermal/intel/therm_throt.c
+++ b/drivers/thermal/intel/therm_throt.c
@@ -14,12 +14,14 @@
  * Credits: Adapted from Zwane Mwaikambo's original code in mce_intel.c.
  *          Inspired by Ross Biro's and Al Borchers' counter code.
  */
+#include <linux/syscore_ops.h>
 #include <linux/interrupt.h>
 #include <linux/notifier.h>
 #include <linux/jiffies.h>
 #include <linux/kernel.h>
 #include <linux/percpu.h>
 #include <linux/export.h>
+#include <linux/delay.h>
 #include <linux/types.h>
 #include <linux/init.h>
 #include <linux/smp.h>
@@ -244,16 +246,23 @@ static void thermal_intr_init_pkg_clear_mask(void)
 	 * IA32_PACKAGE_THERM_STATUS.
 	 */
 
-	/* All bits except BIT 26 depend on CPUID.06H: EAX[6] = 1 */
+	/* All bits except BITs 25 and 26 depend on CPUID.06H: EAX[6] = 1 */
 	if (boot_cpu_has(X86_FEATURE_PTS))
 		therm_intr_pkg_clear_mask = (BIT(1) | BIT(3) | BIT(5) | BIT(7) | BIT(9) | BIT(11));
 
 	/*
-	 * Intel SDM Volume 2A: Thermal and Power Management Leaf
+	 * Intel SDM Volume 1: Thermal and Power Management Leaf
 	 * Bit 26: CPUID.06H: EAX[19] = 1
 	 */
 	if (boot_cpu_has(X86_FEATURE_HFI))
 		therm_intr_pkg_clear_mask |= BIT(26);
+
+	/*
+	 * Intel SDM Volume 1: Thermal and Power Management Leaf
+	 * Bit 25: CPUID.06H: EAX[24] = 1
+	 */
+	if (boot_cpu_has(X86_FEATURE_DPTI))
+		therm_intr_pkg_clear_mask |= BIT(25);
 }
 
 /*
@@ -524,6 +533,254 @@ static void thermal_throttle_remove_dev(struct device *dev)
 	sysfs_remove_group(&dev->kobj, &thermal_attr_group);
 }
 
+static int check_directed_thermal_pkg_intr_ack(void)
+{
+	unsigned int count = 15000;
+	u64 msr_val;
+
+	/*
+	 * Hardware acknowledges the directed interrupt setup in 10ms or less.
+	 * Wait 15ms to be safe.
+	 */
+	do {
+		rdmsrq(MSR_IA32_PACKAGE_THERM_STATUS, msr_val);
+		udelay(1);
+	} while (!(msr_val & PACKAGE_THERM_STATUS_DPTI_ACK) && --count);
+
+	if (!count)
+		return -ETIMEDOUT;
+
+	thermal_clear_package_intr_status(PACKAGE_LEVEL,
+					  PACKAGE_THERM_STATUS_DPTI_ACK);
+
+	return 0;
+}
+
+static void config_directed_thermal_pkg_intr(void *info)
+{
+	bool enable = *((bool *)info);
+	u64 msr_val;
+
+	rdmsrq(MSR_IA32_THERM_INTERRUPT, msr_val);
+
+	if (enable)
+		msr_val |= THERM_INT_DPTI_ENABLE;
+	else
+		msr_val &= ~THERM_INT_DPTI_ENABLE;
+
+	wrmsrq(MSR_IA32_THERM_INTERRUPT, msr_val);
+}
+
+/*
+ * Accessed from CPU hotplug callbacks and from code that runs while CPU
+ * hotplug is inactive: the init and cleanup paths as well as syscore callbacks.
+ * No extra locking needed.
+ */
+static unsigned int *directed_intr_handler_cpus;
+
+static bool directed_thermal_pkg_intr_supported(void)
+{
+	if (!boot_cpu_has(X86_FEATURE_DPTI))
+		return false;
+
+	if (!directed_intr_handler_cpus)
+		return false;
+
+	return true;
+}
+
+/*
+ * Must be called with cpu_hotplug_lock held to prevent CPUs from going offline
+ * while iterating through packages and interrupts must be enabled to avoid
+ * deadlocks in SMP function calls. The syscore shutdown callback also calls
+ * this function, but runs with CPU hotplug disabled (and interrupts enabled).
+ */
+static void disable_directed_thermal_pkg_intr_all(void)
+{
+	bool enable = false;
+	int i;
+
+	if (!directed_thermal_pkg_intr_supported())
+		return;
+
+	for (i = 0; i < topology_max_packages(); i++) {
+		if (directed_intr_handler_cpus[i] == nr_cpu_ids)
+			continue;
+
+		smp_call_function_single(directed_intr_handler_cpus[i],
+					 config_directed_thermal_pkg_intr,
+					 &enable, true);
+	}
+}
+
+static int enable_directed_thermal_pkg_intr(unsigned int cpu)
+{
+	bool enable = true;
+	u16 pkg_id;
+
+	if (!directed_thermal_pkg_intr_supported())
+		return 0;
+
+	pkg_id = topology_logical_package_id(cpu);
+	if (pkg_id >= topology_max_packages())
+		return -EINVAL;
+
+	/* Another CPU in this package already handles the directed interrupt. */
+	if (directed_intr_handler_cpus[pkg_id] != nr_cpu_ids)
+		return 0;
+
+	thermal_clear_package_intr_status(PACKAGE_LEVEL,
+					  PACKAGE_THERM_STATUS_DPTI_ACK);
+
+	config_directed_thermal_pkg_intr(&enable);
+	if (!check_directed_thermal_pkg_intr_ack()) {
+		directed_intr_handler_cpus[pkg_id] = cpu;
+		return 0;
+	}
+
+	/*
+	 * A failure indicates faulty hardware. Roll back completely so that
+	 * no other CPU tries. This is especially important during boot as all
+	 * CPUs may come online and would otherwise keep trying.
+	 */
+	enable = false;
+	config_directed_thermal_pkg_intr(&enable);
+
+	return -ETIMEDOUT;
+}
+
+static void disable_directed_thermal_pkg_intr(unsigned int cpu)
+{
+	unsigned int new_cpu;
+	bool enable;
+	u16 pkg_id;
+
+	if (!directed_thermal_pkg_intr_supported())
+		return;
+
+	pkg_id = topology_logical_package_id(cpu);
+	if (pkg_id >= topology_max_packages())
+		return;
+
+	/* Not the CPU handling the directed interrupt. */
+	if (directed_intr_handler_cpus[pkg_id] != cpu)
+		return;
+
+	/*
+	 * The package-level interrupt must remain directed after this CPU goes
+	 * offline.
+	 */
+	new_cpu = cpumask_any_but(topology_core_cpumask(cpu), cpu);
+	if (new_cpu < nr_cpu_ids) {
+		enable = true;
+		thermal_clear_package_intr_status(PACKAGE_LEVEL,
+						  PACKAGE_THERM_STATUS_DPTI_ACK);
+
+		/*
+		 * We are here via CPU hotplug. Since we are holding the
+		 * cpu_hotplug_lock, @new_cpu cannot go offline and interrupts
+		 * are enabled, so the SMP function call is safe.
+		 *
+		 * The syscore suspend callback runs with interrupts disabled,
+		 * but it does not reach this path because all the secondary
+		 * CPUs are offline.
+		 */
+		smp_call_function_single(new_cpu, config_directed_thermal_pkg_intr,
+					 &enable, true);
+	}
+
+	/*
+	 * If hardware does not acknowledge the directed interrupt setup on
+	 * @new_cpu, disable the redirection. Since no other CPU is configured
+	 * to receive the package-level interrupt, all CPUs in the package will
+	 * receive it.
+	 */
+	enable = false;
+	if (new_cpu < nr_cpu_ids && check_directed_thermal_pkg_intr_ack()) {
+		smp_call_function_single(new_cpu, config_directed_thermal_pkg_intr,
+					 &enable, true);
+
+		pr_warn_once("Failed to redirect package thermal interrupt from CPU%u to CPU%u; reverting to broadcast.\n",
+			     cpu, new_cpu);
+
+		new_cpu = nr_cpu_ids;
+	}
+
+	/*
+	 * Clear the directed interrupt on @cpu. Hardware acknowledgment can be
+	 * ignored since @cpu is going offline.
+	 */
+	config_directed_thermal_pkg_intr(&enable);
+
+	directed_intr_handler_cpus[pkg_id] = (new_cpu < nr_cpu_ids) ? new_cpu : nr_cpu_ids;
+}
+
+/*
+ * CPU0 may be handling the directed interrupt, but the CPU hotplug callbacks
+ * are not called for CPU0 during suspend and resume.
+ */
+static void directed_pkg_intr_syscore_resume(void *data)
+{
+	/*
+	 * We can't do anything to handle errors. If direction fails for CPU0,
+	 * another CPU will take over or disable direction entirely during CPU
+	 * hotplug.
+	 */
+	enable_directed_thermal_pkg_intr(0);
+}
+
+static int directed_pkg_intr_syscore_suspend(void *data)
+{
+	disable_directed_thermal_pkg_intr(0);
+
+	return 0;
+}
+
+static void directed_pkg_intr_syscore_shutdown(void *data)
+{
+	disable_directed_thermal_pkg_intr_all();
+}
+
+static const struct syscore_ops directed_pkg_intr_pm_ops = {
+	.resume = directed_pkg_intr_syscore_resume,
+	.suspend = directed_pkg_intr_syscore_suspend,
+	.shutdown = directed_pkg_intr_syscore_shutdown,
+};
+
+static struct syscore directed_pkg_intr_pm = {
+	.ops = &directed_pkg_intr_pm_ops,
+};
+
+static __init void init_directed_pkg_intr(void)
+{
+	int i;
+
+	if (!boot_cpu_has(X86_FEATURE_DPTI))
+		return;
+
+	directed_intr_handler_cpus = kmalloc_array(topology_max_packages(),
+						   sizeof(*directed_intr_handler_cpus),
+						   GFP_KERNEL);
+	if (!directed_intr_handler_cpus)
+		return;
+
+	for (i = 0; i < topology_max_packages(); i++)
+		directed_intr_handler_cpus[i] = nr_cpu_ids;
+
+	register_syscore(&directed_pkg_intr_pm);
+}
+
+static void cleanup_directed_pkg_thermal_intr(void)
+{
+	if (!directed_thermal_pkg_intr_supported())
+		return;
+
+	unregister_syscore(&directed_pkg_intr_pm);
+	disable_directed_thermal_pkg_intr_all();
+	kfree(directed_intr_handler_cpus);
+	directed_intr_handler_cpus = NULL;
+}
+
 /* Get notified when a cpu comes on/off. Be hotplug friendly. */
 static int thermal_throttle_online(unsigned int cpu)
 {
@@ -549,6 +806,11 @@ static int thermal_throttle_online(unsigned int cpu)
 	 */
 	intel_hfi_online(cpu);
 
+	if (enable_directed_thermal_pkg_intr(cpu)) {
+		pr_info_once("Failed to direct package thermal interrupts. All CPUs will receive it.\n");
+		cleanup_directed_pkg_thermal_intr();
+	}
+
 	/* Unmask the thermal vector after the above workqueues are initialized. */
 	l = apic_read(APIC_LVTTHMR);
 	apic_write(APIC_LVTTHMR, l & ~APIC_LVT_MASKED);
@@ -566,6 +828,8 @@ static int thermal_throttle_offline(unsigned int cpu)
 	l = apic_read(APIC_LVTTHMR);
 	apic_write(APIC_LVTTHMR, l | APIC_LVT_MASKED);
 
+	disable_directed_thermal_pkg_intr(cpu);
+
 	intel_hfi_offline(cpu);
 
 	cancel_delayed_work_sync(&state->package_throttle.therm_work);
@@ -585,12 +849,19 @@ static __init int thermal_throttle_init_device(void)
 	if (!atomic_read(&therm_throt_en))
 		return 0;
 
+	init_directed_pkg_intr();
+
 	intel_hfi_init();
 
 	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/therm:online",
 				thermal_throttle_online,
 				thermal_throttle_offline);
-	return ret < 0 ? ret : 0;
+	if (ret >= 0)
+		return 0;
+
+	cleanup_directed_pkg_thermal_intr();
+
+	return ret;
 }
 device_initcall(thermal_throttle_init_device);
 
@@ -657,7 +928,7 @@ void intel_thermal_interrupt(void)
 {
 	__u64 msr_val;
 
-	if (static_cpu_has(X86_FEATURE_HWP))
+	if (cpu_feature_enabled(X86_FEATURE_HWP))
 		notify_hwp_interrupt();
 
 	rdmsrq(MSR_IA32_THERM_STATUS, msr_val);
@@ -722,8 +993,8 @@ void __init therm_lvt_init(void)
 void intel_init_thermal(struct cpuinfo_x86 *c)
 {
 	unsigned int cpu = smp_processor_id();
+	struct msr val;
 	int tm2 = 0;
-	u32 l, h;
 
 	if (!intel_thermal_supported(c))
 		return;
@@ -733,9 +1004,9 @@ void intel_init_thermal(struct cpuinfo_x86 *c)
 	 * be some SMM goo which handles it, so we can't even put a handler
 	 * since it might be delivered via SMI already:
 	 */
-	rdmsr(MSR_IA32_MISC_ENABLE, l, h);
+	rdmsrq(MSR_IA32_MISC_ENABLE, val.q);
 
-	h = lvtthmr_init;
+	val.h = lvtthmr_init;
 	/*
 	 * The initial value of thermal LVT entries on all APs always reads
 	 * 0x10000 because APs are woken up by BSP issuing INIT-SIPI-SIPI
@@ -746,11 +1017,11 @@ void intel_init_thermal(struct cpuinfo_x86 *c)
 	 * BIOS has programmed on AP based on BSP's info we saved since BIOS
 	 * is always setting the same value for all threads/cores.
 	 */
-	if ((h & APIC_DM_FIXED_MASK) != APIC_DM_FIXED)
+	if ((val.h & APIC_DM_FIXED_MASK) != APIC_DM_FIXED)
 		apic_write(APIC_LVTTHMR, lvtthmr_init);
 
 
-	if ((l & MSR_IA32_MISC_ENABLE_TM1) && (h & APIC_DM_SMI)) {
+	if ((val.l & MSR_IA32_MISC_ENABLE_TM1) && (val.h & APIC_DM_SMI)) {
 		if (system_state == SYSTEM_BOOTING)
 			pr_debug("CPU%d: Thermal monitoring handled by SMI\n", cpu);
 		return;
@@ -759,59 +1030,55 @@ void intel_init_thermal(struct cpuinfo_x86 *c)
 	/* early Pentium M models use different method for enabling TM2 */
 	if (cpu_has(c, X86_FEATURE_TM2)) {
 		if (c->x86 == 6 && (c->x86_model == 9 || c->x86_model == 13)) {
-			rdmsr(MSR_THERM2_CTL, l, h);
-			if (l & MSR_THERM2_CTL_TM_SELECT)
+			rdmsrq(MSR_THERM2_CTL, val.q);
+			if (val.l & MSR_THERM2_CTL_TM_SELECT)
 				tm2 = 1;
-		} else if (l & MSR_IA32_MISC_ENABLE_TM2)
+		} else if (val.l & MSR_IA32_MISC_ENABLE_TM2)
 			tm2 = 1;
 	}
 
 	/* We'll mask the thermal vector in the lapic till we're ready: */
-	h = THERMAL_APIC_VECTOR | APIC_DM_FIXED | APIC_LVT_MASKED;
-	apic_write(APIC_LVTTHMR, h);
+	val.h = THERMAL_APIC_VECTOR | APIC_DM_FIXED | APIC_LVT_MASKED;
+	apic_write(APIC_LVTTHMR, val.h);
 
 	thermal_intr_init_core_clear_mask();
 	thermal_intr_init_pkg_clear_mask();
 
-	rdmsr(MSR_IA32_THERM_INTERRUPT, l, h);
-	if (cpu_has(c, X86_FEATURE_PLN) && !int_pln_enable)
-		wrmsr(MSR_IA32_THERM_INTERRUPT,
-			(l | (THERM_INT_LOW_ENABLE
-			| THERM_INT_HIGH_ENABLE)) & ~THERM_INT_PLN_ENABLE, h);
-	else if (cpu_has(c, X86_FEATURE_PLN) && int_pln_enable)
-		wrmsr(MSR_IA32_THERM_INTERRUPT,
-			l | (THERM_INT_LOW_ENABLE
-			| THERM_INT_HIGH_ENABLE | THERM_INT_PLN_ENABLE), h);
+	rdmsrq(MSR_IA32_THERM_INTERRUPT, val.q);
+	if (cpu_has(c, X86_FEATURE_PLN) && !int_pln_enable) {
+		val.l |= THERM_INT_LOW_ENABLE | THERM_INT_HIGH_ENABLE;
+		val.l &= ~THERM_INT_PLN_ENABLE;
+	} else if (cpu_has(c, X86_FEATURE_PLN) && int_pln_enable)
+		val.l |= THERM_INT_LOW_ENABLE | THERM_INT_HIGH_ENABLE |
+			 THERM_INT_PLN_ENABLE;
 	else
-		wrmsr(MSR_IA32_THERM_INTERRUPT,
-		      l | (THERM_INT_LOW_ENABLE | THERM_INT_HIGH_ENABLE), h);
+		val.l |= THERM_INT_LOW_ENABLE | THERM_INT_HIGH_ENABLE;
+	wrmsrq(MSR_IA32_THERM_INTERRUPT, val.q);
 
 	if (cpu_has(c, X86_FEATURE_PTS)) {
-		rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
-		if (cpu_has(c, X86_FEATURE_PLN) && !int_pln_enable)
-			wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
-				(l | (PACKAGE_THERM_INT_LOW_ENABLE
-				| PACKAGE_THERM_INT_HIGH_ENABLE))
-				& ~PACKAGE_THERM_INT_PLN_ENABLE, h);
-		else if (cpu_has(c, X86_FEATURE_PLN) && int_pln_enable)
-			wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
-				l | (PACKAGE_THERM_INT_LOW_ENABLE
-				| PACKAGE_THERM_INT_HIGH_ENABLE
-				| PACKAGE_THERM_INT_PLN_ENABLE), h);
+		rdmsrq(MSR_IA32_PACKAGE_THERM_INTERRUPT, val.q);
+		if (cpu_has(c, X86_FEATURE_PLN) && !int_pln_enable) {
+			val.l |= PACKAGE_THERM_INT_LOW_ENABLE |
+				 PACKAGE_THERM_INT_HIGH_ENABLE;
+			val.l &= ~PACKAGE_THERM_INT_PLN_ENABLE;
+		} else if (cpu_has(c, X86_FEATURE_PLN) && int_pln_enable)
+			val.l |= PACKAGE_THERM_INT_LOW_ENABLE |
+				 PACKAGE_THERM_INT_HIGH_ENABLE |
+				 PACKAGE_THERM_INT_PLN_ENABLE;
 		else
-			wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
-			      l | (PACKAGE_THERM_INT_LOW_ENABLE
-				| PACKAGE_THERM_INT_HIGH_ENABLE), h);
+			val.l |= PACKAGE_THERM_INT_LOW_ENABLE |
+				 PACKAGE_THERM_INT_HIGH_ENABLE;
+		wrmsrq(MSR_IA32_PACKAGE_THERM_INTERRUPT, val.q);
 
 		if (cpu_has(c, X86_FEATURE_HFI)) {
-			rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
-			wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
-			      l | PACKAGE_THERM_INT_HFI_ENABLE, h);
+			rdmsrq(MSR_IA32_PACKAGE_THERM_INTERRUPT, val.q);
+			wrmsrq(MSR_IA32_PACKAGE_THERM_INTERRUPT,
+			       val.q | PACKAGE_THERM_INT_HFI_ENABLE);
 		}
 	}
 
-	rdmsr(MSR_IA32_MISC_ENABLE, l, h);
-	wrmsr(MSR_IA32_MISC_ENABLE, l | MSR_IA32_MISC_ENABLE_TM1, h);
+	rdmsrq(MSR_IA32_MISC_ENABLE, val.q);
+	wrmsrq(MSR_IA32_MISC_ENABLE, val.q | MSR_IA32_MISC_ENABLE_TM1);
 
 	pr_info_once("CPU0: Thermal monitoring enabled (%s)\n",
 		      tm2 ? "TM2" : "TM1");
diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c
index 688e04c63761a..43fd5bdf1d8d8 100644
--- a/drivers/thermal/intel/x86_pkg_temp_thermal.c
+++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c
@@ -51,8 +51,7 @@ MODULE_PARM_DESC(notify_delay_ms,
 struct zone_device {
 	int				cpu;
 	bool				work_scheduled;
-	u32				msr_pkg_therm_low;
-	u32				msr_pkg_therm_high;
+	u64				msr_pkg_therm;
 	struct delayed_work		work;
 	struct thermal_zone_device	*tzone;
 	struct cpumask			cpumask;
@@ -186,28 +185,28 @@ static bool pkg_thermal_rate_control(void)
 static inline void enable_pkg_thres_interrupt(void)
 {
 	u8 thres_0, thres_1;
-	u32 l, h;
+	struct msr val;
 
-	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
+	rdmsrq(MSR_IA32_PACKAGE_THERM_INTERRUPT, val.q);
 	/* only enable/disable if it had valid threshold value */
-	thres_0 = (l & THERM_MASK_THRESHOLD0) >> THERM_SHIFT_THRESHOLD0;
-	thres_1 = (l & THERM_MASK_THRESHOLD1) >> THERM_SHIFT_THRESHOLD1;
+	thres_0 = (val.l & THERM_MASK_THRESHOLD0) >> THERM_SHIFT_THRESHOLD0;
+	thres_1 = (val.l & THERM_MASK_THRESHOLD1) >> THERM_SHIFT_THRESHOLD1;
 	if (thres_0)
-		l |= THERM_INT_THRESHOLD0_ENABLE;
+		val.l |= THERM_INT_THRESHOLD0_ENABLE;
 	if (thres_1)
-		l |= THERM_INT_THRESHOLD1_ENABLE;
-	wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
+		val.l |= THERM_INT_THRESHOLD1_ENABLE;
+	wrmsrq(MSR_IA32_PACKAGE_THERM_INTERRUPT, val.q);
 }
 
 /* Disable threshold interrupt on local package/cpu */
 static inline void disable_pkg_thres_interrupt(void)
 {
-	u32 l, h;
+	struct msr val;
 
-	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
+	rdmsrq(MSR_IA32_PACKAGE_THERM_INTERRUPT, val.q);
 
-	l &= ~(THERM_INT_THRESHOLD0_ENABLE | THERM_INT_THRESHOLD1_ENABLE);
-	wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
+	val.l &= ~(THERM_INT_THRESHOLD0_ENABLE | THERM_INT_THRESHOLD1_ENABLE);
+	wrmsrq(MSR_IA32_PACKAGE_THERM_INTERRUPT, val.q);
 }
 
 static void pkg_temp_thermal_threshold_work_fn(struct work_struct *work)
@@ -357,8 +356,7 @@ static int pkg_temp_thermal_device_add(unsigned int cpu)
 		goto out_unregister_tz;
 
 	/* Store MSR value for package thermal interrupt, to restore at exit */
-	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, zonedev->msr_pkg_therm_low,
-	      zonedev->msr_pkg_therm_high);
+	rdmsrq(MSR_IA32_PACKAGE_THERM_INTERRUPT, zonedev->msr_pkg_therm);
 
 	cpumask_set_cpu(cpu, &zonedev->cpumask);
 	raw_spin_lock_irq(&pkg_temp_lock);
@@ -426,8 +424,8 @@ static int pkg_thermal_cpu_offline(unsigned int cpu)
 	if (lastcpu) {
 		zones[topology_logical_die_id(cpu)] = NULL;
 		/* After this point nothing touches the MSR anymore. */
-		wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
-		      zonedev->msr_pkg_therm_low, zonedev->msr_pkg_therm_high);
+		wrmsrq(MSR_IA32_PACKAGE_THERM_INTERRUPT,
+		       zonedev->msr_pkg_therm);
 	}
 
 	/*
diff --git a/drivers/thermal/qcom/qcom-spmi-adc-tm5.c b/drivers/thermal/qcom/qcom-spmi-adc-tm5.c
index d7f2e6ca92c2c..bb6222c8cc5f0 100644
--- a/drivers/thermal/qcom/qcom-spmi-adc-tm5.c
+++ b/drivers/thermal/qcom/qcom-spmi-adc-tm5.c
@@ -1069,3 +1069,4 @@ module_platform_driver(adc_tm5_driver);
 
 MODULE_DESCRIPTION("SPMI PMIC Thermal Monitor ADC driver");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/thermal/qcom/qcom-spmi-temp-alarm.c b/drivers/thermal/qcom/qcom-spmi-temp-alarm.c
index f39ca0ddd17b6..fb003ca964546 100644
--- a/drivers/thermal/qcom/qcom-spmi-temp-alarm.c
+++ b/drivers/thermal/qcom/qcom-spmi-temp-alarm.c
@@ -904,3 +904,4 @@ module_platform_driver(qpnp_tm_driver);
 MODULE_ALIAS("platform:spmi-temp-alarm");
 MODULE_DESCRIPTION("QPNP PMIC Temperature Alarm driver");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/thermal/renesas/rzg3s_thermal.c b/drivers/thermal/renesas/rzg3s_thermal.c
index e25e36c99a886..7ced8f76a0ec3 100644
--- a/drivers/thermal/renesas/rzg3s_thermal.c
+++ b/drivers/thermal/renesas/rzg3s_thermal.c
@@ -270,3 +270,4 @@ module_platform_driver(rzg3s_thermal_driver);
 MODULE_DESCRIPTION("Renesas RZ/G3S Thermal Sensor Unit Driver");
 MODULE_AUTHOR("Claudiu Beznea <claudiu.beznea.uj@bp.renesas.com>");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/thermal/thermal-generic-adc.c b/drivers/thermal/thermal-generic-adc.c
index 7c844589b153b..cfdb8e674dd27 100644
--- a/drivers/thermal/thermal-generic-adc.c
+++ b/drivers/thermal/thermal-generic-adc.c
@@ -228,3 +228,4 @@ module_platform_driver(gadc_thermal_driver);
 MODULE_AUTHOR("Laxman Dewangan <ldewangan@nvidia.com>");
 MODULE_DESCRIPTION("Generic ADC thermal driver using IIO framework with DT");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/drivers/thunderbolt/dma_test.c b/drivers/thunderbolt/dma_test.c
index 7877319b1b03a..519c67678b08b 100644
--- a/drivers/thunderbolt/dma_test.c
+++ b/drivers/thunderbolt/dma_test.c
@@ -636,7 +636,7 @@ static void dma_test_debugfs_init(struct tb_service *svc)
 	debugfs_create_file("test", 0200, debugfs_dir, svc, &test_fops);
 }
 
-static int dma_test_probe(struct tb_service *svc, const struct tb_service_id *id)
+static int dma_test_probe(struct tb_service *svc)
 {
 	struct tb_xdomain *xd = tb_service_parent(svc);
 	struct dma_test *dt;
@@ -689,7 +689,7 @@ static const struct dev_pm_ops dma_test_pm_ops = {
 
 static const struct tb_service_id dma_test_ids[] = {
 	{ TB_SERVICE("dma_test", 1) },
-	{ },
+	{ }
 };
 MODULE_DEVICE_TABLE(tbsvc, dma_test_ids);
 
diff --git a/drivers/thunderbolt/domain.c b/drivers/thunderbolt/domain.c
index 479fa4d265c21..24611f05b3cda 100644
--- a/drivers/thunderbolt/domain.c
+++ b/drivers/thunderbolt/domain.c
@@ -77,12 +77,10 @@ static int tb_service_probe(struct device *dev)
 {
 	struct tb_service *svc = tb_to_service(dev);
 	struct tb_service_driver *driver;
-	const struct tb_service_id *id;
 
 	driver = container_of(dev->driver, struct tb_service_driver, driver);
-	id = __tb_service_match(dev, &driver->driver);
 
-	return driver->probe(svc, id);
+	return driver->probe(svc);
 }
 
 static void tb_service_remove(struct device *dev)
diff --git a/drivers/thunderbolt/eeprom.c b/drivers/thunderbolt/eeprom.c
index 5681c17f82ecc..2a13fa6888ba1 100644
--- a/drivers/thunderbolt/eeprom.c
+++ b/drivers/thunderbolt/eeprom.c
@@ -394,9 +394,16 @@ static int tb_drom_parse_entry_port(struct tb_switch *sw,
 			return -EIO;
 		}
 		port->link_nr = entry->link_nr;
-		if (entry->has_dual_link_port)
+		if (entry->has_dual_link_port) {
+			if (entry->dual_link_port_nr > sw->config.max_port_number) {
+				tb_sw_warn(sw,
+					"port entry has invalid dual link port number %u\n",
+					entry->dual_link_port_nr);
+				return -EIO;
+			}
 			port->dual_link_port =
 				&port->sw->ports[entry->dual_link_port_nr];
+		}
 	}
 	return 0;
 }
diff --git a/drivers/thunderbolt/nhi.c b/drivers/thunderbolt/nhi.c
index 0f795ea58756d..698fb124d5291 100644
--- a/drivers/thunderbolt/nhi.c
+++ b/drivers/thunderbolt/nhi.c
@@ -1235,6 +1235,8 @@ int nhi_probe(struct tb_nhi *nhi)
 
 	init_completion(&nhi->domain_released);
 
+	nhi->host_reset = host_reset;
+
 	res = tb_domain_add(tb, host_reset);
 	if (res) {
 		/*
diff --git a/drivers/thunderbolt/pci.c b/drivers/thunderbolt/pci.c
index bbd186c29ef72..dbb6badda867f 100644
--- a/drivers/thunderbolt/pci.c
+++ b/drivers/thunderbolt/pci.c
@@ -230,7 +230,7 @@ static void nhi_pci_ring_release_msix(struct tb_ring *ring)
 	ring->irq = 0;
 }
 
-static void nhi_pci_shutdown(struct tb_nhi *nhi)
+static void nhi_pci_release_irq(struct tb_nhi *nhi)
 {
 	struct tb_nhi_pci *nhi_pci = nhi_to_pci(nhi);
 	struct pci_dev *pdev = to_pci_dev(nhi->dev);
@@ -256,7 +256,7 @@ static const struct tb_nhi_ops pci_nhi_default_ops = {
 	.post_nvm_auth = nhi_pci_complete_dma_port,
 	.request_ring_irq = nhi_pci_ring_request_msix,
 	.release_ring_irq = nhi_pci_ring_release_msix,
-	.shutdown = nhi_pci_shutdown,
+	.shutdown = nhi_pci_release_irq,
 	.is_present = nhi_pci_is_present,
 	.init_interrupts = nhi_pci_init_msi,
 };
@@ -424,7 +424,7 @@ static int icl_nhi_resume(struct tb_nhi *nhi)
 
 static void icl_nhi_shutdown(struct tb_nhi *nhi)
 {
-	nhi_pci_shutdown(nhi);
+	nhi_pci_release_irq(nhi);
 
 	icl_nhi_force_power(nhi, false);
 }
@@ -479,11 +479,19 @@ static int nhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	return nhi_probe(&nhi_pci->nhi);
 }
 
-static void nhi_pci_remove(struct pci_dev *pdev)
+static void nhi_pci_do_remove(struct pci_dev *pdev, bool reset)
 {
 	struct tb *tb = pci_get_drvdata(pdev);
 	struct tb_nhi *nhi = tb->nhi;
 
+	/*
+	 * On system shutdown/reboot force a host router reset so the
+	 * connection manager asserts DPR on connected Thunderbolt 3 devices
+	 * before the router tree is removed (see tb_stop()).
+	 */
+	if (reset)
+		nhi->host_reset = true;
+
 	pm_runtime_get_sync(&pdev->dev);
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
 	pm_runtime_forbid(&pdev->dev);
@@ -493,6 +501,16 @@ static void nhi_pci_remove(struct pci_dev *pdev)
 	nhi_shutdown(nhi);
 }
 
+static void nhi_pci_remove(struct pci_dev *pdev)
+{
+	nhi_pci_do_remove(pdev, false);
+}
+
+static void nhi_pci_shutdown(struct pci_dev *pdev)
+{
+	nhi_pci_do_remove(pdev, true);
+}
+
 static struct pci_device_id nhi_ids[] = {
 	/*
 	 * We have to specify class, the TB bridges use the same device and
@@ -593,7 +611,7 @@ static struct pci_driver nhi_driver = {
 	.id_table = nhi_ids,
 	.probe = nhi_pci_probe,
 	.remove = nhi_pci_remove,
-	.shutdown = nhi_pci_remove,
+	.shutdown = nhi_pci_shutdown,
 	.driver.pm = &nhi_pm_ops,
 };
 
diff --git a/drivers/thunderbolt/stream.c b/drivers/thunderbolt/stream.c
index c1f5c55583d06..7368a07934acd 100644
--- a/drivers/thunderbolt/stream.c
+++ b/drivers/thunderbolt/stream.c
@@ -257,7 +257,7 @@ static void tbstream_ring_free(struct tbstream_ring *ring)
 
 		if (sf->frame.buffer_phy)
 			dma_unmap_page(dma_dev, sf->frame.buffer_phy,
-				       tb_ring_frame_size(&sf->frame), dir);
+				       TB_MAX_FRAME_SIZE, dir);
 		sf->frame.buffer_phy = 0;
 		if (sf->page)
 			__free_page(sf->page);
@@ -1540,7 +1540,7 @@ static void tbstream_group_detach_stream(struct tbstream *stream)
 	config_group_put(&sg->group);
 }
 
-static int tbstream_probe(struct tb_service *svc, const struct tb_service_id *id)
+static int tbstream_probe(struct tb_service *svc)
 {
 	struct tbstream *stream;
 
@@ -1630,7 +1630,7 @@ static const struct dev_pm_ops tbstream_pm_ops = {
 
 static const struct tb_service_id tbstream_ids[] = {
 	{ TB_SERVICE("stream", 1) },
-	{ },
+	{ }
 };
 MODULE_DEVICE_TABLE(tbsvc, tbstream_ids);
 
diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c
index a830c82bb9059..404c0693df50e 100644
--- a/drivers/thunderbolt/switch.c
+++ b/drivers/thunderbolt/switch.c
@@ -682,7 +682,16 @@ int tb_port_disable(struct tb_port *port)
 	return __tb_port_enable(port, false);
 }
 
-static int tb_port_reset(struct tb_port *port)
+/**
+ * tb_port_reset() - Reset the port
+ * @port: Port to reset
+ *
+ * Resets @port. For USB4 ports this issues a USB4 port reset and for
+ * legacy ports the link controller port is reset.
+ *
+ * Return: %0 on success, negative errno otherwise.
+ */
+int tb_port_reset(struct tb_port *port)
 {
 	if (tb_switch_is_usb4(port->sw))
 		return port->cap_usb4 ? usb4_port_reset(port) : 0;
diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c
index 76323255439a8..47753a5c0f2eb 100644
--- a/drivers/thunderbolt/tb.c
+++ b/drivers/thunderbolt/tb.c
@@ -609,7 +609,7 @@ static int tb_consumed_dp_bandwidth(struct tb *tb,
 				    int *consumed_up,
 				    int *consumed_down)
 {
-	int group_reserved[MAX_GROUPS] = {};
+	int group_reserved[MAX_GROUPS + 1] = {};
 	struct tb_cm *tcm = tb_priv(tb);
 	struct tb_tunnel *tunnel;
 	bool downstream;
@@ -2941,7 +2941,9 @@ static void tb_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,
 static void tb_stop(struct tb *tb)
 {
 	struct tb_cm *tcm = tb_priv(tb);
+	struct tb_nhi *nhi = tb->nhi;
 	struct tb_tunnel *tunnel;
+	struct tb_port *port;
 	struct tb_tunnel *n;
 
 	cancel_delayed_work(&tcm->remove_work);
@@ -2956,6 +2958,25 @@ static void tb_stop(struct tb *tb)
 			tb_tunnel_deactivate(tunnel);
 		tb_tunnel_put(tunnel);
 	}
+	/*
+	 * Signal disconnect to connected devices before the router tree is
+	 * removed below. A Thunderbolt 3 device directly connected to a USB4
+	 * host otherwise never receives a disconnect indication, leaving
+	 * firmware to poll the dead link for up to ~60 s which on some
+	 * platforms turns the shutdown into a warm reset. Asserting
+	 * PORT_CS_19.DPR drives SBTX low (USB4 spec section 6.9) so the device
+	 * detects SBRX low and goes to Uninitialized Unplugged immediately.
+	 */
+	if (nhi->host_reset) {
+		tb_switch_for_each_port(tb->root_switch, port) {
+			if (!tb_port_is_null(port) || !tb_port_has_remote(port))
+				continue;
+			if (tb_switch_is_usb4(port->remote->sw))
+				continue;
+			if (tb_port_reset(port))
+				tb_port_dbg(port, "downstream port reset failed, continuing\n");
+		}
+	}
 	tb_switch_remove(tb->root_switch);
 	tb->root_switch = NULL;
 	tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */
diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h
index ec9192b61bc0e..4373336d9425e 100644
--- a/drivers/thunderbolt/tb.h
+++ b/drivers/thunderbolt/tb.h
@@ -1103,6 +1103,7 @@ int tb_port_clear_counter(struct tb_port *port, int counter);
 int tb_port_unlock(struct tb_port *port);
 int tb_port_enable(struct tb_port *port);
 int tb_port_disable(struct tb_port *port);
+int tb_port_reset(struct tb_port *port);
 int tb_port_alloc_in_hopid(struct tb_port *port, int hopid, int max_hopid);
 void tb_port_release_in_hopid(struct tb_port *port, int hopid);
 int tb_port_alloc_out_hopid(struct tb_port *port, int hopid, int max_hopid);
diff --git a/drivers/thunderbolt/xdomain.c b/drivers/thunderbolt/xdomain.c
index 86b2f74746700..c179bd751fe42 100644
--- a/drivers/thunderbolt/xdomain.c
+++ b/drivers/thunderbolt/xdomain.c
@@ -968,6 +968,9 @@ tb_xdp_schedule_request(struct tb *tb, const struct tb_xdp_header *hdr,
  */
 int tb_register_service_driver(struct tb_service_driver *drv)
 {
+	if (!drv->probe)
+		return -EINVAL;
+
 	drv->driver.bus = &tb_bus_type;
 	return driver_register(&drv->driver);
 }
@@ -1811,6 +1814,7 @@ static void tb_xdomain_state_work(struct work_struct *work)
 			tb_xdomain_failed(xd);
 		} else {
 			xd->state = XDOMAIN_STATE_ENUMERATED;
+			tb_xdomain_queue_properties_changed(xd);
 		}
 		break;
 
diff --git a/drivers/tty/amiserial.c b/drivers/tty/amiserial.c
index 81eaca751541a..28af0fd981818 100644
--- a/drivers/tty/amiserial.c
+++ b/drivers/tty/amiserial.c
@@ -443,23 +443,23 @@ static int rs_startup(struct tty_struct *tty, struct serial_state *info)
 	struct tty_port *port = &info->tport;
 	unsigned long flags;
 	int	retval=0;
-	unsigned long page;
+	void *buffer;
 
-	page = get_zeroed_page(GFP_KERNEL);
-	if (!page)
+	buffer = kzalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!buffer)
 		return -ENOMEM;
 
 	local_irq_save(flags);
 
 	if (tty_port_initialized(port)) {
-		free_page(page);
+		kfree(buffer);
 		goto errout;
 	}
 
 	if (info->xmit.buf)
-		free_page(page);
+		kfree(buffer);
 	else
-		info->xmit.buf = (unsigned char *) page;
+		info->xmit.buf = buffer;
 
 #ifdef SERIAL_DEBUG_OPEN
 	printk("starting up ttys%d ...", info->line);
@@ -537,7 +537,7 @@ static void rs_shutdown(struct tty_struct *tty, struct serial_state *info)
 	 */
 	free_irq(IRQ_AMIGA_VERTB, info);
 
-	free_page((unsigned long)info->xmit.buf);
+	kfree(info->xmit.buf);
 	info->xmit.buf = NULL;
 
 	info->IER = 0;
diff --git a/drivers/tty/goldfish.c b/drivers/tty/goldfish.c
index 16edd71a0d8d7..fa723a4ba7b39 100644
--- a/drivers/tty/goldfish.c
+++ b/drivers/tty/goldfish.c
@@ -17,6 +17,7 @@
 #include <linux/mm.h>
 #include <linux/dma-mapping.h>
 #include <linux/serial_core.h>
+#include <linux/wordpart.h>
 
 /* Goldfish tty register's offsets */
 #define	GOLDFISH_TTY_REG_BYTES_READY	0x04
@@ -37,7 +38,6 @@ struct goldfish_tty {
 	spinlock_t lock;
 	void __iomem *base;
 	u32 irq;
-	int opencount;
 	struct console console;
 	u32 version;
 	struct device *dev;
@@ -49,15 +49,23 @@ static u32 goldfish_tty_line_count = 8;
 static u32 goldfish_tty_current_line_count;
 static struct goldfish_tty *goldfish_ttys;
 
+static inline void gf_write_addr(unsigned long addr, void __iomem *portl, void __iomem *porth)
+{
+	gf_iowrite32(lower_32_bits(addr), portl);
+#ifdef CONFIG_64BIT
+	gf_iowrite32(upper_32_bits(addr), porth);
+#endif
+}
+
 static void do_rw_io(struct goldfish_tty *qtty, unsigned long address,
 		     size_t count, bool is_write)
 {
-	unsigned long irq_flags;
 	void __iomem *base = qtty->base;
 
-	spin_lock_irqsave(&qtty->lock, irq_flags);
-	gf_write_ptr((void *)address, base + GOLDFISH_TTY_REG_DATA_PTR,
-		     base + GOLDFISH_TTY_REG_DATA_PTR_HIGH);
+	guard(spinlock_irqsave)(&qtty->lock);
+
+	gf_write_addr(address, base + GOLDFISH_TTY_REG_DATA_PTR,
+		      base + GOLDFISH_TTY_REG_DATA_PTR_HIGH);
 	gf_iowrite32(count, base + GOLDFISH_TTY_REG_DATA_LEN);
 
 	if (is_write)
@@ -66,8 +74,6 @@ static void do_rw_io(struct goldfish_tty *qtty, unsigned long address,
 	else
 		gf_iowrite32(GOLDFISH_TTY_CMD_READ_BUFFER,
 		       base + GOLDFISH_TTY_REG_CMD);
-
-	spin_unlock_irqrestore(&qtty->lock, irq_flags);
 }
 
 static void goldfish_tty_rw(struct goldfish_tty *qtty, unsigned long addr,
@@ -409,7 +415,7 @@ static void goldfish_tty_remove(struct platform_device *pdev)
 {
 	struct goldfish_tty *qtty = platform_get_drvdata(pdev);
 
-	mutex_lock(&goldfish_tty_lock);
+	guard(mutex)(&goldfish_tty_lock);
 
 	unregister_console(&qtty->console);
 	tty_unregister_device(goldfish_tty_driver, qtty->console.index);
@@ -420,7 +426,6 @@ static void goldfish_tty_remove(struct platform_device *pdev)
 	goldfish_tty_current_line_count--;
 	if (goldfish_tty_current_line_count == 0)
 		goldfish_tty_delete_driver();
-	mutex_unlock(&goldfish_tty_lock);
 }
 
 #ifdef CONFIG_GOLDFISH_TTY_EARLY_CONSOLE
diff --git a/drivers/tty/serdev/core.c b/drivers/tty/serdev/core.c
index e9d044a331b03..7500efcdfc218 100644
--- a/drivers/tty/serdev/core.c
+++ b/drivers/tty/serdev/core.c
@@ -651,11 +651,11 @@ static int acpi_serdev_do_lookup(struct acpi_device *adev,
 	INIT_LIST_HEAD(&resource_list);
 	ret = acpi_dev_get_resources(adev, &resource_list,
 				     acpi_serdev_parse_resource, lookup);
-	acpi_dev_free_resource_list(&resource_list);
-
 	if (ret < 0)
 		return -EINVAL;
 
+	acpi_dev_free_resource_list(&resource_list);
+
 	return 0;
 }
 
diff --git a/drivers/tty/serial/8250/8250.h b/drivers/tty/serial/8250/8250.h
index 77fe0588fd6bd..9d1068d0489dc 100644
--- a/drivers/tty/serial/8250/8250.h
+++ b/drivers/tty/serial/8250/8250.h
@@ -334,6 +334,13 @@ int fintek_8250_probe(struct uart_8250_port *uart);
 static inline int fintek_8250_probe(struct uart_8250_port *uart) { return 0; }
 #endif
 
+#ifdef CONFIG_SERIAL_8250_HUB6
+bool hub6_match_port(const struct uart_port *port1, const struct uart_port *port2);
+#else
+static inline bool hub6_match_port(const struct uart_port *port1, const struct uart_port *port2)
+{ return false; }
+#endif
+
 #ifdef CONFIG_ARCH_OMAP1
 #include <linux/soc/ti/omap1-soc.h>
 static inline int is_omap1_8250(struct uart_8250_port *pt)
diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c
index f49862d90eebb..c0e8a4efbdcc8 100644
--- a/drivers/tty/serial/8250/8250_core.c
+++ b/drivers/tty/serial/8250/8250_core.c
@@ -720,8 +720,12 @@ int serial8250_register_8250_port(const struct uart_8250_port *up)
 	/* Preserve specified console flow control. */
 	cons_flow = uart_cons_flow_enabled(&uart->port);
 
-	if (uart->port.dev)
+	if (uart->port.dev) {
+		if (uart_console(&uart->port))
+			uart->port.cons->flags &= ~CON_PRINTBUFFER;
+
 		uart_remove_one_port(&serial8250_reg, &uart->port);
+	}
 
 	uart->port.ctrl_id	= up->port.ctrl_id;
 	uart->port.port_id	= up->port.port_id;
diff --git a/drivers/tty/serial/8250/8250_early.c b/drivers/tty/serial/8250/8250_early.c
index dc0371857ecbe..44ec209f37c47 100644
--- a/drivers/tty/serial/8250/8250_early.c
+++ b/drivers/tty/serial/8250/8250_early.c
@@ -23,6 +23,7 @@
  *	console=uart8250,mmio32,0xff5e0000,115200n8
  */
 
+#include <linux/align.h>
 #include <linux/tty.h>
 #include <linux/init.h>
 #include <linux/console.h>
@@ -177,6 +178,23 @@ OF_EARLYCON_DECLARE(ns16550a, "ns16550a", early_serial8250_setup);
 OF_EARLYCON_DECLARE(uart, "nvidia,tegra20-uart", early_serial8250_setup);
 OF_EARLYCON_DECLARE(uart, "snps,dw-apb-uart", early_serial8250_setup);
 
+static int __init early_serial8250_xscale_setup(struct earlycon_device *device,
+					     const char *options)
+{
+	/*
+	 * Adjust for BE32 register accesses: drop any hardcoded
+	 * address for the big endian byte target, add it explicitly
+	 * if running on BE32.
+	 */
+	device->port.membase = PTR_ALIGN_DOWN(device->port.membase, 4);
+	if (IS_ENABLED(CONFIG_CPU_ENDIAN_BE32))
+		device->port.membase += 3;
+	device->port.regshift = 2;
+
+	return early_serial8250_setup(device, options);
+}
+OF_EARLYCON_DECLARE(uart, "intel,xscale-uart", early_serial8250_xscale_setup);
+
 static int __init early_serial8250_rs2_setup(struct earlycon_device *device,
 					     const char *options)
 {
@@ -184,7 +202,6 @@ static int __init early_serial8250_rs2_setup(struct earlycon_device *device,
 
 	return early_serial8250_setup(device, options);
 }
-OF_EARLYCON_DECLARE(uart, "intel,xscale-uart", early_serial8250_rs2_setup);
 OF_EARLYCON_DECLARE(uart, "mrvl,mmp-uart", early_serial8250_rs2_setup);
 OF_EARLYCON_DECLARE(uart, "mrvl,pxa-uart", early_serial8250_rs2_setup);
 
diff --git a/drivers/tty/serial/8250/8250_exar.c b/drivers/tty/serial/8250/8250_exar.c
index c682c0d0dffa4..f9a14eaa13cba 100644
--- a/drivers/tty/serial/8250/8250_exar.c
+++ b/drivers/tty/serial/8250/8250_exar.c
@@ -1642,14 +1642,14 @@ static const struct exar8250_board pbn_exar_XR17V8358 = {
 	.exit		= pci_xr17v35x_exit,
 };
 
-#define CTI_EXAR_DEVICE(devid, bd) {                    \
-	PCI_DEVICE_SUB(                                 \
-		PCI_VENDOR_ID_EXAR,                     \
-		PCI_DEVICE_ID_EXAR_##devid,             \
-		PCI_SUBVENDOR_ID_CONNECT_TECH,          \
-		PCI_ANY_ID), 0, 0,                      \
-		(kernel_ulong_t)&bd                     \
-	}
+#define CTI_EXAR_DEVICE(devid, bd) {			\
+	PCI_DEVICE_SUB(					\
+		PCI_VENDOR_ID_EXAR,			\
+		PCI_DEVICE_ID_EXAR_##devid,		\
+		PCI_SUBVENDOR_ID_CONNECT_TECH,		\
+		PCI_ANY_ID),				\
+	.driver_data = (kernel_ulong_t)&bd		\
+}
 
 #define EXAR_DEVICE(vend, devid, bd) { PCI_DEVICE_DATA(vend, devid, &bd) }
 
@@ -1658,18 +1658,18 @@ static const struct exar8250_board pbn_exar_XR17V8358 = {
 		PCI_VENDOR_ID_EXAR,			\
 		PCI_DEVICE_ID_EXAR_##devid,		\
 		PCI_SUBVENDOR_ID_IBM,			\
-		PCI_SUBDEVICE_ID_IBM_##sdevid), 0, 0,	\
-		(kernel_ulong_t)&bd			\
-	}
+		PCI_SUBDEVICE_ID_IBM_##sdevid),		\
+	.driver_data = (kernel_ulong_t)&bd		\
+}
 
 #define USR_DEVICE(devid, sdevid, bd) {			\
 	PCI_DEVICE_SUB(					\
 		PCI_VENDOR_ID_USR,			\
 		PCI_DEVICE_ID_EXAR_##devid,		\
 		PCI_VENDOR_ID_EXAR,			\
-		PCI_SUBDEVICE_ID_USR_##sdevid), 0, 0,	\
-		(kernel_ulong_t)&bd			\
-	}
+		PCI_SUBDEVICE_ID_USR_##sdevid),		\
+	.driver_data = (kernel_ulong_t)&bd		\
+}
 
 static const struct pci_device_id exar_pci_tbl[] = {
 	EXAR_DEVICE(ACCESSIO, COM_2S, pbn_exar_XR17C15x),
@@ -1726,7 +1726,7 @@ static const struct pci_device_id exar_pci_tbl[] = {
 	EXAR_DEVICE(COMMTECH, 4224PCI335, pbn_fastcom335_4),
 	EXAR_DEVICE(COMMTECH, 2324PCI335, pbn_fastcom335_4),
 	EXAR_DEVICE(COMMTECH, 2328PCI335, pbn_fastcom335_8),
-	{ 0, }
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, exar_pci_tbl);
 
diff --git a/drivers/tty/serial/8250/8250_hub6.c b/drivers/tty/serial/8250/8250_hub6.c
index 273f59b9bca55..b6767633c966c 100644
--- a/drivers/tty/serial/8250/8250_hub6.c
+++ b/drivers/tty/serial/8250/8250_hub6.c
@@ -7,6 +7,8 @@
 #include <linux/init.h>
 #include <linux/serial_8250.h>
 
+#include "8250.h"
+
 #define HUB6(card, port)						\
 	{								\
 		.iobase		= 0x302,				\
@@ -41,6 +43,12 @@ static struct platform_device hub6_device = {
 	},
 };
 
+bool hub6_match_port(const struct uart_port *port1, const struct uart_port *port2)
+{
+	return port1->iobase == port2->iobase && port1->hub6 == port2->hub6;
+}
+EXPORT_SYMBOL_GPL(hub6_match_port);
+
 static int __init hub6_init(void)
 {
 	return platform_device_register(&hub6_device);
diff --git a/drivers/tty/serial/8250/8250_mid.c b/drivers/tty/serial/8250/8250_mid.c
index f88809ff370b7..82656645b8a64 100644
--- a/drivers/tty/serial/8250/8250_mid.c
+++ b/drivers/tty/serial/8250/8250_mid.c
@@ -318,9 +318,11 @@ static int mid8250_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	if (!uart.port.membase)
 		return -ENOMEM;
 
-	ret = mid->board->setup(mid, &uart.port);
-	if (ret)
-		return ret;
+	if (mid->board->setup) {
+		ret = mid->board->setup(mid, &uart.port);
+		if (ret)
+			return ret;
+	}
 
 	ret = mid8250_dma_setup(mid, &uart);
 	if (ret)
@@ -336,7 +338,8 @@ static int mid8250_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	return 0;
 
 err:
-	mid->board->exit(mid);
+	if (mid->board->exit)
+		mid->board->exit(mid);
 	return ret;
 }
 
@@ -346,7 +349,8 @@ static void mid8250_remove(struct pci_dev *pdev)
 
 	serial8250_unregister_port(mid->line);
 
-	mid->board->exit(mid);
+	if (mid->board->exit)
+		mid->board->exit(mid);
 }
 
 static const struct mid8250_board pnw_board = {
diff --git a/drivers/tty/serial/8250/8250_of.c b/drivers/tty/serial/8250/8250_of.c
index 81644d40b09a1..f0537fb6ef4f7 100644
--- a/drivers/tty/serial/8250/8250_of.c
+++ b/drivers/tty/serial/8250/8250_of.c
@@ -5,6 +5,7 @@
  *    Copyright (C) 2006 Arnd Bergmann <arnd@arndb.de>, IBM Corp.
  */
 
+#include <linux/align.h>
 #include <linux/bits.h>
 #include <linux/console.h>
 #include <linux/math.h>
@@ -122,6 +123,17 @@ static int of_platform_serial_setup(struct platform_device *ofdev,
 	if (ret)
 		goto err_pmruntime;
 
+	if (IS_ENABLED(CONFIG_CPU_XSCALE) && type == PORT_XSCALE) {
+		/*
+		 * Adjust for BE32 register accesses: drop any hardcoded
+		 * address for the big endian byte target, add it explicitly
+		 * if running on BE32.
+		 */
+		port->mapbase = PTR_ALIGN_DOWN(port->mapbase, 4);
+		if (IS_ENABLED(CONFIG_CPU_ENDIAN_BE32))
+			port->mapbase += 3;
+	}
+
 	/* Get clk rate through clk driver if present */
 	if (!port->uartclk) {
 		struct clk *bus_clk;
diff --git a/drivers/tty/serial/8250/8250_platform.c b/drivers/tty/serial/8250/8250_platform.c
index ad3a7bc31d6ff..af946d12e7648 100644
--- a/drivers/tty/serial/8250/8250_platform.c
+++ b/drivers/tty/serial/8250/8250_platform.c
@@ -284,6 +284,7 @@ static struct platform_driver serial8250_isa_driver = {
 	.driver		= {
 		.name	= "serial8250",
 		.acpi_match_table = acpi_platform_serial_table,
+		.probe_type = PROBE_FORCE_SYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/tty/serial/8250/8250_pnp.c b/drivers/tty/serial/8250/8250_pnp.c
index 7a837fdf9df13..6bfdeff5fe222 100644
--- a/drivers/tty/serial/8250/8250_pnp.c
+++ b/drivers/tty/serial/8250/8250_pnp.c
@@ -28,351 +28,351 @@
 static const struct pnp_device_id pnp_dev_table[] = {
 	/* Archtek America Corp. */
 	/* Archtek SmartLink Modem 3334BT Plug & Play */
-	{	"AAC000F",		0	},
+	{ .id = "AAC000F", .driver_data = 0 },
 	/* Anchor Datacomm BV */
 	/* SXPro 144 External Data Fax Modem Plug & Play */
-	{	"ADC0001",		0	},
+	{ .id = "ADC0001", .driver_data = 0 },
 	/* SXPro 288 External Data Fax Modem Plug & Play */
-	{	"ADC0002",		0	},
+	{ .id = "ADC0002", .driver_data = 0 },
 	/* PROLiNK 1456VH ISA PnP K56flex Fax Modem */
-	{	"AEI0250",		0	},
+	{ .id = "AEI0250", .driver_data = 0 },
 	/* Actiontec ISA PNP 56K X2 Fax Modem */
-	{	"AEI1240",		0	},
+	{ .id = "AEI1240", .driver_data = 0 },
 	/* Rockwell 56K ACF II Fax+Data+Voice Modem */
-	{	"AKY1021",		0 /*SPCI_FL_NO_SHIRQ*/	},
+	{ .id ="AKY1021", .driver_data = 0 /*SPCI_FL_NO_SHIRQ*/ },
 	/*
 	 * ALi Fast Infrared Controller
 	 * Native driver (ali-ircc) is broken so at least
 	 * it can be used with irtty-sir.
 	 */
-	{	"ALI5123",		0	},
+	{ .id = "ALI5123", .driver_data = 0 },
 	/* AZT3005 PnP SOUND DEVICE */
-	{	"AZT4001",		0	},
+	{ .id = "AZT4001", .driver_data = 0 },
 	/* Best Data Products Inc. Smart One 336F PnP Modem */
-	{	"BDP3336",		0	},
+	{ .id = "BDP3336", .driver_data = 0 },
 	/*  Boca Research */
 	/* Boca Complete Ofc Communicator 14.4 Data-FAX */
-	{	"BRI0A49",		0	},
+	{ .id = "BRI0A49", .driver_data = 0 },
 	/* Boca Research 33,600 ACF Modem */
-	{	"BRI1400",		0	},
+	{ .id = "BRI1400", .driver_data = 0 },
 	/* Boca 33.6 Kbps Internal FD34FSVD */
-	{	"BRI3400",		0	},
+	{ .id = "BRI3400", .driver_data = 0 },
 	/* Computer Peripherals Inc */
 	/* EuroViVa CommCenter-33.6 SP PnP */
-	{	"CPI4050",		0	},
+	{ .id = "CPI4050", .driver_data = 0 },
 	/* Creative Labs */
 	/* Creative Labs Phone Blaster 28.8 DSVD PnP Voice */
-	{	"CTL3001",		0	},
+	{ .id = "CTL3001", .driver_data = 0 },
 	/* Creative Labs Modem Blaster 28.8 DSVD PnP Voice */
-	{	"CTL3011",		0	},
+	{ .id = "CTL3011", .driver_data = 0 },
 	/* Davicom ISA 33.6K Modem */
-	{	"DAV0336",		0	},
+	{ .id = "DAV0336", .driver_data = 0 },
 	/* Creative */
 	/* Creative Modem Blaster Flash56 DI5601-1 */
-	{	"DMB1032",		0	},
+	{ .id = "DMB1032", .driver_data = 0 },
 	/* Creative Modem Blaster V.90 DI5660 */
-	{	"DMB2001",		0	},
+	{ .id = "DMB2001", .driver_data = 0 },
 	/* E-Tech */
 	/* E-Tech CyberBULLET PC56RVP */
-	{	"ETT0002",		0	},
+	{ .id = "ETT0002", .driver_data = 0 },
 	/* FUJITSU */
 	/* Fujitsu 33600 PnP-I2 R Plug & Play */
-	{	"FUJ0202",		0	},
+	{ .id = "FUJ0202", .driver_data = 0 },
 	/* Fujitsu FMV-FX431 Plug & Play */
-	{	"FUJ0205",		0	},
+	{ .id = "FUJ0205", .driver_data = 0 },
 	/* Fujitsu 33600 PnP-I4 R Plug & Play */
-	{	"FUJ0206",		0	},
+	{ .id = "FUJ0206", .driver_data = 0 },
 	/* Fujitsu Fax Voice 33600 PNP-I5 R Plug & Play */
-	{	"FUJ0209",		0	},
+	{ .id = "FUJ0209", .driver_data = 0 },
 	/* Archtek America Corp. */
 	/* Archtek SmartLink Modem 3334BT Plug & Play */
-	{	"GVC000F",		0	},
+	{ .id = "GVC000F", .driver_data = 0 },
 	/* Archtek SmartLink Modem 3334BRV 33.6K Data Fax Voice */
-	{	"GVC0303",		0	},
+	{ .id = "GVC0303", .driver_data = 0 },
 	/* Hayes */
 	/* Hayes Optima 288 V.34-V.FC + FAX + Voice Plug & Play */
-	{	"HAY0001",		0	},
+	{ .id = "HAY0001", .driver_data = 0 },
 	/* Hayes Optima 336 V.34 + FAX + Voice PnP */
-	{	"HAY000C",		0	},
+	{ .id = "HAY000C", .driver_data = 0 },
 	/* Hayes Optima 336B V.34 + FAX + Voice PnP */
-	{	"HAY000D",		0	},
+	{ .id = "HAY000D", .driver_data = 0 },
 	/* Hayes Accura 56K Ext Fax Modem PnP */
-	{	"HAY5670",		0	},
+	{ .id = "HAY5670", .driver_data = 0 },
 	/* Hayes Accura 56K Ext Fax Modem PnP */
-	{	"HAY5674",		0	},
+	{ .id = "HAY5674", .driver_data = 0 },
 	/* Hayes Accura 56K Fax Modem PnP */
-	{	"HAY5675",		0	},
+	{ .id = "HAY5675", .driver_data = 0 },
 	/* Hayes 288, V.34 + FAX */
-	{	"HAYF000",		0	},
+	{ .id = "HAYF000", .driver_data = 0 },
 	/* Hayes Optima 288 V.34 + FAX + Voice, Plug & Play */
-	{	"HAYF001",		0	},
+	{ .id = "HAYF001", .driver_data = 0 },
 	/* IBM */
 	/* IBM Thinkpad 701 Internal Modem Voice */
-	{	"IBM0033",		0	},
+	{ .id = "IBM0033", .driver_data = 0 },
 	/* Intermec */
 	/* Intermec CV60 touchscreen port */
-	{	"PNP4972",		0	},
+	{ .id = "PNP4972", .driver_data = 0 },
 	/* Intertex */
 	/* Intertex 28k8 33k6 Voice EXT PnP */
-	{	"IXDC801",		0	},
+	{ .id = "IXDC801", .driver_data = 0 },
 	/* Intertex 33k6 56k Voice EXT PnP */
-	{	"IXDC901",		0	},
+	{ .id = "IXDC901", .driver_data = 0 },
 	/* Intertex 28k8 33k6 Voice SP EXT PnP */
-	{	"IXDD801",		0	},
+	{ .id = "IXDD801", .driver_data = 0 },
 	/* Intertex 33k6 56k Voice SP EXT PnP */
-	{	"IXDD901",		0	},
+	{ .id = "IXDD901", .driver_data = 0 },
 	/* Intertex 28k8 33k6 Voice SP INT PnP */
-	{	"IXDF401",		0	},
+	{ .id = "IXDF401", .driver_data = 0 },
 	/* Intertex 28k8 33k6 Voice SP EXT PnP */
-	{	"IXDF801",		0	},
+	{ .id = "IXDF801", .driver_data = 0 },
 	/* Intertex 33k6 56k Voice SP EXT PnP */
-	{	"IXDF901",		0	},
+	{ .id = "IXDF901", .driver_data = 0 },
 	/* Kortex International */
 	/* KORTEX 28800 Externe PnP */
-	{	"KOR4522",		0	},
+	{ .id = "KOR4522", .driver_data = 0 },
 	/* KXPro 33.6 Vocal ASVD PnP */
-	{	"KORF661",		0	},
+	{ .id = "KORF661", .driver_data = 0 },
 	/* Lasat */
 	/* LASAT Internet 33600 PnP */
-	{	"LAS4040",		0	},
+	{ .id = "LAS4040", .driver_data = 0 },
 	/* Lasat Safire 560 PnP */
-	{	"LAS4540",		0	},
+	{ .id = "LAS4540", .driver_data = 0 },
 	/* Lasat Safire 336  PnP */
-	{	"LAS5440",		0	},
+	{ .id = "LAS5440", .driver_data = 0 },
 	/* Microcom, Inc. */
 	/* Microcom TravelPorte FAST V.34 Plug & Play */
-	{	"MNP0281",		0	},
+	{ .id = "MNP0281", .driver_data = 0 },
 	/* Microcom DeskPorte V.34 FAST or FAST+ Plug & Play */
-	{	"MNP0336",		0	},
+	{ .id = "MNP0336", .driver_data = 0 },
 	/* Microcom DeskPorte FAST EP 28.8 Plug & Play */
-	{	"MNP0339",		0	},
+	{ .id = "MNP0339", .driver_data = 0 },
 	/* Microcom DeskPorte 28.8P Plug & Play */
-	{	"MNP0342",		0	},
+	{ .id = "MNP0342", .driver_data = 0 },
 	/* Microcom DeskPorte FAST ES 28.8 Plug & Play */
-	{	"MNP0500",		0	},
+	{ .id = "MNP0500", .driver_data = 0 },
 	/* Microcom DeskPorte FAST ES 28.8 Plug & Play */
-	{	"MNP0501",		0	},
+	{ .id = "MNP0501", .driver_data = 0 },
 	/* Microcom DeskPorte 28.8S Internal Plug & Play */
-	{	"MNP0502",		0	},
+	{ .id = "MNP0502", .driver_data = 0 },
 	/* Motorola */
 	/* Motorola BitSURFR Plug & Play */
-	{	"MOT1105",		0	},
+	{ .id = "MOT1105", .driver_data = 0 },
 	/* Motorola TA210 Plug & Play */
-	{	"MOT1111",		0	},
+	{ .id = "MOT1111", .driver_data = 0 },
 	/* Motorola HMTA 200 (ISDN) Plug & Play */
-	{	"MOT1114",		0	},
+	{ .id = "MOT1114", .driver_data = 0 },
 	/* Motorola BitSURFR Plug & Play */
-	{	"MOT1115",		0	},
+	{ .id = "MOT1115", .driver_data = 0 },
 	/* Motorola Lifestyle 28.8 Internal */
-	{	"MOT1190",		0	},
+	{ .id = "MOT1190", .driver_data = 0 },
 	/* Motorola V.3400 Plug & Play */
-	{	"MOT1501",		0	},
+	{ .id = "MOT1501", .driver_data = 0 },
 	/* Motorola Lifestyle 28.8 V.34 Plug & Play */
-	{	"MOT1502",		0	},
+	{ .id = "MOT1502", .driver_data = 0 },
 	/* Motorola Power 28.8 V.34 Plug & Play */
-	{	"MOT1505",		0	},
+	{ .id = "MOT1505", .driver_data = 0 },
 	/* Motorola ModemSURFR External 28.8 Plug & Play */
-	{	"MOT1509",		0	},
+	{ .id = "MOT1509", .driver_data = 0 },
 	/* Motorola Premier 33.6 Desktop Plug & Play */
-	{	"MOT150A",		0	},
+	{ .id = "MOT150A", .driver_data = 0 },
 	/* Motorola VoiceSURFR 56K External PnP */
-	{	"MOT150F",		0	},
+	{ .id = "MOT150F", .driver_data = 0 },
 	/* Motorola ModemSURFR 56K External PnP */
-	{	"MOT1510",		0	},
+	{ .id = "MOT1510", .driver_data = 0 },
 	/* Motorola ModemSURFR 56K Internal PnP */
-	{	"MOT1550",		0	},
+	{ .id = "MOT1550", .driver_data = 0 },
 	/* Motorola ModemSURFR Internal 28.8 Plug & Play */
-	{	"MOT1560",		0	},
+	{ .id = "MOT1560", .driver_data = 0 },
 	/* Motorola Premier 33.6 Internal Plug & Play */
-	{	"MOT1580",		0	},
+	{ .id = "MOT1580", .driver_data = 0 },
 	/* Motorola OnlineSURFR 28.8 Internal Plug & Play */
-	{	"MOT15B0",		0	},
+	{ .id = "MOT15B0", .driver_data = 0 },
 	/* Motorola VoiceSURFR 56K Internal PnP */
-	{	"MOT15F0",		0	},
+	{ .id = "MOT15F0", .driver_data = 0 },
 	/* Com 1 */
 	/*  Deskline K56 Phone System PnP */
-	{	"MVX00A1",		0	},
+	{ .id = "MVX00A1", .driver_data = 0 },
 	/* PC Rider K56 Phone System PnP */
-	{	"MVX00F2",		0	},
+	{ .id = "MVX00F2", .driver_data = 0 },
 	/* NEC 98NOTE SPEAKER PHONE FAX MODEM(33600bps) */
-	{	"nEC8241",		0	},
+	{ .id = "nEC8241", .driver_data = 0 },
 	/* Pace 56 Voice Internal Plug & Play Modem */
-	{	"PMC2430",		0	},
+	{ .id = "PMC2430", .driver_data = 0 },
 	/* Generic */
 	/* Generic standard PC COM port	 */
-	{	"PNP0500",		0	},
+	{ .id = "PNP0500", .driver_data = 0 },
 	/* Generic 16550A-compatible COM port */
-	{	"PNP0501",		0	},
+	{ .id = "PNP0501", .driver_data = 0 },
 	/* Compaq 14400 Modem */
-	{	"PNPC000",		0	},
+	{ .id = "PNPC000", .driver_data = 0 },
 	/* Compaq 2400/9600 Modem */
-	{	"PNPC001",		0	},
+	{ .id = "PNPC001", .driver_data = 0 },
 	/* Dial-Up Networking Serial Cable between 2 PCs */
-	{	"PNPC031",		0	},
+	{ .id = "PNPC031", .driver_data = 0 },
 	/* Dial-Up Networking Parallel Cable between 2 PCs */
-	{	"PNPC032",		0	},
+	{ .id = "PNPC032", .driver_data = 0 },
 	/* Standard 9600 bps Modem */
-	{	"PNPC100",		0	},
+	{ .id = "PNPC100", .driver_data = 0 },
 	/* Standard 14400 bps Modem */
-	{	"PNPC101",		0	},
+	{ .id = "PNPC101", .driver_data = 0 },
 	/*  Standard 28800 bps Modem*/
-	{	"PNPC102",		0	},
+	{ .id = "PNPC102", .driver_data = 0 },
 	/*  Standard Modem*/
-	{	"PNPC103",		0	},
+	{ .id = "PNPC103", .driver_data = 0 },
 	/*  Standard 9600 bps Modem*/
-	{	"PNPC104",		0	},
+	{ .id = "PNPC104", .driver_data = 0 },
 	/*  Standard 14400 bps Modem*/
-	{	"PNPC105",		0	},
+	{ .id = "PNPC105", .driver_data = 0 },
 	/*  Standard 28800 bps Modem*/
-	{	"PNPC106",		0	},
+	{ .id = "PNPC106", .driver_data = 0 },
 	/*  Standard Modem */
-	{	"PNPC107",		0	},
+	{ .id = "PNPC107", .driver_data = 0 },
 	/* Standard 9600 bps Modem */
-	{	"PNPC108",		0	},
+	{ .id = "PNPC108", .driver_data = 0 },
 	/* Standard 14400 bps Modem */
-	{	"PNPC109",		0	},
+	{ .id = "PNPC109", .driver_data = 0 },
 	/* Standard 28800 bps Modem */
-	{	"PNPC10A",		0	},
+	{ .id = "PNPC10A", .driver_data = 0 },
 	/* Standard Modem */
-	{	"PNPC10B",		0	},
+	{ .id = "PNPC10B", .driver_data = 0 },
 	/* Standard 9600 bps Modem */
-	{	"PNPC10C",		0	},
+	{ .id = "PNPC10C", .driver_data = 0 },
 	/* Standard 14400 bps Modem */
-	{	"PNPC10D",		0	},
+	{ .id = "PNPC10D", .driver_data = 0 },
 	/* Standard 28800 bps Modem */
-	{	"PNPC10E",		0	},
+	{ .id = "PNPC10E", .driver_data = 0 },
 	/* Standard Modem */
-	{	"PNPC10F",		0	},
+	{ .id = "PNPC10F", .driver_data = 0 },
 	/* Standard PCMCIA Card Modem */
-	{	"PNP2000",		0	},
+	{ .id = "PNP2000", .driver_data = 0 },
 	/* Rockwell */
 	/* Modular Technology */
 	/* Rockwell 33.6 DPF Internal PnP */
 	/* Modular Technology 33.6 Internal PnP */
-	{	"ROK0030",		0	},
+	{ .id = "ROK0030", .driver_data = 0 },
 	/* Kortex International */
 	/* KORTEX 14400 Externe PnP */
-	{	"ROK0100",		0	},
+	{ .id = "ROK0100", .driver_data = 0 },
 	/* Rockwell 28.8 */
-	{	"ROK4120",		0	},
+	{ .id = "ROK4120", .driver_data = 0 },
 	/* Viking Components, Inc */
 	/* Viking 28.8 INTERNAL Fax+Data+Voice PnP */
-	{	"ROK4920",		0	},
+	{ .id = "ROK4920", .driver_data = 0 },
 	/* Rockwell */
 	/* British Telecom */
 	/* Modular Technology */
 	/* Rockwell 33.6 DPF External PnP */
 	/* BT Prologue 33.6 External PnP */
 	/* Modular Technology 33.6 External PnP */
-	{	"RSS00A0",		0	},
+	{ .id = "RSS00A0", .driver_data = 0 },
 	/* Viking 56K FAX INT */
-	{	"RSS0262",		0	},
+	{ .id = "RSS0262", .driver_data = 0 },
 	/* K56 par,VV,Voice,Speakphone,AudioSpan,PnP */
-	{       "RSS0250",              0       },
+	{ .id = "RSS0250", .driver_data = 0 },
 	/* SupraExpress 28.8 Data/Fax PnP modem */
-	{	"SUP1310",		0	},
+	{ .id = "SUP1310", .driver_data = 0 },
 	/* SupraExpress 336i PnP Voice Modem */
-	{	"SUP1381",		0	},
+	{ .id = "SUP1381", .driver_data = 0 },
 	/* SupraExpress 33.6 Data/Fax PnP modem */
-	{	"SUP1421",		0	},
+	{ .id = "SUP1421", .driver_data = 0 },
 	/* SupraExpress 33.6 Data/Fax PnP modem */
-	{	"SUP1590",		0	},
+	{ .id = "SUP1590", .driver_data = 0 },
 	/* SupraExpress 336i Sp ASVD */
-	{	"SUP1620",		0	},
+	{ .id = "SUP1620", .driver_data = 0 },
 	/* SupraExpress 33.6 Data/Fax PnP modem */
-	{	"SUP1760",		0	},
+	{ .id = "SUP1760", .driver_data = 0 },
 	/* SupraExpress 56i Sp Intl */
-	{	"SUP2171",		0	},
+	{ .id = "SUP2171", .driver_data = 0 },
 	/* Phoebe Micro */
 	/* Phoebe Micro 33.6 Data Fax 1433VQH Plug & Play */
-	{	"TEX0011",		0	},
+	{ .id = "TEX0011", .driver_data = 0 },
 	/* Archtek America Corp. */
 	/* Archtek SmartLink Modem 3334BT Plug & Play */
-	{	"UAC000F",		0	},
+	{ .id = "UAC000F", .driver_data = 0 },
 	/* 3Com Corp. */
 	/* Gateway Telepath IIvi 33.6 */
-	{	"USR0000",		0	},
+	{ .id = "USR0000", .driver_data = 0 },
 	/* U.S. Robotics Sporster 33.6K Fax INT PnP */
-	{	"USR0002",		0	},
+	{ .id = "USR0002", .driver_data = 0 },
 	/*  Sportster Vi 14.4 PnP FAX Voicemail */
-	{	"USR0004",		0	},
+	{ .id = "USR0004", .driver_data = 0 },
 	/* U.S. Robotics 33.6K Voice INT PnP */
-	{	"USR0006",		0	},
+	{ .id = "USR0006", .driver_data = 0 },
 	/* U.S. Robotics 33.6K Voice EXT PnP */
-	{	"USR0007",		0	},
+	{ .id = "USR0007", .driver_data = 0 },
 	/* U.S. Robotics Courier V.Everything INT PnP */
-	{	"USR0009",		0	},
+	{ .id = "USR0009", .driver_data = 0 },
 	/* U.S. Robotics 33.6K Voice INT PnP */
-	{	"USR2002",		0	},
+	{ .id = "USR2002", .driver_data = 0 },
 	/* U.S. Robotics 56K Voice INT PnP */
-	{	"USR2070",		0	},
+	{ .id = "USR2070", .driver_data = 0 },
 	/* U.S. Robotics 56K Voice EXT PnP */
-	{	"USR2080",		0	},
+	{ .id = "USR2080", .driver_data = 0 },
 	/* U.S. Robotics 56K FAX INT */
-	{	"USR3031",		0	},
+	{ .id = "USR3031", .driver_data = 0 },
 	/* U.S. Robotics 56K FAX INT */
-	{	"USR3050",		0	},
+	{ .id = "USR3050", .driver_data = 0 },
 	/* U.S. Robotics 56K Voice INT PnP */
-	{	"USR3070",		0	},
+	{ .id = "USR3070", .driver_data = 0 },
 	/* U.S. Robotics 56K Voice EXT PnP */
-	{	"USR3080",		0	},
+	{ .id = "USR3080", .driver_data = 0 },
 	/* U.S. Robotics 56K Voice INT PnP */
-	{	"USR3090",		0	},
+	{ .id = "USR3090", .driver_data = 0 },
 	/* U.S. Robotics 56K Message  */
-	{	"USR9100",		0	},
+	{ .id = "USR9100", .driver_data = 0 },
 	/* U.S. Robotics 56K FAX EXT PnP*/
-	{	"USR9160",		0	},
+	{ .id = "USR9160", .driver_data = 0 },
 	/* U.S. Robotics 56K FAX INT PnP*/
-	{	"USR9170",		0	},
+	{ .id = "USR9170", .driver_data = 0 },
 	/* U.S. Robotics 56K Voice EXT PnP*/
-	{	"USR9180",		0	},
+	{ .id = "USR9180", .driver_data = 0 },
 	/* U.S. Robotics 56K Voice INT PnP*/
-	{	"USR9190",		0	},
+	{ .id = "USR9190", .driver_data = 0 },
 	/* Wacom tablets */
-	{	"WACFXXX",		0	},
+	{ .id = "WACFXXX", .driver_data = 0 },
 	/* Compaq touchscreen */
-	{       "FPI2002",              0 },
+	{ .id = "FPI2002", .driver_data = 0 },
 	/* Fujitsu Stylistic touchscreens */
-	{       "FUJ02B2",              0 },
-	{       "FUJ02B3",              0 },
+	{ .id = "FUJ02B2", .driver_data = 0 },
+	{ .id = "FUJ02B3", .driver_data = 0 },
 	/* Fujitsu Stylistic LT touchscreens */
-	{       "FUJ02B4",              0 },
+	{ .id = "FUJ02B4", .driver_data = 0 },
 	/* Passive Fujitsu Stylistic touchscreens */
-	{       "FUJ02B6",              0 },
-	{       "FUJ02B7",              0 },
-	{       "FUJ02B8",              0 },
-	{       "FUJ02B9",              0 },
-	{       "FUJ02BC",              0 },
+	{ .id = "FUJ02B6", .driver_data = 0 },
+	{ .id = "FUJ02B7", .driver_data = 0 },
+	{ .id = "FUJ02B8", .driver_data = 0 },
+	{ .id = "FUJ02B9", .driver_data = 0 },
+	{ .id = "FUJ02BC", .driver_data = 0 },
 	/* Fujitsu Wacom Tablet PC device */
-	{	"FUJ02E5",		0	},
+	{ .id = "FUJ02E5", .driver_data = 0 },
 	/* Fujitsu P-series tablet PC device */
-	{	"FUJ02E6",		0	},
+	{ .id = "FUJ02E6", .driver_data = 0 },
 	/* Fujitsu Wacom 2FGT Tablet PC device */
-	{	"FUJ02E7",		0	},
+	{ .id = "FUJ02E7", .driver_data = 0 },
 	/* Fujitsu Wacom 1FGT Tablet PC device */
-	{	"FUJ02E9",		0	},
+	{ .id = "FUJ02E9", .driver_data = 0 },
 	/*
 	 * LG C1 EXPRESS DUAL (C1-PB11A3) touch screen (actually a FUJ02E6
 	 * in disguise).
 	 */
-	{	"LTS0001",		0       },
+	{ .id = "LTS0001", .driver_data = 0 },
 	/* Rockwell's (PORALiNK) 33600 INT PNP */
-	{	"WCI0003",		0	},
+	{ .id = "WCI0003", .driver_data = 0 },
 	/* Unknown PnP modems */
-	{	"PNPCXXX",		UNKNOWN_DEV	},
+	{ .id = "PNPCXXX", .driver_data = UNKNOWN_DEV },
 	/* More unknown PnP modems */
-	{	"PNPDXXX",		UNKNOWN_DEV	},
+	{ .id = "PNPDXXX", .driver_data = UNKNOWN_DEV },
 	/*
 	 * Winbond CIR port, should not be probed. We should keep track of
 	 * it to prevent the legacy serial driver from probing it.
 	 */
-	{	"WEC1022",		CIR_PORT	},
+	{ .id = "WEC1022", .driver_data = CIR_PORT },
 	/*
 	 * SMSC IrCC SIR/FIR port, should not be probed by serial driver as
 	 * well so its own driver can bind to it.
 	 */
-	{	"SMCF010",		CIR_PORT	},
-	{	"",			0	}
+	{ .id = "SMCF010", .driver_data = CIR_PORT },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pnp, pnp_dev_table);
@@ -521,6 +521,7 @@ static struct pnp_driver serial_pnp_driver = {
 	.remove		= serial_pnp_remove,
 	.driver         = {
 		.pm     = pm_sleep_ptr(&serial_pnp_pm_ops),
+		.probe_type = PROBE_FORCE_SYNCHRONOUS,
 	},
 	.id_table	= pnp_dev_table,
 };
diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c
index e94a0802cbddc..8c241ec7f4f29 100644
--- a/drivers/tty/serial/8250/8250_port.c
+++ b/drivers/tty/serial/8250/8250_port.c
@@ -474,16 +474,10 @@ static void set_io_from_upio(struct uart_port *p)
 static void
 serial_port_out_sync(struct uart_port *p, int offset, int value)
 {
-	switch (p->iotype) {
-	case UPIO_MEM:
-	case UPIO_MEM16:
-	case UPIO_MEM32:
-	case UPIO_MEM32BE:
-	case UPIO_AU:
+	if (uart_iotype_mmio(p->iotype)) {
 		p->serial_out(p, offset, value);
 		p->serial_in(p, UART_LCR);	/* safe, no side-effects */
-		break;
-	default:
+	} else {
 		p->serial_out(p, offset, value);
 	}
 }
@@ -2891,13 +2885,7 @@ static int serial8250_request_std_resource(struct uart_8250_port *up)
 	unsigned int size = serial8250_port_size(up);
 	struct uart_port *port = &up->port;
 
-	switch (port->iotype) {
-	case UPIO_AU:
-	case UPIO_TSI:
-	case UPIO_MEM32:
-	case UPIO_MEM32BE:
-	case UPIO_MEM16:
-	case UPIO_MEM:
+	if (uart_iotype_mmio(port->iotype)) {
 		if (!port->mapbase)
 			return -EINVAL;
 
@@ -2911,14 +2899,9 @@ static int serial8250_request_std_resource(struct uart_8250_port *up)
 				return -ENOMEM;
 			}
 		}
-		return 0;
-	case UPIO_HUB6:
-	case UPIO_PORT:
+	} else if (uart_iotype_io(port->iotype)) {
 		if (!request_region(port->iobase, size, "serial"))
 			return -EBUSY;
-		return 0;
-	case UPIO_UNKNOWN:
-		break;
 	}
 
 	return 0;
@@ -2929,15 +2912,9 @@ static void serial8250_release_std_resource(struct uart_8250_port *up)
 	unsigned int size = serial8250_port_size(up);
 	struct uart_port *port = &up->port;
 
-	switch (port->iotype) {
-	case UPIO_AU:
-	case UPIO_TSI:
-	case UPIO_MEM32:
-	case UPIO_MEM32BE:
-	case UPIO_MEM16:
-	case UPIO_MEM:
+	if (uart_iotype_mmio(port->iotype)) {
 		if (!port->mapbase)
-			break;
+			return;
 
 		if (port->flags & UPF_IOREMAP) {
 			iounmap(port->membase);
@@ -2945,14 +2922,8 @@ static void serial8250_release_std_resource(struct uart_8250_port *up)
 		}
 
 		release_mem_region(port->mapbase, size);
-		break;
-
-	case UPIO_HUB6:
-	case UPIO_PORT:
+	} else if (uart_iotype_io(port->iotype)) {
 		release_region(port->iobase, size);
-		break;
-	case UPIO_UNKNOWN:
-		break;
 	}
 }
 
diff --git a/drivers/tty/serial/8250/8250_rsa.c b/drivers/tty/serial/8250/8250_rsa.c
index fff9395948e33..da971437e89c1 100644
--- a/drivers/tty/serial/8250/8250_rsa.c
+++ b/drivers/tty/serial/8250/8250_rsa.c
@@ -19,35 +19,33 @@ static const struct uart_ops *core_port_base_ops;
 static int rsa8250_request_resource(struct uart_8250_port *up)
 {
 	struct uart_port *port = &up->port;
-	unsigned long start = UART_RSA_BASE << port->regshift;
-	unsigned int size = 8 << port->regshift;
-
-	switch (port->iotype) {
-	case UPIO_HUB6:
-	case UPIO_PORT:
-		start += port->iobase;
-		if (!request_region(start, size, "serial-rsa"))
-			return -EBUSY;
-		return 0;
-	default:
+	unsigned long start;
+	unsigned int size;
+
+	if (!uart_iotype_io(port->iotype))
 		return -EINVAL;
-	}
+
+	start = UART_RSA_BASE << port->regshift;
+	start += port->iobase;
+	size = 8 << port->regshift;
+
+	if (!request_region(start, size, "serial-rsa"))
+		return -EBUSY;
+	return 0;
 }
 
 static void rsa8250_release_resource(struct uart_8250_port *up)
 {
 	struct uart_port *port = &up->port;
-	unsigned long offset = UART_RSA_BASE << port->regshift;
-	unsigned int size = 8 << port->regshift;
-
-	switch (port->iotype) {
-	case UPIO_HUB6:
-	case UPIO_PORT:
-		release_region(port->iobase + offset, size);
-		break;
-	default:
-		break;
-	}
+	unsigned long offset;
+	unsigned int size;
+
+	if (!uart_iotype_io(port->iotype))
+		return;
+
+	offset = UART_RSA_BASE << port->regshift;
+	size = 8 << port->regshift;
+	release_region(port->iobase + offset, size);
 }
 
 static void univ8250_config_port(struct uart_port *port, int flags)
diff --git a/drivers/tty/serial/cpm_uart.c b/drivers/tty/serial/cpm_uart.c
index b778a20ec9b13..39f54bb7b485b 100644
--- a/drivers/tty/serial/cpm_uart.c
+++ b/drivers/tty/serial/cpm_uart.c
@@ -27,7 +27,6 @@
 #include <linux/memblock.h>
 #include <linux/dma-mapping.h>
 #include <linux/of_address.h>
-#include <linux/of_irq.h>
 #include <linux/of_platform.h>
 #include <linux/gpio/consumer.h>
 #include <linux/clk.h>
@@ -1530,16 +1529,14 @@ static int cpm_uart_probe(struct platform_device *ofdev)
 	/* initialize the device pointer for the port */
 	pinfo->port.dev = &ofdev->dev;
 
-	pinfo->port.irq = irq_of_parse_and_map(ofdev->dev.of_node, 0);
-	if (!pinfo->port.irq)
-		return -EINVAL;
+	pinfo->port.irq = platform_get_irq(ofdev, 0);
+	if (pinfo->port.irq < 0)
+		return pinfo->port.irq;
 
 	ret = cpm_uart_init_port(ofdev->dev.of_node, pinfo);
 	if (!ret)
 		return uart_add_one_port(&cpm_reg, &pinfo->port);
 
-	irq_dispose_mapping(pinfo->port.irq);
-
 	return ret;
 }
 
diff --git a/drivers/tty/serial/earlycon.c b/drivers/tty/serial/earlycon.c
index ab9af37f6cda3..ce740cdc7cebf 100644
--- a/drivers/tty/serial/earlycon.c
+++ b/drivers/tty/serial/earlycon.c
@@ -75,19 +75,12 @@ static void __init earlycon_print_info(struct earlycon_device *device)
 {
 	struct console *earlycon = device->con;
 	struct uart_port *port = &device->port;
+	char ioinfos[64];
 
-	if (port->iotype == UPIO_MEM || port->iotype == UPIO_MEM16 ||
-	    port->iotype == UPIO_MEM32 || port->iotype == UPIO_MEM32BE)
-		pr_info("%s%d at MMIO%s %pa (options '%s')\n",
-			earlycon->name, earlycon->index,
-			(port->iotype == UPIO_MEM) ? "" :
-			(port->iotype == UPIO_MEM16) ? "16" :
-			(port->iotype == UPIO_MEM32) ? "32" : "32be",
-			&port->mapbase, device->options);
-	else
-		pr_info("%s%d at I/O port 0x%lx (options '%s')\n",
-			earlycon->name, earlycon->index,
-			port->iobase, device->options);
+	uart_get_ioinfos(port, ioinfos, sizeof(ioinfos));
+
+	pr_info("%s%d%s (options '%s')\n", earlycon->name, earlycon->index,
+		ioinfos, device->options);
 }
 
 static int __init parse_options(struct earlycon_device *device, char *options)
diff --git a/drivers/tty/serial/kgdboc.c b/drivers/tty/serial/kgdboc.c
index 5a955c80a8532..09648d6438979 100644
--- a/drivers/tty/serial/kgdboc.c
+++ b/drivers/tty/serial/kgdboc.c
@@ -363,7 +363,7 @@ static int param_set_kgdboc_var(const char *kmessage,
 
 	mutex_lock(&config_mutex);
 
-	strcpy(config, kmessage);
+	strscpy(config, kmessage);
 	/* Chop out \n char as a result of echo */
 	if (len && config[len - 1] == '\n')
 		config[len - 1] = '\0';
diff --git a/drivers/tty/serial/ma35d1_serial.c b/drivers/tty/serial/ma35d1_serial.c
index 285b0fe41a86a..920fe7ff5083b 100644
--- a/drivers/tty/serial/ma35d1_serial.c
+++ b/drivers/tty/serial/ma35d1_serial.c
@@ -608,8 +608,14 @@ static int __init ma35d1serial_console_setup(struct console *co, char *options)
 	if (!np || !p)
 		return -ENODEV;
 
-	if (of_property_read_u32_array(np, "reg", val32, ARRAY_SIZE(val32)) != 0)
+	if (of_property_read_u32_array(np, "reg", val32, ARRAY_SIZE(val32)) != 0) {
+		of_node_put(np);
+		ma35d1serial_uart_nodes[co->index] = NULL;
 		return -EINVAL;
+	}
+
+	of_node_put(np);
+	ma35d1serial_uart_nodes[co->index] = NULL;
 
 	p->port.iobase = val32[1];
 	p->port.membase = ioremap(p->port.iobase, MA35_UART_REG_SIZE);
@@ -648,8 +654,10 @@ static void ma35d1serial_console_init_port(void)
 			of_node_get(np);
 			ma35d1serial_uart_nodes[i] = np;
 			i++;
-			if (i == MA35_UART_NR)
+			if (i == MA35_UART_NR) {
+				of_node_put(np);
 				break;
+			}
 		}
 	}
 }
diff --git a/drivers/tty/serial/max3100.c b/drivers/tty/serial/max3100.c
index 44b745fa26c6b..a5a5cd68f6962 100644
--- a/drivers/tty/serial/max3100.c
+++ b/drivers/tty/serial/max3100.c
@@ -724,6 +724,7 @@ static int max3100_probe(struct spi_device *spi)
 	max3100s[i]->port.ops = &max3100_ops;
 	max3100s[i]->port.flags = UPF_SKIP_TEST | UPF_BOOT_AUTOCONF;
 	max3100s[i]->port.line = i;
+	max3100s[i]->port.iotype = UPIO_BUS;
 	max3100s[i]->port.type = PORT_MAX3100;
 	max3100s[i]->port.dev = &spi->dev;
 
diff --git a/drivers/tty/serial/max310x.c b/drivers/tty/serial/max310x.c
index 5fe12577ce8b8..41a7bff941c17 100644
--- a/drivers/tty/serial/max310x.c
+++ b/drivers/tty/serial/max310x.c
@@ -1400,30 +1400,18 @@ static int max310x_probe(struct device *dev, const struct max310x_devtype *devty
 
 	dev_dbg(dev, "Reference clock set to %i Hz\n", uartclk);
 
+	/*
+	 * Set up each port's state before registering the gpiochip,
+	 * since the gpiochip callbacks will read s->p[i].regmap as
+	 * soon as gpiolib exposes the controller.
+	 */
 	for (i = 0; i < devtype->nr; i++) {
-		unsigned int line;
-
-		line = find_first_zero_bit(max310x_lines, MAX310X_UART_NRMAX);
-		if (line == MAX310X_UART_NRMAX) {
-			ret = -ERANGE;
-			goto out_uart;
-		}
-
-		/* Initialize port data */
-		s->p[i].port.line	= line;
 		s->p[i].port.dev	= dev;
 		s->p[i].port.irq	= irq;
 		s->p[i].port.type	= PORT_MAX310X;
 		s->p[i].port.fifosize	= MAX310X_FIFO_SIZE;
 		s->p[i].port.flags	= UPF_FIXED_TYPE | UPF_LOW_LATENCY;
-		s->p[i].port.iotype	= UPIO_PORT;
-		s->p[i].port.iobase	= i;
-		/*
-		 * Use all ones as membase to make sure uart_configure_port() in
-		 * serial_core.c does not abort for SPI/I2C devices where the
-		 * membase address is not applicable.
-		 */
-		s->p[i].port.membase	= (void __iomem *)~0;
+		s->p[i].port.iotype	= UPIO_BUS;
 		s->p[i].port.uartclk	= uartclk;
 		s->p[i].port.rs485_config = max310x_rs485_config;
 		s->p[i].port.rs485_supported = max310x_rs485_supported;
@@ -1440,20 +1428,16 @@ static int max310x_probe(struct device *dev, const struct max310x_devtype *devty
 		INIT_WORK(&s->p[i].md_work, max310x_md_proc);
 		/* Initialize queue for changing RS485 mode */
 		INIT_WORK(&s->p[i].rs_work, max310x_rs_proc);
-
-		/* Register port */
-		ret = uart_add_one_port(&max310x_uart, &s->p[i].port);
-		if (ret)
-			goto out_uart;
-
-		set_bit(line, max310x_lines);
-
-		/* Go to suspend mode */
-		max310x_power(&s->p[i].port, 0);
 	}
 
 #ifdef CONFIG_GPIOLIB
-	/* Setup GPIO controller */
+	/*
+	 * Register the GPIO controller before adding the UART ports so
+	 * that consumers referencing the chip's own GPIOs from device
+	 * tree (for example rs485-term-gpios = <&max310x ...>) can
+	 * resolve them at uart_add_one_port() time instead of receiving
+	 * -EPROBE_DEFER from their own provider.
+	 */
 	s->gpio.owner		= THIS_MODULE;
 	s->gpio.parent		= dev;
 	s->gpio.label		= devtype->name;
@@ -1471,6 +1455,64 @@ static int max310x_probe(struct device *dev, const struct max310x_devtype *devty
 		goto out_uart;
 #endif
 
+	for (i = 0; i < devtype->nr; i++) {
+		struct fwnode_handle *saved_fwnode = dev_fwnode(dev);
+		struct device_node *port_np = NULL;
+		struct device_node *child;
+		unsigned int line;
+
+		line = find_first_zero_bit(max310x_lines, MAX310X_UART_NRMAX);
+		if (line == MAX310X_UART_NRMAX) {
+			ret = -ERANGE;
+			goto out_uart;
+		}
+		s->p[i].port.line = line;
+
+		/* Locate the matching "serial@i" DT subnode, if any. */
+		for_each_available_child_of_node(dev->of_node, child) {
+			u32 reg;
+
+			if (!of_node_name_eq(child, "serial"))
+				continue;
+			if (of_property_read_u32(child, "reg", &reg))
+				continue;
+			if (reg == i) {
+				port_np = child;
+				break;
+			}
+		}
+
+		/*
+		 * Temporarily retarget dev's fwnode to the per-port subnode
+		 * so uart_get_rs485_mode() picks up the per-port properties.
+		 * For single-port variants, fall back to the chip's own
+		 * fwnode so legacy DTs that declare rs485 properties at the
+		 * top level keep working.
+		 */
+		if (port_np) {
+			device_set_node(dev, of_fwnode_handle(port_np));
+			ret = uart_get_rs485_mode(&s->p[i].port);
+			device_set_node(dev, saved_fwnode);
+			of_node_put(port_np);
+			if (ret)
+				goto out_uart;
+		} else if (devtype->nr == 1) {
+			ret = uart_get_rs485_mode(&s->p[i].port);
+			if (ret)
+				goto out_uart;
+		}
+
+		/* Register port */
+		ret = uart_add_one_port(&max310x_uart, &s->p[i].port);
+		if (ret)
+			goto out_uart;
+
+		set_bit(line, max310x_lines);
+
+		/* Go to suspend mode */
+		max310x_power(&s->p[i].port, 0);
+	}
+
 	/* Setup interrupt */
 	ret = devm_request_threaded_irq(dev, irq, NULL, max310x_ist,
 					IRQF_ONESHOT | IRQF_SHARED, dev_name(dev), s);
diff --git a/drivers/tty/serial/men_z135_uart.c b/drivers/tty/serial/men_z135_uart.c
index 6fad57fee912c..9138fa29d301b 100644
--- a/drivers/tty/serial/men_z135_uart.c
+++ b/drivers/tty/serial/men_z135_uart.c
@@ -16,6 +16,7 @@
 #include <linux/tty_flip.h>
 #include <linux/bitops.h>
 #include <linux/mcb.h>
+#include <linux/slab.h>
 
 #define MEN_Z135_MAX_PORTS		12
 #define MEN_Z135_BASECLK		29491200
@@ -811,7 +812,7 @@ static int men_z135_probe(struct mcb_device *mdev,
 	if (!uart)
 		return -ENOMEM;
 
-	uart->rxbuf = (unsigned char *)__get_free_page(GFP_KERNEL);
+	uart->rxbuf = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!uart->rxbuf)
 		return -ENOMEM;
 
@@ -841,7 +842,7 @@ static int men_z135_probe(struct mcb_device *mdev,
 	return 0;
 
 err:
-	free_page((unsigned long) uart->rxbuf);
+	kfree(uart->rxbuf);
 	dev_err(dev, "Failed to add UART: %d\n", err);
 
 	return err;
@@ -858,7 +859,7 @@ static void men_z135_remove(struct mcb_device *mdev)
 
 	line--;
 	uart_remove_one_port(&men_z135_driver, &uart->port);
-	free_page((unsigned long) uart->rxbuf);
+	kfree(uart->rxbuf);
 }
 
 static const struct mcb_device_id men_z135_ids[] = {
diff --git a/drivers/tty/serial/mpc52xx_uart.c b/drivers/tty/serial/mpc52xx_uart.c
index 37eb701b0b46e..b566206f42a2b 100644
--- a/drivers/tty/serial/mpc52xx_uart.c
+++ b/drivers/tty/serial/mpc52xx_uart.c
@@ -645,6 +645,8 @@ static irqreturn_t mpc512x_psc_handle_irq(struct uart_port *port)
 
 	/* Check if it is an interrupt for this port */
 	psc_num = (port->mapbase & 0xf00) >> 8;
+	if (psc_num >= ARRAY_SIZE(psc_mclk_clk))
+		return IRQ_NONE;
 	if (test_bit(psc_num, &fifoc_int) ||
 	    test_bit(psc_num + 16, &fifoc_int))
 		return mpc5xxx_uart_process_int(port);
@@ -663,6 +665,8 @@ static int mpc512x_psc_alloc_clock(struct uart_port *port)
 	int err;
 
 	psc_num = (port->mapbase & 0xf00) >> 8;
+	if (psc_num >= ARRAY_SIZE(psc_mclk_clk))
+		return -EINVAL;
 
 	clk = devm_clk_get(port->dev, "mclk");
 	if (IS_ERR(clk)) {
@@ -711,6 +715,8 @@ static void mpc512x_psc_relse_clock(struct uart_port *port)
 	struct clk *clk;
 
 	psc_num = (port->mapbase & 0xf00) >> 8;
+	if (psc_num >= ARRAY_SIZE(psc_mclk_clk))
+		return;
 	clk = psc_mclk_clk[psc_num];
 	if (clk) {
 		clk_disable_unprepare(clk);
@@ -733,6 +739,8 @@ static int mpc512x_psc_endis_clock(struct uart_port *port, int enable)
 		return 0;
 
 	psc_num = (port->mapbase & 0xf00) >> 8;
+	if (psc_num >= ARRAY_SIZE(psc_mclk_clk))
+		return -ENODEV;
 	psc_clk = psc_mclk_clk[psc_num];
 	if (!psc_clk) {
 		dev_err(port->dev, "Failed to get PSC clock entry!\n");
diff --git a/drivers/tty/serial/pch_uart.c b/drivers/tty/serial/pch_uart.c
index 80e31c4d9536d..db5c62b0322f6 100644
--- a/drivers/tty/serial/pch_uart.c
+++ b/drivers/tty/serial/pch_uart.c
@@ -678,6 +678,11 @@ static void pch_request_dma(struct uart_port *port)
 	/* Get DMA's dev information */
 	dma_dev = pci_get_slot(priv->pdev->bus,
 			PCI_DEVFN(PCI_SLOT(priv->pdev->devfn), 0));
+	if (!dma_dev) {
+		dev_err(priv->port.dev, "%s: failed to get DMA device\n",
+			__func__);
+		return;
+	}
 
 	/* Set Tx DMA */
 	param = &priv->param_tx;
@@ -1662,7 +1667,7 @@ static struct eg20t_port *pch_uart_init_port(struct pci_dev *pdev,
 	if (priv == NULL)
 		goto init_port_alloc_err;
 
-	rxbuf = (unsigned char *)__get_free_page(GFP_KERNEL);
+	rxbuf = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!rxbuf)
 		goto init_port_free_txbuf;
 
@@ -1735,7 +1740,7 @@ static struct eg20t_port *pch_uart_init_port(struct pci_dev *pdev,
 #ifdef CONFIG_SERIAL_PCH_UART_CONSOLE
 	pch_uart_ports[board->line_no] = NULL;
 #endif
-	free_page((unsigned long)rxbuf);
+	kfree(rxbuf);
 init_port_free_txbuf:
 	kfree(priv);
 init_port_alloc_err:
@@ -1750,7 +1755,7 @@ static void pch_uart_exit_port(struct eg20t_port *priv)
 	snprintf(name, sizeof(name), "uart%d_regs", priv->port.line);
 	debugfs_lookup_and_remove(name, NULL);
 	uart_remove_one_port(&pch_uart_driver, &priv->port);
-	free_page((unsigned long)priv->rxbuf.buf);
+	kfree(priv->rxbuf.buf);
 }
 
 static void pch_uart_pci_remove(struct pci_dev *pdev)
diff --git a/drivers/tty/serial/qcom_geni_serial.c b/drivers/tty/serial/qcom_geni_serial.c
index 7ead87b4eb65b..e037af56a1596 100644
--- a/drivers/tty/serial/qcom_geni_serial.c
+++ b/drivers/tty/serial/qcom_geni_serial.c
@@ -9,6 +9,7 @@
 
 #include <linux/clk.h>
 #include <linux/console.h>
+#include <linux/dma-mapping.h>
 #include <linux/io.h>
 #include <linux/iopoll.h>
 #include <linux/irq.h>
@@ -143,6 +144,7 @@ struct qcom_geni_serial_port {
 
 	unsigned int tx_remaining;
 	unsigned int tx_queued;
+	bool tx_dma_stale;
 	int wakeup_irq;
 	bool rx_tx_swap;
 	bool cts_rts_swap;
@@ -697,6 +699,7 @@ static void qcom_geni_serial_start_tx_dma(struct uart_port *uport)
 	}
 
 	port->tx_remaining = xmit_size;
+	port->tx_dma_stale = false;
 }
 
 static void qcom_geni_serial_start_tx_fifo(struct uart_port *uport)
@@ -863,37 +866,31 @@ static void qcom_geni_serial_stop_rx_dma(struct uart_port *uport)
 				uport->membase + SE_DMA_RX_IRQ_CLR);
 	}
 
-	if (port->rx_dma_addr) {
-		geni_se_rx_dma_unprep(&port->se, port->rx_dma_addr,
-				      DMA_RX_BUF_SIZE);
-		port->rx_dma_addr = 0;
-	}
 }
 
 static void qcom_geni_serial_start_rx_dma(struct uart_port *uport)
 {
 	struct qcom_geni_serial_port *port = to_dev_port(uport);
-	int ret;
 
 	if (qcom_geni_serial_secondary_active(uport))
 		qcom_geni_serial_stop_rx_dma(uport);
 
 	geni_se_setup_s_cmd(&port->se, UART_START_READ, UART_PARAM_RFR_OPEN);
 
-	ret = geni_se_rx_dma_prep(&port->se, port->rx_buf,
-				  DMA_RX_BUF_SIZE,
-				  &port->rx_dma_addr);
-	if (ret) {
-		dev_err(uport->dev, "unable to start RX SE DMA: %d\n", ret);
-		qcom_geni_serial_stop_rx_dma(uport);
+	if (!port->rx_dma_addr) {
+		dev_err(uport->dev, "RX DMA buffer not mapped\n");
+		return;
 	}
+
+	dma_sync_single_for_device(uport->dev->parent, port->rx_dma_addr,
+				   DMA_RX_BUF_SIZE, DMA_FROM_DEVICE);
+	geni_se_rx_init_dma(&port->se, port->rx_dma_addr, DMA_RX_BUF_SIZE);
 }
 
 static void qcom_geni_serial_handle_rx_dma(struct uart_port *uport, bool drop)
 {
 	struct qcom_geni_serial_port *port = to_dev_port(uport);
 	u32 rx_in;
-	int ret;
 
 	if (!qcom_geni_serial_secondary_active(uport))
 		return;
@@ -901,8 +898,8 @@ static void qcom_geni_serial_handle_rx_dma(struct uart_port *uport, bool drop)
 	if (!port->rx_dma_addr)
 		return;
 
-	geni_se_rx_dma_unprep(&port->se, port->rx_dma_addr, DMA_RX_BUF_SIZE);
-	port->rx_dma_addr = 0;
+	dma_sync_single_for_cpu(uport->dev->parent, port->rx_dma_addr,
+				DMA_RX_BUF_SIZE, DMA_FROM_DEVICE);
 
 	rx_in = readl(uport->membase + SE_DMA_RX_LEN_IN);
 	if (!rx_in)
@@ -910,13 +907,9 @@ static void qcom_geni_serial_handle_rx_dma(struct uart_port *uport, bool drop)
 	else if (!drop)
 		handle_rx_uart(uport, rx_in);
 
-	ret = geni_se_rx_dma_prep(&port->se, port->rx_buf,
-				  DMA_RX_BUF_SIZE,
-				  &port->rx_dma_addr);
-	if (ret) {
-		dev_err(uport->dev, "unable to start RX SE DMA: %d\n", ret);
-		qcom_geni_serial_stop_rx_dma(uport);
-	}
+	dma_sync_single_for_device(uport->dev->parent, port->rx_dma_addr,
+				   DMA_RX_BUF_SIZE, DMA_FROM_DEVICE);
+	geni_se_rx_init_dma(&port->se, port->rx_dma_addr, DMA_RX_BUF_SIZE);
 }
 
 static void qcom_geni_serial_start_rx(struct uart_port *uport)
@@ -1029,6 +1022,7 @@ static void qcom_geni_serial_handle_tx_dma(struct uart_port *uport)
 	struct qcom_geni_serial_port *port = to_dev_port(uport);
 	struct tty_port *tport = &uport->state->port;
 	unsigned int fifo_len = kfifo_len(&tport->xmit_fifo);
+	bool tx_dma_stale = port->tx_dma_stale;
 
 	/*
 	 * Only advance the kfifo if it still contains the bytes that were
@@ -1039,12 +1033,13 @@ static void qcom_geni_serial_handle_tx_dma(struct uart_port *uport)
 	 * kfifo->in, making kfifo_len() wrap to UART_XMIT_SIZE - tx_remaining
 	 * and triggering a spurious large DMA transfer of stale data.
 	 */
-	if (fifo_len >= port->tx_remaining)
+	if (!tx_dma_stale && fifo_len >= port->tx_remaining)
 		uart_xmit_advance(uport, port->tx_remaining);
 
 	geni_se_tx_dma_unprep(&port->se, port->tx_dma_addr, port->tx_remaining);
 	port->tx_dma_addr = 0;
 	port->tx_remaining = 0;
+	port->tx_dma_stale = false;
 
 	if (!kfifo_is_empty(&tport->xmit_fifo))
 		qcom_geni_serial_start_tx_dma(uport);
@@ -1182,6 +1177,10 @@ static void qcom_geni_serial_shutdown(struct uart_port *uport)
 
 static void qcom_geni_serial_flush_buffer(struct uart_port *uport)
 {
+	struct qcom_geni_serial_port *port = to_dev_port(uport);
+
+	if (port->tx_dma_addr)
+		port->tx_dma_stale = true;
 	qcom_geni_serial_cancel_tx_cmd(uport);
 }
 
@@ -1245,6 +1244,7 @@ static int qcom_geni_serial_startup(struct uart_port *uport)
 {
 	int ret;
 	struct qcom_geni_serial_port *port = to_dev_port(uport);
+	struct tty_port *tport = &uport->state->port;
 
 	if (!port->setup) {
 		ret = qcom_geni_serial_port_setup(uport);
@@ -1252,6 +1252,13 @@ static int qcom_geni_serial_startup(struct uart_port *uport)
 			return ret;
 	}
 
+	/*
+	 * Skip the close-time transmit drain for console ports so that
+	 * shutdown can proceed without waiting for pending TX completion.
+	 */
+	if (uart_console(uport))
+		tport->closing_wait = ASYNC_CLOSING_WAIT_NONE;
+
 	uart_port_lock_irq(uport);
 	qcom_geni_serial_start_rx(uport);
 	uart_port_unlock_irq(uport);
@@ -1864,6 +1871,14 @@ static int qcom_geni_serial_probe(struct platform_device *pdev)
 			ret = -ENOMEM;
 			goto error;
 		}
+
+		port->rx_dma_addr = dma_map_single(pdev->dev.parent, port->rx_buf,
+						   DMA_RX_BUF_SIZE, DMA_FROM_DEVICE);
+		if (dma_mapping_error(pdev->dev.parent, port->rx_dma_addr)) {
+			ret = -EIO;
+			dev_err(&pdev->dev, "Failed to map RX DMA buffer: %d\n", ret);
+			goto error;
+		}
 	}
 
 	port->name = devm_kasprintf(uport->dev, GFP_KERNEL,
@@ -1928,6 +1943,11 @@ static int qcom_geni_serial_probe(struct platform_device *pdev)
 	return 0;
 
 error:
+	if (port->rx_dma_addr) {
+		dma_unmap_single(pdev->dev.parent, port->rx_dma_addr,
+				 DMA_RX_BUF_SIZE, DMA_FROM_DEVICE);
+		port->rx_dma_addr = 0;
+	}
 	dev_pm_domain_detach_list(port->pd_list);
 	return ret;
 }
@@ -1942,6 +1962,13 @@ static void qcom_geni_serial_remove(struct platform_device *pdev)
 	device_init_wakeup(&pdev->dev, false);
 	ida_free(&port_ida, uport->line);
 	uart_remove_one_port(drv, &port->uport);
+
+	if (port->rx_dma_addr) {
+		dma_unmap_single(pdev->dev.parent, port->rx_dma_addr,
+				 DMA_RX_BUF_SIZE, DMA_FROM_DEVICE);
+		port->rx_dma_addr = 0;
+	}
+
 	dev_pm_domain_detach_list(port->pd_list);
 }
 
@@ -1974,6 +2001,7 @@ static int qcom_geni_serial_suspend(struct device *dev)
 	struct qcom_geni_serial_port *port = dev_get_drvdata(dev);
 	struct uart_port *uport = &port->uport;
 	struct qcom_geni_private_data *private_data = uport->private_data;
+	int ret;
 
 	/*
 	 * This is done so we can hit the lowest possible state in suspend
@@ -1983,7 +2011,19 @@ static int qcom_geni_serial_suspend(struct device *dev)
 		geni_icc_set_tag(&port->se, QCOM_ICC_TAG_ACTIVE_ONLY);
 		geni_icc_set_bw(&port->se);
 	}
-	return uart_suspend_port(private_data->drv, uport);
+
+	ret = uart_suspend_port(private_data->drv, uport);
+	if (ret)
+		return ret;
+
+	/*
+	 * When no_console_suspend is set the console must remain active
+	 * across system sleep, so skip the force suspend path.
+	 */
+	if (!console_suspend_enabled && uart_console(uport))
+		return 0;
+
+	return pm_runtime_force_suspend(dev);
 }
 
 static int qcom_geni_serial_resume(struct device *dev)
@@ -1993,6 +2033,10 @@ static int qcom_geni_serial_resume(struct device *dev)
 	struct uart_port *uport = &port->uport;
 	struct qcom_geni_private_data *private_data = uport->private_data;
 
+	ret = pm_runtime_force_resume(dev);
+	if (ret)
+		return ret;
+
 	ret = uart_resume_port(private_data->drv, uport);
 	if (uart_console(uport)) {
 		geni_icc_set_tag(&port->se, QCOM_ICC_TAG_ALWAYS);
diff --git a/drivers/tty/serial/rp2.c b/drivers/tty/serial/rp2.c
index 6d99a02dd439d..51e81ec0ffdba 100644
--- a/drivers/tty/serial/rp2.c
+++ b/drivers/tty/serial/rp2.c
@@ -197,7 +197,7 @@ struct rp2_card {
 };
 
 #define RP_ID(prod) PCI_VDEVICE(RP, (prod))
-#define RP_CAP(ports, smpte) (((ports) << 8) | ((smpte) << 0))
+#define RP_CAP(ports, smpte) .driver_data = (((ports) << 8) | ((smpte) << 0))
 
 static inline void rp2_decode_cap(const struct pci_device_id *id,
 				  int *ports, int *smpte)
diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c
index daebd92f32c77..943be318de410 100644
--- a/drivers/tty/serial/sc16is7xx.c
+++ b/drivers/tty/serial/sc16is7xx.c
@@ -1273,6 +1273,17 @@ static int sc16is7xx_gpio_set(struct gpio_chip *chip, unsigned int offset,
 	return 0;
 }
 
+static int sc16is7xx_gpio_get_direction(struct gpio_chip *chip, unsigned int offset)
+{
+	struct sc16is7xx_port *s = gpiochip_get_data(chip);
+	struct uart_port *port = &s->p[0].port;
+	unsigned int val;
+
+	val = sc16is7xx_port_read(port, SC16IS7XX_IODIR_REG);
+
+	return val & BIT(offset) ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN;
+}
+
 static int sc16is7xx_gpio_direction_input(struct gpio_chip *chip,
 					  unsigned offset)
 {
@@ -1350,6 +1361,7 @@ static int sc16is7xx_setup_gpio_chip(struct sc16is7xx_port *s)
 	s->gpio.parent		 = dev;
 	s->gpio.label		 = dev_name(dev);
 	s->gpio.init_valid_mask	 = sc16is7xx_gpio_init_valid_mask;
+	s->gpio.get_direction	 = sc16is7xx_gpio_get_direction;
 	s->gpio.direction_input	 = sc16is7xx_gpio_direction_input;
 	s->gpio.get		 = sc16is7xx_gpio_get;
 	s->gpio.direction_output = sc16is7xx_gpio_direction_output;
@@ -1471,14 +1483,7 @@ static int sc16is7xx_setup_channel(struct sc16is7xx_one *one, int i,
 	port->type	= PORT_SC16IS7XX;
 	port->fifosize	= SC16IS7XX_FIFO_SIZE;
 	port->flags	= UPF_FIXED_TYPE | UPF_LOW_LATENCY;
-	port->iobase	= i;
-	/*
-	 * Use all ones as membase to make sure uart_configure_port() in
-	 * serial_core.c does not abort for SPI/I2C devices where the
-	 * membase address is not applicable.
-	 */
-	port->membase	= (void __iomem *)~0;
-	port->iotype	= UPIO_PORT;
+	port->iotype	= UPIO_BUS;
 	port->rs485_config = sc16is7xx_config_rs485;
 	port->rs485_supported = sc16is7xx_rs485_supported;
 	port->ops	= &sc16is7xx_ops;
diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c
index a530ad372b434..06d7f44643d61 100644
--- a/drivers/tty/serial/serial_core.c
+++ b/drivers/tty/serial/serial_core.c
@@ -33,6 +33,7 @@
 #include <linux/uaccess.h>
 
 #include "serial_base.h"
+#include "8250/8250.h" /* For hub6_match_port() */
 
 /*
  * This is used to lock changes in serial line configuration.
@@ -1962,6 +1963,34 @@ static const char *uart_type(struct uart_port *port)
 	return str;
 }
 
+bool uart_iotype_mmio(enum uart_iotype iotype)
+{
+	switch (iotype) {
+	case UPIO_MEM:
+	case UPIO_MEM32:
+	case UPIO_AU:
+	case UPIO_TSI:
+	case UPIO_MEM32BE:
+	case UPIO_MEM16:
+		return true;
+	default:
+		return false;
+	}
+}
+EXPORT_SYMBOL_GPL(uart_iotype_mmio);
+
+bool uart_iotype_io(enum uart_iotype iotype)
+{
+	switch (iotype) {
+	case UPIO_PORT:
+	case UPIO_HUB6:
+		return true;
+	default:
+		return false;
+	}
+}
+EXPORT_SYMBOL_GPL(uart_iotype_io);
+
 #ifdef CONFIG_PROC_FS
 
 static void uart_line_info(struct seq_file *m, struct uart_state *state)
@@ -1969,9 +1998,9 @@ static void uart_line_info(struct seq_file *m, struct uart_state *state)
 	struct tty_port *port = &state->port;
 	enum uart_pm_state pm_state;
 	struct uart_port *uport;
+	char ioinfos[64];
 	char stat_buf[32];
 	unsigned int status;
-	int mmio;
 
 	guard(mutex)(&port->mutex);
 
@@ -1979,13 +2008,10 @@ static void uart_line_info(struct seq_file *m, struct uart_state *state)
 	if (!uport)
 		return;
 
-	mmio = uport->iotype >= UPIO_MEM;
-	seq_printf(m, "%u: uart:%s %s%08llX irq:%u",
-			uport->line, uart_type(uport),
-			mmio ? "mmio:0x" : "port:",
-			mmio ? (unsigned long long)uport->mapbase
-			     : (unsigned long long)uport->iobase,
-			uport->irq);
+	seq_printf(m, "%u: uart:%s", uport->line, uart_type(uport));
+	uart_get_ioinfos(uport, ioinfos, sizeof(ioinfos));
+	seq_printf(m, "%s", ioinfos);
+	seq_printf(m, " irq:%u", uport->irq);
 
 	if (uport->type == PORT_UNKNOWN) {
 		seq_putc(m, '\n');
@@ -2459,38 +2485,47 @@ int uart_resume_port(struct uart_driver *drv, struct uart_port *uport)
 }
 EXPORT_SYMBOL(uart_resume_port);
 
-static inline void
-uart_report_port(struct uart_driver *drv, struct uart_port *port)
+static const char *uart_get_mmio_width(struct uart_port *port)
 {
-	char address[64];
-
 	switch (port->iotype) {
-	case UPIO_PORT:
-		snprintf(address, sizeof(address), "I/O 0x%lx", port->iobase);
-		break;
-	case UPIO_HUB6:
-		snprintf(address, sizeof(address),
-			 "I/O 0x%lx offset 0x%x", port->iobase, port->hub6);
-		break;
-	case UPIO_MEM:
 	case UPIO_MEM16:
+		return "16";
 	case UPIO_MEM32:
 	case UPIO_MEM32BE:
+		return "32be";
 	case UPIO_AU:
-	case UPIO_TSI:
-		snprintf(address, sizeof(address),
-			 "MMIO 0x%llx", (unsigned long long)port->mapbase);
-		break;
+	case UPIO_MEM:
 	default:
-		strscpy(address, "*unknown*", sizeof(address));
-		break;
+		return "";
+	}
+}
+
+void uart_get_ioinfos(struct uart_port *port, char *buf, size_t size)
+{
+	buf[0] = '\0';
+
+	if (uart_iotype_mmio(port->iotype)) {
+		scnprintf(buf, size, " MMIO%s:%pa", uart_get_mmio_width(port), &port->mapbase);
+	} else if (uart_iotype_io(port->iotype)) {
+		if (port->iotype == UPIO_PORT)
+			scnprintf(buf, size, " I/O:0x%lx", port->iobase);
+		else if (port->iotype == UPIO_HUB6)
+			scnprintf(buf, size, " I/O:0x%lx, offset 0x%x", port->iobase, port->hub6);
 	}
+}
+EXPORT_SYMBOL(uart_get_ioinfos);
+
+static inline void
+uart_report_port(struct uart_driver *drv, struct uart_port *port)
+{
+	char ioinfos[64];
+
+	uart_get_ioinfos(port, ioinfos, sizeof(ioinfos));
 
-	pr_info("%s%s%s at %s (irq = %u, base_baud = %u) is a %s\n",
-	       port->dev ? dev_name(port->dev) : "",
-	       port->dev ? ": " : "",
-	       port->name,
-	       address, port->irq, port->uartclk / 16, uart_type(port));
+	pr_info("%s%s%s%s (irq = %u, base_baud = %u) is a %s\n",
+		port->dev ? dev_name(port->dev) : "",
+		port->dev ? ": " : "",
+		port->name, ioinfos, port->irq, port->uartclk / 16, uart_type(port));
 
 	/* The magic multiplier feature is a bit obscure, so report it too.  */
 	if (port->flags & UPF_MAGIC_MULTIPLIER)
@@ -2507,11 +2542,10 @@ uart_configure_port(struct uart_driver *drv, struct uart_state *state,
 {
 	unsigned int flags;
 
-	/*
-	 * If there isn't a port here, don't do anything further.
-	 */
-	if (!port->iobase && !port->mapbase && !port->membase)
-		return;
+	/* If there isn't a port here, don't do anything further. */
+	if (uart_iotype_mmio(port->iotype) || uart_iotype_io(port->iotype))
+		if (!port->iobase && !port->mapbase && !port->membase)
+			return;
 
 	/*
 	 * Now do the auto configuration stuff.  Note that config_port
@@ -3208,23 +3242,16 @@ bool uart_match_port(const struct uart_port *port1,
 {
 	if (port1->iotype != port2->iotype)
 		return false;
-
-	switch (port1->iotype) {
-	case UPIO_PORT:
+	else if (port1->iotype == UPIO_PORT)
 		return port1->iobase == port2->iobase;
-	case UPIO_HUB6:
-		return port1->iobase == port2->iobase &&
-		       port1->hub6   == port2->hub6;
-	case UPIO_MEM:
-	case UPIO_MEM16:
-	case UPIO_MEM32:
-	case UPIO_MEM32BE:
-	case UPIO_AU:
-	case UPIO_TSI:
+	else if (port1->iotype == UPIO_HUB6)
+		return hub6_match_port(port1, port2);
+	else if (uart_iotype_mmio(port1->iotype))
 		return port1->mapbase == port2->mapbase;
-	default:
+	else if (port1->iotype == UPIO_BUS)
+		return true;
+	else
 		return false;
-	}
 }
 EXPORT_SYMBOL(uart_match_port);
 
diff --git a/drivers/tty/tty_jobctrl.c b/drivers/tty/tty_jobctrl.c
index ef8741c3e6629..37929fb561748 100644
--- a/drivers/tty/tty_jobctrl.c
+++ b/drivers/tty/tty_jobctrl.c
@@ -506,29 +506,23 @@ static int tiocspgrp(struct tty_struct *tty, struct tty_struct *real_tty, pid_t
 	if (pgrp_nr < 0)
 		return -EINVAL;
 
-	spin_lock_irq(&real_tty->ctrl.lock);
+	guard(spinlock_irq)(&real_tty->ctrl.lock);
 	if (!current->signal->tty ||
 	    (current->signal->tty != real_tty) ||
-	    (real_tty->ctrl.session != task_session(current))) {
-		retval = -ENOTTY;
-		goto out_unlock_ctrl;
-	}
-	rcu_read_lock();
+	    (real_tty->ctrl.session != task_session(current)))
+		return -ENOTTY;
+
+	guard(rcu)();
 	pgrp = find_vpid(pgrp_nr);
-	retval = -ESRCH;
 	if (!pgrp)
-		goto out_unlock;
-	retval = -EPERM;
+		return -ESRCH;
 	if (session_of_pgrp(pgrp) != task_session(current))
-		goto out_unlock;
-	retval = 0;
+		return -EPERM;
+
 	put_pid(real_tty->ctrl.pgrp);
 	real_tty->ctrl.pgrp = get_pid(pgrp);
-out_unlock:
-	rcu_read_unlock();
-out_unlock_ctrl:
-	spin_unlock_irq(&real_tty->ctrl.lock);
-	return retval;
+
+	return 0;
 }
 
 /**
@@ -542,7 +536,6 @@ static int tiocspgrp(struct tty_struct *tty, struct tty_struct *real_tty, pid_t
  */
 static int tiocgsid(struct tty_struct *tty, struct tty_struct *real_tty, pid_t __user *p)
 {
-	unsigned long flags;
 	pid_t sid;
 
 	/*
@@ -552,17 +545,13 @@ static int tiocgsid(struct tty_struct *tty, struct tty_struct *real_tty, pid_t _
 	if (tty == real_tty && current->signal->tty != real_tty)
 		return -ENOTTY;
 
-	spin_lock_irqsave(&real_tty->ctrl.lock, flags);
-	if (!real_tty->ctrl.session)
-		goto err;
-	sid = pid_vnr(real_tty->ctrl.session);
-	spin_unlock_irqrestore(&real_tty->ctrl.lock, flags);
+	scoped_guard(spinlock_irqsave, &real_tty->ctrl.lock) {
+		if (!real_tty->ctrl.session)
+			return -ENOTTY;
+		sid = pid_vnr(real_tty->ctrl.session);
+	}
 
 	return put_user(sid, p);
-
-err:
-	spin_unlock_irqrestore(&real_tty->ctrl.lock, flags);
-	return -ENOTTY;
 }
 
 /*
diff --git a/drivers/tty/vt/vc_screen.c b/drivers/tty/vt/vc_screen.c
index 7d40eacc21b3e..bf1502fd5bd4b 100644
--- a/drivers/tty/vt/vc_screen.c
+++ b/drivers/tty/vt/vc_screen.c
@@ -53,8 +53,6 @@
 #define HEADER_SIZE	4u
 #define CON_BUF_SIZE (IS_ENABLED(CONFIG_BASE_SMALL) ? 256 : PAGE_SIZE)
 
-DEFINE_FREE(free_page_ptr, void *, if (_T) free_page((unsigned long)_T));
-
 /*
  * Our minor space:
  *
@@ -371,7 +369,7 @@ vcs_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
 	loff_t pos;
 	bool viewed, attr, uni_mode;
 
-	char *con_buf __free(free_page_ptr) = (char *)__get_free_page(GFP_KERNEL);
+	char *con_buf __free(kfree) = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!con_buf)
 		return -ENOMEM;
 
@@ -596,7 +594,7 @@ vcs_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos)
 	if (use_unicode(inode))
 		return -EOPNOTSUPP;
 
-	char *con_buf __free(free_page_ptr) = (char *)__get_free_page(GFP_KERNEL);
+	char *con_buf __free(kfree) = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!con_buf)
 		return -ENOMEM;
 
diff --git a/drivers/ufs/core/ufs-sysfs.c b/drivers/ufs/core/ufs-sysfs.c
index d9dc4cc3452ee..63e670d1a9d9e 100644
--- a/drivers/ufs/core/ufs-sysfs.c
+++ b/drivers/ufs/core/ufs-sysfs.c
@@ -54,6 +54,7 @@ static const char *ufs_hs_gear_to_string(enum ufs_hs_gear_tag gear)
 	case UFS_HS_G3:	return "HS_GEAR3";
 	case UFS_HS_G4:	return "HS_GEAR4";
 	case UFS_HS_G5:	return "HS_GEAR5";
+	case UFS_HS_G6:	return "HS_GEAR6";
 	default:	return "UNKNOWN";
 	}
 }
diff --git a/drivers/ufs/core/ufs-txeq.c b/drivers/ufs/core/ufs-txeq.c
index aa64f2bf4f1ef..fa5f539632eb0 100644
--- a/drivers/ufs/core/ufs-txeq.c
+++ b/drivers/ufs/core/ufs-txeq.c
@@ -10,6 +10,7 @@
 #include <linux/delay.h>
 #include <linux/errno.h>
 #include <linux/kernel.h>
+#include <linux/sched/mm.h>
 #include <ufs/ufshcd.h>
 #include <ufs/unipro.h>
 #include "ufshcd-priv.h"
@@ -481,7 +482,8 @@ static void ufshcd_evaluate_tx_eqtr_fom(struct ufs_hba *hba,
  * @h_iter: host TX EQTR iterator data structure
  * @d_iter: device TX EQTR iterator data structure
  *
- * Returns 0 on success, negative error code otherwise
+ * Returns 0 on success, negative error code if get_rx_fom vops fails.
+ * RX_FOM DME get failures are logged and treated as 0 FOM for that lane.
  */
 static int ufshcd_get_rx_fom(struct ufs_hba *hba,
 			     struct ufs_pa_layer_attr *pwr_mode,
@@ -496,8 +498,12 @@ static int ufshcd_get_rx_fom(struct ufs_hba *hba,
 		ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB_SEL(RX_FOM,
 					  UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
 					  &fom);
-		if (ret)
-			return ret;
+		if (ret) {
+			h_iter->fom[lane] = 0;
+			dev_dbg(hba->dev, "Failed to get FOM for Host TX Lane %d: %d\n",
+				 lane, ret);
+			continue;
+		}
 
 		h_iter->fom[lane] = (u8)fom;
 	}
@@ -507,8 +513,12 @@ static int ufshcd_get_rx_fom(struct ufs_hba *hba,
 		ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_FOM,
 				     UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)),
 				     &fom);
-		if (ret)
-			return ret;
+		if (ret) {
+			d_iter->fom[lane] = 0;
+			dev_dbg(hba->dev, "Failed to get FOM for Device TX Lane %d: %d\n",
+				 lane, ret);
+			continue;
+		}
 
 		d_iter->fom[lane] = (u8)fom;
 	}
@@ -1216,14 +1226,26 @@ static int ufshcd_tx_eqtr(struct ufs_hba *hba,
 			  struct ufs_pa_layer_attr *pwr_mode)
 {
 	struct ufs_pa_layer_attr old_pwr_info;
+	unsigned int noio_flag;
+	int notify_ret;
 	int ret;
 
+	/*
+	 * ufshcd_tx_eqtr() is called from a power-mode-change context where
+	 * I/O is suspended. Use memalloc_noio_save() to propagate GFP_NOIO
+	 * to all allocations in the call tree instead of tagging each call
+	 * site individually.
+	 */
+	noio_flag = memalloc_noio_save();
+
 	if (!params->eqtr_record) {
 		params->eqtr_record = devm_kzalloc(hba->dev,
 						   sizeof(*params->eqtr_record),
 						   GFP_KERNEL);
-		if (!params->eqtr_record)
-			return -ENOMEM;
+		if (!params->eqtr_record) {
+			ret = -ENOMEM;
+			goto out_noio_restore;
+		}
 	}
 
 	memcpy(&old_pwr_info, &hba->pwr_info, sizeof(struct ufs_pa_layer_attr));
@@ -1231,23 +1253,31 @@ static int ufshcd_tx_eqtr(struct ufs_hba *hba,
 	ret = ufshcd_tx_eqtr_prepare(hba, pwr_mode);
 	if (ret) {
 		dev_err(hba->dev, "Failed to prepare TX EQTR: %d\n", ret);
-		goto out;
+		goto out_unprepare;
 	}
 
 	ret = ufshcd_vops_tx_eqtr_notify(hba, PRE_CHANGE, pwr_mode);
-	if (ret)
-		goto out;
+	if (ret) {
+		dev_err(hba->dev, "TX EQTR PRE_CHANGE notify failed: %d\n", ret);
+		goto out_unprepare;
+	}
 
 	ret = __ufshcd_tx_eqtr(hba, params, pwr_mode);
-	if (ret)
-		goto out;
 
-	ret = ufshcd_vops_tx_eqtr_notify(hba, POST_CHANGE, pwr_mode);
+	notify_ret = ufshcd_vops_tx_eqtr_notify(hba, POST_CHANGE, pwr_mode);
+	if (notify_ret)
+		dev_err(hba->dev, "TX EQTR POST_CHANGE notify failed: %d\n", notify_ret);
 
-out:
+	if (!ret)
+		ret = notify_ret;
+
+out_unprepare:
 	if (ret)
 		ufshcd_tx_eqtr_unprepare(hba, &old_pwr_info);
 
+out_noio_restore:
+	memalloc_noio_restore(noio_flag);
+
 	return ret;
 }
 
@@ -1301,7 +1331,13 @@ int ufshcd_config_tx_eq_settings(struct ufs_hba *hba,
 	}
 
 	params = &hba->tx_eq_params[gear - 1];
-	if (!params->is_valid || force_tx_eqtr) {
+	/*
+	 * TX EQTR must run for the following cases:
+	 * 1. TX EQ settings are invalid.
+	 * 2. TX EQ settings are from Device Tree.
+	 * 3. TX EQTR procedure is forced.
+	 */
+	if (!params->is_valid || params->from_dt || force_tx_eqtr) {
 		int ret;
 
 		ret = ufshcd_tx_eqtr(hba, params, pwr_mode);
@@ -1314,6 +1350,8 @@ int ufshcd_config_tx_eq_settings(struct ufs_hba *hba,
 		/* Mark TX Equalization settings as valid */
 		params->is_valid = true;
 		params->is_trained = true;
+		/* TX EQTR succeeds, clear from_dt flag */
+		params->from_dt = false;
 		params->is_applied = false;
 	}
 
@@ -1499,6 +1537,11 @@ static void ufshcd_extract_tx_eq_settings_attrs(struct ufs_hba *hba, u8 gear)
 	}
 
 	params->is_valid = true;
+	/*
+	 * Optimal TX EQ settings are retrieved from UFS device attributes,
+	 * clear from_dt flag to avoid TX EQTR procedure.
+	 */
+	params->from_dt = false;
 }
 
 void ufshcd_retrieve_tx_eq_settings(struct ufs_hba *hba)
diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
index d3044a3089b53..79331c2bd38de 100644
--- a/drivers/ufs/core/ufshcd.c
+++ b/drivers/ufs/core/ufshcd.c
@@ -1445,7 +1445,7 @@ static int ufshcd_scale_gear(struct ufs_hba *hba, u32 target_gear, bool scale_up
  *
  * Return: 0 upon success; -EBUSY upon timeout.
  */
-static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba, u64 timeout_us)
+static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
 {
 	int ret = 0;
 	/*
@@ -1453,16 +1453,13 @@ static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba, u64 timeout_us)
 	 * clock scaling is in progress
 	 */
 	mutex_lock(&hba->host->scan_mutex);
-	blk_mq_quiesce_tagset(&hba->host->tag_set);
 	mutex_lock(&hba->wb_mutex);
 	down_write(&hba->clk_scaling_lock);
 
-	if (!hba->clk_scaling.is_allowed ||
-	    ufshcd_wait_for_pending_cmds(hba, timeout_us)) {
+	if (!hba->clk_scaling.is_allowed) {
 		ret = -EBUSY;
 		up_write(&hba->clk_scaling_lock);
 		mutex_unlock(&hba->wb_mutex);
-		blk_mq_unquiesce_tagset(&hba->host->tag_set);
 		mutex_unlock(&hba->host->scan_mutex);
 		goto out;
 	}
@@ -1478,7 +1475,6 @@ static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err)
 {
 	up_write(&hba->clk_scaling_lock);
 	mutex_unlock(&hba->wb_mutex);
-	blk_mq_unquiesce_tagset(&hba->host->tag_set);
 	mutex_unlock(&hba->host->scan_mutex);
 
 	/* Enable Write Booster if current gear requires it else disable it */
@@ -1506,7 +1502,7 @@ static int ufshcd_devfreq_scale(struct ufs_hba *hba, unsigned long freq,
 
 	new_gear = ufshcd_vops_freq_to_gear_speed(hba, freq);
 
-	ret = ufshcd_clock_scaling_prepare(hba, 1 * USEC_PER_SEC);
+	ret = ufshcd_clock_scaling_prepare(hba);
 	if (ret)
 		return ret;
 
@@ -2506,7 +2502,10 @@ static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
 	hba->nutmrs =
 	((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
 
-	hba->nortt = FIELD_GET(MASK_NUMBER_OUTSTANDING_RTT, hba->capabilities) + 1;
+	if (hba->vops && hba->vops->get_hba_nortt)
+		hba->nortt = hba->vops->get_hba_nortt(hba);
+	else
+		hba->nortt = FIELD_GET(MASK_NUMBER_OUTSTANDING_RTT, hba->capabilities) + 1;
 
 	/* Read crypto capabilities */
 	err = ufshcd_hba_init_crypto_capabilities(hba);
@@ -6415,8 +6414,8 @@ static bool ufshcd_wb_curr_buff_threshold_check(struct ufs_hba *hba,
 	}
 
 	if (!cur_buf) {
-		dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n",
-			 cur_buf);
+		dev_warn_once(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n",
+			      cur_buf);
 		return false;
 	}
 	/* Let it continue to flush when available buffer exceeds threshold */
@@ -8611,8 +8610,6 @@ static void ufshcd_set_rtt(struct ufs_hba *hba)
 	struct ufs_dev_info *dev_info = &hba->dev_info;
 	u32 rtt = 0;
 	u32 dev_rtt = 0;
-	int host_rtt_cap = hba->vops && hba->vops->max_num_rtt ?
-			   hba->vops->max_num_rtt : hba->nortt;
 
 	/* RTT override makes sense only for UFS-4.0 and above */
 	if (dev_info->wspecversion < 0x400)
@@ -8628,7 +8625,7 @@ static void ufshcd_set_rtt(struct ufs_hba *hba)
 	if (dev_rtt != DEFAULT_MAX_NUM_RTT)
 		return;
 
-	rtt = min_t(int, dev_info->rtt_cap, host_rtt_cap);
+	rtt = min_t(int, dev_info->rtt_cap, hba->nortt);
 
 	if (rtt == dev_rtt)
 		return;
diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c
index 3991a51263a6d..58701ca95edd7 100644
--- a/drivers/ufs/host/ufs-mediatek.c
+++ b/drivers/ufs/host/ufs-mediatek.c
@@ -2183,6 +2183,16 @@ static int ufs_mtk_clk_scale_notify(struct ufs_hba *hba, bool scale_up,
 	return 0;
 }
 
+static int ufs_mtk_get_hba_nortt(struct ufs_hba *hba)
+{
+	struct ufs_mtk_host *host = ufshcd_get_variant(hba);
+
+	if (host->legacy_ip_ver || host->ip_ver < IP_VER_MT6995_B0)
+		return MTK_MAX_NUM_RTT_LEGACY;
+
+	return FIELD_GET(MASK_NUMBER_OUTSTANDING_RTT, hba->capabilities) + 1;
+}
+
 static int ufs_mtk_get_hba_mac(struct ufs_hba *hba)
 {
 	struct ufs_mtk_host *host = ufshcd_get_variant(hba);
@@ -2322,7 +2332,6 @@ static void ufs_mtk_config_scsi_dev(struct scsi_device *sdev)
  */
 static const struct ufs_hba_variant_ops ufs_hba_mtk_vops = {
 	.name                = "mediatek.ufshci",
-	.max_num_rtt         = MTK_MAX_NUM_RTT,
 	.init                = ufs_mtk_init,
 	.get_ufs_hci_version = ufs_mtk_get_ufs_hci_version,
 	.setup_clocks        = ufs_mtk_setup_clocks,
@@ -2339,6 +2348,7 @@ static const struct ufs_hba_variant_ops ufs_hba_mtk_vops = {
 	.event_notify        = ufs_mtk_event_notify,
 	.config_scaling_param = ufs_mtk_config_scaling_param,
 	.clk_scale_notify    = ufs_mtk_clk_scale_notify,
+	.get_hba_nortt       = ufs_mtk_get_hba_nortt,
 	/* mcq vops */
 	.get_hba_mac         = ufs_mtk_get_hba_mac,
 	.op_runtime_config   = ufs_mtk_op_runtime_config,
diff --git a/drivers/ufs/host/ufs-mediatek.h b/drivers/ufs/host/ufs-mediatek.h
index 8547a6f049909..73cdc726f2903 100644
--- a/drivers/ufs/host/ufs-mediatek.h
+++ b/drivers/ufs/host/ufs-mediatek.h
@@ -203,8 +203,8 @@ struct ufs_mtk_host {
 /* MTK delay of autosuspend: 500 ms */
 #define MTK_RPM_AUTOSUSPEND_DELAY_MS 500
 
-/* MTK RTT support number */
-#define MTK_MAX_NUM_RTT 2
+/* MTK RTT support number for platforms before MT6995 B0 */
+#define MTK_MAX_NUM_RTT_LEGACY 2
 
 /* UFSHCI MTK ip version value */
 enum {
diff --git a/drivers/ufs/host/ufs-qcom.c b/drivers/ufs/host/ufs-qcom.c
index 291c434487648..0500320545c43 100644
--- a/drivers/ufs/host/ufs-qcom.c
+++ b/drivers/ufs/host/ufs-qcom.c
@@ -2794,7 +2794,7 @@ static int ufs_qcom_get_rx_fom(struct ufs_hba *hba,
 	if (ret) {
 		dev_err(hba->dev, "%s: Failed to apply TX EQ settings for HS-G%u: %d\n",
 			__func__, gear, ret);
-		return ret;
+		goto link_recover_and_restore;
 	}
 
 	/* Force PMC to target HS Gear to use new TX Equalization settings. */
@@ -2802,16 +2802,15 @@ static int ufs_qcom_get_rx_fom(struct ufs_hba *hba,
 	if (ret) {
 		dev_err(hba->dev, "%s: Failed to change power mode to HS-G%u, Rate-%s: %d\n",
 			__func__, gear, ufs_hs_rate_to_str(rate), ret);
-		return ret;
+		goto link_recover_and_restore;
 	}
 
 	ret = ufs_qcom_host_sw_rx_fom(hba, pwr_mode->lane_rx, fom);
-	if (ret) {
+	if (ret)
 		dev_err(hba->dev, "Failed to get SW FOM of TX (PreShoot: %u, DeEmphasis: %u): %d\n",
 			d_iter->preshoot, d_iter->deemphasis, ret);
-		return ret;
-	}
 
+link_recover_and_restore:
 	/* Restore Device's TX Equalization settings. */
 	ret = ufshcd_apply_tx_eq_settings(hba, &hba->tx_eq_params[gear - 1], gear);
 	if (ret) {
diff --git a/drivers/ufs/host/ufshcd-pltfrm.c b/drivers/ufs/host/ufshcd-pltfrm.c
index c2dafb583cf51..5ac7afe759346 100644
--- a/drivers/ufs/host/ufshcd-pltfrm.c
+++ b/drivers/ufs/host/ufshcd-pltfrm.c
@@ -210,6 +210,158 @@ static void ufshcd_init_lanes_per_dir(struct ufs_hba *hba)
 	}
 }
 
+static int ufshcd_parse_tx_precode_enable(struct ufs_hba *hba,
+					  bool host_precode_en[UFS_MAX_LANES],
+					  bool device_precode_en[UFS_MAX_LANES])
+{
+	const char *prop_name = "tx-precode-enable-g6";
+	u32 num_elems = 2 * hba->lanes_per_direction;
+	const u32 lpd = hba->lanes_per_direction;
+	u32 precode[UFS_MAX_LANES * 2];
+	struct device *dev = hba->dev;
+	int count, err, i;
+
+	count = of_property_count_u32_elems(dev->of_node, prop_name);
+	if (count == -EINVAL || count == -ENODATA)
+		return 0;
+
+	if (count < 0)
+		return count;
+
+	if (count != num_elems) {
+		dev_err(dev, "Property %s has invalid count (%d), expecting %u\n",
+			prop_name, count, num_elems);
+		return -EINVAL;
+	}
+
+	err = of_property_read_u32_array(dev->of_node, prop_name, precode, num_elems);
+	if (err) {
+		dev_err(dev, "Failed to read %s property, %d\n", prop_name, err);
+		return err;
+	}
+
+	for (i = 0; i < num_elems; i++) {
+		if (precode[i] > 1) {
+			dev_err(dev, "Invalid TX precode value (%u) in %s property\n",
+				precode[i], prop_name);
+			return -EINVAL;
+		}
+	}
+
+	for (i = 0; i < lpd; i++) {
+		host_precode_en[i] = precode[i * 2];
+		device_precode_en[i] = precode[i * 2 + 1];
+	}
+
+	return 0;
+}
+
+static int ufshcd_parse_tx_eq_value_array(struct ufs_hba *hba,
+					  const char *prop_name,
+					  const u32 max_value,
+					  u32 values[UFS_MAX_LANES * 2])
+{
+	u32 num_elems = 2 * hba->lanes_per_direction;
+	struct device *dev = hba->dev;
+	int count, err, i;
+
+	count = of_property_count_u32_elems(dev->of_node, prop_name);
+	if (count < 0)
+		return count;
+
+	if (count != num_elems) {
+		dev_err(dev, "Property %s has invalid count (%d), expecting %u\n",
+			prop_name, count, num_elems);
+		return -EINVAL;
+	}
+
+	err = of_property_read_u32_array(dev->of_node, prop_name, values, num_elems);
+	if (err) {
+		dev_err(dev, "Failed to read %s property, %d\n", prop_name, err);
+		return err;
+	}
+
+	for (i = 0; i < num_elems; i++) {
+		if (values[i] >= max_value) {
+			dev_err(dev, "Invalid TX EQ value (%u) in %s property\n",
+				values[i], prop_name);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * ufshcd_parse_tx_eq_settings_for_gear - Parse static TX EQ DT settings for one gear
+ * @hba: per adapter instance
+ * @gear: target HS gear
+ *
+ * Reads the txeq-preshoot-gN, txeq-deemphasis-gN, and (for G6)
+ * tx-precode-enable-g6 device-tree properties.
+ * If all present values are valid, stores them as static TX Equalization
+ * settings for the given gear.
+ */
+static void ufshcd_parse_tx_eq_settings_for_gear(struct ufs_hba *hba, int gear)
+{
+	bool device_precode_en[UFS_MAX_LANES] = { false };
+	bool host_precode_en[UFS_MAX_LANES] = { false };
+	const u32 lpd = hba->lanes_per_direction;
+	struct ufshcd_tx_eq_params *params;
+	u32 deemphasis[UFS_MAX_LANES * 2];
+	u32 preshoot[UFS_MAX_LANES * 2];
+	char prop_name[MAX_PROP_SIZE];
+	int err, lane;
+
+	snprintf(prop_name, MAX_PROP_SIZE, "txeq-preshoot-g%d", gear);
+	err = ufshcd_parse_tx_eq_value_array(hba, prop_name, TX_HS_NUM_PRESHOOT, preshoot);
+	if (err)
+		return;
+
+	snprintf(prop_name, MAX_PROP_SIZE, "txeq-deemphasis-g%d", gear);
+	err = ufshcd_parse_tx_eq_value_array(hba, prop_name, TX_HS_NUM_DEEMPHASIS, deemphasis);
+	if (err)
+		return;
+
+	if (gear == UFS_HS_G6) {
+		err = ufshcd_parse_tx_precode_enable(hba, host_precode_en, device_precode_en);
+		if (err)
+			return;
+	}
+
+	params = &hba->tx_eq_params[gear - 1];
+	for (lane = 0; lane < lpd; lane++) {
+		params->host[lane].preshoot = preshoot[lane * 2];
+		params->host[lane].deemphasis = deemphasis[lane * 2];
+		params->host[lane].precode_en = host_precode_en[lane];
+
+		params->device[lane].preshoot = preshoot[lane * 2 + 1];
+		params->device[lane].deemphasis = deemphasis[lane * 2 + 1];
+		params->device[lane].precode_en = device_precode_en[lane];
+	}
+
+	params->is_valid = true;
+	params->from_dt = true;
+}
+
+static void ufshcd_parse_static_tx_eq_settings(struct ufs_hba *hba)
+{
+	const u32 lpd = hba->lanes_per_direction;
+	int gear;
+
+	if (!lpd)
+		return;
+
+	if (lpd > UFS_MAX_LANES) {
+		dev_warn(hba->dev, "lanes_per_direction (%u) exceeds UFS_MAX_LANES (%u)\n",
+			 lpd, UFS_MAX_LANES);
+		return;
+	}
+
+	for (gear = UFS_HS_G1; gear <= UFS_HS_GEAR_MAX; gear++)
+		ufshcd_parse_tx_eq_settings_for_gear(hba, gear);
+}
+
 /**
  * ufshcd_parse_clock_min_max_freq  - Parse MIN and MAX clocks freq
  * @hba: per adapter instance
@@ -528,6 +680,8 @@ int ufshcd_pltfrm_init(struct platform_device *pdev,
 
 	ufshcd_init_lanes_per_dir(hba);
 
+	ufshcd_parse_static_tx_eq_settings(hba);
+
 	err = ufshcd_parse_operating_points(hba);
 	if (err) {
 		dev_err(dev, "%s: OPP parse failed %d\n", __func__, err);
diff --git a/drivers/usb/atm/ueagle-atm.c b/drivers/usb/atm/ueagle-atm.c
index 4e71ed679a761..4266a0cb7e3b3 100644
--- a/drivers/usb/atm/ueagle-atm.c
+++ b/drivers/usb/atm/ueagle-atm.c
@@ -2549,6 +2549,7 @@ static struct usbatm_driver uea_usbatm_driver = {
 static int uea_probe(struct usb_interface *intf, const struct usb_device_id *id)
 {
 	struct usb_device *usb = interface_to_usbdev(intf);
+	bool single_iface = usb->config->desc.bNumInterfaces == 1;
 	int ret;
 
 	uea_dbg(usb, "ADSL device found with vid (%#X) pid (%#X) Rev (%#X): %s\n",
@@ -2557,6 +2558,22 @@ static int uea_probe(struct usb_interface *intf, const struct usb_device_id *id)
 		le16_to_cpu(usb->descriptor.bcdDevice),
 		chip_name[UEA_CHIP_VERSION(id)]);
 
+	/*
+	 * uea_probe() decides between the pre-firmware and post-firmware case
+	 * from the USB id and stores a different object as interface data in
+	 * each case: a struct completion for a pre-firmware device, a struct
+	 * usbatm_data for a post-firmware one. uea_disconnect() instead tells
+	 * the two apart by the number of interfaces (a pre-firmware device
+	 * exposes a single interface, ADI930 has 2 and eagle has 3). A crafted
+	 * device advertising a pre-firmware id together with a multi-interface
+	 * descriptor (or the other way around) makes the two disagree, so that
+	 * usbatm_usb_disconnect() treats the small completion object as a
+	 * struct usbatm_data and reads out of bounds. Reject such inconsistent
+	 * descriptors so both paths make the same decision.
+	 */
+	if (UEA_IS_PREFIRM(id) != single_iface)
+		return -ENODEV;
+
 	usb_reset_device(usb);
 
 	if (UEA_IS_PREFIRM(id)) {
diff --git a/drivers/usb/atm/xusbatm.c b/drivers/usb/atm/xusbatm.c
index 0befbf63d1cc8..5c1e1f5215555 100644
--- a/drivers/usb/atm/xusbatm.c
+++ b/drivers/usb/atm/xusbatm.c
@@ -79,7 +79,7 @@ static int xusbatm_bind(struct usbatm_data *usbatm,
 			struct usb_interface *intf, const struct usb_device_id *id)
 {
 	struct usb_device *usb_dev = interface_to_usbdev(intf);
-	int drv_ix = id - xusbatm_usb_ids;
+	int drv_ix = id->driver_info;
 	int rx_alt = rx_altsetting[drv_ix];
 	int tx_alt = tx_altsetting[drv_ix];
 	struct usb_interface *rx_intf = xusbatm_find_intf(usb_dev, rx_alt, rx_endpoint[drv_ix]);
@@ -168,7 +168,8 @@ static struct usb_driver xusbatm_usb_driver = {
 	.name		= xusbatm_driver_name,
 	.probe		= xusbatm_usb_probe,
 	.disconnect	= usbatm_usb_disconnect,
-	.id_table	= xusbatm_usb_ids
+	.id_table	= xusbatm_usb_ids,
+	.no_dynamic_id	= 1,
 };
 
 static int __init xusbatm_init(void)
@@ -190,6 +191,7 @@ static int __init xusbatm_init(void)
 		xusbatm_usb_ids[i].match_flags	= USB_DEVICE_ID_MATCH_DEVICE;
 		xusbatm_usb_ids[i].idVendor	= vendor[i];
 		xusbatm_usb_ids[i].idProduct	= product[i];
+		xusbatm_usb_ids[i].driver_info	= i;
 
 		xusbatm_drivers[i].driver_name	= xusbatm_driver_name;
 		xusbatm_drivers[i].bind		= xusbatm_bind;
diff --git a/drivers/usb/cdns3/cdns3-plat.c b/drivers/usb/cdns3/cdns3-plat.c
index bb5405460035e..e3f32c3e95352 100644
--- a/drivers/usb/cdns3/cdns3-plat.c
+++ b/drivers/usb/cdns3/cdns3-plat.c
@@ -147,12 +147,12 @@ static int cdns3_plat_probe(struct platform_device *pdev)
 	cdns->usb2_phy = devm_phy_optional_get(dev, "cdns3,usb2-phy");
 	if (IS_ERR(cdns->usb2_phy))
 		return dev_err_probe(dev, PTR_ERR(cdns->usb2_phy),
-				     "Failed to get cdn3,usb2-phy\n");
+				     "Failed to get cdns3,usb2-phy\n");
 
 	cdns->usb3_phy = devm_phy_optional_get(dev, "cdns3,usb3-phy");
 	if (IS_ERR(cdns->usb3_phy))
 		return dev_err_probe(dev, PTR_ERR(cdns->usb3_phy),
-				     "Failed to get cdn3,usb3-phy\n");
+				     "Failed to get cdns3,usb3-phy\n");
 
 	ret = phy_init(cdns->usb2_phy);
 	if (ret)
diff --git a/drivers/usb/chipidea/ci_hdrc_imx.c b/drivers/usb/chipidea/ci_hdrc_imx.c
index 56d2ba824a0b2..282314eea7fc1 100644
--- a/drivers/usb/chipidea/ci_hdrc_imx.c
+++ b/drivers/usb/chipidea/ci_hdrc_imx.c
@@ -528,7 +528,7 @@ static int ci_hdrc_imx_probe(struct platform_device *pdev)
 	if (data->wakeup_irq > 0) {
 		irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s:wakeup", pdata.name);
 		if (!irq_name) {
-			dev_err_probe(dev, -ENOMEM, "failed to create irq_name\n");
+			ret = dev_err_probe(dev, -ENOMEM, "failed to create irq_name\n");
 			goto err_clk;
 		}
 
diff --git a/drivers/usb/chipidea/core.c b/drivers/usb/chipidea/core.c
index 07563be0013f4..3461dd763a6e2 100644
--- a/drivers/usb/chipidea/core.c
+++ b/drivers/usb/chipidea/core.c
@@ -879,7 +879,7 @@ struct platform_device *ci_hdrc_add_device(struct device *dev,
 	}
 
 	pdev->dev.parent = dev;
-	device_set_of_node_from_dev(&pdev->dev, dev);
+	platform_device_set_of_node_from_dev(pdev, dev);
 
 	ret = platform_device_add_resources(pdev, res, nres);
 	if (ret)
@@ -1250,6 +1250,7 @@ static void ci_hdrc_remove(struct platform_device *pdev)
 		usb_role_switch_unregister(ci->role_switch);
 
 	if (ci->supports_runtime_pm) {
+		pm_runtime_dont_use_autosuspend(&pdev->dev);
 		pm_runtime_get_sync(&pdev->dev);
 		pm_runtime_disable(&pdev->dev);
 		pm_runtime_put_noidle(&pdev->dev);
diff --git a/drivers/usb/class/usbtmc.c b/drivers/usb/class/usbtmc.c
index af9ae55dae14e..51cd9320a7366 100644
--- a/drivers/usb/class/usbtmc.c
+++ b/drivers/usb/class/usbtmc.c
@@ -71,7 +71,6 @@ struct usbtmc_dev_capabilities {
  * allocated for each USBTMC device in the driver's probe function.
  */
 struct usbtmc_device_data {
-	const struct usb_device_id *id;
 	struct usb_device *usb_dev;
 	struct usb_interface *intf;
 	struct list_head file_list;
@@ -2394,7 +2393,6 @@ static int usbtmc_probe(struct usb_interface *intf,
 		return -ENOMEM;
 
 	data->intf = intf;
-	data->id = id;
 	data->usb_dev = usb_get_dev(interface_to_usbdev(intf));
 	usb_set_intfdata(intf, data);
 	kref_init(&data->kref);
diff --git a/drivers/usb/core/config.c b/drivers/usb/core/config.c
index 45e20c6d76c04..cd3231d210902 100644
--- a/drivers/usb/core/config.c
+++ b/drivers/usb/core/config.c
@@ -151,7 +151,7 @@ static void usb_parse_ss_endpoint_companion(struct device *ddev, int cfgno,
 			usb_endpoint_xfer_int(&ep->desc)) &&
 				desc->bmAttributes != 0) {
 		dev_notice(ddev, "%s endpoint with bmAttributes = %d in config %d interface %d altsetting %d ep 0x%X: setting to zero\n",
-				usb_endpoint_xfer_control(&ep->desc) ? "Control" : "Bulk",
+				usb_endpoint_xfer_control(&ep->desc) ? "Control" : "Interrupt",
 				desc->bmAttributes,
 				cfgno, inum, asnum, ep->desc.bEndpointAddress);
 		ep->ss_ep_comp.bmAttributes = 0;
diff --git a/drivers/usb/core/devices.c b/drivers/usb/core/devices.c
index a247da73f34d7..6f0354aba38b3 100644
--- a/drivers/usb/core/devices.c
+++ b/drivers/usb/core/devices.c
@@ -37,6 +37,7 @@
  */
 
 #include <linux/fs.h>
+#include <linux/slab.h>
 #include <linux/mm.h>
 #include <linux/gfp.h>
 #include <linux/usb.h>
@@ -408,7 +409,7 @@ static ssize_t usb_device_dump(char __user **buffer, size_t *nbytes,
 		return 0;
 	/* allocate 2^1 pages = 8K (on i386);
 	 * should be more than enough for one device */
-	pages_start = (char *)__get_free_pages(GFP_NOIO, 1);
+	pages_start = kmalloc(PAGE_SIZE << 1, GFP_NOIO);
 	if (!pages_start)
 		return -ENOMEM;
 
@@ -479,7 +480,7 @@ static ssize_t usb_device_dump(char __user **buffer, size_t *nbytes,
 		if (length > *nbytes)
 			length = *nbytes;
 		if (copy_to_user(*buffer, pages_start + *skip_bytes, length)) {
-			free_pages((unsigned long)pages_start, 1);
+			kfree(pages_start);
 			return -EFAULT;
 		}
 		*nbytes -= length;
@@ -490,7 +491,7 @@ static ssize_t usb_device_dump(char __user **buffer, size_t *nbytes,
 	} else
 		*skip_bytes -= length;
 
-	free_pages((unsigned long)pages_start, 1);
+	kfree(pages_start);
 
 	/* Now look at all of this device's children. */
 	usb_hub_for_each_child(usbdev, chix, childdev) {
diff --git a/drivers/usb/core/devio.c b/drivers/usb/core/devio.c
index e191934623c73..8329d1c7d1b27 100644
--- a/drivers/usb/core/devio.c
+++ b/drivers/usb/core/devio.c
@@ -2329,6 +2329,13 @@ static int proc_ioctl(struct usb_dev_state *ps, struct usbdevfs_ioctl *ctl)
 	if (!connected(ps))
 		return -ENODEV;
 
+	if (ps->dev->state != USB_STATE_CONFIGURED)
+		return -EHOSTUNREACH;
+
+	intf = usb_ifnum_to_if(ps->dev, ctl->ifno);
+	if (!intf)
+		return -EINVAL;
+
 	/* alloc buffer */
 	size = _IOC_SIZE(ctl->ioctl_code);
 	if (size > 0) {
@@ -2345,11 +2352,7 @@ static int proc_ioctl(struct usb_dev_state *ps, struct usbdevfs_ioctl *ctl)
 		}
 	}
 
-	if (ps->dev->state != USB_STATE_CONFIGURED)
-		retval = -EHOSTUNREACH;
-	else if (!(intf = usb_ifnum_to_if(ps->dev, ctl->ifno)))
-		retval = -EINVAL;
-	else switch (ctl->ioctl_code) {
+	switch (ctl->ioctl_code) {
 
 	/* disconnect kernel driver from interface */
 	case USBDEVFS_DISCONNECT:
diff --git a/drivers/usb/core/driver.c b/drivers/usb/core/driver.c
index f63004417058e..7f33fe5ba03bd 100644
--- a/drivers/usb/core/driver.c
+++ b/drivers/usb/core/driver.c
@@ -228,14 +228,16 @@ static void usb_free_dynids(struct usb_driver *usb_drv)
 }
 
 static const struct usb_device_id *usb_match_dynamic_id(struct usb_interface *intf,
-							const struct usb_driver *drv)
+							const struct usb_driver *drv,
+							struct usb_device_id *id_copy)
 {
 	struct usb_dynid *dynid;
 
 	guard(mutex)(&usb_dynids_lock);
 	list_for_each_entry(dynid, &drv->dynids.list, node) {
 		if (usb_match_one_id(intf, &dynid->id)) {
-			return &dynid->id;
+			*id_copy = dynid->id;
+			return id_copy;
 		}
 	}
 	return NULL;
@@ -321,6 +323,7 @@ static int usb_probe_interface(struct device *dev)
 	struct usb_interface *intf = to_usb_interface(dev);
 	struct usb_device *udev = interface_to_usbdev(intf);
 	const struct usb_device_id *id;
+	struct usb_device_id id_copy;
 	int error = -ENODEV;
 	int lpm_disable_error = -ENODEV;
 
@@ -340,7 +343,7 @@ static int usb_probe_interface(struct device *dev)
 		return error;
 	}
 
-	id = usb_match_dynamic_id(intf, driver);
+	id = usb_match_dynamic_id(intf, driver, &id_copy);
 	if (!id)
 		id = usb_match_id(intf, driver->id_table);
 	if (!id)
@@ -892,6 +895,7 @@ static int usb_device_match(struct device *dev, const struct device_driver *drv)
 		struct usb_interface *intf;
 		const struct usb_driver *usb_drv;
 		const struct usb_device_id *id;
+		struct usb_device_id id_copy;
 
 		/* device drivers never match interfaces */
 		if (is_usb_device_driver(drv))
@@ -904,7 +908,7 @@ static int usb_device_match(struct device *dev, const struct device_driver *drv)
 		if (id)
 			return 1;
 
-		id = usb_match_dynamic_id(intf, usb_drv);
+		id = usb_match_dynamic_id(intf, usb_drv, &id_copy);
 		if (id)
 			return 1;
 	}
diff --git a/drivers/usb/core/port.c b/drivers/usb/core/port.c
index b1364f0c384ce..b4452b665f591 100644
--- a/drivers/usb/core/port.c
+++ b/drivers/usb/core/port.c
@@ -740,6 +740,8 @@ static void connector_unbind(struct device *dev, struct device *connector, void
 
 	sysfs_remove_link(&connector->kobj, dev_name(dev));
 	sysfs_remove_link(&dev->kobj, "connector");
+	if (port_dev->child)
+		typec_deattach(port_dev->connector, &port_dev->child->dev);
 	port_dev->connector = NULL;
 }
 
diff --git a/drivers/usb/core/sysfs.c b/drivers/usb/core/sysfs.c
index a07866f1060cf..d22dc78457d79 100644
--- a/drivers/usb/core/sysfs.c
+++ b/drivers/usb/core/sysfs.c
@@ -899,10 +899,15 @@ bos_descriptors_read(struct file *filp, struct kobject *kobj,
 {
 	struct device *dev = kobj_to_dev(kobj);
 	struct usb_device *udev = to_usb_device(dev);
-	struct usb_host_bos *bos = udev->bos;
+	struct usb_host_bos *bos;
 	struct usb_bos_descriptor *desc;
 	size_t desclen, n = 0;
+	int rc;
 
+	rc = usb_lock_device_interruptible(udev);
+	if (rc < 0)
+		return -EINTR;
+	bos = udev->bos;
 	if (bos) {
 		desc = bos->desc;
 		desclen = le16_to_cpu(desc->wTotalLength);
@@ -911,6 +916,7 @@ bos_descriptors_read(struct file *filp, struct kobject *kobj,
 			memcpy(buf, (void *) desc + off, n);
 		}
 	}
+	usb_unlock_device(udev);
 	return n;
 }
 static const BIN_ATTR_RO(bos_descriptors, 65535); /* max-size BOS */
diff --git a/drivers/usb/dwc2/core_intr.c b/drivers/usb/dwc2/core_intr.c
index 7d3e641806f82..9565e6a52725a 100644
--- a/drivers/usb/dwc2/core_intr.c
+++ b/drivers/usb/dwc2/core_intr.c
@@ -334,9 +334,8 @@ static void dwc2_handle_session_req_intr(struct dwc2_hsotg *hsotg)
 
 /**
  * dwc2_wakeup_from_lpm_l1 - Exit the device from LPM L1 state
- *
  * @hsotg: Programming view of DWC_otg controller
- *
+ * @remotewakeup: Whether this is a remote wakeup
  */
 void dwc2_wakeup_from_lpm_l1(struct dwc2_hsotg *hsotg, bool remotewakeup)
 {
diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c
index ceb49f2f80041..fd5c2cd36c595 100644
--- a/drivers/usb/dwc3/core.c
+++ b/drivers/usb/dwc3/core.c
@@ -2188,22 +2188,89 @@ static void dwc3_vbus_draw_work(struct work_struct *work)
 			ret, dwc->current_limit);
 }
 
-static struct power_supply *dwc3_get_usb_power_supply(struct dwc3 *dwc)
+static int dwc3_psy_notifier(struct notifier_block *nb,
+			     unsigned long event, void *data)
 {
-	struct power_supply *usb_psy;
-	const char *usb_psy_name;
+	struct dwc3 *dwc = container_of(nb, struct dwc3, psy_nb);
+	struct power_supply *psy = data;
+	unsigned long flags;
+
+	if (dwc->usb_psy)
+		return NOTIFY_DONE;
+
+	if (strcmp(psy->desc->name, dwc->usb_psy_name) != 0)
+		return NOTIFY_DONE;
+
+	/* Explicitly get the reference for this psy */
+	psy = power_supply_get_by_name(dwc->usb_psy_name);
+	if (!psy)
+		return NOTIFY_DONE;
+
+	spin_lock_irqsave(&dwc->lock, flags);
+	/*
+	 * The USB power_supply may already be set. This can happen if notifier
+	 * callbacks for the USB power_supply race, or if a previous notifier
+	 * callback has already successfully fetched and associated the instance.
+	 * In such cases, release the newly acquired reference and ignore
+	 * subsequent notifications until the notifier is unregistered.
+	 */
+	if (dwc->usb_psy) {
+		spin_unlock_irqrestore(&dwc->lock, flags);
+		power_supply_put(psy);
+		return NOTIFY_DONE;
+	}
+
+	dwc->usb_psy = psy;
+	if (dwc->current_limit != DWC3_CURRENT_UNSPECIFIED)
+		schedule_work(&dwc->vbus_draw_work);
+	spin_unlock_irqrestore(&dwc->lock, flags);
+
+	return NOTIFY_OK;
+}
+
+static void dwc3_get_usb_power_supply(struct dwc3 *dwc)
+{
+	struct power_supply *psy;
+	unsigned long flags;
 	int ret;
 
-	ret = device_property_read_string(dwc->dev, "usb-psy-name", &usb_psy_name);
+	ret = device_property_read_string(dwc->dev, "usb-psy-name", &dwc->usb_psy_name);
 	if (ret < 0)
-		return NULL;
-
-	usb_psy = power_supply_get_by_name(usb_psy_name);
-	if (!usb_psy)
-		return ERR_PTR(-EPROBE_DEFER);
+		return;
 
 	INIT_WORK(&dwc->vbus_draw_work, dwc3_vbus_draw_work);
-	return usb_psy;
+
+	dwc->current_limit = DWC3_CURRENT_UNSPECIFIED;
+	dwc->psy_nb.notifier_call = dwc3_psy_notifier;
+	ret = power_supply_reg_notifier(&dwc->psy_nb);
+	if (ret) {
+		dev_err(dwc->dev, "Failed to register power supply notifier: %d\n", ret);
+		dwc->psy_nb.notifier_call = NULL;
+		return;
+	}
+
+	psy = power_supply_get_by_name(dwc->usb_psy_name);
+	if (!psy)
+		return;
+
+	/* Unregister the notifier now that we have the power supply */
+	power_supply_unreg_notifier(&dwc->psy_nb);
+	dwc->psy_nb.notifier_call = NULL;
+
+	spin_lock_irqsave(&dwc->lock, flags);
+	/*
+	 * It is possible that the notifier callback ran before we reached here
+	 * and successfully fetched the power supply. In that case we need to
+	 * release the above reference.
+	 */
+	if (dwc->usb_psy) {
+		spin_unlock_irqrestore(&dwc->lock, flags);
+		power_supply_put(psy);
+		return;
+	}
+
+	dwc->usb_psy = psy;
+	spin_unlock_irqrestore(&dwc->lock, flags);
 }
 
 int dwc3_core_probe(const struct dwc3_probe_data *data)
@@ -2251,9 +2318,9 @@ int dwc3_core_probe(const struct dwc3_probe_data *data)
 
 	dwc3_get_software_properties(dwc, &data->properties);
 
-	dwc->usb_psy = dwc3_get_usb_power_supply(dwc);
-	if (IS_ERR(dwc->usb_psy))
-		return dev_err_probe(dev, PTR_ERR(dwc->usb_psy), "couldn't get usb power supply\n");
+	spin_lock_init(&dwc->lock);
+
+	dwc3_get_usb_power_supply(dwc);
 
 	if (!data->ignore_clocks_and_resets) {
 		dwc->reset = devm_reset_control_array_get_optional_shared(dev);
@@ -2305,7 +2372,6 @@ int dwc3_core_probe(const struct dwc3_probe_data *data)
 		dwc->num_usb3_ports = 1;
 	}
 
-	spin_lock_init(&dwc->lock);
 	mutex_init(&dwc->mutex);
 
 	pm_runtime_get_noresume(dev);
@@ -2373,6 +2439,8 @@ int dwc3_core_probe(const struct dwc3_probe_data *data)
 err_assert_reset:
 	reset_control_assert(dwc->reset);
 err_put_psy:
+	if (dwc->psy_nb.notifier_call)
+		power_supply_unreg_notifier(&dwc->psy_nb);
 	if (dwc->usb_psy)
 		power_supply_put(dwc->usb_psy);
 
@@ -2429,6 +2497,9 @@ void dwc3_core_remove(struct dwc3 *dwc)
 
 	dwc3_free_event_buffers(dwc);
 
+	if (dwc->psy_nb.notifier_call)
+		power_supply_unreg_notifier(&dwc->psy_nb);
+
 	if (dwc->usb_psy) {
 		cancel_work_sync(&dwc->vbus_draw_work);
 		power_supply_put(dwc->usb_psy);
diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h
index e0dee9d287401..608daeb7ef104 100644
--- a/drivers/usb/dwc3/core.h
+++ b/drivers/usb/dwc3/core.h
@@ -722,7 +722,6 @@ struct dwc3_event_buffer {
  * @cancelled_list: list of cancelled requests for this endpoint
  * @pending_list: list of pending requests for this endpoint
  * @started_list: list of started requests on this endpoint
- * @regs: pointer to first endpoint register
  * @trb_pool: array of transaction buffers
  * @trb_pool_dma: dma address of @trb_pool
  * @trb_enqueue: enqueue 'pointer' into TRB array
@@ -1059,6 +1058,8 @@ struct dwc3_glue_ops {
  * @role_switch_default_mode: default operation mode of controller while
  *			usb role is USB_ROLE_NONE.
  * @usb_psy: pointer to power supply interface.
+ * @usb_psy_name: name of the USB power supply
+ * @psy_nb: power supply notifier block
  * @vbus_draw_work: Work to set the vbus drawing limit
  * @current_limit: How much current to draw from vbus, in milliAmperes.
  * @usb2_phy: pointer to USB2 PHY
@@ -1251,9 +1252,13 @@ struct dwc3 {
 	enum usb_dr_mode	role_switch_default_mode;
 
 	struct power_supply	*usb_psy;
+	const char		*usb_psy_name;
+	struct notifier_block	psy_nb;
 	struct work_struct	vbus_draw_work;
 	unsigned int		current_limit;
 
+#define DWC3_CURRENT_UNSPECIFIED	UINT_MAX
+
 	u32			fladj;
 	u32			ref_clk_per;
 	u32			irq_gadget;
diff --git a/drivers/usb/dwc3/dwc3-am62.c b/drivers/usb/dwc3/dwc3-am62.c
index e11d7643f9669..632634d6e81e1 100644
--- a/drivers/usb/dwc3/dwc3-am62.c
+++ b/drivers/usb/dwc3/dwc3-am62.c
@@ -205,7 +205,9 @@ static int dwc3_ti_init(struct dwc3_am62 *am62)
 
 	dwc3_ti_writel(am62, USBSS_PHY_CONFIG, reg);
 
-	clk_prepare_enable(am62->usb2_refclk);
+	ret = clk_prepare_enable(am62->usb2_refclk);
+	if (ret)
+		return ret;
 
 	/* Set mode valid bit to indicate role is valid */
 	reg = dwc3_ti_readl(am62, USBSS_MODE_CONTROL);
@@ -361,14 +363,19 @@ static int dwc3_ti_resume_common(struct device *dev)
 {
 	struct dwc3_am62 *am62 = dev_get_drvdata(dev);
 	u32 reg;
+	int ret;
 
 	reg = dwc3_ti_readl(am62, USBSS_DEBUG_CFG);
 	if (reg != USBSS_DEBUG_CFG_DISABLED) {
 		/* lost power/context */
-		dwc3_ti_init(am62);
+		ret = dwc3_ti_init(am62);
+		if (ret)
+			return ret;
 	} else {
 		dwc3_ti_writel(am62, USBSS_DEBUG_CFG, USBSS_DEBUG_CFG_OFF);
-		clk_prepare_enable(am62->usb2_refclk);
+		ret = clk_prepare_enable(am62->usb2_refclk);
+		if (ret)
+			return ret;
 	}
 
 	if (device_may_wakeup(dev)) {
diff --git a/drivers/usb/dwc3/dwc3-qcom.c b/drivers/usb/dwc3/dwc3-qcom.c
index f43f73ac36ff1..ac68b4218b56b 100644
--- a/drivers/usb/dwc3/dwc3-qcom.c
+++ b/drivers/usb/dwc3/dwc3-qcom.c
@@ -602,7 +602,7 @@ static void dwc3_qcom_run_stop_notifier(struct dwc3 *dwc, bool is_on)
 	pm_runtime_mark_last_busy(qcom->dev);
 }
 
-struct dwc3_glue_ops dwc3_qcom_glue_ops = {
+static struct dwc3_glue_ops dwc3_qcom_glue_ops = {
 	.pre_set_role	= dwc3_qcom_set_role_notifier,
 	.pre_run_stop	= dwc3_qcom_run_stop_notifier,
 };
diff --git a/drivers/usb/dwc3/dwc3-xilinx.c b/drivers/usb/dwc3/dwc3-xilinx.c
index 9b9525592a851..b832505e1b042 100644
--- a/drivers/usb/dwc3/dwc3-xilinx.c
+++ b/drivers/usb/dwc3/dwc3-xilinx.c
@@ -98,18 +98,10 @@ static int dwc3_xlnx_init_versal(struct dwc3_xlnx *priv_data)
 
 	dwc3_xlnx_mask_phy_rst(priv_data, false);
 
-	/* Assert and De-assert reset */
-	ret = reset_control_assert(crst);
-	if (ret < 0) {
-		dev_err_probe(dev, ret, "failed to assert Reset\n");
-		return ret;
-	}
-
-	ret = reset_control_deassert(crst);
-	if (ret < 0) {
-		dev_err_probe(dev, ret, "failed to De-assert Reset\n");
-		return ret;
-	}
+	/* assert and deassert reset */
+	ret = reset_control_reset(crst);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to assert and deassert reset\n");
 
 	dwc3_xlnx_mask_phy_rst(priv_data, true);
 	dwc3_xlnx_set_coherency(priv_data, XLNX_USB2_TRAFFIC_ROUTE_CONFIG);
@@ -194,7 +186,7 @@ static int dwc3_xlnx_init_zynqmp(struct dwc3_xlnx *priv_data)
 	}
 
 	if (priv_data->usb3_phy) {
-		/* Set PIPE Power Present signal in FPD Power Present Register*/
+		/* Set PIPE Power Present signal in FPD Power Present Register */
 		writel(FPD_POWER_PRSNT_OPTION, priv_data->regs + XLNX_USB_FPD_POWER_PRSNT);
 		/* Set the PIPE Clock Select bit in FPD PIPE Clock register */
 		writel(PIPE_CLK_SELECT, priv_data->regs + XLNX_USB_FPD_PIPE_CLK);
diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c
index 1082e9c9afaa8..fa0f16ffafef4 100644
--- a/drivers/usb/dwc3/gadget.c
+++ b/drivers/usb/dwc3/gadget.c
@@ -3124,15 +3124,26 @@ static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g,
 static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA)
 {
 	struct dwc3		*dwc = gadget_to_dwc(g);
+	unsigned long		flags;
 
 	if (dwc->usb2_phy)
 		return usb_phy_set_power(dwc->usb2_phy, mA);
 
-	if (!dwc->usb_psy)
-		return -EOPNOTSUPP;
+	spin_lock_irqsave(&dwc->lock, flags);
+	if (!dwc->usb_psy) {
+		if (!dwc->psy_nb.notifier_call) {
+			spin_unlock_irqrestore(&dwc->lock, flags);
+			return -EOPNOTSUPP;
+		}
+		dwc->current_limit = mA;
+		spin_unlock_irqrestore(&dwc->lock, flags);
+		dev_dbg(dwc->dev, "Stored VBUS draw: %u mA (power supply not ready)\n", mA);
+		return 0;
+	}
 
 	dwc->current_limit = mA;
 	schedule_work(&dwc->vbus_draw_work);
+	spin_unlock_irqrestore(&dwc->lock, flags);
 
 	return 0;
 }
diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c
index 183a25f65ac89..51df6d1d1487a 100644
--- a/drivers/usb/gadget/configfs.c
+++ b/drivers/usb/gadget/configfs.c
@@ -125,11 +125,11 @@ static int usb_string_copy(const char *s, char **s_copy)
 	if (copy) {
 		str = copy;
 	} else {
-		str = kmalloc(USB_MAX_STRING_WITH_NULL_LEN, GFP_KERNEL);
+		str = kzalloc(USB_MAX_STRING_WITH_NULL_LEN, GFP_KERNEL);
 		if (!str)
 			return -ENOMEM;
 	}
-	strcpy(str, s);
+	memcpy(str, s, ret + 1);
 	if (str[ret - 1] == '\n')
 		str[ret - 1] = '\0';
 	*s_copy = str;
@@ -1177,7 +1177,7 @@ static ssize_t os_desc_qw_sign_show(struct config_item *item, char *page)
 	struct gadget_info *gi = os_desc_item_to_gadget_info(item);
 	int res;
 
-	res = utf16s_to_utf8s((wchar_t *) gi->qw_sign, OS_STRING_QW_SIGN_LEN,
+	res = utf16s_to_utf8s((wchar_t *) gi->qw_sign, OS_STRING_QW_SIGN_LEN / 2,
 			      UTF16_LITTLE_ENDIAN, page, PAGE_SIZE - 1);
 	page[res++] = '\n';
 
@@ -1199,7 +1199,7 @@ static ssize_t os_desc_qw_sign_store(struct config_item *item, const char *page,
 	mutex_lock(&gi->lock);
 	res = utf8s_to_utf16s(page, l,
 			      UTF16_LITTLE_ENDIAN, (wchar_t *) gi->qw_sign,
-			      OS_STRING_QW_SIGN_LEN);
+			      OS_STRING_QW_SIGN_LEN / 2);
 	if (res > 0)
 		res = len;
 	mutex_unlock(&gi->lock);
diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c
index 44218be1e6768..0fa7672f95da8 100644
--- a/drivers/usb/gadget/function/f_fs.c
+++ b/drivers/usb/gadget/function/f_fs.c
@@ -159,7 +159,9 @@ struct ffs_epfile {
 	struct mutex			mutex;
 
 	struct ffs_data			*ffs;
-	struct ffs_ep			*ep;	/* P: ffs->eps_lock */
+	struct ffs_ep			*ep;		/* P: ffs->eps_lock */
+	struct ffs_epfile		*epfile_in;	/* P: ffs->eps_lock */
+	struct ffs_epfile		*epfile_out;	/* P: ffs->eps_lock */
 
 	/*
 	 * Buffer for holding data from partial reads which may happen since
@@ -219,12 +221,13 @@ struct ffs_epfile {
 	struct ffs_buffer		*read_buffer;
 #define READ_BUFFER_DROP ((struct ffs_buffer *)ERR_PTR(-ESHUTDOWN))
 
-	char				name[5];
+	char				name[8];
 
 	unsigned char			in;	/* P: ffs->eps_lock */
 	unsigned char			isoc;	/* P: ffs->eps_lock */
 
-	unsigned char			_pad;
+	u8				zlp_enabled; /* P: ffs->eps_lock */
+	bool				is_rw_proxy;
 
 	/* Protects dmabufs */
 	struct mutex			dmabufs_mutex;
@@ -867,9 +870,15 @@ static void ffs_user_copy_worker(struct work_struct *work)
 	bool kiocb_has_eventfd = io_data->kiocb->ki_flags & IOCB_EVENTFD;
 
 	if (io_data->read && ret > 0) {
-		kthread_use_mm(io_data->mm);
-		ret = ffs_copy_to_iter(io_data->buf, ret, &io_data->data);
-		kthread_unuse_mm(io_data->mm);
+		if (mmget_not_zero(io_data->mm)) {
+			kthread_use_mm(io_data->mm);
+			ret = ffs_copy_to_iter(io_data->buf, ret, &io_data->data);
+			kthread_unuse_mm(io_data->mm);
+			mmput(io_data->mm);
+		} else {
+			ret = -EFAULT;
+		}
+		mmdrop(io_data->mm);
 	}
 
 	io_data->kiocb->ki_complete(io_data->kiocb, ret);
@@ -979,9 +988,8 @@ static ssize_t __ffs_epfile_read_data(struct ffs_epfile *epfile,
 	return ret;
 }
 
-static struct ffs_ep *ffs_epfile_wait_ep(struct file *file)
+static struct ffs_ep *ffs_epfile_wait_ep(struct ffs_epfile *epfile, struct file *file)
 {
-	struct ffs_epfile *epfile = file->private_data;
 	struct ffs_ep *ep;
 	int ret;
 
@@ -1008,17 +1016,22 @@ static ssize_t ffs_epfile_io(struct file *file, struct ffs_io_data *io_data)
 	char *data = NULL;
 	ssize_t ret, data_len = -EINVAL;
 	int halt;
+	bool is_rw_proxy = epfile->is_rw_proxy;
 
 	/* Are we still active? */
 	if (WARN_ON(epfile->ffs->state != FFS_ACTIVE))
 		return -ENODEV;
 
-	ep = ffs_epfile_wait_ep(file);
+	/* Proxy to base endpoint if rw_proxy */
+	if (is_rw_proxy)
+		epfile = io_data->read ? epfile->epfile_out : epfile->epfile_in;
+
+	ep = ffs_epfile_wait_ep(epfile, file);
 	if (IS_ERR(ep))
 		return PTR_ERR(ep);
 
 	/* Do we halt? */
-	halt = (!io_data->read == !epfile->in);
+	halt = is_rw_proxy ? 0 : (!io_data->read == !epfile->in);
 	if (halt && epfile->isoc)
 		return -EINVAL;
 
@@ -1115,6 +1128,8 @@ static ssize_t ffs_epfile_io(struct file *file, struct ffs_io_data *io_data)
 			req->buf = data;
 			req->num_sgs = 0;
 		}
+
+		req->zero = !io_data->read ? epfile->zlp_enabled : 0;
 		req->length = data_len;
 
 		io_data->buf = data;
@@ -1166,6 +1181,8 @@ static ssize_t ffs_epfile_io(struct file *file, struct ffs_io_data *io_data)
 			req->buf = data;
 			req->num_sgs = 0;
 		}
+
+		req->zero = !io_data->read ? epfile->zlp_enabled : 0;
 		req->length = data_len;
 
 		io_data->buf = data;
@@ -1264,16 +1281,20 @@ static ssize_t ffs_epfile_write_iter(struct kiocb *kiocb, struct iov_iter *from)
 
 	kiocb->private = p;
 
-	if (p->aio)
+	if (p->aio) {
+		mmgrab(p->mm);
 		kiocb_set_cancel_fn(kiocb, ffs_aio_cancel);
+	}
 
 	res = ffs_epfile_io(kiocb->ki_filp, p);
 	if (res == -EIOCBQUEUED)
 		return res;
-	if (p->aio)
+	if (p->aio) {
+		mmdrop(p->mm);
 		kfree(p);
-	else
+	} else {
 		*from = p->data;
+	}
 	return res;
 }
 
@@ -1308,14 +1329,17 @@ static ssize_t ffs_epfile_read_iter(struct kiocb *kiocb, struct iov_iter *to)
 
 	kiocb->private = p;
 
-	if (p->aio)
+	if (p->aio) {
+		mmgrab(p->mm);
 		kiocb_set_cancel_fn(kiocb, ffs_aio_cancel);
+	}
 
 	res = ffs_epfile_io(kiocb->ki_filp, p);
 	if (res == -EIOCBQUEUED)
 		return res;
 
 	if (p->aio) {
+		mmdrop(p->mm);
 		kfree(p->to_free);
 		kfree(p);
 	} else {
@@ -1643,7 +1667,7 @@ static int ffs_dmabuf_transfer(struct file *file,
 
 	priv = attach->importer_priv;
 
-	ep = ffs_epfile_wait_ep(file);
+	ep = ffs_epfile_wait_ep(epfile, file);
 	if (IS_ERR(ep)) {
 		ret = PTR_ERR(ep);
 		goto err_attachment_put;
@@ -1682,13 +1706,13 @@ static int ffs_dmabuf_transfer(struct file *file,
 	/* In the meantime, endpoint got disabled or changed. */
 	if (epfile->ep != ep) {
 		ret = -ESHUTDOWN;
-		goto err_fence_put;
+		goto err_fence_free;
 	}
 
 	usb_req = usb_ep_alloc_request(ep->ep, GFP_ATOMIC);
 	if (!usb_req) {
 		ret = -ENOMEM;
-		goto err_fence_put;
+		goto err_fence_free;
 	}
 
 	/*
@@ -1709,6 +1733,7 @@ static int ffs_dmabuf_transfer(struct file *file,
 
 	/* Now that the dma_fence is in place, queue the transfer. */
 
+	usb_req->zero = epfile->zlp_enabled;
 	usb_req->length = req->length;
 	usb_req->buf = NULL;
 	usb_req->sg = priv->sgt->sgl;
@@ -1737,9 +1762,9 @@ static int ffs_dmabuf_transfer(struct file *file,
 
 	return ret;
 
-err_fence_put:
+err_fence_free:
 	spin_unlock_irq(&epfile->ffs->eps_lock);
-	dma_fence_put(&fence->base);
+	kfree(fence);
 err_resv_unlock:
 	dma_resv_unlock(dmabuf->resv);
 err_attachment_put:
@@ -1756,10 +1781,14 @@ static long ffs_epfile_ioctl(struct file *file, unsigned code,
 	struct ffs_epfile *epfile = file->private_data;
 	struct ffs_ep *ep;
 	int ret;
+	__u32 enable_zlp = 0;
 
 	if (WARN_ON(epfile->ffs->state != FFS_ACTIVE))
 		return -ENODEV;
 
+	if (epfile->is_rw_proxy)
+		return -ENOTTY;
+
 	switch (code) {
 	case FUNCTIONFS_DMABUF_ATTACH:
 	{
@@ -1788,12 +1817,29 @@ static long ffs_epfile_ioctl(struct file *file, unsigned code,
 
 		return ffs_dmabuf_transfer(file, &req);
 	}
+	/*
+	 * We handle this IOCTL before ffs_epfile_wait_ep() to allow userspace
+	 * to configure ZLP behavior immediately without blocking indefinitely
+	 * while waiting for the USB host to connect and enable the endpoint.
+	 */
+	case FUNCTIONFS_ENDPOINT_ENABLE_ZLP:
+		if (!epfile->in)
+			return -EINVAL;
+
+		if (copy_from_user(&enable_zlp, (void __user *)value, sizeof(enable_zlp)))
+			return -EFAULT;
+
+		spin_lock_irq(&epfile->ffs->eps_lock);
+		epfile->zlp_enabled = !!enable_zlp;
+		spin_unlock_irq(&epfile->ffs->eps_lock);
+
+		return 0;
 	default:
 		break;
 	}
 
 	/* Wait for endpoint to be enabled */
-	ep = ffs_epfile_wait_ep(file);
+	ep = ffs_epfile_wait_ep(epfile, file);
 	if (IS_ERR(ep))
 		return PTR_ERR(ep);
 
@@ -2191,7 +2237,7 @@ static void ffs_data_closed(struct ffs_data *ffs)
 
 		if (epfiles)
 			ffs_epfiles_destroy(ffs->sb, epfiles,
-					 ffs->eps_count);
+					 ffs->epfiles_count);
 
 		if (ffs->setup_state == FFS_SETUP_PENDING)
 			__ffs_ep0_stall(ffs);
@@ -2250,7 +2296,7 @@ static void ffs_data_clear(struct ffs_data *ffs)
 	 * copy of epfile will save us from use-after-free.
 	 */
 	if (epfiles) {
-		ffs_epfiles_destroy(ffs->sb, epfiles, ffs->eps_count);
+		ffs_epfiles_destroy(ffs->sb, epfiles, ffs->epfiles_count);
 		ffs->epfiles = NULL;
 	}
 
@@ -2348,11 +2394,16 @@ static void functionfs_unbind(struct ffs_data *ffs)
 static int ffs_epfiles_create(struct ffs_data *ffs)
 {
 	struct ffs_epfile *epfile, *epfiles;
-	unsigned i, count;
+	unsigned int i, count, epfiles_count;
 	int err;
 
 	count = ffs->eps_count;
-	epfiles = kzalloc_objs(*epfiles, count);
+	epfiles_count = count;
+	if (ffs->user_flags & FUNCTIONFS_RW_PROXY_EPS)
+		epfiles_count += count / 2;
+	ffs->epfiles_count = epfiles_count;
+
+	epfiles = kzalloc_objs(*epfiles, epfiles_count);
 	if (!epfiles)
 		return -ENOMEM;
 
@@ -2375,6 +2426,32 @@ static int ffs_epfiles_create(struct ffs_data *ffs)
 		}
 	}
 
+	if (ffs->user_flags & FUNCTIONFS_RW_PROXY_EPS) {
+		struct ffs_epfile *comp = epfiles + count;
+
+		for (i = 0; i < count; i += 2, ++comp) {
+			struct ffs_epfile *ep1 = &epfiles[i];
+			struct ffs_epfile *ep2 = &epfiles[i + 1];
+			bool ep1_in = ffs->eps_addrmap[i + 1] & USB_ENDPOINT_DIR_MASK;
+
+			comp->ffs = ffs;
+			comp->is_rw_proxy = true;
+			comp->epfile_in = ep1_in ? ep1 : ep2;
+			comp->epfile_out = ep1_in ? ep2 : ep1;
+			mutex_init(&comp->mutex);
+			mutex_init(&comp->dmabufs_mutex);
+			INIT_LIST_HEAD(&comp->dmabufs);
+			snprintf(comp->name, sizeof(comp->name), "%s_rw",
+				 epfiles[i].name);
+			err = ffs_sb_create_file(ffs->sb, comp->name,
+						 comp, &ffs_epfile_operations);
+			if (err) {
+				ffs_epfiles_destroy(ffs->sb, epfiles, count + (i / 2));
+				return err;
+			}
+		}
+	}
+
 	ffs->epfiles = epfiles;
 	return 0;
 }
@@ -2952,7 +3029,8 @@ static int __ffs_data_got_descs(struct ffs_data *ffs,
 			      FUNCTIONFS_VIRTUAL_ADDR |
 			      FUNCTIONFS_EVENTFD |
 			      FUNCTIONFS_ALL_CTRL_RECIP |
-			      FUNCTIONFS_CONFIG0_SETUP)) {
+			      FUNCTIONFS_CONFIG0_SETUP |
+			      FUNCTIONFS_RW_PROXY_EPS)) {
 			ret = -ENOSYS;
 			goto error;
 		}
@@ -3040,6 +3118,21 @@ static int __ffs_data_got_descs(struct ffs_data *ffs,
 		goto error;
 	}
 
+	if (ffs->user_flags & FUNCTIONFS_RW_PROXY_EPS) {
+		if (ffs->eps_count % 2) {
+			ret = -EINVAL;
+			goto error;
+		}
+
+		for (i = 1; i < ffs->eps_count; i += 2) {
+			if ((ffs->eps_addrmap[i] & USB_ENDPOINT_DIR_MASK) ==
+			    (ffs->eps_addrmap[i + 1] & USB_ENDPOINT_DIR_MASK)) {
+				ret = -EINVAL;
+				goto error;
+			}
+		}
+	}
+
 	ffs->raw_descs_data	= _data;
 	ffs->raw_descs		= raw_descs;
 	ffs->raw_descs_length	= data - raw_descs;
diff --git a/drivers/usb/gadget/function/f_midi.c b/drivers/usb/gadget/function/f_midi.c
index 4d9e4bd700d89..fba8cf787d6c1 100644
--- a/drivers/usb/gadget/function/f_midi.c
+++ b/drivers/usb/gadget/function/f_midi.c
@@ -1309,6 +1309,7 @@ static void f_midi_free(struct usb_function *f)
 	opts = container_of(f->fi, struct f_midi_opts, func_inst);
 	mutex_lock(&opts->lock);
 	if (!--midi->free_ref) {
+		cancel_work_sync(&midi->work);
 		kfree(midi->id);
 		kfifo_free(&midi->in_req_fifo);
 		kfree(midi);
diff --git a/drivers/usb/gadget/function/f_ncm.c b/drivers/usb/gadget/function/f_ncm.c
index c5bf8a448d641..64eabda2f5469 100644
--- a/drivers/usb/gadget/function/f_ncm.c
+++ b/drivers/usb/gadget/function/f_ncm.c
@@ -1189,6 +1189,10 @@ static int ncm_unwrap_ntb(struct gether *port,
 	frame_max = ncm_opts->max_segment_size;
 
 parse_ntb:
+	if (to_process < (int)opts->nth_size) {
+		INFO(port->func.config->cdev, "Packet too small for headers\n");
+		goto err;
+	}
 	tmp = (__le16 *)ntb_ptr;
 
 	/* dwSignature */
@@ -1209,8 +1213,12 @@ static int ncm_unwrap_ntb(struct gether *port,
 	tmp++; /* skip wSequence */
 
 	block_len = get_ncm(&tmp, opts->block_length);
+	if (block_len == 0)
+		block_len = to_process;
+
 	/* (d)wBlockLength */
-	if ((block_len < opts->nth_size + opts->ndp_size) || (block_len > ntb_max)) {
+	if ((block_len < opts->nth_size + opts->ndp_size) || (block_len > ntb_max) ||
+			(block_len > to_process)) {
 		INFO(port->func.config->cdev, "Bad block length: %#X\n", block_len);
 		goto err;
 	}
@@ -1273,7 +1281,7 @@ static int ncm_unwrap_ntb(struct gether *port,
 			index = index2;
 			/* wDatagramIndex[0] */
 			if ((index < opts->nth_size) ||
-					(index > block_len - opts->dpe_size)) {
+					(index > block_len)) {
 				INFO(port->func.config->cdev,
 				     "Bad index: %#X\n", index);
 				goto err;
@@ -1285,7 +1293,8 @@ static int ncm_unwrap_ntb(struct gether *port,
 			 * ethernet hdr + crc or larger than max frame size
 			 */
 			if ((dg_len < 14 + crc_len) ||
-					(dg_len > frame_max)) {
+					(dg_len > frame_max) ||
+					(dg_len > block_len - index)) {
 				INFO(port->func.config->cdev,
 				     "Bad dgram length: %#X\n", dg_len);
 				goto err;
@@ -1310,7 +1319,7 @@ static int ncm_unwrap_ntb(struct gether *port,
 			dg_len2 = get_ncm(&tmp, opts->dgram_item_len);
 
 			/* wDatagramIndex[1] */
-			if (index2 > block_len - opts->dpe_size) {
+			if (index2 > block_len) {
 				INFO(port->func.config->cdev,
 				     "Bad index: %#X\n", index2);
 				goto err;
diff --git a/drivers/usb/gadget/function/f_printer.c b/drivers/usb/gadget/function/f_printer.c
index 837f753d0cae5..1857d786110b4 100644
--- a/drivers/usb/gadget/function/f_printer.c
+++ b/drivers/usb/gadget/function/f_printer.c
@@ -431,7 +431,7 @@ printer_read(struct file *fd, char __user *buf, size_t len, loff_t *ptr)
 {
 	struct printer_dev		*dev = fd->private_data;
 	unsigned long			flags;
-	size_t				size;
+	size_t				size, not_copied, copied;
 	size_t				bytes_copied;
 	struct usb_request		*req;
 	/* This is a pointer to the current USB rx request. */
@@ -524,10 +524,12 @@ printer_read(struct file *fd, char __user *buf, size_t len, loff_t *ptr)
 		else
 			size = len;
 
-		size -= copy_to_user(buf, current_rx_buf, size);
-		bytes_copied += size;
-		len -= size;
-		buf += size;
+		not_copied = copy_to_user(buf, current_rx_buf, size);
+		copied = size - not_copied;
+
+		bytes_copied += copied;
+		len -= copied;
+		buf += copied;
 
 		spin_lock_irqsave(&dev->lock, flags);
 
@@ -542,6 +544,17 @@ printer_read(struct file *fd, char __user *buf, size_t len, loff_t *ptr)
 		if (dev->interface < 0)
 			goto out_disabled;
 
+		if (!copied) {
+			dev->current_rx_req = current_rx_req;
+			dev->current_rx_bytes = current_rx_bytes;
+			dev->current_rx_buf = current_rx_buf;
+			spin_unlock_irqrestore(&dev->lock, flags);
+			mutex_unlock(&dev->lock_printer_io);
+			return bytes_copied ? bytes_copied : -EFAULT;
+		}
+
+		size = copied;
+
 		/* If we not returning all the data left in this RX request
 		 * buffer then adjust the amount of data left in the buffer.
 		 * Othewise if we are done with this RX request buffer then
diff --git a/drivers/usb/gadget/function/f_tcm.c b/drivers/usb/gadget/function/f_tcm.c
index 34d9f49e99874..b3fa5a17fd2db 100644
--- a/drivers/usb/gadget/function/f_tcm.c
+++ b/drivers/usb/gadget/function/f_tcm.c
@@ -2363,31 +2363,158 @@ static int tcm_bind(struct usb_configuration *c, struct usb_function *f)
 	return -ENOTSUPP;
 }
 
-struct guas_setup_wq {
-	struct work_struct work;
-	struct f_uas *fu;
-	unsigned int alt;
-};
+static void tcm_cleanup_old_alt(struct f_uas *fu)
+{
+	if (fu->flags & USBG_IS_UAS)
+		uasp_cleanup_old_alt(fu);
+	else if (fu->flags & USBG_IS_BOT)
+		bot_cleanup_old_alt(fu);
+	fu->flags = 0;
+}
+
+static void tcm_delayed_set_alt_done(struct f_uas *fu)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&fu->delayed_set_alt_lock, flags);
+	fu->delayed_set_alt_state = USBG_DELAYED_SET_ALT_IDLE;
+	fu->delayed_set_alt_cancel = false;
+	spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags);
+}
+
+static bool tcm_delayed_set_alt_cancelled(struct f_uas *fu)
+{
+	bool cancelled;
+	unsigned long flags;
+
+	spin_lock_irqsave(&fu->delayed_set_alt_lock, flags);
+	cancelled = fu->delayed_set_alt_cancel;
+	spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags);
+
+	return cancelled;
+}
+
+static bool tcm_complete_delayed_status(struct f_uas *fu)
+{
+	struct usb_composite_dev *cdev = fu->function.config->cdev;
+	struct usb_request *req = cdev->req;
+	unsigned long cdev_flags;
+	bool cancelled;
+	int ret;
+
+	spin_lock_irqsave(&cdev->lock, cdev_flags);
+	spin_lock(&fu->delayed_set_alt_lock);
+	cancelled = fu->delayed_set_alt_cancel;
+	if (!cancelled) {
+		fu->delayed_set_alt_state = USBG_DELAYED_SET_ALT_IDLE;
+		fu->delayed_set_alt_cancel = false;
+	}
+	spin_unlock(&fu->delayed_set_alt_lock);
+
+	if (cancelled) {
+		spin_unlock_irqrestore(&cdev->lock, cdev_flags);
+		return false;
+	}
+
+	if (cdev->delayed_status == 0) {
+		WARN(cdev, "%s: Unexpected call\n", __func__);
+	} else if (--cdev->delayed_status == 0) {
+		req->length = 0;
+		req->context = cdev;
+		ret = usb_ep_queue(cdev->gadget->ep0, req, GFP_ATOMIC);
+		if (ret == 0) {
+			cdev->setup_pending = true;
+		} else {
+			req->status = 0;
+			req->complete(cdev->gadget->ep0, req);
+		}
+	}
+
+	spin_unlock_irqrestore(&cdev->lock, cdev_flags);
+
+	return true;
+}
+
+static bool tcm_cancel_delayed_set_alt(struct f_uas *fu)
+{
+	bool cleanup = false;
+	bool cancel = false;
+	unsigned long flags;
+
+	spin_lock_irqsave(&fu->delayed_set_alt_lock, flags);
+	switch (fu->delayed_set_alt_state) {
+	case USBG_DELAYED_SET_ALT_IDLE:
+		cleanup = true;
+		break;
+	case USBG_DELAYED_SET_ALT_QUEUED:
+	case USBG_DELAYED_SET_ALT_RUNNING:
+		fu->delayed_set_alt_cancel = true;
+		cancel = true;
+		break;
+	}
+	spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags);
+
+	if (cancel && cancel_work(&fu->delayed_set_alt)) {
+		spin_lock_irqsave(&fu->delayed_set_alt_lock, flags);
+		if (fu->delayed_set_alt_state == USBG_DELAYED_SET_ALT_QUEUED) {
+			fu->delayed_set_alt_state = USBG_DELAYED_SET_ALT_IDLE;
+			fu->delayed_set_alt_cancel = false;
+			cleanup = true;
+		}
+		spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags);
+	}
+
+	return cleanup;
+}
+
+static void tcm_cancel_delayed_set_alt_sync(struct f_uas *fu)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&fu->delayed_set_alt_lock, flags);
+	if (fu->delayed_set_alt_state != USBG_DELAYED_SET_ALT_IDLE)
+		fu->delayed_set_alt_cancel = true;
+	spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags);
+
+	cancel_work_sync(&fu->delayed_set_alt);
+
+	spin_lock_irqsave(&fu->delayed_set_alt_lock, flags);
+	fu->delayed_set_alt_state = USBG_DELAYED_SET_ALT_IDLE;
+	fu->delayed_set_alt_cancel = false;
+	spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags);
+}
 
 static void tcm_delayed_set_alt(struct work_struct *wq)
 {
-	struct guas_setup_wq *work = container_of(wq, struct guas_setup_wq,
-			work);
-	struct f_uas *fu = work->fu;
-	int alt = work->alt;
+	struct f_uas *fu = container_of(wq, struct f_uas, delayed_set_alt);
+	unsigned long flags;
+	unsigned int alt;
 
-	kfree(work);
+	spin_lock_irqsave(&fu->delayed_set_alt_lock, flags);
+	if (fu->delayed_set_alt_state != USBG_DELAYED_SET_ALT_QUEUED) {
+		spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags);
+		return;
+	}
+	fu->delayed_set_alt_state = USBG_DELAYED_SET_ALT_RUNNING;
+	alt = fu->delayed_alt;
+	spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags);
 
-	if (fu->flags & USBG_IS_BOT)
-		bot_cleanup_old_alt(fu);
-	if (fu->flags & USBG_IS_UAS)
-		uasp_cleanup_old_alt(fu);
+	tcm_cleanup_old_alt(fu);
+
+	if (tcm_delayed_set_alt_cancelled(fu))
+		goto out_done;
 
 	if (alt == USB_G_ALT_INT_BBB)
 		bot_set_alt(fu);
 	else if (alt == USB_G_ALT_INT_UAS)
 		uasp_set_alt(fu);
-	usb_composite_setup_continue(fu->function.config->cdev);
+
+	if (tcm_complete_delayed_status(fu))
+		return;
+
+	tcm_cleanup_old_alt(fu);
+out_done:
+	tcm_delayed_set_alt_done(fu);
 }
 
 static int tcm_get_alt(struct usb_function *f, unsigned intf)
@@ -2413,15 +2540,20 @@ static int tcm_set_alt(struct usb_function *f, unsigned intf, unsigned alt)
 		return -EOPNOTSUPP;
 
 	if ((alt == USB_G_ALT_INT_BBB) || (alt == USB_G_ALT_INT_UAS)) {
-		struct guas_setup_wq *work;
+		unsigned long flags;
 
-		work = kmalloc_obj(*work, GFP_ATOMIC);
-		if (!work)
-			return -ENOMEM;
-		INIT_WORK(&work->work, tcm_delayed_set_alt);
-		work->fu = fu;
-		work->alt = alt;
-		schedule_work(&work->work);
+		spin_lock_irqsave(&fu->delayed_set_alt_lock, flags);
+		if (fu->delayed_set_alt_state != USBG_DELAYED_SET_ALT_IDLE) {
+			spin_unlock_irqrestore(&fu->delayed_set_alt_lock,
+					       flags);
+			return -EBUSY;
+		}
+		fu->delayed_alt = alt;
+		fu->delayed_set_alt_cancel = false;
+		fu->delayed_set_alt_state = USBG_DELAYED_SET_ALT_QUEUED;
+		spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags);
+
+		schedule_work(&fu->delayed_set_alt);
 		return USB_GADGET_DELAYED_STATUS;
 	}
 	return -EOPNOTSUPP;
@@ -2431,11 +2563,8 @@ static void tcm_disable(struct usb_function *f)
 {
 	struct f_uas *fu = to_f_uas(f);
 
-	if (fu->flags & USBG_IS_UAS)
-		uasp_cleanup_old_alt(fu);
-	else if (fu->flags & USBG_IS_BOT)
-		bot_cleanup_old_alt(fu);
-	fu->flags = 0;
+	if (tcm_cancel_delayed_set_alt(fu))
+		tcm_cleanup_old_alt(fu);
 }
 
 static int tcm_setup(struct usb_function *f,
@@ -2583,11 +2712,16 @@ static void tcm_free(struct usb_function *f)
 {
 	struct f_uas *tcm = to_f_uas(f);
 
+	tcm_cancel_delayed_set_alt_sync(tcm);
 	kfree(tcm);
 }
 
 static void tcm_unbind(struct usb_configuration *c, struct usb_function *f)
 {
+	struct f_uas *fu = to_f_uas(f);
+
+	tcm_cancel_delayed_set_alt_sync(fu);
+	tcm_cleanup_old_alt(fu);
 	usb_free_all_descriptors(f);
 }
 
@@ -2620,6 +2754,8 @@ static struct usb_function *tcm_alloc(struct usb_function_instance *fi)
 	fu->function.disable = tcm_disable;
 	fu->function.free_func = tcm_free;
 	fu->tpg = tpg_instances[i].tpg;
+	INIT_WORK(&fu->delayed_set_alt, tcm_delayed_set_alt);
+	spin_lock_init(&fu->delayed_set_alt_lock);
 
 	hash_init(fu->stream_hash);
 	mutex_unlock(&tpg_instances_lock);
diff --git a/drivers/usb/gadget/function/f_uac1.c b/drivers/usb/gadget/function/f_uac1.c
index 85c502e98f577..7a81cd176abd3 100644
--- a/drivers/usb/gadget/function/f_uac1.c
+++ b/drivers/usb/gadget/function/f_uac1.c
@@ -1594,7 +1594,8 @@ static ssize_t f_uac1_opts_##name##_store(struct config_item *item,	\
 					  const char *page, size_t len)	\
 {									\
 	struct f_uac1_opts *opts = to_f_uac1_opts(item);		\
-	char *split_page = NULL;					\
+	char *buf = NULL;						\
+	char *split_page;						\
 	int ret = -EINVAL;						\
 	char *token;							\
 	u32 num;							\
@@ -1608,18 +1609,22 @@ static ssize_t f_uac1_opts_##name##_store(struct config_item *item,	\
 									\
 	i = 0;								\
 	memset(opts->name##s, 0x00, sizeof(opts->name##s));		\
-	split_page = kstrdup(page, GFP_KERNEL);				\
+	buf = kstrdup(page, GFP_KERNEL);				\
+	split_page = buf;						\
 	while ((token = strsep(&split_page, ",")) != NULL) {		\
 		ret = kstrtou32(token, 0, &num);			\
 		if (ret)						\
 			goto end;					\
-									\
+		if (i >= UAC_MAX_RATES) {				\
+			ret = -EINVAL;					\
+			goto end;					\
+		}							\
 		opts->name##s[i++] = num;				\
 		ret = len;						\
 	};								\
 									\
 end:									\
-	kfree(split_page);						\
+	kfree(buf);							\
 	mutex_unlock(&opts->lock);					\
 	return ret;							\
 }									\
diff --git a/drivers/usb/gadget/function/f_uac2.c b/drivers/usb/gadget/function/f_uac2.c
index 897787d0803c1..d8cf710085a05 100644
--- a/drivers/usb/gadget/function/f_uac2.c
+++ b/drivers/usb/gadget/function/f_uac2.c
@@ -2012,7 +2012,8 @@ static ssize_t f_uac2_opts_##name##_store(struct config_item *item,	\
 					  const char *page, size_t len)	\
 {									\
 	struct f_uac2_opts *opts = to_f_uac2_opts(item);		\
-	char *split_page = NULL;					\
+	char *buf = NULL;						\
+	char *split_page;						\
 	int ret = -EINVAL;						\
 	char *token;							\
 	u32 num;							\
@@ -2026,18 +2027,22 @@ static ssize_t f_uac2_opts_##name##_store(struct config_item *item,	\
 									\
 	i = 0;								\
 	memset(opts->name##s, 0x00, sizeof(opts->name##s));		\
-	split_page = kstrdup(page, GFP_KERNEL);				\
+	buf = kstrdup(page, GFP_KERNEL);				\
+	split_page = buf;						\
 	while ((token = strsep(&split_page, ",")) != NULL) {		\
 		ret = kstrtou32(token, 0, &num);			\
 		if (ret)						\
 			goto end;					\
-									\
+		if (i >= UAC_MAX_RATES) {				\
+			ret = -EINVAL;					\
+			goto end;					\
+		}							\
 		opts->name##s[i++] = num;				\
 		ret = len;						\
 	};								\
 									\
 end:									\
-	kfree(split_page);						\
+	kfree(buf);							\
 	mutex_unlock(&opts->lock);					\
 	return ret;							\
 }									\
diff --git a/drivers/usb/gadget/function/tcm.h b/drivers/usb/gadget/function/tcm.h
index 009974d81d66b..e1d5a93916127 100644
--- a/drivers/usb/gadget/function/tcm.h
+++ b/drivers/usb/gadget/function/tcm.h
@@ -3,6 +3,7 @@
 #define __TARGET_USB_GADGET_H__
 
 #include <linux/kref.h>
+#include <linux/spinlock.h>
 /* #include <linux/usb/uas.h> */
 #include <linux/hashtable.h>
 #include <linux/usb/composite.h>
@@ -29,6 +30,12 @@ enum {
 
 #define USB_G_DEFAULT_SESSION_TAGS	USBG_NUM_CMDS
 
+enum {
+	USBG_DELAYED_SET_ALT_IDLE = 0,
+	USBG_DELAYED_SET_ALT_QUEUED,
+	USBG_DELAYED_SET_ALT_RUNNING,
+};
+
 struct tcm_usbg_nexus {
 	struct se_session *tvn_se_sess;
 };
@@ -132,6 +139,12 @@ struct f_uas {
 #define USBG_BOT_CMD_PEND	(1 << 4)
 #define USBG_BOT_WEDGED		(1 << 5)
 
+	struct work_struct	delayed_set_alt;
+	spinlock_t		delayed_set_alt_lock; /* protects delayed_set_alt_* */
+	unsigned int		delayed_alt;
+	unsigned int		delayed_set_alt_state;
+	bool			delayed_set_alt_cancel;
+
 	struct usbg_cdb		cmd[USBG_NUM_CMDS];
 	struct usb_ep		*ep_in;
 	struct usb_ep		*ep_out;
diff --git a/drivers/usb/gadget/function/u_audio.c b/drivers/usb/gadget/function/u_audio.c
index e53f2927b5398..ca26bf9c80404 100644
--- a/drivers/usb/gadget/function/u_audio.c
+++ b/drivers/usb/gadget/function/u_audio.c
@@ -641,15 +641,15 @@ int u_audio_start_capture(struct g_audio *audio_dev)
 	ret = config_ep_by_speed(gadget, &audio_dev->func, ep_fback);
 	if (ret < 0) {
 		dev_err(dev, "config_ep_by_speed in_ep_fback failed (%d)\n", ret);
-		return ret; // TODO: Clean up out_ep
+		goto err_out_ep;
 	}
 
-	prm->fb_ep_enabled = true;
 	ret = usb_ep_enable(ep_fback);
 	if (ret < 0) {
 		dev_err(dev, "usb_ep_enable failed for in_ep_fback (%d)\n", ret);
-		return ret; // TODO: Clean up out_ep
+		goto err_out_ep;
 	}
+	prm->fb_ep_enabled = true;
 	req_len = ep_fback->maxpacket;
 
 	req_fback = usb_ep_alloc_request(ep_fback, GFP_ATOMIC);
@@ -680,6 +680,12 @@ int u_audio_start_capture(struct g_audio *audio_dev)
 		dev_err(dev, "%s:%d Error!\n", __func__, __LINE__);
 
 	return 0;
+
+err_out_ep:
+	set_active(prm, false);
+	free_ep(prm, ep);
+
+	return ret;
 }
 EXPORT_SYMBOL_GPL(u_audio_start_capture);
 
diff --git a/drivers/usb/gadget/function/u_fs.h b/drivers/usb/gadget/function/u_fs.h
index 6a80182aadd70..c280c495fbd21 100644
--- a/drivers/usb/gadget/function/u_fs.h
+++ b/drivers/usb/gadget/function/u_fs.h
@@ -252,8 +252,14 @@ struct ffs_data {
 
 	unsigned short			strings_count;
 	unsigned short			interfaces_count;
+
+	/*
+	 * eps_count tracks the number of underlying hardware endpoints.
+	 * epfiles_count tracks the total number of VFS endpoint files.
+	 * When companion endpoints are active, epfiles_count > eps_count.
+	 */
 	unsigned short			eps_count;
-	unsigned short			_pad1;
+	unsigned short			epfiles_count;
 
 	/* filled by __ffs_data_got_strings() */
 	/* ids in stringtabs are set in functionfs_bind() */
diff --git a/drivers/usb/gadget/function/uvc_v4l2.c b/drivers/usb/gadget/function/uvc_v4l2.c
index 514e5930b9ca9..dfa0521a243ac 100644
--- a/drivers/usb/gadget/function/uvc_v4l2.c
+++ b/drivers/usb/gadget/function/uvc_v4l2.c
@@ -200,6 +200,8 @@ uvc_send_response(struct uvc_device *uvc, struct uvc_request_data *data)
 		return usb_ep_set_halt(cdev->gadget->ep0);
 
 	req->length = min_t(unsigned int, uvc->event_length, data->length);
+	if (req->length > sizeof(data->data))
+		req->length = sizeof(data->data);
 	req->zero = data->length < uvc->event_length;
 
 	memcpy(req->buf, data->data, req->length);
diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c
index d87a8ab515107..db961aaa3740d 100644
--- a/drivers/usb/gadget/legacy/inode.c
+++ b/drivers/usb/gadget/legacy/inode.c
@@ -471,11 +471,17 @@ static void ep_user_copy_worker(struct work_struct *work)
 	struct kiocb *iocb = priv->iocb;
 	size_t ret;
 
-	kthread_use_mm(mm);
-	ret = copy_to_iter(priv->buf, priv->actual, &priv->to);
-	kthread_unuse_mm(mm);
-	if (!ret)
+	if (mmget_not_zero(mm)) {
+		kthread_use_mm(mm);
+		ret = copy_to_iter(priv->buf, priv->actual, &priv->to);
+		kthread_unuse_mm(mm);
+		mmput(mm);
+		if (!ret)
+			ret = -EFAULT;
+	} else {
 		ret = -EFAULT;
+	}
+	mmdrop(mm);
 
 	/* completing the iocb can drop the ctx and mm, don't touch mm after */
 	iocb->ki_complete(iocb, ret);
@@ -501,6 +507,7 @@ static void ep_aio_complete(struct usb_ep *ep, struct usb_request *req)
 	 * complete the aio request immediately.
 	 */
 	if (priv->to_free == NULL || unlikely(req->actual == 0)) {
+		mmdrop(priv->mm);
 		kfree(req->buf);
 		kfree(priv->to_free);
 		kfree(priv);
@@ -541,6 +548,7 @@ static ssize_t ep_aio(struct kiocb *iocb,
 	priv->epdata = epdata;
 	priv->actual = 0;
 	priv->mm = current->mm; /* mm teardown waits for iocbs in exit_aio() */
+	mmgrab(priv->mm);
 
 	/* each kiocb is coupled to one usb_request, but we can't
 	 * allocate or submit those if the host disconnected.
@@ -570,6 +578,7 @@ static ssize_t ep_aio(struct kiocb *iocb,
 
 fail:
 	spin_unlock_irq(&epdata->dev->lock);
+	mmdrop(priv->mm);
 	kfree(priv->to_free);
 	kfree(priv);
 	put_ep(epdata);
diff --git a/drivers/usb/gadget/legacy/nokia.c b/drivers/usb/gadget/legacy/nokia.c
index 2e15f9a32ce9f..396605a84cd1a 100644
--- a/drivers/usb/gadget/legacy/nokia.c
+++ b/drivers/usb/gadget/legacy/nokia.c
@@ -50,7 +50,7 @@ static unsigned int fsg_num_buffers = CONFIG_USB_GADGET_STORAGE_NUM_BUFFERS;
  */
 #define fsg_num_buffers	CONFIG_USB_GADGET_STORAGE_NUM_BUFFERS
 
-#endif /* CONFIG_USB_DEBUG */
+#endif /* CONFIG_USB_GADGET_DEBUG_FILES */
 
 FSG_MODULE_PARAMETERS(/* no prefix */, fsg_mod_data);
 
diff --git a/drivers/usb/gadget/udc/Kconfig b/drivers/usb/gadget/udc/Kconfig
index 26460340fbc97..7ddbc2606686a 100644
--- a/drivers/usb/gadget/udc/Kconfig
+++ b/drivers/usb/gadget/udc/Kconfig
@@ -90,7 +90,7 @@ config USB_BCM63XX_UDC
 
 config USB_FSL_USB2
 	tristate "Freescale Highspeed USB DR Peripheral Controller"
-	depends on FSL_SOC
+	depends on FSL_SOC || COMPILE_TEST
 	help
 	   Some of Freescale PowerPC and i.MX processors have a High Speed
 	   Dual-Role(DR) USB controller, which supports device mode.
diff --git a/drivers/usb/gadget/udc/aspeed_udc.c b/drivers/usb/gadget/udc/aspeed_udc.c
index 75f9c831b21a6..54f81e6680094 100644
--- a/drivers/usb/gadget/udc/aspeed_udc.c
+++ b/drivers/usb/gadget/udc/aspeed_udc.c
@@ -1431,25 +1431,12 @@ static void ast_udc_init_hw(struct ast_udc_dev *udc)
 	ast_udc_write(udc, 0, AST_UDC_EP0_CTRL);
 }
 
-static void ast_udc_remove(struct platform_device *pdev)
+static void ast_udc_cleanup(struct platform_device *pdev)
 {
 	struct ast_udc_dev *udc = platform_get_drvdata(pdev);
 	unsigned long flags;
 	u32 ctrl;
 
-	usb_del_gadget_udc(&udc->gadget);
-	if (udc->driver) {
-		/*
-		 * This is broken as only some cleanup is skipped, *udev is
-		 * freed and the register mapping goes away. Any further usage
-		 * probably crashes. Also the device is unbound, so the skipped
-		 * cleanup is never catched up later.
-		 */
-		dev_alert(&pdev->dev,
-			  "Driver is busy and still going away. Fasten your seat belts!\n");
-		return;
-	}
-
 	spin_lock_irqsave(&udc->lock, flags);
 
 	/* Disable upstream port connection */
@@ -1469,6 +1456,26 @@ static void ast_udc_remove(struct platform_device *pdev)
 	udc->ep0_buf = NULL;
 }
 
+static void ast_udc_remove(struct platform_device *pdev)
+{
+	struct ast_udc_dev *udc = platform_get_drvdata(pdev);
+
+	usb_del_gadget_udc(&udc->gadget);
+	if (udc->driver) {
+		/*
+		 * This is broken as only some cleanup is skipped, *udev is
+		 * freed and the register mapping goes away. Any further usage
+		 * probably crashes. Also the device is unbound, so the skipped
+		 * cleanup is never catched up later.
+		 */
+		dev_alert(&pdev->dev,
+			  "Driver is busy and still going away. Fasten your seat belts!\n");
+		return;
+	}
+
+	ast_udc_cleanup(pdev);
+}
+
 static int ast_udc_probe(struct platform_device *pdev)
 {
 	enum usb_device_speed max_speed;
@@ -1521,6 +1528,12 @@ static int ast_udc_probe(struct platform_device *pdev)
 					  AST_UDC_NUM_ENDPOINTS,
 					  &udc->ep0_buf_dma, GFP_KERNEL);
 
+	if (!udc->ep0_buf) {
+		clk_disable_unprepare(udc->clk);
+		rc = -ENOMEM;
+		goto err;
+	}
+
 	udc->gadget.speed = USB_SPEED_UNKNOWN;
 	udc->gadget.max_speed = USB_SPEED_HIGH;
 	udc->creq = udc->reg + AST_UDC_SETUP0;
@@ -1550,20 +1563,20 @@ static int ast_udc_probe(struct platform_device *pdev)
 	udc->irq = platform_get_irq(pdev, 0);
 	if (udc->irq < 0) {
 		rc = udc->irq;
-		goto err;
+		goto err_cleanup;
 	}
 
 	rc = devm_request_irq(&pdev->dev, udc->irq, ast_udc_isr, 0,
 			      KBUILD_MODNAME, udc);
 	if (rc) {
 		dev_err(&pdev->dev, "Failed to request interrupt\n");
-		goto err;
+		goto err_cleanup;
 	}
 
 	rc = usb_add_gadget_udc(&pdev->dev, &udc->gadget);
 	if (rc) {
 		dev_err(&pdev->dev, "Failed to add gadget udc\n");
-		goto err;
+		goto err_cleanup;
 	}
 
 	dev_info(&pdev->dev, "Initialized udc in USB%s mode\n",
@@ -1571,9 +1584,10 @@ static int ast_udc_probe(struct platform_device *pdev)
 
 	return 0;
 
+err_cleanup:
+	ast_udc_cleanup(pdev);
 err:
 	dev_err(&pdev->dev, "Failed to udc probe, rc:0x%x\n", rc);
-	ast_udc_remove(pdev);
 
 	return rc;
 }
diff --git a/drivers/usb/gadget/udc/bdc/bdc_core.c b/drivers/usb/gadget/udc/bdc/bdc_core.c
index 438201dc96ca7..a8dbaef54abae 100644
--- a/drivers/usb/gadget/udc/bdc/bdc_core.c
+++ b/drivers/usb/gadget/udc/bdc/bdc_core.c
@@ -585,9 +585,29 @@ static int bdc_probe(struct platform_device *pdev)
 static void bdc_remove(struct platform_device *pdev)
 {
 	struct bdc *bdc;
+	unsigned long flags;
+	u32 temp;
 
 	bdc  = platform_get_drvdata(pdev);
 	dev_dbg(bdc->dev, "%s ()\n", __func__);
+	/*
+	 * Disable the device interrupt source before freeing the IRQ:
+	 * clear BDC_GIE so the controller stops asserting interrupts,
+	 * then free_irq drains any in-flight handler.
+	 */
+	spin_lock_irqsave(&bdc->lock, flags);
+	temp = bdc_readl(bdc->regs, BDC_BDCSC);
+	temp &= ~BDC_GIE;
+	bdc_writel(bdc->regs, BDC_BDCSC, temp);
+	spin_unlock_irqrestore(&bdc->lock, flags);
+	free_irq(bdc->irq, bdc);
+	/*
+	 * Drain func_wake_notify after free_irq: the IRQ handler arms this
+	 * delayed_work via bdc_sr_uspc -> handle_link_state_change ->
+	 * schedule_delayed_work (self-rearmed in bdc_func_wake_timer), so
+	 * the IRQ must be released first to prevent re-arm after cancel.
+	 */
+	cancel_delayed_work_sync(&bdc->func_wake_notify);
 	bdc_udc_exit(bdc);
 	bdc_hw_exit(bdc);
 	bdc_phy_exit(bdc);
diff --git a/drivers/usb/gadget/udc/bdc/bdc_udc.c b/drivers/usb/gadget/udc/bdc/bdc_udc.c
index 23826fd7a8e69..7a12219edac6f 100644
--- a/drivers/usb/gadget/udc/bdc/bdc_udc.c
+++ b/drivers/usb/gadget/udc/bdc/bdc_udc.c
@@ -530,8 +530,8 @@ int bdc_udc_init(struct bdc *bdc)
 
 
 	bdc->gadget.name = BRCM_BDC_NAME;
-	ret = devm_request_irq(bdc->dev, bdc->irq, bdc_udc_interrupt,
-				IRQF_SHARED, BRCM_BDC_NAME, bdc);
+	ret = request_irq(bdc->irq, bdc_udc_interrupt, IRQF_SHARED,
+			  BRCM_BDC_NAME, bdc);
 	if (ret) {
 		dev_err(bdc->dev,
 			"failed to request irq #%d %d\n",
@@ -542,7 +542,7 @@ int bdc_udc_init(struct bdc *bdc)
 	ret = bdc_init_ep(bdc);
 	if (ret) {
 		dev_err(bdc->dev, "bdc init ep fail: %d\n", ret);
-		return ret;
+		goto err0;
 	}
 
 	ret = usb_add_gadget_udc(bdc->dev, &bdc->gadget);
@@ -571,6 +571,7 @@ int bdc_udc_init(struct bdc *bdc)
 err1:
 	usb_del_gadget_udc(&bdc->gadget);
 err0:
+	free_irq(bdc->irq, bdc);
 	bdc_free_ep(bdc);
 
 	return ret;
diff --git a/drivers/usb/gadget/udc/dummy_hcd.c b/drivers/usb/gadget/udc/dummy_hcd.c
index f47903461ed5c..c0e40fa6dde58 100644
--- a/drivers/usb/gadget/udc/dummy_hcd.c
+++ b/drivers/usb/gadget/udc/dummy_hcd.c
@@ -278,6 +278,7 @@ struct dummy {
 	unsigned			ints_enabled:1;
 	unsigned			udc_suspended:1;
 	unsigned			pullup:1;
+	unsigned			fifo_req_busy:1;
 
 	/*
 	 * HOST side support
@@ -329,6 +330,26 @@ static inline struct dummy *gadget_dev_to_dummy(struct device *dev)
 
 /* DEVICE/GADGET SIDE UTILITY ROUTINES */
 
+/*
+ * Give back a gadget request with dum->lock dropped around the callback.
+ * If @req is the shared fifo_req, clear fifo_req_busy afterward: the flag
+ * was set in dummy_queue() when the shared request was taken and must stay
+ * set until its completion callback has returned; list_del_init() alone
+ * makes the request look idle while the callback is still running.
+ * Caller holds dum->lock and has already done list_del_init() + status.
+ */
+static void dummy_giveback(struct dummy *dum, struct usb_ep *_ep,
+			   struct dummy_request *req)
+{
+	bool fifo = req == &dum->fifo_req;
+
+	spin_unlock(&dum->lock);
+	usb_gadget_giveback_request(_ep, &req->req);
+	spin_lock(&dum->lock);
+	if (fifo)
+		dum->fifo_req_busy = 0;
+}
+
 /* called with spinlock held */
 static void nuke(struct dummy *dum, struct dummy_ep *ep)
 {
@@ -339,9 +360,7 @@ static void nuke(struct dummy *dum, struct dummy_ep *ep)
 		list_del_init(&req->queue);
 		req->req.status = -ESHUTDOWN;
 
-		spin_unlock(&dum->lock);
-		usb_gadget_giveback_request(&ep->ep, &req->req);
-		spin_lock(&dum->lock);
+		dummy_giveback(dum, &ep->ep, req);
 	}
 }
 
@@ -728,10 +747,11 @@ static int dummy_queue(struct usb_ep *_ep, struct usb_request *_req,
 
 	/* implement an emulated single-request FIFO */
 	if (ep->desc && (ep->desc->bEndpointAddress & USB_DIR_IN) &&
-			list_empty(&dum->fifo_req.queue) &&
+			!dum->fifo_req_busy &&
 			list_empty(&ep->queue) &&
 			_req->length <= FIFO_SIZE) {
 		req = &dum->fifo_req;
+		dum->fifo_req_busy = 1;
 		req->req = *_req;
 		req->req.buf = dum->fifo_buf;
 		memcpy(dum->fifo_buf, _req->buf, _req->length);
@@ -785,9 +805,7 @@ static int dummy_dequeue(struct usb_ep *_ep, struct usb_request *_req)
 		dev_dbg(udc_dev(dum),
 				"dequeued req %p from %s, len %d buf %p\n",
 				req, _ep->name, _req->length, _req->buf);
-		spin_unlock(&dum->lock);
-		usb_gadget_giveback_request(_ep, _req);
-		spin_lock(&dum->lock);
+		dummy_giveback(dum, _ep, req);
 	}
 	spin_unlock_irqrestore(&dum->lock, flags);
 	return retval;
@@ -1523,9 +1541,7 @@ static int transfer(struct dummy_hcd *dum_hcd, struct urb *urb,
 		if (req->req.status != -EINPROGRESS) {
 			list_del_init(&req->queue);
 
-			spin_unlock(&dum->lock);
-			usb_gadget_giveback_request(&ep->ep, &req->req);
-			spin_lock(&dum->lock);
+			dummy_giveback(dum, &ep->ep, req);
 
 			/* requests might have been unlinked... */
 			rescan = 1;
@@ -1910,9 +1926,7 @@ static enum hrtimer_restart dummy_timer(struct hrtimer *t)
 				dev_dbg(udc_dev(dum), "stale req = %p\n",
 						req);
 
-				spin_unlock(&dum->lock);
-				usb_gadget_giveback_request(&ep->ep, &req->req);
-				spin_lock(&dum->lock);
+				dummy_giveback(dum, &ep->ep, req);
 				ep->already_seen = 0;
 				goto restart;
 			}
diff --git a/drivers/usb/gadget/udc/fsl_qe_udc.c b/drivers/usb/gadget/udc/fsl_qe_udc.c
index bf87285ad13ca..603c77ff129fd 100644
--- a/drivers/usb/gadget/udc/fsl_qe_udc.c
+++ b/drivers/usb/gadget/udc/fsl_qe_udc.c
@@ -1945,6 +1945,7 @@ static void ch9getstatus(struct qe_udc *udc, u8 request_type, u16 value,
 			u16 index, u16 length)
 {
 	u16 usb_status = 0;
+	struct usb_request *usb_req;
 	struct qe_req *req;
 	struct qe_ep *ep;
 	int status = 0;
@@ -1983,8 +1984,11 @@ static void ch9getstatus(struct qe_udc *udc, u8 request_type, u16 value,
 		}
 	}
 
-	req = container_of(qe_alloc_request(&ep->ep, GFP_KERNEL),
-					struct qe_req, req);
+	usb_req = qe_alloc_request(&ep->ep, GFP_KERNEL);
+	if (!usb_req)
+		goto stall;
+
+	req = container_of(usb_req, struct qe_req, req);
 	req->req.length = 2;
 	req->req.buf = udc->statusbuf;
 	*(u16 *)req->req.buf = cpu_to_le16(usb_status);
diff --git a/drivers/usb/gadget/udc/fsl_udc_core.c b/drivers/usb/gadget/udc/fsl_udc_core.c
index 600ce8cc0fefc..20392409afb15 100644
--- a/drivers/usb/gadget/udc/fsl_udc_core.c
+++ b/drivers/usb/gadget/udc/fsl_udc_core.c
@@ -184,7 +184,7 @@ __acquires(ep->udc->lock)
 	usb_gadget_unmap_request(&ep->udc->gadget, &req->req, ep_is_in(ep));
 
 	if (status && (status != -ESHUTDOWN))
-		dev_vdbg(&udc->gadget.dev, "complete %s req %p stat %d len %u/%u\n",
+		dev_vdbg(udc->dev, "complete %s req %p stat %d len %u/%u\n",
 			 ep->ep.name, &req->req, status,
 			 req->req.actual, req->req.length);
 
@@ -286,7 +286,7 @@ static int dr_controller_setup(struct fsl_udc *udc)
 	timeout = jiffies + FSL_UDC_RESET_TIMEOUT;
 	while (fsl_readl(&dr_regs->usbcmd) & USB_CMD_CTRL_RESET) {
 		if (time_after(jiffies, timeout)) {
-			dev_err(&udc->gadget.dev, "udc reset timeout!\n");
+			dev_err(udc->dev, "udc reset timeout!\n");
 			return -ETIMEDOUT;
 		}
 		cpu_relax();
@@ -309,7 +309,7 @@ static int dr_controller_setup(struct fsl_udc *udc)
 	tmp &= USB_EP_LIST_ADDRESS_MASK;
 	fsl_writel(tmp, &dr_regs->endpointlistaddr);
 
-	dev_vdbg(&udc->gadget.dev,
+	dev_vdbg(udc->dev,
 		 "vir[qh_base] is %p phy[qh_base] is 0x%8x reg is 0x%8x\n",
 		 udc->ep_qh, (int)tmp,
 		 fsl_readl(&dr_regs->endpointlistaddr));
@@ -500,7 +500,7 @@ static void struct_ep_qh_setup(struct fsl_udc *udc, unsigned char ep_num,
 		tmp = max_pkt_len << EP_QUEUE_HEAD_MAX_PKT_LEN_POS;
 		break;
 	default:
-		dev_vdbg(&udc->gadget.dev, "error ep type is %d\n", ep_type);
+		dev_vdbg(udc->dev, "error ep type is %d\n", ep_type);
 		return;
 	}
 	if (zlt)
@@ -613,7 +613,7 @@ static int fsl_ep_enable(struct usb_ep *_ep,
 	spin_unlock_irqrestore(&udc->lock, flags);
 	retval = 0;
 
-	dev_vdbg(&udc->gadget.dev, "enabled %s (ep%d%s) maxpacket %d\n",
+	dev_vdbg(udc->dev, "enabled %s (ep%d%s) maxpacket %d\n",
 		 ep->ep.name, ep->ep.desc->bEndpointAddress & 0x0f,
 		 (desc->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
 		 max);
@@ -634,13 +634,8 @@ static int fsl_ep_disable(struct usb_ep *_ep)
 	int ep_num;
 
 	ep = container_of(_ep, struct fsl_ep, ep);
-	if (!_ep || !ep->ep.desc) {
-		/*
-		 * dev_vdbg(&udc->gadget.dev, "%s not enabled\n",
-		 *	 _ep ? ep->ep.name : NULL);
-		 */
+	if (!_ep || !ep->ep.desc)
 		return -EINVAL;
-	}
 
 	/* disable ep on controller */
 	ep_num = ep_index(ep);
@@ -664,7 +659,7 @@ static int fsl_ep_disable(struct usb_ep *_ep)
 	ep->stopped = 1;
 	spin_unlock_irqrestore(&udc->lock, flags);
 
-	dev_vdbg(&udc->gadget.dev, "disabled %s OK\n", _ep->name);
+	dev_vdbg(udc->dev, "disabled %s OK\n", _ep->name);
 	return 0;
 }
 
@@ -724,9 +719,6 @@ static void fsl_queue_td(struct fsl_ep *ep, struct fsl_req *req)
 {
 	u32 temp, bitmask, tmp_stat;
 
-	/* dev_vdbg(&udc->gadget.dev, "QH addr Register 0x%8x\n", dr_regs->endpointlistaddr);
-	dev_vdbg(&udc->gadget.dev, "ep_qh[%d] addr is 0x%8x\n", i, (u32)&(ep->udc->ep_qh[i])); */
-
 	bitmask = ep_is_in(ep)
 		? (1 << (ep_index(ep) + 16))
 		: (1 << (ep_index(ep)));
@@ -813,7 +805,7 @@ static struct ep_td_struct *fsl_build_dtd(struct fsl_req *req, unsigned *length,
 		*is_last = 0;
 
 	if ((*is_last) == 0)
-		dev_vdbg(&udc_controller->gadget.dev, "multi-dtd request!\n");
+		dev_vdbg(udc_controller->dev, "multi-dtd request!\n");
 	/* Fill in the transfer size; set active bit */
 	swap_temp = ((*length << DTD_LENGTH_BIT_POS) | DTD_STATUS_ACTIVE);
 
@@ -825,7 +817,7 @@ static struct ep_td_struct *fsl_build_dtd(struct fsl_req *req, unsigned *length,
 
 	mb();
 
-	dev_vdbg(&udc_controller->gadget.dev, "length = %d address= 0x%x\n", *length, (int)*dma);
+	dev_vdbg(udc_controller->dev, "length = %d address= 0x%x\n", *length, (int)*dma);
 
 	return dtd;
 }
@@ -876,11 +868,11 @@ fsl_ep_queue(struct usb_ep *_ep, struct usb_request *_req, gfp_t gfp_flags)
 	/* catch various bogus parameters */
 	if (!_req || !req->req.complete || !req->req.buf
 			|| !list_empty(&req->queue)) {
-		dev_vdbg(&udc->gadget.dev, "%s, bad params\n", __func__);
+		dev_vdbg(udc->dev, "%s, bad params\n", __func__);
 		return -EINVAL;
 	}
 	if (unlikely(!ep->ep.desc)) {
-		dev_vdbg(&udc->gadget.dev, "%s, bad ep\n", __func__);
+		dev_vdbg(udc->dev, "%s, bad ep\n", __func__);
 		return -EINVAL;
 	}
 	if (usb_endpoint_xfer_isoc(ep->ep.desc)) {
@@ -1040,7 +1032,7 @@ static int fsl_ep_set_halt(struct usb_ep *_ep, int value)
 		udc->ep0_dir = 0;
 	}
 out:
-	dev_vdbg(&udc->gadget.dev, "%s %s halt stat %d\n", ep->ep.name,
+	dev_vdbg(udc->dev, "%s %s halt stat %d\n", ep->ep.name,
 		 value ?  "set" : "clear", status);
 
 	return status;
@@ -1109,7 +1101,7 @@ static void fsl_ep_fifo_flush(struct usb_ep *_ep)
 		/* Wait until flush complete */
 		while (fsl_readl(&dr_regs->endptflush)) {
 			if (time_after(jiffies, timeout)) {
-				dev_err(&udc_controller->gadget.dev,
+				dev_err(udc_controller->dev,
 					"ep flush timeout\n");
 				return;
 			}
@@ -1182,7 +1174,7 @@ static int fsl_vbus_session(struct usb_gadget *gadget, int is_active)
 
 	udc = container_of(gadget, struct fsl_udc, gadget);
 	spin_lock_irqsave(&udc->lock, flags);
-	dev_vdbg(&gadget->dev, "VBUS %s\n", str_on_off(is_active));
+	dev_vdbg(udc->dev, "VBUS %s\n", str_on_off(is_active));
 	udc->vbus_active = (is_active != 0);
 	if (can_pullup(udc))
 		fsl_writel((fsl_readl(&dr_regs->usbcmd) | USB_CMD_RUN_STOP),
@@ -1548,7 +1540,7 @@ static void ep0_req_complete(struct fsl_udc *udc, struct fsl_ep *ep0,
 		udc->ep0_state = WAIT_FOR_SETUP;
 		break;
 	case WAIT_FOR_SETUP:
-		dev_err(&udc->gadget.dev, "Unexpected ep0 packets\n");
+		dev_err(udc->dev, "Unexpected ep0 packets\n");
 		break;
 	default:
 		ep0stall(udc);
@@ -1617,7 +1609,7 @@ static int process_ep_req(struct fsl_udc *udc, int pipe,
 		errors = hc32_to_cpu(curr_td->size_ioc_sts);
 		if (errors & DTD_ERROR_MASK) {
 			if (errors & DTD_STATUS_HALTED) {
-				dev_err(&udc->gadget.dev, "dTD error %08x QH=%d\n", errors, pipe);
+				dev_err(udc->dev, "dTD error %08x QH=%d\n", errors, pipe);
 				/* Clear the errors and Halt condition */
 				tmp = hc32_to_cpu(curr_qh->size_ioc_int_sts);
 				tmp &= ~errors;
@@ -1628,26 +1620,26 @@ static int process_ep_req(struct fsl_udc *udc, int pipe,
 				break;
 			}
 			if (errors & DTD_STATUS_DATA_BUFF_ERR) {
-				dev_vdbg(&udc->gadget.dev, "Transfer overflow\n");
+				dev_vdbg(udc->dev, "Transfer overflow\n");
 				status = -EPROTO;
 				break;
 			} else if (errors & DTD_STATUS_TRANSACTION_ERR) {
-				dev_vdbg(&udc->gadget.dev, "ISO error\n");
+				dev_vdbg(udc->dev, "ISO error\n");
 				status = -EILSEQ;
 				break;
 			} else
-				dev_err(&udc->gadget.dev,
+				dev_err(udc->dev,
 					"Unknown error has occurred (0x%x)!\n",
 					errors);
 
 		} else if (hc32_to_cpu(curr_td->size_ioc_sts)
 				& DTD_STATUS_ACTIVE) {
-			dev_vdbg(&udc->gadget.dev, "Request not complete\n");
+			dev_vdbg(udc->dev, "Request not complete\n");
 			status = REQ_UNCOMPLETE;
 			return status;
 		} else if (remaining_length) {
 			if (direction) {
-				dev_vdbg(&udc->gadget.dev,
+				dev_vdbg(udc->dev,
 					 "Transmit dTD remaining length not zero\n");
 				status = -EPROTO;
 				break;
@@ -1655,8 +1647,7 @@ static int process_ep_req(struct fsl_udc *udc, int pipe,
 				break;
 			}
 		} else {
-			dev_vdbg(&udc->gadget.dev,
-				 "dTD transmitted successful\n");
+			dev_vdbg(udc->dev, "dTD transmitted successful\n");
 		}
 
 		if (j != curr_req->dtd_count - 1)
@@ -1699,7 +1690,7 @@ static void dtd_complete_irq(struct fsl_udc *udc)
 
 		/* If the ep is configured */
 		if (!curr_ep->ep.name) {
-			dev_warn(&udc->gadget.dev, "Invalid EP?\n");
+			dev_warn(udc->dev, "Invalid EP?\n");
 			continue;
 		}
 
@@ -1708,7 +1699,7 @@ static void dtd_complete_irq(struct fsl_udc *udc)
 				queue) {
 			status = process_ep_req(udc, i, curr_req);
 
-			dev_vdbg(&udc->gadget.dev,
+			dev_vdbg(udc->dev,
 				 "status of process_ep_req= %d, ep = %d\n",
 				 status, ep_num);
 			if (status == REQ_UNCOMPLETE)
@@ -1829,7 +1820,7 @@ static void reset_irq(struct fsl_udc *udc)
 	while (fsl_readl(&dr_regs->endpointprime)) {
 		/* Wait until all endptprime bits cleared */
 		if (time_after(jiffies, timeout)) {
-			dev_err(&udc->gadget.dev, "Timeout for reset\n");
+			dev_err(udc->dev, "Timeout for reset\n");
 			break;
 		}
 		cpu_relax();
@@ -1839,7 +1830,7 @@ static void reset_irq(struct fsl_udc *udc)
 	fsl_writel(0xffffffff, &dr_regs->endptflush);
 
 	if (fsl_readl(&dr_regs->portsc1) & PORTSCX_PORT_RESET) {
-		dev_vdbg(&udc->gadget.dev, "Bus reset\n");
+		dev_vdbg(udc->dev, "Bus reset\n");
 		/* Bus is reseting */
 		udc->bus_reset = 1;
 		/* Reset all the queues, include XD, dTD, EP queue
@@ -1847,7 +1838,7 @@ static void reset_irq(struct fsl_udc *udc)
 		reset_queues(udc, true);
 		udc->usb_state = USB_STATE_DEFAULT;
 	} else {
-		dev_vdbg(&udc->gadget.dev, "Controller reset\n");
+		dev_vdbg(udc->dev, "Controller reset\n");
 		/* initialize usb hw reg except for regs for EP, not
 		 * touch usbintr reg */
 		dr_controller_setup(udc);
@@ -1881,7 +1872,7 @@ static irqreturn_t fsl_udc_irq(int irq, void *_udc)
 	/* Clear notification bits */
 	fsl_writel(irq_src, &dr_regs->usbsts);
 
-	/* dev_vdbg(&udc->gadget.dev, "irq_src [0x%8x]", irq_src); */
+	/* dev_vdbg(udc->dev, "irq_src [0x%8x]", irq_src); */
 
 	/* Need to resume? */
 	if (udc->usb_state == USB_STATE_SUSPENDED)
@@ -1890,7 +1881,7 @@ static irqreturn_t fsl_udc_irq(int irq, void *_udc)
 
 	/* USB Interrupt */
 	if (irq_src & USB_STS_INT) {
-		dev_vdbg(&udc->gadget.dev, "Packet int\n");
+		dev_vdbg(udc->dev, "Packet int\n");
 		/* Setup package, we only support ep0 as control ep */
 		if (fsl_readl(&dr_regs->endptsetupstat) & EP_SETUP_STATUS_EP0) {
 			tripwire_handler(udc, 0,
@@ -1919,7 +1910,7 @@ static irqreturn_t fsl_udc_irq(int irq, void *_udc)
 
 	/* Reset Received */
 	if (irq_src & USB_STS_RESET) {
-		dev_vdbg(&udc->gadget.dev, "reset int\n");
+		dev_vdbg(udc->dev, "reset int\n");
 		reset_irq(udc);
 		status = IRQ_HANDLED;
 	}
@@ -1931,7 +1922,7 @@ static irqreturn_t fsl_udc_irq(int irq, void *_udc)
 	}
 
 	if (irq_src & (USB_STS_ERR | USB_STS_SYS_ERR)) {
-		dev_vdbg(&udc->gadget.dev, "Error IRQ %x\n", irq_src);
+		dev_vdbg(udc->dev, "Error IRQ %x\n", irq_src);
 	}
 
 	spin_unlock_irqrestore(&udc->lock, flags);
@@ -1967,7 +1958,7 @@ static int fsl_udc_start(struct usb_gadget *g,
 					udc_controller->transceiver->otg,
 						    &udc_controller->gadget);
 			if (retval < 0) {
-				dev_err(&udc_controller->gadget.dev, "can't bind to transceiver\n");
+				dev_err(udc_controller->dev, "can't bind to transceiver\n");
 				udc_controller->driver = NULL;
 				return retval;
 			}
@@ -2252,7 +2243,7 @@ static int struct_udc_setup(struct fsl_udc *udc,
 
 	udc->eps = kzalloc_objs(struct fsl_ep, udc->max_ep);
 	if (!udc->eps) {
-		dev_err(&udc->gadget.dev, "kmalloc udc endpoint status failed\n");
+		dev_err(udc->dev, "kmalloc udc endpoint status failed\n");
 		goto eps_alloc_failed;
 	}
 
@@ -2267,7 +2258,7 @@ static int struct_udc_setup(struct fsl_udc *udc,
 	udc->ep_qh = dma_alloc_coherent(&pdev->dev, size,
 					&udc->ep_qh_dma, GFP_KERNEL);
 	if (!udc->ep_qh) {
-		dev_err(&udc->gadget.dev, "malloc QHs for udc failed\n");
+		dev_err(udc->dev, "malloc QHs for udc failed\n");
 		goto ep_queue_alloc_failed;
 	}
 
@@ -2278,14 +2269,14 @@ static int struct_udc_setup(struct fsl_udc *udc,
 	udc->status_req = container_of(fsl_alloc_request(NULL, GFP_KERNEL),
 			struct fsl_req, req);
 	if (!udc->status_req) {
-		dev_err(&udc->gadget.dev, "kzalloc for udc status request failed\n");
+		dev_err(udc->dev, "kzalloc for udc status request failed\n");
 		goto udc_status_alloc_failed;
 	}
 
 	/* allocate a small amount of memory to get valid address */
 	udc->status_req->req.buf = kmalloc(8, GFP_KERNEL);
 	if (!udc->status_req->req.buf) {
-		dev_err(&udc->gadget.dev, "kzalloc for udc request buffer failed\n");
+		dev_err(udc->dev, "kzalloc for udc request buffer failed\n");
 		goto udc_req_buf_alloc_failed;
 	}
 
@@ -2373,6 +2364,7 @@ static int fsl_udc_probe(struct platform_device *pdev)
 	if (udc_controller == NULL)
 		return -ENOMEM;
 
+	udc_controller->dev = &pdev->dev;
 	pdata = dev_get_platdata(&pdev->dev);
 	udc_controller->pdata = pdata;
 	spin_lock_init(&udc_controller->lock);
@@ -2382,7 +2374,7 @@ static int fsl_udc_probe(struct platform_device *pdev)
 	if (pdata->operating_mode == FSL_USB2_DR_OTG) {
 		udc_controller->transceiver = usb_get_phy(USB_PHY_TYPE_USB2);
 		if (IS_ERR_OR_NULL(udc_controller->transceiver)) {
-			dev_err(&udc_controller->gadget.dev, "Can't find OTG driver!\n");
+			dev_err(&pdev->dev, "Can't find OTG driver!\n");
 			ret = -ENODEV;
 			goto err_kfree;
 		}
@@ -2398,7 +2390,7 @@ static int fsl_udc_probe(struct platform_device *pdev)
 	if (pdata->operating_mode == FSL_USB2_DR_DEVICE) {
 		if (!request_mem_region(res->start, resource_size(res),
 					driver_name)) {
-			dev_err(&udc_controller->gadget.dev, "request mem region for %s failed\n", pdev->name);
+			dev_err(&pdev->dev, "failed to request mem region\n");
 			ret = -EBUSY;
 			goto err_kfree;
 		}
@@ -2429,7 +2421,7 @@ static int fsl_udc_probe(struct platform_device *pdev)
 	/* Read Device Controller Capability Parameters register */
 	dccparams = fsl_readl(&dr_regs->dccparams);
 	if (!(dccparams & DCCPARAMS_DC)) {
-		dev_err(&udc_controller->gadget.dev, "This SOC doesn't support device role\n");
+		dev_err(&pdev->dev, "This SOC doesn't support device role\n");
 		ret = -ENODEV;
 		goto err_exit;
 	}
@@ -2447,14 +2439,14 @@ static int fsl_udc_probe(struct platform_device *pdev)
 	ret = request_irq(udc_controller->irq, fsl_udc_irq, IRQF_SHARED,
 			driver_name, udc_controller);
 	if (ret != 0) {
-		dev_err(&udc_controller->gadget.dev, "cannot request irq %d err %d\n",
+		dev_err(&pdev->dev, "cannot request irq %d err %d\n",
 				udc_controller->irq, ret);
 		goto err_exit;
 	}
 
 	/* Initialize the udc structure including QH member and other member */
 	if (struct_udc_setup(udc_controller, pdev)) {
-		dev_err(&udc_controller->gadget.dev, "Can't initialize udc data structure\n");
+		dev_err(&pdev->dev, "Can't initialize udc data structure\n");
 		ret = -ENOMEM;
 		goto err_free_irq;
 	}
@@ -2474,7 +2466,6 @@ static int fsl_udc_probe(struct platform_device *pdev)
 	udc_controller->gadget.name = driver_name;
 
 	/* Setup gadget.dev and register with kernel */
-	dev_set_name(&udc_controller->gadget.dev, "gadget");
 	udc_controller->gadget.dev.of_node = pdev->dev.of_node;
 
 	if (!IS_ERR_OR_NULL(udc_controller->transceiver))
@@ -2548,12 +2539,6 @@ static void fsl_udc_remove(struct platform_device *pdev)
 
 	DECLARE_COMPLETION_ONSTACK(done);
 
-	if (!udc_controller) {
-		dev_err(&pdev->dev,
-			"Driver still in use but removing anyhow\n");
-		return;
-	}
-
 	udc_controller->done = &done;
 	usb_del_gadget_udc(&udc_controller->gadget);
 
diff --git a/drivers/usb/gadget/udc/fsl_usb2_udc.h b/drivers/usb/gadget/udc/fsl_usb2_udc.h
index cc1756f3e89d1..53922bc58ca01 100644
--- a/drivers/usb/gadget/udc/fsl_usb2_udc.h
+++ b/drivers/usb/gadget/udc/fsl_usb2_udc.h
@@ -470,6 +470,7 @@ struct fsl_ep {
 #define EP_DIR_OUT	0
 
 struct fsl_udc {
+	struct device *dev;
 	struct usb_gadget gadget;
 	struct usb_gadget_driver *driver;
 	struct fsl_usb2_platform_data *pdata;
diff --git a/drivers/usb/gadget/udc/pch_udc.c b/drivers/usb/gadget/udc/pch_udc.c
index 0a68864287391..99b3ce28210f9 100644
--- a/drivers/usb/gadget/udc/pch_udc.c
+++ b/drivers/usb/gadget/udc/pch_udc.c
@@ -320,7 +320,6 @@ struct pch_vbus_gpio_data {
  * @lock:		protects all state
  * @stall:		stall requested
  * @prot_stall:		protcol stall requested
- * @registered:		driver registered with system
  * @suspended:		driver in suspended state
  * @connected:		gadget driver associated
  * @vbus_session:	required vbus_session state
diff --git a/drivers/usb/gadget/udc/pxa27x_udc.c b/drivers/usb/gadget/udc/pxa27x_udc.c
index 640f81988c041..df5cca43afbef 100644
--- a/drivers/usb/gadget/udc/pxa27x_udc.c
+++ b/drivers/usb/gadget/udc/pxa27x_udc.c
@@ -2374,9 +2374,10 @@ static int pxa_udc_probe(struct platform_device *pdev)
 	struct pxa_udc *udc = &memory;
 	int retval = 0;
 
-	udc->gpiod = devm_gpiod_get(&pdev->dev, NULL, GPIOD_ASIS);
+	udc->gpiod = devm_gpiod_get_optional(&pdev->dev, NULL, GPIOD_ASIS);
 	if (IS_ERR(udc->gpiod))
-		return PTR_ERR(udc->gpiod);
+		return dev_err_probe(&pdev->dev, PTR_ERR(udc->gpiod),
+				     "Couldn't find or request D+ gpio\n");
 
 	udc->regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(udc->regs))
@@ -2395,11 +2396,6 @@ static int pxa_udc_probe(struct platform_device *pdev)
 		udc->transceiver = usb_get_phy(USB_PHY_TYPE_USB2);
 	}
 
-	if (IS_ERR(udc->gpiod)) {
-		dev_err(&pdev->dev, "Couldn't find or request D+ gpio : %ld\n",
-			PTR_ERR(udc->gpiod));
-		return PTR_ERR(udc->gpiod);
-	}
 	if (udc->gpiod)
 		gpiod_direction_output(udc->gpiod, 0);
 
diff --git a/drivers/usb/gadget/udc/r8a66597-udc.c b/drivers/usb/gadget/udc/r8a66597-udc.c
index e7a5d8553c0ea..d190e16d43fc0 100644
--- a/drivers/usb/gadget/udc/r8a66597-udc.c
+++ b/drivers/usb/gadget/udc/r8a66597-udc.c
@@ -1951,7 +1951,6 @@ static int r8a66597_probe(struct platform_device *pdev)
 	return 0;
 
 err_add_udc:
-	r8a66597_free_request(&r8a66597->ep[0].ep, r8a66597->ep0_req);
 clean_up2:
 	if (r8a66597->pdata->on_chip)
 		clk_disable_unprepare(r8a66597->clk);
diff --git a/drivers/usb/gadget/udc/snps_udc_core.c b/drivers/usb/gadget/udc/snps_udc_core.c
index 0e0db68e0b27a..d506f9d92bcad 100644
--- a/drivers/usb/gadget/udc/snps_udc_core.c
+++ b/drivers/usb/gadget/udc/snps_udc_core.c
@@ -3133,7 +3133,6 @@ int udc_probe(struct udc *dev)
 	/* device struct setup */
 	dev->gadget.ops = &udc_ops;
 
-	dev_set_name(&dev->gadget.dev, "gadget");
 	dev->gadget.name = name;
 	dev->gadget.max_speed = USB_SPEED_HIGH;
 
diff --git a/drivers/usb/host/fsl-mph-dr-of.c b/drivers/usb/host/fsl-mph-dr-of.c
index edfb635430293..7d46586e2e212 100644
--- a/drivers/usb/host/fsl-mph-dr-of.c
+++ b/drivers/usb/host/fsl-mph-dr-of.c
@@ -15,6 +15,7 @@
 #include <linux/clk.h>
 #include <linux/module.h>
 #include <linux/dma-mapping.h>
+#include <linux/regulator/consumer.h>
 
 struct fsl_usb2_dev_data {
 	char *dr_mode;		/* controller mode */
@@ -183,7 +184,7 @@ static int fsl_usb2_mph_dr_of_probe(struct platform_device *ofdev)
 	const struct of_device_id *match;
 	const unsigned char *prop;
 	static unsigned int idx;
-	int i;
+	int i, err;
 
 	if (!of_device_is_available(np))
 		return -ENODEV;
@@ -246,6 +247,10 @@ static int fsl_usb2_mph_dr_of_probe(struct platform_device *ofdev)
 		}
 	}
 
+	err = devm_regulator_get_enable_optional(&ofdev->dev, "vbus");
+	if (err)
+		return dev_err_probe(&ofdev->dev, err, "failed to get vbus regulator\n");
+
 	for (i = 0; i < ARRAY_SIZE(dev_data->drivers); i++) {
 		if (!dev_data->drivers[i])
 			continue;
diff --git a/drivers/usb/host/ohci-dbg.c b/drivers/usb/host/ohci-dbg.c
index 9e0e06bbc5706..23dc9eddc06c4 100644
--- a/drivers/usb/host/ohci-dbg.c
+++ b/drivers/usb/host/ohci-dbg.c
@@ -683,7 +683,7 @@ static int fill_buffer(struct debug_buffer *buf)
 	int ret;
 
 	if (!buf->page)
-		buf->page = (char *)get_zeroed_page(GFP_KERNEL);
+		buf->page = kzalloc(PAGE_SIZE, GFP_KERNEL);
 
 	if (!buf->page) {
 		ret = -ENOMEM;
@@ -729,11 +729,8 @@ static int debug_close(struct inode *inode, struct file *file)
 {
 	struct debug_buffer *buf = file->private_data;
 
-	if (buf) {
-		if (buf->page)
-			free_page((unsigned long)buf->page);
-		kfree(buf);
-	}
+	kfree(buf->page);
+	kfree(buf);
 
 	return 0;
 }
diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c
index 039c26b241d08..6b3fcba44b08d 100644
--- a/drivers/usb/host/xhci-pci.c
+++ b/drivers/usb/host/xhci-pci.c
@@ -448,6 +448,7 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci)
 	if (pdev->vendor == PCI_VENDOR_ID_VIA && pdev->device == PCI_DEVICE_ID_VIA_VL805) {
 		xhci->quirks |= XHCI_LPM_SUPPORT;
 		xhci->quirks |= XHCI_TRB_OVERFETCH;
+		xhci->dma_mask_bits = 36;
 	}
 
 	if (pdev->vendor == PCI_VENDOR_ID_ASMEDIA &&
diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c
index f44ccee5fa078..091c82ca8ee29 100644
--- a/drivers/usb/host/xhci.c
+++ b/drivers/usb/host/xhci.c
@@ -5466,6 +5466,7 @@ int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks)
 	if (xhci->hci_version > 0x100)
 		xhci->hcc_params2 = readl(&xhci->cap_regs->hcc_params2);
 
+	xhci->dma_mask_bits = 64;
 	xhci->max_slots = min(HCS_MAX_SLOTS(hcs_params1), MAX_HC_SLOTS);
 	xhci->max_ports = min(HCS_MAX_PORTS(hcs_params1), MAX_HC_PORTS);
 	/* xhci-plat or xhci-pci might have set max_interrupters already */
@@ -5515,12 +5516,16 @@ int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks)
 	if (xhci->quirks & XHCI_NO_64BIT_SUPPORT)
 		xhci->hcc_params &= ~BIT(0);
 
-	/* Set dma_mask and coherent_dma_mask to 64-bits,
-	 * if xHC supports 64-bit addressing */
+	/*
+	 * Set dma_mask and coherent_dma_mask to 64-bits if xHC supports
+	 * 64-bit addressing, unless a controller-specific quirk callback
+	 * limits the usable address width.
+	 */
 	if ((xhci->hcc_params & HCC_64BIT_ADDR) &&
-			!dma_set_mask(dev, DMA_BIT_MASK(64))) {
-		xhci_dbg(xhci, "Enabling 64-bit DMA addresses.\n");
-		dma_set_coherent_mask(dev, DMA_BIT_MASK(64));
+	    !dma_set_mask(dev, DMA_BIT_MASK(xhci->dma_mask_bits))) {
+		xhci_dbg(xhci, "Enabling %u-bit DMA addresses.\n",
+			 xhci->dma_mask_bits);
+		dma_set_coherent_mask(dev, DMA_BIT_MASK(xhci->dma_mask_bits));
 	} else {
 		/*
 		 * This is to avoid error in cases where a 32-bit USB
diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h
index d02046a573e45..2d3941b5e1e3d 100644
--- a/drivers/usb/host/xhci.h
+++ b/drivers/usb/host/xhci.h
@@ -1526,6 +1526,7 @@ struct xhci_hcd {
 	/* imod_interval in ns (I * 250ns) */
 	u32		imod_interval;
 	u32		page_size;
+	unsigned int	dma_mask_bits;
 	/* MSI-X/MSI vectors */
 	int		nvecs;
 	/* optional clocks */
diff --git a/drivers/usb/misc/usb-ljca.c b/drivers/usb/misc/usb-ljca.c
index 78e94dd89da55..fcb627b49eac6 100644
--- a/drivers/usb/misc/usb-ljca.c
+++ b/drivers/usb/misc/usb-ljca.c
@@ -595,6 +595,9 @@ static int ljca_enumerate_gpio(struct ljca_adapter *adap)
 	if (gpio_num > LJCA_MAX_GPIO_NUM)
 		return -EINVAL;
 
+	if (desc->bank_num > ARRAY_SIZE(valid_pin))
+		return -EINVAL;
+
 	/* construct platform data */
 	gpio_info = kzalloc_obj(*gpio_info);
 	if (!gpio_info)
diff --git a/drivers/usb/mtu3/mtu3_core.c b/drivers/usb/mtu3/mtu3_core.c
index 66dbfe1705d57..a40bf5bad2d5d 100644
--- a/drivers/usb/mtu3/mtu3_core.c
+++ b/drivers/usb/mtu3/mtu3_core.c
@@ -1037,9 +1037,14 @@ int ssusb_gadget_suspend(struct ssusb_mtk *ssusb, pm_message_t msg)
 	if (!mtu->gadget_driver)
 		return 0;
 
-	if (mtu->connected)
+	/* Prevent runtime suspend when active connection exists */
+	if (mtu->connected && PMSG_IS_AUTO(msg))
 		return -EBUSY;
 
+	/* Perform soft disconnect for system suspend */
+	if (mtu->softconnect && !PMSG_IS_AUTO(msg))
+		mtu3_dev_on_off(mtu, 0);
+
 	mtu3_dev_suspend(mtu);
 	synchronize_irq(mtu->irq);
 
@@ -1055,5 +1060,9 @@ int ssusb_gadget_resume(struct ssusb_mtk *ssusb, pm_message_t msg)
 
 	mtu3_dev_resume(mtu);
 
+	/* Restore soft connect for system resume */
+	if (mtu->softconnect && !PMSG_IS_AUTO(msg))
+		mtu3_dev_on_off(mtu, 1);
+
 	return 0;
 }
diff --git a/drivers/usb/mtu3/mtu3_host.c b/drivers/usb/mtu3/mtu3_host.c
index 7c657ea2dabd1..8138b3f3096a9 100644
--- a/drivers/usb/mtu3/mtu3_host.c
+++ b/drivers/usb/mtu3/mtu3_host.c
@@ -46,6 +46,14 @@
 #define WC1_IS_P_95		BIT(12)
 #define WC1_IS_EN_P0_95		BIT(6)
 
+/* mt8196 */
+#define PERI_WK_CTRL0_8196	0x08
+#define WC0_IS_EN_P0_96		BIT(0)
+#define WC0_IS_EN_P1_96		BIT(7)
+
+#define PERI_WK_CTRL1_8196	0x10
+#define WC1_IS_EN_P2_96		BIT(0)
+
 /* mt2712 etc */
 #define PERI_SSUSB_SPM_CTRL	0x0
 #define SSC_IP_SLEEP_EN	BIT(4)
@@ -59,6 +67,9 @@ enum ssusb_uwk_vers {
 	SSUSB_UWK_V1_3,		/* mt8195 IP0 */
 	SSUSB_UWK_V1_5 = 105,	/* mt8195 IP2 */
 	SSUSB_UWK_V1_6,		/* mt8195 IP3 */
+	SSUSB_UWK_V1_7, 	/* mt8196 IP0 */
+	SSUSB_UWK_V1_8, 	/* mt8196 IP1 */
+	SSUSB_UWK_V1_9, 	/* mt8196 IP2 */
 };
 
 /*
@@ -100,6 +111,21 @@ static void ssusb_wakeup_ip_sleep_set(struct ssusb_mtk *ssusb, bool enable)
 		msk = WC0_IS_EN_P3_95 | WC0_IS_C_95(0x7) | WC0_IS_P_95;
 		val = enable ? (WC0_IS_EN_P3_95 | WC0_IS_C_95(0x1)) : 0;
 		break;
+	case SSUSB_UWK_V1_7:
+		reg = ssusb->uwk_reg_base + PERI_WK_CTRL0_8196;
+		msk = WC0_IS_EN_P0_96;
+		val = enable ? msk : 0;
+		break;
+	case SSUSB_UWK_V1_8:
+		reg = ssusb->uwk_reg_base + PERI_WK_CTRL0_8196;
+		msk = WC0_IS_EN_P1_96;
+		val = enable ? msk : 0;
+		break;
+	case SSUSB_UWK_V1_9:
+		reg = ssusb->uwk_reg_base + PERI_WK_CTRL1_8196;
+		msk = WC1_IS_EN_P2_96;
+		val = enable ? msk : 0;
+		break;
 	case SSUSB_UWK_V2:
 		reg = ssusb->uwk_reg_base + PERI_SSUSB_SPM_CTRL;
 		msk = SSC_IP_SLEEP_EN | SSC_SPM_INT_EN;
diff --git a/drivers/usb/mtu3/mtu3_plat.c b/drivers/usb/mtu3/mtu3_plat.c
index cc8a864dbd638..b4f4c776adb98 100644
--- a/drivers/usb/mtu3/mtu3_plat.c
+++ b/drivers/usb/mtu3/mtu3_plat.c
@@ -544,7 +544,8 @@ static int mtu3_suspend_common(struct device *dev, pm_message_t msg)
 
 	ssusb_phy_power_off(ssusb);
 	clk_bulk_disable_unprepare(BULK_CLKS_CNT, ssusb->clks);
-	ssusb_wakeup_set(ssusb, true);
+	if (device_may_wakeup(dev) && (ssusb->is_host || PMSG_IS_AUTO(msg)))
+		ssusb_wakeup_set(ssusb, true);
 	return 0;
 
 sleep_err:
diff --git a/drivers/usb/musb/jz4740.c b/drivers/usb/musb/jz4740.c
index df56c972986f7..c770ba576f05b 100644
--- a/drivers/usb/musb/jz4740.c
+++ b/drivers/usb/musb/jz4740.c
@@ -273,7 +273,7 @@ static int jz4740_probe(struct platform_device *pdev)
 	musb->dev.parent		= dev;
 	musb->dev.dma_mask		= &musb->dev.coherent_dma_mask;
 	musb->dev.coherent_dma_mask	= DMA_BIT_MASK(32);
-	device_set_of_node_from_dev(&musb->dev, dev);
+	platform_device_set_of_node_from_dev(musb, dev);
 
 	glue->pdev			= musb;
 	glue->clk			= clk;
diff --git a/drivers/usb/musb/omap2430.c b/drivers/usb/musb/omap2430.c
index 333ab79f0ca90..6e749faac33cb 100644
--- a/drivers/usb/musb/omap2430.c
+++ b/drivers/usb/musb/omap2430.c
@@ -454,7 +454,6 @@ static int omap2430_probe(struct platform_device *pdev)
 		dev_err(&pdev->dev, "failed to register musb device\n");
 		goto err_disable_rpm;
 	}
-	of_node_put(np);
 
 	return 0;
 
@@ -464,7 +463,6 @@ static int omap2430_probe(struct platform_device *pdev)
 	if (!IS_ERR(glue->control_otghs))
 		put_device(glue->control_otghs);
 err_put_musb:
-	of_node_put(np);
 	platform_device_put(musb);
 
 	return ret;
diff --git a/drivers/usb/renesas_usbhs/common.c b/drivers/usb/renesas_usbhs/common.c
index 8c93bde4b8167..51d3035f82bed 100644
--- a/drivers/usb/renesas_usbhs/common.c
+++ b/drivers/usb/renesas_usbhs/common.c
@@ -813,9 +813,6 @@ static void usbhs_remove(struct platform_device *pdev)
 
 	flush_delayed_work(&priv->notify_hotplug_work);
 
-	usbhs_platform_call(priv, hardware_exit, pdev);
-	reset_control_assert(priv->rsts);
-
 	/*
 	 * Explicitly free the IRQ to ensure the interrupt handler is
 	 * disabled and synchronized before freeing resources.
@@ -832,6 +829,9 @@ static void usbhs_remove(struct platform_device *pdev)
 	if (!usbhs_get_dparam(priv, runtime_pwctrl))
 		usbhsc_power_ctrl(priv, 0);
 
+	usbhs_platform_call(priv, hardware_exit, pdev);
+	reset_control_assert(priv->rsts);
+
 	usbhsc_clk_put(priv);
 	pm_runtime_disable(&pdev->dev);
 }
diff --git a/drivers/usb/serial/digi_acceleport.c b/drivers/usb/serial/digi_acceleport.c
index dea039163661b..e6d14f7d106dc 100644
--- a/drivers/usb/serial/digi_acceleport.c
+++ b/drivers/usb/serial/digi_acceleport.c
@@ -32,10 +32,6 @@
 /* so we can be sure to send the full buffer in one urb */
 #define DIGI_OUT_BUF_SIZE		8
 
-/* port input buffer length -- must be >= transfer buffer length - 3 */
-/* so we can be sure to hold at least one full buffer from one urb */
-#define DIGI_IN_BUF_SIZE		64
-
 /* retry timeout while sleeping */
 #define DIGI_RETRY_TIMEOUT		(HZ/10)
 
@@ -176,10 +172,10 @@
 /* Structures */
 
 struct digi_serial {
-	spinlock_t ds_serial_lock;
+	struct mutex open_mutex;
 	struct usb_serial_port *ds_oob_port;	/* out-of-band port */
 	int ds_oob_port_num;			/* index of out-of-band port */
-	int ds_device_started;
+	int open_count;
 };
 
 struct digi_port {
@@ -194,7 +190,6 @@ struct digi_port {
 	int dp_throttled;
 	int dp_throttle_restart;
 	wait_queue_head_t dp_flush_wait;
-	wait_queue_head_t dp_close_wait;	/* wait queue for close */
 	wait_queue_head_t write_wait;
 	struct usb_serial_port *dp_port;
 };
@@ -227,9 +222,7 @@ static unsigned int digi_chars_in_buffer(struct tty_struct *tty);
 static int digi_open(struct tty_struct *tty, struct usb_serial_port *port);
 static void digi_close(struct usb_serial_port *port);
 static void digi_dtr_rts(struct usb_serial_port *port, int on);
-static int digi_startup_device(struct usb_serial *serial);
 static int digi_startup(struct usb_serial *serial);
-static void digi_disconnect(struct usb_serial *serial);
 static void digi_release(struct usb_serial *serial);
 static int digi_port_probe(struct usb_serial_port *port);
 static void digi_port_remove(struct usb_serial_port *port);
@@ -272,7 +265,7 @@ static struct usb_serial_driver digi_acceleport_2_device = {
 	.dtr_rts =			digi_dtr_rts,
 	.write =			digi_write,
 	.write_room =			digi_write_room,
-	.write_bulk_callback = 		digi_write_bulk_callback,
+	.write_bulk_callback =		digi_write_bulk_callback,
 	.read_bulk_callback =		digi_read_bulk_callback,
 	.chars_in_buffer =		digi_chars_in_buffer,
 	.throttle =			digi_rx_throttle,
@@ -282,7 +275,6 @@ static struct usb_serial_driver digi_acceleport_2_device = {
 	.tiocmget =			digi_tiocmget,
 	.tiocmset =			digi_tiocmset,
 	.attach =			digi_startup,
-	.disconnect =			digi_disconnect,
 	.release =			digi_release,
 	.port_probe =			digi_port_probe,
 	.port_remove =			digi_port_remove,
@@ -301,7 +293,7 @@ static struct usb_serial_driver digi_acceleport_4_device = {
 	.close =			digi_close,
 	.write =			digi_write,
 	.write_room =			digi_write_room,
-	.write_bulk_callback = 		digi_write_bulk_callback,
+	.write_bulk_callback =		digi_write_bulk_callback,
 	.read_bulk_callback =		digi_read_bulk_callback,
 	.chars_in_buffer =		digi_chars_in_buffer,
 	.throttle =			digi_rx_throttle,
@@ -311,7 +303,6 @@ static struct usb_serial_driver digi_acceleport_4_device = {
 	.tiocmget =			digi_tiocmget,
 	.tiocmset =			digi_tiocmset,
 	.attach =			digi_startup,
-	.disconnect =			digi_disconnect,
 	.release =			digi_release,
 	.port_probe =			digi_port_probe,
 	.port_remove =			digi_port_remove,
@@ -335,7 +326,6 @@ static struct usb_serial_driver * const serial_drivers[] = {
  *  interruptible_sleep_on_timeout is deprecated and has been replaced
  *  with the equivalent code.
  */
-
 static long cond_wait_interruptible_timeout_irqrestore(
 	wait_queue_head_t *q, long timeout,
 	spinlock_t *lock, unsigned long flags)
@@ -351,6 +341,13 @@ __releases(lock)
 	return timeout;
 }
 
+static struct usb_serial_port *digi_get_oob_port(struct usb_serial *serial)
+{
+	struct digi_serial *serial_priv = usb_get_serial_data(serial);
+
+	return serial_priv->ds_oob_port;
+}
+
 /*
  *  Digi Write OOB Command
  *
@@ -361,15 +358,14 @@ __releases(lock)
  *  the interruptible flag is true, or a negative error
  *  returned by usb_submit_urb.
  */
-
 static int digi_write_oob_command(struct usb_serial_port *port,
 	unsigned char *buf, int count, int interruptible)
 {
-	int ret = 0;
-	int len;
-	struct usb_serial_port *oob_port = (struct usb_serial_port *)((struct digi_serial *)(usb_get_serial_data(port->serial)))->ds_oob_port;
+	struct usb_serial_port *oob_port = digi_get_oob_port(port->serial);
 	struct digi_port *oob_priv = usb_get_serial_port_data(oob_port);
 	unsigned long flags;
+	int ret = 0;
+	int len;
 
 	dev_dbg(&port->dev,
 		"digi_write_oob_command: TOP: port=%d, count=%d\n",
@@ -406,10 +402,8 @@ static int digi_write_oob_command(struct usb_serial_port *port,
 		dev_err(&port->dev, "%s: usb_submit_urb failed, ret=%d\n",
 			__func__, ret);
 	return ret;
-
 }
 
-
 /*
  *  Digi Write In Band Command
  *
@@ -421,16 +415,15 @@ static int digi_write_oob_command(struct usb_serial_port *port,
  *  timeout ticks.  Returns 0 if successful, or a negative
  *  error returned by digi_write.
  */
-
 static int digi_write_inb_command(struct usb_serial_port *port,
 	unsigned char *buf, int count, unsigned long timeout)
 {
-	int ret = 0;
-	int len;
 	struct digi_port *priv = usb_get_serial_port_data(port);
 	unsigned char *data = port->write_urb->transfer_buffer;
 	unsigned long expire;
 	unsigned long flags;
+	int ret = 0;
+	int len;
 
 	dev_dbg(&port->dev, "digi_write_inb_command: TOP: port=%d, count=%d\n",
 		priv->dp_port_num, count);
@@ -459,7 +452,7 @@ static int digi_write_inb_command(struct usb_serial_port *port,
 		/* len must be a multiple of 4 and small enough to */
 		/* guarantee the write will send buffered data first, */
 		/* so commands are in order with data and not split */
-		len = min(count, port->bulk_out_size-2-priv->dp_out_buf_len);
+		len = min(count, port->bulk_out_size - 2 - priv->dp_out_buf_len);
 		if (len > 4)
 			len &= ~3;
 
@@ -467,11 +460,10 @@ static int digi_write_inb_command(struct usb_serial_port *port,
 		if (priv->dp_out_buf_len > 0) {
 			data[0] = DIGI_CMD_SEND_DATA;
 			data[1] = priv->dp_out_buf_len;
-			memcpy(data + 2, priv->dp_out_buf,
-				priv->dp_out_buf_len);
+			memcpy(data + 2, priv->dp_out_buf, priv->dp_out_buf_len);
 			memcpy(data + 2 + priv->dp_out_buf_len, buf, len);
-			port->write_urb->transfer_buffer_length
-				= priv->dp_out_buf_len + 2 + len;
+			port->write_urb->transfer_buffer_length =
+					priv->dp_out_buf_len + 2 + len;
 		} else {
 			memcpy(data, buf, len);
 			port->write_urb->transfer_buffer_length = len;
@@ -484,7 +476,6 @@ static int digi_write_inb_command(struct usb_serial_port *port,
 			count -= len;
 			buf += len;
 		}
-
 	}
 	spin_unlock_irqrestore(&priv->dp_port_lock, flags);
 
@@ -495,7 +486,6 @@ static int digi_write_inb_command(struct usb_serial_port *port,
 	return ret;
 }
 
-
 /*
  *  Digi Set Modem Signals
  *
@@ -505,17 +495,15 @@ static int digi_write_inb_command(struct usb_serial_port *port,
  *  -EINTR if interrupted while sleeping, or a non-zero error
  *  returned by usb_submit_urb.
  */
-
 static int digi_set_modem_signals(struct usb_serial_port *port,
 	unsigned int modem_signals, int interruptible)
 {
-
-	int ret;
 	struct digi_port *port_priv = usb_get_serial_port_data(port);
-	struct usb_serial_port *oob_port = (struct usb_serial_port *) ((struct digi_serial *)(usb_get_serial_data(port->serial)))->ds_oob_port;
+	struct usb_serial_port *oob_port = digi_get_oob_port(port->serial);
 	struct digi_port *oob_priv = usb_get_serial_port_data(oob_port);
 	unsigned char *data = oob_port->write_urb->transfer_buffer;
 	unsigned long flags;
+	int ret;
 
 	dev_dbg(&port->dev,
 		"digi_set_modem_signals: TOP: port=%d, modem_signals=0x%x\n",
@@ -573,14 +561,13 @@ static int digi_set_modem_signals(struct usb_serial_port *port,
  *  is only called from close, and only one process can be in close on a
  *  port at a time, so its ok.
  */
-
 static int digi_transmit_idle(struct usb_serial_port *port,
 	unsigned long timeout)
 {
-	int ret;
-	unsigned char buf[2];
 	struct digi_port *priv = usb_get_serial_port_data(port);
+	unsigned char buf[2];
 	unsigned long flags;
+	int ret;
 
 	spin_lock_irqsave(&priv->dp_port_lock, flags);
 	priv->dp_transmit_idle = 0;
@@ -607,16 +594,15 @@ static int digi_transmit_idle(struct usb_serial_port *port,
 	}
 	priv->dp_transmit_idle = 0;
 	spin_unlock_irqrestore(&priv->dp_port_lock, flags);
-	return 0;
 
+	return 0;
 }
 
-
 static void digi_rx_throttle(struct tty_struct *tty)
 {
-	unsigned long flags;
 	struct usb_serial_port *port = tty->driver_data;
 	struct digi_port *priv = usb_get_serial_port_data(port);
+	unsigned long flags;
 
 	/* stop receiving characters by not resubmitting the read urb */
 	spin_lock_irqsave(&priv->dp_port_lock, flags);
@@ -625,13 +611,12 @@ static void digi_rx_throttle(struct tty_struct *tty)
 	spin_unlock_irqrestore(&priv->dp_port_lock, flags);
 }
 
-
 static void digi_rx_unthrottle(struct tty_struct *tty)
 {
-	int ret = 0;
-	unsigned long flags;
 	struct usb_serial_port *port = tty->driver_data;
 	struct digi_port *priv = usb_get_serial_port_data(port);
+	unsigned long flags;
+	int ret = 0;
 
 	spin_lock_irqsave(&priv->dp_port_lock, flags);
 
@@ -651,7 +636,6 @@ static void digi_rx_unthrottle(struct tty_struct *tty)
 			__func__, ret, priv->dp_port_num);
 }
 
-
 static void digi_set_termios(struct tty_struct *tty,
 			     struct usb_serial_port *port,
 			     const struct ktermios *old_termios)
@@ -762,7 +746,6 @@ static void digi_set_termios(struct tty_struct *tty,
 
 	/* set stop bits */
 	if ((cflag & CSTOPB) != (old_cflag & CSTOPB)) {
-
 		if ((cflag & CSTOPB))
 			arg = DIGI_STOP_BITS_2;
 		else
@@ -772,7 +755,6 @@ static void digi_set_termios(struct tty_struct *tty,
 		buf[i++] = priv->dp_port_num;
 		buf[i++] = arg;
 		buf[i++] = 0;
-
 	}
 
 	/* set input flow control */
@@ -841,7 +823,6 @@ static void digi_set_termios(struct tty_struct *tty,
 	tty_encode_baud_rate(tty, baud, baud);
 }
 
-
 static int digi_break_ctl(struct tty_struct *tty, int break_state)
 {
 	struct usb_serial_port *port = tty->driver_data;
@@ -855,50 +836,48 @@ static int digi_break_ctl(struct tty_struct *tty, int break_state)
 	return digi_write_inb_command(port, buf, 4, 0);
 }
 
-
 static int digi_tiocmget(struct tty_struct *tty)
 {
 	struct usb_serial_port *port = tty->driver_data;
 	struct digi_port *priv = usb_get_serial_port_data(port);
-	unsigned int val;
 	unsigned long flags;
+	unsigned int val;
 
 	spin_lock_irqsave(&priv->dp_port_lock, flags);
 	val = priv->dp_modem_signals;
 	spin_unlock_irqrestore(&priv->dp_port_lock, flags);
+
 	return val;
 }
 
-
 static int digi_tiocmset(struct tty_struct *tty,
 					unsigned int set, unsigned int clear)
 {
 	struct usb_serial_port *port = tty->driver_data;
 	struct digi_port *priv = usb_get_serial_port_data(port);
-	unsigned int val;
 	unsigned long flags;
+	unsigned int val;
 
 	spin_lock_irqsave(&priv->dp_port_lock, flags);
 	val = (priv->dp_modem_signals & ~clear) | set;
 	spin_unlock_irqrestore(&priv->dp_port_lock, flags);
+
 	return digi_set_modem_signals(port, val, 1);
 }
 
-
 static int digi_write(struct tty_struct *tty, struct usb_serial_port *port,
 					const unsigned char *buf, int count)
 {
-
-	int ret, data_len, new_len;
 	struct digi_port *priv = usb_get_serial_port_data(port);
 	unsigned char *data = port->write_urb->transfer_buffer;
+	int ret, data_len, new_len;
 	unsigned long flags;
 
 	dev_dbg(&port->dev, "digi_write: TOP: port=%d, count=%d\n",
 		priv->dp_port_num, count);
 
 	/* copy user data (which can sleep) before getting spin lock */
-	count = min(count, port->bulk_out_size-2);
+	count = min(count, port->bulk_out_size - 2);
 	count = min(64, count);
 
 	/* be sure only one write proceeds at a time */
@@ -920,7 +899,7 @@ static int digi_write(struct tty_struct *tty, struct usb_serial_port *port,
 
 	/* allow space for any buffered data and for new data, up to */
 	/* transfer buffer size - 2 (for command and length bytes) */
-	new_len = min(count, port->bulk_out_size-2-priv->dp_out_buf_len);
+	new_len = min(count, port->bulk_out_size - 2 - priv->dp_out_buf_len);
 	data_len = new_len + priv->dp_out_buf_len;
 
 	if (data_len == 0) {
@@ -928,7 +907,7 @@ static int digi_write(struct tty_struct *tty, struct usb_serial_port *port,
 		return 0;
 	}
 
-	port->write_urb->transfer_buffer_length = data_len+2;
+	port->write_urb->transfer_buffer_length = data_len + 2;
 
 	*data++ = DIGI_CMD_SEND_DATA;
 	*data++ = data_len;
@@ -954,39 +933,22 @@ static int digi_write(struct tty_struct *tty, struct usb_serial_port *port,
 			"%s: usb_submit_urb failed, ret=%d, port=%d\n",
 			__func__, ret, priv->dp_port_num);
 	dev_dbg(&port->dev, "digi_write: returning %d\n", ret);
-	return ret;
 
+	return ret;
 }
 
 static void digi_write_bulk_callback(struct urb *urb)
 {
-
 	struct usb_serial_port *port = urb->context;
-	struct usb_serial *serial;
-	struct digi_port *priv;
-	struct digi_serial *serial_priv;
+	struct digi_serial *serial_priv = usb_get_serial_data(port->serial);
+	struct digi_port *priv = usb_get_serial_port_data(port);
+	unsigned char *data = urb->transfer_buffer;
 	unsigned long flags;
-	int ret = 0;
-	int status = urb->status;
 	bool wakeup;
-
-	/* port and serial sanity check */
-	if (port == NULL || (priv = usb_get_serial_port_data(port)) == NULL) {
-		pr_err("%s: port or port->private is NULL, status=%d\n",
-			__func__, status);
-		return;
-	}
-	serial = port->serial;
-	if (serial == NULL || (serial_priv = usb_get_serial_data(serial)) == NULL) {
-		dev_err(&port->dev,
-			"%s: serial or serial->private is NULL, status=%d\n",
-			__func__, status);
-		return;
-	}
+	int ret = 0;
 
 	/* handle oob callback */
 	if (priv->dp_port_num == serial_priv->ds_oob_port_num) {
-		dev_dbg(&port->dev, "digi_write_bulk_callback: oob callback\n");
 		spin_lock_irqsave(&priv->dp_port_lock, flags);
 		priv->dp_write_urb_in_use = 0;
 		wake_up_interruptible(&priv->write_wait);
@@ -999,15 +961,13 @@ static void digi_write_bulk_callback(struct urb *urb)
 	spin_lock_irqsave(&priv->dp_port_lock, flags);
 	priv->dp_write_urb_in_use = 0;
 	if (priv->dp_out_buf_len > 0) {
-		*((unsigned char *)(port->write_urb->transfer_buffer))
-			= (unsigned char)DIGI_CMD_SEND_DATA;
-		*((unsigned char *)(port->write_urb->transfer_buffer) + 1)
-			= (unsigned char)priv->dp_out_buf_len;
-		port->write_urb->transfer_buffer_length =
-						priv->dp_out_buf_len + 2;
-		memcpy(port->write_urb->transfer_buffer + 2, priv->dp_out_buf,
-			priv->dp_out_buf_len);
-		ret = usb_submit_urb(port->write_urb, GFP_ATOMIC);
+		data[0] = DIGI_CMD_SEND_DATA;
+		data[1] = priv->dp_out_buf_len;
+		memcpy(data + 2, priv->dp_out_buf, priv->dp_out_buf_len);
+
+		urb->transfer_buffer_length = priv->dp_out_buf_len + 2;
+
+		ret = usb_submit_urb(urb, GFP_ATOMIC);
 		if (ret == 0) {
 			priv->dp_write_urb_in_use = 1;
 			priv->dp_out_buf_len = 0;
@@ -1041,8 +1001,8 @@ static unsigned int digi_write_room(struct tty_struct *tty)
 
 	spin_unlock_irqrestore(&priv->dp_port_lock, flags);
 	dev_dbg(&port->dev, "digi_write_room: port=%d, room=%u\n", priv->dp_port_num, room);
-	return room;
 
+	return room;
 }
 
 static unsigned int digi_chars_in_buffer(struct tty_struct *tty)
@@ -1070,17 +1030,53 @@ static void digi_dtr_rts(struct usb_serial_port *port, int on)
 	digi_set_modem_signals(port, on * (TIOCM_DTR | TIOCM_RTS), 1);
 }
 
+static int digi_open_oob_port(struct usb_serial *serial)
+{
+	struct digi_serial *serial_priv = usb_get_serial_data(serial);
+	struct usb_serial_port *oob_port = serial_priv->ds_oob_port;
+	int ret = 0;
+
+	mutex_lock(&serial_priv->open_mutex);
+
+	if (serial_priv->open_count++ == 0) {
+		ret = usb_submit_urb(oob_port->read_urb, GFP_KERNEL);
+		if (ret) {
+			dev_err(&serial->interface->dev, "failed to submit OOB read urb: %d\n",
+					ret);
+			serial_priv->open_count--;
+		}
+	}
+
+	mutex_unlock(&serial_priv->open_mutex);
+
+	return ret;
+}
+
+static void digi_close_oob_port(struct usb_serial *serial)
+{
+	struct digi_serial *serial_priv = usb_get_serial_data(serial);
+	struct usb_serial_port *oob_port = serial_priv->ds_oob_port;
+
+	mutex_lock(&serial_priv->open_mutex);
+
+	if (serial_priv->open_count-- == 1) {
+		usb_kill_urb(oob_port->read_urb);
+		usb_kill_urb(oob_port->write_urb);
+	}
+
+	mutex_unlock(&serial_priv->open_mutex);
+}
+
 static int digi_open(struct tty_struct *tty, struct usb_serial_port *port)
 {
-	int ret;
-	unsigned char buf[32];
 	struct digi_port *priv = usb_get_serial_port_data(port);
 	struct ktermios not_termios;
-	int throttled;
+	unsigned char buf[32];
+	int ret;
 
-	/* be sure the device is started up */
-	if (digi_startup_device(port->serial) != 0)
-		return -ENXIO;
+	ret = digi_open_oob_port(port->serial);
+	if (ret)
+		return ret;
 
 	/* read modem signals automatically whenever they change */
 	buf[0] = DIGI_CMD_READ_INPUT_SIGNALS;
@@ -1106,32 +1102,34 @@ static int digi_open(struct tty_struct *tty, struct usb_serial_port *port)
 	}
 
 	spin_lock_irq(&priv->dp_port_lock);
-	throttled = priv->dp_throttle_restart;
 	priv->dp_throttled = 0;
 	priv->dp_throttle_restart = 0;
 	spin_unlock_irq(&priv->dp_port_lock);
 
-	if (throttled) {
-		ret = usb_submit_urb(port->read_urb, GFP_KERNEL);
-		if (ret) {
-			dev_err(&port->dev, "failed to submit read urb: %d\n", ret);
-			return ret;
-		}
+	ret = usb_submit_urb(port->read_urb, GFP_KERNEL);
+	if (ret) {
+		dev_err(&port->dev, "failed to submit read urb: %d\n", ret);
+		goto err_close_oob;
 	}
 
 	return 0;
-}
 
+err_close_oob:
+	digi_close_oob_port(port->serial);
+
+	return ret;
+}
 
 static void digi_close(struct usb_serial_port *port)
 {
+	struct digi_port *priv = usb_get_serial_port_data(port);
+	unsigned char buf[32];
 	DEFINE_WAIT(wait);
 	int ret;
-	unsigned char buf[32];
-	struct digi_port *priv = usb_get_serial_port_data(port);
+
+	usb_kill_urb(port->read_urb);
 
 	mutex_lock(&port->serial->disc_mutex);
-	/* if disconnected, just clear flags */
 	if (port->serial->disconnected)
 		goto exit;
 
@@ -1177,53 +1175,13 @@ static void digi_close(struct usb_serial_port *port)
 			TASK_INTERRUPTIBLE);
 	schedule_timeout(DIGI_CLOSE_TIMEOUT);
 	finish_wait(&priv->dp_flush_wait, &wait);
-
-	/* shutdown any outstanding bulk writes */
-	usb_kill_urb(port->write_urb);
 exit:
-	spin_lock_irq(&priv->dp_port_lock);
-	priv->dp_write_urb_in_use = 0;
-	wake_up_interruptible(&priv->dp_close_wait);
-	spin_unlock_irq(&priv->dp_port_lock);
 	mutex_unlock(&port->serial->disc_mutex);
-}
-
-
-/*
- *  Digi Startup Device
- *
- *  Starts reads on all ports.  Must be called AFTER startup, with
- *  urbs initialized.  Returns 0 if successful, non-zero error otherwise.
- */
 
-static int digi_startup_device(struct usb_serial *serial)
-{
-	int i, ret = 0;
-	struct digi_serial *serial_priv = usb_get_serial_data(serial);
-	struct usb_serial_port *port;
+	/* shutdown any outstanding bulk writes */
+	usb_kill_urb(port->write_urb);
 
-	/* be sure this happens exactly once */
-	spin_lock(&serial_priv->ds_serial_lock);
-	if (serial_priv->ds_device_started) {
-		spin_unlock(&serial_priv->ds_serial_lock);
-		return 0;
-	}
-	serial_priv->ds_device_started = 1;
-	spin_unlock(&serial_priv->ds_serial_lock);
-
-	/* start reading from each bulk in endpoint for the device */
-	/* set USB_DISABLE_SPD flag for write bulk urbs */
-	for (i = 0; i < serial->type->num_ports + 1; i++) {
-		port = serial->port[i];
-		ret = usb_submit_urb(port->read_urb, GFP_KERNEL);
-		if (ret != 0) {
-			dev_err(&port->dev,
-				"%s: usb_submit_urb failed, ret=%d, port=%d\n",
-				__func__, ret, i);
-			break;
-		}
-	}
-	return ret;
+	digi_close_oob_port(port->serial);
 }
 
 static int digi_port_init(struct usb_serial_port *port, unsigned port_num)
@@ -1238,7 +1196,6 @@ static int digi_port_init(struct usb_serial_port *port, unsigned port_num)
 	priv->dp_port_num = port_num;
 	init_waitqueue_head(&priv->dp_transmit_idle_wait);
 	init_waitqueue_head(&priv->dp_flush_wait);
-	init_waitqueue_head(&priv->dp_close_wait);
 	init_waitqueue_head(&priv->write_wait);
 	priv->dp_port = port;
 
@@ -1275,7 +1232,8 @@ static int digi_startup(struct usb_serial *serial)
 	if (!serial_priv)
 		return -ENOMEM;
 
-	spin_lock_init(&serial_priv->ds_serial_lock);
+	mutex_init(&serial_priv->open_mutex);
+
 	serial_priv->ds_oob_port_num = oob_port_num;
 	serial_priv->ds_oob_port = serial->port[oob_port_num];
 
@@ -1291,19 +1249,6 @@ static int digi_startup(struct usb_serial *serial)
 	return 0;
 }
 
-
-static void digi_disconnect(struct usb_serial *serial)
-{
-	int i;
-
-	/* stop reads and writes on all ports */
-	for (i = 0; i < serial->type->num_ports + 1; i++) {
-		usb_kill_urb(serial->port[i]->read_urb);
-		usb_kill_urb(serial->port[i]->write_urb);
-	}
-}
-
-
 static void digi_release(struct usb_serial *serial)
 {
 	struct digi_serial *serial_priv;
@@ -1333,30 +1278,25 @@ static void digi_port_remove(struct usb_serial_port *port)
 static void digi_read_bulk_callback(struct urb *urb)
 {
 	struct usb_serial_port *port = urb->context;
-	struct digi_port *priv;
-	struct digi_serial *serial_priv;
-	int ret;
+	struct usb_serial *serial = port->serial;
+	struct digi_serial *serial_priv = usb_get_serial_data(serial);
+	struct digi_port *priv = usb_get_serial_port_data(port);
 	int status = urb->status;
-
-	/* port sanity check, do not resubmit if port is not valid */
-	if (port == NULL)
-		return;
-	priv = usb_get_serial_port_data(port);
-	if (priv == NULL) {
-		dev_err(&port->dev, "%s: port->private is NULL, status=%d\n",
-			__func__, status);
-		return;
-	}
-	if (port->serial == NULL ||
-		(serial_priv = usb_get_serial_data(port->serial)) == NULL) {
-		dev_err(&port->dev, "%s: serial is bad or serial->private "
-			"is NULL, status=%d\n", __func__, status);
-		return;
-	}
+	int ret;
 
 	/* do not resubmit urb if it has any status error */
-	if (status) {
-		dev_err(&port->dev,
+	switch (status) {
+	case 0:
+		break;
+	case -ENOENT:
+	case -ECONNRESET:
+	case -ESHUTDOWN:
+		dev_err(&serial->interface->dev,
+			"%s: nonzero read bulk status: status=%d, port=%d\n",
+			__func__, status, priv->dp_port_num);
+		return;
+	default:
+		dev_err(&serial->interface->dev,
 			"%s: nonzero read bulk status: status=%d, port=%d\n",
 			__func__, status, priv->dp_port_num);
 		return;
@@ -1374,11 +1314,10 @@ static void digi_read_bulk_callback(struct urb *urb)
 	/* continue read */
 	ret = usb_submit_urb(urb, GFP_ATOMIC);
 	if (ret != 0 && ret != -EPERM) {
-		dev_err(&port->dev,
+		dev_err(&serial->interface->dev,
 			"%s: failed resubmitting urb, ret=%d, port=%d\n",
 			__func__, ret, priv->dp_port_num);
 	}
-
 }
 
 /*
@@ -1390,7 +1329,6 @@ static void digi_read_bulk_callback(struct urb *urb)
  *  It returns 0 if successful, 1 if successful but the port is
  *  throttled, and -1 if the sanity checks failed.
  */
-
 static int digi_read_inb_callback(struct urb *urb)
 {
 	struct usb_serial_port *port = urb->context;
@@ -1468,10 +1406,8 @@ static int digi_read_inb_callback(struct urb *urb)
 		dev_dbg(&port->dev, "%s: unknown opcode: %d\n", __func__, opcode);
 
 	return throttled ? 1 : 0;
-
 }
 
-
 /*
  *  Digi Read OOB Callback
  *
@@ -1480,10 +1416,8 @@ static int digi_read_inb_callback(struct urb *urb)
  *  the port->serial is valid.  It returns 0 if successful, and
  *  -1 if the sanity checks failed.
  */
-
 static int digi_read_oob_callback(struct urb *urb)
 {
-
 	struct usb_serial_port *port = urb->context;
 	struct usb_serial *serial = port->serial;
 	struct tty_struct *tty;
@@ -1491,8 +1425,8 @@ static int digi_read_oob_callback(struct urb *urb)
 	unsigned char *buf = urb->transfer_buffer;
 	int opcode, line, status, val;
 	unsigned long flags;
-	int i;
 	unsigned int rts;
+	int i;
 
 	if (urb->actual_length < 4)
 		return -1;
@@ -1504,7 +1438,8 @@ static int digi_read_oob_callback(struct urb *urb)
 		status = buf[i + 2];
 		val = buf[i + 3];
 
-		dev_dbg(&port->dev, "digi_read_oob_callback: opcode=%d, line=%d, status=%d, val=%d\n",
+		dev_dbg(&serial->interface->dev,
+			"digi_read_oob_callback: opcode=%d, line=%d, status=%d, val=%d\n",
 			opcode, line, status, val);
 
 		if (status != 0 || line >= serial->type->num_ports)
@@ -1562,8 +1497,8 @@ static int digi_read_oob_callback(struct urb *urb)
 		}
 		tty_kref_put(tty);
 	}
-	return 0;
 
+	return 0;
 }
 
 module_usb_serial_driver(serial_drivers, id_table_combined);
diff --git a/drivers/usb/serial/ftdi_sio.c b/drivers/usb/serial/ftdi_sio.c
index 88dd32da82c2b..c6ffa23bcc8f7 100644
--- a/drivers/usb/serial/ftdi_sio.c
+++ b/drivers/usb/serial/ftdi_sio.c
@@ -1072,6 +1072,8 @@ static const struct usb_device_id id_table_combined[] = {
 	{ USB_DEVICE_INTERFACE_NUMBER(ALTERA_VID, ALTERA_UB3_602E_PID, 3) },
 	/* Abacus Electrics */
 	{ USB_DEVICE(FTDI_VID, ABACUS_OPTICAL_PROBE_PID) },
+	/* Endress+Hauser AG devices */
+	{ USB_DEVICE(FTDI_VID, FTDI_EH_FXA291_PID) },
 	{ }					/* Terminating entry */
 };
 
diff --git a/drivers/usb/serial/ftdi_sio_ids.h b/drivers/usb/serial/ftdi_sio_ids.h
index 6c76cfebfd0e4..9c83c17853c87 100644
--- a/drivers/usb/serial/ftdi_sio_ids.h
+++ b/drivers/usb/serial/ftdi_sio_ids.h
@@ -313,6 +313,11 @@
 #define FTDI_ELV_UDF77_PID	0xFB5E	/* USB DCF Funkuhr (UDF 77) */
 #define FTDI_ELV_UIO88_PID	0xFB5F	/* USB-I/O Interface (UIO 88) */
 
+/*
+ * Endress+Hauser AG product ids (FTDI_VID)
+ */
+#define FTDI_EH_FXA291_PID	0xE510
+
 /*
  * EVER Eco Pro UPS (http://www.ever.com.pl/)
  */
diff --git a/drivers/usb/serial/keyspan_pda.c b/drivers/usb/serial/keyspan_pda.c
index f05bcce606002..4ee0bdf98c0e1 100644
--- a/drivers/usb/serial/keyspan_pda.c
+++ b/drivers/usb/serial/keyspan_pda.c
@@ -31,8 +31,9 @@
 struct keyspan_pda_private {
 	int			tx_room;
 	struct work_struct	unthrottle_work;
-	struct usb_serial	*serial;
 	struct usb_serial_port	*port;
+	bool			throttled;
+	bool			throttle_req;
 };
 
 static int keyspan_pda_write_start(struct usb_serial_port *port);
@@ -148,6 +149,7 @@ static void keyspan_pda_rx_interrupt(struct urb *urb)
 	int retval;
 	int status = urb->status;
 	struct keyspan_pda_private *priv;
+	bool throttled = false;
 	unsigned long flags;
 
 	priv = usb_get_serial_port_data(port);
@@ -209,16 +211,24 @@ static void keyspan_pda_rx_interrupt(struct urb *urb)
 	}
 
 exit:
-	retval = usb_submit_urb(urb, GFP_ATOMIC);
-	if (retval)
-		dev_err(&port->dev,
-			"%s - usb_submit_urb failed with result %d\n",
-			__func__, retval);
+	spin_lock_irqsave(&port->lock, flags);
+	if (priv->throttle_req) {
+		priv->throttled = true;
+		throttled = true;
+	}
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	if (!throttled) {
+		retval = usb_submit_urb(urb, GFP_ATOMIC);
+		if (retval)
+			dev_err(&port->dev, "failed to resubmit in urb: %d\n", retval);
+	}
 }
 
 static void keyspan_pda_rx_throttle(struct tty_struct *tty)
 {
 	struct usb_serial_port *port = tty->driver_data;
+	struct keyspan_pda_private *priv = usb_get_serial_port_data(port);
 
 	/*
 	 * Stop receiving characters. We just turn off the URB request, and
@@ -228,16 +238,29 @@ static void keyspan_pda_rx_throttle(struct tty_struct *tty)
 	 * send an XOFF, although it might make sense to foist that off upon
 	 * the device too.
 	 */
-	usb_kill_urb(port->interrupt_in_urb);
+	spin_lock_irq(&port->lock);
+	priv->throttle_req = true;
+	spin_unlock_irq(&port->lock);
 }
 
 static void keyspan_pda_rx_unthrottle(struct tty_struct *tty)
 {
 	struct usb_serial_port *port = tty->driver_data;
+	struct keyspan_pda_private *priv = usb_get_serial_port_data(port);
+	bool throttled;
+	int ret;
 
-	/* just restart the receive interrupt URB */
-	if (usb_submit_urb(port->interrupt_in_urb, GFP_KERNEL))
-		dev_dbg(&port->dev, "usb_submit_urb(read urb) failed\n");
+	spin_lock_irq(&port->lock);
+	throttled = priv->throttled;
+	priv->throttled = false;
+	priv->throttle_req = false;
+	spin_unlock_irq(&port->lock);
+
+	if (throttled) {
+		ret = usb_submit_urb(port->interrupt_in_urb, GFP_KERNEL);
+		if (ret)
+			dev_err(&port->dev, "failed to submit in urb: %d\n", ret);
+	}
 }
 
 static speed_t keyspan_pda_setbaud(struct usb_serial *serial, speed_t baud)
@@ -577,6 +600,8 @@ static int keyspan_pda_open(struct tty_struct *tty,
 
 	spin_lock_irq(&port->lock);
 	priv->tx_room = rc;
+	priv->throttled = false;
+	priv->throttle_req = false;
 	spin_unlock_irq(&port->lock);
 
 	rc = usb_submit_urb(port->interrupt_in_urb, GFP_KERNEL);
diff --git a/drivers/usb/serial/metro-usb.c b/drivers/usb/serial/metro-usb.c
index 35473544f1c83..22c5f071f116d 100644
--- a/drivers/usb/serial/metro-usb.c
+++ b/drivers/usb/serial/metro-usb.c
@@ -36,6 +36,7 @@
 struct metrousb_private {
 	spinlock_t lock;
 	int throttled;
+	int throttle_req;
 	unsigned long control_state;
 };
 
@@ -108,7 +109,7 @@ static void metrousb_read_int_callback(struct urb *urb)
 	unsigned char *data = urb->transfer_buffer;
 	unsigned long flags;
 	int throttled = 0;
-	int result = 0;
+	int result;
 
 	dev_dbg(&port->dev, "%s\n", __func__);
 
@@ -143,7 +144,10 @@ static void metrousb_read_int_callback(struct urb *urb)
 
 	/* Set any port variables. */
 	spin_lock_irqsave(&metro_priv->lock, flags);
-	throttled = metro_priv->throttled;
+	if (metro_priv->throttle_req) {
+		metro_priv->throttled = 1;
+		throttled = 1;
+	}
 	spin_unlock_irqrestore(&metro_priv->lock, flags);
 
 	if (throttled)
@@ -169,12 +173,13 @@ static int metrousb_open(struct tty_struct *tty, struct usb_serial_port *port)
 	struct usb_serial *serial = port->serial;
 	struct metrousb_private *metro_priv = usb_get_serial_port_data(port);
 	unsigned long flags;
-	int result = 0;
+	int result;
 
 	/* Set the private data information for the port. */
 	spin_lock_irqsave(&metro_priv->lock, flags);
 	metro_priv->control_state = 0;
 	metro_priv->throttled = 0;
+	metro_priv->throttle_req = 0;
 	spin_unlock_irqrestore(&metro_priv->lock, flags);
 
 	/* Clear the urb pipe. */
@@ -269,7 +274,7 @@ static void metrousb_throttle(struct tty_struct *tty)
 
 	/* Set the private information for the port to stop reading data. */
 	spin_lock_irqsave(&metro_priv->lock, flags);
-	metro_priv->throttled = 1;
+	metro_priv->throttle_req = 1;
 	spin_unlock_irqrestore(&metro_priv->lock, flags);
 }
 
@@ -321,19 +326,23 @@ static void metrousb_unthrottle(struct tty_struct *tty)
 	struct usb_serial_port *port = tty->driver_data;
 	struct metrousb_private *metro_priv = usb_get_serial_port_data(port);
 	unsigned long flags;
-	int result = 0;
+	int throttled;
+	int result;
 
 	/* Set the private information for the port to resume reading data. */
 	spin_lock_irqsave(&metro_priv->lock, flags);
+	throttled = metro_priv->throttled;
 	metro_priv->throttled = 0;
+	metro_priv->throttle_req = 0;
 	spin_unlock_irqrestore(&metro_priv->lock, flags);
 
-	/* Submit the urb to read from the port. */
-	result = usb_submit_urb(port->interrupt_in_urb, GFP_ATOMIC);
-	if (result)
-		dev_err(&port->dev,
-			"failed submitting interrupt in urb error code=%d\n",
-			result);
+	if (throttled) {
+		result = usb_submit_urb(port->interrupt_in_urb, GFP_KERNEL);
+		if (result) {
+			dev_err(&port->dev, "failed to submit interrupt in urb: %d\n",
+				result);
+		}
+	}
 }
 
 static struct usb_serial_driver metrousb_device = {
diff --git a/drivers/usb/serial/spcp8x5.c b/drivers/usb/serial/spcp8x5.c
index c11d64bf08fba..0e7715a02df4c 100644
--- a/drivers/usb/serial/spcp8x5.c
+++ b/drivers/usb/serial/spcp8x5.c
@@ -133,14 +133,14 @@ struct spcp8x5_private {
 static int spcp8x5_probe(struct usb_serial *serial,
 						const struct usb_device_id *id)
 {
-	usb_set_serial_data(serial, (void *)id);
+	usb_set_serial_data(serial, (void *)id->driver_info);
 
 	return 0;
 }
 
 static int spcp8x5_port_probe(struct usb_serial_port *port)
 {
-	const struct usb_device_id *id = usb_get_serial_data(port->serial);
+	unsigned int quirks = (unsigned int)(unsigned long)usb_get_serial_data(port->serial);
 	struct spcp8x5_private *priv;
 
 	priv = kzalloc_obj(*priv);
@@ -148,7 +148,7 @@ static int spcp8x5_port_probe(struct usb_serial_port *port)
 		return -ENOMEM;
 
 	spin_lock_init(&priv->lock);
-	priv->quirks = id->driver_info;
+	priv->quirks = quirks;
 
 	usb_set_serial_port_data(port, priv);
 
diff --git a/drivers/usb/storage/unusual_devs.h b/drivers/usb/storage/unusual_devs.h
index 255968f9ca42a..ac22fa3187343 100644
--- a/drivers/usb/storage/unusual_devs.h
+++ b/drivers/usb/storage/unusual_devs.h
@@ -395,6 +395,13 @@ UNUSUAL_DEV(  0x04b3, 0x4001, 0x0110, 0x0110,
 		USB_SC_DEVICE, USB_PR_CB, NULL,
 		US_FL_MAX_SECTORS_MIN),
 
+/* Reported by Ai Chao <aichao@kylinos.cn> */
+UNUSUAL_DEV(  0x04b4, 0xb708, 0x0000, 0xffff,
+		"Longmai Technologies",
+		"USB Key",
+		USB_SC_SCSI, USB_PR_BULK, NULL,
+		US_FL_NO_ATA_1X),
+
 /*
  * Reported by Simon Levitt <simon@whattf.com>
  * This entry needs Sub and Proto fields
diff --git a/drivers/usb/typec/altmodes/displayport.c b/drivers/usb/typec/altmodes/displayport.c
index 263a89c5f3243..51c92dd887aba 100644
--- a/drivers/usb/typec/altmodes/displayport.c
+++ b/drivers/usb/typec/altmodes/displayport.c
@@ -764,6 +764,7 @@ int dp_altmode_probe(struct typec_altmode *alt)
 	struct typec_altmode *plug = typec_altmode_get_plug(alt, TYPEC_PLUG_SOP_P);
 	struct fwnode_handle *fwnode;
 	struct dp_altmode *dp;
+	u32 cap = DP_CAP_CAPABILITY(alt->vdo);
 
 	/* Port can only be DFP_U. */
 	if (typec_altmode_get_data_role(alt) != TYPEC_HOST)
@@ -778,6 +779,18 @@ int dp_altmode_probe(struct typec_altmode *alt)
 		return -ENODEV;
 	}
 
+	/*
+	 * Make sure the DisplayPort VDO is valid (VESA DPAM v2.1a, Section
+	 * 5.4.1, Table 5-6, DP Capabilities VDO). A device exposing DP on a
+	 * USB-C receptacle must advertise at least one pin assignment for the
+	 * capability it claims, otherwise Alt Mode can never be configured.
+	 */
+	if ((cap == DP_CAP_DFP_D && !DP_CAP_PIN_ASSIGN_DFP_D(alt->vdo)) ||
+	    (cap == DP_CAP_UFP_D && !DP_CAP_PIN_ASSIGN_UFP_D(alt->vdo))) {
+		typec_altmode_put_plug(plug);
+		return -ENODEV;
+	}
+
 	dp = devm_kzalloc(&alt->dev, sizeof(*dp), GFP_KERNEL);
 	if (!dp) {
 		typec_altmode_put_plug(plug);
@@ -790,7 +803,6 @@ int dp_altmode_probe(struct typec_altmode *alt)
 	dp->alt = alt;
 
 	alt->desc = "DisplayPort";
-	typec_altmode_set_ops(alt, &dp_altmode_ops);
 
 	if (plug) {
 		plug->desc = "Displayport";
@@ -811,6 +823,10 @@ int dp_altmode_probe(struct typec_altmode *alt)
 	if (plug)
 		typec_altmode_set_drvdata(plug, dp);
 
+	if ((alt->vdo & DP_CAP_RECEPTACLE) && typec_cable_altmode_unsupported(alt))
+		return 0;
+
+	typec_altmode_set_ops(alt, &dp_altmode_ops);
 	if (!alt->mode_selection) {
 		dp->state = plug ? DP_STATE_ENTER_PRIME : DP_STATE_ENTER;
 		schedule_work(&dp->work);
diff --git a/drivers/usb/typec/altmodes/thunderbolt.c b/drivers/usb/typec/altmodes/thunderbolt.c
index 32250b94262a9..2eccdddf1b1f4 100644
--- a/drivers/usb/typec/altmodes/thunderbolt.c
+++ b/drivers/usb/typec/altmodes/thunderbolt.c
@@ -284,6 +284,10 @@ static int tbt_altmode_probe(struct typec_altmode *alt)
 
 	alt->desc = "Thunderbolt3";
 	typec_altmode_set_drvdata(alt, tbt);
+
+	if (typec_cable_altmode_unsupported(alt))
+		return 0;
+
 	typec_altmode_set_ops(alt, &tbt_altmode_ops);
 
 	if (!alt->mode_selection && tbt_ready(alt)) {
diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c
index 0595e8cb83aa6..54c7810b563be 100644
--- a/drivers/usb/typec/class.c
+++ b/drivers/usb/typec/class.c
@@ -1429,6 +1429,77 @@ int typec_cable_is_active(struct typec_cable *cable)
 }
 EXPORT_SYMBOL_GPL(typec_cable_is_active);
 
+enum typec_cable_altmode_support {
+	CABLE_SUPPORT_UNKNOWN,
+	CABLE_SUPPORTED,
+	CABLE_NOT_SUPPORTED,
+};
+
+static enum typec_cable_altmode_support
+typec_cable_check_altmode_support(struct typec_cable *cable,
+				  struct typec_altmode *alt)
+{
+	struct typec_altmode *plug;
+	u32 speed;
+
+	/*
+	 * Check if the cable has an e-marker, supports modal operation, and the
+	 * SOP' altmode nodes are created.
+	 */
+	plug = typec_altmode_get_plug(alt, TYPEC_PLUG_SOP_P);
+	if (plug) {
+		typec_altmode_put_plug(plug);
+		return CABLE_SUPPORTED;
+	}
+
+	/* The identity is not specified */
+	if (!cable->identity)
+		return CABLE_SUPPORT_UNKNOWN;
+
+	/* Non-e-marked cable */
+	if (!cable->identity->id_header)
+		return CABLE_NOT_SUPPORTED;
+
+	switch (PD_IDH_PTYPE(cable->identity->id_header)) {
+	case IDH_PTYPE_PCABLE:
+		speed = VDO_TYPEC_CABLE_SPEED(cable->identity->vdo[0]);
+		if (speed == CABLE_USB2_ONLY)
+			return CABLE_NOT_SUPPORTED;
+		return CABLE_SUPPORTED;
+	case IDH_PTYPE_ACABLE:
+		/*
+		 * Active cables must establish an SOP' communication
+		 * node. Since that check failed at the beginning of
+		 * this function, this active cable does not support
+		 * this specific altmode.
+		 */
+		return CABLE_NOT_SUPPORTED;
+	}
+
+	return CABLE_SUPPORT_UNKNOWN;
+}
+
+/**
+ * typec_cable_altmode_unsupported - Check if a cable restricts altmode
+ * @alt: The Alternate Mode to evaluate
+ *
+ * Returns true if the connected cable is incapable of handling the altmode.
+ */
+bool typec_cable_altmode_unsupported(struct typec_altmode *alt)
+{
+	enum typec_cable_altmode_support support = CABLE_SUPPORT_UNKNOWN;
+	struct typec_cable *cable;
+
+	cable = typec_cable_get(typec_altmode2port(alt));
+	if (cable) {
+		support = typec_cable_check_altmode_support(cable, alt);
+		typec_cable_put(cable);
+	}
+
+	return support == CABLE_NOT_SUPPORTED;
+}
+EXPORT_SYMBOL_GPL(typec_cable_altmode_unsupported);
+
 /**
  * typec_cable_set_identity - Report result from Discover Identity command
  * @cable: The cable updated identity values
diff --git a/drivers/usb/typec/mux/tusb1046.c b/drivers/usb/typec/mux/tusb1046.c
index d6e1289a49450..34ef81f75693e 100644
--- a/drivers/usb/typec/mux/tusb1046.c
+++ b/drivers/usb/typec/mux/tusb1046.c
@@ -178,6 +178,7 @@ static const struct of_device_id tusb1046_match_table[] = {
 	{.compatible = "ti,tusb1046"},
 	{},
 };
+MODULE_DEVICE_TABLE(of, tusb1046_match_table);
 
 static struct i2c_driver tusb1046_driver = {
 	.driver = {
diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c
index 89eec20a2064c..5e652f3d449bd 100644
--- a/drivers/usb/typec/tcpm/tcpm.c
+++ b/drivers/usb/typec/tcpm/tcpm.c
@@ -2034,7 +2034,7 @@ static void tcpm_register_partner_altmodes(struct tcpm_port *port)
 	if (!port->partner)
 		return;
 
-	for (i = 0; i < modep->altmodes; i++) {
+	for (i = 0; i < modep->altmodes && i < ALTMODE_DISCOVERY_MAX; i++) {
 		altmode = typec_partner_register_altmode(port->partner,
 						&modep->altmode_desc[i]);
 		if (IS_ERR(altmode)) {
@@ -2052,9 +2052,10 @@ static void tcpm_register_plug_altmodes(struct tcpm_port *port)
 	struct typec_altmode *altmode;
 	int i;
 
-	typec_plug_set_num_altmodes(port->plug_prime, modep->altmodes);
+	typec_plug_set_num_altmodes(port->plug_prime,
+				    min(modep->altmodes, ALTMODE_DISCOVERY_MAX));
 
-	for (i = 0; i < modep->altmodes; i++) {
+	for (i = 0; i < modep->altmodes && i < ALTMODE_DISCOVERY_MAX; i++) {
 		altmode = typec_plug_register_altmode(port->plug_prime,
 						&modep->altmode_desc[i]);
 		if (IS_ERR(altmode)) {
@@ -4896,11 +4897,11 @@ static void tcpm_unregister_altmodes(struct tcpm_port *port)
 	struct pd_mode_data *modep_prime = &port->mode_data_prime;
 	int i;
 
-	for (i = 0; i < modep->altmodes; i++) {
+	for (i = 0; i < modep->altmodes && i < ALTMODE_DISCOVERY_MAX; i++) {
 		typec_unregister_altmode(port->partner_altmode[i]);
 		port->partner_altmode[i] = NULL;
 	}
-	for (i = 0; i < modep_prime->altmodes; i++) {
+	for (i = 0; i < modep_prime->altmodes && i < ALTMODE_DISCOVERY_MAX; i++) {
 		typec_unregister_altmode(port->plug_prime_altmode[i]);
 		port->plug_prime_altmode[i] = NULL;
 	}
diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c
index d5ee0af9058b9..b6335b36d3846 100644
--- a/drivers/usb/typec/tipd/core.c
+++ b/drivers/usb/typec/tipd/core.c
@@ -1744,7 +1744,7 @@ static int tps6598x_probe(struct i2c_client *client)
 	struct tps6598x *tps;
 	struct fwnode_handle *fwnode;
 	u32 status;
-	u32 vid;
+	u32 vid = 0;
 	int ret;
 
 	data = i2c_get_match_data(client);
@@ -1772,8 +1772,11 @@ static int tps6598x_probe(struct i2c_client *client)
 
 	if (!device_is_compatible(tps->dev, "ti,tps25750")) {
 		ret = tps6598x_read32(tps, TPS_REG_VID, &vid);
-		if (ret < 0 || !vid)
+		if (ret < 0 || !vid) {
+			dev_err(tps->dev, "failed to read vendor ID: %d, vid: %#x\n",
+				ret, vid);
 			return -ENODEV;
+		}
 	}
 
 	/*
@@ -1851,7 +1854,7 @@ static int tps6598x_probe(struct i2c_client *client)
 						IRQF_SHARED | IRQF_ONESHOT,
 						dev_name(&client->dev), tps);
 	} else {
-		dev_warn(tps->dev, "Unable to find the interrupt, switching to polling\n");
+		dev_dbg(tps->dev, "no IRQ specified, using polling mode\n");
 		INIT_DELAYED_WORK(&tps->wq_poll, tps6598x_poll_work);
 		queue_delayed_work(system_power_efficient_wq, &tps->wq_poll,
 				   msecs_to_jiffies(POLL_INTERVAL));
diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h
index 03edbb77bbd6d..d4140f4da5bb9 100644
--- a/drivers/usb/typec/tipd/tps6598x.h
+++ b/drivers/usb/typec/tipd/tps6598x.h
@@ -142,9 +142,13 @@
 #define TPS_SYSTEM_POWER_STATE_S4	0x04
 #define TPS_SYSTEM_POWER_STATE_S5	0x05
 
-/* TPS_REG_POWER_STATUS bits */
-#define TPS_POWER_STATUS_CONNECTION(x)  TPS_FIELD_GET(BIT(0), (x))
-#define TPS_POWER_STATUS_SOURCESINK(x)	TPS_FIELD_GET(BIT(1), (x))
+/* TPS_REG_POWER_STATUS bits (masks shared by TPS_FIELD_GET accessors and FIELD_PREP) */
+#define TPS_POWER_STATUS_CONNECTION_MASK	BIT(0)
+#define TPS_POWER_STATUS_SOURCESINK_MASK	BIT(1)
+#define TPS_POWER_STATUS_CONNECTION(x) \
+	TPS_FIELD_GET(TPS_POWER_STATUS_CONNECTION_MASK, (x))
+#define TPS_POWER_STATUS_SOURCESINK(x) \
+	TPS_FIELD_GET(TPS_POWER_STATUS_SOURCESINK_MASK, (x))
 #define TPS_POWER_STATUS_BC12_DET(x)	TPS_FIELD_GET(BIT(2), (x))
 
 #define TPS_POWER_STATUS_TYPEC_CURRENT_MASK GENMASK(3, 2)
diff --git a/drivers/usb/typec/ucsi/debugfs.c b/drivers/usb/typec/ucsi/debugfs.c
index ff33a5e7c6b0d..77a0dd75edd3c 100644
--- a/drivers/usb/typec/ucsi/debugfs.c
+++ b/drivers/usb/typec/ucsi/debugfs.c
@@ -82,8 +82,8 @@ static int ucsi_resp_show(struct seq_file *s, void *not_used)
 	if (ucsi->debugfs->status)
 		return ucsi->debugfs->status;
 
-	seq_printf(s, "0x%016llx%016llx\n", ucsi->debugfs->response.high,
-		   ucsi->debugfs->response.low);
+	seq_printf(s, "0x%016llx%016llx%016llx\n", ucsi->debugfs->response.ext,
+		   ucsi->debugfs->response.high, ucsi->debugfs->response.low);
 	return 0;
 }
 DEFINE_SHOW_ATTRIBUTE(ucsi_resp);
@@ -162,6 +162,7 @@ void ucsi_debugfs_unregister(struct ucsi *ucsi)
 
 	debugfs_remove_recursive(ucsi->debugfs->dentry);
 	kfree(ucsi->debugfs);
+	ucsi->debugfs = NULL;
 }
 
 void ucsi_debugfs_init(void)
diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c
index 6a6723e8fb127..d221ca9bc88ec 100644
--- a/drivers/usb/typec/ucsi/ucsi.c
+++ b/drivers/usb/typec/ucsi/ucsi.c
@@ -529,6 +529,129 @@ static int ucsi_register_altmode(struct ucsi_connector *con,
 	return ret;
 }
 
+static void ucsi_dump_duplicate_altmode(struct ucsi_connector *con,
+					u8 recipient, u16 svid,
+					u32 existing_vdo, u32 new_vdo,
+					int offset)
+{
+	static const char * const recipient_names[] = {
+		[UCSI_RECIPIENT_CON]    = "port",
+		[UCSI_RECIPIENT_SOP]    = "partner",
+		[UCSI_RECIPIENT_SOP_P]  = "plug",
+		[UCSI_RECIPIENT_SOP_PP] = "cable plug prime",
+	};
+
+	dev_warn(con->ucsi->dev,
+		 "con%d: Firmware bug: duplicate %s altmode SVID 0x%04x at offset %d, ignoring but please contact the BIOS vendor to fix this issue.\n",
+		 con->num, recipient_names[recipient], svid, offset);
+
+	if (existing_vdo != new_vdo)
+		dev_warn(con->ucsi->dev,
+			 "con%d: VDO mismatch: 0x%08x vs 0x%08x\n",
+			 con->num, existing_vdo, new_vdo);
+}
+
+/* Count altmodes in @altmodes that advertise @svid. */
+static int ucsi_altmode_count_svid(struct typec_altmode **altmodes, u16 svid)
+{
+	int count = 0;
+	int k;
+
+	for (k = 0; k < UCSI_MAX_ALTMODES; k++) {
+		if (!altmodes[k])
+			break;
+		if (altmodes[k]->svid == svid)
+			count++;
+	}
+
+	return count;
+}
+
+/*
+ * Check if an altmode is a duplicate. Some firmware implementations
+ * incorrectly return the same altmode multiple times, causing sysfs errors.
+ * Returns true if the altmode should be skipped.
+ *
+ * The matching rules differ by recipient:
+ *
+ *   - UCSI_RECIPIENT_CON (port) and UCSI_RECIPIENT_SOP_P (plug):
+ *     Two altmodes with identical SVID and VDO are byte-for-byte duplicates
+ *     and the second has no observable function. Drop them.
+ *
+ *   - UCSI_RECIPIENT_SOP (partner):
+ *     The typec class binds each partner altmode to a port altmode of the
+ *     same SVID via altmode_match()/device_find_child(), which returns the
+ *     first port altmode with a matching SVID. If the partner advertises
+ *     more altmodes for SVID X than the port advertises, the surplus
+ *     partner altmode(s) collapse onto an already-paired port altmode and
+ *     trigger a "duplicate filename .../partner" sysfs error during
+ *     typec_altmode_create_links(). Use the port-side altmode count for
+ *     SVID X as the authoritative cap and reject any partner altmode that
+ *     would exceed it. This preserves legitimate multi-Mode partner
+ *     altmodes (e.g. vendor SVIDs that the port really does advertise
+ *     twice) while filtering the firmware-generated duplicates that have
+ *     no port counterpart.
+ */
+static bool ucsi_altmode_is_duplicate(struct ucsi_connector *con, u8 recipient,
+				      const struct ucsi_altmode *alt_batch, int batch_idx,
+				      u16 svid, u32 vdo, int offset)
+{
+	struct typec_altmode **altmodes;
+	int port_count, partner_count;
+	int k;
+
+	/* Check for duplicates within the current batch first */
+	for (k = 0; k < batch_idx; k++) {
+		if (alt_batch[k].svid == svid && alt_batch[k].mid == vdo) {
+			ucsi_dump_duplicate_altmode(con, recipient, svid,
+						    vdo, vdo, offset);
+			return true;
+		}
+	}
+
+	switch (recipient) {
+	case UCSI_RECIPIENT_SOP:
+		/*
+		 * Cap partner altmodes per SVID by the port-side count:
+		 * any further partner altmode for that SVID would alias an
+		 * already-paired port altmode and break typec sysfs.
+		 */
+		port_count = ucsi_altmode_count_svid(con->port_altmode, svid);
+		partner_count = ucsi_altmode_count_svid(con->partner_altmode,
+							svid);
+		if (port_count && partner_count >= port_count) {
+			ucsi_dump_duplicate_altmode(con, recipient, svid,
+						    con->partner_altmode[partner_count - 1]->vdo,
+						    vdo, offset);
+			return true;
+		}
+		return false;
+	case UCSI_RECIPIENT_CON:
+		altmodes = con->port_altmode;
+		break;
+	case UCSI_RECIPIENT_SOP_P:
+		altmodes = con->plug_altmode;
+		break;
+	default:
+		return false;
+	}
+
+	/* CON and SOP_P: drop only exact SVID+VDO duplicates. */
+	for (k = 0; k < UCSI_MAX_ALTMODES; k++) {
+		if (!altmodes[k])
+			break;
+
+		if (altmodes[k]->svid != svid || altmodes[k]->vdo != vdo)
+			continue;
+
+		ucsi_dump_duplicate_altmode(con, recipient, svid,
+					    altmodes[k]->vdo, vdo, offset);
+		return true;
+	}
+
+	return false;
+}
+
 static int
 ucsi_register_altmodes_nvidia(struct ucsi_connector *con, u8 recipient)
 {
@@ -583,19 +706,25 @@ ucsi_register_altmodes_nvidia(struct ucsi_connector *con, u8 recipient)
 
 	/* now register altmodes */
 	for (i = 0; i < max_altmodes; i++) {
-		memset(&desc, 0, sizeof(desc));
-		if (multi_dp) {
-			desc.svid = updated[i].svid;
-			desc.vdo = updated[i].mid;
-		} else {
-			desc.svid = orig[i].svid;
-			desc.vdo = orig[i].mid;
-		}
-		desc.roles = TYPEC_PORT_DRD;
+		struct ucsi_altmode *altmode_array = multi_dp ? updated : orig;
 
-		if (!desc.svid)
+		if (!altmode_array[i].svid)
 			return 0;
 
+		/*
+		 * Check for duplicates in current array and already
+		 * registered altmodes. Skip if duplicate found.
+		 */
+		if (ucsi_altmode_is_duplicate(con, recipient, altmode_array, i,
+					      altmode_array[i].svid,
+					      altmode_array[i].mid, i))
+			continue;
+
+		memset(&desc, 0, sizeof(desc));
+		desc.svid = altmode_array[i].svid;
+		desc.vdo = altmode_array[i].mid;
+		desc.roles = TYPEC_PORT_DRD;
+
 		ret = ucsi_register_altmode(con, &desc, recipient);
 		if (ret)
 			return ret;
@@ -653,6 +782,15 @@ static int ucsi_register_altmodes(struct ucsi_connector *con, u8 recipient)
 			if (!alt[j].svid)
 				return 0;
 
+			/*
+			 * Check for duplicates in current batch and already
+			 * registered altmodes. Skip if duplicate found.
+			 */
+			if (ucsi_altmode_is_duplicate(con, recipient, alt, j,
+						      alt[j].svid, alt[j].mid,
+						      i - num + j))
+				continue;
+
 			memset(&desc, 0, sizeof(desc));
 			desc.vdo = alt[j].mid;
 			desc.svid = alt[j].svid;
@@ -963,8 +1101,8 @@ static int ucsi_register_plug(struct ucsi_connector *con)
 	plug = typec_register_plug(con->cable, &desc);
 	if (IS_ERR(plug)) {
 		dev_err(con->ucsi->dev,
-			"con%d: failed to register plug (%ld)\n", con->num,
-			PTR_ERR(plug));
+			"con%d: failed to register plug (%pe)\n", con->num,
+			plug);
 		return PTR_ERR(plug);
 	}
 
@@ -1022,8 +1160,8 @@ static int ucsi_register_cable(struct ucsi_connector *con)
 	cable = typec_register_cable(con->port, &desc);
 	if (IS_ERR(cable)) {
 		dev_err(con->ucsi->dev,
-			"con%d: failed to register cable (%ld)\n", con->num,
-			PTR_ERR(cable));
+			"con%d: failed to register cable (%pe)\n", con->num,
+			cable);
 		return PTR_ERR(cable);
 	}
 
@@ -1152,8 +1290,8 @@ static int ucsi_register_partner(struct ucsi_connector *con)
 	partner = typec_register_partner(con->port, &desc);
 	if (IS_ERR(partner)) {
 		dev_err(con->ucsi->dev,
-			"con%d: failed to register partner (%ld)\n", con->num,
-			PTR_ERR(partner));
+			"con%d: failed to register partner (%pe)\n", con->num,
+			partner);
 		return PTR_ERR(partner);
 	}
 
@@ -1845,6 +1983,42 @@ static int ucsi_register_port(struct ucsi *ucsi, struct ucsi_connector *con)
 	return ret;
 }
 
+static void ucsi_unregister_port(struct ucsi_connector *con)
+{
+	struct ucsi_work *uwork;
+
+	if (con->wq) {
+		mutex_lock(&con->lock);
+		ucsi_unregister_partner(con);
+		/*
+		 * queue delayed items immediately so they can execute
+		 * and free themselves before the wq is destroyed
+		 */
+		list_for_each_entry(uwork, &con->partner_tasks, node) {
+			if (cancel_delayed_work(&uwork->work))
+				queue_delayed_work(con->wq, &uwork->work, 0);
+		}
+		mutex_unlock(&con->lock);
+
+		destroy_workqueue(con->wq);
+		con->wq = NULL;
+	} else {
+		ucsi_unregister_partner(con);
+	}
+
+	ucsi_unregister_altmodes(con, UCSI_RECIPIENT_CON);
+	ucsi_unregister_port_psy(con);
+
+	usb_power_delivery_unregister_capabilities(con->port_sink_caps);
+	con->port_sink_caps = NULL;
+	usb_power_delivery_unregister_capabilities(con->port_source_caps);
+	con->port_source_caps = NULL;
+	usb_power_delivery_unregister(con->pd);
+	con->pd = NULL;
+	typec_unregister_port(con->port);
+	con->port = NULL;
+}
+
 static u64 ucsi_get_supported_notifications(struct ucsi *ucsi)
 {
 	u16 features = ucsi->cap.features;
@@ -1971,22 +2145,8 @@ static int ucsi_init(struct ucsi *ucsi)
 	for (i = 0; i < ucsi->cap.num_connectors; i++)
 		lockdep_unregister_key(&connector[i].lock_key);
 
-	for (con = connector; con->port; con++) {
-		if (con->wq)
-			destroy_workqueue(con->wq);
-		ucsi_unregister_partner(con);
-		ucsi_unregister_altmodes(con, UCSI_RECIPIENT_CON);
-		ucsi_unregister_port_psy(con);
-
-		usb_power_delivery_unregister_capabilities(con->port_sink_caps);
-		con->port_sink_caps = NULL;
-		usb_power_delivery_unregister_capabilities(con->port_source_caps);
-		con->port_source_caps = NULL;
-		usb_power_delivery_unregister(con->pd);
-		con->pd = NULL;
-		typec_unregister_port(con->port);
-		con->port = NULL;
-	}
+	for (con = connector; con->port; con++)
+		ucsi_unregister_port(con);
 	kfree(connector);
 err_reset:
 	memset(&ucsi->cap, 0, sizeof(ucsi->cap));
@@ -2060,7 +2220,7 @@ static void ucsi_init_work(struct work_struct *work)
 			return;
 		}
 
-		queue_delayed_work(system_long_wq, &ucsi->work,
+		queue_delayed_work(system_dfl_long_wq, &ucsi->work,
 				   UCSI_ROLE_SWITCH_INTERVAL);
 	}
 }
@@ -2184,7 +2344,7 @@ int ucsi_register(struct ucsi *ucsi)
 		UCSI_BCD_GET_MINOR(ucsi->version),
 		UCSI_BCD_GET_SUBMINOR(ucsi->version));
 
-	queue_delayed_work(system_long_wq, &ucsi->work, 0);
+	queue_delayed_work(system_dfl_long_wq, &ucsi->work, 0);
 
 	ucsi_debugfs_register(ucsi);
 	return 0;
@@ -2206,6 +2366,8 @@ void ucsi_unregister(struct ucsi *ucsi)
 	cancel_delayed_work_sync(&ucsi->work);
 	cancel_work_sync(&ucsi->resume_work);
 
+	ucsi_debugfs_unregister(ucsi);
+
 	/* Disable notifications */
 	ucsi->ops->async_control(ucsi, cmd);
 
@@ -2214,33 +2376,7 @@ void ucsi_unregister(struct ucsi *ucsi)
 
 	for (i = 0; i < ucsi->cap.num_connectors; i++) {
 		cancel_work_sync(&ucsi->connector[i].work);
-
-		if (ucsi->connector[i].wq) {
-			struct ucsi_work *uwork;
-
-			mutex_lock(&ucsi->connector[i].lock);
-			/*
-			 * queue delayed items immediately so they can execute
-			 * and free themselves before the wq is destroyed
-			 */
-			list_for_each_entry(uwork, &ucsi->connector[i].partner_tasks, node)
-				mod_delayed_work(ucsi->connector[i].wq, &uwork->work, 0);
-			mutex_unlock(&ucsi->connector[i].lock);
-			destroy_workqueue(ucsi->connector[i].wq);
-		}
-
-		ucsi_unregister_partner(&ucsi->connector[i]);
-		ucsi_unregister_altmodes(&ucsi->connector[i],
-					 UCSI_RECIPIENT_CON);
-		ucsi_unregister_port_psy(&ucsi->connector[i]);
-
-		usb_power_delivery_unregister_capabilities(ucsi->connector[i].port_sink_caps);
-		ucsi->connector[i].port_sink_caps = NULL;
-		usb_power_delivery_unregister_capabilities(ucsi->connector[i].port_source_caps);
-		ucsi->connector[i].port_source_caps = NULL;
-		usb_power_delivery_unregister(ucsi->connector[i].pd);
-		ucsi->connector[i].pd = NULL;
-		typec_unregister_port(ucsi->connector[i].port);
+		ucsi_unregister_port(&ucsi->connector[i]);
 		lockdep_unregister_key(&ucsi->connector[i].lock_key);
 	}
 
diff --git a/drivers/usb/typec/ucsi/ucsi.h b/drivers/usb/typec/ucsi/ucsi.h
index 6e1608d88ec3d..dc594388dcd08 100644
--- a/drivers/usb/typec/ucsi/ucsi.h
+++ b/drivers/usb/typec/ucsi/ucsi.h
@@ -466,6 +466,7 @@ struct ucsi_debugfs_entry {
 	struct ucsi_data {
 		u64 low;
 		u64 high;
+		u64 ext;
 	} response;
 	int status;
 	u8 message_out[MESSAGE_OUT_MAX_LEN];
diff --git a/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c b/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c
index ca1b534cb183d..57b37fca150c2 100644
--- a/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c
+++ b/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c
@@ -105,6 +105,7 @@ struct gaokun_ucsi {
 	struct notifier_block nb;
 	u16 version;
 	u8 num_ports;
+	bool registered;
 };
 
 /* -------------------------------------------------------------------------- */
@@ -482,8 +483,13 @@ static void gaokun_ucsi_register_worker(struct work_struct *work)
 	}
 
 	ret = ucsi_register(ucsi);
-	if (ret)
+	if (ret) {
 		dev_err_probe(ucsi->dev, ret, "ucsi register failed\n");
+		gaokun_ec_unregister_notify(uec->ec, &uec->nb);
+		return;
+	}
+
+	uec->registered = true;
 }
 
 static int gaokun_ucsi_probe(struct auxiliary_device *adev,
@@ -528,8 +534,11 @@ static void gaokun_ucsi_remove(struct auxiliary_device *adev)
 	int i;
 
 	disable_delayed_work_sync(&uec->work);
-	gaokun_ec_unregister_notify(uec->ec, &uec->nb);
-	ucsi_unregister(uec->ucsi);
+	if (uec->registered) {
+		gaokun_ec_unregister_notify(uec->ec, &uec->nb);
+		ucsi_unregister(uec->ucsi);
+	}
+
 	for (i = 0; i < uec->num_ports; ++i)
 		typec_mux_put(uec->ports[i].typec_mux);
 
diff --git a/drivers/usb/typec/ucsi/ucsi_yoga_c630.c b/drivers/usb/typec/ucsi/ucsi_yoga_c630.c
index 1be18d1018426..bb7b09bc7c1c1 100644
--- a/drivers/usb/typec/ucsi/ucsi_yoga_c630.c
+++ b/drivers/usb/typec/ucsi/ucsi_yoga_c630.c
@@ -139,28 +139,6 @@ static int yoga_c630_ucsi_sync_control(struct ucsi *ucsi,
 	return ret;
 }
 
-static bool yoga_c630_ucsi_update_altmodes(struct ucsi *ucsi,
-					   u8 recipient,
-					   struct ucsi_altmode *orig,
-					   struct ucsi_altmode *updated)
-{
-	int i;
-
-	if (orig[0].svid == 0 || recipient != UCSI_RECIPIENT_SOP)
-		return false;
-
-	/* EC is nice and repeats altmodes again and again. Ignore copies. */
-	for (i = 1; i < UCSI_MAX_ALTMODES; i++) {
-		if (orig[i].svid == orig[0].svid) {
-			dev_dbg(ucsi->dev, "Found duplicate altmodes, starting from %d\n", i);
-			memset(&orig[i], 0, (UCSI_MAX_ALTMODES - i) * sizeof(*orig));
-			break;
-		}
-	}
-
-	return false;
-}
-
 static void yoga_c630_ucsi_update_connector(struct ucsi_connector *con)
 {
 	if (con->num == 1)
@@ -174,7 +152,6 @@ static const struct ucsi_operations yoga_c630_ucsi_ops = {
 	.read_message_in = yoga_c630_ucsi_read_message_in,
 	.sync_control = yoga_c630_ucsi_sync_control,
 	.async_control = yoga_c630_ucsi_async_control,
-	.update_altmodes = yoga_c630_ucsi_update_altmodes,
 	.update_connector = yoga_c630_ucsi_update_connector,
 };
 
diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
index 10dcf016bfb06..eb71b91324ef1 100644
--- a/drivers/vdpa/vdpa_user/vduse_dev.c
+++ b/drivers/vdpa/vdpa_user/vduse_dev.c
@@ -9,6 +9,7 @@
  */
 
 #include "linux/virtio_net.h"
+#include <linux/bits.h>
 #include <linux/cleanup.h>
 #include <linux/init.h>
 #include <linux/module.h>
@@ -51,6 +52,10 @@
 
 #define IRQ_UNBOUND -1
 
+/* Supported VDUSE features */
+static const uint64_t vduse_features = BIT_U64(VDUSE_F_QUEUE_READY) |
+				       BIT_U64(VDUSE_F_SUSPEND);
+
 /*
  * VDUSE instance have not asked the vduse API version, so assume 0.
  *
@@ -80,6 +85,7 @@ struct vduse_virtqueue {
 	int irq_effective_cpu;
 	struct cpumask irq_affinity;
 	struct kobject kobj;
+	struct vduse_dev *dev;
 };
 
 struct vduse_dev;
@@ -116,6 +122,7 @@ struct vduse_dev {
 	char *name;
 	struct mutex lock;
 	spinlock_t msg_lock;
+	u64 vduse_features;
 	u64 msg_unique;
 	u32 msg_timeout;
 	wait_queue_head_t waitq;
@@ -128,6 +135,7 @@ struct vduse_dev {
 	int minor;
 	bool broken;
 	bool connected;
+	bool suspended;
 	u64 api_version;
 	u64 device_features;
 	u64 driver_features;
@@ -163,6 +171,7 @@ static DEFINE_IDR(vduse_idr);
 
 static dev_t vduse_major;
 static struct cdev vduse_ctrl_cdev;
+static const struct device *vduse_ctrl_dev;
 static struct cdev vduse_cdev;
 static struct workqueue_struct *vduse_irq_wq;
 static struct workqueue_struct *vduse_irq_bound_wq;
@@ -496,6 +505,7 @@ static void vduse_dev_reset(struct vduse_dev *dev)
 
 	down_write(&dev->rwsem);
 
+	dev->suspended = false;
 	dev->status = 0;
 	dev->driver_features = 0;
 	dev->generation++;
@@ -550,16 +560,18 @@ static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx,
 
 static void vduse_vq_kick(struct vduse_virtqueue *vq)
 {
-	spin_lock(&vq->kick_lock);
+	guard(rwsem_read)(&vq->dev->rwsem);
+	if (vq->dev->suspended)
+		return;
+
+	guard(spinlock)(&vq->kick_lock);
 	if (!vq->ready)
-		goto unlock;
+		return;
 
 	if (vq->kickfd)
 		eventfd_signal(vq->kickfd);
 	else
 		vq->kicked = true;
-unlock:
-	spin_unlock(&vq->kick_lock);
 }
 
 static void vduse_vq_kick_work(struct work_struct *work)
@@ -619,8 +631,29 @@ static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa,
 {
 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 	struct vduse_virtqueue *vq = dev->vqs[idx];
+	struct vduse_dev_msg msg = { 0 };
+	int r;
 
 	vq->ready = ready;
+
+	if (!(dev->vduse_features & BIT_U64(VDUSE_F_QUEUE_READY)))
+		return;
+
+	msg.req.type = VDUSE_SET_VQ_READY;
+	msg.req.vq_ready.num = idx;
+	msg.req.vq_ready.ready = !!ready;
+
+	r = vduse_dev_msg_sync(dev, &msg);
+
+	if (r < 0) {
+		dev_dbg(&vdpa->dev, "device refuses to set vq %u ready %u",
+			idx, ready);
+
+		/* We can't do better than break the device in this case */
+		spin_lock(&dev->msg_lock);
+		vduse_dev_broken(dev);
+		spin_unlock(&dev->msg_lock);
+	}
 }
 
 static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx)
@@ -891,6 +924,27 @@ static int vduse_vdpa_set_map(struct vdpa_device *vdpa,
 	return 0;
 }
 
+static int vduse_vdpa_suspend(struct vdpa_device *vdpa)
+{
+	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
+	struct vduse_dev_msg msg = { 0 };
+	int ret;
+
+	msg.req.type = VDUSE_SUSPEND;
+
+	ret = vduse_dev_msg_sync(dev, &msg);
+	if (ret == 0) {
+		scoped_guard(rwsem_write, &dev->rwsem)
+			dev->suspended = true;
+
+		cancel_work_sync(&dev->inject);
+		for (u32 i = 0; i < dev->vq_num; i++)
+			cancel_work_sync(&dev->vqs[i]->inject);
+	}
+
+	return ret;
+}
+
 static void vduse_vdpa_free(struct vdpa_device *vdpa)
 {
 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
@@ -932,6 +986,41 @@ static const struct vdpa_config_ops vduse_vdpa_config_ops = {
 	.free			= vduse_vdpa_free,
 };
 
+static const struct vdpa_config_ops vduse_vdpa_config_ops_with_suspend = {
+	.set_vq_address		= vduse_vdpa_set_vq_address,
+	.kick_vq		= vduse_vdpa_kick_vq,
+	.set_vq_cb		= vduse_vdpa_set_vq_cb,
+	.set_vq_num             = vduse_vdpa_set_vq_num,
+	.get_vq_size		= vduse_vdpa_get_vq_size,
+	.get_vq_group		= vduse_get_vq_group,
+	.set_vq_ready		= vduse_vdpa_set_vq_ready,
+	.get_vq_ready		= vduse_vdpa_get_vq_ready,
+	.set_vq_state		= vduse_vdpa_set_vq_state,
+	.get_vq_state		= vduse_vdpa_get_vq_state,
+	.get_vq_align		= vduse_vdpa_get_vq_align,
+	.get_device_features	= vduse_vdpa_get_device_features,
+	.set_driver_features	= vduse_vdpa_set_driver_features,
+	.get_driver_features	= vduse_vdpa_get_driver_features,
+	.set_config_cb		= vduse_vdpa_set_config_cb,
+	.get_vq_num_max		= vduse_vdpa_get_vq_num_max,
+	.get_device_id		= vduse_vdpa_get_device_id,
+	.get_vendor_id		= vduse_vdpa_get_vendor_id,
+	.get_status		= vduse_vdpa_get_status,
+	.set_status		= vduse_vdpa_set_status,
+	.get_config_size	= vduse_vdpa_get_config_size,
+	.get_config		= vduse_vdpa_get_config,
+	.set_config		= vduse_vdpa_set_config,
+	.get_generation		= vduse_vdpa_get_generation,
+	.set_vq_affinity	= vduse_vdpa_set_vq_affinity,
+	.get_vq_affinity	= vduse_vdpa_get_vq_affinity,
+	.reset			= vduse_vdpa_reset,
+	.set_map		= vduse_vdpa_set_map,
+	.set_group_asid		= vduse_set_group_asid,
+	.get_vq_map		= vduse_get_vq_map,
+	.suspend		= vduse_vdpa_suspend,
+	.free			= vduse_vdpa_free,
+};
+
 static void vduse_dev_sync_single_for_device(union virtio_map token,
 					     dma_addr_t dma_addr, size_t size,
 					     enum dma_data_direction dir)
@@ -1143,6 +1232,10 @@ static void vduse_dev_irq_inject(struct work_struct *work)
 {
 	struct vduse_dev *dev = container_of(work, struct vduse_dev, inject);
 
+	guard(rwsem_read)(&dev->rwsem);
+	if (dev->suspended)
+		return;
+
 	spin_lock_bh(&dev->irq_lock);
 	if (dev->config_cb.callback)
 		dev->config_cb.callback(dev->config_cb.private);
@@ -1154,6 +1247,10 @@ static void vduse_vq_irq_inject(struct work_struct *work)
 	struct vduse_virtqueue *vq = container_of(work,
 					struct vduse_virtqueue, inject);
 
+	guard(rwsem_read)(&vq->dev->rwsem);
+	if (vq->dev->suspended)
+		return;
+
 	spin_lock_bh(&vq->irq_lock);
 	if (vq->ready && vq->cb.callback)
 		vq->cb.callback(vq->cb.private);
@@ -1183,9 +1280,9 @@ static int vduse_dev_queue_irq_work(struct vduse_dev *dev,
 {
 	int ret = -EINVAL;
 
-	down_read(&dev->rwsem);
-	if (!(dev->status & VIRTIO_CONFIG_S_DRIVER_OK))
-		goto unlock;
+	guard(rwsem_read)(&dev->rwsem);
+	if (dev->suspended || !(dev->status & VIRTIO_CONFIG_S_DRIVER_OK))
+		return ret;
 
 	ret = 0;
 	if (irq_effective_cpu == IRQ_UNBOUND)
@@ -1193,8 +1290,6 @@ static int vduse_dev_queue_irq_work(struct vduse_dev *dev,
 	else
 		queue_work_on(irq_effective_cpu,
 			      vduse_irq_bound_wq, irq_work);
-unlock:
-	up_read(&dev->rwsem);
 
 	return ret;
 }
@@ -1948,6 +2043,7 @@ static int vduse_dev_init_vqs(struct vduse_dev *dev, u32 vq_align, u32 vq_num)
 		}
 
 		dev->vqs[i]->index = i;
+		dev->vqs[i]->dev = dev;
 		dev->vqs[i]->irq_effective_cpu = IRQ_UNBOUND;
 		INIT_WORK(&dev->vqs[i]->inject, vduse_vq_irq_inject);
 		INIT_WORK(&dev->vqs[i]->kick, vduse_vq_kick_work);
@@ -2210,6 +2306,9 @@ static int vduse_create_dev(struct vduse_dev_config *config,
 	dev->device_features = config->features;
 	dev->device_id = config->device_id;
 	dev->vendor_id = config->vendor_id;
+	dev->vduse_features = config->vduse_features;
+	dev_dbg(vduse_ctrl_dev, "Creating device %s with features 0x%llx",
+		config->name, config->vduse_features);
 
 	dev->nas = (dev->api_version < VDUSE_API_VERSION_1) ? 1 : config->nas;
 	dev->as = kzalloc_objs(dev->as[0], dev->nas);
@@ -2341,6 +2440,10 @@ static long vduse_ioctl(struct file *file, unsigned int cmd,
 		ret = vduse_destroy_dev(name);
 		break;
 	}
+	case VDUSE_GET_FEATURES:
+		ret = put_user(vduse_features, (u64 __user *)argp);
+		break;
+
 	default:
 		ret = -EINVAL;
 		break;
@@ -2391,12 +2494,18 @@ static struct vduse_mgmt_dev *vduse_mgmt;
 static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name)
 {
 	struct vduse_vdpa *vdev;
+	const struct vdpa_config_ops *ops;
 
 	if (dev->vdev)
 		return -EEXIST;
 
+	if (dev->vduse_features & BIT_U64(VDUSE_F_SUSPEND))
+		ops = &vduse_vdpa_config_ops_with_suspend;
+	else
+		ops = &vduse_vdpa_config_ops;
+
 	vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev,
-				 &vduse_vdpa_config_ops, &vduse_map_ops,
+				 ops, &vduse_map_ops,
 				 dev->ngroups, dev->nas, name, true);
 	if (IS_ERR(vdev))
 		return PTR_ERR(vdev);
@@ -2531,7 +2640,6 @@ static void vduse_mgmtdev_exit(void)
 static int vduse_init(void)
 {
 	int ret;
-	struct device *dev;
 
 	ret = class_register(&vduse_class);
 	if (ret)
@@ -2548,9 +2656,9 @@ static int vduse_init(void)
 	if (ret)
 		goto err_ctrl_cdev;
 
-	dev = device_create(&vduse_class, NULL, vduse_major, NULL, "control");
-	if (IS_ERR(dev)) {
-		ret = PTR_ERR(dev);
+	vduse_ctrl_dev = device_create(&vduse_class, NULL, vduse_major, NULL, "control");
+	if (IS_ERR(vduse_ctrl_dev)) {
+		ret = PTR_ERR(vduse_ctrl_dev);
 		goto err_device;
 	}
 
diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig
index ceae52fd7586d..b9d6e1c22aed5 100644
--- a/drivers/vfio/Kconfig
+++ b/drivers/vfio/Kconfig
@@ -22,8 +22,7 @@ config VFIO_DEVICE_CDEV
 	  The VFIO device cdev is another way for userspace to get device
 	  access. Userspace gets device fd by opening device cdev under
 	  /dev/vfio/devices/vfioX, and then bind the device fd with an iommufd
-	  to set up secure DMA context for device access.  This interface does
-	  not support noiommu.
+	  to set up secure DMA context for device access.
 
 	  If you don't know what to do here, say N.
 
@@ -62,7 +61,9 @@ endif
 
 config VFIO_NOIOMMU
 	bool "VFIO No-IOMMU support"
-	depends on VFIO_GROUP
+	depends on VFIO_GROUP || (VFIO_DEVICE_CDEV && !GENERIC_ATOMIC64)
+	depends on !VFIO_GROUP || VFIO_CONTAINER || IOMMUFD_VFIO_CONTAINER
+	select IOMMUFD_NOIOMMU if VFIO_DEVICE_CDEV && !GENERIC_ATOMIC64
 	help
 	  VFIO is built on the ability to isolate devices using the IOMMU.
 	  Only with an IOMMU can userspace access to DMA capable devices be
diff --git a/drivers/vfio/device_cdev.c b/drivers/vfio/device_cdev.c
index 54abf312cf049..1d9515c967b01 100644
--- a/drivers/vfio/device_cdev.c
+++ b/drivers/vfio/device_cdev.c
@@ -11,6 +11,10 @@ static dev_t device_devt;
 
 void vfio_init_device_cdev(struct vfio_device *device)
 {
+	if (vfio_device_is_noiommu(device) &&
+	    !IS_ENABLED(CONFIG_IOMMUFD_NOIOMMU))
+		return;
+
 	device->device.devt = MKDEV(MAJOR(device_devt), device->index);
 	cdev_init(&device->cdev, &vfio_device_fops);
 	device->cdev.owner = THIS_MODULE;
@@ -31,6 +35,11 @@ int vfio_device_fops_cdev_open(struct inode *inode, struct file *filep)
 	if (!vfio_device_try_get_registration(device))
 		return -ENODEV;
 
+	if (vfio_device_is_noiommu(device) && !capable(CAP_SYS_RAWIO)) {
+		ret = -EPERM;
+		goto err_put_registration;
+	}
+
 	df = vfio_allocate_device_file(device);
 	if (IS_ERR(df)) {
 		ret = PTR_ERR(df);
diff --git a/drivers/vfio/iommufd.c b/drivers/vfio/iommufd.c
index a38d262c60280..e9893d34d07be 100644
--- a/drivers/vfio/iommufd.c
+++ b/drivers/vfio/iommufd.c
@@ -25,8 +25,8 @@ int vfio_df_iommufd_bind(struct vfio_device_file *df)
 
 	lockdep_assert_held(&vdev->dev_set->lock);
 
-	/* Returns 0 to permit device opening under noiommu mode */
-	if (vfio_device_is_noiommu(vdev))
+	/* Group noiommu via iommufd compat needs no device binding */
+	if (df->group && vfio_device_is_noiommu(vdev))
 		return 0;
 
 	return vdev->ops->bind_iommufd(vdev, ictx, &df->devid);
@@ -40,7 +40,11 @@ int vfio_iommufd_compat_attach_ioas(struct vfio_device *vdev,
 
 	lockdep_assert_held(&vdev->dev_set->lock);
 
-	/* compat noiommu does not need to do ioas attach */
+	/*
+	 * Compat noiommu does not need to do ioas attach. This helper is
+	 * only called from the legacy group/iommufd compat path, so no
+	 * explicit df->group check is needed.
+	 */
 	if (vfio_device_is_noiommu(vdev))
 		return 0;
 
@@ -58,7 +62,7 @@ void vfio_df_iommufd_unbind(struct vfio_device_file *df)
 
 	lockdep_assert_held(&vdev->dev_set->lock);
 
-	if (vfio_device_is_noiommu(vdev))
+	if (df->group && vfio_device_is_noiommu(vdev))
 		return;
 
 	if (vdev->ops->unbind_iommufd)
diff --git a/drivers/vfio/pci/nvgrace-gpu/main.c b/drivers/vfio/pci/nvgrace-gpu/main.c
index d07dcacb76bd2..963fd8ded20d1 100644
--- a/drivers/vfio/pci/nvgrace-gpu/main.c
+++ b/drivers/vfio/pci/nvgrace-gpu/main.c
@@ -11,6 +11,7 @@
 #include <linux/jiffies.h>
 #include <linux/sched.h>
 #include <linux/pci-p2pdma.h>
+#include <linux/pagemap.h>
 #include <linux/pm_runtime.h>
 #include <linux/memory-failure.h>
 
@@ -385,7 +386,7 @@ static unsigned long addr_to_pgoff(struct vm_area_struct *vma,
 	u64 pgoff = vma->vm_pgoff &
 		((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
 
-	return ((addr - vma->vm_start) >> PAGE_SHIFT) + pgoff;
+	return linear_page_delta(vma, addr) + pgoff;
 }
 
 static vm_fault_t nvgrace_gpu_vfio_pci_huge_fault(struct vm_fault *vmf,
diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
index 3f11a9624b9c0..a113c55845e10 100644
--- a/drivers/vfio/pci/vfio_pci_core.c
+++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -20,6 +20,7 @@
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/notifier.h>
+#include <linux/pagemap.h>
 #include <linux/pci.h>
 #include <linux/pm_runtime.h>
 #include <linux/slab.h>
@@ -1780,7 +1781,7 @@ static vm_fault_t vfio_pci_mmap_huge_fault(struct vm_fault *vmf,
 	struct vm_area_struct *vma = vmf->vma;
 	struct vfio_pci_core_device *vdev = vma->vm_private_data;
 	unsigned long addr = vmf->address & ~((PAGE_SIZE << order) - 1);
-	unsigned long pgoff = (addr - vma->vm_start) >> PAGE_SHIFT;
+	unsigned long pgoff = linear_page_delta(vma, addr);
 	unsigned long pfn = vma_to_pfn(vma) + pgoff;
 	vm_fault_t ret = VM_FAULT_FALLBACK;
 
diff --git a/drivers/vfio/vfio.h b/drivers/vfio/vfio.h
index e4b72e79b7e3e..7728bc99b63d1 100644
--- a/drivers/vfio/vfio.h
+++ b/drivers/vfio/vfio.h
@@ -112,11 +112,6 @@ bool vfio_device_has_container(struct vfio_device *device);
 int __init vfio_group_init(void);
 void vfio_group_cleanup(void);
 
-static inline bool vfio_device_is_noiommu(struct vfio_device *vdev)
-{
-	return IS_ENABLED(CONFIG_VFIO_NOIOMMU) &&
-	       vdev->group->type == VFIO_NO_IOMMU;
-}
 #else
 struct vfio_group;
 
@@ -188,11 +183,17 @@ static inline void vfio_group_cleanup(void)
 {
 }
 
+#endif /* CONFIG_VFIO_GROUP */
+
 static inline bool vfio_device_is_noiommu(struct vfio_device *vdev)
 {
-	return false;
+#if IS_ENABLED(CONFIG_VFIO_GROUP)
+	if (vdev->group && vdev->group->type == VFIO_NO_IOMMU)
+		return true;
+#endif
+
+	return IS_ENABLED(CONFIG_IOMMUFD_NOIOMMU) && vdev->noiommu;
 }
-#endif /* CONFIG_VFIO_GROUP */
 
 #if IS_ENABLED(CONFIG_VFIO_CONTAINER)
 /**
@@ -358,19 +359,13 @@ void vfio_init_device_cdev(struct vfio_device *device);
 
 static inline int vfio_device_add(struct vfio_device *device)
 {
-	/* cdev does not support noiommu device */
-	if (vfio_device_is_noiommu(device))
-		return device_add(&device->device);
 	vfio_init_device_cdev(device);
 	return cdev_device_add(&device->cdev, &device->device);
 }
 
 static inline void vfio_device_del(struct vfio_device *device)
 {
-	if (vfio_device_is_noiommu(device))
-		device_del(&device->device);
-	else
-		cdev_device_del(&device->cdev, &device->device);
+	cdev_device_del(&device->cdev, &device->device);
 }
 
 int vfio_device_fops_cdev_open(struct inode *inode, struct file *filep);
diff --git a/drivers/vfio/vfio_main.c b/drivers/vfio/vfio_main.c
index ed538aebb0b8e..423ead48aafe3 100644
--- a/drivers/vfio/vfio_main.c
+++ b/drivers/vfio/vfio_main.c
@@ -321,6 +321,24 @@ static int vfio_init_device(struct vfio_device *device, struct device *dev,
 	return ret;
 }
 
+static int vfio_device_set_noiommu_and_name(struct vfio_device *device, enum vfio_group_type type)
+{
+	if (IS_ENABLED(CONFIG_IOMMUFD_NOIOMMU) && vfio_noiommu &&
+	    !device->dev->iommu && type == VFIO_IOMMU)
+		device->noiommu = true;
+
+	/*
+	 * device->noiommu records no-IOMMU support for the standalone cdev
+	 * interface. VFIO_NOIOMMU enables both group and cdev no-IOMMU; when
+	 * cdev no-IOMMU is available, device->noiommu is set before
+	 * vfio_device_set_group(), so the cdev is named noiommu-vfio%d up
+	 * front. If IOMMUFD_NOIOMMU is unavailable, no-IOMMU devices are
+	 * limited to the group interface and do not receive a device cdev.
+	 */
+	return dev_set_name(&device->device, "%svfio%d",
+		     device->noiommu ? "noiommu-" : "", device->index);
+}
+
 static int __vfio_register_dev(struct vfio_device *device,
 			       enum vfio_group_type type)
 {
@@ -340,7 +358,7 @@ static int __vfio_register_dev(struct vfio_device *device,
 	if (!device->dev_set)
 		vfio_assign_device_set(device, device);
 
-	ret = dev_set_name(&device->device, "vfio%d", device->index);
+	ret = vfio_device_set_noiommu_and_name(device, type);
 	if (ret)
 		return ret;
 
@@ -348,6 +366,12 @@ static int __vfio_register_dev(struct vfio_device *device,
 	if (ret)
 		return ret;
 
+	if (vfio_device_is_noiommu(device) && IS_ENABLED(CONFIG_IOMMUFD_NOIOMMU)) {
+		add_taint(TAINT_USER, LOCKDEP_STILL_OK);
+		dev_warn(device->dev,
+			 "Adding kernel taint for vfio-noiommu cdev\n");
+	}
+
 	/*
 	 * VFIO always sets IOMMU_CACHE because we offer no way for userspace to
 	 * restore cache coherency. It has to be checked here because it is only
diff --git a/drivers/video/backlight/Kconfig b/drivers/video/backlight/Kconfig
index 7aa1c4b21111f..7c66b8840d885 100644
--- a/drivers/video/backlight/Kconfig
+++ b/drivers/video/backlight/Kconfig
@@ -207,6 +207,14 @@ config BACKLIGHT_KTZ8866
 		Say Y to enable the backlight driver for the Kinetic KTZ8866
 		found in Xiaomi Mi Pad 5 series.
 
+config BACKLIGHT_SY7758
+	tristate "Backlight Driver for Silergy SY7758"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  Say Y to enable the backlight driver for the Silergy SY7758
+	  backlight controller found in Ayaneo Pocket S2.
+
 config BACKLIGHT_LM3533
 	tristate "Backlight Driver for LM3533"
 	depends on MFD_LM3533
diff --git a/drivers/video/backlight/Makefile b/drivers/video/backlight/Makefile
index 21c8313cfb121..34469711c6cd9 100644
--- a/drivers/video/backlight/Makefile
+++ b/drivers/video/backlight/Makefile
@@ -57,6 +57,7 @@ obj-$(CONFIG_BACKLIGHT_PWM)		+= pwm_bl.o
 obj-$(CONFIG_BACKLIGHT_QCOM_WLED)	+= qcom-wled.o
 obj-$(CONFIG_BACKLIGHT_RT4831)		+= rt4831-backlight.o
 obj-$(CONFIG_BACKLIGHT_SAHARA)		+= kb3886_bl.o
+obj-$(CONFIG_BACKLIGHT_SY7758)		+= sy7758.o
 obj-$(CONFIG_BACKLIGHT_SKY81452)	+= sky81452-backlight.o
 obj-$(CONFIG_BACKLIGHT_TPS65217)	+= tps65217_bl.o
 obj-$(CONFIG_BACKLIGHT_WM831X)		+= wm831x_bl.o
diff --git a/drivers/video/backlight/sy7758.c b/drivers/video/backlight/sy7758.c
new file mode 100644
index 0000000000000..786589cb8df91
--- /dev/null
+++ b/drivers/video/backlight/sy7758.c
@@ -0,0 +1,259 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Silergy SY7758 6-channel High Efficiency LED Driver
+ *
+ * Copyright (C) 2025 Kancy Joe <kancy2333@outlook.com>
+ * Copyright (C) 2026 Linaro Limited
+ * Author: Neil Armstrong <neil.armstrong@linaro.org>
+ */
+#include <linux/backlight.h>
+#include <linux/module.h>
+#include <linux/i2c.h>
+#include <linux/of.h>
+#include <linux/err.h>
+#include <linux/bits.h>
+#include <linux/delay.h>
+#include <linux/regmap.h>
+#include <linux/bitfield.h>
+#include <linux/gpio/consumer.h>
+#include <linux/regulator/consumer.h>
+
+#define DEFAULT_BRIGHTNESS	1024
+#define MAX_BRIGHTNESS		4080
+#define REG_MAX			0xAE
+
+/* Registers */
+#define REG_DEV_CTL		0x01
+#define REG_DEV_ID		0x03
+#define REG_BRT_12BIT_L		0x10
+#define REG_BRT_12BIT_H		0x11
+
+/* OTP memory */
+#define REG_OTP_CFG0		0xA0
+#define REG_OTP_CFG1		0xA1
+#define REG_OTP_CFG2		0xA2
+#define REG_OTP_CFG5		0xA5
+#define REG_OTP_CFG9		0xA9
+
+/* Fields */
+#define BIT_DEV_CTL_FAST	BIT(7)
+#define MSK_DEV_CTL_BRT_MODE	GENMASK(2, 1)
+#define BIT_DEV_CTL_BL_CTLB	BIT(0)
+
+#define MSK_BRT_12BIT_L		GENMASK(7, 0)
+#define MSK_BRT_12BIT_H		GENMASK(3, 0)
+
+#define MSK_CFG0_CURRENT_LOW	GENMASK(7, 0)
+
+#define BIT_CFG1_PDET_STDBY	BIT(7)
+#define MSK_CFG1_CURRENT_MAX	GENMASK(6, 4)
+#define MSK_CFG1_CURRENT_HIGH	GENMASK(3, 0)
+
+#define BIT_CFG2_UVLO_EN	BIT(5)
+#define BIT_CFG2_UVLO_TH	BIT(4)
+#define BIT_CFG2_BL_ON		BIT(3)
+#define BIT_CFG2_ISET_EN	BIT(2)
+#define BIT_CFG2_BST_ESET_EN	BIT(1)
+
+#define BIT_CFG5_PWM_DIRECT	BIT(7)
+#define MSK_CFG5_PS_MODE	GENMASK(6, 4)
+#define MSK_CFG5_PWM_FREQ	GENMASK(3, 0)
+
+#define MSK_CFG9_VBST_MAX	GENMASK(7, 5)
+#define BIT_CFG9_JUMP_EN	BIT(4)
+#define MSK_CFG9_JUMP_TH	GENMASK(3, 2)
+#define MSK_CFG9_JUMP_VOLTAGE	GENMASK(1, 0)
+
+struct sy7758 {
+	struct i2c_client *client;
+	struct regmap *regmap;
+	struct gpio_desc *gpio;
+	struct backlight_device *bl;
+};
+
+static const struct regmap_config sy7758_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = REG_MAX,
+};
+
+static int sy7758_backlight_update_status(struct backlight_device *backlight_dev)
+{
+	struct sy7758 *sydev = bl_get_data(backlight_dev);
+	unsigned int brightness = backlight_get_brightness(backlight_dev);
+	int ret;
+
+	ret = regmap_write(sydev->regmap, REG_BRT_12BIT_L,
+			   FIELD_PREP(MSK_BRT_12BIT_L,
+				      brightness & 0xff));
+	if (ret)
+		return ret;
+
+	ret = regmap_write(sydev->regmap, REG_BRT_12BIT_H,
+			   FIELD_PREP(MSK_BRT_12BIT_H,
+				      (brightness >> 8) & 0xf));
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static const struct backlight_ops sy7758_backlight_ops = {
+	.options = BL_CORE_SUSPENDRESUME,
+	.update_status = sy7758_backlight_update_status,
+};
+
+static int sy7758_init(struct sy7758 *sydev)
+{
+	int ret = 0;
+
+	ret = regmap_write(sydev->regmap, REG_DEV_CTL,
+			   BIT_DEV_CTL_FAST | BIT_DEV_CTL_BL_CTLB |
+			   FIELD_PREP(MSK_DEV_CTL_BRT_MODE, 2));
+	if (ret)
+		return ret;
+
+	ret = regmap_write(sydev->regmap, REG_BRT_12BIT_L,
+			   FIELD_PREP(MSK_BRT_12BIT_L,
+				      DEFAULT_BRIGHTNESS & 0xff));
+	if (ret)
+		return ret;
+
+	ret = regmap_write(sydev->regmap, REG_BRT_12BIT_H,
+			   FIELD_PREP(MSK_BRT_12BIT_H,
+				      (DEFAULT_BRIGHTNESS >> 8)));
+	if (ret)
+		return ret;
+
+	ret = regmap_write(sydev->regmap, REG_OTP_CFG5,
+			   FIELD_PREP(MSK_CFG5_PS_MODE, 6) |
+			   FIELD_PREP(MSK_CFG5_PWM_FREQ, 4));
+	if (ret)
+		return ret;
+
+	ret = regmap_write(sydev->regmap, REG_OTP_CFG0,
+			   FIELD_PREP(MSK_CFG0_CURRENT_LOW, 85));
+	if (ret)
+		return ret;
+
+	ret = regmap_write(sydev->regmap, REG_OTP_CFG1,
+			   BIT_CFG1_PDET_STDBY |
+			   FIELD_PREP(MSK_CFG1_CURRENT_MAX, 1) |
+			   FIELD_PREP(MSK_CFG1_CURRENT_HIGH, 10));
+	if (ret)
+		return ret;
+
+	ret = regmap_write(sydev->regmap, REG_OTP_CFG9,
+			   FIELD_PREP(MSK_CFG9_VBST_MAX, 4));
+	if (ret)
+		return ret;
+
+	ret = regmap_write(sydev->regmap, REG_OTP_CFG2,
+			   BIT_CFG2_BL_ON | BIT_CFG2_UVLO_EN);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int sy7758_probe(struct i2c_client *client)
+{
+	struct backlight_properties props = { };
+	struct device *dev = &client->dev;
+	struct sy7758 *sydev;
+	unsigned int dev_id;
+	int ret;
+
+	sydev = devm_kzalloc(dev, sizeof(*sydev), GFP_KERNEL);
+	if (!sydev)
+		return -ENOMEM;
+
+	i2c_set_clientdata(client, sydev);
+
+	/* Initialize regmap */
+	sydev->client = client;
+	sydev->regmap = devm_regmap_init_i2c(client, &sy7758_regmap_config);
+	if (IS_ERR(sydev->regmap))
+		return dev_err_probe(dev, PTR_ERR(sydev->regmap),
+				     "failed to init regmap\n");
+
+	/* Get and enable regulator */
+	ret = devm_regulator_get_enable(dev, "vdd");
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to get regulator\n");
+
+	fsleep(100);
+
+	/* Get enable GPIO and set to high */
+	sydev->gpio = devm_gpiod_get(dev, "enable", GPIOD_OUT_HIGH);
+	if (IS_ERR(sydev->gpio))
+		return dev_err_probe(dev, PTR_ERR(sydev->gpio),
+				     "failed to get enable GPIO\n");
+
+	/* Let some time for HW to settle */
+	fsleep(10000);
+
+	/* try read and check device id */
+	ret = regmap_read(sydev->regmap, REG_DEV_ID, &dev_id);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to read device id\n");
+	if (dev_id != 0x63) {
+		dev_err(dev, "unexpected device id: 0x%02x\n", dev_id);
+		return -ENODEV;
+	}
+
+	/* Initialize and set default brightness */
+	ret = sy7758_init(sydev);
+	if (ret)
+		return ret;
+
+	props.type = BACKLIGHT_RAW;
+	props.max_brightness = MAX_BRIGHTNESS;
+	props.brightness = DEFAULT_BRIGHTNESS;
+	props.scale = BACKLIGHT_SCALE_LINEAR;
+
+	sydev->bl = devm_backlight_device_register(dev, "sy7758-backlight",
+						   dev, sydev, &sy7758_backlight_ops,
+						   &props);
+	if (IS_ERR(sydev->bl))
+		return dev_err_probe(dev, PTR_ERR(sydev->bl),
+				     "failed to register backlight device\n");
+
+	return backlight_update_status(sydev->bl);
+}
+
+static void sy7758_remove(struct i2c_client *client)
+{
+	struct sy7758 *sydev = i2c_get_clientdata(client);
+
+	backlight_disable(sydev->bl);
+}
+
+static const struct i2c_device_id sy7758_ids[] = {
+	{ "sy7758" },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, sy7758_ids);
+
+static const struct of_device_id sy7758_match_table[] = {
+	{ .compatible = "silergy,sy7758", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, sy7758_match_table);
+
+static struct i2c_driver sy7758_driver = {
+	.driver = {
+		.name = "sy7758",
+		.of_match_table = sy7758_match_table,
+	},
+	.probe = sy7758_probe,
+	.remove = sy7758_remove,
+	.id_table = sy7758_ids,
+};
+
+module_i2c_driver(sy7758_driver);
+
+MODULE_DESCRIPTION("Silergy SY7758 Backlight Driver");
+MODULE_AUTHOR("Kancy Joe <kancy2333@outlook.com>");
+MODULE_AUTHOR("Neil Armstrong <neil.armstrong@linaro.org>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/video/fbdev/core/fb_io_fops.c b/drivers/video/fbdev/core/fb_io_fops.c
index 6ab60fcd00501..6d0a8c8e141a2 100644
--- a/drivers/video/fbdev/core/fb_io_fops.c
+++ b/drivers/video/fbdev/core/fb_io_fops.c
@@ -161,7 +161,7 @@ int fb_io_mmap(struct fb_info *info, struct vm_area_struct *vma)
 		len = info->fix.mmio_len;
 	}
 
-	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+	vma->vm_page_prot = vma_get_page_prot(vma);
 	vma->vm_page_prot = pgprot_framebuffer(vma->vm_page_prot, vma->vm_start,
 					       vma->vm_end, start);
 
diff --git a/drivers/virt/vmgenid.c b/drivers/virt/vmgenid.c
index 66135eac3abff..2cf0096aa217f 100644
--- a/drivers/virt/vmgenid.c
+++ b/drivers/virt/vmgenid.c
@@ -75,7 +75,8 @@ static int vmgenid_add_acpi(struct device *dev, struct vmgenid_state *state)
 	phys_addr = (obj->package.elements[0].integer.value << 0) |
 		    (obj->package.elements[1].integer.value << 32);
 
-	virt_addr = devm_memremap(&device->dev, phys_addr, VMGENID_SIZE, MEMREMAP_WB);
+	virt_addr = devm_memremap(&device->dev, phys_addr, VMGENID_SIZE,
+				  MEMREMAP_WB | MEMREMAP_DEC);
 	if (IS_ERR(virt_addr)) {
 		ret = PTR_ERR(virt_addr);
 		goto out;
diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
index 088b3a0e6ce6c..581ac799d9749 100644
--- a/drivers/virtio/virtio_balloon.c
+++ b/drivers/virtio/virtio_balloon.c
@@ -1017,10 +1017,6 @@ static int virtballoon_probe(struct virtio_device *vdev)
 		unsigned int capacity;
 
 		capacity = virtqueue_get_vring_size(vb->reporting_vq);
-		if (capacity < PAGE_REPORTING_CAPACITY) {
-			err = -ENOSPC;
-			goto out_unregister_oom;
-		}
 
 		vb->pr_dev_info.order = PAGE_REPORTING_ORDER_UNSPECIFIED;
 
@@ -1041,6 +1037,7 @@ static int virtballoon_probe(struct virtio_device *vdev)
 		vb->pr_dev_info.order = 5;
 #endif
 
+		vb->pr_dev_info.capacity = capacity;
 		err = page_reporting_register(&vb->pr_dev_info);
 		if (err)
 			goto out_unregister_oom;
diff --git a/drivers/w1/masters/ds2482.c b/drivers/w1/masters/ds2482.c
index 0069e6f854d7f..9e57c6e487d1c 100644
--- a/drivers/w1/masters/ds2482.c
+++ b/drivers/w1/masters/ds2482.c
@@ -545,9 +545,17 @@ static const struct i2c_device_id ds2482_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, ds2482_id);
 
+static const struct of_device_id ds2482_of_match[] = {
+	{ .compatible = "maxim,ds2482", },
+	{ .compatible = "maxim,ds2484", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ds2482_of_match);
+
 static struct i2c_driver ds2482_driver = {
 	.driver = {
 		.name	= "ds2482",
+		.of_match_table  = ds2482_of_match,
 	},
 	.probe		= ds2482_probe,
 	.remove		= ds2482_remove,
diff --git a/drivers/w1/slaves/w1_ds28e17.c b/drivers/w1/slaves/w1_ds28e17.c
index e53bc41bde3ca..b638963d4b595 100644
--- a/drivers/w1/slaves/w1_ds28e17.c
+++ b/drivers/w1/slaves/w1_ds28e17.c
@@ -389,6 +389,10 @@ static int w1_f19_i2c_master_transfer(struct i2c_adapter *adapter,
 			 * another simple read in that case.
 			 */
 			if (msgs[i+1].flags & I2C_M_RECV_LEN) {
+				if (msgs[i+1].buf[0] > I2C_SMBUS_BLOCK_MAX) {
+					i = -EPROTO;
+					goto error;
+				}
 				result = w1_f19_i2c_read(sl, msgs[i+1].addr,
 					&(msgs[i+1].buf[1]), msgs[i+1].buf[0]);
 				if (result < 0) {
@@ -415,6 +419,10 @@ static int w1_f19_i2c_master_transfer(struct i2c_adapter *adapter,
 			 * another simple read in that case.
 			 */
 			if (msgs[i].flags & I2C_M_RECV_LEN) {
+				if (msgs[i].buf[0] > I2C_SMBUS_BLOCK_MAX) {
+					i = -EPROTO;
+					goto error;
+				}
 				result = w1_f19_i2c_read(sl,
 					msgs[i].addr,
 					&(msgs[i].buf[1]),
diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig
index 08cb8612d41fe..9f013d774897e 100644
--- a/drivers/watchdog/Kconfig
+++ b/drivers/watchdog/Kconfig
@@ -93,10 +93,19 @@ config WATCHDOG_PRETIMEOUT_GOV_SEL
 	tristate
 	depends on WATCHDOG_PRETIMEOUT_GOV
 	default m
-	select WATCHDOG_PRETIMEOUT_GOV_PANIC if WATCHDOG_PRETIMEOUT_GOV_NOOP=n
+	select WATCHDOG_PRETIMEOUT_GOV_PANIC if \
+		WATCHDOG_PRETIMEOUT_GOV_NOOP=n && WATCHDOG_PRETIMEOUT_GOV_DUMP=n
 
 if WATCHDOG_PRETIMEOUT_GOV
 
+config WATCHDOG_PRETIMEOUT_GOV_DUMP
+	bool "Dump watchdog pretimeout governor"
+	depends on WATCHDOG_CORE=y
+	default WATCHDOG_CORE
+	help
+	  Dump watchdog pretimeout governor, all cpu backtrace is
+	  added to kernel log buffer.
+
 config WATCHDOG_PRETIMEOUT_GOV_NOOP
 	tristate "Noop watchdog pretimeout governor"
 	depends on WATCHDOG_CORE
@@ -121,6 +130,14 @@ choice
 	  The governor takes its action, if a watchdog is capable
 	  to report a pretimeout event.
 
+config WATCHDOG_PRETIMEOUT_DEFAULT_GOV_DUMP
+	bool "dump"
+	depends on WATCHDOG_PRETIMEOUT_GOV_DUMP
+	help
+	  Use dump watchdog pretimeout governor by default. If dump
+	  governor is selected by a user, dump all cpu backtrace to
+	  the kernel log buffer and don't do any system changes.
+
 config WATCHDOG_PRETIMEOUT_DEFAULT_GOV_NOOP
 	bool "noop"
 	depends on WATCHDOG_PRETIMEOUT_GOV_NOOP
@@ -291,6 +308,18 @@ config LENOVO_SE30_WDT
 	  This driver can also be built as a module. If so, the module
 	  will be called lenovo-se30-wdt.
 
+config LENOVO_SE30G2_SE60_WDT
+	tristate "Lenovo SE30 Gen 2, SE60 Watchdog"
+	depends on (X86 && DMI) || COMPILE_TEST
+	depends on HAS_IOPORT
+	select WATCHDOG_CORE
+	help
+	  If you say yes here you get support for the watchdog
+	  functionality for the Lenovo SE30 Gen2 and SE60 platforms.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called lenovo-se30g2-se60-wdt.
+
 config MENF21BMC_WATCHDOG
 	tristate "MEN 14F021P00 BMC Watchdog"
 	depends on MFD_MENF21BMC || COMPILE_TEST
diff --git a/drivers/watchdog/Makefile b/drivers/watchdog/Makefile
index bc1d52220f223..cb78e9932eae6 100644
--- a/drivers/watchdog/Makefile
+++ b/drivers/watchdog/Makefile
@@ -11,6 +11,7 @@ watchdog-objs	+= watchdog_core.o watchdog_dev.o
 watchdog-$(CONFIG_WATCHDOG_PRETIMEOUT_GOV)	+= watchdog_pretimeout.o
 watchdog-$(CONFIG_WATCHDOG_HRTIMER_PRETIMEOUT)	+= watchdog_hrtimer_pretimeout.o
 
+obj-$(CONFIG_WATCHDOG_PRETIMEOUT_GOV_DUMP)	+= pretimeout_dump.o
 obj-$(CONFIG_WATCHDOG_PRETIMEOUT_GOV_NOOP)	+= pretimeout_noop.o
 obj-$(CONFIG_WATCHDOG_PRETIMEOUT_GOV_PANIC)	+= pretimeout_panic.o
 
@@ -127,6 +128,7 @@ obj-$(CONFIG_IE6XX_WDT) += ie6xx_wdt.o
 obj-$(CONFIG_ITCO_WDT) += iTCO_wdt.o
 obj-$(CONFIG_LENOVO_SE10_WDT) += lenovo_se10_wdt.o
 obj-$(CONFIG_LENOVO_SE30_WDT) += lenovo_se30_wdt.o
+obj-$(CONFIG_LENOVO_SE30G2_SE60_WDT) += lenovo_se30g2_se60_wdt.o
 obj-$(CONFIG_IT8712F_WDT) += it8712f_wdt.o
 obj-$(CONFIG_IT87_WDT) += it87_wdt.o
 obj-$(CONFIG_HP_WATCHDOG) += hpwdt.o
diff --git a/drivers/watchdog/airoha_wdt.c b/drivers/watchdog/airoha_wdt.c
index dc8ca11c14d81..4bd333189b87e 100644
--- a/drivers/watchdog/airoha_wdt.c
+++ b/drivers/watchdog/airoha_wdt.c
@@ -147,6 +147,9 @@ static int airoha_wdt_probe(struct platform_device *pdev)
 
 	/* Watchdog ticks at half the bus rate */
 	airoha_wdt->wdt_freq = clk_get_rate(bus_clk) / 2;
+	if (!airoha_wdt->wdt_freq)
+		return dev_err_probe(dev, -EINVAL,
+				     "invalid clock frequency\n");
 
 	/* Initialize struct watchdog device */
 	wdog_dev = &airoha_wdt->wdog_dev;
diff --git a/drivers/watchdog/cros_ec_wdt.c b/drivers/watchdog/cros_ec_wdt.c
index 9a4a59b39ed90..2283419d8b381 100644
--- a/drivers/watchdog/cros_ec_wdt.c
+++ b/drivers/watchdog/cros_ec_wdt.c
@@ -178,8 +178,8 @@ static int __maybe_unused cros_ec_wdt_resume(struct platform_device *pdev)
 }
 
 static const struct platform_device_id cros_ec_wdt_id[] = {
-	{ DRV_NAME, 0 },
-	{}
+	{ .name = DRV_NAME },
+	{ }
 };
 
 static struct platform_driver cros_ec_wdt_driver = {
diff --git a/drivers/watchdog/db8500_wdt.c b/drivers/watchdog/db8500_wdt.c
index 97148ac0aa54a..70ccea13288d8 100644
--- a/drivers/watchdog/db8500_wdt.c
+++ b/drivers/watchdog/db8500_wdt.c
@@ -16,7 +16,7 @@
 #include <linux/watchdog.h>
 #include <linux/platform_device.h>
 
-#include <linux/mfd/dbx500-prcmu.h>
+#include <linux/mfd/db8500-prcmu.h>
 
 #define WATCHDOG_TIMEOUT 600 /* 10 minutes */
 
@@ -37,24 +37,24 @@ MODULE_PARM_DESC(nowayout,
 
 static int db8500_wdt_start(struct watchdog_device *wdd)
 {
-	return prcmu_enable_a9wdog(PRCMU_WDOG_ALL);
+	return db8500_prcmu_enable_a9wdog(PRCMU_WDOG_ALL);
 }
 
 static int db8500_wdt_stop(struct watchdog_device *wdd)
 {
-	return prcmu_disable_a9wdog(PRCMU_WDOG_ALL);
+	return db8500_prcmu_disable_a9wdog(PRCMU_WDOG_ALL);
 }
 
 static int db8500_wdt_keepalive(struct watchdog_device *wdd)
 {
-	return prcmu_kick_a9wdog(PRCMU_WDOG_ALL);
+	return db8500_prcmu_kick_a9wdog(PRCMU_WDOG_ALL);
 }
 
 static int db8500_wdt_set_timeout(struct watchdog_device *wdd,
 				 unsigned int timeout)
 {
 	db8500_wdt_stop(wdd);
-	prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000);
+	db8500_prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000);
 	db8500_wdt_start(wdd);
 
 	return 0;
@@ -91,10 +91,10 @@ static int db8500_wdt_probe(struct platform_device *pdev)
 	watchdog_set_nowayout(&db8500_wdt, nowayout);
 
 	/* disable auto off on sleep */
-	prcmu_config_a9wdog(PRCMU_WDOG_CPU1, false);
+	db8500_prcmu_config_a9wdog(PRCMU_WDOG_CPU1, false);
 
 	/* set HW initial value */
-	prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000);
+	db8500_prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000);
 
 	ret = devm_watchdog_register_device(dev, &db8500_wdt);
 	if (ret)
@@ -110,9 +110,9 @@ static int db8500_wdt_suspend(struct platform_device *pdev,
 {
 	if (watchdog_active(&db8500_wdt)) {
 		db8500_wdt_stop(&db8500_wdt);
-		prcmu_config_a9wdog(PRCMU_WDOG_CPU1, true);
+		db8500_prcmu_config_a9wdog(PRCMU_WDOG_CPU1, true);
 
-		prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000);
+		db8500_prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000);
 		db8500_wdt_start(&db8500_wdt);
 	}
 	return 0;
@@ -122,9 +122,9 @@ static int db8500_wdt_resume(struct platform_device *pdev)
 {
 	if (watchdog_active(&db8500_wdt)) {
 		db8500_wdt_stop(&db8500_wdt);
-		prcmu_config_a9wdog(PRCMU_WDOG_CPU1, false);
+		db8500_prcmu_config_a9wdog(PRCMU_WDOG_CPU1, false);
 
-		prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000);
+		db8500_prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000);
 		db8500_wdt_start(&db8500_wdt);
 	}
 	return 0;
diff --git a/drivers/watchdog/lenovo_se30g2_se60_wdt.c b/drivers/watchdog/lenovo_se30g2_se60_wdt.c
new file mode 100644
index 0000000000000..f31ecdb82ad05
--- /dev/null
+++ b/drivers/watchdog/lenovo_se30g2_se60_wdt.c
@@ -0,0 +1,289 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * WDT driver for Lenovo SE30G2 & SE60.
+ */
+
+#include <linux/delay.h>
+#include <linux/dmi.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/platform_device.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/watchdog.h>
+
+#define CFG_PORT 0x2E
+#define CFG_SIZE 2
+
+#define CFG_LDN	     0x07
+#define CFG_BRAM_LDN 0x10
+
+#define BRAM_SIZE 2
+
+#define BRAM_WDT_REG 0x48
+
+#define DRVNAME "lenovo-se30g2-se60-wdt"
+
+/*The timeout range is 1-255 seconds*/
+#define MIN_TIMEOUT 1
+#define MAX_TIMEOUT 255
+#define WATCHDOG_TIMEOUT 60 /* 60 sec default timeout */
+
+static unsigned short bram_base;
+static struct platform_device *se_30g2_60_pdev;
+
+static int timeout; /* in seconds */
+module_param(timeout, int, 0);
+MODULE_PARM_DESC(timeout,
+		 "Watchdog timeout in seconds. 1 <= timeout <= 255, default="
+		 __MODULE_STRING(WATCHDOG_TIMEOUT) ".");
+
+static bool nowayout = WATCHDOG_NOWAYOUT;
+module_param(nowayout, bool, 0);
+MODULE_PARM_DESC(nowayout,
+		 "Watchdog cannot be stopped once started (default="
+		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+
+struct se_30g2_60_wdt {
+	struct watchdog_device wdd;
+};
+
+static void enter_pnp_mode(void)
+{
+	outb(0x87, CFG_PORT);
+	outb(0x01, CFG_PORT);
+	outb(0x55, CFG_PORT);
+	outb(0x55, CFG_PORT);
+}
+
+static void exit_pnp_mode(void)
+{
+	outb(0x2, CFG_PORT);
+	outb(0x2, CFG_PORT + 1);
+}
+
+static void lpc_write(unsigned char index, unsigned char data)
+{
+	outb(index, CFG_PORT);
+	outb(data, CFG_PORT + 1);
+}
+
+static unsigned char lpc_read(unsigned char index)
+{
+	outb(index, CFG_PORT);
+	return inb(CFG_PORT + 1);
+}
+
+static unsigned short lpc_chip_id(void)
+{
+	unsigned char msb, lsb;
+
+	msb = lpc_read(0x20);
+	lsb = lpc_read(0x21);
+
+	return (msb << 8 | lsb);
+}
+
+static void bram_write(unsigned char reg, unsigned char val)
+{
+	outb(reg | 0x80, bram_base);
+	outb(val, bram_base + 1);
+}
+
+static unsigned char bram_read(unsigned char reg)
+{
+	unsigned char val;
+
+	outb(reg | 0x80, bram_base);
+	val = inb(bram_base + 1);
+	return val;
+}
+
+static int wdt_read(unsigned short *val)
+{
+	if (!request_muxed_region(bram_base, BRAM_SIZE, DRVNAME))
+		return -EACCES;
+
+	*val = ((bram_read(BRAM_WDT_REG) & 0xFF) << 8);
+	*val |= bram_read(BRAM_WDT_REG + 1) & 0xFF;
+	release_region(bram_base, BRAM_SIZE);
+
+	return 0;
+}
+
+static int wdt_write(unsigned short val)
+{
+	if (!request_muxed_region(bram_base, BRAM_SIZE, DRVNAME))
+		return -EACCES;
+
+	bram_write(BRAM_WDT_REG, (val >> 8) & 0xFF);
+	bram_write(BRAM_WDT_REG + 1, val & 0xFF);
+	release_region(bram_base, BRAM_SIZE);
+
+	return 0;
+}
+
+static int wdt_start(struct watchdog_device *wdog)
+{
+	return wdt_write(wdog->timeout);
+}
+
+static int wdt_stop(struct watchdog_device *wdog)
+{
+	return wdt_write(0);
+}
+
+static int wdt_ping(struct watchdog_device *wdog)
+{
+	return wdt_write(wdog->timeout);
+}
+
+static unsigned int wdt_get_timeleft(struct watchdog_device *wdog)
+{
+	unsigned short val;
+	int err;
+
+	err = wdt_read(&val);
+	return err ? 0 : val;
+}
+
+static const struct watchdog_info wdt_info = {
+	.options = WDIOF_SETTIMEOUT | WDIOF_KEEPALIVEPING | WDIOF_MAGICCLOSE,
+	.identity = "Lenovo SE30G2 SE60 WDOG",
+};
+
+static const struct watchdog_ops se_30g2_60_wdt_ops = {
+	.owner = THIS_MODULE,
+	.start = wdt_start,
+	.stop = wdt_stop,
+	.ping = wdt_ping,
+	.get_timeleft = wdt_get_timeleft,
+};
+
+static int se_30g2_60_wdt_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct se_30g2_60_wdt *priv;
+	unsigned int chip_id;
+	int ret;
+
+	if (!request_muxed_region(CFG_PORT, CFG_SIZE, DRVNAME))
+		return -EBUSY;
+
+	/* identify the chip */
+	enter_pnp_mode();
+	chip_id = lpc_chip_id();
+	if (chip_id != 0x5782) {
+		exit_pnp_mode();
+		release_region(CFG_PORT, CFG_SIZE);
+		return -ENODEV;
+	}
+
+	/* probe the BRAM base address */
+	lpc_write(CFG_LDN, CFG_BRAM_LDN);
+	bram_base = (lpc_read(0x60) << 8) | lpc_read(0x61);
+	exit_pnp_mode();
+	release_region(CFG_PORT, CFG_SIZE);
+	dev_info(dev, "Found Lenovo SE30G2 SE60 0x%x\n", chip_id);
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	watchdog_set_drvdata(&priv->wdd, priv);
+
+	priv->wdd.parent = dev;
+	priv->wdd.info = &wdt_info;
+	priv->wdd.ops = &se_30g2_60_wdt_ops;
+	priv->wdd.timeout = WATCHDOG_TIMEOUT; /* Set default timeout */
+	priv->wdd.min_timeout = MIN_TIMEOUT;
+	priv->wdd.max_timeout = MAX_TIMEOUT;
+
+	watchdog_init_timeout(&priv->wdd, timeout, dev);
+	watchdog_set_nowayout(&priv->wdd, nowayout);
+	watchdog_stop_on_reboot(&priv->wdd);
+	watchdog_stop_on_unregister(&priv->wdd);
+
+	ret = devm_watchdog_register_device(dev, &priv->wdd);
+
+	dev_dbg(&pdev->dev, "initialized. timeout=%d sec (nowayout=%d)\n",
+		priv->wdd.timeout, nowayout);
+
+	return ret;
+}
+
+static struct platform_driver se_30g2_60_wdt_driver = {
+	.driver = {
+		.name = DRVNAME,
+	},
+	.probe = se_30g2_60_wdt_probe,
+};
+
+static int se_30g2_60_create_device(const struct dmi_system_id *id)
+{
+	int err;
+
+	se_30g2_60_pdev = platform_device_alloc("lenovo-se30g2-se60-wdt", -1);
+	if (!se_30g2_60_pdev)
+		return -ENOMEM;
+
+	err = platform_device_add(se_30g2_60_pdev);
+	if (err) {
+		platform_device_put(se_30g2_60_pdev);
+		se_30g2_60_pdev = NULL;
+	}
+
+	return err;
+}
+
+static const struct dmi_system_id se_30g2_60[] __initconst = {
+	{
+		.ident = "LENOVO-SE30G2",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_BOARD_NAME, "33BD"),
+		},
+		.callback = se_30g2_60_create_device,
+	},
+	{
+		.ident = "LENOVO-SE60",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_BOARD_NAME, "33BF"),
+		},
+		.callback = se_30g2_60_create_device,
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(dmi, se_30g2_60);
+
+static int __init se_30g2_60_wdt_init(void)
+{
+	int err;
+
+	if (!dmi_check_system(se_30g2_60))
+		return -ENODEV;
+
+	err = platform_driver_register(&se_30g2_60_wdt_driver);
+	if (err && se_30g2_60_pdev) {
+		platform_device_unregister(se_30g2_60_pdev);
+		se_30g2_60_pdev = NULL;
+	}
+
+	return err;
+}
+
+static void __exit se_30g2_60_wdt_exit(void)
+{
+	if (se_30g2_60_pdev)
+		platform_device_unregister(se_30g2_60_pdev);
+	platform_driver_unregister(&se_30g2_60_wdt_driver);
+}
+
+module_init(se_30g2_60_wdt_init);
+module_exit(se_30g2_60_wdt_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Mark Pearson <mpearson-lenovo@squebb.ca>");
+MODULE_DESCRIPTION("WDT driver for Lenovo SE30G2 & SE60");
diff --git a/drivers/watchdog/max63xx_wdt.c b/drivers/watchdog/max63xx_wdt.c
index 3b4f3134d1c4b..e9073df1a7de3 100644
--- a/drivers/watchdog/max63xx_wdt.c
+++ b/drivers/watchdog/max63xx_wdt.c
@@ -245,13 +245,13 @@ static int max63xx_wdt_probe(struct platform_device *pdev)
 }
 
 static const struct platform_device_id max63xx_id_table[] = {
-	{ "max6369_wdt", (kernel_ulong_t)max6369_table, },
-	{ "max6370_wdt", (kernel_ulong_t)max6369_table, },
-	{ "max6371_wdt", (kernel_ulong_t)max6371_table, },
-	{ "max6372_wdt", (kernel_ulong_t)max6371_table, },
-	{ "max6373_wdt", (kernel_ulong_t)max6373_table, },
-	{ "max6374_wdt", (kernel_ulong_t)max6373_table, },
-	{ },
+	{ .name = "max6369_wdt", .driver_data = (kernel_ulong_t)max6369_table },
+	{ .name = "max6370_wdt", .driver_data = (kernel_ulong_t)max6369_table },
+	{ .name = "max6371_wdt", .driver_data = (kernel_ulong_t)max6371_table },
+	{ .name = "max6372_wdt", .driver_data = (kernel_ulong_t)max6371_table },
+	{ .name = "max6373_wdt", .driver_data = (kernel_ulong_t)max6373_table },
+	{ .name = "max6374_wdt", .driver_data = (kernel_ulong_t)max6373_table },
+	{ }
 };
 MODULE_DEVICE_TABLE(platform, max63xx_id_table);
 
diff --git a/drivers/watchdog/max77620_wdt.c b/drivers/watchdog/max77620_wdt.c
index 6ce435741d97b..ed5a98cea525d 100644
--- a/drivers/watchdog/max77620_wdt.c
+++ b/drivers/watchdog/max77620_wdt.c
@@ -235,9 +235,9 @@ static int max77620_wdt_probe(struct platform_device *pdev)
 }
 
 static const struct platform_device_id max77620_wdt_devtype[] = {
-	{ "max77620-watchdog", (kernel_ulong_t)&max77620_wdt_data },
-	{ "max77714-watchdog", (kernel_ulong_t)&max77714_wdt_data },
-	{ },
+	{ .name = "max77620-watchdog", .driver_data = (kernel_ulong_t)&max77620_wdt_data },
+	{ .name = "max77714-watchdog", .driver_data = (kernel_ulong_t)&max77714_wdt_data },
+	{ }
 };
 MODULE_DEVICE_TABLE(platform, max77620_wdt_devtype);
 
diff --git a/drivers/watchdog/ni903x_wdt.c b/drivers/watchdog/ni903x_wdt.c
index 8b1b9baa914e5..c72a9ee9cb8e1 100644
--- a/drivers/watchdog/ni903x_wdt.c
+++ b/drivers/watchdog/ni903x_wdt.c
@@ -183,9 +183,14 @@ static int ni903x_acpi_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct watchdog_device *wdd;
 	struct ni903x_wdt *wdt;
+	acpi_handle handle;
 	acpi_status status;
 	int ret;
 
+	handle = ACPI_HANDLE(dev);
+	if (!handle)
+		return -ENODEV;
+
 	wdt = devm_kzalloc(dev, sizeof(*wdt), GFP_KERNEL);
 	if (!wdt)
 		return -ENOMEM;
@@ -193,7 +198,7 @@ static int ni903x_acpi_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, wdt);
 	wdt->dev = dev;
 
-	status = acpi_walk_resources(ACPI_HANDLE(dev), METHOD_NAME__CRS,
+	status = acpi_walk_resources(handle, METHOD_NAME__CRS,
 				     ni903x_resources, wdt);
 	if (ACPI_FAILURE(status) || wdt->io_base == 0) {
 		dev_err(dev, "failed to get resources\n");
diff --git a/drivers/watchdog/pretimeout_dump.c b/drivers/watchdog/pretimeout_dump.c
new file mode 100644
index 0000000000000..c5d3dac2606c4
--- /dev/null
+++ b/drivers/watchdog/pretimeout_dump.c
@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2026 Google LLC
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/nmi.h>
+#include <linux/watchdog.h>
+
+#include "watchdog_pretimeout.h"
+
+/**
+ * pretimeout_dump - Dump on watchdog pretimeout event
+ * @wdd: watchdog_device
+ *
+ * Dump all cpu backtrace on pretimeout event.
+ */
+static void pretimeout_dump(struct watchdog_device *wdd)
+{
+	pr_alert("watchdog%d: pretimeout event\n", wdd->id);
+	if (!trigger_all_cpu_backtrace())
+		pr_alert("trigger_all_cpu_backtrace() isn't available\n");
+}
+
+static struct watchdog_governor watchdog_gov_dump = {
+	.name		= "dump",
+	.pretimeout	= pretimeout_dump,
+};
+
+static int __init watchdog_gov_dump_register(void)
+{
+	return watchdog_register_governor(&watchdog_gov_dump);
+}
+
+static void __exit watchdog_gov_dump_unregister(void)
+{
+	watchdog_unregister_governor(&watchdog_gov_dump);
+}
+module_init(watchdog_gov_dump_register);
+module_exit(watchdog_gov_dump_unregister);
+
+MODULE_AUTHOR("Tzung-Bi Shih <tzungbi@kernel.org>");
+MODULE_DESCRIPTION("Dump watchdog pretimeout governor");
+MODULE_LICENSE("GPL");
diff --git a/drivers/watchdog/s32g_wdt.c b/drivers/watchdog/s32g_wdt.c
index ad55063060af6..6422a694fc65f 100644
--- a/drivers/watchdog/s32g_wdt.c
+++ b/drivers/watchdog/s32g_wdt.c
@@ -56,8 +56,7 @@ MODULE_PARM_DESC(early_enable,
 
 static const struct watchdog_info s32g_wdt_info = {
 	.identity = "s32g watchdog",
-	.options = WDIOF_KEEPALIVEPING | WDIOF_SETTIMEOUT | WDIOF_MAGICCLOSE |
-	WDIOC_GETTIMEOUT | WDIOC_GETTIMELEFT,
+	.options = WDIOF_KEEPALIVEPING | WDIOF_SETTIMEOUT | WDIOF_MAGICCLOSE,
 };
 
 static struct s32g_wdt_device *wdd_to_s32g_wdt(struct watchdog_device *wdd)
diff --git a/drivers/watchdog/w83627hf_wdt.c b/drivers/watchdog/w83627hf_wdt.c
index bc33b63c5a5df..db77599e43a06 100644
--- a/drivers/watchdog/w83627hf_wdt.c
+++ b/drivers/watchdog/w83627hf_wdt.c
@@ -49,7 +49,7 @@ static int wdt_cfg_leave = 0xAA;/* key to lock configuration space */
 enum chips { w83627hf, w83627s, w83697hf, w83697ug, w83637hf, w83627thf,
 	     w83687thf, w83627ehf, w83627dhg, w83627uhg, w83667hg, w83627dhg_p,
 	     w83667hg_b, nct6775, nct6776, nct6779, nct6791, nct6792, nct6793,
-	     nct6795, nct6796, nct6102, nct6116 };
+	     nct6795, nct6796, nct6102, nct6116, nct6126 };
 
 static int timeout;			/* in seconds */
 module_param(timeout, int, 0);
@@ -94,7 +94,9 @@ MODULE_PARM_DESC(early_disable, "Disable watchdog at boot time (default=0)");
 #define NCT6775_ID		0xb4
 #define NCT6776_ID		0xc3
 #define NCT6102_ID		0xc4
-#define NCT6116_ID		0xd2
+#define NCT6116_ID		0xd2	/* also NCT6126D */
+#define NCT6126_VER_A_LOW_ID	0x83	/* ... version A */
+#define NCT6126_VER_B_LOW_ID	0x84	/* ... version B */
 #define NCT6779_ID		0xc5
 #define NCT6791_ID		0xc8
 #define NCT6792_ID		0xc9
@@ -217,6 +219,7 @@ static int w83627hf_init(struct watchdog_device *wdog, enum chips chip)
 	case nct6796:
 	case nct6102:
 	case nct6116:
+	case nct6126:
 		/*
 		 * These chips have a fixed WDTO# output pin (W83627UHG),
 		 * or support more than one WDTO# output pin.
@@ -241,6 +244,7 @@ static int w83627hf_init(struct watchdog_device *wdog, enum chips chip)
 			pr_info("Watchdog already running. Resetting timeout to %d sec\n",
 				wdog->timeout);
 			superio_outb(cr_wdt_timeout, wdog->timeout);
+			set_bit(WDOG_HW_RUNNING, &wdog->status);
 		}
 	}
 
@@ -313,9 +317,8 @@ static unsigned int wdt_get_time(struct watchdog_device *wdog)
  *	Kernel Interfaces
  */
 
-static const struct watchdog_info wdt_info = {
+static struct watchdog_info wdt_info = {
 	.options = WDIOF_SETTIMEOUT | WDIOF_KEEPALIVEPING | WDIOF_MAGICCLOSE,
-	.identity = "W83627HF Watchdog",
 };
 
 static const struct watchdog_ops wdt_ops = {
@@ -428,7 +431,12 @@ static int wdt_find(int addr)
 		cr_wdt_csr = NCT6102D_WDT_CSR;
 		break;
 	case NCT6116_ID:
-		ret = nct6116;
+		val = superio_inb(0x21);
+		if (val == NCT6126_VER_A_LOW_ID || val == NCT6126_VER_B_LOW_ID)
+			ret = nct6126;
+		else
+			ret = nct6116;
+
 		cr_wdt_timeout = NCT6102D_WDT_TIMEOUT;
 		cr_wdt_control = NCT6102D_WDT_CONTROL;
 		cr_wdt_csr = NCT6102D_WDT_CSR;
@@ -499,6 +507,7 @@ static int __init wdt_init(void)
 		"NCT6796",
 		"NCT6102",
 		"NCT6116",
+		"NCT6126"
 	};
 
 	/* Apply system-specific quirks */
@@ -516,6 +525,9 @@ static int __init wdt_init(void)
 	pr_info("WDT driver for %s Super I/O chip initialising\n",
 		chip_name[chip]);
 
+	snprintf(wdt_info.identity, sizeof(wdt_info.identity), "%s Watchdog",
+		 chip_name[chip]);
+
 	watchdog_init_timeout(&wdt_dev, timeout, NULL);
 	watchdog_set_nowayout(&wdt_dev, nowayout);
 	watchdog_stop_on_reboot(&wdt_dev);
diff --git a/drivers/watchdog/watchdog_pretimeout.c b/drivers/watchdog/watchdog_pretimeout.c
index 19eb2ed2c7cb0..02e09b9e396da 100644
--- a/drivers/watchdog/watchdog_pretimeout.c
+++ b/drivers/watchdog/watchdog_pretimeout.c
@@ -167,6 +167,8 @@ void watchdog_unregister_governor(struct watchdog_governor *gov)
 	}
 
 	spin_lock_irq(&pretimeout_lock);
+	if (default_gov == gov)
+		default_gov = NULL;
 	list_for_each_entry(p, &pretimeout_list, entry)
 		if (p->wdd->gov == gov)
 			p->wdd->gov = default_gov;
diff --git a/drivers/watchdog/watchdog_pretimeout.h b/drivers/watchdog/watchdog_pretimeout.h
index a3f1abc68839a..57aa790e2669f 100644
--- a/drivers/watchdog/watchdog_pretimeout.h
+++ b/drivers/watchdog/watchdog_pretimeout.h
@@ -24,7 +24,9 @@ int watchdog_pretimeout_governor_get(struct watchdog_device *wdd, char *buf);
 int watchdog_pretimeout_governor_set(struct watchdog_device *wdd,
 				     const char *buf);
 
-#if IS_ENABLED(CONFIG_WATCHDOG_PRETIMEOUT_DEFAULT_GOV_NOOP)
+#if IS_ENABLED(CONFIG_WATCHDOG_PRETIMEOUT_DEFAULT_GOV_DUMP)
+#define WATCHDOG_PRETIMEOUT_DEFAULT_GOV		"dump"
+#elif IS_ENABLED(CONFIG_WATCHDOG_PRETIMEOUT_DEFAULT_GOV_NOOP)
 #define WATCHDOG_PRETIMEOUT_DEFAULT_GOV		"noop"
 #elif IS_ENABLED(CONFIG_WATCHDOG_PRETIMEOUT_DEFAULT_GOV_PANIC)
 #define WATCHDOG_PRETIMEOUT_DEFAULT_GOV		"panic"
diff --git a/drivers/xen/xen-acpi-pad.c b/drivers/xen/xen-acpi-pad.c
index 5b98e0e938076..2fa9c7d3581fd 100644
--- a/drivers/xen/xen-acpi-pad.c
+++ b/drivers/xen/xen-acpi-pad.c
@@ -17,8 +17,6 @@
 #include <xen/xen-ops.h>
 #include <asm/xen/hypercall.h>
 
-#define ACPI_PROCESSOR_AGGREGATOR_CLASS	"acpi_pad"
-#define ACPI_PROCESSOR_AGGREGATOR_DEVICE_NAME "Processor Aggregator"
 #define ACPI_PROCESSOR_AGGREGATOR_NOTIFY 0x80
 static DEFINE_MUTEX(xen_cpu_lock);
 
@@ -117,9 +115,6 @@ static int acpi_pad_probe(struct platform_device *pdev)
 	if (!device)
 		return -ENODEV;
 
-	strcpy(acpi_device_name(device), ACPI_PROCESSOR_AGGREGATOR_DEVICE_NAME);
-	strcpy(acpi_device_class(device), ACPI_PROCESSOR_AGGREGATOR_CLASS);
-
 	status = acpi_install_notify_handler(device->handle,
 		ACPI_DEVICE_NOTIFY, acpi_pad_notify, device);
 	if (ACPI_FAILURE(status))
diff --git a/drivers/xen/xen-scsiback.c b/drivers/xen/xen-scsiback.c
index e33f95c91b096..c7036e0e41bda 100644
--- a/drivers/xen/xen-scsiback.c
+++ b/drivers/xen/xen-scsiback.c
@@ -611,6 +611,25 @@ static void scsiback_disconnect(struct vscsibk_info *info)
 	xenbus_unmap_ring_vfree(info->dev, info->ring.sring);
 }
 
+/*
+ * Send the error response for a request that did not reach the target core
+ * and return its tag.  Free the tag before the response drops the v2p
+ * reference that keeps the session alive, and snapshot what the response
+ * needs since returning the tag can let the slot be reused.
+ */
+static void scsiback_resp_and_free(struct vscsibk_pend *pending_req,
+				   int32_t result)
+{
+	struct vscsibk_info *info = pending_req->info;
+	struct v2p_entry *v2p = pending_req->v2p;
+	struct se_session *se_sess = v2p->tpg->tpg_nexus->tvn_se_sess;
+	u16 rqid = pending_req->rqid;
+
+	target_free_tag(se_sess, &pending_req->se_cmd);
+	scsiback_send_response(info, NULL, result, 0, rqid);
+	kref_put(&v2p->kref, scsiback_free_translation_entry);
+}
+
 static void scsiback_device_action(struct vscsibk_pend *pending_req,
 	enum tcm_tmreq_table act, int tag)
 {
@@ -639,7 +658,7 @@ static void scsiback_device_action(struct vscsibk_pend *pending_req,
 	return;
 
 err:
-	scsiback_do_resp_with_sense(NULL, err, 0, pending_req);
+	scsiback_resp_and_free(pending_req, err);
 }
 
 /*
@@ -792,9 +811,8 @@ static int scsiback_do_cmd_fn(struct vscsibk_info *info,
 		case VSCSIIF_ACT_SCSI_CDB:
 			if (scsiback_gnttab_data_map(&ring_req, pending_req)) {
 				scsiback_fast_flush_area(pending_req);
-				scsiback_do_resp_with_sense(NULL,
-						DID_ERROR << 16, 0, pending_req);
-				transport_generic_free_cmd(&pending_req->se_cmd, 0);
+				scsiback_resp_and_free(pending_req,
+						       DID_ERROR << 16);
 			} else {
 				scsiback_cmd_exec(pending_req);
 			}
@@ -808,9 +826,7 @@ static int scsiback_do_cmd_fn(struct vscsibk_info *info,
 			break;
 		default:
 			pr_err_ratelimited("invalid request\n");
-			scsiback_do_resp_with_sense(NULL, DID_ERROR << 16, 0,
-						    pending_req);
-			transport_generic_free_cmd(&pending_req->se_cmd, 0);
+			scsiback_resp_and_free(pending_req, DID_ERROR << 16);
 			break;
 		}
 
diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c
index 5783d0336f96a..3829554ca3692 100644
--- a/fs/9p/vfs_inode.c
+++ b/fs/9p/vfs_inode.c
@@ -645,7 +645,6 @@ v9fs_create(struct v9fs_session_info *v9ses, struct inode *dir,
  * @dir: The parent directory
  * @dentry: The name of file to be created
  * @mode: The UNIX file mode to set
- * @excl: True if the file must not yet exist
  *
  * open(.., O_CREAT) is handled in v9fs_vfs_atomic_open().  This is only called
  * for mknod(2).
@@ -654,7 +653,7 @@ v9fs_create(struct v9fs_session_info *v9ses, struct inode *dir,
 
 static int
 v9fs_vfs_create(struct mnt_idmap *idmap, struct inode *dir,
-		struct dentry *dentry, umode_t mode, bool excl)
+		struct dentry *dentry, umode_t mode)
 {
 	struct v9fs_session_info *v9ses = v9fs_inode2v9ses(dir);
 	u32 perm = unixmode2p9mode(v9ses, mode);
@@ -689,7 +688,7 @@ static struct dentry *v9fs_vfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 
 	p9_debug(P9_DEBUG_VFS, "name %pd\n", dentry);
 	v9ses = v9fs_inode2v9ses(dir);
-	perm = unixmode2p9mode(v9ses, mode | S_IFDIR);
+	perm = unixmode2p9mode(v9ses, mode);
 	fid = v9fs_create(v9ses, dir, dentry, NULL, perm, P9_OREAD);
 	if (IS_ERR(fid))
 		return ERR_CAST(fid);
diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c
index f7396d20cb6cd..116b29e95f21e 100644
--- a/fs/9p/vfs_inode_dotl.c
+++ b/fs/9p/vfs_inode_dotl.c
@@ -213,12 +213,11 @@ int v9fs_open_to_dotl_flags(int flags)
  * @dir: directory inode that is being created
  * @dentry:  dentry that is being deleted
  * @omode: create permissions
- * @excl: True if the file must not yet exist
  *
  */
 static int
 v9fs_vfs_create_dotl(struct mnt_idmap *idmap, struct inode *dir,
-		     struct dentry *dentry, umode_t omode, bool excl)
+		     struct dentry *dentry, umode_t omode)
 {
 	return v9fs_vfs_mknod_dotl(idmap, dir, dentry, omode, 0);
 }
@@ -362,7 +361,6 @@ static struct dentry *v9fs_vfs_mkdir_dotl(struct mnt_idmap *idmap,
 	p9_debug(P9_DEBUG_VFS, "name %pd\n", dentry);
 	v9ses = v9fs_inode2v9ses(dir);
 
-	omode |= S_IFDIR;
 	if (dir->i_mode & S_ISGID)
 		omode |= S_ISGID;
 
diff --git a/fs/Kconfig b/fs/Kconfig
index cf6ae64776e62..64ff193fb42cc 100644
--- a/fs/Kconfig
+++ b/fs/Kconfig
@@ -315,7 +315,6 @@ source "fs/hfs/Kconfig"
 source "fs/hfsplus/Kconfig"
 source "fs/befs/Kconfig"
 source "fs/bfs/Kconfig"
-source "fs/efs/Kconfig"
 source "fs/jffs2/Kconfig"
 # UBIFS File system configuration
 source "fs/ubifs/Kconfig"
diff --git a/fs/Makefile b/fs/Makefile
index 89a8a9d207d1c..aa847be93bc6f 100644
--- a/fs/Makefile
+++ b/fs/Makefile
@@ -92,7 +92,6 @@ obj-$(CONFIG_HPFS_FS)		+= hpfs/
 obj-$(CONFIG_NTFS_FS)		+= ntfs/
 obj-$(CONFIG_NTFS3_FS)		+= ntfs3/
 obj-$(CONFIG_UFS_FS)		+= ufs/
-obj-$(CONFIG_EFS_FS)		+= efs/
 obj-$(CONFIG_JFFS2_FS)		+= jffs2/
 obj-$(CONFIG_UBIFS_FS)		+= ubifs/
 obj-$(CONFIG_AFFS_FS)		+= affs/
diff --git a/fs/affs/affs.h b/fs/affs/affs.h
index 44a3f69d275f9..d6b3393633f2d 100644
--- a/fs/affs/affs.h
+++ b/fs/affs/affs.h
@@ -169,7 +169,7 @@ extern int	affs_hash_name(struct super_block *sb, const u8 *name, unsigned int l
 extern struct dentry *affs_lookup(struct inode *dir, struct dentry *dentry, unsigned int);
 extern int	affs_unlink(struct inode *dir, struct dentry *dentry);
 extern int	affs_create(struct mnt_idmap *idmap, struct inode *dir,
-			struct dentry *dentry, umode_t mode, bool);
+			struct dentry *dentry, umode_t mode);
 extern struct dentry *affs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, umode_t mode);
 extern int	affs_rmdir(struct inode *dir, struct dentry *dentry);
diff --git a/fs/affs/namei.c b/fs/affs/namei.c
index c3c6532da4b0d..5311828b19e8a 100644
--- a/fs/affs/namei.c
+++ b/fs/affs/namei.c
@@ -243,7 +243,7 @@ affs_unlink(struct inode *dir, struct dentry *dentry)
 
 int
 affs_create(struct mnt_idmap *idmap, struct inode *dir,
-	    struct dentry *dentry, umode_t mode, bool excl)
+	    struct dentry *dentry, umode_t mode)
 {
 	struct super_block *sb = dir->i_sb;
 	struct inode	*inode;
@@ -287,7 +287,7 @@ affs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	if (!inode)
 		return ERR_PTR(-ENOSPC);
 
-	inode->i_mode = S_IFDIR | mode;
+	inode->i_mode = mode;
 	affs_mode_to_prot(inode);
 
 	inode->i_op = &affs_dir_inode_operations;
diff --git a/fs/afs/dir.c b/fs/afs/dir.c
index 6df56fe9163fd..81565366d937b 100644
--- a/fs/afs/dir.c
+++ b/fs/afs/dir.c
@@ -34,7 +34,7 @@ static bool afs_lookup_filldir(struct dir_context *ctx, const char *name, int nl
 			       u64 ino, u32 uniquifier);
 #define AFS_LOOKUP ((filldir_t)0x137UL)
 static int afs_create(struct mnt_idmap *idmap, struct inode *dir,
-		      struct dentry *dentry, umode_t mode, bool excl);
+		      struct dentry *dentry, umode_t mode);
 static struct dentry *afs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 				struct dentry *dentry, umode_t mode);
 static int afs_rmdir(struct inode *dir, struct dentry *dentry);
@@ -1333,7 +1333,7 @@ static struct dentry *afs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	op->file[0].modification = true;
 	op->file[0].update_ctime = true;
 	op->dentry	= dentry;
-	op->create.mode	= S_IFDIR | mode;
+	op->create.mode	= mode;
 	op->create.reason = afs_edit_dir_for_mkdir;
 	op->mtime	= current_time(dir);
 	op->ops		= &afs_mkdir_operation;
@@ -1633,7 +1633,7 @@ static const struct afs_operation_ops afs_create_operation = {
  * create a regular file on an AFS filesystem
  */
 static int afs_create(struct mnt_idmap *idmap, struct inode *dir,
-		      struct dentry *dentry, umode_t mode, bool excl)
+		      struct dentry *dentry, umode_t mode)
 {
 	struct afs_operation *op;
 	struct afs_vnode *dvnode = AFS_FS_I(dir);
diff --git a/fs/aio.c b/fs/aio.c
index f57fa21a25035..3de586da197bd 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -604,7 +604,7 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events)
 
 	ctx->mmap_base = do_mmap(ctx->aio_ring_file, 0, ctx->mmap_size,
 				 PROT_READ | PROT_WRITE,
-				 MAP_SHARED, 0, 0, &unused, NULL);
+				 MAP_SHARED, EMPTY_VMA_FLAGS, 0, &unused, NULL);
 	mmap_write_unlock(mm);
 	if (IS_ERR((void *)ctx->mmap_base)) {
 		ctx->mmap_size = 0;
diff --git a/fs/autofs/root.c b/fs/autofs/root.c
index 186e960f1e235..b36439f4521eb 100644
--- a/fs/autofs/root.c
+++ b/fs/autofs/root.c
@@ -741,7 +741,7 @@ static struct dentry *autofs_dir_mkdir(struct mnt_idmap *idmap,
 
 	autofs_del_active(dentry);
 
-	inode = autofs_get_inode(dir->i_sb, S_IFDIR | mode);
+	inode = autofs_get_inode(dir->i_sb, mode);
 	if (!inode)
 		return ERR_PTR(-ENOMEM);
 
diff --git a/fs/bad_inode.c b/fs/bad_inode.c
index acf8613f5e368..486c40f73e51e 100644
--- a/fs/bad_inode.c
+++ b/fs/bad_inode.c
@@ -29,7 +29,7 @@ static const struct file_operations bad_file_ops =
 
 static int bad_inode_create(struct mnt_idmap *idmap,
 			    struct inode *dir, struct dentry *dentry,
-			    umode_t mode, bool excl)
+			    umode_t mode)
 {
 	return -EIO;
 }
diff --git a/fs/bfs/dir.c b/fs/bfs/dir.c
index 5b40ab09a796d..4c3b4db08cdef 100644
--- a/fs/bfs/dir.c
+++ b/fs/bfs/dir.c
@@ -83,7 +83,7 @@ const struct file_operations bfs_dir_operations = {
 };
 
 static int bfs_create(struct mnt_idmap *idmap, struct inode *dir,
-		      struct dentry *dentry, umode_t mode, bool excl)
+		      struct dentry *dentry, umode_t mode)
 {
 	int err;
 	struct inode *inode;
diff --git a/fs/bpf_fs_kfuncs.c b/fs/bpf_fs_kfuncs.c
index f1863a891db64..6efe3e53c7421 100644
--- a/fs/bpf_fs_kfuncs.c
+++ b/fs/bpf_fs_kfuncs.c
@@ -11,6 +11,7 @@
 #include <linux/file.h>
 #include <linux/kernfs.h>
 #include <linux/mm.h>
+#include <linux/net.h>
 #include <linux/xattr.h>
 
 __bpf_kfunc_start_defs();
@@ -359,6 +360,39 @@ __bpf_kfunc int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__s
 }
 #endif /* CONFIG_CGROUPS */
 
+#ifdef CONFIG_NET
+/**
+ * bpf_sock_read_xattr - read xattr of a socket's inode in sockfs
+ * @sock: socket to get xattr from
+ * @name__str: name of the xattr
+ * @value_p: output buffer of the xattr value
+ *
+ * Get xattr *name__str* of *sock* and store the output in *value_p*.
+ *
+ * For security reasons, only *name__str* with prefix "user." is allowed.
+ *
+ * Return: length of the xattr value on success, a negative value on error.
+ */
+__bpf_kfunc int bpf_sock_read_xattr(struct socket *sock, const char *name__str,
+				    struct bpf_dynptr *value_p)
+{
+	struct bpf_dynptr_kern *value_ptr = (struct bpf_dynptr_kern *)value_p;
+	u32 value_len;
+	void *value;
+
+	/* Only allow reading "user.*" xattrs */
+	if (strncmp(name__str, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN))
+		return -EPERM;
+
+	value_len = __bpf_dynptr_size(value_ptr);
+	value = __bpf_dynptr_data_rw(value_ptr, value_len);
+	if (!value)
+		return -EINVAL;
+
+	return sock_read_xattr(sock, name__str, value, value_len);
+}
+#endif /* CONFIG_NET */
+
 /**
  * bpf_real_data_inode - get the real inode hosting a file's data
  * @file: file to resolve
@@ -390,6 +424,9 @@ BTF_ID_FLAGS(func, bpf_get_file_xattr, KF_SLEEPABLE)
 BTF_ID_FLAGS(func, bpf_set_dentry_xattr, KF_SLEEPABLE)
 BTF_ID_FLAGS(func, bpf_remove_dentry_xattr, KF_SLEEPABLE)
 BTF_ID_FLAGS(func, bpf_real_data_inode, KF_SLEEPABLE | KF_RET_NULL)
+#ifdef CONFIG_NET
+BTF_ID_FLAGS(func, bpf_sock_read_xattr, KF_RCU)
+#endif
 BTF_KFUNCS_END(bpf_fs_kfunc_set_ids)
 
 static int bpf_fs_kfuncs_filter(const struct bpf_prog *prog, u32 kfunc_id)
diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c
index 23c3eeb58dc1d..1be632c742bdd 100644
--- a/fs/btrfs/backref.c
+++ b/fs/btrfs/backref.c
@@ -3054,7 +3054,10 @@ void btrfs_backref_free_node(struct btrfs_backref_cache *cache,
 	if (node) {
 		ASSERT(list_empty(&node->list));
 		ASSERT(list_empty(&node->lower));
-		ASSERT(node->eb == NULL, "node->eb->start=%llu", node->eb->start);
+		ASSERT(node->eb == NULL, "node->eb->start=%llu level=%d owner=%llu",
+		       node->eb ? node->eb->start : 0,
+		       node->eb ? btrfs_header_level(node->eb) : 0,
+		       node->eb ? btrfs_header_owner(node->eb) : 0);
 		cache->nr_nodes--;
 		btrfs_put_root(node->root);
 		kfree(node);
diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h
index d5d81f9546c37..7fdc6c3fd0666 100644
--- a/fs/btrfs/btrfs_inode.h
+++ b/fs/btrfs/btrfs_inode.h
@@ -476,6 +476,8 @@ static inline bool btrfs_inode_can_compress(const struct btrfs_inode *inode)
 	if (inode->flags & BTRFS_INODE_NODATACOW ||
 	    inode->flags & BTRFS_INODE_NODATASUM)
 		return false;
+	if (btrfs_is_data_reloc_root(inode->root))
+		return false;
 	return true;
 }
 
diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c
index 318ddb7904292..dc0834f920c3b 100644
--- a/fs/btrfs/dev-replace.c
+++ b/fs/btrfs/dev-replace.c
@@ -247,8 +247,8 @@ static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
 		return -EINVAL;
 	}
 
-	bdev_file = bdev_file_open_by_path(device_path, BLK_OPEN_WRITE,
-					   fs_info->sb, &fs_holder_ops);
+	/* Unfreezable for the whole replace; see btrfs_dev_replace_start(). */
+	bdev_file = btrfs_open_device_deny_freeze(device_path, fs_info->sb);
 	if (IS_ERR(bdev_file)) {
 		btrfs_err(fs_info, "target device %s is invalid!", device_path);
 		return PTR_ERR(bdev_file);
@@ -327,7 +327,8 @@ static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
 	return 0;
 
 error:
-	bdev_fput(bdev_file);
+	/* Undo the open-time freeze deny. */
+	btrfs_release_device_allow_freeze(bdev_file);
 	return ret;
 }
 
@@ -624,6 +625,15 @@ static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info,
 	if (ret)
 		return ret;
 
+	/* Deny the source before mark, so every 'leave' unwinds both denied. */
+	if (src_device->bdev) {
+		ret = bdev_deny_freeze(src_device->bdev);
+		if (ret) {
+			btrfs_destroy_dev_replace_tgtdev(tgt_device, true);
+			return ret;
+		}
+	}
+
 	ret = mark_block_group_to_copy(fs_info, src_device);
 	if (ret)
 		return ret;
@@ -708,7 +718,9 @@ static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info,
 	return ret;
 
 leave:
-	btrfs_destroy_dev_replace_tgtdev(tgt_device);
+	if (src_device->bdev)
+		bdev_allow_freeze(src_device->bdev);
+	btrfs_destroy_dev_replace_tgtdev(tgt_device, true);
 	return ret;
 }
 
@@ -889,6 +901,7 @@ static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
 	 */
 	ret = btrfs_start_delalloc_roots(fs_info, LONG_MAX, false);
 	if (ret) {
+		/* Stays started/resumable; keep both denied. */
 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
 		return ret;
 	}
@@ -902,6 +915,7 @@ static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
 	while (1) {
 		trans = btrfs_start_transaction(root, 0);
 		if (IS_ERR(trans)) {
+			/* Stays started/resumable; keep both denied. */
 			mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
 			return PTR_ERR(trans);
 		}
@@ -954,7 +968,10 @@ static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
 		mutex_unlock(&fs_devices->device_list_mutex);
 		btrfs_rm_dev_replace_blocked(fs_info);
 		if (tgt_device)
-			btrfs_destroy_dev_replace_tgtdev(tgt_device);
+			btrfs_destroy_dev_replace_tgtdev(tgt_device, true);
+		/* The source stays a member; re-allow freezing it. */
+		if (src_device->bdev)
+			bdev_allow_freeze(src_device->bdev);
 		btrfs_rm_dev_replace_unblocked(fs_info);
 		mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
 
@@ -1027,6 +1044,8 @@ static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
 
 	mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
 
+	/* The target is now a member; the source is freed (allow + release). */
+	bdev_allow_freeze(tgt_device->bdev);
 	btrfs_rm_dev_replace_free_srcdev(src_device);
 
 	return 0;
@@ -1155,8 +1174,9 @@ int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info)
 			btrfs_dev_name(src_device), src_device->devid,
 			btrfs_dev_name(tgt_device));
 
+		/* A suspended replace never re-denied freezing; do not allow. */
 		if (tgt_device)
-			btrfs_destroy_dev_replace_tgtdev(tgt_device);
+			btrfs_destroy_dev_replace_tgtdev(tgt_device, false);
 		break;
 	default:
 		up_write(&dev_replace->rwsem);
@@ -1186,6 +1206,11 @@ void btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info *fs_info)
 		dev_replace->time_stopped = ktime_get_real_seconds();
 		dev_replace->item_needs_writeback = 1;
 		btrfs_info(fs_info, "suspending dev_replace for unmount");
+		/* Reopened freezable next mount; resume re-denies. */
+		if (dev_replace->srcdev && dev_replace->srcdev->bdev)
+			bdev_allow_freeze(dev_replace->srcdev->bdev);
+		if (dev_replace->tgtdev && dev_replace->tgtdev->bdev)
+			bdev_allow_freeze(dev_replace->tgtdev->bdev);
 		break;
 	}
 
@@ -1198,6 +1223,7 @@ int btrfs_resume_dev_replace_async(struct btrfs_fs_info *fs_info)
 {
 	struct task_struct *task;
 	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
+	int ret = 0;
 
 	down_write(&dev_replace->rwsem);
 
@@ -1241,8 +1267,33 @@ int btrfs_resume_dev_replace_async(struct btrfs_fs_info *fs_info)
 		return 0;
 	}
 
+	/* Re-deny for the resumed replace; stay suspended if frozen now. */
+	if (dev_replace->srcdev->bdev &&
+	    bdev_deny_freeze(dev_replace->srcdev->bdev))
+		goto suspend;
+	if (bdev_deny_freeze(dev_replace->tgtdev->bdev)) {
+		if (dev_replace->srcdev->bdev)
+			bdev_allow_freeze(dev_replace->srcdev->bdev);
+		goto suspend;
+	}
+
 	task = kthread_run(btrfs_dev_replace_kthread, fs_info, "btrfs-devrepl");
-	return PTR_ERR_OR_ZERO(task);
+	if (IS_ERR(task)) {
+		bdev_allow_freeze(dev_replace->tgtdev->bdev);
+		if (dev_replace->srcdev->bdev)
+			bdev_allow_freeze(dev_replace->srcdev->bdev);
+		/* Undo the deny and suspend, but still fail the mount. */
+		ret = PTR_ERR(task);
+		goto suspend;
+	}
+	return 0;
+
+suspend:
+	btrfs_exclop_finish(fs_info);
+	down_write(&dev_replace->rwsem);
+	dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED;
+	up_write(&dev_replace->rwsem);
+	return ret;
 }
 
 static int btrfs_dev_replace_kthread(void *data)
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 7d604524e83c3..de5785117a47f 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -2004,7 +2004,7 @@ static noinline_for_stack bool lock_extent_buffer_for_io(struct extent_buffer *e
 
 		btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
 		percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
-					 -eb->len,
+					 -(s64)eb->len,
 					 fs_info->dirty_metadata_batch);
 		ret = true;
 	} else {
@@ -3774,7 +3774,7 @@ void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
 		return;
 
 	buffer_tree_clear_mark(eb, PAGECACHE_TAG_DIRTY);
-	percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -eb->len,
+	percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -(s64)eb->len,
 				 fs_info->dirty_metadata_batch);
 
 	for (int i = 0; i < num_extent_folios(eb); i++) {
diff --git a/fs/btrfs/extent_map.c b/fs/btrfs/extent_map.c
index fce9c5cc01228..6ad7b39ae358b 100644
--- a/fs/btrfs/extent_map.c
+++ b/fs/btrfs/extent_map.c
@@ -866,13 +866,13 @@ void btrfs_drop_extent_map_range(struct btrfs_inode *inode, u64 start, u64 end,
 			goto next;
 		}
 
-		flags = em->flags;
 		/*
 		 * In case we split the extent map, we want to preserve the
 		 * EXTENT_FLAG_LOGGING flag on our extent map, but we don't want
 		 * it on the new extent maps.
 		 */
-		em->flags &= ~(EXTENT_FLAG_PINNED | EXTENT_FLAG_LOGGING);
+		flags = em->flags & ~EXTENT_FLAG_LOGGING;
+		em->flags &= ~EXTENT_FLAG_PINNED;
 		modified = !list_empty(&em->list);
 
 		/*
diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c
index 9f6454e9db818..cf50fd623f41a 100644
--- a/fs/btrfs/file-item.c
+++ b/fs/btrfs/file-item.c
@@ -358,6 +358,7 @@ int btrfs_lookup_bio_sums(struct btrfs_bio *bbio)
 	const unsigned int nblocks = orig_len >> fs_info->sectorsize_bits;
 	int ret = 0;
 	u32 bio_offset = 0;
+	bool using_commit_root = false;
 
 	if ((inode->flags & BTRFS_INODE_NODATASUM) ||
 	    test_bit(BTRFS_FS_STATE_NO_DATA_CSUMS, &fs_info->fs_state))
@@ -431,6 +432,7 @@ int btrfs_lookup_bio_sums(struct btrfs_bio *bbio)
 	 * from across transactions.
 	 */
 	if (bbio->csum_search_commit_root) {
+		using_commit_root = true;
 		path->search_commit_root = true;
 		path->skip_locking = true;
 		down_read(&fs_info->commit_root_sem);
@@ -463,6 +465,28 @@ int btrfs_lookup_bio_sums(struct btrfs_bio *bbio)
 		 * assume this is the case.
 		 */
 		if (count == 0) {
+			/*
+			 * If an extent is relocated in the current transaction
+			 * then relocation writes a new csum without updating
+			 * the extent map generation. Until the next commit, we
+			 * will see a hole in that case, so we need to fallback
+			 * to searching the transaction csum root.
+			 *
+			 * Note that a commit root lookup of a referenced extent can
+			 * only miss, not return a stale csum. A freed extent's csum
+			 * is deleted in the same transaction and its bytenr is not
+			 * reusable until that transaction has committed and the
+			 * extent is unpinned.
+			 */
+			if (using_commit_root) {
+				up_read(&fs_info->commit_root_sem);
+				using_commit_root = false;
+				path->search_commit_root = false;
+				path->skip_locking = false;
+				btrfs_release_path(path);
+				continue;
+			}
+
 			memset(csum_dst, 0, csum_size);
 			count = 1;
 
@@ -481,7 +505,7 @@ int btrfs_lookup_bio_sums(struct btrfs_bio *bbio)
 		bio_offset += count * sectorsize;
 	}
 
-	if (bbio->csum_search_commit_root)
+	if (using_commit_root)
 		up_read(&fs_info->commit_root_sem);
 	return ret;
 }
diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c
index 6009b1477232d..e2af75a205ea5 100644
--- a/fs/btrfs/free-space-cache.c
+++ b/fs/btrfs/free-space-cache.c
@@ -551,6 +551,9 @@ static int io_ctl_check_crc(struct btrfs_io_ctl *io_ctl, int index)
 	u32 crc = ~(u32)0;
 	unsigned offset = 0;
 
+	if (index >= io_ctl->num_pages)
+		return -EIO;
+
 	if (index == 0)
 		offset = sizeof(u32) * io_ctl->num_pages;
 
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index 272598f6ae77a..af0f5504a2d43 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -6832,7 +6832,7 @@ static int btrfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 }
 
 static int btrfs_create(struct mnt_idmap *idmap, struct inode *dir,
-			struct dentry *dentry, umode_t mode, bool excl)
+			struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode;
 
@@ -6936,7 +6936,7 @@ static struct dentry *btrfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	inode = new_inode(dir->i_sb);
 	if (!inode)
 		return ERR_PTR(-ENOMEM);
-	inode_init_owner(idmap, inode, dir, S_IFDIR | mode);
+	inode_init_owner(idmap, inode, dir, mode);
 	inode->i_op = &btrfs_dir_inode_operations;
 	inode->i_fop = &btrfs_dir_file_operations;
 	return ERR_PTR(btrfs_create_common(dir, dentry, inode));
@@ -8068,7 +8068,8 @@ static int btrfs_getattr(struct mnt_idmap *idmap,
 	stat->result_mask |= STATX_SUBVOL;
 
 	spin_lock(&BTRFS_I(inode)->lock);
-	delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes;
+	delalloc_bytes = S_ISREG(inode->i_mode) ?
+			 BTRFS_I(inode)->new_delalloc_bytes : 0;
 	inode_bytes = inode_get_bytes(inode);
 	spin_unlock(&BTRFS_I(inode)->lock);
 	stat->blocks = (ALIGN(inode_bytes, blocksize) +
diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
index 9d47d16394fc5..e8fa5cc738180 100644
--- a/fs/btrfs/ioctl.c
+++ b/fs/btrfs/ioctl.c
@@ -289,6 +289,7 @@ int btrfs_fileattr_set(struct mnt_idmap *idmap,
 	int ret;
 	const char *comp = NULL;
 	u32 inode_flags;
+	bool prop_set = false;
 
 	if (btrfs_root_readonly(root))
 		return -EROFS;
@@ -401,16 +402,15 @@ int btrfs_fileattr_set(struct mnt_idmap *idmap,
 	if (comp) {
 		ret = btrfs_set_prop(trans, inode, "btrfs.compression",
 				     comp, strlen(comp), 0);
-		if (unlikely(ret)) {
-			btrfs_abort_transaction(trans, ret);
+		if (ret)
 			goto out_end_trans;
-		}
+		prop_set = true;
 	} else {
 		ret = btrfs_set_prop(trans, inode, "btrfs.compression", NULL, 0, 0);
-		if (unlikely(ret && ret != -ENODATA)) {
-			btrfs_abort_transaction(trans, ret);
+		prop_set = (ret == 0);
+		/* If ret == -ENODATA ignore and proceed to update inode item. */
+		if (ret && ret != -ENODATA)
 			goto out_end_trans;
-		}
 	}
 
 update_flags:
@@ -420,6 +420,12 @@ int btrfs_fileattr_set(struct mnt_idmap *idmap,
 	inode_inc_iversion(&inode->vfs_inode);
 	inode_set_ctime_current(&inode->vfs_inode);
 	ret = btrfs_update_inode(trans, inode);
+	/*
+	 * If we set a property or deleted one, we must abort if we fail to
+	 * update the inode, to avoid persisting an inconsistent state.
+	 */
+	if (unlikely(ret && prop_set))
+		btrfs_abort_transaction(trans, ret);
 
  out_end_trans:
 	btrfs_end_transaction(trans);
@@ -2042,6 +2048,7 @@ static int _btrfs_ioctl_get_subvol_info(struct inode *inode,
 			ret = -ENOENT;
 			goto out;
 		}
+		ret = 0;
 	}
 
 out:
@@ -2622,7 +2629,7 @@ static long btrfs_ioctl_rm_dev_v2(struct file *file, void __user *arg)
 err_drop:
 	mnt_drop_write_file(file);
 	if (bdev_file)
-		bdev_fput(bdev_file);
+		btrfs_release_device_allow_freeze(bdev_file);
 out:
 	btrfs_put_dev_args_from_path(&args);
 	return ret;
@@ -2672,7 +2679,7 @@ static long btrfs_ioctl_rm_dev(struct file *file, void __user *arg)
 
 	mnt_drop_write_file(file);
 	if (bdev_file)
-		bdev_fput(bdev_file);
+		btrfs_release_device_allow_freeze(bdev_file);
 out:
 	btrfs_put_dev_args_from_path(&args);
 	return ret;
diff --git a/fs/btrfs/lzo.c b/fs/btrfs/lzo.c
index 6e4aa22853aba..1531adb117d15 100644
--- a/fs/btrfs/lzo.c
+++ b/fs/btrfs/lzo.c
@@ -552,17 +552,26 @@ int lzo_decompress(struct list_head *ws, const u8 *data_in,
 	size_t max_segment_len = workspace_buf_length(fs_info);
 	int ret;
 
-	if (unlikely(srclen < LZO_LEN || srclen > max_segment_len + LZO_LEN * 2))
+	if (unlikely(srclen < LZO_LEN || srclen > max_segment_len + LZO_LEN * 2)) {
+		btrfs_err(fs_info, "invalid lzo header length, has %zu expect (%u, %zu)",
+			  srclen, LZO_LEN, max_segment_len + LZO_LEN * 2);
 		return -EUCLEAN;
+	}
 
 	in_len = get_unaligned_le32(data_in);
-	if (unlikely(in_len != srclen))
+	if (unlikely(in_len != srclen)) {
+		btrfs_err(fs_info, "invalid lzo header length, has %zu expect %zu",
+			  in_len, srclen);
 		return -EUCLEAN;
+	}
 	data_in += LZO_LEN;
 
 	in_len = get_unaligned_le32(data_in);
-	if (unlikely(in_len != srclen - LZO_LEN * 2))
+	if (unlikely(in_len != srclen - LZO_LEN * 2)) {
+		btrfs_err(fs_info, "invalid lzo segment length, has %zu expect %zu",
+			  in_len, srclen - LZO_LEN * 2);
 		return -EUCLEAN;
+	}
 	data_in += LZO_LEN;
 
 	out_len = sectorsize;
diff --git a/fs/btrfs/print-tree.c b/fs/btrfs/print-tree.c
index 87e60a2d4bd81..53e726119ca70 100644
--- a/fs/btrfs/print-tree.c
+++ b/fs/btrfs/print-tree.c
@@ -449,9 +449,9 @@ void btrfs_print_leaf(const struct extent_buffer *l)
 	nr = btrfs_header_nritems(l);
 
 	btrfs_info(fs_info,
-		   "leaf %llu gen %llu total ptrs %d free space %d owner %llu",
+		   "leaf %llu gen %llu total ptrs %d free space %d owner %lld",
 		   btrfs_header_bytenr(l), btrfs_header_generation(l), nr,
-		   btrfs_leaf_free_space(l), btrfs_header_owner(l));
+		   btrfs_leaf_free_space(l), (s64)btrfs_header_owner(l));
 	print_eb_refs_lock(l);
 	for (i = 0 ; i < nr ; i++) {
 		char key_buf[KEY_TYPE_BUF_SIZE];
@@ -600,10 +600,10 @@ void btrfs_print_tree(const struct extent_buffer *c, bool follow)
 		return;
 	}
 	btrfs_info(fs_info,
-		   "node %llu level %d gen %llu total ptrs %d free spc %u owner %llu",
+		   "node %llu level %d gen %llu total ptrs %d free spc %u owner %lld",
 		   btrfs_header_bytenr(c), level, btrfs_header_generation(c),
 		   nr, (u32)BTRFS_NODEPTRS_PER_BLOCK(fs_info) - nr,
-		   btrfs_header_owner(c));
+		   (s64)btrfs_header_owner(c));
 	print_eb_refs_lock(c);
 	for (i = 0; i < nr; i++) {
 		btrfs_node_key_to_cpu(c, &key, i);
diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c
index adc956432d2f1..bb77d46376d4b 100644
--- a/fs/btrfs/props.c
+++ b/fs/btrfs/props.c
@@ -127,14 +127,24 @@ int btrfs_set_prop(struct btrfs_trans_handle *trans, struct btrfs_inode *inode,
 		return ret;
 	}
 
+	ret = handler->validate(inode, value, value_len);
+	if (ret)
+		return ret;
 	ret = btrfs_setxattr(trans, &inode->vfs_inode, handler->xattr_name, value,
 			     value_len, flags);
 	if (ret)
 		return ret;
 	ret = handler->apply(inode, value, value_len);
-	if (ret) {
-		btrfs_setxattr(trans, &inode->vfs_inode, handler->xattr_name, NULL,
-			       0, flags);
+	/* We validated before, so it should not fail here. */
+	ASSERT(ret == 0);
+	if (unlikely(ret)) {
+		int ret2;
+
+		/* Try to delete xattr, if not possible abort transaction. */
+		ret2 = btrfs_setxattr(trans, &inode->vfs_inode, handler->xattr_name,
+				      NULL, 0, flags);
+		if (unlikely(ret2))
+			btrfs_abort_transaction(trans, ret2);
 		return ret;
 	}
 
diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c
index fb85bc8b345c7..fc5c14b5adad7 100644
--- a/fs/btrfs/relocation.c
+++ b/fs/btrfs/relocation.c
@@ -588,6 +588,7 @@ static int __add_reloc_root(struct btrfs_root *root, struct reloc_control *rc)
 		btrfs_err(fs_info,
 			    "Duplicate root found for start=%llu while inserting into relocation tree",
 			    node->bytenr);
+		kfree(node);
 		return -EEXIST;
 	}
 
@@ -719,21 +720,19 @@ static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans,
 
 	ret = btrfs_insert_root(trans, fs_info->tree_root,
 				&root_key, root_item);
-	if (ret)
-		goto abort;
+	if (unlikely(ret)) {
+		btrfs_abort_transaction(trans, ret);
+		return ERR_PTR(ret);
+	}
 
 	reloc_root = btrfs_read_tree_root(fs_info->tree_root, &root_key);
 	if (IS_ERR(reloc_root)) {
-		ret = PTR_ERR(reloc_root);
-		goto abort;
+		btrfs_abort_transaction(trans, PTR_ERR(reloc_root));
+		return ERR_CAST(reloc_root);
 	}
 	set_bit(BTRFS_ROOT_SHAREABLE, &reloc_root->state);
 	btrfs_set_root_last_trans(reloc_root, trans->transid);
 	return reloc_root;
-
-abort:
-	btrfs_abort_transaction(trans, ret);
-	return ERR_PTR(ret);
 }
 
 /*
@@ -892,6 +891,13 @@ static int get_new_location(struct inode *reloc_inode, u64 *new_bytenr,
 	leaf = path->nodes[0];
 	fi = btrfs_item_ptr(leaf, path->slots[0],
 			    struct btrfs_file_extent_item);
+	if (unlikely(btrfs_file_extent_type(leaf, fi) == BTRFS_FILE_EXTENT_INLINE)) {
+		btrfs_print_leaf(leaf);
+		btrfs_err(fs_info,
+	"unexpected inline file extent item for data reloc inode %llu key offset %llu",
+			  btrfs_ino(BTRFS_I(reloc_inode)), bytenr);
+		return -EUCLEAN;
+	}
 
 	/*
 	 * The cluster-boundary key searched above is always written by
@@ -1520,6 +1526,17 @@ static int insert_dirty_subvol(struct btrfs_trans_handle *trans,
 	return 0;
 }
 
+static void clear_reloc_root(struct btrfs_root *root)
+{
+	root->reloc_root = NULL;
+	/*
+	 * Need barrier to ensure clear_bit() only happens after
+	 * root->reloc_root = NULL. Pairs with have_reloc_root().
+	 */
+	smp_wmb();
+	clear_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state);
+}
+
 static int clean_dirty_subvols(struct reloc_control *rc)
 {
 	struct btrfs_root *root;
@@ -1534,13 +1551,7 @@ static int clean_dirty_subvols(struct reloc_control *rc)
 			struct btrfs_root *reloc_root = root->reloc_root;
 
 			list_del_init(&root->reloc_dirty_list);
-			root->reloc_root = NULL;
-			/*
-			 * Need barrier to ensure clear_bit() only happens after
-			 * root->reloc_root = NULL. Pairs with have_reloc_root.
-			 */
-			smp_wmb();
-			clear_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state);
+			clear_reloc_root(root);
 			if (reloc_root) {
 				/*
 				 * btrfs_drop_snapshot drops our ref we hold for
@@ -1912,24 +1923,43 @@ void merge_reloc_roots(struct reloc_control *rc)
 				 * corruption, e.g. bad reloc tree key offset.
 				 */
 				ret = -EINVAL;
+				btrfs_put_root(root);
 				goto out;
 			}
 			ret = merge_reloc_root(rc, root);
-			btrfs_put_root(root);
 			if (ret) {
-				if (list_empty(&reloc_root->root_list))
+				/*
+				 * Clear the reloc root since below we will call
+				 * free_reloc_roots(), otherwise we leave
+				 * root->reloc_root pointing to a freed reloc
+				 * root and trigger a use-after-free during
+				 * unmount or elsewhere.
+				 */
+				clear_reloc_root(root);
+				btrfs_put_root(root);
+				/*
+				 * We are adding the reloc_root to the local
+				 * reloc_roots list, so we add a ref for this
+				 * list which will be dropped below by the call
+				 * to free_reloc_roots().
+				 */
+				if (list_empty(&reloc_root->root_list)) {
 					list_add_tail(&reloc_root->root_list,
 						      &reloc_roots);
+					btrfs_grab_root(reloc_root);
+				}
+				/* Now drop the ref for root->reloc_root. */
+				btrfs_put_root(reloc_root);
 				goto out;
 			}
+			btrfs_put_root(root);
 		} else {
 			if (!IS_ERR(root)) {
 				if (root->reloc_root == reloc_root) {
-					root->reloc_root = NULL;
+					clear_reloc_root(root);
+					/* Drop the ref for root->reloc_root. */
 					btrfs_put_root(reloc_root);
 				}
-				clear_bit(BTRFS_ROOT_DEAD_RELOC_TREE,
-					  &root->state);
 				btrfs_put_root(root);
 			}
 
diff --git a/fs/btrfs/subpage.c b/fs/btrfs/subpage.c
index 56060acac2e9c..2a9397be8116c 100644
--- a/fs/btrfs/subpage.c
+++ b/fs/btrfs/subpage.c
@@ -359,6 +359,23 @@ void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info,
 	folio_mark_dirty(folio);
 }
 
+static void folio_clear_tags(struct folio *folio)
+{
+	struct address_space *mapping = folio_mapping(folio);
+	XA_STATE(xas, &mapping->i_pages, folio->index);
+	unsigned long flags;
+
+	ASSERT(folio_test_locked(folio));
+	ASSERT(mapping);
+	ASSERT(mapping_use_writeback_tags(mapping));
+
+	xas_lock_irqsave(&xas, flags);
+	xas_load(&xas);
+	xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY);
+	xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE);
+	xas_unlock_irqrestore(&xas, flags);
+}
+
 /*
  * Extra clear_and_test function for subpage dirty bitmap.
  *
@@ -403,7 +420,6 @@ void btrfs_subpage_set_writeback(const struct btrfs_fs_info *fs_info,
 	unsigned int start_bit = subpage_calc_start_bit(fs_info, folio,
 							writeback, start, len);
 	unsigned long flags;
-	bool keep_write;
 
 	spin_lock_irqsave(&bfs->lock, flags);
 	bitmap_set(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
@@ -413,10 +429,14 @@ void btrfs_subpage_set_writeback(const struct btrfs_fs_info *fs_info,
 	 * folio. Doing so can cause WB_SYNC_ALL writepages() to overlook it,
 	 * assume writeback is complete, and exit too early — violating sync
 	 * ordering guarantees.
+	 *
+	 * Instead we manually clear the DIRTY and TOWRITE tags after the folio
+	 * is no longer dirty.
 	 */
-	keep_write = folio_test_dirty(folio);
 	if (!folio_test_writeback(folio))
-		__folio_start_writeback(folio, keep_write);
+		__folio_start_writeback(folio, true);
+	if (!folio_test_dirty(folio))
+		folio_clear_tags(folio);
 	spin_unlock_irqrestore(&bfs->lock, flags);
 }
 
diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c
index cb3e676a81cc4..0ce91396b517f 100644
--- a/fs/btrfs/tree-checker.c
+++ b/fs/btrfs/tree-checker.c
@@ -1923,6 +1923,12 @@ static int check_inode_ref(struct extent_buffer *leaf,
 
 		iref = (struct btrfs_inode_ref *)ptr;
 		namelen = btrfs_inode_ref_name_len(leaf, iref);
+		if (unlikely(namelen == 0 || namelen > BTRFS_NAME_LEN)) {
+			inode_ref_err(leaf, slot,
+				"invalid inode ref name length, has %u expect [1, %u]",
+				namelen, BTRFS_NAME_LEN);
+			return -EUCLEAN;
+		}
 		if (unlikely(ptr + sizeof(*iref) + namelen > end)) {
 			inode_ref_err(leaf, slot,
 				"inode ref overflow, ptr %lu end %lu namelen %u",
diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c
index 6eab4cc73ce44..e68ce323bb06c 100644
--- a/fs/btrfs/volumes.c
+++ b/fs/btrfs/volumes.c
@@ -12,6 +12,7 @@
 #include <linux/uuid.h>
 #include <linux/list_sort.h>
 #include <linux/namei.h>
+#include <linux/fs_struct.h>
 #include "misc.h"
 #include "disk-io.h"
 #include "extent-tree.h"
@@ -480,7 +481,12 @@ btrfs_get_bdev_and_sb(const char *device_path, blk_mode_t flags, void *holder,
 	struct block_device *bdev;
 	int ret;
 
-	*bdev_file = bdev_file_open_by_path(device_path, flags, holder, &fs_holder_ops);
+	if (holder)
+		*bdev_file = fs_bdev_file_open_by_path(device_path, flags,
+						       holder, holder);
+	else
+		*bdev_file = bdev_file_open_by_path(device_path, flags, NULL,
+						    NULL);
 
 	if (IS_ERR(*bdev_file)) {
 		ret = PTR_ERR(*bdev_file);
@@ -495,7 +501,7 @@ btrfs_get_bdev_and_sb(const char *device_path, blk_mode_t flags, void *holder,
 	if (holder) {
 		ret = set_blocksize(*bdev_file, BTRFS_BDEV_BLOCKSIZE);
 		if (ret) {
-			bdev_fput(*bdev_file);
+			fs_bdev_file_release(*bdev_file, holder);
 			goto error;
 		}
 	}
@@ -503,7 +509,10 @@ btrfs_get_bdev_and_sb(const char *device_path, blk_mode_t flags, void *holder,
 	*disk_super = btrfs_read_disk_super(bdev, 0, false);
 	if (IS_ERR(*disk_super)) {
 		ret = PTR_ERR(*disk_super);
-		bdev_fput(*bdev_file);
+		if (holder)
+			fs_bdev_file_release(*bdev_file, holder);
+		else
+			bdev_fput(*bdev_file);
 		goto error;
 	}
 
@@ -727,7 +736,7 @@ static int btrfs_open_one_device(struct btrfs_fs_devices *fs_devices,
 
 error_free_page:
 	btrfs_release_disk_super(disk_super);
-	bdev_fput(bdev_file);
+	fs_bdev_file_release(bdev_file, holder);
 
 	return -EINVAL;
 }
@@ -1087,7 +1096,7 @@ static void __btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices,
 			continue;
 
 		if (device->bdev_file) {
-			bdev_fput(device->bdev_file);
+			fs_bdev_file_release(device->bdev_file, device->bdev_file->private_data);
 			device->bdev = NULL;
 			device->bdev_file = NULL;
 			fs_devices->open_devices--;
@@ -1124,7 +1133,18 @@ void btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices)
 	mutex_unlock(&uuid_mutex);
 }
 
-static void btrfs_close_bdev(struct btrfs_device *device)
+/* Release a device that was made unfreezable for a membership change. */
+void btrfs_release_device_allow_freeze(struct file *bdev_file)
+{
+	struct super_block *sb = bdev_file->private_data;
+
+	/* Unregister before re-allowing (strand-safe); file still open (UAF-safe). */
+	fs_bdev_unregister(bdev_file, sb);
+	bdev_allow_freeze(file_bdev(bdev_file));
+	bdev_fput(bdev_file);
+}
+
+static void btrfs_close_bdev(struct btrfs_device *device, bool allow_freeze)
 {
 	if (!device->bdev)
 		return;
@@ -1134,7 +1154,12 @@ static void btrfs_close_bdev(struct btrfs_device *device)
 		invalidate_bdev(device->bdev);
 	}
 
-	bdev_fput(device->bdev_file);
+	/* @allow_freeze undoes a replace-time deny; unmount-close was never denied. */
+	if (allow_freeze)
+		btrfs_release_device_allow_freeze(device->bdev_file);
+	else
+		fs_bdev_file_release(device->bdev_file,
+				     device->bdev_file->private_data);
 }
 
 static void btrfs_close_one_device(struct btrfs_device *device)
@@ -1155,7 +1180,7 @@ static void btrfs_close_one_device(struct btrfs_device *device)
 		fs_devices->missing_devices--;
 	}
 
-	btrfs_close_bdev(device);
+	btrfs_close_bdev(device, false);
 	if (device->bdev) {
 		fs_devices->open_devices--;
 		device->bdev = NULL;
@@ -2125,8 +2150,16 @@ static int btrfs_add_dev_item(struct btrfs_trans_handle *trans,
 static void update_dev_time(const char *device_path)
 {
 	struct path path;
+	int err;
+
+	if (tsk_is_kthread(current)) {
+		scoped_with_init_fs()
+			err = kern_path(device_path, LOOKUP_FOLLOW, &path);
+	} else {
+		err = kern_path(device_path, LOOKUP_FOLLOW, &path);
+	}
 
-	if (!kern_path(device_path, LOOKUP_FOLLOW, &path)) {
+	if (!err) {
 		vfs_utimes(&path, NULL);
 		path_put(&path);
 	}
@@ -2373,6 +2406,13 @@ int btrfs_rm_device(struct btrfs_fs_info *fs_info,
 	    fs_info->fs_devices->rw_devices == 1)
 		return BTRFS_ERROR_DEV_ONLY_WRITABLE;
 
+	/* Removal and freezing are mutually exclusive; refuse if frozen now. */
+	if (device->bdev) {
+		ret = bdev_deny_freeze(device->bdev);
+		if (ret)
+			return ret;
+	}
+
 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
 		mutex_lock(&fs_info->chunk_mutex);
 		list_del_init(&device->dev_alloc_list);
@@ -2399,6 +2439,8 @@ int btrfs_rm_device(struct btrfs_fs_info *fs_info,
 			   device->devid, ret);
 		btrfs_abort_transaction(trans, ret);
 		btrfs_end_transaction(trans);
+		if (device->bdev)
+			bdev_allow_freeze(device->bdev);
 		return ret;
 	}
 
@@ -2490,6 +2532,8 @@ int btrfs_rm_device(struct btrfs_fs_info *fs_info,
 	return btrfs_commit_transaction(trans);
 
 error_undo:
+	if (device->bdev)
+		bdev_allow_freeze(device->bdev);
 	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
 		mutex_lock(&fs_info->chunk_mutex);
 		list_add(&device->dev_alloc_list,
@@ -2534,7 +2578,8 @@ void btrfs_rm_dev_replace_free_srcdev(struct btrfs_device *srcdev)
 
 	mutex_lock(&uuid_mutex);
 
-	btrfs_close_bdev(srcdev);
+	/* The source was made unfreezable for the replace; undo it. */
+	btrfs_close_bdev(srcdev, true);
 	synchronize_rcu();
 	btrfs_free_device(srcdev);
 
@@ -2555,7 +2600,8 @@ void btrfs_rm_dev_replace_free_srcdev(struct btrfs_device *srcdev)
 	mutex_unlock(&uuid_mutex);
 }
 
-void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev)
+void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev,
+				      bool allow_freeze)
 {
 	struct btrfs_fs_devices *fs_devices = tgtdev->fs_info->fs_devices;
 
@@ -2576,7 +2622,7 @@ void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev)
 
 	btrfs_scratch_superblocks(tgtdev->fs_info, tgtdev);
 
-	btrfs_close_bdev(tgtdev);
+	btrfs_close_bdev(tgtdev, allow_freeze);
 	synchronize_rcu();
 	btrfs_free_device(tgtdev);
 }
@@ -2845,6 +2891,37 @@ static int btrfs_finish_sprout(struct btrfs_trans_handle *trans)
 	return 0;
 }
 
+/*
+ * Open @path for @sb with freezing denied before the holder claim is published,
+ * so a racing bdev_freeze() can never reach a claim a device add or replace may
+ * still abort.  The deny is taken on a throwaway non-holder probe open, then the
+ * holder is opened by the probe's dev_t.  Balanced by the caller.
+ */
+struct file *btrfs_open_device_deny_freeze(const char *path,
+					   struct super_block *sb)
+{
+	struct file *probe_file, *bdev_file;
+	int ret;
+
+	/* WRITE so bdev_file_open_by_path() rejects a read-only device. */
+	probe_file = bdev_file_open_by_path(path, BLK_OPEN_WRITE, NULL, NULL);
+	if (IS_ERR(probe_file))
+		return probe_file;
+
+	ret = bdev_deny_freeze(file_bdev(probe_file));
+	if (ret) {
+		bdev_fput(probe_file);
+		return ERR_PTR(ret);
+	}
+
+	bdev_file = fs_bdev_file_open_by_dev(file_bdev(probe_file)->bd_dev,
+					     BLK_OPEN_WRITE, sb, sb);
+	if (IS_ERR(bdev_file))
+		bdev_allow_freeze(file_bdev(probe_file));
+	bdev_fput(probe_file);
+	return bdev_file;
+}
+
 int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path)
 {
 	struct btrfs_root *root = fs_info->dev_root;
@@ -2863,8 +2940,8 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path
 	if (sb_rdonly(sb) && !fs_devices->seeding)
 		return -EROFS;
 
-	bdev_file = bdev_file_open_by_path(device_path, BLK_OPEN_WRITE,
-					   fs_info->sb, &fs_holder_ops);
+	/* Forbid freezing until the device is a committed member (or unwound). */
+	bdev_file = btrfs_open_device_deny_freeze(device_path, fs_info->sb);
 	if (IS_ERR(bdev_file))
 		return PTR_ERR(bdev_file);
 
@@ -3035,8 +3112,10 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path
 		up_write(&sb->s_umount);
 		locked = false;
 
-		if (ret) /* transaction commit */
+		if (ret) { /* transaction commit */
+			bdev_allow_freeze(file_bdev(bdev_file));
 			return ret;
+		}
 
 		ret = btrfs_relocate_sys_chunks(fs_info);
 		if (ret < 0)
@@ -3044,8 +3123,10 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path
 				    "Failed to relocate sys chunks after device initialization. This can be fixed using the \"btrfs balance\" command.");
 		trans = btrfs_attach_transaction(root);
 		if (IS_ERR(trans)) {
-			if (PTR_ERR(trans) == -ENOENT)
+			if (PTR_ERR(trans) == -ENOENT) {
+				bdev_allow_freeze(file_bdev(bdev_file));
 				return 0;
+			}
 			ret = PTR_ERR(trans);
 			trans = NULL;
 			goto error_sysfs;
@@ -3065,6 +3146,7 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path
 	/* Update ctime/mtime for blkid or udev */
 	update_dev_time(device_path);
 
+	bdev_allow_freeze(file_bdev(bdev_file));
 	return ret;
 
 error_sysfs:
@@ -3094,7 +3176,7 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path
 error_free_device:
 	btrfs_free_device(device);
 error:
-	bdev_fput(bdev_file);
+	btrfs_release_device_allow_freeze(bdev_file);
 	if (locked) {
 		mutex_unlock(&uuid_mutex);
 		up_write(&sb->s_umount);
diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h
index 63be45c3298ca..df2c671ab6fa5 100644
--- a/fs/btrfs/volumes.h
+++ b/fs/btrfs/volumes.h
@@ -744,6 +744,7 @@ int btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
 struct btrfs_device *btrfs_scan_one_device(const char *path, bool mount_arg_dev);
 int btrfs_forget_devices(dev_t devt);
 void btrfs_close_devices(struct btrfs_fs_devices *fs_devices);
+void btrfs_release_device_allow_freeze(struct file *bdev_file);
 void btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices);
 void btrfs_assign_next_active_device(struct btrfs_device *device,
 				     struct btrfs_device *this_dev);
@@ -768,6 +769,8 @@ struct btrfs_device *btrfs_find_device(const struct btrfs_fs_devices *fs_devices
 				       const struct btrfs_dev_lookup_args *args);
 int btrfs_shrink_device(struct btrfs_device *device, u64 new_size);
 int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *path);
+struct file *btrfs_open_device_deny_freeze(const char *path,
+					   struct super_block *sb);
 int btrfs_balance(struct btrfs_fs_info *fs_info,
 		  struct btrfs_balance_control *bctl,
 		  struct btrfs_ioctl_balance_args *bargs);
@@ -788,7 +791,8 @@ int btrfs_init_writeback_bio_size(struct btrfs_fs_info *fs_info);
 int btrfs_run_dev_stats(struct btrfs_trans_handle *trans);
 void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_device *srcdev);
 void btrfs_rm_dev_replace_free_srcdev(struct btrfs_device *srcdev);
-void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev);
+void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev,
+				      bool allow_freeze);
 unsigned long btrfs_full_stripe_len(struct btrfs_fs_info *fs_info,
 				    u64 logical);
 u64 btrfs_calc_stripe_length(const struct btrfs_chunk_map *map);
diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c
index 27ce9e55e9476..b957149abe5a0 100644
--- a/fs/ceph/dir.c
+++ b/fs/ceph/dir.c
@@ -978,7 +978,7 @@ static int ceph_mknod(struct mnt_idmap *idmap, struct inode *dir,
 }
 
 static int ceph_create(struct mnt_idmap *idmap, struct inode *dir,
-		       struct dentry *dentry, umode_t mode, bool excl)
+		       struct dentry *dentry, umode_t mode)
 {
 	return ceph_mknod(idmap, dir, dentry, mode, 0);
 }
@@ -1142,7 +1142,6 @@ static struct dentry *ceph_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 		goto out;
 	}
 
-	mode |= S_IFDIR;
 	req->r_new_inode = ceph_new_inode(dir, dentry, &mode, &as_ctx);
 	if (IS_ERR(req->r_new_inode)) {
 		ret = ERR_CAST(req->r_new_inode);
diff --git a/fs/coda/dir.c b/fs/coda/dir.c
index 835eb7fdfdad0..67148edfadeef 100644
--- a/fs/coda/dir.c
+++ b/fs/coda/dir.c
@@ -134,7 +134,7 @@ static inline void coda_dir_drop_nlink(struct inode *dir)
 
 /* creation routines: create, mknod, mkdir, link, symlink */
 static int coda_create(struct mnt_idmap *idmap, struct inode *dir,
-		       struct dentry *de, umode_t mode, bool excl)
+		       struct dentry *de, umode_t mode)
 {
 	int error;
 	const char *name=de->d_name.name;
@@ -179,7 +179,12 @@ static struct dentry *coda_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	if (is_root_inode(dir) && coda_iscontrol(name, len))
 		return ERR_PTR(-EPERM);
 
-	attrs.va_mode = mode;
+	/*
+	 * vfs_mkdir() now passes S_IFDIR in @mode, but @mode is forwarded
+	 * verbatim to userspace, which has only ever been given the permission
+	 * bits. Strip the type bit until venus is known to cope with it.
+	 */
+	attrs.va_mode = mode & ~S_IFDIR;
 	error = venus_mkdir(dir->i_sb, coda_i2f(dir),
 			       name, len, &newfid, &attrs);
 	if (error)
diff --git a/fs/coredump.c b/fs/coredump.c
index e68a76ff92a38..ac3cd74808c64 100644
--- a/fs/coredump.c
+++ b/fs/coredump.c
@@ -921,15 +921,10 @@ static bool coredump_file(struct core_name *cn, struct coredump_params *cprm,
 		 * with a fully qualified path" rule is to control where
 		 * coredumps may be placed using root privileges,
 		 * current->fs->root must not be used. Instead, use the
-		 * root directory of init_task.
+		 * root directory of PID 1.
 		 */
-		struct path root;
-
-		task_lock(&init_task);
-		get_fs_root(init_task.fs, &root);
-		task_unlock(&init_task);
-		file = file_open_root(&root, cn->corename, open_flags, 0600);
-		path_put(&root);
+		scoped_with_init_fs()
+			file = filp_open(cn->corename, open_flags, 0600);
 	} else {
 		file = filp_open(cn->corename, open_flags, 0600);
 	}
diff --git a/fs/cramfs/inode.c b/fs/cramfs/inode.c
index 4edbfccd0bbea..d4cd03f4f60d5 100644
--- a/fs/cramfs/inode.c
+++ b/fs/cramfs/inode.c
@@ -504,7 +504,7 @@ static void cramfs_kill_sb(struct super_block *sb)
 		sb->s_mtd = NULL;
 	} else if (IS_ENABLED(CONFIG_CRAMFS_BLOCKDEV) && sb->s_bdev) {
 		sync_blockdev(sb->s_bdev);
-		bdev_fput(sb->s_bdev_file);
+		fs_bdev_file_release(sb->s_bdev_file, sb);
 	}
 	kfree(sbi);
 }
diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c
index 570a2231c945a..94dd6c89ddcd0 100644
--- a/fs/crypto/crypto.c
+++ b/fs/crypto/crypto.c
@@ -323,7 +323,6 @@ EXPORT_SYMBOL(fscrypt_decrypt_block_inplace);
  */
 int fscrypt_initialize(struct super_block *sb)
 {
-	int err = 0;
 	mempool_t *pool;
 
 	/* pairs with smp_store_release() below */
@@ -334,20 +333,16 @@ int fscrypt_initialize(struct super_block *sb)
 	if (!sb->s_cop->needs_bounce_pages)
 		return 0;
 
-	mutex_lock(&fscrypt_init_mutex);
+	guard(mutex)(&fscrypt_init_mutex);
 	if (fscrypt_bounce_page_pool)
-		goto out_unlock;
+		return 0;
 
-	err = -ENOMEM;
 	pool = mempool_create_page_pool(num_prealloc_crypto_pages, 0);
 	if (!pool)
-		goto out_unlock;
+		return -ENOMEM;
 	/* pairs with smp_load_acquire() above */
 	smp_store_release(&fscrypt_bounce_page_pool, pool);
-	err = 0;
-out_unlock:
-	mutex_unlock(&fscrypt_init_mutex);
-	return err;
+	return 0;
 }
 
 void fscrypt_msg(const struct inode *inode, const char *level,
@@ -374,15 +369,8 @@ void fscrypt_msg(const struct inode *inode, const char *level,
 	va_end(args);
 }
 
-/**
- * fscrypt_init() - Set up for fs encryption.
- *
- * Return: 0 on success; -errno on failure
- */
 static int __init fscrypt_init(void)
 {
-	int err = -ENOMEM;
-
 	/*
 	 * Use an unbound workqueue to allow bios to be decrypted in parallel
 	 * even when they happen to complete on the same CPU.  This sacrifices
@@ -395,24 +383,12 @@ static int __init fscrypt_init(void)
 						 WQ_UNBOUND | WQ_HIGHPRI,
 						 num_online_cpus());
 	if (!fscrypt_read_workqueue)
-		goto fail;
+		panic("failed to allocate fscrypt_read_queue");
 
 	fscrypt_inode_info_cachep = KMEM_CACHE(fscrypt_inode_info,
-					       SLAB_RECLAIM_ACCOUNT);
-	if (!fscrypt_inode_info_cachep)
-		goto fail_free_queue;
-
-	err = fscrypt_init_keyring();
-	if (err)
-		goto fail_free_inode_info;
-
+					       SLAB_RECLAIM_ACCOUNT |
+					       SLAB_PANIC);
+	fscrypt_init_keyring();
 	return 0;
-
-fail_free_inode_info:
-	kmem_cache_destroy(fscrypt_inode_info_cachep);
-fail_free_queue:
-	destroy_workqueue(fscrypt_read_workqueue);
-fail:
-	return err;
 }
 late_initcall(fscrypt_init)
diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h
index 0053b5c454123..8234ee542476c 100644
--- a/fs/crypto/fscrypt_private.h
+++ b/fs/crypto/fscrypt_private.h
@@ -66,9 +66,6 @@
 #define FSCRYPT_CONTEXT_V1	1
 #define FSCRYPT_CONTEXT_V2	2
 
-/* Keep this in sync with include/uapi/linux/fscrypt.h */
-#define FSCRYPT_MODE_MAX	FSCRYPT_MODE_AES_256_HCTR2
-
 struct fscrypt_context_v1 {
 	u8 version; /* FSCRYPT_CONTEXT_V1 */
 	u8 contents_encryption_mode;
@@ -718,7 +715,7 @@ int fscrypt_add_test_dummy_key(struct super_block *sb,
 int fscrypt_verify_key_added(struct super_block *sb,
 			     const u8 identifier[FSCRYPT_KEY_IDENTIFIER_SIZE]);
 
-int __init fscrypt_init_keyring(void);
+void __init fscrypt_init_keyring(void);
 
 /* keysetup.c */
 
diff --git a/fs/crypto/keyring.c b/fs/crypto/keyring.c
index 38b73e7030733..76e28d1e00640 100644
--- a/fs/crypto/keyring.c
+++ b/fs/crypto/keyring.c
@@ -497,7 +497,7 @@ static int do_add_master_key(struct super_block *sb,
 	struct fscrypt_master_key *mk;
 	int err;
 
-	mutex_lock(&fscrypt_add_key_mutex); /* serialize find + link */
+	guard(mutex)(&fscrypt_add_key_mutex); /* serialize find + link */
 
 	mk = fscrypt_find_master_key(sb, mk_spec);
 	if (!mk) {
@@ -524,7 +524,6 @@ static int do_add_master_key(struct super_block *sb,
 		}
 		fscrypt_put_master_key(mk);
 	}
-	mutex_unlock(&fscrypt_add_key_mutex);
 	return err;
 }
 
@@ -1221,21 +1220,19 @@ int fscrypt_ioctl_get_key_status(struct file *filp, void __user *uarg)
 }
 EXPORT_SYMBOL_GPL(fscrypt_ioctl_get_key_status);
 
-int __init fscrypt_init_keyring(void)
+void __init fscrypt_init_keyring(void)
 {
 	int err;
 
+	/*
+	 * Note that register_key_type() fails only if a key type with the same
+	 * name already exists, which should never happen here.
+	 */
 	err = register_key_type(&key_type_fscrypt_user);
 	if (err)
-		return err;
-
+		panic("failed to register .fscrypt key type (%d)", err);
 	err = register_key_type(&key_type_fscrypt_provisioning);
 	if (err)
-		goto err_unregister_fscrypt_user;
-
-	return 0;
-
-err_unregister_fscrypt_user:
-	unregister_key_type(&key_type_fscrypt_user);
-	return err;
+		panic("failed to register fscrypt-provisioning key type (%d)",
+		      err);
 }
diff --git a/fs/crypto/keysetup.c b/fs/crypto/keysetup.c
index f905f9f94bdd5..cfd348e2252ee 100644
--- a/fs/crypto/keysetup.c
+++ b/fs/crypto/keysetup.c
@@ -83,8 +83,6 @@ static struct fscrypt_mode *
 select_encryption_mode(const union fscrypt_policy *policy,
 		       const struct inode *inode)
 {
-	BUILD_BUG_ON(ARRAY_SIZE(fscrypt_modes) != FSCRYPT_MODE_MAX + 1);
-
 	if (S_ISREG(inode->i_mode))
 		return &fscrypt_modes[fscrypt_policy_contents_mode(policy)];
 
@@ -229,9 +227,6 @@ static int setup_per_mode_enc_key(struct fscrypt_inode_info *ci,
 	bool use_hw_wrapped_key = false;
 	int err;
 
-	if (WARN_ON_ONCE(mode_num > FSCRYPT_MODE_MAX))
-		return -EINVAL;
-
 	if (mk->mk_secret.is_hw_wrapped && S_ISREG(inode->i_mode)) {
 		/* Using a hardware-wrapped key for file contents encryption */
 		if (!fscrypt_using_inline_encryption(ci)) {
@@ -349,18 +344,17 @@ static int fscrypt_setup_iv_ino_lblk_32_key(struct fscrypt_inode_info *ci,
 
 	/* pairs with smp_store_release() below */
 	if (!smp_load_acquire(&mk->mk_ino_hash_key_initialized)) {
-
-		mutex_lock(&fscrypt_mode_key_setup_mutex);
-
-		if (mk->mk_ino_hash_key_initialized)
-			goto unlock;
-
-		fscrypt_derive_siphash_key(mk, HKDF_CONTEXT_INODE_HASH_KEY,
-					   NULL, 0, &mk->mk_ino_hash_key);
-		/* pairs with smp_load_acquire() above */
-		smp_store_release(&mk->mk_ino_hash_key_initialized, true);
-unlock:
-		mutex_unlock(&fscrypt_mode_key_setup_mutex);
+		guard(mutex)(&fscrypt_mode_key_setup_mutex);
+
+		if (!mk->mk_ino_hash_key_initialized) {
+			fscrypt_derive_siphash_key(mk,
+						   HKDF_CONTEXT_INODE_HASH_KEY,
+						   NULL, 0,
+						   &mk->mk_ino_hash_key);
+			/* pairs with smp_load_acquire() above */
+			smp_store_release(&mk->mk_ino_hash_key_initialized,
+					  true);
+		}
 	}
 
 	/*
diff --git a/fs/crypto/policy.c b/fs/crypto/policy.c
index 9915e39362dba..f40fb5924e752 100644
--- a/fs/crypto/policy.c
+++ b/fs/crypto/policy.c
@@ -507,7 +507,6 @@ int fscrypt_ioctl_set_policy(struct file *filp, const void __user *arg)
 	union fscrypt_policy policy;
 	union fscrypt_policy existing_policy;
 	struct inode *inode = file_inode(filp);
-	u8 version;
 	int size;
 	int ret;
 
@@ -518,21 +517,9 @@ int fscrypt_ioctl_set_policy(struct file *filp, const void __user *arg)
 	if (size <= 0)
 		return -EINVAL;
 
-	/*
-	 * We should just copy the remaining 'size - 1' bytes here, but a
-	 * bizarre bug in gcc 7 and earlier (fixed by gcc r255731) causes gcc to
-	 * think that size can be 0 here (despite the check above!) *and* that
-	 * it's a compile-time constant.  Thus it would think copy_from_user()
-	 * is passed compile-time constant ULONG_MAX, causing the compile-time
-	 * buffer overflow check to fail, breaking the build. This only occurred
-	 * when building an i386 kernel with -Os and branch profiling enabled.
-	 *
-	 * Work around it by just copying the first byte again...
-	 */
-	version = policy.version;
-	if (copy_from_user(&policy, arg, size))
+	if (copy_from_user((u8 *)&policy + 1, (const u8 __user *)arg + 1,
+			   size - 1))
 		return -EFAULT;
-	policy.version = version;
 
 	if (!inode_owner_or_capable(&nop_mnt_idmap, inode))
 		return -EACCES;
diff --git a/fs/dax.c b/fs/dax.c
index 6d175cd47a99b..6ba50142eeb2f 100644
--- a/fs/dax.c
+++ b/fs/dax.c
@@ -392,12 +392,12 @@ int dax_folio_reset_order(struct folio *folio)
 	int order = folio_order(folio);
 
 	/*
-	 * DAX maintains the invariant that folio->share != 0 only when
-	 * folio->mapping == NULL (enforced by dax_folio_make_shared()).
-	 * Equivalently: folio->mapping != NULL implies folio->share == 0.
-	 * Callers ensure share has been decremented to zero before
-	 * calling here, so unconditionally clearing both fields is
-	 * correct.
+	 * Clear the mapping and the index/share union word. folio->share
+	 * and folio->index occupy the same union in struct folio. For
+	 * non-shared folios (mapping != NULL), the union holds folio->index
+	 * (file page offset); for shared folios (mapping == NULL), it holds
+	 * folio->share (reference count). Either way, we are releasing the
+	 * folio and both fields should be zeroed.
 	 */
 	folio->mapping = NULL;
 	folio->share = 0;
@@ -1201,7 +1201,7 @@ static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev,
 
 	/* Walk all mappings of a given index of a file and writeprotect them */
 	i_mmap_lock_read(mapping);
-	vma_interval_tree_foreach(vma, &mapping->i_mmap, index, end) {
+	mapping_rmap_tree_foreach(vma, mapping, index, end) {
 		pfn_mkclean_range(pfn, count, index, vma);
 		cond_resched();
 	}
diff --git a/fs/debugfs/file.c b/fs/debugfs/file.c
index edd6aafbfbaaf..08de6652a4f31 100644
--- a/fs/debugfs/file.c
+++ b/fs/debugfs/file.c
@@ -273,7 +273,8 @@ static int debugfs_locked_down(struct inode *inode,
 	    (!real_fops ||
 	     (!real_fops->unlocked_ioctl &&
 	      !real_fops->compat_ioctl &&
-	      !real_fops->mmap)))
+	      !real_fops->mmap &&
+	      !real_fops->mmap_prepare)))
 		return 0;
 
 	if (security_locked_down(LOCKDOWN_DEBUGFS))
diff --git a/fs/ecryptfs/inode.c b/fs/ecryptfs/inode.c
index 7aaf1913f9c6d..525297c7ebd8c 100644
--- a/fs/ecryptfs/inode.c
+++ b/fs/ecryptfs/inode.c
@@ -268,7 +268,7 @@ int ecryptfs_initialize_file(struct dentry *ecryptfs_dentry,
 static int
 ecryptfs_create(struct mnt_idmap *idmap,
 		struct inode *directory_inode, struct dentry *ecryptfs_dentry,
-		umode_t mode, bool excl)
+		umode_t mode)
 {
 	struct inode *ecryptfs_inode;
 	int rc;
diff --git a/fs/efivarfs/inode.c b/fs/efivarfs/inode.c
index 95dcad83da11b..f0d009555fc66 100644
--- a/fs/efivarfs/inode.c
+++ b/fs/efivarfs/inode.c
@@ -75,7 +75,7 @@ static bool efivarfs_valid_name(const char *str, int len)
 }
 
 static int efivarfs_create(struct mnt_idmap *idmap, struct inode *dir,
-			   struct dentry *dentry, umode_t mode, bool excl)
+			   struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode = NULL;
 	struct efivar_entry *var;
diff --git a/fs/efs/Kconfig b/fs/efs/Kconfig
deleted file mode 100644
index 0833e533df9d5..0000000000000
--- a/fs/efs/Kconfig
+++ /dev/null
@@ -1,16 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-config EFS_FS
-	tristate "EFS file system support (read only)"
-	depends on BLOCK
-	select BUFFER_HEAD
-	help
-	  EFS is an older file system used for non-ISO9660 CD-ROMs and hard
-	  disk partitions by SGI's IRIX operating system (IRIX 6.0 and newer
-	  uses the XFS file system for hard disk partitions however).
-
-	  This implementation only offers read-only access. If you don't know
-	  what all this is about, it's safe to say N. For more information
-	  about EFS see its home page at <http://aeschi.ch.eu.org/efs/>.
-
-	  To compile the EFS file system support as a module, choose M here: the
-	  module will be called efs.
diff --git a/fs/efs/Makefile b/fs/efs/Makefile
deleted file mode 100644
index 85e5b88f94712..0000000000000
--- a/fs/efs/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# Makefile for the linux efs-filesystem routines.
-#
-
-obj-$(CONFIG_EFS_FS) += efs.o
-
-efs-objs := super.o inode.o namei.o dir.o file.o symlink.o
diff --git a/fs/efs/dir.c b/fs/efs/dir.c
deleted file mode 100644
index 35ad0092c1154..0000000000000
--- a/fs/efs/dir.c
+++ /dev/null
@@ -1,105 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * dir.c
- *
- * Copyright (c) 1999 Al Smith
- */
-
-#include <linux/buffer_head.h>
-#include <linux/filelock.h>
-#include "efs.h"
-
-static int efs_readdir(struct file *, struct dir_context *);
-
-const struct file_operations efs_dir_operations = {
-	.llseek		= generic_file_llseek,
-	.read		= generic_read_dir,
-	.iterate_shared	= efs_readdir,
-	.setlease	= generic_setlease,
-};
-
-const struct inode_operations efs_dir_inode_operations = {
-	.lookup		= efs_lookup,
-};
-
-static int efs_readdir(struct file *file, struct dir_context *ctx)
-{
-	struct inode *inode = file_inode(file);
-	efs_block_t		block;
-	int			slot;
-
-	if (inode->i_size & (EFS_DIRBSIZE-1))
-		pr_warn("%s(): directory size not a multiple of EFS_DIRBSIZE\n",
-			__func__);
-
-	/* work out where this entry can be found */
-	block = ctx->pos >> EFS_DIRBSIZE_BITS;
-
-	/* each block contains at most 256 slots */
-	slot  = ctx->pos & 0xff;
-
-	/* look at all blocks */
-	while (block < inode->i_blocks) {
-		struct efs_dir		*dirblock;
-		struct buffer_head *bh;
-
-		/* read the dir block */
-		bh = sb_bread(inode->i_sb, efs_bmap(inode, block));
-
-		if (!bh) {
-			pr_err("%s(): failed to read dir block %d\n",
-			       __func__, block);
-			break;
-		}
-
-		dirblock = (struct efs_dir *) bh->b_data; 
-
-		if (be16_to_cpu(dirblock->magic) != EFS_DIRBLK_MAGIC) {
-			pr_err("%s(): invalid directory block\n", __func__);
-			brelse(bh);
-			break;
-		}
-
-		for (; slot < dirblock->slots; slot++) {
-			struct efs_dentry *dirslot;
-			efs_ino_t inodenum;
-			const char *nameptr;
-			int namelen;
-
-			if (dirblock->space[slot] == 0)
-				continue;
-
-			dirslot  = (struct efs_dentry *) (((char *) bh->b_data) + EFS_SLOTAT(dirblock, slot));
-
-			inodenum = be32_to_cpu(dirslot->inode);
-			namelen  = dirslot->namelen;
-			nameptr  = dirslot->name;
-			pr_debug("%s(): block %d slot %d/%d: inode %u, name \"%s\", namelen %u\n",
-				 __func__, block, slot, dirblock->slots-1,
-				 inodenum, nameptr, namelen);
-			if (!namelen)
-				continue;
-			/* found the next entry */
-			ctx->pos = (block << EFS_DIRBSIZE_BITS) | slot;
-
-			/* sanity check */
-			if (nameptr - (char *) dirblock + namelen > EFS_DIRBSIZE) {
-				pr_warn("directory entry %d exceeds directory block\n",
-					slot);
-				continue;
-			}
-
-			/* copy filename and data in dirslot */
-			if (!dir_emit(ctx, nameptr, namelen, inodenum, DT_UNKNOWN)) {
-				brelse(bh);
-				return 0;
-			}
-		}
-		brelse(bh);
-
-		slot = 0;
-		block++;
-	}
-	ctx->pos = (block << EFS_DIRBSIZE_BITS) | slot;
-	return 0;
-}
diff --git a/fs/efs/efs.h b/fs/efs/efs.h
deleted file mode 100644
index 918d2b9abb763..0000000000000
--- a/fs/efs/efs.h
+++ /dev/null
@@ -1,144 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (c) 1999 Al Smith, <Al.Smith@aeschi.ch.eu.org>
- *
- * Portions derived from work (c) 1995,1996 Christian Vogelgsang.
- * Portions derived from IRIX header files (c) 1988 Silicon Graphics
- */
-#ifndef _EFS_EFS_H_
-#define _EFS_EFS_H_
-
-#ifdef pr_fmt
-#undef pr_fmt
-#endif
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/fs.h>
-#include <linux/uaccess.h>
-
-#define EFS_VERSION "1.0a"
-
-/* 1 block is 512 bytes */
-#define	EFS_BLOCKSIZE_BITS	9
-#define	EFS_BLOCKSIZE		(1 << EFS_BLOCKSIZE_BITS)
-
-typedef	int32_t		efs_block_t;
-typedef uint32_t	efs_ino_t;
-
-#define	EFS_DIRECTEXTENTS	12
-
-/*
- * layout of an extent, in memory and on disk. 8 bytes exactly.
- */
-typedef union extent_u {
-	unsigned char raw[8];
-	struct extent_s {
-		unsigned int	ex_magic:8;	/* magic # (zero) */
-		unsigned int	ex_bn:24;	/* basic block */
-		unsigned int	ex_length:8;	/* numblocks in this extent */
-		unsigned int	ex_offset:24;	/* logical offset into file */
-	} cooked;
-} efs_extent;
-
-typedef struct edevs {
-	__be16		odev;
-	__be32		ndev;
-} efs_devs;
-
-/*
- * extent based filesystem inode as it appears on disk.  The efs inode
- * is exactly 128 bytes long.
- */
-struct	efs_dinode {
-	__be16		di_mode;	/* mode and type of file */
-	__be16		di_nlink;	/* number of links to file */
-	__be16		di_uid;		/* owner's user id */
-	__be16		di_gid;		/* owner's group id */
-	__be32		di_size;	/* number of bytes in file */
-	__be32		di_atime;	/* time last accessed */
-	__be32		di_mtime;	/* time last modified */
-	__be32		di_ctime;	/* time created */
-	__be32		di_gen;		/* generation number */
-	__be16		di_numextents;	/* # of extents */
-	u_char		di_version;	/* version of inode */
-	u_char		di_spare;	/* spare - used by AFS */
-	union di_addr {
-		efs_extent	di_extents[EFS_DIRECTEXTENTS];
-		efs_devs	di_dev;	/* device for IFCHR/IFBLK */
-	} di_u;
-};
-
-/* efs inode storage in memory */
-struct efs_inode_info {
-	int		numextents;
-	int		lastextent;
-
-	efs_extent	extents[EFS_DIRECTEXTENTS];
-	struct inode	vfs_inode;
-};
-
-#include <linux/efs_fs_sb.h>
-
-#define EFS_DIRBSIZE_BITS	EFS_BLOCKSIZE_BITS
-#define EFS_DIRBSIZE		(1 << EFS_DIRBSIZE_BITS)
-
-struct efs_dentry {
-	__be32		inode;
-	unsigned char	namelen;
-	char		name[3];
-};
-
-#define EFS_DENTSIZE	(sizeof(struct efs_dentry) - 3 + 1)
-#define EFS_MAXNAMELEN  ((1 << (sizeof(char) * 8)) - 1)
-
-#define EFS_DIRBLK_HEADERSIZE	4
-#define EFS_DIRBLK_MAGIC	0xbeef	/* moo */
-
-struct efs_dir {
-	__be16	magic;
-	unsigned char	firstused;
-	unsigned char	slots;
-
-	unsigned char	space[EFS_DIRBSIZE - EFS_DIRBLK_HEADERSIZE];
-};
-
-#define EFS_MAXENTS \
-	((EFS_DIRBSIZE - EFS_DIRBLK_HEADERSIZE) / \
-	 (EFS_DENTSIZE + sizeof(char)))
-
-#define EFS_SLOTAT(dir, slot) EFS_REALOFF((dir)->space[slot])
-
-#define EFS_REALOFF(offset) ((offset << 1))
-
-
-static inline struct efs_inode_info *INODE_INFO(struct inode *inode)
-{
-	return container_of(inode, struct efs_inode_info, vfs_inode);
-}
-
-static inline struct efs_sb_info *SUPER_INFO(struct super_block *sb)
-{
-	return sb->s_fs_info;
-}
-
-struct statfs;
-struct fid;
-
-extern const struct inode_operations efs_dir_inode_operations;
-extern const struct file_operations efs_dir_operations;
-extern const struct address_space_operations efs_symlink_aops;
-
-extern struct inode *efs_iget(struct super_block *, unsigned long);
-extern efs_block_t efs_map_block(struct inode *, efs_block_t);
-extern int efs_get_block(struct inode *, sector_t, struct buffer_head *, int);
-
-extern struct dentry *efs_lookup(struct inode *, struct dentry *, unsigned int);
-extern struct dentry *efs_fh_to_dentry(struct super_block *sb, struct fid *fid,
-		int fh_len, int fh_type);
-extern struct dentry *efs_fh_to_parent(struct super_block *sb, struct fid *fid,
-		int fh_len, int fh_type);
-extern struct dentry *efs_get_parent(struct dentry *);
-extern int efs_bmap(struct inode *, int);
-
-#endif /* _EFS_EFS_H_ */
diff --git a/fs/efs/file.c b/fs/efs/file.c
deleted file mode 100644
index 9153dfe79bbcc..0000000000000
--- a/fs/efs/file.c
+++ /dev/null
@@ -1,42 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * file.c
- *
- * Copyright (c) 1999 Al Smith
- *
- * Portions derived from work (c) 1995,1996 Christian Vogelgsang.
- */
-
-#include <linux/buffer_head.h>
-#include "efs.h"
-
-int efs_get_block(struct inode *inode, sector_t iblock,
-		  struct buffer_head *bh_result, int create)
-{
-	int error = -EROFS;
-	long phys;
-
-	if (create)
-		return error;
-	if (iblock >= inode->i_blocks)
-		return 0;
-
-	phys = efs_map_block(inode, iblock);
-	if (phys)
-		map_bh(bh_result, inode->i_sb, phys);
-	return 0;
-}
-
-int efs_bmap(struct inode *inode, efs_block_t block) {
-
-	if (block < 0) {
-		pr_warn("%s(): block < 0\n", __func__);
-		return 0;
-	}
-
-	/* are we about to read past the end of a file ? */
-	if (!(block < inode->i_blocks))
-		return 0;
-
-	return efs_map_block(inode, block);
-}
diff --git a/fs/efs/inode.c b/fs/efs/inode.c
deleted file mode 100644
index 4b132729e6385..0000000000000
--- a/fs/efs/inode.c
+++ /dev/null
@@ -1,315 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * inode.c
- *
- * Copyright (c) 1999 Al Smith
- *
- * Portions derived from work (c) 1995,1996 Christian Vogelgsang,
- *              and from work (c) 1998 Mike Shaver.
- */
-
-#include <linux/buffer_head.h>
-#include <linux/module.h>
-#include <linux/fs.h>
-#include "efs.h"
-#include <linux/efs_fs_sb.h>
-
-static int efs_read_folio(struct file *file, struct folio *folio)
-{
-	return block_read_full_folio(folio, efs_get_block);
-}
-
-static sector_t _efs_bmap(struct address_space *mapping, sector_t block)
-{
-	return generic_block_bmap(mapping,block,efs_get_block);
-}
-
-static const struct address_space_operations efs_aops = {
-	.read_folio = efs_read_folio,
-	.bmap = _efs_bmap
-};
-
-static inline void extent_copy(efs_extent *src, efs_extent *dst) {
-	/*
-	 * this is slightly evil. it doesn't just copy
-	 * efs_extent from src to dst, it also mangles
-	 * the bits so that dst ends up in cpu byte-order.
-	 */
-
-	dst->cooked.ex_magic  =  (unsigned int) src->raw[0];
-	dst->cooked.ex_bn     = ((unsigned int) src->raw[1] << 16) |
-				((unsigned int) src->raw[2] <<  8) |
-				((unsigned int) src->raw[3] <<  0);
-	dst->cooked.ex_length =  (unsigned int) src->raw[4];
-	dst->cooked.ex_offset = ((unsigned int) src->raw[5] << 16) |
-				((unsigned int) src->raw[6] <<  8) |
-				((unsigned int) src->raw[7] <<  0);
-	return;
-}
-
-struct inode *efs_iget(struct super_block *super, unsigned long ino)
-{
-	int i, inode_index;
-	dev_t device;
-	u32 rdev;
-	struct buffer_head *bh;
-	struct efs_sb_info    *sb = SUPER_INFO(super);
-	struct efs_inode_info *in;
-	efs_block_t block, offset;
-	struct efs_dinode *efs_inode;
-	struct inode *inode;
-
-	inode = iget_locked(super, ino);
-	if (!inode)
-		return ERR_PTR(-ENOMEM);
-	if (!(inode_state_read_once(inode) & I_NEW))
-		return inode;
-
-	in = INODE_INFO(inode);
-
-	/*
-	** EFS layout:
-	**
-	** |   cylinder group    |   cylinder group    |   cylinder group ..etc
-	** |inodes|data          |inodes|data          |inodes|data       ..etc
-	**
-	** work out the inode block index, (considering initially that the
-	** inodes are stored as consecutive blocks). then work out the block
-	** number of that inode given the above layout, and finally the
-	** offset of the inode within that block.
-	*/
-
-	inode_index = inode->i_ino /
-		(EFS_BLOCKSIZE / sizeof(struct efs_dinode));
-
-	block = sb->fs_start + sb->first_block + 
-		(sb->group_size * (inode_index / sb->inode_blocks)) +
-		(inode_index % sb->inode_blocks);
-
-	offset = (inode->i_ino %
-			(EFS_BLOCKSIZE / sizeof(struct efs_dinode))) *
-		sizeof(struct efs_dinode);
-
-	bh = sb_bread(inode->i_sb, block);
-	if (!bh) {
-		pr_warn("%s() failed at block %d\n", __func__, block);
-		goto read_inode_error;
-	}
-
-	efs_inode = (struct efs_dinode *) (bh->b_data + offset);
-    
-	inode->i_mode  = be16_to_cpu(efs_inode->di_mode);
-	set_nlink(inode, be16_to_cpu(efs_inode->di_nlink));
-	i_uid_write(inode, (uid_t)be16_to_cpu(efs_inode->di_uid));
-	i_gid_write(inode, (gid_t)be16_to_cpu(efs_inode->di_gid));
-	inode->i_size  = be32_to_cpu(efs_inode->di_size);
-	inode_set_atime(inode, be32_to_cpu(efs_inode->di_atime), 0);
-	inode_set_mtime(inode, be32_to_cpu(efs_inode->di_mtime), 0);
-	inode_set_ctime(inode, be32_to_cpu(efs_inode->di_ctime), 0);
-
-	/* this is the number of blocks in the file */
-	if (inode->i_size == 0) {
-		inode->i_blocks = 0;
-	} else {
-		inode->i_blocks = ((inode->i_size - 1) >> EFS_BLOCKSIZE_BITS) + 1;
-	}
-
-	rdev = be16_to_cpu(efs_inode->di_u.di_dev.odev);
-	if (rdev == 0xffff) {
-		rdev = be32_to_cpu(efs_inode->di_u.di_dev.ndev);
-		if (sysv_major(rdev) > 0xfff)
-			device = 0;
-		else
-			device = MKDEV(sysv_major(rdev), sysv_minor(rdev));
-	} else
-		device = old_decode_dev(rdev);
-
-	/* get the number of extents for this object */
-	in->numextents = be16_to_cpu(efs_inode->di_numextents);
-	in->lastextent = 0;
-
-	/* copy the extents contained within the inode to memory */
-	for(i = 0; i < EFS_DIRECTEXTENTS; i++) {
-		extent_copy(&(efs_inode->di_u.di_extents[i]), &(in->extents[i]));
-		if (i < in->numextents && in->extents[i].cooked.ex_magic != 0) {
-			pr_warn("extent %d has bad magic number in inode %llu\n",
-				i, inode->i_ino);
-			brelse(bh);
-			goto read_inode_error;
-		}
-	}
-
-	brelse(bh);
-	pr_debug("efs_iget(): inode %llu, extents %d, mode %o\n",
-		 inode->i_ino, in->numextents, inode->i_mode);
-	switch (inode->i_mode & S_IFMT) {
-		case S_IFDIR: 
-			inode->i_op = &efs_dir_inode_operations; 
-			inode->i_fop = &efs_dir_operations; 
-			break;
-		case S_IFREG:
-			inode->i_fop = &generic_ro_fops;
-			inode->i_data.a_ops = &efs_aops;
-			break;
-		case S_IFLNK:
-			inode->i_op = &page_symlink_inode_operations;
-			inode_nohighmem(inode);
-			inode->i_data.a_ops = &efs_symlink_aops;
-			break;
-		case S_IFCHR:
-		case S_IFBLK:
-		case S_IFIFO:
-			init_special_inode(inode, inode->i_mode, device);
-			break;
-		default:
-			pr_warn("unsupported inode mode %o\n", inode->i_mode);
-			goto read_inode_error;
-			break;
-	}
-
-	unlock_new_inode(inode);
-	return inode;
-        
-read_inode_error:
-	pr_warn("failed to read inode %llu\n", inode->i_ino);
-	iget_failed(inode);
-	return ERR_PTR(-EIO);
-}
-
-static inline efs_block_t
-efs_extent_check(efs_extent *ptr, efs_block_t block, struct efs_sb_info *sb) {
-	efs_block_t start;
-	efs_block_t length;
-	efs_block_t offset;
-
-	/*
-	 * given an extent and a logical block within a file,
-	 * can this block be found within this extent ?
-	 */
-	start  = ptr->cooked.ex_bn;
-	length = ptr->cooked.ex_length;
-	offset = ptr->cooked.ex_offset;
-
-	if ((block >= offset) && (block < offset+length)) {
-		return(sb->fs_start + start + block - offset);
-	} else {
-		return 0;
-	}
-}
-
-efs_block_t efs_map_block(struct inode *inode, efs_block_t block) {
-	struct efs_sb_info    *sb = SUPER_INFO(inode->i_sb);
-	struct efs_inode_info *in = INODE_INFO(inode);
-	struct buffer_head    *bh = NULL;
-
-	int cur, last, first = 1;
-	int ibase, ioffset, dirext, direxts, indext, indexts;
-	efs_block_t iblock, result = 0, lastblock = 0;
-	efs_extent ext, *exts;
-
-	last = in->lastextent;
-
-	if (in->numextents <= EFS_DIRECTEXTENTS) {
-		/* first check the last extent we returned */
-		if ((result = efs_extent_check(&in->extents[last], block, sb)))
-			return result;
-    
-		/* if we only have one extent then nothing can be found */
-		if (in->numextents == 1) {
-			pr_err("%s() failed to map (1 extent)\n", __func__);
-			return 0;
-		}
-
-		direxts = in->numextents;
-
-		/*
-		 * check the stored extents in the inode
-		 * start with next extent and check forwards
-		 */
-		for(dirext = 1; dirext < direxts; dirext++) {
-			cur = (last + dirext) % in->numextents;
-			if ((result = efs_extent_check(&in->extents[cur], block, sb))) {
-				in->lastextent = cur;
-				return result;
-			}
-		}
-
-		pr_err("%s() failed to map block %u (dir)\n", __func__, block);
-		return 0;
-	}
-
-	pr_debug("%s(): indirect search for logical block %u\n",
-		 __func__, block);
-	direxts = in->extents[0].cooked.ex_offset;
-	indexts = in->numextents;
-
-	for(indext = 0; indext < indexts; indext++) {
-		cur = (last + indext) % indexts;
-
-		/*
-		 * work out which direct extent contains `cur'.
-		 *
-		 * also compute ibase: i.e. the number of the first
-		 * indirect extent contained within direct extent `cur'.
-		 *
-		 */
-		ibase = 0;
-		for(dirext = 0; cur < ibase && dirext < direxts; dirext++) {
-			ibase += in->extents[dirext].cooked.ex_length *
-				(EFS_BLOCKSIZE / sizeof(efs_extent));
-		}
-
-		if (dirext == direxts) {
-			/* should never happen */
-			pr_err("couldn't find direct extent for indirect extent %d (block %u)\n",
-			       cur, block);
-			if (bh) brelse(bh);
-			return 0;
-		}
-		
-		/* work out block number and offset of this indirect extent */
-		iblock = sb->fs_start + in->extents[dirext].cooked.ex_bn +
-			(cur - ibase) /
-			(EFS_BLOCKSIZE / sizeof(efs_extent));
-		ioffset = (cur - ibase) %
-			(EFS_BLOCKSIZE / sizeof(efs_extent));
-
-		if (first || lastblock != iblock) {
-			if (bh) brelse(bh);
-
-			bh = sb_bread(inode->i_sb, iblock);
-			if (!bh) {
-				pr_err("%s() failed at block %d\n",
-				       __func__, iblock);
-				return 0;
-			}
-			pr_debug("%s(): read indirect extent block %d\n",
-				 __func__, iblock);
-			first = 0;
-			lastblock = iblock;
-		}
-
-		exts = (efs_extent *) bh->b_data;
-
-		extent_copy(&(exts[ioffset]), &ext);
-
-		if (ext.cooked.ex_magic != 0) {
-			pr_err("extent %d has bad magic number in block %d\n",
-			       cur, iblock);
-			if (bh) brelse(bh);
-			return 0;
-		}
-
-		if ((result = efs_extent_check(&ext, block, sb))) {
-			if (bh) brelse(bh);
-			in->lastextent = cur;
-			return result;
-		}
-	}
-	if (bh) brelse(bh);
-	pr_err("%s() failed to map block %u (indir)\n", __func__, block);
-	return 0;
-}  
-
-MODULE_DESCRIPTION("Extent File System (efs)");
-MODULE_LICENSE("GPL");
diff --git a/fs/efs/namei.c b/fs/efs/namei.c
deleted file mode 100644
index 38961ee1d1af1..0000000000000
--- a/fs/efs/namei.c
+++ /dev/null
@@ -1,120 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * namei.c
- *
- * Copyright (c) 1999 Al Smith
- *
- * Portions derived from work (c) 1995,1996 Christian Vogelgsang.
- */
-
-#include <linux/buffer_head.h>
-#include <linux/string.h>
-#include <linux/exportfs.h>
-#include "efs.h"
-
-
-static efs_ino_t efs_find_entry(struct inode *inode, const char *name, int len)
-{
-	struct buffer_head *bh;
-
-	int			slot, namelen;
-	char			*nameptr;
-	struct efs_dir		*dirblock;
-	struct efs_dentry	*dirslot;
-	efs_ino_t		inodenum;
-	efs_block_t		block;
- 
-	if (inode->i_size & (EFS_DIRBSIZE-1))
-		pr_warn("%s(): directory size not a multiple of EFS_DIRBSIZE\n",
-			__func__);
-
-	for(block = 0; block < inode->i_blocks; block++) {
-
-		bh = sb_bread(inode->i_sb, efs_bmap(inode, block));
-		if (!bh) {
-			pr_err("%s(): failed to read dir block %d\n",
-			       __func__, block);
-			return 0;
-		}
-    
-		dirblock = (struct efs_dir *) bh->b_data;
-
-		if (be16_to_cpu(dirblock->magic) != EFS_DIRBLK_MAGIC) {
-			pr_err("%s(): invalid directory block\n", __func__);
-			brelse(bh);
-			return 0;
-		}
-
-		for (slot = 0; slot < dirblock->slots; slot++) {
-			dirslot  = (struct efs_dentry *) (((char *) bh->b_data) + EFS_SLOTAT(dirblock, slot));
-
-			namelen  = dirslot->namelen;
-			nameptr  = dirslot->name;
-
-			if ((namelen == len) && (!memcmp(name, nameptr, len))) {
-				inodenum = be32_to_cpu(dirslot->inode);
-				brelse(bh);
-				return inodenum;
-			}
-		}
-		brelse(bh);
-	}
-	return 0;
-}
-
-struct dentry *efs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags)
-{
-	efs_ino_t inodenum;
-	struct inode *inode = NULL;
-
-	inodenum = efs_find_entry(dir, dentry->d_name.name, dentry->d_name.len);
-	if (inodenum)
-		inode = efs_iget(dir->i_sb, inodenum);
-
-	return d_splice_alias(inode, dentry);
-}
-
-static struct inode *efs_nfs_get_inode(struct super_block *sb, u64 ino,
-		u32 generation)
-{
-	struct inode *inode;
-
-	if (ino == 0)
-		return ERR_PTR(-ESTALE);
-	inode = efs_iget(sb, ino);
-	if (IS_ERR(inode))
-		return ERR_CAST(inode);
-
-	if (generation && inode->i_generation != generation) {
-		iput(inode);
-		return ERR_PTR(-ESTALE);
-	}
-
-	return inode;
-}
-
-struct dentry *efs_fh_to_dentry(struct super_block *sb, struct fid *fid,
-		int fh_len, int fh_type)
-{
-	return generic_fh_to_dentry(sb, fid, fh_len, fh_type,
-				    efs_nfs_get_inode);
-}
-
-struct dentry *efs_fh_to_parent(struct super_block *sb, struct fid *fid,
-		int fh_len, int fh_type)
-{
-	return generic_fh_to_parent(sb, fid, fh_len, fh_type,
-				    efs_nfs_get_inode);
-}
-
-struct dentry *efs_get_parent(struct dentry *child)
-{
-	struct dentry *parent = ERR_PTR(-ENOENT);
-	efs_ino_t ino;
-
-	ino = efs_find_entry(d_inode(child), "..", 2);
-	if (ino)
-		parent = d_obtain_alias(efs_iget(child->d_sb, ino));
-
-	return parent;
-}
diff --git a/fs/efs/super.c b/fs/efs/super.c
deleted file mode 100644
index 11fea3bbce7ca..0000000000000
--- a/fs/efs/super.c
+++ /dev/null
@@ -1,368 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * super.c
- *
- * Copyright (c) 1999 Al Smith
- *
- * Portions derived from work (c) 1995,1996 Christian Vogelgsang.
- */
-
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/exportfs.h>
-#include <linux/slab.h>
-#include <linux/buffer_head.h>
-#include <linux/vfs.h>
-#include <linux/blkdev.h>
-#include <linux/fs_context.h>
-#include "efs.h"
-#include <linux/efs_vh.h>
-#include <linux/efs_fs_sb.h>
-
-static int efs_statfs(struct dentry *dentry, struct kstatfs *buf);
-static int efs_init_fs_context(struct fs_context *fc);
-
-static void efs_kill_sb(struct super_block *s)
-{
-	struct efs_sb_info *sbi = SUPER_INFO(s);
-	kill_block_super(s);
-	kfree(sbi);
-}
-
-static struct pt_types sgi_pt_types[] = {
-	{0x00,		"SGI vh"},
-	{0x01,		"SGI trkrepl"},
-	{0x02,		"SGI secrepl"},
-	{0x03,		"SGI raw"},
-	{0x04,		"SGI bsd"},
-	{SGI_SYSV,	"SGI sysv"},
-	{0x06,		"SGI vol"},
-	{SGI_EFS,	"SGI efs"},
-	{0x08,		"SGI lv"},
-	{0x09,		"SGI rlv"},
-	{0x0A,		"SGI xfs"},
-	{0x0B,		"SGI xfslog"},
-	{0x0C,		"SGI xlv"},
-	{0x82,		"Linux swap"},
-	{0x83,		"Linux native"},
-	{0,		NULL}
-};
-
-/*
- * File system definition and registration.
- */
-static struct file_system_type efs_fs_type = {
-	.owner			= THIS_MODULE,
-	.name			= "efs",
-	.kill_sb		= efs_kill_sb,
-	.fs_flags		= FS_REQUIRES_DEV,
-	.init_fs_context	= efs_init_fs_context,
-};
-MODULE_ALIAS_FS("efs");
-
-static struct kmem_cache * efs_inode_cachep;
-
-static struct inode *efs_alloc_inode(struct super_block *sb)
-{
-	struct efs_inode_info *ei;
-	ei = alloc_inode_sb(sb, efs_inode_cachep, GFP_KERNEL);
-	if (!ei)
-		return NULL;
-	return &ei->vfs_inode;
-}
-
-static void efs_free_inode(struct inode *inode)
-{
-	kmem_cache_free(efs_inode_cachep, INODE_INFO(inode));
-}
-
-static void init_once(void *foo)
-{
-	struct efs_inode_info *ei = (struct efs_inode_info *) foo;
-
-	inode_init_once(&ei->vfs_inode);
-}
-
-static int __init init_inodecache(void)
-{
-	efs_inode_cachep = kmem_cache_create("efs_inode_cache",
-				sizeof(struct efs_inode_info), 0,
-				SLAB_RECLAIM_ACCOUNT|SLAB_ACCOUNT,
-				init_once);
-	if (efs_inode_cachep == NULL)
-		return -ENOMEM;
-	return 0;
-}
-
-static void destroy_inodecache(void)
-{
-	/*
-	 * Make sure all delayed rcu free inodes are flushed before we
-	 * destroy cache.
-	 */
-	rcu_barrier();
-	kmem_cache_destroy(efs_inode_cachep);
-}
-
-static const struct super_operations efs_superblock_operations = {
-	.alloc_inode	= efs_alloc_inode,
-	.free_inode	= efs_free_inode,
-	.statfs		= efs_statfs,
-};
-
-static const struct export_operations efs_export_ops = {
-	.encode_fh	= generic_encode_ino32_fh,
-	.fh_to_dentry	= efs_fh_to_dentry,
-	.fh_to_parent	= efs_fh_to_parent,
-	.get_parent	= efs_get_parent,
-};
-
-static int __init init_efs_fs(void) {
-	int err;
-	pr_info(EFS_VERSION" - http://aeschi.ch.eu.org/efs/\n");
-	err = init_inodecache();
-	if (err)
-		goto out1;
-	err = register_filesystem(&efs_fs_type);
-	if (err)
-		goto out;
-	return 0;
-out:
-	destroy_inodecache();
-out1:
-	return err;
-}
-
-static void __exit exit_efs_fs(void) {
-	unregister_filesystem(&efs_fs_type);
-	destroy_inodecache();
-}
-
-module_init(init_efs_fs)
-module_exit(exit_efs_fs)
-
-static efs_block_t efs_validate_vh(struct volume_header *vh) {
-	int		i;
-	__be32		cs, *ui;
-	int		csum;
-	efs_block_t	sblock = 0; /* shuts up gcc */
-	struct pt_types	*pt_entry;
-	int		pt_type, slice = -1;
-
-	if (be32_to_cpu(vh->vh_magic) != VHMAGIC) {
-		/*
-		 * assume that we're dealing with a partition and allow
-		 * read_super() to try and detect a valid superblock
-		 * on the next block.
-		 */
-		return 0;
-	}
-
-	ui = ((__be32 *) (vh + 1)) - 1;
-	for(csum = 0; ui >= ((__be32 *) vh);) {
-		cs = *ui--;
-		csum += be32_to_cpu(cs);
-	}
-	if (csum) {
-		pr_warn("SGI disklabel: checksum bad, label corrupted\n");
-		return 0;
-	}
-
-#ifdef DEBUG
-	pr_debug("bf: \"%16s\"\n", vh->vh_bootfile);
-
-	for(i = 0; i < NVDIR; i++) {
-		int	j;
-		char	name[VDNAMESIZE+1];
-
-		for(j = 0; j < VDNAMESIZE; j++) {
-			name[j] = vh->vh_vd[i].vd_name[j];
-		}
-		name[j] = (char) 0;
-
-		if (name[0]) {
-			pr_debug("vh: %8s block: 0x%08x size: 0x%08x\n",
-				name, (int) be32_to_cpu(vh->vh_vd[i].vd_lbn),
-				(int) be32_to_cpu(vh->vh_vd[i].vd_nbytes));
-		}
-	}
-#endif
-
-	for(i = 0; i < NPARTAB; i++) {
-		pt_type = (int) be32_to_cpu(vh->vh_pt[i].pt_type);
-		for(pt_entry = sgi_pt_types; pt_entry->pt_name; pt_entry++) {
-			if (pt_type == pt_entry->pt_type) break;
-		}
-#ifdef DEBUG
-		if (be32_to_cpu(vh->vh_pt[i].pt_nblks)) {
-			pr_debug("pt %2d: start: %08d size: %08d type: 0x%02x (%s)\n",
-				 i, (int)be32_to_cpu(vh->vh_pt[i].pt_firstlbn),
-				 (int)be32_to_cpu(vh->vh_pt[i].pt_nblks),
-				 pt_type, (pt_entry->pt_name) ?
-				 pt_entry->pt_name : "unknown");
-		}
-#endif
-		if (IS_EFS(pt_type)) {
-			sblock = be32_to_cpu(vh->vh_pt[i].pt_firstlbn);
-			slice = i;
-		}
-	}
-
-	if (slice == -1) {
-		pr_notice("partition table contained no EFS partitions\n");
-#ifdef DEBUG
-	} else {
-		pr_info("using slice %d (type %s, offset 0x%x)\n", slice,
-			(pt_entry->pt_name) ? pt_entry->pt_name : "unknown",
-			sblock);
-#endif
-	}
-	return sblock;
-}
-
-static int efs_validate_super(struct efs_sb_info *sb, struct efs_super *super) {
-
-	if (!IS_EFS_MAGIC(be32_to_cpu(super->fs_magic)))
-		return -1;
-
-	sb->fs_magic     = be32_to_cpu(super->fs_magic);
-	sb->total_blocks = be32_to_cpu(super->fs_size);
-	sb->first_block  = be32_to_cpu(super->fs_firstcg);
-	sb->group_size   = be32_to_cpu(super->fs_cgfsize);
-	sb->data_free    = be32_to_cpu(super->fs_tfree);
-	sb->inode_free   = be32_to_cpu(super->fs_tinode);
-	sb->inode_blocks = be16_to_cpu(super->fs_cgisize);
-	sb->total_groups = be16_to_cpu(super->fs_ncg);
-    
-	return 0;    
-}
-
-static int efs_fill_super(struct super_block *s, struct fs_context *fc)
-{
-	struct efs_sb_info *sb;
-	struct buffer_head *bh;
-	struct inode *root;
-
-	sb = kzalloc_obj(struct efs_sb_info);
-	if (!sb)
-		return -ENOMEM;
-	s->s_fs_info = sb;
-	s->s_time_min = 0;
-	s->s_time_max = U32_MAX;
-
-	s->s_magic		= EFS_SUPER_MAGIC;
-	if (!sb_set_blocksize(s, EFS_BLOCKSIZE)) {
-		pr_err("device does not support %d byte blocks\n",
-			EFS_BLOCKSIZE);
-		return invalf(fc, "device does not support %d byte blocks\n",
-			      EFS_BLOCKSIZE);
-	}
-
-	/* read the vh (volume header) block */
-	bh = sb_bread(s, 0);
-
-	if (!bh) {
-		pr_err("cannot read volume header\n");
-		return -EIO;
-	}
-
-	/*
-	 * if this returns zero then we didn't find any partition table.
-	 * this isn't (yet) an error - just assume for the moment that
-	 * the device is valid and go on to search for a superblock.
-	 */
-	sb->fs_start = efs_validate_vh((struct volume_header *) bh->b_data);
-	brelse(bh);
-
-	if (sb->fs_start == -1) {
-		return -EINVAL;
-	}
-
-	bh = sb_bread(s, sb->fs_start + EFS_SUPER);
-	if (!bh) {
-		pr_err("cannot read superblock\n");
-		return -EIO;
-	}
-
-	if (efs_validate_super(sb, (struct efs_super *) bh->b_data)) {
-#ifdef DEBUG
-		pr_warn("invalid superblock at block %u\n",
-			sb->fs_start + EFS_SUPER);
-#endif
-		brelse(bh);
-		return -EINVAL;
-	}
-	brelse(bh);
-
-	if (!sb_rdonly(s)) {
-#ifdef DEBUG
-		pr_info("forcing read-only mode\n");
-#endif
-		s->s_flags |= SB_RDONLY;
-	}
-	s->s_op   = &efs_superblock_operations;
-	s->s_export_op = &efs_export_ops;
-	root = efs_iget(s, EFS_ROOTINODE);
-	if (IS_ERR(root)) {
-		pr_err("get root inode failed\n");
-		return PTR_ERR(root);
-	}
-
-	s->s_root = d_make_root(root);
-	if (!(s->s_root)) {
-		pr_err("get root dentry failed\n");
-		return -ENOMEM;
-	}
-
-	return 0;
-}
-
-static int efs_get_tree(struct fs_context *fc)
-{
-	return get_tree_bdev(fc, efs_fill_super);
-}
-
-static int efs_reconfigure(struct fs_context *fc)
-{
-	sync_filesystem(fc->root->d_sb);
-	fc->sb_flags |= SB_RDONLY;
-
-	return 0;
-}
-
-static const struct fs_context_operations efs_context_opts = {
-	.get_tree	= efs_get_tree,
-	.reconfigure	= efs_reconfigure,
-};
-
-/*
- * Set up the filesystem mount context.
- */
-static int efs_init_fs_context(struct fs_context *fc)
-{
-	fc->ops = &efs_context_opts;
-
-	return 0;
-}
-
-static int efs_statfs(struct dentry *dentry, struct kstatfs *buf) {
-	struct super_block *sb = dentry->d_sb;
-	struct efs_sb_info *sbi = SUPER_INFO(sb);
-	u64 id = huge_encode_dev(sb->s_bdev->bd_dev);
-
-	buf->f_type    = EFS_SUPER_MAGIC;	/* efs magic number */
-	buf->f_bsize   = EFS_BLOCKSIZE;		/* blocksize */
-	buf->f_blocks  = sbi->total_groups *	/* total data blocks */
-			(sbi->group_size - sbi->inode_blocks);
-	buf->f_bfree   = sbi->data_free;	/* free data blocks */
-	buf->f_bavail  = sbi->data_free;	/* free blocks for non-root */
-	buf->f_files   = sbi->total_groups *	/* total inodes */
-			sbi->inode_blocks *
-			(EFS_BLOCKSIZE / sizeof(struct efs_dinode));
-	buf->f_ffree   = sbi->inode_free;	/* free inodes */
-	buf->f_fsid    = u64_to_fsid(id);
-	buf->f_namelen = EFS_MAXNAMELEN;	/* max filename length */
-
-	return 0;
-}
-
diff --git a/fs/efs/symlink.c b/fs/efs/symlink.c
deleted file mode 100644
index 7749feded722b..0000000000000
--- a/fs/efs/symlink.c
+++ /dev/null
@@ -1,50 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * symlink.c
- *
- * Copyright (c) 1999 Al Smith
- *
- * Portions derived from work (c) 1995,1996 Christian Vogelgsang.
- */
-
-#include <linux/string.h>
-#include <linux/pagemap.h>
-#include <linux/buffer_head.h>
-#include "efs.h"
-
-static int efs_symlink_read_folio(struct file *file, struct folio *folio)
-{
-	char *link = folio_address(folio);
-	struct buffer_head *bh;
-	struct inode *inode = folio->mapping->host;
-	efs_block_t size = inode->i_size;
-	int err;
-  
-	err = -ENAMETOOLONG;
-	if (size > 2 * EFS_BLOCKSIZE)
-		goto fail;
-  
-	/* read first 512 bytes of link target */
-	err = -EIO;
-	bh = sb_bread(inode->i_sb, efs_bmap(inode, 0));
-	if (!bh)
-		goto fail;
-	memcpy(link, bh->b_data, (size > EFS_BLOCKSIZE) ? EFS_BLOCKSIZE : size);
-	brelse(bh);
-	if (size > EFS_BLOCKSIZE) {
-		bh = sb_bread(inode->i_sb, efs_bmap(inode, 1));
-		if (!bh)
-			goto fail;
-		memcpy(link + EFS_BLOCKSIZE, bh->b_data, size - EFS_BLOCKSIZE);
-		brelse(bh);
-	}
-	link[size] = '\0';
-	err = 0;
-fail:
-	folio_end_read(folio, err == 0);
-	return err;
-}
-
-const struct address_space_operations efs_symlink_aops = {
-	.read_folio	= efs_symlink_read_folio
-};
diff --git a/fs/erofs/ishare.c b/fs/erofs/ishare.c
index 0868c12fc15b4..a1a20a5ba548f 100644
--- a/fs/erofs/ishare.c
+++ b/fs/erofs/ishare.c
@@ -12,12 +12,6 @@
 
 static struct vfsmount *erofs_ishare_mnt;
 
-static inline bool erofs_is_ishare_inode(struct inode *inode)
-{
-	/* assumed FS_ONDEMAND is excluded with FS_PAGE_CACHE_SHARE feature */
-	return inode->i_sb->s_type == &erofs_anon_fs_type;
-}
-
 static int erofs_ishare_iget5_eq(struct inode *inode, void *data)
 {
 	struct erofs_inode_fingerprint *fp1 = &EROFS_I(inode)->fingerprint;
@@ -179,7 +173,7 @@ struct inode *erofs_real_inode(struct inode *inode, bool *need_iput)
 	struct inode *realinode;
 
 	*need_iput = false;
-	if (!erofs_is_ishare_inode(inode))
+	if (inode->i_sb != erofs_ishare_mnt->mnt_sb)
 		return inode;
 
 	vi_share = EROFS_I(inode);
diff --git a/fs/erofs/super.c b/fs/erofs/super.c
index 86fa5c6a0c708..770808685934e 100644
--- a/fs/erofs/super.c
+++ b/fs/erofs/super.c
@@ -147,8 +147,8 @@ static int erofs_init_device(struct erofs_buf *buf, struct super_block *sb,
 	if (!sbi->devs->flatdev) {
 		file = erofs_is_fileio_mode(sbi) ?
 				filp_open(dif->path, O_RDONLY | O_LARGEFILE, 0) :
-				bdev_file_open_by_path(dif->path,
-						BLK_OPEN_READ, sb->s_type, NULL);
+				fs_bdev_file_open_by_path(dif->path,
+						BLK_OPEN_READ, sb->s_type, sb);
 		if (IS_ERR(file)) {
 			if (file == ERR_PTR(-ENOTBLK))
 				return -EINVAL;
@@ -595,17 +595,6 @@ static const struct export_operations erofs_export_ops = {
 	.get_parent = erofs_get_parent,
 };
 
-static void erofs_set_sysfs_name(struct super_block *sb)
-{
-	struct erofs_sb_info *sbi = EROFS_SB(sb);
-
-	if (erofs_is_fileio_mode(sbi))
-		super_set_sysfs_name_generic(sb, "%s",
-					     bdi_dev_name(sb->s_bdi));
-	else
-		super_set_sysfs_name_id(sb);
-}
-
 static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc)
 {
 	struct inode *inode;
@@ -657,12 +646,14 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc)
 		err = super_setup_bdi(sb);
 		if (err)
 			return err;
+
+		snprintf(sb->s_id, sizeof(sb->s_id),
+			 "%u:%u", MAJOR(sb->s_dev), MINOR(sb->s_dev));
 	} else {
 		if (!sb_set_blocksize(sb, PAGE_SIZE)) {
 			errorfc(fc, "failed to set initial blksize");
 			return -EINVAL;
 		}
-
 		sbi->dif0.dax_dev = fs_dax_get_by_bdev(sb->s_bdev,
 				&sbi->dif0.dax_part_off, NULL, NULL);
 	}
@@ -740,7 +731,7 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc)
 	if (err)
 		return err;
 
-	erofs_set_sysfs_name(sb);
+	super_set_sysfs_name_id(sb);
 	err = erofs_register_sysfs(sb);
 	if (err)
 		return err;
@@ -799,28 +790,34 @@ static int erofs_fc_reconfigure(struct fs_context *fc)
 
 static int erofs_release_device_info(int id, void *ptr, void *data)
 {
+	struct super_block *sb = data;
 	struct erofs_device_info *dif = ptr;
 
 	fs_put_dax(dif->dax_dev, NULL);
-	if (dif->file)
-		fput(dif->file);
+	if (dif->file) {
+		if (S_ISBLK(file_inode(dif->file)->i_mode))
+			fs_bdev_file_release(dif->file, sb);
+		else
+			fput(dif->file);
+	}
 	kfree(dif->path);
 	kfree(dif);
 	return 0;
 }
 
-static void erofs_free_dev_context(struct erofs_dev_context *devs)
+static void erofs_free_dev_context(struct erofs_dev_context *devs,
+				   struct super_block *sb)
 {
 	if (!devs)
 		return;
-	idr_for_each(&devs->tree, &erofs_release_device_info, NULL);
+	idr_for_each(&devs->tree, &erofs_release_device_info, sb);
 	idr_destroy(&devs->tree);
 	kfree(devs);
 }
 
-static void erofs_sb_free(struct erofs_sb_info *sbi)
+static void erofs_sb_free(struct erofs_sb_info *sbi, struct super_block *sb)
 {
-	erofs_free_dev_context(sbi->devs);
+	erofs_free_dev_context(sbi->devs, sb);
 	kfree_sensitive(sbi->domain_id);
 	if (sbi->dif0.file)
 		fput(sbi->dif0.file);
@@ -832,8 +829,13 @@ static void erofs_fc_free(struct fs_context *fc)
 {
 	struct erofs_sb_info *sbi = fc->s_fs_info;
 
-	if (sbi) /* free here if an error occurs before transferring to sb */
-		erofs_sb_free(sbi);
+	/*
+	 * Freed here only if an error occurs before the sb is set up; at that
+	 * point no block-backed device has been claimed (that happens in
+	 * fill_super), so the NULL sb never reaches fs_bdev_file_release().
+	 */
+	if (sbi)
+		erofs_sb_free(sbi, NULL);
 }
 
 static const struct fs_context_operations erofs_context_ops = {
@@ -887,7 +889,7 @@ static void erofs_kill_sb(struct super_block *sb)
 		kill_block_super(sb);
 	erofs_drop_internal_inodes(sbi);
 	fs_put_dax(sbi->dif0.dax_dev, NULL);
-	erofs_sb_free(sbi);
+	erofs_sb_free(sbi, sb);
 	sb->s_fs_info = NULL;
 }
 
@@ -899,7 +901,7 @@ static void erofs_put_super(struct super_block *sb)
 	erofs_shrinker_unregister(sb);
 	erofs_xattr_prefixes_cleanup(sb);
 	erofs_drop_internal_inodes(sbi);
-	erofs_free_dev_context(sbi->devs);
+	erofs_free_dev_context(sbi->devs, sb);
 	sbi->devs = NULL;
 }
 
@@ -1038,7 +1040,7 @@ static int erofs_show_options(struct seq_file *seq, struct dentry *root)
 				",user_xattr" : ",nouser_xattr");
 	if (IS_ENABLED(CONFIG_EROFS_FS_POSIX_ACL))
 		seq_puts(seq, test_opt(opt, POSIX_ACL) ? ",acl" : ",noacl");
-	if (IS_ENABLED(CONFIG_EROFS_FS_ZIP))
+	if (IS_ENABLED(CONFIG_EROFS_FS_ZIP) && sbi->available_compr_algs)
 		seq_printf(seq, ",cache_strategy=%s",
 			  erofs_param_cache_strategy[opt->cache_strategy].name);
 	if (test_opt(opt, DAX_ALWAYS))
diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c
index bab5216135524..5811556a7b71f 100644
--- a/fs/erofs/zmap.c
+++ b/fs/erofs/zmap.c
@@ -732,7 +732,8 @@ static int z_erofs_map_sanity_check(struct inode *inode,
 				  map->m_algorithmformat, EROFS_I(inode)->nid);
 			return -EFSCORRUPTED;
 		}
-		if (EROFS_MAP_FULL(map->m_flags) && map->m_llen < map->m_plen) {
+		if (EROFS_MAP_FULL(map->m_flags) && map->m_llen < map->m_plen &&
+		    map->m_la + map->m_llen < inode->i_size) {
 			erofs_err(inode->i_sb, "too much compressed data @ la %llu of nid %llu",
 				  map->m_la, EROFS_I(inode)->nid);
 			return -EFSCORRUPTED;
diff --git a/fs/exfat/exfat_fs.h b/fs/exfat/exfat_fs.h
index 9be50949ce34f..1f020b041a3d6 100644
--- a/fs/exfat/exfat_fs.h
+++ b/fs/exfat/exfat_fs.h
@@ -294,7 +294,7 @@ struct exfat_inode_info {
 	/* on-disk position of directory entry or 0 */
 	loff_t i_pos;
 	loff_t valid_size;
-	/* page-aligned size that has been zeroed out for mmap */
+	/* block-aligned size zeroed in the page cache (>= valid_size) */
 	loff_t zeroed_size;
 	/* hash by i_location */
 	struct hlist_node i_hash_fat;
diff --git a/fs/exfat/file.c b/fs/exfat/file.c
index 5fc13378d35f7..637d1cfd7903d 100644
--- a/fs/exfat/file.c
+++ b/fs/exfat/file.c
@@ -16,6 +16,7 @@
 #include <linux/falloc.h>
 #include <linux/fileattr.h>
 #include <linux/iomap.h>
+#include <linux/pagemap.h>
 
 #include "exfat_raw.h"
 #include "exfat_fs.h"
@@ -654,6 +655,122 @@ int exfat_file_fsync(struct file *filp, loff_t start, loff_t end, int datasync)
 	return blkdev_issue_flush(inode->i_sb->s_bdev);
 }
 
+/*
+ * exfat_zero_new_range - fill a page-cache gap without clobbering live data
+ *
+ * Used to zero the gap [start, end) below a freshly extended valid_size, both
+ * from the buffered write path (exfat_extend_valid_size()) and from a mmap
+ * write fault (exfat_page_mkwrite()).
+ *
+ * iomap_zero_range() zeroes every block of every folio it touches.  That would
+ * clobber data a racing writer has stored through a shared mapping into a block
+ * past the byte-granular valid_size but below the block-aligned zeroed_size.
+ * So zero with block granularity instead.  Leave uptodate blocks (which hold
+ * the live data, or zeroes for a never-written gap page) intact and only zero
+ * the not-uptodate blocks, which are stale on disk.
+ */
+static int exfat_zero_new_range(struct inode *inode, loff_t start, loff_t end)
+{
+	struct address_space *mapping = inode->i_mapping;
+	unsigned int blocksize = i_blocksize(inode);
+	loff_t pos = start;
+	int err;
+
+	while (pos < end) {
+		loff_t next = min_t(loff_t,
+				round_down(pos, PAGE_SIZE) + PAGE_SIZE, end);
+		struct folio *folio;
+		loff_t bpos;
+
+		folio = filemap_get_folio(mapping, pos >> PAGE_SHIFT);
+		if (IS_ERR(folio)) {
+			err = iomap_zero_range(inode, pos, next - pos, NULL,
+					       &exfat_iomap_ops, NULL, NULL);
+			if (err < 0)
+				return err;
+			pos = next;
+			continue;
+		}
+
+		if (folio_test_uptodate(folio)) {
+			folio_lock(folio);
+			if (folio->mapping == mapping)
+				folio_mark_dirty(folio);
+			folio_unlock(folio);
+			folio_put(folio);
+			pos = next;
+			continue;
+		}
+
+		/*
+		 * Partially uptodate: walk the gap block by block under the
+		 * folio lock and zero only maximal runs of not-uptodate blocks.
+		 * iomap_zero_range() needs the folio unlocked, so drop the lock
+		 * around each run and re-check ->mapping in case the folio was
+		 * reclaimed in between (in which case zero the remainder from
+		 * the on-disk view).
+		 */
+		folio_lock(folio);
+		bpos = pos;
+		while (bpos < next) {
+			loff_t rstart, rend;
+
+			if (folio->mapping != mapping) {
+				folio_unlock(folio);
+				err = iomap_zero_range(inode, bpos, next - bpos,
+						NULL, &exfat_iomap_ops, NULL, NULL);
+				if (err < 0) {
+					folio_put(folio);
+					return err;
+				}
+				folio_lock(folio);
+				break;
+			}
+
+			if (iomap_is_partially_uptodate(folio,
+					offset_in_folio(folio, bpos), blocksize)) {
+				bpos += blocksize;
+				continue;
+			}
+
+			rstart = bpos;
+			rend = min_t(loff_t, bpos + blocksize, next);
+			while (rend < next &&
+			       !iomap_is_partially_uptodate(folio,
+					offset_in_folio(folio, rend), blocksize))
+				rend = min_t(loff_t, rend + blocksize, next);
+
+			folio_unlock(folio);
+			err = iomap_zero_range(inode, rstart, rend - rstart,
+					NULL, &exfat_iomap_ops, NULL, NULL);
+			if (err < 0) {
+				folio_put(folio);
+				return err;
+			}
+			folio_lock(folio);
+			bpos = rend;
+		}
+
+		/*
+		 * Dirty the folio only if every block is now uptodate. The
+		 * zeroed gap blocks and any preserved (mmap-written) blocks are
+		 * uptodate; if other blocks of this boundary folio - below
+		 * gap_start, holding valid on-disk data that is not cached - are
+		 * still not uptodate, a whole-folio dirty would make writeback
+		 * clobber that on-disk data with uninitialised page content. A
+		 * preserved marker block is already dirty from the store, so it
+		 * is written back regardless.
+		 */
+		if (folio->mapping == mapping && folio_test_uptodate(folio))
+			folio_mark_dirty(folio);
+		folio_unlock(folio);
+		folio_put(folio);
+		pos = next;
+	}
+
+	return 0;
+}
+
 static int exfat_extend_valid_size(struct inode *inode, loff_t new_valid_size)
 {
 	struct exfat_inode_info *ei = EXFAT_I(inode);
@@ -661,18 +778,56 @@ static int exfat_extend_valid_size(struct inode *inode, loff_t new_valid_size)
 	int ret = 0;
 
 	if (old_valid_size < new_valid_size) {
+		/*
+		 * Skip the part of [old_valid_size, new_valid_size) that has
+		 * already been zeroed, tracked block-granularly by zeroed_size.
+		 * Re-zeroing it could clobber a marker that an mmap store wrote
+		 * into a block past the byte-granular valid_size but below the
+		 * block-aligned zeroed_size.
+		 */
+		loff_t gap_start = max(old_valid_size, ei->zeroed_size);
+
 		if (i_size_read(inode) < new_valid_size) {
-			i_size_write(inode, new_valid_size);
-			mark_inode_dirty(inode);
+			/*
+			 * The write extends the file past i_size. Allocate the
+			 * clusters up to new_valid_size before zeroing the gap;
+			 * otherwise the gap fill can be skipped where zeroed_size
+			 * already covers the range, leaving i_size beyond the last
+			 * allocated cluster and a later mapping hitting EOF.
+			 */
+			ret = exfat_cont_expand(inode, new_valid_size);
+			if (ret)
+				return ret;
 		}
 
-		ret = iomap_zero_range(inode, old_valid_size,
-				new_valid_size - old_valid_size, NULL,
-				&exfat_write_iomap_ops, NULL, NULL);
+		/*
+		 * Serialize the gap fill against concurrent mmap stores. A
+		 * shared writable mapping can hold a writable PTE for a gap page
+		 * and store into it with no fault (and so no valid_size advance);
+		 * such a store could land in the range we are about to zero and
+		 * be lost. Hold the invalidate lock (which exfat_page_mkwrite()
+		 * takes shared) and revoke the gap's PTEs, so any further store
+		 * re-faults through exfat_page_mkwrite(). As we hold the inode
+		 * lock, that fault blocks until the gap is zeroed and valid_size
+		 * covers it; the re-faulted store then lands in an already-valid
+		 * page and is preserved instead of re-zeroed.
+		 */
+		filemap_invalidate_lock(inode->i_mapping);
+		if (gap_start < new_valid_size)
+			unmap_mapping_range(inode->i_mapping, gap_start,
+					new_valid_size - gap_start, 0);
+		ret = exfat_zero_new_range(inode, gap_start, new_valid_size);
+		filemap_invalidate_unlock(inode->i_mapping);
 		if (ret) {
 			truncate_setsize(inode, old_valid_size);
 			exfat_truncate(inode);
+			return ret;
 		}
+
+		ei->valid_size = new_valid_size;
+		if (ei->zeroed_size < round_up(new_valid_size, i_blocksize(inode)))
+			ei->zeroed_size = round_up(new_valid_size, i_blocksize(inode));
+		mark_inode_dirty(inode);
 	}
 
 	return ret;
@@ -825,39 +980,52 @@ static vm_fault_t exfat_page_mkwrite(struct vm_fault *vmf)
 	struct inode *inode = file_inode(vmf->vma->vm_file);
 	struct exfat_inode_info *ei = EXFAT_I(inode);
 	vm_fault_t ret;
-	loff_t new_valid_size, mmap_valid_size;
+	loff_t new_valid_size, mmap_valid_size, fault_page_start;
 
 	if (!inode_trylock(inode))
 		return VM_FAULT_RETRY;
 
 	mmap_valid_size = ((loff_t)vmf->pgoff + 1) << PAGE_SHIFT;
+	fault_page_start = ((loff_t)vmf->pgoff) << PAGE_SHIFT;
 	new_valid_size = min(mmap_valid_size, i_size_read(inode));
 
 	if (ei->valid_size < new_valid_size) {
-		if (ei->zeroed_size < mmap_valid_size) {
+		if (ei->zeroed_size < fault_page_start) {
 			int err;
 
 			/*
-			 * Only zero the range that hasn't been zeroed yet for
-			 * this mmap write path. zeroed_size tracks the largest
+			 * Only physically zero the gap pages *below* the
+			 * faulting page. zeroed_size tracks the largest
 			 * page-aligned offset that has already been zeroed.
 			 *
-			 * This prevents unnecessarily zeroing out the entire
-			 * tail page on every page fault when userspace writes
-			 * data byte-by-byte through mmap (after a small
-			 * fallocate). It fixes data corruption in the tail page
-			 * while preserving the existing valid_size semantics.
+			 * The faulting folio itself is not zeroed here: it is
+			 * already populated by the read fault (real data below
+			 * valid_size, zeroes above it) and is dirtied by
+			 * iomap_page_mkwrite() below, so writeback persists it
+			 * correctly. Zeroing it here would instead risk
+			 * clobbering data that userspace is about to write
+			 * through the mapping, which is what corrupts the tail
+			 * page when the mmap_prepare-time extend is dropped.
 			 */
-			err = iomap_zero_range(inode, ei->zeroed_size,
-					mmap_valid_size - ei->zeroed_size, NULL,
-					&exfat_iomap_ops, NULL, NULL);
+			err = exfat_zero_new_range(inode, ei->zeroed_size,
+					fault_page_start);
 			if (err < 0) {
 				inode_unlock(inode);
 				return vmf_fs_error(err);
 			}
-			ei->zeroed_size = mmap_valid_size;
 		}
 
+		/*
+		 * Advance zeroed_size to the block boundary of new_valid_size,
+		 * which is clamped to i_size. The block that straddles i_size is
+		 * written back in full with its tail zeroed, so recording it as
+		 * zeroed is safe. Rounding up to the page boundary instead would
+		 * cover blocks entirely beyond i_size that writeback never
+		 * persists (writeback stops at i_size), letting a later
+		 * valid_size extension skip them and expose stale on-disk data.
+		 */
+		if (ei->zeroed_size < round_up(new_valid_size, i_blocksize(inode)))
+			ei->zeroed_size = round_up(new_valid_size, i_blocksize(inode));
 		ei->valid_size = new_valid_size;
 		mark_inode_dirty(inode);
 	}
@@ -866,7 +1034,7 @@ static vm_fault_t exfat_page_mkwrite(struct vm_fault *vmf)
 	file_update_time(vmf->vma->vm_file);
 
 	filemap_invalidate_lock_shared(inode->i_mapping);
-	ret = iomap_page_mkwrite(vmf, &exfat_write_iomap_ops, NULL);
+	ret = iomap_page_mkwrite(vmf, &exfat_iomap_ops, NULL);
 	filemap_invalidate_unlock_shared(inode->i_mapping);
 	sb_end_pagefault(inode->i_sb);
 	inode_unlock(inode);
@@ -876,7 +1044,6 @@ static vm_fault_t exfat_page_mkwrite(struct vm_fault *vmf)
 
 static const struct vm_operations_struct exfat_file_vm_ops = {
 	.fault		= filemap_fault,
-	.map_pages	= filemap_map_pages,
 	.page_mkwrite	= exfat_page_mkwrite,
 };
 
@@ -887,21 +1054,6 @@ static int exfat_file_mmap_prepare(struct vm_area_desc *desc)
 	if (unlikely(exfat_forced_shutdown(file_inode(desc->file)->i_sb)))
 		return -EIO;
 
-	if (vma_desc_test_all(desc, VMA_SHARED_BIT, VMA_MAYWRITE_BIT)) {
-		struct inode *inode = file_inode(file);
-		loff_t from, to;
-		int err;
-
-		from = ((loff_t)desc->pgoff << PAGE_SHIFT);
-		to = min_t(loff_t, i_size_read(inode),
-				from + vma_desc_size(desc));
-		if (EXFAT_I(inode)->valid_size < to) {
-			err = exfat_extend_valid_size(inode, to);
-			if (err)
-				return err;
-		}
-	}
-
 	file_accessed(file);
 	desc->vm_ops = &exfat_file_vm_ops;
 	return 0;
diff --git a/fs/exfat/iomap.c b/fs/exfat/iomap.c
index 190fc6471f849..a847baa7e1473 100644
--- a/fs/exfat/iomap.c
+++ b/fs/exfat/iomap.c
@@ -175,11 +175,29 @@ static int exfat_write_iomap_end(struct inode *inode, loff_t pos, loff_t length,
 
 	if (ei->valid_size < end) {
 		ei->valid_size = end;
-		if (ei->zeroed_size < end)
-			ei->zeroed_size = end;
 		dirtied = true;
 	}
 
+	/*
+	 * If the tail of the last block was zeroed (IOMAP_F_ZERO_TAIL),
+	 * that block is now fully valid (written data followed by zeroes)
+	 * and has been dirtied, so it will be written back as zeroes. Record
+	 * it as zeroed up to the block boundary.
+	 *
+	 * This mirrors the block-granular valid_size advance that the
+	 * pre-iomap exfat_get_block() performed. Without it, the sub-block
+	 * tail between the byte-granular valid_size and the block boundary is
+	 * treated as not-yet-zeroed, and a later valid_size extension
+	 * (exfat_extend_valid_size()) or mmap gap fill would re-zero it -
+	 * clobbering a marker that an mmap store wrote into that tail through
+	 * a fault-around writable PTE (no page_mkwrite, so valid_size was
+	 * never advanced to cover it).
+	 */
+	if (iomap->flags & IOMAP_F_ZERO_TAIL)
+		end = round_up(end, i_blocksize(inode));
+	if (ei->zeroed_size < end)
+		ei->zeroed_size = end;
+
 	if (dirtied || iomap->flags & IOMAP_F_SIZE_CHANGED)
 		mark_inode_dirty(inode);
 
diff --git a/fs/exfat/namei.c b/fs/exfat/namei.c
index b7d5e44ad38ee..cd9c9eca58f86 100644
--- a/fs/exfat/namei.c
+++ b/fs/exfat/namei.c
@@ -538,7 +538,7 @@ static int exfat_add_entry(struct inode *inode, const char *path,
 }
 
 static int exfat_create(struct mnt_idmap *idmap, struct inode *dir,
-			struct dentry *dentry, umode_t mode, bool excl)
+			struct dentry *dentry, umode_t mode)
 {
 	struct super_block *sb = dir->i_sb;
 	struct inode *inode;
diff --git a/fs/ext2/namei.c b/fs/ext2/namei.c
index 0d09d22fe7086..8666233ec63bc 100644
--- a/fs/ext2/namei.c
+++ b/fs/ext2/namei.c
@@ -99,7 +99,7 @@ struct dentry *ext2_get_parent(struct dentry *child)
  */
 static int ext2_create (struct mnt_idmap * idmap,
 			struct inode * dir, struct dentry * dentry,
-			umode_t mode, bool excl)
+			umode_t mode)
 {
 	struct inode *inode;
 	int err;
@@ -236,7 +236,7 @@ static struct dentry *ext2_mkdir(struct mnt_idmap * idmap,
 
 	inode_inc_link_count(dir);
 
-	inode = ext2_new_inode(dir, S_IFDIR | mode, &dentry->d_name);
+	inode = ext2_new_inode(dir, mode, &dentry->d_name);
 	err = PTR_ERR(inode);
 	if (IS_ERR(inode))
 		goto out_dir;
diff --git a/fs/ext4/extents-test.c b/fs/ext4/extents-test.c
index bd7795a82607d..c3836ecb89f9c 100644
--- a/fs/ext4/extents-test.c
+++ b/fs/ext4/extents-test.c
@@ -126,11 +126,6 @@ struct kunit_ext_test_param {
 	struct kunit_ext_data_state exp_data_state[3];
 };
 
-static void ext_kill_sb(struct super_block *sb)
-{
-	generic_shutdown_super(sb);
-}
-
 static int ext_init_fs_context(struct fs_context *fc)
 {
 	return 0;
@@ -138,13 +133,13 @@ static int ext_init_fs_context(struct fs_context *fc)
 
 static int ext_set(struct super_block *sb, struct fs_context *fc)
 {
-	return 0;
+	return set_anon_super_fc(sb, fc);
 }
 
 static struct file_system_type ext_fs_type = {
 	.name		 = "extents test",
 	.init_fs_context = ext_init_fs_context,
-	.kill_sb	 = ext_kill_sb,
+	.kill_sb	 = kill_anon_super,
 };
 
 static void extents_kunit_exit(struct kunit *test)
diff --git a/fs/ext4/mballoc-test.c b/fs/ext4/mballoc-test.c
index 0424b8b0b4c3c..d31780075c21e 100644
--- a/fs/ext4/mballoc-test.c
+++ b/fs/ext4/mballoc-test.c
@@ -59,11 +59,6 @@ static const struct super_operations mbt_sops = {
 	.free_inode	= mbt_free_inode,
 };
 
-static void mbt_kill_sb(struct super_block *sb)
-{
-	generic_shutdown_super(sb);
-}
-
 static int mbt_init_fs_context(struct fs_context *fc)
 {
 	return 0;
@@ -72,7 +67,7 @@ static int mbt_init_fs_context(struct fs_context *fc)
 static struct file_system_type mbt_fs_type = {
 	.name			= "mballoc test",
 	.init_fs_context	= mbt_init_fs_context,
-	.kill_sb		= mbt_kill_sb,
+	.kill_sb		= kill_anon_super,
 };
 
 static int mbt_mb_init(struct super_block *sb)
@@ -136,7 +131,7 @@ static void mbt_mb_release(struct super_block *sb)
 
 static int mbt_set(struct super_block *sb, struct fs_context *fc)
 {
-	return 0;
+	return set_anon_super_fc(sb, fc);
 }
 
 static struct super_block *mbt_ext4_alloc_super_block(void)
diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c
index cc49ae04a6f64..640a03ee02c76 100644
--- a/fs/ext4/namei.c
+++ b/fs/ext4/namei.c
@@ -2811,7 +2811,7 @@ static int ext4_add_nondir(handle_t *handle,
  * with d_instantiate().
  */
 static int ext4_create(struct mnt_idmap *idmap, struct inode *dir,
-		       struct dentry *dentry, umode_t mode, bool excl)
+		       struct dentry *dentry, umode_t mode)
 {
 	handle_t *handle;
 	struct inode *inode;
@@ -3009,7 +3009,7 @@ static struct dentry *ext4_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	credits = (EXT4_DATA_TRANS_BLOCKS(dir->i_sb) +
 		   EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3);
 retry:
-	inode = ext4_new_inode_start_handle(idmap, dir, S_IFDIR | mode,
+	inode = ext4_new_inode_start_handle(idmap, dir, mode,
 					    &dentry->d_name,
 					    0, NULL, EXT4_HT_DIR, credits);
 	handle = ext4_journal_current_handle();
diff --git a/fs/ext4/super.c b/fs/ext4/super.c
index 245f67d10ded3..467aa44109bc5 100644
--- a/fs/ext4/super.c
+++ b/fs/ext4/super.c
@@ -5797,7 +5797,7 @@ failed_mount8: __maybe_unused
 	brelse(sbi->s_sbh);
 	if (sbi->s_journal_bdev_file) {
 		invalidate_bdev(file_bdev(sbi->s_journal_bdev_file));
-		bdev_fput(sbi->s_journal_bdev_file);
+		fs_bdev_file_release(sbi->s_journal_bdev_file, sb);
 	}
 out_fail:
 	invalidate_bdev(sb->s_bdev);
@@ -5981,9 +5981,9 @@ static struct file *ext4_get_journal_blkdev(struct super_block *sb,
 	struct ext4_super_block *es;
 	int errno;
 
-	bdev_file = bdev_file_open_by_dev(j_dev,
+	bdev_file = fs_bdev_file_open_by_dev(j_dev,
 		BLK_OPEN_READ | BLK_OPEN_WRITE | BLK_OPEN_RESTRICT_WRITES,
-		sb, &fs_holder_ops);
+		sb, sb);
 	if (IS_ERR(bdev_file)) {
 		ext4_msg(sb, KERN_ERR,
 			 "failed to open journal device unknown-block(%u,%u) %pe",
@@ -6043,7 +6043,7 @@ static struct file *ext4_get_journal_blkdev(struct super_block *sb,
 out_bh:
 	brelse(bh);
 out_bdev:
-	bdev_fput(bdev_file);
+	fs_bdev_file_release(bdev_file, sb);
 	return ERR_PTR(errno);
 }
 
@@ -6082,7 +6082,7 @@ static journal_t *ext4_open_dev_journal(struct super_block *sb,
 out_journal:
 	ext4_journal_destroy(EXT4_SB(sb), journal);
 out_bdev:
-	bdev_fput(bdev_file);
+	fs_bdev_file_release(bdev_file, sb);
 	return ERR_PTR(errno);
 }
 
@@ -7499,7 +7499,7 @@ static void ext4_kill_sb(struct super_block *sb)
 	kill_block_super(sb);
 
 	if (bdev_file)
-		bdev_fput(bdev_file);
+		fs_bdev_file_release(bdev_file, sb);
 }
 
 static struct file_system_type ext4_fs_type = {
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index a765fda715363..042ed8ad9cc3e 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -1323,10 +1323,11 @@ struct folio *f2fs_get_read_data_folio(struct inode *inode, pgoff_t index,
 
 	if (folio_test_large(folio)) {
 		pgoff_t folio_index = mapping_align_index(mapping, index);
+		unsigned long nr_pages = folio_nr_pages(folio);
 
 		f2fs_folio_put(folio, true);
 		invalidate_inode_pages2_range(mapping, folio_index,
-				folio_index + folio_nr_pages(folio) - 1);
+				folio_index + nr_pages - 1);
 		f2fs_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT);
 		goto retry;
 	}
@@ -3975,7 +3976,7 @@ static int f2fs_write_begin(const struct kiocb *iocb,
 	 * Will wait that below with our IO control.
 	 */
 	folio = f2fs_filemap_get_folio(mapping, index,
-				FGP_LOCK | FGP_WRITE | FGP_CREAT | FGP_NOFS,
+				FGP_LOCK | FGP_WRITE | FGP_CREAT,
 				mapping_gfp_mask(mapping));
 	if (IS_ERR(folio)) {
 		err = PTR_ERR(folio);
diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c
index af88db8fdb711..ff379aff44726 100644
--- a/fs/f2fs/debug.c
+++ b/fs/f2fs/debug.c
@@ -352,10 +352,6 @@ static void update_mem_info(struct f2fs_sb_info *sbi)
 get_cache:
 	si->cache_mem = 0;
 
-	/* build gc */
-	if (sbi->gc_thread)
-		si->cache_mem += sizeof(struct f2fs_gc_kthread);
-
 	/* build merge flush thread */
 	if (SM_I(sbi)->fcc_info)
 		si->cache_mem += sizeof(struct flush_cmd_control);
diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
index 8f3e632f315cc..8011bbdf2c688 100644
--- a/fs/f2fs/f2fs.h
+++ b/fs/f2fs/f2fs.h
@@ -1748,6 +1748,33 @@ struct decompress_io_ctx {
 #define MAX_COMPRESS_LOG_SIZE		8
 #define MAX_COMPRESS_WINDOW_SIZE(log_size)	((PAGE_SIZE) << (log_size))
 
+struct f2fs_gc_kthread {
+	struct task_struct *f2fs_gc_task;
+	wait_queue_head_t gc_wait_queue_head;
+
+	/* for gc sleep time */
+	unsigned int urgent_sleep_time;
+	unsigned int min_sleep_time;
+	unsigned int max_sleep_time;
+	unsigned int no_gc_sleep_time;
+
+	/* for changing gc mode */
+	bool gc_wake;
+
+	/* for GC_MERGE mount option */
+	wait_queue_head_t fggc_wq;		/*
+						 * caller of f2fs_balance_fs()
+						 * will wait on this wait queue.
+						 */
+
+	/* for gc control for zoned devices */
+	unsigned int no_zoned_gc_percent;
+	unsigned int boost_zoned_gc_percent;
+	unsigned int valid_thresh_ratio;
+	unsigned int boost_gc_multiple;
+	unsigned int boost_gc_greedy;
+};
+
 struct f2fs_sb_info {
 	struct super_block *sb;			/* pointer to VFS super block */
 	struct proc_dir_entry *s_proc;		/* proc entry */
@@ -1883,7 +1910,7 @@ struct f2fs_sb_info {
 						 * semaphore for GC, avoid
 						 * race between GC and GC or CP
 						 */
-	struct f2fs_gc_kthread	*gc_thread;	/* GC thread */
+	struct f2fs_gc_kthread gc_thread;	/* GC thread */
 	struct atgc_management am;		/* atgc management */
 	unsigned int cur_victim_sec;		/* current victim section num */
 	unsigned int gc_mode;			/* current GC state */
diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
index 4b52c56d71f07..c54897a25981a 100644
--- a/fs/f2fs/file.c
+++ b/fs/f2fs/file.c
@@ -1107,17 +1107,23 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 			!IS_ALIGNED(attr->ia_size,
 			F2FS_BLK_TO_BYTES(fi->i_cluster_size)))
 			return -EINVAL;
-		/*
-		 * To prevent scattered pin block generation, we don't allow
-		 * smaller/equal size unaligned truncation for pinned file.
-		 * We only support overwrite IO to pinned file, so don't
-		 * care about larger size truncation.
-		 */
-		if (f2fs_is_pinned_file(inode) &&
-			attr->ia_size <= i_size_read(inode) &&
-			!IS_ALIGNED(attr->ia_size,
-			F2FS_BLK_TO_BYTES(CAP_BLKS_PER_SEC(sbi))))
-			return -EINVAL;
+
+		if (f2fs_is_pinned_file(inode)) {
+			/*
+			 * It may break section-aligned fallocate recovery
+			 * mechanism, so do not allow larger size truncation.
+			 */
+			if (attr->ia_size > i_size_read(inode))
+				return -EINVAL;
+			/*
+			 * To prevent scattered pin block generation, we don't
+			 * allow smaller/equal size unaligned truncation for
+			 * pinned file.
+			 */
+			else if (!IS_ALIGNED(attr->ia_size,
+				F2FS_BLK_TO_BYTES(CAP_BLKS_PER_SEC(sbi))))
+				return -EINVAL;
+		}
 	}
 
 	if (is_quota_modification(idmap, inode, attr)) {
diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c
index ffaa7ba76a1bd..d04633f872ef3 100644
--- a/fs/f2fs/gc.c
+++ b/fs/f2fs/gc.c
@@ -31,9 +31,9 @@ static unsigned int count_bits(const unsigned long *addr,
 static int gc_thread_func(void *data)
 {
 	struct f2fs_sb_info *sbi = data;
-	struct f2fs_gc_kthread *gc_th = sbi->gc_thread;
-	wait_queue_head_t *wq = &sbi->gc_thread->gc_wait_queue_head;
-	wait_queue_head_t *fggc_wq = &sbi->gc_thread->fggc_wq;
+	struct f2fs_gc_kthread *gc_th = &sbi->gc_thread;
+	wait_queue_head_t *wq = &sbi->gc_thread.gc_wait_queue_head;
+	wait_queue_head_t *fggc_wq = &sbi->gc_thread.fggc_wq;
 	unsigned int wait_ms;
 	struct f2fs_gc_control gc_control = {
 		.victim_segno = NULL_SEGNO,
@@ -193,13 +193,9 @@ static int gc_thread_func(void *data)
 
 int f2fs_start_gc_thread(struct f2fs_sb_info *sbi)
 {
-	struct f2fs_gc_kthread *gc_th;
+	struct f2fs_gc_kthread *gc_th = &sbi->gc_thread;
 	dev_t dev = sbi->sb->s_bdev->bd_dev;
 
-	gc_th = f2fs_kmalloc(sbi, sizeof(struct f2fs_gc_kthread), GFP_KERNEL);
-	if (!gc_th)
-		return -ENOMEM;
-
 	gc_th->urgent_sleep_time = DEF_GC_THREAD_URGENT_SLEEP_TIME;
 	gc_th->valid_thresh_ratio = DEF_GC_THREAD_VALID_THRESH_RATIO;
 	gc_th->boost_gc_multiple = BOOST_GC_MULTIPLE;
@@ -221,16 +217,14 @@ int f2fs_start_gc_thread(struct f2fs_sb_info *sbi)
 
 	gc_th->gc_wake = false;
 
-	sbi->gc_thread = gc_th;
-	init_waitqueue_head(&sbi->gc_thread->gc_wait_queue_head);
-	init_waitqueue_head(&sbi->gc_thread->fggc_wq);
-	sbi->gc_thread->f2fs_gc_task = kthread_run(gc_thread_func, sbi,
+	init_waitqueue_head(&gc_th->gc_wait_queue_head);
+	init_waitqueue_head(&gc_th->fggc_wq);
+	gc_th->f2fs_gc_task = kthread_run(gc_thread_func, sbi,
 			"f2fs_gc-%u:%u", MAJOR(dev), MINOR(dev));
 	if (IS_ERR(gc_th->f2fs_gc_task)) {
 		int err = PTR_ERR(gc_th->f2fs_gc_task);
 
-		kfree(gc_th);
-		sbi->gc_thread = NULL;
+		gc_th->f2fs_gc_task = NULL;
 		return err;
 	}
 
@@ -241,14 +235,14 @@ int f2fs_start_gc_thread(struct f2fs_sb_info *sbi)
 
 void f2fs_stop_gc_thread(struct f2fs_sb_info *sbi)
 {
-	struct f2fs_gc_kthread *gc_th = sbi->gc_thread;
+	struct f2fs_gc_kthread *gc_th = &sbi->gc_thread;
 
-	if (!gc_th)
+	if (!gc_th->f2fs_gc_task)
 		return;
+
 	kthread_stop(gc_th->f2fs_gc_task);
+	gc_th->f2fs_gc_task = NULL;
 	wake_up_all(&gc_th->fggc_wq);
-	kfree(gc_th);
-	sbi->gc_thread = NULL;
 }
 
 static int select_gc_type(struct f2fs_sb_info *sbi, int gc_type)
@@ -796,7 +790,7 @@ int f2fs_get_victim(struct f2fs_sb_info *sbi, unsigned int *result,
 	if (one_time) {
 		p.one_time_gc = one_time;
 		if (has_enough_free_secs(sbi, 0, NR_PERSISTENT_LOG))
-			valid_thresh_ratio = sbi->gc_thread->valid_thresh_ratio;
+			valid_thresh_ratio = sbi->gc_thread.valid_thresh_ratio;
 	}
 
 retry:
@@ -1807,9 +1801,9 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi,
 
 			if (f2fs_sb_has_blkzoned(sbi) &&
 					!has_enough_free_blocks(sbi,
-					sbi->gc_thread->boost_zoned_gc_percent))
+					sbi->gc_thread.boost_zoned_gc_percent))
 				window_granularity *=
-					sbi->gc_thread->boost_gc_multiple;
+					sbi->gc_thread.boost_gc_multiple;
 
 			end_segno = start_segno + window_granularity;
 		}
diff --git a/fs/f2fs/gc.h b/fs/f2fs/gc.h
index 6c4d4567571eb..b015742fb455b 100644
--- a/fs/f2fs/gc.h
+++ b/fs/f2fs/gc.h
@@ -45,32 +45,7 @@
 
 #define NR_GC_CHECKPOINT_SECS (3)	/* data/node/dentry sections */
 
-struct f2fs_gc_kthread {
-	struct task_struct *f2fs_gc_task;
-	wait_queue_head_t gc_wait_queue_head;
-
-	/* for gc sleep time */
-	unsigned int urgent_sleep_time;
-	unsigned int min_sleep_time;
-	unsigned int max_sleep_time;
-	unsigned int no_gc_sleep_time;
-
-	/* for changing gc mode */
-	bool gc_wake;
-
-	/* for GC_MERGE mount option */
-	wait_queue_head_t fggc_wq;		/*
-						 * caller of f2fs_balance_fs()
-						 * will wait on this wait queue.
-						 */
 
-	/* for gc control for zoned devices */
-	unsigned int no_zoned_gc_percent;
-	unsigned int boost_zoned_gc_percent;
-	unsigned int valid_thresh_ratio;
-	unsigned int boost_gc_multiple;
-	unsigned int boost_gc_greedy;
-};
 
 struct gc_inode_list {
 	struct list_head ilist;
@@ -197,6 +172,6 @@ static inline bool need_to_boost_gc(struct f2fs_sb_info *sbi)
 {
 	if (f2fs_sb_has_blkzoned(sbi))
 		return !has_enough_free_blocks(sbi,
-				sbi->gc_thread->boost_zoned_gc_percent);
+				sbi->gc_thread.boost_zoned_gc_percent);
 	return has_enough_invalid_blocks(sbi);
 }
diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
index cac03b8e91a15..5ae647a352aab 100644
--- a/fs/f2fs/namei.c
+++ b/fs/f2fs/namei.c
@@ -366,7 +366,7 @@ static struct inode *f2fs_new_inode(struct mnt_idmap *idmap,
 }
 
 static int f2fs_create(struct mnt_idmap *idmap, struct inode *dir,
-		       struct dentry *dentry, umode_t mode, bool excl)
+		       struct dentry *dentry, umode_t mode)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
 	struct f2fs_lock_context lc;
@@ -742,7 +742,7 @@ static struct dentry *f2fs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	if (err)
 		return ERR_PTR(err);
 
-	inode = f2fs_new_inode(idmap, dir, S_IFDIR | mode, NULL);
+	inode = f2fs_new_inode(idmap, dir, mode, NULL);
 	if (IS_ERR(inode))
 		return ERR_CAST(inode);
 
diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c
index d71ddb3ee9184..0b706568b0343 100644
--- a/fs/f2fs/segment.c
+++ b/fs/f2fs/segment.c
@@ -452,15 +452,14 @@ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
 	f2fs_submit_merged_write(sbi, DATA);
 	f2fs_submit_all_merged_ipu_writes(sbi);
 
-	if (test_opt(sbi, GC_MERGE) && sbi->gc_thread &&
-				sbi->gc_thread->f2fs_gc_task) {
+	if (test_opt(sbi, GC_MERGE) && sbi->gc_thread.f2fs_gc_task) {
 		DEFINE_WAIT(wait);
 
-		prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait,
+		prepare_to_wait(&sbi->gc_thread.fggc_wq, &wait,
 					TASK_UNINTERRUPTIBLE);
-		wake_up(&sbi->gc_thread->gc_wait_queue_head);
+		wake_up(&sbi->gc_thread.gc_wait_queue_head);
 		io_schedule();
-		finish_wait(&sbi->gc_thread->fggc_wq, &wait);
+		finish_wait(&sbi->gc_thread.fggc_wq, &wait);
 	} else {
 		struct f2fs_gc_control gc_control = {
 			.victim_segno = NULL_SEGNO,
diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
index 2b8d964111562..aa246f8ee79ef 100644
--- a/fs/f2fs/super.c
+++ b/fs/f2fs/super.c
@@ -1971,7 +1971,7 @@ static void destroy_device_list(struct f2fs_sb_info *sbi)
 
 	for (i = 0; i < sbi->s_ndevs; i++) {
 		if (i > 0)
-			bdev_fput(FDEV(i).bdev_file);
+			fs_bdev_file_release(FDEV(i).bdev_file, sbi->sb);
 #ifdef CONFIG_BLK_DEV_ZONED
 		kvfree(FDEV(i).blkz_seq);
 #endif
@@ -2943,11 +2943,11 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb)
 	if ((flags & SB_RDONLY) ||
 			(F2FS_OPTION(sbi).bggc_mode == BGGC_MODE_OFF &&
 			!test_opt(sbi, GC_MERGE))) {
-		if (sbi->gc_thread) {
+		if (sbi->gc_thread.f2fs_gc_task) {
 			f2fs_stop_gc_thread(sbi);
 			need_restart_gc = true;
 		}
-	} else if (!sbi->gc_thread) {
+	} else if (!sbi->gc_thread.f2fs_gc_task) {
 		err = f2fs_start_gc_thread(sbi);
 		if (err)
 			goto restore_opts;
@@ -3168,7 +3168,7 @@ static ssize_t f2fs_quota_read(struct super_block *sb, int type, char *data,
 
 repeat:
 		folio = mapping_read_folio_gfp(mapping, off >> PAGE_SHIFT,
-				GFP_NOFS);
+				GFP_KERNEL);
 		if (IS_ERR(folio)) {
 			if (PTR_ERR(folio) == -ENOMEM) {
 				memalloc_retry_wait(GFP_NOFS);
@@ -4898,8 +4898,8 @@ static int f2fs_scan_devices(struct f2fs_sb_info *sbi)
 				FDEV(i).end_blk = FDEV(i).start_blk +
 						SEGS_TO_BLKS(sbi,
 						FDEV(i).total_segments) - 1;
-				FDEV(i).bdev_file = bdev_file_open_by_path(
-					FDEV(i).path, mode, sbi->sb, NULL);
+				FDEV(i).bdev_file = fs_bdev_file_open_by_path(
+					FDEV(i).path, mode, sbi->sb, sbi->sb);
 			}
 		}
 		if (IS_ERR(FDEV(i).bdev_file))
diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c
index 665687244c939..be92c05a54203 100644
--- a/fs/f2fs/sysfs.c
+++ b/fs/f2fs/sysfs.c
@@ -75,7 +75,7 @@ static ssize_t f2fs_sbi_show(struct f2fs_attr *a,
 static unsigned char *__struct_ptr(struct f2fs_sb_info *sbi, int struct_type)
 {
 	if (struct_type == GC_THREAD)
-		return (unsigned char *)sbi->gc_thread;
+		return (unsigned char *)&sbi->gc_thread;
 	else if (struct_type == SM_INFO)
 		return (unsigned char *)SM_I(sbi);
 	else if (struct_type == DCC_INFO)
@@ -664,20 +664,20 @@ static ssize_t __sbi_store(struct f2fs_attr *a,
 			sbi->gc_mode = GC_NORMAL;
 		} else if (t == 1) {
 			sbi->gc_mode = GC_URGENT_HIGH;
-			if (sbi->gc_thread) {
-				sbi->gc_thread->gc_wake = true;
+			if (sbi->gc_thread.f2fs_gc_task) {
+				sbi->gc_thread.gc_wake = true;
 				wake_up_interruptible_all(
-					&sbi->gc_thread->gc_wait_queue_head);
+					&sbi->gc_thread.gc_wait_queue_head);
 				wake_up_discard_thread(sbi, true);
 			}
 		} else if (t == 2) {
 			sbi->gc_mode = GC_URGENT_LOW;
 		} else if (t == 3) {
 			sbi->gc_mode = GC_URGENT_MID;
-			if (sbi->gc_thread) {
-				sbi->gc_thread->gc_wake = true;
+			if (sbi->gc_thread.f2fs_gc_task) {
+				sbi->gc_thread.gc_wake = true;
 				wake_up_interruptible_all(
-					&sbi->gc_thread->gc_wait_queue_head);
+					&sbi->gc_thread.gc_wait_queue_head);
 			}
 		} else {
 			return -EINVAL;
@@ -934,14 +934,14 @@ static ssize_t __sbi_store(struct f2fs_attr *a,
 	if (!strcmp(a->attr.name, "gc_boost_gc_multiple")) {
 		if (t < 1 || t > SEGS_PER_SEC(sbi))
 			return -EINVAL;
-		sbi->gc_thread->boost_gc_multiple = (unsigned int)t;
+		sbi->gc_thread.boost_gc_multiple = (unsigned int)t;
 		return count;
 	}
 
 	if (!strcmp(a->attr.name, "gc_boost_gc_greedy")) {
 		if (t > GC_GREEDY)
 			return -EINVAL;
-		sbi->gc_thread->boost_gc_greedy = (unsigned int)t;
+		sbi->gc_thread.boost_gc_greedy = (unsigned int)t;
 		return count;
 	}
 
@@ -989,8 +989,8 @@ static ssize_t __sbi_store(struct f2fs_attr *a,
 		if (sbi->cprc_info.f2fs_issue_ckpt)
 			set_user_nice(sbi->cprc_info.f2fs_issue_ckpt,
 					PRIO_TO_NICE(sbi->critical_task_priority));
-		if (sbi->gc_thread && sbi->gc_thread->f2fs_gc_task)
-			set_user_nice(sbi->gc_thread->f2fs_gc_task,
+		if (sbi->gc_thread.f2fs_gc_task)
+			set_user_nice(sbi->gc_thread.f2fs_gc_task,
 					PRIO_TO_NICE(sbi->critical_task_priority));
 		return count;
 	}
diff --git a/fs/fat/fat.h b/fs/fat/fat.h
index 99ed9228a677b..2772675bd35a3 100644
--- a/fs/fat/fat.h
+++ b/fs/fat/fat.h
@@ -249,13 +249,13 @@ static inline unsigned char fat_checksum(const __u8 *name)
 	return s;
 }
 
-static inline sector_t fat_clus_to_blknr(struct msdos_sb_info *sbi, int clus)
+static inline sector_t fat_clus_to_blknr(const struct msdos_sb_info *sbi, int clus)
 {
 	return ((sector_t)clus - FAT_START_ENT) * sbi->sec_per_clus
 		+ sbi->data_start;
 }
 
-static inline void fat_get_blknr_offset(struct msdos_sb_info *sbi,
+static inline void fat_get_blknr_offset(const struct msdos_sb_info *sbi,
 				loff_t i_pos, sector_t *blknr, int *offset)
 {
 	*blknr = i_pos >> sbi->dir_per_block_bits;
diff --git a/fs/fat/fat_test.c b/fs/fat/fat_test.c
index 4eeed9dca5494..9583ce66dca3c 100644
--- a/fs/fat/fat_test.c
+++ b/fs/fat/fat_test.c
@@ -22,7 +22,7 @@ static void fat_checksum_test(struct kunit *test)
 
 static void fat_clus_to_blknr_test(struct kunit *test)
 {
-	struct msdos_sb_info sbi = {
+	static const struct msdos_sb_info sbi = {
 		.sec_per_clus = 4,
 		.data_start = 100,
 	};
@@ -34,7 +34,7 @@ static void fat_clus_to_blknr_test(struct kunit *test)
 
 static void fat_get_blknr_offset_test(struct kunit *test)
 {
-	struct msdos_sb_info sbi = {
+	static const struct msdos_sb_info sbi = {
 		.dir_per_block = 16,
 		.dir_per_block_bits = 4,
 	};
diff --git a/fs/fat/inode.c b/fs/fat/inode.c
index 3aa52481ad5cf..d1e6c9b2f9df8 100644
--- a/fs/fat/inode.c
+++ b/fs/fat/inode.c
@@ -1479,8 +1479,9 @@ static int fat_read_static_bpb(struct super_block *sb,
 	int error = -EINVAL;
 	unsigned i;
 
-	/* 16-bit DOS 1.x reliably wrote bootstrap short-jmp code */
-	if (b->ignored[0] != 0xeb || b->ignored[2] != 0x90) {
+	/* 16-bit DOS 1.x reliably wrote bootstrap short-jmp or near-jmp code */
+	if ((b->ignored[0] != 0xeb || b->ignored[2] != 0x90) &&
+	    (b->ignored[0] != 0xe9)) {
 		if (!silent)
 			fat_msg(sb, KERN_ERR,
 				"%s; no bootstrapping code", notdos1x);
diff --git a/fs/fat/misc.c b/fs/fat/misc.c
index 3027ef53af214..2a0fea26a99a8 100644
--- a/fs/fat/misc.c
+++ b/fs/fat/misc.c
@@ -133,7 +133,11 @@ int fat_chain_add(struct inode *inode, int new_dclus, int nr_cluster)
 		ret = fat_ent_read(inode, &fatent, last);
 		if (ret >= 0) {
 			int wait = inode_needs_sync(inode);
+			int old = ret;
+
 			ret = fat_ent_write(inode, &fatent, new_dclus, wait);
+			if (ret < 0)
+				fat_ent_write(inode, &fatent, old, wait);
 			fatent_brelse(&fatent);
 		}
 		if (ret < 0)
diff --git a/fs/fat/namei_msdos.c b/fs/fat/namei_msdos.c
index 0fd2971ad4b13..ee6824c5d1361 100644
--- a/fs/fat/namei_msdos.c
+++ b/fs/fat/namei_msdos.c
@@ -29,6 +29,9 @@ static int msdos_format_name(const unsigned char *name, int len,
 	unsigned char c;
 	int space;
 
+	if (len > NAME_MAX)
+		return -ENAMETOOLONG;
+
 	if (name[0] == '.') {	/* dotfile because . and .. already done */
 		if (opts->dotsOK) {
 			/* Get rid of dot - test for it elsewhere */
@@ -262,7 +265,7 @@ static int msdos_add_entry(struct inode *dir, const unsigned char *name,
 
 /***** Create a file */
 static int msdos_create(struct mnt_idmap *idmap, struct inode *dir,
-			struct dentry *dentry, umode_t mode, bool excl)
+			struct dentry *dentry, umode_t mode)
 {
 	struct super_block *sb = dir->i_sb;
 	struct inode *inode = NULL;
diff --git a/fs/fat/namei_vfat.c b/fs/fat/namei_vfat.c
index e909447873e36..139d3ef4bfaeb 100644
--- a/fs/fat/namei_vfat.c
+++ b/fs/fat/namei_vfat.c
@@ -755,7 +755,7 @@ static struct dentry *vfat_lookup(struct inode *dir, struct dentry *dentry,
 }
 
 static int vfat_create(struct mnt_idmap *idmap, struct inode *dir,
-		       struct dentry *dentry, umode_t mode, bool excl)
+		       struct dentry *dentry, umode_t mode)
 {
 	struct super_block *sb = dir->i_sb;
 	struct inode *inode;
diff --git a/fs/fat/nfs.c b/fs/fat/nfs.c
index 509eea96a457d..6e1b371711edd 100644
--- a/fs/fat/nfs.c
+++ b/fs/fat/nfs.c
@@ -250,8 +250,10 @@ struct inode *fat_rebuild_parent(struct super_block *sb, int parent_logstart)
 		MSDOS_I(dummy_grand_parent)->i_pos = -1;
 	}
 
-	if (!fat_scan_logstart(dummy_grand_parent, clus_to_match, &sinfo))
+	if (!fat_scan_logstart(dummy_grand_parent, clus_to_match, &sinfo)) {
 		parent = fat_build_inode(sb, sinfo.de, sinfo.i_pos);
+		brelse(sinfo.bh);
+	}
 
 	brelse(parent_bh);
 	iput(dummy_grand_parent);
diff --git a/fs/fs_struct.c b/fs/fs_struct.c
index 394875d06fd60..34699f3b6f881 100644
--- a/fs/fs_struct.c
+++ b/fs/fs_struct.c
@@ -8,6 +8,7 @@
 #include <linux/fs_struct.h>
 #include <linux/init_task.h>
 #include "internal.h"
+#include "mount.h"
 
 /*
  * Replace the fs->{rootmnt,root} with {mnt,dentry}. Put the old values.
@@ -60,8 +61,11 @@ void chroot_fs_refs(const struct path *old_root, const struct path *new_root)
 
 	read_lock(&tasklist_lock);
 	for_each_process_thread(g, p) {
+		if (p->flags & (PF_KTHREAD | PF_EXITING | PF_DUMPCORE))
+			continue;
+
 		task_lock(p);
-		fs = p->fs;
+		fs = p->real_fs;
 		if (fs) {
 			int hits = 0;
 			write_seqlock(&fs->seq);
@@ -89,12 +93,13 @@ void free_fs_struct(struct fs_struct *fs)
 
 void exit_fs(struct task_struct *tsk)
 {
-	struct fs_struct *fs = tsk->fs;
+	struct fs_struct *fs = tsk->real_fs;
 
 	if (fs) {
 		int kill;
 		task_lock(tsk);
 		read_seqlock_excl(&fs->seq);
+		tsk->real_fs = NULL;
 		tsk->fs = NULL;
 		kill = !--fs->users;
 		read_sequnlock_excl(&fs->seq);
@@ -126,7 +131,7 @@ struct fs_struct *copy_fs_struct(struct fs_struct *old)
 
 int unshare_fs_struct(void)
 {
-	struct fs_struct *fs = current->fs;
+	struct fs_struct *fs = current->real_fs;
 	struct fs_struct *new_fs = copy_fs_struct(fs);
 	int kill;
 
@@ -135,8 +140,10 @@ int unshare_fs_struct(void)
 
 	task_lock(current);
 	read_seqlock_excl(&fs->seq);
+	VFS_WARN_ON_ONCE(fs != current->fs);
 	kill = !--fs->users;
 	current->fs = new_fs;
+	current->real_fs = new_fs;
 	read_sequnlock_excl(&fs->seq);
 	task_unlock(current);
 
@@ -147,9 +154,99 @@ int unshare_fs_struct(void)
 }
 EXPORT_SYMBOL_GPL(unshare_fs_struct);
 
+/*
+ * PID 1 may choose to stop sharing fs_struct state with us.
+ * Either via unshare(CLONE_FS) or unshare(CLONE_NEWNS). Of
+ * course, PID 1 could have chosen to create arbitrary process
+ * trees that all share fs_struct state via CLONE_FS. This is a
+ * strong statement: We only care about PID 1 aka the thread-group
+ * leader so subthread's fs_struct state doesn't matter.
+ *
+ * PID 1 unsharing fs_struct state is a bug. PID 1 relies on
+ * various kthreads to be able to perform work based on its
+ * fs_struct state. Breaking that contract sucks for both sides.
+ * So just don't bother with extra work for this. No sane init
+ * system should ever do this.
+ *
+ * On older kernels if PID 1 unshared its filesystem state with us the
+ * kernel simply used the stale fs_struct state implicitly pinning
+ * anything that PID 1 had last used. Even if PID 1 might've moved on to
+ * some completely different fs_struct state and might've even unmounted
+ * the old root.
+ *
+ * This has hilarious consequences: Think continuing to dump coredump
+ * state into an implicitly pinned directory somewhere. Calling random
+ * binaries in the old rootfs via usermodehelpers.
+ *
+ * Be aggressive about this: We simply reject operating on stale
+ * fs_struct state by reverting to nullfs. Every kworker that does
+ * lookups after this point will fail. Every usermodehelper call will
+ * fail. Tough luck but let's be kind and emit a warning to userspace.
+ */
+static inline void validate_fs_switch(struct fs_struct *old_fs)
+{
+	might_sleep();
+
+	if (likely(current->pid != 1))
+		return;
+	/* @old_fs may be dangling but for comparison it's fine */
+	if (old_fs != userspace_init_fs)
+		return;
+	pr_warn("VFS: Pid 1 stopped sharing filesystem state\n");
+	set_fs_root(userspace_init_fs, &init_fs.root);
+	set_fs_pwd(userspace_init_fs, &init_fs.root);
+}
+
+struct fs_struct *switch_fs_struct(struct fs_struct *new_fs)
+{
+	struct fs_struct *fs;
+
+	scoped_guard(task_lock, current) {
+		fs = current->fs;
+		VFS_WARN_ON_ONCE(fs != current->real_fs);
+		read_seqlock_excl(&fs->seq);
+		current->fs = new_fs;
+		current->real_fs = new_fs;
+		if (--fs->users)
+			new_fs = NULL;
+		else
+			new_fs = fs;
+		read_sequnlock_excl(&fs->seq);
+	}
+
+	validate_fs_switch(fs);
+	return new_fs;
+}
+
 /* to be mentioned only in INIT_TASK */
 struct fs_struct init_fs = {
 	.users		= 1,
 	.seq		= __SEQLOCK_UNLOCKED(init_fs.seq),
 	.umask		= 0022,
 };
+
+struct fs_struct *userspace_init_fs __ro_after_init;
+EXPORT_SYMBOL_GPL(userspace_init_fs);
+
+void __init init_userspace_fs(void)
+{
+	struct mount *m;
+	struct path root;
+
+	/* Move PID 1 from nullfs into the initramfs. */
+	m = topmost_overmount(current->nsproxy->mnt_ns->root);
+	root.mnt = &m->mnt;
+	root.dentry = root.mnt->mnt_root;
+
+	VFS_WARN_ON_ONCE(current->pid != 1);
+
+	set_fs_root(current->fs, &root);
+	set_fs_pwd(current->fs, &root);
+
+	/* Hold a reference for the global pointer. */
+	read_seqlock_excl(&current->fs->seq);
+	current->fs->users++;
+	read_sequnlock_excl(&current->fs->seq);
+
+	userspace_init_fs = current->fs;
+}
diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c
index 0e2a1039fa43e..d4e0029810c0f 100644
--- a/fs/fuse/dir.c
+++ b/fs/fuse/dir.c
@@ -1084,7 +1084,7 @@ static int fuse_mknod(struct mnt_idmap *idmap, struct inode *dir,
 }
 
 static int fuse_create(struct mnt_idmap *idmap, struct inode *dir,
-		       struct dentry *entry, umode_t mode, bool excl)
+		       struct dentry *entry, umode_t mode)
 {
 	return fuse_mknod(idmap, dir, entry, mode, 0);
 }
@@ -1117,6 +1117,14 @@ static struct dentry *fuse_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	if (!fm->fc->dont_mask)
 		mode &= ~current_umask();
 
+	/*
+	 * vfs_mkdir() now passes S_IFDIR in @mode, but @mode is forwarded
+	 * verbatim to the userspace server which has only ever been given the
+	 * permission bits. Strip the type bit until the protocol is known to
+	 * cope with it.
+	 */
+	mode &= ~S_IFDIR;
+
 	memset(&inarg, 0, sizeof(inarg));
 	inarg.mode = mode;
 	inarg.umask = current_umask();
diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c
index 8a77794bbd4af..f361876c55833 100644
--- a/fs/gfs2/inode.c
+++ b/fs/gfs2/inode.c
@@ -963,15 +963,14 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
  * @dir: The directory in which to create the file
  * @dentry: The dentry of the new file
  * @mode: The mode of the new file
- * @excl: Force fail if inode exists
  *
  * Returns: errno
  */
 
 static int gfs2_create(struct mnt_idmap *idmap, struct inode *dir,
-		       struct dentry *dentry, umode_t mode, bool excl)
+		       struct dentry *dentry, umode_t mode)
 {
-	return gfs2_create_inode(dir, dentry, NULL, S_IFREG | mode, 0, NULL, 0, excl);
+	return gfs2_create_inode(dir, dentry, NULL, S_IFREG | mode, 0, NULL, 0, 1);
 }
 
 /**
@@ -1351,7 +1350,7 @@ static struct dentry *gfs2_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 {
 	unsigned dsize = gfs2_max_stuffed_size(GFS2_I(dir));
 
-	return ERR_PTR(gfs2_create_inode(dir, dentry, NULL, S_IFDIR | mode, 0, NULL, dsize, 0));
+	return ERR_PTR(gfs2_create_inode(dir, dentry, NULL, mode, 0, NULL, dsize, 0));
 }
 
 /**
diff --git a/fs/hfs/dir.c b/fs/hfs/dir.c
index e13450bb933eb..e1f1fb3514644 100644
--- a/fs/hfs/dir.c
+++ b/fs/hfs/dir.c
@@ -184,7 +184,7 @@ static int hfs_dir_release(struct inode *inode, struct file *file)
  * the directory and the name (and its length) of the new file.
  */
 static int hfs_create(struct mnt_idmap *idmap, struct inode *dir,
-		      struct dentry *dentry, umode_t mode, bool excl)
+		      struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode;
 	int res;
@@ -219,7 +219,7 @@ static struct dentry *hfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	struct inode *inode;
 	int res;
 
-	inode = hfs_new_inode(dir, &dentry->d_name, S_IFDIR | mode);
+	inode = hfs_new_inode(dir, &dentry->d_name, mode);
 	if (IS_ERR(inode))
 		return ERR_CAST(inode);
 
diff --git a/fs/hfsplus/dir.c b/fs/hfsplus/dir.c
index 8bf6c7cdd9a86..51fcba2e6d403 100644
--- a/fs/hfsplus/dir.c
+++ b/fs/hfsplus/dir.c
@@ -562,7 +562,7 @@ static int hfsplus_mknod(struct mnt_idmap *idmap, struct inode *dir,
 }
 
 static int hfsplus_create(struct mnt_idmap *idmap, struct inode *dir,
-			  struct dentry *dentry, umode_t mode, bool excl)
+			  struct dentry *dentry, umode_t mode)
 {
 	return hfsplus_mknod(&nop_mnt_idmap, dir, dentry, mode, 0);
 }
@@ -570,7 +570,7 @@ static int hfsplus_create(struct mnt_idmap *idmap, struct inode *dir,
 static struct dentry *hfsplus_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 				    struct dentry *dentry, umode_t mode)
 {
-	return ERR_PTR(hfsplus_mknod(&nop_mnt_idmap, dir, dentry, mode | S_IFDIR, 0));
+	return ERR_PTR(hfsplus_mknod(&nop_mnt_idmap, dir, dentry, mode, 0));
 }
 
 static int hfsplus_rename(struct mnt_idmap *idmap,
diff --git a/fs/hostfs/hostfs_kern.c b/fs/hostfs/hostfs_kern.c
index abe86d72d9ef7..7add056d47d8f 100644
--- a/fs/hostfs/hostfs_kern.c
+++ b/fs/hostfs/hostfs_kern.c
@@ -593,7 +593,7 @@ static struct inode *hostfs_iget(struct super_block *sb, char *name)
 }
 
 static int hostfs_create(struct mnt_idmap *idmap, struct inode *dir,
-			 struct dentry *dentry, umode_t mode, bool excl)
+			 struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode;
 	char *name;
diff --git a/fs/hpfs/namei.c b/fs/hpfs/namei.c
index 353e13a615f56..9446f4038874c 100644
--- a/fs/hpfs/namei.c
+++ b/fs/hpfs/namei.c
@@ -105,10 +105,10 @@ static struct dentry *hpfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 
 	if (!uid_eq(result->i_uid, current_fsuid()) ||
 	    !gid_eq(result->i_gid, current_fsgid()) ||
-	    result->i_mode != (mode | S_IFDIR)) {
+	    result->i_mode != mode) {
 		result->i_uid = current_fsuid();
 		result->i_gid = current_fsgid();
-		result->i_mode = mode | S_IFDIR;
+		result->i_mode = mode;
 		hpfs_write_inode_nolock(result);
 	}
 	hpfs_update_directory_times(dir);
@@ -129,7 +129,7 @@ static struct dentry *hpfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 }
 
 static int hpfs_create(struct mnt_idmap *idmap, struct inode *dir,
-		       struct dentry *dentry, umode_t mode, bool excl)
+		       struct dentry *dentry, umode_t mode)
 {
 	const unsigned char *name = dentry->d_name.name;
 	unsigned len = dentry->d_name.len;
diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
index 216e1a0dd0b23..fdc8eaaa8cdd8 100644
--- a/fs/hugetlbfs/inode.c
+++ b/fs/hugetlbfs/inode.c
@@ -184,7 +184,8 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
 	if (addr)
 		addr0 = ALIGN(addr, huge_page_size(h));
 
-	return mm_get_unmapped_area_vmflags(file, addr0, len, pgoff, flags, 0);
+	return mm_get_unmapped_area_vmaflags(file, addr0, len, pgoff, flags,
+					     EMPTY_VMA_FLAGS);
 }
 
 /*
@@ -380,7 +381,6 @@ static void hugetlb_unmap_file_folio(struct hstate *h,
 					struct address_space *mapping,
 					struct folio *folio, pgoff_t index)
 {
-	struct rb_root_cached *root = &mapping->i_mmap;
 	struct hugetlb_vma_lock *vma_lock;
 	unsigned long pfn = folio_pfn(folio);
 	struct vm_area_struct *vma;
@@ -394,7 +394,7 @@ static void hugetlb_unmap_file_folio(struct hstate *h,
 	i_mmap_lock_write(mapping);
 retry:
 	vma_lock = NULL;
-	vma_interval_tree_foreach(vma, root, start, end - 1) {
+	mapping_rmap_tree_foreach(vma, mapping, start, end - 1) {
 		v_start = vma_offset_start(vma, start);
 		v_end = vma_offset_end(vma, end);
 
@@ -460,8 +460,8 @@ static void hugetlb_unmap_file_folio(struct hstate *h,
 }
 
 static void
-hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end,
-		      zap_flags_t zap_flags)
+hugetlb_vmdelete_list(struct address_space *mapping, pgoff_t start,
+		      pgoff_t end, zap_flags_t zap_flags)
 {
 	struct vm_area_struct *vma;
 
@@ -470,7 +470,8 @@ hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end,
 	 * unmapped.  Note, end is exclusive, whereas the interval tree takes
 	 * an inclusive "last".
 	 */
-	vma_interval_tree_foreach(vma, root, start, end ? end - 1 : ULONG_MAX) {
+	mapping_rmap_tree_foreach(vma, mapping, start,
+				  end ? end - 1 : ULONG_MAX) {
 		unsigned long v_start;
 		unsigned long v_end;
 
@@ -615,8 +616,7 @@ static void hugetlb_vmtruncate(struct inode *inode, loff_t offset)
 	i_size_write(inode, offset);
 	i_mmap_lock_write(mapping);
 	if (mapping_mapped(mapping))
-		hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0,
-				      ZAP_FLAG_DROP_MARKER);
+		hugetlb_vmdelete_list(mapping, pgoff, 0, ZAP_FLAG_DROP_MARKER);
 	i_mmap_unlock_write(mapping);
 	remove_inode_hugepages(inode, offset, LLONG_MAX);
 }
@@ -676,7 +676,7 @@ static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
 	/* Unmap users of full pages in the hole. */
 	if (hole_end > hole_start) {
 		if (mapping_mapped(mapping))
-			hugetlb_vmdelete_list(&mapping->i_mmap,
+			hugetlb_vmdelete_list(mapping,
 					      hole_start >> PAGE_SHIFT,
 					      hole_end >> PAGE_SHIFT, 0);
 	}
@@ -971,7 +971,7 @@ static struct dentry *hugetlbfs_mkdir(struct mnt_idmap *idmap, struct inode *dir
 				      struct dentry *dentry, umode_t mode)
 {
 	int retval = hugetlbfs_mknod(idmap, dir, dentry,
-				     mode | S_IFDIR, 0);
+				     mode, 0);
 	if (!retval)
 		inc_nlink(dir);
 	return ERR_PTR(retval);
@@ -979,7 +979,7 @@ static struct dentry *hugetlbfs_mkdir(struct mnt_idmap *idmap, struct inode *dir
 
 static int hugetlbfs_create(struct mnt_idmap *idmap,
 			    struct inode *dir, struct dentry *dentry,
-			    umode_t mode, bool excl)
+			    umode_t mode)
 {
 	return hugetlbfs_mknod(idmap, dir, dentry, mode | S_IFREG, 0);
 }
diff --git a/fs/inode.c b/fs/inode.c
index 31c5b9ee3a81d..a31aa7cb47f65 100644
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -763,21 +763,18 @@ void clear_inode(struct inode *inode)
 		fsverity_cleanup_inode(inode);
 
 	/*
-	 * We have to cycle the i_pages lock here because reclaim can be in the
-	 * process of removing the last page (in __filemap_remove_folio())
-	 * and we must not free the mapping under it.
+	 * We have to cycle the i_pages lock here because reclaim
+	 * can be in the process of removing the last page (in
+	 * __filemap_remove_folio()) and we must not free the mapping
+	 * under it.  We also remove nodes which are empty; these
+	 * can occur in two different ways.  The first is that radix
+	 * tree expansion can fail partway and the second is that THP
+	 * collapse_file() can allocate some temporary nodes and not
+	 * clean them up.
 	 */
-	xa_lock_irq(&inode->i_data.i_pages);
+	xa_destroy(&inode->i_data.i_pages);
+
 	BUG_ON(inode->i_data.nrpages);
-	/*
-	 * Almost always, mapping_empty(&inode->i_data) here; but there are
-	 * two known and long-standing ways in which nodes may get left behind
-	 * (when deep radix-tree node allocation failed partway; or when THP
-	 * collapse_file() failed). Until those two known cases are cleaned up,
-	 * or a cleanup function is called here, do not BUG_ON(!mapping_empty),
-	 * nor even WARN_ON(!mapping_empty).
-	 */
-	xa_unlock_irq(&inode->i_data.i_pages);
 	BUG_ON(!(inode_state_read_once(inode) & I_FREEING));
 	BUG_ON(inode_state_read_once(inode) & I_CLEAR);
 	BUG_ON(!list_empty(&inode->i_wb_list));
diff --git a/fs/internal.h b/fs/internal.h
index 355d93f922086..174f063575558 100644
--- a/fs/internal.h
+++ b/fs/internal.h
@@ -137,6 +137,7 @@ extern int reconfigure_super(struct fs_context *);
 extern bool super_trylock_shared(struct super_block *sb);
 struct super_block *user_get_super(dev_t, bool excl);
 void put_super(struct super_block *sb);
+void __init super_dev_init(void);
 extern bool mount_capable(struct fs_context *);
 
 /*
diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c
index 276720bc18dc5..6f07dbe6e8f25 100644
--- a/fs/iomap/buffered-io.c
+++ b/fs/iomap/buffered-io.c
@@ -768,7 +768,7 @@ EXPORT_SYMBOL_GPL(iomap_is_partially_uptodate);
  */
 struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len)
 {
-	fgf_t fgp = FGP_WRITEBEGIN | FGP_NOFS;
+	fgf_t fgp = FGP_WRITEBEGIN;
 
 	if (iter->flags & IOMAP_NOWAIT)
 		fgp |= FGP_NOWAIT;
@@ -1156,7 +1156,6 @@ static bool iomap_write_end(struct iomap_iter *iter, size_t len, size_t copied,
 static int iomap_write_iter(struct iomap_iter *iter, struct iov_iter *i,
 		const struct iomap_write_ops *write_ops)
 {
-	ssize_t total_written = 0;
 	int status = 0;
 	struct address_space *mapping = iter->inode->i_mapping;
 	size_t chunk = mapping_max_folio_size(mapping);
@@ -1252,12 +1251,11 @@ static int iomap_write_iter(struct iomap_iter *iter, struct iov_iter *i,
 				goto retry;
 			}
 		} else {
-			total_written += written;
 			iomap_iter_advance(iter, written);
 		}
 	} while (iov_iter_count(i) && iomap_length(iter));
 
-	return total_written ? 0 : status;
+	return status;
 }
 
 ssize_t
diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
index e2cd5f92babed..64c6b7286c208 100644
--- a/fs/iomap/direct-io.c
+++ b/fs/iomap/direct-io.c
@@ -10,6 +10,7 @@
 #include <linux/iomap.h>
 #include <linux/task_io_accounting_ops.h>
 #include <linux/fserror.h>
+#include <linux/init.h>
 #include "internal.h"
 #include "trace.h"
 
@@ -88,9 +89,9 @@ static inline enum fserror_type iomap_dio_err_type(const struct iomap_dio *dio)
 	return FSERR_DIRECTIO_READ;
 }
 
-static inline bool should_report_dio_fserror(const struct iomap_dio *dio)
+static inline bool should_report_dio_fserror(int error)
 {
-	switch (dio->error) {
+	switch (error) {
 	case 0:
 	case -EAGAIN:
 	case -ENOTBLK:
@@ -110,7 +111,7 @@ ssize_t iomap_dio_complete(struct iomap_dio *dio)
 
 	if (dops && dops->end_io)
 		ret = dops->end_io(iocb, dio->size, ret, dio->flags);
-	if (should_report_dio_fserror(dio))
+	if (should_report_dio_fserror(dio->error))
 		fserror_report_io(file_inode(iocb->ki_filp),
 				  iomap_dio_err_type(dio), offset, dio->size,
 				  dio->error, GFP_NOFS);
@@ -403,6 +404,14 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter,
 	return ret;
 }
 
+static inline unsigned int iomap_dio_alignment(struct inode *inode,
+		struct block_device *bdev, unsigned int dio_flags)
+{
+	if (dio_flags & IOMAP_DIO_FSBLOCK_ALIGNED)
+		return i_blocksize(inode);
+	return bdev_logical_block_size(bdev);
+}
+
 static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
 {
 	const struct iomap *iomap = &iter->iomap;
@@ -421,10 +430,7 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
 	 * File systems that write out of place and always allocate new blocks
 	 * need each bio to be block aligned as that's the unit of allocation.
 	 */
-	if (dio->flags & IOMAP_DIO_FSBLOCK_ALIGNED)
-		alignment = fs_block_size;
-	else
-		alignment = bdev_logical_block_size(iomap->bdev);
+	alignment = iomap_dio_alignment(inode, iomap->bdev, dio->flags);
 
 	if ((pos | length) & (alignment - 1))
 		return -EINVAL;
@@ -893,12 +899,277 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
 }
 EXPORT_SYMBOL_GPL(__iomap_dio_rw);
 
+struct iomap_dio_simple {
+	struct kiocb		*iocb;
+	size_t			size;
+	unsigned int		dio_flags;
+	struct work_struct	work;
+	/*
+	 * Align @bio to a cacheline boundary so that, combined with the
+	 * front_pad passed to bioset_init(), the bio sits at the start of
+	 * a cacheline in memory returned by the (HWCACHE-aligned) bio
+	 * slab.  This keeps the hot fields block layer touches on submit
+	 * and completion (bi_iter, bi_status, ...) within a single line.
+	 */
+	struct bio		bio ____cacheline_aligned_in_smp;
+};
+
+static struct bio_set iomap_dio_simple_pool;
+
+static ssize_t iomap_dio_simple_complete(struct iomap_dio_simple *sr)
+{
+	struct bio *bio = &sr->bio;
+	struct kiocb *iocb = sr->iocb;
+	struct inode *inode = file_inode(iocb->ki_filp);
+	ssize_t ret;
+
+	if (unlikely(bio->bi_status)) {
+		ret = blk_status_to_errno(bio->bi_status);
+		if (should_report_dio_fserror(ret))
+			fserror_report_io(inode, FSERR_DIRECTIO_READ,
+					  iocb->ki_pos, sr->size, ret,
+					  GFP_NOFS);
+	} else {
+		ret = sr->size;
+		iocb->ki_pos += ret;
+	}
+
+	if (sr->dio_flags & IOMAP_DIO_USER_BACKED) {
+		bio_check_pages_dirty(bio);
+	} else {
+		bio_release_pages(bio, false);
+		bio_put(bio);
+	}
+	inode_dio_end(inode);
+	trace_iomap_dio_complete(iocb, ret < 0 ? ret : 0, ret);
+	return ret;
+}
+
+static void iomap_dio_simple_complete_work(struct work_struct *work)
+{
+	struct iomap_dio_simple *sr =
+		container_of(work, struct iomap_dio_simple, work);
+	struct kiocb *iocb = sr->iocb;
+
+	WRITE_ONCE(iocb->private, NULL);
+	iocb->ki_complete(iocb, iomap_dio_simple_complete(sr));
+}
+
+static void iomap_dio_simple_end_io(struct bio *bio)
+{
+	struct iomap_dio_simple *sr =
+		container_of(bio, struct iomap_dio_simple, bio);
+	struct kiocb *iocb = sr->iocb;
+
+	if (unlikely(sr->bio.bi_status)) {
+		struct inode *inode = file_inode(iocb->ki_filp);
+
+		INIT_WORK(&sr->work, iomap_dio_simple_complete_work);
+		queue_work(inode->i_sb->s_dio_done_wq, &sr->work);
+		return;
+	}
+
+	WRITE_ONCE(iocb->private, NULL);
+	iocb->ki_complete(iocb, iomap_dio_simple_complete(sr));
+}
+
+static inline bool
+iomap_dio_simple_supported(struct kiocb *iocb, struct iov_iter *iter,
+			   const struct iomap_dio_ops *dops,
+			   unsigned int dio_flags, size_t done_before)
+{
+	struct inode *inode = file_inode(iocb->ki_filp);
+	size_t count = iov_iter_count(iter);
+
+	if (dops || done_before)
+		return false;
+	if (iov_iter_rw(iter) != READ)
+		return false;
+	if (!count)
+		return false;
+	/*
+	 * Simple dio is an optimization for small IO. Filter out large IO
+	 * early as it's the most common case to fail for typical direct IO
+	 * workloads.
+	 */
+	if (count > inode->i_sb->s_blocksize)
+		return false;
+	if (dio_flags & (IOMAP_DIO_FORCE_WAIT | IOMAP_DIO_PARTIAL |
+			 IOMAP_DIO_BOUNCE))
+		return false;
+	if (iocb->ki_pos + count > i_size_read(inode))
+		return false;
+	if (IS_ENCRYPTED(inode))
+		return false;
+
+	return true;
+}
+
+/*
+ * Fast path for small, block-aligned direct I/Os that map to a single
+ * contiguous on-disk extent.
+ *
+ * iomap_dio_simple_supported() enforces the cheap up-front constraints before
+ * entering this path.
+ *
+ * @dops must be NULL: a non-NULL @dops means the caller wants its
+ * ->end_io / ->submit_io hooks invoked, and in particular wants its bios to be
+ * allocated from the filesystem-private @dops->bio_set (whose front_pad sizes a
+ * filesystem-private wrapper around the bio).  The fast path instead allocates
+ * from the shared iomap_dio_simple_pool, whose front_pad matches struct
+ * iomap_dio_simple; the two wrappers are not interchangeable, so we must fall
+ * back to __iomap_dio_rw() in that case.
+ *
+ * @done_before must be zero: a non-zero caller-accumulated residual cannot be
+ * carried through a single-bio inline completion.
+ *
+ * @iter must describe a non-empty READ no larger than the inode block size:
+ * writes, zero-length I/O, and larger requests need the generic iomap direct
+ * I/O path.
+ *
+ * @dio_flags must not request IOMAP_DIO_FORCE_WAIT, IOMAP_DIO_PARTIAL, or
+ * IOMAP_DIO_BOUNCE: this path does not support forced waiting, partial direct
+ * I/O, or bouncing.  The range must also stay within i_size and encrypted
+ * inodes must use the generic iomap direct I/O path.
+ *
+ * -ENOTBLK is the private sentinel returned by iomap_dio_simple() when it
+ * decides the request does not fit the fast path.  In that case we proceed to
+ * the generic __iomap_dio_rw() slow path.  Any other errno is a real result and
+ * is propagated as-is, in particular -EAGAIN for IOCB_NOWAIT must reach the
+ * caller.
+ */
+static ssize_t
+iomap_dio_simple(struct kiocb *iocb, struct iov_iter *iter,
+		 const struct iomap_ops *ops, void *private,
+		 unsigned int dio_flags)
+{
+	struct inode *inode = file_inode(iocb->ki_filp);
+	size_t count = iov_iter_count(iter);
+	bool wait_for_completion = is_sync_kiocb(iocb);
+	struct iomap_iter iomi = {
+		.inode		= inode,
+		.pos		= iocb->ki_pos,
+		.len		= count,
+		.flags		= IOMAP_DIRECT,
+		.private	= private,
+	};
+	struct iomap_dio_simple *sr;
+	unsigned int alignment;
+	struct bio *bio;
+	ssize_t ret;
+
+	if (iocb->ki_flags & IOCB_NOWAIT)
+		iomi.flags |= IOMAP_NOWAIT;
+
+	ret = kiocb_write_and_wait(iocb, count);
+	if (ret)
+		return ret;
+
+	inode_dio_begin(inode);
+
+	ret = ops->iomap_begin(inode, iomi.pos, count, iomi.flags,
+			       &iomi.iomap, &iomi.srcmap);
+	if (ret) {
+		inode_dio_end(inode);
+		return ret;
+	}
+
+	if (iomi.iomap.type != IOMAP_MAPPED ||
+	    iomi.iomap.offset + iomi.iomap.length < iomi.pos + count ||
+	    (iomi.iomap.flags & IOMAP_F_INTEGRITY)) {
+		ret = -ENOTBLK;
+		goto out_iomap_end;
+	}
+
+	alignment = iomap_dio_alignment(inode, iomi.iomap.bdev, dio_flags);
+	if ((iomi.pos | count) & (alignment - 1)) {
+		ret = -EINVAL;
+		goto out_iomap_end;
+	}
+
+	if (!wait_for_completion && unlikely(!inode->i_sb->s_dio_done_wq)) {
+		ret = sb_init_dio_done_wq(inode->i_sb);
+		if (ret < 0)
+			goto out_iomap_end;
+	}
+
+	trace_iomap_dio_rw_begin(iocb, iter, dio_flags, 0);
+
+	if (user_backed_iter(iter))
+		dio_flags |= IOMAP_DIO_USER_BACKED;
+
+	bio = bio_alloc_bioset(iomi.iomap.bdev,
+			       bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS),
+			       REQ_OP_READ, GFP_KERNEL, &iomap_dio_simple_pool);
+	sr = container_of(bio, struct iomap_dio_simple, bio);
+	sr->iocb = iocb;
+	sr->dio_flags = dio_flags;
+
+	bio->bi_iter.bi_sector = iomap_sector(&iomi.iomap, iomi.pos);
+	bio->bi_ioprio = iocb->ki_ioprio;
+
+	ret = bio_iov_iter_get_pages(bio, iter, alignment - 1);
+	if (unlikely(ret))
+		goto out_bio_put;
+
+	if (bio->bi_iter.bi_size != count) {
+		iov_iter_revert(iter, bio->bi_iter.bi_size);
+		ret = -ENOTBLK;
+		goto out_bio_release_pages;
+	}
+
+	sr->size = bio->bi_iter.bi_size;
+
+	if (dio_flags & IOMAP_DIO_USER_BACKED)
+		bio_set_pages_dirty(bio);
+
+	if (iocb->ki_flags & IOCB_NOWAIT)
+		bio->bi_opf |= REQ_NOWAIT;
+	if ((iocb->ki_flags & IOCB_HIPRI) && !wait_for_completion) {
+		bio->bi_opf |= REQ_POLLED;
+		WRITE_ONCE(iocb->private, bio);
+	}
+
+	if (ops->iomap_end)
+		ops->iomap_end(inode, iomi.pos, count, count, iomi.flags,
+			       &iomi.iomap);
+
+	if (!wait_for_completion) {
+		bio->bi_end_io = iomap_dio_simple_end_io;
+		submit_bio(bio);
+		trace_iomap_dio_rw_queued(inode, iomi.pos, count);
+		return -EIOCBQUEUED;
+	}
+
+	submit_bio_wait(bio);
+	return iomap_dio_simple_complete(sr);
+
+out_bio_release_pages:
+	bio_release_pages(bio, false);
+out_bio_put:
+	bio_put(bio);
+out_iomap_end:
+	if (ops->iomap_end)
+		ops->iomap_end(inode, iomi.pos, count, 0, iomi.flags,
+			       &iomi.iomap);
+	inode_dio_end(inode);
+	return ret;
+}
+
 ssize_t
 iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
 		const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
 		unsigned int dio_flags, void *private, size_t done_before)
 {
 	struct iomap_dio *dio;
+	ssize_t ret;
+
+	if (iomap_dio_simple_supported(iocb, iter, dops, dio_flags,
+				       done_before)) {
+		ret = iomap_dio_simple(iocb, iter, ops, private, dio_flags);
+		if (ret != -ENOTBLK)
+			return ret;
+	}
 
 	dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private,
 			     done_before);
@@ -907,3 +1178,11 @@ iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
 	return iomap_dio_complete(dio);
 }
 EXPORT_SYMBOL_GPL(iomap_dio_rw);
+
+static int __init iomap_dio_init(void)
+{
+	return bioset_init(&iomap_dio_simple_pool, 4,
+			   offsetof(struct iomap_dio_simple, bio),
+			   BIOSET_NEED_BVECS | BIOSET_PERCPU_CACHE);
+}
+fs_initcall(iomap_dio_init);
diff --git a/fs/iomap/ioend.c b/fs/iomap/ioend.c
index 0565328764c17..2ec755a89228b 100644
--- a/fs/iomap/ioend.c
+++ b/fs/iomap/ioend.c
@@ -13,6 +13,7 @@
 
 struct bio_set iomap_ioend_bioset;
 EXPORT_SYMBOL_GPL(iomap_ioend_bioset);
+static struct bio_set iomap_ioend_split_bioset;
 
 struct iomap_ioend *iomap_init_ioend(struct inode *inode,
 		struct bio *bio, loff_t file_offset, u16 ioend_flags)
@@ -486,7 +487,8 @@ struct iomap_ioend *iomap_split_ioend(struct iomap_ioend *ioend,
 	sector_offset = ALIGN_DOWN(sector_offset << SECTOR_SHIFT,
 			i_blocksize(ioend->io_inode)) >> SECTOR_SHIFT;
 
-	split = bio_split(bio, sector_offset, GFP_NOFS, &iomap_ioend_bioset);
+	split = bio_split(bio, sector_offset, GFP_NOFS,
+			&iomap_ioend_split_bioset);
 	if (IS_ERR(split))
 		return ERR_CAST(split);
 	split->bi_private = bio->bi_private;
@@ -509,8 +511,23 @@ EXPORT_SYMBOL_GPL(iomap_split_ioend);
 
 static int __init iomap_ioend_init(void)
 {
-	return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE),
+	const unsigned int nr_mempool_entries = 4 * (PAGE_SIZE / SECTOR_SIZE);
+	int error;
+
+	error = bioset_init(&iomap_ioend_bioset, nr_mempool_entries,
 			   offsetof(struct iomap_ioend, io_bio),
 			   BIOSET_NEED_BVECS);
+	if (error)
+		return error;
+	error = bioset_init(&iomap_ioend_split_bioset, nr_mempool_entries,
+			   offsetof(struct iomap_ioend, io_bio),
+			   BIOSET_NEED_BVECS);
+	if (error)
+		goto out_exit_ioend_bioset;
+	return 0;
+
+out_exit_ioend_bioset:
+	bioset_exit(&iomap_ioend_bioset);
+	return error;
 }
 fs_initcall(iomap_ioend_init);
diff --git a/fs/isofs/compress.c b/fs/isofs/compress.c
index 397568b9c7e7d..3fda92358e225 100644
--- a/fs/isofs/compress.c
+++ b/fs/isofs/compress.c
@@ -65,12 +65,14 @@ static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start,
 	/* Empty block? */
 	if (block_size == 0) {
 		for ( i = 0 ; i < pcount ; i++ ) {
+			unsigned int off = i ? 0 : poffset;
+
 			if (!pages[i])
 				continue;
-			memzero_page(pages[i], 0, PAGE_SIZE);
+			memzero_page(pages[i], off, PAGE_SIZE - off);
 			SetPageUptodate(pages[i]);
 		}
-		return ((loff_t)pcount) << PAGE_SHIFT;
+		return (((loff_t)pcount) << PAGE_SHIFT) - poffset;
 	}
 
 	/* Because zlib is not thread-safe, do all the I/O at the top. */
diff --git a/fs/jffs2/dir.c b/fs/jffs2/dir.c
index c4088c3b4ac03..656c920864c50 100644
--- a/fs/jffs2/dir.c
+++ b/fs/jffs2/dir.c
@@ -26,7 +26,7 @@
 static int jffs2_readdir (struct file *, struct dir_context *);
 
 static int jffs2_create (struct mnt_idmap *, struct inode *,
-		         struct dentry *, umode_t, bool);
+		         struct dentry *, umode_t);
 static struct dentry *jffs2_lookup (struct inode *,struct dentry *,
 				    unsigned int);
 static int jffs2_link (struct dentry *,struct inode *,struct dentry *);
@@ -163,7 +163,7 @@ static int jffs2_readdir(struct file *file, struct dir_context *ctx)
 
 
 static int jffs2_create(struct mnt_idmap *idmap, struct inode *dir_i,
-			struct dentry *dentry, umode_t mode, bool excl)
+			struct dentry *dentry, umode_t mode)
 {
 	struct jffs2_raw_inode *ri;
 	struct jffs2_inode_info *f, *dir_f;
@@ -462,8 +462,6 @@ static struct dentry *jffs2_mkdir (struct mnt_idmap *idmap, struct inode *dir_i,
 	uint32_t alloclen;
 	int ret;
 
-	mode |= S_IFDIR;
-
 	ri = jffs2_alloc_raw_inode();
 	if (!ri)
 		return ERR_PTR(-ENOMEM);
diff --git a/fs/jfs/namei.c b/fs/jfs/namei.c
index 442d626792622..8a36c218f0f7a 100644
--- a/fs/jfs/namei.c
+++ b/fs/jfs/namei.c
@@ -61,7 +61,7 @@ static inline void free_ea_wmap(struct inode *inode)
  *
  */
 static int jfs_create(struct mnt_idmap *idmap, struct inode *dip,
-		      struct dentry *dentry, umode_t mode, bool excl)
+		      struct dentry *dentry, umode_t mode)
 {
 	int rc = 0;
 	tid_t tid;		/* transaction id */
@@ -223,7 +223,7 @@ static struct dentry *jfs_mkdir(struct mnt_idmap *idmap, struct inode *dip,
 	 * block there while holding dtree page, so we allocate the inode &
 	 * begin the transaction before we search the directory.
 	 */
-	ip = ialloc(dip, S_IFDIR | mode);
+	ip = ialloc(dip, mode);
 	if (IS_ERR(ip)) {
 		rc = PTR_ERR(ip);
 		goto out2;
diff --git a/fs/kernel_read_file.c b/fs/kernel_read_file.c
index de32c95d823db..9c2ba92400833 100644
--- a/fs/kernel_read_file.c
+++ b/fs/kernel_read_file.c
@@ -150,18 +150,13 @@ ssize_t kernel_read_file_from_path_initns(const char *path, loff_t offset,
 					  enum kernel_read_file_id id)
 {
 	struct file *file;
-	struct path root;
 	ssize_t ret;
 
 	if (!path || !*path)
 		return -EINVAL;
 
-	task_lock(&init_task);
-	get_fs_root(init_task.fs, &root);
-	task_unlock(&init_task);
-
-	file = file_open_root(&root, path, O_RDONLY, 0);
-	path_put(&root);
+	scoped_with_init_fs()
+		file = filp_open(path, O_RDONLY, 0);
 	if (IS_ERR(file))
 		return PTR_ERR(file);
 
diff --git a/fs/lockd/lockd.h b/fs/lockd/lockd.h
index e418a50c41803..14cc952fe81a2 100644
--- a/fs/lockd/lockd.h
+++ b/fs/lockd/lockd.h
@@ -314,7 +314,7 @@ void		  nsm_release(struct nsm_handle *nsm);
  * This is used in garbage collection and resource reclaim
  * A return value != 0 means destroy the lock/block/share
  */
-typedef int	  (*nlm_host_match_fn_t)(void *cur, struct nlm_host *ref);
+typedef int	  (*nlm_host_match_fn_t)(void *owner, struct nlm_host *ref);
 
 /*
  * Server-side lock handling
diff --git a/fs/lockd/svc.c b/fs/lockd/svc.c
index 490551369ef2a..ee90e743064af 100644
--- a/fs/lockd/svc.c
+++ b/fs/lockd/svc.c
@@ -47,7 +47,7 @@
 
 static struct svc_program	nlmsvc_program;
 
-const struct nlmsvc_binding	*nlmsvc_ops;
+const struct nlmsvc_binding __rcu *nlmsvc_ops;
 EXPORT_SYMBOL_GPL(nlmsvc_ops);
 
 static DEFINE_MUTEX(nlmsvc_mutex);
@@ -142,7 +142,7 @@ lockd(void *vrqstp)
 		nlmsvc_retry_blocked(rqstp);
 		svc_recv(rqstp, 0);
 	}
-	if (nlmsvc_ops)
+	if (rcu_access_pointer(nlmsvc_ops))
 		nlmsvc_invalidate_all();
 	nlm_shutdown_hosts();
 	cancel_delayed_work_sync(&ln->grace_period_end);
diff --git a/fs/lockd/svc4proc.c b/fs/lockd/svc4proc.c
index 78e675470c4b2..080dffce9d8ee 100644
--- a/fs/lockd/svc4proc.c
+++ b/fs/lockd/svc4proc.c
@@ -128,7 +128,7 @@ nlm4svc_lookup_host(struct svc_rqst *rqstp, string caller, bool monitored)
 {
 	struct nlm_host *host;
 
-	if (!nlmsvc_ops)
+	if (!rcu_access_pointer(nlmsvc_ops))
 		return NULL;
 	host = nlmsvc_lookup_host(rqstp, caller.data, caller.len);
 	if (!host)
@@ -894,7 +894,7 @@ static __be32 nlm4svc_proc_granted_res(struct svc_rqst *rqstp)
 {
 	struct nlm4_res_wrapper *argp = rqstp->rq_argp;
 
-	if (!nlmsvc_ops)
+	if (!rcu_access_pointer(nlmsvc_ops))
 		return rpc_success;
 
 	if (nlm4_netobj_to_cookie(&argp->cookie, &argp->xdrgen.cookie))
diff --git a/fs/lockd/svclock.c b/fs/lockd/svclock.c
index e48d31f14a651..e628b5d355071 100644
--- a/fs/lockd/svclock.c
+++ b/fs/lockd/svclock.c
@@ -47,40 +47,6 @@ static const struct rpc_call_ops nlmsvc_grant_ops;
 static LIST_HEAD(nlm_blocked);
 static DEFINE_SPINLOCK(nlm_blocked_lock);
 
-#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
-static const char *nlmdbg_cookie2a(const struct lockd_cookie *cookie)
-{
-	/*
-	 * We can get away with a static buffer because this is only called
-	 * from lockd, which is single-threaded.
-	 */
-	static char buf[2*NLM_MAXCOOKIELEN+1];
-	unsigned int i, len = sizeof(buf);
-	char *p = buf;
-
-	len--;	/* allow for trailing \0 */
-	if (len < 3)
-		return "???";
-	for (i = 0 ; i < cookie->len ; i++) {
-		if (len < 2) {
-			strcpy(p-3, "...");
-			break;
-		}
-		sprintf(p, "%02x", cookie->data[i]);
-		p += 2;
-		len -= 2;
-	}
-	*p = '\0';
-
-	return buf;
-}
-#else
-static inline const char *nlmdbg_cookie2a(const struct lockd_cookie *cookie)
-{
-	return "???";
-}
-#endif
-
 /*
  * Insert a blocked lock into the global list
  */
@@ -155,11 +121,12 @@ nlmsvc_lookup_block(struct nlm_file *file, struct lockd_lock *lock)
 	spin_lock(&nlm_blocked_lock);
 	list_for_each_entry(block, &nlm_blocked, b_list) {
 		fl = &block->b_call->a_args.lock.fl;
-		dprintk("lockd: check f=%p pd=%d %Ld-%Ld ty=%d cookie=%s\n",
+		dprintk("lockd: check f=%p pd=%d %Ld-%Ld ty=%d cookie=%*phN\n",
 				block->b_file, fl->c.flc_pid,
 				(long long)fl->fl_start,
 				(long long)fl->fl_end, fl->c.flc_type,
-				nlmdbg_cookie2a(&block->b_call->a_args.cookie));
+				block->b_call->a_args.cookie.len,
+				block->b_call->a_args.cookie.data);
 		if (block->b_file == file && nlm_compare_locks(fl, &lock->fl)) {
 			kref_get(&block->b_count);
 			spin_unlock(&nlm_blocked_lock);
@@ -198,7 +165,8 @@ nlmsvc_find_block(struct lockd_cookie *cookie)
 	return NULL;
 
 found:
-	dprintk("nlmsvc_find_block(%s): block=%p\n", nlmdbg_cookie2a(cookie), block);
+	dprintk("nlmsvc_find_block(%*phN): block=%p\n",
+		cookie->len, cookie->data, block);
 	kref_get(&block->b_count);
 	spin_unlock(&nlm_blocked_lock);
 	return block;
diff --git a/fs/lockd/svcproc.c b/fs/lockd/svcproc.c
index 4836887f11ef0..dce6f6e3fd406 100644
--- a/fs/lockd/svcproc.c
+++ b/fs/lockd/svcproc.c
@@ -133,7 +133,7 @@ nlm3svc_lookup_host(struct svc_rqst *rqstp, string caller, bool monitored)
 {
 	struct nlm_host *host;
 
-	if (!nlmsvc_ops)
+	if (!rcu_access_pointer(nlmsvc_ops))
 		return NULL;
 	host = nlmsvc_lookup_host(rqstp, caller.data, caller.len);
 	if (!host)
@@ -923,7 +923,7 @@ static __be32 nlmsvc_proc_granted_res(struct svc_rqst *rqstp)
 {
 	struct nlm_res_wrapper *argp = rqstp->rq_argp;
 
-	if (!nlmsvc_ops)
+	if (!rcu_access_pointer(nlmsvc_ops))
 		return rpc_success;
 
 	if (nlm_netobj_to_cookie(&argp->cookie, &argp->xdrgen.cookie))
diff --git a/fs/lockd/svcsubs.c b/fs/lockd/svcsubs.c
index a0d1a6fbf61ee..ffab76278bb26 100644
--- a/fs/lockd/svcsubs.c
+++ b/fs/lockd/svcsubs.c
@@ -90,22 +90,35 @@ int lock_to_openmode(struct file_lock *lock)
 static __be32 nlm_do_fopen(struct svc_rqst *rqstp,
 			   struct nlm_file *file, int mode)
 {
+	const struct nlmsvc_binding *ops;
 	__be32 nlmerr = nlm__int__failed;
 	__be32 deferred = 0;
 	int error;
 	int m;
 
+	rcu_read_lock();
+	ops = rcu_dereference(nlmsvc_ops);
+	if (!ops || !try_module_get(ops->owner)) {
+		rcu_read_unlock();
+		return nlm__int__failed;
+	}
+	rcu_read_unlock();
+
 	for (m = O_RDONLY; m <= O_WRONLY; m++) {
 		struct file **fp = &file->f_file[m];
 
 		if (mode != O_RDWR && mode != m)
 			continue;
-		if (*fp)
+		if (*fp) {
+			module_put(ops->owner);
 			return nlm_granted;
+		}
 
-		error = nlmsvc_ops->fopen(rqstp, &file->f_handle, fp, m);
-		if (!error)
+		error = ops->fopen(rqstp, &file->f_handle, fp, m);
+		if (!error) {
+			module_put(ops->owner);
 			return nlm_granted;
+		}
 
 		dprintk("lockd: open failed (errno %d)\n", error);
 		switch (error) {
@@ -122,6 +135,7 @@ static __be32 nlm_do_fopen(struct svc_rqst *rqstp,
 		}
 	}
 
+	module_put(ops->owner);
 	return deferred ? deferred : nlmerr;
 }
 
@@ -185,6 +199,33 @@ nlm_lookup_file(struct svc_rqst *rqstp, struct nlm_file **result,
 	goto out_unlock;
 }
 
+/*
+ * Release the struct file references held by a nlm_file.
+ */
+static void nlm_release_files(struct nlm_file *file)
+{
+	const struct nlmsvc_binding *ops;
+	bool have_ops;
+
+	rcu_read_lock();
+	ops = rcu_dereference(nlmsvc_ops);
+	have_ops = ops && try_module_get(ops->owner);
+	rcu_read_unlock();
+
+	if (have_ops) {
+		if (file->f_file[O_RDONLY])
+			ops->fclose(file->f_file[O_RDONLY]);
+		if (file->f_file[O_WRONLY])
+			ops->fclose(file->f_file[O_WRONLY]);
+		module_put(ops->owner);
+	} else {
+		if (file->f_file[O_RDONLY])
+			fput(file->f_file[O_RDONLY]);
+		if (file->f_file[O_WRONLY])
+			fput(file->f_file[O_WRONLY]);
+	}
+}
+
 /*
  * Delete a file after having released all locks, blocks and shares
  */
@@ -194,10 +235,7 @@ nlm_delete_file(struct nlm_file *file)
 	nlm_debug_print_file("closing file", file);
 	if (!hlist_unhashed(&file->f_list)) {
 		hlist_del(&file->f_list);
-		if (file->f_file[O_RDONLY])
-			nlmsvc_ops->fclose(file->f_file[O_RDONLY]);
-		if (file->f_file[O_WRONLY])
-			nlmsvc_ops->fclose(file->f_file[O_WRONLY]);
+		nlm_release_files(file);
 		kfree(file);
 	} else {
 		printk(KERN_WARNING "lockd: attempt to release unknown file!\n");
@@ -312,12 +350,10 @@ nlm_file_inuse(struct nlm_file *file)
 	return 0;
 }
 
-static void nlm_close_files(struct nlm_file *file)
+static void nlm_file_release(struct nlm_file *file)
 {
-	if (file->f_file[O_RDONLY])
-		nlmsvc_ops->fclose(file->f_file[O_RDONLY]);
-	if (file->f_file[O_WRONLY])
-		nlmsvc_ops->fclose(file->f_file[O_WRONLY]);
+	if (!nlm_file_inuse(file))
+		nlm_delete_file(file);
 }
 
 /*
@@ -327,32 +363,41 @@ static int
 nlm_traverse_files(void *data, nlm_host_match_fn_t match,
 		int (*is_failover_file)(void *data, struct nlm_file *file))
 {
-	struct hlist_node *next;
-	struct nlm_file	*file;
+	struct nlm_file *file, *next;
 	int i, ret = 0;
 
 	mutex_lock(&nlm_file_mutex);
 	for (i = 0; i < FILE_NRHASH; i++) {
-		hlist_for_each_entry_safe(file, next, &nlm_files[i], f_list) {
-			if (is_failover_file && !is_failover_file(data, file))
-				continue;
+		file = hlist_entry_safe(nlm_files[i].first,
+					struct nlm_file, f_list);
+		if (file)
 			file->f_count++;
-			mutex_unlock(&nlm_file_mutex);
-
-			/* Traverse locks, blocks and shares of this file
-			 * and update file->f_locks count */
-			if (nlm_inspect_file(data, file, match))
-				ret = 1;
+		while (file) {
+			/*
+			 * Pin the next neighbour before we drop the mutex
+			 * for nlm_inspect_file(); a concurrent
+			 * nlm_release_file() under the same mutex would
+			 * otherwise be free to unlink and kfree it during
+			 * the unlock window, leaving us to dereference a
+			 * freed slab when we walked to next afterwards.
+			 */
+			next = hlist_entry_safe(file->f_list.next,
+						struct nlm_file, f_list);
+			if (next)
+				next->f_count++;
+
+			if (!is_failover_file || is_failover_file(data, file)) {
+				mutex_unlock(&nlm_file_mutex);
+
+				if (nlm_inspect_file(data, file, match))
+					ret = 1;
+
+				mutex_lock(&nlm_file_mutex);
+			}
 
-			mutex_lock(&nlm_file_mutex);
 			file->f_count--;
-			/* No more references to this file. Let go of it. */
-			if (list_empty(&file->f_blocks) && !file->f_locks
-			 && !file->f_shares && !file->f_count) {
-				hlist_del(&file->f_list);
-				nlm_close_files(file);
-				kfree(file);
-			}
+			nlm_file_release(file);
+			file = next;
 		}
 	}
 	mutex_unlock(&nlm_file_mutex);
@@ -512,7 +557,7 @@ EXPORT_SYMBOL_GPL(nlmsvc_unlock_all_by_sb);
 static int
 nlmsvc_match_ip(void *datap, struct nlm_host *host)
 {
-	return rpc_cmp_addr(nlm_srcaddr(host), datap);
+	return rpc_cmp_addr(nlm_srcaddr(datap), (struct sockaddr *)host);
 }
 
 /**
diff --git a/fs/minix/namei.c b/fs/minix/namei.c
index 263e4ba8b1c82..5525ba367ed74 100644
--- a/fs/minix/namei.c
+++ b/fs/minix/namei.c
@@ -64,7 +64,7 @@ static int minix_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
 }
 
 static int minix_create(struct mnt_idmap *idmap, struct inode *dir,
-			struct dentry *dentry, umode_t mode, bool excl)
+			struct dentry *dentry, umode_t mode)
 {
 	return minix_mknod(&nop_mnt_idmap, dir, dentry, mode, 0);
 }
@@ -110,7 +110,7 @@ static struct dentry *minix_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	struct inode * inode;
 	int err;
 
-	inode = minix_new_inode(dir, S_IFDIR | mode);
+	inode = minix_new_inode(dir, mode);
 	if (IS_ERR(inode))
 		return ERR_CAST(inode);
 
diff --git a/fs/namei.c b/fs/namei.c
index 19ce43c9a6e66..e4ef48583a3bb 100644
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -4140,11 +4140,6 @@ EXPORT_SYMBOL(end_renaming);
  * after setgid stripping allows the same ordering for both non-POSIX ACL and
  * POSIX ACL supporting filesystems.
  *
- * Note that it's currently valid for @type to be 0 if a directory is created.
- * Filesystems raise that flag individually and we need to check whether each
- * filesystem can deal with receiving S_IFDIR from the vfs before we enforce a
- * non-zero type.
- *
  * Returns: mode to be passed to the filesystem
  */
 static inline umode_t vfs_prepare_mode(struct mnt_idmap *idmap,
@@ -4190,7 +4185,7 @@ int vfs_create(struct mnt_idmap *idmap, struct dentry *dentry, umode_t mode,
 		return error;
 
 	if (!dir->i_op->create)
-		return -EACCES;	/* shouldn't it be ENOSYS? */
+		return -EOPNOTSUPP;
 
 	mode = vfs_prepare_mode(idmap, dir, mode, S_IALLUGO, S_IFREG);
 	error = security_inode_create(dir, dentry, mode);
@@ -4199,7 +4194,7 @@ int vfs_create(struct mnt_idmap *idmap, struct dentry *dentry, umode_t mode,
 	error = try_break_deleg(dir, LEASE_BREAK_DIR_CREATE, di);
 	if (error)
 		return error;
-	error = dir->i_op->create(idmap, dir, dentry, mode, true);
+	error = dir->i_op->create(idmap, dir, dentry, mode);
 	if (!error)
 		fsnotify_create(dir, dentry);
 	return error;
@@ -4501,12 +4496,11 @@ static struct dentry *lookup_open(struct nameidata *nd, struct file *file,
 		file->f_mode |= FMODE_CREATED;
 		audit_inode_child(dir_inode, dentry, AUDIT_TYPE_CHILD_CREATE);
 		if (!dir_inode->i_op->create) {
-			error = -EACCES;
+			error = -EOPNOTSUPP;
 			goto out_dput;
 		}
 
-		error = dir_inode->i_op->create(idmap, dir_inode, dentry,
-						mode, open_flag & O_EXCL);
+		error = dir_inode->i_op->create(idmap, dir_inode, dentry, mode);
 		if (error)
 			goto out_dput;
 	}
@@ -5117,7 +5111,7 @@ int vfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 		return -EPERM;
 
 	if (!dir->i_op->mknod)
-		return -EPERM;
+		return -EOPNOTSUPP;
 
 	mode = vfs_prepare_mode(idmap, dir, mode, mode, mode);
 	error = devcgroup_inode_mknod(mode, dev);
@@ -5256,11 +5250,11 @@ struct dentry *vfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	if (error)
 		goto err;
 
-	error = -EPERM;
+	error = -EOPNOTSUPP;
 	if (!dir->i_op->mkdir)
 		goto err;
 
-	mode = vfs_prepare_mode(idmap, dir, mode, S_IRWXUGO | S_ISVTX, 0);
+	mode = vfs_prepare_mode(idmap, dir, mode, S_IRWXUGO | S_ISVTX, S_IFDIR);
 	error = security_inode_mkdir(dir, dentry, mode);
 	if (error)
 		goto err;
@@ -5360,7 +5354,7 @@ int vfs_rmdir(struct mnt_idmap *idmap, struct inode *dir,
 		return error;
 
 	if (!dir->i_op->rmdir)
-		return -EPERM;
+		return -EOPNOTSUPP;
 
 	dget(dentry);
 	inode_lock(dentry->d_inode);
@@ -5496,7 +5490,7 @@ int vfs_unlink(struct mnt_idmap *idmap, struct inode *dir,
 		return error;
 
 	if (!dir->i_op->unlink)
-		return -EPERM;
+		return -EOPNOTSUPP;
 
 	inode_lock(target);
 	if (IS_SWAPFILE(target))
@@ -5647,7 +5641,7 @@ int vfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
 		return error;
 
 	if (!dir->i_op->symlink)
-		return -EPERM;
+		return -EOPNOTSUPP;
 
 	error = security_inode_symlink(dir, dentry, oldname);
 	if (error)
@@ -5769,7 +5763,7 @@ int vfs_link(struct dentry *old_dentry, struct mnt_idmap *idmap,
 	if (HAS_UNMAPPED_ID(idmap, inode))
 		return -EPERM;
 	if (!dir->i_op->link)
-		return -EPERM;
+		return -EOPNOTSUPP;
 	if (S_ISDIR(inode->i_mode))
 		return -EPERM;
 
@@ -5978,7 +5972,7 @@ int vfs_rename(struct renamedata *rd)
 		return error;
 
 	if (!old_dir->i_op->rename)
-		return -EPERM;
+		return -EOPNOTSUPP;
 
 	/*
 	 * If we are going to change the parent - check write permissions,
diff --git a/fs/namespace.c b/fs/namespace.c
index 3d5cd5bf3b053..6b81b77f93033 100644
--- a/fs/namespace.c
+++ b/fs/namespace.c
@@ -2908,6 +2908,9 @@ static int do_change_type(const struct path *path, int ms_flags)
 	for (m = mnt; m; m = (recurse ? next_mnt(m, mnt) : NULL))
 		change_mnt_propagation(m, type);
 
+	guard(mount_locked_reader)();
+	touch_mnt_namespace(mnt->mnt_ns);
+
 	return 0;
 }
 
@@ -3481,6 +3484,10 @@ static int do_set_group(const struct path *from_path, const struct path *to_path
 		list_add(&to->mnt_share, &from->mnt_share);
 		set_mnt_shared(to);
 	}
+
+	guard(mount_locked_reader)();
+	touch_mnt_namespace(to->mnt_ns);
+
 	return 0;
 }
 
@@ -6184,12 +6191,14 @@ static void __init init_mount_tree(void)
 	struct path root;
 
 	/*
-	 * We create two mounts:
+	 * We create three mounts:
 	 *
 	 * (1) nullfs with mount id 1
 	 * (2) mutable rootfs with mount id 2
+	 * (3) private nullfs for kthreads (SB_KERNMOUNT)
 	 *
-	 * with (2) mounted on top of (1).
+	 * with (2) mounted on top of (1). The init_task's root and pwd
+	 * are pointed at (3) so all kthreads start isolated in nullfs.
 	 */
 	nullfs_mnt = vfs_kern_mount(&nullfs_fs_type, 0, "nullfs", NULL);
 	if (IS_ERR(nullfs_mnt))
@@ -6229,12 +6238,14 @@ static void __init init_mount_tree(void)
 		init_mnt_ns.nr_mounts++;
 	}
 
+	nullfs_mnt = kern_mount(&nullfs_fs_type);
+	if (IS_ERR(nullfs_mnt))
+		panic("VFS: Failed to create private nullfs instance");
+	root.mnt	= nullfs_mnt;
+	root.dentry	= nullfs_mnt->mnt_root;
+
 	init_task.nsproxy->mnt_ns = &init_mnt_ns;
 	get_mnt_ns(&init_mnt_ns);
-
-	/* The root and pwd always point to the mutable rootfs. */
-	root.mnt	= mnt;
-	root.dentry	= mnt->mnt_root;
 	set_fs_pwd(current->fs, &root);
 	set_fs_root(current->fs, &root);
 
@@ -6262,6 +6273,8 @@ void __init mnt_init(void)
 	if (!mount_hashtable || !mountpoint_hashtable)
 		panic("Failed to allocate mount hash table\n");
 
+	super_dev_init();
+
 	kernfs_init();
 
 	err = sysfs_init();
diff --git a/fs/nfs/blocklayout/dev.c b/fs/nfs/blocklayout/dev.c
index bb35f88501cea..368d20daf67b2 100644
--- a/fs/nfs/blocklayout/dev.c
+++ b/fs/nfs/blocklayout/dev.c
@@ -4,6 +4,7 @@
  */
 #include <linux/sunrpc/svc.h>
 #include <linux/blkdev.h>
+#include <linux/fs_struct.h>
 #include <linux/nfs4.h>
 #include <linux/nfs_fs.h>
 #include <linux/nfs_xdr.h>
@@ -363,15 +364,22 @@ static struct file *
 bl_open_path(struct pnfs_block_volume *v, const char *prefix)
 {
 	struct file *bdev_file;
-	const char *devname;
+	const char *devname __free(kfree) = NULL;
 
 	devname = kasprintf(GFP_KERNEL, "/dev/disk/by-id/%s%*phN",
 			prefix, v->scsi.designator_len, v->scsi.designator);
 	if (!devname)
 		return ERR_PTR(-ENOMEM);
 
-	bdev_file = bdev_file_open_by_path(devname,
-			BLK_OPEN_READ | BLK_OPEN_WRITE, NULL, NULL);
+	if (tsk_is_kthread(current)) {
+		scoped_with_init_fs()
+			bdev_file = bdev_file_open_by_path(devname,
+					BLK_OPEN_READ | BLK_OPEN_WRITE,
+					NULL, NULL);
+	} else {
+		bdev_file = bdev_file_open_by_path(devname,
+				BLK_OPEN_READ | BLK_OPEN_WRITE, NULL, NULL);
+	}
 	if (IS_ERR(bdev_file)) {
 		dprintk("failed to open device %s (%ld)\n",
 			devname, PTR_ERR(bdev_file));
@@ -380,7 +388,6 @@ bl_open_path(struct pnfs_block_volume *v, const char *prefix)
 			file_bdev(bdev_file)->bd_disk->disk_name);
 	}
 
-	kfree(devname);
 	return bdev_file;
 }
 
diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c
index ff4e9fd38e83f..bc282b744f34f 100644
--- a/fs/nfs/callback.c
+++ b/fs/nfs/callback.c
@@ -231,8 +231,9 @@ int nfs_callback_up(u32 minorversion, struct rpc_xprt *xprt)
 	cb_info->users++;
 err_net:
 	if (!cb_info->users) {
+		xprt_svc_shutdown_bc(xprt);
 		svc_set_num_threads(cb_info->serv, 0, 0);
-		svc_destroy(&cb_info->serv);
+		xprt_svc_destroy_nullify_bc(xprt, &cb_info->serv);
 	}
 err_create:
 	mutex_unlock(&nfs_callback_mutex);
@@ -254,6 +255,7 @@ void nfs_callback_down(int minorversion, struct net *net, struct rpc_xprt *xprt)
 
 	mutex_lock(&nfs_callback_mutex);
 	serv = cb_info->serv;
+	xprt_svc_shutdown_bc(xprt);
 	nfs_callback_down_net(minorversion, serv, net);
 	cb_info->users--;
 	if (cb_info->users == 0) {
diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c
index c7caffb31935c..36f2e85889220 100644
--- a/fs/nfs/dir.c
+++ b/fs/nfs/dir.c
@@ -2427,9 +2427,9 @@ static int nfs_do_create(struct inode *dir, struct dentry *dentry,
 }
 
 int nfs_create(struct mnt_idmap *idmap, struct inode *dir,
-	       struct dentry *dentry, umode_t mode, bool excl)
+	       struct dentry *dentry, umode_t mode)
 {
-	return nfs_do_create(dir, dentry, mode, excl ? O_EXCL : 0);
+	return nfs_do_create(dir, dentry, mode, O_EXCL);
 }
 EXPORT_SYMBOL_GPL(nfs_create);
 
@@ -2474,7 +2474,7 @@ struct dentry *nfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 			dir->i_sb->s_id, dir->i_ino, dentry);
 
 	attr.ia_valid = ATTR_MODE;
-	attr.ia_mode = mode | S_IFDIR;
+	attr.ia_mode = mode;
 
 	trace_nfs_mkdir_enter(dir, dentry);
 	ret = NFS_PROTO(dir)->mkdir(dir, dentry, &attr);
diff --git a/fs/nfs/file.c b/fs/nfs/file.c
index a0d8f1c1cf103..851d93a099889 100644
--- a/fs/nfs/file.c
+++ b/fs/nfs/file.c
@@ -597,7 +597,7 @@ static int nfs_swap_activate(struct swap_info_struct *sis, struct file *file,
 	ret = rpc_clnt_swap_activate(clnt);
 	if (ret)
 		return ret;
-	ret = add_swap_extent(sis, 0, sis->max, 0);
+	ret = swap_fs_activate(sis);
 	if (ret < 0) {
 		rpc_clnt_swap_deactivate(clnt);
 		return ret;
@@ -607,8 +607,6 @@ static int nfs_swap_activate(struct swap_info_struct *sis, struct file *file,
 
 	if (cl->rpc_ops->enable_swap)
 		cl->rpc_ops->enable_swap(inode);
-
-	sis->flags |= SWP_FS_OPS;
 	return ret;
 }
 
diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index e4533f5836321..7f96a258af76d 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -395,7 +395,7 @@ extern unsigned long nfs_access_cache_scan(struct shrinker *shrink,
 struct dentry *nfs_lookup(struct inode *, struct dentry *, unsigned int);
 void nfs_d_prune_case_insensitive_aliases(struct inode *inode);
 int nfs_create(struct mnt_idmap *, struct inode *, struct dentry *,
-	       umode_t, bool);
+	       umode_t);
 struct dentry *nfs_mkdir(struct mnt_idmap *, struct inode *, struct dentry *,
 			 umode_t);
 int nfs_rmdir(struct inode *, struct dentry *);
diff --git a/fs/nfs_common/nfslocalio.c b/fs/nfs_common/nfslocalio.c
index dd715cdb6c043..85aa03a7b020c 100644
--- a/fs/nfs_common/nfslocalio.c
+++ b/fs/nfs_common/nfslocalio.c
@@ -292,8 +292,22 @@ struct nfsd_file *nfs_open_local_fh(nfs_uuid_t *uuid,
 	localio = nfs_to->nfsd_open_local_fh(net, uuid->dom, rpc_clnt, cred,
 					     nfs_fh, pnf, fmode);
 	if (!IS_ERR(localio) && nfs_uuid_add_file(uuid, nfl) < 0) {
-		/* Delete the cached file when racing with nfs_uuid_put() */
+		/*
+		 * Delete the cached file when racing with nfs_uuid_put().
+		 * Since nfl->nfs_uuid was never published via
+		 * rcu_assign_pointer(), nfs_close_local_fh() will early-return
+		 * and cannot clean up after us.  Drop the slot's file ref and
+		 * its paired net ref, then drop the caller-owned nfsd_file ref
+		 * (+1) and the entry-time nfsd_net ref carried via nf->nf_net,
+		 * and return -ENXIO so the caller never dereferences the
+		 * now-cleared localio.
+		 */
+		struct nfsd_file __rcu *tmp =
+			(struct nfsd_file __force __rcu *)localio;
+
 		nfs_to_nfsd_file_put_local(pnf);
+		nfs_to_nfsd_file_put_local(&tmp);
+		localio = ERR_PTR(-ENXIO);
 	}
 	nfs_to_nfsd_net_put(net);
 
diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c
index 24511c3208db9..b9548eb17c77d 100644
--- a/fs/nfsd/filecache.c
+++ b/fs/nfsd/filecache.c
@@ -55,6 +55,17 @@
 /* We only care about NFSD_MAY_READ/WRITE for this cache */
 #define NFSD_FILE_MAY_MASK	(NFSD_MAY_READ|NFSD_MAY_WRITE|NFSD_MAY_LOCALIO)
 
+/* If the shrinker runs between calls to list_lru_walk_node() in
+ * nfsd_file_gc(), the "remaining" count will be wrong.  This could
+ * result in premature freeing of some files.  This may not matter much
+ * but is easy to fix with this spinlock which temporarily disables
+ * the shrinker.
+ *
+ * It also serializes callers of nfsd_file_dispose_list_delayed()
+ * against per-net shutdown.
+ */
+static DEFINE_SPINLOCK(nfsd_gc_lock);
+
 static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits);
 static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions);
 static DEFINE_PER_CPU(unsigned long, nfsd_file_allocations);
@@ -62,16 +73,12 @@ static DEFINE_PER_CPU(unsigned long, nfsd_file_releases);
 static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age);
 static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions);
 
-struct nfsd_fcache_disposal {
-	spinlock_t lock;
-	struct list_head freeme;
-};
-
 static struct kmem_cache		*nfsd_file_slab;
 static struct kmem_cache		*nfsd_file_mark_slab;
 static struct list_lru			nfsd_file_lru;
 static unsigned long			nfsd_file_flags;
 static struct fsnotify_group		*nfsd_file_fsnotify_group;
+static struct fsnotify_group		*nfsd_dir_fsnotify_group;
 static struct delayed_work		nfsd_filecache_laundrette;
 static struct rhltable			nfsd_file_rhltable
 						____cacheline_aligned_in_smp;
@@ -147,7 +154,7 @@ static void
 nfsd_file_mark_put(struct nfsd_file_mark *nfm)
 {
 	if (refcount_dec_and_test(&nfm->nfm_ref)) {
-		fsnotify_destroy_mark(&nfm->nfm_mark, nfsd_file_fsnotify_group);
+		fsnotify_destroy_mark(&nfm->nfm_mark, nfm->nfm_mark.group);
 		fsnotify_put_mark(&nfm->nfm_mark);
 	}
 }
@@ -155,37 +162,40 @@ nfsd_file_mark_put(struct nfsd_file_mark *nfm)
 static struct nfsd_file_mark *
 nfsd_file_mark_find_or_create(struct inode *inode)
 {
-	int			err;
-	struct fsnotify_mark	*mark;
 	struct nfsd_file_mark	*nfm = NULL, *new;
+	struct fsnotify_group	*group;
+	struct fsnotify_mark	*mark;
+	int			err;
+
+	group = S_ISDIR(inode->i_mode) ? nfsd_dir_fsnotify_group : nfsd_file_fsnotify_group;
 
 	do {
-		fsnotify_group_lock(nfsd_file_fsnotify_group);
-		mark = fsnotify_find_inode_mark(inode,
-						nfsd_file_fsnotify_group);
+		fsnotify_group_lock(group);
+		mark = fsnotify_find_inode_mark(inode, group);
 		if (mark) {
 			nfm = nfsd_file_mark_get(container_of(mark,
 						 struct nfsd_file_mark,
 						 nfm_mark));
-			fsnotify_group_unlock(nfsd_file_fsnotify_group);
+			fsnotify_group_unlock(group);
 			if (nfm) {
 				fsnotify_put_mark(mark);
 				break;
 			}
 			/* Avoid soft lockup race with nfsd_file_mark_put() */
-			fsnotify_destroy_mark(mark, nfsd_file_fsnotify_group);
+			fsnotify_destroy_mark(mark, group);
 			fsnotify_put_mark(mark);
 		} else {
-			fsnotify_group_unlock(nfsd_file_fsnotify_group);
+			fsnotify_group_unlock(group);
 		}
 
 		/* allocate a new nfm */
 		new = kmem_cache_alloc(nfsd_file_mark_slab, GFP_KERNEL);
 		if (!new)
 			return NULL;
-		fsnotify_init_mark(&new->nfm_mark, nfsd_file_fsnotify_group);
+		fsnotify_init_mark(&new->nfm_mark, group);
 		new->nfm_mark.mask = FS_ATTRIB|FS_DELETE_SELF;
 		refcount_set(&new->nfm_ref, 1);
+		mutex_init(&new->nfm_recalc_mutex);
 
 		err = fsnotify_add_inode_mark(&new->nfm_mark, inode, 0);
 
@@ -327,8 +337,11 @@ static void nfsd_file_lru_add(struct nfsd_file *nf)
 	refcount_inc(&nf->nf_ref);
 	if (list_lru_add_obj(&nfsd_file_lru, &nf->nf_lru))
 		trace_nfsd_file_lru_add(nf);
-	else
-		WARN_ON(1);
+	else {
+		refcount_dec(&nf->nf_ref);
+		WARN_ON_ONCE(1);
+		return;
+	}
 	nfsd_file_schedule_laundrette();
 }
 
@@ -419,25 +432,31 @@ nfsd_file_dispose_list(struct list_head *dispose)
 }
 
 /**
- * nfsd_file_dispose_list_delayed - move list of dead files to net's freeme list
+ * nfsd_file_dispose_list_delayed - queue dead files for nfsd thread disposal
  * @dispose: list of nfsd_files to be disposed
  *
- * Transfers each file to the "freeme" list for its nfsd_net, to eventually
- * be disposed of by the per-net garbage collector.
+ * Transfers each file to the dispose list in its nfsd_net and wakes an nfsd
+ * thread to do the actual close.  This keeps the cost of fput() in the nfsd
+ * threads rather than in the shrinker or GC worker.
+ *
+ * All callers must hold nfsd_gc_lock, so that nfsd_file_cache_shutdown_net()
+ * can synchronize against them before draining the per-net dispose list.
+ * This guarantees nf_net is still live when we call net_generic().
  */
 static void
 nfsd_file_dispose_list_delayed(struct list_head *dispose)
 {
-	while(!list_empty(dispose)) {
+	lockdep_assert_held(&nfsd_gc_lock);
+
+	while (!list_empty(dispose)) {
 		struct nfsd_file *nf = list_first_entry(dispose,
 						struct nfsd_file, nf_gc);
 		struct nfsd_net *nn = net_generic(nf->nf_net, nfsd_net_id);
-		struct nfsd_fcache_disposal *l = nn->fcache_disposal;
 		struct svc_serv *serv;
 
-		spin_lock(&l->lock);
-		list_move_tail(&nf->nf_gc, &l->freeme);
-		spin_unlock(&l->lock);
+		spin_lock(&nn->fcache_dispose_lock);
+		list_move_tail(&nf->nf_gc, &nn->fcache_dispose_list);
+		spin_unlock(&nn->fcache_dispose_lock);
 
 		/*
 		 * The filecache laundrette is shut down after the
@@ -461,21 +480,28 @@ nfsd_file_dispose_list_delayed(struct list_head *dispose)
  */
 void nfsd_file_net_dispose(struct nfsd_net *nn)
 {
-	struct nfsd_fcache_disposal *l = nn->fcache_disposal;
-
-	if (!list_empty(&l->freeme)) {
+	if (!list_empty(&nn->fcache_dispose_list)) {
 		LIST_HEAD(dispose);
 		int i;
 
-		spin_lock(&l->lock);
-		for (i = 0; i < 8 && !list_empty(&l->freeme); i++)
-			list_move(l->freeme.next, &dispose);
-		spin_unlock(&l->lock);
-		if (!list_empty(&l->freeme))
-			/* Wake up another thread to share the work
+		spin_lock(&nn->fcache_dispose_lock);
+		for (i = 0; i < 8 && !list_empty(&nn->fcache_dispose_list); i++)
+			list_move(nn->fcache_dispose_list.next, &dispose);
+		spin_unlock(&nn->fcache_dispose_lock);
+		if (!list_empty(&nn->fcache_dispose_list)) {
+			/*
+			 * Wake up another thread to share the work
 			 * *before* doing any actual disposing.
+			 *
+			 * The filecache laundrette is shut down after
+			 * the nn->nfsd_serv pointer is cleared, but
+			 * before the svc_serv is freed.
 			 */
-			svc_wake_up(nn->nfsd_serv);
+			struct svc_serv *serv = nn->nfsd_serv;
+
+			if (serv)
+				svc_wake_up(serv);
+		}
 		nfsd_file_dispose_list(&dispose);
 	}
 }
@@ -552,13 +578,6 @@ nfsd_file_gc_cb(struct list_head *item, struct list_lru_one *lru,
 	return nfsd_file_lru_cb(item, lru, arg);
 }
 
-/* If the shrinker runs between calls to list_lru_walk_node() in
- * nfsd_file_gc(), the "remaining" count will be wrong.  This could
- * result in premature freeing of some files.  This may not matter much
- * but is easy to fix with this spinlock which temporarily disables
- * the shrinker.
- */
-static DEFINE_SPINLOCK(nfsd_gc_lock);
 static void
 nfsd_file_gc(void)
 {
@@ -581,9 +600,9 @@ nfsd_file_gc(void)
 				remaining = 0;
 		}
 	}
+	nfsd_file_dispose_list_delayed(&dispose);
 	spin_unlock(&nfsd_gc_lock);
 	trace_nfsd_file_gc_removed(ret, list_lru_count(&nfsd_file_lru));
-	nfsd_file_dispose_list_delayed(&dispose);
 }
 
 static void
@@ -611,9 +630,9 @@ nfsd_file_lru_scan(struct shrinker *s, struct shrink_control *sc)
 
 	ret = list_lru_shrink_walk(&nfsd_file_lru, sc,
 				   nfsd_file_lru_cb, &dispose);
+	nfsd_file_dispose_list_delayed(&dispose);
 	spin_unlock(&nfsd_gc_lock);
 	trace_nfsd_file_shrinker_removed(ret, list_lru_count(&nfsd_file_lru));
-	nfsd_file_dispose_list_delayed(&dispose);
 	return ret;
 }
 
@@ -686,11 +705,11 @@ nfsd_file_queue_for_close(struct inode *inode, struct list_head *dispose)
 }
 
 /**
- * nfsd_file_close_inode - attempt a delayed close of a nfsd_file
+ * nfsd_file_close_inode - attempt a deferred close of a nfsd_file
  * @inode: inode of the file to attempt to remove
  *
  * Close out any open nfsd_files that can be reaped for @inode. The
- * actual freeing is deferred to the dispose_list_delayed infrastructure.
+ * actual freeing is deferred to the nfsd service threads.
  *
  * This is used by the fsnotify callbacks and setlease notifier.
  */
@@ -699,8 +718,10 @@ nfsd_file_close_inode(struct inode *inode)
 {
 	LIST_HEAD(dispose);
 
+	spin_lock(&nfsd_gc_lock);
 	nfsd_file_queue_for_close(inode, &dispose);
 	nfsd_file_dispose_list_delayed(&dispose);
+	spin_unlock(&nfsd_gc_lock);
 }
 
 /**
@@ -812,12 +833,36 @@ nfsd_file_fsnotify_handle_event(struct fsnotify_mark *mark, u32 mask,
 	return 0;
 }
 
+#ifdef CONFIG_NFSD_V4
+static int
+nfsd_dir_fsnotify_handle_event(struct fsnotify_group *group, u32 mask,
+			       const void *data, int data_type, struct inode *dir,
+			       const struct qstr *name, u32 cookie,
+			       struct fsnotify_iter_info *iter_info)
+{
+	return nfsd_handle_dir_event(mask, dir, data, data_type, name);
+}
+#else
+static int
+nfsd_dir_fsnotify_handle_event(struct fsnotify_group *group, u32 mask,
+			       const void *data, int data_type, struct inode *dir,
+			       const struct qstr *name, u32 cookie,
+			       struct fsnotify_iter_info *iter_info)
+{
+	return 0;
+}
+#endif
 
 static const struct fsnotify_ops nfsd_file_fsnotify_ops = {
 	.handle_inode_event = nfsd_file_fsnotify_handle_event,
 	.free_mark = nfsd_file_mark_free,
 };
 
+static const struct fsnotify_ops nfsd_dir_fsnotify_ops = {
+	.handle_event = nfsd_dir_fsnotify_handle_event,
+	.free_mark = nfsd_file_mark_free,
+};
+
 int
 nfsd_file_cache_init(void)
 {
@@ -869,8 +914,7 @@ nfsd_file_cache_init(void)
 		goto out_shrinker;
 	}
 
-	nfsd_file_fsnotify_group = fsnotify_alloc_group(&nfsd_file_fsnotify_ops,
-							0);
+	nfsd_file_fsnotify_group = fsnotify_alloc_group(&nfsd_file_fsnotify_ops, 0);
 	if (IS_ERR(nfsd_file_fsnotify_group)) {
 		pr_err("nfsd: unable to create fsnotify group: %ld\n",
 			PTR_ERR(nfsd_file_fsnotify_group));
@@ -879,11 +923,23 @@ nfsd_file_cache_init(void)
 		goto out_notifier;
 	}
 
+	nfsd_dir_fsnotify_group = fsnotify_alloc_group(&nfsd_dir_fsnotify_ops, 0);
+	if (IS_ERR(nfsd_dir_fsnotify_group)) {
+		pr_err("nfsd: unable to create fsnotify group: %ld\n",
+			PTR_ERR(nfsd_dir_fsnotify_group));
+		ret = PTR_ERR(nfsd_dir_fsnotify_group);
+		nfsd_dir_fsnotify_group = NULL;
+		goto out_notify_group;
+	}
+
 	INIT_DELAYED_WORK(&nfsd_filecache_laundrette, nfsd_file_gc_worker);
 out:
 	if (ret)
 		clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags);
 	return ret;
+out_notify_group:
+	fsnotify_put_group(nfsd_file_fsnotify_group);
+	nfsd_file_fsnotify_group = NULL;
 out_notifier:
 	lease_unregister_notifier(&nfsd_file_lease_notifier);
 out_shrinker:
@@ -940,42 +996,14 @@ __nfsd_file_cache_purge(struct net *net)
 	nfsd_file_dispose_list(&dispose);
 }
 
-static struct nfsd_fcache_disposal *
-nfsd_alloc_fcache_disposal(void)
-{
-	struct nfsd_fcache_disposal *l;
-
-	l = kmalloc_obj(*l);
-	if (!l)
-		return NULL;
-	spin_lock_init(&l->lock);
-	INIT_LIST_HEAD(&l->freeme);
-	return l;
-}
-
-static void
-nfsd_free_fcache_disposal(struct nfsd_fcache_disposal *l)
-{
-	nfsd_file_dispose_list(&l->freeme);
-	kfree(l);
-}
-
-static void
-nfsd_free_fcache_disposal_net(struct net *net)
-{
-	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
-	struct nfsd_fcache_disposal *l = nn->fcache_disposal;
-
-	nfsd_free_fcache_disposal(l);
-}
-
 int
 nfsd_file_cache_start_net(struct net *net)
 {
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
-	nn->fcache_disposal = nfsd_alloc_fcache_disposal();
-	return nn->fcache_disposal ? 0 : -ENOMEM;
+	spin_lock_init(&nn->fcache_dispose_lock);
+	INIT_LIST_HEAD(&nn->fcache_dispose_list);
+	return 0;
 }
 
 /**
@@ -994,8 +1022,18 @@ nfsd_file_cache_purge(struct net *net)
 void
 nfsd_file_cache_shutdown_net(struct net *net)
 {
+	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
+
 	nfsd_file_cache_purge(net);
-	nfsd_free_fcache_disposal_net(net);
+	/*
+	 * Ensure any in-progress shrinker, GC, or fsnotify/lease callback
+	 * (all of which hold nfsd_gc_lock while calling
+	 * nfsd_file_dispose_list_delayed()) has fully completed before
+	 * draining the per-net dispose list.
+	 */
+	spin_lock(&nfsd_gc_lock);
+	spin_unlock(&nfsd_gc_lock);
+	nfsd_file_dispose_list(&nn->fcache_dispose_list);
 }
 
 void
@@ -1019,6 +1057,8 @@ nfsd_file_cache_shutdown(void)
 	rcu_barrier();
 	fsnotify_put_group(nfsd_file_fsnotify_group);
 	nfsd_file_fsnotify_group = NULL;
+	fsnotify_put_group(nfsd_dir_fsnotify_group);
+	nfsd_dir_fsnotify_group = NULL;
 	kmem_cache_destroy(nfsd_file_slab);
 	nfsd_file_slab = NULL;
 	fsnotify_wait_marks_destroyed();
@@ -1223,11 +1263,9 @@ nfsd_file_do_acquire(struct svc_rqst *rqstp, struct net *net,
 open_file:
 	trace_nfsd_file_alloc(nf);
 
-	if (type == S_IFREG)
-		nf->nf_mark = nfsd_file_mark_find_or_create(inode);
-
-	if (type != S_IFREG || nf->nf_mark) {
-		if (file) {
+	nf->nf_mark = nfsd_file_mark_find_or_create(inode);
+	if (nf->nf_mark) {
+		if (file && (file->f_mode & FMODE_OPENED)) {
 			get_file(file);
 			nf->nf_file = file;
 			status = nfs_ok;
@@ -1374,12 +1412,12 @@ nfsd_file_acquire_local(struct net *net, struct svc_cred *cred,
  * @rqstp: the RPC transaction being executed
  * @fhp: the NFS filehandle of the file just created
  * @may_flags: NFSD_MAY_ settings for the file
- * @file: cached, already-open file (may be NULL)
+ * @file: cached, already-open file (may be NULL or not yet opened)
  * @pnf: OUT: new or found "struct nfsd_file" object
  *
  * Acquire a nfsd_file object that is not GC'ed. If one doesn't already exist,
- * and @file is non-NULL, use it to instantiate a new nfsd_file instead of
- * opening a new one.
+ * and @file has FMODE_OPENED set, use it to instantiate a new nfsd_file
+ * instead of opening a new one.
  *
  * Return values:
  *   %nfs_ok - @pnf points to an nfsd_file with its reference
@@ -1474,3 +1512,54 @@ int nfsd_file_cache_stats_show(struct seq_file *m, void *v)
 		seq_printf(m, "mean age (ms): -\n");
 	return 0;
 }
+
+/**
+ * nfsd_fsnotify_recalc_mask - recalculate the fsnotify mask for a nfsd_file
+ * @nf: nfsd_file to recalculate the mask on
+ *
+ * When a directory nfsd_file has a delegation added or removed, that may
+ * change the events that nfsd requires from the VFS layer. This function
+ * recalculates the fsnotify mask based on the leases present.
+ */
+void nfsd_fsnotify_recalc_mask(struct nfsd_file *nf)
+{
+	struct inode *inode = file_inode(nf->nf_file);
+	u32 lease_mask, set = 0, clear = 0;
+	struct fsnotify_mark *mark;
+
+	/* This is only needed when adding or removing dir delegs */
+	if (!S_ISDIR(inode->i_mode) || !nf->nf_mark)
+		return;
+
+	mark = &nf->nf_mark->nfm_mark;
+
+	/*
+	 * The mark is shared by every nfsd_file on this inode, so concurrent
+	 * delegation add/remove on the same directory can recalc it in
+	 * parallel. Serialize the read of the lease state and the update of
+	 * the mark so that a recalc working from a stale snapshot of the
+	 * lease list can't clobber a concurrent recalc's update.
+	 */
+	mutex_lock(&nf->nf_mark->nfm_recalc_mutex);
+
+	/* Set up notifications for any ignored delegation events */
+	lease_mask = inode_lease_ignore_mask(inode);
+
+	if (lease_mask & FL_IGN_DIR_CREATE)
+		set |= FS_CREATE | FS_MOVED_TO;
+	else
+		clear |= FS_CREATE | FS_MOVED_TO;
+
+	if (lease_mask & FL_IGN_DIR_DELETE)
+		set |= FS_DELETE | FS_MOVED_FROM;
+	else
+		clear |= FS_DELETE | FS_MOVED_FROM;
+
+	if (lease_mask & FL_IGN_DIR_RENAME)
+		set |= FS_RENAME;
+	else
+		clear |= FS_RENAME;
+
+	fsnotify_modify_mark_mask(mark, set, clear);
+	mutex_unlock(&nf->nf_mark->nfm_recalc_mutex);
+}
diff --git a/fs/nfsd/filecache.h b/fs/nfsd/filecache.h
index 683b6437cacc2..b224902b438d7 100644
--- a/fs/nfsd/filecache.h
+++ b/fs/nfsd/filecache.h
@@ -26,6 +26,8 @@
 struct nfsd_file_mark {
 	struct fsnotify_mark	nfm_mark;
 	refcount_t		nfm_ref;
+	/* serializes nfsd_fsnotify_recalc_mask() against itself */
+	struct mutex		nfm_recalc_mutex;
 };
 
 /*
@@ -86,4 +88,5 @@ __be32 nfsd_file_acquire_local(struct net *net, struct svc_cred *cred,
 __be32 nfsd_file_acquire_dir(struct svc_rqst *rqstp, struct svc_fh *fhp,
 		  struct nfsd_file **pnf);
 int nfsd_file_cache_stats_show(struct seq_file *m, void *v);
+void nfsd_fsnotify_recalc_mask(struct nfsd_file *nf);
 #endif /* _FS_NFSD_FILECACHE_H */
diff --git a/fs/nfsd/flexfilelayoutxdr.c b/fs/nfsd/flexfilelayoutxdr.c
index f9f7e38cba13f..374e52d3064a6 100644
--- a/fs/nfsd/flexfilelayoutxdr.c
+++ b/fs/nfsd/flexfilelayoutxdr.c
@@ -30,19 +30,24 @@ nfsd4_ff_encode_layoutget(struct xdr_stream *xdr,
 	struct ff_idmap uid;
 	struct ff_idmap gid;
 
-	fh_len = 4 + fl->fh.size;
+	fh_len = 4 + xdr_align_size(fl->fh.size);
 
 	uid.len = sprintf(uid.buf, "%u", from_kuid(&init_user_ns, fl->uid));
 	gid.len = sprintf(gid.buf, "%u", from_kgid(&init_user_ns, fl->gid));
 
-	/* 8 + len for recording the length, name, and padding */
-	ds_len = 20 + sizeof(stateid_opaque_t) + 4 + fh_len +
-		 8 + uid.len + 8 + gid.len;
+	/* data server entry: deviceid + efficiency + stateid + fh list +
+	 * user + group + flags + stats_collect_hint
+	 */
+	ds_len = 16 + 4 + 4 + sizeof(stateid_opaque_t) + 4 + fh_len +
+		 4 + xdr_align_size(uid.len) +
+		 4 + xdr_align_size(gid.len) +
+		 4 + 4;
 
+	/* mirror: ds_count + ds */
 	mirror_len = 4 + ds_len;
 
-	/* The layout segment */
-	len = 20 + mirror_len;
+	/* stripe_unit + mirror_count + mirror */
+	len = 12 + mirror_len;
 
 	p = xdr_reserve_space(xdr, sizeof(__be32) + len);
 	if (!p)
@@ -94,7 +99,8 @@ nfsd4_ff_encode_getdeviceinfo(struct xdr_stream *xdr,
 	}
 
 	/* len + padding for two strings */
-	addr_len = 16 + da->netaddr.netid_len + da->netaddr.addr_len;
+	addr_len = 8 + xdr_align_size(da->netaddr.netid_len) +
+		   xdr_align_size(da->netaddr.addr_len);
 	ver_len = 20;
 
 	len = 4 + ver_len + 4 + addr_len;
diff --git a/fs/nfsd/localio.c b/fs/nfsd/localio.c
index be710d809a3ba..c3eb0557b3e1b 100644
--- a/fs/nfsd/localio.c
+++ b/fs/nfsd/localio.c
@@ -97,11 +97,15 @@ nfsd_open_local_fh(struct net *net, struct auth_domain *dom,
 		}
 		nfsd_file_get(localio);
 	again:
+		rcu_read_lock();
 		new = unrcu_pointer(cmpxchg(pnf, NULL, RCU_INITIALIZER(localio)));
 		if (new) {
 			/* Some other thread installed an nfsd_file */
-			if (nfsd_file_get(new) == NULL)
+			if (nfsd_file_get(new) == NULL) {
+				rcu_read_unlock();
 				goto again;
+			}
+			rcu_read_unlock();
 			/*
 			 * Drop the ref we were going to install (both file and
 			 * net) and the one we were going to return (only file).
@@ -110,6 +114,8 @@ nfsd_open_local_fh(struct net *net, struct auth_domain *dom,
 			nfsd_net_put(net);
 			nfsd_file_put(localio);
 			localio = new;
+		} else {
+			rcu_read_unlock();
 		}
 	} else
 		nfsd_net_put(net);
diff --git a/fs/nfsd/lockd.c b/fs/nfsd/lockd.c
index 6fe1325815e0e..72a5b499839d8 100644
--- a/fs/nfsd/lockd.c
+++ b/fs/nfsd/lockd.c
@@ -92,6 +92,7 @@ nlm_fclose(struct file *filp)
 }
 
 static const struct nlmsvc_binding nfsd_nlm_ops = {
+	.owner		= THIS_MODULE,
 	.fopen		= nlm_fopen,		/* open file for locking */
 	.fclose		= nlm_fclose,		/* close file */
 };
@@ -100,11 +101,12 @@ void
 nfsd_lockd_init(void)
 {
 	dprintk("nfsd: initializing lockd\n");
-	nlmsvc_ops = &nfsd_nlm_ops;
+	rcu_assign_pointer(nlmsvc_ops, &nfsd_nlm_ops);
 }
 
 void
 nfsd_lockd_shutdown(void)
 {
-	nlmsvc_ops = NULL;
+	RCU_INIT_POINTER(nlmsvc_ops, NULL);
+	synchronize_rcu();
 }
diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h
index 27da1a3edacbf..03724bef10a79 100644
--- a/fs/nfsd/netns.h
+++ b/fs/nfsd/netns.h
@@ -28,6 +28,16 @@ struct cld_net;
 struct nfsd_net_cb;
 struct nfsd4_client_tracking_ops;
 
+enum nfsd_net_flag {
+	NFSD_NET_GRACE_ENDED,
+	NFSD_NET_GRACE_END_FORCED,
+	NFSD_NET_IN_GRACE,
+	NFSD_NET_SOMEBODY_RECLAIMED,
+	NFSD_NET_TRACK_RECLAIM_COMPLETES,
+	NFSD_NET_UP,
+	NFSD_NET_LOCKD_UP,
+};
+
 enum {
 	/* cache misses due only to checksum comparison failures */
 	NFSD_STATS_PAYLOAD_MISSES,
@@ -66,9 +76,9 @@ struct nfsd_net {
 	struct cache_detail *nametoid_cache;
 
 	struct lock_manager nfsd4_manager;
-	bool grace_ended;
-	bool grace_end_forced;
+	unsigned long flags;
 	time64_t boot_time;
+	time64_t boot_time_bt;	/* same instant in CLOCK_BOOTTIME */
 
 	struct dentry *nfsd_client_dir;
 
@@ -84,6 +94,7 @@ struct nfsd_net {
 	 */
 	struct list_head *reclaim_str_hashtbl;
 	int reclaim_str_hashtbl_size;
+	struct rw_semaphore reclaim_str_hashtbl_lock;
 	struct list_head *conf_id_hashtbl;
 	struct rb_root conf_name_tree;
 	struct list_head *unconf_id_hashtbl;
@@ -96,7 +107,10 @@ struct nfsd_net {
 	 * close_lru holds (open) stateowner queue ordered by nfs4_stateowner.so_time
 	 * for last close replay.
 	 *
-	 * All of the above fields are protected by the client_mutex.
+	 * reclaim_str_hashtbl[], reclaim_str_hashtbl_size are protected by
+	 * reclaim_str_hashtbl_lock.
+	 *
+	 * All of the remaining fields are protected by the client_lock.
 	 */
 	struct list_head client_lru;
 	struct list_head close_lru;
@@ -117,19 +131,13 @@ struct nfsd_net {
 	spinlock_t blocked_locks_lock;
 
 	struct file *rec_file;
-	bool in_grace;
 	const struct nfsd4_client_tracking_ops *client_tracking_ops;
 
 	time64_t nfsd4_lease;
 	time64_t nfsd4_grace;
-	bool somebody_reclaimed;
 
-	bool track_reclaim_completes;
 	atomic_t nr_reclaim_complete;
 
-	bool nfsd_net_up;
-	bool lockd_up;
-
 	seqlock_t writeverf_lock;
 	unsigned char writeverf[8];
 
@@ -209,7 +217,8 @@ struct nfsd_net {
 	/* utsname taken from the process that starts the server */
 	char			nfsd_name[UNX_MAXNODENAME+1];
 
-	struct nfsd_fcache_disposal *fcache_disposal;
+	spinlock_t		fcache_dispose_lock;
+	struct list_head	fcache_dispose_list;
 
 	siphash_key_t		siphash_key;
 
diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c
index 76305b86c1a95..827f90194c436 100644
--- a/fs/nfsd/nfs2acl.c
+++ b/fs/nfsd/nfs2acl.c
@@ -115,14 +115,19 @@ static __be32 nfsacld_proc_setacl(struct svc_rqst *rqstp)
 
 	inode_lock(inode);
 
-	error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, ACL_TYPE_ACCESS,
-			      argp->acl_access);
-	if (error)
-		goto out_drop_lock;
-	error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, ACL_TYPE_DEFAULT,
-			      argp->acl_default);
-	if (error)
-		goto out_drop_lock;
+	error = 0;
+	if (argp->mask & NFS_ACL) {
+		error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry,
+				      ACL_TYPE_ACCESS, argp->acl_access);
+		if (error)
+			goto out_drop_lock;
+	}
+	if (argp->mask & NFS_DFACL) {
+		error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry,
+				      ACL_TYPE_DEFAULT, argp->acl_default);
+		if (error)
+			goto out_drop_lock;
+	}
 
 	inode_unlock(inode);
 
diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c
index e87731380be85..a87f9d7f32be2 100644
--- a/fs/nfsd/nfs3acl.c
+++ b/fs/nfsd/nfs3acl.c
@@ -105,12 +105,17 @@ static __be32 nfsd3_proc_setacl(struct svc_rqst *rqstp)
 
 	inode_lock(inode);
 
-	error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, ACL_TYPE_ACCESS,
-			      argp->acl_access);
-	if (error)
-		goto out_drop_lock;
-	error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, ACL_TYPE_DEFAULT,
-			      argp->acl_default);
+	error = 0;
+	if (argp->mask & NFS_ACL) {
+		error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry,
+				      ACL_TYPE_ACCESS, argp->acl_access);
+		if (error)
+			goto out_drop_lock;
+	}
+	if (argp->mask & NFS_DFACL) {
+		error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry,
+				      ACL_TYPE_DEFAULT, argp->acl_default);
+	}
 
 out_drop_lock:
 	inode_unlock(inode);
diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c
index aeda7a802bdf7..617a70d132926 100644
--- a/fs/nfsd/nfs3proc.c
+++ b/fs/nfsd/nfs3proc.c
@@ -29,6 +29,25 @@ static int	nfs3_ftypes[] = {
 	S_IFIFO,		/* NF3FIFO */
 };
 
+/*
+ * Reject a client-supplied atime or mtime whose nanoseconds field is out
+ * of range. Such a value is well-formed on the wire but is not a valid
+ * timespec64, and storing it verbatim can corrupt on-disk timestamps.
+ * tv_nsec is a long, so it is cast to unsigned long (the same width) to
+ * catch both an over-large value and one that became negative when an
+ * out-of-range u32 wire nseconds was assigned to a 32-bit long.
+ */
+static bool nfsd3_time_in_range(const struct iattr *iap)
+{
+	if ((iap->ia_valid & ATTR_ATIME_SET) &&
+	    (unsigned long)iap->ia_atime.tv_nsec >= NSEC_PER_SEC)
+		return false;
+	if ((iap->ia_valid & ATTR_MTIME_SET) &&
+	    (unsigned long)iap->ia_mtime.tv_nsec >= NSEC_PER_SEC)
+		return false;
+	return true;
+}
+
 static __be32 nfsd3_map_status(__be32 status)
 {
 	switch (status) {
@@ -101,9 +120,14 @@ nfsd3_proc_setattr(struct svc_rqst *rqstp)
 				SVCFH_fmt(&argp->fh));
 
 	fh_copy(&resp->fh, &argp->fh);
+	if (!nfsd3_time_in_range(&argp->attrs)) {
+		resp->status = nfserr_inval;
+		goto out;
+	}
 	if (argp->check_guard)
 		guardtime = &argp->guardtime;
 	resp->status = nfsd_setattr(rqstp, &resp->fh, &attrs, guardtime);
+out:
 	resp->status = nfsd3_map_status(resp->status);
 	return rpc_success;
 }
@@ -265,6 +289,8 @@ nfsd3_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp,
 
 	trace_nfsd_vfs_create(rqstp, fhp, S_IFREG, argp->name, argp->len);
 
+	if (!nfsd3_time_in_range(iap))
+		return nfserr_inval;
 	if (isdotent(argp->name, argp->len))
 		return nfserr_exist;
 	if (!(iap->ia_valid & ATTR_MODE))
@@ -400,8 +426,13 @@ nfsd3_proc_mkdir(struct svc_rqst *rqstp)
 	argp->attrs.ia_valid &= ~ATTR_SIZE;
 	fh_copy(&resp->dirfh, &argp->fh);
 	fh_init(&resp->fh, NFS3_FHSIZE);
+	if (!nfsd3_time_in_range(&argp->attrs)) {
+		resp->status = nfserr_inval;
+		goto out;
+	}
 	resp->status = nfsd_create(rqstp, &resp->dirfh, argp->name, argp->len,
 				   &attrs, S_IFDIR, 0, &resp->fh);
+out:
 	resp->status = nfsd3_map_status(resp->status);
 	return rpc_success;
 }
@@ -415,6 +446,10 @@ nfsd3_proc_symlink(struct svc_rqst *rqstp)
 		.na_iattr	= &argp->attrs,
 	};
 
+	if (!nfsd3_time_in_range(&argp->attrs)) {
+		resp->status = nfserr_inval;
+		goto out;
+	}
 	if (argp->tlen == 0) {
 		resp->status = nfserr_inval;
 		goto out;
@@ -471,6 +506,11 @@ nfsd3_proc_mknod(struct svc_rqst *rqstp)
 		goto out;
 	}
 
+	if (!nfsd3_time_in_range(&argp->attrs)) {
+		resp->status = nfserr_inval;
+		goto out;
+	}
+
 	type = nfs3_ftypes[argp->ftype];
 	resp->status = nfsd_create(rqstp, &resp->dirfh, argp->name, argp->len,
 				   &attrs, type, rdev, &resp->fh);
diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c
index 50827405468d2..71dcb448fa0ab 100644
--- a/fs/nfsd/nfs4callback.c
+++ b/fs/nfsd/nfs4callback.c
@@ -108,6 +108,8 @@ static int decode_cb_fattr4(struct xdr_stream *xdr, uint32_t *bitmap,
 
 		if (!xdrgen_decode_fattr4_time_deleg_access(xdr, &access))
 			return -EIO;
+		if (access.nseconds >= NSEC_PER_SEC)
+			return -EIO;
 		fattr->ncf_cb_atime.tv_sec = access.seconds;
 		fattr->ncf_cb_atime.tv_nsec = access.nseconds;
 
@@ -117,6 +119,8 @@ static int decode_cb_fattr4(struct xdr_stream *xdr, uint32_t *bitmap,
 
 		if (!xdrgen_decode_fattr4_time_deleg_modify(xdr, &modify))
 			return -EIO;
+		if (modify.nseconds >= NSEC_PER_SEC)
+			return -EIO;
 		fattr->ncf_cb_mtime.tv_sec = modify.seconds;
 		fattr->ncf_cb_mtime.tv_nsec = modify.nseconds;
 
@@ -456,13 +460,20 @@ static void encode_cb_sequence4args(struct xdr_stream *xdr,
 				    const struct nfsd4_callback *cb,
 				    struct nfs4_cb_compound_hdr *hdr)
 {
-	struct nfsd4_session *session = cb->cb_clp->cl_cb_session;
+	struct nfsd4_session *session;
 	struct nfsd4_referring_call_list *rcl;
 	__be32 *p;
 
 	if (hdr->minorversion == 0)
 		return;
 
+	rcu_read_lock();
+	session = rcu_dereference(cb->cb_clp->cl_cb_session);
+	if (!session) {
+		rcu_read_unlock();
+		return;
+	}
+
 	encode_nfs_cb_opnum4(xdr, OP_CB_SEQUENCE);
 	encode_sessionid4(xdr, session);
 
@@ -478,6 +489,7 @@ static void encode_cb_sequence4args(struct xdr_stream *xdr,
 		encode_referring_call_list4(xdr, rcl);
 
 	hdr->nops++;
+	rcu_read_unlock();
 }
 
 static void update_cb_slot_table(struct nfsd4_session *ses, u32 target)
@@ -529,21 +541,32 @@ static void update_cb_slot_table(struct nfsd4_session *ses, u32 target)
 static int decode_cb_sequence4resok(struct xdr_stream *xdr,
 				    struct nfsd4_callback *cb)
 {
-	struct nfsd4_session *session = cb->cb_clp->cl_cb_session;
+	struct nfsd4_session *session;
 	int status = -ESERVERFAULT;
 	__be32 *p;
 	u32 seqid, slotid, target;
 
+	rcu_read_lock();
+	session = rcu_dereference(cb->cb_clp->cl_cb_session);
+	if (!session) {
+		rcu_read_unlock();
+		cb->cb_seq_status = -NFS4ERR_BADSESSION;
+		return -NFS4ERR_BADSESSION;
+	}
+
 	/*
 	 * If the server returns different values for sessionID, slotID or
 	 * sequence number, the server is looney tunes.
 	 */
 	p = xdr_inline_decode(xdr, NFS4_MAX_SESSIONID_LEN + 4 + 4 + 4 + 4);
-	if (unlikely(p == NULL))
+	if (unlikely(p == NULL)) {
+		rcu_read_unlock();
 		goto out_overflow;
+	}
 
 	if (memcmp(p, session->se_sessionid.data, NFS4_MAX_SESSIONID_LEN)) {
 		dprintk("NFS: %s Invalid session id\n", __func__);
+		rcu_read_unlock();
 		goto out;
 	}
 	p += XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN);
@@ -551,12 +574,14 @@ static int decode_cb_sequence4resok(struct xdr_stream *xdr,
 	seqid = be32_to_cpup(p++);
 	if (seqid != session->se_cb_seq_nr[cb->cb_held_slot]) {
 		dprintk("NFS: %s Invalid sequence number\n", __func__);
+		rcu_read_unlock();
 		goto out;
 	}
 
 	slotid = be32_to_cpup(p++);
 	if (slotid != cb->cb_held_slot) {
 		dprintk("NFS: %s Invalid slotid\n", __func__);
+		rcu_read_unlock();
 		goto out;
 	}
 
@@ -564,6 +589,7 @@ static int decode_cb_sequence4resok(struct xdr_stream *xdr,
 
 	target = be32_to_cpup(p++);
 	update_cb_slot_table(session, target);
+	rcu_read_unlock();
 	status = 0;
 out:
 	cb->cb_seq_status = status;
@@ -865,6 +891,84 @@ static void encode_stateowner(struct xdr_stream *xdr, struct nfs4_stateowner *so
 	xdr_encode_opaque(p, so->so_owner.data, so->so_owner.len);
 }
 
+static void nfs4_xdr_enc_cb_notify(struct rpc_rqst *req,
+				   struct xdr_stream *xdr,
+				   const void *data)
+{
+	const struct nfsd4_callback *cb = data;
+	struct nfsd4_cb_notify *ncn = container_of(cb, struct nfsd4_cb_notify, ncn_cb);
+	struct nfs4_delegation *dp = container_of(ncn, struct nfs4_delegation, dl_cb_notify);
+	struct nfs4_cb_compound_hdr hdr = {
+		.ident = 0,
+		.minorversion = cb->cb_clp->cl_minorversion,
+	};
+	struct CB_NOTIFY4args args;
+	unsigned int start;
+
+	WARN_ON_ONCE(hdr.minorversion == 0);
+
+	encode_cb_compound4args(xdr, &hdr);
+	encode_cb_sequence4args(xdr, cb, &hdr);
+
+	/*
+	 * nfsd4_cb_notify_prepare() sized the payload against a single page,
+	 * but did not account for the compound, sequence, stateid, and
+	 * filehandle encoded here. If the variable-length encode overflows the
+	 * backchannel send buffer, roll back to before the operation so that a
+	 * truncated CB_NOTIFY is never placed on the wire.
+	 */
+	start = xdr_stream_pos(xdr);
+
+	if (xdr_stream_encode_u32(xdr, OP_CB_NOTIFY) < 0)
+		goto out_err;
+
+	args.cna_stateid.seqid = dp->dl_stid.sc_stateid.si_generation;
+	memcpy(&args.cna_stateid.other, &dp->dl_stid.sc_stateid.si_opaque,
+	       ARRAY_SIZE(args.cna_stateid.other));
+	args.cna_fh.len = dp->dl_stid.sc_file->fi_fhandle.fh_size;
+	args.cna_fh.data = dp->dl_stid.sc_file->fi_fhandle.fh_raw;
+	args.cna_changes.count = ncn->ncn_nf_cnt;
+	args.cna_changes.element = ncn->ncn_nf;
+	if (!xdrgen_encode_CB_NOTIFY4args(xdr, &args))
+		goto out_err;
+
+	hdr.nops++;
+	encode_cb_nops(&hdr);
+	return;
+
+out_err:
+	/*
+	 * Drop the CB_NOTIFY op and emit a valid CB_SEQUENCE-only compound so
+	 * the client still advances its slot. Flag the failure so the done
+	 * handler recalls the delegation and the missed notification is not
+	 * silently lost. The flag is written here in the transmit path and read
+	 * in the done handler; the two are serialized phases of the same
+	 * rpc_task, so no additional barrier is needed.
+	 */
+	ncn->ncn_encode_err = true;
+	xdr_truncate_encode(xdr, start);
+	encode_cb_nops(&hdr);
+}
+
+static int nfs4_xdr_dec_cb_notify(struct rpc_rqst *rqstp,
+				  struct xdr_stream *xdr,
+				  void *data)
+{
+	struct nfsd4_callback *cb = data;
+	struct nfs4_cb_compound_hdr hdr;
+	int status;
+
+	status = decode_cb_compound4res(xdr, &hdr);
+	if (unlikely(status))
+		return status;
+
+	status = decode_cb_sequence4res(xdr, cb);
+	if (unlikely(status || cb->cb_seq_status))
+		return status;
+
+	return decode_cb_op_status(xdr, OP_CB_NOTIFY, &cb->cb_status);
+}
+
 static void nfs4_xdr_enc_cb_notify_lock(struct rpc_rqst *req,
 					struct xdr_stream *xdr,
 					const void *data)
@@ -1026,6 +1130,7 @@ static const struct rpc_procinfo nfs4_cb_procedures[] = {
 #ifdef CONFIG_NFSD_PNFS
 	PROC(CB_LAYOUT,	COMPOUND,	cb_layout,	cb_layout),
 #endif
+	PROC(CB_NOTIFY,		COMPOUND,	cb_notify,	cb_notify),
 	PROC(CB_NOTIFY_LOCK,	COMPOUND,	cb_notify_lock,	cb_notify_lock),
 	PROC(CB_OFFLOAD,	COMPOUND,	cb_offload,	cb_offload),
 	PROC(CB_RECALL_ANY,	COMPOUND,	cb_recall_any,	cb_recall_any),
@@ -1150,9 +1255,8 @@ static int setup_callback_client(struct nfs4_client *clp, struct nfs4_cb_conn *c
 	} else {
 		if (!conn->cb_xprt || !ses)
 			return -EINVAL;
-		clp->cl_cb_session = ses;
 		args.bc_xprt = conn->cb_xprt;
-		args.prognumber = clp->cl_cb_session->se_cb_prog;
+		args.prognumber = ses->se_cb_prog;
 		args.protocol = conn->cb_xprt->xpt_class->xcl_ident |
 				XPRT_TRANSPORT_BC;
 		args.authflavor = ses->se_cb_sec.flavor;
@@ -1170,8 +1274,10 @@ static int setup_callback_client(struct nfs4_client *clp, struct nfs4_cb_conn *c
 		return -ENOMEM;
 	}
 
-	if (clp->cl_minorversion != 0)
+	if (clp->cl_minorversion != 0) {
 		clp->cl_cb_conn.cb_xprt = conn->cb_xprt;
+		rcu_assign_pointer(clp->cl_cb_session, ses);
+	}
 	clp->cl_cb_client = client;
 	clp->cl_cb_cred = cred;
 	rcu_read_lock();
@@ -1278,18 +1384,33 @@ static int grab_slot(struct nfsd4_session *ses)
 static bool nfsd41_cb_get_slot(struct nfsd4_callback *cb, struct rpc_task *task)
 {
 	struct nfs4_client *clp = cb->cb_clp;
-	struct nfsd4_session *ses = clp->cl_cb_session;
+	struct nfsd4_session *ses;
 
 	if (cb->cb_held_slot >= 0)
 		return true;
+
+	rcu_read_lock();
+	ses = rcu_dereference(clp->cl_cb_session);
+	if (!ses) {
+		rcu_read_unlock();
+		rpc_sleep_on(&clp->cl_cb_waitq, task, NULL);
+		return false;
+	}
 	cb->cb_held_slot = grab_slot(ses);
 	if (cb->cb_held_slot < 0) {
+		rcu_read_unlock();
 		rpc_sleep_on(&clp->cl_cb_waitq, task, NULL);
 		/* Race breaker */
-		cb->cb_held_slot = grab_slot(ses);
+		rcu_read_lock();
+		ses = rcu_dereference(clp->cl_cb_session);
+		if (ses)
+			cb->cb_held_slot = grab_slot(ses);
+		rcu_read_unlock();
 		if (cb->cb_held_slot < 0)
 			return false;
 		rpc_wake_up_queued_task(&clp->cl_cb_waitq, task);
+	} else {
+		rcu_read_unlock();
 	}
 	return true;
 }
@@ -1297,12 +1418,17 @@ static bool nfsd41_cb_get_slot(struct nfsd4_callback *cb, struct rpc_task *task)
 static void nfsd41_cb_release_slot(struct nfsd4_callback *cb)
 {
 	struct nfs4_client *clp = cb->cb_clp;
-	struct nfsd4_session *ses = clp->cl_cb_session;
+	struct nfsd4_session *ses;
 
 	if (cb->cb_held_slot >= 0) {
-		spin_lock(&ses->se_lock);
-		ses->se_cb_slot_avail |= BIT(cb->cb_held_slot);
-		spin_unlock(&ses->se_lock);
+		rcu_read_lock();
+		ses = rcu_dereference(clp->cl_cb_session);
+		if (ses) {
+			spin_lock(&ses->se_lock);
+			ses->se_cb_slot_avail |= BIT(cb->cb_held_slot);
+			spin_unlock(&ses->se_lock);
+		}
+		rcu_read_unlock();
 		cb->cb_held_slot = -1;
 		rpc_wake_up_next(&clp->cl_cb_waitq);
 	}
@@ -1319,6 +1445,16 @@ static void nfsd41_destroy_cb(struct nfsd4_callback *cb)
 	else
 		clear_bit(NFSD4_CALLBACK_RUNNING, &cb->cb_flags);
 
+	/*
+	 * Order the clear of NFSD4_CALLBACK_RUNNING above before the ->release()
+	 * callback below. A release op may re-check producer-side state to decide
+	 * whether to requeue itself (see nfsd4_cb_notify_release()), and that
+	 * check must not be reordered ahead of the clear. The plain clear_bit()
+	 * path carries no ordering; clear_and_wake_up_bit() already issues this
+	 * barrier internally, so the extra one is harmless there.
+	 */
+	smp_mb__after_atomic();
+
 	if (cb->cb_ops && cb->cb_ops->release)
 		cb->cb_ops->release(cb);
 	nfsd41_cb_inflight_end(clp);
@@ -1434,22 +1570,35 @@ static void nfsd4_cb_prepare(struct rpc_task *task, void *calldata)
 	trace_nfsd_cb_rpc_prepare(clp);
 	cb->cb_seq_status = 1;
 	cb->cb_status = 0;
-	if (minorversion && !nfsd41_cb_get_slot(cb, task))
-		return;
+	if (minorversion) {
+		if (!rcu_access_pointer(clp->cl_cb_session)) {
+			rpc_exit(task, -EIO);
+			return;
+		}
+		if (!nfsd41_cb_get_slot(cb, task))
+			return;
+	}
 	rpc_call_start(task);
 }
 
 /* Returns true if CB_COMPOUND processing should continue */
 static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback *cb)
 {
-	struct nfsd4_session *session = cb->cb_clp->cl_cb_session;
+	struct nfsd4_session *session;
 	bool ret = false;
 
 	if (cb->cb_held_slot < 0)
 		goto requeue;
 
+	rcu_read_lock();
+	session = rcu_dereference(cb->cb_clp->cl_cb_session);
+	if (!session) {
+		rcu_read_unlock();
+		goto requeue;
+	}
+
 	/* This is the operation status code for CB_SEQUENCE */
-	trace_nfsd_cb_seq_status(task, cb);
+	trace_nfsd_cb_seq_status(task, cb, session);
 	switch (cb->cb_seq_status) {
 	case 0:
 		/*
@@ -1481,12 +1630,16 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback
 		fallthrough;
 	case -NFS4ERR_BADSESSION:
 		nfsd4_mark_cb_fault(cb->cb_clp);
+		rcu_read_unlock();
 		goto requeue;
 	case -NFS4ERR_DELAY:
 		cb->cb_seq_status = 1;
-		if (RPC_SIGNALLED(task) || !rpc_restart_call(task))
+		if (RPC_SIGNALLED(task) || !rpc_restart_call(task)) {
+			rcu_read_unlock();
 			goto requeue;
+		}
 		rpc_delay(task, 2 * HZ);
+		rcu_read_unlock();
 		return false;
 	case -NFS4ERR_SEQ_MISORDERED:
 	case -NFS4ERR_BADSLOT:
@@ -1498,11 +1651,13 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback
 		 */
 		nfsd4_mark_cb_fault(cb->cb_clp);
 		cb->cb_held_slot = -1;
+		rcu_read_unlock();
 		goto retry_nowait;
 	default:
 		nfsd4_mark_cb_fault(cb->cb_clp);
 	}
-	trace_nfsd_cb_free_slot(task, cb);
+	trace_nfsd_cb_free_slot(task, cb, session);
+	rcu_read_unlock();
 	nfsd41_cb_release_slot(cb);
 	return ret;
 retry_nowait:
@@ -1624,7 +1779,15 @@ static struct nfsd4_conn * __nfsd4_find_backchannel(struct nfs4_client *clp)
  * Note there isn't a lot of locking in this code; instead we depend on
  * the fact that it is run from clp->cl_callback_wq, which won't run two
  * work items at once.  So, for example, clp->cl_callback_wq handles all
- * access of cl_cb_client and all calls to rpc_create or rpc_shutdown_client.
+ * access of cl_cb_client, and all calls to rpc_create or
+ * rpc_shutdown_client.
+ *
+ * cl_cb_session is written only from cl_callback_wq (via
+ * rcu_assign_pointer) and read from rpciod under rcu_read_lock (via
+ * rcu_dereference) by encode_cb_sequence4args(), decode_cb_sequence4resok(),
+ * nfsd4_cb_sequence_done(), and the cb-slot helpers.  Sessions are freed
+ * with kfree_rcu() so that rpciod readers in an RCU read-side critical
+ * section never dereference a freed session.
  */
 static void nfsd4_process_cb_update(struct nfsd4_callback *cb)
 {
@@ -1676,6 +1839,7 @@ static void nfsd4_process_cb_update(struct nfsd4_callback *cb)
 		nfsd4_mark_cb_down(clp);
 		if (c)
 			svc_xprt_put(c->cn_xprt);
+		rcu_assign_pointer(clp->cl_cb_session, ses);
 		return;
 	}
 }
@@ -1715,7 +1879,10 @@ nfsd4_run_cb_work(struct work_struct *work)
 
 	if (!test_and_clear_bit(NFSD4_CALLBACK_REQUEUE, &cb->cb_flags)) {
 		if (cb->cb_ops && cb->cb_ops->prepare)
-			cb->cb_ops->prepare(cb);
+			if (!cb->cb_ops->prepare(cb)) {
+				nfsd41_destroy_cb(cb);
+				return;
+			}
 	}
 
 	cb->cb_msg.rpc_cred = clp->cl_cb_cred;
diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c
index f34320e4c2f4a..22bcb6d09f703 100644
--- a/fs/nfsd/nfs4layouts.c
+++ b/fs/nfsd/nfs4layouts.c
@@ -247,13 +247,21 @@ nfsd4_alloc_layout_stateid(struct nfsd4_compound_state *cstate,
 	nfsd4_init_cb(&ls->ls_recall, clp, &nfsd4_cb_layout_ops,
 			NFSPROC4_CLNT_CB_LAYOUT);
 
-	if (parent->sc_type == SC_TYPE_DELEG)
-		ls->ls_file = nfsd_file_get(fp->fi_deleg_file);
-	else
+	if (parent->sc_type == SC_TYPE_DELEG) {
+		rcu_read_lock();
+		ls->ls_file = nfsd_file_get(rcu_dereference(fp->fi_deleg_file));
+		rcu_read_unlock();
+	} else {
 		ls->ls_file = find_any_file(fp);
-	BUG_ON(!ls->ls_file);
+	}
+
+	if (!ls->ls_file) {
+		nfs4_put_stid(stp);
+		return NULL;
+	}
 
 	ls->ls_fenced = false;
+	ls->ls_fence_inflight = false;
 	ls->ls_fence_delay = 0;
 	INIT_DELAYED_WORK(&ls->ls_fence_work, nfsd4_layout_fence_worker);
 
@@ -652,7 +660,7 @@ nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls, struct nfsd_file *file)
 	}
 }
 
-static void
+static bool
 nfsd4_cb_layout_prepare(struct nfsd4_callback *cb)
 {
 	struct nfs4_layout_stateid *ls =
@@ -661,6 +669,7 @@ nfsd4_cb_layout_prepare(struct nfsd4_callback *cb)
 	mutex_lock(&ls->ls_mutex);
 	nfs4_inc_and_copy_stateid(&ls->ls_recall_sid, &ls->ls_stid);
 	mutex_unlock(&ls->ls_mutex);
+	return true;
 }
 
 static int
@@ -791,15 +800,6 @@ nfsd4_layout_fence_worker(struct work_struct *work)
 	struct nfs4_client *clp;
 	struct nfsd_net *nn;
 
-	/*
-	 * The workqueue clears WORK_STRUCT_PENDING before invoking
-	 * this callback. Re-arm immediately so that
-	 * delayed_work_pending() returns true while the fence
-	 * operation is in progress, preventing
-	 * lm_breaker_timedout() from taking a duplicate reference.
-	 */
-	mod_delayed_work(system_dfl_wq, &ls->ls_fence_work, 0);
-
 	spin_lock(&ls->ls_lock);
 	if (list_empty(&ls->ls_layouts)) {
 		spin_unlock(&ls->ls_lock);
@@ -809,6 +809,9 @@ nfsd4_layout_fence_worker(struct work_struct *work)
 		nfsd4_close_layout(ls);
 
 		ls->ls_fenced = true;
+		spin_lock(&ls->ls_lock);
+		ls->ls_fence_inflight = false;
+		spin_unlock(&ls->ls_lock);
 		nfs4_put_stid(&ls->ls_stid);
 		return;
 	}
@@ -894,18 +897,26 @@ nfsd4_layout_lm_breaker_timedout(struct file_lease *fl)
 	if ((!nfsd4_layout_ops[ls->ls_layout_type]->fence_client) ||
 			ls->ls_fenced)
 		return true;
-	if (delayed_work_pending(&ls->ls_fence_work))
-		return false;
 	/*
 	 * Make sure layout has not been returned yet before
-	 * taking a reference count on the layout stateid.
+	 * taking a reference count on the layout stateid. The
+	 * ls_fence_inflight flag is set together with the sc_count
+	 * increment under ls_lock so that a fence worker invocation
+	 * already in progress (which has cleared WORK_STRUCT_PENDING
+	 * but not yet reached dispose:) cannot be coalesced with a
+	 * fresh schedule that takes an extra unmatched reference.
 	 */
 	spin_lock(&ls->ls_lock);
+	if (ls->ls_fence_inflight) {
+		spin_unlock(&ls->ls_lock);
+		return false;
+	}
 	if (list_empty(&ls->ls_layouts) ||
 			!refcount_inc_not_zero(&ls->ls_stid.sc_count)) {
 		spin_unlock(&ls->ls_lock);
 		return true;
 	}
+	ls->ls_fence_inflight = true;
 	spin_unlock(&ls->ls_lock);
 
 	mod_delayed_work(system_dfl_wq, &ls->ls_fence_work, 0);
diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 8561540ab2db9..c413ed0810b9c 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -306,10 +306,6 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp,
 			goto out;
 	}
 
-	status = fh_compose(resfhp, fhp->fh_export, child, fhp);
-	if (status != nfs_ok)
-		goto out;
-
 	v_mtime = 0;
 	v_atime = 0;
 	if (nfsd4_create_is_exclusive(open->op_createmode)) {
@@ -335,6 +331,10 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp,
 		if (status != nfs_ok)
 			goto out;
 
+		status = fh_compose(resfhp, fhp->fh_export, child, fhp);
+		if (status != nfs_ok)
+			goto out;
+
 		switch (open->op_createmode) {
 		case NFS4_CREATE_UNCHECKED:
 			if (!d_is_reg(child))
@@ -385,6 +385,10 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp,
 	open->op_created = true;
 	fh_fill_post_attrs(fhp);
 
+	status = fh_compose(resfhp, fhp->fh_export, child, fhp);
+	if (status != nfs_ok)
+		goto out;
+
 	/* A newly created file already has a file size of zero. */
 	if ((iap->ia_valid & ATTR_SIZE) && (iap->ia_size == 0))
 		iap->ia_valid &= ~ATTR_SIZE;
@@ -663,7 +667,7 @@ nfsd4_open(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		pr_warn("nfsd4_process_open2 failed to open newly-created file: status=%u\n",
 			be32_to_cpu(status));
 	if (reclaim && !status)
-		nn->somebody_reclaimed = true;
+		set_bit(NFSD_NET_SOMEBODY_RECLAIMED, &nn->flags);
 out:
 	if (open->op_filp) {
 		fput(open->op_filp);
@@ -677,8 +681,6 @@ nfsd4_open(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	nfsd4_cleanup_open_state(cstate, open);
 	nfsd4_bump_seqid(cstate, status);
 out_err:
-	posix_acl_release(open->op_dpacl);
-	posix_acl_release(open->op_pacl);
 	return status;
 }
 
@@ -700,6 +702,13 @@ static __be32 nfsd4_open_omfg(struct svc_rqst *rqstp, struct nfsd4_compound_stat
 	return nfsd4_open(rqstp, cstate, &op->u);
 }
 
+static void
+nfsd4_open_release(union nfsd4_op_u *u)
+{
+	posix_acl_release(u->open.op_dpacl);
+	posix_acl_release(u->open.op_pacl);
+}
+
 /*
  * filehandle-manipulating ops.
  */
@@ -839,6 +848,20 @@ nfsd4_create(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	if (status)
 		goto out_aftermask;
 
+	/* Sanitize cr_type to avoid returning ATTRNOTSUPP. */
+	switch (create->cr_type) {
+	case NF4LNK:
+	case NF4BLK:
+	case NF4CHR:
+	case NF4SOCK:
+	case NF4FIFO:
+	case NF4DIR:
+		break;
+	default:
+		status = nfserr_badtype;
+		goto out_aftermask;
+	}
+
 	if (create->cr_acl) {
 		if (attrs.na_dpacl || attrs.na_pacl) {
 			status = nfserr_inval;
@@ -846,6 +869,8 @@ nfsd4_create(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		}
 		status = nfsd4_acl_to_attr(create->cr_type, create->cr_acl,
 								&attrs);
+		if (status != nfs_ok)
+			goto out_aftermask;
 	}
 	current->fs->umask = create->cr_umask;
 	switch (create->cr_type) {
@@ -1253,7 +1278,7 @@ nfsd4_setattr(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 
 	if (deleg_attrs) {
 		status = nfserr_bad_stateid;
-		if (st->sc_type & SC_TYPE_DELEG) {
+		if (st && (st->sc_type & SC_TYPE_DELEG)) {
 			struct nfs4_delegation *dp = delegstateid(st);
 
 			/* Only for *_ATTRS_DELEG flavors */
@@ -1562,6 +1587,11 @@ static bool nfsd4_copy_on_sb(const struct nfsd4_copy *copy,
  * nfsd4_cancel_copy_by_sb - cancel async copy operations on @sb
  * @net: net namespace containing the copy operations
  * @sb: targeted superblock
+ *
+ * Context: Caller must hold nfsd_mutex with nn->nfsd_serv confirmed
+ *          non-NULL.  nfs4_state_destroy_net() frees conf_id_hashtbl
+ *          at server shutdown without clearing the pointer, so a
+ *          walk without these guarantees iterates freed slab memory.
  */
 void nfsd4_cancel_copy_by_sb(struct net *net, struct super_block *sb)
 {
@@ -1571,6 +1601,7 @@ void nfsd4_cancel_copy_by_sb(struct net *net, struct super_block *sb)
 	unsigned int idhashval;
 	LIST_HEAD(to_cancel);
 
+	lockdep_assert_held(&nfsd_mutex);
 	spin_lock(&nn->client_lock);
 	for (idhashval = 0; idhashval < CLIENT_HASH_SIZE; idhashval++) {
 		struct list_head *head = &nn->conf_id_hashtbl[idhashval];
@@ -1950,6 +1981,7 @@ static ssize_t _nfsd_copy_file_range(struct nfsd4_copy *copy,
 	/* See RFC 7862 p.67: */
 	if (bytes_total == 0)
 		bytes_total = ULLONG_MAX;
+	since = READ_ONCE(dst->f_wb_err);
 	do {
 		/* Only async copies can be stopped here */
 		if (kthread_should_stop())
@@ -1965,13 +1997,14 @@ static ssize_t _nfsd_copy_file_range(struct nfsd4_copy *copy,
 	} while (bytes_total > 0 && nfsd4_copy_is_async(copy));
 	/* for a non-zero asynchronous copy do a commit of data */
 	if (nfsd4_copy_is_async(copy) && copy->cp_res.wr_bytes_written > 0) {
-		since = READ_ONCE(dst->f_wb_err);
 		end = copy->cp_dst_pos + copy->cp_res.wr_bytes_written - 1;
 		status = vfs_fsync_range(dst, copy->cp_dst_pos, end, 0);
 		if (!status)
 			status = filemap_check_wb_err(dst->f_mapping, since);
 		if (!status)
 			set_bit(NFSD4_COPY_F_COMMITTED, &copy->cp_flags);
+		else if (status != -EAGAIN && status != -ESTALE)
+			nfsd_reset_write_verifier(copy->cp_nn);
 	}
 	return bytes_copied;
 }
@@ -2153,16 +2186,14 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		}
 		status = nfsd4_setup_inter_ssc(rqstp, cstate, copy);
 		if (status) {
-			trace_nfsd_copy_done(copy, status);
-			return nfserr_offload_denied;
+			status = nfserr_offload_denied;
+			goto out;
 		}
 	} else {
 		trace_nfsd_copy_intra(copy);
 		status = nfsd4_setup_intra_ssc(rqstp, cstate, copy);
-		if (status) {
-			trace_nfsd_copy_done(copy, status);
-			return status;
-		}
+		if (status)
+			goto out;
 	}
 
 	memcpy(&copy->fh, &cstate->current_fh.fh_handle,
@@ -2521,12 +2552,19 @@ nfsd4_verify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	return status == nfserr_same ? nfs_ok : status;
 }
 
+#define SUPPORTED_NOTIFY_MASK	(BIT(NOTIFY4_CHANGE_DIR_ATTRS) |	\
+				 BIT(NOTIFY4_REMOVE_ENTRY) |		\
+				 BIT(NOTIFY4_ADD_ENTRY) |		\
+				 BIT(NOTIFY4_RENAME_ENTRY) |		\
+				 BIT(NOTIFY4_GFLAG_EXTEND))
+
 static __be32
 nfsd4_get_dir_delegation(struct svc_rqst *rqstp,
 			 struct nfsd4_compound_state *cstate,
 			 union nfsd4_op_u *u)
 {
 	struct nfsd4_get_dir_delegation *gdd = &u->get_dir_delegation;
+	u32 requested = gdd->gdda_notification_types[0];
 	struct nfs4_delegation *dd;
 	struct nfsd_file *nf;
 	__be32 status;
@@ -2535,6 +2573,21 @@ nfsd4_get_dir_delegation(struct svc_rqst *rqstp,
 	if (status != nfs_ok)
 		return status;
 
+	/*
+	 * Offer no notifications to an order-aware client. RFC8881bis section
+	 * 16.2.13 defines order-aware as NOTIFY4_CFLAG_ORDER being set or
+	 * NOTIFY4_GFLAG_EXTEND being reset. Such a client expects cookie and
+	 * previous-entry information with its notifications (e.g. 27.4.5), and
+	 * nfsd does not track or emit directory offset information. Per
+	 * 16.2.11.3 the alternative would be to recall the delegation, so it's
+	 * simpler to just decline the notifications here.
+	 */
+	if (!(requested & BIT(NOTIFY4_GFLAG_EXTEND)) ||
+	    (requested & BIT(NOTIFY4_CFLAG_ORDER)))
+		requested = 0;
+
+	gdd->gddr_notification[0] = requested & SUPPORTED_NOTIFY_MASK;
+
 	/*
 	 * RFC 8881, section 18.39.3 says:
 	 *
@@ -2556,6 +2609,10 @@ nfsd4_get_dir_delegation(struct svc_rqst *rqstp,
 
 	gdd->gddrnf_status = GDD4_OK;
 	memcpy(&gdd->gddr_stateid, &dd->dl_stid.sc_stateid, sizeof(gdd->gddr_stateid));
+	gdd->gddr_child_attributes[0] = dd->dl_child_attrs[0];
+	gdd->gddr_child_attributes[1] = dd->dl_child_attrs[1];
+	gdd->gddr_dir_attributes[0] = dd->dl_dir_attrs[0];
+	gdd->gddr_dir_attributes[1] = dd->dl_dir_attrs[1];
 	nfs4_put_stid(&dd->dl_stid);
 	return nfs_ok;
 }
@@ -3119,9 +3176,22 @@ nfsd4_proc_compound(struct svc_rqst *rqstp)
 				op->status = nfsd4_open_omfg(rqstp, cstate, op);
 			goto encode_op;
 		}
-		if (!current_fh->fh_dentry &&
-				!HAS_FH_FLAG(current_fh, NFSD4_FH_FOREIGN)) {
-			if (!(op->opdesc->op_flags & ALLOWED_WITHOUT_FH)) {
+		if (!current_fh->fh_dentry) {
+			if (HAS_FH_FLAG(current_fh, NFSD4_FH_FOREIGN)) {
+				/*
+				 * FOREIGN fh from inter-SSC PUTFH: only
+				 * SAVEFH may proceed with a NULL fh_dentry.
+				 * Per RFC 7862 S15.2.3, validation of a
+				 * foreign fh is deferred to the operation
+				 * that consumes it, and NFS4ERR_STALE is
+				 * returned at that point.
+				 */
+				if (op->opnum != OP_SAVEFH &&
+				    !(op->opdesc->op_flags & ALLOWED_WITHOUT_FH)) {
+					op->status = nfserr_stale;
+					goto encode_op;
+				}
+			} else if (!(op->opdesc->op_flags & ALLOWED_WITHOUT_FH)) {
 				op->status = nfserr_nofilehandle;
 				goto encode_op;
 			}
@@ -3185,6 +3255,9 @@ nfsd4_proc_compound(struct svc_rqst *rqstp)
 			status = op->status;
 		}
 
+		if (op->opdesc && op->opdesc->op_release)
+			op->opdesc->op_release(&op->u);
+
 		trace_nfsd_compound_status(args->client_opcnt, resp->opcnt,
 					   status, nfsd4_op_name(op->opnum));
 
@@ -3506,8 +3579,8 @@ static u32 nfsd4_get_dir_delegation_rsize(const struct svc_rqst *rqstp,
 		op_encode_verifier_maxsz +
 		op_encode_stateid_maxsz +
 		2 /* gddr_notification */ +
-		2 /* gddr_child_attributes */ +
-		2 /* gddr_dir_attributes */);
+		3 /* gddr_child_attributes */ +
+		3 /* gddr_dir_attributes */) * sizeof(__be32);
 }
 
 #ifdef CONFIG_NFSD_PNFS
@@ -3684,6 +3757,7 @@ static const struct nfsd4_operation nfsd4_ops[] = {
 	},
 	[OP_OPEN] = {
 		.op_func = nfsd4_open,
+		.op_release = nfsd4_open_release,
 		.op_flags = OP_HANDLES_WRONGSEC | OP_MODIFIES_SOMETHING,
 		.op_name = "OP_OPEN",
 		.op_rsize_bop = nfsd4_open_rsize,
diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c
index 6ea25a52d2f4e..d513971fb119d 100644
--- a/fs/nfsd/nfs4recover.c
+++ b/fs/nfsd/nfs4recover.c
@@ -167,7 +167,7 @@ nfsd4_create_clid_dir(struct nfs4_client *clp)
 	end_creating(dentry);
 out:
 	if (status == 0) {
-		if (nn->in_grace)
+		if (test_bit(NFSD_NET_IN_GRACE, &nn->flags))
 			__nfsd4_create_reclaim_record_grace(clp, dname, nn);
 		vfs_fsync(nn->rec_file, 0);
 	} else {
@@ -285,10 +285,12 @@ __nfsd4_remove_reclaim_record_grace(const char *dname, int len,
 		return;
 	}
 	name.len = len;
+	down_write(&nn->reclaim_str_hashtbl_lock);
 	crp = nfsd4_find_reclaim_client(name, nn);
-	kfree(name.data);
 	if (crp)
 		nfs4_remove_reclaim_record(crp, nn);
+	up_write(&nn->reclaim_str_hashtbl_lock);
+	kfree(name.data);
 }
 
 static void
@@ -317,7 +319,7 @@ nfsd4_remove_clid_dir(struct nfs4_client *clp)
 	nfs4_reset_creds(original_cred);
 	if (status == 0) {
 		vfs_fsync(nn->rec_file, 0);
-		if (nn->in_grace)
+		if (test_bit(NFSD_NET_IN_GRACE, &nn->flags))
 			__nfsd4_remove_reclaim_record_grace(dname,
 					HEXDIR_LEN, nn);
 	}
@@ -373,7 +375,7 @@ nfsd4_recdir_purge_old(struct nfsd_net *nn)
 {
 	int status;
 
-	nn->in_grace = false;
+	clear_bit(NFSD_NET_IN_GRACE, &nn->flags);
 	if (!nn->rec_file)
 		return;
 	status = mnt_want_write_file(nn->rec_file);
@@ -455,7 +457,7 @@ nfsd4_init_recdir(struct net *net)
 
 	nfs4_reset_creds(original_cred);
 	if (!status)
-		nn->in_grace = true;
+		set_bit(NFSD_NET_IN_GRACE, &nn->flags);
 	return status;
 }
 
@@ -484,6 +486,7 @@ nfs4_legacy_state_init(struct net *net)
 	for (i = 0; i < CLIENT_HASH_SIZE; i++)
 		INIT_LIST_HEAD(&nn->reclaim_str_hashtbl[i]);
 	nn->reclaim_str_hashtbl_size = 0;
+	init_rwsem(&nn->reclaim_str_hashtbl_lock);
 
 	return 0;
 }
@@ -598,13 +601,16 @@ nfsd4_check_legacy_client(struct nfs4_client *clp)
 		goto out_enoent;
 	}
 	name.len = HEXDIR_LEN;
+	down_read(&nn->reclaim_str_hashtbl_lock);
 	crp = nfsd4_find_reclaim_client(name, nn);
-	kfree(name.data);
 	if (crp) {
 		set_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags);
 		crp->cr_clp = clp;
-		return 0;
 	}
+	up_read(&nn->reclaim_str_hashtbl_lock);
+	kfree(name.data);
+	if (crp)
+		return 0;
 
 out_enoent:
 	return -ENOENT;
@@ -1176,6 +1182,7 @@ nfsd4_cld_check(struct nfs4_client *clp)
 		return 0;
 
 	/* look for it in the reclaim hashtable otherwise */
+	down_read(&nn->reclaim_str_hashtbl_lock);
 	crp = nfsd4_find_reclaim_client(clp->cl_name, nn);
 	if (crp)
 		goto found;
@@ -1191,6 +1198,7 @@ nfsd4_cld_check(struct nfs4_client *clp)
 		if (!name.data) {
 			dprintk("%s: failed to allocate memory for name.data!\n",
 				__func__);
+			up_read(&nn->reclaim_str_hashtbl_lock);
 			return -ENOENT;
 		}
 		name.len = HEXDIR_LEN;
@@ -1201,9 +1209,11 @@ nfsd4_cld_check(struct nfs4_client *clp)
 
 	}
 #endif
+	up_read(&nn->reclaim_str_hashtbl_lock);
 	return -ENOENT;
 found:
 	crp->cr_clp = clp;
+	up_read(&nn->reclaim_str_hashtbl_lock);
 	return 0;
 }
 
@@ -1215,6 +1225,7 @@ nfsd4_cld_check_v2(struct nfs4_client *clp)
 	struct cld_net *cn = nn->cld_net;
 #endif
 	struct nfs4_client_reclaim *crp;
+	unsigned int princhashlen;
 	char *principal = NULL;
 
 	/* did we already find that this client is stable? */
@@ -1222,6 +1233,7 @@ nfsd4_cld_check_v2(struct nfs4_client *clp)
 		return 0;
 
 	/* look for it in the reclaim hashtable otherwise */
+	down_read(&nn->reclaim_str_hashtbl_lock);
 	crp = nfsd4_find_reclaim_client(clp->cl_name, nn);
 	if (crp)
 		goto found;
@@ -1237,6 +1249,7 @@ nfsd4_cld_check_v2(struct nfs4_client *clp)
 		if (!name.data) {
 			dprintk("%s: failed to allocate memory for name.data\n",
 					__func__);
+			up_read(&nn->reclaim_str_hashtbl_lock);
 			return -ENOENT;
 		}
 		name.len = HEXDIR_LEN;
@@ -1247,23 +1260,31 @@ nfsd4_cld_check_v2(struct nfs4_client *clp)
 
 	}
 #endif
+	up_read(&nn->reclaim_str_hashtbl_lock);
 	return -ENOENT;
 found:
-	if (crp->cr_princhash.len) {
+	princhashlen = crp->cr_princhash.len;
+	if (princhashlen) {
 		u8 digest[SHA256_DIGEST_SIZE];
+		u8 *pdata;
 
 		if (clp->cl_cred.cr_raw_principal)
 			principal = clp->cl_cred.cr_raw_principal;
 		else if (clp->cl_cred.cr_principal)
 			principal = clp->cl_cred.cr_principal;
-		if (principal == NULL)
+		if (principal == NULL) {
+			up_read(&nn->reclaim_str_hashtbl_lock);
 			return -ENOENT;
+		}
 		sha256(principal, strlen(principal), digest);
-		if (memcmp(crp->cr_princhash.data, digest,
-				crp->cr_princhash.len))
+		pdata = crp->cr_princhash.data;
+		if (memcmp(pdata, digest, princhashlen)) {
+			up_read(&nn->reclaim_str_hashtbl_lock);
 			return -ENOENT;
+		}
 	}
 	crp->cr_clp = clp;
+	up_read(&nn->reclaim_str_hashtbl_lock);
 	return 0;
 }
 
@@ -1362,7 +1383,8 @@ nfs4_cld_state_init(struct net *net)
 	for (i = 0; i < CLIENT_HASH_SIZE; i++)
 		INIT_LIST_HEAD(&nn->reclaim_str_hashtbl[i]);
 	nn->reclaim_str_hashtbl_size = 0;
-	nn->track_reclaim_completes = true;
+	init_rwsem(&nn->reclaim_str_hashtbl_lock);
+	set_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags);
 	atomic_set(&nn->nr_reclaim_complete, 0);
 
 	return 0;
@@ -1373,7 +1395,7 @@ nfs4_cld_state_shutdown(struct net *net)
 {
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
-	nn->track_reclaim_completes = false;
+	clear_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags);
 	kfree(nn->reclaim_str_hashtbl);
 }
 
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index a42f34842d776..4ae5d65c056ab 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -55,6 +55,7 @@
 #include "netns.h"
 #include "pnfs.h"
 #include "filecache.h"
+#include "nfs4xdr_gen.h"
 #include "trace.h"
 
 #define NFSDDBG_FACILITY                NFSDDBG_PROC
@@ -91,6 +92,8 @@ static void _free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_stat
 static void nfsd4_file_hash_remove(struct nfs4_file *fi);
 static void deleg_reaper(struct nfsd_net *nn);
 
+static const struct lease_manager_operations nfsd_lease_mng_ops;
+
 /* Locking: */
 
 enum nfsd4_st_mutex_lock_subclass {
@@ -124,6 +127,7 @@ static void free_session(struct nfsd4_session *);
 static const struct nfsd4_callback_ops nfsd4_cb_recall_ops;
 static const struct nfsd4_callback_ops nfsd4_cb_notify_lock_ops;
 static const struct nfsd4_callback_ops nfsd4_cb_getattr_ops;
+static const struct nfsd4_callback_ops nfsd4_cb_notify_ops;
 
 static struct workqueue_struct *laundry_wq;
 
@@ -355,12 +359,13 @@ remove_blocked_locks(struct nfs4_lockowner *lo)
 	}
 }
 
-static void
+static bool
 nfsd4_cb_notify_lock_prepare(struct nfsd4_callback *cb)
 {
 	struct nfsd4_blocked_lock	*nbl = container_of(cb,
 						struct nfsd4_blocked_lock, nbl_cb);
 	locks_delete_block(&nbl->nbl_lock);
+	return true;
 }
 
 static int
@@ -1120,29 +1125,31 @@ static void block_delegations(struct knfsd_fh *fh)
 }
 
 static struct nfs4_delegation *
-alloc_init_deleg(struct nfs4_client *clp, struct nfs4_file *fp,
-		 struct nfs4_clnt_odstate *odstate, u32 dl_type)
+__alloc_init_deleg(struct nfs4_client *clp, struct nfs4_file *fp,
+		   struct nfs4_clnt_odstate *odstate, u32 dl_type,
+		   void (*sc_free)(struct nfs4_stid *))
 {
 	struct nfs4_delegation *dp;
 	struct nfs4_stid *stid;
 	long n;
 
-	dprintk("NFSD alloc_init_deleg\n");
+	if (delegation_blocked(&fp->fi_fhandle))
+		return NULL;
+
 	n = atomic_long_inc_return(&num_delegations);
 	if (n < 0 || n > max_delegations)
 		goto out_dec;
-	if (delegation_blocked(&fp->fi_fhandle))
-		goto out_dec;
-	stid = nfs4_alloc_stid(clp, deleg_slab, nfs4_free_deleg);
+
+	stid = nfs4_alloc_stid(clp, deleg_slab, sc_free);
 	if (stid == NULL)
 		goto out_dec;
-	dp = delegstateid(stid);
 
 	/*
 	 * delegation seqid's are never incremented.  The 4.1 special
 	 * meaning of seqid 0 isn't meaningful, really, but let's avoid
-	 * 0 anyway just for consistency and use 1:
+	 * 0 anyway just for consistency and use 1.
 	 */
+	dp = delegstateid(stid);
 	dp->dl_stid.sc_stateid.si_generation = 1;
 	INIT_LIST_HEAD(&dp->dl_perfile);
 	INIT_LIST_HEAD(&dp->dl_perclnt);
@@ -1152,19 +1159,79 @@ alloc_init_deleg(struct nfs4_client *clp, struct nfs4_file *fp,
 	dp->dl_type = dl_type;
 	dp->dl_retries = 1;
 	dp->dl_recalled = false;
-	nfsd4_init_cb(&dp->dl_recall, dp->dl_stid.sc_client,
-		      &nfsd4_cb_recall_ops, NFSPROC4_CLNT_CB_RECALL);
-	nfsd4_init_cb(&dp->dl_cb_fattr.ncf_getattr, dp->dl_stid.sc_client,
-			&nfsd4_cb_getattr_ops, NFSPROC4_CLNT_CB_GETATTR);
-	dp->dl_cb_fattr.ncf_file_modified = false;
 	get_nfs4_file(fp);
 	dp->dl_stid.sc_file = fp;
+	nfsd4_init_cb(&dp->dl_recall, dp->dl_stid.sc_client,
+		      &nfsd4_cb_recall_ops, NFSPROC4_CLNT_CB_RECALL);
 	return dp;
 out_dec:
 	atomic_long_dec(&num_delegations);
 	return NULL;
 }
 
+static struct nfs4_delegation *
+alloc_init_deleg(struct nfs4_client *clp, struct nfs4_file *fp,
+		 struct nfs4_clnt_odstate *odstate, u32 dl_type)
+{
+	struct nfs4_delegation *dp;
+
+	dp = __alloc_init_deleg(clp, fp, odstate, dl_type, nfs4_free_deleg);
+	if (!dp)
+		return NULL;
+
+	nfsd4_init_cb(&dp->dl_cb_fattr.ncf_getattr, dp->dl_stid.sc_client,
+			&nfsd4_cb_getattr_ops, NFSPROC4_CLNT_CB_GETATTR);
+	dp->dl_cb_fattr.ncf_file_modified = false;
+	return dp;
+}
+
+static void nfs4_free_dir_deleg(struct nfs4_stid *stid)
+{
+	struct nfs4_delegation	*dp = delegstateid(stid);
+	struct nfsd4_cb_notify *ncn = &dp->dl_cb_notify;
+	int i;
+
+	for (i = 0; i < ncn->ncn_evt_cnt; ++i)
+		nfsd_notify_event_put(ncn->ncn_evt[i]);
+	kfree(ncn->ncn_nf);
+	for (i = 0; i < NOTIFY4_PAGE_ARRAY_SIZE; i++) {
+		if (!ncn->ncn_pages[i])
+			break;
+		put_page(ncn->ncn_pages[i]);
+	}
+	nfs4_free_deleg(stid);
+}
+
+static struct nfs4_delegation *
+alloc_init_dir_deleg(struct nfs4_client *clp, struct nfs4_file *fp)
+{
+	struct nfs4_delegation *dp;
+	struct nfsd4_cb_notify *ncn;
+	int npages;
+
+	dp = __alloc_init_deleg(clp, fp, NULL, NFS4_OPEN_DELEGATE_READ, nfs4_free_dir_deleg);
+	if (!dp)
+		return NULL;
+
+	ncn = &dp->dl_cb_notify;
+
+	npages = alloc_pages_bulk(GFP_KERNEL, NOTIFY4_PAGE_ARRAY_SIZE, ncn->ncn_pages);
+	if (npages != NOTIFY4_PAGE_ARRAY_SIZE) {
+		nfs4_put_stid(&dp->dl_stid);
+		return NULL;
+	}
+
+	ncn->ncn_nf = kcalloc(NOTIFY4_EVENT_QUEUE_SIZE, sizeof(*ncn->ncn_nf), GFP_KERNEL);
+	if (!ncn->ncn_nf) {
+		nfs4_put_stid(&dp->dl_stid);
+		return NULL;
+	}
+	spin_lock_init(&ncn->ncn_lock);
+	nfsd4_init_cb(&ncn->ncn_cb, dp->dl_stid.sc_client,
+			&nfsd4_cb_notify_ops, NFSPROC4_CLNT_CB_NOTIFY);
+	return dp;
+}
+
 void
 nfs4_put_stid(struct nfs4_stid *s)
 {
@@ -1209,7 +1276,9 @@ static void put_deleg_file(struct nfs4_file *fp)
 
 	spin_lock(&fp->fi_lock);
 	if (--fp->fi_delegees == 0) {
-		swap(nf, fp->fi_deleg_file);
+		nf = rcu_dereference_protected(fp->fi_deleg_file,
+					       lockdep_is_held(&fp->fi_lock));
+		RCU_INIT_POINTER(fp->fi_deleg_file, NULL);
 		swap(rnf, fp->fi_rdeleg_file);
 	}
 	spin_unlock(&fp->fi_lock);
@@ -1247,12 +1316,13 @@ static void nfsd4_finalize_deleg_timestamps(struct nfs4_delegation *dp, struct f
 static void nfs4_unlock_deleg_lease(struct nfs4_delegation *dp)
 {
 	struct nfs4_file *fp = dp->dl_stid.sc_file;
-	struct nfsd_file *nf = fp->fi_deleg_file;
+	struct nfsd_file *nf = rcu_dereference_protected(fp->fi_deleg_file, 1);
 
 	WARN_ON_ONCE(!fp->fi_delegees);
 
 	nfsd4_finalize_deleg_timestamps(dp, nf->nf_file);
 	kernel_setlease(nf->nf_file, F_UNLCK, NULL, (void **)&dp);
+	nfsd_fsnotify_recalc_mask(nf);
 	put_deleg_file(fp);
 }
 
@@ -1873,14 +1943,21 @@ static void revoke_one_stid(struct nfsd_net *nn, struct nfs4_client *clp,
  * being released.  Thus nfsd will no longer prevent the filesystem from being
  * unmounted.
  *
- * The clients which own the states will subsequently being notified that the
+ * The clients which own the states will subsequently be notified that the
  * states have been "admin-revoked".
+ *
+ * Context: Caller must hold nfsd_mutex with nn->nfsd_serv confirmed
+ *          non-NULL.  nfs4_state_destroy_net() frees conf_id_hashtbl
+ *          at server shutdown without clearing the pointer, so a
+ *          walk without these guarantees iterates freed slab memory.
  */
 void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb)
 {
 	unsigned int idhashval;
 	unsigned int sc_types;
 
+	lockdep_assert_held(&nfsd_mutex);
+
 	sc_types = SC_TYPE_OPEN | SC_TYPE_LOCK | SC_TYPE_DELEG | SC_TYPE_LAYOUT;
 
 	spin_lock(&nn->client_lock);
@@ -1946,12 +2023,19 @@ static struct nfs4_stid *find_one_export_stid(struct nfs4_client *clp,
  *
  * Userspace (exportfs -u) sends this after removing the last client
  * for a path, enabling the underlying filesystem to be unmounted.
+ *
+ * Context: Caller must hold nfsd_mutex with nn->nfsd_serv confirmed
+ *          non-NULL.  nfs4_state_destroy_net() frees conf_id_hashtbl
+ *          at server shutdown without clearing the pointer, so a
+ *          walk without these guarantees iterates freed slab memory.
  */
 void nfsd4_revoke_export_states(struct nfsd_net *nn, const struct path *path)
 {
 	unsigned int idhashval;
 	unsigned int sc_types;
 
+	lockdep_assert_held(&nfsd_mutex);
+
 	sc_types = SC_TYPE_OPEN | SC_TYPE_LOCK | SC_TYPE_DELEG | SC_TYPE_LAYOUT;
 
 	spin_lock(&nn->client_lock);
@@ -2052,12 +2136,10 @@ gen_sessionid(struct nfsd4_session *ses)
 static struct shrinker *nfsd_slot_shrinker;
 static DEFINE_SPINLOCK(nfsd_session_list_lock);
 static LIST_HEAD(nfsd_session_list);
-/* The sum of "target_slots-1" on every session.  The shrinker can push this
- * down, though it can take a little while for the memory to actually
- * be freed.  The "-1" is because we can never free slot 0 while the
- * session is active.
- */
+/* The sum of "target_slots" on every session, slot 0 included. */
 static atomic_t nfsd_total_target_slots = ATOMIC_INIT(0);
+/* Session count, subtracted from the sum to exclude slot 0. */
+static atomic_t nfsd_total_sessions = ATOMIC_INIT(0);
 
 static void
 free_session_slots(struct nfsd4_session *ses, int from)
@@ -2081,26 +2163,21 @@ free_session_slots(struct nfsd4_session *ses, int from)
 	}
 	ses->se_fchannel.maxreqs = from;
 	if (ses->se_target_maxslots > from) {
-		int new_target = from ?: 1;
-		atomic_sub(ses->se_target_maxslots - new_target, &nfsd_total_target_slots);
-		ses->se_target_maxslots = new_target;
+		int delta = ses->se_target_maxslots - from;
+
+		atomic_sub(delta, &nfsd_total_target_slots);
+		/* Retain one slot so the session can make forward progress. */
+		ses->se_target_maxslots = from ?: 1;
 	}
 }
 
-/**
- * reduce_session_slots - reduce the target max-slots of a session if possible
- * @ses:  The session to affect
- * @dec:  how much to decrease the target by
- *
+/*
  * This interface can be used by a shrinker to reduce the target max-slots
  * for a session so that some slots can eventually be freed.
  * It uses spin_trylock() as it may be called in a context where another
  * spinlock is held that has a dependency on client_lock.  As shrinkers are
- * best-effort, skiping a session is client_lock is already held has no
- * great coast
- *
- * Return value:
- *   The number of slots that the target was reduced by.
+ * best-effort, skipping a session with the client_lock already held has no
+ * great cost.
  */
 static int
 reduce_session_slots(struct nfsd4_session *ses, int dec)
@@ -2179,7 +2256,7 @@ static struct nfsd4_session *alloc_session(struct nfsd4_channel_attrs *fattrs,
 	fattrs->maxreqs = i;
 	memcpy(&new->se_fchannel, fattrs, sizeof(struct nfsd4_channel_attrs));
 	new->se_target_maxslots = i;
-	atomic_add(i - 1, &nfsd_total_target_slots);
+	atomic_add(i, &nfsd_total_target_slots);
 	new->se_cb_slot_avail = ~0U;
 	new->se_cb_highest_slot = min(battrs->maxreqs - 1,
 				      NFSD_BC_SLOT_TABLE_SIZE - 1);
@@ -2295,7 +2372,7 @@ static void __free_session(struct nfsd4_session *ses)
 {
 	free_session_slots(ses, 0);
 	xa_destroy(&ses->se_slots);
-	kfree(ses);
+	kfree_rcu(ses, rcu_head);
 }
 
 static void free_session(struct nfsd4_session *ses)
@@ -2304,21 +2381,51 @@ static void free_session(struct nfsd4_session *ses)
 	__free_session(ses);
 }
 
+/**
+ * nfsd_slot_shrinker_count - report reclaimable DRC slots
+ * @s: shrinker descriptor (unused)
+ * @sc: shrink control (unused)
+ *
+ * Return: a positive count of reclaimable slots, or SHRINK_EMPTY when
+ * there is nothing to reclaim.
+ */
 static unsigned long
-nfsd_slot_count(struct shrinker *s, struct shrink_control *sc)
+nfsd_slot_shrinker_count(struct shrinker *s, struct shrink_control *sc)
 {
-	unsigned long cnt = atomic_read(&nfsd_total_target_slots);
+	int count;
+
+	/*
+	 * To prevent session deadlock, one slot of each session (slot 0)
+	 * is not reclaimable while the session is active. Thus the number
+	 * of sessions is subtracted from the total number of target slots.
+	 */
+	count = atomic_read(&nfsd_total_target_slots) -
+		atomic_read(&nfsd_total_sessions);
 
-	return cnt ? cnt : SHRINK_EMPTY;
+	return count > 0 ? count : SHRINK_EMPTY;
 }
 
+/**
+ * nfsd_slot_shrinker_scan - reclaim DRC slots under memory pressure
+ * @s: shrinker descriptor (unused)
+ * @sc: shrink control; @sc->nr_to_scan bounds the sessions visited,
+ *      @sc->nr_scanned reports how many were visited
+ *
+ * Return: the number of session slots NFSD will release.
+ */
 static unsigned long
-nfsd_slot_scan(struct shrinker *s, struct shrink_control *sc)
+nfsd_slot_shrinker_scan(struct shrinker *s, struct shrink_control *sc)
 {
 	struct nfsd4_session *ses;
 	unsigned long scanned = 0;
 	unsigned long freed = 0;
 
+	/*
+	 * Each visited session releases at most one slot. After
+	 * nr_to_scan sessions have been visited, the list head is
+	 * rotated past the last visited session so the next scan
+	 * resumes from there.
+	 */
 	spin_lock(&nfsd_session_list_lock);
 	list_for_each_entry(ses, &nfsd_session_list, se_all_sessions) {
 		freed += reduce_session_slots(ses, 1);
@@ -2360,6 +2467,7 @@ static void init_session(struct svc_rqst *rqstp, struct nfsd4_session *new, stru
 
 	spin_lock(&nfsd_session_list_lock);
 	list_add_tail(&new->se_all_sessions, &nfsd_session_list);
+	atomic_inc(&nfsd_total_sessions);
 	spin_unlock(&nfsd_session_list_lock);
 
 	{
@@ -2433,6 +2541,7 @@ unhash_session(struct nfsd4_session *ses)
 	spin_unlock(&ses->se_client->cl_lock);
 	spin_lock(&nfsd_session_list_lock);
 	list_del(&ses->se_all_sessions);
+	atomic_dec(&nfsd_total_sessions);
 	spin_unlock(&nfsd_session_list_lock);
 }
 
@@ -2581,7 +2690,17 @@ unhash_client_locked(struct nfs4_client *clp)
 	spin_lock(&nfsd_session_list_lock);
 	list_for_each_entry(ses, &clp->cl_sessions, se_perclnt) {
 		list_del_init(&ses->se_hash);
-		list_del_init(&ses->se_all_sessions);
+		/*
+		 * unhash_client_locked() can run more than once for a
+		 * client; the session stays on cl_sessions across calls.
+		 * The first pass empties se_all_sessions via
+		 * list_del_init(), so skip the decrement on later passes
+		 * to keep nfsd_total_sessions from being double-counted.
+		 */
+		if (!list_empty(&ses->se_all_sessions)) {
+			list_del_init(&ses->se_all_sessions);
+			atomic_dec(&nfsd_total_sessions);
+		}
 	}
 	spin_unlock(&nfsd_session_list_lock);
 	spin_unlock(&clp->cl_lock);
@@ -2675,16 +2794,23 @@ static void inc_reclaim_complete(struct nfs4_client *clp)
 {
 	struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
 
-	if (!nn->track_reclaim_completes)
+	if (!test_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags))
 		return;
-	if (!nfsd4_find_reclaim_client(clp->cl_name, nn))
+
+	down_read(&nn->reclaim_str_hashtbl_lock);
+	if (!nfsd4_find_reclaim_client(clp->cl_name, nn)) {
+		up_read(&nn->reclaim_str_hashtbl_lock);
 		return;
+	}
 	if (atomic_inc_return(&nn->nr_reclaim_complete) ==
 			nn->reclaim_str_hashtbl_size) {
+		up_read(&nn->reclaim_str_hashtbl_lock);
 		printk(KERN_INFO "NFSD: all clients done reclaiming, ending NFSv4 grace period (net %x)\n",
 				clp->net->ns.inum);
 		nfsd4_end_grace(nn);
+		return;
 	}
+	up_read(&nn->reclaim_str_hashtbl_lock);
 }
 
 static void expire_client(struct nfs4_client *clp)
@@ -3140,7 +3266,8 @@ static int nfs4_show_deleg(struct seq_file *s, struct nfs4_stid *st)
 	/* XXX: lease time, whether it's being recalled. */
 
 	spin_lock(&nf->fi_lock);
-	file = nf->fi_deleg_file;
+	file = rcu_dereference_protected(nf->fi_deleg_file,
+					 lockdep_is_held(&nf->fi_lock));
 	if (file) {
 		seq_puts(s, ", ");
 		nfs4_show_superblock(s, file);
@@ -3359,6 +3486,212 @@ nfsd4_cb_getattr_release(struct nfsd4_callback *cb)
 	nfs4_put_stid(&dp->dl_stid);
 }
 
+static void nfsd_break_one_deleg(struct nfs4_delegation *dp)
+{
+	bool queued;
+
+	if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &dp->dl_recall.cb_flags))
+		return;
+
+	/*
+	 * When called from the lease break (nfsd_break_deleg_cb()) the state
+	 * code is serialized by the flc_lock and the lease has not been
+	 * removed yet, so sc_count is known to be nonzero. The CB_NOTIFY
+	 * callback paths reach here from a workqueue without the flc_lock,
+	 * where the delegation may already be unhashed with sc_count at zero.
+	 * Use refcount_inc_not_zero() so both cases are safe, and bail if the
+	 * delegation is already being torn down.
+	 */
+	if (!refcount_inc_not_zero(&dp->dl_stid.sc_count)) {
+		clear_bit(NFSD4_CALLBACK_RUNNING, &dp->dl_recall.cb_flags);
+		return;
+	}
+	queued = nfsd4_run_cb(&dp->dl_recall);
+	WARN_ON_ONCE(!queued);
+	if (!queued) {
+		refcount_dec(&dp->dl_stid.sc_count);
+		clear_bit(NFSD4_CALLBACK_RUNNING, &dp->dl_recall.cb_flags);
+	}
+}
+
+static bool
+nfsd4_cb_notify_prepare(struct nfsd4_callback *cb)
+{
+	struct nfsd4_cb_notify *ncn = container_of(cb, struct nfsd4_cb_notify, ncn_cb);
+	struct nfs4_delegation *dp = container_of(ncn, struct nfs4_delegation, dl_cb_notify);
+	struct nfsd_notify_event *events[NOTIFY4_EVENT_QUEUE_SIZE];
+	struct xdr_buf xdr = { .buflen = PAGE_SIZE * NOTIFY4_PAGE_ARRAY_SIZE,
+			       .pages  = ncn->ncn_pages };
+	int limit = NOTIFY4_EVENT_QUEUE_SIZE;
+	struct xdr_stream stream;
+	struct nfsd_file *nf;
+	bool error = false;
+	int count, i;
+
+	/* Save a slot for dir attr update if requested */
+	if (dp->dl_notify_mask & BIT(NOTIFY4_CHANGE_DIR_ATTRS))
+		--limit;
+
+	/* Clear any failure recorded by a previous transmit. */
+	ncn->ncn_encode_err = false;
+
+	xdr_init_encode_pages(&stream, &xdr);
+
+	spin_lock(&ncn->ncn_lock);
+	count = ncn->ncn_evt_cnt;
+
+	/* spurious queueing? */
+	if (count == 0) {
+		spin_unlock(&ncn->ncn_lock);
+		return false;
+	}
+
+	memcpy(events, ncn->ncn_evt, sizeof(*events) * count);
+	ncn->ncn_evt_cnt = 0;
+	spin_unlock(&ncn->ncn_lock);
+
+	/*
+	 * We can't keep up! Drop the queued events and recall. The queue must
+	 * be drained here: out_recall leaves ncn_evt_cnt at 0, so the release
+	 * op won't see leftover events and requeue this callback forever.
+	 */
+	if (count > limit) {
+		for (i = 0; i < count; ++i)
+			nfsd_notify_event_put(events[i]);
+		goto out_recall;
+	}
+
+	rcu_read_lock();
+	nf = nfsd_file_get(rcu_dereference(dp->dl_stid.sc_file->fi_deleg_file));
+	rcu_read_unlock();
+	if (!nf) {
+		for (i = 0; i < count; ++i)
+			nfsd_notify_event_put(events[i]);
+		goto out_recall;
+	}
+
+	for (i = 0; i < count; ++i) {
+		struct nfsd_notify_event *nne = events[i];
+
+		if (!error) {
+			u32 *maskp = (u32 *)xdr_reserve_space(&stream, sizeof(*maskp));
+			u8 *p;
+
+			if (!maskp) {
+				error = true;
+				goto put_event;
+			}
+
+			p = nfsd4_encode_notify_event(&stream, nne, dp, nf, maskp);
+			if (!p) {
+				pr_notice("Could not generate CB_NOTIFY from fsnotify mask 0x%x\n",
+					  nne->ne_mask);
+				error = true;
+				goto put_event;
+			}
+
+			ncn->ncn_nf[i].notify_mask.count = 1;
+			ncn->ncn_nf[i].notify_mask.element = maskp;
+			ncn->ncn_nf[i].notify_vals.data = p;
+			ncn->ncn_nf[i].notify_vals.len = (u8 *)stream.p - p;
+		}
+put_event:
+		nfsd_notify_event_put(nne);
+	}
+	if (!error && (dp->dl_notify_mask & BIT(NOTIFY4_CHANGE_DIR_ATTRS))) {
+		u32 *maskp = (u32 *)xdr_reserve_space(&stream, sizeof(*maskp));
+		u8 *p;
+
+		if (maskp)
+			p = nfsd4_encode_dir_attr_change(&stream, dp, nf);
+		else
+			p = ERR_PTR(-ENOBUFS);
+
+		if (IS_ERR(p)) {
+			/*
+			 * The client asked to be told about dir attr changes
+			 * but the change could not be encoded. RFC 8881
+			 * s10.9.4 requires the server to recall the delegation
+			 * rather than drop a requested notification, so fall
+			 * through to recall. A NULL return instead means there
+			 * were no attributes to report, so omit the event in
+			 * that case.
+			 */
+			error = true;
+		} else if (p) {
+			*maskp = BIT(NOTIFY4_CHANGE_DIR_ATTRS);
+			ncn->ncn_nf[count].notify_mask.count = 1;
+			ncn->ncn_nf[count].notify_mask.element = maskp;
+			ncn->ncn_nf[count].notify_vals.data = p;
+			ncn->ncn_nf[count].notify_vals.len = (u8 *)stream.p - p;
+			++count;
+		}
+	}
+	if (!error) {
+		ncn->ncn_nf_cnt = count;
+		nfsd_file_put(nf);
+		return true;
+	}
+	nfsd_file_put(nf);
+out_recall:
+	nfsd_break_one_deleg(dp);
+	return false;
+}
+
+static int
+nfsd4_cb_notify_done(struct nfsd4_callback *cb,
+				struct rpc_task *task)
+{
+	struct nfsd4_cb_notify *ncn = container_of(cb, struct nfsd4_cb_notify, ncn_cb);
+	struct nfs4_delegation *dp = container_of(ncn, struct nfs4_delegation, dl_cb_notify);
+
+	if (dp->dl_stid.sc_status)
+		return 1;
+
+	/*
+	 * The CB_NOTIFY op overflowed the send buffer and was dropped from the
+	 * compound. The notification is lost, so recall the delegation rather
+	 * than leaving the client unaware of the directory change.
+	 */
+	if (ncn->ncn_encode_err) {
+		nfsd_break_one_deleg(dp);
+		return 1;
+	}
+
+	switch (task->tk_status) {
+	case -NFS4ERR_DELAY:
+		rpc_delay(task, 2 * HZ);
+		return 0;
+	default:
+		/* For any other hard error, recall the deleg */
+		nfsd_break_one_deleg(dp);
+		fallthrough;
+	case 0:
+		return 1;
+	}
+}
+
+static void nfsd4_run_cb_notify(struct nfsd4_cb_notify *ncn);
+
+static void
+nfsd4_cb_notify_release(struct nfsd4_callback *cb)
+{
+	struct nfsd4_cb_notify *ncn =
+			container_of(cb, struct nfsd4_cb_notify, ncn_cb);
+	struct nfs4_delegation *dp =
+			container_of(ncn, struct nfs4_delegation, dl_cb_notify);
+
+	/*
+	 * Drain events that arrived while this callback was in flight, but
+	 * don't requeue against a revoked delegation: there's no point in
+	 * notifying a client that no longer holds it, and doing so can pin the
+	 * stid and spin the workqueue.
+	 */
+	if (!dp->dl_stid.sc_status && READ_ONCE(ncn->ncn_evt_cnt) > 0)
+		nfsd4_run_cb_notify(ncn);
+	nfs4_put_stid(&dp->dl_stid);
+}
+
 static const struct nfsd4_callback_ops nfsd4_cb_recall_any_ops = {
 	.done		= nfsd4_cb_recall_any_done,
 	.release	= nfsd4_cb_recall_any_release,
@@ -3371,6 +3704,13 @@ static const struct nfsd4_callback_ops nfsd4_cb_getattr_ops = {
 	.opcode		= OP_CB_GETATTR,
 };
 
+static const struct nfsd4_callback_ops nfsd4_cb_notify_ops = {
+	.prepare	= nfsd4_cb_notify_prepare,
+	.done		= nfsd4_cb_notify_done,
+	.release	= nfsd4_cb_notify_release,
+	.opcode		= OP_CB_NOTIFY,
+};
+
 static void nfs4_cb_getattr(struct nfs4_cb_fattr *ncf)
 {
 	struct nfs4_delegation *dp =
@@ -3414,7 +3754,7 @@ static struct nfs4_client *create_client(struct xdr_netobj name,
 	clp->cl_time = ktime_get_boottime_seconds();
 	copy_verf(clp, verf);
 	memcpy(&clp->cl_addr, sa, sizeof(struct sockaddr_storage));
-	clp->cl_cb_session = NULL;
+	RCU_INIT_POINTER(clp->cl_cb_session, NULL);
 	clp->net = net;
 	clp->cl_nfsd_dentry = nfsd_client_mkdir(
 		nn, &clp->cl_nfsdfs,
@@ -4496,6 +4836,19 @@ static void nfsd4_construct_sequence_response(struct nfsd4_session *session,
 		seq->status_flags |= SEQ4_STATUS_ADMIN_STATE_REVOKED;
 }
 
+static bool nfsd4_slots_inuse(struct nfsd4_session *ses, int from)
+{
+	int i;
+
+	for (i = from; i < ses->se_fchannel.maxreqs; i++) {
+		struct nfsd4_slot *slot = xa_load(&ses->se_slots, i);
+
+		if (slot->sl_flags & NFSD4_SLOT_INUSE)
+			return true;
+	}
+	return false;
+}
+
 __be32
 nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		union nfsd4_op_u *u)
@@ -4575,7 +4928,9 @@ nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 
 	if (session->se_target_maxslots < session->se_fchannel.maxreqs &&
 	    slot->sl_generation == session->se_slot_gen &&
-	    seq->maxslots <= session->se_target_maxslots)
+	    seq->maxslots <= session->se_target_maxslots &&
+	    seq->slotid < session->se_target_maxslots &&
+	    !nfsd4_slots_inuse(session, session->se_target_maxslots))
 		/* Client acknowledged our reduce maxreqs */
 		free_session_slots(session, session->se_target_maxslots);
 
@@ -4608,15 +4963,26 @@ nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	 * gently try to allocate another 20%.  This allows
 	 * fairly quick growth without grossly over-shooting what
 	 * the client might use.
+	 *
+	 * Bound that growth by the service's thread ceiling:
+	 * slots beyond the nfsd thread count cannot raise this
+	 * client's throughput, only deepen its backlog.  Cap each
+	 * session independently, since a session cannot use
+	 * another's slots; a shared budget would let idle sessions
+	 * pin an active client small.  Compare against the
+	 * configured maximum, not the running thread count, so a
+	 * client resuming from idle can grow back before the pool
+	 * scales up.
 	 */
 	if (seq->slotid == session->se_fchannel.maxreqs - 1 &&
-	    session->se_target_maxslots >= session->se_fchannel.maxreqs &&
-	    session->se_fchannel.maxreqs < NFSD_MAX_SLOTS_PER_SESSION) {
+	    session->se_target_maxslots >= session->se_fchannel.maxreqs) {
 		int s = session->se_fchannel.maxreqs;
-		int cnt = DIV_ROUND_UP(s, 5);
+		int ceiling = min_t(int, NFSD_MAX_SLOTS_PER_SESSION,
+				    svc_serv_maxthreads(rqstp->rq_server));
+		int cnt = min(DIV_ROUND_UP(s, 5), ceiling - s);
 		void *prev_slot;
 
-		do {
+		while (cnt-- > 0) {
 			/*
 			 * GFP_NOWAIT both allows allocation under a
 			 * spinlock, and only succeeds if there is
@@ -4624,13 +4990,14 @@ nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 			 */
 			slot = nfsd4_alloc_slot(&session->se_fchannel, s,
 						GFP_NOWAIT);
+			if (!slot)
+				break;
 			prev_slot = xa_load(&session->se_slots, s);
-			if (xa_is_value(prev_slot) && slot) {
+			if (xa_is_value(prev_slot)) {
 				slot->sl_seqid = xa_to_value(prev_slot);
 				slot->sl_flags |= NFSD4_SLOT_REUSED;
 			}
-			if (slot &&
-			    !xa_is_err(xa_store(&session->se_slots, s, slot,
+			if (!xa_is_err(xa_store(&session->se_slots, s, slot,
 						GFP_NOWAIT))) {
 				s += 1;
 				session->se_fchannel.maxreqs = s;
@@ -4639,9 +5006,9 @@ nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 				session->se_target_maxslots = s;
 			} else {
 				kfree(slot);
-				slot = NULL;
+				break;
 			}
-		} while (slot && --cnt > 0);
+		}
 	}
 
 out:
@@ -4922,7 +5289,7 @@ static void nfsd4_file_init(const struct svc_fh *fh, struct nfs4_file *fp)
 	INIT_LIST_HEAD(&fp->fi_delegations);
 	INIT_LIST_HEAD(&fp->fi_clnt_odstate);
 	fh_copy_shallow(&fp->fi_fhandle, &fh->fh_handle);
-	fp->fi_deleg_file = NULL;
+	RCU_INIT_POINTER(fp->fi_deleg_file, NULL);
 	fp->fi_rdeleg_file = NULL;
 	fp->fi_had_conflict = false;
 	fp->fi_share_deny = 0;
@@ -5018,8 +5385,6 @@ nfsd4_init_leases_net(struct nfsd_net *nn)
 
 	nn->nfsd4_lease = 90;	/* default lease time */
 	nn->nfsd4_grace = 90;
-	nn->somebody_reclaimed = false;
-	nn->track_reclaim_completes = false;
 	nn->clverifier_counter = get_random_u32();
 	nn->clientid_base = get_random_u32();
 	nn->clientid_counter = nn->clientid_base + 1;
@@ -5172,6 +5537,7 @@ static void nfsd4_drop_revoked_stid(struct nfs4_stid *s)
 	case SC_TYPE_DELEG:
 		dp = delegstateid(s);
 		list_del_init(&dp->dl_recall_lru);
+		s->sc_status |= SC_STATUS_FREED;
 		spin_unlock(&cl->cl_lock);
 		nfs4_put_stid(s);
 		break;
@@ -5528,7 +5894,7 @@ bool nfsd_wait_for_delegreturn(struct svc_rqst *rqstp, struct inode *inode)
 	return timeo > 0;
 }
 
-static void nfsd4_cb_recall_prepare(struct nfsd4_callback *cb)
+static bool nfsd4_cb_recall_prepare(struct nfsd4_callback *cb)
 {
 	struct nfs4_delegation *dp = cb_to_delegation(cb);
 	struct nfsd_net *nn = net_generic(dp->dl_stid.sc_client->net,
@@ -5549,6 +5915,7 @@ static void nfsd4_cb_recall_prepare(struct nfsd4_callback *cb)
 		list_add_tail(&dp->dl_recall_lru, &nn->del_recall_lru);
 	}
 	spin_unlock(&nn->deleg_lock);
+	return true;
 }
 
 static int nfsd4_cb_recall_done(struct nfsd4_callback *cb,
@@ -5598,27 +5965,6 @@ static const struct nfsd4_callback_ops nfsd4_cb_recall_ops = {
 	.opcode		= OP_CB_RECALL,
 };
 
-static void nfsd_break_one_deleg(struct nfs4_delegation *dp)
-{
-	bool queued;
-
-	if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &dp->dl_recall.cb_flags))
-		return;
-
-	/*
-	 * We're assuming the state code never drops its reference
-	 * without first removing the lease.  Since we're in this lease
-	 * callback (and since the lease code is serialized by the
-	 * flc_lock) we know the server hasn't removed the lease yet, and
-	 * we know it's safe to take a reference.
-	 */
-	refcount_inc(&dp->dl_stid.sc_count);
-	queued = nfsd4_run_cb(&dp->dl_recall);
-	WARN_ON_ONCE(!queued);
-	if (!queued)
-		refcount_dec(&dp->dl_stid.sc_count);
-}
-
 /* Called from break_lease() with flc_lock held. */
 static bool
 nfsd_break_deleg_cb(struct file_lease *fl)
@@ -5664,6 +6010,10 @@ static bool nfsd_breaker_owns_lease(struct file_lease *fl)
 	struct svc_rqst *rqst;
 	struct nfs4_client *clp;
 
+	/* Only nfsd leases */
+	if (fl->fl_lmops != &nfsd_lease_mng_ops)
+		return false;
+
 	rqst = nfsd_current_rqst();
 	if (!nfsd_v4client(rqst))
 		return false;
@@ -6043,7 +6393,22 @@ static bool nfsd4_cb_channel_good(struct nfs4_client *clp)
 	return clp->cl_minorversion && clp->cl_cb_state == NFSD4_CB_UNKNOWN;
 }
 
-static struct file_lease *nfs4_alloc_init_lease(struct nfs4_delegation *dp)
+static unsigned int
+nfsd_notify_to_ignore(u32 notify)
+{
+	unsigned int mask = 0;
+
+	if (notify & BIT(NOTIFY4_REMOVE_ENTRY))
+		mask |= FL_IGN_DIR_DELETE;
+	if (notify & BIT(NOTIFY4_ADD_ENTRY))
+		mask |= FL_IGN_DIR_CREATE;
+	if (notify & BIT(NOTIFY4_RENAME_ENTRY))
+		mask |= FL_IGN_DIR_RENAME;
+
+	return mask;
+}
+
+static struct file_lease *nfs4_alloc_init_lease(struct nfs4_delegation *dp, u32 notify)
 {
 	struct file_lease *fl;
 
@@ -6051,11 +6416,11 @@ static struct file_lease *nfs4_alloc_init_lease(struct nfs4_delegation *dp)
 	if (!fl)
 		return NULL;
 	fl->fl_lmops = &nfsd_lease_mng_ops;
-	fl->c.flc_flags = FL_DELEG;
+	fl->c.flc_flags = FL_DELEG | nfsd_notify_to_ignore(notify);
 	fl->c.flc_type = deleg_is_read(dp->dl_type) ? F_RDLCK : F_WRLCK;
 	fl->c.flc_owner = (fl_owner_t)dp;
 	fl->c.flc_pid = current->tgid;
-	fl->c.flc_file = dp->dl_stid.sc_file->fi_deleg_file->nf_file;
+	fl->c.flc_file = rcu_dereference_protected(dp->dl_stid.sc_file->fi_deleg_file, 1)->nf_file;
 	return fl;
 }
 
@@ -6063,7 +6428,7 @@ static int nfsd4_check_conflicting_opens(struct nfs4_client *clp,
 					 struct nfs4_file *fp)
 {
 	struct nfs4_ol_stateid *st;
-	struct file *f = fp->fi_deleg_file->nf_file;
+	struct file *f = rcu_dereference_protected(fp->fi_deleg_file, 1)->nf_file;
 	struct inode *ino = file_inode(f);
 	int writes;
 
@@ -6140,7 +6505,7 @@ nfsd4_verify_deleg_dentry(struct nfsd4_open *open, struct nfs4_file *fp,
 
 	exp_put(exp);
 	dput(child);
-	if (child != file_dentry(fp->fi_deleg_file->nf_file))
+	if (child != file_dentry(rcu_dereference_protected(fp->fi_deleg_file, 1)->nf_file))
 		return -EAGAIN;
 
 	return 0;
@@ -6246,8 +6611,9 @@ nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp,
 		status = -EAGAIN;
 	else if (nfsd4_verify_setuid_write(open, nf))
 		status = -EAGAIN;
-	else if (!fp->fi_deleg_file) {
-		fp->fi_deleg_file = nf;
+	else if (!rcu_dereference_protected(fp->fi_deleg_file,
+					    lockdep_is_held(&fp->fi_lock))) {
+		rcu_assign_pointer(fp->fi_deleg_file, nf);
 		/* increment early to prevent fi_deleg_file from being
 		 * cleared */
 		fp->fi_delegees = 1;
@@ -6268,11 +6634,11 @@ nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp,
 	if (stp->st_stid.sc_export)
 		dp->dl_stid.sc_export = exp_get(stp->st_stid.sc_export);
 
-	fl = nfs4_alloc_init_lease(dp);
+	fl = nfs4_alloc_init_lease(dp, 0);
 	if (!fl)
 		goto out_clnt_odstate;
 
-	status = kernel_setlease(fp->fi_deleg_file->nf_file,
+	status = kernel_setlease(rcu_dereference_protected(fp->fi_deleg_file, 1)->nf_file,
 				      fl->c.flc_type, &fl, NULL);
 	if (fl)
 		locks_free_lease(fl);
@@ -6293,7 +6659,7 @@ nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp,
 	 * Now that the deleg is set, check again to ensure that nothing
 	 * raced in and changed the mode while we weren't looking.
 	 */
-	status = nfsd4_verify_setuid_write(open, fp->fi_deleg_file);
+	status = nfsd4_verify_setuid_write(open, rcu_dereference_protected(fp->fi_deleg_file, 1));
 	if (status)
 		goto out_unlock;
 
@@ -6314,7 +6680,8 @@ nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp,
 
 	return dp;
 out_unlock:
-	kernel_setlease(fp->fi_deleg_file->nf_file, F_UNLCK, NULL, (void **)&dp);
+	kernel_setlease(rcu_dereference_protected(fp->fi_deleg_file, 1)->nf_file,
+			F_UNLCK, NULL, (void **)&dp);
 out_clnt_odstate:
 	put_clnt_odstate(dp->dl_clnt_odstate);
 	nfs4_put_stid(&dp->dl_stid);
@@ -6471,8 +6838,9 @@ nfs4_open_delegation(struct svc_rqst *rqstp, struct nfsd4_open *open,
 	memcpy(&open->op_delegate_stateid, &dp->dl_stid.sc_stateid, sizeof(dp->dl_stid.sc_stateid));
 
 	if (open->op_share_access & NFS4_SHARE_ACCESS_WRITE) {
-		struct file *f = dp->dl_stid.sc_file->fi_deleg_file->nf_file;
+		struct file *f;
 
+		f = rcu_dereference_protected(dp->dl_stid.sc_file->fi_deleg_file, 1)->nf_file;
 		if (!nfsd4_add_rdaccess_to_wrdeleg(rqstp, open, fh, stp) ||
 				!nfs4_delegation_stat(dp, currentfh, &stat)) {
 			nfs4_put_stid(&dp->dl_stid);
@@ -6728,12 +7096,21 @@ nfsd4_renew(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 static void
 nfsd4_end_grace(struct nfsd_net *nn)
 {
-	/* do nothing if grace period already ended */
-	if (nn->grace_ended)
+	/*
+	 * nfsd4_end_grace() can be entered concurrently from the
+	 * laundromat workqueue and from an nfsd compound thread
+	 * handling RECLAIM_COMPLETE.  Without serialization, both
+	 * callers can observe NFSD_NET_GRACE_ENDED clear and proceed
+	 * into nfsd4_record_grace_done().  For tracking ops whose
+	 * grace_done drains reclaim_str_hashtbl, that results in
+	 * list corruption and a double free of every
+	 * nfs4_client_reclaim entry.  Use an atomic test-and-set so
+	 * exactly one caller proceeds.
+	 */
+	if (test_and_set_bit(NFSD_NET_GRACE_ENDED, &nn->flags))
 		return;
 
 	trace_nfsd_grace_complete(nn);
-	nn->grace_ended = true;
 	/*
 	 * If the server goes down again right now, an NFSv4
 	 * client will still be allowed to reclaim after it comes back up,
@@ -6774,10 +7151,10 @@ bool nfsd4_force_end_grace(struct nfsd_net *nn)
 {
 	if (!nn->client_tracking_ops)
 		return false;
-	if (READ_ONCE(nn->grace_ended))
+	if (test_bit(NFSD_NET_GRACE_ENDED, &nn->flags))
 		return false;
 	/* laundromat_work must be initialised now, though it might be disabled */
-	WRITE_ONCE(nn->grace_end_forced, true);
+	set_bit(NFSD_NET_GRACE_END_FORCED, &nn->flags);
 	/* mod_delayed_work() doesn't queue work after
 	 * nfs4_state_shutdown_net() has called disable_delayed_work_sync()
 	 */
@@ -6791,18 +7168,22 @@ bool nfsd4_force_end_grace(struct nfsd_net *nn)
  */
 static bool clients_still_reclaiming(struct nfsd_net *nn)
 {
-	time64_t double_grace_period_end = nn->boot_time +
+	time64_t double_grace_period_end = nn->boot_time_bt +
 					   2 * nn->nfsd4_lease;
 
-	if (READ_ONCE(nn->grace_end_forced))
+	if (test_bit(NFSD_NET_GRACE_END_FORCED, &nn->flags))
 		return false;
-	if (nn->track_reclaim_completes &&
-			atomic_read(&nn->nr_reclaim_complete) ==
-			nn->reclaim_str_hashtbl_size)
-		return false;
-	if (!nn->somebody_reclaimed)
+	if (test_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags)) {
+		int size;
+
+		down_read(&nn->reclaim_str_hashtbl_lock);
+		size = nn->reclaim_str_hashtbl_size;
+		up_read(&nn->reclaim_str_hashtbl_lock);
+		if (atomic_read(&nn->nr_reclaim_complete) == size)
+			return false;
+	}
+	if (!test_and_clear_bit(NFSD_NET_SOMEBODY_RECLAIMED, &nn->flags))
 		return false;
-	nn->somebody_reclaimed = false;
 	/*
 	 * If we've given them *two* lease times to reclaim, and they're
 	 * still not done, give up:
@@ -6859,30 +7240,36 @@ static void nfsd4_ssc_shutdown_umount(struct nfsd_net *nn)
 static void nfsd4_ssc_expire_umount(struct nfsd_net *nn)
 {
 	bool do_wakeup = false;
-	struct nfsd4_ssc_umount_item *ni = NULL;
-	struct nfsd4_ssc_umount_item *tmp;
+	struct nfsd4_ssc_umount_item *ni;
 
+restart:
 	spin_lock(&nn->nfsd_ssc_lock);
-	list_for_each_entry_safe(ni, tmp, &nn->nfsd_ssc_mount_list, nsui_list) {
-		if (time_after(jiffies, ni->nsui_expire)) {
-			if (refcount_read(&ni->nsui_refcnt) > 1)
-				continue;
+	list_for_each_entry(ni, &nn->nfsd_ssc_mount_list, nsui_list) {
+		if (!time_after(jiffies, ni->nsui_expire))
+			break;
+		if (refcount_read(&ni->nsui_refcnt) > 1)
+			continue;
 
-			/* mark being unmount */
-			ni->nsui_busy = true;
-			spin_unlock(&nn->nfsd_ssc_lock);
-			mntput(ni->nsui_vfsmount);
-			spin_lock(&nn->nfsd_ssc_lock);
+		/* Prevent concurrent setup during unmount */
+		ni->nsui_busy = true;
+		spin_unlock(&nn->nfsd_ssc_lock);
+		mntput(ni->nsui_vfsmount);
+		spin_lock(&nn->nfsd_ssc_lock);
 
-			/* waiters need to start from begin of list */
-			list_del(&ni->nsui_list);
-			kfree(ni);
+		/* Force concurrent scanners to restart */
+		list_del(&ni->nsui_list);
+		kfree(ni);
 
-			/* wakeup ssc_connect waiters */
-			do_wakeup = true;
-			continue;
-		}
-		break;
+		/* wakeup ssc_connect waiters */
+		do_wakeup = true;
+		/*
+		 * Concurrent nfsd4_ssc_cancel_dul() can free any item
+		 * on the list under nfsd_ssc_lock while mntput() runs
+		 * above.  Restart from the head; the list is short and
+		 * the expire worker is periodic, so this is cheap.
+		 */
+		spin_unlock(&nn->nfsd_ssc_lock);
+		goto restart;
 	}
 	if (do_wakeup)
 		wake_up_all(&nn->nfsd_ssc_waitq);
@@ -7195,12 +7582,12 @@ deleg_reaper(struct nfsd_net *nn)
 			continue;
 		if (atomic_read(&clp->cl_delegs_in_recall))
 			continue;
-		if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &clp->cl_ra->ra_cb.cb_flags))
-			continue;
 		if (ktime_get_boottime_seconds() - clp->cl_ra_time < 5)
 			continue;
 		if (clp->cl_cb_state != NFSD4_CB_UP)
 			continue;
+		if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &clp->cl_ra->ra_cb.cb_flags))
+			continue;
 
 		/* release in nfsd4_cb_recall_any_release */
 		kref_get(&clp->cl_nfsdfs.cl_ref);
@@ -7787,7 +8174,7 @@ nfs4_preprocess_seqid_op(struct nfsd4_compound_state *cstate, u32 seqid,
 		return status;
 	stp = openlockstateid(s);
 	if (nfsd4_cstate_assign_replay(cstate, stp->st_stateowner) == -EAGAIN) {
-		nfs4_put_stateowner(stp->st_stateowner);
+		nfs4_put_stid(&stp->st_stid);
 		goto retry;
 	}
 
@@ -8036,6 +8423,10 @@ nfsd4_delegreturn(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	if (status)
 		goto put_stateid;
 
+	status = nfs4_check_fh(&cstate->current_fh, &dp->dl_stid);
+	if (status)
+		goto put_stateid;
+
 	trace_nfsd_deleg_return(stateid);
 	destroy_delegation(dp);
 	smp_mb__after_atomic();
@@ -8506,6 +8897,9 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	status = nfserr_no_grace;
 	if (!locks_in_grace(net) && lock->lk_reclaim)
 		goto out;
+	if (lock->lk_reclaim &&
+	    test_bit(NFSD4_CLIENT_RECLAIM_COMPLETE, &cstate->clp->cl_flags))
+		goto out;
 
 	if (lock->lk_reclaim)
 		flags |= FL_RECLAIM;
@@ -8542,10 +8936,11 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		goto out;
 	}
 
-	if (lock->lk_type & (NFS4_READW_LT | NFS4_WRITEW_LT) &&
-		nfsd4_has_session(cstate) &&
-		locks_can_async_lock(nf->nf_file->f_op))
-			flags |= FL_SLEEP;
+	if ((lock->lk_type == NFS4_READW_LT ||
+	     lock->lk_type == NFS4_WRITEW_LT) &&
+	    nfsd4_has_session(cstate) &&
+	    locks_can_async_lock(nf->nf_file->f_op))
+		flags |= FL_SLEEP;
 
 	nbl = find_or_allocate_block(lock_sop, &fp->fi_fhandle, nn);
 	if (!nbl) {
@@ -8587,7 +8982,7 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		nfs4_inc_and_copy_stateid(&lock->lk_resp_stateid, &lock_stp->st_stid);
 		status = 0;
 		if (lock->lk_reclaim)
-			nn->somebody_reclaimed = true;
+			set_bit(NFSD_NET_SOMEBODY_RECLAIMED, &nn->flags);
 		break;
 	case FILE_LOCK_DEFERRED:
 		kref_put(&nbl->nbl_kref, free_nbl);
@@ -8963,9 +9358,13 @@ bool
 nfs4_has_reclaimed_state(struct xdr_netobj name, struct nfsd_net *nn)
 {
 	struct nfs4_client_reclaim *crp;
+	bool found;
 
+	down_read(&nn->reclaim_str_hashtbl_lock);
 	crp = nfsd4_find_reclaim_client(name, nn);
-	return (crp && crp->cr_clp);
+	found = (crp && crp->cr_clp);
+	up_read(&nn->reclaim_str_hashtbl_lock);
+	return found;
 }
 
 /*
@@ -8978,10 +9377,39 @@ nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash,
 	unsigned int strhashval;
 	struct nfs4_client_reclaim *crp;
 
+	down_write(&nn->reclaim_str_hashtbl_lock);
+
+	/*
+	 * A reclaim record for this client name may already exist (for
+	 * example, populated at boot from the recovery directory before
+	 * an in-grace RECLAIM_COMPLETE or an nfsdcld downcall delivers
+	 * the same name). Dedup here so reclaim_str_hashtbl_size stays
+	 * equal to the number of distinct client names; inc_reclaim_complete
+	 * relies on that equality to end the grace period via the fast path.
+	 */
+	crp = nfsd4_find_reclaim_client(name, nn);
+	if (crp) {
+		if (princhash.len && crp->cr_princhash.len == 0) {
+			void *pdata = kmemdup(princhash.data, princhash.len,
+					      GFP_KERNEL);
+			if (pdata) {
+				crp->cr_princhash.data = pdata;
+				crp->cr_princhash.len = princhash.len;
+			} else {
+				dprintk("%s: failed to allocate memory for princhash.data!\n",
+					__func__);
+				crp = NULL;
+			}
+		}
+		up_write(&nn->reclaim_str_hashtbl_lock);
+		return crp;
+	}
+
 	name.data = kmemdup(name.data, name.len, GFP_KERNEL);
 	if (!name.data) {
 		dprintk("%s: failed to allocate memory for name.data!\n",
 			__func__);
+		up_write(&nn->reclaim_str_hashtbl_lock);
 		return NULL;
 	}
 	if (princhash.len) {
@@ -8990,6 +9418,7 @@ nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash,
 			dprintk("%s: failed to allocate memory for princhash.data!\n",
 				__func__);
 			kfree(name.data);
+			up_write(&nn->reclaim_str_hashtbl_lock);
 			return NULL;
 		}
 	} else
@@ -9009,6 +9438,7 @@ nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash,
 		kfree(name.data);
 		kfree(princhash.data);
 	}
+	up_write(&nn->reclaim_str_hashtbl_lock);
 	return crp;
 }
 
@@ -9028,6 +9458,7 @@ nfs4_release_reclaim(struct nfsd_net *nn)
 	struct nfs4_client_reclaim *crp = NULL;
 	int i;
 
+	down_write(&nn->reclaim_str_hashtbl_lock);
 	for (i = 0; i < CLIENT_HASH_SIZE; i++) {
 		while (!list_empty(&nn->reclaim_str_hashtbl[i])) {
 			crp = list_entry(nn->reclaim_str_hashtbl[i].next,
@@ -9036,6 +9467,7 @@ nfs4_release_reclaim(struct nfsd_net *nn)
 		}
 	}
 	WARN_ON_ONCE(nn->reclaim_str_hashtbl_size);
+	up_write(&nn->reclaim_str_hashtbl_lock);
 }
 
 /*
@@ -9113,8 +9545,9 @@ static int nfs4_state_create_net(struct net *net)
 	nn->conf_name_tree = RB_ROOT;
 	nn->unconf_name_tree = RB_ROOT;
 	nn->boot_time = ktime_get_real_seconds();
-	nn->grace_ended = false;
-	nn->grace_end_forced = false;
+	nn->boot_time_bt = ktime_get_boottime_seconds();
+	clear_bit(NFSD_NET_GRACE_ENDED, &nn->flags);
+	clear_bit(NFSD_NET_GRACE_END_FORCED, &nn->flags);
 	nn->nfsd4_manager.block_opens = true;
 	INIT_LIST_HEAD(&nn->nfsd4_manager.list);
 	INIT_LIST_HEAD(&nn->client_lru);
@@ -9200,7 +9633,8 @@ nfs4_state_start_net(struct net *net)
 	nfsd4_client_tracking_init(net);
 	/* safe for laundromat to run now */
 	enable_delayed_work(&nn->laundromat_work);
-	if (nn->track_reclaim_completes && nn->reclaim_str_hashtbl_size == 0)
+	if (test_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags) &&
+	    nn->reclaim_str_hashtbl_size == 0)
 		goto skip_grace;
 	printk(KERN_INFO "NFSD: starting %lld-second grace period (net %x)\n",
 	       nn->nfsd4_grace, net->ns.inum);
@@ -9231,8 +9665,8 @@ nfs4_state_start(void)
 		rhltable_destroy(&nfs4_file_rhltable);
 		return -ENOMEM;
 	}
-	nfsd_slot_shrinker->count_objects = nfsd_slot_count;
-	nfsd_slot_shrinker->scan_objects = nfsd_slot_scan;
+	nfsd_slot_shrinker->count_objects = nfsd_slot_shrinker_count;
+	nfsd_slot_shrinker->scan_objects = nfsd_slot_shrinker_scan;
 	shrinker_register(nfsd_slot_shrinker);
 
 	set_max_delegations();
@@ -9569,6 +10003,30 @@ nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry,
 	return status;
 }
 
+#define GDD_WORD0_CHILD_ATTRS	(FATTR4_WORD0_TYPE |		\
+				 FATTR4_WORD0_CHANGE |		\
+				 FATTR4_WORD0_SIZE |		\
+				 FATTR4_WORD0_FILEID |		\
+				 FATTR4_WORD0_FILEHANDLE)
+
+#define GDD_WORD1_CHILD_ATTRS	(FATTR4_WORD1_MODE |		\
+				 FATTR4_WORD1_NUMLINKS |	\
+				 FATTR4_WORD1_RAWDEV |		\
+				 FATTR4_WORD1_SPACE_USED |	\
+				 FATTR4_WORD1_TIME_ACCESS |	\
+				 FATTR4_WORD1_TIME_METADATA |	\
+				 FATTR4_WORD1_TIME_MODIFY |	\
+				 FATTR4_WORD1_TIME_CREATE)
+
+#define GDD_WORD0_DIR_ATTRS	(FATTR4_WORD0_CHANGE |		\
+				 FATTR4_WORD0_SIZE)
+
+#define GDD_WORD1_DIR_ATTRS	(FATTR4_WORD1_NUMLINKS |	\
+				 FATTR4_WORD1_SPACE_USED |	\
+				 FATTR4_WORD1_TIME_ACCESS |	\
+				 FATTR4_WORD1_TIME_METADATA |	\
+				 FATTR4_WORD1_TIME_MODIFY)
+
 /**
  * nfsd_get_dir_deleg - attempt to get a directory delegation
  * @cstate: compound state
@@ -9576,8 +10034,7 @@ nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry,
  * @nf: nfsd_file opened on the directory
  *
  * Given a GET_DIR_DELEGATION request @gdd, attempt to acquire a delegation
- * on the directory to which @nf refers. Note that this does not set up any
- * sort of async notifications for the delegation.
+ * on the directory to which @nf refers.
  */
 struct nfs4_delegation *
 nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate,
@@ -9614,8 +10071,9 @@ nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate,
 	/* existing delegation? */
 	if (nfs4_delegation_exists(clp, fp)) {
 		status = -EAGAIN;
-	} else if (!fp->fi_deleg_file) {
-		fp->fi_deleg_file = nfsd_file_get(nf);
+	} else if (!rcu_dereference_protected(fp->fi_deleg_file,
+					      lockdep_is_held(&fp->fi_lock))) {
+		rcu_assign_pointer(fp->fi_deleg_file, nfsd_file_get(nf));
 		fp->fi_delegees = 1;
 	} else {
 		++fp->fi_delegees;
@@ -9630,14 +10088,24 @@ nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate,
 
 	/* Try to set up the lease */
 	status = -ENOMEM;
-	dp = alloc_init_deleg(clp, fp, NULL, NFS4_OPEN_DELEGATE_READ);
+	dp = alloc_init_dir_deleg(clp, fp);
 	if (!dp)
 		goto out_delegees;
 	if (cstate->current_fh.fh_export)
 		dp->dl_stid.sc_export =
 			exp_get(cstate->current_fh.fh_export);
 
-	fl = nfs4_alloc_init_lease(dp);
+	/*
+	 * NB: gddr_notification[0] represents the notifications that
+	 * will be granted to the client
+	 */
+	dp->dl_notify_mask = gdd->gddr_notification[0];
+	dp->dl_child_attrs[0] = gdd->gdda_child_attributes[0] & GDD_WORD0_CHILD_ATTRS;
+	dp->dl_child_attrs[1] = gdd->gdda_child_attributes[1] & GDD_WORD1_CHILD_ATTRS;
+	dp->dl_dir_attrs[0] = gdd->gdda_dir_attributes[0] & GDD_WORD0_DIR_ATTRS;
+	dp->dl_dir_attrs[1] = gdd->gdda_dir_attributes[1] & GDD_WORD1_DIR_ATTRS;
+
+	fl = nfs4_alloc_init_lease(dp, dp->dl_notify_mask);
 	if (!fl)
 		goto out_put_stid;
 
@@ -9663,11 +10131,22 @@ nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate,
 
 	if (!status) {
 		put_nfs4_file(fp);
+		nfsd_fsnotify_recalc_mask(nf);
 		return dp;
 	}
 
-	/* Something failed. Drop the lease and clean up the stid */
-	kernel_setlease(fp->fi_deleg_file->nf_file, F_UNLCK, NULL, (void **)&dp);
+	/*
+	 * Something failed after the lease was set. Drop the lease and clean
+	 * up the stid. The lease's flc_file is the fi_deleg_file (see
+	 * nfs4_alloc_init_lease()), which is not necessarily this client's
+	 * @nf when an earlier client already holds a delegation on @fp.
+	 * generic_delete_lease() matches on flc_file, so unlock against
+	 * fi_deleg_file or the lease will be leaked (and later freed with the
+	 * stid, leading to a use-after-free when it's eventually broken).
+	 */
+	kernel_setlease(rcu_dereference_protected(fp->fi_deleg_file, 1)->nf_file,
+			F_UNLCK, NULL, (void **)&dp);
+	nfsd_fsnotify_recalc_mask(nf);
 out_put_stid:
 	nfs4_put_stid(&dp->dl_stid);
 out_delegees:
@@ -9703,3 +10182,170 @@ void nfsd_update_cmtime_attr(struct file *f, unsigned int flags)
 				      MINOR(inode->i_sb->s_dev),
 				      inode->i_ino, ret);
 }
+
+static void
+nfsd4_run_cb_notify(struct nfsd4_cb_notify *ncn)
+{
+	struct nfs4_delegation *dp = container_of(ncn, struct nfs4_delegation, dl_cb_notify);
+
+	if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &ncn->ncn_cb.cb_flags))
+		return;
+
+	if (!refcount_inc_not_zero(&dp->dl_stid.sc_count))
+		clear_bit(NFSD4_CALLBACK_RUNNING, &ncn->ncn_cb.cb_flags);
+	else
+		nfsd4_run_cb(&ncn->ncn_cb);
+}
+
+static struct nfsd_notify_event *
+alloc_nfsd_notify_event(u32 mask, const struct qstr *q, struct dentry *dentry,
+			struct inode *target)
+{
+	struct nfsd_notify_event *ne;
+	struct name_snapshot newname;
+	u32 newnamelen = 0;
+
+	/*
+	 * For a rename, @q is the old name and the live dentry carries the new
+	 * name. Snapshot the new name now, while it is guaranteed to describe
+	 * this event: the dentry can be renamed again before the CB_NOTIFY work
+	 * runs, which would corrupt a late read in nfsd4_encode_notify_event().
+	 */
+	if (mask & FS_RENAME) {
+		take_dentry_name_snapshot(&newname, dentry);
+		newnamelen = newname.name.len;
+	}
+
+	ne = kmalloc(struct_size(ne, ne_name, q->len + 1 +
+				 (newnamelen ? newnamelen + 1 : 0)), GFP_NOFS);
+	if (!ne)
+		goto out;
+
+	memcpy(ne->ne_name, q->name, q->len);
+	ne->ne_name[q->len] = '\0';
+	ne->ne_namelen = q->len;
+
+	ne->ne_newnamelen = newnamelen;
+	if (newnamelen) {
+		char *p = nfsd_notify_event_newname(ne);
+
+		memcpy(p, newname.name.name, newnamelen);
+		p[newnamelen] = '\0';
+	}
+
+	refcount_set(&ne->ne_ref, 1);
+	ne->ne_mask = mask;
+	ne->ne_dentry = dget(dentry);
+	ne->ne_target = target;
+	if (ne->ne_target)
+		ihold(ne->ne_target);
+out:
+	if (mask & FS_RENAME)
+		release_dentry_name_snapshot(&newname);
+	return ne;
+}
+
+static bool
+should_notify_deleg(u32 mask, struct file_lease *fl)
+{
+	/* Don't notify the client generating the event */
+	if (nfsd_breaker_owns_lease(fl))
+		return false;
+
+	/* Skip if this event wasn't ignored by the lease */
+	if ((mask & FS_DELETE) && !(fl->c.flc_flags & FL_IGN_DIR_DELETE))
+		return false;
+	if ((mask & FS_CREATE) && !(fl->c.flc_flags & FL_IGN_DIR_CREATE))
+		return false;
+	if ((mask & FS_RENAME) && !(fl->c.flc_flags & FL_IGN_DIR_RENAME))
+		return false;
+
+	return true;
+}
+
+static void
+nfsd_recall_all_dir_delegs(const struct inode *dir)
+{
+	struct file_lock_context *ctx = locks_inode_context(dir);
+	struct file_lock_core *flc;
+
+	spin_lock(&ctx->flc_lock);
+	list_for_each_entry(flc, &ctx->flc_lease, flc_list) {
+		struct file_lease *fl = container_of(flc, struct file_lease, c);
+
+		if (fl->fl_lmops == &nfsd_lease_mng_ops)
+			nfsd_break_deleg_cb(fl);
+	}
+	spin_unlock(&ctx->flc_lock);
+}
+
+int
+nfsd_handle_dir_event(u32 mask, const struct inode *dir, const void *data,
+		      int data_type, const struct qstr *name)
+{
+	struct dentry *dentry = fsnotify_data_dentry(data, data_type);
+	struct inode *target = fsnotify_data_rename_target(data, data_type);
+	struct file_lock_context *ctx;
+	struct file_lock_core *flc;
+	struct nfsd_notify_event *evt;
+
+	trace_nfsd_handle_dir_event(mask, dir, name);
+
+	/* Normalize cross-dir rename events to create/delete */
+	if (mask & FS_MOVED_FROM) {
+		mask &= ~FS_MOVED_FROM;
+		mask |= FS_DELETE;
+	}
+	if (mask & FS_MOVED_TO) {
+		mask &= ~FS_MOVED_TO;
+		mask |= FS_CREATE;
+	}
+
+	/*
+	 * FS_RENAME fires on the source directory even for a cross-dir
+	 * rename, where the moved entry now lives under a different parent.
+	 * NOTIFY4_RENAME_ENTRY describes an in-place rename, so reporting it
+	 * here would advertise a name absent from this directory.
+	 */
+	if ((mask & FS_RENAME) && dentry && d_inode(dentry->d_parent) != dir)
+		mask &= ~FS_RENAME;
+
+	/* Don't do anything if this is not an expected event */
+	if (!(mask & (FS_CREATE|FS_DELETE|FS_RENAME)))
+		return 0;
+
+	ctx = locks_inode_context(dir);
+	if (!ctx || list_empty(&ctx->flc_lease))
+		return 0;
+
+	evt = alloc_nfsd_notify_event(mask, name, dentry, target);
+	if (!evt) {
+		nfsd_recall_all_dir_delegs(dir);
+		return 0;
+	}
+
+	spin_lock(&ctx->flc_lock);
+	list_for_each_entry(flc, &ctx->flc_lease, flc_list) {
+		struct file_lease *fl = container_of(flc, struct file_lease, c);
+		struct nfs4_delegation *dp = flc->flc_owner;
+		struct nfsd4_cb_notify *ncn = &dp->dl_cb_notify;
+
+		if (!should_notify_deleg(mask, fl))
+			continue;
+
+		spin_lock(&ncn->ncn_lock);
+		if (ncn->ncn_evt_cnt >= NOTIFY4_EVENT_QUEUE_SIZE) {
+			/* We're generating notifications too fast. Recall. */
+			spin_unlock(&ncn->ncn_lock);
+			nfsd_break_deleg_cb(fl);
+			continue;
+		}
+		ncn->ncn_evt[ncn->ncn_evt_cnt++] = nfsd_notify_event_get(evt);
+		spin_unlock(&ncn->ncn_lock);
+
+		nfsd4_run_cb_notify(ncn);
+	}
+	spin_unlock(&ctx->flc_lock);
+	nfsd_notify_event_put(evt);
+	return 0;
+}
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index e17488a911f76..3affb144c66a8 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -244,7 +244,7 @@ nfsd4_decode_nfstime4(struct nfsd4_compoundargs *argp, struct timespec64 *tv)
 		return nfserr_bad_xdr;
 	p = xdr_decode_hyper(p, &tv->tv_sec);
 	tv->tv_nsec = be32_to_cpup(p++);
-	if (tv->tv_nsec >= (u32)1000000000)
+	if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
 		return nfserr_inval;
 	return nfs_ok;
 }
@@ -449,9 +449,18 @@ nfsd4_decode_posixacl(struct nfsd4_compoundargs *argp, struct posix_acl **acl)
 	if (xdr_stream_decode_u32(argp->xdr, &count) < 0)
 		return nfserr_bad_xdr;
 
+	/*
+	 * The NFSv4 POSIX ACL draft doesn't define a max number of ACE's, but
+	 * the NFSACL spec does. For NFSv4, cap the number of entries to the v3
+	 * limit, as we want to ensure that ACLs set via NFSv4 POSIX ACL
+	 * extensions are retrievable via NFSACL.
+	 */
+	if (count > NFS_ACL_MAX_ENTRIES)
+		return nfserr_inval;
+
 	*acl = posix_acl_alloc(count, GFP_KERNEL);
 	if (*acl == NULL)
-		return nfserr_resource;
+		return nfserr_jukebox;
 
 	(*acl)->a_count = count;
 	for (ace = (*acl)->a_entries; ace < (*acl)->a_entries + count; ace++) {
@@ -628,6 +637,8 @@ nfsd4_decode_fattr4(struct nfsd4_compoundargs *argp, u32 *bmval, u32 bmlen,
 
 		if (!xdrgen_decode_fattr4_time_deleg_access(argp->xdr, &access))
 			return nfserr_bad_xdr;
+		if (access.nseconds >= NSEC_PER_SEC)
+			return nfserr_inval;
 		iattr->ia_atime.tv_sec = access.seconds;
 		iattr->ia_atime.tv_nsec = access.nseconds;
 		iattr->ia_valid |= ATTR_ATIME | ATTR_ATIME_SET | ATTR_DELEG;
@@ -637,6 +648,8 @@ nfsd4_decode_fattr4(struct nfsd4_compoundargs *argp, u32 *bmval, u32 bmlen,
 
 		if (!xdrgen_decode_fattr4_time_deleg_modify(argp->xdr, &modify))
 			return nfserr_bad_xdr;
+		if (modify.nseconds >= NSEC_PER_SEC)
+			return nfserr_inval;
 		iattr->ia_mtime.tv_sec = modify.seconds;
 		iattr->ia_mtime.tv_nsec = modify.nseconds;
 		iattr->ia_ctime.tv_sec = modify.seconds;
@@ -955,6 +968,10 @@ nfsd4_decode_create(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u)
 	case NF4LNK:
 		if (xdr_stream_decode_u32(argp->xdr, &create->cr_datalen) < 0)
 			return nfserr_bad_xdr;
+		if (create->cr_datalen == 0)
+			return nfserr_inval;
+		if (create->cr_datalen > NFS4_MAXPATHLEN)
+			return nfserr_nametoolong;
 		p = xdr_inline_decode(argp->xdr, create->cr_datalen);
 		if (!p)
 			return nfserr_bad_xdr;
@@ -2702,7 +2719,7 @@ nfsd4_decode_compound(struct nfsd4_compoundargs *argp)
 }
 
 static __be32 nfsd4_encode_nfs_fh4(struct xdr_stream *xdr,
-				   struct knfsd_fh *fh_handle)
+				   const struct knfsd_fh *fh_handle)
 {
 	return nfsd4_encode_opaque(xdr, fh_handle->fh_raw, fh_handle->fh_size);
 }
@@ -3142,9 +3159,9 @@ nfsd4_encode_bitmap4(struct xdr_stream *xdr, u32 bmval0, u32 bmval1, u32 bmval2)
 
 struct nfsd4_fattr_args {
 	struct svc_rqst		*rqstp;
-	struct svc_fh		*fhp;
 	struct svc_export	*exp;
 	struct dentry		*dentry;
+	struct knfsd_fh		fhandle;
 	struct kstat		stat;
 	struct kstatfs		statfs;
 	struct nfs4_acl		*acl;
@@ -3260,7 +3277,7 @@ static __be32 nfsd4_encode_fattr4_change(struct xdr_stream *xdr,
 {
 	const struct svc_export *exp = args->exp;
 
-	if (unlikely(exp->ex_flags & NFSEXP_V4ROOT)) {
+	if (exp && unlikely(exp->ex_flags & NFSEXP_V4ROOT)) {
 		u32 flush_time = convert_to_wallclock(exp->cd->flush_time);
 
 		if (xdr_stream_encode_u32(xdr, flush_time) != XDR_UNIT)
@@ -3292,7 +3309,7 @@ static __be32 nfsd4_encode_fattr4_fsid(struct xdr_stream *xdr,
 		xdr_encode_hyper(p, NFS4_REFERRAL_FSID_MINOR);
 		return nfs_ok;
 	}
-	switch (fsid_source(args->fhp)) {
+	switch (fsid_source_fh(&args->fhandle, args->exp)) {
 	case FSIDSOURCE_FSID:
 		p = xdr_encode_hyper(p, (u64)args->exp->ex_fsid);
 		xdr_encode_hyper(p, (u64)0);
@@ -3389,7 +3406,7 @@ static __be32 nfsd4_encode_fattr4_homogeneous(struct xdr_stream *xdr,
 static __be32 nfsd4_encode_fattr4_filehandle(struct xdr_stream *xdr,
 					     const struct nfsd4_fattr_args *args)
 {
-	return nfsd4_encode_nfs_fh4(xdr, &args->fhp->fh_handle);
+	return nfsd4_encode_nfs_fh4(xdr, &args->fhandle);
 }
 
 static __be32 nfsd4_encode_fattr4_fileid(struct xdr_stream *xdr,
@@ -3882,6 +3899,22 @@ static const nfsd4_enc_attr nfsd4_enc_fattr4_encode_ops[] = {
 #endif
 };
 
+static __be32
+nfsd4_encode_attr_vals(struct xdr_stream *xdr, u32 *attrmask, struct nfsd4_fattr_args *args)
+{
+	DECLARE_BITMAP(attr_bitmap, ARRAY_SIZE(nfsd4_enc_fattr4_encode_ops));
+	unsigned long bit;
+	__be32 status;
+
+	bitmap_from_arr32(attr_bitmap, attrmask, ARRAY_SIZE(nfsd4_enc_fattr4_encode_ops));
+	for_each_set_bit(bit, attr_bitmap, ARRAY_SIZE(nfsd4_enc_fattr4_encode_ops)) {
+		status = nfsd4_enc_fattr4_encode_ops[bit](xdr, args);
+		if (status != nfs_ok)
+			return status;
+	}
+	return nfs_ok;
+}
+
 /*
  * Note: @fhp can be NULL; in this case, we might have to compose the filehandle
  * ourselves. @case_cache is NULL for callers that encode a single dentry
@@ -3895,7 +3928,6 @@ nfsd4_encode_fattr4(struct svc_rqst *rqstp, struct xdr_stream *xdr,
 		    int ignore_crossmnt,
 		    struct nfsd_case_attrs_cache *case_cache)
 {
-	DECLARE_BITMAP(attr_bitmap, ARRAY_SIZE(nfsd4_enc_fattr4_encode_ops));
 	struct nfs4_delegation *dp = NULL;
 	struct nfsd4_fattr_args args;
 	struct svc_fh *tempfh = NULL;
@@ -3910,7 +3942,6 @@ nfsd4_encode_fattr4(struct svc_rqst *rqstp, struct xdr_stream *xdr,
 		.mnt	= exp->ex_path.mnt,
 		.dentry	= dentry,
 	};
-	unsigned long bit;
 
 	WARN_ON_ONCE(bmval[1] & NFSD_WRITEONLY_ATTRS_WORD1);
 	WARN_ON_ONCE(!nfsd_attrs_supported(minorversion, bmval));
@@ -3988,19 +4019,22 @@ nfsd4_encode_fattr4(struct svc_rqst *rqstp, struct xdr_stream *xdr,
 		if (err)
 			goto out_nfserr;
 	}
-	if ((attrmask[0] & (FATTR4_WORD0_FILEHANDLE | FATTR4_WORD0_FSID)) &&
-	    !fhp) {
-		tempfh = kmalloc_obj(struct svc_fh);
-		status = nfserr_jukebox;
-		if (!tempfh)
-			goto out;
-		fh_init(tempfh, NFS4_FHSIZE);
-		status = fh_compose(tempfh, exp, dentry, NULL);
-		if (status)
-			goto out;
-		args.fhp = tempfh;
-	} else
-		args.fhp = fhp;
+
+	if ((attrmask[0] & (FATTR4_WORD0_FILEHANDLE | FATTR4_WORD0_FSID))) {
+		if (!fhp) {
+			tempfh = kmalloc_obj(struct svc_fh);
+			status = nfserr_jukebox;
+			if (!tempfh)
+				goto out;
+			fh_init(tempfh, NFS4_FHSIZE);
+			status = fh_compose(tempfh, exp, dentry, NULL);
+			if (status)
+				goto out;
+			fhp = tempfh;
+		}
+		fh_copy_shallow(&args.fhandle, &fhp->fh_handle);
+	}
+
 	if (attrmask[0] & (FATTR4_WORD0_CASE_INSENSITIVE |
 			   FATTR4_WORD0_CASE_PRESERVING)) {
 		/*
@@ -4124,27 +4158,22 @@ nfsd4_encode_fattr4(struct svc_rqst *rqstp, struct xdr_stream *xdr,
 #endif /* CONFIG_NFSD_V4_POSIX_ACLS */
 
 	/* attrmask */
-	status = nfsd4_encode_bitmap4(xdr, attrmask[0], attrmask[1],
-				      attrmask[2]);
+	status = nfsd4_encode_bitmap4(xdr, attrmask[0], attrmask[1], attrmask[2]);
 	if (status)
 		goto out;
 
 	/* attr_vals */
 	attrlen_offset = xdr->buf->len;
-	if (unlikely(!xdr_reserve_space(xdr, XDR_UNIT)))
-		goto out_resource;
-	bitmap_from_arr32(attr_bitmap, attrmask,
-			  ARRAY_SIZE(nfsd4_enc_fattr4_encode_ops));
-	for_each_set_bit(bit, attr_bitmap,
-			 ARRAY_SIZE(nfsd4_enc_fattr4_encode_ops)) {
-		status = nfsd4_enc_fattr4_encode_ops[bit](xdr, &args);
-		if (status != nfs_ok)
-			goto out;
+	if (unlikely(!xdr_reserve_space(xdr, XDR_UNIT))) {
+		status = nfserr_resource;
+		goto out;
 	}
-	attrlen = cpu_to_be32(xdr->buf->len - attrlen_offset - XDR_UNIT);
-	write_bytes_to_xdr_buf(xdr->buf, attrlen_offset, &attrlen, XDR_UNIT);
-	status = nfs_ok;
 
+	status = nfsd4_encode_attr_vals(xdr, attrmask, &args);
+	if (status == nfs_ok) {
+		attrlen = cpu_to_be32(xdr->buf->len - attrlen_offset - XDR_UNIT);
+		write_bytes_to_xdr_buf(xdr->buf, attrlen_offset, &attrlen, XDR_UNIT);
+	}
 out:
 #ifdef CONFIG_NFSD_V4_POSIX_ACLS
 	if (args.dpacl)
@@ -4167,9 +4196,286 @@ nfsd4_encode_fattr4(struct svc_rqst *rqstp, struct xdr_stream *xdr,
 out_nfserr:
 	status = nfserrno(err);
 	goto out;
-out_resource:
-	status = nfserr_resource;
-	goto out;
+}
+
+static bool
+setup_notify_fhandle(struct dentry *dentry, struct nfs4_delegation *dp,
+		     struct nfsd_file *nf, struct nfsd4_fattr_args *args)
+{
+	struct nfs4_file *fi = dp->dl_stid.sc_file;
+	struct nfs4_client *clp = dp->dl_stid.sc_client;
+	int fileid_type, fsid_len, maxsize, flags = 0;
+	struct knfsd_fh *fhp = &args->fhandle;
+	struct inode *inode = d_inode(dentry);
+	struct inode *parent = NULL;
+	struct svc_export *exp;
+	struct fid *fid;
+	bool ret = false;
+
+	/*
+	 * drop_stid_export() can clear sc_export under cl_lock and drop its
+	 * reference when the delegation is admin-revoked, concurrently with
+	 * this callback. Grab our own reference under cl_lock so the export
+	 * can be neither NULL-raced nor freed while we encode.
+	 */
+	spin_lock(&clp->cl_lock);
+	exp = dp->dl_stid.sc_export;
+	if (exp)
+		exp_get(exp);
+	spin_unlock(&clp->cl_lock);
+
+	fsid_len = key_len(fi->fi_fhandle.fh_fsid_type);
+	fhp->fh_size = 4 + fsid_len;
+
+	/* Copy first 4 bytes + fsid */
+	memcpy(&fhp->fh_raw, &fi->fi_fhandle.fh_raw, fhp->fh_size);
+
+	fid = (struct fid *)(fh_fsid(fhp) + fsid_len/4);
+	maxsize = (NFS4_FHSIZE - fhp->fh_size)/4;
+
+	/*
+	 * Subtree-checking exports need a connectable filehandle so the
+	 * parent can be resolved at decode time. Derive this from the
+	 * delegation's export rather than the shared nfs4_file, which may
+	 * have been initialized under a different export.
+	 */
+	if (exp && !(exp->ex_flags & NFSEXP_NOSUBTREECHECK) &&
+	    !S_ISDIR(inode->i_mode)) {
+		parent = d_inode(nf->nf_file->f_path.dentry);
+		flags = EXPORT_FH_CONNECTABLE;
+	}
+
+	fileid_type = exportfs_encode_inode_fh(inode, fid, &maxsize, parent, flags);
+	if (fileid_type < 0 || fileid_type == FILEID_INVALID)
+		goto out;
+
+	fhp->fh_fileid_type = fileid_type;
+	fhp->fh_size += maxsize * 4;
+
+	if (exp && (exp->ex_flags & NFSEXP_SIGN_FH))
+		if (!fh_append_mac(fhp, NFS4_FHSIZE, exp->cd->net))
+			goto out;
+
+	ret = true;
+out:
+	if (exp)
+		exp_put(exp);
+	return ret;
+}
+
+#define CB_NOTIFY_STATX_REQUEST_MASK (STATX_BASIC_STATS   | \
+				      STATX_BTIME	  | \
+				      STATX_CHANGE_COOKIE)
+
+static bool
+nfsd4_setup_notify_entry4(struct notify_entry4 *ne, struct xdr_stream *xdr,
+			  struct dentry *dentry, struct nfs4_delegation *dp,
+			  struct nfsd_file *nf, char *name, u32 namelen)
+{
+	struct path path = nf->nf_file->f_path;
+	struct nfsd4_fattr_args args = { };
+	const u32 *reqmask;
+	uint32_t *attrmask;
+	__be32 status;
+	bool parent;
+	int ret;
+
+	/* Reserve space for attrmask */
+	attrmask = xdr_reserve_space(xdr, 3 * sizeof(uint32_t));
+	if (!attrmask)
+		return false;
+
+	ne->ne_file.data = name;
+	ne->ne_file.len = namelen;
+	ne->ne_attrs.attrmask.element = attrmask;
+
+	parent = (dentry == path.dentry);
+	path.dentry = dentry;
+	reqmask = parent ? dp->dl_dir_attrs : dp->dl_child_attrs;
+
+	/*
+	 * A NULL or negative dentry has no attributes to report (expected,
+	 * e.g. for the old entry of a rename or an entry already removed).
+	 * The client may also have been granted the notification while
+	 * requesting no attributes for this entry. Both cases encode an
+	 * empty attribute set rather than failing: the vfs_getattr() and
+	 * nfsd4_encode_attr_vals() failures below recall the delegation, so
+	 * a case with nothing to fetch must short-circuit ahead of them.
+	 */
+	if (!path.dentry || !d_inode(path.dentry) ||
+	    (!reqmask[0] && !reqmask[1])) {
+		attrmask[0] = 0;
+		attrmask[1] = 0;
+		attrmask[2] = 0;
+		ne->ne_attrs.attr_vals.data = NULL;
+		ne->ne_attrs.attr_vals.len = 0;
+		ne->ne_attrs.attrmask.count = 1;
+		return true;
+	}
+
+	/*
+	 * It is possible that the client was granted a delegation when a file
+	 * was created. Note that we don't issue a CB_GETATTR here since stale
+	 * attributes are presumably ok.
+	 */
+	ret = vfs_getattr(&path, &args.stat, CB_NOTIFY_STATX_REQUEST_MASK, AT_STATX_SYNC_AS_STAT);
+	if (ret)
+		return false;
+
+	args.change_attr = nfsd4_change_attribute(&args.stat);
+
+	if (parent) {
+		attrmask[0] = dp->dl_dir_attrs[0];
+		attrmask[1] = dp->dl_dir_attrs[1];
+	} else {
+		attrmask[0] = dp->dl_child_attrs[0];
+		attrmask[1] = dp->dl_child_attrs[1];
+
+		if (!setup_notify_fhandle(dentry, dp, nf, &args))
+			attrmask[0] &= ~FATTR4_WORD0_FILEHANDLE;
+
+		if (!(args.stat.result_mask & STATX_BTIME))
+			attrmask[1] &= ~FATTR4_WORD1_TIME_CREATE;
+	}
+	attrmask[2] = 0;
+
+	ne->ne_attrs.attrmask.count = 2;
+	ne->ne_attrs.attr_vals.data = (u8 *)xdr->p;
+
+	status = nfsd4_encode_attr_vals(xdr, attrmask, &args);
+	if (status != nfs_ok)
+		return false;
+
+	ne->ne_attrs.attr_vals.len = (u8 *)xdr->p - ne->ne_attrs.attr_vals.data;
+	return true;
+}
+
+/**
+ * nfsd4_encode_notify_event - encode a notify
+ * @xdr: stream to which to encode the fattr4
+ * @nne: nfsd_notify_event to encode
+ * @dp: delegation where the event occurred
+ * @nf: nfsd_file on which event occurred
+ * @notify_mask: pointer to word where notification mask should be set
+ *
+ * Encode @nne into @xdr. The matching bit in @notify_mask is set on
+ * success.
+ *
+ * Return: pointer to the start of the encoded event, or NULL if the
+ * event could not be encoded.
+ */
+u8 *nfsd4_encode_notify_event(struct xdr_stream *xdr, struct nfsd_notify_event *nne,
+			      struct nfs4_delegation *dp, struct nfsd_file *nf,
+			      u32 *notify_mask)
+{
+	u8 *p = NULL;
+
+	*notify_mask = 0;
+
+	if (nne->ne_mask & FS_DELETE) {
+		struct notify_remove4 nr = { };
+
+		if (!nfsd4_setup_notify_entry4(&nr.nrm_old_entry, xdr, nne->ne_dentry, dp,
+					       nf, nne->ne_name, nne->ne_namelen))
+			goto out_err;
+		p = (u8 *)xdr->p;
+		if (!xdrgen_encode_notify_remove4(xdr, &nr))
+			goto out_err;
+		*notify_mask |= BIT(NOTIFY4_REMOVE_ENTRY);
+	} else if (nne->ne_mask & FS_CREATE) {
+		struct notify_add4 na = { };
+		struct notify_remove4 old = { };
+
+		if (!nfsd4_setup_notify_entry4(&na.nad_new_entry, xdr, nne->ne_dentry, dp,
+					       nf, nne->ne_name, nne->ne_namelen))
+			goto out_err;
+
+		/* If a file was overwritten, report it in nad_old_entry */
+		if (nne->ne_target) {
+			if (!nfsd4_setup_notify_entry4(&old.nrm_old_entry, xdr,
+						       NULL, dp, nf,
+						       nne->ne_name, nne->ne_namelen))
+				goto out_err;
+			na.nad_old_entry.count = 1;
+			na.nad_old_entry.element = &old;
+		}
+
+		p = (u8 *)xdr->p;
+		if (!xdrgen_encode_notify_add4(xdr, &na))
+			goto out_err;
+
+		*notify_mask |= BIT(NOTIFY4_ADD_ENTRY);
+	} else if (nne->ne_mask & FS_RENAME) {
+		struct notify_rename4 nr = { };
+		struct notify_remove4 old = { };
+		char *newname = nfsd_notify_event_newname(nne);
+
+		/* Don't send any attributes in the old_entry since they're the same in new */
+		if (!nfsd4_setup_notify_entry4(&nr.nrn_old_entry.nrm_old_entry, xdr,
+					       NULL, dp, nf, nne->ne_name,
+					       nne->ne_namelen))
+			goto out_err;
+
+		if (!nfsd4_setup_notify_entry4(&nr.nrn_new_entry.nad_new_entry, xdr,
+					       nne->ne_dentry, dp, nf, newname,
+					       nne->ne_newnamelen))
+			goto out_err;
+
+		/* If a file was overwritten, report it in nad_old_entry */
+		if (nne->ne_target) {
+			if (!nfsd4_setup_notify_entry4(&old.nrm_old_entry, xdr,
+						       NULL, dp, nf, newname,
+						       nne->ne_newnamelen))
+				goto out_err;
+			nr.nrn_new_entry.nad_old_entry.count = 1;
+			nr.nrn_new_entry.nad_old_entry.element = &old;
+		}
+
+		p = (u8 *)xdr->p;
+		if (!xdrgen_encode_notify_rename4(xdr, &nr))
+			goto out_err;
+		*notify_mask |= BIT(NOTIFY4_RENAME_ENTRY);
+	}
+	return p;
+out_err:
+	pr_warn("nfsd: unable to marshal notify event to xdr stream\n");
+	return NULL;
+}
+
+/**
+ * nfsd4_encode_dir_attr_change
+ * @xdr: stream to which to encode the fattr4
+ * @dp: delegation where the event occurred
+ * @nf: nfsd_file opened on the directory
+ *
+ * Encode a dir attr change event.
+ *
+ * Return: a pointer to the start of the encoded event on success; NULL
+ * if there were no requested attributes to report, in which case the
+ * caller should omit the event; or an ERR_PTR if the event was requested
+ * but could not be marshalled into @xdr, in which case the caller should
+ * recall the delegation.
+ */
+u8 *nfsd4_encode_dir_attr_change(struct xdr_stream *xdr, struct nfs4_delegation *dp,
+				 struct nfsd_file *nf)
+{
+	struct dentry *dentry = nf->nf_file->f_path.dentry;
+	struct notify_attr4 na = { };
+	u8 *p;
+
+	/* RFC 8881 s10.4.3: ne_file must be a zero-length string for dir attrs */
+	if (!nfsd4_setup_notify_entry4(&na.na_changed_entry, xdr,
+				       dentry, dp, nf, "", 0))
+		return ERR_PTR(-ENOBUFS);
+
+	/* No requested attributes to report; omit the event */
+	if (!na.na_changed_entry.ne_attrs.attr_vals.len)
+		return NULL;
+
+	p = (u8 *)xdr->p;
+	if (!xdrgen_encode_notify_attr4(xdr, &na))
+		return ERR_PTR(-ENOBUFS);
+	return p;
 }
 
 static void svcxdr_init_encode_from_buffer(struct xdr_stream *xdr,
@@ -6390,9 +6696,6 @@ nfsd4_encode_operation(struct nfsd4_compoundres *resp, struct nfsd4_op *op)
 	write_bytes_to_xdr_buf(xdr->buf, op_status_offset,
 			       &op->status, XDR_UNIT);
 release:
-	if (opdesc && opdesc->op_release)
-		opdesc->op_release(&op->u);
-
 	/*
 	 * Account for pages consumed while encoding this operation.
 	 * The xdr_stream primitives don't manage rq_next_page.
@@ -6424,9 +6727,12 @@ void nfsd4_release_compoundargs(struct svc_rqst *rqstp)
 {
 	struct nfsd4_compoundargs *args = rqstp->rq_argp;
 
+	args->opcnt = 0;
 	if (args->ops != args->iops) {
-		vfree(args->ops);
+		void *old_ops = args->ops;
+
 		args->ops = args->iops;
+		kvfree_rcu_mightsleep(old_ops);
 	}
 	while (args->to_free) {
 		struct svcxdr_tmpbuf *tb = args->to_free;
diff --git a/fs/nfsd/nfs4xdr_gen.c b/fs/nfsd/nfs4xdr_gen.c
index 824497051b876..a6725c773768d 100644
--- a/fs/nfsd/nfs4xdr_gen.c
+++ b/fs/nfsd/nfs4xdr_gen.c
@@ -1,16 +1,16 @@
 // SPDX-License-Identifier: GPL-2.0
 // Generated by xdrgen. Manual edits will be lost.
 // XDR specification file: ../../Documentation/sunrpc/xdr/nfs4_1.x
-// XDR specification modification time: Thu Jan  8 23:12:07 2026
+// XDR specification modification time: Wed Mar 25 11:40:02 2026
 
 #include <linux/sunrpc/svc.h>
 
 #include "nfs4xdr_gen.h"
 
 static bool __maybe_unused
-xdrgen_decode_int64_t(struct xdr_stream *xdr, int64_t *ptr)
+xdrgen_decode_int32_t(struct xdr_stream *xdr, int32_t *ptr)
 {
-	return xdrgen_decode_hyper(xdr, ptr);
+	return xdrgen_decode_int(xdr, ptr);
 }
 
 static bool __maybe_unused
@@ -19,6 +19,154 @@ xdrgen_decode_uint32_t(struct xdr_stream *xdr, uint32_t *ptr)
 	return xdrgen_decode_unsigned_int(xdr, ptr);
 }
 
+static bool __maybe_unused
+xdrgen_decode_int64_t(struct xdr_stream *xdr, int64_t *ptr)
+{
+	return xdrgen_decode_hyper(xdr, ptr);
+}
+
+static bool __maybe_unused
+xdrgen_decode_uint64_t(struct xdr_stream *xdr, uint64_t *ptr)
+{
+	return xdrgen_decode_unsigned_hyper(xdr, ptr);
+}
+
+static bool __maybe_unused
+xdrgen_decode_nfsstat4(struct xdr_stream *xdr, nfsstat4 *ptr)
+{
+	u32 val;
+
+	if (xdr_stream_decode_u32(xdr, &val) < 0)
+		return false;
+	/* Compiler may optimize to a range check for dense enums */
+	switch (val) {
+	case NFS4_OK:
+	case NFS4ERR_PERM:
+	case NFS4ERR_NOENT:
+	case NFS4ERR_IO:
+	case NFS4ERR_NXIO:
+	case NFS4ERR_ACCESS:
+	case NFS4ERR_EXIST:
+	case NFS4ERR_XDEV:
+	case NFS4ERR_NOTDIR:
+	case NFS4ERR_ISDIR:
+	case NFS4ERR_INVAL:
+	case NFS4ERR_FBIG:
+	case NFS4ERR_NOSPC:
+	case NFS4ERR_ROFS:
+	case NFS4ERR_MLINK:
+	case NFS4ERR_NAMETOOLONG:
+	case NFS4ERR_NOTEMPTY:
+	case NFS4ERR_DQUOT:
+	case NFS4ERR_STALE:
+	case NFS4ERR_BADHANDLE:
+	case NFS4ERR_BAD_COOKIE:
+	case NFS4ERR_NOTSUPP:
+	case NFS4ERR_TOOSMALL:
+	case NFS4ERR_SERVERFAULT:
+	case NFS4ERR_BADTYPE:
+	case NFS4ERR_DELAY:
+	case NFS4ERR_SAME:
+	case NFS4ERR_DENIED:
+	case NFS4ERR_EXPIRED:
+	case NFS4ERR_LOCKED:
+	case NFS4ERR_GRACE:
+	case NFS4ERR_FHEXPIRED:
+	case NFS4ERR_SHARE_DENIED:
+	case NFS4ERR_WRONGSEC:
+	case NFS4ERR_CLID_INUSE:
+	case NFS4ERR_RESOURCE:
+	case NFS4ERR_MOVED:
+	case NFS4ERR_NOFILEHANDLE:
+	case NFS4ERR_MINOR_VERS_MISMATCH:
+	case NFS4ERR_STALE_CLIENTID:
+	case NFS4ERR_STALE_STATEID:
+	case NFS4ERR_OLD_STATEID:
+	case NFS4ERR_BAD_STATEID:
+	case NFS4ERR_BAD_SEQID:
+	case NFS4ERR_NOT_SAME:
+	case NFS4ERR_LOCK_RANGE:
+	case NFS4ERR_SYMLINK:
+	case NFS4ERR_RESTOREFH:
+	case NFS4ERR_LEASE_MOVED:
+	case NFS4ERR_ATTRNOTSUPP:
+	case NFS4ERR_NO_GRACE:
+	case NFS4ERR_RECLAIM_BAD:
+	case NFS4ERR_RECLAIM_CONFLICT:
+	case NFS4ERR_BADXDR:
+	case NFS4ERR_LOCKS_HELD:
+	case NFS4ERR_OPENMODE:
+	case NFS4ERR_BADOWNER:
+	case NFS4ERR_BADCHAR:
+	case NFS4ERR_BADNAME:
+	case NFS4ERR_BAD_RANGE:
+	case NFS4ERR_LOCK_NOTSUPP:
+	case NFS4ERR_OP_ILLEGAL:
+	case NFS4ERR_DEADLOCK:
+	case NFS4ERR_FILE_OPEN:
+	case NFS4ERR_ADMIN_REVOKED:
+	case NFS4ERR_CB_PATH_DOWN:
+	case NFS4ERR_BADIOMODE:
+	case NFS4ERR_BADLAYOUT:
+	case NFS4ERR_BAD_SESSION_DIGEST:
+	case NFS4ERR_BADSESSION:
+	case NFS4ERR_BADSLOT:
+	case NFS4ERR_COMPLETE_ALREADY:
+	case NFS4ERR_CONN_NOT_BOUND_TO_SESSION:
+	case NFS4ERR_DELEG_ALREADY_WANTED:
+	case NFS4ERR_BACK_CHAN_BUSY:
+	case NFS4ERR_LAYOUTTRYLATER:
+	case NFS4ERR_LAYOUTUNAVAILABLE:
+	case NFS4ERR_NOMATCHING_LAYOUT:
+	case NFS4ERR_RECALLCONFLICT:
+	case NFS4ERR_UNKNOWN_LAYOUTTYPE:
+	case NFS4ERR_SEQ_MISORDERED:
+	case NFS4ERR_SEQUENCE_POS:
+	case NFS4ERR_REQ_TOO_BIG:
+	case NFS4ERR_REP_TOO_BIG:
+	case NFS4ERR_REP_TOO_BIG_TO_CACHE:
+	case NFS4ERR_RETRY_UNCACHED_REP:
+	case NFS4ERR_UNSAFE_COMPOUND:
+	case NFS4ERR_TOO_MANY_OPS:
+	case NFS4ERR_OP_NOT_IN_SESSION:
+	case NFS4ERR_HASH_ALG_UNSUPP:
+	case NFS4ERR_CLIENTID_BUSY:
+	case NFS4ERR_PNFS_IO_HOLE:
+	case NFS4ERR_SEQ_FALSE_RETRY:
+	case NFS4ERR_BAD_HIGH_SLOT:
+	case NFS4ERR_DEADSESSION:
+	case NFS4ERR_ENCR_ALG_UNSUPP:
+	case NFS4ERR_PNFS_NO_LAYOUT:
+	case NFS4ERR_NOT_ONLY_OP:
+	case NFS4ERR_WRONG_CRED:
+	case NFS4ERR_WRONG_TYPE:
+	case NFS4ERR_DIRDELEG_UNAVAIL:
+	case NFS4ERR_REJECT_DELEG:
+	case NFS4ERR_RETURNCONFLICT:
+	case NFS4ERR_DELEG_REVOKED:
+	case NFS4ERR_PARTNER_NOTSUPP:
+	case NFS4ERR_PARTNER_NO_AUTH:
+	case NFS4ERR_UNION_NOTSUPP:
+	case NFS4ERR_OFFLOAD_DENIED:
+	case NFS4ERR_WRONG_LFS:
+	case NFS4ERR_BADLABEL:
+	case NFS4ERR_OFFLOAD_NO_REQS:
+	case NFS4ERR_NOXATTR:
+	case NFS4ERR_XATTR2BIG:
+		break;
+	default:
+		return false;
+	}
+	*ptr = val;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_attrlist4(struct xdr_stream *xdr, attrlist4 *ptr)
+{
+	return xdrgen_decode_opaque(xdr, ptr, 0);
+}
+
 static bool __maybe_unused
 xdrgen_decode_bitmap4(struct xdr_stream *xdr, bitmap4 *ptr)
 {
@@ -30,6 +178,24 @@ xdrgen_decode_bitmap4(struct xdr_stream *xdr, bitmap4 *ptr)
 	return true;
 }
 
+static bool __maybe_unused
+xdrgen_decode_verifier4(struct xdr_stream *xdr, verifier4 *ptr)
+{
+	return xdr_stream_decode_opaque_fixed(xdr, ptr, NFS4_VERIFIER_SIZE) == 0;
+}
+
+static bool __maybe_unused
+xdrgen_decode_nfs_cookie4(struct xdr_stream *xdr, nfs_cookie4 *ptr)
+{
+	return xdrgen_decode_uint64_t(xdr, ptr);
+}
+
+static bool __maybe_unused
+xdrgen_decode_nfs_fh4(struct xdr_stream *xdr, nfs_fh4 *ptr)
+{
+	return xdrgen_decode_opaque(xdr, ptr, NFS4_FHSIZE);
+}
+
 static bool __maybe_unused
 xdrgen_decode_utf8string(struct xdr_stream *xdr, utf8string *ptr)
 {
@@ -54,6 +220,29 @@ xdrgen_decode_utf8str_mixed(struct xdr_stream *xdr, utf8str_mixed *ptr)
 	return xdrgen_decode_utf8string(xdr, ptr);
 }
 
+static bool __maybe_unused
+xdrgen_decode_component4(struct xdr_stream *xdr, component4 *ptr)
+{
+	return xdrgen_decode_utf8str_cs(xdr, ptr);
+}
+
+static bool __maybe_unused
+xdrgen_decode_linktext4(struct xdr_stream *xdr, linktext4 *ptr)
+{
+	return xdrgen_decode_utf8str_cs(xdr, ptr);
+}
+
+static bool __maybe_unused
+xdrgen_decode_pathname4(struct xdr_stream *xdr, pathname4 *ptr)
+{
+	if (xdr_stream_decode_u32(xdr, &ptr->count) < 0)
+		return false;
+	for (u32 i = 0; i < ptr->count; i++)
+		if (!xdrgen_decode_component4(xdr, &ptr->element[i]))
+			return false;
+	return true;
+}
+
 static bool __maybe_unused
 xdrgen_decode_nfstime4(struct xdr_stream *xdr, struct nfstime4 *ptr)
 {
@@ -64,6 +253,26 @@ xdrgen_decode_nfstime4(struct xdr_stream *xdr, struct nfstime4 *ptr)
 	return true;
 }
 
+static bool __maybe_unused
+xdrgen_decode_fattr4(struct xdr_stream *xdr, struct fattr4 *ptr)
+{
+	if (!xdrgen_decode_bitmap4(xdr, &ptr->attrmask))
+		return false;
+	if (!xdrgen_decode_attrlist4(xdr, &ptr->attr_vals))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_stateid4(struct xdr_stream *xdr, struct stateid4 *ptr)
+{
+	if (!xdrgen_decode_uint32_t(xdr, &ptr->seqid))
+		return false;
+	if (xdr_stream_decode_opaque_fixed(xdr, ptr->other, 12) < 0)
+		return false;
+	return true;
+}
+
 static bool __maybe_unused
 xdrgen_decode_fattr4_offline(struct xdr_stream *xdr, fattr4_offline *ptr)
 {
@@ -366,9 +575,171 @@ xdrgen_decode_fattr4_posix_access_acl(struct xdr_stream *xdr, fattr4_posix_acces
  */
 
 static bool __maybe_unused
-xdrgen_encode_int64_t(struct xdr_stream *xdr, const int64_t value)
+xdrgen_decode_notify_type4(struct xdr_stream *xdr, notify_type4 *ptr)
 {
-	return xdrgen_encode_hyper(xdr, value);
+	u32 val;
+
+	if (xdr_stream_decode_u32(xdr, &val) < 0)
+		return false;
+	/* Compiler may optimize to a range check for dense enums */
+	switch (val) {
+	case NOTIFY4_CHANGE_CHILD_ATTRS:
+	case NOTIFY4_CHANGE_DIR_ATTRS:
+	case NOTIFY4_REMOVE_ENTRY:
+	case NOTIFY4_ADD_ENTRY:
+	case NOTIFY4_RENAME_ENTRY:
+	case NOTIFY4_CHANGE_COOKIE_VERIFIER:
+	case NOTIFY4_GFLAG_EXTEND:
+	case NOTIFY4_AUFLAG_VALID:
+	case NOTIFY4_AUFLAG_USER:
+	case NOTIFY4_AUFLAG_GROUP:
+	case NOTIFY4_AUFLAG_OTHER:
+	case NOTIFY4_CHANGE_AUTH:
+	case NOTIFY4_CFLAG_ORDER:
+	case NOTIFY4_AUFLAG_GANOW:
+	case NOTIFY4_AUFLAG_GALATER:
+	case NOTIFY4_CHANGE_GA:
+	case NOTIFY4_CHANGE_AMASK:
+		break;
+	default:
+		return false;
+	}
+	*ptr = val;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_notify_entry4(struct xdr_stream *xdr, struct notify_entry4 *ptr)
+{
+	if (!xdrgen_decode_component4(xdr, &ptr->ne_file))
+		return false;
+	if (!xdrgen_decode_fattr4(xdr, &ptr->ne_attrs))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_prev_entry4(struct xdr_stream *xdr, struct prev_entry4 *ptr)
+{
+	if (!xdrgen_decode_notify_entry4(xdr, &ptr->pe_prev_entry))
+		return false;
+	if (!xdrgen_decode_nfs_cookie4(xdr, &ptr->pe_prev_entry_cookie))
+		return false;
+	return true;
+}
+
+bool
+xdrgen_decode_notify_remove4(struct xdr_stream *xdr, struct notify_remove4 *ptr)
+{
+	if (!xdrgen_decode_notify_entry4(xdr, &ptr->nrm_old_entry))
+		return false;
+	if (!xdrgen_decode_nfs_cookie4(xdr, &ptr->nrm_old_entry_cookie))
+		return false;
+	return true;
+}
+
+bool
+xdrgen_decode_notify_add4(struct xdr_stream *xdr, struct notify_add4 *ptr)
+{
+	if (xdr_stream_decode_u32(xdr, &ptr->nad_old_entry.count) < 0)
+		return false;
+	if (ptr->nad_old_entry.count > 1)
+		return false;
+	for (u32 i = 0; i < ptr->nad_old_entry.count; i++)
+		if (!xdrgen_decode_notify_remove4(xdr, &ptr->nad_old_entry.element[i]))
+			return false;
+	if (!xdrgen_decode_notify_entry4(xdr, &ptr->nad_new_entry))
+		return false;
+	if (xdr_stream_decode_u32(xdr, &ptr->nad_new_entry_cookie.count) < 0)
+		return false;
+	if (ptr->nad_new_entry_cookie.count > 1)
+		return false;
+	for (u32 i = 0; i < ptr->nad_new_entry_cookie.count; i++)
+		if (!xdrgen_decode_nfs_cookie4(xdr, &ptr->nad_new_entry_cookie.element[i]))
+			return false;
+	if (xdr_stream_decode_u32(xdr, &ptr->nad_prev_entry.count) < 0)
+		return false;
+	if (ptr->nad_prev_entry.count > 1)
+		return false;
+	for (u32 i = 0; i < ptr->nad_prev_entry.count; i++)
+		if (!xdrgen_decode_prev_entry4(xdr, &ptr->nad_prev_entry.element[i]))
+			return false;
+	if (!xdrgen_decode_bool(xdr, &ptr->nad_last_entry))
+		return false;
+	return true;
+}
+
+bool
+xdrgen_decode_notify_attr4(struct xdr_stream *xdr, struct notify_attr4 *ptr)
+{
+	if (!xdrgen_decode_notify_entry4(xdr, &ptr->na_changed_entry))
+		return false;
+	return true;
+}
+
+bool
+xdrgen_decode_notify_rename4(struct xdr_stream *xdr, struct notify_rename4 *ptr)
+{
+	if (!xdrgen_decode_notify_remove4(xdr, &ptr->nrn_old_entry))
+		return false;
+	if (!xdrgen_decode_notify_add4(xdr, &ptr->nrn_new_entry))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_notify_verifier4(struct xdr_stream *xdr, struct notify_verifier4 *ptr)
+{
+	if (!xdrgen_decode_verifier4(xdr, &ptr->nv_old_cookieverf))
+		return false;
+	if (!xdrgen_decode_verifier4(xdr, &ptr->nv_new_cookieverf))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_notifylist4(struct xdr_stream *xdr, notifylist4 *ptr)
+{
+	return xdrgen_decode_opaque(xdr, ptr, 0);
+}
+
+static bool __maybe_unused
+xdrgen_decode_notify4(struct xdr_stream *xdr, struct notify4 *ptr)
+{
+	if (!xdrgen_decode_bitmap4(xdr, &ptr->notify_mask))
+		return false;
+	if (!xdrgen_decode_notifylist4(xdr, &ptr->notify_vals))
+		return false;
+	return true;
+}
+
+bool
+xdrgen_decode_CB_NOTIFY4args(struct xdr_stream *xdr, struct CB_NOTIFY4args *ptr)
+{
+	if (!xdrgen_decode_stateid4(xdr, &ptr->cna_stateid))
+		return false;
+	if (!xdrgen_decode_nfs_fh4(xdr, &ptr->cna_fh))
+		return false;
+	if (xdr_stream_decode_u32(xdr, &ptr->cna_changes.count) < 0)
+		return false;
+	for (u32 i = 0; i < ptr->cna_changes.count; i++)
+		if (!xdrgen_decode_notify4(xdr, &ptr->cna_changes.element[i]))
+			return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_decode_CB_NOTIFY4res(struct xdr_stream *xdr, struct CB_NOTIFY4res *ptr)
+{
+	if (!xdrgen_decode_nfsstat4(xdr, &ptr->cnr_status))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_int32_t(struct xdr_stream *xdr, const int32_t value)
+{
+	return xdrgen_encode_int(xdr, value);
 }
 
 static bool __maybe_unused
@@ -377,6 +748,30 @@ xdrgen_encode_uint32_t(struct xdr_stream *xdr, const uint32_t value)
 	return xdrgen_encode_unsigned_int(xdr, value);
 }
 
+static bool __maybe_unused
+xdrgen_encode_int64_t(struct xdr_stream *xdr, const int64_t value)
+{
+	return xdrgen_encode_hyper(xdr, value);
+}
+
+static bool __maybe_unused
+xdrgen_encode_uint64_t(struct xdr_stream *xdr, const uint64_t value)
+{
+	return xdrgen_encode_unsigned_hyper(xdr, value);
+}
+
+static bool __maybe_unused
+xdrgen_encode_nfsstat4(struct xdr_stream *xdr, nfsstat4 value)
+{
+	return xdr_stream_encode_u32(xdr, value) == XDR_UNIT;
+}
+
+static bool __maybe_unused
+xdrgen_encode_attrlist4(struct xdr_stream *xdr, const attrlist4 value)
+{
+	return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0;
+}
+
 static bool __maybe_unused
 xdrgen_encode_bitmap4(struct xdr_stream *xdr, const bitmap4 value)
 {
@@ -388,6 +783,24 @@ xdrgen_encode_bitmap4(struct xdr_stream *xdr, const bitmap4 value)
 	return true;
 }
 
+static bool __maybe_unused
+xdrgen_encode_verifier4(struct xdr_stream *xdr, const verifier4 value)
+{
+	return xdr_stream_encode_opaque_fixed(xdr, value, NFS4_VERIFIER_SIZE) >= 0;
+}
+
+static bool __maybe_unused
+xdrgen_encode_nfs_cookie4(struct xdr_stream *xdr, const nfs_cookie4 value)
+{
+	return xdrgen_encode_uint64_t(xdr, value);
+}
+
+static bool __maybe_unused
+xdrgen_encode_nfs_fh4(struct xdr_stream *xdr, const nfs_fh4 value)
+{
+	return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0;
+}
+
 static bool __maybe_unused
 xdrgen_encode_utf8string(struct xdr_stream *xdr, const utf8string value)
 {
@@ -412,6 +825,29 @@ xdrgen_encode_utf8str_mixed(struct xdr_stream *xdr, const utf8str_mixed value)
 	return xdrgen_encode_utf8string(xdr, value);
 }
 
+static bool __maybe_unused
+xdrgen_encode_component4(struct xdr_stream *xdr, const component4 value)
+{
+	return xdrgen_encode_utf8str_cs(xdr, value);
+}
+
+static bool __maybe_unused
+xdrgen_encode_linktext4(struct xdr_stream *xdr, const linktext4 value)
+{
+	return xdrgen_encode_utf8str_cs(xdr, value);
+}
+
+static bool __maybe_unused
+xdrgen_encode_pathname4(struct xdr_stream *xdr, const pathname4 value)
+{
+	if (xdr_stream_encode_u32(xdr, value.count) != XDR_UNIT)
+		return false;
+	for (u32 i = 0; i < value.count; i++)
+		if (!xdrgen_encode_component4(xdr, value.element[i]))
+			return false;
+	return true;
+}
+
 static bool __maybe_unused
 xdrgen_encode_nfstime4(struct xdr_stream *xdr, const struct nfstime4 *value)
 {
@@ -422,6 +858,26 @@ xdrgen_encode_nfstime4(struct xdr_stream *xdr, const struct nfstime4 *value)
 	return true;
 }
 
+static bool __maybe_unused
+xdrgen_encode_fattr4(struct xdr_stream *xdr, const struct fattr4 *value)
+{
+	if (!xdrgen_encode_bitmap4(xdr, value->attrmask))
+		return false;
+	if (!xdrgen_encode_attrlist4(xdr, value->attr_vals))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_stateid4(struct xdr_stream *xdr, const struct stateid4 *value)
+{
+	if (!xdrgen_encode_uint32_t(xdr, value->seqid))
+		return false;
+	if (xdr_stream_encode_opaque_fixed(xdr, value->other, 12) < 0)
+		return false;
+	return true;
+}
+
 static bool __maybe_unused
 xdrgen_encode_fattr4_offline(struct xdr_stream *xdr, const fattr4_offline value)
 {
@@ -567,3 +1023,137 @@ xdrgen_encode_fattr4_posix_access_acl(struct xdr_stream *xdr, const fattr4_posix
 			return false;
 	return true;
 }
+
+static bool __maybe_unused
+xdrgen_encode_notify_type4(struct xdr_stream *xdr, notify_type4 value)
+{
+	return xdr_stream_encode_u32(xdr, value) == XDR_UNIT;
+}
+
+static bool __maybe_unused
+xdrgen_encode_notify_entry4(struct xdr_stream *xdr, const struct notify_entry4 *value)
+{
+	if (!xdrgen_encode_component4(xdr, value->ne_file))
+		return false;
+	if (!xdrgen_encode_fattr4(xdr, &value->ne_attrs))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_prev_entry4(struct xdr_stream *xdr, const struct prev_entry4 *value)
+{
+	if (!xdrgen_encode_notify_entry4(xdr, &value->pe_prev_entry))
+		return false;
+	if (!xdrgen_encode_nfs_cookie4(xdr, value->pe_prev_entry_cookie))
+		return false;
+	return true;
+}
+
+bool
+xdrgen_encode_notify_remove4(struct xdr_stream *xdr, const struct notify_remove4 *value)
+{
+	if (!xdrgen_encode_notify_entry4(xdr, &value->nrm_old_entry))
+		return false;
+	if (!xdrgen_encode_nfs_cookie4(xdr, value->nrm_old_entry_cookie))
+		return false;
+	return true;
+}
+
+bool
+xdrgen_encode_notify_add4(struct xdr_stream *xdr, const struct notify_add4 *value)
+{
+	if (value->nad_old_entry.count > 1)
+		return false;
+	if (xdr_stream_encode_u32(xdr, value->nad_old_entry.count) != XDR_UNIT)
+		return false;
+	for (u32 i = 0; i < value->nad_old_entry.count; i++)
+		if (!xdrgen_encode_notify_remove4(xdr, &value->nad_old_entry.element[i]))
+			return false;
+	if (!xdrgen_encode_notify_entry4(xdr, &value->nad_new_entry))
+		return false;
+	if (value->nad_new_entry_cookie.count > 1)
+		return false;
+	if (xdr_stream_encode_u32(xdr, value->nad_new_entry_cookie.count) != XDR_UNIT)
+		return false;
+	for (u32 i = 0; i < value->nad_new_entry_cookie.count; i++)
+		if (!xdrgen_encode_nfs_cookie4(xdr, value->nad_new_entry_cookie.element[i]))
+			return false;
+	if (value->nad_prev_entry.count > 1)
+		return false;
+	if (xdr_stream_encode_u32(xdr, value->nad_prev_entry.count) != XDR_UNIT)
+		return false;
+	for (u32 i = 0; i < value->nad_prev_entry.count; i++)
+		if (!xdrgen_encode_prev_entry4(xdr, &value->nad_prev_entry.element[i]))
+			return false;
+	if (!xdrgen_encode_bool(xdr, value->nad_last_entry))
+		return false;
+	return true;
+}
+
+bool
+xdrgen_encode_notify_attr4(struct xdr_stream *xdr, const struct notify_attr4 *value)
+{
+	if (!xdrgen_encode_notify_entry4(xdr, &value->na_changed_entry))
+		return false;
+	return true;
+}
+
+bool
+xdrgen_encode_notify_rename4(struct xdr_stream *xdr, const struct notify_rename4 *value)
+{
+	if (!xdrgen_encode_notify_remove4(xdr, &value->nrn_old_entry))
+		return false;
+	if (!xdrgen_encode_notify_add4(xdr, &value->nrn_new_entry))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_notify_verifier4(struct xdr_stream *xdr, const struct notify_verifier4 *value)
+{
+	if (!xdrgen_encode_verifier4(xdr, value->nv_old_cookieverf))
+		return false;
+	if (!xdrgen_encode_verifier4(xdr, value->nv_new_cookieverf))
+		return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_notifylist4(struct xdr_stream *xdr, const notifylist4 value)
+{
+	return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0;
+}
+
+static bool __maybe_unused
+xdrgen_encode_notify4(struct xdr_stream *xdr, const struct notify4 *value)
+{
+	if (!xdrgen_encode_bitmap4(xdr, value->notify_mask))
+		return false;
+	if (!xdrgen_encode_notifylist4(xdr, value->notify_vals))
+		return false;
+	return true;
+}
+
+bool
+xdrgen_encode_CB_NOTIFY4args(struct xdr_stream *xdr, const struct CB_NOTIFY4args *value)
+{
+	if (!xdrgen_encode_stateid4(xdr, &value->cna_stateid))
+		return false;
+	if (!xdrgen_encode_nfs_fh4(xdr, value->cna_fh))
+		return false;
+	if (xdr_stream_encode_u32(xdr, value->cna_changes.count) != XDR_UNIT)
+		return false;
+	for (u32 i = 0; i < value->cna_changes.count; i++)
+		if (!xdrgen_encode_notify4(xdr, &value->cna_changes.element[i]))
+			return false;
+	return true;
+}
+
+static bool __maybe_unused
+xdrgen_encode_CB_NOTIFY4res(struct xdr_stream *xdr, const struct CB_NOTIFY4res *value)
+{
+	if (!xdrgen_encode_nfsstat4(xdr, value->cnr_status))
+		return false;
+	return true;
+}
diff --git a/fs/nfsd/nfs4xdr_gen.h b/fs/nfsd/nfs4xdr_gen.h
index 1c487f1a11ab2..f6a458a074067 100644
--- a/fs/nfsd/nfs4xdr_gen.h
+++ b/fs/nfsd/nfs4xdr_gen.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /* Generated by xdrgen. Manual edits will be lost. */
 /* XDR specification file: ../../Documentation/sunrpc/xdr/nfs4_1.x */
-/* XDR specification modification time: Thu Jan  8 23:12:07 2026 */
+/* XDR specification modification time: Wed Mar 25 11:40:02 2026 */
 
 #ifndef _LINUX_XDRGEN_NFS4_1_DECL_H
 #define _LINUX_XDRGEN_NFS4_1_DECL_H
@@ -32,4 +32,19 @@ bool xdrgen_decode_posixaceperm4(struct xdr_stream *xdr, posixaceperm4 *ptr);
 bool xdrgen_encode_posixaceperm4(struct xdr_stream *xdr, const posixaceperm4 value);
 
 
+bool xdrgen_decode_notify_remove4(struct xdr_stream *xdr, struct notify_remove4 *ptr);
+bool xdrgen_encode_notify_remove4(struct xdr_stream *xdr, const struct notify_remove4 *value);
+
+bool xdrgen_decode_notify_add4(struct xdr_stream *xdr, struct notify_add4 *ptr);
+bool xdrgen_encode_notify_add4(struct xdr_stream *xdr, const struct notify_add4 *value);
+
+bool xdrgen_decode_notify_attr4(struct xdr_stream *xdr, struct notify_attr4 *ptr);
+bool xdrgen_encode_notify_attr4(struct xdr_stream *xdr, const struct notify_attr4 *value);
+
+bool xdrgen_decode_notify_rename4(struct xdr_stream *xdr, struct notify_rename4 *ptr);
+bool xdrgen_encode_notify_rename4(struct xdr_stream *xdr, const struct notify_rename4 *value);
+
+bool xdrgen_decode_CB_NOTIFY4args(struct xdr_stream *xdr, struct CB_NOTIFY4args *ptr);
+bool xdrgen_encode_CB_NOTIFY4args(struct xdr_stream *xdr, const struct CB_NOTIFY4args *value);
+
 #endif /* _LINUX_XDRGEN_NFS4_1_DECL_H */
diff --git a/fs/nfsd/nfscache.c b/fs/nfsd/nfscache.c
index 154468ceccdc1..18f8556d33dd4 100644
--- a/fs/nfsd/nfscache.c
+++ b/fs/nfsd/nfscache.c
@@ -200,14 +200,14 @@ int nfsd_reply_cache_init(struct nfsd_net *nn)
 	nn->nfsd_reply_cache_shrinker->seeks = 1;
 	nn->nfsd_reply_cache_shrinker->private_data = nn;
 
-	shrinker_register(nn->nfsd_reply_cache_shrinker);
-
 	for (i = 0; i < hashsize; i++) {
 		INIT_LIST_HEAD(&nn->drc_hashtbl[i].lru_head);
 		spin_lock_init(&nn->drc_hashtbl[i].cache_lock);
 	}
 	nn->drc_hashsize = hashsize;
 
+	shrinker_register(nn->nfsd_reply_cache_shrinker);
+
 	return 0;
 out_shrinker:
 	kvfree(nn->drc_hashtbl);
diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
index d0486f4a47ba5..caf59421f8f4a 100644
--- a/fs/nfsd/nfsctl.c
+++ b/fs/nfsd/nfsctl.c
@@ -296,14 +296,15 @@ static ssize_t write_unlock_fs(struct file *file, char *buf, size_t size)
 	 * 2.  Is that directory a mount point, or
 	 * 3.  Is that directory the root of an exported file system?
 	 */
-	nfsd4_cancel_copy_by_sb(netns(file), path.dentry->d_sb);
 	error = nlmsvc_unlock_all_by_sb(path.dentry->d_sb);
 	mutex_lock(&nfsd_mutex);
 	nn = net_generic(netns(file), nfsd_net_id);
-	if (nn->nfsd_serv)
+	if (nn->nfsd_serv) {
+		nfsd4_cancel_copy_by_sb(netns(file), path.dentry->d_sb);
 		nfsd4_revoke_states(nn, path.dentry->d_sb);
-	else
+	} else {
 		error = -EINVAL;
+	}
 	mutex_unlock(&nfsd_mutex);
 
 	path_put(&path);
@@ -420,6 +421,7 @@ static ssize_t write_threads(struct file *file, char *buf, size_t size)
 	char *mesg = buf;
 	int rv;
 	struct net *net = netns(file);
+	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
 	if (size > 0) {
 		int newthreads;
@@ -430,7 +432,10 @@ static ssize_t write_threads(struct file *file, char *buf, size_t size)
 			return -EINVAL;
 		trace_nfsd_ctl_threads(net, newthreads);
 		mutex_lock(&nfsd_mutex);
-		rv = nfsd_svc(1, &newthreads, net, file->f_cred, NULL);
+		if (newthreads > 0 || nn->nfsd_serv != NULL)
+			rv = nfsd_svc(1, &newthreads, net, file->f_cred, NULL);
+		else
+			rv = 0;
 		mutex_unlock(&nfsd_mutex);
 		if (rv < 0)
 			return rv;
@@ -1111,7 +1116,7 @@ static ssize_t write_v4_end_grace(struct file *file, char *buf, size_t size)
 	}
 
 	return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%c\n",
-			 nn->grace_ended ? 'Y' : 'N');
+			 test_bit(NFSD_NET_GRACE_ENDED, &nn->flags) ? 'Y' : 'N');
 }
 
 #endif
@@ -1414,8 +1419,8 @@ static int create_proc_exports_entry(void)
 unsigned int nfsd_net_id;
 
 struct nfsd_genl_rqstp {
-	struct sockaddr		rq_daddr;
-	struct sockaddr		rq_saddr;
+	struct sockaddr_storage	rq_daddr;
+	struct sockaddr_storage	rq_saddr;
 	unsigned long		rq_flags;
 	ktime_t			rq_stime;
 	__be32			rq_xid;
@@ -1448,9 +1453,9 @@ static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb,
 	    nla_put_s64(skb, NFSD_A_RPC_STATUS_SERVICE_TIME,
 			ktime_to_us(genl_rqstp->rq_stime),
 			NFSD_A_RPC_STATUS_PAD))
-		return -ENOBUFS;
+		goto out_cancel;
 
-	switch (genl_rqstp->rq_saddr.sa_family) {
+	switch (genl_rqstp->rq_saddr.ss_family) {
 	case AF_INET: {
 		const struct sockaddr_in *s_in, *d_in;
 
@@ -1464,7 +1469,7 @@ static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb,
 				 s_in->sin_port) ||
 		    nla_put_be16(skb, NFSD_A_RPC_STATUS_DPORT,
 				 d_in->sin_port))
-			return -ENOBUFS;
+			goto out_cancel;
 		break;
 	}
 	case AF_INET6: {
@@ -1480,7 +1485,7 @@ static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb,
 				 s_in->sin6_port) ||
 		    nla_put_be16(skb, NFSD_A_RPC_STATUS_DPORT,
 				 d_in->sin6_port))
-			return -ENOBUFS;
+			goto out_cancel;
 		break;
 	}
 	}
@@ -1488,10 +1493,14 @@ static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb,
 	for (i = 0; i < genl_rqstp->rq_opcnt; i++)
 		if (nla_put_u32(skb, NFSD_A_RPC_STATUS_COMPOUND_OPS,
 				genl_rqstp->rq_opnum[i]))
-			return -ENOBUFS;
+			goto out_cancel;
 
 	genlmsg_end(skb, hdr);
 	return 0;
+
+out_cancel:
+	genlmsg_cancel(skb, hdr);
+	return -ENOBUFS;
 }
 
 /**
@@ -1519,18 +1528,28 @@ int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb,
 
 	for (i = 0; i < nn->nfsd_serv->sv_nrpools; i++) {
 		struct svc_rqst *rqstp;
+		long thread_skip = 0;
 
 		if (i < cb->args[0]) /* already consumed */
 			continue;
 
+		/*
+		 * The saved thread index only applies to the pool the dump
+		 * was resumed in. Subsequent pools must start from thread 0,
+		 * otherwise their first cb->args[1] threads are silently
+		 * skipped.
+		 */
+		if (i == cb->args[0])
+			thread_skip = cb->args[1];
+
 		rqstp_index = 0;
 		list_for_each_entry_rcu(rqstp,
 				&nn->nfsd_serv->sv_pools[i].sp_all_threads,
 				rq_all) {
-			struct nfsd_genl_rqstp genl_rqstp;
+			struct nfsd_genl_rqstp genl_rqstp = {};
 			unsigned int status_counter;
 
-			if (rqstp_index++ < cb->args[1]) /* already consumed */
+			if (rqstp_index++ < thread_skip) /* already consumed */
 				continue;
 			/*
 			 * Acquire rq_status_counter before parsing the rqst
@@ -1551,9 +1570,9 @@ int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb,
 			genl_rqstp.rq_stime = rqstp->rq_stime;
 			genl_rqstp.rq_opcnt = 0;
 			memcpy(&genl_rqstp.rq_daddr, svc_daddr(rqstp),
-			       sizeof(struct sockaddr));
+			       sizeof(struct sockaddr_storage));
 			memcpy(&genl_rqstp.rq_saddr, svc_addr(rqstp),
-			       sizeof(struct sockaddr));
+			       sizeof(struct sockaddr_storage));
 
 #ifdef CONFIG_NFSD_V4
 			if (rqstp->rq_vers == NFS4_VERSION &&
@@ -1572,17 +1591,26 @@ int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb,
 #endif /* CONFIG_NFSD_V4 */
 
 			/*
-			 * Acquire rq_status_counter before reporting the rqst
-			 * fields to the user.
+			 * Read-side load-load fence: order the field reads
+			 * above before the counter re-read below, mirroring
+			 * the smp_rmb() in the standard seqcount retry. The
+			 * begin-side smp_load_acquire() above pairs with the
+			 * smp_store_release() in nfsd_dispatch().
 			 */
-			if (smp_load_acquire(&rqstp->rq_status_counter) !=
-			    status_counter)
+			smp_rmb();
+			if (READ_ONCE(rqstp->rq_status_counter) != status_counter)
 				continue;
 
 			ret = nfsd_genl_rpc_status_compose_msg(skb, cb,
 							       &genl_rqstp);
-			if (ret)
+			if (ret) {
+				if (skb->len) {
+					cb->args[0] = i;
+					cb->args[1] = rqstp_index - 1;
+					ret = skb->len;
+				}
 				goto out;
+			}
 		}
 	}
 
@@ -1943,6 +1971,60 @@ int nfsd_nl_version_get_doit(struct sk_buff *skb, struct genl_info *info)
 	return err;
 }
 
+/**
+ * nfsd_nl_validate_listeners - sanity-check the listener list from userland
+ * @info: netlink metadata and command arguments
+ *
+ * Walk every NFSD_A_SERVER_SOCK_ADDR attribute and confirm that each entry
+ * is well-formed: it parses against the policy, carries both an address and
+ * a transport name, and the address is long enough for its family. Doing
+ * this up front lets the callers below assume every entry is valid and
+ * guarantees we make no changes when the request is malformed.
+ *
+ * Return: 0 if every entry is valid, or a negative errno otherwise.
+ */
+static int nfsd_nl_validate_listeners(struct genl_info *info)
+{
+	const struct nlattr *attr;
+	int rem;
+
+	nlmsg_for_each_attr_type(attr, NFSD_A_SERVER_SOCK_ADDR, info->nlhdr,
+				 GENL_HDRLEN, rem) {
+		struct nlattr *tb[NFSD_A_SOCK_MAX + 1];
+		struct sockaddr *sa;
+		int err;
+
+		err = nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr,
+				       nfsd_sock_nl_policy, info->extack);
+		if (err < 0)
+			return err;
+
+		if (!tb[NFSD_A_SOCK_ADDR] || !tb[NFSD_A_SOCK_TRANSPORT_NAME])
+			return -EINVAL;
+
+		sa = nla_data(tb[NFSD_A_SOCK_ADDR]);
+		if (nla_len(tb[NFSD_A_SOCK_ADDR]) < sizeof(sa->sa_family))
+			return -EINVAL;
+
+		switch (sa->sa_family) {
+		case AF_INET:
+			if (nla_len(tb[NFSD_A_SOCK_ADDR]) <
+			    sizeof(struct sockaddr_in))
+				return -EINVAL;
+			break;
+		case AF_INET6:
+			if (nla_len(tb[NFSD_A_SOCK_ADDR]) <
+			    sizeof(struct sockaddr_in6))
+				return -EINVAL;
+			break;
+		default:
+			return -EAFNOSUPPORT;
+		}
+	}
+
+	return 0;
+}
+
 /**
  * nfsd_nl_listener_set_doit - set the nfs running sockets
  * @skb: reply buffer
@@ -1961,6 +2043,15 @@ int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info)
 	bool delete = false;
 	int err, rem;
 
+	/*
+	 * Validate the entire listener list before making any changes, so a
+	 * malformed request fails cleanly without creating a serv or touching
+	 * the existing listeners.
+	 */
+	err = nfsd_nl_validate_listeners(info);
+	if (err)
+		return err;
+
 	mutex_lock(&nfsd_mutex);
 
 	err = nfsd_create_serv(net);
@@ -1987,16 +2078,11 @@ int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info)
 		const char *xcl_name;
 		struct sockaddr *sa;
 
+		/* validated up front in nfsd_nl_validate_listeners() */
 		if (nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr,
 				     nfsd_sock_nl_policy, info->extack) < 0)
 			continue;
 
-		if (!tb[NFSD_A_SOCK_ADDR] || !tb[NFSD_A_SOCK_TRANSPORT_NAME])
-			continue;
-
-		if (nla_len(tb[NFSD_A_SOCK_ADDR]) < sizeof(*sa))
-			continue;
-
 		xcl_name = nla_data(tb[NFSD_A_SOCK_TRANSPORT_NAME]);
 		sa = nla_data(tb[NFSD_A_SOCK_ADDR]);
 
@@ -2048,16 +2134,11 @@ int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info)
 		struct sockaddr *sa;
 		int ret;
 
+		/* validated up front in nfsd_nl_validate_listeners() */
 		if (nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr,
 				     nfsd_sock_nl_policy, info->extack) < 0)
 			continue;
 
-		if (!tb[NFSD_A_SOCK_ADDR] || !tb[NFSD_A_SOCK_TRANSPORT_NAME])
-			continue;
-
-		if (nla_len(tb[NFSD_A_SOCK_ADDR]) < sizeof(*sa))
-			continue;
-
 		xcl_name = nla_data(tb[NFSD_A_SOCK_TRANSPORT_NAME]);
 		sa = nla_data(tb[NFSD_A_SOCK_ADDR]);
 
@@ -2340,14 +2421,15 @@ int nfsd_nl_unlock_filesystem_doit(struct sk_buff *skb,
 	if (error)
 		return error;
 
-	nfsd4_cancel_copy_by_sb(net, path.dentry->d_sb);
 	error = nlmsvc_unlock_all_by_sb(path.dentry->d_sb);
 
 	mutex_lock(&nfsd_mutex);
-	if (nn->nfsd_serv)
+	if (nn->nfsd_serv) {
+		nfsd4_cancel_copy_by_sb(net, path.dentry->d_sb);
 		nfsd4_revoke_states(nn, path.dentry->d_sb);
-	else
+	} else {
 		error = -EINVAL;
+	}
 	mutex_unlock(&nfsd_mutex);
 
 	path_put(&path);
@@ -2511,11 +2593,12 @@ static int __init init_nfsd(void)
 {
 	int retval;
 
-	nfsd_debugfs_init();
-
 	retval = nfsd4_init_slabs();
 	if (retval)
 		return retval;
+
+	nfsd_debugfs_init();
+
 	retval = nfsd4_init_pnfs();
 	if (retval)
 		goto out_free_slabs;
@@ -2560,8 +2643,8 @@ static int __init init_nfsd(void)
 out_free_pnfs:
 	nfsd4_exit_pnfs();
 out_free_slabs:
-	nfsd4_free_slabs();
 	nfsd_debugfs_exit();
+	nfsd4_free_slabs();
 	return retval;
 }
 
@@ -2576,9 +2659,9 @@ static void __exit exit_nfsd(void)
 	unregister_pernet_subsys(&nfsd_net_ops);
 	nfsd_drc_slab_free();
 	nfsd_lockd_shutdown();
-	nfsd4_free_slabs();
 	nfsd4_exit_pnfs();
 	nfsd_debugfs_exit();
+	nfsd4_free_slabs();
 }
 
 MODULE_AUTHOR("Olaf Kirch <okir@monad.swb.de>");
diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h
index 11bce03b90316..d33d49b61e6af 100644
--- a/fs/nfsd/nfsd.h
+++ b/fs/nfsd/nfsd.h
@@ -332,13 +332,14 @@ void		nfsd_lockd_shutdown(void);
 #define nfserr_noxattr			cpu_to_be32(NFS4ERR_NOXATTR)
 
 /*
- * Error codes for internal use.  We use enum to choose numbers that are
- * not already assigned, then covert to be32 resulting in a number that
- * cannot conflict with any existing be32 nfserr value.
+ * Error codes for internal use.  These are based at an impossible
+ * nfsstat4 value so that, once converted to be32, they cannot conflict
+ * with any value defined by the protocol (compare the nlm__int__* codes
+ * in fs/lockd/lockd.h).
  */
 enum {
 /* end-of-file indicator in readdir */
-	NFSERR_EOF = NFS4ERR_FIRST_FREE,
+	NFSERR_EOF = 30000,
 #define	nfserr_eof		cpu_to_be32(NFSERR_EOF)
 
 /* replay detected */
diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c
index 429ca5c6ec087..8b1a95e1d0589 100644
--- a/fs/nfsd/nfsfh.c
+++ b/fs/nfsd/nfsfh.c
@@ -70,10 +70,8 @@ nfsd_mode_check(struct dentry *dentry, umode_t requested)
 	if (requested == 0) /* the caller doesn't care */
 		return nfs_ok;
 	if (mode == requested) {
-		if (mode == S_IFDIR && !d_can_lookup(dentry)) {
-			WARN_ON_ONCE(1);
+		if (mode == S_IFDIR && !d_can_lookup(dentry))
 			return nfserr_notdir;
-		}
 		return nfs_ok;
 	}
 	if (mode == S_IFLNK) {
@@ -144,16 +142,15 @@ static inline __be32 check_pseudo_root(struct dentry *dentry,
 /* Size of a file handle MAC, in 4-octet words */
 #define FH_MAC_WORDS (sizeof(__le64) / 4)
 
-static bool fh_append_mac(struct svc_fh *fhp, struct net *net)
+bool fh_append_mac(struct knfsd_fh *fh, int fh_maxsize, struct net *net)
 {
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
-	struct knfsd_fh *fh = &fhp->fh_handle;
 	siphash_key_t *fh_key = nn->fh_key;
 	__le64 hash;
 
 	if (!fh_key)
 		goto out_no_key;
-	if (fh->fh_size + sizeof(hash) > fhp->fh_maxsize)
+	if (fh->fh_size + sizeof(hash) > fh_maxsize)
 		goto out_no_space;
 
 	hash = cpu_to_le64(siphash(&fh->fh_raw, fh->fh_size, fh_key));
@@ -167,7 +164,7 @@ static bool fh_append_mac(struct svc_fh *fhp, struct net *net)
 
 out_no_space:
 	pr_warn_ratelimited("NFSD: unable to sign filehandles, fh_size %zu would be greater than fh_maxsize %d.\n",
-			    fh->fh_size + sizeof(hash), fhp->fh_maxsize);
+			    fh->fh_size + sizeof(hash), fh_maxsize);
 	return false;
 }
 
@@ -344,15 +341,19 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct net *net,
 		if (dentry->d_sb->s_export_op->flags & EXPORT_OP_NOWCC)
 			fhp->fh_no_wcc = true;
 		fhp->fh_64bit_cookies = true;
-		if (exp->ex_flags & NFSEXP_V4ROOT)
+		if (exp->ex_flags & NFSEXP_V4ROOT) {
+			dput(dentry);
 			goto out;
+		}
 		break;
 	case NFS_FHSIZE:
 		fhp->fh_no_wcc = true;
 		if (EX_WGATHER(exp))
 			fhp->fh_use_wgather = true;
-		if (exp->ex_flags & NFSEXP_V4ROOT)
+		if (exp->ex_flags & NFSEXP_V4ROOT) {
+			dput(dentry);
 			goto out;
+		}
 	}
 
 	fhp->fh_dentry = dentry;
@@ -562,7 +563,8 @@ static void _fh_update(struct svc_fh *fhp, struct svc_export *exp,
 		fhp->fh_handle.fh_size += maxsize * 4;
 
 		if (exp->ex_flags & NFSEXP_SIGN_FH)
-			if (!fh_append_mac(fhp, exp->cd->net))
+			if (!fh_append_mac(&fhp->fh_handle, fhp->fh_maxsize,
+					   exp->cd->net))
 				fhp->fh_handle.fh_fileid_type = FILEID_INVALID;
 	} else {
 		fhp->fh_handle.fh_fileid_type = FILEID_ROOT;
@@ -892,19 +894,20 @@ char * SVCFH_fmt(struct svc_fh *fhp)
 	return buf;
 }
 
-enum fsid_source fsid_source(const struct svc_fh *fhp)
+enum fsid_source fsid_source_fh(const struct knfsd_fh *fh,
+				struct svc_export *exp)
 {
-	if (fhp->fh_handle.fh_version != 1)
+	if (fh->fh_version != 1)
 		return FSIDSOURCE_DEV;
-	switch(fhp->fh_handle.fh_fsid_type) {
+	switch (fh->fh_fsid_type) {
 	case FSID_DEV:
 	case FSID_ENCODE_DEV:
 	case FSID_MAJOR_MINOR:
-		if (exp_sb(fhp->fh_export)->s_type->fs_flags & FS_REQUIRES_DEV)
+		if (exp_sb(exp)->s_type->fs_flags & FS_REQUIRES_DEV)
 			return FSIDSOURCE_DEV;
 		break;
 	case FSID_NUM:
-		if (fhp->fh_export->ex_flags & NFSEXP_FSID)
+		if (exp->ex_flags & NFSEXP_FSID)
 			return FSIDSOURCE_FSID;
 		break;
 	default:
@@ -913,13 +916,18 @@ enum fsid_source fsid_source(const struct svc_fh *fhp)
 	/* either a UUID type filehandle, or the filehandle doesn't
 	 * match the export.
 	 */
-	if (fhp->fh_export->ex_flags & NFSEXP_FSID)
+	if (exp->ex_flags & NFSEXP_FSID)
 		return FSIDSOURCE_FSID;
-	if (fhp->fh_export->ex_uuid)
+	if (exp->ex_uuid)
 		return FSIDSOURCE_UUID;
 	return FSIDSOURCE_DEV;
 }
 
+enum fsid_source fsid_source(const struct svc_fh *fhp)
+{
+	return fsid_source_fh(&fhp->fh_handle, fhp->fh_export);
+}
+
 /**
  * nfsd4_change_attribute - Generate an NFSv4 change_attribute value
  * @stat: inode attributes
diff --git a/fs/nfsd/nfsfh.h b/fs/nfsd/nfsfh.h
index 5ef7191f8ad81..cdeb5eea65a89 100644
--- a/fs/nfsd/nfsfh.h
+++ b/fs/nfsd/nfsfh.h
@@ -131,6 +131,8 @@ enum fsid_source {
 	FSIDSOURCE_FSID,
 	FSIDSOURCE_UUID,
 };
+extern enum fsid_source fsid_source_fh(const struct knfsd_fh *fh,
+				       struct svc_export *exp);
 extern enum fsid_source fsid_source(const struct svc_fh *fhp);
 
 
@@ -226,6 +228,7 @@ __be32	fh_getattr(const struct svc_fh *fhp, struct kstat *stat);
 __be32	fh_compose(struct svc_fh *, struct svc_export *, struct dentry *, struct svc_fh *);
 __be32	fh_update(struct svc_fh *);
 void	fh_put(struct svc_fh *);
+bool	fh_append_mac(struct knfsd_fh *fh, int fh_maxsize, struct net *net);
 
 static __inline__ struct svc_fh *
 fh_copy(struct svc_fh *dst, const struct svc_fh *src)
diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c
index 8873033d1e82f..a73d5c259cd93 100644
--- a/fs/nfsd/nfsproc.c
+++ b/fs/nfsd/nfsproc.c
@@ -82,6 +82,7 @@ nfsd_proc_setattr(struct svc_rqst *rqstp)
 		.na_iattr	= iap,
 	};
 	struct svc_fh *fhp;
+	int hosterr;
 
 	dprintk("nfsd: SETATTR  %s, valid=%x, size=%ld\n",
 		SVCFH_fmt(&argp->fh),
@@ -117,6 +118,12 @@ nfsd_proc_setattr(struct svc_rqst *rqstp)
 		if (resp->status != nfs_ok)
 			goto out;
 
+		hosterr = fh_want_write(fhp);
+		if (hosterr) {
+			resp->status = nfserrno(hosterr);
+			goto out;
+		}
+
 		if (delta < 0)
 			delta = -delta;
 		if (delta < MAX_TOUCH_TIME_ERROR &&
diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c
index 4f1ab3222a4d4..a8ea4dbfa56b4 100644
--- a/fs/nfsd/nfssvc.c
+++ b/fs/nfsd/nfssvc.c
@@ -237,15 +237,21 @@ static void nfsd_net_free(struct percpu_ref *ref)
  */
 #define	NFSD_MAXSERVS		8192
 
+/**
+ * nfsd_nrthreads - report a namespace's configured nfsd thread count
+ * @net: network namespace to query
+ *
+ * Return: the configured thread ceiling, or 0 when no service runs.
+ */
 int nfsd_nrthreads(struct net *net)
 {
-	int i, rv = 0;
+	int rv = 0;
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
+	/* nfsd_mutex keeps nn->nfsd_serv valid across the read. */
 	mutex_lock(&nfsd_mutex);
 	if (nn->nfsd_serv)
-		for (i = 0; i < nn->nfsd_serv->sv_nrpools; ++i)
-			rv += nn->nfsd_serv->sv_pools[i].sp_nrthrmax;
+		rv = svc_serv_maxthreads(nn->nfsd_serv);
 	mutex_unlock(&nfsd_mutex);
 	return rv;
 }
@@ -351,7 +357,7 @@ static int nfsd_startup_net(struct net *net, const struct cred *cred)
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 	int ret;
 
-	if (nn->nfsd_net_up)
+	if (test_bit(NFSD_NET_UP, &nn->flags))
 		return 0;
 
 	ret = nfsd_startup_generic();
@@ -364,11 +370,11 @@ static int nfsd_startup_net(struct net *net, const struct cred *cred)
 		goto out_socks;
 	}
 
-	if (nfsd_needs_lockd(nn) && !nn->lockd_up) {
+	if (nfsd_needs_lockd(nn) && !test_bit(NFSD_NET_LOCKD_UP, &nn->flags)) {
 		ret = lockd_up(net, cred);
 		if (ret)
 			goto out_socks;
-		nn->lockd_up = true;
+		set_bit(NFSD_NET_LOCKD_UP, &nn->flags);
 	}
 
 	ret = nfsd_file_cache_start_net(net);
@@ -386,7 +392,7 @@ static int nfsd_startup_net(struct net *net, const struct cred *cred)
 	if (ret)
 		goto out_reply_cache;
 
-	nn->nfsd_net_up = true;
+	set_bit(NFSD_NET_UP, &nn->flags);
 	return 0;
 
 out_reply_cache:
@@ -394,9 +400,9 @@ static int nfsd_startup_net(struct net *net, const struct cred *cred)
 out_filecache:
 	nfsd_file_cache_shutdown_net(net);
 out_lockd:
-	if (nn->lockd_up) {
+	if (test_bit(NFSD_NET_LOCKD_UP, &nn->flags)) {
 		lockd_down(net);
-		nn->lockd_up = false;
+		clear_bit(NFSD_NET_LOCKD_UP, &nn->flags);
 	}
 out_socks:
 	nfsd_shutdown_generic();
@@ -407,7 +413,7 @@ static void nfsd_shutdown_net(struct net *net)
 {
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
-	if (nn->nfsd_net_up) {
+	if (test_bit(NFSD_NET_UP, &nn->flags)) {
 		percpu_ref_kill_and_confirm(&nn->nfsd_net_ref, nfsd_net_done);
 		wait_for_completion(&nn->nfsd_net_confirm_done);
 
@@ -415,18 +421,18 @@ static void nfsd_shutdown_net(struct net *net)
 		nfs4_state_shutdown_net(net);
 		nfsd_reply_cache_shutdown(nn);
 		nfsd_file_cache_shutdown_net(net);
-		if (nn->lockd_up) {
+		if (test_bit(NFSD_NET_LOCKD_UP, &nn->flags)) {
 			lockd_down(net);
-			nn->lockd_up = false;
+			clear_bit(NFSD_NET_LOCKD_UP, &nn->flags);
 		}
 		wait_for_completion(&nn->nfsd_net_free_done);
 	}
 
 	percpu_ref_exit(&nn->nfsd_net_ref);
 
-	if (nn->nfsd_net_up)
+	if (test_bit(NFSD_NET_UP, &nn->flags))
 		nfsd_shutdown_generic();
-	nn->nfsd_net_up = false;
+	clear_bit(NFSD_NET_UP, &nn->flags);
 }
 
 static DEFINE_SPINLOCK(nfsd_notifier_lock);
@@ -815,7 +821,7 @@ nfsd_acl_init_request(struct svc_rqst *rqstp,
 
 	ret->mismatch.lovers = NFSD_ACL_NRVERS;
 	for (i = NFSD_ACL_MINVERS; i < NFSD_ACL_NRVERS; i++) {
-		if (nfsd_support_acl_version(rqstp->rq_vers) &&
+		if (nfsd_support_acl_version(i) &&
 		    nfsd_vers(nn, i, NFSD_TEST)) {
 			ret->mismatch.lovers = i;
 			break;
@@ -825,7 +831,7 @@ nfsd_acl_init_request(struct svc_rqst *rqstp,
 		return rpc_prog_unavail;
 	ret->mismatch.hivers = NFSD_ACL_MINVERS;
 	for (i = NFSD_ACL_NRVERS - 1; i >= NFSD_ACL_MINVERS; i--) {
-		if (nfsd_support_acl_version(rqstp->rq_vers) &&
+		if (nfsd_support_acl_version(i) &&
 		    nfsd_vers(nn, i, NFSD_TEST)) {
 			ret->mismatch.hivers = i;
 			break;
@@ -960,6 +966,20 @@ nfsd(void *vrqstp)
 	return 0;
 }
 
+/*
+ * Set rq_status_counter back to an even value, indicating that the rqstp
+ * fields are no longer meaningful to a lockless reader. This pairs with the
+ * odd-valued store made once the request has been decoded, and must run on
+ * every return path that follows it so that the seq-lock like protocol used
+ * by nfsd_nl_rpc_status_get_dumpit() is not left permanently odd. The store
+ * also advances the counter so a concurrent reader detects the transition.
+ */
+static void nfsd_status_counter_set_idle(struct svc_rqst *rqstp)
+{
+	smp_store_release(&rqstp->rq_status_counter,
+			  (rqstp->rq_status_counter | 1) + 1);
+}
+
 /**
  * nfsd_dispatch - Process an NFS or NFSACL or LOCALIO Request
  * @rqstp: incoming request
@@ -1022,14 +1042,9 @@ int nfsd_dispatch(struct svc_rqst *rqstp)
 	if (!proc->pc_encode(rqstp, &rqstp->rq_res_stream))
 		goto out_encode_err;
 
-	/*
-	 * Release rq_status_counter setting it to an even value after the rpc
-	 * request has been properly processed.
-	 */
-	smp_store_release(&rqstp->rq_status_counter, rqstp->rq_status_counter + 1);
-
 	nfsd_cache_update(rqstp, rp, ntli->ntli_cachetype, nfs_reply);
 out_cached_reply:
+	nfsd_status_counter_set_idle(rqstp);
 	return 1;
 
 out_decode_err:
@@ -1040,12 +1055,14 @@ int nfsd_dispatch(struct svc_rqst *rqstp)
 out_update_drop:
 	nfsd_cache_update(rqstp, rp, RC_NOCACHE, NULL);
 out_dropit:
+	nfsd_status_counter_set_idle(rqstp);
 	return 0;
 
 out_encode_err:
 	trace_nfsd_cant_encode_err(rqstp);
 	nfsd_cache_update(rqstp, rp, RC_NOCACHE, NULL);
 	*statp = rpc_system_err;
+	nfsd_status_counter_set_idle(rqstp);
 	return 1;
 }
 
diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c
index ae71e06213175..86c9dd4b334bd 100644
--- a/fs/nfsd/nfsxdr.c
+++ b/fs/nfsd/nfsxdr.c
@@ -9,6 +9,16 @@
 #include "xdr.h"
 #include "auth.h"
 
+/*
+ * Sun convention: a sattr time-useconds field of one full second (an
+ * otherwise out-of-range value) means "set this time to the current
+ * server time." It's needed to make permissions checks for the "touch"
+ * program across NFSv2 mounts work correctly. See description of
+ * sattr in section 6.1 of "NFS Illustrated" by Brent Callaghan,
+ * Addison-Wesley, ISBN 0-201-32750-5
+ */
+#define NFS2_SATTR_SET_TO_SERVER_TIME	(1000000)
+
 /*
  * Mapping of S_IF* types to NFS file types
  */
@@ -172,27 +182,29 @@ svcxdr_decode_sattr(struct svc_rqst *rqstp, struct xdr_stream *xdr,
 	tmp1 = be32_to_cpup(p++);
 	tmp2 = be32_to_cpup(p++);
 	if (tmp1 != (u32)-1 && tmp2 != (u32)-1) {
+		/*
+		 * Range test here to prevent the multiplication from
+		 * wrapping to a valid (but incorrect) value on 32-bit
+		 * platforms.
+		 */
+		if (tmp2 > NFS2_SATTR_SET_TO_SERVER_TIME)
+			return false;
 		iap->ia_valid |= ATTR_ATIME | ATTR_ATIME_SET;
 		iap->ia_atime.tv_sec = tmp1;
 		iap->ia_atime.tv_nsec = tmp2 * NSEC_PER_USEC;
+		if (tmp2 == NFS2_SATTR_SET_TO_SERVER_TIME)
+			iap->ia_valid &= ~ATTR_ATIME_SET;
 	}
 
 	tmp1 = be32_to_cpup(p++);
 	tmp2 = be32_to_cpup(p++);
 	if (tmp1 != (u32)-1 && tmp2 != (u32)-1) {
+		if (tmp2 > NFS2_SATTR_SET_TO_SERVER_TIME)
+			return false;
 		iap->ia_valid |= ATTR_MTIME | ATTR_MTIME_SET;
 		iap->ia_mtime.tv_sec = tmp1;
 		iap->ia_mtime.tv_nsec = tmp2 * NSEC_PER_USEC;
-		/*
-		 * Passing the invalid value useconds=1000000 for mtime
-		 * is a Sun convention for "set both mtime and atime to
-		 * current server time".  It's needed to make permissions
-		 * checks for the "touch" program across v2 mounts to
-		 * Solaris and Irix boxes work correctly. See description of
-		 * sattr in section 6.1 of "NFS Illustrated" by
-		 * Brent Callaghan, Addison-Wesley, ISBN 0-201-32750-5
-		 */
-		if (tmp2 == 1000000)
+		if (tmp2 == NFS2_SATTR_SET_TO_SERVER_TIME)
 			iap->ia_valid &= ~(ATTR_ATIME_SET|ATTR_MTIME_SET);
 	}
 
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index dec83e92650d1..bc0181ef1cb6e 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -98,9 +98,9 @@ struct nfsd4_callback {
 };
 
 struct nfsd4_callback_ops {
-	void (*prepare)(struct nfsd4_callback *);
-	int (*done)(struct nfsd4_callback *, struct rpc_task *);
-	void (*release)(struct nfsd4_callback *);
+	bool (*prepare)(struct nfsd4_callback *cb);
+	int (*done)(struct nfsd4_callback *cb, struct rpc_task *task);
+	void (*release)(struct nfsd4_callback *cb);
 	uint32_t opcode;
 };
 
@@ -190,6 +190,66 @@ struct nfs4_cb_fattr {
 	u64 ncf_cur_fsize;
 };
 
+/*
+ * FIXME: the current backchannel encoder can't handle a send buffer longer
+ *        than a single page (see bc_malloc/bc_free).
+ */
+#define NOTIFY4_EVENT_QUEUE_SIZE	3
+#define NOTIFY4_PAGE_ARRAY_SIZE		1
+
+struct nfsd_notify_event {
+	refcount_t	ne_ref;		// refcount
+	u32		ne_mask;	// FS_* mask from fsnotify callback
+	struct dentry	*ne_dentry;	// dentry reference to target
+	struct inode	*ne_target;	// inode overwritten by rename, or NULL
+	u32		ne_namelen;	// length of ne_name (old name for a rename)
+	u32		ne_newnamelen;	// length of new name (rename only), else 0
+	char		ne_name[];	// entry name, then new name (rename only)
+};
+
+/*
+ * For a rename, the new name is snapshotted at event-alloc time and stored
+ * immediately after the (NUL-terminated) old name in ne_name[]. ne_dentry can
+ * be renamed again before the CB_NOTIFY work runs, so the new name must not be
+ * read from the live dentry at encode time.
+ */
+static inline char *nfsd_notify_event_newname(struct nfsd_notify_event *ne)
+{
+	return ne->ne_name + ne->ne_namelen + 1;
+}
+
+static inline struct nfsd_notify_event *nfsd_notify_event_get(struct nfsd_notify_event *ne)
+{
+	refcount_inc(&ne->ne_ref);
+	return ne;
+}
+
+static inline void nfsd_notify_event_put(struct nfsd_notify_event *ne)
+{
+	if (refcount_dec_and_test(&ne->ne_ref)) {
+		iput(ne->ne_target);
+		dput(ne->ne_dentry);
+		kfree(ne);
+	}
+}
+
+/*
+ * Represents a directory delegation. The callback is for handling CB_NOTIFYs.
+ * As notifications from fsnotify come in, allocate a new event, take the ncn_lock,
+ * and add it to the ncn_evt queue. The CB_NOTIFY prepare handler will take the
+ * lock, clean out the list and process it.
+ */
+struct nfsd4_cb_notify {
+	spinlock_t			ncn_lock;	// protects the evt queue and count
+	int				ncn_evt_cnt;	// count of events in ncn_evt
+	int				ncn_nf_cnt;	// count of valid entries in ncn_nf
+	struct nfsd_notify_event	*ncn_evt[NOTIFY4_EVENT_QUEUE_SIZE]; // list of events
+	struct page			*ncn_pages[NOTIFY4_PAGE_ARRAY_SIZE]; // for encoding
+	struct notify4			*ncn_nf;	// array of notify4's to be sent
+	bool				ncn_encode_err;	// did encoding fail?
+	struct nfsd4_callback		ncn_cb;		// notify4 callback
+};
+
 /*
  * Represents a delegation stateid. The nfs4_client holds references to these
  * and they are put when it is being destroyed or when the delegation is
@@ -226,13 +286,22 @@ struct nfs4_delegation {
 	bool			dl_written;
 	bool			dl_setattr;
 
-	/* for CB_GETATTR */
-	struct nfs4_cb_fattr    dl_cb_fattr;
+	union {
+		/* for CB_GETATTR */
+		struct nfs4_cb_fattr    dl_cb_fattr;
+		/* for CB_NOTIFY */
+		struct nfsd4_cb_notify	dl_cb_notify;
+	};
 
 	/* For delegated timestamps */
 	struct timespec64	dl_atime;
 	struct timespec64	dl_mtime;
 	struct timespec64	dl_ctime;
+
+	/* For dir delegations */
+	u32			dl_notify_mask;
+	u32			dl_child_attrs[2];
+	u32			dl_dir_attrs[2];
 };
 
 static inline bool deleg_is_read(u32 dl_type)
@@ -384,6 +453,7 @@ struct nfsd4_session {
 	u16			se_slot_gen;
 	bool			se_dead;
 	u32			se_target_maxslots;
+	struct rcu_head		rcu_head;
 };
 
 /* formatted contents of nfs4_sessionid */
@@ -496,7 +566,7 @@ struct nfs4_client {
 #define NFSD4_CB_FAULT		3
 	int			cl_cb_state;
 	struct nfsd4_callback	cl_cb_null;
-	struct nfsd4_session	*cl_cb_session;
+	struct nfsd4_session	__rcu *cl_cb_session;
 
 	/* for all client information that callback code might need: */
 	spinlock_t		cl_lock;
@@ -691,7 +761,7 @@ struct nfs4_file {
 	 */
 	atomic_t		fi_access[2];
 	u32			fi_share_deny;
-	struct nfsd_file	*fi_deleg_file;
+	struct nfsd_file __rcu	*fi_deleg_file;
 	struct nfsd_file	*fi_rdeleg_file;
 	int			fi_delegees;
 	struct knfsd_fh		fi_fhandle;
@@ -754,6 +824,7 @@ struct nfs4_layout_stateid {
 	struct delayed_work		ls_fence_work;
 	unsigned int			ls_fence_delay;
 	bool				ls_fenced;
+	bool				ls_fence_inflight;
 };
 
 static inline struct nfs4_layout_stateid *layoutstateid(struct nfs4_stid *s)
@@ -774,6 +845,7 @@ enum nfsd4_cb_op {
 	NFSPROC4_CLNT_CB_NOTIFY_LOCK,
 	NFSPROC4_CLNT_CB_RECALL_ANY,
 	NFSPROC4_CLNT_CB_GETATTR,
+	NFSPROC4_CLNT_CB_NOTIFY,
 };
 
 /* Returns true iff a is later than b: */
@@ -848,6 +920,8 @@ void nfsd_update_cmtime_attr(struct file *f, unsigned int flags);
 extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(struct xdr_netobj name,
 				struct xdr_netobj princhash, struct nfsd_net *nn);
 extern bool nfs4_has_reclaimed_state(struct xdr_netobj name, struct nfsd_net *nn);
+int nfsd_handle_dir_event(u32 mask, const struct inode *dir, const void *data,
+			  int data_type, const struct qstr *name);
 
 void put_nfs4_file(struct nfs4_file *fi);
 extern void nfs4_put_cpntf_state(struct nfsd_net *nn,
diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h
index 1c5a1e50f9465..db0a0dc706603 100644
--- a/fs/nfsd/trace.h
+++ b/fs/nfsd/trace.h
@@ -12,6 +12,7 @@
 #include <linux/sunrpc/clnt.h>
 #include <linux/sunrpc/xprt.h>
 #include <trace/misc/fs.h>
+#include <trace/misc/fsnotify.h>
 #include <trace/misc/nfs.h>
 #include <trace/misc/sunrpc.h>
 
@@ -271,7 +272,7 @@ TRACE_EVENT_CONDITION(nfsd_fh_verify,
 	TP_CONDITION(rqstp != NULL),
 	TP_STRUCT__entry(
 		__field(unsigned int, netns_ino)
-		__sockaddr(server, rqstp->rq_xprt->xpt_remotelen)
+		__sockaddr(server, rqstp->rq_xprt->xpt_locallen)
 		__sockaddr(client, rqstp->rq_xprt->xpt_remotelen)
 		__field(u32, xid)
 		__field(u32, fh_hash)
@@ -310,7 +311,7 @@ TRACE_EVENT_CONDITION(nfsd_fh_verify_err,
 	TP_CONDITION(rqstp != NULL && error),
 	TP_STRUCT__entry(
 		__field(unsigned int, netns_ino)
-		__sockaddr(server, rqstp->rq_xprt->xpt_remotelen)
+		__sockaddr(server, rqstp->rq_xprt->xpt_locallen)
 		__sockaddr(client, rqstp->rq_xprt->xpt_remotelen)
 		__field(u32, xid)
 		__field(u32, fh_hash)
@@ -1377,6 +1378,28 @@ TRACE_EVENT(nfsd_file_fsnotify_handle_event,
 			__entry->nlink, __entry->mode, __entry->mask)
 );
 
+TRACE_EVENT(nfsd_handle_dir_event,
+	TP_PROTO(u32 mask, const struct inode *dir, const struct qstr *name),
+	TP_ARGS(mask, dir, name),
+	TP_STRUCT__entry(
+		__field(u32, mask)
+		__field(dev_t, s_dev)
+		__field(u64, i_ino)
+		__string_len(name, name ? name->name : NULL,
+				   name ? name->len : 0)
+	),
+	TP_fast_assign(
+		__entry->mask = mask;
+		__entry->s_dev = dir ? dir->i_sb->s_dev : 0;
+		__entry->i_ino = dir ? dir->i_ino : 0;
+		__assign_str(name);
+	),
+	TP_printk("inode=0x%x:0x%x:0x%llx mask=%s name=%s",
+			MAJOR(__entry->s_dev), MINOR(__entry->s_dev),
+			__entry->i_ino, show_fsnotify_mask(__entry->mask),
+			__get_str(name))
+);
+
 DECLARE_EVENT_CLASS(nfsd_file_gc_class,
 	TP_PROTO(
 		const struct nfsd_file *nf
@@ -1677,6 +1700,7 @@ TRACE_EVENT(nfsd_cb_setup_err,
 		{ OP_CB_RECALL,			"CB_RECALL" },		\
 		{ OP_CB_LAYOUTRECALL,		"CB_LAYOUTRECALL" },	\
 		{ OP_CB_RECALL_ANY,		"CB_RECALL_ANY" },	\
+		{ OP_CB_NOTIFY,			"CB_NOTIFY" },		\
 		{ OP_CB_NOTIFY_LOCK,		"CB_NOTIFY_LOCK" },	\
 		{ OP_CB_OFFLOAD,		"CB_OFFLOAD" })
 
@@ -1727,9 +1751,10 @@ DEFINE_NFSD_CB_LIFETIME_EVENT(bc_shutdown);
 TRACE_EVENT(nfsd_cb_seq_status,
 	TP_PROTO(
 		const struct rpc_task *task,
-		const struct nfsd4_callback *cb
+		const struct nfsd4_callback *cb,
+		const struct nfsd4_session *session
 	),
-	TP_ARGS(task, cb),
+	TP_ARGS(task, cb, session),
 	TP_STRUCT__entry(
 		__field(unsigned int, task_id)
 		__field(unsigned int, client_id)
@@ -1741,8 +1766,6 @@ TRACE_EVENT(nfsd_cb_seq_status,
 		__field(int, seq_status)
 	),
 	TP_fast_assign(
-		const struct nfs4_client *clp = cb->cb_clp;
-		const struct nfsd4_session *session = clp->cl_cb_session;
 		const struct nfsd4_sessionid *sid =
 			(struct nfsd4_sessionid *)&session->se_sessionid;
 
@@ -1768,9 +1791,10 @@ TRACE_EVENT(nfsd_cb_seq_status,
 TRACE_EVENT(nfsd_cb_free_slot,
 	TP_PROTO(
 		const struct rpc_task *task,
-		const struct nfsd4_callback *cb
+		const struct nfsd4_callback *cb,
+		const struct nfsd4_session *session
 	),
-	TP_ARGS(task, cb),
+	TP_ARGS(task, cb, session),
 	TP_STRUCT__entry(
 		__field(unsigned int, task_id)
 		__field(unsigned int, client_id)
@@ -1781,8 +1805,6 @@ TRACE_EVENT(nfsd_cb_free_slot,
 		__field(u32, slot_seqno)
 	),
 	TP_fast_assign(
-		const struct nfs4_client *clp = cb->cb_clp;
-		const struct nfsd4_session *session = clp->cl_cb_session;
 		const struct nfsd4_sessionid *sid =
 			(struct nfsd4_sessionid *)&session->se_sessionid;
 
diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c
index 1e89c7ff9493b..c81aea23363af 100644
--- a/fs/nfsd/vfs.c
+++ b/fs/nfsd/vfs.c
@@ -139,16 +139,17 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp,
 	err = follow_down(&path, follow_flags);
 	if (err < 0)
 		goto out;
+
 	if (path.mnt == exp->ex_path.mnt && path.dentry == dentry &&
 	    nfsd_mountpoint(dentry, exp) == 2) {
 		/* This is only a mountpoint in some other namespace */
-		path_put(&path);
 		goto out;
 	}
 
 	exp2 = rqst_exp_get_by_name(rqstp, &path);
 	if (IS_ERR(exp2)) {
 		err = PTR_ERR(exp2);
+		exp2 = NULL;
 		/*
 		 * We normally allow NFS clients to continue
 		 * "underneath" a mountpoint that is not exported.
@@ -158,10 +159,7 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp,
 		 */
 		if (err == -ENOENT && !(exp->ex_flags & NFSEXP_V4ROOT))
 			err = 0;
-		path_put(&path);
-		goto out;
-	}
-	if (nfsd_v4client(rqstp) ||
+	} else if (nfsd_v4client(rqstp) ||
 		(exp->ex_flags & NFSEXP_CROSSMOUNT) || EX_NOHIDE(exp2)) {
 		/* successfully crossed mount point */
 		/*
@@ -175,9 +173,10 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp,
 		*expp = exp2;
 		exp2 = exp;
 	}
-	path_put(&path);
-	exp_put(exp2);
 out:
+	path_put(&path);
+	if (exp2)
+		exp_put(exp2);
 	return err;
 }
 
@@ -419,21 +418,22 @@ nfsd_sanitize_attrs(struct inode *inode, struct iattr *iap)
 }
 
 static __be32
-nfsd_get_write_access(struct svc_rqst *rqstp, struct svc_fh *fhp,
-		struct iattr *iap)
+nfsd_may_truncate(struct svc_rqst *rqstp, struct svc_fh *fhp,
+		  struct iattr *iap)
 {
 	struct inode *inode = d_inode(fhp->fh_dentry);
 
-	if (iap->ia_size < inode->i_size) {
-		__be32 err;
+	if (iap->ia_size >= i_size_read(inode))
+		return nfs_ok;
 
-		err = nfsd_permission(&rqstp->rq_cred,
-				      fhp->fh_export, fhp->fh_dentry,
-				      NFSD_MAY_TRUNC | NFSD_MAY_OWNER_OVERRIDE);
-		if (err)
-			return err;
-	}
-	return nfserrno(get_write_access(inode));
+	return nfsd_permission(&rqstp->rq_cred, fhp->fh_export, fhp->fh_dentry,
+			       NFSD_MAY_TRUNC | NFSD_MAY_OWNER_OVERRIDE);
+}
+
+static __be32
+nfsd_get_write_access(struct svc_fh *fhp)
+{
+	return nfserrno(get_write_access(d_inode(fhp->fh_dentry)));
 }
 
 static int __nfsd_setattr(struct dentry *dentry, struct iattr *iap)
@@ -560,12 +560,17 @@ nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp,
 	 * setattr call.
 	 */
 	if (size_change) {
-		err = nfsd_get_write_access(rqstp, fhp, iap);
+		err = nfsd_get_write_access(fhp);
 		if (err)
 			return err;
 	}
 
 	inode_lock(inode);
+	if (size_change) {
+		err = nfsd_may_truncate(rqstp, fhp, iap);
+		if (err)
+			goto out_unlock;
+	}
 	err = fh_fill_pre_attrs(fhp);
 	if (err)
 		goto out_unlock;
@@ -1374,6 +1379,7 @@ nfsd_direct_write(struct svc_rqst *rqstp, struct svc_fh *fhp,
 	struct file *file = nf->nf_file;
 	unsigned int nsegs, i;
 	ssize_t host_err;
+	size_t expected;
 
 	nsegs = nfsd_write_dio_iters_init(nf, rqstp->rq_bvec, nvecs,
 					  kiocb, *cnt, segments);
@@ -1395,11 +1401,13 @@ nfsd_direct_write(struct svc_rqst *rqstp, struct svc_fh *fhp,
 				kiocb->ki_flags |= IOCB_DONTCACHE;
 		}
 
+		expected = iov_iter_count(&segments[i].iter);
+
 		host_err = vfs_iocb_iter_write(file, kiocb, &segments[i].iter);
 		if (host_err < 0)
 			return host_err;
 		*cnt += host_err;
-		if (host_err < segments[i].iter.count)
+		if (host_err < (ssize_t)expected)
 			break;	/* partial write */
 	}
 
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index 85574b2a139af..805c7122eb93f 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -970,6 +970,11 @@ __be32 nfsd4_encode_fattr_to_buf(__be32 **p, int words,
 		struct svc_fh *fhp, struct svc_export *exp,
 		struct dentry *dentry,
 		u32 *bmval, struct svc_rqst *, int ignore_crossmnt);
+u8 *nfsd4_encode_notify_event(struct xdr_stream *xdr, struct nfsd_notify_event *nne,
+			      struct nfs4_delegation *dd, struct nfsd_file *nf,
+			      u32 *notify_mask);
+u8 *nfsd4_encode_dir_attr_change(struct xdr_stream *xdr, struct nfs4_delegation *dp,
+				 struct nfsd_file *nf);
 extern __be32 nfsd4_setclientid(struct svc_rqst *rqstp,
 		struct nfsd4_compound_state *, union nfsd4_op_u *u);
 extern __be32 nfsd4_setclientid_confirm(struct svc_rqst *rqstp,
diff --git a/fs/nfsd/xdr4cb.h b/fs/nfsd/xdr4cb.h
index f4e29c0c701c9..b06d0170d7c43 100644
--- a/fs/nfsd/xdr4cb.h
+++ b/fs/nfsd/xdr4cb.h
@@ -33,6 +33,18 @@
 					cb_sequence_dec_sz +            \
 					op_dec_sz)
 
+#define NFS4_enc_cb_notify_sz		(cb_compound_enc_hdr_sz +       \
+					cb_sequence_enc_sz +            \
+					1 + enc_stateid_sz +            \
+					enc_nfs4_fh_sz +		\
+					1 +				\
+					NOTIFY4_EVENT_QUEUE_SIZE *	\
+					(2 + (NFS4_OPAQUE_LIMIT >> 2)))
+
+#define NFS4_dec_cb_notify_sz		(cb_compound_dec_hdr_sz  +      \
+					cb_sequence_dec_sz +            \
+					op_dec_sz)
+
 #define NFS4_enc_cb_notify_lock_sz	(cb_compound_enc_hdr_sz +        \
 					cb_sequence_enc_sz +             \
 					2 + 1 +				 \
diff --git a/fs/nilfs2/namei.c b/fs/nilfs2/namei.c
index e2fe95de3d71c..77c5f7f74fbf4 100644
--- a/fs/nilfs2/namei.c
+++ b/fs/nilfs2/namei.c
@@ -86,7 +86,7 @@ nilfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags)
  * with d_instantiate().
  */
 static int nilfs_create(struct mnt_idmap *idmap, struct inode *dir,
-			struct dentry *dentry, umode_t mode, bool excl)
+			struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode;
 	struct nilfs_transaction_info ti;
@@ -231,7 +231,7 @@ static struct dentry *nilfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 
 	inc_nlink(dir);
 
-	inode = nilfs_new_inode(dir, S_IFDIR | mode);
+	inode = nilfs_new_inode(dir, mode);
 	err = PTR_ERR(inode);
 	if (IS_ERR(inode))
 		goto out_dir;
diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c
index a3555bebad63e..b59f0aa43c4be 100644
--- a/fs/notify/fanotify/fanotify.c
+++ b/fs/notify/fanotify/fanotify.c
@@ -599,6 +599,7 @@ static struct fanotify_event *fanotify_alloc_perm_event(const void *data,
 	pevent->hdr.pad = 0;
 	pevent->hdr.len = 0;
 	pevent->state = FAN_EVENT_INIT;
+	pevent->watchdog_cnt = 0;
 	pevent->path = *path;
 	/* NULL ppos means no range info */
 	pevent->ppos = range ? &range->pos : NULL;
diff --git a/fs/ntfs/attrib.c b/fs/ntfs/attrib.c
index 239b7bcbaedf4..d354c3b0fae10 100644
--- a/fs/ntfs/attrib.c
+++ b/fs/ntfs/attrib.c
@@ -697,6 +697,11 @@ static bool ntfs_non_resident_attr_value_is_valid(const struct attr_record *a)
 	attr_len = le32_to_cpu(a->length);
 	min_len = offsetof(struct attr_record, data.non_resident.initialized_size) +
 		  sizeof(a->data.non_resident.initialized_size);
+
+	/* Sparse and compressed attributes have the extra compressed_size field */
+	if (a->flags & (ATTR_IS_SPARSE | ATTR_COMPRESSION_MASK))
+		min_len += sizeof(a->data.non_resident.compressed_size);
+
 	if (attr_len < min_len)
 		return false;
 
@@ -4293,6 +4298,16 @@ static int ntfs_non_resident_attr_shrink(struct ntfs_inode *ni, const s64 newsiz
 		ni->initialized_size = newsize;
 		ctx->attr->data.non_resident.initialized_size = cpu_to_le64(newsize);
 	}
+
+	/*
+	 * Drop any page-cache folios that now lie beyond the shrunk
+	 * attribute. The clusters backing them have just been freed and the
+	 * runlist truncated, so leaving stale dirty folios around makes a
+	 * later writeback map a vcn past the new allocation, which fails with
+	 * -ENOENT and loses the write.
+	 */
+	truncate_inode_pages(VFS_I(ni)->i_mapping, newsize);
+
 	/* Update data size in the index. */
 	if (ni->type == AT_DATA && ni->name == AT_UNNAMED)
 		NInoSetFileNameDirty(ni);
diff --git a/fs/ntfs/compress.c b/fs/ntfs/compress.c
index 76bd806b41edd..fe1877b86f490 100644
--- a/fs/ntfs/compress.c
+++ b/fs/ntfs/compress.c
@@ -96,26 +96,6 @@ void free_compression_buffers(void)
 	mutex_unlock(&ntfs_cb_lock);
 }
 
-/*
- * zero_partial_compressed_page - zero out of bounds compressed page region
- * @page: page to zero
- * @initialized_size: initialized size of the attribute
- */
-static void zero_partial_compressed_page(struct page *page,
-		const s64 initialized_size)
-{
-	u8 *kp = page_address(page);
-	unsigned int kp_ofs;
-
-	ntfs_debug("Zeroing page region outside initialized size.");
-	if (((s64)page->__folio_index << PAGE_SHIFT) >= initialized_size) {
-		clear_page(kp);
-		return;
-	}
-	kp_ofs = initialized_size & ~PAGE_MASK;
-	memset(kp + kp_ofs, 0, PAGE_SIZE - kp_ofs);
-}
-
 /*
  * handle_bounds_compressed_page - test for&handle out of bounds compressed page
  * @page: page to check and handle
@@ -125,9 +105,21 @@ static void zero_partial_compressed_page(struct page *page,
 static inline void handle_bounds_compressed_page(struct page *page,
 		const loff_t i_size, const s64 initialized_size)
 {
-	if ((page->__folio_index >= (initialized_size >> PAGE_SHIFT)) &&
-			(initialized_size < i_size))
-		zero_partial_compressed_page(page, initialized_size);
+	loff_t pos = page_offset(page);
+
+	if ((pos + PAGE_SIZE > initialized_size) &&
+			(initialized_size < i_size)) {
+		size_t offset;
+
+		ntfs_debug("Zeroing page region outside initialized size.");
+		if (pos >= initialized_size)
+			offset = 0;
+		else
+			offset = offset_in_page(initialized_size);
+		zero_user_segment(page, offset, PAGE_SIZE);
+	} else {
+		flush_dcache_page(page);
+	}
 }
 
 /*
@@ -185,6 +177,7 @@ static int ntfs_decompress(struct page *dest_pages[], int completed_pages[],
 
 	/* Variables for uncompressed data / destination. */
 	struct page *dp;	/* Current destination page being worked on. */
+	u8 *dp_kaddr;		/* Local kmap for the current destination page. */
 	u8 *dp_addr;		/* Current pointer into dp. */
 	u8 *dp_sb_start;	/* Start of current sub-block in dp. */
 	u8 *dp_sb_end;		/* End of current sb in dp (dp_sb_start + NTFS_SB_SIZE). */
@@ -199,6 +192,7 @@ static int ntfs_decompress(struct page *dest_pages[], int completed_pages[],
 	/* Default error code. */
 	int err = -EOVERFLOW;
 
+	dp_kaddr = NULL;
 	ntfs_debug("Entering, cb_size = 0x%x.", cb_size);
 do_next_sb:
 	ntfs_debug("Beginning sub-block at offset = 0x%zx in the cb.",
@@ -231,8 +225,6 @@ static int ntfs_decompress(struct page *dest_pages[], int completed_pages[],
 				 */
 				handle_bounds_compressed_page(dp, i_size,
 						initialized_size);
-				flush_dcache_page(dp);
-				kunmap_local(page_address(dp));
 				SetPageUptodate(dp);
 				unlock_page(dp);
 				if (di == xpage)
@@ -278,7 +270,8 @@ static int ntfs_decompress(struct page *dest_pages[], int completed_pages[],
 	}
 
 	/* We have a valid destination page. Setup the destination pointers. */
-	dp_addr = (u8 *)page_address(dp) + do_sb_start;
+	dp_kaddr = kmap_local_page(dp);
+	dp_addr = dp_kaddr + do_sb_start;
 
 	/* Now, we are ready to process the current sub-block (sb). */
 	if (!(le16_to_cpup((__le16 *)cb) & NTFS_SB_IS_COMPRESSED)) {
@@ -299,6 +292,8 @@ static int ntfs_decompress(struct page *dest_pages[], int completed_pages[],
 		/* Advance destination position to next sub-block. */
 		*dest_ofs += NTFS_SB_SIZE;
 		*dest_ofs &= ~PAGE_MASK;
+		kunmap_local(dp_kaddr);
+		dp_kaddr = NULL;
 		if (!(*dest_ofs)) {
 finalize_page:
 			/*
@@ -333,6 +328,8 @@ static int ntfs_decompress(struct page *dest_pages[], int completed_pages[],
 		}
 		/* We have finished the current sub-block. */
 		*dest_ofs &= ~PAGE_MASK;
+		kunmap_local(dp_kaddr);
+		dp_kaddr = NULL;
 		if (!(*dest_ofs))
 			goto finalize_page;
 		goto do_next_sb;
@@ -438,6 +435,8 @@ static int ntfs_decompress(struct page *dest_pages[], int completed_pages[],
 	goto do_next_tag;
 
 return_overflow:
+	if (dp_kaddr)
+		kunmap_local(dp_kaddr);
 	ntfs_error(NULL, "Failed. Returning -EOVERFLOW.");
 	goto return_error;
 }
@@ -465,14 +464,14 @@ int ntfs_read_compressed_block(struct folio *folio)
 	struct page *page = &folio->page;
 	loff_t i_size;
 	s64 initialized_size;
-	struct address_space *mapping = page->mapping;
+	struct address_space *mapping = folio->mapping;
 	struct ntfs_inode *ni = NTFS_I(mapping->host);
 	struct ntfs_volume *vol = ni->vol;
 	struct super_block *sb = vol->sb;
 	struct runlist_element *rl;
 	unsigned long flags;
 	u8 *cb, *cb_pos, *cb_end;
-	unsigned long offset, index = page->__folio_index;
+	unsigned long offset, index = folio->index;
 	u32 cb_size = ni->itype.compressed.block_size;
 	u64 cb_size_mask = cb_size - 1UL;
 	s64 vcn;
@@ -566,7 +565,6 @@ int ntfs_read_compressed_block(struct folio *folio)
 			 * least wasting our time.
 			 */
 			if (!PageDirty(page) && (!PageUptodate(page))) {
-				kmap_local_page(page);
 				continue;
 			}
 			unlock_page(page);
@@ -652,8 +650,7 @@ int ntfs_read_compressed_block(struct folio *folio)
 		}
 
 		lock_page(lpage);
-		memcpy(cb_pos, page_address(lpage) + page_ofs,
-		       vol->cluster_size);
+		memcpy_from_page(cb_pos, lpage, page_ofs, vol->cluster_size);
 		unlock_page(lpage);
 		put_page(lpage);
 		cb_pos += vol->cluster_size;
@@ -692,14 +689,7 @@ int ntfs_read_compressed_block(struct folio *folio)
 		for (; cur_page < cb_max_page; cur_page++) {
 			page = pages[cur_page];
 			if (page) {
-				if (likely(!cur_ofs))
-					clear_page(page_address(page));
-				else
-					memset(page_address(page) + cur_ofs, 0,
-							PAGE_SIZE -
-							cur_ofs);
-				flush_dcache_page(page);
-				kunmap_local(page_address(page));
+				memzero_page(page, cur_ofs, PAGE_SIZE - cur_ofs);
 				SetPageUptodate(page);
 				unlock_page(page);
 				if (cur_page == xpage)
@@ -717,8 +707,7 @@ int ntfs_read_compressed_block(struct folio *folio)
 		if (cb_max_ofs && cb_pos < cb_end) {
 			page = pages[cur_page];
 			if (page)
-				memset(page_address(page) + cur_ofs, 0,
-						cb_max_ofs - cur_ofs);
+				memzero_page(page, cur_ofs, cb_max_ofs - cur_ofs);
 			/*
 			 * No need to update cb_pos at this stage:
 			 *	cb_pos += cb_max_ofs - cur_ofs;
@@ -739,7 +728,7 @@ int ntfs_read_compressed_block(struct folio *folio)
 		for (; cur_page < cb_max_page; cur_page++) {
 			page = pages[cur_page];
 			if (page)
-				memcpy(page_address(page) + cur_ofs, cb_pos,
+				memcpy_to_page(page, cur_ofs, cb_pos,
 						PAGE_SIZE - cur_ofs);
 			cb_pos += PAGE_SIZE - cur_ofs;
 			cur_ofs = 0;
@@ -750,7 +739,7 @@ int ntfs_read_compressed_block(struct folio *folio)
 		if (cb_max_ofs && cb_pos < cb_end) {
 			page = pages[cur_page];
 			if (page)
-				memcpy(page_address(page) + cur_ofs, cb_pos,
+				memcpy_to_page(page, cur_ofs, cb_pos,
 						cb_max_ofs - cur_ofs);
 			cb_pos += cb_max_ofs - cur_ofs;
 			cur_ofs = cb_max_ofs;
@@ -767,8 +756,6 @@ int ntfs_read_compressed_block(struct folio *folio)
 				 */
 				handle_bounds_compressed_page(page, i_size,
 						initialized_size);
-				flush_dcache_page(page);
-				kunmap_local(page_address(page));
 				SetPageUptodate(page);
 				unlock_page(page);
 				if (cur2_page == xpage)
@@ -804,7 +791,6 @@ int ntfs_read_compressed_block(struct folio *folio)
 				page = pages[prev_cur_page];
 				if (page) {
 					flush_dcache_page(page);
-					kunmap_local(page_address(page));
 					unlock_page(page);
 					if (prev_cur_page != xpage)
 						put_page(page);
@@ -822,14 +808,15 @@ int ntfs_read_compressed_block(struct folio *folio)
 	for (cur_page = 0; cur_page < max_page; cur_page++) {
 		page = pages[cur_page];
 		if (page) {
+			folio = page_folio(page);
+
 			ntfs_error(vol->sb,
 				"Still have pages left! Terminating them with extreme prejudice.  Inode 0x%llx, page index 0x%lx.",
-				ni->mft_no, page->__folio_index);
-			flush_dcache_page(page);
-			kunmap_local(page_address(page));
-			unlock_page(page);
+				ni->mft_no, folio->index);
+			flush_dcache_folio(folio);
+			folio_unlock(folio);
 			if (cur_page != xpage)
-				put_page(page);
+				folio_put(folio);
 			pages[cur_page] = NULL;
 		}
 	}
@@ -864,7 +851,6 @@ int ntfs_read_compressed_block(struct folio *folio)
 		page = pages[i];
 		if (page) {
 			flush_dcache_page(page);
-			kunmap_local(page_address(page));
 			unlock_page(page);
 			if (i != xpage)
 				put_page(page);
@@ -1316,7 +1302,6 @@ static int ntfs_write_cb(struct ntfs_inode *ni, loff_t pos, struct page **pages,
 		}
 		pages_disk[i] = pg;
 		lock_page(pg);
-		kmap_local_page(pg);
 	}
 
 	outbuf = vmap(pages_disk, pages_count, VM_MAP, PAGE_KERNEL);
@@ -1451,7 +1436,6 @@ static int ntfs_write_cb(struct ntfs_inode *ni, loff_t pos, struct page **pages,
 	for (i = 0; i < pages_count; i++) {
 		pg = pages_disk[i];
 		if (pg) {
-			kunmap_local(page_address(pg));
 			unlock_page(pg);
 			put_page(pg);
 		}
@@ -1475,16 +1459,6 @@ int ntfs_compress_write(struct ntfs_inode *ni, loff_t pos, size_t count,
 	size_t written = 0;
 	struct address_space *mapping = VFS_I(ni)->i_mapping;
 
-	if (NInoCompressed(ni) && pos + count > ni->allocated_size) {
-		int err;
-		loff_t end = pos + count;
-
-		err = ntfs_attr_expand(ni, end,
-				round_up(end, ni->itype.compressed.block_size));
-		if (err)
-			return err;
-	}
-
 	pages = kmalloc_array(pages_per_cb, sizeof(struct page *), GFP_NOFS);
 	if (!pages)
 		return -ENOMEM;
diff --git a/fs/ntfs/dir.c b/fs/ntfs/dir.c
index 6fa9ae3377cb8..2d594cbb4ebe3 100644
--- a/fs/ntfs/dir.c
+++ b/fs/ntfs/dir.c
@@ -966,13 +966,14 @@ static int ntfs_readdir(struct file *file, struct dir_context *actor)
 			 */
 			private = file->private_data;
 			kfree(private->key);
-			private->key = kmalloc(le16_to_cpu(next->key_length), GFP_KERNEL);
+			private->key = kmemdup(&next->key.file_name,
+					le16_to_cpu(next->key_length),
+					GFP_KERNEL);
 			if (!private->key) {
 				err = -ENOMEM;
 				goto out;
 			}
 
-			memcpy(private->key, &next->key.file_name, le16_to_cpu(next->key_length));
 			private->key_length = next->key_length;
 			break;
 		}
diff --git a/fs/ntfs/ea.c b/fs/ntfs/ea.c
index 0cd192752b7cd..d0044c61152af 100644
--- a/fs/ntfs/ea.c
+++ b/fs/ntfs/ea.c
@@ -196,6 +196,9 @@ static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len,
 	struct ea_attr *p_ea;
 	u32 ea_info_qsize = 0;
 	char *ea_buf = NULL;
+	char *new_ea_buf;
+	char *old_ea_buf = NULL;
+	struct ea_information old_ea_info;
 	size_t new_ea_size = ALIGN(struct_size(p_ea, ea_name, 1 + name_len + val_size), 4);
 	s64 ea_off, ea_info_size, all_ea_size, ea_size;
 
@@ -249,6 +252,22 @@ static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len,
 			err = -EEXIST;
 			goto out;
 		}
+		if ((flags & XATTR_REPLACE) && !val_size) {
+			old_ea_info = *p_ea_info;
+			old_ea_buf = kvmemdup(ea_buf, all_ea_size, GFP_NOFS);
+			if (!old_ea_buf) {
+				err = -ENOMEM;
+				goto out;
+			}
+		}
+
+		/* Check the final $EA size before removing the old entry. */
+		if (val_size &&
+		    ntfs_attr_size_bounds_check(ni->vol, AT_EA,
+					ea_info_qsize - ea_size + new_ea_size)) {
+			err = -EFBIG;
+			goto out;
+		}
 
 		p_ea = (struct ea_attr *)(ea_buf + ea_off);
 
@@ -267,17 +286,39 @@ static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len,
 		ea_info_qsize -= ea_size;
 		p_ea_info->ea_query_length = cpu_to_le32(ea_info_qsize);
 
-		err = ntfs_write_ea(ni, AT_EA_INFORMATION, (char *)p_ea_info, 0,
-				sizeof(struct ea_information), false);
-		if (err)
-			goto out;
+		if ((flags & XATTR_REPLACE) && !val_size && !ea_info_qsize) {
+			err = ntfs_attr_remove(ni, AT_EA, AT_UNNAMED, 0);
+			if (err)
+				goto out;
 
-		err = ntfs_write_ea(ni, AT_EA, ea_buf, 0, ea_info_qsize, true);
-		if (err)
+			err = ntfs_attr_remove(ni, AT_EA_INFORMATION, AT_UNNAMED, 0);
+			if (err) {
+				/* Restore the original $EA if $EA_INFORMATION removal failed. */
+				ntfs_attr_add(ni, AT_EA, AT_UNNAMED, 0, old_ea_buf,
+					      all_ea_size);
+				ea_info_qsize = le32_to_cpu(old_ea_info.ea_query_length);
+			}
 			goto out;
+		}
 
 		if ((flags & XATTR_REPLACE) && !val_size) {
-			/* Remove xattr. */
+			err = ntfs_write_ea(ni, AT_EA, ea_buf, 0, ea_info_qsize,
+					true);
+			if (err) {
+				ntfs_write_ea(ni, AT_EA, old_ea_buf, 0,
+					      all_ea_size, false);
+				goto out;
+			}
+
+			err = ntfs_write_ea(ni, AT_EA_INFORMATION, (char *)p_ea_info,
+					0, sizeof(struct ea_information), false);
+			if (err) {
+				ntfs_write_ea(ni, AT_EA, old_ea_buf, 0,
+					      all_ea_size, false);
+				ntfs_write_ea(ni, AT_EA_INFORMATION,
+					      (char *)&old_ea_info, 0,
+					      sizeof(old_ea_info), false);
+			}
 			goto out;
 		}
 	} else {
@@ -285,22 +326,30 @@ static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len,
 			err = -ENODATA;
 			goto out;
 		}
-	}
-	kvfree(ea_buf);
 
+		if (ntfs_attr_size_bounds_check(ni->vol, AT_EA,
+					ea_info_qsize + new_ea_size)) {
+			err = -EFBIG;
+			goto out;
+		}
+	}
 alloc_new_ea:
-	ea_buf = kzalloc(new_ea_size, GFP_NOFS);
-	if (!ea_buf) {
+	new_ea_buf = kvzalloc(ea_info_qsize + new_ea_size, GFP_NOFS);
+	if (!new_ea_buf) {
 		err = -ENOMEM;
 		goto out;
 	}
+	if (ea_info_qsize)
+		memcpy(new_ea_buf, ea_buf, ea_info_qsize);
+	kvfree(ea_buf);
+	ea_buf = new_ea_buf;
+	p_ea = (struct ea_attr *)(ea_buf + ea_info_qsize);
 
 	/*
 	 * EA and REPARSE_POINT compatibility not checked any more,
 	 * required by Windows 10, but having both may lead to
 	 * problems with earlier versions.
 	 */
-	p_ea = (struct ea_attr *)ea_buf;
 	memcpy(p_ea->ea_name, name, name_len);
 	p_ea->ea_name_length = name_len;
 	p_ea->ea_name[name_len] = 0;
@@ -312,8 +361,7 @@ static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len,
 	p_ea_info->ea_length = cpu_to_le16(ea_packed);
 	p_ea_info->ea_query_length = cpu_to_le32(ea_info_qsize + new_ea_size);
 
-	if (ea_packed > 0xffff ||
-	    ntfs_attr_size_bounds_check(ni->vol, AT_EA, new_ea_size)) {
+	if (ea_packed > 0xffff) {
 		err = -EFBIG;
 		goto out;
 	}
@@ -322,13 +370,13 @@ static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len,
 	 * no EA or EA_INFORMATION : add them
 	 */
 	if (!ntfs_attr_exist(ni, AT_EA, AT_UNNAMED, 0)) {
-		err = ntfs_attr_add(ni, AT_EA, AT_UNNAMED, 0, (char *)p_ea,
-				new_ea_size);
+		err = ntfs_attr_add(ni, AT_EA, AT_UNNAMED, 0, ea_buf,
+				ea_info_qsize + new_ea_size);
 		if (err)
 			goto out;
 	} else {
-		err = ntfs_write_ea(ni, AT_EA, (char *)p_ea, ea_info_qsize,
-				new_ea_size, false);
+		err = ntfs_write_ea(ni, AT_EA, ea_buf, 0,
+				ea_info_qsize + new_ea_size, true);
 		if (err)
 			goto out;
 	}
@@ -348,6 +396,7 @@ static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len,
 		NInoClearHasEA(ni);
 
 	kvfree(ea_buf);
+	kvfree(old_ea_buf);
 	kvfree(p_ea_info);
 
 	return err;
diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c
index 6a7b638e523d9..c8e49f83fd928 100644
--- a/fs/ntfs/file.c
+++ b/fs/ntfs/file.c
@@ -268,23 +268,19 @@ static int ntfs_setattr_size(struct inode *vi, struct iattr *attr)
 		return err;
 
 	inode_dio_wait(vi);
-	truncate_setsize(vi, attr->ia_size);
+	if (attr->ia_size > old_size) {
+		truncate_pagecache(vi, old_size);
+		i_size_write(vi, attr->ia_size);
+		pagecache_isize_extended(vi, old_size, attr->ia_size);
+	} else
+		truncate_setsize(vi, attr->ia_size);
+
 	err = ntfs_truncate_vfs(vi, attr->ia_size, old_size);
 	if (err) {
 		i_size_write(vi, old_size);
 		return err;
 	}
 
-	if (NInoNonResident(ni) && attr->ia_size > old_size &&
-	    old_size % PAGE_SIZE != 0) {
-		loff_t len = min_t(loff_t,
-				round_up(old_size, PAGE_SIZE) - old_size,
-				attr->ia_size - old_size);
-		err = iomap_zero_range(vi, old_size, len,
-				NULL, &ntfs_seek_iomap_ops,
-				&ntfs_iomap_folio_ops, NULL);
-	}
-
 	return err;
 }
 
@@ -535,6 +531,31 @@ static ssize_t ntfs_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	return ret;
 }
 
+static int ntfs_expand_for_write(struct ntfs_inode *ni, loff_t end)
+{
+	struct ntfs_volume *vol = ni->vol;
+	loff_t prealloc_size = 0;
+	int err;
+
+	if (end <= ni->data_size)
+		return 0;
+
+	if (NInoCompressed(ni)) {
+		if (end > ni->allocated_size)
+			prealloc_size = round_up(end,
+						 ni->itype.compressed.block_size);
+	} else if (end > ni->allocated_size &&
+		   end < ni->allocated_size + vol->preallocated_size) {
+		prealloc_size = ni->allocated_size + vol->preallocated_size;
+	}
+
+	mutex_lock(&ni->mrec_lock);
+	err = ntfs_attr_expand(ni, end, prealloc_size);
+	mutex_unlock(&ni->mrec_lock);
+
+	return err;
+}
+
 static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 {
 	struct file *file = iocb->ki_filp;
@@ -543,7 +564,7 @@ static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	struct ntfs_volume *vol = ni->vol;
 	ssize_t ret;
 	ssize_t count;
-	loff_t pos;
+	loff_t pos, end;
 	int err;
 	loff_t old_data_size, old_init_size;
 
@@ -580,10 +601,24 @@ static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 
 	pos = iocb->ki_pos;
 	count = ret;
+	end = pos + count;
 
 	old_data_size = ni->data_size;
 	old_init_size = ni->initialized_size;
 
+	if (end > old_data_size) {
+		ret = ntfs_expand_for_write(ni, end);
+		if (ret < 0)
+			goto out;
+	}
+
+	if (NInoNonResident(ni) && !NInoCompressed(ni) &&
+	    end > old_init_size) {
+		ret = ntfs_extend_initialized_size(vi, pos, end);
+		if (ret < 0)
+			goto out;
+	}
+
 	if (NInoNonResident(ni) && NInoCompressed(ni)) {
 		ret = ntfs_compress_write(ni, pos, count, from);
 		if (ret > 0)
@@ -655,7 +690,7 @@ static int ntfs_file_mmap_prepare(struct vm_area_desc *desc)
 			   from + desc->end - desc->start);
 
 		if (NTFS_I(inode)->initialized_size < to) {
-			err = ntfs_extend_initialized_size(inode, to, to, false);
+			err = ntfs_extend_initialized_size(inode, to, to);
 			if (err)
 				return err;
 		}
@@ -1126,13 +1161,9 @@ static long ntfs_fallocate(struct file *file, int mode, loff_t offset, loff_t le
 		filemap_invalidate_unlock(vi->i_mapping);
 	if (!err) {
 		if (mode == 0 && NInoNonResident(ni) &&
-		    offset > old_size && old_size % PAGE_SIZE != 0) {
-			loff_t len = min_t(loff_t,
-					   round_up(old_size, PAGE_SIZE) - old_size,
-					   offset - old_size);
-			err = iomap_zero_range(vi, old_size, len, NULL,
-					       &ntfs_seek_iomap_ops,
-					       &ntfs_iomap_folio_ops, NULL);
+		    offset > old_size) {
+			truncate_pagecache(vi, old_size);
+			pagecache_isize_extended(vi, old_size, offset);
 		}
 		NInoSetFileNameDirty(ni);
 		inode_set_mtime_to_ts(vi, inode_set_ctime_current(vi));
diff --git a/fs/ntfs/inode.c b/fs/ntfs/inode.c
index 7381a18cfadd9..50e244aa372e7 100644
--- a/fs/ntfs/inode.c
+++ b/fs/ntfs/inode.c
@@ -2401,7 +2401,7 @@ int ntfs_show_options(struct seq_file *sf, struct dentry *root)
 }
 
 int ntfs_extend_initialized_size(struct inode *vi, const loff_t offset,
-				 const loff_t new_size, bool bsync)
+				 const loff_t new_size)
 {
 	struct ntfs_inode *ni = NTFS_I(vi);
 	loff_t old_init_size;
@@ -2428,10 +2428,6 @@ int ntfs_extend_initialized_size(struct inode *vi, const loff_t offset,
 				       &ntfs_iomap_folio_ops, NULL);
 		if (err)
 			return err;
-		if (bsync)
-			err = filemap_write_and_wait_range(vi->i_mapping,
-							   old_init_size,
-							   offset - 1);
 	}
 
 
diff --git a/fs/ntfs/inode.h b/fs/ntfs/inode.h
index 9aacd5787ffea..c6d065aaecd56 100644
--- a/fs/ntfs/inode.h
+++ b/fs/ntfs/inode.h
@@ -352,7 +352,7 @@ static inline void ntfs_commit_inode(struct inode *vi)
 
 int ntfs_inode_sync_filename(struct ntfs_inode *ni);
 int ntfs_extend_initialized_size(struct inode *vi, const loff_t offset,
-		const loff_t new_size, bool bsync);
+		const loff_t new_size);
 void ntfs_set_vfs_operations(struct inode *inode, mode_t mode, dev_t dev);
 struct folio *ntfs_get_locked_folio(struct address_space *mapping,
 		pgoff_t index, pgoff_t end_index, struct file_ra_state *ra);
diff --git a/fs/ntfs/iomap.c b/fs/ntfs/iomap.c
index 52eecf5cb256a..26a1831a2c189 100644
--- a/fs/ntfs/iomap.c
+++ b/fs/ntfs/iomap.c
@@ -675,21 +675,7 @@ static int ntfs_write_iomap_begin_non_resident(struct inode *inode, loff_t offse
 					       loff_t length, unsigned int flags,
 					       struct iomap *iomap, int ntfs_iomap_flags)
 {
-	struct ntfs_inode *ni = NTFS_I(inode);
-
-	if (ntfs_iomap_flags & (NTFS_IOMAP_FLAGS_BEGIN | NTFS_IOMAP_FLAGS_DIO) &&
-	    offset + length > ni->initialized_size) {
-		int ret;
-
-		ret = ntfs_extend_initialized_size(inode, offset,
-						   offset + length,
-						   ntfs_iomap_flags &
-						   NTFS_IOMAP_FLAGS_DIO);
-		if (ret < 0)
-			return ret;
-	}
-
-	mutex_lock(&ni->mrec_lock);
+	mutex_lock(&NTFS_I(inode)->mrec_lock);
 	if (ntfs_iomap_flags & NTFS_IOMAP_FLAGS_BEGIN)
 		return  ntfs_write_simple_iomap_begin_non_resident(inode, offset,
 								   length, iomap);
@@ -705,28 +691,10 @@ static int __ntfs_write_iomap_begin(struct inode *inode, loff_t offset,
 				    struct iomap *iomap, int ntfs_iomap_flags)
 {
 	struct ntfs_inode *ni = NTFS_I(inode);
-	loff_t end = offset + length;
 
 	if (NVolShutdown(ni->vol))
 		return -EIO;
 
-	if (ntfs_iomap_flags & (NTFS_IOMAP_FLAGS_BEGIN | NTFS_IOMAP_FLAGS_DIO) &&
-	    end > ni->data_size) {
-		struct ntfs_volume *vol = ni->vol;
-		int ret;
-
-		mutex_lock(&ni->mrec_lock);
-		if (end > ni->allocated_size &&
-		    end < ni->allocated_size + vol->preallocated_size)
-			ret = ntfs_attr_expand(ni, end,
-					ni->allocated_size + vol->preallocated_size);
-		else
-			ret = ntfs_attr_expand(ni, end, 0);
-		mutex_unlock(&ni->mrec_lock);
-		if (ret)
-			return ret;
-	}
-
 	if (!NInoNonResident(ni)) {
 		mutex_lock(&ni->mrec_lock);
 		return ntfs_write_iomap_begin_resident(inode, offset, iomap);
diff --git a/fs/ntfs/mft.c b/fs/ntfs/mft.c
index fd20d7abd6f5e..f95e433885a00 100644
--- a/fs/ntfs/mft.c
+++ b/fs/ntfs/mft.c
@@ -2420,7 +2420,7 @@ int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode,
 		 * record.
 		 */
 
-		(*ni)->mrec = kmalloc(vol->mft_record_size, GFP_NOFS);
+		(*ni)->mrec = kmemdup(m, vol->mft_record_size, GFP_NOFS);
 		if (!(*ni)->mrec) {
 			folio_unlock(folio);
 			kunmap_local(m);
@@ -2429,7 +2429,6 @@ int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode,
 			goto undo_mftbmp_alloc;
 		}
 
-		memcpy((*ni)->mrec, m, vol->mft_record_size);
 		post_read_mst_fixup((struct ntfs_record *)(*ni)->mrec, vol->mft_record_size);
 		ntfs_mft_mark_dirty(folio);
 		folio_unlock(folio);
diff --git a/fs/ntfs/namei.c b/fs/ntfs/namei.c
index 5ff25e9aaa325..8364d7bb571d0 100644
--- a/fs/ntfs/namei.c
+++ b/fs/ntfs/namei.c
@@ -685,7 +685,8 @@ static struct ntfs_inode *__ntfs_create(struct mnt_idmap *idmap, struct inode *d
 	mutex_unlock(&dir_ni->mrec_lock);
 	mutex_unlock(&ni->mrec_lock);
 
-	ni->flags = fn->file_attributes;
+	ni->flags = fn->file_attributes |
+		    (ni->flags & FILE_ATTRIBUTE_RECALL_ON_OPEN);
 	/* Set the sequence number. */
 	vi->i_generation = ni->seq_no;
 	set_nlink(vi, 1);
@@ -736,7 +737,7 @@ static struct ntfs_inode *__ntfs_create(struct mnt_idmap *idmap, struct inode *d
 }
 
 static int ntfs_create(struct mnt_idmap *idmap, struct inode *dir,
-		struct dentry *dentry, umode_t mode, bool excl)
+		struct dentry *dentry, umode_t mode)
 {
 	struct ntfs_volume *vol = NTFS_SB(dir->i_sb);
 	struct ntfs_inode *ni;
@@ -1082,7 +1083,7 @@ static struct dentry *ntfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	if (!(vol->vol_flags & VOLUME_IS_DIRTY))
 		ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY);
 
-	ni = __ntfs_create(idmap, dir, uname, uname_len, S_IFDIR | mode, 0, NULL, 0);
+	ni = __ntfs_create(idmap, dir, uname, uname_len, mode, 0, NULL, 0);
 	kmem_cache_free(ntfs_name_cache, uname);
 	if (IS_ERR(ni)) {
 		err = PTR_ERR(ni);
diff --git a/fs/ntfs/reparse.c b/fs/ntfs/reparse.c
index fa523dc3691ea..ced4c675d91fc 100644
--- a/fs/ntfs/reparse.c
+++ b/fs/ntfs/reparse.c
@@ -317,8 +317,7 @@ unsigned int ntfs_make_symlink(struct ntfs_inode *ni)
 	} else
 		ni->flags &= ~FILE_ATTR_REPARSE_POINT;
 
-	if (reparse_attr)
-		kvfree(reparse_attr);
+	kvfree(reparse_attr);
 
 	return mode;
 }
@@ -358,8 +357,7 @@ unsigned int ntfs_reparse_tag_dt_types(struct ntfs_volume *vol, unsigned long mr
 		}
 	}
 
-	if (reparse_attr)
-		kvfree(reparse_attr);
+	kvfree(reparse_attr);
 
 	iput(vi);
 	return dt_type;
diff --git a/fs/ntfs/runlist.c b/fs/ntfs/runlist.c
index cbb6576cf7258..8e0fd400e7f78 100644
--- a/fs/ntfs/runlist.c
+++ b/fs/ntfs/runlist.c
@@ -71,29 +71,46 @@ static inline void ntfs_rl_mc(struct runlist_element *dstbase, int dst,
  * On success, return a pointer to the newly allocated, or recycled, memory.
  * On error, return -errno.
  */
-struct runlist_element *ntfs_rl_realloc(struct runlist_element *rl,
-		int old_size, int new_size)
+static inline struct runlist_element *ntfs_rl_realloc_gfp(struct runlist_element *rl,
+		int old_size, int new_size, gfp_t gfp)
 {
 	struct runlist_element *new_rl;
+	size_t new_bytes;
+
+	if (old_size < 0 || new_size < 0)
+		return ERR_PTR(-EINVAL);
 
-	old_size = old_size * sizeof(*rl);
-	new_size = new_size * sizeof(*rl);
 	if (old_size == new_size)
 		return rl;
 
-	new_rl = kvzalloc(new_size, GFP_NOFS);
+	if (check_mul_overflow(new_size, sizeof(*rl), &new_bytes))
+		return ERR_PTR(-EINVAL);
+
+	new_rl = kvzalloc(new_bytes, gfp);
 	if (unlikely(!new_rl))
 		return ERR_PTR(-ENOMEM);
 
 	if (likely(rl != NULL)) {
-		if (unlikely(old_size > new_size))
-			old_size = new_size;
-		memcpy(new_rl, rl, old_size);
+		size_t old_bytes;
+
+		if (check_mul_overflow(old_size, sizeof(*rl), &old_bytes)) {
+			kvfree(new_rl);
+			return ERR_PTR(-EINVAL);
+		}
+		if (unlikely(old_bytes > new_bytes))
+			old_bytes = new_bytes;
+		memcpy(new_rl, rl, old_bytes);
 		kvfree(rl);
 	}
 	return new_rl;
 }
 
+struct runlist_element *ntfs_rl_realloc(struct runlist_element *rl,
+		int old_size, int new_size)
+{
+	return ntfs_rl_realloc_gfp(rl, old_size, new_size, GFP_NOFS);
+}
+
 /*
  * ntfs_rl_realloc_nofail - Reallocate memory for runlists
  * @rl:		original runlist
@@ -118,21 +135,8 @@ struct runlist_element *ntfs_rl_realloc(struct runlist_element *rl,
 static inline struct runlist_element *ntfs_rl_realloc_nofail(struct runlist_element *rl,
 		int old_size, int new_size)
 {
-	struct runlist_element *new_rl;
-
-	old_size = old_size * sizeof(*rl);
-	new_size = new_size * sizeof(*rl);
-	if (old_size == new_size)
-		return rl;
-
-	new_rl = kvmalloc(new_size, GFP_NOFS | __GFP_NOFAIL);
-	if (likely(rl != NULL)) {
-		if (unlikely(old_size > new_size))
-			old_size = new_size;
-		memcpy(new_rl, rl, old_size);
-		kvfree(rl);
-	}
-	return new_rl;
+	return ntfs_rl_realloc_gfp(rl, old_size, new_size,
+			GFP_NOFS | __GFP_NOFAIL);
 }
 
 /*
diff --git a/fs/ntfs3/dir.c b/fs/ntfs3/dir.c
index 873d52233003a..62482c2352ad9 100644
--- a/fs/ntfs3/dir.c
+++ b/fs/ntfs3/dir.c
@@ -273,6 +273,12 @@ struct inode *dir_search_u(struct inode *dir, const struct cpu_str *uni,
 	return err == -ENOENT ? NULL : err ? ERR_PTR(err) : inode;
 }
 
+static inline bool de_fname_fits(const struct NTFS_DE *e, u32 e_size,
+				 const struct ATTR_FILE_NAME *fname)
+{
+	return sizeof(struct NTFS_DE) + fname_full_size(fname) <= e_size;
+}
+
 /*
  * returns false if 'ctx' if full
  */
@@ -281,7 +287,7 @@ static inline bool ntfs_dir_emit(struct ntfs_sb_info *sbi,
 				 u8 *name, struct dir_context *ctx)
 {
 	const struct ATTR_FILE_NAME *fname;
-	unsigned long ino;
+	u64 ino;
 	int name_len;
 	u32 dt_type;
 
@@ -305,15 +311,13 @@ static inline bool ntfs_dir_emit(struct ntfs_sb_info *sbi,
 	if (sbi->options->nohidden && (fname->dup.fa & FILE_ATTRIBUTE_HIDDEN))
 		return true;
 
-	if (sizeof(struct NTFS_DE) +
-	    offsetof(struct ATTR_FILE_NAME, name) +
-	    fname->name_len * sizeof(short) > le16_to_cpu(e->size))
+	if (!de_fname_fits(e, le16_to_cpu(e->size), fname))
 		return true;
 
 	name_len = ntfs_utf16_to_nls(sbi, fname->name, fname->name_len, name,
 				     PATH_MAX);
 	if (name_len <= 0) {
-		ntfs_warn(sbi->sb, "failed to convert name for inode %lx.",
+		ntfs_warn(sbi->sb, "failed to convert name for inode %llx.",
 			  ino);
 		return true;
 	}
@@ -576,6 +580,23 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx)
 	return err;
 }
 
+/*
+ * Return fname when @e passes the same checks as ntfs_dir_emit() before
+ * exposing an entry (valid key, non-DOS, fname fits in e->size).
+ */
+static inline const struct ATTR_FILE_NAME *
+de_countable_fname(const struct NTFS_DE *e, u32 e_size)
+{
+	const struct ATTR_FILE_NAME *fname;
+
+	fname = de_get_fname(e);
+	if (!fname || fname->type == FILE_NAME_DOS ||
+	    !de_fname_fits(e, e_size, fname))
+		return NULL;
+
+	return fname;
+}
+
 static int ntfs_dir_count(struct inode *dir, bool *is_empty, size_t *dirs,
 			  size_t *files)
 {
@@ -615,13 +636,10 @@ static int ntfs_dir_count(struct inode *dir, bool *is_empty, size_t *dirs,
 			if (de_is_last(e))
 				break;
 
-			fname = de_get_fname(e);
+			fname = de_countable_fname(e, e_size);
 			if (!fname)
 				continue;
 
-			if (fname->type == FILE_NAME_DOS)
-				continue;
-
 			if (is_empty) {
 				*is_empty = false;
 				if (!dirs && !files)
diff --git a/fs/ntfs3/file.c b/fs/ntfs3/file.c
index d601f088618c0..fa8e2e56ff3f6 100644
--- a/fs/ntfs3/file.c
+++ b/fs/ntfs3/file.c
@@ -894,7 +894,7 @@ static ssize_t ntfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
 
 out:
 	inode_unlock_shared(inode);
-	file_accessed(iocb->ki_filp);
+	file_accessed(file);
 	return err;
 }
 
diff --git a/fs/ntfs3/frecord.c b/fs/ntfs3/frecord.c
index 2b49bc0775582..3ad309d38fe8b 100644
--- a/fs/ntfs3/frecord.c
+++ b/fs/ntfs3/frecord.c
@@ -169,7 +169,7 @@ int ni_load_mi_ex(struct ntfs_inode *ni, CLST rno, struct mft_inode **mi)
 int ni_load_mi(struct ntfs_inode *ni, const struct ATTR_LIST_ENTRY *le,
 	       struct mft_inode **mi)
 {
-	CLST rno;
+	u64 rno;
 
 	if (!le) {
 		*mi = &ni->mi;
@@ -768,10 +768,23 @@ int ni_create_attr_list(struct ntfs_inode *ni)
 	rs = sbi->record_size;
 
 	/*
-	 * Skip estimating exact memory requirement.
-	 * Looks like one record_size is always enough.
+	 * Compute the exact size of the attribute list.  Each attribute in the
+	 * record yields one ATTR_LIST_ENTRY of le_size(name_len) bytes.  The
+	 * minimum on-disk attribute is SIZEOF_RESIDENT (0x18) bytes, but an
+	 * unnamed one expands to le_size(0) (0x20) here, so a record crafted
+	 * with many such attributes needs more than a single record_size; the
+	 * previous fixed kzalloc(record_size) could therefore be overflowed by
+	 * an attacker-controlled record.
 	 */
-	le = kzalloc(al_aligned(rs), GFP_NOFS);
+	lsize = 0;
+	attr = NULL;
+	while ((attr = mi_enum_attr(ni, &ni->mi, attr)))
+		lsize += le_size(attr->name_len);
+
+	if (!lsize)
+		return -EINVAL;
+
+	le = kzalloc(al_aligned(lsize), GFP_NOFS);
 	if (!le)
 		return -ENOMEM;
 
@@ -781,7 +794,6 @@ int ni_create_attr_list(struct ntfs_inode *ni)
 	attr = NULL;
 	nb = 0;
 	free_b = 0;
-	attr = NULL;
 
 	for (; (attr = mi_enum_attr(ni, &ni->mi, attr)); le = Add2Ptr(le, sz)) {
 		sz = le_size(attr->name_len);
@@ -2919,7 +2931,6 @@ loff_t ni_seek_data_or_hole(struct ntfs_inode *ni, loff_t offset, bool data)
 				break;
 			}
 		}
-
 	}
 
 	vbo = (u64)vcn << cluster_bits;
@@ -2961,8 +2972,8 @@ int ni_write_parents(struct ntfs_inode *ni, int sync)
 		if (IS_ERR(dir)) {
 			ntfs_inode_warn(
 				&ni->vfs_inode,
-				"failed to open parent directory r=%lx to write",
-				(long)ino_get(&fname->home));
+				"failed to open parent directory r=%llx to write",
+				(u64)ino_get(&fname->home));
 			continue;
 		}
 
@@ -3081,8 +3092,8 @@ static bool ni_update_parent(struct ntfs_inode *ni, struct NTFS_DUP_INFO *dup,
 		if (IS_ERR(dir)) {
 			ntfs_inode_warn(
 				&ni->vfs_inode,
-				"failed to open parent directory r=%lx to update",
-				(long)ino_get(&fname->home));
+				"failed to open parent directory r=%llx to update",
+				(u64)ino_get(&fname->home));
 			continue;
 		}
 
diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c
index f038c799e7ac4..cc24c23e4db9d 100644
--- a/fs/ntfs3/fslog.c
+++ b/fs/ntfs3/fslog.c
@@ -2613,7 +2613,6 @@ bool check_index_header(const struct INDEX_HDR *hdr, size_t bytes)
 	const bool has_subnode = hdr_has_subnode(hdr);
 	__le16 mask;
 	u32 min_de, de_off, used, total;
-	const struct NTFS_DE *e;
 
 	if (has_subnode) {
 		min_de = sizeof(struct NTFS_DE) + sizeof(u64);
@@ -2632,8 +2631,8 @@ bool check_index_header(const struct INDEX_HDR *hdr, size_t bytes)
 		return false;
 	}
 
-	e = (const struct NTFS_DE *)((const u8 *)hdr + de_off);
 	for (;;) {
+		const struct NTFS_DE *e = Add2Ptr(hdr, de_off);
 		u16 esize = le16_to_cpu(e->size);
 		u16 key_size = le16_to_cpu(e->key_size);
 		u16 data_size;
@@ -2649,7 +2648,6 @@ bool check_index_header(const struct INDEX_HDR *hdr, size_t bytes)
 		if (de_is_last(e)) {
 			if (key_size)
 				return false;
-
 			break;
 		}
 
@@ -2658,7 +2656,6 @@ bool check_index_header(const struct INDEX_HDR *hdr, size_t bytes)
 			return false;
 
 		de_off += esize;
-		e = (const struct NTFS_DE *)((const u8 *)hdr + de_off);
 	}
 
 	return true;
@@ -3544,8 +3541,7 @@ static int do_action(struct ntfs_log *log, struct OPEN_ATTR_ENRTY *oe,
 		 * bound here so the memmove cannot reach past the entry.
 		 */
 		if (le16_to_cpu(e->view.data_off) > le16_to_cpu(e->size) ||
-		    le16_to_cpu(e->view.data_off) + dlen >
-			    le16_to_cpu(e->size))
+		    le16_to_cpu(e->view.data_off) + dlen > le16_to_cpu(e->size))
 			goto dirty_vol;
 
 		memmove(Add2Ptr(e, le16_to_cpu(e->view.data_off)), data, dlen);
@@ -3756,8 +3752,7 @@ static int do_action(struct ntfs_log *log, struct OPEN_ATTR_ENRTY *oe,
 
 		/* See UpdateRecordDataRoot for the rationale. */
 		if (le16_to_cpu(e->view.data_off) > le16_to_cpu(e->size) ||
-		    le16_to_cpu(e->view.data_off) + dlen >
-			    le16_to_cpu(e->size))
+		    le16_to_cpu(e->view.data_off) + dlen > le16_to_cpu(e->size))
 			goto dirty_vol;
 
 		memmove(Add2Ptr(e, le16_to_cpu(e->view.data_off)), data, dlen);
@@ -4653,11 +4648,11 @@ int log_replay(struct ntfs_inode *ni, bool *initialized)
 		}
 
 		/*
-         * find_dp() only validates that target_vcn is the first
-         * cluster covered by dp.  The walk through lrh->lcns_follow
-         * further entries must stay within the allocated
-         * dp->page_lcns[] array, which is sized by dp->lcns_follow.
-         */
+		 * find_dp() only validates that target_vcn is the first
+		 * cluster covered by dp.  The walk through lrh->lcns_follow
+		 * further entries must stay within the allocated
+		 * dp->page_lcns[] array, which is sized by dp->lcns_follow.
+		 */
 		if (le64_to_cpu(lrh->target_vcn) - le64_to_cpu(dp->vcn) + t16 >
 		    le32_to_cpu(dp->lcns_follow)) {
 			err = -EINVAL;
diff --git a/fs/ntfs3/fsntfs.c b/fs/ntfs3/fsntfs.c
index bc7469d0a34d4..7c4db816c43d3 100644
--- a/fs/ntfs3/fsntfs.c
+++ b/fs/ntfs3/fsntfs.c
@@ -2302,8 +2302,8 @@ int ntfs_reparse_init(struct ntfs_sb_info *sbi)
 		goto out;
 	}
 
-	root_r = resident_data(attr);
-	if (root_r->type != ATTR_ZERO ||
+	root_r = resident_data_ex(attr, sizeof(struct INDEX_ROOT));
+	if (!root_r || root_r->type != ATTR_ZERO ||
 	    root_r->rule != NTFS_COLLATION_TYPE_UINTS) {
 		err = -EINVAL;
 		goto out;
@@ -2340,8 +2340,8 @@ int ntfs_objid_init(struct ntfs_sb_info *sbi)
 		goto out;
 	}
 
-	root = resident_data(attr);
-	if (root->type != ATTR_ZERO ||
+	root = resident_data_ex(attr, sizeof(struct INDEX_ROOT));
+	if (!root || root->type != ATTR_ZERO ||
 	    root->rule != NTFS_COLLATION_TYPE_UINTS) {
 		err = -EINVAL;
 		goto out;
diff --git a/fs/ntfs3/index.c b/fs/ntfs3/index.c
index 2b439ac043563..4afacfc6ff893 100644
--- a/fs/ntfs3/index.c
+++ b/fs/ntfs3/index.c
@@ -612,8 +612,8 @@ static const struct NTFS_DE *hdr_insert_head(struct INDEX_HDR *hdr,
 static bool index_hdr_check(const struct INDEX_HDR *hdr, u32 bytes)
 {
 	const bool has_subnode = hdr_has_subnode(hdr);
-	const u16 min_size = sizeof(struct NTFS_DE) +
-			     (has_subnode ? sizeof(u64) : 0);
+	const u16 min_size =
+		sizeof(struct NTFS_DE) + (has_subnode ? sizeof(u64) : 0);
 	u32 end = le32_to_cpu(hdr->used);
 	u32 tot = le32_to_cpu(hdr->total);
 	u32 off = le32_to_cpu(hdr->de_off);
@@ -2131,8 +2131,7 @@ static struct indx_node *indx_find_buffer(struct ntfs_index *indx,
 			if (err)
 				return ERR_PTR(err);
 
-			r = indx_find_buffer(indx, ni, root, vbn, n,
-					     depth + 1);
+			r = indx_find_buffer(indx, ni, root, vbn, n, depth + 1);
 			if (r)
 				return r;
 		}
diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c
index 0c9bd669117de..4ecd7ebaa655c 100644
--- a/fs/ntfs3/inode.c
+++ b/fs/ntfs3/inode.c
@@ -36,7 +36,7 @@ static struct inode *ntfs_read_mft(struct inode *inode,
 	bool is_match = false;
 	bool is_root = false;
 	bool is_dir;
-	unsigned long ino = inode->i_ino;
+	u64 ino = inode->i_ino;
 	u32 rp_fa = 0, asize, t32;
 	u16 roff, rsize, names = 0, links = 0;
 	const struct ATTR_FILE_NAME *fname = NULL;
@@ -79,7 +79,7 @@ static struct inode *ntfs_read_mft(struct inode *inode,
 		;
 	} else if (ref->seq != rec->seq) {
 		err = -EINVAL;
-		ntfs_err(sb, "MFT: r=%lx, expect seq=%x instead of %x!", ino,
+		ntfs_err(sb, "MFT: r=%llx, expect seq=%x instead of %x!", ino,
 			 le16_to_cpu(ref->seq), le16_to_cpu(rec->seq));
 		goto out;
 	} else if (!is_rec_inuse(rec)) {
@@ -127,10 +127,16 @@ static struct inode *ntfs_read_mft(struct inode *inode,
 
 	if (le && le->vcn) {
 		/* This is non primary attribute segment. Ignore if not MFT. */
-		if (ino != MFT_REC_MFT || attr->type != ATTR_DATA)
+		if (ino != MFT_REC_MFT)
+			goto next_attr;
+
+		if (attr->type == ATTR_DATA)
+			run = &ni->file.run;
+		else if (attr->type == ATTR_BITMAP)
+			run = &sbi->mft.bitmap.run;
+		else
 			goto next_attr;
 
-		run = &ni->file.run;
 		asize = le32_to_cpu(attr->size);
 		goto attr_unpack_run;
 	}
@@ -606,15 +612,17 @@ static void ntfs_iomap_read_end_io(struct bio *bio)
 }
 
 static void ntfs_iomap_bio_submit_read(const struct iomap_iter *iter,
-		struct iomap_read_folio_ctx *ctx)
+				       struct iomap_read_folio_ctx *ctx)
 {
 	iomap_bio_submit_read_endio(iter, ctx, ntfs_iomap_read_end_io);
 }
 
+// clang-format off
 static const struct iomap_read_ops ntfs_iomap_bio_read_ops = {
 	.read_folio_range	= iomap_bio_read_folio_range,
 	.submit_read		= ntfs_iomap_bio_submit_read,
 };
+// clang-format on
 
 static int ntfs_read_folio(struct file *file, struct folio *folio)
 {
diff --git a/fs/ntfs3/lznt.c b/fs/ntfs3/lznt.c
index f818d97850049..5dcb7674790cb 100644
--- a/fs/ntfs3/lznt.c
+++ b/fs/ntfs3/lznt.c
@@ -240,8 +240,10 @@ static inline ssize_t decompress_chunk(u8 *unc, u8 *unc_end, const u8 *cmpr,
 		if (up - unc > LZNT_CHUNK_SIZE)
 			return -EINVAL;
 		/* Correct index */
-		while (index < ARRAY_SIZE(s_max_off) - 1 && unc + s_max_off[index] < up)
+		while (index < ARRAY_SIZE(s_max_off) - 1 &&
+		       unc + s_max_off[index] < up) {
 			index += 1;
+		}
 
 		/* Check the current flag for zero. */
 		if (!(ch & (1 << bit))) {
diff --git a/fs/ntfs3/namei.c b/fs/ntfs3/namei.c
index c59de5f2fa977..7bf889a4ca350 100644
--- a/fs/ntfs3/namei.c
+++ b/fs/ntfs3/namei.c
@@ -22,7 +22,7 @@ int fill_name_de(struct ntfs_sb_info *sbi, void *buf, const struct qstr *name,
 {
 	int err;
 	struct NTFS_DE *e = buf;
-	u16 data_size;
+	u16 data_size, real_size, aligned_size;
 	struct ATTR_FILE_NAME *fname = (struct ATTR_FILE_NAME *)(e + 1);
 
 #ifndef CONFIG_NTFS3_64BIT_CLUSTER
@@ -53,7 +53,12 @@ int fill_name_de(struct ntfs_sb_info *sbi, void *buf, const struct qstr *name,
 	fname->type = FILE_NAME_POSIX;
 	data_size = fname_full_size(fname);
 
-	e->size = cpu_to_le16(ALIGN(data_size, 8) + sizeof(struct NTFS_DE));
+	real_size = data_size + sizeof(struct NTFS_DE);
+	aligned_size = ALIGN(data_size, 8) + sizeof(struct NTFS_DE);
+	if (aligned_size > real_size)
+		memset((char *)buf + real_size, 0, aligned_size - real_size);
+
+	e->size = cpu_to_le16(aligned_size);
 	e->key_size = cpu_to_le16(data_size);
 	e->flags = 0;
 	e->res = 0;
@@ -105,7 +110,7 @@ static struct dentry *ntfs_lookup(struct inode *dir, struct dentry *dentry,
  * ntfs_create - inode_operations::create
  */
 static int ntfs_create(struct mnt_idmap *idmap, struct inode *dir,
-		       struct dentry *dentry, umode_t mode, bool excl)
+		       struct dentry *dentry, umode_t mode)
 {
 	return ntfs_create_inode(idmap, dir, dentry, NULL, S_IFREG | mode, 0,
 				 NULL, 0, NULL);
@@ -213,7 +218,7 @@ static struct dentry *ntfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 				 struct dentry *dentry, umode_t mode)
 {
 	return ERR_PTR(ntfs_create_inode(idmap, dir, dentry, NULL,
-					 S_IFDIR | mode, 0, NULL, 0, NULL));
+					 mode, 0, NULL, 0, NULL));
 }
 
 /*
diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h
index d98d7e474476c..6bd1a439e92f3 100644
--- a/fs/ntfs3/ntfs_fs.h
+++ b/fs/ntfs3/ntfs_fs.h
@@ -401,7 +401,7 @@ struct ntfs_inode {
 			struct rw_semaphore run_lock;
 			/* Unpacked runs from just one record. */
 			struct runs_tree run;
-			/* 
+			/*
 			 * Pairs [vcn, len] for all delay allocated clusters.
 			 * Normal file always contains delayed clusters in one fragment.
 			 * TODO: use 2 CLST per pair instead of 3.
@@ -886,8 +886,8 @@ int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino,
 #else
 #define run_unpack_ex run_unpack
 #endif
-int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size, 
-		       u64 *highest_vcn);
+int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size,
+			u64 *highest_vcn);
 int run_clone(const struct runs_tree *run, struct runs_tree *new_run);
 bool run_remove_range(struct runs_tree *run, CLST vcn, CLST len, CLST *done);
 CLST run_len(const struct runs_tree *run);
diff --git a/fs/ntfs3/run.c b/fs/ntfs3/run.c
index 3ebf0154eda39..6e3ef89fc6660 100644
--- a/fs/ntfs3/run.c
+++ b/fs/ntfs3/run.c
@@ -1265,8 +1265,8 @@ int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino,
  * Return the highest vcn from a mapping pairs array
  * it used while replaying log file.
  */
-int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size, 
-		       u64 *highest_vcn)
+int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size,
+			u64 *highest_vcn)
 {
 	const u8 *run_last = run_buf + run_buf_size;
 	u64 vcn64 = vcn;
@@ -1279,7 +1279,7 @@ int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size,
 		if (size_size > 8 || offset_size > 8)
 			return -EINVAL;
 
-		if (run_buf + size_size + offset_size > run_last) 
+		if (run_buf + size_size + offset_size > run_last)
 			return -EINVAL;
 
 		len = run_unpack_s64(run_buf, size_size, 0);
@@ -1357,7 +1357,8 @@ bool run_remove_range(struct runs_tree *run, CLST vcn, CLST len, CLST *done)
 		if (r_end > end) {
 			/* Remove a middle part, split. */
 			CLST tail_lcn = r->lcn == SPARSE_LCN ?
-					SPARSE_LCN : (r->lcn + (end - r->vcn));
+						SPARSE_LCN :
+						(r->lcn + (end - r->vcn));
 
 			*done += len;
 			r->len = d;
diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c
index 3305fe406cb22..446b37f0de795 100644
--- a/fs/ntfs3/super.c
+++ b/fs/ntfs3/super.c
@@ -1458,7 +1458,10 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc)
 					    Add2Ptr(a, roff),
 					    le32_to_cpu(a->size) - roff);
 			if (err < 0) {
-				ntfs_err(sb, "Failed to unpack $MFT bitmap extent (%d).", err);
+				ntfs_err(
+					sb,
+					"Failed to unpack $MFT bitmap extent (%d).",
+					err);
 				goto put_inode_out;
 			}
 			err = 0;
@@ -1866,8 +1869,7 @@ static int ntfs_init_fs_context(struct fs_context *fc)
 	/* Default options. */
 	opts->fs_uid = current_uid();
 	opts->fs_gid = current_gid();
-	opts->fs_fmask_inv = ~current_umask();
-	opts->fs_dmask_inv = ~current_umask();
+	opts->fs_fmask_inv = opts->fs_dmask_inv = ~current_umask();
 	opts->prealloc = 1;
 
 #ifdef CONFIG_NTFS3_FS_POSIX_ACL
@@ -1928,7 +1930,6 @@ static struct file_system_type ntfs_fs_type = {
 	.kill_sb		= ntfs3_kill_sb,
 	.fs_flags		= FS_REQUIRES_DEV | FS_ALLOW_IDMAP,
 };
-
 // clang-format on
 
 static int __init init_ntfs_fs(void)
diff --git a/fs/ntfs3/xattr.c b/fs/ntfs3/xattr.c
index 04814dd293752..7a81369a11734 100644
--- a/fs/ntfs3/xattr.c
+++ b/fs/ntfs3/xattr.c
@@ -660,7 +660,6 @@ static noinline int ntfs_set_acl_ex(struct mnt_idmap *idmap,
 			inode->i_mode = old_mode;
 			goto out;
 		}
-		inode->i_mode = mode;
 	}
 	set_cached_acl(inode, type, acl);
 	inode_set_ctime_current(inode);
diff --git a/fs/nullfs.c b/fs/nullfs.c
index fdbd3e5d3d71f..e06352c7b2ccf 100644
--- a/fs/nullfs.c
+++ b/fs/nullfs.c
@@ -4,6 +4,8 @@
 #include <linux/fs_context.h>
 #include <linux/magic.h>
 
+#include "mount.h"
+
 static const struct super_operations nullfs_super_operations = {
 	.statfs	= simple_statfs,
 };
@@ -40,14 +42,9 @@ static int nullfs_fs_fill_super(struct super_block *s, struct fs_context *fc)
 	return 0;
 }
 
-/*
- * For now this is a single global instance. If needed we can make it
- * mountable by userspace at which point we will need to make it
- * multi-instance.
- */
 static int nullfs_fs_get_tree(struct fs_context *fc)
 {
-	return get_tree_single(fc, nullfs_fs_fill_super);
+	return get_tree_nodev(fc, nullfs_fs_fill_super);
 }
 
 static const struct fs_context_operations nullfs_fs_context_ops = {
@@ -57,9 +54,8 @@ static const struct fs_context_operations nullfs_fs_context_ops = {
 static int nullfs_init_fs_context(struct fs_context *fc)
 {
 	fc->ops		= &nullfs_fs_context_ops;
-	fc->global	= true;
-	fc->sb_flags	= SB_NOUSER;
-	fc->s_iflags	= SB_I_NOEXEC | SB_I_NODEV;
+	fc->sb_flags	|= SB_NOUSER;
+	fc->s_iflags	|= SB_I_NOEXEC | SB_I_NODEV;
 	return 0;
 }
 
diff --git a/fs/ocfs2/cluster/heartbeat.c b/fs/ocfs2/cluster/heartbeat.c
index d12784aaaa4b1..29542edbc992c 100644
--- a/fs/ocfs2/cluster/heartbeat.c
+++ b/fs/ocfs2/cluster/heartbeat.c
@@ -203,6 +203,7 @@ struct o2hb_region {
 
 	/* protected by the hr_callback_sem */
 	struct task_struct 	*hr_task;
+	u8			hr_node_num;
 
 	unsigned int		hr_blocks;
 	unsigned long long	hr_start_block;
@@ -271,6 +272,9 @@ struct o2hb_bio_wait_ctxt {
 	atomic_t          wc_num_reqs;
 	struct completion wc_io_complete;
 	int               wc_error;
+	/* On-stack bio used by the synchronous write path only. */
+	struct bio        wc_write_bio;
+	struct bio_vec    wc_write_bvec;
 };
 
 #define O2HB_NEGO_TIMEOUT_MS (O2HB_MAX_WRITE_TIMEOUT_MS/2)
@@ -350,12 +354,12 @@ static void o2hb_disarm_timeout(struct o2hb_region *reg)
 	cancel_delayed_work_sync(&reg->hr_nego_timeout_work);
 }
 
-static int o2hb_send_nego_msg(int key, int type, u8 target)
+static int o2hb_send_nego_msg(int key, int type, u8 target, u8 node_num)
 {
 	struct o2hb_nego_msg msg;
 	int status, ret;
 
-	msg.node_num = o2nm_this_node();
+	msg.node_num = node_num;
 again:
 	ret = o2net_send_message(type, key, &msg, sizeof(msg),
 			target, &status);
@@ -373,8 +377,10 @@ static void o2hb_nego_timeout(struct work_struct *work)
 	unsigned long live_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)];
 	int master_node, i, ret;
 	struct o2hb_region *reg;
+	u8 node_num;
 
 	reg = container_of(work, struct o2hb_region, hr_nego_timeout_work.work);
+	node_num = reg->hr_node_num;
 	/* don't negotiate timeout if last hb failed since it is very
 	 * possible io failed. Should let write timeout fence self.
 	 */
@@ -385,10 +391,10 @@ static void o2hb_nego_timeout(struct work_struct *work)
 	/* lowest node as master node to make negotiate decision. */
 	master_node = find_first_bit(live_node_bitmap, O2NM_MAX_NODES);
 
-	if (master_node == o2nm_this_node()) {
+	if (master_node == node_num) {
 		if (!test_bit(master_node, reg->hr_nego_node_bitmap)) {
 			printk(KERN_NOTICE "o2hb: node %d hb write hung for %ds on region %s (%pg).\n",
-				o2nm_this_node(), O2HB_NEGO_TIMEOUT_MS/1000,
+				node_num, O2HB_NEGO_TIMEOUT_MS / 1000,
 				config_item_name(&reg->hr_item), reg_bdev(reg));
 			set_bit(master_node, reg->hr_nego_node_bitmap);
 		}
@@ -417,7 +423,7 @@ static void o2hb_nego_timeout(struct work_struct *work)
 
 			mlog(ML_HEARTBEAT, "send NEGO_APPROVE msg to node %d\n", i);
 			ret = o2hb_send_nego_msg(reg->hr_key,
-					O2HB_NEGO_APPROVE_MSG, i);
+					O2HB_NEGO_APPROVE_MSG, i, node_num);
 			if (ret)
 				mlog(ML_ERROR, "send NEGO_APPROVE msg to node %d fail %d\n",
 					i, ret);
@@ -425,10 +431,10 @@ static void o2hb_nego_timeout(struct work_struct *work)
 	} else {
 		/* negotiate timeout with master node. */
 		printk(KERN_NOTICE "o2hb: node %d hb write hung for %ds on region %s (%pg), negotiate timeout with node %d.\n",
-			o2nm_this_node(), O2HB_NEGO_TIMEOUT_MS/1000, config_item_name(&reg->hr_item),
+			node_num, O2HB_NEGO_TIMEOUT_MS / 1000, config_item_name(&reg->hr_item),
 			reg_bdev(reg), master_node);
 		ret = o2hb_send_nego_msg(reg->hr_key, O2HB_NEGO_TIMEOUT_MSG,
-				master_node);
+				master_node, node_num);
 		if (ret)
 			mlog(ML_ERROR, "send NEGO_TIMEOUT msg to node %d fail %d\n",
 				master_node, ret);
@@ -504,6 +510,23 @@ static void o2hb_bio_end_io(struct bio *bio)
 	bio_put(bio);
 }
 
+/*
+ * End I/O for the synchronous write path. The write bio is embedded in
+ * the wait ctxt rather than allocated, so it must not be freed here; it
+ * is torn down with bio_uninit() once the caller has waited on it.
+ */
+static void o2hb_write_bio_end_io(struct bio *bio)
+{
+	struct o2hb_bio_wait_ctxt *wc = bio->bi_private;
+
+	if (bio->bi_status) {
+		mlog(ML_ERROR, "IO Error %d\n", bio->bi_status);
+		wc->wc_error = blk_status_to_errno(bio->bi_status);
+	}
+
+	o2hb_bio_wait_dec(wc, 1);
+}
+
 /* Setup a Bio to cover I/O against num_slots slots starting at
  * start_slot. */
 static struct bio *o2hb_setup_one_bio(struct o2hb_region *reg,
@@ -519,16 +542,12 @@ static struct bio *o2hb_setup_one_bio(struct o2hb_region *reg,
 	struct bio *bio;
 	struct page *page;
 
-	/* Testing has shown this allocation to take long enough under
-	 * GFP_KERNEL that the local node can get fenced. It would be
-	 * nicest if we could pre-allocate these bios and avoid this
-	 * all together. */
-	bio = bio_alloc(reg_bdev(reg), 16, opf, GFP_ATOMIC);
-	if (!bio) {
-		mlog(ML_ERROR, "Could not alloc slots BIO!\n");
-		bio = ERR_PTR(-ENOMEM);
-		goto bail;
-	}
+	/*
+	 * The heartbeat runs in process context and can sleep, so use
+	 * GFP_NOFS. It is backed by the fs_bio_set mempool and thus cannot
+	 * fail, while avoiding recursion back into the filesystem.
+	 */
+	bio = bio_alloc(reg_bdev(reg), 16, opf, GFP_NOFS);
 
 	/* Must put everything in 512 byte sectors for the bio... */
 	bio->bi_iter.bi_sector = (reg->hr_start_block + cs) << (bits - 9);
@@ -553,7 +572,6 @@ static struct bio *o2hb_setup_one_bio(struct o2hb_region *reg,
 		vec_start = 0;
 	}
 
-bail:
 	*current_slot = cs;
 	return bio;
 }
@@ -563,7 +581,6 @@ static int o2hb_read_slots(struct o2hb_region *reg,
 			   unsigned int max_slots)
 {
 	unsigned int current_slot = begin_slot;
-	int status;
 	struct o2hb_bio_wait_ctxt wc;
 	struct bio *bio;
 
@@ -572,51 +589,51 @@ static int o2hb_read_slots(struct o2hb_region *reg,
 	while(current_slot < max_slots) {
 		bio = o2hb_setup_one_bio(reg, &wc, &current_slot, max_slots,
 					 REQ_OP_READ);
-		if (IS_ERR(bio)) {
-			status = PTR_ERR(bio);
-			mlog_errno(status);
-			goto bail_and_wait;
-		}
-
 		atomic_inc(&wc.wc_num_reqs);
 		submit_bio(bio);
 	}
 
-	status = 0;
-
-bail_and_wait:
 	o2hb_wait_on_io(&wc);
-	if (wc.wc_error && !status)
-		status = wc.wc_error;
 
-	return status;
+	return wc.wc_error;
 }
 
 static int o2hb_issue_node_write(struct o2hb_region *reg,
 				 struct o2hb_bio_wait_ctxt *write_wc)
 {
-	int status;
 	unsigned int slot;
-	struct bio *bio;
+	unsigned int bits = reg->hr_block_bits;
+	unsigned int spp = reg->hr_slots_per_page;
+	unsigned int vec_start, vec_len;
+	struct page *page;
+	struct bio *bio = &write_wc->wc_write_bio;
 
 	o2hb_bio_wait_init(write_wc);
 
-	slot = o2nm_this_node();
+	slot = reg->hr_node_num;
+	if (slot >= O2NM_MAX_NODES)
+		return -EINVAL;
 
-	bio = o2hb_setup_one_bio(reg, write_wc, &slot, slot+1,
-				 REQ_OP_WRITE | REQ_SYNC);
-	if (IS_ERR(bio)) {
-		status = PTR_ERR(bio);
-		mlog_errno(status);
-		goto bail;
-	}
+	/*
+	 * The heartbeat write always covers our own single slot, i.e. one
+	 * block that lives within a single page. Use an on-stack bio (embedded
+	 * in write_wc) so this fence-critical path never has to allocate.
+	 */
+	bio_init(bio, reg_bdev(reg), &write_wc->wc_write_bvec, 1,
+		 REQ_OP_WRITE | REQ_SYNC);
+	bio->bi_iter.bi_sector = (reg->hr_start_block + slot) << (bits - 9);
+	bio->bi_private = write_wc;
+	bio->bi_end_io = o2hb_write_bio_end_io;
+
+	page = reg->hr_slot_data[slot / spp];
+	vec_start = (slot << bits) % PAGE_SIZE;
+	vec_len = PAGE_SIZE / spp;
+	__bio_add_page(bio, page, vec_len, vec_start);
 
 	atomic_inc(&write_wc->wc_num_reqs);
 	submit_bio(bio);
 
-	status = 0;
-bail:
-	return status;
+	return 0;
 }
 
 static u32 o2hb_compute_block_crc_le(struct o2hb_region *reg,
@@ -670,8 +687,12 @@ static int o2hb_check_own_slot(struct o2hb_region *reg)
 	struct o2hb_disk_slot *slot;
 	struct o2hb_disk_heartbeat_block *hb_block;
 	char *errstr;
+	u8 node_num = reg->hr_node_num;
+
+	if (node_num >= O2NM_MAX_NODES)
+		return 0;
 
-	slot = &reg->hr_slots[o2nm_this_node()];
+	slot = &reg->hr_slots[node_num];
 	/* Don't check on our 1st timestamp */
 	if (!slot->ds_last_time)
 		return 0;
@@ -712,7 +733,10 @@ static inline void o2hb_prepare_block(struct o2hb_region *reg,
 	struct o2hb_disk_slot *slot;
 	struct o2hb_disk_heartbeat_block *hb_block;
 
-	node_num = o2nm_this_node();
+	node_num = reg->hr_node_num;
+	if (node_num >= O2NM_MAX_NODES)
+		return;
+
 	slot = &reg->hr_slots[node_num];
 
 	hb_block = (struct o2hb_disk_heartbeat_block *)slot->ds_raw_block;
@@ -1146,6 +1170,7 @@ static int o2hb_do_disk_heartbeat(struct o2hb_region *reg)
 	 * people we find in our steady state have seen us.
 	 */
 	o2hb_wait_on_io(&write_wc);
+	bio_uninit(&write_wc.wc_write_bio);
 	if (write_wc.wc_error) {
 		/* Do not re-arm the write timeout on I/O error - we
 		 * can't be sure that the new block ever made it to
@@ -1206,7 +1231,7 @@ static int o2hb_thread(void *data)
 	set_user_nice(current, MIN_NICE);
 
 	/* Pin node */
-	ret = o2nm_depend_this_node();
+	ret = o2nm_depend_node(reg->hr_node_num);
 	if (ret) {
 		mlog(ML_ERROR, "Node has been deleted, ret = %d\n", ret);
 		reg->hr_node_deleted = 1;
@@ -1215,7 +1240,8 @@ static int o2hb_thread(void *data)
 	}
 
 	while (!kthread_should_stop() &&
-	       !reg->hr_unclean_stop && !reg->hr_aborted_start) {
+	       !reg->hr_unclean_stop && !reg->hr_aborted_start &&
+	       o2nm_this_node() == reg->hr_node_num) {
 		/* We track the time spent inside
 		 * o2hb_do_disk_heartbeat so that we avoid more than
 		 * hr_timeout_ms between disk writes. On busy systems
@@ -1257,14 +1283,16 @@ static int o2hb_thread(void *data)
 	if (!reg->hr_unclean_stop && !reg->hr_aborted_start) {
 		o2hb_prepare_block(reg, 0);
 		ret = o2hb_issue_node_write(reg, &write_wc);
-		if (ret == 0)
+		if (ret == 0) {
 			o2hb_wait_on_io(&write_wc);
-		else
+			bio_uninit(&write_wc.wc_write_bio);
+		} else {
 			mlog_errno(ret);
+		}
 	}
 
 	/* Unpin node */
-	o2nm_undepend_this_node();
+	o2nm_undepend_node(reg->hr_node_num);
 
 	mlog(ML_HEARTBEAT|ML_KTHREAD, "o2hb thread exiting\n");
 
@@ -1791,7 +1819,8 @@ static ssize_t o2hb_region_dev_store(struct config_item *item,
 
 	/* We can't heartbeat without having had our node number
 	 * configured yet. */
-	if (o2nm_this_node() == O2NM_MAX_NODES)
+	reg->hr_node_num = o2nm_this_node();
+	if (reg->hr_node_num == O2NM_MAX_NODES)
 		return -EINVAL;
 
 	ret = kstrtol(p, 0, &fd);
@@ -2036,6 +2065,7 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *g
 		ret = -ENAMETOOLONG;
 		goto free;
 	}
+	reg->hr_node_num = O2NM_MAX_NODES;
 
 	spin_lock(&o2hb_live_lock);
 	reg->hr_region_num = 0;
diff --git a/fs/ocfs2/cluster/nodemanager.c b/fs/ocfs2/cluster/nodemanager.c
index 4025631545508..e1f8f577ce5d6 100644
--- a/fs/ocfs2/cluster/nodemanager.c
+++ b/fs/ocfs2/cluster/nodemanager.c
@@ -367,6 +367,7 @@ static ssize_t o2nm_node_local_store(struct config_item *item, const char *page,
 	if (!tmp && cluster->cl_has_local &&
 	    cluster->cl_local_node == node->nd_num) {
 		o2net_stop_listening(node);
+		cluster->cl_has_local = 0;
 		cluster->cl_local_node = O2NM_INVALID_NODE_NUM;
 	}
 
@@ -782,12 +783,12 @@ void o2nm_undepend_item(struct config_item *item)
 	configfs_undepend_item(item);
 }
 
-int o2nm_depend_this_node(void)
+int o2nm_depend_node(u8 node_num)
 {
 	int ret = 0;
 	struct o2nm_node *local_node;
 
-	local_node = o2nm_get_node_by_num(o2nm_this_node());
+	local_node = o2nm_get_node_by_num(node_num);
 	if (!local_node) {
 		ret = -EINVAL;
 		goto out;
@@ -800,17 +801,27 @@ int o2nm_depend_this_node(void)
 	return ret;
 }
 
-void o2nm_undepend_this_node(void)
+void o2nm_undepend_node(u8 node_num)
 {
 	struct o2nm_node *local_node;
 
-	local_node = o2nm_get_node_by_num(o2nm_this_node());
+	local_node = o2nm_get_node_by_num(node_num);
 	BUG_ON(!local_node);
 
 	o2nm_undepend_item(&local_node->nd_item);
 	o2nm_node_put(local_node);
 }
 
+int o2nm_depend_this_node(void)
+{
+	return o2nm_depend_node(o2nm_this_node());
+}
+
+void o2nm_undepend_this_node(void)
+{
+	o2nm_undepend_node(o2nm_this_node());
+}
+
 
 static void __exit exit_o2nm(void)
 {
diff --git a/fs/ocfs2/cluster/nodemanager.h b/fs/ocfs2/cluster/nodemanager.h
index 3490e77a952d5..39006005427b6 100644
--- a/fs/ocfs2/cluster/nodemanager.h
+++ b/fs/ocfs2/cluster/nodemanager.h
@@ -65,6 +65,8 @@ void o2nm_node_put(struct o2nm_node *node);
 
 int o2nm_depend_item(struct config_item *item);
 void o2nm_undepend_item(struct config_item *item);
+int o2nm_depend_node(u8 node_num);
+void o2nm_undepend_node(u8 node_num);
 int o2nm_depend_this_node(void);
 void o2nm_undepend_this_node(void);
 
diff --git a/fs/ocfs2/dir.c b/fs/ocfs2/dir.c
index 8e6b032383278..047d1b486f4a5 100644
--- a/fs/ocfs2/dir.c
+++ b/fs/ocfs2/dir.c
@@ -302,10 +302,11 @@ static int ocfs2_check_dir_entry(struct inode *dir,
 				 unsigned long offset)
 {
 	const char *error_msg = NULL;
+	unsigned long buf_offset = (char *)de - buf;
 	unsigned long next_offset;
 	int rlen;
 
-	if (offset > size - OCFS2_DIR_REC_LEN(1)) {
+	if (buf_offset > size || size - buf_offset < OCFS2_DIR_REC_LEN(1)) {
 		/* Dirent is (maybe partially) beyond the buffer
 		 * boundaries so touching 'de' members is unsafe.
 		 */
@@ -316,7 +317,7 @@ static int ocfs2_check_dir_entry(struct inode *dir,
 	}
 
 	rlen = le16_to_cpu(de->rec_len);
-	next_offset = ((char *) de - buf) + rlen;
+	next_offset = buf_offset + rlen;
 
 	if (unlikely(rlen < OCFS2_DIR_REC_LEN(1)))
 		error_msg = "rec_len is smaller than minimal";
@@ -1866,6 +1867,7 @@ static int ocfs2_dir_foreach_blk_el(struct inode *inode,
 	struct super_block * sb = inode->i_sb;
 	unsigned int ra_sectors = 16;
 	int stored = 0;
+	int ret;
 
 	bh = NULL;
 
@@ -1873,9 +1875,13 @@ static int ocfs2_dir_foreach_blk_el(struct inode *inode,
 
 	while (ctx->pos < i_size_read(inode)) {
 		blk = ctx->pos >> sb->s_blocksize_bits;
-		if (ocfs2_read_dir_block(inode, blk, &bh, 0)) {
+		ret = ocfs2_read_dir_block(inode, blk, &bh, 0);
+		if (ret) {
+			if (persist)
+				return ret;
 			/* Skip the corrupt dirblock and keep trying */
 			ctx->pos += sb->s_blocksize - offset;
+			offset = 0;
 			continue;
 		}
 
@@ -1969,8 +1975,7 @@ static int ocfs2_dir_foreach_blk(struct inode *inode, u64 *f_version,
 int ocfs2_dir_foreach(struct inode *inode, struct dir_context *ctx)
 {
 	u64 version = inode_query_iversion(inode);
-	ocfs2_dir_foreach_blk(inode, &version, ctx, true);
-	return 0;
+	return ocfs2_dir_foreach_blk(inode, &version, ctx, true);
 }
 
 /*
@@ -2167,7 +2172,7 @@ static int ocfs2_empty_dir_dx(struct inode *inode,
 /*
  * routine to check that the specified directory is empty (for rmdir)
  *
- * Returns 1 if dir is empty, zero otherwise.
+ * Returns 1 if dir is empty, zero if not, and a negative errno on error.
  *
  * XXX: This is a performance problem for unindexed directories.
  */
@@ -2180,8 +2185,10 @@ int ocfs2_empty_dir(struct inode *inode)
 
 	if (ocfs2_dir_indexed(inode)) {
 		ret = ocfs2_empty_dir_dx(inode, &priv);
-		if (ret)
+		if (ret) {
 			mlog_errno(ret);
+			return ret;
+		}
 		/*
 		 * We still run ocfs2_dir_foreach to get the checks
 		 * for "." and "..".
@@ -2189,8 +2196,10 @@ int ocfs2_empty_dir(struct inode *inode)
 	}
 
 	ret = ocfs2_dir_foreach(inode, &priv.ctx);
-	if (ret)
+	if (ret) {
 		mlog_errno(ret);
+		return ret;
+	}
 
 	if (!priv.seen_dot || !priv.seen_dot_dot) {
 		mlog(ML_ERROR, "bad directory (dir #%llu) - no `.' or `..'\n",
diff --git a/fs/ocfs2/dlm/dlmmaster.c b/fs/ocfs2/dlm/dlmmaster.c
index aee3b4c56dcc6..612969867ff92 100644
--- a/fs/ocfs2/dlm/dlmmaster.c
+++ b/fs/ocfs2/dlm/dlmmaster.c
@@ -3099,6 +3099,12 @@ int dlm_migrate_request_handler(struct o2net_msg *msg, u32 len, void *data,
 
 	name = migrate->name;
 	namelen = migrate->namelen;
+	if (namelen > DLM_LOCKID_NAME_MAX) {
+		mlog(ML_ERROR, "%s: invalid name length %u in migrate request\n",
+		     dlm->name, namelen);
+		ret = -EINVAL;
+		goto leave;
+	}
 	hash = dlm_lockid_hash(name, namelen);
 
 	/* preallocate.. if this fails, abort */
diff --git a/fs/ocfs2/dlm/dlmrecovery.c b/fs/ocfs2/dlm/dlmrecovery.c
index 9b97bf73df228..9d4a2695b9594 100644
--- a/fs/ocfs2/dlm/dlmrecovery.c
+++ b/fs/ocfs2/dlm/dlmrecovery.c
@@ -1357,6 +1357,15 @@ int dlm_mig_lockres_handler(struct o2net_msg *msg, u32 len, void *data,
 	if (!dlm_grab(dlm))
 		return -EINVAL;
 
+	if (mres->lockname_len > DLM_LOCKID_NAME_MAX ||
+	    mres->num_locks > DLM_MAX_MIGRATABLE_LOCKS ||
+	    be16_to_cpu(msg->data_len) < struct_size(mres, ml, mres->num_locks)) {
+		mlog(ML_ERROR, "%s: invalid lockres migration message from %u\n",
+		     dlm->name, mres->master);
+		dlm_put(dlm);
+		return -EINVAL;
+	}
+
 	if (!dlm_joined(dlm)) {
 		mlog(ML_ERROR, "Domain %s not joined! "
 			  "lockres %.*s, master %u\n",
diff --git a/fs/ocfs2/dlmfs/dlmfs.c b/fs/ocfs2/dlmfs/dlmfs.c
index 5821e33df78fd..53df5dd10ad05 100644
--- a/fs/ocfs2/dlmfs/dlmfs.c
+++ b/fs/ocfs2/dlmfs/dlmfs.c
@@ -422,7 +422,7 @@ static struct dentry *dlmfs_mkdir(struct mnt_idmap * idmap,
 		goto bail;
 	}
 
-	inode = dlmfs_get_inode(dir, dentry, mode | S_IFDIR);
+	inode = dlmfs_get_inode(dir, dentry, mode);
 	if (!inode) {
 		status = -ENOMEM;
 		mlog_errno(status);
@@ -453,8 +453,7 @@ static struct dentry *dlmfs_mkdir(struct mnt_idmap * idmap,
 static int dlmfs_create(struct mnt_idmap *idmap,
 			struct inode *dir,
 			struct dentry *dentry,
-			umode_t mode,
-			bool excl)
+			umode_t mode)
 {
 	int status = 0;
 	struct inode *inode;
diff --git a/fs/ocfs2/inode.c b/fs/ocfs2/inode.c
index 662dbc845b8bd..41db7dd39ed96 100644
--- a/fs/ocfs2/inode.c
+++ b/fs/ocfs2/inode.c
@@ -1608,6 +1608,10 @@ int ocfs2_validate_inode_block(struct super_block *sb,
 		goto bail;
 	}
 
+	rc = ocfs2_validate_inode_xattr(sb, bh->b_blocknr, di);
+	if (rc)
+		goto bail;
+
 	if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) {
 		struct ocfs2_inline_data *data = &di->id2.i_data;
 
@@ -1948,8 +1952,6 @@ int ocfs2_read_inode_block_full(struct inode *inode, struct buffer_head **bh,
 	rc = ocfs2_read_blocks(INODE_CACHE(inode), OCFS2_I(inode)->ip_blkno,
 			       1, &tmp, flags, ocfs2_validate_inode_block);
 
-	if (rc < 0)
-		make_bad_inode(inode);
 	/* If ocfs2_read_blocks() got us a new bh, pass it up. */
 	if (!rc && !*bh)
 		*bh = tmp;
diff --git a/fs/ocfs2/namei.c b/fs/ocfs2/namei.c
index 1277666c77cd2..10ce03e4557e5 100644
--- a/fs/ocfs2/namei.c
+++ b/fs/ocfs2/namei.c
@@ -657,7 +657,7 @@ static struct dentry *ocfs2_mkdir(struct mnt_idmap *idmap,
 
 	trace_ocfs2_mkdir(dir, dentry, dentry->d_name.len, dentry->d_name.name,
 			  OCFS2_I(dir)->ip_blkno, mode);
-	ret = ocfs2_mknod(&nop_mnt_idmap, dir, dentry, mode | S_IFDIR, 0);
+	ret = ocfs2_mknod(&nop_mnt_idmap, dir, dentry, mode, 0);
 	if (ret)
 		mlog_errno(ret);
 
@@ -667,8 +667,7 @@ static struct dentry *ocfs2_mkdir(struct mnt_idmap *idmap,
 static int ocfs2_create(struct mnt_idmap *idmap,
 			struct inode *dir,
 			struct dentry *dentry,
-			umode_t mode,
-			bool excl)
+			umode_t mode)
 {
 	int ret;
 
@@ -945,7 +944,10 @@ static int ocfs2_unlink(struct inode *dir,
 	child_locked = 1;
 
 	if (S_ISDIR(inode->i_mode)) {
-		if (inode->i_nlink != 2 || !ocfs2_empty_dir(inode)) {
+		status = ocfs2_empty_dir(inode);
+		if (status < 0)
+			goto leave;
+		if (inode->i_nlink != 2 || !status) {
 			status = -ENOTEMPTY;
 			goto leave;
 		}
@@ -1499,8 +1501,10 @@ static int ocfs2_rename(struct mnt_idmap *idmap,
 
 	if (target_exists) {
 		if (S_ISDIR(new_inode->i_mode)) {
-			if (new_inode->i_nlink != 2 ||
-			    !ocfs2_empty_dir(new_inode)) {
+			status = ocfs2_empty_dir(new_inode);
+			if (status < 0)
+				goto bail;
+			if (new_inode->i_nlink != 2 || !status) {
 				status = -ENOTEMPTY;
 				goto bail;
 			}
@@ -2126,7 +2130,7 @@ static int ocfs2_lookup_lock_orphan_dir(struct ocfs2_super *osb,
 		return ret;
 	}
 
-	inode_lock(orphan_dir_inode);
+	inode_lock_nested(orphan_dir_inode, I_MUTEX_NONDIR2);
 
 	ret = ocfs2_inode_lock(orphan_dir_inode, &orphan_dir_bh, 1);
 	if (ret < 0) {
@@ -2725,7 +2729,7 @@ int ocfs2_del_inode_from_orphan(struct ocfs2_super *osb,
 		goto bail;
 	}
 
-	inode_lock(orphan_dir_inode);
+	inode_lock_nested(orphan_dir_inode, I_MUTEX_NONDIR2);
 	status = ocfs2_inode_lock(orphan_dir_inode, &orphan_dir_bh, 1);
 	if (status < 0) {
 		inode_unlock(orphan_dir_inode);
@@ -2838,7 +2842,7 @@ int ocfs2_mv_orphaned_inode_to_new(struct inode *dir,
 		goto leave;
 	}
 
-	inode_lock(orphan_dir_inode);
+	inode_lock_nested(orphan_dir_inode, I_MUTEX_NONDIR2);
 
 	status = ocfs2_inode_lock(orphan_dir_inode, &orphan_dir_bh, 1);
 	if (status < 0) {
diff --git a/fs/ocfs2/refcounttree.c b/fs/ocfs2/refcounttree.c
index 7323bde70caaf..63d6cb326e302 100644
--- a/fs/ocfs2/refcounttree.c
+++ b/fs/ocfs2/refcounttree.c
@@ -116,6 +116,33 @@ static int ocfs2_validate_refcount_block(struct super_block *sb,
 				 le32_to_cpu(rb->rf_fs_generation));
 		goto out;
 	}
+
+	/*
+	 * rf_records (rl_count/rl_used/rl_recs[]) is only meaningful when
+	 * this block is not an interior tree block (OCFS2_REFCOUNT_TREE_FL);
+	 * in that case the same union bytes hold an extent list (rf_list)
+	 * instead, which is validated by ocfs2_validate_extent_block().
+	 */
+	if (!(le32_to_cpu(rb->rf_flags) & OCFS2_REFCOUNT_TREE_FL)) {
+		if (le16_to_cpu(rb->rf_records.rl_count) !=
+		    ocfs2_refcount_recs_per_rb(sb)) {
+			rc = ocfs2_error(sb,
+					 "Refcount block #%llu has an invalid rl_count of %u\n",
+					 (unsigned long long)bh->b_blocknr,
+					 le16_to_cpu(rb->rf_records.rl_count));
+			goto out;
+		}
+
+		if (le16_to_cpu(rb->rf_records.rl_used) >
+		    le16_to_cpu(rb->rf_records.rl_count)) {
+			rc = ocfs2_error(sb,
+					 "Refcount block #%llu has an invalid rl_used of %u (rl_count %u)\n",
+					 (unsigned long long)bh->b_blocknr,
+					 le16_to_cpu(rb->rf_records.rl_used),
+					 le16_to_cpu(rb->rf_records.rl_count));
+			goto out;
+		}
+	}
 out:
 	return rc;
 }
diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c
index bcac614ff02a8..c62e389d4dd65 100644
--- a/fs/ocfs2/super.c
+++ b/fs/ocfs2/super.c
@@ -1882,7 +1882,6 @@ static void ocfs2_dismount_volume(struct super_block *sb, int mnt_err)
 
 	ocfs2_delete_osb(osb);
 	kfree(osb);
-	sb->s_dev = 0;
 	sb->s_fs_info = NULL;
 }
 
diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c
index fcddd3c13acdd..d23364c30e1a7 100644
--- a/fs/ocfs2/xattr.c
+++ b/fs/ocfs2/xattr.c
@@ -390,6 +390,12 @@ static int ocfs2_init_xattr_bucket(struct ocfs2_xattr_bucket *bucket,
 	return rc;
 }
 
+static int ocfs2_validate_xattr_entries_flat(struct super_block *sb, u64 blkno,
+					     struct ocfs2_xattr_header *xh,
+					     size_t region_size);
+static int ocfs2_validate_xattr_bucket(struct ocfs2_xattr_bucket *bucket,
+				       u64 blkno);
+
 /* Read the xattr bucket at xb_blkno */
 static int ocfs2_read_xattr_bucket(struct ocfs2_xattr_bucket *bucket,
 				   u64 xb_blkno)
@@ -408,6 +414,8 @@ static int ocfs2_read_xattr_bucket(struct ocfs2_xattr_bucket *bucket,
 		spin_unlock(&OCFS2_SB(bucket->bu_inode->i_sb)->osb_xattr_lock);
 		if (rc)
 			mlog_errno(rc);
+		else
+			rc = ocfs2_validate_xattr_bucket(bucket, xb_blkno);
 	}
 
 	if (rc)
@@ -509,6 +517,22 @@ static int ocfs2_validate_xattr_block(struct super_block *sb,
 				   le32_to_cpu(xb->xb_fs_generation));
 	}
 
+	if (!(le16_to_cpu(xb->xb_flags) & OCFS2_XATTR_INDEXED)) {
+		size_t region_offset =
+			offsetof(struct ocfs2_xattr_block, xb_attrs.xb_header);
+
+		if (bh->b_size < region_offset)
+			return ocfs2_error(sb,
+					   "Invalid xattr block %llu: block size %zu is too small\n",
+					   (unsigned long long)bh->b_blocknr,
+					   bh->b_size);
+
+		return ocfs2_validate_xattr_entries_flat(sb, bh->b_blocknr,
+							 &xb->xb_attrs.xb_header,
+							 bh->b_size -
+							 region_offset);
+	}
+
 	return 0;
 }
 
@@ -950,41 +974,206 @@ static int ocfs2_xattr_list_entries(struct inode *inode,
 	return result;
 }
 
-static int ocfs2_xattr_ibody_lookup_header(struct inode *inode,
-					   struct ocfs2_dinode *di,
-					   struct ocfs2_xattr_header **header)
+static int ocfs2_validate_xattr_entries_flat(struct super_block *sb, u64 blkno,
+					     struct ocfs2_xattr_header *xh,
+					     size_t region_size)
+{
+	u16 xattr_count = le16_to_cpu(xh->xh_count);
+	size_t entries_limit = region_size;
+	size_t nv_limit = region_size;
+	size_t max_entries;
+	int i;
+
+	if (region_size < sizeof(*xh))
+		return ocfs2_error(sb,
+				   "Invalid xattr in block %llu: region size %zu is too small\n",
+				   (unsigned long long)blkno, region_size);
+
+	max_entries = (entries_limit - sizeof(*xh)) /
+		      sizeof(struct ocfs2_xattr_entry);
+
+	if (xattr_count > max_entries)
+		return ocfs2_error(sb,
+				   "Invalid xattr in block %llu: entry count %u exceeds maximum %zu\n",
+				   (unsigned long long)blkno,
+				   xattr_count, max_entries);
+
+	for (i = 0; i < xattr_count; i++) {
+		struct ocfs2_xattr_entry *xe = &xh->xh_entries[i];
+		size_t name_offset = le16_to_cpu(xe->xe_name_offset);
+		size_t value_offset;
+
+		if (name_offset > nv_limit ||
+		    xe->xe_name_len > nv_limit - name_offset)
+			return ocfs2_error(sb,
+					   "Invalid xattr in block %llu: entry %d name is out of bounds\n",
+					   (unsigned long long)blkno, i);
+
+		value_offset = name_offset + OCFS2_XATTR_SIZE(xe->xe_name_len);
+		if (value_offset > nv_limit)
+			return ocfs2_error(sb,
+					   "Invalid xattr in block %llu: entry %d value starts out of bounds\n",
+					   (unsigned long long)blkno, i);
+
+		if (ocfs2_xattr_is_local(xe)) {
+			if (le64_to_cpu(xe->xe_value_size) >
+			    nv_limit - value_offset)
+				return ocfs2_error(sb,
+						   "Invalid xattr in block %llu: entry %d value is out of bounds\n",
+						   (unsigned long long)blkno,
+						   i);
+		} else if (sizeof(struct ocfs2_xattr_value_root) >
+			   nv_limit - value_offset) {
+			return ocfs2_error(sb,
+					   "Invalid xattr in block %llu: entry %d value root is out of bounds\n",
+					   (unsigned long long)blkno, i);
+		}
+	}
+
+	return 0;
+}
+
+static int ocfs2_xattr_ibody_lookup_header_raw(struct super_block *sb,
+					       u64 blkno,
+					       struct ocfs2_dinode *di,
+					       struct ocfs2_xattr_header **header,
+					       u16 *inline_size_ret)
 {
+	struct ocfs2_xattr_header *xh;
 	u16 xattr_count;
 	size_t max_entries;
 	u16 inline_size = le16_to_cpu(di->i_xattr_inline_size);
 
-	if (inline_size > inode->i_sb->s_blocksize ||
+	if (inline_size > sb->s_blocksize ||
 	    inline_size < sizeof(struct ocfs2_xattr_header)) {
-		ocfs2_error(inode->i_sb,
-			    "Invalid xattr inline size %u in inode %llu\n",
-			    inline_size,
-			    (unsigned long long)OCFS2_I(inode)->ip_blkno);
+		ocfs2_error(sb,
+			    "Invalid inode %llu: xattr inline size %u\n",
+			    (unsigned long long)blkno, inline_size);
 		return -EFSCORRUPTED;
 	}
 
-	*header = (struct ocfs2_xattr_header *)
-		((void *)di + inode->i_sb->s_blocksize - inline_size);
+	xh = (struct ocfs2_xattr_header *)
+		((void *)di + sb->s_blocksize - inline_size);
 
-	xattr_count = le16_to_cpu((*header)->xh_count);
+	xattr_count = le16_to_cpu(xh->xh_count);
 	max_entries = (inline_size - sizeof(struct ocfs2_xattr_header)) /
 		      sizeof(struct ocfs2_xattr_entry);
 
 	if (xattr_count > max_entries) {
-		ocfs2_error(inode->i_sb,
+		ocfs2_error(sb,
 			    "xattr entry count %u exceeds maximum %zu in inode %llu\n",
 			    xattr_count, max_entries,
-			    (unsigned long long)OCFS2_I(inode)->ip_blkno);
+			    (unsigned long long)blkno);
 		return -EFSCORRUPTED;
 	}
 
+	*header = xh;
+	if (inline_size_ret)
+		*inline_size_ret = inline_size;
+
+	return 0;
+}
+
+int ocfs2_validate_inode_xattr(struct super_block *sb, u64 blkno,
+			       struct ocfs2_dinode *di)
+{
+	struct ocfs2_xattr_header *xh;
+	u16 inline_size;
+	int ret;
+
+	if (!(le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_XATTR_FL))
+		return 0;
+
+	ret = ocfs2_xattr_ibody_lookup_header_raw(sb, blkno, di, &xh,
+						  &inline_size);
+	if (ret)
+		return ret;
+
+	return ocfs2_validate_xattr_entries_flat(sb, blkno, xh, inline_size);
+}
+
+static int ocfs2_validate_xattr_bucket(struct ocfs2_xattr_bucket *bucket,
+				       u64 blkno)
+{
+	struct super_block *sb = bucket->bu_inode->i_sb;
+	struct ocfs2_xattr_header *xh = bucket_xh(bucket);
+	u16 xattr_count = le16_to_cpu(xh->xh_count);
+	size_t region_size = (size_t)sb->s_blocksize * bucket->bu_blocks;
+	size_t entries_limit = sb->s_blocksize;
+	size_t nv_limit = sb->s_blocksize;
+	size_t max_entries;
+	int i;
+
+	if (region_size < sizeof(*xh))
+		return ocfs2_error(sb,
+				   "Invalid xattr bucket %llu: region size %zu is too small\n",
+				   (unsigned long long)blkno, region_size);
+
+	if (entries_limit < sizeof(*xh))
+		return ocfs2_error(sb,
+				   "Invalid xattr bucket %llu: entries limit %zu is too small\n",
+				   (unsigned long long)blkno,
+				   entries_limit);
+
+	max_entries = (entries_limit - sizeof(*xh)) /
+		      sizeof(struct ocfs2_xattr_entry);
+
+	if (xattr_count > max_entries)
+		return ocfs2_error(sb,
+				   "Invalid xattr bucket %llu: entry count %u exceeds maximum %zu\n",
+				   (unsigned long long)blkno,
+				   xattr_count, max_entries);
+
+	for (i = 0; i < xattr_count; i++) {
+		struct ocfs2_xattr_entry *xe = &xh->xh_entries[i];
+		size_t name_offset = le16_to_cpu(xe->xe_name_offset);
+		size_t block_off = name_offset >> sb->s_blocksize_bits;
+		size_t block_offset = name_offset % nv_limit;
+		size_t value_offset;
+
+		if (name_offset >= region_size || block_off >= bucket->bu_blocks)
+			return ocfs2_error(sb,
+					   "Invalid xattr bucket %llu: entry %d name is out of bounds\n",
+					   (unsigned long long)blkno, i);
+
+		if (xe->xe_name_len > nv_limit - block_offset)
+			return ocfs2_error(sb,
+					   "Invalid xattr bucket %llu: entry %d name crosses block boundary\n",
+					   (unsigned long long)blkno, i);
+
+		value_offset = block_offset + OCFS2_XATTR_SIZE(xe->xe_name_len);
+		if (value_offset > nv_limit)
+			return ocfs2_error(sb,
+					   "Invalid xattr bucket %llu: entry %d value starts out of bounds\n",
+					   (unsigned long long)blkno, i);
+
+		if (ocfs2_xattr_is_local(xe)) {
+			if (le64_to_cpu(xe->xe_value_size) >
+			    nv_limit - value_offset)
+				return ocfs2_error(sb,
+						   "Invalid xattr bucket %llu: entry %d value is out of bounds\n",
+						   (unsigned long long)blkno,
+						   i);
+		} else if (sizeof(struct ocfs2_xattr_value_root) >
+			   nv_limit - value_offset) {
+			return ocfs2_error(sb,
+					   "Invalid xattr bucket %llu: entry %d value root is out of bounds\n",
+					   (unsigned long long)blkno, i);
+		}
+	}
+
 	return 0;
 }
 
+static int ocfs2_xattr_ibody_lookup_header(struct inode *inode,
+					   struct ocfs2_dinode *di,
+					   struct ocfs2_xattr_header **header)
+{
+	return ocfs2_xattr_ibody_lookup_header_raw(inode->i_sb,
+						   OCFS2_I(inode)->ip_blkno,
+						   di, header, NULL);
+}
+
 int ocfs2_has_inline_xattr_value_outside(struct inode *inode,
 					 struct ocfs2_dinode *di)
 {
diff --git a/fs/ocfs2/xattr.h b/fs/ocfs2/xattr.h
index 65e9aa7439199..6b7589941315e 100644
--- a/fs/ocfs2/xattr.h
+++ b/fs/ocfs2/xattr.h
@@ -43,6 +43,8 @@ int ocfs2_xattr_set_handle(handle_t *, struct inode *, struct buffer_head *,
 			   struct ocfs2_alloc_context *);
 int ocfs2_has_inline_xattr_value_outside(struct inode *inode,
 					 struct ocfs2_dinode *di);
+int ocfs2_validate_inode_xattr(struct super_block *sb, u64 blkno,
+			       struct ocfs2_dinode *di);
 int ocfs2_xattr_remove(struct inode *, struct buffer_head *);
 int ocfs2_init_security_get(struct inode *, struct inode *,
 			    const struct qstr *,
diff --git a/fs/omfs/dir.c b/fs/omfs/dir.c
index 2ed541fccf331..692297cf84e74 100644
--- a/fs/omfs/dir.c
+++ b/fs/omfs/dir.c
@@ -282,11 +282,11 @@ static int omfs_add_node(struct inode *dir, struct dentry *dentry, umode_t mode)
 static struct dentry *omfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 				 struct dentry *dentry, umode_t mode)
 {
-	return ERR_PTR(omfs_add_node(dir, dentry, mode | S_IFDIR));
+	return ERR_PTR(omfs_add_node(dir, dentry, mode));
 }
 
 static int omfs_create(struct mnt_idmap *idmap, struct inode *dir,
-		       struct dentry *dentry, umode_t mode, bool excl)
+		       struct dentry *dentry, umode_t mode)
 {
 	return omfs_add_node(dir, dentry, mode | S_IFREG);
 }
diff --git a/fs/orangefs/namei.c b/fs/orangefs/namei.c
index 75e65e72c2d64..8ebc34e112d51 100644
--- a/fs/orangefs/namei.c
+++ b/fs/orangefs/namei.c
@@ -18,8 +18,7 @@
 static int orangefs_create(struct mnt_idmap *idmap,
 			struct inode *dir,
 			struct dentry *dentry,
-			umode_t mode,
-			bool exclusive)
+			umode_t mode)
 {
 	struct orangefs_inode_s *parent = ORANGEFS_I(dir);
 	struct orangefs_kernel_op_s *new_op;
@@ -333,7 +332,7 @@ static struct dentry *orangefs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 
 	ref = new_op->downcall.resp.mkdir.refn;
 
-	inode = orangefs_new_inode(dir->i_sb, dir, S_IFDIR | mode, 0, &ref);
+	inode = orangefs_new_inode(dir->i_sb, dir, mode, 0, &ref);
 	if (IS_ERR(inode)) {
 		gossip_err("*** Failed to allocate orangefs dir inode\n");
 		ret = PTR_ERR(inode);
diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c
index a033743dbf515..610c0116252c1 100644
--- a/fs/overlayfs/dir.c
+++ b/fs/overlayfs/dir.c
@@ -689,8 +689,8 @@ static int ovl_create_or_link(struct dentry *dentry, struct inode *inode,
 	return err;
 }
 
-static int ovl_create_object(struct dentry *dentry, int mode, dev_t rdev,
-			     const char *link)
+static int ovl_create_object(struct mnt_idmap *idmap, struct dentry *dentry,
+			     int mode, dev_t rdev, const char *link)
 {
 	int err;
 	struct inode *inode;
@@ -717,7 +717,7 @@ static int ovl_create_object(struct dentry *dentry, int mode, dev_t rdev,
 	inode_state_set(inode, I_CREATING);
 	spin_unlock(&inode->i_lock);
 
-	inode_init_owner(&nop_mnt_idmap, inode, dentry->d_parent->d_inode, mode);
+	inode_init_owner(idmap, inode, dentry->d_parent->d_inode, mode);
 	attr.mode = inode->i_mode;
 
 	err = ovl_create_or_link(dentry, inode, &attr, false);
@@ -732,15 +732,15 @@ static int ovl_create_object(struct dentry *dentry, int mode, dev_t rdev,
 }
 
 static int ovl_create(struct mnt_idmap *idmap, struct inode *dir,
-		      struct dentry *dentry, umode_t mode, bool excl)
+		      struct dentry *dentry, umode_t mode)
 {
-	return ovl_create_object(dentry, (mode & 07777) | S_IFREG, 0, NULL);
+	return ovl_create_object(idmap, dentry, (mode & 07777) | S_IFREG, 0, NULL);
 }
 
 static struct dentry *ovl_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 				struct dentry *dentry, umode_t mode)
 {
-	return ERR_PTR(ovl_create_object(dentry, (mode & 07777) | S_IFDIR, 0, NULL));
+	return ERR_PTR(ovl_create_object(idmap, dentry, (mode & 07777) | S_IFDIR, 0, NULL));
 }
 
 static int ovl_mknod(struct mnt_idmap *idmap, struct inode *dir,
@@ -750,13 +750,13 @@ static int ovl_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	if (S_ISCHR(mode) && rdev == WHITEOUT_DEV)
 		return -EPERM;
 
-	return ovl_create_object(dentry, mode, rdev, NULL);
+	return ovl_create_object(idmap, dentry, mode, rdev, NULL);
 }
 
 static int ovl_symlink(struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, const char *link)
 {
-	return ovl_create_object(dentry, S_IFLNK, 0, link);
+	return ovl_create_object(idmap, dentry, S_IFLNK, 0, link);
 }
 
 static int ovl_set_link_redirect(struct dentry *dentry)
@@ -1444,7 +1444,7 @@ static int ovl_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
 	if (!inode)
 		goto drop_write;
 
-	inode_init_owner(&nop_mnt_idmap, inode, dir, mode);
+	inode_init_owner(idmap, inode, dir, mode);
 	err = ovl_create_tmpfile(file, dentry, inode, inode->i_mode);
 	if (err)
 		goto put_inode;
diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c
index bc71231cad532..401cb8c75520c 100644
--- a/fs/overlayfs/inode.c
+++ b/fs/overlayfs/inode.c
@@ -26,10 +26,18 @@ int ovl_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 	bool full_copy_up = false;
 	struct dentry *upperdentry;
 
-	err = setattr_prepare(&nop_mnt_idmap, dentry, attr);
+	err = setattr_prepare(idmap, dentry, attr);
 	if (err)
 		return err;
 
+	/* Rebase ownership from the mount idmap into overlay id space. */
+	if (attr->ia_valid & ATTR_UID)
+		attr->ia_vfsuid = VFSUIDT_INIT(from_vfsuid(idmap,
+				i_user_ns(d_inode(dentry)), attr->ia_vfsuid));
+	if (attr->ia_valid & ATTR_GID)
+		attr->ia_vfsgid = VFSGIDT_INIT(from_vfsgid(idmap,
+				i_user_ns(d_inode(dentry)), attr->ia_vfsgid));
+
 	if (attr->ia_valid & ATTR_SIZE) {
 		/* Truncate should trigger data copy up as well */
 		full_copy_up = true;
@@ -172,6 +180,8 @@ int ovl_getattr(struct mnt_idmap *idmap, const struct path *path,
 	int fsid = 0;
 	int err;
 	bool metacopy_blocks = false;
+	vfsuid_t vfsuid;
+	vfsgid_t vfsgid;
 
 	metacopy_blocks = ovl_is_metacopy_dentry(dentry);
 
@@ -284,6 +294,12 @@ int ovl_getattr(struct mnt_idmap *idmap, const struct path *path,
 	if (!is_dir && ovl_test_flag(OVL_INDEX, d_inode(dentry)))
 		stat->nlink = dentry->d_inode->i_nlink;
 
+	/* Map ownership of the real inode through the overlay mount idmap. */
+	vfsuid = make_vfsuid(idmap, i_user_ns(inode), stat->uid);
+	vfsgid = make_vfsgid(idmap, i_user_ns(inode), stat->gid);
+	stat->uid = vfsuid_into_kuid(vfsuid);
+	stat->gid = vfsgid_into_kgid(vfsgid);
+
 	return err;
 }
 
@@ -306,7 +322,7 @@ int ovl_permission(struct mnt_idmap *idmap,
 	 * Check overlay inode with the creds of task and underlying inode
 	 * with creds of mounter
 	 */
-	err = generic_permission(&nop_mnt_idmap, inode, mask);
+	err = generic_permission(idmap, inode, mask);
 	if (err)
 		return err;
 
@@ -534,7 +550,7 @@ int ovl_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
 		return -EOPNOTSUPP;
 	if (type == ACL_TYPE_DEFAULT && !S_ISDIR(inode->i_mode))
 		return acl ? -EACCES : 0;
-	if (!inode_owner_or_capable(&nop_mnt_idmap, inode))
+	if (!inode_owner_or_capable(idmap, inode))
 		return -EPERM;
 
 	/*
@@ -542,8 +558,8 @@ int ovl_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
 	 * be done with mounter's capabilities and so that won't do it for us).
 	 */
 	if (unlikely(inode->i_mode & S_ISGID) && type == ACL_TYPE_ACCESS &&
-	    !in_group_p(inode->i_gid) &&
-	    !capable_wrt_inode_uidgid(&nop_mnt_idmap, inode, CAP_FSETID)) {
+	    !in_group_or_capable(idmap, inode,
+				 i_gid_into_vfsgid(idmap, inode))) {
 		struct iattr iattr = { .ia_valid = ATTR_KILL_SGID };
 
 		err = ovl_setattr(&nop_mnt_idmap, dentry, &iattr);
diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h
index b75df37f70ac3..e0d8c6152e9fc 100644
--- a/fs/overlayfs/overlayfs.h
+++ b/fs/overlayfs/overlayfs.h
@@ -320,6 +320,7 @@ static inline int ovl_do_setxattr(struct ovl_fs *ofs, struct dentry *dentry,
 				  const char *name, const void *value,
 				  size_t size, int flags)
 {
+	/* Use vfs_setxattr(), not __vfs_setxattr(): it idmaps the security.capability rootid. */
 	int err = vfs_setxattr(ovl_upper_mnt_idmap(ofs), dentry, name,
 			       value, size, flags);
 
diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c
index 60f0b7ceef0a6..f2889cf9bc07b 100644
--- a/fs/overlayfs/super.c
+++ b/fs/overlayfs/super.c
@@ -1573,7 +1573,7 @@ struct file_system_type ovl_fs_type = {
 	.name			= "overlay",
 	.init_fs_context	= ovl_init_fs_context,
 	.parameters		= ovl_parameter_spec,
-	.fs_flags		= FS_USERNS_MOUNT,
+	.fs_flags		= FS_USERNS_MOUNT | FS_ALLOW_IDMAP,
 	.kill_sb		= kill_anon_super,
 };
 MODULE_ALIAS_FS("overlay");
diff --git a/fs/overlayfs/xattrs.c b/fs/overlayfs/xattrs.c
index aa95855c70237..811c94d2d9e98 100644
--- a/fs/overlayfs/xattrs.c
+++ b/fs/overlayfs/xattrs.c
@@ -84,6 +84,7 @@ static int ovl_xattr_get(struct dentry *dentry, struct inode *inode, const char
 	struct path realpath;
 
 	ovl_i_path_real(inode, &realpath);
+	/* Use vfs_getxattr(), not __vfs_getxattr(): it idmaps the security.capability rootid. */
 	with_ovl_creds(dentry->d_sb)
 		return vfs_getxattr(mnt_idmap(realpath.mnt), realpath.dentry, name, value, size);
 }
diff --git a/fs/pipe.c b/fs/pipe.c
index 429b0714ec575..8f98ca864cb4b 100644
--- a/fs/pipe.c
+++ b/fs/pipe.c
@@ -1589,7 +1589,7 @@ static int u2k_pipe_maxsz(const ulong *u_ptr, uint *k_ptr)
 	return proc_uint_u2k_conv_uop(u_ptr, k_ptr, round_pipe_size_ul);
 }
 
-static int do_proc_uint_conv_pipe_maxsz(ulong *u_ptr, uint *k_ptr,
+static int do_proc_uint_conv_pipe_maxsz(bool *negp, ulong *u_ptr, uint *k_ptr,
 					int dir, const struct ctl_table *table)
 {
 	return proc_uint_conv(u_ptr, k_ptr, dir, table, true,
diff --git a/fs/posix_acl.c b/fs/posix_acl.c
index b4bfe4ddf64ea..6df5de23aac42 100644
--- a/fs/posix_acl.c
+++ b/fs/posix_acl.c
@@ -740,8 +740,6 @@ static int posix_acl_fix_xattr_common(const void *value, size_t size)
 	count = posix_acl_xattr_count(size);
 	if (count < 0)
 		return -EINVAL;
-	if (count == 0)
-		return 0;
 
 	return count;
 }
diff --git a/fs/proc/array.c b/fs/proc/array.c
index 479ea8cb4ef43..f6f75d206762a 100644
--- a/fs/proc/array.c
+++ b/fs/proc/array.c
@@ -168,8 +168,8 @@ static inline void task_state(struct seq_file *m, struct pid_namespace *ns,
 	cred = get_task_cred(p);
 
 	task_lock(p);
-	if (p->fs)
-		umask = p->fs->umask;
+	if (p->real_fs)
+		umask = p->real_fs->umask;
 	if (p->files)
 		max_fds = files_fdtable(p->files)->max_fds;
 	task_unlock(p);
diff --git a/fs/proc/base.c b/fs/proc/base.c
index 780f812590527..6a39de424f62a 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -211,8 +211,8 @@ static int get_task_root(struct task_struct *task, struct path *root)
 	int result = -ENOENT;
 
 	task_lock(task);
-	if (task->fs) {
-		get_fs_root(task->fs, root);
+	if (task->real_fs) {
+		get_fs_root(task->real_fs, root);
 		result = 0;
 	}
 	task_unlock(task);
@@ -225,8 +225,8 @@ static int proc_cwd_link(struct dentry *dentry, struct path *path,
 	int result = -ENOENT;
 
 	task_lock(task);
-	if (task->fs) {
-		get_fs_pwd(task->fs, path);
+	if (task->real_fs) {
+		get_fs_pwd(task->real_fs, path);
 		result = 0;
 	}
 	task_unlock(task);
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index d32408f7cd5ed..817e3e0f91943 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -1237,6 +1237,9 @@ static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
 		[ilog2(VM_UFFD_MINOR)]	= "ui",
 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */
+#ifdef CONFIG_USERFAULTFD_RWP
+		[ilog2(VM_UFFD_RWP)]	= "ur",
+#endif
 #ifdef CONFIG_ARCH_HAS_USER_SHADOW_STACK
 		[ilog2(VM_SHADOW_STACK)] = "ss",
 #endif
@@ -2035,14 +2038,14 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
 		page = vm_normal_page(vma, addr, pte);
 		if (pte_soft_dirty(pte))
 			flags |= PM_SOFT_DIRTY;
-		if (pte_uffd_wp(pte))
+		if (pte_uffd(pte))
 			flags |= PM_UFFD_WP;
 	} else {
 		softleaf_t entry;
 
 		if (pte_swp_soft_dirty(pte))
 			flags |= PM_SOFT_DIRTY;
-		if (pte_swp_uffd_wp(pte))
+		if (pte_swp_uffd(pte))
 			flags |= PM_UFFD_WP;
 		entry = softleaf_from_pte(pte);
 		if (pm->show_pfn) {
@@ -2108,7 +2111,7 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr,
 		flags |= PM_PRESENT;
 		if (pmd_soft_dirty(pmd))
 			flags |= PM_SOFT_DIRTY;
-		if (pmd_uffd_wp(pmd))
+		if (pmd_uffd(pmd))
 			flags |= PM_UFFD_WP;
 		if (pm->show_pfn)
 			frame = pmd_pfn(pmd) + idx;
@@ -2127,9 +2130,10 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr,
 		flags |= PM_SWAP;
 		if (pmd_swp_soft_dirty(pmd))
 			flags |= PM_SOFT_DIRTY;
-		if (pmd_swp_uffd_wp(pmd))
+		if (pmd_swp_uffd(pmd))
 			flags |= PM_UFFD_WP;
-		page = softleaf_to_page(entry);
+		if (softleaf_has_pfn(entry))
+			page = softleaf_to_page(entry);
 	}
 
 	if (page) {
@@ -2232,14 +2236,14 @@ static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask,
 		    !hugetlb_pmd_shared(ptep))
 			flags |= PM_MMAP_EXCLUSIVE;
 
-		if (huge_pte_uffd_wp(pte))
+		if (huge_pte_uffd(pte))
 			flags |= PM_UFFD_WP;
 
 		flags |= PM_PRESENT;
 		if (pm->show_pfn)
 			frame = pte_pfn(pte) +
 				((addr & ~hmask) >> PAGE_SHIFT);
-	} else if (pte_swp_uffd_wp_any(pte)) {
+	} else if (pte_swp_uffd_any(pte)) {
 		flags |= PM_UFFD_WP;
 	}
 
@@ -2280,7 +2284,7 @@ static const struct mm_walk_ops pagemap_ops = {
  * Bits 5-54  swap offset if swapped
  * Bit  55    pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst)
  * Bit  56    page exclusively mapped
- * Bit  57    pte is uffd-wp write-protected
+ * Bit  57    pte is tracked by userfaultfd (uffd-wp or RWP)
  * Bit  58    pte is a guard region
  * Bits 59-60 zero
  * Bit  61    page is file-page or shared-anon
@@ -2415,7 +2419,7 @@ static int pagemap_release(struct inode *inode, struct file *file)
 				 PAGE_IS_FILE |	PAGE_IS_PRESENT |	\
 				 PAGE_IS_SWAPPED | PAGE_IS_PFNZERO |	\
 				 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY |	\
-				 PAGE_IS_GUARD)
+				 PAGE_IS_GUARD | PAGE_IS_ACCESSED)
 #define PM_SCAN_FLAGS		(PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC)
 
 struct pagemap_scan_private {
@@ -2432,16 +2436,32 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p,
 {
 	unsigned long categories;
 
-	if (pte_none(pte))
+	if (pte_none(pte)) {
+		/*
+		 * An unpopulated pte carries no uffd bit, i.e. it is not
+		 * write-protected. The PAGE_IS_WRITTEN fast path in
+		 * pagemap_scan_pmd_entry() is now gated on a VM_UFFD_WP VMA;
+		 * gate the pte_none report here the same way so the two paths
+		 * still agree. RWP has no such fast path and an unpopulated
+		 * page is not part of the RWP working set, so it is reported as
+		 * neither.
+		 */
+		if (userfaultfd_wp(vma))
+			return PAGE_IS_WRITTEN;
 		return 0;
+	}
 
 	if (pte_present(pte)) {
 		struct page *page;
 
 		categories = PAGE_IS_PRESENT;
 
-		if (!pte_uffd_wp(pte))
-			categories |= PAGE_IS_WRITTEN;
+		if (!pte_uffd(pte)) {
+			if (userfaultfd_wp(vma))
+				categories |= PAGE_IS_WRITTEN;
+			if (userfaultfd_rwp(vma))
+				categories |= PAGE_IS_ACCESSED;
+		}
 
 		if (p->masks_of_interest & PAGE_IS_FILE) {
 			page = vm_normal_page(vma, addr, pte);
@@ -2458,8 +2478,12 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p,
 
 		categories = PAGE_IS_SWAPPED;
 
-		if (!pte_swp_uffd_wp_any(pte))
-			categories |= PAGE_IS_WRITTEN;
+		if (!pte_swp_uffd_any(pte)) {
+			if (userfaultfd_wp(vma))
+				categories |= PAGE_IS_WRITTEN;
+			if (userfaultfd_rwp(vma))
+				categories |= PAGE_IS_ACCESSED;
+		}
 
 		entry = softleaf_from_pte(pte);
 		if (softleaf_is_guard_marker(entry))
@@ -2483,13 +2507,13 @@ static void make_uffd_wp_pte(struct vm_area_struct *vma,
 		pte_t old_pte;
 
 		old_pte = ptep_modify_prot_start(vma, addr, pte);
-		ptent = pte_mkuffd_wp(old_pte);
+		ptent = pte_mkuffd(old_pte);
 		ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent);
 	} else if (pte_none(ptent)) {
 		set_pte_at(vma->vm_mm, addr, pte,
 			   make_pte_marker(PTE_MARKER_UFFD_WP));
 	} else {
-		ptent = pte_swp_mkuffd_wp(ptent);
+		ptent = pte_swp_mkuffd(ptent);
 		set_pte_at(vma->vm_mm, addr, pte, ptent);
 	}
 }
@@ -2508,8 +2532,12 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p,
 		struct page *page;
 
 		categories |= PAGE_IS_PRESENT;
-		if (!pmd_uffd_wp(pmd))
-			categories |= PAGE_IS_WRITTEN;
+		if (!pmd_uffd(pmd)) {
+			if (userfaultfd_wp(vma))
+				categories |= PAGE_IS_WRITTEN;
+			if (userfaultfd_rwp(vma))
+				categories |= PAGE_IS_ACCESSED;
+		}
 
 		if (p->masks_of_interest & PAGE_IS_FILE) {
 			page = vm_normal_page_pmd(vma, addr, pmd);
@@ -2523,8 +2551,12 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p,
 			categories |= PAGE_IS_SOFT_DIRTY;
 	} else {
 		categories |= PAGE_IS_SWAPPED;
-		if (!pmd_swp_uffd_wp(pmd))
-			categories |= PAGE_IS_WRITTEN;
+		if (!pmd_swp_uffd(pmd)) {
+			if (userfaultfd_wp(vma))
+				categories |= PAGE_IS_WRITTEN;
+			if (userfaultfd_rwp(vma))
+				categories |= PAGE_IS_ACCESSED;
+		}
 		if (pmd_swp_soft_dirty(pmd))
 			categories |= PAGE_IS_SOFT_DIRTY;
 
@@ -2547,17 +2579,18 @@ static void make_uffd_wp_pmd(struct vm_area_struct *vma,
 
 	if (pmd_present(pmd)) {
 		old = pmdp_invalidate_ad(vma, addr, pmdp);
-		pmd = pmd_mkuffd_wp(old);
+		pmd = pmd_mkuffd(old);
 		set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
 	} else if (pmd_is_migration_entry(pmd)) {
-		pmd = pmd_swp_mkuffd_wp(pmd);
+		pmd = pmd_swp_mkuffd(pmd);
 		set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
 	}
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
 #ifdef CONFIG_HUGETLB_PAGE
-static unsigned long pagemap_hugetlb_category(pte_t pte)
+static unsigned long pagemap_hugetlb_category(struct vm_area_struct *vma,
+					      pte_t pte)
 {
 	unsigned long categories = PAGE_IS_HUGE;
 
@@ -2572,8 +2605,12 @@ static unsigned long pagemap_hugetlb_category(pte_t pte)
 	if (pte_present(pte)) {
 		categories |= PAGE_IS_PRESENT;
 
-		if (!huge_pte_uffd_wp(pte))
-			categories |= PAGE_IS_WRITTEN;
+		if (!huge_pte_uffd(pte)) {
+			if (userfaultfd_wp(vma))
+				categories |= PAGE_IS_WRITTEN;
+			if (userfaultfd_rwp(vma))
+				categories |= PAGE_IS_ACCESSED;
+		}
 		if (!PageAnon(pte_page(pte)))
 			categories |= PAGE_IS_FILE;
 		if (is_zero_pfn(pte_pfn(pte)))
@@ -2583,8 +2620,12 @@ static unsigned long pagemap_hugetlb_category(pte_t pte)
 	} else {
 		categories |= PAGE_IS_SWAPPED;
 
-		if (!pte_swp_uffd_wp_any(pte))
-			categories |= PAGE_IS_WRITTEN;
+		if (!pte_swp_uffd_any(pte)) {
+			if (userfaultfd_wp(vma))
+				categories |= PAGE_IS_WRITTEN;
+			if (userfaultfd_rwp(vma))
+				categories |= PAGE_IS_ACCESSED;
+		}
 		if (pte_swp_soft_dirty(pte))
 			categories |= PAGE_IS_SOFT_DIRTY;
 	}
@@ -2611,12 +2652,12 @@ static void make_uffd_wp_huge_pte(struct vm_area_struct *vma,
 
 	if (softleaf_is_migration(entry)) {
 		set_huge_pte_at(vma->vm_mm, addr, ptep,
-				pte_swp_mkuffd_wp(ptent), psize);
+				pte_swp_mkuffd(ptent), psize);
 	} else {
 		pte_t old_pte, new_pte;
 
 		old_pte = huge_ptep_modify_prot_start(vma, addr, ptep);
-		new_pte = huge_pte_mkuffd_wp(old_pte);
+		new_pte = huge_pte_mkuffd(old_pte);
 		huge_ptep_modify_prot_commit(vma, addr, ptep, old_pte, new_pte);
 	}
 }
@@ -2849,8 +2890,8 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,
 		for (addr = start; addr != end; pte++, addr += PAGE_SIZE) {
 			pte_t ptent = ptep_get(pte);
 
-			if ((pte_present(ptent) && pte_uffd_wp(ptent)) ||
-			    pte_swp_uffd_wp_any(ptent))
+			if ((pte_present(ptent) && pte_uffd(ptent)) ||
+			    pte_swp_uffd_any(ptent))
 				continue;
 			make_uffd_wp_pte(vma, addr, pte, ptent);
 			if (!flush_end)
@@ -2860,15 +2901,16 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,
 		goto flush_and_return;
 	}
 
-	if (!p->arg.category_anyof_mask && !p->arg.category_inverted &&
+	if (userfaultfd_wp(vma) && !p->arg.category_anyof_mask &&
+	    !p->arg.category_inverted &&
 	    p->arg.category_mask == PAGE_IS_WRITTEN &&
 	    p->arg.return_mask == PAGE_IS_WRITTEN) {
 		for (addr = start; addr < end; pte++, addr += PAGE_SIZE) {
 			unsigned long next = addr + PAGE_SIZE;
 			pte_t ptent = ptep_get(pte);
 
-			if ((pte_present(ptent) && pte_uffd_wp(ptent)) ||
-			    pte_swp_uffd_wp_any(ptent))
+			if ((pte_present(ptent) && pte_uffd(ptent)) ||
+			    pte_swp_uffd_any(ptent))
 				continue;
 			ret = pagemap_scan_output(p->cur_vma_category | PAGE_IS_WRITTEN,
 						  p, addr, &next);
@@ -2935,7 +2977,8 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask,
 		/* Go the short route when not write-protecting pages. */
 
 		pte = huge_ptep_get(walk->mm, start, ptep);
-		categories = p->cur_vma_category | pagemap_hugetlb_category(pte);
+		categories = p->cur_vma_category |
+			     pagemap_hugetlb_category(vma, pte);
 
 		if (!pagemap_scan_is_interesting_page(categories, p))
 			return 0;
@@ -2947,7 +2990,7 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask,
 	ptl = huge_pte_lock(hstate_vma(vma), vma->vm_mm, ptep);
 
 	pte = huge_ptep_get(walk->mm, start, ptep);
-	categories = p->cur_vma_category | pagemap_hugetlb_category(pte);
+	categories = p->cur_vma_category | pagemap_hugetlb_category(vma, pte);
 
 	if (!pagemap_scan_is_interesting_page(categories, p))
 		goto out_unlock;
@@ -3039,12 +3082,28 @@ static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end,
 {
 	struct pagemap_scan_private *p = walk->private;
 	struct vm_area_struct *vma = walk->vma;
+	unsigned long categories;
 	int ret, err;
 
-	if (!vma || !pagemap_scan_is_interesting_page(p->cur_vma_category, p))
+	if (!vma)
+		return 0;
+
+	/*
+	 * In a uffd-wp VMA an unpopulated range is treated as written:
+	 * uffd-wp registration populates page tables and installs markers
+	 * with WP_UNPOPULATED, so a missing marker means the range was
+	 * zapped. See the pte_none() handling in pagemap_page_category().
+	 *
+	 * hugetlb differs, see pagemap_hugetlb_category().
+	 */
+	categories = p->cur_vma_category;
+	if (userfaultfd_wp(vma) && !is_vm_hugetlb_page(vma))
+		categories |= PAGE_IS_WRITTEN;
+
+	if (!pagemap_scan_is_interesting_page(categories, p))
 		return 0;
 
-	ret = pagemap_scan_output(p->cur_vma_category, p, addr, &end);
+	ret = pagemap_scan_output(categories, p, addr, &end);
 	if (addr == end)
 		return ret;
 
diff --git a/fs/proc_namespace.c b/fs/proc_namespace.c
index 5c555db68aa21..036356c0a55bb 100644
--- a/fs/proc_namespace.c
+++ b/fs/proc_namespace.c
@@ -254,13 +254,13 @@ static int mounts_open_common(struct inode *inode, struct file *file,
 	}
 	ns = nsp->mnt_ns;
 	get_mnt_ns(ns);
-	if (!task->fs) {
+	if (!task->real_fs) {
 		task_unlock(task);
 		put_task_struct(task);
 		ret = -ENOENT;
 		goto err_put_ns;
 	}
-	get_fs_root(task->fs, &root);
+	get_fs_root(task->real_fs, &root);
 	task_unlock(task);
 	put_task_struct(task);
 
diff --git a/fs/ramfs/file-nommu.c b/fs/ramfs/file-nommu.c
index 2f79bcb89d2e5..fb471bf88ab7a 100644
--- a/fs/ramfs/file-nommu.c
+++ b/fs/ramfs/file-nommu.c
@@ -69,6 +69,9 @@ int ramfs_nommu_expand_for_mapping(struct inode *inode, size_t newsize)
 	gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
 
 	/* make various checks */
+	if (!newsize)
+		return 0;
+
 	order = get_order(newsize);
 	if (unlikely(order > MAX_PAGE_ORDER))
 		return -EFBIG;
diff --git a/fs/ramfs/inode.c b/fs/ramfs/inode.c
index 3987639ed132c..0a88ede48e0ad 100644
--- a/fs/ramfs/inode.c
+++ b/fs/ramfs/inode.c
@@ -121,14 +121,14 @@ ramfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 static struct dentry *ramfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 				 struct dentry *dentry, umode_t mode)
 {
-	int retval = ramfs_mknod(&nop_mnt_idmap, dir, dentry, mode | S_IFDIR, 0);
+	int retval = ramfs_mknod(&nop_mnt_idmap, dir, dentry, mode, 0);
 	if (!retval)
 		inc_nlink(dir);
 	return ERR_PTR(retval);
 }
 
 static int ramfs_create(struct mnt_idmap *idmap, struct inode *dir,
-			struct dentry *dentry, umode_t mode, bool excl)
+			struct dentry *dentry, umode_t mode)
 {
 	return ramfs_mknod(&nop_mnt_idmap, dir, dentry, mode | S_IFREG, 0);
 }
diff --git a/fs/resctrl/ctrlmondata.c b/fs/resctrl/ctrlmondata.c
index 9a7dfc48cb2e2..2b29fb5a87022 100644
--- a/fs/resctrl/ctrlmondata.c
+++ b/fs/resctrl/ctrlmondata.c
@@ -261,7 +261,7 @@ static int parse_line(char *line, struct resctrl_schema *s,
 		return -EINVAL;
 	}
 	dom = strim(dom);
-	list_for_each_entry(d, &r->ctrl_domains, hdr.list) {
+	list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) {
 		if (d->hdr.id == dom_id) {
 			data.buf = dom;
 			data.closid = rdtgrp->closid;
@@ -397,7 +397,7 @@ static void show_doms(struct seq_file *s, struct resctrl_schema *schema,
 
 	if (resource_name)
 		seq_printf(s, "%*s:", max_name_width, resource_name);
-	list_for_each_entry(dom, &r->ctrl_domains, hdr.list) {
+	list_for_each_entry_rcu(dom, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) {
 		if (sep)
 			seq_puts(s, ";");
 
@@ -535,6 +535,8 @@ struct rdt_domain_hdr *resctrl_find_domain(struct list_head *h, int id,
 	struct rdt_domain_hdr *d;
 	struct list_head *l;
 
+	lockdep_assert_cpus_held();
+
 	list_for_each(l, h) {
 		d = list_entry(l, struct rdt_domain_hdr, list);
 		/* When id is found, return its domain. */
@@ -717,7 +719,7 @@ int rdtgroup_mondata_show(struct seq_file *m, void *arg)
 		 * struct mon_data. Search all domains in the resource for
 		 * one that matches this cache id.
 		 */
-		list_for_each_entry(d, &r->mon_domains, hdr.list) {
+		list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
 			if (d->ci_id == domid) {
 				cpu = cpumask_any(&d->hdr.cpu_mask);
 				ci = get_cpu_cacheinfo_level(cpu, RESCTRL_L3_CACHE);
@@ -769,10 +771,12 @@ int rdtgroup_mondata_show(struct seq_file *m, void *arg)
 int resctrl_io_alloc_show(struct kernfs_open_file *of, struct seq_file *seq, void *v)
 {
 	struct resctrl_schema *s = rdt_kn_parent_priv(of->kn);
-	struct rdt_resource *r = s->res;
+	struct rdt_resource *r;
 
-	mutex_lock(&rdtgroup_mutex);
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 
+	r = s->res;
 	if (r->cache.io_alloc_capable) {
 		if (resctrl_arch_get_io_alloc_enabled(r))
 			seq_puts(seq, "enabled\n");
@@ -782,7 +786,7 @@ int resctrl_io_alloc_show(struct kernfs_open_file *of, struct seq_file *seq, voi
 		seq_puts(seq, "not supported\n");
 	}
 
-	mutex_unlock(&rdtgroup_mutex);
+	info_kn_unlock(of->kn);
 
 	return 0;
 }
@@ -817,7 +821,7 @@ static int resctrl_io_alloc_init_cbm(struct resctrl_schema *s, u32 closid)
 	/* Keep CDP_CODE and CDP_DATA of io_alloc CLOSID's CBM in sync. */
 	if (resctrl_arch_get_cdp_enabled(r->rid)) {
 		peer_type = resctrl_peer_type(s->conf_type);
-		list_for_each_entry(d, &s->res->ctrl_domains, hdr.list)
+		list_for_each_entry_rcu(d, &s->res->ctrl_domains, hdr.list, lockdep_is_cpus_held())
 			memcpy(&d->staged_config[peer_type],
 			       &d->staged_config[s->conf_type],
 			       sizeof(d->staged_config[0]));
@@ -847,7 +851,7 @@ ssize_t resctrl_io_alloc_write(struct kernfs_open_file *of, char *buf,
 			       size_t nbytes, loff_t off)
 {
 	struct resctrl_schema *s = rdt_kn_parent_priv(of->kn);
-	struct rdt_resource *r = s->res;
+	struct rdt_resource *r;
 	char const *grp_name;
 	u32 io_alloc_closid;
 	bool enable;
@@ -857,9 +861,10 @@ ssize_t resctrl_io_alloc_write(struct kernfs_open_file *of, char *buf,
 	if (ret)
 		return ret;
 
-	cpus_read_lock();
-	mutex_lock(&rdtgroup_mutex);
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 
+	r = s->res;
 	rdt_last_cmd_clear();
 
 	if (!r->cache.io_alloc_capable) {
@@ -907,8 +912,7 @@ ssize_t resctrl_io_alloc_write(struct kernfs_open_file *of, char *buf,
 	}
 
 out_unlock:
-	mutex_unlock(&rdtgroup_mutex);
-	cpus_read_unlock();
+	info_kn_unlock(of->kn);
 
 	return ret ?: nbytes;
 }
@@ -916,14 +920,15 @@ ssize_t resctrl_io_alloc_write(struct kernfs_open_file *of, char *buf,
 int resctrl_io_alloc_cbm_show(struct kernfs_open_file *of, struct seq_file *seq, void *v)
 {
 	struct resctrl_schema *s = rdt_kn_parent_priv(of->kn);
-	struct rdt_resource *r = s->res;
+	struct rdt_resource *r;
 	int ret = 0;
 
-	cpus_read_lock();
-	mutex_lock(&rdtgroup_mutex);
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 
 	rdt_last_cmd_clear();
 
+	r = s->res;
 	if (!r->cache.io_alloc_capable) {
 		rdt_last_cmd_printf("io_alloc is not supported on %s\n", s->name);
 		ret = -ENODEV;
@@ -945,8 +950,7 @@ int resctrl_io_alloc_cbm_show(struct kernfs_open_file *of, struct seq_file *seq,
 	show_doms(seq, s, NULL, resctrl_io_alloc_closid(r));
 
 out_unlock:
-	mutex_unlock(&rdtgroup_mutex);
-	cpus_read_unlock();
+	info_kn_unlock(of->kn);
 	return ret;
 }
 
@@ -980,7 +984,7 @@ static int resctrl_io_alloc_parse_line(char *line,  struct rdt_resource *r,
 	}
 
 	dom = strim(dom);
-	list_for_each_entry(d, &r->ctrl_domains, hdr.list) {
+	list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) {
 		if (update_all || d->hdr.id == dom_id) {
 			data.buf = dom;
 			data.mode = RDT_MODE_SHAREABLE;
@@ -1013,7 +1017,7 @@ ssize_t resctrl_io_alloc_cbm_write(struct kernfs_open_file *of, char *buf,
 				   size_t nbytes, loff_t off)
 {
 	struct resctrl_schema *s = rdt_kn_parent_priv(of->kn);
-	struct rdt_resource *r = s->res;
+	struct rdt_resource *r;
 	u32 io_alloc_closid;
 	int ret = 0;
 
@@ -1023,10 +1027,11 @@ ssize_t resctrl_io_alloc_cbm_write(struct kernfs_open_file *of, char *buf,
 
 	buf[nbytes - 1] = '\0';
 
-	cpus_read_lock();
-	mutex_lock(&rdtgroup_mutex);
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 	rdt_last_cmd_clear();
 
+	r = s->res;
 	if (!r->cache.io_alloc_capable) {
 		rdt_last_cmd_printf("io_alloc is not supported on %s\n", s->name);
 		ret = -ENODEV;
@@ -1051,8 +1056,7 @@ ssize_t resctrl_io_alloc_cbm_write(struct kernfs_open_file *of, char *buf,
 out_clear_configs:
 	rdt_staged_configs_clear();
 out_unlock:
-	mutex_unlock(&rdtgroup_mutex);
-	cpus_read_unlock();
+	info_kn_unlock(of->kn);
 
 	return ret ?: nbytes;
 }
diff --git a/fs/resctrl/internal.h b/fs/resctrl/internal.h
index 48af75b9dc855..e62a277dee850 100644
--- a/fs/resctrl/internal.h
+++ b/fs/resctrl/internal.h
@@ -335,8 +335,9 @@ __printf(1, 2)
 void rdt_last_cmd_printf(const char *fmt, ...);
 
 struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn);
-
 void rdtgroup_kn_unlock(struct kernfs_node *kn);
+bool info_kn_lock(struct kernfs_node *kn);
+void info_kn_unlock(struct kernfs_node *kn);
 
 int rdtgroup_kn_mode_restrict(struct rdtgroup *r, const char *name);
 
diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c
index a932a1fea8182..7340b1d17f177 100644
--- a/fs/resctrl/monitor.c
+++ b/fs/resctrl/monitor.c
@@ -309,7 +309,7 @@ static void add_rmid_to_limbo(struct rmid_entry *entry)
 	idx = resctrl_arch_rmid_idx_encode(entry->closid, entry->rmid);
 
 	entry->busy = 0;
-	list_for_each_entry(d, &r->mon_domains, hdr.list) {
+	list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
 		/*
 		 * For the first limbo RMID in the domain,
 		 * setup up the limbo worker.
@@ -507,6 +507,11 @@ static int __l3_mon_event_count_sum(struct rdtgroup *rdtgrp, struct rmid_read *r
 	 * all domains fail for any reason.
 	 */
 	ret = -EINVAL;
+	/*
+	 * RCU list being traversed with CPU hotplug lock held. lockdep
+	 * unable to help prove this here since this work is scheduled via
+	 * smp_call*(). Not called from MBM overflow handler.
+	 */
 	list_for_each_entry(d, &rr->r->mon_domains, hdr.list) {
 		if (d->ci_id != rr->ci->id)
 			continue;
@@ -628,14 +633,22 @@ void mon_event_count(void *info)
 		rr->err = 0;
 }
 
-static struct rdt_ctrl_domain *get_ctrl_domain_from_cpu(int cpu,
-							struct rdt_resource *r)
+/*
+ * Find the software controller's ctrl domain that contains @cpu on resource @r.
+ *
+ * Only called from the mbm_over worker via update_mba_bw() where the returned
+ * domain is kept alive by cancel_delayed_work_sync() in
+ * resctrl_offline_ctrl_domain(). This drains this worker and then waits on
+ * rdtgroup_mutex held here before the architecture can free the ctrl domain.
+ *
+ * Context: Call from RCU read-side critical section.
+ */
+static struct rdt_ctrl_domain *get_sc_ctrl_domain_from_cpu(int cpu,
+							   struct rdt_resource *r)
 {
 	struct rdt_ctrl_domain *d;
 
-	lockdep_assert_cpus_held();
-
-	list_for_each_entry(d, &r->ctrl_domains, hdr.list) {
+	list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list) {
 		/* Find the domain that contains this CPU */
 		if (cpumask_test_cpu(cpu, &d->hdr.cpu_mask))
 			return d;
@@ -696,7 +709,8 @@ static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_l3_mon_domain *dom_m
 	if (WARN_ON_ONCE(!pmbm_data))
 		return;
 
-	dom_mba = get_ctrl_domain_from_cpu(smp_processor_id(), r_mba);
+	guard(rcu)();
+	dom_mba = get_sc_ctrl_domain_from_cpu(smp_processor_id(), r_mba);
 	if (!dom_mba) {
 		pr_warn_once("Failure to get domain for MBA update\n");
 		return;
@@ -799,11 +813,25 @@ void cqm_handle_limbo(struct work_struct *work)
 	unsigned long delay = msecs_to_jiffies(CQM_LIMBOCHECK_INTERVAL);
 	struct rdt_l3_mon_domain *d;
 
-	cpus_read_lock();
+	/*
+	 * Safe to run without CPU hotplug lock. Work is guaranteed to be
+	 * canceled before the domain structure is removed.
+	 */
 	mutex_lock(&rdtgroup_mutex);
 
+	/*
+	 * Ensure the worker is dedicated to a CPU as intended and not
+	 * relocated by workqueue subsystem as part of CPU going offline.
+	 */
+	if (!is_percpu_thread())
+		goto out_unlock;
+
 	d = container_of(work, struct rdt_l3_mon_domain, cqm_limbo.work);
 
+	/* Domain is going offline */
+	if (cpumask_empty(&d->hdr.cpu_mask))
+		goto out_unlock;
+
 	__check_limbo(d, false);
 
 	if (has_busy_rmid(d)) {
@@ -813,8 +841,8 @@ void cqm_handle_limbo(struct work_struct *work)
 					 delay);
 	}
 
+out_unlock:
 	mutex_unlock(&rdtgroup_mutex);
-	cpus_read_unlock();
 }
 
 /**
@@ -846,7 +874,10 @@ void mbm_handle_overflow(struct work_struct *work)
 	struct list_head *head;
 	struct rdt_resource *r;
 
-	cpus_read_lock();
+	/*
+	 * Safe to run without CPU hotplug lock. Work is guaranteed to be
+	 * canceled before the domain structure is removed.
+	 */
 	mutex_lock(&rdtgroup_mutex);
 
 	/*
@@ -856,9 +887,24 @@ void mbm_handle_overflow(struct work_struct *work)
 	if (!resctrl_mounted || !resctrl_arch_mon_capable())
 		goto out_unlock;
 
+	/*
+	 * Ensure the worker is dedicated to a CPU and not relocated by
+	 * workqueue subsystem as part of CPU going offline since reading
+	 * events depend on smp_processor_id(). After passing this check
+	 * smp_processor_id() is valid for entire duration of this worker
+	 * since it runs with rdtgroup_mutex held and the offline handler needs
+	 * rdtgroup_mutex to offline the CPU being run on here.
+	 */
+	if (!is_percpu_thread())
+		goto out_unlock;
+
 	r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
 	d = container_of(work, struct rdt_l3_mon_domain, mbm_over.work);
 
+	/* Domain is going offline */
+	if (cpumask_empty(&d->hdr.cpu_mask))
+		goto out_unlock;
+
 	list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) {
 		mbm_update(r, d, prgrp);
 
@@ -880,7 +926,6 @@ void mbm_handle_overflow(struct work_struct *work)
 
 out_unlock:
 	mutex_unlock(&rdtgroup_mutex);
-	cpus_read_unlock();
 }
 
 /**
@@ -1057,7 +1102,8 @@ int event_filter_show(struct kernfs_open_file *of, struct seq_file *seq, void *v
 	bool sep = false;
 	int ret = 0, i;
 
-	mutex_lock(&rdtgroup_mutex);
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 	rdt_last_cmd_clear();
 
 	r = resctrl_arch_get_resource(mevt->rid);
@@ -1078,7 +1124,7 @@ int event_filter_show(struct kernfs_open_file *of, struct seq_file *seq, void *v
 	seq_putc(seq, '\n');
 
 out_unlock:
-	mutex_unlock(&rdtgroup_mutex);
+	info_kn_unlock(of->kn);
 
 	return ret;
 }
@@ -1089,7 +1135,8 @@ int resctrl_mbm_assign_on_mkdir_show(struct kernfs_open_file *of, struct seq_fil
 	struct rdt_resource *r = rdt_kn_parent_priv(of->kn);
 	int ret = 0;
 
-	mutex_lock(&rdtgroup_mutex);
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 	rdt_last_cmd_clear();
 
 	if (!resctrl_arch_mbm_cntr_assign_enabled(r)) {
@@ -1101,7 +1148,7 @@ int resctrl_mbm_assign_on_mkdir_show(struct kernfs_open_file *of, struct seq_fil
 	seq_printf(s, "%u\n", r->mon.mbm_assign_on_mkdir);
 
 out_unlock:
-	mutex_unlock(&rdtgroup_mutex);
+	info_kn_unlock(of->kn);
 
 	return ret;
 }
@@ -1117,7 +1164,8 @@ ssize_t resctrl_mbm_assign_on_mkdir_write(struct kernfs_open_file *of, char *buf
 	if (ret)
 		return ret;
 
-	mutex_lock(&rdtgroup_mutex);
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 	rdt_last_cmd_clear();
 
 	if (!resctrl_arch_mbm_cntr_assign_enabled(r)) {
@@ -1129,7 +1177,7 @@ ssize_t resctrl_mbm_assign_on_mkdir_write(struct kernfs_open_file *of, char *buf
 	r->mon.mbm_assign_on_mkdir = value;
 
 out_unlock:
-	mutex_unlock(&rdtgroup_mutex);
+	info_kn_unlock(of->kn);
 
 	return ret ?: nbytes;
 }
@@ -1231,7 +1279,7 @@ static int rdtgroup_assign_cntr_event(struct rdt_l3_mon_domain *d, struct rdtgro
 	int ret = 0;
 
 	if (!d) {
-		list_for_each_entry(d, &r->mon_domains, hdr.list) {
+		list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
 			int err;
 
 			err = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt);
@@ -1303,7 +1351,7 @@ static void rdtgroup_unassign_cntr_event(struct rdt_l3_mon_domain *d, struct rdt
 	struct rdt_resource *r = resctrl_arch_get_resource(mevt->rid);
 
 	if (!d) {
-		list_for_each_entry(d, &r->mon_domains, hdr.list)
+		list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held())
 			rdtgroup_free_unassign_cntr(r, d, rdtgrp, mevt);
 	} else {
 		rdtgroup_free_unassign_cntr(r, d, rdtgrp, mevt);
@@ -1375,7 +1423,7 @@ static void rdtgroup_update_cntr_event(struct rdt_resource *r, struct rdtgroup *
 	struct rdt_l3_mon_domain *d;
 	int cntr_id;
 
-	list_for_each_entry(d, &r->mon_domains, hdr.list) {
+	list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
 		cntr_id = mbm_cntr_get(r, d, rdtgrp, evtid);
 		if (cntr_id >= 0)
 			rdtgroup_assign_cntr(r, d, evtid, rdtgrp->mon.rmid,
@@ -1419,8 +1467,8 @@ ssize_t event_filter_write(struct kernfs_open_file *of, char *buf, size_t nbytes
 
 	buf[nbytes - 1] = '\0';
 
-	cpus_read_lock();
-	mutex_lock(&rdtgroup_mutex);
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 
 	rdt_last_cmd_clear();
 
@@ -1443,8 +1491,7 @@ ssize_t event_filter_write(struct kernfs_open_file *of, char *buf, size_t nbytes
 	}
 
 out_unlock:
-	mutex_unlock(&rdtgroup_mutex);
-	cpus_read_unlock();
+	info_kn_unlock(of->kn);
 
 	return ret ?: nbytes;
 }
@@ -1455,7 +1502,8 @@ int resctrl_mbm_assign_mode_show(struct kernfs_open_file *of,
 	struct rdt_resource *r = rdt_kn_parent_priv(of->kn);
 	bool enabled;
 
-	mutex_lock(&rdtgroup_mutex);
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 	enabled = resctrl_arch_mbm_cntr_assign_enabled(r);
 
 	if (r->mon.mbm_cntr_assignable) {
@@ -1474,7 +1522,7 @@ int resctrl_mbm_assign_mode_show(struct kernfs_open_file *of,
 		seq_puts(s, "[default]\n");
 	}
 
-	mutex_unlock(&rdtgroup_mutex);
+	info_kn_unlock(of->kn);
 
 	return 0;
 }
@@ -1493,8 +1541,8 @@ ssize_t resctrl_mbm_assign_mode_write(struct kernfs_open_file *of, char *buf,
 
 	buf[nbytes - 1] = '\0';
 
-	cpus_read_lock();
-	mutex_lock(&rdtgroup_mutex);
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 
 	rdt_last_cmd_clear();
 
@@ -1545,15 +1593,14 @@ ssize_t resctrl_mbm_assign_mode_write(struct kernfs_open_file *of, char *buf,
 		/*
 		 * Reset all the non-achitectural RMID state and assignable counters.
 		 */
-		list_for_each_entry(d, &r->mon_domains, hdr.list) {
+		list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
 			mbm_cntr_free_all(r, d);
 			resctrl_reset_rmid_all(r, d);
 		}
 	}
 
 out_unlock:
-	mutex_unlock(&rdtgroup_mutex);
-	cpus_read_unlock();
+	info_kn_unlock(of->kn);
 
 	return ret ?: nbytes;
 }
@@ -1565,10 +1612,10 @@ int resctrl_num_mbm_cntrs_show(struct kernfs_open_file *of,
 	struct rdt_l3_mon_domain *dom;
 	bool sep = false;
 
-	cpus_read_lock();
-	mutex_lock(&rdtgroup_mutex);
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 
-	list_for_each_entry(dom, &r->mon_domains, hdr.list) {
+	list_for_each_entry_rcu(dom, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
 		if (sep)
 			seq_putc(s, ';');
 
@@ -1577,8 +1624,7 @@ int resctrl_num_mbm_cntrs_show(struct kernfs_open_file *of,
 	}
 	seq_putc(s, '\n');
 
-	mutex_unlock(&rdtgroup_mutex);
-	cpus_read_unlock();
+	info_kn_unlock(of->kn);
 	return 0;
 }
 
@@ -1591,8 +1637,8 @@ int resctrl_available_mbm_cntrs_show(struct kernfs_open_file *of,
 	u32 cntrs, i;
 	int ret = 0;
 
-	cpus_read_lock();
-	mutex_lock(&rdtgroup_mutex);
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 
 	rdt_last_cmd_clear();
 
@@ -1602,7 +1648,7 @@ int resctrl_available_mbm_cntrs_show(struct kernfs_open_file *of,
 		goto out_unlock;
 	}
 
-	list_for_each_entry(dom, &r->mon_domains, hdr.list) {
+	list_for_each_entry_rcu(dom, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
 		if (sep)
 			seq_putc(s, ';');
 
@@ -1618,8 +1664,7 @@ int resctrl_available_mbm_cntrs_show(struct kernfs_open_file *of,
 	seq_putc(s, '\n');
 
 out_unlock:
-	mutex_unlock(&rdtgroup_mutex);
-	cpus_read_unlock();
+	info_kn_unlock(of->kn);
 
 	return ret;
 }
@@ -1652,7 +1697,7 @@ int mbm_L3_assignments_show(struct kernfs_open_file *of, struct seq_file *s, voi
 
 		sep = false;
 		seq_printf(s, "%s:", mevt->name);
-		list_for_each_entry(d, &r->mon_domains, hdr.list) {
+		list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
 			if (sep)
 				seq_putc(s, ';');
 
@@ -1750,7 +1795,7 @@ static int resctrl_parse_mbm_assignment(struct rdt_resource *r, struct rdtgroup
 	}
 
 	/* Verify if the dom_id is valid */
-	list_for_each_entry(d, &r->mon_domains, hdr.list) {
+	list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
 		if (d->hdr.id == dom_id) {
 			ret = rdtgroup_modify_assign_state(dom_str, d, rdtgrp, mevt);
 			if (ret) {
diff --git a/fs/resctrl/pseudo_lock.c b/fs/resctrl/pseudo_lock.c
index d1cb0986006eb..dea2b4bf966fa 100644
--- a/fs/resctrl/pseudo_lock.c
+++ b/fs/resctrl/pseudo_lock.c
@@ -656,7 +656,7 @@ bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_ctrl_domain *d)
 	 * associated with them.
 	 */
 	for_each_alloc_capable_rdt_resource(r) {
-		list_for_each_entry(d_i, &r->ctrl_domains, hdr.list) {
+		list_for_each_entry_rcu(d_i, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) {
 			if (d_i->plr)
 				cpumask_or(cpu_with_psl, cpu_with_psl,
 					   &d_i->hdr.cpu_mask);
diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c
index cc9966ff6cdfb..30df902ea776e 100644
--- a/fs/resctrl/rdtgroup.c
+++ b/fs/resctrl/rdtgroup.c
@@ -74,8 +74,6 @@ static int rdtgroup_setup_root(struct rdt_fs_context *ctx);
 
 static void rdtgroup_destroy_root(void);
 
-static void mon_put_kn_priv(void);
-
 struct dentry *debugfs_resctrl;
 
 /*
@@ -119,7 +117,7 @@ void rdt_staged_configs_clear(void)
 	lockdep_assert_held(&rdtgroup_mutex);
 
 	for_each_alloc_capable_rdt_resource(r) {
-		list_for_each_entry(dom, &r->ctrl_domains, hdr.list)
+		list_for_each_entry_rcu(dom, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held())
 			memset(dom->staged_config, 0, sizeof(dom->staged_config));
 	}
 }
@@ -979,13 +977,14 @@ static int rdt_last_cmd_status_show(struct kernfs_open_file *of,
 {
 	int len;
 
-	mutex_lock(&rdtgroup_mutex);
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 	len = seq_buf_used(&last_cmd_status);
 	if (len)
 		seq_printf(seq, "%.*s", len, last_cmd_status_buf);
 	else
 		seq_puts(seq, "ok\n");
-	mutex_unlock(&rdtgroup_mutex);
+	info_kn_unlock(of->kn);
 	return 0;
 }
 
@@ -1004,7 +1003,11 @@ static int rdt_num_closids_show(struct kernfs_open_file *of,
 {
 	struct resctrl_schema *s = rdt_kn_parent_priv(of->kn);
 
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 	seq_printf(seq, "%u\n", s->num_closid);
+	info_kn_unlock(of->kn);
+
 	return 0;
 }
 
@@ -1012,9 +1015,14 @@ static int rdt_default_ctrl_show(struct kernfs_open_file *of,
 				 struct seq_file *seq, void *v)
 {
 	struct resctrl_schema *s = rdt_kn_parent_priv(of->kn);
-	struct rdt_resource *r = s->res;
+	struct rdt_resource *r;
 
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
+	r = s->res;
 	seq_printf(seq, "%x\n", resctrl_get_default_ctrl(r));
+	info_kn_unlock(of->kn);
+
 	return 0;
 }
 
@@ -1022,9 +1030,14 @@ static int rdt_min_cbm_bits_show(struct kernfs_open_file *of,
 				 struct seq_file *seq, void *v)
 {
 	struct resctrl_schema *s = rdt_kn_parent_priv(of->kn);
-	struct rdt_resource *r = s->res;
+	struct rdt_resource *r;
 
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
+	r = s->res;
 	seq_printf(seq, "%u\n", r->cache.min_cbm_bits);
+	info_kn_unlock(of->kn);
+
 	return 0;
 }
 
@@ -1032,9 +1045,14 @@ static int rdt_shareable_bits_show(struct kernfs_open_file *of,
 				   struct seq_file *seq, void *v)
 {
 	struct resctrl_schema *s = rdt_kn_parent_priv(of->kn);
-	struct rdt_resource *r = s->res;
+	struct rdt_resource *r;
 
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
+	r = s->res;
 	seq_printf(seq, "%x\n", r->cache.shareable_bits);
+	info_kn_unlock(of->kn);
+
 	return 0;
 }
 
@@ -1062,16 +1080,17 @@ static int rdt_bit_usage_show(struct kernfs_open_file *of,
 	 */
 	unsigned long sw_shareable = 0, hw_shareable = 0;
 	unsigned long exclusive = 0, pseudo_locked = 0;
-	struct rdt_resource *r = s->res;
 	struct rdt_ctrl_domain *dom;
 	int i, hwb, swb, excl, psl;
+	struct rdt_resource *r;
 	enum rdtgrp_mode mode;
 	bool sep = false;
 	u32 ctrl_val;
 
-	cpus_read_lock();
-	mutex_lock(&rdtgroup_mutex);
-	list_for_each_entry(dom, &r->ctrl_domains, hdr.list) {
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
+	r = s->res;
+	list_for_each_entry_rcu(dom, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) {
 		if (sep)
 			seq_putc(seq, ';');
 		hw_shareable = r->cache.shareable_bits;
@@ -1146,8 +1165,7 @@ static int rdt_bit_usage_show(struct kernfs_open_file *of,
 		sep = true;
 	}
 	seq_putc(seq, '\n');
-	mutex_unlock(&rdtgroup_mutex);
-	cpus_read_unlock();
+	info_kn_unlock(of->kn);
 	return 0;
 }
 
@@ -1155,9 +1173,14 @@ static int rdt_min_bw_show(struct kernfs_open_file *of,
 			   struct seq_file *seq, void *v)
 {
 	struct resctrl_schema *s = rdt_kn_parent_priv(of->kn);
-	struct rdt_resource *r = s->res;
+	struct rdt_resource *r;
 
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
+	r = s->res;
 	seq_printf(seq, "%u\n", r->membw.min_bw);
+	info_kn_unlock(of->kn);
+
 	return 0;
 }
 
@@ -1166,8 +1189,12 @@ static int rdt_num_rmids_show(struct kernfs_open_file *of,
 {
 	struct rdt_resource *r = rdt_kn_parent_priv(of->kn);
 
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 	seq_printf(seq, "%u\n", r->mon.num_rmid);
 
+	info_kn_unlock(of->kn);
+
 	return 0;
 }
 
@@ -1177,6 +1204,8 @@ static int rdt_mon_features_show(struct kernfs_open_file *of,
 	struct rdt_resource *r = rdt_kn_parent_priv(of->kn);
 	struct mon_evt *mevt;
 
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 	for_each_mon_event(mevt) {
 		if (mevt->rid != r->rid || !mevt->enabled)
 			continue;
@@ -1186,6 +1215,8 @@ static int rdt_mon_features_show(struct kernfs_open_file *of,
 			seq_printf(seq, "%s_config\n", mevt->name);
 	}
 
+	info_kn_unlock(of->kn);
+
 	return 0;
 }
 
@@ -1193,9 +1224,14 @@ static int rdt_bw_gran_show(struct kernfs_open_file *of,
 			    struct seq_file *seq, void *v)
 {
 	struct resctrl_schema *s = rdt_kn_parent_priv(of->kn);
-	struct rdt_resource *r = s->res;
+	struct rdt_resource *r;
 
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
+	r = s->res;
 	seq_printf(seq, "%u\n", r->membw.bw_gran);
+	info_kn_unlock(of->kn);
+
 	return 0;
 }
 
@@ -1203,16 +1239,24 @@ static int rdt_delay_linear_show(struct kernfs_open_file *of,
 				 struct seq_file *seq, void *v)
 {
 	struct resctrl_schema *s = rdt_kn_parent_priv(of->kn);
-	struct rdt_resource *r = s->res;
+	struct rdt_resource *r;
 
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
+	r = s->res;
 	seq_printf(seq, "%u\n", r->membw.delay_linear);
+	info_kn_unlock(of->kn);
+
 	return 0;
 }
 
 static int max_threshold_occ_show(struct kernfs_open_file *of,
 				  struct seq_file *seq, void *v)
 {
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 	seq_printf(seq, "%u\n", resctrl_rmid_realloc_threshold);
+	info_kn_unlock(of->kn);
 
 	return 0;
 }
@@ -1221,22 +1265,28 @@ static int rdt_thread_throttle_mode_show(struct kernfs_open_file *of,
 					 struct seq_file *seq, void *v)
 {
 	struct resctrl_schema *s = rdt_kn_parent_priv(of->kn);
-	struct rdt_resource *r = s->res;
+	struct rdt_resource *r;
 
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
+
+	r = s->res;
 	switch (r->membw.throttle_mode) {
 	case THREAD_THROTTLE_PER_THREAD:
 		seq_puts(seq, "per-thread\n");
-		return 0;
+		break;
 	case THREAD_THROTTLE_MAX:
 		seq_puts(seq, "max\n");
-		return 0;
+		break;
 	case THREAD_THROTTLE_UNDEFINED:
 		seq_puts(seq, "undefined\n");
-		return 0;
+		break;
+	default:
+		WARN_ON_ONCE(1);
+		break;
 	}
 
-	WARN_ON_ONCE(1);
-
+	info_kn_unlock(of->kn);
 	return 0;
 }
 
@@ -1250,12 +1300,20 @@ static ssize_t max_threshold_occ_write(struct kernfs_open_file *of,
 	if (ret)
 		return ret;
 
-	if (bytes > resctrl_rmid_realloc_limit)
-		return -EINVAL;
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
+
+	if (bytes > resctrl_rmid_realloc_limit) {
+		ret = -EINVAL;
+		goto out_unlock;
+	}
 
 	resctrl_rmid_realloc_threshold = resctrl_arch_round_mon_val(bytes);
 
-	return nbytes;
+out_unlock:
+	info_kn_unlock(of->kn);
+
+	return ret ?: nbytes;
 }
 
 /*
@@ -1295,10 +1353,15 @@ static int rdt_has_sparse_bitmasks_show(struct kernfs_open_file *of,
 					struct seq_file *seq, void *v)
 {
 	struct resctrl_schema *s = rdt_kn_parent_priv(of->kn);
-	struct rdt_resource *r = s->res;
+	struct rdt_resource *r;
 
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
+	r = s->res;
 	seq_printf(seq, "%u\n", r->cache.arch_has_sparse_bitmasks);
 
+	info_kn_unlock(of->kn);
+
 	return 0;
 }
 
@@ -1423,7 +1486,7 @@ static bool rdtgroup_mode_test_exclusive(struct rdtgroup *rdtgrp)
 		if (r->rid == RDT_RESOURCE_MBA || r->rid == RDT_RESOURCE_SMBA)
 			continue;
 		has_cache = true;
-		list_for_each_entry(d, &r->ctrl_domains, hdr.list) {
+		list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) {
 			ctrl = resctrl_arch_get_config(r, d, closid,
 						       s->conf_type);
 			if (rdtgroup_cbm_overlaps(s, d, ctrl, closid, false)) {
@@ -1612,7 +1675,7 @@ static int rdtgroup_size_show(struct kernfs_open_file *of,
 		type = schema->conf_type;
 		sep = false;
 		seq_printf(s, "%*s:", max_name_width, schema->name);
-		list_for_each_entry(d, &r->ctrl_domains, hdr.list) {
+		list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) {
 			if (sep)
 				seq_putc(s, ';');
 			if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
@@ -1654,10 +1717,10 @@ static int mbm_config_show(struct seq_file *s, struct rdt_resource *r, u32 evtid
 	struct rdt_l3_mon_domain *dom;
 	bool sep = false;
 
-	cpus_read_lock();
-	mutex_lock(&rdtgroup_mutex);
+	lockdep_assert_cpus_held();
+	lockdep_assert_held(&rdtgroup_mutex);
 
-	list_for_each_entry(dom, &r->mon_domains, hdr.list) {
+	list_for_each_entry_rcu(dom, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
 		if (sep)
 			seq_puts(s, ";");
 
@@ -1672,9 +1735,6 @@ static int mbm_config_show(struct seq_file *s, struct rdt_resource *r, u32 evtid
 	}
 	seq_puts(s, "\n");
 
-	mutex_unlock(&rdtgroup_mutex);
-	cpus_read_unlock();
-
 	return 0;
 }
 
@@ -1683,8 +1743,12 @@ static int mbm_total_bytes_config_show(struct kernfs_open_file *of,
 {
 	struct rdt_resource *r = rdt_kn_parent_priv(of->kn);
 
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
+
 	mbm_config_show(seq, r, QOS_L3_MBM_TOTAL_EVENT_ID);
 
+	info_kn_unlock(of->kn);
 	return 0;
 }
 
@@ -1693,8 +1757,12 @@ static int mbm_local_bytes_config_show(struct kernfs_open_file *of,
 {
 	struct rdt_resource *r = rdt_kn_parent_priv(of->kn);
 
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
+
 	mbm_config_show(seq, r, QOS_L3_MBM_LOCAL_EVENT_ID);
 
+	info_kn_unlock(of->kn);
 	return 0;
 }
 
@@ -1771,7 +1839,7 @@ static int mon_config_write(struct rdt_resource *r, char *tok, u32 evtid)
 		return -EINVAL;
 	}
 
-	list_for_each_entry(d, &r->mon_domains, hdr.list) {
+	list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
 		if (d->hdr.id == dom_id) {
 			mbm_config_write_domain(r, d, evtid, val);
 			goto next;
@@ -1792,8 +1860,8 @@ static ssize_t mbm_total_bytes_config_write(struct kernfs_open_file *of,
 	if (nbytes == 0 || buf[nbytes - 1] != '\n')
 		return -EINVAL;
 
-	cpus_read_lock();
-	mutex_lock(&rdtgroup_mutex);
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 
 	rdt_last_cmd_clear();
 
@@ -1801,8 +1869,7 @@ static ssize_t mbm_total_bytes_config_write(struct kernfs_open_file *of,
 
 	ret = mon_config_write(r, buf, QOS_L3_MBM_TOTAL_EVENT_ID);
 
-	mutex_unlock(&rdtgroup_mutex);
-	cpus_read_unlock();
+	info_kn_unlock(of->kn);
 
 	return ret ?: nbytes;
 }
@@ -1818,8 +1885,8 @@ static ssize_t mbm_local_bytes_config_write(struct kernfs_open_file *of,
 	if (nbytes == 0 || buf[nbytes - 1] != '\n')
 		return -EINVAL;
 
-	cpus_read_lock();
-	mutex_lock(&rdtgroup_mutex);
+	if (!info_kn_lock(of->kn))
+		return -ENOENT;
 
 	rdt_last_cmd_clear();
 
@@ -1827,8 +1894,7 @@ static ssize_t mbm_local_bytes_config_write(struct kernfs_open_file *of,
 
 	ret = mon_config_write(r, buf, QOS_L3_MBM_LOCAL_EVENT_ID);
 
-	mutex_unlock(&rdtgroup_mutex);
-	cpus_read_unlock();
+	info_kn_unlock(of->kn);
 
 	return ret ?: nbytes;
 }
@@ -2562,7 +2628,7 @@ static int set_mba_sc(bool mba_sc)
 
 	rdtgroup_default.mba_mbps_event = mba_mbps_default_event;
 
-	list_for_each_entry(d, &r->ctrl_domains, hdr.list) {
+	list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) {
 		for (i = 0; i < num_closid; i++)
 			d->mbps_val[i] = MBA_MAX_MBPS;
 	}
@@ -2608,15 +2674,24 @@ static void rdtgroup_kn_get(struct rdtgroup *rdtgrp, struct kernfs_node *kn)
 
 static void rdtgroup_kn_put(struct rdtgroup *rdtgrp, struct kernfs_node *kn)
 {
-	if (atomic_dec_and_test(&rdtgrp->waitcount) &&
-	    (rdtgrp->flags & RDT_DELETED)) {
+	bool needs_free;
+
+	if (!atomic_dec_and_mutex_lock(&rdtgrp->waitcount, &rdtgroup_mutex)) {
+		kernfs_unbreak_active_protection(kn);
+		return;
+	}
+
+	needs_free = rdtgrp->flags & RDT_DELETED;
+
+	mutex_unlock(&rdtgroup_mutex);
+
+	kernfs_unbreak_active_protection(kn);
+
+	if (needs_free) {
 		if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP ||
 		    rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED)
 			rdtgroup_pseudo_lock_remove(rdtgrp);
-		kernfs_unbreak_active_protection(kn);
 		rdtgroup_remove(rdtgrp);
-	} else {
-		kernfs_unbreak_active_protection(kn);
 	}
 }
 
@@ -2652,6 +2727,58 @@ void rdtgroup_kn_unlock(struct kernfs_node *kn)
 	rdtgroup_kn_put(rdtgrp, kn);
 }
 
+/*
+ * Accessing the kn after breaking active protection is safe since the open
+ * of resctrl file holds a kernfs base reference (different from active
+ * protection) on the kn ensuring that it remains accessible even if it was
+ * unlinked. Each kn in turn holds base reference to parent so the kn's
+ * genealogy remains in memory until all base references dropped.
+ */
+static bool is_active_resctrl_node(struct kernfs_node *kn)
+{
+	struct kernfs_node *p;
+	bool match = false;
+
+	guard(rcu)();
+	p = kn;
+	while (p) {
+		if (p == rdtgroup_default.kn) {
+			match = true;
+			break;
+		}
+		p = rcu_dereference(p->__parent);
+	}
+
+	return match;
+}
+
+bool info_kn_lock(struct kernfs_node *kn)
+{
+	kernfs_break_active_protection(kn);
+	cpus_read_lock();
+	mutex_lock(&rdtgroup_mutex);
+
+	/*
+	 * Check both if resctrl is torn down (!rdtgroup_default.kn) and
+	 * if the reader's kernfs_node originates from a dead mount.
+	 */
+	if (!rdtgroup_default.kn || !is_active_resctrl_node(kn)) {
+		mutex_unlock(&rdtgroup_mutex);
+		cpus_read_unlock();
+		kernfs_unbreak_active_protection(kn);
+		return false;
+	}
+
+	return true;
+}
+
+void info_kn_unlock(struct kernfs_node *kn)
+{
+	mutex_unlock(&rdtgroup_mutex);
+	cpus_read_unlock();
+	kernfs_unbreak_active_protection(kn);
+}
+
 static int mkdir_mondata_all(struct kernfs_node *parent_kn,
 			     struct rdtgroup *prgrp,
 			     struct kernfs_node **mon_data_kn);
@@ -2800,73 +2927,288 @@ static void schemata_list_destroy(void)
 	}
 }
 
-static int rdt_get_tree(struct fs_context *fc)
+/*
+ * Move tasks from one to the other group. If @from is NULL, then all tasks
+ * in the systems are moved unconditionally (used for teardown).
+ *
+ * If @mask is not NULL the cpus on which moved tasks are running are set
+ * in that mask so the update smp function call is restricted to affected
+ * cpus.
+ */
+static void rdt_move_group_tasks(struct rdtgroup *from, struct rdtgroup *to,
+				 struct cpumask *mask)
 {
-	struct rdt_fs_context *ctx = rdt_fc2context(fc);
-	unsigned long flags = RFTYPE_CTRL_BASE;
-	struct rdt_l3_mon_domain *dom;
-	struct rdt_resource *r;
-	int ret;
+	struct task_struct *p, *t;
 
-	DO_ONCE_SLEEPABLE(resctrl_arch_pre_mount);
+	read_lock(&tasklist_lock);
+	for_each_process_thread(p, t) {
+		if (!from || is_closid_match(t, from) ||
+		    is_rmid_match(t, from)) {
+			resctrl_arch_set_closid_rmid(t, to->closid,
+						     to->mon.rmid);
 
-	cpus_read_lock();
-	mutex_lock(&rdtgroup_mutex);
-	/*
-	 * resctrl file system can only be mounted once.
-	 */
-	if (resctrl_mounted) {
-		ret = -EBUSY;
-		goto out;
-	}
+			/*
+			 * Order the closid/rmid stores above before the loads
+			 * in task_curr(). This pairs with the full barrier
+			 * between the rq->curr update and
+			 * resctrl_arch_sched_in() during context switch.
+			 */
+			smp_mb();
 
-	/* Avoid races from pending operations from a previous mount */
-	if (atomic_read(&rdtgroup_default.waitcount) != 0) {
-		ret = -EBUSY;
-		goto out;
+			/*
+			 * If the task is on a CPU, set the CPU in the mask.
+			 * The detection is inaccurate as tasks might move or
+			 * schedule before the smp function call takes place.
+			 * In such a case the function call is pointless, but
+			 * there is no other side effect.
+			 */
+			if (IS_ENABLED(CONFIG_SMP) && mask && task_curr(t))
+				cpumask_set_cpu(task_cpu(t), mask);
+		}
 	}
+	read_unlock(&tasklist_lock);
+}
 
-	ret = setup_rmid_lru_list();
-	if (ret)
-		goto out;
-
-	ret = rdtgroup_setup_root(ctx);
-	if (ret)
-		goto out;
+static void free_all_child_rdtgrp(struct rdtgroup *rdtgrp)
+{
+	struct rdtgroup *sentry, *stmp;
+	struct list_head *head;
 
-	ret = rdt_enable_ctx(ctx);
-	if (ret)
-		goto out_root;
+	head = &rdtgrp->mon.crdtgrp_list;
+	list_for_each_entry_safe(sentry, stmp, head, mon.crdtgrp_list) {
+		rdtgroup_unassign_cntrs(sentry);
+		free_rmid(sentry->closid, sentry->mon.rmid);
+		list_del(&sentry->mon.crdtgrp_list);
 
-	ret = schemata_list_create();
-	if (ret)
-		goto out_schemata_free;
+		if (atomic_read(&sentry->waitcount) != 0)
+			sentry->flags = RDT_DELETED;
+		else
+			rdtgroup_remove(sentry);
+	}
+}
 
-	ret = closid_init();
-	if (ret)
-		goto out_schemata_free;
+/*
+ * Forcibly remove all of subdirectories under root.
+ */
+static void rmdir_all_sub(void)
+{
+	struct rdtgroup *rdtgrp, *tmp;
 
-	if (resctrl_arch_mon_capable())
-		flags |= RFTYPE_MON;
+	/* Move all tasks to the default resource group */
+	rdt_move_group_tasks(NULL, &rdtgroup_default, NULL);
 
-	ret = rdtgroup_add_files(rdtgroup_default.kn, flags);
-	if (ret)
-		goto out_closid_exit;
+	list_for_each_entry_safe(rdtgrp, tmp, &rdt_all_groups, rdtgroup_list) {
+		/* Free any child rmids */
+		free_all_child_rdtgrp(rdtgrp);
 
-	kernfs_activate(rdtgroup_default.kn);
+		/* Remove each rdtgroup other than root */
+		if (rdtgrp == &rdtgroup_default)
+			continue;
 
-	ret = rdtgroup_create_info_dir(rdtgroup_default.kn);
-	if (ret < 0)
-		goto out_closid_exit;
+		/*
+		 * Give any CPUs back to the default group. We cannot copy
+		 * cpu_online_mask because a CPU might have executed the
+		 * offline callback already, but is still marked online.
+		 */
+		cpumask_or(&rdtgroup_default.cpu_mask,
+			   &rdtgroup_default.cpu_mask, &rdtgrp->cpu_mask);
 
-	if (resctrl_arch_mon_capable()) {
-		ret = mongroup_create_dir(rdtgroup_default.kn,
-					  &rdtgroup_default, "mon_groups",
-					  &kn_mongrp);
-		if (ret < 0)
-			goto out_info;
+		rdtgroup_unassign_cntrs(rdtgrp);
 
-		rdtgroup_assign_cntrs(&rdtgroup_default);
+		if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP ||
+		    rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) {
+			rdtgroup_pseudo_lock_remove(rdtgrp);
+		} else {
+			/* Pseudo-locked group's RMID is freed during setup. */
+			free_rmid(rdtgrp->closid, rdtgrp->mon.rmid);
+		}
+
+		kernfs_remove(rdtgrp->kn);
+		list_del(&rdtgrp->rdtgroup_list);
+
+		if (atomic_read(&rdtgrp->waitcount) != 0)
+			rdtgrp->flags = RDT_DELETED;
+		else
+			rdtgroup_remove(rdtgrp);
+	}
+	/* Notify online CPUs to update per cpu storage and PQR_ASSOC MSR */
+	update_closid_rmid(cpu_online_mask, &rdtgroup_default);
+
+	kernfs_remove(kn_info);
+	kernfs_remove(kn_mongrp);
+	kernfs_remove(kn_mondata);
+}
+
+/**
+ * mon_get_kn_priv() - Get the mon_data priv data for this event.
+ *
+ * The same values are used across the mon_data directories of all control and
+ * monitor groups for the same event in the same domain. Keep a list of
+ * allocated structures and re-use an existing one with the same values for
+ * @rid, @domid, etc.
+ *
+ * @rid:    The resource id for the event file being created.
+ * @domid:  The domain id for the event file being created.
+ * @mevt:   The type of event file being created.
+ * @do_sum: Whether SNC summing monitors are being created. Only set
+ *	    when @rid == RDT_RESOURCE_L3.
+ *
+ * Return: Pointer to mon_data private data of the event, NULL on failure.
+ */
+static struct mon_data *mon_get_kn_priv(enum resctrl_res_level rid, int domid,
+					struct mon_evt *mevt,
+					bool do_sum)
+{
+	struct mon_data *priv;
+
+	lockdep_assert_held(&rdtgroup_mutex);
+
+	list_for_each_entry(priv, &mon_data_kn_priv_list, list) {
+		if (priv->rid == rid && priv->domid == domid &&
+		    priv->sum == do_sum && priv->evt == mevt)
+			return priv;
+	}
+
+	priv = kzalloc_obj(*priv);
+	if (!priv)
+		return NULL;
+
+	priv->rid = rid;
+	priv->domid = domid;
+	priv->sum = do_sum;
+	priv->evt = mevt;
+	list_add_tail(&priv->list, &mon_data_kn_priv_list);
+
+	return priv;
+}
+
+/**
+ * mon_put_kn_priv() - Free all allocated mon_data structures.
+ *
+ * Called when resctrl file system is unmounted.
+ */
+static void mon_put_kn_priv(void)
+{
+	struct mon_data *priv, *tmp;
+
+	lockdep_assert_held(&rdtgroup_mutex);
+
+	list_for_each_entry_safe(priv, tmp, &mon_data_kn_priv_list, list) {
+		list_del(&priv->list);
+		kfree(priv);
+	}
+}
+
+static void resctrl_fs_teardown(void)
+{
+	lockdep_assert_held(&rdtgroup_mutex);
+
+	/* Cleared by rdtgroup_destroy_root() */
+	if (!rdtgroup_default.kn)
+		return;
+
+	rmdir_all_sub();
+	rdtgroup_unassign_cntrs(&rdtgroup_default);
+	mon_put_kn_priv();
+	rdt_pseudo_lock_release();
+	rdtgroup_default.mode = RDT_MODE_SHAREABLE;
+	rdtgroup_default.flags = RDT_DELETED;
+	closid_exit();
+	schemata_list_destroy();
+	rdtgroup_destroy_root();
+}
+
+static void resctrl_unmount(void)
+{
+	struct rdt_resource *r;
+
+	cpus_read_lock();
+	mutex_lock(&rdtgroup_mutex);
+
+	rdt_disable_ctx();
+
+	/* Put everything back to default values. */
+	for_each_alloc_capable_rdt_resource(r)
+		resctrl_arch_reset_all_ctrls(r);
+
+	resctrl_fs_teardown();
+	if (resctrl_arch_alloc_capable())
+		resctrl_arch_disable_alloc();
+	if (resctrl_arch_mon_capable())
+		resctrl_arch_disable_mon();
+	resctrl_mounted = false;
+	mutex_unlock(&rdtgroup_mutex);
+	cpus_read_unlock();
+}
+
+static int rdt_get_tree(struct fs_context *fc)
+{
+	struct rdt_fs_context *ctx = rdt_fc2context(fc);
+	unsigned long flags = RFTYPE_CTRL_BASE;
+	struct kernfs_node *rdt_root_kn;
+	struct rdt_l3_mon_domain *dom;
+	struct rdt_resource *r;
+	int ret;
+
+	DO_ONCE_SLEEPABLE(resctrl_arch_pre_mount);
+
+	cpus_read_lock();
+	mutex_lock(&rdtgroup_mutex);
+	/*
+	 * resctrl file system can only be mounted once.
+	 */
+	if (resctrl_mounted) {
+		ret = -EBUSY;
+		goto out;
+	}
+
+	/* Avoid races from pending operations from a previous mount */
+	if (atomic_read(&rdtgroup_default.waitcount) != 0) {
+		ret = -EBUSY;
+		goto out;
+	}
+
+	ret = setup_rmid_lru_list();
+	if (ret)
+		goto out;
+
+	ret = rdtgroup_setup_root(ctx);
+	if (ret)
+		goto out;
+
+	ret = rdt_enable_ctx(ctx);
+	if (ret)
+		goto out_root;
+
+	ret = schemata_list_create();
+	if (ret)
+		goto out_schemata_free;
+
+	ret = closid_init();
+	if (ret)
+		goto out_schemata_free;
+
+	if (resctrl_arch_mon_capable())
+		flags |= RFTYPE_MON;
+
+	ret = rdtgroup_add_files(rdtgroup_default.kn, flags);
+	if (ret)
+		goto out_closid_exit;
+
+	kernfs_activate(rdtgroup_default.kn);
+
+	ret = rdtgroup_create_info_dir(rdtgroup_default.kn);
+	if (ret < 0)
+		goto out_closid_exit;
+
+	if (resctrl_arch_mon_capable()) {
+		ret = mongroup_create_dir(rdtgroup_default.kn,
+					  &rdtgroup_default, "mon_groups",
+					  &kn_mongrp);
+		if (ret < 0)
+			goto out_info;
+
+		rdtgroup_assign_cntrs(&rdtgroup_default);
 
 		ret = mkdir_mondata_all(rdtgroup_default.kn,
 					&rdtgroup_default, &kn_mondata);
@@ -2879,10 +3221,6 @@ static int rdt_get_tree(struct fs_context *fc)
 	if (ret)
 		goto out_mondata;
 
-	ret = kernfs_get_tree(fc);
-	if (ret < 0)
-		goto out_psl;
-
 	if (resctrl_arch_alloc_capable())
 		resctrl_arch_enable_alloc();
 	if (resctrl_arch_mon_capable())
@@ -2893,15 +3231,43 @@ static int rdt_get_tree(struct fs_context *fc)
 
 	if (resctrl_is_mbm_enabled()) {
 		r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
-		list_for_each_entry(dom, &r->mon_domains, hdr.list)
+		list_for_each_entry_rcu(dom, &r->mon_domains, hdr.list, lockdep_is_cpus_held())
 			mbm_setup_overflow_handler(dom, MBM_OVERFLOW_INTERVAL,
 						   RESCTRL_PICK_ANY_CPU);
 	}
 
-	goto out;
+	/*
+	 * Ensure root remains accessible after mutex is unlocked so that
+	 * kernfs_kill_sb() can run safely if called by kernfs_get_tree()'s
+	 * failure path after creating a superblock but before taking reference
+	 * on root kn (for example, if unable to get inode for root kn).
+	 */
+	kernfs_get(rdtgroup_default.kn);
+
+	/*
+	 * Make backup of the current root kn being created to be used in
+	 * kernfs_put(). The additional reference taken above will prevent the
+	 * kn from being freed before kernfs_kill_sb() can run but
+	 * rdtgroup_default.kn may be set to NULL via rdtgroup_destroy_root()
+	 * and its backing root (rdt_root) could be overwritten before
+	 * kernfs_put() can run.
+	 */
+	rdt_root_kn = rdtgroup_default.kn;
+
+	rdt_last_cmd_clear();
+	mutex_unlock(&rdtgroup_mutex);
+	cpus_read_unlock();
+
+	ret = kernfs_get_tree(fc);
+	/*
+	 * resctrl can only be mounted once, new superblock only expected
+	 * to be created once.
+	 */
+	if (!ctx->kfc.new_sb_created)
+		resctrl_unmount();
+	kernfs_put(rdt_root_kn);
+	return ret;
 
-out_psl:
-	rdt_pseudo_lock_release();
 out_mondata:
 	if (resctrl_arch_mon_capable())
 		kernfs_remove(kn_mondata);
@@ -2921,7 +3287,6 @@ static int rdt_get_tree(struct fs_context *fc)
 out_root:
 	rdtgroup_destroy_root();
 out:
-	rdt_last_cmd_clear();
 	mutex_unlock(&rdtgroup_mutex);
 	cpus_read_unlock();
 	return ret;
@@ -3006,219 +3371,10 @@ static int rdt_init_fs_context(struct fs_context *fc)
 	return 0;
 }
 
-/*
- * Move tasks from one to the other group. If @from is NULL, then all tasks
- * in the systems are moved unconditionally (used for teardown).
- *
- * If @mask is not NULL the cpus on which moved tasks are running are set
- * in that mask so the update smp function call is restricted to affected
- * cpus.
- */
-static void rdt_move_group_tasks(struct rdtgroup *from, struct rdtgroup *to,
-				 struct cpumask *mask)
-{
-	struct task_struct *p, *t;
-
-	read_lock(&tasklist_lock);
-	for_each_process_thread(p, t) {
-		if (!from || is_closid_match(t, from) ||
-		    is_rmid_match(t, from)) {
-			resctrl_arch_set_closid_rmid(t, to->closid,
-						     to->mon.rmid);
-
-			/*
-			 * Order the closid/rmid stores above before the loads
-			 * in task_curr(). This pairs with the full barrier
-			 * between the rq->curr update and
-			 * resctrl_arch_sched_in() during context switch.
-			 */
-			smp_mb();
-
-			/*
-			 * If the task is on a CPU, set the CPU in the mask.
-			 * The detection is inaccurate as tasks might move or
-			 * schedule before the smp function call takes place.
-			 * In such a case the function call is pointless, but
-			 * there is no other side effect.
-			 */
-			if (IS_ENABLED(CONFIG_SMP) && mask && task_curr(t))
-				cpumask_set_cpu(task_cpu(t), mask);
-		}
-	}
-	read_unlock(&tasklist_lock);
-}
-
-static void free_all_child_rdtgrp(struct rdtgroup *rdtgrp)
-{
-	struct rdtgroup *sentry, *stmp;
-	struct list_head *head;
-
-	head = &rdtgrp->mon.crdtgrp_list;
-	list_for_each_entry_safe(sentry, stmp, head, mon.crdtgrp_list) {
-		rdtgroup_unassign_cntrs(sentry);
-		free_rmid(sentry->closid, sentry->mon.rmid);
-		list_del(&sentry->mon.crdtgrp_list);
-
-		if (atomic_read(&sentry->waitcount) != 0)
-			sentry->flags = RDT_DELETED;
-		else
-			rdtgroup_remove(sentry);
-	}
-}
-
-/*
- * Forcibly remove all of subdirectories under root.
- */
-static void rmdir_all_sub(void)
-{
-	struct rdtgroup *rdtgrp, *tmp;
-
-	/* Move all tasks to the default resource group */
-	rdt_move_group_tasks(NULL, &rdtgroup_default, NULL);
-
-	list_for_each_entry_safe(rdtgrp, tmp, &rdt_all_groups, rdtgroup_list) {
-		/* Free any child rmids */
-		free_all_child_rdtgrp(rdtgrp);
-
-		/* Remove each rdtgroup other than root */
-		if (rdtgrp == &rdtgroup_default)
-			continue;
-
-		/*
-		 * Give any CPUs back to the default group. We cannot copy
-		 * cpu_online_mask because a CPU might have executed the
-		 * offline callback already, but is still marked online.
-		 */
-		cpumask_or(&rdtgroup_default.cpu_mask,
-			   &rdtgroup_default.cpu_mask, &rdtgrp->cpu_mask);
-
-		rdtgroup_unassign_cntrs(rdtgrp);
-
-		if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP ||
-		    rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) {
-			rdtgroup_pseudo_lock_remove(rdtgrp);
-		} else {
-			/* Pseudo-locked group's RMID is freed during setup. */
-			free_rmid(rdtgrp->closid, rdtgrp->mon.rmid);
-		}
-
-		kernfs_remove(rdtgrp->kn);
-		list_del(&rdtgrp->rdtgroup_list);
-
-		if (atomic_read(&rdtgrp->waitcount) != 0)
-			rdtgrp->flags = RDT_DELETED;
-		else
-			rdtgroup_remove(rdtgrp);
-	}
-	/* Notify online CPUs to update per cpu storage and PQR_ASSOC MSR */
-	update_closid_rmid(cpu_online_mask, &rdtgroup_default);
-
-	kernfs_remove(kn_info);
-	kernfs_remove(kn_mongrp);
-	kernfs_remove(kn_mondata);
-}
-
-/**
- * mon_get_kn_priv() - Get the mon_data priv data for this event.
- *
- * The same values are used across the mon_data directories of all control and
- * monitor groups for the same event in the same domain. Keep a list of
- * allocated structures and re-use an existing one with the same values for
- * @rid, @domid, etc.
- *
- * @rid:    The resource id for the event file being created.
- * @domid:  The domain id for the event file being created.
- * @mevt:   The type of event file being created.
- * @do_sum: Whether SNC summing monitors are being created. Only set
- *	    when @rid == RDT_RESOURCE_L3.
- *
- * Return: Pointer to mon_data private data of the event, NULL on failure.
- */
-static struct mon_data *mon_get_kn_priv(enum resctrl_res_level rid, int domid,
-					struct mon_evt *mevt,
-					bool do_sum)
-{
-	struct mon_data *priv;
-
-	lockdep_assert_held(&rdtgroup_mutex);
-
-	list_for_each_entry(priv, &mon_data_kn_priv_list, list) {
-		if (priv->rid == rid && priv->domid == domid &&
-		    priv->sum == do_sum && priv->evt == mevt)
-			return priv;
-	}
-
-	priv = kzalloc_obj(*priv);
-	if (!priv)
-		return NULL;
-
-	priv->rid = rid;
-	priv->domid = domid;
-	priv->sum = do_sum;
-	priv->evt = mevt;
-	list_add_tail(&priv->list, &mon_data_kn_priv_list);
-
-	return priv;
-}
-
-/**
- * mon_put_kn_priv() - Free all allocated mon_data structures.
- *
- * Called when resctrl file system is unmounted.
- */
-static void mon_put_kn_priv(void)
-{
-	struct mon_data *priv, *tmp;
-
-	lockdep_assert_held(&rdtgroup_mutex);
-
-	list_for_each_entry_safe(priv, tmp, &mon_data_kn_priv_list, list) {
-		list_del(&priv->list);
-		kfree(priv);
-	}
-}
-
-static void resctrl_fs_teardown(void)
-{
-	lockdep_assert_held(&rdtgroup_mutex);
-
-	/* Cleared by rdtgroup_destroy_root() */
-	if (!rdtgroup_default.kn)
-		return;
-
-	rmdir_all_sub();
-	rdtgroup_unassign_cntrs(&rdtgroup_default);
-	mon_put_kn_priv();
-	rdt_pseudo_lock_release();
-	rdtgroup_default.mode = RDT_MODE_SHAREABLE;
-	rdtgroup_default.flags = RDT_DELETED;
-	closid_exit();
-	schemata_list_destroy();
-	rdtgroup_destroy_root();
-}
-
 static void rdt_kill_sb(struct super_block *sb)
 {
-	struct rdt_resource *r;
-
-	cpus_read_lock();
-	mutex_lock(&rdtgroup_mutex);
-
-	rdt_disable_ctx();
-
-	/* Put everything back to default values. */
-	for_each_alloc_capable_rdt_resource(r)
-		resctrl_arch_reset_all_ctrls(r);
-
-	resctrl_fs_teardown();
-	if (resctrl_arch_alloc_capable())
-		resctrl_arch_disable_alloc();
-	if (resctrl_arch_mon_capable())
-		resctrl_arch_disable_mon();
-	resctrl_mounted = false;
+	resctrl_unmount();
 	kernfs_kill_sb(sb);
-	mutex_unlock(&rdtgroup_mutex);
-	cpus_read_unlock();
 }
 
 static struct file_system_type rdt_fs_type = {
@@ -3453,7 +3609,7 @@ static int mkdir_mondata_subdir_alldom(struct kernfs_node *parent_kn,
 	/* Walking r->domains, ensure it can't race with cpuhp */
 	lockdep_assert_cpus_held();
 
-	list_for_each_entry(hdr, &r->mon_domains, list) {
+	list_for_each_entry_rcu(hdr, &r->mon_domains, list, lockdep_is_cpus_held()) {
 		ret = mkdir_mondata_subdir(parent_kn, hdr, r, prgrp);
 		if (ret)
 			return ret;
@@ -3638,7 +3794,7 @@ int rdtgroup_init_cat(struct resctrl_schema *s, u32 closid)
 	struct rdt_ctrl_domain *d;
 	int ret;
 
-	list_for_each_entry(d, &s->res->ctrl_domains, hdr.list) {
+	list_for_each_entry_rcu(d, &s->res->ctrl_domains, hdr.list, lockdep_is_cpus_held()) {
 		ret = __init_one_rdt_domain(d, s, closid);
 		if (ret < 0)
 			return ret;
@@ -3653,7 +3809,7 @@ static void rdtgroup_init_mba(struct rdt_resource *r, u32 closid)
 	struct resctrl_staged_config *cfg;
 	struct rdt_ctrl_domain *d;
 
-	list_for_each_entry(d, &r->ctrl_domains, hdr.list) {
+	list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) {
 		if (is_mba_sc(r)) {
 			d->mbps_val[closid] = MBA_MAX_MBPS;
 			continue;
@@ -4333,6 +4489,29 @@ static void domain_destroy_l3_mon_state(struct rdt_l3_mon_domain *d)
 
 void resctrl_offline_ctrl_domain(struct rdt_resource *r, struct rdt_ctrl_domain *d)
 {
+	/*
+	 * mbm_handle_overflow() may dereference this ctrl domain via
+	 * update_mba_bw()->get_sc_ctrl_domain_from_cpu(). The architecture has
+	 * unlinked the domain from the RCU list and waited a grace period, so
+	 * no new worker iteration can find it; drain any worker that already
+	 * holds a pointer to it before the architecture frees the domain.
+	 *
+	 * Software controller is enabled/disabled on mount/unmount with
+	 * cpus_read_lock() held. Running here with cpus_write_lock() so
+	 * there are no concurrent changes to software controller status.
+	 */
+	if (r->rid == RDT_RESOURCE_MBA && is_mba_sc(r)) {
+		struct rdt_resource *l3 = resctrl_arch_get_resource(RDT_RESOURCE_L3);
+		struct rdt_l3_mon_domain *mon_d;
+
+		list_for_each_entry_rcu(mon_d, &l3->mon_domains, hdr.list, lockdep_is_cpus_held()) {
+			if (mon_d->hdr.id == d->hdr.id) {
+				cancel_delayed_work_sync(&mon_d->mbm_over);
+				break;
+			}
+		}
+	}
+
 	mutex_lock(&rdtgroup_mutex);
 
 	if (supports_mba_mbps() && r->rid == RDT_RESOURCE_MBA)
@@ -4345,6 +4524,24 @@ void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_domain_hdr *h
 {
 	struct rdt_l3_mon_domain *d;
 
+	/*
+	 * Called by architecture under CPU hotplug lock as it prepares to remove
+	 * the domain which is guaranteed to be accessible here.
+	 * The domain has been unlinked from the RCU list and a grace period
+	 * has elapsed, so no new worker can be scheduled. Drain any worker that
+	 * is in flight or pending before letting architecture proceed to free
+	 * the domain that has the workers' struct delayed_work embedded.
+	 * Do so before taking rdtgroup_mutex since the workers also acquire it.
+	 */
+	if (r->rid == RDT_RESOURCE_L3 &&
+	    domain_header_is_valid(hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) {
+		d = container_of(hdr, struct rdt_l3_mon_domain, hdr);
+		if (resctrl_is_mbm_enabled())
+			cancel_delayed_work_sync(&d->mbm_over);
+		if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID))
+			cancel_delayed_work_sync(&d->cqm_limbo);
+	}
+
 	mutex_lock(&rdtgroup_mutex);
 
 	/*
@@ -4361,8 +4558,6 @@ void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_domain_hdr *h
 		goto out_unlock;
 
 	d = container_of(hdr, struct rdt_l3_mon_domain, hdr);
-	if (resctrl_is_mbm_enabled())
-		cancel_delayed_work(&d->mbm_over);
 	if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID) && has_busy_rmid(d)) {
 		/*
 		 * When a package is going down, forcefully
@@ -4373,7 +4568,6 @@ void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_domain_hdr *h
 		 * package never comes back.
 		 */
 		__check_limbo(d, true);
-		cancel_delayed_work(&d->cqm_limbo);
 	}
 
 	domain_destroy_l3_mon_state(d);
@@ -4525,7 +4719,7 @@ static struct rdt_l3_mon_domain *get_mon_domain_from_cpu(int cpu,
 
 	lockdep_assert_cpus_held();
 
-	list_for_each_entry(d, &r->mon_domains, hdr.list) {
+	list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) {
 		/* Find the domain that contains this CPU */
 		if (cpumask_test_cpu(cpu, &d->hdr.cpu_mask))
 			return d;
@@ -4554,12 +4748,16 @@ void resctrl_offline_cpu(unsigned int cpu)
 	d = get_mon_domain_from_cpu(cpu, l3);
 	if (d) {
 		if (resctrl_is_mbm_enabled() && cpu == d->mbm_work_cpu) {
-			cancel_delayed_work(&d->mbm_over);
+			mutex_unlock(&rdtgroup_mutex);
+			cancel_delayed_work_sync(&d->mbm_over);
+			mutex_lock(&rdtgroup_mutex);
 			mbm_setup_overflow_handler(d, 0, cpu);
 		}
 		if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID) &&
 		    cpu == d->cqm_work_cpu && has_busy_rmid(d)) {
-			cancel_delayed_work(&d->cqm_limbo);
+			mutex_unlock(&rdtgroup_mutex);
+			cancel_delayed_work_sync(&d->cqm_limbo);
+			mutex_lock(&rdtgroup_mutex);
 			cqm_setup_limbo_handler(d, 0, cpu);
 		}
 	}
diff --git a/fs/romfs/super.c b/fs/romfs/super.c
index ac55193bf3988..7f783d6173e8a 100644
--- a/fs/romfs/super.c
+++ b/fs/romfs/super.c
@@ -240,6 +240,8 @@ static struct dentry *romfs_lookup(struct inode *dir, struct dentry *dentry,
 			if ((be32_to_cpu(ri.next) & ROMFH_TYPE) == ROMFH_HRD)
 				offset = be32_to_cpu(ri.spec) & ROMFH_MASK;
 			inode = romfs_iget(dir->i_sb, offset);
+			if (IS_ERR(inode))
+				return ERR_CAST(inode);
 			break;
 		}
 
@@ -262,6 +264,8 @@ static const struct inode_operations romfs_dir_inode_operations = {
 	.lookup		= romfs_lookup,
 };
 
+#define ROMFS_MAX_HARDLINK_DEPTH 64
+
 /*
  * get a romfs inode based on its position in the image (which doubles as the
  * inode number)
@@ -273,6 +277,7 @@ static struct inode *romfs_iget(struct super_block *sb, unsigned long pos)
 	struct inode *i;
 	unsigned long nlen;
 	unsigned nextfh;
+	unsigned int depth = 0;
 	int ret;
 	umode_t mode;
 
@@ -289,6 +294,9 @@ static struct inode *romfs_iget(struct super_block *sb, unsigned long pos)
 		if ((nextfh & ROMFH_TYPE) != ROMFH_HRD)
 			break;
 
+		if (++depth > ROMFS_MAX_HARDLINK_DEPTH)
+			return ERR_PTR(-ELOOP);
+
 		pos = be32_to_cpu(ri.spec) & ROMFH_MASK;
 	}
 
@@ -587,7 +595,7 @@ static void romfs_kill_sb(struct super_block *sb)
 #ifdef CONFIG_ROMFS_ON_BLOCK
 	if (sb->s_bdev) {
 		sync_blockdev(sb->s_bdev);
-		bdev_fput(sb->s_bdev_file);
+		fs_bdev_file_release(sb->s_bdev_file, sb);
 	}
 #endif
 }
diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c
index 4df6ca03a8de2..66b9104e7ca20 100644
--- a/fs/smb/client/cifsfs.c
+++ b/fs/smb/client/cifsfs.c
@@ -1490,9 +1490,13 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off,
 		}
 	}
 
-	/* force revalidate of size and timestamps of target file now
-	   that target is updated on the server */
-	CIFS_I(target_inode)->time = 0;
+	/*
+	 * On success, duplicate_extents already updated the target inode attrs
+	 * or marked them stale if the refresh failed.  On failure, mark attrs
+	 * stale because EOF may have changed before the clone failed.
+	 */
+	if (rc)
+		CIFS_I(target_inode)->time = 0;
 unlock:
 	/* although unlocking in the reverse order from locking is not
 	   strictly necessary here it is a little cleaner to be consistent */
diff --git a/fs/smb/client/cifsfs.h b/fs/smb/client/cifsfs.h
index 854e672a4e37a..287c632392c4b 100644
--- a/fs/smb/client/cifsfs.h
+++ b/fs/smb/client/cifsfs.h
@@ -54,7 +54,7 @@ void cifs_sb_deactive(struct super_block *sb);
 extern const struct inode_operations cifs_dir_inode_ops;
 struct inode *cifs_root_iget(struct super_block *sb);
 int cifs_create(struct mnt_idmap *idmap, struct inode *dir,
-		struct dentry *direntry, umode_t mode, bool excl);
+		struct dentry *direntry, umode_t mode);
 int cifs_atomic_open(struct inode *dir, struct dentry *direntry,
 		     struct file *file, unsigned int oflags, umode_t mode);
 int cifs_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
diff --git a/fs/smb/client/dir.c b/fs/smb/client/dir.c
index 88a4a1787ff04..7803bd5bd01f8 100644
--- a/fs/smb/client/dir.c
+++ b/fs/smb/client/dir.c
@@ -645,7 +645,7 @@ int cifs_atomic_open(struct inode *dir, struct dentry *direntry,
  * hashed-positive by calling d_instantiate().
  */
 int cifs_create(struct mnt_idmap *idmap, struct inode *dir,
-		struct dentry *direntry, umode_t mode, bool excl)
+		struct dentry *direntry, umode_t mode)
 {
 	struct cifs_sb_info *cifs_sb = CIFS_SB(dir);
 	int rc;
diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c
index 968740e7c9c3a..081bb16abeb4b 100644
--- a/fs/smb/client/file.c
+++ b/fs/smb/client/file.c
@@ -3392,9 +3392,7 @@ static int cifs_swap_activate(struct swap_info_struct *sis,
 	 * but we could add call to grab a byte range lock to prevent others
 	 * from reading or writing the file
 	 */
-
-	sis->flags |= SWP_FS_OPS;
-	return add_swap_extent(sis, 0, sis->max, 0);
+	return swap_fs_activate(sis);
 }
 
 static void cifs_swap_deactivate(struct file *file)
diff --git a/fs/smb/client/inode.c b/fs/smb/client/inode.c
index deed04dd9b914..4e8e4a23e29df 100644
--- a/fs/smb/client/inode.c
+++ b/fs/smb/client/inode.c
@@ -2285,6 +2285,13 @@ struct dentry *cifs_mkdir(struct mnt_idmap *idmap, struct inode *inode,
 	const char *full_path;
 	void *page;
 
+	/*
+	 * vfs_mkdir() now passes S_IFDIR in @mode, but @mode is forwarded
+	 * verbatim to the server and in the past only contained permission
+	 * bits. Strip the type bit until SMB is verified to deal with it.
+	 */
+	mode &= ~S_IFDIR;
+
 	cifs_dbg(FYI, "In cifs_mkdir, mode = %04ho inode = 0x%p\n",
 		 mode, inode);
 
diff --git a/fs/smb/client/smb1maperror.c b/fs/smb/client/smb1maperror.c
index 74530088d17d0..ab3d09613c919 100644
--- a/fs/smb/client/smb1maperror.c
+++ b/fs/smb/client/smb1maperror.c
@@ -10,6 +10,9 @@
  */
 
 #include <linux/bsearch.h>
+
+#include <kunit/visibility.h>
+
 #include "cifsproto.h"
 #include "smb1proto.h"
 #include "smberr.h"
@@ -239,48 +242,45 @@ int __init smb1_init_maperror(void)
 }
 
 #if IS_ENABLED(CONFIG_SMB1_KUNIT_TESTS)
-#define EXPORT_SYMBOL_FOR_SMB_TEST(sym) \
-	EXPORT_SYMBOL_FOR_MODULES(sym, "smb1maperror_test")
-
 const struct ntstatus_to_dos_err *
 search_ntstatus_to_dos_map_test(__u32 ntstatus)
 {
 	return search_ntstatus_to_dos_map(ntstatus);
 }
-EXPORT_SYMBOL_FOR_SMB_TEST(search_ntstatus_to_dos_map_test);
+EXPORT_SYMBOL_IF_KUNIT(search_ntstatus_to_dos_map_test);
 
 const struct ntstatus_to_dos_err *
 ntstatus_to_dos_map_test = ntstatus_to_dos_map;
-EXPORT_SYMBOL_FOR_SMB_TEST(ntstatus_to_dos_map_test);
+EXPORT_SYMBOL_IF_KUNIT(ntstatus_to_dos_map_test);
 
 unsigned int ntstatus_to_dos_num = ARRAY_SIZE(ntstatus_to_dos_map);
-EXPORT_SYMBOL_FOR_SMB_TEST(ntstatus_to_dos_num);
+EXPORT_SYMBOL_IF_KUNIT(ntstatus_to_dos_num);
 
 const struct smb_to_posix_error *
 search_mapping_table_ERRDOS_test(__u16 smb_err)
 {
 	return search_mapping_table_ERRDOS(smb_err);
 }
-EXPORT_SYMBOL_FOR_SMB_TEST(search_mapping_table_ERRDOS_test);
+EXPORT_SYMBOL_IF_KUNIT(search_mapping_table_ERRDOS_test);
 
 const struct smb_to_posix_error *
 mapping_table_ERRDOS_test = mapping_table_ERRDOS;
-EXPORT_SYMBOL_FOR_SMB_TEST(mapping_table_ERRDOS_test);
+EXPORT_SYMBOL_IF_KUNIT(mapping_table_ERRDOS_test);
 
 unsigned int mapping_table_ERRDOS_num = ARRAY_SIZE(mapping_table_ERRDOS);
-EXPORT_SYMBOL_FOR_SMB_TEST(mapping_table_ERRDOS_num);
+EXPORT_SYMBOL_IF_KUNIT(mapping_table_ERRDOS_num);
 
 const struct smb_to_posix_error *
 search_mapping_table_ERRSRV_test(__u16 smb_err)
 {
 	return search_mapping_table_ERRSRV(smb_err);
 }
-EXPORT_SYMBOL_FOR_SMB_TEST(search_mapping_table_ERRSRV_test);
+EXPORT_SYMBOL_IF_KUNIT(search_mapping_table_ERRSRV_test);
 
 const struct smb_to_posix_error *
 mapping_table_ERRSRV_test = mapping_table_ERRSRV;
-EXPORT_SYMBOL_FOR_SMB_TEST(mapping_table_ERRSRV_test);
+EXPORT_SYMBOL_IF_KUNIT(mapping_table_ERRSRV_test);
 
 unsigned int mapping_table_ERRSRV_num = ARRAY_SIZE(mapping_table_ERRSRV);
-EXPORT_SYMBOL_FOR_SMB_TEST(mapping_table_ERRSRV_num);
+EXPORT_SYMBOL_IF_KUNIT(mapping_table_ERRSRV_num);
 #endif
diff --git a/fs/smb/client/smb1maperror_test.c b/fs/smb/client/smb1maperror_test.c
index 2caaf11228ef5..903c46f712913 100644
--- a/fs/smb/client/smb1maperror_test.c
+++ b/fs/smb/client/smb1maperror_test.c
@@ -75,3 +75,4 @@ kunit_test_suite(maperror_suite);
 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("KUnit tests of SMB1 maperror");
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
diff --git a/fs/smb/client/smb2maperror.c b/fs/smb/client/smb2maperror.c
index 9ed21f7b618c5..d86f2460d0e5e 100644
--- a/fs/smb/client/smb2maperror.c
+++ b/fs/smb/client/smb2maperror.c
@@ -8,6 +8,9 @@
  *
  */
 #include <linux/errno.h>
+
+#include <kunit/visibility.h>
+
 #include "cifsproto.h"
 #include "cifs_debug.h"
 #include "smb2proto.h"
@@ -109,18 +112,15 @@ int __init smb2_init_maperror(void)
 }
 
 #if IS_ENABLED(CONFIG_SMB_KUNIT_TESTS)
-#define EXPORT_SYMBOL_FOR_SMB_TEST(sym) \
-	EXPORT_SYMBOL_FOR_MODULES(sym, "smb2maperror_test")
-
 const struct status_to_posix_error *smb2_get_err_map_test(__u32 smb2_status)
 {
 	return smb2_get_err_map(smb2_status);
 }
-EXPORT_SYMBOL_FOR_SMB_TEST(smb2_get_err_map_test);
+EXPORT_SYMBOL_IF_KUNIT(smb2_get_err_map_test);
 
 const struct status_to_posix_error *smb2_error_map_table_test = smb2_error_map_table;
-EXPORT_SYMBOL_FOR_SMB_TEST(smb2_error_map_table_test);
+EXPORT_SYMBOL_IF_KUNIT(smb2_error_map_table_test);
 
 unsigned int smb2_error_map_num = ARRAY_SIZE(smb2_error_map_table);
-EXPORT_SYMBOL_FOR_SMB_TEST(smb2_error_map_num);
+EXPORT_SYMBOL_IF_KUNIT(smb2_error_map_num);
 #endif
diff --git a/fs/smb/client/smb2maperror_test.c b/fs/smb/client/smb2maperror_test.c
index 0f8a44a5ed3cf..44dc5e899cad0 100644
--- a/fs/smb/client/smb2maperror_test.c
+++ b/fs/smb/client/smb2maperror_test.c
@@ -47,3 +47,4 @@ kunit_test_suite(maperror_suite);
 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("KUnit tests of SMB2 maperror");
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
diff --git a/fs/smb/client/smb2misc.c b/fs/smb/client/smb2misc.c
index 6270b33147d25..9068175e57cd0 100644
--- a/fs/smb/client/smb2misc.c
+++ b/fs/smb/client/smb2misc.c
@@ -19,7 +19,8 @@
 #include "nterr.h"
 #include "cached_dir.h"
 
-static unsigned int __smb2_calc_size(void *buf, bool *have_data);
+static unsigned int __smb2_calc_size(void *buf, bool *have_data,
+				     bool *data_area_overlap);
 
 static int
 check_smb2_hdr(struct smb2_hdr *shdr, __u64 mid)
@@ -148,6 +149,7 @@ smb2_check_message(char *buf, unsigned int pdu_len, unsigned int len,
 	__u32 calc_len; /* calculated length */
 	__u64 mid;
 	bool have_data;
+	bool data_area_overlap;
 
 	/* If server is a channel, select the primary channel */
 	pserver = SERVER_IS_CHAN(server) ? server->primary_server : server;
@@ -232,7 +234,12 @@ smb2_check_message(char *buf, unsigned int pdu_len, unsigned int len,
 	}
 
 	have_data = false;
-	calc_len = __smb2_calc_size(buf, &have_data);
+	data_area_overlap = false;
+	calc_len = __smb2_calc_size(buf, &have_data, &data_area_overlap);
+
+	/* Reject responses whose data area overlaps the fixed area. */
+	if (data_area_overlap)
+		return 1;
 
 	/* For SMB2_IOCTL, OutputOffset and OutputLength are optional, so might
 	 * be 0, and not a real miscalculation */
@@ -416,14 +423,15 @@ smb2_get_data_area_len(int *off, int *len, struct smb2_hdr *shdr)
 }
 
 /*
- * Calculate the size of the SMB message based on the fixed header
- * portion, the number of word parameters and the data portion of the message.
- * If have_data is non-NULL, it is set to true when a non-empty data area was
- * found (data_length > 0), allowing callers to distinguish the implied bcc[0]
- * case (no data area) from an overreported data length.
+ * Calculate the size of the SMB message based on the fixed header, fixed
+ * parameter area, and variable data area.
+ *
+ * If have_data is not NULL, it is set when a non-empty data area is found.
+ * If data_area_overlap is not NULL, it is set when the data area overlaps
+ * the fixed area.
  */
 static unsigned int
-__smb2_calc_size(void *buf, bool *have_data)
+__smb2_calc_size(void *buf, bool *have_data, bool *data_area_overlap)
 {
 	struct smb2_pdu *pdu = buf;
 	struct smb2_hdr *shdr = &pdu->hdr;
@@ -432,6 +440,11 @@ __smb2_calc_size(void *buf, bool *have_data)
 	/* Structure Size has already been checked to make sure it is 64 */
 	int len = le16_to_cpu(shdr->StructureSize);
 
+	if (have_data)
+		*have_data = false;
+	if (data_area_overlap)
+		*data_area_overlap = false;
+
 	/*
 	 * StructureSize2, ie length of fixed parameter area has already
 	 * been checked to make sure it is the correct length.
@@ -454,7 +467,10 @@ __smb2_calc_size(void *buf, bool *have_data)
 		if (offset + 1 < len) {
 			cifs_dbg(VFS, "data area offset %d overlaps SMB2 header %d\n",
 				 offset + 1, len);
+			if (data_area_overlap)
+				*data_area_overlap = true;
 			data_length = 0;
+			goto calc_size_exit;
 		} else {
 			len = offset + data_length;
 		}
@@ -469,7 +485,7 @@ __smb2_calc_size(void *buf, bool *have_data)
 unsigned int
 smb2_calc_size(void *buf)
 {
-	return __smb2_calc_size(buf, NULL);
+	return __smb2_calc_size(buf, NULL, NULL);
 }
 
 /* Note: caller must free return buffer */
diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c
index 6bce44b171e43..cbd51a08e97e6 100644
--- a/fs/smb/client/smb2ops.c
+++ b/fs/smb/client/smb2ops.c
@@ -2193,10 +2193,14 @@ smb2_duplicate_extents(const unsigned int xid,
 			u64 len, u64 dest_off)
 {
 	int rc;
+	int qrc;
 	unsigned int ret_data_len;
 	struct inode *inode;
+	struct smb2_file_all_info file_inf;
 	struct duplicate_extents_to_file dup_ext_buf;
+	struct timespec64 ts;
 	struct cifs_tcon *tcon = tlink_tcon(trgtfile->tlink);
+	u64 asize;
 
 	/* server fileays advertise duplicate extent support with this flag */
 	if ((le32_to_cpu(tcon->fsAttrInfo.Attributes) &
@@ -2232,6 +2236,32 @@ smb2_duplicate_extents(const unsigned int xid,
 	if (ret_data_len > 0)
 		cifs_dbg(FYI, "Non-zero response length in duplicate extents\n");
 
+	if (rc == 0) {
+		qrc = SMB2_query_info(xid, tcon, trgtfile->fid.persistent_fid,
+				      trgtfile->fid.volatile_fid, &file_inf);
+		spin_lock(&inode->i_lock);
+		if (qrc == 0) {
+			asize = le64_to_cpu(file_inf.AllocationSize);
+			CIFS_I(inode)->time = jiffies;
+			if (file_inf.LastWriteTime) {
+				ts = cifs_NTtimeToUnix(file_inf.LastWriteTime);
+				inode_set_mtime_to_ts(inode, ts);
+			}
+			if (file_inf.ChangeTime) {
+				ts = cifs_NTtimeToUnix(file_inf.ChangeTime);
+				inode_set_ctime_to_ts(inode, ts);
+			}
+			if (file_inf.LastAccessTime) {
+				ts = cifs_NTtimeToUnix(file_inf.LastAccessTime);
+				inode_set_atime_to_ts(inode, ts);
+			}
+			inode->i_blocks = CIFS_INO_BLOCKS(asize);
+		} else {
+			CIFS_I(inode)->time = 0; /* force reval */
+		}
+		spin_unlock(&inode->i_lock);
+	}
+
 duplicate_extents_out:
 	if (rc)
 		trace_smb3_clone_err(xid, srcfile->fid.volatile_fid,
@@ -3489,6 +3519,15 @@ static long smb3_punch_hole(struct file *file, struct cifs_tcon *tcon,
 		goto out;
 
 	filemap_invalidate_lock(inode->i_mapping);
+	/*
+	 * Flush dirty data first, otherwise a dirty folio spanning the punched
+	 * range may be written back after the ioctl and refill the hole.
+	 */
+	rc = filemap_write_and_wait_range(inode->i_mapping, offset,
+					  offset + len - 1);
+	if (rc < 0)
+		goto unlock;
+
 	/*
 	 * We implement the punch hole through ioctl, so we need remove the page
 	 * caches first, otherwise the data may be inconsistent with the server.
@@ -3543,7 +3582,7 @@ static int smb3_simple_fallocate_write_range(unsigned int xid,
 					     char *buf)
 {
 	struct cifs_io_parms io_parms = {0};
-	int nbytes;
+	unsigned int nbytes;
 	int rc = 0;
 	struct kvec iov[2];
 
@@ -3564,9 +3603,10 @@ static int smb3_simple_fallocate_write_range(unsigned int xid,
 		rc = SMB2_write(xid, &io_parms, &nbytes, iov, 1);
 		if (rc)
 			break;
+		if (!nbytes)
+			return -EIO;
 		if (nbytes > len)
 			return -EINVAL;
-		buf += nbytes;
 		off += nbytes;
 		len -= nbytes;
 	}
@@ -3579,11 +3619,25 @@ static int smb3_simple_fallocate_range(unsigned int xid,
 				       loff_t off, loff_t len)
 {
 	struct file_allocated_range_buffer in_data, *out_data = NULL, *tmp_data;
+	struct inode *inode = d_inode(cfile->dentry);
 	u32 out_data_len;
 	char *buf = NULL;
+	u64 range_start, range_len, range_end;
 	loff_t l;
 	int rc;
 
+	buf = kvzalloc(min_t(loff_t, len, SMB2_MAX_BUFFER_SIZE), GFP_KERNEL);
+	if (!buf) {
+		rc = -ENOMEM;
+		goto out;
+	}
+
+	if (off >= i_size_read(inode)) {
+		rc = smb3_simple_fallocate_write_range(xid, tcon, cfile,
+						       off, len, buf);
+		goto out;
+	}
+
 	in_data.file_offset = cpu_to_le64(off);
 	in_data.length = cpu_to_le64(len);
 	rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid,
@@ -3595,12 +3649,6 @@ static int smb3_simple_fallocate_range(unsigned int xid,
 	if (rc)
 		goto out;
 
-	buf = kzalloc(1024 * 1024, GFP_KERNEL);
-	if (buf == NULL) {
-		rc = -ENOMEM;
-		goto out;
-	}
-
 	tmp_data = out_data;
 	while (len) {
 		/*
@@ -3617,13 +3665,21 @@ static int smb3_simple_fallocate_range(unsigned int xid,
 			goto out;
 		}
 
-		if (off < le64_to_cpu(tmp_data->file_offset)) {
+		range_start = le64_to_cpu(tmp_data->file_offset);
+		range_len = le64_to_cpu(tmp_data->length);
+		if (check_add_overflow(range_start, range_len, &range_end) ||
+		    range_end > S64_MAX) {
+			rc = -EINVAL;
+			goto out;
+		}
+
+		if (off < range_start) {
 			/*
 			 * We are at a hole. Write until the end of the region
 			 * or until the next allocated data,
 			 * whichever comes next.
 			 */
-			l = le64_to_cpu(tmp_data->file_offset) - off;
+			l = range_start - off;
 			if (len < l)
 				l = len;
 			rc = smb3_simple_fallocate_write_range(xid, tcon,
@@ -3640,11 +3696,13 @@ static int smb3_simple_fallocate_range(unsigned int xid,
 		 * until the end of the data or the end of the region
 		 * we are supposed to fallocate, whichever comes first.
 		 */
-		l = le64_to_cpu(tmp_data->length);
-		if (len < l)
-			l = len;
-		off += l;
-		len -= l;
+		if (off < range_end) {
+			l = range_end - off;
+			if (len < l)
+				l = len;
+			off += l;
+			len -= l;
+		}
 
 		tmp_data = &tmp_data[1];
 		out_data_len -= sizeof(struct file_allocated_range_buffer);
@@ -3652,7 +3710,7 @@ static int smb3_simple_fallocate_range(unsigned int xid,
 
  out:
 	kfree(out_data);
-	kfree(buf);
+	kvfree(buf);
 	return rc;
 }
 
@@ -3665,18 +3723,22 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon,
 	struct cifsFileInfo *cfile = file->private_data;
 	long rc = -EOPNOTSUPP;
 	unsigned int xid;
-	loff_t new_eof;
+	loff_t old_eof, new_eof;
+	struct smb2_file_all_info file_inf;
+	u64 asize;
+	int qrc;
 
 	xid = get_xid();
 
 	inode = d_inode(cfile->dentry);
 	cifsi = CIFS_I(inode);
+	old_eof = i_size_read(inode);
 
 	trace_smb3_falloc_enter(xid, cfile->fid.persistent_fid, tcon->tid,
 				tcon->ses->Suid, off, len);
 	/* if file not oplocked can't be sure whether asking to extend size */
 	if (!CIFS_CACHE_READ(cifsi))
-		if (keep_size == false) {
+		if (!keep_size) {
 			trace_smb3_falloc_err(xid, cfile->fid.persistent_fid,
 				tcon->tid, tcon->ses->Suid, off, len, rc);
 			free_xid(xid);
@@ -3686,21 +3748,98 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon,
 	/*
 	 * Extending the file
 	 */
-	if ((keep_size == false) && i_size_read(inode) < off + len) {
+	if (!keep_size && old_eof < off + len) {
 		rc = inode_newsize_ok(inode, off + len);
 		if (rc)
 			goto out;
 
+		/*
+		 * A small range at or beyond EOF can be allocated by writing
+		 * zeroes.  For off > old_eof, this preserves the intervening
+		 * hole instead of allocating from offset 0.
+		 */
+		if (off > old_eof ||
+		    (off == old_eof && old_eof != 0 &&
+		     (cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE))) {
+			if (len > 1024 * 1024) {
+				rc = -EOPNOTSUPP;
+				goto out;
+			}
+
+			rc = smb3_simple_fallocate_range(xid, tcon, cfile,
+							 off, len);
+			if (rc) {
+				spin_lock(&inode->i_lock);
+				cifsi->time = 0;
+				spin_unlock(&inode->i_lock);
+				goto out;
+			}
+
+			new_eof = off + len;
+			netfs_resize_file(&cifsi->netfs, new_eof, true);
+			cifs_setsize(inode, new_eof);
+
+			qrc = SMB2_query_info(xid, tcon,
+					      cfile->fid.persistent_fid,
+					      cfile->fid.volatile_fid, &file_inf);
+			spin_lock(&inode->i_lock);
+			if (qrc == 0) {
+				asize = le64_to_cpu(file_inf.AllocationSize);
+				inode->i_blocks = CIFS_INO_BLOCKS(asize);
+			} else {
+				cifsi->time = 0;
+			}
+			spin_unlock(&inode->i_lock);
+			goto out;
+		}
+
 		if (cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE)
 			smb2_set_sparse(xid, tcon, cfile, inode, false);
 
 		new_eof = off + len;
+
+		qrc = SMB2_query_info(xid, tcon,
+				      cfile->fid.persistent_fid,
+				      cfile->fid.volatile_fid, &file_inf);
+		if (qrc == 0)
+			asize = le64_to_cpu(file_inf.AllocationSize);
+
+		/*
+		 * FILE_ALLOCATION_INFORMATION can only describe allocation up to
+		 * new_eof. Some servers may accept it without allocating blocks,
+		 * so refresh AllocationSize before updating i_blocks.
+		 */
+		if (off == 0 || off == old_eof) {
+			if (qrc || asize < new_eof) {
+				rc = SMB2_set_allocation(xid, tcon,
+							 cfile->fid.persistent_fid,
+							 cfile->fid.volatile_fid,
+							 cfile->pid, new_eof);
+				if (rc)
+					goto out;
+			}
+		}
+
 		rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid,
 				  cfile->fid.volatile_fid, cfile->pid, new_eof);
-		if (rc == 0) {
-			netfs_resize_file(&cifsi->netfs, new_eof, true);
-			cifs_setsize(inode, new_eof);
+		if (rc)
+			goto out;
+
+		netfs_resize_file(&cifsi->netfs, new_eof, true);
+		cifs_setsize(inode, new_eof);
+
+		qrc = SMB2_query_info(xid, tcon,
+				      cfile->fid.persistent_fid,
+				      cfile->fid.volatile_fid, &file_inf);
+		spin_lock(&inode->i_lock);
+		if (qrc == 0) {
+			asize = le64_to_cpu(file_inf.AllocationSize);
+			if (asize >= new_eof)
+				inode->i_blocks = CIFS_INO_BLOCKS(asize);
+		} else {
+			cifsi->time = 0;
 		}
+		spin_unlock(&inode->i_lock);
 		goto out;
 	}
 
diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c
index 8f83ab377db14..4ce165e40657f 100644
--- a/fs/smb/client/smb2pdu.c
+++ b/fs/smb/client/smb2pdu.c
@@ -5949,6 +5949,25 @@ SMB2_set_eof(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid,
 			0, 1, &data, &size);
 }
 
+int
+SMB2_set_allocation(const unsigned int xid, struct cifs_tcon *tcon,
+		    u64 persistent_fid, u64 volatile_fid, u32 pid,
+		    loff_t allocation_size)
+{
+	struct smb2_file_alloc_info info;
+	void *data;
+	unsigned int size;
+
+	info.AllocationSize = cpu_to_le64(allocation_size);
+
+	data = &info;
+	size = sizeof(struct smb2_file_alloc_info);
+
+	return send_set_info(xid, tcon, persistent_fid, volatile_fid,
+			pid, FILE_ALLOCATION_INFORMATION, SMB2_O_INFO_FILE,
+			0, 1, &data, &size);
+}
+
 int
 SMB2_set_acl(const unsigned int xid, struct cifs_tcon *tcon,
 		u64 persistent_fid, u64 volatile_fid,
diff --git a/fs/smb/client/smb2proto.h b/fs/smb/client/smb2proto.h
index e01effe45ae2e..2e9f700968251 100644
--- a/fs/smb/client/smb2proto.h
+++ b/fs/smb/client/smb2proto.h
@@ -204,6 +204,9 @@ void SMB2_query_directory_free(struct smb_rqst *rqst);
 int SMB2_set_eof(const unsigned int xid, struct cifs_tcon *tcon,
 		 u64 persistent_fid, u64 volatile_fid, u32 pid,
 		 loff_t new_eof);
+int SMB2_set_allocation(const unsigned int xid, struct cifs_tcon *tcon,
+			u64 persistent_fid, u64 volatile_fid, u32 pid,
+			loff_t allocation_size);
 int SMB2_set_info_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server,
 		       struct smb_rqst *rqst, u64 persistent_fid,
 		       u64 volatile_fid, u32 pid, u8 info_class, u8 info_type,
diff --git a/fs/smb/common/fscc.h b/fs/smb/common/fscc.h
index 859849a42fec5..941db5a95564b 100644
--- a/fs/smb/common/fscc.h
+++ b/fs/smb/common/fscc.h
@@ -283,6 +283,11 @@ struct smb2_file_eof_info { /* encoding of request for level 10 */
 	__le64 EndOfFile; /* new end of file value */
 } __packed; /* level 20 Set */
 
+/* See MS-FSCC 2.4.4 */
+struct smb2_file_alloc_info { /* encoding of request for level 19 */
+	__le64 AllocationSize;
+} __packed;
+
 /* See MS-FSCC 2.4.15 */
 typedef struct {
 	__le32 NextEntryOffset;
diff --git a/fs/smb/server/compress.c b/fs/smb/server/compress.c
index f8cf515b9c30a..95e48fa6b4486 100644
--- a/fs/smb/server/compress.c
+++ b/fs/smb/server/compress.c
@@ -56,7 +56,8 @@ int ksmbd_decompress_request(struct ksmbd_conn *conn)
 	}
 
 	max_allowed_pdu_size = SMB3_MAX_MSGSIZE + conn->vals->max_write_size;
-	if (out_size > max_allowed_pdu_size ||
+	if (out_size < sizeof(struct smb2_pdu) ||
+	    out_size > max_allowed_pdu_size ||
 	    out_size > MAX_STREAM_PROT_LEN)
 		return -EINVAL;
 
diff --git a/fs/smb/server/connection.c b/fs/smb/server/connection.c
index 9e8fdb39e5a2a..dee8e4aced994 100644
--- a/fs/smb/server/connection.c
+++ b/fs/smb/server/connection.c
@@ -441,6 +441,8 @@ bool ksmbd_conn_alive(struct ksmbd_conn *conn)
 /* "+2" for BCC field (ByteCount, 2 bytes) */
 #define SMB1_MIN_SUPPORTED_PDU_SIZE (sizeof(struct smb_hdr) + 2)
 #define SMB2_MIN_SUPPORTED_PDU_SIZE (sizeof(struct smb2_pdu))
+#define SMB2_TRANSFORM_MIN_SUPPORTED_PDU_SIZE	\
+	(sizeof(struct smb2_transform_hdr) + sizeof(struct smb2_hdr))
 
 /**
  * ksmbd_conn_handler_loop() - session thread to listen on new smb requests
@@ -455,6 +457,7 @@ int ksmbd_conn_handler_loop(void *p)
 	struct ksmbd_conn *conn = (struct ksmbd_conn *)p;
 	struct ksmbd_transport *t = conn->transport;
 	unsigned int pdu_size, max_allowed_pdu_size, max_req;
+	__le32 proto;
 	char hdr_buf[4] = {0,};
 	int size;
 
@@ -546,11 +549,14 @@ int ksmbd_conn_handler_loop(void *p)
 		if (!ksmbd_smb_request(conn))
 			break;
 
-		if (((struct smb2_hdr *)smb_get_msg(conn->request_buf))->ProtocolId ==
-		    SMB2_PROTO_NUMBER) {
-			if (pdu_size < SMB2_MIN_SUPPORTED_PDU_SIZE)
-				break;
-		}
+		proto = *(__le32 *)smb_get_msg(conn->request_buf);
+		if (proto == SMB2_PROTO_NUMBER &&
+		    pdu_size < SMB2_MIN_SUPPORTED_PDU_SIZE)
+			break;
+
+		if (proto == SMB2_TRANSFORM_PROTO_NUM &&
+		    pdu_size < SMB2_TRANSFORM_MIN_SUPPORTED_PDU_SIZE)
+			break;
 
 		if (!default_conn_ops.process_fn) {
 			pr_err("No connection request callback\n");
diff --git a/fs/smb/server/ksmbd_netlink.h b/fs/smb/server/ksmbd_netlink.h
index 8ccd57fd904bc..c9e1b0b689d7f 100644
--- a/fs/smb/server/ksmbd_netlink.h
+++ b/fs/smb/server/ksmbd_netlink.h
@@ -377,6 +377,7 @@ enum KSMBD_TREE_CONN_STATUS {
 #define KSMBD_SHARE_FLAG_UPDATE				BIT(14)
 #define KSMBD_SHARE_FLAG_CROSSMNT			BIT(15)
 #define KSMBD_SHARE_FLAG_CONTINUOUS_AVAILABILITY	BIT(16)
+#define KSMBD_SHARE_FLAG_HIDE_UNREADABLE		BIT(17)
 
 /*
  * Tree connect request flags.
diff --git a/fs/smb/server/mgmt/share_config.c b/fs/smb/server/mgmt/share_config.c
index 6f97f8d39657c..e00aee1559353 100644
--- a/fs/smb/server/mgmt/share_config.c
+++ b/fs/smb/server/mgmt/share_config.c
@@ -9,6 +9,7 @@
 #include <linux/rwsem.h>
 #include <linux/parser.h>
 #include <linux/namei.h>
+#include <linux/fs_struct.h>
 #include <linux/sched.h>
 #include <linux/mm.h>
 
@@ -193,7 +194,8 @@ static struct ksmbd_share_config *share_config_request(struct ksmbd_work *work,
 				goto out;
 			}
 
-			ret = kern_path(share->path, 0, &share->vfs_path);
+			scoped_with_init_fs()
+				ret = kern_path(share->path, 0, &share->vfs_path);
 			ksmbd_revert_fsids(work);
 			if (ret) {
 				ksmbd_debug(SMB, "failed to access '%s'\n",
diff --git a/fs/smb/server/mgmt/user_session.c b/fs/smb/server/mgmt/user_session.c
index d6331184ebfcb..f99c86284ba3d 100644
--- a/fs/smb/server/mgmt/user_session.c
+++ b/fs/smb/server/mgmt/user_session.c
@@ -457,22 +457,19 @@ void ksmbd_sessions_deregister(struct ksmbd_conn *conn)
 {
 	struct ksmbd_session *sess;
 	unsigned long id;
+	struct hlist_node *tmp;
+	int bkt;
 
 	down_write(&sessions_table_lock);
-	if (conn->binding) {
-		int bkt;
-		struct hlist_node *tmp;
-
-		hash_for_each_safe(sessions_table, bkt, tmp, sess, hlist) {
-			if (!ksmbd_chann_del(conn, sess) &&
-			    xa_empty(&sess->ksmbd_chann_list)) {
-				hash_del(&sess->hlist);
-				down_write(&conn->session_lock);
-				xa_erase(&conn->sessions, sess->id);
-				up_write(&conn->session_lock);
-				if (atomic_dec_and_test(&sess->refcnt))
-					ksmbd_session_destroy(sess);
-			}
+	hash_for_each_safe(sessions_table, bkt, tmp, sess, hlist) {
+		if (!ksmbd_chann_del(conn, sess) &&
+		    xa_empty(&sess->ksmbd_chann_list)) {
+			hash_del(&sess->hlist);
+			down_write(&conn->session_lock);
+			xa_erase(&conn->sessions, sess->id);
+			up_write(&conn->session_lock);
+			if (atomic_dec_and_test(&sess->refcnt))
+				ksmbd_session_destroy(sess);
 		}
 	}
 
diff --git a/fs/smb/server/mgmt/user_session.h b/fs/smb/server/mgmt/user_session.h
index 8893a9aaede79..4637a8c8436d0 100644
--- a/fs/smb/server/mgmt/user_session.h
+++ b/fs/smb/server/mgmt/user_session.h
@@ -19,7 +19,7 @@
 struct ksmbd_file_table;
 
 struct channel {
-	char			sess_key[SMB2_NTLMV2_SESSKEY_SIZE];
+	char			sess_key[CIFS_KEY_SIZE];
 	__u8			smb3signingkey[SMB3_SIGN_KEY_SIZE];
 	struct ksmbd_conn	*conn;
 };
diff --git a/fs/smb/server/oplock.c b/fs/smb/server/oplock.c
index 3c55ae5d6a11b..79787099afdc6 100644
--- a/fs/smb/server/oplock.c
+++ b/fs/smb/server/oplock.c
@@ -875,6 +875,7 @@ static void __smb2_oplock_break_noti(struct work_struct *wk)
 out:
 	ksmbd_free_work_struct(work);
 	ksmbd_conn_r_count_dec(conn);
+	ksmbd_conn_put(conn);
 }
 
 /**
@@ -910,7 +911,7 @@ static int smb2_oplock_break_noti(struct oplock_info *opinfo)
 	br_info->open_trunc = opinfo->open_trunc;
 
 	work->request_buf = (char *)br_info;
-	work->conn = conn;
+	work->conn = ksmbd_conn_get(conn);
 	work->sess = opinfo->sess;
 
 	ksmbd_conn_r_count_inc(conn);
@@ -985,6 +986,7 @@ static void __smb2_lease_break_noti(struct work_struct *wk)
 out:
 	ksmbd_free_work_struct(work);
 	ksmbd_conn_r_count_dec(conn);
+	ksmbd_conn_put(conn);
 }
 
 /**
@@ -1034,7 +1036,7 @@ static int smb2_lease_break_noti(struct oplock_info *opinfo, bool wait_ack,
 	memcpy(br_info->lease_key, lease->lease_key, SMB2_LEASE_KEY_SIZE);
 
 	work->request_buf = (char *)br_info;
-	work->conn = conn;
+	work->conn = ksmbd_conn_get(conn);
 	work->sess = opinfo->sess;
 
 	ksmbd_conn_r_count_inc(conn);
diff --git a/fs/smb/server/smb2misc.c b/fs/smb/server/smb2misc.c
index c0c4edd092c2b..9f3629c86291b 100644
--- a/fs/smb/server/smb2misc.c
+++ b/fs/smb/server/smb2misc.c
@@ -407,6 +407,11 @@ int ksmbd_smb2_check_message(struct ksmbd_work *work)
 		return 1;
 	}
 
+	if (len < __SMB2_HEADER_STRUCTURE_SIZE + sizeof(__le16)) {
+		ksmbd_debug(SMB, "Message is too small for StructureSize2\n");
+		return 1;
+	}
+
 	if (smb2_req_struct_sizes[command] != pdu->StructureSize2) {
 		if (!(command == SMB2_OPLOCK_BREAK_HE &&
 		    (le16_to_cpu(pdu->StructureSize2) == OP_BREAK_STRUCT_SIZE_20 ||
diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c
index b73167785e875..6c1e407ef33f5 100644
--- a/fs/smb/server/smb2pdu.c
+++ b/fs/smb/server/smb2pdu.c
@@ -9,6 +9,7 @@
 #include <net/addrconf.h>
 #include <linux/syscalls.h>
 #include <linux/namei.h>
+#include <linux/fs_struct.h>
 #include <linux/statfs.h>
 #include <linux/ethtool.h>
 #include <linux/falloc.h>
@@ -64,6 +65,9 @@ static void __wbuf(struct ksmbd_work *work, void **req, void **rsp)
 /* Windows reports automatic write-time updates at roughly 15 ms resolution. */
 #define KSMBD_WRITE_TIME_RESOLUTION	(15ULL * 10000)
 
+/* MAXFILESIZE in [MS-FSA] 2.1.5.3 Server Requests a Write. */
+#define SMB2_MAX_FILE_SIZE		0xfffffff0000ULL
+
 /**
  * check_session_id() - check for valid session id in smb header
  * @conn:	connection instance
@@ -1689,7 +1693,7 @@ static int ntlm_authenticate(struct ksmbd_work *work,
 	struct ksmbd_conn *conn = work->conn;
 	struct ksmbd_session *sess = work->sess;
 	struct ksmbd_user *user;
-	char channel_key[SMB2_NTLMV2_SESSKEY_SIZE] = {};
+	char channel_key[CIFS_KEY_SIZE] = {};
 	char *auth_key = conn->binding ? channel_key : sess->sess_key;
 	u64 prev_id;
 	bool binding = conn->binding;
@@ -1717,11 +1721,6 @@ static int ntlm_authenticate(struct ksmbd_work *work,
 		return -EPERM;
 	}
 
-	/* Check for previous session */
-	prev_id = le64_to_cpu(req->PreviousSessionId);
-	if (prev_id && prev_id != sess->id)
-		destroy_previous_session(conn, user, prev_id);
-
 	if (sess->state == SMB2_SESSION_VALID) {
 		/*
 		 * Reuse session if anonymous try to connect
@@ -1761,6 +1760,10 @@ static int ntlm_authenticate(struct ksmbd_work *work,
 		}
 	}
 
+	prev_id = le64_to_cpu(req->PreviousSessionId);
+	if (prev_id && prev_id != sess->id)
+		destroy_previous_session(conn, sess->user, prev_id);
+
 	/*
 	 * If session state is SMB2_SESSION_VALID, We can assume
 	 * that it is reauthentication. And the user/password
@@ -1826,7 +1829,7 @@ static int krb5_authenticate(struct ksmbd_work *work,
 	struct ksmbd_conn *conn = work->conn;
 	struct ksmbd_session *sess = work->sess;
 	char *in_blob, *out_blob;
-	char channel_key[SMB2_NTLMV2_SESSKEY_SIZE] = {};
+	char channel_key[CIFS_KEY_SIZE] = {};
 	char *auth_key = conn->binding ? channel_key : sess->sess_key;
 	u64 prev_sess_id;
 	bool binding = conn->binding;
@@ -1971,6 +1974,21 @@ int smb2_sess_setup(struct ksmbd_work *work)
 			goto out_err;
 		}
 
+		if (conn->dialect == SMB311_PROT_ID) {
+			struct channel *chann;
+			unsigned long index;
+
+			down_read(&sess->chann_lock);
+			xa_for_each(&sess->ksmbd_chann_list, index, chann) {
+				if (conn->cipher_type != chann->conn->cipher_type)
+					rc = -EINVAL;
+				break;
+			}
+			up_read(&sess->chann_lock);
+			if (rc)
+				goto out_err;
+		}
+
 		if (!(req->hdr.Flags & SMB2_FLAGS_SIGNED)) {
 			rc = -EINVAL;
 			goto out_err;
@@ -2335,6 +2353,10 @@ int smb2_tree_connect(struct ksmbd_work *work)
 	if (conn->dialect == SMB311_PROT_ID &&
 	    conn->compress_algorithm != SMB3_COMPRESS_NONE)
 		rsp->ShareFlags |= cpu_to_le32(SMB2_SHAREFLAG_COMPRESS_DATA);
+	if (share && test_share_config_flag(share,
+					  KSMBD_SHARE_FLAG_HIDE_UNREADABLE))
+		rsp->ShareFlags |=
+			cpu_to_le32(SMB2_SHAREFLAG_ACCESS_BASED_DIRECTORY_ENUM);
 
 	rc = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_tree_connect_rsp));
 	if (rc)
@@ -2643,6 +2665,22 @@ static noinline int create_smb2_pipe(struct ksmbd_work *work)
 	return err;
 }
 
+static bool smb2_is_private_ea(const char *name, size_t name_len)
+{
+	if (name_len == SD_PREFIX_LEN &&
+	    !strncasecmp(name, SD_PREFIX, SD_PREFIX_LEN))
+		return true;
+	if (name_len == DOS_ATTRIBUTE_PREFIX_LEN &&
+	    !strncasecmp(name, DOS_ATTRIBUTE_PREFIX,
+			   DOS_ATTRIBUTE_PREFIX_LEN))
+		return true;
+	if (name_len >= STREAM_PREFIX_LEN &&
+	    !strncasecmp(name, STREAM_PREFIX, STREAM_PREFIX_LEN))
+		return true;
+
+	return false;
+}
+
 /**
  * smb2_set_ea() - handler for setting extended attributes using set
  *		info command
@@ -2684,6 +2722,10 @@ static int smb2_set_ea(struct smb2_ea_info *eabuf, unsigned int buf_len,
 			rc = -EINVAL;
 			break;
 		}
+		if (smb2_is_private_ea(eabuf->name, eabuf->EaNameLength)) {
+			rc = -EACCES;
+			break;
+		}
 
 		memcpy(attr_name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN);
 		memcpy(&attr_name[XATTR_USER_PREFIX_LEN], eabuf->name,
@@ -3519,6 +3561,8 @@ int smb2_open(struct ksmbd_work *work)
 		file_present = true;
 
 		if (req->CreateOptions & FILE_DELETE_ON_CLOSE_LE) {
+			struct xattr_dos_attrib da;
+
 			/*
 			 * If file exists with under flags, return access
 			 * denied error.
@@ -3532,6 +3576,16 @@ int smb2_open(struct ksmbd_work *work)
 			if (!test_tree_conn_flag(tcon, KSMBD_TREE_CONN_FLAG_WRITABLE)) {
 				ksmbd_debug(SMB,
 					    "User does not have write permission\n");
+					rc = -EACCES;
+					goto err_out;
+				}
+
+			if (test_share_config_flag(tcon->share_conf,
+						   KSMBD_SHARE_FLAG_STORE_DOS_ATTRS) &&
+			    ksmbd_vfs_get_dos_attrib_xattr(mnt_idmap(path.mnt),
+							 path.dentry, &da) > 0 &&
+			    da.attr & FILE_ATTRIBUTE_READONLY) {
+				rsp->hdr.Status = STATUS_CANNOT_DELETE;
 				rc = -EACCES;
 				goto err_out;
 			}
@@ -3549,6 +3603,13 @@ int smb2_open(struct ksmbd_work *work)
 		rc = 0;
 	}
 
+	if (!file_present && req->CreateOptions & FILE_DELETE_ON_CLOSE_LE &&
+	    req->FileAttributes & FILE_ATTRIBUTE_READONLY_LE) {
+		rsp->hdr.Status = STATUS_CANNOT_DELETE;
+		rc = -EACCES;
+		goto err_out;
+	}
+
 	/*
 	 * An explicit ::$DATA suffix names the unnamed data stream and is
 	 * canonicalized to a NULL stream name (base file), but the request
@@ -3608,9 +3669,18 @@ int smb2_open(struct ksmbd_work *work)
 
 	if (file_present && !(req->CreateOptions & FILE_DELETE_ON_CLOSE_LE)) {
 		rc = smb_check_perm_dacl(conn, &path, &daccess,
-					 sess->user->uid);
+					 req->DesiredAccess,
+					 sess->user->uid, false);
 		if (rc)
 			goto err_out;
+
+		if (maximal_access_ctxt) {
+			maximal_access = FILE_MAXIMAL_ACCESS_LE;
+			rc = smb_check_perm_dacl(conn, &path, &maximal_access,
+						 0, sess->user->uid, false);
+			if (rc)
+				goto err_out;
+		}
 	}
 
 	if (daccess & FILE_MAXIMAL_ACCESS_LE) {
@@ -4180,8 +4250,13 @@ int smb2_open(struct ksmbd_work *work)
 			rsp->hdr.Status = STATUS_INVALID_PARAMETER;
 		else if (rc == -EOPNOTSUPP)
 			rsp->hdr.Status = STATUS_NOT_SUPPORTED;
-		else if (rc == -EACCES || rc == -ESTALE || rc == -EXDEV)
-			rsp->hdr.Status = STATUS_ACCESS_DENIED;
+		else if ((rc == -EACCES || rc == -ESTALE || rc == -EXDEV) &&
+			 !rsp->hdr.Status) {
+			if (req->DesiredAccess & FILE_ACCESS_SYSTEM_SECURITY_LE)
+				rsp->hdr.Status = STATUS_PRIVILEGE_NOT_HELD;
+			else
+				rsp->hdr.Status = STATUS_ACCESS_DENIED;
+		}
 		else if (rc == -ENOENT)
 			rsp->hdr.Status = STATUS_OBJECT_NAME_INVALID;
 		else if (rc == -EPERM)
@@ -4579,6 +4654,7 @@ static int process_query_dir_entries(struct smb2_query_dir_private *priv)
 
 	for (i = 0; i < priv->d_info->num_entry; i++) {
 		struct dentry *dent;
+		struct path path;
 
 		if (dentry_name(priv->d_info, priv->info_level))
 			return -EINVAL;
@@ -4601,6 +4677,23 @@ static int process_query_dir_entries(struct smb2_query_dir_private *priv)
 			continue;
 		}
 
+		if (test_share_config_flag(priv->work->tcon->share_conf,
+					   KSMBD_SHARE_FLAG_HIDE_UNREADABLE)) {
+			__le32 daccess = FILE_READ_DATA_LE | FILE_READ_EA_LE |
+				FILE_READ_ATTRIBUTES_LE;
+
+			path.mnt = priv->dir_fp->filp->f_path.mnt;
+			path.dentry = dent;
+			rc = smb_check_perm_dacl(priv->work->conn, &path,
+						 &daccess, daccess,
+						 priv->work->sess->user->uid,
+						 true);
+			if (rc) {
+				dput(dent);
+				continue;
+			}
+		}
+
 		ksmbd_kstat.kstat = &kstat;
 		if (priv->info_level != FILE_NAMES_INFORMATION) {
 			rc = ksmbd_vfs_fill_dentry_attrs(priv->work,
@@ -5029,21 +5122,30 @@ int smb2_query_dir(struct ksmbd_work *work)
 /**
  * buffer_check_err() - helper function to check buffer errors
  * @reqOutputBufferLength:	max buffer length expected in command response
+ * @fixed_len:			minimum fixed response length
  * @rsp:		query info response buffer contains output buffer length
  * @rsp_org:		base response buffer pointer in case of chained response
  *
  * Return:	0 on success, otherwise error
  */
 static int buffer_check_err(int reqOutputBufferLength,
+			    unsigned int fixed_len,
 			    struct smb2_query_info_rsp *rsp,
 			    void *rsp_org)
 {
-	if (reqOutputBufferLength < le32_to_cpu(rsp->OutputBufferLength)) {
+	unsigned int output_len = le32_to_cpu(rsp->OutputBufferLength);
+
+	if (reqOutputBufferLength < fixed_len) {
 		pr_err("Invalid Buffer Size Requested\n");
 		rsp->hdr.Status = STATUS_INFO_LENGTH_MISMATCH;
 		*(__be32 *)rsp_org = cpu_to_be32(sizeof(struct smb2_hdr));
 		return -EINVAL;
 	}
+
+	if (reqOutputBufferLength < output_len) {
+		rsp->hdr.Status = STATUS_BUFFER_OVERFLOW;
+		rsp->OutputBufferLength = cpu_to_le32(reqOutputBufferLength);
+	}
 	return 0;
 }
 
@@ -5106,11 +5208,13 @@ static int smb2_get_info_file_pipe(struct ksmbd_session *sess,
 	case FILE_STANDARD_INFORMATION:
 		get_standard_info_pipe(rsp, rsp_org);
 		rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength),
+				      le32_to_cpu(rsp->OutputBufferLength),
 				      rsp, rsp_org);
 		break;
 	case FILE_INTERNAL_INFORMATION:
 		get_internal_info_pipe(rsp, id, rsp_org);
 		rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength),
+				      le32_to_cpu(rsp->OutputBufferLength),
 				      rsp, rsp_org);
 		break;
 	default:
@@ -5208,17 +5312,13 @@ static int smb2_get_ea(struct ksmbd_work *work, struct ksmbd_file *fp,
 		if (strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN))
 			continue;
 
-		if (!strncmp(&name[XATTR_USER_PREFIX_LEN], STREAM_PREFIX,
-			     STREAM_PREFIX_LEN))
-			continue;
-
 		if (req->InputBufferLength &&
 		    strncmp(&name[XATTR_USER_PREFIX_LEN], ea_req->name,
 			    ea_req->EaNameLength))
 			continue;
 
-		if (!strncmp(&name[XATTR_USER_PREFIX_LEN],
-			     DOS_ATTRIBUTE_PREFIX, DOS_ATTRIBUTE_PREFIX_LEN))
+		if (smb2_is_private_ea(&name[XATTR_USER_PREFIX_LEN],
+				       name_len - XATTR_USER_PREFIX_LEN))
 			continue;
 
 		if (!strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN))
@@ -5931,6 +6031,7 @@ static int smb2_get_info_file(struct ksmbd_work *work,
 	}
 	if (!rc)
 		rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength),
+				      le32_to_cpu(rsp->OutputBufferLength),
 				      rsp, work->response_buf);
 	ksmbd_fd_put(work, fp);
 
@@ -5952,11 +6053,13 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work,
 	struct kstatfs stfs;
 	struct path path;
 	int rc = 0, len;
+	unsigned int fixed_len = 0;
 
 	if (!share->path)
 		return -EIO;
 
-	rc = kern_path(share->path, LOOKUP_NO_SYMLINKS, &path);
+	scoped_with_init_fs()
+		rc = kern_path(share->path, LOOKUP_NO_SYMLINKS, &path);
 	if (rc) {
 		pr_err("cannot create vfs path\n");
 		return -EIO;
@@ -5986,6 +6089,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work,
 			info->DeviceCharacteristics |=
 				cpu_to_le32(FILE_READ_ONLY_DEVICE);
 		rsp->OutputBufferLength = cpu_to_le32(8);
+		fixed_len = 8;
 		break;
 	}
 	case FS_ATTRIBUTE_INFORMATION:
@@ -6038,6 +6142,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work,
 		info->FileSystemNameLen = cpu_to_le32(len);
 		sz = sizeof(FILE_SYSTEM_ATTRIBUTE_INFO) + len;
 		rsp->OutputBufferLength = cpu_to_le32(sz);
+		fixed_len = 16;
 		break;
 	}
 	case FS_VOLUME_INFORMATION:
@@ -6065,6 +6170,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work,
 		info->SupportsObjects = 0;
 		sz = sizeof(struct filesystem_vol_info) + len;
 		rsp->OutputBufferLength = cpu_to_le32(sz);
+		fixed_len = 24;
 		break;
 	}
 	case FS_SIZE_INFORMATION:
@@ -6077,6 +6183,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work,
 		info->SectorsPerAllocationUnit = cpu_to_le32(1);
 		info->BytesPerSector = cpu_to_le32(stfs.f_bsize);
 		rsp->OutputBufferLength = cpu_to_le32(24);
+		fixed_len = 24;
 		break;
 	}
 	case FS_FULL_SIZE_INFORMATION:
@@ -6092,6 +6199,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work,
 		info->SectorsPerAllocationUnit = cpu_to_le32(1);
 		info->BytesPerSector = cpu_to_le32(stfs.f_bsize);
 		rsp->OutputBufferLength = cpu_to_le32(32);
+		fixed_len = 32;
 		break;
 	}
 	case FS_OBJECT_ID_INFORMATION:
@@ -6112,6 +6220,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work,
 		info->extended_info.rel_date = 0;
 		memcpy(info->extended_info.version_string, "1.1.0", strlen("1.1.0"));
 		rsp->OutputBufferLength = cpu_to_le32(64);
+		fixed_len = 64;
 		break;
 	}
 	case FS_SECTOR_SIZE_INFORMATION:
@@ -6133,6 +6242,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work,
 		info->ByteOffsetForSectorAlignment = 0;
 		info->ByteOffsetForPartitionAlignment = 0;
 		rsp->OutputBufferLength = cpu_to_le32(28);
+		fixed_len = 28;
 		break;
 	}
 	case FS_CONTROL_INFORMATION:
@@ -6153,6 +6263,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work,
 		info->DefaultQuotaLimit = cpu_to_le64(SMB2_NO_FID);
 		info->Padding = 0;
 		rsp->OutputBufferLength = cpu_to_le32(48);
+		fixed_len = 48;
 		break;
 	}
 	case FS_POSIX_INFORMATION:
@@ -6173,6 +6284,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work,
 			info->TotalFileNodes = cpu_to_le64(stfs.f_files);
 			info->FreeFileNodes = cpu_to_le64(stfs.f_ffree);
 			rsp->OutputBufferLength = cpu_to_le32(56);
+			fixed_len = 56;
 		}
 		break;
 	}
@@ -6181,6 +6293,7 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work,
 		return -EOPNOTSUPP;
 	}
 	rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength),
+			      fixed_len,
 			      rsp, work->response_buf);
 	path_put(&path);
 
@@ -6282,6 +6395,7 @@ static int smb2_get_info_sec(struct ksmbd_work *work,
 
 	rsp->OutputBufferLength = cpu_to_le32(secdesclen);
 	rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength),
+			      le32_to_cpu(rsp->OutputBufferLength),
 			      rsp, work->response_buf);
 	if (rc)
 		goto err_out;
@@ -6597,9 +6711,8 @@ static int smb2_rename(struct ksmbd_work *work,
 			pr_err("failed to store stream name in xattr: %d\n",
 			       rc);
 			rc = -EINVAL;
-			goto out;
 		}
-
+		kfree(xattr_stream_name);
 		goto out;
 	}
 
@@ -6781,6 +6894,7 @@ static int set_file_allocation_info(struct ksmbd_work *work,
 	 */
 
 	loff_t alloc_blks;
+	u64 alloc_size;
 	struct inode *inode;
 	struct kstat stat;
 	int rc;
@@ -6796,7 +6910,19 @@ static int set_file_allocation_info(struct ksmbd_work *work,
 	if (rc)
 		return rc;
 
-	alloc_blks = (le64_to_cpu(file_alloc_info->AllocationSize) + 511) >> 9;
+	/*
+	 * AllocationSize is fully client-controlled (the caller only
+	 * validates the fixed 8-byte buffer length). Reject values that
+	 * would overflow the "round up to 512-byte blocks" conversion
+	 * below instead of silently wrapping it to a tiny block count,
+	 * which would truncate the file to a size the client never
+	 * asked for.
+	 */
+	alloc_size = le64_to_cpu(file_alloc_info->AllocationSize);
+	if (alloc_size > MAX_LFS_FILESIZE - 511)
+		return -EINVAL;
+
+	alloc_blks = (alloc_size + 511) >> 9;
 	inode = file_inode(fp->filp);
 
 	if (alloc_blks > stat.blocks) {
@@ -6895,6 +7021,9 @@ static int set_file_disposition_info(struct ksmbd_work *work,
 		return -EACCES;
 	}
 
+	if (fp->f_ci->m_fattr & FILE_ATTRIBUTE_READONLY_LE)
+		return -EACCES;
+
 	inode = file_inode(fp->filp);
 	if (file_info->DeletePending) {
 		if (ksmbd_has_stream_without_delete_share(fp))
@@ -7165,8 +7294,14 @@ int smb2_set_info(struct ksmbd_work *work)
 	return 0;
 
 err_out:
-	if (rc == -EACCES || rc == -EPERM || rc == -EXDEV)
-		rsp->hdr.Status = STATUS_ACCESS_DENIED;
+	if (rc == -EACCES || rc == -EPERM || rc == -EXDEV) {
+		if (fp && req->InfoType == SMB2_O_INFO_FILE &&
+		    req->FileInfoClass == FILE_DISPOSITION_INFORMATION &&
+		    fp->f_ci->m_fattr & FILE_ATTRIBUTE_READONLY_LE)
+			rsp->hdr.Status = STATUS_CANNOT_DELETE;
+		else
+			rsp->hdr.Status = STATUS_ACCESS_DENIED;
+	}
 	else if (rc == -EINVAL)
 		rsp->hdr.Status = STATUS_INVALID_PARAMETER;
 	else if (rc == -EMSGSIZE)
@@ -7433,6 +7568,15 @@ int smb2_read(struct ksmbd_work *work)
 		goto out;
 	}
 
+	/*
+	 * ksmbd_vfs_read() fills only nbytes; the [nbytes, ALIGN(nbytes, 8))
+	 * tail of the un-zeroed buffer is transmitted as compound-response
+	 * alignment padding, leaking uninitialized kernel memory to the
+	 * client.  Zero just that tail.
+	 */
+	if (nbytes & 7)
+		memset(aux_payload_buf + nbytes, 0, ALIGN(nbytes, 8) - nbytes);
+
 	if ((nbytes == 0 && length != 0) || nbytes < mincount) {
 		kvfree(aux_payload_buf);
 		rsp->hdr.Status = STATUS_END_OF_FILE;
@@ -7656,8 +7800,10 @@ int smb2_write(struct ksmbd_work *work)
 	}
 
 	offset = le64_to_cpu(req->Offset);
-	if (offset < 0)
-		return -EINVAL;
+	if (offset < 0) {
+		err = -EINVAL;
+		goto out;
+	}
 	length = le32_to_cpu(req->Length);
 
 	if (req->Channel == SMB2_CHANNEL_RDMA_V1 ||
@@ -7671,6 +7817,19 @@ int smb2_write(struct ksmbd_work *work)
 		length = le32_to_cpu(req->RemainingBytes);
 	}
 
+	if (length) {
+		u64 end = (u64)offset + length;
+
+		if (end > SMB2_MAX_FILE_SIZE) {
+			err = -EINVAL;
+			goto out;
+		}
+		if (end == SMB2_MAX_FILE_SIZE) {
+			err = -EFBIG;
+			goto out;
+		}
+	}
+
 	if (is_rdma_channel == true) {
 		unsigned int ch_offset = le16_to_cpu(req->WriteChannelInfoOffset);
 
@@ -9787,22 +9946,21 @@ void smb3_preauth_hash_rsp(struct ksmbd_work *work)
 	}
 
 	if (le16_to_cpu(rsp->Command) == SMB2_SESSION_SETUP_HE && sess) {
-		__u8 *hash_value;
+		ksmbd_conn_lock(conn);
 
 		if (conn->binding) {
 			struct preauth_session *preauth_sess;
 
 			preauth_sess = ksmbd_preauth_session_lookup(conn, sess->id);
-			if (!preauth_sess)
-				return;
-			hash_value = preauth_sess->Preauth_HashValue;
-		} else {
-			hash_value = sess->Preauth_HashValue;
-			if (!hash_value)
-				return;
+			if (preauth_sess)
+				ksmbd_gen_preauth_integrity_hash(conn,
+					work->response_buf,
+					preauth_sess->Preauth_HashValue);
+		} else if (sess->Preauth_HashValue) {
+			ksmbd_gen_preauth_integrity_hash(conn, work->response_buf,
+					 sess->Preauth_HashValue);
 		}
-		ksmbd_gen_preauth_integrity_hash(conn, work->response_buf,
-						 hash_value);
+		ksmbd_conn_unlock(conn);
 	}
 }
 
diff --git a/fs/smb/server/smb2pdu.h b/fs/smb/server/smb2pdu.h
index c2512dbcdec85..aa06c8c905f1f 100644
--- a/fs/smb/server/smb2pdu.h
+++ b/fs/smb/server/smb2pdu.h
@@ -212,10 +212,6 @@ struct smb2_file_ea_info {
 	__le32 EASize;
 } __packed;
 
-struct smb2_file_alloc_info {
-	__le64 AllocationSize;
-} __packed;
-
 struct smb2_file_disposition_info {
 	__u8 DeletePending;
 } __packed;
diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c
index 67b39b4d218cc..b5db6dcfbaa4f 100644
--- a/fs/smb/server/smbacl.c
+++ b/fs/smb/server/smbacl.c
@@ -27,6 +27,9 @@ static const struct smb_sid creator_owner = {
 /* security id for everyone/world system group */
 static const struct smb_sid creator_group = {
 	1, 1, {0, 0, 0, 0, 0, 3}, {cpu_to_le32(1)} };
+/* security id for owner rights */
+static const struct smb_sid sid_owner_rights = {
+	1, 1, {0, 0, 0, 0, 0, 3}, {cpu_to_le32(4)} };
 
 /* security id for everyone/world system group */
 static const struct smb_sid sid_everyone = {
@@ -608,7 +611,8 @@ static void parse_dacl(struct mnt_idmap *idmap,
 static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap,
 				       struct smb_ace *pndace,
 				       struct smb_fattr *fattr, u16 *num_aces,
-				       u16 *size, u32 nt_aces_num)
+				       u16 *size, u16 existing_nt_aces,
+				       bool had_nt_aces)
 {
 	struct posix_acl_entry *pace;
 	struct smb_sid *sid;
@@ -640,14 +644,14 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap,
 
 			gid = posix_acl_gid_translate(idmap, pace);
 			id_to_sid(gid, SIDUNIX_GROUP, sid);
-		} else if (pace->e_tag == ACL_OTHER && !nt_aces_num) {
+		} else if (pace->e_tag == ACL_OTHER && !had_nt_aces) {
 			smb_copy_sid(sid, &sid_everyone);
 		} else {
 			kfree(sid);
 			continue;
 		}
 		ntace = pndace;
-		for (j = 0; j < nt_aces_num; j++) {
+		for (j = 0; j < existing_nt_aces; j++) {
 			if (ntace->sid.sub_auth[ntace->sid.num_subauth - 1] ==
 					sid->sub_auth[sid->num_subauth - 1])
 				goto pass_same_sid;
@@ -662,6 +666,7 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap,
 		ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, flags,
 				pace->e_perm, 0777);
 		if (check_add_overflow(*size, ace_sz, size)) {
+			*size -= ace_sz;
 			kfree(sid);
 			break;
 		}
@@ -676,6 +681,7 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap,
 			ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED,
 					0x03, pace->e_perm, 0777);
 			if (check_add_overflow(*size, ace_sz, size)) {
+				*size -= ace_sz;
 				kfree(sid);
 				break;
 			}
@@ -689,7 +695,7 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap,
 		kfree(sid);
 	}
 
-	if (nt_aces_num)
+	if (had_nt_aces)
 		return;
 
 posix_default_acl:
@@ -721,6 +727,7 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap,
 		ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, 0x0b,
 				pace->e_perm, 0777);
 		if (check_add_overflow(*size, ace_sz, size)) {
+			*size -= ace_sz;
 			kfree(sid);
 			break;
 		}
@@ -742,6 +749,7 @@ static void set_ntacl_dacl(struct mnt_idmap *idmap,
 {
 	struct smb_ace *ntace, *pndace;
 	u16 nt_num_aces = le16_to_cpu(nt_dacl->num_aces), num_aces = 0;
+	u16 copied_nt_aces;
 	unsigned short size = 0;
 	int i;
 
@@ -751,20 +759,29 @@ static void set_ntacl_dacl(struct mnt_idmap *idmap,
 		for (i = 0; i < nt_num_aces; i++) {
 			unsigned short nt_ace_size;
 
-			if (offsetof(struct smb_ace, access_req) > aces_size)
+			if (aces_size < offsetof(struct smb_ace, sid) +
+					CIFS_SID_BASE_SIZE)
 				break;
 
 			nt_ace_size = le16_to_cpu(ntace->size);
-			if (nt_ace_size > aces_size)
+			if (nt_ace_size > aces_size ||
+			    nt_ace_size < offsetof(struct smb_ace, sid) +
+					  CIFS_SID_BASE_SIZE)
 				break;
 
 			if (ntace->sid.num_subauth == 0 ||
-			    ntace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES)
+			    ntace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES ||
+			    nt_ace_size < offsetof(struct smb_ace, sid) +
+					  CIFS_SID_BASE_SIZE +
+					  sizeof(__le32) *
+					  ntace->sid.num_subauth)
 				goto next_ace;
 
 			memcpy((char *)pndace + size, ntace, nt_ace_size);
-			if (check_add_overflow(size, nt_ace_size, &size))
+			if (check_add_overflow(size, nt_ace_size, &size)) {
+				size -= nt_ace_size;
 				break;
+			}
 			num_aces++;
 
 next_ace:
@@ -773,8 +790,10 @@ static void set_ntacl_dacl(struct mnt_idmap *idmap,
 		}
 	}
 
+	copied_nt_aces = num_aces;
 	set_posix_acl_entries_dacl(idmap, pndace, fattr,
-				   &num_aces, &size, nt_num_aces);
+				   &num_aces, &size, copied_nt_aces,
+				   nt_num_aces != 0);
 	pndacl->num_aces = cpu_to_le16(num_aces);
 	pndacl->size = cpu_to_le16(le16_to_cpu(pndacl->size) + size);
 }
@@ -792,7 +811,7 @@ static void set_mode_dacl(struct mnt_idmap *idmap,
 
 	if (fattr->cf_acls) {
 		set_posix_acl_entries_dacl(idmap, pndace, fattr,
-					   &num_aces, &size, num_aces);
+					   &num_aces, &size, num_aces, false);
 		goto out;
 	}
 
@@ -1416,7 +1435,8 @@ bool smb_inherit_flags(int flags, bool is_dir)
 }
 
 int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path,
-			__le32 *pdaccess, int uid)
+			__le32 *pdaccess, __le32 raw_daccess, int uid,
+			bool strict)
 {
 	struct mnt_idmap *idmap = mnt_idmap(path->mnt);
 	struct smb_ntsd *pntsd = NULL;
@@ -1426,14 +1446,17 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path,
 	unsigned int dacl_offset;
 	size_t dacl_struct_end;
 	struct smb_sid sid;
-	int granted = le32_to_cpu(*pdaccess & ~FILE_MAXIMAL_ACCESS_LE);
+	int requested = le32_to_cpu(*pdaccess & ~FILE_MAXIMAL_ACCESS_LE);
+	int granted = requested;
 	struct smb_ace *ace;
 	int i, found = 0;
-	unsigned int access_bits = 0;
+	unsigned int access_bits = 0, denied = 0;
 	struct smb_ace *others_ace = NULL;
 	struct posix_acl_entry *pa_entry;
 	unsigned int sid_type = SIDOWNER;
 	unsigned short ace_size;
+	bool is_owner, owner_rights = false;
+	vfsuid_t vfsuid;
 
 	ksmbd_debug(SMB, "check permission using windows acl\n");
 	pntsd_size = ksmbd_vfs_get_sd_xattr(conn, idmap,
@@ -1463,10 +1486,13 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path,
 		goto err_out;
 	}
 
-	if (*pdaccess & FILE_MAXIMAL_ACCESS_LE) {
-		granted = READ_CONTROL | WRITE_DAC | FILE_READ_ATTRIBUTES |
-			DELETE;
+	if (!uid)
+		sid_type = SIDUNIX_USER;
+	id_to_sid(uid, sid_type, &sid);
+	vfsuid = i_uid_into_vfsuid(idmap, d_inode(path->dentry));
+	is_owner = uid == from_kuid(&init_user_ns, vfsuid_into_kuid(vfsuid));
 
+	if (*pdaccess & FILE_MAXIMAL_ACCESS_LE) {
 		ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl));
 		aces_size = acl_size - sizeof(struct smb_acl);
 		for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) {
@@ -1479,15 +1505,51 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path,
 				       CIFS_SID_BASE_SIZE)
 				break;
 			aces_size -= ace_size;
-			granted |= le32_to_cpu(ace->access_req);
+
+			if (ace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES ||
+			    ace_size < offsetof(struct smb_ace, sid) +
+				      CIFS_SID_BASE_SIZE +
+				      sizeof(__le32) * ace->sid.num_subauth)
+				break;
+
+			if (!compare_sids(&sid_owner_rights, &ace->sid)) {
+				owner_rights = true;
+				if (!is_owner)
+					goto next_ace;
+			}
+
+			if (ace->flags & INHERIT_ONLY_ACE ||
+			    (compare_sids(&sid, &ace->sid) &&
+			     compare_sids(&sid_unix_NFS_mode, &ace->sid) &&
+			     compare_sids(&sid_everyone, &ace->sid) &&
+			     compare_sids(&sid_authusers, &ace->sid) &&
+			     compare_sids(&sid_owner_rights, &ace->sid)))
+				goto next_ace;
+
+			switch (ace->type) {
+			case ACCESS_ALLOWED_ACE_TYPE:
+				access_bits |= le32_to_cpu(ace->access_req);
+				break;
+			case ACCESS_DENIED_ACE_TYPE:
+			case ACCESS_DENIED_CALLBACK_ACE_TYPE:
+				denied |= ~access_bits &
+					le32_to_cpu(ace->access_req);
+				break;
+			}
+next_ace:
 			ace = (struct smb_ace *)((char *)ace + le16_to_cpu(ace->size));
 		}
+		if (is_owner && !owner_rights)
+			access_bits |= READ_CONTROL | WRITE_DAC |
+				FILE_READ_ATTRIBUTES | DELETE;
+		access_bits &= ~denied;
+		if ((raw_daccess & FILE_GENERIC_EXECUTE_LE) &&
+		    S_ISREG(d_inode(path->dentry)->i_mode) &&
+		    (access_bits & GENERIC_READ_FLAGS) == GENERIC_READ_FLAGS)
+			access_bits |= FILE_EXECUTE;
+		granted = requested | access_bits;
 	}
 
-	if (!uid)
-		sid_type = SIDUNIX_USER;
-	id_to_sid(uid, sid_type, &sid);
-
 	ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl));
 	aces_size = acl_size - sizeof(struct smb_acl);
 	for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) {
@@ -1511,25 +1573,16 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path,
 			found = 1;
 			break;
 		}
-		if (!compare_sids(&sid_everyone, &ace->sid))
+		if (!compare_sids(&sid_everyone, &ace->sid) ||
+		    !compare_sids(&sid_authusers, &ace->sid))
 			others_ace = ace;
 
 		ace = (struct smb_ace *)((char *)ace + le16_to_cpu(ace->size));
 	}
 
-	if (*pdaccess & FILE_MAXIMAL_ACCESS_LE && found) {
-		granted = READ_CONTROL | WRITE_DAC | FILE_READ_ATTRIBUTES |
-			DELETE;
-
-		granted |= le32_to_cpu(ace->access_req);
-
-		if (!pdacl->num_aces)
-			granted = GENERIC_ALL_FLAGS;
-	}
-
 	if (IS_ENABLED(CONFIG_FS_POSIX_ACL)) {
 		posix_acls = get_inode_acl(d_inode(path->dentry), ACL_TYPE_ACCESS);
-		if (!IS_ERR_OR_NULL(posix_acls) && !found) {
+		if (!IS_ERR_OR_NULL(posix_acls) && !found && !others_ace) {
 			unsigned int id = -1;
 
 			pa_entry = posix_acls->a_entries;
@@ -1567,19 +1620,27 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path,
 		}
 	}
 
-	switch (ace->type) {
-	case ACCESS_ALLOWED_ACE_TYPE:
-		access_bits = le32_to_cpu(ace->access_req);
-		break;
-	case ACCESS_DENIED_ACE_TYPE:
-	case ACCESS_DENIED_CALLBACK_ACE_TYPE:
-		access_bits = le32_to_cpu(~ace->access_req);
-		break;
+	if (!(*pdaccess & FILE_MAXIMAL_ACCESS_LE)) {
+		switch (ace->type) {
+		case ACCESS_ALLOWED_ACE_TYPE:
+			access_bits = le32_to_cpu(ace->access_req);
+			break;
+		case ACCESS_DENIED_ACE_TYPE:
+		case ACCESS_DENIED_CALLBACK_ACE_TYPE:
+			access_bits = le32_to_cpu(~ace->access_req);
+			break;
+		}
 	}
 
 check_access_bits:
-	if (granted &
-	    ~(access_bits | FILE_READ_ATTRIBUTES | READ_CONTROL | WRITE_DAC | DELETE)) {
+	if (strict) {
+		access_bits &= granted;
+	} else {
+		access_bits |= FILE_READ_ATTRIBUTES | READ_CONTROL |
+			WRITE_DAC | DELETE;
+	}
+
+	if (granted & ~access_bits) {
 		ksmbd_debug(SMB, "Access denied with winACL, granted : %x, access_req : %x\n",
 			    granted, le32_to_cpu(ace->access_req));
 		rc = -EACCES;
diff --git a/fs/smb/server/smbacl.h b/fs/smb/server/smbacl.h
index ab21ba2cd4df3..01810c16cc046 100644
--- a/fs/smb/server/smbacl.h
+++ b/fs/smb/server/smbacl.h
@@ -95,7 +95,8 @@ bool smb_inherit_flags(int flags, bool is_dir);
 int smb_inherit_dacl(struct ksmbd_conn *conn, const struct path *path,
 		     unsigned int uid, unsigned int gid);
 int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path,
-			__le32 *pdaccess, int uid);
+			__le32 *pdaccess, __le32 raw_daccess, int uid,
+			bool strict);
 int set_info_sec(struct ksmbd_conn *conn, struct ksmbd_tree_connect *tcon,
 		 const struct path *path, struct smb_ntsd *pntsd, int ntsd_len,
 		 bool type_check, bool get_write);
diff --git a/fs/smb/server/vfs.c b/fs/smb/server/vfs.c
index d0a0ad15d8037..8cfea4a3fdada 100644
--- a/fs/smb/server/vfs.c
+++ b/fs/smb/server/vfs.c
@@ -7,6 +7,7 @@
 #include <crypto/sha2.h>
 #include <linux/kernel.h>
 #include <linux/fs.h>
+#include <linux/fs_struct.h>
 #include <linux/filelock.h>
 #include <linux/uaccess.h>
 #include <linux/backing-dev.h>
@@ -67,8 +68,9 @@ static int ksmbd_vfs_path_lookup(struct ksmbd_share_config *share_conf,
 	}
 
 	CLASS(filename_kernel, filename)(pathname);
-	err = vfs_path_parent_lookup(filename, flags, path, &last,
-				     root_share_path);
+	scoped_with_init_fs()
+		err = vfs_path_parent_lookup(filename, flags, path, &last,
+					     root_share_path);
 	if (err)
 		return err;
 
@@ -297,9 +299,6 @@ static int check_lock_range(struct file *filp, loff_t start, loff_t end,
 	struct file_lock_context *ctx = locks_inode_context(file_inode(filp));
 	int error = 0;
 
-	if (start == end)
-		return 0;
-
 	if (!ctx || list_empty_careful(&ctx->flc_posix))
 		return 0;
 
@@ -345,7 +344,7 @@ int ksmbd_vfs_read(struct ksmbd_work *work, struct ksmbd_file *fp, size_t count,
 	ssize_t nbytes = 0;
 	struct inode *inode = file_inode(filp);
 
-	if (S_ISDIR(inode->i_mode))
+	if (S_ISDIR(inode->i_mode) && !ksmbd_stream_fd(fp))
 		return -EISDIR;
 
 	if (unlikely(count == 0))
@@ -474,7 +473,8 @@ int ksmbd_vfs_write(struct ksmbd_work *work, struct ksmbd_file *fp,
 
 	if (work->conn->connection_type) {
 		if (!(fp->daccess & (FILE_WRITE_DATA_LE | FILE_APPEND_DATA_LE)) ||
-		    S_ISDIR(file_inode(fp->filp)->i_mode)) {
+		    (S_ISDIR(file_inode(fp->filp)->i_mode) &&
+		     !ksmbd_stream_fd(fp))) {
 			pr_err("no right to write(%pD)\n", fp->filp);
 			err = -EACCES;
 			goto out;
@@ -623,7 +623,8 @@ int ksmbd_vfs_link(struct ksmbd_work *work, const char *oldname,
 	if (ksmbd_override_fsids(work))
 		return -ENOMEM;
 
-	err = kern_path(oldname, LOOKUP_NO_SYMLINKS, &oldpath);
+	scoped_with_init_fs()
+		err = kern_path(oldname, LOOKUP_NO_SYMLINKS, &oldpath);
 	if (err) {
 		pr_err("cannot get linux path for %s, err = %d\n",
 		       oldname, err);
diff --git a/fs/smb/smbdirect/accept.c b/fs/smb/smbdirect/accept.c
index 5297400058385..44b681a207253 100644
--- a/fs/smb/smbdirect/accept.c
+++ b/fs/smb/smbdirect/accept.c
@@ -32,8 +32,9 @@ int smbdirect_accept_connect_request(struct smbdirect_socket *sc,
 	/*
 	 * First set what the we as server are able to support
 	 */
-	sp->initiator_depth = min_t(u8, sp->initiator_depth,
-				    sc->ib.dev->attrs.max_qp_rd_atom);
+	sp->initiator_depth = min3(sp->initiator_depth,
+				   sc->ib.dev->attrs.max_qp_rd_atom,
+				   U8_MAX);
 
 	peer_initiator_depth = param->initiator_depth;
 	peer_responder_resources = param->responder_resources;
diff --git a/fs/smb/smbdirect/connect.c b/fs/smb/smbdirect/connect.c
index cd726b399afec..34a3e72c38fbb 100644
--- a/fs/smb/smbdirect/connect.c
+++ b/fs/smb/smbdirect/connect.c
@@ -182,8 +182,9 @@ static int smbdirect_connect_rdma_connect(struct smbdirect_socket *sc)
 	if (sc->ib.dev->attrs.kernel_cap_flags & IBK_SG_GAPS_REG)
 		sc->mr_io.type = IB_MR_TYPE_SG_GAPS;
 
-	sp->responder_resources = min_t(u8, sp->responder_resources,
-					sc->ib.dev->attrs.max_qp_rd_atom);
+	sp->responder_resources = min3(sp->responder_resources,
+				       sc->ib.dev->attrs.max_qp_rd_atom,
+				       U8_MAX);
 	smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_INFO,
 		"responder_resources=%d\n",
 		sp->responder_resources);
diff --git a/fs/smb/smbdirect/connection.c b/fs/smb/smbdirect/connection.c
index 8adf580975344..690acb84e1b59 100644
--- a/fs/smb/smbdirect/connection.c
+++ b/fs/smb/smbdirect/connection.c
@@ -287,7 +287,7 @@ int smbdirect_connection_create_qp(struct smbdirect_socket *sc)
 	    qp_cap.max_send_wr > sc->ib.dev->attrs.max_qp_wr) {
 		pr_err("Possible CQE overrun: max_send_wr %d\n",
 		       qp_cap.max_send_wr);
-		pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n",
+		pr_err("device %.*s reporting max_cqe %u max_qp_wr %u\n",
 		       IB_DEVICE_NAME_MAX,
 		       sc->ib.dev->name,
 		       sc->ib.dev->attrs.max_cqe,
@@ -302,7 +302,7 @@ int smbdirect_connection_create_qp(struct smbdirect_socket *sc)
 	     max_send_wr >= sc->ib.dev->attrs.max_qp_wr)) {
 		pr_err("Possible CQE overrun: rdma_send_wr %d + max_send_wr %d = %d\n",
 		       rdma_send_wr, qp_cap.max_send_wr, max_send_wr);
-		pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n",
+		pr_err("device %.*s reporting max_cqe %u max_qp_wr %u\n",
 		       IB_DEVICE_NAME_MAX,
 		       sc->ib.dev->name,
 		       sc->ib.dev->attrs.max_cqe,
@@ -316,7 +316,7 @@ int smbdirect_connection_create_qp(struct smbdirect_socket *sc)
 	    qp_cap.max_recv_wr > sc->ib.dev->attrs.max_qp_wr) {
 		pr_err("Possible CQE overrun: max_recv_wr %d\n",
 		       qp_cap.max_recv_wr);
-		pr_err("device %.*s reporting max_cqe %d max_qp_wr %d\n",
+		pr_err("device %.*s reporting max_cqe %u max_qp_wr %u\n",
 		       IB_DEVICE_NAME_MAX,
 		       sc->ib.dev->name,
 		       sc->ib.dev->attrs.max_cqe,
@@ -328,7 +328,7 @@ int smbdirect_connection_create_qp(struct smbdirect_socket *sc)
 
 	if (qp_cap.max_send_sge > sc->ib.dev->attrs.max_send_sge ||
 	    qp_cap.max_recv_sge > sc->ib.dev->attrs.max_recv_sge) {
-		pr_err("device %.*s max_send_sge/max_recv_sge = %d/%d too small\n",
+		pr_err("device %.*s max_send_sge/max_recv_sge = %u/%u too small\n",
 		       IB_DEVICE_NAME_MAX,
 		       sc->ib.dev->name,
 		       sc->ib.dev->attrs.max_send_sge,
diff --git a/fs/super.c b/fs/super.c
index a8fd61136aafe..9f82e6b38f07e 100644
--- a/fs/super.c
+++ b/fs/super.c
@@ -24,6 +24,8 @@
 #include <linux/export.h>
 #include <linux/slab.h>
 #include <linux/blkdev.h>
+#include <linux/memcontrol.h>
+#include <linux/rhashtable.h>
 #include <linux/mount.h>
 #include <linux/security.h>
 #include <linux/writeback.h>		/* for the emergency remount stuff */
@@ -102,7 +104,7 @@ static bool super_flags(const struct super_block *sb, unsigned int flags)
  * creation will succeed and SB_BORN is set by vfs_get_tree() or we're
  * woken and we'll see SB_DYING.
  *
- * The caller must have acquired a temporary reference on @sb->s_count.
+ * The caller must have acquired a temporary reference on @sb->s_passive.
  *
  * Return: The function returns true if SB_BORN was set and with
  *         s_umount held. The function returns false if SB_DYING was
@@ -169,6 +171,19 @@ static void super_wake(struct super_block *sb, unsigned int flag)
 	wake_up_var(&sb->s_flags);
 }
 
+/*
+ * The s_op->nr_cached_objects hooks (used for example by btrfs and xfs)
+ * operate on filesystem-global state and ignore sc->memcg. Driving them
+ * from per-memcg shrink_slab_memcg() invocations only burns CPU walking
+ * per-cpu counters and queueing duplicate work: the actual reclaim happens on
+ * the global path (kswapd or root direct reclaim) regardless. Restrict them
+ * to that path.
+ */
+static inline bool super_fs_objects_eligible(struct shrink_control *sc)
+{
+	return !sc->memcg || mem_cgroup_is_root(sc->memcg);
+}
+
 /*
  * One thing we have to be careful of with a per-sb shrinker is that we don't
  * drop the last active reference to the superblock from within the shrinker.
@@ -198,7 +213,7 @@ static unsigned long super_cache_scan(struct shrinker *shrink,
 	if (!super_trylock_shared(sb))
 		return SHRINK_STOP;
 
-	if (sb->s_op->nr_cached_objects)
+	if (sb->s_op->nr_cached_objects && super_fs_objects_eligible(sc))
 		fs_objects = sb->s_op->nr_cached_objects(sb, sc);
 
 	inodes = list_lru_shrink_count(&sb->s_inode_lru, sc);
@@ -259,7 +274,8 @@ static unsigned long super_cache_count(struct shrinker *shrink,
 		return 0;
 	smp_rmb();
 
-	if (sb->s_op && sb->s_op->nr_cached_objects)
+	if (sb->s_op && sb->s_op->nr_cached_objects &&
+	    super_fs_objects_eligible(sc))
 		total_objects = sb->s_op->nr_cached_objects(sb, sc);
 
 	total_objects += list_lru_shrink_count(&sb->s_dentry_lru, sc);
@@ -272,6 +288,8 @@ static unsigned long super_cache_count(struct shrinker *shrink,
 	return total_objects;
 }
 
+static struct super_dev *super_dev_alloc(dev_t dev, struct super_block *sb);
+
 static void destroy_super_work(struct work_struct *work)
 {
 	struct super_block *s = container_of(work, struct super_block,
@@ -279,6 +297,8 @@ static void destroy_super_work(struct work_struct *work)
 	fsnotify_sb_free(s);
 	security_sb_free(s);
 	put_user_ns(s->s_user_ns);
+	/* Only an unregistered entry is still owned by the superblock. */
+	kfree(s->s_super_dev);
 	kfree(s->s_subtype);
 	for (int i = 0; i < SB_FREEZE_LEVELS; i++)
 		percpu_free_rwsem(&s->s_writers.rw_sem[i]);
@@ -367,7 +387,7 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags,
 	spin_lock_init(&s->s_inode_wblist_lock);
 	fserror_mount(s);
 
-	s->s_count = 1;
+	refcount_set(&s->s_passive, 1);
 	atomic_set(&s->s_active, 1);
 	mutex_init(&s->s_vfs_rename_mutex);
 	lockdep_set_class(&s->s_vfs_rename_mutex, &type->s_vfs_rename_key);
@@ -392,6 +412,10 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags,
 		goto fail;
 	if (list_lru_init_memcg(&s->s_inode_lru, s->s_shrink))
 		goto fail;
+	s->s_super_dev = super_dev_alloc(0, s);
+	if (!s->s_super_dev)
+		goto fail;
+
 	s->s_min_writeback_pages = MIN_WRITEBACK_PAGES;
 	return s;
 
@@ -403,12 +427,17 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags,
 /* Superblock refcounting  */
 
 /*
- * Drop a superblock's refcount.  The caller must hold sb_lock.
+ * Drop a superblock's passive reference.  Must be called WITHOUT sb_lock held;
+ * put_super() acquires sb_lock itself when the final reference is dropped.
  */
-static void __put_super(struct super_block *s)
+void put_super(struct super_block *s)
 {
-	if (!--s->s_count) {
+	if (refcount_dec_and_test(&s->s_passive)) {
+
+		spin_lock(&sb_lock);
 		list_del_init(&s->s_list);
+		spin_unlock(&sb_lock);
+
 		WARN_ON(s->s_dentry_lru.node);
 		WARN_ON(s->s_inode_lru.node);
 		WARN_ON(s->s_mounts);
@@ -416,18 +445,109 @@ static void __put_super(struct super_block *s)
 	}
 }
 
-/**
- *	put_super	-	drop a temporary reference to superblock
- *	@sb: superblock in question
- *
- *	Drops a temporary reference, frees superblock if there's no
- *	references left.
- */
-void put_super(struct super_block *sb)
+struct super_dev {
+	dev_t			sd_dev;
+	struct super_block	*sd_sb;
+	refcount_t		sd_ref;
+	struct rhlist_head	sd_node;
+	struct rcu_head		sd_rcu;
+};
+
+static struct rhltable super_dev_table;
+static const struct rhashtable_params super_dev_params = {
+	.key_len	= sizeof(dev_t),
+	.key_offset	= offsetof(struct super_dev, sd_dev),
+	.head_offset	= offsetof(struct super_dev, sd_node),
+};
+
+static struct super_dev *super_dev_alloc(dev_t dev, struct super_block *sb)
 {
-	spin_lock(&sb_lock);
-	__put_super(sb);
-	spin_unlock(&sb_lock);
+	struct super_dev *fsd;
+
+	fsd = kzalloc_obj(*fsd);
+	if (!fsd)
+		return NULL;
+	fsd->sd_dev = dev;
+	fsd->sd_sb = sb;
+	refcount_set(&fsd->sd_ref, 1);
+	return fsd;
+}
+
+static void super_dev_put(struct super_dev *fsd)
+{
+	/* Unlink only once unpinned, so a cursor never resumes from a removed node. */
+	if (fsd && refcount_dec_and_test(&fsd->sd_ref)) {
+		rhltable_remove(&super_dev_table, &fsd->sd_node, super_dev_params);
+		put_super(fsd->sd_sb);
+		kfree_rcu(fsd, sd_rcu);
+	}
+}
+
+void __init super_dev_init(void)
+{
+	if (rhltable_init(&super_dev_table, &super_dev_params))
+		panic("VFS: Cannot initialise super_dev_table\n");
+}
+
+static int super_dev_insert(struct super_dev *fsd)
+{
+	int err;
+
+	err = rhltable_insert(&super_dev_table, &fsd->sd_node, super_dev_params);
+	if (!err)
+		refcount_inc(&fsd->sd_sb->s_passive);
+	return err;
+}
+
+/* Register @sb under @sb->s_dev as the final fallible act of a set callback. */
+static int super_dev_register(struct super_block *sb)
+{
+	struct super_dev *fsd = sb->s_super_dev;
+	int err;
+
+	lockdep_assert_held(&sb_lock);
+	VFS_WARN_ON_ONCE(!sb->s_dev);
+	VFS_WARN_ON_ONCE(!fsd || fsd->sd_dev);
+
+	fsd->sd_dev = sb->s_dev;
+	err = super_dev_insert(fsd);
+	if (err)
+		fsd->sd_dev = 0;
+	return err;
+}
+
+static struct super_dev *super_dev_get(struct rhlist_head *pos)
+{
+	struct super_dev *sb_dev;
+
+	for (; pos; pos = rcu_dereference_all(pos->next)) {
+		sb_dev = container_of(pos, struct super_dev, sd_node);
+		if (refcount_inc_not_zero(&sb_dev->sd_ref))
+			return sb_dev;
+	}
+	return NULL;
+}
+
+static struct super_dev *super_dev_first(dev_t dev)
+{
+	struct super_dev *sb_dev;
+
+	rcu_read_lock();
+	sb_dev = super_dev_get(rhltable_lookup(&super_dev_table, &dev, super_dev_params));
+	rcu_read_unlock();
+	return sb_dev;
+}
+
+static struct super_dev *super_dev_next(struct super_dev *prev)
+{
+	struct super_dev *sb_dev;
+
+	rcu_read_lock();
+	sb_dev = super_dev_get(rcu_dereference_all(prev->sd_node.next));
+	rcu_read_unlock();
+
+	super_dev_put(prev);
+	return sb_dev;
 }
 
 static void kill_super_notify(struct super_block *sb)
@@ -449,6 +569,12 @@ static void kill_super_notify(struct super_block *sb)
 	hlist_del_init(&sb->s_instances);
 	spin_unlock(&sb_lock);
 
+	/* Drop sget_fc()'s claim; a never-registered entry stays with the sb. */
+	if (sb->s_super_dev->sd_dev) {
+		super_dev_put(sb->s_super_dev);
+		sb->s_super_dev = NULL;
+	}
+
 	/*
 	 * Let concurrent mounts know that this thing is really dead.
 	 * We don't need @sb->s_umount here as every concurrent caller
@@ -478,11 +604,7 @@ void deactivate_locked_super(struct super_block *s)
 
 		kill_super_notify(s);
 
-		/*
-		 * Since list_lru_destroy() may sleep, we cannot call it from
-		 * put_super(), where we hold the sb_lock. Therefore we destroy
-		 * the lru lists right now.
-		 */
+		/* list_lru_destroy() may sleep; put_super() callers may not. */
 		list_lru_destroy(&s->s_dentry_lru);
 		list_lru_destroy(&s->s_inode_lru);
 
@@ -529,7 +651,7 @@ static bool grab_super(struct super_block *sb)
 {
 	bool locked;
 
-	sb->s_count++;
+	refcount_inc(&sb->s_passive);
 	spin_unlock(&sb_lock);
 	locked = super_lock_excl(sb);
 	if (locked) {
@@ -556,7 +678,7 @@ static bool grab_super(struct super_block *sb)
  *	lock held in read mode in case of success. On successful return,
  *	the caller must drop the s_umount lock when done.
  *
- *	Note that unlike get_super() et.al. this one does *not* bump ->s_count.
+ *	Note that unlike get_super() et.al. this one does *not* bump ->s_passive.
  *	The reason why it's safe is that we are OK with doing trylock instead
  *	of down_read().  There's a couple of places that are OK with that, but
  *	it's very much not a general-purpose interface.
@@ -763,6 +885,7 @@ struct super_block *sget_fc(struct fs_context *fc,
 	}
 	if (!s) {
 		spin_unlock(&sb_lock);
+
 		s = alloc_super(fc->fs_type, fc->sb_flags, user_ns);
 		if (!s)
 			return ERR_PTR(-ENOMEM);
@@ -772,11 +895,13 @@ struct super_block *sget_fc(struct fs_context *fc,
 	s->s_fs_info = fc->s_fs_info;
 	err = set(s, fc);
 	if (err) {
+		VFS_WARN_ON_ONCE(s->s_super_dev->sd_dev);
 		s->s_fs_info = NULL;
 		spin_unlock(&sb_lock);
 		destroy_unused_super(s);
 		return ERR_PTR(err);
 	}
+	VFS_WARN_ON_ONCE(!s->s_super_dev->sd_dev);
 	fc->s_fs_info = NULL;
 	s->s_type = fc->fs_type;
 	s->s_iflags |= fc->s_iflags;
@@ -851,14 +976,17 @@ static void __iterate_supers(void (*f)(struct super_block *, void *), void *arg,
 	struct super_block *sb, *p = NULL;
 	bool excl = flags & SUPER_ITER_EXCL;
 
-	guard(spinlock)(&sb_lock);
+	spin_lock(&sb_lock);
 
 	for (sb = first_super(flags);
 	     !list_entry_is_head(sb, &super_blocks, s_list);
 	     sb = next_super(sb, flags)) {
 		if (super_flags(sb, SB_DYING))
 			continue;
-		sb->s_count++;
+
+		if (!refcount_inc_not_zero(&sb->s_passive))
+			continue;
+
 		spin_unlock(&sb_lock);
 
 		if (flags & SUPER_ITER_UNLOCKED) {
@@ -868,13 +996,14 @@ static void __iterate_supers(void (*f)(struct super_block *, void *), void *arg,
 			super_unlock(sb, excl);
 		}
 
-		spin_lock(&sb_lock);
 		if (p)
-			__put_super(p);
+			put_super(p);
 		p = sb;
+		spin_lock(&sb_lock);
 	}
+	spin_unlock(&sb_lock);
 	if (p)
-		__put_super(p);
+		put_super(p);
 }
 
 void iterate_supers(void (*f)(struct super_block *, void *), void *arg)
@@ -903,7 +1032,9 @@ void iterate_supers_type(struct file_system_type *type,
 		if (super_flags(sb, SB_DYING))
 			continue;
 
-		sb->s_count++;
+		if (!refcount_inc_not_zero(&sb->s_passive))
+			continue;
+
 		spin_unlock(&sb_lock);
 
 		locked = super_lock_shared(sb);
@@ -912,41 +1043,33 @@ void iterate_supers_type(struct file_system_type *type,
 			super_unlock_shared(sb);
 		}
 
-		spin_lock(&sb_lock);
 		if (p)
-			__put_super(p);
+			put_super(p);
 		p = sb;
+		spin_lock(&sb_lock);
 	}
-	if (p)
-		__put_super(p);
 	spin_unlock(&sb_lock);
+	if (p)
+		put_super(p);
 }
 
 EXPORT_SYMBOL(iterate_supers_type);
 
 struct super_block *user_get_super(dev_t dev, bool excl)
 {
-	struct super_block *sb;
+	struct super_dev *sb_dev;
 
-	spin_lock(&sb_lock);
-	list_for_each_entry(sb, &super_blocks, s_list) {
-		bool locked;
+	for (sb_dev = super_dev_first(dev); sb_dev; sb_dev = super_dev_next(sb_dev)) {
+		struct super_block *sb = sb_dev->sd_sb;
 
-		if (sb->s_dev != dev)
+		if (!super_lock(sb, excl))
 			continue;
 
-		sb->s_count++;
-		spin_unlock(&sb_lock);
-
-		locked = super_lock(sb, excl);
-		if (locked)
-			return sb;
-
-		spin_lock(&sb_lock);
-		__put_super(sb);
-		break;
+		/* The pinned entry holds a passive reference, take our own. */
+		refcount_inc(&sb->s_passive);
+		super_dev_put(sb_dev);
+		return sb;
 	}
-	spin_unlock(&sb_lock);
 	return NULL;
 }
 
@@ -1222,7 +1345,16 @@ EXPORT_SYMBOL(free_anon_bdev);
 
 int set_anon_super(struct super_block *s, void *data)
 {
-	return get_anon_bdev(&s->s_dev);
+	int error;
+
+	error = get_anon_bdev(&s->s_dev);
+	if (error)
+		return error;
+
+	error = super_dev_register(s);
+	if (error)
+		free_anon_bdev(s->s_dev);
+	return error;
 }
 EXPORT_SYMBOL(set_anon_super);
 
@@ -1308,7 +1440,7 @@ EXPORT_SYMBOL(get_tree_keyed);
 static int set_bdev_super(struct super_block *s, void *data)
 {
 	s->s_dev = *(dev_t *)data;
-	return 0;
+	return super_dev_register(s);
 }
 
 static int super_s_dev_set(struct super_block *s, struct fs_context *fc)
@@ -1350,197 +1482,313 @@ struct super_block *sget_dev(struct fs_context *fc, dev_t dev)
 EXPORT_SYMBOL(sget_dev);
 
 #ifdef CONFIG_BLOCK
-/*
- * Lock the superblock that is holder of the bdev. Returns the superblock
- * pointer if we successfully locked the superblock and it is alive. Otherwise
- * we return NULL and just unlock bdev->bd_holder_lock.
- *
- * The function must be called with bdev->bd_holder_lock and releases it.
- */
-static struct super_block *bdev_super_lock(struct block_device *bdev, bool excl)
-	__releases(&bdev->bd_holder_lock)
+static int fs_super_freeze(struct super_block *sb)
 {
-	struct super_block *sb = bdev->bd_holder;
-	bool locked;
-
-	lockdep_assert_held(&bdev->bd_holder_lock);
-	lockdep_assert_not_held(&sb->s_umount);
-	lockdep_assert_not_held(&bdev->bd_disk->open_mutex);
-
-	/* Make sure sb doesn't go away from under us */
-	spin_lock(&sb_lock);
-	sb->s_count++;
-	spin_unlock(&sb_lock);
-
-	mutex_unlock(&bdev->bd_holder_lock);
-
-	locked = super_lock(sb, excl);
-
-	/*
-	 * If the superblock wasn't already SB_DYING then we hold
-	 * s_umount and can safely drop our temporary reference.
-         */
-	put_super(sb);
-
-	if (!locked)
-		return NULL;
-
-	if (!sb->s_root || !(sb->s_flags & SB_ACTIVE)) {
-		super_unlock(sb, excl);
-		return NULL;
-	}
+	if (sb->s_op->freeze_super)
+		return sb->s_op->freeze_super(sb,
+				FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL);
+	return freeze_super(sb, FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL);
+}
 
-	return sb;
+static int fs_super_thaw(struct super_block *sb)
+{
+	if (sb->s_op->thaw_super)
+		return sb->s_op->thaw_super(sb,
+				FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL);
+	return thaw_super(sb, FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL);
 }
 
 static void fs_bdev_mark_dead(struct block_device *bdev, bool surprise)
 {
-	struct super_block *sb;
+	struct super_dev *sb_dev;
+	dev_t dev = bdev->bd_dev;
 
-	sb = bdev_super_lock(bdev, false);
-	if (!sb)
-		return;
+	mutex_unlock(&bdev->bd_holder_lock);
 
-	if (sb->s_op->remove_bdev) {
-		int ret;
+	for (sb_dev = super_dev_first(dev); sb_dev; sb_dev = super_dev_next(sb_dev)) {
+		struct super_block *sb = sb_dev->sd_sb;
 
-		ret = sb->s_op->remove_bdev(sb, bdev);
-		if (!ret) {
-			super_unlock_shared(sb);
-			return;
+		if (!super_lock_shared(sb))
+			continue;
+		if (sb->s_root && (sb->s_flags & SB_ACTIVE)) {
+			if (!sb->s_op->remove_bdev ||
+			    sb->s_op->remove_bdev(sb, bdev)) {
+				if (!surprise)
+					sync_filesystem(sb);
+				shrink_dcache_sb(sb);
+				evict_inodes(sb);
+				if (sb->s_op->shutdown)
+					sb->s_op->shutdown(sb);
+			}
 		}
-		/* Fallback to shutdown. */
+		super_unlock_shared(sb);
 	}
-
-	if (!surprise)
-		sync_filesystem(sb);
-	shrink_dcache_sb(sb);
-	evict_inodes(sb);
-	if (sb->s_op->shutdown)
-		sb->s_op->shutdown(sb);
-
-	super_unlock_shared(sb);
 }
 
 static void fs_bdev_sync(struct block_device *bdev)
 {
-	struct super_block *sb;
-
-	sb = bdev_super_lock(bdev, false);
-	if (!sb)
-		return;
+	struct super_dev *sb_dev;
+	dev_t dev = bdev->bd_dev;
 
-	sync_filesystem(sb);
-	super_unlock_shared(sb);
-}
+	mutex_unlock(&bdev->bd_holder_lock);
 
-static struct super_block *get_bdev_super(struct block_device *bdev)
-{
-	bool active = false;
-	struct super_block *sb;
+	for (sb_dev = super_dev_first(dev); sb_dev; sb_dev = super_dev_next(sb_dev)) {
+		struct super_block *sb = sb_dev->sd_sb;
 
-	sb = bdev_super_lock(bdev, true);
-	if (sb) {
-		active = atomic_inc_not_zero(&sb->s_active);
-		super_unlock_excl(sb);
+		if (!super_lock_shared(sb))
+			continue;
+		if (sb->s_root && (sb->s_flags & SB_ACTIVE))
+			sync_filesystem(sb);
+		super_unlock_shared(sb);
 	}
-	if (!active)
-		return NULL;
-	return sb;
 }
 
 /**
- * fs_bdev_freeze - freeze owning filesystem of block device
+ * fs_bdev_freeze - freeze every superblock using a block device
  * @bdev: block device
  *
- * Freeze the filesystem that owns this block device if it is still
- * active.
- *
- * A filesystem that owns multiple block devices may be frozen from each
- * block device and won't be unfrozen until all block devices are
- * unfrozen. Each block device can only freeze the filesystem once as we
- * nest freezes for block devices in the block layer.
+ * Freeze each live superblock using @bdev.  A superblock owning several block
+ * devices is frozen once per device and stays frozen until all are thawed; the
+ * block layer nests these freezes so the count stays balanced.
  *
- * Return: If the freeze was successful zero is returned. If the freeze
- *         failed a negative error code is returned.
+ * Return: 0, or the error from the one superblock on a single-fs device.  When
+ *         several superblocks share @bdev a per-superblock failure is swallowed
+ *         (see below), but a sync_blockdev() failure is always reported.
  */
 static int fs_bdev_freeze(struct block_device *bdev)
 {
-	struct super_block *sb;
-	int error = 0;
+	dev_t dev = bdev->bd_dev;
+	struct super_dev *sb_dev;
+	unsigned int count = 0;
+	int error = 0, err;
 
 	lockdep_assert_held(&bdev->bd_fsfreeze_mutex);
 
-	sb = get_bdev_super(bdev);
-	if (!sb)
-		return -EINVAL;
+	mutex_unlock(&bdev->bd_holder_lock);
 
-	if (sb->s_op->freeze_super)
-		error = sb->s_op->freeze_super(sb,
-				FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL);
-	else
-		error = freeze_super(sb,
-				FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL);
+	for (sb_dev = super_dev_first(dev); sb_dev; sb_dev = super_dev_next(sb_dev)) {
+		if (!get_active_super(sb_dev->sd_sb))
+			continue;
+		err = fs_super_freeze(sb_dev->sd_sb);
+		if (err && !error)
+			error = err;
+		deactivate_super(sb_dev->sd_sb);
+		count++;
+	}
+
+	/*
+	 * When several superblocks share the device, keep it frozen even if some
+	 * of them failed to freeze and swallow the error: rolling the rest back
+	 * via thaw_super() can fail too, so neither is a clear win. A single
+	 * filesystem (count == 1) still reports its error.
+	 */
+	if (error && count > 1)
+		error = 0;
 	if (!error)
 		error = sync_blockdev(bdev);
-	deactivate_super(sb);
 	return error;
 }
 
 /**
- * fs_bdev_thaw - thaw owning filesystem of block device
+ * fs_bdev_thaw - thaw every superblock using a block device
  * @bdev: block device
  *
- * Thaw the filesystem that owns this block device.
- *
- * A filesystem that owns multiple block devices may be frozen from each
- * block device and won't be unfrozen until all block devices are
- * unfrozen. Each block device can only freeze the filesystem once as we
- * nest freezes for block devices in the block layer.
+ * The counterpart to fs_bdev_freeze(): thaw each live superblock using @bdev.
+ * A zero return does not imply a superblock is fully unfrozen; it may have been
+ * frozen more than once (by the kernel or via another device).
  *
- * Return: If the thaw was successful zero is returned. If the thaw
- *         failed a negative error code is returned. If this function
- *         returns zero it doesn't mean that the filesystem is unfrozen
- *         as it may have been frozen multiple times (kernel may hold a
- *         freeze or might be frozen from other block devices).
+ * Return: 0, or the first error on a single-fs device; a shared device swallows
+ *         per-superblock errors, as fs_bdev_freeze() does.
  */
 static int fs_bdev_thaw(struct block_device *bdev)
 {
-	struct super_block *sb;
-	int error;
+	dev_t dev = bdev->bd_dev;
+	struct super_dev *sb_dev;
+	unsigned int count = 0;
+	int error = 0, err;
 
 	lockdep_assert_held(&bdev->bd_fsfreeze_mutex);
 
-	/*
-	 * The block device may have been frozen before it was claimed by a
-	 * filesystem. Concurrently another process might try to mount that
-	 * frozen block device and has temporarily claimed the block device for
-	 * that purpose causing a concurrent fs_bdev_thaw() to end up here. The
-	 * mounter is already about to abort mounting because they still saw an
-	 * elevanted bdev->bd_fsfreeze_count so get_bdev_super() will return
-	 * NULL in that case.
-	 */
-	sb = get_bdev_super(bdev);
-	if (!sb)
-		return -EINVAL;
+	mutex_unlock(&bdev->bd_holder_lock);
 
-	if (sb->s_op->thaw_super)
-		error = sb->s_op->thaw_super(sb,
-				FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL);
-	else
-		error = thaw_super(sb,
-				FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL);
-	deactivate_super(sb);
+	for (sb_dev = super_dev_first(dev); sb_dev; sb_dev = super_dev_next(sb_dev)) {
+		if (!get_active_super(sb_dev->sd_sb))
+			continue;
+		err = fs_super_thaw(sb_dev->sd_sb);
+		if (err && !error)
+			error = err;
+		deactivate_super(sb_dev->sd_sb);
+		count++;
+	}
+
+	/* Shared device: swallow per-superblock errors, like fs_bdev_freeze(). */
+	if (error && count > 1)
+		error = 0;
 	return error;
 }
 
-const struct blk_holder_ops fs_holder_ops = {
+static const struct blk_holder_ops fs_holder_ops = {
 	.mark_dead		= fs_bdev_mark_dead,
 	.sync			= fs_bdev_sync,
 	.freeze			= fs_bdev_freeze,
 	.thaw			= fs_bdev_thaw,
 };
-EXPORT_SYMBOL_GPL(fs_holder_ops);
+
+static struct super_dev *super_dev_lookup(dev_t dev, struct super_block *sb)
+{
+	struct super_dev *it;
+	struct rhlist_head *list, *pos;
+
+	RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "suspicious super_dev_lookup() usage");
+	VFS_WARN_ON_ONCE(!dev);
+	VFS_WARN_ON_ONCE(!sb);
+
+	list = rhltable_lookup(&super_dev_table, &dev, super_dev_params);
+	rhl_for_each_entry_rcu(it, pos, list, sd_node) {
+		if (it->sd_sb == sb)
+			return it;
+	}
+
+	return NULL;
+}
+
+static int fs_bdev_register(struct file *bdev_file, struct super_block *sb)
+{
+	struct super_dev *sb_dev __free(kfree) = NULL;
+	dev_t dev = file_bdev(bdev_file)->bd_dev;
+	int err;
+
+	scoped_guard(rcu) {
+		sb_dev = super_dev_lookup(dev, sb);
+		if (sb_dev && refcount_inc_not_zero(&sb_dev->sd_ref)) {
+			retain_and_null_ptr(sb_dev);
+			return 0;
+		}
+	}
+
+	sb_dev = super_dev_alloc(dev, sb);
+	if (!sb_dev)
+		return -ENOMEM;
+
+	err = super_dev_insert(sb_dev);
+	if (err)
+		return err;
+
+	/* Publish the entry before reading the count; pairs with bdev_freeze(). */
+	smp_mb();
+	if (atomic_read(&file_bdev(bdev_file)->bd_fsfreeze_count) > 0) {
+		err = -EBUSY;
+		super_dev_put(sb_dev);
+	}
+
+	retain_and_null_ptr(sb_dev);
+	return err;
+}
+
+/**
+ * fs_bdev_file_open_by_dev - claim a block device on behalf of a superblock
+ * @dev: block device number
+ * @mode: open mode
+ * @holder: block-layer exclusivity token (a superblock, or the file_system_type
+ *          when the device may be shared by several superblocks of that type)
+ * @sb: superblock to drive fs_holder_ops events for
+ *
+ * Open @dev with &fs_holder_ops and register that @sb uses it, so device
+ * removal/sync/freeze/thaw are propagated to @sb (and any other superblock
+ * sharing @dev).  Must be paired with fs_bdev_file_release().
+ *
+ * Return: an opened block-device file or an ERR_PTR().
+ */
+struct file *fs_bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder,
+				      struct super_block *sb)
+{
+	struct file *bdev_file;
+	int err;
+
+	bdev_file = bdev_file_open_by_dev(dev, mode, holder, &fs_holder_ops);
+	if (IS_ERR(bdev_file))
+		return bdev_file;
+
+	err = fs_bdev_register(bdev_file, sb);
+	if (err) {
+		bdev_fput(bdev_file);
+		return ERR_PTR(err);
+	}
+	return bdev_file;
+}
+EXPORT_SYMBOL_GPL(fs_bdev_file_open_by_dev);
+
+/**
+ * fs_bdev_file_open_by_path - claim a block device on behalf of a superblock
+ * @path: path to the block device
+ * @mode: open mode
+ * @holder: block-layer exclusivity token (a superblock, or the file_system_type
+ *          when the device may be shared by several superblocks of that type)
+ * @sb: superblock to drive fs_holder_ops events for
+ *
+ * Open the block device at @path with &fs_holder_ops and register that @sb
+ * uses it, so device removal/sync/freeze/thaw are propagated to @sb (and any
+ * other superblock sharing the device).  Must be paired with
+ * fs_bdev_file_release().
+ *
+ * Return: an opened block-device file or an ERR_PTR().
+ */
+struct file *fs_bdev_file_open_by_path(const char *path, blk_mode_t mode,
+				       void *holder, struct super_block *sb)
+{
+	struct file *bdev_file;
+	int err;
+
+	bdev_file = bdev_file_open_by_path(path, mode, holder, &fs_holder_ops);
+	if (IS_ERR(bdev_file))
+		return bdev_file;
+
+	err = fs_bdev_register(bdev_file, sb);
+	if (err) {
+		bdev_fput(bdev_file);
+		return ERR_PTR(err);
+	}
+	return bdev_file;
+}
+EXPORT_SYMBOL_GPL(fs_bdev_file_open_by_path);
+
+/**
+ * fs_bdev_unregister - drop a superblock's claim on a block device
+ * @bdev_file: file returned by fs_bdev_file_open_by_{dev,path}()
+ * @sb: superblock the device was claimed for
+ *
+ * The inverse of fs_bdev_register(): drop one claim on the {dev, @sb} entry
+ * (the last claim unregisters it; a pinning cursor defers the actual unlink)
+ * without closing the device.  A caller that must act on the still-open device
+ * between unregistering and closing - e.g. re-allow freezing one denied for a
+ * membership change - pairs this with bdev_fput().  fs_bdev_file_release() is
+ * the common unregister-and-close.
+ */
+void fs_bdev_unregister(struct file *bdev_file, struct super_block *sb)
+{
+	dev_t dev = file_bdev(bdev_file)->bd_dev;
+	struct super_dev *sb_dev;
+
+	rcu_read_lock();
+	sb_dev = super_dev_lookup(dev, sb);
+	rcu_read_unlock();
+	super_dev_put(sb_dev);
+}
+EXPORT_SYMBOL_GPL(fs_bdev_unregister);
+
+/**
+ * fs_bdev_file_release - release a block device claimed for a superblock
+ * @bdev_file: file returned by fs_bdev_file_open_by_{dev,path}()
+ * @sb: superblock the device was claimed for
+ *
+ * Unregister the {dev, @sb} entry, then close the block device.
+ */
+void fs_bdev_file_release(struct file *bdev_file, struct super_block *sb)
+{
+	fs_bdev_unregister(bdev_file, sb);
+	bdev_fput(bdev_file);
+}
+EXPORT_SYMBOL_GPL(fs_bdev_file_release);
 
 int setup_bdev_super(struct super_block *sb, int sb_flags,
 		struct fs_context *fc)
@@ -1549,7 +1797,7 @@ int setup_bdev_super(struct super_block *sb, int sb_flags,
 	struct file *bdev_file;
 	struct block_device *bdev;
 
-	bdev_file = bdev_file_open_by_dev(sb->s_dev, mode, sb, &fs_holder_ops);
+	bdev_file = fs_bdev_file_open_by_dev(sb->s_dev, mode, sb, sb);
 	if (IS_ERR(bdev_file)) {
 		if (fc)
 			errorf(fc, "%s: Can't open blockdev", fc->source);
@@ -1563,20 +1811,19 @@ int setup_bdev_super(struct super_block *sb, int sb_flags,
 	 * writable from userspace even for a read-only block device.
 	 */
 	if ((mode & BLK_OPEN_WRITE) && bdev_read_only(bdev)) {
-		bdev_fput(bdev_file);
+		fs_bdev_file_release(bdev_file, sb);
 		return -EACCES;
 	}
 
-	/*
-	 * It is enough to check bdev was not frozen before we set
-	 * s_bdev as freezing will wait until SB_BORN is set.
-	 */
+	/* The sget_fc() entry is already published; pairs with bdev_freeze(). */
+	smp_mb();
 	if (atomic_read(&bdev->bd_fsfreeze_count) > 0) {
 		if (fc)
 			warnf(fc, "%pg: Can't mount, blockdev is frozen", bdev);
-		bdev_fput(bdev_file);
+		fs_bdev_file_release(bdev_file, sb);
 		return -EBUSY;
 	}
+
 	spin_lock(&sb_lock);
 	sb->s_bdev_file = bdev_file;
 	sb->s_bdev = bdev;
@@ -1665,7 +1912,7 @@ void kill_block_super(struct super_block *sb)
 	generic_shutdown_super(sb);
 	if (bdev) {
 		sync_blockdev(bdev);
-		bdev_fput(sb->s_bdev_file);
+		fs_bdev_file_release(sb->s_bdev_file, sb);
 	}
 }
 
diff --git a/fs/ubifs/dir.c b/fs/ubifs/dir.c
index 86d41e077e4d6..23ec924162d63 100644
--- a/fs/ubifs/dir.c
+++ b/fs/ubifs/dir.c
@@ -303,7 +303,7 @@ static int ubifs_prepare_create(struct inode *dir, struct dentry *dentry,
 }
 
 static int ubifs_create(struct mnt_idmap *idmap, struct inode *dir,
-			struct dentry *dentry, umode_t mode, bool excl)
+			struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode;
 	struct ubifs_info *c = dir->i_sb->s_fs_info;
@@ -1031,7 +1031,7 @@ static struct dentry *ubifs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 
 	sz_change = CALC_DENT_SIZE(fname_len(&nm));
 
-	inode = ubifs_new_inode(c, dir, S_IFDIR | mode, false);
+	inode = ubifs_new_inode(c, dir, mode, false);
 	if (IS_ERR(inode)) {
 		err = PTR_ERR(inode);
 		goto out_fname;
diff --git a/fs/udf/balloc.c b/fs/udf/balloc.c
index cc6dc6e1d84de..30cec56001491 100644
--- a/fs/udf/balloc.c
+++ b/fs/udf/balloc.c
@@ -502,6 +502,8 @@ static int udf_table_prealloc_blocks(struct super_block *sb,
 	int8_t etype = -1;
 	struct udf_inode_info *iinfo;
 	int ret = 0;
+	/* AED block freed by udf_delete_aext(), released after unlock */
+	struct kernel_lb_addr freed = { .partitionReferenceNum = 0xFFFF };
 
 	if (first_block >= sbi->s_partmaps[partition].s_partition_len)
 		return 0;
@@ -541,7 +543,7 @@ static int udf_table_prealloc_blocks(struct super_block *sb,
 			udf_write_aext(table, &epos, &eloc,
 					(etype << 30) | elen, 1);
 		} else
-			udf_delete_aext(table, epos);
+			udf_delete_aext(table, epos, &freed);
 	} else {
 		alloc_count = 0;
 	}
@@ -552,6 +554,8 @@ static int udf_table_prealloc_blocks(struct super_block *sb,
 	if (alloc_count)
 		udf_add_free_space(sb, partition, -alloc_count);
 	mutex_unlock(&sbi->s_alloc_mutex);
+	if (freed.partitionReferenceNum != 0xFFFF)
+		udf_free_blocks(sb, table, &freed, 0, 1);
 	return alloc_count;
 }
 
@@ -560,6 +564,8 @@ static udf_pblk_t udf_table_new_block(struct super_block *sb,
 			       uint32_t goal, int *err)
 {
 	struct udf_sb_info *sbi = UDF_SB(sb);
+	/* AED block freed by udf_delete_aext(), released after unlock */
+	struct kernel_lb_addr freed = { .partitionReferenceNum = 0xFFFF };
 	uint32_t spread = 0xFFFFFFFF, nspread = 0xFFFFFFFF;
 	udf_pblk_t newblock = 0;
 	uint32_t adsize;
@@ -643,12 +649,14 @@ static udf_pblk_t udf_table_new_block(struct super_block *sb,
 	if (goal_elen)
 		udf_write_aext(table, &goal_epos, &goal_eloc, goal_elen, 1);
 	else
-		udf_delete_aext(table, goal_epos);
+		udf_delete_aext(table, goal_epos, &freed);
 	brelse(goal_epos.bh);
 
 	udf_add_free_space(sb, partition, -1);
 
 	mutex_unlock(&sbi->s_alloc_mutex);
+	if (freed.partitionReferenceNum != 0xFFFF)
+		udf_free_blocks(sb, table, &freed, 0, 1);
 	*err = 0;
 	return newblock;
 }
diff --git a/fs/udf/inode.c b/fs/udf/inode.c
index 67bcf83758c8f..c97914aa8d8be 100644
--- a/fs/udf/inode.c
+++ b/fs/udf/inode.c
@@ -1204,7 +1204,7 @@ static int udf_update_extents(struct inode *inode, struct kernel_long_ad *laarr,
 
 	if (startnum > endnum) {
 		for (i = 0; i < (startnum - endnum); i++)
-			udf_delete_aext(inode, *epos);
+			udf_delete_aext(inode, *epos, NULL);
 	} else if (startnum < endnum) {
 		for (i = 0; i < (endnum - startnum); i++) {
 			err = udf_insert_aext(inode, *epos,
@@ -2299,6 +2299,13 @@ int udf_current_aext(struct inode *inode, struct extent_position *epos,
 		return -EINVAL;
 	}
 
+	if (eloc->partitionReferenceNum >= UDF_SB(inode->i_sb)->s_partitions) {
+		udf_debug("invalid partition reference %u (partitions %u)\n",
+			  eloc->partitionReferenceNum,
+			  UDF_SB(inode->i_sb)->s_partitions);
+		return -EFSCORRUPTED;
+	}
+
 	return 1;
 }
 
@@ -2328,7 +2335,8 @@ static int udf_insert_aext(struct inode *inode, struct extent_position epos,
 	return ret;
 }
 
-int8_t udf_delete_aext(struct inode *inode, struct extent_position epos)
+int8_t udf_delete_aext(struct inode *inode, struct extent_position epos,
+			struct kernel_lb_addr *freed)
 {
 	struct extent_position oepos;
 	int adsize;
@@ -2378,7 +2386,19 @@ int8_t udf_delete_aext(struct inode *inode, struct extent_position epos)
 	elen = 0;
 
 	if (epos.bh != oepos.bh) {
-		udf_free_blocks(inode->i_sb, inode, &epos.block, 0, 1);
+		/*
+		 * The block that held the now-empty allocation extent must be
+		 * returned to free space.  When the caller already holds
+		 * s_alloc_mutex (the space-table allocator in balloc.c),
+		 * freeing it inline would recurse through udf_free_blocks()
+		 * into udf_table_free_blocks() and deadlock re-acquiring
+		 * s_alloc_mutex.  In that case report the block to the caller,
+		 *  which frees it after dropping the lock.
+		 */
+		if (freed)
+			*freed = epos.block;
+		else
+			udf_free_blocks(inode->i_sb, inode, &epos.block, 0, 1);
 		udf_write_aext(inode, &oepos, &eloc, elen, 1);
 		udf_write_aext(inode, &oepos, &eloc, elen, 1);
 		if (!oepos.bh) {
diff --git a/fs/udf/namei.c b/fs/udf/namei.c
index 9a3b7cef36064..b90841ac0a403 100644
--- a/fs/udf/namei.c
+++ b/fs/udf/namei.c
@@ -371,7 +371,7 @@ static int udf_add_nondir(struct dentry *dentry, struct inode *inode)
 }
 
 static int udf_create(struct mnt_idmap *idmap, struct inode *dir,
-		      struct dentry *dentry, umode_t mode, bool excl)
+		      struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode = udf_new_inode(dir, mode);
 
@@ -428,7 +428,7 @@ static struct dentry *udf_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	struct udf_inode_info *dinfo = UDF_I(dir);
 	struct udf_inode_info *iinfo;
 
-	inode = udf_new_inode(dir, S_IFDIR | mode);
+	inode = udf_new_inode(dir, mode);
 	if (IS_ERR(inode))
 		return ERR_CAST(inode);
 
diff --git a/fs/udf/partition.c b/fs/udf/partition.c
index 2b85c9501bed8..ad8dcedca263d 100644
--- a/fs/udf/partition.c
+++ b/fs/udf/partition.c
@@ -55,7 +55,7 @@ uint32_t udf_get_pblock_virt15(struct super_block *sb, uint32_t block,
 	map = &sbi->s_partmaps[partition];
 	vdata = &map->s_type_specific.s_virtual;
 
-	if (block > vdata->s_num_entries) {
+	if (block >= vdata->s_num_entries) {
 		udf_debug("Trying to access block beyond end of VAT (%u max %u)\n",
 			  block, vdata->s_num_entries);
 		return 0xFFFFFFFF;
diff --git a/fs/udf/super.c b/fs/udf/super.c
index 7b85f5a2b79fc..9686078bba649 100644
--- a/fs/udf/super.c
+++ b/fs/udf/super.c
@@ -2054,6 +2054,17 @@ static int udf_load_vrs(struct super_block *sb, struct udf_options *uopt,
 	return 0;
 }
 
+static void udf_mark_buffer_dirty(struct buffer_head *bh)
+{
+	/*
+	 * We set buffer uptodate unconditionally here to avoid spurious
+	 * warnings from mark_buffer_dirty() when previous EIO has marked
+	 * the buffer as !uptodate
+	 */
+	set_buffer_uptodate(bh);
+	mark_buffer_dirty(bh);
+}
+
 static void udf_finalize_lvid(struct logicalVolIntegrityDesc *lvid)
 {
 	struct timespec64 ts;
@@ -2089,7 +2100,7 @@ static void udf_open_lvid(struct super_block *sb)
 		UDF_SET_FLAG(sb, UDF_FLAG_INCONSISTENT);
 
 	udf_finalize_lvid(lvid);
-	mark_buffer_dirty(bh);
+	udf_mark_buffer_dirty(bh);
 	sbi->s_lvid_dirty = 0;
 	mutex_unlock(&sbi->s_alloc_mutex);
 	/* Make opening of filesystem visible on the media immediately */
@@ -2122,14 +2133,8 @@ static void udf_close_lvid(struct super_block *sb)
 	if (!UDF_QUERY_FLAG(sb, UDF_FLAG_INCONSISTENT))
 		lvid->integrityType = cpu_to_le32(LVID_INTEGRITY_TYPE_CLOSE);
 
-	/*
-	 * We set buffer uptodate unconditionally here to avoid spurious
-	 * warnings from mark_buffer_dirty() when previous EIO has marked
-	 * the buffer as !uptodate
-	 */
-	set_buffer_uptodate(bh);
 	udf_finalize_lvid(lvid);
-	mark_buffer_dirty(bh);
+	udf_mark_buffer_dirty(bh);
 	sbi->s_lvid_dirty = 0;
 	mutex_unlock(&sbi->s_alloc_mutex);
 	/* Make closing of filesystem visible on the media immediately */
@@ -2411,7 +2416,7 @@ static int udf_sync_fs(struct super_block *sb, int wait)
 		 * Blockdevice will be synced later so we don't have to submit
 		 * the buffer for IO
 		 */
-		mark_buffer_dirty(bh);
+		udf_mark_buffer_dirty(bh);
 		sbi->s_lvid_dirty = 0;
 	}
 	mutex_unlock(&sbi->s_alloc_mutex);
diff --git a/fs/udf/truncate.c b/fs/udf/truncate.c
index 41b2bfd30449b..0990f94b85518 100644
--- a/fs/udf/truncate.c
+++ b/fs/udf/truncate.c
@@ -159,7 +159,7 @@ void udf_discard_prealloc(struct inode *inode)
 
 	if (etype == (EXT_NOT_RECORDED_ALLOCATED >> 30)) {
 		lbcount -= elen;
-		udf_delete_aext(inode, prev_epos);
+		udf_delete_aext(inode, prev_epos, NULL);
 		udf_free_blocks(inode->i_sb, inode, &eloc, 0,
 				DIV_ROUND_UP(elen, bsize));
 	}
diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h
index 6d951e05c0046..21de6925fc68e 100644
--- a/fs/udf/udfdecl.h
+++ b/fs/udf/udfdecl.h
@@ -170,7 +170,8 @@ extern int udf_add_aext(struct inode *, struct extent_position *,
 			struct kernel_lb_addr *, uint32_t, int);
 extern void udf_write_aext(struct inode *, struct extent_position *,
 			   struct kernel_lb_addr *, uint32_t, int);
-extern int8_t udf_delete_aext(struct inode *, struct extent_position);
+extern int8_t udf_delete_aext(struct inode *, struct extent_position,
+			       struct kernel_lb_addr *);
 extern int udf_next_aext(struct inode *inode, struct extent_position *epos,
 			 struct kernel_lb_addr *eloc, uint32_t *elen,
 			 int8_t *etype, int inc);
diff --git a/fs/ufs/namei.c b/fs/ufs/namei.c
index 5b3c85c932429..6703f3bcf76fc 100644
--- a/fs/ufs/namei.c
+++ b/fs/ufs/namei.c
@@ -70,8 +70,7 @@ static struct dentry *ufs_lookup(struct inode * dir, struct dentry *dentry, unsi
  * with d_instantiate(). 
  */
 static int ufs_create (struct mnt_idmap * idmap,
-		struct inode * dir, struct dentry * dentry, umode_t mode,
-		bool excl)
+		struct inode * dir, struct dentry * dentry, umode_t mode)
 {
 	struct inode *inode;
 
@@ -174,7 +173,7 @@ static struct dentry *ufs_mkdir(struct mnt_idmap * idmap, struct inode * dir,
 
 	inode_inc_link_count(dir);
 
-	inode = ufs_new_inode(dir, S_IFDIR|mode);
+	inode = ufs_new_inode(dir, mode);
 	err = PTR_ERR(inode);
 	if (IS_ERR(inode))
 		goto out_dir;
diff --git a/fs/unicode/utf8-core.c b/fs/unicode/utf8-core.c
index 543c60c124615..8c4b8e27e9562 100644
--- a/fs/unicode/utf8-core.c
+++ b/fs/unicode/utf8-core.c
@@ -138,28 +138,6 @@ int utf8_casefold_hash(const struct unicode_map *um, const void *salt,
 }
 EXPORT_SYMBOL(utf8_casefold_hash);
 
-int utf8_normalize(const struct unicode_map *um, const struct qstr *str,
-		   unsigned char *dest, size_t dlen)
-{
-	struct utf8cursor cur;
-	ssize_t nlen = 0;
-
-	if (utf8ncursor(&cur, um, UTF8_NFDI, str->name, str->len) < 0)
-		return -EINVAL;
-
-	for (nlen = 0; nlen < dlen; nlen++) {
-		int c = utf8byte(&cur);
-
-		dest[nlen] = c;
-		if (!c)
-			return nlen;
-		if (c == -1)
-			break;
-	}
-	return -EINVAL;
-}
-EXPORT_SYMBOL(utf8_normalize);
-
 static const struct utf8data *find_table_version(const struct utf8data *table,
 		size_t nr_entries, unsigned int version)
 {
diff --git a/fs/vboxsf/dir.c b/fs/vboxsf/dir.c
index c5bd3271aa961..0b9eab157432d 100644
--- a/fs/vboxsf/dir.c
+++ b/fs/vboxsf/dir.c
@@ -298,9 +298,9 @@ static int vboxsf_dir_create(struct inode *parent, struct dentry *dentry,
 
 static int vboxsf_dir_mkfile(struct mnt_idmap *idmap,
 			     struct inode *parent, struct dentry *dentry,
-			     umode_t mode, bool excl)
+			     umode_t mode)
 {
-	return vboxsf_dir_create(parent, dentry, mode, false, excl, NULL);
+	return vboxsf_dir_create(parent, dentry, mode, false, true, NULL);
 }
 
 static struct dentry *vboxsf_dir_mkdir(struct mnt_idmap *idmap,
diff --git a/fs/xfs/libxfs/xfs_defer.c b/fs/xfs/libxfs/xfs_defer.c
index c6909716b0415..89501e8bd2f84 100644
--- a/fs/xfs/libxfs/xfs_defer.c
+++ b/fs/xfs/libxfs/xfs_defer.c
@@ -878,7 +878,7 @@ xfs_defer_add_barrier(
 	if (dfp)
 		return;
 
-	xfs_defer_alloc(&tp->t_dfops, &xfs_barrier_defer_type);
+	dfp = xfs_defer_alloc(&tp->t_dfops, &xfs_barrier_defer_type);
 
 	trace_xfs_defer_add_item(tp->t_mountp, dfp, NULL);
 }
diff --git a/fs/xfs/scrub/cow_repair.c b/fs/xfs/scrub/cow_repair.c
index 0075b6d5a1b5f..8dd9c0266e216 100644
--- a/fs/xfs/scrub/cow_repair.c
+++ b/fs/xfs/scrub/cow_repair.c
@@ -29,6 +29,7 @@
 #include "xfs_rtalloc.h"
 #include "xfs_rtbitmap.h"
 #include "xfs_rtgroup.h"
+#include "xfs_bmap_util.h"
 #include "scrub/xfs_scrub.h"
 #include "scrub/scrub.h"
 #include "scrub/common.h"
@@ -80,12 +81,6 @@ struct xrep_cow {
 	unsigned int		next_bno;
 };
 
-/* CoW staging extent. */
-struct xrep_cow_extent {
-	xfs_fsblock_t		fsbno;
-	xfs_extlen_t		len;
-};
-
 /*
  * Mark the part of the file range that corresponds to the given physical
  * space.  Caller must ensure that the physical range is within xc->irec.
@@ -230,6 +225,29 @@ xrep_cow_mark_missing_staging_rmap(
 			xfs_gbno_to_fsb(cur->bc_group, rec_bno), rec_len);
 }
 
+/*
+ * Trim the start and end of the current mapping by up to 1/4 of the length
+ * and mark that as "bad" to test the cow fork repair mechanism.
+ */
+static inline int
+xrep_cow_debug_replacement(
+	struct xrep_cow		*xc)
+{
+	xfs_fsblock_t		fsbno = xc->irec.br_startblock;
+	xfs_extlen_t		len = xc->irec.br_blockcount;
+	uint32_t		trim;
+
+	/* get_random_u32_below requires a nonzero argument */
+	trim = len > 4 ? get_random_u32_below(len / 4) : 0;
+	len -= trim;
+
+	trim = len > 4 ? get_random_u32_below(len / 4) : 0;
+	fsbno += trim;
+	len -= trim;
+
+	return xrep_cow_mark_file_range(xc, fsbno, len);
+}
+
 /*
  * Find any part of the CoW fork mapping that isn't a single-owner CoW staging
  * extent and mark the corresponding part of the file range in the bitmap.
@@ -299,8 +317,9 @@ xrep_cow_find_bad(
 	 * If userspace is forcing us to rebuild the CoW fork or someone turned
 	 * on the debugging knob, replace everything in the CoW fork.
 	 */
-	if ((sc->sm->sm_flags & XFS_SCRUB_IFLAG_FORCE_REBUILD) ||
-	    XFS_TEST_ERROR(sc->mp, XFS_ERRTAG_FORCE_SCRUB_REPAIR))
+	if (XFS_TEST_ERROR(sc->mp, XFS_ERRTAG_FORCE_SCRUB_REPAIR))
+		error = xrep_cow_debug_replacement(xc);
+	else if (sc->sm->sm_flags & XFS_SCRUB_IFLAG_FORCE_REBUILD)
 		error = xrep_cow_mark_file_range(xc, xc->irec.br_startblock,
 				xc->irec.br_blockcount);
 
@@ -381,8 +400,9 @@ xrep_cow_find_bad_rt(
 	 * turned on the debugging knob, replace everything in the
 	 * CoW fork and then scan for staging extents in the refcountbt.
 	 */
-	if ((sc->sm->sm_flags & XFS_SCRUB_IFLAG_FORCE_REBUILD) ||
-	    XFS_TEST_ERROR(sc->mp, XFS_ERRTAG_FORCE_SCRUB_REPAIR))
+	if (XFS_TEST_ERROR(sc->mp, XFS_ERRTAG_FORCE_SCRUB_REPAIR))
+		error = xrep_cow_debug_replacement(xc);
+	else if (sc->sm->sm_flags & XFS_SCRUB_IFLAG_FORCE_REBUILD)
 		error = xrep_cow_mark_file_range(xc, xc->irec.br_startblock,
 				xc->irec.br_blockcount);
 
@@ -401,22 +421,21 @@ xrep_cow_find_bad_rt(
 STATIC int
 xrep_cow_alloc(
 	struct xfs_scrub	*sc,
-	xfs_extlen_t		maxlen,
-	struct xrep_cow_extent	*repl)
+	struct xfs_bmbt_irec	*del)
 {
 	struct xfs_alloc_arg	args = {
 		.tp		= sc->tp,
 		.mp		= sc->mp,
 		.oinfo		= XFS_RMAP_OINFO_SKIP_UPDATE,
 		.minlen		= 1,
-		.maxlen		= maxlen,
+		.maxlen		= del->br_blockcount,
 		.prod		= 1,
 		.resv		= XFS_AG_RESV_NONE,
 		.datatype	= XFS_ALLOC_USERDATA,
 	};
 	int			error;
 
-	error = xfs_trans_reserve_more(sc->tp, maxlen, 0);
+	error = xfs_trans_reserve_more(sc->tp, del->br_blockcount, 0);
 	if (error)
 		return error;
 
@@ -428,8 +447,8 @@ xrep_cow_alloc(
 
 	xfs_refcount_alloc_cow_extent(sc->tp, false, args.fsbno, args.len);
 
-	repl->fsbno = args.fsbno;
-	repl->len = args.len;
+	del->br_startblock = args.fsbno;
+	del->br_blockcount = args.len;
 	return 0;
 }
 
@@ -440,10 +459,12 @@ xrep_cow_alloc(
 STATIC int
 xrep_cow_alloc_rt(
 	struct xfs_scrub	*sc,
-	xfs_extlen_t		maxlen,
-	struct xrep_cow_extent	*repl)
+	struct xfs_bmbt_irec	*del)
 {
-	xfs_rtxlen_t		maxrtx = xfs_rtb_to_rtx(sc->mp, maxlen);
+	xfs_fsblock_t		fsbno;
+	xfs_rtxlen_t		maxrtx =
+		min(U32_MAX, xfs_blen_to_rtbxlen(sc->mp, del->br_blockcount));
+	xfs_extlen_t		len;
 	int			error;
 
 	error = xfs_trans_reserve_more(sc->tp, 0, maxrtx);
@@ -451,11 +472,14 @@ xrep_cow_alloc_rt(
 		return error;
 
 	error = xfs_rtallocate_rtgs(sc->tp, NULLRTBLOCK, 1, maxrtx, 1, false,
-			false, &repl->fsbno, &repl->len);
+			false, &fsbno, &len);
 	if (error)
 		return error;
 
-	xfs_refcount_alloc_cow_extent(sc->tp, true, repl->fsbno, repl->len);
+	xfs_refcount_alloc_cow_extent(sc->tp, true, fsbno, len);
+
+	del->br_startblock = fsbno;
+	del->br_blockcount = len;
 	return 0;
 }
 
@@ -469,19 +493,19 @@ static inline int
 xrep_cow_find_mapping(
 	struct xrep_cow		*xc,
 	struct xfs_iext_cursor	*icur,
-	xfs_fileoff_t		startoff,
-	struct xfs_bmbt_irec	*got)
+	xfs_fileoff_t		badoff,
+	xfs_extlen_t		badlen,
+	struct xfs_bmbt_irec	*got,
+	struct xfs_bmbt_irec	*rep)
 {
 	struct xfs_inode	*ip = xc->sc->ip;
 	struct xfs_ifork	*ifp = xfs_ifork_ptr(ip, XFS_COW_FORK);
 
-	if (!xfs_iext_lookup_extent(ip, ifp, startoff, icur, got))
+	if (!xfs_iext_lookup_extent(ip, ifp, badoff, icur, got))
 		goto bad;
+	memcpy(rep, got, sizeof(*rep));
 
-	if (got->br_startoff > startoff)
-		goto bad;
-
-	if (got->br_blockcount == 0)
+	if (got->br_startoff > badoff)
 		goto bad;
 
 	if (isnullstartblock(got->br_startblock))
@@ -490,56 +514,28 @@ xrep_cow_find_mapping(
 	if (xfs_bmap_is_written_extent(got))
 		goto bad;
 
-	return 0;
-bad:
-	ASSERT(0);
-	return -EFSCORRUPTED;
-}
+	if (got->br_startoff < badoff) {
+		const int64_t	delta = badoff - got->br_startoff;
 
-#define REPLACE_LEFT_SIDE	(1U << 0)
-#define REPLACE_RIGHT_SIDE	(1U << 1)
-
-/*
- * Given a CoW fork mapping @got and a replacement mapping @repl, remap the
- * beginning of @got with the space described by @rep.
- */
-static inline void
-xrep_cow_replace_mapping(
-	struct xfs_inode		*ip,
-	struct xfs_iext_cursor		*icur,
-	const struct xfs_bmbt_irec	*got,
-	const struct xrep_cow_extent	*repl)
-{
-	struct xfs_bmbt_irec		new = *got; /* struct copy */
-
-	ASSERT(repl->len > 0);
-	ASSERT(!isnullstartblock(got->br_startblock));
+		rep->br_blockcount -= delta;
+		rep->br_startoff += delta;
+		rep->br_startblock += delta;
+	}
 
-	trace_xrep_cow_replace_mapping(ip, got, repl->fsbno, repl->len);
+	if (got->br_startoff + got->br_blockcount > badoff + badlen) {
+		const int64_t	delta = (got->br_startoff + got->br_blockcount) -
+					(badoff + badlen);
 
-	if (got->br_blockcount == repl->len) {
-		/*
-		 * The new extent is a complete replacement for the existing
-		 * extent.  Update the COW fork record.
-		 */
-		new.br_startblock = repl->fsbno;
-		xfs_iext_update_extent(ip, BMAP_COWFORK, icur, &new);
-		return;
+		rep->br_blockcount -= delta;
 	}
 
-	/*
-	 * The new extent can replace the beginning of the COW fork record.
-	 * Move the left side of @got upwards, then insert the new record.
-	 */
-	new.br_startoff += repl->len;
-	new.br_startblock += repl->len;
-	new.br_blockcount -= repl->len;
-	xfs_iext_update_extent(ip, BMAP_COWFORK, icur, &new);
-
-	new.br_startoff = got->br_startoff;
-	new.br_startblock = repl->fsbno;
-	new.br_blockcount = repl->len;
-	xfs_iext_insert(ip, icur, &new, BMAP_COWFORK);
+	if (got->br_blockcount == 0)
+		goto bad;
+
+	return 0;
+bad:
+	ASSERT(0);
+	return -EFSCORRUPTED;
 }
 
 /*
@@ -553,33 +549,30 @@ xrep_cow_replace_range(
 	xfs_extlen_t		*blockcount)
 {
 	struct xfs_iext_cursor	icur;
-	struct xrep_cow_extent	repl;
-	struct xfs_bmbt_irec	got;
+	struct xfs_bmbt_irec	got, rep;
 	struct xfs_scrub	*sc = xc->sc;
-	xfs_fileoff_t		nextoff;
-	xfs_extlen_t		alloc_len;
+	xfs_fsblock_t		old_fsbno;
 	int			error;
 
 	/*
-	 * Put the existing CoW fork mapping in @got.  If @got ends before
-	 * @rep, truncate @rep so we only replace one extent mapping at a time.
+	 * Put the existing CoW fork mapping in @got, and put in @rep the
+	 * contents of @got trimmed to @startoff/@blockcount.  We only want
+	 * to replace the bad region, and only one mapping at a time.
 	 */
-	error = xrep_cow_find_mapping(xc, &icur, startoff, &got);
+	error = xrep_cow_find_mapping(xc, &icur, startoff, *blockcount, &got,
+			&rep);
 	if (error)
 		return error;
-	nextoff = min(startoff + *blockcount,
-		      got.br_startoff + got.br_blockcount);
+	old_fsbno = rep.br_startblock;
 
 	/*
 	 * Allocate a replacement extent.  If we don't fill all the blocks,
 	 * shorten the quantity that will be deleted in this step.
 	 */
-	alloc_len = min_t(xfs_fileoff_t, XFS_MAX_BMBT_EXTLEN,
-			  nextoff - startoff);
 	if (XFS_IS_REALTIME_INODE(sc->ip))
-		error = xrep_cow_alloc_rt(sc, alloc_len, &repl);
+		error = xrep_cow_alloc_rt(sc, &rep);
 	else
-		error = xrep_cow_alloc(sc, alloc_len, &repl);
+		error = xrep_cow_alloc(sc, &rep);
 	if (error)
 		return error;
 
@@ -587,7 +580,7 @@ xrep_cow_replace_range(
 	 * Replace the old mapping with the new one, and commit the metadata
 	 * changes made so far.
 	 */
-	xrep_cow_replace_mapping(sc->ip, &icur, &got, &repl);
+	xfs_bmap_replace_cow_mapping(sc->ip, &icur, &got, &rep);
 
 	xfs_inode_set_cowblocks_tag(sc->ip);
 	error = xfs_defer_finish(&sc->tp);
@@ -596,15 +589,15 @@ xrep_cow_replace_range(
 
 	/* Note the old CoW staging extents; we'll reap them all later. */
 	if (XFS_IS_REALTIME_INODE(sc->ip))
-		error = xrtb_bitmap_set(&xc->old_cowfork_rtblocks,
-				got.br_startblock, repl.len);
+		error = xrtb_bitmap_set(&xc->old_cowfork_rtblocks, old_fsbno,
+				rep.br_blockcount);
 	else
-		error = xfsb_bitmap_set(&xc->old_cowfork_fsblocks,
-				got.br_startblock, repl.len);
+		error = xfsb_bitmap_set(&xc->old_cowfork_fsblocks, old_fsbno,
+				rep.br_blockcount);
 	if (error)
 		return error;
 
-	*blockcount = repl.len;
+	*blockcount = rep.br_blockcount;
 	return 0;
 }
 
diff --git a/fs/xfs/scrub/dirtree.c b/fs/xfs/scrub/dirtree.c
index c6a210f7508fc..b2cf6e5439d91 100644
--- a/fs/xfs/scrub/dirtree.c
+++ b/fs/xfs/scrub/dirtree.c
@@ -383,6 +383,14 @@ xchk_dirpath_step_up(
 		goto out_scanlock;
 	}
 
+	/* The handle encoded in the parent pointer must match. */
+	if (VFS_I(dp)->i_generation != be32_to_cpu(dl->pptr_rec.p_gen)) {
+		trace_xchk_dirpath_badgen(dl->sc, dp, path->path_nr,
+				path->nr_steps, &dl->xname, &dl->pptr_rec);
+		error = -EFSCORRUPTED;
+		goto out_scanlock;
+	}
+
 	/* We've reached the root directory; the path is ok. */
 	if (parent_ino == dl->root_ino) {
 		xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_OK);
@@ -411,14 +419,6 @@ xchk_dirpath_step_up(
 		goto out_scanlock;
 	}
 
-	/* The handle encoded in the parent pointer must match. */
-	if (VFS_I(dp)->i_generation != be32_to_cpu(dl->pptr_rec.p_gen)) {
-		trace_xchk_dirpath_badgen(dl->sc, dp, path->path_nr,
-				path->nr_steps, &dl->xname, &dl->pptr_rec);
-		error = -EFSCORRUPTED;
-		goto out_scanlock;
-	}
-
 	/* Parent pointer must point up to a directory. */
 	if (!S_ISDIR(VFS_I(dp)->i_mode)) {
 		trace_xchk_dirpath_nondir_parent(dl->sc, dp, path->path_nr,
diff --git a/fs/xfs/scrub/dqiterate.c b/fs/xfs/scrub/dqiterate.c
index 10950e4bd4c3c..079dc4e691a01 100644
--- a/fs/xfs/scrub/dqiterate.c
+++ b/fs/xfs/scrub/dqiterate.c
@@ -205,7 +205,7 @@ xchk_dquot_iter(
 	if (error)
 		return error;
 
-	cursor->id = dq->q_id + 1;
+	cursor->id = (uint64_t)dq->q_id + 1;
 	*dqpp = dq;
 	return 1;
 }
diff --git a/fs/xfs/scrub/inode_repair.c b/fs/xfs/scrub/inode_repair.c
index 493dcf5cc6c15..3ec41c1983511 100644
--- a/fs/xfs/scrub/inode_repair.c
+++ b/fs/xfs/scrub/inode_repair.c
@@ -921,7 +921,7 @@ xrep_dinode_bad_bmbt_fork(
 
 	if (nrecs == 0 || xfs_bmdr_space_calc(nrecs) > dfork_size)
 		return true;
-	if (level == 0 || level >= XFS_BM_MAXLEVELS(sc->mp, whichfork))
+	if (level == 0 || level > XFS_BM_MAXLEVELS(sc->mp, whichfork))
 		return true;
 
 	dmxr = xfs_bmdr_maxrecs(dfork_size, 0);
@@ -1757,7 +1757,7 @@ xrep_clamp_timestamp(
 	struct xfs_inode	*ip,
 	struct timespec64	*ts)
 {
-	ts->tv_nsec = clamp_t(long, ts->tv_nsec, 0, NSEC_PER_SEC);
+	ts->tv_nsec = clamp_t(long, ts->tv_nsec, 0, NSEC_PER_SEC - 1);
 	*ts = timestamp_truncate(*ts, VFS_I(ip));
 }
 
diff --git a/fs/xfs/scrub/rgsuper.c b/fs/xfs/scrub/rgsuper.c
index 482f899a518a8..2bd2c0351b35f 100644
--- a/fs/xfs/scrub/rgsuper.c
+++ b/fs/xfs/scrub/rgsuper.c
@@ -23,6 +23,8 @@ int
 xchk_setup_rgsuperblock(
 	struct xfs_scrub	*sc)
 {
+	if (xchk_need_intent_drain(sc))
+		xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
 	return xchk_trans_alloc(sc, 0);
 }
 
@@ -43,6 +45,7 @@ xchk_rgsuperblock(
 	struct xfs_scrub	*sc)
 {
 	xfs_rgnumber_t		rgno = sc->sm->sm_agno;
+	unsigned int		flags;
 	int			error;
 
 	/*
@@ -63,7 +66,12 @@ xchk_rgsuperblock(
 	if (!xchk_xref_process_error(sc, 0, 0, &error))
 		return error;
 
-	error = xchk_rtgroup_lock(sc, &sc->sr, XFS_RTGLOCK_BITMAP_SHARED);
+	if (xfs_has_rtrmapbt(sc->mp))
+		flags = XFS_RTGLOCK_BITMAP | XFS_RTGLOCK_RMAP;
+	else
+		flags = XFS_RTGLOCK_BITMAP_SHARED;
+
+	error = xchk_rtgroup_lock(sc, &sc->sr, flags);
 	if (error)
 		return error;
 
@@ -80,9 +88,13 @@ int
 xrep_rgsuperblock(
 	struct xfs_scrub	*sc)
 {
+	struct xfs_buf		*sb_bp;
+
 	ASSERT(rtg_rgno(sc->sr.rtg) == 0);
 
+	sb_bp = xfs_trans_getsb(sc->tp);
 	xfs_log_sb(sc->tp);
+	xfs_log_rtsb(sc->tp, sb_bp);
 	return 0;
 }
 #endif /* CONFIG_XFS_ONLINE_REPAIR */
diff --git a/fs/xfs/scrub/rtbitmap.c b/fs/xfs/scrub/rtbitmap.c
index de3f22f310f7e..52c24d3d4be6c 100644
--- a/fs/xfs/scrub/rtbitmap.c
+++ b/fs/xfs/scrub/rtbitmap.c
@@ -258,7 +258,7 @@ xchk_rtbitmap(
 	 * the last free extent we saw and the last possible extent in the rt
 	 * group.
 	 */
-	last_rgbno = rtg->rtg_extents * mp->m_sb.sb_rextsize - 1;
+	last_rgbno = rtg->rtg_extents * mp->m_sb.sb_rextsize;
 	if (rtb->next_free_rgbno < last_rgbno)
 		xchk_xref_has_rt_owner(sc, rtb->next_free_rgbno,
 				last_rgbno - rtb->next_free_rgbno);
diff --git a/fs/xfs/scrub/rtrefcount.c b/fs/xfs/scrub/rtrefcount.c
index 0d10ce2910c2c..4e7c540c8d230 100644
--- a/fs/xfs/scrub/rtrefcount.c
+++ b/fs/xfs/scrub/rtrefcount.c
@@ -607,7 +607,7 @@ xchk_xref_is_rt_cow_staging(
 
 	/* CoW lookup returned a shared extent record? */
 	if (rc.rc_domain != XFS_REFC_DOMAIN_COW)
-		xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0);
+		xchk_btree_xref_set_corrupt(sc, sc->sr.refc_cur, 0);
 
 	/* Must be at least as long as what was passed in */
 	if (rc.rc_blockcount < len)
diff --git a/fs/xfs/scrub/rtrmap.c b/fs/xfs/scrub/rtrmap.c
index 043be93c71488..564d19a97a2f8 100644
--- a/fs/xfs/scrub/rtrmap.c
+++ b/fs/xfs/scrub/rtrmap.c
@@ -87,6 +87,9 @@ xchk_rtrmapbt_is_shareable(
 		return false;
 	if (irec->rm_flags & XFS_RMAP_UNWRITTEN)
 		return false;
+	if (irec->rm_owner == XFS_RMAP_OWN_COW ||
+	    irec->rm_owner == XFS_RMAP_OWN_FS)
+		return false;
 	return true;
 }
 
@@ -146,6 +149,9 @@ xchk_rtrmap_mergeable(
 		return false;
 	if (r1->rm_flags != r2->rm_flags)
 		return false;
+	if (r1->rm_owner == XFS_RMAP_OWN_COW ||
+	    r1->rm_owner == XFS_RMAP_OWN_FS)
+		return true;
 	return r1->rm_offset + r1->rm_blockcount == r2->rm_offset;
 }
 
@@ -209,7 +215,7 @@ xchk_rtrmapbt_xref(
 			xfs_rgbno_to_rtb(sc->sr.rtg, irec->rm_startblock),
 			irec->rm_blockcount);
 	if (irec->rm_owner == XFS_RMAP_OWN_COW)
-		xchk_xref_is_cow_staging(sc, irec->rm_startblock,
+		xchk_xref_is_rt_cow_staging(sc, irec->rm_startblock,
 				irec->rm_blockcount);
 	else
 		xchk_rtrmapbt_xref_rtrefc(sc, irec);
diff --git a/fs/xfs/scrub/scrub.h b/fs/xfs/scrub/scrub.h
index a3f1abc913903..6d7d3523b71f2 100644
--- a/fs/xfs/scrub/scrub.h
+++ b/fs/xfs/scrub/scrub.h
@@ -11,7 +11,7 @@ struct xfs_scrub;
 struct xchk_relax {
 	unsigned long	next_resched;
 	unsigned int	resched_nr;
-	bool		interruptible;
+	bool		killable;
 };
 
 /* Yield to the scheduler at most 10x per second. */
@@ -21,7 +21,7 @@ struct xchk_relax {
 	(struct xchk_relax){ \
 		.next_resched	= XCHK_RELAX_NEXT, \
 		.resched_nr	= 0, \
-		.interruptible	= true, \
+		.killable	= true, \
 	}
 
 /*
@@ -45,7 +45,7 @@ static inline int xchk_maybe_relax(struct xchk_relax *widget)
 		widget->next_resched = XCHK_RELAX_NEXT;
 	}
 
-	if (widget->interruptible && fatal_signal_pending(current))
+	if (widget->killable && fatal_signal_pending(current))
 		return -EINTR;
 
 	return 0;
diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h
index 1b7d9e07a27d3..d5d39d82749e5 100644
--- a/fs/xfs/scrub/trace.h
+++ b/fs/xfs/scrub/trace.h
@@ -2671,41 +2671,6 @@ TRACE_EVENT(xrep_cow_mark_file_range,
 		  __entry->blockcount)
 );
 
-TRACE_EVENT(xrep_cow_replace_mapping,
-	TP_PROTO(struct xfs_inode *ip, const struct xfs_bmbt_irec *irec,
-		 xfs_fsblock_t new_startblock, xfs_extlen_t new_blockcount),
-	TP_ARGS(ip, irec, new_startblock, new_blockcount),
-	TP_STRUCT__entry(
-		__field(dev_t, dev)
-		__field(xfs_ino_t, ino)
-		__field(xfs_fsblock_t, startblock)
-		__field(xfs_fileoff_t, startoff)
-		__field(xfs_filblks_t, blockcount)
-		__field(xfs_exntst_t, state)
-		__field(xfs_fsblock_t, new_startblock)
-		__field(xfs_extlen_t, new_blockcount)
-	),
-	TP_fast_assign(
-		__entry->dev = ip->i_mount->m_super->s_dev;
-		__entry->ino = I_INO(ip);
-		__entry->startoff = irec->br_startoff;
-		__entry->startblock = irec->br_startblock;
-		__entry->blockcount = irec->br_blockcount;
-		__entry->state = irec->br_state;
-		__entry->new_startblock = new_startblock;
-		__entry->new_blockcount = new_blockcount;
-	),
-	TP_printk("dev %d:%d ino 0x%llx startoff 0x%llx startblock 0x%llx fsbcount 0x%llx state 0x%x new_startblock 0x%llx new_fsbcount 0x%x",
-		  MAJOR(__entry->dev), MINOR(__entry->dev),
-		  __entry->ino,
-		  __entry->startoff,
-		  __entry->startblock,
-		  __entry->blockcount,
-		  __entry->state,
-		  __entry->new_startblock,
-		  __entry->new_blockcount)
-);
-
 TRACE_EVENT(xrep_cow_free_staging,
 	TP_PROTO(const struct xfs_perag *pag, xfs_agblock_t agbno,
 		 xfs_extlen_t blockcount),
diff --git a/fs/xfs/scrub/xfarray.c b/fs/xfs/scrub/xfarray.c
index c7c4a71b6fa7c..2ce24bfe4c0fa 100644
--- a/fs/xfs/scrub/xfarray.c
+++ b/fs/xfs/scrub/xfarray.c
@@ -487,8 +487,7 @@ xfarray_sortinfo_alloc(
 	xfarray_sortinfo_lo(si)[0] = 0;
 	xfarray_sortinfo_hi(si)[0] = array->nr - 1;
 	si->relax = INIT_XCHK_RELAX;
-	if (flags & XFARRAY_SORT_KILLABLE)
-		si->relax.interruptible = false;
+	si->relax.killable = !!(flags & XFARRAY_SORT_KILLABLE);
 
 	trace_xfarray_sort(si, nr_bytes);
 	*infop = si;
diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c
index 3b9f262f8e912..c88b9ade7389d 100644
--- a/fs/xfs/xfs_bmap_util.c
+++ b/fs/xfs/xfs_bmap_util.c
@@ -1744,3 +1744,92 @@ xfs_swap_extents(
 	xfs_trans_cancel(tp);
 	goto out_unlock_ilock;
 }
+
+/*
+ * Given a CoW fork mapping @got and a replacement mapping @rep, map the space
+ * described by @rep into the cow fork, pushing aside @got as necessary.  @icur
+ * must point to iext tree leaf containing @got.
+ */
+void
+xfs_bmap_replace_cow_mapping(
+	struct xfs_inode	*ip,
+	struct xfs_iext_cursor	*icur,
+	struct xfs_bmbt_irec	*got,
+	struct xfs_bmbt_irec	*rep)
+{
+	struct xfs_ifork	*ifp = xfs_ifork_ptr(ip, XFS_COW_FORK);
+	xfs_fileoff_t		rep_endoff =
+			rep->br_startoff + rep->br_blockcount;
+	xfs_fileoff_t		got_endoff =
+			got->br_startoff + got->br_blockcount;
+	uint32_t		state = BMAP_COWFORK;
+
+	ASSERT(rep->br_blockcount > 0);
+	ASSERT(!isnullstartblock(got->br_startblock));
+	ASSERT(got->br_startoff <= rep->br_startoff);
+	ASSERT(got_endoff >= rep_endoff);
+
+	trace_xfs_bmap_replace_cow_mapping(ip, got, rep);
+
+	if (got->br_startoff == rep->br_startoff)
+		state |= BMAP_LEFT_FILLING;
+	if (got_endoff == rep_endoff)
+		state |= BMAP_RIGHT_FILLING;
+
+	switch (state & (BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING)) {
+	case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING:
+		/*
+		 * Replacement matches the whole mapping, update the record.
+		 */
+		xfs_iext_update_extent(ip, state, icur, rep);
+		break;
+	case BMAP_LEFT_FILLING:
+		/*
+		 * Replace the first part of the mapping: Update the cursor
+		 * position with the new mapping, then add a record with the
+		 * tail of the old mapping.
+		 */
+		got->br_startoff = rep_endoff;
+		got->br_blockcount -= rep->br_blockcount;
+		got->br_startblock += rep->br_blockcount;
+
+		xfs_iext_update_extent(ip, state, icur, rep);
+		xfs_iext_next(ifp, icur);
+		xfs_iext_insert(ip, icur, got, state);
+		break;
+	case BMAP_RIGHT_FILLING:
+		/*
+		 * Replacing the last part of the mapping.  Shorten the current
+		 * mapping then add a record with the new mapping.
+		 */
+		got->br_blockcount -= rep->br_blockcount;
+
+		xfs_iext_update_extent(ip, state, icur, got);
+		xfs_iext_next(ifp, icur);
+		xfs_iext_insert(ip, icur, rep, state);
+		break;
+	case 0:
+		/*
+		 * Replacing the middle of the extent.  Shorten the current
+		 * mapping, add a new record with the new mapping, and add a
+		 * second new record with the tail of the old mapping.
+		 */
+		got->br_blockcount = rep->br_startoff - got->br_startoff;
+
+		struct xfs_bmbt_irec	new = {
+			.br_startoff	= rep_endoff,
+			.br_blockcount	= got_endoff - rep_endoff,
+			.br_state	= got->br_state,
+			.br_startblock	= got->br_startblock +
+						rep->br_blockcount +
+						got->br_blockcount,
+		};
+
+		xfs_iext_update_extent(ip, state, icur, got);
+		xfs_iext_next(ifp, icur);
+		xfs_iext_insert(ip, icur, rep, state);
+		xfs_iext_next(ifp, icur);
+		xfs_iext_insert(ip, icur, &new, state);
+		break;
+	}
+}
diff --git a/fs/xfs/xfs_bmap_util.h b/fs/xfs/xfs_bmap_util.h
index c477b33616304..eaaf094154b9f 100644
--- a/fs/xfs/xfs_bmap_util.h
+++ b/fs/xfs/xfs_bmap_util.h
@@ -81,4 +81,8 @@ int xfs_bmap_count_blocks(struct xfs_trans *tp, struct xfs_inode *ip,
 int	xfs_flush_unmap_range(struct xfs_inode *ip, xfs_off_t offset,
 			      xfs_off_t len);
 
+void xfs_bmap_replace_cow_mapping(struct xfs_inode *ip,
+		struct xfs_iext_cursor *icur, struct xfs_bmbt_irec *got,
+		struct xfs_bmbt_irec *rep);
+
 #endif	/* __XFS_BMAP_UTIL_H__ */
diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c
index e1465e950acc7..8c210ebb66bc3 100644
--- a/fs/xfs/xfs_buf.c
+++ b/fs/xfs/xfs_buf.c
@@ -1631,7 +1631,7 @@ xfs_free_buftarg(
 	fs_put_dax(btp->bt_daxdev, btp->bt_mount);
 	/* the main block device is closed by kill_block_super */
 	if (btp->bt_bdev != btp->bt_mount->m_super->s_bdev)
-		bdev_fput(btp->bt_file);
+		fs_bdev_file_release(btp->bt_file, btp->bt_mount->m_super);
 	kfree(btp);
 }
 
diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c
index 6339f4956ecb3..4a3299abf774f 100644
--- a/fs/xfs/xfs_iops.c
+++ b/fs/xfs/xfs_iops.c
@@ -293,8 +293,7 @@ xfs_vn_create(
 	struct mnt_idmap	*idmap,
 	struct inode		*dir,
 	struct dentry		*dentry,
-	umode_t			mode,
-	bool			flags)
+	umode_t			mode)
 {
 	return xfs_generic_create(idmap, dir, dentry, mode, 0, NULL);
 }
@@ -306,7 +305,7 @@ xfs_vn_mkdir(
 	struct dentry		*dentry,
 	umode_t			mode)
 {
-	return ERR_PTR(xfs_generic_create(idmap, dir, dentry, mode | S_IFDIR, 0, NULL));
+	return ERR_PTR(xfs_generic_create(idmap, dir, dentry, mode, 0, NULL));
 }
 
 STATIC struct dentry *
@@ -338,7 +337,7 @@ STATIC struct dentry *
 xfs_vn_ci_lookup(
 	struct inode	*dir,
 	struct dentry	*dentry,
-	unsigned int flags)
+	unsigned int	flags)
 {
 	struct xfs_inode *ip;
 	struct xfs_name	xname;
diff --git a/fs/xfs/xfs_log_cil.c b/fs/xfs/xfs_log_cil.c
index edc368938f305..639f875a8fb25 100644
--- a/fs/xfs/xfs_log_cil.c
+++ b/fs/xfs/xfs_log_cil.c
@@ -1710,7 +1710,7 @@ xlog_cil_push_background(
 static void
 xlog_cil_push_now(
 	struct xlog	*log,
-	xfs_lsn_t	push_seq,
+	xfs_csn_t	push_seq,
 	bool		async)
 {
 	struct xfs_cil	*cil = log->l_cilp;
diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c
index 5f984bf5698a8..fdb011e6ef600 100644
--- a/fs/xfs/xfs_log_recover.c
+++ b/fs/xfs/xfs_log_recover.c
@@ -1907,18 +1907,20 @@ xlog_recover_reorder_trans(
 	list_for_each_entry_safe(item, n, &sort_list, ri_list) {
 		enum xlog_recover_reorder	fate = XLOG_REORDER_ITEM_LIST;
 
+		/* a committed item with no regions has a NULL ri_buf[0] */
+		if (!item->ri_cnt || !item->ri_buf) {
+			xfs_warn(log->l_mp,
+				"%s: committed log item has no regions",
+				__func__);
+			error = -EFSCORRUPTED;
+			break;
+		}
+
 		item->ri_ops = xlog_find_item_ops(item);
 		if (!item->ri_ops) {
 			xfs_warn(log->l_mp,
 				"%s: unrecognized type of log operation (%d)",
 				__func__, ITEM_TYPE(item));
-			ASSERT(0);
-			/*
-			 * return the remaining items back to the transaction
-			 * item list so they can be freed in caller.
-			 */
-			if (!list_empty(&sort_list))
-				list_splice_init(&sort_list, &trans->r_itemq);
 			error = -EFSCORRUPTED;
 			break;
 		}
@@ -1946,7 +1948,15 @@ xlog_recover_reorder_trans(
 		}
 	}
 
-	ASSERT(list_empty(&sort_list));
+	/*
+	 * Return the remaining items back to the transaction item list so they
+	 * can be freed in caller.  This should only happen when we encounter
+	 * an error.
+	 */
+	if (!list_empty(&sort_list)) {
+		ASSERT(error);
+		list_splice_init(&sort_list, &trans->r_itemq);
+	}
 	if (!list_empty(&buffer_list))
 		list_splice(&buffer_list, &trans->r_itemq);
 	if (!list_empty(&item_list))
diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c
index a5c188b781382..4801361366359 100644
--- a/fs/xfs/xfs_reflink.c
+++ b/fs/xfs/xfs_reflink.c
@@ -440,6 +440,7 @@ xfs_reflink_fill_cow_hole(
 	struct xfs_mount	*mp = ip->i_mount;
 	struct xfs_trans	*tp;
 	xfs_filblks_t		resaligned;
+	unsigned int		seq_before = READ_ONCE(ip->i_df.if_seq);
 	unsigned int		dblocks = 0, rblocks = 0;
 	int			nimaps;
 	int			error;
@@ -465,6 +466,22 @@ xfs_reflink_fill_cow_hole(
 
 	*lockmode = XFS_ILOCK_EXCL;
 
+	/*
+	 * The data fork mapping may have changed while we dropped the ILOCK
+	 * (a racing O_DIRECT writer under IOLOCK_SHARED can complete a full
+	 * CoW cycle including xfs_reflink_end_cow(), which remaps this offset
+	 * and drops the refcount of the old shared block).  Re-read it so the
+	 * shared-status recheck below and the caller's in-place iomap both
+	 * operate on the current mapping rather than a stale physical block.
+	 */
+	if (seq_before != READ_ONCE(ip->i_df.if_seq)) {
+		nimaps = 1;
+		error = xfs_bmapi_read(ip, imap->br_startoff,
+				imap->br_blockcount, imap, &nimaps, 0);
+		if (error)
+			goto out_trans_cancel;
+	}
+
 	error = xfs_find_trim_cow_extent(ip, imap, cmap, shared, &found);
 	if (error || !*shared)
 		goto out_trans_cancel;
@@ -511,6 +528,8 @@ xfs_reflink_fill_delalloc(
 	bool			found;
 
 	do {
+		unsigned int	seq_before = READ_ONCE(ip->i_df.if_seq);
+
 		xfs_iunlock(ip, *lockmode);
 		*lockmode = 0;
 
@@ -521,6 +540,23 @@ xfs_reflink_fill_delalloc(
 
 		*lockmode = XFS_ILOCK_EXCL;
 
+		/*
+		 * The data fork mapping may have changed while we dropped the
+		 * ILOCK (a racing O_DIRECT writer under IOLOCK_SHARED can
+		 * complete a full CoW cycle including xfs_reflink_end_cow(),
+		 * which remaps this offset and drops the refcount of the old
+		 * shared block).  Re-read it so the shared-status recheck
+		 * below and the caller's in-place iomap both operate on the
+		 * current mapping rather than a stale physical block.
+		 */
+		if (seq_before != READ_ONCE(ip->i_df.if_seq)) {
+			nimaps = 1;
+			error = xfs_bmapi_read(ip, imap->br_startoff,
+					imap->br_blockcount, imap, &nimaps, 0);
+			if (error)
+				goto out_trans_cancel;
+		}
+
 		error = xfs_find_trim_cow_extent(ip, imap, cmap, shared,
 				&found);
 		if (error || !*shared)
diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c
index 8531d526fc440..d1c622f0a957f 100644
--- a/fs/xfs/xfs_super.c
+++ b/fs/xfs/xfs_super.c
@@ -400,8 +400,8 @@ xfs_blkdev_get(
 	blk_mode_t		mode;
 
 	mode = sb_open_mode(mp->m_super->s_flags);
-	*bdev_filep = bdev_file_open_by_path(name, mode,
-			mp->m_super, &fs_holder_ops);
+	*bdev_filep = fs_bdev_file_open_by_path(name, mode,
+			mp->m_super, mp->m_super);
 	if (IS_ERR(*bdev_filep)) {
 		error = PTR_ERR(*bdev_filep);
 		*bdev_filep = NULL;
@@ -526,7 +526,7 @@ xfs_open_devices(
 		mp->m_logdev_targp = mp->m_ddev_targp;
 		/* Handle won't be used, drop it */
 		if (logdev_file)
-			bdev_fput(logdev_file);
+			fs_bdev_file_release(logdev_file, mp->m_super);
 	}
 
 	return 0;
@@ -541,10 +541,10 @@ xfs_open_devices(
 	mp->m_ddev_targp = NULL;
  out_close_rtdev:
 	 if (rtdev_file)
-		bdev_fput(rtdev_file);
+		fs_bdev_file_release(rtdev_file, mp->m_super);
  out_close_logdev:
 	if (logdev_file)
-		bdev_fput(logdev_file);
+		fs_bdev_file_release(logdev_file, mp->m_super);
 	return error;
 }
 
diff --git a/fs/xfs/xfs_sysfs.c b/fs/xfs/xfs_sysfs.c
index 676777064c2d7..b627121873241 100644
--- a/fs/xfs/xfs_sysfs.c
+++ b/fs/xfs/xfs_sysfs.c
@@ -780,6 +780,23 @@ static const struct kobj_type xfs_zoned_ktype = {
 	.default_groups = xfs_zoned_groups,
 };
 
+int
+xfs_zoned_sysfs_init(struct xfs_mount *mp)
+{
+	if (!IS_ENABLED(CONFIG_XFS_RT) || !xfs_has_zoned(mp))
+		return 0;
+
+	return xfs_sysfs_init(&mp->m_zoned_kobj, &xfs_zoned_ktype,
+			&mp->m_kobj, "zoned");
+}
+
+void
+xfs_zoned_sysfs_del(struct xfs_mount *mp)
+{
+	if (IS_ENABLED(CONFIG_XFS_RT) && xfs_has_zoned(mp))
+		xfs_sysfs_del(&mp->m_zoned_kobj);
+}
+
 int
 xfs_mount_sysfs_init(
 	struct xfs_mount	*mp)
@@ -820,14 +837,6 @@ xfs_mount_sysfs_init(
 	if (error)
 		goto out_remove_error_dir;
 
-	if (IS_ENABLED(CONFIG_XFS_RT) && xfs_has_zoned(mp)) {
-		/* .../xfs/<dev>/zoned/ */
-		error = xfs_sysfs_init(&mp->m_zoned_kobj, &xfs_zoned_ktype,
-					&mp->m_kobj, "zoned");
-		if (error)
-			goto out_remove_error_dir;
-	}
-
 	return 0;
 
 out_remove_error_dir:
@@ -846,9 +855,6 @@ xfs_mount_sysfs_del(
 	struct xfs_error_cfg	*cfg;
 	int			i, j;
 
-	if (IS_ENABLED(CONFIG_XFS_RT) && xfs_has_zoned(mp))
-		xfs_sysfs_del(&mp->m_zoned_kobj);
-
 	for (i = 0; i < XFS_ERR_CLASS_MAX; i++) {
 		for (j = 0; j < XFS_ERR_ERRNO_MAX; j++) {
 			cfg = &mp->m_error_cfg[i][j];
diff --git a/fs/xfs/xfs_sysfs.h b/fs/xfs/xfs_sysfs.h
index 1622fe80ad3ed..25e5f8fae2f3a 100644
--- a/fs/xfs/xfs_sysfs.h
+++ b/fs/xfs/xfs_sysfs.h
@@ -53,6 +53,8 @@ xfs_sysfs_del(
 }
 
 int	xfs_mount_sysfs_init(struct xfs_mount *mp);
+int	xfs_zoned_sysfs_init(struct xfs_mount *mp);
+void	xfs_zoned_sysfs_del(struct xfs_mount *mp);
 void	xfs_mount_sysfs_del(struct xfs_mount *mp);
 
 #endif	/* __XFS_SYSFS_H__ */
diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h
index d478693674f95..aeb89ac53bf19 100644
--- a/fs/xfs/xfs_trace.h
+++ b/fs/xfs/xfs_trace.h
@@ -6439,6 +6439,47 @@ TRACE_EVENT(xfs_verify_media_error,
 		  __entry->error)
 );
 
+TRACE_EVENT(xfs_bmap_replace_cow_mapping,
+	TP_PROTO(struct xfs_inode *ip, const struct xfs_bmbt_irec *got,
+		 const struct xfs_bmbt_irec *rep),
+	TP_ARGS(ip, got, rep),
+	TP_STRUCT__entry(
+		__field(dev_t, dev)
+		__field(xfs_ino_t, ino)
+		__field(xfs_fsblock_t, startblock)
+		__field(xfs_fileoff_t, startoff)
+		__field(xfs_filblks_t, blockcount)
+		__field(xfs_exntst_t, state)
+		__field(xfs_fileoff_t, new_startoff)
+		__field(xfs_fsblock_t, new_startblock)
+		__field(xfs_extlen_t, new_blockcount)
+		__field(xfs_exntst_t, new_state)
+	),
+	TP_fast_assign(
+		__entry->dev = ip->i_mount->m_super->s_dev;
+		__entry->ino = I_INO(ip);
+		__entry->startoff = got->br_startoff;
+		__entry->startblock = got->br_startblock;
+		__entry->blockcount = got->br_blockcount;
+		__entry->state = got->br_state;
+		__entry->new_startoff = rep->br_startoff;
+		__entry->new_startblock = rep->br_startblock;
+		__entry->new_blockcount = rep->br_blockcount;
+		__entry->new_state = rep->br_state;
+	),
+	TP_printk("dev %d:%d ino 0x%llx startoff 0x%llx startblock 0x%llx fsbcount 0x%llx state 0x%x new_startoff 0x%llx new_startblock 0x%llx new_fsbcount 0x%x new_state 0x%x",
+		  MAJOR(__entry->dev), MINOR(__entry->dev),
+		  __entry->ino,
+		  __entry->startoff,
+		  __entry->startblock,
+		  __entry->blockcount,
+		  __entry->state,
+		  __entry->new_startoff,
+		  __entry->new_startblock,
+		  __entry->new_blockcount,
+		  __entry->new_state)
+);
+
 #endif /* _TRACE_XFS_H */
 
 #undef TRACE_INCLUDE_PATH
diff --git a/fs/xfs/xfs_zone_alloc.c b/fs/xfs/xfs_zone_alloc.c
index 08d8b34f467e7..7d13fa7ab30a7 100644
--- a/fs/xfs/xfs_zone_alloc.c
+++ b/fs/xfs/xfs_zone_alloc.c
@@ -21,6 +21,7 @@
 #include "xfs_rtbitmap.h"
 #include "xfs_rtrmap_btree.h"
 #include "xfs_zone_alloc.h"
+#include "xfs_sysfs.h"
 #include "xfs_zone_priv.h"
 #include "xfs_zones.h"
 #include "xfs_trace.h"
@@ -1420,11 +1421,17 @@ xfs_mount_zones(
 	if (error)
 		goto out_free_zone_info;
 
+	error = xfs_zoned_sysfs_init(mp);
+	if (error)
+		goto out_zone_gc_unmount;
+
 	xfs_info(mp, "%u zones of %u blocks (%u max open zones)",
 		 mp->m_sb.sb_rgcount, iz.zone_capacity, mp->m_max_open_zones);
 	trace_xfs_zones_mount(mp);
 	return 0;
 
+out_zone_gc_unmount:
+	xfs_zone_gc_unmount(mp);
 out_free_zone_info:
 	xfs_free_zone_info(mp->m_zone_info);
 	return error;
@@ -1434,6 +1441,7 @@ void
 xfs_unmount_zones(
 	struct xfs_mount	*mp)
 {
+	xfs_zoned_sysfs_del(mp);
 	xfs_zone_gc_unmount(mp);
 	xfs_free_zone_info(mp->m_zone_info);
 }
diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h
index 714d111d80530..7b8051baed756 100644
--- a/include/acpi/acpi_bus.h
+++ b/include/acpi/acpi_bus.h
@@ -108,7 +108,6 @@ enum acpi_bus_device_type {
 	ACPI_BUS_DEVICE_TYPE_COUNT
 };
 
-struct acpi_driver;
 struct acpi_device;
 
 /*
@@ -158,32 +157,6 @@ struct acpi_hotplug_context {
 	acpi_hp_fixup fixup;
 };
 
-/*
- * ACPI Driver
- * -----------
- */
-
-typedef int (*acpi_op_add) (struct acpi_device * device);
-typedef void (*acpi_op_remove) (struct acpi_device *device);
-typedef void (*acpi_op_notify) (struct acpi_device * device, u32 event);
-
-struct acpi_device_ops {
-	acpi_op_add add;
-	acpi_op_remove remove;
-	acpi_op_notify notify;
-};
-
-#define ACPI_DRIVER_ALL_NOTIFY_EVENTS	0x1	/* system AND device events */
-
-struct acpi_driver {
-	char name[80];
-	char class[80];
-	const struct acpi_device_id *ids; /* Supported Hardware IDs */
-	unsigned int flags;
-	struct acpi_device_ops ops;
-	struct device_driver drv;
-};
-
 /*
  * ACPI Device
  * -----------
@@ -211,7 +184,6 @@ struct acpi_device_flags {
 	u32 removable:1;
 	u32 ejectable:1;
 	u32 power_manageable:1;
-	u32 match_driver:1;
 	u32 initialized:1;
 	u32 visited:1;
 	u32 hotplug_notify:1;
@@ -221,7 +193,7 @@ struct acpi_device_flags {
 	u32 cca_seen:1;
 	u32 enumeration_by_parent:1;
 	u32 honor_deps:1;
-	u32 reserved:18;
+	u32 reserved:19;
 };
 
 /* File System */
@@ -570,7 +542,6 @@ static inline void *acpi_driver_data(struct acpi_device *d)
 }
 
 #define to_acpi_device(d)	container_of(d, struct acpi_device, dev)
-#define to_acpi_driver(d)	container_of_const(d, struct acpi_driver, drv)
 
 static inline struct acpi_device *acpi_dev_parent(struct acpi_device *adev)
 {
@@ -676,13 +647,6 @@ void acpi_scan_lock_release(void);
 void acpi_lock_hp_context(void);
 void acpi_unlock_hp_context(void);
 int acpi_scan_add_handler(struct acpi_scan_handler *handler);
-/*
- * use a macro to avoid include chaining to get THIS_MODULE
- */
-#define acpi_bus_register_driver(drv) \
-	__acpi_bus_register_driver(drv, THIS_MODULE)
-int __acpi_bus_register_driver(struct acpi_driver *driver, struct module *owner);
-void acpi_bus_unregister_driver(struct acpi_driver *driver);
 int acpi_bus_scan(acpi_handle handle);
 void acpi_bus_trim(struct acpi_device *start);
 acpi_status acpi_bus_get_ejd(acpi_handle handle, acpi_handle * ejd);
@@ -696,18 +660,6 @@ static inline bool acpi_device_enumerated(struct acpi_device *adev)
 	return adev && adev->flags.initialized && adev->flags.visited;
 }
 
-/**
- * module_acpi_driver(acpi_driver) - Helper macro for registering an ACPI driver
- * @__acpi_driver: acpi_driver struct
- *
- * Helper macro for ACPI drivers which do not do anything special in module
- * init/exit. This eliminates a lot of boilerplate. Each module may only
- * use this macro once, and calling it replaces module_init() and module_exit()
- */
-#define module_acpi_driver(__acpi_driver) \
-	module_driver(__acpi_driver, acpi_bus_register_driver, \
-		      acpi_bus_unregister_driver)
-
 /*
  * Bind physical devices with ACPI devices
  */
diff --git a/include/asm-generic/audit_change_attr.h b/include/asm-generic/audit_change_attr.h
index ddd90bbe40dfc..94388da3490ce 100644
--- a/include/asm-generic/audit_change_attr.h
+++ b/include/asm-generic/audit_change_attr.h
@@ -40,3 +40,6 @@ __NR_link,
 #ifdef __NR_linkat
 __NR_linkat,
 #endif
+#ifdef __NR_file_setattr
+__NR_file_setattr,
+#endif
diff --git a/include/asm-generic/audit_read.h b/include/asm-generic/audit_read.h
index fb9991f53fb6f..d8dc3dd6bf63c 100644
--- a/include/asm-generic/audit_read.h
+++ b/include/asm-generic/audit_read.h
@@ -3,6 +3,9 @@
 __NR_readlink,
 #endif
 __NR_quotactl,
+#ifdef __NR_quotactl_fd
+__NR_quotactl_fd,
+#endif
 __NR_listxattr,
 #ifdef __NR_listxattrat
 __NR_listxattrat,
@@ -18,3 +21,31 @@ __NR_fgetxattr,
 #ifdef __NR_readlinkat
 __NR_readlinkat,
 #endif
+#ifdef __NR_file_getattr
+__NR_file_getattr,
+#endif
+#ifdef __NR_stat
+__NR_stat,
+#endif
+#ifdef __NR_stat64
+__NR_stat64,
+#endif
+#ifdef __NR_lstat
+__NR_lstat,
+#endif
+#ifdef __NR_lstat64
+__NR_lstat64,
+#endif
+#ifdef __NR_fstat
+__NR_fstat,
+#endif
+#ifdef __NR_fstat64
+__NR_fstat64,
+#endif
+#ifdef __NR_newfstatat
+__NR_newfstatat,
+#endif
+#ifdef __NR_fstatat64
+__NR_fstatat64,
+#endif
+__NR_statx,
diff --git a/include/asm-generic/audit_write.h b/include/asm-generic/audit_write.h
index f9f1d0ae11d9c..378128dc31e38 100644
--- a/include/asm-generic/audit_write.h
+++ b/include/asm-generic/audit_write.h
@@ -5,6 +5,9 @@ __NR_acct,
 __NR_swapon,
 #endif
 __NR_quotactl,
+#ifdef __NR_quotactl_fd
+__NR_quotactl_fd,
+#endif
 #ifdef __NR_truncate
 __NR_truncate,
 #endif
diff --git a/include/asm-generic/bitops/le.h b/include/asm-generic/bitops/le.h
index d51beff603755..e3b0da9a25f1b 100644
--- a/include/asm-generic/bitops/le.h
+++ b/include/asm-generic/bitops/le.h
@@ -16,47 +16,47 @@
 #endif
 
 
-static inline int test_bit_le(int nr, const void *addr)
+static inline int test_bit_le(unsigned long nr, const void *addr)
 {
 	return test_bit(nr ^ BITOP_LE_SWIZZLE, addr);
 }
 
-static inline void set_bit_le(int nr, void *addr)
+static inline void set_bit_le(unsigned long nr, void *addr)
 {
 	set_bit(nr ^ BITOP_LE_SWIZZLE, addr);
 }
 
-static inline void clear_bit_le(int nr, void *addr)
+static inline void clear_bit_le(unsigned long nr, void *addr)
 {
 	clear_bit(nr ^ BITOP_LE_SWIZZLE, addr);
 }
 
-static inline void __set_bit_le(int nr, void *addr)
+static inline void __set_bit_le(unsigned long nr, void *addr)
 {
 	__set_bit(nr ^ BITOP_LE_SWIZZLE, addr);
 }
 
-static inline void __clear_bit_le(int nr, void *addr)
+static inline void __clear_bit_le(unsigned long nr, void *addr)
 {
 	__clear_bit(nr ^ BITOP_LE_SWIZZLE, addr);
 }
 
-static inline int test_and_set_bit_le(int nr, void *addr)
+static inline int test_and_set_bit_le(unsigned long nr, void *addr)
 {
 	return test_and_set_bit(nr ^ BITOP_LE_SWIZZLE, addr);
 }
 
-static inline int test_and_clear_bit_le(int nr, void *addr)
+static inline int test_and_clear_bit_le(unsigned long nr, void *addr)
 {
 	return test_and_clear_bit(nr ^ BITOP_LE_SWIZZLE, addr);
 }
 
-static inline int __test_and_set_bit_le(int nr, void *addr)
+static inline int __test_and_set_bit_le(unsigned long nr, void *addr)
 {
 	return __test_and_set_bit(nr ^ BITOP_LE_SWIZZLE, addr);
 }
 
-static inline int __test_and_clear_bit_le(int nr, void *addr)
+static inline int __test_and_clear_bit_le(unsigned long nr, void *addr)
 {
 	return __test_and_clear_bit(nr ^ BITOP_LE_SWIZZLE, addr);
 }
diff --git a/include/asm-generic/fixmap.h b/include/asm-generic/fixmap.h
index 29cab7947980a..3ff832ebcea50 100644
--- a/include/asm-generic/fixmap.h
+++ b/include/asm-generic/fixmap.h
@@ -21,7 +21,7 @@
 #define __fix_to_virt(x)	(FIXADDR_TOP - ((x) << PAGE_SHIFT))
 #define __virt_to_fix(x)	((FIXADDR_TOP - ((x)&PAGE_MASK)) >> PAGE_SHIFT)
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 /*
  * 'index to address' translation. If anyone tries to use the idx
  * directly without translation, we catch the bug with a NULL-deference
@@ -97,5 +97,5 @@ static inline unsigned long virt_to_fix(const unsigned long vaddr)
 #define set_fixmap_io(idx, phys) \
 	__set_fixmap(idx, phys, FIXMAP_PAGE_IO)
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 #endif /* __ASM_GENERIC_FIXMAP_H */
diff --git a/include/asm-generic/getorder.h b/include/asm-generic/getorder.h
index f2979e3a96b60..875ccae196832 100644
--- a/include/asm-generic/getorder.h
+++ b/include/asm-generic/getorder.h
@@ -2,7 +2,7 @@
 #ifndef __ASM_GENERIC_GETORDER_H
 #define __ASM_GENERIC_GETORDER_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/compiler.h>
 #include <linux/log2.h>
@@ -47,6 +47,6 @@ static __always_inline __attribute_const__ int get_order(unsigned long size)
 #endif
 }
 
-#endif	/* __ASSEMBLY__ */
+#endif	/* __ASSEMBLER__ */
 
 #endif	/* __ASM_GENERIC_GETORDER_H */
diff --git a/include/asm-generic/hugetlb.h b/include/asm-generic/hugetlb.h
index e1a2e1b7c8e7b..635c41cc34797 100644
--- a/include/asm-generic/hugetlb.h
+++ b/include/asm-generic/hugetlb.h
@@ -37,24 +37,24 @@ static inline pte_t huge_pte_modify(pte_t pte, pgprot_t newprot)
 	return pte_modify(pte, newprot);
 }
 
-#ifndef __HAVE_ARCH_HUGE_PTE_MKUFFD_WP
-static inline pte_t huge_pte_mkuffd_wp(pte_t pte)
+#ifndef __HAVE_ARCH_HUGE_PTE_MKUFFD
+static inline pte_t huge_pte_mkuffd(pte_t pte)
 {
-	return huge_pte_wrprotect(pte_mkuffd_wp(pte));
+	return huge_pte_wrprotect(pte_mkuffd(pte));
 }
 #endif
 
-#ifndef __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD_WP
-static inline pte_t huge_pte_clear_uffd_wp(pte_t pte)
+#ifndef __HAVE_ARCH_HUGE_PTE_CLEAR_UFFD
+static inline pte_t huge_pte_clear_uffd(pte_t pte)
 {
-	return pte_clear_uffd_wp(pte);
+	return pte_clear_uffd(pte);
 }
 #endif
 
-#ifndef __HAVE_ARCH_HUGE_PTE_UFFD_WP
-static inline int huge_pte_uffd_wp(pte_t pte)
+#ifndef __HAVE_ARCH_HUGE_PTE_UFFD
+static inline int huge_pte_uffd(pte_t pte)
 {
-	return pte_uffd_wp(pte);
+	return pte_uffd(pte);
 }
 #endif
 
diff --git a/include/asm-generic/kexec_handover.h b/include/asm-generic/kexec_handover.h
new file mode 100644
index 0000000000000..50839fb5ee8ed
--- /dev/null
+++ b/include/asm-generic/kexec_handover.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ASM_GENERIC_KEXEC_HANDOVER_H
+#define __ASM_GENERIC_KEXEC_HANDOVER_H
+
+#include <linux/types.h>
+
+struct kho_scratch {
+	phys_addr_t addr;
+	phys_addr_t size;
+};
+
+#endif /* __ASM_GENERIC_KEXEC_HANDOVER_H */
diff --git a/include/asm-generic/kprobes.h b/include/asm-generic/kprobes.h
index 5290a2b2e15a0..16f16963d503d 100644
--- a/include/asm-generic/kprobes.h
+++ b/include/asm-generic/kprobes.h
@@ -2,7 +2,7 @@
 #ifndef _ASM_GENERIC_KPROBES_H
 #define _ASM_GENERIC_KPROBES_H
 
-#if defined(__KERNEL__) && !defined(__ASSEMBLY__)
+#if defined(__KERNEL__) && !defined(__ASSEMBLER__)
 #ifdef CONFIG_KPROBES
 /*
  * Blacklist ganerating macro. Specify functions which is not probed
@@ -21,6 +21,6 @@ static unsigned long __used					\
 # define __kprobes
 # define nokprobe_inline	inline
 #endif
-#endif /* defined(__KERNEL__) && !defined(__ASSEMBLY__) */
+#endif /* defined(__KERNEL__) && !defined(__ASSEMBLER__) */
 
 #endif /* _ASM_GENERIC_KPROBES_H */
diff --git a/include/asm-generic/memory_model.h b/include/asm-generic/memory_model.h
index efa6610acbc79..c6b4eafaf4cb2 100644
--- a/include/asm-generic/memory_model.h
+++ b/include/asm-generic/memory_model.h
@@ -4,7 +4,7 @@
 
 #include <linux/pfn.h>
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 /*
  * supports 3 memory models.
@@ -53,7 +53,7 @@ static inline int pfn_valid(unsigned long pfn)
  */
 #define __page_to_pfn(pg)					\
 ({	const struct page *__pg = (pg);				\
-	int __sec = memdesc_section(__pg->flags);		\
+	int __sec = memdesc_section(&__pg->flags);		\
 	(unsigned long)(__pg - __section_mem_map_addr(__nr_to_section(__sec)));	\
 })
 
@@ -86,6 +86,6 @@ static inline int pfn_valid(unsigned long pfn)
 #endif /* CONFIG_DEBUG_VIRTUAL */
 #define phys_to_page(phys)	pfn_to_page(PHYS_PFN(phys))
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif
diff --git a/include/asm-generic/mmu.h b/include/asm-generic/mmu.h
index 0618380375429..5f78971e3ac2c 100644
--- a/include/asm-generic/mmu.h
+++ b/include/asm-generic/mmu.h
@@ -6,7 +6,7 @@
  * This is the mmu.h header for nommu implementations.
  * Architectures with an MMU need something more complex.
  */
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 typedef struct {
 	unsigned long		end_brk;
 
diff --git a/include/asm-generic/pgtable-nop4d.h b/include/asm-generic/pgtable-nop4d.h
index 03b7dae47dd43..89c21f84cffbe 100644
--- a/include/asm-generic/pgtable-nop4d.h
+++ b/include/asm-generic/pgtable-nop4d.h
@@ -2,7 +2,7 @@
 #ifndef _PGTABLE_NOP4D_H
 #define _PGTABLE_NOP4D_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #define __PAGETABLE_P4D_FOLDED 1
 
@@ -54,5 +54,5 @@ static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
 #undef  p4d_addr_end
 #define p4d_addr_end(addr, end)			(end)
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 #endif /* _PGTABLE_NOP4D_H */
diff --git a/include/asm-generic/pgtable-nopmd.h b/include/asm-generic/pgtable-nopmd.h
index 8ffd64e7a24cb..36b6490ed1808 100644
--- a/include/asm-generic/pgtable-nopmd.h
+++ b/include/asm-generic/pgtable-nopmd.h
@@ -2,7 +2,7 @@
 #ifndef _PGTABLE_NOPMD_H
 #define _PGTABLE_NOPMD_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <asm-generic/pgtable-nopud.h>
 
@@ -68,6 +68,6 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
 #undef  pmd_addr_end
 #define pmd_addr_end(addr, end)			(end)
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* _PGTABLE_NOPMD_H */
diff --git a/include/asm-generic/pgtable-nopud.h b/include/asm-generic/pgtable-nopud.h
index eb70c6d7ceff2..356cbfbaab247 100644
--- a/include/asm-generic/pgtable-nopud.h
+++ b/include/asm-generic/pgtable-nopud.h
@@ -2,7 +2,7 @@
 #ifndef _PGTABLE_NOPUD_H
 #define _PGTABLE_NOPUD_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <asm-generic/pgtable-nop4d.h>
 
@@ -62,5 +62,5 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
 #undef  pud_addr_end
 #define pud_addr_end(addr, end)			(end)
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 #endif /* _PGTABLE_NOPUD_H */
diff --git a/include/asm-generic/pgtable_uffd.h b/include/asm-generic/pgtable_uffd.h
index 0d85791efdf77..30e88fc1de2f0 100644
--- a/include/asm-generic/pgtable_uffd.h
+++ b/include/asm-generic/pgtable_uffd.h
@@ -2,79 +2,79 @@
 #define _ASM_GENERIC_PGTABLE_UFFD_H
 
 /*
- * Some platforms can customize the uffd-wp bit, making it unavailable
+ * Some platforms can customize the uffd PTE bit, making it unavailable
  * even if the architecture provides the resource.
  * Adding this API allows architectures to add their own checks for the
  * devices on which the kernel is running.
  * Note: When overriding it, please make sure the
  * CONFIG_HAVE_ARCH_USERFAULTFD_WP is part of this macro.
  */
-#ifndef pgtable_supports_uffd_wp
-#define pgtable_supports_uffd_wp()	IS_ENABLED(CONFIG_HAVE_ARCH_USERFAULTFD_WP)
+#ifndef pgtable_supports_uffd
+#define pgtable_supports_uffd()	IS_ENABLED(CONFIG_HAVE_ARCH_USERFAULTFD_WP)
 #endif
 
 static inline bool uffd_supports_wp_marker(void)
 {
-	return pgtable_supports_uffd_wp() && IS_ENABLED(CONFIG_PTE_MARKER_UFFD_WP);
+	return pgtable_supports_uffd() && IS_ENABLED(CONFIG_PTE_MARKER_UFFD_WP);
 }
 
 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP
-static __always_inline int pte_uffd_wp(pte_t pte)
+static __always_inline int pte_uffd(pte_t pte)
 {
 	return 0;
 }
 
-static __always_inline int pmd_uffd_wp(pmd_t pmd)
+static __always_inline int pmd_uffd(pmd_t pmd)
 {
 	return 0;
 }
 
-static __always_inline pte_t pte_mkuffd_wp(pte_t pte)
+static __always_inline pte_t pte_mkuffd(pte_t pte)
 {
 	return pte;
 }
 
-static __always_inline pmd_t pmd_mkuffd_wp(pmd_t pmd)
+static __always_inline pmd_t pmd_mkuffd(pmd_t pmd)
 {
 	return pmd;
 }
 
-static __always_inline pte_t pte_clear_uffd_wp(pte_t pte)
+static __always_inline pte_t pte_clear_uffd(pte_t pte)
 {
 	return pte;
 }
 
-static __always_inline pmd_t pmd_clear_uffd_wp(pmd_t pmd)
+static __always_inline pmd_t pmd_clear_uffd(pmd_t pmd)
 {
 	return pmd;
 }
 
-static __always_inline pte_t pte_swp_mkuffd_wp(pte_t pte)
+static __always_inline pte_t pte_swp_mkuffd(pte_t pte)
 {
 	return pte;
 }
 
-static __always_inline int pte_swp_uffd_wp(pte_t pte)
+static __always_inline int pte_swp_uffd(pte_t pte)
 {
 	return 0;
 }
 
-static __always_inline pte_t pte_swp_clear_uffd_wp(pte_t pte)
+static __always_inline pte_t pte_swp_clear_uffd(pte_t pte)
 {
 	return pte;
 }
 
-static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd)
+static inline pmd_t pmd_swp_mkuffd(pmd_t pmd)
 {
 	return pmd;
 }
 
-static inline int pmd_swp_uffd_wp(pmd_t pmd)
+static inline int pmd_swp_uffd(pmd_t pmd)
 {
 	return 0;
 }
 
-static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd)
+static inline pmd_t pmd_swp_clear_uffd(pmd_t pmd)
 {
 	return pmd;
 }
diff --git a/include/asm-generic/syscall.h b/include/asm-generic/syscall.h
index c5a3ad53beec6..b53bdd3899489 100644
--- a/include/asm-generic/syscall.h
+++ b/include/asm-generic/syscall.h
@@ -58,8 +58,8 @@ void syscall_set_nr(struct task_struct *task, struct pt_regs *regs, int nr);
  *
  * It's only valid to call this when @task is stopped for system
  * call exit tracing (due to %SYSCALL_WORK_SYSCALL_TRACE or
- * %SYSCALL_WORK_SYSCALL_AUDIT), after ptrace_report_syscall_entry()
- * returned nonzero to prevent the system call from taking place.
+ * %SYSCALL_WORK_SYSCALL_AUDIT), after ptrace_report_syscall_permit_entry()
+ * returned False to prevent the system call from taking place.
  *
  * This rolls back the register state in @regs so it's as if the
  * system call instruction was a no-op.  The registers containing
diff --git a/include/asm-generic/vdso/vsyscall.h b/include/asm-generic/vdso/vsyscall.h
index 5c6d9799f4e74..a6b03cfba0e24 100644
--- a/include/asm-generic/vdso/vsyscall.h
+++ b/include/asm-generic/vdso/vsyscall.h
@@ -2,7 +2,7 @@
 #ifndef __ASM_GENERIC_VSYSCALL_H
 #define __ASM_GENERIC_VSYSCALL_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #ifndef __arch_get_vdso_u_time_data
 static __always_inline const struct vdso_time_data *__arch_get_vdso_u_time_data(void)
@@ -30,6 +30,6 @@ static __always_inline void __arch_sync_vdso_time_data(struct vdso_time_data *vd
 }
 #endif /* __arch_sync_vdso_time_data */
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 #endif /* __ASM_GENERIC_VSYSCALL_H */
diff --git a/include/crypto/aes.h b/include/crypto/aes.h
index 16fbfd93e2bd0..3279cfa546085 100644
--- a/include/crypto/aes.h
+++ b/include/crypto/aes.h
@@ -259,7 +259,7 @@ int aes_p8_set_decrypt_key(const u8 *userKey, const int bits,
 void aes_p8_encrypt(const u8 *in, u8 *out, const struct p8_aes_key *key);
 void aes_p8_decrypt(const u8 *in, u8 *out, const struct p8_aes_key *key);
 void aes_p8_cbc_encrypt(const u8 *in, u8 *out, size_t len,
-			const struct p8_aes_key *key, u8 *iv, const int enc);
+			const struct p8_aes_key *key, u8 *iv, bool enc);
 void aes_p8_ctr32_encrypt_blocks(const u8 *in, u8 *out, size_t len,
 				 const struct p8_aes_key *key, const u8 *iv);
 void aes_p8_xts_encrypt(const u8 *in, u8 *out, size_t len,
diff --git a/include/crypto/if_alg.h b/include/crypto/if_alg.h
index 7643ba954125a..4e9ed8e734030 100644
--- a/include/crypto/if_alg.h
+++ b/include/crypto/if_alg.h
@@ -161,9 +161,17 @@ struct af_alg_ctx {
 	unsigned int inflight;
 };
 
+struct af_alg_allowlist_entry {
+	const char *name;
+	bool privileged;
+};
+
 int af_alg_register_type(const struct af_alg_type *type);
 int af_alg_unregister_type(const struct af_alg_type *type);
 
+int af_alg_check_restriction(const char *name,
+			     const struct af_alg_allowlist_entry allowlist[]);
+
 int af_alg_release(struct socket *sock);
 void af_alg_release_parent(struct sock *sk);
 int af_alg_accept(struct sock *sk, struct socket *newsock,
diff --git a/include/cxl/cxl.h b/include/cxl/cxl.h
index 016c74fb747cf..802b143de83db 100644
--- a/include/cxl/cxl.h
+++ b/include/cxl/cxl.h
@@ -226,4 +226,6 @@ struct cxl_dev_state *_devm_cxl_dev_state_create(struct device *dev,
 
 struct cxl_memdev *devm_cxl_probe_mem(struct cxl_dev_state *cxlds,
 				      struct range *range);
+
+int cxl_set_capacity(struct cxl_dev_state *cxlds, u64 capacity);
 #endif /* __CXL_CXL_H__ */
diff --git a/include/cxl/mailbox.h b/include/cxl/mailbox.h
index c4e99e2e3a9d4..d008b9db07aa6 100644
--- a/include/cxl/mailbox.h
+++ b/include/cxl/mailbox.h
@@ -50,6 +50,7 @@ struct cxl_mbox_cmd {
  * @payload_size: Size of space for payload
  *                (CXL 3.1 8.2.8.4.3 Mailbox Capabilities Register)
  * @mbox_mutex: mutex protects device mailbox and firmware
+ * @feat_mutex: serializes multi-part Get/Set Feature transfers
  * @mbox_wait: rcuwait for mailbox
  * @mbox_send: @dev specific transport for transmitting mailbox commands
  * @feat_cap: Features capability
@@ -60,6 +61,7 @@ struct cxl_mailbox {
 	DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
 	size_t payload_size;
 	struct mutex mbox_mutex; /* lock to protect mailbox context */
+	struct mutex feat_mutex;
 	struct rcuwait mbox_wait;
 	int (*mbox_send)(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd *cmd);
 	enum cxl_features_capability feat_cap;
diff --git a/include/cxl/pci.h b/include/cxl/pci.h
new file mode 100644
index 0000000000000..3e0000015871a
--- /dev/null
+++ b/include/cxl/pci.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
+
+#ifndef __CXL_CXL_PCI_H__
+#define __CXL_CXL_PCI_H__
+
+/* Register Block Identifier (RBI) */
+enum cxl_regloc_type {
+	CXL_REGLOC_RBI_EMPTY = 0,
+	CXL_REGLOC_RBI_COMPONENT,
+	CXL_REGLOC_RBI_VIRT,
+	CXL_REGLOC_RBI_MEMDEV,
+	CXL_REGLOC_RBI_PMU,
+	CXL_REGLOC_RBI_TYPES
+};
+
+struct cxl_register_map;
+struct pci_dev;
+
+int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
+		       struct cxl_register_map *map);
+#endif
diff --git a/include/drm/bridge/analogix_dp.h b/include/drm/bridge/analogix_dp.h
index 854af692229b9..0e0b87abee590 100644
--- a/include/drm/bridge/analogix_dp.h
+++ b/include/drm/bridge/analogix_dp.h
@@ -16,12 +16,21 @@ enum analogix_dp_devtype {
 	EXYNOS_DP,
 	RK3288_DP,
 	RK3399_EDP,
+	RK3576_EDP,
 	RK3588_EDP,
 };
 
-static inline bool is_rockchip(enum analogix_dp_devtype type)
+static inline bool analogix_dp_is_rockchip(enum analogix_dp_devtype type)
 {
-	return type == RK3288_DP || type == RK3399_EDP || type == RK3588_EDP;
+	switch (type) {
+	case RK3288_DP:
+	case RK3399_EDP:
+	case RK3576_EDP:
+	case RK3588_EDP:
+		return true;
+	default:
+		return false;
+	}
 }
 
 struct analogix_dp_plat_data {
diff --git a/include/drm/bridge/dw_dp.h b/include/drm/bridge/dw_dp.h
index 25363541e69d5..22105c3e8e4d6 100644
--- a/include/drm/bridge/dw_dp.h
+++ b/include/drm/bridge/dw_dp.h
@@ -24,4 +24,5 @@ struct dw_dp_plat_data {
 
 struct dw_dp *dw_dp_bind(struct device *dev, struct drm_encoder *encoder,
 			 const struct dw_dp_plat_data *plat_data);
+void dw_dp_unbind(struct dw_dp *dp);
 #endif /* __DW_DP__ */
diff --git a/include/drm/display/drm_dp.h b/include/drm/display/drm_dp.h
index 829e4d98d61cd..7154216e0aff9 100644
--- a/include/drm/display/drm_dp.h
+++ b/include/drm/display/drm_dp.h
@@ -354,15 +354,24 @@
 # define DP_DSC_20_PER_DP_DSC_SINK          (1 << 1)
 # define DP_DSC_24_PER_DP_DSC_SINK          (1 << 2)
 
-#define DP_DSC_BITS_PER_PIXEL_INC           0x06F
+#define DP_DSC_MAX_BPP_DELTA_VERSION_1		0x06E
 # define DP_DSC_RGB_YCbCr444_MAX_BPP_DELTA_MASK 0x1f
-# define DP_DSC_RGB_YCbCr420_MAX_BPP_DELTA_MASK 0xe0
+# define DP_DSC_NATIVE_YCbCr420_MAX_BPP_DELTA_MASK 0xe0
+
+# define DP_DSC_BPP_DELTA_444			16
+# define DP_DSC_BPP_DELTA_420			12
+# define DP_DSC_BPP_DELTA_SHIFT_420		5
+
+#define DP_DSC_BITS_PER_PIXEL_INC           0x06F
 # define DP_DSC_BITS_PER_PIXEL_1_16         0x0
 # define DP_DSC_BITS_PER_PIXEL_1_8          0x1
 # define DP_DSC_BITS_PER_PIXEL_1_4          0x2
 # define DP_DSC_BITS_PER_PIXEL_1_2          0x3
 # define DP_DSC_BITS_PER_PIXEL_1_1          0x4
 # define DP_DSC_BITS_PER_PIXEL_MASK         0x7
+# define DP_DSC_NATIVE_YCbCr422_MAX_BPP_DELTA_MASK  0x78
+# define DP_DSC_BPP_DELTA_NATIVE_SHIFT_422	3
+# define DP_DSC_BPP_DELTA_NATIVE_422		16
 
 #define DP_PSR_SUPPORT                      0x070   /* XXX 1.2? */
 # define DP_PSR_IS_SUPPORTED                1
diff --git a/include/drm/display/drm_hdmi_state_helper.h b/include/drm/display/drm_hdmi_state_helper.h
index 0adc30c55ec91..13375bd0f4ae1 100644
--- a/include/drm/display/drm_hdmi_state_helper.h
+++ b/include/drm/display/drm_hdmi_state_helper.h
@@ -11,8 +11,8 @@ struct hdmi_audio_infoframe;
 
 enum drm_connector_status;
 
-void __drm_atomic_helper_connector_hdmi_reset(struct drm_connector *connector,
-					      struct drm_connector_state *new_conn_state);
+void __drm_atomic_helper_connector_hdmi_state_init(struct drm_connector *connector,
+						   struct drm_connector_state *new_conn_state);
 
 int drm_atomic_helper_connector_hdmi_check(struct drm_connector *connector,
 					   struct drm_atomic_commit *state);
diff --git a/include/drm/drm_atomic.h b/include/drm/drm_atomic.h
index 1a80a8cdf2693..88087910ab1a3 100644
--- a/include/drm/drm_atomic.h
+++ b/include/drm/drm_atomic.h
@@ -265,7 +265,10 @@ struct drm_private_state_funcs {
 	 * @atomic_create_state:
 	 *
 	 * Allocates a pristine, initialized, state for the private
-	 * object and returns it.
+	 * object and returns it. This callback must have no side
+	 * effects: in particular, the returned state must not be
+	 * assigned to the object's state pointer and it must not affect
+	 * the hardware state.
 	 *
 	 * RETURNS:
 	 *
diff --git a/include/drm/drm_atomic_helper.h b/include/drm/drm_atomic_helper.h
index b84152810abb9..4cfeec70d648a 100644
--- a/include/drm/drm_atomic_helper.h
+++ b/include/drm/drm_atomic_helper.h
@@ -295,4 +295,11 @@ drm_atomic_helper_bridge_propagate_bus_fmt(struct drm_bridge *bridge,
 					u32 output_fmt,
 					unsigned int *num_input_fmts);
 
+u32 *
+drm_atomic_helper_bridge_get_hdmi_output_bus_fmts(struct drm_bridge *bridge,
+					struct drm_bridge_state *bridge_state,
+					struct drm_crtc_state *crtc_state,
+					struct drm_connector_state *conn_state,
+					unsigned int *num_output_fmts);
+
 #endif /* DRM_ATOMIC_HELPER_H_ */
diff --git a/include/drm/drm_atomic_state_helper.h b/include/drm/drm_atomic_state_helper.h
index 61a3b38ad49fd..34a599c3d86d4 100644
--- a/include/drm/drm_atomic_state_helper.h
+++ b/include/drm/drm_atomic_state_helper.h
@@ -40,11 +40,13 @@ struct drm_private_state;
 struct drm_modeset_acquire_ctx;
 struct drm_device;
 
-void __drm_atomic_helper_crtc_state_reset(struct drm_crtc_state *state,
+void __drm_atomic_helper_crtc_state_init(struct drm_crtc_state *state,
 					  struct drm_crtc *crtc);
 void __drm_atomic_helper_crtc_reset(struct drm_crtc *crtc,
 				    struct drm_crtc_state *state);
 void drm_atomic_helper_crtc_reset(struct drm_crtc *crtc);
+struct drm_crtc_state *
+drm_atomic_helper_crtc_create_state(struct drm_crtc *crtc);
 void __drm_atomic_helper_crtc_duplicate_state(struct drm_crtc *crtc,
 					      struct drm_crtc_state *state);
 struct drm_crtc_state *
@@ -53,8 +55,10 @@ void __drm_atomic_helper_crtc_destroy_state(struct drm_crtc_state *state);
 void drm_atomic_helper_crtc_destroy_state(struct drm_crtc *crtc,
 					  struct drm_crtc_state *state);
 
-void __drm_atomic_helper_plane_state_reset(struct drm_plane_state *state,
+void __drm_atomic_helper_plane_state_init(struct drm_plane_state *state,
 					   struct drm_plane *plane);
+struct drm_plane_state *
+drm_atomic_helper_plane_create_state(struct drm_plane *plane);
 void __drm_atomic_helper_plane_reset(struct drm_plane *plane,
 				     struct drm_plane_state *state);
 void drm_atomic_helper_plane_reset(struct drm_plane *plane);
@@ -66,11 +70,13 @@ void __drm_atomic_helper_plane_destroy_state(struct drm_plane_state *state);
 void drm_atomic_helper_plane_destroy_state(struct drm_plane *plane,
 					  struct drm_plane_state *state);
 
-void __drm_atomic_helper_connector_state_reset(struct drm_connector_state *conn_state,
+void __drm_atomic_helper_connector_state_init(struct drm_connector_state *conn_state,
 					       struct drm_connector *connector);
 void __drm_atomic_helper_connector_reset(struct drm_connector *connector,
 					 struct drm_connector_state *conn_state);
 void drm_atomic_helper_connector_reset(struct drm_connector *connector);
+struct drm_connector_state *
+drm_atomic_helper_connector_create_state(struct drm_connector *connector);
 void drm_atomic_helper_connector_tv_reset(struct drm_connector *connector);
 int drm_atomic_helper_connector_tv_check(struct drm_connector *connector,
 					 struct drm_atomic_commit *state);
@@ -96,7 +102,7 @@ struct drm_bridge_state *
 drm_atomic_helper_bridge_duplicate_state(struct drm_bridge *bridge);
 void drm_atomic_helper_bridge_destroy_state(struct drm_bridge *bridge,
 					    struct drm_bridge_state *state);
-void __drm_atomic_helper_bridge_reset(struct drm_bridge *bridge,
-				      struct drm_bridge_state *state);
+void __drm_atomic_helper_bridge_state_init(struct drm_bridge_state *state,
+					   struct drm_bridge *bridge);
 struct drm_bridge_state *
-drm_atomic_helper_bridge_reset(struct drm_bridge *bridge);
+drm_atomic_helper_bridge_create_state(struct drm_bridge *bridge);
diff --git a/include/drm/drm_bridge.h b/include/drm/drm_bridge.h
index 4ba3a5deef9a6..3ac84ed57ab2a 100644
--- a/include/drm/drm_bridge.h
+++ b/include/drm/drm_bridge.h
@@ -173,52 +173,6 @@ struct drm_bridge_funcs {
 	bool (*mode_fixup)(struct drm_bridge *bridge,
 			   const struct drm_display_mode *mode,
 			   struct drm_display_mode *adjusted_mode);
-	/**
-	 * @disable:
-	 *
-	 * This callback should disable the bridge. It is called right before
-	 * the preceding element in the display pipe is disabled. If the
-	 * preceding element is a bridge this means it's called before that
-	 * bridge's @disable vfunc. If the preceding element is a &drm_encoder
-	 * it's called right before the &drm_encoder_helper_funcs.disable,
-	 * &drm_encoder_helper_funcs.prepare or &drm_encoder_helper_funcs.dpms
-	 * hook.
-	 *
-	 * The bridge can assume that the display pipe (i.e. clocks and timing
-	 * signals) feeding it is still running when this callback is called.
-	 *
-	 * The @disable callback is optional.
-	 *
-	 * NOTE:
-	 *
-	 * This is deprecated, do not use!
-	 * New drivers shall use &drm_bridge_funcs.atomic_disable.
-	 */
-	void (*disable)(struct drm_bridge *bridge);
-
-	/**
-	 * @post_disable:
-	 *
-	 * This callback should disable the bridge. It is called right after the
-	 * preceding element in the display pipe is disabled. If the preceding
-	 * element is a bridge this means it's called after that bridge's
-	 * @post_disable function. If the preceding element is a &drm_encoder
-	 * it's called right after the encoder's
-	 * &drm_encoder_helper_funcs.disable, &drm_encoder_helper_funcs.prepare
-	 * or &drm_encoder_helper_funcs.dpms hook.
-	 *
-	 * The bridge must assume that the display pipe (i.e. clocks and timing
-	 * signals) feeding it is no longer running when this callback is
-	 * called.
-	 *
-	 * The @post_disable callback is optional.
-	 *
-	 * NOTE:
-	 *
-	 * This is deprecated, do not use!
-	 * New drivers shall use &drm_bridge_funcs.atomic_post_disable.
-	 */
-	void (*post_disable)(struct drm_bridge *bridge);
 
 	/**
 	 * @mode_set:
@@ -249,55 +203,6 @@ struct drm_bridge_funcs {
 	void (*mode_set)(struct drm_bridge *bridge,
 			 const struct drm_display_mode *mode,
 			 const struct drm_display_mode *adjusted_mode);
-	/**
-	 * @pre_enable:
-	 *
-	 * This callback should enable the bridge. It is called right before
-	 * the preceding element in the display pipe is enabled. If the
-	 * preceding element is a bridge this means it's called before that
-	 * bridge's @pre_enable function. If the preceding element is a
-	 * &drm_encoder it's called right before the encoder's
-	 * &drm_encoder_helper_funcs.enable, &drm_encoder_helper_funcs.commit or
-	 * &drm_encoder_helper_funcs.dpms hook.
-	 *
-	 * The display pipe (i.e. clocks and timing signals) feeding this bridge
-	 * will not yet be running when this callback is called. The bridge must
-	 * not enable the display link feeding the next bridge in the chain (if
-	 * there is one) when this callback is called.
-	 *
-	 * The @pre_enable callback is optional.
-	 *
-	 * NOTE:
-	 *
-	 * This is deprecated, do not use!
-	 * New drivers shall use &drm_bridge_funcs.atomic_pre_enable.
-	 */
-	void (*pre_enable)(struct drm_bridge *bridge);
-
-	/**
-	 * @enable:
-	 *
-	 * This callback should enable the bridge. It is called right after
-	 * the preceding element in the display pipe is enabled. If the
-	 * preceding element is a bridge this means it's called after that
-	 * bridge's @enable function. If the preceding element is a
-	 * &drm_encoder it's called right after the encoder's
-	 * &drm_encoder_helper_funcs.enable, &drm_encoder_helper_funcs.commit or
-	 * &drm_encoder_helper_funcs.dpms hook.
-	 *
-	 * The bridge can assume that the display pipe (i.e. clocks and timing
-	 * signals) feeding it is running when this callback is called. This
-	 * callback must enable the display link feeding the next bridge in the
-	 * chain if there is one.
-	 *
-	 * The @enable callback is optional.
-	 *
-	 * NOTE:
-	 *
-	 * This is deprecated, do not use!
-	 * New drivers shall use &drm_bridge_funcs.atomic_enable.
-	 */
-	void (*enable)(struct drm_bridge *bridge);
 
 	/**
 	 * @atomic_pre_enable:
@@ -504,31 +409,20 @@ struct drm_bridge_funcs {
 			    struct drm_connector_state *conn_state);
 
 	/**
-	 * @atomic_reset:
+	 * @atomic_create_state:
 	 *
-	 * Reset the bridge to a predefined state (or retrieve its current
-	 * state) and return a &drm_bridge_state object matching this state.
-	 * This function is called at attach time.
-	 *
-	 * The atomic_reset hook is mandatory if the bridge implements any of
-	 * the atomic hooks, and should be left unassigned otherwise. For
-	 * bridges that don't subclass &drm_bridge_state, the
-	 * drm_atomic_helper_bridge_reset() helper function shall be used to
-	 * implement this hook.
-	 *
-	 * Note that the atomic_reset() semantics is not exactly matching the
-	 * reset() semantics found on other components (connector, plane, ...).
-	 *
-	 * 1. The reset operation happens when the bridge is attached, not when
-	 *    drm_mode_config_reset() is called
-	 * 2. It's meant to be used exclusively on bridges that have been
-	 *    converted to the ATOMIC API
+	 * Allocate a pristine, initialized, state for the bridge
+	 * object and return it. This callback must have no side
+	 * effects: in particular, the returned state must not be
+	 * assigned to the object's state pointer and it must not affect
+	 * the hardware state.
 	 *
 	 * RETURNS:
-	 * A valid drm_bridge_state object in case of success, an ERR_PTR()
-	 * giving the reason of the failure otherwise.
+	 *
+	 * A new, pristine, bridge state instance or an error pointer
+	 * on failure.
 	 */
-	struct drm_bridge_state *(*atomic_reset)(struct drm_bridge *bridge);
+	struct drm_bridge_state *(*atomic_create_state)(struct drm_bridge *bridge);
 
 	/**
 	 * @detect:
@@ -1256,6 +1150,10 @@ struct drm_bridge {
 	 * @hpd_mutex: Protects the @hpd_cb and @hpd_data fields.
 	 */
 	struct mutex hpd_mutex;
+	/**
+	 * @hpd_state_mutex: Protects the HPD en/disablement state for the bridge.
+	 */
+	struct mutex hpd_state_mutex;
 	/**
 	 * @hpd_cb: Hot plug detection callback, registered with
 	 * drm_bridge_hpd_enable().
@@ -1366,14 +1264,6 @@ drm_bridge_get_current_state(struct drm_bridge *bridge)
 	if (!bridge)
 		return NULL;
 
-	/*
-	 * Only atomic bridges will have bridge->base initialized by
-	 * drm_atomic_private_obj_init(), so we need to make sure we're
-	 * working with one before we try to use the lock.
-	 */
-	if (!bridge->funcs || !bridge->funcs->atomic_reset)
-		return NULL;
-
 	drm_modeset_lock_assert_held(&bridge->base.lock);
 
 	if (!bridge->base.state)
@@ -1481,9 +1371,9 @@ static inline struct drm_bridge *__drm_for_each_bridge_in_chain_next(struct drm_
 DEFINE_FREE(__drm_for_each_bridge_in_chain_cleanup, struct drm_bridge *,
 	if (_T) { mutex_unlock(&_T->encoder->bridge_chain_mutex); drm_bridge_put(_T); })
 
-/* Internal to drm_for_each_bridge_in_chain_scoped() */
+/* Internal to drm_for_each_bridge_in_chain() */
 static inline struct drm_bridge *
-__drm_for_each_bridge_in_chain_scoped_start(struct drm_encoder *encoder)
+__drm_for_each_bridge_in_chain_start(struct drm_encoder *encoder)
 {
 	mutex_lock(&encoder->bridge_chain_mutex);
 
@@ -1496,8 +1386,7 @@ __drm_for_each_bridge_in_chain_scoped_start(struct drm_encoder *encoder)
 }
 
 /**
- * drm_for_each_bridge_in_chain_scoped - iterate over all bridges attached
- *                                       to an encoder
+ * drm_for_each_bridge_in_chain - iterate over all bridges attached to an encoder
  * @encoder: the encoder to iterate bridges on
  * @bridge: a bridge pointer updated to point to the current bridge at each
  *	    iteration
@@ -1507,9 +1396,9 @@ __drm_for_each_bridge_in_chain_scoped_start(struct drm_encoder *encoder)
  * Automatically gets/puts the bridge reference while iterating and locks
  * the encoder chain mutex to prevent chain modifications while iterating.
  */
-#define drm_for_each_bridge_in_chain_scoped(encoder, bridge)				\
+#define drm_for_each_bridge_in_chain(encoder, bridge)					\
 	for (struct drm_bridge *bridge __free(__drm_for_each_bridge_in_chain_cleanup) =	\
-		__drm_for_each_bridge_in_chain_scoped_start((encoder));			\
+		__drm_for_each_bridge_in_chain_start((encoder));			\
 	     bridge;									\
 	     bridge = __drm_for_each_bridge_in_chain_next(bridge))			\
 
diff --git a/include/drm/drm_colorop.h b/include/drm/drm_colorop.h
index d08a6a8a83922..224fae40ed2b2 100644
--- a/include/drm/drm_colorop.h
+++ b/include/drm/drm_colorop.h
@@ -423,6 +423,8 @@ int drm_plane_colorop_3dlut_init(struct drm_device *dev, struct drm_colorop *col
 				 enum drm_colorop_lut3d_interpolation_type interpolation,
 				 uint32_t flags);
 
+struct drm_colorop_state *
+drm_atomic_helper_colorop_create_state(struct drm_colorop *colorop);
 struct drm_colorop_state *
 drm_atomic_helper_colorop_duplicate_state(struct drm_colorop *colorop);
 
diff --git a/include/drm/drm_connector.h b/include/drm/drm_connector.h
index 5f5ca023cd654..af075c38f4db3 100644
--- a/include/drm/drm_connector.h
+++ b/include/drm/drm_connector.h
@@ -571,12 +571,80 @@ enum drm_colorspace {
  *   YCbCr 4:2:2 output format (ie. with horizontal subsampling)
  * @DRM_OUTPUT_COLOR_FORMAT_YCBCR420:
  *   YCbCr 4:2:0 output format (ie. with horizontal and vertical subsampling)
+ * @DRM_OUTPUT_COLOR_FORMAT_COUNT:
+ *   Number of valid output color format values in this enum
  */
 enum drm_output_color_format {
 	DRM_OUTPUT_COLOR_FORMAT_RGB444 = 0,
 	DRM_OUTPUT_COLOR_FORMAT_YCBCR444,
 	DRM_OUTPUT_COLOR_FORMAT_YCBCR422,
 	DRM_OUTPUT_COLOR_FORMAT_YCBCR420,
+	DRM_OUTPUT_COLOR_FORMAT_COUNT,
+};
+
+/**
+ * enum drm_connector_color_format - Connector Color Format Request
+ *
+ * This enum, unlike &enum drm_output_color_format, is used to specify requests
+ * for a specific color format on a connector through the DRM "color format"
+ * property. The difference is that it has an "AUTO" value to specify that
+ * no specific choice has been made.
+ */
+enum drm_connector_color_format {
+	/**
+	 * @DRM_CONNECTOR_COLOR_FORMAT_AUTO: The driver or display protocol
+	 * helpers should pick a suitable color format. All implementations of a
+	 * specific display protocol must behave the same way with "AUTO", but
+	 * different display protocols do not necessarily have the same "AUTO"
+	 * semantics.
+	 *
+	 * For HDMI, "AUTO" picks RGB, but falls back to YCbCr 4:2:0 if the
+	 * bandwidth required for full-scale RGB is not available, or the mode
+	 * is YCbCr 4:2:0-only, as long as the mode and output both support
+	 * YCbCr 4:2:0.
+	 *
+	 * For display protocols other than HDMI, the recursive bridge chain
+	 * format selection picks the first chain of bridge formats that works,
+	 * as has already been the case before the introduction of the "color
+	 * format" property. Non-HDMI bridges should therefore either sort their
+	 * bus output formats by preference, or agree on a unified auto format
+	 * selection logic that's implemented in a common state helper (like
+	 * how HDMI does it).
+	 */
+	DRM_CONNECTOR_COLOR_FORMAT_AUTO = 0,
+
+	/**
+	 * @DRM_CONNECTOR_COLOR_FORMAT_RGB444: RGB output format. The
+	 * quantization range depends on the value of the "Broadcast RGB"
+	 * property if it is present on the connector.
+	 */
+	DRM_CONNECTOR_COLOR_FORMAT_RGB444,
+
+	/**
+	 * @DRM_CONNECTOR_COLOR_FORMAT_YCBCR444: YCbCr 4:4:4 output format (ie.
+	 * not subsampled). Quantization range is "Limited" by default.
+	 */
+	DRM_CONNECTOR_COLOR_FORMAT_YCBCR444,
+
+	/**
+	 * @DRM_CONNECTOR_COLOR_FORMAT_YCBCR422: YCbCr 4:2:2 output format (ie.
+	 * with horizontal subsampling). Quantization range is "Limited" by
+	 * default.
+	 */
+	DRM_CONNECTOR_COLOR_FORMAT_YCBCR422,
+
+	/**
+	 * @DRM_CONNECTOR_COLOR_FORMAT_YCBCR420: YCbCr 4:2:0 output format (ie.
+	 * with horizontal and vertical subsampling). Quantization range is
+	 * "Limited" by default.
+	 */
+	DRM_CONNECTOR_COLOR_FORMAT_YCBCR420,
+
+	/**
+	 * @DRM_CONNECTOR_COLOR_FORMAT_COUNT: Number of valid connector color
+	 * format values in this enum
+	 */
+	DRM_CONNECTOR_COLOR_FORMAT_COUNT,
 };
 
 const char *
@@ -921,6 +989,12 @@ struct drm_display_info {
 	 * @amd_vsdb: AMD-specific VSDB information.
 	 */
 	struct drm_amd_vsdb_info amd_vsdb;
+
+	/**
+	 * @panel_type: Panel type from DisplayID Display Parameters
+	 * Data Block (tag 0x21). Uses DRM_MODE_PANEL_TYPE_* constants.
+	 */
+	u8 panel_type;
 };
 
 int drm_display_info_set_bus_formats(struct drm_display_info *info,
@@ -1167,6 +1241,13 @@ struct drm_connector_state {
 	 */
 	enum drm_colorspace colorspace;
 
+	/**
+	 * @color_format: State variable for Connector property to request
+	 * color format change on Sink. This is most commonly used to switch
+	 * between RGB to YUV and vice-versa.
+	 */
+	enum drm_connector_color_format color_format;
+
 	/**
 	 * @writeback_job: Writeback job for writeback connectors
 	 *
@@ -1571,6 +1652,22 @@ struct drm_connector_funcs {
 	 */
 	void (*destroy)(struct drm_connector *connector);
 
+	/**
+	 * @atomic_create_state:
+	 *
+	 * Allocate a pristine, initialized, state for the connector
+	 * object and return it. This callback must have no side
+	 * effects: in particular, the returned state must not be
+	 * assigned to the object's state pointer and it must not affect
+	 * the hardware state.
+	 *
+	 * RETURNS:
+	 *
+	 * A new, pristine, connector state instance or an error pointer
+	 * on failure.
+	 */
+	struct drm_connector_state *(*atomic_create_state)(struct drm_connector *connector);
+
 	/**
 	 * @atomic_duplicate_state:
 	 *
@@ -1712,6 +1809,16 @@ struct drm_connector_funcs {
 	 * Allows connectors to create connector-specific debugfs files.
 	 */
 	void (*debugfs_init)(struct drm_connector *connector, struct dentry *root);
+
+	/**
+	 * @color_format:
+	 *
+	 * Allows connectors to return a connector color format other than
+	 * @conn_state.color_format for purposes of e.g. display protocol
+	 * specific helper logic having already mapped it to an output format.
+	 */
+	enum drm_connector_color_format (*color_format)(
+		const struct drm_connector_state *conn_state);
 };
 
 /**
@@ -2165,6 +2272,12 @@ struct drm_connector {
 	 */
 	struct drm_property *colorspace_property;
 
+	/**
+	 * @color_format_property: Connector property to set the suitable
+	 * color format supported by the sink.
+	 */
+	struct drm_property *color_format_property;
+
 	/**
 	 * @path_blob_ptr:
 	 *
@@ -2522,6 +2635,8 @@ drm_connector_is_unregistered(struct drm_connector *connector)
 
 void drm_connector_oob_hotplug_event(struct fwnode_handle *connector_fwnode,
 				     enum drm_connector_status status);
+enum drm_connector_color_format
+drm_connector_get_color_format(const struct drm_connector_state *conn_state);
 const char *drm_get_connector_type_name(unsigned int connector_type);
 const char *drm_get_connector_status_name(enum drm_connector_status status);
 const char *drm_get_subpixel_order_name(enum subpixel_order order);
@@ -2648,6 +2763,9 @@ bool drm_connector_has_possible_encoder(struct drm_connector *connector,
 					struct drm_encoder *encoder);
 const char *drm_get_colorspace_name(enum drm_colorspace colorspace);
 
+int drm_connector_attach_color_format_property(struct drm_connector *connector,
+					       unsigned long supported_color_formats);
+
 /**
  * drm_for_each_connector_iter - connector_list iterator macro
  * @connector: &struct drm_connector pointer used as cursor
diff --git a/include/drm/drm_crtc.h b/include/drm/drm_crtc.h
index c6dbe8b7db9ee..152349f973e33 100644
--- a/include/drm/drm_crtc.h
+++ b/include/drm/drm_crtc.h
@@ -638,6 +638,22 @@ struct drm_crtc_funcs {
 	int (*set_property)(struct drm_crtc *crtc,
 			    struct drm_property *property, uint64_t val);
 
+	/**
+	 * @atomic_create_state:
+	 *
+	 * Allocate a pristine, initialized, state for the CRTC object
+	 * and return it. This callback must have no side effects: in
+	 * particular, the returned state must not be assigned to the
+	 * object's state pointer and it must not affect the hardware
+	 * state.
+	 *
+	 * RETURNS:
+	 *
+	 * A new, pristine, CRTC state instance or an error pointer
+	 * on failure.
+	 */
+	struct drm_crtc_state *(*atomic_create_state)(struct drm_crtc *crtc);
+
 	/**
 	 * @atomic_duplicate_state:
 	 *
diff --git a/include/drm/drm_debugfs.h b/include/drm/drm_debugfs.h
index ea8cba94208ab..eb93512b0f230 100644
--- a/include/drm/drm_debugfs.h
+++ b/include/drm/drm_debugfs.h
@@ -48,7 +48,7 @@
  * For each DRM GPU VA space drivers should call drm_debugfs_gpuva_info() from
  * their @show callback.
  */
-#define DRM_DEBUGFS_GPUVA_INFO(show, data) {"gpuvas", show, DRIVER_GEM_GPUVA, data}
+#define DRM_DEBUGFS_GPUVA_INFO(show, data) {"gpuvas", show, 0, data}
 
 /**
  * struct drm_info_list - debugfs info list entry
diff --git a/include/drm/drm_drv.h b/include/drm/drm_drv.h
index 42fc085f986de..e09559495c5bf 100644
--- a/include/drm/drm_drv.h
+++ b/include/drm/drm_drv.h
@@ -107,12 +107,6 @@ enum drm_driver_feature {
 	 * acceleration should be handled by two drivers that are connected using auxiliary bus.
 	 */
 	DRIVER_COMPUTE_ACCEL            = BIT(7),
-	/**
-	 * @DRIVER_GEM_GPUVA:
-	 *
-	 * Driver supports user defined GPU VA bindings for GEM objects.
-	 */
-	DRIVER_GEM_GPUVA		= BIT(8),
 	/**
 	 * @DRIVER_CURSOR_HOTSPOT:
 	 *
diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h
index 8a704f6a65c15..885244e375d32 100644
--- a/include/drm/drm_gem.h
+++ b/include/drm/drm_gem.h
@@ -661,9 +661,6 @@ static inline bool drm_gem_is_imported(const struct drm_gem_object *obj)
  *
  * This initializes the &drm_gem_object's &drm_gpuvm_bo list.
  *
- * Calling this function is only necessary for drivers intending to support the
- * &drm_driver_feature DRIVER_GEM_GPUVA.
- *
  * See also drm_gem_gpuva_set_lock().
  */
 static inline void drm_gem_gpuva_init(struct drm_gem_object *obj)
diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h
index b2c23af628e1a..bcbb92f3fe9ba 100644
--- a/include/drm/drm_gem_shmem_helper.h
+++ b/include/drm/drm_gem_shmem_helper.h
@@ -142,6 +142,10 @@ struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem)
 void drm_gem_shmem_print_info(const struct drm_gem_shmem_object *shmem,
 			      struct drm_printer *p, unsigned int indent);
 
+int drm_gem_shmem_create_with_handle(struct drm_file *file_priv,
+				     struct drm_device *dev, size_t size,
+				     uint32_t *handle);
+
 extern const struct vm_operations_struct drm_gem_shmem_vm_ops;
 
 /*
diff --git a/include/drm/drm_gpusvm.h b/include/drm/drm_gpusvm.h
index 8a4d7134a9a72..b7d987bf76aaa 100644
--- a/include/drm/drm_gpusvm.h
+++ b/include/drm/drm_gpusvm.h
@@ -109,9 +109,7 @@ struct drm_gpusvm_notifier {
 /**
  * struct drm_gpusvm_pages_flags - Structure representing a GPU SVM pages flags
  *
- * @migrate_devmem: Flag indicating whether the pages can be migrated to device memory
  * @unmapped: Flag indicating if the pages has been unmapped
- * @partial_unmap: Flag indicating if the pages has been partially unmapped
  * @has_devmem_pages: Flag indicating if the pages has devmem pages
  * @has_dma_mapping: Flag indicating if the pages has a DMA mapping
  * @__flags: Flags for pages in u16 form (used for READ_ONCE)
@@ -119,11 +117,8 @@ struct drm_gpusvm_notifier {
 struct drm_gpusvm_pages_flags {
 	union {
 		struct {
-			/* All flags below must be set upon creation */
-			u16 migrate_devmem : 1;
 			/* All flags below must be set / cleared under notifier lock */
 			u16 unmapped : 1;
-			u16 partial_unmap : 1;
 			u16 has_devmem_pages : 1;
 			u16 has_dma_mapping : 1;
 		};
@@ -134,6 +129,7 @@ struct drm_gpusvm_pages_flags {
 /**
  * struct drm_gpusvm_pages - Structure representing a GPU SVM mapped pages
  *
+ * @drm: The DRM device that owns the dma mappings
  * @dma_addr: Device address array
  * @dpagemap: The struct drm_pagemap of the device pages we're dma-mapping.
  *            Note this is assuming only one drm_pagemap per range is allowed.
@@ -143,6 +139,7 @@ struct drm_gpusvm_pages_flags {
  * @flags: Flags for the range; see &struct drm_gpusvm_pages_flags
  */
 struct drm_gpusvm_pages {
+	struct drm_device *drm;
 	struct drm_pagemap_addr *dma_addr;
 	struct drm_pagemap *dpagemap;
 	struct dma_iova_state state;
@@ -151,6 +148,27 @@ struct drm_gpusvm_pages {
 	struct drm_gpusvm_pages_flags flags;
 };
 
+/**
+ * struct drm_gpusvm_range_flags - Range-level GPU SVM flags
+ *
+ * @migrate_devmem: Flag indicating whether the range can be migrated to device memory
+ * @unmapped: Flag indicating if the range has been unmapped
+ * @partial_unmap: Flag indicating if the range has been partially unmapped
+ * @__flags: All flags in u16 form (used for READ_ONCE)
+ */
+struct drm_gpusvm_range_flags {
+	union {
+		struct {
+			/* All flags below must be set upon creation */
+			u16 migrate_devmem	: 1;
+			/* All flags below must be set / cleared under notifier lock */
+			u16 unmapped		: 1;
+			u16 partial_unmap	: 1;
+		};
+		u16 __flags;
+	};
+};
+
 /**
  * struct drm_gpusvm_range - Structure representing a GPU SVM range
  *
@@ -159,7 +177,7 @@ struct drm_gpusvm_pages {
  * @refcount: Reference count for the range
  * @itree: Interval tree node for the range (inserted in GPU SVM notifier)
  * @entry: List entry to fast interval tree traversal
- * @pages: The pages for this range.
+ * @flags: Flags for range see &struct drm_gpusvm_range_flags
  *
  * This structure represents a GPU SVM range used for tracking memory ranges
  * mapped in a DRM device.
@@ -170,14 +188,13 @@ struct drm_gpusvm_range {
 	struct kref refcount;
 	struct interval_tree_node itree;
 	struct list_head entry;
-	struct drm_gpusvm_pages pages;
+	struct drm_gpusvm_range_flags flags;
 };
 
 /**
  * struct drm_gpusvm - GPU SVM structure
  *
  * @name: Name of the GPU SVM
- * @drm: Pointer to the DRM device structure
  * @mm: Pointer to the mm_struct for the address space
  * @mm_start: Start address of GPU SVM
  * @mm_range: Range of the GPU SVM
@@ -201,7 +218,6 @@ struct drm_gpusvm_range {
  */
 struct drm_gpusvm {
 	const char *name;
-	struct drm_device *drm;
 	struct mm_struct *mm;
 	unsigned long mm_start;
 	unsigned long mm_range;
@@ -253,7 +269,7 @@ struct drm_gpusvm_ctx {
 };
 
 int drm_gpusvm_init(struct drm_gpusvm *gpusvm,
-		    const char *name, struct drm_device *drm,
+		    const char *name,
 		    struct mm_struct *mm,
 		    unsigned long mm_start, unsigned long mm_range,
 		    unsigned long notifier_size,
@@ -287,16 +303,8 @@ drm_gpusvm_range_get(struct drm_gpusvm_range *range);
 
 void drm_gpusvm_range_put(struct drm_gpusvm_range *range);
 
-bool drm_gpusvm_range_pages_valid(struct drm_gpusvm *gpusvm,
-				  struct drm_gpusvm_range *range);
-
-int drm_gpusvm_range_get_pages(struct drm_gpusvm *gpusvm,
-			       struct drm_gpusvm_range *range,
-			       const struct drm_gpusvm_ctx *ctx);
-
-void drm_gpusvm_range_unmap_pages(struct drm_gpusvm *gpusvm,
-				  struct drm_gpusvm_range *range,
-				  const struct drm_gpusvm_ctx *ctx);
+bool drm_gpusvm_pages_valid(struct drm_gpusvm *gpusvm,
+			    struct drm_gpusvm_pages *svm_pages);
 
 bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start,
 			    unsigned long end);
@@ -310,6 +318,8 @@ drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start,
 		      unsigned long end);
 
 void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range,
+				   struct drm_gpusvm_pages *pages,
+				   unsigned int pages_count,
 				   const struct mmu_notifier_range *mmu_range);
 
 int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
@@ -328,6 +338,23 @@ void drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm,
 			   struct drm_gpusvm_pages *svm_pages,
 			   unsigned long npages);
 
+/**
+ * drm_gpusvm_init_pages() - Initialize a freshly allocated drm_gpusvm_pages
+ * @svm_pages: Pointer to the drm_gpusvm_pages to initialize.
+ * @drm: The DRM device that will own DMA mappings for this pages object.
+ *
+ * Drivers that embed one or more drm_gpusvm_pages in their own range
+ * structure must call this once on each pages instance after allocation,
+ * before the first drm_gpusvm_get_pages() / unmap / free.
+ */
+static inline void drm_gpusvm_init_pages(struct drm_gpusvm_pages *svm_pages,
+					 struct drm_device *drm)
+{
+	memset(svm_pages, 0, sizeof(*svm_pages));
+	svm_pages->drm = drm;
+	svm_pages->notifier_seq = LONG_MAX;
+}
+
 /**
  * enum drm_gpusvm_scan_result - Scan result from the drm_gpusvm_scan_mm() function.
  * @DRM_GPUSVM_SCAN_UNPOPULATED: At least one page was not present or inaccessible.
diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h
index 655bd9104ffb2..38221d83285b2 100644
--- a/include/drm/drm_gpuvm.h
+++ b/include/drm/drm_gpuvm.h
@@ -159,8 +159,6 @@ struct drm_gpuva *drm_gpuva_find(struct drm_gpuvm *gpuvm,
 				 u64 addr, u64 range);
 struct drm_gpuva *drm_gpuva_find_first(struct drm_gpuvm *gpuvm,
 				       u64 addr, u64 range);
-struct drm_gpuva *drm_gpuva_find_prev(struct drm_gpuvm *gpuvm, u64 start);
-struct drm_gpuva *drm_gpuva_find_next(struct drm_gpuvm *gpuvm, u64 end);
 
 /**
  * drm_gpuva_invalidate() - sets whether the backing GEM of this &drm_gpuva is
@@ -930,6 +928,8 @@ struct drm_gpuva_op_unmap {
  * If either a new mapping's start address is aligned with the start address
  * of the old mapping or the new mapping's end address is aligned with the
  * end address of the old mapping, either @prev or @next is NULL.
+ * This will also be the case when the requested mapping begins before the
+ * old mapping's start address or stretches beyond its end address.
  *
  * Note, the reason for a dedicated remap operation, rather than arbitrary
  * unmap and map operations, is to give drivers the chance of extracting driver
diff --git a/include/drm/drm_managed.h b/include/drm/drm_managed.h
index 72bfac002c062..72d0d68be2261 100644
--- a/include/drm/drm_managed.h
+++ b/include/drm/drm_managed.h
@@ -18,7 +18,7 @@ typedef void (*drmres_release_t)(struct drm_device *dev, void *res);
  * @action: function which should be called when @dev is released
  * @data: opaque pointer, passed to @action
  *
- * This function adds the @release action with optional parameter @data to the
+ * This function adds the release @action with optional parameter @data to the
  * list of cleanup actions for @dev. The cleanup actions will be run in reverse
  * order in the final drm_dev_put() call for @dev.
  */
diff --git a/include/drm/drm_mipi_dsi.h b/include/drm/drm_mipi_dsi.h
index b429acde4f71a..7ff43967251e0 100644
--- a/include/drm/drm_mipi_dsi.h
+++ b/include/drm/drm_mipi_dsi.h
@@ -138,6 +138,8 @@ struct mipi_dsi_host *of_find_mipi_dsi_host_by_node(struct device_node *node);
 #define MIPI_DSI_MODE_LPM		BIT(11)
 /* transmit data ending at the same time for all lanes within one hsync */
 #define MIPI_DSI_HS_PKT_END_ALIGNED	BIT(12)
+/* pack all DSC slices for a line into a single packet */
+#define MIPI_DSI_MODE_DSC_ALL_SLICES_IN_PKT	BIT(13)
 
 enum mipi_dsi_pixel_format {
 	MIPI_DSI_FMT_RGB888,
diff --git a/include/drm/drm_mode_config.h b/include/drm/drm_mode_config.h
index e584652ddf676..d8f5b7e9673ee 100644
--- a/include/drm/drm_mode_config.h
+++ b/include/drm/drm_mode_config.h
@@ -1007,6 +1007,7 @@ static inline int drm_mode_config_init(struct drm_device *dev)
 	return drmm_mode_config_init(dev);
 }
 
+int drm_mode_config_create_initial_state(struct drm_device *dev);
 void drm_mode_config_reset(struct drm_device *dev);
 void drm_mode_config_cleanup(struct drm_device *dev);
 
diff --git a/include/drm/drm_modes.h b/include/drm/drm_modes.h
index b9bb92e4b0295..6e3eccc3c349c 100644
--- a/include/drm/drm_modes.h
+++ b/include/drm/drm_modes.h
@@ -193,6 +193,7 @@ enum drm_mode_status {
 #define DRM_MODE_MATCH_FLAGS (1 << 2)
 #define DRM_MODE_MATCH_3D_FLAGS (1 << 3)
 #define DRM_MODE_MATCH_ASPECT_RATIO (1 << 4)
+#define DRM_MODE_MATCH_TIMINGS_VRR (1 << 5)
 
 /**
  * struct drm_display_mode - DRM kernel-internal display mode structure
diff --git a/include/drm/drm_of.h b/include/drm/drm_of.h
index 7bcc0ccfe0f4b..ebebed14c611e 100644
--- a/include/drm/drm_of.h
+++ b/include/drm/drm_of.h
@@ -20,6 +20,8 @@ struct device_node;
 struct mipi_dsi_device_info;
 struct mipi_dsi_host;
 
+enum drm_panel_orientation;
+
 /**
  * enum drm_lvds_dual_link_pixels - Pixel order of an LVDS dual-link connection
  * @DRM_LVDS_DUAL_LINK_EVEN_ODD_PIXELS: Even pixels are expected to be generated
@@ -47,6 +49,8 @@ int drm_of_component_probe(struct device *dev,
 int drm_of_encoder_active_endpoint(struct device_node *node,
 				   struct drm_encoder *encoder,
 				   struct of_endpoint *endpoint);
+int drm_of_get_panel_orientation(const struct device_node *np,
+				 enum drm_panel_orientation *orientation);
 int drm_of_find_panel_or_bridge(const struct device_node *np,
 				int port, int endpoint,
 				struct drm_panel **panel,
@@ -101,6 +105,13 @@ static inline int drm_of_encoder_active_endpoint(struct device_node *node,
 {
 	return -EINVAL;
 }
+
+static inline int drm_of_get_panel_orientation(const struct device_node *np,
+					       enum drm_panel_orientation *orientation)
+{
+	return -EINVAL;
+}
+
 static inline int drm_of_find_panel_or_bridge(const struct device_node *np,
 					      int port, int endpoint,
 					      struct drm_panel **panel,
diff --git a/include/drm/drm_panel.h b/include/drm/drm_panel.h
index 86b3f9c65c926..b87323443f49c 100644
--- a/include/drm/drm_panel.h
+++ b/include/drm/drm_panel.h
@@ -337,19 +337,11 @@ int drm_panel_get_modes(struct drm_panel *panel, struct drm_connector *connector
 
 #if defined(CONFIG_OF) && defined(CONFIG_DRM_PANEL)
 struct drm_panel *of_drm_find_panel(const struct device_node *np);
-int of_drm_get_panel_orientation(const struct device_node *np,
-				 enum drm_panel_orientation *orientation);
 #else
 static inline struct drm_panel *of_drm_find_panel(const struct device_node *np)
 {
 	return ERR_PTR(-ENODEV);
 }
-
-static inline int of_drm_get_panel_orientation(const struct device_node *np,
-					       enum drm_panel_orientation *orientation)
-{
-	return -ENODEV;
-}
 #endif
 
 #if defined(CONFIG_DRM_PANEL)
diff --git a/include/drm/drm_plane.h b/include/drm/drm_plane.h
index 419c88c873a6f..2c5a5a70a71b8 100644
--- a/include/drm/drm_plane.h
+++ b/include/drm/drm_plane.h
@@ -388,6 +388,22 @@ struct drm_plane_funcs {
 	int (*set_property)(struct drm_plane *plane,
 			    struct drm_property *property, uint64_t val);
 
+	/**
+	 * @atomic_create_state:
+	 *
+	 * Allocate a pristine, initialized, state for the plane object
+	 * and return it. This callback must have no side effects: in
+	 * particular, the returned state must not be assigned to the
+	 * object's state pointer and it must not affect the hardware
+	 * state.
+	 *
+	 * RETURNS:
+	 *
+	 * A new, pristine, plane state instance or an error pointer
+	 * on failure.
+	 */
+	struct drm_plane_state *(*atomic_create_state)(struct drm_plane *plane);
+
 	/**
 	 * @atomic_duplicate_state:
 	 *
diff --git a/include/drm/drm_print.h b/include/drm/drm_print.h
index ab017b05e1751..2adc5ac688e11 100644
--- a/include/drm/drm_print.h
+++ b/include/drm/drm_print.h
@@ -87,7 +87,7 @@ extern unsigned long __drm_debug;
  *  - drm.debug=0x2 will enable DRIVER messages
  *  - drm.debug=0x3 will enable CORE and DRIVER messages
  *  - ...
- *  - drm.debug=0x1ff will enable all messages
+ *  - drm.debug=0x3ff will enable all messages
  *
  * An interesting feature is that it's possible to enable verbose logging at
  * run-time by echoing the debug value in its sysfs node::
diff --git a/include/drm/drm_utils.h b/include/drm/drm_utils.h
index 6a46f755daba0..7e077484c5bbf 100644
--- a/include/drm/drm_utils.h
+++ b/include/drm/drm_utils.h
@@ -19,6 +19,7 @@ int drm_get_panel_orientation_quirk(int width, int height);
 struct drm_panel_backlight_quirk {
 	u16 min_brightness;
 	u32 brightness_mask;
+	bool force_pwm;
 };
 
 const struct drm_panel_backlight_quirk *
diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
index d61c19e78182c..363d13fc929f6 100644
--- a/include/drm/gpu_scheduler.h
+++ b/include/drm/gpu_scheduler.h
@@ -217,8 +217,7 @@ struct drm_sched_entity {
 	 * @stopped:
 	 *
 	 * Marks the enity as removed from rq and destined for
-	 * termination. This is set by calling drm_sched_entity_flush() and by
-	 * drm_sched_fini().
+	 * termination. This is set by calling drm_sched_entity_flush().
 	 */
 	bool 				stopped;
 
diff --git a/include/drm/intel/display_parent_interface.h b/include/drm/intel/display_parent_interface.h
index 39991afeb173d..de395df9ca301 100644
--- a/include/drm/intel/display_parent_interface.h
+++ b/include/drm/intel/display_parent_interface.h
@@ -167,7 +167,9 @@ struct intel_display_overlay_interface {
 
 struct intel_display_panic_interface {
 	struct intel_panic *(*alloc)(void);
-	int (*setup)(struct intel_panic *panic, struct drm_scanout_buffer *sb);
+	int (*setup)(struct intel_panic *panic, struct drm_scanout_buffer *sb,
+		     struct drm_gem_object *obj,
+		     unsigned int (*tiling)(unsigned int x, unsigned int y, unsigned int width));
 	void (*finish)(struct intel_panic *panic);
 };
 
diff --git a/include/drm/intel/mchbar_regs.h b/include/drm/intel/mchbar_regs.h
index ca0d421be16c3..66498ca5e40bb 100644
--- a/include/drm/intel/mchbar_regs.h
+++ b/include/drm/intel/mchbar_regs.h
@@ -6,8 +6,6 @@
 #ifndef __INTEL_MCHBAR_REGS__
 #define __INTEL_MCHBAR_REGS__
 
-#include "i915_reg_defs.h"
-
 /*
  * MCHBAR mirror.
  *
diff --git a/include/drm/intel/pciids.h b/include/drm/intel/pciids.h
index e32ef763427c7..dff389b56eb3b 100644
--- a/include/drm/intel/pciids.h
+++ b/include/drm/intel/pciids.h
@@ -893,8 +893,9 @@
 	MACRO__(0xD741, ## __VA_ARGS__), \
 	MACRO__(0xD742, ## __VA_ARGS__), \
 	MACRO__(0xD743, ## __VA_ARGS__), \
-	MACRO__(0xD744, ## __VA_ARGS__), \
-	MACRO__(0xD745, ## __VA_ARGS__)
+	MACRO__(0xD745, ## __VA_ARGS__), \
+	MACRO__(0xD74A, ## __VA_ARGS__), \
+	MACRO__(0xD74B, ## __VA_ARGS__)
 
 /* CRI */
 #define INTEL_CRI_IDS(MACRO__, ...) \
diff --git a/include/dt-bindings/arm/qcom,ids.h b/include/dt-bindings/arm/qcom,ids.h
index 1af73c0ad41cd..2cf433f61d5e7 100644
--- a/include/dt-bindings/arm/qcom,ids.h
+++ b/include/dt-bindings/arm/qcom,ids.h
@@ -184,6 +184,7 @@
 #define QCOM_ID_IPQ8078			344
 #define QCOM_ID_SDM636			345
 #define QCOM_ID_SDA636			346
+#define QCOM_ID_SDM850			348
 #define QCOM_ID_SDM632			349
 #define QCOM_ID_SDA632			350
 #define QCOM_ID_SDA450			351
diff --git a/include/dt-bindings/clock/qcom,glymur-camcc.h b/include/dt-bindings/clock/qcom,glymur-camcc.h
new file mode 100644
index 0000000000000..0c93fc77ef268
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,glymur-camcc.h
@@ -0,0 +1,122 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_CAM_CC_GLYMUR_H
+#define _DT_BINDINGS_CLK_QCOM_CAM_CC_GLYMUR_H
+
+/* CAM_CC clocks */
+#define CAM_CC_BPS_AHB_CLK					0
+#define CAM_CC_BPS_CLK						1
+#define CAM_CC_BPS_CLK_SRC					2
+#define CAM_CC_BPS_FAST_AHB_CLK					3
+#define CAM_CC_CAMNOC_AXI_NRT_CLK				4
+#define CAM_CC_CAMNOC_AXI_RT_CLK				5
+#define CAM_CC_CAMNOC_AXI_RT_CLK_SRC				6
+#define CAM_CC_CAMNOC_DCD_XO_CLK				7
+#define CAM_CC_CAMNOC_XO_CLK					8
+#define CAM_CC_CCI_0_CLK					9
+#define CAM_CC_CCI_0_CLK_SRC					10
+#define CAM_CC_CCI_1_CLK					11
+#define CAM_CC_CCI_1_CLK_SRC					12
+#define CAM_CC_CORE_AHB_CLK					13
+#define CAM_CC_CPAS_AHB_CLK					14
+#define CAM_CC_CPAS_BPS_CLK					15
+#define CAM_CC_CPAS_FAST_AHB_CLK				16
+#define CAM_CC_CPAS_IFE_0_CLK					17
+#define CAM_CC_CPAS_IFE_1_CLK					18
+#define CAM_CC_CPAS_IFE_LITE_CLK				19
+#define CAM_CC_CPAS_IPE_NPS_CLK					20
+#define CAM_CC_CPHY_RX_CLK_SRC					21
+#define CAM_CC_CSI0PHYTIMER_CLK					22
+#define CAM_CC_CSI0PHYTIMER_CLK_SRC				23
+#define CAM_CC_CSI1PHYTIMER_CLK					24
+#define CAM_CC_CSI1PHYTIMER_CLK_SRC				25
+#define CAM_CC_CSI4PHYTIMER_CLK					26
+#define CAM_CC_CSI4PHYTIMER_CLK_SRC				27
+#define CAM_CC_CSID_CLK						28
+#define CAM_CC_CSID_CLK_SRC					29
+#define CAM_CC_CSID_CSIPHY_RX_CLK				30
+#define CAM_CC_CSIPHY0_CLK					31
+#define CAM_CC_CSIPHY1_CLK					32
+#define CAM_CC_CSIPHY4_CLK					33
+#define CAM_CC_FAST_AHB_CLK_SRC					34
+#define CAM_CC_GDSC_CLK						35
+#define CAM_CC_ICP_AHB_CLK					36
+#define CAM_CC_ICP_CLK						37
+#define CAM_CC_ICP_CLK_SRC					38
+#define CAM_CC_IFE_0_CLK					39
+#define CAM_CC_IFE_0_CLK_SRC					40
+#define CAM_CC_IFE_0_DSP_CLK					41
+#define CAM_CC_IFE_0_FAST_AHB_CLK				42
+#define CAM_CC_IFE_1_CLK					43
+#define CAM_CC_IFE_1_CLK_SRC					44
+#define CAM_CC_IFE_1_DSP_CLK					45
+#define CAM_CC_IFE_1_FAST_AHB_CLK				46
+#define CAM_CC_IFE_LITE_AHB_CLK					47
+#define CAM_CC_IFE_LITE_CLK					48
+#define CAM_CC_IFE_LITE_CLK_SRC					49
+#define CAM_CC_IFE_LITE_CPHY_RX_CLK				50
+#define CAM_CC_IFE_LITE_CSID_CLK				51
+#define CAM_CC_IFE_LITE_CSID_CLK_SRC				52
+#define CAM_CC_IPE_NPS_AHB_CLK					53
+#define CAM_CC_IPE_NPS_CLK					54
+#define CAM_CC_IPE_NPS_CLK_SRC					55
+#define CAM_CC_IPE_NPS_FAST_AHB_CLK				56
+#define CAM_CC_IPE_PPS_CLK					57
+#define CAM_CC_IPE_PPS_FAST_AHB_CLK				58
+#define CAM_CC_JPEG_CLK						59
+#define CAM_CC_JPEG_CLK_SRC					60
+#define CAM_CC_MCLK0_CLK					61
+#define CAM_CC_MCLK0_CLK_SRC					62
+#define CAM_CC_MCLK1_CLK					63
+#define CAM_CC_MCLK1_CLK_SRC					64
+#define CAM_CC_MCLK2_CLK					65
+#define CAM_CC_MCLK2_CLK_SRC					66
+#define CAM_CC_MCLK3_CLK					67
+#define CAM_CC_MCLK3_CLK_SRC					68
+#define CAM_CC_MCLK4_CLK					69
+#define CAM_CC_MCLK4_CLK_SRC					70
+#define CAM_CC_MCLK5_CLK					71
+#define CAM_CC_MCLK5_CLK_SRC					72
+#define CAM_CC_MCLK6_CLK					73
+#define CAM_CC_MCLK6_CLK_SRC					74
+#define CAM_CC_MCLK7_CLK					75
+#define CAM_CC_MCLK7_CLK_SRC					76
+#define CAM_CC_PLL0						77
+#define CAM_CC_PLL0_OUT_EVEN					78
+#define CAM_CC_PLL0_OUT_ODD					79
+#define CAM_CC_PLL1						80
+#define CAM_CC_PLL1_OUT_EVEN					81
+#define CAM_CC_PLL2						82
+#define CAM_CC_PLL3						83
+#define CAM_CC_PLL3_OUT_EVEN					84
+#define CAM_CC_PLL4						85
+#define CAM_CC_PLL4_OUT_EVEN					86
+#define CAM_CC_PLL5						87
+#define CAM_CC_PLL5_OUT_EVEN					88
+#define CAM_CC_QDSS_DEBUG_CLK					89
+#define CAM_CC_QDSS_DEBUG_CLK_SRC				90
+#define CAM_CC_QDSS_DEBUG_XO_CLK				91
+#define CAM_CC_SLEEP_CLK					92
+#define CAM_CC_SLEEP_CLK_SRC					93
+#define CAM_CC_SLOW_AHB_CLK_SRC					94
+#define CAM_CC_XO_CLK_SRC					95
+
+/* CAM_CC power domains */
+#define CAM_CC_BPS_GDSC						0
+#define CAM_CC_IFE_0_GDSC					1
+#define CAM_CC_IFE_1_GDSC					2
+#define CAM_CC_IPE_0_GDSC					3
+#define CAM_CC_TITAN_TOP_GDSC					4
+
+/* CAM_CC resets */
+#define CAM_CC_BPS_BCR						0
+#define CAM_CC_ICP_BCR						1
+#define CAM_CC_IFE_0_BCR					2
+#define CAM_CC_IFE_1_BCR					3
+#define CAM_CC_IPE_0_BCR					4
+#define CAM_CC_QDSS_DEBUG_BCR					5
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,glymur-evacc.h b/include/dt-bindings/clock/qcom,glymur-evacc.h
new file mode 100644
index 0000000000000..35a7b45503516
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,glymur-evacc.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_EVACC_GLYMUR_H
+#define _DT_BINDINGS_CLK_QCOM_EVACC_GLYMUR_H
+
+/* EVA_CC clocks */
+#define EVA_CC_AHB_CLK					0
+#define EVA_CC_AHB_CLK_SRC				1
+#define EVA_CC_MVS0_CLK					2
+#define EVA_CC_MVS0_CLK_SRC				3
+#define EVA_CC_MVS0_DIV_CLK_SRC				4
+#define EVA_CC_MVS0_FREERUN_CLK				5
+#define EVA_CC_MVS0_SHIFT_CLK				6
+#define EVA_CC_MVS0C_CLK				7
+#define EVA_CC_MVS0C_DIV2_DIV_CLK_SRC			8
+#define EVA_CC_MVS0C_FREERUN_CLK			9
+#define EVA_CC_MVS0C_SHIFT_CLK				10
+#define EVA_CC_PLL0					11
+#define EVA_CC_SLEEP_CLK				12
+#define EVA_CC_SLEEP_CLK_SRC				13
+#define EVA_CC_XO_CLK					14
+#define EVA_CC_XO_CLK_SRC				15
+
+/* EVA_CC power domains */
+#define EVA_CC_MVS0_GDSC				0
+#define EVA_CC_MVS0C_GDSC				1
+
+/* EVA_CC resets */
+#define EVA_CC_INTERFACE_BCR				0
+#define EVA_CC_MVS0_BCR					1
+#define EVA_CC_MVS0C_CLK_ARES				2
+#define EVA_CC_MVS0C_BCR				3
+#define EVA_CC_MVS0C_FREERUN_CLK_ARES			4
+
+#endif /* _DT_BINDINGS_CLK_QCOM_EVACC_GLYMUR_H */
diff --git a/include/dt-bindings/clock/qcom,hawi-videocc.h b/include/dt-bindings/clock/qcom,hawi-videocc.h
new file mode 100644
index 0000000000000..8c97079ff1a75
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,hawi-videocc.h
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_VIDEO_CC_HAWI_H
+#define _DT_BINDINGS_CLK_QCOM_VIDEO_CC_HAWI_H
+
+/* VIDEO_CC clocks */
+#define VIDEO_CC_AHB_CLK					0
+#define VIDEO_CC_AHB_CLK_SRC					1
+#define VIDEO_CC_CX_AXI0_CLK					2
+#define VIDEO_CC_CX_DBGCH_XO_CLK				3
+#define VIDEO_CC_CX_XO_CLK					4
+#define VIDEO_CC_DBGCH_XO_CLK					5
+#define VIDEO_CC_MVS0_CLK					6
+#define VIDEO_CC_MVS0_CLK_SRC					7
+#define VIDEO_CC_MVS0_SHIFT_CLK					8
+#define VIDEO_CC_MVS0_VPP0_CLK					9
+#define VIDEO_CC_MVS0_VPP0_VPP1_GATING_CLK			10
+#define VIDEO_CC_MVS0_VPP1_CLK					11
+#define VIDEO_CC_MVS0A_CLK					12
+#define VIDEO_CC_MVS0A_CLK_SRC					13
+#define VIDEO_CC_MVS0B_CLK					14
+#define VIDEO_CC_MVS0B_CLK_SRC					15
+#define VIDEO_CC_MVS0C_CLK					16
+#define VIDEO_CC_MVS0C_CLK_SRC					17
+#define VIDEO_CC_MVS0C_CTL_FREERUN_CLK				18
+#define VIDEO_CC_MVS0C_DEBUG_CLK				19
+#define VIDEO_CC_MVS0C_FREERUN_CLK				20
+#define VIDEO_CC_MVS0C_SHIFT_CLK				21
+#define VIDEO_CC_PLL0						22
+#define VIDEO_CC_PLL0_OUT_EVEN					23
+#define VIDEO_CC_PLL1						24
+#define VIDEO_CC_PLL2						25
+#define VIDEO_CC_PLL3						26
+#define VIDEO_CC_SLEEP_CLK					27
+#define VIDEO_CC_XO_CLK						28
+#define VIDEO_CC_XO_CLK_SRC					29
+
+/* VIDEO_CC power domains */
+#define VIDEO_CC_AXI0_CX_INT_GDSC				0
+#define VIDEO_CC_MM_INT_GDSC					1
+#define VIDEO_CC_MVS0_GDSC					2
+#define VIDEO_CC_MVS0_VPP0_GDSC					3
+#define VIDEO_CC_MVS0_VPP1_GDSC					4
+#define VIDEO_CC_MVS0A_GDSC					5
+#define VIDEO_CC_MVS0C_GDSC					6
+
+/* VIDEO_CC resets */
+#define VIDEO_CC_AXI0_CX_INT_BCR				0
+#define VIDEO_CC_INTERFACE_BCR					1
+#define VIDEO_CC_MM_INT_BCR					2
+#define VIDEO_CC_MVS0_BCR					3
+#define VIDEO_CC_MVS0_VPP0_BCR					4
+#define VIDEO_CC_MVS0_VPP1_BCR					5
+#define VIDEO_CC_MVS0A_BCR					6
+#define VIDEO_CC_MVS0C_CLK_ARES					7
+#define VIDEO_CC_MVS0C_BCR					8
+#define VIDEO_CC_MVS0C_CTL_FREERUN_CLK_ARES			9
+#define VIDEO_CC_MVS0C_FREERUN_CLK_ARES				10
+#define VIDEO_CC_XO_CLK_ARES					11
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,ipq9650-gcc.h b/include/dt-bindings/clock/qcom,ipq9650-gcc.h
index afd17c00d96ee..2d43ca36c09cc 100644
--- a/include/dt-bindings/clock/qcom,ipq9650-gcc.h
+++ b/include/dt-bindings/clock/qcom,ipq9650-gcc.h
@@ -169,4 +169,9 @@
 #define GPLL2							160
 #define GPLL2_OUT_MAIN						161
 #define GPLL4							162
+#define GCC_REFGEN_CORE_CLK_SRC					163
+#define GCC_REFGEN_PCIE_CORE_CLK				164
+#define GCC_REFGEN_PCIE_HCLK					165
+#define GCC_REFGEN_CMN_UPHY_HCLK				166
+#define GCC_REFGEN_CMN_UPHY_CORE_CLK				167
 #endif
diff --git a/include/dt-bindings/clock/qcom,shikra-gcc.h b/include/dt-bindings/clock/qcom,shikra-gcc.h
new file mode 100644
index 0000000000000..656c959c7e125
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,shikra-gcc.h
@@ -0,0 +1,263 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_SHIKRA_H
+#define _DT_BINDINGS_CLK_QCOM_GCC_SHIKRA_H
+
+/* GCC clocks */
+#define GPLL0							0
+#define GPLL0_OUT_AUX2						1
+#define GPLL1							2
+#define GPLL10							3
+#define GPLL11							4
+#define GPLL12							5
+#define GPLL12_OUT_AUX2						6
+#define GPLL3							7
+#define GPLL3_OUT_MAIN						8
+#define GPLL4							9
+#define GPLL5							10
+#define GPLL6							11
+#define GPLL6_OUT_MAIN						12
+#define GPLL7							13
+#define GPLL8							14
+#define GPLL8_OUT_MAIN						15
+#define GPLL9							16
+#define GPLL9_OUT_MAIN						17
+#define GCC_AHB2PHY_CSI_CLK					18
+#define GCC_AHB2PHY_USB_CLK					19
+#define GCC_BOOT_ROM_AHB_CLK					20
+#define GCC_CAM_THROTTLE_NRT_CLK				21
+#define GCC_CAM_THROTTLE_RT_CLK					22
+#define GCC_CAMERA_AHB_CLK					23
+#define GCC_CAMERA_XO_CLK					24
+#define GCC_CAMSS_AXI_CLK					25
+#define GCC_CAMSS_AXI_CLK_SRC					26
+#define GCC_CAMSS_CAMNOC_ATB_CLK				27
+#define GCC_CAMSS_CAMNOC_DRAGONLINK_ATB_CLK			28
+#define GCC_CAMSS_CAMNOC_NTS_XO_CLK				29
+#define GCC_CAMSS_CCI_0_CLK					30
+#define GCC_CAMSS_CCI_CLK_SRC					31
+#define GCC_CAMSS_CPHY_0_CLK					32
+#define GCC_CAMSS_CPHY_1_CLK					33
+#define GCC_CAMSS_CSI0PHYTIMER_CLK				34
+#define GCC_CAMSS_CSI0PHYTIMER_CLK_SRC				35
+#define GCC_CAMSS_CSI1PHYTIMER_CLK				36
+#define GCC_CAMSS_CSI1PHYTIMER_CLK_SRC				37
+#define GCC_CAMSS_MCLK0_CLK					38
+#define GCC_CAMSS_MCLK0_CLK_SRC					39
+#define GCC_CAMSS_MCLK1_CLK					40
+#define GCC_CAMSS_MCLK1_CLK_SRC					41
+#define GCC_CAMSS_MCLK2_CLK					42
+#define GCC_CAMSS_MCLK2_CLK_SRC					43
+#define GCC_CAMSS_MCLK3_CLK					44
+#define GCC_CAMSS_MCLK3_CLK_SRC					45
+#define GCC_CAMSS_NRT_AXI_CLK					46
+#define GCC_CAMSS_OPE_AHB_CLK					47
+#define GCC_CAMSS_OPE_AHB_CLK_SRC				48
+#define GCC_CAMSS_OPE_CLK					49
+#define GCC_CAMSS_OPE_CLK_SRC					50
+#define GCC_CAMSS_RT_AXI_CLK					51
+#define GCC_CAMSS_TFE_0_CLK					52
+#define GCC_CAMSS_TFE_0_CLK_SRC					53
+#define GCC_CAMSS_TFE_0_CPHY_RX_CLK				54
+#define GCC_CAMSS_TFE_0_CSID_CLK				55
+#define GCC_CAMSS_TFE_0_CSID_CLK_SRC				56
+#define GCC_CAMSS_TFE_1_CLK					57
+#define GCC_CAMSS_TFE_1_CLK_SRC					58
+#define GCC_CAMSS_TFE_1_CPHY_RX_CLK				59
+#define GCC_CAMSS_TFE_1_CSID_CLK				60
+#define GCC_CAMSS_TFE_1_CSID_CLK_SRC				61
+#define GCC_CAMSS_TFE_CPHY_RX_CLK_SRC				62
+#define GCC_CAMSS_TOP_AHB_CLK					63
+#define GCC_CAMSS_TOP_AHB_CLK_SRC				64
+#define GCC_CFG_NOC_USB2_PRIM_AXI_CLK				65
+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK				66
+#define GCC_DDRSS_GPU_AXI_CLK					67
+#define GCC_DDRSS_MEMNOC_PCIE_SF_CLK				68
+#define GCC_DISP_AHB_CLK					69
+#define GCC_DISP_GPLL0_CLK_SRC					70
+#define GCC_DISP_GPLL0_DIV_CLK_SRC				71
+#define GCC_DISP_HF_AXI_CLK					72
+#define GCC_DISP_THROTTLE_CORE_CLK				73
+#define GCC_DISP_XO_CLK						74
+#define GCC_EMAC0_AHB_CLK					75
+#define GCC_EMAC0_AXI_CLK					76
+#define GCC_EMAC0_AXI_CLK_SRC					77
+#define GCC_EMAC0_AXI_SYS_NOC_CLK				78
+#define GCC_EMAC0_CC_SGMIIPHY_RX_CLK				79
+#define GCC_EMAC0_CC_SGMIIPHY_RX_CLK_SRC			80
+#define GCC_EMAC0_CC_SGMIIPHY_TX_CLK				81
+#define GCC_EMAC0_CC_SGMIIPHY_TX_CLK_SRC			82
+#define GCC_EMAC0_PHY_AUX_CLK					83
+#define GCC_EMAC0_PHY_AUX_CLK_SRC				84
+#define GCC_EMAC0_PTP_CLK					85
+#define GCC_EMAC0_PTP_CLK_SRC					86
+#define GCC_EMAC0_RGMII_CLK					87
+#define GCC_EMAC0_RGMII_CLK_SRC					88
+#define GCC_EMAC1_AHB_CLK					89
+#define GCC_EMAC1_AXI_CLK					90
+#define GCC_EMAC1_AXI_CLK_SRC					91
+#define GCC_EMAC1_AXI_SYS_NOC_CLK				92
+#define GCC_EMAC1_CC_SGMIIPHY_RX_CLK				93
+#define GCC_EMAC1_CC_SGMIIPHY_RX_CLK_SRC			94
+#define GCC_EMAC1_CC_SGMIIPHY_TX_CLK				95
+#define GCC_EMAC1_CC_SGMIIPHY_TX_CLK_SRC			96
+#define GCC_EMAC1_PHY_AUX_CLK					97
+#define GCC_EMAC1_PHY_AUX_CLK_SRC				98
+#define GCC_EMAC1_PTP_CLK					99
+#define GCC_EMAC1_PTP_CLK_SRC					100
+#define GCC_EMAC1_RGMII_CLK					101
+#define GCC_EMAC1_RGMII_CLK_SRC					102
+#define GCC_GP1_CLK						103
+#define GCC_GP1_CLK_SRC						104
+#define GCC_GP2_CLK						105
+#define GCC_GP2_CLK_SRC						106
+#define GCC_GP3_CLK						107
+#define GCC_GP3_CLK_SRC						108
+#define GCC_GPU_CFG_AHB_CLK					109
+#define GCC_GPU_GPLL0_CLK_SRC					110
+#define GCC_GPU_GPLL0_DIV_CLK_SRC				111
+#define GCC_GPU_IREF_CLK					112
+#define GCC_GPU_MEMNOC_GFX_CLK					113
+#define GCC_GPU_SMMU_VOTE_CLK					114
+#define GCC_GPU_SNOC_DVM_GFX_CLK				115
+#define GCC_GPU_THROTTLE_CORE_CLK				116
+#define GCC_LPASS_CONFIG_CLK					117
+#define GCC_LPASS_CORE_AXIM_CLK					118
+#define GCC_MMU_TCU_VOTE_CLK					119
+#define GCC_PCIE_AUX_CLK					120
+#define GCC_PCIE_AUX_CLK_SRC					121
+#define GCC_PCIE_AUX_PHY_CLK_SRC				122
+#define GCC_PCIE_CFG_AHB_CLK					123
+#define GCC_PCIE_CLKREF_EN					124
+#define GCC_PCIE_MSTR_AXI_CLK					125
+#define GCC_PCIE_PIPE_CLK					126
+#define GCC_PCIE_PIPE_CLK_SRC					127
+#define GCC_PCIE_RCHNG_PHY_CLK					128
+#define GCC_PCIE_RCHNG_PHY_CLK_SRC				129
+#define GCC_PCIE_SLEEP_CLK					130
+#define GCC_PCIE_SLV_AXI_CLK					131
+#define GCC_PCIE_SLV_Q2A_AXI_CLK				132
+#define GCC_PCIE_TBU_CLK					133
+#define GCC_PCIE_THROTTLE_CORE_CLK				134
+#define GCC_PCIE_THROTTLE_XO_CLK				135
+#define GCC_PCIE_TILE_AXI_SYS_NOC_CLK				136
+#define GCC_PDM2_CLK						137
+#define GCC_PDM2_CLK_SRC					138
+#define GCC_PDM_AHB_CLK						139
+#define GCC_PDM_XO4_CLK						140
+#define GCC_PWM0_XO512_CLK					141
+#define GCC_QMIP_CAMERA_NRT_AHB_CLK				142
+#define GCC_QMIP_CAMERA_RT_AHB_CLK				143
+#define GCC_QMIP_DISP_AHB_CLK					144
+#define GCC_QMIP_GPU_CFG_AHB_CLK				145
+#define GCC_QMIP_PCIE_CFG_AHB_CLK				146
+#define GCC_QMIP_VIDEO_VCODEC_AHB_CLK				147
+#define GCC_QUPV3_WRAP0_CORE_2X_CLK				148
+#define GCC_QUPV3_WRAP0_CORE_CLK				149
+#define GCC_QUPV3_WRAP0_S0_CLK					150
+#define GCC_QUPV3_WRAP0_S0_CLK_SRC				151
+#define GCC_QUPV3_WRAP0_S1_CLK					152
+#define GCC_QUPV3_WRAP0_S1_CLK_SRC				153
+#define GCC_QUPV3_WRAP0_S2_CLK					154
+#define GCC_QUPV3_WRAP0_S2_CLK_SRC				155
+#define GCC_QUPV3_WRAP0_S3_CLK					156
+#define GCC_QUPV3_WRAP0_S3_CLK_SRC				157
+#define GCC_QUPV3_WRAP0_S4_CLK					158
+#define GCC_QUPV3_WRAP0_S4_CLK_SRC				159
+#define GCC_QUPV3_WRAP0_S5_CLK					160
+#define GCC_QUPV3_WRAP0_S5_CLK_SRC				161
+#define GCC_QUPV3_WRAP0_S6_CLK					162
+#define GCC_QUPV3_WRAP0_S6_CLK_SRC				163
+#define GCC_QUPV3_WRAP0_S7_CLK					164
+#define GCC_QUPV3_WRAP0_S7_CLK_SRC				165
+#define GCC_QUPV3_WRAP0_S8_CLK					166
+#define GCC_QUPV3_WRAP0_S8_CLK_SRC				167
+#define GCC_QUPV3_WRAP0_S9_CLK					168
+#define GCC_QUPV3_WRAP0_S9_CLK_SRC				169
+#define GCC_QUPV3_WRAP_0_M_AHB_CLK				170
+#define GCC_QUPV3_WRAP_0_S_AHB_CLK				171
+#define GCC_SDCC1_AHB_CLK					172
+#define GCC_SDCC1_APPS_CLK					173
+#define GCC_SDCC1_APPS_CLK_SRC					174
+#define GCC_SDCC1_ICE_CORE_CLK					175
+#define GCC_SDCC1_ICE_CORE_CLK_SRC				176
+#define GCC_SDCC2_AHB_CLK					177
+#define GCC_SDCC2_APPS_CLK					178
+#define GCC_SDCC2_APPS_CLK_SRC					179
+#define GCC_SYS_NOC_CPUSS_AHB_CLK				180
+#define GCC_SYS_NOC_USB2_PRIM_AXI_CLK				181
+#define GCC_SYS_NOC_USB3_PRIM_AXI_CLK				182
+#define GCC_TSCSS_AHB_CLK					183
+#define GCC_TSCSS_CLK_SRC					184
+#define GCC_TSCSS_CNTR_CLK					185
+#define GCC_TSCSS_ETU_CLK					186
+#define GCC_UFS_CLKREF_EN					187
+#define GCC_USB20_MASTER_CLK					188
+#define GCC_USB20_MASTER_CLK_SRC				189
+#define GCC_USB20_MOCK_UTMI_CLK					190
+#define GCC_USB20_MOCK_UTMI_CLK_SRC				191
+#define GCC_USB20_MOCK_UTMI_POSTDIV_CLK_SRC			192
+#define GCC_USB20_SLEEP_CLK					193
+#define GCC_USB30_PRIM_MASTER_CLK				194
+#define GCC_USB30_PRIM_MASTER_CLK_SRC				195
+#define GCC_USB30_PRIM_MOCK_UTMI_CLK				196
+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC			197
+#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC		198
+#define GCC_USB30_PRIM_SLEEP_CLK				199
+#define GCC_USB3_PRIM_CLKREF_EN					200
+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC				201
+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK				202
+#define GCC_USB3_PRIM_PHY_PIPE_CLK				203
+#define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC				204
+#define GCC_VCODEC0_AXI_CLK					205
+#define GCC_VENUS_AHB_CLK					206
+#define GCC_VENUS_CTL_AXI_CLK					207
+#define GCC_VIDEO_AHB_CLK					208
+#define GCC_VIDEO_AXI0_CLK					209
+#define GCC_VIDEO_THROTTLE_CORE_CLK				210
+#define GCC_VIDEO_VCODEC0_SYS_CLK				211
+#define GCC_VIDEO_VENUS_CLK_SRC					212
+#define GCC_VIDEO_VENUS_CTL_CLK					213
+#define GCC_VIDEO_XO_CLK					214
+
+/* GCC power domains */
+#define GCC_CAMSS_TOP_GDSC					0
+#define GCC_EMAC0_GDSC						1
+#define GCC_EMAC1_GDSC						2
+#define GCC_PCIE_GDSC						3
+#define GCC_USB20_GDSC						4
+#define GCC_USB30_PRIM_GDSC					5
+#define GCC_VCODEC0_GDSC					6
+#define GCC_VENUS_GDSC						7
+
+/* GCC resets */
+#define GCC_CAMSS_OPE_BCR					0
+#define GCC_CAMSS_TFE_BCR					1
+#define GCC_CAMSS_TOP_BCR					2
+#define GCC_EMAC0_BCR						3
+#define GCC_EMAC1_BCR						4
+#define GCC_GPU_BCR						5
+#define GCC_MMSS_BCR						6
+#define GCC_PCIE_BCR						7
+#define GCC_PCIE_PHY_BCR					8
+#define GCC_PDM_BCR						9
+#define GCC_QUPV3_WRAPPER_0_BCR					10
+#define GCC_QUSB2PHY_PRIM_BCR					11
+#define GCC_QUSB2PHY_SEC_BCR					12
+#define GCC_SDCC1_BCR						13
+#define GCC_SDCC2_BCR						14
+#define GCC_TSCSS_BCR						15
+#define GCC_USB20_BCR						16
+#define GCC_USB30_PRIM_BCR					17
+#define GCC_USB3PHY_PHY_PRIM_SP0_BCR				18
+#define GCC_USB3_DP_PHY_PRIM_BCR				19
+#define GCC_USB3_PHY_PRIM_SP0_BCR				20
+#define GCC_USB_PHY_CFG_AHB2PHY_BCR				21
+#define GCC_VCODEC0_BCR						22
+#define GCC_VENUS_BCR						23
+#define GCC_VIDEO_INTERFACE_BCR					24
+
+#endif
diff --git a/include/dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h b/include/dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h
index c4863e444458f..aa47685f329af 100644
--- a/include/dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h
+++ b/include/dt-bindings/clock/renesas,r9a09g077-cpg-mssr.h
@@ -34,5 +34,7 @@
 #define R9A09G077_XSPI_CLK0		22
 #define R9A09G077_XSPI_CLK1		23
 #define R9A09G077_PCLKCAN		24
+#define R9A09G077_LCDC_CLKD		25
+#define R9A09G077_PCLKRTC		26
 
 #endif /* __DT_BINDINGS_CLOCK_RENESAS_R9A09G077_CPG_H__ */
diff --git a/include/dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h b/include/dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h
index 0d53f1e650776..1c73d0dcef184 100644
--- a/include/dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h
+++ b/include/dt-bindings/clock/renesas,r9a09g087-cpg-mssr.h
@@ -34,5 +34,7 @@
 #define R9A09G087_XSPI_CLK0		22
 #define R9A09G087_XSPI_CLK1		23
 #define R9A09G087_PCLKCAN		24
+#define R9A09G087_LCDC_CLKD		25
+#define R9A09G087_PCLKRTC		26
 
 #endif /* __DT_BINDINGS_CLOCK_RENESAS_R9A09G087_CPG_H__ */
diff --git a/include/dt-bindings/clock/samsung,exynos990.h b/include/dt-bindings/clock/samsung,exynos990.h
index 47540307cb52e..c06f591d9d906 100644
--- a/include/dt-bindings/clock/samsung,exynos990.h
+++ b/include/dt-bindings/clock/samsung,exynos990.h
@@ -434,5 +434,6 @@
 #define CLK_GOUT_PERIS_TMU_TOP_PCLK		17
 #define CLK_GOUT_PERIS_OTP_CON_BIRA_OSCCLK	18
 #define CLK_GOUT_PERIS_OTP_CON_TOP_OSCCLK	19
+#define CLK_GOUT_PERIS_TMU_SUB_PCLK		20
 
 #endif
diff --git a/include/dt-bindings/clock/spacemit,k3-clocks.h b/include/dt-bindings/clock/spacemit,k3-clocks.h
index dfae52547cda4..69978ab0b29f8 100644
--- a/include/dt-bindings/clock/spacemit,k3-clocks.h
+++ b/include/dt-bindings/clock/spacemit,k3-clocks.h
@@ -385,6 +385,7 @@
 #define CLK_APMU_PCIE_PORTC_DBI  91
 #define CLK_APMU_PCIE_PORTD_DBI  92
 #define CLK_APMU_PCIE_PORTE_DBI  93
+#define CLK_APMU_UFS_REFCLK      94
 
 /* DCIU clocks */
 #define CLK_DCIU_HDMA            0
diff --git a/include/dt-bindings/clock/sun6i-rtc.h b/include/dt-bindings/clock/sun6i-rtc.h
index 3bd3aa3d57ce1..5132a393ca4b6 100644
--- a/include/dt-bindings/clock/sun6i-rtc.h
+++ b/include/dt-bindings/clock/sun6i-rtc.h
@@ -6,5 +6,9 @@
 #define CLK_OSC32K		0
 #define CLK_OSC32K_FANOUT	1
 #define CLK_IOSC		2
+#define CLK_HOSC_UFS		8
+#define CLK_HOSC_HDMI		9
+#define CLK_HOSC_SERDES0	10
+#define CLK_HOSC_SERDES1	11
 
 #endif /* _DT_BINDINGS_CLK_SUN6I_RTC_H_ */
diff --git a/include/dt-bindings/iio/adc/mediatek,mt6323-auxadc.h b/include/dt-bindings/iio/adc/mediatek,mt6323-auxadc.h
new file mode 100644
index 0000000000000..6ee9a9ecffc10
--- /dev/null
+++ b/include/dt-bindings/iio/adc/mediatek,mt6323-auxadc.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+
+#ifndef _DT_BINDINGS_MEDIATEK_MT6323_AUXADC_H
+#define _DT_BINDINGS_MEDIATEK_MT6323_AUXADC_H
+
+#define MT6323_AUXADC_BATON2		0
+#define MT6323_AUXADC_CH6		1
+#define MT6323_AUXADC_BAT_TEMP		2
+#define MT6323_AUXADC_CHIP_TEMP		3
+#define MT6323_AUXADC_VCDT		4
+#define MT6323_AUXADC_BATON1		5
+#define MT6323_AUXADC_ISENSE		6
+#define MT6323_AUXADC_BATSNS		7
+#define MT6323_AUXADC_ACCDET		8
+#define MT6323_AUXADC_AUDIO0		9
+#define MT6323_AUXADC_AUDIO1		10
+#define MT6323_AUXADC_AUDIO2		11
+#define MT6323_AUXADC_AUDIO3		12
+#define MT6323_AUXADC_AUDIO4		13
+#define MT6323_AUXADC_AUDIO5		14
+#define MT6323_AUXADC_AUDIO6		15
+#define MT6323_AUXADC_AUDIO7		16
+
+#endif
diff --git a/include/dt-bindings/interconnect/qcom,maili-rpmh.h b/include/dt-bindings/interconnect/qcom,maili-rpmh.h
new file mode 100644
index 0000000000000..ae3e48b14eabb
--- /dev/null
+++ b/include/dt-bindings/interconnect/qcom,maili-rpmh.h
@@ -0,0 +1,171 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_MAILI_H
+#define __DT_BINDINGS_INTERCONNECT_QCOM_MAILI_H
+
+#define MASTER_QSPI_0				0
+#define MASTER_QUP_2				1
+#define MASTER_QUP_3				2
+#define MASTER_QUP_4				3
+#define MASTER_QUP_5				4
+#define MASTER_CRYPTO				5
+#define MASTER_IPA				6
+#define MASTER_QUP_1				7
+#define MASTER_SOCCP_PROC			8
+#define MASTER_QDSS_ETR				9
+#define MASTER_QDSS_ETR_1			10
+#define MASTER_SDCC_2				11
+#define MASTER_SDCC_4				12
+#define MASTER_UFS_MEM				13
+#define MASTER_USB3				14
+#define SLAVE_A1NOC_SNOC			15
+
+#define MASTER_DDR_EFF_VETO			0
+#define MASTER_QUP_CORE_0			1
+#define MASTER_QUP_CORE_1			2
+#define MASTER_QUP_CORE_2			3
+#define MASTER_QUP_CORE_3			4
+#define MASTER_QUP_CORE_4			5
+#define MASTER_QUP_CORE_5			6
+#define SLAVE_DDR_EFF_VETO			7
+#define SLAVE_QUP_CORE_0			8
+#define SLAVE_QUP_CORE_1			9
+#define SLAVE_QUP_CORE_2			10
+#define SLAVE_QUP_CORE_3			11
+#define SLAVE_QUP_CORE_4			12
+#define SLAVE_QUP_CORE_5			13
+
+#define MASTER_GEM_NOC_CNOC			0
+#define MASTER_GEM_NOC_PCIE_SNOC		1
+#define SLAVE_AOSS				2
+#define SLAVE_IPA_CFG				3
+#define SLAVE_IPC_ROUTER_FENCE			4
+#define SLAVE_SOCCP				5
+#define SLAVE_TME_CFG				6
+#define SLAVE_CNOC_CFG				7
+#define SLAVE_DDRSS_CFG				8
+#define SLAVE_IMEM				9
+#define SLAVE_PCIE_0				10
+#define SLAVE_PCIE_1				11
+
+#define MASTER_GIC				0
+#define MASTER_GPU_TCU				1
+#define MASTER_SYS_TCU				2
+#define MASTER_APPSS_PROC			3
+#define MASTER_GFX3D				4
+#define MASTER_LPASS_GEM_NOC			5
+#define MASTER_MSS_PROC				6
+#define MASTER_MNOC_HF_MEM_NOC			7
+#define MASTER_MNOC_SF_MEM_NOC			8
+#define MASTER_COMPUTE_NOC			9
+#define MASTER_ANOC_PCIE_GEM_NOC		10
+#define MASTER_QPACE				11
+#define MASTER_SNOC_SF_MEM_NOC			12
+#define MASTER_WLAN_Q6				13
+#define SLAVE_GEM_NOC_CNOC			14
+#define SLAVE_LLCC				15
+#define SLAVE_MEM_NOC_PCIE_SNOC			16
+
+#define MASTER_LPIAON_NOC_LLCLPI_NOC		0
+#define SLAVE_LPASS_LPI_CC			1
+#define SLAVE_LLCC_ISLAND			2
+#define SLAVE_SERVICE_LLCLPI_NOC		3
+#define SLAVE_SERVICE_LLCLPI_NOC_CHIPCX		4
+
+#define MASTER_LPIAON_NOC			0
+#define SLAVE_LPASS_GEM_NOC			1
+
+#define MASTER_LPASS_LPINOC			0
+#define SLAVE_LPIAON_NOC_LLCLPI_NOC		1
+#define SLAVE_LPIAON_NOC_LPASS_AG_NOC		2
+
+#define MASTER_LPASS_PROC			0
+#define SLAVE_LPICX_NOC_LPIAON_NOC		1
+
+#define MASTER_LLCC				0
+#define MASTER_DDR_RT				1
+#define SLAVE_EBI1				2
+#define SLAVE_DDR_RT				3
+
+#define MASTER_CAMNOC_HF			0
+#define MASTER_CAMNOC_NRT_ICP_SF		1
+#define MASTER_CAMNOC_RT_CDM_SF			2
+#define MASTER_CAMNOC_SF			3
+#define MASTER_MDP				4
+#define MASTER_MDSS_DCP				5
+#define MASTER_CDSP_HCP				6
+#define MASTER_VIDEO_CV_PROC			7
+#define MASTER_VIDEO_EVA			8
+#define MASTER_VIDEO_MVP			9
+#define MASTER_VIDEO_V_PROC			10
+#define SLAVE_MNOC_HF_MEM_NOC			11
+#define SLAVE_MNOC_SF_MEM_NOC			12
+
+#define MASTER_CDSP_PROC			0
+#define SLAVE_CDSP_MEM_NOC			1
+
+#define MASTER_PCIE_ANOC_CFG			0
+#define MASTER_PCIE_0				1
+#define MASTER_PCIE_1				2
+#define SLAVE_ANOC_PCIE_GEM_NOC			3
+#define SLAVE_SERVICE_PCIE_ANOC			4
+
+#define MASTER_CFG_CENTER			0
+#define MASTER_CFG_EAST				1
+#define MASTER_CFG_MM_HF			2
+#define MASTER_CFG_MM_SF			3
+#define MASTER_CFG_NORTH			4
+#define MASTER_CFG_SOUTH			5
+#define MASTER_CFG_WEST				6
+#define SLAVE_AHB2PHY_SOUTH			7
+#define SLAVE_BOOT_ROM				8
+#define SLAVE_CAMERA_CFG			9
+#define SLAVE_CLK_CTL				10
+#define SLAVE_CRYPTO_CFG			11
+#define SLAVE_DISPLAY_CFG			12
+#define SLAVE_EVA_CFG				13
+#define SLAVE_GFX3D_CFG				14
+#define SLAVE_I2C				15
+#define SLAVE_IMEM_CFG				16
+#define SLAVE_IPC_ROUTER_CFG			17
+#define SLAVE_IRIS_CFG				18
+#define SLAVE_CNOC_MSS				19
+#define SLAVE_PCIE_0_CFG			20
+#define SLAVE_PCIE_1_CFG			21
+#define SLAVE_PRNG				22
+#define SLAVE_QSPI_0				23
+#define SLAVE_QUP_1				24
+#define SLAVE_QUP_2				25
+#define SLAVE_QUP_3				26
+#define SLAVE_QUP_4				27
+#define SLAVE_QUP_5				28
+#define SLAVE_SDCC_2				29
+#define SLAVE_SDCC_4				30
+#define SLAVE_TLMM				31
+#define SLAVE_UFS_MEM_CFG			32
+#define SLAVE_USB3				33
+#define SLAVE_VSENSE_CTRL_CFG			34
+#define SLAVE_PCIE_ANOC_CFG			35
+#define SLAVE_QDSS_CFG				36
+#define SLAVE_QDSS_STM				37
+#define SLAVE_TCSR				38
+#define SLAVE_TCU				39
+
+#define MASTER_CNOC_STARDUST			0
+#define SLAVE_STARDUST_CENTER_CFG		1
+#define SLAVE_STARDUST_EAST_CFG			2
+#define SLAVE_STARDUST_MM_HF_CFG		3
+#define SLAVE_STARDUST_MM_SF_CFG		4
+#define SLAVE_STARDUST_NORTH_CFG		5
+#define SLAVE_STARDUST_SOUTH_CFG		6
+#define SLAVE_STARDUST_WEST_CFG			7
+
+#define MASTER_A1NOC_SNOC			0
+#define MASTER_APSS_NOC				1
+#define MASTER_CNOC_SNOC			2
+#define SLAVE_SNOC_GEM_NOC_SF			3
+
+#endif
diff --git a/include/dt-bindings/power/fsl,imx93-power.h b/include/dt-bindings/power/fsl,imx93-power.h
index 17f9f015bf7df..071221fe5c57d 100644
--- a/include/dt-bindings/power/fsl,imx93-power.h
+++ b/include/dt-bindings/power/fsl,imx93-power.h
@@ -11,5 +11,6 @@
 #define IMX93_MEDIABLK_PD_PXP			2
 #define IMX93_MEDIABLK_PD_LCDIF			3
 #define IMX93_MEDIABLK_PD_ISI			4
+#define IMX93_MEDIABLK_PD_MIPI_PHY		5
 
 #endif
diff --git a/include/dt-bindings/power/mediatek,mt8196-power.h b/include/dt-bindings/power/mediatek,mt8196-power.h
index 0f622a93c8076..085790bf81241 100644
--- a/include/dt-bindings/power/mediatek,mt8196-power.h
+++ b/include/dt-bindings/power/mediatek,mt8196-power.h
@@ -30,6 +30,10 @@
 #define MT8196_POWER_DOMAIN_MM_PROC_DORMANT		0
 #define MT8196_POWER_DOMAIN_SSR				1
 
+/* HFRPSYS Multimedia Power Control (MMPC) - Direct Control */
+#define MT8196_POWER_DOMAIN_EDPTX			0
+#define MT8196_POWER_DOMAIN_DPTX			1
+
 /* HFRPSYS MultiMedia Power Control (MMPC) - HW Voter */
 #define MT8196_POWER_DOMAIN_VDE0			0
 #define MT8196_POWER_DOMAIN_VDE1			1
diff --git a/include/dt-bindings/reset/altr,rst-mgr-s10.h b/include/dt-bindings/reset/altr,rst-mgr-s10.h
index 04c4d0c6fd34b..c2505b9eb63e5 100644
--- a/include/dt-bindings/reset/altr,rst-mgr-s10.h
+++ b/include/dt-bindings/reset/altr,rst-mgr-s10.h
@@ -22,7 +22,7 @@
 #define USB0_RESET		35
 #define USB1_RESET		36
 #define NAND_RESET		37
-/* 38 is empty */
+#define COMBOPHY_RESET		38
 #define SDMMC_RESET		39
 #define EMAC0_OCP_RESET		40
 #define EMAC1_OCP_RESET		41
diff --git a/include/kunit/fwnode.h b/include/kunit/fwnode.h
new file mode 100644
index 0000000000000..e250bb87e144c
--- /dev/null
+++ b/include/kunit/fwnode.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * KUnit resource management helpers for firmware nodes.
+ *
+ * Copyright (C) Qualcomm Technologies, Inc. and/or its subsidiaries
+ */
+
+#ifndef _KUNIT_FWNODE_H
+#define _KUNIT_FWNODE_H
+
+struct device;
+struct fwnode_handle;
+struct kunit;
+struct property_entry;
+struct software_node;
+
+struct fwnode_handle *
+kunit_fwnode_create_software_node(struct kunit *test,
+				  const struct property_entry *properties,
+				  const struct fwnode_handle *parent);
+struct fwnode_handle *
+kunit_software_node_register(struct kunit *test,
+			     const struct software_node *node);
+int kunit_software_node_register_node_group(struct kunit *test,
+					    const struct software_node *const *nodes);
+int kunit_device_add_software_node(struct kunit *test, struct device *dev,
+				   const struct software_node *node);
+
+#endif /* _KUNIT_FWNODE_H */
diff --git a/include/kunit/test.h b/include/kunit/test.h
index e52452e58305a..da5312e0dfa57 100644
--- a/include/kunit/test.h
+++ b/include/kunit/test.h
@@ -285,6 +285,7 @@ struct kunit_suite {
 	struct string_stream *log;
 	int suite_init_err;
 	bool is_init;
+	enum kunit_status status;
 };
 
 /* Stores an array of suites, end points one past the end */
diff --git a/include/linux/acpi.h b/include/linux/acpi.h
index 60ab50cb8930c..8d20a25f1a62f 100644
--- a/include/linux/acpi.h
+++ b/include/linux/acpi.h
@@ -94,6 +94,12 @@ static inline void acpi_preset_companion(struct device *dev,
 	ACPI_COMPANION_SET(dev, acpi_find_child_device(parent, addr, false));
 }
 
+static inline void acpi_device_clear_deps(struct device *dev)
+{
+	if (has_acpi_companion(dev))
+		acpi_dev_clear_dependencies(ACPI_COMPANION(dev));
+}
+
 static inline const char *acpi_dev_name(struct acpi_device *adev)
 {
 	return dev_name(&adev->dev);
@@ -315,6 +321,19 @@ static inline int acpi_processor_evaluate_cst(acpi_handle handle, u32 cpu,
 }
 #endif
 
+#ifdef CONFIG_ACPI_PROCESSOR_IDLE
+int acpi_processor_extract_lpi_info(acpi_handle pr_handle,
+				    struct acpi_processor_power *pr_power,
+				    bool strict);
+#else
+static inline int acpi_processor_extract_lpi_info(acpi_handle pr_handle,
+				    struct acpi_processor_power *pr_power,
+				    bool strict)
+{
+	return -ENODEV;
+}
+#endif
+
 #ifdef CONFIG_ACPI_HOTPLUG_CPU
 /* Arch dependent functions for cpu hotplug support */
 int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, u32 acpi_id,
@@ -360,9 +379,10 @@ int acpi_gsi_to_irq (u32 gsi, unsigned int *irq);
 int acpi_isa_irq_to_gsi (unsigned isa_irq, u32 *gsi);
 
 typedef struct fwnode_handle *(*acpi_gsi_domain_disp_fn)(u32);
+typedef acpi_handle (*acpi_gsi_handle_disp_fn)(u32);
 
 void acpi_set_irq_model(enum acpi_irq_model_id model,
-			acpi_gsi_domain_disp_fn fn);
+			acpi_gsi_domain_disp_fn fn, acpi_gsi_handle_disp_fn gsi_dep_fn);
 acpi_gsi_domain_disp_fn acpi_get_gsi_dispatcher(void);
 void acpi_set_gsi_to_irq_fallback(u32 (*)(u32));
 
@@ -372,6 +392,8 @@ struct irq_domain *acpi_irq_create_hierarchy(unsigned int flags,
 					     const struct irq_domain_ops *ops,
 					     void *host_data);
 
+u32 acpi_irq_add_auto_dep(acpi_handle handle);
+
 #ifdef CONFIG_X86_IO_APIC
 extern int acpi_get_override_irq(u32 gsi, int *trigger, int *polarity);
 #else
@@ -908,6 +930,8 @@ static inline void acpi_preset_companion(struct device *dev,
 {
 }
 
+static inline void acpi_device_clear_deps(struct device *dev) {}
+
 static inline const char *acpi_dev_name(struct acpi_device *adev)
 {
 	return NULL;
diff --git a/include/linux/acpi_iort.h b/include/linux/acpi_iort.h
index 17bb3374f4ca7..931eaa7bbf6a5 100644
--- a/include/linux/acpi_iort.h
+++ b/include/linux/acpi_iort.h
@@ -27,7 +27,8 @@ int iort_register_domain_token(int trans_id, phys_addr_t base,
 			       struct fwnode_handle *fw_node);
 void iort_deregister_domain_token(int trans_id);
 struct fwnode_handle *iort_find_domain_token(int trans_id);
-struct fwnode_handle *iort_iwb_handle(u32 iwb_id);
+acpi_handle iort_iwb_handle(u32 iwb_id);
+struct fwnode_handle *iort_iwb_handle_fwnode(u32 iwb_id);
 
 #ifdef CONFIG_ACPI_IORT
 u32 iort_msi_map_id(struct device *dev, u32 id);
diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h
index 068ba2e77c5d6..7f2d80a597924 100644
--- a/include/linux/alloc_tag.h
+++ b/include/linux/alloc_tag.h
@@ -134,6 +134,8 @@ static inline bool mem_alloc_profiling_enabled(void)
 				   &mem_alloc_profiling_key);
 }
 
+bool mem_alloc_profiling_permanently_disabled(void);
+
 static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag)
 {
 	struct alloc_tag_counters v = { 0, 0 };
@@ -163,11 +165,11 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref)
 {
 	WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n");
 }
-void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags);
+void alloc_tag_add_early_pfn(unsigned long pfn, unsigned int alloc_flags);
 #else
 static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {}
 static inline void alloc_tag_sub_check(union codetag_ref *ref) {}
-static inline void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags) {}
+static inline void alloc_tag_add_early_pfn(unsigned long pfn, unsigned int alloc_flags) {}
 #endif
 
 /* Caller should verify both ref and tag to be valid */
@@ -239,6 +241,7 @@ static inline bool alloc_tag_is_inaccurate(struct alloc_tag *tag)
 
 #define DEFINE_ALLOC_TAG(_alloc_tag)
 static inline bool mem_alloc_profiling_enabled(void) { return false; }
+static inline bool mem_alloc_profiling_permanently_disabled(void) { return true; }
 static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag,
 				 size_t bytes) {}
 static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {}
diff --git a/include/linux/amd-pmf-io.h b/include/linux/amd-pmf-io.h
index 55198d2875cc0..e014d4ce5a204 100644
--- a/include/linux/amd-pmf-io.h
+++ b/include/linux/amd-pmf-io.h
@@ -52,15 +52,6 @@ struct amd_sfh_info {
 	u32 laptop_placement;
 };
 
-enum laptop_placement {
-	LP_UNKNOWN = 0,
-	ON_TABLE,
-	ON_LAP_MOTION,
-	IN_BAG,
-	OUT_OF_BAG,
-	LP_UNDEFINED,
-};
-
 /**
  * struct amd_pmf_npu_metrics: Get NPU metrics data from PMF driver
  * @npuclk_freq: NPU clock frequency [MHz]
diff --git a/include/linux/arm_ffa.h b/include/linux/arm_ffa.h
index 17eca3dfc59e2..e71d83ee0aef3 100644
--- a/include/linux/arm_ffa.h
+++ b/include/linux/arm_ffa.h
@@ -421,6 +421,13 @@ struct ffa_mem_region {
 #define FFA_EMAD_HAS_IMPDEF_FIELD(version)	((version) >= FFA_VERSION_1_2)
 #define FFA_MEM_REGION_HAS_EP_MEM_OFFSET(version) ((version) > FFA_VERSION_1_0)
 
+/* The layout changed from FFA_VERSION_1_0 and the region includes an
+ * ep_mem_offset.
+ */
+#define FFA_MEM_REGION_SZ(version)		(!FFA_MEM_REGION_HAS_EP_MEM_OFFSET((version)) ?\
+						 offsetof(struct ffa_mem_region, ep_mem_offset) :\
+						 sizeof(struct ffa_mem_region))
+
 static inline u32 ffa_emad_size_get(u32 ffa_version)
 {
 	u32 sz;
@@ -445,7 +452,7 @@ ffa_mem_desc_offset(struct ffa_mem_region *buf, int count, u32 ffa_version)
 	if (!FFA_MEM_REGION_HAS_EP_MEM_OFFSET(ffa_version))
 		offset += offsetof(struct ffa_mem_region, ep_mem_offset);
 	else
-		offset += sizeof(struct ffa_mem_region);
+		offset += buf->ep_mem_offset;
 
 	return offset;
 }
diff --git a/include/linux/ata.h b/include/linux/ata.h
index 8fd48bcb2a46c..fc21a2417b250 100644
--- a/include/linux/ata.h
+++ b/include/linux/ata.h
@@ -75,6 +75,7 @@ enum {
 	ATA_ID_HW_CONFIG	= 93,
 	ATA_ID_SPG		= 98,
 	ATA_ID_LBA_CAPACITY_2	= 100,
+	ATA_ID_MAX_PAGES_PER_DSM	= 105,
 	ATA_ID_SECTOR_SIZE	= 106,
 	ATA_ID_WWN		= 108,
 	ATA_ID_LOGICAL_SECTOR_SIZE	= 117,	/* and 118 */
@@ -288,6 +289,10 @@ enum {
 	ATA_CMD_SANITIZE_DEVICE = 0xB4,
 	ATA_CMD_ZAC_MGMT_IN	= 0x4A,
 	ATA_CMD_ZAC_MGMT_OUT	= 0x9F,
+	ATA_CMD_GET_PHYS_ELEMENT_STATUS = 0x12,
+	ATA_CMD_REMOVE_ELEMENT_AND_TRUNCATE = 0x7c,
+	ATA_CMD_RESTORE_ELEMENTS_AND_REBUILD = 0x7d,
+	ATA_CMD_REMOVE_ELEMENT_AND_MODIFY_ZONES = 0x7e,
 
 	/* marked obsolete in the ATA/ATAPI-7 spec */
 	ATA_CMD_RESTORE		= 0x10,
@@ -928,6 +933,18 @@ static inline bool ata_id_has_trim(const u16 *id)
 	return false;
 }
 
+static inline u16 ata_id_dsm_max_pages(const u16 *id)
+{
+	/*
+	 * IDENTIFY DEVICE word 105: MAX PAGES PER DSM COMMAND. Maximum number
+	 * of 512-byte pages of LBA Range Entries the device accepts in a
+	 * single DATA SET MANAGEMENT command. Zero means the device does not
+	 * specify a limit. The field is reserved unless TRIM is supported, so
+	 * callers must gate on ata_id_has_trim().
+	 */
+	return id[ATA_ID_MAX_PAGES_PER_DSM];
+}
+
 static inline bool ata_id_has_zero_after_trim(const u16 *id)
 {
 	/* DSM supported, deterministic read, and read zero after trim set */
diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h
index 5b7d12b40d5e7..c2284466e7aa6 100644
--- a/include/linux/backing-dev.h
+++ b/include/linux/backing-dev.h
@@ -76,8 +76,6 @@ static inline s64 wb_stat_sum(struct bdi_writeback *wb, enum wb_stat_item item)
 	return percpu_counter_sum_positive(&wb->stat[item]);
 }
 
-extern void wb_writeout_inc(struct bdi_writeback *wb);
-
 /*
  * maximal error of a stat counter.
  */
diff --git a/include/linux/bcma/bcma.h b/include/linux/bcma/bcma.h
index f02cb3909375e..17fc50190014e 100644
--- a/include/linux/bcma/bcma.h
+++ b/include/linux/bcma/bcma.h
@@ -486,4 +486,6 @@ extern u32 bcma_core_dma_translation(struct bcma_device *core);
 
 extern unsigned int bcma_core_irq(struct bcma_device *core, int num);
 
+extern const struct software_node bcma_gpio_swnode;
+
 #endif /* LINUX_BCMA_H_ */
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 8f33f717b14f5..3a0e9cb6c2f60 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -256,6 +256,12 @@ static inline struct folio *bio_first_folio_all(struct bio *bio)
 	return page_folio(bio_first_page_all(bio));
 }
 
+static inline struct bio_vec *bio_last_bvec_all(struct bio *bio)
+{
+	WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
+	return &bio->bi_io_vec[bio->bi_vcnt - 1];
+}
+
 /**
  * struct folio_iter - State for iterating all folios in a bio.
  * @folio: The current folio we're iterating.  NULL after the last folio.
diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h
index b007d54a9036a..8854acf77869a 100644
--- a/include/linux/bitmap.h
+++ b/include/linux/bitmap.h
@@ -2,7 +2,7 @@
 #ifndef __LINUX_BITMAP_H
 #define __LINUX_BITMAP_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/align.h>
 #include <linux/bitops.h>
@@ -894,6 +894,6 @@ void bitmap_write(unsigned long *map, unsigned long value,
 #define bitmap_set_value8(map, value, start)		\
 	bitmap_write(map, value, start, BITS_PER_BYTE)
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* __LINUX_BITMAP_H */
diff --git a/include/linux/bits.h b/include/linux/bits.h
index a40cc861b3a7c..b7509f15718e6 100644
--- a/include/linux/bits.h
+++ b/include/linux/bits.h
@@ -17,7 +17,7 @@
  * position @h. For example
  * GENMASK_ULL(39, 21) gives us the 64bit vector 0x000000ffffe00000.
  */
-#if !defined(__ASSEMBLY__)
+#if !defined(__ASSEMBLER__)
 
 /*
  * Missing asm support
@@ -75,7 +75,7 @@
 #define BIT_U32(nr)	BIT_TYPE(u32, nr)
 #define BIT_U64(nr)	BIT_TYPE(u64, nr)
 
-#else /* defined(__ASSEMBLY__) */
+#else /* defined(__ASSEMBLER__) */
 
 /*
  * BUILD_BUG_ON_ZERO is not available in h files included from asm files,
@@ -84,6 +84,6 @@
 #define GENMASK(h, l)		__GENMASK(h, l)
 #define GENMASK_ULL(h, l)	__GENMASK_ULL(h, l)
 
-#endif /* !defined(__ASSEMBLY__) */
+#endif /* !defined(__ASSEMBLER__) */
 
 #endif	/* __LINUX_BITS_H */
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index 8808ee76e73c0..5a725a0cd35f4 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -66,7 +66,7 @@ struct block_device {
 	int			bd_holders;
 	struct kobject		*bd_holder_dir;
 
-	atomic_t		bd_fsfreeze_count; /* number of freeze requests */
+	atomic_t		bd_fsfreeze_count; /* >0 freeze requests, <0 freeze deniers */
 	struct mutex		bd_fsfreeze_mutex; /* serialize freeze/thaw */
 
 	struct partition_meta_info *bd_meta_info;
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 9213a5716f95a..dbb549cdfb778 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -126,8 +126,6 @@ struct blk_integrity {
 	unsigned char				pi_tuple_size;
 };
 
-typedef unsigned int __bitwise blk_mode_t;
-
 /* open for reading */
 #define BLK_OPEN_READ		((__force blk_mode_t)(1 << 0))
 /* open for writing */
@@ -1770,13 +1768,6 @@ struct blk_holder_ops {
 		__releases(&bdev->bd_holder_lock);
 };
 
-/*
- * For filesystems using @fs_holder_ops, the @holder argument passed to
- * helpers used to open and claim block devices via
- * bd_prepare_to_claim() must point to a superblock.
- */
-extern const struct blk_holder_ops fs_holder_ops;
-
 /*
  * Return the correct open flags for blkdev_get_by_* for super block flags
  * as stored in sb->s_flags.
@@ -1837,7 +1828,10 @@ static inline int early_lookup_bdev(const char *pathname, dev_t *dev)
 
 int bdev_freeze(struct block_device *bdev);
 int bdev_thaw(struct block_device *bdev);
+int bdev_deny_freeze(struct block_device *bdev);
+void bdev_allow_freeze(struct block_device *bdev);
 void bdev_fput(struct file *bdev_file);
+void bdev_yield_claim(struct file *bdev_file);
 
 struct io_comp_batch {
 	struct rq_list req_list;
diff --git a/include/linux/bootconfig.h b/include/linux/bootconfig.h
index 1c7f3b74ffcf3..deda507500da2 100644
--- a/include/linux/bootconfig.h
+++ b/include/linux/bootconfig.h
@@ -308,4 +308,18 @@ static inline const char *xbc_get_embedded_bootconfig(size_t *size)
 }
 #endif
 
+/* Bootconfig opt-in detection, shared by setup_arch() and setup_boot_config() */
+#ifdef CONFIG_BOOT_CONFIG
+bool __init bootconfig_cmdline_requested(const char *boot_cmdline, int *end_offset);
+#endif
+
+/* Build-time-rendered bootconfig cmdline prepended in setup_arch() */
+#ifdef CONFIG_CMDLINE_FROM_BOOTCONFIG
+void __init xbc_prepend_embedded_cmdline(char *dst, size_t size);
+bool __init xbc_embedded_cmdline_applied(void);
+#else
+static inline void xbc_prepend_embedded_cmdline(char *dst, size_t size) { }
+static inline bool xbc_embedded_cmdline_applied(void) { return false; }
+#endif
+
 #endif
diff --git a/include/linux/bootmem_info.h b/include/linux/bootmem_info.h
deleted file mode 100644
index f724340755e55..0000000000000
--- a/include/linux/bootmem_info.h
+++ /dev/null
@@ -1,89 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __LINUX_BOOTMEM_INFO_H
-#define __LINUX_BOOTMEM_INFO_H
-
-#include <linux/mm.h>
-#include <linux/kmemleak.h>
-
-/*
- * Types for free bootmem stored in the low bits of page->private.
- */
-enum bootmem_type {
-	MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE = 1,
-	SECTION_INFO = MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE,
-	MIX_SECTION_INFO,
-	NODE_INFO,
-	MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE = NODE_INFO,
-};
-
-#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
-void __init register_page_bootmem_info_node(struct pglist_data *pgdat);
-void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
-				  unsigned long nr_pages);
-
-void get_page_bootmem(unsigned long info, struct page *page,
-		enum bootmem_type type);
-void put_page_bootmem(struct page *page);
-
-static inline enum bootmem_type bootmem_type(const struct page *page)
-{
-	return (unsigned long)page->private & 0xf;
-}
-
-static inline unsigned long bootmem_info(const struct page *page)
-{
-	return (unsigned long)page->private >> 4;
-}
-
-/*
- * Any memory allocated via the memblock allocator and not via the
- * buddy will be marked reserved already in the memmap. For those
- * pages, we can call this function to free it to buddy allocator.
- */
-static inline void free_bootmem_page(struct page *page)
-{
-	enum bootmem_type type = bootmem_type(page);
-
-	VM_BUG_ON_PAGE(page_ref_count(page) != 2, page);
-
-	if (type == SECTION_INFO || type == MIX_SECTION_INFO)
-		put_page_bootmem(page);
-	else
-		VM_BUG_ON_PAGE(1, page);
-}
-#else
-static inline void register_page_bootmem_info_node(struct pglist_data *pgdat)
-{
-}
-
-static inline void register_page_bootmem_memmap(unsigned long section_nr,
-		struct page *map, unsigned long nr_pages)
-{
-}
-
-static inline void put_page_bootmem(struct page *page)
-{
-}
-
-static inline enum bootmem_type bootmem_type(const struct page *page)
-{
-	return SECTION_INFO;
-}
-
-static inline unsigned long bootmem_info(const struct page *page)
-{
-	return 0;
-}
-
-static inline void get_page_bootmem(unsigned long info, struct page *page,
-				    enum bootmem_type type)
-{
-}
-
-static inline void free_bootmem_page(struct page *page)
-{
-	free_reserved_page(page);
-}
-#endif
-
-#endif /* __LINUX_BOOTMEM_INFO_H */
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 7719f65284456..31181e0c2b801 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -299,7 +299,6 @@ struct bpf_map_owner {
 
 struct bpf_map {
 	u8 sha[SHA256_DIGEST_SIZE];
-	u32 excl;
 	const struct bpf_map_ops *ops;
 	struct bpf_map *inner_map_meta;
 #ifdef CONFIG_SECURITY
@@ -570,7 +569,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec,
 
 	if (IS_ERR_OR_NULL(rec)) {
 		if (long_memcpy)
-			bpf_long_memcpy(dst, src, round_up(size, 8));
+			bpf_long_memcpy(dst, src, size);
 		else
 			memcpy(dst, src, size);
 		return;
@@ -593,7 +592,7 @@ static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
 
 static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src)
 {
-	bpf_obj_memcpy(map->record, dst, src, map->value_size, true);
+	bpf_obj_memcpy(map->record, dst, src, round_up(map->value_size, 8), true);
 }
 
 static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src)
@@ -1865,8 +1864,9 @@ struct bpf_prog_aux {
 
 struct bpf_prog {
 	u16			pages;		/* Number of allocated pages */
-	u16			jited:1,	/* Is our filter JIT'ed? */
+	u32			jited:1,	/* Is our filter JIT'ed? */
 				jit_requested:1,/* archs need to JIT the prog */
+				jit_required:1,	/* program strictly requires JIT compiler */
 				gpl_compatible:1, /* Is filter GPL compatible? */
 				cb_access:1,	/* Is control block accessed? */
 				dst_needed:1,	/* Do we need dst entry? */
@@ -3146,7 +3146,7 @@ int btf_struct_access(struct bpf_verifier_log *log,
 bool btf_struct_ids_match(struct bpf_verifier_log *log,
 			  const struct btf *btf, u32 id, int off,
 			  const struct btf *need_btf, u32 need_type_id,
-			  bool strict);
+			  bool strict, bool walk_flex_arrays);
 
 int btf_distill_func_proto(struct bpf_verifier_log *log,
 			   struct btf *btf,
@@ -3170,7 +3170,6 @@ const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id,
 						 const struct bpf_prog *prog);
 void bpf_task_storage_free(struct task_struct *task);
 void bpf_cgrp_storage_free(struct cgroup *cgroup);
-bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
 const struct btf_func_model *
 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
 			 const struct bpf_insn *insn);
@@ -3509,11 +3508,6 @@ static inline void bpf_task_storage_free(struct task_struct *task)
 {
 }
 
-static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
-{
-	return false;
-}
-
 static inline const struct btf_func_model *
 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
 			 const struct bpf_insn *insn)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 39a851e690ec4..317e99b9acc0a 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -898,6 +898,14 @@ struct bpf_scc_info {
 
 struct bpf_liveness;
 
+struct bpf_fd_array {
+	union {
+		struct bpf_map *map;
+		struct btf *btf;
+		unsigned long val;
+	};
+};
+
 /* single container for all structs
  * one verifier_env per bpf_check() call
  */
@@ -939,6 +947,7 @@ struct bpf_verifier_env {
 	bool bypass_spec_v4;
 	bool seen_direct_write;
 	bool seen_exception;
+	bool signature;
 	struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */
 	const struct bpf_line_info *prev_linfo;
 	struct bpf_verifier_log log;
@@ -989,7 +998,19 @@ struct bpf_verifier_env {
 	u32 free_list_size;
 	u32 explored_states_size;
 	u32 num_backedges;
-	bpfptr_t fd_array;
+	/*
+	 * The program's fd_array comes in two shapes, told apart by whether
+	 * the caller passed fd_array_cnt. They are mutually exclusive:
+	 *  - continuous (fd_array_cnt given): ->fd_array holds every entry
+	 *    resolved to its object up front, indexed by fd_array position,
+	 *    with ->fd_array_cnt slots; ->fd_array_raw is unused.
+	 *  - sparse (no fd_array_cnt): ->fd_array is NULL, and entries are
+	 *    read from ->fd_array_raw (the caller's fd_array) and resolved
+	 *    on the spot at each reference.
+	 */
+	struct bpf_fd_array *fd_array;
+	u32 fd_array_cnt;
+	bpfptr_t fd_array_raw;
 
 	/* bit mask to keep track of whether a register has been accessed
 	 * since the last time the function state was printed
@@ -1243,6 +1264,11 @@ static inline void bpf_bt_set_frame_slot(struct backtrack_state *bt, u32 frame,
 	bt->stack_masks[frame] |= 1ull << slot;
 }
 
+static inline void bpf_bt_set_frame_slot_mask(struct backtrack_state *bt, u32 frame, u64 mask)
+{
+	bt->stack_masks[frame] |= mask;
+}
+
 static inline void bt_set_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot)
 {
 	bt->stack_arg_masks[frame] |= 1 << slot;
diff --git a/include/linux/btf.h b/include/linux/btf.h
index 240401d9b25b9..c09b7994de4eb 100644
--- a/include/linux/btf.h
+++ b/include/linux/btf.h
@@ -578,6 +578,7 @@ const char *btf_str_by_offset(const struct btf *btf, u32 offset);
 struct btf *btf_parse_vmlinux(void);
 struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog);
 u32 *btf_kfunc_flags(const struct btf *btf, u32 kfunc_btf_id, const struct bpf_prog *prog);
+int btf_kfunc_check_flag(const struct btf *btf, u32 kfunc_btf_id, u32 flag);
 bool btf_kfunc_is_allowed(const struct btf *btf, u32 kfunc_btf_id, const struct bpf_prog *prog);
 u32 *btf_kfunc_is_modify_return(const struct btf *btf, u32 kfunc_btf_id,
 				const struct bpf_prog *prog);
diff --git a/include/linux/can/dev/peak_canfd.h b/include/linux/can/dev/peak_canfd.h
index d3788a3d0942b..056e0efa649fd 100644
--- a/include/linux/can/dev/peak_canfd.h
+++ b/include/linux/can/dev/peak_canfd.h
@@ -3,7 +3,7 @@
  * CAN driver for PEAK System micro-CAN based adapters
  *
  * Copyright (C) 2003-2025 PEAK System-Technik GmbH
- * Author: Stéphane Grosjean <stephane.grosjean@hms-networks.com>
+ * Author: Stéphane Grosjean <s.grosjean@peak-system.fr>
  */
 #ifndef PUCAN_H
 #define PUCAN_H
diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h
index de2cd6238c2ae..7a631a257613b 100644
--- a/include/linux/cgroup-defs.h
+++ b/include/linux/cgroup-defs.h
@@ -896,7 +896,7 @@ static inline void cgroup_threadgroup_change_begin(struct task_struct *tsk)
  * cgroup_threadgroup_change_end - threadgroup exclusion for cgroups
  * @tsk: target task
  *
- * Counterpart of cgroup_threadcgroup_change_begin().
+ * Counterpart of cgroup_threadgroup_change_begin().
  */
 static inline void cgroup_threadgroup_change_end(struct task_struct *tsk)
 {
diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
index f2aa46a4f871e..b905208942bfc 100644
--- a/include/linux/cgroup.h
+++ b/include/linux/cgroup.h
@@ -480,7 +480,7 @@ static inline void cgroup_unlock(void)
 		rcu_read_lock_sched_held() ||				\
 		lockdep_is_held(&cgroup_mutex) ||			\
 		lockdep_is_held(&css_set_lock) ||			\
-		((task)->flags & PF_EXITING) || (__c))
+		(data_race((task)->flags) & PF_EXITING) || (__c))
 #else
 #define task_css_set_check(task, __c)					\
 	rcu_dereference((task)->cgroups)
diff --git a/include/linux/clk/renesas.h b/include/linux/clk/renesas.h
index 0949400f44de4..c9495558cd5c0 100644
--- a/include/linux/clk/renesas.h
+++ b/include/linux/clk/renesas.h
@@ -53,6 +53,9 @@ static inline void rzg2l_cpg_dsi_div_set_divider(u8 divider, int target) { }
  * various parameters used to configure a PLL. These limits ensure
  * the PLL operates within valid and stable ranges.
  *
+ * @input_fref: Reference input frequency to the PLL (in Hz). If set
+ * to 0, a default value of 24MHz is used.
+ *
  * @fout: Output frequency range (in MHz)
  * @fout.min: Minimum allowed output frequency
  * @fout.max: Maximum allowed output frequency
@@ -78,6 +81,8 @@ static inline void rzg2l_cpg_dsi_div_set_divider(u8 divider, int target) { }
  * @k.max: Maximum delta-sigma value
  */
 struct rzv2h_pll_limits {
+	u32 input_fref;
+
 	struct {
 		u32 min;
 		u32 max;
@@ -184,28 +189,31 @@ struct rzv2h_pll_div_pars {
 		.k = { .min = -32768, .max = 32767 },			\
 	}								\
 
-#ifdef CONFIG_CLK_RZV2H
-bool rzv2h_get_pll_pars(const struct rzv2h_pll_limits *limits,
-			struct rzv2h_pll_pars *pars, u64 freq_millihz);
+#ifdef CONFIG_CLK_RZV2H_CPG_LIB
+bool rzv2h_cpg_get_pll_pars(const struct rzv2h_pll_limits *limits,
+			    struct rzv2h_pll_pars *pars, u64 freq_millihz);
 
-bool rzv2h_get_pll_divs_pars(const struct rzv2h_pll_limits *limits,
-			     struct rzv2h_pll_div_pars *pars,
-			     const u8 *table, u8 table_size, u64 freq_millihz);
+bool rzv2h_cpg_get_pll_divs_pars(const struct rzv2h_pll_limits *limits,
+				 struct rzv2h_pll_div_pars *pars,
+				 const u8 *table, u8 table_size, u64 freq_millihz);
 #else
-static inline bool rzv2h_get_pll_pars(const struct rzv2h_pll_limits *limits,
-				      struct rzv2h_pll_pars *pars,
-				      u64 freq_millihz)
+static inline bool rzv2h_cpg_get_pll_pars(const struct rzv2h_pll_limits *limits,
+					  struct rzv2h_pll_pars *pars,
+					  u64 freq_millihz)
 {
 	return false;
 }
 
-static inline bool rzv2h_get_pll_divs_pars(const struct rzv2h_pll_limits *limits,
-					   struct rzv2h_pll_div_pars *pars,
-					   const u8 *table, u8 table_size,
-					   u64 freq_millihz)
+static inline bool rzv2h_cpg_get_pll_divs_pars(const struct rzv2h_pll_limits *limits,
+					       struct rzv2h_pll_div_pars *pars,
+					       const u8 *table, u8 table_size,
+					       u64 freq_millihz)
 {
 	return false;
 }
 #endif
 
+#define rzv2h_get_pll_pars	rzv2h_cpg_get_pll_pars
+#define rzv2h_get_pll_divs_pars	rzv2h_cpg_get_pll_divs_pars
+
 #endif
diff --git a/include/linux/codetag.h b/include/linux/codetag.h
index ddae7484ca456..a25a085c2df19 100644
--- a/include/linux/codetag.h
+++ b/include/linux/codetag.h
@@ -77,6 +77,8 @@ struct codetag_iterator {
 void codetag_lock_module_list(struct codetag_type *cttype);
 bool codetag_trylock_module_list(struct codetag_type *cttype);
 void codetag_unlock_module_list(struct codetag_type *cttype);
+unsigned long codetag_get_content_id(struct codetag_type *cttype);
+unsigned int codetag_get_count(struct codetag_type *cttype);
 struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype);
 struct codetag *codetag_next_ct(struct codetag_iterator *iter);
 
diff --git a/include/linux/container_of.h b/include/linux/container_of.h
index 1f6ebf27d962f..28db38e9ee3e3 100644
--- a/include/linux/container_of.h
+++ b/include/linux/container_of.h
@@ -17,11 +17,10 @@
  * Do not use container_of() in new code.
  */
 #define container_of(ptr, type, member) ({				\
-	void *__mptr = (void *)(ptr);					\
-	static_assert(__same_type(*(ptr), ((type *)0)->member) ||	\
+	static_assert(__same_type(*(ptr), typeof_member(type, member)) || \
 		      __same_type(*(ptr), void),			\
 		      "pointer type mismatch in container_of()");	\
-	((type *)(__mptr - offsetof(type, member))); })
+	(type *)((void *)(ptr) - offsetof(type, member)); })
 
 /**
  * container_of_const - cast a member of a structure out to the containing
diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h
index ae9d1ce4f49c4..35ce665edfd82 100644
--- a/include/linux/cpufreq.h
+++ b/include/linux/cpufreq.h
@@ -379,6 +379,7 @@ struct cpufreq_driver {
 	void		(*adjust_perf)(struct cpufreq_policy *policy,
 				       unsigned long min_perf,
 				       unsigned long target_perf,
+				       unsigned long max_perf,
 				       unsigned long capacity);
 
 	/*
@@ -624,6 +625,7 @@ unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
 void cpufreq_driver_adjust_perf(struct cpufreq_policy *policy,
 				unsigned long min_perf,
 				unsigned long target_perf,
+				unsigned long max_perf,
 				unsigned long capacity);
 bool cpufreq_driver_has_adjust_perf(void);
 int cpufreq_driver_target(struct cpufreq_policy *policy,
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 0fb3a2a62eb00..feb32949aeea6 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -180,7 +180,6 @@ enum cpuhp_state {
 	CPUHP_AP_DUMMY_TIMER_STARTING,
 	CPUHP_AP_ARM_XEN_STARTING,
 	CPUHP_AP_ARM_XEN_RUNSTATE_STARTING,
-	CPUHP_AP_ARM_CORESIGHT_CTI_STARTING,
 	CPUHP_AP_ARM64_ISNDEP_STARTING,
 	CPUHP_AP_SMPCFD_DYING,
 	CPUHP_AP_HRTIMERS_DYING,
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index d3cda05449541..4c8bb6953107d 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -13,7 +13,6 @@
 #include <linux/cpumask_types.h>
 #include <linux/gfp_types.h>
 #include <linux/numa.h>
-#include <linux/sprintf.h>
 #include <linux/threads.h>
 #include <linux/types.h>
 #include <vdso/page.h>
@@ -1315,24 +1314,6 @@ static __always_inline bool cpu_dying(unsigned int cpu)
 }
 #endif /* NR_CPUS > BITS_PER_LONG */
 
-/**
- * cpumap_print_to_pagebuf  - copies the cpumask into the buffer either
- *	as comma-separated list of cpus or hex values of cpumask
- * @list: indicates whether the cpumap must be list
- * @mask: the cpumask to copy
- * @buf: the buffer to copy into
- *
- * Return: the length of the (null-terminated) @buf string, zero if
- * nothing is copied.
- */
-static __always_inline ssize_t
-cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
-{
-	/* Opencode offset_in_page(buf) to not include linux/mm.h */
-	return scnprintf(buf, PAGE_SIZE - ((unsigned long)buf & ~PAGE_MASK),
-			 list ? "%*pbl\n" : "%*pb\n", cpumask_pr_args(mask));
-}
-
 /**
  * cpumap_print_bitmask_to_buf  - copies the cpumask into the buffer as
  *	hex values of cpumask
diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h
index 65d76a38974ba..a67ba1b32d168 100644
--- a/include/linux/cpuset.h
+++ b/include/linux/cpuset.h
@@ -80,10 +80,11 @@ extern void lockdep_assert_cpuset_lock_held(void);
 extern void cpuset_cpus_allowed_locked(struct task_struct *p, struct cpumask *mask);
 extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask);
 extern bool cpuset_cpus_allowed_fallback(struct task_struct *p);
+extern int cpuset_num_cpus(struct cgroup *cgroup);
 extern nodemask_t cpuset_mems_allowed(struct task_struct *p);
 #define cpuset_current_mems_allowed (current->mems_allowed)
 void cpuset_init_current_mems_allowed(void);
-int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask);
+int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask);
 
 extern bool cpuset_current_node_allowed(int node, gfp_t gfp_mask);
 
@@ -216,6 +217,11 @@ static inline bool cpuset_cpus_allowed_fallback(struct task_struct *p)
 	return false;
 }
 
+static inline int cpuset_num_cpus(struct cgroup *cgroup)
+{
+	return num_online_cpus();
+}
+
 static inline nodemask_t cpuset_mems_allowed(struct task_struct *p)
 {
 	return node_possible_map;
@@ -224,7 +230,7 @@ static inline nodemask_t cpuset_mems_allowed(struct task_struct *p)
 #define cpuset_current_mems_allowed (node_states[N_MEMORY])
 static inline void cpuset_init_current_mems_allowed(void) {}
 
-static inline int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
+static inline int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask)
 {
 	return 1;
 }
diff --git a/include/linux/crash_core.h b/include/linux/crash_core.h
index c1dee3f971a91..bc087124cd78e 100644
--- a/include/linux/crash_core.h
+++ b/include/linux/crash_core.h
@@ -59,6 +59,9 @@ extern int crash_exclude_mem_range(struct crash_mem *mem,
 				   unsigned long long mend);
 extern int crash_prepare_elf64_headers(struct crash_mem *mem, int need_kernel_map,
 				       void **addr, unsigned long *sz);
+extern int crash_prepare_headers(int need_kernel_map, void **addr,
+				 unsigned long *sz, unsigned long *nr_mem_ranges);
+extern int crash_exclude_core_ranges(struct crash_mem **cmem);
 
 struct kimage;
 struct kexec_segment;
@@ -76,6 +79,12 @@ int kexec_should_crash(struct task_struct *p);
 int kexec_crash_loaded(void);
 void crash_save_cpu(struct pt_regs *regs, int cpu);
 extern int kimage_crash_copy_vmcoreinfo(struct kimage *image);
+extern unsigned int arch_get_system_nr_ranges(void);
+extern int arch_crash_populate_cmem(struct crash_mem *cmem);
+extern int arch_crash_exclude_ranges(struct crash_mem *cmem);
+extern int arch_crash_exclude_mem_range(struct crash_mem **mem,
+					unsigned long long mstart,
+					unsigned long long mend);
 
 #else /* !CONFIG_CRASH_DUMP*/
 struct pt_regs;
diff --git a/include/linux/crash_reserve.h b/include/linux/crash_reserve.h
index f0dc03d94ca2c..30864d90d7f50 100644
--- a/include/linux/crash_reserve.h
+++ b/include/linux/crash_reserve.h
@@ -14,9 +14,11 @@
 extern struct resource crashk_res;
 extern struct resource crashk_low_res;
 extern struct range crashk_cma_ranges[];
+
+#define CRASHK_CMA_RANGES_MAX 4
 #if defined(CONFIG_CMA) && defined(CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION)
 #define CRASHKERNEL_CMA
-#define CRASHKERNEL_CMA_RANGES_MAX 4
+#define CRASHKERNEL_CMA_RANGES_MAX (CRASHK_CMA_RANGES_MAX)
 extern int crashk_cma_cnt;
 #else
 #define crashk_cma_cnt 0
diff --git a/include/linux/cred.h b/include/linux/cred.h
index c6676265a9859..6ef1750c93e2a 100644
--- a/include/linux/cred.h
+++ b/include/linux/cred.h
@@ -371,7 +371,6 @@ DEFINE_FREE(put_cred, struct cred *, if (!IS_ERR_OR_NULL(_T)) put_cred(_T))
 })
 
 #define task_uid(task)		(task_cred_xxx((task), uid))
-#define task_euid(task)		(task_cred_xxx((task), euid))
 #define task_ucounts(task)	(task_cred_xxx((task), ucounts))
 
 #define current_cred_xxx(xxx)			\
diff --git a/include/linux/damon.h b/include/linux/damon.h
index 02ac34537df9a..327f534696aa8 100644
--- a/include/linux/damon.h
+++ b/include/linux/damon.h
@@ -1,8 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
  * DAMON api
- *
- * Author: SeongJae Park <sj@kernel.org>
  */
 
 #ifndef _DAMON_H_
@@ -47,13 +45,11 @@ struct damon_size_range {
  * @ar:			The address range of the region.
  * @sampling_addr:	Address of the sample for the next access check.
  * @nr_accesses:	Access frequency of this region.
- * @nr_accesses_bp:	@nr_accesses in basis point (0.01%) that updated for
- *			each sampling interval.
  * @probe_hits:		Number of probe-positive region samples.
- * @list:		List head for siblings.
  * @age:		Age of this region.
  *
- * For any use case, @ar should be non-zero positive size.
+ * For any use case, @ar should be non-zero positive size.  damon_set_regions()
+ * does the validation.
  *
  * @nr_accesses is reset to zero for every &damon_attrs->aggr_interval and be
  * increased for every &damon_attrs->sample_interval if an access to the region
@@ -61,13 +57,6 @@ struct damon_size_range {
  * not be done with direct access but with the helper function,
  * damon_update_region_access_rate().
  *
- * @nr_accesses_bp is another representation of @nr_accesses in basis point
- * (1 in 10,000) that updated for every &damon_attrs->sample_interval in a
- * manner similar to moving sum.  By the algorithm, this value becomes
- * @nr_accesses * 10000 for every &struct damon_attrs->aggr_interval.  This can
- * be used when the aggregation interval is too huge and therefore cannot wait
- * for it before getting the access monitoring results.
- *
  * @age is initially zero, increased for each aggregation interval, and reset
  * to zero again if the access frequency is significantly changed.  If two
  * regions are merged into a new region, both @nr_accesses and @age of the new
@@ -77,21 +66,19 @@ struct damon_region {
 	struct damon_addr_range ar;
 	unsigned long sampling_addr;
 	unsigned int nr_accesses;
-	unsigned int nr_accesses_bp;
 	unsigned char probe_hits[DAMON_MAX_PROBES];
-	struct list_head list;
-
 	unsigned int age;
-/* private: Internal value for age calculation. */
+/* private: internal use only. */
+	/* List head for siblings. */
+	struct list_head list;
+	/* for age calculation. */
 	unsigned int last_nr_accesses;
+	unsigned char last_probe_hits[DAMON_MAX_PROBES];
 };
 
 /**
  * struct damon_target - Represents a monitoring target.
  * @pid:		The PID of the virtual address space to monitor.
- * @nr_regions:		Number of monitoring target regions of this target.
- * @regions_list:	Head of the monitoring target regions of this target.
- * @list:		List head for siblings.
  * @obsolete:		Whether the commit destination target is obsolete.
  *
  * Each monitoring context could have multiple targets.  For example, a context
@@ -105,10 +92,14 @@ struct damon_region {
  */
 struct damon_target {
 	struct pid *pid;
+	bool obsolete;
+/* private: */
+	/* Number of monitoring target regions of this target. */
 	unsigned int nr_regions;
+	/* Head of the monitoring target regions of this target. */
 	struct list_head regions_list;
+	/* List head for siblings. */
 	struct list_head list;
-	bool obsolete;
 };
 
 /**
@@ -184,10 +175,8 @@ enum damos_quota_goal_metric {
  * @metric:		Metric to be used for representing the goal.
  * @target_value:	Target value of @metric to achieve with the tuning.
  * @current_value:	Current value of @metric.
- * @last_psi_total:	Last measured total PSI
  * @nid:		Node id.
  * @memcg_id:		Memcg id.
- * @list:		List head for siblings.
  *
  * Data structure for getting the current score of the quota tuning goal.  The
  * score is calculated by how close @current_value and @target_value are.  Then
@@ -210,12 +199,17 @@ struct damos_quota_goal {
 	unsigned long current_value;
 	/* metric-dependent fields */
 	union {
-		u64 last_psi_total;
 		struct {
 			int nid;
 			u64 memcg_id;
 		};
+/* private: */
+		/* Last measured total PSI */
+		u64 last_psi_total;
+
 	};
+/* private: */
+	/* List head for siblings. */
 	struct list_head list;
 };
 
@@ -234,7 +228,6 @@ enum damos_quota_goal_tuner {
  * @reset_interval:	Charge reset interval in milliseconds.
  * @ms:			Maximum milliseconds that the scheme can use.
  * @sz:			Maximum bytes of memory that the action can be applied.
- * @goals:		Head of quota tuning goals (&damos_quota_goal) list.
  * @goal_tuner:		Goal-based @esz tuning algorithm to use.
  * @esz:		Effective size quota in bytes.
  * @fail_charge_num:	Failed regions charge rate numerator.
@@ -261,7 +254,7 @@ enum damos_quota_goal_tuner {
  * the scheme's action.  DAMON then compares it against &sz and uses smaller
  * one as the effective quota.
  *
- * If @goals is not empty, DAMON calculates yet another size quota based on the
+ * If goals is not empty, DAMON calculates yet another size quota based on the
  * goals using its internal feedback loop algorithm, for every @reset_interval.
  * Then, if the new size quota is smaller than the effective quota, it uses the
  * new size quota as the effective quota.
@@ -282,7 +275,7 @@ struct damos_quota {
 	unsigned long reset_interval;
 	unsigned long ms;
 	unsigned long sz;
-	struct list_head goals;
+
 	enum damos_quota_goal_tuner goal_tuner;
 	unsigned long esz;
 
@@ -294,6 +287,8 @@ struct damos_quota {
 	unsigned int weight_age;
 
 /* private: */
+	/* Head of quota tuning goals (&damos_quota_goal) list. */
+	struct list_head goals;
 	/* For throughput estimation */
 	unsigned long total_charged_sz;
 	unsigned long total_charged_ns;
@@ -437,7 +432,6 @@ enum damos_filter_type {
  *		&damon_ctx->adaptive_targets if @type is
  *		DAMOS_FILTER_TYPE_TARGET.
  * @sz_range:	Size range if @type is DAMOS_FILTER_TYPE_HUGEPAGE_SIZE.
- * @list:	List head for siblings.
  *
  * Before applying the &damos->action to a memory region, DAMOS checks if each
  * byte of the region matches to this given condition and avoid applying the
@@ -455,6 +449,8 @@ struct damos_filter {
 		int target_idx;
 		struct damon_size_range sz_range;
 	};
+/* private: */
+	/* List head for siblings. */
 	struct list_head list;
 };
 
@@ -526,12 +522,8 @@ struct damos_migrate_dests {
  * @wmarks:		Watermarks for automated (in)activation of this scheme.
  * @migrate_dests:	Destination nodes if @action is "migrate_{hot,cold}".
  * @target_nid:		Destination node if @action is "migrate_{hot,cold}".
- * @core_filters:	Additional set of &struct damos_filter for &action.
- * @ops_filters:	ops layer handling &struct damos_filter objects list.
- * @last_applied:	Last @action applied ops-managing entity.
  * @stat:		Statistics of this scheme.
  * @max_nr_snapshots:	Upper limit of nr_snapshots stat.
- * @list:		List head for siblings.
  *
  * For each @apply_interval_us, DAMON finds regions which fit in the
  * &pattern and applies &action to those. To avoid consuming too much
@@ -553,16 +545,7 @@ struct damos_migrate_dests {
  *
  * Before applying the &action to a memory region, &struct damon_operations
  * implementation could check pages of the region and skip &action to respect
- * &core_filters
- *
- * The minimum entity that @action can be applied depends on the underlying
- * &struct damon_operations.  Since it may not be aligned with the core layer
- * abstract, namely &struct damon_region, &struct damon_operations could apply
- * @action to same entity multiple times.  Large folios that underlying on
- * multiple &struct damon region objects could be such examples.  The &struct
- * damon_operations can use @last_applied to avoid that.  DAMOS core logic
- * unsets @last_applied when each regions walking for applying the scheme is
- * finished.
+ * &struct damos_filter.
  *
  * After applying the &action to each region, &stat is updated.
  *
@@ -573,6 +556,16 @@ struct damos {
 	struct damos_access_pattern pattern;
 	enum damos_action action;
 	unsigned long apply_interval_us;
+	struct damos_quota quota;
+	struct damos_watermarks wmarks;
+	union {
+		struct {
+			int target_nid;
+			struct damos_migrate_dests migrate_dests;
+		};
+	};
+	struct damos_stat stat;
+	unsigned long max_nr_snapshots;
 /* private: internal use only */
 	/*
 	 * number of sample intervals that should be passed before applying
@@ -589,20 +582,25 @@ struct damos {
 	/* whether to reject core/ops filters umatched regions */
 	bool core_filters_default_reject;
 	bool ops_filters_default_reject;
-/* public: */
-	struct damos_quota quota;
-	struct damos_watermarks wmarks;
-	union {
-		struct {
-			int target_nid;
-			struct damos_migrate_dests migrate_dests;
-		};
-	};
+	/* Additional set of &struct damos_filter for &action. */
 	struct list_head core_filters;
+	/* ops layer handling &struct damos_filter objects list. */
 	struct list_head ops_filters;
+	/*
+	 * Last @action applied ops-managing entity.
+	 *
+	 * The minimum entity that @action can be applied depends on the
+	 * underlying &struct damon_operations.  Since it may not be aligned
+	 * with the core layer abstract, namely &struct damon_region, &struct
+	 * damon_operations could apply @action to same entity multiple times.
+	 * Large folios that underlying on multiple &struct damon region
+	 * objects could be such examples.  The &struct damon_operations can
+	 * use @last_applied to avoid that.  DAMOS core logic unsets
+	 * @last_applied when each regions walking for applying the scheme is
+	 * finished.
+	 */
 	void *last_applied;
-	struct damos_stat stat;
-	unsigned long max_nr_snapshots;
+	/* List head for siblings. */
 	struct list_head list;
 };
 
@@ -658,7 +656,10 @@ enum damon_ops_id {
  * It should also return max number of observed accesses that made as a result
  * of its update.  The value will be used for regions adjustment threshold.
  * @apply_probes should apply the data attribute probes to each region and
- * accordingly update the probe hits counter of the region.
+ * accordingly update the probe hits counter of the region.  It should also
+ * set &damon_region->sampling_addr of each region if ``set_samples`` is true.
+ * It should also return maximum probe hits weighted sum of regions if
+ * ``return_max_wsum`` is true.
  * @get_scheme_score should return the priority score of a region for a scheme
  * as an integer in [0, &DAMOS_MAX_SCORE].
  * @apply_scheme is called from @kdamond when a region for user provided
@@ -676,7 +677,8 @@ struct damon_operations {
 	void (*update)(struct damon_ctx *context);
 	void (*prepare_access_checks)(struct damon_ctx *context);
 	unsigned int (*check_accesses)(struct damon_ctx *context);
-	void (*apply_probes)(struct damon_ctx *context);
+	unsigned int (*apply_probes)(struct damon_ctx *context,
+			bool set_samples, bool return_max_wsum);
 	int (*get_scheme_score)(struct damon_ctx *context,
 			struct damon_region *r, struct damos *scheme);
 	unsigned long (*apply_scheme)(struct damon_ctx *context,
@@ -756,7 +758,6 @@ enum damon_filter_type {
  * @matching:	Whether this filter is for the type-matching ones.
  * @allow:	Whether the @type-@matching ones should pass this filter.
  * @memcg_id:	Memcg id of the question if @type is DAMON_FILTER_MEMCG.
- * @list:	Siblings list.
  */
 struct damon_filter {
 	enum damon_filter_type type;
@@ -765,17 +766,22 @@ struct damon_filter {
 	union {
 		u64 memcg_id;
 	};
+/* private: */
+	/* Siblings list. */
 	struct list_head list;
 };
 
 /**
  * struct damon_probe - Data region attribute probe.
  *
- * @filters:	Filters for assessing if a given region is for this probe.
- * @list:	Siblings list.
+ * @weight:	Relative priority of the attribute for this probe.
  */
 struct damon_probe {
+	unsigned int weight;
+/* private: */
+	/* Filters for assessing if a given region is for this probe. */
 	struct list_head filters;
+	/* Siblings list. */
 	struct list_head list;
 };
 
@@ -842,14 +848,9 @@ struct damon_attrs {
  * thread other than the kdamond should be made using safe DAMON APIs,
  * including damon_call() and damos_walk().
  *
- * @ops:	Set of monitoring operations for given use cases.
- * @probes:	Head of probes (&damon_probe) list.
  * @addr_unit:	Scale factor for core to ops address conversion.
  * @min_region_sz:	Minimum region size.
  * @pause:	Pause kdamond main loop.
- * @adaptive_targets:	Head of monitoring targets (&damon_target) list.
- * @schemes:		Head of schemes (&damos) list.
- * @rnd_state:	Per-ctx PRNG state for damon_rand().
  */
 struct damon_ctx {
 	struct damon_attrs attrs;
@@ -898,15 +899,21 @@ struct damon_ctx {
 	struct mutex kdamond_lock;
 
 /* public: */
-	struct damon_operations ops;
-	struct list_head probes;
 	unsigned long addr_unit;
 	unsigned long min_region_sz;
 	bool pause;
 
+/* private: */
+	/* Set of monitoring operations for given use cases. */
+	struct damon_operations ops;
+	/* Head of monitoring targets (&damon_target) list. */
 	struct list_head adaptive_targets;
+	/* Head of probes (&damon_probe) list. */
+	struct list_head probes;
+	/* Head of schemes (&damos) list. */
 	struct list_head schemes;
 
+	/* @rnd_state:	Per-ctx PRNG state for damon_rand(). */
 	struct rnd_state rnd_state;
 };
 
@@ -1013,11 +1020,16 @@ struct damon_probe *damon_new_probe(void);
 void damon_add_probe(struct damon_ctx *ctx, struct damon_probe *probe);
 
 struct damon_region *damon_new_region(unsigned long start, unsigned long end);
+unsigned int damon_nr_accesses_mvsum(struct damon_region *r,
+		struct damon_ctx *ctx);
+unsigned char damon_probe_hits_mvsum(int probe_idx, struct damon_region *r,
+		struct damon_ctx *ctx);
+unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last,
+		struct damon_ctx *ctx);
 
 int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges,
 		unsigned int nr_ranges, unsigned long min_region_sz);
-void damon_update_region_access_rate(struct damon_region *r, bool accessed,
-		struct damon_attrs *attrs);
+void damon_update_region_access_rate(struct damon_region *r, bool accessed);
 
 struct damos_filter *damos_new_filter(enum damos_filter_type type,
 		bool matching, bool allow);
@@ -1064,21 +1076,23 @@ static inline bool damon_target_has_pid(const struct damon_ctx *ctx)
 	return ctx->ops.id == DAMON_OPS_VADDR || ctx->ops.id == DAMON_OPS_FVADDR;
 }
 
-static inline unsigned int damon_max_nr_accesses(const struct damon_attrs *attrs)
+/* Returns number of samples per aggregation interval */
+static inline unsigned int damon_nr_samples_per_aggr(
+		const struct damon_attrs *attrs)
 {
 	unsigned long sample_interval;
-	unsigned long max_nr_accesses;
+	unsigned long nr_samples;
 
 	sample_interval = attrs->sample_interval ? : 1;
-	max_nr_accesses = min(attrs->aggr_interval / sample_interval,
+	nr_samples = min(attrs->aggr_interval / sample_interval,
 			(unsigned long)UINT_MAX);
-	return max_nr_accesses ? : 1;
+	return nr_samples ? : 1;
 }
 
 
 bool damon_initialized(void);
 int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive);
-int damon_stop(struct damon_ctx **ctxs, int nr_ctxs);
+void damon_stop(struct damon_ctx **ctxs, int nr_ctxs);
 bool damon_is_running(struct damon_ctx *ctx);
 int damon_kdamond_pid(struct damon_ctx *ctx);
 
diff --git a/include/linux/device-id/zorro.h b/include/linux/device-id/zorro.h
index 5fdac81689839..26827b9b90b67 100644
--- a/include/linux/device-id/zorro.h
+++ b/include/linux/device-id/zorro.h
@@ -13,7 +13,11 @@ typedef unsigned long kernel_ulong_t;
 
 struct zorro_device_id {
 	__u32 id;			/* Device ID or ZORRO_WILDCARD */
-	kernel_ulong_t driver_data;	/* Data private to the driver */
+	union {
+		/* Data private to the driver */
+		kernel_ulong_t driver_data;
+		const void *driver_data_ptr;
+	};
 };
 
 #endif /* ifndef LINUX_DEVICE_ID_ZORRO_H */
diff --git a/include/linux/device.h b/include/linux/device.h
index 7b2baffdd2f55..aee79fd6b32b4 100644
--- a/include/linux/device.h
+++ b/include/linux/device.h
@@ -99,7 +99,9 @@ struct device_type {
  * struct device_attribute - Interface for exporting device attributes.
  * @attr: sysfs attribute definition.
  * @show: Show handler.
+ * @show_const: Show handler (read-only).
  * @store: Store handler.
+ * @store_const: Store handler (read-only).
  */
 struct device_attribute {
 	struct attribute	attr;
diff --git a/include/linux/dma-fence-unwrap.h b/include/linux/dma-fence-unwrap.h
index 62df222fe0f16..7bfacdf79de21 100644
--- a/include/linux/dma-fence-unwrap.h
+++ b/include/linux/dma-fence-unwrap.h
@@ -8,6 +8,8 @@
 #ifndef __LINUX_DMA_FENCE_UNWRAP_H
 #define __LINUX_DMA_FENCE_UNWRAP_H
 
+#include <linux/types.h>
+
 struct dma_fence;
 
 /**
@@ -48,11 +50,11 @@ struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor);
 	for (fence = dma_fence_unwrap_first(head, cursor); fence;	\
 	     fence = dma_fence_unwrap_next(cursor))
 
-struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences,
+struct dma_fence *__dma_fence_unwrap_merge(size_t num_fences,
 					   struct dma_fence **fences,
 					   struct dma_fence_unwrap *cursors);
 
-int dma_fence_dedup_array(struct dma_fence **array, int num_fences);
+size_t dma_fence_dedup_array(struct dma_fence **array, size_t num_fences);
 
 /**
  * dma_fence_unwrap_merge - unwrap and merge fences
diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h
index b52ab692b22e8..158cd609f1036 100644
--- a/include/linux/dma-fence.h
+++ b/include/linux/dma-fence.h
@@ -448,7 +448,7 @@ int dma_fence_add_callback(struct dma_fence *fence,
 			   dma_fence_func_t func);
 bool dma_fence_remove_callback(struct dma_fence *fence,
 			       struct dma_fence_cb *cb);
-void dma_fence_enable_sw_signaling(struct dma_fence *fence);
+void dma_fence_enable_signaling(struct dma_fence *fence);
 
 /**
  * DOC: Safe external access to driver provided object members
@@ -534,7 +534,7 @@ dma_fence_is_signaled_locked(struct dma_fence *fence)
  * Returns true if the fence was already signaled, false if not. Since this
  * function doesn't enable signaling, it is not guaranteed to ever return
  * true if dma_fence_add_callback(), dma_fence_wait() or
- * dma_fence_enable_sw_signaling() haven't been called before.
+ * dma_fence_enable_signaling() haven't been called before.
  *
  * It's recommended for seqno fences to call dma_fence_signal when the
  * operation is complete, it makes it possible to prevent issues from
diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h
index 1fafd5b0e3153..da7a5cc93ad43 100644
--- a/include/linux/dma/edma.h
+++ b/include/linux/dma/edma.h
@@ -14,6 +14,8 @@
 
 #define EDMA_MAX_WR_CH                                  8
 #define EDMA_MAX_RD_CH                                  8
+#define HDMA_MAX_WR_CH                                  64
+#define HDMA_MAX_RD_CH                                  64
 
 struct dw_edma;
 
@@ -89,12 +91,12 @@ struct dw_edma_chip {
 	u16			ll_wr_cnt;
 	u16			ll_rd_cnt;
 	/* link list address */
-	struct dw_edma_region	ll_region_wr[EDMA_MAX_WR_CH];
-	struct dw_edma_region	ll_region_rd[EDMA_MAX_RD_CH];
+	struct dw_edma_region	ll_region_wr[HDMA_MAX_WR_CH];
+	struct dw_edma_region	ll_region_rd[HDMA_MAX_RD_CH];
 
 	/* data region */
-	struct dw_edma_region	dt_region_wr[EDMA_MAX_WR_CH];
-	struct dw_edma_region	dt_region_rd[EDMA_MAX_RD_CH];
+	struct dw_edma_region	dt_region_wr[HDMA_MAX_WR_CH];
+	struct dw_edma_region	dt_region_rd[HDMA_MAX_RD_CH];
 
 	/* interrupt emulation */
 	int			db_irq;
diff --git a/include/linux/dma/qcom_bam_dma.h b/include/linux/dma/qcom_bam_dma.h
index 68fc0e643b1b9..d9d07a9ab3132 100644
--- a/include/linux/dma/qcom_bam_dma.h
+++ b/include/linux/dma/qcom_bam_dma.h
@@ -13,9 +13,12 @@
  * supported by BAM DMA Engine.
  *
  * @cmd_and_addr - upper 8 bits command and lower 24 bits register address.
- * @data - for write command: content to be written into peripheral register.
- *	   for read command: dest addr to write peripheral register value.
- * @mask - register mask.
+ * @data - For write command: content to be written into peripheral register.
+ *	   For read command: lower 32 bits of destination address.
+ * @mask - For write command: register write mask.
+ *	   For read command on BAM v1.6.0+: upper 4 bits of destination address.
+ *	   For read command on BAM < v1.6.0: ignored by hardware.
+ *	   Setting to 0 ensures 32-bit addressing compatibility.
  * @reserved - for future usage.
  *
  */
@@ -42,6 +45,10 @@ enum bam_command_type {
  * @addr: target address
  * @cmd: BAM command
  * @data: actual data for write and dest addr for read in le32
+ *
+ * For BAM v1.6.0+, the mask field behavior depends on command type:
+ * - Write commands: mask = write mask (typically 0xffffffff)
+ * - Read commands: mask = upper 4 bits of destination address (0 for 32-bit)
  */
 static inline void
 bam_prep_ce_le32(struct bam_cmd_element *bam_ce, u32 addr,
@@ -50,7 +57,11 @@ bam_prep_ce_le32(struct bam_cmd_element *bam_ce, u32 addr,
 	bam_ce->cmd_and_addr =
 		cpu_to_le32((addr & 0xffffff) | ((cmd & 0xff) << 24));
 	bam_ce->data = data;
-	bam_ce->mask = cpu_to_le32(0xffffffff);
+	if (cmd == BAM_READ_COMMAND)
+		bam_ce->mask = cpu_to_le32(0x0); /* 32-bit addressing */
+	else
+		bam_ce->mask = cpu_to_le32(0xffffffff); /* Write mask */
+	bam_ce->reserved = 0;
 }
 
 /*
@@ -60,7 +71,7 @@ bam_prep_ce_le32(struct bam_cmd_element *bam_ce, u32 addr,
  * @bam_ce: BAM command element
  * @addr: target address
  * @cmd: BAM command
- * @data: actual data for write and dest addr for read
+ * @data: actual data for write and destination address for read
  */
 static inline void
 bam_prep_ce(struct bam_cmd_element *bam_ce, u32 addr,
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
index b3d251c9734e9..fe33a20abc614 100644
--- a/include/linux/dmaengine.h
+++ b/include/linux/dmaengine.h
@@ -322,6 +322,8 @@ struct dma_router {
  * @slave: ptr to the device using this channel
  * @cookie: last cookie value returned to client
  * @completed_cookie: last completed cookie for this channel
+ * @lock: protect between config and prepare transfer when driver have not
+ *	  implemented callback device_prep_config_sg().
  * @chan_id: channel ID for sysfs
  * @dev: class device for sysfs
  * @name: backlink name for sysfs
@@ -341,6 +343,12 @@ struct dma_chan {
 	dma_cookie_t cookie;
 	dma_cookie_t completed_cookie;
 
+	/*
+	 * protect between config and prepare transfer because *_prep() may be
+	 * called from complete callback, which is in GFP_NOSLEEP context.
+	 */
+	spinlock_t lock;
+
 	/* sysfs */
 	int chan_id;
 	struct dma_chan_dev *dev;
@@ -623,7 +631,7 @@ struct dma_async_tx_descriptor {
 	void *callback_param;
 	struct dmaengine_unmap_data *unmap;
 	enum dma_desc_metadata_mode desc_metadata_mode;
-	struct dma_descriptor_metadata_ops *metadata_ops;
+	const struct dma_descriptor_metadata_ops *metadata_ops;
 #ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
 	struct dma_async_tx_descriptor *next;
 	struct dma_async_tx_descriptor *parent;
@@ -835,6 +843,7 @@ struct dma_filter {
  *	where the address and size of each segment is located in one entry of
  *	the dma_vec array.
  * @device_prep_slave_sg: prepares a slave dma operation
+ * @device_prep_config_sg: prepares a slave DMA operation with dma_slave_config
  * @device_prep_dma_cyclic: prepare a cyclic dma operation suitable for audio.
  *	The function takes a buffer of size buf_len. The callback function will
  *	be called after period_len bytes have been transferred.
@@ -934,6 +943,10 @@ struct dma_device {
 		struct dma_chan *chan, struct scatterlist *sgl,
 		unsigned int sg_len, enum dma_transfer_direction direction,
 		unsigned long flags, void *context);
+	struct dma_async_tx_descriptor *(*device_prep_config_sg)(
+		struct dma_chan *chan, struct scatterlist *sgl,
+		unsigned int sg_len, enum dma_transfer_direction direction,
+		unsigned long flags, struct dma_slave_config *config);
 	struct dma_async_tx_descriptor *(*device_prep_dma_cyclic)(
 		struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
 		size_t period_len, enum dma_transfer_direction direction,
@@ -974,22 +987,44 @@ static inline bool is_slave_direction(enum dma_transfer_direction direction)
 	       (direction == DMA_DEV_TO_DEV);
 }
 
-static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_single(
-	struct dma_chan *chan, dma_addr_t buf, size_t len,
-	enum dma_transfer_direction dir, unsigned long flags)
+static inline struct dma_async_tx_descriptor *
+dmaengine_prep_config_single(struct dma_chan *chan, dma_addr_t buf, size_t len,
+			     enum dma_transfer_direction dir,
+			     unsigned long flags,
+			     struct dma_slave_config *config)
 {
 	struct scatterlist sg;
+
+	if (!chan || !chan->device)
+		return NULL;
+
 	sg_init_table(&sg, 1);
 	sg_dma_address(&sg) = buf;
 	sg_dma_len(&sg) = len;
 
-	if (!chan || !chan->device || !chan->device->device_prep_slave_sg)
+	if (chan->device->device_prep_config_sg)
+		return chan->device->device_prep_config_sg(chan, &sg, 1, dir,
+							   flags, config);
+
+	if (config)
+		if (dmaengine_slave_config(chan, config))
+			return NULL;
+
+	if (!chan->device->device_prep_slave_sg)
 		return NULL;
 
 	return chan->device->device_prep_slave_sg(chan, &sg, 1,
 						  dir, flags, NULL);
 }
 
+static inline struct dma_async_tx_descriptor *
+dmaengine_prep_slave_single(struct dma_chan *chan, dma_addr_t buf, size_t len,
+			    enum dma_transfer_direction dir,
+			    unsigned long flags)
+{
+	return dmaengine_prep_config_single(chan, buf, len, dir, flags, NULL);
+}
+
 /**
  * dmaengine_prep_peripheral_dma_vec() - Prepare a DMA scatter-gather descriptor
  * @chan: The channel to be used for this descriptor
@@ -1010,17 +1045,115 @@ static inline struct dma_async_tx_descriptor *dmaengine_prep_peripheral_dma_vec(
 							    dir, flags);
 }
 
-static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_sg(
-	struct dma_chan *chan, struct scatterlist *sgl,	unsigned int sg_len,
-	enum dma_transfer_direction dir, unsigned long flags)
+static inline struct dma_async_tx_descriptor *
+dmaengine_prep_config_sg(struct dma_chan *chan, struct scatterlist *sgl,
+			 unsigned int sg_len, enum dma_transfer_direction dir,
+			 unsigned long flags, struct dma_slave_config *config)
 {
-	if (!chan || !chan->device || !chan->device->device_prep_slave_sg)
+	if (!chan || !chan->device)
+		return NULL;
+
+	if (chan->device->device_prep_config_sg)
+		return chan->device->device_prep_config_sg(chan, sgl, sg_len,
+				dir, flags, config);
+
+	if (config)
+		if (dmaengine_slave_config(chan, config))
+			return NULL;
+
+	if (!chan->device->device_prep_slave_sg)
 		return NULL;
 
 	return chan->device->device_prep_slave_sg(chan, sgl, sg_len,
 						  dir, flags, NULL);
 }
 
+static inline struct dma_async_tx_descriptor *
+dmaengine_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
+			unsigned int sg_len, enum dma_transfer_direction dir,
+			unsigned long flags)
+{
+	return dmaengine_prep_config_sg(chan, sgl, sg_len, dir, flags, NULL);
+}
+
+/**
+ * dmaengine_prep_config_sg_safe - prepare a scatter-gather DMA transfer
+ *                                 with atomic slave configuration update
+ * @chan: DMA channel
+ * @sgl: scatterlist for the transfer
+ * @sg_len: number of entries in @sgl
+ * @dir: DMA transfer direction
+ * @flags: transfer preparation flags
+ * @config: DMA slave configuration for this transfer
+ *
+ * Prepare a DMA scatter-gather transfer together with a corresponding slave
+ * configuration update in a re-entrant and race-safe manner.
+ *
+ * DMA engine drivers may implement the optional
+ * device_prep_config_sg() callback to perform both the slave configuration
+ * and descriptor preparation atomically. In this case, the operation is
+ * fully handled by the DMA engine driver.
+ *
+ * If the DMA engine driver does not implement device_prep_config_sg(), falls
+ * back to calling dmaengine_slave_config() followed by dmaengine_prep_slave_sg().
+ * The fallback path is protected by a per-channel spinlock to ensure that
+ * concurrent callers cannot interleave configuration and descriptor preparation
+ * on the same DMA channel.
+ *
+ * Return: Pointer to a prepared DMA async transaction descriptor on success,
+ * or %NULL if the transfer could not be prepared.
+ */
+static inline struct dma_async_tx_descriptor *
+dmaengine_prep_config_sg_safe(struct dma_chan *chan, struct scatterlist *sgl,
+			      unsigned int sg_len,
+			      enum dma_transfer_direction dir,
+			      unsigned long flags,
+			      struct dma_slave_config *config)
+{
+	struct dma_async_tx_descriptor *tx;
+	unsigned long spinlock_flags;
+
+	if (!chan || !chan->device)
+		return NULL;
+
+	if (!chan->device->device_prep_config_sg)
+		spin_lock_irqsave(&chan->lock, spinlock_flags);
+
+	tx = dmaengine_prep_config_sg(chan, sgl, sg_len, dir, flags, config);
+
+	if (!chan->device->device_prep_config_sg)
+		spin_unlock_irqrestore(&chan->lock, spinlock_flags);
+
+	return tx;
+}
+
+/**
+ * dmaengine_prep_config_single_safe - prepare a single-buffer DMA transfer
+ *                                     with atomic slave configuration update
+ * @chan: DMA channel
+ * @buf: DMA buffer address
+ * @len: length of the transfer in bytes
+ * @dir: DMA transfer direction
+ * @flags: transfer preparation flags
+ * @config: DMA slave configuration for this transfer
+ *
+ * Detail see dmaengine_prep_config_sg_safe().
+ */
+static inline struct dma_async_tx_descriptor *
+dmaengine_prep_config_single_safe(struct dma_chan *chan, dma_addr_t buf,
+				  size_t len, enum dma_transfer_direction dir,
+				  unsigned long flags,
+				  struct dma_slave_config *config)
+{
+	struct scatterlist sg;
+
+	sg_init_table(&sg, 1);
+	sg_dma_address(&sg) = buf;
+	sg_dma_len(&sg) = len;
+
+	return dmaengine_prep_config_sg_safe(chan, &sg, 1, dir, flags, config);
+}
+
 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
 struct rio_dma_ext;
 static inline struct dma_async_tx_descriptor *dmaengine_prep_rio_sg(
diff --git a/include/linux/edac.h b/include/linux/edac.h
index e6b4e51130e5f..f7a8218f9cc0f 100644
--- a/include/linux/edac.h
+++ b/include/linux/edac.h
@@ -598,9 +598,6 @@ struct mem_ctl_info {
 	int op_state;
 
 	struct dentry *debugfs;
-	u8 fake_inject_layer[EDAC_MAX_LAYERS];
-	bool fake_inject_ue;
-	u16 fake_inject_count;
 
 	/*
 	 * Memory Controller hierarchy
diff --git a/include/linux/efi.h b/include/linux/efi.h
index ccbc35479684a..b3c83516593d1 100644
--- a/include/linux/efi.h
+++ b/include/linux/efi.h
@@ -1212,8 +1212,8 @@ efi_call_acpi_prm_handler(efi_status_t (__efiapi *handler_addr)(u64, void *),
 
 /*
  * efi_runtime_service() function identifiers.
- * "NONE" is used by efi_recover_from_page_fault() to check if the page
- * fault happened while executing an efi runtime service.
+ * "NONE" is used by efi_crash_gracefully_on_page_fault() to check if the
+ * page fault happened while executing an efi runtime service.
  */
 enum efi_rts_ids {
 	EFI_NONE,
@@ -1256,6 +1256,8 @@ extern struct efi_runtime_work efi_rts_work;
 /* Workqueue to queue EFI Runtime Services */
 extern struct workqueue_struct *efi_rts_wq;
 
+void __noreturn efi_rts_park_worker(void);
+
 struct linux_efi_memreserve {
 	int		size;			// allocated size of the array
 	atomic_t	count;			// number of entries used
diff --git a/include/linux/efs_vh.h b/include/linux/efs_vh.h
deleted file mode 100644
index 206c5270f7b89..0000000000000
--- a/include/linux/efs_vh.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * efs_vh.h
- *
- * Copyright (c) 1999 Al Smith
- *
- * Portions derived from IRIX header files (c) 1985 MIPS Computer Systems, Inc.
- */
-
-#ifndef __EFS_VH_H__
-#define __EFS_VH_H__
-
-#define VHMAGIC		0xbe5a941	/* volume header magic number */
-#define NPARTAB		16		/* 16 unix partitions */
-#define NVDIR		15		/* max of 15 directory entries */
-#define BFNAMESIZE	16		/* max 16 chars in boot file name */
-#define VDNAMESIZE	8
-
-struct volume_directory {
-	char	vd_name[VDNAMESIZE];	/* name */
-	__be32	vd_lbn;			/* logical block number */
-	__be32	vd_nbytes;		/* file length in bytes */
-};
-
-struct partition_table {	/* one per logical partition */
-	__be32	pt_nblks;	/* # of logical blks in partition */
-	__be32	pt_firstlbn;	/* first lbn of partition */
-	__be32	pt_type;	/* use of partition */
-};
-
-struct volume_header {
-	__be32	vh_magic;			/* identifies volume header */
-	__be16	vh_rootpt;			/* root partition number */
-	__be16	vh_swappt;			/* swap partition number */
-	char	vh_bootfile[BFNAMESIZE];	/* name of file to boot */
-	char	pad[48];			/* device param space */
-	struct volume_directory vh_vd[NVDIR];	/* other vol hdr contents */
-	struct partition_table  vh_pt[NPARTAB];	/* device partition layout */
-	__be32	vh_csum;			/* volume header checksum */
-	__be32	vh_fill;			/* fill out to 512 bytes */
-};
-
-/* partition type sysv is used for EFS format CD-ROM partitions */
-#define SGI_SYSV	0x05
-#define SGI_EFS		0x07
-#define IS_EFS(x)	(((x) == SGI_EFS) || ((x) == SGI_SYSV))
-
-struct pt_types {
-	int	pt_type;
-	char	*pt_name;
-};
-
-#endif /* __EFS_VH_H__ */
-
diff --git a/include/linux/entry-common.h b/include/linux/entry-common.h
index 416a3352261fe..6574b7183c017 100644
--- a/include/linux/entry-common.h
+++ b/include/linux/entry-common.h
@@ -6,9 +6,11 @@
 #include <linux/irq-entry-common.h>
 #include <linux/livepatch.h>
 #include <linux/ptrace.h>
+#include <linux/randomize_kstack.h>
 #include <linux/resume_user_mode.h>
 #include <linux/seccomp.h>
 #include <linux/sched.h>
+#include <linux/syscall_user_dispatch.h>
 
 #include <asm/entry-common.h>
 #include <asm/syscall.h>
@@ -18,7 +20,7 @@
 #endif
 
 /*
- * SYSCALL_WORK flags handled in syscall_enter_from_user_mode()
+ * SYSCALL_WORK flags handled in syscall_enter_from_user_mode_work()
  */
 #define SYSCALL_WORK_ENTER	(SYSCALL_WORK_SECCOMP |			\
 				 SYSCALL_WORK_SYSCALL_TRACEPOINT |	\
@@ -37,42 +39,32 @@
 				 SYSCALL_WORK_SYSCALL_EXIT_TRAP)
 
 /**
- * arch_ptrace_report_syscall_entry - Architecture specific ptrace_report_syscall_entry() wrapper
+ * arch_ptrace_report_syscall_permit_entry - Architecture specific wrapper for
+ *					     ptrace_report_syscall_permit_entry()
  * @regs: Pointer to the register state at syscall entry
  *
- * Invoked from syscall_trace_enter() to wrap ptrace_report_syscall_entry().
+ * Invoked from syscall_trace_enter() to wrap ptrace_report_syscall_permit_entry().
  *
- * This allows architecture specific ptrace_report_syscall_entry()
+ * This allows architecture specific ptrace_report_syscall_permit_entry()
  * implementations. If not defined by the architecture this falls back to
- * to ptrace_report_syscall_entry().
+ * to ptrace_report_syscall_permit_entry().
  */
-static __always_inline int arch_ptrace_report_syscall_entry(struct pt_regs *regs);
+static __always_inline bool arch_ptrace_report_syscall_permit_entry(struct pt_regs *regs);
 
-#ifndef arch_ptrace_report_syscall_entry
-static __always_inline int arch_ptrace_report_syscall_entry(struct pt_regs *regs)
+#ifndef arch_ptrace_report_syscall_permit_entry
+static __always_inline bool arch_ptrace_report_syscall_permit_entry(struct pt_regs *regs)
 {
-	return ptrace_report_syscall_entry(regs);
+	return ptrace_report_syscall_permit_entry(regs);
 }
 #endif
 
-bool syscall_user_dispatch(struct pt_regs *regs);
-long trace_syscall_enter(struct pt_regs *regs, long syscall);
+void trace_syscall_enter(struct pt_regs *regs);
 void trace_syscall_exit(struct pt_regs *regs, long ret);
+void syscall_enter_audit(struct pt_regs *regs);
 
-static inline void syscall_enter_audit(struct pt_regs *regs, long syscall)
+static __always_inline long syscall_trace_enter(struct pt_regs *regs, unsigned long work,
+						long syscall)
 {
-	if (unlikely(audit_context())) {
-		unsigned long args[6];
-
-		syscall_get_arguments(current, regs, args);
-		audit_syscall_entry(syscall, args[0], args[1], args[2], args[3]);
-	}
-}
-
-static __always_inline long syscall_trace_enter(struct pt_regs *regs, unsigned long work)
-{
-	long syscall, ret = 0;
-
 	/*
 	 * Handle Syscall User Dispatch.  This must comes first, since
 	 * the ABI here can be something that doesn't make sense for
@@ -80,7 +72,7 @@ static __always_inline long syscall_trace_enter(struct pt_regs *regs, unsigned l
 	 */
 	if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) {
 		if (syscall_user_dispatch(regs))
-			return -1L;
+			return false;
 	}
 
 	/*
@@ -89,31 +81,31 @@ static __always_inline long syscall_trace_enter(struct pt_regs *regs, unsigned l
 	 * through hrtimer_interrupt().
 	 */
 	if (work & SYSCALL_WORK_SYSCALL_RSEQ_SLICE)
-		rseq_syscall_enter_work(syscall_get_nr(current, regs));
+		rseq_syscall_enter_work(syscall);
 
 	/* Handle ptrace */
 	if (work & (SYSCALL_WORK_SYSCALL_TRACE | SYSCALL_WORK_SYSCALL_EMU)) {
-		ret = arch_ptrace_report_syscall_entry(regs);
-		if (ret || (work & SYSCALL_WORK_SYSCALL_EMU))
-			return -1L;
+		if (!arch_ptrace_report_syscall_permit_entry(regs) ||
+		    (work & SYSCALL_WORK_SYSCALL_EMU))
+			return false;
+
+		/* ptrace might have changed work flags */
+		work = READ_ONCE(current_thread_info()->syscall_work);
 	}
 
 	/* Do seccomp after ptrace, to catch any tracer changes. */
 	if (work & SYSCALL_WORK_SECCOMP) {
-		ret = __secure_computing();
-		if (ret == -1L)
-			return ret;
+		if (!__seccomp_permit_syscall())
+			return false;
 	}
 
-	/* Either of the above might have changed the syscall number */
-	syscall = syscall_get_nr(current, regs);
-
 	if (unlikely(work & SYSCALL_WORK_SYSCALL_TRACEPOINT))
-		syscall = trace_syscall_enter(regs, syscall);
+		trace_syscall_enter(regs);
 
-	syscall_enter_audit(regs, syscall);
+	if (unlikely(audit_context()))
+		syscall_enter_audit(regs);
 
-	return ret ? : syscall;
+	return true;
 }
 
 /**
@@ -122,36 +114,63 @@ static __always_inline long syscall_trace_enter(struct pt_regs *regs, unsigned l
  * @regs:	Pointer to currents pt_regs
  * @syscall:	The syscall number
  *
- * Invoked from architecture specific syscall entry code with interrupts
- * enabled after invoking enter_from_user_mode(), enabling interrupts and
- * extra architecture specific work.
+ * Invoked from architecture specific syscall entry code with interrupts enabled
+ * after invoking enter_from_user_mode(), enabling interrupts and extra
+ * architecture specific work with the syscall return value preset to -ENOSYS.
  *
- * Returns: The original or a modified syscall number
+ * Returns: True if the syscall should be invoked, False otherwise.
  *
- * If the returned syscall number is -1 then the syscall should be
- * skipped. In this case the caller may invoke syscall_set_error() or
- * syscall_set_return_value() first.  If neither of those are called and -1
- * is returned, then the syscall will fail with ENOSYS.
+ * If the return value is false, the caller must skip the syscall and leave the
+ * syscall return value unmodified as it might have been set by one of the entry
+ * work functions.
  *
  * It handles the following work items:
  *
  *  1) syscall_work flag dependent invocations of
- *     ptrace_report_syscall_entry(), __secure_computing(), trace_sys_enter()
+ *     ptrace_report_syscall_permit_entry(), __seccomp_permit_syscall(), trace_sys_enter()
  *  2) Invocation of audit_syscall_entry()
  */
-static __always_inline long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall)
+static __always_inline bool syscall_enter_from_user_mode_work(struct pt_regs *regs, long *syscall)
 {
 	unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
 
-	if (work & SYSCALL_WORK_ENTER)
-		syscall = syscall_trace_enter(regs, work);
+	if (!(work & SYSCALL_WORK_ENTER))
+		return true;
+
+	if (unlikely(!syscall_trace_enter(regs, work, *syscall)))
+		return false;
 
-	return syscall;
+	/* Reread the syscall number as it might have been modified */
+	*syscall = syscall_get_nr(current, regs);
+
+	return true;
 }
 
 /**
- * syscall_enter_from_user_mode - Establish state and check and handle work
- *				  before invoking a syscall
+ * enter_from_user_mode_randomize_stack - Establish state and add stack randomization
+ *					  before invoking syscall_enter_from_user_mode_work()
+ * @regs:	Pointer to currents pt_regs
+ *
+ * Invoked from architecture specific syscall entry code with interrupts
+ * disabled. The calling code has to be non-instrumentable. When the function
+ * returns all state is correct, interrupts are still disabled and the
+ * subsequent functions can be instrumented.
+ *
+ * Implemented as a macro so that the stack randomization is effective
+ * throughout the function in which it is invoked. An inline would only make it
+ * effective in the scope of the inline function.
+ */
+#define enter_from_user_mode_randomize_stack(regs)			\
+do {									\
+	enter_from_user_mode(regs);					\
+	instrumentation_begin();					\
+	add_random_kstack_offset_irqsoff();				\
+	instrumentation_end();						\
+} while (0)
+
+/**
+ * syscall_enter_from_user_mode_randomize_stack - Establish state and check and handle work
+ *						  before invoking a syscall
  * @regs:	Pointer to currents pt_regs
  * @syscall:	The syscall number
  *
@@ -160,31 +179,32 @@ static __always_inline long syscall_enter_from_user_mode_work(struct pt_regs *re
  * function returns all state is correct, interrupts are enabled and the
  * subsequent functions can be instrumented.
  *
- * This is the combination of enter_from_user_mode() and
+ * This is the combination of enter_from_user_mode_randomize_stack() and
  * syscall_enter_from_user_mode_work() to be used when there is no
  * architecture specific work to be done between the two.
  *
  * Returns: The original or a modified syscall number. See
  * syscall_enter_from_user_mode_work() for further explanation.
+ *
+ * Implemented as a macro to make stack randomization effective in the calling
+ * scope.
  */
-static __always_inline long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall)
-{
-	long ret;
-
-	enter_from_user_mode(regs);
-
-	instrumentation_begin();
-	local_irq_enable();
-	ret = syscall_enter_from_user_mode_work(regs, syscall);
-	instrumentation_end();
-
-	return ret;
-}
+#define syscall_enter_from_user_mode_randomize_stack(regs, syscall)	\
+({									\
+	enter_from_user_mode_randomize_stack(regs);			\
+									\
+	instrumentation_begin();					\
+	local_irq_enable();						\
+	long _ret = syscall_enter_from_user_mode_work(regs, syscall);	\
+	instrumentation_end();						\
+									\
+	_ret;								\
+})
 
 /*
- * If SYSCALL_EMU is set, then the only reason to report is when
- * SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP).  This syscall
- * instruction has been already reported in syscall_enter_from_user_mode().
+ * If SYSCALL_EMU is set, then the only reason to report is when SINGLESTEP is
+ * set (i.e. PTRACE_SYSEMU_SINGLESTEP).  This syscall instruction has been
+ * already reported in syscall_enter_from_user_mode_work().
  */
 static __always_inline bool report_single_step(unsigned long work)
 {
@@ -232,10 +252,8 @@ static __always_inline void syscall_exit_work(struct pt_regs *regs, unsigned lon
 	 * of these syscalls is unknown.
 	 */
 	if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) {
-		if (unlikely(current->syscall_dispatch.on_dispatch)) {
-			current->syscall_dispatch.on_dispatch = false;
+		if (syscall_user_dispatch_clear_on_dispatch())
 			return;
-		}
 	}
 
 	audit_syscall_exit(regs);
diff --git a/include/linux/firmware/qcom/qcom_pas.h b/include/linux/firmware/qcom/qcom_pas.h
new file mode 100644
index 0000000000000..65b1c9564458f
--- /dev/null
+++ b/include/linux/firmware/qcom/qcom_pas.h
@@ -0,0 +1,43 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2010-2015, 2018-2019 The Linux Foundation. All rights reserved.
+ * Copyright (C) 2015 Linaro Ltd.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __QCOM_PAS_H
+#define __QCOM_PAS_H
+
+#include <linux/err.h>
+#include <linux/types.h>
+
+struct qcom_pas_context {
+	struct device *dev;
+	u32 pas_id;
+	phys_addr_t mem_phys;
+	size_t mem_size;
+	void *ptr;
+	dma_addr_t phys;
+	ssize_t size;
+	bool use_tzmem;
+};
+
+bool qcom_pas_is_available(void);
+struct qcom_pas_context *devm_qcom_pas_context_alloc(struct device *dev,
+						     u32 pas_id,
+						     phys_addr_t mem_phys,
+						     size_t mem_size);
+int qcom_pas_init_image(u32 pas_id, const void *metadata, size_t size,
+			struct qcom_pas_context *ctx);
+struct resource_table *qcom_pas_get_rsc_table(struct qcom_pas_context *ctx,
+					      void *input_rt, size_t input_rt_size,
+					      size_t *output_rt_size);
+int qcom_pas_mem_setup(u32 pas_id, phys_addr_t addr, phys_addr_t size);
+int qcom_pas_auth_and_reset(u32 pas_id);
+int qcom_pas_prepare_and_auth_reset(struct qcom_pas_context *ctx);
+int qcom_pas_set_remote_state(u32 state, u32 pas_id);
+int qcom_pas_shutdown(u32 pas_id);
+bool qcom_pas_supported(u32 pas_id);
+void qcom_pas_metadata_release(struct qcom_pas_context *ctx);
+
+#endif /* __QCOM_PAS_H */
diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h
index 7e27b0f7bf7ed..347df66ee176a 100644
--- a/include/linux/firmware/xlnx-zynqmp.h
+++ b/include/linux/firmware/xlnx-zynqmp.h
@@ -644,6 +644,8 @@ int zynqmp_pm_get_node_status(const u32 node, u32 *const status,
 			      u32 *const requirements, u32 *const usage);
 int zynqmp_pm_get_rpu_node_status(const u32 node, u32 *const status,
 				  u32 *const requirements, u32 *const usage);
+int zynqmp_pm_start_rpu(const u32 node, const u64 bootaddr);
+int zynqmp_pm_stop_rpu(const u32 node);
 int zynqmp_pm_set_sd_config(u32 node, enum pm_sd_config_type config, u32 value);
 int zynqmp_pm_set_gem_config(u32 node, enum pm_gem_config_type config,
 			     u32 value);
@@ -960,6 +962,16 @@ static inline int zynqmp_pm_get_rpu_node_status(const u32 node, u32 *const statu
 	return -ENODEV;
 }
 
+static inline int zynqmp_pm_start_rpu(const u32 node, const u64 bootaddr)
+{
+	return -ENODEV;
+}
+
+static inline int zynqmp_pm_stop_rpu(const u32 node)
+{
+	return -ENODEV;
+}
+
 static inline int zynqmp_pm_set_sd_config(u32 node,
 					  enum pm_sd_config_type config,
 					  u32 value)
diff --git a/include/linux/font.h b/include/linux/font.h
index 6845f02d739a0..ea23b727388ba 100644
--- a/include/linux/font.h
+++ b/include/linux/font.h
@@ -101,6 +101,9 @@ font_data_t *font_data_import(const struct console_font *font, unsigned int vpit
 void font_data_get(font_data_t *fd);
 bool font_data_put(font_data_t *fd);
 unsigned int font_data_size(font_data_t *fd);
+const unsigned char *font_data_glyph_buf(font_data_t *fd,
+					 unsigned int width, unsigned int vpitch,
+					 unsigned int c);
 bool font_data_is_equal(font_data_t *lhs, font_data_t *rhs);
 int font_data_export(font_data_t *fd, struct console_font *font, unsigned int vpitch);
 
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 50ce731a2b78f..6e4b8e19af05f 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -1916,8 +1916,6 @@ struct dir_context {
 struct io_uring_cmd;
 struct offset_ctx;
 
-typedef unsigned int __bitwise fop_flags_t;
-
 struct file_operations {
 	struct module *owner;
 	fop_flags_t fop_flags;
@@ -2002,7 +2000,7 @@ struct inode_operations {
 	int (*readlink) (struct dentry *, char __user *,int);
 
 	int (*create) (struct mnt_idmap *, struct inode *,struct dentry *,
-		       umode_t, bool);
+		       umode_t);
 	int (*link) (struct dentry *,struct inode *,struct dentry *);
 	int (*unlink) (struct inode *,struct dentry *);
 	int (*symlink) (struct mnt_idmap *, struct inode *,struct dentry *,
diff --git a/include/linux/fs/super.h b/include/linux/fs/super.h
index 4056126781151..733d439f01edb 100644
--- a/include/linux/fs/super.h
+++ b/include/linux/fs/super.h
@@ -237,4 +237,12 @@ int thaw_super(struct super_block *super, enum freeze_holder who,
 
 int sb_init_dio_done_wq(struct super_block *sb);
 
+struct file;
+struct file *fs_bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder,
+				      struct super_block *sb);
+struct file *fs_bdev_file_open_by_path(const char *path, blk_mode_t mode,
+				       void *holder, struct super_block *sb);
+void fs_bdev_unregister(struct file *bdev_file, struct super_block *sb);
+void fs_bdev_file_release(struct file *bdev_file, struct super_block *sb);
+
 #endif /* _LINUX_FS_SUPER_H */
diff --git a/include/linux/fs/super_types.h b/include/linux/fs/super_types.h
index ef7941e9dc79a..c8172558750fb 100644
--- a/include/linux/fs/super_types.h
+++ b/include/linux/fs/super_types.h
@@ -30,6 +30,7 @@ struct mount;
 struct mtd_info;
 struct quotactl_ops;
 struct shrinker;
+struct super_dev;
 struct unicode_map;
 struct user_namespace;
 struct workqueue_struct;
@@ -132,6 +133,7 @@ struct super_operations {
 struct super_block {
 	struct list_head			s_list;		/* Keep this first */
 	dev_t					s_dev;		/* search index; _not_ kdev_t */
+	struct super_dev			*s_super_dev;	/* sget_fc()'s device table claim */
 	unsigned char				s_blocksize_bits;
 	unsigned long				s_blocksize;
 	loff_t					s_maxbytes;	/* Max file size */
@@ -145,7 +147,7 @@ struct super_block {
 	unsigned long				s_magic;
 	struct dentry				*s_root;
 	struct rw_semaphore			s_umount;
-	int					s_count;
+	refcount_t				s_passive;
 	atomic_t				s_active;
 #ifdef CONFIG_SECURITY
 	void					*s_security;
diff --git a/include/linux/fs_struct.h b/include/linux/fs_struct.h
index 0070764b790a3..97eef8d3863dd 100644
--- a/include/linux/fs_struct.h
+++ b/include/linux/fs_struct.h
@@ -6,6 +6,7 @@
 #include <linux/path.h>
 #include <linux/spinlock.h>
 #include <linux/seqlock.h>
+#include <linux/vfsdebug.h>
 
 struct fs_struct {
 	int users;
@@ -16,6 +17,7 @@ struct fs_struct {
 } __randomize_layout;
 
 extern struct kmem_cache *fs_cachep;
+extern struct fs_struct *userspace_init_fs;
 
 extern void exit_fs(struct task_struct *);
 extern void set_fs_root(struct fs_struct *, const struct path *);
@@ -40,6 +42,8 @@ static inline void get_fs_pwd(struct fs_struct *fs, struct path *pwd)
 	read_sequnlock_excl(&fs->seq);
 }
 
+struct fs_struct *switch_fs_struct(struct fs_struct *new_fs);
+
 extern bool current_chrooted(void);
 
 static inline int current_umask(void)
@@ -47,4 +51,34 @@ static inline int current_umask(void)
 	return current->fs->umask;
 }
 
+/*
+ * Temporarily use userspace_init_fs for path resolution in kthreads.
+ * Callers should use scoped_with_init_fs() which automatically
+ * restores the original fs_struct at scope exit.
+ */
+static inline struct fs_struct *__override_init_fs(void)
+{
+	struct fs_struct *old_fs;
+
+	old_fs = current->fs;
+	WRITE_ONCE(current->fs, userspace_init_fs);
+	return old_fs;
+}
+
+static inline void __revert_init_fs(struct fs_struct *old_fs)
+{
+	VFS_WARN_ON_ONCE(current->fs != userspace_init_fs);
+	WRITE_ONCE(current->fs, old_fs);
+}
+
+DEFINE_CLASS(__override_init_fs,
+	     struct fs_struct *,
+	     __revert_init_fs(_T),
+	     __override_init_fs(), void)
+
+#define scoped_with_init_fs() \
+	scoped_class(__override_init_fs, __UNIQUE_ID(label))
+
+void __init init_userspace_fs(void);
+
 #endif /* _LINUX_FS_STRUCT_H */
diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index cdf95a9f0b87c..4d57e9c0bf204 100644
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -17,28 +17,6 @@ struct mempolicy;
 #define __default_gfp(a,b,...) b
 #define default_gfp(...) __default_gfp(,##__VA_ARGS__,GFP_KERNEL)
 
-/* Convert GFP flags to their corresponding migrate type */
-#define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE)
-#define GFP_MOVABLE_SHIFT 3
-
-static inline int gfp_migratetype(const gfp_t gfp_flags)
-{
-	VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK);
-	BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE);
-	BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE);
-	BUILD_BUG_ON((___GFP_RECLAIMABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_RECLAIMABLE);
-	BUILD_BUG_ON(((___GFP_MOVABLE | ___GFP_RECLAIMABLE) >>
-		      GFP_MOVABLE_SHIFT) != MIGRATE_HIGHATOMIC);
-
-	if (unlikely(page_group_by_mobility_disabled))
-		return MIGRATE_UNMOVABLE;
-
-	/* Group based on mobility */
-	return (__force unsigned long)(gfp_flags & GFP_MOVABLE_MASK) >> GFP_MOVABLE_SHIFT;
-}
-#undef GFP_MOVABLE_MASK
-#undef GFP_MOVABLE_SHIFT
-
 static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags)
 {
 	return !!(gfp_flags & __GFP_DIRECT_RECLAIM);
@@ -226,10 +204,6 @@ static inline void arch_free_page(struct page *page, int order) { }
 static inline void arch_alloc_page(struct page *page, int order) { }
 #endif
 
-struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, int preferred_nid,
-		nodemask_t *nodemask);
-#define __alloc_pages(...)			alloc_hooks(__alloc_pages_noprof(__VA_ARGS__))
-
 struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_nid,
 		nodemask_t *nodemask);
 #define __folio_alloc(...)			alloc_hooks(__folio_alloc_noprof(__VA_ARGS__))
@@ -278,21 +252,6 @@ static inline void warn_if_node_offline(int this_node, gfp_t gfp_mask)
 	dump_stack();
 }
 
-/*
- * Allocate pages, preferring the node given as nid. The node must be valid and
- * online. For more general interface, see alloc_pages_node().
- */
-static inline struct page *
-__alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order)
-{
-	VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES);
-	warn_if_node_offline(nid, gfp_mask);
-
-	return __alloc_pages_noprof(gfp_mask, order, nid, NULL);
-}
-
-#define  __alloc_pages_node(...)		alloc_hooks(__alloc_pages_node_noprof(__VA_ARGS__))
-
 static inline
 struct folio *__folio_alloc_node_noprof(gfp_t gfp, unsigned int order, int nid)
 {
@@ -309,14 +268,7 @@ struct folio *__folio_alloc_node_noprof(gfp_t gfp, unsigned int order, int nid)
  * prefer the current CPU's closest node. Otherwise node must be valid and
  * online.
  */
-static inline struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask,
-						   unsigned int order)
-{
-	if (nid == NUMA_NO_NODE)
-		nid = numa_mem_id();
-
-	return __alloc_pages_node_noprof(nid, gfp_mask, order);
-}
+struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order);
 
 #define  alloc_pages_node(...)			alloc_hooks(alloc_pages_node_noprof(__VA_ARGS__))
 
@@ -395,10 +347,6 @@ extern void free_pages(unsigned long addr, unsigned int order);
 #define __free_page(page) __free_pages((page), 0)
 #define free_page(addr) free_pages((addr), 0)
 
-void page_alloc_init_cpuhp(void);
-bool decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp);
-void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp);
-void drain_all_pages(struct zone *zone);
 void drain_local_pages(struct zone *zone);
 
 void page_alloc_init_late(void);
diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h
index 54ca0c88bab6e..190191411009f 100644
--- a/include/linux/gfp_types.h
+++ b/include/linux/gfp_types.h
@@ -55,7 +55,6 @@ enum {
 #ifdef CONFIG_LOCKDEP
 	___GFP_NOLOCKDEP_BIT,
 #endif
-	___GFP_NO_OBJ_EXT_BIT,
 	___GFP_LAST_BIT
 };
 
@@ -96,7 +95,6 @@ enum {
 #else
 #define ___GFP_NOLOCKDEP	0
 #endif
-#define ___GFP_NO_OBJ_EXT       BIT(___GFP_NO_OBJ_EXT_BIT)
 
 /*
  * Physical address zone modifiers (see linux/mmzone.h - low four bits)
@@ -136,18 +134,14 @@ enum {
  * %__GFP_THISNODE forces the allocation to be satisfied from the requested
  * node with no fallbacks or placement policy enforcements.
  *
- * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg.
- *
- * %__GFP_NO_OBJ_EXT causes slab allocation to have no object extension.
- * mark_obj_codetag_empty() should be called upon freeing for objects allocated
- * with this flag to indicate that their NULL tags are expected and normal.
+ * %__GFP_ACCOUNT causes the allocation to be accounted to the active
+ * cgroup context.
  */
 #define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE)
 #define __GFP_WRITE	((__force gfp_t)___GFP_WRITE)
 #define __GFP_HARDWALL   ((__force gfp_t)___GFP_HARDWALL)
 #define __GFP_THISNODE	((__force gfp_t)___GFP_THISNODE)
 #define __GFP_ACCOUNT	((__force gfp_t)___GFP_ACCOUNT)
-#define __GFP_NO_OBJ_EXT   ((__force gfp_t)___GFP_NO_OBJ_EXT)
 
 /**
  * DOC: Watermark modifiers
@@ -320,7 +314,7 @@ enum {
  * %ZONE_NORMAL or a lower zone for direct access but can direct reclaim.
  *
  * %GFP_KERNEL_ACCOUNT is the same as GFP_KERNEL, except the allocation is
- * accounted to kmemcg.
+ * accounted to the active cgroup context.
  *
  * %GFP_NOWAIT is for kernel allocations that should not stall for direct
  * reclaim, start physical IO or use any filesystem callback.  It is very
diff --git a/include/linux/goldfish.h b/include/linux/goldfish.h
index bcc17f95b9066..98a4719c6776c 100644
--- a/include/linux/goldfish.h
+++ b/include/linux/goldfish.h
@@ -2,8 +2,6 @@
 #ifndef __LINUX_GOLDFISH_H
 #define __LINUX_GOLDFISH_H
 
-#include <linux/kernel.h>
-#include <linux/types.h>
 #include <linux/io.h>
 
 /* Helpers for Goldfish virtual platform */
@@ -15,26 +13,4 @@
 #define gf_iowrite32 iowrite32
 #endif
 
-static inline void gf_write_ptr(const void *ptr, void __iomem *portl,
-				void __iomem *porth)
-{
-	const unsigned long addr = (unsigned long)ptr;
-
-	gf_iowrite32(lower_32_bits(addr), portl);
-#ifdef CONFIG_64BIT
-	gf_iowrite32(upper_32_bits(addr), porth);
-#endif
-}
-
-static inline void gf_write_dma_addr(const dma_addr_t addr,
-				     void __iomem *portl,
-				     void __iomem *porth)
-{
-	gf_iowrite32(lower_32_bits(addr), portl);
-#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
-	gf_iowrite32(upper_32_bits(addr), porth);
-#endif
-}
-
-
 #endif /* __LINUX_GOLDFISH_H */
diff --git a/include/linux/gpio/machine.h b/include/linux/gpio/machine.h
index 5eb88f5d0630f..1a141056716f8 100644
--- a/include/linux/gpio/machine.h
+++ b/include/linux/gpio/machine.h
@@ -54,7 +54,7 @@ static struct gpiod_lookup_table _name = {				\
 	.dev_id = _dev_id,						\
 	.table = {							\
 		GPIO_LOOKUP(_key, _chip_hwnum, _con_id, _flags),	\
-		{},							\
+		{ }							\
 	},								\
 }
 
diff --git a/include/linux/gpu_buddy.h b/include/linux/gpu_buddy.h
index 71941a0396485..e037714563d82 100644
--- a/include/linux/gpu_buddy.h
+++ b/include/linux/gpu_buddy.h
@@ -173,6 +173,21 @@ struct gpu_buddy {
 	 * that fits in the remaining space.
 	 */
 	struct gpu_buddy_block **roots;
+	/*
+	 * Per-order free block scoreboard: free_scoreboard[order] holds the
+	 * number of blocks of that order currently in the free state.
+	 * Incremented in mark_free(), decremented wherever rbtree_remove() is
+	 * called on a free block.
+	 */
+	u64 *free_scoreboard;
+	/*
+	 * Per-order used block scoreboard: used_scoreboard[order] holds the
+	 * number of blocks of that order currently in the allocated state.
+	 * Incremented in mark_allocated(), decremented in mark_free() (guarded
+	 * by gpu_buddy_block_is_allocated()) and in __gpu_buddy_free() when an
+	 * allocated block is consumed directly during buddy coalescing.
+	 */
+	u64 *used_scoreboard;
 /* public: */
 	unsigned int n_roots;
 	unsigned int max_order;
diff --git a/include/linux/hmm.h b/include/linux/hmm.h
index db75ffc949a7a..6f04e3932f5b0 100644
--- a/include/linux/hmm.h
+++ b/include/linux/hmm.h
@@ -123,6 +123,8 @@ struct hmm_range {
  * Please see Documentation/mm/hmm.rst for how to use the range API.
  */
 int hmm_range_fault(struct hmm_range *range);
+int hmm_range_fault_unlocked_timeout(struct hmm_range *range,
+				     unsigned long timeout);
 
 /*
  * HMM_RANGE_DEFAULT_TIMEOUT - default timeout (ms) when waiting for a range
diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h
index 6862dea0acc52..29072d89e5cb3 100644
--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -12,7 +12,6 @@
 #ifndef _LINUX_HRTIMER_H
 #define _LINUX_HRTIMER_H
 
-#include <linux/hrtimer_defs.h>
 #include <linux/hrtimer_rearm.h>
 #include <linux/hrtimer_types.h>
 #include <linux/init.h>
@@ -287,37 +286,8 @@ static inline bool hrtimer_is_queued(struct hrtimer *timer)
 	return READ_ONCE(timer->is_queued);
 }
 
-/*
- * Helper function to check, whether the timer is running the callback
- * function
- */
-static inline int hrtimer_callback_running(struct hrtimer *timer)
-{
-	return timer->base->running == timer;
-}
-
-/**
- * hrtimer_update_function - Update the timer's callback function
- * @timer:	Timer to update
- * @function:	New callback function
- *
- * Only safe to call if the timer is not enqueued. Can be called in the callback function if the
- * timer is not enqueued at the same time (see the comments above HRTIMER_STATE_ENQUEUED).
- */
-static inline void hrtimer_update_function(struct hrtimer *timer,
-					   enum hrtimer_restart (*function)(struct hrtimer *))
-{
-#ifdef CONFIG_PROVE_LOCKING
-	guard(raw_spinlock_irqsave)(&timer->base->cpu_base->lock);
-
-	if (WARN_ON_ONCE(hrtimer_is_queued(timer)))
-		return;
-
-	if (WARN_ON_ONCE(!function))
-		return;
-#endif
-	ACCESS_PRIVATE(timer, function) = function;
-}
+void hrtimer_update_function(struct hrtimer *timer,
+			     enum hrtimer_restart (*function)(struct hrtimer *));
 
 /* Forward a hrtimer so it expires after now: */
 extern u64
diff --git a/include/linux/hrtimer_bases.h b/include/linux/hrtimer_bases.h
new file mode 100644
index 0000000000000..70b99e651168b
--- /dev/null
+++ b/include/linux/hrtimer_bases.h
@@ -0,0 +1,123 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_HRTIMER_BASES_H
+#define _LINUX_HRTIMER_BASES_H
+
+#include <linux/hrtimer.h>
+#include <linux/ktime.h>
+#include <linux/timerqueue.h>
+#include <linux/seqlock.h>
+
+#ifdef CONFIG_64BIT
+# define __hrtimer_clock_base_align	____cacheline_aligned
+#else
+# define __hrtimer_clock_base_align
+#endif
+
+/**
+ * struct hrtimer_clock_base - the timer base for a specific clock
+ * @cpu_base:		per cpu clock base
+ * @index:		clock type index for per_cpu support when moving a
+ *			timer to a base on another cpu.
+ * @clockid:		clock id for per_cpu support
+ * @seq:		seqcount around __run_hrtimer
+ * @expires_next:	Absolute time of the next event in this clock base
+ * @running:		pointer to the currently running hrtimer
+ * @active:		red black tree root node for the active timers
+ * @offset:		offset of this clock to the monotonic base
+ */
+struct hrtimer_clock_base {
+	struct hrtimer_cpu_base		*cpu_base;
+	const unsigned int		index;
+	const clockid_t			clockid;
+	seqcount_raw_spinlock_t		seq;
+	ktime_t				expires_next;
+	struct hrtimer			*running;
+	struct timerqueue_linked_head	active;
+	ktime_t				offset;
+} __hrtimer_clock_base_align;
+
+enum hrtimer_base_type {
+	HRTIMER_BASE_MONOTONIC,
+	HRTIMER_BASE_REALTIME,
+	HRTIMER_BASE_BOOTTIME,
+	HRTIMER_BASE_TAI,
+	HRTIMER_BASE_MONOTONIC_SOFT,
+	HRTIMER_BASE_REALTIME_SOFT,
+	HRTIMER_BASE_BOOTTIME_SOFT,
+	HRTIMER_BASE_TAI_SOFT,
+	HRTIMER_MAX_CLOCK_BASES
+};
+
+/**
+ * struct hrtimer_cpu_base - the per cpu clock bases
+ * @lock:			lock protecting the base and associated clock bases and timers
+ * @cpu:			cpu number
+ * @active_bases:		Bitfield to mark bases with active timers
+ * @clock_was_set_seq:		Sequence counter of clock was set events
+ * @hres_active:		State of high resolution mode
+ * @deferred_rearm:		A deferred rearm is pending
+ * @deferred_needs_update:	The deferred rearm must re-evaluate the first timer
+ * @hang_detected:		The last hrtimer interrupt detected a hang
+ * @softirq_activated:		displays, if the softirq is raised - update of softirq
+ *				related settings is not required then.
+ * @nr_events:			Total number of hrtimer interrupt events
+ * @nr_retries:			Total number of hrtimer interrupt retries
+ * @nr_hangs:			Total number of hrtimer interrupt hangs
+ * @max_hang_time:		Maximum time spent in hrtimer_interrupt
+ * @softirq_expiry_lock:	Lock which is taken while softirq based hrtimer are expired
+ * @online:			CPU is online from an hrtimers point of view
+ * @timer_waiters:		A hrtimer_cancel() waiters for the timer callback to finish.
+ * @expires_next:		Absolute time of the next event, is required for remote
+ *				hrtimer enqueue; it is the total first expiry time (hard
+ *				and soft hrtimer are taken into account)
+ * @next_timer:			Pointer to the first expiring timer
+ * @softirq_expires_next:	Time to check, if soft queues needs also to be expired
+ * @softirq_next_timer:		Pointer to the first expiring softirq based timer
+ * @deferred_expires_next:	Cached expires next value for deferred rearm
+ * @clock_base:			Array of clock bases for this cpu
+ *
+ * Note: next_timer is just an optimization for __remove_hrtimer().
+ *	 Do not dereference the pointer because it is not reliable on
+ *	 cross cpu removals.
+ */
+struct hrtimer_cpu_base {
+	raw_spinlock_t			lock;
+	unsigned int			cpu;
+	unsigned int			active_bases;
+	unsigned int			clock_was_set_seq;
+	bool				hres_active;
+	bool				deferred_rearm;
+	bool				deferred_needs_update;
+	bool				hang_detected;
+	bool				softirq_activated;
+	bool				online;
+#ifdef CONFIG_HIGH_RES_TIMERS
+	unsigned int			nr_events;
+	unsigned short			nr_retries;
+	unsigned short			nr_hangs;
+	unsigned int			max_hang_time;
+#endif
+#ifdef CONFIG_PREEMPT_RT
+	spinlock_t			softirq_expiry_lock;
+	atomic_t			timer_waiters;
+#endif
+	ktime_t				expires_next;
+	struct hrtimer			*next_timer;
+	ktime_t				softirq_expires_next;
+	struct hrtimer			*softirq_next_timer;
+	ktime_t				deferred_expires_next;
+	struct hrtimer_clock_base	clock_base[HRTIMER_MAX_CLOCK_BASES];
+	call_single_data_t		csd;
+} ____cacheline_aligned;
+
+
+/*
+ * Helper function to check, whether the timer is running the callback
+ * function
+ */
+static inline int hrtimer_callback_running(struct hrtimer *timer)
+{
+	return timer->base->running == timer;
+}
+
+#endif
diff --git a/include/linux/hrtimer_defs.h b/include/linux/hrtimer_defs.h
deleted file mode 100644
index 52ed9e46ff136..0000000000000
--- a/include/linux/hrtimer_defs.h
+++ /dev/null
@@ -1,113 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _LINUX_HRTIMER_DEFS_H
-#define _LINUX_HRTIMER_DEFS_H
-
-#include <linux/ktime.h>
-#include <linux/timerqueue.h>
-#include <linux/seqlock.h>
-
-#ifdef CONFIG_64BIT
-# define __hrtimer_clock_base_align	____cacheline_aligned
-#else
-# define __hrtimer_clock_base_align
-#endif
-
-/**
- * struct hrtimer_clock_base - the timer base for a specific clock
- * @cpu_base:		per cpu clock base
- * @index:		clock type index for per_cpu support when moving a
- *			timer to a base on another cpu.
- * @clockid:		clock id for per_cpu support
- * @seq:		seqcount around __run_hrtimer
- * @expires_next:	Absolute time of the next event in this clock base
- * @running:		pointer to the currently running hrtimer
- * @active:		red black tree root node for the active timers
- * @offset:		offset of this clock to the monotonic base
- */
-struct hrtimer_clock_base {
-	struct hrtimer_cpu_base		*cpu_base;
-	const unsigned int		index;
-	const clockid_t			clockid;
-	seqcount_raw_spinlock_t		seq;
-	ktime_t				expires_next;
-	struct hrtimer			*running;
-	struct timerqueue_linked_head	active;
-	ktime_t				offset;
-} __hrtimer_clock_base_align;
-
-enum hrtimer_base_type {
-	HRTIMER_BASE_MONOTONIC,
-	HRTIMER_BASE_REALTIME,
-	HRTIMER_BASE_BOOTTIME,
-	HRTIMER_BASE_TAI,
-	HRTIMER_BASE_MONOTONIC_SOFT,
-	HRTIMER_BASE_REALTIME_SOFT,
-	HRTIMER_BASE_BOOTTIME_SOFT,
-	HRTIMER_BASE_TAI_SOFT,
-	HRTIMER_MAX_CLOCK_BASES
-};
-
-/**
- * struct hrtimer_cpu_base - the per cpu clock bases
- * @lock:			lock protecting the base and associated clock bases and timers
- * @cpu:			cpu number
- * @active_bases:		Bitfield to mark bases with active timers
- * @clock_was_set_seq:		Sequence counter of clock was set events
- * @hres_active:		State of high resolution mode
- * @deferred_rearm:		A deferred rearm is pending
- * @deferred_needs_update:	The deferred rearm must re-evaluate the first timer
- * @hang_detected:		The last hrtimer interrupt detected a hang
- * @softirq_activated:		displays, if the softirq is raised - update of softirq
- *				related settings is not required then.
- * @nr_events:			Total number of hrtimer interrupt events
- * @nr_retries:			Total number of hrtimer interrupt retries
- * @nr_hangs:			Total number of hrtimer interrupt hangs
- * @max_hang_time:		Maximum time spent in hrtimer_interrupt
- * @softirq_expiry_lock:	Lock which is taken while softirq based hrtimer are expired
- * @online:			CPU is online from an hrtimers point of view
- * @timer_waiters:		A hrtimer_cancel() waiters for the timer callback to finish.
- * @expires_next:		Absolute time of the next event, is required for remote
- *				hrtimer enqueue; it is the total first expiry time (hard
- *				and soft hrtimer are taken into account)
- * @next_timer:			Pointer to the first expiring timer
- * @softirq_expires_next:	Time to check, if soft queues needs also to be expired
- * @softirq_next_timer:		Pointer to the first expiring softirq based timer
- * @deferred_expires_next:	Cached expires next value for deferred rearm
- * @clock_base:			Array of clock bases for this cpu
- *
- * Note: next_timer is just an optimization for __remove_hrtimer().
- *	 Do not dereference the pointer because it is not reliable on
- *	 cross cpu removals.
- */
-struct hrtimer_cpu_base {
-	raw_spinlock_t			lock;
-	unsigned int			cpu;
-	unsigned int			active_bases;
-	unsigned int			clock_was_set_seq;
-	bool				hres_active;
-	bool				deferred_rearm;
-	bool				deferred_needs_update;
-	bool				hang_detected;
-	bool				softirq_activated;
-	bool				online;
-#ifdef CONFIG_HIGH_RES_TIMERS
-	unsigned int			nr_events;
-	unsigned short			nr_retries;
-	unsigned short			nr_hangs;
-	unsigned int			max_hang_time;
-#endif
-#ifdef CONFIG_PREEMPT_RT
-	spinlock_t			softirq_expiry_lock;
-	atomic_t			timer_waiters;
-#endif
-	ktime_t				expires_next;
-	struct hrtimer			*next_timer;
-	ktime_t				softirq_expires_next;
-	struct hrtimer			*softirq_next_timer;
-	ktime_t				deferred_expires_next;
-	struct hrtimer_clock_base	clock_base[HRTIMER_MAX_CLOCK_BASES];
-	call_single_data_t		csd;
-} ____cacheline_aligned;
-
-
-#endif
diff --git a/include/linux/hrtimer_rearm.h b/include/linux/hrtimer_rearm.h
index a6f2e5d5e1c7d..17a81826bd9a9 100644
--- a/include/linux/hrtimer_rearm.h
+++ b/include/linux/hrtimer_rearm.h
@@ -2,7 +2,12 @@
 #ifndef _LINUX_HRTIMER_REARM_H
 #define _LINUX_HRTIMER_REARM_H
 
+#include <linux/types.h>
+
 #ifdef CONFIG_HRTIMER_REARM_DEFERRED
+#include <linux/irqflags.h>
+#include <linux/lockdep.h>
+#include <linux/preempt.h>
 #include <linux/thread_info.h>
 
 void __hrtimer_rearm_deferred(void);
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index ad20f7f8c1794..9e43e138c0b77 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -230,6 +230,7 @@ static inline bool thp_vma_suitable_order(struct vm_area_struct *vma,
 
 	/* Don't have to check pgoff for anonymous vma */
 	if (!vma_is_anonymous(vma)) {
+		/* vma_start_pgoff() in mm.h so not available. */
 		if (!IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff,
 				hpage_size >> PAGE_SHIFT))
 			return false;
@@ -390,9 +391,9 @@ static inline bool thp_disabled_by_hw(void)
 
 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
 		unsigned long len, unsigned long pgoff, unsigned long flags);
-unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr,
+unsigned long thp_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr,
 		unsigned long len, unsigned long pgoff, unsigned long flags,
-		vm_flags_t vm_flags);
+		vma_flags_t vma_flags);
 
 enum split_type {
 	SPLIT_TYPE_UNIFORM,
@@ -529,6 +530,8 @@ static inline bool folio_test_pmd_mappable(struct folio *folio)
 
 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf);
 
+vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf);
+
 vm_fault_t do_huge_pmd_device_private(struct vm_fault *vmf);
 
 extern struct folio *huge_zero_folio;
@@ -567,7 +570,7 @@ static inline struct folio *get_persistent_huge_zero_folio(void)
 
 static inline bool thp_migration_supported(void)
 {
-	return IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION);
+	return IS_ENABLED(CONFIG_ARCH_HAS_PMD_SOFTLEAVES);
 }
 
 void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address,
@@ -614,9 +617,9 @@ static inline unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma,
 #define thp_get_unmapped_area	NULL
 
 static inline unsigned long
-thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr,
-			      unsigned long len, unsigned long pgoff,
-			      unsigned long flags, vm_flags_t vm_flags)
+thp_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr,
+			       unsigned long len, unsigned long pgoff,
+			       unsigned long flags, vma_flags_t vma_flags)
 {
 	return 0;
 }
@@ -723,6 +726,11 @@ static inline spinlock_t *pud_trans_huge_lock(pud_t *pud,
 	return NULL;
 }
 
+static inline vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf)
+{
+	return 0;
+}
+
 static inline vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf)
 {
 	return 0;
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 2abaf99321e90..16c4c4caa126c 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -2,6 +2,7 @@
 #ifndef _LINUX_HUGETLB_H
 #define _LINUX_HUGETLB_H
 
+#include <linux/mempolicy.h>
 #include <linux/mm.h>
 #include <linux/mm_types.h>
 #include <linux/mmdebug.h>
@@ -154,7 +155,8 @@ long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
 bool folio_isolate_hugetlb(struct folio *folio, struct list_head *list);
 int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison);
 void folio_putback_hugetlb(struct folio *folio);
-void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason);
+void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio,
+		enum migrate_reason reason);
 void hugetlb_fix_reserve_counts(struct inode *inode);
 extern struct mutex *hugetlb_fault_mutex_table;
 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx);
@@ -171,6 +173,7 @@ extern int movable_gigantic_pages __read_mostly;
 extern int sysctl_hugetlb_shm_group __read_mostly;
 extern struct list_head huge_boot_pages[MAX_NUMNODES];
 
+void hugetlb_bootmem_struct_page_init(void);
 void hugetlb_bootmem_alloc(void);
 extern nodemask_t hugetlb_bootmem_nodes;
 void hugetlb_bootmem_set_nodes(void);
@@ -424,7 +427,7 @@ static inline void folio_putback_hugetlb(struct folio *folio)
 }
 
 static inline void move_hugetlb_state(struct folio *old_folio,
-					struct folio *new_folio, int reason)
+		struct folio *new_folio, enum migrate_reason reason)
 {
 }
 
@@ -673,24 +676,30 @@ struct hstate {
 	char name[HSTATE_NAME_LEN];
 };
 
-struct cma;
-
-struct huge_bootmem_page {
-	struct list_head list;
-	struct hstate *hstate;
-	unsigned long flags;
-	struct cma *cma;
-};
-
 #define HUGE_BOOTMEM_HVO		0x0001
 #define HUGE_BOOTMEM_ZONES_VALID	0x0002
 #define HUGE_BOOTMEM_CMA		0x0004
 
-bool hugetlb_bootmem_page_zones_valid(int nid, struct huge_bootmem_page *m);
-
 int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list);
 int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn);
 void wait_for_freed_hugetlb_folios(void);
+
+struct mempolicy_interpreted {
+	int nid;
+	nodemask_t *nodemask;
+	enum mempolicy_mode mode;
+};
+
+enum hugetlb_alloc_flag {
+	HUGETLB_ALLOC_CHARGE_CGROUP_RSVD_BIT = 0,
+	HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS_BIT,
+};
+
+#define HUGETLB_ALLOC_CHARG_CGROUP_RSVD BIT(HUGETLB_ALLOC_CHARGE_CGROUP_RSVD_BIT)
+#define HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS BIT(HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS_BIT)
+
+struct folio *hugetlb_alloc_folio(struct hstate *h,
+		struct mempolicy_interpreted *mpoli, u8 alloc_flags);
 struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
 				unsigned long addr, bool cow_from_owner);
 struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid,
@@ -705,8 +714,8 @@ void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma,
 				unsigned long address, struct folio *folio);
 
 /* arch callback */
-int __init __alloc_bootmem_huge_page(struct hstate *h, int nid);
-int __init alloc_bootmem_huge_page(struct hstate *h, int nid);
+void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid);
+void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid);
 bool __init hugetlb_node_alloc_supported(void);
 
 void __init hugetlb_add_hstate(unsigned order);
@@ -792,8 +801,7 @@ static inline pgoff_t hugetlb_linear_page_index(struct vm_area_struct *vma,
 {
 	struct hstate *h = hstate_vma(vma);
 
-	return ((address - vma->vm_start) >> huge_page_shift(h)) +
-		(vma->vm_pgoff >> huge_page_order(h));
+	return linear_page_index(vma, address) >> huge_page_order(h);
 }
 
 static inline bool order_is_gigantic(unsigned int order)
@@ -956,7 +964,7 @@ static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask)
 	return modified_mask;
 }
 
-static inline bool htlb_allow_alloc_fallback(int reason)
+static inline bool htlb_allow_alloc_fallback(enum migrate_reason reason)
 {
 	bool allowed_fallback = false;
 
@@ -1137,9 +1145,9 @@ alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid,
 	return NULL;
 }
 
-static inline int __alloc_bootmem_huge_page(struct hstate *h)
+static inline void *__alloc_bootmem_huge_page(struct hstate *h, int nid)
 {
-	return 0;
+	return NULL;
 }
 
 static inline struct hstate *hstate_file(struct file *f)
@@ -1238,7 +1246,7 @@ static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask)
 	return 0;
 }
 
-static inline bool htlb_allow_alloc_fallback(int reason)
+static inline bool htlb_allow_alloc_fallback(enum migrate_reason reason)
 {
 	return false;
 }
@@ -1261,6 +1269,9 @@ static inline void hugetlb_count_sub(long l, struct mm_struct *mm)
 {
 }
 
+pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
+unsigned long huge_pte_dirty(pte_t pte);
+
 static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
 					  unsigned long addr, pte_t *ptep)
 {
@@ -1293,6 +1304,10 @@ static inline bool hugetlbfs_pagecache_present(
 static inline void hugetlb_bootmem_alloc(void)
 {
 }
+
+static inline void hugetlb_bootmem_struct_page_init(void)
+{
+}
 #endif	/* CONFIG_HUGETLB_PAGE */
 
 static inline spinlock_t *huge_pte_lock(struct hstate *h,
diff --git a/include/linux/hugetlb_cgroup.h b/include/linux/hugetlb_cgroup.h
index e5d64b8b59c20..16d72c8c71f69 100644
--- a/include/linux/hugetlb_cgroup.h
+++ b/include/linux/hugetlb_cgroup.h
@@ -267,5 +267,5 @@ static inline void hugetlb_cgroup_migrate(struct folio *old_folio,
 {
 }
 
-#endif  /* CONFIG_MEM_RES_CTLR_HUGETLB */
+#endif  /* CONFIG_CGROUP_HUGETLB */
 #endif
diff --git a/include/linux/ieee80211-eht.h b/include/linux/ieee80211-eht.h
index 18f9c662cf4cb..c109722b1969a 100644
--- a/include/linux/ieee80211-eht.h
+++ b/include/linux/ieee80211-eht.h
@@ -857,7 +857,7 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len)
 	const struct ieee80211_multi_link_elem *mle = (const void *)data;
 	u8 fixed = sizeof(*mle);
 	u8 common = 0;
-	bool check_common_len = false;
+	u8 common_len;
 	u16 control;
 
 	if (!data || len < fixed)
@@ -868,7 +868,6 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len)
 	switch (u16_get_bits(control, IEEE80211_ML_CONTROL_TYPE)) {
 	case IEEE80211_ML_CONTROL_TYPE_BASIC:
 		common += sizeof(struct ieee80211_mle_basic_common_info);
-		check_common_len = true;
 		if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID)
 			common += 1;
 		if (control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT)
@@ -888,9 +887,9 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len)
 		common += sizeof(struct ieee80211_mle_preq_common_info);
 		if (control & IEEE80211_MLC_PREQ_PRES_MLD_ID)
 			common += 1;
-		check_common_len = true;
 		break;
 	case IEEE80211_ML_CONTROL_TYPE_RECONF:
+		common += 1;
 		if (control & IEEE80211_MLC_RECONF_PRES_MLD_MAC_ADDR)
 			common += ETH_ALEN;
 		if (control & IEEE80211_MLC_RECONF_PRES_EML_CAPA)
@@ -902,7 +901,6 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len)
 		break;
 	case IEEE80211_ML_CONTROL_TYPE_TDLS:
 		common += sizeof(struct ieee80211_mle_tdls_common_info);
-		check_common_len = true;
 		break;
 	case IEEE80211_ML_CONTROL_TYPE_PRIO_ACCESS:
 		common = ETH_ALEN + 1;
@@ -915,11 +913,9 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len)
 	if (len < fixed + common)
 		return false;
 
-	if (!check_common_len)
-		return true;
+	common_len = mle->variable[0];
 
-	/* if present, common length is the first octet there */
-	return mle->variable[0] >= common;
+	return common_len >= common && common_len <= len - fixed;
 }
 
 /**
diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h
index d40484451e9a2..26e6740388654 100644
--- a/include/linux/ieee80211.h
+++ b/include/linux/ieee80211.h
@@ -556,6 +556,17 @@ static inline bool ieee80211_is_reassoc_resp(__le16 fc)
 	       cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_REASSOC_RESP);
 }
 
+/**
+ * ieee80211_is_assoc - check if (Re)association request/response frame
+ * @fc: frame control bytes in little-endian byteorder
+ * Return: whether or not the frame is an (re)association request or response
+ */
+static inline bool ieee80211_is_assoc(__le16 fc)
+{
+	return ieee80211_is_assoc_req(fc) || ieee80211_is_reassoc_req(fc) ||
+	       ieee80211_is_assoc_resp(fc) || ieee80211_is_reassoc_resp(fc);
+}
+
 /**
  * ieee80211_is_probe_req - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_PROBE_REQ
  * @fc: frame control bytes in little-endian byteorder
@@ -2616,6 +2627,7 @@ static inline int ieee80211_get_tdls_action(struct sk_buff *skb)
 /* convert frequencies */
 #define MHZ_TO_KHZ(freq) ((freq) * 1000)
 #define KHZ_TO_MHZ(freq) ((freq) / 1000)
+#define KHZ_TO_HZ(x) ((x) * 1000)
 #define PR_KHZ(f) KHZ_TO_MHZ(f), f % 1000
 #define KHZ_F "%d.%03d"
 
diff --git a/include/linux/iio/types.h b/include/linux/iio/types.h
index 4e3099defc1d3..924ac9dc6893f 100644
--- a/include/linux/iio/types.h
+++ b/include/linux/iio/types.h
@@ -7,6 +7,9 @@
 #ifndef _IIO_TYPES_H_
 #define _IIO_TYPES_H_
 
+#include <linux/types.h>
+#include <linux/wordpart.h>
+
 #include <uapi/linux/iio/types.h>
 
 enum iio_event_info {
@@ -34,6 +37,23 @@ enum iio_event_info {
 #define IIO_VAL_FRACTIONAL_LOG2 11
 #define IIO_VAL_CHAR 12
 
+#define IIO_VAL_DECIMAL64_BASE		32
+#define IIO_VAL_DECIMAL64_MILLI		(IIO_VAL_DECIMAL64_BASE + 3)
+#define IIO_VAL_DECIMAL64_MICRO		(IIO_VAL_DECIMAL64_BASE + 6)
+#define IIO_VAL_DECIMAL64_NANO		(IIO_VAL_DECIMAL64_BASE + 9)
+#define IIO_VAL_DECIMAL64_PICO		(IIO_VAL_DECIMAL64_BASE + 12)
+
+static inline s64 iio_val_s64_compose(s32 val0, s32 val1)
+{
+	return (s64)(((u64)val1 << 32) | (u32)val0);
+}
+
+static inline void iio_val_s64_decompose(s64 dec64, s32 *val0, s32 *val1)
+{
+	*val0 = lower_32_bits(dec64);
+	*val1 = upper_32_bits(dec64);
+}
+
 enum iio_available_type {
 	IIO_AVAIL_LIST,
 	IIO_AVAIL_RANGE,
diff --git a/include/linux/init_task.h b/include/linux/init_task.h
index a6cb241ea00c3..61536be773f59 100644
--- a/include/linux/init_task.h
+++ b/include/linux/init_task.h
@@ -24,6 +24,7 @@
 
 extern struct files_struct init_files;
 extern struct fs_struct init_fs;
+extern struct fs_struct *userspace_init_fs;
 extern struct nsproxy init_nsproxy;
 
 #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 87151a5b62c1b..a2c623a67a251 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -534,6 +534,8 @@ struct io_ring_ctx {
 	struct io_mapped_region		ring_region;
 	/* used for optimised request parameter and wait argument passing  */
 	struct io_mapped_region		param_region;
+
+	struct kcov_common_handle_id	kcov_handle;
 };
 
 /*
diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h
index 1fabf0f5ea8e7..0bb6c03481fad 100644
--- a/include/linux/irq-entry-common.h
+++ b/include/linux/irq-entry-common.h
@@ -49,9 +49,9 @@
  * Defaults to an empty implementation. Can be replaced by architecture
  * specific code.
  *
- * Invoked from syscall_enter_from_user_mode() in the non-instrumentable
- * section. Use __always_inline so the compiler cannot push it out of line
- * and make it instrumentable.
+ * Invoked from enter_from_user_mode() in the non-instrumentable section. Use
+ * __always_inline so the compiler cannot push it out of line and make it
+ * instrumentable.
  */
 static __always_inline void arch_enter_from_user_mode(struct pt_regs *regs);
 
diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h
index 8080db17c1b1b..779d6023c5c29 100644
--- a/include/linux/irqdesc.h
+++ b/include/linux/irqdesc.h
@@ -146,7 +146,7 @@ extern struct irq_desc irq_desc[NR_IRQS];
 static inline unsigned int irq_desc_kstat_cpu(struct irq_desc *desc,
 					      unsigned int cpu)
 {
-	return desc->kstat_irqs ? per_cpu(desc->kstat_irqs->cnt, cpu) : 0;
+	return per_cpu(desc->kstat_irqs->cnt, cpu);
 }
 
 static inline struct irq_desc *irq_data_to_desc(struct irq_data *data)
diff --git a/include/linux/kexec.h b/include/linux/kexec.h
index 8a22bc9b8c6c8..0af8ae4fdd087 100644
--- a/include/linux/kexec.h
+++ b/include/linux/kexec.h
@@ -13,7 +13,7 @@
 #define IND_SOURCE       (1 << IND_SOURCE_BIT)
 #define IND_FLAGS (IND_DESTINATION | IND_INDIRECTION | IND_DONE | IND_SOURCE)
 
-#if !defined(__ASSEMBLY__)
+#if !defined(__ASSEMBLER__)
 
 #include <linux/vmcore_info.h>
 #include <linux/crash_reserve.h>
diff --git a/include/linux/kexec_handover.h b/include/linux/kexec_handover.h
index 8968c56d2d73e..48a9793c2b76c 100644
--- a/include/linux/kexec_handover.h
+++ b/include/linux/kexec_handover.h
@@ -5,11 +5,7 @@
 #include <linux/err.h>
 #include <linux/errno.h>
 #include <linux/types.h>
-
-struct kho_scratch {
-	phys_addr_t addr;
-	phys_addr_t size;
-};
+#include <asm-generic/kexec_handover.h>
 
 struct kho_vmalloc;
 
diff --git a/include/linux/kobject.h b/include/linux/kobject.h
index bcb5d4e320015..55e37a5d405ef 100644
--- a/include/linux/kobject.h
+++ b/include/linux/kobject.h
@@ -138,12 +138,79 @@ struct kset_uevent_ops {
 
 struct kobj_attribute {
 	struct attribute attr;
-	ssize_t (*show)(struct kobject *kobj, struct kobj_attribute *attr,
-			char *buf);
-	ssize_t (*store)(struct kobject *kobj, struct kobj_attribute *attr,
-			 const char *buf, size_t count);
+	__SYSFS_FUNCTION_ALTERNATIVE(
+		ssize_t (*show)(struct kobject *kobj, struct kobj_attribute *attr, char *buf);
+		ssize_t (*show_const)(struct kobject *kobj, const struct kobj_attribute *attr,
+				      char *buf);
+	);
+	__SYSFS_FUNCTION_ALTERNATIVE(
+		ssize_t (*store)(struct kobject *kobj, struct kobj_attribute *attr,
+				 const char *buf, size_t count);
+		ssize_t (*store_const)(struct kobject *kobj, const struct kobj_attribute *attr,
+				       const char *buf, size_t count);
+	);
 };
 
+typedef ssize_t __kobj_show_handler_const(struct kobject *kobj, const struct kobj_attribute *attr,
+					  char *buf);
+typedef ssize_t __kobj_store_handler_const(struct kobject *kobj, const struct kobj_attribute *attr,
+					   const char *buf, size_t count);
+
+#ifdef CONFIG_CFI
+
+#define __KOBJ_ATTR_SHOW_STORE(_show, _store)						\
+	.show		= _Generic(_show,						\
+			  __kobj_show_handler_const * : NULL,				\
+			  default : _show						\
+	),										\
+	.show_const	= _Generic(_show,						\
+			  __kobj_show_handler_const * : _show,				\
+			  default : NULL						\
+	),										\
+	.store		= _Generic(_store,						\
+			  __kobj_store_handler_const * : NULL,				\
+			  default : _store						\
+	),										\
+	.store_const	= _Generic(_store,						\
+			  __kobj_store_handler_const * : _store,			\
+			  default : NULL \
+	),
+
+#else
+
+#define __KOBJ_ATTR_SHOW_STORE(_show, _store)						\
+	.show		= _Generic(_show,						\
+			  __kobj_show_handler_const * : (void *)_show,			\
+			  default : _show						\
+	),										\
+	.store		= _Generic(_store,						\
+			  __kobj_store_handler_const * : (void *)_store,		\
+			  default : _store						\
+	),										\
+
+#endif
+
+#define __KOBJ_ATTR(_name, _mode, _show, _store) {			\
+	.attr	= { .name = __stringify(_name),				\
+		    .mode = VERIFY_OCTAL_PERMISSIONS(_mode) },		\
+	__KOBJ_ATTR_SHOW_STORE(_show, _store)				\
+}
+
+#define __KOBJ_ATTR_RO_MODE(_name, _mode)				\
+	__KOBJ_ATTR(_name, _mode, _name##_show, NULL)
+
+#define __KOBJ_ATTR_RO(_name)						\
+	__KOBJ_ATTR_RO_MODE(_name, 0444)
+
+#define __KOBJ_ATTR_RW_MODE(_name, _mode)				\
+	__KOBJ_ATTR(_name, _mode, _name##_show, _name##_store)
+
+#define __KOBJ_ATTR_WO(_name)						\
+	__KOBJ_ATTR(_name, 0200, NULL, _name##_store)
+
+#define __KOBJ_ATTR_RW(_name)						\
+	__KOBJ_ATTR(_name, 0644, _name##_show, _name##_store)
+
 extern const struct sysfs_ops kobj_sysfs_ops;
 
 struct sock;
diff --git a/include/linux/kstrtox.h b/include/linux/kstrtox.h
index 6c92828667704..b41b9736886ff 100644
--- a/include/linux/kstrtox.h
+++ b/include/linux/kstrtox.h
@@ -97,6 +97,9 @@ int __must_check kstrtou8(const char *s, unsigned int base, u8 *res);
 int __must_check kstrtos8(const char *s, unsigned int base, s8 *res);
 int __must_check kstrtobool(const char *s, bool *res);
 
+int __must_check kstrtoudec64(const char *s, unsigned int scale, u64 *res);
+int __must_check kstrtodec64(const char *s, unsigned int scale, s64 *res);
+
 int __must_check kstrtoull_from_user(const char __user *s, size_t count, unsigned int base, unsigned long long *res);
 int __must_check kstrtoll_from_user(const char __user *s, size_t count, unsigned int base, long long *res);
 int __must_check kstrtoul_from_user(const char __user *s, size_t count, unsigned int base, unsigned long *res);
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index ab8cfaec82d31..9db6eb4023c48 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -989,6 +989,13 @@ static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx)
 					 lockdep_is_held(&kvm->slots_lock));
 }
 
+static inline void kvm_lockdep_assert_vcpu_is_locked_or_unreachable(struct kvm_vcpu *vcpu)
+{
+	lockdep_assert_once(lockdep_is_held(&vcpu->mutex) ||
+			    vcpu->vcpu_idx < 0 ||
+			    !refcount_read(&vcpu->kvm->users_count));
+}
+
 static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i)
 {
 	int num_vcpus = atomic_read(&kvm->online_vcpus);
@@ -2608,6 +2615,7 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src,
 
 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
 void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end);
+void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range);
 #endif
 
 #ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY
diff --git a/include/linux/leafops.h b/include/linux/leafops.h
index 992cd8bd8ed0e..4c1476ae32343 100644
--- a/include/linux/leafops.h
+++ b/include/linux/leafops.h
@@ -81,7 +81,7 @@ static inline pte_t softleaf_to_pte(softleaf_t entry)
 	return swp_entry_to_pte(entry);
 }
 
-#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
 /**
  * softleaf_from_pmd() - Obtain a leaf entry from a PMD entry.
  * @pmd: PMD entry.
@@ -100,14 +100,29 @@ static inline softleaf_t softleaf_from_pmd(pmd_t pmd)
 
 	if (pmd_swp_soft_dirty(pmd))
 		pmd = pmd_swp_clear_soft_dirty(pmd);
-	if (pmd_swp_uffd_wp(pmd))
-		pmd = pmd_swp_clear_uffd_wp(pmd);
+	if (pmd_swp_uffd(pmd))
+		pmd = pmd_swp_clear_uffd(pmd);
 	arch_entry = __pmd_to_swp_entry(pmd);
 
 	/* Temporary until swp_entry_t eliminated. */
 	return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
 }
 
+/**
+ * softleaf_to_pmd() - Obtain a PMD entry from a leaf entry.
+ * @entry: Leaf entry.
+ *
+ * This generates an architecture-specific PMD entry that can be utilised to
+ * encode the metadata the leaf entry encodes.
+ *
+ * Returns: Architecture-specific PMD entry encoding leaf entry.
+ */
+static inline pmd_t softleaf_to_pmd(softleaf_t entry)
+{
+	/* Temporary until swp_entry_t eliminated. */
+	return swp_entry_to_pmd(entry);
+}
+
 #else
 
 static inline softleaf_t softleaf_from_pmd(pmd_t pmd)
@@ -115,6 +130,11 @@ static inline softleaf_t softleaf_from_pmd(pmd_t pmd)
 	return softleaf_mk_none();
 }
 
+static inline pmd_t softleaf_to_pmd(softleaf_t entry)
+{
+	return __pmd(0);
+}
+
 #endif
 
 /**
@@ -567,7 +587,7 @@ static inline bool pte_is_uffd_marker(pte_t pte)
 	return false;
 }
 
-#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_ENABLE_THP_MIGRATION)
+#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_HAS_PMD_SOFTLEAVES)
 
 /**
  * pmd_is_device_private_entry() - Check if PMD contains a device private swap
@@ -586,14 +606,14 @@ static inline bool pmd_is_device_private_entry(pmd_t pmd)
 	return softleaf_is_device_private(softleaf_from_pmd(pmd));
 }
 
-#else  /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */
+#else  /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_HAS_PMD_SOFTLEAVES */
 
 static inline bool pmd_is_device_private_entry(pmd_t pmd)
 {
 	return false;
 }
 
-#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */
+#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_HAS_PMD_SOFTLEAVES */
 
 /**
  * pmd_is_migration_entry() - Does this PMD entry encode a migration entry?
diff --git a/include/linux/leds.h b/include/linux/leds.h
index b16b803cc1ac5..bf31c246d9e2f 100644
--- a/include/linux/leds.h
+++ b/include/linux/leds.h
@@ -607,6 +607,10 @@ enum led_trigger_netdev_modes {
 	TRIGGER_NETDEV_LINK_2500,
 	TRIGGER_NETDEV_LINK_5000,
 	TRIGGER_NETDEV_LINK_10000,
+	TRIGGER_NETDEV_LINK_25000,
+	TRIGGER_NETDEV_LINK_40000,
+	TRIGGER_NETDEV_LINK_50000,
+	TRIGGER_NETDEV_LINK_100000,
 	TRIGGER_NETDEV_HALF_DUPLEX,
 	TRIGGER_NETDEV_FULL_DUPLEX,
 	TRIGGER_NETDEV_TX,
diff --git a/include/linux/libata.h b/include/linux/libata.h
index 96e626d6a7cab..18edb36c29fc2 100644
--- a/include/linux/libata.h
+++ b/include/linux/libata.h
@@ -121,6 +121,55 @@ enum {
 	ATA_QUIRK_NO_FUA		= BIT_ULL(__ATA_QUIRK_NO_FUA),
 };
 
+/*
+ * struct ata_device flags
+ */
+enum {
+	ATA_DFLAG_LBA		= (1UL << 0), /* device supports LBA */
+	ATA_DFLAG_LBA48		= (1UL << 1), /* device supports LBA48 */
+	ATA_DFLAG_CDB_INTR	= (1UL << 2), /* device asserts INTRQ when ready for CDB */
+	ATA_DFLAG_NCQ		= (1UL << 3), /* device supports NCQ */
+	ATA_DFLAG_FLUSH_EXT	= (1UL << 4), /* do FLUSH_EXT instead of FLUSH */
+	ATA_DFLAG_ACPI_PENDING	= (1UL << 5), /* ACPI resume action pending */
+	ATA_DFLAG_ACPI_FAILED	= (1UL << 6), /* ACPI on devcfg has failed */
+	ATA_DFLAG_AN		= (1UL << 7), /* AN configured */
+	ATA_DFLAG_TRUSTED	= (1UL << 8), /* device supports trusted send/recv */
+	ATA_DFLAG_FUA		= (1UL << 9), /* device supports FUA */
+	ATA_DFLAG_DMADIR	= (1UL << 10), /* device requires DMADIR */
+	ATA_DFLAG_NCQ_SEND_RECV = (1UL << 11), /* device supports NCQ SEND and RECV */
+	ATA_DFLAG_NCQ_PRIO	= (1UL << 12), /* device supports NCQ priority */
+	ATA_DFLAG_CDL		= (1UL << 13), /* supports cmd duration limits */
+	ATA_DFLAG_DEPOP		= (1UL << 14), /* supports depopulation capability */
+	ATA_DFLAG_DEPOP_RESTORE	= (1UL << 15), /* supports depopulation restoration */
+	ATA_DFLAG_DEPOP_MODIFY	= (1UL << 16), /* supports zoned depopulation */
+	ATA_DFLAG_CFG_MASK	= (1UL << 17) - 1,
+
+	ATA_DFLAG_PIO		= (1UL << 17), /* device limited to PIO mode */
+	ATA_DFLAG_NCQ_OFF	= (1UL << 18), /* device limited to non-NCQ mode */
+	ATA_DFLAG_SLEEPING	= (1UL << 19), /* device is sleeping */
+	ATA_DFLAG_DUBIOUS_XFER	= (1UL << 20), /* data transfer not verified */
+	ATA_DFLAG_NO_UNLOAD	= (1UL << 21), /* device doesn't support unload */
+	ATA_DFLAG_UNLOCK_HPA	= (1UL << 22), /* unlock HPA */
+	ATA_DFLAG_INIT_MASK	= (1UL << 23) - 1,
+
+	ATA_DFLAG_NCQ_PRIO_ENABLED = (1UL << 23), /* Priority cmds sent to dev */
+	ATA_DFLAG_CDL_ENABLED	= (1UL << 24), /* cmd duration limits is enabled */
+	ATA_DFLAG_RESUMING	= (1UL << 25),  /* Device is resuming */
+	ATA_DFLAG_DETACH	= (1UL << 26),
+	ATA_DFLAG_DETACHED	= (1UL << 27),
+	ATA_DFLAG_DA		= (1UL << 28), /* device supports Device Attention */
+	ATA_DFLAG_DEVSLP	= (1UL << 29), /* device supports Device Sleep */
+	ATA_DFLAG_ACPI_DISABLED = (1UL << 30), /* ACPI for the device is disabled */
+	ATA_DFLAG_D_SENSE	= (1UL << 31), /* Descriptor sense requested */
+
+	ATA_DFLAG_FEATURES_MASK	= (ATA_DFLAG_TRUSTED | ATA_DFLAG_DA |	\
+				   ATA_DFLAG_DEVSLP | ATA_DFLAG_NCQ_SEND_RECV | \
+				   ATA_DFLAG_NCQ_PRIO | ATA_DFLAG_FUA | \
+				   ATA_DFLAG_CDL | ATA_DFLAG_DEPOP | \
+				   ATA_DFLAG_DEPOP_RESTORE |
+				   ATA_DFLAG_DEPOP_MODIFY)
+};
+
 enum {
 	/* various global constants */
 	LIBATA_MAX_PRD		= ATA_MAX_PRD / 2,
@@ -146,46 +195,7 @@ enum {
 	ATA_TFLAG_FUA		= (1 << 5), /* enable FUA */
 	ATA_TFLAG_POLLING	= (1 << 6), /* set nIEN to 1 and use polling */
 
-	/* struct ata_device stuff */
-	ATA_DFLAG_LBA		= (1 << 0), /* device supports LBA */
-	ATA_DFLAG_LBA48		= (1 << 1), /* device supports LBA48 */
-	ATA_DFLAG_CDB_INTR	= (1 << 2), /* device asserts INTRQ when ready for CDB */
-	ATA_DFLAG_NCQ		= (1 << 3), /* device supports NCQ */
-	ATA_DFLAG_FLUSH_EXT	= (1 << 4), /* do FLUSH_EXT instead of FLUSH */
-	ATA_DFLAG_ACPI_PENDING	= (1 << 5), /* ACPI resume action pending */
-	ATA_DFLAG_ACPI_FAILED	= (1 << 6), /* ACPI on devcfg has failed */
-	ATA_DFLAG_AN		= (1 << 7), /* AN configured */
-	ATA_DFLAG_TRUSTED	= (1 << 8), /* device supports trusted send/recv */
-	ATA_DFLAG_FUA		= (1 << 9), /* device supports FUA */
-	ATA_DFLAG_DMADIR	= (1 << 10), /* device requires DMADIR */
-	ATA_DFLAG_NCQ_SEND_RECV = (1 << 11), /* device supports NCQ SEND and RECV */
-	ATA_DFLAG_NCQ_PRIO	= (1 << 12), /* device supports NCQ priority */
-	ATA_DFLAG_CDL		= (1 << 13), /* supports cmd duration limits */
-	ATA_DFLAG_CFG_MASK	= (1 << 14) - 1,
-
-	ATA_DFLAG_PIO		= (1 << 14), /* device limited to PIO mode */
-	ATA_DFLAG_NCQ_OFF	= (1 << 15), /* device limited to non-NCQ mode */
-	ATA_DFLAG_SLEEPING	= (1 << 16), /* device is sleeping */
-	ATA_DFLAG_DUBIOUS_XFER	= (1 << 17), /* data transfer not verified */
-	ATA_DFLAG_NO_UNLOAD	= (1 << 18), /* device doesn't support unload */
-	ATA_DFLAG_UNLOCK_HPA	= (1 << 19), /* unlock HPA */
-	ATA_DFLAG_INIT_MASK	= (1 << 20) - 1,
-
-	ATA_DFLAG_NCQ_PRIO_ENABLED = (1 << 20), /* Priority cmds sent to dev */
-	ATA_DFLAG_CDL_ENABLED	= (1 << 21), /* cmd duration limits is enabled */
-	ATA_DFLAG_RESUMING	= (1 << 22),  /* Device is resuming */
-	ATA_DFLAG_DETACH	= (1 << 24),
-	ATA_DFLAG_DETACHED	= (1 << 25),
-	ATA_DFLAG_DA		= (1 << 26), /* device supports Device Attention */
-	ATA_DFLAG_DEVSLP	= (1 << 27), /* device supports Device Sleep */
-	ATA_DFLAG_ACPI_DISABLED = (1 << 28), /* ACPI for the device is disabled */
-	ATA_DFLAG_D_SENSE	= (1 << 29), /* Descriptor sense requested */
-
-	ATA_DFLAG_FEATURES_MASK	= (ATA_DFLAG_TRUSTED | ATA_DFLAG_DA |	\
-				   ATA_DFLAG_DEVSLP | ATA_DFLAG_NCQ_SEND_RECV | \
-				   ATA_DFLAG_NCQ_PRIO | ATA_DFLAG_FUA | \
-				   ATA_DFLAG_CDL),
-
+	/* sturct ata_device class. */
 	ATA_DEV_UNKNOWN		= 0,	/* unknown device */
 	ATA_DEV_ATA		= 1,	/* ATA device */
 	ATA_DEV_ATA_UNSUP	= 2,	/* ATA device (unsupported) */
@@ -1418,9 +1428,6 @@ extern int ata_port_freeze(struct ata_port *ap);
 extern void ata_eh_freeze_port(struct ata_port *ap);
 extern void ata_eh_thaw_port(struct ata_port *ap);
 
-extern void ata_eh_qc_complete(struct ata_queued_cmd *qc);
-extern void ata_eh_qc_retry(struct ata_queued_cmd *qc);
-
 extern void ata_std_error_handler(struct ata_port *ap)
 	__must_hold(&ap->host->eh_mutex);
 extern void ata_std_sched_eh(struct ata_port *ap);
diff --git a/include/linux/lockd/bind.h b/include/linux/lockd/bind.h
index b614e0deea726..db8207d4059f3 100644
--- a/include/linux/lockd/bind.h
+++ b/include/linux/lockd/bind.h
@@ -16,17 +16,23 @@ struct svc_rqst;
 struct rpc_task;
 struct rpc_clnt;
 struct super_block;
+struct module;
 
-/*
- * This is the set of functions for lockd->nfsd communication
+/**
+ * struct nlmsvc_binding - lockd -> nfsd callback table
+ * @owner:  module that provides this binding.
+ * @fopen:  open a file by NFS file handle on behalf of an NLM request.
+ * @fclose: close a file that was previously opened via @fopen.
+ *          Implementations MUST be semantically equivalent to fput().
  */
 struct nlmsvc_binding {
+	struct module	*owner;
 	int		(*fopen)(struct svc_rqst *rqstp, struct nfs_fh *f,
 				 struct file **filp, int flags);
 	void		(*fclose)(struct file *filp);
 };
 
-extern const struct nlmsvc_binding *nlmsvc_ops;
+extern const struct nlmsvc_binding __rcu *nlmsvc_ops;
 
 /*
  * Similar to nfs_client_initdata, but without the NFS-specific
diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h
index 621566345406d..a6451ecbbe9a0 100644
--- a/include/linux/lockdep.h
+++ b/include/linux/lockdep.h
@@ -273,6 +273,9 @@ extern struct pin_cookie lock_pin_lock(struct lockdep_map *lock);
 extern void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie);
 extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie);
 
+extern u32 lock_sequence(struct lockdep_map *lock);
+#define lockdep_sequence(lock)	lock_sequence(&(lock)->dep_map)
+
 #define lockdep_depth(tsk)	(debug_locks ? (tsk)->lockdep_depth : 0)
 
 #define lockdep_assert(cond)		\
diff --git a/include/linux/lockdep_types.h b/include/linux/lockdep_types.h
index eae115a264885..55c4b152fedf7 100644
--- a/include/linux/lockdep_types.h
+++ b/include/linux/lockdep_types.h
@@ -253,7 +253,8 @@ struct held_lock {
 	unsigned int hardirqs_off:1;
 	unsigned int sync:1;
 	unsigned int references:11;					/* 32 bits */
-	unsigned int pin_count;
+	unsigned int pin_count:24;
+	unsigned int seq_count:8;
 };
 
 #else /* !CONFIG_LOCKDEP */
diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h
index b4f8cad53ddb0..c4488c4a6d8a9 100644
--- a/include/linux/lsm_hooks.h
+++ b/include/linux/lsm_hooks.h
@@ -167,6 +167,7 @@ enum lsm_order {
  * @initcall_fs: LSM callback for fs_initcall setup, optional
  * @initcall_device: LSM callback for device_initcall() setup, optional
  * @initcall_late: LSM callback for late_initcall() setup, optional
+ * @initcall_late_sync: LSM callback for late_initcall_sync() setup, optional
  */
 struct lsm_info {
 	const struct lsm_id *id;
@@ -182,6 +183,7 @@ struct lsm_info {
 	int (*initcall_fs)(void);
 	int (*initcall_device)(void);
 	int (*initcall_late)(void);
+	int (*initcall_late_sync)(void);
 };
 
 #define DEFINE_LSM(lsm)							\
diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h
index 4a5631906affb..b723384b3bf49 100644
--- a/include/linux/maple_tree.h
+++ b/include/linux/maple_tree.h
@@ -11,7 +11,6 @@
 #include <linux/kernel.h>
 #include <linux/rcupdate.h>
 #include <linux/spinlock.h>
-/* #define CONFIG_MAPLE_RCU_DISABLED */
 
 /*
  * Allocated nodes are mutable until they have been inserted into the tree,
@@ -29,13 +28,11 @@
 #define MAPLE_NODE_SLOTS	31	/* 256 bytes including ->parent */
 #define MAPLE_RANGE64_SLOTS	16	/* 256 bytes */
 #define MAPLE_ARANGE64_SLOTS	10	/* 240 bytes */
-#define MAPLE_ALLOC_SLOTS	(MAPLE_NODE_SLOTS - 1)
 #else
 /* 32bit sizes */
 #define MAPLE_NODE_SLOTS	63	/* 256 bytes including ->parent */
 #define MAPLE_RANGE64_SLOTS	32	/* 256 bytes */
 #define MAPLE_ARANGE64_SLOTS	21	/* 240 bytes */
-#define MAPLE_ALLOC_SLOTS	(MAPLE_NODE_SLOTS - 2)
 #endif /* defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) */
 
 #define MAPLE_NODE_MASK		255UL
@@ -485,6 +482,12 @@ struct ma_state {
 	unsigned char mas_flags;
 	unsigned char end;		/* The end of the node */
 	enum store_type store_type;	/* The type of store needed for this operation */
+#ifdef CONFIG_LOCKDEP
+	u32 ld_seq;
+#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
+	unsigned long rcu_gp;
+#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */
+#endif /* CONFIG_LOCKDEP */
 };
 
 struct ma_wr_state {
@@ -572,7 +575,7 @@ void maple_tree_init(void);
 void mas_destroy(struct ma_state *mas);
 
 void *mas_prev(struct ma_state *mas, unsigned long min);
-void *mas_prev_range(struct ma_state *mas, unsigned long max);
+void *mas_prev_range(struct ma_state *mas, unsigned long min);
 void *mas_next(struct ma_state *mas, unsigned long max);
 void *mas_next_range(struct ma_state *mas, unsigned long max);
 
@@ -864,9 +867,6 @@ static inline void mt_init(struct maple_tree *mt)
 
 static inline bool mt_in_rcu(struct maple_tree *mt)
 {
-#ifdef CONFIG_MAPLE_RCU_DISABLED
-	return false;
-#endif
 	return mt->ma_flags & MT_FLAGS_USE_RCU;
 }
 
diff --git a/include/linux/math64.h b/include/linux/math64.h
index cc305206d89ff..99189410d4bb7 100644
--- a/include/linux/math64.h
+++ b/include/linux/math64.h
@@ -57,6 +57,20 @@ static inline u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder)
 	return dividend / divisor;
 }
 
+/**
+ * div64_s64_rem - signed 64bit divide with 64bit divisor and remainder
+ * @dividend: signed 64bit dividend
+ * @divisor: signed 64bit divisor
+ * @remainder: pointer to signed 64bit remainder
+ *
+ * Return: sets ``*remainder``, then returns dividend / divisor
+ */
+static inline s64 div64_s64_rem(s64 dividend, s64 divisor, s64 *remainder)
+{
+	*remainder = dividend % divisor;
+	return dividend / divisor;
+}
+
 /**
  * div64_u64 - unsigned 64bit divide with 64bit divisor
  * @dividend: unsigned 64bit dividend
@@ -102,6 +116,10 @@ extern s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder);
 extern u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder);
 #endif
 
+#ifndef div64_s64_rem
+extern s64 div64_s64_rem(s64 dividend, s64 divisor, s64 *remainder);
+#endif
+
 #ifndef div64_u64
 extern u64 div64_u64(u64 dividend, u64 divisor);
 #endif
diff --git a/include/linux/mdio.h b/include/linux/mdio.h
index f4f9d96094488..300805e66592d 100644
--- a/include/linux/mdio.h
+++ b/include/linux/mdio.h
@@ -8,7 +8,6 @@
 
 #include <uapi/linux/mdio.h>
 #include <linux/bitfield.h>
-#include <linux/mod_devicetable.h>
 
 struct gpio_desc;
 struct mii_bus;
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index e1f46a0016fcf..9572606776781 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -1472,6 +1472,31 @@ static inline void lruvec_lock_irq(struct lruvec *lruvec)
 	spin_lock_irq(&lruvec->lru_lock);
 }
 
+static inline struct lruvec *lruvec_live_lock_irq(struct lruvec *lruvec)
+{
+#ifdef CONFIG_MEMCG
+	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
+	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
+
+	rcu_read_lock();
+
+	/*
+	 * The memcg can be NULL when the memory controller is disabled.
+	 * Otherwise, the caller keeps the memcg owning @lruvec alive.
+	 */
+	while (unlikely(memcg && css_is_dying(&memcg->css))) {
+		memcg = parent_mem_cgroup(memcg);
+		lruvec = mem_cgroup_lruvec(memcg, pgdat);
+	}
+
+	spin_lock_irq(&lruvec->lru_lock);
+#else
+	lruvec_lock_irq(lruvec);
+#endif
+
+	return lruvec;
+}
+
 static inline void lruvec_unlock(struct lruvec *lruvec)
 {
 	spin_unlock(&lruvec->lru_lock);
diff --git a/include/linux/memfd.h b/include/linux/memfd.h
index b4fda09dab9f6..c159e40e3f34a 100644
--- a/include/linux/memfd.h
+++ b/include/linux/memfd.h
@@ -14,9 +14,9 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx);
  * to sealing, or 0 otherwise.
  *
  * We also update VMA flags if appropriate by manipulating the VMA flags pointed
- * to by vm_flags_ptr.
+ * to by vma_flags_ptr.
  */
-int memfd_check_seals_mmap(struct file *file, vm_flags_t *vm_flags_ptr);
+int memfd_check_seals_mmap(struct file *file, vma_flags_t *vma_flags_ptr);
 struct file *memfd_alloc_file(const char *name, unsigned int flags);
 int memfd_get_seals(struct file *file);
 int memfd_add_seals(struct file *file, unsigned int seals);
@@ -30,7 +30,7 @@ static inline struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx)
 	return ERR_PTR(-EINVAL);
 }
 static inline int memfd_check_seals_mmap(struct file *file,
-					 vm_flags_t *vm_flags_ptr)
+					 vma_flags_t *vma_flags_ptr)
 {
 	return 0;
 }
diff --git a/include/linux/memory.h b/include/linux/memory.h
index 463dc02f6cff0..1783299073e47 100644
--- a/include/linux/memory.h
+++ b/include/linux/memory.h
@@ -20,6 +20,7 @@
 #include <linux/compiler.h>
 #include <linux/mutex.h>
 #include <linux/memory_hotplug.h>
+#include <linux/range.h>
 
 #define MIN_MEMORY_BLOCK_SIZE     (1UL << SECTION_SIZE_BITS)
 
@@ -100,6 +101,32 @@ int arch_get_memory_phys_device(unsigned long start_pfn);
 unsigned long memory_block_size_bytes(void);
 int set_memory_block_size_order(unsigned int order);
 
+/**
+ * memory_block_aligned_range - align a physical address range to memory blocks
+ * @range: the input range to align
+ *
+ * Aligns the start address up and the end address down to memory block
+ * boundaries. This is required for memory hotplug operations which must
+ * operate on memory-block aligned ranges.
+ *
+ * Returns the aligned range. Callers should check that the returned
+ * range is valid (aligned.start < aligned.end) before using it.
+ */
+static inline struct range memory_block_aligned_range(const struct range *range)
+{
+	struct range aligned;
+
+	aligned.start = ALIGN(range->start, memory_block_size_bytes());
+	aligned.end = ALIGN_DOWN(range->end + 1, memory_block_size_bytes());
+	/* No whole block fits (e.g. range below the first boundary): empty. */
+	if (aligned.end <= aligned.start)
+		aligned.start = aligned.end;
+	else
+		aligned.end -= 1;
+
+	return aligned;
+}
+
 struct memory_notify {
 	unsigned long start_pfn;
 	unsigned long nr_pages;
diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h
index 7c9d66729c609..b39605d308963 100644
--- a/include/linux/memory_hotplug.h
+++ b/include/linux/memory_hotplug.h
@@ -127,6 +127,7 @@ extern int arch_add_memory(int nid, u64 start, u64 size,
 extern u64 max_mem_size;
 
 extern int mhp_online_type_from_str(const char *str);
+const char *mhp_online_type_to_str(int online_type);
 
 /* If movable_node boot option specified */
 extern bool movable_node_enabled;
@@ -267,6 +268,8 @@ extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
 extern int remove_memory(u64 start, u64 size);
 extern void __remove_memory(u64 start, u64 size);
 extern int offline_and_remove_memory(u64 start, u64 size);
+int offline_and_remove_memory_ranges(const struct range *ranges,
+		unsigned int nr_ranges);
 
 #else
 static inline void try_offline_node(int nid) {}
@@ -283,17 +286,26 @@ static inline int remove_memory(u64 start, u64 size)
 }
 
 static inline void __remove_memory(u64 start, u64 size) {}
+
+static inline int offline_and_remove_memory_ranges(const struct range *ranges,
+		unsigned int nr_ranges)
+{
+	return -EBUSY;
+}
 #endif /* CONFIG_MEMORY_HOTREMOVE */
 
 #ifdef CONFIG_MEMORY_HOTPLUG
 /* Default online_type (MMOP_*) when new memory blocks are added. */
 extern enum mmop mhp_get_default_online_type(void);
 extern void mhp_set_default_online_type(enum mmop online_type);
-extern void __ref free_area_init_core_hotplug(struct pglist_data *pgdat);
+int __ref free_area_init_core_hotplug(struct pglist_data *pgdat);
 extern int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags);
 extern int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags);
 extern int add_memory_resource(int nid, struct resource *resource,
 			       mhp_t mhp_flags);
+int __add_memory_driver_managed(int nid, u64 start, u64 size,
+				const char *resource_name, mhp_t mhp_flags,
+				enum mmop online_type);
 extern int add_memory_driver_managed(int nid, u64 start, u64 size,
 				     const char *resource_name,
 				     mhp_t mhp_flags);
@@ -316,6 +328,8 @@ extern struct zone *zone_for_pfn_range(enum mmop online_type,
 extern int arch_create_linear_mapping(int nid, u64 start, u64 size,
 				      struct mhp_params *params);
 void arch_remove_linear_mapping(u64 start, u64 size);
+#else
+static inline enum mmop mhp_get_default_online_type(void) { return MMOP_OFFLINE; }
 #endif /* CONFIG_MEMORY_HOTPLUG */
 
 #endif /* __LINUX_MEMORY_HOTPLUG_H */
diff --git a/include/linux/mfd/88pm886.h b/include/linux/mfd/88pm886.h
index 38892ba7b8a42..2c24dd3032ab3 100644
--- a/include/linux/mfd/88pm886.h
+++ b/include/linux/mfd/88pm886.h
@@ -11,6 +11,7 @@
 
 #define PM886_PAGE_OFFSET_REGULATORS	1
 #define PM886_PAGE_OFFSET_GPADC		2
+#define PM886_PAGE_OFFSET_BATTERY	3
 
 #define PM886_REG_ID			0x00
 
@@ -128,9 +129,13 @@
 #define PM886_GPADC_BIAS_LEVELS		16
 #define PM886_GPADC_INDEX_TO_BIAS_uA(i)	(1 + (i) * 5)
 
+/* Battery block register definitions */
+#define PM886_REG_CLS_CONFIG1		0x71
+
 struct pm886_chip {
 	struct i2c_client *client;
 	unsigned int chip_id;
 	struct regmap *regmap;
+	struct regmap *regmap_battery;
 };
 #endif /* __MFD_88PM886_H */
diff --git a/include/linux/mfd/cs42l43.h b/include/linux/mfd/cs42l43.h
index ff0f7e365a19f..8e993fb535e68 100644
--- a/include/linux/mfd/cs42l43.h
+++ b/include/linux/mfd/cs42l43.h
@@ -86,7 +86,6 @@ struct cs42l43 {
 	struct regmap_irq_chip_data *irq_data;
 
 	struct work_struct boot_work;
-	struct completion device_attach;
 	struct completion device_detach;
 	struct completion firmware_download;
 	int firmware_error;
@@ -96,7 +95,6 @@ struct cs42l43 {
 	struct mutex pll_lock;
 
 	bool sdw_pll_active;
-	bool attached;
 	bool hw_lock;
 	long variant_id;
 };
diff --git a/include/linux/mfd/db8500-prcmu.h b/include/linux/mfd/db8500-prcmu.h
index a62de3d155edc..c939c9a1170a0 100644
--- a/include/linux/mfd/db8500-prcmu.h
+++ b/include/linux/mfd/db8500-prcmu.h
@@ -12,6 +12,9 @@
 
 #include <linux/interrupt.h>
 #include <linux/bitops.h>
+#include <linux/err.h>
+
+#include <dt-bindings/mfd/dbx500-prcmu.h> /* For clock identifiers */
 
 /*
  * Registers
@@ -24,6 +27,38 @@
 #define DB8500_PRCM_DSI_SW_RESET_DSI1_SW_RESETN BIT(1)
 #define DB8500_PRCM_DSI_SW_RESET_DSI2_SW_RESETN BIT(2)
 
+/* Offset for the firmware version within the TCPM */
+#define DB8500_PRCMU_FW_VERSION_OFFSET 0xA4
+
+#define DB8500_PRCMU_LEGACY_OFFSET		0xDD4
+
+/*
+ * CLKOUT sources
+ */
+#define PRCMU_CLKSRC_CLK38M		0x00
+#define PRCMU_CLKSRC_ACLK		0x01
+#define PRCMU_CLKSRC_SYSCLK		0x02
+#define PRCMU_CLKSRC_LCDCLK		0x03
+#define PRCMU_CLKSRC_SDMMCCLK		0x04
+#define PRCMU_CLKSRC_TVCLK		0x05
+#define PRCMU_CLKSRC_TIMCLK		0x06
+#define PRCMU_CLKSRC_CLK009		0x07
+/* These are only valid for CLKOUT1: */
+#define PRCMU_CLKSRC_SIAMMDSPCLK	0x40
+#define PRCMU_CLKSRC_I2CCLK		0x41
+#define PRCMU_CLKSRC_MSP02CLK		0x42
+#define PRCMU_CLKSRC_ARMPLL_OBSCLK	0x43
+#define PRCMU_CLKSRC_HSIRXCLK		0x44
+#define PRCMU_CLKSRC_HSITXCLK		0x45
+#define PRCMU_CLKSRC_ARMCLKFIX		0x46
+#define PRCMU_CLKSRC_HDMICLK		0x47
+
+/*
+ * Definitions for controlling ESRAM0 in deep sleep.
+ */
+#define ESRAM0_DEEP_SLEEP_STATE_OFF 1
+#define ESRAM0_DEEP_SLEEP_STATE_RET 2
+
 /* This portion previously known as <mach/prcmu-fw-defs_v1.h> */
 
 /**
@@ -451,10 +486,173 @@ enum prcmu_power_status {
 	PRCMU_ARMPENDINGIT_ER		= 0x93,
 };
 
+/* PRCMU Wakeup defines */
+enum prcmu_wakeup_index {
+	PRCMU_WAKEUP_INDEX_RTC,
+	PRCMU_WAKEUP_INDEX_RTT0,
+	PRCMU_WAKEUP_INDEX_RTT1,
+	PRCMU_WAKEUP_INDEX_HSI0,
+	PRCMU_WAKEUP_INDEX_HSI1,
+	PRCMU_WAKEUP_INDEX_USB,
+	PRCMU_WAKEUP_INDEX_ABB,
+	PRCMU_WAKEUP_INDEX_ABB_FIFO,
+	PRCMU_WAKEUP_INDEX_ARM,
+	PRCMU_WAKEUP_INDEX_CD_IRQ,
+	NUM_PRCMU_WAKEUP_INDICES
+};
+
+#define PRCMU_WAKEUP(_name) (BIT(PRCMU_WAKEUP_INDEX_##_name))
+
+/**
+ * enum prcmu_wdog_id - PRCMU watchdog IDs
+ * @PRCMU_WDOG_ALL: use all timers
+ * @PRCMU_WDOG_CPU1: use first CPU timer only
+ * @PRCMU_WDOG_CPU2: use second CPU timer conly
+ */
+enum prcmu_wdog_id {
+	PRCMU_WDOG_ALL = 0x00,
+	PRCMU_WDOG_CPU1 = 0x01,
+	PRCMU_WDOG_CPU2 = 0x02,
+};
+
+/**
+ * enum ape_opp - APE OPP states definition
+ * @APE_OPP_INIT:
+ * @APE_NO_CHANGE: The APE operating point is unchanged
+ * @APE_100_OPP: The new APE operating point is ape100opp
+ * @APE_50_OPP: 50%
+ * @APE_50_PARTLY_25_OPP: 50%, except some clocks at 25%.
+ */
+enum ape_opp {
+	APE_OPP_INIT = 0x00,
+	APE_NO_CHANGE = 0x01,
+	APE_100_OPP = 0x02,
+	APE_50_OPP = 0x03,
+	APE_50_PARTLY_25_OPP = 0xFF,
+};
+
+/**
+ * enum arm_opp - ARM OPP states definition
+ * @ARM_OPP_INIT:
+ * @ARM_NO_CHANGE: The ARM operating point is unchanged
+ * @ARM_100_OPP: The new ARM operating point is arm100opp
+ * @ARM_50_OPP: The new ARM operating point is arm50opp
+ * @ARM_MAX_OPP: Operating point is "max" (more than 100)
+ * @ARM_MAX_FREQ100OPP: Set max opp if available, else 100
+ * @ARM_EXTCLK: The new ARM operating point is armExtClk
+ */
+enum arm_opp {
+	ARM_OPP_INIT = 0x00,
+	ARM_NO_CHANGE = 0x01,
+	ARM_100_OPP = 0x02,
+	ARM_50_OPP = 0x03,
+	ARM_MAX_OPP = 0x04,
+	ARM_MAX_FREQ100OPP = 0x05,
+	ARM_EXTCLK = 0x07
+};
+
+/**
+ * enum ddr_opp - DDR OPP states definition
+ * @DDR_100_OPP: The new DDR operating point is ddr100opp
+ * @DDR_50_OPP: The new DDR operating point is ddr50opp
+ * @DDR_25_OPP: The new DDR operating point is ddr25opp
+ */
+enum ddr_opp {
+	DDR_100_OPP = 0x00,
+	DDR_50_OPP = 0x01,
+	DDR_25_OPP = 0x02,
+};
+
+/**
+ * enum ddr_pwrst - DDR power states definition
+ * @DDR_PWR_STATE_UNCHANGED: SDRAM and DDR controller state is unchanged
+ * @DDR_PWR_STATE_ON:
+ * @DDR_PWR_STATE_OFFLOWLAT:
+ * @DDR_PWR_STATE_OFFHIGHLAT:
+ */
+enum ddr_pwrst {
+	DDR_PWR_STATE_UNCHANGED     = 0x00,
+	DDR_PWR_STATE_ON            = 0x01,
+	DDR_PWR_STATE_OFFLOWLAT     = 0x02,
+	DDR_PWR_STATE_OFFHIGHLAT    = 0x03
+};
+
 /*
  * Definitions for autonomous power management configuration.
  */
 
+/* EPOD (power domain) IDs */
+
+/*
+ * DB8500 EPODs
+ * - EPOD_ID_SVAMMDSP: power domain for SVA MMDSP
+ * - EPOD_ID_SVAPIPE: power domain for SVA pipe
+ * - EPOD_ID_SIAMMDSP: power domain for SIA MMDSP
+ * - EPOD_ID_SIAPIPE: power domain for SIA pipe
+ * - EPOD_ID_SGA: power domain for SGA
+ * - EPOD_ID_B2R2_MCDE: power domain for B2R2 and MCDE
+ * - EPOD_ID_ESRAM12: power domain for ESRAM 1 and 2
+ * - EPOD_ID_ESRAM34: power domain for ESRAM 3 and 4
+ * - NUM_EPOD_ID: number of power domains
+ *
+ * TODO: These should be prefixed.
+ */
+#define EPOD_ID_SVAMMDSP	0
+#define EPOD_ID_SVAPIPE		1
+#define EPOD_ID_SIAMMDSP	2
+#define EPOD_ID_SIAPIPE		3
+#define EPOD_ID_SGA		4
+#define EPOD_ID_B2R2_MCDE	5
+#define EPOD_ID_ESRAM12		6
+#define EPOD_ID_ESRAM34		7
+#define NUM_EPOD_ID		8
+
+/*
+ * state definition for EPOD (power domain)
+ * - EPOD_STATE_NO_CHANGE: The EPOD should remain unchanged
+ * - EPOD_STATE_OFF: The EPOD is switched off
+ * - EPOD_STATE_RAMRET: The EPOD is switched off with its internal RAM in
+ *                         retention
+ * - EPOD_STATE_ON_CLK_OFF: The EPOD is switched on, clock is still off
+ * - EPOD_STATE_ON: Same as above, but with clock enabled
+ */
+#define EPOD_STATE_NO_CHANGE	0x00
+#define EPOD_STATE_OFF		0x01
+#define EPOD_STATE_RAMRET	0x02
+#define EPOD_STATE_ON_CLK_OFF	0x03
+#define EPOD_STATE_ON		0x04
+
+#define PRCMU_FW_PROJECT_U8500		2
+#define PRCMU_FW_PROJECT_U8400		3
+#define PRCMU_FW_PROJECT_U9500		4 /* Customer specific */
+#define PRCMU_FW_PROJECT_U8500_MBB	5
+#define PRCMU_FW_PROJECT_U8500_C1	6
+#define PRCMU_FW_PROJECT_U8500_C2	7
+#define PRCMU_FW_PROJECT_U8500_C3	8
+#define PRCMU_FW_PROJECT_U8500_C4	9
+#define PRCMU_FW_PROJECT_U9500_MBL	10
+#define PRCMU_FW_PROJECT_U8500_SSG1	11 /* Samsung specific */
+#define PRCMU_FW_PROJECT_U8500_MBL2	12 /* Customer specific */
+#define PRCMU_FW_PROJECT_U8520		13
+#define PRCMU_FW_PROJECT_U8420		14
+#define PRCMU_FW_PROJECT_U8500_SSG2	15 /* Samsung specific */
+#define PRCMU_FW_PROJECT_U8420_SYSCLK	17
+#define PRCMU_FW_PROJECT_A9420		20
+/* [32..63] 9540 and derivatives */
+#define PRCMU_FW_PROJECT_U9540		32
+/* [64..95] 8540 and derivatives */
+#define PRCMU_FW_PROJECT_L8540		64
+/* [96..126] 8580 and derivatives */
+#define PRCMU_FW_PROJECT_L8580		96
+
+#define PRCMU_FW_PROJECT_NAME_LEN	20
+
+/* PRCMU QoS APE OPP class */
+#define PRCMU_QOS_APE_OPP 1
+#define PRCMU_QOS_DDR_OPP 2
+#define PRCMU_QOS_ARM_OPP 3
+#define PRCMU_QOS_DEFAULT_VALUE -1
+
 #define PRCMU_AUTO_PM_OFF 0
 #define PRCMU_AUTO_PM_ON 1
 
@@ -469,6 +667,14 @@ enum prcmu_auto_pm_policy {
 	PRCMU_AUTO_PM_POLICY_DSP_CLK_OFF_HWP_CLK_OFF,
 };
 
+struct prcmu_fw_version {
+	u32 project; /* Notice, project shifted with 8 on ux540 */
+	u8 api_version;
+	u8 func_version;
+	u8 errata;
+	char project_name[PRCMU_FW_PROJECT_NAME_LEN];
+};
+
 /**
  * struct prcmu_auto_pm_config - Autonomous power management configuration.
  * @sia_auto_pm_enable: SIA autonomous pm enable. (PRCMU_AUTO_PM_{OFF,ON})
@@ -501,6 +707,9 @@ void prcmu_configure_auto_pm(struct prcmu_auto_pm_config *sleep,
 bool prcmu_is_auto_pm_enabled(void);
 
 int prcmu_config_clkout(u8 clkout, u8 source, u8 div);
+unsigned long prcmu_clock_rate(u8 clock);
+long prcmu_round_clock_rate(u8 clock, unsigned long rate);
+int prcmu_set_clock_rate(u8 clock, unsigned long rate);
 int prcmu_set_clock_divider(u8 clock, u8 divider);
 int db8500_prcmu_config_hotdog(u8 threshold);
 int db8500_prcmu_config_hotmon(u8 low, u8 high);
@@ -508,6 +717,8 @@ int db8500_prcmu_start_temp_sense(u16 cycles32k);
 int db8500_prcmu_stop_temp_sense(void);
 int prcmu_abb_read(u8 slave, u8 reg, u8 *value, u8 size);
 int prcmu_abb_write(u8 slave, u8 reg, u8 *value, u8 size);
+int prcmu_abb_write_masked(u8 slave, u8 reg, u8 *value,
+			   u8 *mask, u8 size);
 
 int prcmu_ac_wake_req(void);
 void prcmu_ac_sleep_req(void);
@@ -610,6 +821,21 @@ static inline int prcmu_config_clkout(u8 clkout, u8 source, u8 div)
 	return 0;
 }
 
+static inline unsigned long prcmu_clock_rate(u8 clock)
+{
+	return 0;
+}
+
+static inline long prcmu_round_clock_rate(u8 clock, unsigned long rate)
+{
+	return 0;
+}
+
+static inline int prcmu_set_clock_rate(u8 clock, unsigned long rate)
+{
+	return 0;
+}
+
 static inline int prcmu_set_clock_divider(u8 clock, u8 divider)
 {
 	return 0;
@@ -637,12 +863,18 @@ static inline int db8500_prcmu_stop_temp_sense(void)
 
 static inline int prcmu_abb_read(u8 slave, u8 reg, u8 *value, u8 size)
 {
-	return -ENOSYS;
+	return -EINVAL;
 }
 
 static inline int prcmu_abb_write(u8 slave, u8 reg, u8 *value, u8 size)
 {
-	return -ENOSYS;
+	return -EINVAL;
+}
+
+static inline int prcmu_abb_write_masked(u8 slave, u8 reg,
+					 u8 *value, u8 *mask, u8 size)
+{
+	return -EINVAL;
 }
 
 static inline int prcmu_ac_wake_req(void)
@@ -745,4 +977,20 @@ static inline void db8500_prcmu_write_masked(unsigned int reg, u32 mask,
 
 #endif /* !CONFIG_MFD_DB8500_PRCMU */
 
+static inline int prcmu_qos_add_requirement(int prcmu_qos_class,
+					    char *name, s32 value)
+{
+	return 0;
+}
+
+static inline int prcmu_qos_update_requirement(int prcmu_qos_class,
+					       char *name, s32 new_value)
+{
+	return 0;
+}
+
+static inline void prcmu_qos_remove_requirement(int prcmu_qos_class, char *name)
+{
+}
+
 #endif /* __MFD_DB8500_PRCMU_H */
diff --git a/include/linux/mfd/dbx500-prcmu.h b/include/linux/mfd/dbx500-prcmu.h
deleted file mode 100644
index 828362b7860c6..0000000000000
--- a/include/linux/mfd/dbx500-prcmu.h
+++ /dev/null
@@ -1,575 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) ST Ericsson SA 2011
- *
- * STE Ux500 PRCMU API
- */
-#ifndef __MACH_PRCMU_H
-#define __MACH_PRCMU_H
-
-#include <linux/interrupt.h>
-#include <linux/notifier.h>
-#include <linux/err.h>
-
-#include <dt-bindings/mfd/dbx500-prcmu.h> /* For clock identifiers */
-
-/* Offset for the firmware version within the TCPM */
-#define DB8500_PRCMU_FW_VERSION_OFFSET 0xA4
-#define DBX540_PRCMU_FW_VERSION_OFFSET 0xA8
-
-/* PRCMU Wakeup defines */
-enum prcmu_wakeup_index {
-	PRCMU_WAKEUP_INDEX_RTC,
-	PRCMU_WAKEUP_INDEX_RTT0,
-	PRCMU_WAKEUP_INDEX_RTT1,
-	PRCMU_WAKEUP_INDEX_HSI0,
-	PRCMU_WAKEUP_INDEX_HSI1,
-	PRCMU_WAKEUP_INDEX_USB,
-	PRCMU_WAKEUP_INDEX_ABB,
-	PRCMU_WAKEUP_INDEX_ABB_FIFO,
-	PRCMU_WAKEUP_INDEX_ARM,
-	PRCMU_WAKEUP_INDEX_CD_IRQ,
-	NUM_PRCMU_WAKEUP_INDICES
-};
-#define PRCMU_WAKEUP(_name) (BIT(PRCMU_WAKEUP_INDEX_##_name))
-
-/* EPOD (power domain) IDs */
-
-/*
- * DB8500 EPODs
- * - EPOD_ID_SVAMMDSP: power domain for SVA MMDSP
- * - EPOD_ID_SVAPIPE: power domain for SVA pipe
- * - EPOD_ID_SIAMMDSP: power domain for SIA MMDSP
- * - EPOD_ID_SIAPIPE: power domain for SIA pipe
- * - EPOD_ID_SGA: power domain for SGA
- * - EPOD_ID_B2R2_MCDE: power domain for B2R2 and MCDE
- * - EPOD_ID_ESRAM12: power domain for ESRAM 1 and 2
- * - EPOD_ID_ESRAM34: power domain for ESRAM 3 and 4
- * - NUM_EPOD_ID: number of power domains
- *
- * TODO: These should be prefixed.
- */
-#define EPOD_ID_SVAMMDSP	0
-#define EPOD_ID_SVAPIPE		1
-#define EPOD_ID_SIAMMDSP	2
-#define EPOD_ID_SIAPIPE		3
-#define EPOD_ID_SGA		4
-#define EPOD_ID_B2R2_MCDE	5
-#define EPOD_ID_ESRAM12		6
-#define EPOD_ID_ESRAM34		7
-#define NUM_EPOD_ID		8
-
-/*
- * state definition for EPOD (power domain)
- * - EPOD_STATE_NO_CHANGE: The EPOD should remain unchanged
- * - EPOD_STATE_OFF: The EPOD is switched off
- * - EPOD_STATE_RAMRET: The EPOD is switched off with its internal RAM in
- *                         retention
- * - EPOD_STATE_ON_CLK_OFF: The EPOD is switched on, clock is still off
- * - EPOD_STATE_ON: Same as above, but with clock enabled
- */
-#define EPOD_STATE_NO_CHANGE	0x00
-#define EPOD_STATE_OFF		0x01
-#define EPOD_STATE_RAMRET	0x02
-#define EPOD_STATE_ON_CLK_OFF	0x03
-#define EPOD_STATE_ON		0x04
-
-/*
- * CLKOUT sources
- */
-#define PRCMU_CLKSRC_CLK38M		0x00
-#define PRCMU_CLKSRC_ACLK		0x01
-#define PRCMU_CLKSRC_SYSCLK		0x02
-#define PRCMU_CLKSRC_LCDCLK		0x03
-#define PRCMU_CLKSRC_SDMMCCLK		0x04
-#define PRCMU_CLKSRC_TVCLK		0x05
-#define PRCMU_CLKSRC_TIMCLK		0x06
-#define PRCMU_CLKSRC_CLK009		0x07
-/* These are only valid for CLKOUT1: */
-#define PRCMU_CLKSRC_SIAMMDSPCLK	0x40
-#define PRCMU_CLKSRC_I2CCLK		0x41
-#define PRCMU_CLKSRC_MSP02CLK		0x42
-#define PRCMU_CLKSRC_ARMPLL_OBSCLK	0x43
-#define PRCMU_CLKSRC_HSIRXCLK		0x44
-#define PRCMU_CLKSRC_HSITXCLK		0x45
-#define PRCMU_CLKSRC_ARMCLKFIX		0x46
-#define PRCMU_CLKSRC_HDMICLK		0x47
-
-/**
- * enum prcmu_wdog_id - PRCMU watchdog IDs
- * @PRCMU_WDOG_ALL: use all timers
- * @PRCMU_WDOG_CPU1: use first CPU timer only
- * @PRCMU_WDOG_CPU2: use second CPU timer conly
- */
-enum prcmu_wdog_id {
-	PRCMU_WDOG_ALL = 0x00,
-	PRCMU_WDOG_CPU1 = 0x01,
-	PRCMU_WDOG_CPU2 = 0x02,
-};
-
-/**
- * enum ape_opp - APE OPP states definition
- * @APE_OPP_INIT:
- * @APE_NO_CHANGE: The APE operating point is unchanged
- * @APE_100_OPP: The new APE operating point is ape100opp
- * @APE_50_OPP: 50%
- * @APE_50_PARTLY_25_OPP: 50%, except some clocks at 25%.
- */
-enum ape_opp {
-	APE_OPP_INIT = 0x00,
-	APE_NO_CHANGE = 0x01,
-	APE_100_OPP = 0x02,
-	APE_50_OPP = 0x03,
-	APE_50_PARTLY_25_OPP = 0xFF,
-};
-
-/**
- * enum arm_opp - ARM OPP states definition
- * @ARM_OPP_INIT:
- * @ARM_NO_CHANGE: The ARM operating point is unchanged
- * @ARM_100_OPP: The new ARM operating point is arm100opp
- * @ARM_50_OPP: The new ARM operating point is arm50opp
- * @ARM_MAX_OPP: Operating point is "max" (more than 100)
- * @ARM_MAX_FREQ100OPP: Set max opp if available, else 100
- * @ARM_EXTCLK: The new ARM operating point is armExtClk
- */
-enum arm_opp {
-	ARM_OPP_INIT = 0x00,
-	ARM_NO_CHANGE = 0x01,
-	ARM_100_OPP = 0x02,
-	ARM_50_OPP = 0x03,
-	ARM_MAX_OPP = 0x04,
-	ARM_MAX_FREQ100OPP = 0x05,
-	ARM_EXTCLK = 0x07
-};
-
-/**
- * enum ddr_opp - DDR OPP states definition
- * @DDR_100_OPP: The new DDR operating point is ddr100opp
- * @DDR_50_OPP: The new DDR operating point is ddr50opp
- * @DDR_25_OPP: The new DDR operating point is ddr25opp
- */
-enum ddr_opp {
-	DDR_100_OPP = 0x00,
-	DDR_50_OPP = 0x01,
-	DDR_25_OPP = 0x02,
-};
-
-/*
- * Definitions for controlling ESRAM0 in deep sleep.
- */
-#define ESRAM0_DEEP_SLEEP_STATE_OFF 1
-#define ESRAM0_DEEP_SLEEP_STATE_RET 2
-
-/**
- * enum ddr_pwrst - DDR power states definition
- * @DDR_PWR_STATE_UNCHANGED: SDRAM and DDR controller state is unchanged
- * @DDR_PWR_STATE_ON:
- * @DDR_PWR_STATE_OFFLOWLAT:
- * @DDR_PWR_STATE_OFFHIGHLAT:
- */
-enum ddr_pwrst {
-	DDR_PWR_STATE_UNCHANGED     = 0x00,
-	DDR_PWR_STATE_ON            = 0x01,
-	DDR_PWR_STATE_OFFLOWLAT     = 0x02,
-	DDR_PWR_STATE_OFFHIGHLAT    = 0x03
-};
-
-#define DB8500_PRCMU_LEGACY_OFFSET		0xDD4
-
-#define PRCMU_FW_PROJECT_U8500		2
-#define PRCMU_FW_PROJECT_U8400		3
-#define PRCMU_FW_PROJECT_U9500		4 /* Customer specific */
-#define PRCMU_FW_PROJECT_U8500_MBB	5
-#define PRCMU_FW_PROJECT_U8500_C1	6
-#define PRCMU_FW_PROJECT_U8500_C2	7
-#define PRCMU_FW_PROJECT_U8500_C3	8
-#define PRCMU_FW_PROJECT_U8500_C4	9
-#define PRCMU_FW_PROJECT_U9500_MBL	10
-#define PRCMU_FW_PROJECT_U8500_SSG1	11 /* Samsung specific */
-#define PRCMU_FW_PROJECT_U8500_MBL2	12 /* Customer specific */
-#define PRCMU_FW_PROJECT_U8520		13
-#define PRCMU_FW_PROJECT_U8420		14
-#define PRCMU_FW_PROJECT_U8500_SSG2	15 /* Samsung specific */
-#define PRCMU_FW_PROJECT_U8420_SYSCLK	17
-#define PRCMU_FW_PROJECT_A9420		20
-/* [32..63] 9540 and derivatives */
-#define PRCMU_FW_PROJECT_U9540		32
-/* [64..95] 8540 and derivatives */
-#define PRCMU_FW_PROJECT_L8540		64
-/* [96..126] 8580 and derivatives */
-#define PRCMU_FW_PROJECT_L8580		96
-
-#define PRCMU_FW_PROJECT_NAME_LEN	20
-struct prcmu_fw_version {
-	u32 project; /* Notice, project shifted with 8 on ux540 */
-	u8 api_version;
-	u8 func_version;
-	u8 errata;
-	char project_name[PRCMU_FW_PROJECT_NAME_LEN];
-};
-
-#include <linux/mfd/db8500-prcmu.h>
-
-#if defined(CONFIG_UX500_SOC_DB8500)
-
-static inline void __init prcmu_early_init(void)
-{
-	db8500_prcmu_early_init();
-}
-
-static inline int prcmu_set_power_state(u8 state, bool keep_ulp_clk,
-		bool keep_ap_pll)
-{
-	return db8500_prcmu_set_power_state(state, keep_ulp_clk,
-		keep_ap_pll);
-}
-
-static inline u8 prcmu_get_power_state_result(void)
-{
-	return db8500_prcmu_get_power_state_result();
-}
-
-static inline int prcmu_set_epod(u16 epod_id, u8 epod_state)
-{
-	return db8500_prcmu_set_epod(epod_id, epod_state);
-}
-
-static inline void prcmu_enable_wakeups(u32 wakeups)
-{
-	db8500_prcmu_enable_wakeups(wakeups);
-}
-
-static inline void prcmu_disable_wakeups(void)
-{
-	prcmu_enable_wakeups(0);
-}
-
-static inline void prcmu_config_abb_event_readout(u32 abb_events)
-{
-	db8500_prcmu_config_abb_event_readout(abb_events);
-}
-
-static inline void prcmu_get_abb_event_buffer(void __iomem **buf)
-{
-	db8500_prcmu_get_abb_event_buffer(buf);
-}
-
-int prcmu_abb_read(u8 slave, u8 reg, u8 *value, u8 size);
-int prcmu_abb_write(u8 slave, u8 reg, u8 *value, u8 size);
-int prcmu_abb_write_masked(u8 slave, u8 reg, u8 *value, u8 *mask, u8 size);
-
-int prcmu_config_clkout(u8 clkout, u8 source, u8 div);
-
-static inline int prcmu_request_clock(u8 clock, bool enable)
-{
-	return db8500_prcmu_request_clock(clock, enable);
-}
-
-unsigned long prcmu_clock_rate(u8 clock);
-long prcmu_round_clock_rate(u8 clock, unsigned long rate);
-int prcmu_set_clock_rate(u8 clock, unsigned long rate);
-
-static inline int prcmu_get_ddr_opp(void)
-{
-	return db8500_prcmu_get_ddr_opp();
-}
-
-static inline int prcmu_set_arm_opp(u8 opp)
-{
-	return db8500_prcmu_set_arm_opp(opp);
-}
-
-static inline int prcmu_get_arm_opp(void)
-{
-	return db8500_prcmu_get_arm_opp();
-}
-
-static inline int prcmu_set_ape_opp(u8 opp)
-{
-	return db8500_prcmu_set_ape_opp(opp);
-}
-
-static inline int prcmu_get_ape_opp(void)
-{
-	return db8500_prcmu_get_ape_opp();
-}
-
-static inline int prcmu_request_ape_opp_100_voltage(bool enable)
-{
-	return db8500_prcmu_request_ape_opp_100_voltage(enable);
-}
-
-static inline void prcmu_system_reset(u16 reset_code)
-{
-	db8500_prcmu_system_reset(reset_code);
-}
-
-static inline u16 prcmu_get_reset_code(void)
-{
-	return db8500_prcmu_get_reset_code();
-}
-
-int prcmu_ac_wake_req(void);
-void prcmu_ac_sleep_req(void);
-static inline void prcmu_modem_reset(void)
-{
-	db8500_prcmu_modem_reset();
-}
-
-static inline bool prcmu_is_ac_wake_requested(void)
-{
-	return db8500_prcmu_is_ac_wake_requested();
-}
-
-static inline int prcmu_config_esram0_deep_sleep(u8 state)
-{
-	return db8500_prcmu_config_esram0_deep_sleep(state);
-}
-
-static inline int prcmu_config_hotdog(u8 threshold)
-{
-	return db8500_prcmu_config_hotdog(threshold);
-}
-
-static inline int prcmu_config_hotmon(u8 low, u8 high)
-{
-	return db8500_prcmu_config_hotmon(low, high);
-}
-
-static inline int prcmu_start_temp_sense(u16 cycles32k)
-{
-	return  db8500_prcmu_start_temp_sense(cycles32k);
-}
-
-static inline int prcmu_stop_temp_sense(void)
-{
-	return  db8500_prcmu_stop_temp_sense();
-}
-
-static inline u32 prcmu_read(unsigned int reg)
-{
-	return db8500_prcmu_read(reg);
-}
-
-static inline void prcmu_write(unsigned int reg, u32 value)
-{
-	db8500_prcmu_write(reg, value);
-}
-
-static inline void prcmu_write_masked(unsigned int reg, u32 mask, u32 value)
-{
-	db8500_prcmu_write_masked(reg, mask, value);
-}
-
-static inline int prcmu_enable_a9wdog(u8 id)
-{
-	return db8500_prcmu_enable_a9wdog(id);
-}
-
-static inline int prcmu_disable_a9wdog(u8 id)
-{
-	return db8500_prcmu_disable_a9wdog(id);
-}
-
-static inline int prcmu_kick_a9wdog(u8 id)
-{
-	return db8500_prcmu_kick_a9wdog(id);
-}
-
-static inline int prcmu_load_a9wdog(u8 id, u32 timeout)
-{
-	return db8500_prcmu_load_a9wdog(id, timeout);
-}
-
-static inline int prcmu_config_a9wdog(u8 num, bool sleep_auto_off)
-{
-	return db8500_prcmu_config_a9wdog(num, sleep_auto_off);
-}
-#else
-
-static inline void prcmu_early_init(void) {}
-
-static inline int prcmu_set_power_state(u8 state, bool keep_ulp_clk,
-	bool keep_ap_pll)
-{
-	return 0;
-}
-
-static inline int prcmu_set_epod(u16 epod_id, u8 epod_state)
-{
-	return 0;
-}
-
-static inline void prcmu_enable_wakeups(u32 wakeups) {}
-
-static inline void prcmu_disable_wakeups(void) {}
-
-static inline int prcmu_abb_read(u8 slave, u8 reg, u8 *value, u8 size)
-{
-	return -ENOSYS;
-}
-
-static inline int prcmu_abb_write(u8 slave, u8 reg, u8 *value, u8 size)
-{
-	return -ENOSYS;
-}
-
-static inline int prcmu_abb_write_masked(u8 slave, u8 reg, u8 *value, u8 *mask,
-	u8 size)
-{
-	return -ENOSYS;
-}
-
-static inline int prcmu_config_clkout(u8 clkout, u8 source, u8 div)
-{
-	return 0;
-}
-
-static inline int prcmu_request_clock(u8 clock, bool enable)
-{
-	return 0;
-}
-
-static inline long prcmu_round_clock_rate(u8 clock, unsigned long rate)
-{
-	return 0;
-}
-
-static inline int prcmu_set_clock_rate(u8 clock, unsigned long rate)
-{
-	return 0;
-}
-
-static inline unsigned long prcmu_clock_rate(u8 clock)
-{
-	return 0;
-}
-
-static inline int prcmu_set_ape_opp(u8 opp)
-{
-	return 0;
-}
-
-static inline int prcmu_get_ape_opp(void)
-{
-	return APE_100_OPP;
-}
-
-static inline int prcmu_request_ape_opp_100_voltage(bool enable)
-{
-	return 0;
-}
-
-static inline int prcmu_set_arm_opp(u8 opp)
-{
-	return 0;
-}
-
-static inline int prcmu_get_arm_opp(void)
-{
-	return ARM_100_OPP;
-}
-
-static inline int prcmu_get_ddr_opp(void)
-{
-	return DDR_100_OPP;
-}
-
-static inline void prcmu_system_reset(u16 reset_code) {}
-
-static inline u16 prcmu_get_reset_code(void)
-{
-	return 0;
-}
-
-static inline int prcmu_ac_wake_req(void)
-{
-	return 0;
-}
-
-static inline void prcmu_ac_sleep_req(void) {}
-
-static inline void prcmu_modem_reset(void) {}
-
-static inline bool prcmu_is_ac_wake_requested(void)
-{
-	return false;
-}
-
-static inline int prcmu_config_esram0_deep_sleep(u8 state)
-{
-	return 0;
-}
-
-static inline void prcmu_config_abb_event_readout(u32 abb_events) {}
-
-static inline void prcmu_get_abb_event_buffer(void __iomem **buf)
-{
-	*buf = NULL;
-}
-
-static inline int prcmu_config_hotdog(u8 threshold)
-{
-	return 0;
-}
-
-static inline int prcmu_config_hotmon(u8 low, u8 high)
-{
-	return 0;
-}
-
-static inline int prcmu_start_temp_sense(u16 cycles32k)
-{
-	return 0;
-}
-
-static inline int prcmu_stop_temp_sense(void)
-{
-	return 0;
-}
-
-static inline u32 prcmu_read(unsigned int reg)
-{
-	return 0;
-}
-
-static inline void prcmu_write(unsigned int reg, u32 value) {}
-
-static inline void prcmu_write_masked(unsigned int reg, u32 mask, u32 value) {}
-
-#endif
-
-static inline void prcmu_set(unsigned int reg, u32 bits)
-{
-	prcmu_write_masked(reg, bits, bits);
-}
-
-static inline void prcmu_clear(unsigned int reg, u32 bits)
-{
-	prcmu_write_masked(reg, bits, 0);
-}
-
-/* PRCMU QoS APE OPP class */
-#define PRCMU_QOS_APE_OPP 1
-#define PRCMU_QOS_DDR_OPP 2
-#define PRCMU_QOS_ARM_OPP 3
-#define PRCMU_QOS_DEFAULT_VALUE -1
-
-static inline int prcmu_qos_add_requirement(int prcmu_qos_class,
-					    char *name, s32 value)
-{
-	return 0;
-}
-
-static inline int prcmu_qos_update_requirement(int prcmu_qos_class,
-					       char *name, s32 new_value)
-{
-	return 0;
-}
-
-static inline void prcmu_qos_remove_requirement(int prcmu_qos_class, char *name)
-{
-}
-
-#endif /* __MACH_PRCMU_H */
diff --git a/include/linux/mfd/max14577-private.h b/include/linux/mfd/max14577-private.h
index dd51a37fa37fa..5957e15b568ee 100644
--- a/include/linux/mfd/max14577-private.h
+++ b/include/linux/mfd/max14577-private.h
@@ -350,6 +350,9 @@ enum max77836_pmic_reg {
 #define MAX77836_CNFG1_LDO_PWRMD_SHIFT		6
 #define MAX77836_CNFG1_LDO_TV_SHIFT		0
 #define MAX77836_CNFG1_LDO_PWRMD_MASK		(0x3 << MAX77836_CNFG1_LDO_PWRMD_SHIFT)
+#define MAX77836_CNFG1_LDO_PWRMD_OFF		(0x0 << MAX77836_CNFG1_LDO_PWRMD_SHIFT)
+#define MAX77836_CNFG1_LDO_PWRMD_LPM		(0x1 << MAX77836_CNFG1_LDO_PWRMD_SHIFT)
+#define MAX77836_CNFG1_LDO_PWRMD_NORMAL		(0x3 << MAX77836_CNFG1_LDO_PWRMD_SHIFT)
 #define MAX77836_CNFG1_LDO_TV_MASK		(0x3f << MAX77836_CNFG1_LDO_TV_SHIFT)
 
 /* LDO1/LDO2 CONFIG2 register */
diff --git a/include/linux/mfd/si476x-core.h b/include/linux/mfd/si476x-core.h
index e913b2cdf77d5..d9e3a322134cd 100644
--- a/include/linux/mfd/si476x-core.h
+++ b/include/linux/mfd/si476x-core.h
@@ -14,6 +14,7 @@
 
 #include <linux/kfifo.h>
 #include <linux/atomic.h>
+#include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/regmap.h>
 #include <linux/mutex.h>
@@ -104,7 +105,7 @@ enum si476x_power_state {
  * @power_state: Current power state of the device.
  * @supplies: Structure containing handles to all power supplies used
  * by the device (NULL ones are ignored).
- * @gpio_reset: GPIO pin connectet to the RSTB pin of the chip.
+ * @reset: GPIO connected to the RSTB pin of the chip.
  * @pinmux: Chip's configurable pins configuration.
  * @diversity_mode: Chips role when functioning in diversity mode.
  * @is_alive: Chip is initialized and active.
@@ -142,7 +143,7 @@ struct si476x_core {
 
 	struct regulator_bulk_data supplies[4];
 
-	int gpio_reset;
+	struct gpio_desc *reset;
 
 	struct si476x_pinmux pinmux;
 	enum si476x_phase_diversity_mode diversity_mode;
diff --git a/include/linux/mfd/si476x-platform.h b/include/linux/mfd/si476x-platform.h
index cb99e16ca9473..f9e1f6b272772 100644
--- a/include/linux/mfd/si476x-platform.h
+++ b/include/linux/mfd/si476x-platform.h
@@ -246,8 +246,6 @@ enum si476x_phase_diversity_mode {
  * Platform dependent definition
  */
 struct si476x_platform_data {
-	int gpio_reset; /* < 0 if not used */
-
 	struct si476x_power_up_args power_up_parameters;
 	enum si476x_phase_diversity_mode diversity_mode;
 
diff --git a/include/linux/migrate.h b/include/linux/migrate.h
index d5af2b7f577b3..78424b3824c29 100644
--- a/include/linux/migrate.h
+++ b/include/linux/migrate.h
@@ -57,8 +57,9 @@ void putback_movable_pages(struct list_head *l);
 int migrate_folio(struct address_space *mapping, struct folio *dst,
 		struct folio *src, enum migrate_mode mode);
 int migrate_pages(struct list_head *l, new_folio_t new, free_folio_t free,
-		  unsigned long private, enum migrate_mode mode, int reason,
-		  unsigned int *ret_succeeded);
+		unsigned long private, enum migrate_mode mode,
+		enum migrate_reason reason,
+		unsigned int *ret_succeeded);
 struct folio *alloc_migration_target(struct folio *src, unsigned long private);
 bool isolate_movable_ops_page(struct page *page, isolate_mode_t mode);
 bool isolate_folio_to_list(struct folio *folio, struct list_head *list);
@@ -77,7 +78,8 @@ int set_movable_ops(const struct movable_operations *ops, enum pagetype type);
 static inline void putback_movable_pages(struct list_head *l) {}
 static inline int migrate_pages(struct list_head *l, new_folio_t new,
 		free_folio_t free, unsigned long private,
-		enum migrate_mode mode, int reason, unsigned int *ret_succeeded)
+		enum migrate_mode mode, enum migrate_reason reason,
+		unsigned int *ret_succeeded)
 	{ return -ENOSYS; }
 static inline struct folio *alloc_migration_target(struct folio *src,
 		unsigned long private)
diff --git a/include/linux/migrate_mode.h b/include/linux/migrate_mode.h
index 265c4328b36ad..05102d4d2490e 100644
--- a/include/linux/migrate_mode.h
+++ b/include/linux/migrate_mode.h
@@ -25,6 +25,7 @@ enum migrate_reason {
 	MR_LONGTERM_PIN,
 	MR_DEMOTION,
 	MR_DAMON,
+	MR_NEVER,		/* page has never been migrated */
 	MR_TYPES
 };
 
diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h
index 07a25f2642921..8cb321a9fb3d1 100644
--- a/include/linux/mlx5/device.h
+++ b/include/linux/mlx5/device.h
@@ -1171,6 +1171,7 @@ enum {
 	MLX5_MATCH_MISC_PARAMETERS_3	= 1 << 4,
 	MLX5_MATCH_MISC_PARAMETERS_4	= 1 << 5,
 	MLX5_MATCH_MISC_PARAMETERS_5	= 1 << 6,
+	MLX5_MATCH_MISC_PARAMETERS_6	= 1 << 7,
 };
 
 enum {
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index 695c86ee6d7a3..c7206a9d67318 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -508,7 +508,8 @@ struct mlx5_ifc_flow_table_prop_layout_bits {
 	u8         reformat_l2_to_l3_audp_tunnel[0x1];
 	u8         reformat_l3_audp_tunnel_to_l2[0x1];
 	u8         ignore_flow_level_rtc_valid[0x1];
-	u8         reserved_at_70[0x8];
+	u8         reserved_at_70[0x7];
+	u8         reformat_del_psp_transport[0x1];
 	u8         log_max_ft_num[0x8];
 
 	u8         reserved_at_80[0x10];
@@ -798,6 +799,15 @@ struct mlx5_ifc_fte_match_set_misc5_bits {
 	u8         reserved_at_100[0x100];
 };
 
+struct mlx5_ifc_fte_match_set_misc6_bits {
+	u8         reserved_at_0[0x1a];
+	u8         psp_version[0x4];
+	u8         reserved_at_1e[0x2];
+
+	u8         reserved_at_20[0x1e0];
+};
+
+
 struct mlx5_ifc_cmd_pas_bits {
 	u8         pa_h[0x20];
 
@@ -1042,20 +1052,6 @@ struct mlx5_ifc_wqe_based_flow_table_cap_bits {
 	u8         reserved_at_1c1[0x1f];
 };
 
-struct mlx5_ifc_esw_cap_bits {
-	u8         reserved_at_0[0x1d];
-	u8         merged_eswitch[0x1];
-	u8         reserved_at_1e[0x2];
-
-	u8         reserved_at_20[0x40];
-
-	u8         esw_manager_vport_number_valid[0x1];
-	u8         reserved_at_61[0xf];
-	u8         esw_manager_vport_number[0x10];
-
-	u8         reserved_at_80[0x780];
-};
-
 enum {
 	MLX5_COUNTER_SOURCE_ESWITCH = 0x0,
 	MLX5_COUNTER_FLOW_ESWITCH   = 0x1,
@@ -1096,7 +1092,11 @@ struct mlx5_ifc_e_switch_cap_bits {
 	u8         log_max_esw_sf[0x5];
 	u8         esw_sf_base_id[0x10];
 
-	u8         reserved_at_60[0x7a0];
+	u8         esw_manager_vport_number_valid[0x1];
+	u8         reserved_at_61[0xf];
+	u8         esw_manager_vport_number[0x10];
+
+	u8         reserved_at_80[0x780];
 
 };
 
@@ -2352,7 +2352,7 @@ struct mlx5_ifc_fte_match_param_bits {
 
 	struct mlx5_ifc_fte_match_set_misc5_bits misc_parameters_5;
 
-	u8         reserved_at_e00[0x200];
+	struct mlx5_ifc_fte_match_set_misc6_bits misc_parameters_6;
 };
 
 enum {
@@ -3859,7 +3859,6 @@ union mlx5_ifc_hca_cap_union_bits {
 	struct mlx5_ifc_flow_table_nic_cap_bits flow_table_nic_cap;
 	struct mlx5_ifc_flow_table_eswitch_cap_bits flow_table_eswitch_cap;
 	struct mlx5_ifc_wqe_based_flow_table_cap_bits wqe_based_flow_table_cap;
-	struct mlx5_ifc_esw_cap_bits esw_cap;
 	struct mlx5_ifc_e_switch_cap_bits e_switch_cap;
 	struct mlx5_ifc_port_selection_cap_bits port_selection_cap;
 	struct mlx5_ifc_qos_cap_bits qos_cap;
@@ -6999,6 +6998,7 @@ enum {
 	MLX5_QUERY_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_3 = 0x4,
 	MLX5_QUERY_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_4 = 0x5,
 	MLX5_QUERY_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_5 = 0x6,
+	MLX5_QUERY_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_6 = 0x7,
 };
 
 struct mlx5_ifc_query_flow_group_out_bits {
@@ -7260,6 +7260,7 @@ enum mlx5_reformat_ctx_type {
 	MLX5_REFORMAT_TYPE_REMOVE_HDR = 0x10,
 	MLX5_REFORMAT_TYPE_ADD_MACSEC = 0x11,
 	MLX5_REFORMAT_TYPE_DEL_MACSEC = 0x12,
+	MLX5_REFORMAT_TYPE_REMOVE_PSP_TRANSPORT = 0x16,
 };
 
 struct mlx5_ifc_alloc_packet_reformat_context_in_bits {
@@ -7383,6 +7384,7 @@ enum {
 	MLX5_ACTION_IN_FIELD_OUT_EMD_47_32     = 0x6F,
 	MLX5_ACTION_IN_FIELD_OUT_EMD_31_0      = 0x70,
 	MLX5_ACTION_IN_FIELD_PSP_SYNDROME      = 0x71,
+	MLX5_ACTION_IN_FIELD_PSP_HEADER_1      = 0x78,
 };
 
 struct mlx5_ifc_alloc_modify_header_context_out_bits {
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 485df9c2dbddb..550fb92957d18 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -37,6 +37,7 @@
 #include <linux/bitmap.h>
 #include <linux/bitops.h>
 #include <linux/iommu-debug-pagealloc.h>
+#include <linux/kcsan-checks.h>
 
 struct mempolicy;
 struct anon_vma;
@@ -353,6 +354,7 @@ enum {
 #endif
 	DECLARE_VMA_BIT(UFFD_MINOR, 41),
 	DECLARE_VMA_BIT(SEALED, 42),
+	DECLARE_VMA_BIT(UFFD_RWP, 43),
 	/* Flags that reuse flags above. */
 	DECLARE_VMA_BIT_ALIAS(PKEY_BIT0, HIGH_ARCH_0),
 	DECLARE_VMA_BIT_ALIAS(PKEY_BIT1, HIGH_ARCH_1),
@@ -440,8 +442,10 @@ enum {
 #define VM_STACK	INIT_VM_FLAG(STACK)
 #ifdef CONFIG_STACK_GROWSUP
 #define VM_STACK_EARLY	INIT_VM_FLAG(STACK_EARLY)
+#define VMA_STACK_EARLY mk_vma_flags(VMA_STACK_EARLY_BIT)
 #else
 #define VM_STACK_EARLY	VM_NONE
+#define VMA_STACK_EARLY EMPTY_VMA_FLAGS
 #endif
 #ifdef CONFIG_ARCH_HAS_PKEYS
 #define VM_PKEY_SHIFT ((__force int)VMA_HIGH_ARCH_0_BIT)
@@ -463,15 +467,18 @@ enum {
 #if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_ARM64_GCS) || \
 	defined(CONFIG_RISCV_USER_CFI)
 #define VM_SHADOW_STACK	INIT_VM_FLAG(SHADOW_STACK)
+#define VMA_SHADOW_STACK mk_vma_flags(VMA_SHADOW_STACK_BIT)
 #define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT, VMA_SHADOW_STACK_BIT)
 #else
 #define VM_SHADOW_STACK	VM_NONE
+#define VMA_SHADOW_STACK EMPTY_VMA_FLAGS
 #define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT)
 #endif
 #if defined(CONFIG_PPC64)
 #define VM_SAO		INIT_VM_FLAG(SAO)
 #elif defined(CONFIG_PARISC)
 #define VM_GROWSUP	INIT_VM_FLAG(GROWSUP)
+#define VMA_GROWSUP	mk_vma_flags(VMA_GROWSUP_BIT)
 #elif defined(CONFIG_SPARC64)
 #define VM_SPARC_ADI	INIT_VM_FLAG(SPARC_ADI)
 #define VM_ARCH_CLEAR	INIT_VM_FLAG(ARCH_CLEAR)
@@ -483,6 +490,7 @@ enum {
 #endif
 #ifndef VM_GROWSUP
 #define VM_GROWSUP	VM_NONE
+#define VMA_GROWSUP	EMPTY_VMA_FLAGS
 #endif
 #ifdef CONFIG_ARM64_MTE
 #define VM_MTE		INIT_VM_FLAG(MTE)
@@ -496,12 +504,17 @@ enum {
 #else
 #define VM_UFFD_MINOR	VM_NONE
 #endif
+#ifdef CONFIG_USERFAULTFD_RWP
+#define VM_UFFD_RWP		INIT_VM_FLAG(UFFD_RWP)
+#else
+#define VM_UFFD_RWP		VM_NONE
+#endif
 
 /*
- * vma_flags_t masks for the userfaultfd VMA flags. VMA_UFFD_MINOR is gated on
- * the same config as VM_UFFD_MINOR -- which implies 64BIT, where the bit fits
- * -- so an out-of-range bit is never fed to mk_vma_flags() on a build whose
- * bitmap cannot hold it.
+ * vma_flags_t masks for the userfaultfd VMA flags. The two high-bit modes are
+ * gated on the same configs as their VM_* flags above -- both of which imply
+ * 64BIT -- so an out-of-range bit is never fed to mk_vma_flags() on a build
+ * whose bitmap cannot hold it.
  */
 #define VMA_UFFD_MISSING	mk_vma_flags(VMA_UFFD_MISSING_BIT)
 #define VMA_UFFD_WP		mk_vma_flags(VMA_UFFD_WP_BIT)
@@ -510,6 +523,11 @@ enum {
 #else
 #define VMA_UFFD_MINOR		EMPTY_VMA_FLAGS
 #endif
+#ifdef CONFIG_USERFAULTFD_RWP
+#define VMA_UFFD_RWP		mk_vma_flags(VMA_UFFD_RWP_BIT)
+#else
+#define VMA_UFFD_RWP		EMPTY_VMA_FLAGS
+#endif
 
 #ifdef CONFIG_64BIT
 #define VM_ALLOW_ANY_UNCACHED	INIT_VM_FLAG(ALLOW_ANY_UNCACHED)
@@ -528,6 +546,8 @@ enum {
 
 /* Bits set in the VMA until the stack is in its final location */
 #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY)
+#define VMA_STACK_INCOMPLETE_SETUP append_vma_flags(		\
+	VMA_STACK_EARLY, VMA_RAND_READ_BIT, VMA_SEQ_READ_BIT)
 
 #define TASK_EXEC_BIT ((current->personality & READ_IMPLIES_EXEC) ? \
 		       VMA_EXEC_BIT : VMA_READ_BIT)
@@ -648,29 +668,32 @@ enum {
  * reconsistuted upon page fault, so necessitate page table copying upon fork.
  *
  * Note that these flags should be compared with the DESTINATION VMA not the
- * source, as VM_UFFD_WP may not be propagated to destination, while all other
- * flags will be.
+ * source: VM_UFFD_WP and VM_UFFD_RWP may be cleared on the destination
+ * (dup_userfaultfd() -> userfaultfd_reset_ctx() when the parent context did
+ * not negotiate UFFD_FEATURE_EVENT_FORK), while all other flags propagate.
  *
  * VM_PFNMAP / VM_MIXEDMAP - These contain kernel-mapped data which cannot be
  *                           reasonably reconstructed on page fault.
  *
  *              VM_UFFD_WP - Encodes metadata about an installed uffd
- *                           write protect handler, which cannot be
- *                           reconstructed on page fault.
+ *              VM_UFFD_RWP  write- or read-write-protect handler, which
+ *                           cannot be reconstructed on page fault.
  *
- *                           We always copy pgtables when dst_vma has uffd-wp
- *                           enabled even if it's file-backed
- *                           (e.g. shmem). Because when uffd-wp is enabled,
- *                           pgtable contains uffd-wp protection information,
- *                           that's something we can't retrieve from page cache,
- *                           and skip copying will lose those info.
+ *                           We always copy pgtables when dst_vma has the
+ *                           uffd PTE bit in use even if it's file-backed
+ *                           (e.g. shmem). Because when the uffd bit is
+ *                           in use, the pgtable contains the protection
+ *                           information, that's something we can't
+ *                           retrieve from page cache, and skip copying
+ *                           will lose those info.
  *
  *          VM_MAYBE_GUARD - Could contain page guard region markers which
  *                           by design are a property of the page tables
  *                           only and thus cannot be reconstructed on page
  *                           fault.
  */
-#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_MAYBE_GUARD)
+#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_UFFD_RWP | \
+			 VM_MAYBE_GUARD)
 
 /*
  * mapping from the currently active vm_flags protection bits (the
@@ -1563,11 +1586,24 @@ static inline bool vma_is_initial_stack(const struct vm_area_struct *vma)
 		vma->vm_end >= vma->vm_mm->start_stack;
 }
 
-static inline bool vma_is_temporary_stack(const struct vm_area_struct *vma)
+static inline bool vma_flags_can_grow(const vma_flags_t *flags)
 {
-	int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP);
+	if (vma_flags_test_single_mask(flags, VMA_GROWSUP))
+		return true;
+	if (vma_flags_test(flags, VMA_GROWSDOWN_BIT))
+		return true;
 
-	if (!maybe_stack)
+	return false;
+}
+
+static inline bool vma_can_grow(const struct vm_area_struct *vma)
+{
+	return vma_flags_can_grow(&vma->flags);
+}
+
+static inline bool vma_is_temporary_stack(const struct vm_area_struct *vma)
+{
+	if (!vma_can_grow(vma))
 		return false;
 
 	if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) ==
@@ -2286,22 +2322,30 @@ static inline int page_zone_id(struct page *page)
 }
 
 #ifdef NODE_NOT_IN_PAGE_FLAGS
-int memdesc_nid(memdesc_flags_t mdf);
+int memdesc_nid(const memdesc_flags_t *mdf);
 #else
-static inline int memdesc_nid(memdesc_flags_t mdf)
+#ifdef CONFIG_NUMA
+static inline int memdesc_nid(const memdesc_flags_t *mdf)
+{
+	ASSERT_EXCLUSIVE_BITS(mdf->f, NODES_MASK << NODES_PGSHIFT);
+	return (mdf->f >> NODES_PGSHIFT) & NODES_MASK;
+}
+#else
+static inline int memdesc_nid(const memdesc_flags_t *mdf)
 {
-	return (mdf.f >> NODES_PGSHIFT) & NODES_MASK;
+	return 0;
 }
 #endif
+#endif
 
 static inline int page_to_nid(const struct page *page)
 {
-	return memdesc_nid(PF_POISONED_CHECK(page)->flags);
+	return memdesc_nid(&(PF_POISONED_CHECK(page)->flags));
 }
 
 static inline int folio_nid(const struct folio *folio)
 {
-	return memdesc_nid(folio->flags);
+	return memdesc_nid(&folio->flags);
 }
 
 #ifdef CONFIG_NUMA_BALANCING
@@ -2541,12 +2585,13 @@ static inline void set_page_section(struct page *page, unsigned long section)
 	page->flags.f |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
 }
 
-static inline unsigned long memdesc_section(memdesc_flags_t mdf)
+static inline unsigned long memdesc_section(const memdesc_flags_t *mdf)
 {
-	return (mdf.f >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
+	ASSERT_EXCLUSIVE_BITS(mdf->f, SECTIONS_MASK << SECTIONS_PGSHIFT);
+	return (mdf->f >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
 }
 #else /* !SECTION_IN_PAGE_FLAGS */
-static inline unsigned long memdesc_section(memdesc_flags_t mdf)
+static inline unsigned long memdesc_section(const memdesc_flags_t *mdf)
 {
 	return 0;
 }
@@ -3317,6 +3362,11 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen);
 #define  MM_CP_UFFD_WP_RESOLVE             (1UL << 3) /* Resolve wp */
 #define  MM_CP_UFFD_WP_ALL                 (MM_CP_UFFD_WP | \
 					    MM_CP_UFFD_WP_RESOLVE)
+/* Whether this change is for uffd RWP */
+#define  MM_CP_UFFD_RWP                    (1UL << 4) /* do rwp */
+#define  MM_CP_UFFD_RWP_RESOLVE            (1UL << 5) /* resolve rwp */
+#define  MM_CP_UFFD_RWP_ALL                (MM_CP_UFFD_RWP | \
+					    MM_CP_UFFD_RWP_RESOLVE)
 
 bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
 			     pte_t pte);
@@ -3967,8 +4017,12 @@ extern unsigned long free_reserved_area(void *start, void *end,
 
 extern void adjust_managed_page_count(struct page *page, long count);
 
-/* Free the reserved page into the buddy system, so it gets managed. */
-void free_reserved_page(struct page *page);
+void free_reserved_pages(struct page *page, unsigned int order);
+
+static inline void free_reserved_page(struct page *page)
+{
+	free_reserved_pages(page, 0);
+}
 
 static inline void mark_page_reserved(struct page *page)
 {
@@ -4042,7 +4096,7 @@ extern int __meminit early_pfn_to_nid(unsigned long pfn);
 extern void mem_init(void);
 extern void __init mmap_init(void);
 
-extern void __show_mem(unsigned int flags, nodemask_t *nodemask, int max_zone_idx);
+extern void __show_mem(unsigned int flags, const nodemask_t *nodemask, int max_zone_idx);
 static inline void show_mem(void)
 {
 	__show_mem(0, NULL, MAX_NR_ZONES - 1);
@@ -4052,7 +4106,7 @@ extern void si_meminfo(struct sysinfo * val);
 extern void si_meminfo_node(struct sysinfo *val, int nid);
 
 extern __printf(3, 4)
-void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...);
+void warn_alloc(gfp_t gfp_mask, const nodemask_t *nodemask, const char *fmt, ...);
 
 extern void setup_per_cpu_pageset(void);
 
@@ -4061,44 +4115,46 @@ extern atomic_long_t mmap_pages_allocated;
 extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
 
 /* interval_tree.c */
-void vma_interval_tree_insert(struct vm_area_struct *node,
-			      struct rb_root_cached *root);
-void vma_interval_tree_insert_after(struct vm_area_struct *node,
+void mapping_rmap_tree_insert(struct vm_area_struct *vma,
+			      struct address_space *mapping);
+void mapping_rmap_tree_insert_after(struct vm_area_struct *vma,
 				    struct vm_area_struct *prev,
-				    struct rb_root_cached *root);
-void vma_interval_tree_remove(struct vm_area_struct *node,
-			      struct rb_root_cached *root);
-struct vm_area_struct *vma_interval_tree_subtree_search(struct vm_area_struct *node,
-				unsigned long start, unsigned long last);
-struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root_cached *root,
-				unsigned long start, unsigned long last);
-struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,
-				unsigned long start, unsigned long last);
-
-#define vma_interval_tree_foreach(vma, root, start, last)		\
-	for (vma = vma_interval_tree_iter_first(root, start, last);	\
-	     vma; vma = vma_interval_tree_iter_next(vma, start, last))
-
-void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
-				   struct rb_root_cached *root);
-void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
-				   struct rb_root_cached *root);
+				    struct address_space *mapping);
+void mapping_rmap_tree_remove(struct vm_area_struct *vma,
+			      struct address_space *mapping);
+struct vm_area_struct *
+mapping_rmap_tree_iter_first(struct address_space *mapping,
+			     pgoff_t pgoff_start, pgoff_t pgoff_last);
+struct vm_area_struct *
+mapping_rmap_tree_iter_next(struct vm_area_struct *vma,
+			    pgoff_t pgoff_start, pgoff_t pgoff_last);
+
+#define mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_last) \
+	for (vma = mapping_rmap_tree_iter_first(mapping, pgoff_start,	 \
+						pgoff_last);		 \
+	     vma; vma = mapping_rmap_tree_iter_next(vma, pgoff_start,	 \
+						    pgoff_last))
+
+void anon_rmap_tree_insert(struct anon_vma_chain *avc,
+			   struct anon_vma *anon_vma);
+void anon_rmap_tree_remove(struct anon_vma_chain *avc,
+			   struct anon_vma *anon_vma);
+struct anon_vma_chain *
+anon_rmap_tree_iter_first(struct anon_vma *anon_vma,
+			  pgoff_t pgoff_start, pgoff_t pgoff_last);
 struct anon_vma_chain *
-anon_vma_interval_tree_iter_first(struct rb_root_cached *root,
-				  unsigned long start, unsigned long last);
-struct anon_vma_chain *anon_vma_interval_tree_iter_next(
-	struct anon_vma_chain *node, unsigned long start, unsigned long last);
+anon_rmap_tree_iter_next(struct anon_vma_chain *avc,
+			 pgoff_t pgoff_start, pgoff_t pgoff_last);
 #ifdef CONFIG_DEBUG_VM_RB
-void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
+void anon_rmap_tree_verify(struct anon_vma_chain *avc);
 #endif
 
-#define anon_vma_interval_tree_foreach(avc, root, start, last)		 \
-	for (avc = anon_vma_interval_tree_iter_first(root, start, last); \
-	     avc; avc = anon_vma_interval_tree_iter_next(avc, start, last))
+#define anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_last)		 \
+	for (avc = anon_rmap_tree_iter_first(anon_vma, pgoff_start, pgoff_last); \
+	     avc; avc = anon_rmap_tree_iter_next(avc, pgoff_start, pgoff_last))
 
 /* mmap.c */
 extern int __vm_enough_memory(const struct mm_struct *mm, long pages, int cap_sys_admin);
-extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
 extern void exit_mmap(struct mm_struct *);
 bool mmap_read_lock_maybe_expand(struct mm_struct *mm, struct vm_area_struct *vma,
 				 unsigned long addr, bool write);
@@ -4139,18 +4195,20 @@ unsigned long randomize_page(unsigned long start, unsigned long range);
 
 unsigned long
 __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
-		    unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags);
+		    unsigned long pgoff, unsigned long flags,
+		    vma_flags_t vma_flags);
 
 static inline unsigned long
 get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
 		  unsigned long pgoff, unsigned long flags)
 {
-	return __get_unmapped_area(file, addr, len, pgoff, flags, 0);
+	return __get_unmapped_area(file, addr, len, pgoff, flags,
+				   EMPTY_VMA_FLAGS);
 }
 
-extern unsigned long do_mmap(struct file *file, unsigned long addr,
+unsigned long do_mmap(struct file *file, unsigned long addr,
 	unsigned long len, unsigned long prot, unsigned long flags,
-	vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate,
+	vma_flags_t vma_flags, unsigned long pgoff, unsigned long *populate,
 	struct list_head *uf);
 extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
 			 unsigned long start, size_t len, struct list_head *uf,
@@ -4278,9 +4336,61 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma)
 	return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
 }
 
-static inline unsigned long vma_last_pgoff(struct vm_area_struct *vma)
+/**
+ * vma_start_pgoff() - Get the page offset of the start of @vma
+ * @vma: The VMA whose page offset is required.
+ *
+ * If the VMA is file-backed, this is the page offset into the file.
+ *
+ * If @vma is anonymous, this is the virtual page offset of the start of the
+ * VMA - if unfaulted, then vma->vm_start >> PAGE_SHIFT, if faulted then the
+ * virtual page offset at the time of first fault.
+ *
+ * If @vma is a MAP_PRIVATE file-backed mapping, then this returns the
+ * page offset within the file.
+ *
+ * Edge cases: nommu does not abide by these, MAP_PRIVATE-/dev/zero satisfies
+ * vma_is_anonymous() but has file-backed page offset, and MAP_PRIVATE-pfnmap
+ * regions have their page offset set to the first PFN in the range.
+ *
+ * Returns: The page offset of the start of @vma.
+ */
+static inline pgoff_t vma_start_pgoff(const struct vm_area_struct *vma)
 {
-	return vma->vm_pgoff + vma_pages(vma) - 1;
+	return vma->vm_pgoff;
+}
+
+/**
+ * vma_end_pgoff() - Get the page offset of the exclusive end of @vma
+ * @vma: The VMA whose end page offset is required.
+ *
+ * This returns the exclusive end page offset of @vma, which is useful for
+ * expressing page offset ranges.
+ *
+ * See the description of vma_start_pgoff() for a description of VMA page
+ * offsets.
+ *
+ * Returns: The exclusive end page offset of @vma.
+ */
+static inline pgoff_t vma_end_pgoff(const struct vm_area_struct *vma)
+{
+	return vma_start_pgoff(vma) + vma_pages(vma);
+}
+
+/**
+ * vma_last_pgoff() - Get the page offset of the last page in @vma
+ * @vma: The VMA whose last page offset is required.
+ *
+ * This returns the last page offset contained within @vma.
+ *
+ * See the description of vma_start_pgoff() for a description of VMA page
+ * offsets.
+ *
+ * Returns: The last page offset of @vma.
+ */
+static inline pgoff_t vma_last_pgoff(const struct vm_area_struct *vma)
+{
+	return vma_end_pgoff(vma) - 1;
 }
 
 static inline unsigned long vma_desc_size(const struct vm_area_desc *desc)
@@ -4497,26 +4607,35 @@ static inline bool range_in_vma_desc(const struct vm_area_desc *desc,
 #ifdef CONFIG_MMU
 pgprot_t vm_get_page_prot(vm_flags_t vm_flags);
 
-static inline pgprot_t vma_get_page_prot(vma_flags_t vma_flags)
+static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags)
 {
 	const vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags);
 
 	return vm_get_page_prot(vm_flags);
 }
 
+static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma)
+{
+	return vma_flags_to_page_prot(vma->flags);
+}
+
 void vma_set_page_prot(struct vm_area_struct *vma);
 #else
 static inline pgprot_t vm_get_page_prot(vm_flags_t vm_flags)
 {
 	return __pgprot(0);
 }
-static inline pgprot_t vma_get_page_prot(vma_flags_t vma_flags)
+static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags)
+{
+	return __pgprot(0);
+}
+static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma)
 {
 	return __pgprot(0);
 }
 static inline void vma_set_page_prot(struct vm_area_struct *vma)
 {
-	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+	vma->vm_page_prot = vma_get_page_prot(vma);
 }
 #endif
 
@@ -4625,11 +4744,25 @@ static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags)
 
 /*
  * Indicates whether GUP can follow a PROT_NONE mapped page, or whether
- * a (NUMA hinting) fault is required.
+ * a (NUMA hinting or userfaultfd RWP) fault is required.
  */
 static inline bool gup_can_follow_protnone(const struct vm_area_struct *vma,
 					   unsigned int flags)
 {
+	/*
+	 * VM_UFFD_RWP uses protnone as an access-tracking marker, not for
+	 * NUMA hinting. GUP must always take a fault so the access is
+	 * delivered to userfaultfd, regardless of FOLL_HONOR_NUMA_FAULT.
+	 *
+	 * Only do so while the VMA is accessible. If it has been made
+	 * inaccessible (e.g. mprotect(PROT_NONE)), fall through to the guard
+	 * below: forcing a fault there would loop, as handle_mm_fault() makes
+	 * no progress on protnone in an inaccessible VMA, and the access is
+	 * denied regardless of RWP anyway.
+	 */
+	if (vma_test_single_mask(vma, VMA_UFFD_RWP) && vma_is_accessible(vma))
+		return false;
+
 	/*
 	 * If callers don't want to honor NUMA hinting faults, no need to
 	 * determine if we would actually have to trigger a NUMA hinting fault.
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index a8430a7ae0544..b5c4dc0f3fe32 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -600,14 +600,14 @@ pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr,
 		return false;
 
 	/* A uffd-wp wr-protected normal pte */
-	if (unlikely(pte_present(pteval) && pte_uffd_wp(pteval)))
+	if (unlikely(pte_present(pteval) && pte_uffd(pteval)))
 		arm_uffd_pte = true;
 
 	/*
 	 * A uffd-wp wr-protected swap pte.  Note: this should even cover an
 	 * existing pte marker with uffd-wp bit set.
 	 */
-	if (unlikely(pte_swp_uffd_wp_any(pteval)))
+	if (unlikely(pte_swp_uffd_any(pteval)))
 		arm_uffd_pte = true;
 
 	if (unlikely(arm_uffd_pte)) {
@@ -650,7 +650,7 @@ static inline bool vma_has_recency(const struct vm_area_struct *vma)
 static inline size_t num_pages_contiguous(struct page **pages, size_t nr_pages)
 {
 	struct page *cur_page = pages[0];
-	unsigned long section = memdesc_section(cur_page->flags);
+	unsigned long section = memdesc_section(&cur_page->flags);
 	size_t i;
 
 	for (i = 1; i < nr_pages; i++) {
@@ -660,7 +660,7 @@ static inline size_t num_pages_contiguous(struct page **pages, size_t nr_pages)
 		 * In unproblematic kernel configs, page_to_section() == 0 and
 		 * the whole check will get optimized out.
 		 */
-		if (memdesc_section(cur_page->flags) != section)
+		if (memdesc_section(&cur_page->flags) != section)
 			break;
 	}
 
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index b18c2b2e7d2c6..939b5ea8c9e0b 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -368,6 +368,8 @@ typedef unsigned short mm_id_t;
  *    dax_associate_entry.
  * @private: Filesystem per-folio data (see folio_attach_private()).
  * @swap: Used for swp_entry_t if folio_test_swapcache().
+ * @migrate_info: Stores migration state (anon_vma pointer and
+ *    FOLIO_WAS_* markers).
  * @_mapcount: Do not access this member directly.  Use folio_mapcount() to
  *    find out how many times this folio is mapped by userspace.
  * @_refcount: Do not access this member directly.  Use folio_ref_count()
@@ -427,6 +429,7 @@ struct folio {
 			union {
 				void *private;
 				swp_entry_t swap;
+				unsigned long migrate_info;
 			};
 			atomic_t _mapcount;
 			atomic_t _refcount;
diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h
index 04b8f61ece5d8..87f77e3da77f0 100644
--- a/include/linux/mmap_lock.h
+++ b/include/linux/mmap_lock.h
@@ -506,6 +506,8 @@ static inline __must_check
 int vma_start_write_killable(struct vm_area_struct *vma) { return 0; }
 static inline void vma_assert_write_locked(struct vm_area_struct *vma)
 		{ mmap_assert_write_locked(vma->vm_mm); }
+static inline bool vma_is_attached(struct vm_area_struct *vma)
+		{ return true; }
 static inline void vma_assert_attached(struct vm_area_struct *vma) {}
 static inline void vma_assert_detached(struct vm_area_struct *vma) {}
 static inline void vma_mark_attached(struct vm_area_struct *vma) {}
@@ -530,6 +532,12 @@ static inline void vma_assert_stabilised(struct vm_area_struct *vma)
 
 #endif /* CONFIG_PER_VMA_LOCK */
 
+static inline void vma_assert_can_modify(struct vm_area_struct *vma)
+{
+	if (vma_is_attached(vma))
+		vma_assert_write_locked(vma);
+}
+
 static inline void mmap_write_lock(struct mm_struct *mm)
 {
 	__mmap_lock_trace_start_locking(mm, true);
@@ -621,6 +629,9 @@ static inline void mmap_read_unlock(struct mm_struct *mm)
 
 DEFINE_GUARD(mmap_read_lock, struct mm_struct *,
 	     mmap_read_lock(_T), mmap_read_unlock(_T))
+DEFINE_GUARD(mmap_write_lock, struct mm_struct *,
+	     mmap_write_lock(_T), mmap_write_unlock(_T))
+DEFINE_GUARD_COND(mmap_read_lock, _try, mmap_read_trylock(_T))
 
 static inline void mmap_read_unlock_non_owner(struct mm_struct *mm)
 {
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index ca27121871475..0507193b3ae34 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -2,7 +2,7 @@
 #ifndef _LINUX_MMZONE_H
 #define _LINUX_MMZONE_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #ifndef __GENERATING_BOUNDS_H
 
 #include <linux/spinlock.h>
@@ -1272,31 +1272,33 @@ static inline bool zone_is_empty(const struct zone *zone)
 #define KASAN_TAG_MASK		((1UL << KASAN_TAG_WIDTH) - 1)
 #define ZONEID_MASK		((1UL << ZONEID_SHIFT) - 1)
 
-static inline enum zone_type memdesc_zonenum(memdesc_flags_t flags)
+static inline enum zone_type memdesc_zonenum(const memdesc_flags_t *flags)
 {
-	ASSERT_EXCLUSIVE_BITS(flags.f, ZONES_MASK << ZONES_PGSHIFT);
-	return (flags.f >> ZONES_PGSHIFT) & ZONES_MASK;
+#if ZONES_WIDTH != 0
+	ASSERT_EXCLUSIVE_BITS(flags->f, ZONES_MASK << ZONES_PGSHIFT);
+#endif
+	return (flags->f >> ZONES_PGSHIFT) & ZONES_MASK;
 }
 
 static inline enum zone_type page_zonenum(const struct page *page)
 {
-	return memdesc_zonenum(page->flags);
+	return memdesc_zonenum(&page->flags);
 }
 
 static inline enum zone_type folio_zonenum(const struct folio *folio)
 {
-	return memdesc_zonenum(folio->flags);
+	return memdesc_zonenum(&folio->flags);
 }
 
 #ifdef CONFIG_ZONE_DEVICE
-static inline bool memdesc_is_zone_device(memdesc_flags_t mdf)
+static inline bool memdesc_is_zone_device(const memdesc_flags_t *mdf)
 {
 	return memdesc_zonenum(mdf) == ZONE_DEVICE;
 }
 
 static inline struct dev_pagemap *page_pgmap(const struct page *page)
 {
-	VM_WARN_ON_ONCE_PAGE(!memdesc_is_zone_device(page->flags), page);
+	VM_WARN_ON_ONCE_PAGE(!memdesc_is_zone_device(&page->flags), page);
 	return page_folio(page)->pgmap;
 }
 
@@ -1311,9 +1313,9 @@ static inline struct dev_pagemap *page_pgmap(const struct page *page)
 static inline bool zone_device_pages_have_same_pgmap(const struct page *a,
 						     const struct page *b)
 {
-	if (memdesc_is_zone_device(a->flags) != memdesc_is_zone_device(b->flags))
+	if (memdesc_is_zone_device(&a->flags) != memdesc_is_zone_device(&b->flags))
 		return false;
-	if (!memdesc_is_zone_device(a->flags))
+	if (!memdesc_is_zone_device(&a->flags))
 		return true;
 	return page_pgmap(a) == page_pgmap(b);
 }
@@ -1321,7 +1323,7 @@ static inline bool zone_device_pages_have_same_pgmap(const struct page *a,
 extern void memmap_init_zone_device(struct zone *, unsigned long,
 				    unsigned long, struct dev_pagemap *);
 #else
-static inline bool memdesc_is_zone_device(memdesc_flags_t mdf)
+static inline bool memdesc_is_zone_device(const memdesc_flags_t *mdf)
 {
 	return false;
 }
@@ -1338,12 +1340,12 @@ static inline struct dev_pagemap *page_pgmap(const struct page *page)
 
 static inline bool is_zone_device_page(const struct page *page)
 {
-	return memdesc_is_zone_device(page->flags);
+	return memdesc_is_zone_device(&page->flags);
 }
 
 static inline bool folio_is_zone_device(const struct folio *folio)
 {
-	return memdesc_is_zone_device(folio->flags);
+	return memdesc_is_zone_device(&folio->flags);
 }
 
 static inline bool is_zone_movable_page(const struct page *page)
@@ -1815,7 +1817,7 @@ static inline int zonelist_node_idx(const struct zoneref *zoneref)
 
 struct zoneref *__next_zones_zonelist(struct zoneref *z,
 					enum zone_type highest_zoneidx,
-					nodemask_t *nodes);
+					const nodemask_t *nodes);
 
 /**
  * next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodemask using a cursor within a zonelist as a starting point
@@ -1834,7 +1836,7 @@ struct zoneref *__next_zones_zonelist(struct zoneref *z,
  */
 static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z,
 					enum zone_type highest_zoneidx,
-					nodemask_t *nodes)
+					const nodemask_t *nodes)
 {
 	if (likely(!nodes && zonelist_zone_idx(z) <= highest_zoneidx))
 		return z;
@@ -1860,7 +1862,7 @@ static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z,
  */
 static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist,
 					enum zone_type highest_zoneidx,
-					nodemask_t *nodes)
+					const nodemask_t *nodes)
 {
 	return next_zones_zonelist(zonelist->_zonerefs,
 							highest_zoneidx, nodes);
@@ -2156,8 +2158,6 @@ static inline int preinited_vmemmap_section(const struct mem_section *section)
 }
 
 void sparse_vmemmap_init_nid_early(int nid);
-void sparse_vmemmap_init_nid_late(int nid);
-
 #else
 static inline int preinited_vmemmap_section(const struct mem_section *section)
 {
@@ -2166,10 +2166,6 @@ static inline int preinited_vmemmap_section(const struct mem_section *section)
 static inline void sparse_vmemmap_init_nid_early(int nid)
 {
 }
-
-static inline void sparse_vmemmap_init_nid_late(int nid)
-{
-}
 #endif
 
 static inline int online_section_nr(unsigned long nr)
@@ -2374,7 +2370,6 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr)
 
 #else
 #define sparse_vmemmap_init_nid_early(_nid) do {} while (0)
-#define sparse_vmemmap_init_nid_late(_nid) do {} while (0)
 #define pfn_in_present_section pfn_valid
 #endif /* CONFIG_SPARSEMEM */
 
@@ -2389,5 +2384,5 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr)
 #endif
 
 #endif /* !__GENERATING_BOUNDS.H */
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 #endif /* _LINUX_MMZONE_H */
diff --git a/include/linux/module.h b/include/linux/module.h
index 7566815fabbe8..96cc98568eea5 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -477,10 +477,6 @@ struct module {
 	struct module_notes_attrs *notes_attrs;
 #endif
 
-	/* The command line arguments (may be mangled).  People like
-	   keeping pointers to this stuff */
-	char *args;
-
 #ifdef CONFIG_SMP
 	/* Per-cpu data. */
 	void __percpu *percpu;
diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h
index ec6efcfeef834..5f4c00ae72a72 100644
--- a/include/linux/mtd/spinand.h
+++ b/include/linux/mtd/spinand.h
@@ -437,6 +437,7 @@ extern const struct spinand_manufacturer esmt_c8_spinand_manufacturer;
 extern const struct spinand_manufacturer fmsh_spinand_manufacturer;
 extern const struct spinand_manufacturer foresee_spinand_manufacturer;
 extern const struct spinand_manufacturer gigadevice_spinand_manufacturer;
+extern const struct spinand_manufacturer heyangtek_spinand_manufacturer;
 extern const struct spinand_manufacturer macronix_spinand_manufacturer;
 extern const struct spinand_manufacturer micron_spinand_manufacturer;
 extern const struct spinand_manufacturer paragon_spinand_manufacturer;
diff --git a/include/linux/net.h b/include/linux/net.h
index f268f395ce473..3d82966e2243b 100644
--- a/include/linux/net.h
+++ b/include/linux/net.h
@@ -47,6 +47,29 @@ typedef struct sockopt {
 	int optlen;
 } sockopt_t;
 
+/*
+ * Initialize a user-backed sockopt_t from the (optval, optlen) __user pair of
+ * a getsockopt() callback. Used by transitional __user getsockopt wrappers
+ * while the proto-layer callbacks are converted to take a sockopt_t; the
+ * caller writes opt->optlen back to the user optlen after the callback.
+ */
+static inline int sockopt_init_user(sockopt_t *opt, char __user *optval,
+				    int __user *optlen)
+{
+	int len;
+
+	if (get_user(len, optlen))
+		return -EFAULT;
+	if (len < 0)
+		return -EINVAL;
+
+	iov_iter_ubuf(&opt->iter_out, ITER_DEST, optval, len);
+	iov_iter_ubuf(&opt->iter_in, ITER_SOURCE, optval, len);
+	opt->optlen = len;
+
+	return 0;
+}
+
 struct poll_table_struct;
 struct pipe_inode_info;
 struct inode;
@@ -285,6 +308,7 @@ int sock_recvmsg(struct socket *sock, struct msghdr *msg, int flags);
 struct file *sock_alloc_file(struct socket *sock, int flags, const char *dname);
 struct socket *sockfd_lookup(int fd, int *err);
 struct socket *sock_from_file(struct file *file);
+int sock_read_xattr(struct socket *sock, const char *name, void *value, size_t size);
 #define		     sockfd_put(sock) fput(sock->file)
 int net_ratelimit(void);
 
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 9981d637f8b54..8db25b79573e3 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -1845,6 +1845,8 @@ enum netdev_reg_state {
  *	@napi_list:	List entry used for polling NAPI devices
  *	@unreg_list:	List entry  when we are unregistering the
  *			device; see the function unregister_netdev
+ *	@unreg_list_net:List entry when we are unregistering the cross-netns
+ *			device; see the function unregister_netdevice_queue_net()
  *	@close_list:	List entry used when we are closing the device
  *	@ptype_all:     Device-specific packet handlers for all protocols
  *	@ptype_specific: Device-specific, protocol-specific packet handlers
@@ -2241,6 +2243,9 @@ struct net_device {
 	struct list_head	dev_list;
 	struct list_head	napi_list;
 	struct list_head	unreg_list;
+#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
+	struct list_head	unreg_list_net;
+#endif
 	struct list_head	close_list;
 	struct list_head	ptype_all;
 
@@ -3472,6 +3477,25 @@ static inline void unregister_netdevice(struct net_device *dev)
 	unregister_netdevice_queue(dev, NULL);
 }
 
+#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
+void unregister_netdevice_queue_net(struct net *net, struct net_device *dev,
+				    struct list_head *head);
+void unregister_netdevice_many_net(struct net *net);
+void unregister_netdevice_queue_many_net(struct net *net, struct list_head *head);
+#else
+static inline void unregister_netdevice_queue_net(struct net *net,
+						  struct net_device *dev,
+						  struct list_head *head)
+{
+	unregister_netdevice_queue(dev, head);
+}
+
+static inline void unregister_netdevice_queue_many_net(struct net *net,
+						       struct list_head *head)
+{
+}
+#endif
+
 int netdev_refcnt_read(const struct net_device *dev);
 void free_netdev(struct net_device *dev);
 
diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h
index efbbfa770d661..e99afc1414cd3 100644
--- a/include/linux/netfilter.h
+++ b/include/linux/netfilter.h
@@ -93,6 +93,7 @@ enum nf_hook_ops_type {
 	NF_HOOK_OP_NF_TABLES,
 	NF_HOOK_OP_BPF,
 	NF_HOOK_OP_NFT_FT,
+	NF_HOOK_OP_NAT,
 };
 
 struct nf_hook_ops {
@@ -140,6 +141,12 @@ struct nf_hook_entries {
 	 */
 };
 
+struct nf_nat_lookup_hook_priv {
+	struct nf_hook_entries __rcu *entries;
+
+	struct rcu_head rcu_head;
+};
+
 #ifdef CONFIG_NETFILTER
 static inline struct nf_hook_ops **nf_hook_entries_get_hook_ops(const struct nf_hook_entries *e)
 {
diff --git a/include/linux/netfilter/nf_conntrack_h323.h b/include/linux/netfilter/nf_conntrack_h323.h
index 81286c4993252..b15f37604cde4 100644
--- a/include/linux/netfilter/nf_conntrack_h323.h
+++ b/include/linux/netfilter/nf_conntrack_h323.h
@@ -9,8 +9,6 @@
 #include <net/netfilter/nf_conntrack_expect.h>
 #include <uapi/linux/netfilter/nf_conntrack_tuple_common.h>
 
-#define RAS_PORT 1719
-#define Q931_PORT 1720
 #define H323_RTP_CHANNEL_MAX 4	/* Audio, video, FAX and other */
 
 /* This structure exists only once per master */
diff --git a/include/linux/netfilter/nf_conntrack_pptp.h b/include/linux/netfilter/nf_conntrack_pptp.h
index c3bdb43709384..c0b305ce7c3c6 100644
--- a/include/linux/netfilter/nf_conntrack_pptp.h
+++ b/include/linux/netfilter/nf_conntrack_pptp.h
@@ -50,8 +50,6 @@ struct nf_nat_pptp {
 	__be16 pac_call_id;			/* NAT'ed PAC call id */
 };
 
-#define PPTP_CONTROL_PORT	1723
-
 #define PPTP_PACKET_CONTROL	1
 #define PPTP_PACKET_MGMT	2
 
diff --git a/include/linux/netfilter/nf_conntrack_sane.h b/include/linux/netfilter/nf_conntrack_sane.h
index 46c7acd1b4a70..8501035d73355 100644
--- a/include/linux/netfilter/nf_conntrack_sane.h
+++ b/include/linux/netfilter/nf_conntrack_sane.h
@@ -3,8 +3,6 @@
 #define _NF_CONNTRACK_SANE_H
 /* SANE tracking. */
 
-#define SANE_PORT	6566
-
 enum sane_state {
 	SANE_STATE_NORMAL,
 	SANE_STATE_START_REQUESTED,
diff --git a/include/linux/netfilter/nf_conntrack_tftp.h b/include/linux/netfilter/nf_conntrack_tftp.h
index 90b334bbce3ce..e3d1739c557d1 100644
--- a/include/linux/netfilter/nf_conntrack_tftp.h
+++ b/include/linux/netfilter/nf_conntrack_tftp.h
@@ -2,8 +2,6 @@
 #ifndef _NF_CONNTRACK_TFTP_H
 #define _NF_CONNTRACK_TFTP_H
 
-#define TFTP_PORT 69
-
 #include <linux/netfilter.h>
 #include <linux/skbuff.h>
 #include <linux/types.h>
diff --git a/include/linux/netpoll.h b/include/linux/netpoll.h
index 88f7daa8560e5..79315461a7b1e 100644
--- a/include/linux/netpoll.h
+++ b/include/linux/netpoll.h
@@ -21,20 +21,6 @@ union inet_addr {
 	struct in6_addr	in6;
 };
 
-/*
- * Maximum payload netpoll's preallocated skb pool can carry. Keep this in
- * sync with the buffer size used by refill_skbs() in net/core/netpoll.c;
- * callers (e.g. netconsole) use it to detect requests the pool can never
- * satisfy and avoid dequeuing a pooled skb that would later trip
- * skb_over_panic() in skb_put().
- */
-#define MAX_UDP_CHUNK	1460
-#define MAX_SKB_SIZE						\
-	(sizeof(struct ethhdr) +				\
-	 sizeof(struct iphdr) +					\
-	 sizeof(struct udphdr) +				\
-	 MAX_UDP_CHUNK)
-
 struct netpoll {
 	struct net_device *dev;
 	netdevice_tracker dev_tracker;
@@ -49,10 +35,6 @@ struct netpoll {
 
 	union inet_addr local_ip, remote_ip;
 	bool ipv6;
-	u16 local_port, remote_port;
-	u8 remote_mac[ETH_ALEN];
-	struct sk_buff_head skb_pool;
-	struct work_struct refill_wq;
 };
 
 #define np_info(np, fmt, ...)				\
diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h
index d87be1f25273a..44e5e9fa12e1e 100644
--- a/include/linux/nfs4.h
+++ b/include/linux/nfs4.h
@@ -171,133 +171,6 @@ Needs to be updated if more operations are defined in future.*/
 #define LAST_NFS42_OP	OP_REMOVEXATTR
 #define LAST_NFS4_OP	LAST_NFS42_OP
 
-enum nfsstat4 {
-	NFS4_OK = 0,
-	NFS4ERR_PERM = 1,
-	NFS4ERR_NOENT = 2,
-	NFS4ERR_IO = 5,
-	NFS4ERR_NXIO = 6,
-	NFS4ERR_ACCESS = 13,
-	NFS4ERR_EXIST = 17,
-	NFS4ERR_XDEV = 18,
-	/* Unused/reserved 19 */
-	NFS4ERR_NOTDIR = 20,
-	NFS4ERR_ISDIR = 21,
-	NFS4ERR_INVAL = 22,
-	NFS4ERR_FBIG = 27,
-	NFS4ERR_NOSPC = 28,
-	NFS4ERR_ROFS = 30,
-	NFS4ERR_MLINK = 31,
-	NFS4ERR_NAMETOOLONG = 63,
-	NFS4ERR_NOTEMPTY = 66,
-	NFS4ERR_DQUOT = 69,
-	NFS4ERR_STALE = 70,
-	NFS4ERR_BADHANDLE = 10001,
-	NFS4ERR_BAD_COOKIE = 10003,
-	NFS4ERR_NOTSUPP = 10004,
-	NFS4ERR_TOOSMALL = 10005,
-	NFS4ERR_SERVERFAULT = 10006,
-	NFS4ERR_BADTYPE = 10007,
-	NFS4ERR_DELAY = 10008,
-	NFS4ERR_SAME = 10009,
-	NFS4ERR_DENIED = 10010,
-	NFS4ERR_EXPIRED = 10011,
-	NFS4ERR_LOCKED = 10012,
-	NFS4ERR_GRACE = 10013,
-	NFS4ERR_FHEXPIRED = 10014,
-	NFS4ERR_SHARE_DENIED = 10015,
-	NFS4ERR_WRONGSEC = 10016,
-	NFS4ERR_CLID_INUSE = 10017,
-	NFS4ERR_RESOURCE = 10018,
-	NFS4ERR_MOVED = 10019,
-	NFS4ERR_NOFILEHANDLE = 10020,
-	NFS4ERR_MINOR_VERS_MISMATCH = 10021,
-	NFS4ERR_STALE_CLIENTID = 10022,
-	NFS4ERR_STALE_STATEID = 10023,
-	NFS4ERR_OLD_STATEID = 10024,
-	NFS4ERR_BAD_STATEID = 10025,
-	NFS4ERR_BAD_SEQID = 10026,
-	NFS4ERR_NOT_SAME = 10027,
-	NFS4ERR_LOCK_RANGE = 10028,
-	NFS4ERR_SYMLINK = 10029,
-	NFS4ERR_RESTOREFH = 10030,
-	NFS4ERR_LEASE_MOVED = 10031,
-	NFS4ERR_ATTRNOTSUPP = 10032,
-	NFS4ERR_NO_GRACE = 10033,
-	NFS4ERR_RECLAIM_BAD = 10034,
-	NFS4ERR_RECLAIM_CONFLICT = 10035,
-	NFS4ERR_BADXDR = 10036,
-	NFS4ERR_LOCKS_HELD = 10037,
-	NFS4ERR_OPENMODE = 10038,
-	NFS4ERR_BADOWNER = 10039,
-	NFS4ERR_BADCHAR = 10040,
-	NFS4ERR_BADNAME = 10041,
-	NFS4ERR_BAD_RANGE = 10042,
-	NFS4ERR_LOCK_NOTSUPP = 10043,
-	NFS4ERR_OP_ILLEGAL = 10044,
-	NFS4ERR_DEADLOCK = 10045,
-	NFS4ERR_FILE_OPEN = 10046,
-	NFS4ERR_ADMIN_REVOKED = 10047,
-	NFS4ERR_CB_PATH_DOWN = 10048,
-
-	/* nfs41 */
-	NFS4ERR_BADIOMODE	= 10049,
-	NFS4ERR_BADLAYOUT	= 10050,
-	NFS4ERR_BAD_SESSION_DIGEST = 10051,
-	NFS4ERR_BADSESSION	= 10052,
-	NFS4ERR_BADSLOT		= 10053,
-	NFS4ERR_COMPLETE_ALREADY = 10054,
-	NFS4ERR_CONN_NOT_BOUND_TO_SESSION = 10055,
-	NFS4ERR_DELEG_ALREADY_WANTED = 10056,
-	NFS4ERR_BACK_CHAN_BUSY	= 10057,	/* backchan reqs outstanding */
-	NFS4ERR_LAYOUTTRYLATER	= 10058,
-	NFS4ERR_LAYOUTUNAVAILABLE = 10059,
-	NFS4ERR_NOMATCHING_LAYOUT = 10060,
-	NFS4ERR_RECALLCONFLICT	= 10061,
-	NFS4ERR_UNKNOWN_LAYOUTTYPE = 10062,
-	NFS4ERR_SEQ_MISORDERED = 10063, 	/* unexpected seq.id in req */
-	NFS4ERR_SEQUENCE_POS	= 10064,	/* [CB_]SEQ. op not 1st op */
-	NFS4ERR_REQ_TOO_BIG	= 10065,	/* request too big */
-	NFS4ERR_REP_TOO_BIG	= 10066,	/* reply too big */
-	NFS4ERR_REP_TOO_BIG_TO_CACHE = 10067,	/* rep. not all cached */
-	NFS4ERR_RETRY_UNCACHED_REP = 10068,	/* retry & rep. uncached */
-	NFS4ERR_UNSAFE_COMPOUND = 10069,	/* retry/recovery too hard */
-	NFS4ERR_TOO_MANY_OPS	= 10070,	/* too many ops in [CB_]COMP */
-	NFS4ERR_OP_NOT_IN_SESSION = 10071,	/* op needs [CB_]SEQ. op */
-	NFS4ERR_HASH_ALG_UNSUPP = 10072,	/* hash alg. not supp. */
-						/* Error 10073 is unused. */
-	NFS4ERR_CLIENTID_BUSY	= 10074,	/* clientid has state */
-	NFS4ERR_PNFS_IO_HOLE	= 10075,	/* IO to _SPARSE file hole */
-	NFS4ERR_SEQ_FALSE_RETRY	= 10076,	/* retry not original */
-	NFS4ERR_BAD_HIGH_SLOT	= 10077,	/* sequence arg bad */
-	NFS4ERR_DEADSESSION	= 10078,	/* persistent session dead */
-	NFS4ERR_ENCR_ALG_UNSUPP = 10079,	/* SSV alg mismatch */
-	NFS4ERR_PNFS_NO_LAYOUT	= 10080,	/* direct I/O with no layout */
-	NFS4ERR_NOT_ONLY_OP	= 10081,	/* bad compound */
-	NFS4ERR_WRONG_CRED	= 10082,	/* permissions:state change */
-	NFS4ERR_WRONG_TYPE	= 10083,	/* current operation mismatch */
-	NFS4ERR_DIRDELEG_UNAVAIL = 10084,	/* no directory delegation */
-	NFS4ERR_REJECT_DELEG	= 10085,	/* on callback */
-	NFS4ERR_RETURNCONFLICT	= 10086,	/* outstanding layoutreturn */
-	NFS4ERR_DELEG_REVOKED	= 10087,	/* deleg./layout revoked */
-
-	/* nfs42 */
-	NFS4ERR_PARTNER_NOTSUPP	= 10088,
-	NFS4ERR_PARTNER_NO_AUTH	= 10089,
-	NFS4ERR_UNION_NOTSUPP	= 10090,
-	NFS4ERR_OFFLOAD_DENIED	= 10091,
-	NFS4ERR_WRONG_LFS	= 10092,
-	NFS4ERR_BADLABEL	= 10093,
-	NFS4ERR_OFFLOAD_NO_REQS	= 10094,
-
-	/* xattr (RFC8276) */
-	NFS4ERR_NOXATTR		= 10095,
-	NFS4ERR_XATTR2BIG	= 10096,
-
-	/* can be used for internal errors */
-	NFS4ERR_FIRST_FREE
-};
-
 /* error codes for internal client use */
 #define NFS4ERR_RESET_TO_MDS   12001
 #define NFS4ERR_RESET_TO_PNFS  12002
diff --git a/include/linux/nmi.h b/include/linux/nmi.h
index bc1162895f355..89c5465e25242 100644
--- a/include/linux/nmi.h
+++ b/include/linux/nmi.h
@@ -77,9 +77,9 @@ static inline void reset_hung_task_detector(void) { }
  * detectors are 'suspended' while 'watchdog_thresh' is equal zero.
  */
 #define WATCHDOG_HARDLOCKUP_ENABLED_BIT  0
-#define WATCHDOG_SOFTOCKUP_ENABLED_BIT   1
+#define WATCHDOG_SOFTLOCKUP_ENABLED_BIT  1
 #define WATCHDOG_HARDLOCKUP_ENABLED     (1 << WATCHDOG_HARDLOCKUP_ENABLED_BIT)
-#define WATCHDOG_SOFTOCKUP_ENABLED      (1 << WATCHDOG_SOFTOCKUP_ENABLED_BIT)
+#define WATCHDOG_SOFTLOCKUP_ENABLED     (1 << WATCHDOG_SOFTLOCKUP_ENABLED_BIT)
 
 #if defined(CONFIG_HARDLOCKUP_DETECTOR)
 extern void hardlockup_detector_disable(void);
diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h
index b842aa5255464..607373bd83ee6 100644
--- a/include/linux/nodemask.h
+++ b/include/linux/nodemask.h
@@ -95,6 +95,14 @@
 
 extern nodemask_t _unused_nodemask_arg_;
 
+#if MAX_NUMNODES > 1
+extern unsigned int nr_node_ids;
+extern unsigned int nr_online_nodes;
+#else
+#define nr_node_ids		1U
+#define nr_online_nodes		1U
+#endif
+
 /**
  * nodemask_pr_args - printf args to output a nodemask
  * @maskp: nodemask to be printed
@@ -105,7 +113,7 @@ extern nodemask_t _unused_nodemask_arg_;
 				__nodemask_pr_bits(maskp)
 static __always_inline unsigned int __nodemask_pr_numnodes(const nodemask_t *m)
 {
-	return m ? MAX_NUMNODES : 0;
+	return m ? nr_node_ids : 0;
 }
 static __always_inline const unsigned long *__nodemask_pr_bits(const nodemask_t *m)
 {
@@ -438,9 +446,6 @@ static __always_inline unsigned int next_memory_node(int nid)
 	return next_node(nid, node_states[N_MEMORY]);
 }
 
-extern unsigned int nr_node_ids;
-extern unsigned int nr_online_nodes;
-
 static __always_inline void node_set_online(int nid)
 {
 	node_set_state(nid, N_ONLINE);
@@ -480,8 +485,6 @@ static __always_inline int num_node_state(enum node_states state)
 #define first_memory_node	0
 #define next_online_node(nid)	(MAX_NUMNODES)
 #define next_memory_node(nid)	(MAX_NUMNODES)
-#define nr_node_ids		1U
-#define nr_online_nodes		1U
 
 #define node_set_online(node)	   node_set_state((node), N_ONLINE)
 #define node_set_offline(node)	   node_clear_state((node), N_ONLINE)
diff --git a/include/linux/nvmem-provider.h b/include/linux/nvmem-provider.h
index f3b13da78aac9..6063fe5b7784a 100644
--- a/include/linux/nvmem-provider.h
+++ b/include/linux/nvmem-provider.h
@@ -214,6 +214,12 @@ static inline int nvmem_layout_register(struct nvmem_layout *layout)
 
 static inline void nvmem_layout_unregister(struct nvmem_layout *layout) {}
 
+static inline int nvmem_add_cells_from_dt(struct nvmem_device *nvmem,
+					  struct device_node *np)
+{
+	return -EOPNOTSUPP;
+}
+
 #endif /* CONFIG_NVMEM */
 
 #if IS_ENABLED(CONFIG_NVMEM) && IS_ENABLED(CONFIG_OF)
diff --git a/include/linux/oom.h b/include/linux/oom.h
index 7b02bc1d0a7ea..00da05d227e66 100644
--- a/include/linux/oom.h
+++ b/include/linux/oom.h
@@ -30,7 +30,7 @@ struct oom_control {
 	struct zonelist *zonelist;
 
 	/* Used to determine mempolicy */
-	nodemask_t *nodemask;
+	const nodemask_t *nodemask;
 
 	/* Memory cgroup in which oom is invoked, or NULL for global oom */
 	struct mem_cgroup *memcg;
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 7223f6f4e2b40..7a863572adce7 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -879,20 +879,6 @@ FOLIO_FLAG_FALSE(partially_mapped)
 
 #define PG_head_mask ((1UL << PG_head))
 
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
-/*
- * PageTransCompound returns true for both transparent huge pages
- * and hugetlbfs pages, so it should only be called when it's known
- * that hugetlbfs pages aren't involved.
- */
-static inline int PageTransCompound(const struct page *page)
-{
-	return PageCompound(page);
-}
-#else
-TESTPAGEFLAG_FALSE(TransCompound, transcompound)
-#endif
-
 #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
 /*
  * PageHasHWPoisoned indicates that at least one subpage is hwpoisoned in the
diff --git a/include/linux/page_owner.h b/include/linux/page_owner.h
index 3328357f6dba7..8188ddc5c4127 100644
--- a/include/linux/page_owner.h
+++ b/include/linux/page_owner.h
@@ -3,6 +3,7 @@
 #define __LINUX_PAGE_OWNER_H
 
 #include <linux/jump_label.h>
+#include <linux/migrate_mode.h>
 
 #ifdef CONFIG_PAGE_OWNER
 extern struct static_key_false page_owner_inited;
@@ -14,7 +15,7 @@ extern void __set_page_owner(struct page *page,
 extern void __split_page_owner(struct page *page, int old_order,
 			int new_order);
 extern void __folio_copy_owner(struct folio *newfolio, struct folio *old);
-extern void __folio_set_owner_migrate_reason(struct folio *folio, int reason);
+void __folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason);
 extern void __dump_page_owner(const struct page *page);
 extern void pagetypeinfo_showmixedcount_print(struct seq_file *m,
 					pg_data_t *pgdat, struct zone *zone);
@@ -43,7 +44,7 @@ static inline void folio_copy_owner(struct folio *newfolio, struct folio *old)
 	if (static_branch_unlikely(&page_owner_inited))
 		__folio_copy_owner(newfolio, old);
 }
-static inline void folio_set_owner_migrate_reason(struct folio *folio, int reason)
+static inline void folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason)
 {
 	if (static_branch_unlikely(&page_owner_inited))
 		__folio_set_owner_migrate_reason(folio, reason);
@@ -68,7 +69,7 @@ static inline void split_page_owner(struct page *page, int old_order,
 static inline void folio_copy_owner(struct folio *newfolio, struct folio *folio)
 {
 }
-static inline void folio_set_owner_migrate_reason(struct folio *folio, int reason)
+static inline void folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason)
 {
 }
 static inline void dump_page_owner(const struct page *page)
diff --git a/include/linux/page_reporting.h b/include/linux/page_reporting.h
index 9d4ca5c218a0f..272b1274efdc1 100644
--- a/include/linux/page_reporting.h
+++ b/include/linux/page_reporting.h
@@ -5,7 +5,6 @@
 #include <linux/mmzone.h>
 #include <linux/scatterlist.h>
 
-/* This value should always be a power of 2, see page_reporting_cycle() */
 #define PAGE_REPORTING_CAPACITY		32
 #define PAGE_REPORTING_ORDER_UNSPECIFIED	-1
 
@@ -22,6 +21,9 @@ struct page_reporting_dev_info {
 
 	/* Minimal order of page reporting */
 	unsigned int order;
+
+	/* Max pages per report batch; 0 (default) means PAGE_REPORTING_CAPACITY */
+	unsigned int capacity;
 };
 
 /* Tear-down and bring-up for page reporting devices */
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index 2c3718d592d6d..c6fc783aaee57 100644
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -1063,12 +1063,41 @@ static inline pgoff_t folio_pgoff(const struct folio *folio)
 	return folio->index;
 }
 
+/**
+ * linear_page_delta() - Determine the relative page offset of @address within
+ * @vma.
+ * @vma: The VMA in which @address resides.
+ * @address: The address whose relative page offset is required.
+ *
+ * The result is identical for both file-backed and anonymous mappings and
+ * simply determines how many pages @address lies from @vma->vm_start.
+ *
+ * Returns: The number of pages @address is offset by within @vma.
+ */
+static inline pgoff_t linear_page_delta(const struct vm_area_struct *vma,
+					const unsigned long address)
+{
+	return (address - vma->vm_start) >> PAGE_SHIFT;
+}
+
+/**
+ * linear_page_index() - Determine the absolute page offset of @address within
+ * @vma.
+ * @vma: The VMA in which @address resides.
+ * @address: The address whose absolute page offset is required.
+ *
+ * See the comment for vma_start_pgoff() for a description of what the page
+ * offset signifies.
+ *
+ * Returns: The absolute page offset of @address within @vma.
+ */
 static inline pgoff_t linear_page_index(const struct vm_area_struct *vma,
 					const unsigned long address)
 {
 	pgoff_t pgoff;
-	pgoff = (address - vma->vm_start) >> PAGE_SHIFT;
-	pgoff += vma->vm_pgoff;
+
+	pgoff = linear_page_delta(vma, address);
+	pgoff += vma_start_pgoff(vma);
 	return pgoff;
 }
 
@@ -1219,7 +1248,7 @@ static inline vm_fault_t folio_lock_or_retry(struct folio *folio,
 void folio_wait_bit(struct folio *folio, int bit_nr);
 int folio_wait_bit_killable(struct folio *folio, int bit_nr);
 
-/* 
+/*
  * Wait for a folio to be unlocked.
  *
  * This must be called with the caller "holding" the folio,
diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h
index 43c854a273c3a..2cba7cc2b01f9 100644
--- a/include/linux/percpu-defs.h
+++ b/include/linux/percpu-defs.h
@@ -203,7 +203,7 @@
 /*
  * Accessors and operations.
  */
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 /*
  * __verify_pcpu_ptr() verifies @ptr is a percpu pointer without evaluating
@@ -514,5 +514,5 @@ do {									\
 #define this_cpu_inc_return(pcp)	this_cpu_add_return(pcp, 1)
 #define this_cpu_dec_return(pcp)	this_cpu_add_return(pcp, -1)
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 #endif /* _LINUX_PERCPU_DEFS_H */
diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h
index d73a1c08c3e3c..1e3212e2c8271 100644
--- a/include/linux/percpu-refcount.h
+++ b/include/linux/percpu-refcount.h
@@ -12,8 +12,8 @@
  * start shutting down you call percpu_ref_kill() _before_ dropping the initial
  * refcount.
  *
- * The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less
- * than an atomic_t - this is because of the way shutdown works, see
+ * The refcount will have a range of 0 to LONG_MAX, i.e. one bit less
+ * than an atomic_long_t - this is because of the way shutdown works, see
  * percpu_ref_kill()/PERCPU_COUNT_BIAS.
  *
  * Before you call percpu_ref_kill(), percpu_ref_put() does not check for the
@@ -269,6 +269,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref)
 /**
  * percpu_ref_tryget_live_rcu - same as percpu_ref_tryget_live() but the
  * caller is responsible for taking RCU.
+ * @ref: percpu_ref to try-get
  *
  * This function is safe to call as long as @ref is between init and exit.
  */
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 48d851fbd8ea5..5842552294c19 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -1791,22 +1791,8 @@ static inline int perf_is_paranoid(void)
 }
 
 extern int perf_allow_kernel(void);
-
-static inline int perf_allow_cpu(void)
-{
-	if (sysctl_perf_event_paranoid > 0 && !perfmon_capable())
-		return -EACCES;
-
-	return security_perf_event_open(PERF_SECURITY_CPU);
-}
-
-static inline int perf_allow_tracepoint(void)
-{
-	if (sysctl_perf_event_paranoid > -1 && !perfmon_capable())
-		return -EPERM;
-
-	return security_perf_event_open(PERF_SECURITY_TRACEPOINT);
-}
+extern int perf_allow_cpu(void);
+extern int perf_allow_tracepoint(void);
 
 extern int perf_exclude_event(struct perf_event *event, struct pt_regs *regs);
 
@@ -2023,6 +2009,19 @@ perf_event_pause(struct perf_event *event, bool reset)			{ return 0; }
 static inline int
 perf_exclude_event(struct perf_event *event, struct pt_regs *regs)	{ return 0; }
 
+static inline int perf_allow_kernel(void)
+{
+	return perfmon_capable() ? 0 : -EACCES;
+}
+static inline int perf_allow_cpu(void)
+{
+	return perfmon_capable() ? 0 : -EACCES;
+}
+static inline int perf_allow_tracepoint(void)
+{
+	return perfmon_capable() ? 0 : -EPERM;
+}
+
 #endif /* !CONFIG_PERF_EVENTS */
 
 #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)
diff --git a/include/linux/pfn.h b/include/linux/pfn.h
index b90ca0b6c331e..cfedf0f61bb3c 100644
--- a/include/linux/pfn.h
+++ b/include/linux/pfn.h
@@ -2,7 +2,7 @@
 #ifndef _LINUX_PFN_H_
 #define _LINUX_PFN_H_
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <linux/types.h>
 #endif
 
diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
index 2981e386da7b3..8c093c119e5a8 100644
--- a/include/linux/pgtable.h
+++ b/include/linux/pgtable.h
@@ -8,7 +8,7 @@
 #define PMD_ORDER	(PMD_SHIFT - PAGE_SHIFT)
 #define PUD_ORDER	(PUD_SHIFT - PAGE_SHIFT)
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #ifdef CONFIG_MMU
 
 #include <linux/mm_types.h>
@@ -988,49 +988,30 @@ static inline void update_mmu_tlb(struct vm_area_struct *vma,
 	update_mmu_tlb_range(vma, address, ptep, 1);
 }
 
-/*
- * Some architectures may be able to avoid expensive synchronization
- * primitives when modifications are made to PTE's which are already
- * not present, or in the process of an address space destruction.
- */
-#ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL
-static inline void pte_clear_not_present_full(struct mm_struct *mm,
-					      unsigned long address,
-					      pte_t *ptep,
-					      int full)
-{
-	pte_clear(mm, address, ptep);
-}
-#endif
-
-#ifndef clear_not_present_full_ptes
 /**
- * clear_not_present_full_ptes - Clear multiple not present PTEs which are
- *				 consecutive in the pgtable.
+ * clear_nonpresent_ptes - Clear multiple non-present PTEs which are
+ *			    consecutive in the pgtable.
  * @mm: Address space the ptes represent.
  * @addr: Address of the first pte.
  * @ptep: Page table pointer for the first entry.
  * @nr: Number of entries to clear.
- * @full: Whether we are clearing a full mm.
- *
- * May be overridden by the architecture; otherwise, implemented as a simple
- * loop over pte_clear_not_present_full().
  *
  * Context: The caller holds the page table lock.  The PTEs are all not present.
  * The PTEs are all in the same PMD.
  */
-static inline void clear_not_present_full_ptes(struct mm_struct *mm,
-		unsigned long addr, pte_t *ptep, unsigned int nr, int full)
+static inline void clear_nonpresent_ptes(struct mm_struct *mm,
+		unsigned long addr, pte_t *ptep, unsigned int nr)
 {
+	(void)addr;
+
 	for (;;) {
-		pte_clear_not_present_full(mm, addr, ptep, full);
+		pte_clear(mm, addr, ptep);
 		if (--nr == 0)
 			break;
 		ptep++;
 		addr += PAGE_SIZE;
 	}
 }
-#endif
 
 #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
 extern pte_t ptep_clear_flush(struct vm_area_struct *vma,
@@ -1858,7 +1839,7 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
 #endif
 
 #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
-#ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#ifndef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
 {
 	return pmd;
@@ -2129,18 +2110,26 @@ static inline int pud_trans_unstable(pud_t *pud)
 	return 0;
 }
 
-#ifndef CONFIG_NUMA_BALANCING
+#ifndef CONFIG_ARCH_HAS_PTE_PROTNONE
 /*
- * In an inaccessible (PROT_NONE) VMA, pte_protnone() may indicate "yes". It is
- * perfectly valid to indicate "no" in that case, which is why our default
- * implementation defaults to "always no".
+ * In an inaccessible (PROT_NONE) VMA, pte_protnone() may indicate "yes". It
+ * is perfectly valid to indicate "no" in that case, which is why our
+ * default implementation defaults to "always no".
+ *
+ * In an accessible VMA, pte_protnone() reliably indicates a present
+ * PROT_NONE page protection. Today the kernel uses such PTEs for two
+ * purposes: NUMA hinting faults, and userfaultfd RWP tracking on
+ * VM_UFFD_RWP VMAs. The two are distinguished by the uffd PTE bit and
+ * the VMA flag; see include/linux/userfaultfd_k.h.
  *
- * In an accessible VMA, however, pte_protnone() reliably indicates PROT_NONE
- * page protection due to NUMA hinting. NUMA hinting faults only apply in
- * accessible VMAs.
+ * So, to reliably identify PROT_NONE PTEs that require kernel handling,
+ * looking at the VMA accessibility (and the uffd bit on RWP VMAs) is
+ * sufficient.
  *
- * So, to reliably identify PROT_NONE PTEs that require a NUMA hinting fault,
- * looking at the VMA accessibility is sufficient.
+ * Architectures without CONFIG_ARCH_HAS_PTE_PROTNONE get the always-zero
+ * stubs below; PAGE_NONE references that survive to runtime fire the
+ * BUILD_BUG() fallback, since callers should have folded such paths to
+ * dead code via IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE).
  */
 static inline int pte_protnone(pte_t pte)
 {
@@ -2151,7 +2140,11 @@ static inline int pmd_protnone(pmd_t pmd)
 {
 	return 0;
 }
-#endif /* CONFIG_NUMA_BALANCING */
+
+#ifndef PAGE_NONE
+#define PAGE_NONE ({ BUILD_BUG(); (pgprot_t){0}; })
+#endif
+#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */
 
 #endif /* CONFIG_MMU */
 
@@ -2320,7 +2313,7 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level)
 	}
 }
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT)
 #ifdef CONFIG_PHYS_ADDR_T_64BIT
diff --git a/include/linux/platform_data/x86/asus-wmi.h b/include/linux/platform_data/x86/asus-wmi.h
index c29962d5baac6..b5ed8c83ace1c 100644
--- a/include/linux/platform_data/x86/asus-wmi.h
+++ b/include/linux/platform_data/x86/asus-wmi.h
@@ -203,6 +203,7 @@ int asus_wmi_evaluate_method(u32 method_id, u32 arg0, u32 arg1, u32 *retval);
 int asus_hid_register_listener(struct asus_hid_listener *cdev);
 void asus_hid_unregister_listener(struct asus_hid_listener *cdev);
 int asus_hid_event(enum asus_hid_event event);
+bool asus_wmi_custom_fan_curve_is_enabled(void);
 #else
 static inline void set_ally_mcu_hack(enum asus_ally_mcu_hack status)
 {
@@ -234,6 +235,10 @@ static inline int asus_hid_event(enum asus_hid_event event)
 {
 	return -ENODEV;
 }
+static inline bool asus_wmi_custom_fan_curve_is_enabled(void)
+{
+	return false;
+}
 #endif
 
 #endif	/* __PLATFORM_DATA_X86_ASUS_WMI_H */
diff --git a/include/linux/platform_device.h b/include/linux/platform_device.h
index 8c566f09d04ef..3d5bbcbae7308 100644
--- a/include/linux/platform_device.h
+++ b/include/linux/platform_device.h
@@ -19,6 +19,8 @@
 struct irq_affinity;
 struct mfd_cell;
 struct property_entry;
+struct device_node;
+struct fwnode_handle;
 
 struct platform_device {
 	const char	*name;
@@ -262,6 +264,12 @@ extern int platform_device_add_resources(struct platform_device *pdev,
 					 unsigned int num);
 extern int platform_device_add_data(struct platform_device *pdev,
 				    const void *data, size_t size);
+void platform_device_set_of_node(struct platform_device *pdev,
+				 struct device_node *np);
+void platform_device_set_fwnode(struct platform_device *pdev,
+				struct fwnode_handle *fwnode);
+void platform_device_set_of_node_from_dev(struct platform_device *pdev,
+					  const struct device *dev2);
 extern int platform_device_add(struct platform_device *pdev);
 extern void platform_device_del(struct platform_device *pdev);
 extern void platform_device_put(struct platform_device *pdev);
diff --git a/include/linux/pps_kernel.h b/include/linux/pps_kernel.h
index 9f088c9023b1c..00b840970d560 100644
--- a/include/linux/pps_kernel.h
+++ b/include/linux/pps_kernel.h
@@ -60,6 +60,7 @@ struct pps_device {
 	struct device dev;
 	struct fasync_struct *async_queue;	/* fasync method */
 	spinlock_t lock;
+	bool kc_removed;
 };
 
 /*
diff --git a/include/linux/psi.h b/include/linux/psi.h
index e0745873e3f26..7966e3ac03b99 100644
--- a/include/linux/psi.h
+++ b/include/linux/psi.h
@@ -25,7 +25,9 @@ void psi_memstall_leave(unsigned long *flags);
 int psi_show(struct seq_file *s, struct psi_group *group, enum psi_res res);
 struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf,
 				       enum psi_res res, struct file *file,
-				       struct kernfs_open_file *of);
+				       struct kernfs_open_file *of,
+				       bool *need_rtpoll_worker);
+int psi_trigger_create_rtpoll_worker(struct psi_group *group);
 void psi_trigger_destroy(struct psi_trigger *t);
 
 __poll_t psi_trigger_poll(void **trigger_ptr, struct file *file,
diff --git a/include/linux/psp-sev.h b/include/linux/psp-sev.h
index ce16bbc0b3083..03a79786df1d7 100644
--- a/include/linux/psp-sev.h
+++ b/include/linux/psp-sev.h
@@ -129,6 +129,7 @@ enum sev_cmd {
 	SEV_CMD_SNP_LAUNCH_FINISH	= 0x0A2,
 	SEV_CMD_SNP_DBG_DECRYPT		= 0x0B0,
 	SEV_CMD_SNP_DBG_ENCRYPT		= 0x0B1,
+	SEV_CMD_SNP_VERIFY_MITIGATION	= 0x0B2,
 	SEV_CMD_SNP_PAGE_SWAP_OUT	= 0x0C0,
 	SEV_CMD_SNP_PAGE_SWAP_IN	= 0x0C1,
 	SEV_CMD_SNP_PAGE_MOVE		= 0x0C2,
@@ -898,10 +899,60 @@ struct snp_feature_info {
 #define SNP_CIPHER_TEXT_HIDING_SUPPORTED	BIT(3)
 #define SNP_AES_256_XTS_POLICY_SUPPORTED	BIT(4)
 #define SNP_CXL_ALLOW_POLICY_SUPPORTED		BIT(5)
+#define SNP_VERIFY_MITIGATION_SUPPORTED		BIT(13)
 
 /* Feature bits in EBX */
 #define SNP_SEV_TIO_SUPPORTED			BIT(1)
 
+#define SNP_MIT_SUBCMD_REQ_STATUS      0x0
+#define SNP_MIT_SUBCMD_REQ_VERIFY      0x1
+
+/**
+ * struct sev_data_snp_verify_mitigation - SNP_VERIFY_MITIGATION command params
+ *
+ * @length: Length of the command buffer read by the PSP
+ * @subcommand: Mitigation sub-command for the firmware to execute.
+ *              REQ_STATUS: 0x0 - Request status about currently supported and
+ *                                verified mitigations
+ *              REQ_VERIFY: 0x1 - Request to initiate verification mitigation
+ *                                operation on a specific mitigation
+ * @rsvd: Reserved
+ * @vector: Bit specifying the vulnerability mitigation to process
+ * @dst_paddr_en: Destination paddr enabled
+ * @src_paddr_en: Source paddr enabled
+ * @rsvd1: Reserved
+ * @rsvd2: Reserved
+ * @src_paddr: Source address for optional input data
+ * @dst_paddr: Destination address to write the result
+ * @rsvd3: Reserved
+ */
+struct sev_data_snp_verify_mitigation {
+	u32 length;
+	u16 subcommand;
+	u16 rsvd;
+	u64 vector;
+	u32 dst_paddr_en : 1,
+	    src_paddr_en : 1,
+	    rsvd1 : 30;
+	u8 rsvd2[4];
+	u64 src_paddr;
+	u64 dst_paddr;
+	u8 rsvd3[24];
+} __packed;
+
+/**
+ * struct sev_data_snp_verify_mitigation_dst - mitigation result vectors
+ *
+ * @mit_verified_vector: Bit vector of vulnerability mitigations verified
+ * @mit_supported_vector: Bit vector of vulnerability mitigations supported
+ * @mit_failure_status: Status of the verification operation
+ */
+struct sev_data_snp_verify_mitigation_dst {
+	u64 mit_verified_vector;                /* OUT */
+	u64 mit_supported_vector;               /* OUT */
+	u32 mit_failure_status;                 /* OUT */
+} __packed;
+
 /**
  * struct sev_snp_tcb_version_genoa_milan
  *
diff --git a/include/linux/ptrace.h b/include/linux/ptrace.h
index ef314f7a9ecc5..2839d5c8eaa0f 100644
--- a/include/linux/ptrace.h
+++ b/include/linux/ptrace.h
@@ -405,13 +405,13 @@ extern void sigaction_compat_abi(struct k_sigaction *act, struct k_sigaction *oa
 /*
  * ptrace report for syscall entry and exit looks identical.
  */
-static inline int ptrace_report_syscall(unsigned long message)
+static inline bool ptrace_report_syscall(unsigned long message)
 {
 	int ptrace = current->ptrace;
 	int signr;
 
 	if (!(ptrace & PT_PTRACED))
-		return 0;
+		return true;
 
 	signr = ptrace_notify(SIGTRAP | ((ptrace & PT_TRACESYSGOOD) ? 0x80 : 0),
 			      message);
@@ -424,11 +424,11 @@ static inline int ptrace_report_syscall(unsigned long message)
 	if (signr)
 		send_sig(signr, current, 1);
 
-	return fatal_signal_pending(current);
+	return !fatal_signal_pending(current);
 }
 
 /**
- * ptrace_report_syscall_entry - task is about to attempt a system call
+ * ptrace_report_syscall_permit_entry - task is about to attempt a system call
  * @regs:		user register state of current task
  *
  * This will be called if %SYSCALL_WORK_SYSCALL_TRACE or
@@ -438,7 +438,7 @@ static inline int ptrace_report_syscall(unsigned long message)
  * call number and arguments to be tried.  It is safe to block here,
  * preventing the system call from beginning.
  *
- * Returns zero normally, or nonzero if the calling arch code should abort
+ * Returns True normally, or False if the calling architecture code should abort
  * the system call.  That must prevent normal entry so no system call is
  * made.  If @task ever returns to user mode after this, its register state
  * is unspecified, but should be something harmless like an %ENOSYS error
@@ -447,8 +447,7 @@ static inline int ptrace_report_syscall(unsigned long message)
  *
  * Called without locks, just after entering kernel mode.
  */
-static inline __must_check int ptrace_report_syscall_entry(
-	struct pt_regs *regs)
+static inline __must_check bool ptrace_report_syscall_permit_entry(struct pt_regs *regs)
 {
 	return ptrace_report_syscall(PTRACE_EVENTMSG_SYSCALL_ENTRY);
 }
diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h
index eae67015ce51a..057edc4cbb6ef 100644
--- a/include/linux/radix-tree.h
+++ b/include/linux/radix-tree.h
@@ -111,7 +111,7 @@ struct radix_tree_iter {
 };
 
 /**
- * Radix-tree synchronization
+ * DOC: Radix-tree synchronization
  *
  * The radix-tree API requires that users provide all synchronisation (with
  * specific exceptions, noted below).
@@ -182,6 +182,7 @@ static inline void *radix_tree_deref_slot(void __rcu **slot)
 /**
  * radix_tree_deref_slot_protected - dereference a slot with tree lock held
  * @slot: slot pointer, returned by radix_tree_lookup_slot
+ * @treelock: caller must hold this spinlock
  *
  * Similar to radix_tree_deref_slot.  The caller does not hold the RCU read
  * lock but it must hold the tree lock to prevent parallel updates.
@@ -306,7 +307,7 @@ radix_tree_iter_init(struct radix_tree_iter *iter, unsigned long start)
  * Also it fills @iter with data about chunk: position in the tree (index),
  * its end (next_index), and constructs a bit mask for tagged iterating (tags).
  */
-void __rcu **radix_tree_next_chunk(const struct radix_tree_root *,
+void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root,
 			     struct radix_tree_iter *iter, unsigned flags);
 
 /**
diff --git a/include/linux/randomize_kstack.h b/include/linux/randomize_kstack.h
index 024fc20e77622..d8b939f50bba7 100644
--- a/include/linux/randomize_kstack.h
+++ b/include/linux/randomize_kstack.h
@@ -77,8 +77,27 @@ static __always_inline u32 get_kstack_offset(void)
 	}								\
 } while (0)
 
+/**
+ * add_random_kstack_offset_irqsoff - Increase stack utilization by a random offset.
+ *
+ * This should be used in the syscall entry path after user registers have been
+ * stored to the stack. Interrupts must be still disabled.
+ */
+#define add_random_kstack_offset_irqsoff()					\
+do {										\
+	lockdep_assert_irqs_disabled();						\
+	if (static_branch_maybe(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT,		\
+				&randomize_kstack_offset)) {			\
+		u32 offset = prandom_u32_state(raw_cpu_ptr(&kstack_rnd_state));	\
+		u8 *ptr = __kstack_alloca(KSTACK_OFFSET_MAX(offset));		\
+		/* Keep allocation even after "ptr" loses scope. */		\
+		asm volatile("" :: "r"(ptr) : "memory");			\
+	}									\
+} while (0)
+
 #else /* CONFIG_RANDOMIZE_KSTACK_OFFSET */
 #define add_random_kstack_offset()		do { } while (0)
+#define add_random_kstack_offset_irqsoff()	do { } while (0)
 #endif /* CONFIG_RANDOMIZE_KSTACK_OFFSET */
 
 #endif
diff --git a/include/linux/regmap.h b/include/linux/regmap.h
index df44cb30f53b9..370baa19db873 100644
--- a/include/linux/regmap.h
+++ b/include/linux/regmap.h
@@ -1770,6 +1770,8 @@ struct regmap_irq_chip {
 			       void *irq_drv_data);
 	unsigned int (*get_irq_reg)(struct regmap_irq_chip_data *data,
 				    unsigned int base, int index);
+	int (*irq_reqres)(void *irq_drv_data, irq_hw_number_t hwirq);
+	void (*irq_relres)(void *irq_drv_data, irq_hw_number_t hwirq);
 	void *irq_drv_data;
 };
 
diff --git a/include/linux/remoteproc.h b/include/linux/remoteproc.h
index 7c1546d480082..17ed75a11e152 100644
--- a/include/linux/remoteproc.h
+++ b/include/linux/remoteproc.h
@@ -259,6 +259,7 @@ enum rproc_features {
  * @subdevs: list of subdevices, to following the running state
  * @notifyids: idr for dynamically assigning rproc-wide unique notify ids
  * @index: index of this rproc device
+ * @attach_work: workqueue for attaching rproc
  * @crash_handler: workqueue for handling a crash
  * @crash_cnt: crash counter
  * @recovery_disabled: flag that state if recovery was disabled
@@ -271,6 +272,7 @@ enum rproc_features {
  * @has_iommu: flag to indicate if remote processor is behind an MMU
  * @auto_boot: flag to indicate if remote processor should be auto-started
  * @sysfs_read_only: flag to make remoteproc sysfs files read only
+ * @subdevs_started: flag to indicate if subdevs have started
  * @dump_segments: list of segments in the firmware
  * @nb_vdev: number of vdev currently handled by rproc
  * @elf_class: firmware ELF class
@@ -301,6 +303,7 @@ struct rproc {
 	struct list_head subdevs;
 	struct idr notifyids;
 	int index;
+	struct work_struct attach_work;
 	struct work_struct crash_handler;
 	unsigned int crash_cnt;
 	bool recovery_disabled;
@@ -312,6 +315,7 @@ struct rproc {
 	bool has_iommu;
 	bool auto_boot;
 	bool sysfs_read_only;
+	bool subdevs_started;
 	struct list_head dump_segments;
 	int nb_vdev;
 	u8 elf_class;
diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h
index 79f83b6eec272..57a2a29bef0e8 100644
--- a/include/linux/rhashtable.h
+++ b/include/linux/rhashtable.h
@@ -23,7 +23,6 @@
 #include <linux/irq_work.h>
 #include <linux/jhash.h>
 #include <linux/list_nulls.h>
-#include <linux/workqueue.h>
 #include <linux/rculist.h>
 #include <linux/bit_spinlock.h>
 
diff --git a/include/linux/rtnetlink.h b/include/linux/rtnetlink.h
index ea39dd23a1976..95729339e7a54 100644
--- a/include/linux/rtnetlink.h
+++ b/include/linux/rtnetlink.h
@@ -115,6 +115,10 @@ bool rtnl_net_is_locked(struct net *net);
 
 bool lockdep_rtnl_net_is_held(struct net *net);
 
+void rtnl_net_queue_work(struct net *net);
+void rtnl_net_flush_workqueue(void);
+void rtnl_net_work_func(struct work_struct *work);
+
 #define rcu_dereference_rtnl_net(net, p)				\
 	rcu_dereference_check(p, lockdep_rtnl_net_is_held(net))
 #define rtnl_net_dereference(net, p)					\
@@ -150,6 +154,10 @@ static inline void ASSERT_RTNL_NET(struct net *net)
 	ASSERT_RTNL();
 }
 
+static inline void rtnl_net_flush_workqueue(void)
+{
+}
+
 #define rcu_dereference_rtnl_net(net, p)		\
 	rcu_dereference_rtnl(p)
 #define rtnl_net_dereference(net, p)			\
diff --git a/include/linux/rtsx_usb.h b/include/linux/rtsx_usb.h
index 276b509c03e36..0fc5a74700a8b 100644
--- a/include/linux/rtsx_usb.h
+++ b/include/linux/rtsx_usb.h
@@ -61,6 +61,9 @@ struct rtsx_ucr {
 
 	struct timer_list	sg_timer;
 	struct mutex		dev_mutex;
+
+	u16			card_status_cache;
+	bool			card_status_valid;
 };
 
 /* buffer size */
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 373bcc0598d10..bcd101db428e4 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -550,7 +550,6 @@ struct sched_statistics {
 	s64				exec_max;
 	u64				slice_max;
 
-	u64				nr_migrations_cold;
 	u64				nr_failed_migrations_affine;
 	u64				nr_failed_migrations_running;
 	u64				nr_failed_migrations_hot;
@@ -563,8 +562,6 @@ struct sched_statistics {
 	u64				nr_wakeups_remote;
 	u64				nr_wakeups_affine;
 	u64				nr_wakeups_affine_attempts;
-	u64				nr_wakeups_passive;
-	u64				nr_wakeups_idle;
 
 #ifdef CONFIG_SCHED_CORE
 	u64				core_forceidle_sum;
@@ -575,6 +572,7 @@ struct sched_statistics {
 struct sched_entity {
 	/* For load-balancing: */
 	struct load_weight		load;
+	struct load_weight		h_load;
 	struct rb_node			run_node;
 	u64				deadline;
 	u64				min_vruntime;
@@ -1191,6 +1189,7 @@ struct task_struct {
 	unsigned long			last_switch_time;
 #endif
 	/* Filesystem information: */
+	struct fs_struct		*real_fs;
 	struct fs_struct		*fs;
 
 	/* Open file information: */
@@ -1288,6 +1287,7 @@ struct task_struct {
 	u64				curr_chain_key;
 	int				lockdep_depth;
 	unsigned int			lockdep_recursion;
+	unsigned int			lockdep_seq;
 	struct held_lock		held_locks[MAX_LOCK_DEPTH];
 #endif
 
diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h
index 20b2343aa344b..803da0f1e5092 100644
--- a/include/linux/sched/ext.h
+++ b/include/linux/sched/ext.h
@@ -58,6 +58,7 @@ enum scx_dsq_id_flags {
 	SCX_DSQ_GLOBAL		= SCX_DSQ_FLAG_BUILTIN | 1,
 	SCX_DSQ_LOCAL		= SCX_DSQ_FLAG_BUILTIN | 2,
 	SCX_DSQ_BYPASS		= SCX_DSQ_FLAG_BUILTIN | 3,
+	SCX_DSQ_REJECT		= SCX_DSQ_FLAG_BUILTIN | 4,	/* internal - see find_dsq_for_dispatch() */
 	SCX_DSQ_LOCAL_ON	= SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON,
 	SCX_DSQ_LOCAL_CPU_MASK	= 0xffffffffLLU,
 };
@@ -124,7 +125,7 @@ enum scx_ent_flags {
 	SCX_TASK_DEAD		= 5 << SCX_TASK_STATE_SHIFT,
 
 	/*
-	 * Bits 12 and 13 are used to carry reenqueue reason. In addition to
+	 * Bits 12 to 14 are used to carry reenqueue reason. In addition to
 	 * %SCX_ENQ_REENQ flag, ops.enqueue() can also test for
 	 * %SCX_TASK_REENQ_REASON_NONE to distinguish reenqueues.
 	 *
@@ -132,15 +133,17 @@ enum scx_ent_flags {
 	 * KFUNC	reenqueued by scx_bpf_dsq_reenq() and friends
 	 * IMMED	reenqueued due to failed ENQ_IMMED
 	 * PREEMPTED	preempted while running
+	 * CAP		sub-sched cap miss, see p->scx.reenq_reason_*
 	 */
 	SCX_TASK_REENQ_REASON_SHIFT = 12,
-	SCX_TASK_REENQ_REASON_BITS = 2,
+	SCX_TASK_REENQ_REASON_BITS = 3,
 	SCX_TASK_REENQ_REASON_MASK = ((1 << SCX_TASK_REENQ_REASON_BITS) - 1) << SCX_TASK_REENQ_REASON_SHIFT,
 
 	SCX_TASK_REENQ_NONE	= 0 << SCX_TASK_REENQ_REASON_SHIFT,
 	SCX_TASK_REENQ_KFUNC	= 1 << SCX_TASK_REENQ_REASON_SHIFT,
 	SCX_TASK_REENQ_IMMED	= 2 << SCX_TASK_REENQ_REASON_SHIFT,
 	SCX_TASK_REENQ_PREEMPTED = 3 << SCX_TASK_REENQ_REASON_SHIFT,
+	SCX_TASK_REENQ_CAP	= 4 << SCX_TASK_REENQ_REASON_SHIFT,
 
 	/* iteration cursor, not a task */
 	SCX_TASK_CURSOR		= 1 << 31,
@@ -198,6 +201,7 @@ struct sched_ext_entity {
 	s32			sticky_cpu;
 	s32			holding_cpu;
 	s32			selected_cpu;
+	s32			runnable_cpu;	/* cpu @p is runnable on, -1 if not */
 	struct task_struct	*kf_tasks[2];	/* see SCX_CALL_OP_TASK() */
 
 	struct list_head	runnable_node;	/* rq->scx.runnable_list */
@@ -219,10 +223,11 @@ struct sched_ext_entity {
 	/* BPF scheduler modifiable fields */
 
 	/*
-	 * Runtime budget in nsecs. This is usually set through
-	 * scx_bpf_dsq_insert() but can also be modified directly by the BPF
-	 * scheduler. Automatically decreased by SCX as the task executes. On
-	 * depletion, a scheduling event is triggered.
+	 * Runtime budget in nsecs - how long the task may hold its cpu. Owned
+	 * by the task's scheduler. Set it when enqueuing via
+	 * scx_bpf_dsq_insert(), or otherwise via scx_bpf_task_set_slice().
+	 * Automatically decreased as the task executes. On depletion a
+	 * scheduling event is triggered.
 	 *
 	 * This value is cleared to zero if the task is preempted by
 	 * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the
@@ -239,6 +244,22 @@ struct sched_ext_entity {
 	 */
 	u64			dsq_vtime;
 
+	/*
+	 * Out-of-band slice request from scx_bpf_task_set_slice() when the
+	 * caller does not hold the rq lock, applied under the rq lock at the
+	 * next slice consideration. One atomic64 packs the pending flag, the
+	 * issuing sch's id, and the requested slice. See scx_slice_oob_consts.
+	 */
+	atomic64_t		slice_oob;
+
+	/*
+	 * Sub-sched cap rejected reenq context, valid only while
+	 * %SCX_TASK_REENQ_CAP is set. @reenq_reason_caps is the SCX_CAP_* bits
+	 * that were needed but missing. @reenq_reason_cid is the target cid.
+	 */
+	u64			reenq_reason_caps;
+	s32			reenq_reason_cid;
+
 	/*
 	 * If set, reject future sched_setscheduler(2) calls updating the policy
 	 * to %SCHED_EXT with -%EACCES.
@@ -263,7 +284,7 @@ void sched_ext_dead(struct task_struct *p);
 void print_scx_info(const char *log_lvl, struct task_struct *p);
 void scx_softlockup(u32 dur_s);
 bool scx_hardlockup(int cpu);
-bool scx_rcu_cpu_stall(void);
+bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask);
 
 #else	/* !CONFIG_SCHED_CLASS_EXT */
 
@@ -271,7 +292,7 @@ static inline void sched_ext_dead(struct task_struct *p) {}
 static inline void print_scx_info(const char *log_lvl, struct task_struct *p) {}
 static inline void scx_softlockup(u32 dur_s) {}
 static inline bool scx_hardlockup(int cpu) { return false; }
-static inline bool scx_rcu_cpu_stall(void) { return false; }
+static inline bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask) { return false; }
 
 #endif	/* CONFIG_SCHED_CLASS_EXT */
 
diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h
index 95d0040df5841..10be8a54b416f 100644
--- a/include/linux/sched/mm.h
+++ b/include/linux/sched/mm.h
@@ -193,21 +193,18 @@ unsigned long mm_get_unmapped_area(struct file *filp, unsigned long addr,
 				   unsigned long len, unsigned long pgoff,
 				   unsigned long flags);
 
-unsigned long mm_get_unmapped_area_vmflags(struct file *filp,
-					   unsigned long addr,
-					   unsigned long len,
-					   unsigned long pgoff,
-					   unsigned long flags,
-					   vm_flags_t vm_flags);
+unsigned long mm_get_unmapped_area_vmaflags(struct file *filp,
+		unsigned long addr, unsigned long len, unsigned long pgoff,
+		unsigned long flags, vma_flags_t vma_flags);
 
 unsigned long
 generic_get_unmapped_area(struct file *filp, unsigned long addr,
 			  unsigned long len, unsigned long pgoff,
-			  unsigned long flags, vm_flags_t vm_flags);
+			  unsigned long flags, vma_flags_t vma_flags);
 unsigned long
 generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
 				  unsigned long len, unsigned long pgoff,
-				  unsigned long flags, vm_flags_t vm_flags);
+				  unsigned long flags, vma_flags_t vma_flags);
 #else
 static inline void arch_pick_mmap_layout(struct mm_struct *mm,
 					 const struct rlimit *rlim_stack) {}
diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h
index 41ed884cffc96..e0c1ca8c6a188 100644
--- a/include/linux/sched/task.h
+++ b/include/linux/sched/task.h
@@ -31,6 +31,7 @@ struct kernel_clone_args {
 	u32 io_thread:1;
 	u32 user_worker:1;
 	u32 no_files:1;
+	u32 umh:1;
 	unsigned long stack;
 	unsigned long stack_size;
 	unsigned long tls;
diff --git a/include/linux/seccomp.h b/include/linux/seccomp.h
index 9b959972bf4a2..fcb3eb9825e50 100644
--- a/include/linux/seccomp.h
+++ b/include/linux/seccomp.h
@@ -22,14 +22,14 @@
 #include <linux/atomic.h>
 #include <asm/seccomp.h>
 
-extern int __secure_computing(void);
+extern bool __seccomp_permit_syscall(void);
 
 #ifdef CONFIG_HAVE_ARCH_SECCOMP_FILTER
-static inline int secure_computing(void)
+static __always_inline bool seccomp_permit_syscall(void)
 {
 	if (unlikely(test_syscall_work(SECCOMP)))
-		return  __secure_computing();
-	return 0;
+		return  __seccomp_permit_syscall();
+	return true;
 }
 #else
 extern void secure_computing_strict(int this_syscall);
@@ -50,11 +50,11 @@ static inline int seccomp_mode(struct seccomp *s)
 struct seccomp_data;
 
 #ifdef CONFIG_HAVE_ARCH_SECCOMP_FILTER
-static inline int secure_computing(void) { return 0; }
+static inline bool seccomp_permit_syscall(void) { return true; }
 #else
 static inline void secure_computing_strict(int this_syscall) { return; }
 #endif
-static inline int __secure_computing(void) { return 0; }
+static inline bool __seccomp_permit_syscall(void) { return true; }
 
 static inline long prctl_get_seccomp(void)
 {
diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h
index bdc214386e4a5..c4cc4f66af4ba 100644
--- a/include/linux/serial_core.h
+++ b/include/linux/serial_core.h
@@ -437,6 +437,7 @@ enum uart_iotype {
 	UPIO_TSI	= SERIAL_IO_TSI,	/* Tsi108/109 type IO */
 	UPIO_MEM32BE	= SERIAL_IO_MEM32BE,	/* 32b big endian */
 	UPIO_MEM16	= SERIAL_IO_MEM16,	/* 16b little endian */
+	UPIO_BUS	= SERIAL_IO_BUS,	/* Serial bus I/O access (ex: SPI, I2C) */
 };
 
 struct uart_port {
@@ -1338,4 +1339,9 @@ static inline int uart_handle_break(struct uart_port *port)
 					 !((cflag) & CLOCAL))
 
 int uart_get_rs485_mode(struct uart_port *port);
+
+void uart_get_ioinfos(struct uart_port *port, char *buf, size_t size);
+bool uart_iotype_mmio(enum uart_iotype iotype);
+bool uart_iotype_io(enum uart_iotype iotype);
+
 #endif /* LINUX_SERIAL_CORE_H */
diff --git a/include/linux/serial_s3c.h b/include/linux/serial_s3c.h
index 102aa33d956c4..f54cb6e23f85e 100644
--- a/include/linux/serial_s3c.h
+++ b/include/linux/serial_s3c.h
@@ -269,7 +269,7 @@
 #define APPLE_S5L_UTRSTAT_RXTO		BIT(9)
 #define APPLE_S5L_UTRSTAT_ALL_FLAGS	GENMASK(9, 3)
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/serial_core.h>
 
@@ -294,7 +294,7 @@ struct s3c2410_uartcfg {
 	unsigned long	   ufcon;	 /* value of ufcon for port */
 };
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* __ASM_ARM_REGS_SERIAL_H */
 
diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index e729b9b0e38d4..5663dff53186e 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -12,8 +12,6 @@
 #include <linux/userfaultfd_k.h>
 #include <linux/bits.h>
 
-struct swap_iocb;
-
 /* inode in-kernel data */
 
 #ifdef CONFIG_TMPFS_QUOTA
@@ -123,8 +121,7 @@ static inline bool shmem_mapping(const struct address_space *mapping)
 void shmem_unlock_mapping(struct address_space *mapping);
 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
 					pgoff_t index, gfp_t gfp_mask);
-int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
-		struct list_head *folio_list);
+int shmem_write_folio(struct folio *folio);
 void shmem_truncate_range(struct inode *inode, loff_t start, uoff_t end);
 int shmem_unuse(unsigned int type);
 
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 22eda1d54a0e8..4431b026e429d 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -3573,7 +3573,7 @@ static inline struct page *__dev_alloc_pages_noprof(gfp_t gfp_mask,
 	 * 3.  If requesting a order 0 page it will not be compound
 	 *     due to the check to see if order has a value in prep_new_page
 	 * 4.  __GFP_MEMALLOC is ignored if __GFP_NOMEMALLOC is set due to
-	 *     code in gfp_to_alloc_flags that should be enforcing this.
+	 *     code in alloc_flags_slowpath() that should be enforcing this.
 	 */
 	gfp_mask |= __GFP_COMP | __GFP_MEMALLOC;
 
diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h
index a8553401b1c93..d5e35f24738d5 100644
--- a/include/linux/skmsg.h
+++ b/include/linux/skmsg.h
@@ -551,20 +551,6 @@ static inline void psock_progs_drop(struct sk_psock_progs *progs)
 	psock_set_prog(&progs->skb_verdict, NULL);
 }
 
-/* for tcp only, sk is locked */
-static inline ssize_t sk_psock_msg_inq(struct sock *sk)
-{
-	struct sk_psock *psock;
-	ssize_t inq = 0;
-
-	psock = sk_psock_get(sk);
-	if (likely(psock)) {
-		inq = sk_psock_get_msg_len_nolock(psock);
-		sk_psock_put(sk, psock);
-	}
-	return inq;
-}
-
 /* for udp only, sk is not locked */
 static inline ssize_t sk_msg_first_len(struct sock *sk)
 {
diff --git a/include/linux/slab.h b/include/linux/slab.h
index 51f03f18c9a7b..3a14df50766ab 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -58,10 +58,13 @@ enum _slab_flag_bits {
 #endif
 	_SLAB_OBJECT_POISON,
 	_SLAB_CMPXCHG_DOUBLE,
+#ifdef CONFIG_SLAB_OBJ_EXT
 	_SLAB_NO_OBJ_EXT,
-#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT)
+#ifdef CONFIG_64BIT
 	_SLAB_OBJ_EXT_IN_OBJ,
 #endif
+#endif
+	_SLAB_NO_SHEAVES,
 	_SLAB_FLAGS_LAST_BIT
 };
 
@@ -239,8 +242,14 @@ enum _slab_flag_bits {
 #endif
 #define SLAB_TEMPORARY		SLAB_RECLAIM_ACCOUNT	/* Objects are short-lived */
 
-/* Slab created using create_boot_cache */
+/* Slab caches without obj_exts array */
+#ifdef CONFIG_SLAB_OBJ_EXT
 #define SLAB_NO_OBJ_EXT		__SLAB_FLAG_BIT(_SLAB_NO_OBJ_EXT)
+#else
+#define SLAB_NO_OBJ_EXT		__SLAB_FLAG_UNUSED
+#endif
+
+#define SLAB_NO_SHEAVES		__SLAB_FLAG_BIT(_SLAB_NO_SHEAVES)
 
 #if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT)
 #define SLAB_OBJ_EXT_IN_OBJ	__SLAB_FLAG_BIT(_SLAB_OBJ_EXT_IN_OBJ)
@@ -699,6 +708,9 @@ enum kmalloc_cache_type {
 #endif
 #ifndef CONFIG_MEMCG
 	KMALLOC_CGROUP = KMALLOC_NORMAL,
+#endif
+#ifndef CONFIG_SLAB_OBJ_EXT
+	KMALLOC_NO_OBJ_EXT = KMALLOC_NORMAL,
 #endif
 	KMALLOC_PARTITION_START = KMALLOC_NORMAL,
 	KMALLOC_PARTITION_END = KMALLOC_PARTITION_START + KMALLOC_PARTITION_CACHES_NR,
@@ -712,6 +724,9 @@ enum kmalloc_cache_type {
 #endif
 #ifdef CONFIG_MEMCG
 	KMALLOC_CGROUP,
+#endif
+#ifdef CONFIG_SLAB_OBJ_EXT
+	KMALLOC_NO_OBJ_EXT,
 #endif
 	NR_KMALLOC_TYPES
 };
diff --git a/include/linux/soc/pxa/cpu.h b/include/linux/soc/pxa/cpu.h
index 5782450ee45c4..38bacdae684f9 100644
--- a/include/linux/soc/pxa/cpu.h
+++ b/include/linux/soc/pxa/cpu.h
@@ -46,14 +46,6 @@
  *  PXA31x	A2	0x69056892	0x2E649013
  *  PXA32x	B1	0x69056825	0x5E642013
  *  PXA32x	B2	0x69056826	0x6E642013
- *
- *  PXA930	B0	0x69056835	0x5E643013
- *  PXA930	B1	0x69056837	0x7E643013
- *  PXA930	B2	0x69056838	0x8E643013
- *
- *  PXA935	A0	0x56056931	0x1E653013
- *  PXA935	B0	0x56056936	0x6E653013
- *  PXA935	B1	0x56056938	0x8E653013
  */
 #ifdef CONFIG_PXA25x
 #define __cpu_is_pxa210(id)				\
@@ -126,26 +118,6 @@
 #define __cpu_is_pxa320(id)	(0)
 #endif
 
-#ifdef CONFIG_CPU_PXA930
-#define __cpu_is_pxa930(id)				\
-	({						\
-		unsigned int _id = (id) >> 4 & 0xfff;	\
-		_id == 0x683;				\
-	 })
-#else
-#define __cpu_is_pxa930(id)	(0)
-#endif
-
-#ifdef CONFIG_CPU_PXA935
-#define __cpu_is_pxa935(id)				\
-	({						\
-		unsigned int _id = (id) >> 4 & 0xfff;	\
-		_id == 0x693;				\
-	 })
-#else
-#define __cpu_is_pxa935(id)	(0)
-#endif
-
 #define cpu_is_pxa210()					\
 	({						\
 		__cpu_is_pxa210(read_cpuid_id());	\
@@ -186,18 +158,6 @@
 		__cpu_is_pxa320(read_cpuid_id());	\
 	 })
 
-#define cpu_is_pxa930()					\
-	({						\
-		__cpu_is_pxa930(read_cpuid_id());	\
-	 })
-
-#define cpu_is_pxa935()					\
-	({						\
-		__cpu_is_pxa935(read_cpuid_id());	\
-	 })
-
-
-
 /*
  * CPUID Core Generation Bit
  * <= 0x2 for pxa21x/pxa25x/pxa26x/pxa27x
@@ -218,22 +178,11 @@
 		__cpu_is_pxa300(id)			\
 			|| __cpu_is_pxa310(id)		\
 			|| __cpu_is_pxa320(id)		\
-			|| __cpu_is_pxa93x(id);		\
 	 })
 #else
 #define __cpu_is_pxa3xx(id)	(0)
 #endif
 
-#if defined(CONFIG_CPU_PXA930) || defined(CONFIG_CPU_PXA935)
-#define __cpu_is_pxa93x(id)				\
-	({						\
-		__cpu_is_pxa930(id)			\
-			|| __cpu_is_pxa935(id);		\
-	 })
-#else
-#define __cpu_is_pxa93x(id)	(0)
-#endif
-
 #define cpu_is_pxa2xx()					\
 	({						\
 		__cpu_is_pxa2xx(read_cpuid_id());	\
@@ -244,9 +193,4 @@
 		__cpu_is_pxa3xx(read_cpuid_id());	\
 	 })
 
-#define cpu_is_pxa93x()					\
-	({						\
-		__cpu_is_pxa93x(read_cpuid_id());	\
-	 })
-
 #endif
diff --git a/include/linux/soc/qcom/mdt_loader.h b/include/linux/soc/qcom/mdt_loader.h
index 82372e0db0a18..142409555425f 100644
--- a/include/linux/soc/qcom/mdt_loader.h
+++ b/include/linux/soc/qcom/mdt_loader.h
@@ -10,7 +10,7 @@
 
 struct device;
 struct firmware;
-struct qcom_scm_pas_context;
+struct qcom_pas_context;
 
 #if IS_ENABLED(CONFIG_QCOM_MDT_LOADER)
 
@@ -20,7 +20,7 @@ int qcom_mdt_load(struct device *dev, const struct firmware *fw,
 		  phys_addr_t mem_phys, size_t mem_size,
 		  phys_addr_t *reloc_base);
 
-int qcom_mdt_pas_load(struct qcom_scm_pas_context *ctx, const struct firmware *fw,
+int qcom_mdt_pas_load(struct qcom_pas_context *ctx, const struct firmware *fw,
 		      const char *firmware, void *mem_region, phys_addr_t *reloc_base);
 
 int qcom_mdt_load_no_init(struct device *dev, const struct firmware *fw,
@@ -45,7 +45,7 @@ static inline int qcom_mdt_load(struct device *dev, const struct firmware *fw,
 	return -ENODEV;
 }
 
-static inline int qcom_mdt_pas_load(struct qcom_scm_pas_context *ctx,
+static inline int qcom_mdt_pas_load(struct qcom_pas_context *ctx,
 				    const struct firmware *fw, const char *firmware,
 				    void *mem_region, phys_addr_t *reloc_base)
 {
diff --git a/include/linux/soc/qcom/ubwc.h b/include/linux/soc/qcom/ubwc.h
index 83d2c2a7116c0..a7372d9c25fbc 100644
--- a/include/linux/soc/qcom/ubwc.h
+++ b/include/linux/soc/qcom/ubwc.h
@@ -13,20 +13,6 @@
 
 struct qcom_ubwc_cfg_data {
 	u32 ubwc_enc_version;
-	/* Can be read from MDSS_BASE + 0x58 */
-	u32 ubwc_dec_version;
-
-	/**
-	 * @ubwc_swizzle: Whether to enable level 1, 2 & 3 bank swizzling.
-	 *
-	 * UBWC 1.0 always enables all three levels.
-	 * UBWC 2.0 removes level 1 bank swizzling, leaving levels 2 & 3.
-	 * UBWC 4.0 adds the optional ability to disable levels 2 & 3.
-	 */
-	u32 ubwc_swizzle;
-#define UBWC_SWIZZLE_ENABLE_LVL1	BIT(0)
-#define UBWC_SWIZZLE_ENABLE_LVL2	BIT(1)
-#define UBWC_SWIZZLE_ENABLE_LVL3	BIT(2)
 
 	/**
 	 * @highest_bank_bit: Highest Bank Bit
@@ -35,16 +21,10 @@ struct qcom_ubwc_cfg_data {
 	 * DDR bank.  This should ideally use DRAM type detection.
 	 */
 	int highest_bank_bit;
-	bool ubwc_bank_spread;
 
-	/**
-	 * @macrotile_mode: Macrotile Mode
-	 *
-	 * Whether to use 4-channel macrotiling mode or the newer
-	 * 8-channel macrotiling mode introduced in UBWC 3.1. 0 is
-	 * 4-channel and 1 is 8-channel.
-	 */
-	bool macrotile_mode;
+	unsigned int flags;
+#define UBWC_FLAG_DISABLE_SWIZZLE_LVL2	BIT(0)
+#define UBWC_FLAG_DISABLE_SWIZZLE_LVL3	BIT(1)
 };
 
 #define UBWC_1_0 0x10000000
@@ -67,37 +47,62 @@ static inline const struct qcom_ubwc_cfg_data *qcom_ubwc_config_get_data(void)
 
 static inline bool qcom_ubwc_get_ubwc_mode(const struct qcom_ubwc_cfg_data *cfg)
 {
-	bool ret = cfg->ubwc_enc_version == UBWC_1_0;
-
-	if (ret && !(cfg->ubwc_swizzle & UBWC_SWIZZLE_ENABLE_LVL1))
-		pr_err("UBWC config discrepancy - level 1 swizzling disabled on UBWC 1.0\n");
-
-	return ret;
+	return cfg->ubwc_enc_version == UBWC_1_0;
 }
 
-/*
- * This is the best guess, based on the MDSS driver, which worked so far.
- */
 static inline bool qcom_ubwc_min_acc_length_64b(const struct qcom_ubwc_cfg_data *cfg)
 {
-	return cfg->ubwc_enc_version == UBWC_1_0 &&
-		(cfg->ubwc_dec_version == UBWC_2_0 ||
-		 cfg->ubwc_dec_version == UBWC_3_0);
+	return cfg->ubwc_enc_version == UBWC_1_0;
 }
 
+/*
+ * @qcom_ubwc_macrotile_mode: whether to use 4-channel or 8-channel macrotiling
+ *
+ * The 8-channel macrotiling mode was introduced in UBWC 3.1.
+ *
+ * Returns: false for the 4-channel and true for 8-channel.
+ */
 static inline bool qcom_ubwc_macrotile_mode(const struct qcom_ubwc_cfg_data *cfg)
 {
-	return cfg->macrotile_mode;
+	return cfg->ubwc_enc_version >= UBWC_3_1;
 }
 
 static inline bool qcom_ubwc_bank_spread(const struct qcom_ubwc_cfg_data *cfg)
 {
-	return cfg->ubwc_bank_spread;
+	return true;
 }
 
+#define UBWC_SWIZZLE_ENABLE_LVL1	BIT(0)
+#define UBWC_SWIZZLE_ENABLE_LVL2	BIT(1)
+#define UBWC_SWIZZLE_ENABLE_LVL3	BIT(2)
+
+/**
+ * @qcom_ubwc_swizzle: Whether to enable level 1, 2 & 3 bank swizzling.
+ *
+ * UBWC 1.0 always enables all three levels.
+ * UBWC 2.0 removes level 1 bank swizzling, leaving levels 2 & 3.
+ * UBWC 4.0 adds the optional ability to disable levels 2 & 3.
+ */
 static inline u32 qcom_ubwc_swizzle(const struct qcom_ubwc_cfg_data *cfg)
 {
-	return cfg->ubwc_swizzle;
+	if (cfg->ubwc_enc_version == 0)
+		return 0;
+
+	if (cfg->ubwc_enc_version == UBWC_1_0)
+		return UBWC_SWIZZLE_ENABLE_LVL1 |
+		       UBWC_SWIZZLE_ENABLE_LVL2 |
+		       UBWC_SWIZZLE_ENABLE_LVL3;
+
+	u32 ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 |
+			   UBWC_SWIZZLE_ENABLE_LVL3;
+
+	if (cfg->flags & UBWC_FLAG_DISABLE_SWIZZLE_LVL2)
+		ubwc_swizzle &= ~UBWC_SWIZZLE_ENABLE_LVL2;
+
+	if (cfg->flags & UBWC_FLAG_DISABLE_SWIZZLE_LVL3)
+		ubwc_swizzle &= ~UBWC_SWIZZLE_ENABLE_LVL3;
+
+	return ubwc_swizzle;
 }
 
 static inline u32 qcom_ubwc_version_tag(const struct qcom_ubwc_cfg_data *cfg)
diff --git a/include/linux/ssb/ssb.h b/include/linux/ssb/ssb.h
index 7fee9afa94588..67cb66f6f5ed6 100644
--- a/include/linux/ssb/ssb.h
+++ b/include/linux/ssb/ssb.h
@@ -671,4 +671,6 @@ int ssb_pcibios_plat_dev_init(struct pci_dev *dev);
 int ssb_pcibios_map_irq(const struct pci_dev *dev, u8 slot, u8 pin);
 #endif /* CONFIG_SSB_EMBEDDED */
 
+extern const struct software_node ssb_gpio_swnode;
+
 #endif /* LINUX_SSB_H_ */
diff --git a/include/linux/sunrpc/bc_xprt.h b/include/linux/sunrpc/bc_xprt.h
index 98939cb664cfa..59d0cc889beba 100644
--- a/include/linux/sunrpc/bc_xprt.h
+++ b/include/linux/sunrpc/bc_xprt.h
@@ -32,6 +32,7 @@ int xprt_setup_bc(struct rpc_xprt *xprt, unsigned int min_reqs);
 void xprt_destroy_bc(struct rpc_xprt *xprt, unsigned int max_reqs);
 void xprt_free_bc_rqst(struct rpc_rqst *req);
 unsigned int xprt_bc_max_slots(struct rpc_xprt *xprt);
+void xprt_svc_shutdown_bc(struct rpc_xprt *xprt);
 void xprt_svc_destroy_nullify_bc(struct rpc_xprt *xprt, struct svc_serv **serv);
 
 /*
@@ -71,6 +72,10 @@ static inline void xprt_free_bc_request(struct rpc_rqst *req)
 {
 }
 
+static inline void xprt_svc_shutdown_bc(struct rpc_xprt *xprt)
+{
+}
+
 static inline void xprt_svc_destroy_nullify_bc(struct rpc_xprt *xprt, struct svc_serv **serv)
 {
 	svc_destroy(serv);
diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
index 4be6204f66305..3a0152d926fb6 100644
--- a/include/linux/sunrpc/svc.h
+++ b/include/linux/sunrpc/svc.h
@@ -469,6 +469,7 @@ int		   svc_set_pool_threads(struct svc_serv *serv, struct svc_pool *pool,
 					unsigned int min_threads, unsigned int max_threads);
 int		   svc_set_num_threads(struct svc_serv *serv, unsigned int min_threads,
 				       unsigned int nrservs);
+unsigned int	   svc_serv_maxthreads(const struct svc_serv *serv);
 int		   svc_pool_stats_open(struct svc_info *si, struct file *file);
 void		   svc_process(struct svc_rqst *rqstp);
 void		   svc_process_bc(struct rpc_rqst *req, struct svc_rqst *rqstp);
diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
index 5aadb47b3b0e9..76aa5ec4ab402 100644
--- a/include/linux/sunrpc/svc_rdma.h
+++ b/include/linux/sunrpc/svc_rdma.h
@@ -77,8 +77,8 @@ struct svcxprt_rdma {
 	struct rdma_cm_id    *sc_cm_id;		/* RDMA connection id */
 	struct list_head     sc_accept_q;	/* Conn. waiting accept */
 	struct rpcrdma_notification sc_rn;	/* removal notification */
-	int		     sc_ord;		/* RDMA read limit */
-	int                  sc_max_send_sges;
+	u32		     sc_ord;		/* RDMA read limit */
+	unsigned int         sc_max_send_sges;
 	bool		     sc_snd_w_inv;	/* OK to use Send With Invalidate */
 
 	atomic_t             sc_sq_avail;	/* SQEs ready to be consumed */
diff --git a/include/linux/sunrpc/svc_rdma_pcl.h b/include/linux/sunrpc/svc_rdma_pcl.h
index 7516ad0fae808..6346d8cf25873 100644
--- a/include/linux/sunrpc/svc_rdma_pcl.h
+++ b/include/linux/sunrpc/svc_rdma_pcl.h
@@ -97,7 +97,7 @@ pcl_next_chunk(const struct svc_rdma_pcl *pcl, struct svc_rdma_chunk *chunk)
  */
 #define pcl_for_each_segment(pos, chunk) \
 	for (pos = &(chunk)->ch_segments[0]; \
-	     pos <= &(chunk)->ch_segments[(chunk)->ch_segcount - 1]; \
+	     pos < &(chunk)->ch_segments[(chunk)->ch_segcount]; \
 	     pos++)
 
 /**
@@ -119,6 +119,8 @@ extern bool pcl_alloc_call(struct svc_rdma_recv_ctxt *rctxt, __be32 *p);
 extern bool pcl_alloc_read(struct svc_rdma_recv_ctxt *rctxt, __be32 *p);
 extern bool pcl_alloc_write(struct svc_rdma_recv_ctxt *rctxt,
 			    struct svc_rdma_pcl *pcl, __be32 *p);
+extern bool pcl_check_read_chunk_positions(struct svc_rdma_recv_ctxt *rctxt,
+					   unsigned int inline_len);
 extern int pcl_process_nonpayloads(const struct svc_rdma_pcl *pcl,
 				   const struct xdr_buf *xdr,
 				   int (*actor)(const struct xdr_buf *,
diff --git a/include/linux/sunrpc/xdrgen/nfs4_1.h b/include/linux/sunrpc/xdrgen/nfs4_1.h
index 4ac54bdbd335b..356c3da9f4e09 100644
--- a/include/linux/sunrpc/xdrgen/nfs4_1.h
+++ b/include/linux/sunrpc/xdrgen/nfs4_1.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /* Generated by xdrgen. Manual edits will be lost. */
 /* XDR specification file: ../../Documentation/sunrpc/xdr/nfs4_1.x */
-/* XDR specification modification time: Thu Jan  8 23:12:07 2026 */
+/* XDR specification modification time: Wed Mar 25 11:40:02 2026 */
 
 #ifndef _LINUX_XDRGEN_NFS4_1_DEF_H
 #define _LINUX_XDRGEN_NFS4_1_DEF_H
@@ -9,15 +9,149 @@
 #include <linux/types.h>
 #include <linux/sunrpc/xdrgen/_defs.h>
 
-typedef s64 int64_t;
+typedef s32 int32_t;
 
 typedef u32 uint32_t;
 
+typedef s64 int64_t;
+
+typedef u64 uint64_t;
+
+enum { NFS4_VERIFIER_SIZE = 8 };
+
+enum { NFS4_FHSIZE = 128 };
+
+enum nfsstat4 {
+	NFS4_OK = 0,
+	NFS4ERR_PERM = 1,
+	NFS4ERR_NOENT = 2,
+	NFS4ERR_IO = 5,
+	NFS4ERR_NXIO = 6,
+	NFS4ERR_ACCESS = 13,
+	NFS4ERR_EXIST = 17,
+	NFS4ERR_XDEV = 18,
+	NFS4ERR_NOTDIR = 20,
+	NFS4ERR_ISDIR = 21,
+	NFS4ERR_INVAL = 22,
+	NFS4ERR_FBIG = 27,
+	NFS4ERR_NOSPC = 28,
+	NFS4ERR_ROFS = 30,
+	NFS4ERR_MLINK = 31,
+	NFS4ERR_NAMETOOLONG = 63,
+	NFS4ERR_NOTEMPTY = 66,
+	NFS4ERR_DQUOT = 69,
+	NFS4ERR_STALE = 70,
+	NFS4ERR_BADHANDLE = 10001,
+	NFS4ERR_BAD_COOKIE = 10003,
+	NFS4ERR_NOTSUPP = 10004,
+	NFS4ERR_TOOSMALL = 10005,
+	NFS4ERR_SERVERFAULT = 10006,
+	NFS4ERR_BADTYPE = 10007,
+	NFS4ERR_DELAY = 10008,
+	NFS4ERR_SAME = 10009,
+	NFS4ERR_DENIED = 10010,
+	NFS4ERR_EXPIRED = 10011,
+	NFS4ERR_LOCKED = 10012,
+	NFS4ERR_GRACE = 10013,
+	NFS4ERR_FHEXPIRED = 10014,
+	NFS4ERR_SHARE_DENIED = 10015,
+	NFS4ERR_WRONGSEC = 10016,
+	NFS4ERR_CLID_INUSE = 10017,
+	NFS4ERR_RESOURCE = 10018,
+	NFS4ERR_MOVED = 10019,
+	NFS4ERR_NOFILEHANDLE = 10020,
+	NFS4ERR_MINOR_VERS_MISMATCH = 10021,
+	NFS4ERR_STALE_CLIENTID = 10022,
+	NFS4ERR_STALE_STATEID = 10023,
+	NFS4ERR_OLD_STATEID = 10024,
+	NFS4ERR_BAD_STATEID = 10025,
+	NFS4ERR_BAD_SEQID = 10026,
+	NFS4ERR_NOT_SAME = 10027,
+	NFS4ERR_LOCK_RANGE = 10028,
+	NFS4ERR_SYMLINK = 10029,
+	NFS4ERR_RESTOREFH = 10030,
+	NFS4ERR_LEASE_MOVED = 10031,
+	NFS4ERR_ATTRNOTSUPP = 10032,
+	NFS4ERR_NO_GRACE = 10033,
+	NFS4ERR_RECLAIM_BAD = 10034,
+	NFS4ERR_RECLAIM_CONFLICT = 10035,
+	NFS4ERR_BADXDR = 10036,
+	NFS4ERR_LOCKS_HELD = 10037,
+	NFS4ERR_OPENMODE = 10038,
+	NFS4ERR_BADOWNER = 10039,
+	NFS4ERR_BADCHAR = 10040,
+	NFS4ERR_BADNAME = 10041,
+	NFS4ERR_BAD_RANGE = 10042,
+	NFS4ERR_LOCK_NOTSUPP = 10043,
+	NFS4ERR_OP_ILLEGAL = 10044,
+	NFS4ERR_DEADLOCK = 10045,
+	NFS4ERR_FILE_OPEN = 10046,
+	NFS4ERR_ADMIN_REVOKED = 10047,
+	NFS4ERR_CB_PATH_DOWN = 10048,
+	NFS4ERR_BADIOMODE = 10049,
+	NFS4ERR_BADLAYOUT = 10050,
+	NFS4ERR_BAD_SESSION_DIGEST = 10051,
+	NFS4ERR_BADSESSION = 10052,
+	NFS4ERR_BADSLOT = 10053,
+	NFS4ERR_COMPLETE_ALREADY = 10054,
+	NFS4ERR_CONN_NOT_BOUND_TO_SESSION = 10055,
+	NFS4ERR_DELEG_ALREADY_WANTED = 10056,
+	NFS4ERR_BACK_CHAN_BUSY = 10057,
+	NFS4ERR_LAYOUTTRYLATER = 10058,
+	NFS4ERR_LAYOUTUNAVAILABLE = 10059,
+	NFS4ERR_NOMATCHING_LAYOUT = 10060,
+	NFS4ERR_RECALLCONFLICT = 10061,
+	NFS4ERR_UNKNOWN_LAYOUTTYPE = 10062,
+	NFS4ERR_SEQ_MISORDERED = 10063,
+	NFS4ERR_SEQUENCE_POS = 10064,
+	NFS4ERR_REQ_TOO_BIG = 10065,
+	NFS4ERR_REP_TOO_BIG = 10066,
+	NFS4ERR_REP_TOO_BIG_TO_CACHE = 10067,
+	NFS4ERR_RETRY_UNCACHED_REP = 10068,
+	NFS4ERR_UNSAFE_COMPOUND = 10069,
+	NFS4ERR_TOO_MANY_OPS = 10070,
+	NFS4ERR_OP_NOT_IN_SESSION = 10071,
+	NFS4ERR_HASH_ALG_UNSUPP = 10072,
+	NFS4ERR_CLIENTID_BUSY = 10074,
+	NFS4ERR_PNFS_IO_HOLE = 10075,
+	NFS4ERR_SEQ_FALSE_RETRY = 10076,
+	NFS4ERR_BAD_HIGH_SLOT = 10077,
+	NFS4ERR_DEADSESSION = 10078,
+	NFS4ERR_ENCR_ALG_UNSUPP = 10079,
+	NFS4ERR_PNFS_NO_LAYOUT = 10080,
+	NFS4ERR_NOT_ONLY_OP = 10081,
+	NFS4ERR_WRONG_CRED = 10082,
+	NFS4ERR_WRONG_TYPE = 10083,
+	NFS4ERR_DIRDELEG_UNAVAIL = 10084,
+	NFS4ERR_REJECT_DELEG = 10085,
+	NFS4ERR_RETURNCONFLICT = 10086,
+	NFS4ERR_DELEG_REVOKED = 10087,
+	NFS4ERR_PARTNER_NOTSUPP = 10088,
+	NFS4ERR_PARTNER_NO_AUTH = 10089,
+	NFS4ERR_UNION_NOTSUPP = 10090,
+	NFS4ERR_OFFLOAD_DENIED = 10091,
+	NFS4ERR_WRONG_LFS = 10092,
+	NFS4ERR_BADLABEL = 10093,
+	NFS4ERR_OFFLOAD_NO_REQS = 10094,
+	NFS4ERR_NOXATTR = 10095,
+	NFS4ERR_XATTR2BIG = 10096,
+};
+
+typedef enum nfsstat4 nfsstat4;
+
+typedef opaque attrlist4;
+
 typedef struct {
 	u32 count;
 	uint32_t *element;
 } bitmap4;
 
+typedef u8 verifier4[NFS4_VERIFIER_SIZE];
+
+typedef uint64_t nfs_cookie4;
+
+typedef opaque nfs_fh4;
+
 typedef opaque utf8string;
 
 typedef utf8string utf8str_cis;
@@ -26,11 +160,30 @@ typedef utf8string utf8str_cs;
 
 typedef utf8string utf8str_mixed;
 
+typedef utf8str_cs component4;
+
+typedef utf8str_cs linktext4;
+
+typedef struct {
+	u32 count;
+	component4 *element;
+} pathname4;
+
 struct nfstime4 {
 	int64_t seconds;
 	uint32_t nseconds;
 };
 
+struct fattr4 {
+	bitmap4 attrmask;
+	attrlist4 attr_vals;
+};
+
+struct stateid4 {
+	uint32_t seqid;
+	u8 other[12];
+};
+
 typedef bool fattr4_offline;
 
 enum { FATTR4_OFFLINE = 83 };
@@ -216,11 +369,109 @@ enum { FATTR4_POSIX_DEFAULT_ACL = 91 };
 
 enum { FATTR4_POSIX_ACCESS_ACL = 92 };
 
-#define NFS4_int64_t_sz                 \
-	(XDR_hyper)
+enum notify_type4 {
+	NOTIFY4_CHANGE_CHILD_ATTRS = 0,
+	NOTIFY4_CHANGE_DIR_ATTRS = 1,
+	NOTIFY4_REMOVE_ENTRY = 2,
+	NOTIFY4_ADD_ENTRY = 3,
+	NOTIFY4_RENAME_ENTRY = 4,
+	NOTIFY4_CHANGE_COOKIE_VERIFIER = 5,
+	NOTIFY4_GFLAG_EXTEND = 6,
+	NOTIFY4_AUFLAG_VALID = 7,
+	NOTIFY4_AUFLAG_USER = 8,
+	NOTIFY4_AUFLAG_GROUP = 9,
+	NOTIFY4_AUFLAG_OTHER = 10,
+	NOTIFY4_CHANGE_AUTH = 11,
+	NOTIFY4_CFLAG_ORDER = 12,
+	NOTIFY4_AUFLAG_GANOW = 13,
+	NOTIFY4_AUFLAG_GALATER = 14,
+	NOTIFY4_CHANGE_GA = 15,
+	NOTIFY4_CHANGE_AMASK = 16,
+};
+
+typedef enum notify_type4 notify_type4;
+
+struct notify_entry4 {
+	component4 ne_file;
+	struct fattr4 ne_attrs;
+};
+
+struct prev_entry4 {
+	struct notify_entry4 pe_prev_entry;
+	nfs_cookie4 pe_prev_entry_cookie;
+};
+
+struct notify_remove4 {
+	struct notify_entry4 nrm_old_entry;
+	nfs_cookie4 nrm_old_entry_cookie;
+};
+
+struct notify_add4 {
+	struct {
+		u32 count;
+		struct notify_remove4 *element;
+	} nad_old_entry;
+	struct notify_entry4 nad_new_entry;
+	struct {
+		u32 count;
+		nfs_cookie4 *element;
+	} nad_new_entry_cookie;
+	struct {
+		u32 count;
+		struct prev_entry4 *element;
+	} nad_prev_entry;
+	bool nad_last_entry;
+};
+
+struct notify_attr4 {
+	struct notify_entry4 na_changed_entry;
+};
+
+struct notify_rename4 {
+	struct notify_remove4 nrn_old_entry;
+	struct notify_add4 nrn_new_entry;
+};
+
+struct notify_verifier4 {
+	verifier4 nv_old_cookieverf;
+	verifier4 nv_new_cookieverf;
+};
+
+typedef opaque notifylist4;
+
+struct notify4 {
+	bitmap4 notify_mask;
+	notifylist4 notify_vals;
+};
+
+struct CB_NOTIFY4args {
+	struct stateid4 cna_stateid;
+	nfs_fh4 cna_fh;
+	struct {
+		u32 count;
+		struct notify4 *element;
+	} cna_changes;
+};
+
+struct CB_NOTIFY4res {
+	nfsstat4 cnr_status;
+};
+
+#define NFS4_int32_t_sz                 \
+	(XDR_int)
 #define NFS4_uint32_t_sz                \
 	(XDR_unsigned_int)
+#define NFS4_int64_t_sz                 \
+	(XDR_hyper)
+#define NFS4_uint64_t_sz                \
+	(XDR_unsigned_hyper)
+#define NFS4_nfsstat4_sz                (XDR_int)
+#define NFS4_attrlist4_sz               (XDR_unsigned_int)
 #define NFS4_bitmap4_sz                 (XDR_unsigned_int)
+#define NFS4_verifier4_sz               (XDR_QUADLEN(NFS4_VERIFIER_SIZE))
+#define NFS4_nfs_cookie4_sz             \
+	(NFS4_uint64_t_sz)
+#define NFS4_nfs_fh4_sz                 (XDR_unsigned_int + XDR_QUADLEN(NFS4_FHSIZE))
 #define NFS4_utf8string_sz              (XDR_unsigned_int)
 #define NFS4_utf8str_cis_sz             \
 	(NFS4_utf8string_sz)
@@ -228,8 +479,17 @@ enum { FATTR4_POSIX_ACCESS_ACL = 92 };
 	(NFS4_utf8string_sz)
 #define NFS4_utf8str_mixed_sz           \
 	(NFS4_utf8string_sz)
+#define NFS4_component4_sz              \
+	(NFS4_utf8str_cs_sz)
+#define NFS4_linktext4_sz               \
+	(NFS4_utf8str_cs_sz)
+#define NFS4_pathname4_sz               (XDR_unsigned_int)
 #define NFS4_nfstime4_sz                \
 	(NFS4_int64_t_sz + NFS4_uint32_t_sz)
+#define NFS4_fattr4_sz                  \
+	(NFS4_bitmap4_sz + NFS4_attrlist4_sz)
+#define NFS4_stateid4_sz                \
+	(NFS4_uint32_t_sz + XDR_QUADLEN(12))
 #define NFS4_fattr4_offline_sz          \
 	(XDR_bool)
 #define NFS4_open_arguments4_sz         \
@@ -259,5 +519,27 @@ enum { FATTR4_POSIX_ACCESS_ACL = 92 };
 	(NFS4_aclscope4_sz)
 #define NFS4_fattr4_posix_default_acl_sz (XDR_unsigned_int)
 #define NFS4_fattr4_posix_access_acl_sz (XDR_unsigned_int)
+#define NFS4_notify_type4_sz            (XDR_int)
+#define NFS4_notify_entry4_sz           \
+	(NFS4_component4_sz + NFS4_fattr4_sz)
+#define NFS4_prev_entry4_sz             \
+	(NFS4_notify_entry4_sz + NFS4_nfs_cookie4_sz)
+#define NFS4_notify_remove4_sz          \
+	(NFS4_notify_entry4_sz + NFS4_nfs_cookie4_sz)
+#define NFS4_notify_add4_sz             \
+	(XDR_unsigned_int + (1 * (NFS4_notify_remove4_sz)) + NFS4_notify_entry4_sz + XDR_unsigned_int + (1 * (NFS4_nfs_cookie4_sz)) + XDR_unsigned_int + (1 * (NFS4_prev_entry4_sz)) + XDR_bool)
+#define NFS4_notify_attr4_sz            \
+	(NFS4_notify_entry4_sz)
+#define NFS4_notify_rename4_sz          \
+	(NFS4_notify_remove4_sz + NFS4_notify_add4_sz)
+#define NFS4_notify_verifier4_sz        \
+	(NFS4_verifier4_sz + NFS4_verifier4_sz)
+#define NFS4_notifylist4_sz             (XDR_unsigned_int)
+#define NFS4_notify4_sz                 \
+	(NFS4_bitmap4_sz + NFS4_notifylist4_sz)
+#define NFS4_CB_NOTIFY4args_sz          \
+	(NFS4_stateid4_sz + NFS4_nfs_fh4_sz + XDR_unsigned_int)
+#define NFS4_CB_NOTIFY4res_sz           \
+	(NFS4_nfsstat4_sz)
 
 #endif /* _LINUX_XDRGEN_NFS4_1_DEF_H */
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 8f0f68e245baa..0544b2ec4c565 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -16,10 +16,6 @@
 #include <uapi/linux/mempolicy.h>
 #include <asm/page.h>
 
-struct notifier_block;
-
-struct bio;
-
 #define SWAP_FLAG_PREFER	0x8000	/* set if swap priority specified */
 #define SWAP_FLAG_PRIO_MASK	0x7fff
 #define SWAP_FLAG_DISCARD	0x10000 /* enable discard for swap */
@@ -29,7 +25,6 @@ struct bio;
 #define SWAP_FLAGS_VALID	(SWAP_FLAG_PRIO_MASK | SWAP_FLAG_PREFER | \
 				 SWAP_FLAG_DISCARD | SWAP_FLAG_DISCARD_ONCE | \
 				 SWAP_FLAG_DISCARD_PAGES)
-#define SWAP_BATCH 64
 
 static inline int current_is_kswapd(void)
 {
@@ -175,7 +170,6 @@ static inline void mm_account_reclaimed_pages(unsigned long pages)
 
 struct address_space;
 struct sysinfo;
-struct writeback_control;
 struct zone;
 
 /*
@@ -208,7 +202,6 @@ enum {
 	SWP_SOLIDSTATE	= (1 << 4),	/* blkdev seeks are cheap */
 	SWP_BLKDEV	= (1 << 6),	/* its a block device */
 	SWP_ACTIVATED	= (1 << 7),	/* set after swap_activate success */
-	SWP_FS_OPS	= (1 << 8),	/* swapfile operations go through fs */
 	SWP_AREA_DISCARD = (1 << 9),	/* single-time swap area discards */
 	SWP_PAGE_DISCARD = (1 << 10),	/* freed swap page-cluster discards */
 	SWP_STABLE_WRITES = (1 << 11),	/* no overwrite PG_writeback pages */
@@ -282,6 +275,7 @@ struct swap_info_struct {
 	struct work_struct reclaim_work; /* reclaim worker */
 	struct list_head discard_clusters; /* discard clusters list */
 	struct plist_node avail_list;   /* entry in swap_avail_head */
+	const struct swap_ops *ops;
 };
 
 static inline swp_entry_t page_swap_entry(struct page *page)
@@ -293,39 +287,19 @@ static inline swp_entry_t page_swap_entry(struct page *page)
 	return entry;
 }
 
-/* linux/mm/workingset.c */
-bool workingset_test_recent(void *shadow, bool file, bool *workingset,
-				bool flush);
-void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages);
-void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg);
-void workingset_refault(struct folio *folio, void *shadow);
-void workingset_activation(struct folio *folio);
-
 /* linux/mm/page_alloc.c */
 extern unsigned long totalreserve_pages;
 
 /* Definition of global_zone_page_state not available yet */
 #define nr_free_pages() global_zone_page_state(NR_FREE_PAGES)
 
+/* linux/mm/folio.c */
+void folio_add_lru(struct folio *folio);
+void folio_mark_accessed(struct folio *folio);
+void lru_add_drain_all(void);
 
-/* linux/mm/swap.c */
-void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file,
-		unsigned int nr_io, unsigned int nr_rotated);
-void lru_note_cost_refault(struct folio *);
-void folio_add_lru(struct folio *);
-void folio_add_lru_vma(struct folio *, struct vm_area_struct *);
-void mark_page_accessed(struct page *);
-void folio_mark_accessed(struct folio *);
-
-static inline bool folio_may_be_lru_cached(struct folio *folio)
-{
-	/*
-	 * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting.
-	 * Holding small numbers of low-order mTHP folios in per-CPU LRU cache
-	 * will be sensible, but nobody has implemented and tested that yet.
-	 */
-	return !folio_test_large(folio);
-}
+/* linux/mm/folio-compat.c */
+void mark_page_accessed(struct page *page);
 
 extern atomic_t lru_disable_count;
 
@@ -334,43 +308,6 @@ static inline bool lru_cache_disabled(void)
 	return atomic_read(&lru_disable_count);
 }
 
-static inline void lru_cache_enable(void)
-{
-	atomic_dec(&lru_disable_count);
-}
-
-extern void lru_cache_disable(void);
-extern void lru_add_drain(void);
-extern void lru_add_drain_cpu(int cpu);
-extern void lru_add_drain_cpu_zone(struct zone *zone);
-extern void lru_add_drain_all(void);
-void folio_deactivate(struct folio *folio);
-void folio_mark_lazyfree(struct folio *folio);
-extern void swap_setup(void);
-
-/* linux/mm/vmscan.c */
-extern unsigned long zone_reclaimable_pages(struct zone *zone);
-extern unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
-					gfp_t gfp_mask, nodemask_t *mask);
-unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx);
-
-#define MEMCG_RECLAIM_MAY_SWAP (1 << 1)
-#define MEMCG_RECLAIM_PROACTIVE (1 << 2)
-#define MIN_SWAPPINESS 0
-#define MAX_SWAPPINESS 200
-
-/* Just reclaim from anon folios in proactive memory reclaim */
-#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1)
-
-extern unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
-						  unsigned long nr_pages,
-						  gfp_t gfp_mask,
-						  unsigned int reclaim_options,
-						  int *swappiness);
-extern unsigned long mem_cgroup_shrink_node(struct mem_cgroup *mem,
-						gfp_t gfp_mask, bool noswap,
-						pg_data_t *pgdat,
-						unsigned long *nr_scanned);
 extern unsigned long shrink_all_memory(unsigned long nr_pages);
 extern int vm_swappiness;
 long remove_mapping(struct address_space *mapping, struct folio *folio);
@@ -391,11 +328,6 @@ static inline void reclaim_unregister_node(struct node *node)
 }
 #endif /* CONFIG_SYSFS && CONFIG_NUMA */
 
-#ifdef CONFIG_NUMA
-extern int sysctl_min_unmapped_ratio;
-extern int sysctl_min_slab_ratio;
-#endif
-
 void check_move_unevictable_folios(struct folio_batch *fbatch);
 
 extern void __meminit kswapd_run(int nid);
@@ -403,6 +335,7 @@ extern void __meminit kswapd_stop(int nid);
 
 #ifdef CONFIG_SWAP
 
+int swap_fs_activate(struct swap_info_struct *sis);
 int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
 		unsigned long nr_pages, sector_t start_block);
 int generic_swapfile_activate(struct swap_info_struct *, struct file *,
@@ -442,7 +375,6 @@ extern sector_t swapdev_block(int, pgoff_t);
 extern int __swap_count(swp_entry_t entry);
 extern bool swap_entry_swapped(struct swap_info_struct *si, swp_entry_t entry);
 extern int swp_swapcount(swp_entry_t entry);
-struct backing_dev_info;
 extern struct swap_info_struct *get_swap_device(swp_entry_t entry);
 sector_t swap_folio_sector(struct folio *folio);
 
@@ -529,6 +461,10 @@ static inline bool folio_free_swap(struct folio *folio)
 	return false;
 }
 
+static inline int swap_fs_activate(struct swap_info_struct *sis)
+{
+	return -EINVAL;
+}
 static inline int add_swap_extent(struct swap_info_struct *sis,
 				  unsigned long start_page,
 				  unsigned long nr_pages, sector_t start_block)
@@ -552,7 +488,7 @@ static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg)
 
 void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid);
 #else
-static inline int mem_cgroup_swappiness(struct mem_cgroup *mem)
+static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg)
 {
 	return READ_ONCE(vm_swappiness);
 }
diff --git a/include/linux/swapops.h b/include/linux/swapops.h
index 8cfc966eae48e..c956bc445ee01 100644
--- a/include/linux/swapops.h
+++ b/include/linux/swapops.h
@@ -73,8 +73,8 @@ static inline pte_t pte_swp_clear_flags(pte_t pte)
 		pte = pte_swp_clear_exclusive(pte);
 	if (pte_swp_soft_dirty(pte))
 		pte = pte_swp_clear_soft_dirty(pte);
-	if (pte_swp_uffd_wp(pte))
-		pte = pte_swp_clear_uffd_wp(pte);
+	if (pte_swp_uffd(pte))
+		pte = pte_swp_clear_uffd(pte);
 	return pte;
 }
 
@@ -321,7 +321,7 @@ static inline swp_entry_t make_guard_swp_entry(void)
 
 struct page_vma_mapped_walk;
 
-#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
 extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
 		struct page *page);
 
@@ -338,7 +338,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry)
 	return __swp_entry_to_pmd(arch_entry);
 }
 
-#else  /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
+#else  /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */
 static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
 		struct page *page)
 {
@@ -358,7 +358,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry)
 	return __pmd(0);
 }
 
-#endif  /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
+#endif  /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */
 
 #endif /* CONFIG_MMU */
 #endif /* _LINUX_SWAPOPS_H */
diff --git a/include/linux/syscall_user_dispatch.h b/include/linux/syscall_user_dispatch.h
index 3858a6ffdd5c8..3dd30f4b27997 100644
--- a/include/linux/syscall_user_dispatch.h
+++ b/include/linux/syscall_user_dispatch.h
@@ -6,9 +6,23 @@
 #define _SYSCALL_USER_DISPATCH_H
 
 #include <linux/thread_info.h>
+#include <linux/sched.h>
 #include <linux/syscall_user_dispatch_types.h>
 
-#ifdef CONFIG_GENERIC_ENTRY
+struct pt_regs;
+
+#ifdef CONFIG_SYSCALL_USER_DISPATCH
+
+bool syscall_user_dispatch(struct pt_regs *regs);
+
+static __always_inline bool syscall_user_dispatch_clear_on_dispatch(void)
+{
+	if (likely(!current->syscall_dispatch.on_dispatch))
+		return false;
+
+	current->syscall_dispatch.on_dispatch = false;
+	return true;
+}
 
 int set_syscall_user_dispatch(unsigned long mode, unsigned long offset,
 			      unsigned long len, char __user *selector);
@@ -24,6 +38,16 @@ int syscall_user_dispatch_set_config(struct task_struct *task, unsigned long siz
 
 #else
 
+static __always_inline bool syscall_user_dispatch(struct pt_regs *regs)
+{
+	return false;
+}
+
+static __always_inline bool syscall_user_dispatch_clear_on_dispatch(void)
+{
+	return false;
+}
+
 static inline int set_syscall_user_dispatch(unsigned long mode, unsigned long offset,
 					    unsigned long len, char __user *selector)
 {
@@ -46,6 +70,6 @@ static inline int syscall_user_dispatch_set_config(struct task_struct *task,
 	return -EINVAL;
 }
 
-#endif /* CONFIG_GENERIC_ENTRY */
+#endif /* CONFIG_SYSCALL_USER_DISPATCH */
 
 #endif /* _SYSCALL_USER_DISPATCH_H */
diff --git a/include/linux/syscall_user_dispatch_types.h b/include/linux/syscall_user_dispatch_types.h
index 3be36b06c7d70..c0bdd4f760d3e 100644
--- a/include/linux/syscall_user_dispatch_types.h
+++ b/include/linux/syscall_user_dispatch_types.h
@@ -4,7 +4,7 @@
 
 #include <linux/types.h>
 
-#ifdef CONFIG_GENERIC_ENTRY
+#ifdef CONFIG_SYSCALL_USER_DISPATCH
 
 struct syscall_user_dispatch {
 	char __user	*selector;
diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h
index 2886fbceb5d63..39cf1bf9703fc 100644
--- a/include/linux/sysctl.h
+++ b/include/linux/sysctl.h
@@ -74,53 +74,108 @@ extern const int sysctl_vals[];
 
 extern const unsigned long sysctl_long_vals[];
 
-typedef int proc_handler(const struct ctl_table *ctl, int write, void *buffer,
-		size_t *lenp, loff_t *ppos);
-
-int proc_dostring(const struct ctl_table *, int, void *, size_t *, loff_t *);
-int proc_dobool(const struct ctl_table *table, int write, void *buffer,
-		size_t *lenp, loff_t *ppos);
+typedef int proc_handler(const struct ctl_table *ctl, int dir, void *buf,
+			 size_t *lenp, loff_t *ppos);
 
-int proc_dointvec(const struct ctl_table *, int, void *, size_t *, loff_t *);
-int proc_dointvec_minmax(const struct ctl_table *table, int dir, void *buffer,
+/* proc_handler functions */
+int proc_dostring(const struct ctl_table *ctl, int dir, void *buf, size_t *lenp,
+		  loff_t *ppos);
+int proc_dobool(const struct ctl_table *ctl, int dir, void *buf, size_t *lenp,
+		loff_t *ppos);
+int proc_dointvec(const struct ctl_table *ctl, int dir, void *buf, size_t *lenp,
+		  loff_t *ppos);
+int proc_dointvec_minmax(const struct ctl_table *ctl, int dir, void *buf,
 			 size_t *lenp, loff_t *ppos);
-int proc_dointvec_conv(const struct ctl_table *table, int dir, void *buffer,
+int proc_douintvec(const struct ctl_table *ctl, int dir, void *buf, size_t *lenp,
+		   loff_t *ppos);
+int proc_douintvec_minmax(const struct ctl_table *ctl, int dir, void *buf,
+			  size_t *lenp, loff_t *ppos);
+int proc_dou8vec_minmax(const struct ctl_table *ctl, int dir, void *buf,
+			size_t *lenp, loff_t *ppos);
+int proc_doulongvec_minmax(const struct ctl_table *ctl, int dir, void *buf,
+			   size_t *lenp, loff_t *ppos);
+int proc_do_large_bitmap(const struct ctl_table *ctl, int dir, void *buf,
+			 size_t *lenp, loff_t *ppos);
+int proc_do_static_key(const struct ctl_table *ctl, int dir, void *buf,
+		       size_t *lenp, loff_t *ppos);
+
+/*
+ * proc_handler aggregators
+ *
+ * Create a proc_handler with a custom converter. Use when the user space
+ * value is a transformation of the kernel value. Cannot be passed as
+ * proc_handlers.
+ *
+ * Example of creating your custom proc handler:
+ * int custom_converter(bool *negp, ulong *u_ptr, uint *k_ptr,
+ *                      int dir, const struct ctl_table *ctl) {...}
+ * int custom_proc_handler(const struct ctl_table *ctl, int dir,
+ *                         void *buf, size_t *lenp, loff_t *ppos
+ * { return proc_dointvec_conv(ctl, dir, buf, lenp, ppos, custom_converter); }
+ */
+int proc_dointvec_conv(const struct ctl_table *ctl, int dir, void *buf,
 		       size_t *lenp, loff_t *ppos,
 		       int (*conv)(bool *negp, unsigned long *u_ptr, int *k_ptr,
-				   int dir, const struct ctl_table *table));
-int proc_int_k2u_conv_kop(ulong *u_ptr, const int *k_ptr, bool *negp,
-			  ulong (*k_ptr_op)(const ulong));
-int proc_int_u2k_conv_uop(const ulong *u_ptr, int *k_ptr, const bool *negp,
-			  ulong (*u_ptr_op)(const ulong));
+				   int dir, const struct ctl_table *ctl));
+int proc_douintvec_conv(const struct ctl_table *ctl, int dir, void *buf,
+			size_t *lenp, loff_t *ppos,
+			int (*conv)(bool *negp, ulong *u_ptr, uint *k_ptr,
+				    int dir, const struct ctl_table *ctl));
+int proc_doulongvec_conv(const struct ctl_table *ctl, int dir, void *buf,
+			 size_t *lenp, loff_t *ppos,
+			 int (*conv)(bool *negp, ulong *u_ptr, ulong *k_ptr,
+				     int dir, const struct ctl_table *ctl));
+
+/*
+ * bi-directional converter functions
+ *
+ * Specify the converter function for both directions (user to kernel & kernel
+ * to user). Use when you want to change the value of the variable before
+ * assignment. Used to create custom proc_handler aggregators.
+ *
+ * Example of creating your custom bi-directional converter:
+ * int custom_u2k(ulong *u_ptr, const uint *k_ptr) { ... }
+ * int custom_converter(bool *negp, ulong *u_ptr, uint *k_ptr,
+ *                      int dir, const struct ctl_table *ctl)
+ * { return proc_uint_conv(u_ptr, k_ptr, dir, ctl, true,
+ *                         custom_u2k, proc_uint_k2u_conv}
+ */
 int proc_int_conv(bool *negp, ulong *u_ptr, int *k_ptr, int dir,
 		  const struct ctl_table *tbl, bool k_ptr_range_check,
 		  int (*user_to_kern)(const bool *negp, const ulong *u_ptr, int *k_ptr),
 		  int (*kern_to_user)(bool *negp, ulong *u_ptr, const int *k_ptr));
-
-int proc_douintvec(const struct ctl_table *, int, void *, size_t *, loff_t *);
-int proc_douintvec_minmax(const struct ctl_table *table, int write, void *buffer,
-		size_t *lenp, loff_t *ppos);
-int proc_douintvec_conv(const struct ctl_table *table, int write, void *buffer,
-			size_t *lenp, loff_t *ppos,
-			int (*conv)(unsigned long *lvalp, unsigned int *valp,
-				    int write, const struct ctl_table *table));
-int proc_uint_k2u_conv(ulong *u_ptr, const uint *k_ptr);
-int proc_uint_u2k_conv_uop(const ulong *u_ptr, uint *k_ptr,
-			   ulong (*u_ptr_op)(const ulong));
 int proc_uint_conv(ulong *u_ptr, uint *k_ptr, int dir,
 		   const struct ctl_table *tbl, bool k_ptr_range_check,
 		   int (*user_to_kern)(const ulong *u_ptr, uint *k_ptr),
 		   int (*kern_to_user)(ulong *u_ptr, const uint *k_ptr));
+int proc_ulong_conv(ulong *u_ptr, ulong *k_ptr, int dir,
+		    const struct ctl_table *tbl, bool k_ptr_range_check,
+		    int (*user_to_kern)(const ulong *u_ptr, ulong *k_ptr),
+		    int (*kern_to_user)(ulong *u_ptr, const ulong *k_ptr));
 
-int proc_dou8vec_minmax(const struct ctl_table *table, int write, void *buffer,
-			size_t *lenp, loff_t *ppos);
-int proc_doulongvec_minmax(const struct ctl_table *, int, void *, size_t *, loff_t *);
-int proc_doulongvec_minmax_conv(const struct ctl_table *table, int dir,
-				void *buffer, size_t *lenp, loff_t *ppos,
-				unsigned long convmul, unsigned long convdiv);
-int proc_do_large_bitmap(const struct ctl_table *, int, void *, size_t *, loff_t *);
-int proc_do_static_key(const struct ctl_table *table, int write, void *buffer,
-		size_t *lenp, loff_t *ppos);
+/*
+ * uni-directional converter functions
+ *
+ * Specify the converter function for one directions (user to kernel or
+ * kernel to user). Use to call the actual value conversion. Used to Create
+ * bi-directional converters.
+ *
+ * Example of creating a uni-directional converter:
+ * ulong op(const ulong val) { ... }
+ * int custom_unidir_conv(ulong *u_ptr, const uint *k_ptr)
+ * { return proc_uint_k2u_conv_kop(u_ptr, k_ptr, op); }
+ */
+int proc_int_k2u_conv_kop(ulong *u_ptr, const int *k_ptr, bool *negp,
+			  ulong (*k_ptr_op)(const ulong));
+int proc_int_u2k_conv_uop(const ulong *u_ptr, int *k_ptr, const bool *negp,
+			  ulong (*u_ptr_op)(const ulong));
+int proc_uint_k2u_conv(ulong *u_ptr, const uint *k_ptr);
+int proc_uint_u2k_conv_uop(const ulong *u_ptr, uint *k_ptr,
+			   ulong (*u_ptr_op)(const ulong));
+int proc_ulong_u2k_conv_uop(const ulong *u_ptr, ulong *k_ptr,
+			    ulong (*u_ptr_op)(const ulong));
+int proc_ulong_k2u_conv_kop(ulong *u_ptr, const ulong *k_ptr,
+			    ulong (*k_ptr_op)(const ulong));
 
 /*
  * Register a set of sysctl names by calling register_sysctl
diff --git a/include/linux/sysfb.h b/include/linux/sysfb.h
index 5226efde9ad4d..ed23d65162232 100644
--- a/include/linux/sysfb.h
+++ b/include/linux/sysfb.h
@@ -118,7 +118,7 @@ struct platform_device *sysfb_create_simplefb(const struct screen_info *si,
 					      const struct simplefb_platform_data *mode,
 					      struct device *parent);
 
-#else /* CONFIG_SYSFB_SIMPLE */
+#else /* CONFIG_SYSFB_SIMPLEFB */
 
 static inline bool sysfb_parse_mode(const struct screen_info *si,
 				    struct simplefb_platform_data *mode)
@@ -133,6 +133,6 @@ static inline struct platform_device *sysfb_create_simplefb(const struct screen_
 	return ERR_PTR(-EINVAL);
 }
 
-#endif /* CONFIG_SYSFB_SIMPLE */
+#endif /* CONFIG_SYSFB_SIMPLEFB */
 
 #endif /* _LINUX_SYSFB_H */
diff --git a/include/linux/taskstats_kern.h b/include/linux/taskstats_kern.h
index dbb4d124c7d7c..995cd19b56c1f 100644
--- a/include/linux/taskstats_kern.h
+++ b/include/linux/taskstats_kern.h
@@ -14,7 +14,6 @@
 
 #ifdef CONFIG_TASKSTATS
 extern struct kmem_cache *taskstats_cache;
-extern struct mutex taskstats_exit_mutex;
 
 static inline void taskstats_tgid_free(struct signal_struct *sig)
 {
diff --git a/include/linux/thunderbolt.h b/include/linux/thunderbolt.h
index 557288c0274b3..56aa4d6202796 100644
--- a/include/linux/thunderbolt.h
+++ b/include/linux/thunderbolt.h
@@ -465,7 +465,7 @@ static inline struct tb_service *tb_to_service(struct device *dev)
  */
 struct tb_service_driver {
 	struct device_driver driver;
-	int (*probe)(struct tb_service *svc, const struct tb_service_id *id);
+	int (*probe)(struct tb_service *svc);
 	void (*remove)(struct tb_service *svc);
 	void (*shutdown)(struct tb_service *svc);
 	const struct tb_service_id *id_table;
@@ -514,6 +514,11 @@ void tb_service_properties_changed(struct tb_service *svc);
  * @hop_count: Number of rings (end point hops) supported by NHI.
  * @quirks: NHI specific quirks if any
  * @domain_released: Completed when domain has been fully released
+ * @host_reset: Host router was reset on driver load, or forced on system
+ *		shutdown/reboot. When set, tb_stop() asserts DPR on connected
+ *		downstream ports to signal disconnect before tearing down the
+ *		router tree. Only Thunderbolt 3 devices are reset; USB4
+ *		routers are skipped.
  */
 struct tb_nhi {
 	spinlock_t lock;
@@ -528,6 +533,7 @@ struct tb_nhi {
 	u32 hop_count;
 	unsigned long quirks;
 	struct completion domain_released;
+	bool host_reset;
 };
 
 /**
diff --git a/include/linux/tick.h b/include/linux/tick.h
index 1cf4651f09ad6..a43b7bd8ba3fb 100644
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -44,9 +44,6 @@ static inline void tick_unfreeze(void) { }
 
 #ifdef CONFIG_TICK_ONESHOT
 extern void tick_irq_enter(void);
-#  ifndef arch_needs_cpu
-#   define arch_needs_cpu() (0)
-#  endif
 # else
 static inline void tick_irq_enter(void) { }
 #endif
diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h
index 4486dfd5d0ded..9c198f65621ba 100644
--- a/include/linux/timekeeper_internal.h
+++ b/include/linux/timekeeper_internal.h
@@ -84,8 +84,6 @@ struct tk_read_base {
  * @cycle_interval:		Number of clock cycles in one NTP interval
  * @xtime_interval:		Number of clock shifted nano seconds in one NTP
  *				interval.
- * @xtime_remainder:		Shifted nano seconds left over when rounding
- *				@cycle_interval
  * @raw_interval:		Shifted raw nano seconds accumulated per NTP interval.
  * @next_leap_ktime:		CLOCK_MONOTONIC time value of a pending leap-second
  * @ntp_tick:			The ntp_tick_length() value currently being
@@ -99,6 +97,10 @@ struct tk_read_base {
  * @ntp_error_shift:		Shift conversion between clock shifted nano seconds and
  *				ntp shifted nano seconds.
  * @ntp_err_mult:		Multiplication factor for scaled math conversion
+ * @cs_tick_adj:		Per-second adjustment handed to NTP via ntp_clear()
+ *				accounting for the difference between the nominal
+ *				NTP interval and the real time taken by the
+ *				clocksource's integer @cycle_interval (upscaled).
  * @skip_second_overflow:	Flag used to avoid updating NTP twice with same second
  * @tai_offset:			The current UTC to TAI offset in seconds
  *
@@ -178,7 +180,6 @@ struct timekeeper {
 
 	u64			cycle_interval;
 	u64			xtime_interval;
-	s64			xtime_remainder;
 	u64			raw_interval;
 
 	ktime_t			next_leap_ktime;
@@ -186,29 +187,10 @@ struct timekeeper {
 	s64			ntp_error;
 	u32			ntp_error_shift;
 	u32			ntp_err_mult;
+	s64			cs_tick_adj;
 	u32			skip_second_overflow;
+	s64			skew_delta;
 	s32			tai_offset;
 };
 
-#ifdef CONFIG_GENERIC_GETTIMEOFDAY
-
-extern void update_vsyscall(struct timekeeper *tk);
-extern void update_vsyscall_tz(void);
-
-#else
-
-static inline void update_vsyscall(struct timekeeper *tk)
-{
-}
-static inline void update_vsyscall_tz(void)
-{
-}
-#endif
-
-#if defined(CONFIG_GENERIC_GETTIMEOFDAY) && defined(CONFIG_POSIX_AUX_CLOCKS)
-extern void vdso_time_update_aux(struct timekeeper *tk);
-#else
-static inline void vdso_time_update_aux(struct timekeeper *tk) { }
-#endif
-
 #endif /* _LINUX_TIMEKEEPER_INTERNAL_H */
diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h
index 984a866d293b6..b4191a873c390 100644
--- a/include/linux/timekeeping.h
+++ b/include/linux/timekeeping.h
@@ -276,7 +276,7 @@ static inline bool ktime_get_aux_ts64(clockid_t id, struct timespec64 *kt) { ret
 #endif
 
 /**
- * struct system_time_snapshot - Simultaneous time capture of CLOCK_MONOTONIC_RAW,
+ * struct system_time_snapshot - Simultaneous time capture of monotonic raw time,
  *				 a selected CLOCK_* and the clocksource counter value
  * @cycles:		Clocksource counter value to produce the system times
  * @hw_cycles:		For derived clocksources, the hardware counter value from
@@ -289,6 +289,10 @@ static inline bool ktime_get_aux_ts64(clockid_t id, struct timespec64 *kt) { ret
  * @clock_was_set_seq:	The sequence number of clock-was-set events
  * @cs_was_changed_seq:	The sequence number of clocksource change events
  * @valid:		True if the snapshot is valid
+ *
+ * @monoraw is CLOCK_MONOTONIC_RAW for system time CLOCK ids. For CLOCK_AUX$N
+ * clock ids it's the monotonic raw time related to the AUX clock, which is
+ * CLOCK_MONOTONIC_RAW plus a AUX clock specific offset.
  */
 struct system_time_snapshot {
 	u64			cycles;
@@ -326,6 +330,10 @@ struct system_counterval_t {
  * @sys_counter:	Clocksource counter value simultaneous with device time
  * @sys_systime:	System time for @clock_id
  * @sys_monoraw:	Monotonic raw simultaneous with device time
+ *
+ * @sys_monoraw is CLOCK_MONOTONIC_RAW for system time CLOCK ids. For
+ * CLOCK_AUX$N clock ids it's the monotonic raw time related to the AUX clock,
+ * which is CLOCK_MONOTONIC_RAW plus a AUX clock specific offset.
  */
 struct system_device_crosststamp {
 	clockid_t			clock_id;
diff --git a/include/linux/types.h b/include/linux/types.h
index 93166b0b0617a..bc5dda2a3d861 100644
--- a/include/linux/types.h
+++ b/include/linux/types.h
@@ -163,6 +163,8 @@ typedef u32 dma_addr_t;
 typedef unsigned int __bitwise gfp_t;
 typedef unsigned int __bitwise slab_flags_t;
 typedef unsigned int __bitwise fmode_t;
+typedef unsigned int __bitwise blk_mode_t;
+typedef unsigned int __bitwise fop_flags_t;
 
 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 typedef u64 phys_addr_t;
diff --git a/include/linux/unicode.h b/include/linux/unicode.h
index 5e6b212a2aeda..64fa44fe180c4 100644
--- a/include/linux/unicode.h
+++ b/include/linux/unicode.h
@@ -66,9 +66,6 @@ int utf8_strncasecmp_folded(const struct unicode_map *um,
 			    const struct qstr *cf,
 			    const struct qstr *s1);
 
-int utf8_normalize(const struct unicode_map *um, const struct qstr *str,
-		   unsigned char *dest, size_t dlen);
-
 int utf8_casefold(const struct unicode_map *um, const struct qstr *str,
 		  unsigned char *dest, size_t dlen);
 
diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h
index f548fea2adec8..18be159bbc341 100644
--- a/include/linux/uprobes.h
+++ b/include/linux/uprobes.h
@@ -186,9 +186,6 @@ struct xol_area;
 
 struct uprobes_state {
 	struct xol_area		*xol_area;
-#ifdef CONFIG_X86_64
-	struct hlist_head	head_tramps;
-#endif
 };
 
 typedef int (*uprobe_write_verify_t)(struct page *page, unsigned long vaddr,
@@ -238,8 +235,6 @@ extern void uprobe_handle_trampoline(struct pt_regs *regs);
 extern void *arch_uretprobe_trampoline(unsigned long *psize);
 extern unsigned long uprobe_get_trampoline_vaddr(void);
 extern void uprobe_copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len);
-extern void arch_uprobe_clear_state(struct mm_struct *mm);
-extern void arch_uprobe_init_state(struct mm_struct *mm);
 extern void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr);
 extern void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr);
 extern unsigned long arch_uprobe_get_xol_area(void);
diff --git a/include/linux/usb.h b/include/linux/usb.h
index 1da4ad1610bca..49ab8dbb885f6 100644
--- a/include/linux/usb.h
+++ b/include/linux/usb.h
@@ -1185,7 +1185,8 @@ extern ssize_t usb_show_dynids(struct usb_dynids *dynids, char *buf);
  *	interface.  It may also use usb_set_interface() to specify the
  *	appropriate altsetting.  If unwilling to manage the interface,
  *	return -ENODEV, if genuine IO errors occurred, an appropriate
- *	negative errno value.
+ *	negative errno value.  The usb_device_id parameter is only valid during
+ *	probe.
  * @disconnect: Called when the interface is no longer accessible, usually
  *	because its device has been (or is being) disconnected or the
  *	driver module is being unloaded.
diff --git a/include/linux/usb/pd.h b/include/linux/usb/pd.h
index 337a5485af7c7..ee360dedeaa65 100644
--- a/include/linux/usb/pd.h
+++ b/include/linux/usb/pd.h
@@ -493,7 +493,7 @@ static inline unsigned int pdo_epr_avs_apdo_min_voltage_mv(u32 pdo)
 
 static inline unsigned int pdo_epr_avs_apdo_max_voltage_mv(u32 pdo)
 {
-	return FIELD_GET(PDO_EPR_AVS_APDO_MIN_VOLT, pdo) * 100;
+	return FIELD_GET(PDO_EPR_AVS_APDO_MAX_VOLT, pdo) * 100;
 }
 
 static inline unsigned int pdo_epr_avs_apdo_pdp_w(u32 pdo)
diff --git a/include/linux/usb/typec.h b/include/linux/usb/typec.h
index d61ec38216fa9..10a783b738efd 100644
--- a/include/linux/usb/typec.h
+++ b/include/linux/usb/typec.h
@@ -337,6 +337,7 @@ void typec_unregister_cable(struct typec_cable *cable);
 struct typec_cable *typec_cable_get(struct typec_port *port);
 void typec_cable_put(struct typec_cable *cable);
 int typec_cable_is_active(struct typec_cable *cable);
+bool typec_cable_altmode_unsupported(struct typec_altmode *alt);
 
 struct typec_plug *typec_register_plug(struct typec_cable *cable,
 				       struct typec_plug_desc *desc);
diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h
index 68edac4dcd78b..bfbd6a59909f0 100644
--- a/include/linux/userfaultfd_k.h
+++ b/include/linux/userfaultfd_k.h
@@ -21,10 +21,11 @@
 #include <linux/hugetlb_inline.h>
 
 /* The set of all possible UFFD-related VM flags. */
-#define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR)
+#define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_MINOR | \
+			 VM_UFFD_WP | VM_UFFD_RWP)
 
 #define __VMA_UFFD_FLAGS mk_vma_flags_from_masks(VMA_UFFD_MISSING, VMA_UFFD_WP, \
-						 VMA_UFFD_MINOR)
+						 VMA_UFFD_MINOR, VMA_UFFD_RWP)
 
 /*
  * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining
@@ -149,6 +150,8 @@ static inline uffd_flags_t uffd_flags_set_mode(uffd_flags_t flags, enum mfill_at
 
 extern long uffd_wp_range(struct vm_area_struct *vma,
 			  unsigned long start, unsigned long len, bool enable_wp);
+extern int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
+			    unsigned long len, bool enable_rwp);
 
 /* move_pages */
 void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2);
@@ -168,7 +171,8 @@ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma,
 /*
  * Never enable huge pmd sharing on some uffd registered vmas:
  *
- * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry.
+ * - VM_UFFD_WP and VM_UFFD_RWP VMAs, because the write protect / access
+ *   tracking information is per pgtable entry.
  *
  * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for
  *   VMAs which share huge pmds. (If you have two mappings to the same
@@ -178,51 +182,80 @@ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma,
  */
 static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma)
 {
-	return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR);
+	return vma_test_any_mask(vma,
+		mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_RWP,
+					VMA_UFFD_MINOR));
 }
 
 /*
- * Don't do fault around for either WP or MINOR registered uffd range.  For
+ * Don't do fault around for WP, RWP or MINOR registered uffd range.  For
  * MINOR registered range, fault around will be a total disaster and ptes can
  * be installed without notifications; for WP it should mostly be fine as long
  * as the fault around checks for pte_none() before the installation, however
- * to be super safe we just forbid it.
+ * to be super safe we just forbid it; for RWP, pre-faulted neighbours would
+ * be indistinguishable from accessed pages in PAGEMAP_SCAN (PAGE_IS_ACCESSED)
+ * and pollute the tracked working set, so each page must be populated by its
+ * own fault.
  */
 static inline bool uffd_disable_fault_around(struct vm_area_struct *vma)
 {
-	return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR);
+	return vma_test_any_mask(vma,
+		mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_RWP,
+					VMA_UFFD_MINOR));
 }
 
 static inline bool userfaultfd_missing(struct vm_area_struct *vma)
 {
-	return vma->vm_flags & VM_UFFD_MISSING;
+	return vma_test_any_mask(vma, VMA_UFFD_MISSING);
 }
 
 static inline bool userfaultfd_wp(struct vm_area_struct *vma)
 {
-	return vma->vm_flags & VM_UFFD_WP;
+	return vma_test_any_mask(vma, VMA_UFFD_WP);
 }
 
 static inline bool userfaultfd_minor(struct vm_area_struct *vma)
 {
-	return vma->vm_flags & VM_UFFD_MINOR;
+	return vma_test_any_mask(vma, VMA_UFFD_MINOR);
+}
+
+static inline bool userfaultfd_rwp(struct vm_area_struct *vma)
+{
+	return vma_test_single_mask(vma, VMA_UFFD_RWP);
+}
+
+static inline bool userfaultfd_protected(struct vm_area_struct *vma)
+{
+	return userfaultfd_wp(vma) || userfaultfd_rwp(vma);
 }
 
 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma,
 				      pte_t pte)
 {
-	return userfaultfd_wp(vma) && pte_uffd_wp(pte);
+	return userfaultfd_wp(vma) && pte_uffd(pte);
 }
 
 static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma,
 					   pmd_t pmd)
 {
-	return userfaultfd_wp(vma) && pmd_uffd_wp(pmd);
+	return userfaultfd_wp(vma) && pmd_uffd(pmd);
+}
+
+static inline bool userfaultfd_pte_rwp(struct vm_area_struct *vma,
+				       pte_t pte)
+{
+	return userfaultfd_rwp(vma) && pte_uffd(pte);
+}
+
+static inline bool userfaultfd_huge_pmd_rwp(struct vm_area_struct *vma,
+					    pmd_t pmd)
+{
+	return userfaultfd_rwp(vma) && pmd_uffd(pmd);
 }
 
 static inline bool userfaultfd_armed(struct vm_area_struct *vma)
 {
-	return vma->vm_flags & __VM_UFFD_FLAGS;
+	return vma_test_any_mask(vma, __VMA_UFFD_FLAGS);
 }
 
 static inline bool vma_has_uffd_without_event_remap(struct vm_area_struct *vma)
@@ -253,6 +286,7 @@ extern void userfaultfd_unmap_complete(struct mm_struct *mm,
 				       struct list_head *uf);
 extern bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma);
 extern bool userfaultfd_wp_async(struct vm_area_struct *vma);
+extern bool userfaultfd_rwp_async(struct vm_area_struct *vma);
 
 static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma)
 {
@@ -272,10 +306,10 @@ static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma)
 }
 
 /*
- * Returns true if this is a swap pte and was uffd-wp wr-protected in either
- * forms (pte marker or a normal swap pte), false otherwise.
+ * Returns true if this swap pte carries uffd-tracked state in either
+ * form (pte marker or a normal swap pte), false otherwise.
  */
-static inline bool pte_swp_uffd_wp_any(pte_t pte)
+static inline bool pte_swp_uffd_any(pte_t pte)
 {
 	if (!uffd_supports_wp_marker())
 		return false;
@@ -283,7 +317,7 @@ static inline bool pte_swp_uffd_wp_any(pte_t pte)
 	if (pte_present(pte))
 		return false;
 
-	if (pte_swp_uffd_wp(pte))
+	if (pte_swp_uffd(pte))
 		return true;
 
 	if (pte_is_uffd_wp_marker(pte))
@@ -328,6 +362,16 @@ static inline bool userfaultfd_minor(struct vm_area_struct *vma)
 	return false;
 }
 
+static inline bool userfaultfd_rwp(struct vm_area_struct *vma)
+{
+	return false;
+}
+
+static inline bool userfaultfd_protected(struct vm_area_struct *vma)
+{
+	return false;
+}
+
 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma,
 				      pte_t pte)
 {
@@ -340,6 +384,17 @@ static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma,
 	return false;
 }
 
+static inline bool userfaultfd_pte_rwp(struct vm_area_struct *vma,
+				       pte_t pte)
+{
+	return false;
+}
+
+static inline bool userfaultfd_huge_pmd_rwp(struct vm_area_struct *vma,
+					    pmd_t pmd)
+{
+	return false;
+}
 
 static inline bool userfaultfd_armed(struct vm_area_struct *vma)
 {
@@ -410,6 +465,11 @@ static inline bool userfaultfd_wp_async(struct vm_area_struct *vma)
 	return false;
 }
 
+static inline bool userfaultfd_rwp_async(struct vm_area_struct *vma)
+{
+	return false;
+}
+
 static inline bool vma_has_uffd_without_event_remap(struct vm_area_struct *vma)
 {
 	return false;
@@ -421,10 +481,10 @@ static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma)
 }
 
 /*
- * Returns true if this is a swap pte and was uffd-wp wr-protected in either
- * forms (pte marker or a normal swap pte), false otherwise.
+ * Returns true if this swap pte carries uffd-tracked state in either
+ * form (pte marker or a normal swap pte), false otherwise.
  */
-static inline bool pte_swp_uffd_wp_any(pte_t pte)
+static inline bool pte_swp_uffd_any(pte_t pte)
 {
 	return false;
 }
diff --git a/include/linux/vdso_datastore.h b/include/linux/vdso_datastore.h
index 3dfba9502d78f..13b01baf34972 100644
--- a/include/linux/vdso_datastore.h
+++ b/include/linux/vdso_datastore.h
@@ -7,10 +7,10 @@
 extern const struct vm_special_mapping vdso_vvar_mapping;
 struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr);
 
-#ifdef CONFIG_HAVE_GENERIC_VDSO
+#ifdef CONFIG_VDSO_DATASTORE
 void __init vdso_setup_data_pages(void);
-#else /* !CONFIG_HAVE_GENERIC_VDSO */
+#else /* !CONFIG_VDSO_DATASTORE */
 static inline void vdso_setup_data_pages(void) { }
-#endif /* CONFIG_HAVE_GENERIC_VDSO */
+#endif /* CONFIG_VDSO_DATASTORE */
 
 #endif /* _LINUX_VDSO_DATASTORE_H */
diff --git a/include/linux/vfio.h b/include/linux/vfio.h
index 31b826efba000..45f08986359e4 100644
--- a/include/linux/vfio.h
+++ b/include/linux/vfio.h
@@ -74,6 +74,7 @@ struct vfio_device {
 	u8 iommufd_attached:1;
 #endif
 	u8 cdev_opened:1;
+	u8 noiommu:1;
 	/*
 	 * debug_root is a static property of the vfio_device
 	 * which must be set prior to registering the vfio_device.
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 03fe95f5a0201..2628ccda076a0 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -175,6 +175,10 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
 		KSTACK_REST,
 #endif
 #endif /* CONFIG_DEBUG_STACK_USAGE */
+#ifdef CONFIG_SWAP
+		NRSWPIN,
+		NRSWPOUT,
+#endif /* CONFIG_SWAP */
 		NR_VM_EVENT_ITEMS
 };
 
diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
index d87dc7f77f4e8..e4d8d0a9f30f9 100644
--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -310,14 +310,14 @@ static inline void set_vm_flush_reset_perms(void *addr) {}
 #if defined(CONFIG_MMU) && defined(CONFIG_SMP)
 struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
 				     const size_t *sizes, int nr_vms,
-				     size_t align);
+				     size_t align, gfp_t gfp);
 
 void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms);
 # else
 static inline struct vm_struct **
 pcpu_get_vm_areas(const unsigned long *offsets,
 		const size_t *sizes, int nr_vms,
-		size_t align)
+		size_t align, gfp_t gfp)
 {
 	return NULL;
 }
diff --git a/include/linux/vmpressure.h b/include/linux/vmpressure.h
index faecd55224017..b4d13457bc2ad 100644
--- a/include/linux/vmpressure.h
+++ b/include/linux/vmpressure.h
@@ -13,18 +13,31 @@
 struct vmpressure {
 	unsigned long scanned;
 	unsigned long reclaimed;
+	/* The lock is used to keep the scanned/reclaimed in sync. */
+	spinlock_t sr_lock;
 
+#ifdef CONFIG_MEMCG_V1
+	/*
+	 * tree=true accumulators feed the v1 userspace eventfd interface
+	 * (memory.pressure_level). Drained by @work. v2 has no equivalent
+	 * interface, so this state is omitted on CONFIG_MEMCG_V1=n builds.
+	 */
 	unsigned long tree_scanned;
 	unsigned long tree_reclaimed;
-	/* The lock is used to keep the scanned/reclaimed above in sync. */
-	spinlock_t sr_lock;
-
 	/* The list of vmpressure_event structs. */
 	struct list_head events;
 	/* Have to grab the lock on events traversal or modifications. */
 	struct mutex events_lock;
 
 	struct work_struct work;
+#endif
+};
+
+enum vmpressure_levels {
+	VMPRESSURE_LOW = 0,
+	VMPRESSURE_MEDIUM,
+	VMPRESSURE_CRITICAL,
+	VMPRESSURE_NUM_LEVELS,
 };
 
 struct mem_cgroup;
@@ -32,18 +45,41 @@ struct mem_cgroup;
 #ifdef CONFIG_MEMCG
 void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree,
 		unsigned long scanned, unsigned long reclaimed);
-extern void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio);
-
 extern void vmpressure_init(struct vmpressure *vmpr);
 extern void vmpressure_cleanup(struct vmpressure *vmpr);
 extern struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg);
 extern struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr);
+
+/* Shared with the v1 vmpressure block in mm/memcontrol-v1.c. */
+extern const unsigned long vmpressure_win;
+extern enum vmpressure_levels vmpressure_calc_level(unsigned long scanned,
+						    unsigned long reclaimed);
+
+#ifdef CONFIG_MEMCG_V1
+extern void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio);
 extern int vmpressure_register_event(struct mem_cgroup *memcg,
 				     struct eventfd_ctx *eventfd,
 				     const char *args);
 extern void vmpressure_unregister_event(struct mem_cgroup *memcg,
 					struct eventfd_ctx *eventfd);
+
+/* v1 hooks called from mm/vmpressure.c; no-ops below when !MEMCG_V1. */
+extern void vmpressure_v1_init(struct vmpressure *vmpr);
+extern void vmpressure_v1_cleanup(struct vmpressure *vmpr);
+extern void vmpressure_v1_account_tree(struct vmpressure *vmpr,
+				       unsigned long scanned,
+				       unsigned long reclaimed);
 #else
+static inline void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg,
+				   int prio) {}
+static inline void vmpressure_v1_init(struct vmpressure *vmpr) {}
+static inline void vmpressure_v1_cleanup(struct vmpressure *vmpr) {}
+static inline void vmpressure_v1_account_tree(struct vmpressure *vmpr,
+					      unsigned long scanned,
+					      unsigned long reclaimed) {}
+#endif /* CONFIG_MEMCG_V1 */
+
+#else /* !CONFIG_MEMCG */
 static inline void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg,
 			      bool tree, unsigned long scanned,
 			      unsigned long reclaimed) {}
diff --git a/include/linux/writeback.h b/include/linux/writeback.h
index 62552a2ce5b9e..b749a9a5a5ee0 100644
--- a/include/linux/writeback.h
+++ b/include/linux/writeback.h
@@ -244,7 +244,7 @@ void wbc_attach_fdatawrite_inode(struct writeback_control *wbc,
 		struct inode *inode);
 
 /**
- * wbc_init_bio - writeback specific initializtion of bio
+ * wbc_init_bio - writeback specific initialization of bio
  * @wbc: writeback_control for the writeback in progress
  * @bio: bio to be initialized
  *
diff --git a/include/media/cec.h b/include/media/cec.h
index 0c8e86115b6f5..4c8b0950b3779 100644
--- a/include/media/cec.h
+++ b/include/media/cec.h
@@ -85,7 +85,6 @@ struct cec_event_entry {
 	struct cec_event	ev;
 };
 
-#define CEC_NUM_CORE_EVENTS 2
 #define CEC_NUM_EVENTS CEC_EVENT_PIN_5V_HIGH
 
 struct cec_fh {
@@ -101,7 +100,6 @@ struct cec_fh {
 	struct list_head	events[CEC_NUM_EVENTS]; /* queued events */
 	u16			queued_events[CEC_NUM_EVENTS];
 	unsigned int		total_queued_events;
-	struct cec_event_entry	core_events[CEC_NUM_CORE_EVENTS];
 	struct list_head	msgs; /* queued messages */
 	unsigned int		queued_msgs;
 };
@@ -223,6 +221,8 @@ struct cec_adap_ops {
  * @tx_error_log_cnt:	number of logged Error transmits since the adapter was
  *                      enabled. Used to avoid flooding the kernel log if this
  *                      happens a lot.
+ * @error_inj_tx_timeouts: error injection: the next @error_inj_tx_timeouts
+ *			transmits will time out.
  * @notifier:		CEC notifier
  * @pin:		CEC pin status struct
  * @cec_dir:		debugfs cec directory
@@ -283,6 +283,8 @@ struct cec_adapter {
 	u32 tx_low_drive_log_cnt;
 	u32 tx_error_log_cnt;
 
+	u32 error_inj_tx_timeouts;
+
 #ifdef CONFIG_CEC_NOTIFIER
 	struct cec_notifier *notifier;
 #endif
diff --git a/include/media/drv-intf/s3c_camif.h b/include/media/drv-intf/s3c_camif.h
index f746851a5ce60..00d83620ea5ff 100644
--- a/include/media/drv-intf/s3c_camif.h
+++ b/include/media/drv-intf/s3c_camif.h
@@ -31,8 +31,6 @@ struct s3c_camif_sensor_info {
 
 struct s3c_camif_plat_data {
 	struct s3c_camif_sensor_info sensor;
-	int (*gpio_get)(void);
-	int (*gpio_put)(void);
 };
 
 #endif /* MEDIA_S3C_CAMIF_ */
diff --git a/include/media/drv-intf/saa7146.h b/include/media/drv-intf/saa7146.h
index 71ce63c99cb43..b8f2fe5680f0f 100644
--- a/include/media/drv-intf/saa7146.h
+++ b/include/media/drv-intf/saa7146.h
@@ -77,13 +77,10 @@ struct saa7146_pci_extension_data {
 	void *ext_priv;			/* most likely a name string */
 };
 
-#define MAKE_EXTENSION_PCI(x_var, x_vendor, x_device)		\
-	{							\
-		.vendor    = PCI_VENDOR_ID_PHILIPS,		\
-		.device	   = PCI_DEVICE_ID_PHILIPS_SAA7146,	\
-		.subvendor = x_vendor,				\
-		.subdevice = x_device,				\
-		.driver_data = (unsigned long)& x_var,		\
+#define MAKE_EXTENSION_PCI(x_var, x_vendor, x_device)						\
+	{											\
+		PCI_VDEVICE_SUB(PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7146, (x_vendor), (x_device)),\
+		.driver_data = (unsigned long)&(x_var),						\
 	}
 
 struct saa7146_extension
diff --git a/include/media/media-entity.h b/include/media/media-entity.h
index d9b72cd87d524..fa393e8406694 100644
--- a/include/media/media-entity.h
+++ b/include/media/media-entity.h
@@ -1387,6 +1387,7 @@ void media_remove_intf_links(struct media_interface *intf);
  * @entity: entity where the @operation will be called
  * @operation: type of the operation. Should be the name of a member of
  *	struct &media_entity_operations.
+ * @args: arguments for the operation.
  *
  * This helper function will check if @operation is not %NULL. On such case,
  * it will issue a call to @operation\(@entity, @args\).
diff --git a/include/media/rc-map.h b/include/media/rc-map.h
index d90e4611b0664..d95ed3e96de28 100644
--- a/include/media/rc-map.h
+++ b/include/media/rc-map.h
@@ -262,7 +262,6 @@ struct rc_map *rc_map_get(const char *name);
 #define RC_MAP_GENIUS_TVGO_A11MCE        "rc-genius-tvgo-a11mce"
 #define RC_MAP_GOTVIEW7135               "rc-gotview7135"
 #define RC_MAP_HAUPPAUGE                 "rc-hauppauge"
-#define RC_MAP_HAUPPAUGE_NEW             "rc-hauppauge"
 #define RC_MAP_HISI_POPLAR               "rc-hisi-poplar"
 #define RC_MAP_HISI_TV_DEMO              "rc-hisi-tv-demo"
 #define RC_MAP_IMON_MCE                  "rc-imon-mce"
@@ -309,7 +308,6 @@ struct rc_map *rc_map_get(const char *name);
 #define RC_MAP_PROTEUS_2309              "rc-proteus-2309"
 #define RC_MAP_PURPLETV                  "rc-purpletv"
 #define RC_MAP_PV951                     "rc-pv951"
-#define RC_MAP_RC5_TV                    "rc-rc5-tv"
 #define RC_MAP_RC6_MCE                   "rc-rc6-mce"
 #define RC_MAP_REAL_AUDIO_220_32_KEYS    "rc-real-audio-220-32-keys"
 #define RC_MAP_REDDO                     "rc-reddo"
diff --git a/include/media/v4l2-ioctl.h b/include/media/v4l2-ioctl.h
index 54c83b18d555b..a13dca7c609f1 100644
--- a/include/media/v4l2-ioctl.h
+++ b/include/media/v4l2-ioctl.h
@@ -736,8 +736,8 @@ long int video_usercopy(struct file *file, unsigned int cmd,
  * @cmd: Ioctl name.
  * @arg: Ioctl argument.
  *
- * Method used to hancle an ioctl. Should be used to fill the
- * &v4l2_ioctl_ops.unlocked_ioctl on all V4L2 drivers.
+ * Method used to handle an ioctl. Should be used to fill the
+ * &v4l2_file_operations.unlocked_ioctl on all V4L2 drivers.
  */
 long int video_ioctl2(struct file *file,
 		      unsigned int cmd, unsigned long int arg);
diff --git a/include/media/videobuf2-v4l2.h b/include/media/videobuf2-v4l2.h
index 71d2864fb235f..d51e10c10fc04 100644
--- a/include/media/videobuf2-v4l2.h
+++ b/include/media/videobuf2-v4l2.h
@@ -72,6 +72,19 @@ struct vb2_v4l2_buffer {
  */
 struct vb2_buffer *vb2_find_buffer(struct vb2_queue *q, u64 timestamp);
 
+/**
+ * vb2_querybuf() - Query video buffer information
+ * @q:		pointer to &struct vb2_queue with videobuf2 queue.
+ * @b:		buffer structure passed from userspace to
+ *		&v4l2_ioctl_ops->vidioc_querybuf handler in driver
+ *
+ * Should be called from vidioc_querybuf ioctl handler in driver.
+ * This function will verify the passed v4l2_buffer structure and fill the
+ * relevant information for the userspace.
+ *
+ * The return values from this function are intended to be directly returned
+ * from vidioc_querybuf handler in driver.
+ */
 int vb2_querybuf(struct vb2_queue *q, struct v4l2_buffer *b);
 
 /**
diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h
index 4ca09298e11ae..e7133ff87fbfa 100644
--- a/include/net/bluetooth/hci_core.h
+++ b/include/net/bluetooth/hci_core.h
@@ -2430,6 +2430,7 @@ void mgmt_new_link_key(struct hci_dev *hdev, struct link_key *key,
 		       bool persistent);
 void mgmt_device_connected(struct hci_dev *hdev, struct hci_conn *conn,
 			   u8 *name, u8 name_len);
+u8 hci_to_mgmt_reason(u8 err);
 void mgmt_device_disconnected(struct hci_dev *hdev, bdaddr_t *bdaddr,
 			      u8 link_type, u8 addr_type, u8 reason,
 			      bool mgmt_connected);
diff --git a/include/net/bluetooth/hci_sync.h b/include/net/bluetooth/hci_sync.h
index 73e494b2591de..0756d6fe77d44 100644
--- a/include/net/bluetooth/hci_sync.h
+++ b/include/net/bluetooth/hci_sync.h
@@ -59,6 +59,7 @@ int __hci_cmd_sync_status(struct hci_dev *hdev, u16 opcode, u32 plen,
 int __hci_cmd_sync_status_sk(struct hci_dev *hdev, u16 opcode, u32 plen,
 			     const void *param, u8 event, u32 timeout,
 			     struct sock *sk);
+int __hci_reset_sync(struct hci_dev *hdev);
 int hci_cmd_sync_status(struct hci_dev *hdev, u16 opcode, u32 plen,
 			const void *param, u32 timeout);
 
@@ -84,9 +85,6 @@ void hci_cmd_sync_cancel_entry(struct hci_dev *hdev,
 			       struct hci_cmd_sync_work_entry *entry);
 bool hci_cmd_sync_dequeue(struct hci_dev *hdev, hci_cmd_sync_work_func_t func,
 			  void *data, hci_cmd_sync_work_destroy_t destroy);
-bool hci_cmd_sync_dequeue_once(struct hci_dev *hdev,
-			      hci_cmd_sync_work_func_t func, void *data,
-			      hci_cmd_sync_work_destroy_t destroy);
 
 int hci_update_eir_sync(struct hci_dev *hdev);
 int hci_update_class_sync(struct hci_dev *hdev);
diff --git a/include/net/bond_3ad.h b/include/net/bond_3ad.h
index 05572c19e14b7..ef667dff29729 100644
--- a/include/net/bond_3ad.h
+++ b/include/net/bond_3ad.h
@@ -302,8 +302,8 @@ void bond_3ad_state_machine_handler(struct work_struct *);
 void bond_3ad_initiate_agg_selection(struct bonding *bond, int timeout);
 void bond_3ad_adapter_speed_duplex_changed(struct slave *slave);
 void bond_3ad_handle_link_change(struct slave *slave, char link);
-int  bond_3ad_get_active_agg_info(struct bonding *bond, struct ad_info *ad_info);
-int  __bond_3ad_get_active_agg_info(struct bonding *bond,
+int  bond_3ad_get_active_agg_info(const struct bonding *bond, struct ad_info *ad_info);
+int  __bond_3ad_get_active_agg_info(const struct bonding *bond,
 				    struct ad_info *ad_info);
 int bond_3ad_lacpdu_recv(const struct sk_buff *skb, struct bonding *bond,
 			 struct slave *slave);
diff --git a/include/net/bonding.h b/include/net/bonding.h
index 2c54a36a8477b..598d56b1bc970 100644
--- a/include/net/bonding.h
+++ b/include/net/bonding.h
@@ -345,14 +345,14 @@ static inline bool bond_mode_uses_primary(int mode)
 	       mode == BOND_MODE_ALB;
 }
 
-static inline bool bond_uses_primary(struct bonding *bond)
+static inline bool bond_uses_primary(const struct bonding *bond)
 {
 	return bond_mode_uses_primary(BOND_MODE(bond));
 }
 
-static inline struct net_device *bond_option_active_slave_get_rcu(struct bonding *bond)
+static inline struct net_device *bond_option_active_slave_get_rcu(const struct bonding *bond)
 {
-	struct slave *slave = rcu_dereference_rtnl(bond->curr_active_slave);
+	const struct slave *slave = rcu_dereference_rtnl(bond->curr_active_slave);
 
 	return bond_uses_primary(bond) && slave ? slave->dev : NULL;
 }
@@ -703,7 +703,7 @@ void bond_setup(struct net_device *bond_dev);
 unsigned int bond_get_num_tx_queues(void);
 int bond_netlink_init(void);
 void bond_netlink_fini(void);
-struct net_device *bond_option_active_slave_get_rcu(struct bonding *bond);
+struct net_device *bond_option_active_slave_get_rcu(const struct bonding *bond);
 const char *bond_slave_link_status(s8 link);
 struct bond_vlan_tag *bond_verify_device_path(struct net_device *start_dev,
 					      struct net_device *end_dev,
diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h
index 8188ad200de53..4ec64aff88fff 100644
--- a/include/net/cfg80211.h
+++ b/include/net/cfg80211.h
@@ -1236,6 +1236,26 @@ ieee80211_chandef_max_power(struct cfg80211_chan_def *chandef)
 	return chandef->chan->max_power;
 }
 
+/**
+ * cfg80211_chandef_s1g_pri_width - return S1G primary width in MHz
+ *
+ * An S1G interface may have a primary channel width of either 1
+ * or 2MHz depending on whether chandef::s1g_primary_2mhz is set.
+ *
+ * Note: There is _always_ a 1MHz primary subchannel, regardless
+ * of the primary width. So chandef::chan always points to this
+ * 1MHz primary channel.
+ *
+ * @chandef: the chandef to use
+ *
+ * Returns: width in MHz of the S1G primary channel in use
+ */
+static inline int
+cfg80211_chandef_s1g_pri_width(struct cfg80211_chan_def *chandef)
+{
+	return chandef->s1g_primary_2mhz ? 2 : 1;
+}
+
 /**
  * cfg80211_any_usable_channels - check for usable channels
  * @wiphy: the wiphy to check for
@@ -5086,8 +5106,8 @@ struct mgmt_frame_regs {
  * @tdls_mgmt: Transmit a TDLS management frame.
  * @tdls_oper: Perform a high-level TDLS operation (e.g. TDLS link setup).
  *
- * @probe_client: probe an associated client, must return a cookie that it
- *	later passes to cfg80211_probe_status().
+ * @probe_peer: probe a connected peer (AP: STA MAC required; STA: no MAC),
+ *	must return a cookie that is later passed to cfg80211_probe_status().
  *
  * @set_noack_map: Set the NoAck Map for the TIDs.
  *
@@ -5488,8 +5508,8 @@ struct cfg80211_ops {
 	int	(*tdls_oper)(struct wiphy *wiphy, struct net_device *dev,
 			     const u8 *peer, enum nl80211_tdls_operation oper);
 
-	int	(*probe_client)(struct wiphy *wiphy, struct net_device *dev,
-				const u8 *peer, u64 *cookie);
+	int	(*probe_peer)(struct wiphy *wiphy, struct net_device *dev,
+			      const u8 *peer, u64 *cookie);
 
 	int	(*set_noack_map)(struct wiphy *wiphy,
 				  struct net_device *dev,
@@ -5690,7 +5710,6 @@ struct cfg80211_ops {
  *	set this flag to update channels on beacon hints.
  * @WIPHY_FLAG_SUPPORTS_NSTR_NONPRIMARY: support connection to non-primary link
  *	of an NSTR mobile AP MLD.
- * @WIPHY_FLAG_DISABLE_WEXT: disable wireless extensions for this device
  */
 enum wiphy_flags {
 	WIPHY_FLAG_SUPPORTS_EXT_KEK_KCK		= BIT(0),
@@ -5702,7 +5721,7 @@ enum wiphy_flags {
 	WIPHY_FLAG_4ADDR_STATION		= BIT(6),
 	WIPHY_FLAG_CONTROL_PORT_PROTOCOL	= BIT(7),
 	WIPHY_FLAG_IBSS_RSN			= BIT(8),
-	WIPHY_FLAG_DISABLE_WEXT			= BIT(9),
+	/* reuse bit 9 */
 	WIPHY_FLAG_MESH_AUTH			= BIT(10),
 	WIPHY_FLAG_SUPPORTS_EXT_KCK_32          = BIT(11),
 	WIPHY_FLAG_SUPPORTS_NSTR_NONPRIMARY	= BIT(12),
@@ -7228,7 +7247,7 @@ struct wireless_dev {
 	enum ieee80211_bss_type conn_bss_type;
 	u32 conn_owner_nlportid;
 
-	struct work_struct disconnect_wk;
+	struct wiphy_work disconnect_wk;
 	u8 disconnect_bssid[ETH_ALEN];
 
 	struct list_head event_list;
@@ -7265,7 +7284,7 @@ struct wireless_dev {
 
 	struct list_head pmsr_list;
 	spinlock_t pmsr_lock;
-	struct work_struct pmsr_free_wk;
+	struct wiphy_work pmsr_free_wk;
 
 	unsigned long unprot_beacon_reported;
 
@@ -9846,15 +9865,17 @@ bool cfg80211_rx_unexpected_4addr_frame(struct net_device *dev, const u8 *addr,
 /**
  * cfg80211_probe_status - notify userspace about probe status
  * @dev: the device the probe was sent on
- * @addr: the address of the peer
- * @cookie: the cookie filled in @probe_client previously
+ * @peer: The peer MAC address (or MLD address for MLO) or %NULL if not
+ *	applicable (e.g. for STA/P2P-client)
+ * @cookie: the cookie filled in @probe_peer previously
+ * @link_id: The link ID on which the probe was sent (or -1 for non-MLO)
  * @acked: indicates whether probe was acked or not
  * @ack_signal: signal strength (in dBm) of the ACK frame.
  * @is_valid_ack_signal: indicates the ack_signal is valid or not.
  * @gfp: allocation flags
  */
-void cfg80211_probe_status(struct net_device *dev, const u8 *addr,
-			   u64 cookie, bool acked, s32 ack_signal,
+void cfg80211_probe_status(struct net_device *dev, const u8 *peer, u64 cookie,
+			   int link_id, bool acked, s32 ack_signal,
 			   bool is_valid_ack_signal, gfp_t gfp);
 
 /**
diff --git a/include/net/devlink.h b/include/net/devlink.h
index dd546dbd57cf0..4830aba4087af 100644
--- a/include/net/devlink.h
+++ b/include/net/devlink.h
@@ -36,7 +36,7 @@ struct devlink_port_phys_attrs {
  * struct devlink_port_pci_pf_attrs - devlink port's PCI PF attributes
  * @controller: Associated controller number
  * @pf: associated PCI function number for the devlink port instance
- * @external: when set, indicates if a port is for an external controller
+ * @external: when set, indicates if a port is for an external host controller.
  */
 struct devlink_port_pci_pf_attrs {
 	u32 controller;
@@ -50,7 +50,7 @@ struct devlink_port_pci_pf_attrs {
  * @pf: associated PCI function number for the devlink port instance
  * @vf: associated PCI VF number of a PF for the devlink port instance;
  *	VF number starts from 0 for the first PCI virtual function
- * @external: when set, indicates if a port is for an external controller
+ * @external: when set, indicates if a port is for an external host controller.
  */
 struct devlink_port_pci_vf_attrs {
 	u32 controller;
@@ -64,7 +64,7 @@ struct devlink_port_pci_vf_attrs {
  * @controller: Associated controller number
  * @sf: associated SF number of a PF for the devlink port instance
  * @pf: associated PCI function number for the devlink port instance
- * @external: when set, indicates if a port is for an external controller
+ * @external: when set, indicates if a port is for an external host controller.
  */
 struct devlink_port_pci_sf_attrs {
 	u32 controller;
@@ -1594,6 +1594,15 @@ struct devlink_ops {
 				    struct devlink_rate *parent,
 				    void *priv_child, void *priv_parent,
 				    struct netlink_ext_ack *extack);
+	/* Indicates if cross-device rate nodes are supported.
+	 * This also requires a shared common ancestor object all devices that
+	 * could share rate nodes are nested in.
+	 * If enabled, rate operations may be called on an instance with only
+	 * the common ancestor lock held and *without that instance lock held*.
+	 * It is the driver's responsibility to ensure proper serialization
+	 * with other operations.
+	 */
+	bool supported_cross_device_rate_nodes;
 	/**
 	 * selftests_check() - queries if selftest is supported
 	 * @devlink: devlink instance
diff --git a/include/net/fib_rules.h b/include/net/fib_rules.h
index 7dee0ae616e32..c6b94790fa81e 100644
--- a/include/net/fib_rules.h
+++ b/include/net/fib_rules.h
@@ -82,7 +82,7 @@ struct fib_rules_ops {
 					     struct fib_rule_hdr *,
 					     struct nlattr **,
 					     struct netlink_ext_ack *);
-	int			(*delete)(struct fib_rule *);
+	void			(*delete)(struct fib_rule *);
 	int			(*compare)(struct fib_rule *,
 					   struct fib_rule_hdr *,
 					   struct nlattr **);
@@ -93,11 +93,13 @@ struct fib_rules_ops {
 	/* Called after modifications to the rules set, must flush
 	 * the route cache if one exists. */
 	void			(*flush_cache)(struct fib_rules_ops *ops);
+	bool			(*need_rtnl)(struct net *net);
 
 	int			nlgroup;
 	struct list_head	rules_list;
 	struct module		*owner;
 	struct net		*fro_net;
+	struct mutex		lock;
 	struct rcu_head		rcu;
 };
 
diff --git a/include/net/ieee80211_radiotap.h b/include/net/ieee80211_radiotap.h
index c60867e7e43c9..8bbaf77da7cfc 100644
--- a/include/net/ieee80211_radiotap.h
+++ b/include/net/ieee80211_radiotap.h
@@ -95,6 +95,8 @@ enum ieee80211_radiotap_presence {
 	IEEE80211_RADIOTAP_EXT = 31,
 	IEEE80211_RADIOTAP_EHT_USIG = 33,
 	IEEE80211_RADIOTAP_EHT = 34,
+	IEEE80211_RADIOTAP_UHR_ELR = 37,
+	IEEE80211_RADIOTAP_UHR = 38,
 };
 
 /* for IEEE80211_RADIOTAP_FLAGS */
@@ -602,6 +604,194 @@ enum ieee80211_radiotap_eht_usig_tb {
 	IEEE80211_RADIOTAP_EHT_USIG2_TB_B20_B25_TAIL		= 0xfc000000,
 };
 
+/*
+ * ieee80211_radiotap_uhr_elr - content of UHR-ELR TLV (type 37)
+ * see https://www.radiotap.org/fields/UHR-ELR for details
+ */
+struct ieee80211_radiotap_uhr_elr {
+	__le32 known;
+	__le32 sig1, sig2, mark;
+} __packed;
+
+enum ieee80211_radiotap_uhr_elr_known {
+	IEEE80211_RADIOTAP_UHR_ELR_KNOWN_VERSION_ID		= 0x00000001,
+	IEEE80211_RADIOTAP_UHR_ELR_KNOWN_UL_DL			= 0x00000002,
+	IEEE80211_RADIOTAP_UHR_ELR_KNOWN_MCS			= 0x00000004,
+	IEEE80211_RADIOTAP_UHR_ELR_KNOWN_CODING			= 0x00000008,
+	IEEE80211_RADIOTAP_UHR_ELR_KNOWN_LENGTH			= 0x00000010,
+	IEEE80211_RADIOTAP_UHR_ELR_KNOWN_LDPC_EXTRA_OFDM_SYM	= 0x00000020,
+	IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_1_CRC		= 0x00000040,
+	IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_1_TAIL		= 0x00000080,
+	IEEE80211_RADIOTAP_UHR_ELR_KNOWN_STA_ID			= 0x00000100,
+	IEEE80211_RADIOTAP_UHR_ELR_KNOWN_DISREGARD		= 0x00000200,
+	IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_2_CRC		= 0x00000400,
+	IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_2_TAIL		= 0x00000800,
+	IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_1_CRC_CHECKED	= 0x00001000,
+	IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_2_CRC_CHECKED	= 0x00002000,
+	IEEE80211_RADIOTAP_UHR_ELR_KNOWN_MARK_BSS_COLOR		= 0x00010000,
+};
+
+enum ieee80211_radiotap_uhr_elr_sig1 {
+	IEEE80211_RADIOTAP_UHR_ELR_SIG1_VERSION_ID		= 0x00000001,
+	IEEE80211_RADIOTAP_UHR_ELR_SIG1_UL_DL			= 0x00000002,
+	IEEE80211_RADIOTAP_UHR_ELR_SIG1_MCS			= 0x00000004,
+	IEEE80211_RADIOTAP_UHR_ELR_SIG1_CODING			= 0x00000008,
+	IEEE80211_RADIOTAP_UHR_ELR_SIG1_LENGTH			= 0x00001FF0,
+	IEEE80211_RADIOTAP_UHR_ELR_SIG1_LDPC_EXTRA_OFDM_SYM	= 0x00002000,
+	IEEE80211_RADIOTAP_UHR_ELR_SIG1_CRC			= 0x0003C000,
+	IEEE80211_RADIOTAP_UHR_ELR_SIG1_TAIL			= 0x00FC0000,
+	IEEE80211_RADIOTAP_UHR_ELR_SIG1_CRC_VALID		= 0x80000000,
+};
+
+enum ieee80211_radiotap_uhr_elr_sig2 {
+	IEEE80211_RADIOTAP_UHR_ELR_SIG2_STA_ID			= 0x000007FF,
+	IEEE80211_RADIOTAP_UHR_ELR_SIG2_DISREGARD		= 0x00003800,
+	IEEE80211_RADIOTAP_UHR_ELR_SIG2_CRC			= 0x0003C000,
+	IEEE80211_RADIOTAP_UHR_ELR_SIG2_TAIL			= 0x00FC0000,
+	IEEE80211_RADIOTAP_UHR_ELR_SIG2_CRC_VALID		= 0x80000000,
+};
+
+enum ieee80211_radiotap_uhr_elr_mark {
+	IEEE80211_RADIOTAP_UHR_ELR_MARK_BSS_COLOR		= 0x0000003F,
+};
+
+/*
+ * ieee80211_radiotap_uhr - content of UHR TLV (type 38)
+ * see https://www.radiotap.org/fields/UHR for details
+ */
+struct ieee80211_radiotap_uhr {
+	__le32 known;
+	__le32 data[9];
+	struct {
+		__le32 known, info;
+	} user[];
+} __packed;
+
+enum ieee80211_radiotap_uhr_known {
+	IEEE80211_RADIOTAP_UHR_KNOWN_SPATIAL_REUSE		= 0x00000001,
+	IEEE80211_RADIOTAP_UHR_KNOWN_GI_LTF_SIZE		= 0x00000002,
+	IEEE80211_RADIOTAP_UHR_KNOWN_NUMBER_OF_UHR_LTF_SYMBOLS	= 0x00000004,
+	IEEE80211_RADIOTAP_UHR_KNOWN_LDPC_EXTRA_SYMBOL_SEGMENT	= 0x00000008,
+	IEEE80211_RADIOTAP_UHR_KNOWN_PRE_FEC_PADDING_FACTOR	= 0x00000010,
+	IEEE80211_RADIOTAP_UHR_KNOWN_PE_DISAMBIGUITY		= 0x00000020,
+	IEEE80211_RADIOTAP_UHR_KNOWN_DISREGARD_OFDMA		= 0x00000040,
+	IEEE80211_RADIOTAP_UHR_KNOWN_CRC1			= 0x00000080,
+	IEEE80211_RADIOTAP_UHR_KNOWN_TAIL1			= 0x00000100,
+	IEEE80211_RADIOTAP_UHR_KNOWN_CRC2			= 0x00000200,
+	IEEE80211_RADIOTAP_UHR_KNOWN_TAIL2			= 0x00000400,
+	IEEE80211_RADIOTAP_UHR_KNOWN_INTERFERENCE_MITIGATION	= 0x00000800,
+	IEEE80211_RADIOTAP_UHR_KNOWN_DISREGARD_NON_OFDMA	= 0x00001000,
+	IEEE80211_RADIOTAP_UHR_KNOWN_NUMBER_OF_NON_OFDMA_USERS	= 0x00002000,
+	IEEE80211_RADIOTAP_UHR_KNOWN_COMMON_ENCODING_BLOCK_CRC	= 0x00004000,
+	IEEE80211_RADIOTAP_UHR_KNOWN_COMMON_ENCODING_BLOCK_TAIL	= 0x00008000,
+	IEEE80211_RADIOTAP_UHR_KNOWN_RU_MRU_DRU_SIZE		= 0x00010000,
+	IEEE80211_RADIOTAP_UHR_KNOWN_RU_MRU_INDEX		= 0x00020000,
+	IEEE80211_RADIOTAP_UHR_KNOWN_DRU_RRU_ALLOC_TB_FMT	= 0x00040000,
+	IEEE80211_RADIOTAP_UHR_KNOWN_PRI80_CHAN_POS		= 0x00080000,
+};
+
+enum ieee80211_radiotap_uhr_data {
+	/* data[0] */
+	IEEE80211_RADIOTAP_UHR_DATA0_SPATIAL_REUSE		= 0x0000000F,
+	IEEE80211_RADIOTAP_UHR_DATA0_GI_LTF_SIZE		= 0x00000030,
+	IEEE80211_RADIOTAP_UHR_DATA0_NUMBER_OF_LTF_SYMBOLS	= 0x00000700,
+	IEEE80211_RADIOTAP_UHR_DATA0_LDPC_EXTRA_SYMBOL_SEGMENT	= 0x00000800,
+	IEEE80211_RADIOTAP_UHR_DATA0_PRE_FEC_PADDING_FACTOR	= 0x00003000,
+	IEEE80211_RADIOTAP_UHR_DATA0_PE_DISAMBIGUITY		= 0x00004000,
+	IEEE80211_RADIOTAP_UHR_DATA0_DISREGARD_OFDMA		= 0x00078000,
+	IEEE80211_RADIOTAP_UHR_DATA0_CRC1			= 0x00780000,
+	IEEE80211_RADIOTAP_UHR_DATA0_TAIL1			= 0x1f800000,
+	/* data[1] */
+	IEEE80211_RADIOTAP_UHR_DATA1_RU_MRU_DRU_SIZE		= 0x0000001f,
+	IEEE80211_RADIOTAP_UHR_DATA1_RU_MRU_INDEX		= 0x00001fe0,
+	IEEE80211_RADIOTAP_UHR_DATA1_RU_ALLOC_CC_1_1_1		= 0x003fe000,
+	IEEE80211_RADIOTAP_UHR_DATA1_RU_ALLOC_CC_1_1_1_KNOWN	= 0x00400000,
+	IEEE80211_RADIOTAP_UHR_DATA1_PRI80_CHAN_POS		= 0xc0000000,
+	/* data[2] */
+	IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_2_1_1		= 0x000001ff,
+	IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_2_1_1_KNOWN	= 0x00000200,
+	IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_1_1_2		= 0x0007fc00,
+	IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_1_1_2_KNOWN	= 0x00080000,
+	IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_2_1_2		= 0x1ff00000,
+	IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_2_1_2_KNOWN	= 0x20000000,
+	/* data[3] */
+	IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_1_2_1		= 0x000001ff,
+	IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_1_2_1_KNOWN	= 0x00000200,
+	IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_2_2_1		= 0x0007fc00,
+	IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_2_2_1_KNOWN	= 0x00080000,
+	IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_1_2_2		= 0x1ff00000,
+	IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_1_2_2_KNOWN	= 0x20000000,
+	/* data[4] */
+	IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_2_2_2		= 0x000001ff,
+	IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_2_2_2_KNOWN	= 0x00000200,
+	IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_1_2_3		= 0x0007fc00,
+	IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_1_2_3_KNOWN	= 0x00080000,
+	IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_2_2_3		= 0x1ff00000,
+	IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_2_2_3_KNOWN	= 0x20000000,
+	/* data[5] */
+	IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_1_2_4		= 0x000001ff,
+	IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_1_2_4_KNOWN	= 0x00000200,
+	IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_2_2_4		= 0x0007fc00,
+	IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_2_2_4_KNOWN	= 0x00080000,
+	IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_1_2_5		= 0x1ff00000,
+	IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_1_2_5_KNOWN	= 0x20000000,
+	/* data[6] */
+	IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_2_2_5		= 0x000001ff,
+	IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_2_2_5_KNOWN	= 0x00000200,
+	IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_1_2_6		= 0x0007fc00,
+	IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_1_2_6_KNOWN	= 0x00080000,
+	IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_2_2_6		= 0x1ff00000,
+	IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_2_2_6_KNOWN	= 0x20000000,
+	/* data[7] */
+	IEEE80211_RADIOTAP_UHR_DATA7_CRC2			= 0x0000000f,
+	IEEE80211_RADIOTAP_UHR_DATA7_TAIL2			= 0x000003f0,
+	IEEE80211_RADIOTAP_UHR_DATA7_INTERFERENCE_MITIGATION	= 0x00000400,
+	IEEE80211_RADIOTAP_UHR_DATA7_DISREGARD_NON_OFDMA	= 0x00001800,
+	IEEE80211_RADIOTAP_UHR_DATA7_NUMBER_OF_NON_OFDMA_USERS	= 0x0000e000,
+	IEEE80211_RADIOTAP_UHR_DATA7_COMMON_ENCODING_BLOCK_CRC	= 0x000f0000,
+	IEEE80211_RADIOTAP_UHR_DATA7_COMMON_ENCODING_BLOCK_TAIL	= 0x03f00000,
+	/* data[8] */
+	IEEE80211_RADIOTAP_UHR_DATA8_DRU_RRU_ALLOC_TB_FMT_PS_160= 0x00000001,
+	IEEE80211_RADIOTAP_UHR_DATA8_DRU_RRU_ALLOC_TB_FMT_B0	= 0x00000002,
+	IEEE80211_RADIOTAP_UHR_DATA8_DRU_RRU_ALLOC_TB_FMT_B7_B1	= 0x000001fc,
+	IEEE80211_RADIOTAP_UHR_DATA8_DRU_RRU_INDICATION		= 0x00000200,
+};
+
+enum ieee80211_radiotap_uhr_user_known {
+	IEEE80211_RADIOTAP_UHR_USER_KNOWN_STA_ID		= 0x00000001,
+	IEEE80211_RADIOTAP_UHR_USER_KNOWN_MCS			= 0x00000002,
+	IEEE80211_RADIOTAP_UHR_USER_KNOWN_NSS			= 0x00000004,
+	IEEE80211_RADIOTAP_UHR_USER_KNOWN_UEQM			= 0x00000008,
+	IEEE80211_RADIOTAP_UHR_USER_KNOWN_BF			= 0x00000010,
+	IEEE80211_RADIOTAP_UHR_USER_KNOWN_CODING		= 0x00000020,
+	IEEE80211_RADIOTAP_UHR_USER_KNOWN_UEQM_PATTERN		= 0x00000040,
+	IEEE80211_RADIOTAP_UHR_USER_KNOWN_2X_LDPC		= 0x00000080,
+	IEEE80211_RADIOTAP_UHR_USER_KNOWN_SPATIAL_CONFIG	= 0x00000100,
+	IEEE80211_RADIOTAP_UHR_USER_KNOWN_DISREGARD		= 0x00000200,
+	IEEE80211_RADIOTAP_UHR_USER_KNOWN_BSS_COLOR_INDICATION	= 0x00000400,
+	IEEE80211_RADIOTAP_UHR_USER_KNOWN_USR_ENC_BLK_CRC	= 0x00000800,
+	IEEE80211_RADIOTAP_UHR_USER_KNOWN_USR_ENC_BLK_TAIL	= 0x00001000,
+	/* really 'known' but actual data */
+	IEEE80211_RADIOTAP_UHR_USER_KNOWN_DATA_USR_ENC_BLK_CRC	= 0x000f0000,
+	IEEE80211_RADIOTAP_UHR_USER_KNOWN_DATA_USR_ENC_BLK_TAIL	= 0x03f00000,
+	/* indicates this user was captured */
+	IEEE80211_RADIOTAP_UHR_USER_KNOWN_USER_CAPTURED		= 0x80000000,
+};
+
+enum ieee80211_radiotap_uhr_user_info {
+	IEEE80211_RADIOTAP_UHR_USER_INFO_STA_ID			= 0x000007ff,
+	IEEE80211_RADIOTAP_UHR_USER_INFO_MCS			= 0x0000f800,
+	IEEE80211_RADIOTAP_UHR_USER_INFO_NSS			= 0x00070000,
+	IEEE80211_RADIOTAP_UHR_USER_INFO_SPATIAL_CONFIG		= 0x000f0000,
+	IEEE80211_RADIOTAP_UHR_USER_INFO_UEQM			= 0x00100000,
+	IEEE80211_RADIOTAP_UHR_USER_INFO_DISREGARD		= 0x00100000,
+	IEEE80211_RADIOTAP_UHR_USER_INFO_BF			= 0x00200000,
+	IEEE80211_RADIOTAP_UHR_USER_INFO_BSS_COLOR_INDICATION	= 0x00200000,
+	IEEE80211_RADIOTAP_UHR_USER_INFO_UEQM_PATTERN		= 0x00c00000,
+	IEEE80211_RADIOTAP_UHR_USER_INFO_CODING			= 0x01000000,
+	IEEE80211_RADIOTAP_UHR_USER_INFO_2X_LDPC		= 0x02000000,
+};
+
 /**
  * ieee80211_get_radiotap_len - get radiotap header length
  * @data: pointer to the header
diff --git a/include/net/ip_fib.h b/include/net/ip_fib.h
index c63a3c4967ae6..0a35355fb0f33 100644
--- a/include/net/ip_fib.h
+++ b/include/net/ip_fib.h
@@ -302,7 +302,8 @@ static inline struct fib_table *fib_get_table(struct net *net, u32 id)
 		&net->ipv4.fib_table_hash[TABLE_LOCAL_INDEX] :
 		&net->ipv4.fib_table_hash[TABLE_MAIN_INDEX];
 
-	tb_hlist = rcu_dereference_rtnl(hlist_first_rcu(ptr));
+	/* Only fib4_rules_init() adds fib_table. */
+	tb_hlist = rcu_dereference_protected(hlist_first_rcu(ptr), true);
 
 	return hlist_entry(tb_hlist, struct fib_table, tb_hlist);
 }
diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h
index 417ff51f62fc8..1737fbfce2d0f 100644
--- a/include/net/ip_vs.h
+++ b/include/net/ip_vs.h
@@ -2124,7 +2124,7 @@ void ip_vs_update_conntrack(struct sk_buff *skb, struct ip_vs_conn *cp,
 			    int outin);
 int ip_vs_confirm_conntrack(struct sk_buff *skb);
 void ip_vs_nfct_expect_related(struct sk_buff *skb, struct nf_conn *ct,
-			       struct ip_vs_conn *cp, u_int8_t proto,
+			       struct ip_vs_conn *cp, u8 proto,
 			       const __be16 port, int from_rs);
 void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp);
 
diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h
index 0c395917b2144..8529cef0d7c45 100644
--- a/include/net/mana/gdma.h
+++ b/include/net/mana/gdma.h
@@ -47,6 +47,7 @@ enum gdma_queue_type {
 	GDMA_RQ,
 	GDMA_CQ,
 	GDMA_EQ,
+	GDMA_DIM,
 };
 
 enum gdma_work_request_flags {
@@ -126,6 +127,17 @@ union gdma_doorbell_entry {
 		u64 tail_ptr	: 31;
 		u64 arm		: 1;
 	} eq;
+
+	struct {
+		u64 id           : 24;
+		u64 reserved     : 8;
+		u64 mod_usec     : 10;
+		u64 reserve1     : 5;
+		u64 mod_usec_vld : 1;
+		u64 mod_comps    : 8;
+		u64 reserve2     : 7;
+		u64 mod_comps_vld: 1;
+	} dim;
 }; /* HW DATA */
 
 struct gdma_msg_hdr {
@@ -502,6 +514,9 @@ void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit);
 
 int mana_schedule_serv_work(struct gdma_context *gc, enum gdma_eqe_type type);
 
+void mana_gd_ring_dim(struct gdma_queue *cq, u32 mod_usec, bool mod_usec_vld,
+		      u32 mod_comps, bool mod_comps_vld);
+
 struct gdma_wqe {
 	u32 reserved	:24;
 	u32 last_vbytes	:8;
@@ -650,6 +665,9 @@ enum {
 /* Driver supports self recovery on Hardware Channel timeouts */
 #define GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY BIT(25)
 
+/* Driver supports dynamic interrupt moderation - DIM */
+#define GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION BIT(28)
+
 #define GDMA_DRV_CAP_FLAGS1 \
 	(GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \
 	 GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX | \
@@ -665,7 +683,8 @@ enum {
 	 GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY | \
 	 GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY | \
 	 GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY | \
-	 GDMA_DRV_CAP_FLAG_1_EQ_MSI_UNSHARE_MULTI_VPORT)
+	 GDMA_DRV_CAP_FLAG_1_EQ_MSI_UNSHARE_MULTI_VPORT | \
+	 GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION)
 
 #define GDMA_DRV_CAP_FLAGS2 0
 
@@ -701,6 +720,9 @@ struct gdma_verify_ver_req {
 	u8 os_ver_str4[128];
 }; /* HW DATA */
 
+/* HW supports dynamic interrupt moderation - DIM */
+#define GDMA_PF_CAP_FLAG_1_DYN_INTERRUPT_MODERATION BIT(15)
+
 struct gdma_verify_ver_resp {
 	struct gdma_resp_hdr hdr;
 	u64 gdma_protocol_ver;
diff --git a/include/net/mana/mana.h b/include/net/mana/mana.h
index 04acb6791dbd9..4d041fb8437f5 100644
--- a/include/net/mana/mana.h
+++ b/include/net/mana/mana.h
@@ -4,6 +4,7 @@
 #ifndef _MANA_H
 #define _MANA_H
 
+#include <linux/dim.h>
 #include <net/xdp.h>
 #include <net/net_shaper.h>
 
@@ -64,6 +65,19 @@ enum TRI_STATE {
 /* Maximum number of packets per coalesced CQE */
 #define MANA_RXCOMP_OOB_NUM_PPI 4
 
+/* Default/max interrupt moderation settings */
+#define MANA_INTR_MODR_USEC_DEF 0
+#define MANA_INTR_MODR_COMP_DEF 0
+
+#define MANA_ADAPTIVE_RX_DEF true
+#define MANA_ADAPTIVE_TX_DEF true
+
+/* DIM doorbell value field layout */
+#define MANA_INTR_MODR_USEC_MAX    GENMASK(9, 0)
+#define MANA_INTR_MODR_USEC_VLD    BIT(15)
+#define MANA_INTR_MODR_COMP_MAX    GENMASK(7, 0)
+#define MANA_INTR_MODR_COMP_MASK   GENMASK(23, 16)
+
 /* Update this count whenever the respective structures are changed */
 #define MANA_STATS_RX_COUNT (6 + MANA_RXCOMP_OOB_NUM_PPI - 1)
 #define MANA_STATS_TX_COUNT 11
@@ -297,6 +311,17 @@ struct mana_cq {
 	int work_done;
 	int work_done_since_doorbell;
 	int budget;
+
+	/* DIM - Dynamic Interrupt Moderation */
+	struct dim dim;
+	u16 dim_event_ctr;
+
+	/* Cumulative TX completions fed to DIM. Updated and read only in
+	 * NAPI context (mana_poll_tx_cq() / mana_update_tx_dim()), so they
+	 * measure the hardware completion rate and need no u64_stats_sync.
+	 */
+	u64 tx_dim_pkts;
+	u64 tx_dim_bytes;
 };
 
 struct mana_recv_buf_oob {
@@ -505,6 +530,9 @@ struct mana_port_context {
 	struct net_device *ndev;
 	struct work_struct queue_reset_work;
 
+	/* Debug knob to log TX timeout but skip recovery reset */
+	bool tx_timeout_skip_reset;
+
 	u8 mac_addr[ETH_ALEN];
 
 	struct mana_eq *eqs;
@@ -581,6 +609,15 @@ struct mana_port_context {
 	u8 cqe_coalescing_enable;
 	u32 cqe_coalescing_timeout_ns;
 
+	/* Interrupt moderation settings */
+	u16 intr_modr_rx_usec;
+	u16 intr_modr_rx_comp;
+	u16 intr_modr_tx_usec;
+	u16 intr_modr_tx_comp;
+
+	bool rx_dim_enabled;
+	bool tx_dim_enabled;
+
 	struct mana_ethtool_stats eth_stats;
 
 	struct mana_ethtool_phy_stats phy_stats;
@@ -606,6 +643,8 @@ int mana_alloc_queues(struct net_device *ndev);
 int mana_attach(struct net_device *ndev);
 int mana_detach(struct net_device *ndev, bool from_close);
 
+void mana_dim_change(struct mana_cq *cq, bool enable);
+
 int mana_probe(struct gdma_dev *gd, bool resuming);
 void mana_remove(struct gdma_dev *gd, bool suspending);
 
@@ -641,6 +680,9 @@ struct mana_obj_spec {
 	u32 queue_size;
 	u32 attached_eq;
 	u32 modr_ctx_id;
+	u8 req_cq_moderation;
+	u16 cq_moderation_comp;
+	u16 cq_moderation_usec;
 };
 
 enum mana_command_code {
@@ -772,6 +814,15 @@ struct mana_create_wqobj_req {
 	u32 cq_size;
 	u32 cq_moderation_ctx_id;
 	u32 cq_parent_qid;
+
+	/* V2 */
+	u8 allow_rqwqe_chain;
+
+	/* V3 */
+	u8 req_cq_moderation;
+	u16 cq_moderation_comp;
+	u16 cq_moderation_usec;
+	u8 reserved2[2];
 }; /* HW DATA */
 
 struct mana_create_wqobj_resp {
@@ -779,6 +830,12 @@ struct mana_create_wqobj_resp {
 	u32 wq_id;
 	u32 cq_id;
 	mana_handle_t wq_obj;
+
+	/* V2 */
+	u16 cq_moderation_comp;
+	u16 cq_moderation_usec;
+	u8 cq_moderation_enabled;
+	u8 reserved1[3];
 }; /* HW DATA */
 
 /* Destroy WQ Object */
diff --git a/include/net/net_namespace.h b/include/net/net_namespace.h
index 80de5e98a66d6..501af1999fe83 100644
--- a/include/net/net_namespace.h
+++ b/include/net/net_namespace.h
@@ -197,6 +197,9 @@ struct net {
 #ifdef CONFIG_DEBUG_NET_SMALL_RTNL
 	/* Move to a better place when the config guard is removed. */
 	struct mutex		rtnl_mutex;
+	struct work_struct	rtnl_work;
+	struct list_head	dev_unreg_head;
+	spinlock_t		dev_unreg_lock;
 #endif
 #if IS_ENABLED(CONFIG_VSOCKETS)
 	struct netns_vsock	vsock;
diff --git a/include/net/netfilter/nf_conntrack_helper.h b/include/net/netfilter/nf_conntrack_helper.h
index c761cd8158b25..bc5427d239f4c 100644
--- a/include/net/netfilter/nf_conntrack_helper.h
+++ b/include/net/netfilter/nf_conntrack_helper.h
@@ -43,11 +43,10 @@ struct nf_conntrack_helper {
 
 	refcount_t ct_refcnt;
 
-	/* Tuple of things we will help (compared against server response) */
-	struct nf_conntrack_tuple tuple;
+	u8 nfproto;	/* NFPROTO_*, can be NFPROTO_UNSPEC */
+	u8 l4proto;	/* IPPROTO_UDP/TCP */
 
-	/* Function to call when data passes; return verdict, or -1 to
-           invalidate. */
+	/* Function to call when data passes; return verdict */
 	int __rcu (*help)(struct sk_buff *skb, unsigned int protoff,
 			  struct nf_conn *ct,
 			  enum ip_conntrack_info conntrackinfo);
@@ -94,8 +93,7 @@ struct nf_conntrack_helper *nf_conntrack_helper_try_module_get(const char *name,
 void nf_conntrack_helper_put(struct nf_conntrack_helper *helper);
 
 void nf_ct_helper_init(struct nf_conntrack_helper *helper,
-		       u16 l3num, u16 protonum, const char *name,
-		       u16 default_port, u16 spec_port, u32 id,
+		       u8 l3num, u16 protonum, const char *name,
 		       const struct nf_conntrack_expect_policy *exp_pol,
 		       u32 expect_class_max,
 		       int (*help)(struct sk_buff *skb, unsigned int protoff,
diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h
index 6e27c56514df5..cb7f8bf15671c 100644
--- a/include/net/netns/ipv4.h
+++ b/include/net/netns/ipv4.h
@@ -118,6 +118,7 @@ struct netns_ipv4 {
 	struct fib_rules_ops	*rules_ops;
 	struct fib_table __rcu	*fib_main;
 	struct fib_table __rcu	*fib_default;
+	spinlock_t		fib_table_hash_lock;
 	unsigned int		fib_rules_require_fldissect;
 	bool			fib_has_custom_rules;
 #endif
diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h
index affee44bd38e3..cccc662561aac 100644
--- a/include/net/sctp/structs.h
+++ b/include/net/sctp/structs.h
@@ -312,7 +312,8 @@ struct sctp_cookie {
 
 	__u8 auth_random[sizeof(struct sctp_paramhdr) +
 			 SCTP_AUTH_RANDOM_LENGTH];
-	__u8 auth_hmacs[SCTP_AUTH_NUM_HMACS * sizeof(__u16) + 2];
+	__u8 auth_hmacs[sizeof(struct sctp_paramhdr) +
+			SCTP_AUTH_NUM_HMACS * sizeof(__u16)];
 	__u8 auth_chunks[sizeof(struct sctp_paramhdr) + SCTP_AUTH_MAX_CHUNKS];
 
 	/* This is a shim for my peer's INIT packet, followed by
diff --git a/include/net/udp.h b/include/net/udp.h
index 8262e2b215b4e..1fee17274745f 100644
--- a/include/net/udp.h
+++ b/include/net/udp.h
@@ -430,7 +430,7 @@ struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb,
 				       netdev_features_t features,
 				       bool is_ipv6);
 int udp_lib_getsockopt(struct sock *sk, int level, int optname,
-		       char __user *optval, int __user *optlen);
+		       sockopt_t *opt);
 int udp_lib_setsockopt(struct sock *sk, int level, int optname,
 		       sockptr_t optval, unsigned int optlen,
 		       int (*push_pending_frames)(struct sock *));
diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index 519a0156a05ca..a6d69aaa6cd2d 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -162,6 +162,8 @@ struct xfrm_dev_offload {
 	 */
 	struct net_device	*real_dev;
 	unsigned long		offload_handle;
+	/* Snapshot the attached device index for dump paths. */
+	int			ifindex;
 	u8			dir : 2;
 	u8			type : 2;
 	u8			flags : 2;
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 794746de8db0b..8d82d303b723a 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -407,36 +407,36 @@ struct ib_device_attr {
 	u32			vendor_id;
 	u32			vendor_part_id;
 	u32			hw_ver;
-	int			max_qp;
-	int			max_qp_wr;
+	u32			max_qp;
+	u32			max_qp_wr;
 	u64			device_cap_flags;
 	u64			kernel_cap_flags;
-	int			max_send_sge;
-	int			max_recv_sge;
-	int			max_sge_rd;
-	int			max_cq;
-	int			max_cqe;
-	int			max_mr;
-	int			max_pd;
-	int			max_qp_rd_atom;
-	int			max_ee_rd_atom;
-	int			max_res_rd_atom;
-	int			max_qp_init_rd_atom;
-	int			max_ee_init_rd_atom;
+	u32			max_send_sge;
+	u32			max_recv_sge;
+	u32			max_sge_rd;
+	u32			max_cq;
+	u32			max_cqe;
+	u32			max_mr;
+	u32			max_pd;
+	u32			max_qp_rd_atom;
+	u32			max_ee_rd_atom;
+	u32			max_res_rd_atom;
+	u32			max_qp_init_rd_atom;
+	u32			max_ee_init_rd_atom;
 	enum ib_atomic_cap	atomic_cap;
 	enum ib_atomic_cap	masked_atomic_cap;
-	int			max_ee;
-	int			max_rdd;
-	int			max_mw;
-	int			max_raw_ipv6_qp;
-	int			max_raw_ethy_qp;
-	int			max_mcast_grp;
-	int			max_mcast_qp_attach;
-	int			max_total_mcast_qp_attach;
-	int			max_ah;
-	int			max_srq;
-	int			max_srq_wr;
-	int			max_srq_sge;
+	u32			max_ee;
+	u32			max_rdd;
+	u32			max_mw;
+	u32			max_raw_ipv6_qp;
+	u32			max_raw_ethy_qp;
+	u32			max_mcast_grp;
+	u32			max_mcast_qp_attach;
+	u32			max_total_mcast_qp_attach;
+	u32			max_ah;
+	u32			max_srq;
+	u32			max_srq_wr;
+	u32			max_srq_sge;
 	unsigned int		max_fast_reg_page_list_len;
 	unsigned int		max_pi_fast_reg_page_list_len;
 	u16			max_pkeys;
diff --git a/include/rdma/restrack.h b/include/rdma/restrack.h
index 451f99e3717d8..c081384740ce9 100644
--- a/include/rdma/restrack.h
+++ b/include/rdma/restrack.h
@@ -123,7 +123,7 @@ struct rdma_restrack_entry {
 	u32 id;
 };
 
-int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type,
+u32 rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type,
 			bool show_details);
 /**
  * rdma_is_kernel_res() - check the owner of resource
diff --git a/include/rdma/uverbs_ioctl.h b/include/rdma/uverbs_ioctl.h
index 24fd362130233..80f3ba6663d06 100644
--- a/include/rdma/uverbs_ioctl.h
+++ b/include/rdma/uverbs_ioctl.h
@@ -1151,4 +1151,24 @@ static inline int ib_respond_empty_udata(struct ib_udata *udata)
 	return 0;
 }
 
+/**
+ * ib_no_udata_io - Ensure no input data and zero fill the response buffer
+ * @udata: The system call's ib_udata struct
+ *
+ * Driver ops which do not accept any input data and do not provide any response
+ * data may call this at the beginning of their handler to fully adhere to the
+ * uAPI forward/backward compatibility rules.
+ *
+ * Return: Negative failure code if the op should be denied, 0 otherwise.
+ */
+static inline int ib_no_udata_io(struct ib_udata *udata)
+{
+	int ret = ib_is_udata_in_empty(udata);
+
+	if (ret)
+		return ret;
+
+	return ib_respond_empty_udata(udata);
+}
+
 #endif
diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h
index 029f5115b2ea0..8694eeadd753e 100644
--- a/include/scsi/scsi_device.h
+++ b/include/scsi/scsi_device.h
@@ -408,7 +408,6 @@ void scsi_attach_vpd(struct scsi_device *sdev);
 void scsi_cdl_check(struct scsi_device *sdev);
 int scsi_cdl_enable(struct scsi_device *sdev, bool enable);
 
-extern struct scsi_device *scsi_device_from_queue(struct request_queue *q);
 extern int __must_check scsi_device_get(struct scsi_device *);
 extern void scsi_device_put(struct scsi_device *);
 extern struct scsi_device *scsi_device_lookup(struct Scsi_Host *,
diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h
index 7e2011830ba4b..f6b286fa59f21 100644
--- a/include/scsi/scsi_host.h
+++ b/include/scsi/scsi_host.h
@@ -750,6 +750,9 @@ struct Scsi_Host {
 	 */
 	struct device *dma_dev;
 
+	/* Used for an rcu-synchronizing eh wakeup */
+	struct work_struct eh_work;
+
 	/* Delay for runtime autosuspend */
 	int rpm_autosuspend_delay;
 
diff --git a/include/scsi/scsi_proto.h b/include/scsi/scsi_proto.h
index f64385cde5b93..c6b4a4dc8d9ca 100644
--- a/include/scsi/scsi_proto.h
+++ b/include/scsi/scsi_proto.h
@@ -129,6 +129,10 @@
 #define SAI_GET_LBA_STATUS    0x12
 #define SAI_REPORT_REFERRALS  0x13
 #define SAI_GET_STREAM_STATUS 0x16
+#define SAI_GET_PHYSICAL_ELEMENT_STATUS 0x17
+#define SAI_REMOVE_ELEMENT_AND_TRUNCATE 0x18
+#define SAI_RESTORE_ELEMENTS_AND_REBUILD 0x19
+#define SAI_REMOVE_ELEMENT_AND_MODIFY_ZONES 0x1a
 /* values for maintenance in */
 #define MI_REPORT_IDENTIFYING_INFORMATION 0x05
 #define MI_REPORT_TARGET_PGS  0x0a
@@ -233,6 +237,57 @@ enum sam_status {
 #define MISCOMPARE          0x0e
 #define COMPLETED	    0x0f
 
+/*
+ * Additional Sense Codes (ASC).
+ */
+#define NO_ADDITIONAL_SENSE		0x00
+#define OVERLAP_ATOMIC_COMMAND_ASC	0x00
+#define LOGICAL_UNIT_NOT_READY		0x04
+#define LOGICAL_UNIT_COMMUNICATION_FAILURE 0x8
+#define WRITE_ERROR_ASC			0x0c
+#define UNRECOVERED_READ_ERR		0x11
+#define PARAMETER_LIST_LENGTH_ERR	0x1a
+#define MISCOMPARE_VERIFY_ASC		0x1d
+#define INVALID_OPCODE			0x20
+#define LBA_OUT_OF_RANGE		0x21
+#define INVALID_FIELD_IN_CDB		0x24
+#define INVALID_FIELD_IN_PARAM_LIST	0x26
+#define WRITE_PROTECTED			0x27
+#define UA_READY_ASC			0x28
+#define UA_RESET_ASC			0x29
+#define UA_CHANGED_ASC			0x2a
+#define TOO_MANY_IN_PARTITION_ASC	0x3b
+#define TARGET_CHANGED_ASC		0x3f
+#define SAVING_PARAMS_UNSUP		0x39
+#define TRANSPORT_PROBLEM		0x4b
+#define INSUFF_RES_ASC			0x55
+#define LOW_POWER_COND_ON		0x5e
+#define THRESHOLD_EXCEEDED		0x5d
+
+/*
+ * Additional Sense Code Qualifiers (ASCQ).
+ */
+#define POWER_ON_RESET_ASCQ		0x00
+#define MODE_CHANGED_ASCQ		0x01	/* mode parameters changed */
+#define FILEMARK_DETECTED_ASCQ		0x01
+#define POWER_ON_OCCURRED_ASCQ		0x01
+#define MICROCODE_CHANGED_ASCQ		0x01	/* with TARGET_CHANGED_ASC */
+#define BUS_RESET_ASCQ			0x02	/* scsi bus reset occurred */
+#define EOP_EOM_DETECTED_ASCQ		0x02
+#define INSUFF_RES_ASCQ			0x03
+#define BEGINNING_OF_P_M_DETECTED_ASCQ	0x04
+#define UNALIGNED_WRITE_ASCQ		0x04
+#define EOD_DETECTED_ASCQ		0x05
+#define WRITE_BOUNDARY_ASCQ		0x05
+#define READ_INVDATA_ASCQ		0x06
+#define READ_BOUNDARY_ASCQ		0x07
+#define CAPACITY_CHANGED_ASCQ		0x09
+#define ATTEMPT_ACCESS_GAP		0x09
+#define LUNS_CHANGED_ASCQ		0x0e
+#define INSUFF_ZONE_ASCQ		0x0e
+#define MICROCODE_CHANGED_WO_RESET_ASCQ 0x16
+#define OVERLAP_ATOMIC_COMMAND_ASCQ	0x23
+
 /*
  *  DEVICE TYPES
  *  Please keep them in 0x%02x format for $MODALIAS to work
@@ -413,6 +468,25 @@ enum zbc_zone_alignment_method {
 	ZBC_CONSTANT_ZONE_START_OFFSET	= 0x8,
 };
 
+/* SCSI physical element types */
+enum scsi_phys_element_type {
+	SCSI_PHYS_ELEM_TYPE_ALL_ACCESS_STORAGE	= 0x1,
+	SCSI_PHYS_ELEM_TYPE_FRAC_ACCESS_STORAGE	= 0x2,
+};
+
+/* SCSI physical element health. */
+enum scsi_phys_element_health {
+	SCSI_PHYS_ELEM_HEALTH_NOT_REPORTED		= 0x00,
+	SCSI_PHYS_ELEM_HEALTH_WITHIN_SPEC_LIMITS	= 0x01,
+	SCSI_PHYS_ELEM_HEALTH_AT_SPEC_LIMITS		= 0x64,
+	SCSI_PHYS_ELEM_HEALTH_OUTSIDE_SPEC_LIMITS	= 0x65,
+	SCSI_PHYS_ELEM_HEALTH_DEPOP_REVOKE_ERR		= 0xFB,
+	SCSI_PHYS_ELEM_HEALTH_DEPOP_REVOKE_IN_PROGRESS	= 0xFC,
+	SCSI_PHYS_ELEM_HEALTH_DEPOP_ERR			= 0xFD,
+	SCSI_PHYS_ELEM_HEALTH_DEPOP_IN_PROGRESS		= 0xFE,
+	SCSI_PHYS_ELEM_HEALTH_DEPOP_OK			= 0xFF,
+};
+
 /* Version descriptor values for INQUIRY */
 enum scsi_version_descriptor {
 	SCSI_VERSION_DESCRIPTOR_FCP4	= 0x0a40,
diff --git a/include/soc/tegra/pmc.h b/include/soc/tegra/pmc.h
index 4bcbf19d75ace..53f6d02889dbd 100644
--- a/include/soc/tegra/pmc.h
+++ b/include/soc/tegra/pmc.h
@@ -210,7 +210,6 @@ tegra_pmc_io_pad_power_disable(struct tegra_pmc *pmc, enum tegra_io_pad id)
 bool tegra_pmc_cpu_is_powered(unsigned int cpuid);
 int tegra_pmc_cpu_power_on(unsigned int cpuid);
 int tegra_pmc_cpu_remove_clamping(unsigned int cpuid);
-bool tegra_pmc_core_domain_state_synced(void);
 
 #if defined(CONFIG_SOC_TEGRA_PMC) && defined(CONFIG_PM_SLEEP)
 enum tegra_suspend_mode tegra_pmc_get_suspend_mode(void);
@@ -230,6 +229,10 @@ static inline void tegra_pmc_enter_suspend_mode(enum tegra_suspend_mode mode)
 {
 }
 #endif
+#endif
+
+#if defined(CONFIG_ARM) && defined(CONFIG_SOC_TEGRA_PMC)
+bool tegra_pmc_core_domain_state_synced(void);
 #else
 /* needed for COMPILE_TEST */
 static inline bool tegra_pmc_core_domain_state_synced(void)
diff --git a/include/sound/ac97/codec.h b/include/sound/ac97/codec.h
index 69b404c354f50..69aa7ee15b910 100644
--- a/include/sound/ac97/codec.h
+++ b/include/sound/ac97/codec.h
@@ -33,7 +33,7 @@ struct ac97_id {
 };
 
 /**
- * ac97_codec_device - a ac97 codec
+ * struct ac97_codec_device - a ac97 codec
  * @dev: the core device
  * @vendor_id: the vendor_id of the codec, as sensed on the AC-link
  * @num: the codec number, 0 is primary, 1 is first slave, etc ...
@@ -53,7 +53,7 @@ struct ac97_codec_device {
 };
 
 /**
- * ac97_codec_driver - a ac97 codec driver
+ * struct ac97_codec_driver - a ac97 codec driver
  * @driver: the device driver structure
  * @probe: the function called when a ac97_codec_device is matched
  * @remove: the function called when the device is unbound/removed
diff --git a/include/sound/hda_codec.h b/include/sound/hda_codec.h
index b4e3278777398..1d05f991f2ce7 100644
--- a/include/sound/hda_codec.h
+++ b/include/sound/hda_codec.h
@@ -256,6 +256,7 @@ struct hda_codec {
 	unsigned int link_down_at_suspend:1; /* link down at runtime suspend */
 	unsigned int relaxed_resume:1;	/* don't resume forcibly for jack */
 	unsigned int forced_resume:1; /* forced resume for jack */
+	unsigned int acomp_requested_resume:1; /* resume requested by acomp */
 	unsigned int no_stream_clean_at_suspend:1; /* do not clean streams at suspend */
 	unsigned int ctl_dev_id:1; /* old control element id build behaviour */
 	unsigned int eld_jack_detect:1;	/* Machine jack-detection by ELD */
diff --git a/include/sound/hda_regmap.h b/include/sound/hda_regmap.h
index 4c1b9bebbd60f..e06e4847ed282 100644
--- a/include/sound/hda_regmap.h
+++ b/include/sound/hda_regmap.h
@@ -69,11 +69,14 @@ void snd_hdac_regmap_sync(struct hdac_device *codec);
 
 /**
  * snd_hdac_regmap_write - Write a verb with caching
+ * @codec: HD-audio codec base device
  * @nid: codec NID
- * @reg: verb to write
+ * @verb: verb to write
  * @val: value to write
  *
  * For writing an amp value, use snd_hdac_regmap_update_amp().
+ *
+ * Returns: %0 if successful or a negative error code.
  */
 static inline int
 snd_hdac_regmap_write(struct hdac_device *codec, hda_nid_t nid,
@@ -85,13 +88,16 @@ snd_hdac_regmap_write(struct hdac_device *codec, hda_nid_t nid,
 }
 
 /**
- * snd_hda_regmap_update - Update a verb value with caching
+ * snd_hdac_regmap_update - Update a verb value with caching
+ * @codec: HD-audio codec
  * @nid: codec NID
  * @verb: verb to update
  * @mask: bit mask to update
  * @val: value to update
  *
  * For updating an amp value, use snd_hdac_regmap_update_amp().
+ *
+ * Returns: %0 if successful or a negative error code.
  */
 static inline int
 snd_hdac_regmap_update(struct hdac_device *codec, hda_nid_t nid,
@@ -104,12 +110,15 @@ snd_hdac_regmap_update(struct hdac_device *codec, hda_nid_t nid,
 }
 
 /**
- * snd_hda_regmap_read - Read a verb with caching
+ * snd_hdac_regmap_read - Read a verb with caching
+ * @codec: HD-audio codec
  * @nid: codec NID
  * @verb: verb to read
  * @val: pointer to store the value
  *
  * For reading an amp value, use snd_hda_regmap_get_amp().
+ *
+ * Returns: %0 if successful or a negative error code.
  */
 static inline int
 snd_hdac_regmap_read(struct hdac_device *codec, hda_nid_t nid,
@@ -125,12 +134,12 @@ snd_hdac_regmap_read(struct hdac_device *codec, hda_nid_t nid,
  * @codec: HD-audio codec
  * @nid: NID to read the AMP value
  * @ch: channel (left=0 or right=1)
- * @direction: #HDA_INPUT or #HDA_OUTPUT
- * @index: the index value (only for input direction)
- * @val: the pointer to store the value
+ * @dir: #HDA_INPUT or #HDA_OUTPUT
+ * @idx: the index value (only for input direction)
  *
  * Read AMP value.  The volume is between 0 to 0x7f, 0x80 = mute bit.
- * Returns the value or a negative error.
+ *
+ * Returns: the value or a negative error.
  */
 static inline int
 snd_hdac_regmap_get_amp(struct hdac_device *codec, hda_nid_t nid,
@@ -148,13 +157,14 @@ snd_hdac_regmap_get_amp(struct hdac_device *codec, hda_nid_t nid,
  * @codec: HD-audio codec
  * @nid: NID to read the AMP value
  * @ch: channel (left=0 or right=1)
- * @direction: #HDA_INPUT or #HDA_OUTPUT
+ * @dir: #HDA_INPUT or #HDA_OUTPUT
  * @idx: the index value (only for input direction)
  * @mask: bit mask to set
  * @val: the bits value to set
  *
  * Update the AMP value with a bit mask.
- * Returns 0 if the value is unchanged, 1 if changed, or a negative error.
+ *
+ * Returns: 0 if the value is unchanged, 1 if changed, or a negative error.
  */
 static inline int
 snd_hdac_regmap_update_amp(struct hdac_device *codec, hda_nid_t nid,
@@ -169,13 +179,12 @@ snd_hdac_regmap_update_amp(struct hdac_device *codec, hda_nid_t nid,
  * snd_hdac_regmap_get_amp_stereo - Read stereo AMP values
  * @codec: HD-audio codec
  * @nid: NID to read the AMP value
- * @ch: channel (left=0 or right=1)
- * @direction: #HDA_INPUT or #HDA_OUTPUT
- * @index: the index value (only for input direction)
- * @val: the pointer to store the value
+ * @dir: #HDA_INPUT or #HDA_OUTPUT
+ * @idx: the index value (only for input direction)
  *
  * Read stereo AMP values.  The lower byte is left, the upper byte is right.
- * Returns the value or a negative error.
+ *
+ * Returns: the value or a negative error.
  */
 static inline int
 snd_hdac_regmap_get_amp_stereo(struct hdac_device *codec, hda_nid_t nid,
@@ -192,14 +201,15 @@ snd_hdac_regmap_get_amp_stereo(struct hdac_device *codec, hda_nid_t nid,
  * snd_hdac_regmap_update_amp_stereo - update the stereo AMP value
  * @codec: HD-audio codec
  * @nid: NID to read the AMP value
- * @direction: #HDA_INPUT or #HDA_OUTPUT
+ * @dir: #HDA_INPUT or #HDA_OUTPUT
  * @idx: the index value (only for input direction)
  * @mask: bit mask to set
  * @val: the bits value to set
  *
  * Update the stereo AMP value with a bit mask.
  * The lower byte is left, the upper byte is right.
- * Returns 0 if the value is unchanged, 1 if changed, or a negative error.
+ *
+ * Returns: 0 if the value is unchanged, 1 if changed, or a negative error.
  */
 static inline int
 snd_hdac_regmap_update_amp_stereo(struct hdac_device *codec, hda_nid_t nid,
diff --git a/include/sound/simple_card_utils.h b/include/sound/simple_card_utils.h
index 915e6ae5f68d7..bd8c3a0335773 100644
--- a/include/sound/simple_card_utils.h
+++ b/include/sound/simple_card_utils.h
@@ -181,7 +181,7 @@ void simple_util_canonicalize_platform(struct snd_soc_dai_link_component *platfo
 void simple_util_canonicalize_cpu(struct snd_soc_dai_link_component *cpus,
 				  int is_single_links);
 
-void simple_util_clean_reference(struct snd_soc_card *card);
+void simple_util_clean_reference(struct simple_util_priv *priv);
 
 void simple_util_parse_convert(struct device_node *np, char *prefix,
 			       struct simple_util_data *data);
@@ -189,18 +189,37 @@ bool simple_util_is_convert_required(const struct simple_util_data *data);
 
 int simple_util_get_sample_fmt(struct simple_util_data *data);
 
-int simple_util_parse_routing(struct snd_soc_card *card,
-				      char *prefix);
-int simple_util_parse_widgets(struct snd_soc_card *card,
-				      char *prefix);
-int simple_util_parse_pin_switches(struct snd_soc_card *card,
-				   char *prefix);
+int simple_util_parse_property(struct simple_util_priv *priv,
+			       int (*func)(struct snd_soc_card *card, const char *propname),
+			       char *prefix, char *property);
+static inline int simple_util_parse_routing(struct simple_util_priv *priv, char *prefix)
+{
+	return simple_util_parse_property(priv, snd_soc_of_parse_audio_routing,
+					  prefix, "routing");
+}
+
+static inline int simple_util_parse_widgets(struct simple_util_priv *priv, char *prefix)
+{
+	return simple_util_parse_property(priv, snd_soc_of_parse_audio_simple_widgets,
+					  prefix, "widgets");
+}
+
+static inline int simple_util_parse_pin_switches(struct simple_util_priv *priv, char *prefix)
+{
+	return simple_util_parse_property(priv, snd_soc_of_parse_pin_switches,
+					  prefix, "pin-switches");
+}
+
+static inline int simple_util_parse_aux_devs(struct simple_util_priv *priv, char *prefix)
+{
+	return simple_util_parse_property(priv, snd_soc_of_parse_aux_devs,
+					  prefix, "aux-devs");
+}
 
 int simple_util_init_jack(struct snd_soc_card *card,
 			       struct simple_util_jack *sjack,
 			       int is_hp, char *prefix, char *pin);
-int simple_util_init_aux_jacks(struct simple_util_priv *priv,
-				char *prefix);
+int simple_util_init_aux_jacks(struct snd_soc_card *card, char *prefix);
 int simple_util_init_priv(struct simple_util_priv *priv,
 			       struct link_info *li);
 void simple_util_remove(struct platform_device *pdev);
diff --git a/include/sound/soc_sdw_utils.h b/include/sound/soc_sdw_utils.h
index 79c21966220b2..9b28e9aef4f1d 100644
--- a/include/sound/soc_sdw_utils.h
+++ b/include/sound/soc_sdw_utils.h
@@ -44,6 +44,18 @@
 
 struct asoc_sdw_codec_info;
 
+struct asoc_sdw_mc_private {
+	struct snd_soc_card card;
+	struct snd_soc_jack sdw_headset;
+	struct device *headset_codec_dev; /* only one headset per card */
+	struct device *amp_dev1, *amp_dev2;
+	bool append_dai_type;
+	bool ignore_internal_dmic;
+	void *private;
+	unsigned long mc_quirk;
+	int codec_info_list_count;
+};
+
 struct asoc_sdw_dai_info {
 	const bool direction[2]; /* playback & capture support */
 	const char *codec_name;
@@ -88,25 +100,13 @@ struct asoc_sdw_codec_info {
 
 	int (*codec_card_late_probe)(struct snd_soc_card *card);
 
-	int  (*count_sidecar)(struct snd_soc_card *card,
+	int  (*count_sidecar)(struct asoc_sdw_mc_private *ctx,
 			      int *num_dais, int *num_devs);
 	int  (*add_sidecar)(struct snd_soc_card *card,
 			    struct snd_soc_dai_link **dai_links,
 			    struct snd_soc_codec_conf **codec_conf);
 };
 
-struct asoc_sdw_mc_private {
-	struct snd_soc_card card;
-	struct snd_soc_jack sdw_headset;
-	struct device *headset_codec_dev; /* only one headset per card */
-	struct device *amp_dev1, *amp_dev2;
-	bool append_dai_type;
-	bool ignore_internal_dmic;
-	void *private;
-	unsigned long mc_quirk;
-	int codec_info_list_count;
-};
-
 struct asoc_sdw_endpoint {
 	struct list_head list;
 
@@ -182,7 +182,8 @@ struct asoc_sdw_dailink *asoc_sdw_find_dailink(struct asoc_sdw_dailink *dailinks
 					       const struct snd_soc_acpi_endpoint *new);
 int asoc_sdw_get_dai_type(u32 type);
 
-int asoc_sdw_parse_sdw_endpoints(struct snd_soc_card *card,
+int asoc_sdw_parse_sdw_endpoints(struct device *dev,
+				 struct asoc_sdw_mc_private *ctx,
 				 struct snd_soc_aux_dev *soc_aux,
 				 struct asoc_sdw_dailink *soc_dais,
 				 struct asoc_sdw_endpoint *soc_ends,
@@ -235,7 +236,7 @@ int asoc_sdw_es9356_amp_init(struct snd_soc_card *card,
 int asoc_sdw_es9356_exit(struct snd_soc_card *card, struct snd_soc_dai_link *dai_link);
 
 /* CS AMP support */
-int asoc_sdw_bridge_cs35l56_count_sidecar(struct snd_soc_card *card,
+int asoc_sdw_bridge_cs35l56_count_sidecar(struct asoc_sdw_mc_private *ctx,
 					  int *num_dais, int *num_devs);
 int asoc_sdw_bridge_cs35l56_add_sidecar(struct snd_soc_card *card,
 					struct snd_soc_dai_link **dai_links,
diff --git a/include/sound/sof/dai-amd.h b/include/sound/sof/dai-amd.h
index 59cd014392c1b..e2107da6558f0 100644
--- a/include/sound/sof/dai-amd.h
+++ b/include/sound/sof/dai-amd.h
@@ -18,6 +18,7 @@ struct sof_ipc_dai_acp_params {
 	uint32_t fsync_rate;    /* FSYNC frequency in Hz */
 	uint32_t tdm_slots;
 	uint32_t tdm_mode;
+	uint32_t format;
 } __packed;
 
 /* ACPDMIC Configuration Request - SOF_IPC_DAI_AMD_CONFIG */
diff --git a/include/sound/sof/dai.h b/include/sound/sof/dai.h
index 36809f7127236..e3fe492e78f57 100644
--- a/include/sound/sof/dai.h
+++ b/include/sound/sof/dai.h
@@ -90,6 +90,8 @@ enum sof_ipc_dai_type {
 	SOF_DAI_AMD_HS_VIRTUAL,		/**< AMD ACP HS VIRTUAL */
 	SOF_DAI_IMX_MICFIL,		/** < i.MX MICFIL PDM */
 	SOF_DAI_AMD_SDW,		/**< AMD ACP SDW */
+	SOF_DAI_INTEL_UAOL,		/**< Intel UAOL */
+	SOF_DAI_AMD_I2S,		/**< AMD ACP I2S */
 };
 
 /* general purpose DAI configuration */
@@ -121,6 +123,7 @@ struct sof_ipc_dai_config {
 		struct sof_ipc_dai_mtk_afe_params afe;
 		struct sof_ipc_dai_micfil_params micfil;
 		struct sof_ipc_dai_acp_sdw_params acp_sdw;
+		struct sof_ipc_dai_acp_params acp_i2s;
 	};
 } __packed;
 
diff --git a/include/trace/events/damon.h b/include/trace/events/damon.h
index 78388538acf44..8851727ae1627 100644
--- a/include/trace/events/damon.h
+++ b/include/trace/events/damon.h
@@ -78,9 +78,11 @@ TRACE_EVENT_CONDITION(damos_before_apply,
 
 	TP_PROTO(unsigned int context_idx, unsigned int scheme_idx,
 		unsigned int target_idx, struct damon_region *r,
-		unsigned int nr_regions, bool do_trace),
+		unsigned int nr_accesses, unsigned int nr_regions,
+		bool do_trace),
 
-	TP_ARGS(context_idx, scheme_idx, target_idx, r, nr_regions, do_trace),
+	TP_ARGS(context_idx, scheme_idx, target_idx, r, nr_accesses,
+		nr_regions, do_trace),
 
 	TP_CONDITION(do_trace),
 
@@ -101,7 +103,7 @@ TRACE_EVENT_CONDITION(damos_before_apply,
 		__entry->target_idx = target_idx;
 		__entry->start = r->ar.start;
 		__entry->end = r->ar.end;
-		__entry->nr_accesses = r->nr_accesses_bp / 10000;
+		__entry->nr_accesses = nr_accesses;
 		__entry->age = r->age;
 		__entry->nr_regions = nr_regions;
 	),
diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h
index 291fae364c62d..5a48c5406cce4 100644
--- a/include/trace/events/huge_memory.h
+++ b/include/trace/events/huge_memory.h
@@ -16,7 +16,7 @@
 	EM( SCAN_EXCEED_SWAP_PTE,	"exceed_swap_pte")		\
 	EM( SCAN_EXCEED_SHARED_PTE,	"exceed_shared_pte")		\
 	EM( SCAN_PTE_NON_PRESENT,	"pte_non_present")		\
-	EM( SCAN_PTE_UFFD_WP,		"pte_uffd_wp")			\
+	EM( SCAN_PTE_UFFD,		"pte_uffd_wp")			\
 	EM( SCAN_PTE_MAPPED_HUGEPAGE,	"pte_mapped_hugepage")		\
 	EM( SCAN_LACK_REFERENCED_PAGE,	"lack_referenced_page")		\
 	EM( SCAN_PAGE_NULL,		"page_null")			\
diff --git a/include/trace/events/memory-failure.h b/include/trace/events/memory-failure.h
index aa57cc8f896be..7a8ee5d1a44e4 100644
--- a/include/trace/events/memory-failure.h
+++ b/include/trace/events/memory-failure.h
@@ -1,6 +1,10 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #undef TRACE_SYSTEM
-#define TRACE_SYSTEM memory_failure
+/*
+ * For historical versions, memory_failure_event is in ras subsystem,
+ * some user programs depend on it.
+ */
+#define TRACE_SYSTEM ras
 #define TRACE_INCLUDE_FILE memory-failure
 
 #if !defined(_TRACE_MEMORY_FAILURE_H) || defined(TRACE_HEADER_MULTI_READ)
diff --git a/include/trace/events/migrate.h b/include/trace/events/migrate.h
index cd01dd7b3640d..15ee2ef201b5f 100644
--- a/include/trace/events/migrate.h
+++ b/include/trace/events/migrate.h
@@ -23,7 +23,8 @@
 	EM( MR_CONTIG_RANGE,	"contig_range")			\
 	EM( MR_LONGTERM_PIN,	"longterm_pin")			\
 	EM( MR_DEMOTION,	"demotion")			\
-	EMe(MR_DAMON,		"damon")
+	EM( MR_DAMON,		"damon")			\
+	EMe(MR_NEVER,		"never_migrated")
 
 /*
  * First define the enums in the above macros to be exported to userspace
@@ -51,7 +52,7 @@ TRACE_EVENT(mm_migrate_pages,
 	TP_PROTO(unsigned long succeeded, unsigned long failed,
 		 unsigned long thp_succeeded, unsigned long thp_failed,
 		 unsigned long thp_split, unsigned long large_folio_split,
-		 enum migrate_mode mode, int reason),
+		 enum migrate_mode mode, enum migrate_reason reason),
 
 	TP_ARGS(succeeded, failed, thp_succeeded, thp_failed,
 		thp_split, large_folio_split, mode, reason),
@@ -64,7 +65,7 @@ TRACE_EVENT(mm_migrate_pages,
 		__field(	unsigned long,		thp_split)
 		__field(	unsigned long,		large_folio_split)
 		__field(	enum migrate_mode,	mode)
-		__field(	int,			reason)
+		__field(	enum migrate_reason,	reason)
 	),
 
 	TP_fast_assign(
@@ -91,13 +92,13 @@ TRACE_EVENT(mm_migrate_pages,
 
 TRACE_EVENT(mm_migrate_pages_start,
 
-	TP_PROTO(enum migrate_mode mode, int reason),
+	TP_PROTO(enum migrate_mode mode, enum migrate_reason reason),
 
 	TP_ARGS(mode, reason),
 
 	TP_STRUCT__entry(
 		__field(enum migrate_mode, mode)
-		__field(int, reason)
+		__field(enum migrate_reason, reason)
 	),
 
 	TP_fast_assign(
diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h
index a6e5a44c9b429..935893e5ea53b 100644
--- a/include/trace/events/mmflags.h
+++ b/include/trace/events/mmflags.h
@@ -54,18 +54,10 @@
 # define TRACE_GFP_FLAGS_LOCKDEP
 #endif
 
-#ifdef CONFIG_SLAB_OBJ_EXT
-# define TRACE_GFP_FLAGS_SLAB			\
-	TRACE_GFP_EM(NO_OBJ_EXT)
-#else
-# define TRACE_GFP_FLAGS_SLAB
-#endif
-
 #define TRACE_GFP_FLAGS				\
 	TRACE_GFP_FLAGS_GENERAL			\
 	TRACE_GFP_FLAGS_KASAN			\
-	TRACE_GFP_FLAGS_LOCKDEP			\
-	TRACE_GFP_FLAGS_SLAB
+	TRACE_GFP_FLAGS_LOCKDEP
 
 #undef TRACE_GFP_EM
 #define TRACE_GFP_EM(a) TRACE_DEFINE_ENUM(___GFP_##a##_BIT);
@@ -194,6 +186,12 @@ IF_HAVE_PG_ARCH_3(arch_3)
 # define IF_HAVE_UFFD_MINOR(flag, name)
 #endif
 
+#ifdef CONFIG_USERFAULTFD_RWP
+# define IF_HAVE_UFFD_RWP(flag, name) {flag, name},
+#else
+# define IF_HAVE_UFFD_RWP(flag, name)
+#endif
+
 #if defined(CONFIG_64BIT) || defined(CONFIG_PPC32)
 # define IF_HAVE_VM_DROPPABLE(flag, name) {flag, name},
 #else
@@ -215,6 +213,7 @@ IF_HAVE_UFFD_MINOR(VM_UFFD_MINOR,	"uffd_minor"	)		\
 	{VM_PFNMAP,			"pfnmap"	},		\
 	{VM_MAYBE_GUARD,		"maybe_guard"	},		\
 	{VM_UFFD_WP,			"uffd_wp"	},		\
+IF_HAVE_UFFD_RWP(VM_UFFD_RWP,		"uffd_rwp"	)		\
 	{VM_LOCKED,			"locked"	},		\
 	{VM_IO,				"io"		},		\
 	{VM_SEQ_READ,			"seqread"	},		\
diff --git a/include/trace/events/pagemap.h b/include/trace/events/pagemap.h
index 171524d3526db..36c3a90f0acca 100644
--- a/include/trace/events/pagemap.h
+++ b/include/trace/events/pagemap.h
@@ -77,6 +77,14 @@ TRACE_EVENT(mm_lru_activate,
 	TP_printk("folio=%p pfn=0x%lx", __entry->folio, __entry->pfn)
 );
 
+DECLARE_TRACE(mm_lru_add_drain,
+	      TP_PROTO(int cpu, unsigned int nr_folios),
+	      TP_ARGS(cpu, nr_folios));
+
+DECLARE_TRACE(mm_lru_add_drain_all,
+	      TP_PROTO(bool force_all_cpus),
+	      TP_ARGS(force_all_cpus));
+
 #endif /* _TRACE_PAGEMAP_H */
 
 /* This part must be outside protection */
diff --git a/include/trace/events/qcom_geni_serial.h b/include/trace/events/qcom_geni_serial.h
index 417ec01f9fc8e..e1aa551d525e5 100644
--- a/include/trace/events/qcom_geni_serial.h
+++ b/include/trace/events/qcom_geni_serial.h
@@ -97,18 +97,17 @@ DECLARE_EVENT_CLASS(geni_serial_data,
 		    TP_ARGS(dev, buf, len),
 
 		    TP_STRUCT__entry(__string(name, dev_name(dev))
-				     __field(unsigned int, len)
 				     __dynamic_array(u8, data, len)
 		    ),
 
 		    TP_fast_assign(__assign_str(name);
-				   __entry->len = len;
 				   memcpy(__get_dynamic_array(data), buf, len);
 		    ),
 
 		    TP_printk("%s: len=%u data=%s",
-			      __get_str(name), __entry->len,
-			      __print_hex(__get_dynamic_array(data), __entry->len))
+			      __get_str(name), __get_dynamic_array_len(data),
+			      __print_hex(__get_dynamic_array(data),
+					  __get_dynamic_array_len(data)))
 );
 
 DEFINE_EVENT(geni_serial_data, geni_serial_tx_data,
diff --git a/include/trace/events/sched_ext.h b/include/trace/events/sched_ext.h
index d1bf5acd59c56..1e54f9564ef39 100644
--- a/include/trace/events/sched_ext.h
+++ b/include/trace/events/sched_ext.h
@@ -84,6 +84,34 @@ TRACE_EVENT(sched_ext_bypass_lb,
 	)
 );
 
+TRACE_EVENT(sched_ext_exit,
+
+	TP_PROTO(struct scx_sched *sch, __u32 kind),
+
+	TP_ARGS(sch, kind),
+
+	TP_STRUCT__entry(
+		__string(	name,		sch->ops.name		)
+		__field(	__s32,		level			)
+		__field(	__u64,		sub_cgroup_id		)
+		__string(	cgrp_path,	sch_cgrp_path(sch)	)
+		__field(	__u32,		kind			)
+	),
+
+	TP_fast_assign(
+		__assign_str(name);
+		__entry->level		= sch->level;
+		__entry->sub_cgroup_id	= sch->ops.sub_cgroup_id;
+		__assign_str(cgrp_path);
+		__entry->kind		= kind;
+	),
+
+	TP_printk("sched %s level %d sub_cgroup_id %llu cgrp_path %s kind %u",
+		  __get_str(name), __entry->level, __entry->sub_cgroup_id,
+		  __get_str(cgrp_path), __entry->kind
+	)
+);
+
 #endif /* _TRACE_SCHED_EXT_H */
 
 /* This part must be outside protection */
diff --git a/include/uapi/asm-generic/errno.h b/include/uapi/asm-generic/errno.h
index bd78e69e0a43f..c84ebf89c8b6e 100644
--- a/include/uapi/asm-generic/errno.h
+++ b/include/uapi/asm-generic/errno.h
@@ -76,7 +76,7 @@
 #define	ENOPROTOOPT	92	/* Protocol not available */
 #define	EPROTONOSUPPORT	93	/* Protocol not supported */
 #define	ESOCKTNOSUPPORT	94	/* Socket type not supported */
-#define	EOPNOTSUPP	95	/* Operation not supported on transport endpoint */
+#define	EOPNOTSUPP	95	/* Operation not supported */
 #define	EPFNOSUPPORT	96	/* Protocol family not supported */
 #define	EAFNOSUPPORT	97	/* Address family not supported by protocol */
 #define	EADDRINUSE	98	/* Address already in use */
diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
index 9f3090db2f163..b32c72a662b61 100644
--- a/include/uapi/drm/amdgpu_drm.h
+++ b/include/uapi/drm/amdgpu_drm.h
@@ -58,6 +58,7 @@ extern "C" {
 #define DRM_AMDGPU_USERQ_SIGNAL		0x17
 #define DRM_AMDGPU_USERQ_WAIT		0x18
 #define DRM_AMDGPU_GEM_LIST_HANDLES	0x19
+#define DRM_AMDGPU_PROC_OPTIONS		0x1A
 
 #define DRM_IOCTL_AMDGPU_GEM_CREATE	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_CREATE, union drm_amdgpu_gem_create)
 #define DRM_IOCTL_AMDGPU_GEM_MMAP	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_MMAP, union drm_amdgpu_gem_mmap)
@@ -79,6 +80,7 @@ extern "C" {
 #define DRM_IOCTL_AMDGPU_USERQ_SIGNAL	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_USERQ_SIGNAL, struct drm_amdgpu_userq_signal)
 #define DRM_IOCTL_AMDGPU_USERQ_WAIT	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_USERQ_WAIT, struct drm_amdgpu_userq_wait)
 #define DRM_IOCTL_AMDGPU_GEM_LIST_HANDLES DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_LIST_HANDLES, struct drm_amdgpu_gem_list_handles)
+#define DRM_IOCTL_AMDGPU_PROC_OPTIONS	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_PROC_OPTIONS, struct drm_amdgpu_proc_options)
 
 /**
  * DOC: memory domains
@@ -1673,6 +1675,25 @@ struct drm_amdgpu_info_uq_metadata {
 #define AMDGPU_FAMILY_GC_11_5_4			154 /* GC 11.5.4 */
 #define AMDGPU_FAMILY_GC_12_0_0			152 /* GC 12.0.0 */
 
+/*
+ * Definition of user options
+ *
+ * option: AMDGPU_PROC_OPTIONS_OP_KFD_SIGBUS_DELAY
+ *    0:          Disable sigbus delay - SIGBUS will be raised immediately
+ *    0xFFFFFFFF: SIGBUS will not be raised
+ *    other:      Set the sigbus delay in milliseconds
+ */
+#define AMDGPU_PROC_OPTIONS_OP_KFD_SIGBUS_DELAY		0
+
+#define AMDGPU_PROC_OPTIONS_KFD_SIGBUS_DELAY_DISABLED	0xFFFFFFFFu
+
+struct drm_amdgpu_proc_options {
+	__u32 op;
+	struct {
+		__u32 value;
+	} kfd_sigbus_delay;
+};
+
 #if defined(__cplusplus)
 }
 #endif
diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h
index 381a3e857d4e9..bd435effdcee6 100644
--- a/include/uapi/drm/drm_mode.h
+++ b/include/uapi/drm/drm_mode.h
@@ -155,6 +155,7 @@ extern "C" {
 /* Panel type property */
 #define DRM_MODE_PANEL_TYPE_UNKNOWN	0
 #define DRM_MODE_PANEL_TYPE_OLED	1
+#define DRM_MODE_PANEL_TYPE_LCD		2
 
 /*
  * DRM_MODE_ROTATE_<degrees>
diff --git a/include/uapi/drm/ethosu_accel.h b/include/uapi/drm/ethosu_accel.h
index af78bb4686d70..5b97d59a7806c 100644
--- a/include/uapi/drm/ethosu_accel.h
+++ b/include/uapi/drm/ethosu_accel.h
@@ -43,6 +43,11 @@ enum drm_ethosu_ioctl_id {
 
 	/** @DRM_ETHOSU_SUBMIT: Submit a job and BOs to run. */
 	DRM_ETHOSU_SUBMIT,
+
+	DRM_ETHOSU_PERFMON_CREATE,
+	DRM_ETHOSU_PERFMON_DESTROY,
+	DRM_ETHOSU_PERFMON_GET_VALUES,
+	DRM_ETHOSU_PERFMON_SET_GLOBAL,
 };
 
 /**
@@ -79,6 +84,7 @@ struct drm_ethosu_npu_info {
 	__u32 config;
 
 	__u32 sram_size;
+	__u32 pmu_counters;
 };
 
 /**
@@ -220,10 +226,54 @@ struct drm_ethosu_submit {
 	/** Input: Number of jobs passed in. */
 	__u32 job_count;
 
-	/** Reserved, must be zero. */
+	/** Input: Id returned by DRM_ETHOSU_PERFMON_CREATE */
+	__u32 perfmon_id;
+};
+
+#define DRM_ETHOSU_MAX_PERF_EVENT_COUNTERS	8
+#define DRM_ETHOSU_MAX_PERF_COUNTERS \
+	(DRM_ETHOSU_MAX_PERF_EVENT_COUNTERS + 1)
+
+struct drm_ethosu_perfmon_create {
+	__u32 id;
+	__u32 ncounters;
+	__u16 counters[DRM_ETHOSU_MAX_PERF_EVENT_COUNTERS];
+};
+
+struct drm_ethosu_perfmon_destroy {
+	__u32 id;
 	__u32 pad;
 };
 
+/*
+ * Returns the values of the performance counters tracked by this
+ * perfmon (as an array of (ncounters + 1) u64 values).
+ *
+ * No implicit synchronization is performed, so the user has to
+ * guarantee that any jobs using this perfmon have already been
+ * completed.
+ */
+struct drm_ethosu_perfmon_get_values {
+	__u32 id;
+	__u32 pad;
+	__u64 values_ptr;
+};
+
+#define DRM_ETHOSU_PERFMON_CLEAR_GLOBAL    0x0001
+
+/**
+ * struct drm_ethosu_perfmon_set_global - ioctl to define a global performance
+ * monitor
+ *
+ * The global performance monitor will be used for all jobs. If a global
+ * performance monitor is defined, jobs with a self-defined performance
+ * monitor won't be allowed.
+ */
+struct drm_ethosu_perfmon_set_global {
+	__u32 flags;
+	__u32 id;
+};
+
 /**
  * DRM_IOCTL_ETHOSU() - Build a ethosu IOCTL number
  * @__access: Access type. Must be R, W or RW.
@@ -252,6 +302,14 @@ enum {
 		DRM_IOCTL_ETHOSU(WR, CMDSTREAM_BO_CREATE, cmdstream_bo_create),
 	DRM_IOCTL_ETHOSU_SUBMIT =
 		DRM_IOCTL_ETHOSU(WR, SUBMIT, submit),
+	DRM_IOCTL_ETHOSU_PERFMON_CREATE =
+		DRM_IOCTL_ETHOSU(WR, PERFMON_CREATE, perfmon_create),
+	DRM_IOCTL_ETHOSU_PERFMON_DESTROY =
+		DRM_IOCTL_ETHOSU(WR, PERFMON_DESTROY, perfmon_destroy),
+	DRM_IOCTL_ETHOSU_PERFMON_GET_VALUES =
+		DRM_IOCTL_ETHOSU(WR, PERFMON_GET_VALUES, perfmon_get_values),
+	DRM_IOCTL_ETHOSU_PERFMON_SET_GLOBAL =
+		DRM_IOCTL_ETHOSU(WR, PERFMON_SET_GLOBAL, perfmon_set_global),
 };
 
 #if defined(__cplusplus)
diff --git a/include/uapi/drm/panthor_drm.h b/include/uapi/drm/panthor_drm.h
index 0e455d91e77d4..a2ff0f4ec6915 100644
--- a/include/uapi/drm/panthor_drm.h
+++ b/include/uapi/drm/panthor_drm.h
@@ -253,6 +253,9 @@ enum drm_panthor_dev_query_type {
 	 * @DRM_PANTHOR_DEV_QUERY_GROUP_PRIORITIES_INFO: Query allowed group priorities information.
 	 */
 	DRM_PANTHOR_DEV_QUERY_GROUP_PRIORITIES_INFO,
+
+	/** @DRM_PANTHOR_DEV_QUERY_MMU_INFO: Query MMU information. */
+	DRM_PANTHOR_DEV_QUERY_MMU_INFO,
 };
 
 /**
@@ -487,6 +490,16 @@ struct drm_panthor_timestamp_info {
 	__u64 cpu_timestamp_nsec;
 };
 
+/**
+ * struct drm_panthor_mmu_info - MMU information
+ *
+ * Structure grouping all queryable information relating to the MMU.
+ */
+struct drm_panthor_mmu_info {
+	/** @page_size_bitmap: Allowed page sizes */
+	__u64 page_size_bitmap;
+};
+
 /**
  * struct drm_panthor_group_priorities_info - Group priorities information
  *
@@ -601,6 +614,18 @@ enum drm_panthor_vm_bind_op_flags {
 	 */
 	DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED = 1 << 2,
 
+	/**
+	 * @DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE: Sparsely map a virtual memory range
+	 *
+	 * Only valid with DRM_PANTHOR_VM_BIND_OP_TYPE_MAP.
+	 *
+	 * When this flag is set, the whole vm_bind range is mapped over a dummy object in a cyclic
+	 * fashion, and all GPU reads from addresses in the range return undefined values. This flag
+	 * being set means drm_panthor_vm_bind_op::bo_offset and drm_panthor_vm_bind_op::bo_handle
+	 * must both be set to 0. DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC must also be set.
+	 */
+	DRM_PANTHOR_VM_BIND_OP_MAP_SPARSE = 1 << 3,
+
 	/**
 	 * @DRM_PANTHOR_VM_BIND_OP_TYPE_MASK: Mask used to determine the type of operation.
 	 */
@@ -664,7 +689,6 @@ struct drm_panthor_vm_bind_op {
 	 * This array shall not be empty for sync-only operations.
 	 */
 	struct drm_panthor_obj_array syncs;
-
 };
 
 /**
diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h
index 48e9f1fdb78dd..509202a7b13eb 100644
--- a/include/uapi/drm/xe_drm.h
+++ b/include/uapi/drm/xe_drm.h
@@ -2537,21 +2537,21 @@ struct drm_xe_exec_queue_set_property {
  * Refer to Documentation/netlink/specs/drm_ras.yaml for complete interface specification.
  *
  * Node Registration
- * =================
+ * -----------------
  *
  * The driver registers DRM RAS nodes for each error severity level.
  * enum drm_xe_ras_error_severity defines the node-id, while DRM_XE_RAS_ERROR_SEVERITY_NAMES maps
  * node-id to node-name.
  *
  * Error Classification
- * ====================
+ * --------------------
  *
  * Each node contains a list of error counters. Each error is identified by a error-id and
  * an error-name. enum drm_xe_ras_error_component defines the error-id, while
  * DRM_XE_RAS_ERROR_COMPONENT_NAMES maps error-id to error-name.
  *
  * User Interface
- * ==============
+ * --------------
  *
  * To retrieve error values of a error counter, userspace applications should
  * follow the below steps:
@@ -2589,6 +2589,12 @@ enum drm_xe_ras_error_component {
 	DRM_XE_RAS_ERR_COMP_CORE_COMPUTE = 1,
 	/** @DRM_XE_RAS_ERR_COMP_SOC_INTERNAL: SoC Internal Error */
 	DRM_XE_RAS_ERR_COMP_SOC_INTERNAL,
+	/** @DRM_XE_RAS_ERR_COMP_DEVICE_MEMORY: Device Memory Error */
+	DRM_XE_RAS_ERR_COMP_DEVICE_MEMORY,
+	/** @DRM_XE_RAS_ERR_COMP_PCIE: PCIe Subsystem Error */
+	DRM_XE_RAS_ERR_COMP_PCIE,
+	/** @DRM_XE_RAS_ERR_COMP_FABRIC: Fabric Subsystem Error */
+	DRM_XE_RAS_ERR_COMP_FABRIC,
 	/** @DRM_XE_RAS_ERR_COMP_MAX: Max Error */
 	DRM_XE_RAS_ERR_COMP_MAX	/* non-ABI */
 };
@@ -2606,7 +2612,10 @@ enum drm_xe_ras_error_component {
  */
 #define DRM_XE_RAS_ERROR_COMPONENT_NAMES {				\
 	[DRM_XE_RAS_ERR_COMP_CORE_COMPUTE] = "core-compute",		\
-	[DRM_XE_RAS_ERR_COMP_SOC_INTERNAL] = "soc-internal"		\
+	[DRM_XE_RAS_ERR_COMP_SOC_INTERNAL] = "soc-internal",		\
+	[DRM_XE_RAS_ERR_COMP_DEVICE_MEMORY] = "device-memory",		\
+	[DRM_XE_RAS_ERR_COMP_PCIE] = "pcie",				\
+	[DRM_XE_RAS_ERR_COMP_FABRIC] = "fabric",			\
 }
 
 #if defined(__cplusplus)
diff --git a/include/uapi/linux/alloc_tag.h b/include/uapi/linux/alloc_tag.h
new file mode 100644
index 0000000000000..270f693b18220
--- /dev/null
+++ b/include/uapi/linux/alloc_tag.h
@@ -0,0 +1,99 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * alloc_tag IOCTL API definition
+ *
+ * Copyright (C) 2026 Google, LLC.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef _UAPI_ALLOC_TAG_H
+#define _UAPI_ALLOC_TAG_H
+
+#include <linux/types.h>
+
+/*
+ * Function, file and module names often have the same prefixes, therefore
+ * when filtering by these criteria, we compare the last 64 characters to
+ * minimize the chances of name collisions
+ */
+#define ALLOCINFO_STR_SIZE	64
+
+struct allocinfo_content_id {
+	__u64 id;
+};
+
+struct allocinfo_tag {
+	/* Longer names are trimmed */
+	char modname[ALLOCINFO_STR_SIZE];
+	char function[ALLOCINFO_STR_SIZE];
+	char filename[ALLOCINFO_STR_SIZE];
+	__u64 lineno;
+	/* filter criteria only; see allocinfo_counter.accurate for actual accuracy */
+	__u64 inaccurate;
+};
+
+/* The alignment ensures 32-bit compatible interfaces are not broken */
+struct allocinfo_counter {
+	__u64 bytes;
+	__u64 calls;
+	__u8 accurate;
+} __attribute__((aligned(8)));
+
+struct allocinfo_tag_data {
+	struct allocinfo_tag tag;
+	struct allocinfo_counter counter;
+};
+
+enum {
+	ALLOCINFO_FILTER_MODNAME,
+	ALLOCINFO_FILTER_FUNCTION,
+	ALLOCINFO_FILTER_FILENAME,
+	ALLOCINFO_FILTER_LINENO,
+	ALLOCINFO_FILTER_INACCURATE,
+	ALLOCINFO_FILTER_MIN_SIZE,
+	ALLOCINFO_FILTER_MAX_SIZE,
+	__ALLOCINFO_FILTER_LAST = ALLOCINFO_FILTER_MAX_SIZE
+};
+
+#define ALLOCINFO_FILTER_MASK_MODNAME		(1 << ALLOCINFO_FILTER_MODNAME)
+#define ALLOCINFO_FILTER_MASK_FUNCTION		(1 << ALLOCINFO_FILTER_FUNCTION)
+#define ALLOCINFO_FILTER_MASK_FILENAME		(1 << ALLOCINFO_FILTER_FILENAME)
+#define ALLOCINFO_FILTER_MASK_LINENO		(1 << ALLOCINFO_FILTER_LINENO)
+#define ALLOCINFO_FILTER_MASK_INACCURATE	(1 << ALLOCINFO_FILTER_INACCURATE)
+#define ALLOCINFO_FILTER_MASK_MIN_SIZE		(1 << ALLOCINFO_FILTER_MIN_SIZE)
+#define ALLOCINFO_FILTER_MASK_MAX_SIZE		(1 << ALLOCINFO_FILTER_MAX_SIZE)
+
+#define ALLOCINFO_FILTER_MASKS \
+	((1 << (__ALLOCINFO_FILTER_LAST + 1)) - 1)
+
+struct allocinfo_filter {
+	__u64 mask; /* bitmask of the filter fields used */
+	struct allocinfo_tag fields;
+	__u64 min_size;
+	__u64 max_size;
+};
+
+struct allocinfo_get_at {
+	/* inputs */
+	__u64 pos;
+	struct allocinfo_filter filter;
+	/* output */
+	struct allocinfo_tag_data data;
+};
+
+#define _ALLOCINFO_IOC_CONTENT_ID	0
+#define _ALLOCINFO_IOC_GET_AT		1
+#define _ALLOCINFO_IOC_GET_NEXT		2
+
+#define ALLOCINFO_IOC_BASE		0xA6
+#define ALLOCINFO_IOC_CONTENT_ID	_IOR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_CONTENT_ID,	\
+					     struct allocinfo_content_id)
+#define ALLOCINFO_IOC_GET_AT		_IOWR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_GET_AT,	\
+					      struct allocinfo_get_at)
+#define ALLOCINFO_IOC_GET_NEXT		_IOR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_GET_NEXT,	\
+					     struct allocinfo_tag_data)
+
+#endif /* _UAPI_ALLOC_TAG_H */
diff --git a/include/uapi/linux/amd-pmf.h b/include/uapi/linux/amd-pmf.h
new file mode 100644
index 0000000000000..24cf9926dc590
--- /dev/null
+++ b/include/uapi/linux/amd-pmf.h
@@ -0,0 +1,204 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later WITH Linux-syscall-note */
+/*
+ * AMD Platform Management Framework (PMF) UAPI Header
+ *
+ * Copyright (c) 2026, Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * This file defines the user-space API for interacting with the AMD PMF
+ * driver. It provides ioctl interfaces to query platform-specific metrics
+ * such as power source, slider position, platform type, laptop placement,
+ * and various BIOS input/output parameters.
+ */
+
+#ifndef _UAPI_LINUX_AMD_PMF_H
+#define _UAPI_LINUX_AMD_PMF_H
+
+#include <linux/bits.h>
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+/**
+ * AMD_PMF_IOC_MAGIC - Magic number for AMD PMF ioctl commands
+ *
+ * This magic number uniquely identifies AMD PMF ioctl operations.
+ */
+#define AMD_PMF_IOC_MAGIC	'p'
+
+/**
+ * IOCTL_AMD_PMF_POPULATE_DATA - ioctl command to retrieve PMF metrics data
+ *
+ * This ioctl command is used to populate the amd_pmf_info structure
+ * with the requested PMF metrics information.
+ */
+#define IOCTL_AMD_PMF_POPULATE_DATA		_IOWR(AMD_PMF_IOC_MAGIC, 0x00, __u64)
+
+#define AMD_PMF_BIOS_PARAMS_MAX		10
+
+/* AMD PMF feature flags - bitmask indicating supported features */
+#define AMD_PMF_FEAT_AUTO_MODE			BIT(0)
+#define AMD_PMF_FEAT_STATIC_POWER_SLIDER	BIT(1)
+#define AMD_PMF_FEAT_POLICY_BUILDER		BIT(2)
+#define AMD_PMF_FEAT_DYNAMIC_POWER_SLIDER_AC	BIT(3)
+#define AMD_PMF_FEAT_DYNAMIC_POWER_SLIDER_DC	BIT(4)
+
+/**
+ * enum amd_pmf_laptop_placement - Describes the physical placement of the laptop
+ * @AMD_PMF_LP_UNKNOWN: Placement cannot be determined
+ * @AMD_PMF_ON_TABLE: Laptop is placed on a stable surface like a table or desk
+ * @AMD_PMF_ON_LAP_MOTION: Laptop is on a lap with detected motion
+ * @AMD_PMF_IN_BAG: Laptop is detected to be inside a bag or case
+ * @AMD_PMF_OUT_OF_BAG: Laptop has been removed from bag or case
+ * @AMD_PMF_LP_UNDEFINED: Placement state is undefined
+ *
+ * This enumeration represents the physical placement state of the laptop
+ * as detected by platform sensors. Used for adaptive power management
+ * and thermal policies.
+ */
+enum amd_pmf_laptop_placement {
+	AMD_PMF_LP_UNKNOWN,
+	AMD_PMF_ON_TABLE,
+	AMD_PMF_ON_LAP_MOTION,
+	AMD_PMF_IN_BAG,
+	AMD_PMF_OUT_OF_BAG,
+	AMD_PMF_LP_UNDEFINED,
+};
+
+/**
+ * enum amd_pmf_ta_slider - Trusted Application power slider positions
+ * @AMD_PMF_TA_BEST_BATTERY: Maximum battery savings, minimal performance
+ * @AMD_PMF_TA_BETTER_BATTERY: Balanced towards battery life
+ * @AMD_PMF_TA_BETTER_PERFORMANCE: Balanced towards performance
+ * @AMD_PMF_TA_BEST_PERFORMANCE: Maximum performance, higher power consumption
+ * @AMD_PMF_TA_MAX: Sentinel value indicating maximum enum value
+ *
+ * This enumeration defines the power slider positions used by the
+ * AMD PMF Trusted Application for dynamic power management decisions.
+ * These correspond to the Windows power slider UI positions.
+ */
+enum amd_pmf_ta_slider {
+	AMD_PMF_TA_BEST_BATTERY,
+	AMD_PMF_TA_BETTER_BATTERY,
+	AMD_PMF_TA_BETTER_PERFORMANCE,
+	AMD_PMF_TA_BEST_PERFORMANCE,
+	AMD_PMF_TA_MAX,
+};
+
+/**
+ * enum amd_pmf_platform_type - Describes the physical form factor orientation
+ * @AMD_PMF_PTYPE_UNKNOWN: Platform type cannot be determined
+ * @AMD_PMF_LID_CLOSE: Laptop lid is closed
+ * @AMD_PMF_CLAMSHELL: Traditional laptop mode with keyboard and screen
+ * @AMD_PMF_FLAT: Device is lying flat on a surface
+ * @AMD_PMF_TENT: Device is in tent mode (keyboard folded back, standing)
+ * @AMD_PMF_STAND: Device is propped up in stand orientation
+ * @AMD_PMF_TABLET: Device is in tablet mode with keyboard hidden
+ * @AMD_PMF_BOOK: Device is in book reading orientation
+ * @AMD_PMF_PRESENTATION: Device is in presentation mode
+ * @AMD_PMF_PULL_FWD: Screen is pulled forward towards user
+ * @AMD_PMF_PTYPE_INVALID: Invalid platform type marker
+ *
+ * This enumeration describes the current physical orientation or form
+ * factor of convertible/2-in-1 devices. Used for optimizing power and
+ * thermal management based on device posture.
+ */
+enum amd_pmf_platform_type {
+	AMD_PMF_PTYPE_UNKNOWN,
+	AMD_PMF_LID_CLOSE,
+	AMD_PMF_CLAMSHELL,
+	AMD_PMF_FLAT,
+	AMD_PMF_TENT,
+	AMD_PMF_STAND,
+	AMD_PMF_TABLET,
+	AMD_PMF_BOOK,
+	AMD_PMF_PRESENTATION,
+	AMD_PMF_PULL_FWD,
+	AMD_PMF_PTYPE_INVALID = 0xf,
+};
+
+static inline const char *amd_pmf_get_platform_type(unsigned int platform_type)
+{
+	switch (platform_type) {
+	case AMD_PMF_CLAMSHELL:
+		return "CLAMSHELL";
+	case AMD_PMF_LID_CLOSE:
+		return "LID_CLOSE";
+	case AMD_PMF_FLAT:
+		return "FLAT";
+	case AMD_PMF_TENT:
+		return "TENT";
+	case AMD_PMF_STAND:
+		return "STAND";
+	case AMD_PMF_TABLET:
+		return "TABLET";
+	case AMD_PMF_BOOK:
+		return "BOOK";
+	case AMD_PMF_PRESENTATION:
+		return "PRESENTATION";
+	case AMD_PMF_PULL_FWD:
+		return "PULL_FWD";
+	default:
+		return "UNKNOWN";
+	}
+}
+
+static inline const char *amd_pmf_get_laptop_placement(unsigned int device_state)
+{
+	switch (device_state) {
+	case AMD_PMF_ON_TABLE:
+		return "ON_TABLE";
+	case AMD_PMF_ON_LAP_MOTION:
+		return "ON_LAP_MOTION";
+	case AMD_PMF_IN_BAG:
+		return "IN_BAG";
+	case AMD_PMF_OUT_OF_BAG:
+		return "OUT_OF_BAG";
+	default:
+		return "UNKNOWN";
+	}
+}
+
+static inline const char *amd_pmf_get_slider_position(unsigned int state)
+{
+	switch (state) {
+	case AMD_PMF_TA_BEST_PERFORMANCE:
+		return "PERFORMANCE";
+	case AMD_PMF_TA_BETTER_PERFORMANCE:
+		return "BALANCED";
+	case AMD_PMF_TA_BEST_BATTERY:
+		return "POWER_SAVER";
+	case AMD_PMF_TA_BETTER_BATTERY:
+		return "BALANCED_BATTERY";
+	default:
+		return "Unknown TA Slider State";
+	}
+}
+
+struct amd_pmf_info {
+	__u64 size;
+
+	/* Feature info */
+	__u32 features_supported;
+
+	/* Power and state info */
+	__u32 platform_type;
+	__u32 power_source;
+	__u32 laptop_placement;
+	__u32 lid_state;
+	__u32 user_presence;
+	__u32 slider_position;
+
+	/* Thermal and power metrics */
+	__s32 skin_temp;
+	__u32 gfx_busy;
+	__s32 ambient_light;
+	__u32 avg_c0_residency;
+	__u32 max_c0_residency;
+	__u32 socket_power;
+
+	/* BIOS parameters */
+	__u32 bios_input[AMD_PMF_BIOS_PARAMS_MAX];
+	__u32 bios_output[AMD_PMF_BIOS_PARAMS_MAX];
+};
+
+#endif /* _UAPI_LINUX_AMD_PMF_H */
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 89b36de5fdbb6..2f1d24fef8573 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -5079,17 +5079,19 @@ union bpf_attr {
  * 	Description
  * 		Redirect the packet to another net device of index *ifindex*.
  * 		This helper is somewhat similar to **bpf_redirect**\ (), except
- * 		that the redirection happens to the *ifindex*' peer device and
- * 		the netns switch takes place from ingress to ingress without
- * 		going through the CPU's backlog queue.
+ * 		that the redirection happens to the *ifindex*' peer device. If
+ * 		*flags* is 0, the netns switch takes place from ingress to
+ * 		ingress without going through the CPU's backlog queue. If the
+ * 		**BPF_F_EGRESS** flag is provided then redirection happens in
+ * 		the egress direction of the peer device.
  *
  * 		*skb*\ **->mark** and *skb*\ **->tstamp** are not cleared during
  * 		the netns switch.
  *
- * 		The *flags* argument is reserved and must be 0. The helper is
- * 		currently only supported for tc BPF program types at the
- * 		ingress hook and for veth and netkit target device types. The
- * 		peer device must reside in a different network namespace.
+ * 		If the *flags* argument is 0, the helper is currently only
+ * 		supported for tc BPF program types at the ingress hook and for
+ * 		veth and netkit target device types. The peer device must reside
+ * 		in a different network namespace.
  * 	Return
  * 		The helper returns **TC_ACT_REDIRECT** on success or
  * 		**TC_ACT_SHOT** on error.
@@ -6336,9 +6338,10 @@ enum {
 /* Flags for bpf_redirect and bpf_redirect_map helpers */
 enum {
 	BPF_F_INGRESS		= (1ULL << 0), /* used for skb path */
+	BPF_F_EGRESS		= (1ULL << 1), /* used for skb path */
 	BPF_F_BROADCAST		= (1ULL << 3), /* used for XDP path */
 	BPF_F_EXCLUDE_INGRESS	= (1ULL << 4), /* used for XDP path */
-#define BPF_F_REDIRECT_FLAGS (BPF_F_INGRESS | BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS)
+#define BPF_F_REDIRECT_FLAGS (BPF_F_INGRESS | BPF_F_EGRESS | BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS)
 };
 
 #define __bpf_md_ptr(type, name)	\
@@ -6839,6 +6842,15 @@ struct bpf_link_info {
 			__u32 flags;
 			__u32 pid;
 		} uprobe_multi;
+		struct {
+			__u32 attach_type;
+			__u32 count; /* in/out: tracing_multi target count */
+			__u32 btf_obj_id;
+			__u32 :32;
+			__aligned_u64 ids;
+			__aligned_u64 addrs;
+			__aligned_u64 cookies;
+		} tracing_multi;
 		struct {
 			__u32 type; /* enum bpf_perf_event_type */
 			__u32 :32;
diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h
index 9b576603b3f14..0a13baf3d8d14 100644
--- a/include/uapi/linux/btrfs.h
+++ b/include/uapi/linux/btrfs.h
@@ -598,7 +598,7 @@ struct btrfs_ioctl_search_args_v2 {
 	__u64 buf_size;		   /* in - size of buffer
 					    * out - on EOVERFLOW: needed size
 					    *       to store item */
-	__u64 buf[];                       /* out - found items */
+	__u8 buf[];                        /* out - found items */
 };
 
 /* With a @src_length of zero, the range from @src_offset->EOF is cloned! */
diff --git a/include/uapi/linux/cec.h b/include/uapi/linux/cec.h
index 81a05c9c07060..fdfc97a6e4ec0 100644
--- a/include/uapi/linux/cec.h
+++ b/include/uapi/linux/cec.h
@@ -403,6 +403,8 @@ struct cec_log_addrs {
 #define CEC_LOG_ADDRS_FL_ALLOW_RC_PASSTHRU	(1 << 1)
 /* CDC-Only device: supports only CDC messages */
 #define CEC_LOG_ADDRS_FL_CDC_ONLY		(1 << 2)
+/* Configuration failed */
+#define CEC_LOG_ADDRS_FL_CONFIG_FAILED		(1 << 3)
 
 /**
  * struct cec_drm_connector_info - tells which drm connector is
diff --git a/include/uapi/linux/devlink.h b/include/uapi/linux/devlink.h
index ca713bcc47b9d..a6801feb77445 100644
--- a/include/uapi/linux/devlink.h
+++ b/include/uapi/linux/devlink.h
@@ -648,6 +648,8 @@ enum devlink_attr {
 	DEVLINK_ATTR_INDEX,			/* uint */
 	DEVLINK_ATTR_RESOURCE_SCOPE_MASK,	/* u32 */
 
+	DEVLINK_ATTR_PARENT_DEV,		/* nested */
+
 	/* Add new attributes above here, update the spec in
 	 * Documentation/netlink/specs/devlink.yaml and re-generate
 	 * net/devlink/netlink_gen.c.
diff --git a/include/uapi/linux/dpll.h b/include/uapi/linux/dpll.h
index 55eaa82f5f986..85b898b1db5ee 100644
--- a/include/uapi/linux/dpll.h
+++ b/include/uapi/linux/dpll.h
@@ -129,6 +129,9 @@ enum dpll_type {
  * @DPLL_PIN_TYPE_SYNCE_ETH_PORT: ethernet port PHY's recovered clock
  * @DPLL_PIN_TYPE_INT_OSCILLATOR: device internal oscillator
  * @DPLL_PIN_TYPE_GNSS: GNSS recovered clock
+ * @DPLL_PIN_TYPE_INT_NCO: Device internal numerically controlled oscillator.
+ *   When connected as a DPLL input, the DPLL enters NCO mode where the output
+ *   frequency is adjusted by the host via the PTP clock interface.
  */
 enum dpll_pin_type {
 	DPLL_PIN_TYPE_MUX = 1,
@@ -136,6 +139,7 @@ enum dpll_pin_type {
 	DPLL_PIN_TYPE_SYNCE_ETH_PORT,
 	DPLL_PIN_TYPE_INT_OSCILLATOR,
 	DPLL_PIN_TYPE_GNSS,
+	DPLL_PIN_TYPE_INT_NCO,
 
 	/* private: */
 	__DPLL_PIN_TYPE_MAX,
@@ -208,11 +212,15 @@ enum dpll_pin_operstate {
  * @DPLL_PIN_CAPABILITIES_DIRECTION_CAN_CHANGE: pin direction can be changed
  * @DPLL_PIN_CAPABILITIES_PRIORITY_CAN_CHANGE: pin priority can be changed
  * @DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE: pin state can be changed
+ * @DPLL_PIN_CAPABILITIES_STATE_CONNECTED_OVERRIDE: pin state can be set to
+ *   connected regardless of current DPLL device mode, overriding the active
+ *   input selection. Requires state-can-change to be set as well.
  */
 enum dpll_pin_capabilities {
 	DPLL_PIN_CAPABILITIES_DIRECTION_CAN_CHANGE = 1,
 	DPLL_PIN_CAPABILITIES_PRIORITY_CAN_CHANGE = 2,
 	DPLL_PIN_CAPABILITIES_STATE_CAN_CHANGE = 4,
+	DPLL_PIN_CAPABILITIES_STATE_CONNECTED_OVERRIDE = 8,
 };
 
 #define DPLL_PHASE_OFFSET_DIVIDER		1000
diff --git a/include/uapi/linux/efs_fs_sb.h b/include/uapi/linux/efs_fs_sb.h
deleted file mode 100644
index 6bad29a10faa4..0000000000000
--- a/include/uapi/linux/efs_fs_sb.h
+++ /dev/null
@@ -1,63 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-/*
- * efs_fs_sb.h
- *
- * Copyright (c) 1999 Al Smith
- *
- * Portions derived from IRIX header files (c) 1988 Silicon Graphics
- */
-
-#ifndef __EFS_FS_SB_H__
-#define __EFS_FS_SB_H__
-
-#include <linux/types.h>
-#include <linux/magic.h>
-
-/* EFS superblock magic numbers */
-#define EFS_MAGIC	0x072959
-#define EFS_NEWMAGIC	0x07295a
-
-#define IS_EFS_MAGIC(x)	((x == EFS_MAGIC) || (x == EFS_NEWMAGIC))
-
-#define EFS_SUPER		1
-#define EFS_ROOTINODE		2
-
-/* efs superblock on disk */
-struct efs_super {
-	__be32		fs_size;        /* size of filesystem, in sectors */
-	__be32		fs_firstcg;     /* bb offset to first cg */
-	__be32		fs_cgfsize;     /* size of cylinder group in bb's */
-	__be16		fs_cgisize;     /* bb's of inodes per cylinder group */
-	__be16		fs_sectors;     /* sectors per track */
-	__be16		fs_heads;       /* heads per cylinder */
-	__be16		fs_ncg;         /* # of cylinder groups in filesystem */
-	__be16		fs_dirty;       /* fs needs to be fsck'd */
-	__be32		fs_time;        /* last super-block update */
-	__be32		fs_magic;       /* magic number */
-	char		fs_fname[6];    /* file system name */
-	char		fs_fpack[6];    /* file system pack name */
-	__be32		fs_bmsize;      /* size of bitmap in bytes */
-	__be32		fs_tfree;       /* total free data blocks */
-	__be32		fs_tinode;      /* total free inodes */
-	__be32		fs_bmblock;     /* bitmap location. */
-	__be32		fs_replsb;      /* Location of replicated superblock. */
-	__be32		fs_lastialloc;  /* last allocated inode */
-	char		fs_spare[20];   /* space for expansion - MUST BE ZERO */
-	__be32		fs_checksum;    /* checksum of volume portion of fs */
-};
-
-/* efs superblock information in memory */
-struct efs_sb_info {
-	__u32	fs_magic;	/* superblock magic number */
-	__u32	fs_start;	/* first block of filesystem */
-	__u32	first_block;	/* first data block in filesystem */
-	__u32	total_blocks;	/* total number of blocks in filesystem */
-	__u32	group_size;	/* # of blocks a group consists of */ 
-	__u32	data_free;	/* # of free data blocks */
-	__u32	inode_free;	/* # of free inodes */
-	__u16	inode_blocks;	/* # of blocks used for inodes in every grp */
-	__u16	total_groups;	/* # of groups */
-};
-
-#endif /* __EFS_FS_SB_H__ */
-
diff --git a/include/uapi/linux/ethtool.h b/include/uapi/linux/ethtool.h
index a2091d4e00f3f..986d70caec335 100644
--- a/include/uapi/linux/ethtool.h
+++ b/include/uapi/linux/ethtool.h
@@ -2013,7 +2013,13 @@ enum ethtool_link_mode_bit_indices {
 	ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT	= 39,
 	ETHTOOL_LINK_MODE_50000baseSR2_Full_BIT		= 40,
 	ETHTOOL_LINK_MODE_1000baseX_Full_BIT	= 41,
+
+	/* Despite the "baseCR" in 10000baseCR, this is not an IEEE 802.3 baseCR
+	 * It represents SFF-8431 Appendix-E SFP+ Direct Attach (10G-SFI-DA).
+	 * The name is kept as-is for uAPI backward compatibility.
+	 */
 	ETHTOOL_LINK_MODE_10000baseCR_Full_BIT	= 42,
+
 	ETHTOOL_LINK_MODE_10000baseSR_Full_BIT	= 43,
 	ETHTOOL_LINK_MODE_10000baseLR_Full_BIT	= 44,
 	ETHTOOL_LINK_MODE_10000baseLRM_Full_BIT	= 45,
diff --git a/include/uapi/linux/fs.h b/include/uapi/linux/fs.h
index bd87262f2e349..34c6f219462a5 100644
--- a/include/uapi/linux/fs.h
+++ b/include/uapi/linux/fs.h
@@ -471,6 +471,7 @@ typedef int __bitwise __kernel_rwf_t;
 #define PAGE_IS_HUGE		(1 << 6)
 #define PAGE_IS_SOFT_DIRTY	(1 << 7)
 #define PAGE_IS_GUARD		(1 << 8)
+#define PAGE_IS_ACCESSED	(1 << 9)
 
 /*
  * struct page_region - Page region with flags
diff --git a/include/uapi/linux/fscrypt.h b/include/uapi/linux/fscrypt.h
index 3aff99f2696a0..84507280b3ead 100644
--- a/include/uapi/linux/fscrypt.h
+++ b/include/uapi/linux/fscrypt.h
@@ -30,7 +30,6 @@
 #define FSCRYPT_MODE_SM4_CTS			8
 #define FSCRYPT_MODE_ADIANTUM			9
 #define FSCRYPT_MODE_AES_256_HCTR2		10
-/* If adding a mode number > 10, update FSCRYPT_MODE_MAX in fscrypt_private.h */
 
 /*
  * Legacy policy version; ad-hoc KDF and no key verification.
diff --git a/include/uapi/linux/if_addr.h b/include/uapi/linux/if_addr.h
index aa7958b4e41d0..7fb630b7fe311 100644
--- a/include/uapi/linux/if_addr.h
+++ b/include/uapi/linux/if_addr.h
@@ -36,6 +36,7 @@ enum {
 	IFA_RT_PRIORITY,	/* u32, priority/metric for prefix route */
 	IFA_TARGET_NETNSID,
 	IFA_PROTO,		/* u8, address protocol */
+	IFA_MC_USERS,		/* u32, multicast group users */
 	__IFA_MAX,
 };
 
diff --git a/include/uapi/linux/iommufd.h b/include/uapi/linux/iommufd.h
index 0425d452d41ed..fddaf14cfdd20 100644
--- a/include/uapi/linux/iommufd.h
+++ b/include/uapi/linux/iommufd.h
@@ -57,6 +57,7 @@ enum {
 	IOMMUFD_CMD_IOAS_CHANGE_PROCESS = 0x92,
 	IOMMUFD_CMD_VEVENTQ_ALLOC = 0x93,
 	IOMMUFD_CMD_HW_QUEUE_ALLOC = 0x94,
+	IOMMUFD_CMD_IOAS_NOIOMMU_GET_PA = 0x95,
 };
 
 /**
@@ -219,6 +220,33 @@ struct iommu_ioas_map {
 };
 #define IOMMU_IOAS_MAP _IO(IOMMUFD_TYPE, IOMMUFD_CMD_IOAS_MAP)
 
+/**
+ * struct iommu_ioas_noiommu_get_pa - ioctl(IOMMU_IOAS_NOIOMMU_GET_PA)
+ * @size: sizeof(struct iommu_ioas_noiommu_get_pa)
+ * @flags: Reserved, must be 0 for now
+ * @ioas_id: IOAS ID to query IOVA to PA mapping from
+ * @__reserved: Must be 0
+ * @iova: IOVA to query
+ * @length: On input, non-zero maximum number of bytes to query starting from
+ *          @iova. On output, number of physically contiguous bytes starting
+ *          from @out_phys, capped by the input length.
+ * @out_phys: Output physical address the IOVA maps to
+ *
+ * Query the physical address backing an IOVA range. The beginning of the
+ * range must be mapped already and length must be non-zero. For noiommu
+ * devices doing unsafe DMA only.
+ */
+struct iommu_ioas_noiommu_get_pa {
+	__u32 size;
+	__u32 flags;
+	__u32 ioas_id;
+	__u32 __reserved;
+	__aligned_u64 iova;
+	__aligned_u64 length;
+	__aligned_u64 out_phys;
+};
+#define IOMMU_IOAS_NOIOMMU_GET_PA _IO(IOMMUFD_TYPE, IOMMUFD_CMD_IOAS_NOIOMMU_GET_PA)
+
 /**
  * struct iommu_ioas_map_file - ioctl(IOMMU_IOAS_MAP_FILE)
  * @size: sizeof(struct iommu_ioas_map_file)
diff --git a/include/uapi/linux/landlock.h b/include/uapi/linux/landlock.h
index 7ffe2ef127ee7..272f047df4382 100644
--- a/include/uapi/linux/landlock.h
+++ b/include/uapi/linux/landlock.h
@@ -315,16 +315,16 @@ struct landlock_net_port_attr {
  *   :manpage:`connect(2)` as well as calls to :manpage:`sendmsg(2)` with an
  *   explicit recipient address.
  *
- *   This access right only applies to connections to UNIX server sockets which
+ *   This access right applies only to connections to UNIX server sockets which
  *   were created outside of the newly created Landlock domain (e.g. from within
  *   a parent domain or from an unrestricted process).  Newly created UNIX
  *   servers within the same Landlock domain continue to be accessible.  In this
  *   regard, %LANDLOCK_ACCESS_FS_RESOLVE_UNIX has the same semantics as the
  *   ``LANDLOCK_SCOPE_*`` flags.
  *
- *   If a resolve attempt is denied, the operation returns an ``EACCES`` error,
- *   in line with other filesystem access rights (but different to denials for
- *   abstract UNIX domain sockets).
+ *   If a resolution attempt is denied, the operation returns an ``EACCES``
+ *   error, in line with other filesystem access rights (but different to
+ *   denials for abstract UNIX domain sockets).
  *
  *   This access right is available since the ninth version of the Landlock ABI.
  *
diff --git a/include/uapi/linux/mempolicy.h b/include/uapi/linux/mempolicy.h
index 6c962d866e864..7f6fc9599693b 100644
--- a/include/uapi/linux/mempolicy.h
+++ b/include/uapi/linux/mempolicy.h
@@ -16,7 +16,7 @@
  */
 
 /* Policies */
-enum {
+enum mempolicy_mode {
 	MPOL_DEFAULT,
 	MPOL_PREFERRED,
 	MPOL_BIND,
diff --git a/include/uapi/linux/nfs4.h b/include/uapi/linux/nfs4.h
index 4273e0249fcbb..289205b53a085 100644
--- a/include/uapi/linux/nfs4.h
+++ b/include/uapi/linux/nfs4.h
@@ -17,11 +17,9 @@
 #include <linux/types.h>
 
 #define NFS4_BITMAP_SIZE	3
-#define NFS4_VERIFIER_SIZE	8
 #define NFS4_STATEID_SEQID_SIZE 4
 #define NFS4_STATEID_OTHER_SIZE 12
 #define NFS4_STATEID_SIZE	(NFS4_STATEID_SEQID_SIZE + NFS4_STATEID_OTHER_SIZE)
-#define NFS4_FHSIZE		128
 #define NFS4_MAXPATHLEN		PATH_MAX
 #define NFS4_MAXNAMLEN		NAME_MAX
 #define NFS4_OPAQUE_LIMIT	1024
diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h
index 9998f6c0a665c..020387d764122 100644
--- a/include/uapi/linux/nl80211.h
+++ b/include/uapi/linux/nl80211.h
@@ -922,13 +922,15 @@
  *	and wasn't already in a 4-addr VLAN. The event will be sent similarly
  *	to the %NL80211_CMD_UNEXPECTED_FRAME event, to the same listener.
  *
- * @NL80211_CMD_PROBE_CLIENT: Probe an associated station on an AP interface
- *	by sending a null data frame to it and reporting when the frame is
- *	acknowledged. This is used to allow timing out inactive clients. Uses
- *	%NL80211_ATTR_IFINDEX and %NL80211_ATTR_MAC. The command returns a
- *	direct reply with an %NL80211_ATTR_COOKIE that is later used to match
- *	up the event with the request. The event includes the same data and
- *	has %NL80211_ATTR_ACK set if the frame was ACKed.
+ * @NL80211_CMD_PROBE_PEER: Probe a connected peer by sending a null data
+ *	frame and reporting when the frame is acknowledged.
+ *	In AP/GO mode, %NL80211_ATTR_MAC is required to identify the client.
+ *	In STA/P2P-client mode, %NL80211_ATTR_MAC must be omitted (the AP is
+ *	implied); the driver must advertise %NL80211_EXT_FEATURE_PROBE_AP.
+ *	The command returns a direct reply with an %NL80211_ATTR_COOKIE that
+ *	is later used to match up the event with the request. The event
+ *	includes the same data and has %NL80211_ATTR_ACK set if the frame
+ *	was ACKed.
  *
  * @NL80211_CMD_REGISTER_BEACONS: Register this socket to receive beacons from
  *	other BSSes when any interfaces are in AP mode. This helps implement
@@ -1558,7 +1560,7 @@ enum nl80211_commands {
 
 	NL80211_CMD_UNEXPECTED_FRAME,
 
-	NL80211_CMD_PROBE_CLIENT,
+	NL80211_CMD_PROBE_PEER,
 
 	NL80211_CMD_REGISTER_BEACONS,
 
@@ -1729,6 +1731,7 @@ enum nl80211_commands {
 #define NL80211_CMD_GET_MESH_PARAMS NL80211_CMD_GET_MESH_CONFIG
 #define NL80211_CMD_SET_MESH_PARAMS NL80211_CMD_SET_MESH_CONFIG
 #define NL80211_MESH_SETUP_VENDOR_PATH_SEL_IE NL80211_MESH_SETUP_IE
+#define NL80211_CMD_PROBE_CLIENT NL80211_CMD_PROBE_PEER
 
 /**
  * enum nl80211_attrs - nl80211 netlink attributes
@@ -3165,6 +3168,23 @@ enum nl80211_commands {
  * @NL80211_ATTR_NPCA_PRIMARY_FREQ: NPCA primary channel (u32)
  * @NL80211_ATTR_NPCA_PUNCT_BITMAP: NPCA puncturing bitmap (u32)
  *
+ * @NL80211_ATTR_STA_DUMP_LINK_STATS: Request flag for %NL80211_CMD_GET_STATION
+ *	(dump mode only). When set on an MLD station, the dump produces two
+ *	%NL80211_CMD_NEW_STATION messages per station per dump call:
+ *
+ *	1. An aggregated-stats message whose top-level %NL80211_ATTR_STA_INFO
+ *	   contains MLO-combined statistics (same content as a dump without
+ *	   this flag).
+ *
+ *	2. For each active link, a per-link message containing
+ *	   %NL80211_ATTR_MLO_LINKS with a single link entry. Each entry holds
+ *	   %NL80211_ATTR_MLO_LINK_ID, the link-specific %NL80211_ATTR_MAC,
+ *	   and %NL80211_ATTR_STA_INFO with per-link statistics (see
+ *	   &enum nl80211_sta_info).
+ *
+ *	The aggregated message always precedes the per-link messages for the
+ *	same station within a dump sequence.
+ *
  * @NUM_NL80211_ATTR: total number of nl80211_attrs available
  * @NL80211_ATTR_MAX: highest attribute number currently defined
  * @__NL80211_ATTR_AFTER_LAST: internal use
@@ -3763,6 +3783,8 @@ enum nl80211_attrs {
 	NL80211_ATTR_NPCA_PRIMARY_FREQ,
 	NL80211_ATTR_NPCA_PUNCT_BITMAP,
 
+	NL80211_ATTR_STA_DUMP_LINK_STATS,
+
 	/* add attributes here, update the policy in nl80211.c */
 
 	__NL80211_ATTR_AFTER_LAST,
@@ -4474,8 +4496,8 @@ enum nl80211_mpath_info {
  *     capabilities IE
  * @NL80211_BAND_IFTYPE_ATTR_HE_CAP_PPE: HE PPE thresholds information as
  *     defined in HE capabilities IE
- * @NL80211_BAND_IFTYPE_ATTR_HE_6GHZ_CAPA: HE 6GHz band capabilities (__le16),
- *	given for all 6 GHz band channels
+ * @NL80211_BAND_IFTYPE_ATTR_HE_6GHZ_CAPA: HE 6GHz band capabilities,
+ *	given for all 6 GHz band channels (binary, element content)
  * @NL80211_BAND_IFTYPE_ATTR_VENDOR_ELEMS: vendor element capabilities that are
  *	advertised on this band/for this iftype (binary)
  * @NL80211_BAND_IFTYPE_ATTR_EHT_CAP_MAC: EHT MAC capabilities as in EHT
@@ -7085,6 +7107,9 @@ enum nl80211_feature_flags {
  *	LTF key seed via %NL80211_KEY_LTF_SEED. The seed is used to generate
  *	secure LTF keys for secure LTF measurement sessions.
  *
+ * @NL80211_EXT_FEATURE_PROBE_AP: Driver supports probing the associated AP
+ *	in STA mode using @NL80211_CMD_PROBE_PEER.
+ *
  * @NUM_NL80211_EXT_FEATURES: number of extended features.
  * @MAX_NL80211_EXT_FEATURES: highest extended feature index.
  */
@@ -7166,6 +7191,7 @@ enum nl80211_ext_feature_index {
 	NL80211_EXT_FEATURE_IEEE8021X_AUTH,
 	NL80211_EXT_FEATURE_ROC_ADDR_FILTER,
 	NL80211_EXT_FEATURE_SET_KEY_LTF_SEED,
+	NL80211_EXT_FEATURE_PROBE_AP,
 
 	/* add new features before the definition below */
 	NUM_NL80211_EXT_FEATURES,
diff --git a/include/uapi/linux/serial.h b/include/uapi/linux/serial.h
index de9b4733607e6..e6f61538fc283 100644
--- a/include/uapi/linux/serial.h
+++ b/include/uapi/linux/serial.h
@@ -72,6 +72,7 @@ struct serial_struct {
 #define SERIAL_IO_TSI	  5
 #define SERIAL_IO_MEM32BE 6
 #define SERIAL_IO_MEM16	7
+#define SERIAL_IO_BUS	8
 
 #define UART_CLEAR_FIFO		0x01
 #define UART_USE_FIFO		0x02
diff --git a/include/uapi/linux/time_types.h b/include/uapi/linux/time_types.h
index bcc0002115d39..03a0d8aaadca5 100644
--- a/include/uapi/linux/time_types.h
+++ b/include/uapi/linux/time_types.h
@@ -30,7 +30,7 @@ struct __kernel_old_timeval {
 
 struct __kernel_old_timespec {
 	__kernel_old_time_t	tv_sec;		/* seconds */
-	long			tv_nsec;	/* nanoseconds */
+	__kernel_long_t		tv_nsec;	/* nanoseconds */
 };
 
 struct __kernel_old_itimerval {
diff --git a/include/uapi/linux/usb/functionfs.h b/include/uapi/linux/usb/functionfs.h
index beef1752e36e1..290308c9dd92b 100644
--- a/include/uapi/linux/usb/functionfs.h
+++ b/include/uapi/linux/usb/functionfs.h
@@ -25,6 +25,7 @@ enum functionfs_flags {
 	FUNCTIONFS_EVENTFD = 32,
 	FUNCTIONFS_ALL_CTRL_RECIP = 64,
 	FUNCTIONFS_CONFIG0_SETUP = 128,
+	FUNCTIONFS_RW_PROXY_EPS = 256,
 };
 
 /* Descriptor of an non-audio endpoint */
@@ -414,4 +415,27 @@ struct usb_functionfs_event {
 #define FUNCTIONFS_DMABUF_TRANSFER	_IOW('g', 133, \
 					     struct usb_ffs_dmabuf_transfer_req)
 
+/*
+ * Enable or disable automatic zero-length packet (ZLP) appending for the
+ * endpoint. The argument is a pointer to a __u32: 0 to disable, non-zero to
+ * enable.
+ *
+ * When enabled, the kernel will automatically append a ZLP at the end of
+ * a transfer if the payload length is an exact multiple of the endpoint's
+ * max packet size.
+ *
+ * This is useful for compatibility with legacy protocols which require
+ * automatic ZLP appending to data written from userspace.
+ *
+ * This ioctl can only be used on IN endpoints. It can be called at any time
+ * after the FunctionFS instance is active, even before the host has connected
+ * or enabled the endpoint.
+ *
+ * Returns zero on success, or a negative errno value on error:
+ * -ENODEV: The FunctionFS instance is not active.
+ * -EINVAL: The endpoint is not an IN endpoint.
+ * -EFAULT: Invalid user space pointer for the argument.
+ */
+#define	FUNCTIONFS_ENDPOINT_ENABLE_ZLP	_IOW('g', 134, __u32)
+
 #endif /* _UAPI__LINUX_FUNCTIONFS_H__ */
diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h
index 2841e4ea8f2ce..cea11aad6b544 100644
--- a/include/uapi/linux/userfaultfd.h
+++ b/include/uapi/linux/userfaultfd.h
@@ -25,7 +25,8 @@
 #define UFFD_API ((__u64)0xAA)
 #define UFFD_API_REGISTER_MODES (UFFDIO_REGISTER_MODE_MISSING |	\
 				 UFFDIO_REGISTER_MODE_WP |	\
-				 UFFDIO_REGISTER_MODE_MINOR)
+				 UFFDIO_REGISTER_MODE_MINOR |	\
+				 UFFDIO_REGISTER_MODE_RWP)
 #define UFFD_API_FEATURES (UFFD_FEATURE_PAGEFAULT_FLAG_WP |	\
 			   UFFD_FEATURE_EVENT_FORK |		\
 			   UFFD_FEATURE_EVENT_REMAP |		\
@@ -42,10 +43,13 @@
 			   UFFD_FEATURE_WP_UNPOPULATED |	\
 			   UFFD_FEATURE_POISON |		\
 			   UFFD_FEATURE_WP_ASYNC |		\
-			   UFFD_FEATURE_MOVE)
+			   UFFD_FEATURE_MOVE |			\
+			   UFFD_FEATURE_RWP |			\
+			   UFFD_FEATURE_RWP_ASYNC)
 #define UFFD_API_IOCTLS				\
 	((__u64)1 << _UFFDIO_REGISTER |		\
 	 (__u64)1 << _UFFDIO_UNREGISTER |	\
+	 (__u64)1 << _UFFDIO_SET_MODE |		\
 	 (__u64)1 << _UFFDIO_API)
 #define UFFD_API_RANGE_IOCTLS			\
 	((__u64)1 << _UFFDIO_WAKE |		\
@@ -54,13 +58,15 @@
 	 (__u64)1 << _UFFDIO_MOVE |		\
 	 (__u64)1 << _UFFDIO_WRITEPROTECT |	\
 	 (__u64)1 << _UFFDIO_CONTINUE |		\
-	 (__u64)1 << _UFFDIO_POISON)
+	 (__u64)1 << _UFFDIO_POISON |		\
+	 (__u64)1 << _UFFDIO_RWPROTECT)
 #define UFFD_API_RANGE_IOCTLS_BASIC		\
 	((__u64)1 << _UFFDIO_WAKE |		\
 	 (__u64)1 << _UFFDIO_COPY |		\
 	 (__u64)1 << _UFFDIO_WRITEPROTECT |	\
 	 (__u64)1 << _UFFDIO_CONTINUE |		\
-	 (__u64)1 << _UFFDIO_POISON)
+	 (__u64)1 << _UFFDIO_POISON |		\
+	 (__u64)1 << _UFFDIO_RWPROTECT)
 
 /*
  * Valid ioctl command number range with this API is from 0x00 to
@@ -79,6 +85,8 @@
 #define _UFFDIO_WRITEPROTECT		(0x06)
 #define _UFFDIO_CONTINUE		(0x07)
 #define _UFFDIO_POISON			(0x08)
+#define _UFFDIO_RWPROTECT		(0x09)
+#define _UFFDIO_SET_MODE		(0x0A)
 #define _UFFDIO_API			(0x3F)
 
 /* userfaultfd ioctl ids */
@@ -103,6 +111,10 @@
 				      struct uffdio_continue)
 #define UFFDIO_POISON		_IOWR(UFFDIO, _UFFDIO_POISON, \
 				      struct uffdio_poison)
+#define UFFDIO_RWPROTECT	_IOWR(UFFDIO, _UFFDIO_RWPROTECT,	\
+				      struct uffdio_rwprotect)
+#define UFFDIO_SET_MODE		_IOW(UFFDIO, _UFFDIO_SET_MODE,	\
+				     struct uffdio_set_mode)
 
 /* read() structure */
 struct uffd_msg {
@@ -158,6 +170,7 @@ struct uffd_msg {
 #define UFFD_PAGEFAULT_FLAG_WRITE	(1<<0)	/* If this was a write fault */
 #define UFFD_PAGEFAULT_FLAG_WP		(1<<1)	/* If reason is VM_UFFD_WP */
 #define UFFD_PAGEFAULT_FLAG_MINOR	(1<<2)	/* If reason is VM_UFFD_MINOR */
+#define UFFD_PAGEFAULT_FLAG_RWP		(1<<3)	/* If reason is VM_UFFD_RWP */
 
 struct uffdio_api {
 	/* userland asks for an API number and the features to enable */
@@ -230,6 +243,18 @@ struct uffdio_api {
 	 *
 	 * UFFD_FEATURE_MOVE indicates that the kernel supports moving an
 	 * existing page contents from userspace.
+	 *
+	 * UFFD_FEATURE_RWP indicates that the kernel supports
+	 * UFFDIO_REGISTER_MODE_RWP for read-write protection tracking.
+	 * Pages are made inaccessible via UFFDIO_RWPROTECT and faults
+	 * are delivered when the pages are re-accessed.
+	 *
+	 * UFFD_FEATURE_RWP_ASYNC indicates asynchronous mode for
+	 * UFFDIO_REGISTER_MODE_RWP.  When set, faults on read-write
+	 * protected pages are auto-resolved by the kernel (PTE
+	 * permissions restored immediately) without delivering a message
+	 * to the userfaultfd handler.  Use PAGEMAP_SCAN with inverted
+	 * PAGE_IS_ACCESSED to find pages that were not re-accessed.
 	 */
 #define UFFD_FEATURE_PAGEFAULT_FLAG_WP		(1<<0)
 #define UFFD_FEATURE_EVENT_FORK			(1<<1)
@@ -248,6 +273,8 @@ struct uffdio_api {
 #define UFFD_FEATURE_POISON			(1<<14)
 #define UFFD_FEATURE_WP_ASYNC			(1<<15)
 #define UFFD_FEATURE_MOVE			(1<<16)
+#define UFFD_FEATURE_RWP			(1<<17)
+#define UFFD_FEATURE_RWP_ASYNC			(1<<18)
 	__u64 features;
 
 	__u64 ioctls;
@@ -263,6 +290,7 @@ struct uffdio_register {
 #define UFFDIO_REGISTER_MODE_MISSING	((__u64)1<<0)
 #define UFFDIO_REGISTER_MODE_WP		((__u64)1<<1)
 #define UFFDIO_REGISTER_MODE_MINOR	((__u64)1<<2)
+#define UFFDIO_REGISTER_MODE_RWP	((__u64)1<<3)
 	__u64 mode;
 
 	/*
@@ -356,6 +384,14 @@ struct uffdio_poison {
 	__s64 updated;
 };
 
+struct uffdio_rwprotect {
+	struct uffdio_range range;
+	/* !RWP means undo RWP-protection */
+#define UFFDIO_RWPROTECT_MODE_RWP		((__u64)1<<0)
+#define UFFDIO_RWPROTECT_MODE_DONTWAKE		((__u64)1<<1)
+	__u64 mode;
+};
+
 struct uffdio_move {
 	__u64 dst;
 	__u64 src;
@@ -374,6 +410,16 @@ struct uffdio_move {
 	__s64 move;
 };
 
+struct uffdio_set_mode {
+	/*
+	 * Toggle async mode for features at runtime.
+	 * Supported: UFFD_FEATURE_RWP_ASYNC.
+	 * Setting a bit in both enable and disable is invalid.
+	 */
+	__u64 enable;
+	__u64 disable;
+};
+
 /*
  * Flags for the userfaultfd(2) system call itself.
  */
diff --git a/include/uapi/linux/vduse.h b/include/uapi/linux/vduse.h
index 361eea511c218..8c616895c5116 100644
--- a/include/uapi/linux/vduse.h
+++ b/include/uapi/linux/vduse.h
@@ -14,6 +14,12 @@
 
 #define VDUSE_API_VERSION_1	1
 
+/* The VDUSE instance expects a request for vq ready */
+#define VDUSE_F_QUEUE_READY	0
+
+/* The VDUSE instance expects a request for suspend */
+#define VDUSE_F_SUSPEND		1
+
 /*
  * Get the version of VDUSE API that kernel supported (VDUSE_API_VERSION).
  * This is used for future extension.
@@ -33,6 +39,7 @@
  * @vq_align: the allocation alignment of virtqueue's metadata
  * @ngroups: number of vq groups that VDUSE device declares
  * @nas: number of address spaces that VDUSE device declares
+ * @vduse_features: VDUSE features
  * @reserved: for future use, needs to be initialized to zero
  * @config_size: the size of the configuration space
  * @config: the buffer of the configuration space
@@ -49,7 +56,8 @@ struct vduse_dev_config {
 	__u32 vq_align;
 	__u32 ngroups; /* if VDUSE_API_VERSION >= 1 */
 	__u32 nas; /* if VDUSE_API_VERSION >= 1 */
-	__u32 reserved[11];
+	__u64 vduse_features; /* if VDUSE_GET_FEATURES is not EINVAL */
+	__u32 reserved[9];
 	__u32 config_size;
 	__u8 config[];
 };
@@ -63,6 +71,9 @@ struct vduse_dev_config {
  */
 #define VDUSE_DESTROY_DEV	_IOW(VDUSE_BASE, 0x03, char[VDUSE_NAME_MAX])
 
+/* Get the VDUSE supported features */
+#define VDUSE_GET_FEATURES	_IOR(VDUSE_BASE, 0x04, __u64)
+
 /* The ioctls for VDUSE device (/dev/vduse/$NAME) */
 
 /**
@@ -325,6 +336,8 @@ enum vduse_req_type {
 	VDUSE_SET_STATUS,
 	VDUSE_UPDATE_IOTLB,
 	VDUSE_SET_VQ_GROUP_ASID,
+	VDUSE_SET_VQ_READY,
+	VDUSE_SUSPEND,
 };
 
 /**
@@ -372,6 +385,15 @@ struct vduse_iova_range_v2 {
 	__u32 padding;
 };
 
+/**
+ * struct vduse_vq_ready - Virtqueue ready request message
+ * @num: Virtqueue number
+ */
+struct vduse_vq_ready {
+	__u32 num;
+	__u32 ready;
+};
+
 /**
  * struct vduse_dev_request - control request
  * @type: request type
@@ -382,6 +404,7 @@ struct vduse_iova_range_v2 {
  * @iova: IOVA range for updating
  * @iova_v2: IOVA range for updating if API_VERSION >= 1
  * @vq_group_asid: ASID of a virtqueue group
+ * @vq_ready: Virtqueue ready request
  * @padding: padding
  *
  * Structure used by read(2) on /dev/vduse/$NAME.
@@ -399,6 +422,10 @@ struct vduse_dev_request {
 		 */
 		struct vduse_iova_range_v2 iova_v2;
 		struct vduse_vq_group_asid vq_group_asid;
+
+		/* Only if VDUSE_F_QUEUE_READY is negotiated */
+		struct vduse_vq_ready vq_ready;
+
 		__u32 padding[32];
 	};
 };
diff --git a/include/uapi/rdma/rdma_netlink.h b/include/uapi/rdma/rdma_netlink.h
index aac9782ddc09a..3af946ecbac3e 100644
--- a/include/uapi/rdma/rdma_netlink.h
+++ b/include/uapi/rdma/rdma_netlink.h
@@ -604,6 +604,11 @@ enum rdma_nldev_attr {
 	RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES,	/* u32 */
 	RDMA_NLDEV_ATTR_FRMR_POOL_KEY_KERNEL_VENDOR_KEY,	/* u64 */
 
+	/*
+	 * Resource summary entry maximum value.
+	 */
+	RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_MAX,		/* u64 */
+
 	/*
 	 * Always the end
 	 */
diff --git a/include/uapi/sound/firewire.h b/include/uapi/sound/firewire.h
index 1e86872c151f5..1235d89d2bff7 100644
--- a/include/uapi/sound/firewire.h
+++ b/include/uapi/sound/firewire.h
@@ -79,11 +79,12 @@ struct snd_firewire_event_motu_register_dsp_change {
  *
  * @type: Fixed to SNDRV_FIREWIRE_EVENT_FF400_MESSAGE.
  * @message_count: The number of messages.
+ * @messages: Array of @message_count messages
  * @messages.message: The messages expressing hardware knob operation.
  * @messages.tstamp: The isochronous cycle at which the request subaction of asynchronous
- *		     transaction was sent to deliver the message. It has 16 bit unsigned integer
+ *		     transaction was sent to deliver the message. It has 16-bit unsigned integer
  *		     value. The higher 3 bits of value expresses the lower three bits of second
- *		     field in the format of CYCLE_TIME, up to 7. The rest 13 bits expresses cycle
+ *		     field in the format of CYCLE_TIME, up to 7. The remaining 13 bits express cycle
  *		     field up to 7999.
  *
  * The structure expresses message transmitted by Fireface 400 when operating hardware knob.
@@ -191,8 +192,9 @@ struct snd_firewire_motu_register_dsp_meter {
 #define SNDRV_FIREWIRE_MOTU_REGISTER_DSP_ALIGNED_INPUT_COUNT	(SNDRV_FIREWIRE_MOTU_REGISTER_DSP_INPUT_COUNT + 2)
 
 /**
- * snd_firewire_motu_register_dsp_parameter - the container for parameters of DSP controlled
- *					      by register access.
+ * struct snd_firewire_motu_register_dsp_parameter - the container for parameters
+ *                                                   of DSP controlled by register access.
+ * @mixer: aggregate of @mixer.source and @mixer.output
  * @mixer.source.gain: The gain of source to mixer.
  * @mixer.source.pan: The L/R balance of source to mixer.
  * @mixer.source.flag: The flag of source to mixer, including mute, solo.
@@ -200,21 +202,25 @@ struct snd_firewire_motu_register_dsp_meter {
  *				 Audio Express.
  * @mixer.source.paired_width: The width of paired source to mixer, only for 4 pre and
  *			       Audio Express.
+ * @mixer.output: FIXME
  * @mixer.output.paired_volume: The volume of paired output from mixer.
  * @mixer.output.paired_flag: The flag of paired output from mixer.
+ * @output: output parameters
  * @output.main_paired_volume: The volume of paired main output.
  * @output.hp_paired_volume: The volume of paired hp output.
  * @output.hp_paired_assignment: The source assigned to paired hp output.
- * @output.reserved: Padding for 32 bit alignment for future extension.
+ * @output.reserved: Padding for 32-bit alignment for future extension.
+ * @line_input: line input parameters
  * @line_input.boost_flag: The flags of boost for line inputs, only for 828mk2 and Traveler.
  * @line_input.nominal_level_flag: The flags of nominal level for line inputs, only for 828mk2 and
  *				   Traveler.
- * @line_input.reserved: Padding for 32 bit alignment for future extension.
+ * @line_input.reserved: Padding for 32-bit alignment for future extension.
+ * @input: input parameters
  * @input.gain_and_invert: The value including gain and invert for input, only for Ultralite, 4 pre
  *			   and Audio Express.
  * @input.flag: The flag of input; e.g. jack detection, phantom power, and pad, only for Ultralite,
  *		4 pre and Audio express.
- * @reserved: Padding so that the size of structure is kept to 512 byte, but for future extension.
+ * @reserved: Padding so that the size of structure is kept to 512 bytes, but for future extension.
  *
  * The structure expresses the set of parameters for DSP controlled by register access.
  */
@@ -272,8 +278,8 @@ struct snd_firewire_motu_register_dsp_parameter {
  *						controlled by command
  * @data: Signal level meters. The mapping between position and signal channel is model-dependent.
  *
- * The structure expresses the part of DSP status for hardware meter. The 32 bit storage is
- * estimated to include IEEE 764 32 bit single precision floating point (binary32) value. It is
+ * The structure expresses the part of DSP status for hardware meter. The 32-bit storage is
+ * estimated to include IEEE 764 32-bit single precision floating point (binary32) value. It is
  * expected to be linear value (not logarithm) for audio signal level between 0.0 and +1.0.
  */
 struct snd_firewire_motu_command_dsp_meter {
diff --git a/include/uapi/sound/sof/tokens.h b/include/uapi/sound/sof/tokens.h
index f4a7baadb44da..cc694a3979870 100644
--- a/include/uapi/sound/sof/tokens.h
+++ b/include/uapi/sound/sof/tokens.h
@@ -223,6 +223,7 @@
 #define SOF_TKN_AMD_ACPI2S_RATE			1700
 #define SOF_TKN_AMD_ACPI2S_CH			1701
 #define SOF_TKN_AMD_ACPI2S_TDM_MODE		1702
+#define SOF_TKN_AMD_ACPI2S_FORMAT		1703
 
 /* MICFIL PDM */
 #define SOF_TKN_IMX_MICFIL_RATE			2000
diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h
index 248d0a5bef407..1c732601afcea 100644
--- a/include/ufs/ufshcd.h
+++ b/include/ufs/ufshcd.h
@@ -359,6 +359,7 @@ struct ufshcd_tx_eqtr_record {
  * @is_valid: True if parameter contains valid TX Equalization settings
  * @is_applied: True if settings have been applied to UniPro of both sides
  * @is_trained: True if parameters obtained from TX EQTR procedure
+ * @from_dt: True if settings are from Device Tree
  */
 struct ufshcd_tx_eq_params {
 	struct ufshcd_tx_eq_settings host[UFS_MAX_LANES];
@@ -367,12 +368,12 @@ struct ufshcd_tx_eq_params {
 	bool is_valid;
 	bool is_applied;
 	bool is_trained;
+	bool from_dt;
 };
 
 /**
  * struct ufs_hba_variant_ops - variant specific callbacks
  * @name: variant name
- * @max_num_rtt: maximum RTT supported by the host
  * @init: called when the driver is initialized
  * @exit: called to cleanup everything done in init
  * @set_dma_mask: For setting another DMA mask than indicated by the 64AS
@@ -417,10 +418,11 @@ struct ufshcd_tx_eq_params {
  * @get_rx_fom: called to get Figure of Merit (FOM) value.
  * @tx_eqtr_notify: called before and after TX Equalization Training procedure
  *	to allow platform vendor specific configs to take place.
+ * @get_hba_nortt: called to get maximum number of outstanding RTTs supported by
+ *	the controller.
  */
 struct ufs_hba_variant_ops {
 	const char *name;
-	int	max_num_rtt;
 	int	(*init)(struct ufs_hba *);
 	void    (*exit)(struct ufs_hba *);
 	u32	(*get_ufs_hci_version)(struct ufs_hba *);
@@ -479,6 +481,7 @@ struct ufs_hba_variant_ops {
 	int	(*tx_eqtr_notify)(struct ufs_hba *hba,
 				  enum ufs_notify_change_status status,
 				  struct ufs_pa_layer_attr *pwr_mode);
+	int	(*get_hba_nortt)(struct ufs_hba *hba);
 };
 
 /* clock gating state  */
diff --git a/include/vdso/datapage.h b/include/vdso/datapage.h
index 5977723fb3b5d..09897f76ae07a 100644
--- a/include/vdso/datapage.h
+++ b/include/vdso/datapage.h
@@ -2,7 +2,7 @@
 #ifndef __VDSO_DATAPAGE_H
 #define __VDSO_DATAPAGE_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/types.h>
 
@@ -176,7 +176,7 @@ enum vdso_pages {
 	VDSO_NR_PAGES
 };
 
-#else /* !__ASSEMBLY__ */
+#else /* !__ASSEMBLER__ */
 
 #ifdef CONFIG_VDSO_GETRANDOM
 #define __vdso_u_rng_data	PROVIDE(vdso_u_rng_data = vdso_u_data + 2 * PAGE_SIZE);
@@ -197,6 +197,6 @@ enum vdso_pages {
 	__vdso_u_arch_data					\
 
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 #endif /* __VDSO_DATAPAGE_H */
diff --git a/include/vdso/helpers.h b/include/vdso/helpers.h
index a3bf4f1c0d37d..65151b681c4f5 100644
--- a/include/vdso/helpers.h
+++ b/include/vdso/helpers.h
@@ -2,7 +2,7 @@
 #ifndef __VDSO_HELPERS_H
 #define __VDSO_HELPERS_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <asm/barrier.h>
 #include <vdso/datapage.h>
@@ -111,6 +111,6 @@ static __always_inline void vdso_write_end(struct vdso_time_data *vd)
 	vdso_write_seq_end(&vc[CS_RAW]);
 }
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 #endif /* __VDSO_HELPERS_H */
diff --git a/include/vdso/processor.h b/include/vdso/processor.h
index fbe8265ea3c49..cc781912a696e 100644
--- a/include/vdso/processor.h
+++ b/include/vdso/processor.h
@@ -5,10 +5,10 @@
 #ifndef __VDSO_PROCESSOR_H
 #define __VDSO_PROCESSOR_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <asm/vdso/processor.h>
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 #endif /* __VDSO_PROCESSOR_H */
diff --git a/include/vdso/vsyscall.h b/include/vdso/vsyscall.h
index b0fdc9c6bf439..c5c2a2c078571 100644
--- a/include/vdso/vsyscall.h
+++ b/include/vdso/vsyscall.h
@@ -2,13 +2,13 @@
 #ifndef __VDSO_VSYSCALL_H
 #define __VDSO_VSYSCALL_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <asm/vdso/vsyscall.h>
 
 unsigned long vdso_update_begin(void);
 void vdso_update_end(unsigned long flags);
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 
 #endif /* __VDSO_VSYSCALL_H */
diff --git a/init/Kconfig b/init/Kconfig
index 5230d4879b1c8..08ff0a99a0759 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -190,6 +190,9 @@ config RUSTC_HAS_FILE_WITH_NUL
 config RUSTC_HAS_FILE_AS_C_STR
 	def_bool RUSTC_VERSION >= 109100
 
+config RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS
+	def_bool RUSTC_VERSION >= 109800
+
 config PAHOLE_VERSION
 	int
 	default "$(PAHOLE_VERSION)"
@@ -944,6 +947,13 @@ config SCHED_PROXY_EXEC
 
 endmenu
 
+#
+# For architectures that support present-but-inaccessible (PROT_NONE) page
+# table entries detectable via pte_protnone() / pmd_protnone():
+#
+config ARCH_HAS_PTE_PROTNONE
+	bool
+
 #
 # For architectures that want to enable the support for NUMA-affine scheduler
 # balancing logic:
@@ -1010,6 +1020,7 @@ config ARCH_WANT_NUMA_VARIABLE_LOCALITY
 config NUMA_BALANCING
 	bool "Memory placement aware NUMA scheduler"
 	depends on ARCH_SUPPORTS_NUMA_BALANCING
+	depends on ARCH_HAS_PTE_PROTNONE
 	depends on !ARCH_WANT_NUMA_VARIABLE_LOCALITY
 	depends on SMP && NUMA_MIGRATION && !PREEMPT_RT
 	help
@@ -1566,6 +1577,42 @@ config BOOT_CONFIG_EMBED_FILE
 	  This bootconfig will be used if there is no initrd or no other
 	  bootconfig in the initrd.
 
+config ARCH_SUPPORTS_CMDLINE_FROM_BOOTCONFIG
+	bool
+	help
+	  Silent symbol; no C code reads it directly. Architectures
+	  select it once their setup_arch() calls
+	  xbc_prepend_embedded_cmdline() before parse_early_param().
+	  Its only role is to gate the user-visible
+	  CMDLINE_FROM_BOOTCONFIG option per-arch, the same
+	  ARCH_SUPPORTS_* idiom used by ARCH_SUPPORTS_CFI, etc.
+
+config CMDLINE_FROM_BOOTCONFIG
+	bool "Render embedded bootconfig as kernel cmdline at build time"
+	depends on BOOT_CONFIG_EMBED_FILE != ""
+	depends on ARCH_SUPPORTS_CMDLINE_FROM_BOOTCONFIG
+	depends on CMDLINE = ""
+	default n
+	help
+	  Render the "kernel" subtree of the embedded bootconfig file into a
+	  flat cmdline string at kernel build time and prepend it to
+	  boot_command_line during early architecture setup. This makes
+	  early_param() handlers (e.g. mem=, earlycon=, loglevel=) see the
+	  values supplied via the embedded bootconfig.
+
+	  The runtime bootconfig parser is unaffected, so tree-structured
+	  consumers such as ftrace boot-time tracing keep working.
+
+	  Note: when an initrd also carries a bootconfig, its "kernel"
+	  subtree is still parsed at runtime, but the embedded "kernel"
+	  keys remain in boot_command_line for parse_early_param() and
+	  end up later than the initrd keys in saved_command_line, so
+	  parse_args() last-wins favors the embedded values. If you need
+	  initrd to override embedded kernel.* keys, leave this option
+	  off.
+
+	  If unsure, say N.
+
 config CMDLINE_LOG_WRAP_IDEAL_LEN
 	int "Length to try to wrap the cmdline when logged at boot"
 	default 1021
@@ -1857,7 +1904,7 @@ config HAVE_FUTEX_ROBUST_UNLOCK
 	bool
 
 config FUTEX_ROBUST_UNLOCK
-	def_bool FUTEX && HAVE_GENERIC_VDSO && GENERIC_IRQ_ENTRY && RSEQ && HAVE_FUTEX_ROBUST_UNLOCK
+	def_bool FUTEX && GENERIC_IRQ_ENTRY && RSEQ && HAVE_FUTEX_ROBUST_UNLOCK
 
 config EPOLL
 	bool "Enable eventpoll support" if EXPERT
diff --git a/init/init_task.c b/init/init_task.c
index b67ef6040a655..adb207cd987c5 100644
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -145,6 +145,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
 		.dsq_list.node	= LIST_HEAD_INIT(init_task.scx.dsq_list.node),
 		.sticky_cpu	= -1,
 		.holding_cpu	= -1,
+		.runnable_cpu	= -1,
 		.runnable_node	= LIST_HEAD_INIT(init_task.scx.runnable_node),
 		.runnable_at	= INITIAL_JIFFIES,
 		.ddsp_dsq_id	= SCX_DSQ_INVALID,
@@ -162,6 +163,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
 	RCU_POINTER_INITIALIZER(cred, &init_cred),
 	.comm		= INIT_TASK_COMM,
 	.thread		= INIT_THREAD,
+	.real_fs	= &init_fs,
 	.fs		= &init_fs,
 	.files		= &init_files,
 #ifdef CONFIG_IO_URING
diff --git a/init/initramfs.c b/init/initramfs.c
index 20a18fcda48ee..4e27b97a88440 100644
--- a/init/initramfs.c
+++ b/init/initramfs.c
@@ -6,6 +6,7 @@
 #include <linux/fcntl.h>
 #include <linux/file.h>
 #include <linux/fs.h>
+#include <linux/fs_struct.h>
 #include <linux/hex.h>
 #include <linux/init.h>
 #include <linux/init_syscalls.h>
@@ -716,7 +717,7 @@ static void __init populate_initrd_image(char *err)
 }
 #endif /* CONFIG_BLK_DEV_RAM */
 
-static void __init do_populate_rootfs(void *unused, async_cookie_t cookie)
+static void __init unpack_initramfs(async_cookie_t cookie)
 {
 	/* Load the built in initramfs */
 	char *err = unpack_to_rootfs(__initramfs_start, __initramfs_size);
@@ -724,7 +725,7 @@ static void __init do_populate_rootfs(void *unused, async_cookie_t cookie)
 		panic_show_mem("%s", err); /* Failed to decompress INTERNAL initramfs */
 
 	if (!initrd_start || IS_ENABLED(CONFIG_INITRAMFS_FORCE))
-		goto done;
+		return;
 
 	if (IS_ENABLED(CONFIG_BLK_DEV_RAM))
 		printk(KERN_INFO "Trying to unpack rootfs image as initramfs...\n");
@@ -739,9 +740,14 @@ static void __init do_populate_rootfs(void *unused, async_cookie_t cookie)
 		printk(KERN_EMERG "Initramfs unpacking failed: %s\n", err);
 #endif
 	}
+}
 
-done:
-	security_initramfs_populated();
+static void __init do_populate_rootfs(void *unused, async_cookie_t cookie)
+{
+	scoped_with_init_fs() {
+		unpack_initramfs(cookie);
+		security_initramfs_populated();
+	}
 
 	/*
 	 * If the initrd region is overlapped with crashkernel reserved region,
diff --git a/init/initramfs_test.c b/init/initramfs_test.c
index bc55306d226dc..1fc990a66563e 100644
--- a/init/initramfs_test.c
+++ b/init/initramfs_test.c
@@ -3,6 +3,7 @@
 #include <linux/fcntl.h>
 #include <linux/file.h>
 #include <linux/fs.h>
+#include <linux/fs_struct.h>
 #include <linux/init.h>
 #include <linux/init_syscalls.h>
 #include <linux/initrd.h>
@@ -116,38 +117,41 @@ static void __init initramfs_test_extract(struct kunit *test)
 			      GFP_KERNEL);
 	len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf);
 
-	ktime_get_real_ts64(&ts_before);
-	err = unpack_to_rootfs(cpio_srcbuf, len);
-	ktime_get_real_ts64(&ts_after);
-	if (err) {
-		KUNIT_FAIL(test, "unpack failed %s", err);
-		goto out;
+	/* Tests run in a nullfs kthread; borrow the init fs for path resolution. */
+	scoped_with_init_fs() {
+		ktime_get_real_ts64(&ts_before);
+		err = unpack_to_rootfs(cpio_srcbuf, len);
+		ktime_get_real_ts64(&ts_after);
+		if (err) {
+			KUNIT_FAIL(test, "unpack failed %s", err);
+			goto out;
+		}
+
+		KUNIT_EXPECT_EQ(test, init_stat(c[0].fname, &st, 0), 0);
+		KUNIT_EXPECT_TRUE(test, S_ISREG(st.mode));
+		KUNIT_EXPECT_TRUE(test, uid_eq(st.uid, KUIDT_INIT(c[0].uid)));
+		KUNIT_EXPECT_TRUE(test, gid_eq(st.gid, KGIDT_INIT(c[0].gid)));
+		KUNIT_EXPECT_EQ(test, st.nlink, 1);
+		if (IS_ENABLED(CONFIG_INITRAMFS_PRESERVE_MTIME)) {
+			KUNIT_EXPECT_EQ(test, st.mtime.tv_sec, c[0].mtime);
+		} else {
+			KUNIT_EXPECT_GE(test, st.mtime.tv_sec, ts_before.tv_sec);
+			KUNIT_EXPECT_LE(test, st.mtime.tv_sec, ts_after.tv_sec);
+		}
+		KUNIT_EXPECT_EQ(test, st.blocks, c[0].filesize);
+
+		KUNIT_EXPECT_EQ(test, init_stat(c[1].fname, &st, 0), 0);
+		KUNIT_EXPECT_TRUE(test, S_ISDIR(st.mode));
+		if (IS_ENABLED(CONFIG_INITRAMFS_PRESERVE_MTIME)) {
+			KUNIT_EXPECT_EQ(test, st.mtime.tv_sec, c[1].mtime);
+		} else {
+			KUNIT_EXPECT_GE(test, st.mtime.tv_sec, ts_before.tv_sec);
+			KUNIT_EXPECT_LE(test, st.mtime.tv_sec, ts_after.tv_sec);
+		}
+
+		KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0);
+		KUNIT_EXPECT_EQ(test, init_rmdir(c[1].fname), 0);
 	}
-
-	KUNIT_EXPECT_EQ(test, init_stat(c[0].fname, &st, 0), 0);
-	KUNIT_EXPECT_TRUE(test, S_ISREG(st.mode));
-	KUNIT_EXPECT_TRUE(test, uid_eq(st.uid, KUIDT_INIT(c[0].uid)));
-	KUNIT_EXPECT_TRUE(test, gid_eq(st.gid, KGIDT_INIT(c[0].gid)));
-	KUNIT_EXPECT_EQ(test, st.nlink, 1);
-	if (IS_ENABLED(CONFIG_INITRAMFS_PRESERVE_MTIME)) {
-		KUNIT_EXPECT_EQ(test, st.mtime.tv_sec, c[0].mtime);
-	} else {
-		KUNIT_EXPECT_GE(test, st.mtime.tv_sec, ts_before.tv_sec);
-		KUNIT_EXPECT_LE(test, st.mtime.tv_sec, ts_after.tv_sec);
-	}
-	KUNIT_EXPECT_EQ(test, st.blocks, c[0].filesize);
-
-	KUNIT_EXPECT_EQ(test, init_stat(c[1].fname, &st, 0), 0);
-	KUNIT_EXPECT_TRUE(test, S_ISDIR(st.mode));
-	if (IS_ENABLED(CONFIG_INITRAMFS_PRESERVE_MTIME)) {
-		KUNIT_EXPECT_EQ(test, st.mtime.tv_sec, c[1].mtime);
-	} else {
-		KUNIT_EXPECT_GE(test, st.mtime.tv_sec, ts_before.tv_sec);
-		KUNIT_EXPECT_LE(test, st.mtime.tv_sec, ts_after.tv_sec);
-	}
-
-	KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0);
-	KUNIT_EXPECT_EQ(test, init_rmdir(c[1].fname), 0);
 out:
 	kfree(cpio_srcbuf);
 }
@@ -197,8 +201,11 @@ static void __init initramfs_test_fname_overrun(struct kunit *test)
 		suffix_off--;
 	}
 
-	err = unpack_to_rootfs(cpio_srcbuf, len);
-	KUNIT_EXPECT_NOT_NULL(test, err);
+	/* Tests run in a nullfs kthread; borrow the init fs for path resolution. */
+	scoped_with_init_fs() {
+		err = unpack_to_rootfs(cpio_srcbuf, len);
+		KUNIT_EXPECT_NOT_NULL(test, err);
+	}
 
 	kfree(cpio_srcbuf);
 }
@@ -233,22 +240,25 @@ static void __init initramfs_test_data(struct kunit *test)
 
 	len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf);
 
-	err = unpack_to_rootfs(cpio_srcbuf, len);
-	KUNIT_EXPECT_NULL(test, err);
+	/* Tests run in a nullfs kthread; borrow the init fs for path resolution. */
+	scoped_with_init_fs() {
+		err = unpack_to_rootfs(cpio_srcbuf, len);
+		KUNIT_EXPECT_NULL(test, err);
 
-	file = filp_open(c[0].fname, O_RDONLY, 0);
-	if (IS_ERR(file)) {
-		KUNIT_FAIL(test, "open failed");
-		goto out;
-	}
+		file = filp_open(c[0].fname, O_RDONLY, 0);
+		if (IS_ERR(file)) {
+			KUNIT_FAIL(test, "open failed");
+			goto out;
+		}
 
-	/* read back file contents into @cpio_srcbuf and confirm match */
-	len = kernel_read(file, cpio_srcbuf, c[0].filesize, NULL);
-	KUNIT_EXPECT_EQ(test, len, c[0].filesize);
-	KUNIT_EXPECT_MEMEQ(test, cpio_srcbuf, c[0].data, len);
+		/* read back file contents into @cpio_srcbuf and confirm match */
+		len = kernel_read(file, cpio_srcbuf, c[0].filesize, NULL);
+		KUNIT_EXPECT_EQ(test, len, c[0].filesize);
+		KUNIT_EXPECT_MEMEQ(test, cpio_srcbuf, c[0].data, len);
 
-	fput(file);
-	KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0);
+		fput(file);
+		KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0);
+	}
 out:
 	kfree(cpio_srcbuf);
 }
@@ -288,25 +298,28 @@ static void __init initramfs_test_csum(struct kunit *test)
 
 	len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf);
 
-	err = unpack_to_rootfs(cpio_srcbuf, len);
-	KUNIT_EXPECT_NULL(test, err);
+	/* Tests run in a nullfs kthread; borrow the init fs for path resolution. */
+	scoped_with_init_fs() {
+		err = unpack_to_rootfs(cpio_srcbuf, len);
+		KUNIT_EXPECT_NULL(test, err);
 
-	KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0);
-	KUNIT_EXPECT_EQ(test, init_unlink(c[1].fname), 0);
+		KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0);
+		KUNIT_EXPECT_EQ(test, init_unlink(c[1].fname), 0);
 
-	/* mess up the csum and confirm that unpack fails */
-	c[0].csum--;
-	len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf);
+		/* mess up the csum and confirm that unpack fails */
+		c[0].csum--;
+		len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf);
 
-	err = unpack_to_rootfs(cpio_srcbuf, len);
-	KUNIT_EXPECT_NOT_NULL(test, err);
+		err = unpack_to_rootfs(cpio_srcbuf, len);
+		KUNIT_EXPECT_NOT_NULL(test, err);
 
-	/*
-	 * file (with content) is still retained in case of bad-csum abort.
-	 * Perhaps we should change this.
-	 */
-	KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0);
-	KUNIT_EXPECT_EQ(test, init_unlink(c[1].fname), -ENOENT);
+		/*
+		 * file (with content) is still retained in case of bad-csum abort.
+		 * Perhaps we should change this.
+		 */
+		KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0);
+		KUNIT_EXPECT_EQ(test, init_unlink(c[1].fname), -ENOENT);
+	}
 	kfree(cpio_srcbuf);
 }
 
@@ -344,17 +357,20 @@ static void __init initramfs_test_hardlink(struct kunit *test)
 
 	len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf);
 
-	err = unpack_to_rootfs(cpio_srcbuf, len);
-	KUNIT_EXPECT_NULL(test, err);
+	/* Tests run in a nullfs kthread; borrow the init fs for path resolution. */
+	scoped_with_init_fs() {
+		err = unpack_to_rootfs(cpio_srcbuf, len);
+		KUNIT_EXPECT_NULL(test, err);
 
-	KUNIT_EXPECT_EQ(test, init_stat(c[0].fname, &st0, 0), 0);
-	KUNIT_EXPECT_EQ(test, init_stat(c[1].fname, &st1, 0), 0);
-	KUNIT_EXPECT_EQ(test, st0.ino, st1.ino);
-	KUNIT_EXPECT_EQ(test, st0.nlink, 2);
-	KUNIT_EXPECT_EQ(test, st1.nlink, 2);
+		KUNIT_EXPECT_EQ(test, init_stat(c[0].fname, &st0, 0), 0);
+		KUNIT_EXPECT_EQ(test, init_stat(c[1].fname, &st1, 0), 0);
+		KUNIT_EXPECT_EQ(test, st0.ino, st1.ino);
+		KUNIT_EXPECT_EQ(test, st0.nlink, 2);
+		KUNIT_EXPECT_EQ(test, st1.nlink, 2);
 
-	KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0);
-	KUNIT_EXPECT_EQ(test, init_unlink(c[1].fname), 0);
+		KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0);
+		KUNIT_EXPECT_EQ(test, init_unlink(c[1].fname), 0);
+	}
 
 	kfree(cpio_srcbuf);
 }
@@ -387,12 +403,15 @@ static void __init initramfs_test_many(struct kunit *test)
 	}
 
 	len = p - cpio_srcbuf;
-	err = unpack_to_rootfs(cpio_srcbuf, len);
-	KUNIT_EXPECT_NULL(test, err);
-
-	for (i = 0; i < INITRAMFS_TEST_MANY_LIMIT; i++) {
-		sprintf(thispath, "initramfs_test_many-%d", i);
-		KUNIT_EXPECT_EQ(test, init_unlink(thispath), 0);
+	/* Tests run in a nullfs kthread; borrow the init fs for path resolution. */
+	scoped_with_init_fs() {
+		err = unpack_to_rootfs(cpio_srcbuf, len);
+		KUNIT_EXPECT_NULL(test, err);
+
+		for (i = 0; i < INITRAMFS_TEST_MANY_LIMIT; i++) {
+			sprintf(thispath, "initramfs_test_many-%d", i);
+			KUNIT_EXPECT_EQ(test, init_unlink(thispath), 0);
+		}
 	}
 
 	kfree(cpio_srcbuf);
@@ -439,22 +458,25 @@ static void __init initramfs_test_fname_pad(struct kunit *test)
 	memcpy(tbufs->padded_fname, "padded_fname", sizeof("padded_fname"));
 	len = fill_cpio(c, ARRAY_SIZE(c), false, tbufs->cpio_srcbuf);
 
-	err = unpack_to_rootfs(tbufs->cpio_srcbuf, len);
-	KUNIT_EXPECT_NULL(test, err);
+	/* Tests run in a nullfs kthread; borrow the init fs for path resolution. */
+	scoped_with_init_fs() {
+		err = unpack_to_rootfs(tbufs->cpio_srcbuf, len);
+		KUNIT_EXPECT_NULL(test, err);
 
-	file = filp_open(c[0].fname, O_RDONLY, 0);
-	if (IS_ERR(file)) {
-		KUNIT_FAIL(test, "open failed");
-		goto out;
-	}
+		file = filp_open(c[0].fname, O_RDONLY, 0);
+		if (IS_ERR(file)) {
+			KUNIT_FAIL(test, "open failed");
+			goto out;
+		}
 
-	/* read back file contents into @cpio_srcbuf and confirm match */
-	len = kernel_read(file, tbufs->cpio_srcbuf, c[0].filesize, NULL);
-	KUNIT_EXPECT_EQ(test, len, c[0].filesize);
-	KUNIT_EXPECT_MEMEQ(test, tbufs->cpio_srcbuf, c[0].data, len);
+		/* read back file contents into @cpio_srcbuf and confirm match */
+		len = kernel_read(file, tbufs->cpio_srcbuf, c[0].filesize, NULL);
+		KUNIT_EXPECT_EQ(test, len, c[0].filesize);
+		KUNIT_EXPECT_MEMEQ(test, tbufs->cpio_srcbuf, c[0].data, len);
 
-	fput(file);
-	KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0);
+		fput(file);
+		KUNIT_EXPECT_EQ(test, init_unlink(c[0].fname), 0);
+	}
 out:
 	kfree(tbufs);
 }
@@ -495,13 +517,16 @@ static void __init initramfs_test_fname_path_max(struct kunit *test)
 	memcpy(tbufs->fname_ok, "fname_ok", sizeof("fname_ok") - 1);
 	len = fill_cpio(c, ARRAY_SIZE(c), false, tbufs->cpio_src);
 
-	/* unpack skips over fname_oversize instead of returning an error */
-	err = unpack_to_rootfs(tbufs->cpio_src, len);
-	KUNIT_EXPECT_NULL(test, err);
+	/* Tests run in a nullfs kthread; borrow the init fs for path resolution. */
+	scoped_with_init_fs() {
+		/* unpack skips over fname_oversize instead of returning an error */
+		err = unpack_to_rootfs(tbufs->cpio_src, len);
+		KUNIT_EXPECT_NULL(test, err);
 
-	KUNIT_EXPECT_EQ(test, init_stat("fname_oversize", &st0, 0), -ENOENT);
-	KUNIT_EXPECT_EQ(test, init_stat("fname_ok", &st1, 0), 0);
-	KUNIT_EXPECT_EQ(test, init_rmdir("fname_ok"), 0);
+		KUNIT_EXPECT_EQ(test, init_stat("fname_oversize", &st0, 0), -ENOENT);
+		KUNIT_EXPECT_EQ(test, init_stat("fname_ok", &st1, 0), 0);
+		KUNIT_EXPECT_EQ(test, init_rmdir("fname_ok"), 0);
+	}
 
 	kfree(tbufs);
 }
@@ -539,8 +564,11 @@ static void __init initramfs_test_hdr_hex(struct kunit *test)
 	/* inject_ox=true to add "0x" cpio field prefixes */
 	len = fill_cpio(c, ARRAY_SIZE(c), true, tbufs->cpio_src);
 
-	err = unpack_to_rootfs(tbufs->cpio_src, len);
-	KUNIT_EXPECT_NOT_NULL(test, err);
+	/* Tests run in a nullfs kthread; borrow the init fs for path resolution. */
+	scoped_with_init_fs() {
+		err = unpack_to_rootfs(tbufs->cpio_src, len);
+		KUNIT_EXPECT_NOT_NULL(test, err);
+	}
 
 	kfree(tbufs);
 }
diff --git a/init/main.c b/init/main.c
index e363232b428b4..2613d3f9b3ce9 100644
--- a/init/main.c
+++ b/init/main.c
@@ -103,6 +103,7 @@
 #include <linux/stackdepot.h>
 #include <linux/randomize_kstack.h>
 #include <linux/pidfs.h>
+#include <linux/fs_struct.h>
 #include <linux/ptdump.h>
 #include <linux/time_namespace.h>
 #include <linux/unaligned.h>
@@ -356,45 +357,33 @@ static char * __init xbc_make_cmdline(const char *key)
 	return new_cmdline;
 }
 
-static int __init bootconfig_params(char *param, char *val,
-				    const char *unused, void *arg)
-{
-	if (strcmp(param, "bootconfig") == 0) {
-		bootconfig_found = true;
-	}
-	return 0;
-}
-
 static int __init warn_bootconfig(char *str)
 {
-	/* The 'bootconfig' has been handled by bootconfig_params(). */
+	/* The 'bootconfig' option is handled by setup_boot_config(). */
 	return 0;
 }
 
 static void __init setup_boot_config(void)
 {
-	static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;
 	const char *msg, *data;
-	int pos, ret;
+	int pos, ret, offs;
 	size_t size;
-	char *err;
+	bool from_embedded = false;
 
 	/* Cut out the bootconfig data even if we have no bootconfig option */
 	data = get_boot_config_from_initrd(&size);
 	/* If there is no bootconfig in initrd, try embedded one. */
-	if (!data)
+	if (!data) {
 		data = xbc_get_embedded_bootconfig(&size);
+		from_embedded = true;
+	}
 
-	strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
-	err = parse_args("bootconfig", tmp_cmdline, NULL, 0, 0, 0, NULL,
-			 bootconfig_params);
-
-	if (IS_ERR(err) || !(bootconfig_found || IS_ENABLED(CONFIG_BOOT_CONFIG_FORCE)))
+	bootconfig_found = bootconfig_cmdline_requested(boot_command_line, &offs);
+	if (!(bootconfig_found || IS_ENABLED(CONFIG_BOOT_CONFIG_FORCE)))
 		return;
 
-	/* parse_args() stops at the next param of '--' and returns an address */
-	if (err)
-		initargs_offs = err - tmp_cmdline;
+	/* Offset of the init arguments after a "--", located by the helper. */
+	initargs_offs = offs;
 
 	if (!data) {
 		/* If user intended to use bootconfig, show an error level message */
@@ -421,8 +410,24 @@ static void __init setup_boot_config(void)
 	} else {
 		xbc_get_info(&ret, NULL);
 		pr_info("Load bootconfig: %ld bytes %d nodes\n", (long)size, ret);
-		/* keys starting with "kernel." are passed via cmdline */
-		extra_command_line = xbc_make_cmdline("kernel");
+		/*
+		 * keys starting with "kernel." are passed via cmdline. When
+		 * this bootconfig came from the embedded source and
+		 * setup_arch() already prepended the rendered "kernel" subtree
+		 * to boot_command_line, rendering again here would duplicate
+		 * the keys in saved_command_line and make accumulating handlers
+		 * (console=, earlycon=, ...) re-register the same value. Skip
+		 * only when the prepend really happened.
+		 *
+		 * On arches that do not select ARCH_SUPPORTS_CMDLINE_FROM_BOOTCONFIG,
+		 * CONFIG_CMDLINE_FROM_BOOTCONFIG is unselectable and
+		 * xbc_embedded_cmdline_applied() collapses to a stub returning
+		 * false, so this path still runs and the embedded "kernel"
+		 * keys reach the cmdline via the runtime parser exactly as
+		 * before this series.
+		 */
+		if (!from_embedded || !xbc_embedded_cmdline_applied())
+			extra_command_line = xbc_make_cmdline("kernel");
 		/* Also, "init." keys are init arguments */
 		extra_init_args = xbc_make_cmdline("init");
 	}
@@ -670,6 +675,11 @@ static __initdata DECLARE_COMPLETION(kthreadd_done);
 
 static noinline void __ref __noreturn rest_init(void)
 {
+	struct kernel_clone_args init_args = {
+		.flags		= (CLONE_VM | CLONE_UNTRACED),
+		.fn		= kernel_init,
+		.fn_arg		= NULL,
+	};
 	struct task_struct *tsk;
 	int pid;
 
@@ -679,7 +689,7 @@ static noinline void __ref __noreturn rest_init(void)
 	 * the init task will end up wanting to create kthreads, which, if
 	 * we schedule it before we create kthreadd, will OOPS.
 	 */
-	pid = user_mode_thread(kernel_init, NULL, CLONE_FS);
+	pid = kernel_clone(&init_args);
 	/*
 	 * Pin init on the boot CPU. Task migration is not properly working
 	 * until sched_init_smp() has been run. It will set the allowed
@@ -1540,6 +1550,8 @@ static int __ref kernel_init(void *unused)
 {
 	int ret;
 
+	init_userspace_fs();
+
 	/*
 	 * Wait until kthreadd is all set-up.
 	 */
diff --git a/io_uring/fs.c b/io_uring/fs.c
index d0580c754bf86..26ea841a22e70 100644
--- a/io_uring/fs.c
+++ b/io_uring/fs.c
@@ -110,7 +110,8 @@ int io_unlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 	const char __user *fname;
 	int err;
 
-	if (sqe->off || sqe->len || sqe->buf_index || sqe->splice_fd_in)
+	if (sqe->off || sqe->len || sqe->buf_index || sqe->splice_fd_in ||
+	    sqe->addr3 || sqe->__pad2[0])
 		return -EINVAL;
 	if (unlikely(req->flags & REQ_F_FIXED_FILE))
 		return -EBADF;
diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c
index 2e14880eef92a..be8d75731d246 100644
--- a/io_uring/io-wq.c
+++ b/io_uring/io-wq.c
@@ -19,6 +19,7 @@
 #include <linux/mmu_context.h>
 #include <linux/sched/sysctl.h>
 #include <uapi/linux/io_uring.h>
+#include <linux/kcov.h>
 
 #include "io-wq.h"
 #include "slist.h"
@@ -643,13 +644,17 @@ static void io_worker_handle_work(struct io_wq_acct *acct,
 			unsigned int hash = __io_wq_is_hashed(work_flags)
 				? __io_get_work_hash(work_flags)
 				: -1U;
+			struct io_kiocb *req;
 
 			next_hashed = wq_next_work(work);
 
 			if (do_kill &&
 			    (work_flags & IO_WQ_WORK_UNBOUND))
 				atomic_or(IO_WQ_WORK_CANCEL, &work->flags);
+			req = container_of(work, struct io_kiocb, work);
+			kcov_remote_start_common(req->ctx->kcov_handle);
 			io_wq_submit_work(work);
+			kcov_remote_stop();
 			io_assign_current_work(worker, NULL);
 
 			linked = io_wq_free_work(work);
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 1ea2fca34a36f..1279e27c2c6d2 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -59,6 +59,7 @@
 #include <linux/audit.h>
 #include <linux/security.h>
 #include <linux/jump_label.h>
+#include <linux/kcov.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/io_uring.h>
@@ -293,6 +294,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
 	INIT_HLIST_HEAD(&ctx->cancelable_uring_cmd);
 	io_napi_init(ctx);
 	mutex_init(&ctx->mmap_lock);
+	ctx->kcov_handle = kcov_common_handle();
 
 	return ctx;
 
diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c
index b6b969b55e122..de0129bceaba3 100644
--- a/io_uring/kbuf.c
+++ b/io_uring/kbuf.c
@@ -328,8 +328,8 @@ static int io_ring_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg,
 		buf = io_ring_head_to_buf(br, ++head, bl->mask);
 	} while (--nr_iovs);
 
-	if (arg->mode & KBUF_MODE_FREE)
-		kfree(arg->iovs);
+	if (arg->iovs != org_iovs && (arg->mode & KBUF_MODE_FREE))
+		kfree(org_iovs);
 
 	if (head == tail)
 		req->flags |= REQ_F_BL_EMPTY;
diff --git a/io_uring/sqpoll.c b/io_uring/sqpoll.c
index 2460bd6052665..ad42e8eb10029 100644
--- a/io_uring/sqpoll.c
+++ b/io_uring/sqpoll.c
@@ -13,6 +13,7 @@
 #include <linux/cpuset.h>
 #include <linux/sched/cputime.h>
 #include <linux/io_uring.h>
+#include <linux/kcov.h>
 
 #include <uapi/linux/io_uring.h>
 
@@ -332,10 +333,14 @@ static int io_sq_thread(void *data)
 
 		cap_entries = !list_is_singular(&sqd->ctx_list);
 		list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
-			int ret = __io_sq_thread(ctx, sqd, cap_entries, &ist);
+			int ret;
+
+			kcov_remote_start_common(ctx->kcov_handle);
+			ret = __io_sq_thread(ctx, sqd, cap_entries, &ist);
 
 			if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list)))
 				sqt_spin = true;
+			kcov_remote_stop();
 		}
 		if (io_sq_tw(IORING_TW_CAP_ENTRIES_VALUE))
 			sqt_spin = true;
diff --git a/ipc/ipc_sysctl.c b/ipc/ipc_sysctl.c
index 9b087ebeb643b..d038d944257f6 100644
--- a/ipc/ipc_sysctl.c
+++ b/ipc/ipc_sysctl.c
@@ -28,7 +28,7 @@ static int proc_ipc_dointvec_minmax_orphans(const struct ctl_table *table, int w
 
 	if (err < 0)
 		return err;
-	if (ns->shm_rmid_forced)
+	if (write && ns->shm_rmid_forced)
 		shm_destroy_orphaned(ns);
 	return err;
 }
diff --git a/ipc/mqueue.c b/ipc/mqueue.c
index 4798b375972bf..d1dd36a651b0d 100644
--- a/ipc/mqueue.c
+++ b/ipc/mqueue.c
@@ -608,7 +608,7 @@ static int mqueue_create_attr(struct dentry *dentry, umode_t mode, void *arg)
 }
 
 static int mqueue_create(struct mnt_idmap *idmap, struct inode *dir,
-			 struct dentry *dentry, umode_t mode, bool excl)
+			 struct dentry *dentry, umode_t mode)
 {
 	return mqueue_create_attr(dentry, mode, NULL);
 }
@@ -790,7 +790,6 @@ static void __do_notify(struct mqueue_inode_info *info)
 			struct kernel_siginfo sig_i;
 			struct task_struct *task;
 
-			/* do_mq_notify() accepts sigev_signo == 0, why?? */
 			if (!info->notify.sigev_signo)
 				break;
 
@@ -1281,9 +1280,9 @@ static int do_mq_notify(mqd_t mqdes, const struct sigevent *notification)
 			     notification->sigev_notify != SIGEV_THREAD))
 			return -EINVAL;
 		if (notification->sigev_notify == SIGEV_SIGNAL &&
-			!valid_signal(notification->sigev_signo)) {
+		    (!notification->sigev_signo ||
+		     !valid_signal(notification->sigev_signo)))
 			return -EINVAL;
-		}
 		if (notification->sigev_notify == SIGEV_THREAD) {
 			long timeo;
 
diff --git a/ipc/shm.c b/ipc/shm.c
index b3e8a58e177d3..bb1a721a3e746 100644
--- a/ipc/shm.c
+++ b/ipc/shm.c
@@ -1661,7 +1661,8 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg,
 			goto invalid;
 	}
 
-	addr = do_mmap(file, addr, size, prot, flags, 0, 0, &populate, NULL);
+	addr = do_mmap(file, addr, size, prot, flags, EMPTY_VMA_FLAGS, 0,
+		       &populate, NULL);
 	*raddr = addr;
 	err = 0;
 	if (IS_ERR_VALUE(addr))
diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c
index 1433809bb166a..3983b4ce73c81 100644
--- a/kernel/bpf/bpf_lsm.c
+++ b/kernel/bpf/bpf_lsm.c
@@ -295,7 +295,6 @@ BTF_ID(func, bpf_lsm_bpf_map_create)
 BTF_ID(func, bpf_lsm_bpf_map_free)
 BTF_ID(func, bpf_lsm_bpf_prog)
 BTF_ID(func, bpf_lsm_bpf_prog_load)
-BTF_ID(func, bpf_lsm_bpf_prog_free)
 BTF_ID(func, bpf_lsm_bpf_token_create)
 BTF_ID(func, bpf_lsm_bpf_token_free)
 BTF_ID(func, bpf_lsm_bpf_token_cmd)
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 64572f85edc83..cbb1e49b9bcbf 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -6451,7 +6451,7 @@ struct btf *btf_parse_vmlinux(void)
 	if (IS_ERR(btf))
 		goto err_out;
 
-	/* btf_parse_vmlinux() runs under bpf_verifier_lock */
+	/* btf_parse_vmlinux() runs under btf_vmlinux_lock */
 	bpf_ctx_convert.t = btf_type_by_id(btf, bpf_ctx_convert_btf_id[0]);
 	err = btf_alloc_id(btf);
 	if (err) {
@@ -7108,7 +7108,7 @@ enum bpf_struct_walk_result {
 static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf,
 			   const struct btf_type *t, int off, int size,
 			   u32 *next_btf_id, enum bpf_type_flag *flag,
-			   const char **field_name)
+			   const char **field_name, bool walk_flex_arrays)
 {
 	u32 i, moff, mtrue_end, msize = 0, total_nelems = 0;
 	const struct btf_type *mtype, *elem_type = NULL;
@@ -7135,11 +7135,14 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf,
 		*flag |= PTR_UNTRUSTED;
 
 	if (off + size > t->size) {
+		struct btf_array *array_elem;
+
+		if (!walk_flex_arrays)
+			goto error;
+
 		/* If the last element is a variable size array, we may
 		 * need to relax the rule.
 		 */
-		struct btf_array *array_elem;
-
 		if (vlen == 0)
 			goto error;
 
@@ -7404,7 +7407,8 @@ int btf_struct_access(struct bpf_verifier_log *log,
 
 	t = btf_type_by_id(btf, id);
 	do {
-		err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, field_name);
+		err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag,
+				      field_name, !type_is_alloc(reg->type));
 
 		switch (err) {
 		case WALK_PTR:
@@ -7463,7 +7467,7 @@ bool btf_types_are_same(const struct btf *btf1, u32 id1,
 bool btf_struct_ids_match(struct bpf_verifier_log *log,
 			  const struct btf *btf, u32 id, int off,
 			  const struct btf *need_btf, u32 need_type_id,
-			  bool strict)
+			  bool strict, bool walk_flex_arrays)
 {
 	const struct btf_type *type;
 	enum bpf_type_flag flag = 0;
@@ -7482,7 +7486,8 @@ bool btf_struct_ids_match(struct bpf_verifier_log *log,
 	type = btf_type_by_id(btf, id);
 	if (!type)
 		return false;
-	err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL);
+	err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL,
+			      walk_flex_arrays);
 	if (err != WALK_STRUCT)
 		return false;
 
@@ -9114,6 +9119,35 @@ u32 *btf_kfunc_flags(const struct btf *btf, u32 kfunc_btf_id, const struct bpf_p
 	return btf_kfunc_id_set_contains(btf, hook, kfunc_btf_id);
 }
 
+/*
+ * Check a single KF_* @flag on a kfunc across all of its hook sets.
+ * Returns:
+ *   * 1 if @flag is set
+ *   * 0 if @flag is not set
+ *   * -EINVAL if @flag is set inconsistently across the sets
+ *   * -ENOENT if kfunc_btf_id is not a registered kfunc
+ */
+int btf_kfunc_check_flag(const struct btf *btf, u32 kfunc_btf_id, u32 flag)
+{
+	enum btf_kfunc_hook hook;
+	int res = -ENOENT;
+	bool is_set;
+	u32 *flags;
+
+	for (hook = 0; hook < BTF_KFUNC_HOOK_MAX; hook++) {
+		flags = btf_kfunc_id_set_contains(btf, hook, kfunc_btf_id);
+		if (!flags)
+			continue;
+		is_set = *flags & flag;
+		if (res < 0)
+			res = is_set;
+		else if (res != is_set)
+			return -EINVAL;
+	}
+
+	return res;
+}
+
 u32 *btf_kfunc_is_modify_return(const struct btf *btf, u32 kfunc_btf_id,
 				const struct bpf_prog *prog)
 {
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 6e19a030da6f1..47fe047ad30b8 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -126,6 +126,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag
 	fp->aux->main_prog_aux = aux;
 	fp->aux->prog = fp;
 	fp->jit_requested = ebpf_jit_enabled();
+	fp->jit_required = IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON);
 	fp->blinding_requested = bpf_jit_blinding_enabled(fp);
 #ifdef CONFIG_CGROUP_BPF
 	aux->cgroup_atype = CGROUP_BPF_ATTACH_TYPE_INVALID;
@@ -305,7 +306,7 @@ int bpf_prog_calc_tag(struct bpf_prog *fp)
 	bool was_ld_map;
 	u32 i;
 
-	dst = vmalloc(size);
+	dst = __vmalloc(size, GFP_KERNEL_ACCOUNT);
 	if (!dst)
 		return -ENOMEM;
 
@@ -2670,15 +2671,11 @@ struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct
 	/* In case of BPF to BPF calls, verifier did all the prep
 	 * work with regards to JITing, etc.
 	 */
-	bool jit_needed = false;
+	bool jit_needed = fp->jit_required;
 
 	if (fp->bpf_func)
 		goto finalize;
 
-	if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) ||
-	    bpf_prog_has_kfunc_call(fp))
-		jit_needed = true;
-
 	if (!bpf_prog_select_interpreter(fp))
 		jit_needed = true;
 
diff --git a/kernel/bpf/cpumask.c b/kernel/bpf/cpumask.c
index b8c805b4b06a0..1336a4efa7553 100644
--- a/kernel/bpf/cpumask.c
+++ b/kernel/bpf/cpumask.c
@@ -449,12 +449,12 @@ __bpf_kfunc u32 bpf_cpumask_weight(const struct cpumask *cpumask)
  * @src__sz: Length of the BPF memory region in bytes.
  *
  * Return:
- * * 0 if the struct cpumask * instance was populated successfully.
+ * * 0 if the struct bpf_cpumask * instance was populated successfully.
  * * -EACCES if the memory region is too small to populate the cpumask.
  * * -EINVAL if the memory region is not aligned to the size of a long
  *   and the architecture does not support efficient unaligned accesses.
  */
-__bpf_kfunc int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t src__sz)
+__bpf_kfunc int bpf_cpumask_populate(struct bpf_cpumask *cpumask, void *src, size_t src__sz)
 {
 	unsigned long source = (unsigned long)src;
 
@@ -467,7 +467,7 @@ __bpf_kfunc int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t
 		!IS_ALIGNED(source, sizeof(long)))
 		return -EINVAL;
 
-	bitmap_copy(cpumask_bits(cpumask), src, nr_cpu_ids);
+	bitmap_copy(cpumask_bits(&cpumask->cpumask), src, nr_cpu_ids);
 
 	return 0;
 }
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 3cf2cc6e3ab66..d3be972714b22 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -1378,7 +1378,6 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env)
 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
 	struct bpf_prog *prog = env->prog;
 	struct bpf_insn *insn = prog->insnsi;
-	bool has_kfunc_call = bpf_prog_has_kfunc_call(prog);
 	int depth;
 #endif
 	int i, err = 0;
@@ -1404,8 +1403,8 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env)
 			return err;
 	}
 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
-	if (has_kfunc_call) {
-		verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");
+	if (prog->jit_required) {
+		verbose(env, "program requires BPF JIT compiler but it is not available\n");
 		return -EINVAL;
 	}
 	for (i = 0; i < env->subprog_cnt; i++) {
@@ -1841,8 +1840,10 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
 		}
 
 		/* Skip inlining the helper call if the JIT does it. */
-		if (bpf_jit_inlines_helper_call(insn->imm))
+		if (bpf_jit_inlines_helper_call(insn->imm)) {
+			prog->jit_required = 1;
 			goto next_insn;
+		}
 
 		if (insn->imm == BPF_FUNC_get_route_realm)
 			prog->dst_needed = 1;
@@ -2338,7 +2339,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
 				     func_id_name(insn->imm), insn->imm);
 			return -EFAULT;
 		}
-		insn->imm = fn->func - __bpf_call_base;
+		insn->imm = BPF_CALL_IMM(fn->func);
 next_insn:
 		if (subprogs[cur_subprog + 1].start == i + delta + 1) {
 			subprogs[cur_subprog].stack_depth += stack_depth_extra;
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 6db306d23b479..0ff9e3aa293d5 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -40,7 +40,6 @@
 #include <linux/tracepoint.h>
 #include <linux/overflow.h>
 #include <linux/cookie.h>
-#include <linux/verification.h>
 #include <linux/btf_ids.h>
 
 #include <net/netfilter/nf_bpf_link.h>
@@ -1599,13 +1598,6 @@ static int map_create_alloc(union bpf_attr *attr, bpfptr_t uattr, struct bpf_ver
 			err = -EFAULT;
 			goto free_map;
 		}
-
-		/* See libbpf: emit_signature_match() */
-		BUILD_BUG_ON(offsetof(struct bpf_map, excl) != SHA256_DIGEST_SIZE);
-		BUILD_BUG_ON(!__same_type(map->excl, u32));
-		BUILD_BUG_ON(offsetof(struct bpf_map, sha)  != 0);
-		BUILD_BUG_ON(!__same_type(map->sha, u8[SHA256_DIGEST_SIZE]));
-		map->excl = 1;
 	} else if (attr->excl_prog_hash_size) {
 		bpf_log(log, "Invalid excl_prog_hash_size.\n");
 		err = -EINVAL;
@@ -1657,7 +1649,7 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_common_at
 
 	err = security_bpf_map_create(map, attr, token, uattr.is_kernel);
 	if (err)
-		goto free_map_sec;
+		goto free_map;
 
 	err = bpf_map_alloc_id(map);
 	if (err)
@@ -2886,64 +2878,6 @@ static bool is_perfmon_prog_type(enum bpf_prog_type prog_type)
 	}
 }
 
-static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id)
-{
-	switch (keyring_id) {
-	case 0:
-		return BPF_SIG_KEYRING_BUILTIN;
-	case (s32)(unsigned long)VERIFY_USE_SECONDARY_KEYRING:
-		return BPF_SIG_KEYRING_SECONDARY;
-	case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING:
-		return BPF_SIG_KEYRING_PLATFORM;
-	default:
-		return BPF_SIG_KEYRING_USER;
-	}
-}
-
-static int bpf_prog_verify_signature(struct bpf_prog *prog, union bpf_attr *attr,
-				     bool is_kernel, s32 *keyring_serial)
-{
-	bpfptr_t usig = make_bpfptr(attr->signature, is_kernel);
-	struct bpf_dynptr_kern sig_ptr, insns_ptr;
-	struct bpf_key *key = NULL;
-	void *sig;
-	int err = 0;
-
-	/*
-	 * Don't attempt to use kmalloc_large or vmalloc for signatures.
-	 * Practical signature for BPF program should be below this limit.
-	 */
-	if (attr->signature_size > KMALLOC_MAX_CACHE_SIZE)
-		return -EINVAL;
-
-	if (system_keyring_id_check(attr->keyring_id) == 0)
-		key = bpf_lookup_system_key(attr->keyring_id);
-	else
-		key = bpf_lookup_user_key(attr->keyring_id, 0);
-
-	if (!key)
-		return -EINVAL;
-
-	sig = kvmemdup_bpfptr(usig, attr->signature_size);
-	if (IS_ERR(sig)) {
-		bpf_key_put(key);
-		return PTR_ERR(sig);
-	}
-
-	bpf_dynptr_init(&sig_ptr, sig, BPF_DYNPTR_TYPE_LOCAL, 0,
-			attr->signature_size);
-	bpf_dynptr_init(&insns_ptr, prog->insnsi, BPF_DYNPTR_TYPE_LOCAL, 0,
-			prog->len * sizeof(struct bpf_insn));
-
-	err = bpf_verify_pkcs7_signature((struct bpf_dynptr *)&insns_ptr,
-					 (struct bpf_dynptr *)&sig_ptr, key);
-	if (!err)
-		*keyring_serial = bpf_key_serial(key);
-	bpf_key_put(key);
-	kvfree(sig);
-	return err;
-}
-
 static int bpf_prog_mark_insn_arrays_ready(struct bpf_prog *prog)
 {
 	int err;
@@ -3133,17 +3067,8 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at
 
 	/* eBPF programs must be GPL compatible to use GPL-ed functions */
 	prog->gpl_compatible = license_is_gpl_compatible(license) ? 1 : 0;
-	if (attr->signature) {
-		err = bpf_prog_verify_signature(prog, attr, uattr.is_kernel,
-						&prog->aux->sig.keyring_serial);
-		if (err)
-			goto free_prog;
-		prog->aux->sig.keyring_type = bpf_classify_keyring(attr->keyring_id);
-		prog->aux->sig.verdict = BPF_SIG_VERIFIED;
-	} else {
-		prog->aux->sig.keyring_type = BPF_SIG_KEYRING_NONE;
-		prog->aux->sig.verdict = BPF_SIG_UNSIGNED;
-	}
+	prog->aux->sig.keyring_type = BPF_SIG_KEYRING_NONE;
+	prog->aux->sig.verdict = BPF_SIG_UNSIGNED;
 	prog->orig_prog = NULL;
 	prog->jited = 0;
 
@@ -3189,10 +3114,6 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at
 	if (err < 0)
 		goto free_prog;
 
-	err = security_bpf_prog_load(prog, attr, token, uattr.is_kernel);
-	if (err)
-		goto free_prog;
-
 	/* run eBPF verifier */
 	err = bpf_check(&prog, attr, uattr, attr_log);
 	if (err < 0)
diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c
index e791ae065c39b..b256fb9c1214e 100644
--- a/kernel/bpf/task_iter.c
+++ b/kernel/bpf/task_iter.c
@@ -756,6 +756,7 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start,
 	struct mmap_unlock_irq_work *work = NULL;
 	struct vm_area_struct *vma;
 	bool irq_work_busy = false;
+	bool __maybe_unused mmput_needed = false;
 	struct mm_struct *mm;
 	int ret = -ENOENT;
 
@@ -765,14 +766,38 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start,
 	if (!task)
 		return -ENOENT;
 
-	mm = task->mm;
+	if (task == current) {
+		mm = task->mm;
+	} else {
+		/*
+		 * Foreign task: pin task->mm against a concurrent exit_mm().
+		 * Use trylock on alloc_lock instead of get_task_mm()'s
+		 * blocking task_lock() to avoid deadlocking the target task.
+		 */
+		if (!IS_ENABLED(CONFIG_MMU))
+			return -EOPNOTSUPP;
+		if (irqs_disabled())
+			return -EBUSY;
+		if (!spin_trylock(&task->alloc_lock))
+			return -EBUSY;
+		mm = task->mm;
+		if (mm && !(task->flags & PF_KTHREAD)) {
+			mmget(mm);
+			mmput_needed = true;
+		} else {
+			mm = NULL;
+		}
+		spin_unlock(&task->alloc_lock);
+	}
 	if (!mm)
 		return -ENOENT;
 
 	irq_work_busy = bpf_mmap_unlock_get_irq_work(&work);
 
-	if (irq_work_busy || !mmap_read_trylock(mm))
-		return -EBUSY;
+	if (irq_work_busy || !mmap_read_trylock(mm)) {
+		ret = -EBUSY;
+		goto out;
+	}
 
 	vma = find_vma(mm, start);
 
@@ -782,6 +807,11 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start,
 		ret = 0;
 	}
 	bpf_mmap_unlock_mm(work, mm);
+out:
+#ifdef CONFIG_MMU
+	if (mmput_needed)
+		mmput_async(mm);
+#endif
 	return ret;
 }
 
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 1a721fc4bef56..6eadf64f7ec90 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -1536,6 +1536,7 @@ static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_ima
 	if (bpf_trampoline_use_jmp(tr->flags))
 		addr = ftrace_jmp_set(addr);
 
+	tr->func.ftrace_managed = true;
 	ftrace_hash_add(data->reg, data->entry, ip, addr);
 	tr->cur_image = im;
 	return 0;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 6515d4d3c0032..6fb298ed9c14a 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -22,6 +22,8 @@
 #include <linux/ctype.h>
 #include <linux/error-injection.h>
 #include <linux/bpf_lsm.h>
+#include <linux/security.h>
+#include <linux/verification.h>
 #include <linux/btf_ids.h>
 #include <linux/poison.h>
 #include <linux/module.h>
@@ -322,6 +324,7 @@ static const char *btf_type_name(const struct btf *btf, u32 id)
 }
 
 static DEFINE_MUTEX(bpf_verifier_lock);
+static DEFINE_MUTEX(btf_vmlinux_lock);
 static DEFINE_MUTEX(bpf_percpu_ma_lock);
 
 __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...)
@@ -2490,6 +2493,83 @@ int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
 	return 0;
 }
 
+#define BPF_FD_SLOT_BTF	1UL
+
+static void fd_slot_set_map(struct bpf_fd_array *slot, struct bpf_map *map)
+{
+	slot->val = (unsigned long)map;
+}
+
+static void fd_slot_set_btf(struct bpf_fd_array *slot, struct btf *btf)
+{
+	slot->val = (unsigned long)btf | BPF_FD_SLOT_BTF;
+}
+
+static struct bpf_map *fd_slot_map(struct bpf_fd_array slot)
+{
+	if (slot.val & BPF_FD_SLOT_BTF)
+		return NULL;
+	return (struct bpf_map *)slot.val;
+}
+
+static struct btf *fd_slot_btf(struct bpf_fd_array slot)
+{
+	if (!(slot.val & BPF_FD_SLOT_BTF))
+		return NULL;
+	return (struct btf *)(slot.val & ~BPF_FD_SLOT_BTF);
+}
+
+static struct btf *
+fd_array_get_btf_continuous(struct bpf_verifier_env *env, u32 idx)
+{
+	struct btf *btf;
+
+	if (idx >= env->fd_array_cnt) {
+		verbose(env, "kfunc fd_idx %u out of bounds, fd_array_cnt %u\n",
+			idx, env->fd_array_cnt);
+		return ERR_PTR(-EINVAL);
+	}
+	btf = fd_slot_btf(env->fd_array[idx]);
+	if (!btf) {
+		verbose(env, "kfunc fd_idx %u is not a module BTF\n", idx);
+		return ERR_PTR(-EINVAL);
+	}
+	btf_get(btf);
+	return btf;
+}
+
+static struct btf *
+fd_array_get_btf_sparse(struct bpf_verifier_env *env, u32 idx)
+{
+	struct btf *btf;
+	int btf_fd;
+
+	if (copy_from_bpfptr_offset(&btf_fd, env->fd_array_raw,
+				    (size_t)idx * sizeof(btf_fd), sizeof(btf_fd)))
+		return ERR_PTR(-EFAULT);
+	btf = btf_get_by_fd(btf_fd);
+	if (IS_ERR(btf)) {
+		verbose(env, "invalid module BTF fd specified\n");
+		return btf;
+	}
+	return btf;
+}
+
+static struct btf *fd_array_get_btf(struct bpf_verifier_env *env, u32 idx)
+{
+	if (env->signature) {
+		verbose(env, "signed program cannot bind any BTF\n");
+		return ERR_PTR(-EACCES);
+	}
+	if (env->fd_array)
+		return fd_array_get_btf_continuous(env, idx);
+	if (!bpfptr_is_null(env->fd_array_raw))
+		return fd_array_get_btf_sparse(env, idx);
+
+	verbose(env, "kfunc offset > 0 without fd_array is invalid\n");
+	return ERR_PTR(-EPROTO);
+}
+
 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,
 					 s16 offset)
 {
@@ -2498,7 +2578,6 @@ static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,
 	struct bpf_kfunc_btf *b;
 	struct module *mod;
 	struct btf *btf;
-	int btf_fd;
 
 	tab = env->prog->aux->kfunc_btf_tab;
 	b = bsearch(&kf_btf, tab->descs, tab->nr_descs,
@@ -2509,22 +2588,9 @@ static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,
 			return ERR_PTR(-E2BIG);
 		}
 
-		if (bpfptr_is_null(env->fd_array)) {
-			verbose(env, "kfunc offset > 0 without fd_array is invalid\n");
-			return ERR_PTR(-EPROTO);
-		}
-
-		if (copy_from_bpfptr_offset(&btf_fd, env->fd_array,
-					    offset * sizeof(btf_fd),
-					    sizeof(btf_fd)))
-			return ERR_PTR(-EFAULT);
-
-		btf = btf_get_by_fd(btf_fd);
-		if (IS_ERR(btf)) {
-			verbose(env, "invalid module BTF fd specified\n");
+		btf = fd_array_get_btf(env, offset);
+		if (IS_ERR(btf))
 			return btf;
-		}
-
 		if (!btf_is_module(btf)) {
 			verbose(env, "BTF fd for kfunc is not a module BTF\n");
 			btf_put(btf);
@@ -2584,24 +2650,25 @@ static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset)
 
 #define KF_IMPL_SUFFIX "_impl"
 
-static const struct btf_type *find_kfunc_impl_proto(struct bpf_verifier_env *env,
+static const struct btf_type *find_kfunc_impl_proto(struct bpf_verifier_log *log,
 						    struct btf *btf,
 						    const char *func_name)
 {
-	char *buf = env->tmp_str_buf;
 	const struct btf_type *func;
+	char buf[KSYM_NAME_LEN];
 	s32 impl_id;
 	int len;
 
-	len = snprintf(buf, TMP_STR_BUF_LEN, "%s%s", func_name, KF_IMPL_SUFFIX);
-	if (len < 0 || len >= TMP_STR_BUF_LEN) {
-		verbose(env, "function name %s%s is too long\n", func_name, KF_IMPL_SUFFIX);
+	len = snprintf(buf, sizeof(buf), "%s%s", func_name, KF_IMPL_SUFFIX);
+	if (len < 0 || len >= sizeof(buf)) {
+		bpf_log(log, "function name %s%s is too long\n",
+			func_name, KF_IMPL_SUFFIX);
 		return NULL;
 	}
 
 	impl_id = btf_find_by_name_kind(btf, buf, BTF_KIND_FUNC);
 	if (impl_id <= 0) {
-		verbose(env, "cannot find function %s in BTF\n", buf);
+		bpf_log(log, "cannot find function %s in BTF\n", buf);
 		return NULL;
 	}
 
@@ -2653,7 +2720,7 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env,
 	 * can be found through the counterpart _impl kfunc.
 	 */
 	if (kfunc_flags && (*kfunc_flags & KF_IMPLICIT_ARGS))
-		func_proto = find_kfunc_impl_proto(env, btf, func_name);
+		func_proto = find_kfunc_impl_proto(&env->log, btf, func_name);
 	else
 		func_proto = btf_type_by_id(btf, func->type);
 
@@ -2714,6 +2781,8 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset)
 		prog_aux->kfunc_tab = tab;
 	}
 
+	env->prog->jit_required = 1;
+
 	/* func_id == 0 is always invalid, but instead of returning an error, be
 	 * conservative and wait until the code elimination pass before returning
 	 * error, so that invalid calls that get pruned out can be in BPF programs
@@ -2768,11 +2837,6 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset)
 	return 0;
 }
 
-bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
-{
-	return !!prog->aux->kfunc_tab;
-}
-
 static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
 {
 	struct bpf_subprog_info *subprog = env->subprog_info;
@@ -3304,34 +3368,6 @@ static int mark_chain_precision_batch(struct bpf_verifier_env *env,
 	return bpf_mark_chain_precision(env, starting_state, -1, NULL);
 }
 
-static bool is_spillable_regtype(enum bpf_reg_type type)
-{
-	switch (base_type(type)) {
-	case PTR_TO_MAP_VALUE:
-	case PTR_TO_STACK:
-	case PTR_TO_CTX:
-	case PTR_TO_PACKET:
-	case PTR_TO_PACKET_META:
-	case PTR_TO_PACKET_END:
-	case PTR_TO_FLOW_KEYS:
-	case CONST_PTR_TO_MAP:
-	case PTR_TO_SOCKET:
-	case PTR_TO_SOCK_COMMON:
-	case PTR_TO_TCP_SOCK:
-	case PTR_TO_XDP_SOCK:
-	case PTR_TO_BTF_ID:
-	case PTR_TO_BUF:
-	case PTR_TO_MEM:
-	case PTR_TO_FUNC:
-	case PTR_TO_MAP_KEY:
-	case PTR_TO_ARENA:
-		return true;
-	default:
-		return false;
-	}
-}
-
-
 /* check if register is a constant scalar value */
 static bool is_reg_const(struct bpf_reg_state *reg, bool subreg32)
 {
@@ -3345,13 +3381,18 @@ static u64 reg_const_value(struct bpf_reg_state *reg, bool subreg32)
 	return subreg32 ? tnum_subreg(reg->var_off).value : reg->var_off.value;
 }
 
+static bool is_pointer_regtype(enum bpf_reg_type type)
+{
+	return type != SCALAR_VALUE && type != NOT_INIT;
+}
+
 static bool __is_pointer_value(bool allow_ptr_leaks,
 			       const struct bpf_reg_state *reg)
 {
 	if (allow_ptr_leaks)
 		return false;
 
-	return reg->type != SCALAR_VALUE;
+	return is_pointer_regtype(reg->type);
 }
 
 static void clear_scalar_id(struct bpf_reg_state *reg)
@@ -3476,7 +3517,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
 	if (value_regno >= 0)
 		reg = &cur->regs[value_regno];
 	if (!env->bypass_spec_v4) {
-		bool sanitize = reg && is_spillable_regtype(reg->type);
+		bool sanitize = reg && is_pointer_regtype(reg->type);
 
 		for (i = 0; i < size; i++) {
 			u8 type = state->stack[spi].slot_type[(slot - i) %
@@ -3517,7 +3558,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
 		__mark_reg_known(tmp_reg, insn->imm);
 		tmp_reg->type = SCALAR_VALUE;
 		save_register_state(env, state, spi, tmp_reg, size);
-	} else if (reg && is_spillable_regtype(reg->type)) {
+	} else if (reg && is_pointer_regtype(reg->type)) {
 		/* register containing pointer is being spilled into stack */
 		if (size != BPF_REG_SIZE) {
 			verbose_linfo(env, insn_idx, "; ");
@@ -3702,14 +3743,21 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env,
  * SCALAR. This function does not deal with register filling; the caller must
  * ensure that all spilled registers in the stack range have been marked as
  * read.
+ *
+ * STACK_SPILL bytes backed by spilled scalar const zeroes are also considered
+ * zero bytes. In that case, mark the contributing stack slots precise so
+ * pruning cannot reuse a zero-spill state for a later non-zero spill state.
+ *
+ * Returns an error if precision backtracking fails.
  */
-static void mark_reg_stack_read(struct bpf_verifier_env *env,
-				/* func where src register points to */
-				struct bpf_func_state *ptr_state,
-				int min_off, int max_off, int dst_regno)
+static int mark_reg_stack_read(struct bpf_verifier_env *env,
+			       /* func where src register points to */
+			       struct bpf_func_state *ptr_state,
+			       int min_off, int max_off, int dst_regno)
 {
 	struct bpf_verifier_state *vstate = env->cur_state;
 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
+	u64 zero_spill_mask = 0;
 	int i, slot, spi;
 	u8 *stype;
 	int zeros = 0;
@@ -3719,19 +3767,33 @@ static void mark_reg_stack_read(struct bpf_verifier_env *env,
 		spi = slot / BPF_REG_SIZE;
 		mark_stack_slot_scratched(env, spi);
 		stype = ptr_state->stack[spi].slot_type;
-		if (stype[slot % BPF_REG_SIZE] != STACK_ZERO)
-			break;
-		zeros++;
+		if (stype[slot % BPF_REG_SIZE] == STACK_ZERO) {
+			zeros++;
+			continue;
+		}
+		if (stype[slot % BPF_REG_SIZE] == STACK_SPILL &&
+		    bpf_register_is_null(&ptr_state->stack[spi].spilled_ptr)) {
+			zero_spill_mask |= 1ull << spi;
+			zeros++;
+			continue;
+		}
+		break;
 	}
 	if (zeros == max_off - min_off) {
 		/* Any access_size read into register is zero extended,
 		 * so the whole register == const_zero.
 		 */
 		__mark_reg_const_zero(env, &state->regs[dst_regno]);
+		if (zero_spill_mask) {
+			bpf_bt_set_frame_slot_mask(&env->bt, ptr_state->frameno, zero_spill_mask);
+			return mark_chain_precision_batch(env, env->cur_state);
+		}
 	} else {
 		/* have read misc data from the stack */
 		mark_reg_unknown(env, state->regs, dst_regno);
 	}
+
+	return 0;
 }
 
 /* Read the stack at 'off' and put the results into the register indicated by
@@ -3753,6 +3815,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
 	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE;
 	struct bpf_reg_state *reg;
 	u8 *stype, type;
+	int err;
 	int insn_flags = INSN_F_STACK_ACCESS;
 	int hist_spi = spi, hist_frame = reg_state->frameno;
 
@@ -3835,7 +3898,10 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
 					__mark_reg_const_zero(env, &state->regs[dst_regno]);
 					insn_flags = 0; /* not restoring original register state */
 				} else {
-					mark_reg_unknown(env, state->regs, dst_regno);
+					err = mark_reg_stack_read(env, reg_state, off, off + size,
+								  dst_regno);
+					if (err)
+						return err;
 					insn_flags = 0; /* not restoring original register state */
 				}
 			}
@@ -3880,8 +3946,11 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
 			}
 			return -EACCES;
 		}
-		if (dst_regno >= 0)
-			mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);
+		if (dst_regno >= 0) {
+			err = mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);
+			if (err)
+				return err;
+		}
 		insn_flags = 0; /* we are not restoring spilled register */
 	}
 	if (insn_flags)
@@ -3935,7 +4004,10 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg
 
 	min_off = reg_smin(reg) + off;
 	max_off = reg_smax(reg) + off;
-	mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
+	err = mark_reg_stack_read(env, ptr_state, min_off, max_off + size,
+				  dst_regno);
+	if (err)
+		return err;
 	check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off);
 	return 0;
 }
@@ -4348,7 +4420,8 @@ static int map_kptr_match_type(struct bpf_verifier_env *env,
 	 */
 	if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value,
 				  kptr_field->kptr.btf, kptr_field->kptr.btf_id,
-				  kptr_field->type != BPF_KPTR_UNREF))
+				  kptr_field->type != BPF_KPTR_UNREF,
+				  !type_is_alloc(reg->type)))
 		goto bad_type;
 	return 0;
 bad_type:
@@ -5326,14 +5399,11 @@ static int check_max_stack_depth(struct bpf_verifier_env *env)
 static int __check_buffer_access(struct bpf_verifier_env *env,
 				 const char *buf_info,
 				 const struct bpf_reg_state *reg,
-				 argno_t argno, int off, int size)
+				 argno_t argno, int off, int size,
+				 u32 *access_end)
 {
-	if (off < 0) {
-		verbose(env,
-			"%s invalid %s buffer access: off=%d, size=%d\n",
-			reg_arg_name(env, argno), buf_info, off, size);
-		return -EACCES;
-	}
+	s64 start;
+
 	if (!tnum_is_const(reg->var_off)) {
 		char tn_buf[48];
 
@@ -5344,6 +5414,15 @@ static int __check_buffer_access(struct bpf_verifier_env *env,
 		return -EACCES;
 	}
 
+	start = (s64)reg->var_off.value + off;
+	if (start < 0) {
+		verbose(env,
+			"%s invalid negative %s buffer offset: off=%d, var_off=%lld\n",
+			reg_arg_name(env, argno), buf_info, off, (s64)reg->var_off.value);
+		return -EACCES;
+	}
+
+	*access_end = start + size;
 	return 0;
 }
 
@@ -5351,14 +5430,14 @@ static int check_tp_buffer_access(struct bpf_verifier_env *env,
 				  const struct bpf_reg_state *reg,
 				  argno_t argno, int off, int size)
 {
+	u32 access_end;
 	int err;
 
-	err = __check_buffer_access(env, "tracepoint", reg, argno, off, size);
+	err = __check_buffer_access(env, "tracepoint", reg, argno, off, size, &access_end);
 	if (err)
 		return err;
 
-	env->prog->aux->max_tp_access = max(reg->var_off.value + off + size,
-					    env->prog->aux->max_tp_access);
+	env->prog->aux->max_tp_access = max(access_end, env->prog->aux->max_tp_access);
 
 	return 0;
 }
@@ -5370,13 +5449,14 @@ static int check_buffer_access(struct bpf_verifier_env *env,
 			       u32 *max_access)
 {
 	const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr";
+	u32 access_end;
 	int err;
 
-	err = __check_buffer_access(env, buf_info, reg, argno, off, size);
+	err = __check_buffer_access(env, buf_info, reg, argno, off, size, &access_end);
 	if (err)
 		return err;
 
-	*max_access = max(reg->var_off.value + off + size, *max_access);
+	*max_access = max(access_end, *max_access);
 
 	return 0;
 }
@@ -5781,6 +5861,11 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
 		return -EACCES;
 	}
 
+	if (atype != BPF_READ && (type_flag(reg->type) & PTR_UNTRUSTED)) {
+		verbose(env, "only read is supported\n");
+		return -EACCES;
+	}
+
 	if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) {
 		if (!btf_is_kernel(reg->btf)) {
 			verifier_bug(env, "reg->btf must be kernel btf");
@@ -5793,8 +5878,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
 				reg_arg_name(env, argno), tname, off, size);
 	} else {
 		/* Writes are permitted with default btf_struct_access for
-		 * program allocated objects (which always have id > 0),
-		 * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC.
+		 * program allocated objects (which always have id > 0).
 		 */
 		if (atype != BPF_READ && !type_is_ptr_alloc_obj(reg->type)) {
 			verbose(env, "only read is supported\n");
@@ -6318,11 +6402,23 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b
 
 	if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ &&
 	    regs[value_regno].type == SCALAR_VALUE) {
-		if (!is_ldsx)
+		if (!is_ldsx) {
 			/* b/h/w load zero-extends, mark upper bits as known 0 */
 			coerce_reg_to_size(&regs[value_regno], size);
-		else
+		} else {
+			/*
+			 * Sign-extension can change the register value relative
+			 * to a scalar it is linked with by id (e.g. a zero-
+			 * extending fill of the same spilled stack slot), thus
+			 * drop the shared id in that case.
+			 */
+			bool no_sext = reg_umax(&regs[value_regno]) <
+					(1ULL << (size * BITS_PER_BYTE - 1));
+
 			coerce_reg_to_size_sx(&regs[value_regno], size);
+			if (!no_sext)
+				clear_scalar_id(&regs[value_regno]);
+		}
 	}
 	return err;
 }
@@ -7939,7 +8035,7 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re
 
 			if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id,
 						  reg->var_off.value, btf_vmlinux, *arg_btf_id,
-						  strict_type_match)) {
+						  strict_type_match, !type_is_alloc(reg->type))) {
 				verbose(env, "%s is of type %s but %s is expected\n",
 					reg_arg_name(env, argno),
 					btf_type_name(reg->btf, reg->btf_id),
@@ -8464,12 +8560,7 @@ static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
 		if (func_id == BPF_FUNC_map_delete_elem)
 			return true;
 		break;
-	case BPF_PROG_TYPE_SOCKET_FILTER:
-	case BPF_PROG_TYPE_SCHED_CLS:
-	case BPF_PROG_TYPE_SCHED_ACT:
-	case BPF_PROG_TYPE_XDP:
 	case BPF_PROG_TYPE_SK_REUSEPORT:
-	case BPF_PROG_TYPE_FLOW_DISSECTOR:
 	case BPF_PROG_TYPE_SK_LOOKUP:
 		return true;
 	default:
@@ -11405,7 +11496,8 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env,
 	reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, &reg_ref_id);
 	reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off);
 	struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->var_off.value,
-					   meta->btf, ref_id, strict_type_match);
+					   meta->btf, ref_id, strict_type_match,
+					   !type_is_alloc(reg->type));
 	/* If kfunc is accepting a projection type (ie. __sk_buff), it cannot
 	 * actually use it -- it must cast to the underlying type. So we allow
 	 * caller to pass in the underlying type.
@@ -11852,7 +11944,8 @@ __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env,
 	et = btf_type_by_id(field->graph_root.btf, field->graph_root.value_btf_id);
 	t = btf_type_by_id(reg->btf, reg->btf_id);
 	if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf,
-				  field->graph_root.value_btf_id, true)) {
+				  field->graph_root.value_btf_id, true,
+				  !type_is_alloc(reg->type))) {
 		verbose(env, "operation on %s expects arg#1 %s at offset=%d "
 			"in struct %s, but arg is at offset=%d in struct %s\n",
 			btf_field_type_name(head_field_type),
@@ -12024,6 +12117,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 				}
 
 				meta->r0_size = reg->var_off.value;
+				if (meta->r0_size > U32_MAX) {
+					verbose(env, "%s rdonly/rdwr_buf_size exceeds u32 max\n",
+						reg_arg_name(env, argno));
+					return -EINVAL;
+				}
 				if (regno >= 0)
 					ret = mark_chain_precision(env, regno);
 				else
@@ -17558,6 +17656,11 @@ static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf)
 		if (env->used_btfs[i].btf == btf)
 			goto ret_put;
 
+	if (env->signature) {
+		verbose(env, "signed program cannot bind any BTF\n");
+		ret = -EACCES;
+		goto ret_put;
+	}
 	if (env->used_btf_cnt >= MAX_USED_BTFS) {
 		verbose(env, "The total number of btfs per program has reached the limit of %u\n",
 			MAX_USED_BTFS);
@@ -17840,6 +17943,12 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map)
 		if (env->used_maps[i] == map)
 			return i;
 
+	if (env->signature &&
+	    env->prog->aux->sig.verdict == BPF_SIG_VERIFIED) {
+		verbose(env, "signed program cannot bind map '%s' not covered by the signature\n",
+			map->name);
+		return -EACCES;
+	}
 	if (env->used_map_cnt >= MAX_USED_MAPS) {
 		verbose(env, "The total number of maps per program has reached the limit of %u\n",
 			MAX_USED_MAPS);
@@ -17892,6 +18001,48 @@ static int add_used_map(struct bpf_verifier_env *env, int fd)
 	return __add_used_map(env, map);
 }
 
+static int fd_array_get_map_idx_continuous(struct bpf_verifier_env *env, u32 idx)
+{
+	struct bpf_map *map;
+
+	if (idx >= env->fd_array_cnt) {
+		verbose(env, "fd_idx %u out of bounds, fd_array_cnt %u\n",
+			idx, env->fd_array_cnt);
+		return -EINVAL;
+	}
+	map = fd_slot_map(env->fd_array[idx]);
+	if (!map) {
+		verbose(env, "fd_idx %u is not a map\n", idx);
+		return -EINVAL;
+	}
+	return __add_used_map(env, map);
+}
+
+static int fd_array_get_map_idx_sparse(struct bpf_verifier_env *env, u32 idx)
+{
+	int fd;
+
+	if (copy_from_bpfptr_offset(&fd, env->fd_array_raw,
+				    (size_t)idx * sizeof(fd), sizeof(fd)))
+		return -EFAULT;
+	return add_used_map(env, fd);
+}
+
+static int fd_array_get_map_idx(struct bpf_verifier_env *env, u32 idx)
+{
+	if (env->fd_array)
+		return fd_array_get_map_idx_continuous(env, idx);
+	if (env->signature) {
+		verbose(env, "signed program must bind maps via a continuous fd_array (fd_array_cnt)\n");
+		return -EACCES;
+	}
+	if (!bpfptr_is_null(env->fd_array_raw))
+		return fd_array_get_map_idx_sparse(env, idx);
+
+	verbose(env, "fd_idx without fd_array is invalid\n");
+	return -EPROTO;
+}
+
 static int check_alu_fields(struct bpf_verifier_env *env, struct bpf_insn *insn)
 {
 	u8 class = BPF_CLASS(insn->code);
@@ -18109,7 +18260,6 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env)
 			struct bpf_map *map;
 			int map_idx;
 			u64 addr;
-			u32 fd;
 
 			if (i == insn_cnt - 1 || insn[1].code != 0 ||
 			    insn[1].dst_reg != 0 || insn[1].src_reg != 0 ||
@@ -18161,21 +18311,17 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env)
 			switch (insn[0].src_reg) {
 			case BPF_PSEUDO_MAP_IDX_VALUE:
 			case BPF_PSEUDO_MAP_IDX:
-				if (bpfptr_is_null(env->fd_array)) {
-					verbose(env, "fd_idx without fd_array is invalid\n");
-					return -EPROTO;
-				}
-				if (copy_from_bpfptr_offset(&fd, env->fd_array,
-							    insn[0].imm * sizeof(fd),
-							    sizeof(fd)))
-					return -EFAULT;
+				map_idx = fd_array_get_map_idx(env, insn[0].imm);
 				break;
 			default:
-				fd = insn[0].imm;
+				if (env->signature) {
+					verbose(env, "signed program cannot reference a map by fd, only via fd_array index\n");
+					return -EINVAL;
+				}
+				map_idx = add_used_map(env, insn[0].imm);
 				break;
 			}
 
-			map_idx = add_used_map(env, fd);
 			if (map_idx < 0)
 				return map_idx;
 			map = env->used_maps[map_idx];
@@ -18873,6 +19019,47 @@ static int btf_id_allow_sleepable(u32 btf_id, unsigned long addr, const struct b
 	return -EINVAL;
 }
 
+/*
+ * Resolve the prototype describing a trace target's real ABI. A
+ * KF_IMPLICIT_ARGS kfunc has its injected args stripped from the public
+ * prototype, so use the _impl prototype; other targets use their own.
+ */
+static const struct btf_type *
+btf_attach_func_proto(struct bpf_verifier_log *log, struct btf *btf, u32 func_id)
+{
+	const struct btf_type *func;
+	struct module *mod = NULL;
+	const char *name;
+	int implicit;
+
+	func = btf_type_by_id(btf, func_id);
+	if (!func || !btf_type_is_func(func))
+		return NULL;
+	name = btf_name_by_offset(btf, func->name_off);
+
+	/*
+	 * btf_kfunc_check_flag() reads kfunc_set_tab, which for a module is
+	 * stable only once it is live; hold a module ref across the read to
+	 * exclude a concurrent module load.
+	 */
+	if (btf_is_module(btf)) {
+		mod = btf_try_get_module(btf);
+		if (!mod)
+			return NULL;
+	}
+	implicit = btf_kfunc_check_flag(btf, func_id, KF_IMPLICIT_ARGS);
+	module_put(mod);
+
+	if (implicit == -EINVAL) {
+		bpf_log(log, "kfunc %s has inconsistent KF_IMPLICIT_ARGS\n", name);
+		return NULL;
+	}
+	if (implicit > 0)
+		return find_kfunc_impl_proto(log, btf, name);
+
+	return btf_type_by_id(btf, func->type);
+}
+
 int bpf_check_attach_target(struct bpf_verifier_log *log,
 			    const struct bpf_prog *prog,
 			    const struct bpf_prog *tgt_prog,
@@ -19121,8 +19308,8 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
 		if (prog_extension &&
 		    btf_check_type_match(log, prog, btf, t))
 			return -EINVAL;
-		t = btf_type_by_id(btf, t->type);
-		if (!btf_type_is_func_proto(t))
+		t = btf_attach_func_proto(log, btf, btf_id);
+		if (!t || !btf_type_is_func_proto(t))
 			return -EINVAL;
 
 		if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) &&
@@ -19405,10 +19592,8 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt
 	tname = btf_name_by_offset(btf, t->name_off);
 	if (!tname)
 		return -EINVAL;
-	if (!btf_type_is_func(t))
-		return -EINVAL;
-	t = btf_type_by_id(btf, t->type);
-	if (!btf_type_is_func_proto(t))
+	t = btf_attach_func_proto(NULL, btf, btf_id);
+	if (!t || !btf_type_is_func_proto(t))
 		return -EINVAL;
 	err = btf_distill_func_proto(NULL, btf, t, tname, &tgt_info->fmodel);
 	if (err < 0)
@@ -19436,13 +19621,25 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt
 
 struct btf *bpf_get_btf_vmlinux(void)
 {
-	if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {
-		mutex_lock(&bpf_verifier_lock);
-		if (!btf_vmlinux)
-			btf_vmlinux = btf_parse_vmlinux();
-		mutex_unlock(&bpf_verifier_lock);
+	/* Pairs with the smp_store_release() on the parse path below. */
+	struct btf *btf = smp_load_acquire(&btf_vmlinux);
+
+	if (!btf && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {
+		mutex_lock(&btf_vmlinux_lock);
+		btf = btf_vmlinux;
+		if (!btf) {
+			btf = btf_parse_vmlinux();
+			/*
+			 * Order the parsed BTF contents and the globals the
+			 * parse populated (e.g. bpf_ctx_convert.t) before
+			 * the pointer publication. Pairs with the acquire
+			 * on the lockless fast path above.
+			 */
+			smp_store_release(&btf_vmlinux, btf);
+		}
+		mutex_unlock(&btf_vmlinux_lock);
 	}
-	return btf_vmlinux;
+	return btf;
 }
 
 /*
@@ -19450,7 +19647,7 @@ struct btf *bpf_get_btf_vmlinux(void)
  * this case expect that every file descriptor in the array is either a map or
  * a BTF. Everything else is considered to be trash.
  */
-static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd)
+static int add_fd_from_fd_array(struct bpf_verifier_env *env, u32 idx, int fd)
 {
 	struct bpf_map *map;
 	struct btf *btf;
@@ -19462,51 +19659,83 @@ static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd)
 		err = __add_used_map(env, map);
 		if (err < 0)
 			return err;
+		fd_slot_set_map(&env->fd_array[idx], map);
 		return 0;
 	}
 
 	btf = __btf_get_by_fd(f);
 	if (!IS_ERR(btf)) {
 		btf_get(btf);
-		return __add_used_btf(env, btf);
+		err = __add_used_btf(env, btf);
+		if (err < 0)
+			return err;
+		fd_slot_set_btf(&env->fd_array[idx], btf);
+		return 0;
 	}
 
 	verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd);
 	return PTR_ERR(map);
 }
 
-static int process_fd_array(struct bpf_verifier_env *env, union bpf_attr *attr, bpfptr_t uattr)
+/*
+ * A continuous fd_array is resolved into an in-memory cache with one slot
+ * per entry. The bound here is deliberately generous and not derived from
+ * the per-program object limits: Duplicate entries /are/ permitted, and
+ * the number of distinct maps and BTFs a program can bind is enforced when
+ * each entry is resolved by __add_used_map() and __add_used_btf().
+ */
+#define MAX_FD_ARRAY_CNT 4096
+
+static int process_fd_array_continuous(struct bpf_verifier_env *env,
+				       bpfptr_t fd_array, u32 cnt)
 {
-	size_t size = sizeof(int);
-	int ret;
-	int fd;
+	int fd, ret;
 	u32 i;
 
-	env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel);
-
-	/*
-	 * The only difference between old (no fd_array_cnt is given) and new
-	 * APIs is that in the latter case the fd_array is expected to be
-	 * continuous and is scanned for map fds right away
-	 */
-	if (!attr->fd_array_cnt)
-		return 0;
-
-	/* Check for integer overflow */
-	if (attr->fd_array_cnt >= (U32_MAX / size)) {
-		verbose(env, "fd_array_cnt is too big (%u)\n", attr->fd_array_cnt);
-		return -EINVAL;
+	if (cnt > MAX_FD_ARRAY_CNT) {
+		verbose(env, "fd_array has too many entries (%u, max %u)\n",
+			cnt, MAX_FD_ARRAY_CNT);
+		return -E2BIG;
 	}
 
-	for (i = 0; i < attr->fd_array_cnt; i++) {
-		if (copy_from_bpfptr_offset(&fd, env->fd_array, i * size, size))
+	env->fd_array = kvcalloc(cnt, sizeof(*env->fd_array),
+				 GFP_KERNEL_ACCOUNT);
+	if (!env->fd_array)
+		return -ENOMEM;
+	env->fd_array_cnt = cnt;
+	for (i = 0; i < cnt; i++) {
+		if (copy_from_bpfptr_offset(&fd, fd_array,
+					    (size_t)i * sizeof(fd), sizeof(fd)))
 			return -EFAULT;
-
-		ret = add_fd_from_fd_array(env, fd);
+		ret = add_fd_from_fd_array(env, i, fd);
 		if (ret)
 			return ret;
 	}
+	return 0;
+}
 
+static int process_fd_array(struct bpf_verifier_env *env,
+			    union bpf_attr *attr, bpfptr_t uattr)
+{
+	bpfptr_t fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel);
+
+	if (bpfptr_is_null(fd_array)) {
+		if (attr->fd_array_cnt) {
+			verbose(env, "fd_array_cnt %u without fd_array is invalid\n",
+				attr->fd_array_cnt);
+			return -EINVAL;
+		}
+		return 0;
+	}
+	/*
+	 * New API: the caller passes fd_array_cnt and a continuous array that
+	 * is resolved and bound up front. Legacy API (no fd_array_cnt): keep
+	 * the caller's array and resolve entries on the spot at each reference.
+	 */
+	if (attr->fd_array_cnt)
+		return process_fd_array_continuous(env, fd_array,
+						   attr->fd_array_cnt);
+	env->fd_array_raw = fd_array;
 	return 0;
 }
 
@@ -19721,6 +19950,146 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	return 0;
 }
 
+static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id)
+{
+	switch (keyring_id) {
+	case 0:
+		return BPF_SIG_KEYRING_BUILTIN;
+	case (s32)(unsigned long)VERIFY_USE_SECONDARY_KEYRING:
+		return BPF_SIG_KEYRING_SECONDARY;
+	case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING:
+		return BPF_SIG_KEYRING_PLATFORM;
+	default:
+		return BPF_SIG_KEYRING_USER;
+	}
+}
+
+/*
+ * Verify the PKCS#7 signature of a loaded program. Called from bpf_check()
+ * once the program's metadata maps have been resolved into used_maps, so
+ * the exact maps folded into the signature are the ones the program binds.
+ *
+ * The signature covers the instructions followed by the frozen contents of
+ * each map, in @maps order: insns || map_0 || map_1 || [...]. On success the
+ * verdict and keyring info are recorded on prog->aux.
+ */
+static int bpf_prog_verify_signature(struct bpf_verifier_env *env,
+				     union bpf_attr *attr, bool is_kernel)
+{
+	bpfptr_t usig = make_bpfptr(attr->signature, is_kernel);
+	struct bpf_dynptr_kern sig_ptr, data_ptr;
+	struct bpf_prog *prog = env->prog;
+	struct bpf_map **maps = env->used_maps;
+	struct bpf_key *key = NULL;
+	void *sig, *data = NULL;
+	u32 map_cnt = env->used_map_cnt;
+	u32 i, off, insns_sz;
+	u64 data_sz;
+	int err = 0;
+
+	/*
+	 * Don't attempt to use kmalloc_large or vmalloc for signatures.
+	 * Practical signature for BPF program should be below this limit.
+	 */
+	if (!attr->signature_size ||
+	    attr->signature_size > KMALLOC_MAX_CACHE_SIZE)
+		return -EINVAL;
+	if (system_keyring_id_check(attr->keyring_id) == 0)
+		key = bpf_lookup_system_key(attr->keyring_id);
+	else
+		key = bpf_lookup_user_key(attr->keyring_id, 0);
+	if (!key) {
+		verbose(env, "cannot resolve signing keyring with keyring_id %d\n",
+			attr->keyring_id);
+		return -EINVAL;
+	}
+
+	sig = kvmemdup_bpfptr(usig, attr->signature_size);
+	if (IS_ERR(sig)) {
+		bpf_key_put(key);
+		return PTR_ERR(sig);
+	}
+
+	insns_sz = prog->len * sizeof(struct bpf_insn);
+	data_sz = insns_sz;
+	for (i = 0; i < map_cnt; i++) {
+		struct bpf_map *map = maps[i];
+
+		if (map->map_type != BPF_MAP_TYPE_ARRAY ||
+		    !map->ops->map_direct_value_addr) {
+			verbose(env, "signed program metadata map '%s' must be an array\n",
+				map->name);
+			err = -EINVAL;
+			goto out;
+		}
+		if (!READ_ONCE(map->frozen)) {
+			verbose(env, "signed program metadata map '%s' must be frozen\n",
+				map->name);
+			err = -EPERM;
+			goto out;
+		}
+		if (bpf_map_write_active(map)) {
+			verbose(env, "signed program metadata map '%s' has active writers\n",
+				map->name);
+			err = -EBUSY;
+			goto out;
+		}
+		if (!map->excl_prog_sha) {
+			verbose(env, "signed program metadata map '%s' must be exclusive\n",
+				map->name);
+			err = -EPERM;
+			goto out;
+		}
+		data_sz += map->value_size;
+	}
+	if (bpf_dynptr_check_size(data_sz)) {
+		verbose(env, "signed payload too large: %llu bytes\n", data_sz);
+		err = -E2BIG;
+		goto out;
+	}
+	data = kvmalloc(data_sz, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
+	if (!data) {
+		err = -ENOMEM;
+		goto out;
+	}
+	memcpy(data, prog->insnsi, insns_sz);
+	off = insns_sz;
+	for (i = 0; i < map_cnt; i++) {
+		struct bpf_map *map = maps[i];
+		u64 addr;
+
+		err = map->ops->map_direct_value_addr(map, &addr, 0);
+		if (err) {
+			verbose(env, "failed to read signed metadata map '%s': %d\n",
+				map->name, err);
+			goto out;
+		}
+		memcpy(data + off, (void *)(unsigned long)addr,
+		       map->value_size);
+		off += map->value_size;
+	}
+
+	bpf_dynptr_init(&data_ptr, data, BPF_DYNPTR_TYPE_LOCAL, 0, data_sz);
+	bpf_dynptr_init(&sig_ptr, sig, BPF_DYNPTR_TYPE_LOCAL, 0,
+			attr->signature_size);
+
+	err = bpf_verify_pkcs7_signature((struct bpf_dynptr *)&data_ptr,
+					 (struct bpf_dynptr *)&sig_ptr, key);
+	if (err) {
+		verbose(env, "signature verification failed: %d\n", err);
+	} else {
+		verbose(env, "signature verification passed\n");
+		prog->aux->sig.keyring_serial = bpf_key_serial(key);
+		prog->aux->sig.keyring_type = bpf_classify_keyring(attr->keyring_id);
+		prog->aux->sig.verdict = BPF_SIG_VERIFIED;
+	}
+out:
+	kvfree(data);
+	bpf_key_put(key);
+	kvfree(sig);
+	return err;
+}
+
 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	      struct bpf_log_attr *attr_log)
 {
@@ -19743,18 +20112,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 		return -ENOMEM;
 
 	env->bt.env = env;
-
-	len = (*prog)->len;
-	env->insn_aux_data =
-		vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len));
-	ret = -ENOMEM;
-	if (!env->insn_aux_data)
-		goto err_free_env;
-	for (i = 0; i < len; i++)
-		env->insn_aux_data[i].orig_idx = i;
-	env->succ = bpf_iarray_realloc(NULL, 2);
-	if (!env->succ)
-		goto err_free_env;
 	env->prog = *prog;
 	env->ops = bpf_verifier_ops[env->prog->type];
 
@@ -19763,22 +20120,52 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token);
 	env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token);
 	env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF);
-
-	bpf_get_btf_vmlinux();
-
-	/* grab the mutex to protect few globals used by verifier */
-	if (!is_priv)
-		mutex_lock(&bpf_verifier_lock);
+	env->signature = attr->signature;
 
 	/* user could have requested verbose verifier output
 	 * and supplied buffer to store the verification trace
 	 */
 	ret = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size);
 	if (ret)
-		goto err_unlock;
+		goto err_free_env;
+	if (env->signature) {
+		ret = bpf_prog_calc_tag(env->prog);
+		if (ret < 0)
+			goto err_prep;
+	}
 
 	ret = process_fd_array(env, attr, uattr);
 	if (ret)
+		goto err_prep;
+
+	if (env->signature) {
+		ret = bpf_prog_verify_signature(env, attr, uattr.is_kernel);
+		if (ret)
+			goto err_prep;
+	}
+
+	ret = security_bpf_prog_load(env->prog, attr, env->prog->aux->token,
+				     uattr.is_kernel);
+	if (ret)
+		goto err_prep;
+
+	bpf_get_btf_vmlinux();
+
+	/* Serialize verification of unprivileged programs. */
+	if (!is_priv)
+		mutex_lock(&bpf_verifier_lock);
+
+	len = env->prog->len;
+	env->insn_aux_data =
+		__vmalloc(array_size(sizeof(struct bpf_insn_aux_data), len),
+			  GFP_KERNEL_ACCOUNT | __GFP_ZERO);
+	ret = -ENOMEM;
+	if (!env->insn_aux_data)
+		goto skip_full_check;
+	for (i = 0; i < len; i++)
+		env->insn_aux_data[i].orig_idx = i;
+	env->succ = bpf_iarray_realloc(NULL, 2);
+	if (!env->succ)
 		goto skip_full_check;
 
 	mark_verifier_state_clean(env);
@@ -20002,17 +20389,26 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	*prog = env->prog;
 
 	module_put(env->attach_btf_mod);
-err_unlock:
 	if (!is_priv)
 		mutex_unlock(&bpf_verifier_lock);
-	bpf_clear_insn_aux_data(env, 0, env->prog->len);
+	goto err_free_env;
+err_prep:
+	err = bpf_log_attr_finalize(attr_log, &env->log);
+	if (err)
+		ret = err;
+	release_insn_arrays(env);
+	release_maps(env);
+	release_btfs(env);
 err_free_env:
+	if (env->insn_aux_data)
+		bpf_clear_insn_aux_data(env, 0, env->prog->len);
+	vfree(env->insn_aux_data);
+	kvfree(env->fd_array);
 	bpf_stack_liveness_free(env);
 	kvfree(env->cfg.insn_postorder);
 	kvfree(env->scc_info);
 	kvfree(env->succ);
 	kvfree(env->gotox_tmp_buf);
-	vfree(env->insn_aux_data);
 	kvfree(env);
 	return ret;
 }
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 38f8d9df8fbc4..b5b461d4418b1 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -3996,6 +3996,7 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf,
 	struct psi_trigger *new;
 	struct cgroup *cgrp;
 	struct psi_group *psi;
+	bool need_rtpoll_worker;
 	ssize_t ret = 0;
 
 	cgrp = cgroup_kn_lock_live(of->kn, false);
@@ -4015,12 +4016,32 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf,
 	}
 
 	psi = cgroup_psi(cgrp);
-	new = psi_trigger_create(psi, buf, res, of->file, of);
+	new = psi_trigger_create(psi, buf, res, of->file, of,
+				 &need_rtpoll_worker);
 	if (IS_ERR(new)) {
 		ret = PTR_ERR(new);
 		goto out_unlock;
 	}
 
+	/*
+	 * The worker fork must run with neither cgroup_mutex nor the file's
+	 * kernfs active reference held. The latter is broken since
+	 * cgroup_kn_lock_live(). @of->priv may be released while unlocked, so
+	 * recheck before publishing @new.
+	 */
+	if (need_rtpoll_worker) {
+		cgroup_unlock();
+		ret = psi_trigger_create_rtpoll_worker(psi);
+		cgroup_lock();
+
+		if (!ret && !of->priv)
+			ret = -ENODEV;
+		if (ret) {
+			psi_trigger_destroy(new);
+			goto out_unlock;
+		}
+	}
+
 	smp_store_release(&ctx->psi.trigger, new);
 
 out_unlock:
diff --git a/kernel/cgroup/cpuset-internal.h b/kernel/cgroup/cpuset-internal.h
index f7aaf01f7cd5e..e7d010661fd32 100644
--- a/kernel/cgroup/cpuset-internal.h
+++ b/kernel/cgroup/cpuset-internal.h
@@ -146,10 +146,9 @@ struct cpuset {
 	nodemask_t old_mems_allowed;
 
 	/*
-	 * Tasks are being attached to this cpuset.  Used to prevent
-	 * zeroing cpus/mems_allowed between ->can_attach() and ->attach().
+	 * For linking impacted cpusets during an attach operation.
 	 */
-	int attach_in_progress;
+	struct llist_node attach_node;
 
 	/* partition root state */
 	int partition_root_state;
@@ -165,7 +164,7 @@ struct cpuset {
 	 * number of SCHED_DEADLINE tasks attached to this cpuset, so that we
 	 * know when to rebuild associated root domain bandwidth information.
 	 */
-	int nr_deadline_tasks;
+	atomic_t nr_deadline_tasks;
 	int nr_migrate_dl_tasks;
 	/* DL bandwidth that needs destination reservation for this attach. */
 	u64 sum_migrate_dl_bw;
@@ -269,10 +268,7 @@ static inline int nr_cpusets(void)
 static inline bool cpuset_is_populated(struct cpuset *cs)
 {
 	lockdep_assert_cpuset_lock_held();
-
-	/* Cpusets in the process of attaching should be considered as populated */
-	return cgroup_is_populated(cs->css.cgroup) ||
-		cs->attach_in_progress;
+	return cgroup_is_populated(cs->css.cgroup);
 }
 
 /**
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index 591e3aa487fc1..03e82d6026f57 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -37,6 +37,7 @@
 #include <linux/wait.h>
 #include <linux/workqueue.h>
 #include <linux/task_work.h>
+#include <linux/llist.h>
 
 DEFINE_STATIC_KEY_FALSE(cpusets_pre_enable_key);
 DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key);
@@ -222,14 +223,14 @@ void inc_dl_tasks_cs(struct task_struct *p)
 {
 	struct cpuset *cs = task_cs(p);
 
-	cs->nr_deadline_tasks++;
+	atomic_inc(&cs->nr_deadline_tasks);
 }
 
 void dec_dl_tasks_cs(struct task_struct *p)
 {
 	struct cpuset *cs = task_cs(p);
 
-	cs->nr_deadline_tasks--;
+	atomic_dec(&cs->nr_deadline_tasks);
 }
 
 static inline bool is_partition_valid(const struct cpuset *cs)
@@ -356,6 +357,41 @@ static struct workqueue_struct *cpuset_migrate_mm_wq;
 
 static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq);
 
+/*
+ * Cpuset task attach context
+ * Protected by cpuset_mutex
+ */
+static struct {
+	int in_progress;
+	bool cpus_updated;
+	bool mems_updated;
+	bool task_work_queued;
+	bool many_dest_cs;	/* Have many destination cpusets */
+	struct cpuset *old_cs;	/* Source cpuset */
+	nodemask_t nodemask_to;
+} attach_ctx;
+static LLIST_HEAD(src_cs_head);
+static LLIST_HEAD(dst_cs_head);
+
+/*
+ * Wait if task attach is in progress until it is done and then acquire
+ * cpuset_mutex before returning.
+ */
+static void wait_attach_done_lock(void)
+	    __acquires(&cpuset_mutex)
+{
+	for (;;) {
+		mutex_lock(&cpuset_mutex);
+		if (!attach_ctx.in_progress)
+			return;
+
+		mutex_unlock(&cpuset_mutex);
+
+		/* Wait until attach operation is done to prevent racing */
+		wait_event(cpuset_attach_wq, attach_ctx.in_progress == 0);
+	}
+}
+
 static inline void check_insane_mems_config(nodemask_t *nodes)
 {
 	if (!cpusets_insane_config() &&
@@ -368,22 +404,22 @@ static inline void check_insane_mems_config(nodemask_t *nodes)
 }
 
 /*
- * decrease cs->attach_in_progress.
- * wake_up cpuset_attach_wq if cs->attach_in_progress==0.
+ * decrease attach_ctx.in_progress.
+ * wake_up cpuset_attach_wq if attach_ctx.in_progress==0.
  */
-static inline void dec_attach_in_progress_locked(struct cpuset *cs)
+static inline void dec_attach_in_progress_locked(void)
 {
 	lockdep_assert_cpuset_lock_held();
 
-	cs->attach_in_progress--;
-	if (!cs->attach_in_progress)
+	attach_ctx.in_progress--;
+	if (!attach_ctx.in_progress)
 		wake_up(&cpuset_attach_wq);
 }
 
-static inline void dec_attach_in_progress(struct cpuset *cs)
+static inline void dec_attach_in_progress(void)
 {
 	mutex_lock(&cpuset_mutex);
-	dec_attach_in_progress_locked(cs);
+	dec_attach_in_progress_locked();
 	mutex_unlock(&cpuset_mutex);
 }
 
@@ -432,8 +468,7 @@ static inline bool partition_is_populated(struct cpuset *cs,
 	 * nr_populated_domain_children may include populated
 	 * csets from descendants that are partitions.
 	 */
-	if (cgroup_has_tasks(cs->css.cgroup) ||
-	    cs->attach_in_progress)
+	if (cgroup_has_tasks(cs->css.cgroup))
 		return true;
 
 	rcu_read_lock();
@@ -489,7 +524,10 @@ static void guarantee_active_cpus(struct task_struct *tsk,
  * Return in *pmask the portion of a cpusets's mems_allowed that
  * are online, with memory.  If none are online with memory, walk
  * up the cpuset hierarchy until we find one that does have some
- * online mems.  The top cpuset always has some mems online.
+ * online mems.  The top cpuset always has some mems online. With v2,
+ * effective_mems should always contain online memory nodes except
+ * during the transition period where a memory node hotunplug operation
+ * is in progress.
  *
  * One way or another, we guarantee to return some non-empty subset
  * of node_states[N_MEMORY].
@@ -581,6 +619,7 @@ static struct cpuset *dup_or_alloc_cpuset(struct cpuset *cs)
 		return NULL;
 
 	trial->dl_bw_cpu = -1;
+	init_llist_node(&trial->attach_node);
 
 	/* Setup cpumask pointer array */
 	cpumask_var_t *pmask[4] = {
@@ -918,7 +957,7 @@ static void dl_update_tasks_root_domain(struct cpuset *cs)
 	struct css_task_iter it;
 	struct task_struct *task;
 
-	if (cs->nr_deadline_tasks == 0)
+	if (atomic_read(&cs->nr_deadline_tasks) == 0)
 		return;
 
 	css_task_iter_start(&cs->css, 0, &it);
@@ -1089,12 +1128,35 @@ void cpuset_update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus)
  * @cs: the cpuset the need to recompute the new effective_cpus mask
  * @parent: the parent cpuset
  *
+ * For v2, the parent's effective_cpus is inherited if cpumask is empty.
  * The result is valid only if the given cpuset isn't a partition root.
  */
 static void compute_effective_cpumask(struct cpumask *new_cpus,
 				      struct cpuset *cs, struct cpuset *parent)
 {
-	cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus);
+	bool has_cpus;
+
+	has_cpus = cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus);
+	if (!has_cpus && is_in_v2_mode())
+		cpumask_copy(new_cpus, parent->effective_cpus);
+}
+
+/**
+ * compute_effective_nodemask - Compute the effective nodemask of the cpuset
+ * @new_mems: the temp variable for the new effective_mems mask
+ * @cs: the cpuset the need to recompute the new effective_mems mask
+ * @parent: the parent cpuset
+ *
+ * For v2, the parent's effective_mems is inherited if nodemask is empty.
+ */
+static void compute_effective_nodemask(nodemask_t *new_mems,
+				       struct cpuset *cs, struct cpuset *parent)
+{
+	bool has_mems;
+
+	has_mems = nodes_and(*new_mems, cs->mems_allowed, parent->effective_mems);
+	if (!has_mems && is_in_v2_mode())
+		nodes_copy(*new_mems, parent->effective_mems);
 }
 
 /*
@@ -2143,15 +2205,6 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp,
 			goto update_parent_effective;
 		}
 
-		/*
-		 * If it becomes empty, inherit the effective mask of the
-		 * parent, which is guaranteed to have some CPUs unless
-		 * it is a partition root that has explicitly distributed
-		 * out all its CPUs.
-		 */
-		if (is_in_v2_mode() && !remote && cpumask_empty(tmp->new_cpus))
-			cpumask_copy(tmp->new_cpus, parent->effective_cpus);
-
 		/*
 		 * Skip the whole subtree if
 		 * 1) the cpumask remains the same,
@@ -2619,6 +2672,14 @@ static void *cpuset_being_rebound;
  * Iterate through each task of @cs updating its mems_allowed to the
  * effective cpuset's.  As this function is called with cpuset_mutex held,
  * cpuset membership stays stable.
+ *
+ * - cpuset_change_task_nodemask(): guarantee_online_mems()
+ * - mpol_rebind_mm(): effective_mems
+ * - cpuset_migrate_mm(): guarantee_online_mems()
+ * - old_mems_allowed: guarantee_online_mems()
+ *
+ * For v2, guarantee_online_mems() should return a node mask that is the same
+ * as the effective_mems of current cpuset.
  */
 void cpuset_update_tasks_nodemask(struct cpuset *cs)
 {
@@ -2627,7 +2688,6 @@ void cpuset_update_tasks_nodemask(struct cpuset *cs)
 	struct task_struct *task;
 
 	cpuset_being_rebound = cs;		/* causes mpol_dup() rebind */
-
 	guarantee_online_mems(cs, &newmems);
 
 	/*
@@ -2647,13 +2707,22 @@ void cpuset_update_tasks_nodemask(struct cpuset *cs)
 
 		cpuset_change_task_nodemask(task, &newmems);
 
+		/* Rebind and migrate mm only for thread group leader */
+		if (!thread_group_leader(task))
+			continue;
+
 		mm = get_task_mm(task);
 		if (!mm)
 			continue;
 
 		migrate = is_memory_migrate(cs);
 
-		mpol_rebind_mm(mm, &cs->mems_allowed);
+		/*
+		 * For v1 we can have empty effective_mems, but we cannot
+		 * attach any tasks (see cpuset_can_attach_check()). For v2,
+		 * effective_mems is guaranteed to not be empty.
+		 */
+		mpol_rebind_mm(mm, &cs->effective_mems);
 		if (migrate)
 			cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems);
 		else
@@ -2692,14 +2761,7 @@ static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems)
 	cpuset_for_each_descendant_pre(cp, pos_css, cs) {
 		struct cpuset *parent = parent_cs(cp);
 
-		bool has_mems = nodes_and(*new_mems, cp->mems_allowed, parent->effective_mems);
-
-		/*
-		 * If it becomes empty, inherit the effective mask of the
-		 * parent, which is guaranteed to have some MEMs.
-		 */
-		if (is_in_v2_mode() && !has_mems)
-			*new_mems = parent->effective_mems;
+		compute_effective_nodemask(new_mems, cp, parent);
 
 		/* Skip the whole subtree if the nodemask remains the same. */
 		if (nodes_equal(*new_mems, cp->effective_mems)) {
@@ -2968,27 +3030,104 @@ static int update_prstate(struct cpuset *cs, int new_prs)
 	return 0;
 }
 
-static struct cpuset *cpuset_attach_old_cs;
-
 /*
  * Check to see if a cpuset can accept a new task
  * For v1, cpus_allowed and mems_allowed can't be empty.
  * For v2, effective_cpus can't be empty.
  * Note that in v1, effective_cpus = cpus_allowed.
+ *
+ * Also set the boolean flag passed in by @psetsched depending on if
+ * security_task_setscheduler() call is needed and @oldcs is not NULL.
  */
-static int cpuset_can_attach_check(struct cpuset *cs)
+static int cpuset_can_attach_check(struct cpuset *cs, struct cpuset *oldcs,
+				   bool *psetsched)
 {
+	bool cpus_updated, mems_updated;
+
 	if (cpumask_empty(cs->effective_cpus) ||
 	   (!is_in_v2_mode() && nodes_empty(cs->mems_allowed)))
 		return -ENOSPC;
+
+	if (!oldcs)
+		return 0;
+
+	if (!llist_on_list(&oldcs->attach_node))
+		llist_add(&oldcs->attach_node, &src_cs_head);
+
+	if (!llist_on_list(&cs->attach_node))
+		llist_add(&cs->attach_node, &dst_cs_head);
+
+	cpus_updated = !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus);
+	mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems);
+
+	if (cpus_updated)
+		attach_ctx.cpus_updated = true;
+	if (mems_updated)
+		attach_ctx.mems_updated = true;
+
+	/*
+	 * Skip rights over task setsched check in v2 when nothing changes for
+	 * the current oldcs/cs pair, migration permission derives from
+	 * hierarchy ownership in cgroup_procs_write_permission()).
+	 */
+	*psetsched = !cpuset_v2() || cpus_updated || mems_updated;
+
+	/*
+	 * A v1 cpuset with tasks will have no CPU left only when CPU hotplug
+	 * brings the last online CPU offline as users are not allowed to empty
+	 * cpuset.cpus when there are active tasks inside. When that happens,
+	 * we should allow tasks to migrate out without security check to make
+	 * sure they will be able to run after migration.
+	 */
+	if (!is_in_v2_mode() && cpumask_empty(oldcs->effective_cpus))
+		*psetsched = false;
+
 	return 0;
 }
 
-static void reset_migrate_dl_data(struct cpuset *cs)
+static int cpuset_reserve_dl_bw(void)
 {
-	cs->nr_migrate_dl_tasks = 0;
-	cs->sum_migrate_dl_bw = 0;
-	cs->dl_bw_cpu = -1;
+	struct cpuset *cs;
+	int cpu, ret;
+
+	llist_for_each_entry(cs, dst_cs_head.first, attach_node) {
+		if (!cs->sum_migrate_dl_bw)
+			continue;
+
+		cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus);
+		if (unlikely(cpu >= nr_cpu_ids))
+			return -EINVAL;
+
+		ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw);
+		if (ret)
+			return ret;
+
+		cs->dl_bw_cpu = cpu;
+	}
+	return 0;
+}
+
+/*
+ * Clear and optionally apply (@cancel is false) the attach related data in the
+ * source or destination cpuset.
+ */
+static void clear_attach_data(struct llist_head *head, bool cancel)
+{
+	struct cpuset *cs, *next;
+	struct llist_node *lnode = __llist_del_all(head);
+
+	llist_for_each_entry_safe(cs, next, lnode, attach_node) {
+		init_llist_node(&cs->attach_node);
+		if (cs->nr_migrate_dl_tasks) {
+			if (!cancel)
+				atomic_add(cs->nr_migrate_dl_tasks, &cs->nr_deadline_tasks);
+			else if (cs->dl_bw_cpu >= 0) /* && cancel */
+				dl_bw_free(cs->dl_bw_cpu, cs->sum_migrate_dl_bw);
+			cs->nr_migrate_dl_tasks = 0;
+			cs->sum_migrate_dl_bw = 0;
+			cs->dl_bw_cpu = -1;
+		}
+	}
 }
 
 /* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */
@@ -2998,44 +3137,66 @@ static int cpuset_can_attach(struct cgroup_taskset *tset)
 	struct cpuset *cs, *oldcs;
 	struct task_struct *task;
 	bool setsched_check;
-	int cpu, ret;
-
-	/* used later by cpuset_attach() */
-	cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset, &css));
-	oldcs = cpuset_attach_old_cs;
-	cs = css_cs(css);
+	int ret;
 
+	cs = oldcs = NULL;
 	mutex_lock(&cpuset_mutex);
-
-	/* Check to see if task is allowed in the cpuset */
-	ret = cpuset_can_attach_check(cs);
-	if (ret)
-		goto out_unlock;
+	attach_ctx.old_cs = NULL;	/* Used later in cpuset_attach_task() */
+	attach_ctx.cpus_updated = false;
+	attach_ctx.mems_updated = false;
+	attach_ctx.many_dest_cs = false;
 
 	/*
-	 * Skip rights over task setsched check in v2 when nothing changes,
-	 * migration permission derives from hierarchy ownership in
-	 * cgroup_procs_write_permission()).
+	 * The attach_ctx.old_cs is used mainly by cpuset_migrate_mm() to get
+	 * the old_mems_allowed value. There are two ways that many-to-one
+	 * cpuset migration can happen:
+	 * 1) A multithread application with threads in different cpusets is
+	 *    wholely migrated to a new cpuset.
+	 * 2) Disabling v2 cpuset controller will move all the tasks in child
+	 *    cpusets to the parent cpuset.
+	 *
+	 * In the former case, it is the mm setting of the group leader that
+	 * really matters. So attach_ctx.old_cs should track the oldcs of the
+	 * group leader. It falls back to the oldcs of the first task if there
+	 * is no group leader in the taskset. In the latter case, effective_mems
+	 * of child cpusets must always be a subset of the parent. So no real
+	 * page migration will be necessary no matter which child cpuset is
+	 * selected as attach_ctx.old_cs.
+	 *
+	 * For a v2 threaded subtree where cpuset isn't enabled in some of the
+	 * cgroups, it is possible that oldcs == cs for some of the tasks.
+	 * In this case, we can skip checking on those tasks as there is no
+	 * actual migration wrt cpuset.
 	 */
-	setsched_check = !cpuset_v2() ||
-		!cpumask_equal(cs->effective_cpus, oldcs->effective_cpus) ||
-		!nodes_equal(cs->effective_mems, oldcs->effective_mems);
+	cgroup_taskset_for_each(task, css, tset) {
+		struct cpuset *new_cs = css_cs(css);
+		struct cpuset *new_oldcs = task_cs(task);
+
+		if ((new_oldcs != oldcs) || (new_cs != cs)) {
+			if (cs && (new_cs != cs))
+				attach_ctx.many_dest_cs = true;
+			cs = new_cs;
+			oldcs = new_oldcs;
+			if (oldcs == cs)
+				continue;
+			if (!attach_ctx.old_cs)
+				attach_ctx.old_cs = oldcs;
+			ret = cpuset_can_attach_check(cs, oldcs, &setsched_check);
+			if (ret)
+				goto out_unlock;
+		}
 
-	/*
-	 * A v1 cpuset with tasks will have no CPU left only when CPU hotplug
-	 * brings the last online CPU offline as users are not allowed to empty
-	 * cpuset.cpus when there are active tasks inside. When that happens,
-	 * we should allow tasks to migrate out without security check to make
-	 * sure they will be able to run after migration.
-	 */
-	if (!is_in_v2_mode() && cpumask_empty(oldcs->effective_cpus))
-		setsched_check = false;
+		if (oldcs == cs)
+			continue;
 
-	cgroup_taskset_for_each(task, css, tset) {
 		ret = task_can_attach(task);
 		if (ret)
 			goto out_unlock;
 
+		/* Update attach_ctx.old_cs to the latest group leader */
+		if (task == task->group_leader)
+			attach_ctx.old_cs = task_cs(task);
+
 		if (setsched_check) {
 			ret = security_task_setscheduler(task);
 			if (ret)
@@ -3049,57 +3210,48 @@ static int cpuset_can_attach(struct cgroup_taskset *tset)
 			 * contribute to sum_migrate_dl_bw.
 			 */
 			cs->nr_migrate_dl_tasks++;
+			oldcs->nr_migrate_dl_tasks--;
 			if (dl_task_needs_bw_move(task, cs->effective_cpus))
 				cs->sum_migrate_dl_bw += task->dl.dl_bw;
 		}
 	}
 
-	if (!cs->sum_migrate_dl_bw)
-		goto out_success;
-
-	cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus);
-	if (unlikely(cpu >= nr_cpu_ids)) {
+	/*
+	 * The only case where there are multiple destination cpusets for
+	 * task migration is when enabling a v2 cpuset controllers where
+	 * tasks will be migrated to multiple child cpusets from a parent
+	 * cpuset with the same effective CPUs and memory nodes. IOW,
+	 * both attach_cpus_updated and attach_mems_updated should be false.
+	 * If not, it is a condition that the current code cannot handle.
+	 * Print a warning and abort the attach operation as further code
+	 * change may be needed.
+	 */
+	if (WARN_ON_ONCE(attach_ctx.many_dest_cs && (!cpuset_v2() ||
+			 attach_ctx.cpus_updated || attach_ctx.mems_updated))) {
 		ret = -EINVAL;
 		goto out_unlock;
 	}
 
-	ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw);
-	if (ret)
-		goto out_unlock;
-
-	cs->dl_bw_cpu = cpu;
-
-out_success:
-	/*
-	 * Mark attach is in progress.  This makes validate_change() fail
-	 * changes which zero cpus/mems_allowed.
-	 */
-	cs->attach_in_progress++;
+	ret = cpuset_reserve_dl_bw();
 
 out_unlock:
-	if (ret)
-		reset_migrate_dl_data(cs);
+	if (ret) {
+		clear_attach_data(&src_cs_head, true);
+		clear_attach_data(&dst_cs_head, true);
+	} else {
+		attach_ctx.in_progress++;
+	}
+
 	mutex_unlock(&cpuset_mutex);
 	return ret;
 }
 
 static void cpuset_cancel_attach(struct cgroup_taskset *tset)
 {
-	struct cgroup_subsys_state *css;
-	struct cpuset *cs;
-
-	cgroup_taskset_first(tset, &css);
-	cs = css_cs(css);
-
 	mutex_lock(&cpuset_mutex);
-	dec_attach_in_progress_locked(cs);
-
-	if (cs->dl_bw_cpu >= 0)
-		dl_bw_free(cs->dl_bw_cpu, cs->sum_migrate_dl_bw);
-
-	if (cs->nr_migrate_dl_tasks)
-		reset_migrate_dl_data(cs);
-
+	dec_attach_in_progress_locked();
+	clear_attach_data(&src_cs_head, true);
+	clear_attach_data(&dst_cs_head, true);
 	mutex_unlock(&cpuset_mutex);
 }
 
@@ -3109,10 +3261,11 @@ static void cpuset_cancel_attach(struct cgroup_taskset *tset)
  * allocate from cpuset_init().
  */
 static cpumask_var_t cpus_attach;
-static nodemask_t cpuset_attach_nodemask_to;
 
 static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task)
 {
+	struct mm_struct *mm;
+
 	lockdep_assert_cpuset_lock_held();
 
 	if (cs != &top_cpuset)
@@ -3126,90 +3279,88 @@ static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task)
 	 */
 	WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpus_attach));
 
-	cpuset_change_task_nodemask(task, &cpuset_attach_nodemask_to);
+	if (cpuset_v2() && !attach_ctx.mems_updated)
+		return;
+
+	cpuset_change_task_nodemask(task, &attach_ctx.nodemask_to);
 	cpuset1_update_task_spread_flags(cs, task);
+
+	if ((task != task->group_leader) || !attach_ctx.mems_updated)
+		return;
+
+	/*
+	 * Change mm for threadgroup leader. This is expensive and may
+	 * sleep and should be moved outside migration path proper.
+	 */
+	mm = get_task_mm(task);
+	if (mm) {
+		struct cpuset *oldcs = attach_ctx.old_cs;
+
+		mpol_rebind_mm(mm, &cs->effective_mems);
+
+		/*
+		 * old_mems_allowed is the same with mems_allowed
+		 * here, except if this task is being moved
+		 * automatically due to hotplug.  In that case
+		 * @mems_allowed has been updated and is empty, so
+		 * @old_mems_allowed is the right nodesets that we
+		 * migrate mm from.
+		 */
+		if (is_memory_migrate(cs)) {
+			cpuset_migrate_mm(mm, &oldcs->old_mems_allowed,
+					  &attach_ctx.nodemask_to);
+			attach_ctx.task_work_queued = true;
+		} else {
+			mmput(mm);
+		}
+	}
 }
 
 static void cpuset_attach(struct cgroup_taskset *tset)
 {
 	struct task_struct *task;
-	struct task_struct *leader;
 	struct cgroup_subsys_state *css;
 	struct cpuset *cs;
-	struct cpuset *oldcs = cpuset_attach_old_cs;
-	bool cpus_updated, mems_updated;
-	bool queue_task_work = false;
 
 	cgroup_taskset_first(tset, &css);
 	cs = css_cs(css);
 
 	lockdep_assert_cpus_held();	/* see cgroup_attach_lock() */
 	mutex_lock(&cpuset_mutex);
-	cpus_updated = !cpumask_equal(cs->effective_cpus,
-				      oldcs->effective_cpus);
-	mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems);
+	attach_ctx.task_work_queued = false;
+	guarantee_online_mems(cs, &attach_ctx.nodemask_to);
 
 	/*
-	 * In the default hierarchy, enabling cpuset in the child cgroups
-	 * will trigger a number of cpuset_attach() calls with no change
-	 * in effective cpus and mems. In that case, we can optimize out
-	 * by skipping the task iteration and update.
+	 * attach_ctx.old_cs can only be NULL if no task is actually migrating.
+	 * This is highly unlikely. If it happens at all, we can skip task
+	 * iteration and setting old_mems_allowed.
 	 */
-	if (cpuset_v2() && !cpus_updated && !mems_updated) {
-		cpuset_attach_nodemask_to = cs->effective_mems;
+	if (unlikely(!attach_ctx.old_cs))
 		goto out;
-	}
-
-	guarantee_online_mems(cs, &cpuset_attach_nodemask_to);
-
-	cgroup_taskset_for_each(task, css, tset)
-		cpuset_attach_task(cs, task);
 
 	/*
-	 * Change mm for all threadgroup leaders. This is expensive and may
-	 * sleep and should be moved outside migration path proper. Skip it
-	 * if there is no change in effective_mems and CS_MEMORY_MIGRATE is
-	 * not set.
+	 * In the default hierarchy, enabling cpuset in the child cgroups
+	 * will trigger a cpuset_attach() call with no change in effective cpus
+	 * and mems. In that case, we can optimize out by skipping the task
+	 * iteration and the destination cpuset list is iterated to set
+	 * old_mems_allowed.
 	 */
-	cpuset_attach_nodemask_to = cs->effective_mems;
-	if (!is_memory_migrate(cs) && !mems_updated)
+	if (cpuset_v2() && !attach_ctx.cpus_updated && !attach_ctx.mems_updated) {
+		llist_for_each_entry(cs, dst_cs_head.first, attach_node)
+			cs->old_mems_allowed = attach_ctx.nodemask_to;
 		goto out;
-
-	cgroup_taskset_for_each_leader(leader, css, tset) {
-		struct mm_struct *mm = get_task_mm(leader);
-
-		if (mm) {
-			mpol_rebind_mm(mm, &cpuset_attach_nodemask_to);
-
-			/*
-			 * old_mems_allowed is the same with mems_allowed
-			 * here, except if this task is being moved
-			 * automatically due to hotplug.  In that case
-			 * @mems_allowed has been updated and is empty, so
-			 * @old_mems_allowed is the right nodesets that we
-			 * migrate mm from.
-			 */
-			if (is_memory_migrate(cs)) {
-				cpuset_migrate_mm(mm, &oldcs->old_mems_allowed,
-						  &cpuset_attach_nodemask_to);
-				queue_task_work = true;
-			} else
-				mmput(mm);
-		}
 	}
 
-out:
-	if (queue_task_work)
-		schedule_flush_migrate_mm();
-	cs->old_mems_allowed = cpuset_attach_nodemask_to;
-
-	if (cs->nr_migrate_dl_tasks) {
-		cs->nr_deadline_tasks += cs->nr_migrate_dl_tasks;
-		oldcs->nr_deadline_tasks -= cs->nr_migrate_dl_tasks;
-		reset_migrate_dl_data(cs);
-	}
+	cgroup_taskset_for_each(task, css, tset)
+		cpuset_attach_task(cs, task);
 
-	dec_attach_in_progress_locked(cs);
+	if (attach_ctx.task_work_queued)
+		schedule_flush_migrate_mm();
+	cs->old_mems_allowed = attach_ctx.nodemask_to;
+out:
+	clear_attach_data(&src_cs_head, false);
+	clear_attach_data(&dst_cs_head, false);
+	dec_attach_in_progress_locked();
 
 	mutex_unlock(&cpuset_mutex);
 }
@@ -3229,7 +3380,12 @@ ssize_t cpuset_write_resmask(struct kernfs_open_file *of,
 		return -EACCES;
 
 	buf = strstrip(buf);
-	cpuset_full_lock();
+
+	/* cpuset_mutex acquired in wait_attach_done_lock() */
+	mutex_lock(&cpuset_top_mutex);
+	cpus_read_lock();
+	wait_attach_done_lock();
+
 	if (!is_cpuset_online(cs))
 		goto out_unlock;
 
@@ -3360,7 +3516,10 @@ static ssize_t cpuset_partition_write(struct kernfs_open_file *of, char *buf,
 	else
 		return -EINVAL;
 
-	cpuset_full_lock();
+	mutex_lock(&cpuset_top_mutex);
+	cpus_read_lock();
+	wait_attach_done_lock();
+
 	if (is_cpuset_online(cs))
 		retval = update_prstate(cs, val);
 	cpuset_update_sd_hk_unlock();
@@ -3587,7 +3746,7 @@ static int cpuset_can_fork(struct task_struct *task, struct css_set *cset)
 	mutex_lock(&cpuset_mutex);
 
 	/* Check to see if task is allowed in the cpuset */
-	ret = cpuset_can_attach_check(cs);
+	ret = cpuset_can_attach_check(cs, NULL, NULL);
 	if (ret)
 		goto out_unlock;
 
@@ -3599,11 +3758,7 @@ static int cpuset_can_fork(struct task_struct *task, struct css_set *cset)
 	if (ret)
 		goto out_unlock;
 
-	/*
-	 * Mark attach is in progress.  This makes validate_change() fail
-	 * changes which zero cpus/mems_allowed.
-	 */
-	cs->attach_in_progress++;
+	attach_ctx.in_progress++;
 out_unlock:
 	mutex_unlock(&cpuset_mutex);
 	return ret;
@@ -3621,7 +3776,7 @@ static void cpuset_cancel_fork(struct task_struct *task, struct css_set *cset)
 	if (same_cs)
 		return;
 
-	dec_attach_in_progress(cs);
+	dec_attach_in_progress();
 }
 
 /*
@@ -3631,15 +3786,14 @@ static void cpuset_cancel_fork(struct task_struct *task, struct css_set *cset)
  */
 static void cpuset_fork(struct task_struct *task)
 {
-	struct cpuset *cs;
-	bool same_cs;
+	struct cpuset *cs, *oldcs;
 
 	rcu_read_lock();
 	cs = task_cs(task);
-	same_cs = (cs == task_cs(current));
+	oldcs = task_cs(current);
 	rcu_read_unlock();
 
-	if (same_cs) {
+	if (cs == oldcs) {
 		if (cs == &top_cpuset)
 			return;
 
@@ -3650,10 +3804,22 @@ static void cpuset_fork(struct task_struct *task)
 
 	/* CLONE_INTO_CGROUP */
 	mutex_lock(&cpuset_mutex);
-	guarantee_online_mems(cs, &cpuset_attach_nodemask_to);
+	guarantee_online_mems(cs, &attach_ctx.nodemask_to);
+	cs->old_mems_allowed = attach_ctx.nodemask_to;
+
+	/*
+	 * Assume CPUs and memory nodes are updated
+	 * A CLONE_INTO_CGROUP operation should have taken the cgroup mutex
+	 * and so there shouldn't be a competing cpuset_attach() operation.
+	 */
+	attach_ctx.cpus_updated = attach_ctx.mems_updated = true;
+	attach_ctx.task_work_queued = false;
+	attach_ctx.old_cs = oldcs;
 	cpuset_attach_task(cs, task);
+	if (attach_ctx.task_work_queued)
+		schedule_flush_migrate_mm();
 
-	dec_attach_in_progress_locked(cs);
+	dec_attach_in_progress_locked();
 	mutex_unlock(&cpuset_mutex);
 }
 
@@ -3700,6 +3866,7 @@ int __init cpuset_init(void)
 	cpumask_setall(top_cpuset.effective_xcpus);
 	cpumask_setall(top_cpuset.exclusive_cpus);
 	nodes_setall(top_cpuset.effective_mems);
+	init_llist_node(&top_cpuset.attach_node);
 
 	cpuset1_init(&top_cpuset);
 
@@ -3757,23 +3924,11 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
 	bool remote;
 	int partcmd = -1;
 	struct cpuset *parent;
-retry:
-	wait_event(cpuset_attach_wq, cs->attach_in_progress == 0);
-
-	mutex_lock(&cpuset_mutex);
-
-	/*
-	 * We have raced with task attaching. We wait until attaching
-	 * is finished, so we won't attach a task to an empty cpuset.
-	 */
-	if (cs->attach_in_progress) {
-		mutex_unlock(&cpuset_mutex);
-		goto retry;
-	}
 
+	wait_attach_done_lock();
 	parent = parent_cs(cs);
 	compute_effective_cpumask(&new_cpus, cs, parent);
-	nodes_and(new_mems, cs->mems_allowed, parent->effective_mems);
+	compute_effective_nodemask(&new_mems, cs, parent);
 
 	if (!tmp || !cs->partition_root_state)
 		goto update_tasks;
@@ -4124,6 +4279,28 @@ bool cpuset_cpus_allowed_fallback(struct task_struct *tsk)
 	return changed;
 }
 
+/*
+ * Returns the number of CPUs available for this cgroup.
+ *
+ * This only really works for cgroup-v2 where all the controllers are mounted
+ * in the same hierarchy. If not cgroup-v2 or no cpuset controller is
+ * configured it reverts to num_online_cpus().
+ */
+int cpuset_num_cpus(struct cgroup *cgrp)
+{
+	int nr = num_online_cpus();
+	struct cpuset *cs;
+
+	if (is_in_v2_mode()) {
+		guard(rcu)();
+		cs = css_cs(cgroup_e_css(cgrp, &cpuset_cgrp_subsys));
+		if (cs)
+			nr = cpumask_weight(cs->effective_cpus);
+	}
+
+	return nr;
+}
+
 void __init cpuset_init_current_mems_allowed(void)
 {
 	nodes_setall(current->mems_allowed);
@@ -4157,7 +4334,7 @@ nodemask_t cpuset_mems_allowed(struct task_struct *tsk)
  *
  * Are any of the nodes in the nodemask allowed in current->mems_allowed?
  */
-int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
+int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask)
 {
 	return nodes_intersects(*nodemask, current->mems_allowed);
 }
diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c
index 4753a67d0f0f2..6430c7ce1e037 100644
--- a/kernel/cgroup/dmem.c
+++ b/kernel/cgroup/dmem.c
@@ -182,6 +182,11 @@ static u64 get_resource_current(struct dmem_cgroup_pool_state *pool)
 	return pool ? page_counter_read(&pool->cnt) : 0;
 }
 
+static u64 get_resource_peak(struct dmem_cgroup_pool_state *pool)
+{
+	return pool ? READ_ONCE(pool->cnt.watermark) : 0;
+}
+
 static void reset_all_resource_limits(struct dmem_cgroup_pool_state *rpool)
 {
 	set_resource_min(rpool, 0);
@@ -808,6 +813,11 @@ static int dmemcg_limit_show(struct seq_file *sf, void *v,
 	return 0;
 }
 
+static int dmem_cgroup_region_peak_show(struct seq_file *sf, void *v)
+{
+	return dmemcg_limit_show(sf, v, get_resource_peak);
+}
+
 static int dmem_cgroup_region_current_show(struct seq_file *sf, void *v)
 {
 	return dmemcg_limit_show(sf, v, get_resource_current);
@@ -856,6 +866,11 @@ static struct cftype files[] = {
 		.name = "current",
 		.seq_show = dmem_cgroup_region_current_show,
 	},
+	{
+		.name = "peak",
+		.seq_show = dmem_cgroup_region_peak_show,
+		.flags = CFTYPE_NOT_ON_ROOT,
+	},
 	{
 		.name = "min",
 		.write = dmem_cgroup_region_min_write,
diff --git a/kernel/crash_core.c b/kernel/crash_core.c
index 4f21fc3b108b8..2b36aa9fade06 100644
--- a/kernel/crash_core.c
+++ b/kernel/crash_core.c
@@ -168,9 +168,6 @@ static inline resource_size_t crash_resource_size(const struct resource *res)
 	return !res->end ? 0 : resource_size(res);
 }
 
-
-
-
 int crash_prepare_elf64_headers(struct crash_mem *mem, int need_kernel_map,
 			  void **addr, unsigned long *sz)
 {
@@ -272,6 +269,92 @@ int crash_prepare_elf64_headers(struct crash_mem *mem, int need_kernel_map,
 	return 0;
 }
 
+static struct crash_mem *alloc_cmem(unsigned int nr_ranges)
+{
+	struct crash_mem *cmem;
+
+	cmem = kvzalloc_flex(*cmem, ranges, nr_ranges);
+	if (!cmem)
+		return NULL;
+
+	cmem->max_nr_ranges = nr_ranges;
+	return cmem;
+}
+
+unsigned int __weak arch_get_system_nr_ranges(void) { return 0; }
+int __weak arch_crash_populate_cmem(struct crash_mem *cmem) { return -1; }
+int __weak arch_crash_exclude_ranges(struct crash_mem *cmem) { return 0; }
+
+int __weak arch_crash_exclude_mem_range(struct crash_mem **mem,
+					unsigned long long mstart,
+					unsigned long long mend)
+{
+	return crash_exclude_mem_range(*mem, mstart, mend);
+}
+
+int crash_exclude_core_ranges(struct crash_mem **cmem)
+{
+	int ret, i;
+
+	/* Exclude crashkernel region */
+	ret = arch_crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end);
+	if (ret)
+		return ret;
+
+	if (crashk_low_res.end) {
+		ret = arch_crash_exclude_mem_range(cmem, crashk_low_res.start, crashk_low_res.end);
+		if (ret)
+			return ret;
+	}
+
+	for (i = 0; i < crashk_cma_cnt; ++i) {
+		ret = arch_crash_exclude_mem_range(cmem, crashk_cma_ranges[i].start,
+						   crashk_cma_ranges[i].end);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+int crash_prepare_headers(int need_kernel_map, void **addr, unsigned long *sz,
+			  unsigned long *nr_mem_ranges)
+{
+	unsigned int max_nr_ranges;
+	struct crash_mem *cmem;
+	int ret;
+
+	max_nr_ranges = arch_get_system_nr_ranges();
+	if (!max_nr_ranges)
+		return -ENOMEM;
+
+	cmem = alloc_cmem(max_nr_ranges);
+	if (!cmem)
+		return -ENOMEM;
+
+	ret = arch_crash_populate_cmem(cmem);
+	if (ret)
+		goto out;
+
+	ret = crash_exclude_core_ranges(&cmem);
+	if (ret)
+		goto out;
+
+	ret = arch_crash_exclude_ranges(cmem);
+	if (ret)
+		goto out;
+
+	/* Return the computed number of memory ranges, for hotplug usage */
+	if (nr_mem_ranges)
+		*nr_mem_ranges = cmem->nr_ranges;
+
+	ret = crash_prepare_elf64_headers(cmem, need_kernel_map, addr, sz);
+
+out:
+	kvfree(cmem);
+	return ret;
+}
+
 /**
  * crash_exclude_mem_range - exclude a mem range for existing ranges
  * @mem: mem->range contains an array of ranges sorted in ascending order
diff --git a/kernel/crash_dump_dm_crypt.c b/kernel/crash_dump_dm_crypt.c
index cb875ddb6ba68..c685497cd470e 100644
--- a/kernel/crash_dump_dm_crypt.c
+++ b/kernel/crash_dump_dm_crypt.c
@@ -81,7 +81,6 @@ static int add_key_to_keyring(struct dm_crypt_key *dm_key,
 		kexec_dprintk("Error when adding key");
 	}
 
-	key_ref_put(keyring_ref);
 	return r;
 }
 
@@ -104,6 +103,7 @@ static int restore_dm_crypt_keys_to_thread_keyring(void)
 	struct dm_crypt_key *key;
 	size_t keys_header_size;
 	key_ref_t keyring_ref;
+	int ret = 0;
 	u64 addr;
 
 	/* find the target keyring (which must be writable) */
@@ -118,7 +118,8 @@ static int restore_dm_crypt_keys_to_thread_keyring(void)
 	dm_crypt_keys_read((char *)&key_count, sizeof(key_count), &addr);
 	if (key_count > KEY_NUM_MAX) {
 		kexec_dprintk("Failed to read the number of dm-crypt keys\n");
-		return -1;
+		ret = -1;
+		goto out;
 	}
 
 	kexec_dprintk("There are %u keys\n", key_count);
@@ -126,8 +127,10 @@ static int restore_dm_crypt_keys_to_thread_keyring(void)
 
 	keys_header_size = get_keys_header_size(key_count);
 	keys_header = kzalloc(keys_header_size, GFP_KERNEL);
-	if (!keys_header)
-		return -ENOMEM;
+	if (!keys_header) {
+		ret = -ENOMEM;
+		goto out;
+	}
 
 	dm_crypt_keys_read((char *)keys_header, keys_header_size, &addr);
 
@@ -137,7 +140,9 @@ static int restore_dm_crypt_keys_to_thread_keyring(void)
 		add_key_to_keyring(key, keyring_ref);
 	}
 
-	return 0;
+out:
+	key_ref_put(keyring_ref);
+	return ret;
 }
 
 static int read_key_from_user_keyring(struct dm_crypt_key *dm_key)
diff --git a/kernel/dma/coherent.c b/kernel/dma/coherent.c
index bcdc0f76d2e80..45bbae947f4be 100644
--- a/kernel/dma/coherent.c
+++ b/kernel/dma/coherent.c
@@ -28,7 +28,7 @@ static inline struct dma_coherent_mem *dev_get_coherent_memory(struct device *de
 }
 
 static inline dma_addr_t dma_get_device_base(struct device *dev,
-					     struct dma_coherent_mem * mem)
+					     struct dma_coherent_mem *mem)
 {
 	if (mem->use_dev_dma_pfn_offset)
 		return phys_to_dma(dev, PFN_PHYS(mem->pfn_base));
@@ -69,8 +69,8 @@ static struct dma_coherent_mem *dma_init_coherent_memory(phys_addr_t phys_addr,
 	kfree(dma_mem);
 out_unmap_membase:
 	memunmap(mem_base);
-	pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %zd MiB\n",
-		&phys_addr, size / SZ_1M);
+	pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %zu KiB\n",
+		&phys_addr, size / SZ_1K);
 	return ERR_PTR(-ENOMEM);
 }
 
@@ -236,14 +236,15 @@ static int __dma_mmap_from_coherent(struct dma_coherent_mem *mem,
 {
 	if (mem && vaddr >= mem->virt_base && vaddr + size <=
 		   (mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) {
-		unsigned long off = vma->vm_pgoff;
+		const pgoff_t pgoff_start = vma_start_pgoff(vma);
+		const pgoff_t pgoff_end = vma_end_pgoff(vma);
 		int start = (vaddr - mem->virt_base) >> PAGE_SHIFT;
 		unsigned long user_count = vma_pages(vma);
 		int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 
 		*ret = -ENXIO;
-		if (off < count && user_count <= count - off) {
-			unsigned long pfn = mem->pfn_base + start + off;
+		if (pgoff_start < count && pgoff_end <= count) {
+			unsigned long pfn = mem->pfn_base + start + pgoff_start;
 			*ret = remap_pfn_range(vma, vma->vm_start, pfn,
 					       user_count << PAGE_SHIFT,
 					       vma->vm_page_prot);
@@ -384,8 +385,8 @@ static int __init rmem_dma_setup(unsigned long node, struct reserved_mem *rmem)
 	}
 #endif
 
-	pr_info("Reserved memory: created DMA memory pool at %pa, size %ld MiB\n",
-		&rmem->base, (unsigned long)rmem->size / SZ_1M);
+	pr_info("Reserved memory: created DMA memory pool at %pa, size %llu KiB\n",
+		&rmem->base, (unsigned long long)(rmem->size / SZ_1K));
 	return 0;
 }
 
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 4391b797d4db8..6cb18cfa92ff7 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -489,9 +489,8 @@ int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
 		case PCI_P2PDMA_MAP_BUS_ADDR:
 			sg->dma_address = pci_p2pdma_bus_addr_map(
 				p2pdma_state.mem, sg_phys(sg));
-			sg_dma_len(sg) = sg->length;
 			sg_dma_mark_bus_address(sg);
-			continue;
+			break;
 		default:
 			ret = -EREMOTEIO;
 			goto out_unmap;
@@ -534,6 +533,8 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma,
 	unsigned long user_count = vma_pages(vma);
 	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 	unsigned long pfn = PHYS_PFN(dma_to_phys(dev, dma_addr));
+	const pgoff_t pgoff_start = vma_start_pgoff(vma);
+	const pgoff_t pgoff_end = vma_end_pgoff(vma);
 	int ret = -ENXIO;
 
 	vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);
@@ -545,9 +546,9 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma,
 	if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret))
 		return ret;
 
-	if (vma->vm_pgoff >= count || user_count > count - vma->vm_pgoff)
+	if (pgoff_start >= count || pgoff_end > count)
 		return -ENXIO;
-	return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,
+	return remap_pfn_range(vma, vma->vm_start, pfn + pgoff_start,
 			user_count << PAGE_SHIFT, vma->vm_page_prot);
 }
 
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index 4fe04669e5e66..c986639044e92 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -761,12 +761,14 @@ EXPORT_SYMBOL_GPL(dma_free_pages);
 int dma_mmap_pages(struct device *dev, struct vm_area_struct *vma,
 		size_t size, struct page *page)
 {
-	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
+	const pgoff_t pgoff_start = vma_start_pgoff(vma);
+	const pgoff_t pgoff_end = vma_end_pgoff(vma);
+	const unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 
-	if (vma->vm_pgoff >= count || vma_pages(vma) > count - vma->vm_pgoff)
+	if (pgoff_start >= count || pgoff_end > count)
 		return -ENXIO;
 	return remap_pfn_range(vma, vma->vm_start,
-			       page_to_pfn(page) + vma->vm_pgoff,
+			       page_to_pfn(page) + pgoff_start,
 			       vma_pages(vma) << PAGE_SHIFT, vma->vm_page_prot);
 }
 EXPORT_SYMBOL_GPL(dma_mmap_pages);
diff --git a/kernel/dma/ops_helpers.c b/kernel/dma/ops_helpers.c
index 20caf9cabf699..6b5f9208d31c0 100644
--- a/kernel/dma/ops_helpers.c
+++ b/kernel/dma/ops_helpers.c
@@ -39,7 +39,7 @@ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
 #ifdef CONFIG_MMU
 	unsigned long user_count = vma_pages(vma);
 	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
-	unsigned long off = vma->vm_pgoff;
+	unsigned long off = vma_start_pgoff(vma);
 	struct page *page = dma_common_vaddr_to_page(cpu_addr);
 	int ret = -ENXIO;
 
@@ -52,7 +52,7 @@ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
 		return -ENXIO;
 
 	return remap_pfn_range(vma, vma->vm_start,
-			page_to_pfn(page) + vma->vm_pgoff,
+			page_to_pfn(page) + vma_start_pgoff(vma),
 			user_count << PAGE_SHIFT, vma->vm_page_prot);
 #else
 	return -ENXIO;
diff --git a/kernel/entry/Makefile b/kernel/entry/Makefile
index 2333d70802e44..f220bae86b123 100644
--- a/kernel/entry/Makefile
+++ b/kernel/entry/Makefile
@@ -13,5 +13,6 @@ CFLAGS_REMOVE_common.o	 = -fstack-protector -fstack-protector-strong
 CFLAGS_common.o		+= -fno-stack-protector
 
 obj-$(CONFIG_GENERIC_IRQ_ENTRY) 	+= common.o
-obj-$(CONFIG_GENERIC_SYSCALL) 		+= syscall-common.o syscall_user_dispatch.o
+obj-$(CONFIG_GENERIC_SYSCALL) 		+= syscall-common.o
+obj-$(CONFIG_SYSCALL_USER_DISPATCH)	+= syscall_user_dispatch.o
 obj-$(CONFIG_VIRT_XFER_TO_GUEST_WORK)	+= virt.o
diff --git a/kernel/entry/syscall-common.c b/kernel/entry/syscall-common.c
index cd4967a9c53ee..b8eac9efb6fdd 100644
--- a/kernel/entry/syscall-common.c
+++ b/kernel/entry/syscall-common.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 
+#include <linux/audit.h>
 #include <linux/entry-common.h>
 
 #define CREATE_TRACE_POINTS
@@ -7,17 +8,23 @@
 
 /* Out of line to prevent tracepoint code duplication */
 
-long trace_syscall_enter(struct pt_regs *regs, long syscall)
+void trace_syscall_enter(struct pt_regs *regs)
 {
-	trace_sys_enter(regs, syscall);
-	/*
-	 * Probes or BPF hooks in the tracepoint may have changed the
-	 * system call number. Reread it.
-	 */
-	return syscall_get_nr(current, regs);
+	trace_sys_enter(regs, syscall_get_nr(current, regs));
 }
 
 void trace_syscall_exit(struct pt_regs *regs, long ret)
 {
 	trace_sys_exit(regs, ret);
 }
+
+#ifdef CONFIG_AUDITSYSCALL
+void syscall_enter_audit(struct pt_regs *regs)
+{
+	long syscall = syscall_get_nr(current, regs);
+	unsigned long args[6];
+
+	syscall_get_arguments(current, regs, args);
+	__audit_syscall_entry(syscall, args[0], args[1], args[2], args[3]);
+}
+#endif
diff --git a/kernel/entry/syscall_user_dispatch.c b/kernel/entry/syscall_user_dispatch.c
index d89dffcc2d642..2002c7aae4358 100644
--- a/kernel/entry/syscall_user_dispatch.c
+++ b/kernel/entry/syscall_user_dispatch.c
@@ -2,21 +2,22 @@
 /*
  * Copyright (C) 2020 Collabora Ltd.
  */
-
+#include <linux/elf.h>
 #include <linux/entry-common.h>
-#include <linux/sched.h>
 #include <linux/prctl.h>
 #include <linux/ptrace.h>
-#include <linux/syscall_user_dispatch.h>
-#include <linux/uaccess.h>
-#include <linux/signal.h>
-#include <linux/elf.h>
-
+#include <linux/sched.h>
 #include <linux/sched/signal.h>
 #include <linux/sched/task_stack.h>
+#include <linux/signal.h>
+#include <linux/syscall_user_dispatch.h>
+#include <linux/sysctl.h>
+#include <linux/uaccess.h>
 
 #include <asm/syscall.h>
 
+static bool syscall_user_dispatch_allowed __read_mostly = true;
+
 static void trigger_sigsys(struct pt_regs *regs)
 {
 	struct kernel_siginfo info;
@@ -102,6 +103,10 @@ static int task_set_syscall_user_dispatch(struct task_struct *task, unsigned lon
 		return -EINVAL;
 	}
 
+	/* Arming can be denied at runtime via sysctl, disarming is allowed */
+	if (mode != PR_SYS_DISPATCH_OFF && !syscall_user_dispatch_allowed)
+		return -EPERM;
+
 	/*
 	 * access_ok() will clear memory tags for tagged addresses
 	 * if current has memory tagging enabled.
@@ -172,3 +177,22 @@ int syscall_user_dispatch_set_config(struct task_struct *task, unsigned long siz
 	return task_set_syscall_user_dispatch(task, cfg.mode, cfg.offset, cfg.len,
 					      (char __user *)(uintptr_t)cfg.selector);
 }
+
+#ifdef CONFIG_PROC_SYSCTL
+static const struct ctl_table syscall_user_dispatch_sysctls[] = {
+	{
+		.procname	= "syscall_user_dispatch",
+		.data		= &syscall_user_dispatch_allowed,
+		.maxlen		= sizeof(syscall_user_dispatch_allowed),
+		.mode		= 0644,
+		.proc_handler	= proc_dobool,
+	},
+};
+
+static int __init syscall_user_dispatch_sysctl_init(void)
+{
+	register_sysctl_init("kernel", syscall_user_dispatch_sysctls);
+	return 0;
+}
+late_initcall(syscall_user_dispatch_sysctl_init);
+#endif /* CONFIG_PROC_SYSCTL */
diff --git a/kernel/events/core.c b/kernel/events/core.c
index ba5bd6a78fe7b..51c1200ea3fdd 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -6998,7 +6998,7 @@ static void perf_mmap_open(struct vm_area_struct *vma)
 	refcount_inc(&event->mmap_count);
 	refcount_inc(&event->rb->mmap_count);
 
-	if (vma->vm_pgoff)
+	if (vma_start_pgoff(vma))
 		refcount_inc(&event->rb->aux_mmap_count);
 
 	if (mapped)
@@ -7032,7 +7032,7 @@ static void perf_mmap_close(struct vm_area_struct *vma)
 	 * The AUX buffer is strictly a sub-buffer, serialize using aux_mutex
 	 * to avoid complications.
 	 */
-	if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff &&
+	if (rb_has_aux(rb) && vma_start_pgoff(vma) == rb->aux_pgoff &&
 	    refcount_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) {
 		/*
 		 * Stop all AUX events that are writing to this buffer,
@@ -7192,7 +7192,8 @@ static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma)
 	 */
 	for (pagenum = 0; pagenum < nr_pages; pagenum++) {
 		unsigned long va = vma->vm_start + PAGE_SIZE * pagenum;
-		struct page *page = perf_mmap_to_page(rb, vma->vm_pgoff + pagenum);
+		struct page *page = perf_mmap_to_page(rb,
+				vma_start_pgoff(vma) + pagenum);
 
 		if (page == NULL) {
 			err = -EINVAL;
@@ -7346,6 +7347,7 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event,
 static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event,
 			 unsigned long nr_pages)
 {
+	const pgoff_t pgoff_start = vma_start_pgoff(vma);
 	long extra = 0, user_extra = nr_pages;
 	u64 aux_offset, aux_size;
 	struct perf_buffer *rb;
@@ -7368,11 +7370,11 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event,
 	if (aux_offset < perf_data_size(rb) + PAGE_SIZE)
 		return -EINVAL;
 
-	if (aux_offset != vma->vm_pgoff << PAGE_SHIFT)
+	if (aux_offset != pgoff_start << PAGE_SHIFT)
 		return -EINVAL;
 
 	/* already mapped with a different offset */
-	if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff)
+	if (rb_has_aux(rb) && rb->aux_pgoff != pgoff_start)
 		return -EINVAL;
 
 	if (aux_size != nr_pages * PAGE_SIZE)
@@ -7402,7 +7404,7 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event,
 		if (vma->vm_flags & VM_WRITE)
 			rb_flags |= RING_BUFFER_WRITABLE;
 
-		ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages,
+		ret = rb_alloc_aux(rb, event, pgoff_start, nr_pages,
 				   event->attr.aux_watermark, rb_flags);
 		if (ret) {
 			refcount_dec(&rb->mmap_count);
@@ -7459,7 +7461,7 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma)
 		if (event->state <= PERF_EVENT_STATE_REVOKED)
 			return -ENODEV;
 
-		if (vma->vm_pgoff == 0)
+		if (!vma_start_pgoff(vma))
 			ret = perf_mmap_rb(vma, event, nr_pages);
 		else
 			ret = perf_mmap_aux(vma, event, nr_pages);
@@ -7793,10 +7795,20 @@ unsigned long perf_misc_flags(struct perf_event *event,
 unsigned long perf_instruction_pointer(struct perf_event *event,
 				       struct pt_regs *regs)
 {
-	if (should_sample_guest(event))
-		return perf_guest_get_ip();
+	/*
+	 * Hardware skid can lead to a scenario where a PMI is
+	 * delivered after the CPU has already entered kernel mode.
+	 * In that case, user-space sampling must not expose kernel
+	 * register state.
+	 */
+	if (should_sample_guest(event)) {
+		return event->attr.exclude_kernel &&
+		       !(perf_guest_state() & PERF_GUEST_USER) ?
+			0 : perf_guest_get_ip();
+	}
 
-	return perf_arch_instruction_pointer(regs);
+	return event->attr.exclude_kernel && !user_mode(regs) ?
+		0 : perf_arch_instruction_pointer(regs);
 }
 
 static void
@@ -7830,10 +7842,22 @@ static void perf_sample_regs_user(struct perf_regs *regs_user,
 }
 
 static void perf_sample_regs_intr(struct perf_regs *regs_intr,
-				  struct pt_regs *regs)
+				  struct pt_regs *regs,
+				  bool exclude_kernel)
 {
-	regs_intr->regs = regs;
-	regs_intr->abi  = perf_reg_abi(current);
+	/*
+	 * Hardware skid can lead to a scenario where a PMI is
+	 * delivered after the CPU has already entered kernel mode.
+	 * In that case, user-space sampling must not expose kernel
+	 * register state.
+	 */
+	if (exclude_kernel && !user_mode(regs)) {
+		regs_intr->abi = PERF_SAMPLE_REGS_ABI_NONE;
+		regs_intr->regs = NULL;
+	} else {
+		regs_intr->regs = regs;
+		regs_intr->abi = perf_reg_abi(current);
+	}
 }
 
 
@@ -8724,7 +8748,8 @@ void perf_prepare_sample(struct perf_sample_data *data,
 		/* regs dump ABI info */
 		int size = sizeof(u64);
 
-		perf_sample_regs_intr(&data->regs_intr, regs);
+		perf_sample_regs_intr(&data->regs_intr, regs,
+				      event->attr.exclude_kernel);
 
 		if (data->regs_intr.regs) {
 			u64 mask = event->attr.sample_regs_intr;
@@ -9886,7 +9911,7 @@ static bool perf_addr_filter_vma_adjust(struct perf_addr_filter *filter,
 					struct perf_addr_filter_range *fr)
 {
 	unsigned long vma_size = vma->vm_end - vma->vm_start;
-	unsigned long off = vma->vm_pgoff << PAGE_SHIFT;
+	unsigned long off = vma_start_pgoff(vma) << PAGE_SHIFT;
 	struct file *file = vma->vm_file;
 
 	if (!perf_addr_filter_match(filter, file, off, vma_size))
@@ -9976,7 +10001,7 @@ void perf_event_mmap(struct vm_area_struct *vma)
 			/* .tid */
 			.start  = vma->vm_start,
 			.len    = vma->vm_end - vma->vm_start,
-			.pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
+			.pgoff  = (u64)vma_start_pgoff(vma) << PAGE_SHIFT,
 		},
 		/* .maj (attr_mmap2 only) */
 		/* .min (attr_mmap2 only) */
@@ -12696,7 +12721,7 @@ static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
 	struct cpumask *mask = perf_scope_cpumask(pmu->scope);
 
 	if (mask)
-		return cpumap_print_to_pagebuf(true, buf, mask);
+		return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(mask));
 	return 0;
 }
 
@@ -13911,7 +13936,9 @@ SYSCALL_DEFINE5(perf_event_open,
 	if (err)
 		return err;
 
-	if (!attr.exclude_kernel) {
+	if (!attr.exclude_kernel ||
+	    ((attr.sample_type & PERF_SAMPLE_CALLCHAIN) &&
+	     !attr.exclude_callchain_kernel)) {
 		err = perf_allow_kernel();
 		if (err)
 			return err;
@@ -14771,6 +14798,24 @@ int perf_allow_kernel(void)
 }
 EXPORT_SYMBOL_GPL(perf_allow_kernel);
 
+int perf_allow_cpu(void)
+{
+	if (sysctl_perf_event_paranoid > 0 && !perfmon_capable())
+		return -EACCES;
+
+	return security_perf_event_open(PERF_SECURITY_CPU);
+}
+EXPORT_SYMBOL_GPL(perf_allow_cpu);
+
+int perf_allow_tracepoint(void)
+{
+	if (sysctl_perf_event_paranoid > -1 && !perfmon_capable())
+		return -EPERM;
+
+	return security_perf_event_open(PERF_SECURITY_TRACEPOINT);
+}
+EXPORT_SYMBOL_GPL(perf_allow_tracepoint);
+
 /*
  * Inherit an event from parent task to child task.
  *
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 4084e926e2844..07f69dd3093d5 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -144,12 +144,14 @@ static bool valid_vma(struct vm_area_struct *vma, bool is_register)
 
 static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset)
 {
-	return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
+	return vma->vm_start + offset -
+		((loff_t)vma_start_pgoff(vma) << PAGE_SHIFT);
 }
 
 static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr)
 {
-	return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start);
+	return ((loff_t)vma_start_pgoff(vma) << PAGE_SHIFT) +
+		(vaddr - vma->vm_start);
 }
 
 /**
@@ -1210,7 +1212,7 @@ build_map_info(struct address_space *mapping, loff_t offset, bool is_register)
 
  again:
 	i_mmap_lock_read(mapping);
-	vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
+	mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) {
 		if (!valid_vma(vma, is_register))
 			continue;
 
@@ -1482,7 +1484,7 @@ static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm)
 		    file_inode(vma->vm_file) != uprobe->inode)
 			continue;
 
-		offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT;
+		offset = (loff_t)vma_start_pgoff(vma) << PAGE_SHIFT;
 		if (uprobe->offset <  offset ||
 		    uprobe->offset >= offset + vma->vm_end - vma->vm_start)
 			continue;
@@ -1806,14 +1808,6 @@ static struct xol_area *get_xol_area(void)
 	return area;
 }
 
-void __weak arch_uprobe_clear_state(struct mm_struct *mm)
-{
-}
-
-void __weak arch_uprobe_init_state(struct mm_struct *mm)
-{
-}
-
 /*
  * uprobe_clear_state - Free the area allocated for slots.
  */
@@ -1825,8 +1819,6 @@ void uprobe_clear_state(struct mm_struct *mm)
 	delayed_uprobe_remove(NULL, mm);
 	mutex_unlock(&delayed_uprobe_lock);
 
-	arch_uprobe_clear_state(mm);
-
 	if (!area)
 		return;
 
@@ -2453,7 +2445,8 @@ static struct uprobe *find_active_uprobe_speculative(unsigned long bp_vaddr)
 	if (!vm_file)
 		return NULL;
 
-	offset = (loff_t)(vma->vm_pgoff << PAGE_SHIFT) + (bp_vaddr - vma->vm_start);
+	offset = (loff_t)(vma_start_pgoff(vma) << PAGE_SHIFT) +
+		(bp_vaddr - vma->vm_start);
 	uprobe = find_uprobe_rcu(vm_file->f_inode, offset);
 	if (!uprobe)
 		return NULL;
diff --git a/kernel/exit.c b/kernel/exit.c
index 2c0b1c02920f4..f812df279ea75 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -1116,7 +1116,7 @@ void __noreturn make_task_dead(int signr)
 
 SYSCALL_DEFINE1(exit, int, error_code)
 {
-	do_exit((error_code&0xff)<<8);
+	do_exit((error_code & 0xff) << 8);
 }
 
 /*
diff --git a/kernel/fork.c b/kernel/fork.c
index f0e2e131a9a5a..f3f378a85be4c 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -143,7 +143,7 @@
 unsigned long total_forks;	/* Handle normal Linux uptimes. */
 int nr_threads;			/* The idle threads do not count.. */
 
-static int max_threads;		/* tunable limit on nr_threads */
+static int max_threads __read_mostly;		/* tunable limit on nr_threads */
 
 #define NAMED_ARRAY_INDEX(x)	[x] = __stringify(x)
 
@@ -1069,7 +1069,6 @@ static void mm_init_uprobes_state(struct mm_struct *mm)
 {
 #ifdef CONFIG_UPROBES
 	mm->uprobes_state.xol_area = NULL;
-	arch_uprobe_init_state(mm);
 #endif
 }
 
@@ -1618,9 +1617,27 @@ static int copy_exec_state(u64 clone_flags, struct task_struct *tsk)
 	return task_exec_state_copy(tsk);
 }
 
-static int copy_fs(u64 clone_flags, struct task_struct *tsk)
+static int copy_fs(u64 clone_flags, struct task_struct *tsk, bool umh)
 {
-	struct fs_struct *fs = current->fs;
+	struct fs_struct *fs;
+
+	/*
+	 * Usermodehelper may copy userspace_init_fs filesystem state but
+	 * they don't get to create mount namespaces, share the
+	 * filesystem state, or be started from a non-initial mount
+	 * namespace.
+	 */
+	if (umh) {
+		if (clone_flags & (CLONE_NEWNS | CLONE_FS))
+			return -EINVAL;
+		if (current->nsproxy->mnt_ns != &init_mnt_ns)
+			return -EINVAL;
+		fs = userspace_init_fs;
+	} else {
+		fs = current->fs;
+		VFS_WARN_ON_ONCE(current->fs != current->real_fs);
+	}
+
 	if (clone_flags & CLONE_FS) {
 		/* tsk->fs is already what we want */
 		read_seqlock_excl(&fs->seq);
@@ -1633,7 +1650,7 @@ static int copy_fs(u64 clone_flags, struct task_struct *tsk)
 		read_sequnlock_excl(&fs->seq);
 		return 0;
 	}
-	tsk->fs = copy_fs_struct(fs);
+	tsk->real_fs = tsk->fs = copy_fs_struct(fs);
 	if (!tsk->fs)
 		return -ENOMEM;
 	return 0;
@@ -2277,7 +2294,7 @@ __latent_entropy struct task_struct *copy_process(
 	retval = copy_files(clone_flags, p, args->no_files);
 	if (retval)
 		goto bad_fork_cleanup_semundo;
-	retval = copy_fs(clone_flags, p);
+	retval = copy_fs(clone_flags, p, args->umh);
 	if (retval)
 		goto bad_fork_cleanup_files;
 	retval = copy_sighand(clone_flags, p);
@@ -2819,6 +2836,7 @@ pid_t user_mode_thread(int (*fn)(void *), void *arg, unsigned long flags)
 		.exit_signal	= (flags & CSIGNAL),
 		.fn		= fn,
 		.fn_arg		= arg,
+		.umh		= 1,
 	};
 
 	return kernel_clone(&args);
@@ -3216,7 +3234,7 @@ static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp
  */
 int ksys_unshare(unsigned long unshare_flags)
 {
-	struct fs_struct *fs, *new_fs = NULL;
+	struct fs_struct *new_fs = NULL;
 	struct files_struct *new_fd = NULL;
 	struct cred *new_cred = NULL;
 	struct nsproxy *new_nsproxy = NULL;
@@ -3247,6 +3265,10 @@ int ksys_unshare(unsigned long unshare_flags)
 	if (unshare_flags & CLONE_NEWNS)
 		unshare_flags |= CLONE_FS;
 
+	/* No unsharing with overriden fs state */
+	VFS_WARN_ON_ONCE(unshare_flags & (CLONE_NEWNS | CLONE_FS) &&
+			 current->fs != current->real_fs);
+
 	err = check_unshare_flags(unshare_flags);
 	if (err)
 		goto bad_unshare_out;
@@ -3294,23 +3316,13 @@ int ksys_unshare(unsigned long unshare_flags)
 			new_nsproxy = NULL;
 		}
 
-		task_lock(current);
+		if (new_fs)
+			new_fs = switch_fs_struct(new_fs);
 
-		if (new_fs) {
-			fs = current->fs;
-			read_seqlock_excl(&fs->seq);
-			current->fs = new_fs;
-			if (--fs->users)
-				new_fs = NULL;
-			else
-				new_fs = fs;
-			read_sequnlock_excl(&fs->seq);
-		}
-
-		if (new_fd)
+		if (new_fd) {
+			guard(task_lock)(current);
 			swap(current->files, new_fd);
-
-		task_unlock(current);
+		}
 
 		if (new_cred) {
 			/* Install the new user namespace */
diff --git a/kernel/futex/core.c b/kernel/futex/core.c
index 179b26e9c9341..233b926f5624e 100644
--- a/kernel/futex/core.c
+++ b/kernel/futex/core.c
@@ -1746,8 +1746,7 @@ void futex_hash_free(struct mm_struct *mm)
 	free_percpu(mm->futex.phash.ref);
 	kvfree(mm->futex.phash.hash_new);
 	fph = rcu_dereference_raw(mm->futex.phash.hash);
-	if (fph)
-		kvfree(fph);
+	kvfree(fph);
 }
 
 static bool futex_pivot_pending(struct mm_struct *mm)
diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c
index 80ef4e27dcf47..3a818f07a1013 100644
--- a/kernel/irq/irqdesc.c
+++ b/kernel/irq/irqdesc.c
@@ -1004,7 +1004,7 @@ unsigned int kstat_irqs_cpu(unsigned int irq, int cpu)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 
-	return desc && desc->kstat_irqs ? per_cpu(desc->kstat_irqs->cnt, cpu) : 0;
+	return desc ? irq_desc_kstat_cpu(desc, cpu) : 0;
 }
 
 static unsigned int kstat_irqs_desc(struct irq_desc *desc, const struct cpumask *cpumask)
@@ -1026,7 +1026,7 @@ static unsigned int kstat_irqs(unsigned int irq)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 
-	if (!desc || !desc->kstat_irqs)
+	if (!desc)
 		return 0;
 	return kstat_irqs_desc(desc, cpu_possible_mask);
 }
@@ -1038,18 +1038,15 @@ void kstat_snapshot_irqs(void)
 	struct irq_desc *desc;
 	unsigned int irq;
 
-	for_each_irq_desc(irq, desc) {
-		if (!desc->kstat_irqs)
-			continue;
+	for_each_irq_desc(irq, desc)
 		this_cpu_write(desc->kstat_irqs->ref, this_cpu_read(desc->kstat_irqs->cnt));
-	}
 }
 
 unsigned int kstat_get_irq_since_snapshot(unsigned int irq)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 
-	if (!desc || !desc->kstat_irqs)
+	if (!desc)
 		return 0;
 	return this_cpu_read(desc->kstat_irqs->cnt) - this_cpu_read(desc->kstat_irqs->ref);
 }
diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c
index f15c9f1223bb3..57c819da30c21 100644
--- a/kernel/irq/irqdomain.c
+++ b/kernel/irq/irqdomain.c
@@ -1658,7 +1658,7 @@ static int irq_domain_alloc_irqs_locked(struct irq_domain *domain, int irq_base,
 	for (i = 0; i < nr_irqs; i++) {
 		ret = irq_domain_trim_hierarchy(virq + i);
 		if (ret)
-			goto out_free_irq_data;
+			goto out_free_irqs;
 	}
 
 	for (i = 0; i < nr_irqs; i++)
@@ -1666,6 +1666,8 @@ static int irq_domain_alloc_irqs_locked(struct irq_domain *domain, int irq_base,
 
 	return virq;
 
+out_free_irqs:
+	irq_domain_free_irqs_hierarchy(domain, virq, nr_irqs);
 out_free_irq_data:
 	irq_domain_free_irq_data(virq, nr_irqs);
 out_free_desc:
@@ -1963,7 +1965,7 @@ EXPORT_SYMBOL_GPL(irq_domain_free_irqs_parent);
 
 static void __irq_domain_deactivate_irq(struct irq_data *irq_data)
 {
-	if (irq_data && irq_data->domain) {
+	if (irq_data->domain) {
 		struct irq_domain *domain = irq_data->domain;
 
 		if (domain->ops->deactivate)
@@ -1977,7 +1979,7 @@ static int __irq_domain_activate_irq(struct irq_data *irqd, bool reserve)
 {
 	int ret = 0;
 
-	if (irqd && irqd->domain) {
+	if (irqd->domain) {
 		struct irq_domain *domain = irqd->domain;
 
 		if (irqd->parent_data)
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index 7eb07e3bdb4c2..2fbff2618a1e2 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -171,6 +171,7 @@ int irq_can_set_affinity(unsigned int irq)
 {
 	return __irq_can_set_affinity(irq_to_desc(irq));
 }
+EXPORT_SYMBOL_GPL(irq_can_set_affinity);
 
 /**
  * irq_can_set_affinity_usr - Check if affinity of a irq can be set from user space
@@ -2703,11 +2704,7 @@ static int __irq_get_irqchip_state(struct irq_data *data, enum irqchip_irq_state
 			return -ENODEV;
 		if (chip->irq_get_irqchip_state)
 			break;
-#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
-		data = data->parent_data;
-#else
-		data = NULL;
-#endif
+		data = irqd_get_parent_data(data);
 	} while (data);
 
 	if (data)
diff --git a/kernel/kcmp.c b/kernel/kcmp.c
index 7c1a65bd5f8d3..76476aeee067b 100644
--- a/kernel/kcmp.c
+++ b/kernel/kcmp.c
@@ -186,7 +186,7 @@ SYSCALL_DEFINE5(kcmp, pid_t, pid1, pid_t, pid2, int, type,
 		ret = kcmp_ptr(task1->files, task2->files, KCMP_FILES);
 		break;
 	case KCMP_FS:
-		ret = kcmp_ptr(task1->fs, task2->fs, KCMP_FS);
+		ret = kcmp_ptr(task1->real_fs, task2->real_fs, KCMP_FS);
 		break;
 	case KCMP_SIGHAND:
 		ret = kcmp_ptr(task1->sighand, task2->sighand, KCMP_SIGHAND);
diff --git a/kernel/kcov.c b/kernel/kcov.c
index 1df373fb562bc..b19b473c366a0 100644
--- a/kernel/kcov.c
+++ b/kernel/kcov.c
@@ -512,7 +512,7 @@ static int kcov_mmap(struct file *filep, struct vm_area_struct *vma)
 
 	spin_lock_irqsave(&kcov->lock, flags);
 	size = kcov->size * sizeof(unsigned long);
-	if (kcov->area == NULL || vma->vm_pgoff != 0 ||
+	if (kcov->area == NULL || vma_start_pgoff(vma) ||
 	    vma->vm_end - vma->vm_start != size) {
 		res = -EINVAL;
 		goto exit;
diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c
index 2bfbb2d144e69..59fb9d71e9d86 100644
--- a/kernel/kexec_file.c
+++ b/kernel/kexec_file.c
@@ -27,6 +27,7 @@
 #include <linux/syscalls.h>
 #include <linux/vmalloc.h>
 #include <linux/dma-map-ops.h>
+#include <linux/kexec_handover.h>
 #include "kexec_internal.h"
 
 #ifdef CONFIG_KEXEC_SIG
@@ -798,6 +799,16 @@ int kexec_add_buffer(struct kexec_buf *kbuf)
 	return 0;
 }
 
+static bool kexec_only_cma_segments(struct kimage *image)
+{
+	for (int i = 0; i < image->nr_segments; i++) {
+		if (!image->segment_cma[i])
+			return false;
+	}
+
+	return true;
+}
+
 /* Calculate and store the digest of segments */
 static int kexec_calculate_store_digests(struct kimage *image)
 {
@@ -822,6 +833,21 @@ static int kexec_calculate_store_digests(struct kimage *image)
 
 	sha256_init(&sctx);
 
+	/*
+	 * If KHO is enabled, the destinations are located in KHO scratch.
+	 * KHO scratch can only contain early boot allocations and movable
+	 * allocations. That means there is no risk of memory corruption by
+	 * uncancelled DMA.
+	 *
+	 * If all segments were loaded into contiguous memory, there will be no
+	 * relocations at all, so also no risk of corruption.
+	 */
+	if (image->type != KEXEC_TYPE_CRASH &&
+	    (kho_is_enabled() || kexec_only_cma_segments(image))) {
+		pr_debug("disabling checksum verification in purgatory\n");
+		goto skip_checksum;
+	}
+
 	for (j = i = 0; i < image->nr_segments; i++) {
 		struct kexec_segment *ksegment;
 
@@ -867,6 +893,7 @@ static int kexec_calculate_store_digests(struct kimage *image)
 		j++;
 	}
 
+skip_checksum:
 	sha256_final(&sctx, digest);
 
 	ret = kexec_purgatory_get_set_symbol(image, "purgatory_sha_regions",
diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c
index 4834a809985ab..78ff267bf51e5 100644
--- a/kernel/liveupdate/kexec_handover.c
+++ b/kernel/liveupdate/kexec_handover.c
@@ -34,7 +34,8 @@
  * KHO is tightly coupled with mm init and needs access to some of mm
  * internal APIs.
  */
-#include "../../mm/internal.h"
+#include "../../mm/mm_init.h"
+#include "../../mm/vmalloc.h"
 #include "../kexec_internal.h"
 #include "kexec_handover_internal.h"
 
diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c
index b79b2a4889741..f48e9a4185f97 100644
--- a/kernel/liveupdate/luo_session.c
+++ b/kernel/liveupdate/luo_session.c
@@ -154,7 +154,6 @@ static struct luo_session *luo_session_alloc(const char *name)
 		return ERR_PTR(-ENOMEM);
 
 	strscpy(session->name, name, sizeof(session->name));
-	INIT_LIST_HEAD(&session->file_set.files_list);
 	luo_file_set_init(&session->file_set);
 	INIT_LIST_HEAD(&session->list);
 	mutex_init(&session->mutex);
@@ -316,8 +315,12 @@ static int luo_session_finish(struct luo_session *session,
 			      struct luo_ucmd *ucmd)
 {
 	struct liveupdate_session_finish *argp = ucmd->cmd;
-	int err = luo_session_finish_one(session);
+	int err;
+
+	if (argp->reserved)
+		return -EINVAL;
 
+	err = luo_session_finish_one(session);
 	if (err)
 		return err;
 
@@ -378,7 +381,7 @@ static const struct luo_ioctl_op luo_session_ioctl_ops[] = {
 	IOCTL_OP(LIVEUPDATE_SESSION_RETRIEVE_FD, luo_session_retrieve_fd,
 		 struct liveupdate_session_retrieve_fd, token, LUO_IOCTL_INCOMING),
 	IOCTL_OP(LIVEUPDATE_SESSION_GET_NAME, luo_session_get_name,
-		 struct liveupdate_session_retrieve_fd, token, LUO_IOCTL_ALL),
+		 struct liveupdate_session_get_name, name, LUO_IOCTL_ALL),
 };
 
 static bool luo_ioctl_type_valid(struct luo_session *session,
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index 2d4c5bab5af88..6dd11ffbd6c39 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -787,17 +787,33 @@ static void lockdep_print_held_locks(struct task_struct *p)
 {
 	int i, depth = READ_ONCE(p->lockdep_depth);
 
-	if (!depth)
-		printk("no locks held by %s/%d.\n", p->comm, task_pid_nr(p));
-	else
-		printk("%d lock%s held by %s/%d:\n", depth,
-		       str_plural(depth), p->comm, task_pid_nr(p));
 	/*
-	 * It's not reliable to print a task's held locks if it's not sleeping
-	 * and it's not the current task.
+	 * Note that it's always somewhat unreliable to print held locks
+	 * of a task that is running on another CPU, but we cannot guarantee
+	 * the stability of ->held_locks without actually stopping all active
+	 * remote CPUs, which we absolutely do not want to do because it's
+	 * very intrusive and thus slow.
+	 *
+	 * So we do the next best thing here: we print out the held lock
+	 * array on a best-effort basis, without crashing even if the
+	 * fields are being modified on another CPU. Note the careful
+	 * construction of print_lock() so that it never crashes.
+	 *
+	 * We also print out the CPU the task is or was last running on, with
+	 * the message saying 'on CPU...' if the task is running, and
+	 * 'last CPU' if it's not.
+	 *
+	 * Also note that the task_is_running(p) information is fundamentally
+	 * racy: even if the message says the task is 'on CPU', the task may
+	 * have scheduled out already, or if it says 'last CPU', it may just
+	 * have scheduled in on another CPU. But even with these limitations
+	 * it's still useful debuggining information.
 	 */
-	if (p != current && task_is_running(p))
-		return;
+	printk("locks held by %s/%d: %d, %s CPU#%d%s\n",
+		p->comm, task_pid_nr(p), depth,
+		task_is_running(p) ? "last" : "on", task_cpu(p),
+		depth > 0 ? ":" : "");
+
 	for (i = 0; i < depth; i++) {
 		printk(" #%d: ", i);
 		print_lock(p->held_locks + i);
@@ -5077,7 +5093,7 @@ static int __lock_is_held(const struct lockdep_map *lock, int read);
 static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
 			  int trylock, int read, int check, int hardirqs_off,
 			  struct lockdep_map *nest_lock, unsigned long ip,
-			  int references, int pin_count, int sync)
+			  int references, int pin_count, int sync, int seq)
 {
 	struct task_struct *curr = current;
 	struct lock_class *class = NULL;
@@ -5183,6 +5199,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
 	hlock->holdtime_stamp = lockstat_clock();
 #endif
 	hlock->pin_count = pin_count;
+	hlock->seq_count = seq;
 
 	if (check_wait_context(curr, hlock))
 		return 0;
@@ -5388,7 +5405,7 @@ static int reacquire_held_locks(struct task_struct *curr, unsigned int depth,
 				    hlock->read, hlock->check,
 				    hlock->hardirqs_off,
 				    hlock->nest_lock, hlock->acquire_ip,
-				    hlock->references, hlock->pin_count, 0)) {
+				    hlock->references, hlock->pin_count, 0, hlock->seq_count)) {
 		case 0:
 			return 1;
 		case 1:
@@ -5669,14 +5686,17 @@ static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie
 		struct held_lock *hlock = curr->held_locks + i;
 
 		if (match_held_lock(hlock, lock)) {
+			int pin_count;
+
 			if (WARN(!hlock->pin_count, "unpinning an unpinned lock\n"))
 				return;
 
-			hlock->pin_count -= cookie.val;
+			pin_count = hlock->pin_count - cookie.val;
 
-			if (WARN((int)hlock->pin_count < 0, "pin count corrupted\n"))
-				hlock->pin_count = 0;
+			if (WARN(pin_count < 0, "pin count corrupted\n"))
+				pin_count = 0;
 
+			hlock->pin_count = pin_count;
 			return;
 		}
 	}
@@ -5684,6 +5704,24 @@ static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie
 	WARN(1, "unpinning an unheld lock\n");
 }
 
+static u32 __lock_sequence(struct lockdep_map *lock)
+{
+	struct task_struct *curr = current;
+	int i;
+
+	if (unlikely(!debug_locks))
+		return ~0;
+
+	for (i = 0; i < curr->lockdep_depth; i++) {
+		struct held_lock *hlock = curr->held_locks + i;
+
+		if (match_held_lock(hlock, lock))
+			return hlock->seq_count;
+	}
+
+	return ~0;
+}
+
 /*
  * Check whether we follow the irq-flags state precisely:
  */
@@ -5866,7 +5904,8 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
 
 	lockdep_recursion_inc();
 	__lock_acquire(lock, subclass, trylock, read, check,
-		       irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 0);
+		       irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 0,
+		       ++current->lockdep_seq);
 	lockdep_recursion_finish();
 	raw_local_irq_restore(flags);
 }
@@ -5914,7 +5953,8 @@ void lock_sync(struct lockdep_map *lock, unsigned subclass, int read,
 
 	lockdep_recursion_inc();
 	__lock_acquire(lock, subclass, 0, read, check,
-		       irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 1);
+		       irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 1,
+		       ++current->lockdep_seq);
 	check_chain_key(current);
 	lockdep_recursion_finish();
 	raw_local_irq_restore(flags);
@@ -6000,6 +6040,26 @@ void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
 }
 EXPORT_SYMBOL_GPL(lock_unpin_lock);
 
+u32 lock_sequence(struct lockdep_map *lock)
+{
+	unsigned long flags;
+	u32 seq = ~0;
+
+	if (unlikely(!lockdep_enabled()))
+		return seq;
+
+	raw_local_irq_save(flags);
+	check_flags(flags);
+
+	lockdep_recursion_inc();
+	seq = __lock_sequence(lock);
+	lockdep_recursion_finish();
+	raw_local_irq_restore(flags);
+
+	return seq;
+}
+EXPORT_SYMBOL_GPL(lock_sequence);
+
 #ifdef CONFIG_LOCK_STAT
 static void print_lock_contention_bug(struct task_struct *curr,
 				      struct lockdep_map *lock,
diff --git a/kernel/module/dups.c b/kernel/module/dups.c
index 1d720a5311ba0..8b6f8f177de4f 100644
--- a/kernel/module/dups.c
+++ b/kernel/module/dups.c
@@ -129,7 +129,7 @@ bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret)
 	if (!new_kmod_req)
 		return false;
 
-	memcpy(new_kmod_req->name, module_name, strlen(module_name));
+	strscpy(new_kmod_req->name, module_name);
 	INIT_WORK(&new_kmod_req->complete_work, kmod_dup_request_complete);
 	INIT_DELAYED_WORK(&new_kmod_req->delete_work, kmod_dup_request_delete);
 	init_completion(&new_kmod_req->first_req_done);
diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c
index 0fc11e45df9b9..f23126d804b25 100644
--- a/kernel/module/kallsyms.c
+++ b/kernel/module/kallsyms.c
@@ -258,17 +258,25 @@ static const char *find_kallsyms_symbol(struct module *mod,
 	unsigned int i, best = 0;
 	unsigned long nextval, bestval;
 	struct mod_kallsyms *kallsyms = rcu_dereference(mod->kallsyms);
-	struct module_memory *mod_mem;
+	struct module_memory *mod_mem = NULL;
 
-	/* At worse, next value is at end of module */
-	if (within_module_init(addr, mod))
-		mod_mem = &mod->mem[MOD_INIT_TEXT];
-	else
-		mod_mem = &mod->mem[MOD_TEXT];
+	for_each_mod_mem_type(type) {
+#ifndef CONFIG_KALLSYMS_ALL
+		if (!mod_mem_type_is_text(type))
+			continue;
+#endif
+		if (within_module_mem_type(addr, mod, type)) {
+			mod_mem = &mod->mem[type];
+			break;
+		}
+	}
 
-	nextval = (unsigned long)mod_mem->base + mod_mem->size;
+	if (!mod_mem)
+		return NULL;
 
-	bestval = kallsyms_symbol_value(&kallsyms->symtab[best]);
+	/* Initialize bounds within memory region the address belongs to. */
+	nextval = (unsigned long)mod_mem->base + mod_mem->size;
+	bestval = (unsigned long)mod_mem->base - 1;
 
 	/*
 	 * Scan for closest preceding symbol, and next symbol. (ELF
diff --git a/kernel/module/main.c b/kernel/module/main.c
index 46dd8d25a6058..528690ba160bd 100644
--- a/kernel/module/main.c
+++ b/kernel/module/main.c
@@ -1458,7 +1458,6 @@ static void free_module(struct module *mod)
 
 	/* This may be empty, but that's OK */
 	module_arch_freeing_init(mod);
-	kfree(mod->args);
 	percpu_modfree(mod);
 
 	free_mod_mem(mod);
@@ -3425,7 +3424,7 @@ static int load_module(struct load_info *info, const char __user *uargs,
 	struct module *mod;
 	bool module_allocated = false;
 	long err = 0;
-	char *after_dashes;
+	char *args = NULL, *after_dashes;
 
 	/*
 	 * Do the signature check (if any) first. All that
@@ -3523,9 +3522,9 @@ static int load_module(struct load_info *info, const char __user *uargs,
 	flush_module_icache(mod);
 
 	/* Now copy in args */
-	mod->args = strndup_user(uargs, ~0UL >> 1);
-	if (IS_ERR(mod->args)) {
-		err = PTR_ERR(mod->args);
+	args = strndup_user(uargs, ~0UL >> 1);
+	if (IS_ERR(args)) {
+		err = PTR_ERR(args);
 		goto free_arch_cleanup;
 	}
 
@@ -3546,7 +3545,7 @@ static int load_module(struct load_info *info, const char __user *uargs,
 	mod->async_probe_requested = async_probe;
 
 	/* Module is ready to execute: parsing args may do that. */
-	after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp,
+	after_dashes = parse_args(mod->name, args, mod->kp, mod->num_kp,
 				  -32768, 32767, mod,
 				  unknown_module_param_cb);
 	if (IS_ERR(after_dashes)) {
@@ -3556,6 +3555,8 @@ static int load_module(struct load_info *info, const char __user *uargs,
 		pr_warn("%s: parameters '%s' after `--' ignored\n",
 		       mod->name, after_dashes);
 	}
+	kfree(args);
+	args = NULL;
 
 	/* Link in to sysfs. */
 	err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp);
@@ -3597,7 +3598,7 @@ static int load_module(struct load_info *info, const char __user *uargs,
  ddebug_cleanup:
 	ftrace_release_mod(mod);
 	synchronize_rcu();
-	kfree(mod->args);
+	kfree(args);
  free_arch_cleanup:
 	module_arch_cleanup(mod);
  free_modinfo:
diff --git a/kernel/module/procfs.c b/kernel/module/procfs.c
index 0a4841e88adb4..90712aa9dd13d 100644
--- a/kernel/module/procfs.c
+++ b/kernel/module/procfs.c
@@ -64,7 +64,7 @@ static void m_stop(struct seq_file *m, void *p)
 
 static unsigned int module_total_size(struct module *mod)
 {
-	int size = 0;
+	unsigned int size = 0;
 
 	for_each_mod_mem_type(type)
 		size += mod->mem[type].size;
diff --git a/kernel/module/stats.c b/kernel/module/stats.c
index 3a9672f93a8ef..78b59bc05387f 100644
--- a/kernel/module/stats.c
+++ b/kernel/module/stats.c
@@ -253,7 +253,7 @@ int try_add_failed_module(const char *name, enum fail_dup_mod_reason reason)
 	mod_fail = kzalloc_obj(*mod_fail);
 	if (!mod_fail)
 		return -ENOMEM;
-	memcpy(mod_fail->name, name, strlen(name));
+	strscpy(mod_fail->name, name);
 	__set_bit(reason, &mod_fail->dup_fail_mask);
 	atomic_long_inc(&mod_fail->count);
 	list_add_rcu(&mod_fail->list, &dup_failed_modules);
diff --git a/kernel/params.c b/kernel/params.c
index a668863a4bb63..d3a4ecab4f278 100644
--- a/kernel/params.c
+++ b/kernel/params.c
@@ -136,8 +136,8 @@ static int parse_one(char *param,
 			if (!val &&
 			    !(params[i].ops->flags & KERNEL_PARAM_OPS_FL_NOARG))
 				return -EINVAL;
-			pr_debug("handling %s with %p\n", param,
-				params[i].ops->set);
+			pr_debug("handling %s with value '%s'\n", param,
+				val ? val : "no-arg");
 			kernel_param_lock(params[i].mod);
 			if (param_check_unsafe(&params[i]))
 				err = params[i].ops->set(val, &params[i]);
@@ -538,7 +538,7 @@ const struct kernel_param_ops param_ops_string = {
 };
 EXPORT_SYMBOL(param_ops_string);
 
-/* sysfs output in /sys/modules/XYZ/parameters/ */
+/* sysfs output in /sys/module/XYZ/parameters/ */
 #define to_module_attr(n) container_of_const(n, struct module_attribute, attr)
 #define to_module_kobject(n) container_of(n, struct module_kobject, kobj)
 
diff --git a/kernel/power/wakelock.c b/kernel/power/wakelock.c
index fd763da06a873..a8b6bd5ec46b4 100644
--- a/kernel/power/wakelock.c
+++ b/kernel/power/wakelock.c
@@ -63,7 +63,7 @@ static unsigned int number_of_wakelocks;
 
 static inline bool wakelocks_limit_exceeded(void)
 {
-	return number_of_wakelocks > CONFIG_PM_WAKELOCKS_LIMIT;
+	return number_of_wakelocks >= CONFIG_PM_WAKELOCKS_LIMIT;
 }
 
 static inline void increment_wakelocks_number(void)
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index 2fe9a963c823a..3fcdf4b4e2e53 100644
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -3264,10 +3264,8 @@ static bool console_flush_one_record(bool do_cond_resched, u64 *next_seq, bool *
 		if (flags & CON_NBCON) {
 			progress = nbcon_legacy_emit_next_record(con, handover, cookie,
 								 !do_cond_resched);
-			printk_seq = nbcon_seq_read(con);
 		} else {
 			progress = console_emit_next_record(con, handover, cookie);
-			printk_seq = con->seq;
 		}
 
 		/*
@@ -3277,6 +3275,15 @@ static bool console_flush_one_record(bool do_cond_resched, u64 *next_seq, bool *
 		if (*handover)
 			goto fail;
 
+		/*
+		 * @con can be used here now that it is certain that this
+		 * context is still holding the SRCU read lock.
+		 */
+		if (flags & CON_NBCON)
+			printk_seq = nbcon_seq_read(con);
+		else
+			printk_seq = con->seq;
+
 		/* Track the next of the highest seq flushed. */
 		if (printk_seq > *next_seq)
 			*next_seq = printk_seq;
@@ -3890,9 +3897,6 @@ static int console_call_setup(struct console *newcon, char *options)
  * the newly registered console with any of the ones selected
  * by either the command line or add_preferred_console() and
  * setup/enable it.
- *
- * Care need to be taken with consoles that are statically
- * enabled such as netconsole
  */
 static int try_enable_preferred_console(struct console *newcon,
 					bool user_specified)
@@ -3933,14 +3937,6 @@ static int try_enable_preferred_console(struct console *newcon,
 		return 0;
 	}
 
-	/*
-	 * Some consoles, such as pstore and netconsole, can be enabled even
-	 * without matching. Accept the pre-enabled consoles only when match()
-	 * and setup() had a chance to be called.
-	 */
-	if (newcon->flags & CON_ENABLED && c->user_specified ==	user_specified)
-		return 0;
-
 	return -ENOENT;
 }
 
@@ -4123,6 +4119,14 @@ void register_console(struct console *newcon)
 	if (err == -ENOENT)
 		err = try_enable_preferred_console(newcon, false);
 
+	/*
+	 * Some consoles, such as pstore and netconsole, can be enabled even
+	 * without matching. Accept them at this stage when they had a chance
+	 * to match() and call setup().
+	 */
+	if (err == -ENOENT && (newcon->flags & CON_ENABLED))
+		err = 0;
+
 	/* printk() messages are not printed to the Braille console. */
 	if (err || newcon->flags & CON_BRL) {
 		if (newcon->flags & CON_NBCON)
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 03a43d3d26160..415583c35f8c8 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -4900,6 +4900,9 @@ static void __init rcu_dump_rcu_node_tree(void)
 
 struct workqueue_struct *rcu_gp_wq;
 
+static struct cpumask rcu_stall_cpumask;
+static struct cpumask rcu_exp_stall_cpumask;
+
 void __init rcu_init(void)
 {
 	int cpu = smp_processor_id();
diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h
index 82cada459e5d0..46b6907f1b09c 100644
--- a/kernel/rcu/tree_exp.h
+++ b/kernel/rcu/tree_exp.h
@@ -578,6 +578,7 @@ static void synchronize_rcu_expedited_stall(unsigned long jiffies_start, unsigne
 			if (!(READ_ONCE(rnp->expmask) & mask))
 				continue;
 			ndetected++;
+			cpumask_set_cpu(cpu, &rcu_exp_stall_cpumask);
 			rdp = per_cpu_ptr(&rcu_data, cpu);
 			pr_cont(" %d-%c%c%c%c", cpu,
 				"O."[!!cpu_online(cpu)],
@@ -665,6 +666,8 @@ static void synchronize_rcu_expedited_wait(void)
 		if (rcu_stall_is_suppressed())
 			continue;
 
+		cpumask_clear(&rcu_exp_stall_cpumask);
+
 		nbcon_cpu_emergency_enter();
 
 		j = jiffies;
@@ -675,7 +678,7 @@ static void synchronize_rcu_expedited_wait(void)
 
 		nbcon_cpu_emergency_exit();
 
-		panic_on_rcu_stall();
+		panic_on_rcu_stall(&rcu_exp_stall_cpumask);
 	}
 }
 
diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h
index cf7ae51cba403..ebf381936eb16 100644
--- a/kernel/rcu/tree_stall.h
+++ b/kernel/rcu/tree_stall.h
@@ -159,7 +159,7 @@ static int __init check_cpu_stall_init(void)
 early_initcall(check_cpu_stall_init);
 
 /* If so specified via sysctl, panic, yielding cleaner stall-warning output. */
-static void panic_on_rcu_stall(void)
+static void panic_on_rcu_stall(const struct cpumask *stalled_mask)
 {
 	static int cpu_stall;
 
@@ -167,7 +167,7 @@ static void panic_on_rcu_stall(void)
 	 * Attempt to kick out the BPF scheduler if it's installed and defer
 	 * the panic to give the system a chance to recover.
 	 */
-	if (scx_rcu_cpu_stall())
+	if (scx_rcu_cpu_stall(stalled_mask))
 		return;
 
 	if (++cpu_stall < sysctl_max_rcu_stall_to_panic)
@@ -644,6 +644,8 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps)
 	if (rcu_stall_is_suppressed())
 		return;
 
+	cpumask_clear(&rcu_stall_cpumask);
+
 	nbcon_cpu_emergency_enter();
 
 	/*
@@ -659,6 +661,7 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps)
 			for_each_leaf_node_possible_cpu(rnp, cpu)
 				if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) {
 					print_cpu_stall_info(cpu);
+					cpumask_set_cpu(cpu, &rcu_stall_cpumask);
 					ndetected++;
 				}
 		}
@@ -700,7 +703,7 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps)
 
 	nbcon_cpu_emergency_exit();
 
-	panic_on_rcu_stall();
+	panic_on_rcu_stall(&rcu_stall_cpumask);
 
 	rcu_force_quiescent_state();  /* Kick them all. */
 }
@@ -753,7 +756,9 @@ static void print_cpu_stall(unsigned long gp_seq, unsigned long gps)
 
 	nbcon_cpu_emergency_exit();
 
-	panic_on_rcu_stall();
+	cpumask_clear(&rcu_stall_cpumask);
+	cpumask_set_cpu(smp_processor_id(), &rcu_stall_cpumask);
+	panic_on_rcu_stall(&rcu_stall_cpumask);
 
 	/*
 	 * Attempt to revive the RCU machinery by forcing a context switch.
diff --git a/kernel/reboot.c b/kernel/reboot.c
index 695c33e75efd9..bed6967bfa965 100644
--- a/kernel/reboot.c
+++ b/kernel/reboot.c
@@ -938,16 +938,18 @@ void orderly_reboot(void)
 }
 EXPORT_SYMBOL_GPL(orderly_reboot);
 
+/* DEFAULT is NULL: a sentinel, not a real action. */
+static const char *const hw_protection_action_strs[] = {
+	[HWPROT_ACT_SHUTDOWN] = "shutdown",
+	[HWPROT_ACT_REBOOT]   = "reboot",
+};
+
 static const char *hw_protection_action_str(enum hw_protection_action action)
 {
-	switch (action) {
-	case HWPROT_ACT_SHUTDOWN:
-		return "shutdown";
-	case HWPROT_ACT_REBOOT:
-		return "reboot";
-	default:
+	if (action >= ARRAY_SIZE(hw_protection_action_strs) ||
+	    !hw_protection_action_strs[action])
 		return "undefined";
-	}
+	return hw_protection_action_strs[action];
 }
 
 static enum hw_protection_action hw_failure_emergency_action;
@@ -1055,14 +1057,17 @@ EXPORT_SYMBOL_GPL(__hw_protection_trigger);
 static bool hw_protection_action_parse(const char *str,
 				       enum hw_protection_action *action)
 {
-	if (sysfs_streq(str, "shutdown"))
-		*action = HWPROT_ACT_SHUTDOWN;
-	else if (sysfs_streq(str, "reboot"))
-		*action = HWPROT_ACT_REBOOT;
-	else
-		return false;
+	unsigned int i;
 
-	return true;
+	for (i = 0; i < ARRAY_SIZE(hw_protection_action_strs); i++) {
+		if (hw_protection_action_strs[i] &&
+		    sysfs_streq(str, hw_protection_action_strs[i])) {
+			*action = i;
+			return true;
+		}
+	}
+
+	return false;
 }
 
 static int __init hw_protection_setup(char *str)
@@ -1177,68 +1182,43 @@ __setup("reboot=", reboot_setup);
 
 #ifdef CONFIG_SYSFS
 
-#define REBOOT_COLD_STR		"cold"
-#define REBOOT_WARM_STR		"warm"
-#define REBOOT_HARD_STR		"hard"
-#define REBOOT_SOFT_STR		"soft"
-#define REBOOT_GPIO_STR		"gpio"
 #define REBOOT_UNDEFINED_STR	"undefined"
 
-#define BOOT_TRIPLE_STR		"triple"
-#define BOOT_KBD_STR		"kbd"
-#define BOOT_BIOS_STR		"bios"
-#define BOOT_ACPI_STR		"acpi"
-#define BOOT_EFI_STR		"efi"
-#define BOOT_PCI_STR		"pci"
+static const char *const reboot_mode_strs[] = {
+	[REBOOT_COLD]	= "cold",
+	[REBOOT_WARM]	= "warm",
+	[REBOOT_HARD]	= "hard",
+	[REBOOT_SOFT]	= "soft",
+	[REBOOT_GPIO]	= "gpio",
+};
 
 static ssize_t mode_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 {
-	const char *val;
+	const char *val = REBOOT_UNDEFINED_STR;
 
-	switch (reboot_mode) {
-	case REBOOT_COLD:
-		val = REBOOT_COLD_STR;
-		break;
-	case REBOOT_WARM:
-		val = REBOOT_WARM_STR;
-		break;
-	case REBOOT_HARD:
-		val = REBOOT_HARD_STR;
-		break;
-	case REBOOT_SOFT:
-		val = REBOOT_SOFT_STR;
-		break;
-	case REBOOT_GPIO:
-		val = REBOOT_GPIO_STR;
-		break;
-	default:
-		val = REBOOT_UNDEFINED_STR;
-	}
+	if ((unsigned int)reboot_mode < ARRAY_SIZE(reboot_mode_strs))
+		val = reboot_mode_strs[reboot_mode];
 
 	return sysfs_emit(buf, "%s\n", val);
 }
 static ssize_t mode_store(struct kobject *kobj, struct kobj_attribute *attr,
 			  const char *buf, size_t count)
 {
+	unsigned int i;
+
 	if (!capable(CAP_SYS_BOOT))
 		return -EPERM;
 
-	if (!strncmp(buf, REBOOT_COLD_STR, strlen(REBOOT_COLD_STR)))
-		reboot_mode = REBOOT_COLD;
-	else if (!strncmp(buf, REBOOT_WARM_STR, strlen(REBOOT_WARM_STR)))
-		reboot_mode = REBOOT_WARM;
-	else if (!strncmp(buf, REBOOT_HARD_STR, strlen(REBOOT_HARD_STR)))
-		reboot_mode = REBOOT_HARD;
-	else if (!strncmp(buf, REBOOT_SOFT_STR, strlen(REBOOT_SOFT_STR)))
-		reboot_mode = REBOOT_SOFT;
-	else if (!strncmp(buf, REBOOT_GPIO_STR, strlen(REBOOT_GPIO_STR)))
-		reboot_mode = REBOOT_GPIO;
-	else
-		return -EINVAL;
-
-	reboot_default = 0;
+	for (i = 0; i < ARRAY_SIZE(reboot_mode_strs); i++) {
+		if (!strncmp(buf, reboot_mode_strs[i],
+			     strlen(reboot_mode_strs[i]))) {
+			reboot_mode = i;
+			reboot_default = 0;
+			return count;
+		}
+	}
 
-	return count;
+	return -EINVAL;
 }
 static struct kobj_attribute reboot_mode_attr = __ATTR_RW(mode);
 
@@ -1265,31 +1245,28 @@ static ssize_t force_store(struct kobject *kobj, struct kobj_attribute *attr,
 }
 static struct kobj_attribute reboot_force_attr = __ATTR_RW(force);
 
+static const struct {
+	enum reboot_type type;
+	const char *str;
+} reboot_type_strs[] = {
+	{ BOOT_TRIPLE,		"triple" },
+	{ BOOT_KBD,		"kbd" },
+	{ BOOT_BIOS,		"bios" },
+	{ BOOT_ACPI,		"acpi" },
+	{ BOOT_EFI,		"efi" },
+	{ BOOT_CF9_FORCE,	"pci" },
+};
+
 static ssize_t type_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 {
-	const char *val;
+	const char *val = REBOOT_UNDEFINED_STR;
+	unsigned int i;
 
-	switch (reboot_type) {
-	case BOOT_TRIPLE:
-		val = BOOT_TRIPLE_STR;
-		break;
-	case BOOT_KBD:
-		val = BOOT_KBD_STR;
-		break;
-	case BOOT_BIOS:
-		val = BOOT_BIOS_STR;
-		break;
-	case BOOT_ACPI:
-		val = BOOT_ACPI_STR;
-		break;
-	case BOOT_EFI:
-		val = BOOT_EFI_STR;
-		break;
-	case BOOT_CF9_FORCE:
-		val = BOOT_PCI_STR;
-		break;
-	default:
-		val = REBOOT_UNDEFINED_STR;
+	for (i = 0; i < ARRAY_SIZE(reboot_type_strs); i++) {
+		if (reboot_type == reboot_type_strs[i].type) {
+			val = reboot_type_strs[i].str;
+			break;
+		}
 	}
 
 	return sysfs_emit(buf, "%s\n", val);
@@ -1297,27 +1274,21 @@ static ssize_t type_show(struct kobject *kobj, struct kobj_attribute *attr, char
 static ssize_t type_store(struct kobject *kobj, struct kobj_attribute *attr,
 			  const char *buf, size_t count)
 {
+	unsigned int i;
+
 	if (!capable(CAP_SYS_BOOT))
 		return -EPERM;
 
-	if (!strncmp(buf, BOOT_TRIPLE_STR, strlen(BOOT_TRIPLE_STR)))
-		reboot_type = BOOT_TRIPLE;
-	else if (!strncmp(buf, BOOT_KBD_STR, strlen(BOOT_KBD_STR)))
-		reboot_type = BOOT_KBD;
-	else if (!strncmp(buf, BOOT_BIOS_STR, strlen(BOOT_BIOS_STR)))
-		reboot_type = BOOT_BIOS;
-	else if (!strncmp(buf, BOOT_ACPI_STR, strlen(BOOT_ACPI_STR)))
-		reboot_type = BOOT_ACPI;
-	else if (!strncmp(buf, BOOT_EFI_STR, strlen(BOOT_EFI_STR)))
-		reboot_type = BOOT_EFI;
-	else if (!strncmp(buf, BOOT_PCI_STR, strlen(BOOT_PCI_STR)))
-		reboot_type = BOOT_CF9_FORCE;
-	else
-		return -EINVAL;
-
-	reboot_default = 0;
+	for (i = 0; i < ARRAY_SIZE(reboot_type_strs); i++) {
+		if (!strncmp(buf, reboot_type_strs[i].str,
+			     strlen(reboot_type_strs[i].str))) {
+			reboot_type = reboot_type_strs[i].type;
+			reboot_default = 0;
+			return count;
+		}
+	}
 
-	return count;
+	return -EINVAL;
 }
 static struct kobj_attribute reboot_type_attr = __ATTR_RW(type);
 #endif
diff --git a/kernel/resource.c b/kernel/resource.c
index 3d17e3196a3ec..e60539a55541d 100644
--- a/kernel/resource.c
+++ b/kernel/resource.c
@@ -1859,7 +1859,7 @@ int iomem_map_sanity_check(resource_size_t addr, unsigned long size)
 		if (p->flags & IORESOURCE_BUSY)
 			continue;
 
-		pr_warn("resource sanity check: requesting [mem %pa-%pa], which spans more than %s %pR\n",
+		pr_debug("resource sanity check: requesting [mem %pa-%pa], which spans more than %s %pR\n",
 			&addr, &end, p->name, p);
 		err = -1;
 		break;
diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c
index d74b54f819927..2a828725a7f96 100644
--- a/kernel/sched/build_policy.c
+++ b/kernel/sched/build_policy.c
@@ -66,10 +66,13 @@
 # include "ext/cid.h"
 # include "ext/arena.h"
 # include "ext/idle.h"
+# include "ext/sub.h"
+# include "ext/inlines.h"
 # include "ext/ext.c"
 # include "ext/cid.c"
 # include "ext/arena.c"
 # include "ext/idle.c"
+# include "ext/sub.c"
 #endif
 
 #include "syscalls.c"
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 96226707c2f61..2e7cde033a319 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5657,11 +5657,8 @@ EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
  */
 static inline void prefetch_curr_exec_start(struct task_struct *p)
 {
-#ifdef CONFIG_FAIR_GROUP_SCHED
-	struct sched_entity *curr = p->se.cfs_rq->curr;
-#else
 	struct sched_entity *curr = task_rq(p)->cfs.curr;
-#endif
+
 	prefetch(curr);
 	prefetch(&curr->exec_start);
 }
diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
index a4e689eefdfbd..dc0835ceee982 100644
--- a/kernel/sched/cpufreq_schedutil.c
+++ b/kernel/sched/cpufreq_schedutil.c
@@ -50,6 +50,7 @@ struct sugov_cpu {
 
 	unsigned long		util;
 	unsigned long		bw_min;
+	unsigned long		bw_max;
 
 	/* The field below is for single-CPU policies only: */
 #ifdef CONFIG_NO_HZ_COMMON
@@ -232,6 +233,7 @@ static void sugov_get_util(struct sugov_cpu *sg_cpu, unsigned long boost)
 	util = effective_cpu_util(sg_cpu->cpu, util, &min, &max);
 	util = max(util, boost);
 	sg_cpu->bw_min = min;
+	sg_cpu->bw_max = max;
 	sg_cpu->util = sugov_effective_cpu_perf(sg_cpu->cpu, util, min, max);
 }
 
@@ -314,7 +316,7 @@ static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time,
  * A CPU running a task which woken up after an IO operation can have its
  * utilization boosted to speed up the completion of those IO operations.
  * The IO boost value is increased each time a task wakes up from IO, in
- * sugov_iowait_apply(), and it's instead decreased by this function,
+ * sugov_iowait_boost(), and it's instead decreased by this function,
  * each time an increase has not been requested (!iowait_boost_pending).
  *
  * A CPU which also appears to have been idle for at least one tick has also
@@ -484,7 +486,7 @@ static void sugov_update_single_perf(struct update_util_data *hook, u64 time,
 		sg_cpu->util = prev_util;
 
 	cpufreq_driver_adjust_perf(sg_policy->policy, sg_cpu->bw_min,
-				   sg_cpu->util, max_cap);
+				   sg_cpu->util, sg_cpu->bw_max, max_cap);
 
 	sg_policy->need_freq_update = false;
 	sg_policy->last_freq_update_time = time;
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 0f858b98c9aa3..a3003b0f45226 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -2733,15 +2733,17 @@ static int balance_dl(struct rq *rq, struct rq_flags *rf)
  */
 static void wakeup_preempt_dl(struct rq *rq, struct task_struct *p, int flags)
 {
+	struct task_struct *donor = rq->donor;
 	/*
 	 * Can only get preempted by stop-class, and those should be
 	 * few and short lived, doesn't really make sense to push
 	 * anything away for that.
 	 */
-	if (p->sched_class != &dl_sched_class)
+	if (p->sched_class != &dl_sched_class ||
+	    donor->sched_class != &dl_sched_class)
 		return;
 
-	if (dl_entity_preempt(&p->dl, &rq->donor->dl)) {
+	if (dl_entity_preempt(&p->dl, &donor->dl)) {
 		resched_curr(rq);
 		return;
 	}
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 40584b27ea0ce..72236db67983b 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -633,6 +633,82 @@ static void debugfs_fair_server_init(void)
 	}
 }
 
+#ifdef CONFIG_FAIR_GROUP_SCHED
+static int cgroup_mode = 2;
+
+/* See __sched_cgroup_mode_update(). */
+static const char *cgroup_mode_str[] = {
+	"up",
+	"smp",
+	"concur",
+	"max",
+	"tasks",
+};
+
+static int sched_cgroup_mode(const char *str)
+{
+	for (int i = 0; i < ARRAY_SIZE(cgroup_mode_str); i++) {
+		if (!strcmp(str, cgroup_mode_str[i]))
+			return i;
+	}
+	return -EINVAL;
+}
+
+static ssize_t sched_cgroup_write(struct file *filp, const char __user *ubuf,
+				   size_t cnt, loff_t *ppos)
+{
+	char buf[16];
+	int mode;
+
+	if (cnt > 15)
+		cnt = 15;
+
+	if (copy_from_user(buf, ubuf, cnt))
+		return -EFAULT;
+
+	buf[cnt] = 0;
+	mode = sched_cgroup_mode(strstrip(buf));
+	if (mode < 0)
+		return mode;
+
+	__sched_cgroup_mode_update(mode);
+	WRITE_ONCE(cgroup_mode, mode);
+
+	*ppos += cnt;
+	return cnt;
+}
+
+static int sched_cgroup_show(struct seq_file *m, void *v)
+{
+	int mode = READ_ONCE(cgroup_mode);
+
+	for (int i = 0; i < ARRAY_SIZE(cgroup_mode_str); i++) {
+		if (mode == i)
+			seq_puts(m, "(");
+		seq_puts(m, cgroup_mode_str[i]);
+		if (mode == i)
+			seq_puts(m, ")");
+
+		seq_puts(m, " ");
+	}
+	seq_puts(m, "\n");
+	return 0;
+}
+
+static int sched_cgroup_open(struct inode *inode, struct file *filp)
+{
+	return single_open(filp, sched_cgroup_show, NULL);
+}
+
+static const struct file_operations sched_cgroup_fops = {
+	.open		= sched_cgroup_open,
+	.write		= sched_cgroup_write,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+#endif
+
 static __init int sched_init_debug(void)
 {
 	struct dentry __maybe_unused *numa, *llc;
@@ -686,6 +762,10 @@ static __init int sched_init_debug(void)
 
 	debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);
 
+#ifdef CONFIG_FAIR_GROUP_SCHED
+	debugfs_create_file("cgroup_mode", 0644, debugfs_sched, NULL, &sched_cgroup_fops);
+#endif
+
 	debugfs_fair_server_init();
 #ifdef CONFIG_SCHED_CLASS_EXT
 	debugfs_ext_server_init();
@@ -895,10 +975,11 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p)
 	else
 		SEQ_printf(m, " %c", task_state_to_char(p));
 
-	SEQ_printf(m, " %15s %5d %9Ld.%06ld   %c   %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld   %5d ",
+	SEQ_printf(m, " %15s %5d %10ld %9Ld.%06ld   %c   %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld   %5d ",
 		p->comm, task_pid_nr(p),
+		p->se.h_load.weight,
 		SPLIT_NS(p->se.vruntime),
-		entity_eligible(cfs_rq_of(&p->se), &p->se) ? 'E' : 'N',
+		entity_eligible(&rq->cfs, &p->se) ? 'E' : 'N',
 		SPLIT_NS(p->se.deadline),
 		p->se.custom_slice ? 'S' : ' ',
 		SPLIT_NS(p->se.slice),
@@ -927,7 +1008,7 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu)
 
 	SEQ_printf(m, "\n");
 	SEQ_printf(m, "runnable tasks:\n");
-	SEQ_printf(m, " S            task   PID       vruntime   eligible    "
+	SEQ_printf(m, " S            task   PID     weight       vruntime   eligible    "
 		   "deadline             slice          sum-exec      switches  "
 		   "prio         wait-time        sum-sleep       sum-block"
 #ifdef CONFIG_NUMA_BALANCING
@@ -1035,6 +1116,8 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
 			cfs_rq->tg_load_avg_contrib);
 	SEQ_printf(m, "  .%-30s: %ld\n", "tg_load_avg",
 			atomic_long_read(&cfs_rq->tg->load_avg));
+	SEQ_printf(m, "  .%-30s: %lu\n", "h_load",
+			cfs_rq->h_load);
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 #ifdef CONFIG_CFS_BANDWIDTH
 	SEQ_printf(m, "  .%-30s: %d\n", "throttled",
@@ -1359,7 +1442,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
 		P_SCHEDSTAT(wait_count);
 		PN_SCHEDSTAT(iowait_sum);
 		P_SCHEDSTAT(iowait_count);
-		P_SCHEDSTAT(nr_migrations_cold);
 		P_SCHEDSTAT(nr_failed_migrations_affine);
 		P_SCHEDSTAT(nr_failed_migrations_running);
 		P_SCHEDSTAT(nr_failed_migrations_hot);
@@ -1371,8 +1453,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
 		P_SCHEDSTAT(nr_wakeups_remote);
 		P_SCHEDSTAT(nr_wakeups_affine);
 		P_SCHEDSTAT(nr_wakeups_affine_attempts);
-		P_SCHEDSTAT(nr_wakeups_passive);
-		P_SCHEDSTAT(nr_wakeups_idle);
 
 		avg_atom = p->se.sum_exec_runtime;
 		if (nr_switches)
diff --git a/kernel/sched/ext/arena.c b/kernel/sched/ext/arena.c
index 5783694ec21da..b0e6a0be49139 100644
--- a/kernel/sched/ext/arena.c
+++ b/kernel/sched/ext/arena.c
@@ -70,8 +70,6 @@ void scx_arena_pool_destroy(struct scx_sched *sch)
  */
 static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt)
 {
-	u64 kern_vm_start;
-	u32 uaddr32;
 	void *p;
 	int ret;
 
@@ -83,15 +81,8 @@ static int scx_arena_grow(struct scx_sched *sch, u32 page_cnt)
 	if (!p)
 		return -ENOMEM;
 
-	uaddr32 = (u32)(unsigned long)p;
-	/* arena.o, which defines these, is built only on MMU && 64BIT */
-#if defined(CONFIG_MMU) && defined(CONFIG_64BIT)
-	kern_vm_start = bpf_arena_map_kern_vm_start(sch->arena_map);
-#else
-	kern_vm_start = 0;
-#endif
-
-	ret = gen_pool_add(sch->arena_pool, kern_vm_start + uaddr32,
+	ret = gen_pool_add(sch->arena_pool,
+			   (unsigned long)scx_arena_to_kaddr(sch, p),
 			   page_cnt * PAGE_SIZE, NUMA_NO_NODE);
 	if (ret) {
 		bpf_arena_free_pages_non_sleepable(sch->arena_map, p, page_cnt);
diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c
index af83084ec740a..9dfd242be34ff 100644
--- a/kernel/sched/ext/cid.c
+++ b/kernel/sched/ext/cid.c
@@ -13,18 +13,22 @@
 /*
  * cid tables.
  *
- * Pointers are published once on first enable and never revoked. The default
- * mapping is populated before ops.init() runs; scx_bpf_cid_override() commits
- * before it returns. As long as the BPF scheduler only uses the tables from
- * those points onward, it sees a consistent view.
+ * Pointers are allocated on first enable and never freed. During root enable,
+ * the default mapping is populated and then ops.init_cids() is called which can
+ * use scx_bpf_cid_override() to change the mapping. The mapping stays stable
+ * until the root is disabled.
  */
+u32 scx_nr_cid_shards;
 s16 *scx_cid_to_cpu_tbl;
 s16 *scx_cpu_to_cid_tbl;
+s32 *scx_cid_to_shard;
+s32 *scx_shard_node;
+struct scx_cid_shard *scx_cid_shard_ranges;
 struct scx_cid_topo *scx_cid_topo;
 
 #define SCX_CID_TOPO_NEG	(struct scx_cid_topo) {				\
 	.core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1,		\
-	.node_cid = -1, .node_idx = -1,						\
+	.node_cid = -1, .node_idx = -1, .shard_cid = -1, .shard_idx = -1,	\
 }
 
 /*
@@ -43,11 +47,40 @@ static const struct cpumask *cpu_llc_mask(int cpu, struct cpumask *fallbacks)
 	return &ci->info_list[ci->num_leaves - 1].shared_cpu_map;
 }
 
+/*
+ * Compute per-LLC shard layout. Each shard holds at most @shard_size cids, and
+ * in any case no more than SCX_CID_SHARD_MAX_CPUS. Cores are spread as evenly
+ * as possible across shards so cpu count is balanced: the first *@nr_large_p
+ * shards get (*@cores_per_shard_p + 1) cores, the rest get *@cores_per_shard_p.
+ */
+static void calc_shard_layout(const struct cpumask *llc_cpus, u32 shard_size,
+			      u32 *cores_per_shard_p, u32 *nr_large_p)
+{
+	u32 nr_cores = 0, nr_cpus = 0, nr_shards;
+	int cpu;
+
+	for_each_cpu(cpu, llc_cpus) {
+		nr_cpus++;
+		if (cpumask_first(topology_sibling_cpumask(cpu)) == cpu)
+			nr_cores++;
+	}
+
+	nr_shards = max_t(u32, 1, DIV_ROUND_UP(nr_cpus, shard_size));
+	nr_shards = max_t(u32, nr_shards,
+			  DIV_ROUND_UP(nr_cpus, SCX_CID_SHARD_MAX_CPUS));
+
+	*cores_per_shard_p = nr_cores / nr_shards;
+	*nr_large_p = nr_cores % nr_shards;
+}
+
 /* Allocate the cid tables once on first enable; never freed. */
 static s32 scx_cid_arrays_alloc(void)
 {
 	u32 npossible = num_possible_cpus();
 	s16 *cid_to_cpu, *cpu_to_cid;
+	s32 *cid_to_shard;
+	s32 *shard_node;
+	struct scx_cid_shard *cid_shard_ranges;
 	struct scx_cid_topo *cid_topo;
 
 	if (scx_cid_to_cpu_tbl)
@@ -55,17 +88,27 @@ static s32 scx_cid_arrays_alloc(void)
 
 	cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL);
 	cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL);
+	cid_to_shard = kzalloc_objs(*scx_cid_to_shard, npossible, GFP_KERNEL);
+	shard_node = kmalloc_objs(*scx_shard_node, npossible, GFP_KERNEL);
+	cid_shard_ranges = kzalloc_objs(*scx_cid_shard_ranges, npossible, GFP_KERNEL);
 	cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL);
 
-	if (!cid_to_cpu || !cpu_to_cid || !cid_topo) {
+	if (!cid_to_cpu || !cpu_to_cid || !cid_to_shard || !shard_node ||
+	    !cid_shard_ranges || !cid_topo) {
 		kfree(cid_to_cpu);
 		kfree(cpu_to_cid);
+		kfree(cid_to_shard);
+		kfree(shard_node);
+		kfree(cid_shard_ranges);
 		kfree(cid_topo);
 		return -ENOMEM;
 	}
 
 	WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu);
 	WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid);
+	WRITE_ONCE(scx_cid_to_shard, cid_to_shard);
+	WRITE_ONCE(scx_shard_node, shard_node);
+	WRITE_ONCE(scx_cid_shard_ranges, cid_shard_ranges);
 	WRITE_ONCE(scx_cid_topo, cid_topo);
 	return 0;
 }
@@ -90,17 +133,29 @@ s32 scx_cid_init(struct scx_sched *sch)
 	cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL;
 	u32 next_cid = 0;
 	s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0;
-	s32 cpu, ret;
+	s32 next_shard_idx = 0;
+	u32 shard_size, max_cids;
+	u32 notopo_in_shard;
+	s32 notopo_shard_cid, notopo_shard_idx;
+	s32 cpu, cid, si, ret;
 
 	/* CMASK_MAX_WORDS in cid.bpf.h covers NR_CPUS up to 8192 */
 	BUILD_BUG_ON(NR_CPUS > 8192);
 
 	lockdep_assert_cpus_held();
 
+	shard_size = sch->ops.cid_shard_size ?: SCX_CID_SHARD_SIZE_DFL;
+	max_cids = min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS);
+
 	ret = scx_cid_arrays_alloc();
 	if (ret)
 		return ret;
 
+	/* clear shard ranges and reset shard_node for repopulate */
+	memset(scx_cid_shard_ranges, 0, num_possible_cpus() * sizeof(*scx_cid_shard_ranges));
+	for (si = 0; si < num_possible_cpus(); si++)
+		scx_shard_node[si] = NUMA_NO_NODE;
+
 	if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) ||
 	    !zalloc_cpumask_var(&node_scratch, GFP_KERNEL) ||
 	    !zalloc_cpumask_var(&llc_scratch, GFP_KERNEL) ||
@@ -142,29 +197,60 @@ s32 scx_cid_init(struct scx_sched *sch)
 			const struct cpumask *llc_mask = cpu_llc_mask(ncpu, llc_fallback);
 			s32 llc_cid = next_cid;
 			s32 llc_idx = next_llc_idx++;
+			u32 cores_per_shard, nr_large;
+			u32 shard_local = 0, cores_in_shard = 0, cids_in_shard = 0;
+			s32 shard_cid, shard_idx;
 
 			/* llc_scratch = node_scratch & this llc */
 			cpumask_and(llc_scratch, node_scratch, llc_mask);
 			if (WARN_ON_ONCE(!cpumask_test_cpu(ncpu, llc_scratch)))
 				return -EINVAL;
 
+			calc_shard_layout(llc_scratch, shard_size, &cores_per_shard, &nr_large);
+			shard_cid = next_cid;
+			shard_idx = next_shard_idx++;
+			scx_shard_node[shard_idx] = nid;
+
 			while (!cpumask_empty(llc_scratch)) {
 				s32 lcpu = cpumask_first(llc_scratch);
 				const struct cpumask *sib = topology_sibling_cpumask(lcpu);
 				s32 core_cid = next_cid;
 				s32 core_idx = next_core_idx++;
 				s32 ccpu;
+				u32 max_cores, cids_in_core;
 
 				/* core_scratch = llc_scratch & this core */
 				cpumask_and(core_scratch, llc_scratch, sib);
 				if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch)))
 					return -EINVAL;
 
+				/*
+				 * Advance to a new shard when either core or
+				 * cid count reaches max. The latter bounds
+				 * shard sizes under uneven SMT. Never start an
+				 * empty shard.
+				 */
+				cids_in_core = cpumask_weight(core_scratch);
+				max_cores = cores_per_shard + (shard_local < nr_large ? 1 : 0);
+				if (cores_in_shard &&
+				    (cores_in_shard >= max_cores ||
+				     cids_in_shard + cids_in_core > max_cids)) {
+					shard_local++;
+					cores_in_shard = 0;
+					cids_in_shard = 0;
+					shard_cid = next_cid;
+					shard_idx = next_shard_idx++;
+					scx_shard_node[shard_idx] = nid;
+				}
+				cores_in_shard++;
+				cids_in_shard += cids_in_core;
+
 				for_each_cpu(ccpu, core_scratch) {
 					s32 cid = next_cid++;
 
 					scx_cid_to_cpu_tbl[cid] = ccpu;
 					scx_cpu_to_cid_tbl[ccpu] = cid;
+					scx_cid_to_shard[cid] = shard_idx;
 					scx_cid_topo[cid] = (struct scx_cid_topo){
 						.core_cid = core_cid,
 						.core_idx = core_idx,
@@ -172,6 +258,8 @@ s32 scx_cid_init(struct scx_sched *sch)
 						.llc_idx = llc_idx,
 						.node_cid = node_cid,
 						.node_idx = node_idx,
+						.shard_cid = shard_cid,
+						.shard_idx = shard_idx,
 					};
 
 					cpumask_clear_cpu(ccpu, llc_scratch);
@@ -184,12 +272,17 @@ s32 scx_cid_init(struct scx_sched *sch)
 
 	/*
 	 * No-topo section: any possible cpu without a cid - normally just the
-	 * not-online ones. Collect any currently-online cpus that land here in
-	 * @online_no_topo so we can warn about them at the end.
+	 * not-online ones. Pack into shards of up to min(@shard_size,
+	 * SCX_CID_SHARD_MAX_CPUS) cids so that every cid has a valid shard
+	 * assignment and the hard cap holds even with a large @shard_size.
+	 * Collect any currently-online cpus that land here in @online_no_topo
+	 * so we can warn about them at the end.
 	 */
-	for_each_cpu(cpu, cpu_possible_mask) {
-		s32 cid;
+	notopo_in_shard = min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS);
+	notopo_shard_cid = -1;
+	notopo_shard_idx = -1;
 
+	for_each_cpu(cpu, cpu_possible_mask) {
 		if (__scx_cpu_to_cid(cpu) != -1)
 			continue;
 		if (cpu_online(cpu))
@@ -198,7 +291,18 @@ s32 scx_cid_init(struct scx_sched *sch)
 		cid = next_cid++;
 		scx_cid_to_cpu_tbl[cid] = cpu;
 		scx_cpu_to_cid_tbl[cpu] = cid;
+
+		if (notopo_in_shard >= min_t(u32, shard_size, SCX_CID_SHARD_MAX_CPUS)) {
+			notopo_shard_cid = cid;
+			notopo_shard_idx = next_shard_idx++;
+			notopo_in_shard = 0;
+		}
+		notopo_in_shard++;
+
+		scx_cid_to_shard[cid] = notopo_shard_idx;
 		scx_cid_topo[cid] = SCX_CID_TOPO_NEG;
+		scx_cid_topo[cid].shard_cid = notopo_shard_cid;
+		scx_cid_topo[cid].shard_idx = notopo_shard_idx;
 	}
 
 	if (!cpumask_empty(llc_fallback))
@@ -208,6 +312,20 @@ s32 scx_cid_init(struct scx_sched *sch)
 		pr_warn("scx_cid: online cpus with no usable topology: %*pbl\n",
 			cpumask_pr_args(online_no_topo));
 
+	/*
+	 * Fill cid_shard_ranges[] from cid_to_shard[]. Shards are contiguous
+	 * cid ranges by construction: base_cid is the first cid landing in a
+	 * shard, nr_cids is the count.
+	 */
+	for (cid = 0; cid < next_cid; cid++) {
+		s32 sidx = scx_cid_to_shard[cid];
+
+		if (scx_cid_shard_ranges[sidx].nr_cids == 0)
+			scx_cid_shard_ranges[sidx].base_cid = cid;
+		scx_cid_shard_ranges[sidx].nr_cids++;
+	}
+
+	scx_nr_cid_shards = next_shard_idx;
 	return 0;
 }
 
@@ -274,29 +392,66 @@ void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst)
 	}
 }
 
+/*
+ * Return the index of the largest entry in @counts, or NUMA_NO_NODE if all
+ * entries are zero. Ties resolve to the lowest index.
+ */
+static s32 pick_max_node(const u32 *counts, u32 n)
+{
+	s32 best = NUMA_NO_NODE;
+	u32 best_count = 0, i;
+
+	for (i = 0; i < n; i++) {
+		if (counts[i] > best_count) {
+			best_count = counts[i];
+			best = i;
+		}
+	}
+	return best;
+}
+
 __bpf_kfunc_start_defs();
 
 /**
- * scx_bpf_cid_override - Install an explicit cpu->cid mapping
- * @cpu_to_cid: array of nr_cpu_ids s32 entries (cid for each cpu)
- * @cpu_to_cid__sz: must be nr_cpu_ids * sizeof(s32) bytes
+ * scx_bpf_cid_override - Install an explicit cpu->cid mapping with shard info
+ * @cpu_to_cid_src: array of nr_cpu_ids s32 entries (cid for each cpu)
+ * @cpu_to_cid_src__sz: must be nr_cpu_ids * sizeof(s32) bytes
+ * @shard_start_src: array of first-cid-of-each-shard, strictly increasing from 0
+ * @shard_start_src__sz: nr_shards * sizeof(s32) bytes
  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
- * May only be called from ops.init() of the root scheduler. Replace the
- * topology-probed cid mapping with the caller-provided one. Each possible cpu
- * must map to a unique cid in [0, num_possible_cpus()). Topo info is cleared.
- * On invalid input, trigger scx_error() to abort the scheduler.
+ * May only be called from ops.init_cids() of the root scheduler. Replace the
+ * topology-probed cid mapping and shard layout with caller-provided ones. Each
+ * possible cpu must map to a unique cid in [0, num_possible_cpus()). The shard
+ * starts must be strictly increasing with the first entry 0 and all values <
+ * num_possible_cpus(). The last shard extends to num_possible_cpus() and no
+ * shard may span more than SCX_CID_SHARD_MAX_CPUS cids. Topo info
+ * (core/LLC/node) is cleared and the shard layout is set from the input. On
+ * invalid input, abort the scheduler.
  */
-__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz,
-				      const struct bpf_prog_aux *aux)
+__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid_src, u32 cpu_to_cid_src__sz,
+				       const s32 *shard_start_src, u32 shard_start_src__sz,
+				       const struct bpf_prog_aux *aux)
 {
 	cpumask_var_t seen __free(free_cpumask_var) = CPUMASK_VAR_NULL;
+	u32 *node_counts __free(kfree) = NULL;
+	s32 *cpu_to_cid __free(kfree) = NULL;
+	s32 *shard_start __free(kfree) = NULL;
+	u32 npossible = num_possible_cpus();
 	struct scx_sched *sch;
+	u32 nr_shards;
 	bool alloced;
-	s32 cpu, cid;
+	s32 cpu, cid, si;
 
-	/* GFP_KERNEL alloc must happen before the rcu read section */
+	/*
+	 * GFP_KERNEL allocs must happen before the rcu read section. Snapshot
+	 * the BPF-supplied arrays so a concurrent map mutation can't change
+	 * them between validation and use.
+	 */
 	alloced = zalloc_cpumask_var(&seen, GFP_KERNEL);
+	node_counts = kcalloc(nr_node_ids, sizeof(*node_counts), GFP_KERNEL);
+	cpu_to_cid = kmemdup(cpu_to_cid_src, cpu_to_cid_src__sz, GFP_KERNEL);
+	shard_start = kmemdup(shard_start_src, shard_start_src__sz, GFP_KERNEL);
 
 	guard(rcu)();
 
@@ -304,22 +459,57 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz,
 	if (unlikely(!sch))
 		return;
 
-	if (!alloced) {
-		scx_error(sch, "scx_bpf_cid_override: failed to allocate cpumask");
+	if (!alloced || !node_counts || !cpu_to_cid || !shard_start) {
+		scx_error(sch, "scx_bpf_cid_override: allocation failed");
 		return;
 	}
 
-	if (scx_parent(sch)) {
-		scx_error(sch, "scx_bpf_cid_override() only allowed from root sched");
+	if (cpu_to_cid_src__sz != nr_cpu_ids * sizeof(s32)) {
+		scx_error(sch, "scx_bpf_cid_override: cpu_to_cid expected %zu bytes, got %u",
+			  nr_cpu_ids * sizeof(s32), cpu_to_cid_src__sz);
 		return;
 	}
 
-	if (cpu_to_cid__sz != nr_cpu_ids * sizeof(s32)) {
-		scx_error(sch, "scx_bpf_cid_override: expected %zu bytes, got %u",
-			  nr_cpu_ids * sizeof(s32), cpu_to_cid__sz);
+	if (!shard_start_src__sz || shard_start_src__sz % sizeof(s32)) {
+		scx_error(sch, "scx_bpf_cid_override: invalid shard_start size %u",
+			  shard_start_src__sz);
 		return;
 	}
 
+	nr_shards = shard_start_src__sz / sizeof(s32);
+
+	/* validate shard_start[]: starts at 0, strictly increasing, in range */
+	if (shard_start[0] != 0) {
+		scx_error(sch, "scx_bpf_cid_override: shard_start[0] must be 0, got %d",
+			  shard_start[0]);
+		return;
+	}
+	for (si = 1; si < nr_shards; si++) {
+		if (shard_start[si] <= shard_start[si - 1]) {
+			scx_error(sch, "scx_bpf_cid_override: shard_start not increasing at [%d]",
+				  si);
+			return;
+		}
+		if (shard_start[si] >= npossible) {
+			scx_error(sch, "scx_bpf_cid_override: shard_start[%d]=%d >= %u",
+				  si, shard_start[si], npossible);
+			return;
+		}
+		if (shard_start[si] - shard_start[si - 1] > SCX_CID_SHARD_MAX_CPUS) {
+			scx_error(sch, "scx_bpf_cid_override: shard[%d] span %d exceeds max %d",
+				  si - 1, shard_start[si] - shard_start[si - 1],
+				  SCX_CID_SHARD_MAX_CPUS);
+			return;
+		}
+	}
+	if (npossible - shard_start[nr_shards - 1] > SCX_CID_SHARD_MAX_CPUS) {
+		scx_error(sch, "scx_bpf_cid_override: shard[%d] span %d exceeds max %d",
+			  nr_shards - 1, npossible - shard_start[nr_shards - 1],
+			  SCX_CID_SHARD_MAX_CPUS);
+		return;
+	}
+
+	/* Validate first so that invalid input leaves globals untouched. */
 	for_each_possible_cpu(cpu) {
 		s32 c = cpu_to_cid[cpu];
 
@@ -329,13 +519,56 @@ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz,
 			scx_error(sch, "cid %d assigned to multiple cpus", c);
 			return;
 		}
+	}
+
+	for_each_possible_cpu(cpu) {
+		s32 c = cpu_to_cid[cpu];
+
 		scx_cpu_to_cid_tbl[cpu] = c;
 		scx_cid_to_cpu_tbl[c] = cpu;
 	}
 
-	/* Invalidate stale topo info - the override carries no topology. */
-	for (cid = 0; cid < num_possible_cpus(); cid++)
+	/*
+	 * Derive scx_shard_node[] by majority count: an overridden shard may
+	 * span NUMA nodes, so assign each to the node that owns the most cpus.
+	 */
+	for (si = 0; si < nr_shards; si++) {
+		u32 end = (si + 1 < nr_shards) ? shard_start[si + 1] : npossible;
+
+		memset(node_counts, 0, nr_node_ids * sizeof(*node_counts));
+		for (cid = shard_start[si]; cid < end; cid++) {
+			s32 node = cpu_to_node(scx_cid_to_cpu_tbl[cid]);
+
+			if (numa_valid_node(node))
+				node_counts[node]++;
+		}
+		scx_shard_node[si] = pick_max_node(node_counts, nr_node_ids);
+	}
+
+	/*
+	 * Invalidate stale topo info and install shard layout from
+	 * @shard_start. Walk shards to derive shard_cid/shard_idx for each cid.
+	 */
+	si = 0;
+	for (cid = 0; cid < npossible; cid++) {
+		if (si + 1 < nr_shards && cid >= shard_start[si + 1])
+			si++;
+		scx_cid_to_shard[cid] = si;
 		scx_cid_topo[cid] = SCX_CID_TOPO_NEG;
+		scx_cid_topo[cid].shard_cid = shard_start[si];
+		scx_cid_topo[cid].shard_idx = si;
+	}
+
+	/* Rebuild scx_cid_shard_ranges[] for the new layout. */
+	memset(scx_cid_shard_ranges, 0, npossible * sizeof(*scx_cid_shard_ranges));
+	for (si = 0; si < nr_shards; si++) {
+		u32 end = (si + 1 < nr_shards) ? shard_start[si + 1] : npossible;
+
+		scx_cid_shard_ranges[si].base_cid = shard_start[si];
+		scx_cid_shard_ranges[si].nr_cids = end - shard_start[si];
+	}
+
+	scx_nr_cid_shards = nr_shards;
 }
 
 /**
@@ -395,21 +628,25 @@ __bpf_kfunc s32 scx_bpf_cpu_to_cid(s32 cpu, const struct bpf_prog_aux *aux)
  * bits outside stay untouched. In particular, scx_cmask_copy() does NOT zero
  * @dst bits that lie outside @src's range.
  *
- * The _RACY variants are otherwise identical to their non-racy counterpart but
- * read @src word-by-word via data_race(). Memory ordering with concurrent
- * writers is the caller's responsibility.
+ * Word accesses use READ_ONCE/WRITE_ONCE so a caller may read @src
+ * locklessly. Memory ordering against concurrent writers is the caller's
+ * responsibility.
  */
 enum cmask_op2 {
 	/* mutating */
 	CMASK_OP2_AND,
 	CMASK_OP2_OR,
-	CMASK_OP2_OR_RACY,
 	CMASK_OP2_COPY,
-	CMASK_OP2_COPY_RACY,
 	CMASK_OP2_ANDNOT,
 	/* predicates - short-circuit when the per-word result is true */
 	CMASK_OP2_SUBSET,
 	CMASK_OP2_INTERSECTS,
+	/*
+	 * @a is a BPF-arena cmask. Words on @a use READ_ONCE/WRITE_ONCE since
+	 * BPF may read/write concurrently. See scx_cmask_ref_or() / _copy().
+	 */
+	CMASK_OP2_REF_OR,
+	CMASK_OP2_REF_COPY,
 };
 
 static __always_inline bool cmask_op2_is_pred(const enum cmask_op2 op)
@@ -422,28 +659,28 @@ static __always_inline bool cmask_word_op2(u64 *av, const u64 *bp, u64 mask,
 {
 	switch (op) {
 	case CMASK_OP2_AND:
-		*av &= ~mask | *bp;
+		WRITE_ONCE(*av, *av & (~mask | READ_ONCE(*bp)));
 		return false;
 	case CMASK_OP2_OR:
-		*av |= *bp & mask;
-		return false;
-	case CMASK_OP2_OR_RACY:
-		*av |= data_race(*bp) & mask;
+		WRITE_ONCE(*av, *av | (READ_ONCE(*bp) & mask));
 		return false;
 	case CMASK_OP2_COPY:
-		*av = (*av & ~mask) | (*bp & mask);
-		return false;
-	case CMASK_OP2_COPY_RACY:
-		*av = (*av & ~mask) | (data_race(*bp) & mask);
+		WRITE_ONCE(*av, (*av & ~mask) | (READ_ONCE(*bp) & mask));
 		return false;
 	case CMASK_OP2_ANDNOT:
-		*av &= ~(*bp & mask);
+		WRITE_ONCE(*av, *av & ~(READ_ONCE(*bp) & mask));
 		return false;
 	case CMASK_OP2_SUBSET:
 		/* stop on the first bit in @sub not set in @super */
-		return (*bp & ~*av) & mask;
+		return (READ_ONCE(*bp) & ~READ_ONCE(*av)) & mask;
 	case CMASK_OP2_INTERSECTS:
-		return (*av & *bp) & mask;
+		return (READ_ONCE(*av) & READ_ONCE(*bp)) & mask;
+	case CMASK_OP2_REF_OR:
+		WRITE_ONCE(*av, READ_ONCE(*av) | (READ_ONCE(*bp) & mask));
+		return false;
+	case CMASK_OP2_REF_COPY:
+		WRITE_ONCE(*av, (READ_ONCE(*av) & ~mask) | (READ_ONCE(*bp) & mask));
+		return false;
 	}
 	unreachable();
 }
@@ -504,7 +741,7 @@ static __always_inline bool cmask_word_op1(const u64 *ap, u64 mask,
 {
 	switch (op) {
 	case CMASK_OP1_ANY_SET:
-		return *ap & mask;
+		return READ_ONCE(*ap) & mask;
 	}
 	unreachable();
 }
@@ -556,39 +793,12 @@ void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src)
 		       src->bits, src->base, src->nr_cids, CMASK_OP2_OR);
 }
 
-/**
- * scx_cmask_or_racy - OR @src into @dst, reading @src without locking
- *
- * @src is read word-by-word through data_race(). Same per-bit independence
- * rationale as scx_cmask_copy_racy(). Memory ordering with writers is the
- * caller's responsibility.
- */
-void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src)
-{
-	cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
-		       src->bits, src->base, src->nr_cids, CMASK_OP2_OR_RACY);
-}
-
 void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src)
 {
 	cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
 		       src->bits, src->base, src->nr_cids, CMASK_OP2_COPY);
 }
 
-/**
- * scx_cmask_copy_racy - Snapshot @src into @dst without locking
- *
- * @src is read word-by-word through data_race(). Head/tail masking matches
- * scx_cmask_copy(). Each bit in a cmask is independent, so partial updates
- * just leave some bits fresher than others. Memory ordering with writers is
- * the caller's responsibility.
- */
-void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src)
-{
-	cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
-		       src->bits, src->base, src->nr_cids, CMASK_OP2_COPY_RACY);
-}
-
 void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src)
 {
 	cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
@@ -680,13 +890,13 @@ __bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit,
 
 __bpf_kfunc_end_defs();
 
-BTF_KFUNCS_START(scx_kfunc_ids_init)
+BTF_KFUNCS_START(scx_kfunc_ids_init_cids)
 BTF_ID_FLAGS(func, scx_bpf_cid_override, KF_IMPLICIT_ARGS | KF_SLEEPABLE)
-BTF_KFUNCS_END(scx_kfunc_ids_init)
+BTF_KFUNCS_END(scx_kfunc_ids_init_cids)
 
-static const struct btf_kfunc_id_set scx_kfunc_set_init = {
+static const struct btf_kfunc_id_set scx_kfunc_set_init_cids = {
 	.owner	= THIS_MODULE,
-	.set	= &scx_kfunc_ids_init,
+	.set	= &scx_kfunc_ids_init_cids,
 	.filter	= scx_kfunc_context_filter,
 };
 
@@ -701,9 +911,200 @@ static const struct btf_kfunc_id_set scx_kfunc_set_cid = {
 	.set	= &scx_kfunc_ids_cid,
 };
 
+/**
+ * scx_cmask_ref_init - Bind a scx_cmask_ref to a BPF-arena cmask
+ * @sch: scheduler whose arena hosts @src
+ * @src: BPF-supplied cmask pointer
+ * @ref: output ref
+ *
+ * Snapshot @src's @base, @nr_cids and @alloc_words. The snapshot is necessary
+ * because BPF may mutate the live header asynchronously.
+ *
+ * Return 0 on success, -EINVAL if the range is out of bounds or @alloc_words
+ * doesn't cover it.
+ */
+int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src,
+		       struct scx_cmask_ref *ref)
+{
+	struct scx_cmask *kern_src = scx_arena_to_kaddr(sch, src);
+	u32 base, nr_cids, alloc_words, npossible = num_possible_cpus();
+
+	base = READ_ONCE(kern_src->base);
+	nr_cids = READ_ONCE(kern_src->nr_cids);
+	alloc_words = READ_ONCE(kern_src->alloc_words);
+
+	if (unlikely(base >= npossible || nr_cids > npossible - base ||
+		     SCX_CMASK_NR_WORDS(nr_cids) > alloc_words))
+		return -EINVAL;
+
+	ref->sch = sch;
+	ref->src = kern_src;
+	ref->base = base;
+	ref->nr_cids = nr_cids;
+
+	ref->shard_first = scx_cid_to_shard[base];
+	if (likely(nr_cids))
+		ref->shard_end = scx_cid_to_shard[base + nr_cids - 1] + 1;
+	else
+		ref->shard_end = ref->shard_first;
+
+	return 0;
+}
+
+/**
+ * scx_cmask_ref_init_kern - Bind a scx_cmask_ref to a kernel-owned cmask
+ * @sch: scheduler the cmask belongs to
+ * @m: kernel address of the target cmask, storage sized for @nr_cids at @base
+ * @base: first cid of the active range
+ * @nr_cids: active range length
+ * @ref: output ref
+ *
+ * Like scx_cmask_ref_init() but the geometry is supplied by the caller, not
+ * read from @m's header, so a concurrent BPF write to the header can't steer
+ * later sizing or offsets. Rewrite the header from the trusted geometry and
+ * bind @ref to it.
+ */
+void scx_cmask_ref_init_kern(struct scx_sched *sch, struct scx_cmask *m,
+			     u32 base, u32 nr_cids, struct scx_cmask_ref *ref)
+{
+	WRITE_ONCE(m->base, base);
+	WRITE_ONCE(m->nr_cids, nr_cids);
+	WRITE_ONCE(m->alloc_words, SCX_CMASK_NR_WORDS(nr_cids));
+
+	ref->sch = sch;
+	ref->src = m;
+	ref->base = base;
+	ref->nr_cids = nr_cids;
+
+	ref->shard_first = scx_cid_to_shard[base];
+	if (likely(nr_cids))
+		ref->shard_end = scx_cid_to_shard[base + nr_cids - 1] + 1;
+	else
+		ref->shard_end = ref->shard_first;
+}
+
+/**
+ * scx_cmask_ref_shard - Read one shard from @ref into @out
+ * @ref: validated ref
+ * @shard_idx: target shard, in [@ref->shard_first, @ref->shard_end)
+ * @out: output cmask whose @out->alloc_words must hold the shard
+ *
+ * Set @out to the intersection of @ref's range with @shard_idx's cid range,
+ * with bits[] read from @ref->src via READ_ONCE. Empty intersection sets
+ * @out->nr_cids to 0. scx_error()s on @ref's sched if @out can't hold the
+ * shard.
+ */
+void scx_cmask_ref_shard(const struct scx_cmask_ref *ref, s32 shard_idx,
+			 struct scx_cmask *out)
+{
+	const struct scx_cid_shard *shard = &scx_cid_shard_ranges[shard_idx];
+	u32 shard_base = shard->base_cid;
+	u32 shard_end = shard_base + shard->nr_cids;
+	u32 isect_base, isect_end, nr_words, src_off, wi;
+	u64 head_mask, tail_mask;
+
+	isect_base = max(ref->base, shard_base);
+	isect_end = min(ref->base + ref->nr_cids, shard_end);
+
+	if (isect_base >= isect_end) {
+		out->base = shard_base;
+		out->nr_cids = 0;
+		return;
+	}
+
+	nr_words = ((isect_end - 1) / 64) - (isect_base / 64) + 1;
+	if (nr_words > out->alloc_words) {
+		scx_error(ref->sch, "scx_cmask_ref_shard: out alloc_words=%u < %u for shard %d",
+			  out->alloc_words, nr_words, shard_idx);
+		out->base = shard_base;
+		out->nr_cids = 0;
+		return;
+	}
+
+	out->base = isect_base;
+	out->nr_cids = isect_end - isect_base;
+	src_off = (isect_base / 64) - (ref->base / 64);
+
+	for (wi = 0; wi < nr_words; wi++)
+		out->bits[wi] = READ_ONCE(ref->src->bits[src_off + wi]);
+
+	head_mask = GENMASK_U64(63, isect_base & 63);
+	out->bits[0] &= head_mask;
+	tail_mask = GENMASK_U64((isect_end - 1) & 63, 0);
+	out->bits[nr_words - 1] &= tail_mask;
+}
+
+/**
+ * scx_cmask_ref_or - OR @src into the arena cmask referenced by @ref
+ * @ref: validated ref
+ * @src: stable kernel cmask
+ *
+ * Bits inside the intersection of @ref's snapshotted range with @src's range
+ * are OR'd into @ref->src and bits outside are left unchanged. Stores on
+ * @ref->src use WRITE_ONCE since BPF may read/write concurrently.
+ */
+void scx_cmask_ref_or(const struct scx_cmask_ref *ref, const struct scx_cmask *src)
+{
+	cmask_walk_op2(ref->src->bits, ref->base, ref->nr_cids,
+		       src->bits, src->base, src->nr_cids, CMASK_OP2_REF_OR);
+}
+
+/**
+ * scx_cmask_ref_copy - Copy @src into the arena cmask referenced by @ref
+ * @ref: validated ref
+ * @src: stable kernel cmask
+ *
+ * Bits inside the intersection of @ref's snapshotted range with @src's range
+ * take @src's values and bits outside are left unchanged. Stores on @ref->src
+ * use WRITE_ONCE since BPF may read/write concurrently.
+ */
+void scx_cmask_ref_copy(const struct scx_cmask_ref *ref, const struct scx_cmask *src)
+{
+	cmask_walk_op2(ref->src->bits, ref->base, ref->nr_cids,
+		       src->bits, src->base, src->nr_cids, CMASK_OP2_REF_COPY);
+}
+
+/**
+ * scx_cmask_ref_from_cpumask - Populate @ref's arena cmask from a cpumask
+ * @ref: kern-bound ref, see scx_cmask_ref_init_kern()
+ * @cpumask: cpus to translate into cids
+ *
+ * Write @ref's active range one word at a time, setting each cid's bit when
+ * its cpu is in @cpumask. Offsets and length come from @ref's trusted geometry
+ * and stores use WRITE_ONCE since BPF may read concurrently, so the arena
+ * header is never read.
+ */
+void scx_cmask_ref_from_cpumask(const struct scx_cmask_ref *ref,
+				const struct cpumask *cpumask)
+{
+	struct scx_cmask *m = ref->src;
+	u32 base = ref->base, nr_cids = ref->nr_cids;
+	u32 wi, nr_words;
+
+	if (!nr_cids)
+		return;
+
+	nr_words = (base + nr_cids - 1) / 64 - base / 64 + 1;
+	for (wi = 0; wi < nr_words; wi++) {
+		u32 word_first_cid = (base / 64 + wi) * 64;
+		u64 word = 0;
+		u32 bit;
+
+		for (bit = 0; bit < 64; bit++) {
+			u32 cid = word_first_cid + bit;
+
+			if (cid < base || cid >= base + nr_cids)
+				continue;
+			if (cpumask_test_cpu(__scx_cid_to_cpu(cid), cpumask))
+				word |= BIT_U64(bit);
+		}
+		WRITE_ONCE(m->bits[wi], word);
+	}
+}
+
 int scx_cid_kfunc_init(void)
 {
-	return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init) ?:
+	return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init_cids) ?:
 		register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_cid) ?:
 		register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_cid) ?:
 		register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_cid);
diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h
index 9c4f4b907f12d..b36a1a28eac85 100644
--- a/kernel/sched/ext/cid.h
+++ b/kernel/sched/ext/cid.h
@@ -48,18 +48,20 @@ struct scx_sched;
  * See the comment above the table definitions in cid.c for the
  * memory-ordering and visibility contract.
  */
+extern u32 scx_nr_cid_shards;
 extern s16 *scx_cid_to_cpu_tbl;
 extern s16 *scx_cpu_to_cid_tbl;
+extern s32 *scx_cid_to_shard;
+extern s32 *scx_shard_node;
+extern struct scx_cid_shard *scx_cid_shard_ranges;
 extern struct scx_cid_topo *scx_cid_topo;
-extern struct btf_id_set8 scx_kfunc_ids_init;
+extern struct btf_id_set8 scx_kfunc_ids_init_cids;
 
 void scx_cmask_clear(struct scx_cmask *m);
 void scx_cmask_fill(struct scx_cmask *m);
 void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src);
 void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src);
-void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src);
 void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src);
-void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src);
 void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src);
 bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super);
 bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b);
@@ -291,4 +293,15 @@ static inline s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid)
 	return scx_cid_to_cpu(sch, cpu_or_cid);
 }
 
+int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src,
+		       struct scx_cmask_ref *ref);
+void scx_cmask_ref_init_kern(struct scx_sched *sch, struct scx_cmask *m,
+			     u32 base, u32 nr_cids, struct scx_cmask_ref *ref);
+void scx_cmask_ref_shard(const struct scx_cmask_ref *ref, s32 shard_idx,
+			 struct scx_cmask *out);
+void scx_cmask_ref_from_cpumask(const struct scx_cmask_ref *ref,
+				const struct cpumask *cpumask);
+void scx_cmask_ref_or(const struct scx_cmask_ref *ref, const struct scx_cmask *src);
+void scx_cmask_ref_copy(const struct scx_cmask_ref *ref, const struct scx_cmask *src);
+
 #endif /* _KERNEL_SCHED_EXT_CID_H */
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 691d53fe0f648..7fcfcc43462de 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -19,8 +19,10 @@
 #include "cid.h"
 #include "arena.h"
 #include "idle.h"
+#include "sub.h"
+#include "inlines.h"
 
-static DEFINE_RAW_SPINLOCK(scx_sched_lock);
+DEFINE_RAW_SPINLOCK(scx_sched_lock);
 
 /*
  * NOTE: sched_ext is in the process of growing multiple scheduler support and
@@ -39,14 +41,14 @@ struct scx_sched __rcu *scx_root;
 static LIST_HEAD(scx_sched_all);
 
 #ifdef CONFIG_EXT_SUB_SCHED
-static const struct rhashtable_params scx_sched_hash_params = {
+const struct rhashtable_params scx_sched_hash_params = {
 	.key_len		= sizeof_field(struct scx_sched, ops.sub_cgroup_id),
 	.key_offset		= offsetof(struct scx_sched, ops.sub_cgroup_id),
 	.head_offset		= offsetof(struct scx_sched, hash_node),
 	.insecure_elasticity	= true,	/* inserted under scx_sched_lock */
 };
 
-static struct rhashtable scx_sched_hash;
+struct rhashtable scx_sched_hash;
 #endif
 
 /* see SCX_OPS_TID_TO_TASK */
@@ -68,9 +70,9 @@ static DEFINE_RAW_SPINLOCK(scx_tasks_lock);
 static LIST_HEAD(scx_tasks);
 
 /* ops enable/disable */
-static DEFINE_MUTEX(scx_enable_mutex);
+DEFINE_MUTEX(scx_enable_mutex);
 DEFINE_STATIC_KEY_FALSE(__scx_enabled);
-DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem);
+DEFINE_PERCPU_RWSEM(scx_fork_rwsem);
 static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED);
 static DEFINE_RAW_SPINLOCK(scx_bypass_lock);
 static bool scx_init_task_enabled;
@@ -95,13 +97,28 @@ static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0);
 /* Global cursor for the per-CPU tid allocator. Starts at 1; tid 0 is reserved. */
 static atomic64_t scx_tid_cursor = ATOMIC64_INIT(1);
 
+/* is @dsq synchronized by the containing rq lock instead of dsq->lock? */
+static bool dsq_is_rq_owned(struct scx_dispatch_q *dsq)
+{
+	switch (dsq->id) {
+	case SCX_DSQ_LOCAL:
+	case SCX_DSQ_REJECT:
+		return true;
+	default:
+		return false;
+	}
+}
+
+/* Cursor for unique scx_sched instance ids. id 0 is reserved. */
+static atomic64_t scx_sched_id_cursor = ATOMIC64_INIT(0);
+
 #ifdef CONFIG_EXT_SUB_SCHED
 /*
  * The sub sched being enabled. Used by scx_disable_and_exit_task() to exit
  * tasks for the sub-sched being enabled. Use a global variable instead of a
  * per-task field as all enables are serialized.
  */
-static struct scx_sched *scx_enabling_sub_sched;
+struct scx_sched *scx_enabling_sub_sched;
 #else
 #define scx_enabling_sub_sched	(struct scx_sched *)NULL
 #endif	/* CONFIG_EXT_SUB_SCHED */
@@ -171,14 +188,8 @@ static const struct rhashtable_params dsq_hash_params = {
 
 static LLIST_HEAD(dsqs_to_free);
 
-/* string formatting from BPF */
-struct scx_bstr_buf {
-	u64			data[MAX_BPRINTF_VARARGS];
-	char			line[SCX_EXIT_MSG_LEN];
-};
-
-static DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock);
-static struct scx_bstr_buf scx_exit_bstr_buf;
+DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock);
+struct scx_bstr_buf scx_exit_bstr_buf;
 
 /* ops debug dump */
 static DEFINE_RAW_SPINLOCK(scx_dump_lock);
@@ -242,7 +253,6 @@ MODULE_PARM_DESC(bypass_lb_intv_us, "bypass load balance interval in microsecond
 
 static void run_deferred(struct rq *rq);
 static bool task_dead_and_done(struct task_struct *p);
-static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags);
 static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind);
 
 __printf(5, 6) bool __scx_exit(struct scx_sched *sch,
@@ -272,58 +282,6 @@ static bool u32_before(u32 a, u32 b)
 	return (s32)(a - b) < 0;
 }
 
-#ifdef CONFIG_EXT_SUB_SCHED
-/**
- * scx_next_descendant_pre - find the next descendant for pre-order walk
- * @pos: the current position (%NULL to initiate traversal)
- * @root: sched whose descendants to walk
- *
- * To be used by scx_for_each_descendant_pre(). Find the next descendant to
- * visit for pre-order traversal of @root's descendants. @root is included in
- * the iteration and the first node to be visited.
- */
-static struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos,
-						 struct scx_sched *root)
-{
-	struct scx_sched *next;
-
-	lockdep_assert(lockdep_is_held(&scx_enable_mutex) ||
-		       lockdep_is_held(&scx_sched_lock));
-
-	/* if first iteration, visit @root */
-	if (!pos)
-		return root;
-
-	/* visit the first child if exists */
-	next = list_first_entry_or_null(&pos->children, struct scx_sched, sibling);
-	if (next)
-		return next;
-
-	/* no child, visit my or the closest ancestor's next sibling */
-	while (pos != root) {
-		if (!list_is_last(&pos->sibling, &scx_parent(pos)->children))
-			return list_next_entry(pos, sibling);
-		pos = scx_parent(pos);
-	}
-
-	return NULL;
-}
-
-static struct scx_sched *scx_find_sub_sched(u64 cgroup_id)
-{
-	return rhashtable_lookup(&scx_sched_hash, &cgroup_id,
-				 scx_sched_hash_params);
-}
-
-static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch)
-{
-	rcu_assign_pointer(p->scx.sched, sch);
-}
-#else	/* CONFIG_EXT_SUB_SCHED */
-static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }
-static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {}
-#endif	/* CONFIG_EXT_SUB_SCHED */
-
 /**
  * scx_is_descendant - Test whether sched is a descendant
  * @sch: sched to test
@@ -331,26 +289,13 @@ static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *s
  *
  * Test whether @sch is a descendant of @ancestor.
  */
-static bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor)
+bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor)
 {
 	if (sch->level < ancestor->level)
 		return false;
 	return sch->ancestors[ancestor->level] == ancestor;
 }
 
-/**
- * scx_for_each_descendant_pre - pre-order walk of a sched's descendants
- * @pos: iteration cursor
- * @root: sched to walk the descendants of
- *
- * Walk @root's descendants. @root is included in the iteration and the first
- * node to be visited. Must be called with either scx_enable_mutex or
- * scx_sched_lock held.
- */
-#define scx_for_each_descendant_pre(pos, root)					\
-	for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos);		\
-	     (pos) = scx_next_descendant_pre((pos), (root)))
-
 static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch, s32 cpu)
 {
 	return &sch->pnode[cpu_to_node(cpu)]->global_dsq;
@@ -369,11 +314,6 @@ static const struct sched_class *scx_setscheduler_class(struct task_struct *p)
 	return __setscheduler_class(p->policy, p->prio);
 }
 
-static struct scx_dispatch_q *bypass_dsq(struct scx_sched *sch, s32 cpu)
-{
-	return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq;
-}
-
 static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu)
 {
 #ifdef CONFIG_EXT_SUB_SCHED
@@ -392,27 +332,7 @@ static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 c
 		sch = scx_parent(sch);
 #endif	/* CONFIG_EXT_SUB_SCHED */
 
-	return bypass_dsq(sch, cpu);
-}
-
-/**
- * bypass_dsp_enabled - Check if bypass dispatch path is enabled
- * @sch: scheduler to check
- *
- * When a descendant scheduler enters bypass mode, bypassed tasks are scheduled
- * by the nearest non-bypassing ancestor, or the root scheduler if all ancestors
- * are bypassing. In the former case, the ancestor is not itself bypassing but
- * its bypass DSQs will be populated with bypassed tasks from descendants. Thus,
- * the ancestor's bypass dispatch path must be active even though its own
- * bypass_depth remains zero.
- *
- * This function checks bypass_dsp_enable_depth which is managed separately from
- * bypass_depth to enable this decoupling. See enable_bypass_dsp() and
- * disable_bypass_dsp().
- */
-static bool bypass_dsp_enabled(struct scx_sched *sch)
-{
-	return unlikely(atomic_read(&sch->bypass_dsp_enable_depth));
+	return scx_bypass_dsq(sch, cpu);
 }
 
 /**
@@ -479,6 +399,18 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags)
  */
 DEFINE_PER_CPU(struct rq *, scx_locked_rq_state);
 
+static void switch_rq_lock(struct rq *from, struct rq *to)
+{
+	bool tracked = scx_locked_rq() == from;
+
+	if (tracked)
+		update_locked_rq(NULL);
+	raw_spin_rq_unlock(from);
+	raw_spin_rq_lock(to);
+	if (tracked)
+		update_locked_rq(to);
+}
+
 /*
  * Flipped on enable per sch->is_cid_type. Declared in internal.h so
  * subsystem inlines can read it.
@@ -500,27 +432,22 @@ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq,
 					   struct task_struct *task,
 					   const struct cpumask *cpumask)
 {
-	WARN_ON_ONCE(current->scx.kf_tasks[0]);
-	current->scx.kf_tasks[0] = task;
-	if (rq)
-		update_locked_rq(rq);
-
 	if (scx_is_cid_type()) {
 		struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch);
+		struct scx_cmask_ref ref;
+
 		/*
-		 * Build the per-CPU arena cmask and hand BPF its arena address.
-		 * Caller holds the rq lock with IRQs disabled, which makes us
-		 * the sole user of the scratch area.
+		 * Build the per-cpu arena cmask from kernel geometry via @ref,
+		 * never reading its BPF-writable header, and hand BPF the arena
+		 * address. The rq lock makes this cpu the sole kernel writer.
 		 */
-		scx_cpumask_to_cmask(cpumask, kern_va);
-		sch->ops_cid.set_cmask(task, scx_kaddr_to_arena(sch, kern_va));
+		scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref);
+		scx_cmask_ref_from_cpumask(&ref, cpumask);
+		SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task,
+				     scx_kaddr_to_arena(sch, kern_va));
 	} else {
-		sch->ops.set_cpumask(task, cpumask);
+		SCX_CALL_OP_TASK(sch, set_cpumask, rq, task, cpumask);
 	}
-
-	if (rq)
-		update_locked_rq(NULL);
-	current->scx.kf_tasks[0] = NULL;
 }
 
 enum scx_dsq_iter_flags {
@@ -676,12 +603,12 @@ struct bpf_iter_scx_dsq {
 } __attribute__((aligned(8)));
 
 
-static u32 scx_get_task_state(const struct task_struct *p)
+u32 scx_get_task_state(const struct task_struct *p)
 {
 	return p->scx.flags & SCX_TASK_STATE_MASK;
 }
 
-static void scx_set_task_state(struct task_struct *p, u32 state)
+void scx_set_task_state(struct task_struct *p, u32 state)
 {
 	u32 prev_state = scx_get_task_state(p);
 	bool warn = false;
@@ -721,23 +648,6 @@ static void scx_set_task_state(struct task_struct *p, u32 state)
 	p->scx.flags |= state;
 }
 
-/*
- * SCX task iterator.
- */
-struct scx_task_iter {
-	struct sched_ext_entity		cursor;
-	struct task_struct		*locked_task;
-	struct rq			*rq;
-	struct rq_flags			rf;
-	u32				cnt;
-	bool				list_locked;
-#ifdef CONFIG_EXT_SUB_SCHED
-	struct cgroup			*cgrp;
-	struct cgroup_subsys_state	*css_pos;
-	struct css_task_iter		css_iter;
-#endif
-};
-
 /**
  * scx_task_iter_start - Lock scx_tasks_lock and start a task iteration
  * @iter: iterator to init
@@ -766,7 +676,7 @@ struct scx_task_iter {
  * All tasks which existed when the iteration started are guaranteed to be
  * visited as long as they are not dead.
  */
-static void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp)
+void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp)
 {
 	memset(iter, 0, sizeof(*iter));
 
@@ -805,7 +715,7 @@ static void __scx_task_iter_rq_unlock(struct scx_task_iter *iter)
  * This function can be safely called anytime during an iteration. The next
  * iterator operation will automatically restore the necessary locking.
  */
-static void scx_task_iter_unlock(struct scx_task_iter *iter)
+void scx_task_iter_unlock(struct scx_task_iter *iter)
 {
 	__scx_task_iter_rq_unlock(iter);
 	if (iter->list_locked) {
@@ -848,7 +758,7 @@ static void scx_task_iter_relock(struct scx_task_iter *iter,
  * which is released on return. If the iterator holds a task's rq lock, that rq
  * lock is also released. See scx_task_iter_start() for details.
  */
-static void scx_task_iter_stop(struct scx_task_iter *iter)
+void scx_task_iter_stop(struct scx_task_iter *iter)
 {
 #ifdef CONFIG_EXT_SUB_SCHED
 	if (iter->cgrp) {
@@ -923,7 +833,7 @@ static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter)
  * whether they would like to filter out dead tasks. See scx_task_iter_start()
  * for details.
  */
-static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter)
+struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter)
 {
 	struct task_struct *p;
 
@@ -978,42 +888,6 @@ static struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter)
 	return NULL;
 }
 
-/**
- * scx_add_event - Increase an event counter for 'name' by 'cnt'
- * @sch: scx_sched to account events for
- * @name: an event name defined in struct scx_event_stats
- * @cnt: the number of the event occurred
- *
- * This can be used when preemption is not disabled.
- */
-#define scx_add_event(sch, name, cnt) do {					\
-	this_cpu_add((sch)->pcpu->event_stats.name, (cnt));			\
-	trace_sched_ext_event(#name, (cnt));					\
-} while(0)
-
-/**
- * __scx_add_event - Increase an event counter for 'name' by 'cnt'
- * @sch: scx_sched to account events for
- * @name: an event name defined in struct scx_event_stats
- * @cnt: the number of the event occurred
- *
- * This should be used only when preemption is disabled.
- */
-#define __scx_add_event(sch, name, cnt) do {					\
-	__this_cpu_add((sch)->pcpu->event_stats.name, (cnt));			\
-	trace_sched_ext_event(#name, cnt);					\
-} while(0)
-
-/**
- * scx_agg_event - Aggregate an event counter 'kind' from 'src_e' to 'dst_e'
- * @dst_e: destination event stats
- * @src_e: source event stats
- * @kind: a kind of event to be aggregated
- */
-#define scx_agg_event(dst_e, src_e, kind) do {					\
-	(dst_e)->kind += READ_ONCE((src_e)->kind);				\
-} while(0)
-
 /**
  * scx_dump_event - Dump an event 'kind' in 'events' to 's'
  * @s: output seq_buf
@@ -1088,28 +962,6 @@ bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where)
 	}
 }
 
-/**
- * ops_sanitize_err - Sanitize a -errno value
- * @sch: scx_sched to error out on error
- * @ops_name: operation to blame on failure
- * @err: -errno value to sanitize
- *
- * Verify @err is a valid -errno. If not, trigger scx_error() and return
- * -%EPROTO. This is necessary because returning a rogue -errno up the chain can
- * cause misbehaviors. For an example, a large negative return from
- * ops.init_task() triggers an oops when passed up the call chain because the
- * value fails IS_ERR() test after being encoded with ERR_PTR() and then is
- * handled as a pointer.
- */
-static int ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err)
-{
-	if (err < 0 && err >= -MAX_ERRNO)
-		return err;
-
-	scx_error(sch, "ops.%s() returned an invalid errno %d", ops_name, err);
-	return -EPROTO;
-}
-
 static void deferred_bal_cb_workfn(struct rq *rq)
 {
 	run_deferred(rq);
@@ -1196,8 +1048,8 @@ static void schedule_deferred_locked(struct rq *rq)
 	schedule_deferred(rq);
 }
 
-static void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
-			       u64 reenq_flags, struct rq *locked_rq)
+void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
+			u64 reenq_flags, struct rq *locked_rq)
 {
 	struct rq *rq;
 
@@ -1261,16 +1113,6 @@ static void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq
 		schedule_deferred(rq);
 }
 
-static void schedule_reenq_local(struct rq *rq, u64 reenq_flags)
-{
-	struct scx_sched *root = rcu_dereference_sched(scx_root);
-
-	if (WARN_ON_ONCE(!root))
-		return;
-
-	schedule_dsq_reenq(root, &rq->scx.local_dsq, reenq_flags, rq);
-}
-
 /**
  * touch_core_sched - Update timestamp used for core-sched task ordering
  * @rq: rq to read clock from, must be locked
@@ -1318,11 +1160,145 @@ static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p)
 #endif
 }
 
+/*
+ * p->scx.slice_oob packs an out-of-band slice request into one atomic64. A zero
+ * word means no request. Otherwise the fields are:
+ *
+ *   63      SCX_SLICE_OOB_PENDING, set on every request
+ *   62-43   lower bits of issuing scheduler's id
+ *   42-0    requested slice duration in nsecs
+ *
+ * A duration of SCX_SLICE_OOB_DUR_MASK means SCX_SLICE_INF. A finite dur
+ * saturates at SCX_SLICE_OOB_DUR_MASK - 1. The id is used to detect and ignore
+ * a request that outlived a task ownership change.
+ *
+ * Only the low 20 bits of sch->id are packed, which is enough to make
+ * collisions practically impossible. A theoretical collision just lets a stale
+ * request through once.
+ */
+enum scx_slice_oob_consts {
+	SCX_SLICE_OOB_DUR_BITS	= 43,
+	SCX_SLICE_OOB_ID_BITS	= 64 - SCX_SLICE_OOB_DUR_BITS - 1,
+
+	SCX_SLICE_OOB_DUR_MASK	= (1LLU << SCX_SLICE_OOB_DUR_BITS) - 1,
+	SCX_SLICE_OOB_ID_SHIFT	= SCX_SLICE_OOB_DUR_BITS,
+	SCX_SLICE_OOB_ID_MASK	= (1LLU << SCX_SLICE_OOB_ID_BITS) - 1,
+	SCX_SLICE_OOB_PENDING	= 1LLU << 63,
+};
+
+/*
+ * Slice write rules
+ *
+ * A task's slice - how long it may hold its cpu - is an occupancy grant owned
+ * by the task's scheduler. How it may be written depends on whether the task is
+ * running.
+ *
+ * Queued, not running: the slice grants no occupancy yet and nothing consumes
+ * it, so the owner writes it directly - via scx_bpf_dsq_insert(), the dsq move
+ * kfuncs, or scx_bpf_task_set_slice(). Serializing its own writers is then the
+ * scheduler's job, not the kernel's.
+ *
+ * Running: the slice must be changed under the task's rq lock, because:
+ *
+ * - Raising it extends occupancy, allowed only with %SCX_CAP_BASE on the cpu,
+ *   and that cap check is coherent only under the rq lock. Shortening is always
+ *   allowed.
+ *
+ * - The kernel decrements it there as the task runs. The decrement is a
+ *   read-modify-write, so a racing write can be clobbered.
+ *
+ * scx_bpf_task_set_slice() writes directly only when @p is queued or running
+ * on the rq lock it holds. That is the only state where the lock keeps us @p's
+ * owner: @p can't move to another rq without it. A task that isn't queued here
+ * can instead be woken onto a different rq without taking this lock, and that
+ * dispatch sets its slice - so a direct write would race. Those cases stash
+ * into p->scx.slice_oob to be applied under @p's actual rq lock. A later in-band
+ * write supersedes a stash, and a stash whose scheduler id no longer matches
+ * @p's owner is dropped.
+ */
+
+/* clear a pending slice request */
+static void clear_task_slice_oob(struct task_struct *p)
+{
+	if (unlikely(atomic64_read(&p->scx.slice_oob)))
+		atomic64_set(&p->scx.slice_oob, 0);
+}
+
+/* set @p's slice, leaving any pending out-of-band request in place */
+static void set_task_slice_keep_oob(struct task_struct *p, u64 slice)
+{
+	p->scx.slice = slice;
+}
+
+/* set @p's slice, superseding any pending out-of-band request */
+static void set_task_slice(struct task_struct *p, u64 slice)
+{
+	set_task_slice_keep_oob(p, slice);
+	clear_task_slice_oob(p);
+}
+
+/* request @p's slice to be set to @slice, see the slice write rules above */
+static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 slice)
+{
+	u64 dur;
+
+	if (slice == SCX_SLICE_INF) {
+		dur = SCX_SLICE_OOB_DUR_MASK;
+	} else if (unlikely(slice >= SCX_SLICE_OOB_DUR_MASK)) {
+		dur = SCX_SLICE_OOB_DUR_MASK - 1;
+		scx_add_event(sch, SCX_EV_SLICE_CLAMPED, 1);
+	} else {
+		dur = slice;
+	}
+
+	atomic64_set(&p->scx.slice_oob, SCX_SLICE_OOB_PENDING |
+		     ((sch->id & SCX_SLICE_OOB_ID_MASK) << SCX_SLICE_OOB_ID_SHIFT) | dur);
+}
+
+/*
+ * Apply a pending out-of-band slice request under @rq's lock. A request whose
+ * packed id no longer matches @p's current owner is dropped. An extension needs
+ * baseline cpu access on @p's cid. %SCX_EV_SLICE_DENIED counts the denials.
+ * Shortening is always allowed. See the slice write rules above.
+ */
+static void apply_task_slice_oob(struct rq *rq, struct task_struct *p)
+{
+	u64 oob, dur, slice;
+
+	lockdep_assert_rq_held(rq);
+
+	if (likely(!atomic64_read(&p->scx.slice_oob)))
+		return;
+
+	oob = atomic64_xchg(&p->scx.slice_oob, 0);
+	if (unlikely(!oob))
+		return;
+
+	/* the issuing scheduler no longer owns @p, drop the request */
+	if (unlikely(((oob >> SCX_SLICE_OOB_ID_SHIFT) & SCX_SLICE_OOB_ID_MASK) !=
+		     (scx_task_sched(p)->id & SCX_SLICE_OOB_ID_MASK)))
+		return;
+
+	dur = oob & SCX_SLICE_OOB_DUR_MASK;
+	slice = dur == SCX_SLICE_OOB_DUR_MASK ? SCX_SLICE_INF : dur;
+
+	if (slice > p->scx.slice &&
+	    unlikely(scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE))) {
+		__scx_add_event(scx_task_sched(p), SCX_EV_SLICE_DENIED, 1);
+		return;
+	}
+
+	p->scx.slice = slice;
+}
+
 static void update_curr_scx(struct rq *rq)
 {
 	struct task_struct *curr = rq->curr;
 	s64 delta_exec;
 
+	/* apply even on 0 delta_exec, callers may still act on the slice */
+	apply_task_slice_oob(rq, curr);
+
 	delta_exec = update_curr_common(rq);
 	if (unlikely(delta_exec <= 0))
 		return;
@@ -1357,8 +1333,8 @@ static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 en
 	 * to the CPU or dequeued. In both cases, the only way @p can go back to
 	 * the BPF sched is through enqueueing. If being inserted into a local
 	 * DSQ with IMMED, persist the state until the next enqueueing event in
-	 * do_enqueue_task() so that we can maintain IMMED protection through
-	 * e.g. SAVE/RESTORE cycles and slice extensions.
+	 * scx_do_enqueue_task() so that we can maintain IMMED protection
+	 * through e.g. SAVE/RESTORE cycles and slice extensions.
 	 */
 	if (enq_flags & SCX_ENQ_IMMED) {
 		if (unlikely(dsq->id != SCX_DSQ_LOCAL)) {
@@ -1381,7 +1357,7 @@ static void dsq_inc_nr(struct scx_dispatch_q *dsq, struct task_struct *p, u64 en
 		 * done yet, @p can't go on the CPU immediately. Re-enqueue.
 		 */
 		if (unlikely(dsq->nr > 1 || !rq_is_open(rq, enq_flags)))
-			schedule_reenq_local(rq, 0);
+			scx_schedule_reenq_local(rq, 0);
 	}
 }
 
@@ -1403,7 +1379,11 @@ static void dsq_dec_nr(struct scx_dispatch_q *dsq, struct task_struct *p)
 
 static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p)
 {
-	p->scx.slice = READ_ONCE(sch->slice_dfl);
+	/*
+	 * A default refill is not an explicit request, so it must not drop a
+	 * pending out-of-band one, which is applied when @p next runs.
+	 */
+	set_task_slice_keep_oob(p, READ_ONCE(sch->slice_dfl));
 	__scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1);
 }
 
@@ -1438,13 +1418,18 @@ static void call_task_dequeue(struct scx_sched *sch, struct rq *rq,
 	p->scx.flags &= ~SCX_TASK_IN_CUSTODY;
 }
 
-static void local_dsq_post_enq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
-			       struct task_struct *p, u64 enq_flags)
+static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq,
+			      struct scx_dispatch_q *dsq, struct task_struct *p,
+			      u64 enq_flags)
 {
-	struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
-
 	call_task_dequeue(sch, rq, p, 0);
 
+	/* rejected: kick the deferred reenq, skip wakeup/preemption */
+	if (unlikely(dsq->id == SCX_DSQ_REJECT)) {
+		schedule_deferred_locked(rq);
+		return;
+	}
+
 	/*
 	 * Note that @rq's lock may be dropped between this enqueue and @p
 	 * actually getting on CPU. This gives higher-class tasks (e.g. RT)
@@ -1493,22 +1478,27 @@ static void local_dsq_post_enq(struct scx_sched *sch, struct scx_dispatch_q *dsq
 
 	if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&
 	    rq->curr->sched_class == &ext_sched_class) {
-		rq->curr->scx.slice = 0;
+		set_task_slice(rq->curr, 0);
 		resched_curr(rq);
 	}
 }
 
-static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
-			     struct scx_dispatch_q *dsq, struct task_struct *p,
-			     u64 enq_flags)
+static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
+				 struct scx_dispatch_q *dsq, struct task_struct *p,
+				 u64 enq_flags)
 {
-	bool is_local = dsq->id == SCX_DSQ_LOCAL;
+	bool is_rq_owned = false;
+
+	if (dsq->id == SCX_DSQ_LOCAL) {
+		dsq = scx_local_or_reject_dsq(sch, rq, p, &enq_flags);
+		is_rq_owned = true;
+	}
 
 	WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node));
 	WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) ||
 		     !RB_EMPTY_NODE(&p->scx.dsq_priq));
 
-	if (!is_local) {
+	if (!is_rq_owned) {
 		raw_spin_lock_nested(&dsq->lock,
 			(enq_flags & SCX_ENQ_NESTED) ? SINGLE_DEPTH_NESTING : 0);
 
@@ -1603,12 +1593,16 @@ static void dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
 	 * ops_state first, both sides would modify p->scx.flags
 	 * concurrently in a non-atomic way.
 	 */
-	if (is_local) {
-		local_dsq_post_enq(sch, dsq, p, enq_flags);
+	if (is_rq_owned) {
+		rq_owned_post_enq(sch, rq, dsq, p, enq_flags);
 	} else {
 		/*
-		 * Task on global/bypass DSQ: leave custody, task on
-		 * non-terminal DSQ: enter custody.
+		 * Global and bypass DSQs are terminal - the task leaves the
+		 * scheduler's custody, so ops.dequeue() fires here. It can run
+		 * without @p's rq lock (finish_dispatch() passes the dispatch
+		 * rq); that's safe because dequeue_task_scx() waits on
+		 * SCX_OPSS_DISPATCHING (see the ops_state note above) and so
+		 * can't race it. A non-terminal DSQ keeps the task in custody.
 		 */
 		if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS)
 			call_task_dequeue(sch, rq, p, 0);
@@ -1648,10 +1642,10 @@ static void task_unlink_from_dsq(struct task_struct *p,
 	}
 }
 
-static void dispatch_dequeue(struct rq *rq, struct task_struct *p)
+void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p)
 {
 	struct scx_dispatch_q *dsq = p->scx.dsq;
-	bool is_local = dsq == &rq->scx.local_dsq;
+	bool is_rq_owned = dsq && dsq_is_rq_owned(dsq);
 
 	lockdep_assert_rq_held(rq);
 
@@ -1675,7 +1669,7 @@ static void dispatch_dequeue(struct rq *rq, struct task_struct *p)
 		return;
 	}
 
-	if (!is_local)
+	if (!is_rq_owned)
 		raw_spin_lock(&dsq->lock);
 
 	/*
@@ -1697,13 +1691,13 @@ static void dispatch_dequeue(struct rq *rq, struct task_struct *p)
 	}
 	p->scx.dsq = NULL;
 
-	if (!is_local)
+	if (!is_rq_owned)
 		raw_spin_unlock(&dsq->lock);
 }
 
 /*
- * Abbreviated version of dispatch_dequeue() that can be used when both @p's rq
- * and dsq are locked.
+ * Abbreviated version of scx_dispatch_dequeue() that can be used when both
+ * @p's rq and dsq are locked.
  */
 static void dispatch_dequeue_locked(struct task_struct *p,
 				    struct scx_dispatch_q *dsq)
@@ -1738,6 +1732,10 @@ static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch,
 	else
 		dsq = find_user_dsq(sch, dsq_id);
 
+	/*
+	 * Built-in DSQs are never inserted into dsq_hash, so REJECT hits the
+	 * error below. It cannot be reached with an ID.
+	 */
 	if (unlikely(!dsq)) {
 		scx_error(sch, "non-existent DSQ 0x%llx", dsq_id);
 		return find_global_dsq(sch, tcpu);
@@ -1784,10 +1782,10 @@ static void mark_direct_dispatch(struct scx_sched *sch,
  *  - direct_dispatch(): cleared on the synchronous enqueue path, deferred
  *    dispatch keeps the state until consumed
  *  - process_ddsp_deferred_locals(): cleared after consuming deferred state,
- *  - do_enqueue_task(): cleared on enqueue fallbacks where the dispatch
+ *  - scx_do_enqueue_task(): cleared on enqueue fallbacks where the dispatch
  *    verdict is ignored (local/global/bypass)
- *  - dequeue_task_scx(): cleared after dispatch_dequeue(), covering deferred
- *    cancellation and holding_cpu races
+ *  - dequeue_task_scx(): cleared after scx_dispatch_dequeue(), covering
+ *    deferred cancellation and holding_cpu races
  *  - scx_disable_task(): cleared for queued wakeup tasks, which are excluded by
  *    the scx_bypass() loop, so that stale state is not reused by a subsequent
  *    scheduler instance
@@ -1848,10 +1846,10 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p,
 	ddsp_enq_flags = p->scx.ddsp_enq_flags;
 	clear_direct_dispatch(p);
 
-	dispatch_enqueue(sch, rq, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS);
+	scx_dispatch_enqueue(sch, rq, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS);
 }
 
-static bool scx_rq_online(struct rq *rq)
+bool scx_rq_online(struct rq *rq)
 {
 	/*
 	 * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE indicates
@@ -1863,8 +1861,8 @@ static bool scx_rq_online(struct rq *rq)
 	return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq)));
 }
 
-static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
-			    int sticky_cpu)
+void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
+			 int sticky_cpu)
 {
 	struct scx_sched *sch = scx_task_sched(p);
 	struct task_struct **ddsp_taskp;
@@ -1951,7 +1949,7 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
 	direct_dispatch(sch, p, enq_flags);
 	return;
 local_norefill:
-	dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, enq_flags);
+	scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, enq_flags);
 	return;
 local:
 	dsq = &rq->scx.local_dsq;
@@ -1972,7 +1970,7 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
 	touch_core_sched(rq, p);
 	refill_task_slice_dfl(sch, p);
 	clear_direct_dispatch(p);
-	dispatch_enqueue(sch, rq, dsq, p, enq_flags);
+	scx_dispatch_enqueue(sch, rq, dsq, p, enq_flags);
 }
 
 static bool task_runnable(const struct task_struct *p)
@@ -1994,11 +1992,19 @@ static void set_task_runnable(struct rq *rq, struct task_struct *p)
 	 * appended to the runnable_list.
 	 */
 	list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list);
+
+	/*
+	 * Record the rq @p is runnable on, maintained under the rq lock so it
+	 * stays valid unlike task_cpu(), which a remote wakeup can move under
+	 * pi_lock alone.
+	 */
+	WRITE_ONCE(p->scx.runnable_cpu, cpu_of(rq));
 }
 
 static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at)
 {
 	list_del_init(&p->scx.runnable_node);
+	WRITE_ONCE(p->scx.runnable_cpu, -1);
 	if (reset_runnable_at)
 		p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT;
 }
@@ -2007,7 +2013,7 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_
 {
 	struct scx_sched *sch = scx_task_sched(p);
 	int sticky_cpu = p->scx.sticky_cpu;
-	u64 enq_flags = core_enq_flags | rq->scx.extra_enq_flags;
+	u64 enq_flags = core_enq_flags | rq->scx.remote_activate_enq_flags;
 
 	if (enq_flags & ENQUEUE_WAKEUP)
 		rq->scx.flags |= SCX_RQ_IN_WAKEUP;
@@ -2016,10 +2022,13 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_
 	 * Restoring a running task will be immediately followed by
 	 * set_next_task_scx() which expects the task to not be on the BPF
 	 * scheduler as tasks can only start running through local DSQs. Force
-	 * direct-dispatch into the local DSQ by setting the sticky_cpu.
+	 * direct-dispatch into the local DSQ by setting the sticky_cpu. Mark
+	 * IGNORE_CAPS to force entry into the local DSQ.
 	 */
-	if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p))
+	if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) {
 		sticky_cpu = cpu_of(rq);
+		enq_flags |= SCX_ENQ_IGNORE_CAPS;
+	}
 
 	if (p->scx.flags & SCX_TASK_QUEUED) {
 		WARN_ON_ONCE(!task_runnable(p));
@@ -2041,7 +2050,7 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_
 	if (rq->scx.nr_running == 1)
 		dl_server_start(&rq->ext_server);
 
-	do_enqueue_task(rq, p, enq_flags, sticky_cpu);
+	scx_do_enqueue_task(rq, p, enq_flags, sticky_cpu);
 
 	if (sticky_cpu >= 0)
 		p->scx.sticky_cpu = -1;
@@ -2177,7 +2186,7 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_
 	rq->scx.nr_running--;
 	sub_nr_running(rq, 1);
 
-	dispatch_dequeue(rq, p);
+	scx_dispatch_dequeue(rq, p);
 	clear_direct_dispatch(p);
 	return true;
 }
@@ -2190,7 +2199,7 @@ static void yield_task_scx(struct rq *rq)
 	if (SCX_HAS_OP(sch, yield))
 		SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL);
 	else
-		p->scx.slice = 0;
+		set_task_slice(p, 0);
 }
 
 static bool yield_to_task_scx(struct rq *rq, struct task_struct *to)
@@ -2225,7 +2234,7 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl
 	 * - A higher-priority wakes up while SCX dispatch is in progress.
 	 */
 	if (rq->scx.nr_immed)
-		schedule_reenq_local(rq, 0);
+		scx_schedule_reenq_local(rq, 0);
 }
 
 static void move_local_task_to_local_dsq(struct scx_sched *sch,
@@ -2233,7 +2242,7 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch,
 					 struct scx_dispatch_q *src_dsq,
 					 struct rq *dst_rq)
 {
-	struct scx_dispatch_q *dst_dsq = &dst_rq->scx.local_dsq;
+	struct scx_dispatch_q *dst_dsq = scx_local_or_reject_dsq(sch, dst_rq, p, &enq_flags);
 
 	/* @dsq is locked and @p is on @dst_rq */
 	lockdep_assert_held(&src_dsq->lock);
@@ -2249,11 +2258,12 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch,
 	dsq_inc_nr(dst_dsq, p, enq_flags);
 	p->scx.dsq = dst_dsq;
 
-	local_dsq_post_enq(sch, dst_dsq, p, enq_flags);
+	rq_owned_post_enq(sch, dst_rq, dst_dsq, p, enq_flags);
 }
 
 /**
  * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ
+ * @sch: scheduler placing @p
  * @p: task to move
  * @enq_flags: %SCX_ENQ_*
  * @src_rq: rq to move the task from, locked on entry, released on return
@@ -2261,7 +2271,8 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch,
  *
  * Move @p which is currently on @src_rq to @dst_rq's local DSQ.
  */
-static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
+static void move_remote_task_to_local_dsq(struct scx_sched *sch,
+					  struct task_struct *p, u64 enq_flags,
 					  struct rq *src_rq, struct rq *dst_rq)
 {
 	lockdep_assert_rq_held(src_rq);
@@ -2274,19 +2285,22 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
 	deactivate_task(src_rq, p, 0);
 	set_task_cpu(p, cpu_of(dst_rq));
 
-	raw_spin_rq_unlock(src_rq);
-	raw_spin_rq_lock(dst_rq);
+	switch_rq_lock(src_rq, dst_rq);
 
 	/*
-	 * We want to pass scx-specific enq_flags but activate_task() will
-	 * truncate the upper 32 bit. As we own @rq, we can pass them through
-	 * @rq->scx.extra_enq_flags instead.
+	 * activate_task() below truncates enq_flags to 32 bits and re-derives
+	 * @p's owner, dropping our scx flags and the placing @sch. We own @rq,
+	 * so stash both across the call. The enqueue reads them back, keeping
+	 * the scx flags and checking caps against the placer, not the owner.
 	 */
 	WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr));
-	WARN_ON_ONCE(dst_rq->scx.extra_enq_flags);
-	dst_rq->scx.extra_enq_flags = enq_flags;
+	WARN_ON_ONCE(dst_rq->scx.remote_activate_enq_flags ||
+		     dst_rq->scx.remote_activate_sch);
+	dst_rq->scx.remote_activate_enq_flags = enq_flags;
+	dst_rq->scx.remote_activate_sch = sch;
 	activate_task(dst_rq, p, 0);
-	dst_rq->scx.extra_enq_flags = 0;
+	dst_rq->scx.remote_activate_enq_flags = 0;
+	dst_rq->scx.remote_activate_sch = NULL;
 }
 
 /*
@@ -2307,6 +2321,7 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
  *   no to the BPF scheduler initiated migrations while offline.
  *
  * The caller must ensure that @p and @rq are on different CPUs.
+ * If enforce == true, caller must hold @p's rq lock.
  */
 static bool task_can_run_on_remote_rq(struct scx_sched *sch,
 				      struct task_struct *p, struct rq *rq,
@@ -2314,6 +2329,14 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch,
 {
 	s32 cpu = cpu_of(rq);
 
+	/*
+	 * To prevent races with @p still running on its old CPU while switching
+	 * out, make sure we're holding @p's rq lock so as not to risk
+	 * erroneously killing the BPF scheduler.
+	 */
+	if (enforce)
+		lockdep_assert_rq_held(task_rq(p));
+
 	WARN_ON_ONCE(task_cpu(p) == cpu);
 
 	/*
@@ -2358,13 +2381,14 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch,
 }
 
 /**
- * unlink_dsq_and_lock_src_rq() - Unlink task from its DSQ and lock its task_rq
+ * unlink_dsq_and_switch_rq_lock() - Unlink task and switch to its rq lock
  * @p: target task
  * @dsq: locked DSQ @p is currently on
+ * @locked_rq: currently locked rq
  * @src_rq: rq @p is currently on, stable with @dsq locked
  *
- * Called with @dsq locked but no rq's locked. We want to move @p to a different
- * DSQ, including any local DSQ, but are not locking @src_rq. Locking @src_rq is
+ * Called with @dsq and @locked_rq locked. We want to move @p to a different DSQ,
+ * including any local DSQ, but are not locking @src_rq. Locking @src_rq is
  * required when transferring into a local DSQ. Even when transferring into a
  * non-local DSQ, it's better to use the same mechanism to protect against
  * dequeues and maintain the invariant that @p->scx.dsq can only change while
@@ -2381,43 +2405,42 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch,
  * values afterwards, as this operation can't be preempted or recurse, the
  * holding_cpu can never become this CPU again before we're done. Thus, we can
  * tell whether we lost to dequeue by testing whether the holding_cpu still
- * points to this CPU. See dispatch_dequeue() for the counterpart.
+ * points to this CPU. See scx_dispatch_dequeue() for the counterpart.
  *
  * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is
  * still valid. %false if lost to dequeue.
  */
-static bool unlink_dsq_and_lock_src_rq(struct task_struct *p,
-				       struct scx_dispatch_q *dsq,
-				       struct rq *src_rq)
+static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p,
+					  struct scx_dispatch_q *dsq,
+					  struct rq *locked_rq,
+					  struct rq *src_rq)
 {
 	s32 cpu = raw_smp_processor_id();
 
 	lockdep_assert_held(&dsq->lock);
+	lockdep_assert_rq_held(locked_rq);
 
 	WARN_ON_ONCE(p->scx.holding_cpu >= 0);
 	task_unlink_from_dsq(p, dsq);
 	p->scx.holding_cpu = cpu;
 
 	raw_spin_unlock(&dsq->lock);
-	raw_spin_rq_lock(src_rq);
+	switch_rq_lock(locked_rq, src_rq);
 
 	/* task_rq couldn't have changed if we're still the holding cpu */
 	return likely(p->scx.holding_cpu == cpu) &&
 		!WARN_ON_ONCE(src_rq != task_rq(p));
 }
 
-static bool consume_remote_task(struct rq *this_rq,
+static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq,
 				struct task_struct *p, u64 enq_flags,
 				struct scx_dispatch_q *dsq, struct rq *src_rq)
 {
-	raw_spin_rq_unlock(this_rq);
-
-	if (unlink_dsq_and_lock_src_rq(p, dsq, src_rq)) {
-		move_remote_task_to_local_dsq(p, enq_flags, src_rq, this_rq);
+	if (unlink_dsq_and_switch_rq_lock(p, dsq, this_rq, src_rq)) {
+		move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, this_rq);
 		return true;
 	} else {
-		raw_spin_rq_unlock(src_rq);
-		raw_spin_rq_lock(this_rq);
+		switch_rq_lock(src_rq, this_rq);
 		return false;
 	}
 }
@@ -2475,8 +2498,7 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch,
 			raw_spin_unlock(&src_dsq->lock);
 		} else {
 			raw_spin_unlock(&src_dsq->lock);
-			move_remote_task_to_local_dsq(p, enq_flags,
-						      src_rq, dst_rq);
+			move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq);
 		}
 	} else {
 		/*
@@ -2486,14 +2508,14 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch,
 		dispatch_dequeue_locked(p, src_dsq);
 		raw_spin_unlock(&src_dsq->lock);
 
-		dispatch_enqueue(sch, dst_rq, dst_dsq, p, enq_flags);
+		scx_dispatch_enqueue(sch, dst_rq, dst_dsq, p, enq_flags);
 	}
 
 	return dst_rq;
 }
 
-static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq,
-			       struct scx_dispatch_q *dsq, u64 enq_flags)
+bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq,
+			    struct scx_dispatch_q *dsq, u64 enq_flags)
 {
 	struct task_struct *p;
 retry:
@@ -2529,7 +2551,7 @@ static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq,
 		}
 
 		if (task_can_run_on_remote_rq(sch, p, rq, false)) {
-			if (likely(consume_remote_task(rq, p, enq_flags, dsq, task_rq)))
+			if (likely(consume_remote_task(sch, rq, p, enq_flags, dsq, task_rq)))
 				return true;
 			goto retry;
 		}
@@ -2539,11 +2561,11 @@ static bool consume_dispatch_q(struct scx_sched *sch, struct rq *rq,
 	return false;
 }
 
-static bool consume_global_dsq(struct scx_sched *sch, struct rq *rq)
+bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq)
 {
 	int node = cpu_to_node(cpu_of(rq));
 
-	return consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0);
+	return scx_consume_dispatch_q(sch, rq, &sch->pnode[node]->global_dsq, 0);
 }
 
 /**
@@ -2576,15 +2598,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
 	 * If dispatching to @rq that @p is already on, no lock dancing needed.
 	 */
 	if (rq == src_rq && rq == dst_rq) {
-		dispatch_enqueue(sch, rq, dst_dsq, p,
-				 enq_flags | SCX_ENQ_CLEAR_OPSS);
-		return;
-	}
-
-	if (src_rq != dst_rq &&
-	    unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
-		dispatch_enqueue(sch, rq, find_global_dsq(sch, task_cpu(p)), p,
-				 enq_flags | SCX_ENQ_CLEAR_OPSS | SCX_ENQ_GDSQ_FALLBACK);
+		scx_dispatch_enqueue(sch, rq, dst_dsq, p,
+				     enq_flags | SCX_ENQ_CLEAR_OPSS);
 		return;
 	}
 
@@ -2597,7 +2612,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
 	 * As DISPATCHING guarantees that @p is wholly ours, we can pretend that
 	 * we're moving from a DSQ and use the same mechanism - mark the task
 	 * under transfer with holding_cpu, release DISPATCHING and then follow
-	 * the same protocol. See unlink_dsq_and_lock_src_rq().
+	 * the same protocol. See unlink_dsq_and_switch_rq_lock().
 	 */
 	p->scx.holding_cpu = raw_smp_processor_id();
 
@@ -2606,14 +2621,14 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
 
 	/* switch to @src_rq lock */
 	if (locked_rq != src_rq) {
-		raw_spin_rq_unlock(locked_rq);
+		switch_rq_lock(locked_rq, src_rq);
 		locked_rq = src_rq;
-		raw_spin_rq_lock(src_rq);
 	}
 
 	/* task_rq couldn't have changed if we're still the holding cpu */
 	if (likely(p->scx.holding_cpu == raw_smp_processor_id()) &&
 	    !WARN_ON_ONCE(src_rq != task_rq(p))) {
+		bool fallback = false;
 		/*
 		 * If @p is staying on the same rq, there's no need to go
 		 * through the full deactivate/activate cycle. Optimize by
@@ -2621,25 +2636,27 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
 		 */
 		if (src_rq == dst_rq) {
 			p->scx.holding_cpu = -1;
-			dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p,
-					 enq_flags);
+			scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p,
+					     enq_flags);
+		} else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
+			p->scx.holding_cpu = -1;
+			fallback = true;
+			scx_dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)),
+					     p, enq_flags | SCX_ENQ_GDSQ_FALLBACK);
 		} else {
-			move_remote_task_to_local_dsq(p, enq_flags,
-						      src_rq, dst_rq);
+			move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq);
 			/* task has been moved to dst_rq, which is now locked */
 			locked_rq = dst_rq;
 		}
 
 		/* if the destination CPU is idle, wake it up */
-		if (sched_class_above(p->sched_class, dst_rq->curr->sched_class))
+		if (!fallback && sched_class_above(p->sched_class, dst_rq->curr->sched_class))
 			resched_curr(dst_rq);
 	}
 
 	/* switch back to @rq lock */
-	if (locked_rq != rq) {
-		raw_spin_rq_unlock(locked_rq);
-		raw_spin_rq_lock(rq);
-	}
+	if (locked_rq != rq)
+		switch_rq_lock(locked_rq, rq);
 }
 
 /**
@@ -2710,10 +2727,10 @@ static void finish_dispatch(struct scx_sched *sch, struct rq *rq,
 		goto retry;
 	case SCX_OPSS_QUEUEING:
 		/*
-		 * do_enqueue_task() is in the process of transferring the task
-		 * to the BPF scheduler while holding @p's rq lock. As we aren't
-		 * holding any kernel or BPF resource that the enqueue path may
-		 * depend upon, it's safe to wait.
+		 * scx_do_enqueue_task() is in the process of transferring the
+		 * task to the BPF scheduler while holding @p's rq lock. As we
+		 * aren't holding any kernel or BPF resource that the enqueue
+		 * path may depend upon, it's safe to wait.
 		 */
 		wait_ops_state(p, opss);
 		goto retry;
@@ -2726,10 +2743,10 @@ static void finish_dispatch(struct scx_sched *sch, struct rq *rq,
 	if (dsq->id == SCX_DSQ_LOCAL)
 		dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags);
 	else
-		dispatch_enqueue(sch, rq, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS);
+		scx_dispatch_enqueue(sch, rq, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS);
 }
 
-static void flush_dispatch_buf(struct scx_sched *sch, struct rq *rq)
+void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq)
 {
 	struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
 	u32 u;
@@ -2758,115 +2775,6 @@ static inline void maybe_queue_balance_callback(struct rq *rq)
 	rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING;
 }
 
-/*
- * One user of this function is scx_bpf_dispatch() which can be called
- * recursively as sub-sched dispatches nest. Always inline to reduce stack usage
- * from the call frame.
- */
-static __always_inline bool
-scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
-		   struct task_struct *prev, bool nested)
-{
-	struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
-	int nr_loops = SCX_DSP_MAX_LOOPS;
-	s32 cpu = cpu_of(rq);
-	bool prev_on_sch = (prev->sched_class == &ext_sched_class) &&
-		scx_task_on_sched(sch, prev);
-
-	if (consume_global_dsq(sch, rq))
-		return true;
-
-	if (bypass_dsp_enabled(sch)) {
-		/* if @sch is bypassing, only the bypass DSQs are active */
-		if (scx_bypassing(sch, cpu))
-			return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0);
-
-#ifdef CONFIG_EXT_SUB_SCHED
-		/*
-		 * If @sch isn't bypassing but its children are, @sch is
-		 * responsible for making forward progress for both its own
-		 * tasks that aren't bypassing and the bypassing descendants'
-		 * tasks. The following implements a simple built-in behavior -
-		 * let each CPU try to run the bypass DSQ every Nth time.
-		 *
-		 * Later, if necessary, we can add an ops flag to suppress the
-		 * auto-consumption and a kfunc to consume the bypass DSQ and,
-		 * so that the BPF scheduler can fully control scheduling of
-		 * bypassed tasks.
-		 */
-		struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
-
-		if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) &&
-		    consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0)) {
-			__scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1);
-			return true;
-		}
-#endif	/* CONFIG_EXT_SUB_SCHED */
-	}
-
-	if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq))
-		return false;
-
-	dspc->rq = rq;
-
-	/*
-	 * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock,
-	 * the local DSQ might still end up empty after a successful
-	 * ops.dispatch(). If the local DSQ is empty even after ops.dispatch()
-	 * produced some tasks, retry. The BPF scheduler may depend on this
-	 * looping behavior to simplify its implementation.
-	 */
-	do {
-		dspc->nr_tasks = 0;
-
-		if (nested) {
-			SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
-				    prev_on_sch ? prev : NULL);
-		} else {
-			/* stash @prev so that nested invocations can access it */
-			rq->scx.sub_dispatch_prev = prev;
-			SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
-				    prev_on_sch ? prev : NULL);
-			rq->scx.sub_dispatch_prev = NULL;
-		}
-
-		flush_dispatch_buf(sch, rq);
-
-		if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) {
-			rq->scx.flags |= SCX_RQ_BAL_KEEP;
-			return true;
-		}
-		if (rq->scx.local_dsq.nr)
-			return true;
-		if (consume_global_dsq(sch, rq))
-			return true;
-
-		/*
-		 * ops.dispatch() can trap us in this loop by repeatedly
-		 * dispatching ineligible tasks. Break out once in a while to
-		 * allow the watchdog to run. As IRQ can't be enabled in
-		 * balance(), we want to complete this scheduling cycle and then
-		 * start a new one. IOW, we want to call resched_curr() on the
-		 * next, most likely idle, task, not the current one. Use
-		 * __scx_bpf_kick_cpu() for deferred kicking.
-		 */
-		if (unlikely(!--nr_loops)) {
-			scx_kick_cpu(sch, cpu, 0);
-			break;
-		}
-	} while (dspc->nr_tasks);
-
-	/*
-	 * Prevent the CPU from going idle while bypassed descendants have tasks
-	 * queued. Without this fallback, bypassed tasks could stall if the host
-	 * scheduler's ops.dispatch() doesn't yield any tasks.
-	 */
-	if (bypass_dsp_enabled(sch))
-		return consume_dispatch_q(sch, rq, bypass_dsq(sch, cpu), 0);
-
-	return false;
-}
-
 static int balance_one(struct rq *rq, struct task_struct *prev)
 {
 	struct scx_sched *sch = scx_root;
@@ -2876,6 +2784,8 @@ static int balance_one(struct rq *rq, struct task_struct *prev)
 	rq->scx.flags |= SCX_RQ_IN_BALANCE;
 	rq->scx.flags &= ~SCX_RQ_BAL_KEEP;
 
+	scx_process_sync_ecaps(rq, prev);
+
 	if ((sch->ops.flags & SCX_OPS_HAS_CPU_PREEMPT) &&
 	    unlikely(rq->scx.cpu_released)) {
 		/*
@@ -2921,7 +2831,8 @@ static int balance_one(struct rq *rq, struct task_struct *prev)
 	 * %SCX_OPS_ENQ_LAST is in effect.
 	 */
 	if ((prev->scx.flags & SCX_TASK_QUEUED) &&
-	    (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu))) {
+	    (!(sch->ops.flags & SCX_OPS_ENQ_LAST) || scx_bypassing(sch, cpu)) &&
+	    scx_task_can_stay_on_cpu(rq, prev)) {
 		rq->scx.flags |= SCX_RQ_BAL_KEEP;
 		__scx_add_event(sch, SCX_EV_DISPATCH_KEEP_LAST, 1);
 		goto has_tasks;
@@ -2941,7 +2852,7 @@ static int balance_one(struct rq *rq, struct task_struct *prev)
 	 *   between the IMMED queueing and the subsequent scheduling event.
 	 */
 	if (unlikely(rq->scx.local_dsq.nr > 1 && rq->scx.nr_immed))
-		schedule_reenq_local(rq, 0);
+		scx_schedule_reenq_local(rq, 0);
 
 	rq->scx.flags &= ~SCX_RQ_IN_BALANCE;
 	return true;
@@ -2957,7 +2868,7 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
 		 * dispatched. Call ops_dequeue() to notify the BPF scheduler.
 		 */
 		ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC);
-		dispatch_dequeue(rq, p);
+		scx_dispatch_dequeue(rq, p);
 	}
 
 	p->se.exec_start = rq_clock_task(rq);
@@ -2968,26 +2879,43 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
 
 	clr_task_runnable(p, true);
 
+	/* apply any pending out-of-band slice request before the tick decision */
+	apply_task_slice_oob(rq, p);
+
 	/*
 	 * @p is getting newly scheduled or got kicked after someone updated its
-	 * slice. Refresh whether tick can be stopped. See scx_can_stop_tick().
+	 * slice. Update SCX_RQ_CAN_STOP_TICK to reflect whether the tick can be
+	 * stopped. See scx_can_stop_tick().
+	 *
+	 * Moreover, refresh the load_avgs just when transitioning in and out of
+	 * nohz. In the future, we might want to add a mechanism to update
+	 * load_avgs periodically on tick-stopped CPUs.
 	 */
-	if ((p->scx.slice == SCX_SLICE_INF) !=
-	    (bool)(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) {
-		if (p->scx.slice == SCX_SLICE_INF)
+	if (p->scx.slice == SCX_SLICE_INF) {
+		if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) {
+			/*
+			 * Bypass mode always assigns finite slices, so @p
+			 * can't have an infinite slice while bypassing.
+			 * Therefore, sched_update_tick_dependency() can safely
+			 * evaluate the outgoing task.
+			 */
 			rq->scx.flags |= SCX_RQ_CAN_STOP_TICK;
-		else
-			rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
+			sched_update_tick_dependency(rq);
 
-		sched_update_tick_dependency(rq);
+			update_other_load_avgs(rq);
+		}
+	} else {
+		if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) {
+			rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
+			update_other_load_avgs(rq);
+		}
 
 		/*
-		 * For now, let's refresh the load_avgs just when transitioning
-		 * in and out of nohz. In the future, we might want to add a
-		 * mechanism which calls the following periodically on
-		 * tick-stopped CPUs.
+		 * @rq still references the outgoing scheduling context. A finite
+		 * slice is sufficient by itself to require the tick.
 		 */
-		update_other_load_avgs(rq);
+		if (tick_nohz_full_cpu(cpu_of(rq)))
+			tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED);
 	}
 }
 
@@ -3069,10 +2997,10 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
 		if (p->scx.slice && !scx_bypassing(sch, cpu_of(rq))) {
 			if (p->scx.flags & SCX_TASK_IMMED) {
 				p->scx.flags |= SCX_TASK_REENQ_PREEMPTED;
-				do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
+				scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
 				p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
 			} else {
-				dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, SCX_ENQ_HEAD);
+				scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, SCX_ENQ_HEAD);
 			}
 			goto switch_class;
 		}
@@ -3082,12 +3010,19 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
 		 * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell
 		 * ops.enqueue() that @p is the only one available for this cpu,
 		 * which should trigger an explicit follow-up scheduling event.
+		 * This doesn't apply if the baseline access on the CPU is lost.
+		 *
+		 * Core scheduling can force this CPU idle while @p stays
+		 * runnable. @p's cookie then won't match the core's, so skip
+		 * the warning in that case.
 		 */
-		if (next && sched_class_above(&ext_sched_class, next->sched_class)) {
-			WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_LAST));
-			do_enqueue_task(rq, p, SCX_ENQ_LAST, -1);
+		if (next && sched_class_above(&ext_sched_class, next->sched_class) &&
+		    scx_task_can_stay_on_cpu(rq, p)) {
+			WARN_ON_ONCE(sched_cpu_cookie_match(rq, p) &&
+				     !(sch->ops.flags & SCX_OPS_ENQ_LAST));
+			scx_do_enqueue_task(rq, p, SCX_ENQ_LAST, -1);
 		} else {
-			do_enqueue_task(rq, p, 0, -1);
+			scx_do_enqueue_task(rq, p, 0, -1);
 		}
 	}
 
@@ -3204,7 +3139,7 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx)
 		if (!p)
 			return NULL;
 
-		if (unlikely(!p->scx.slice)) {
+		if (unlikely(!p->scx.slice) && scx_task_can_stay_on_cpu(rq, p)) {
 			struct scx_sched *sch = scx_task_sched(p);
 
 			if (!scx_bypassing(sch, cpu_of(rq)) &&
@@ -3394,6 +3329,11 @@ static void handle_hotplug(struct rq *rq, bool online)
 	if (scx_enabled())
 		scx_idle_update_selcpu_topology(&sch->ops);
 
+	if (online)
+		scx_online_ecaps(rq);
+	else
+		scx_offline_ecaps(rq);
+
 	if (online && SCX_HAS_OP(sch, cpu_online))
 		SCX_CALL_OP(sch, cpu_online, NULL, scx_cpu_arg(cpu));
 	else if (!online && SCX_HAS_OP(sch, cpu_offline))
@@ -3513,7 +3453,7 @@ static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued)
 	 * we can't trust the slice management or ops.core_sched_before().
 	 */
 	if (scx_bypassing(sch, cpu_of(rq))) {
-		curr->scx.slice = 0;
+		set_task_slice(curr, 0);
 		touch_core_sched(rq, curr);
 	} else if (SCX_HAS_OP(sch, tick)) {
 		SCX_CALL_OP_TASK(sch, tick, rq, curr);
@@ -3545,7 +3485,7 @@ static struct cgroup *tg_cgrp(struct task_group *tg)
 
 #endif	/* CONFIG_EXT_GROUP_SCHED */
 
-static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork)
+int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork)
 {
 	int ret;
 
@@ -3559,7 +3499,7 @@ static int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fo
 
 		ret = SCX_CALL_OP_RET(sch, init_task, NULL, p, &args);
 		if (unlikely(ret)) {
-			ret = ops_sanitize_err(sch, "init_task", ret);
+			ret = scx_ops_sanitize_err(sch, "init_task", ret);
 			return ret;
 		}
 	}
@@ -3628,7 +3568,7 @@ static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p)
 		SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight);
 }
 
-static void scx_enable_task(struct scx_sched *sch, struct task_struct *p)
+void scx_enable_task(struct scx_sched *sch, struct task_struct *p)
 {
 	__scx_enable_task(sch, p);
 	scx_set_task_state(p, SCX_TASK_ENABLED);
@@ -3647,6 +3587,13 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p)
 		SCX_CALL_OP_TASK(sch, disable, rq, p);
 	scx_set_task_state(p, SCX_TASK_READY);
 
+	/*
+	 * Reset the SCX-managed fields when @p leaves the BPF scheduler's
+	 * control, after ops.disable() has observed their final values.
+	 */
+	p->scx.dsq_vtime = 0;
+	set_task_slice(p, 0);
+
 	/*
 	 * Verify the task is not in BPF scheduler's custody. If flag
 	 * transitions are consistent, the flag should always be clear
@@ -3655,8 +3602,7 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p)
 	WARN_ON_ONCE(p->scx.flags & SCX_TASK_IN_CUSTODY);
 }
 
-static void __scx_disable_and_exit_task(struct scx_sched *sch,
-					struct task_struct *p)
+void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p)
 {
 	struct scx_exit_task_args args = {
 		.cancelled = false,
@@ -3690,19 +3636,19 @@ static void __scx_disable_and_exit_task(struct scx_sched *sch,
  * ran. The task state has not been transitioned, so this mirrors the
  * SCX_TASK_INIT branch in __scx_disable_and_exit_task().
  */
-static void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p)
+void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p)
 {
 	struct scx_exit_task_args args = { .cancelled = true };
 
 	lockdep_assert_held(&p->pi_lock);
 	lockdep_assert_rq_held(task_rq(p));
 
+	/* @p was never associated with @sch, dispatch on the explicit @sch */
 	if (SCX_HAS_OP(sch, exit_task))
-		SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args);
+		__SCX_CALL_OP_TASK(sch, ops, exit_task, task_rq(p), p, &args);
 }
 
-static void scx_disable_and_exit_task(struct scx_sched *sch,
-				      struct task_struct *p)
+void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p)
 {
 	__scx_disable_and_exit_task(sch, p);
 
@@ -3730,6 +3676,7 @@ void init_scx_entity(struct sched_ext_entity *scx)
 	RB_CLEAR_NODE(&scx->dsq_priq);
 	scx->sticky_cpu = -1;
 	scx->holding_cpu = -1;
+	scx->runnable_cpu = -1;
 	INIT_LIST_HEAD(&scx->runnable_node);
 	scx->runnable_at = jiffies;
 	scx->ddsp_dsq_id = SCX_DSQ_INVALID;
@@ -3925,6 +3872,17 @@ static void reweight_task_scx(struct rq *rq, struct task_struct *p,
 	if (task_dead_and_done(p))
 		return;
 
+	/*
+	 * When switching sched_class away from SCX, reweight_task_scx()
+	 * is called _after_ scx_disable_task(). Skip calling ops.set_weight()
+	 * since the BPF scheduler may have already forgotten the task in
+	 * ops.disable().
+	 * p->scx.weight will be recalculated in scx_enable_task() if the task
+	 * ever returns to SCX class.
+	 */
+	if (scx_get_task_state(p) != SCX_TASK_ENABLED)
+		return;
+
 	p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight));
 	if (SCX_HAS_OP(sch, set_weight))
 		SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight);
@@ -4035,7 +3993,8 @@ static void process_ddsp_deferred_locals(struct rq *rq)
  * another reenq cycle. Repetitions are bounded by %SCX_REENQ_LOCAL_MAX_REPEAT
  * in process_deferred_reenq_locals().
  */
-static bool local_task_should_reenq(struct task_struct *p, u64 *reenq_flags, u32 *reason)
+static bool local_task_should_reenq(struct rq *rq, struct task_struct *p,
+				    u64 *reenq_flags, u32 *reason)
 {
 	bool first;
 
@@ -4051,6 +4010,12 @@ static bool local_task_should_reenq(struct task_struct *p, u64 *reenq_flags, u32
 		return true;
 	}
 
+	if ((*reenq_flags & SCX_REENQ_CAP_REVOKE) &&
+	    scx_task_reenq_on_cap_revoke(rq, p)) {
+		*reason = SCX_TASK_REENQ_CAP;
+		return true;
+	}
+
 	return *reenq_flags & SCX_REENQ_ANY;
 }
 
@@ -4094,10 +4059,10 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
 		if (!scx_is_descendant(task_sch, sch))
 			continue;
 
-		if (!local_task_should_reenq(p, &reenq_flags, &reason))
+		if (!local_task_should_reenq(rq, p, &reenq_flags, &reason))
 			continue;
 
-		dispatch_dequeue(rq, p);
+		scx_dispatch_dequeue(rq, p);
 
 		if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
 			p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
@@ -4109,12 +4074,26 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
 	list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
 		list_del_init(&p->scx.dsq_list.node);
 
-		do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
+		scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
 
 		p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
 		nr_enqueued++;
 	}
 
+	/*
+	 * The revoke that scheduled this scan may have raced the pick: curr
+	 * may be a now-capless task, either one that kept running or one
+	 * promoted off the local DSQ between the ecaps sync and this scan.
+	 * Zero the slice to evict it. The enqueue gate blocks new capless
+	 * inserts, so no later pick can slip through after the scan.
+	 */
+	if ((reenq_flags & SCX_REENQ_CAP_REVOKE) &&
+	    rq->curr->sched_class == &ext_sched_class &&
+	    scx_task_reenq_on_cap_revoke(rq, rq->curr)) {
+		set_task_slice(rq->curr, 0);
+		resched_curr(rq);
+	}
+
 	return nr_enqueued;
 }
 
@@ -4224,7 +4203,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag
 			p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
 		p->scx.flags |= reason;
 
-		do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1);
+		scx_do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1);
 
 		p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
 
@@ -4290,6 +4269,8 @@ static void run_deferred(struct rq *rq)
 
 	if (!list_empty(&rq->scx.deferred_reenq_users))
 		process_deferred_reenq_users(rq);
+
+	scx_reenq_reject(rq);
 }
 
 #ifdef CONFIG_NO_HZ_FULL
@@ -4301,6 +4282,15 @@ bool scx_can_stop_tick(struct rq *rq)
 	if (p->sched_class != &ext_sched_class)
 		return true;
 
+	/*
+	 * @rq->curr may still reference an outgoing EXT task after it has been
+	 * dequeued. If no EXT tasks are accounted on @rq, ignore its stale
+	 * slice state. If another task is dispatched from a DSQ,
+	 * set_next_task_scx() will update the dependency for the incoming task.
+	 */
+	if (!rq->scx.nr_running)
+		return true;
+
 	if (scx_bypassing(sch, cpu_of(rq)))
 		return false;
 
@@ -4344,7 +4334,7 @@ int scx_tg_online(struct task_group *tg)
 			ret = SCX_CALL_OP_RET(sch, cgroup_init,
 					      NULL, tg->css.cgroup, &args);
 			if (ret)
-				ret = ops_sanitize_err(sch, "cgroup_init", ret);
+				ret = scx_ops_sanitize_err(sch, "cgroup_init", ret);
 		}
 		if (ret == 0)
 			tg->scx.flags |= SCX_TG_ONLINE | SCX_TG_INITED;
@@ -4412,7 +4402,7 @@ int scx_cgroup_can_attach(struct cgroup_taskset *tset)
 		p->scx.cgrp_moving_from = NULL;
 	}
 
-	return ops_sanitize_err(sch, "cgroup_prep_move", ret);
+	return scx_ops_sanitize_err(sch, "cgroup_prep_move", ret);
 }
 
 void scx_cgroup_move_task(struct task_struct *p)
@@ -4427,12 +4417,13 @@ void scx_cgroup_move_task(struct task_struct *p)
 	 * cgroup changes. Migration keys off css rather than cgroup identity,
 	 * so it can hand an unchanged-cgroup task here with cgrp_moving_from
 	 * NULL. Nothing to report to the BPF scheduler then, so skip it and
-	 * keep prep_move and move paired.
+	 * keep prep_move and move paired. Cgroup ops run on the root sched,
+	 * dispatch on the explicit @sch.
 	 */
 	if (SCX_HAS_OP(sch, cgroup_move) && p->scx.cgrp_moving_from)
-		SCX_CALL_OP_TASK(sch, cgroup_move, task_rq(p),
-				 p, p->scx.cgrp_moving_from,
-				 tg_cgrp(task_group(p)));
+		__SCX_CALL_OP_TASK(sch, ops, cgroup_move, task_rq(p),
+				   p, p->scx.cgrp_moving_from,
+				   tg_cgrp(task_group(p)));
 	p->scx.cgrp_moving_from = NULL;
 }
 
@@ -4515,7 +4506,7 @@ static struct cgroup *root_cgroup(void)
 	return &cgrp_dfl_root.cgrp;
 }
 
-static void scx_cgroup_lock(void)
+void scx_cgroup_lock(void)
 {
 #ifdef CONFIG_EXT_GROUP_SCHED
 	percpu_down_write(&scx_cgroup_ops_rwsem);
@@ -4523,7 +4514,7 @@ static void scx_cgroup_lock(void)
 	cgroup_lock();
 }
 
-static void scx_cgroup_unlock(void)
+void scx_cgroup_unlock(void)
 {
 	cgroup_unlock();
 #ifdef CONFIG_EXT_GROUP_SCHED
@@ -4536,26 +4527,6 @@ static inline void scx_cgroup_lock(void) {}
 static inline void scx_cgroup_unlock(void) {}
 #endif	/* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */
 
-#ifdef CONFIG_EXT_SUB_SCHED
-static struct cgroup *sch_cgroup(struct scx_sched *sch)
-{
-	return sch->cgrp;
-}
-
-/* for each descendant of @cgrp including self, set ->scx_sched to @sch */
-static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch)
-{
-	struct cgroup *pos;
-	struct cgroup_subsys_state *css;
-
-	cgroup_for_each_live_descendant_pre(pos, css, cgrp)
-		rcu_assign_pointer(pos->scx_sched, sch);
-}
-#else	/* CONFIG_EXT_SUB_SCHED */
-static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; }
-static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {}
-#endif	/* CONFIG_EXT_SUB_SCHED */
-
 /*
  * Omitted operations:
  *
@@ -4690,7 +4661,7 @@ static void destroy_dsq(struct scx_sched *sch, u64 dsq_id)
 		goto out_unlock_dsq;
 
 	/*
-	 * Mark dead by invalidating ->id to prevent dispatch_enqueue() from
+	 * Mark dead by invalidating ->id to prevent scx_dispatch_enqueue() from
 	 * queueing more tasks. As this function can be called from anywhere,
 	 * freeing is bounced through an irq work to avoid nesting RCU
 	 * operations inside scheduler locks.
@@ -4838,8 +4809,10 @@ static const struct attribute_group scx_global_attr_group = {
 
 static void free_pnode(struct scx_sched_pnode *pnode);
 static void free_exit_info(struct scx_exit_info *ei);
+static const char *scx_exit_reason(enum scx_exit_kind kind);
+static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind);
 
-static s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch)
+s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch)
 {
 	size_t size = struct_size_t(struct scx_cmask, bits,
 				    SCX_CMASK_NR_WORDS(num_possible_cpus()));
@@ -4894,6 +4867,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
 	timer_shutdown_sync(&sch->bypass_lb_timer);
 	free_cpumask_var(sch->bypass_lb_donee_cpumask);
 	free_cpumask_var(sch->bypass_lb_resched_cpumask);
+	free_cpumask_var(sch->stall_cpus);
 
 #ifdef CONFIG_EXT_SUB_SCHED
 	kfree(sch->cgrp_path);
@@ -4901,6 +4875,8 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
 		cgroup_put(sch_cgroup(sch));
 	if (sch->sub_kset)
 		kobject_put(&sch->sub_kset->kobj);
+	if (scx_parent(sch))
+		kobject_put(&scx_parent(sch)->kobj);
 #endif	/* CONFIG_EXT_SUB_SCHED */
 
 	for_each_possible_cpu(cpu) {
@@ -4913,7 +4889,21 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
 		 */
 		WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node));
 
-		exit_dsq(bypass_dsq(sch, cpu));
+		/* remove the queued ecaps sync so the pcpu can be freed */
+		scx_discard_ecaps_to_sync(cpu, pcpu);
+
+		/*
+		 * Bypass blocks new kicks. Flush the kick irq_work so this
+		 * pcpu's to_kick_node is off the list before it is freed.
+		 */
+		irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work);
+		WARN_ON_ONCE(!list_empty(&pcpu->to_kick_node));
+		free_cpumask_var(pcpu->cpus_to_kick);
+		free_cpumask_var(pcpu->cpus_to_kick_if_idle);
+		free_cpumask_var(pcpu->cpus_to_preempt);
+		free_cpumask_var(pcpu->cpus_to_wait);
+
+		exit_dsq(scx_bypass_dsq(sch, cpu));
 	}
 
 	free_percpu(sch->pcpu);
@@ -4922,6 +4912,8 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
 		free_pnode(sch->pnode[node]);
 	kfree(sch->pnode);
 
+	scx_free_pshards(sch);
+
 	rhashtable_walk_enter(&sch->dsq_hash, &rht_iter);
 	do {
 		rhashtable_walk_start(&rht_iter);
@@ -4939,6 +4931,10 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
 	scx_arena_pool_destroy(sch);
 	if (sch->arena_map)
 		bpf_map_put(sch->arena_map);
+
+	/* @sch is completely inactive by now */
+	scx_dec_has_subs(sch);
+
 	kfree(sch);
 }
 
@@ -4971,26 +4967,61 @@ static ssize_t scx_attr_events_show(struct kobject *kobj,
 	int at = 0;
 
 	scx_read_events(sch, &events);
-	at += scx_attr_event_show(buf, at, &events, SCX_EV_SELECT_CPU_FALLBACK);
-	at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE);
-	at += scx_attr_event_show(buf, at, &events, SCX_EV_DISPATCH_KEEP_LAST);
-	at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_EXITING);
-	at += scx_attr_event_show(buf, at, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED);
-	at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_IMMED);
-	at += scx_attr_event_show(buf, at, &events, SCX_EV_REENQ_LOCAL_REPEAT);
-	at += scx_attr_event_show(buf, at, &events, SCX_EV_REFILL_SLICE_DFL);
-	at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DURATION);
-	at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_DISPATCH);
-	at += scx_attr_event_show(buf, at, &events, SCX_EV_BYPASS_ACTIVATE);
-	at += scx_attr_event_show(buf, at, &events, SCX_EV_INSERT_NOT_OWNED);
-	at += scx_attr_event_show(buf, at, &events, SCX_EV_SUB_BYPASS_DISPATCH);
+#define SCX_EVENT(name)	(at += scx_attr_event_show(buf, at, &events, name))
+	SCX_EVENTS_LIST(SCX_EVENT);
+#undef SCX_EVENT
 	return at;
 }
 SCX_ATTR(events);
 
+#ifdef CONFIG_EXT_SUB_SCHED
+static const char *scx_cap_names[__SCX_NR_CAPS] = {
+	[__SCX_CAP_ENQ_IMMED]	= "enq_immed",
+	[__SCX_CAP_ENQ]		= "enq",
+	[__SCX_CAP_PREEMPT]	= "preempt",
+};
+
+static ssize_t scx_attr_caps_show(struct kobject *kobj,
+				  struct kobj_attribute *ka, char *buf)
+{
+	struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj);
+	u32 npossible = num_possible_cpus();
+	struct scx_cmask *agg __free(kfree) =
+		kzalloc(struct_size(agg, bits, SCX_CMASK_NR_WORDS(npossible)), GFP_KERNEL);
+	unsigned long *agg_bm __free(bitmap) = bitmap_zalloc(npossible, GFP_KERNEL);
+	ssize_t count = 0;
+	s32 cap, si;
+
+	if (!agg || !agg_bm)
+		return -ENOMEM;
+
+	for (cap = 0; cap < __SCX_NR_CAPS; cap++) {
+		SCX_CMASK_DEFINE(snap, 0, SCX_CID_SHARD_MAX_CPUS);
+
+		scx_cmask_init(agg, 0, npossible);
+		for (si = 0; si < sch->nr_pshards; si++) {
+			struct scx_cmask *cm = &sch->pshard[si]->caps[cap].cmask;
+
+			scx_cmask_reframe(snap, cm->base, cm->nr_cids);
+			scx_cmask_copy(snap, cm);
+			scx_cmask_or(agg, snap);
+		}
+		/* %*pbl takes unsigned long bitmap layout, convert from u64 */
+		bitmap_from_arr64(agg_bm, agg->bits, npossible);
+		count += sysfs_emit_at(buf, count, "%s: %*pbl\n",
+				       scx_cap_names[cap], npossible, agg_bm);
+	}
+	return count;
+}
+SCX_ATTR(caps);
+#endif	/* CONFIG_EXT_SUB_SCHED */
+
 static struct attribute *scx_sched_attrs[] = {
 	&scx_attr_ops.attr,
 	&scx_attr_events.attr,
+#ifdef CONFIG_EXT_SUB_SCHED
+	&scx_attr_caps.attr,
+#endif
 	NULL,
 };
 ATTRIBUTE_GROUPS(scx_sched);
@@ -5077,6 +5108,7 @@ bool scx_allow_ttwu_queue(const struct task_struct *p)
 
 /**
  * handle_lockup - sched_ext common lockup handler
+ * @exit_cpu: CPU to record in exit_info. Pass the stalled/hung CPU, not current.
  * @fmt: format string
  *
  * Called on system stall or lockup condition and initiates abort of sched_ext
@@ -5086,7 +5118,7 @@ bool scx_allow_ttwu_queue(const struct task_struct *p)
  * resolve the lockup. %false if sched_ext is not enabled or abort was already
  * initiated by someone else.
  */
-static __printf(1, 2) bool handle_lockup(const char *fmt, ...)
+static __printf(2, 3) bool handle_lockup(int exit_cpu, const char *fmt, ...)
 {
 	struct scx_sched *sch;
 	va_list args;
@@ -5102,7 +5134,7 @@ static __printf(1, 2) bool handle_lockup(const char *fmt, ...)
 	case SCX_ENABLING:
 	case SCX_ENABLED:
 		va_start(args, fmt);
-		ret = scx_verror(sch, fmt, args);
+		ret = scx_vexit(sch, SCX_EXIT_ERROR, 0, exit_cpu, fmt, args);
 		va_end(args);
 		return ret;
 	default:
@@ -5122,9 +5154,46 @@ static __printf(1, 2) bool handle_lockup(const char *fmt, ...)
  * resolve the reported RCU stall. %false if sched_ext is not enabled or someone
  * else already initiated abort.
  */
-bool scx_rcu_cpu_stall(void)
+bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask)
 {
-	return handle_lockup("RCU CPU stall detected!");
+	struct scx_sched *sch;
+	struct scx_exit_info *ei;
+	int exit_cpu;
+
+	guard(rcu)();
+
+	sch = rcu_dereference(scx_root);
+	if (unlikely(!sch))
+		return false;
+
+	switch (scx_enable_state()) {
+	case SCX_ENABLING:
+	case SCX_ENABLED:
+		break;
+	default:
+		return false;
+	}
+
+	exit_cpu = cpumask_empty(stalled_mask) ? -1 : (int)cpumask_first(stalled_mask);
+	ei = sch->exit_info;
+
+	guard(preempt)();
+
+	if (!scx_claim_exit(sch, SCX_EXIT_ERROR))
+		return false;
+
+#ifdef CONFIG_STACKTRACE
+	ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1);
+#endif
+	scnprintf(ei->msg, SCX_EXIT_MSG_LEN, "RCU CPU stall on CPUs (%*pbl)",
+		  cpumask_pr_args(stalled_mask));
+	ei->kind = SCX_EXIT_ERROR;
+	ei->reason = scx_exit_reason(SCX_EXIT_ERROR);
+	ei->exit_cpu = exit_cpu;
+	cpumask_copy(sch->stall_cpus, stalled_mask);
+
+	irq_work_queue(&sch->disable_irq_work);
+	return true;
 }
 
 /**
@@ -5139,11 +5208,13 @@ bool scx_rcu_cpu_stall(void)
  */
 void scx_softlockup(u32 dur_s)
 {
-	if (!handle_lockup("soft lockup - CPU %d stuck for %us", smp_processor_id(), dur_s))
+	int cpu = smp_processor_id();
+
+	if (!handle_lockup(cpu, "soft lockup - CPU %d stuck for %us", cpu, dur_s))
 		return;
 
 	printk_deferred(KERN_ERR "sched_ext: Soft lockup - CPU %d stuck for %us, disabling BPF scheduler\n",
-			smp_processor_id(), dur_s);
+			cpu, dur_s);
 }
 
 /*
@@ -5158,7 +5229,7 @@ static void scx_hardlockup_irq_workfn(struct irq_work *work)
 {
 	int cpu = atomic_xchg(&scx_hardlockup_cpu, -1);
 
-	if (cpu >= 0 && handle_lockup("hard lockup - CPU %d", cpu))
+	if (cpu >= 0 && handle_lockup(cpu, "hard lockup - CPU %d", cpu))
 		printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n",
 				cpu);
 }
@@ -5194,7 +5265,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor,
 			 u32 nr_donor_target, u32 nr_donee_target)
 {
 	struct rq *donor_rq = cpu_rq(donor);
-	struct scx_dispatch_q *donor_dsq = bypass_dsq(sch, donor);
+	struct scx_dispatch_q *donor_dsq = scx_bypass_dsq(sch, donor);
 	struct task_struct *p, *n;
 	struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, donor_dsq, 0);
 	s32 delta = READ_ONCE(donor_dsq->nr) - nr_donor_target;
@@ -5242,7 +5313,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor,
 		if (donee >= nr_cpu_ids)
 			continue;
 
-		donee_dsq = bypass_dsq(sch, donee);
+		donee_dsq = scx_bypass_dsq(sch, donee);
 
 		/*
 		 * $p's rq is not locked but $p's DSQ lock protects its
@@ -5263,7 +5334,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor,
 		 * between bypass DSQs.
 		 */
 		dispatch_dequeue_locked(p, donor_dsq);
-		dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, SCX_ENQ_NESTED);
+		scx_dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, SCX_ENQ_NESTED);
 
 		/*
 		 * $donee might have been idle and need to be woken up. No need
@@ -5306,7 +5377,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node)
 
 	/* count the target tasks and CPUs */
 	for_each_cpu_and(cpu, cpu_online_mask, node_mask) {
-		u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr);
+		u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr);
 
 		nr_tasks += nr;
 		nr_cpus++;
@@ -5328,7 +5399,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node)
 
 	cpumask_clear(donee_mask);
 	for_each_cpu_and(cpu, cpu_online_mask, node_mask) {
-		if (READ_ONCE(bypass_dsq(sch, cpu)->nr) < nr_target)
+		if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) < nr_target)
 			cpumask_set_cpu(cpu, donee_mask);
 	}
 
@@ -5339,7 +5410,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node)
 			break;
 		if (cpumask_test_cpu(cpu, donee_mask))
 			continue;
-		if (READ_ONCE(bypass_dsq(sch, cpu)->nr) <= nr_donor_target)
+		if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) <= nr_donor_target)
 			continue;
 
 		nr_balanced += bypass_lb_cpu(sch, cpu, donee_mask, resched_mask,
@@ -5350,7 +5421,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node)
 		resched_cpu(cpu);
 
 	for_each_cpu_and(cpu, cpu_online_mask, node_mask) {
-		u32 nr = READ_ONCE(bypass_dsq(sch, cpu)->nr);
+		u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr);
 
 		after_min = min(nr, after_min);
 		after_max = max(nr, after_max);
@@ -5376,7 +5447,7 @@ static void scx_bypass_lb_timerfn(struct timer_list *timer)
 	int node;
 	u32 intv_us;
 
-	if (!bypass_dsp_enabled(sch))
+	if (!scx_bypass_dsp_enabled(sch))
 		return;
 
 	for_each_node_with_cpus(node)
@@ -5442,9 +5513,9 @@ static void enable_bypass_dsp(struct scx_sched *sch)
 	 * dispatch enabled while a descendant is bypassing, which is all that's
 	 * required.
 	 *
-	 * bypass_dsp_enabled() test is used to determine whether to enter the
-	 * bypass dispatch handling path from both bypassing and hosting scheds.
-	 * Bump enable depth on both @sch and bypass dispatch host.
+	 * scx_bypass_dsp_enabled() test is used to determine whether to enter
+	 * the bypass dispatch handling path from both bypassing and hosting
+	 * scheds. Bump enable depth on both @sch and bypass dispatch host.
 	 */
 	ret = atomic_inc_return(&sch->bypass_dsp_enable_depth);
 	WARN_ON_ONCE(ret <= 0);
@@ -5464,7 +5535,7 @@ static void enable_bypass_dsp(struct scx_sched *sch)
 }
 
 /* may be called without holding scx_bypass_lock */
-static void disable_bypass_dsp(struct scx_sched *sch)
+void scx_disable_bypass_dsp(struct scx_sched *sch)
 {
 	s32 ret;
 
@@ -5480,6 +5551,38 @@ static void disable_bypass_dsp(struct scx_sched *sch)
 	}
 }
 
+/**
+ * unbypass_renotify_idle - Arm an idle re-notify for a sched leaving bypass
+ * @rq: rq of the cpu leaving bypass
+ * @pos: scheduler that just left bypass on @rq's cpu
+ * @pcpu: @pos's per-cpu state for @rq's cpu
+ *
+ * A sched leaving bypass is owed the ops.update_idle() calls suppressed while
+ * bypassing. A cpu that goes idle during the bypass window and stays idle won't
+ * produce a notification. Arm a re-notify that scx_bypass()'s resched flushes
+ * on the next idle pick.
+ *
+ * An acute case is ops.sub_attach(). If the parent grants the child cids while
+ * attaching, when attach is complete and bypass is lifted, the child may hold
+ * idle cids it never saw go idle.
+ *
+ * The root is no exception as bypass suppresses its notifications the same way.
+ * However, the root uses a separate per-rq flag so its re-notify keeps working
+ * even when !CONFIG_EXT_SUB_SCHED.
+ */
+static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos,
+				   struct scx_sched_pcpu *pcpu)
+{
+	if (pos == scx_root) {
+		rq->scx.flags |= SCX_RQ_ROOT_IDLE_RENOTIFY;
+		return;
+	}
+#ifdef CONFIG_EXT_SUB_SCHED
+	pcpu->idle_renotify = true;
+	rq->scx.flags |= SCX_RQ_SUB_IDLE_RENOTIFY;
+#endif
+}
+
 /**
  * scx_bypass - [Un]bypass scx_ops and guarantee forward progress
  * @sch: sched to bypass
@@ -5512,7 +5615,7 @@ static void disable_bypass_dsp(struct scx_sched *sch)
  *
  * - scx_prio_less() reverts to the default core_sched_at order.
  */
-static void scx_bypass(struct scx_sched *sch, bool bypass)
+void scx_bypass(struct scx_sched *sch, bool bypass)
 {
 	struct scx_sched *pos;
 	unsigned long flags;
@@ -5563,11 +5666,17 @@ static void scx_bypass(struct scx_sched *sch, bool bypass)
 
 		scx_for_each_descendant_pre(pos, sch) {
 			struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu);
+			bool was_bypassing = pcpu->flags & SCX_SCHED_PCPU_BYPASSING;
 
-			if (pos->bypass_depth)
+			if (pos->bypass_depth) {
 				pcpu->flags |= SCX_SCHED_PCPU_BYPASSING;
-			else
+			} else {
 				pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING;
+				if (was_bypassing) {
+					unbypass_renotify_idle(rq, pos, pcpu);
+					scx_unbypass_replay_ecaps(rq, pos);
+				}
+			}
 		}
 
 		raw_spin_unlock(&scx_sched_lock);
@@ -5609,7 +5718,7 @@ static void scx_bypass(struct scx_sched *sch, bool bypass)
 
 	/* disarming must come after moving all tasks out of the bypass DSQs */
 	if (!bypass)
-		disable_bypass_dsp(sch);
+		scx_disable_bypass_dsp(sch);
 unlock:
 	raw_spin_unlock_irqrestore(&scx_bypass_lock, flags);
 }
@@ -5656,6 +5765,8 @@ static const char *scx_exit_reason(enum scx_exit_kind kind)
 		return "disabled by sysrq-S";
 	case SCX_EXIT_PARENT:
 		return "parent exiting";
+	case SCX_EXIT_PARENT_KILL:
+		return "killed by parent scheduler";
 	case SCX_EXIT_ERROR:
 		return "runtime error";
 	case SCX_EXIT_ERROR_BPF:
@@ -5672,9 +5783,11 @@ static void free_kick_syncs(void)
 	int cpu;
 
 	for_each_possible_cpu(cpu) {
-		struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
+		struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
 		struct scx_kick_syncs *to_free;
 
+		/* flush the pending kick before freeing @ksyncs */
+		irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work);
 		to_free = rcu_replace_pointer(*ksyncs, NULL, true);
 		if (to_free)
 			kvfree_rcu(to_free, rcu);
@@ -5701,12 +5814,13 @@ static void refresh_watchdog(void)
 		cancel_delayed_work_sync(&scx_watchdog_work);
 }
 
-static s32 scx_link_sched(struct scx_sched *sch)
+s32 scx_link_sched(struct scx_sched *sch)
 {
 	const char *err_msg = "";
 	s32 ret = 0;
 
-	scoped_guard(raw_spinlock_irq, &scx_sched_lock) {
+	scoped_guard(raw_spinlock_irqsave, &scx_bypass_lock)	/* for the parent bypass check */
+	scoped_guard(raw_spinlock, &scx_sched_lock) {
 #ifdef CONFIG_EXT_SUB_SCHED
 		struct scx_sched *parent = scx_parent(sch);
 
@@ -5723,6 +5837,17 @@ static s32 scx_link_sched(struct scx_sched *sch)
 				break;
 			}
 
+			/*
+			 * Bypass state is spread across per-cpu flags and a
+			 * depth count, so inheriting it is tricky and has no
+			 * valid use case. Refuse it.
+			 */
+			if (READ_ONCE(parent->bypass_depth)) {
+				err_msg = "parent bypassing";
+				ret = -EBUSY;
+				break;
+			}
+
 			ret = rhashtable_lookup_insert_fast(&scx_sched_hash,
 					&sch->hash_node, scx_sched_hash_params);
 			if (ret) {
@@ -5730,7 +5855,7 @@ static s32 scx_link_sched(struct scx_sched *sch)
 				break;
 			}
 
-			list_add_tail(&sch->sibling, &parent->children);
+			list_add_tail_rcu(&sch->sibling, &parent->children);
 		}
 #endif	/* CONFIG_EXT_SUB_SCHED */
 
@@ -5750,14 +5875,14 @@ static s32 scx_link_sched(struct scx_sched *sch)
 	return 0;
 }
 
-static void scx_unlink_sched(struct scx_sched *sch)
+void scx_unlink_sched(struct scx_sched *sch)
 {
 	scoped_guard(raw_spinlock_irq, &scx_sched_lock) {
 #ifdef CONFIG_EXT_SUB_SCHED
 		if (scx_parent(sch)) {
 			rhashtable_remove_fast(&scx_sched_hash, &sch->hash_node,
 					       scx_sched_hash_params);
-			list_del_init(&sch->sibling);
+			list_del_rcu(&sch->sibling);
 		}
 #endif	/* CONFIG_EXT_SUB_SCHED */
 		list_del_rcu(&sch->all);
@@ -5771,13 +5896,13 @@ static void scx_unlink_sched(struct scx_sched *sch)
  * @sch. Once @sch becomes empty during disable, there's no point in dumping it.
  * This prevents calling dump ops on a dead sch.
  */
-static void scx_disable_dump(struct scx_sched *sch)
+void scx_disable_dump(struct scx_sched *sch)
 {
 	guard(raw_spinlock_irqsave)(&scx_dump_lock);
 	sch->dump_disabled = true;
 }
 
-static void scx_log_sched_disable(struct scx_sched *sch)
+void scx_log_sched_disable(struct scx_sched *sch)
 {
 	struct scx_exit_info *ei = sch->exit_info;
 	const char *type = scx_parent(sch) ? "sub-scheduler" : "scheduler";
@@ -5797,202 +5922,6 @@ static void scx_log_sched_disable(struct scx_sched *sch)
 	}
 }
 
-#ifdef CONFIG_EXT_SUB_SCHED
-static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq);
-
-static void drain_descendants(struct scx_sched *sch)
-{
-	/*
-	 * Child scheds that finished the critical part of disabling will take
-	 * themselves off @sch->children. Wait for it to drain. As propagation
-	 * is recursive, empty @sch->children means that all proper descendant
-	 * scheds reached unlinking stage.
-	 */
-	wait_event(scx_unlink_waitq, list_empty(&sch->children));
-}
-
-static void scx_fail_parent(struct scx_sched *sch,
-			    struct task_struct *failed, s32 fail_code)
-{
-	struct scx_sched *parent = scx_parent(sch);
-	struct scx_task_iter sti;
-	struct task_struct *p;
-
-	scx_error(parent, "ops.init_task() failed (%d) for %s[%d] while disabling a sub-scheduler",
-		  fail_code, failed->comm, failed->pid);
-
-	/*
-	 * Once $parent is bypassed, it's safe to put SCX_TASK_NONE tasks into
-	 * it. This may cause downstream failures on the BPF side but $parent is
-	 * dying anyway.
-	 */
-	scx_bypass(parent, true);
-
-	scx_task_iter_start(&sti, sch->cgrp);
-	while ((p = scx_task_iter_next_locked(&sti))) {
-		if (scx_task_on_sched(parent, p))
-			continue;
-
-		scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
-			scx_disable_and_exit_task(sch, p);
-			scx_set_task_sched(p, parent);
-		}
-	}
-	scx_task_iter_stop(&sti);
-}
-
-static void scx_sub_disable(struct scx_sched *sch)
-{
-	struct scx_sched *parent = scx_parent(sch);
-	struct scx_task_iter sti;
-	struct task_struct *p;
-	int ret;
-
-	/*
-	 * Guarantee forward progress and wait for descendants to be disabled.
-	 * To limit disruptions, $parent is not bypassed. Tasks are fully
-	 * prepped and then inserted back into $parent.
-	 */
-	scx_bypass(sch, true);
-	drain_descendants(sch);
-
-	/*
-	 * Here, every runnable task is guaranteed to make forward progress and
-	 * we can safely use blocking synchronization constructs. Actually
-	 * disable ops.
-	 */
-	mutex_lock(&scx_enable_mutex);
-	percpu_down_write(&scx_fork_rwsem);
-	scx_cgroup_lock();
-
-	set_cgroup_sched(sch_cgroup(sch), parent);
-
-	scx_task_iter_start(&sti, sch->cgrp);
-	while ((p = scx_task_iter_next_locked(&sti))) {
-		struct rq *rq;
-		struct rq_flags rf;
-
-		/* filter out duplicate visits */
-		if (scx_task_on_sched(parent, p))
-			continue;
-
-		/*
-		 * By the time control reaches here, all descendant schedulers
-		 * should already have been disabled.
-		 */
-		WARN_ON_ONCE(!scx_task_on_sched(sch, p));
-
-		/*
-		 * @p is pinned by the iter: css_task_iter_next() takes a
-		 * reference and holds it until the next iter_next() call, so
-		 * @p->usage is guaranteed > 0.
-		 */
-		get_task_struct(p);
-
-		scx_task_iter_unlock(&sti);
-
-		/*
-		 * $p is READY or ENABLED on @sch. Initialize for $parent,
-		 * disable and exit from @sch, and then switch over to $parent.
-		 *
-		 * If a task fails to initialize for $parent, the only available
-		 * action is disabling $parent too. While this allows disabling
-		 * of a child sched to cause the parent scheduler to fail, the
-		 * failure can only originate from ops.init_task() of the
-		 * parent. A child can't directly affect the parent through its
-		 * own failures.
-		 */
-		ret = __scx_init_task(parent, p, false);
-		if (ret) {
-			scx_fail_parent(sch, p, ret);
-			put_task_struct(p);
-			break;
-		}
-
-		rq = task_rq_lock(p, &rf);
-
-		if (scx_get_task_state(p) == SCX_TASK_DEAD) {
-			/*
-			 * sched_ext_dead() raced us between __scx_init_task()
-			 * and this rq lock and ran exit_task() on @sch (the
-			 * sched @p was on at that point), not on $parent.
-			 * $parent's just-completed init is owed an exit_task()
-			 * and we issue it here.
-			 */
-			scx_sub_init_cancel_task(parent, p);
-			task_rq_unlock(rq, p, &rf);
-			put_task_struct(p);
-			continue;
-		}
-
-		scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
-			/*
-			 * $p is initialized for $parent and still attached to
-			 * @sch. Disable and exit for @sch, switch over to
-			 * $parent, override the state to READY to account for
-			 * $p having already been initialized, and then enable.
-			 */
-			scx_disable_and_exit_task(sch, p);
-			scx_set_task_state(p, SCX_TASK_INIT_BEGIN);
-			scx_set_task_state(p, SCX_TASK_INIT);
-			scx_set_task_sched(p, parent);
-			scx_set_task_state(p, SCX_TASK_READY);
-			scx_enable_task(parent, p);
-		}
-
-		task_rq_unlock(rq, p, &rf);
-		put_task_struct(p);
-	}
-	scx_task_iter_stop(&sti);
-
-	scx_disable_dump(sch);
-
-	scx_cgroup_unlock();
-	percpu_up_write(&scx_fork_rwsem);
-
-	/*
-	 * All tasks are moved off of @sch but there may still be on-going
-	 * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use
-	 * the expedited version as ancestors may be waiting in bypass mode.
-	 * Also, tell the parent that there is no need to keep running bypass
-	 * DSQs for us.
-	 */
-	synchronize_rcu_expedited();
-	disable_bypass_dsp(sch);
-
-	scx_unlink_sched(sch);
-
-	mutex_unlock(&scx_enable_mutex);
-
-	/*
-	 * @sch is now unlinked from the parent's children list. Notify and call
-	 * ops.sub_detach/exit(). Note that ops.sub_detach/exit() must be called
-	 * after unlinking and releasing all locks. See scx_claim_exit().
-	 */
-	wake_up_all(&scx_unlink_waitq);
-
-	if (parent->ops.sub_detach && sch->sub_attached) {
-		struct scx_sub_detach_args sub_detach_args = {
-			.ops = &sch->ops,
-			.cgroup_path = sch->cgrp_path,
-		};
-		SCX_CALL_OP(parent, sub_detach, NULL,
-			    &sub_detach_args);
-	}
-
-	scx_log_sched_disable(sch);
-
-	if (sch->ops.exit)
-		SCX_CALL_OP(sch, exit, NULL, sch->exit_info);
-	if (sch->sub_kset)
-		kobject_del(&sch->sub_kset->kobj);
-	kobject_del(&sch->kobj);
-}
-#else	/* CONFIG_EXT_SUB_SCHED */
-static inline void drain_descendants(struct scx_sched *sch) { }
-static inline void scx_sub_disable(struct scx_sched *sch) { }
-#endif	/* CONFIG_EXT_SUB_SCHED */
-
 static void scx_root_disable(struct scx_sched *sch)
 {
 	struct scx_task_iter sti;
@@ -6121,6 +6050,14 @@ static void scx_root_disable(struct scx_sched *sch)
 	if (sch->ops.exit)
 		SCX_CALL_OP(sch, exit, NULL, sch->exit_info);
 
+	/*
+	 * @sch's non-ops programs such as timers and tracers can fire after
+	 * ops.exit(). Now that exit is complete, stop scx_prog_sched() from
+	 * resolving to @sch and drain in-flight resolvers.
+	 */
+	WRITE_ONCE(sch->dead, true);
+	synchronize_rcu();
+
 	scx_unlink_sched(sch);
 
 	/*
@@ -6140,7 +6077,9 @@ static void scx_root_disable(struct scx_sched *sch)
 	if (sch->sub_kset)
 		kobject_del(&sch->sub_kset->kobj);
 #endif
-	kobject_del(&sch->kobj);
+	/* not added if enable failed before scx_sched_sysfs_add() */
+	if (sch->kobj.state_in_sysfs)
+		kobject_del(&sch->kobj);
 
 	free_kick_syncs();
 
@@ -6177,6 +6116,8 @@ static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind)
 	 */
 	WRITE_ONCE(sch->aborting, true);
 
+	trace_sched_ext_exit(sch, kind);
+
 	/*
 	 * Propagate exits to descendants immediately. Each has a dedicated
 	 * helper kthread and can run in parallel. While most of disabling is
@@ -6240,7 +6181,7 @@ static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind)
  * as a noop. Syncing the irq_work first is required to guarantee the
  * kthread work has been queued before waiting for it.
  */
-static void scx_flush_disable_work(struct scx_sched *sch)
+void scx_flush_disable_work(struct scx_sched *sch)
 {
 	int kind;
 
@@ -6422,6 +6363,7 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s,
 			 bool dump_all_tasks)
 {
 	struct rq *rq = cpu_rq(cpu);
+	struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
 	struct rq_flags rf;
 	struct task_struct *p;
 	struct seq_buf ns;
@@ -6454,18 +6396,18 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s,
 	dump_line(&ns, "          curr=%s[%d] class=%ps",
 		  rq->curr->comm, rq->curr->pid,
 		  rq->curr->sched_class);
-	if (!cpumask_empty(rq->scx.cpus_to_kick))
+	if (!cpumask_empty(pcpu->cpus_to_kick))
 		dump_line(&ns, "  cpus_to_kick   : %*pb",
-			  cpumask_pr_args(rq->scx.cpus_to_kick));
-	if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle))
+			  cpumask_pr_args(pcpu->cpus_to_kick));
+	if (!cpumask_empty(pcpu->cpus_to_kick_if_idle))
 		dump_line(&ns, "  idle_to_kick   : %*pb",
-			  cpumask_pr_args(rq->scx.cpus_to_kick_if_idle));
-	if (!cpumask_empty(rq->scx.cpus_to_preempt))
+			  cpumask_pr_args(pcpu->cpus_to_kick_if_idle));
+	if (!cpumask_empty(pcpu->cpus_to_preempt))
 		dump_line(&ns, "  cpus_to_preempt: %*pb",
-			  cpumask_pr_args(rq->scx.cpus_to_preempt));
-	if (!cpumask_empty(rq->scx.cpus_to_wait))
+			  cpumask_pr_args(pcpu->cpus_to_preempt));
+	if (!cpumask_empty(pcpu->cpus_to_wait))
 		dump_line(&ns, "  cpus_to_wait   : %*pb",
-			  cpumask_pr_args(rq->scx.cpus_to_wait));
+			  cpumask_pr_args(pcpu->cpus_to_wait));
 	if (!cpumask_empty(rq->scx.cpus_to_sync))
 		dump_line(&ns, "  cpus_to_sync   : %*pb",
 			  cpumask_pr_args(rq->scx.cpus_to_sync));
@@ -6569,14 +6511,23 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei,
 	dump_line(&s, "----------");
 
 	/*
-	 * Dump the exit CPU first so it isn't lost to dump truncation, then
-	 * walk the rest in order, skipping the one already dumped.
+	 * Dump stalled CPUs first so they aren't lost to dump truncation, then
+	 * walk the rest in order. Fall back to exit_cpu if no stall mask set.
 	 */
-	if (ei->exit_cpu >= 0)
-		scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks);
-	for_each_possible_cpu(cpu) {
-		if (cpu != ei->exit_cpu)
+	if (!cpumask_empty(sch->stall_cpus)) {
+		for_each_cpu(cpu, sch->stall_cpus)
 			scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks);
+		for_each_possible_cpu(cpu) {
+			if (!cpumask_test_cpu(cpu, sch->stall_cpus))
+				scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks);
+		}
+	} else {
+		if (ei->exit_cpu >= 0)
+			scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks);
+		for_each_possible_cpu(cpu) {
+			if (cpu != ei->exit_cpu)
+				scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks);
+		}
 	}
 
 	dump_newline(&s);
@@ -6584,19 +6535,9 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei,
 	dump_line(&s, "--------------");
 
 	scx_read_events(sch, &events);
-	scx_dump_event(s, &events, SCX_EV_SELECT_CPU_FALLBACK);
-	scx_dump_event(s, &events, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE);
-	scx_dump_event(s, &events, SCX_EV_DISPATCH_KEEP_LAST);
-	scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_EXITING);
-	scx_dump_event(s, &events, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED);
-	scx_dump_event(s, &events, SCX_EV_REENQ_IMMED);
-	scx_dump_event(s, &events, SCX_EV_REENQ_LOCAL_REPEAT);
-	scx_dump_event(s, &events, SCX_EV_REFILL_SLICE_DFL);
-	scx_dump_event(s, &events, SCX_EV_BYPASS_DURATION);
-	scx_dump_event(s, &events, SCX_EV_BYPASS_DISPATCH);
-	scx_dump_event(s, &events, SCX_EV_BYPASS_ACTIVATE);
-	scx_dump_event(s, &events, SCX_EV_INSERT_NOT_OWNED);
-	scx_dump_event(s, &events, SCX_EV_SUB_BYPASS_DISPATCH);
+#define SCX_EVENT(name)	scx_dump_event(s, &events, name)
+	SCX_EVENTS_LIST(SCX_EVENT);
+#undef SCX_EVENT
 
 	if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker))
 		memcpy(ei->dump + dump_len - sizeof(trunc_marker),
@@ -6653,7 +6594,7 @@ static int alloc_kick_syncs(void)
 	 * can exceed percpu allocator limits on large machines.
 	 */
 	for_each_possible_cpu(cpu) {
-		struct scx_kick_syncs **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
+		struct scx_kick_syncs __rcu **ksyncs = per_cpu_ptr(&scx_kick_syncs, cpu);
 		struct scx_kick_syncs *new_ksyncs;
 
 		WARN_ON_ONCE(rcu_access_pointer(*ksyncs));
@@ -6695,31 +6636,13 @@ static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node)
 	return pnode;
 }
 
-/*
- * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid
- * starvation. During the READY -> ENABLED task switching loop, the calling
- * thread's sched_class gets switched from fair to ext. As fair has higher
- * priority than ext, the calling thread can be indefinitely starved under
- * fair-class saturation, leading to a system hang.
- */
-struct scx_enable_cmd {
-	struct kthread_work	work;
-	union {
-		struct sched_ext_ops		*ops;
-		struct sched_ext_ops_cid	*ops_cid;
-	};
-	bool			is_cid_type;
-	struct bpf_map		*arena_map;	/* arena ref to transfer to sch */
-	int			ret;
-};
-
 /*
  * Allocate and initialize a new scx_sched. @cgrp's reference is always
  * consumed whether the function succeeds or fails.
  */
-static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
-						 struct cgroup *cgrp,
-						 struct scx_sched *parent)
+struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
+					  struct cgroup *cgrp,
+					  struct scx_sched *parent)
 {
 	struct sched_ext_ops *ops = cmd->ops;
 	struct scx_sched *sch;
@@ -6766,7 +6689,7 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
 	}
 
 	for_each_possible_cpu(cpu) {
-		ret = init_dsq(bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch);
+		ret = init_dsq(scx_bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch);
 		if (ret) {
 			bypass_fail_cpu = cpu;
 			goto err_free_pcpu;
@@ -6776,8 +6699,20 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
 	for_each_possible_cpu(cpu) {
 		struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
 
+		node = cpu_to_node(cpu);
 		pcpu->sch = sch;
 		INIT_LIST_HEAD(&pcpu->deferred_reenq_local.node);
+#ifdef CONFIG_EXT_SUB_SCHED
+		init_llist_node(&pcpu->ecaps_to_sync_node);
+#endif
+		INIT_LIST_HEAD(&pcpu->to_kick_node);
+		if (!zalloc_cpumask_var_node(&pcpu->cpus_to_kick, GFP_KERNEL, node) ||
+		    !zalloc_cpumask_var_node(&pcpu->cpus_to_kick_if_idle, GFP_KERNEL, node) ||
+		    !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt, GFP_KERNEL, node) ||
+		    !zalloc_cpumask_var_node(&pcpu->cpus_to_wait, GFP_KERNEL, node)) {
+			ret = -ENOMEM;
+			goto err_free_pcpu;
+		}
 	}
 
 	sch->helper = kthread_run_worker(0, "sched_ext_helper");
@@ -6793,6 +6728,7 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
 		       level * sizeof(parent->ancestors[0]));
 	sch->ancestors[level] = sch;
 	sch->level = level;
+	sch->id = atomic64_inc_return(&scx_sched_id_cursor);
 
 	if (ops->timeout_ms)
 		sch->watchdog_timeout = msecs_to_jiffies(ops->timeout_ms);
@@ -6813,6 +6749,10 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
 		ret = -ENOMEM;
 		goto err_free_lb_cpumask;
 	}
+	if (!zalloc_cpumask_var(&sch->stall_cpus, GFP_KERNEL)) {
+		ret = -ENOMEM;
+		goto err_free_lb_resched_cpumask;
+	}
 	/*
 	 * Copy ops through the right union view. For cid-form the source is
 	 * struct sched_ext_ops_cid which lacks the trailing cpu_acquire/
@@ -6825,11 +6765,6 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
 		sch->ops = *cmd->ops;
 	}
 
-	rcu_assign_pointer(ops->priv, sch);
-
-	sch->kobj.kset = scx_kset;
-	INIT_LIST_HEAD(&sch->all);
-
 #ifdef CONFIG_EXT_SUB_SCHED
 	char *buf = kzalloc(PATH_MAX, GFP_KERNEL);
 	if (!buf) {
@@ -6847,37 +6782,35 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
 	sch->cgrp = cgrp;
 	INIT_LIST_HEAD(&sch->children);
 	INIT_LIST_HEAD(&sch->sibling);
+#endif	/* CONFIG_EXT_SUB_SCHED */
 
-	if (parent)
-		ret = kobject_init_and_add(&sch->kobj, &scx_ktype,
-					   &parent->sub_kset->kobj,
-					   "sub-%llu", cgroup_id(cgrp));
-	else
-		ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root");
+	/*
+	 * Publishing makes @sch visible to scx_prog_sched() readers. Failure
+	 * paths after this point must free @sch through kobject_put() whose
+	 * release path defers the actual freeing by an RCU grace period.
+	 */
+	rcu_assign_pointer(ops->priv, sch);
 
-	if (ret < 0) {
-		RCU_INIT_POINTER(ops->priv, NULL);
-		kobject_put(&sch->kobj);
-		return ERR_PTR(ret);
-	}
+	sch->kobj.kset = scx_kset;
+	INIT_LIST_HEAD(&sch->all);
 
-	if (ops->sub_attach) {
-		sch->sub_kset = kset_create_and_add("sub", NULL, &sch->kobj);
-		if (!sch->sub_kset) {
-			RCU_INIT_POINTER(ops->priv, NULL);
-			kobject_put(&sch->kobj);
-			return ERR_PTR(-ENOMEM);
-		}
-	}
-#else	/* CONFIG_EXT_SUB_SCHED */
-	ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root");
-	if (ret < 0) {
-		RCU_INIT_POINTER(ops->priv, NULL);
-		kobject_put(&sch->kobj);
-		return ERR_PTR(ret);
+#ifdef CONFIG_EXT_SUB_SCHED
+	if (parent) {
+		/*
+		 * Pin @parent for @sch's lifetime. The kobject hierarchy pins
+		 * it only via @parent->sub_kset, which is dropped during
+		 * disable. Released in scx_sched_free_rcu_work().
+		 */
+		kobject_get(&parent->kobj);
 	}
 #endif	/* CONFIG_EXT_SUB_SCHED */
 
+	/*
+	 * Init the kobj but don't add to sysfs yet. The enable path calls
+	 * scx_sched_sysfs_add() once @sch's sysfs-visible state is initialized.
+	 */
+	kobject_init(&sch->kobj, &scx_ktype);
+
 	/*
 	 * Consume the arena_map ref bpf_scx_reg_cid() took. Defer to here so
 	 * earlier failure paths leave cmd->arena_map set and bpf_scx_reg_cid
@@ -6895,18 +6828,27 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
 
 #ifdef CONFIG_EXT_SUB_SCHED
 err_free_lb_resched:
-	RCU_INIT_POINTER(ops->priv, NULL);
-	free_cpumask_var(sch->bypass_lb_resched_cpumask);
+	free_cpumask_var(sch->stall_cpus);
 #endif
+err_free_lb_resched_cpumask:
+	free_cpumask_var(sch->bypass_lb_resched_cpumask);
 err_free_lb_cpumask:
 	free_cpumask_var(sch->bypass_lb_donee_cpumask);
 err_stop_helper:
 	kthread_destroy_worker(sch->helper);
 err_free_pcpu:
+	for_each_possible_cpu(cpu) {
+		struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
+
+		free_cpumask_var(pcpu->cpus_to_kick);
+		free_cpumask_var(pcpu->cpus_to_kick_if_idle);
+		free_cpumask_var(pcpu->cpus_to_preempt);
+		free_cpumask_var(pcpu->cpus_to_wait);
+	}
 	for_each_possible_cpu(cpu) {
 		if (cpu == bypass_fail_cpu)
 			break;
-		exit_dsq(bypass_dsq(sch, cpu));
+		exit_dsq(scx_bypass_dsq(sch, cpu));
 	}
 	free_percpu(sch->pcpu);
 err_free_pnode:
@@ -6926,12 +6868,42 @@ static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
 	return ERR_PTR(ret);
 }
 
-static int check_hotplug_seq(struct scx_sched *sch,
-			      const struct sched_ext_ops *ops)
+/*
+ * Add @sch's kobject to sysfs, and create its sub_kset if the scheduler
+ * implements ops.sub_attach. Called by the enable workfns once @sch's
+ * sysfs-visible state is initialized.
+ */
+int scx_sched_sysfs_add(struct scx_sched *sch)
 {
-	unsigned long long global_hotplug_seq;
-
-	/*
+#ifdef CONFIG_EXT_SUB_SCHED
+	struct scx_sched *parent = scx_parent(sch);
+	int ret;
+
+	if (parent)
+		ret = kobject_add(&sch->kobj, &parent->sub_kset->kobj,
+				  "sub-%llu", cgroup_id(sch_cgroup(sch)));
+	else
+		ret = kobject_add(&sch->kobj, NULL, "root");
+	if (ret < 0)
+		return ret;
+
+	if (sch->ops.sub_attach) {
+		sch->sub_kset = kset_create_and_add("sub", NULL, &sch->kobj);
+		if (!sch->sub_kset)
+			return -ENOMEM;
+	}
+	return 0;
+#else
+	return kobject_add(&sch->kobj, NULL, "root");
+#endif
+}
+
+static int check_hotplug_seq(struct scx_sched *sch,
+			      const struct sched_ext_ops *ops)
+{
+	unsigned long long global_hotplug_seq;
+
+	/*
 	 * If a hotplug event has occurred between when a scheduler was
 	 * initialized, and when we were able to attach, exit and notify user
 	 * space about it.
@@ -6950,7 +6922,7 @@ static int check_hotplug_seq(struct scx_sched *sch,
 	return 0;
 }
 
-static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops)
+int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops)
 {
 	/*
 	 * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the
@@ -6988,7 +6960,7 @@ static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops)
 	 * run past the BPF allocation. Skip for cid-form.
 	 */
 	if (!sch->is_cid_type && (ops->cpu_acquire || ops->cpu_release))
-		pr_warn("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n");
+		pr_warn_ratelimited("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n");
 
 	/*
 	 * Sub-scheduler support is tied to the cid-form struct_ops. A sub-sched
@@ -7078,6 +7050,8 @@ static void scx_root_enable_workfn(struct kthread_work *work)
 		rq->scx.cpuperf_target = SCX_CPUPERF_ONE;
 	}
 
+	scx_discard_stale_ecaps_syncs();
+
 	/*
 	 * Keep CPUs stable during enable so that the BPF scheduler can track
 	 * online CPUs by watching ->on/offline_cpu() after ->init().
@@ -7110,15 +7084,19 @@ static void scx_root_enable_workfn(struct kthread_work *work)
 
 	scx_idle_enable(ops);
 
-	if (sch->ops.init) {
-		ret = SCX_CALL_OP_RET(sch, init, NULL);
+	/*
+	 * A cid-form scheduler finalizes its cid layout in ops.init_cids(),
+	 * which may call scx_bpf_cid_override(). Run it before the caps and
+	 * shard state are built so the final layout is in effect.
+	 */
+	if (sch->is_cid_type && sch->ops_cid.init_cids) {
+		ret = SCX_CALL_OP_RET(sch, init_cids, NULL);
 		if (ret) {
-			ret = ops_sanitize_err(sch, "init", ret);
+			ret = scx_ops_sanitize_err(sch, "init_cids", ret);
 			cpus_read_unlock();
-			scx_error(sch, "ops.init() failed (%d)", ret);
+			scx_error(sch, "ops.init_cids() failed (%d)", ret);
 			goto err_disable;
 		}
-		sch->exit_info->flags |= SCX_EFLAG_INITIALIZED;
 	}
 
 	ret = scx_arena_pool_init(sch);
@@ -7133,6 +7111,32 @@ static void scx_root_enable_workfn(struct kthread_work *work)
 		goto err_disable;
 	}
 
+	ret = scx_alloc_pshards(sch);
+	if (ret) {
+		cpus_read_unlock();
+		goto err_disable;
+	}
+
+	scx_init_root_caps(sch);
+
+	/* the cid caps and shards are live now, so ops.init() can query them */
+	if (sch->ops.init) {
+		ret = SCX_CALL_OP_RET(sch, init, NULL);
+		if (ret) {
+			ret = scx_ops_sanitize_err(sch, "init", ret);
+			cpus_read_unlock();
+			scx_error(sch, "ops.init() failed (%d)", ret);
+			goto err_disable;
+		}
+		sch->exit_info->flags |= SCX_EFLAG_INITIALIZED;
+	}
+
+	ret = scx_sched_sysfs_add(sch);
+	if (ret) {
+		cpus_read_unlock();
+		goto err_disable;
+	}
+
 	for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++)
 		if (((void (**)(void))ops)[i])
 			set_bit(i, sch->has_op);
@@ -7146,7 +7150,7 @@ static void scx_root_enable_workfn(struct kthread_work *work)
 
 	cpus_read_unlock();
 
-	ret = validate_ops(sch, ops);
+	ret = scx_validate_ops(sch, ops);
 	if (ret)
 		goto err_disable;
 
@@ -7267,468 +7271,127 @@ static void scx_root_enable_workfn(struct kthread_work *work)
 			 */
 			scx_sub_init_cancel_task(sch, p);
 		} else {
-			scx_set_task_state(p, SCX_TASK_INIT);
-			scx_set_task_sched(p, sch);
-			scx_set_task_state(p, SCX_TASK_READY);
-		}
-
-		/*
-		 * Insert into the tid hash. scx_tasks_lock is held by the iter;
-		 * list_empty() guards against sched_ext_dead() having taken @p
-		 * off the list while init ran unlocked.
-		 */
-		if (scx_tid_to_task_enabled() && !list_empty(&p->scx.tasks_node))
-			scx_tid_hash_insert(p);
-
-		put_task_struct(p);
-	}
-	scx_task_iter_stop(&sti);
-	scx_cgroup_unlock();
-	percpu_up_write(&scx_fork_rwsem);
-
-	/*
-	 * All tasks are READY. It's safe to turn on scx_enabled() and switch
-	 * all eligible tasks.
-	 */
-	WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL));
-	static_branch_enable(&__scx_enabled);
-
-	/*
-	 * We're fully committed and can't fail. The task READY -> ENABLED
-	 * transitions here are synchronized against sched_ext_free() through
-	 * scx_tasks_lock.
-	 */
-	percpu_down_write(&scx_fork_rwsem);
-	scx_task_iter_start(&sti, NULL);
-	while ((p = scx_task_iter_next_locked(&sti))) {
-		unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE;
-		const struct sched_class *old_class = p->sched_class;
-		const struct sched_class *new_class = scx_setscheduler_class(p);
-
-		if (scx_get_task_state(p) != SCX_TASK_READY)
-			continue;
-
-		if (old_class != new_class)
-			queue_flags |= DEQUEUE_CLASS;
-
-		scoped_guard (sched_change, p, queue_flags) {
-			p->scx.slice = READ_ONCE(sch->slice_dfl);
-			p->sched_class = new_class;
-		}
-	}
-	scx_task_iter_stop(&sti);
-	percpu_up_write(&scx_fork_rwsem);
-
-	scx_bypass(sch, false);
-
-	if (!scx_tryset_enable_state(SCX_ENABLED, SCX_ENABLING)) {
-		WARN_ON_ONCE(atomic_read(&sch->exit_kind) == SCX_EXIT_NONE);
-		goto err_disable;
-	}
-
-	if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL))
-		static_branch_enable(&__scx_switched_all);
-
-	/*
-	 * Detach the fair_server bandwidth reservation now that the switch
-	 * is fully committed. In full mode (!SCX_OPS_SWITCH_PARTIAL) no
-	 * task will ever run in the fair class, so give that bandwidth
-	 * back to the RT class. The matching ext_server attach already
-	 * happened earlier; this only releases bandwidth and cannot fail.
-	 *
-	 * In partial mode keep fair_server attached.
-	 */
-	if (scx_switched_all()) {
-		for_each_possible_cpu(cpu) {
-			struct rq *rq = cpu_rq(cpu);
-
-			guard(rq_lock_irqsave)(rq);
-			update_rq_clock(rq);
-			dl_server_detach_bw(&rq->fair_server);
-		}
-	}
-
-	pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n",
-		sch->ops.name, scx_switched_all() ? "" : " (partial)");
-	kobject_uevent(&sch->kobj, KOBJ_ADD);
-	mutex_unlock(&scx_enable_mutex);
-
-	atomic_long_inc(&scx_enable_seq);
-
-	cmd->ret = 0;
-	return;
-
-err_free_tid_hash:
-	if (ops->flags & SCX_OPS_TID_TO_TASK)
-		rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL);
-err_free_ksyncs:
-	free_kick_syncs();
-err_unlock:
-	mutex_unlock(&scx_enable_mutex);
-	cmd->ret = ret;
-	return;
-
-err_disable_unlock_all:
-	scx_cgroup_unlock();
-	percpu_up_write(&scx_fork_rwsem);
-	/* we'll soon enter disable path, keep bypass on */
-err_disable:
-	mutex_unlock(&scx_enable_mutex);
-	/*
-	 * Returning an error code here would not pass all the error information
-	 * to userspace. Record errno using scx_error() for cases scx_error()
-	 * wasn't already invoked and exit indicating success so that the error
-	 * is notified through ops.exit() with all the details.
-	 *
-	 * Flush scx_disable_work to ensure that error is reported before init
-	 * completion. sch's base reference will be put by bpf_scx_unreg().
-	 */
-	scx_error(sch, "scx_root_enable() failed (%d)", ret);
-	scx_flush_disable_work(sch);
-	cmd->ret = 0;
-}
-
-#ifdef CONFIG_EXT_SUB_SCHED
-/* verify that a scheduler can be attached to @cgrp and return the parent */
-static struct scx_sched *find_parent_sched(struct cgroup *cgrp)
-{
-	struct scx_sched *parent = cgrp->scx_sched;
-	struct scx_sched *pos;
-
-	lockdep_assert_held(&scx_sched_lock);
-
-	/* can't attach twice to the same cgroup */
-	if (parent->cgrp == cgrp)
-		return ERR_PTR(-EBUSY);
-
-	/* does $parent allow sub-scheds? */
-	if (!parent->ops.sub_attach)
-		return ERR_PTR(-EOPNOTSUPP);
-
-	/* can't insert between $parent and its exiting children */
-	list_for_each_entry(pos, &parent->children, sibling)
-		if (cgroup_is_descendant(pos->cgrp, cgrp))
-			return ERR_PTR(-EBUSY);
-
-	return parent;
-}
-
-static bool assert_task_ready_or_enabled(struct task_struct *p)
-{
-	u32 state = scx_get_task_state(p);
-
-	switch (state) {
-	case SCX_TASK_READY:
-	case SCX_TASK_ENABLED:
-		return true;
-	default:
-		WARN_ONCE(true, "sched_ext: Invalid task state %d for %s[%d] during enabling sub sched",
-			  state, p->comm, p->pid);
-		return false;
-	}
-}
-
-static void scx_sub_enable_workfn(struct kthread_work *work)
-{
-	struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work);
-	struct sched_ext_ops *ops = cmd->ops;
-	struct cgroup *cgrp;
-	struct scx_sched *parent, *sch;
-	struct scx_task_iter sti;
-	struct task_struct *p;
-	s32 i, ret;
-
-	mutex_lock(&scx_enable_mutex);
-
-	if (!scx_enabled()) {
-		ret = -ENODEV;
-		goto out_unlock;
-	}
-
-	/* See scx_root_enable_workfn() for the @ops->priv check. */
-	if (rcu_access_pointer(ops->priv)) {
-		ret = -EBUSY;
-		goto out_unlock;
-	}
-
-	cgrp = cgroup_get_from_id(ops->sub_cgroup_id);
-	if (IS_ERR(cgrp)) {
-		ret = PTR_ERR(cgrp);
-		goto out_unlock;
-	}
-
-	raw_spin_lock_irq(&scx_sched_lock);
-	parent = find_parent_sched(cgrp);
-	if (IS_ERR(parent)) {
-		raw_spin_unlock_irq(&scx_sched_lock);
-		ret = PTR_ERR(parent);
-		goto out_put_cgrp;
-	}
-	kobject_get(&parent->kobj);
-	raw_spin_unlock_irq(&scx_sched_lock);
-
-	/* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */
-	sch = scx_alloc_and_add_sched(cmd, cgrp, parent);
-	kobject_put(&parent->kobj);
-	if (IS_ERR(sch)) {
-		ret = PTR_ERR(sch);
-		goto out_unlock;
-	}
-
-	ret = scx_link_sched(sch);
-	if (ret)
-		goto err_disable;
-
-	if (sch->level >= SCX_SUB_MAX_DEPTH) {
-		scx_error(sch, "max nesting depth %d violated",
-			  SCX_SUB_MAX_DEPTH);
-		goto err_disable;
-	}
-
-	if (sch->ops.init) {
-		ret = SCX_CALL_OP_RET(sch, init, NULL);
-		if (ret) {
-			ret = ops_sanitize_err(sch, "init", ret);
-			scx_error(sch, "ops.init() failed (%d)", ret);
-			goto err_disable;
-		}
-		sch->exit_info->flags |= SCX_EFLAG_INITIALIZED;
-	}
-
-	ret = scx_arena_pool_init(sch);
-	if (ret)
-		goto err_disable;
-
-	ret = scx_set_cmask_scratch_alloc(sch);
-	if (ret)
-		goto err_disable;
-
-	if (validate_ops(sch, ops))
-		goto err_disable;
-
-	struct scx_sub_attach_args sub_attach_args = {
-		.ops = &sch->ops,
-		.cgroup_path = sch->cgrp_path,
-	};
-
-	ret = SCX_CALL_OP_RET(parent, sub_attach, NULL,
-			      &sub_attach_args);
-	if (ret) {
-		ret = ops_sanitize_err(sch, "sub_attach", ret);
-		scx_error(sch, "parent rejected (%d)", ret);
-		goto err_disable;
-	}
-	sch->sub_attached = true;
-
-	scx_bypass(sch, true);
-
-	for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++)
-		if (((void (**)(void))ops)[i])
-			set_bit(i, sch->has_op);
-
-	percpu_down_write(&scx_fork_rwsem);
-	scx_cgroup_lock();
-
-	/*
-	 * Set cgroup->scx_sched's and check CSS_ONLINE. Either we see
-	 * !CSS_ONLINE or scx_cgroup_lifetime_notify() sees and shoots us down.
-	 */
-	set_cgroup_sched(sch_cgroup(sch), sch);
-	if (!(cgrp->self.flags & CSS_ONLINE)) {
-		scx_error(sch, "cgroup is not online");
-		goto err_unlock_and_disable;
-	}
-
-	/*
-	 * Initialize tasks for the new child $sch without exiting them for
-	 * $parent so that the tasks can always be reverted back to $parent
-	 * sched on child init failure.
-	 */
-	WARN_ON_ONCE(scx_enabling_sub_sched);
-	scx_enabling_sub_sched = sch;
-
-	scx_task_iter_start(&sti, sch->cgrp);
-	while ((p = scx_task_iter_next_locked(&sti))) {
-		struct rq *rq;
-		struct rq_flags rf;
-
-		/*
-		 * Task iteration may visit the same task twice when racing
-		 * against exiting. Use %SCX_TASK_SUB_INIT to mark tasks which
-		 * finished __scx_init_task() and skip if set.
-		 *
-		 * A task may exit and get freed between __scx_init_task()
-		 * completion and scx_enable_task(). In such cases,
-		 * scx_disable_and_exit_task() must exit the task for both the
-		 * parent and child scheds.
-		 */
-		if (p->scx.flags & SCX_TASK_SUB_INIT)
-			continue;
-
-		/* @p is pinned by the iter; see scx_sub_disable() */
-		get_task_struct(p);
-
-		if (!assert_task_ready_or_enabled(p)) {
-			ret = -EINVAL;
-			goto abort;
-		}
-
-		scx_task_iter_unlock(&sti);
-
-		/*
-		 * As $p is still on $parent, it can't be transitioned to INIT.
-		 * Let's worry about task state later. Use __scx_init_task().
-		 */
-		ret = __scx_init_task(sch, p, false);
-		if (ret)
-			goto abort;
-
-		rq = task_rq_lock(p, &rf);
-
-		if (scx_get_task_state(p) == SCX_TASK_DEAD) {
-			/*
-			 * sched_ext_dead() raced us between __scx_init_task()
-			 * and this rq lock and ran exit_task() on $parent (the
-			 * sched @p was on at that point), not on @sch. @sch's
-			 * just-completed init is owed an exit_task() and we
-			 * issue it here.
-			 */
-			scx_sub_init_cancel_task(sch, p);
-			task_rq_unlock(rq, p, &rf);
-			put_task_struct(p);
-			continue;
+			scx_set_task_state(p, SCX_TASK_INIT);
+			scx_set_task_sched(p, sch);
+			scx_set_task_state(p, SCX_TASK_READY);
 		}
 
-		p->scx.flags |= SCX_TASK_SUB_INIT;
-		task_rq_unlock(rq, p, &rf);
+		/*
+		 * Insert into the tid hash. scx_tasks_lock is held by the iter;
+		 * list_empty() guards against sched_ext_dead() having taken @p
+		 * off the list while init ran unlocked.
+		 */
+		if (scx_tid_to_task_enabled() && !list_empty(&p->scx.tasks_node))
+			scx_tid_hash_insert(p);
 
 		put_task_struct(p);
 	}
 	scx_task_iter_stop(&sti);
+	scx_cgroup_unlock();
+	percpu_up_write(&scx_fork_rwsem);
+
+	/*
+	 * All tasks are READY. It's safe to turn on scx_enabled() and switch
+	 * all eligible tasks.
+	 */
+	WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL));
+	static_branch_enable(&__scx_enabled);
 
 	/*
-	 * All tasks are prepped. Disable/exit tasks for $parent and enable for
-	 * the new @sch.
+	 * We're fully committed and can't fail. The task READY -> ENABLED
+	 * transitions here are synchronized against sched_ext_free() through
+	 * scx_tasks_lock.
 	 */
-	scx_task_iter_start(&sti, sch->cgrp);
+	percpu_down_write(&scx_fork_rwsem);
+	scx_task_iter_start(&sti, NULL);
 	while ((p = scx_task_iter_next_locked(&sti))) {
-		/*
-		 * Use clearing of %SCX_TASK_SUB_INIT to detect and skip
-		 * duplicate iterations.
-		 */
-		if (!(p->scx.flags & SCX_TASK_SUB_INIT))
-			continue;
+		unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE;
+		const struct sched_class *old_class = p->sched_class;
+		const struct sched_class *new_class = scx_setscheduler_class(p);
 
-		scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
-			/*
-			 * $p must be either READY or ENABLED. If ENABLED,
-			 * __scx_disabled_and_exit_task() first disables and
-			 * makes it READY. However, after exiting $p, it will
-			 * leave $p as READY.
-			 */
-			assert_task_ready_or_enabled(p);
-			__scx_disable_and_exit_task(parent, p);
+		if (scx_get_task_state(p) != SCX_TASK_READY)
+			continue;
 
-			/*
-			 * $p is now only initialized for @sch and READY, which
-			 * is what we want. Assign it to @sch and enable.
-			 */
-			scx_set_task_sched(p, sch);
-			scx_enable_task(sch, p);
+		if (old_class != new_class)
+			queue_flags |= DEQUEUE_CLASS;
 
-			p->scx.flags &= ~SCX_TASK_SUB_INIT;
+		scoped_guard (sched_change, p, queue_flags) {
+			set_task_slice(p, READ_ONCE(sch->slice_dfl));
+			p->sched_class = new_class;
 		}
 	}
 	scx_task_iter_stop(&sti);
-
-	scx_enabling_sub_sched = NULL;
-
-	scx_cgroup_unlock();
 	percpu_up_write(&scx_fork_rwsem);
 
 	scx_bypass(sch, false);
 
-	pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name);
-	kobject_uevent(&sch->kobj, KOBJ_ADD);
-	ret = 0;
-	goto out_unlock;
-
-out_put_cgrp:
-	cgroup_put(cgrp);
-out_unlock:
-	mutex_unlock(&scx_enable_mutex);
-	cmd->ret = ret;
-	return;
+	if (!scx_tryset_enable_state(SCX_ENABLED, SCX_ENABLING)) {
+		WARN_ON_ONCE(atomic_read(&sch->exit_kind) == SCX_EXIT_NONE);
+		goto err_disable;
+	}
 
-abort:
-	put_task_struct(p);
-	scx_task_iter_stop(&sti);
+	if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL))
+		static_branch_enable(&__scx_switched_all);
 
 	/*
-	 * Undo __scx_init_task() for tasks we marked. scx_enable_task() never
-	 * ran for @sch on them, so calling scx_disable_task() here would invoke
-	 * ops.disable() without a matching ops.enable(). scx_enabling_sub_sched
-	 * must stay set until SUB_INIT is cleared from every marked task -
-	 * scx_disable_and_exit_task() reads it when a task exits concurrently.
+	 * Detach the fair_server bandwidth reservation now that the switch
+	 * is fully committed. In full mode (!SCX_OPS_SWITCH_PARTIAL) no
+	 * task will ever run in the fair class, so give that bandwidth
+	 * back to the RT class. The matching ext_server attach already
+	 * happened earlier; this only releases bandwidth and cannot fail.
+	 *
+	 * In partial mode keep fair_server attached.
 	 */
-	scx_task_iter_start(&sti, sch->cgrp);
-	while ((p = scx_task_iter_next_locked(&sti))) {
-		if (p->scx.flags & SCX_TASK_SUB_INIT) {
-			scx_sub_init_cancel_task(sch, p);
-			p->scx.flags &= ~SCX_TASK_SUB_INIT;
+	if (scx_switched_all()) {
+		for_each_possible_cpu(cpu) {
+			struct rq *rq = cpu_rq(cpu);
+
+			guard(rq_lock_irqsave)(rq);
+			update_rq_clock(rq);
+			dl_server_detach_bw(&rq->fair_server);
 		}
 	}
-	scx_task_iter_stop(&sti);
-	scx_enabling_sub_sched = NULL;
-err_unlock_and_disable:
-	/* we'll soon enter disable path, keep bypass on */
+
+	pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n",
+		sch->ops.name, scx_switched_all() ? "" : " (partial)");
+	kobject_uevent(&sch->kobj, KOBJ_ADD);
+	mutex_unlock(&scx_enable_mutex);
+
+	atomic_long_inc(&scx_enable_seq);
+
+	cmd->ret = 0;
+	return;
+
+err_free_tid_hash:
+	if (ops->flags & SCX_OPS_TID_TO_TASK)
+		rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL);
+err_free_ksyncs:
+	free_kick_syncs();
+err_unlock:
+	mutex_unlock(&scx_enable_mutex);
+	cmd->ret = ret;
+	return;
+
+err_disable_unlock_all:
 	scx_cgroup_unlock();
 	percpu_up_write(&scx_fork_rwsem);
+	/* we'll soon enter disable path, keep bypass on */
 err_disable:
 	mutex_unlock(&scx_enable_mutex);
+	/*
+	 * Returning an error code here would not pass all the error information
+	 * to userspace. Record errno using scx_error() for cases scx_error()
+	 * wasn't already invoked and exit indicating success so that the error
+	 * is notified through ops.exit() with all the details.
+	 *
+	 * Flush scx_disable_work to ensure that error is reported before init
+	 * completion. sch's base reference will be put by bpf_scx_unreg().
+	 */
+	scx_error(sch, "scx_root_enable() failed (%d)", ret);
 	scx_flush_disable_work(sch);
 	cmd->ret = 0;
 }
 
-static s32 scx_cgroup_lifetime_notify(struct notifier_block *nb,
-				      unsigned long action, void *data)
-{
-	struct cgroup *cgrp = data;
-	struct cgroup *parent = cgroup_parent(cgrp);
-
-	if (!cgroup_on_dfl(cgrp))
-		return NOTIFY_OK;
-
-	switch (action) {
-	case CGROUP_LIFETIME_ONLINE:
-		/* inherit ->scx_sched from $parent */
-		if (parent)
-			rcu_assign_pointer(cgrp->scx_sched, parent->scx_sched);
-		break;
-	case CGROUP_LIFETIME_OFFLINE:
-		/* if there is a sched attached, shoot it down */
-		if (cgrp->scx_sched && cgrp->scx_sched->cgrp == cgrp)
-			scx_exit(cgrp->scx_sched, SCX_EXIT_UNREG_KERN,
-				 SCX_ECODE_RSN_CGROUP_OFFLINE,
-				 "cgroup %llu going offline", cgroup_id(cgrp));
-		break;
-	}
-
-	return NOTIFY_OK;
-}
-
-static struct notifier_block scx_cgroup_lifetime_nb = {
-	.notifier_call = scx_cgroup_lifetime_notify,
-};
-
-static s32 __init scx_cgroup_lifetime_notifier_init(void)
-{
-	return blocking_notifier_chain_register(&cgroup_lifetime_notifier,
-						&scx_cgroup_lifetime_nb);
-}
-core_initcall(scx_cgroup_lifetime_notifier_init);
-#endif	/* CONFIG_EXT_SUB_SCHED */
-
 static s32 scx_enable(struct scx_enable_cmd *cmd, struct bpf_link *link)
 {
 	static struct kthread_worker *helper;
@@ -7791,6 +7454,21 @@ static bool bpf_scx_is_valid_access(int off, int size,
 	return btf_ctx_access(off, size, type, prog, info);
 }
 
+/* common to both forms: only scx.disallow is writable */
+static int bpf_scx_btf_struct_access_common(const struct bpf_reg_state *reg,
+					    int off, int size)
+{
+	const struct btf_type *t;
+
+	t = btf_type_by_id(reg->btf, reg->btf_id);
+	if (t == task_struct_type &&
+	    off >= offsetof(struct task_struct, scx.disallow) &&
+	    off + size <= offsetofend(struct task_struct, scx.disallow))
+		return SCALAR_VALUE;
+
+	return -EACCES;
+}
+
 static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log,
 				     const struct bpf_reg_state *reg, int off,
 				     int size)
@@ -7799,23 +7477,22 @@ static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log,
 
 	t = btf_type_by_id(reg->btf, reg->btf_id);
 	if (t == task_struct_type) {
-		/*
-		 * COMPAT: Will be removed in v6.23.
-		 */
 		if ((off >= offsetof(struct task_struct, scx.slice) &&
 		     off + size <= offsetofend(struct task_struct, scx.slice)) ||
 		    (off >= offsetof(struct task_struct, scx.dsq_vtime) &&
-		     off + size <= offsetofend(struct task_struct, scx.dsq_vtime))) {
-			pr_warn("sched_ext: Writing directly to p->scx.slice/dsq_vtime is deprecated, use scx_bpf_task_set_slice/dsq_vtime()");
-			return SCALAR_VALUE;
-		}
-
-		if (off >= offsetof(struct task_struct, scx.disallow) &&
-		    off + size <= offsetofend(struct task_struct, scx.disallow))
+		     off + size <= offsetofend(struct task_struct, scx.dsq_vtime)))
 			return SCALAR_VALUE;
 	}
 
-	return -EACCES;
+	return bpf_scx_btf_struct_access_common(reg, off, size);
+}
+
+/* cid-form rejects direct slice and dsq_vtime writes in favor of the kfuncs */
+static int bpf_scx_cid_btf_struct_access(struct bpf_verifier_log *log,
+					 const struct bpf_reg_state *reg, int off,
+					 int size)
+{
+	return bpf_scx_btf_struct_access_common(reg, off, size);
 }
 
 static const struct bpf_verifier_ops bpf_scx_verifier_ops = {
@@ -7824,6 +7501,12 @@ static const struct bpf_verifier_ops bpf_scx_verifier_ops = {
 	.btf_struct_access = bpf_scx_btf_struct_access,
 };
 
+static const struct bpf_verifier_ops bpf_scx_cid_verifier_ops = {
+	.get_func_proto = bpf_base_func_proto,
+	.is_valid_access = bpf_scx_is_valid_access,
+	.btf_struct_access = bpf_scx_cid_btf_struct_access,
+};
+
 static int bpf_scx_init_member(const struct btf_type *t,
 			       const struct btf_member *member,
 			       void *kdata, const void *udata)
@@ -7865,6 +7548,9 @@ static int bpf_scx_init_member(const struct btf_type *t,
 	case offsetof(struct sched_ext_ops, hotplug_seq):
 		ops->hotplug_seq = *(u64 *)(udata + moff);
 		return 1;
+	case offsetof(struct sched_ext_ops, cid_shard_size):
+		ops->cid_shard_size = *(u32 *)(udata + moff);
+		return 1;
 #ifdef CONFIG_EXT_SUB_SCHED
 	case offsetof(struct sched_ext_ops, sub_cgroup_id):
 		ops->sub_cgroup_id = *(u64 *)(udata + moff);
@@ -7875,20 +7561,6 @@ static int bpf_scx_init_member(const struct btf_type *t,
 	return 0;
 }
 
-#ifdef CONFIG_EXT_SUB_SCHED
-static void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog)
-{
-	struct scx_sched *sch;
-
-	guard(rcu)();
-	sch = scx_prog_sched(prog->aux);
-	if (unlikely(!sch))
-		return;
-
-	scx_error(sch, "dispatch recursion detected");
-}
-#endif	/* CONFIG_EXT_SUB_SCHED */
-
 static int bpf_scx_check_member(const struct btf_type *t,
 				const struct btf_member *member,
 				const struct bpf_prog *prog)
@@ -7904,6 +7576,7 @@ static int bpf_scx_check_member(const struct btf_type *t,
 #endif
 	case offsetof(struct sched_ext_ops, cpu_online):
 	case offsetof(struct sched_ext_ops, cpu_offline):
+	case offsetof(struct sched_ext_ops, init_cids):
 	case offsetof(struct sched_ext_ops, init):
 	case offsetof(struct sched_ext_ops, exit):
 	case offsetof(struct sched_ext_ops, sub_attach):
@@ -7927,6 +7600,11 @@ static int bpf_scx_check_member(const struct btf_type *t,
 	case offsetof(struct sched_ext_ops, dispatch):
 		prog->aux->priv_stack_requested = true;
 		prog->aux->recursion_detected = scx_pstack_recursion_on_dispatch;
+		break;
+	case offsetof(struct sched_ext_ops, sub_caps_updated):
+		prog->aux->priv_stack_requested = true;
+		prog->aux->recursion_detected = scx_pstack_recursion_on_caps_updated;
+		break;
 	}
 #endif	/* CONFIG_EXT_SUB_SCHED */
 
@@ -8062,6 +7740,7 @@ static s32 sched_ext_ops__sub_attach(struct scx_sub_attach_args *args) { return
 static void sched_ext_ops__sub_detach(struct scx_sub_detach_args *args) {}
 static void sched_ext_ops__cpu_online(s32 cpu) {}
 static void sched_ext_ops__cpu_offline(s32 cpu) {}
+static s32 sched_ext_ops__init_cids(void) { return -EINVAL; }
 static s32 sched_ext_ops__init(void) { return -EINVAL; }
 static void sched_ext_ops__exit(struct scx_exit_info *info) {}
 static void sched_ext_ops__dump(struct scx_dump_ctx *ctx) {}
@@ -8103,6 +7782,7 @@ static struct sched_ext_ops __bpf_ops_sched_ext_ops = {
 	.sub_detach		= sched_ext_ops__sub_detach,
 	.cpu_online		= sched_ext_ops__cpu_online,
 	.cpu_offline		= sched_ext_ops__cpu_offline,
+	.init_cids		= sched_ext_ops__init_cids,
 	.init			= sched_ext_ops__init,
 	.exit			= sched_ext_ops__exit,
 	.dump			= sched_ext_ops__dump,
@@ -8126,11 +7806,14 @@ static struct bpf_struct_ops bpf_sched_ext_ops = {
 
 /*
  * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types
- * identical, only param names differ across structs) are reused; only
- * set_cmask needs a fresh stub since the second argument type differs.
+ * identical, only param names differ across structs) are reused. Some need
+ * fresh stubs, set_cmask due to an argument type difference and the sub-sched
+ * notifiers because no cpu-form stub exists to reuse.
  */
 static void sched_ext_ops_cid__set_cmask(struct task_struct *p,
 					 const struct scx_cmask *cmask) {}
+static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask, u64 caps) {}
+static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {}
 
 static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = {
 	.select_cid		= sched_ext_ops__select_cpu,
@@ -8163,8 +7846,11 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = {
 #endif
 	.sub_attach		= sched_ext_ops__sub_attach,
 	.sub_detach		= sched_ext_ops__sub_detach,
+	.sub_caps_updated	= sched_ext_ops__sub_caps_updated,
+	.sub_ecaps_updated	= sched_ext_ops__sub_ecaps_updated,
 	.cid_online		= sched_ext_ops__cpu_online,
 	.cid_offline		= sched_ext_ops__cpu_offline,
+	.init_cids		= sched_ext_ops__init_cids,
 	.init			= sched_ext_ops__init,
 	.exit			= sched_ext_ops__exit,
 	.dump			= sched_ext_ops__dump,
@@ -8178,7 +7864,7 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = {
  * verified to match by the BUILD_BUG_ON checks in scx_init().
  */
 static struct bpf_struct_ops bpf_sched_ext_ops_cid = {
-	.verifier_ops = &bpf_scx_verifier_ops,
+	.verifier_ops = &bpf_scx_cid_verifier_ops,
 	.reg = bpf_scx_reg_cid,
 	.unreg = bpf_scx_unreg,
 	.check_member = bpf_scx_check_member,
@@ -8251,7 +7937,8 @@ static bool can_skip_idle_kick(struct rq *rq)
 	return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_BALANCE);
 }
 
-static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs)
+static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_rq,
+			 unsigned long *ksyncs)
 {
 	struct rq *rq = cpu_rq(cpu);
 	struct scx_rq *this_scx = &this_rq->scx;
@@ -8266,29 +7953,38 @@ static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs)
 	 * During CPU hotplug, a CPU may depend on kicking itself to make
 	 * forward progress. Allow kicking self regardless of online state. If
 	 * @cpu is running a higher class task, we have no control over @cpu.
-	 * Skip kicking.
+	 * Skip kicking. A sub-sched lacking baseline access on @cid has no
+	 * business forcing a reschedule there - skip. This is the authoritative
+	 * cap check: ecaps is read here under @rq's lock.
 	 */
 	if ((cpu_online(cpu) || cpu == cpu_of(this_rq)) &&
-	    !sched_class_above(cur_class, &ext_sched_class)) {
-		if (cpumask_test_cpu(cpu, this_scx->cpus_to_preempt)) {
-			if (cur_class == &ext_sched_class)
-				rq->curr->scx.slice = 0;
-			cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt);
+	    !sched_class_above(cur_class, &ext_sched_class) &&
+	    !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) {
+		if (cpumask_test_cpu(cpu, pcpu->cpus_to_preempt)) {
+			if (cur_class == &ext_sched_class) {
+				if (likely(!scx_missing_caps(pcpu->sch, cpu,
+							     scx_caps_for_preempt(pcpu->sch, rq))))
+					set_task_slice(rq->curr, 0);
+				else
+					__scx_add_event(pcpu->sch,
+							SCX_EV_SUB_PREEMPT_DENIED, 1);
+			}
+			cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt);
 		}
 
-		if (cpumask_test_cpu(cpu, this_scx->cpus_to_wait)) {
+		if (cpumask_test_cpu(cpu, pcpu->cpus_to_wait)) {
 			if (cur_class == &ext_sched_class) {
 				cpumask_set_cpu(cpu, this_scx->cpus_to_sync);
 				ksyncs[cpu] = rq->scx.kick_sync;
 				should_wait = true;
 			}
-			cpumask_clear_cpu(cpu, this_scx->cpus_to_wait);
+			cpumask_clear_cpu(cpu, pcpu->cpus_to_wait);
 		}
 
 		resched_curr(rq);
 	} else {
-		cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt);
-		cpumask_clear_cpu(cpu, this_scx->cpus_to_wait);
+		cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt);
+		cpumask_clear_cpu(cpu, pcpu->cpus_to_wait);
 	}
 
 	raw_spin_rq_unlock_irqrestore(rq, flags);
@@ -8296,15 +7992,18 @@ static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs)
 	return should_wait;
 }
 
-static void kick_one_cpu_if_idle(s32 cpu, struct rq *this_rq)
+static void kick_one_cpu_if_idle(s32 cpu, struct scx_sched_pcpu *pcpu,
+				 struct rq *this_rq)
 {
 	struct rq *rq = cpu_rq(cpu);
 	unsigned long flags;
 
 	raw_spin_rq_lock_irqsave(rq, flags);
 
+	/* idle kicks need baseline access too, see kick_one_cpu() */
 	if (!can_skip_idle_kick(rq) &&
-	    (cpu_online(cpu) || cpu == cpu_of(this_rq)))
+	    (cpu_online(cpu) || cpu == cpu_of(this_rq)) &&
+	    !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE))
 		resched_curr(rq);
 
 	raw_spin_rq_unlock_irqrestore(rq, flags);
@@ -8315,6 +8014,7 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work)
 	struct rq *this_rq = this_rq();
 	struct scx_rq *this_scx = &this_rq->scx;
 	struct scx_kick_syncs __rcu *ksyncs_pcpu = __this_cpu_read(scx_kick_syncs);
+	struct scx_sched_pcpu *pcpu, *tmp;
 	bool should_wait = false;
 	unsigned long *ksyncs;
 	s32 cpu;
@@ -8325,15 +8025,24 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work)
 
 	ksyncs = rcu_dereference_bh(ksyncs_pcpu)->syncs;
 
-	for_each_cpu(cpu, this_scx->cpus_to_kick) {
-		should_wait |= kick_one_cpu(cpu, this_rq, ksyncs);
-		cpumask_clear_cpu(cpu, this_scx->cpus_to_kick);
-		cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle);
-	}
+	/*
+	 * Walk scheds with pending kicks on this cpu. scx_kick_cpu() adds to
+	 * the list under local_irq_save() and only this irq_work consumes it.
+	 * A plain list without locking is sufficient.
+	 */
+	list_for_each_entry_safe(pcpu, tmp, &this_scx->sched_pcpus_to_kick, to_kick_node) {
+		list_del_init(&pcpu->to_kick_node);
+
+		for_each_cpu(cpu, pcpu->cpus_to_kick) {
+			should_wait |= kick_one_cpu(cpu, pcpu, this_rq, ksyncs);
+			cpumask_clear_cpu(cpu, pcpu->cpus_to_kick);
+			cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle);
+		}
 
-	for_each_cpu(cpu, this_scx->cpus_to_kick_if_idle) {
-		kick_one_cpu_if_idle(cpu, this_rq);
-		cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle);
+		for_each_cpu(cpu, pcpu->cpus_to_kick_if_idle) {
+			kick_one_cpu_if_idle(cpu, pcpu, this_rq);
+			cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle);
+		}
 	}
 
 	/*
@@ -8454,15 +8163,15 @@ void __init init_sched_ext_class(void)
 
 		/* local_dsq's sch will be set during scx_root_enable() */
 		BUG_ON(init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL));
+#ifdef CONFIG_EXT_SUB_SCHED
+		BUG_ON(init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL));
+#endif
 
 		INIT_LIST_HEAD(&rq->scx.runnable_list);
 		INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals);
 
-		BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick, GFP_KERNEL, n));
-		BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL, n));
-		BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_preempt, GFP_KERNEL, n));
-		BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_wait, GFP_KERNEL, n));
 		BUG_ON(!zalloc_cpumask_var_node(&rq->scx.cpus_to_sync, GFP_KERNEL, n));
+		INIT_LIST_HEAD(&rq->scx.sched_pcpus_to_kick);
 		raw_spin_lock_init(&rq->scx.deferred_reenq_lock);
 		INIT_LIST_HEAD(&rq->scx.deferred_reenq_locals);
 		INIT_LIST_HEAD(&rq->scx.deferred_reenq_users);
@@ -8572,7 +8281,7 @@ __bpf_kfunc_start_defs();
  * When called from ops.select_cpu() or ops.enqueue(), it's for direct dispatch
  * and @p must match the task being enqueued.
  *
- * When called from ops.select_cpu(), @enq_flags and @dsp_id are stored, and @p
+ * When called from ops.select_cpu(), @enq_flags and @dsq_id are stored, and @p
  * will be directly inserted into the corresponding dispatch queue after
  * ops.select_cpu() returns. If @p is inserted into SCX_DSQ_LOCAL, it will be
  * inserted into the local DSQ of the CPU returned by ops.select_cpu().
@@ -8612,9 +8321,9 @@ __bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id,
 		return false;
 
 	if (slice)
-		p->scx.slice = slice;
+		set_task_slice(p, slice);
 	else
-		p->scx.slice = p->scx.slice ?: 1;
+		set_task_slice_keep_oob(p, p->scx.slice ?: 1);
 
 	scx_dsq_insert_commit(sch, p, dsq_id, enq_flags);
 
@@ -8638,9 +8347,9 @@ static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p,
 		return false;
 
 	if (slice)
-		p->scx.slice = slice;
+		set_task_slice(p, slice);
 	else
-		p->scx.slice = p->scx.slice ?: 1;
+		set_task_slice_keep_oob(p, p->scx.slice ?: 1);
 
 	p->scx.dsq_vtime = vtime;
 
@@ -8773,7 +8482,7 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
 
 	/*
 	 * If the BPF scheduler keeps calling this function repeatedly, it can
-	 * cause similar live-lock conditions as consume_dispatch_q().
+	 * cause similar live-lock conditions as scx_consume_dispatch_q().
 	 */
 	if (unlikely(READ_ONCE(sch->aborting)))
 		return false;
@@ -8796,10 +8505,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
 	in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE;
 
 	if (in_balance) {
-		if (this_rq != src_rq) {
-			raw_spin_rq_unlock(this_rq);
-			raw_spin_rq_lock(src_rq);
-		}
+		if (this_rq != src_rq)
+			switch_rq_lock(this_rq, src_rq);
 	} else {
 		raw_spin_rq_lock(src_rq);
 	}
@@ -8824,17 +8531,15 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
 	if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME)
 		p->scx.dsq_vtime = kit->vtime;
 	if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE)
-		p->scx.slice = kit->slice;
+		set_task_slice(p, kit->slice);
 
 	/* execute move */
 	locked_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq);
 	dispatched = true;
 out:
 	if (in_balance) {
-		if (this_rq != locked_rq) {
-			raw_spin_rq_unlock(locked_rq);
-			raw_spin_rq_lock(this_rq);
-		}
+		if (this_rq != locked_rq)
+			switch_rq_lock(locked_rq, this_rq);
 	} else {
 		raw_spin_rq_unlock_irqrestore(locked_rq, flags);
 	}
@@ -8934,7 +8639,7 @@ __bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags,
 
 	dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
 
-	flush_dispatch_buf(sch, dspc->rq);
+	scx_flush_dispatch_buf(sch, dspc->rq);
 
 	dsq = find_user_dsq(sch, dsq_id);
 	if (unlikely(!dsq)) {
@@ -8942,7 +8647,7 @@ __bpf_kfunc bool scx_bpf_dsq_move_to_local___v2(u64 dsq_id, u64 enq_flags,
 		return false;
 	}
 
-	if (consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) {
+	if (scx_consume_dispatch_q(sch, dspc->rq, dsq, enq_flags)) {
 		/*
 		 * A successfully consumed task can be dequeued before it starts
 		 * running while the CPU is trying to migrate other dispatched
@@ -9059,45 +8764,6 @@ __bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter,
 			    p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ);
 }
 
-#ifdef CONFIG_EXT_SUB_SCHED
-/**
- * scx_bpf_sub_dispatch - Trigger dispatching on a child scheduler
- * @cgroup_id: cgroup ID of the child scheduler to dispatch
- * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
- *
- * Allows a parent scheduler to trigger dispatching on one of its direct
- * child schedulers. The child scheduler runs its dispatch operation to
- * move tasks from dispatch queues to the local runqueue.
- *
- * Returns: true on success, false if cgroup_id is invalid, not a direct
- * child, or caller lacks dispatch permission.
- */
-__bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux)
-{
-	struct rq *this_rq = this_rq();
-	struct scx_sched *parent, *child;
-
-	guard(rcu)();
-	parent = scx_prog_sched(aux);
-	if (unlikely(!parent))
-		return false;
-
-	child = scx_find_sub_sched(cgroup_id);
-
-	if (unlikely(!child))
-		return false;
-
-	if (unlikely(scx_parent(child) != parent)) {
-		scx_error(parent, "trying to dispatch a distant sub-sched on cgroup %llu",
-			  cgroup_id);
-		return false;
-	}
-
-	return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev,
-				  true);
-}
-#endif	/* CONFIG_EXT_SUB_SCHED */
-
 __bpf_kfunc_end_defs();
 
 BTF_KFUNCS_START(scx_kfunc_ids_dispatch)
@@ -9245,20 +8911,48 @@ __bpf_kfunc_start_defs();
  * @slice: time slice to set in nsecs
  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
- * Set @p's time slice to @slice. Returns %true on success, %false if the
- * calling scheduler doesn't have authority over @p.
+ * Set @p's time slice. @p must be on the calling scheduler. The value is
+ * applied whether or not the caller holds @p's rq lock - see the slice write
+ * rules above for the ownership model.
+ *
+ * Raising the slice is honored only while the scheduler holds %SCX_CAP_BASE on
+ * @p's cpu, otherwise it is counted in %SCX_EV_SLICE_DENIED. Shortening is
+ * always allowed. On the stashed path the slice is packed into an atomic64_t
+ * with the scheduler id and a flag bit, so a slice too large to fit is clamped
+ * and counted in %SCX_EV_SLICE_CLAMPED. %SCX_SLICE_INF is preserved.
+ *
+ * Return %true on success, %false if @p is not on the calling scheduler.
  */
 __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice,
 					const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
+	struct rq *locked_rq;
 
 	guard(rcu)();
 	sch = scx_prog_sched(aux);
 	if (unlikely(!sch || !scx_task_on_sched(sch, p)))
 		return false;
 
-	p->scx.slice = slice;
+	/*
+	 * Directly write only when we hold the lock of the rq @p is queued or
+	 * running on. See the slice write rules above.
+	 */
+	locked_rq = scx_locked_rq();
+	if (!locked_rq ||
+	    (READ_ONCE(p->scx.runnable_cpu) != cpu_of(locked_rq) &&
+	     !task_current(locked_rq, p))) {
+		set_task_slice_oob(sch, p, slice);
+		return true;
+	}
+
+	/* under the rq lock: apply now, extensions gated on baseline access */
+	if (slice > p->scx.slice &&
+	    unlikely(scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE)))
+		__scx_add_event(sch, SCX_EV_SLICE_DENIED, 1);
+	else
+		set_task_slice(p, slice);
+
 	return true;
 }
 
@@ -9285,14 +8979,29 @@ __bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime,
 	return true;
 }
 
-static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags)
+void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags)
 {
+	struct scx_sched_pcpu *pcpu;
 	struct rq *this_rq;
 	unsigned long irq_flags;
 
+	/*
+	 * The per-cpu kick list is guarded only by local_irq_save(), which does
+	 * not mask NMIs, so kicking from NMI could corrupt it and is unsupported.
+	 */
+	if (unlikely(in_nmi())) {
+		if (!sch->warned_nmi_kick) {
+			sch->warned_nmi_kick = true;
+			pr_warn("sched_ext: %s: scx_bpf_kick_cpu() from NMI ignored\n",
+				sch->ops.name);
+		}
+		return;
+	}
+
 	local_irq_save(irq_flags);
 
 	this_rq = this_rq();
+	pcpu = this_cpu_ptr(sch->pcpu);
 
 	/*
 	 * While bypassing for PM ops, IRQ handling may not be online which can
@@ -9306,6 +9015,9 @@ static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags)
 	 * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting
 	 * rq locks. We can probably be smarter and avoid bouncing if called
 	 * from ops which don't hold a rq lock.
+	 *
+	 * The kick masks are owned by @sch->pcpu, so that a preempt kick can be
+	 * attributed to @sch.
 	 */
 	if (flags & SCX_KICK_IDLE) {
 		struct rq *target_rq = cpu_rq(cpu);
@@ -9320,16 +9032,18 @@ static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags)
 			}
 			raw_spin_rq_unlock(target_rq);
 		}
-		cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick_if_idle);
+		cpumask_set_cpu(cpu, pcpu->cpus_to_kick_if_idle);
 	} else {
-		cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick);
+		cpumask_set_cpu(cpu, pcpu->cpus_to_kick);
 
 		if (flags & SCX_KICK_PREEMPT)
-			cpumask_set_cpu(cpu, this_rq->scx.cpus_to_preempt);
+			cpumask_set_cpu(cpu, pcpu->cpus_to_preempt);
 		if (flags & SCX_KICK_WAIT)
-			cpumask_set_cpu(cpu, this_rq->scx.cpus_to_wait);
+			cpumask_set_cpu(cpu, pcpu->cpus_to_wait);
 	}
 
+	if (list_empty(&pcpu->to_kick_node))
+		list_add_tail(&pcpu->to_kick_node, &this_rq->scx.sched_pcpus_to_kick);
 	irq_work_queue(&this_rq->scx.kick_cpus_irq_work);
 out:
 	local_irq_restore(irq_flags);
@@ -9362,10 +9076,13 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux
  * @flags: %SCX_KICK_* flags
  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
- * cid-addressed equivalent of scx_bpf_kick_cpu(). Return 0 on success,
- * -errno otherwise.
+ * cid-addressed equivalent of scx_bpf_kick_cpu(). An invalid @cid aborts the
+ * scheduler via scx_cid_to_cpu(). Caps are enforced on the delivery path: a
+ * kick is dropped if the caller lacks baseline access on @cid, and a
+ * %SCX_KICK_PREEMPT degrades to a plain reschedule if the caller lacks
+ * %SCX_CAP_PREEMPT for a task outside its subtree.
  */
-__bpf_kfunc s32 scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux)
+__bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux)
 {
 	struct scx_sched *sch;
 	s32 cpu;
@@ -9373,12 +9090,11 @@ __bpf_kfunc s32 scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *
 	guard(rcu)();
 	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
-		return -ENODEV;
+		return;
 	cpu = scx_cid_to_cpu(sch, cid);
 	if (cpu < 0)
-		return cpu;
+		return;
 	scx_kick_cpu(sch, cpu, flags);
-	return 0;
 }
 
 /**
@@ -9666,8 +9382,8 @@ static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf,
 }
 
 __printf(3, 0)
-static s32 bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf,
-		       char *fmt, unsigned long long *data, u32 data__sz)
+s32 scx_bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf,
+		    char *fmt, unsigned long long *data, u32 data__sz)
 {
 	return __bstr_format(sch, buf->data, buf->line, sizeof(buf->line),
 			     fmt, data, data__sz);
@@ -9697,7 +9413,7 @@ __bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt,
 	raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags);
 	sch = scx_prog_sched(aux);
 	if (likely(sch) &&
-	    bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0)
+	    scx_bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0)
 		scx_exit(sch, SCX_EXIT_UNREG_BPF, exit_code, "%s", scx_exit_bstr_buf.line);
 	raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags);
 }
@@ -9722,7 +9438,7 @@ __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data,
 	raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags);
 	sch = scx_prog_sched(aux);
 	if (likely(sch) &&
-	    bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0)
+	    scx_bstr_format(sch, &scx_exit_bstr_buf, fmt, data, data__sz) >= 0)
 		scx_exit(sch, SCX_EXIT_ERROR_BPF, 0, "%s", scx_exit_bstr_buf.line);
 	raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags);
 }
@@ -10094,34 +9810,6 @@ __bpf_kfunc s32 scx_bpf_task_cid(const struct task_struct *p)
 	return tbl[task_cpu(p)];
 }
 
-/**
- * scx_bpf_cpu_rq - Fetch the rq of a CPU
- * @cpu: CPU of the rq
- * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
- */
-__bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu, const struct bpf_prog_aux *aux)
-{
-	struct scx_sched *sch;
-
-	guard(rcu)();
-
-	sch = scx_prog_sched(aux);
-	if (unlikely(!sch))
-		return NULL;
-
-	if (!scx_cpu_valid(sch, cpu, NULL))
-		return NULL;
-
-	if (!sch->warned_deprecated_rq) {
-		printk_deferred(KERN_WARNING "sched_ext: %s() is deprecated; "
-				"use scx_bpf_locked_rq() when holding rq lock "
-				"or scx_bpf_cpu_curr() to read remote curr safely.\n", __func__);
-		sch->warned_deprecated_rq = true;
-	}
-
-	return cpu_rq(cpu);
-}
-
 /**
  * scx_bpf_locked_rq - Return the rq currently locked by SCX
  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
@@ -10288,26 +9976,15 @@ __bpf_kfunc u64 scx_bpf_now(void)
 
 static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events)
 {
-	struct scx_event_stats *e_cpu;
 	int cpu;
 
 	/* Aggregate per-CPU event counters into @events. */
 	memset(events, 0, sizeof(*events));
 	for_each_possible_cpu(cpu) {
-		e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats;
-		scx_agg_event(events, e_cpu, SCX_EV_SELECT_CPU_FALLBACK);
-		scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE);
-		scx_agg_event(events, e_cpu, SCX_EV_DISPATCH_KEEP_LAST);
-		scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_EXITING);
-		scx_agg_event(events, e_cpu, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED);
-		scx_agg_event(events, e_cpu, SCX_EV_REENQ_IMMED);
-		scx_agg_event(events, e_cpu, SCX_EV_REENQ_LOCAL_REPEAT);
-		scx_agg_event(events, e_cpu, SCX_EV_REFILL_SLICE_DFL);
-		scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DURATION);
-		scx_agg_event(events, e_cpu, SCX_EV_BYPASS_DISPATCH);
-		scx_agg_event(events, e_cpu, SCX_EV_BYPASS_ACTIVATE);
-		scx_agg_event(events, e_cpu, SCX_EV_INSERT_NOT_OWNED);
-		scx_agg_event(events, e_cpu, SCX_EV_SUB_BYPASS_DISPATCH);
+		struct scx_event_stats *e_cpu = &per_cpu_ptr(sch->pcpu, cpu)->event_stats;
+#define SCX_EVENT(name)	(events->name += READ_ONCE(e_cpu->name))
+		SCX_EVENTS_LIST(SCX_EVENT);
+#undef SCX_EVENT
 	}
 }
 
@@ -10377,6 +10054,35 @@ __bpf_kfunc struct cgroup *scx_bpf_task_cgroup(struct task_struct *p,
 }
 #endif	/* CONFIG_CGROUP_SCHED */
 
+#ifndef CONFIG_EXT_SUB_SCHED
+__bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps,
+				  const struct scx_cmask *cmask__ign,
+				  struct scx_cmask *denied_out__ign,
+				  const struct bpf_prog_aux *aux)
+{
+	return -EOPNOTSUPP;
+}
+
+__bpf_kfunc void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps,
+				    const struct scx_cmask *cmask__ign,
+				    const struct bpf_prog_aux *aux)
+{
+}
+
+__bpf_kfunc s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out__ign,
+				 const struct bpf_prog_aux *aux)
+{
+	return -EOPNOTSUPP;
+}
+
+__bpf_kfunc s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt,
+				      unsigned long long *data, u32 data__sz,
+				      const struct bpf_prog_aux *aux)
+{
+	return -EOPNOTSUPP;
+}
+#endif	/* !CONFIG_EXT_SUB_SCHED */
+
 __bpf_kfunc_end_defs();
 
 BTF_KFUNCS_START(scx_kfunc_ids_any)
@@ -10412,7 +10118,6 @@ BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE)
 BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU)
 BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU)
 BTF_ID_FLAGS(func, scx_bpf_task_cid, KF_RCU)
-BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS | KF_RET_NULL)
 BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED)
 BTF_ID_FLAGS(func, scx_bpf_cid_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED)
@@ -10422,6 +10127,10 @@ BTF_ID_FLAGS(func, scx_bpf_events)
 #ifdef CONFIG_CGROUP_SCHED
 BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE)
 #endif
+BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_sub_caps, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_sub_kill_bstr, KF_IMPLICIT_ARGS)
 BTF_KFUNCS_END(scx_kfunc_ids_any)
 
 static const struct btf_kfunc_id_set scx_kfunc_set_any = {
@@ -10447,7 +10156,6 @@ static const struct btf_kfunc_id_set scx_kfunc_set_any = {
 BTF_KFUNCS_START(scx_kfunc_ids_cpu_only)
 BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU)
-BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED)
 BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS)
@@ -10480,7 +10188,7 @@ BTF_KFUNCS_END(scx_kfunc_ids_cpu_only)
  */
 enum scx_kf_allow_flags {
 	SCX_KF_ALLOW_UNLOCKED		= 1 << 0,
-	SCX_KF_ALLOW_INIT		= 1 << 1,
+	SCX_KF_ALLOW_INIT_CIDS		= 1 << 1,
 	SCX_KF_ALLOW_CPU_RELEASE	= 1 << 2,
 	SCX_KF_ALLOW_DISPATCH		= 1 << 3,
 	SCX_KF_ALLOW_ENQUEUE		= 1 << 4,
@@ -10510,9 +10218,11 @@ static const u32 scx_kf_allow_flags[] = {
 #endif	/* CONFIG_EXT_GROUP_SCHED */
 	[SCX_OP_IDX(sub_attach)]	= SCX_KF_ALLOW_UNLOCKED,
 	[SCX_OP_IDX(sub_detach)]	= SCX_KF_ALLOW_UNLOCKED,
+	[SCX_OP_IDX(sub_ecaps_updated)]	= SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH,
 	[SCX_OP_IDX(cpu_online)]	= SCX_KF_ALLOW_UNLOCKED,
 	[SCX_OP_IDX(cpu_offline)]	= SCX_KF_ALLOW_UNLOCKED,
-	[SCX_OP_IDX(init)]		= SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT,
+	[SCX_OP_IDX(init_cids)]		= SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT_CIDS,
+	[SCX_OP_IDX(init)]		= SCX_KF_ALLOW_UNLOCKED,
 	[SCX_OP_IDX(exit)]		= SCX_KF_ALLOW_UNLOCKED,
 };
 
@@ -10527,7 +10237,7 @@ static const u32 scx_kf_allow_flags[] = {
 int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id)
 {
 	bool in_unlocked = btf_id_set8_contains(&scx_kfunc_ids_unlocked, kfunc_id);
-	bool in_init = btf_id_set8_contains(&scx_kfunc_ids_init, kfunc_id);
+	bool in_init_cids = btf_id_set8_contains(&scx_kfunc_ids_init_cids, kfunc_id);
 	bool in_select_cpu = btf_id_set8_contains(&scx_kfunc_ids_select_cpu, kfunc_id);
 	bool in_enqueue = btf_id_set8_contains(&scx_kfunc_ids_enqueue_dispatch, kfunc_id);
 	bool in_dispatch = btf_id_set8_contains(&scx_kfunc_ids_dispatch, kfunc_id);
@@ -10538,7 +10248,7 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id)
 	u32 moff, flags;
 
 	/* Not an SCX kfunc - allow. */
-	if (!(in_unlocked || in_init || in_select_cpu || in_enqueue || in_dispatch ||
+	if (!(in_unlocked || in_init_cids || in_select_cpu || in_enqueue || in_dispatch ||
 	      in_cpu_release || in_idle || in_any))
 		return 0;
 
@@ -10590,7 +10300,7 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id)
 
 	if ((flags & SCX_KF_ALLOW_UNLOCKED) && in_unlocked)
 		return 0;
-	if ((flags & SCX_KF_ALLOW_INIT) && in_init)
+	if ((flags & SCX_KF_ALLOW_INIT_CIDS) && in_init_cids)
 		return 0;
 	if ((flags & SCX_KF_ALLOW_CPU_RELEASE) && in_cpu_release)
 		return 0;
@@ -10624,6 +10334,7 @@ static int __init scx_init(void)
 	CID_OFFSET_MATCH(timeout_ms, timeout_ms);
 	CID_OFFSET_MATCH(exit_dump_len, exit_dump_len);
 	CID_OFFSET_MATCH(hotplug_seq, hotplug_seq);
+	CID_OFFSET_MATCH(cid_shard_size, cid_shard_size);
 	CID_OFFSET_MATCH(sub_cgroup_id, sub_cgroup_id);
 	/* shared callbacks: the union view requires byte-for-byte offset match */
 	CID_OFFSET_MATCH(enqueue, enqueue);
@@ -10646,6 +10357,9 @@ static int __init scx_init(void)
 	CID_OFFSET_MATCH(dump_task, dump_task);
 	CID_OFFSET_MATCH(sub_attach, sub_attach);
 	CID_OFFSET_MATCH(sub_detach, sub_detach);
+	CID_OFFSET_MATCH(sub_caps_updated, sub_caps_updated);
+	CID_OFFSET_MATCH(sub_ecaps_updated, sub_ecaps_updated);
+	CID_OFFSET_MATCH(init_cids, init_cids);
 	CID_OFFSET_MATCH(init, init);
 	CID_OFFSET_MATCH(exit, exit);
 #ifdef CONFIG_EXT_GROUP_SCHED
@@ -10667,7 +10381,7 @@ static int __init scx_init(void)
 	/* @priv tail must align since both share the same data block */
 	CID_OFFSET_MATCH(priv, priv);
 	/*
-	 * cid-form must end exactly at @priv - validate_ops() skips
+	 * cid-form must end exactly at @priv - scx_validate_ops() skips
 	 * cpu_acquire/cpu_release for cid-form because reading those fields
 	 * past the BPF allocation would be UB.
 	 */
diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c
index 8e8c6201b7dfd..d08166de03d8d 100644
--- a/kernel/sched/ext/idle.c
+++ b/kernel/sched/ext/idle.c
@@ -12,6 +12,7 @@
 #include "internal.h"
 #include "cid.h"
 #include "idle.h"
+#include "sub.h"
 
 /* Enable/disable built-in idle CPU selection policy */
 static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled);
@@ -730,6 +731,56 @@ static void update_builtin_idle(int cpu, bool idle)
 	}
 }
 
+/*
+ * Notify schedulers of an idle transition on @cpu's cid, delivering to every
+ * sched that holds %SCX_CAP_BASE on the cid (the root holds every cap). A real
+ * transition (@do_notify) reaches all holders. A forced one (@root_renotify for
+ * the root, a sub-sched's idle_renotify marker for a sub) reaches only the owed
+ * scheds.
+ */
+static void scx_idle_notify(struct rq *rq, bool idle, bool do_notify, bool root_renotify)
+{
+	s32 cpu = cpu_of(rq);
+	s32 cid = scx_cpu_arg(cpu);
+	struct scx_sched *pos;
+
+	lockdep_assert_rq_held(rq);
+
+	/* with no sub-sched, only the root can be owed a notification */
+	if (!scx_has_subs()) {
+		struct scx_sched *sch = scx_root;
+
+		if ((do_notify || root_renotify) &&
+		    SCX_HAS_OP(sch, update_idle) && !scx_bypassing(sch, cpu))
+			SCX_CALL_OP(sch, update_idle, rq, cid, idle);
+		return;
+	}
+
+	pos = scx_next_descendant_pre(NULL, scx_root);
+	while (pos) {
+		bool forced = false;
+
+		if (unlikely(scx_missing_caps(pos, cpu, SCX_CAP_BASE))) {
+			pos = scx_skip_subtree_pre(pos, scx_root);
+			continue;
+		}
+
+		if (pos == scx_root) {
+			forced = root_renotify;
+		}
+#ifdef CONFIG_EXT_SUB_SCHED
+		else if (per_cpu_ptr(pos->pcpu, cpu)->idle_renotify) {
+			per_cpu_ptr(pos->pcpu, cpu)->idle_renotify = false;
+			forced = true;
+		}
+#endif
+		if ((do_notify || forced) && SCX_HAS_OP(pos, update_idle) &&
+		    !scx_bypassing(pos, cpu))
+			SCX_CALL_OP(pos, update_idle, rq, cid, idle);
+		pos = scx_next_descendant_pre(pos, scx_root);
+	}
+}
+
 /*
  * Update the idle state of a CPU to @idle.
  *
@@ -748,44 +799,39 @@ static void update_builtin_idle(int cpu, bool idle)
  */
 void __scx_update_idle(struct rq *rq, bool idle, bool do_notify)
 {
-	struct scx_sched *sch = scx_root;
 	int cpu = cpu_of(rq);
 
 	lockdep_assert_rq_held(rq);
 
 	/*
-	 * Update the idle masks:
-	 * - for real idle transitions (do_notify == true)
-	 * - for idle-to-idle transitions (indicated by the previous task
-	 *   being the idle thread, managed by pick_task_idle())
-	 *
-	 * Skip updating idle masks if the previous task is not the idle
-	 * thread, since set_next_task_idle() has already handled it when
-	 * transitioning from a task to the idle thread (calling this
-	 * function with do_notify == true).
-	 *
-	 * In this way we can avoid updating the idle masks twice,
-	 * unnecessarily.
+	 * pick_task_idle() calls here only on an idle-to-idle re-pick and the
+	 * transitions call with @do_notify, so every reaching call updates the
+	 * masks.
 	 */
 	if (static_branch_likely(&scx_builtin_idle_enabled))
-		if (do_notify || is_idle_task(rq->curr))
-			update_builtin_idle(cpu, idle);
+		update_builtin_idle(cpu, idle);
 
 	/*
-	 * Trigger ops.update_idle() only when transitioning from a task to
-	 * the idle thread and vice versa.
+	 * ops.update_idle() fires on real idle transitions, indicated by
+	 * @do_notify and managed by put_prev_task_idle()/set_next_task_idle().
+	 * An idle pick also fires it to flush a forced notify owed to a sched
+	 * that missed transitions while bypassed or on a cid it just gained.
+	 * unbypass_renotify_idle() and scx_process_sync_ecaps() arm the per-rq
+	 * gates, and scx_idle_notify() targets the owed scheds.
 	 *
-	 * Idle transitions are indicated by do_notify being set to true,
-	 * managed by put_prev_task_idle()/set_next_task_idle().
-	 *
-	 * This must come after builtin idle update so that BPF schedulers can
-	 * create interlocking between ops.update_idle() and ops.enqueue() -
+	 * This must come after the builtin idle update so that BPF schedulers
+	 * can create interlocking between ops.update_idle() and ops.enqueue() -
 	 * either enqueue() sees the idle bit or update_idle() sees the task
 	 * that enqueue() queued.
 	 */
-	if (SCX_HAS_OP(sch, update_idle) && do_notify &&
-	    !scx_bypassing(sch, cpu_of(rq)))
-		SCX_CALL_OP(sch, update_idle, rq, scx_cpu_arg(cpu_of(rq)), idle);
+	if (do_notify ||
+	    (idle && (rq->scx.flags &
+		      (SCX_RQ_SUB_IDLE_RENOTIFY | SCX_RQ_ROOT_IDLE_RENOTIFY)))) {
+		bool root_renotify = rq->scx.flags & SCX_RQ_ROOT_IDLE_RENOTIFY;
+
+		rq->scx.flags &= ~(SCX_RQ_SUB_IDLE_RENOTIFY | SCX_RQ_ROOT_IDLE_RENOTIFY);
+		scx_idle_notify(rq, idle, do_notify, root_renotify);
+	}
 }
 
 static void reset_idle_masks(struct sched_ext_ops *ops)
diff --git a/kernel/sched/ext/inlines.h b/kernel/sched/ext/inlines.h
new file mode 100644
index 0000000000000..d2d074cee1cb3
--- /dev/null
+++ b/kernel/sched/ext/inlines.h
@@ -0,0 +1,128 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
+ *
+ * Inline definitions layered on top of internal.h and cid.h.
+ *
+ * Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2026 Tejun Heo <tj@kernel.org>
+ */
+#ifndef _KERNEL_SCHED_EXT_INLINES_H
+#define _KERNEL_SCHED_EXT_INLINES_H
+
+#include "internal.h"
+#include "cid.h"
+
+/*
+ * One user of this function is scx_bpf_dispatch() which can be called
+ * recursively as sub-sched dispatches nest. Always inline to reduce stack usage
+ * from the call frame.
+ */
+static __always_inline bool
+scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
+		   struct task_struct *prev, bool nested)
+{
+	struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
+	int nr_loops = SCX_DSP_MAX_LOOPS;
+	s32 cpu = cpu_of(rq);
+	bool prev_on_sch = (prev->sched_class == &ext_sched_class) &&
+		scx_task_on_sched(sch, prev);
+
+	if (scx_consume_global_dsq(sch, rq))
+		return true;
+
+	if (scx_bypass_dsp_enabled(sch)) {
+		/* if @sch is bypassing, only the bypass DSQs are active */
+		if (scx_bypassing(sch, cpu))
+			return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0);
+
+#ifdef CONFIG_EXT_SUB_SCHED
+		/*
+		 * If @sch isn't bypassing but its children are, @sch is
+		 * responsible for making forward progress for both its own
+		 * tasks that aren't bypassing and the bypassing descendants'
+		 * tasks. The following implements a simple built-in behavior -
+		 * let each CPU try to run the bypass DSQ every Nth time.
+		 *
+		 * Later, if necessary, we can add an ops flag to suppress the
+		 * auto-consumption and a kfunc to consume the bypass DSQ and,
+		 * so that the BPF scheduler can fully control scheduling of
+		 * bypassed tasks.
+		 */
+		struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
+
+		if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) &&
+		    scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) {
+			__scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1);
+			return true;
+		}
+#endif	/* CONFIG_EXT_SUB_SCHED */
+	}
+
+	if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq))
+		return false;
+
+	dspc->rq = rq;
+
+	/*
+	 * The dispatch loop. Because scx_flush_dispatch_buf() may drop the rq
+	 * lock, the local DSQ might still end up empty after a successful
+	 * ops.dispatch(). If the local DSQ is empty even after ops.dispatch()
+	 * produced some tasks, retry. The BPF scheduler may depend on this
+	 * looping behavior to simplify its implementation.
+	 */
+	do {
+		dspc->nr_tasks = 0;
+
+#ifdef CONFIG_EXT_SUB_SCHED
+		/* stash @prev so that nested invocations can access it */
+		if (!nested)
+			rq->scx.sub_dispatch_prev = prev;
+#endif
+
+		SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
+			    prev_on_sch ? prev : NULL);
+
+#ifdef CONFIG_EXT_SUB_SCHED
+		if (!nested)
+			rq->scx.sub_dispatch_prev = NULL;
+#endif
+
+		scx_flush_dispatch_buf(sch, rq);
+
+		if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) {
+			rq->scx.flags |= SCX_RQ_BAL_KEEP;
+			return true;
+		}
+		if (rq->scx.local_dsq.nr)
+			return true;
+		if (scx_consume_global_dsq(sch, rq))
+			return true;
+
+		/*
+		 * ops.dispatch() can trap us in this loop by repeatedly
+		 * dispatching ineligible tasks. Break out once in a while to
+		 * allow the watchdog to run. As IRQ can't be enabled in
+		 * balance(), we want to complete this scheduling cycle and then
+		 * start a new one. IOW, we want to call resched_curr() on the
+		 * next, most likely idle, task, not the current one. Use
+		 * __scx_bpf_kick_cpu() for deferred kicking.
+		 */
+		if (unlikely(!--nr_loops)) {
+			scx_kick_cpu(sch, cpu, 0);
+			break;
+		}
+	} while (dspc->nr_tasks);
+
+	/*
+	 * Prevent the CPU from going idle while bypassed descendants have tasks
+	 * queued. Without this fallback, bypassed tasks could stall if the host
+	 * scheduler's ops.dispatch() doesn't yield any tasks.
+	 */
+	if (scx_bypass_dsp_enabled(sch))
+		return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0);
+
+	return false;
+}
+
+#endif /* _KERNEL_SCHED_EXT_INLINES_H */
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 145272cb4d8a7..4f4130f0d1211 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -11,6 +11,34 @@
 #include "../sched.h"
 #include "types.h"
 
+#include <trace/events/sched_ext.h>
+
+/**
+ * scx_add_event - Increase an event counter for 'name' by 'cnt'
+ * @sch: scx_sched to account events for
+ * @name: an event name defined in struct scx_event_stats
+ * @cnt: the number of the event occurred
+ *
+ * This can be used when preemption is not disabled.
+ */
+#define scx_add_event(sch, name, cnt) do {					\
+	this_cpu_add((sch)->pcpu->event_stats.name, (cnt));			\
+	trace_sched_ext_event(#name, (cnt));					\
+} while(0)
+
+/**
+ * __scx_add_event - Increase an event counter for 'name' by 'cnt'
+ * @sch: scx_sched to account events for
+ * @name: an event name defined in struct scx_event_stats
+ * @cnt: the number of the event occurred
+ *
+ * This should be used only when preemption is disabled.
+ */
+#define __scx_add_event(sch, name, cnt) do {					\
+	__this_cpu_add((sch)->pcpu->event_stats.name, (cnt));			\
+	trace_sched_ext_event(#name, cnt);					\
+} while(0)
+
 #define SCX_OP_IDX(op)		(offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void)))
 #define SCX_MOFF_IDX(moff)	((moff) / sizeof(void (*)(void)))
 
@@ -23,6 +51,7 @@ enum scx_exit_kind {
 	SCX_EXIT_UNREG_KERN,	/* kernel-initiated unregistration */
 	SCX_EXIT_SYSRQ,		/* requested by 'S' sysrq */
 	SCX_EXIT_PARENT,	/* parent exiting */
+	SCX_EXIT_PARENT_KILL,	/* killed by parent scheduler */
 
 	SCX_EXIT_ERROR = 1024,	/* runtime error, error msg contains details */
 	SCX_EXIT_ERROR_BPF,	/* ERROR but triggered through scx_bpf_error() */
@@ -372,8 +401,9 @@ struct sched_ext_ops {
 	 * @p: task running currently
 	 *
 	 * This operation is called every 1/HZ seconds on CPUs which are
-	 * executing an SCX task. Setting @p->scx.slice to 0 will trigger an
-	 * immediate dispatch cycle on the CPU.
+	 * executing an SCX task. Setting a slice of 0 for @p with
+	 * scx_bpf_task_set_slice() will trigger an immediate dispatch cycle on
+	 * the CPU.
 	 */
 	void (*tick)(struct task_struct *p);
 
@@ -729,6 +759,39 @@ struct sched_ext_ops {
 	 */
 	void (*sub_detach)(struct scx_sub_detach_args *args);
 
+	/**
+	 * @sub_caps_updated: Caps on this sub-sched's shard changed
+	 * @cmask: cids whose caps changed (cmask->base identifies the shard)
+	 * @caps: SCX_CAP_* that changed
+	 *
+	 * Invoked after grant or revoke modifies caps on a shard. There can be
+	 * only one in-flight invocation per shard. @cmask and @caps coalesce
+	 * all changes since the last delivery. Direction (set vs cleared) isn't
+	 * encoded. Query current state with scx_bpf_sub_caps().
+	 *
+	 * Delivered asynchronously after the change is recorded, and may run
+	 * before it takes effect on any given cpu. Use it to track which caps
+	 * the sub-sched holds and propagate to its own children, not to decide
+	 * if a task can run on a cpu now. sub_ecaps_updated() reports that per
+	 * cpu, once it is in effect.
+	 *
+	 * May call scx_bpf_sub_grant() / scx_bpf_sub_revoke() on children.
+	 */
+	void (*sub_caps_updated)(const struct scx_cmask *cmask, u64 caps);
+
+	/**
+	 * @sub_ecaps_updated: This sub-sched's effective caps on a cid changed
+	 * @cid: the cid whose effective caps changed
+	 * @before: effective caps as of the last delivery
+	 * @after: effective caps now
+	 *
+	 * Invoked when this sub-sched's effective caps on @cid change, once the
+	 * change is in effect on the cpu. Runs in dispatch context with rq lock
+	 * held, and can perform all operations allowed in ops.dispatch()
+	 * including inserting/moving tasks.
+	 */
+	void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after);
+
 	/*
 	 * All online ops must come before ops.cpu_online().
 	 */
@@ -752,8 +815,17 @@ struct sched_ext_ops {
 	void (*cpu_offline)(s32 cpu);
 
 	/*
-	 * All CPU hotplug ops must come before ops.init().
+	 * All CPU hotplug ops must come before ops.init_cids().
+	 */
+
+	/**
+	 * @init_cids: Finalize the cid layout (cid-form only)
+	 *
+	 * Runs after the default cid layout is built, before caps and shards
+	 * are finalized. A cid-form scheduler may call scx_bpf_cid_override()
+	 * here for a custom layout. Ignored for cpu-form schedulers.
 	 */
+	s32 (*init_cids)(void);
 
 	/**
 	 * @init: Initialize the BPF scheduler
@@ -808,6 +880,18 @@ struct sched_ext_ops {
 	 */
 	u64 hotplug_seq;
 
+	/**
+	 * @cid_shard_size: Target number of CIDs per shard
+	 *
+	 * Shards are contiguous CID ranges used as operation and locking
+	 * domains for sub-scheduling. Each LLC is divided into ceil(nr_cpus /
+	 * @cid_shard_size) shards, then cores are distributed across them
+	 * evenly. If one core has more logical CPUs than @cid_shard_size, its
+	 * shard will become larger than @cid_shard_size. Values above
+	 * SCX_CID_SHARD_MAX_CPUS are capped. 0 means use the default (24).
+	 */
+	u32 cid_shard_size;
+
 	/**
 	 * @cgroup_id: When >1, attach the scheduler as a sub-scheduler on the
 	 * specified cgroup.
@@ -928,8 +1012,11 @@ struct sched_ext_ops_cid {
 #endif	/* CONFIG_EXT_GROUP_SCHED */
 	s32 (*sub_attach)(struct scx_sub_attach_args *args);
 	void (*sub_detach)(struct scx_sub_detach_args *args);
+	void (*sub_caps_updated)(const struct scx_cmask *cmask, u64 caps);
+	void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after);
 	void (*cid_online)(s32 cid);
 	void (*cid_offline)(s32 cid);
+	s32 (*init_cids)(void);
 	s32 (*init)(void);
 	void (*exit)(struct scx_exit_info *info);
 
@@ -939,6 +1026,7 @@ struct sched_ext_ops_cid {
 	u32 timeout_ms;
 	u32 exit_dump_len;
 	u64 hotplug_seq;
+	u32 cid_shard_size;
 	u64 sub_cgroup_id;
 	char name[SCX_OPS_NAME_LEN];
 
@@ -954,8 +1042,8 @@ enum scx_opi {
 	SCX_OPI_NORMAL_BEGIN		= 0,
 	SCX_OPI_NORMAL_END		= SCX_OP_IDX(cpu_online),
 	SCX_OPI_CPU_HOTPLUG_BEGIN	= SCX_OP_IDX(cpu_online),
-	SCX_OPI_CPU_HOTPLUG_END		= SCX_OP_IDX(init),
-	SCX_OPI_END			= SCX_OP_IDX(init),
+	SCX_OPI_CPU_HOTPLUG_END		= SCX_OP_IDX(init_cids),
+	SCX_OPI_END			= SCX_OP_IDX(init_cids),
 };
 
 /*
@@ -1017,6 +1105,18 @@ struct scx_event_stats {
 	 */
 	s64		SCX_EV_REFILL_SLICE_DFL;
 
+	/*
+	 * The number of times an out-of-band slice request exceeded the maximum
+	 * representable value and was clamped.
+	 */
+	s64		SCX_EV_SLICE_CLAMPED;
+
+	/*
+	 * The number of times a slice extension was denied because the
+	 * scheduler lacked baseline cpu access on the task's cpu.
+	 */
+	s64		SCX_EV_SLICE_DENIED;
+
 	/*
 	 * The total duration of bypass modes in nanoseconds.
 	 */
@@ -1049,8 +1149,41 @@ struct scx_event_stats {
 	 * from sub_bypass_dsq's.
 	 */
 	s64		SCX_EV_SUB_BYPASS_DISPATCH;
+
+	/*
+	 * The number of times a migration-disabled task lacking the cap for its
+	 * cid was allowed onto the local DSQ. It must run on its pinned CPU, so
+	 * it can't be rejected. The violation is counted here.
+	 */
+	s64		SCX_EV_SUB_FORCED_ADMIT;
+
+	/*
+	 * The number of times a preempting kick was refused because the
+	 * sub-sched lacked SCX_CAP_PREEMPT for a task outside its subtree. The
+	 * kick degrades to a plain reschedule.
+	 */
+	s64		SCX_EV_SUB_PREEMPT_DENIED;
 };
 
+#define SCX_EVENTS_LIST(SCX_EVENT)					\
+	SCX_EVENT(SCX_EV_SELECT_CPU_FALLBACK);				\
+	SCX_EVENT(SCX_EV_DISPATCH_LOCAL_DSQ_OFFLINE);			\
+	SCX_EVENT(SCX_EV_DISPATCH_KEEP_LAST);				\
+	SCX_EVENT(SCX_EV_ENQ_SKIP_EXITING);				\
+	SCX_EVENT(SCX_EV_ENQ_SKIP_MIGRATION_DISABLED);			\
+	SCX_EVENT(SCX_EV_REENQ_IMMED);					\
+	SCX_EVENT(SCX_EV_REENQ_LOCAL_REPEAT);				\
+	SCX_EVENT(SCX_EV_REFILL_SLICE_DFL);				\
+	SCX_EVENT(SCX_EV_SLICE_CLAMPED);				\
+	SCX_EVENT(SCX_EV_SLICE_DENIED);					\
+	SCX_EVENT(SCX_EV_BYPASS_DURATION);				\
+	SCX_EVENT(SCX_EV_BYPASS_DISPATCH);				\
+	SCX_EVENT(SCX_EV_BYPASS_ACTIVATE);				\
+	SCX_EVENT(SCX_EV_INSERT_NOT_OWNED);				\
+	SCX_EVENT(SCX_EV_SUB_BYPASS_DISPATCH);				\
+	SCX_EVENT(SCX_EV_SUB_FORCED_ADMIT);				\
+	SCX_EVENT(SCX_EV_SUB_PREEMPT_DENIED)
+
 struct scx_sched;
 
 enum scx_sched_pcpu_flags {
@@ -1083,6 +1216,41 @@ struct scx_sched_pcpu {
 	struct scx_sched	*sch;
 	u64			flags;	/* protected by rq lock */
 
+	/*
+	 * Kick state owned by this cpu for this sched. scx_kick_cpu() records
+	 * targets here and links @to_kick_node onto the cpu's
+	 * rq->scx.sched_pcpus_to_kick. The cpu's single kick irq_work walks
+	 * that list and kicks each sched's targets on its behalf. Per-sched so
+	 * a kick stays attributed to its scheduler.
+	 */
+	cpumask_var_t		cpus_to_kick;
+	cpumask_var_t		cpus_to_kick_if_idle;
+	cpumask_var_t		cpus_to_preempt;
+	cpumask_var_t		cpus_to_wait;
+	struct list_head	to_kick_node;
+
+#ifdef CONFIG_EXT_SUB_SCHED
+	/*
+	 * pshard->caps[cap_bit] is the set of cids the sched holds that one
+	 * cap on. ecaps is its transpose: the set of SCX_CAP_* bits the sched
+	 * effectively holds on this cpu, with implied caps folded in, so that
+	 * the hot-path check is a single read.
+	 *
+	 * While pshard->caps[] under pshard->lock is the target configuration,
+	 * ecaps is the effective copy owned by the cpu. It is written under the
+	 * rq lock while processing rq->ecaps_to_sync. Can also be read with
+	 * READ_ONCE() outside rq lock.
+	 *
+	 * See queue_sync_ecaps() and scx_process_sync_ecaps().
+	 */
+	u64			ecaps;
+	struct llist_node	ecaps_to_sync_node;
+	/* owed a forced update_idle() re-notify on this cpu */
+	bool			idle_renotify;
+	/* effective caps as of the last sub_ecaps_updated() delivery */
+	u64			reported_ecaps;
+#endif
+
 	/*
 	 * The event counters are in a per-CPU variable to minimize the
 	 * accounting overhead. A system-wide view on the event counter is
@@ -1104,6 +1272,139 @@ struct scx_sched_pnode {
 	struct scx_dispatch_q	global_dsq;
 };
 
+/*
+ * Sub-sched capability delegation.
+ *
+ * Caps are per-cid permissions parents delegate to direct children via
+ * scx_bpf_sub_grant() / scx_bpf_sub_revoke(). A child's cap set is always a
+ * subset of its parent's. A sub-sched checks its caps locally, and cross-sched
+ * communication is needed only when the delegation set itself changes.
+ *
+ * Caps are used to implement sub-sched scheduling on the enqueue path. Picking
+ * a cid for a task at a leaf depends on which cids the leaf is allowed to use.
+ * Resolving that programmatically on every enqueue would mean a cross-sched
+ * round-trip call chain, possibly retrying if the request can't be granted
+ * as-is.
+ *
+ * The dispatch path is different - it runs as top-down recursion via
+ * scx_bpf_sub_dispatch(): a sched's dispatch op invokes a child's dispatch op
+ * on the local rq, and the subtree dispatches in a single pass.
+ *
+ * Locking is per shard. cid space is split into shards, and each sub-sched has
+ * its own pshard->lock for each shard. Operations are broken up on shard
+ * boundaries. Different shards never contend. Shards are expected to be
+ * topology-aligned and likely to serve as the locality unit when cids are
+ * allocated to schedulers, so per-shard lock granularity scales naturally with
+ * the allocation pattern.
+ *
+ * ENQ_IMMED  insert an IMMED task onto the cid's local DSQ
+ *            - kick the cid's cpu (except SCX_KICK_PREEMPT)
+ *
+ * ENQ        insert any task onto the cid's local DSQ (implies ENQ_IMMED)
+ *
+ * PREEMPT    preempt any task running on the cid regardless of the owning
+ *            sched (implies ENQ). Preempting a task in the sched's own subtree
+ *            doesn't require any cap.
+ *            - SCX_ENQ_PREEMPT inserts
+ *            - SCX_KICK_PREEMPT kicks
+ *
+ * Implied caps apply to the holder's own use of a cid, not to delegation.
+ * scx_bpf_sub_grant() delegates literally-held caps, so a cap held only through
+ * implication is usable but cannot be re-delegated to a child. When granting a
+ * cap, it usually makes sense to delegate its implied caps explicitly alongside
+ * it.
+ */
+enum scx_cap_flags {
+	__SCX_CAP_ENQ_IMMED		= 0,
+	__SCX_CAP_ENQ			= 1,
+	__SCX_CAP_PREEMPT		= 2,
+
+	__SCX_NR_CAPS,
+	__SCX_CAP_ALL			= BIT_U64(__SCX_NR_CAPS) - 1,
+
+	SCX_CAP_ENQ_IMMED		= BIT_U64(__SCX_CAP_ENQ_IMMED),
+	SCX_CAP_ENQ			= BIT_U64(__SCX_CAP_ENQ),
+	SCX_CAP_PREEMPT			= BIT_U64(__SCX_CAP_PREEMPT),
+
+	/* alias for minimal cap to make any use of a cpu */
+	SCX_CAP_BASE			= SCX_CAP_ENQ_IMMED,
+
+	/* caps whose loss strands queued tasks, see scx_process_sync_ecaps() */
+	SCX_CAPS_REENQ_ON_LOSS		= SCX_CAP_ENQ_IMMED | SCX_CAP_ENQ,
+};
+
+#ifdef CONFIG_EXT_SUB_SCHED
+/* iterate set bits in a u64 cap mask */
+#define scx_for_each_cap_bit(cap_bit, caps)				\
+	for (u64 __caps = (caps);					\
+	     __caps && ((cap_bit) = __ffs64(__caps), true);		\
+	     __caps &= __caps - 1)
+
+/*
+ * Sub-cap update notifier.
+ *
+ * ops_cid.sub_caps_updated() notifies sub-scheds when their cap state changes
+ * so they can refresh internal state without polling scx_bpf_sub_caps() per
+ * enqueue.
+ *
+ * Three constraints shape the design:
+ *
+ *   1. Static memory. Deliveries use a fixed-size buffer, both for runtime
+ *      efficiency and so notifications can't be lost under memory pressure.
+ *
+ *   2. High-frequency updates. Grant/revoke can mutate caps in bursts, and the
+ *      notifier path must absorb that without amplifying it.
+ *
+ *   3. Recursive grant/revoke from the callback. A child receiving a
+ *      notification can call grant/revoke on its own children, which can
+ *      cascade recursively down its subtree.
+ *
+ * (1) and (2) lead to coalescing into a fixed payload. Each delivery carries a
+ * single (cmask, caps) pair covering every change since the previous one.
+ * Direction (set vs cleared) isn't encoded as it doesn't fit in the fixed-size
+ * summary. The callback queries scx_bpf_sub_caps() for current state. Only one
+ * delivery is in flight per shard. Further changes fold into the same buffer
+ * and ship as the next callback, so a shard's callbacks fire in order.
+ *
+ * (3) leads to deferred delivery. Events accumulate during grant/revoke and are
+ * delivered after the shard lock is released.
+ */
+struct scx_caps_updated {
+	raw_spinlock_t		lock;
+	u64			caps;
+	struct scx_cmask	*cmask_arena_out;
+	struct list_head	node_in_flight;
+	/* Kernel-side accumulator. Access as &cu->cmask. */
+	TRAILING_OVERLAP(struct scx_cmask, cmask, bits,
+			 u64 _bits[SCX_CMASK_NR_WORDS(SCX_CID_SHARD_MAX_CPUS)];
+	);
+};
+
+struct scx_pshard {
+	raw_spinlock_t		lock;		/* serializes caps */
+	struct scx_sched	*sch;		/* backpointer */
+	struct scx_caps_updated	caps_updated;
+
+	/*
+	 * Per-cap cmask, inline via TRAILING_OVERLAP so cmask.bits[] overlaps
+	 * the trailing _bits[] storage. Access as &caps[i].cmask. See
+	 * scx_sched_pcpu->ecaps.
+	 */
+	TRAILING_OVERLAP(struct scx_cmask, cmask, bits,
+			 u64 _bits[SCX_CMASK_NR_WORDS(SCX_CID_SHARD_MAX_CPUS)];
+	) caps[__SCX_NR_CAPS];
+
+	/*
+	 * Shard geometry captured at alloc. cmask_arena_out's own header is
+	 * bpf-writable and the live shard range can change before the
+	 * rcu-deferred free, so re-init and size cmask_arena_out from these
+	 * trusted copies instead.
+	 */
+	u32			base;
+	u32			nr_cids;
+};
+#endif
+
 struct scx_sched {
 	/*
 	 * cpu-form and cid-form ops share field offsets up to .priv (verified
@@ -1117,6 +1418,7 @@ struct scx_sched {
 		struct sched_ext_ops_cid	ops_cid;
 	};
 	bool			is_cid_type;	/* true if registered via bpf_sched_ext_ops_cid */
+	bool			dead;		/* set after ops.exit(), gates scx_prog_sched() */
 
 	/*
 	 * Arena map auto-discovered from member progs at struct_ops attach.
@@ -1151,13 +1453,16 @@ struct scx_sched {
 	 */
 	struct rhashtable	dsq_hash;
 	struct scx_sched_pnode	**pnode;
+#ifdef CONFIG_EXT_SUB_SCHED
+	struct scx_pshard	**pshard;	/* indexed by shard_idx */
+#endif
 	struct scx_sched_pcpu __percpu *pcpu;
 
 	u64			slice_dfl;
 	u64			bypass_timestamp;
 	s32			bypass_depth;
 
-	/* bypass dispatch path enable state, see bypass_dsp_enabled() */
+	/* bypass dispatch path enable state, see scx_bypass_dsp_enabled() */
 	unsigned long		bypass_dsp_claim;
 	atomic_t		bypass_dsp_enable_depth;
 
@@ -1166,16 +1471,28 @@ struct scx_sched {
 	u32			dsp_max_batch;
 	s32			level;
 
+#ifdef CONFIG_EXT_SUB_SCHED
+	/*
+	 * pshard[] size captured at enable for the async RCU free path -
+	 * scx_nr_cid_shards may be rewritten by a later scx_cid_init() before
+	 * free runs. While sch is active, use the global.
+	 */
+	u32			nr_pshards;
+#endif
+
 	/*
 	 * Updates to the following warned bitfields can race causing RMW issues
 	 * but it doesn't really matter.
 	 */
 	bool			warned_zero_slice:1;
-	bool			warned_deprecated_rq:1;
 	bool			warned_unassoc_progs:1;
+	bool			warned_nmi_kick:1;
 
 	struct list_head	all;
 
+	/* unique instance id, monotonic and never reused */
+	u64			id;
+
 #ifdef CONFIG_EXT_SUB_SCHED
 	struct rhash_head	hash_node;
 
@@ -1206,6 +1523,7 @@ struct scx_sched {
 	struct timer_list	bypass_lb_timer;
 	cpumask_var_t		bypass_lb_donee_cpumask;
 	cpumask_var_t		bypass_lb_resched_cpumask;
+	cpumask_var_t		stall_cpus;
 	struct rcu_work		rcu_work;
 
 	/* all ancestors including self */
@@ -1303,6 +1621,7 @@ enum scx_enq_flags {
 	SCX_ENQ_DSQ_PRIQ	= 1LLU << 57,
 	SCX_ENQ_NESTED		= 1LLU << 58,
 	SCX_ENQ_GDSQ_FALLBACK	= 1LLU << 59,	/* fell back to global DSQ */
+	SCX_ENQ_IGNORE_CAPS	= 1LLU << 60,	/* admit to local DSQ ignoring caps */
 };
 
 enum scx_deq_flags {
@@ -1329,6 +1648,9 @@ enum scx_reenq_flags {
 	/* low 16bits determine which tasks should be reenqueued */
 	SCX_REENQ_ANY		= 1LLU << 0,	/* all tasks */
 
+	/* internal: kernel-issued on cap revoke, not accepted from BPF */
+	SCX_REENQ_CAP_REVOKE	= 1LLU << 1,
+
 	__SCX_REENQ_FILTER_MASK	= 0xffffLLU,
 
 	__SCX_REENQ_USER_MASK	= SCX_REENQ_ANY,
@@ -1469,21 +1791,24 @@ static const char *scx_enable_state_str[] = {
  * The sched_ext core uses a "lock dancing" protocol coordinated by
  * p->scx.holding_cpu. When moving a task to a different rq:
  *
- *   1. Verify task can be moved (CPU affinity, migration_disabled, etc.)
- *   2. Set p->scx.holding_cpu to the current CPU
- *   3. Set task state to %SCX_OPSS_NONE; dequeue waits while DISPATCHING
+ *   1. Set p->scx.holding_cpu to the current CPU
+ *   2. Set task state to %SCX_OPSS_NONE; dequeue waits while DISPATCHING
  *      is set, so clearing DISPATCHING first prevents the circular wait
  *      (safe to lock the rq we need)
- *   4. Unlock the current CPU's rq
- *   5. Lock src_rq (where the task currently lives)
- *   6. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the
+ *   3. Unlock the current CPU's rq
+ *   4. Lock src_rq (where the task currently lives)
+ *   5. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the
  *      race (dequeue clears holding_cpu to -1 when it takes the task), in
  *      this case migration is aborted
- *   7. If src_rq == dst_rq: clear holding_cpu and enqueue directly
+ *   6. If src_rq == dst_rq: clear holding_cpu and enqueue directly
  *      into dst_rq's local DSQ (no lock swap needed)
- *   8. Otherwise: call move_remote_task_to_local_dsq(), which releases
- *      src_rq, locks dst_rq, and performs the deactivate/activate
- *      migration cycle (dst_rq is held on return)
+ *   7. Otherwise, verify under src_rq lock that the task can be moved to dst_rq
+ *      (CPU affinity, migration_disabled, etc.). If not, clear holding_cpu,
+ *      leave the task on src_rq, and enqueue it on the fallback DSQ.
+ *   8. Otherwise (i.e. if the task can be moved to dst_rq), call
+ *      move_remote_task_to_local_dsq(), which releases src_rq, locks dst_rq,
+ *      and performs the deactivate/activate migration cycle
+ *      (dst_rq is held on return)
  *   9. Unlock dst_rq and re-lock the current CPU's rq to restore
  *      the lock state expected by the caller
  *
@@ -1517,6 +1842,47 @@ enum scx_ops_state {
 #define SCX_OPSS_STATE_MASK	((1LU << SCX_OPSS_QSEQ_SHIFT) - 1)
 #define SCX_OPSS_QSEQ_MASK	(~SCX_OPSS_STATE_MASK)
 
+/*
+ * SCX task iterator.
+ */
+struct scx_task_iter {
+	struct sched_ext_entity		cursor;
+	struct task_struct		*locked_task;
+	struct rq			*rq;
+	struct rq_flags			rf;
+	u32				cnt;
+	bool				list_locked;
+#ifdef CONFIG_EXT_SUB_SCHED
+	struct cgroup			*cgrp;
+	struct cgroup_subsys_state	*css_pos;
+	struct css_task_iter		css_iter;
+#endif
+};
+
+/*
+ * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid
+ * starvation. During the READY -> ENABLED task switching loop, the calling
+ * thread's sched_class gets switched from fair to ext. As fair has higher
+ * priority than ext, the calling thread can be indefinitely starved under
+ * fair-class saturation, leading to a system hang.
+ */
+struct scx_enable_cmd {
+	struct kthread_work	work;
+	union {
+		struct sched_ext_ops		*ops;
+		struct sched_ext_ops_cid	*ops_cid;
+	};
+	bool			is_cid_type;
+	struct bpf_map		*arena_map;	/* arena ref to transfer to sch */
+	int			ret;
+};
+
+/* string formatting from BPF */
+struct scx_bstr_buf {
+	u64			data[MAX_BPRINTF_VARARGS];
+	char			line[SCX_EXIT_MSG_LEN];
+};
+
 extern struct scx_sched __rcu *scx_root;
 DECLARE_PER_CPU(struct rq *, scx_locked_rq_state);
 
@@ -1537,12 +1903,121 @@ __printf(5, 0) bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind,
 __printf(5, 6) bool __scx_exit(struct scx_sched *sch, enum scx_exit_kind kind,
 			       s64 exit_code, s32 exit_cpu, const char *fmt, ...);
 
+u32 scx_get_task_state(const struct task_struct *p);
+void scx_set_task_state(struct task_struct *p, u32 state);
+void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp);
+void scx_task_iter_unlock(struct scx_task_iter *iter);
+void scx_task_iter_stop(struct scx_task_iter *iter);
+struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter);
+void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p);
+void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
+			 int sticky_cpu);
+bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq,
+			    struct scx_dispatch_q *dsq, u64 enq_flags);
+bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq);
+bool scx_rq_online(struct rq *rq);
+void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq);
+void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags);
+void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
+			u64 reenq_flags, struct rq *locked_rq);
+int __scx_init_task(struct scx_sched *sch, struct task_struct *p, bool fork);
+void scx_enable_task(struct scx_sched *sch, struct task_struct *p);
+void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p);
+void scx_sub_init_cancel_task(struct scx_sched *sch, struct task_struct *p);
+void scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p);
+#if defined(CONFIG_EXT_GROUP_SCHED) || defined(CONFIG_EXT_SUB_SCHED)
+void scx_cgroup_lock(void);
+void scx_cgroup_unlock(void);
+#endif
+s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch);
+void scx_disable_bypass_dsp(struct scx_sched *sch);
+void scx_bypass(struct scx_sched *sch, bool bypass);
+s32 scx_link_sched(struct scx_sched *sch);
+void scx_unlink_sched(struct scx_sched *sch);
+void scx_disable_dump(struct scx_sched *sch);
+void scx_log_sched_disable(struct scx_sched *sch);
+void scx_flush_disable_work(struct scx_sched *sch);
+struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
+					  struct cgroup *cgrp,
+					  struct scx_sched *parent);
+int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops);
+int scx_sched_sysfs_add(struct scx_sched *sch);
+bool scx_is_descendant(struct scx_sched *sch, struct scx_sched *ancestor);
+__printf(3, 0) s32 scx_bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf,
+				   char *fmt, unsigned long long *data, u32 data__sz);
+
+extern raw_spinlock_t scx_sched_lock;
+extern struct mutex scx_enable_mutex;
+extern struct percpu_rw_semaphore scx_fork_rwsem;
+extern raw_spinlock_t scx_exit_bstr_buf_lock;
+extern struct scx_bstr_buf scx_exit_bstr_buf;
+#ifdef CONFIG_EXT_SUB_SCHED
+extern const struct rhashtable_params scx_sched_hash_params;
+extern struct rhashtable scx_sched_hash;
+extern struct scx_sched *scx_enabling_sub_sched;
+#endif
+
 #define scx_exit(sch, kind, exit_code, fmt, args...)				\
 	__scx_exit(sch, kind, exit_code, raw_smp_processor_id(), fmt, ##args)
 #define scx_error(sch, fmt, args...)						\
 	scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args)
-#define scx_verror(sch, fmt, args)						\
-	scx_vexit((sch), SCX_EXIT_ERROR, 0, raw_smp_processor_id(), fmt, args)
+
+static inline struct scx_dispatch_q *scx_bypass_dsq(struct scx_sched *sch, s32 cpu)
+{
+	return &per_cpu_ptr(sch->pcpu, cpu)->bypass_dsq;
+}
+
+/**
+ * scx_bypass_dsp_enabled - Check if bypass dispatch path is enabled
+ * @sch: scheduler to check
+ *
+ * When a descendant scheduler enters bypass mode, bypassed tasks are scheduled
+ * by the nearest non-bypassing ancestor, or the root scheduler if all ancestors
+ * are bypassing. In the former case, the ancestor is not itself bypassing but
+ * its bypass DSQs will be populated with bypassed tasks from descendants. Thus,
+ * the ancestor's bypass dispatch path must be active even though its own
+ * bypass_depth remains zero.
+ *
+ * This function checks bypass_dsp_enable_depth which is managed separately from
+ * bypass_depth to enable this decoupling. See enable_bypass_dsp() and
+ * scx_disable_bypass_dsp().
+ */
+static inline bool scx_bypass_dsp_enabled(struct scx_sched *sch)
+{
+	return unlikely(atomic_read(&sch->bypass_dsp_enable_depth));
+}
+
+/**
+ * scx_ops_sanitize_err - Sanitize a -errno value
+ * @sch: scx_sched to error out on error
+ * @ops_name: operation to blame on failure
+ * @err: -errno value to sanitize
+ *
+ * Verify @err is a valid -errno. If not, trigger scx_error() and return
+ * -%EPROTO. This is necessary because returning a rogue -errno up the chain can
+ * cause misbehaviors. For an example, a large negative return from
+ * ops.init_task() triggers an oops when passed up the call chain because the
+ * value fails IS_ERR() test after being encoded with ERR_PTR() and then is
+ * handled as a pointer.
+ */
+static inline int scx_ops_sanitize_err(struct scx_sched *sch, const char *ops_name, s32 err)
+{
+	if (err < 0 && err >= -MAX_ERRNO)
+		return err;
+
+	scx_error(sch, "ops.%s() returned an invalid errno %d", ops_name, err);
+	return -EPROTO;
+}
+
+static inline void scx_schedule_reenq_local(struct rq *rq, u64 reenq_flags)
+{
+	struct scx_sched *root = rcu_dereference_sched(scx_root);
+
+	if (WARN_ON_ONCE(!root))
+		return;
+
+	schedule_dsq_reenq(root, &rq->scx.local_dsq, reenq_flags, rq);
+}
 
 /*
  * Return the rq currently locked from an scx callback, or NULL if no rq is
@@ -1570,8 +2045,11 @@ static inline void update_locked_rq(struct rq *rq)
 /*
  * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not
  * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit.
+ *
+ * @ops is the ops table to dispatch through: ops for the cpu form, ops_cid
+ * for the cid form.
  */
-#define SCX_CALL_OP(sch, op, locked_rq, args...)				\
+#define __SCX_CALL_OP(sch, ops, op, locked_rq, args...)				\
 do {										\
 	struct rq *__prev_locked_rq;						\
 										\
@@ -1584,6 +2062,9 @@ do {										\
 		update_locked_rq(__prev_locked_rq);				\
 } while (0)
 
+#define SCX_CALL_OP(sch, op, locked_rq, args...)				\
+	__SCX_CALL_OP(sch, ops, op, locked_rq, ##args)
+
 #define SCX_CALL_OP_RET(sch, op, locked_rq, args...)				\
 ({										\
 	struct rq *__prev_locked_rq;						\
@@ -1615,17 +2096,36 @@ do {										\
  * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants
  * while a previous one is still in progress.
  */
-#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...)			\
+#define __SCX_CALL_OP_TASK(sch, ops, op, locked_rq, task, args...)		\
 do {										\
 	WARN_ON_ONCE(current->scx.kf_tasks[0]);					\
 	current->scx.kf_tasks[0] = task;					\
-	SCX_CALL_OP((sch), op, locked_rq, task, ##args);			\
+	__SCX_CALL_OP((sch), ops, op, locked_rq, task, ##args);			\
 	current->scx.kf_tasks[0] = NULL;					\
 } while (0)
 
+/*
+ * A per-task op runs on @task's owner - WARN if @sch isn't it. Sites that must
+ * target a different scheduler call __SCX_CALL_OP_TASK() directly.
+ */
+#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...)			\
+do {										\
+	WARN_ON_ONCE(scx_has_subs() && (sch) != scx_task_sched_rcu(task));	\
+	__SCX_CALL_OP_TASK((sch), ops, op, locked_rq, task, ##args);		\
+} while (0)
+
+/*
+ * Dispatch a task op through the cid-form ops_cid table. Only set_cmask() needs
+ * this: it takes an arena cmask address instead of a cpumask, so it cannot be
+ * invoked via its cpu-form set_cpumask() slot.
+ */
+#define SCX_CALL_CID_OP_TASK(sch, op, locked_rq, task, args...)			\
+	__SCX_CALL_OP_TASK(sch, ops_cid, op, locked_rq, task, ##args)
+
 #define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...)			\
 ({										\
 	__typeof__((sch)->ops.op(task, ##args)) __ret;				\
+	WARN_ON_ONCE(scx_has_subs() && (sch) != scx_task_sched_rcu(task));	\
 	WARN_ON_ONCE(current->scx.kf_tasks[0]);					\
 	current->scx.kf_tasks[0] = task;					\
 	__ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args);		\
@@ -1665,6 +2165,19 @@ static inline bool scx_bypassing(struct scx_sched *sch, s32 cpu)
 }
 
 #ifdef CONFIG_EXT_SUB_SCHED
+DECLARE_STATIC_KEY_FALSE(__scx_has_subs);
+
+/**
+ * scx_has_subs - Whether any sub-scheduler exists
+ *
+ * Gates the sub-sched portions of hot paths so that a root-only system doesn't
+ * pay for them. See scx_sub_enable_workfn() and scx_sched_free_rcu_work().
+ */
+static inline bool scx_has_subs(void)
+{
+	return static_branch_unlikely(&__scx_has_subs);
+}
+
 /**
  * scx_task_sched - Find scx_sched scheduling a task
  * @p: task of interest
@@ -1714,14 +2227,20 @@ static inline bool scx_task_on_sched(struct scx_sched *sch,
 static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux)
 {
 	struct sched_ext_ops *ops;
-	struct scx_sched *root;
+	struct scx_sched *sch, *root;
 
 	ops = bpf_prog_get_assoc_struct_ops(aux);
-	if (likely(ops))
-		return rcu_dereference_all(ops->priv);
+	if (likely(ops)) {
+		sch = rcu_dereference_all(ops->priv);
+		if (sch && unlikely(READ_ONCE(sch->dead)))
+			return NULL;
+		return sch;
+	}
 
 	root = rcu_dereference_all(scx_root);
 	if (root) {
+		if (unlikely(READ_ONCE(root->dead)))
+			return NULL;
 		/*
 		 * COMPAT-v6.19: Schedulers built before sub-sched support was
 		 * introduced may have unassociated non-struct_ops programs.
@@ -1753,6 +2272,8 @@ static inline struct scx_sched *scx_parent(struct scx_sched *sch)
 		return NULL;
 }
 #else	/* CONFIG_EXT_SUB_SCHED */
+static inline bool scx_has_subs(void) { return false; }
+
 static inline struct scx_sched *scx_task_sched(const struct task_struct *p)
 {
 	return rcu_dereference_protected(scx_root,
@@ -1773,7 +2294,11 @@ static inline bool scx_task_on_sched(struct scx_sched *sch,
 
 static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux)
 {
-	return rcu_dereference_all(scx_root);
+	struct scx_sched *root = rcu_dereference_all(scx_root);
+
+	if (root && unlikely(READ_ONCE(root->dead)))
+		return NULL;
+	return root;
 }
 
 static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; }
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
new file mode 100644
index 0000000000000..3cc6d2633f73c
--- /dev/null
+++ b/kernel/sched/ext/sub.c
@@ -0,0 +1,1791 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
+ *
+ * Sub-scheduler hierarchy support.
+ *
+ * A sub-scheduler is an scx_sched attached to a cgroup subtree under another
+ * scx_sched. This file holds the sub-scheduler implementation: the scheduler
+ * tree walk, capability delegation, per-shard cap state and its sync, and the
+ * sub-scheduler enable/disable paths. The core dispatch/enqueue machinery it
+ * builds on lives in ext.c.
+ *
+ * Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2026 Tejun Heo <tj@kernel.org>
+ */
+#include <linux/rhashtable.h>
+#include "internal.h"
+#include "cid.h"
+#include "arena.h"
+#include "sub.h"
+#include "inlines.h"
+
+#ifdef CONFIG_EXT_SUB_SCHED
+
+/*
+ * On while any sub-scheduler exists so that a root-only system doesn't pay for
+ * the sub-sched portions of hot paths. See scx_has_subs().
+ */
+DEFINE_STATIC_KEY_FALSE(__scx_has_subs);
+
+/**
+ * scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk
+ * @pos: current position
+ * @root: walk root
+ *
+ * In a walk started by scx_next_descendant_pre(), continue past @pos's subtree:
+ * return @pos's next sibling, or the closest ancestor's next sibling, or NULL
+ * if @pos's subtree is the last under @root. Same locking rules.
+ */
+struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root)
+{
+	struct scx_sched *next;
+
+	lockdep_assert(lockdep_is_held(&scx_enable_mutex) ||
+		       lockdep_is_held(&scx_sched_lock) ||
+		       rcu_read_lock_any_held());
+
+	while (pos != root) {
+		next = list_next_or_null_rcu(&scx_parent(pos)->children, &pos->sibling,
+					     struct scx_sched, sibling);
+		if (next)
+			return next;
+		pos = scx_parent(pos);
+	}
+	return NULL;
+}
+
+/**
+ * scx_next_descendant_pre - find the next descendant for pre-order walk
+ * @pos: the current position (%NULL to initiate traversal)
+ * @root: sched whose descendants to walk
+ *
+ * To be used by scx_for_each_descendant_pre(). Find the next descendant to
+ * visit for pre-order traversal of @root's descendants. @root is included in
+ * the iteration and the first node to be visited.
+ */
+struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root)
+{
+	struct scx_sched *next;
+
+	lockdep_assert(lockdep_is_held(&scx_enable_mutex) ||
+		       lockdep_is_held(&scx_sched_lock) ||
+		       rcu_read_lock_any_held());
+
+	/* if first iteration, visit @root */
+	if (!pos)
+		return root;
+
+	/* visit the first child if exists */
+	next = list_first_or_null_rcu(&pos->children, struct scx_sched, sibling);
+	if (next)
+		return next;
+
+	/* no child, visit my or the closest ancestor's next sibling */
+	return scx_skip_subtree_pre(pos, root);
+}
+
+static struct scx_sched *scx_find_sub_sched(u64 cgroup_id)
+{
+	return rhashtable_lookup(&scx_sched_hash, &cgroup_id,
+				 scx_sched_hash_params);
+}
+
+void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch)
+{
+	rcu_assign_pointer(p->scx.sched, sch);
+}
+
+struct cgroup *sch_cgroup(struct scx_sched *sch)
+{
+	return sch->cgrp;
+}
+
+/* for each descendant of @cgrp including self, set ->scx_sched to @sch */
+void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch)
+{
+	struct cgroup *pos;
+	struct cgroup_subsys_state *css;
+
+	cgroup_for_each_live_descendant_pre(pos, css, cgrp)
+		rcu_assign_pointer(pos->scx_sched, sch);
+}
+
+static void free_pshard(struct scx_pshard *pshard)
+{
+	struct scx_caps_updated *cu;
+
+	if (!pshard)
+		return;
+	cu = &pshard->caps_updated;
+	if (cu->cmask_arena_out)
+		scx_arena_free(pshard->sch, cu->cmask_arena_out,
+			       struct_size_t(struct scx_cmask, bits,
+					     SCX_CMASK_NR_WORDS(pshard->nr_cids)));
+	kfree(pshard);
+}
+
+void scx_free_pshards(struct scx_sched *sch)
+{
+	s32 si;
+
+	if (!sch->pshard)
+		return;
+	for (si = 0; si < sch->nr_pshards; si++)
+		free_pshard(sch->pshard[si]);
+	kfree(sch->pshard);
+}
+
+static struct scx_pshard *alloc_pshard(struct scx_sched *sch, s32 shard_idx, s32 node)
+{
+	const struct scx_cid_shard *shard = &scx_cid_shard_ranges[shard_idx];
+	size_t cmask_size = struct_size_t(struct scx_cmask, bits,
+					  SCX_CMASK_NR_WORDS(shard->nr_cids));
+	struct scx_pshard *pshard;
+	struct scx_caps_updated *cu;
+	s32 i;
+
+	pshard = kzalloc_node(sizeof(*pshard), GFP_KERNEL, node);
+	if (!pshard)
+		return NULL;
+
+	raw_spin_lock_init(&pshard->lock);
+	pshard->sch = sch;
+	pshard->base = shard->base_cid;
+	pshard->nr_cids = shard->nr_cids;
+
+	for (i = 0; i < __SCX_NR_CAPS; i++)
+		scx_cmask_init(&pshard->caps[i].cmask, shard->base_cid, shard->nr_cids);
+
+	cu = &pshard->caps_updated;
+	raw_spin_lock_init(&cu->lock);
+	INIT_LIST_HEAD(&cu->node_in_flight);
+	__scx_cmask_init(&cu->cmask, shard->base_cid, shard->nr_cids, SCX_CID_SHARD_MAX_CPUS);
+
+	cu->cmask_arena_out = scx_arena_alloc(sch, cmask_size);
+	if (!cu->cmask_arena_out) {
+		free_pshard(pshard);
+		return NULL;
+	}
+
+	scx_cmask_init(cu->cmask_arena_out, shard->base_cid, shard->nr_cids);
+
+	return pshard;
+}
+
+s32 scx_alloc_pshards(struct scx_sched *sch)
+{
+	struct scx_pshard **pshard;
+	s32 si;
+
+	if (!sch->is_cid_type || !sch->arena_pool)
+		return 0;
+
+	pshard = kzalloc_objs(pshard[0], scx_nr_cid_shards, GFP_KERNEL);
+	if (!pshard)
+		return -ENOMEM;
+
+	for (si = 0; si < scx_nr_cid_shards; si++) {
+		pshard[si] = alloc_pshard(sch, si, scx_shard_node[si]);
+		if (!pshard[si]) {
+			while (--si >= 0)
+				free_pshard(pshard[si]);
+			kfree(pshard);
+			return -ENOMEM;
+		}
+	}
+
+	sch->nr_pshards = scx_nr_cid_shards;
+	/*
+	 * Publish only after every entry is built so a reader observing
+	 * @sch->pshard never sees a partially-filled array. Pair the store
+	 * with a barrier and READ_ONCE() on the read side.
+	 */
+	smp_wmb();
+	WRITE_ONCE(sch->pshard, pshard);
+	return 0;
+}
+
+/*
+ * Seed the root's caps fully. Root owns all cids on all caps at enable time.
+ * Children acquire caps via scx_bpf_sub_grant().
+ */
+void scx_init_root_caps(struct scx_sched *sch)
+{
+	s32 si, i;
+
+	for (si = 0; si < sch->nr_pshards; si++) {
+		struct scx_pshard *ps = sch->pshard[si];
+
+		for (i = 0; i < __SCX_NR_CAPS; i++)
+			scx_cmask_fill(&ps->caps[i].cmask);
+	}
+}
+
+/**
+ * scx_local_or_reject_dsq - Pick the local or reject DSQ for an insert
+ * @sch: enqueuing sub-sched
+ * @rq: rq whose local DSQ @p targets
+ * @p: task being inserted
+ * @enq_flags: in/out, unhonored flags are cleared
+ *
+ * Return @rq's local DSQ if @sch holds the required caps on @rq's cid,
+ * otherwise @rq's reject DSQ after recording the reenq reason on @p.
+ *
+ * %SCX_ENQ_IMMED and %SCX_ENQ_PREEMPT are cleared when diverting to reject.
+ * %SCX_ENQ_PREEMPT is also cleared on a fallback migration-disabled admission.
+ *
+ * Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued
+ * to and run by its nearest non-bypassing ancestor. If root is bypassing, it
+ * always holds all caps.
+ */
+struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq,
+					       struct task_struct *p, u64 *enq_flags)
+{
+	if (!scx_has_subs())
+		return &rq->scx.local_dsq;
+
+	s32 cid = __scx_cpu_to_cid(cpu_of(rq));
+	struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch;
+	u64 needed = scx_caps_for_enq(*enq_flags);
+	u64 missing;
+
+	/*
+	 * On a remote activation the scheduling sched (@asch) differs from
+	 * @p's owner (@sch). Check caps against the scheduling sched.
+	 */
+	if (*enq_flags & SCX_ENQ_PREEMPT)
+		needed |= scx_caps_for_preempt(asch, rq);
+	missing = scx_missing_caps(asch, cpu_of(rq), needed);
+
+	/* requirements met */
+	if (likely(!missing))
+		return &rq->scx.local_dsq;
+
+	/*
+	 * The task must run on this CPU regardless of caps: the rq is draining
+	 * offline (BPF scheduler bypassed), the task is migration-disabled, or a
+	 * migration is pending. Admit despite the missing caps and count it.
+	 * Refuse preemptions.
+	 */
+	if (unlikely(!scx_rq_online(rq) || is_migration_disabled(p) ||
+		     p->migration_pending)) {
+		__scx_add_event(sch, SCX_EV_SUB_FORCED_ADMIT, 1);
+		*enq_flags &= ~SCX_ENQ_PREEMPT;
+		return &rq->scx.local_dsq;
+	}
+
+	p->scx.reenq_reason_caps = missing;
+	p->scx.reenq_reason_cid = cid;
+
+	/*
+	 * Only local DSQ can honor IMMED and dsq_inc_nr() WARNs on IMMED into
+	 * others. Strip both the enq flag and the sticky task flag - the
+	 * latter can carry in from an earlier admitted IMMED insert. Strip
+	 * PREEMPT too.
+	 */
+	*enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT);
+	p->scx.flags &= ~SCX_TASK_IMMED;
+
+	return &rq->scx.reject_dsq;
+}
+
+/* @p lost the caps needed to stay on @rq's local DSQ? Record reason if so. */
+bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p)
+{
+	u64 missing;
+
+	/* migration-disabled tasks are admitted regardless of caps */
+	if (is_migration_disabled(p))
+		return false;
+
+	missing = scx_missing_caps(scx_task_sched(p), cpu_of(rq), scx_caps_for_task(p));
+	if (likely(!missing))
+		return false;
+
+	p->scx.reenq_reason_caps = missing;
+	p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu_of(rq));
+	return true;
+}
+
+/*
+ * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides
+ * from p->scx.reenq_reason_*.
+ *
+ * A task can be re-rejected repeatedly, and there's no repeat limit here.
+ * Rejection can't happen for root, and sub-scheds can be safely ejected after
+ * triggering the stall watchdog.
+ */
+void scx_reenq_reject(struct rq *rq)
+{
+	LIST_HEAD(tasks);
+	struct task_struct *p, *n;
+
+	lockdep_assert_rq_held(rq);
+
+	if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list))
+		return;
+
+	/*
+	 * Move to a private list so a task re-rejected by the
+	 * scx_do_enqueue_task() below isn't revisited this round.
+	 */
+	list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) {
+		/* migration_pending tasks should have bypassed to local DSQ */
+		if (WARN_ON_ONCE(p->migration_pending))
+			continue;
+
+		scx_dispatch_dequeue(rq, p);
+
+		if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
+			p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
+		p->scx.flags |= SCX_TASK_REENQ_CAP;
+
+		list_add_tail(&p->scx.dsq_list.node, &tasks);
+	}
+
+	list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
+		list_del_init(&p->scx.dsq_list.node);
+
+		scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
+
+		p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
+	}
+}
+
+/* record a caps change, see struct scx_caps_updated */
+static void caps_updated_record(struct scx_pshard *ps, const struct scx_cmask *cids, u64 caps,
+				struct list_head *to_deliver)
+{
+	struct scx_caps_updated *cu = &ps->caps_updated;
+
+	guard(raw_spinlock)(&cu->lock);
+	scx_cmask_or(&cu->cmask, cids);
+	cu->caps |= caps;
+	if (list_empty(&cu->node_in_flight))
+		list_add_tail(&cu->node_in_flight, to_deliver);
+}
+
+/* deliver queued caps_updated callbacks, see struct scx_caps_updated */
+static void caps_updated_deliver(struct list_head *to_deliver)
+{
+	struct scx_caps_updated *cu, *tmp;
+
+	list_for_each_entry_safe(cu, tmp, to_deliver, node_in_flight) {
+		struct scx_pshard *ps = container_of(cu, struct scx_pshard, caps_updated);
+		struct scx_sched *sch = ps->sch;
+
+		while (true) {
+			u64 caps = 0;
+
+			/*
+			 * During enable, has_op is set after ops.sub_attach(),
+			 * so !has_op means the op is absent or the sched isn't
+			 * live yet - e.g. caps grant from ops.sub_attach().
+			 * Either way don't consume - leave for
+			 * scx_sub_seed_caps() to deliver once live.
+			 */
+			scoped_guard (raw_spinlock, &cu->lock) {
+				if (cu->caps && SCX_HAS_OP(sch, sub_caps_updated) &&
+				    likely(!READ_ONCE(sch->aborting))) {
+					struct scx_cmask_ref ref;
+
+					caps = cu->caps;
+					scx_cmask_ref_init_kern(sch, cu->cmask_arena_out,
+								ps->base, ps->nr_cids, &ref);
+					scx_cmask_ref_copy(&ref, &cu->cmask);
+					scx_cmask_clear(&cu->cmask);
+					cu->caps = 0;
+				} else {
+					list_del_init(&cu->node_in_flight);
+				}
+			}
+			if (!caps)
+				break;
+
+			/* caps != 0 only when deliverable (has_op, above) */
+			SCX_CALL_OP(sch, sub_caps_updated, NULL,
+				    scx_kaddr_to_arena(sch, cu->cmask_arena_out),
+				    caps);
+		}
+	}
+}
+
+/*
+ * Deliver caps owed to @sch that couldn't be delivered earlier (e.g. a grant
+ * taken during its sub_attach(), before has_op was set). Called once @sch is
+ * enabled.
+ */
+static void scx_sub_seed_caps(struct scx_sched *sch)
+{
+	LIST_HEAD(to_deliver);
+	s32 si;
+
+	guard(irqsave)();
+
+	for (si = 0; si < sch->nr_pshards; si++) {
+		struct scx_pshard *ps = sch->pshard[si];
+		struct scx_caps_updated *cu = &ps->caps_updated;
+
+		scoped_guard (raw_spinlock, &cu->lock) {
+			if (cu->caps && list_empty(&cu->node_in_flight))
+				list_add_tail(&cu->node_in_flight, &to_deliver);
+		}
+	}
+	caps_updated_deliver(&to_deliver);
+}
+
+static u64 calc_effective_caps(struct scx_pshard *ps, s32 cid)
+{
+	u64 ecaps = 0;
+	u32 cap_bit;
+
+	for (cap_bit = 0; cap_bit < __SCX_NR_CAPS; cap_bit++)
+		if (scx_cmask_test(cid, &ps->caps[cap_bit].cmask))
+			ecaps |= BIT_U64(cap_bit) | scx_caps_implied(BIT_U64(cap_bit));
+	return ecaps;
+}
+
+/**
+ * queue_sync_ecaps - Queue ecaps update for a (sch, cid) pair
+ * @sch: sched to update
+ * @cid: cid to update
+ *
+ * Queue an ecaps update for @sch's @cid and kick the cpu so that it syncs in
+ * balance_one().
+ */
+static void queue_sync_ecaps(struct scx_sched *sch, s32 cid)
+{
+	s32 cpu = __scx_cid_to_cpu(cid);
+	struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
+
+	/*
+	 * Pairs with smp_mb() in scx_process_sync_ecaps(). Either the check
+	 * below sees the node off the list and queues it, or the in-flight sync
+	 * sees the caps[] update made before this call.
+	 */
+	smp_mb();
+
+	/* @cid's pshard->lock excludes concurrent queueing attempts */
+	if (llist_on_list(&pcpu->ecaps_to_sync_node))
+		return;
+	if (llist_add(&pcpu->ecaps_to_sync_node, &cpu_rq(cpu)->scx.ecaps_to_sync))
+		scx_kick_cpu(scx_root, cpu, 0);
+}
+
+/* discard @rq's queued ecaps syncs */
+static void discard_queued_syncs(struct rq *rq)
+{
+	struct llist_node *pos, *tmp;
+
+	lockdep_assert_rq_held(rq);
+
+	llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync))
+		init_llist_node(pos);
+}
+
+/**
+ * scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[]
+ * @rq: the cid's cpu rq
+ * @prev: @rq's previous task from the in-progress balance
+ *
+ * pshard->caps[] is the target configuration. pcpu->ecaps is the effective
+ * transposed copy owned by the cid's cpu and written only here under @rq's
+ * lock.
+ *
+ * A sched that newly gains baseline access here is owed an update_idle() so it
+ * learns the cid's idle state. Such a gain arms the per-rq
+ * %SCX_RQ_SUB_IDLE_RENOTIFY gate so the next idle pick delivers it.
+ */
+void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
+{
+	s32 cpu = cpu_of(rq);
+	s32 cid, shard;
+	struct llist_node *batch, *pos, *tmp;
+	u64 lost_all = 0;
+
+	lockdep_assert_rq_held(rq);
+
+	if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync)))
+		return;
+
+	/*
+	 * ecaps are zeroed while the cpu is inactive and must stay zero.
+	 * Discard queued syncs instead of processing them - the
+	 * scx_online_ecaps() reseed re-syncs every sched on activation.
+	 * cpu_active() clears before the offline zeroing and sets before the
+	 * reseed is queued, so this test can neither miss a racing sync nor
+	 * eat the reseed.
+	 */
+	if (unlikely(!cpu_active(cpu))) {
+		discard_queued_syncs(rq);
+		return;
+	}
+
+	/* @cid is valid here: the cpu is active with queued syncs */
+	cid = __scx_cpu_to_cid(cpu);
+	shard = scx_cid_to_shard[cid];
+
+	batch = llist_del_all(&rq->scx.ecaps_to_sync);
+	llist_for_each_safe(pos, tmp, batch) {
+		struct scx_sched_pcpu *pcpu =
+			container_of(pos, struct scx_sched_pcpu, ecaps_to_sync_node);
+		struct scx_pshard *ps = pcpu->sch->pshard[shard];
+		u64 old, ecaps, lost, gained;
+
+		init_llist_node(pos);
+
+		/* pairs with smp_mb() in queue_sync_ecaps(), see there */
+		smp_mb();
+
+		old = READ_ONCE(pcpu->ecaps);
+		ecaps = calc_effective_caps(ps, cid);
+		WRITE_ONCE(pcpu->ecaps, ecaps);
+
+		lost = old & ~ecaps;
+		gained = ecaps & ~old;
+		lost_all |= lost;
+
+		/*
+		 * Tell the sched its effective caps on this cid changed. The
+		 * invocation is equivalent to the dispatch path and may drop
+		 * and re-acquire the rq lock temporarily while the rest of
+		 * @batch is held privately, see scx_discard_ecaps_to_sync().
+		 */
+		if (ecaps != pcpu->reported_ecaps &&
+		    SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) &&
+		    !scx_bypassing(pcpu->sch, cpu)) {
+			struct scx_dsp_ctx *dspc = &pcpu->dsp_ctx;
+
+			dspc->rq = rq;
+			/* stash @prev so nested dispatches can access it */
+			rq->scx.sub_dispatch_prev = prev;
+			SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, scx_cpu_arg(cpu),
+				    pcpu->reported_ecaps, ecaps);
+			rq->scx.sub_dispatch_prev = NULL;
+			scx_flush_dispatch_buf(pcpu->sch, rq);
+			pcpu->reported_ecaps = ecaps;
+		}
+
+		/*
+		 * Gaining baseline access owes an update_idle() so the sched
+		 * learns the cpu's idle state. Arm the per-rq gate so the next
+		 * idle pick flushes it. Losing access drops any pending notify.
+		 */
+		if (gained & SCX_CAP_BASE) {
+			pcpu->idle_renotify = true;
+			rq->scx.flags |= SCX_RQ_SUB_IDLE_RENOTIFY;
+		} else if (lost & SCX_CAP_BASE) {
+			pcpu->idle_renotify = false;
+		}
+	}
+
+	/*
+	 * Losing a cap can strand already-queued tasks. Schedule a reenq scan
+	 * to move the now-capless ones off the local DSQ. The scan tests
+	 * against the effective caps and thus must come after the ecaps sync.
+	 */
+	if (lost_all & SCX_CAPS_REENQ_ON_LOSS)
+		scx_schedule_reenq_local(rq, SCX_REENQ_CAP_REVOKE);
+}
+
+/**
+ * scx_unbypass_replay_ecaps - Replay a bypass-suppressed ecaps notification
+ * @rq: rq of the cpu leaving bypass
+ * @sch: scheduler that just left bypass on @rq's cpu
+ *
+ * scx_process_sync_ecaps() consumes syncs while bypassing without delivering
+ * ops.sub_ecaps_updated(), leaving reported_ecaps stale. Nothing re-queues a
+ * sync when bypass lifts, so without a replay a cid that never changes again
+ * would never be notified. The attach-time initial grants are the acute case
+ * as they are consumed during the enable bypass window. Re-queue a sync for
+ * any undelivered delta so the next balance delivers it.
+ */
+void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch)
+{
+	s32 cpu = cpu_of(rq);
+	struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
+	struct scx_pshard *ps;
+	s32 cid;
+
+	lockdep_assert_rq_held(rq);
+
+	/* root holds every cap and never uses ecaps */
+	if (!sch->level)
+		return;
+
+	if (READ_ONCE(pcpu->ecaps) == pcpu->reported_ecaps)
+		return;
+
+	cid = __scx_cpu_to_cid(cpu);
+	ps = sch->pshard[scx_cid_to_shard[cid]];
+
+	guard(raw_spinlock)(&ps->lock);
+	queue_sync_ecaps(sch, cid);
+}
+
+/*
+ * A cpu came back. Re-seed each sub-sched's ecaps on the cpu's cid. The sync
+ * recomputes effective caps from the pshard and fires ops.sub_ecaps_updated()
+ * only on a real change since offline.
+ */
+void scx_online_ecaps(struct rq *rq)
+{
+	s32 cid = __scx_cpu_to_cid(cpu_of(rq));
+	s32 shard = scx_cid_to_shard[cid];
+	struct scx_sched *pos;
+
+	guard(rq_lock_irqsave)(rq);
+
+	scx_for_each_descendant_pre(pos, scx_root) {
+		struct scx_pshard *ps;
+
+		/* root holds every cap and never uses ecaps */
+		if (pos == scx_root)
+			continue;
+
+		ps = pos->pshard[shard];
+		guard(raw_spinlock)(&ps->lock);
+		queue_sync_ecaps(pos, cid);
+	}
+}
+
+/*
+ * A cpu is going down. Zero each sub-sched's in-effect ecaps so cap checks
+ * treat the cpu as capless while offline. Pending and late-queued syncs are
+ * discarded at consumption by scx_process_sync_ecaps() while the cpu is
+ * inactive. Leave reported_ecaps. Ownership is unchanged, so the
+ * scx_online_ecaps() reseed reports only a genuine delta. No callback fires
+ * here.
+ */
+void scx_offline_ecaps(struct rq *rq)
+{
+	s32 cpu = cpu_of(rq);
+	struct scx_sched *pos;
+
+	guard(rq_lock_irqsave)(rq);
+
+	scx_for_each_descendant_pre(pos, scx_root) {
+		/* root holds every cap and never uses ecaps */
+		if (pos == scx_root)
+			continue;
+
+		WRITE_ONCE(per_cpu_ptr(pos->pcpu, cpu)->ecaps, 0);
+	}
+}
+
+/*
+ * @pcpu's sched was unhashed before the grace period, so nothing re-queues its
+ * sync node. Remove the node from @rq's pending list so the pcpu can be freed.
+ */
+void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+	struct llist_node *head = NULL, *tail = NULL;
+	struct llist_node *pos, *tmp;
+
+	/*
+	 * llist can't unlink a single node. Take all queued nodes, drop @pcpu's
+	 * and resplice the rest. Nodes in the taken batch read as on-list
+	 * throughout, so queue_sync_ecaps() stays correct.
+	 */
+	if (llist_on_list(&pcpu->ecaps_to_sync_node)) {
+		scoped_guard (rq_lock_irqsave, rq) {
+			llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) {
+				if (pos == &pcpu->ecaps_to_sync_node) {
+					init_llist_node(pos);
+				} else {
+					pos->next = head;
+					head = pos;
+					if (!tail)
+						tail = pos;
+				}
+			}
+			if (head)
+				llist_add_batch(head, tail, &rq->scx.ecaps_to_sync);
+		}
+	}
+
+	/*
+	 * An in-flight scx_process_sync_ecaps() batch may still hold the node
+	 * privately across dispatch-induced rq unlocks, reading as on-list.
+	 *
+	 * Because a bypassing sched gets no op call, init_llist_node() and all
+	 * @pcpu accesses share one contiguous lock hold, off-list under the rq
+	 * lock means @pcpu won't be accessed again.
+	 */
+	while (true) {
+		scoped_guard (rq_lock_irqsave, rq) {
+			if (!llist_on_list(&pcpu->ecaps_to_sync_node))
+				return;
+		}
+		cpu_relax();
+	}
+}
+
+/**
+ * scx_discard_stale_ecaps_syncs - Discard ecaps syncs from earlier schedulers
+ *
+ * To be called during root enable before the scheduler goes live. An earlier
+ * root's sub-sched may not have gone through its RCU free path yet (e.g. a
+ * still-open link fd defers it) and can leave queued ecaps syncs behind.
+ * Processing them would decode the dead sched's pshards with the current cid
+ * layout. Discard them instead. The backing scx_sched_pcpu's are still
+ * allocated as the free path removes ecaps_to_sync_node before freeing.
+ */
+void scx_discard_stale_ecaps_syncs(void)
+{
+	s32 cpu;
+
+	for_each_possible_cpu(cpu) {
+		struct rq *rq = cpu_rq(cpu);
+
+		guard(rq_lock_irqsave)(rq);
+		discard_queued_syncs(rq);
+	}
+}
+
+static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq);
+
+void drain_descendants(struct scx_sched *sch)
+{
+	/*
+	 * Child scheds that finished the critical part of disabling will take
+	 * themselves off @sch->children. Wait for it to drain. As propagation
+	 * is recursive, empty @sch->children means that all proper descendant
+	 * scheds reached unlinking stage.
+	 */
+	wait_event(scx_unlink_waitq, list_empty(&sch->children));
+}
+
+static void scx_fail_parent(struct scx_sched *sch,
+			    struct task_struct *failed, s32 fail_code)
+{
+	struct scx_sched *parent = scx_parent(sch);
+	struct scx_task_iter sti;
+	struct task_struct *p;
+
+	scx_error(parent, "ops.init_task() failed (%d) for %s[%d] while disabling a sub-scheduler",
+		  fail_code, failed->comm, failed->pid);
+
+	/*
+	 * Once $parent is bypassed, it's safe to put SCX_TASK_NONE tasks into
+	 * it. This may cause downstream failures on the BPF side but $parent is
+	 * dying anyway.
+	 */
+	scx_bypass(parent, true);
+
+	scx_task_iter_start(&sti, sch->cgrp);
+	while ((p = scx_task_iter_next_locked(&sti))) {
+		if (scx_task_on_sched(parent, p))
+			continue;
+
+		scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
+			scx_disable_and_exit_task(sch, p);
+			scx_set_task_sched(p, parent);
+		}
+	}
+	scx_task_iter_stop(&sti);
+}
+
+void scx_sub_disable(struct scx_sched *sch)
+{
+	struct scx_sched *parent = scx_parent(sch);
+	struct scx_task_iter sti;
+	struct task_struct *p;
+	int ret;
+
+	/*
+	 * Guarantee forward progress and wait for descendants to be disabled.
+	 * To limit disruptions, $parent is not bypassed. Tasks are fully
+	 * prepped and then inserted back into $parent.
+	 */
+	scx_bypass(sch, true);
+	drain_descendants(sch);
+
+	/*
+	 * Here, every runnable task is guaranteed to make forward progress and
+	 * we can safely use blocking synchronization constructs. Actually
+	 * disable ops.
+	 */
+	mutex_lock(&scx_enable_mutex);
+	percpu_down_write(&scx_fork_rwsem);
+	scx_cgroup_lock();
+
+	set_cgroup_sched(sch_cgroup(sch), parent);
+
+	scx_task_iter_start(&sti, sch->cgrp);
+	while ((p = scx_task_iter_next_locked(&sti))) {
+		struct rq *rq;
+		struct rq_flags rf;
+
+		/* filter out duplicate visits */
+		if (scx_task_on_sched(parent, p))
+			continue;
+
+		/*
+		 * By the time control reaches here, all descendant schedulers
+		 * should already have been disabled.
+		 */
+		WARN_ON_ONCE(!scx_task_on_sched(sch, p));
+
+		/*
+		 * @p is pinned by the iter: css_task_iter_next() takes a
+		 * reference and holds it until the next iter_next() call, so
+		 * @p->usage is guaranteed > 0.
+		 */
+		get_task_struct(p);
+
+		scx_task_iter_unlock(&sti);
+
+		/*
+		 * $p is READY or ENABLED on @sch. Initialize for $parent,
+		 * disable and exit from @sch, and then switch over to $parent.
+		 *
+		 * If a task fails to initialize for $parent, the only available
+		 * action is disabling $parent too. While this allows disabling
+		 * of a child sched to cause the parent scheduler to fail, the
+		 * failure can only originate from ops.init_task() of the
+		 * parent. A child can't directly affect the parent through its
+		 * own failures.
+		 */
+		ret = __scx_init_task(parent, p, false);
+		if (ret) {
+			scx_fail_parent(sch, p, ret);
+			put_task_struct(p);
+			break;
+		}
+
+		rq = task_rq_lock(p, &rf);
+
+		if (scx_get_task_state(p) == SCX_TASK_DEAD) {
+			/*
+			 * sched_ext_dead() raced us between __scx_init_task()
+			 * and this rq lock and ran exit_task() on @sch (the
+			 * sched @p was on at that point), not on $parent.
+			 * $parent's just-completed init is owed an exit_task()
+			 * and we issue it here.
+			 */
+			scx_sub_init_cancel_task(parent, p);
+			task_rq_unlock(rq, p, &rf);
+			put_task_struct(p);
+			continue;
+		}
+
+		scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
+			/*
+			 * $p is initialized for $parent and still attached to
+			 * @sch. Disable and exit for @sch, switch over to
+			 * $parent, override the state to READY to account for
+			 * $p having already been initialized, and then enable.
+			 */
+			scx_disable_and_exit_task(sch, p);
+			scx_set_task_state(p, SCX_TASK_INIT_BEGIN);
+			scx_set_task_state(p, SCX_TASK_INIT);
+			scx_set_task_sched(p, parent);
+			scx_set_task_state(p, SCX_TASK_READY);
+			scx_enable_task(parent, p);
+		}
+
+		task_rq_unlock(rq, p, &rf);
+		put_task_struct(p);
+	}
+	scx_task_iter_stop(&sti);
+
+	scx_disable_dump(sch);
+
+	scx_cgroup_unlock();
+	percpu_up_write(&scx_fork_rwsem);
+
+	/*
+	 * All tasks are moved off of @sch but there may still be on-going
+	 * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use
+	 * the expedited version as ancestors may be waiting in bypass mode.
+	 * Also, tell the parent that there is no need to keep running bypass
+	 * DSQs for us.
+	 */
+	synchronize_rcu_expedited();
+	scx_disable_bypass_dsp(sch);
+
+	scx_unlink_sched(sch);
+
+	mutex_unlock(&scx_enable_mutex);
+
+	/*
+	 * @sch is now unlinked from the parent's children list. Notify and call
+	 * ops.sub_detach/exit(). Note that ops.sub_detach/exit() must be called
+	 * after unlinking and releasing all locks. See scx_claim_exit().
+	 */
+	wake_up_all(&scx_unlink_waitq);
+
+	if (parent->ops.sub_detach && sch->sub_attached) {
+		struct scx_sub_detach_args sub_detach_args = {
+			.ops = &sch->ops,
+			.cgroup_path = sch->cgrp_path,
+		};
+		SCX_CALL_OP(parent, sub_detach, NULL,
+			    &sub_detach_args);
+	}
+
+	scx_log_sched_disable(sch);
+
+	if (sch->ops.exit)
+		SCX_CALL_OP(sch, exit, NULL, sch->exit_info);
+
+	/*
+	 * @sch's non-ops programs such as timers and tracers can fire after
+	 * ops.exit(). Now that exit is complete, stop scx_prog_sched() from
+	 * resolving to @sch and drain in-flight resolvers.
+	 */
+	WRITE_ONCE(sch->dead, true);
+	synchronize_rcu();
+
+	if (sch->sub_kset)
+		kobject_del(&sch->sub_kset->kobj);
+	/* not added if enable failed before scx_sched_sysfs_add() */
+	if (sch->kobj.state_in_sysfs)
+		kobject_del(&sch->kobj);
+}
+
+/* verify that a scheduler can be attached to @cgrp and return the parent */
+static struct scx_sched *find_parent_sched(struct cgroup *cgrp)
+{
+	struct scx_sched *parent = cgrp->scx_sched;
+	struct scx_sched *pos;
+
+	lockdep_assert_held(&scx_sched_lock);
+
+	/* can't attach twice to the same cgroup */
+	if (parent->cgrp == cgrp)
+		return ERR_PTR(-EBUSY);
+
+	/* does $parent allow sub-scheds? */
+	if (!parent->ops.sub_attach)
+		return ERR_PTR(-EOPNOTSUPP);
+
+	/* can't insert between $parent and its exiting children */
+	list_for_each_entry(pos, &parent->children, sibling)
+		if (cgroup_is_descendant(pos->cgrp, cgrp))
+			return ERR_PTR(-EBUSY);
+
+	return parent;
+}
+
+static bool assert_task_ready_or_enabled(struct task_struct *p)
+{
+	u32 state = scx_get_task_state(p);
+
+	switch (state) {
+	case SCX_TASK_READY:
+	case SCX_TASK_ENABLED:
+		return true;
+	default:
+		WARN_ONCE(true, "sched_ext: Invalid task state %d for %s[%d] during enabling sub sched",
+			  state, p->comm, p->pid);
+		return false;
+	}
+}
+
+void scx_sub_enable_workfn(struct kthread_work *work)
+{
+	struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work);
+	struct sched_ext_ops *ops = cmd->ops;
+	struct cgroup *cgrp;
+	struct scx_sched *parent, *sch;
+	struct scx_task_iter sti;
+	struct task_struct *p;
+	s32 i, ret;
+
+	mutex_lock(&scx_enable_mutex);
+
+	if (!scx_enabled()) {
+		ret = -ENODEV;
+		goto out_unlock;
+	}
+
+	/* See scx_root_enable_workfn() for the @ops->priv check. */
+	if (rcu_access_pointer(ops->priv)) {
+		ret = -EBUSY;
+		goto out_unlock;
+	}
+
+	cgrp = cgroup_get_from_id(ops->sub_cgroup_id);
+	if (IS_ERR(cgrp)) {
+		ret = PTR_ERR(cgrp);
+		goto out_unlock;
+	}
+
+	raw_spin_lock_irq(&scx_sched_lock);
+	parent = find_parent_sched(cgrp);
+	if (IS_ERR(parent)) {
+		raw_spin_unlock_irq(&scx_sched_lock);
+		ret = PTR_ERR(parent);
+		goto out_put_cgrp;
+	}
+	kobject_get(&parent->kobj);
+	raw_spin_unlock_irq(&scx_sched_lock);
+
+	/*
+	 * Flip the hot-path gates before ops->priv is published - the sub's
+	 * programs can e.g. kick cpus from that point on. The matching dec is
+	 * at the end of scx_sched_free_rcu_work().
+	 */
+	static_branch_inc(&__scx_has_subs);
+
+	/* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */
+	sch = scx_alloc_and_add_sched(cmd, cgrp, parent);
+	kobject_put(&parent->kobj);
+	if (IS_ERR(sch)) {
+		static_branch_dec(&__scx_has_subs);
+		ret = PTR_ERR(sch);
+		goto out_unlock;
+	}
+
+	/*
+	 * Validate before scx_link_sched() publishes @sch, so an invalid sub
+	 * never becomes visible with an unallocated pshard.
+	 */
+	ret = scx_validate_ops(sch, ops);
+	if (ret)
+		goto err_disable;
+
+	/*
+	 * Allocate pshard[] before scx_link_sched() publishes @sch into the
+	 * parent's RCU children list. A concurrent revoke walking the tree
+	 * would otherwise dereference sch->pshard[si] while it's still NULL.
+	 * Unlike the root path, the cid shard layout is stable at this point.
+	 *
+	 * scx_alloc_pshards() skips allocation when @sch's arena pool isn't
+	 * initialized, so scx_arena_pool_init() must run first.
+	 */
+	ret = scx_arena_pool_init(sch);
+	if (ret)
+		goto err_disable;
+
+	ret = scx_alloc_pshards(sch);
+	if (ret)
+		goto err_disable;
+
+	ret = scx_link_sched(sch);
+	if (ret)
+		goto err_disable;
+
+	ret = scx_sched_sysfs_add(sch);
+	if (ret)
+		goto err_disable;
+
+	if (sch->level >= SCX_SUB_MAX_DEPTH) {
+		scx_error(sch, "max nesting depth %d violated",
+			  SCX_SUB_MAX_DEPTH);
+		goto err_disable;
+	}
+
+	if (sch->ops.init) {
+		ret = SCX_CALL_OP_RET(sch, init, NULL);
+		if (ret) {
+			ret = scx_ops_sanitize_err(sch, "init", ret);
+			scx_error(sch, "ops.init() failed (%d)", ret);
+			goto err_disable;
+		}
+		sch->exit_info->flags |= SCX_EFLAG_INITIALIZED;
+	}
+
+	ret = scx_set_cmask_scratch_alloc(sch);
+	if (ret)
+		goto err_disable;
+
+	struct scx_sub_attach_args sub_attach_args = {
+		.ops = &sch->ops,
+		.cgroup_path = sch->cgrp_path,
+	};
+
+	ret = SCX_CALL_OP_RET(parent, sub_attach, NULL,
+			      &sub_attach_args);
+	if (ret) {
+		ret = scx_ops_sanitize_err(sch, "sub_attach", ret);
+		scx_error(sch, "parent rejected (%d)", ret);
+		goto err_disable;
+	}
+	sch->sub_attached = true;
+
+	scx_bypass(sch, true);
+
+	for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++)
+		if (((void (**)(void))ops)[i])
+			set_bit(i, sch->has_op);
+
+	percpu_down_write(&scx_fork_rwsem);
+	scx_cgroup_lock();
+
+	/*
+	 * Set cgroup->scx_sched's and check CSS_ONLINE. Either we see
+	 * !CSS_ONLINE or scx_cgroup_lifetime_notify() sees and shoots us down.
+	 */
+	set_cgroup_sched(sch_cgroup(sch), sch);
+	if (!(cgrp->self.flags & CSS_ONLINE)) {
+		scx_error(sch, "cgroup is not online");
+		goto err_unlock_and_disable;
+	}
+
+	/*
+	 * Initialize tasks for the new child $sch without exiting them for
+	 * $parent so that the tasks can always be reverted back to $parent
+	 * sched on child init failure.
+	 */
+	WARN_ON_ONCE(scx_enabling_sub_sched);
+	scx_enabling_sub_sched = sch;
+
+	scx_task_iter_start(&sti, sch->cgrp);
+	while ((p = scx_task_iter_next_locked(&sti))) {
+		struct rq *rq;
+		struct rq_flags rf;
+
+		/*
+		 * Task iteration may visit the same task twice when racing
+		 * against exiting. Use %SCX_TASK_SUB_INIT to mark tasks which
+		 * finished __scx_init_task() and skip if set.
+		 *
+		 * A task may exit and get freed between __scx_init_task()
+		 * completion and scx_enable_task(). In such cases,
+		 * scx_disable_and_exit_task() must exit the task for both the
+		 * parent and child scheds.
+		 */
+		if (p->scx.flags & SCX_TASK_SUB_INIT)
+			continue;
+
+		/* @p is pinned by the iter; see scx_sub_disable() */
+		get_task_struct(p);
+
+		if (!assert_task_ready_or_enabled(p)) {
+			ret = -EINVAL;
+			goto abort;
+		}
+
+		scx_task_iter_unlock(&sti);
+
+		/*
+		 * As $p is still on $parent, it can't be transitioned to INIT.
+		 * Let's worry about task state later. Use __scx_init_task().
+		 */
+		ret = __scx_init_task(sch, p, false);
+		if (ret)
+			goto abort;
+
+		rq = task_rq_lock(p, &rf);
+
+		if (scx_get_task_state(p) == SCX_TASK_DEAD) {
+			/*
+			 * sched_ext_dead() raced us between __scx_init_task()
+			 * and this rq lock and ran exit_task() on $parent (the
+			 * sched @p was on at that point), not on @sch. @sch's
+			 * just-completed init is owed an exit_task() and we
+			 * issue it here.
+			 */
+			scx_sub_init_cancel_task(sch, p);
+			task_rq_unlock(rq, p, &rf);
+			put_task_struct(p);
+			continue;
+		}
+
+		p->scx.flags |= SCX_TASK_SUB_INIT;
+		task_rq_unlock(rq, p, &rf);
+
+		put_task_struct(p);
+	}
+	scx_task_iter_stop(&sti);
+
+	/*
+	 * All tasks are prepped. Disable/exit tasks for $parent and enable for
+	 * the new @sch.
+	 */
+	scx_task_iter_start(&sti, sch->cgrp);
+	while ((p = scx_task_iter_next_locked(&sti))) {
+		/*
+		 * Use clearing of %SCX_TASK_SUB_INIT to detect and skip
+		 * duplicate iterations.
+		 */
+		if (!(p->scx.flags & SCX_TASK_SUB_INIT))
+			continue;
+
+		scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
+			/*
+			 * $p must be either READY or ENABLED. If ENABLED,
+			 * __scx_disabled_and_exit_task() first disables and
+			 * makes it READY. However, after exiting $p, it will
+			 * leave $p as READY.
+			 */
+			assert_task_ready_or_enabled(p);
+			__scx_disable_and_exit_task(parent, p);
+
+			/*
+			 * $p is now only initialized for @sch and READY, which
+			 * is what we want. Assign it to @sch and enable.
+			 */
+			scx_set_task_sched(p, sch);
+			scx_enable_task(sch, p);
+
+			p->scx.flags &= ~SCX_TASK_SUB_INIT;
+		}
+	}
+	scx_task_iter_stop(&sti);
+
+	scx_enabling_sub_sched = NULL;
+
+	scx_cgroup_unlock();
+	percpu_up_write(&scx_fork_rwsem);
+
+	scx_bypass(sch, false);
+
+	/* @sch is enabled; deliver any caps owed since its sub_attach() */
+	scx_sub_seed_caps(sch);
+
+	pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name);
+	kobject_uevent(&sch->kobj, KOBJ_ADD);
+	ret = 0;
+	goto out_unlock;
+
+out_put_cgrp:
+	cgroup_put(cgrp);
+out_unlock:
+	mutex_unlock(&scx_enable_mutex);
+	cmd->ret = ret;
+	return;
+
+abort:
+	put_task_struct(p);
+	scx_task_iter_stop(&sti);
+
+	/*
+	 * Undo __scx_init_task() for tasks we marked. scx_enable_task() never
+	 * ran for @sch on them, so calling scx_disable_task() here would invoke
+	 * ops.disable() without a matching ops.enable(). scx_enabling_sub_sched
+	 * must stay set until SUB_INIT is cleared from every marked task -
+	 * scx_disable_and_exit_task() reads it when a task exits concurrently.
+	 */
+	scx_task_iter_start(&sti, sch->cgrp);
+	while ((p = scx_task_iter_next_locked(&sti))) {
+		if (p->scx.flags & SCX_TASK_SUB_INIT) {
+			scx_sub_init_cancel_task(sch, p);
+			p->scx.flags &= ~SCX_TASK_SUB_INIT;
+		}
+	}
+	scx_task_iter_stop(&sti);
+	scx_enabling_sub_sched = NULL;
+err_unlock_and_disable:
+	/* we'll soon enter disable path, keep bypass on */
+	scx_cgroup_unlock();
+	percpu_up_write(&scx_fork_rwsem);
+err_disable:
+	mutex_unlock(&scx_enable_mutex);
+	/*
+	 * Some enable failures only return an errno (e.g. -ENOMEM from an
+	 * allocation) without calling scx_error(). Record it so
+	 * scx_flush_disable_work() runs the disable and ops.exit() fires.
+	 */
+	scx_error(sch, "scx_sub_enable() failed (%d)", ret);
+	scx_flush_disable_work(sch);
+	cmd->ret = 0;
+}
+
+static s32 scx_cgroup_lifetime_notify(struct notifier_block *nb,
+				      unsigned long action, void *data)
+{
+	struct cgroup *cgrp = data;
+	struct cgroup *parent = cgroup_parent(cgrp);
+
+	if (!cgroup_on_dfl(cgrp))
+		return NOTIFY_OK;
+
+	switch (action) {
+	case CGROUP_LIFETIME_ONLINE:
+		/* inherit ->scx_sched from $parent */
+		if (parent)
+			rcu_assign_pointer(cgrp->scx_sched, parent->scx_sched);
+		break;
+	case CGROUP_LIFETIME_OFFLINE:
+		/* if there is a sched attached, shoot it down */
+		if (cgrp->scx_sched && cgrp->scx_sched->cgrp == cgrp)
+			scx_exit(cgrp->scx_sched, SCX_EXIT_UNREG_KERN,
+				 SCX_ECODE_RSN_CGROUP_OFFLINE,
+				 "cgroup %llu going offline", cgroup_id(cgrp));
+		break;
+	}
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block scx_cgroup_lifetime_nb = {
+	.notifier_call = scx_cgroup_lifetime_notify,
+};
+
+static s32 __init scx_cgroup_lifetime_notifier_init(void)
+{
+	return blocking_notifier_chain_register(&cgroup_lifetime_notifier,
+						&scx_cgroup_lifetime_nb);
+}
+core_initcall(scx_cgroup_lifetime_notifier_init);
+
+static void scx_pstack_recursion(struct bpf_prog *prog, const char *op)
+{
+	struct scx_sched *sch;
+
+	guard(rcu)();
+	sch = scx_prog_sched(prog->aux);
+	if (unlikely(!sch))
+		return;
+
+	scx_error(sch, "%s recursion detected", op);
+}
+
+void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog)
+{
+	scx_pstack_recursion(prog, "dispatch");
+}
+
+void scx_pstack_recursion_on_caps_updated(struct bpf_prog *prog)
+{
+	scx_pstack_recursion(prog, "sub_caps_updated");
+}
+
+__bpf_kfunc_start_defs();
+
+/**
+ * scx_bpf_sub_dispatch - Trigger dispatching on a child scheduler
+ * @cgroup_id: cgroup ID of the child scheduler to dispatch
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * Allows a parent scheduler to trigger dispatching on one of its direct
+ * child schedulers. The child scheduler runs its dispatch operation to
+ * move tasks from dispatch queues to the local runqueue.
+ *
+ * Returns: true on success, false if cgroup_id is invalid, not a direct
+ * child, or caller lacks dispatch permission.
+ */
+__bpf_kfunc bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux)
+{
+	struct rq *this_rq = this_rq();
+	struct scx_sched *parent, *child;
+
+	guard(rcu)();
+	parent = scx_prog_sched(aux);
+	if (unlikely(!parent))
+		return false;
+
+	child = scx_find_sub_sched(cgroup_id);
+
+	if (unlikely(!child))
+		return false;
+
+	if (unlikely(scx_parent(child) != parent)) {
+		scx_error(parent, "trying to dispatch a distant sub-sched on cgroup %llu",
+			  cgroup_id);
+		return false;
+	}
+
+	/*
+	 * Skip a child that does not effectively hold the base cap on this cpu:
+	 * its inserts would only be rejected. ecaps are synced at the top of
+	 * balance_one() before dispatch, so this reflects the in-effect state.
+	 */
+	if (scx_missing_caps(child, cpu_of(this_rq), SCX_CAP_BASE))
+		return false;
+
+	return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev,
+				  true);
+}
+
+/* Validate common inputs. On success, *parent_out and *child_out are set. */
+static s32 sub_cap_preamble(u64 cgroup_id, u64 caps, const struct bpf_prog_aux *aux,
+			    struct scx_sched **parent_out, struct scx_sched **child_out)
+{
+	struct scx_sched *parent, *child;
+
+	parent = scx_prog_sched(aux);
+	if (unlikely(!parent))
+		return -ENODEV;
+
+	if (!scx_is_cid_type()) {
+		scx_error(parent, "sub-cap kfuncs require a cid-form scheduler");
+		return -EOPNOTSUPP;
+	}
+
+	child = scx_find_sub_sched(cgroup_id);
+	if (unlikely(!child))
+		return -ENODEV;
+
+	if (unlikely(scx_parent(child) != parent)) {
+		scx_error(parent, "%s: sub-%llu is not a direct child",
+			  parent->cgrp_path, cgroup_id);
+		return -EINVAL;
+	}
+
+	if (unlikely(caps & ~__SCX_CAP_ALL)) {
+		scx_error(parent, "invalid caps 0x%llx", caps);
+		return -EINVAL;
+	}
+
+	*parent_out = parent;
+	*child_out = child;
+	return 0;
+}
+
+/**
+ * scx_bpf_sub_grant - Grant @caps on @cmask__ign's cids to a direct child
+ * @cgroup_id: cgroup id of the direct child sub-sched
+ * @caps: bitmask of SCX_CAP_* to grant
+ * @cmask__ign: cid cmask to grant @caps on (arena pointer)
+ * @denied_out__ign: optional arena cmask accumulating refused cids
+ * @aux: implicit BPF argument
+ *
+ * A cid in @cmask__ign is granted to the child only if the parent holds every
+ * requested cap on it. Refused cids are OR'd into @denied_out__ign when
+ * provided. Refusals outside @denied_out__ign's range are not recorded.
+ *
+ * All-or-nothing keeps the caller-visible result binary per cid, so
+ * @denied_out__ign is one mask to interpret rather than a per-cap matrix.
+ *
+ * Return 0 on full success, -EPERM if any cid was refused, or a negative
+ * errno on other failures.
+ */
+__bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps,
+				  const struct scx_cmask *cmask__ign,
+				  struct scx_cmask *denied_out__ign,
+				  const struct bpf_prog_aux *aux)
+{
+	struct scx_cmask_ref ref, denied_ref;
+	struct scx_sched *parent, *child;
+	bool any_denied = false;
+	LIST_HEAD(to_deliver);
+	s32 si, ret;
+
+	guard(irqsave)();
+
+	ret = sub_cap_preamble(cgroup_id, caps, aux, &parent, &child);
+	if (ret)
+		return ret;
+
+	ret = scx_cmask_ref_init(parent, cmask__ign, &ref);
+	if (ret) {
+		scx_error(parent, "invalid cmask (%d)", ret);
+		return ret;
+	}
+
+	if (denied_out__ign) {
+		ret = scx_cmask_ref_init(parent, denied_out__ign, &denied_ref);
+		if (ret) {
+			scx_error(parent, "invalid denied_out (%d)", ret);
+			return ret;
+		}
+	}
+
+	/* apply the grant one shard at a time */
+	for (si = ref.shard_first; si < ref.shard_end; si++) {
+		SCX_CMASK_DEFINE_SHARD(slice, 0, SCX_CID_SHARD_MAX_CPUS);
+		struct scx_pshard *pps = parent->pshard[si];
+		struct scx_pshard *cps = child->pshard[si];
+		u64 granted_caps = 0;
+		u32 cap_bit;
+
+		scx_cmask_ref_shard(&ref, si, slice);
+		if (scx_cmask_empty(slice))
+			continue;
+
+		SCX_CMASK_DEFINE_SHARD(granted_cids, slice->base, slice->nr_cids);
+		SCX_CMASK_DEFINE_SHARD(changed_cids, slice->base, slice->nr_cids);
+		SCX_CMASK_DEFINE_SHARD(delta, slice->base, slice->nr_cids);
+
+		scx_cmask_copy(granted_cids, slice);
+
+		scoped_guard (raw_spinlock, &pps->lock) {
+			guard(raw_spinlock_nested)(&cps->lock);
+
+			/*
+			 * Narrow granted_cids to cids the parent holds every
+			 * requested cap on. All-or-nothing per cid.
+			 */
+			scx_for_each_cap_bit(cap_bit, caps)
+				scx_cmask_and(granted_cids, &pps->caps[cap_bit].cmask);
+
+			/*
+			 * For each requested cap, fold the newly-set cids into
+			 * the child and accumulate the delta.
+			 */
+			scx_for_each_cap_bit(cap_bit, caps) {
+				struct scx_cmask *ccm = &cps->caps[cap_bit].cmask;
+
+				scx_cmask_copy(delta, granted_cids);
+				scx_cmask_andnot(delta, ccm);
+				if (scx_cmask_empty(delta))
+					continue;
+
+				scx_cmask_or(ccm, delta);
+				scx_cmask_or(changed_cids, delta);
+				granted_caps |= BIT_U64(cap_bit);
+			}
+
+			if (granted_caps) {
+				s32 cid;
+
+				caps_updated_record(cps, changed_cids, granted_caps,
+						    &to_deliver);
+				/*
+				 * The sync arms an update_idle() re-notify if
+				 * the cid gains baseline access, so the holder
+				 * learns of an already-idle cid.
+				 */
+				scx_cmask_for_each_cid(cid, changed_cids)
+					queue_sync_ecaps(child, cid);
+			}
+		}
+
+		/* record cids that didn't make it through into @denied_out */
+		if (!scx_cmask_subset(slice, granted_cids)) {
+			any_denied = true;
+			if (denied_out__ign) {
+				SCX_CMASK_DEFINE_SHARD(denied, slice->base, slice->nr_cids);
+
+				scx_cmask_copy(denied, slice);
+				scx_cmask_andnot(denied, granted_cids);
+				scx_cmask_ref_or(&denied_ref, denied);
+			}
+		}
+	}
+
+	caps_updated_deliver(&to_deliver);
+
+	return any_denied ? -EPERM : 0;
+}
+
+/**
+ * scx_bpf_sub_revoke - Revoke @caps on @cmask__ign's cids from @child
+ * @cgroup_id: cgroup id of the direct child sub-sched
+ * @caps: bitmask of SCX_CAP_* to revoke
+ * @cmask__ign: cid cmask to revoke @caps on (arena pointer)
+ * @aux: implicit BPF argument
+ *
+ * Clear @caps bits on @cmask__ign from the child named by @cgroup_id and all
+ * its descendants. The origin parent's pshard lock is held across the subtree
+ * walk so a concurrent grant from the origin parent observes the revoked
+ * state.
+ */
+__bpf_kfunc void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps,
+				    const struct scx_cmask *cmask__ign,
+				    const struct bpf_prog_aux *aux)
+{
+	struct scx_cmask_ref ref;
+	struct scx_sched *parent, *child, *pos;
+	LIST_HEAD(to_deliver);
+	s32 si, ret;
+
+	guard(irqsave)();
+
+	if (sub_cap_preamble(cgroup_id, caps, aux, &parent, &child))
+		return;
+
+	ret = scx_cmask_ref_init(parent, cmask__ign, &ref);
+	if (ret) {
+		scx_error(parent, "invalid cmask (%d)", ret);
+		return;
+	}
+
+	/* per-shard, walk child's subtree and clear @caps */
+	for (si = ref.shard_first; si < ref.shard_end; si++) {
+		SCX_CMASK_DEFINE_SHARD(slice, 0, SCX_CID_SHARD_MAX_CPUS);
+
+		scx_cmask_ref_shard(&ref, si, slice);
+		if (scx_cmask_empty(slice))
+			continue;
+
+		/*
+		 * Pre-order with subtree skip: a descendant that cleared
+		 * nothing means no descendant of it can hold @caps on these
+		 * cids either.
+		 */
+		guard(raw_spinlock)(&parent->pshard[si]->lock);
+		pos = scx_next_descendant_pre(NULL, child);
+		while (pos) {
+			struct scx_pshard *ps = pos->pshard[si];
+			SCX_CMASK_DEFINE_SHARD(changed_cids, slice->base, slice->nr_cids);
+			SCX_CMASK_DEFINE_SHARD(delta, slice->base, slice->nr_cids);
+			u64 revoked_caps = 0;
+			u32 cap_bit;
+
+			scoped_guard (raw_spinlock_nested, &ps->lock) {
+				/*
+				 * For each cap, clear lost cids and accumulate
+				 * the per-cap diff for notification.
+				 */
+				scx_for_each_cap_bit(cap_bit, caps) {
+					struct scx_cmask *cm = &ps->caps[cap_bit].cmask;
+
+					scx_cmask_copy(delta, cm);
+					scx_cmask_and(delta, slice);
+					if (scx_cmask_empty(delta))
+						continue;
+
+					scx_cmask_andnot(cm, delta);
+					scx_cmask_or(changed_cids, delta);
+					revoked_caps |= BIT_U64(cap_bit);
+				}
+
+				if (revoked_caps) {
+					s32 cid;
+
+					caps_updated_record(ps, changed_cids, revoked_caps,
+							    &to_deliver);
+					scx_cmask_for_each_cid(cid, changed_cids)
+						queue_sync_ecaps(pos, cid);
+				}
+			}
+
+			if (revoked_caps)
+				pos = scx_next_descendant_pre(pos, child);
+			else
+				pos = scx_skip_subtree_pre(pos, child);
+		}
+	}
+
+	caps_updated_deliver(&to_deliver);
+}
+
+/**
+ * scx_bpf_sub_caps - Read self's or a direct child's cap cmasks
+ * @cgroup_id: 0 for self, or a direct child's cgroup id
+ * @caps: one or more SCX_CAP_* bits
+ * @out__ign: arena cmask to receive the union of @caps within its range
+ * @aux: implicit BPF argument
+ *
+ * Read the cap cmasks granted on each cid for self (@cgroup_id 0) or a direct
+ * child - the literal granted set. A sched can read only itself or a direct
+ * child.
+ *
+ * Return 0, -ENODEV if @cgroup_id names no direct child, or -EINVAL on bad
+ * inputs.
+ */
+__bpf_kfunc s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out__ign,
+				 const struct bpf_prog_aux *aux)
+{
+	struct scx_cmask_ref ref;
+	struct scx_sched *sch, *target;
+	struct scx_pshard **pshard;
+	s32 si, ret;
+
+	guard(irqsave)();
+
+	sch = scx_prog_sched(aux);
+	if (unlikely(!sch))
+		return -ENODEV;
+
+	if (!scx_is_cid_type()) {
+		scx_error(sch, "sub-cap kfuncs require a cid-form scheduler");
+		return -EOPNOTSUPP;
+	}
+
+	if (unlikely(caps & ~__SCX_CAP_ALL)) {
+		scx_error(sch, "invalid caps 0x%llx", caps);
+		return -EINVAL;
+	}
+
+	/* @cgroup_id 0 reads self, otherwise a direct child */
+	if (cgroup_id) {
+		target = scx_find_sub_sched(cgroup_id);
+		if (unlikely(!target))
+			return -ENODEV;
+		if (unlikely(scx_parent(target) != sch)) {
+			scx_error(sch, "%s: sub-%llu is not a direct child",
+				  sch->cgrp_path, cgroup_id);
+			return -EINVAL;
+		}
+	} else {
+		target = sch;
+	}
+
+	/*
+	 * The target's caps storage may not be set up yet (e.g. a self-read
+	 * during ops.init_cids()). Pairs with the publish in
+	 * scx_alloc_pshards(): a non-NULL pshard has every element set.
+	 */
+	pshard = READ_ONCE(target->pshard);
+	if (unlikely(!pshard)) {
+		scx_error(sch, "scx_bpf_sub_caps() called before caps storage is initialized");
+		return -ENODEV;
+	}
+
+	ret = scx_cmask_ref_init(sch, out__ign, &ref);
+	if (ret) {
+		scx_error(sch, "invalid out (%d)", ret);
+		return ret;
+	}
+
+	for (si = ref.shard_first; si < ref.shard_end; si++) {
+		const struct scx_cid_shard *shard = &scx_cid_shard_ranges[si];
+		SCX_CMASK_DEFINE_SHARD(local_out, shard->base_cid, shard->nr_cids);
+		u32 cap_bit;
+
+		scx_for_each_cap_bit(cap_bit, caps)
+			scx_cmask_or(local_out, &pshard[si]->caps[cap_bit].cmask);
+		scx_cmask_ref_copy(&ref, local_out);
+	}
+	return 0;
+}
+
+/**
+ * scx_bpf_sub_kill_bstr - Kill a direct child sub-scheduler
+ * @cgroup_id: cgroup id of the direct child to kill
+ * @fmt: reason message format string
+ * @data: format string parameters packaged using ___bpf_fill() macro
+ * @data__sz: @data len, must end in '__sz' for the verifier
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * Evict a direct child sub-scheduler, disabling it with the supplied reason.
+ * The child and its subtree are torn down asynchronously through the usual
+ * disable path.
+ *
+ * Unlike scx_bpf_exit(), no exit code is taken: the child is a separate
+ * scheduler with its own exit-code semantics, so a code chosen by the parent
+ * would have no defined meaning. The reason string carries the intent.
+ *
+ * Return 0 on success or -ENODEV if @cgroup_id names no sub-scheduler, which
+ * can race with the child detaching on its own and so is not a scheduler error.
+ * Naming a sched that exists but is not a direct child aborts the parent.
+ */
+__printf(2, 0)
+__bpf_kfunc s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt,
+				      unsigned long long *data, u32 data__sz,
+				      const struct bpf_prog_aux *aux)
+{
+	struct scx_sched *parent, *child;
+	s32 ret;
+
+	guard(rcu)();
+
+	parent = scx_prog_sched(aux);
+	if (unlikely(!parent))
+		return -ENODEV;
+
+	if (!scx_is_cid_type()) {
+		scx_error(parent, "sub-cap kfuncs require a cid-form scheduler");
+		return -EOPNOTSUPP;
+	}
+
+	child = scx_find_sub_sched(cgroup_id);
+	if (unlikely(!child))
+		return -ENODEV;
+
+	if (unlikely(scx_parent(child) != parent)) {
+		scx_error(parent, "%s: sub-%llu is not a direct child",
+			  parent->cgrp_path, cgroup_id);
+		return -EINVAL;
+	}
+
+	guard(raw_spinlock_irqsave)(&scx_exit_bstr_buf_lock);
+	ret = scx_bstr_format(parent, &scx_exit_bstr_buf, fmt, data, data__sz);
+	if (ret < 0)
+		return ret;
+	scx_exit(child, SCX_EXIT_PARENT_KILL, 0, "%s", scx_exit_bstr_buf.line);
+	return 0;
+}
+
+__bpf_kfunc_end_defs();
+
+#endif	/* CONFIG_EXT_SUB_SCHED */
diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h
new file mode 100644
index 0000000000000..625d7ce334aa8
--- /dev/null
+++ b/kernel/sched/ext/sub.h
@@ -0,0 +1,189 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
+ *
+ * Sub-scheduler hierarchy support.
+ *
+ * Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2026 Tejun Heo <tj@kernel.org>
+ */
+#ifndef _KERNEL_SCHED_EXT_SUB_H
+#define _KERNEL_SCHED_EXT_SUB_H
+
+#include "internal.h"
+
+#ifdef CONFIG_EXT_SUB_SCHED
+
+struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root);
+struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root);
+void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch);
+struct cgroup *sch_cgroup(struct scx_sched *sch);
+void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch);
+void scx_pstack_recursion_on_dispatch(struct bpf_prog *prog);
+void scx_pstack_recursion_on_caps_updated(struct bpf_prog *prog);
+void drain_descendants(struct scx_sched *sch);
+void scx_sub_disable(struct scx_sched *sch);
+void scx_sub_enable_workfn(struct kthread_work *work);
+bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux);
+void scx_free_pshards(struct scx_sched *sch);
+s32 scx_alloc_pshards(struct scx_sched *sch);
+void scx_init_root_caps(struct scx_sched *sch);
+void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev);
+void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch);
+void scx_online_ecaps(struct rq *rq);
+void scx_offline_ecaps(struct rq *rq);
+void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu);
+void scx_discard_stale_ecaps_syncs(void);
+struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq,
+					       struct task_struct *p, u64 *enq_flags);
+bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p);
+void scx_reenq_reject(struct rq *rq);
+
+static inline const char *sch_cgrp_path(struct scx_sched *sch)
+{
+	return sch->cgrp_path;
+}
+
+/* a dying sub's hot-path influence ends in scx_sched_free_rcu_work() */
+static inline void scx_dec_has_subs(struct scx_sched *sch)
+{
+	if (sch->level)
+		static_branch_dec(&__scx_has_subs);
+}
+
+#else	/* CONFIG_EXT_SUB_SCHED */
+
+static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }
+static inline struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root) { return NULL; }
+static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {}
+static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; }
+static inline const char *sch_cgrp_path(struct scx_sched *sch) { return "/"; }
+static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {}
+static inline void drain_descendants(struct scx_sched *sch) { }
+static inline void scx_sub_disable(struct scx_sched *sch) { }
+static inline void scx_free_pshards(struct scx_sched *sch) {}
+static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; }
+static inline void scx_init_root_caps(struct scx_sched *sch) {}
+static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) {}
+static inline void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) {}
+static inline void scx_online_ecaps(struct rq *rq) {}
+static inline void scx_offline_ecaps(struct rq *rq) {}
+static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {}
+static inline void scx_discard_stale_ecaps_syncs(void) {}
+static inline struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; }
+static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; }
+static inline void scx_reenq_reject(struct rq *rq) {}
+static inline void scx_dec_has_subs(struct scx_sched *sch) {}
+
+#endif	/* CONFIG_EXT_SUB_SCHED */
+
+/**
+ * scx_for_each_descendant_pre - pre-order walk of a sched's descendants
+ * @pos: iteration cursor
+ * @root: sched to walk the descendants of
+ *
+ * Walk @root's descendants. @root is included in the iteration and the first
+ * node to be visited. Must be called with scx_enable_mutex, scx_sched_lock, or
+ * RCU read lock.
+ */
+#define scx_for_each_descendant_pre(pos, root)					\
+	for ((pos) = scx_next_descendant_pre(NULL, (root)); (pos);		\
+	     (pos) = scx_next_descendant_pre((pos), (root)))
+
+#ifdef CONFIG_EXT_SUB_SCHED
+
+/**
+ * scx_missing_caps - The caps in @needed that @sch lacks on @cpu
+ * @sch: sched to test
+ * @cpu: cpu to test on
+ * @needed: bitmask of SCX_CAP_* values
+ *
+ * Return the caps in @needed that @sch lacks for @cpu, 0 if it holds them all.
+ */
+static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed)
+{
+	u64 ecaps;
+
+	/* no sub-scheds, no missing caps */
+	if (!scx_has_subs())
+		return 0;
+
+	/* root holds every cap on every cpu */
+	if (!sch->level)
+		return 0;
+
+	ecaps = READ_ONCE(per_cpu_ptr(sch->pcpu, cpu)->ecaps);
+
+	return needed & ~ecaps;
+}
+
+/*
+ * Cap semantics: which caps an action requires, and which caps a cap implies.
+ * Keep all such mappings collected here.
+ */
+
+/* map @enq_flags to the SCX_CAP_* bit required for the local-DSQ insert */
+static inline u64 scx_caps_for_enq(u64 enq_flags)
+{
+	/* a restored task must be put into the local DSQ regardless of caps */
+	if (enq_flags & SCX_ENQ_IGNORE_CAPS)
+		return 0;
+	if (enq_flags & SCX_ENQ_IMMED)
+		return SCX_CAP_ENQ_IMMED;
+	return SCX_CAP_ENQ;
+}
+
+/* map queued @p to the SCX_CAP_* bit required to stay on its local DSQ */
+static inline u64 scx_caps_for_task(struct task_struct *p)
+{
+	if (p->scx.flags & SCX_TASK_IMMED)
+		return SCX_CAP_ENQ_IMMED;
+	return SCX_CAP_ENQ;
+}
+
+/* the cap @sch needs to preempt @rq's current task, 0 if none */
+static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq)
+{
+	struct task_struct *curr = rq->curr;
+
+	/* a non-ext task can't be preempted by ext, own-subtree needs no cap */
+	if (curr->sched_class != &ext_sched_class ||
+	    scx_is_descendant(scx_task_sched(curr), sch))
+		return 0;
+	return SCX_CAP_PREEMPT;
+}
+
+/* caps implied by holding @cap */
+static inline u64 scx_caps_implied(u64 cap)
+{
+	switch (cap) {
+	case SCX_CAP_PREEMPT:
+		return SCX_CAP_ENQ | SCX_CAP_ENQ_IMMED;
+	case SCX_CAP_ENQ:
+		return SCX_CAP_ENQ_IMMED;
+	}
+	return 0;
+}
+
+/* may @p keep running on @rq's cpu? requires baseline cpu access */
+static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p)
+{
+	if (!scx_has_subs())
+		return true;
+
+	/* a migration-disabled task is let in without caps, keep it likewise */
+	if (unlikely(is_migration_disabled(p)))
+		return true;
+
+	return likely(!scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE));
+}
+
+#else	/* CONFIG_EXT_SUB_SCHED */
+
+static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; }
+static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq) { return 0; }
+static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { return true; }
+
+#endif	/* CONFIG_EXT_SUB_SCHED */
+
+#endif /* _KERNEL_SCHED_EXT_SUB_H */
diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h
index bc74eafd43f12..a1a05820725e9 100644
--- a/kernel/sched/ext/types.h
+++ b/kernel/sched/ext/types.h
@@ -47,11 +47,16 @@ enum scx_consts {
 };
 
 /*
- * Per-cid topology info. For each topology level (core, LLC, node), records
- * the first cid in the unit and its global index. Global indices are
- * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges
- * over [0, nr_cores_at_init) with no gaps. No-topo cids have all fields set
- * to -1.
+ * Per-cid topology info. For each topology level (core, LLC, node) and shard,
+ * records the first cid in the unit and its global index. Global indices are
+ * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges over
+ * [0, nr_cores_at_init) with no gaps. No-topo cids have core/LLC/node fields
+ * set to -1 but always have valid shard assignments.
+ *
+ * Shards are contiguous CID ranges used as scalable locking/work domains for
+ * sub-scheduler operations. By default each LLC becomes one shard, split into
+ * smaller shards if the LLC exceeds the target size. No-topo cids are packed
+ * into their own max-sized shards.
  *
  * @core_cid: first cid of this cid's core (smt-sibling group)
  * @core_idx: global index of that core, in [0, nr_cores_at_init)
@@ -59,6 +64,8 @@ enum scx_consts {
  * @llc_idx: global index of that LLC, in [0, nr_llcs_at_init)
  * @node_cid: first cid of this cid's NUMA node
  * @node_idx: global index of that node, in [0, nr_nodes_at_init)
+ * @shard_cid: first cid of this cid's shard
+ * @shard_idx: global index of that shard, in [0, scx_nr_cid_shards)
  */
 struct scx_cid_topo {
 	s32 core_cid;
@@ -67,6 +74,24 @@ struct scx_cid_topo {
 	s32 llc_idx;
 	s32 node_cid;
 	s32 node_idx;
+	s32 shard_cid;
+	s32 shard_idx;
+};
+
+enum scx_cid_consts {
+	SCX_CID_SHARD_SIZE_DFL		= 24,
+	SCX_CID_SHARD_MAX_CPUS		= 512,
+};
+
+/*
+ * Per-shard metadata for O(1) shard->cid-range lookup.
+ *
+ * @base_cid: first cid of the shard
+ * @nr_cids: number of cids in the shard
+ */
+struct scx_cid_shard {
+	s32			base_cid;
+	s32			nr_cids;
 };
 
 /*
@@ -91,7 +116,7 @@ struct scx_cmask {
 	u32 base;
 	u32 nr_cids;
 	u32 alloc_words;
-	u64 bits[] __counted_by(alloc_words);
+	u64 bits[];
 };
 
 /*
@@ -147,4 +172,41 @@ struct scx_cmask {
 #define SCX_CMASK_DEFINE_SHARD(NAME, BASE, NR_CIDS)				\
 	__SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, SCX_CID_SHARD_MAX_CPUS)
 
+/*
+ * scx_cmask_ref: validated reference to a BPF-arena cmask.
+ *
+ * scx_cmask_ref_init() normalizes the pointer into the arena and snapshots
+ * @base/@nr_cids. The snapshot is what downstream code uses for sizing - the
+ * live header can be mutated concurrently by BPF.
+ *
+ * scx_cmask_ref_shard() reads one shard into a cmask. scx_cmask_ref_or() and
+ * scx_cmask_ref_copy() write back into the referenced arena cmask, bounded by
+ * the snapshot.
+ *
+ * Typical input use:
+ *
+ *	struct scx_cmask_ref ref;
+ *	SCX_CMASK_DEFINE(shard, 0, SCX_CID_SHARD_MAX_CPUS);
+ *	s32 idx, ret;
+ *
+ *	ret = scx_cmask_ref_init(sch, src, &ref);
+ *	if (ret < 0)
+ *		return ret;
+ *
+ *	for (idx = ref.shard_first; idx < ref.shard_end; idx++) {
+ *		scx_cmask_ref_shard(&ref, idx, shard);
+ *		if (!shard->nr_cids)
+ *			continue;
+ *		... use idx and shard ...
+ *	}
+ */
+struct scx_cmask_ref {
+	struct scx_sched	*sch;
+	struct scx_cmask	*src;
+	u32			base;
+	u32			nr_cids;
+	s32			shard_first;
+	s32			shard_end;
+};
+
 #endif /* _KERNEL_SCHED_EXT_TYPES_H */
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index d78467ec6ee13..37001c63452e5 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -26,6 +26,8 @@
 #include <linux/jiffies.h>
 #include <linux/mm_api.h>
 #include <linux/highmem.h>
+#include <linux/hrtimer.h>
+#include <linux/hrtimer_bases.h>
 #include <linux/spinlock_api.h>
 #include <linux/cpumask_api.h>
 #include <linux/lockdep_api.h>
@@ -38,6 +40,7 @@
 #include <linux/sched/isolation.h>
 #include <linux/sched/nohz.h>
 #include <linux/sched/prio.h>
+#include <linux/static_call.h>
 
 #include <linux/cpuidle.h>
 #include <linux/interrupt.h>
@@ -72,7 +75,10 @@
 unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG;
 
 /*
- * Minimal preemption granularity for CPU-bound tasks:
+ * Default base time slice (request size r_i) for SCHED_NORMAL/SCHED_BATCH:
+ *
+ * Under EEVDF this is the request size used to compute the virtual
+ * deadline; see update_deadline().
  *
  * (default: 0.70 msec * (1 + ilog(ncpus)), units: nanoseconds)
  */
@@ -296,8 +302,8 @@ static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight
  */
 static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se)
 {
-	if (unlikely(se->load.weight != NICE_0_LOAD))
-		delta = __calc_delta(delta, NICE_0_LOAD, &se->load);
+	if (se->h_load.weight != NICE_0_LOAD)
+		delta = __calc_delta(delta, NICE_0_LOAD, &se->h_load);
 
 	return delta;
 }
@@ -427,38 +433,6 @@ static inline struct sched_entity *parent_entity(const struct sched_entity *se)
 	return se->parent;
 }
 
-static void
-find_matching_se(struct sched_entity **se, struct sched_entity **pse)
-{
-	int se_depth, pse_depth;
-
-	/*
-	 * preemption test can be made between sibling entities who are in the
-	 * same cfs_rq i.e who have a common parent. Walk up the hierarchy of
-	 * both tasks until we find their ancestors who are siblings of common
-	 * parent.
-	 */
-
-	/* First walk up until both entities are at same depth */
-	se_depth = (*se)->depth;
-	pse_depth = (*pse)->depth;
-
-	while (se_depth > pse_depth) {
-		se_depth--;
-		*se = parent_entity(*se);
-	}
-
-	while (pse_depth > se_depth) {
-		pse_depth--;
-		*pse = parent_entity(*pse);
-	}
-
-	while (!is_same_group(*se, *pse)) {
-		*se = parent_entity(*se);
-		*pse = parent_entity(*pse);
-	}
-}
-
 static int tg_is_idle(struct task_group *tg)
 {
 	return tg->idle > 0;
@@ -502,11 +476,6 @@ static inline struct sched_entity *parent_entity(struct sched_entity *se)
 	return NULL;
 }
 
-static inline void
-find_matching_se(struct sched_entity **se, struct sched_entity **pse)
-{
-}
-
 static inline int tg_is_idle(struct task_group *tg)
 {
 	return 0;
@@ -685,7 +654,7 @@ static inline unsigned long avg_vruntime_weight(struct cfs_rq *cfs_rq, unsigned
 static inline void
 __sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-	unsigned long weight = avg_vruntime_weight(cfs_rq, se->load.weight);
+	unsigned long weight = avg_vruntime_weight(cfs_rq, se->h_load.weight);
 	s64 w_vruntime, key = entity_key(cfs_rq, se);
 
 	w_vruntime = key * weight;
@@ -702,7 +671,7 @@ sum_w_vruntime_add_paranoid(struct cfs_rq *cfs_rq, struct sched_entity *se)
 	s64 key, tmp;
 
 again:
-	weight = avg_vruntime_weight(cfs_rq, se->load.weight);
+	weight = avg_vruntime_weight(cfs_rq, se->h_load.weight);
 	key = entity_key(cfs_rq, se);
 
 	if (check_mul_overflow(key, weight, &key))
@@ -748,7 +717,7 @@ sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se)
 static void
 sum_w_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-	unsigned long weight = avg_vruntime_weight(cfs_rq, se->load.weight);
+	unsigned long weight = avg_vruntime_weight(cfs_rq, se->h_load.weight);
 	s64 key = entity_key(cfs_rq, se);
 
 	cfs_rq->sum_w_vruntime -= key * weight;
@@ -790,7 +759,7 @@ u64 avg_vruntime(struct cfs_rq *cfs_rq)
 		s64 runtime = cfs_rq->sum_w_vruntime;
 
 		if (curr) {
-			unsigned long w = avg_vruntime_weight(cfs_rq, curr->load.weight);
+			unsigned long w = avg_vruntime_weight(cfs_rq, curr->h_load.weight);
 
 			runtime += entity_key(cfs_rq, curr) * w;
 			weight += w;
@@ -813,6 +782,67 @@ u64 avg_vruntime(struct cfs_rq *cfs_rq)
 	return cfs_rq->zero_vruntime;
 }
 
+/*
+ *     \Sum (v_i - v0)*w_i
+ * V = ------------------- + v0
+ *          \Sum w_i
+ *
+ * Let W = \Sum w_i, and move v_j such that 'v_j == V', thus:
+ *
+ * V = 1/W * {(v_j - v0)*w_j + \Sum_i!=j (v_i - v0)*w_i} + v0
+ *
+ * v_j = 1/W * {(v_j - v0)*w_j + \Sum_i!=j (v_i - v0)*w_i} + v0
+ *
+ * v_j = 1/W * (v_j - v0)*w_j + 1/W * \Sum_i!=j (v_i - v0)*w_i + v0
+ *
+ * v_j - 1/W * (v_j - v0)*w_j = 1/W * \Sum_i!=j (v_i - v0)*w_i + v0
+ *
+ * v_j*W - (v_j - v0)*w_j = \Sum_i!=j (v_i - v0)*w_i + v0*W
+ *
+ * v_j*(W - w_j) + v0*w_j = \Sum_i!=j (v_i - v0)*w_i + v0*W
+ *
+ * v_j*(W - w_j) = \Sum_i!=j (v_i - v0)*w_i + v0*(W - w_j)
+ *
+ *       \Sum_i!=j (v_i - v0)*w_i
+ * v_j = ------------------------ + v0
+ *               W - w_j
+ *
+ * When v_j happens to be curr, then '\Sum_i!=j (v_i - v0)*w_i'
+ * is cfs_rq->sum_w_runtime, and 'W - w_j' is cfs_rq->sum_weight, since curr
+ * is not included in the sum.
+ */
+static u64 ineligible_vruntime(struct cfs_rq *cfs_rq)
+{
+	struct sched_entity *curr = cfs_rq->curr;
+	long weight = cfs_rq->sum_weight;
+	s64 delta = 0;
+
+	if (curr && !curr->on_rq)
+		curr = NULL;
+
+	/*
+	 * This is called from set_next_task_fair(.first=true) /
+	 * set_protect_slice() so curr had better be set and on_rq.
+	 */
+	WARN_ON_ONCE(!curr);
+
+	if (weight) {
+		s64 runtime = cfs_rq->sum_w_vruntime;
+
+		/*
+		 * Do not add @curr to obtain the effective '- w_j' terms.
+		 */
+
+		/* sign flips effective floor / ceiling */
+		if (runtime < 0)
+			runtime -= (weight - 1);
+
+		delta = div64_long(runtime, weight);
+	}
+
+	return cfs_rq->zero_vruntime + delta + 1;
+}
+
 static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq);
 
 /*
@@ -861,8 +891,6 @@ bool update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se)
 	u64 avruntime = avg_vruntime(cfs_rq);
 	s64 vlag = entity_lag(cfs_rq, se, avruntime);
 
-	WARN_ON_ONCE(!se->on_rq);
-
 	if (se->sched_delayed) {
 		/* previous vlag < 0 otherwise se would not be delayed */
 		vlag = max(vlag, se->vlag);
@@ -898,7 +926,7 @@ static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime)
 	long load = cfs_rq->sum_weight;
 
 	if (curr && curr->on_rq) {
-		unsigned long weight = avg_vruntime_weight(cfs_rq, curr->load.weight);
+		unsigned long weight = avg_vruntime_weight(cfs_rq, curr->h_load.weight);
 
 		avg += entity_key(cfs_rq, curr) * weight;
 		load += weight;
@@ -1039,6 +1067,9 @@ RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity,
  */
 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
+	WARN_ON_ONCE(&rq_of(cfs_rq)->cfs != cfs_rq);
+	WARN_ON_ONCE(!entity_is_task(se));
+
 	sum_w_vruntime_add(cfs_rq, se);
 	se->min_vruntime = se->vruntime;
 	se->min_slice = se->slice;
@@ -1048,6 +1079,9 @@ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
 
 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
+	WARN_ON_ONCE(&rq_of(cfs_rq)->cfs != cfs_rq);
+	WARN_ON_ONCE(!entity_is_task(se));
+
 	rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline,
 				  &min_vruntime_cb);
 	sum_w_vruntime_sub(cfs_rq, se);
@@ -1090,8 +1124,14 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity
 		slice = cfs_rq_min_slice(cfs_rq);
 
 	slice = min(slice, se->slice);
-	if (slice != se->slice)
-		vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se));
+
+	/* If there are shorter slices than se's one */
+	if (slice != se->slice) {
+		if (sched_feat(PREEMPT_SHORT))
+			vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq));
+		else
+			vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se));
+	}
 
 	se->vprot = vprot;
 }
@@ -1099,8 +1139,9 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity
 static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
 	u64 slice = cfs_rq_min_slice(cfs_rq);
+	u64 vruntime = min_vruntime(se->vruntime, avg_vruntime(cfs_rq));
 
-	se->vprot = min_vruntime(se->vprot, se->vruntime + calc_delta_fair(slice, se));
+	se->vprot = min_vruntime(se->vprot, vruntime + calc_delta_fair(slice, se));
 }
 
 static inline bool protect_slice(struct sched_entity *se)
@@ -1144,7 +1185,7 @@ static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq, bool protect)
 	 * We can safely skip eligibility check if there is only one entity
 	 * in this cfs_rq, saving some cycles.
 	 */
-	if (cfs_rq->nr_queued == 1)
+	if (cfs_rq->h_nr_queued == 1)
 		return curr && curr->on_rq ? curr : se;
 
 	/*
@@ -1394,8 +1435,6 @@ static s64 update_se(struct rq *rq, struct sched_entity *se)
 	return delta_exec;
 }
 
-static void set_next_buddy(struct sched_entity *se);
-
 #ifdef CONFIG_SCHED_CACHE
 
 /*
@@ -1990,7 +2029,7 @@ static void update_curr(struct cfs_rq *cfs_rq)
 	 * not necessarily be the actual task running
 	 * (rq->curr.se). This is easy to confuse!
 	 */
-	struct sched_entity *curr = cfs_rq->curr;
+	struct sched_entity *curr = cfs_rq->h_curr;
 	struct rq *rq = rq_of(cfs_rq);
 	s64 delta_exec;
 	bool resched;
@@ -2002,26 +2041,29 @@ static void update_curr(struct cfs_rq *cfs_rq)
 	if (unlikely(delta_exec <= 0))
 		return;
 
+	account_cfs_rq_runtime(cfs_rq, delta_exec);
+
+	if (!entity_is_task(curr))
+		return;
+
+	cfs_rq = &rq->cfs;
+
 	curr->vruntime += calc_delta_fair(delta_exec, curr);
 	resched = update_deadline(cfs_rq, curr);
 
-	if (entity_is_task(curr)) {
-		/*
-		 * If the fair_server is active, we need to account for the
-		 * fair_server time whether or not the task is running on
-		 * behalf of fair_server or not:
-		 *  - If the task is running on behalf of fair_server, we need
-		 *    to limit its time based on the assigned runtime.
-		 *  - Fair task that runs outside of fair_server should account
-		 *    against fair_server such that it can account for this time
-		 *    and possibly avoid running this period.
-		 */
-		dl_server_update(&rq->fair_server, delta_exec);
-	}
-
-	account_cfs_rq_runtime(cfs_rq, delta_exec);
+	/*
+	 * If the fair_server is active, we need to account for the
+	 * fair_server time whether or not the task is running on
+	 * behalf of fair_server or not:
+	 *  - If the task is running on behalf of fair_server, we need
+	 *    to limit its time based on the assigned runtime.
+	 *  - Fair task that runs outside of fair_server should account
+	 *    against fair_server such that it can account for this time
+	 *    and possibly avoid running this period.
+	 */
+	dl_server_update(&rq->fair_server, delta_exec);
 
-	if (cfs_rq->nr_queued == 1)
+	if (cfs_rq->h_nr_queued == 1)
 		return;
 
 	if (resched || !protect_slice(curr)) {
@@ -2032,7 +2074,10 @@ static void update_curr(struct cfs_rq *cfs_rq)
 
 static void update_curr_fair(struct rq *rq)
 {
-	update_curr(cfs_rq_of(&rq->donor->se));
+	struct sched_entity *se = &rq->donor->se;
+
+	for_each_sched_entity(se)
+		update_curr(cfs_rq_of(se));
 }
 
 static inline void
@@ -2108,7 +2153,7 @@ update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int fl
 	 * Are we enqueueing a waiting task? (for current tasks
 	 * a dequeue/enqueue event is a NOP)
 	 */
-	if (se != cfs_rq->curr)
+	if (se != cfs_rq->h_curr)
 		update_stats_wait_start_fair(cfs_rq, se);
 
 	if (flags & ENQUEUE_WAKEUP)
@@ -2126,7 +2171,7 @@ update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int fl
 	 * Mark the end of the wait period if dequeueing a
 	 * waiting task:
 	 */
-	if (se != cfs_rq->curr)
+	if (se != cfs_rq->h_curr)
 		update_stats_wait_end_fair(cfs_rq, se);
 
 	if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) {
@@ -4467,6 +4512,7 @@ static inline void update_scan_period(struct task_struct *p, int new_cpu)
 static void
 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
+	WARN_ON_ONCE(cfs_rq != cfs_rq_of(se));
 	update_load_add(&cfs_rq->load, se->load.weight);
 	if (entity_is_task(se)) {
 		struct task_struct *p = task_of(se);
@@ -4482,6 +4528,7 @@ account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 static void
 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
+	WARN_ON_ONCE(cfs_rq != cfs_rq_of(se));
 	update_load_sub(&cfs_rq->load, se->load.weight);
 	if (entity_is_task(se)) {
 		struct task_struct *p = task_of(se);
@@ -4563,7 +4610,7 @@ dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
 static void
 rescale_entity(struct sched_entity *se, unsigned long weight, bool rel_vprot)
 {
-	unsigned long old_weight = se->load.weight;
+	long old_weight = se->h_load.weight;
 
 	/*
 	 * VRUNTIME
@@ -4663,16 +4710,17 @@ rescale_entity(struct sched_entity *se, unsigned long weight, bool rel_vprot)
 		se->vprot = div64_long(se->vprot * old_weight, weight);
 }
 
-static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
-			    unsigned long weight)
+static void reweight_eevdf(struct cfs_rq *cfs_rq, struct sched_entity *se,
+			   unsigned long weight, bool on_rq)
 {
 	bool curr = cfs_rq->curr == se;
 	bool rel_vprot = false;
 	u64 avruntime = 0;
 
-	if (se->on_rq) {
-		/* commit outstanding execution time */
-		update_curr(cfs_rq);
+	if (se->h_load.weight == weight)
+		return;
+
+	if (on_rq) {
 		avruntime = avg_vruntime(cfs_rq);
 		se->vlag = entity_lag(cfs_rq, se, avruntime);
 		se->deadline -= avruntime;
@@ -4682,46 +4730,90 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
 			rel_vprot = true;
 		}
 
-		cfs_rq->nr_queued--;
+		cfs_rq->h_nr_queued--;
 		if (!curr)
 			__dequeue_entity(cfs_rq, se);
-		update_load_sub(&cfs_rq->load, se->load.weight);
 	}
-	dequeue_load_avg(cfs_rq, se);
 
 	rescale_entity(se, weight, rel_vprot);
 
-	update_load_set(&se->load, weight);
+	update_load_set(&se->h_load, weight);
 
-	do {
-		u32 divider = get_pelt_divider(&se->avg);
-		se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider);
-	} while (0);
-
-	enqueue_load_avg(cfs_rq, se);
-	if (se->on_rq) {
+	if (on_rq) {
 		if (rel_vprot)
 			se->vprot += avruntime;
 		se->deadline += avruntime;
 		se->rel_deadline = 0;
 		se->vruntime = avruntime - se->vlag;
 
-		update_load_add(&cfs_rq->load, se->load.weight);
 		if (!curr)
 			__enqueue_entity(cfs_rq, se);
-		cfs_rq->nr_queued++;
+		cfs_rq->h_nr_queued++;
 	}
 }
 
+static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
+			    unsigned long weight)
+{
+	if (se->load.weight == weight)
+		return;
+
+	if (se->on_rq) {
+		WARN_ON_ONCE(cfs_rq != cfs_rq_of(se));
+		update_load_sub(&cfs_rq->load, se->load.weight);
+	}
+	dequeue_load_avg(cfs_rq, se);
+
+	update_load_set(&se->load, weight);
+
+	do {
+		u32 divider = get_pelt_divider(&se->avg);
+		se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider);
+	} while (0);
+
+	enqueue_load_avg(cfs_rq, se);
+
+	if (se->on_rq)
+		update_load_add(&cfs_rq->load, se->load.weight);
+}
+
+/*
+ * weight = NICE_0_LOAD;
+ * for_each_entity_se(se)
+ *   weight = __calc_prop_weight(cfs_rq_of(se), se, weight);
+ */
+static __always_inline
+unsigned long __calc_prop_weight(struct cfs_rq *cfs_rq, struct sched_entity *se,
+				 unsigned long weight)
+{
+	weight *= se->load.weight;
+	if (parent_entity(se))
+		weight /= cfs_rq->load.weight;
+	else
+		weight /= NICE_0_LOAD;
+
+	return max(weight, MIN_SHARES);
+}
+
 static void reweight_task_fair(struct rq *rq, struct task_struct *p,
 			       const struct load_weight *lw)
 {
 	struct sched_entity *se = &p->se;
-	struct cfs_rq *cfs_rq = cfs_rq_of(se);
-	struct load_weight *load = &se->load;
+	unsigned long weight = NICE_0_LOAD;
+
+	if (se->on_rq)
+		update_curr_fair(rq);
+
+	reweight_entity(cfs_rq_of(se), se, lw->weight);
+	se->load.inv_weight = lw->inv_weight;
 
-	reweight_entity(cfs_rq, se, lw->weight);
-	load->inv_weight = lw->inv_weight;
+	if (!se->on_rq)
+		return;
+
+	for_each_sched_entity(se)
+		weight = __calc_prop_weight(cfs_rq_of(se), se, weight);
+
+	reweight_eevdf(&rq->cfs, &p->se, weight, p->se.on_rq);
 }
 
 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
@@ -4800,12 +4892,10 @@ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
  *
  * hence icky!
  */
-static long calc_group_shares(struct cfs_rq *cfs_rq)
+static long __calc_smp_shares(struct cfs_rq *cfs_rq, long tg_shares, long shares_max)
 {
-	long tg_weight, tg_shares, load, shares;
 	struct task_group *tg = cfs_rq->tg;
-
-	tg_shares = READ_ONCE(tg->shares);
+	long tg_weight, load, shares;
 
 	load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg);
 
@@ -4831,7 +4921,115 @@ static long calc_group_shares(struct cfs_rq *cfs_rq)
 	 * case no task is runnable on a CPU MIN_SHARES=2 should be returned
 	 * instead of 0.
 	 */
-	return clamp_t(long, shares, MIN_SHARES, tg_shares);
+	return clamp_t(long, shares, MIN_SHARES, shares_max);
+}
+
+static int tg_cpus(struct task_group *tg)
+{
+	int nr = num_online_cpus();
+
+	if (cpusets_enabled()) {
+		struct cgroup *cgrp = tg->css.cgroup;
+		if (cgrp)
+			nr = cpuset_num_cpus(cgrp);
+	}
+
+	return nr;
+}
+
+static inline int tg_tasks(struct task_group *tg)
+{
+	return max(1, atomic_long_read(&tg->runnable_avg) >> SCHED_CAPACITY_SHIFT);
+}
+
+/*
+ * Func: fraction(nr_tasks * tg->shares)
+ *
+ * Scale tg->shares by the number of tasks.
+ */
+static long calc_tasks_shares(struct cfs_rq *cfs_rq)
+{
+	struct task_group *tg = cfs_rq->tg;
+	int nr = tg_tasks(tg);
+	long tg_shares = READ_ONCE(tg->shares);
+	return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares);
+}
+
+/*
+ * Func: min(fraction(nr_cpus * tg->shares), nice -20)
+ *
+ * Scale tg->shares by the maximal number of CPUs; but clip the max shares at
+ * nice -20, otherwise a single spinner on a 512 CPU machine would result in
+ * 512*NICE_0_LOAD, which is also crazy.
+ */
+static long calc_max_shares(struct cfs_rq *cfs_rq)
+{
+	struct task_group *tg = cfs_rq->tg;
+	int nr = tg_cpus(tg);
+	long tg_shares = READ_ONCE(tg->shares);
+	long max_shares = scale_load(sched_prio_to_weight[0]);
+	return __calc_smp_shares(cfs_rq, tg_shares * nr, max_shares);
+}
+
+/*
+ * Func: fraction(nr * tg->shares); nr = min(nr_tasks, nr_cpus)
+ *
+ * Scales between "smp" and "max" in a natural way. No longer needs clipping
+ * since there are no unnatural inflations like with "max".
+ */
+static long calc_concur_shares(struct cfs_rq *cfs_rq)
+{
+	struct task_group *tg = cfs_rq->tg;
+	int nr = min(tg_tasks(tg), tg_cpus(tg));
+	long tg_shares = READ_ONCE(tg->shares);
+	return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares);
+}
+
+/*
+ * Func: fraction(tg->shares)
+ *
+ * This infamously results in tiny shares when you have many CPUs.
+ */
+static long calc_smp_shares(struct cfs_rq *cfs_rq)
+{
+	struct task_group *tg = cfs_rq->tg;
+	long tg_shares = READ_ONCE(tg->shares);
+	return __calc_smp_shares(cfs_rq, tg_shares, tg_shares);
+}
+
+/*
+ * Ignore this pesky SMP stuff, use (4).
+ */
+static long calc_up_shares(struct cfs_rq *cfs_rq)
+{
+	struct task_group *tg = cfs_rq->tg;
+	return READ_ONCE(tg->shares);
+}
+
+DEFINE_STATIC_CALL(calc_group_shares, calc_concur_shares);
+
+void __sched_cgroup_mode_update(int mode)
+{
+	long (*func)(struct cfs_rq *);
+	switch (mode) {
+	case 0:
+		func = &calc_up_shares;
+		break;
+	case 1:
+		func = &calc_smp_shares;
+		break;
+	case 2:
+	default:
+		func = &calc_concur_shares;
+		break;
+	case 3:
+		func = &calc_max_shares;
+		break;
+	case 4:
+		func = &calc_tasks_shares;
+		break;
+	}
+	static_call_update(calc_group_shares, func);
 }
 
 /*
@@ -4850,9 +5048,8 @@ static void update_cfs_group(struct sched_entity *se)
 	if (!gcfs_rq || !gcfs_rq->load.weight)
 		return;
 
-	shares = calc_group_shares(gcfs_rq);
-	if (unlikely(se->load.weight != shares))
-		reweight_entity(cfs_rq_of(se), se, shares);
+	shares = static_call(calc_group_shares)(gcfs_rq);
+	reweight_entity(cfs_rq_of(se), se, shares);
 }
 
 #else /* !CONFIG_FAIR_GROUP_SCHED: */
@@ -4970,11 +5167,11 @@ static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq)
  * differential update where we store the last value we propagated. This in
  * turn allows skipping updates if the differential is 'small'.
  *
- * Updating tg's load_avg is necessary before update_cfs_share().
+ * Updating tg's load_avg is necessary before update_cfs_group().
  */
 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq)
 {
-	long delta;
+	long dl, dr;
 	u64 now;
 
 	/*
@@ -4995,17 +5192,21 @@ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq)
 	if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC)
 		return;
 
-	delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib;
-	if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) {
-		atomic_long_add(delta, &cfs_rq->tg->load_avg);
+	dl = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib;
+	dr = cfs_rq->avg.runnable_avg - cfs_rq->tg_runnable_avg_contrib;
+	if (abs(dl) > cfs_rq->tg_load_avg_contrib / 64 ||
+	    abs(dr) > cfs_rq->tg_runnable_avg_contrib / 64) {
+		atomic_long_add(dl, &cfs_rq->tg->load_avg);
+		atomic_long_add(dr, &cfs_rq->tg->runnable_avg);
 		cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg;
+		cfs_rq->tg_runnable_avg_contrib = cfs_rq->avg.runnable_avg;
 		cfs_rq->last_update_tg_load_avg = now;
 	}
 }
 
 static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq)
 {
-	long delta;
+	long dl, dr;
 	u64 now;
 
 	/*
@@ -5015,9 +5216,12 @@ static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq)
 		return;
 
 	now = rq_clock(rq_of(cfs_rq));
-	delta = 0 - cfs_rq->tg_load_avg_contrib;
-	atomic_long_add(delta, &cfs_rq->tg->load_avg);
+	dl = 0 - cfs_rq->tg_load_avg_contrib;
+	dr = 0 - cfs_rq->tg_runnable_avg_contrib;
+	atomic_long_add(dl, &cfs_rq->tg->load_avg);
+	atomic_long_add(dr, &cfs_rq->tg->runnable_avg);
 	cfs_rq->tg_load_avg_contrib = 0;
+	cfs_rq->tg_runnable_avg_contrib = 0;
 	cfs_rq->last_update_tg_load_avg = now;
 }
 
@@ -5174,7 +5378,8 @@ static inline void
 update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
 {
 	long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg;
-	u32 new_sum, divider;
+	u64 new_sum;
+	u32 divider;
 
 	/* Nothing to update */
 	if (!delta_avg)
@@ -5188,7 +5393,7 @@ update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cf
 
 	/* Set new sched_entity's runnable */
 	se->avg.runnable_avg = gcfs_rq->avg.runnable_avg;
-	new_sum = se->avg.runnable_avg * divider;
+	new_sum = (u64)se->avg.runnable_avg * divider;
 	delta_sum = (long)new_sum - (long)se->avg.runnable_sum;
 	se->avg.runnable_sum = new_sum;
 
@@ -5430,7 +5635,7 @@ static void migrate_se_pelt_lag(struct sched_entity *se) {}
  * The cfs_rq avg is the direct sum of all its entities (blocked and runnable)
  * avg. The immediate corollary is that all (fair) tasks must be attached.
  *
- * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example.
+ * cfs_rq->avg is used for task_h_load() and update_cfs_group() for example.
  *
  * Return: true if the load decayed or we removed load.
  *
@@ -5968,6 +6173,7 @@ static void
 place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 {
 	u64 vslice, vruntime = avg_vruntime(cfs_rq);
+	unsigned int nr_queued = cfs_rq->h_nr_queued;
 	bool update_zero = false;
 	s64 lag = 0;
 
@@ -5975,6 +6181,9 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 		se->slice = sysctl_sched_base_slice;
 	vslice = calc_delta_fair(se->slice, se);
 
+	if (flags & ENQUEUE_QUEUED)
+		nr_queued -= 1;
+
 	/*
 	 * Due to how V is constructed as the weighted average of entities,
 	 * adding tasks with positive lag, or removing tasks with negative lag
@@ -5983,7 +6192,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 	 *
 	 * EEVDF: placement strategy #1 / #2
 	 */
-	if (sched_feat(PLACE_LAG) && cfs_rq->nr_queued && se->vlag) {
+	if (sched_feat(PLACE_LAG) && nr_queued && se->vlag) {
 		struct sched_entity *curr = cfs_rq->curr;
 		long load, weight;
 
@@ -6043,9 +6252,9 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 		 */
 		load = cfs_rq->sum_weight;
 		if (curr && curr->on_rq)
-			load += avg_vruntime_weight(cfs_rq, curr->load.weight);
+			load += avg_vruntime_weight(cfs_rq, curr->h_load.weight);
 
-		weight = avg_vruntime_weight(cfs_rq, se->load.weight);
+		weight = avg_vruntime_weight(cfs_rq, se->h_load.weight);
 		lag *= load + weight;
 		if (WARN_ON_ONCE(!load))
 			load = 1;
@@ -6103,23 +6312,9 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 static void check_enqueue_throttle(struct cfs_rq *cfs_rq);
 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq);
 
-static void
-requeue_delayed_entity(struct sched_entity *se);
-
 static void
 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 {
-	bool curr = cfs_rq->curr == se;
-
-	/*
-	 * If we're the current task, we must renormalise before calling
-	 * update_curr().
-	 */
-	if (curr)
-		place_entity(cfs_rq, se, flags);
-
-	update_curr(cfs_rq);
-
 	/*
 	 * When enqueuing a sched_entity, we must:
 	 *   - Update loads to have both entity and cfs_rq synced with now.
@@ -6138,13 +6333,6 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 	 */
 	update_cfs_group(se);
 
-	/*
-	 * XXX now that the entity has been re-weighted, and it's lag adjusted,
-	 * we can place the entity.
-	 */
-	if (!curr)
-		place_entity(cfs_rq, se, flags);
-
 	account_entity_enqueue(cfs_rq, se);
 
 	/* Entity has migrated, no longer consider this task hot */
@@ -6153,8 +6341,6 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 
 	check_schedstat_required();
 	update_stats_enqueue_fair(cfs_rq, se, flags);
-	if (!curr)
-		__enqueue_entity(cfs_rq, se);
 	se->on_rq = 1;
 
 	if (cfs_rq->nr_queued == 1) {
@@ -6172,21 +6358,19 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 	}
 }
 
-static void __clear_buddies_next(struct sched_entity *se)
+static void set_next_buddy(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-	for_each_sched_entity(se) {
-		struct cfs_rq *cfs_rq = cfs_rq_of(se);
-		if (cfs_rq->next != se)
-			break;
-
-		cfs_rq->next = NULL;
-	}
+	if (WARN_ON_ONCE(!se->on_rq || se->sched_delayed))
+		return;
+	if (se_is_idle(se))
+		return;
+	cfs_rq->next = se;
 }
 
 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
 	if (cfs_rq->next == se)
-		__clear_buddies_next(se);
+		cfs_rq->next = NULL;
 }
 
 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
@@ -6197,7 +6381,7 @@ static void set_delayed(struct sched_entity *se)
 
 	/*
 	 * Delayed se of cfs_rq have no tasks queued on them.
-	 * Do not adjust h_nr_runnable since dequeue_entities()
+	 * Do not adjust h_nr_runnable since __dequeue_task()
 	 * will account it for blocked tasks.
 	 */
 	if (!entity_is_task(se))
@@ -6230,45 +6414,16 @@ static void clear_delayed(struct sched_entity *se)
 	}
 }
 
-static bool
+static void
 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 {
-	bool sleep = flags & DEQUEUE_SLEEP;
-	int action = 0;
-
-	update_curr(cfs_rq);
-	clear_buddies(cfs_rq, se);
-
-	if (flags & DEQUEUE_DELAYED) {
-		WARN_ON_ONCE(!se->sched_delayed);
-	} else {
-		bool delay = sleep;
-		/*
-		 * DELAY_DEQUEUE relies on spurious wakeups, special task
-		 * states must not suffer spurious wakeups, excempt them.
-		 */
-		if (flags & (DEQUEUE_SPECIAL | DEQUEUE_THROTTLE))
-			delay = false;
-
-		WARN_ON_ONCE(delay && se->sched_delayed);
-
-		if (sched_feat(DELAY_DEQUEUE) && delay &&
-		    !entity_eligible(cfs_rq, se)) {
-			if (entity_is_task(se))
-				action |= UPDATE_UTIL_EST;
-			update_load_avg(cfs_rq, se, action);
-			update_entity_lag(cfs_rq, se);
-			set_delayed(se);
-			return false;
-		}
-	}
+	int action = UPDATE_TG;
 
-	action = UPDATE_TG;
 	if (entity_is_task(se)) {
 		if (task_on_rq_migrating(task_of(se)))
 			action |= DO_DETACH;
 
-		if (sleep && !(flags & DEQUEUE_DELAYED))
+		if ((flags & DEQUEUE_SLEEP) && !(flags & DEQUEUE_DELAYED))
 			action |= UPDATE_UTIL_EST;
 	}
 
@@ -6286,14 +6441,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 
 	update_stats_dequeue_fair(cfs_rq, se, flags);
 
-	update_entity_lag(cfs_rq, se);
-	if (sched_feat(PLACE_REL_DEADLINE) && !sleep) {
-		se->deadline -= se->vruntime;
-		se->rel_deadline = 1;
-	}
-
-	if (se != cfs_rq->curr)
-		__dequeue_entity(cfs_rq, se);
 	se->on_rq = 0;
 	account_entity_dequeue(cfs_rq, se);
 
@@ -6302,9 +6449,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 
 	update_cfs_group(se);
 
-	if (flags & DEQUEUE_DELAYED)
-		clear_delayed(se);
-
 	if (cfs_rq->nr_queued == 0) {
 		update_idle_cfs_rq_clock_pelt(cfs_rq);
 #ifdef CONFIG_CFS_BANDWIDTH
@@ -6317,15 +6461,11 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 		}
 #endif
 	}
-
-	return true;
 }
 
 static void
-set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first)
+set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-	clear_buddies(cfs_rq, se);
-
 	/* 'current' is not kept within the tree. */
 	if (se->on_rq) {
 		/*
@@ -6334,16 +6474,12 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first)
 		 * runqueue.
 		 */
 		update_stats_wait_end_fair(cfs_rq, se);
-		__dequeue_entity(cfs_rq, se);
 		update_load_avg(cfs_rq, se, UPDATE_TG);
-
-		if (first)
-			set_protect_slice(cfs_rq, se);
 	}
 
 	update_stats_curr_start(cfs_rq, se);
-	WARN_ON_ONCE(cfs_rq->curr);
-	cfs_rq->curr = se;
+	WARN_ON_ONCE(cfs_rq->h_curr);
+	cfs_rq->h_curr = se;
 
 	/*
 	 * Track our maximum slice length, if the CPU's load is at
@@ -6363,23 +6499,17 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first)
 	se->prev_sum_exec_runtime = se->sum_exec_runtime;
 }
 
-static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags);
+static bool __dequeue_task(struct rq *rq, struct task_struct *p, int flags);
 
-/*
- * Pick the next process, keeping these things in mind, in this order:
- * 1) keep things fair between processes/task groups
- * 2) pick the "next" process, since someone really wants that to run
- * 3) pick the "last" process, for cache locality
- * 4) do not run the "skip" process, if something else is available
- */
 static struct sched_entity *
-pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq, bool protect)
+pick_next_entity(struct rq *rq, bool protect)
 {
+	struct cfs_rq *cfs_rq = &rq->cfs;
 	struct sched_entity *se;
 
 	se = pick_eevdf(cfs_rq, protect);
 	if (se->sched_delayed) {
-		dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED);
+		__dequeue_task(rq, task_of(se), DEQUEUE_SLEEP | DEQUEUE_DELAYED);
 		/*
 		 * Must not reference @se again, see __block_task().
 		 */
@@ -6399,13 +6529,11 @@ static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
 
 	if (prev->on_rq) {
 		update_stats_wait_start_fair(cfs_rq, prev);
-		/* Put 'current' back into the tree. */
-		__enqueue_entity(cfs_rq, prev);
 		/* in !on_rq case, update occurred at dequeue */
 		update_load_avg(cfs_rq, prev, 0);
 	}
-	WARN_ON_ONCE(cfs_rq->curr != prev);
-	cfs_rq->curr = NULL;
+	WARN_ON_ONCE(cfs_rq->h_curr != prev);
+	cfs_rq->h_curr = NULL;
 }
 
 static void
@@ -6960,7 +7088,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
 	assert_list_leaf_cfs_rq(rq);
 
 	/* Determine whether we need to wake up potentially idle CPU: */
-	if (rq->curr == rq->idle && rq->cfs.nr_queued)
+	if (rq->curr == rq->idle && rq->cfs.h_nr_queued)
 		resched_curr(rq);
 }
 
@@ -7295,7 +7423,7 @@ static void check_enqueue_throttle(struct cfs_rq *cfs_rq)
 		return;
 
 	/* an active group must be handled by the update_curr() path */
-	if (!cfs_rq->runtime_enabled || cfs_rq->curr)
+	if (!cfs_rq->runtime_enabled || cfs_rq->h_curr)
 		return;
 
 	/* ensure the group is not already throttled */
@@ -7667,7 +7795,7 @@ static void hrtick_start_fair(struct rq *rq, struct task_struct *p)
 			resched_curr(rq);
 		return;
 	}
-	delta = (se->load.weight * vdelta) / NICE_0_LOAD;
+	delta = (se->h_load.weight * vdelta) / NICE_0_LOAD;
 
 	/*
 	 * Correct for instantaneous load of other classes.
@@ -7767,10 +7895,8 @@ static int choose_idle_cpu(int cpu, struct task_struct *p)
 }
 
 static void
-requeue_delayed_entity(struct sched_entity *se)
+requeue_delayed_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-	struct cfs_rq *cfs_rq = cfs_rq_of(se);
-
 	/*
 	 * se->sched_delayed should imply: se->on_rq == 1.
 	 * Because a delayed entity is one that is still on
@@ -7780,34 +7906,72 @@ requeue_delayed_entity(struct sched_entity *se)
 	WARN_ON_ONCE(!se->on_rq);
 
 	if (update_entity_lag(cfs_rq, se)) {
-		cfs_rq->nr_queued--;
+		cfs_rq->h_nr_queued--;
 		if (se != cfs_rq->curr)
 			__dequeue_entity(cfs_rq, se);
 		place_entity(cfs_rq, se, 0);
 		if (se != cfs_rq->curr)
 			__enqueue_entity(cfs_rq, se);
-		cfs_rq->nr_queued++;
+		cfs_rq->h_nr_queued++;
 	}
 
 	update_load_avg(cfs_rq, se, 0);
 	clear_delayed(se);
 }
 
+static unsigned long enqueue_hierarchy(struct task_struct *p, int flags)
+{
+	unsigned long weight = NICE_0_LOAD;
+	int task_new = !(flags & ENQUEUE_WAKEUP);
+	struct sched_entity *se = &p->se;
+	int h_nr_idle = task_has_idle_policy(p);
+	int h_nr_runnable = 1;
+
+	if (task_new && se->sched_delayed)
+		h_nr_runnable = 0;
+
+	for_each_sched_entity(se) {
+		struct cfs_rq *cfs_rq = cfs_rq_of(se);
+
+		update_curr(cfs_rq);
+
+		if (!se->on_rq) {
+			enqueue_entity(cfs_rq, se, flags);
+		} else {
+			update_load_avg(cfs_rq, se, UPDATE_TG);
+			se_update_runnable(se);
+			update_cfs_group(se);
+		}
+
+		cfs_rq->h_nr_runnable += h_nr_runnable;
+		cfs_rq->h_nr_queued++;
+		cfs_rq->h_nr_idle += h_nr_idle;
+
+		if (cfs_rq_is_idle(cfs_rq))
+			h_nr_idle = 1;
+
+		weight = __calc_prop_weight(cfs_rq, se, weight);
+
+		flags = ENQUEUE_WAKEUP;
+	}
+
+	return weight;
+}
+
 /*
  * The enqueue_task method is called before nr_running is
  * increased. Here we update the fair scheduling stats and
  * then put the task into the rbtree:
  */
 static void
-enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
-{
-	struct cfs_rq *cfs_rq;
-	struct sched_entity *se = &p->se;
-	int h_nr_idle = task_has_idle_policy(p);
-	int h_nr_runnable = 1;
-	int task_new = !(flags & ENQUEUE_WAKEUP);
+enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
+{
 	int rq_h_nr_queued = rq->cfs.h_nr_queued;
-	u64 slice = 0;
+	int task_new = !(flags & ENQUEUE_WAKEUP);
+	struct sched_entity *se = &p->se;
+	struct cfs_rq *cfs_rq = &rq->cfs;
+	unsigned long weight;
+	bool curr;
 
 	if (task_is_throttled(p) && enqueue_throttled_task(p))
 		return;
@@ -7819,10 +7983,10 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 	 * estimated utilization, before we update schedutil.
 	 */
 	if (!p->se.sched_delayed || (flags & ENQUEUE_DELAYED))
-		util_est_enqueue(&rq->cfs, p);
+		util_est_enqueue(cfs_rq, p);
 
 	if (flags & ENQUEUE_DELAYED) {
-		requeue_delayed_entity(se);
+		requeue_delayed_entity(cfs_rq, se);
 		return;
 	}
 
@@ -7834,57 +7998,22 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 	if (p->in_iowait)
 		cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT);
 
-	if (task_new && se->sched_delayed)
-		h_nr_runnable = 0;
-
-	for_each_sched_entity(se) {
-		if (se->on_rq) {
-			if (se->sched_delayed)
-				requeue_delayed_entity(se);
-			break;
-		}
-		cfs_rq = cfs_rq_of(se);
-
-		/*
-		 * Basically set the slice of group entries to the min_slice of
-		 * their respective cfs_rq. This ensures the group can service
-		 * its entities in the desired time-frame.
-		 */
-		if (slice) {
-			se->slice = slice;
-			se->custom_slice = 1;
-		}
-		enqueue_entity(cfs_rq, se, flags);
-		slice = cfs_rq_min_slice(cfs_rq);
-
-		cfs_rq->h_nr_runnable += h_nr_runnable;
-		cfs_rq->h_nr_queued++;
-		cfs_rq->h_nr_idle += h_nr_idle;
-
-		if (cfs_rq_is_idle(cfs_rq))
-			h_nr_idle = 1;
-
-		flags = ENQUEUE_WAKEUP;
-	}
-
-	for_each_sched_entity(se) {
-		cfs_rq = cfs_rq_of(se);
-
-		update_load_avg(cfs_rq, se, UPDATE_TG);
-		se_update_runnable(se);
-		update_cfs_group(se);
+	/*
+	 * XXX comment on the curr thing
+	 */
+	curr = (cfs_rq->curr == se);
+	if (curr)
+		place_entity(cfs_rq, se, flags);
 
-		se->slice = slice;
-		if (se != cfs_rq->curr)
-			min_vruntime_cb_propagate(&se->run_node, NULL);
-		slice = cfs_rq_min_slice(cfs_rq);
+	if (se->on_rq && se->sched_delayed)
+		requeue_delayed_entity(cfs_rq, se);
 
-		cfs_rq->h_nr_runnable += h_nr_runnable;
-		cfs_rq->h_nr_queued++;
-		cfs_rq->h_nr_idle += h_nr_idle;
+	weight = enqueue_hierarchy(p, flags);
 
-		if (cfs_rq_is_idle(cfs_rq))
-			h_nr_idle = 1;
+	if (!curr) {
+		reweight_eevdf(cfs_rq, se, weight, false);
+		place_entity(cfs_rq, se, flags | ENQUEUE_QUEUED);
+		__enqueue_entity(cfs_rq, se);
 	}
 
 	if (!rq_h_nr_queued && rq->cfs.h_nr_queued)
@@ -7915,105 +8044,109 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 	hrtick_update(rq);
 }
 
-/*
- * Basically dequeue_task_fair(), except it can deal with dequeue_entity()
- * failing half-way through and resume the dequeue later.
- *
- * Returns:
- * -1 - dequeue delayed
- *  0 - dequeue throttled
- *  1 - dequeue complete
- */
-static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags)
+static void dequeue_hierarchy(struct task_struct *p, int flags)
 {
-	bool was_sched_idle = sched_idle_rq(rq);
+	struct sched_entity *se = &p->se;
 	bool task_sleep = flags & DEQUEUE_SLEEP;
 	bool task_delayed = flags & DEQUEUE_DELAYED;
 	bool task_throttled = flags & DEQUEUE_THROTTLE;
-	struct task_struct *p = NULL;
-	int h_nr_idle = 0;
-	int h_nr_queued = 0;
 	int h_nr_runnable = 0;
-	struct cfs_rq *cfs_rq;
-	u64 slice = 0;
+	int h_nr_idle = task_has_idle_policy(p);
+	bool dequeue = true;
 
-	if (entity_is_task(se)) {
-		p = task_of(se);
-		h_nr_queued = 1;
-		h_nr_idle = task_has_idle_policy(p);
-		if (task_sleep || task_delayed || !se->sched_delayed)
-			h_nr_runnable = 1;
-	}
+	if (task_sleep || task_delayed || !se->sched_delayed)
+		h_nr_runnable = 1;
 
 	for_each_sched_entity(se) {
-		cfs_rq = cfs_rq_of(se);
+		struct cfs_rq *cfs_rq = cfs_rq_of(se);
 
-		if (!dequeue_entity(cfs_rq, se, flags)) {
-			if (p && &p->se == se)
-				return -1;
+		update_curr(cfs_rq);
 
-			slice = cfs_rq_min_slice(cfs_rq);
-			break;
+		if (dequeue) {
+			dequeue_entity(cfs_rq, se, flags);
+			/* Don't dequeue parent if it has other entities besides us */
+			if (cfs_rq->load.weight)
+				dequeue = false;
+		} else {
+			update_load_avg(cfs_rq, se, UPDATE_TG);
+			se_update_runnable(se);
+			update_cfs_group(se);
 		}
 
 		cfs_rq->h_nr_runnable -= h_nr_runnable;
-		cfs_rq->h_nr_queued -= h_nr_queued;
+		cfs_rq->h_nr_queued--;
 		cfs_rq->h_nr_idle -= h_nr_idle;
 
 		if (cfs_rq_is_idle(cfs_rq))
-			h_nr_idle = h_nr_queued;
+			h_nr_idle = 1;
 
 		if (throttled_hierarchy(cfs_rq) && task_throttled)
 			record_throttle_clock(cfs_rq);
 
-		/* Don't dequeue parent if it has other entities besides us */
-		if (cfs_rq->load.weight) {
-			slice = cfs_rq_min_slice(cfs_rq);
-
-			/* Avoid re-evaluating load for this entity: */
-			se = parent_entity(se);
-			/*
-			 * Bias pick_next to pick a task from this cfs_rq, as
-			 * p is sleeping when it is within its sched_slice.
-			 */
-			if (task_sleep && se)
-				set_next_buddy(se);
-			break;
-		}
 		flags |= DEQUEUE_SLEEP;
 		flags &= ~(DEQUEUE_DELAYED | DEQUEUE_SPECIAL);
 	}
+}
 
-	for_each_sched_entity(se) {
-		cfs_rq = cfs_rq_of(se);
+/*
+ * The part of dequeue_task_fair() that is needed to dequeue delayed tasks.
+ *
+ * Returns:
+ *   true  - dequeued
+ *   false - delayed
+ */
+static bool __dequeue_task(struct rq *rq, struct task_struct *p, int flags)
+{
+	struct sched_entity *se = &p->se;
+	struct cfs_rq *cfs_rq = &rq->cfs;
+	bool was_sched_idle = sched_idle_rq(rq);
+	bool task_sleep = flags & DEQUEUE_SLEEP;
+	bool task_delayed = flags & DEQUEUE_DELAYED;
 
-		update_load_avg(cfs_rq, se, UPDATE_TG);
-		se_update_runnable(se);
-		update_cfs_group(se);
+	clear_buddies(cfs_rq, se);
 
-		se->slice = slice;
-		if (se != cfs_rq->curr)
-			min_vruntime_cb_propagate(&se->run_node, NULL);
-		slice = cfs_rq_min_slice(cfs_rq);
+	update_curr(cfs_rq_of(se));
+	update_entity_lag(cfs_rq, se);
 
-		cfs_rq->h_nr_runnable -= h_nr_runnable;
-		cfs_rq->h_nr_queued -= h_nr_queued;
-		cfs_rq->h_nr_idle -= h_nr_idle;
+	if (flags & DEQUEUE_DELAYED) {
+		WARN_ON_ONCE(!se->sched_delayed);
+	} else {
+		bool delay = task_sleep;
+		/*
+		 * DELAY_DEQUEUE relies on spurious wakeups, special task
+		 * states must not suffer spurious wakeups, excempt them.
+		 */
+		if (flags & (DEQUEUE_SPECIAL | DEQUEUE_THROTTLE))
+			delay = false;
 
-		if (cfs_rq_is_idle(cfs_rq))
-			h_nr_idle = h_nr_queued;
+		WARN_ON_ONCE(delay && se->sched_delayed);
 
-		if (throttled_hierarchy(cfs_rq) && task_throttled)
-			record_throttle_clock(cfs_rq);
+		if (sched_feat(DELAY_DEQUEUE) && delay &&
+		    !entity_eligible(cfs_rq, se)) {
+			update_load_avg(cfs_rq_of(se), se, UPDATE_UTIL_EST);
+			set_delayed(se);
+			return false;
+		}
+	}
+
+	dequeue_hierarchy(p, flags);
+
+	if (sched_feat(PLACE_REL_DEADLINE) && !task_sleep) {
+		se->deadline -= se->vruntime;
+		se->rel_deadline = 1;
 	}
+	if (se != cfs_rq->curr)
+		__dequeue_entity(cfs_rq, se);
 
-	sub_nr_running(rq, h_nr_queued);
+	sub_nr_running(rq, 1);
 
 	/* balance early to pull high priority tasks */
 	if (unlikely(!was_sched_idle && sched_idle_rq(rq)))
 		rq->next_balance = jiffies;
 
-	if (p && task_delayed) {
+	if (task_delayed) {
+		clear_delayed(se);
+
 		WARN_ON_ONCE(!task_sleep);
 		WARN_ON_ONCE(p->on_rq != 1);
 
@@ -8025,7 +8158,7 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags)
 		__block_task(rq, p);
 	}
 
-	return 1;
+	return true;
 }
 
 /*
@@ -8043,11 +8176,11 @@ static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 	if (!p->se.sched_delayed)
 		util_est_dequeue(&rq->cfs, p);
 
-	if (dequeue_entities(rq, &p->se, flags) < 0)
+	if (!__dequeue_task(rq, p, flags))
 		return false;
 
 	/*
-	 * Must not reference @p after dequeue_entities(DEQUEUE_DELAYED).
+	 * Must not reference @p after __dequeue_task(DEQUEUE_DELAYED).
 	 */
 	return true;
 }
@@ -9642,19 +9775,6 @@ static void migrate_task_rq_fair(struct task_struct *p, int new_cpu)
 static void task_dead_fair(struct task_struct *p)
 {
 	struct sched_entity *se = &p->se;
-
-	if (se->sched_delayed) {
-		struct rq_flags rf;
-		struct rq *rq;
-
-		rq = task_rq_lock(p, &rf);
-		if (se->sched_delayed) {
-			update_rq_clock(rq);
-			dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED);
-		}
-		task_rq_unlock(rq, p, &rf);
-	}
-
 	remove_entity_load_avg(se);
 }
 
@@ -9688,27 +9808,14 @@ static void set_cpus_allowed_fair(struct task_struct *p, struct affinity_context
 	set_task_max_allowed_capacity(p);
 }
 
-static void set_next_buddy(struct sched_entity *se)
-{
-	for_each_sched_entity(se) {
-		if (WARN_ON_ONCE(!se->on_rq))
-			return;
-		if (se_is_idle(se))
-			return;
-		cfs_rq_of(se)->next = se;
-	}
-}
-
 enum preempt_wakeup_action {
 	PREEMPT_WAKEUP_NONE,	/* No preemption. */
 	PREEMPT_WAKEUP_SHORT,	/* Ignore slice protection. */
-	PREEMPT_WAKEUP_PICK,	/* Let __pick_eevdf() decide. */
+	PREEMPT_WAKEUP_PICK,	/* Let pick_eevdf() decide. */
 	PREEMPT_WAKEUP_RESCHED,	/* Force reschedule. */
 };
 
-static inline bool
-set_preempt_buddy(struct cfs_rq *cfs_rq, int wake_flags,
-		  struct sched_entity *pse, struct sched_entity *se)
+static inline bool set_preempt_buddy(struct cfs_rq *cfs_rq, struct sched_entity *pse)
 {
 	/*
 	 * Keep existing buddy if the deadline is sooner than pse.
@@ -9719,7 +9826,16 @@ set_preempt_buddy(struct cfs_rq *cfs_rq, int wake_flags,
 	if (cfs_rq->next && entity_before(cfs_rq->next, pse))
 		return false;
 
-	set_next_buddy(pse);
+	set_next_buddy(cfs_rq, pse);
+	return true;
+}
+
+static inline bool set_short_buddy(struct cfs_rq *cfs_rq, struct sched_entity *pse)
+{
+	if (cfs_rq->next && cfs_rq->next->slice < pse->slice)
+		return false;
+
+	set_next_buddy(cfs_rq, pse);
 	return true;
 }
 
@@ -9772,13 +9888,14 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f
 	enum preempt_wakeup_action preempt_action = PREEMPT_WAKEUP_PICK;
 	struct task_struct *donor = rq->donor;
 	struct sched_entity *nse, *se = &donor->se, *pse = &p->se;
-	struct cfs_rq *cfs_rq = task_cfs_rq(donor);
+	struct cfs_rq *cfs_rq = &rq->cfs;
 	int cse_is_idle, pse_is_idle;
 
 	/*
 	 * XXX Getting preempted by higher class, try and find idle CPU?
 	 */
-	if (p->sched_class != &fair_sched_class)
+	if (p->sched_class != &fair_sched_class ||
+	    donor->sched_class != &fair_sched_class)
 		return;
 
 	if (unlikely(se == pse))
@@ -9803,42 +9920,43 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f
 	 * prevents us from potentially nominating it as a false LAST_BUDDY
 	 * below.
 	 */
-	if (test_tsk_need_resched(rq->curr))
+	if (!sched_feat(PREEMPT_SHORT) && test_tsk_need_resched(rq->curr))
 		return;
 
 	if (!sched_feat(WAKEUP_PREEMPTION))
 		return;
 
-	find_matching_se(&se, &pse);
 	WARN_ON_ONCE(!pse);
 
 	cse_is_idle = se_is_idle(se);
 	pse_is_idle = se_is_idle(pse);
 
+	nse = se;
 	/*
 	 * Preempt an idle entity in favor of a non-idle entity (and don't preempt
 	 * in the inverse case).
 	 */
-	if (cse_is_idle && !pse_is_idle) {
-		/*
-		 * When non-idle entity preempt an idle entity,
-		 * don't give idle entity slice protection.
-		 */
-		preempt_action = PREEMPT_WAKEUP_SHORT;
+	if (cse_is_idle && !pse_is_idle)
 		goto preempt;
-	}
+
+	update_curr_fair(rq);
 
 	if (cse_is_idle != pse_is_idle)
-		return;
+		goto update;
 
 	/*
 	 * BATCH and IDLE tasks do not preempt others.
 	 */
 	if (unlikely(!normal_policy(p->policy)))
-		return;
+		goto update;
+
+	/*
+	 * Do not preempt for tasks that are sched_delayed as it would violate
+	 * EEVDF to forcibly queue an ineligible task.
+	 */
+	if (pse->sched_delayed)
+		goto update;
 
-	cfs_rq = cfs_rq_of(se);
-	update_curr(cfs_rq);
 	/*
 	 * If @p has a shorter slice than current and @p is eligible, override
 	 * current's slice protection in order to allow preemption.
@@ -9850,17 +9968,13 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f
 
 	/*
 	 * Ignore wakee preemption on WF_FORK as it is less likely that
-	 * there is shared data as exec often follow fork. Do not
-	 * preempt for tasks that are sched_delayed as it would violate
-	 * EEVDF to forcibly queue an ineligible task.
+	 * there is shared data as exec often follow fork.
 	 */
-	if ((wake_flags & WF_FORK) || pse->sched_delayed)
-		return;
+	if (wake_flags & WF_FORK)
+		goto update;
 
 	/* Prefer picking wakee soon if appropriate. */
-	if (sched_feat(NEXT_BUDDY) &&
-	    set_preempt_buddy(cfs_rq, wake_flags, pse, se)) {
-
+	if (sched_feat(NEXT_BUDDY) && set_preempt_buddy(cfs_rq, pse)) {
 		/*
 		 * Decide whether to obey WF_SYNC hint for a new buddy. Old
 		 * buddies are ignored as they may not be relevant to the
@@ -9882,29 +9996,33 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f
 	}
 
 pick:
-	nse = pick_next_entity(rq, cfs_rq, preempt_action != PREEMPT_WAKEUP_SHORT);
-	/* If @p has become the most eligible task, force preemption */
-	if (nse == pse)
-		goto preempt;
+	if (cfs_rq->h_nr_queued) {
+		nse = pick_next_entity(rq, preempt_action != PREEMPT_WAKEUP_SHORT);
+		if (unlikely(!nse))
+			goto pick;
+
+		/* If @p has become the most eligible task, force preemption */
+		if (nse == pse)
+			goto preempt;
+	}
 
 	/*
-	 * Because p is enqueued, nse being null can only mean that we
-	 * dequeued a delayed task. If there are still entities queued in
-	 * cfs, check if the next one will be p.
+	 * If @p is eligible but not the next task to run then cancel protection
+	 * to prevent large scheduling latency
 	 */
-	if (!nse && cfs_rq->nr_queued)
-		goto pick;
-
+	if (preempt_action == PREEMPT_WAKEUP_SHORT && entity_eligible(cfs_rq, pse))
+		goto preempt;
+update:
 	if (sched_feat(RUN_TO_PARITY))
 		update_protect_slice(cfs_rq, se);
 
 	return;
 
 preempt:
-	if (preempt_action == PREEMPT_WAKEUP_SHORT) {
-		cancel_protect_slice(se);
-		clear_buddies(cfs_rq, se);
-	}
+	cancel_protect_slice(se);
+
+	if (preempt_action == PREEMPT_WAKEUP_SHORT)
+		set_short_buddy(cfs_rq, pse);
 
 	resched_curr_lazy(rq);
 }
@@ -9912,33 +10030,24 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f
 struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf)
 	__must_hold(__rq_lockp(rq))
 {
+	struct cfs_rq *cfs_rq = &rq->cfs;
 	struct sched_entity *se;
-	struct cfs_rq *cfs_rq;
 	struct task_struct *p;
-	bool throttled;
 	int new_tasks;
 
 again:
-	cfs_rq = &rq->cfs;
-	if (!cfs_rq->nr_queued)
+	if (!cfs_rq->h_nr_queued)
 		goto idle;
 
-	throttled = false;
-
-	do {
-		/* Might not have done put_prev_entity() */
-		if (cfs_rq->curr && cfs_rq->curr->on_rq)
-			update_curr(cfs_rq);
+	/* Might not have done put_prev_entity() */
+	if (cfs_rq->curr && cfs_rq->curr->on_rq)
+		update_curr(cfs_rq);
 
-		se = pick_next_entity(rq, cfs_rq, true);
-		if (!se)
-			goto again;
-		cfs_rq = group_cfs_rq(se);
-	} while (cfs_rq);
+	se = pick_next_entity(rq, true);
+	if (!se)
+		goto again;
 
 	p = task_of(se);
-	if (unlikely(throttled))
-		task_throttle_setup_work(p);
 	return p;
 
 idle:
@@ -9975,7 +10084,7 @@ void fair_server_init(struct rq *rq)
 static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct task_struct *next)
 {
 	struct sched_entity *se = &prev->se;
-	struct cfs_rq *cfs_rq;
+	struct cfs_rq *cfs_rq = &rq->cfs;
 	struct sched_entity *nse = NULL;
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
@@ -9985,7 +10094,7 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t
 
 	while (se) {
 		cfs_rq = cfs_rq_of(se);
-		if (!nse || cfs_rq->curr)
+		if (!nse || cfs_rq->h_curr)
 			put_prev_entity(cfs_rq, se);
 #ifdef CONFIG_FAIR_GROUP_SCHED
 		if (nse) {
@@ -10004,6 +10113,14 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t
 #endif
 		se = parent_entity(se);
 	}
+
+	/* Put 'current' back into the tree. */
+	cfs_rq = &rq->cfs;
+	se = &prev->se;
+	WARN_ON_ONCE(cfs_rq->curr != se);
+	cfs_rq->curr = NULL;
+	if (se->on_rq)
+		__enqueue_entity(cfs_rq, se);
 }
 
 /*
@@ -10012,8 +10129,8 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t
 static void yield_task_fair(struct rq *rq)
 {
 	struct task_struct *curr = rq->donor;
-	struct cfs_rq *cfs_rq = task_cfs_rq(curr);
 	struct sched_entity *se = &curr->se;
+	struct cfs_rq *cfs_rq = &rq->cfs;
 
 	/*
 	 * Are we the only task in the tree?
@@ -10054,11 +10171,11 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p)
 	struct sched_entity *se = &p->se;
 
 	/* !se->on_rq also covers throttled task */
-	if (!se->on_rq)
+	if (!se->on_rq || se->sched_delayed)
 		return false;
 
 	/* Tell the scheduler that we'd really like se to run next. */
-	set_next_buddy(se);
+	set_next_buddy(&task_rq(p)->cfs, se);
 
 	yield_task_fair(rq);
 
@@ -10397,15 +10514,10 @@ static inline long migrate_degrades_locality(struct task_struct *p,
  */
 static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_cpu)
 {
-	struct cfs_rq *dst_cfs_rq;
+	struct cfs_rq *dst_cfs_rq = &cpu_rq(dest_cpu)->cfs;
 
-#ifdef CONFIG_FAIR_GROUP_SCHED
-	dst_cfs_rq = tg_cfs_rq(task_group(p), dest_cpu);
-#else
-	dst_cfs_rq = &cpu_rq(dest_cpu)->cfs;
-#endif
-	if (sched_feat(PLACE_LAG) && dst_cfs_rq->nr_queued &&
-	    !entity_eligible(task_cfs_rq(p), &p->se))
+	if (sched_feat(PLACE_LAG) && dst_cfs_rq->h_nr_queued &&
+	    !entity_eligible(&task_rq(p)->cfs, &p->se))
 		return 1;
 
 	return 0;
@@ -11188,7 +11300,7 @@ static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)
 	while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) {
 		load = cfs_rq->h_load;
 		load = div64_ul(load * se->avg.load_avg,
-			cfs_rq_load_avg(cfs_rq) + 1);
+				cfs_rq_load_avg(cfs_rq) + 1);
 		cfs_rq = group_cfs_rq(se);
 		cfs_rq->h_load = load;
 		cfs_rq->last_h_load_update = now;
@@ -13437,75 +13549,78 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq,
 		}
 	}
 
-	if (!ld_moved) {
-		schedstat_inc(sd->lb_failed[idle]);
-		/*
-		 * Increment the failure counter only on periodic balance.
-		 * We do not want newidle balance, which can be very
-		 * frequent, pollute the failure counter causing
-		 * excessive cache_hot migrations and active balances.
-		 *
-		 * Similarly for migration_misfit which is not related to
-		 * load/util migration, don't pollute nr_balance_failed.
-		 *
-		 * The same for cache aware scheduling's allowance for
-		 * load imbalance. If regular load balance does not
-		 * migrate task due to LLC locality, it is a expected
-		 * behavior and don't pollute nr_balance_failed.
-		 * See can_migrate_task().
-		 */
-		if (idle != CPU_NEWLY_IDLE &&
-		    env.migration_type != migrate_misfit &&
-		    !(env.flags & LBF_LLC_PINNED))
-			sd->nr_balance_failed++;
+	if (ld_moved) {
+		sd->nr_balance_failed = 0;
+		goto out_unbalanced;
+	}
 
-		if (need_active_balance(&env)) {
-			unsigned long flags;
+	schedstat_inc(sd->lb_failed[idle]);
+	/*
+	 * Increment the failure counter only on periodic balance.
+	 * We do not want newidle balance, which can be very
+	 * frequent, pollute the failure counter causing
+	 * excessive cache_hot migrations and active balances.
+	 *
+	 * Similarly for migration_misfit which is not related to
+	 * load/util migration, don't pollute nr_balance_failed.
+	 *
+	 * The same for cache aware scheduling's allowance for
+	 * load imbalance. If regular load balance does not
+	 * migrate task due to LLC locality, it is a expected
+	 * behavior and don't pollute nr_balance_failed.
+	 * See can_migrate_task().
+	 */
+	if (idle != CPU_NEWLY_IDLE &&
+	    env.migration_type != migrate_misfit &&
+	    !(env.flags & LBF_LLC_PINNED))
+		sd->nr_balance_failed++;
 
-			raw_spin_rq_lock_irqsave(busiest, flags);
+	if (!need_active_balance(&env))
+		goto out_unbalanced;
 
-			/*
-			 * Don't kick the active_load_balance_cpu_stop,
-			 * if the curr task on busiest CPU can't be
-			 * moved to this_cpu:
-			 */
-			if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) {
-				raw_spin_rq_unlock_irqrestore(busiest, flags);
-				goto out_one_pinned;
-			}
+	scoped_guard (raw_spin_rq_lock_irqsave, busiest) {
+		/*
+		 * Don't kick the active_load_balance_cpu_stop,
+		 * if the curr task on busiest CPU can't be
+		 * moved to this_cpu:
+		 */
+		if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr))
+			goto out_one_pinned;
 
-			/* Record that we found at least one task that could run on this_cpu */
-			env.flags &= ~LBF_ALL_PINNED;
+		/* Record that we found at least one task that could run on this_cpu */
+		env.flags &= ~LBF_ALL_PINNED;
 
-			/*
-			 * ->active_balance synchronizes accesses to
-			 * ->active_balance_work.  Once set, it's cleared
-			 * only after active load balance is finished.
-			 */
-			if (!busiest->active_balance) {
-				busiest->active_balance = 1;
-				busiest->push_cpu = this_cpu;
-				active_balance = 1;
-			}
+		/*
+		 * ->active_balance synchronizes accesses to
+		 * ->active_balance_work.  Once set, it's cleared
+		 * only after active load balance is finished.
+		 */
+		if (busiest->active_balance)
+			goto out_unbalanced;
 
-			preempt_disable();
-			raw_spin_rq_unlock_irqrestore(busiest, flags);
-			if (active_balance) {
-				stop_one_cpu_nowait(cpu_of(busiest),
-					active_load_balance_cpu_stop, busiest,
-					&busiest->active_balance_work);
-			}
-			preempt_enable();
-		}
-	} else {
-		sd->nr_balance_failed = 0;
-	}
+		/*
+		 * @busiest dropped its rq_lock in the middle of
+		 * scheduling out its ->curr task (->on_rq := 0), no
+		 * need to forcefully punt it away with active balance.
+		 */
+		if (!busiest->curr->on_rq)
+			goto out_unbalanced;
 
-	if (likely(!active_balance) || need_active_balance(&env)) {
-		/* We were unbalanced, so reset the balancing interval */
-		sd->balance_interval = sd->min_interval;
+		busiest->active_balance = 1;
+		busiest->push_cpu = this_cpu;
+		active_balance = 1;
+		preempt_disable();
+	}
+	if (active_balance) {
+		stop_one_cpu_nowait(cpu_of(busiest),
+				    active_load_balance_cpu_stop, busiest,
+				    &busiest->active_balance_work);
 	}
+	preempt_enable();
 
+out_unbalanced:
+	/* We were unbalanced, so reset the balancing interval */
+	sd->balance_interval = sd->min_interval;
 	goto out;
 
 out_balanced:
@@ -14571,7 +14686,7 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr)
 	 * if the sibling is forced idle, then trigger schedule to
 	 * give forced idle task a chance.
 	 *
-	 * sched_slice() considers only this active rq and it gets the
+	 * __entity_slice_used() considers only this active rq and it gets the
 	 * whole slice. But during force idle, we have siblings acting
 	 * like a single runqueue and hence we need to consider runnable
 	 * tasks on this CPU and the forced idle CPU. Ideally, we should
@@ -14580,7 +14695,7 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr)
 	 * MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check
 	 * if we need to give up the CPU.
 	 */
-	if (rq->core->core_forceidle_count && rq->cfs.nr_queued == 1 &&
+	if (rq->core->core_forceidle_count && rq->cfs.h_nr_queued == 1 &&
 	    __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE))
 		resched_curr(rq);
 }
@@ -14789,30 +14904,8 @@ bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b,
 
 	WARN_ON_ONCE(task_rq(b)->core != rq->core);
 
-#ifdef CONFIG_FAIR_GROUP_SCHED
-	/*
-	 * Find an se in the hierarchy for tasks a and b, such that the se's
-	 * are immediate siblings.
-	 */
-	while (sea->cfs_rq->tg != seb->cfs_rq->tg) {
-		int sea_depth = sea->depth;
-		int seb_depth = seb->depth;
-
-		if (sea_depth >= seb_depth)
-			sea = parent_entity(sea);
-		if (sea_depth <= seb_depth)
-			seb = parent_entity(seb);
-	}
-
-	se_fi_update(sea, rq->core->core_forceidle_seq, in_fi);
-	se_fi_update(seb, rq->core->core_forceidle_seq, in_fi);
-
-	cfs_rqa = sea->cfs_rq;
-	cfs_rqb = seb->cfs_rq;
-#else /* !CONFIG_FAIR_GROUP_SCHED: */
 	cfs_rqa = &task_rq(a)->cfs;
 	cfs_rqb = &task_rq(b)->cfs;
-#endif /* !CONFIG_FAIR_GROUP_SCHED */
 
 	/*
 	 * Find delta after normalizing se's vruntime with its cfs_rq's
@@ -14851,11 +14944,20 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {}
 static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
 {
 	struct sched_entity *se = &curr->se;
-	struct cfs_rq *cfs_rq;
 
-	for_each_sched_entity(se) {
-		cfs_rq = cfs_rq_of(se);
-		entity_tick(cfs_rq, se, queued);
+	if (se->on_rq) {
+		unsigned long weight = NICE_0_LOAD;
+		struct cfs_rq *cfs_rq;
+
+		for_each_sched_entity(se) {
+			cfs_rq = cfs_rq_of(se);
+			entity_tick(cfs_rq, se, queued);
+
+			weight = __calc_prop_weight(cfs_rq, se, weight);
+		}
+
+		se = &curr->se;
+		reweight_eevdf(cfs_rq, se, weight, se->on_rq);
 	}
 
 	if (queued)
@@ -14895,7 +14997,7 @@ prio_changed_fair(struct rq *rq, struct task_struct *p, u64 oldprio)
 	if (p->prio == oldprio)
 		return;
 
-	if (rq->cfs.nr_queued == 1)
+	if (rq->cfs.h_nr_queued == 1)
 		return;
 
 	/*
@@ -15024,33 +15126,44 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p)
 	}
 }
 
-/*
- * Account for a task changing its policy or group.
- *
- * This routine is mostly called to set cfs_rq->curr field when a task
- * migrates between groups/classes.
- */
 static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)
 {
 	struct sched_entity *se = &p->se;
 	bool throttled = false;
+	struct cfs_rq *cfs_rq = &rq->cfs;
+	unsigned long weight = NICE_0_LOAD;
+	bool on_rq = se->on_rq;
+
+	clear_buddies(cfs_rq, se);
+
+	if (on_rq)
+		__dequeue_entity(cfs_rq, se);
 
 	for_each_sched_entity(se) {
-		struct cfs_rq *cfs_rq = cfs_rq_of(se);
+		cfs_rq = cfs_rq_of(se);
 
-		if (IS_ENABLED(CONFIG_FAIR_GROUP_SCHED) &&
-		    first && cfs_rq->curr)
-			break;
+		if (!IS_ENABLED(CONFIG_FAIR_GROUP_SCHED) ||
+		    !first || !cfs_rq->h_curr)
+			set_next_entity(cfs_rq, se);
 
-		set_next_entity(cfs_rq, se, first);
 		/* ensure bandwidth has been allocated on our new cfs_rq */
 		throttled |= account_cfs_rq_runtime(cfs_rq, 0);
+
+		if (on_rq)
+			weight = __calc_prop_weight(cfs_rq, se, weight);
 	}
 
 	if (throttled)
 		task_throttle_setup_work(p);
 
 	se = &p->se;
+	cfs_rq->curr = se;
+
+	if (on_rq) {
+		reweight_eevdf(cfs_rq, se, weight, se->on_rq);
+		if (first)
+			set_protect_slice(cfs_rq, se);
+	}
 
 	if (task_on_rq_queued(p)) {
 		/*
@@ -15163,17 +15276,8 @@ void unregister_fair_sched_group(struct task_group *tg)
 		struct sched_entity *se = tg_se(tg, cpu);
 		struct rq *rq = cpu_rq(cpu);
 
-		if (se) {
-			if (se->sched_delayed) {
-				guard(rq_lock_irqsave)(rq);
-				if (se->sched_delayed) {
-					update_rq_clock(rq);
-					dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED);
-				}
-				list_del_leaf_cfs_rq(cfs_rq);
-			}
+		if (se)
 			remove_entity_load_avg(se);
-		}
 
 		/*
 		 * Only empty task groups can be destroyed; so we can speculatively
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index 052435f4d3e3c..eb73b65ce6c47 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -503,7 +503,13 @@ static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool fir
 
 struct task_struct *pick_task_idle(struct rq *rq, struct rq_flags *rf)
 {
-	scx_update_idle(rq, true, false);
+	/*
+	 * Notify scx only on an idle-to-idle re-pick (the cpu was already idle).
+	 * A real task->idle transition is delivered by set_next_task_idle(), so
+	 * calling here too would duplicate it.
+	 */
+	if (scx_enabled() && is_idle_task(rq->curr))
+		scx_update_idle(rq, true, false);
 	return rq->idle;
 }
 
diff --git a/kernel/sched/pelt.c b/kernel/sched/pelt.c
index 897790889ba38..779eb58a42614 100644
--- a/kernel/sched/pelt.c
+++ b/kernel/sched/pelt.c
@@ -206,7 +206,7 @@ ___update_load_sum(u64 now, struct sched_avg *sa,
 	/*
 	 * running is a subset of runnable (weight) so running can't be set if
 	 * runnable is clear. But there are some corner cases where the current
-	 * se has been already dequeued but cfs_rq->curr still points to it.
+	 * se has been already dequeued but cfs_rq->h_curr still points to it.
 	 * This means that weight will be 0 but not running for a sched_entity
 	 * but also for a cfs_rq if the latter becomes idle. As an example,
 	 * this happens during sched_balance_newidle() which calls
@@ -307,7 +307,7 @@ int __update_load_avg_blocked_se(u64 now, struct sched_entity *se)
 int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
 	if (___update_load_sum(now, &se->avg, !!se->on_rq, se_runnable(se),
-				cfs_rq->curr == se)) {
+				cfs_rq->h_curr == se)) {
 
 		___update_load_avg(&se->avg, se_weight(se));
 		cfs_se_util_change(&se->avg);
@@ -323,7 +323,7 @@ int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq)
 	if (___update_load_sum(now, &cfs_rq->avg,
 				scale_load_down(cfs_rq->load.weight),
 				cfs_rq->h_nr_runnable,
-				cfs_rq->curr != NULL)) {
+				cfs_rq->h_curr != NULL)) {
 
 		___update_load_avg(&cfs_rq->avg, 1);
 		trace_pelt_cfs_tp(cfs_rq);
diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c
index d9c9d9480a45b..4e152410653de 100644
--- a/kernel/sched/psi.c
+++ b/kernel/sched/psi.c
@@ -1023,7 +1023,7 @@ void psi_account_irqtime(struct rq *rq, struct task_struct *curr, struct task_st
 
 	irq = irq_time_read(cpu);
 	delta = (s64)(irq - rq->psi_irq_time);
-	if (delta < 0)
+	if (delta <= 0)
 		return;
 	rq->psi_irq_time = irq;
 
@@ -1134,6 +1134,12 @@ void psi_cgroup_free(struct cgroup *cgroup)
 		return;
 
 	cancel_delayed_work_sync(&cgroup->psi->avgs_work);
+	/*
+	 * A psi_schedule_rtpoll_work() call racing the last trigger's
+	 * destruction may have re-armed the timer after psi_trigger_destroy()
+	 * deleted it. Spurious firing while the group is alive is harmless.
+	 */
+	timer_shutdown_sync(&cgroup->psi->rtpoll_timer);
 	free_percpu(cgroup->psi->pcpu);
 	/* All triggers must be removed by now */
 	WARN_ONCE(cgroup->psi->rtpoll_states, "psi: trigger leak\n");
@@ -1292,9 +1298,44 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res)
 	return 0;
 }
 
+/*
+ * Create @group's rtpoll worker after psi_trigger_create() reported the need
+ * for one. kthread creation depends on the whole fork path and we don't want
+ * all of that nested inside cgroup_mutex, so the caller must drop it and any
+ * other lock that forks can wait behind. If two callers race, the loser stops
+ * its never-woken kthread.
+ */
+int psi_trigger_create_rtpoll_worker(struct psi_group *group)
+{
+	struct task_struct *task;
+
+	task = kthread_create(psi_rtpoll_worker, group, "psimon");
+	if (IS_ERR(task))
+		return PTR_ERR(task);
+
+	scoped_guard(mutex, &group->rtpoll_trigger_lock) {
+		if (!rcu_access_pointer(group->rtpoll_task)) {
+			atomic_set(&group->rtpoll_wakeup, 0);
+			wake_up_process(task);
+			rcu_assign_pointer(group->rtpoll_task, task);
+
+			/*
+			 * Poll once to catch up on scheduling attempts dropped
+			 * while there was no rtpoll worker.
+			 */
+			psi_schedule_rtpoll_work(group, 1, true);
+			return 0;
+		}
+	}
+
+	kthread_stop(task);
+	return 0;
+}
+
 struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf,
 				       enum psi_res res, struct file *file,
-				       struct kernfs_open_file *of)
+				       struct kernfs_open_file *of,
+				       bool *need_rtpoll_worker)
 {
 	struct psi_trigger *t;
 	enum psi_states state;
@@ -1302,6 +1343,8 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf,
 	bool privileged;
 	u32 window_us;
 
+	*need_rtpoll_worker = false;
+
 	if (static_branch_likely(&psi_disabled))
 		return ERR_PTR(-EOPNOTSUPP);
 
@@ -1362,26 +1405,14 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf,
 	if (privileged) {
 		mutex_lock(&group->rtpoll_trigger_lock);
 
-		if (!rcu_access_pointer(group->rtpoll_task)) {
-			struct task_struct *task;
-
-			task = kthread_create(psi_rtpoll_worker, group, "psimon");
-			if (IS_ERR(task)) {
-				kfree(t);
-				mutex_unlock(&group->rtpoll_trigger_lock);
-				return ERR_CAST(task);
-			}
-			atomic_set(&group->rtpoll_wakeup, 0);
-			wake_up_process(task);
-			rcu_assign_pointer(group->rtpoll_task, task);
-		}
-
 		list_add(&t->node, &group->rtpoll_triggers);
 		group->rtpoll_min_period = min(group->rtpoll_min_period,
 			div_u64(t->win.size, UPDATES_PER_WINDOW));
 		group->rtpoll_nr_triggers[t->state]++;
 		group->rtpoll_states |= (1 << t->state);
 
+		*need_rtpoll_worker = !rcu_access_pointer(group->rtpoll_task);
+
 		mutex_unlock(&group->rtpoll_trigger_lock);
 	} else {
 		mutex_lock(&group->avgs_lock);
@@ -1541,6 +1572,8 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf,
 	size_t buf_size;
 	struct seq_file *seq;
 	struct psi_trigger *new;
+	bool need_rtpoll_worker;
+	int ret;
 
 	if (static_branch_likely(&psi_disabled))
 		return -EOPNOTSUPP;
@@ -1565,12 +1598,22 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf,
 		return -EBUSY;
 	}
 
-	new = psi_trigger_create(&psi_system, buf, res, file, NULL);
+	new = psi_trigger_create(&psi_system, buf, res, file, NULL,
+				 &need_rtpoll_worker);
 	if (IS_ERR(new)) {
 		mutex_unlock(&seq->lock);
 		return PTR_ERR(new);
 	}
 
+	if (need_rtpoll_worker) {
+		ret = psi_trigger_create_rtpoll_worker(&psi_system);
+		if (ret) {
+			psi_trigger_destroy(new);
+			mutex_unlock(&seq->lock);
+			return ret;
+		}
+	}
+
 	smp_store_release(&seq->private, new);
 	mutex_unlock(&seq->lock);
 
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index e474c31d8fe66..e6e5f8a2caafb 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -1629,7 +1629,8 @@ static void wakeup_preempt_rt(struct rq *rq, struct task_struct *p, int flags)
 	/*
 	 * XXX If we're preempted by DL, queue a push?
 	 */
-	if (p->sched_class != &rt_sched_class)
+	if (p->sched_class != &rt_sched_class ||
+	    donor->sched_class != &rt_sched_class)
 		return;
 
 	if (p->prio < donor->prio) {
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 56acf502ba260..e3e4e68bb0c5c 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -493,6 +493,8 @@ struct task_group {
 	 * will also be accessed at each tick.
 	 */
 	atomic_long_t		load_avg ____cacheline_aligned;
+	atomic_long_t		runnable_avg;
+
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 
 #ifdef CONFIG_RT_GROUP_SCHED
@@ -528,21 +530,8 @@ struct task_group {
 
 };
 
-#ifdef CONFIG_GROUP_SCHED_WEIGHT
 #define ROOT_TASK_GROUP_LOAD	NICE_0_LOAD
 
-/*
- * A weight of 0 or 1 can cause arithmetics problems.
- * A weight of a cfs_rq is the sum of weights of which entities
- * are queued on this cfs_rq, so a weight of a entity should not be
- * too large, so as the shares value of a task group.
- * (The default weight is 1024 - so there's no practical
- *  limitation from this.)
- */
-#define MIN_SHARES		(1UL <<  1)
-#define MAX_SHARES		(1UL << 18)
-#endif
-
 typedef int (*tg_visitor)(struct task_group *, void *);
 
 extern int walk_tg_tree_from(struct task_group *from,
@@ -571,6 +560,7 @@ extern void free_fair_sched_group(struct task_group *tg);
 extern int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent);
 extern void online_fair_sched_group(struct task_group *tg);
 extern void unregister_fair_sched_group(struct task_group *tg);
+extern void __sched_cgroup_mode_update(int mode);
 #else /* !CONFIG_FAIR_GROUP_SCHED: */
 static inline void free_fair_sched_group(struct task_group *tg) { }
 static inline int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
@@ -628,6 +618,17 @@ static inline bool cfs_task_bw_constrained(struct task_struct *p) { return false
 
 #endif /* !CONFIG_CGROUP_SCHED */
 
+/*
+ * A weight of 0 or 1 can cause arithmetics problems.
+ * A weight of a cfs_rq is the sum of weights of which entities
+ * are queued on this cfs_rq, so a weight of a entity should not be
+ * too large, so as the shares value of a task group.
+ * (The default weight is 1024 - so there's no practical
+ *  limitation from this.)
+ */
+#define MIN_SHARES		(1UL <<  1)
+#define MAX_SHARES		(1UL << 18)
+
 extern void unregister_rt_sched_group(struct task_group *tg);
 extern void free_rt_sched_group(struct task_group *tg);
 extern int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent);
@@ -706,6 +707,7 @@ struct cfs_rq {
 	/*
 	 * CFS load tracking
 	 */
+	struct sched_entity	*h_curr;
 	struct sched_avg	avg;
 #ifndef CONFIG_64BIT
 	u64			last_update_time_copy;
@@ -721,6 +723,7 @@ struct cfs_rq {
 #ifdef CONFIG_FAIR_GROUP_SCHED
 	u64			last_update_tg_load_avg;
 	unsigned long		tg_load_avg_contrib;
+	unsigned long		tg_runnable_avg_contrib;
 	long			propagate;
 	long			prop_runnable_sum;
 
@@ -787,6 +790,8 @@ enum scx_rq_flags {
 	SCX_RQ_BAL_KEEP		= 1 << 3, /* balance decided to keep current */
 	SCX_RQ_CLK_VALID	= 1 << 5, /* RQ clock is fresh and valid */
 	SCX_RQ_BAL_CB_PENDING	= 1 << 6, /* must queue a cb after dispatching */
+	SCX_RQ_SUB_IDLE_RENOTIFY	= 1 << 7, /* sub-scheds are owed update_idle() */
+	SCX_RQ_ROOT_IDLE_RENOTIFY	= 1 << 8, /* the root is owed update_idle() */
 
 	SCX_RQ_IN_WAKEUP	= 1 << 16,
 	SCX_RQ_IN_BALANCE	= 1 << 17,
@@ -794,10 +799,15 @@ enum scx_rq_flags {
 
 struct scx_rq {
 	struct scx_dispatch_q	local_dsq;
+#ifdef CONFIG_EXT_SUB_SCHED
+	struct scx_dispatch_q	reject_dsq;		/* staging for cap-rejected tasks */
+#endif
 	struct list_head	runnable_list;		/* runnable tasks on this rq */
 	struct list_head	ddsp_deferred_locals;	/* deferred ddsps from enq */
 	unsigned long		ops_qseq;
-	u64			extra_enq_flags;	/* see move_task_to_local_dsq() */
+	/* both stashed across the activate_task() in move_remote_task_to_local_dsq() */
+	u64			remote_activate_enq_flags;
+	struct scx_sched	*remote_activate_sch;
 	u32			nr_running;
 	u32			cpuperf_target;		/* [0, SCHED_CAPACITY_SCALE] */
 	bool			in_select_cpu;
@@ -805,15 +815,15 @@ struct scx_rq {
 	u32			flags;
 	u32			nr_immed;		/* ENQ_IMMED tasks on local_dsq */
 	u64			clock;			/* current per-rq clock -- see scx_bpf_now() */
-	cpumask_var_t		cpus_to_kick;
-	cpumask_var_t		cpus_to_kick_if_idle;
-	cpumask_var_t		cpus_to_preempt;
-	cpumask_var_t		cpus_to_wait;
+#ifdef CONFIG_EXT_SUB_SCHED
+	struct llist_head	ecaps_to_sync;		/* pending ecaps syncs */
+	struct task_struct	*sub_dispatch_prev;
+#endif
 	cpumask_var_t		cpus_to_sync;
 	bool			kick_sync_pending;
 	unsigned long		kick_sync;
 
-	struct task_struct	*sub_dispatch_prev;
+	struct list_head	sched_pcpus_to_kick;	/* see kick_cpus_irq_workfn() */
 
 	raw_spinlock_t		deferred_reenq_lock;
 	u64			deferred_reenq_locals_seq;
@@ -2018,7 +2028,8 @@ DEFINE_LOCK_GUARD_1(rq_lock, struct rq,
 		    rq_unlock(_T->lock, &_T->rf),
 		    struct rq_flags rf)
 
-DECLARE_LOCK_GUARD_1_ATTRS(rq_lock, __acquires(__rq_lockp(_T)), __releases(__rq_lockp(*(struct rq **)_T)));
+DECLARE_LOCK_GUARD_1_ATTRS(rq_lock, __acquires(__rq_lockp(_T)),
+			   __releases(__rq_lockp(*(struct rq **)_T)));
 #define class_rq_lock_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rq_lock, _T)
 
 DEFINE_LOCK_GUARD_1(rq_lock_irq, struct rq,
@@ -2026,7 +2037,8 @@ DEFINE_LOCK_GUARD_1(rq_lock_irq, struct rq,
 		    rq_unlock_irq(_T->lock, &_T->rf),
 		    struct rq_flags rf)
 
-DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irq, __acquires(__rq_lockp(_T)), __releases(__rq_lockp(*(struct rq **)_T)));
+DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irq, __acquires(__rq_lockp(_T)),
+			   __releases(__rq_lockp(*(struct rq **)_T)));
 #define class_rq_lock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rq_lock_irq, _T)
 
 DEFINE_LOCK_GUARD_1(rq_lock_irqsave, struct rq,
@@ -2034,9 +2046,20 @@ DEFINE_LOCK_GUARD_1(rq_lock_irqsave, struct rq,
 		    rq_unlock_irqrestore(_T->lock, &_T->rf),
 		    struct rq_flags rf)
 
-DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irqsave, __acquires(__rq_lockp(_T)), __releases(__rq_lockp(*(struct rq **)_T)));
+DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irqsave, __acquires(__rq_lockp(_T)),
+			   __releases(__rq_lockp(*(struct rq **)_T)));
 #define class_rq_lock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rq_lock_irqsave, _T)
 
+DEFINE_LOCK_GUARD_1(raw_spin_rq_lock_irqsave, struct rq,
+		    raw_spin_rq_lock_irqsave(_T->lock, _T->flags),
+		    raw_spin_rq_unlock_irqrestore(_T->lock, _T->flags),
+		    unsigned long flags)
+
+DECLARE_LOCK_GUARD_1_ATTRS(raw_spin_rq_lock_irqsave, __acquires(__rq_lockp(_T)),
+			   __releases(__rq_lockp(*(struct rq **)_T)));
+#define class_raw_spin_rq_lock_irqsave_constructor(_T) \
+	WITH_LOCK_GUARD_1_ATTRS(raw_spin_rq_lock_irqsave, _T)
+
 #define this_rq_lock_irq(...) __acquire_ret(_this_rq_lock_irq(__VA_ARGS__), __rq_lockp(__ret))
 static inline struct rq *_this_rq_lock_irq(struct rq_flags *rf) __acquires_ret
 {
@@ -2571,6 +2594,7 @@ extern const u32		sched_prio_to_wmult[40];
 #define ENQUEUE_MIGRATED	0x00040000
 #define ENQUEUE_INITIAL		0x00080000
 #define ENQUEUE_RQ_SELECTED	0x00100000
+#define ENQUEUE_QUEUED		0x00200000
 
 #define RETRY_TASK		((void *)-1UL)
 
diff --git a/kernel/seccomp.c b/kernel/seccomp.c
index 066909393c38f..86cf4460d69ea 100644
--- a/kernel/seccomp.c
+++ b/kernel/seccomp.c
@@ -1100,12 +1100,13 @@ void secure_computing_strict(int this_syscall)
 	else
 		BUG();
 }
-int __secure_computing(void)
+
+bool __seccomp_permit_syscall(void)
 {
 	int this_syscall = syscall_get_nr(current, current_pt_regs());
 
 	secure_computing_strict(this_syscall);
-	return 0;
+	return true;
 }
 #else
 
@@ -1256,7 +1257,7 @@ static int seccomp_do_user_notification(int this_syscall,
 	return -1;
 }
 
-static int __seccomp_filter(int this_syscall, const bool recheck_after_trace)
+static bool __seccomp_filter(int this_syscall, const bool recheck_after_trace)
 {
 	u32 filter_ret, action;
 	struct seccomp_data sd;
@@ -1294,7 +1295,7 @@ static int __seccomp_filter(int this_syscall, const bool recheck_after_trace)
 	case SECCOMP_RET_TRACE:
 		/* We've been put in this state by the ptracer already. */
 		if (recheck_after_trace)
-			return 0;
+			return true;
 
 		/* ENOSYS these calls if there is no tracer attached. */
 		if (!ptrace_event_enabled(current, PTRACE_EVENT_SECCOMP)) {
@@ -1329,20 +1330,17 @@ static int __seccomp_filter(int this_syscall, const bool recheck_after_trace)
 		 * a reload of all registers. This does not goto skip since
 		 * a skip would have already been reported.
 		 */
-		if (__seccomp_filter(this_syscall, true))
-			return -1;
-
-		return 0;
+		return __seccomp_filter(this_syscall, true);
 
 	case SECCOMP_RET_USER_NOTIF:
 		if (seccomp_do_user_notification(this_syscall, match, &sd))
 			goto skip;
 
-		return 0;
+		return true;
 
 	case SECCOMP_RET_LOG:
 		seccomp_log(this_syscall, 0, action, true);
-		return 0;
+		return true;
 
 	case SECCOMP_RET_ALLOW:
 		/*
@@ -1350,7 +1348,7 @@ static int __seccomp_filter(int this_syscall, const bool recheck_after_trace)
 		 * this action since SECCOMP_RET_ALLOW is the starting
 		 * state in seccomp_run_filters().
 		 */
-		return 0;
+		return true;
 
 	case SECCOMP_RET_KILL_THREAD:
 	case SECCOMP_RET_KILL_PROCESS:
@@ -1367,46 +1365,46 @@ static int __seccomp_filter(int this_syscall, const bool recheck_after_trace)
 		} else {
 			do_exit(SIGSYS);
 		}
-		return -1; /* skip the syscall go directly to signal handling */
+		return false; /* skip the syscall go directly to signal handling */
 	}
 
 	unreachable();
 
 skip:
 	seccomp_log(this_syscall, 0, action, match ? match->log : false);
-	return -1;
+	return false;
 }
 #else
-static int __seccomp_filter(int this_syscall, const bool recheck_after_trace)
+static bool __seccomp_filter(int this_syscall, const bool recheck_after_trace)
 {
 	BUG();
 
-	return -1;
+	return false;
 }
 #endif
 
-int __secure_computing(void)
+bool __seccomp_permit_syscall(void)
 {
 	int mode = current->seccomp.mode;
 	int this_syscall;
 
 	if (IS_ENABLED(CONFIG_CHECKPOINT_RESTORE) &&
 	    unlikely(current->ptrace & PT_SUSPEND_SECCOMP))
-		return 0;
+		return true;
 
 	this_syscall = syscall_get_nr(current, current_pt_regs());
 
 	switch (mode) {
 	case SECCOMP_MODE_STRICT:
 		__secure_computing_strict(this_syscall);  /* may call do_exit */
-		return 0;
+		return true;
 	case SECCOMP_MODE_FILTER:
 		return __seccomp_filter(this_syscall, false);
 	/* Surviving SECCOMP_RET_KILL_* must be proactively impossible. */
 	case SECCOMP_MODE_DEAD:
 		WARN_ON_ONCE(1);
 		do_exit(SIGKILL);
-		return -1;
+		return false;
 	default:
 		BUG();
 	}
diff --git a/kernel/signal.c b/kernel/signal.c
index bbc0fd4cc4d7c..fdee0b012a117 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -130,28 +130,10 @@ static bool sig_ignored(struct task_struct *t, int sig, bool force)
  */
 static inline bool has_pending_signals(sigset_t *signal, sigset_t *blocked)
 {
-	unsigned long ready;
-	long i;
-
-	switch (_NSIG_WORDS) {
-	default:
-		for (i = _NSIG_WORDS, ready = 0; --i >= 0 ;)
-			ready |= signal->sig[i] &~ blocked->sig[i];
-		break;
-
-	case 4: ready  = signal->sig[3] &~ blocked->sig[3];
-		ready |= signal->sig[2] &~ blocked->sig[2];
-		ready |= signal->sig[1] &~ blocked->sig[1];
-		ready |= signal->sig[0] &~ blocked->sig[0];
-		break;
-
-	case 2: ready  = signal->sig[1] &~ blocked->sig[1];
-		ready |= signal->sig[0] &~ blocked->sig[0];
-		break;
-
-	case 1: ready  = signal->sig[0] &~ blocked->sig[0];
-	}
-	return ready !=	0;
+	unsigned long ready = 0;
+	for (long i = 0; i < _NSIG_WORDS; i++)
+		ready |= signal->sig[i] & ~blocked->sig[i];
+	return ready != 0;
 }
 
 #define PENDING(p,b) has_pending_signals(&(p)->signal, (b))
@@ -1181,6 +1163,7 @@ static inline bool has_si_pid_and_uid(struct kernel_siginfo *info)
 int send_signal_locked(int sig, struct kernel_siginfo *info,
 		       struct task_struct *t, enum pid_type type)
 {
+	struct kernel_siginfo __maybe_unused rewritten;
 	/* Should SIGKILL or SIGSTOP be received by a pid namespace init? */
 	bool force = false;
 
@@ -1192,24 +1175,34 @@ int send_signal_locked(int sig, struct kernel_siginfo *info,
 		force = true;
 	} else if (has_si_pid_and_uid(info)) {
 		/* SIGKILL and SIGSTOP is special or has ids */
+#ifdef CONFIG_USER_NS
 		struct user_namespace *t_user_ns;
+		kuid_t uid;
 
 		rcu_read_lock();
 		t_user_ns = task_cred_xxx(t, user_ns);
 		if (current_user_ns() != t_user_ns) {
-			kuid_t uid = make_kuid(current_user_ns(), info->si_uid);
-			info->si_uid = from_kuid_munged(t_user_ns, uid);
+			rewritten = *info;
+			info = &rewritten;
+			uid = make_kuid(current_user_ns(), info->si_uid);
+			rewritten.si_uid = from_kuid_munged(t_user_ns, uid);
 		}
 		rcu_read_unlock();
-
+#endif
 		/* A kernel generated signal? */
 		force = (info->si_code == SI_KERNEL);
 
+#ifdef CONFIG_PID_NS
 		/* From an ancestor pid namespace? */
 		if (!task_pid_nr_ns(current, task_active_pid_ns(t))) {
-			info->si_pid = 0;
+			if (info != &rewritten) {
+				rewritten = *info;
+				info = &rewritten;
+			}
+			rewritten.si_pid = 0;
 			force = true;
 		}
+#endif
 	}
 	return __send_signal_locked(sig, info, t, type, force);
 }
@@ -3958,11 +3951,7 @@ static void prepare_kill_siginfo(int sig, struct kernel_siginfo *info,
  */
 SYSCALL_DEFINE2(kill, pid_t, pid, int, sig)
 {
-	struct kernel_siginfo info;
-
-	prepare_kill_siginfo(sig, &info, PIDTYPE_TGID);
-
-	return kill_something_info(sig, &info, pid);
+	return kill_something_info(sig, SEND_SIG_NOINFO, pid);
 }
 
 /*
diff --git a/kernel/smp.c b/kernel/smp.c
index a0bb56bd8ddad..b718f7ab0b48b 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -137,10 +137,10 @@ csd_do_func(smp_call_func_t func, void *info, call_single_data_t *csd)
 	trace_csd_function_exit(func, csd);
 }
 
-#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
-
 static DEFINE_STATIC_KEY_MAYBE(CONFIG_CSD_LOCK_WAIT_DEBUG_DEFAULT, csdlock_debug_enabled);
 
+#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
+
 /*
  * Parse the csdlock_debug= kernel boot parameter.
  *
@@ -182,16 +182,22 @@ static atomic_t csd_bug_count = ATOMIC_INIT(0);
 static void __csd_lock_record(call_single_data_t *csd)
 {
 	if (!csd) {
-		smp_mb(); /* NULL cur_csd after unlock. */
-		__this_cpu_write(cur_csd, NULL);
+		/*
+		 * Pairs with smp_load_acquire() of cur_csd in
+		 * csd_lock_wait_toolong(): orders any preceding CSD
+		 * callback/unlock before a remote reader observes NULL.
+		 */
+		smp_store_release(this_cpu_ptr(&cur_csd), NULL);
 		return;
 	}
 	__this_cpu_write(cur_csd_func, csd->func);
 	__this_cpu_write(cur_csd_info, csd->info);
-	smp_wmb(); /* func and info before csd. */
-	__this_cpu_write(cur_csd, csd);
-	smp_mb(); /* Update cur_csd before function call. */
-		  /* Or before unlock, as the case may be. */
+	/*
+	 * Pairs with smp_load_acquire() of cur_csd in
+	 * csd_lock_wait_toolong(): publishes cur_csd_func and
+	 * cur_csd_info before the non-NULL pointer becomes visible.
+	 */
+	smp_store_release(this_cpu_ptr(&cur_csd), csd);
 }
 
 static __always_inline void csd_lock_record(call_single_data_t *csd)
@@ -272,7 +278,13 @@ static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, in
 		cpux = 0;
 	else
 		cpux = cpu;
-	cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* Before func and info. */
+	/*
+	 * Pairs with smp_store_release() of cur_csd in __csd_lock_record():
+	 * a non-NULL cur_csd here implies cur_csd_func and cur_csd_info
+	 * are the matching publication; a NULL value is ordered after any
+	 * preceding CSD callback/unlock on the remote CPU.
+	 */
+	cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux));
 	/* How long since this CSD lock was stuck. */
 	ts_delta = ts2 - ts0;
 	pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %lld ns for CPU#%02d %pS(%ps).\n",
@@ -342,6 +354,10 @@ static __always_inline void csd_lock_wait(call_single_data_t *csd)
 	smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));
 }
 #else
+static __always_inline void __csd_lock_wait(call_single_data_t *csd)
+{
+}
+
 static void csd_lock_record(call_single_data_t *csd)
 {
 }
@@ -354,8 +370,22 @@ static __always_inline void csd_lock_wait(call_single_data_t *csd)
 
 static __always_inline void csd_lock(call_single_data_t *csd)
 {
-	csd_lock_wait(csd);
-	csd->node.u_flags |= CSD_FLAG_LOCK;
+	if (IS_ENABLED(CONFIG_CSD_LOCK_WAIT_DEBUG) &&
+	    static_branch_unlikely(&csdlock_debug_enabled)) {
+		unsigned int flags;
+
+		for (;;) {
+			__csd_lock_wait(csd);
+			flags = READ_ONCE(csd->node.u_flags);
+			if (!(flags & CSD_FLAG_LOCK) &&
+			    try_cmpxchg_acquire(&csd->node.u_flags, &flags,
+						flags | CSD_FLAG_LOCK))
+				break;
+		}
+	} else {
+		csd_lock_wait(csd);
+		csd->node.u_flags |= CSD_FLAG_LOCK;
+	}
 
 	/*
 	 * prevent CPU from reordering the above assignment
@@ -380,7 +410,8 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data);
 #ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
 static call_single_data_t *get_single_csd_data(int cpu)
 {
-	if (static_branch_unlikely(&csdlock_debug_enabled))
+	if (static_branch_unlikely(&csdlock_debug_enabled) &&
+	    (unsigned int)cpu < nr_cpu_ids)
 		return per_cpu_ptr(&csd_data, cpu);
 	return this_cpu_ptr(&csd_data);
 }
diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c
index add3032da16f5..c8be0abaa4077 100644
--- a/kernel/sys_ni.c
+++ b/kernel/sys_ni.c
@@ -351,6 +351,10 @@ COND_SYSCALL(ppoll_time32);
 COND_SYSCALL_COMPAT(ppoll_time32);
 COND_SYSCALL(utimensat_time32);
 COND_SYSCALL(clock_adjtime32);
+COND_SYSCALL(gettimeofday);
+COND_SYSCALL_COMPAT(gettimeofday);
+COND_SYSCALL(time);
+COND_SYSCALL(stime);
 
 /*
  * The syscalls below are not found in include/uapi/asm-generic/unistd.h
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index c9efb17cc255c..0cb5d1759d426 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -148,7 +148,7 @@ static void warn_sysctl_write(const struct ctl_table *table)
  * @ppos: file position
  * @table: the sysctl table
  *
- * Returns true if the first position is non-zero and the sysctl_writes_strict
+ * Returns: true if the first position is non-zero and the sysctl_writes_strict
  * mode indicates this is not allowed for numeric input types. String proc
  * handlers can ignore the return value.
  */
@@ -184,7 +184,7 @@ static bool proc_first_pos_non_zero_ignore(loff_t *ppos,
  * and a newline '\n' is added. It is truncated if the buffer is
  * not large enough.
  *
- * Returns 0 on success.
+ * Returns: %0 on success.
  */
 int proc_dostring(const struct ctl_table *table, int dir,
 		  void *buffer, size_t *lenp, loff_t *ppos)
@@ -225,11 +225,14 @@ static void proc_skip_char(char **buf, size_t *size, const char v)
  * @base: the base to use
  * @res: where the parsed integer will be stored
  *
- * In case of success 0 is returned and @res will contain the parsed integer,
- * @endp will hold any trailing characters.
  * This function will fail the parse on overflow. If there wasn't an overflow
  * the function will defer the decision what characters count as invalid to the
  * caller.
+ *
+ * Returns:
+ * * %0 on success and @res will contain the parsed integer,
+ *   @endp will hold any trailing characters.
+ * * %-ERANGE on overflow.
  */
 static int strtoul_lenient(const char *cp, char **endp, unsigned int base,
 			   unsigned long *res)
@@ -263,10 +266,12 @@ static int strtoul_lenient(const char *cp, char **endp, unsigned int base,
  * @perm_tr_len: size of the perm_tr vector
  * @tr: pointer to store the trailer character
  *
- * In case of success %0 is returned and @buf and @size are updated with
- * the amount of bytes read. If @tr is non-NULL and a trailing
- * character exists (size is non-zero after returning from this
- * function), @tr is updated with the trailing character.
+ * Returns:
+ * * %0 on success and @buf and @size are updated with
+ *   the amount of bytes read. If @tr is non-NULL and a trailing
+ *   character exists (size is non-zero after returning from this
+ *   function), @tr is updated with the trailing character.
+ * * %-EINVAL on failure.
  */
 static int proc_get_long(char **buf, size_t *size,
 			  unsigned long *val, bool *neg,
@@ -365,7 +370,7 @@ static void proc_put_char(void **buf, size_t *size, char c)
  * not NULL. Check that the values are less than UINT_MAX to avoid
  * having to support wrap around from userspace.
  *
- * returns 0 on success.
+ * Returns: %0 on success.
  */
 int proc_uint_u2k_conv_uop(const ulong *u_ptr, uint *k_ptr,
 			   ulong (*u_ptr_op)(const ulong))
@@ -386,7 +391,7 @@ int proc_uint_u2k_conv_uop(const ulong *u_ptr, uint *k_ptr,
  *
  * Uses READ_ONCE to assign value to u_ptr.
  *
- * returns 0 on success.
+ * Returns: %0 on success.
  */
 int proc_uint_k2u_conv(ulong *u_ptr, const uint *k_ptr)
 {
@@ -409,7 +414,7 @@ int proc_uint_k2u_conv(ulong *u_ptr, const uint *k_ptr)
  * When direction is kernel to user, then the u_ptr is modified.
  * When direction is user to kernel, then the k_ptr is modified.
  *
- * Returns 0 on success
+ * Returns: %0 on success
  */
 int proc_uint_conv(ulong *u_ptr, uint *k_ptr, int dir,
 		   const struct ctl_table *tbl, bool k_ptr_range_check,
@@ -444,15 +449,15 @@ static int proc_uint_u2k_conv(const ulong *u_ptr, uint *k_ptr)
 	return proc_uint_u2k_conv_uop(u_ptr, k_ptr, NULL);
 }
 
-static int do_proc_uint_conv(ulong *u_ptr, uint *k_ptr, int dir,
+static int do_proc_uint_conv(bool *negp, ulong *u_ptr, uint *k_ptr, int dir,
 			     const struct ctl_table *tbl)
 {
 	return proc_uint_conv(u_ptr, k_ptr, dir, tbl, false,
 			      proc_uint_u2k_conv, proc_uint_k2u_conv);
 }
 
-static int do_proc_uint_conv_minmax(ulong *u_ptr, uint *k_ptr, int dir,
-				    const struct ctl_table *tbl)
+static int do_proc_uint_conv_minmax(bool *negp, ulong *u_ptr, uint *k_ptr,
+				    int dir, const struct ctl_table *tbl)
 {
 	return proc_uint_conv(u_ptr, k_ptr, dir, tbl, true,
 			      proc_uint_u2k_conv, proc_uint_k2u_conv);
@@ -515,6 +520,23 @@ int proc_int_u2k_conv_uop(const ulong *u_ptr, int *k_ptr, const bool *negp,
 	return 0;
 }
 
+/**
+ * proc_int_conv - Change user or kernel pointer based on direction
+ *
+ * @negp: will be passed to uni-directional converters
+ * @u_ptr: pointer to user variable
+ * @k_ptr: pointer to kernel variable
+ * @dir: %TRUE if this is a write to the sysctl file
+ * @tbl: the sysctl table
+ * @k_ptr_range_check: Check range for k_ptr when %TRUE
+ * @user_to_kern: Callback used to assign value from user to kernel var
+ * @kern_to_user: Callback used to assign value from kernel to user var
+ *
+ * When direction is kernel to user, then the u_ptr is modified.
+ * When direction is user to kernel, then the k_ptr is modified.
+ *
+ * Returns: 0 on success
+ */
 int proc_int_conv(bool *negp, ulong *u_ptr, int *k_ptr, int dir,
 		  const struct ctl_table *tbl, bool k_ptr_range_check,
 		  int (*user_to_kern)(const bool *negp, const ulong *u_ptr, int *k_ptr),
@@ -572,14 +594,80 @@ static int do_proc_int_conv_minmax(bool *negp, unsigned long *u_ptr, int *k_ptr,
 
 static const char proc_wspace_sep[] = { ' ', '\t', '\n' };
 
-static int do_proc_dointvec(const struct ctl_table *table, int dir,
-		  void *buffer, size_t *lenp, loff_t *ppos,
-		  int (*conv)(bool *negp, unsigned long *u_ptr, int *k_ptr,
-			      int dir, const struct ctl_table *table))
+/*
+ * Element type processed by do_proc_vec(). The tag selects the element size
+ * and signedness, and it selects which member of union proc_vec_conv is live.
+ */
+enum proc_vec_type {
+	PROC_VEC_INT,
+	PROC_VEC_UINT,
+	PROC_VEC_ULONG,
+};
+
+/*
+ * Converter passed to do_proc_vec(). Only the member matching the
+ * enum proc_vec_type tag is ever read, so every dispatch stays fully typed and
+ * no void * converter pointer is needed.
+ */
+union proc_vec_conv {
+	int (*int_conv)(bool *negp, ulong *u_ptr, int *k_ptr,
+			int dir, const struct ctl_table *table);
+	int (*uint_conv)(bool *negp, ulong *u_ptr, uint *k_ptr,
+			 int dir, const struct ctl_table *table);
+	int (*ulong_conv)(bool *negp, ulong *u_ptr, ulong *k_ptr,
+			  int dir, const struct ctl_table *table);
+};
+
+/*
+ * Dispatch to the converter member selected by @type. @k_ptr walks
+ * table->data as raw bytes and is cast back to the element type here.
+ */
+static int proc_vec_conv(enum proc_vec_type type, union proc_vec_conv conv,
+			 bool *negp, ulong *u_ptr, char *k_ptr, int dir,
+			 const struct ctl_table *table)
+{
+	switch (type) {
+	case PROC_VEC_INT:
+		return conv.int_conv(negp, u_ptr, (int *)k_ptr, dir, table);
+	case PROC_VEC_UINT:
+		return conv.uint_conv(negp, u_ptr, (uint *)k_ptr, dir, table);
+	case PROC_VEC_ULONG:
+		return conv.ulong_conv(negp, u_ptr, (ulong *)k_ptr, dir, table);
+	}
+	return -EINVAL;
+}
+
+/*
+ * Read/write a vector of @type elements. The element size and signedness are
+ * derived from @type, so a single runtime function replaces the per-type
+ * variants. table->data is walked as raw bytes (@i) advanced by @size; the
+ * converter performs the actual typed load/store.
+ */
+static int do_proc_vec(const struct ctl_table *table, int dir,
+		       void *buffer, size_t *lenp, loff_t *ppos,
+		       enum proc_vec_type type, union proc_vec_conv conv)
 {
-	int *i, vleft, first = 1, err = 0;
-	size_t left;
-	char *p;
+	int vleft, first = 1, err = 0;
+	size_t left, size;
+	bool is_unsigned;
+	char *i, *p;
+
+	switch (type) {
+	case PROC_VEC_INT:
+		size = sizeof(int);
+		is_unsigned = false;
+		break;
+	case PROC_VEC_UINT:
+		size = sizeof(uint);
+		is_unsigned = true;
+		break;
+	case PROC_VEC_ULONG:
+		size = sizeof(ulong);
+		is_unsigned = true;
+		break;
+	default:
+		return -EINVAL;
+	}
 
 	if (!table->data || !table->maxlen || !*lenp ||
 	    (*ppos && SYSCTL_KERN_TO_USER(dir))) {
@@ -587,12 +675,13 @@ static int do_proc_dointvec(const struct ctl_table *table, int dir,
 		return 0;
 	}
 
-	i = (int *) table->data;
-	vleft = table->maxlen / sizeof(*i);
+	i = table->data;
+	vleft = table->maxlen / size;
 	left = *lenp;
 
-	if (!conv)
-		conv = do_proc_int_conv;
+	/* uint arrays are not supported, *Do not* add support for them. */
+	if (type == PROC_VEC_UINT && vleft != 1)
+		return -EINVAL;
 
 	if (SYSCTL_USER_TO_KERN(dir)) {
 		if (proc_first_pos_non_zero_ignore(ppos, table))
@@ -603,9 +692,9 @@ static int do_proc_dointvec(const struct ctl_table *table, int dir,
 		p = buffer;
 	}
 
-	for (; left && vleft--; i++, first=0) {
+	for (; left && vleft--; i += size, first = 0) {
 		unsigned long lval;
-		bool neg;
+		bool neg = false;
 
 		if (SYSCTL_USER_TO_KERN(dir)) {
 			proc_skip_spaces(&p, &left);
@@ -613,16 +702,18 @@ static int do_proc_dointvec(const struct ctl_table *table, int dir,
 			if (!left)
 				break;
 			err = proc_get_long(&p, &left, &lval, &neg,
-					     proc_wspace_sep,
-					     sizeof(proc_wspace_sep), NULL);
+					    proc_wspace_sep,
+					    sizeof(proc_wspace_sep), NULL);
+			if (!err && neg && is_unsigned)
+				err = -EINVAL;
 			if (err)
 				break;
-			if (conv(&neg, &lval, i, 1, table)) {
+			if (proc_vec_conv(type, conv, &neg, &lval, i, dir, table)) {
 				err = -EINVAL;
 				break;
 			}
 		} else {
-			if (conv(&neg, &lval, i, 0, table)) {
+			if (proc_vec_conv(type, conv, &neg, &lval, i, dir, table)) {
 				err = -EINVAL;
 				break;
 			}
@@ -644,122 +735,6 @@ static int do_proc_dointvec(const struct ctl_table *table, int dir,
 	return err;
 }
 
-static int do_proc_douintvec_w(const struct ctl_table *table, void *buffer,
-			       size_t *lenp, loff_t *ppos,
-			       int (*conv)(unsigned long *u_ptr,
-					   unsigned int *k_ptr, int dir,
-					   const struct ctl_table *table))
-{
-	unsigned long lval;
-	int err = 0;
-	size_t left;
-	bool neg;
-	char *p = buffer;
-
-	left = *lenp;
-
-	if (proc_first_pos_non_zero_ignore(ppos, table))
-		goto bail_early;
-
-	if (left > PAGE_SIZE - 1)
-		left = PAGE_SIZE - 1;
-
-	proc_skip_spaces(&p, &left);
-	if (!left) {
-		err = -EINVAL;
-		goto out_free;
-	}
-
-	err = proc_get_long(&p, &left, &lval, &neg,
-			     proc_wspace_sep,
-			     sizeof(proc_wspace_sep), NULL);
-	if (err || neg) {
-		err = -EINVAL;
-		goto out_free;
-	}
-
-	if (conv(&lval, (unsigned int *) table->data, 1, table)) {
-		err = -EINVAL;
-		goto out_free;
-	}
-
-	if (!err && left)
-		proc_skip_spaces(&p, &left);
-
-out_free:
-	if (err)
-		return -EINVAL;
-
-	return 0;
-
-bail_early:
-	*ppos += *lenp;
-	return err;
-}
-
-static int do_proc_douintvec_r(const struct ctl_table *table, void *buffer,
-			       size_t *lenp, loff_t *ppos,
-			       int (*conv)(unsigned long *u_ptr,
-					   unsigned int *k_ptr, int dir,
-					   const struct ctl_table *table))
-{
-	unsigned long lval;
-	int err = 0;
-	size_t left;
-
-	left = *lenp;
-
-	if (conv(&lval, (unsigned int *) table->data, 0, table)) {
-		err = -EINVAL;
-		goto out;
-	}
-
-	proc_put_long(&buffer, &left, lval, false);
-	if (!left)
-		goto out;
-
-	proc_put_char(&buffer, &left, '\n');
-
-out:
-	*lenp -= left;
-	*ppos += *lenp;
-
-	return err;
-}
-
-static int do_proc_douintvec(const struct ctl_table *table, int dir,
-			     void *buffer, size_t *lenp, loff_t *ppos,
-			      int (*conv)(unsigned long *u_ptr,
-					  unsigned int *k_ptr, int dir,
-					  const struct ctl_table *table))
-{
-	unsigned int vleft;
-
-	if (!table->data || !table->maxlen || !*lenp ||
-	    (*ppos && SYSCTL_KERN_TO_USER(dir))) {
-		*lenp = 0;
-		return 0;
-	}
-
-	vleft = table->maxlen / sizeof(unsigned int);
-
-	/*
-	 * Arrays are not supported, keep this simple. *Do not* add
-	 * support for them.
-	 */
-	if (vleft != 1) {
-		*lenp = 0;
-		return -EINVAL;
-	}
-
-	if (!conv)
-		conv = do_proc_uint_conv;
-
-	if (SYSCTL_USER_TO_KERN(dir))
-		return do_proc_douintvec_w(table, buffer, lenp, ppos, conv);
-	return do_proc_douintvec_r(table, buffer, lenp, ppos, conv);
-}
-
 /**
  * proc_douintvec_conv - read a vector of unsigned ints with a custom converter
  *
@@ -774,14 +749,19 @@ static int do_proc_douintvec(const struct ctl_table *table, int dir,
  * values from/to the user buffer, treated as an ASCII string. Negative
  * strings are not allowed.
  *
- * Returns 0 on success
+ * Returns: %0 on success
  */
 int proc_douintvec_conv(const struct ctl_table *table, int dir, void *buffer,
 			size_t *lenp, loff_t *ppos,
-			int (*conv)(unsigned long *u_ptr, unsigned int *k_ptr,
+			int (*conv)(bool *negp, ulong *u_ptr, uint *k_ptr,
 				    int dir, const struct ctl_table *table))
 {
-	return do_proc_douintvec(table, dir, buffer, lenp, ppos, conv);
+
+	if (!conv)
+		conv = do_proc_uint_conv;
+
+	return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_UINT,
+			   (union proc_vec_conv){ .uint_conv = conv });
 }
 
 /**
@@ -798,7 +778,7 @@ int proc_douintvec_conv(const struct ctl_table *table, int dir, void *buffer,
  * table->data must point to a bool variable and table->maxlen must
  * be sizeof(bool).
  *
- * Returns 0 on success.
+ * Returns: %0 on success.
  */
 int proc_dobool(const struct ctl_table *table, int dir, void *buffer,
 		size_t *lenp, loff_t *ppos)
@@ -835,12 +815,13 @@ int proc_dobool(const struct ctl_table *table, int dir, void *buffer,
  * Reads/writes up to table->maxlen/sizeof(unsigned int) integer
  * values from/to the user buffer, treated as an ASCII string.
  *
- * Returns 0 on success.
+ * Returns: %0 on success.
  */
 int proc_dointvec(const struct ctl_table *table, int dir, void *buffer,
 		  size_t *lenp, loff_t *ppos)
 {
-	return do_proc_dointvec(table, dir, buffer, lenp, ppos, NULL);
+	return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_INT,
+			   (union proc_vec_conv){ .int_conv = do_proc_int_conv });
 }
 
 /**
@@ -854,13 +835,13 @@ int proc_dointvec(const struct ctl_table *table, int dir, void *buffer,
  * Reads/writes up to table->maxlen/sizeof(unsigned int) unsigned integer
  * values from/to the user buffer, treated as an ASCII string.
  *
- * Returns 0 on success.
+ * Returns: %0 on success.
  */
 int proc_douintvec(const struct ctl_table *table, int dir, void *buffer,
 		size_t *lenp, loff_t *ppos)
 {
-	return do_proc_douintvec(table, dir, buffer, lenp, ppos,
-				 do_proc_uint_conv);
+	return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_UINT,
+			   (union proc_vec_conv){ .uint_conv = do_proc_uint_conv });
 }
 
 /**
@@ -877,14 +858,14 @@ int proc_douintvec(const struct ctl_table *table, int dir, void *buffer,
  * This routine will ensure the values are within the range specified by
  * table->extra1 (min) and table->extra2 (max).
  *
- * Returns 0 on success or -EINVAL when the range check fails and
+ * Returns: %0 on success or -EINVAL when the range check fails and
  * SYSCTL_USER_TO_KERN(dir) == true
  */
 int proc_dointvec_minmax(const struct ctl_table *table, int dir,
 		  void *buffer, size_t *lenp, loff_t *ppos)
 {
-	return do_proc_dointvec(table, dir, buffer, lenp, ppos,
-				do_proc_int_conv_minmax);
+	return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_INT,
+			   (union proc_vec_conv){ .int_conv = do_proc_int_conv_minmax });
 }
 
 /**
@@ -904,14 +885,14 @@ int proc_dointvec_minmax(const struct ctl_table *table, int dir,
  * (max). And Check that the values are less than UINT_MAX to avoid having to
  * support wrap around uses from userspace.
  *
- * Returns 0 on success or -ERANGE when range check failes and
+ * Returns: %0 on success or -ERANGE when range check failes and
  * SYSCTL_USER_TO_KERN(dir) == true
  */
 int proc_douintvec_minmax(const struct ctl_table *table, int dir,
 			  void *buffer, size_t *lenp, loff_t *ppos)
 {
-	return do_proc_douintvec(table, dir, buffer, lenp, ppos,
-				 do_proc_uint_conv_minmax);
+	return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_UINT,
+			   (union proc_vec_conv){ .uint_conv = do_proc_uint_conv_minmax });
 }
 
 /**
@@ -929,7 +910,7 @@ int proc_douintvec_minmax(const struct ctl_table *table, int dir,
  * This routine will ensure the values are within the range specified by
  * table->extra1 (min) and table->extra2 (max).
  *
- * Returns 0 on success or an error on SYSCTL_USER_TO_KERN(dir) == true
+ * Returns: %0 on success or an error on SYSCTL_USER_TO_KERN(dir) == true
  * and the range check fails.
  */
 int proc_dou8vec_minmax(const struct ctl_table *table, int dir,
@@ -954,8 +935,8 @@ int proc_dou8vec_minmax(const struct ctl_table *table, int dir,
 		tmp.extra2 = (unsigned int *) &max;
 
 	val = READ_ONCE(*data);
-	res = do_proc_douintvec(&tmp, dir, buffer, lenp, ppos,
-				do_proc_uint_conv_minmax);
+	res = do_proc_vec(&tmp, dir, buffer, lenp, ppos, PROC_VEC_UINT,
+			  (union proc_vec_conv){ .uint_conv = do_proc_uint_conv_minmax });
 	if (res)
 		return res;
 	if (SYSCTL_USER_TO_KERN(dir))
@@ -964,87 +945,129 @@ int proc_dou8vec_minmax(const struct ctl_table *table, int dir,
 }
 EXPORT_SYMBOL_GPL(proc_dou8vec_minmax);
 
-static int do_proc_doulongvec_minmax(const struct ctl_table *table, int dir,
-				     void *buffer, size_t *lenp, loff_t *ppos,
-				     unsigned long convmul,
-				     unsigned long convdiv)
+/**
+ * proc_ulong_conv - Change user or kernel pointer based on direction
+ *
+ * @u_ptr: pointer to user variable
+ * @k_ptr: pointer to kernel variable
+ * @dir: %TRUE if this is a write to the sysctl file
+ * @tbl: the sysctl table
+ * @k_ptr_range_check: Check range for k_ptr when %TRUE
+ * @user_to_kern: Callback used to assign value from user to kernel var
+ * @kern_to_user: Callback used to assign value from kernel to user var
+ *
+ * When direction is kernel to user, then the u_ptr is modified.
+ * When direction is user to kernel, then the k_ptr is modified.
+ *
+ * Returns: 0 on success
+ */
+int proc_ulong_conv(ulong *u_ptr, ulong *k_ptr, int dir,
+		    const struct ctl_table *tbl, bool k_ptr_range_check,
+		    int (*user_to_kern)(const ulong *u_ptr, ulong *k_ptr),
+		    int (*kern_to_user)(ulong *u_ptr, const ulong *k_ptr))
 {
-	unsigned long *i, *min, *max;
-	int vleft, first = 1, err = 0;
-	size_t left;
-	char *p;
-
-	if (!table->data || !table->maxlen || !*lenp ||
-	    (*ppos && SYSCTL_KERN_TO_USER(dir))) {
-		*lenp = 0;
-		return 0;
-	}
-
-	i = table->data;
-	min = table->extra1;
-	max = table->extra2;
-	vleft = table->maxlen / sizeof(unsigned long);
-	left = *lenp;
+	if (SYSCTL_KERN_TO_USER(dir))
+		return kern_to_user(u_ptr, k_ptr);
 
-	if (SYSCTL_USER_TO_KERN(dir)) {
-		if (proc_first_pos_non_zero_ignore(ppos, table))
-			goto out;
+	if (k_ptr_range_check) {
+		ulong tmp_k;
+		int ret;
 
-		if (left > PAGE_SIZE - 1)
-			left = PAGE_SIZE - 1;
-		p = buffer;
-	}
+		if (!tbl)
+			return -EINVAL;
+		ret = user_to_kern(u_ptr, &tmp_k);
+		if (ret)
+			return ret;
+		if ((tbl->extra1 && *(ulong *)tbl->extra1 > tmp_k) ||
+		    (tbl->extra2 && *(ulong *)tbl->extra2 < tmp_k))
+			return -ERANGE;
+		WRITE_ONCE(*k_ptr, tmp_k);
+	} else
+		return user_to_kern(u_ptr, k_ptr);
+	return 0;
+}
 
-	for (; left && vleft--; i++, first = 0) {
-		unsigned long val;
+/**
+ * proc_ulong_u2k_conv_uop - Assign user value to a kernel pointer
+ *
+ * @u_ptr: pointer to user space variable
+ * @k_ptr: pointer to kernel variable
+ * @u_ptr_op: execute this function before assigning to k_ptr
+ *
+ * Uses WRITE_ONCE to assign value to k_ptr. Executes u_ptr_op if
+ * not NULL.
+ *
+ * Returns: 0 on success.
+ */
+int proc_ulong_u2k_conv_uop(const ulong *u_ptr, ulong *k_ptr,
+			    ulong (*u_ptr_op)(const ulong))
+{
+	ulong u = u_ptr_op ? u_ptr_op(*u_ptr) : *u_ptr;
 
-		if (SYSCTL_USER_TO_KERN(dir)) {
-			bool neg;
+	WRITE_ONCE(*k_ptr, u);
+	return 0;
+}
 
-			proc_skip_spaces(&p, &left);
-			if (!left)
-				break;
+static int proc_ulong_u2k_conv(const ulong *u_ptr, ulong *k_ptr)
+{
+	return proc_ulong_u2k_conv_uop(u_ptr, k_ptr, NULL);
+}
 
-			err = proc_get_long(&p, &left, &val, &neg,
-					     proc_wspace_sep,
-					     sizeof(proc_wspace_sep), NULL);
-			if (err || neg) {
-				err = -EINVAL;
-				break;
-			}
+/**
+ * proc_ulong_k2u_conv_kop - Assign kernel value to a user space pointer
+ *
+ * @u_ptr: pointer to user space variable
+ * @k_ptr: pointer to kernel variable
+ * @k_ptr_op: Operation applied to k_ptr before assignment
+ *
+ * Uses READ_ONCE to assign value to u_ptr. Executes k_ptr_op if
+ * not NULL.
+ *
+ * Returns: 0 on success.
+ */
+int proc_ulong_k2u_conv_kop(ulong *u_ptr, const ulong *k_ptr,
+			    ulong (*k_ptr_op)(const ulong))
+{
+	ulong val = k_ptr_op ? k_ptr_op(READ_ONCE(*k_ptr)) : READ_ONCE(*k_ptr);
+	*u_ptr = (ulong)val;
+	return 0;
+}
 
-			val = convmul * val / convdiv;
-			if ((min && val < *min) || (max && val > *max)) {
-				err = -EINVAL;
-				break;
-			}
-			WRITE_ONCE(*i, val);
-		} else {
-			val = convdiv * READ_ONCE(*i) / convmul;
-			if (!first)
-				proc_put_char(&buffer, &left, '\t');
-			proc_put_long(&buffer, &left, val, false);
-		}
-	}
+static int proc_ulong_k2u_conv(ulong *u_ptr, const ulong *k_ptr)
+{
+	return proc_ulong_k2u_conv_kop(u_ptr, k_ptr, NULL);
+}
 
-	if (SYSCTL_KERN_TO_USER(dir) && !first && left && !err)
-		proc_put_char(&buffer, &left, '\n');
-	if (SYSCTL_USER_TO_KERN(dir) && !err)
-		proc_skip_spaces(&p, &left);
-	if (SYSCTL_USER_TO_KERN(dir) && first)
-		return err ? : -EINVAL;
-	*lenp -= left;
-out:
-	*ppos += *lenp;
-	return err;
+static int do_proc_ulong_conv(bool *negp, ulong *u_ptr, ulong *k_ptr, int dir,
+			      const struct ctl_table *tbl)
+{
+	return proc_ulong_conv(u_ptr, k_ptr, dir, tbl, true,
+			       proc_ulong_u2k_conv, proc_ulong_k2u_conv);
 }
 
-int proc_doulongvec_minmax_conv(const struct ctl_table *table, int dir,
+/**
+ * proc_doulongvec_conv - read a vector of unsigned longs with a custom converter
+ *
+ * @table: the sysctl table
+ * @dir: %TRUE if this is a write to the sysctl file
+ * @buffer: the user buffer
+ * @lenp: the size of the user buffer
+ * @ppos: file position
+ * @conv: Custom converter call back
+ *
+ * Reads/writes up to table->maxlen/sizeof(unsigned long) unsigned long
+ * values from/to the user buffer, treated as an ASCII string. Negative
+ * strings are not allowed.
+ *
+ * Returns: 0 on success
+ */
+int proc_doulongvec_conv(const struct ctl_table *table, int dir,
 				void *buffer, size_t *lenp, loff_t *ppos,
-				unsigned long convmul, unsigned long convdiv)
+				int (*conv)(bool *negp, ulong *u_ptr, ulong *k_ptr,
+					    int dir, const struct ctl_table *table))
 {
-	return do_proc_doulongvec_minmax(table, dir, buffer, lenp, ppos,
-					 convmul, convdiv);
+	return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_ULONG,
+			   (union proc_vec_conv){ .ulong_conv = conv });
 }
 
 /**
@@ -1061,12 +1084,13 @@ int proc_doulongvec_minmax_conv(const struct ctl_table *table, int dir,
  * This routine will ensure the values are within the range specified by
  * table->extra1 (min) and table->extra2 (max).
  *
- * Returns 0 on success.
+ * Returns: %0 on success.
  */
 int proc_doulongvec_minmax(const struct ctl_table *table, int dir,
 			   void *buffer, size_t *lenp, loff_t *ppos)
 {
-	return proc_doulongvec_minmax_conv(table, dir, buffer, lenp, ppos, 1l, 1l);
+	return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_ULONG,
+			   (union proc_vec_conv){ .ulong_conv = do_proc_ulong_conv });
 }
 
 /**
@@ -1076,21 +1100,22 @@ int proc_doulongvec_minmax(const struct ctl_table *table, int dir,
  * @buffer: the user buffer
  * @lenp: the size of the user buffer
  * @ppos: file position
- * @conv: Custom converter call back
+ * @conv: Custom converter call back. Defaults to do_proc_int_conv
  *
- * Reads/writes up to table->maxlen/sizeof(unsigned int) unsigned integer
- * values from/to the user buffer, treated as an ASCII string. Negative
- * strings are not allowed.
+ * Reads/writes up to table->maxlen/sizeof(int) integer values from/to the
+ * user buffer, treated as an ASCII string.
  *
  * Returns: 0 on success
  */
-
 int proc_dointvec_conv(const struct ctl_table *table, int dir, void *buffer,
 		       size_t *lenp, loff_t *ppos,
 		       int (*conv)(bool *negp, unsigned long *u_ptr, int *k_ptr,
 				   int dir, const struct ctl_table *table))
 {
-	return do_proc_dointvec(table, dir, buffer, lenp, ppos, conv);
+	if (!conv)
+		conv = do_proc_int_conv;
+	return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_INT,
+			   (union proc_vec_conv){ .int_conv = conv });
 }
 
 /**
@@ -1108,7 +1133,7 @@ int proc_dointvec_conv(const struct ctl_table *table, int dir, void *buffer,
  * large bitmaps may be represented in a compact manner. Writing into
  * the file will clear the bitmap then update it with the given input.
  *
- * Returns 0 on success.
+ * Returns: %0 on success.
  */
 int proc_do_large_bitmap(const struct ctl_table *table, int dir,
 			 void *buffer, size_t *lenp, loff_t *ppos)
@@ -1280,7 +1305,7 @@ int proc_douintvec_minmax(const struct ctl_table *table, int dir,
 
 int proc_douintvec_conv(const struct ctl_table *table, int write, void *buffer,
 			size_t *lenp, loff_t *ppos,
-			int (*conv)(unsigned long *lvalp, unsigned int *valp,
+			int (*conv)(bool *negp, ulong *lvalp, uint *valp,
 				    int write, const struct ctl_table *table))
 {
 	return -ENOSYS;
@@ -1317,9 +1342,10 @@ int proc_doulongvec_minmax(const struct ctl_table *table, int dir,
 	return -ENOSYS;
 }
 
-int proc_doulongvec_minmax_conv(const struct ctl_table *table, int dir,
+int proc_doulongvec_conv(const struct ctl_table *table, int dir,
 				void *buffer, size_t *lenp, loff_t *ppos,
-				unsigned long convmul, unsigned long convdiv)
+				int (*conv)(bool *negp, ulong *u_ptr, ulong *k_ptr,
+					    int dir, const struct ctl_table *table))
 {
 	return -ENOSYS;
 }
diff --git a/kernel/taskstats.c b/kernel/taskstats.c
index 2cd0172d05160..8d115670a3ca6 100644
--- a/kernel/taskstats.c
+++ b/kernel/taskstats.c
@@ -423,7 +423,7 @@ static int cgroupstats_user_cmd(struct sk_buff *skb, struct genl_info *info)
 	fd = nla_get_u32(info->attrs[CGROUPSTATS_CMD_ATTR_FD]);
 	CLASS(fd, f)(fd);
 	if (fd_empty(f))
-		return 0;
+		return -EBADF;
 
 	size = nla_total_size(sizeof(struct cgroupstats));
 
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index e48c4d379a7ce..f1253f5795c6e 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -123,7 +123,6 @@ static atomic_t watchdog_reset_pending;
 
 /* Watchdog interval: 0.5sec. */
 #define WATCHDOG_INTERVAL		(HZ >> 1)
-#define WATCHDOG_INTERVAL_NS		(WATCHDOG_INTERVAL * (NSEC_PER_SEC / HZ))
 
 /* Maximum time between two reference watchdog readouts */
 #define WATCHDOG_READOUT_MAX_NS		(50U * NSEC_PER_USEC)
@@ -1566,8 +1565,12 @@ static int __init init_clocksource_sysfs(void)
 {
 	int error = subsys_system_register(&clocksource_subsys, NULL);
 
-	if (!error)
-		error = device_register(&device_clocksource);
+	if (error)
+		return error;
+
+	error = device_register(&device_clocksource);
+	if (error)
+		bus_unregister(&clocksource_subsys);
 
 	return error;
 }
diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index 313dcea127fe4..01890b7a5f982 100644
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -26,6 +26,7 @@
 #include <linux/export.h>
 #include <linux/percpu.h>
 #include <linux/hrtimer.h>
+#include <linux/hrtimer_bases.h>
 #include <linux/notifier.h>
 #include <linux/syscalls.h>
 #include <linux/interrupt.h>
@@ -707,7 +708,7 @@ static inline void hrtimer_rearm_event(ktime_t expires_next, bool deferred)
 	tick_program_event(expires_next, 1);
 }
 
-static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base, struct hrtimer *next_timer,
+static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,
 				ktime_t expires_next)
 {
 	cpu_base->expires_next = expires_next;
@@ -743,7 +744,7 @@ static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, bool skip
 	if (skip_equal && expires_next == cpu_base->expires_next)
 		return;
 
-	__hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next);
+	__hrtimer_reprogram(cpu_base, expires_next);
 }
 
 /* High resolution timer related functions */
@@ -896,7 +897,7 @@ static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
 
 	cpu_base->next_timer = timer;
 
-	__hrtimer_reprogram(cpu_base, timer, expires);
+	__hrtimer_reprogram(cpu_base, expires);
 }
 
 static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base, unsigned int active)
@@ -1039,6 +1040,30 @@ static inline void unlock_hrtimer_base(const struct hrtimer *timer, unsigned lon
 	raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);
 }
 
+/**
+ * hrtimer_update_function - Update the timer's callback function
+ * @timer:	Timer to update
+ * @function:	New callback function
+ *
+ * Only safe to call if the timer is not enqueued. Can be called in the callback function if the
+ * timer is not enqueued at the same time (see the comments above HRTIMER_STATE_ENQUEUED).
+ */
+void hrtimer_update_function(struct hrtimer *timer,
+			     enum hrtimer_restart (*function)(struct hrtimer *))
+{
+#ifdef CONFIG_PROVE_LOCKING
+	guard(raw_spinlock_irqsave)(&timer->base->cpu_base->lock);
+
+	if (WARN_ON_ONCE(hrtimer_is_queued(timer)))
+		return;
+
+	if (WARN_ON_ONCE(!function))
+		return;
+#endif
+	ACCESS_PRIVATE(timer, function) = function;
+}
+EXPORT_SYMBOL_GPL(hrtimer_update_function);
+
 /**
  * hrtimer_forward() - forward the timer expiry
  * @timer:	hrtimer to forward
@@ -1786,13 +1811,21 @@ EXPORT_SYMBOL_GPL(__hrtimer_get_remaining);
 ktime_t hrtimer_get_next_event(void)
 {
 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
-	ktime_t expires = KTIME_MAX;
 
-	guard(raw_spinlock_irqsave)(&cpu_base->lock);
-	if (!hrtimer_hres_active(cpu_base))
-		expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
+	/*
+	 * When HRES is active cmp_next_hrtimer_event() expects KTIME_MAX.
+	 *
+	 * cpu_base->hres_active is written only by the local CPU in
+	 * hrtimer_switch_to_hres() from hard interrupt context and in
+	 * hrtimers_cpu_starting() during CPU bring-up, and all callers reach
+	 * this with interrupts disabled on the same CPU, so an unlocked read is
+	 * stable without holding the lock.
+	 */
+	if (hrtimer_hres_active(cpu_base))
+		return KTIME_MAX;
 
-	return expires;
+	guard(raw_spinlock_irqsave)(&cpu_base->lock);
+	return __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
 }
 
 /**
@@ -2060,13 +2093,6 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, struct hrtimer_cloc
 	base->running = NULL;
 }
 
-static __always_inline struct hrtimer *clock_base_next_timer_safe(struct hrtimer_clock_base *base)
-{
-	struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active);
-
-	return next ? hrtimer_from_timerqueue_node(next) : NULL;
-}
-
 static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
 				 unsigned long flags, unsigned int active_mask)
 {
diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c
index d51428867a339..af529d558fa92 100644
--- a/kernel/time/jiffies.c
+++ b/kernel/time/jiffies.c
@@ -185,6 +185,24 @@ static int do_proc_int_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr,
 			     sysctl_u2k_int_conv_ms, sysctl_k2u_int_conv_ms);
 }
 
+static int sysctl_u2k_ulong_conv_ms(const ulong *u_ptr, ulong *k_ptr)
+{
+	return proc_ulong_u2k_conv_uop(u_ptr, k_ptr, sysctl_msecs_to_jiffies);
+}
+
+static int sysctl_k2u_ulong_conv_ms(ulong *u_ptr, const ulong *k_ptr)
+{
+	return proc_ulong_k2u_conv_kop(u_ptr, k_ptr, sysctl_jiffies_to_msecs);
+}
+
+static int do_proc_ulong_conv_ms_jiffies(bool *negp, ulong *u_ptr, ulong *k_ptr,
+					 int dir, const struct ctl_table *tbl)
+{
+	return proc_ulong_conv(u_ptr, k_ptr, dir, tbl, false,
+			       sysctl_u2k_ulong_conv_ms, sysctl_k2u_ulong_conv_ms);
+}
+
+
 #else // CONFIG_PROC_SYSCTL
 static int do_proc_int_conv_jiffies(bool *negp, ulong *u_ptr, int *k_ptr,
 				    int dir, const struct ctl_table *tbl)
@@ -211,6 +229,12 @@ static int do_proc_int_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr,
 {
 	return -ENOSYS;
 }
+
+static int do_proc_ulong_conv_ms_jiffies(bool *negp, ulong *u_ptr, ulong *k_ptr,
+					 int dir, const struct ctl_table *tbl)
+{
+	return -ENOSYS;
+}
 #endif
 
 /**
@@ -309,8 +333,8 @@ int proc_dointvec_ms_jiffies_minmax(const struct ctl_table *table, int dir,
 int proc_doulongvec_ms_jiffies_minmax(const struct ctl_table *table, int dir,
 				      void *buffer, size_t *lenp, loff_t *ppos)
 {
-	return proc_doulongvec_minmax_conv(table, dir, buffer, lenp, ppos,
-					   HZ, 1000l);
+	return proc_doulongvec_conv(table, dir, buffer, lenp, ppos,
+				    do_proc_ulong_conv_ms_jiffies);
 }
 EXPORT_SYMBOL(proc_doulongvec_ms_jiffies_minmax);
 
diff --git a/kernel/time/namespace.c b/kernel/time/namespace.c
index 5fa0af66cf3f7..3aff27bb0a154 100644
--- a/kernel/time/namespace.c
+++ b/kernel/time/namespace.c
@@ -293,10 +293,12 @@ int proc_timens_set_offset(struct file *file, struct task_struct *p,
 			return -EINVAL;
 		}
 
-		if (off->val.tv_sec > KTIME_SEC_MAX ||
-		    off->val.tv_sec < -KTIME_SEC_MAX)
+		if (off->val.tv_sec > KTIME_SEC_MAX || off->val.tv_sec < -KTIME_SEC_MAX)
 			return -ERANGE;
 
+		if (off->val.tv_nsec < 0 || off->val.tv_nsec >= NSEC_PER_SEC)
+			return -EINVAL;
+
 		tp = timespec64_add(tp, off->val);
 		/*
 		 * KTIME_SEC_MAX is divided by 2 to be sure that KTIME_MAX is
diff --git a/kernel/time/namespace_vdso.c b/kernel/time/namespace_vdso.c
index 0d74d160eec98..5ac7b6a6d3a87 100644
--- a/kernel/time/namespace_vdso.c
+++ b/kernel/time/namespace_vdso.c
@@ -74,11 +74,8 @@ struct page *find_timens_vvar_page(struct vm_area_struct *vma)
 		return current->nsproxy->time_ns->vvar_page;
 
 	/*
-	 * VM_PFNMAP | VM_IO protect .fault() handler from being called
-	 * through interfaces like /proc/$pid/mem or
-	 * process_vm_{readv,writev}() as long as there's no .access()
-	 * in special_mapping_vmops().
-	 * For more details check_vma_flags() and __access_remote_vm()
+	 * vvar_fault() protects this from being called through remote interfaces like
+	 * /proc/$pid/mem or process_vm_{readv,writev}().
 	 */
 
 	WARN(1, "vvar_page accessed remotely");
diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c
index 97fa99b96dd0b..d22b532ec5363 100644
--- a/kernel/time/ntp.c
+++ b/kernel/time/ntp.c
@@ -26,11 +26,13 @@
 /**
  * struct ntp_data - Structure holding all NTP related state
  * @tick_usec:		USER_HZ period in microseconds
- * @tick_length:	Adjusted tick length
- * @tick_length_base:	Base value for @tick_length
+ * @tick_length:	Tick length in ns << NTP_SCALE_SHIFT
  * @time_state:		State of the clock synchronization
  * @time_status:	Clock status bits
  * @time_offset:	Time adjustment in nanoseconds
+ * @skew_delta:		Per-tick phase slew rate for the coming second, in
+ *			@time_offset units (shifted-ns / HZ). Set by
+ *			second_overflow().
  * @time_constant:	PLL time constant
  * @time_maxerror:	Maximum error in microseconds holding the NTP sync distance
  *			(NTP dispersion + delay / 2)
@@ -38,7 +40,13 @@
  * @time_freq:		Frequency offset scaled nsecs/secs
  * @time_reftime:	Time at last adjustment in seconds
  * @time_adjust:	Adjustment value
+ * @time_adjust_frac:	Sub-microsecond remainder of @time_adjust being
+ *			delivered, in ns << NTP_SCALE_SHIFT (not divided by HZ).
  * @ntp_tick_adj:	Constant boot-param configurable NTP tick adjustment (upscaled)
+ * @cs_tick_adj:	Fixed per-second adjustment compensating for the difference
+ *			between the nominal NTP interval and the real time taken
+ *			by the clocksource's integer @cycle_interval (upscaled).
+ *			Set by the timekeeping core via ntp_clear().
  * @ntp_next_leap_sec:	Second value of the next pending leapsecond, or TIME64_MAX if no leap
  *
  * @pps_valid:		PPS signal watchdog counter
@@ -59,17 +67,19 @@
 struct ntp_data {
 	unsigned long		tick_usec;
 	u64			tick_length;
-	u64			tick_length_base;
 	int			time_state;
 	int			time_status;
 	s64			time_offset;
+	s64			skew_delta;
 	long			time_constant;
 	long			time_maxerror;
 	long			time_esterror;
 	s64			time_freq;
 	time64_t		time_reftime;
 	long			time_adjust;
+	s64			time_adjust_frac;
 	s64			ntp_tick_adj;
+	s64			cs_tick_adj;
 	time64_t		ntp_next_leap_sec;
 #ifdef CONFIG_NTP_PPS
 	int			pps_valid;
@@ -101,6 +111,9 @@ static struct ntp_data tk_ntp_data[TIMEKEEPERS_MAX] = {
 
 #define SECS_PER_DAY		86400
 #define MAX_TICKADJ		500LL		/* usecs */
+/* One microsecond of phase, in plain shifted-ns (ns << NTP_SCALE_SHIFT) */
+#define ONE_US_NS		((s64)NSEC_PER_USEC << NTP_SCALE_SHIFT)
+/* Per-tick MAX_TICKADJ slew, in plain shifted-ns */
 #define MAX_TICKADJ_SCALED \
 	(((MAX_TICKADJ * NSEC_PER_USEC) << NTP_SCALE_SHIFT) / NTP_INTERVAL_FREQ)
 #define MAX_TAI_OFFSET		100000
@@ -245,8 +258,7 @@ static inline void pps_fill_timex(struct ntp_data *ntpdata, struct __kernel_time
 #endif /* CONFIG_NTP_PPS */
 
 /*
- * Update tick_length and tick_length_base, based on tick_usec, ntp_tick_adj and
- * time_freq:
+ * Update tick_length based on tick_usec, ntp_tick_adj and time_freq:
  */
 static void ntp_update_frequency(struct ntp_data *ntpdata)
 {
@@ -255,6 +267,7 @@ static void ntp_update_frequency(struct ntp_data *ntpdata)
 	second_length		 = (u64)(tick_usec * NSEC_PER_USEC * USER_HZ) << NTP_SCALE_SHIFT;
 
 	second_length		+= ntpdata->ntp_tick_adj;
+	second_length		+= ntpdata->cs_tick_adj;
 	second_length		+= ntpdata->time_freq;
 
 	new_base		 = div_u64(second_length, NTP_INTERVAL_FREQ);
@@ -263,8 +276,7 @@ static void ntp_update_frequency(struct ntp_data *ntpdata)
 	 * Don't wait for the next second_overflow, apply the change to the
 	 * tick length immediately:
 	 */
-	ntpdata->tick_length		+= new_base - ntpdata->tick_length_base;
-	ntpdata->tick_length_base	 = new_base;
+	ntpdata->tick_length	 = new_base;
 }
 
 static inline s64 ntp_update_offset_fll(struct ntp_data *ntpdata, s64 offset64, long secs)
@@ -335,14 +347,15 @@ static void __ntp_clear(struct ntp_data *ntpdata)
 {
 	/* Stop active adjtime() */
 	ntpdata->time_adjust	= 0;
+	ntpdata->time_adjust_frac = 0;
 	ntpdata->time_status	|= STA_UNSYNC;
 	ntpdata->time_maxerror	= NTP_PHASE_LIMIT;
 	ntpdata->time_esterror	= NTP_PHASE_LIMIT;
 
 	ntp_update_frequency(ntpdata);
 
-	ntpdata->tick_length	= ntpdata->tick_length_base;
 	ntpdata->time_offset	= 0;
+	ntpdata->skew_delta	= 0;
 
 	ntpdata->ntp_next_leap_sec = TIME64_MAX;
 	/* Clear PPS state variables */
@@ -350,11 +363,26 @@ static void __ntp_clear(struct ntp_data *ntpdata)
 }
 
 /**
- * ntp_clear - Clears the NTP state variables
- * @tkid:	Timekeeper ID to be able to select proper ntp data array member
+ * ntp_clear - Clear NTP state and set the clocksource quantisation adjustment
+ * @tkid:		Timekeeper ID
+ * @cs_tick_adj:	Per-second adjustment in ns << NTP_SCALE_SHIFT
+ *
+ * The timekeeping core uses an integer number of cycles (@cycle_interval)
+ * per NTP interval, so the real time that interval represents differs from
+ * the nominal NTP_INTERVAL_LENGTH by up to half a counter period. Folding
+ * this fixed offset into @cs_tick_adj makes it an explicit part of the NTP
+ * tick_length computation in ntp.c, instead of being applied during
+ * timekeeping accumulation where the NTP code never saw it. Like
+ * @ntp_tick_adj it stays internal to the kernel; userspace still sees the
+ * nominal tick via adjtimex. NTP retains its full symmetric ±MAXFREQ range
+ * around the corrected base rate.
+ *
+ * Called whenever the clocksource is (re)configured, which is also when the
+ * rest of the NTP state must be cleared, so the two are done together.
  */
-void ntp_clear(unsigned int tkid)
+void ntp_clear(unsigned int tkid, s64 cs_tick_adj)
 {
+	tk_ntp_data[tkid].cs_tick_adj = cs_tick_adj;
 	__ntp_clear(&tk_ntp_data[tkid]);
 }
 
@@ -364,6 +392,186 @@ u64 ntp_tick_length(unsigned int tkid)
 	return tk_ntp_data[tkid].tick_length;
 }
 
+s64 ntp_get_skew_delta(unsigned int tkid)
+{
+	return tk_ntp_data[tkid].skew_delta;
+}
+
+/* Sign of @x as +1 or -1 (zero counts as positive; callers pass nonzero). */
+static inline int signof(s64 x)
+{
+	return x < 0 ? -1 : 1;
+}
+
+static s64 ntp_drain_time_offset(unsigned int tkid, s64 amount)
+{
+	struct ntp_data *ntpdata = &tk_ntp_data[tkid];
+
+	/* Only drain if amount and time_offset have the same sign */
+	if (!amount || signof(amount) != signof(ntpdata->time_offset))
+		return amount;
+
+	/* Clamp: don't overshoot zero */
+	if (abs(amount) > abs(ntpdata->time_offset)) {
+		s64 undrained = amount - ntpdata->time_offset;
+
+		ntpdata->time_offset = 0;
+		return undrained;
+	}
+
+	ntpdata->time_offset -= amount;
+	return 0;
+}
+
+/*
+ * Drain the legacy adjtime() correction (time_adjust) as it is delivered.
+ *
+ * @amount is the total intentional per-tick skew for this accumulation
+ * (skew_delta << shift), in time_offset units (shifted_ns / HZ); it covers
+ * both the exponential time_offset slew and the linear adjtime slew. This
+ * function claims only the adjtime share — capped at the MAX_TICKADJ rate —
+ * and returns the remainder for ntp_drain_time_offset().
+ *
+ * time_adjust is in whole µs. The sub-µs remainder being delivered lives in
+ * time_adjust_frac (plain shifted-ns, i.e. ns << NTP_SCALE_SHIFT -- unlike
+ * time_offset these are NOT pre-divided by HZ); we top it up by borrowing
+ * whole microseconds from time_adjust as the drain consumes it.
+ */
+static s64 ntp_drain_time_adjust(unsigned int tkid, s64 amount, unsigned int shift)
+{
+	struct ntp_data *ntpdata = &tk_ntp_data[tkid];
+	/* Sign reference: time_adjust if any whole us remain, else the drawer */
+	s64 ref = ntpdata->time_adjust ? (s64)ntpdata->time_adjust
+				       : ntpdata->time_adjust_frac;
+	s64 deliver, deficit, claimed;
+
+	if (!amount || !ref || signof(amount) != signof(ref))
+		return amount;
+
+	/*
+	 * Phase to deliver this accumulation, in plain shifted-ns. The drain
+	 * @amount is in ÷HZ units, so multiply by HZ first, then clamp to the
+	 * MAX_TICKADJ rate (MAX_TICKADJ_SCALED is the per-tick slew in
+	 * shifted-ns). Multiply-then-clamp avoids an s64 divide for the cap.
+	 */
+	deliver = min(abs(amount) * NTP_INTERVAL_FREQ,
+		      (s64)MAX_TICKADJ_SCALED << shift);
+
+	/* Top up the sub-µs drawer from whole-µs time_adjust as needed */
+	deficit = deliver - abs(ntpdata->time_adjust_frac);
+	if (deficit > 0 && ntpdata->time_adjust) {
+		long borrow = div64_u64(deficit + ONE_US_NS - 1, ONE_US_NS);
+
+		if (ntpdata->time_adjust > 0) {
+			borrow = min(borrow, ntpdata->time_adjust);
+			ntpdata->time_adjust	  -= borrow;
+			ntpdata->time_adjust_frac += (s64)borrow * ONE_US_NS;
+		} else {
+			/* Clamp without negating time_adjust (UB for LONG_MIN) */
+			if (ntpdata->time_adjust > -borrow)
+				borrow = -ntpdata->time_adjust;
+			ntpdata->time_adjust	  += borrow;
+			ntpdata->time_adjust_frac -= (s64)borrow * ONE_US_NS;
+		}
+	}
+
+	/* Never deliver more than the drawer holds */
+	deliver = min(deliver, abs(ntpdata->time_adjust_frac));
+	if (ntpdata->time_adjust_frac > 0)
+		ntpdata->time_adjust_frac -= deliver;
+	else
+		ntpdata->time_adjust_frac += deliver;
+
+	/* Return the unclaimed remainder in ÷HZ drain units for time_offset */
+	claimed = div_s64(deliver, NTP_INTERVAL_FREQ);
+	return amount - signof(amount) * claimed;
+}
+
+/*
+ * Drain one accumulation's worth of intentional skew as it is delivered.
+ *
+ * @amount is the total intentional per-tick skew for this accumulation
+ * (skew_delta << shift), in time_offset units (shifted_ns / HZ). The
+ * adjtime() linear share is taken from time_adjust first (capped at the
+ * MAX_TICKADJ rate, hence @shift), then the exponential remainder from
+ * time_offset. Returns the amount actually claimed (same ÷HZ units).
+ */
+s64 ntp_drain_skew(unsigned int tkid, s64 amount, unsigned int shift)
+{
+	s64 unclaimed = ntp_drain_time_adjust(tkid, amount, shift);
+
+	unclaimed = ntp_drain_time_offset(tkid, unclaimed);
+
+	/*
+	 * Return the amount actually drained from the intentional
+	 * phase offset in time_offset and/or time_adjust.
+	 */
+	return amount - unclaimed;
+}
+
+/*
+ * time_offset (drained exponentially) and time_adjust (drained linearly at the
+ * MAX_TICKADJ rate) can be asked to slew the clock in opposite directions.
+ * second_overflow() only folds their *net* into skew_delta, so the cancelling
+ * part would never be drained from either tracker via the per-tick code -- and
+ * if they cancel exactly, skew_delta is zero and neither converges at all.
+ *
+ * Settle that cancelling phase directly between the two here. No clock motion
+ * results (the opposing slews annihilate), but both move toward zero so neither
+ * stalls. @amount is the phase to take off time_offset, in its (÷HZ) units and
+ * with its sign; the same real magnitude comes off time_adjust in the opposite
+ * direction. Clamped so neither tracker is driven past zero.
+ */
+static void ntp_transfer_offset_adjust(struct ntp_data *ntpdata, s64 amount)
+{
+	s64 frac_delta, carry;
+
+	/*
+	 * Don't drain time_offset past zero. @amount shares its sign and is
+	 * normally bounded below it by ntp_offset_chunk(), but the ±1 skew_delta
+	 * floor for a tiny time_offset can exceed it, so clamp.
+	 */
+	if (abs(amount) > abs(ntpdata->time_offset))
+		amount = ntpdata->time_offset;
+	if (!amount)
+		return;
+
+	/*
+	 * Remove the matching phase from time_adjust, in plain shifted-ns. No
+	 * clamp against time_adjust's zero is needed: @amount is bounded by the
+	 * adjtime chunk, which second_overflow() never lets exceed time_adjust's
+	 * own pending phase, so this cannot overshoot.
+	 */
+	frac_delta = amount * NTP_INTERVAL_FREQ;
+
+	ntpdata->time_offset -= amount;
+
+	/* Add the matching phase to time_adjust, carrying whole µs (O(1)). */
+	ntpdata->time_adjust_frac += frac_delta;
+	if (ntpdata->time_adjust_frac >= ONE_US_NS ||
+	    ntpdata->time_adjust_frac <= -ONE_US_NS) {
+		carry = div64_s64(ntpdata->time_adjust_frac, ONE_US_NS);
+		ntpdata->time_adjust	  += carry;
+		ntpdata->time_adjust_frac -= carry * ONE_US_NS;
+	}
+
+	/*
+	 * Keep time_adjust and its sub-µs remainder the same sign. The
+	 * truncating carry above can leave them opposed (e.g. +4 µs paired
+	 * with -250 ns), and ntp_drain_time_adjust() treats abs(time_adjust_frac)
+	 * as same-direction drawer capacity -- an opposing remainder there makes
+	 * it over-deliver phase that was never removed from the pile. Borrow or
+	 * repay a single whole µs to realign; the total phase is unchanged.
+	 */
+	if (ntpdata->time_adjust > 0 && ntpdata->time_adjust_frac < 0) {
+		ntpdata->time_adjust--;
+		ntpdata->time_adjust_frac += ONE_US_NS;
+	} else if (ntpdata->time_adjust < 0 && ntpdata->time_adjust_frac > 0) {
+		ntpdata->time_adjust++;
+		ntpdata->time_adjust_frac -= ONE_US_NS;
+	}
+}
+
 /**
  * ntp_get_next_leap - Returns the next leapsecond in CLOCK_REALTIME ktime_t
  * @tkid:	Timekeeper ID
@@ -398,7 +606,6 @@ ktime_t ntp_get_next_leap(unsigned int tkid)
 int second_overflow(unsigned int tkid, time64_t secs)
 {
 	struct ntp_data *ntpdata = &tk_ntp_data[tkid];
-	s64 delta;
 	int leap = 0;
 	s32 rem;
 
@@ -458,35 +665,70 @@ int second_overflow(unsigned int tkid, time64_t secs)
 	}
 
 	/* Compute the phase adjustment for the next second */
-	ntpdata->tick_length	 = ntpdata->tick_length_base;
-
-	delta			 = ntp_offset_chunk(ntpdata, ntpdata->time_offset);
-	ntpdata->time_offset	-= delta;
-	ntpdata->tick_length	+= delta;
 
 	/* Check PPS signal */
 	pps_dec_valid(ntpdata);
 
-	if (!ntpdata->time_adjust)
-		goto out;
+	/*
+	 * Set the per-tick skew rate for the next second. This is in
+	 * the same units as time_offset: (ns << NTP_SCALE_SHIFT) / HZ.
+	 * If the result is so low that the skew imparted would round
+	 * to zero, pass the bare minimum ±1 to ensure that it *does*
+	 * actually drain completely to zero. It won't overshoot because
+	 * logarithmic_accumulation() only drains what it can from
+	 * time_offset or time_adjust, and the rest ends up in ntp_error
+	 * which drives the selection of 'mult' immediately each tick.
+	 */
+	if (ntpdata->time_offset || ntpdata->time_adjust ||
+	    ntpdata->time_adjust_frac) {
+		s64 off_chunk = ntp_offset_chunk(ntpdata, ntpdata->time_offset);
+		s64 adj_chunk = 0, net;
 
-	if (ntpdata->time_adjust > MAX_TICKADJ) {
-		ntpdata->time_adjust -= MAX_TICKADJ;
-		ntpdata->tick_length += MAX_TICKADJ_SCALED;
-		goto out;
-	}
+		/*
+		 * Once the exponential chunk rounds to zero, deliver the last
+		 * remaining offset this second so it converges to zero instead
+		 * of stalling just above it.
+		 */
+		if (!off_chunk)
+			off_chunk = ntpdata->time_offset;
+
+		if (ntpdata->time_adjust || ntpdata->time_adjust_frac) {
+			s64 adj;
+
+			if (ntpdata->time_adjust >= MAX_TICKADJ)
+				adj = MAX_TICKADJ * ONE_US_NS;
+			else if (ntpdata->time_adjust <= -MAX_TICKADJ)
+				adj = -MAX_TICKADJ * ONE_US_NS;
+			else
+				adj = ntpdata->time_adjust * ONE_US_NS +
+					ntpdata->time_adjust_frac;
+
+			adj_chunk = div_s64(adj, NTP_INTERVAL_FREQ);
+			if (!adj_chunk)
+				adj_chunk = signof(ntpdata->time_adjust_frac);
+		}
 
-	if (ntpdata->time_adjust < -MAX_TICKADJ) {
-		ntpdata->time_adjust += MAX_TICKADJ;
-		ntpdata->tick_length -= MAX_TICKADJ_SCALED;
-		goto out;
-	}
+		/*
+		 * If the two slews oppose, only their net would drive the
+		 * per-tick drain, so the cancelling part would never drain from
+		 * either tracker and an exact cancellation would stall both.
+		 * Settle that overlap directly between them (no clock motion).
+		 */
+		if (off_chunk && adj_chunk && signof(off_chunk) != signof(adj_chunk)) {
+			s64 conflict = min(abs(off_chunk), abs(adj_chunk));
 
-	ntpdata->tick_length += (s64)(ntpdata->time_adjust * NSEC_PER_USEC / NTP_INTERVAL_FREQ)
-				<< NTP_SCALE_SHIFT;
-	ntpdata->time_adjust = 0;
+			ntp_transfer_offset_adjust(ntpdata, signof(off_chunk) * conflict);
+		}
+
+		/* Net is what the clock delivers; reduce to per-tick, then floor. */
+		net = off_chunk + adj_chunk;
+		ntpdata->skew_delta = div_s64(net, NTP_INTERVAL_FREQ);
+		if (!ntpdata->skew_delta && net)
+			ntpdata->skew_delta = signof(net);
+	} else {
+		ntpdata->skew_delta = 0;
+	}
 
-out:
 	return leap;
 }
 
@@ -779,6 +1021,7 @@ int ntp_adjtimex(unsigned int tkid, struct __kernel_timex *txc, const struct tim
 		if (!(txc->modes & ADJ_OFFSET_READONLY)) {
 			/* adjtime() is independent from ntp_adjtime() */
 			ntpdata->time_adjust = txc->offset;
+			ntpdata->time_adjust_frac = 0;
 			ntp_update_frequency(ntpdata);
 
 			audit_ntp_set_old(ad, AUDIT_NTP_ADJUST,	save_adjust);
@@ -1020,6 +1263,7 @@ static void hardpps_update_phase(struct ntp_data *ntpdata, long error)
 					       NTP_INTERVAL_FREQ);
 		/* Cancel running adjtime() */
 		ntpdata->time_adjust = 0;
+		ntpdata->time_adjust_frac = 0;
 	}
 	/* Update jitter */
 	ntpdata->pps_jitter += (jitter - ntpdata->pps_jitter) >> PPS_INTMIN;
diff --git a/kernel/time/ntp_internal.h b/kernel/time/ntp_internal.h
index 7084d839c2074..0474a761bafc3 100644
--- a/kernel/time/ntp_internal.h
+++ b/kernel/time/ntp_internal.h
@@ -3,9 +3,11 @@
 #define _LINUX_NTP_INTERNAL_H
 
 extern void ntp_init(void);
-extern void ntp_clear(unsigned int tkid);
+extern void ntp_clear(unsigned int tkid, s64 cs_tick_adj);
 /* Returns how long ticks are at present, in ns / 2^NTP_SCALE_SHIFT. */
 extern u64 ntp_tick_length(unsigned int tkid);
+extern s64 ntp_get_skew_delta(unsigned int tkid);
+extern s64 ntp_drain_skew(unsigned int tkid, s64 amount, unsigned int shift);
 extern ktime_t ntp_get_next_leap(unsigned int tkid);
 extern int second_overflow(unsigned int tkid, time64_t secs);
 extern int ntp_adjtimex(unsigned int tkid, struct __kernel_timex *txc, const struct timespec64 *ts,
diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c
index a7d3e8229c4ba..d73d31c7994fe 100644
--- a/kernel/time/posix-cpu-timers.c
+++ b/kernel/time/posix-cpu-timers.c
@@ -1357,8 +1357,11 @@ static void handle_posix_cpu_timers(struct task_struct *tsk)
 	unsigned long flags, start;
 	LIST_HEAD(firing);
 
-	if (!lock_task_sighand(tsk, &flags))
-		return;
+	/*
+	 * tsk is current and ->sighand is stable, see the
+	 * tsk->exit_state check in run_posix_cpu_timers()
+	 */
+	spin_lock_irqsave(&tsk->sighand->siglock, flags);
 
 	do {
 		/*
@@ -1418,7 +1421,7 @@ static void handle_posix_cpu_timers(struct task_struct *tsk)
 	 * that gets the timer lock before we do will give it up and
 	 * spin until we've taken care of that timer below.
 	 */
-	unlock_task_sighand(tsk, &flags);
+	spin_unlock_irqrestore(&tsk->sighand->siglock, flags);
 
 	/*
 	 * Now that all the timers on our list have the firing flag,
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h
index 597d816d22e88..182974c4f21b5 100644
--- a/kernel/time/tick-internal.h
+++ b/kernel/time/tick-internal.h
@@ -3,6 +3,7 @@
  * tick internal variable and functions used by low/high res code
  */
 #include <linux/hrtimer.h>
+#include <linux/hrtimer_bases.h>
 #include <linux/tick.h>
 
 #include "timekeeping.h"
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 98a9cae915c09..6c3fea3867139 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -828,8 +828,7 @@ static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
 	ts->timer_expires_base = basemono;
 
 	/*
-	 * Keep the periodic tick, when RCU, architecture or irq_work
-	 * requests it.
+	 * Keep the periodic tick, when RCU or irq_work requests it.
 	 * Aside of that, check whether the local timer softirq is
 	 * pending. If so, its a bad idea to call get_next_timer_interrupt(),
 	 * because there is an already expired timer, so it will request
@@ -837,8 +836,8 @@ static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
 	 * minimal delta, which brings us back to this place
 	 * immediately. Lather, rinse and repeat...
 	 */
-	if (rcu_needs_cpu() || arch_needs_cpu() ||
-	    irq_work_needs_cpu() || local_timer_softirq_pending()) {
+	if (rcu_needs_cpu() || irq_work_needs_cpu() ||
+	    local_timer_softirq_pending()) {
 		next_tick = basemono + TICK_NSEC;
 	} else {
 		/*
diff --git a/kernel/time/time.c b/kernel/time/time.c
index 0dd63a91e7c58..079ab34f61db6 100644
--- a/kernel/time/time.c
+++ b/kernel/time/time.c
@@ -42,6 +42,13 @@
 
 #include <generated/timeconst.h>
 #include "timekeeping.h"
+#include "timekeeping_internal.h"
+
+#if defined(CONFIG_64BIT) || defined(CONFIG_COMPAT_32BIT_TIME)
+#define __WANT_OLD_TIME_TYPE_SYSCALL 1
+#endif
+
+static_assert(sizeof(__kernel_old_time_t) == 8 ? IS_ENABLED(__WANT_OLD_TIME_TYPE_SYSCALL) : true);
 
 /*
  * The timezone where the local system is located.  Used as a default by some
@@ -51,7 +58,7 @@ struct timezone sys_tz;
 
 EXPORT_SYMBOL(sys_tz);
 
-#ifdef __ARCH_WANT_SYS_TIME
+#if defined(__ARCH_WANT_SYS_TIME) && defined(__WANT_OLD_TIME_TYPE_SYSCALL)
 
 /*
  * sys_time() can be implemented in user-level using
@@ -96,7 +103,7 @@ SYSCALL_DEFINE1(stime, __kernel_old_time_t __user *, tptr)
 	return 0;
 }
 
-#endif /* __ARCH_WANT_SYS_TIME */
+#endif /* __ARCH_WANT_SYS_TIME && __WANT_OLD_TIME_TYPE_SYSCALL */
 
 #ifdef CONFIG_COMPAT_32BIT_TIME
 #ifdef __ARCH_WANT_SYS_TIME32
@@ -137,6 +144,7 @@ SYSCALL_DEFINE1(stime32, old_time32_t __user *, tptr)
 #endif /* __ARCH_WANT_SYS_TIME32 */
 #endif
 
+#ifdef __WANT_OLD_TIME_TYPE_SYSCALL
 SYSCALL_DEFINE2(gettimeofday, struct __kernel_old_timeval __user *, tv,
 		struct timezone __user *, tz)
 {
@@ -154,6 +162,7 @@ SYSCALL_DEFINE2(gettimeofday, struct __kernel_old_timeval __user *, tv,
 	}
 	return 0;
 }
+#endif /* __WANT_OLD_TIME_TYPE_SYSCALL */
 
 /*
  * In case for some reason the CMOS clock has not already been running
@@ -203,6 +212,9 @@ SYSCALL_DEFINE2(settimeofday, struct __kernel_old_timeval __user *, tv,
 	struct timezone new_tz;
 
 	if (tv) {
+		if (!IS_ENABLED(__WANT_OLD_TIME_TYPE_SYSCALL))
+			return -EINVAL;
+
 		if (get_user(new_ts.tv_sec, &tv->tv_sec) ||
 		    get_user(new_ts.tv_nsec, &tv->tv_usec))
 			return -EFAULT;
@@ -220,7 +232,7 @@ SYSCALL_DEFINE2(settimeofday, struct __kernel_old_timeval __user *, tv,
 	return do_sys_settimeofday64(tv ? &new_ts : NULL, tz ? &new_tz : NULL);
 }
 
-#ifdef CONFIG_COMPAT
+#ifdef CONFIG_COMPAT_32BIT_TIME
 COMPAT_SYSCALL_DEFINE2(gettimeofday, struct old_timeval32 __user *, tv,
 		       struct timezone __user *, tz)
 {
@@ -239,7 +251,9 @@ COMPAT_SYSCALL_DEFINE2(gettimeofday, struct old_timeval32 __user *, tv,
 
 	return 0;
 }
+#endif /* CONFIG_COMPAT_32BIT_TIME */
 
+#ifdef CONFIG_COMPAT
 COMPAT_SYSCALL_DEFINE2(settimeofday, struct old_timeval32 __user *, tv,
 		       struct timezone __user *, tz)
 {
@@ -247,6 +261,9 @@ COMPAT_SYSCALL_DEFINE2(settimeofday, struct old_timeval32 __user *, tv,
 	struct timezone new_tz;
 
 	if (tv) {
+		if (!IS_ENABLED(CONFIG_COMPAT_32BIT_TIME))
+			return -EINVAL;
+
 		if (get_user(new_ts.tv_sec, &tv->tv_sec) ||
 		    get_user(new_ts.tv_nsec, &tv->tv_usec))
 			return -EFAULT;
@@ -263,7 +280,7 @@ COMPAT_SYSCALL_DEFINE2(settimeofday, struct old_timeval32 __user *, tv,
 
 	return do_sys_settimeofday64(tv ? &new_ts : NULL, tz ? &new_tz : NULL);
 }
-#endif
+#endif /* CONFIG_COMPAT */
 
 #ifdef CONFIG_64BIT
 SYSCALL_DEFINE1(adjtimex, struct __kernel_timex __user *, txc_p)
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index b1b5ec43c0f27..97db2e9393f9d 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -339,7 +339,6 @@ static inline void clocksource_enable_inline_read(void) { }
 static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
 {
 	u64 interval;
-	u64 tmp, ntpinterval;
 	struct clocksource *old_clock;
 
 	++tk->cs_was_changed_seq;
@@ -353,20 +352,16 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
 	tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;
 
 	/* Do the ns -> cycle conversion first, using original mult */
-	tmp = NTP_INTERVAL_LENGTH;
-	tmp <<= clock->shift;
-	ntpinterval = tmp;
-	tmp += clock->mult/2;
-	do_div(tmp, clock->mult);
-	if (tmp == 0)
-		tmp = 1;
-
-	interval = (u64) tmp;
+	interval = (u64)NTP_INTERVAL_LENGTH << clock->shift;
+	interval += clock->mult / 2;
+	do_div(interval, clock->mult);
+	if (interval == 0)
+		interval = 1;
+
 	tk->cycle_interval = interval;
 
 	/* Go back from cycles -> shifted ns */
 	tk->xtime_interval = interval * clock->mult;
-	tk->xtime_remainder = ntpinterval - tk->xtime_interval;
 	tk->raw_interval = interval * clock->mult;
 
 	 /* if changing clocks, convert xtime_nsec shift units */
@@ -386,7 +381,38 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
 
 	tk->ntp_error = 0;
 	tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
-	tk->ntp_tick = ntpinterval << tk->ntp_error_shift;
+
+	/*
+	 * ntp_tick is the tick length that NTP disciplines (its ±500 PPM
+	 * scales only this part), in NTP-shifted ns: the real interval of
+	 * a whole number of counter cycles. Because cycle_interval is
+	 * rounded to an integer number of cycles, this ntp_tick differs
+	 * from the true intended 1/HZ tick length by up to half a cycle
+	 * period.
+	 */
+	tk->ntp_tick = (u64)tk->xtime_interval << tk->ntp_error_shift;
+
+	/*
+	 * cs_tick_adj is the constant difference between the disciplined
+	 * ntp_tick above and the true 1/HZ tick, expressed per-second to
+	 * match the ntp_update_frequency() addends and handed to NTP via
+	 * ntp_clear() to be explicitly included in its tick_length.
+	 *
+	 * Worked example: HZ=1000, ACPI PM timer at 3.579545 MHz, which
+	 * has 3579.545 cycles in 1ms, rounded to cycle_interval = 3580.
+	 *
+	 * So ntp_tick is actually 1.000127ms, as that is the amount of
+	 * time that 3580 cycles will take at the nominal frequency. This
+	 * is the part that NTP disciplines, causing each 3580 counts to
+	 * advance the clock by up to NTP's ±500PPM of that amount.
+	 *
+	 * The "extra" 127ns/tick is what's stored in cs_tick_adj and
+	 * applied as a constant correction by ntp_update_frequency() so
+	 * that NTP *believes* it's disciplining a 1ms tick.
+	 */
+	tk->cs_tick_adj = (s64)tk->ntp_tick -
+			  ((s64)NTP_INTERVAL_LENGTH << NTP_SCALE_SHIFT);
+	tk->cs_tick_adj *= NTP_INTERVAL_FREQ;
 
 	/*
 	 * The timekeeper keeps its own mult values for the currently
@@ -397,6 +423,7 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
 	tk->tkr_raw.mult = clock->mult;
 	tk->ntp_err_mult = 0;
 	tk->skip_second_overflow = 0;
+	tk->skew_delta = 0;
 
 	tk->cs_id = clock->id;
 
@@ -803,7 +830,7 @@ static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int act
 
 	if (action & TK_CLEAR_NTP) {
 		tk->ntp_error = 0;
-		ntp_clear(tk->id);
+		ntp_clear(tk->id, tk->cs_tick_adj);
 	}
 
 	tk_update_leap_state(tk);
@@ -1202,10 +1229,21 @@ static inline u64 tk_clock_read_snapshot(const struct tk_read_base *tkr,
 
 /**
  * ktime_get_snapshot_id -  Simultaneously snapshot a given clock ID with
- *			    CLOCK_MONOTONIC_RAW and the underlying
+ *			    the corresponding monotonic raw and the underlying
  *			    clocksource counter value.
  * @clock_id:		The clock ID to snapshot
  * @systime_snapshot:	Pointer to struct receiving the system time snapshot
+ *
+ * For the system time keeping clocks (REALTIME, MONOTONIC and BOOTTIME) the
+ * monotonic raw clock is CLOCK_MONOTONIC_RAW. For AUX clocks this is the
+ * monotonic raw clock related to the AUX clock. These AUX clock related
+ * monotonic raw clocks have a strict linear offset to the system time
+ * CLOCK_MONOTONIC_RAW:
+ *
+ *	MONOTONIC_RAW(AUX$N) = CLOCK_MONOTONIC_RAW(system) + offset(AUX$N)
+ *
+ * The offset is established when a AUX clock is initialized, but it is
+ * currently not accessible.
  */
 void ktime_get_snapshot_id(clockid_t clock_id, struct system_time_snapshot *systime_snapshot)
 {
@@ -1512,6 +1550,9 @@ EXPORT_SYMBOL_GPL(ktime_real_to_base_clock);
  * @xtstamp:		Receives simultaneously captured system and device time
  *
  * Reads a timestamp from a device and correlates it to system time
+ *
+ * See documentation for ktime_get_snapshot_id() for information about the raw
+ * monotonic time stamp which is used here.
  */
 int get_device_system_crosststamp(int (*get_time_fn)
 				  (ktime_t *device_time,
@@ -2076,7 +2117,12 @@ void __init timekeeping_init(void)
 
 	tk_set_wall_to_mono(tks, wall_to_mono);
 
-	timekeeping_update_from_shadow(&tk_core, TK_CLOCK_WAS_SET);
+	/*
+	 * Use TK_UPDATE_ALL so the NTP layer picks up the clocksource's
+	 * cs_tick_adj via ntp_clear(). Clearing NTP here is otherwise
+	 * redundant as ntp_init() already initialised it above.
+	 */
+	timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
 }
 
 /* time in seconds when suspend began for persistent clock */
@@ -2390,6 +2436,11 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
 	 *	xtime_nsec_2 = xtime_nsec_1 - offset
 	 * Which simplifies to:
 	 *	xtime_nsec -= offset
+	 *
+	 * When subtracting offset from xtime_nsec, the same amount
+	 * (in appropriate units) has to be added to ntp_error, in
+	 * order to correctly track the delta between the time
+	 * reported in xtime_nsec, and the intended time.
 	 */
 	if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) {
 		/* NTP adjustment caused clocksource mult overflow */
@@ -2400,6 +2451,7 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
 	tk->tkr_mono.mult += mult_adj;
 	tk->xtime_interval += interval;
 	tk->tkr_mono.xtime_nsec -= offset;
+	tk->ntp_error += offset << tk->ntp_error_shift;
 }
 
 /*
@@ -2409,18 +2461,27 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
 static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
 {
 	u64 ntp_tl = ntp_tick_length(tk->id);
+	s64 skew = ntp_get_skew_delta(tk->id);
 	u32 mult;
 
 	/*
-	 * Determine the multiplier from the current NTP tick length.
-	 * Avoid expensive division when the tick length doesn't change.
+	 * Determine the multiplier from the current NTP tick length plus
+	 * skew_delta. The skew biases mult so that ±1 dithering can deliver
+	 * the time_offset slew rate. Recompute when either changes.
 	 */
-	if (likely(tk->ntp_tick == ntp_tl)) {
+	if (likely(tk->ntp_tick == ntp_tl && tk->skew_delta == skew)) {
+		/* Revert to the base mult rate. */
 		mult = tk->tkr_mono.mult - tk->ntp_err_mult;
 	} else {
 		tk->ntp_tick = ntp_tl;
-		mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) -
-				 tk->xtime_remainder, tk->cycle_interval);
+		tk->skew_delta = skew;
+		/*
+		 * skew_delta is stored pre-divided by HZ (matching time_offset);
+		 * scale it back up to the full per-tick rate for the mult bias.
+		 */
+		skew *= NTP_INTERVAL_FREQ;
+		mult = div64_u64((tk->ntp_tick + skew) >> tk->ntp_error_shift,
+				 tk->cycle_interval);
 	}
 
 	/*
@@ -2545,8 +2606,25 @@ static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset,
 
 	/* Accumulate error between NTP and clock interval */
 	tk->ntp_error += tk->ntp_tick << shift;
-	tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
-						(tk->ntp_error_shift + shift);
+	tk->ntp_error -= tk->xtime_interval << (tk->ntp_error_shift + shift);
+
+	/*
+	 * When skewing, do so by adjusting ntp_error to impart an extra
+	 * target delta into ntp_error per tick, limited to what can be
+	 * drained from time_offset / time_adjust to avoid overshoot.
+	 *
+	 * The base 'mult' value was calculated with the skew taken into
+	 * account, such that the per-tick choice of 'mult' vs. 'mult+1'
+	 * allows for the desired effective rate and ntp_error does not
+	 * grow unbounded.
+	 *
+	 * Once the full desired phase offset is delivered, any remaining
+	 * skew imparted by the adjusted 'mult', accounted above, remains
+	 * in ntp_error and will be compensated by the dithering over time.
+	 */
+	if (tk->skew_delta)
+		tk->ntp_error += ntp_drain_skew(tk->id, tk->skew_delta << shift,
+						shift) * NTP_INTERVAL_FREQ;
 
 	return offset;
 }
@@ -3313,7 +3391,9 @@ static const struct attribute_group aux_clock_enable_attr_group = {
 static int __init tk_aux_sysfs_init(void)
 {
 	struct kobject *auxo, *tko = kobject_create_and_add("time", kernel_kobj);
+	struct kobject *clks[MAX_AUX_CLOCKS];
 	int ret = -ENOMEM;
+	int i;
 
 	if (!tko)
 		return ret;
@@ -3322,21 +3402,28 @@ static int __init tk_aux_sysfs_init(void)
 	if (!auxo)
 		goto err_clean;
 
-	for (int i = 0; i < MAX_AUX_CLOCKS; i++) {
+	for (i = 0; i < MAX_AUX_CLOCKS; i++) {
 		char id[2] = { [0] = '0' + i, };
-		struct kobject *clk = kobject_create_and_add(id, auxo);
+		clks[i] = kobject_create_and_add(id, auxo);
 
-		if (!clk) {
+		if (!clks[i]) {
 			ret = -ENOMEM;
-			goto err_clean;
+			goto err_clks;
 		}
 
-		ret = sysfs_create_group(clk, &aux_clock_enable_attr_group);
+		ret = sysfs_create_group(clks[i], &aux_clock_enable_attr_group);
 		if (ret)
-			goto err_clean;
+			goto err_clk;
 	}
 	return 0;
 
+err_clk:
+	kobject_put(clks[i]);
+err_clks:
+	while (--i >= 0) {
+		sysfs_remove_group(clks[i], &aux_clock_enable_attr_group);
+		kobject_put(clks[i]);
+	}
 err_clean:
 	kobject_put(auxo);
 	kobject_put(tko);
diff --git a/kernel/time/timekeeping_internal.h b/kernel/time/timekeeping_internal.h
index 973ede670a36e..6d719b8e5ea2d 100644
--- a/kernel/time/timekeeping_internal.h
+++ b/kernel/time/timekeeping_internal.h
@@ -6,6 +6,8 @@
 #include <linux/spinlock.h>
 #include <linux/time.h>
 
+struct timekeeper;
+
 /*
  * timekeeping debug functions
  */
@@ -48,4 +50,23 @@ void timekeeper_unlock_irqrestore(unsigned long flags);
 /* NTP specific interface to access the current seconds value */
 long ktime_get_ntp_seconds(unsigned int id);
 
+#ifdef CONFIG_GENERIC_GETTIMEOFDAY
+
+extern void update_vsyscall(struct timekeeper *tk);
+extern void update_vsyscall_tz(void);
+extern void vdso_time_update_aux(struct timekeeper *tk);
+
+#else
+
+static inline void update_vsyscall(struct timekeeper *tk)
+{
+}
+static inline void update_vsyscall_tz(void)
+{
+}
+static inline void vdso_time_update_aux(struct timekeeper *tk)
+{
+}
+#endif
+
 #endif /* _TIMEKEEPING_INTERNAL_H */
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c
index 514802def1e02..8823e7dcfce6b 100644
--- a/kernel/time/timer_list.c
+++ b/kernel/time/timer_list.c
@@ -118,12 +118,15 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now)
 		SEQ_printf(m, " clock %d:\n", i);
 		print_base(m, cpu_base->clock_base + i, now);
 	}
-#define P(x) \
+
+#define DIAG_READ(x) data_race(READ_ONCE(x))
+
+#define P(x)				     \
 	SEQ_printf(m, "  .%-15s: %Lu\n", #x, \
-		   (unsigned long long)(cpu_base->x))
+		   (unsigned long long)DIAG_READ(cpu_base->x))
 #define P_ns(x) \
 	SEQ_printf(m, "  .%-15s: %Lu nsecs\n", #x, \
-		   (unsigned long long)(ktime_to_ns(cpu_base->x)))
+		   (unsigned long long)ktime_to_ns(DIAG_READ(cpu_base->x)))
 
 #ifdef CONFIG_HIGH_RES_TIMERS
 	P_ns(expires_next);
@@ -139,12 +142,12 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now)
 #ifdef CONFIG_TICK_ONESHOT
 # define P(x) \
 	SEQ_printf(m, "  .%-15s: %Lu\n", #x, \
-		   (unsigned long long)(ts->x))
+		   (unsigned long long)DIAG_READ(ts->x))
 # define P_ns(x) \
 	SEQ_printf(m, "  .%-15s: %Lu nsecs\n", #x, \
-		   (unsigned long long)(ktime_to_ns(ts->x)))
+		   (unsigned long long)ktime_to_ns(DIAG_READ(ts->x)))
 # define P_flag(x, f)			    \
-	SEQ_printf(m, "  .%-15s: %d\n", #x, !!(ts->flags & (f)))
+	SEQ_printf(m, "  .%-15s: %d\n", #x, !!(DIAG_READ(ts->flags) & (f)))
 
 	{
 		struct tick_sched *ts = tick_get_tick_sched(cpu);
@@ -166,6 +169,8 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now)
 
 #undef P
 #undef P_ns
+#undef P_flag
+#undef DIAG_READ
 	SEQ_printf(m, "\n");
 }
 
diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c
index 806c23cf71fc9..059d43355e650 100644
--- a/kernel/time/timer_migration.c
+++ b/kernel/time/timer_migration.c
@@ -1847,8 +1847,10 @@ static int tmigr_setup_groups(struct tmigr_hierarchy *hier, unsigned int cpu,
 	}
 
 	/* Assert single root without parent */
-	if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels))
+	if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels)) {
+		kfree(stack);
 		return -EINVAL;
+	}
 
 	for (; i >= start_lvl; i--) {
 		group = stack[i];
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 084f34dc6c9f3..0ab5916575a97 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -779,6 +779,18 @@ config PROBE_EVENTS_BTF_ARGS
 	  kernel function entry or a tracepoint.
 	  This is available only if BTF (BPF Type Format) support is enabled.
 
+config PROBE_EVENTS_DUMP_FETCHARG
+	bool "Dump of dynamic probe event fetch-arguments"
+	depends on PROBE_EVENTS
+	default n
+	help
+	  This shows the dump of fetch-arguments of dynamic probe events
+	  alongside their event definitions in the dynamic_events file
+	  as comment lines. This is useful to debug the probe events.
+	  Since this exposes the raw values in the dynamic_events file,
+	  it might be a security risk. Only enable it if you need to debug
+	  probe events themselves.
+
 config KPROBE_EVENTS
 	depends on KPROBES
 	depends on HAVE_REGS_AND_STACK_ACCESS_API
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index 75495a5c35079..76ab51deaa6b9 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -3700,6 +3700,60 @@ static void bpf_tracing_multi_link_dealloc(struct bpf_link *link)
 	kvfree(tr_link);
 }
 
+static int bpf_tracing_multi_link_fill_link_info(const struct bpf_link *link,
+						 struct bpf_link_info *info)
+{
+	u64 __user *ucookies = u64_to_user_ptr(info->tracing_multi.cookies);
+	u64 __user *uaddrs = u64_to_user_ptr(info->tracing_multi.addrs);
+	u32 __user *uids = u64_to_user_ptr(info->tracing_multi.ids);
+	struct bpf_tracing_multi_link *tr_link;
+	u32 ucount = info->tracing_multi.count;
+	bool has_cookies, show_addrs;
+	int err = 0;
+
+	if ((uids || ucookies || uaddrs) && !ucount)
+		return -EINVAL;
+
+	tr_link = container_of(link, struct bpf_tracing_multi_link, link);
+
+	info->tracing_multi.attach_type = tr_link->link.attach_type;
+	info->tracing_multi.count = tr_link->nodes_cnt;
+	info->tracing_multi.btf_obj_id = btf_obj_id(tr_link->link.prog->aux->attach_btf);
+
+	if (!uids && !ucookies && !uaddrs)
+		return 0;
+
+	if (ucount < tr_link->nodes_cnt)
+		err = -ENOSPC;
+	else
+		ucount = tr_link->nodes_cnt;
+
+	has_cookies = !!tr_link->cookies;
+	show_addrs = kallsyms_show_value(current_cred());
+
+	for (int i = 0; i < ucount; i++) {
+		struct bpf_tracing_multi_node *mnode = &tr_link->nodes[i];
+		u64 addr, cookie;
+		u32 id;
+
+		bpf_trampoline_unpack_key(mnode->trampoline->key, NULL, &id);
+
+		addr = show_addrs ? mnode->trampoline->ip : 0;
+		cookie = has_cookies ? tr_link->cookies[i] : 0;
+
+		if (uids && put_user(id, uids + i))
+			return -EFAULT;
+		if (uaddrs && put_user(addr, uaddrs + i))
+			return -EFAULT;
+		if (ucookies && put_user(cookie, ucookies + i))
+			return -EFAULT;
+
+		cond_resched();
+	}
+
+	return err;
+}
+
 #ifdef CONFIG_PROC_FS
 static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link,
 					  struct seq_file *seq)
@@ -3730,6 +3784,7 @@ static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link,
 static const struct bpf_link_ops bpf_tracing_multi_link_lops = {
 	.release = bpf_tracing_multi_link_release,
 	.dealloc_deferred = bpf_tracing_multi_link_dealloc,
+	.fill_link_info = bpf_tracing_multi_link_fill_link_info,
 #ifdef CONFIG_PROC_FS
 	.show_fdinfo = bpf_tracing_multi_show_fdinfo,
 #endif
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 804ccae694d21..e863eb0d10f78 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -7611,7 +7611,8 @@ static int __rb_inc_dec_mapped(struct ring_buffer_per_cpu *cpu_buffer,
 static int __rb_map_vma(struct ring_buffer_per_cpu *cpu_buffer,
 			struct vm_area_struct *vma)
 {
-	unsigned long nr_subbufs, nr_pages, nr_vma_pages, pgoff = vma->vm_pgoff;
+	unsigned long nr_subbufs, nr_pages, nr_vma_pages;
+	pgoff_t pgoff = vma_start_pgoff(vma);
 	unsigned int subbuf_pages, subbuf_order;
 	struct page **pages __free(kfree) = NULL;
 	int p = 0, s = 0;
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 18710c190c924..4eb177a0a9271 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -4320,14 +4320,16 @@ static const char readme_msg[] =
 	"\t     args: <name>=fetcharg[:type]\n"
 	"\t fetcharg: (%<register>|$<efield>), @<address>, @<symbol>[+|-<offset>],\n"
 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
-	"\t           $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
+	"\t           $stack<index>, $stack, $retval, $comm, $arg<N>, $current\n"
 #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
-	"\t           <argname>[->field[->field|.field...]],\n"
+	"\t           [(structname[,field])]<argname>[->field[->field|.field...]],\n"
+	"\t           [(structname[,field])](fetcharg)->field[->field|.field...],\n"
 #endif
 #else
-	"\t           $stack<index>, $stack, $retval, $comm,\n"
+	"\t           $stack<index>, $stack, $retval, $comm, $current\n"
 #endif
 	"\t           +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
+	"\t           this_cpu_read(<fetcharg>), this_cpu_ptr(<fetcharg>)\n"
 	"\t     kernel return probes support: $retval, $arg<N>, $comm\n"
 	"\t     type: s8/16/32/64, u8/16/32/64, x8/16/32/64, char, string, symbol,\n"
 	"\t           b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
diff --git a/kernel/trace/trace_eprobe.c b/kernel/trace/trace_eprobe.c
index 50518b0714144..462c311457331 100644
--- a/kernel/trace/trace_eprobe.c
+++ b/kernel/trace/trace_eprobe.c
@@ -87,6 +87,8 @@ static int eprobe_dyn_event_show(struct seq_file *m, struct dyn_event *ev)
 		seq_printf(m, " %s=%s", ep->tp.args[i].name, ep->tp.args[i].comm);
 	seq_putc(m, '\n');
 
+	trace_probe_dump_args(m, &ep->tp);
+
 	return 0;
 }
 
diff --git a/kernel/trace/trace_fprobe.c b/kernel/trace/trace_fprobe.c
index 9f5f08c0e7c2f..536781cd4c47a 100644
--- a/kernel/trace/trace_fprobe.c
+++ b/kernel/trace/trace_fprobe.c
@@ -764,7 +764,7 @@ static int unregister_fprobe_event(struct trace_fprobe *tf)
 	return trace_probe_unregister_event_call(&tf->tp);
 }
 
-static int __regsiter_tracepoint_fprobe(struct trace_fprobe *tf)
+static int __register_tracepoint_fprobe(struct trace_fprobe *tf)
 {
 	struct tracepoint_user *tuser __free(tuser_put) = NULL;
 	struct module *mod __free(module_put) = NULL;
@@ -836,7 +836,7 @@ static int __register_trace_fprobe(struct trace_fprobe *tf)
 	tf->fp.flags &= ~FPROBE_FL_DISABLED;
 
 	if (trace_fprobe_is_tracepoint(tf))
-		return __regsiter_tracepoint_fprobe(tf);
+		return __register_tracepoint_fprobe(tf);
 
 	/* TODO: handle filter, nofilter or symbol list */
 	return register_fprobe(&tf->fp, tf->symbol, NULL);
@@ -1449,6 +1449,8 @@ static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev)
 		seq_printf(m, " %s=%s", tf->tp.args[i].name, tf->tp.args[i].comm);
 	seq_putc(m, '\n');
 
+	trace_probe_dump_args(m, &tf->tp);
+
 	return 0;
 }
 
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index a8420e6abb56a..cfa807d8e760f 100644
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -1320,6 +1320,8 @@ static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev)
 		seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);
 	seq_putc(m, '\n');
 
+	trace_probe_dump_args(m, &tk->tp);
+
 	return 0;
 }
 
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c
index d17cfee77d9ce..18c2121223440 100644
--- a/kernel/trace/trace_probe.c
+++ b/kernel/trace/trace_probe.c
@@ -345,6 +345,109 @@ static int parse_trace_event(char *arg, struct fetch_insn *code,
 	return -EINVAL;
 }
 
+/* this_cpu_* parser */
+#define THIS_CPU_PTR_PREFIX "this_cpu_ptr("
+#define THIS_CPU_READ_PREFIX "this_cpu_read("
+#define THIS_CPU_PTR_LEN (sizeof(THIS_CPU_PTR_PREFIX) - 1)
+#define THIS_CPU_READ_LEN (sizeof(THIS_CPU_READ_PREFIX) - 1)
+
+static int
+parse_probe_arg(char *arg, const struct fetch_type *type,
+		struct fetch_insn **pcode, struct fetch_insn *end,
+		struct traceprobe_parse_context *ctx);
+
+/* handle dereference nested call */
+static inline int handle_dereference(char *arg, struct fetch_insn **pcode,
+	struct fetch_insn *end, struct traceprobe_parse_context *ctx,
+	int deref, long offset)
+{
+	const struct fetch_type *type = find_fetch_type(NULL, ctx->flags);
+	struct fetch_insn *code = *pcode;
+	int cur_offs = ctx->offset;
+	char *tmp;
+	int ret;
+
+	tmp = strrchr(arg, ')');
+	if (!tmp) {
+		trace_probe_log_err(ctx->offset + strlen(arg),
+					DEREF_OPEN_BRACE);
+		return -EINVAL;
+	}
+
+	*tmp = '\0';
+	ret = parse_probe_arg(arg, type, &code, end, ctx);
+	if (ret)
+		return ret;
+	ctx->offset = cur_offs;
+	if (code->op == FETCH_OP_COMM || code->op == FETCH_OP_IMMSTR) {
+		trace_probe_log_err(ctx->offset, COMM_CANT_DEREF);
+		return -EINVAL;
+	}
+
+	/*
+	 * this_cpu_ptr(@SYM) does not use SYM value, but use SYM address.
+	 * So we overwrite the last FETCH_OP_DEREF with FETCH_OP_CPU_PTR.
+	 */
+	if (!(deref == FETCH_OP_CPU_PTR && *arg == '@')) {
+		code++;
+		if (code == end) {
+			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
+			return -EINVAL;
+		}
+	}
+	*pcode = code;
+
+	code->op = deref;
+	code->offset = offset;
+	/* Reset the last type if used */
+	ctx->last_type = NULL;
+	return 0;
+}
+
+static int parse_this_cpu(char *arg, struct fetch_insn **pcode,
+			  struct fetch_insn *end,
+			  struct traceprobe_parse_context *ctx)
+{
+	struct fetch_insn *code;
+	bool is_ptr = false;
+	int ret;
+
+	/*
+	 * This is only for kernel probes, excluding eprobe, because per-cpu
+	 * pointer should not be recorded by events.
+	 */
+	if (!(ctx->flags & TPARG_FL_KERNEL) ||
+	    (ctx->flags & TPARG_FL_TEVENT)) {
+		trace_probe_log_err(ctx->offset, NOSUP_PERCPU);
+		return -EINVAL;
+	}
+	if (str_has_prefix(arg, THIS_CPU_PTR_PREFIX)) {
+		arg += THIS_CPU_PTR_LEN;
+		ctx->offset += THIS_CPU_PTR_LEN;
+		is_ptr = true;
+	} else if (str_has_prefix(arg, THIS_CPU_READ_PREFIX)) {
+		arg += THIS_CPU_READ_LEN;
+		ctx->offset += THIS_CPU_READ_LEN;
+	} else
+		return -EINVAL;
+
+	ret = handle_dereference(arg, pcode, end, ctx, FETCH_OP_CPU_PTR, 0);
+	if (ret || is_ptr)
+		return ret;
+
+	/* this_cpu_read(VAR) -> +0(this_cpu_ptr(VAR)) */
+	code = *pcode;
+	code++;
+	if (code == end) {
+		trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
+		return -EINVAL;
+	}
+	code->op = FETCH_OP_DEREF;
+	code->offset = 0;
+	*pcode = code;
+	return 0;
+}
+
 #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
 
 static u32 btf_type_int(const struct btf_type *t)
@@ -356,9 +459,8 @@ static bool btf_type_is_char_ptr(struct btf *btf, const struct btf_type *type)
 {
 	const struct btf_type *real_type;
 	u32 intdata;
-	s32 tid;
 
-	real_type = btf_type_skip_modifiers(btf, type->type, &tid);
+	real_type = btf_type_skip_modifiers(btf, type->type, NULL);
 	if (!real_type)
 		return false;
 
@@ -375,14 +477,13 @@ static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type)
 	const struct btf_type *real_type;
 	const struct btf_array *array;
 	u32 intdata;
-	s32 tid;
 
 	if (BTF_INFO_KIND(type->info) != BTF_KIND_ARRAY)
 		return false;
 
 	array = (const struct btf_array *)(type + 1);
 
-	real_type = btf_type_skip_modifiers(btf, array->type, &tid);
+	real_type = btf_type_skip_modifiers(btf, array->type, NULL);
 
 	intdata = btf_type_int(real_type);
 	return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
@@ -570,6 +671,64 @@ static int split_next_field(char *varname, char **next_field,
 	return ret;
 }
 
+/* Inner loop for solving dot operator ('.'). Return bit-offset of the given field */
+static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **ptype,
+				  struct traceprobe_parse_context *ctx)
+{
+	const struct btf_type *type = *ptype;
+	const struct btf_member *field;
+	struct btf *btf = ctx_btf(ctx);
+	char *fieldname = *pfieldname;
+	int bitoffs = 0;
+	u32 anon_offs;
+	char *next;
+	int is_ptr;
+
+	do {
+		next = NULL;
+		is_ptr = split_next_field(fieldname, &next, ctx);
+		if (is_ptr < 0)
+			return is_ptr;
+
+		anon_offs = 0;
+		field = btf_find_struct_member(btf, type, fieldname,
+						&anon_offs);
+		if (IS_ERR(field)) {
+			trace_probe_log_err(ctx->offset, BAD_BTF_TID);
+			return PTR_ERR(field);
+		}
+		if (!field) {
+			trace_probe_log_err(ctx->offset, NO_BTF_FIELD);
+			return -ENOENT;
+		}
+		/* Add anonymous structure/union offset */
+		bitoffs += anon_offs;
+
+		/* Accumulate the bit-offsets of the dot-connected fields */
+		if (btf_type_kflag(type)) {
+			bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset);
+			ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset);
+		} else {
+			bitoffs += field->offset;
+			ctx->last_bitsize = 0;
+		}
+
+			type = btf_type_skip_modifiers(btf, field->type, NULL);
+			if (!type) {
+				trace_probe_log_err(ctx->offset, BAD_BTF_TID);
+				return -EINVAL;
+			}
+
+		if (next)
+			ctx->offset += next - fieldname;
+		fieldname = next;
+	} while (!is_ptr && fieldname);
+
+	*pfieldname = fieldname;
+	*ptype = type;
+
+	return bitoffs;
+}
 /*
  * Parse the field of data structure. The @type must be a pointer type
  * pointing the target data structure type.
@@ -579,16 +738,13 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type,
 			   struct traceprobe_parse_context *ctx)
 {
 	struct fetch_insn *code = *pcode;
-	const struct btf_member *field;
-	u32 bitoffs, anon_offs;
-	bool is_struct = ctx->struct_btf != NULL;
 	struct btf *btf = ctx_btf(ctx);
-	char *next;
-	int is_ptr;
-	s32 tid;
+	bool is_first_field = true;
+	int bitoffs;
 
 	do {
-		if (!is_struct) {
+		/* For the first field of typecast, @type will be the target structure type. */
+		if (!(is_first_field && ctx->struct_btf)) {
 			/* Outer loop for solving arrow operator ('->') */
 			if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) {
 				trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
@@ -596,66 +752,31 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type,
 			}
 
 			/* Convert a struct pointer type to a struct type */
-			type = btf_type_skip_modifiers(btf, type->type, &tid);
+			type = btf_type_skip_modifiers(btf, type->type, NULL);
 			if (!type) {
 				trace_probe_log_err(ctx->offset, BAD_BTF_TID);
 				return -EINVAL;
 			}
 		}
-		/* Only the first type can skip being a pointer */
-		is_struct = false;
-
-		bitoffs = 0;
-		do {
-			/* Inner loop for solving dot operator ('.') */
-			next = NULL;
-			is_ptr = split_next_field(fieldname, &next, ctx);
-			if (is_ptr < 0)
-				return is_ptr;
-
-			anon_offs = 0;
-			field = btf_find_struct_member(btf, type, fieldname,
-						       &anon_offs);
-			if (IS_ERR(field)) {
-				trace_probe_log_err(ctx->offset, BAD_BTF_TID);
-				return PTR_ERR(field);
-			}
-			if (!field) {
-				trace_probe_log_err(ctx->offset, NO_BTF_FIELD);
-				return -ENOENT;
-			}
-			/* Add anonymous structure/union offset */
-			bitoffs += anon_offs;
-
-			/* Accumulate the bit-offsets of the dot-connected fields */
-			if (btf_type_kflag(type)) {
-				bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset);
-				ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset);
-			} else {
-				bitoffs += field->offset;
-				ctx->last_bitsize = 0;
-			}
-
-			type = btf_type_skip_modifiers(btf, field->type, &tid);
-			if (!type) {
-				trace_probe_log_err(ctx->offset, BAD_BTF_TID);
-				return -EINVAL;
-			}
-
-			ctx->offset += next - fieldname;
-			fieldname = next;
-		} while (!is_ptr && fieldname);
 
+		bitoffs = get_bitoffset_of_field(&fieldname, &type, ctx);
+		if (bitoffs < 0)
+			return bitoffs;
 		if (++code == end) {
 			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
 			return -EINVAL;
 		}
 		code->op = FETCH_OP_DEREF;	/* TODO: user deref support */
 		code->offset = bitoffs / 8;
+		if (is_first_field && ctx->struct_btf) {
+			/* The first field can be typecasted with field option. */
+			code->offset -= ctx->prefix_byteoffs;
+		}
 		*pcode = code;
 
 		ctx->last_bitoffs = bitoffs % 8;
 		ctx->last_type = type;
+		is_first_field = false;
 	} while (fieldname);
 
 	return 0;
@@ -674,7 +795,9 @@ static int parse_btf_arg(char *varname,
 	int i, is_ptr, ret;
 	u32 tid;
 
-	if (!ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT))
+	/* Note: field is not separated at this point, so check prefix. */
+	if (!str_has_prefix(varname, "$current") &&
+	    !ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT))
 		return -EINVAL;
 
 	is_ptr = split_next_field(varname, &field, ctx);
@@ -687,22 +810,23 @@ static int parse_btf_arg(char *varname,
 		return -EOPNOTSUPP;
 	}
 
-	if (ctx->flags & TPARG_FL_TEVENT) {
-		ret = parse_trace_event(varname, code, ctx);
+	if (!strcmp(varname, "$current")) {
+		code->op = FETCH_OP_CURRENT;
+		/* If no typecast is specified for $current, use task_struct by default */
+		ret = bpf_find_btf_id("task_struct", BTF_KIND_STRUCT, &ctx->struct_btf);
 		if (ret < 0) {
-			trace_probe_log_err(ctx->offset, BAD_ATTACH_ARG);
-			return ret;
+			trace_probe_log_err(ctx->offset, NO_BTF_ENTRY);
+			return -ENOENT;
 		}
-		/* TEVENT is only here via a typecast */
-		if (WARN_ON_ONCE(ctx->struct_btf == NULL))
-			return -EINVAL;
-		type = ctx->last_struct;
+		tid = (u32)ret;
+		type = ctx->last_struct =
+			btf_type_skip_modifiers(ctx->struct_btf, tid, NULL);
 		goto found_type;
 	}
 
 	if (ctx->flags & TPARG_FL_RETURN && !strcmp(varname, "$retval")) {
 		code->op = FETCH_OP_RETVAL;
-		/* Check whether the function return type is not void */
+		/* Check whether the function return type is not void, even with typecast. */
 		if (query_btf_context(ctx) == 0) {
 			if (ctx->proto->type == 0) {
 				trace_probe_log_err(ctx->offset, NO_RETVAL);
@@ -755,7 +879,7 @@ static int parse_btf_arg(char *varname,
 	return -ENOENT;
 
 found:
-	type = btf_type_skip_modifiers(ctx->btf, tid, &tid);
+	type = btf_type_skip_modifiers(ctx->btf, tid, NULL);
 found_type:
 	if (!type) {
 		trace_probe_log_err(ctx->offset, BAD_BTF_TID);
@@ -825,17 +949,89 @@ static int query_btf_struct(const char *sname, struct traceprobe_parse_context *
 	return 0;
 }
 
+static int parse_btf_casttype(char *casttype, struct traceprobe_parse_context *ctx)
+{
+	char *field;
+	int ret;
+
+	/* Field option - evaluated later. */
+	field = strchr(casttype, ',');
+	if (field)
+		*field++ = '\0';
+
+	ret = query_btf_struct(casttype, ctx);
+	if (ret < 0) {
+		trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
+		return -EINVAL;
+	}
+
+	if (field) {
+		struct btf_type *type = (struct btf_type *)ctx->last_struct;
+
+		ctx->offset += field - casttype;
+		ret = get_bitoffset_of_field(&field, &ctx->last_struct, ctx);
+		if (ret < 0)
+			return ret;
+		if (ret % 8) {
+			trace_probe_log_err(ctx->offset, TYPECAST_NOT_ALIGNED);
+			return -EINVAL;
+		}
+		if (field != NULL) {
+			/* this means @field skips an arrow operator ("->"). */
+			trace_probe_log_err(ctx->offset - 2, TYPECAST_BAD_ARROW);
+			return -EINVAL;
+		}
+		ctx->prefix_byteoffs = ret / 8;
+		/* Restore the original struct type (overwritten by get_bitoffset_of_field) */
+		ctx->last_struct = type;
+	}
+
+	return ret;
+}
+
+/* Find the matching closing parenthesis for a given opening parenthesis. */
+static char *find_matched_close_paren(char *s)
+{
+	char *p = s;
+	int count = 0;
+
+	while (*p) {
+		if (*p == '(')
+			count++;
+		else if (*p == ')') {
+			if (--count == 0)
+				return p;
+		}
+		p++;
+	}
+	return NULL;
+}
+
 static int handle_typecast(char *arg, struct fetch_insn **pcode,
 			   struct fetch_insn *end,
 			   struct traceprobe_parse_context *ctx)
 {
+	int orig_offset = ctx->offset;
+	char *close;
 	char *tmp;
 	int ret;
 
-	/* Currently this only works for eprobes */
-	if (!(ctx->flags & TPARG_FL_TEVENT)) {
-		trace_probe_log_err(ctx->offset, TYPECAST_NOT_EVENT);
-		return -EINVAL;
+	if (!(tparg_is_event_probe(ctx->flags) ||
+	      tparg_is_function_entry(ctx->flags) ||
+	      tparg_is_function_return(ctx->flags))) {
+		trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
+		return -EOPNOTSUPP;
+	}
+
+	/*
+	 * Always consider the token after typecast as a nested call
+	 * For example: (STRUCT)VAR->FIELD and (STRUCT)(VAR)->FIELD are same.
+	 * VAR is solved in the nested call.
+	 */
+	ctx->nested_level++;
+	if (ctx->nested_level > TRACEPROBE_MAX_NESTED_LEVEL) {
+		trace_probe_log_err(ctx->offset, TOO_MANY_NESTED);
+		return -E2BIG;
 	}
 
 	tmp = strchr(arg, ')');
@@ -844,19 +1040,92 @@ static int handle_typecast(char *arg, struct fetch_insn **pcode,
 				    DEREF_OPEN_BRACE);
 		return -EINVAL;
 	}
-	*tmp = '\0';
-	ret = query_btf_struct(arg + 1, ctx);
-	*tmp = ')';
+	*tmp++ = '\0';
 
-	if (ret < 0) {
-		trace_probe_log_err(ctx->offset + 1, NO_PTR_STRCT);
-		return -EINVAL;
+	ctx->offset += tmp - arg;
+	if (*tmp == '(') {
+		close = find_matched_close_paren(tmp);
+
+		if (!close) {
+			trace_probe_log_err(ctx->offset, DEREF_OPEN_BRACE);
+			return -EINVAL;
+		}
+		/* We expect a field access for typecast */
+		if (close[1] != '-' || close[2] != '>') {
+			trace_probe_log_err(ctx->offset + close - tmp + 1,
+					    TYPECAST_REQ_FIELD);
+			return -EINVAL;
+		}
+		/* Skip '(' */
+		ctx->offset += 1;
+		tmp++;
+	} else if (*tmp == '+' || *tmp == '-' ||
+		   str_has_prefix(tmp, THIS_CPU_PTR_PREFIX) ||
+		   str_has_prefix(tmp, THIS_CPU_READ_PREFIX)) {
+		/* Dereference can have another field access inside it. */
+		char *open = strchr(tmp + 1, '(');
+
+		if (!open) {
+			trace_probe_log_err(ctx->offset,
+					    DEREF_NEED_BRACE);
+			return -EINVAL;
+		}
+		close = find_matched_close_paren(open);
+		if (!close) {
+			trace_probe_log_err(ctx->offset + strlen(tmp),
+					    DEREF_OPEN_BRACE);
+			return -EINVAL;
+		}
+		close++;
+		/* We expect a field access for typecast */
+		if (close[0] != '-' || close[1] != '>') {
+			trace_probe_log_err(ctx->offset + close - tmp,
+					    TYPECAST_REQ_FIELD);
+			return -EINVAL;
+		}
+	} else {
+		if (tmp[0] == '@') {
+			/* @sym+offset is not allowed without parenthesized */
+			close = strpbrk(tmp, "+-");
+			if (close && isdigit(close[1])) {
+				trace_probe_log_err(ctx->offset,
+						    TYPECAST_SYM_OFFSET);
+				return -EINVAL;
+			}
+		}
+		/* Inner variable name */
+		close = strchr(tmp, '-');
+		if (!close || close[1] != '>') {
+			trace_probe_log_err(ctx->offset + strlen(tmp),
+					    TYPECAST_REQ_FIELD);
+			return -EINVAL;
+		}
 	}
+	*close = '\0';
 
-	tmp++;
+	/* We need to parse the nested one */
+	ret = parse_probe_arg(tmp, find_fetch_type(NULL, ctx->flags),
+			      pcode, end, ctx);
+	if (ret < 0)
+		return ret;
+	ctx->nested_level--;
+	clear_struct_btf(ctx);
 
-	ctx->offset += tmp - arg;
-	ret = parse_btf_arg(tmp, pcode, end, ctx);
+	/* Let tmp point the field name. */
+	if (close[1] == '-')
+		tmp = close + 3; /* Skip "->" after closing parenthesis */
+	else
+		tmp = close + 2; /* Skip ">" after inner variable name */
+
+	/* resolve the typecast struct name */
+	ctx->offset = orig_offset + 1; /* for the '(' */
+	ret = parse_btf_casttype(arg + 1, ctx);
+	if (ret < 0)
+		return ret;
+
+	ctx->offset = orig_offset + tmp - arg;
+	ret = parse_btf_field(tmp, ctx->last_struct, pcode, end, ctx);
+	ctx->prefix_byteoffs = 0;
 	return ret;
 }
 
@@ -1118,6 +1387,24 @@ static int parse_probe_vars(char *orig_arg, const struct fetch_type *t,
 		return 0;
 	}
 
+	/* $current returns the address of the current task_struct. */
+	if (str_has_prefix(arg, "current")) {
+		/* $current is only supported by kernel probe. */
+		if (!(ctx->flags & TPARG_FL_KERNEL)) {
+			err = TP_ERR_BAD_VAR;
+			goto inval;
+		}
+		arg += strlen("current");
+		if (*arg == '-' && IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS))
+			return parse_btf_arg(orig_arg, pcode, end, ctx);
+
+		if (*arg != '\0')
+			goto inval;
+
+		code->op = FETCH_OP_CURRENT;
+		return 0;
+	}
+
 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
 	len = str_has_prefix(arg, "arg");
 	if (len) {
@@ -1230,7 +1517,7 @@ parse_probe_arg(char *arg, const struct fetch_type *type,
 			code->op = FETCH_OP_IMM;
 			code->immediate = param;
 		} else if (arg[1] == '+') {
-			/* kprobes don't support file offsets */
+			/* Kernel probes do not support file offsets */
 			if (ctx->flags & TPARG_FL_KERNEL) {
 				trace_probe_log_err(ctx->offset, FILE_ON_KPROBE);
 				return -EINVAL;
@@ -1294,43 +1581,16 @@ parse_probe_arg(char *arg, const struct fetch_type *type,
 		}
 		ctx->offset += (tmp + 1 - arg) + (arg[0] != '-' ? 1 : 0);
 		arg = tmp + 1;
-		tmp = strrchr(arg, ')');
-		if (!tmp) {
-			trace_probe_log_err(ctx->offset + strlen(arg),
-					    DEREF_OPEN_BRACE);
-			return -EINVAL;
-		} else {
-			const struct fetch_type *t2 = find_fetch_type(NULL, ctx->flags);
-			int cur_offs = ctx->offset;
-
-			*tmp = '\0';
-			ret = parse_probe_arg(arg, t2, &code, end, ctx);
-			if (ret)
-				break;
-			ctx->offset = cur_offs;
-			if (code->op == FETCH_OP_COMM ||
-			    code->op == FETCH_OP_DATA) {
-				trace_probe_log_err(ctx->offset, COMM_CANT_DEREF);
-				return -EINVAL;
-			}
-			if (++code == end) {
-				trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
-				return -EINVAL;
-			}
-			*pcode = code;
-
-			code->op = deref;
-			code->offset = offset;
-			/* Reset the last type if used */
-			ctx->last_type = NULL;
-		}
+		ret = handle_dereference(arg, pcode, end, ctx, deref, offset);
+		if (ret < 0)
+			return ret;
 		break;
 	case '\\':	/* Immediate value */
 		if (arg[1] == '"') {	/* Immediate string */
 			ret = __parse_imm_string(arg + 2, &tmp, ctx->offset + 2);
 			if (ret)
 				break;
-			code->op = FETCH_OP_DATA;
+			code->op = FETCH_OP_IMMSTR;
 			code->data = tmp;
 		} else {
 			ret = str_to_immediate(arg + 1, &code->immediate);
@@ -1344,15 +1604,28 @@ parse_probe_arg(char *arg, const struct fetch_type *type,
 		ret = handle_typecast(arg, pcode, end, ctx);
 		break;
 	default:
-		if (isalpha(arg[0]) || arg[0] == '_') {	/* BTF variable */
+		if (str_has_prefix(arg, THIS_CPU_PTR_PREFIX) ||
+		    str_has_prefix(arg, THIS_CPU_READ_PREFIX)) {
+			ret = parse_this_cpu(arg, pcode, end, ctx);
+		} else if (isalpha(arg[0]) || arg[0] == '_') {
+			/* BTF variable or event field*/
+			if (ctx->flags & TPARG_FL_TEVENT) {
+				ret = parse_trace_event(arg, *pcode, ctx);
+				if (ret < 0) {
+					trace_probe_log_err(ctx->offset,
+							    NO_EVENT_FIELD);
+					return -EINVAL;
+				}
+				break;
+			}
 			if (!tparg_is_function_entry(ctx->flags) &&
 			    !tparg_is_function_return(ctx->flags)) {
 				trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
 				return -EINVAL;
 			}
 			ret = parse_btf_arg(arg, pcode, end, ctx);
-			break;
 		}
+		break;
 	}
 	if (!ret && code->op == FETCH_OP_NOP) {
 		/* Parsed, but do not find fetch method */
@@ -1485,7 +1758,7 @@ static int finalize_fetch_insn(struct fetch_insn *code,
 		} else {
 			if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_UDEREF &&
 			    code->op != FETCH_OP_IMM && code->op != FETCH_OP_COMM &&
-			    code->op != FETCH_OP_DATA && code->op != FETCH_OP_TP_ARG) {
+			    code->op != FETCH_OP_IMMSTR && code->op != FETCH_OP_TP_ARG) {
 				trace_probe_log_err(ctx->offset + type_offset,
 						    BAD_STRING);
 				return -EINVAL;
@@ -1494,7 +1767,7 @@ static int finalize_fetch_insn(struct fetch_insn *code,
 
 		if (!strcmp(parg->type->name, "symstr") ||
 		    (code->op == FETCH_OP_IMM || code->op == FETCH_OP_COMM ||
-		     code->op == FETCH_OP_DATA) || code->op == FETCH_OP_TP_ARG ||
+		     code->op == FETCH_OP_IMMSTR) || code->op == FETCH_OP_TP_ARG ||
 		     parg->count) {
 			/*
 			 * IMM, DATA and COMM is pointing actual address, those
@@ -1620,6 +1893,11 @@ static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size,
 			      ctx);
 	if (ret < 0)
 		goto fail;
+	/* nested_level must be 0 here, otherwise there is a bug. */
+	if (WARN_ON_ONCE(ctx->nested_level)) {
+		ret = -EINVAL;
+		goto fail;
+	}
 
 	/* Update storing type if BTF is available */
 	if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
@@ -1670,7 +1948,7 @@ static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size,
 	if (ret < 0) {
 		for (code = tmp; code < tmp + FETCH_INSN_MAX; code++)
 			if (code->op == FETCH_NOP_SYMBOL ||
-			    code->op == FETCH_OP_DATA)
+			    code->op == FETCH_OP_IMMSTR)
 				kfree(code->data);
 	}
 	kfree(tmp);
@@ -1769,7 +2047,7 @@ void traceprobe_free_probe_arg(struct probe_arg *arg)
 
 	while (code && code->op != FETCH_OP_END) {
 		if (code->op == FETCH_NOP_SYMBOL ||
-		    code->op == FETCH_OP_DATA)
+		    code->op == FETCH_OP_IMMSTR)
 			kfree(code->data);
 		code++;
 	}
@@ -2396,3 +2674,99 @@ int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_a
 	}
 	return 0;
 }
+
+#ifdef CONFIG_PROBE_EVENTS_DUMP_FETCHARG
+
+struct fetch_op_decode {
+	const char *name;
+	void (*decode)(struct seq_file *m, struct fetch_insn *insn);
+};
+
+static const struct fetch_op_decode fetch_op_decode[];
+
+static void fetcharg_decode_none(struct seq_file *m, struct fetch_insn *insn)
+{
+	seq_puts(m, fetch_op_decode[insn->op].name);
+}
+
+static void fetcharg_decode_param(struct seq_file *m, struct fetch_insn *insn)
+{
+	seq_printf(m, "%s(%u)", fetch_op_decode[insn->op].name, insn->param);
+}
+
+static void fetcharg_decode_imm(struct seq_file *m, struct fetch_insn *insn)
+{
+	seq_printf(m, "%s(0x%lx)", fetch_op_decode[insn->op].name, insn->immediate);
+}
+
+static void fetcharg_decode_string(struct seq_file *m, struct fetch_insn *insn)
+{
+	seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, (char *)insn->data);
+}
+
+static void fetcharg_decode_symbol(struct seq_file *m, struct fetch_insn *insn)
+{
+	seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, (char *)insn->data);
+}
+
+static void fetcharg_decode_offset(struct seq_file *m, struct fetch_insn *insn)
+{
+	seq_printf(m, "%s(offset=%d)", fetch_op_decode[insn->op].name, insn->offset);
+}
+
+static void fetcharg_decode_store(struct seq_file *m, struct fetch_insn *insn)
+{
+	if (insn->op == FETCH_OP_ST_RAW)
+		seq_printf(m, "%s(size=%u)", fetch_op_decode[insn->op].name, insn->size);
+	else
+		seq_printf(m, "%s(offset=%d,size=%u)", fetch_op_decode[insn->op].name,
+			  insn->offset, insn->size);
+}
+
+static void fetcharg_decode_bf(struct seq_file *m, struct fetch_insn *insn)
+{
+	seq_printf(m, "%s(basesize=%u,lshift=%u,rshift=%u)",
+		   fetch_op_decode[insn->op].name, insn->basesize, insn->lshift, insn->rshift);
+}
+
+static void fetcharg_decode_tp_arg(struct seq_file *m, struct fetch_insn *insn)
+{
+	struct ftrace_event_field *field = insn->data;
+
+	seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, field->name);
+}
+
+#define FETCH_OP(opname, decode_fn) \
+	[FETCH_OP_##opname] = { .name = #opname, .decode = fetcharg_decode_##decode_fn }
+
+static const struct fetch_op_decode fetch_op_decode[] = FETCH_OP_LIST;
+#undef FETCH_OP
+
+static void trace_probe_dump_arg(struct seq_file *m, struct probe_arg *parg)
+{
+	int i;
+
+	seq_printf(m, "#  %s: ", parg->name);
+	for (i = 0; i < FETCH_INSN_MAX; i++) {
+		struct fetch_insn *insn = parg->code + i;
+
+		if (insn->op >= ARRAY_SIZE(fetch_op_decode) || !fetch_op_decode[insn->op].decode)
+			seq_printf(m, "unknown(%d)", insn->op);
+		else
+			fetch_op_decode[insn->op].decode(m, insn);
+
+		if (insn->op == FETCH_OP_END)
+			break;
+		seq_puts(m, " -> ");
+	}
+	seq_putc(m, '\n');
+}
+
+void trace_probe_dump_args(struct seq_file *m, struct trace_probe *tp)
+{
+	int i;
+
+	for (i = 0; i < tp->nr_args; i++)
+		trace_probe_dump_arg(m, &tp->args[i]);
+}
+#endif /* CONFIG_PROBE_EVENTS_DUMP_FETCHARG */
diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h
index 0f09f7aaf93fb..e6268a8dc378f 100644
--- a/kernel/trace/trace_probe.h
+++ b/kernel/trace/trace_probe.h
@@ -83,38 +83,48 @@ static nokprobe_inline u32 update_data_loc(u32 loc, int consumed)
 /* Printing function type */
 typedef int (*print_type_func_t)(struct trace_seq *, void *, void *);
 
-enum fetch_op {
-	FETCH_OP_NOP = 0,
-	// Stage 1 (load) ops
-	FETCH_OP_REG,		/* Register : .param = offset */
-	FETCH_OP_STACK,		/* Stack : .param = index */
-	FETCH_OP_STACKP,	/* Stack pointer */
-	FETCH_OP_RETVAL,	/* Return value */
-	FETCH_OP_IMM,		/* Immediate : .immediate */
-	FETCH_OP_COMM,		/* Current comm */
-	FETCH_OP_ARG,		/* Function argument : .param */
-	FETCH_OP_FOFFS,		/* File offset: .immediate */
-	FETCH_OP_DATA,		/* Allocated data: .data */
-	FETCH_OP_EDATA,		/* Entry data: .offset */
-	// Stage 2 (dereference) op
-	FETCH_OP_DEREF,		/* Dereference: .offset */
-	FETCH_OP_UDEREF,	/* User-space Dereference: .offset */
-	// Stage 3 (store) ops
-	FETCH_OP_ST_RAW,	/* Raw: .size */
-	FETCH_OP_ST_MEM,	/* Mem: .offset, .size */
-	FETCH_OP_ST_UMEM,	/* Mem: .offset, .size */
-	FETCH_OP_ST_STRING,	/* String: .offset, .size */
-	FETCH_OP_ST_USTRING,	/* User String: .offset, .size */
-	FETCH_OP_ST_SYMSTR,	/* Kernel Symbol String: .offset, .size */
-	FETCH_OP_ST_EDATA,	/* Store Entry Data: .offset */
-	// Stage 4 (modify) op
-	FETCH_OP_MOD_BF,	/* Bitfield: .basesize, .lshift, .rshift */
-	// Stage 5 (loop) op
-	FETCH_OP_LP_ARRAY,	/* Array: .param = loop count */
-	FETCH_OP_TP_ARG,	/* Trace Point argument */
-	FETCH_OP_END,
-	FETCH_NOP_SYMBOL,	/* Unresolved Symbol holder */
-};
+#define FETCH_OP_LIST	{						\
+	/* Stage 1 (load) ops */					\
+	FETCH_OP(NOP, none),		/* NOP */			\
+	FETCH_OP(REG, param),		/* Register: .param = offset */	\
+	FETCH_OP(STACK, param),		/* Stack: .param = index */	\
+	FETCH_OP(STACKP, none),		/* Stack pointer */		\
+	FETCH_OP(RETVAL, none),		/* Return value */		\
+	FETCH_OP(IMM, imm),		/* Immediate: .immediate */	\
+	FETCH_OP(COMM, none),		/* Current comm */		\
+	FETCH_OP(CURRENT, none),	/* Current task_struct address */\
+	FETCH_OP(ARG, param),		/* Argument: .param = index */	\
+	FETCH_OP(FOFFS, imm),		/* File offset: .immediate */	\
+	FETCH_OP(IMMSTR, string),	/* Allocated string: .data */	\
+	FETCH_OP(EDATA, offset),	/* Entry data: .offset */	\
+	FETCH_OP(TP_ARG, tp_arg),	/* Tracepoint argument: .data */\
+	/* Stage 2 (dereference) ops */					\
+	FETCH_OP(DEREF, offset),	/* Dereference: .offset */	\
+	FETCH_OP(UDEREF, offset),	/* User-space dereference: .offset */\
+	FETCH_OP(CPU_PTR, none),	/* Per-CPU pointer: .offset */	\
+	/* Stage 3 (store) ops */					\
+	FETCH_OP(ST_RAW, store),	/* Raw value: .size */		\
+	FETCH_OP(ST_MEM, store),	/* Memory: .offset, .size */	\
+	FETCH_OP(ST_UMEM, store),	/* User memory: .offset, .size */\
+	FETCH_OP(ST_STRING, store),	/* String: .offset, .size */	\
+	FETCH_OP(ST_USTRING, store),	/* User string: .offset, .size */\
+	FETCH_OP(ST_SYMSTR, store),	/* Symbol name: .offset, .size */\
+	FETCH_OP(ST_EDATA, offset),	/* Entry data: .offset */	\
+	/* Stage 4 (modify) op */					\
+	FETCH_OP(MOD_BF, bf),		/* Bitfield: .basesize, .lshift, .rshift*/\
+	/* Stage 5 (loop) op */						\
+	FETCH_OP(LP_ARRAY, param),	/* Loop array: .param = count */\
+	/* End */							\
+	FETCH_OP(END, none),						\
+	/* Unresolved Symbol holder */					\
+	FETCH_OP(NOP_SYMBOL, symbol),	/* Non loaded symbol: .data = symbol name */\
+}
+
+#define FETCH_OP(opname, decode_fn) FETCH_OP_##opname
+enum fetch_op FETCH_OP_LIST;
+#undef FETCH_OP
+
+#define FETCH_NOP_SYMBOL FETCH_OP_NOP_SYMBOL
 
 struct fetch_insn {
 	enum fetch_op op;
@@ -370,6 +380,13 @@ bool trace_probe_match_command_args(struct trace_probe *tp,
 int trace_probe_create(const char *raw_command, int (*createfn)(int, const char **));
 int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
 		 u8 *data, void *field);
+#ifdef CONFIG_PROBE_EVENTS_DUMP_FETCHARG
+void trace_probe_dump_args(struct seq_file *m, struct trace_probe *tp);
+#else
+static inline void trace_probe_dump_args(struct seq_file *m, struct trace_probe *tp)
+{
+}
+#endif
 
 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
 int traceprobe_get_entry_data_size(struct trace_probe *tp);
@@ -414,6 +431,11 @@ static inline bool tparg_is_function_return(unsigned int flags)
 	return (flags & TPARG_FL_LOC_MASK) == (TPARG_FL_KERNEL | TPARG_FL_RETURN);
 }
 
+static inline bool tparg_is_event_probe(unsigned int flags)
+{
+	return !!(flags & TPARG_FL_TEVENT);
+}
+
 struct traceprobe_parse_context {
 	struct trace_event_call *event;
 	/* BTF related parameters */
@@ -430,8 +452,13 @@ struct traceprobe_parse_context {
 	struct trace_probe *tp;
 	unsigned int flags;
 	int offset;
+	int nested_level;
+	int prefix_byteoffs;	/* The byte offset of the prefix field of typecast */
 };
 
+/* Each typecast consumes nested level. So the max number of typecast is 3. */
+#define TRACEPROBE_MAX_NESTED_LEVEL 3
+
 extern int traceprobe_parse_probe_arg(struct trace_probe *tp, int i,
 				      const char *argv,
 				      struct traceprobe_parse_context *ctx);
@@ -516,7 +543,7 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call,
 	C(BAD_MEM_ADDR,		"Invalid memory address"),		\
 	C(BAD_IMM,		"Invalid immediate value"),		\
 	C(IMMSTR_NO_CLOSE,	"String is not closed with '\"'"),	\
-	C(FILE_ON_KPROBE,	"File offset is not available with kprobe"), \
+	C(FILE_ON_KPROBE,	"File offset is not available for kernel probes"), \
 	C(BAD_FILE_OFFS,	"Invalid file offset value"),		\
 	C(SYM_ON_UPROBE,	"Symbol is not available with uprobe"),	\
 	C(TOO_MANY_OPS,		"Dereference is too much nested"), 	\
@@ -559,6 +586,7 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call,
 	C(NO_PTR_STRCT,		"This is not a pointer to union/structure."),	\
 	C(NOSUP_DAT_ARG,	"Non pointer structure/union argument is not supported."),\
 	C(BAD_HYPHEN,		"Failed to parse single hyphen. Forgot '>'?"),	\
+	C(NO_EVENT_FIELD,	"This event field is not found."),	\
 	C(NO_BTF_FIELD,		"This field is not found."),	\
 	C(BAD_BTF_TID,		"Failed to get BTF type info."),\
 	C(BAD_TYPE4STR,		"This type does not fit for string."),\
@@ -566,7 +594,13 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call,
 	C(TOO_MANY_ARGS,	"Too many arguments are specified"),	\
 	C(TOO_MANY_EARGS,	"Too many entry arguments specified"),	\
 	C(EVENT_TOO_BIG,	"Event too big (too many fields?)"),  \
-	C(TYPECAST_NOT_EVENT,	"Typecasts are only for eprobe fields"),
+	C(TYPECAST_NOT_EVENT,	"Typecasts are only for eprobe fields"), \
+	C(TYPECAST_REQ_FIELD,	"Typecast requires a field access"),	\
+	C(TOO_MANY_NESTED,	"Too many nested typecasts/dereferences"), \
+	C(TYPECAST_SYM_OFFSET,	"@SYM+/-OFFSET with typecast needs parentheses"), \
+	C(TYPECAST_NOT_ALIGNED,	"Typecast field option is not byte-aligned"), \
+	C(TYPECAST_BAD_ARROW,	"Typecast field option does not support -> operator"), \
+	C(NOSUP_PERCPU,		"Per-cpu variable access is only for kernel probes"),
 
 #undef C
 #define C(a, b)		TP_ERR_##a
diff --git a/kernel/trace/trace_probe_tmpl.h b/kernel/trace/trace_probe_tmpl.h
index f39b37fcdb3b5..8db12f758fdae 100644
--- a/kernel/trace/trace_probe_tmpl.h
+++ b/kernel/trace/trace_probe_tmpl.h
@@ -109,9 +109,12 @@ process_common_fetch_insn(struct fetch_insn *code, unsigned long *val)
 	case FETCH_OP_COMM:
 		*val = (unsigned long)current->comm;
 		break;
-	case FETCH_OP_DATA:
+	case FETCH_OP_IMMSTR:
 		*val = (unsigned long)code->data;
 		break;
+	case FETCH_OP_CURRENT:
+		*val = (unsigned long)current;
+		break;
 	default:
 		return -EILSEQ;
 	}
@@ -126,25 +129,35 @@ process_fetch_insn_bottom(struct fetch_insn *code, unsigned long val,
 	struct fetch_insn *s3 = NULL;
 	int total = 0, ret = 0, i = 0;
 	u32 loc = 0;
-	unsigned long lval = val;
+	unsigned long lval, llval = val;
 
 stage2:
 	/* 2nd stage: dereference memory if needed */
 	do {
-		if (code->op == FETCH_OP_DEREF) {
-			lval = val;
+		lval = val;
+		switch (code->op) {
+		case FETCH_OP_DEREF:
 			ret = probe_mem_read(&val, (void *)val + code->offset,
 					     sizeof(val));
-		} else if (code->op == FETCH_OP_UDEREF) {
-			lval = val;
+			break;
+		case FETCH_OP_UDEREF:
 			ret = probe_mem_read_user(&val,
 				 (void *)val + code->offset, sizeof(val));
-		} else
 			break;
+		case FETCH_OP_CPU_PTR:
+			val = (unsigned long)this_cpu_ptr((void __percpu *)val);
+			ret = 0;
+			break;
+		default:
+			lval = llval;
+			goto out;
+		}
 		if (ret)
 			return ret;
+		llval = lval;
 		code++;
 	} while (1);
+out:
 
 	s3 = code;
 stage3:
diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c
index c274346853d17..b2e264a4b96ce 100644
--- a/kernel/trace/trace_uprobe.c
+++ b/kernel/trace/trace_uprobe.c
@@ -765,6 +765,9 @@ static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev)
 		seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm);
 
 	seq_putc(m, '\n');
+
+	trace_probe_dump_args(m, &tu->tp);
+
 	return 0;
 }
 
diff --git a/kernel/umh.c b/kernel/umh.c
index 48117c569e1aa..6e2c7bb315c68 100644
--- a/kernel/umh.c
+++ b/kernel/umh.c
@@ -71,10 +71,8 @@ static int call_usermodehelper_exec_async(void *data)
 	spin_unlock_irq(&current->sighand->siglock);
 
 	/*
-	 * Initial kernel threads share ther FS with init, in order to
-	 * get the init root directory. But we've now created a new
-	 * thread that is going to execve a user process and has its own
-	 * 'struct fs_struct'. Reset umask to the default.
+	 * Usermodehelper threads get a copy of userspace init's
+	 * fs_struct. Reset umask to the default.
 	 */
 	current->fs->umask = 0022;
 
diff --git a/kernel/watchdog.c b/kernel/watchdog.c
index 87dd5e0f6968d..e5134ad7b6634 100644
--- a/kernel/watchdog.c
+++ b/kernel/watchdog.c
@@ -359,14 +359,14 @@ static void lockup_detector_update_enable(void)
 	if (watchdog_hardlockup_available && watchdog_hardlockup_user_enabled)
 		watchdog_enabled |= WATCHDOG_HARDLOCKUP_ENABLED;
 	if (watchdog_softlockup_user_enabled)
-		watchdog_enabled |= WATCHDOG_SOFTOCKUP_ENABLED;
+		watchdog_enabled |= WATCHDOG_SOFTLOCKUP_ENABLED;
 }
 
 #ifdef CONFIG_SOFTLOCKUP_DETECTOR
 
 /*
- * Delay the soflockup report when running a known slow code.
- * It does _not_ affect the timestamp of the last successdul reschedule.
+ * Delay the softlockup report when running a known slow code.
+ * It does _not_ affect the timestamp of the last successful reschedule.
  */
 #define SOFTLOCKUP_DELAY_REPORT	ULONG_MAX
 
@@ -742,7 +742,7 @@ static int is_softlockup(unsigned long touch_ts,
 			 unsigned long period_ts,
 			 unsigned long now)
 {
-	if ((watchdog_enabled & WATCHDOG_SOFTOCKUP_ENABLED) && watchdog_thresh) {
+	if ((watchdog_enabled & WATCHDOG_SOFTLOCKUP_ENABLED) && watchdog_thresh) {
 		/*
 		 * If period_ts has not been updated during a sample_period, then
 		 * in the subsequent few sample_periods, period_ts might also not
@@ -1098,11 +1098,11 @@ static void proc_watchdog_update(bool thresh_changed)
  * caller             | table->data points to            | 'which'
  * -------------------|----------------------------------|-------------------------------
  * proc_watchdog      | watchdog_user_enabled            | WATCHDOG_HARDLOCKUP_ENABLED |
- *                    |                                  | WATCHDOG_SOFTOCKUP_ENABLED
+ *                    |                                  | WATCHDOG_SOFTLOCKUP_ENABLED
  * -------------------|----------------------------------|-------------------------------
  * proc_nmi_watchdog  | watchdog_hardlockup_user_enabled | WATCHDOG_HARDLOCKUP_ENABLED
  * -------------------|----------------------------------|-------------------------------
- * proc_soft_watchdog | watchdog_softlockup_user_enabled | WATCHDOG_SOFTOCKUP_ENABLED
+ * proc_soft_watchdog | watchdog_softlockup_user_enabled | WATCHDOG_SOFTLOCKUP_ENABLED
  */
 static int proc_watchdog_common(int which, const struct ctl_table *table, int write,
 				void *buffer, size_t *lenp, loff_t *ppos)
@@ -1136,7 +1136,7 @@ static int proc_watchdog(const struct ctl_table *table, int write,
 			 void *buffer, size_t *lenp, loff_t *ppos)
 {
 	return proc_watchdog_common(WATCHDOG_HARDLOCKUP_ENABLED |
-				    WATCHDOG_SOFTOCKUP_ENABLED,
+				    WATCHDOG_SOFTLOCKUP_ENABLED,
 				    table, write, buffer, lenp, ppos);
 }
 
@@ -1159,7 +1159,7 @@ static int proc_nmi_watchdog(const struct ctl_table *table, int write,
 static int proc_soft_watchdog(const struct ctl_table *table, int write,
 			      void *buffer, size_t *lenp, loff_t *ppos)
 {
-	return proc_watchdog_common(WATCHDOG_SOFTOCKUP_ENABLED,
+	return proc_watchdog_common(WATCHDOG_SOFTLOCKUP_ENABLED,
 				    table, write, buffer, lenp, ppos);
 }
 #endif
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 78068ae8f28a6..26d5680c751c6 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -226,6 +226,8 @@ struct worker_pool {
 						/* L: hash of busy workers */
 
 	struct worker		*manager;	/* L: purely informational */
+	/* L: last worker woken by kick_pool() */
+	struct worker		*last_woken_worker;
 	struct list_head	workers;	/* A: attached workers */
 
 	struct ida		worker_ida;	/* worker IDs for task name */
@@ -1258,19 +1260,27 @@ static void kick_bh_pool(struct worker_pool *pool)
 }
 
 /**
- * kick_pool - wake up an idle worker if necessary
+ * kick_pool_pick - select an idle worker to kick, deferring the wakeup
  * @pool: pool to kick
+ * @wakep: out-param, set to the task to wake after pool->lock is dropped
  *
- * @pool may have pending work items. Wake up worker if necessary. Returns
- * whether a worker was woken up.
+ * Like kick_pool() but, for a regular (non-BH) pool, returns the picked
+ * worker's task via @wakep instead of waking it, so the caller can issue the
+ * wakeup after dropping pool->lock (the wakeup takes rq->lock). Worker
+ * selection, wake_cpu setup and the BH kick still happen under the lock.
+ * Returns whether a worker was selected or kicked.
+ *
+ * Must be called with @pool->lock held.
  */
-static bool kick_pool(struct worker_pool *pool)
+static bool kick_pool_pick(struct worker_pool *pool, struct task_struct **wakep)
 {
 	struct worker *worker = first_idle_worker(pool);
 	struct task_struct *p;
 
 	lockdep_assert_held(&pool->lock);
 
+	*wakep = NULL;
+
 	if (!need_more_worker(pool) || !worker)
 		return false;
 
@@ -1310,10 +1320,30 @@ static bool kick_pool(struct worker_pool *pool)
 		}
 	}
 #endif
-	wake_up_process(p);
+	/* Track the last idle worker woken, used for stall diagnostics. */
+	pool->last_woken_worker = worker;
+
+	*wakep = p;
 	return true;
 }
 
+/**
+ * kick_pool - wake up an idle worker if necessary
+ * @pool: pool to kick
+ *
+ * @pool may have pending work items. Wake up worker if necessary. Returns
+ * whether a worker was woken up.
+ */
+static bool kick_pool(struct worker_pool *pool)
+{
+	struct task_struct *p;
+	bool kicked = kick_pool_pick(pool, &p);
+
+	if (p)
+		wake_up_process(p);
+	return kicked;
+}
+
 #ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT
 
 /*
@@ -1438,7 +1468,7 @@ void wq_worker_running(struct task_struct *task)
 	 * CPU intensive auto-detection cares about how long a work item hogged
 	 * CPU without sleeping. Reset the starting timestamp on wakeup.
 	 */
-	worker->current_at = worker->task->se.sum_exec_runtime;
+	worker->current_at = READ_ONCE(worker->task->se.sum_exec_runtime);
 
 	WRITE_ONCE(worker->sleeping, 0);
 }
@@ -1505,7 +1535,11 @@ void wq_worker_tick(struct task_struct *task)
 	if (!pwq)
 		return;
 
-	pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC;
+	/*
+	 * @pwq is shared across CPUs for unbound wqs and this advisory stat is
+	 * bumped outside pool->lock, so the update is intentionally racy.
+	 */
+	data_race(pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC);
 
 	if (!wq_cpu_intensive_thresh_us)
 		return;
@@ -1523,7 +1557,7 @@ void wq_worker_tick(struct task_struct *task)
 	 * We probably want to make this prettier in the future.
 	 */
 	if ((worker->flags & WORKER_NOT_RUNNING) || READ_ONCE(worker->sleeping) ||
-	    worker->task->se.sum_exec_runtime - worker->current_at <
+	    READ_ONCE(worker->task->se.sum_exec_runtime) - worker->current_at <
 	    wq_cpu_intensive_thresh_us * NSEC_PER_USEC)
 		return;
 
@@ -2277,6 +2311,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq,
 {
 	struct pool_workqueue *pwq;
 	struct worker_pool *last_pool, *pool;
+	struct task_struct *wake_task = NULL;
 	unsigned int work_flags;
 	unsigned int req_cpu = cpu;
 
@@ -2399,7 +2434,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq,
 
 		trace_workqueue_activate_work(work);
 		insert_work(pwq, work, &pool->worklist, work_flags);
-		kick_pool(pool);
+		kick_pool_pick(pool, &wake_task);
 	} else {
 		work_flags |= WORK_STRUCT_INACTIVE;
 		insert_work(pwq, work, &pwq->inactive_works, work_flags);
@@ -2407,6 +2442,8 @@ static void __queue_work(int cpu, struct workqueue_struct *wq,
 
 out:
 	raw_spin_unlock(&pool->lock);
+	if (wake_task)
+		wake_up_process(wake_task);
 	rcu_read_unlock();
 }
 
@@ -2948,6 +2985,13 @@ static void set_worker_dying(struct worker *worker, struct list_head *list)
 	pool->nr_workers--;
 	pool->nr_idle--;
 
+	/*
+	 * Clear last_woken_worker if it points to this worker, so that
+	 * show_cpu_pool_busy_workers() cannot dereference a freed worker.
+	 */
+	if (pool->last_woken_worker == worker)
+		pool->last_woken_worker = NULL;
+
 	worker->flags |= WORKER_DIE;
 
 	list_move(&worker->entry, list);
@@ -3223,6 +3267,7 @@ __acquires(&pool->lock)
 {
 	struct pool_workqueue *pwq = get_work_pwq(work);
 	struct worker_pool *pool = worker->pool;
+	struct task_struct *wake_task = NULL;
 	unsigned long work_data;
 	int lockdep_start_depth, rcu_start_depth;
 	bool bh_draining = pool->flags & POOL_BH_DRAINING;
@@ -3249,7 +3294,7 @@ __acquires(&pool->lock)
 	worker->current_func = work->func;
 	worker->current_pwq = pwq;
 	if (worker->task)
-		worker->current_at = worker->task->se.sum_exec_runtime;
+		worker->current_at = READ_ONCE(worker->task->se.sum_exec_runtime);
 	worker->current_start = jiffies;
 	work_data = *work_data_bits(work);
 	worker->current_color = get_work_color(work_data);
@@ -3276,8 +3321,11 @@ __acquires(&pool->lock)
 	 * since nr_running would always be >= 1 at this point. This is used to
 	 * chain execution of the pending work items for WORKER_NOT_RUNNING
 	 * workers such as the UNBOUND and CPU_INTENSIVE ones.
+	 *
+	 * Select the worker under pool->lock; the wakeup is deferred until
+	 * after the lock is dropped, guarded by the rcu_read_lock() below.
 	 */
-	kick_pool(pool);
+	kick_pool_pick(pool, &wake_task);
 
 	/*
 	 * Record the last pool and clear PENDING which should be the last
@@ -3288,7 +3336,12 @@ __acquires(&pool->lock)
 	set_work_pool_and_clear_pending(work, pool->id, pool_offq_flags(pool));
 
 	pwq->stats[PWQ_STAT_STARTED]++;
+
+	rcu_read_lock();
 	raw_spin_unlock_irq(&pool->lock);
+	if (wake_task)
+		wake_up_process(wake_task);
+	rcu_read_unlock();
 
 	rcu_start_depth = rcu_preempt_depth();
 	lockdep_start_depth = lockdep_depth(current);
@@ -7689,20 +7742,58 @@ module_param_named(panic_on_stall_time, wq_panic_on_stall_time, uint, 0644);
 MODULE_PARM_DESC(panic_on_stall_time, "Panic if stall exceeds this many seconds (0=disabled)");
 
 /*
- * Show workers that might prevent the processing of pending work items.
- * A busy worker that is not running on the CPU (e.g. sleeping in
- * wait_event_idle() with PF_WQ_WORKER cleared) can stall the pool just as
- * effectively as a CPU-bound one, so dump every in-flight worker.
+ * Report that a pool has no worker in running state, which is a sign that the
+ * pool may be stuck. Print pool info. Must be called with pool->lock held and
+ * inside a printk_deferred_enter/exit region.
+ */
+static void show_pool_no_running_worker(struct worker_pool *pool)
+{
+	lockdep_assert_held(&pool->lock);
+
+	printk_deferred_enter();
+	pr_info("pool %d: no worker in running state, cpu=%d is %s (nr_workers=%d nr_idle=%d)\n",
+		pool->id, pool->cpu,
+		idle_cpu(pool->cpu) ? "idle" : "busy",
+		pool->nr_workers, pool->nr_idle);
+	pr_info("The pool might have trouble waking an idle worker.\n");
+	/*
+	 * last_woken_worker and its task are valid here: set_worker_dying()
+	 * clears it under pool->lock before setting WORKER_DIE, so if
+	 * last_woken_worker is non-NULL the kthread has not yet exited and
+	 * worker->task is still alive.
+	 */
+	if (pool->last_woken_worker) {
+		pr_info("Backtrace of last woken worker:\n");
+		sched_show_task(pool->last_woken_worker->task);
+	} else {
+		pr_info("Last woken worker empty\n");
+	}
+	printk_deferred_exit();
+}
+
+/*
+ * Show running workers that might prevent the processing of pending work items.
+ * If no running worker is found, the pool may be stuck waiting for an idle
+ * worker to be woken, so report the pool state and the last woken worker.
  */
 static void show_cpu_pool_busy_workers(struct worker_pool *pool)
 {
+	bool found_running = false;
 	struct worker *worker;
 	unsigned long irq_flags;
-	int bkt;
+	int cpu, bkt;
 
 	raw_spin_lock_irqsave(&pool->lock, irq_flags);
 
+	/* Snapshot cpu inside the lock to safely use it after unlock. */
+	cpu = pool->cpu;
+
 	hash_for_each(pool->busy_hash, bkt, worker, hentry) {
+		/* Skip workers that are not actively running on the CPU. */
+		if (!task_is_running(worker->task))
+			continue;
+
+		found_running = true;
 		/*
 		 * Defer printing to avoid deadlocks in console
 		 * drivers that queue work while holding locks
@@ -7716,7 +7807,24 @@ static void show_cpu_pool_busy_workers(struct worker_pool *pool)
 		printk_deferred_exit();
 	}
 
+	/*
+	 * If no running worker was found, the pool is likely stuck. Print pool
+	 * state and the backtrace of the last woken worker, which is the prime
+	 * suspect for the stall.
+	 */
+	if (!found_running)
+		show_pool_no_running_worker(pool);
+
 	raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
+
+	/*
+	 * Trigger a backtrace on the stalled CPU to capture what it is
+	 * currently executing. Skip an offline CPU, whose NMI is never acked
+	 * and would make the backtrace busy-wait until it times out. Done
+	 * after releasing the lock to avoid issues with NMI delivery.
+	 */
+	if (!found_running && cpu_online(cpu))
+		trigger_single_cpu_backtrace(cpu);
 }
 
 static void show_cpu_pools_busy_workers(void)
@@ -7923,12 +8031,12 @@ static inline void wq_watchdog_init(void) { }
 
 static void bh_pool_kick_normal(struct irq_work *irq_work)
 {
-	raise_softirq_irqoff(TASKLET_SOFTIRQ);
+	raise_softirq(TASKLET_SOFTIRQ);
 }
 
 static void bh_pool_kick_highpri(struct irq_work *irq_work)
 {
-	raise_softirq_irqoff(HI_SOFTIRQ);
+	raise_softirq(HI_SOFTIRQ);
 }
 
 static void __init restrict_unbound_cpumask(const char *name, const struct cpumask *mask)
diff --git a/lib/Kconfig b/lib/Kconfig
index 55748b68714e0..4e6b34c3346d5 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -159,12 +159,6 @@ config AUDIT_COMPAT_GENERIC
 	depends on AUDIT_GENERIC && AUDIT_ARCH_COMPAT_GENERIC && COMPAT
 	default y
 
-config RANDOM32_SELFTEST
-	bool "PRNG perform self test on init"
-	help
-	  This option enables the 32 bit PRNG library functions to perform a
-	  self test on initialization.
-
 #
 # compression support is select'ed if needed
 #
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 1244dcac2294a..bd961278dd458 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1048,34 +1048,6 @@ config CODE_TAGGING
 	bool
 	select KALLSYMS
 
-config MEM_ALLOC_PROFILING
-	bool "Enable memory allocation profiling"
-	default n
-	depends on MMU
-	depends on PROC_FS
-	depends on !DEBUG_FORCE_WEAK_PER_CPU
-	select CODE_TAGGING
-	select PAGE_EXTENSION
-	select SLAB_OBJ_EXT
-	help
-	  Track allocation source code and record total allocation size
-	  initiated at that code location. The mechanism can be used to track
-	  memory leaks with a low performance and memory impact.
-
-config MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
-	bool "Enable memory allocation profiling by default"
-	default y
-	depends on MEM_ALLOC_PROFILING
-
-config MEM_ALLOC_PROFILING_DEBUG
-	bool "Memory allocation profiler debugging"
-	default n
-	depends on MEM_ALLOC_PROFILING
-	select MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
-	help
-	  Adds warnings with helpful error messages for memory allocation
-	  profiling.
-
 source "lib/Kconfig.kasan"
 source "lib/Kconfig.kfence"
 source "lib/Kconfig.kmsan"
@@ -3504,6 +3476,23 @@ config GCD_KUNIT_TEST
 
 	  If unsure, say N
 
+config POLYNOMIAL_KUNIT_TEST
+	tristate "Polynomial calculation (polynomial_calc) test" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	select POLYNOMIAL
+	default KUNIT_ALL_TESTS
+	help
+	  This option enables the KUnit test suite for the polynomial_calc()
+	  function, which evaluates integer polynomials using factor
+	  redistribution to avoid overflow.
+
+	  The test suite verifies correctness for constant, linear, and
+	  quadratic polynomials, negative coefficients, per-step dividers,
+	  divider_leftover, total_divider scaling, and a real sensor
+	  N-to-temperature conversion polynomial.
+
+	  If unsure, say N
+
 config PRIME_NUMBERS_KUNIT_TEST
 	tristate "Prime number generator test" if !KUNIT_ALL_TESTS
 	depends on KUNIT
@@ -3531,6 +3520,15 @@ config GLOB_KUNIT_TEST
 
 	  If unsure, say N
 
+config PRANDOM_KUNIT_TEST
+	tristate "KUnit test for prandom" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	default KUNIT_ALL_TESTS
+	help
+	  Enable this option to test the prandom functions at runtime.
+
+	  If unsure, say N
+
 endif # RUNTIME_TESTING_MENU
 
 config ARCH_USE_MEMTEST
diff --git a/lib/Makefile b/lib/Makefile
index 7f75cc6edf94a..132147b3c5474 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -196,7 +196,6 @@ obj-$(CONFIG_OF_RECONFIG_NOTIFIER_ERROR_INJECT) += \
 obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
 
 obj-$(CONFIG_CODE_TAGGING) += codetag.o
-obj-$(CONFIG_MEM_ALLOC_PROFILING) += alloc_tag.o
 
 lib-$(CONFIG_GENERIC_BUG) += bug.o
 
@@ -273,6 +272,22 @@ filechk_defbconf = cat $(or $(real-prereqs), /dev/null)
 $(obj)/default.bconf: $(CONFIG_BOOT_CONFIG_EMBED_FILE) FORCE
 	$(call filechk,defbconf)
 
+obj-$(CONFIG_CMDLINE_FROM_BOOTCONFIG) += embedded-cmdline.o
+$(obj)/embedded-cmdline.o: $(obj)/embedded_cmdline.bin
+
+# Render the bootconfig "kernel" subtree to a flat cmdline string using
+# the userspace tools/bootconfig parser (-C mode). The runtime prepend
+# helper enforces COMMAND_LINE_SIZE at boot, so no build-time size
+# check is performed here (COMMAND_LINE_SIZE is an arch header
+# constant, not a Kconfig value).
+quiet_cmd_render_cmdline = BCONF2C $@
+      cmd_render_cmdline = \
+	$(objtree)/tools/bootconfig/bootconfig -C $< > $@
+
+targets += embedded_cmdline.bin
+$(obj)/embedded_cmdline.bin: $(obj)/default.bconf $(objtree)/tools/bootconfig/bootconfig FORCE
+	$(call if_changed,render_cmdline)
+
 obj-$(CONFIG_RBTREE_TEST) += rbtree_test.o
 obj-$(CONFIG_INTERVAL_TREE_TEST) += interval_tree_test.o
 
diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c
deleted file mode 100644
index d9be1cf5187d9..0000000000000
--- a/lib/alloc_tag.c
+++ /dev/null
@@ -1,1020 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-#include <linux/alloc_tag.h>
-#include <linux/execmem.h>
-#include <linux/fs.h>
-#include <linux/gfp.h>
-#include <linux/kallsyms.h>
-#include <linux/module.h>
-#include <linux/page_ext.h>
-#include <linux/pgalloc_tag.h>
-#include <linux/proc_fs.h>
-#include <linux/rcupdate.h>
-#include <linux/seq_buf.h>
-#include <linux/seq_file.h>
-#include <linux/string_choices.h>
-#include <linux/vmalloc.h>
-#include <linux/kmemleak.h>
-
-#define ALLOCINFO_FILE_NAME		"allocinfo"
-#define MODULE_ALLOC_TAG_VMAP_SIZE	(100000UL * sizeof(struct alloc_tag))
-#define SECTION_START(NAME)		(CODETAG_SECTION_START_PREFIX NAME)
-#define SECTION_STOP(NAME)		(CODETAG_SECTION_STOP_PREFIX NAME)
-
-#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
-static bool mem_profiling_support = true;
-#else
-static bool mem_profiling_support;
-#endif
-
-static struct codetag_type *alloc_tag_cttype;
-
-#ifdef CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU
-DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
-EXPORT_SYMBOL(_shared_alloc_tag);
-#endif
-
-DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
-			mem_alloc_profiling_key);
-EXPORT_SYMBOL(mem_alloc_profiling_key);
-
-DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed);
-
-struct alloc_tag_kernel_section kernel_tags = { NULL, 0 };
-unsigned long alloc_tag_ref_mask;
-int alloc_tag_ref_offs;
-
-struct allocinfo_private {
-	struct codetag_iterator iter;
-	struct codetag_iterator reported_iter;
-	bool print_header;
-};
-
-static void *allocinfo_start(struct seq_file *m, loff_t *pos)
-{
-	struct allocinfo_private *priv;
-	loff_t node = *pos;
-
-	priv = (struct allocinfo_private *)m->private;
-	codetag_lock_module_list(alloc_tag_cttype);
-	if (node == 0) {
-		priv->print_header = true;
-		priv->iter = codetag_get_ct_iter(alloc_tag_cttype);
-	} else {
-		priv->iter = priv->reported_iter;
-	}
-	codetag_next_ct(&priv->iter);
-	return priv->iter.ct ? priv : NULL;
-}
-
-static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos)
-{
-	struct allocinfo_private *priv = (struct allocinfo_private *)arg;
-	struct codetag *ct;
-
-	priv->reported_iter = priv->iter;
-	ct = codetag_next_ct(&priv->iter);
-	(*pos)++;
-	if (!ct)
-		return NULL;
-
-	return priv;
-}
-
-static void allocinfo_stop(struct seq_file *m, void *arg)
-{
-	codetag_unlock_module_list(alloc_tag_cttype);
-}
-
-static void print_allocinfo_header(struct seq_buf *buf)
-{
-	/* Output format version, so we can change it. */
-	seq_buf_printf(buf, "allocinfo - version: 2.0\n");
-	seq_buf_printf(buf, "#     <size>  <calls> <tag info>\n");
-}
-
-static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct)
-{
-	struct alloc_tag *tag = ct_to_alloc_tag(ct);
-	struct alloc_tag_counters counter = alloc_tag_read(tag);
-	s64 bytes = counter.bytes;
-
-	seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls);
-	codetag_to_text(out, ct);
-	if (unlikely(alloc_tag_is_inaccurate(tag)))
-		seq_buf_printf(out, " accurate:no");
-	seq_buf_putc(out, ' ');
-	seq_buf_putc(out, '\n');
-}
-
-static int allocinfo_show(struct seq_file *m, void *arg)
-{
-	struct allocinfo_private *priv = (struct allocinfo_private *)arg;
-	char *bufp;
-	size_t n = seq_get_buf(m, &bufp);
-	struct seq_buf buf;
-
-	seq_buf_init(&buf, bufp, n);
-	if (priv->print_header) {
-		print_allocinfo_header(&buf);
-		priv->print_header = false;
-	}
-	alloc_tag_to_text(&buf, priv->iter.ct);
-	seq_commit(m, seq_buf_used(&buf));
-	return 0;
-}
-
-static const struct seq_operations allocinfo_seq_op = {
-	.start	= allocinfo_start,
-	.next	= allocinfo_next,
-	.stop	= allocinfo_stop,
-	.show	= allocinfo_show,
-};
-
-size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)
-{
-	struct codetag_iterator iter;
-	struct codetag *ct;
-	struct codetag_bytes n;
-	unsigned int i, nr = 0;
-
-	if (IS_ERR_OR_NULL(alloc_tag_cttype))
-		return 0;
-
-	if (can_sleep)
-		codetag_lock_module_list(alloc_tag_cttype);
-	else if (!codetag_trylock_module_list(alloc_tag_cttype))
-		return 0;
-
-	iter = codetag_get_ct_iter(alloc_tag_cttype);
-	while ((ct = codetag_next_ct(&iter))) {
-		struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct));
-
-		n.ct	= ct;
-		n.bytes = counter.bytes;
-
-		for (i = 0; i < nr; i++)
-			if (n.bytes > tags[i].bytes)
-				break;
-
-		if (i < count) {
-			nr -= nr == count;
-			memmove(&tags[i + 1],
-				&tags[i],
-				sizeof(tags[0]) * (nr - i));
-			nr++;
-			tags[i] = n;
-		}
-	}
-
-	codetag_unlock_module_list(alloc_tag_cttype);
-
-	return nr;
-}
-
-void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
-{
-	int i;
-	struct alloc_tag *tag;
-	unsigned int nr_pages = 1 << new_order;
-
-	if (!mem_alloc_profiling_enabled())
-		return;
-
-	tag = __pgalloc_tag_get(&folio->page);
-	if (!tag)
-		return;
-
-	for (i = nr_pages; i < (1 << old_order); i += nr_pages) {
-		union pgtag_ref_handle handle;
-		union codetag_ref ref;
-
-		if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) {
-			/* Set new reference to point to the original tag */
-			alloc_tag_ref_set(&ref, tag);
-			update_page_tag_ref(handle, &ref);
-			put_page_tag_ref(handle);
-		}
-	}
-}
-
-void pgalloc_tag_swap(struct folio *new, struct folio *old)
-{
-	union pgtag_ref_handle handle_old, handle_new;
-	union codetag_ref ref_old, ref_new;
-	struct alloc_tag *tag_old, *tag_new;
-
-	if (!mem_alloc_profiling_enabled())
-		return;
-
-	tag_old = __pgalloc_tag_get(&old->page);
-	if (!tag_old)
-		return;
-	tag_new = __pgalloc_tag_get(&new->page);
-	if (!tag_new)
-		return;
-
-	if (!get_page_tag_ref(&old->page, &ref_old, &handle_old))
-		return;
-	if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) {
-		put_page_tag_ref(handle_old);
-		return;
-	}
-
-	/*
-	 * Clear tag references to avoid debug warning when using
-	 * __alloc_tag_ref_set() with non-empty reference.
-	 */
-	set_codetag_empty(&ref_old);
-	set_codetag_empty(&ref_new);
-
-	/* swap tags */
-	__alloc_tag_ref_set(&ref_old, tag_new);
-	update_page_tag_ref(handle_old, &ref_old);
-	__alloc_tag_ref_set(&ref_new, tag_old);
-	update_page_tag_ref(handle_new, &ref_new);
-
-	put_page_tag_ref(handle_old);
-	put_page_tag_ref(handle_new);
-}
-
-static void shutdown_mem_profiling(bool remove_file)
-{
-	if (mem_alloc_profiling_enabled())
-		static_branch_disable(&mem_alloc_profiling_key);
-
-	if (!mem_profiling_support)
-		return;
-
-	if (remove_file)
-		remove_proc_entry(ALLOCINFO_FILE_NAME, NULL);
-	mem_profiling_support = false;
-}
-
-void __init alloc_tag_sec_init(void)
-{
-	struct alloc_tag *last_codetag;
-
-	if (!mem_profiling_support)
-		return;
-
-	if (!static_key_enabled(&mem_profiling_compressed))
-		return;
-
-	kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name(
-					SECTION_START(ALLOC_TAG_SECTION_NAME));
-	last_codetag = (struct alloc_tag *)kallsyms_lookup_name(
-					SECTION_STOP(ALLOC_TAG_SECTION_NAME));
-	kernel_tags.count = last_codetag - kernel_tags.first_tag;
-
-	/* Check if kernel tags fit into page flags */
-	if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) {
-		shutdown_mem_profiling(false); /* allocinfo file does not exist yet */
-		pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n",
-			kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS);
-		return;
-	}
-
-	alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS);
-	alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1);
-	pr_debug("Memory allocation profiling compression is using %d page flag bits!\n",
-		 NR_UNUSED_PAGEFLAG_BITS);
-}
-
-#ifdef CONFIG_MODULES
-
-static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE);
-static struct vm_struct *vm_module_tags;
-/* A dummy object used to indicate an unloaded module */
-static struct module unloaded_mod;
-/* A dummy object used to indicate a module prepended area */
-static struct module prepend_mod;
-
-struct alloc_tag_module_section module_tags;
-
-static inline unsigned long alloc_tag_align(unsigned long val)
-{
-	if (!static_key_enabled(&mem_profiling_compressed)) {
-		/* No alignment requirements when we are not indexing the tags */
-		return val;
-	}
-
-	if (val % sizeof(struct alloc_tag) == 0)
-		return val;
-	return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag);
-}
-
-static bool ensure_alignment(unsigned long align, unsigned int *prepend)
-{
-	if (!static_key_enabled(&mem_profiling_compressed)) {
-		/* No alignment requirements when we are not indexing the tags */
-		return true;
-	}
-
-	/*
-	 * If alloc_tag size is not a multiple of required alignment, tag
-	 * indexing does not work.
-	 */
-	if (!IS_ALIGNED(sizeof(struct alloc_tag), align))
-		return false;
-
-	/* Ensure prepend consumes multiple of alloc_tag-sized blocks */
-	if (*prepend)
-		*prepend = alloc_tag_align(*prepend);
-
-	return true;
-}
-
-static inline bool tags_addressable(void)
-{
-	unsigned long tag_idx_count;
-
-	if (!static_key_enabled(&mem_profiling_compressed))
-		return true; /* with page_ext tags are always addressable */
-
-	tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count +
-			module_tags.size / sizeof(struct alloc_tag);
-
-	return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS);
-}
-
-static bool needs_section_mem(struct module *mod, unsigned long size)
-{
-	if (!mem_profiling_support)
-		return false;
-
-	return size >= sizeof(struct alloc_tag);
-}
-
-static bool clean_unused_counters(struct alloc_tag *start_tag,
-				  struct alloc_tag *end_tag)
-{
-	struct alloc_tag *tag;
-	bool ret = true;
-
-	for (tag = start_tag; tag <= end_tag; tag++) {
-		struct alloc_tag_counters counter;
-
-		if (!tag->counters)
-			continue;
-
-		counter = alloc_tag_read(tag);
-		if (!counter.bytes) {
-			free_percpu(tag->counters);
-			tag->counters = NULL;
-		} else {
-			ret = false;
-		}
-	}
-
-	return ret;
-}
-
-/* Called with mod_area_mt locked */
-static void clean_unused_module_areas_locked(void)
-{
-	MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
-	struct module *val;
-
-	mas_for_each(&mas, val, module_tags.size) {
-		struct alloc_tag *start_tag;
-		struct alloc_tag *end_tag;
-
-		if (val != &unloaded_mod)
-			continue;
-
-		/* Release area if all tags are unused */
-		start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
-		end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
-		if (clean_unused_counters(start_tag, end_tag))
-			mas_erase(&mas);
-	}
-}
-
-/* Called with mod_area_mt locked */
-static bool find_aligned_area(struct ma_state *mas, unsigned long section_size,
-			      unsigned long size, unsigned int prepend, unsigned long align)
-{
-	bool cleanup_done = false;
-
-repeat:
-	/* Try finding exact size and hope the start is aligned */
-	if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) {
-		if (IS_ALIGNED(mas->index + prepend, align))
-			return true;
-
-		/* Try finding larger area to align later */
-		mas_reset(mas);
-		if (!mas_empty_area(mas, 0, section_size - 1,
-				    size + prepend + align - 1))
-			return true;
-	}
-
-	/* No free area, try cleanup stale data and repeat the search once */
-	if (!cleanup_done) {
-		clean_unused_module_areas_locked();
-		cleanup_done = true;
-		mas_reset(mas);
-		goto repeat;
-	}
-
-	return false;
-}
-
-static int vm_module_tags_populate(void)
-{
-	unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) +
-				 (vm_module_tags->nr_pages << PAGE_SHIFT);
-	unsigned long new_end = module_tags.start_addr + module_tags.size;
-
-	if (phys_end < new_end) {
-		struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages;
-		unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN);
-		unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN);
-		unsigned long more_pages;
-		unsigned long nr = 0;
-
-		more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT;
-		while (nr < more_pages) {
-			unsigned long allocated;
-
-			allocated = alloc_pages_bulk_node(GFP_KERNEL | __GFP_NOWARN,
-				NUMA_NO_NODE, more_pages - nr, next_page + nr);
-
-			if (!allocated)
-				break;
-			nr += allocated;
-		}
-
-		if (nr < more_pages ||
-		    vmap_pages_range(phys_end, phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL,
-				     next_page, PAGE_SHIFT) < 0) {
-			release_pages_arg arg = { .pages = next_page };
-
-			/* Clean up and error out */
-			release_pages(arg, nr);
-			return -ENOMEM;
-		}
-
-		vm_module_tags->nr_pages += nr;
-
-		/*
-		 * Kasan allocates 1 byte of shadow for every 8 bytes of data.
-		 * When kasan_alloc_module_shadow allocates shadow memory,
-		 * its unit of allocation is a page.
-		 * Therefore, here we need to align to MODULE_ALIGN.
-		 */
-		if (old_shadow_end < new_shadow_end)
-			kasan_alloc_module_shadow((void *)old_shadow_end,
-						  new_shadow_end - old_shadow_end,
-						  GFP_KERNEL);
-	}
-
-	/*
-	 * Mark the pages as accessible, now that they are mapped.
-	 * With hardware tag-based KASAN, marking is skipped for
-	 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
-	 */
-	kasan_unpoison_vmalloc((void *)module_tags.start_addr,
-				new_end - module_tags.start_addr,
-				KASAN_VMALLOC_PROT_NORMAL);
-
-	return 0;
-}
-
-static void *reserve_module_tags(struct module *mod, unsigned long size,
-				 unsigned int prepend, unsigned long align)
-{
-	unsigned long section_size = module_tags.end_addr - module_tags.start_addr;
-	MA_STATE(mas, &mod_area_mt, 0, section_size - 1);
-	unsigned long offset;
-	void *ret = NULL;
-
-	/* If no tags return error */
-	if (size < sizeof(struct alloc_tag))
-		return ERR_PTR(-EINVAL);
-
-	/*
-	 * align is always power of 2, so we can use IS_ALIGNED and ALIGN.
-	 * align 0 or 1 means no alignment, to simplify set to 1.
-	 */
-	if (!align)
-		align = 1;
-
-	if (!ensure_alignment(align, &prepend)) {
-		shutdown_mem_profiling(true);
-		pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n",
-			mod->name, align);
-		return ERR_PTR(-EINVAL);
-	}
-
-	mas_lock(&mas);
-	if (!find_aligned_area(&mas, section_size, size, prepend, align)) {
-		ret = ERR_PTR(-ENOMEM);
-		goto unlock;
-	}
-
-	/* Mark found area as reserved */
-	offset = mas.index;
-	offset += prepend;
-	offset = ALIGN(offset, align);
-	if (offset != mas.index) {
-		unsigned long pad_start = mas.index;
-
-		mas.last = offset - 1;
-		mas_store(&mas, &prepend_mod);
-		if (mas_is_err(&mas)) {
-			ret = ERR_PTR(xa_err(mas.node));
-			goto unlock;
-		}
-		mas.index = offset;
-		mas.last = offset + size - 1;
-		mas_store(&mas, mod);
-		if (mas_is_err(&mas)) {
-			mas.index = pad_start;
-			mas_erase(&mas);
-			ret = ERR_PTR(xa_err(mas.node));
-		}
-	} else {
-		mas.last = offset + size - 1;
-		mas_store(&mas, mod);
-		if (mas_is_err(&mas))
-			ret = ERR_PTR(xa_err(mas.node));
-	}
-unlock:
-	mas_unlock(&mas);
-
-	if (IS_ERR(ret))
-		return ret;
-
-	if (module_tags.size < offset + size) {
-		int grow_res;
-
-		module_tags.size = offset + size;
-		if (mem_alloc_profiling_enabled() && !tags_addressable()) {
-			shutdown_mem_profiling(true);
-			pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n",
-				mod->name, NR_UNUSED_PAGEFLAG_BITS);
-		}
-
-		grow_res = vm_module_tags_populate();
-		if (grow_res) {
-			shutdown_mem_profiling(true);
-			pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n",
-			       mod->name);
-			return ERR_PTR(grow_res);
-		}
-	}
-
-	return (struct alloc_tag *)(module_tags.start_addr + offset);
-}
-
-static void release_module_tags(struct module *mod, bool used)
-{
-	MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size);
-	struct alloc_tag *start_tag;
-	struct alloc_tag *end_tag;
-	struct module *val;
-
-	mas_lock(&mas);
-	mas_for_each_rev(&mas, val, 0)
-		if (val == mod)
-			break;
-
-	if (!val) /* module not found */
-		goto out;
-
-	if (!used)
-		goto release_area;
-
-	start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
-	end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
-	if (!clean_unused_counters(start_tag, end_tag)) {
-		struct alloc_tag *tag;
-
-		for (tag = start_tag; tag <= end_tag; tag++) {
-			struct alloc_tag_counters counter;
-
-			if (!tag->counters)
-				continue;
-
-			counter = alloc_tag_read(tag);
-			pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n",
-				tag->ct.filename, tag->ct.lineno, tag->ct.modname,
-				tag->ct.function, counter.bytes);
-		}
-	} else {
-		used = false;
-	}
-release_area:
-	mas_store(&mas, used ? &unloaded_mod : NULL);
-	val = mas_prev_range(&mas, 0);
-	if (val == &prepend_mod)
-		mas_store(&mas, NULL);
-out:
-	mas_unlock(&mas);
-}
-
-static int load_module(struct module *mod, struct codetag *start, struct codetag *stop)
-{
-	/* Allocate module alloc_tag percpu counters */
-	struct alloc_tag *start_tag;
-	struct alloc_tag *stop_tag;
-	struct alloc_tag *tag;
-
-	/* percpu counters for core allocations are already statically allocated */
-	if (!mod)
-		return 0;
-
-	start_tag = ct_to_alloc_tag(start);
-	stop_tag = ct_to_alloc_tag(stop);
-	for (tag = start_tag; tag < stop_tag; tag++) {
-		WARN_ON(tag->counters);
-		tag->counters = alloc_percpu(struct alloc_tag_counters);
-		if (!tag->counters) {
-			while (--tag >= start_tag) {
-				free_percpu(tag->counters);
-				tag->counters = NULL;
-			}
-			pr_err("Failed to allocate memory for allocation tag percpu counters in the module %s\n",
-			       mod->name);
-			return -ENOMEM;
-		}
-
-		/*
-		 * Avoid a kmemleak false positive. The pointer to the counters is stored
-		 * in the alloc_tag section of the module and cannot be directly accessed.
-		 */
-		kmemleak_ignore_percpu(tag->counters);
-	}
-	return 0;
-}
-
-static void replace_module(struct module *mod, struct module *new_mod)
-{
-	MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
-	struct module *val;
-
-	mas_lock(&mas);
-	mas_for_each(&mas, val, module_tags.size) {
-		if (val != mod)
-			continue;
-
-		mas_store_gfp(&mas, new_mod, GFP_KERNEL);
-		break;
-	}
-	mas_unlock(&mas);
-}
-
-static int __init alloc_mod_tags_mem(void)
-{
-	/* Map space to copy allocation tags */
-	vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE);
-	if (!vm_module_tags) {
-		pr_err("Failed to map %lu bytes for module allocation tags\n",
-			MODULE_ALLOC_TAG_VMAP_SIZE);
-		module_tags.start_addr = 0;
-		return -ENOMEM;
-	}
-
-	vm_module_tags->pages = kmalloc_objs(struct page *,
-					     get_vm_area_size(vm_module_tags) >> PAGE_SHIFT,
-					     GFP_KERNEL | __GFP_ZERO);
-	if (!vm_module_tags->pages) {
-		free_vm_area(vm_module_tags);
-		return -ENOMEM;
-	}
-
-	module_tags.start_addr = (unsigned long)vm_module_tags->addr;
-	module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE;
-	/* Ensure the base is alloc_tag aligned when required for indexing */
-	module_tags.start_addr = alloc_tag_align(module_tags.start_addr);
-
-	return 0;
-}
-
-static void __init free_mod_tags_mem(void)
-{
-	release_pages_arg arg = { .pages = vm_module_tags->pages };
-
-	module_tags.start_addr = 0;
-	release_pages(arg, vm_module_tags->nr_pages);
-	kfree(vm_module_tags->pages);
-	free_vm_area(vm_module_tags);
-}
-
-#else /* CONFIG_MODULES */
-
-static inline int alloc_mod_tags_mem(void) { return 0; }
-static inline void free_mod_tags_mem(void) {}
-
-#endif /* CONFIG_MODULES */
-
-/* See: Documentation/mm/allocation-profiling.rst */
-static int __init setup_early_mem_profiling(char *str)
-{
-	bool compressed = false;
-	bool enable;
-
-	if (!str || !str[0])
-		return -EINVAL;
-
-	if (!strncmp(str, "never", 5)) {
-		enable = false;
-		mem_profiling_support = false;
-		pr_info("Memory allocation profiling is disabled!\n");
-	} else {
-		char *token = strsep(&str, ",");
-
-		if (kstrtobool(token, &enable))
-			return -EINVAL;
-
-		if (str) {
-
-			if (strcmp(str, "compressed"))
-				return -EINVAL;
-
-			compressed = true;
-		}
-		mem_profiling_support = true;
-		pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n",
-			compressed ? "with" : "without", str_on_off(enable));
-	}
-
-	if (enable != mem_alloc_profiling_enabled()) {
-		if (enable)
-			static_branch_enable(&mem_alloc_profiling_key);
-		else
-			static_branch_disable(&mem_alloc_profiling_key);
-	}
-	if (compressed != static_key_enabled(&mem_profiling_compressed)) {
-		if (compressed)
-			static_branch_enable(&mem_profiling_compressed);
-		else
-			static_branch_disable(&mem_profiling_compressed);
-	}
-
-	return 0;
-}
-early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling);
-
-static __init bool need_page_alloc_tagging(void)
-{
-	if (static_key_enabled(&mem_profiling_compressed))
-		return false;
-
-	return mem_profiling_support;
-}
-
-#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
-/*
- * Track page allocations before page_ext is initialized.
- * Some pages are allocated before page_ext becomes available, leaving
- * their codetag uninitialized. Track these early PFNs so we can clear
- * their codetag refs later to avoid warnings when they are freed.
- *
- * Each page is cast to a pfn_pool: the first few bytes hold metadata
- * (next pointer and slot count), the remainder stores PFNs.
- */
-struct pfn_pool {
-	struct pfn_pool *next;
-	atomic_t count;
-	unsigned long pfns[];
-};
-
-#define PFN_POOL_SIZE			((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \
-					 sizeof(unsigned long))
-
-/*
- * Skip early PFN recording for a page allocation.  Reuses the
- * %__GFP_NO_OBJ_EXT bit.  Used by __alloc_tag_add_early_pfn() to avoid
- * recursion when allocating pages for the early PFN tracking list
- * itself.
- *
- * Codetags of the pages allocated with __GFP_NO_CODETAG should be
- * cleared (via clear_page_tag_ref()) before freeing the pages to prevent
- * alloc_tag_sub_check() from triggering a warning.
- */
-#define __GFP_NO_CODETAG		__GFP_NO_OBJ_EXT
-
-static struct pfn_pool *current_pfn_pool __initdata;
-
-static void __init __alloc_tag_add_early_pfn(unsigned long pfn)
-{
-	struct pfn_pool *pool;
-	int idx;
-
-	do {
-		pool = READ_ONCE(current_pfn_pool);
-		if (!pool || atomic_read(&pool->count) >= PFN_POOL_SIZE) {
-			struct page *new_page = alloc_page(__GFP_HIGH | __GFP_NO_CODETAG);
-			struct pfn_pool *new;
-
-			if (!new_page) {
-				pr_warn_once("early PFN tracking page allocation failed\n");
-				return;
-			}
-			new = page_address(new_page);
-			new->next = pool;
-			atomic_set(&new->count, 0);
-			if (cmpxchg(&current_pfn_pool, pool, new) != pool) {
-				clear_page_tag_ref(new_page);
-				__free_page(new_page);
-				continue;
-			}
-			pool = new;
-		}
-		idx = atomic_read(&pool->count);
-		if (idx >= PFN_POOL_SIZE)
-			continue;
-		if (atomic_cmpxchg(&pool->count, idx, idx + 1) == idx)
-			break;
-	} while (1);
-
-	pool->pfns[idx] = pfn;
-}
-
-typedef void alloc_tag_add_func(unsigned long pfn);
-static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata =
-	RCU_INITIALIZER(__alloc_tag_add_early_pfn);
-
-void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags)
-{
-	alloc_tag_add_func *alloc_tag_add;
-
-	if (static_key_enabled(&mem_profiling_compressed))
-		return;
-
-	/* Skip allocations for the tracking list itself to avoid recursion. */
-	if (gfp_flags & __GFP_NO_CODETAG)
-		return;
-
-	rcu_read_lock();
-	alloc_tag_add = rcu_dereference(alloc_tag_add_early_pfn_ptr);
-	if (alloc_tag_add)
-		alloc_tag_add(pfn);
-	rcu_read_unlock();
-}
-
-static void __init clear_early_alloc_pfn_tag_refs(void)
-{
-	struct pfn_pool *pool, *next;
-	struct page *page;
-	int i;
-
-	if (static_key_enabled(&mem_profiling_compressed))
-		return;
-
-	rcu_assign_pointer(alloc_tag_add_early_pfn_ptr, NULL);
-	/* Make sure we are not racing with __alloc_tag_add_early_pfn() */
-	synchronize_rcu();
-
-	for (pool = current_pfn_pool; pool; pool = next) {
-		int nr_pfns = atomic_read(&pool->count);
-
-		for (i = 0; i < nr_pfns; i++) {
-			unsigned long pfn = pool->pfns[i];
-
-			if (pfn_valid(pfn)) {
-				union pgtag_ref_handle handle;
-				union codetag_ref ref;
-
-				if (get_page_tag_ref(pfn_to_page(pfn), &ref, &handle)) {
-					/*
-					 * An early-allocated page could be freed and reallocated
-					 * after its page_ext is initialized but before we clear it.
-					 * In that case, it already has a valid tag set.
-					 * We should not overwrite that valid tag
-					 * with CODETAG_EMPTY.
-					 *
-					 * Note: there is still a small race window between checking
-					 * ref.ct and calling set_codetag_empty(). We accept this
-					 * race as it's unlikely and the extra complexity of atomic
-					 * cmpxchg is not worth it for this debug-only code path.
-					 */
-					if (ref.ct) {
-						put_page_tag_ref(handle);
-						continue;
-					}
-
-					set_codetag_empty(&ref);
-					update_page_tag_ref(handle, &ref);
-					put_page_tag_ref(handle);
-				}
-			}
-		}
-
-		next = pool->next;
-		page = virt_to_page(pool);
-		clear_page_tag_ref(page);
-		__free_page(page);
-	}
-}
-#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */
-static inline void __init clear_early_alloc_pfn_tag_refs(void) {}
-#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
-
-static __init void init_page_alloc_tagging(void)
-{
-	clear_early_alloc_pfn_tag_refs();
-}
-
-struct page_ext_operations page_alloc_tagging_ops = {
-	.size = sizeof(union codetag_ref),
-	.need = need_page_alloc_tagging,
-	.init = init_page_alloc_tagging,
-};
-EXPORT_SYMBOL(page_alloc_tagging_ops);
-
-#ifdef CONFIG_SYSCTL
-/*
- * Not using proc_do_static_key() directly to prevent enabling profiling
- * after it was shut down.
- */
-static int proc_mem_profiling_handler(const struct ctl_table *table, int write,
-				      void *buffer, size_t *lenp, loff_t *ppos)
-{
-	if (write) {
-		/*
-		 * Call from do_sysctl_args() which is a no-op since the same
-		 * value was already set by setup_early_mem_profiling.
-		 * Return success to avoid warnings from do_sysctl_args().
-		 */
-		if (!current->mm)
-			return 0;
-
-#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
-		/* User can't toggle profiling while debugging */
-		return -EACCES;
-#endif
-		if (!mem_profiling_support)
-			return -EINVAL;
-	}
-
-	return proc_do_static_key(table, write, buffer, lenp, ppos);
-}
-
-
-static const struct ctl_table memory_allocation_profiling_sysctls[] = {
-	{
-		.procname	= "mem_profiling",
-		.data		= &mem_alloc_profiling_key,
-		.mode		= 0644,
-		.proc_handler	= proc_mem_profiling_handler,
-	},
-};
-
-static void __init sysctl_init(void)
-{
-	register_sysctl_init("vm", memory_allocation_profiling_sysctls);
-}
-#else /* CONFIG_SYSCTL */
-static inline void sysctl_init(void) {}
-#endif /* CONFIG_SYSCTL */
-
-static int __init alloc_tag_init(void)
-{
-	const struct codetag_type_desc desc = {
-		.section		= ALLOC_TAG_SECTION_NAME,
-		.tag_size		= sizeof(struct alloc_tag),
-#ifdef CONFIG_MODULES
-		.needs_section_mem	= needs_section_mem,
-		.alloc_section_mem	= reserve_module_tags,
-		.free_section_mem	= release_module_tags,
-		.module_load		= load_module,
-		.module_replaced	= replace_module,
-#endif
-	};
-	int res;
-
-	sysctl_init();
-
-	if (!mem_profiling_support) {
-		pr_info("Memory allocation profiling is not supported!\n");
-		return 0;
-	}
-
-	if (!proc_create_seq_private(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op,
-				     sizeof(struct allocinfo_private), NULL)) {
-		pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME);
-		shutdown_mem_profiling(false);
-		return -ENOMEM;
-	}
-
-	res = alloc_mod_tags_mem();
-	if (res) {
-		pr_err("Failed to reserve address space for module tags, errno = %d\n", res);
-		shutdown_mem_profiling(true);
-		return res;
-	}
-
-	alloc_tag_cttype = codetag_register_type(&desc);
-	if (IS_ERR(alloc_tag_cttype)) {
-		pr_err("Allocation tags registration failed, errno = %pe\n", alloc_tag_cttype);
-		free_mod_tags_mem();
-		shutdown_mem_profiling(true);
-		return PTR_ERR(alloc_tag_cttype);
-	}
-
-	return 0;
-}
-module_init(alloc_tag_init);
diff --git a/lib/bitmap-str.c b/lib/bitmap-str.c
index 26d36c938c6af..dd9aa0635fa56 100644
--- a/lib/bitmap-str.c
+++ b/lib/bitmap-str.c
@@ -75,8 +75,7 @@ static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp,
  * @off: in the string from which we are copying, We copy to @buf
  * @count: the maximum number of bytes to print
  *
- * The sprintf("%*pb[l]") is used indirectly via its cpumap wrapper
- * cpumap_print_to_pagebuf() or directly by drivers to export hexadecimal
+ * The sprintf("%*pb[l]") format is used by drivers to export hexadecimal
  * bitmask and decimal list to userspace by sysfs ABI.
  * Drivers might be using a normal attribute for this kind of ABIs. A
  * normal attribute typically has show entry as below::
@@ -115,9 +114,9 @@ static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp,
  * parameters such as off, count from bin_attribute show entry to this API.
  *
  * The role of cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf()
- * is similar with cpumap_print_to_pagebuf(),  the difference is that
- * scnprintf("%*pb[l]") mainly serves sysfs attribute with the assumption
- * the destination buffer is exactly one page and won't be more than one page.
+ * is similar to direct sysfs_emit("%*pb[l]") formatting, but the latter
+ * assumes the destination buffer is exactly one page and won't be more than
+ * one page.
  * cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf(), on the other
  * hand, mainly serves bin_attribute which doesn't work with exact one page,
  * and it can break the size limit of converted decimal list and hexadecimal
diff --git a/lib/bitmap.c b/lib/bitmap.c
index b9bfa157e0950..b464d843f4eb4 100644
--- a/lib/bitmap.c
+++ b/lib/bitmap.c
@@ -432,22 +432,28 @@ unsigned long bitmap_find_next_zero_area_off(unsigned long *map,
 					     unsigned long align_mask,
 					     unsigned long align_offset)
 {
-	unsigned long index, end, i;
-again:
-	index = find_next_zero_bit(map, size, start);
-
-	/* Align allocation */
-	index = __ALIGN_MASK(index + align_offset, align_mask) - align_offset;
-
-	end = index + nr;
-	if (end > size)
-		return end;
-	i = find_next_bit(map, end, index);
-	if (i < end) {
-		start = i + 1;
-		goto again;
+	unsigned long end, i, off;
+
+	for_each_clear_bit_from(start, map, size) {
+		start = __ALIGN_MASK(start + align_offset, align_mask) - align_offset;
+		end = start + nr;
+		if (end > size)
+			break;
+
+		off = round_down(start, BITS_PER_LONG);
+		i = find_last_bit(map + start / BITS_PER_LONG, end - off) + off;
+		if (i >= end || i < start)
+			return start;
+
+		start = i;
 	}
-	return index;
+
+	/*
+	 * Here, returning size + 1 is to maintain consistency
+	 * with the old version, where the return value is always
+	 * greater than size when no zero areas are found.
+	 */
+	return size + 1;
 }
 EXPORT_SYMBOL(bitmap_find_next_zero_area_off);
 
diff --git a/lib/bootconfig.c b/lib/bootconfig.c
index 2ed9ee3dc81c7..89c88e359179f 100644
--- a/lib/bootconfig.c
+++ b/lib/bootconfig.c
@@ -19,9 +19,13 @@
 #include <linux/errno.h>
 #include <linux/cache.h>
 #include <linux/compiler.h>
+#include <linux/init.h>
+#include <linux/moduleparam.h>
+#include <linux/printk.h>
 #include <linux/sprintf.h>
 #include <linux/memblock.h>
 #include <linux/string.h>
+#include <asm/setup.h>		/* COMMAND_LINE_SIZE */
 
 #ifdef CONFIG_BOOT_CONFIG_EMBED
 /* embedded_bootconfig_data is defined in bootconfig-data.S */
@@ -34,7 +38,129 @@ const char * __init xbc_get_embedded_bootconfig(size_t *size)
 	return (*size) ? embedded_bootconfig_data : NULL;
 }
 #endif
-#endif
+
+#ifdef CONFIG_CMDLINE_FROM_BOOTCONFIG
+/* embedded_kernel_cmdline is defined in embedded-cmdline.S */
+extern __visible const char embedded_kernel_cmdline[];
+extern __visible const char embedded_kernel_cmdline_end[];
+
+/* Set once the embedded cmdline has actually been prepended. */
+static bool xbc_cmdline_applied __initdata;
+
+/*
+ * str_prepend() - Prepend @src in front of the string in @dst, in place
+ * @dst: NUL-terminated destination buffer, currently @dst_len bytes long
+ * @dst_len: length of the current @dst string (excluding its NUL)
+ * @src: bytes to prepend (not NUL-terminated)
+ * @src_len: number of bytes from @src to prepend
+ *
+ * The caller must guarantee @dst has room for src_len + dst_len + 1 bytes.
+ * Moving dst_len + 1 bytes carries @dst's NUL terminator too, so an empty
+ * @dst needs no special case.
+ */
+static void __init str_prepend(char *dst, size_t dst_len,
+			       const char *src, size_t src_len)
+{
+	memmove(dst + src_len, dst, dst_len + 1);
+	memcpy(dst, src, src_len);
+}
+
+/**
+ * xbc_prepend_embedded_cmdline() - Prepend embedded bootconfig cmdline
+ * @dst: cmdline buffer to prepend into (must already contain a NUL byte)
+ * @size: total capacity of @dst in bytes
+ *
+ * Prepend the build-time-rendered "kernel" subtree of the embedded
+ * bootconfig to @dst. The rendered string already ends with a single
+ * space (the xbc_snprint_cmdline() invariant), which serves as the
+ * separator between the embedded keys and any existing content of @dst.
+ * On overflow, log an error and leave @dst untouched rather than
+ * silently truncating: booting without the embedded values is better
+ * than refusing to boot, and the error message tells the user why
+ * their embedded keys are missing.
+ *
+ * Intended to be called from setup_arch() before parse_early_param() so
+ * that early_param() handlers see the embedded values.
+ */
+void __init xbc_prepend_embedded_cmdline(char *dst, size_t size)
+{
+	size_t embed_len = embedded_kernel_cmdline_end - embedded_kernel_cmdline;
+	size_t dst_len;
+
+	if (!size || embed_len <= 1)	/* trailing NUL only */
+		return;
+	embed_len--;			/* exclude trailing NUL byte */
+
+	dst_len = strnlen(dst, size);
+	if (embed_len + dst_len + 1 > size) {
+		pr_err("embedded bootconfig cmdline (%zu bytes) does not fit in COMMAND_LINE_SIZE with %zu bytes already used; ignoring embedded values\n",
+		       embed_len, dst_len);
+		return;
+	}
+
+	str_prepend(dst, dst_len, embedded_kernel_cmdline, embed_len);
+	xbc_cmdline_applied = true;
+}
+
+/**
+ * xbc_embedded_cmdline_applied() - Did the embedded cmdline get prepended?
+ *
+ * Return true if xbc_prepend_embedded_cmdline() actually prepended the
+ * embedded "kernel" subtree. setup_boot_config() uses this to avoid
+ * rendering the same keys a second time.
+ */
+bool __init xbc_embedded_cmdline_applied(void)
+{
+	return xbc_cmdline_applied;
+}
+#endif	/* CONFIG_CMDLINE_FROM_BOOTCONFIG */
+
+/* parse_args() callback: flag when the "bootconfig" parameter is present. */
+static int __init bootconfig_optin(char *param, char *val,
+				   const char *unused, void *arg)
+{
+	if (!strcmp(param, "bootconfig"))
+		*(bool *)arg = true;
+	return 0;
+}
+
+/**
+ * bootconfig_cmdline_requested() - Was "bootconfig" passed on the cmdline?
+ * @boot_cmdline: kernel command line to inspect (not modified)
+ * @end_offset: if non-NULL, set to the offset of the init arguments that
+ *		follow a "--" separator, or 0 when there is none
+ *
+ * Parse a private copy of @boot_cmdline (parse_args() is destructive) and
+ * report whether "bootconfig" is present before the "--" separator.
+ * setup_arch() uses this to gate prepending the build-time embedded cmdline;
+ * setup_boot_config() uses it for the runtime opt-in and to locate the init
+ * arguments via @end_offset. Sharing one parser keeps the early and late
+ * paths agreeing on what counts as opt-in. CONFIG_BOOT_CONFIG_FORCE is not
+ * folded in here; callers apply it where they need it.
+ */
+bool __init bootconfig_cmdline_requested(const char *boot_cmdline, int *end_offset)
+{
+	static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;
+	bool found = false;
+	char *err;
+
+	if (end_offset)
+		*end_offset = 0;
+
+	strscpy(tmp_cmdline, boot_cmdline, COMMAND_LINE_SIZE);
+	err = parse_args("bootconfig", tmp_cmdline, NULL, 0, 0, 0,
+			 &found, bootconfig_optin);
+	if (IS_ERR(err))
+		return false;
+
+	/* parse_args() stops at "--" and returns the address of the rest. */
+	if (end_offset && err)
+		*end_offset = err - tmp_cmdline;
+
+	return found;
+}
+
+#endif	/* __KERNEL__ */
 
 /*
  * Extra Boot Config (XBC) is given as tree-structured ascii text of
@@ -440,6 +566,17 @@ int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root)
 	 * itself is well defined and returns the would-be length.
 	 */
 	xbc_node_for_each_key_value(root, knode, val) {
+		/*
+		 * An empty or value-only @root (e.g. "kernel {}" or
+		 * "kernel = x", possibly alongside "kernel.foo = bar")
+		 * yields @root itself here. Skip it: composing a key for it
+		 * would fail with -EINVAL, yet any real descendant keys must
+		 * still be rendered. An entirely empty subtree then renders
+		 * nothing and returns 0 rather than an error.
+		 */
+		if (knode == root)
+			continue;
+
 		ret = xbc_node_compose_key_after(root, knode,
 					xbc_namebuf, XBC_KEYLEN_MAX);
 		if (ret < 0)
diff --git a/lib/bug.c b/lib/bug.c
index 292420f45811a..7c1c2c27f58e7 100644
--- a/lib/bug.c
+++ b/lib/bug.c
@@ -219,14 +219,12 @@ static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long buga
 	no_cut   = bug->flags & BUGFLAG_NO_CUT_HERE;
 	has_args = bug->flags & BUGFLAG_ARGS;
 
-#ifdef CONFIG_KUNIT
 	/*
 	 * Before the once logic so suppressed warnings do not consume
 	 * the single-fire budget of WARN_ON_ONCE().
 	 */
 	if (warning && kunit_is_suppressed_warning(true))
 		return BUG_TRAP_TYPE_WARN;
-#endif
 
 	disable_trace_on_warning();
 
diff --git a/lib/codetag.c b/lib/codetag.c
index 4001a7ea66754..a9cda4c962a30 100644
--- a/lib/codetag.c
+++ b/lib/codetag.c
@@ -19,6 +19,8 @@ struct codetag_type {
 	struct codetag_type_desc desc;
 	/* generates unique sequence number for module load */
 	unsigned long next_mod_seq;
+	/* bumped on every module load and unload */
+	unsigned long content_id;
 };
 
 struct codetag_range {
@@ -50,6 +52,20 @@ void codetag_unlock_module_list(struct codetag_type *cttype)
 	up_read(&cttype->mod_lock);
 }
 
+unsigned long codetag_get_content_id(struct codetag_type *cttype)
+{
+	lockdep_assert_held(&cttype->mod_lock);
+
+	return cttype->content_id;
+}
+
+unsigned int codetag_get_count(struct codetag_type *cttype)
+{
+	lockdep_assert_held(&cttype->mod_lock);
+
+	return cttype->count;
+}
+
 struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype)
 {
 	struct codetag_iterator iter = {
@@ -204,6 +220,7 @@ static int codetag_module_init(struct codetag_type *cttype, struct module *mod)
 
 	down_write(&cttype->mod_lock);
 	cmod->mod_seq = ++cttype->next_mod_seq;
+	++cttype->content_id;
 	mod_id = idr_alloc(&cttype->mod_idr, cmod, 0, 0, GFP_KERNEL);
 	if (mod_id >= 0) {
 		if (cttype->desc.module_load) {
@@ -368,6 +385,7 @@ void codetag_unload_module(struct module *mod)
 			cttype->count -= range_size(cttype, &cmod->range);
 			idr_remove(&cttype->mod_idr, mod_id);
 			kfree(cmod);
+			++cttype->content_id;
 		}
 		up_write(&cttype->mod_lock);
 		if (found && cttype->desc.free_section_mem)
diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
index 591c1c2a7fb31..83d4c95e079ef 100644
--- a/lib/crypto/Kconfig
+++ b/lib/crypto/Kconfig
@@ -8,8 +8,7 @@ config CRYPTO_LIB_UTILS
 
 config CRYPTO_LIB_AES
 	tristate
-	# Select dependencies of modes that are part of libaes.
-	select CRYPTO_LIB_UTILS if CRYPTO_LIB_AES_CBC_MACS
+	select CRYPTO_LIB_UTILS
 
 config CRYPTO_LIB_AES_ARCH
 	bool
diff --git a/lib/crypto/md5.c b/lib/crypto/md5.c
index 6bf130cfbbf98..3d2b017a0525a 100644
--- a/lib/crypto/md5.c
+++ b/lib/crypto/md5.c
@@ -298,19 +298,5 @@ void hmac_md5_usingrawkey(const u8 *raw_key, size_t raw_key_len,
 }
 EXPORT_SYMBOL_GPL(hmac_md5_usingrawkey);
 
-#ifdef md5_mod_init_arch
-static int __init md5_mod_init(void)
-{
-	md5_mod_init_arch();
-	return 0;
-}
-subsys_initcall(md5_mod_init);
-
-static void __exit md5_mod_exit(void)
-{
-}
-module_exit(md5_mod_exit);
-#endif
-
 MODULE_DESCRIPTION("MD5 and HMAC-MD5 library functions");
 MODULE_LICENSE("GPL");
diff --git a/lib/embedded-cmdline.S b/lib/embedded-cmdline.S
new file mode 100644
index 0000000000000..bda81b4a42bea
--- /dev/null
+++ b/lib/embedded-cmdline.S
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Embed the build-time-rendered bootconfig "kernel" subtree as a flat
+ * cmdline string. setup_arch() prepends this to boot_command_line on
+ * architectures that select ARCH_SUPPORTS_CMDLINE_FROM_BOOTCONFIG.
+ *
+ * Copyright (c) 2026 Meta Platforms, Inc. and affiliates
+ * Copyright (c) 2026 Breno Leitao <leitao@debian.org>
+ */
+	.section .init.rodata.embed_cmdline, "a", %progbits
+	.global embedded_kernel_cmdline
+embedded_kernel_cmdline:
+	.incbin "lib/embedded_cmdline.bin"
+	.byte 0
+	.global embedded_kernel_cmdline_end
+embedded_kernel_cmdline_end:
diff --git a/lib/find_bit_benchmark.c b/lib/find_bit_benchmark.c
index 00d9dc61cd464..05305e655f99c 100644
--- a/lib/find_bit_benchmark.c
+++ b/lib/find_bit_benchmark.c
@@ -149,6 +149,21 @@ static int __init test_find_next_and_bit(const void *bitmap,
 	return 0;
 }
 
+static int __init
+test_bitmap_find_next_zero_area_off(unsigned long *bitmap, unsigned long len)
+{
+	unsigned long i, cnt;
+	ktime_t time;
+
+	time = ktime_get();
+	for (cnt = i = 0; i < BITMAP_LEN; cnt++)
+		i = bitmap_find_next_zero_area_off(bitmap, BITMAP_LEN, i, 8, 0, 0) + 1;
+	time = ktime_get() - time;
+	pr_err("bitmap_find_next_zero_area_off:%7llu ns, %6ld iterations\n", time, cnt);
+
+	return 0;
+}
+
 static int __init find_bit_test(void)
 {
 	unsigned long nbits = BITMAP_LEN / SPARSE;
@@ -158,6 +173,7 @@ static int __init find_bit_test(void)
 	get_random_bytes(bitmap, sizeof(bitmap));
 	get_random_bytes(bitmap2, sizeof(bitmap2));
 
+	test_bitmap_find_next_zero_area_off(bitmap, BITMAP_LEN);
 	test_find_next_bit(bitmap, BITMAP_LEN);
 	test_find_next_zero_bit(bitmap, BITMAP_LEN);
 	test_find_last_bit(bitmap, BITMAP_LEN);
@@ -181,6 +197,7 @@ static int __init find_bit_test(void)
 		__set_bit(get_random_u32_below(BITMAP_LEN), bitmap2);
 	}
 
+	test_bitmap_find_next_zero_area_off(bitmap, BITMAP_LEN);
 	test_find_next_bit(bitmap, BITMAP_LEN);
 	test_find_next_zero_bit(bitmap, BITMAP_LEN);
 	test_find_last_bit(bitmap, BITMAP_LEN);
diff --git a/lib/fonts/fonts.c b/lib/fonts/fonts.c
index f5d5333450a02..4fc66722d00d9 100644
--- a/lib/fonts/fonts.c
+++ b/lib/fonts/fonts.c
@@ -178,6 +178,37 @@ unsigned int font_data_size(font_data_t *fd)
 }
 EXPORT_SYMBOL_GPL(font_data_size);
 
+static unsigned int font_data_num_glyphs(font_data_t *fd, unsigned int width, unsigned int height)
+{
+	return font_data_size(fd) / font_glyph_size(width, height);
+}
+
+/**
+ * font_data_glyph_buf() - Returns the glyph for a specific character as raw bytes
+ * @fd: The font data
+ * @width: The glyph width in bits per scanline
+ * @vpitch: The number of scanlines per glyph
+ * @c: The character
+ *
+ * Glyphs start at fixed intervals within the font data. font_data_glyph_buf()
+ * returns the glyph shape of the specified character. If no such glyph
+ * exists in the font, it returns NULL.
+ *
+ * Returns:
+ * The character's raw glyph shape, or NULL if no glyph exists for the character. The
+ * provided buffer is read-only.
+ */
+const unsigned char *font_data_glyph_buf(font_data_t *fd,
+					 unsigned int width, unsigned int vpitch,
+					 unsigned int c)
+{
+	if (c >= font_data_num_glyphs(fd, width, vpitch))
+		return NULL;
+
+	return font_data_buf(fd) + font_glyph_size(width, vpitch) * c;
+}
+EXPORT_SYMBOL_GPL(font_data_glyph_buf);
+
 /**
  * font_data_is_equal - Compares font data for equality
  * @lhs: Left-hand side font data
diff --git a/lib/idr.c b/lib/idr.c
index 69bee5369670c..1649f41016e7c 100644
--- a/lib/idr.c
+++ b/lib/idr.c
@@ -495,10 +495,9 @@ int ida_find_first_range(struct ida *ida, unsigned int min, unsigned int max)
 	unsigned long index = min / IDA_BITMAP_BITS;
 	unsigned int offset = min % IDA_BITMAP_BITS;
 	unsigned long *addr, size, bit;
-	unsigned long tmp = 0;
+	unsigned long tmp;
 	unsigned long flags;
 	void *entry;
-	int ret;
 
 	if ((int)min < 0)
 		return -EINVAL;
@@ -508,40 +507,34 @@ int ida_find_first_range(struct ida *ida, unsigned int min, unsigned int max)
 	xa_lock_irqsave(&ida->xa, flags);
 
 	entry = xa_find(&ida->xa, &index, max / IDA_BITMAP_BITS, XA_PRESENT);
-	if (!entry) {
-		ret = -ENOENT;
-		goto err_unlock;
-	}
-
-	if (index > min / IDA_BITMAP_BITS)
-		offset = 0;
-	if (index * IDA_BITMAP_BITS + offset > max) {
-		ret = -ENOENT;
-		goto err_unlock;
-	}
-
-	if (xa_is_value(entry)) {
-		tmp = xa_to_value(entry);
-		addr = &tmp;
-		size = BITS_PER_XA_VALUE;
-	} else {
-		addr = ((struct ida_bitmap *)entry)->bitmap;
-		size = IDA_BITMAP_BITS;
-	}
-
-	bit = find_next_bit(addr, size, offset);
+	while (entry) {
+		if (index > min / IDA_BITMAP_BITS)
+			offset = 0;
+		if (index * IDA_BITMAP_BITS + offset > max)
+			break;
 
-	xa_unlock_irqrestore(&ida->xa, flags);
+		if (xa_is_value(entry)) {
+			tmp = xa_to_value(entry);
+			addr = &tmp;
+			size = BITS_PER_XA_VALUE;
+		} else {
+			addr = ((struct ida_bitmap *)entry)->bitmap;
+			size = IDA_BITMAP_BITS;
+		}
 
-	if (bit == size ||
-	    index * IDA_BITMAP_BITS + bit > max)
-		return -ENOENT;
+		bit = find_next_bit(addr, size, offset);
+		if (bit < size &&
+		    index * IDA_BITMAP_BITS + bit <= max) {
+			xa_unlock_irqrestore(&ida->xa, flags);
+			return index * IDA_BITMAP_BITS + bit;
+		}
 
-	return index * IDA_BITMAP_BITS + bit;
+		entry = xa_find_after(&ida->xa, &index,
+				      max / IDA_BITMAP_BITS, XA_PRESENT);
+	}
 
-err_unlock:
 	xa_unlock_irqrestore(&ida->xa, flags);
-	return ret;
+	return -ENOENT;
 }
 EXPORT_SYMBOL(ida_find_first_range);
 
diff --git a/lib/kobject.c b/lib/kobject.c
index 9c9ff0f5175fb..e7b010a989fb9 100644
--- a/lib/kobject.c
+++ b/lib/kobject.c
@@ -823,9 +823,11 @@ static ssize_t kobj_attr_show(struct kobject *kobj, struct attribute *attr,
 	struct kobj_attribute *kattr;
 	ssize_t ret = -EIO;
 
-	kattr = container_of(attr, struct kobj_attribute, attr);
+	kattr = container_of_const(attr, struct kobj_attribute, attr);
 	if (kattr->show)
 		ret = kattr->show(kobj, kattr, buf);
+	else if (kattr->show_const)
+		ret = kattr->show_const(kobj, kattr, buf);
 	return ret;
 }
 
@@ -835,9 +837,11 @@ static ssize_t kobj_attr_store(struct kobject *kobj, struct attribute *attr,
 	struct kobj_attribute *kattr;
 	ssize_t ret = -EIO;
 
-	kattr = container_of(attr, struct kobj_attribute, attr);
+	kattr = container_of_const(attr, struct kobj_attribute, attr);
 	if (kattr->store)
 		ret = kattr->store(kobj, kattr, buf, count);
+	else if (kattr->store_const)
+		ret = kattr->store_const(kobj, kattr, buf, count);
 	return ret;
 }
 
diff --git a/lib/kstrtox.c b/lib/kstrtox.c
index edc4eb7c1bca1..bac1c057e1b0d 100644
--- a/lib/kstrtox.c
+++ b/lib/kstrtox.c
@@ -17,6 +17,7 @@
 #include <linux/export.h>
 #include <linux/kstrtox.h>
 #include <linux/math64.h>
+#include <linux/overflow.h>
 #include <linux/types.h>
 #include <linux/uaccess.h>
 
@@ -45,6 +46,7 @@ const char *_parse_integer_fixup_radix(const char *s, unsigned int *base)
  * @base: Radix
  * @p: Where to store result
  * @max_chars: Maximum amount of characters to convert
+ * @init: Initial value of the multiply-accumulate result
  *
  * Convert non-negative integer string representation in explicitly given
  * radix to an integer. If overflow occurs, value at @p is set to ULLONG_MAX.
@@ -56,12 +58,12 @@ const char *_parse_integer_fixup_radix(const char *s, unsigned int *base)
  */
 noinline
 unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned long long *p,
-				  size_t max_chars)
+				  size_t max_chars, unsigned long long init)
 {
 	unsigned int rv, overflow = 0;
 	unsigned long long res;
 
-	res = 0;
+	res = init;
 	for (rv = 0; rv < max_chars; rv++, s++) {
 		unsigned int c = *s;
 		unsigned int lc = _tolower(c);
@@ -94,12 +96,6 @@ unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned lon
 	return rv | overflow;
 }
 
-noinline
-unsigned int _parse_integer(const char *s, unsigned int base, unsigned long long *p)
-{
-	return _parse_integer_limit(s, base, p, INT_MAX);
-}
-
 static int _kstrtoull(const char *s, unsigned int base, unsigned long long *res)
 {
 	unsigned long long _res;
@@ -399,6 +395,109 @@ int kstrtobool(const char *s, bool *res)
 }
 EXPORT_SYMBOL(kstrtobool);
 
+static int _kstrtoudec64(const char *s, unsigned int scale, u64 *res)
+{
+	unsigned int rv_int, rv_frac;
+	u64 _res = 0;
+
+	rv_int = _parse_integer(s, 10, &_res);
+	if (rv_int & KSTRTOX_OVERFLOW)
+		return -ERANGE;
+	s += rv_int;
+
+	if (*s == '.')
+		s++; /* skip decimal point */
+
+	rv_frac = _parse_integer(s, 10, &_res, scale, _res);
+	if (rv_frac & KSTRTOX_OVERFLOW)
+		return -ERANGE;
+	s += rv_frac;
+
+	/*
+	 * Check input beyond rv_int and rv_frac to cover cases like ".5" with
+	 * scale 0, which is considered a valid input, being parsed as 0.
+	 */
+	if (!rv_int && !rv_frac && !isdigit(*s))
+		return -EINVAL;
+
+	while (isdigit(*s)) /* truncate digits */
+		s++;
+
+	if (*s == '\n')
+		s++;
+	if (*s)
+		return -EINVAL;
+
+	if (_res && ((scale - rv_frac) > 19 /* log10(2^64) = 19.26 */ ||
+		     check_mul_overflow(_res, int_pow(10, scale - rv_frac), &_res)))
+		return -ERANGE;
+
+	*res = _res;
+	return 0;
+}
+
+/**
+ * kstrtoudec64() - Convert a string to an unsigned 64-bit scaled decimal value.
+ * @s: The start of the string. The string must be null-terminated, and may also
+ *  include a single newline before its terminating null. The first character
+ *  may also be a plus sign, but not a minus sign.
+ * @scale: The number of digits to the right of the decimal point.
+ * @res: Where to write the result of the conversion on success.
+ *
+ * For example, a scale of 3 with input "123.45" results in 123450. Note that
+ * trailing zeros in the fractional part input to match the scale are not
+ * required. Also, digits beyond the specified scale are ignored.
+ *
+ * Return: 0 on success, -ERANGE on overflow and -EINVAL on parsing error.
+ */
+noinline
+int kstrtoudec64(const char *s, unsigned int scale, u64 *res)
+{
+	if (s[0] == '+')
+		s++;
+	return _kstrtoudec64(s, scale, res);
+}
+EXPORT_SYMBOL(kstrtoudec64);
+
+/**
+ * kstrtodec64() - Convert a string to a signed 64-bit scaled decimal value.
+ * @s: The start of the string. The string must be null-terminated, and may also
+ *  include a single newline before its terminating null. The first character
+ *  may also be a plus sign or a minus sign.
+ * @scale: The number of digits to the right of the decimal point.
+ * @res: Where to write the result of the conversion on success.
+ *
+ * For example, a scale of 4 with input "-3.141592" results in -31415. Note
+ * that digits beyond the specified scale are ignored. Also, trailing zeros in
+ * the fractional part input to match the scale are not required.
+ *
+ * Return: 0 on success, -ERANGE on overflow and -EINVAL on parsing error.
+ */
+noinline
+int kstrtodec64(const char *s, unsigned int scale, s64 *res)
+{
+	u64 tmp;
+	int rv;
+
+	if (s[0] == '-') {
+		rv = _kstrtoudec64(s + 1, scale, &tmp);
+		if (rv < 0)
+			return rv;
+		if ((s64)-tmp > 0)
+			return -ERANGE;
+		*res = -tmp;
+	} else {
+		rv = kstrtoudec64(s, scale, &tmp);
+		if (rv < 0)
+			return rv;
+		if ((s64)tmp < 0)
+			return -ERANGE;
+		*res = tmp;
+	}
+	return 0;
+}
+EXPORT_SYMBOL(kstrtodec64);
+
 /*
  * Since "base" would be a nonsense argument, this open-codes the
  * _from_user helper instead of using the helper macro below.
diff --git a/lib/kstrtox.h b/lib/kstrtox.h
index 158c400ca8658..73dee79fd8ed2 100644
--- a/lib/kstrtox.h
+++ b/lib/kstrtox.h
@@ -2,10 +2,23 @@
 #ifndef _LIB_KSTRTOX_H
 #define _LIB_KSTRTOX_H
 
+#include <linux/args.h>
+
 #define KSTRTOX_OVERFLOW	(1U << 31)
 const char *_parse_integer_fixup_radix(const char *s, unsigned int *base);
 unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned long long *res,
-				  size_t max_chars);
-unsigned int _parse_integer(const char *s, unsigned int base, unsigned long long *res);
+				  size_t max_chars, unsigned long long init);
+
+#define _parse_integer0(s, base, res, ...)						\
+	_parse_integer_limit(s, base, res, INT_MAX, 0)
+
+#define _parse_integer1(s, base, res, max_chars, ...)					\
+	_parse_integer_limit(s, base, res, max_chars, 0)
+
+#define _parse_integer2(s, base, res, max_chars, init, ...)				\
+	_parse_integer_limit(s, base, res, max_chars, init)
+
+#define _parse_integer(s, base, res, ...)						\
+	CONCATENATE(_parse_integer, COUNT_ARGS(__VA_ARGS__))(s, base, res, __VA_ARGS__)
 
 #endif
diff --git a/lib/kunit/Makefile b/lib/kunit/Makefile
index 2e8a6b71a2ab0..204e02b10eba1 100644
--- a/lib/kunit/Makefile
+++ b/lib/kunit/Makefile
@@ -11,6 +11,7 @@ kunit-objs +=				test.o \
 					attributes.o \
 					device.o \
 					platform.o \
+					fwnode.o \
 					bug.o
 
 ifeq ($(CONFIG_KUNIT_DEBUGFS),y)
diff --git a/lib/kunit/debugfs.c b/lib/kunit/debugfs.c
index 9c326f1837bdf..442b2ceb955bb 100644
--- a/lib/kunit/debugfs.c
+++ b/lib/kunit/debugfs.c
@@ -76,18 +76,30 @@ static int debugfs_print_results(struct seq_file *seq, void *v)
 	seq_puts(seq, "KTAP version 1\n");
 	seq_puts(seq, "1..1\n");
 
-	/* Print suite header because it is not stored in the test logs. */
-	seq_puts(seq, KUNIT_SUBTEST_INDENT "KTAP version 1\n");
-	seq_printf(seq, KUNIT_SUBTEST_INDENT "# Subtest: %s\n", suite->name);
-	seq_printf(seq, KUNIT_SUBTEST_INDENT "1..%zd\n", kunit_suite_num_test_cases(suite));
-
-	kunit_suite_for_each_test_case(suite, test_case)
-		debugfs_print_result(seq, test_case->log);
+	if (suite->status != KUNIT_SKIPPED) {
+		/* Print suite header because it is not stored in the test logs. */
+		seq_puts(seq,
+			 KUNIT_SUBTEST_INDENT "KTAP version 1\n");
+		seq_printf(seq,
+			   KUNIT_SUBTEST_INDENT "# Subtest: %s\n",
+			   suite->name);
+		seq_printf(seq,
+			   KUNIT_SUBTEST_INDENT "1..%zd\n",
+			   kunit_suite_num_test_cases(suite));
+
+		kunit_suite_for_each_test_case(suite, test_case)
+			debugfs_print_result(seq, test_case->log);
+	}
 
 	debugfs_print_result(seq, suite->log);
 
-	seq_printf(seq, "%s %d %s\n",
-		   kunit_status_to_ok_not_ok(success), 1, suite->name);
+	if (suite->status != KUNIT_SKIPPED)
+		seq_printf(seq, "%s %d %s\n",
+			   kunit_status_to_ok_not_ok(success), 1, suite->name);
+	else
+		seq_printf(seq, "%s %d %s # SKIP %s\n",
+			   kunit_status_to_ok_not_ok(success), 1, suite->name,
+			   suite->status_comment);
 	return 0;
 }
 
diff --git a/lib/kunit/fwnode.c b/lib/kunit/fwnode.c
new file mode 100644
index 0000000000000..a58ce0984d76b
--- /dev/null
+++ b/lib/kunit/fwnode.c
@@ -0,0 +1,146 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) Qualcomm Technologies, Inc. and/or its subsidiaries
+ */
+
+#include <kunit/fwnode.h>
+#include <kunit/test.h>
+
+#include <linux/fwnode.h>
+#include <linux/property.h>
+
+KUNIT_DEFINE_ACTION_WRAPPER(fwnode_remove_software_node_wrapper,
+			    fwnode_remove_software_node,
+			    struct fwnode_handle *);
+
+/**
+ * kunit_fwnode_create_software_node() - Create a kunit-managed software node
+ * @test: Test context
+ * @properties: Properties to use to create the new software node
+ * @parent: Parent of this software node
+ *
+ * Create a test-managed software node and return its firmware node handle.
+ * The software node is removed after the test case completes.
+ *
+ * Returns:
+ * Firmware node handle of the newly created software node or IS_ERR() on
+ * failure.
+ */
+struct fwnode_handle *
+kunit_fwnode_create_software_node(struct kunit *test,
+				  const struct property_entry *properties,
+				  const struct fwnode_handle *parent)
+{
+	struct fwnode_handle *fwnode;
+	int ret;
+
+	fwnode = fwnode_create_software_node(properties, parent);
+	if (IS_ERR(fwnode))
+		return fwnode;
+
+	ret = kunit_add_action_or_reset(test, fwnode_remove_software_node_wrapper,
+					fwnode);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return fwnode;
+}
+EXPORT_SYMBOL_GPL(kunit_fwnode_create_software_node);
+
+KUNIT_DEFINE_ACTION_WRAPPER(software_node_unregister_wrapper,
+			    software_node_unregister,
+			    const struct software_node *);
+
+/**
+ * kunit_software_node_register() - Register a kunit-managed software node
+ * @test: Test context
+ * @swnode: Software node to register
+ *
+ * Register a test-managed software node and return its firmware node handle.
+ * The software node is unregistered after the test case completes.
+ *
+ * Returns:
+ * Firmware node handle of the registered software node or IS_ERR() on failure.
+ */
+struct fwnode_handle *
+kunit_software_node_register(struct kunit *test,
+			     const struct software_node *swnode)
+{
+	struct fwnode_handle *fwnode;
+	int ret;
+
+	ret = software_node_register(swnode);
+	if (ret)
+		return ERR_PTR(ret);
+
+	fwnode = software_node_fwnode(swnode);
+	if (WARN_ON(!fwnode))
+		return ERR_PTR(-ENOENT);
+
+	ret = kunit_add_action_or_reset(test, software_node_unregister_wrapper,
+					(void *)swnode);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return fwnode;
+}
+EXPORT_SYMBOL_GPL(kunit_software_node_register);
+
+KUNIT_DEFINE_ACTION_WRAPPER(software_node_unregister_node_group_wrapper,
+			    software_node_unregister_node_group,
+			    const struct software_node *const *);
+
+/**
+ * kunit_software_node_register_node_group() - Register a kunit-managed software node group
+ * @test: Test context
+ * @nodes: Software node group to register
+ *
+ * Register a test-managed software node group. The nodes are unregistered
+ * after the test case completes.
+ *
+ * Returns:
+ * 0 on success, negative error number on failure.
+ */
+int kunit_software_node_register_node_group(struct kunit *test,
+					    const struct software_node *const *nodes)
+{
+	int ret;
+
+	ret = software_node_register_node_group(nodes);
+	if (ret)
+		return ret;
+
+	return kunit_add_action_or_reset(test, software_node_unregister_node_group_wrapper,
+					 (void *)nodes);
+}
+EXPORT_SYMBOL_GPL(kunit_software_node_register_node_group);
+
+KUNIT_DEFINE_ACTION_WRAPPER(device_remove_software_node_wrapper,
+			    device_remove_software_node,
+			    struct device *);
+
+/**
+ * kunit_device_add_software_node() - Assign a kunit-managed software node to a device
+ * @test: Test context
+ * @dev: Device to assign the software node for
+ * @node: The software node to assign
+ *
+ * Make @node the secondary firmware node of @dev. If @dev has no primary
+ * firmware node, @node will become the primary node. The software node will
+ * be automatically removed from @dev when the test case completes.
+ *
+ * Returns:
+ * 0 on success, negative error number on failure.
+ */
+int kunit_device_add_software_node(struct kunit *test, struct device *dev,
+				   const struct software_node *node)
+{
+	int ret;
+
+	ret = device_add_software_node(dev, node);
+	if (ret)
+		return ret;
+
+	return kunit_add_action_or_reset(test, device_remove_software_node_wrapper, dev);
+}
+EXPORT_SYMBOL_GPL(kunit_device_add_software_node);
diff --git a/lib/kunit/kunit-example-test.c b/lib/kunit/kunit-example-test.c
index 0bae7b7ca0b05..b8ded54fa46d8 100644
--- a/lib/kunit/kunit-example-test.c
+++ b/lib/kunit/kunit-example-test.c
@@ -591,5 +591,34 @@ static struct kunit_suite example_init_test_suite = {
  */
 kunit_test_init_section_suites(&example_init_test_suite);
 
+/*
+ * This test should always be skipped.
+ */
+static void example_skip_suite_test(struct kunit *test)
+{
+	/* This line should never be seen */
+	KUNIT_FAIL(test, "You should not see a this.");
+}
+
+static struct kunit_case  example_skip_suite_test_cases[] = {
+	KUNIT_CASE(example_skip_suite_test),
+	{}
+};
+
+static int example_skip_suite_init(struct kunit_suite *suite)
+{
+	kunit_mark_skipped(suite, "Test suite expected to be skipped");
+	return 0;
+}
+
+static struct kunit_suite example_test_skip_suite = {
+	.name = "example_skip_suite",
+	.suite_init = example_skip_suite_init,
+	.test_cases = example_skip_suite_test_cases,
+};
+
+/* This registers a test suite that will be skipped */
+kunit_test_suite(example_test_skip_suite);
+
 MODULE_DESCRIPTION("Example KUnit test suite");
 MODULE_LICENSE("GPL v2");
diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c
index 0d8f1b30559b5..51ba40ebf19ff 100644
--- a/lib/kunit/string-stream.c
+++ b/lib/kunit/string-stream.c
@@ -9,6 +9,7 @@
 #include <kunit/static_stub.h>
 #include <kunit/test.h>
 #include <linux/list.h>
+#include <linux/seq_buf.h>
 #include <linux/slab.h>
 
 #include "string-stream.h"
@@ -74,7 +75,8 @@ int string_stream_vadd(struct string_stream *stream,
 
 		/* Append newline if necessary. */
 		if (frag_container->fragment[result_len - 1] != '\n')
-			result_len = strlcat(frag_container->fragment, "\n", buf_len);
+			result_len += strscpy(frag_container->fragment + result_len,
+					      "\n", buf_len - result_len);
 	} else {
 		result_len = vsnprintf(frag_container->fragment, buf_len, fmt, args);
 	}
@@ -118,15 +120,18 @@ char *string_stream_get_string(struct string_stream *stream)
 {
 	struct string_stream_fragment *frag_container;
 	size_t buf_len = stream->length + 1; /* +1 for null byte. */
+	struct seq_buf sb;
 	char *buf;
 
 	buf = kzalloc(buf_len, stream->gfp);
 	if (!buf)
 		return NULL;
 
+	seq_buf_init(&sb, buf, buf_len);
+
 	spin_lock(&stream->lock);
 	list_for_each_entry(frag_container, &stream->fragments, node)
-		strlcat(buf, frag_container->fragment, buf_len);
+		seq_buf_puts(&sb, frag_container->fragment);
 	spin_unlock(&stream->lock);
 
 	return buf;
diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index 99773e000e1b7..09e3dabfac0c9 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c
@@ -214,12 +214,18 @@ enum kunit_status kunit_suite_has_succeeded(struct kunit_suite *suite)
 	const struct kunit_case *test_case;
 	enum kunit_status status = KUNIT_SKIPPED;
 
+	if (suite->status == KUNIT_SKIPPED)
+		return KUNIT_SKIPPED;
+
 	if (suite->suite_init_err)
 		return KUNIT_FAILURE;
 
 	kunit_suite_for_each_test_case(suite, test_case) {
-		if (test_case->status == KUNIT_FAILURE)
+		if (test_case->status == KUNIT_FAILURE) {
+			/* Update the kunit_suite status also */
+			suite->status = KUNIT_FAILURE;
 			return KUNIT_FAILURE;
+		}
 		else if (test_case->status == KUNIT_SUCCESS)
 			status = KUNIT_SUCCESS;
 	}
@@ -795,12 +801,20 @@ int kunit_run_tests(struct kunit_suite *suite)
 	/* Taint the kernel so we know we've run tests. */
 	add_taint(TAINT_TEST, LOCKDEP_STILL_OK);
 
+	if (suite->status == KUNIT_SKIPPED)
+		goto suite_end;
+
 	if (suite->suite_init) {
 		suite->suite_init_err = suite->suite_init(suite);
 		if (suite->suite_init_err) {
+			suite->status = KUNIT_FAILURE;
 			kunit_err(suite, KUNIT_SUBTEST_INDENT
 				  "# failed to initialize (%d)", suite->suite_init_err);
 			goto suite_end;
+
+		} else if (suite->status == KUNIT_SKIPPED) {
+			/* Skip this kunit suite */
+			goto suite_end;
 		}
 	}
 
@@ -825,6 +839,7 @@ static void kunit_init_suite(struct kunit_suite *suite)
 	kunit_debugfs_create_suite(suite);
 	suite->status_comment[0] = '\0';
 	suite->suite_init_err = 0;
+	suite->status = KUNIT_SUCCESS;
 
 	if (suite->log)
 		string_stream_clear(suite->log);
diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index e52876435b77f..e8eb2e1219dbb 100644
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -261,6 +261,12 @@ static inline bool mas_is_underflow(struct ma_state *mas)
 	return mas->status == ma_underflow;
 }
 
+static inline void mas_make_walkable(struct ma_state *mas)
+{
+	if (!mas_is_active(mas) && !mas_is_start(mas))
+		mas->status = ma_start;
+}
+
 static __always_inline struct maple_node *mte_to_node(
 		const struct maple_enode *entry)
 {
@@ -455,46 +461,6 @@ enum maple_type mas_parent_type(struct ma_state *mas, struct maple_enode *enode)
 	return 0;
 }
 
-/*
- * mas_set_parent() - Set the parent node and encode the slot
- * @mas: The maple state
- * @enode: The encoded maple node.
- * @parent: The encoded maple node that is the parent of @enode.
- * @slot: The slot that @enode resides in @parent.
- *
- * Slot number is encoded in the enode->parent bit 3-6 or 2-6, depending on the
- * parent type.
- */
-static inline
-void mas_set_parent(struct ma_state *mas, struct maple_enode *enode,
-		    const struct maple_enode *parent, unsigned char slot)
-{
-	unsigned long val = (unsigned long)parent;
-	unsigned long shift;
-	unsigned long type;
-	enum maple_type p_type = mte_node_type(parent);
-
-	MAS_BUG_ON(mas, p_type == maple_dense);
-	MAS_BUG_ON(mas, p_type == maple_leaf_64);
-
-	switch (p_type) {
-	case maple_range_64:
-	case maple_arange_64:
-		shift = MAPLE_PARENT_SLOT_SHIFT;
-		type = MAPLE_PARENT_RANGE64;
-		break;
-	default:
-	case maple_dense:
-	case maple_leaf_64:
-		shift = type = 0;
-		break;
-	}
-
-	val &= ~MAPLE_NODE_MASK; /* Clear all node metadata in parent */
-	val |= (slot << shift) | type;
-	mte_to_node(enode)->parent = ma_parent_ptr(val);
-}
-
 /*
  * mte_parent_slot() - get the parent slot of @enode.
  * @enode: The encoded maple node.
@@ -876,6 +842,42 @@ static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt,
 	meta->gap = offset;
 }
 
+/*
+ * mas_set_parent_slots() - Bulk operation to set many slot parent pointers
+ * @mas: The maple state
+ * @parent: The encoded maple node that is the parent of @enode.
+ * @slot: The slot that of the @enode.
+ * @start_slot: The offset into @slot
+ * @count: The number of slots to set (eg: exclusive)
+ */
+static inline
+void mas_set_parent_slots(struct ma_state *mas, struct maple_enode *parent,
+	void __rcu **slots, unsigned char start_slot, unsigned char count)
+{
+	unsigned long val;
+	unsigned long shift;
+	unsigned long type;
+	enum maple_type p_type = mte_node_type(parent);
+	unsigned char i;
+
+	MAS_BUG_ON(mas, p_type != maple_range_64 &&
+			p_type != maple_arange_64);
+
+	shift = MAPLE_PARENT_SLOT_SHIFT;
+	type = MAPLE_PARENT_RANGE64;
+
+	val = (unsigned long)parent;
+	val &= ~MAPLE_NODE_MASK;
+
+	for (i = 0; i < count; i++) {
+		unsigned long pval = val | ((start_slot + i) << shift) | type;
+		struct maple_enode *child;
+
+		child = mt_slot_locked(mas->tree, slots, i);
+		mte_to_node(child)->parent = ma_parent_ptr(pval);
+	}
+}
+
 /*
  * mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes.
  * @mat: the ma_topiary, a linked list of dead nodes.
@@ -1153,6 +1155,73 @@ static inline void mas_free(struct ma_state *mas, struct maple_enode *used)
 	ma_free_rcu(mte_to_node(used));
 }
 
+
+#ifdef CONFIG_LOCKDEP
+static struct lockdep_map *mas_lockdep_map(struct ma_state *mas)
+{
+	struct maple_tree *mt = mas->tree;
+
+	if (mt_external_lock(mt))
+		return mt->ma_external_lock;
+
+	return &(mt->ma_lock).dep_map;
+}
+
+#endif
+
+static void mas_lock_check(struct ma_state *mas)
+{
+#ifdef CONFIG_LOCKDEP
+	struct lockdep_map *map;
+	if (!mas_is_active(mas))
+		return;
+
+#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
+	if (!mt_locked(mas->tree)) {
+		if (mt_in_rcu(mas->tree))
+			WARN_ON_ONCE(poll_state_synchronize_rcu(mas->rcu_gp));
+	}
+#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */
+
+	map = mas_lockdep_map(mas);
+	if (map && lock_is_held(map))
+		WARN_ON_ONCE(mas->ld_seq != lock_sequence(map));
+#endif /* CONFIG_LOCKDEP */
+
+}
+
+static void mas_init_lock_check(struct ma_state *mas)
+{
+#ifdef CONFIG_LOCKDEP
+	struct lockdep_map *map;
+#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
+	if (!mt_locked(mas->tree)) {
+		if (mt_in_rcu(mas->tree))
+			mas->rcu_gp = get_state_synchronize_rcu();
+		return;
+	}
+#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */
+
+	map = mas_lockdep_map(mas);
+	if (map && lock_is_held(map))
+		mas->ld_seq = lock_sequence(mas_lockdep_map(mas));
+#endif /* CONFIG_LOCKDEP */
+
+}
+
+static void mas_may_init_lock_check(struct ma_state *mas)
+{
+#ifdef CONFIG_LOCKDEP
+#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
+	if (mas_is_start(mas) || mas_is_paused(mas)) {
+		mas_init_lock_check(mas);
+		return;
+	}
+#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */
+	mas_lock_check(mas);
+#endif /* CONFIG_LOCKDEP */
+}
+
 /*
  * mas_start() - Sets up maple state for operations.
  * @mas: The maple state.
@@ -1171,6 +1240,7 @@ static inline struct maple_enode *mas_start(struct ma_state *mas)
 	if (likely(mas_is_start(mas))) {
 		struct maple_enode *root;
 
+		mas_init_lock_check(mas);
 		mas->min = 0;
 		mas->max = ULONG_MAX;
 
@@ -1501,14 +1571,26 @@ static inline void mas_parent_gap(struct ma_state *mas, unsigned char offset,
 	goto ascend;
 }
 
+static __always_inline void mas_update_gap_known(struct ma_state *mas,
+		unsigned long gap)
+{
+	unsigned char pslot;
+	unsigned long p_gap;
+
+	pslot = mte_parent_slot(mas->node);
+	p_gap = ma_gaps(mte_parent(mas->node),
+			mas_parent_type(mas, mas->node))[pslot];
+
+	if (p_gap != gap)
+		mas_parent_gap(mas, pslot, gap);
+}
+
 /*
  * mas_update_gap() - Update a nodes gaps and propagate up if necessary.
  * @mas: the maple state.
  */
 static inline void mas_update_gap(struct ma_state *mas)
 {
-	unsigned char pslot;
-	unsigned long p_gap;
 	unsigned long max_gap;
 
 	if (!mt_is_alloc(mas->tree))
@@ -1518,13 +1600,7 @@ static inline void mas_update_gap(struct ma_state *mas)
 		return;
 
 	max_gap = mas_max_gap(mas);
-
-	pslot = mte_parent_slot(mas->node);
-	p_gap = ma_gaps(mte_parent(mas->node),
-			mas_parent_type(mas, mas->node))[pslot];
-
-	if (p_gap != max_gap)
-		mas_parent_gap(mas, pslot, max_gap);
+	mas_update_gap_known(mas, max_gap);
 }
 
 /*
@@ -1540,14 +1616,10 @@ static inline void mas_adopt_children(struct ma_state *mas,
 	struct maple_node *node = mte_to_node(parent);
 	void __rcu **slots = ma_slots(node, type);
 	unsigned long *pivots = ma_pivots(node, type);
-	struct maple_enode *child;
-	unsigned char offset;
+	unsigned char end;
 
-	offset = ma_data_end(node, type, pivots, mas->max);
-	do {
-		child = mas_slot_locked(mas, slots, offset);
-		mas_set_parent(mas, child, parent, offset);
-	} while (offset--);
+	end = ma_data_end(node, type, pivots, mas->max);
+	mas_set_parent_slots(mas, parent, slots, 0, end + 1);
 }
 
 /*
@@ -1929,15 +2001,10 @@ unsigned long node_copy(struct ma_state *mas, struct maple_node *src,
 	s_slots = ma_slots(src, s_mt) + start;
 	s_pivots = ma_pivots(src, s_mt) + start;
 	memcpy(d_slots, s_slots, size * sizeof(void __rcu *));
-	if (!ma_is_leaf(d_mt) && s_mt == maple_copy) {
-		struct maple_enode *edst = mt_mk_node(dst, d_mt);
 
-
-		for (int i = 0; i < size; i++)
-			mas_set_parent(mas,
-				       mt_slot_locked(mas->tree, d_slots, i),
-				       edst, d_start + i);
-	}
+	if (!ma_is_leaf(d_mt) && s_mt == maple_copy)
+		mas_set_parent_slots(mas, mt_mk_node(dst, d_mt),
+				     d_slots, d_start, size);
 
 	d_gaps = ma_gaps(dst, d_mt);
 	if (d_gaps) {
@@ -2081,8 +2148,8 @@ static inline void mas_wmb_replace(struct ma_state *mas, struct maple_copy *cp)
 	mas->node = mt_slot_locked(mas->tree, cp->slot, 0);
 	/* Insert the new data in the tree */
 	mas_topiary_replace(mas, old_enode, cp->height);
-	if (!mte_is_leaf(mas->node))
-		mas_update_gap(mas);
+	if (mt_is_alloc(mas->tree) && !mte_is_root(mas->node))
+		mas_update_gap_known(mas, cp->gap[0]);
 
 	mtree_range_walk(mas);
 }
@@ -2928,13 +2995,6 @@ static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas,
 		if (r_mas->last < r_wr_mas->r_max)
 			r_mas->last = r_wr_mas->r_max;
 		r_mas->offset++;
-	} else if ((r_mas->last == r_wr_mas->r_max) &&
-	    (r_mas->last < r_mas->max) &&
-	    !mas_slot_locked(r_mas, r_wr_mas->slots, r_mas->offset + 1)) {
-		r_mas->last = mas_safe_pivot(r_mas, r_wr_mas->pivots,
-					     r_wr_mas->type, r_mas->offset + 1);
-		r_mas->offset++;
-		r_wr_mas->r_max = r_mas->last;
 	}
 }
 
@@ -3125,7 +3185,7 @@ static void mas_wr_spanning_store(struct ma_wr_state *wr_mas)
 static inline void mas_wr_node_store(struct ma_wr_state *wr_mas)
 {
 	unsigned char dst_offset, offset_end;
-	unsigned char copy_size, node_pivots;
+	unsigned char copy_size, node_pivots, node_slots;
 	struct maple_node reuse, *newnode;
 	unsigned long *dst_pivots;
 	void __rcu **dst_slots;
@@ -3138,6 +3198,7 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas)
 	in_rcu = mt_in_rcu(mas->tree);
 	offset_end = wr_mas->offset_end;
 	node_pivots = mt_pivots[wr_mas->type];
+	node_slots = mt_slots[wr_mas->type];
 	/* Assume last adds an entry */
 	new_end = mas->end + 1 - offset_end + mas->offset;
 	if (mas->last == wr_mas->end_piv) {
@@ -3149,7 +3210,6 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas)
 	if (in_rcu) {
 		newnode = mas_pop_node(mas);
 	} else {
-		memset(&reuse, 0, sizeof(struct maple_node));
 		newnode = &reuse;
 	}
 
@@ -3193,7 +3253,21 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas)
 		dst_pivots[new_end] = mas->max;
 
 done:
-	mas_leaf_set_meta(newnode, maple_leaf_64, new_end);
+	if (!in_rcu && new_end + 2 < node_slots) {
+		unsigned char clear_from = new_end + 1;
+
+		/*
+		 * Note that the last slot is never cleared, since the metadata
+		 * will be stored there or it has a value.
+		 */
+		memset(dst_slots + clear_from, 0,
+		       sizeof(void __rcu *) * (node_slots - clear_from));
+		if (clear_from < node_pivots)
+			memset(dst_pivots + clear_from, 0,
+			       sizeof(unsigned long) * (node_pivots - clear_from));
+	}
+
+	mas_leaf_set_meta(newnode, wr_mas->type, new_end);
 	if (in_rcu) {
 		struct maple_enode *old_enode = mas->node;
 
@@ -3218,7 +3292,7 @@ static inline void mas_wr_slot_store(struct ma_wr_state *wr_mas)
 	void __rcu **slots = wr_mas->slots;
 	bool gap = false;
 
-	gap |= !mt_slot_locked(mas->tree, slots, offset);
+	gap |= !wr_mas->content;
 	gap |= !mt_slot_locked(mas->tree, slots, offset + 1);
 
 	if (wr_mas->offset_end - offset == 1) {
@@ -3661,6 +3735,9 @@ static inline enum store_type mas_wr_store_type(struct ma_wr_state *wr_mas)
 {
 	struct ma_state *mas = wr_mas->mas;
 	unsigned char new_end;
+	bool appending;
+	bool one_slot;
+	bool in_rcu;
 
 	if (unlikely(mas_is_none(mas) || mas_is_ptr(mas)))
 		return wr_store_root;
@@ -3680,21 +3757,30 @@ static inline enum store_type mas_wr_store_type(struct ma_wr_state *wr_mas)
 		return wr_new_root;
 
 	new_end = mas_wr_new_end(wr_mas);
+	in_rcu = mt_in_rcu(mas->tree);
+	appending = mas->offset == mas->end;
+	one_slot = wr_mas->offset_end - mas->offset == 1;
+
 	/* Potential spanning rebalance collapsing a node */
 	if (new_end < mt_min_slots[wr_mas->type]) {
 		if (!mte_is_root(mas->node))
 			return  wr_rebalance;
+		if (!in_rcu) {
+			if (appending)
+				return wr_append;
+			else if (mas->end == new_end && one_slot)
+				return wr_slot_store;
+		}
 		return wr_node_store;
 	}
 
 	if (new_end >= mt_slots[wr_mas->type])
 		return wr_split_store;
 
-	if (!mt_in_rcu(mas->tree) && (mas->offset == mas->end))
+	if (!in_rcu && appending)
 		return wr_append;
 
-	if ((new_end == mas->end) && (!mt_in_rcu(mas->tree) ||
-		(wr_mas->offset_end - mas->offset == 1)))
+	if (new_end == mas->end && (!in_rcu || one_slot))
 		return wr_slot_store;
 
 	return wr_node_store;
@@ -3793,35 +3879,40 @@ int mas_alloc_cyclic(struct ma_state *mas, unsigned long *startp,
 		void *entry, unsigned long range_lo, unsigned long range_hi,
 		unsigned long *next, gfp_t gfp)
 {
-	unsigned long min = range_lo;
-	int ret = 0;
-
-	range_lo = max(min, *next);
-	ret = mas_empty_area(mas, range_lo, range_hi, 1);
-	if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) {
-		mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED;
-		ret = 1;
-	}
-	if (ret < 0 && range_lo > min) {
-		mas_reset(mas);
-		ret = mas_empty_area(mas, min, range_hi, 1);
-		if (ret == 0)
-			ret = 1;
-	}
-	if (ret < 0)
-		return ret;
+	int ret;
+	unsigned long min;
 
+	min = range_lo;
 	do {
+		range_lo = max(min, *next);
+		ret = mas_empty_area(mas, range_lo, range_hi, 1);
+		if (ret < 0 && range_lo > min) {
+			mas_reset(mas);
+			ret = mas_empty_area(mas, min, range_hi, 1);
+			if (ret == 0)
+				ret = 1;
+		}
+		if (ret < 0)
+			goto out;
+
 		mas_insert(mas, entry);
 	} while (mas_nomem(mas, gfp));
-	if (mas_is_err(mas))
-		return xa_err(mas->node);
 
+	if (mas_is_err(mas)) {
+		ret = xa_err(mas->node);
+		goto out;
+	}
+
+	if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) {
+		mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED;
+		ret = 1;
+	}
 	*startp = mas->index;
 	*next = *startp + 1;
 	if (*next == 0)
 		mas->tree->ma_flags |= MT_FLAGS_ALLOC_WRAPPED;
 
+out:
 	mas_destroy(mas);
 	return ret;
 }
@@ -4360,8 +4451,8 @@ void *mas_walk(struct ma_state *mas)
 {
 	void *entry;
 
-	if (!mas_is_active(mas) && !mas_is_start(mas))
-		mas->status = ma_start;
+	mas_may_init_lock_check(mas);
+	mas_make_walkable(mas);
 retry:
 	entry = mas_state_walk(mas);
 	if (mas_is_start(mas)) {
@@ -4831,6 +4922,7 @@ void *mas_store(struct ma_state *mas, void *entry)
 {
 	MA_WR_STATE(wr_mas, mas, entry);
 
+	mas_may_init_lock_check(mas);
 	trace_ma_write(TP_FCT, mas, 0, entry);
 #ifdef CONFIG_DEBUG_MAPLE_TREE
 	if (MAS_WARN_ON(mas, mas->index > mas->last))
@@ -4889,6 +4981,7 @@ int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp)
 	MA_WR_STATE(wr_mas, mas, entry);
 	int ret = 0;
 
+	mas_may_init_lock_check(mas);
 retry:
 	mas_wr_preallocate(&wr_mas, entry);
 	if (unlikely(mas_nomem(mas, gfp))) {
@@ -4919,6 +5012,7 @@ void mas_store_prealloc(struct ma_state *mas, void *entry)
 {
 	MA_WR_STATE(wr_mas, mas, entry);
 
+	mas_lock_check(mas);
 	if (mas->store_type == wr_store_root) {
 		mas_wr_prealloc_setup(&wr_mas);
 		goto store;
@@ -4951,6 +5045,7 @@ int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp)
 {
 	MA_WR_STATE(wr_mas, mas, entry);
 
+	mas_may_init_lock_check(mas);
 	mas_wr_prealloc_setup(&wr_mas);
 	mas->store_type = mas_wr_store_type(&wr_mas);
 	mas_prealloc_calc(&wr_mas, entry);
@@ -4997,6 +5092,7 @@ static void mas_may_activate(struct ma_state *mas)
 		mas->status = ma_start;
 	} else {
 		mas->status = ma_active;
+		mas_lock_check(mas);
 	}
 }
 
@@ -5074,6 +5170,7 @@ void *mas_next(struct ma_state *mas, unsigned long max)
 {
 	void *entry = NULL;
 
+	mas_may_init_lock_check(mas);
 	if (mas_next_setup(mas, max, &entry))
 		return entry;
 
@@ -5097,6 +5194,7 @@ void *mas_next_range(struct ma_state *mas, unsigned long max)
 {
 	void *entry = NULL;
 
+	mas_may_init_lock_check(mas);
 	if (mas_next_setup(mas, max, &entry))
 		return entry;
 
@@ -5205,6 +5303,7 @@ void *mas_prev(struct ma_state *mas, unsigned long min)
 {
 	void *entry = NULL;
 
+	mas_may_init_lock_check(mas);
 	if (mas_prev_setup(mas, min, &entry))
 		return entry;
 
@@ -5228,6 +5327,7 @@ void *mas_prev_range(struct ma_state *mas, unsigned long min)
 {
 	void *entry = NULL;
 
+	mas_may_init_lock_check(mas);
 	if (mas_prev_setup(mas, min, &entry))
 		return entry;
 
@@ -5274,6 +5374,7 @@ EXPORT_SYMBOL_GPL(mt_prev);
  */
 void mas_pause(struct ma_state *mas)
 {
+	mas_lock_check(mas);
 	mas->status = ma_pause;
 	mas->node = NULL;
 }
@@ -5382,6 +5483,7 @@ void *mas_find(struct ma_state *mas, unsigned long max)
 {
 	void *entry = NULL;
 
+	mas_may_init_lock_check(mas);
 	if (mas_find_setup(mas, max, &entry))
 		return entry;
 
@@ -5409,6 +5511,7 @@ void *mas_find_range(struct ma_state *mas, unsigned long max)
 {
 	void *entry = NULL;
 
+	mas_may_init_lock_check(mas);
 	if (mas_find_setup(mas, max, &entry))
 		return entry;
 
@@ -5428,7 +5531,6 @@ EXPORT_SYMBOL_GPL(mas_find_range);
 static bool mas_find_rev_setup(struct ma_state *mas, unsigned long min,
 		void **entry)
 {
-
 	switch (mas->status) {
 	case ma_active:
 		goto active;
@@ -5521,6 +5623,7 @@ void *mas_find_rev(struct ma_state *mas, unsigned long min)
 {
 	void *entry = NULL;
 
+	mas_may_init_lock_check(mas);
 	if (mas_find_rev_setup(mas, min, &entry))
 		return entry;
 
@@ -5547,6 +5650,7 @@ void *mas_find_range_rev(struct ma_state *mas, unsigned long min)
 {
 	void *entry = NULL;
 
+	mas_may_init_lock_check(mas);
 	if (mas_find_rev_setup(mas, min, &entry))
 		return entry;
 
@@ -5564,6 +5668,10 @@ EXPORT_SYMBOL_GPL(mas_find_range_rev);
  * Searches for @mas->index, sets @mas->index and @mas->last to the range and
  * erases that range.
  *
+ * Note that erase requires allocations and will use GFP_KERNEL to do so if
+ * necessary.  If the allocation fails, the internal lock will be dropped to
+ * retry.
+ *
  * Return: the entry that was erased or %NULL, @mas->index and @mas->last are updated.
  */
 void *mas_erase(struct ma_state *mas)
@@ -5572,13 +5680,19 @@ void *mas_erase(struct ma_state *mas)
 	unsigned long index = mas->index;
 	MA_WR_STATE(wr_mas, mas, NULL);
 
-	if (!mas_is_active(mas) || !mas_is_start(mas))
-		mas->status = ma_start;
+	/*
+	 * In low memory situations, the allocation is retried with the gfp flag
+	 * GFP_KERNEL.  The internal spinlock is dropped in mas_nomem(), however
+	 * the external lock is not dropped.
+	 */
+	if (mt_external_lock(mas->tree))
+		might_alloc(GFP_KERNEL);
 
+	mas_make_walkable(mas);
 write_retry:
 	entry = mas_state_walk(mas);
 	if (!entry)
-		return NULL;
+		goto out;
 
 	/* Must reset to ensure spanning writes of last slot are detected */
 	mas_reset(mas);
@@ -5589,8 +5703,10 @@ void *mas_erase(struct ma_state *mas)
 		goto write_retry;
 	}
 
-	if (mas_is_err(mas))
+	if (mas_is_err(mas)) {
+		entry = NULL;
 		goto out;
+	}
 
 	mas_wr_store_entry(&wr_mas);
 out:
@@ -5601,10 +5717,11 @@ EXPORT_SYMBOL_GPL(mas_erase);
 
 /**
  * mas_nomem() - Check if there was an error allocating and do the allocation
- * if necessary If there are allocations, then free them.
+ * if necessary.
+ *
  * @mas: The maple state
  * @gfp: The GFP_FLAGS to use for allocations
- * Return: true on allocation, false otherwise.
+ * Return: False on no memory.  True otherwise (partial success as well)
  */
 bool mas_nomem(struct ma_state *mas, gfp_t gfp)
 	__must_hold(mas->tree->ma_lock)
@@ -5612,6 +5729,10 @@ bool mas_nomem(struct ma_state *mas, gfp_t gfp)
 	if (likely(mas->node != MA_ERROR(-ENOMEM)))
 		return false;
 
+	/* Allocations can fail, don't do this. */
+	WARN_ON_ONCE(!gfpflags_allow_blocking(gfp) &&
+		     mt_external_lock(mas->tree));
+
 	if (gfpflags_allow_blocking(gfp) && !mt_external_lock(mas->tree)) {
 		mtree_unlock(mas->tree);
 		mas_alloc_nodes(mas, gfp);
@@ -5620,10 +5741,17 @@ bool mas_nomem(struct ma_state *mas, gfp_t gfp)
 		mas_alloc_nodes(mas, gfp);
 	}
 
-	if (!mas->sheaf && !mas->alloc)
+	/*
+	 * Return false on zero forward progress.  Partial allocations are kept
+	 * so the retry path will attempt to get the rest.  The failure should
+	 * not happen as we try our best to reclaim.  The user would need an
+	 * external lock with a non-blocking gfp in a low memory situation -
+	 * which would have triggered the first warning in this function.
+	 */
+	if (WARN_ON_ONCE(!mas->sheaf && !mas->alloc))
 		return false;
 
-	mas->status = ma_start;
+	mas_reset(mas);
 	return true;
 }
 
@@ -5913,6 +6041,10 @@ EXPORT_SYMBOL(mtree_alloc_rrange);
  * Erasing is the same as a walk to an entry then a store of a NULL to that
  * ENTIRE range.  In fact, it is implemented as such using the advanced API.
  *
+ * Note that erase requires allocations and will use GFP_KERNEL to do so if
+ * necessary.  If the allocation fails, the internal lock will be dropped to
+ * retry.
+ *
  * Return: The entry stored at the @index or %NULL
  */
 void *mtree_erase(struct maple_tree *mt, unsigned long index)
@@ -5922,6 +6054,7 @@ void *mtree_erase(struct maple_tree *mt, unsigned long index)
 	MA_STATE(mas, mt, index, index);
 	trace_ma_op(TP_FCT, &mas);
 
+	might_alloc(GFP_KERNEL);
 	mtree_lock(mt);
 	entry = mas_erase(&mas);
 	mtree_unlock(mt);
@@ -6447,7 +6580,7 @@ static void mt_dump_range(unsigned long min, unsigned long max,
 {
 	static const char spaces[] = "                                ";
 
-	switch(format) {
+	switch (format) {
 	case mt_dump_hex:
 		if (min == max)
 			pr_info("%.*s%lx: ", depth * 2, spaces, min);
@@ -6489,7 +6622,7 @@ static void mt_dump_range64(const struct maple_tree *mt, void *entry,
 
 	pr_cont(" contents: ");
 	for (i = 0; i < MAPLE_RANGE64_SLOTS - 1; i++) {
-		switch(format) {
+		switch (format) {
 		case mt_dump_hex:
 			pr_cont(PTR_FMT " %lX ", node->slot[i], node->pivot[i]);
 			break;
@@ -6517,7 +6650,7 @@ static void mt_dump_range64(const struct maple_tree *mt, void *entry,
 		if (last == max)
 			break;
 		if (last > max) {
-			switch(format) {
+			switch (format) {
 			case mt_dump_hex:
 				pr_err("node " PTR_FMT " last (%lx) > max (%lx) at pivot %d!\n",
 					node, last, max, i);
@@ -6576,7 +6709,7 @@ static void mt_dump_arange64(const struct maple_tree *mt, void *entry,
 		if (last == max)
 			break;
 		if (last > max) {
-			switch(format) {
+			switch (format) {
 			case mt_dump_hex:
 				pr_err("node " PTR_FMT " last (%lx) > max (%lx) at pivot %d!\n",
 					node, last, max, i);
diff --git a/lib/math/div64.c b/lib/math/div64.c
index d1e92ea24fce5..0b10ded09a9bc 100644
--- a/lib/math/div64.c
+++ b/lib/math/div64.c
@@ -158,6 +158,21 @@ u64 div64_u64(u64 dividend, u64 divisor)
 EXPORT_SYMBOL(div64_u64);
 #endif
 
+#ifndef div64_s64_rem
+s64 div64_s64_rem(s64 dividend, s64 divisor, s64 *remainder)
+{
+	s64 quot, t, rem;
+
+	quot = div64_u64_rem(abs(dividend), abs(divisor), (u64 *)&rem);
+	t = dividend >> 63;
+	*remainder = (rem ^ t) - t;
+	t = (dividend ^ divisor) >> 63;
+
+	return (quot ^ t) - t;
+}
+EXPORT_SYMBOL(div64_s64_rem);
+#endif
+
 #ifndef div64_s64
 s64 div64_s64(s64 dividend, s64 divisor)
 {
diff --git a/lib/math/test_mul_u64_u64_div_u64.c b/lib/math/test_mul_u64_u64_div_u64.c
index 338d014f0c733..d12dc05938fb5 100644
--- a/lib/math/test_mul_u64_u64_div_u64.c
+++ b/lib/math/test_mul_u64_u64_div_u64.c
@@ -157,6 +157,7 @@ static void __exit test_exit(void)
 #define __div64_32 __div64_32
 #define div_s64_rem div_s64_rem
 #define div64_u64_rem div64_u64_rem
+#define div64_s64_rem div64_s64_rem
 #define div64_u64 div64_u64
 #define div64_s64 div64_s64
 #define iter_div_u64_rem iter_div_u64_rem
diff --git a/lib/math/tests/Makefile b/lib/math/tests/Makefile
index 13dc96e48408b..85e1ad59f29da 100644
--- a/lib/math/tests/Makefile
+++ b/lib/math/tests/Makefile
@@ -4,5 +4,6 @@ obj-$(CONFIG_GCD_KUNIT_TEST)		+= gcd_kunit.o
 obj-$(CONFIG_INT_LOG_KUNIT_TEST)	+= int_log_kunit.o
 obj-$(CONFIG_INT_POW_KUNIT_TEST)	+= int_pow_kunit.o
 obj-$(CONFIG_INT_SQRT_KUNIT_TEST)	+= int_sqrt_kunit.o
+obj-$(CONFIG_POLYNOMIAL_KUNIT_TEST)	+= polynomial_kunit.o
 obj-$(CONFIG_PRIME_NUMBERS_KUNIT_TEST)	+= prime_numbers_kunit.o
 obj-$(CONFIG_RATIONAL_KUNIT_TEST)	+= rational_kunit.o
diff --git a/lib/math/tests/polynomial_kunit.c b/lib/math/tests/polynomial_kunit.c
new file mode 100644
index 0000000000000..ef443b57fc123
--- /dev/null
+++ b/lib/math/tests/polynomial_kunit.c
@@ -0,0 +1,270 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <kunit/test.h>
+#include <linux/polynomial.h>
+
+struct polynomial_test_param {
+	const struct polynomial *poly;
+	long data;
+	long expected;
+	const char *name;
+};
+
+/* f(x) = 5 */
+static const struct polynomial poly_constant = {
+	.total_divider = 1,
+	.terms = {
+		{0, 5, 1, 1},
+	}
+};
+
+/* f(x) = 2x^2 + 3x + 5 */
+static const struct polynomial poly_simple = {
+	.total_divider = 1,
+	.terms = {
+		{2, 2, 1, 1},
+		{1, 3, 1, 1},
+		{0, 5, 1, 1},
+	}
+};
+
+/* f(x) = -5x + 100 */
+static const struct polynomial poly_negative_coef = {
+	.total_divider = 1,
+	.terms = {
+		{1, -5, 1, 1},
+		{0, 100, 1, 1},
+	}
+};
+
+/* f(x) = (150x + 50) / 10 */
+static const struct polynomial poly_total_divider = {
+	.total_divider = 10,
+	.terms = {
+		{1, 150, 1, 1},
+		{0,  50, 1, 1},
+	}
+};
+
+/*
+ * f(x) = x / 2
+ * divider=2 applied once per multiply: mult_frac(coef, data, 2) = coef*data/2
+ */
+static const struct polynomial poly_step_divider = {
+	.total_divider = 1,
+	.terms = {
+		{1, 1, 2, 1},
+		{0, 0, 1, 1},
+	}
+};
+
+/*
+ * f(x) = (100/500) * x^2 = 0.2 * x^2
+ * Encoded as coef=100, divider=10, divider_leftover=5:
+ *   denom = 10^2 * 5 = 500
+ */
+static const struct polynomial poly_leftover = {
+	.total_divider = 1,
+	.terms = {
+		{2, 100, 10, 5},
+		{0,   0,  1, 1},
+	}
+};
+
+/*
+ * f(x) = 2x^3  (single high-degree term, no constant)
+ * Used to exercise the power loop alone.
+ */
+static const struct polynomial poly_cubic = {
+	.total_divider = 1,
+	.terms = {
+		{3, 2, 1, 1},
+		{0, 0, 1, 1},
+	}
+};
+
+/*
+ * f(x) = 4x + 1  with a zero-coefficient quadratic term.
+ * The deg-2 term contributes nothing regardless of input.
+ */
+static const struct polynomial poly_zero_coef = {
+	.total_divider = 1,
+	.terms = {
+		{2, 0, 1, 1},
+		{1, 4, 1, 1},
+		{0, 1, 1, 1},
+	}
+};
+
+/*
+ * f(x) = 9  with total_divider = 0.
+ * The implementation treats 0 as 1 via `total_divider ?: 1`, so the
+ * result must equal the constant term unchanged.
+ */
+static const struct polynomial poly_zero_total_divider = {
+	.total_divider = 0,
+	.terms = {
+		{0, 9, 1, 1},
+	}
+};
+
+
+static const struct polynomial_test_param test_params[] = {
+	{
+		.poly     = &poly_constant,
+		.data     = 0,
+		.expected = 5,
+		.name     = "Constant polynomial at x=0",
+	},
+	{
+		.poly     = &poly_constant,
+		.data     = 42,
+		.expected = 5,
+		.name     = "Constant polynomial is independent of input",
+	},
+	{
+		.poly     = &poly_simple,
+		.data     = 0,
+		.expected = 5,	/* zero input collapses all power terms */
+		.name     = "Zero input yields constant term only",
+	},
+	{
+		.poly     = &poly_simple,
+		.data     = 10,
+		.expected = 235,	/* 2*100 + 3*10 + 5 */
+		.name     = "Simple quadratic at x=10",
+	},
+	{
+		.poly     = &poly_negative_coef,
+		.data     = 10,
+		.expected = 50,		/* -5*10 + 100 */
+		.name     = "Negative coefficient at x=10",
+	},
+	{
+		.poly     = &poly_negative_coef,
+		.data     = 20,
+		.expected = 0,		/* -5*20 + 100 = 0 */
+		.name     = "Negative coefficient result is zero",
+	},
+	{
+		.poly     = &poly_total_divider,
+		.data     = 3,
+		.expected = 50,		/* (150*3 + 50) / 10 = 500/10 */
+		.name     = "total_divider scales the final sum",
+	},
+	{
+		.poly     = &poly_step_divider,
+		.data     = 100,
+		.expected = 50,		/* 1*100/2 */
+		.name     = "Per-step divider halves input",
+	},
+	{
+		.poly     = &poly_leftover,
+		.data     = 30,
+		.expected = 180,	/* 100*30^2 / (10^2 * 5) = 90000/500 */
+		.name     = "divider_leftover with quadratic term",
+	},
+	/* Boundary: unit and negative-unit input */
+	{
+		/*
+		 * data=1: each mult_frac(tmp, 1, divider) strips one factor of
+		 * divider from coef per degree, so coef is left-shifted right
+		 * until intermediate precision is exhausted.
+		 * 2*1 + 3*1 + 5 = 10
+		 */
+		.poly     = &poly_simple,
+		.data     = 1,
+		.expected = 10,
+		.name     = "Boundary: data=1 (unit input)",
+	},
+	{
+		/*
+		 * data=-1: even degrees produce positive contributions,
+		 * odd degrees produce negative ones.
+		 * 2*(-1)^2 + 3*(-1) + 5 = 2 - 3 + 5 = 4
+		 */
+		.poly     = &poly_simple,
+		.data     = -1,
+		.expected = 4,
+		.name     = "Boundary: data=-1 (negative unit input)",
+	},
+
+	/* Boundary: negative non-trivial input */
+	{
+		/*
+		 * 2*(-3)^2 + 3*(-3) + 5 = 18 - 9 + 5 = 14
+		 * Verifies sign handling for negative data across all degrees.
+		 */
+		.poly     = &poly_simple,
+		.data     = -3,
+		.expected = 14,
+		.name     = "Boundary: negative data with quadratic",
+	},
+
+	/* Boundary: total_divider = 0 is treated as 1 */
+	{
+		.poly     = &poly_zero_total_divider,
+		.data     = 42,
+		.expected = 9,
+		.name     = "Boundary: total_divider=0 defaults to 1",
+	},
+
+	/* Boundary: zero-coefficient high-degree term */
+	{
+		/*
+		 * The deg-2 term has coef=0, so it contributes 0 regardless
+		 * of data. Result: 0 + 4*10 + 1 = 41
+		 */
+		.poly     = &poly_zero_coef,
+		.data     = 10,
+		.expected = 41,
+		.name     = "Boundary: zero-coefficient term is inert",
+	},
+
+	/* Boundary: single high-degree term, no constant */
+	{
+		/* 2 * 5^3 = 250; also verifies the loop terminates on deg-0 */
+		.poly     = &poly_cubic,
+		.data     = 5,
+		.expected = 250,
+		.name     = "Boundary: single cubic term",
+	},
+	{
+		/* 2 * (-2)^3 = -16; odd power preserves sign of negative data */
+		.poly     = &poly_cubic,
+		.data     = -2,
+		.expected = -16,
+		.name     = "Boundary: single cubic term, negative data",
+	},
+
+};
+
+static void get_desc(const struct polynomial_test_param *param, char *desc)
+{
+	strscpy(desc, param->name, KUNIT_PARAM_DESC_SIZE);
+}
+
+KUNIT_ARRAY_PARAM(polynomial, test_params, get_desc);
+
+static void polynomial_calc_test(struct kunit *test)
+{
+	const struct polynomial_test_param *param = test->param_value;
+
+	KUNIT_EXPECT_EQ(test, polynomial_calc(param->poly, param->data),
+			param->expected);
+}
+
+static struct kunit_case polynomial_test_cases[] = {
+	KUNIT_CASE_PARAM(polynomial_calc_test, polynomial_gen_params),
+	{}
+};
+
+static struct kunit_suite polynomial_test_suite = {
+	.name = "math-polynomial",
+	.test_cases = polynomial_test_cases,
+};
+
+kunit_test_suites(&polynomial_test_suite);
+
+MODULE_DESCRIPTION("math.polynomial_calc KUnit test suite");
+MODULE_LICENSE("GPL");
diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c
index 97772e42b9b28..f8d90689af9f5 100644
--- a/lib/percpu-refcount.c
+++ b/lib/percpu-refcount.c
@@ -289,7 +289,7 @@ static void __percpu_ref_switch_mode(struct percpu_ref *ref,
  * @confirm_switch: optional confirmation callback
  *
  * There's no reason to use this function for the usual reference counting.
- * Use percpu_ref_kill[_and_confirm]().
+ * Use percpu_ref_kill() or percpu_ref_kill_and_confirm().
  *
  * Schedule switching of @ref to atomic mode.  All its percpu counts will
  * be collected to the main atomic counter.  On completion, when all CPUs
diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index 976b9bd02a1b5..cb5ca920fddb6 100644
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -455,6 +455,8 @@ static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp,
 		node->slots[0] = (void __rcu *)entry;
 		entry = node_to_entry(node);
 		rcu_assign_pointer(root->xa_head, entry);
+		/* new head may be missed by an in-progress kmemleak scan */
+		kmemleak_transient_leak(node);
 		shift += RADIX_TREE_MAP_SHIFT;
 	} while (shift <= maxshift);
 out:
@@ -495,8 +497,11 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root)
 		if (!node->shift && is_idr(root))
 			break;
 
-		if (radix_tree_is_internal_node(child))
+		if (radix_tree_is_internal_node(child)) {
 			entry_to_node(child)->parent = NULL;
+			/* new head may be missed by an in-progress kmemleak scan */
+			kmemleak_transient_leak(entry_to_node(child));
+		}
 
 		/*
 		 * We don't need rcu_assign_pointer(), since we are simply
diff --git a/lib/raid/Kconfig b/lib/raid/Kconfig
index 978cd6ba08ac6..01f007b2522cf 100644
--- a/lib/raid/Kconfig
+++ b/lib/raid/Kconfig
@@ -29,6 +29,12 @@ config XOR_KUNIT_TEST
 	  This is intended to help people writing architecture-specific
 	  optimized versions.  If unsure, say N.
 
+config XOR_BENCHMARK
+	bool "Benchmark for xor_gen"
+	depends on XOR_KUNIT_TEST
+	help
+	  Include benchmarks in the KUnit test suite for xor_gen.
+
 config RAID6_PQ
 	tristate
 
@@ -54,6 +60,12 @@ config RAID6_PQ_KUNIT_TEST
 	  This is intended to help people writing architecture-specific
 	  optimized versions.  If unsure, say N.
 
+config RAID6_PQ_KUNIT_BENCHMARK
+	bool "Benchmark for RAID6 PQ"
+	depends on RAID6_PQ_KUNIT_TEST
+	help
+	  Include benchmarks in the KUnit test suite for raid P/Q generation.
+
 config RAID6_PQ_BENCHMARK
 	bool "Automatically choose fastest RAID6 PQ functions"
 	depends on RAID6_PQ
diff --git a/lib/raid/raid6/Makefile b/lib/raid/raid6/Makefile
index 038d6c74d1ba1..5cdb7223de2af 100644
--- a/lib/raid/raid6/Makefile
+++ b/lib/raid/raid6/Makefile
@@ -1,5 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 
+CONTEXT_ANALYSIS := y
+
 ccflags-y			+= -I $(src)
 
 ifeq ($(CONFIG_RAID6_PQ_ARCH),y)
diff --git a/lib/raid/raid6/algos.c b/lib/raid/raid6/algos.c
index 6f5c89ab2b17b..011aa9d0eb3c5 100644
--- a/lib/raid/raid6/algos.c
+++ b/lib/raid/raid6/algos.c
@@ -152,40 +152,37 @@ void raid6_recov_datap(int disks, size_t bytes, int faila, void **ptrs)
 }
 EXPORT_SYMBOL_GPL(raid6_recov_datap);
 
-#define RAID6_TIME_JIFFIES_LG2	4
-#define RAID6_TEST_DISKS	8
+#define BENCH_SIZE	SZ_4K
+#define NR_SRCS		8
+#define NR_DISKS	(NR_SRCS + 2)
+#define REPS		800U
 
-static int raid6_choose_gen(void *(*const dptrs)[RAID6_TEST_DISKS],
-		const int disks)
+static int raid6_choose_gen(void *dptrs[NR_DISKS], const int disks)
 {
-	/* work on the second half of the disks */
-	int start = (disks >> 1) - 1, stop = disks - 3;
 	const struct raid6_calls *best = NULL;
 	unsigned long bestgenperf = 0;
 	unsigned int i;
 
 	for (i = 0; i < raid6_nr_algos; i++) {
 		const struct raid6_calls *algo = raid6_algos[i];
-		unsigned long perf = 0, j0, j1;
+		unsigned long perf = 0;
+		u64 t;
+		int i;
 
 		preempt_disable();
-		j0 = jiffies;
-		while ((j1 = jiffies) == j0)
-			cpu_relax();
-		while (time_before(jiffies,
-				    j1 + (1<<RAID6_TIME_JIFFIES_LG2))) {
-			algo->gen_syndrome(disks, PAGE_SIZE, *dptrs);
-			perf++;
-		}
+		t = ktime_get_ns();
+		for (i = 0; i < REPS; i++)
+			algo->gen_syndrome(disks, BENCH_SIZE, dptrs);
+		t = max(ktime_get_ns() - t, 1);
 		preempt_enable();
 
+		/* bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s] */
+		perf = div64_u64((u64)BENCH_SIZE * REPS * NR_SRCS * 1000, t);
 		if (perf > bestgenperf) {
 			bestgenperf = perf;
 			best = algo;
 		}
-		pr_info("raid6: %-8s gen() %5ld MB/s\n", algo->name,
-			(perf * HZ * (disks-2)) >>
-			(20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2));
+		pr_info("raid6: %-8s gen() %5lu MB/s\n", algo->name, perf);
 	}
 
 	if (!best) {
@@ -197,28 +194,24 @@ static int raid6_choose_gen(void *(*const dptrs)[RAID6_TEST_DISKS],
 	static_call_update(raid6_xor_syndrome_impl, best->xor_syndrome);
 
 	pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
-		best->name,
-		(bestgenperf * HZ * (disks - 2)) >>
-		(20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2));
+		best->name, bestgenperf);
 
 	if (best->xor_syndrome) {
-		unsigned long perf = 0, j0, j1;
+		/* work on the second half of the disks */
+		int start = (disks / 2) - 1, stop = disks - 3;
+		u64 t;
 
 		preempt_disable();
-		j0 = jiffies;
-		while ((j1 = jiffies) == j0)
-			cpu_relax();
-		while (time_before(jiffies,
-				   j1 + (1 << RAID6_TIME_JIFFIES_LG2))) {
-			best->xor_syndrome(disks, start, stop,
-					   PAGE_SIZE, *dptrs);
-			perf++;
-		}
+		t = ktime_get_ns();
+		for (i = 0; i < REPS; i++)
+			best->xor_syndrome(disks, start, stop, BENCH_SIZE,
+				dptrs);
+		t = max(ktime_get_ns() - t, 1);
 		preempt_enable();
 
-		pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n",
-			(perf * HZ * (disks - 2)) >>
-			(20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1));
+		pr_info("raid6: .... xor() %llu MB/s, rmw enabled\n",
+			div64_u64((u64)BENCH_SIZE * REPS * NR_SRCS / 2 * 1000,
+				t));
 	}
 
 	return 0;
@@ -230,24 +223,20 @@ static int raid6_choose_gen(void *(*const dptrs)[RAID6_TEST_DISKS],
 
 static int __init raid6_select_algo(void)
 {
-	const int disks = RAID6_TEST_DISKS;
+	const int disks = NR_DISKS;
+	void *dptrs[NR_DISKS];
 	char *disk_ptr, *p;
-	void *dptrs[RAID6_TEST_DISKS];
 	int i, cycle;
 	int error;
 
 	if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK) || raid6_nr_algos == 1) {
 		pr_info("raid6: skipped pq benchmark and selected %s\n",
 			raid6_algos[raid6_nr_algos - 1]->name);
-		static_call_update(raid6_gen_syndrome_impl,
-				raid6_algos[raid6_nr_algos - 1]->gen_syndrome);
-		static_call_update(raid6_xor_syndrome_impl,
-				raid6_algos[raid6_nr_algos - 1]->xor_syndrome);
 		return 0;
 	}
 
 	/* prepare the buffer and fill it circularly with gfmul table */
-	disk_ptr = kmalloc(PAGE_SIZE * RAID6_TEST_DISKS, GFP_KERNEL);
+	disk_ptr = kmalloc_array(NR_DISKS, BENCH_SIZE, GFP_KERNEL);
 	if (!disk_ptr) {
 		pr_err("raid6: Yikes!  No memory available.\n");
 		return -ENOMEM;
@@ -255,19 +244,19 @@ static int __init raid6_select_algo(void)
 
 	p = disk_ptr;
 	for (i = 0; i < disks; i++)
-		dptrs[i] = p + PAGE_SIZE * i;
+		dptrs[i] = p + BENCH_SIZE * i;
 
-	cycle = ((disks - 2) * PAGE_SIZE) / 65536;
+	cycle = ((disks - 2) * BENCH_SIZE) / 65536;
 	for (i = 0; i < cycle; i++) {
 		memcpy(p, raid6_gfmul, 65536);
 		p += 65536;
 	}
 
-	if ((disks - 2) * PAGE_SIZE % 65536)
-		memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536);
+	if ((disks - 2) * BENCH_SIZE % 65536)
+		memcpy(p, raid6_gfmul, (disks - 2) * BENCH_SIZE % 65536);
 
 	/* select raid gen_syndrome function */
-	error = raid6_choose_gen(&dptrs, disks);
+	error = raid6_choose_gen(dptrs, disks);
 
 	kfree(disk_ptr);
 
@@ -329,13 +318,34 @@ static int __init raid6_init(void)
 	static_call_update(raid6_recov_datap_impl, raid6_recov_algo->datap);
 	pr_info("raid6: using %s recovery algorithm\n", raid6_recov_algo->name);
 
+	/*
+	 * Pick the last registered implementation as the temporary default until
+	 * calibration happens.
+	 */
+	static_call_update(raid6_gen_syndrome_impl,
+			raid6_algos[raid6_nr_algos - 1]->gen_syndrome);
+	static_call_update(raid6_xor_syndrome_impl,
+			raid6_algos[raid6_nr_algos - 1]->xor_syndrome);
+
+#ifdef MODULE
 	return raid6_select_algo();
+#else
+	return 0;
+#endif
 }
 
 static void __exit raid6_exit(void)
 {
 }
 
+/*
+ * When built-in we must register the default implementation before md
+ * initializes, but we don't want calibration to run that early as that
+ * would delay the boot process.
+ */
+#ifndef MODULE
+device_initcall(raid6_select_algo);
+#endif
 subsys_initcall(raid6_init);
 module_exit(raid6_exit);
 MODULE_LICENSE("GPL");
diff --git a/lib/raid/raid6/tests/raid6_kunit.c b/lib/raid/raid6/tests/raid6_kunit.c
index 9f3e671a12241..7d0a1ec980013 100644
--- a/lib/raid/raid6/tests/raid6_kunit.c
+++ b/lib/raid/raid6/tests/raid6_kunit.c
@@ -18,6 +18,7 @@ MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
 #define RAID6_KUNIT_MAX_BUFFERS		64 /* Including P and Q */
 #define RAID6_KUNIT_MAX_FAILURES	2
 #define RAID6_KUNIT_MAX_BYTES		PAGE_SIZE
+#define RAID6_KUNIT_ALLOC_BYTES		SZ_16K
 
 static struct rnd_state rng;
 static void *test_buffers[RAID6_KUNIT_MAX_BUFFERS];
@@ -229,6 +230,68 @@ static void raid6_test(struct kunit *test)
 		raid6_test_one(test);
 }
 
+static void raid6_benchmark(struct kunit *test)
+{
+	static const unsigned int nr_to_test[] = {
+		4, 5, 6, 7, 8, 10, 12, 15, 16, 32,
+	};
+	static const unsigned int len_to_test[] = {
+		SZ_4K, SZ_16K,
+	};
+	unsigned int i, j, l;
+	u64 t;
+
+	if (!IS_ENABLED(CONFIG_RAID6_PQ_KUNIT_BENCHMARK))
+		kunit_skip(test, "not enabled");
+
+	/* warm-up */
+	for (i = 0; i < ARRAY_SIZE(nr_to_test); i++) {
+		for (j = 0; j < ARRAY_SIZE(len_to_test); j++) {
+			for (l = 0; l < 10; l++) {
+				raid6_gen_syndrome(nr_to_test[i],
+						len_to_test[j], test_buffers);
+			}
+		}
+	}
+
+	/*
+	 * Preferably this would be a loop over len_to_test, but the kunit
+	 * logging always adds a newline to each logged format string.
+	 */
+	static_assert(ARRAY_SIZE(len_to_test) == 2);
+	kunit_info(test, "          \t%5u bytes\t%5u bytes\n",
+			len_to_test[0], len_to_test[1]);
+
+	for (i = 0; i < ARRAY_SIZE(nr_to_test); i++) {
+		unsigned int nr = nr_to_test[i];
+		u64 speed[ARRAY_SIZE(len_to_test)];
+
+		KUNIT_ASSERT_LE(test, nr, RAID6_KUNIT_MAX_BUFFERS);
+
+		for (j = 0; j < ARRAY_SIZE(len_to_test); j++) {
+			unsigned int len = len_to_test[j];
+			const unsigned long num_iters = 1000;
+
+			KUNIT_ASSERT_GT(test, len, 0);
+			KUNIT_ASSERT_LE(test, len, RAID6_KUNIT_ALLOC_BYTES);
+
+			preempt_disable();
+			t = ktime_get_ns();
+			for (l = 0; l < num_iters; l++)
+				raid6_gen_syndrome(nr_to_test[i],
+						len_to_test[j], test_buffers);
+			t = max(ktime_get_ns() - t, 1);
+			preempt_enable();
+
+			speed[j] = div64_u64((u64)len * num_iters * nr, t);
+		}
+
+		static_assert(ARRAY_SIZE(len_to_test) == 2);
+		kunit_info(test, "%3u disks:\t%5llu  GB/s\t%5llu  GB/s\n",
+				nr, speed[0], speed[1]);
+	}
+}
+
 static const void *raid6_gen_params(struct kunit *test, const void *prev,
 		char *desc)
 {
@@ -256,6 +319,7 @@ static const void *raid6_gen_params(struct kunit *test, const void *prev,
 
 static struct kunit_case raid6_test_cases[] = {
 	KUNIT_CASE_PARAM(raid6_test, raid6_gen_params),
+	KUNIT_CASE(raid6_benchmark),
 	{},
 };
 
@@ -270,7 +334,7 @@ static int raid6_suite_init(struct kunit_suite *suite)
 	 * so that it is immediately followed by a guard page.  This allows
 	 * buffer overreads to be detected, even in assembly code.
 	 */
-	test_buflen = round_up(RAID6_KUNIT_MAX_BYTES, PAGE_SIZE);
+	test_buflen = round_up(RAID6_KUNIT_ALLOC_BYTES, PAGE_SIZE);
 	for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) {
 		test_recov_buffers[i] = vmalloc(test_buflen);
 		if (!test_recov_buffers[i])
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
index e8ecec3c09f9f..9b0fad459cdb6 100644
--- a/lib/raid/xor/Makefile
+++ b/lib/raid/xor/Makefile
@@ -1,5 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 
+CONTEXT_ANALYSIS := y
+
 ccflags-y			+= -I $(src)
 
 obj-$(CONFIG_XOR_BLOCKS)	+= xor.o
diff --git a/lib/raid/xor/tests/Makefile b/lib/raid/xor/tests/Makefile
index 661e8f6ffd1f3..1cce833cd7fd0 100644
--- a/lib/raid/xor/tests/Makefile
+++ b/lib/raid/xor/tests/Makefile
@@ -1,3 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0-only
 
+CONTEXT_ANALYSIS := y
+
 obj-$(CONFIG_XOR_KUNIT_TEST) += xor_kunit.o
diff --git a/lib/raid/xor/tests/xor_kunit.c b/lib/raid/xor/tests/xor_kunit.c
index 0c2a3a420bf94..1fb30b2c7c471 100644
--- a/lib/raid/xor/tests/xor_kunit.c
+++ b/lib/raid/xor/tests/xor_kunit.c
@@ -85,7 +85,7 @@ static void xor_test(struct kunit *test)
 			xor_generate_random_data();
 
 		/*
-		 * If we're not using the entire buffer size, inject randomize
+		 * If we're not using the entire buffer size, inject randomized
 		 * alignment into the buffer.
 		 */
 		max_alignment = XOR_KUNIT_MAX_BYTES - len;
@@ -125,8 +125,70 @@ static void xor_test(struct kunit *test)
 	}
 }
 
+static void xor_benchmark(struct kunit *test)
+{
+	static const unsigned int nr_to_test[] = {
+		4, 5, 6, 7, 8, 10, 12, 15, 16, 32,
+	};
+	static const unsigned int len_to_test[] = {
+		SZ_4K, SZ_16K,
+	};
+	unsigned int i, j, l;
+	u64 t;
+
+	if (!IS_ENABLED(CONFIG_XOR_BENCHMARK))
+		kunit_skip(test, "not enabled");
+
+	/* warm-up */
+	for (i = 0; i < ARRAY_SIZE(nr_to_test); i++) {
+		for (j = 0; j < ARRAY_SIZE(len_to_test); j++) {
+			for (l = 0; l < 10; l++) {
+				xor_gen(test_dest, test_buffers, nr_to_test[i],
+						len_to_test[j]);
+			}
+		}
+	}
+
+	/*
+	 * Preferably this would be a loop over len_to_test, but the kunit
+	 * logging always adds a newline to each logged format string.
+	 */
+	static_assert(ARRAY_SIZE(len_to_test) == 2);
+	kunit_info(test, "          \t%5u bytes\t%5u bytes\n",
+			len_to_test[0], len_to_test[1]);
+
+	for (i = 0; i < ARRAY_SIZE(nr_to_test); i++) {
+		unsigned int nr = nr_to_test[i];
+		u64 speed[ARRAY_SIZE(len_to_test)];
+
+		KUNIT_ASSERT_LE(test, nr, XOR_KUNIT_MAX_BUFFERS);
+
+		for (j = 0; j < ARRAY_SIZE(len_to_test); j++) {
+			unsigned int len = len_to_test[j];
+			const unsigned long num_iters = 1000;
+
+			KUNIT_ASSERT_GT(test, len, 0);
+			KUNIT_ASSERT_LE(test, len, XOR_KUNIT_MAX_BYTES);
+
+			preempt_disable();
+			t = ktime_get_ns();
+			for (l = 0; l < num_iters; l++)
+				xor_gen(test_dest, test_buffers, nr, len);
+			t = max(ktime_get_ns() - t, 1);
+			preempt_enable();
+
+			speed[j] = div64_u64((u64)len * num_iters * nr, t);
+		}
+
+		static_assert(ARRAY_SIZE(len_to_test) == 2);
+		kunit_info(test, "%3u disks:\t%5llu  GB/s\t%5llu  GB/s\n",
+				nr, speed[0], speed[1]);
+	}
+}
+
 static struct kunit_case xor_test_cases[] = {
 	KUNIT_CASE(xor_test),
+	KUNIT_CASE(xor_benchmark),
 	{},
 };
 
diff --git a/lib/raid/xor/xor-core.c b/lib/raid/xor/xor-core.c
index 50931fbf0324e..f9d61b096b117 100644
--- a/lib/raid/xor/xor-core.c
+++ b/lib/raid/xor/xor-core.c
@@ -10,7 +10,6 @@
 #include <linux/gfp.h>
 #include <linux/slab.h>
 #include <linux/raid/xor.h>
-#include <linux/jiffies.h>
 #include <linux/preempt.h>
 #include <linux/static_call.h>
 #include "xor_impl.h"
@@ -73,59 +72,56 @@ void __init xor_force(struct xor_block_template *tmpl)
 	forced_template = tmpl;
 }
 
-#define BENCH_SIZE	4096
+#define BENCH_SIZE	SZ_4K
+#define NR_SRCS		4
 #define REPS		800U
 
-static void __init
-do_xor_speed(struct xor_block_template *tmpl, void *b1, void *b2)
+static void __init do_xor_speed(struct xor_block_template *tmpl, void *dest,
+		void *srcs[NR_SRCS])
 {
-	int speed;
-	unsigned long reps;
-	ktime_t min, start, t0;
-	void *srcs[1] = { b2 };
+	u64 t;
+	int i;
 
 	preempt_disable();
-
-	reps = 0;
-	t0 = ktime_get();
-	/* delay start until time has advanced */
-	while ((start = ktime_get()) == t0)
-		cpu_relax();
-	do {
+	t = ktime_get_ns();
+	for (i = 0; i < REPS; i++) {
 		mb(); /* prevent loop optimization */
-		tmpl->xor_gen(b1, srcs, 1, BENCH_SIZE);
+		tmpl->xor_gen(dest, srcs, NR_SRCS, BENCH_SIZE);
 		mb();
-	} while (reps++ < REPS || (t0 = ktime_get()) == start);
-	min = ktime_sub(t0, start);
-
+	}
+	t = max(ktime_get_ns() - t, 1);
 	preempt_enable();
 
-	// bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s]
-	speed = (1000 * reps * BENCH_SIZE) / (unsigned int)ktime_to_ns(min);
-	tmpl->speed = speed;
+	/* bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s] */
+	tmpl->speed = div64_u64((u64)BENCH_SIZE * REPS * NR_SRCS * 1000, t);
 
-	pr_info("   %-16s: %5d MB/sec\n", tmpl->name, speed);
+	pr_info("   %-16s: %5d MB/sec\n", tmpl->name, tmpl->speed);
 }
 
 static int __init calibrate_xor_blocks(void)
 {
-	void *b1, *b2;
 	struct xor_block_template *f, *fastest;
+	void *srcs[NR_SRCS];
+	void *buf, *dest;
+	int i;
 
 	if (forced_template)
 		return 0;
 
-	b1 = kmalloc(PAGE_SIZE * 4, GFP_KERNEL);
-	if (!b1) {
+	buf = kmalloc(BENCH_SIZE * (NR_SRCS + 1), GFP_KERNEL);
+	if (!buf) {
 		pr_warn("xor: Yikes!  No memory available.\n");
 		return -ENOMEM;
 	}
-	b2 = b1 + 2*PAGE_SIZE + BENCH_SIZE;
+	get_random_bytes(buf, BENCH_SIZE * (NR_SRCS + 1));
+	dest = buf;
+	for (i = 0; i < NR_SRCS; i++)
+		srcs[i] = buf + (i + 1) * BENCH_SIZE;
 
 	pr_info("xor: measuring software checksum speed\n");
 	fastest = template_list;
 	for (f = template_list; f; f = f->next) {
-		do_xor_speed(f, b1, b2);
+		do_xor_speed(f, dest, srcs);
 		if (f->speed > fastest->speed)
 			fastest = f;
 	}
@@ -133,9 +129,10 @@ static int __init calibrate_xor_blocks(void)
 	pr_info("xor: using function: %s (%d MB/sec)\n",
 	       fastest->name, fastest->speed);
 
-	kfree(b1);
+	kfree(buf);
 	return 0;
 }
+#undef NR_SRCS
 
 #ifdef CONFIG_XOR_BLOCKS_ARCH
 #include "xor_arch.h" /* $SRCARCH/xor_arch.h */
diff --git a/lib/random32.c b/lib/random32.c
index 24e7acd9343f6..dad90219c3511 100644
--- a/lib/random32.c
+++ b/lib/random32.c
@@ -41,6 +41,7 @@
 #include <linux/bitops.h>
 #include <linux/slab.h>
 #include <linux/unaligned.h>
+#include <kunit/visibility.h>
 
 /**
  *	prandom_u32_state - seeded pseudo-random number generator.
@@ -92,7 +93,14 @@ void prandom_bytes_state(struct rnd_state *state, void *buf, size_t bytes)
 }
 EXPORT_SYMBOL(prandom_bytes_state);
 
-static void prandom_warmup(struct rnd_state *state)
+/*
+ * Only declared here so that it has a prototype when made
+ * non-static for KUnit testing (avoids -Wmissing-prototypes).
+ */
+#if IS_ENABLED(CONFIG_KUNIT)
+void prandom_warmup(struct rnd_state *state);
+#endif
+VISIBLE_IF_KUNIT void prandom_warmup(struct rnd_state *state)
 {
 	/* Calling RNG ten times to satisfy recurrence condition */
 	prandom_u32_state(state);
@@ -106,6 +114,7 @@ static void prandom_warmup(struct rnd_state *state)
 	prandom_u32_state(state);
 	prandom_u32_state(state);
 }
+EXPORT_SYMBOL_IF_KUNIT(prandom_warmup);
 
 void prandom_seed_full_state(struct rnd_state __percpu *pcpu_state)
 {
@@ -125,176 +134,3 @@ void prandom_seed_full_state(struct rnd_state __percpu *pcpu_state)
 	}
 }
 EXPORT_SYMBOL(prandom_seed_full_state);
-
-#ifdef CONFIG_RANDOM32_SELFTEST
-static struct prandom_test1 {
-	u32 seed;
-	u32 result;
-} test1[] = {
-	{ 1U, 3484351685U },
-	{ 2U, 2623130059U },
-	{ 3U, 3125133893U },
-	{ 4U,  984847254U },
-};
-
-static struct prandom_test2 {
-	u32 seed;
-	u32 iteration;
-	u32 result;
-} test2[] = {
-	/* Test cases against taus113 from GSL library. */
-	{  931557656U, 959U, 2975593782U },
-	{ 1339693295U, 876U, 3887776532U },
-	{ 1545556285U, 961U, 1615538833U },
-	{  601730776U, 723U, 1776162651U },
-	{ 1027516047U, 687U,  511983079U },
-	{  416526298U, 700U,  916156552U },
-	{ 1395522032U, 652U, 2222063676U },
-	{  366221443U, 617U, 2992857763U },
-	{ 1539836965U, 714U, 3783265725U },
-	{  556206671U, 994U,  799626459U },
-	{  684907218U, 799U,  367789491U },
-	{ 2121230701U, 931U, 2115467001U },
-	{ 1668516451U, 644U, 3620590685U },
-	{  768046066U, 883U, 2034077390U },
-	{ 1989159136U, 833U, 1195767305U },
-	{  536585145U, 996U, 3577259204U },
-	{ 1008129373U, 642U, 1478080776U },
-	{ 1740775604U, 939U, 1264980372U },
-	{ 1967883163U, 508U,   10734624U },
-	{ 1923019697U, 730U, 3821419629U },
-	{  442079932U, 560U, 3440032343U },
-	{ 1961302714U, 845U,  841962572U },
-	{ 2030205964U, 962U, 1325144227U },
-	{ 1160407529U, 507U,  240940858U },
-	{  635482502U, 779U, 4200489746U },
-	{ 1252788931U, 699U,  867195434U },
-	{ 1961817131U, 719U,  668237657U },
-	{ 1071468216U, 983U,  917876630U },
-	{ 1281848367U, 932U, 1003100039U },
-	{  582537119U, 780U, 1127273778U },
-	{ 1973672777U, 853U, 1071368872U },
-	{ 1896756996U, 762U, 1127851055U },
-	{  847917054U, 500U, 1717499075U },
-	{ 1240520510U, 951U, 2849576657U },
-	{ 1685071682U, 567U, 1961810396U },
-	{ 1516232129U, 557U,    3173877U },
-	{ 1208118903U, 612U, 1613145022U },
-	{ 1817269927U, 693U, 4279122573U },
-	{ 1510091701U, 717U,  638191229U },
-	{  365916850U, 807U,  600424314U },
-	{  399324359U, 702U, 1803598116U },
-	{ 1318480274U, 779U, 2074237022U },
-	{  697758115U, 840U, 1483639402U },
-	{ 1696507773U, 840U,  577415447U },
-	{ 2081979121U, 981U, 3041486449U },
-	{  955646687U, 742U, 3846494357U },
-	{ 1250683506U, 749U,  836419859U },
-	{  595003102U, 534U,  366794109U },
-	{   47485338U, 558U, 3521120834U },
-	{  619433479U, 610U, 3991783875U },
-	{  704096520U, 518U, 4139493852U },
-	{ 1712224984U, 606U, 2393312003U },
-	{ 1318233152U, 922U, 3880361134U },
-	{  855572992U, 761U, 1472974787U },
-	{   64721421U, 703U,  683860550U },
-	{  678931758U, 840U,  380616043U },
-	{  692711973U, 778U, 1382361947U },
-	{  677703619U, 530U, 2826914161U },
-	{   92393223U, 586U, 1522128471U },
-	{ 1222592920U, 743U, 3466726667U },
-	{  358288986U, 695U, 1091956998U },
-	{ 1935056945U, 958U,  514864477U },
-	{  735675993U, 990U, 1294239989U },
-	{ 1560089402U, 897U, 2238551287U },
-	{   70616361U, 829U,   22483098U },
-	{  368234700U, 731U, 2913875084U },
-	{   20221190U, 879U, 1564152970U },
-	{  539444654U, 682U, 1835141259U },
-	{ 1314987297U, 840U, 1801114136U },
-	{ 2019295544U, 645U, 3286438930U },
-	{  469023838U, 716U, 1637918202U },
-	{ 1843754496U, 653U, 2562092152U },
-	{  400672036U, 809U, 4264212785U },
-	{  404722249U, 965U, 2704116999U },
-	{  600702209U, 758U,  584979986U },
-	{  519953954U, 667U, 2574436237U },
-	{ 1658071126U, 694U, 2214569490U },
-	{  420480037U, 749U, 3430010866U },
-	{  690103647U, 969U, 3700758083U },
-	{ 1029424799U, 937U, 3787746841U },
-	{ 2012608669U, 506U, 3362628973U },
-	{ 1535432887U, 998U,   42610943U },
-	{ 1330635533U, 857U, 3040806504U },
-	{ 1223800550U, 539U, 3954229517U },
-	{ 1322411537U, 680U, 3223250324U },
-	{ 1877847898U, 945U, 2915147143U },
-	{ 1646356099U, 874U,  965988280U },
-	{  805687536U, 744U, 4032277920U },
-	{ 1948093210U, 633U, 1346597684U },
-	{  392609744U, 783U, 1636083295U },
-	{  690241304U, 770U, 1201031298U },
-	{ 1360302965U, 696U, 1665394461U },
-	{ 1220090946U, 780U, 1316922812U },
-	{  447092251U, 500U, 3438743375U },
-	{ 1613868791U, 592U,  828546883U },
-	{  523430951U, 548U, 2552392304U },
-	{  726692899U, 810U, 1656872867U },
-	{ 1364340021U, 836U, 3710513486U },
-	{ 1986257729U, 931U,  935013962U },
-	{  407983964U, 921U,  728767059U },
-};
-
-static void prandom_state_selftest_seed(struct rnd_state *state, u32 seed)
-{
-#define LCG(x)	 ((x) * 69069U)	/* super-duper LCG */
-	state->s1 = __seed(LCG(seed),        2U);
-	state->s2 = __seed(LCG(state->s1),   8U);
-	state->s3 = __seed(LCG(state->s2),  16U);
-	state->s4 = __seed(LCG(state->s3), 128U);
-}
-
-static int __init prandom_state_selftest(void)
-{
-	int i, j, errors = 0, runs = 0;
-	bool error = false;
-
-	for (i = 0; i < ARRAY_SIZE(test1); i++) {
-		struct rnd_state state;
-
-		prandom_state_selftest_seed(&state, test1[i].seed);
-		prandom_warmup(&state);
-
-		if (test1[i].result != prandom_u32_state(&state))
-			error = true;
-	}
-
-	if (error)
-		pr_warn("prandom: seed boundary self test failed\n");
-	else
-		pr_info("prandom: seed boundary self test passed\n");
-
-	for (i = 0; i < ARRAY_SIZE(test2); i++) {
-		struct rnd_state state;
-
-		prandom_state_selftest_seed(&state, test2[i].seed);
-		prandom_warmup(&state);
-
-		for (j = 0; j < test2[i].iteration - 1; j++)
-			prandom_u32_state(&state);
-
-		if (test2[i].result != prandom_u32_state(&state))
-			errors++;
-
-		runs++;
-		cond_resched();
-	}
-
-	if (errors)
-		pr_warn("prandom: %d/%d self tests failed\n", errors, runs);
-	else
-		pr_info("prandom: %d self tests passed\n", runs);
-	return 0;
-}
-core_initcall(prandom_state_selftest);
-#endif
diff --git a/lib/rhashtable.c b/lib/rhashtable.c
index 40cfb38ac919d..40f1c1f2e3387 100644
--- a/lib/rhashtable.c
+++ b/lib/rhashtable.c
@@ -14,6 +14,7 @@
 #include <linux/atomic.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
+#include <linux/irq_work.h>
 #include <linux/log2.h>
 #include <linux/sched.h>
 #include <linux/rculist.h>
@@ -25,6 +26,7 @@
 #include <linux/rhashtable.h>
 #include <linux/err.h>
 #include <linux/export.h>
+#include <linux/workqueue.h>
 
 #define HASH_DEFAULT_SIZE	64UL
 #define HASH_MIN_SIZE		4U
@@ -878,6 +880,7 @@ int rhashtable_walk_start_check(struct rhashtable_iter *iter)
 		iter->walker.tbl = rht_dereference_rcu(ht->tbl, ht);
 		iter->slot = 0;
 		iter->skip = 0;
+		iter->p = NULL;
 		return -EAGAIN;
 	}
 
diff --git a/lib/string.c b/lib/string.c
index a4e8ad23577d8..a3778d5aab4af 100644
--- a/lib/string.c
+++ b/lib/string.c
@@ -821,7 +821,8 @@ void *memchr_inv(const void *start, int c, size_t bytes)
 {
 	u8 value = c;
 	u64 value64;
-	unsigned int words, prefix;
+	size_t words;
+	unsigned int prefix;
 
 	if (bytes <= 16)
 		return check_bytes8(start, value, bytes);
diff --git a/lib/test-kstrtox.c b/lib/test-kstrtox.c
index 811128d0df16f..9255234752db0 100644
--- a/lib/test-kstrtox.c
+++ b/lib/test-kstrtox.c
@@ -709,6 +709,182 @@ static void __init test_kstrtos8_fail(void)
 	TEST_FAIL(kstrtos8, s8, "%hhd", test_s8_fail);
 }
 
+static void __init test_kstrtoudec64_ok(void)
+{
+	DECLARE_TEST_OK(u64, struct test_udec64);
+	static DEFINE_TEST_OK(struct test_udec64, test_udec64_ok) = {
+		/* basic: integer.fraction, exact digits */
+		{"0.0",			1,	0},
+		{"1.5",			1,	15},
+		{"1.234",		3,	1234},
+		{"42.0",		1,	420},
+		/* zero */
+		{"0.0",			1,	0},
+		{"0.000",		3,	0},
+		/* integer only */
+		{"0",			1,	0},
+		{"42",			3,	42000},
+		{"123.",		2,	12300},
+		{"1",			1,	10},
+		/* fractional only (leading dot) */
+		{".5",			1,	5},
+		{".5",			0,	0},
+		{".123",		3,	123},
+		{".001",		3,	1},
+		/* zero padding: fewer fractional digits than scale */
+		{"1.2",			3,	1200},
+		{"1.2",			6,	1200000},
+		{"0.01",		3,	10},
+		{"0.1",			9,	100000000ULL},
+		{"0.01",		9,	10000000},
+		/* truncation: more fractional digits than scale */
+		{"1.23456",		3,	1234},
+		{"3.1415926535",	6,	3141592},
+		{"0.999999999",		3,	999},
+		{"1.99",		1,	19},
+		{"1.234",		0,	1},
+		/* trailing newline */
+		{"1.5\n",		1,	15},
+		{"42\n",		3,	42000},
+		/* plus sign */
+		{"+1.5",		1,	15},
+		{"+.5",			1,	5},
+		/* scale progression */
+		{"1.",			0,	1},
+		{"1.0",			1,	10},
+		{"1.00",		2,	100},
+		{"1.000",		3,	1000},
+		{"1.000000",		6,	1000000},
+		{"1.000000000",		9,	1000000000ULL},
+		/* max limit check */
+		{"18446744073.709551615",	9,	ULLONG_MAX},
+		{"18446744073709.551615",	6,	ULLONG_MAX},
+		{"0.18446744073709551615",	20,	ULLONG_MAX},
+		/* scale > 19: representable when integer part is small */
+		{"0.00000000000000000001",	20,	1},
+		{"0.1",				20,	10000000000000000000ULL},
+		{"0.00000000000000000000001",	23,	1},
+		/* truncation with scale > 19 */
+		{"0.0000000000000000000012345",	23,	123},
+		/* truncation with many excess digits */
+		{"0.00000000000000000000000000000000423",	34,	42},
+		{"1.99999999999999999999999999999999999",	3,	1999},
+	};
+	TEST_OK(kstrtoudec64, u64, "%llu", test_udec64_ok);
+}
+
+static void __init test_kstrtoudec64_fail(void)
+{
+	static DEFINE_TEST_FAIL(test_udec64_fail) = {
+		/* empty / whitespace */
+		{"",		3},
+		{"\n",		3},
+		/* minus sign (unsigned) */
+		{"-1.5",	1},
+		{"-0.5",	1},
+		/* only a decimal point */
+		{".",		3},
+		{".",		0},
+		/* only a sign */
+		{"+",		3},
+		/* non-digit characters */
+		{"abc",		3},
+		{"1.2x",	3},
+		/* leading/trailing space */
+		{" 1.5",	1},
+		{"1.5 ",	1},
+		/* overflow */
+		{"18446744073710.551615",	6},
+		{"99999999999999999999",	1},
+		/* overflow with scale > 19 */
+		{"1.0",				21},
+		{"0.2",				20},
+		{"0.18446744073709551616",	20},
+		{"1",				20},
+	};
+	TEST_FAIL(kstrtoudec64, u64, "%llu", test_udec64_fail);
+}
+
+static void __init test_kstrtodec64_ok(void)
+{
+	DECLARE_TEST_OK(s64, struct test_dec64);
+	static DEFINE_TEST_OK(struct test_dec64, test_dec64_ok) = {
+		/* basic positive */
+		{"0.0",			1,	0},
+		{"1.5",			1,	15},
+		{"1.234",		3,	1234},
+		/* basic negative */
+		{"-1.5",		1,	-15},
+		{"-1.234",		3,	-1234},
+		{"-0.5",		1,	-5},
+		{"-0.001",		3,	-1},
+		/* zero (signed) */
+		{"-0",			1,	0},
+		{"-0.0",		1,	0},
+		{"0.000",		3,	0},
+		/* integer only */
+		{"42",			3,	42000},
+		{"-42",			3,	-42000},
+		/* fractional only */
+		{".5",			1,	5},
+		{"-.5",			1,	-5},
+		/* zero padding */
+		{"1.2",			3,	1200},
+		{"-1.2",		3,	-1200},
+		{"0.01",		3,	10},
+		{"-0.01",		3,	-10},
+		/* truncation */
+		{"1.23456",		3,	1234},
+		{"-1.23456",		3,	-1234},
+		{"0.999999999",		3,	999},
+		{"-0.999999999",	3,	-999},
+		/* trailing newline */
+		{"1.5\n",		1,	15},
+		{"-1.5\n",		1,	-15},
+		/* plus sign */
+		{"+1.5",		1,	15},
+		/* limits */
+		{"9223372036.854775807",	9,	LLONG_MAX},
+		{"-9223372036.854775808",	9,	LLONG_MIN},
+		/* scale > 19 */
+		{"0.0",				20,	0},
+		{"-0.0",			20,	0},
+		{"0.00000000000000000001",	20,	1},
+		{"-0.00000000000000000001",	20,	-1},
+		{"0.009223372036854775807",	21,	LLONG_MAX},
+		{"-0.009223372036854775808",	21,	LLONG_MIN},
+	};
+	TEST_OK(kstrtodec64, s64, "%lld", test_dec64_ok);
+}
+
+static void __init test_kstrtodec64_fail(void)
+{
+	static DEFINE_TEST_FAIL(test_dec64_fail) = {
+		/* empty / whitespace */
+		{"",		3},
+		{"\n",		3},
+		/* no digits after dot */
+		{".",		3},
+		{"-.",		3},
+		/* no digits at all */
+		{"+",		3},
+		{"-",		3},
+		/* non-digit characters */
+		{"abc",		3},
+		{"-1.2x",	3},
+		/* signed overflow */
+		{"9223372036.854775808",	9},
+		{"-9223372036.854775809",	9},
+		{"99999999999999999999",	1},
+		/* signed overflow with scale > 19 */
+		{"0.1",				20},
+		{"-0.1",			20},
+		{"0.09223372036854775808",	20},
+		{"-0.09223372036854775809",	20},
+	};
+	TEST_FAIL(kstrtodec64, s64, "%lld", test_dec64_fail);
+}
+
 static int __init test_kstrtox_init(void)
 {
 	test_kstrtoull_ok();
@@ -735,6 +911,12 @@ static int __init test_kstrtox_init(void)
 	test_kstrtou8_fail();
 	test_kstrtos8_ok();
 	test_kstrtos8_fail();
+
+	test_kstrtoudec64_ok();
+	test_kstrtoudec64_fail();
+	test_kstrtodec64_ok();
+	test_kstrtodec64_fail();
+
 	return -EINVAL;
 }
 module_init(test_kstrtox_init);
diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c
index 69813c10e6c0b..56bd23059b26a 100644
--- a/lib/test_bitmap.c
+++ b/lib/test_bitmap.c
@@ -234,6 +234,43 @@ static void __init test_find_nth_bit(void)
 	}
 }
 
+static void __init
+test_bitmap_find_next_zero_area_off(void)
+{
+	DECLARE_BITMAP(bmap, 192);
+
+	bitmap_set(bmap, 0, 192);
+
+	bitmap_clear(bmap, 0, 8);
+	__clear_bit(50, bmap);
+	bitmap_clear(bmap, 60, 18);
+	__set_bit(69, bmap);
+	__clear_bit(80, bmap);
+	bitmap_clear(bmap, 100, 10);
+	__clear_bit(120, bmap);
+	bitmap_clear(bmap, 145, 8);
+	bitmap_clear(bmap, 160, 32);
+
+	expect_eq_uint(0,
+		bitmap_find_next_zero_area_off(bmap, 192, 0, 8, 0, 0));
+	expect_eq_uint(0,
+		bitmap_find_next_zero_area_off(bmap, 192, 0, 8, 3, 0));
+	expect_eq_uint(163,
+		bitmap_find_next_zero_area_off(bmap, 192, 0, 8, 3, 1));
+	expect_eq_uint(60,
+		bitmap_find_next_zero_area_off(bmap, 192, 1, 8, 0, 0));
+	expect_eq_uint(160,
+		bitmap_find_next_zero_area_off(bmap, 192, 1, 8, 7, 0));
+	expect_eq_uint(60,
+		bitmap_find_next_zero_area_off(bmap, 192, 1, 8, 7, 4));
+	expect_eq_uint(100,
+		bitmap_find_next_zero_area_off(bmap, 192, 0, 10, 0, 0));
+	expect_eq_uint(160,
+		bitmap_find_next_zero_area_off(bmap, 192, 0, 32, 0, 0));
+	expect_eq_uint(1,
+		!!(bitmap_find_next_zero_area_off(bmap, 192, 0, 33, 0, 0) >= 192));
+}
+
 static void __init test_fill_set(void)
 {
 	DECLARE_BITMAP(bmap, 1024);
@@ -392,6 +429,7 @@ static void __init test_bitmap_sg(void)
 
 	/* Scatter/gather relationship */
 	bitmap_zero(bmap_tmp, 100);
+	bitmap_zero(bmap_res, 100);
 	bitmap_gather(bmap_tmp, bmap_scatter, sg_mask, nbits);
 	bitmap_scatter(bmap_res, bmap_tmp, sg_mask, nbits);
 	expect_eq_bitmap(bmap_scatter, bmap_res, 100);
@@ -1559,6 +1597,7 @@ static void __init selftest(void)
 	test_for_each_clear_bitrange_from();
 	test_for_each_set_clump8();
 	test_for_each_set_bit_wrap();
+	test_bitmap_find_next_zero_area_off();
 }
 
 KSTM_MODULE_LOADERS(test_bitmap);
diff --git a/lib/test_fortify/Makefile b/lib/test_fortify/Makefile
index 399cae880e1dc..44cd5df41a810 100644
--- a/lib/test_fortify/Makefile
+++ b/lib/test_fortify/Makefile
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 
 ccflags-y := $(call cc-disable-warning,fortify-source)
+ccflags-y += $(call cc-disable-warning,stringop-overread)
 
 quiet_cmd_test_fortify = TEST    $@
       cmd_test_fortify = $(CONFIG_SHELL) $(src)/test_fortify.sh \
diff --git a/lib/test_hmm.c b/lib/test_hmm.c
index 9c59d1ceb5b57..6205fb313bd05 100644
--- a/lib/test_hmm.c
+++ b/lib/test_hmm.c
@@ -389,6 +389,67 @@ static int dmirror_range_fault(struct dmirror *dmirror,
 	return ret;
 }
 
+static int dmirror_range_fault_unlocked(struct dmirror *dmirror,
+					struct hmm_range *range,
+					unsigned long timeout)
+{
+	int ret;
+
+	while (true) {
+		ret = hmm_range_fault_unlocked_timeout(range, timeout);
+		if (ret)
+			goto out;
+
+		mutex_lock(&dmirror->mutex);
+		if (mmu_interval_read_retry(range->notifier,
+					    range->notifier_seq)) {
+			mutex_unlock(&dmirror->mutex);
+			continue;
+		}
+		break;
+	}
+
+	ret = dmirror_do_fault(dmirror, range);
+
+	mutex_unlock(&dmirror->mutex);
+out:
+	return ret;
+}
+
+static int dmirror_fault_unlocked(struct dmirror *dmirror,
+				  unsigned long start,
+				  unsigned long end, bool write,
+				  unsigned long timeout)
+{
+	struct mm_struct *mm = dmirror->notifier.mm;
+	unsigned long addr;
+	unsigned long pfns[32];
+	struct hmm_range range = {
+		.notifier = &dmirror->notifier,
+		.hmm_pfns = pfns,
+		.pfn_flags_mask = 0,
+		.default_flags =
+			HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0),
+		.dev_private_owner = dmirror->mdevice,
+	};
+	int ret = 0;
+
+	if (!mmget_not_zero(mm))
+		return -EFAULT;
+
+	for (addr = start; addr < end; addr = range.end) {
+		range.start = addr;
+		range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
+
+		ret = dmirror_range_fault_unlocked(dmirror, &range, timeout);
+		if (ret)
+			break;
+	}
+
+	mmput(mm);
+	return ret;
+}
+
 static int dmirror_fault(struct dmirror *dmirror, unsigned long start,
 			 unsigned long end, bool write)
 {
@@ -407,7 +468,7 @@ static int dmirror_fault(struct dmirror *dmirror, unsigned long start,
 
 	/* Since the mm is for the mirrored process, get a reference first. */
 	if (!mmget_not_zero(mm))
-		return 0;
+		return -EFAULT;
 
 	for (addr = start; addr < end; addr = range.end) {
 		range.start = addr;
@@ -488,6 +549,48 @@ static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
 	return ret;
 }
 
+static int dmirror_read_unlocked(struct dmirror *dmirror,
+				 struct hmm_dmirror_cmd *cmd,
+				 unsigned long timeout)
+{
+	struct dmirror_bounce bounce;
+	unsigned long start, end;
+	unsigned long size = cmd->npages << PAGE_SHIFT;
+	int ret;
+
+	start = cmd->addr;
+	end = start + size;
+	if (end < start)
+		return -EINVAL;
+
+	ret = dmirror_bounce_init(&bounce, start, size);
+	if (ret)
+		return ret;
+
+	while (1) {
+		mutex_lock(&dmirror->mutex);
+		ret = dmirror_do_read(dmirror, start, end, &bounce);
+		mutex_unlock(&dmirror->mutex);
+		if (ret != -ENOENT)
+			break;
+
+		start = cmd->addr + (bounce.cpages << PAGE_SHIFT);
+		ret = dmirror_fault_unlocked(dmirror, start, end, false, timeout);
+		if (ret)
+			break;
+		cmd->faults++;
+	}
+
+	if (ret == 0) {
+		if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
+				 bounce.size))
+			ret = -EFAULT;
+	}
+	cmd->cpages = bounce.cpages;
+	dmirror_bounce_fini(&bounce);
+	return ret;
+}
+
 static int dmirror_do_write(struct dmirror *dmirror, unsigned long start,
 			    unsigned long end, struct dmirror_bounce *bounce)
 {
@@ -581,7 +684,7 @@ static int dmirror_allocate_chunk(struct dmirror_device *mdevice,
 		devmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
 		break;
 	case HMM_DMIRROR_MEMORY_DEVICE_COHERENT:
-		devmem->pagemap.range.start = (MINOR(mdevice->cdevice.dev) - 2) ?
+		devmem->pagemap.range.start = (MINOR(mdevice->device.devt) - 2) ?
 							spm_addr_dev0 :
 							spm_addr_dev1;
 		devmem->pagemap.range.end = devmem->pagemap.range.start +
@@ -1572,7 +1675,9 @@ static long dmirror_fops_unlocked_ioctl(struct file *filp,
 		dmirror->flags = cmd.npages;
 		ret = 0;
 		break;
-
+	case HMM_DMIRROR_READ_UNLOCKED:
+		ret = dmirror_read_unlocked(dmirror, &cmd, 0);
+		break;
 	default:
 		return -EINVAL;
 	}
diff --git a/lib/test_hmm_uapi.h b/lib/test_hmm_uapi.h
index f94c6d4573382..ea9b0ec404fb5 100644
--- a/lib/test_hmm_uapi.h
+++ b/lib/test_hmm_uapi.h
@@ -38,6 +38,7 @@ struct hmm_dmirror_cmd {
 #define HMM_DMIRROR_CHECK_EXCLUSIVE	_IOWR('H', 0x06, struct hmm_dmirror_cmd)
 #define HMM_DMIRROR_RELEASE		_IOWR('H', 0x07, struct hmm_dmirror_cmd)
 #define HMM_DMIRROR_FLAGS		_IOWR('H', 0x08, struct hmm_dmirror_cmd)
+#define HMM_DMIRROR_READ_UNLOCKED	_IOWR('H', 0x09, struct hmm_dmirror_cmd)
 
 #define HMM_DMIRROR_FLAG_FAIL_ALLOC	(1ULL << 0)
 
diff --git a/lib/test_ida.c b/lib/test_ida.c
index 63078f8dc13f5..c400b24f88b61 100644
--- a/lib/test_ida.c
+++ b/lib/test_ida.c
@@ -256,6 +256,20 @@ static void ida_check_find_first(struct ida *ida)
 	ida_free(ida, (1 << 20) - 1);
 
 	IDA_BUG_ON(ida, !ida_is_empty(ida));
+
+	/*
+	 * Test cross-chunk search.
+	 * Allocate ID in chunk 0 and ID in chunk 1.
+	 * Search for ID >= 1. min=1 maps to chunk 0. Chunk 0 has no IDs >= 1.
+	 * It should continue to chunk 1 and return 1024.
+	 */
+	IDA_BUG_ON(ida, ida_alloc_min(ida, 0, GFP_KERNEL) != 0);
+	IDA_BUG_ON(ida, ida_alloc_min(ida, 1024, GFP_KERNEL) != 1024);
+	IDA_BUG_ON(ida, ida_find_first_range(ida, 1, INT_MAX) != 1024);
+	ida_free(ida, 0);
+	ida_free(ida, 1024);
+
+	IDA_BUG_ON(ida, !ida_is_empty(ida));
 }
 
 static DEFINE_IDA(ida);
diff --git a/lib/tests/Makefile b/lib/tests/Makefile
index 4ead57602eac4..3cac3b63a7522 100644
--- a/lib/tests/Makefile
+++ b/lib/tests/Makefile
@@ -41,8 +41,7 @@ obj-$(CONFIG_MEMCPY_KUNIT_TEST) += memcpy_kunit.o
 obj-$(CONFIG_MIN_HEAP_KUNIT_TEST) += min_heap_kunit.o
 CFLAGS_overflow_kunit.o = $(call cc-disable-warning, tautological-constant-out-of-range-compare)
 obj-$(CONFIG_OVERFLOW_KUNIT_TEST) += overflow_kunit.o
-# GCC < 12.1 can miscompile errptr() test when branch profiling is enabled.
-CFLAGS_printf_kunit.o += -DDISABLE_BRANCH_PROFILING
+obj-$(CONFIG_PRANDOM_KUNIT_TEST) += random32_kunit.o
 obj-$(CONFIG_PRINTF_KUNIT_TEST) += printf_kunit.o
 obj-$(CONFIG_RANDSTRUCT_KUNIT_TEST) += randstruct_kunit.o
 obj-$(CONFIG_SCANF_KUNIT_TEST) += scanf_kunit.o
diff --git a/lib/tests/random32_kunit.c b/lib/tests/random32_kunit.c
new file mode 100644
index 0000000000000..0b4af2b09c011
--- /dev/null
+++ b/lib/tests/random32_kunit.c
@@ -0,0 +1,182 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Test cases for random32 functions.
+ */
+
+#include <linux/prandom.h>
+#include <kunit/test.h>
+
+/* prandom_warmup() is static in lib/random32.c; exposed for testing only. */
+void prandom_warmup(struct rnd_state *state);
+
+static const struct prandom_test1 {
+	u32 seed;
+	u32 result;
+} test1[] = {
+	{ 1U, 3484351685U },
+	{ 2U, 2623130059U },
+	{ 3U, 3125133893U },
+	{ 4U,  984847254U },
+};
+
+static const struct prandom_test2 {
+	u32 seed;
+	u32 iteration;
+	u32 result;
+} test2[] = {
+	/* Test cases against taus113 from GSL library. */
+	{  931557656U, 959U, 2975593782U },
+	{ 1339693295U, 876U, 3887776532U },
+	{ 1545556285U, 961U, 1615538833U },
+	{  601730776U, 723U, 1776162651U },
+	{ 1027516047U, 687U,  511983079U },
+	{  416526298U, 700U,  916156552U },
+	{ 1395522032U, 652U, 2222063676U },
+	{  366221443U, 617U, 2992857763U },
+	{ 1539836965U, 714U, 3783265725U },
+	{  556206671U, 994U,  799626459U },
+	{  684907218U, 799U,  367789491U },
+	{ 2121230701U, 931U, 2115467001U },
+	{ 1668516451U, 644U, 3620590685U },
+	{  768046066U, 883U, 2034077390U },
+	{ 1989159136U, 833U, 1195767305U },
+	{  536585145U, 996U, 3577259204U },
+	{ 1008129373U, 642U, 1478080776U },
+	{ 1740775604U, 939U, 1264980372U },
+	{ 1967883163U, 508U,   10734624U },
+	{ 1923019697U, 730U, 3821419629U },
+	{  442079932U, 560U, 3440032343U },
+	{ 1961302714U, 845U,  841962572U },
+	{ 2030205964U, 962U, 1325144227U },
+	{ 1160407529U, 507U,  240940858U },
+	{  635482502U, 779U, 4200489746U },
+	{ 1252788931U, 699U,  867195434U },
+	{ 1961817131U, 719U,  668237657U },
+	{ 1071468216U, 983U,  917876630U },
+	{ 1281848367U, 932U, 1003100039U },
+	{  582537119U, 780U, 1127273778U },
+	{ 1973672777U, 853U, 1071368872U },
+	{ 1896756996U, 762U, 1127851055U },
+	{  847917054U, 500U, 1717499075U },
+	{ 1240520510U, 951U, 2849576657U },
+	{ 1685071682U, 567U, 1961810396U },
+	{ 1516232129U, 557U,    3173877U },
+	{ 1208118903U, 612U, 1613145022U },
+	{ 1817269927U, 693U, 4279122573U },
+	{ 1510091701U, 717U,  638191229U },
+	{  365916850U, 807U,  600424314U },
+	{  399324359U, 702U, 1803598116U },
+	{ 1318480274U, 779U, 2074237022U },
+	{  697758115U, 840U, 1483639402U },
+	{ 1696507773U, 840U,  577415447U },
+	{ 2081979121U, 981U, 3041486449U },
+	{  955646687U, 742U, 3846494357U },
+	{ 1250683506U, 749U,  836419859U },
+	{  595003102U, 534U,  366794109U },
+	{   47485338U, 558U, 3521120834U },
+	{  619433479U, 610U, 3991783875U },
+	{  704096520U, 518U, 4139493852U },
+	{ 1712224984U, 606U, 2393312003U },
+	{ 1318233152U, 922U, 3880361134U },
+	{  855572992U, 761U, 1472974787U },
+	{   64721421U, 703U,  683860550U },
+	{  678931758U, 840U,  380616043U },
+	{  692711973U, 778U, 1382361947U },
+	{  677703619U, 530U, 2826914161U },
+	{   92393223U, 586U, 1522128471U },
+	{ 1222592920U, 743U, 3466726667U },
+	{  358288986U, 695U, 1091956998U },
+	{ 1935056945U, 958U,  514864477U },
+	{  735675993U, 990U, 1294239989U },
+	{ 1560089402U, 897U, 2238551287U },
+	{   70616361U, 829U,   22483098U },
+	{  368234700U, 731U, 2913875084U },
+	{   20221190U, 879U, 1564152970U },
+	{  539444654U, 682U, 1835141259U },
+	{ 1314987297U, 840U, 1801114136U },
+	{ 2019295544U, 645U, 3286438930U },
+	{  469023838U, 716U, 1637918202U },
+	{ 1843754496U, 653U, 2562092152U },
+	{  400672036U, 809U, 4264212785U },
+	{  404722249U, 965U, 2704116999U },
+	{  600702209U, 758U,  584979986U },
+	{  519953954U, 667U, 2574436237U },
+	{ 1658071126U, 694U, 2214569490U },
+	{  420480037U, 749U, 3430010866U },
+	{  690103647U, 969U, 3700758083U },
+	{ 1029424799U, 937U, 3787746841U },
+	{ 2012608669U, 506U, 3362628973U },
+	{ 1535432887U, 998U,   42610943U },
+	{ 1330635533U, 857U, 3040806504U },
+	{ 1223800550U, 539U, 3954229517U },
+	{ 1322411537U, 680U, 3223250324U },
+	{ 1877847898U, 945U, 2915147143U },
+	{ 1646356099U, 874U,  965988280U },
+	{  805687536U, 744U, 4032277920U },
+	{ 1948093210U, 633U, 1346597684U },
+	{  392609744U, 783U, 1636083295U },
+	{  690241304U, 770U, 1201031298U },
+	{ 1360302965U, 696U, 1665394461U },
+	{ 1220090946U, 780U, 1316922812U },
+	{  447092251U, 500U, 3438743375U },
+	{ 1613868791U, 592U,  828546883U },
+	{  523430951U, 548U, 2552392304U },
+	{  726692899U, 810U, 1656872867U },
+	{ 1364340021U, 836U, 3710513486U },
+	{ 1986257729U, 931U,  935013962U },
+	{  407983964U, 921U,  728767059U },
+};
+
+static void prandom_state_test_seed(struct rnd_state *state, u32 seed)
+{
+#define LCG(x)	 ((x) * 69069U)	/* super-duper LCG */
+	state->s1 = __seed(LCG(seed),        2U);
+	state->s2 = __seed(LCG(state->s1),   8U);
+	state->s3 = __seed(LCG(state->s2),  16U);
+	state->s4 = __seed(LCG(state->s3), 128U);
+}
+
+static void test_prandom_seed_boundary(struct kunit *test)
+{
+	int i;
+	struct rnd_state state;
+
+	for (i = 0; i < ARRAY_SIZE(test1); i++) {
+		prandom_state_test_seed(&state, test1[i].seed);
+		prandom_warmup(&state);
+		KUNIT_EXPECT_EQ(test, test1[i].result, prandom_u32_state(&state));
+	}
+}
+
+static void test_prandom_taus113(struct kunit *test)
+{
+	int i, j;
+	struct rnd_state state;
+
+	for (i = 0; i < ARRAY_SIZE(test2); i++) {
+		prandom_state_test_seed(&state, test2[i].seed);
+		prandom_warmup(&state);
+
+		for (j = 0; j < test2[i].iteration - 1; j++)
+			prandom_u32_state(&state);
+
+		KUNIT_EXPECT_EQ(test, test2[i].result, prandom_u32_state(&state));
+	}
+}
+
+static struct kunit_case prandom_test_cases[] = {
+	KUNIT_CASE(test_prandom_seed_boundary),
+	KUNIT_CASE(test_prandom_taus113),
+	{}
+};
+
+static struct kunit_suite prandom_test_suite = {
+	.name = "prandom",
+	.test_cases = prandom_test_cases,
+};
+
+kunit_test_suite(prandom_test_suite);
+
+MODULE_DESCRIPTION("KUnit test for prandom");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
diff --git a/lib/tests/test_ratelimit.c b/lib/tests/test_ratelimit.c
index 33cea5f3d28b2..e244f8cd47d74 100644
--- a/lib/tests/test_ratelimit.c
+++ b/lib/tests/test_ratelimit.c
@@ -68,7 +68,6 @@ static void test_ratelimit_smoke(struct kunit *test)
 static struct ratelimit_state stressrl = RATELIMIT_STATE_INIT_FLAGS("stressrl", HZ / 10, 3,
 								    RATELIMIT_MSG_ON_RELEASE);
 
-static int doneflag;
 static const int stress_duration = 2 * HZ;
 
 struct stress_kthread {
@@ -84,9 +83,8 @@ static int test_ratelimit_stress_child(void *arg)
 	struct stress_kthread *sktp = arg;
 
 	set_user_nice(current, MAX_NICE);
-	WARN_ON_ONCE(!sktp->tp);
 
-	while (!READ_ONCE(doneflag)) {
+	while (!kthread_should_stop()) {
 		sktp->nattempts++;
 		if (___ratelimit(&stressrl, __func__))
 			sktp->nunlimited++;
@@ -105,26 +103,37 @@ static void test_ratelimit_stress(struct kunit *test)
 	const int n_stress_kthread = cpumask_weight(cpu_online_mask);
 	struct stress_kthread skt = { 0 };
 	struct stress_kthread *sktp = kzalloc_objs(*sktp, n_stress_kthread);
+	int n_started = 0;
 
-	KUNIT_EXPECT_NOT_NULL_MSG(test, sktp, "Memory allocation failure");
+	KUNIT_ASSERT_NOT_NULL_MSG(test, sktp, "Memory allocation failure");
 	for (i = 0; i < n_stress_kthread; i++) {
 		sktp[i].tp = kthread_run(test_ratelimit_stress_child, &sktp[i], "%s/%i",
 					 "test_ratelimit_stress_child", i);
-		KUNIT_EXPECT_NOT_NULL_MSG(test, sktp, "kthread creation failure");
+		if (IS_ERR(sktp[i].tp)) {
+			KUNIT_FAIL(test, "kthread_run failed: %ld", PTR_ERR(sktp[i].tp));
+			goto out_stop;
+		}
+		n_started++;
 		pr_alert("Spawned test_ratelimit_stress_child %d\n", i);
 	}
 	schedule_timeout_idle(stress_duration);
-	WRITE_ONCE(doneflag, 1);
-	for (i = 0; i < n_stress_kthread; i++) {
+
+out_stop:
+	for (i = 0; i < n_started; i++) {
 		kthread_stop(sktp[i].tp);
 		skt.nattempts += sktp[i].nattempts;
 		skt.nunlimited += sktp[i].nunlimited;
 		skt.nlimited += sktp[i].nlimited;
 		skt.nmissed += sktp[i].nmissed;
 	}
-	KUNIT_ASSERT_EQ_MSG(test, skt.nunlimited + skt.nlimited, skt.nattempts,
-			    "Outcomes not equal to attempts");
-	KUNIT_ASSERT_EQ_MSG(test, skt.nlimited, skt.nmissed, "Misses not equal to limits");
+	if (n_started == n_stress_kthread) {
+		KUNIT_ASSERT_EQ_MSG(test, skt.nunlimited + skt.nlimited, skt.nattempts,
+				    "Outcomes not equal to attempts");
+		KUNIT_ASSERT_EQ_MSG(test, skt.nlimited, skt.nmissed,
+				    "Misses not equal to limits");
+	}
+
+	kfree(sktp);
 }
 
 static struct kunit_case ratelimit_test_cases[] = {
diff --git a/lib/vdso/Kconfig b/lib/vdso/Kconfig
index db87ba34ef192..597f5f0f96819 100644
--- a/lib/vdso/Kconfig
+++ b/lib/vdso/Kconfig
@@ -1,12 +1,11 @@
 # SPDX-License-Identifier: GPL-2.0
 
-config HAVE_GENERIC_VDSO
+config VDSO_DATASTORE
 	bool
 
-if HAVE_GENERIC_VDSO
-
 config GENERIC_GETTIMEOFDAY
 	bool
+	select VDSO_DATASTORE
 	help
 	  This is a generic implementation of gettimeofday vdso.
 	  Each architecture that enables this feature has to
@@ -21,7 +20,6 @@ config GENERIC_VDSO_OVERFLOW_PROTECT
 
 config VDSO_GETRANDOM
 	bool
+	select VDSO_DATASTORE
 	help
 	  Selected by architectures that support vDSO getrandom().
-
-endif
diff --git a/lib/vdso/Makefile b/lib/vdso/Makefile
index 405f743253d72..ac304def42d6d 100644
--- a/lib/vdso/Makefile
+++ b/lib/vdso/Makefile
@@ -1,3 +1,3 @@
 # SPDX-License-Identifier: GPL-2.0-only
 
-obj-$(CONFIG_HAVE_GENERIC_VDSO) += datastore.o
+obj-$(CONFIG_VDSO_DATASTORE) += datastore.o
diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c
index 17d37b82ebc68..1426bf4e0c127 100644
--- a/lib/vdso/datastore.c
+++ b/lib/vdso/datastore.c
@@ -29,10 +29,11 @@ struct vdso_arch_data *vdso_k_arch_data __ro_after_init =
 	(void *)&vdso_initdata[VDSO_ARCH_PAGES_START * PAGE_SIZE];
 #endif /* CONFIG_ARCH_HAS_VDSO_ARCH_DATA */
 
+static struct page *vdso_data_pages __ro_after_init;
+
 void __init vdso_setup_data_pages(void)
 {
 	unsigned int order = get_order(VDSO_NR_PAGES * PAGE_SIZE);
-	struct page *pages;
 
 	/*
 	 * Allocate the data pages dynamically. SPARC does not support mapping
@@ -42,24 +43,24 @@ void __init vdso_setup_data_pages(void)
 	 * Do not use folios. In time namespaces the pages are mapped in a different order
 	 * to userspace, which is not handled by the folio optimizations in finish_fault().
 	 */
-	pages = alloc_pages(GFP_KERNEL, order);
-	if (!pages)
+	vdso_data_pages = alloc_pages(GFP_KERNEL, order);
+	if (!vdso_data_pages)
 		panic("Unable to allocate VDSO storage pages");
 
 	/* The pages are mapped one-by-one into userspace and each one needs to be refcounted. */
-	split_page(pages, order);
+	split_page(vdso_data_pages, order);
 
 	/* Move the data already written by other subsystems to the new pages */
-	memcpy(page_address(pages), vdso_initdata, VDSO_NR_PAGES * PAGE_SIZE);
+	memcpy(page_address(vdso_data_pages), vdso_initdata, VDSO_NR_PAGES * PAGE_SIZE);
 
 	if (IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY))
-		vdso_k_time_data = page_address(pages + VDSO_TIME_PAGE_OFFSET);
+		vdso_k_time_data = page_address(vdso_data_pages + VDSO_TIME_PAGE_OFFSET);
 
 	if (IS_ENABLED(CONFIG_VDSO_GETRANDOM))
-		vdso_k_rng_data = page_address(pages + VDSO_RNG_PAGE_OFFSET);
+		vdso_k_rng_data = page_address(vdso_data_pages + VDSO_RNG_PAGE_OFFSET);
 
 	if (IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA))
-		vdso_k_arch_data = page_address(pages + VDSO_ARCH_PAGES_START);
+		vdso_k_arch_data = page_address(vdso_data_pages + VDSO_ARCH_PAGES_START);
 }
 
 static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
@@ -67,27 +68,28 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
 {
 	struct page *page, *timens_page;
 
+	if (unlikely(vmf->flags & FAULT_FLAG_REMOTE))
+		return VM_FAULT_SIGBUS;
+
+	page = vdso_data_pages + vmf->pgoff;
 	timens_page = find_timens_vvar_page(vma);
 
 	switch (vmf->pgoff) {
 	case VDSO_TIME_PAGE_OFFSET:
-		if (!IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY))
-			return VM_FAULT_SIGBUS;
-		page = virt_to_page(vdso_k_time_data);
-		if (timens_page) {
-			/*
-			 * Fault in VVAR page too, since it will be accessed
-			 * to get clock data anyway.
-			 */
-			unsigned long addr;
-			vm_fault_t err;
-
-			addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE;
-			err = vmf_insert_page(vma, addr, page);
-			if (unlikely(err & VM_FAULT_ERROR))
-				return err;
-			page = timens_page;
-		}
+		if (!IS_ENABLED(CONFIG_GENERIC_GETTIMEOFDAY) || !timens_page)
+			break;
+		/*
+		 * Fault in VVAR page too, since it will be accessed
+		 * to get clock data anyway.
+		 */
+		unsigned long addr;
+		vm_fault_t err;
+
+		addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE;
+		err = vmf_insert_page(vma, addr, page);
+		if (unlikely(err & VM_FAULT_ERROR))
+			return err;
+		page = timens_page;
 		break;
 	case VDSO_TIMENS_PAGE_OFFSET:
 		/*
@@ -98,18 +100,11 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
 		 * See also the comment near timens_setup_vdso_data().
 		 */
 		if (!IS_ENABLED(CONFIG_TIME_NS) || !timens_page)
-			return VM_FAULT_SIGBUS;
-		page = virt_to_page(vdso_k_time_data);
+			break;
+		page = vdso_data_pages + VDSO_TIME_PAGE_OFFSET;
 		break;
 	case VDSO_RNG_PAGE_OFFSET:
-		if (!IS_ENABLED(CONFIG_VDSO_GETRANDOM))
-			return VM_FAULT_SIGBUS;
-		page = virt_to_page(vdso_k_rng_data);
-		break;
 	case VDSO_ARCH_PAGES_START ... VDSO_ARCH_PAGES_END:
-		if (!IS_ENABLED(CONFIG_ARCH_HAS_VDSO_ARCH_DATA))
-			return VM_FAULT_SIGBUS;
-		page = virt_to_page(vdso_k_arch_data) + vmf->pgoff - VDSO_ARCH_PAGES_START;
 		break;
 	default:
 		return VM_FAULT_SIGBUS;
@@ -128,7 +123,7 @@ const struct vm_special_mapping vdso_vvar_mapping = {
 struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr)
 {
 	return _install_special_mapping(mm, addr, VDSO_NR_PAGES * PAGE_SIZE,
-					VM_READ | VM_MAYREAD | VM_IO | VM_DONTDUMP |
+					VM_READ | VM_MAYREAD | VM_DONTDUMP |
 					VM_MIXEDMAP | VM_SEALED_SYSMAP,
 					&vdso_vvar_mapping);
 }
diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c
index e0f289d3d1106..f7a591aba59f1 100644
--- a/lib/vdso/gettimeofday.c
+++ b/lib/vdso/gettimeofday.c
@@ -12,6 +12,8 @@
 #include <vdso/time32.h>
 #include <vdso/time64.h>
 
+#include <uapi/linux/unistd.h>
+
 /*
  * The generic vDSO implementation requires that gettimeofday.h
  * provides:
@@ -23,6 +25,8 @@
  */
 #include <asm/vdso/gettimeofday.h>
 
+#include <linux/build_bug.h>
+
 /* Bring in default accessors */
 #include <vdso/vsyscall.h>
 
@@ -323,6 +327,8 @@ __cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock,
 	struct __kernel_timespec ts;
 	bool ok;
 
+	BUILD_BUG_ON(!IS_ENABLED(CONFIG_COMPAT_32BIT_TIME));
+
 	ok = __cvdso_clock_gettime_common(vd, clock, &ts);
 
 	if (unlikely(!ok))
@@ -348,6 +354,12 @@ __cvdso_gettimeofday_data(const struct vdso_time_data *vd,
 {
 	const struct vdso_clock *vc = vd->clock_data;
 
+#ifndef __NR_gettimeofday
+	BUILD_BUG();
+#endif
+
+	BUILD_BUG_ON(sizeof(tv->tv_sec) != 8 && !IS_ENABLED(CONFIG_COMPAT_32BIT_TIME));
+
 	if (likely(tv != NULL)) {
 		struct __kernel_timespec ts;
 
@@ -382,6 +394,12 @@ __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time)
 	const struct vdso_clock *vc = vd->clock_data;
 	__kernel_old_time_t t;
 
+#ifndef __NR_time
+	BUILD_BUG();
+#endif
+
+	BUILD_BUG_ON(sizeof(*time) != 8 && !IS_ENABLED(CONFIG_COMPAT_32BIT_TIME));
+
 	if (vdso_is_timens_clock(vc)) {
 		vd = vdso_timens_data(vd);
 		vc = vd->clock_data;
@@ -471,6 +489,8 @@ __cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t cloc
 	struct __kernel_timespec ts;
 	bool ok;
 
+	BUILD_BUG_ON(!IS_ENABLED(CONFIG_COMPAT_32BIT_TIME));
+
 	ok = __cvdso_clock_getres_common(vd, clock, &ts);
 
 	if (unlikely(!ok))
diff --git a/lib/vsprintf.c b/lib/vsprintf.c
index 2bc6ef483576c..1d95ab9314bf7 100644
--- a/lib/vsprintf.c
+++ b/lib/vsprintf.c
@@ -86,7 +86,7 @@ static unsigned long long simple_strntoull(const char *startp, char **endp, unsi
 	cp = _parse_integer_fixup_radix(startp, &base);
 	prefix_chars = cp - startp;
 	if (prefix_chars < max_chars) {
-		rv = _parse_integer_limit(cp, base, &result, max_chars - prefix_chars);
+		rv = _parse_integer(cp, base, &result, max_chars - prefix_chars);
 		/* FIXME */
 		cp += (rv & ~KSTRTOX_OVERFLOW);
 	} else {
@@ -2360,6 +2360,9 @@ void __init hash_pointers_finalize(bool slub_debug)
 
 static int __init hash_pointers_mode_parse(char *str)
 {
+	/* Avoid stale no_hash_pointers state when hash_pointers overrides it */
+	no_hash_pointers = false;
+
 	if (!str) {
 		pr_warn("Hash pointers mode empty; falling back to auto.\n");
 		hash_pointers_mode = HASH_PTR_AUTO;
@@ -2369,6 +2372,7 @@ static int __init hash_pointers_mode_parse(char *str)
 	} else if (strcmp(str, "never") == 0) {
 		pr_info("Hash pointers mode set to never.\n");
 		hash_pointers_mode = HASH_PTR_NEVER;
+		no_hash_pointers = true;
 	} else if (strcmp(str, "always") == 0) {
 		pr_info("Hash pointers mode set to always.\n");
 		hash_pointers_mode = HASH_PTR_ALWAYS;
diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c
index cc49a300a5b2f..88922323f96e1 100644
--- a/lib/xz/xz_dec_bcj.c
+++ b/lib/xz/xz_dec_bcj.c
@@ -466,7 +466,7 @@ static void bcj_flush(struct xz_dec_bcj *s, struct xz_buf *b)
 {
 	size_t copy_size;
 
-	copy_size = min_t(size_t, s->temp.filtered, b->out_size - b->out_pos);
+	copy_size = min(s->temp.filtered, b->out_size - b->out_pos);
 	memcpy(b->out + b->out_pos, s->temp.buf, copy_size);
 	b->out_pos += copy_size;
 
diff --git a/lib/xz/xz_dec_lzma2.c b/lib/xz/xz_dec_lzma2.c
index 4b783ac94e713..46c2df6ad6f5e 100644
--- a/lib/xz/xz_dec_lzma2.c
+++ b/lib/xz/xz_dec_lzma2.c
@@ -135,14 +135,16 @@ struct lzma_dec {
 	uint32_t rep2;
 	uint32_t rep3;
 
-	/* Types of the most recently seen LZMA symbols */
-	enum lzma_state state;
-
 	/*
 	 * Length of a match. This is updated so that dict_repeat can
-	 * be called again to finish repeating the whole match.
+	 * be called again to finish repeating the whole match. This is
+	 * size_t because a pointer to this is passed to dict_repeat,
+	 * and there it's nicer to have size_t instead of uint32_t.
 	 */
-	uint32_t len;
+	size_t len;
+
+	/* Types of the most recently seen LZMA symbols */
+	enum lzma_state state;
 
 	/*
 	 * LZMA properties or related bit masks (number of literal
@@ -228,13 +230,13 @@ struct lzma2_dec {
 	enum lzma2_seq next_sequence;
 
 	/* Uncompressed size of LZMA chunk (2 MiB at maximum) */
-	uint32_t uncompressed;
+	size_t uncompressed;
 
 	/*
 	 * Compressed size of LZMA chunk or compressed/uncompressed
 	 * size of uncompressed chunk (64 KiB at maximum)
 	 */
-	uint32_t compressed;
+	size_t compressed;
 
 	/*
 	 * True if dictionary reset is needed. This is false before
@@ -269,11 +271,11 @@ struct xz_dec_lzma2 {
 	struct lzma_dec lzma;
 
 	/*
-	 * Temporary buffer which holds small number of input bytes between
+	 * Temporary buffer which holds a small number of input bytes between
 	 * decoder calls. See lzma2_lzma() for details.
 	 */
 	struct {
-		uint32_t size;
+		size_t size;
 		uint8_t buf[3 * LZMA_IN_REQUIRED];
 	} temp;
 };
@@ -320,7 +322,7 @@ static inline bool dict_has_space(const struct dictionary *dict)
  * still empty. This special case is needed for single-call decoding to
  * avoid writing a '\0' to the end of the destination buffer.
  */
-static inline uint32_t dict_get(const struct dictionary *dict, uint32_t dist)
+static inline uint32_t dict_get(const struct dictionary *dict, size_t dist)
 {
 	size_t offset = dict->pos - dist - 1;
 
@@ -346,15 +348,15 @@ static inline void dict_put(struct dictionary *dict, uint8_t byte)
  * invalid, false is returned. On success, true is returned and *len is
  * updated to indicate how many bytes were left to be repeated.
  */
-static bool dict_repeat(struct dictionary *dict, uint32_t *len, uint32_t dist)
+static bool dict_repeat(struct dictionary *dict, size_t *len, size_t dist)
 {
 	size_t back;
-	uint32_t left;
+	size_t left;
 
 	if (dist >= dict->full || dist >= dict->size)
 		return false;
 
-	left = min_t(size_t, dict->limit - dict->pos, *len);
+	left = min(dict->limit - dict->pos, *len);
 	*len -= left;
 
 	back = dict->pos - dist - 1;
@@ -375,7 +377,7 @@ static bool dict_repeat(struct dictionary *dict, uint32_t *len, uint32_t dist)
 
 /* Copy uncompressed data as is from input to dictionary and output buffers. */
 static void dict_uncompressed(struct dictionary *dict, struct xz_buf *b,
-			      uint32_t *left)
+			      size_t *left)
 {
 	size_t copy_size;
 
@@ -433,7 +435,7 @@ static void dict_uncompressed(struct dictionary *dict, struct xz_buf *b,
  * enough space in b->out. This is guaranteed because caller uses dict_limit()
  * before decoding data into the dictionary.
  */
-static uint32_t dict_flush(struct dictionary *dict, struct xz_buf *b)
+static size_t dict_flush(struct dictionary *dict, struct xz_buf *b)
 {
 	size_t copy_size = dict->pos - dict->start;
 
@@ -755,8 +757,8 @@ static bool lzma_main(struct xz_dec_lzma2 *s)
 	uint32_t pos_state;
 
 	/*
-	 * If the dictionary was reached during the previous call, try to
-	 * finish the possibly pending repeat in the dictionary.
+	 * If the dictionary write limit was reached during the previous call,
+	 * try to finish the possibly pending repeat in the dictionary.
 	 */
 	if (dict_has_space(&s->dict) && s->lzma.len > 0)
 		dict_repeat(&s->dict, &s->lzma.len, s->lzma.rep0);
@@ -878,7 +880,7 @@ static bool lzma_props(struct xz_dec_lzma2 *s, uint8_t props)
 static bool lzma2_lzma(struct xz_dec_lzma2 *s, struct xz_buf *b)
 {
 	size_t in_avail;
-	uint32_t tmp;
+	size_t tmp;
 
 	in_avail = b->in_size - b->in_pos;
 	if (s->temp.size > 0 || s->lzma2.compressed == 0) {
@@ -976,7 +978,7 @@ enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b)
 			 *          an uncompressed chunk
 			 *   0x02   Uncompressed chunk (no dictionary reset)
 			 *
-			 * Highest three bits (s->control & 0xE0):
+			 * Highest three bits (tmp & 0xE0):
 			 *   0xE0   Dictionary reset, new properties and state
 			 *          reset, followed by LZMA compressed chunk
 			 *   0xC0   New properties and state reset, followed
@@ -988,7 +990,7 @@ enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b)
 			 *   0x80   LZMA chunk (no dictionary or state reset)
 			 *
 			 * For LZMA compressed chunks, the lowest five bits
-			 * (s->control & 1F) are the highest bits of the
+			 * (tmp & 1F) are the highest bits of the
 			 * uncompressed size (bits 16-20).
 			 *
 			 * A new LZMA2 stream must begin with a dictionary
@@ -1046,25 +1048,23 @@ enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b)
 
 		case SEQ_UNCOMPRESSED_1:
 			s->lzma2.uncompressed
-					+= (uint32_t)b->in[b->in_pos++] << 8;
+					+= (size_t)b->in[b->in_pos++] << 8;
 			s->lzma2.sequence = SEQ_UNCOMPRESSED_2;
 			break;
 
 		case SEQ_UNCOMPRESSED_2:
 			s->lzma2.uncompressed
-					+= (uint32_t)b->in[b->in_pos++] + 1;
+					+= (size_t)b->in[b->in_pos++] + 1;
 			s->lzma2.sequence = SEQ_COMPRESSED_0;
 			break;
 
 		case SEQ_COMPRESSED_0:
-			s->lzma2.compressed
-					= (uint32_t)b->in[b->in_pos++] << 8;
+			s->lzma2.compressed = (size_t)b->in[b->in_pos++] << 8;
 			s->lzma2.sequence = SEQ_COMPRESSED_1;
 			break;
 
 		case SEQ_COMPRESSED_1:
-			s->lzma2.compressed
-					+= (uint32_t)b->in[b->in_pos++] + 1;
+			s->lzma2.compressed += (size_t)b->in[b->in_pos++] + 1;
 			s->lzma2.sequence = s->lzma2.next_sequence;
 			break;
 
@@ -1091,16 +1091,15 @@ enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s, struct xz_buf *b)
 		case SEQ_LZMA_RUN:
 			/*
 			 * Set dictionary limit to indicate how much we want
-			 * to be encoded at maximum. Decode new data into the
+			 * to be decoded at maximum. Decode new data into the
 			 * dictionary. Flush the new data from dictionary to
 			 * b->out. Check if we finished decoding this chunk.
 			 * In case the dictionary got full but we didn't fill
 			 * the output buffer yet, we may run this loop
 			 * multiple times without changing s->lzma2.sequence.
 			 */
-			dict_limit(&s->dict, min_t(size_t,
-					b->out_size - b->out_pos,
-					s->lzma2.uncompressed));
+			dict_limit(&s->dict, min(b->out_size - b->out_pos,
+						 s->lzma2.uncompressed));
 			if (!lzma2_lzma(s, b))
 				return XZ_DATA_ERROR;
 
@@ -1260,8 +1259,8 @@ enum xz_ret xz_dec_microlzma_run(struct xz_dec_microlzma *s_ptr,
 		s->dict.end = b->out_size - b->out_pos;
 
 	while (true) {
-		dict_limit(&s->dict, min_t(size_t, b->out_size - b->out_pos,
-					   s->lzma2.uncompressed));
+		dict_limit(&s->dict, min(b->out_size - b->out_pos,
+					 s->lzma2.uncompressed));
 
 		if (!lzma2_lzma(s, b))
 			return XZ_DATA_ERROR;
diff --git a/lib/xz/xz_dec_stream.c b/lib/xz/xz_dec_stream.c
index 59bfd54ffee78..0bed6daefac23 100644
--- a/lib/xz/xz_dec_stream.c
+++ b/lib/xz/xz_dec_stream.c
@@ -155,8 +155,8 @@ static const uint8_t check_sizes[16] = {
  */
 static bool fill_temp(struct xz_dec *s, struct xz_buf *b)
 {
-	size_t copy_size = min_t(size_t,
-			b->in_size - b->in_pos, s->temp.size - s->temp.pos);
+	size_t copy_size = min(b->in_size - b->in_pos,
+			       s->temp.size - s->temp.pos);
 
 	memcpy(s->temp.buf + s->temp.pos, b->in + b->in_pos, copy_size);
 	b->in_pos += copy_size;
diff --git a/lib/xz/xz_lzma2.h b/lib/xz/xz_lzma2.h
index d2632b7dfb9c8..a612ce4fd450a 100644
--- a/lib/xz/xz_lzma2.h
+++ b/lib/xz/xz_lzma2.h
@@ -143,7 +143,7 @@ static inline bool lzma_state_is_literal(enum lzma_state state)
  * Get the index of the appropriate probability array for decoding
  * the distance slot.
  */
-static inline uint32_t lzma_get_dist_state(uint32_t len)
+static inline size_t lzma_get_dist_state(size_t len)
 {
 	return len < DIST_STATES + MATCH_LEN_MIN
 			? len - MATCH_LEN_MIN : DIST_STATES - 1;
diff --git a/localversion-next b/localversion-next
new file mode 100644
index 0000000000000..590fee5c5e75e
--- /dev/null
+++ b/localversion-next
@@ -0,0 +1 @@
+-next-20260717
diff --git a/mm/Kconfig b/mm/Kconfig
index 9e0ca48249054..3cec14092b289 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -125,8 +125,6 @@ config ZSWAP_COMPRESSOR_DEFAULT
 config ZSMALLOC
 	tristate
 
-if ZSMALLOC
-
 menu "Zsmalloc allocator options"
 	depends on ZSMALLOC
 
@@ -161,8 +159,6 @@ config ZSMALLOC_CHAIN_SIZE
 
 endmenu
 
-endif
-
 menu "Slab allocator options"
 
 config SLUB
@@ -509,13 +505,6 @@ config EXCLUSIVE_SYSTEM_RAM
 	def_bool y
 	depends on !DEVMEM || STRICT_DEVMEM
 
-#
-# Only be set on architectures that have completely implemented memory hotplug
-# feature. If you are not sure, don't touch it.
-#
-config HAVE_BOOTMEM_INFO_NODE
-	def_bool n
-
 config ARCH_ENABLE_MEMORY_HOTPLUG
 	bool
 
@@ -590,13 +579,10 @@ endchoice
 
 config MEMORY_HOTREMOVE
 	bool "Allow for memory hot remove"
-	select HAVE_BOOTMEM_INFO_NODE if X86_64
-	depends on MEMORY_HOTPLUG
 	select MIGRATION
 
 config MHP_MEMMAP_ON_MEMORY
 	def_bool y
-	depends on MEMORY_HOTPLUG && SPARSEMEM_VMEMMAP
 	depends on ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE
 
 endif # MEMORY_HOTPLUG
@@ -703,7 +689,7 @@ config DEVICE_MIGRATION
 config ARCH_ENABLE_HUGEPAGE_MIGRATION
 	bool
 
-config ARCH_ENABLE_THP_MIGRATION
+config ARCH_HAS_PMD_SOFTLEAVES
 	bool
 
 config HUGETLB_PAGE_SIZE_VARIABLE
@@ -1235,9 +1221,7 @@ config ZONE_DMA32
 
 config ZONE_DEVICE
 	bool "Device memory (pmem, HMM, etc...) hotplug support"
-	depends on MEMORY_HOTPLUG
 	depends on MEMORY_HOTREMOVE
-	depends on SPARSEMEM_VMEMMAP
 	select XARRAY_MULTI
 
 	help
@@ -1356,7 +1340,7 @@ config MEMFD_CREATE
 config SECRETMEM
 	default y
 	bool "Enable memfd_secret() system call" if EXPERT
-	depends on ARCH_HAS_SET_DIRECT_MAP
+	depends on ARCH_HAS_SET_DIRECT_MAP && !HIGHMEM
 	help
 	  Enable the memfd_secret() system call with the ability to create
 	  memory areas visible only in the context of the owning process and
@@ -1386,6 +1370,15 @@ config HAVE_ARCH_USERFAULTFD_MINOR
 	help
 	  Arch has userfaultfd minor fault support
 
+config USERFAULTFD_RWP
+	def_bool y
+	depends on 64BIT && ARCH_HAS_PTE_PROTNONE && HAVE_ARCH_USERFAULTFD_WP
+	help
+	  Userfaultfd read-write protection (UFFDIO_RWPROTECT) delivers a
+	  userfaultfd notification on every access -- read or write -- to a
+	  protected range, letting userspace observe the working set of a
+	  process.
+
 menuconfig USERFAULTFD
 	bool "Enable userfaultfd() system call"
 	depends on MMU
@@ -1393,17 +1386,15 @@ menuconfig USERFAULTFD
 	  Enable the userfaultfd() system call that allows to intercept and
 	  handle page faults in userland.
 
-if USERFAULTFD
 config PTE_MARKER_UFFD_WP
 	bool "Userfaultfd write protection support for shmem/hugetlbfs"
 	default y
-	depends on HAVE_ARCH_USERFAULTFD_WP
+	depends on USERFAULTFD && HAVE_ARCH_USERFAULTFD_WP
 
 	help
 	  Allows to create marker PTEs for userfaultfd write protection
 	  purposes.  It is required to enable userfaultfd write protection on
 	  file-backed memory types like shmem and hugetlbfs.
-endif # USERFAULTFD
 
 # multi-gen LRU {
 config LRU_GEN
diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug
index 91b3e027b7535..5737a504efbbe 100644
--- a/mm/Kconfig.debug
+++ b/mm/Kconfig.debug
@@ -320,3 +320,31 @@ config PER_VMA_LOCK_STATS
 	  overhead in the page fault path.
 
 	  If in doubt, say N.
+
+config MEM_ALLOC_PROFILING
+	bool "Enable memory allocation profiling"
+	default n
+	depends on MMU
+	depends on PROC_FS
+	depends on !DEBUG_FORCE_WEAK_PER_CPU
+	select CODE_TAGGING
+	select PAGE_EXTENSION
+	select SLAB_OBJ_EXT
+	help
+	  Track allocation source code and record total allocation size
+	  initiated at that code location. The mechanism can be used to track
+	  memory leaks with a low performance and memory impact.
+
+config MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
+	bool "Enable memory allocation profiling by default"
+	default y
+	depends on MEM_ALLOC_PROFILING
+
+config MEM_ALLOC_PROFILING_DEBUG
+	bool "Memory allocation profiler debugging"
+	default n
+	depends on MEM_ALLOC_PROFILING
+	select MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
+	help
+	  Adds warnings with helpful error messages for memory allocation
+	  profiling.
diff --git a/mm/Makefile b/mm/Makefile
index eff9f9e7e061c..ab37ef428d98d 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -50,7 +50,7 @@ endif
 
 obj-y			:= filemap.o mempool.o oom_kill.o fadvise.o \
 			   maccess.o page-writeback.o folio-compat.o \
-			   readahead.o swap.o truncate.o vmscan.o shrinker.o \
+			   readahead.o folio.o truncate.o vmscan.o shrinker.o \
 			   shmem.o util.o mmzone.o vmstat.o backing-dev.o \
 			   mm_init.o percpu.o slab_common.o \
 			   compaction.o show_mem.o \
@@ -141,9 +141,9 @@ obj-$(CONFIG_MEMFD_CREATE) += memfd.o
 obj-$(CONFIG_MAPPING_DIRTY_HELPERS) += mapping_dirty_helpers.o
 obj-$(CONFIG_PTDUMP) += ptdump.o
 obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o
-obj-$(CONFIG_HAVE_BOOTMEM_INFO_NODE) += bootmem_info.o
 obj-$(CONFIG_GENERIC_IOREMAP) += ioremap.o
 obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o
 obj-$(CONFIG_EXECMEM) += execmem.o
 obj-$(CONFIG_TMPFS_QUOTA) += shmem_quota.o
 obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) += tests/lazy_mmu_mode_kunit.o
+obj-$(CONFIG_MEM_ALLOC_PROFILING) += alloc_tag.o
diff --git a/mm/alloc_tag.c b/mm/alloc_tag.c
new file mode 100644
index 0000000000000..52aece27b00e0
--- /dev/null
+++ b/mm/alloc_tag.c
@@ -0,0 +1,1361 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/alloc_tag.h>
+#include <linux/execmem.h>
+#include <linux/fs.h>
+#include <linux/gfp.h>
+#include <linux/kallsyms.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/compat.h>
+#include <linux/page_ext.h>
+#include <linux/pgalloc_tag.h>
+#include <linux/proc_fs.h>
+#include <linux/rcupdate.h>
+#include <linux/seq_buf.h>
+#include <linux/seq_file.h>
+#include <linux/string_choices.h>
+#include <linux/vmalloc.h>
+#include <linux/kmemleak.h>
+#include <uapi/linux/alloc_tag.h>
+
+#include "internal.h"
+#include "page_alloc.h"
+
+#define ALLOCINFO_FILE_NAME		"allocinfo"
+#define MODULE_ALLOC_TAG_VMAP_SIZE	(100000UL * sizeof(struct alloc_tag))
+#define SECTION_START(NAME)		(CODETAG_SECTION_START_PREFIX NAME)
+#define SECTION_STOP(NAME)		(CODETAG_SECTION_STOP_PREFIX NAME)
+
+#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
+static bool mem_profiling_support = true;
+#else
+static bool mem_profiling_support;
+#endif
+
+/*
+ * Memory allocation profiling is permanently disabled and cannot be enabled.
+ * Must be called after setup_early_mem_profiling().
+ */
+bool mem_alloc_profiling_permanently_disabled(void)
+{
+	return !mem_profiling_support;
+}
+
+static struct codetag_type *alloc_tag_cttype;
+
+#ifdef CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU
+DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
+EXPORT_SYMBOL(_shared_alloc_tag);
+#endif
+
+DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
+			mem_alloc_profiling_key);
+EXPORT_SYMBOL(mem_alloc_profiling_key);
+
+DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed);
+
+struct alloc_tag_kernel_section kernel_tags = { NULL, 0 };
+unsigned long alloc_tag_ref_mask;
+int alloc_tag_ref_offs;
+
+struct allocinfo_private {
+	struct codetag_iterator iter;
+	struct codetag_iterator reported_iter;
+	bool print_header;
+	struct allocinfo_filter filter;
+	/* ioctl uses a separate iterator not to interfere with reads */
+	struct codetag_iterator ioctl_iter;
+	bool positioned; /* seq_open_private() sets to 0 */
+	struct mutex ioctl_lock;
+};
+
+static void *allocinfo_start(struct seq_file *m, loff_t *pos)
+{
+	struct allocinfo_private *priv;
+	loff_t node = *pos;
+
+	priv = (struct allocinfo_private *)m->private;
+	codetag_lock_module_list(alloc_tag_cttype);
+	if (node == 0) {
+		priv->print_header = true;
+		priv->iter = codetag_get_ct_iter(alloc_tag_cttype);
+	} else {
+		priv->iter = priv->reported_iter;
+	}
+	codetag_next_ct(&priv->iter);
+	return priv->iter.ct ? priv : NULL;
+}
+
+static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos)
+{
+	struct allocinfo_private *priv = (struct allocinfo_private *)arg;
+	struct codetag *ct;
+
+	priv->reported_iter = priv->iter;
+	ct = codetag_next_ct(&priv->iter);
+	(*pos)++;
+	if (!ct)
+		return NULL;
+
+	return priv;
+}
+
+static void allocinfo_stop(struct seq_file *m, void *arg)
+{
+	codetag_unlock_module_list(alloc_tag_cttype);
+}
+
+static void print_allocinfo_header(struct seq_buf *buf)
+{
+	/* Output format version, so we can change it. */
+	seq_buf_printf(buf, "allocinfo - version: 2.0\n");
+	seq_buf_printf(buf, "#     <size>  <calls> <tag info>\n");
+}
+
+static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct)
+{
+	struct alloc_tag *tag = ct_to_alloc_tag(ct);
+	struct alloc_tag_counters counter = alloc_tag_read(tag);
+	s64 bytes = counter.bytes;
+
+	seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls);
+	codetag_to_text(out, ct);
+	if (unlikely(alloc_tag_is_inaccurate(tag)))
+		seq_buf_printf(out, " accurate:no");
+	seq_buf_putc(out, ' ');
+	seq_buf_putc(out, '\n');
+}
+
+static int allocinfo_show(struct seq_file *m, void *arg)
+{
+	struct allocinfo_private *priv = (struct allocinfo_private *)arg;
+	char *bufp;
+	size_t n = seq_get_buf(m, &bufp);
+	struct seq_buf buf;
+
+	seq_buf_init(&buf, bufp, n);
+	if (priv->print_header) {
+		print_allocinfo_header(&buf);
+		priv->print_header = false;
+	}
+	alloc_tag_to_text(&buf, priv->iter.ct);
+	seq_commit(m, seq_buf_used(&buf));
+	return 0;
+}
+
+static const struct seq_operations allocinfo_seq_op = {
+	.start	= allocinfo_start,
+	.next	= allocinfo_next,
+	.stop	= allocinfo_stop,
+	.show	= allocinfo_show,
+};
+
+/*
+ * Initializes seq_file operations and allocates private state when opening
+ * the /proc/allocinfo procfs entry.
+ */
+static int allocinfo_open(struct inode *inode, struct file *file)
+{
+	int ret;
+
+	ret = seq_open_private(file, &allocinfo_seq_op,
+			       sizeof(struct allocinfo_private));
+	if (!ret) {
+		struct seq_file *m = file->private_data;
+		struct allocinfo_private *priv = m->private;
+
+		mutex_init(&priv->ioctl_lock);
+	}
+	return ret;
+}
+
+/*
+ * Cleans up the seq_file state and frees up the private state allocated in
+ * allocinfo_open() when closing the /proc/allocinfo file descriptor.
+ */
+static int allocinfo_release(struct inode *inode, struct file *file)
+{
+	struct seq_file *m = file->private_data;
+	struct allocinfo_private *priv = m->private;
+
+	mutex_destroy(&priv->ioctl_lock);
+	return seq_release_private(inode, file);
+}
+
+/*
+ * Returns a pointer to the suffix of a string so that its length fits within
+ * ALLOCINFO_STR_SIZE, preserving the trailing characters.
+ * Function, file and module names often have the same prefixes, therefore
+ * when filtering by these criteria, we compare the last 64 characters to
+ * minimize the chances of name collisions
+ */
+static const char *allocinfo_str(const char *str)
+{
+	size_t len = strlen(str);
+
+	/* Keep an extra space for the trailing NULL. */
+	if (len >= ALLOCINFO_STR_SIZE)
+		str += (len - ALLOCINFO_STR_SIZE) + 1;
+	return str;
+}
+
+/* Copy a string and trim from the beginning if it's too long */
+static void allocinfo_copy_str(char *dest, const char *src)
+{
+	strscpy_pad(dest, allocinfo_str(src), ALLOCINFO_STR_SIZE);
+}
+
+/* Compare two strings and only consider the trimmed suffix if s1 is too long */
+static int allocinfo_cmp_str(const char *str, const char *template)
+{
+	return strncmp(allocinfo_str(str), template, ALLOCINFO_STR_SIZE);
+}
+
+/* Fetch the per-CPU counters */
+static inline struct alloc_tag_counters allocinfo_prefetch_counters(struct codetag *ct)
+{
+	return alloc_tag_read(ct_to_alloc_tag(ct));
+}
+
+/*
+ * Populates the UAPI allocinfo_tag_data structure with active runtime
+ * profiling counters extracted from the given kernel codetag.
+ */
+static void allocinfo_to_params(struct codetag *ct,
+				struct allocinfo_tag_data *data,
+				struct alloc_tag_counters *counters)
+{
+	if (ct->modname)
+		allocinfo_copy_str(data->tag.modname, ct->modname);
+	else
+		data->tag.modname[0] = '\0';
+	allocinfo_copy_str(data->tag.function, ct->function);
+	allocinfo_copy_str(data->tag.filename, ct->filename);
+	data->tag.lineno = ct->lineno;
+	data->counter.bytes = counters->bytes;
+	data->counter.calls = counters->calls;
+	data->counter.accurate = !alloc_tag_is_inaccurate(ct_to_alloc_tag(ct));
+}
+
+/*
+ * Retrieves the unique content ID representing the current allocation tag module
+ * layout, allowing userspace to detect if modules were loaded / unloaded.
+ */
+static int allocinfo_ioctl_get_content_id(struct seq_file *m, void __user *arg)
+{
+	struct allocinfo_content_id params;
+
+	codetag_lock_module_list(alloc_tag_cttype);
+	params.id = codetag_get_content_id(alloc_tag_cttype);
+	codetag_unlock_module_list(alloc_tag_cttype);
+	if (copy_to_user(arg, &params, sizeof(params)))
+		return -EFAULT;
+
+	return 0;
+}
+
+/*
+ * Verifies whether a given codetag satisfies the active filtering criteria by
+ * matching its characteristics against the specified filter.
+ */
+static bool matches_filter(struct codetag *ct, struct allocinfo_filter *filter,
+			   struct alloc_tag_counters *counters,
+			   bool *fetched_counters)
+{
+	bool inaccurate;
+
+	if (!filter || !filter->mask)
+		return true;
+
+	if (filter->mask & ALLOCINFO_FILTER_MASK_MODNAME) {
+		/* user wants to filter by modname but ct->modname is NULL */
+		if (!ct->modname) {
+			/* validate if user was attempting to filter for built-in allocations */
+			if (filter->fields.modname[0] != '\0')
+				return false;
+		} else if (allocinfo_cmp_str(ct->modname, filter->fields.modname))
+			return false;
+	}
+
+	if ((filter->mask & ALLOCINFO_FILTER_MASK_FUNCTION) &&
+	    ct->function && allocinfo_cmp_str(ct->function, filter->fields.function))
+		return false;
+
+	if ((filter->mask & ALLOCINFO_FILTER_MASK_FILENAME) &&
+	    ct->filename && allocinfo_cmp_str(ct->filename, filter->fields.filename))
+		return false;
+
+	if ((filter->mask & ALLOCINFO_FILTER_MASK_LINENO) &&
+	    ct->lineno != filter->fields.lineno)
+		return false;
+
+	if (filter->mask & ALLOCINFO_FILTER_MASK_INACCURATE) {
+		inaccurate = !!(ct->flags & CODETAG_FLAG_INACCURATE);
+		if (inaccurate != !!(filter->fields.inaccurate))
+			return false;
+	}
+
+	if (filter->mask & (ALLOCINFO_FILTER_MASK_MIN_SIZE | ALLOCINFO_FILTER_MASK_MAX_SIZE)) {
+		if (!*fetched_counters) {
+			*counters = allocinfo_prefetch_counters(ct);
+			*fetched_counters = true;
+		}
+		if ((filter->mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) &&
+		    counters->bytes < filter->min_size)
+			return false;
+		if ((filter->mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) &&
+		    counters->bytes > filter->max_size)
+			return false;
+	}
+
+	return true;
+}
+
+/*
+ * Seeks the ioctl iterator to the specified 0-indexed tag position, reads its
+ * profiling data and returns it to userspace.
+ */
+static int allocinfo_ioctl_get_at(struct seq_file *m, void __user *arg)
+{
+	struct allocinfo_private *priv;
+	struct codetag *ct;
+	struct allocinfo_get_at params = {0};
+	__u64 skip_count;
+	struct alloc_tag_counters counters;
+	bool fetched_counters;
+
+	if (copy_from_user(&params, arg, sizeof(params)))
+		return -EFAULT;
+
+	if (params.filter.mask & ~ALLOCINFO_FILTER_MASKS)
+		return -EINVAL;
+
+	if ((params.filter.mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) &&
+	    (params.filter.mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) &&
+	    params.filter.min_size > params.filter.max_size)
+		return -EINVAL;
+
+	priv = m->private;
+
+	mutex_lock(&priv->ioctl_lock);
+	codetag_lock_module_list(alloc_tag_cttype);
+
+	if (params.pos >= codetag_get_count(alloc_tag_cttype)) {
+		codetag_unlock_module_list(alloc_tag_cttype);
+		mutex_unlock(&priv->ioctl_lock);
+		return -ENOENT;
+	}
+
+	skip_count = params.pos;
+
+	if (params.filter.mask)
+		priv->filter = params.filter;
+	else
+		priv->filter.mask = 0;
+
+	/* Find the codetag */
+	priv->ioctl_iter = codetag_get_ct_iter(alloc_tag_cttype);
+	ct = codetag_next_ct(&priv->ioctl_iter);
+
+	while (ct) {
+		fetched_counters = false;
+		if (matches_filter(ct, &priv->filter, &counters, &fetched_counters)) {
+			if (skip_count == 0)
+				break;
+			skip_count--;
+		}
+		ct = codetag_next_ct(&priv->ioctl_iter);
+	}
+
+	if (ct) {
+		if (!fetched_counters)
+			counters = allocinfo_prefetch_counters(ct);
+		allocinfo_to_params(ct, &params.data, &counters);
+		priv->positioned = true;
+	}
+
+	codetag_unlock_module_list(alloc_tag_cttype);
+	mutex_unlock(&priv->ioctl_lock);
+
+	if (!ct)
+		return -ENOENT;
+
+	if (copy_to_user(arg, &params, sizeof(params)))
+		return -EFAULT;
+
+	return 0;
+}
+
+/*
+ * Advances the ioctl iterator to the next allocation tag in the sequence and
+ * returns its profiling data to userspace.
+ */
+static int allocinfo_ioctl_get_next(struct seq_file *m, void __user *arg)
+{
+	struct allocinfo_private *priv;
+	struct codetag *ct;
+	struct allocinfo_tag_data params;
+	int ret = 0;
+	struct alloc_tag_counters counters;
+	bool fetched_counters;
+
+	memset(&params, 0, sizeof(params));
+	priv = m->private;
+
+	mutex_lock(&priv->ioctl_lock);
+	codetag_lock_module_list(alloc_tag_cttype);
+
+	if (!priv->positioned) {
+		priv->ioctl_iter = codetag_get_ct_iter(alloc_tag_cttype);
+		priv->positioned = true;
+	}
+
+	ct = codetag_next_ct(&priv->ioctl_iter);
+	while (ct) {
+		fetched_counters = false;
+		if (matches_filter(ct, &priv->filter, &counters, &fetched_counters))
+			break;
+		ct = codetag_next_ct(&priv->ioctl_iter);
+	}
+
+	if (ct) {
+		if (!fetched_counters)
+			counters = allocinfo_prefetch_counters(ct);
+		allocinfo_to_params(ct, &params, &counters);
+	}
+	if (!ct) {
+		priv->positioned = false;
+		ret = -ENOENT;
+	}
+	codetag_unlock_module_list(alloc_tag_cttype);
+	mutex_unlock(&priv->ioctl_lock);
+
+	if (ret == 0) {
+		if (copy_to_user(arg, &params, sizeof(params)))
+			return -EFAULT;
+	}
+	return ret;
+}
+
+/*
+ * Entry point ioctl function for /proc/allocinfo routing requests to fetch the
+ * layout content ID, seek to a specific tag, or read sequential tags.
+ */
+static long allocinfo_ioctl(struct file *file, unsigned int cmd,
+			    unsigned long __arg)
+{
+	void __user *arg = (void __user *)__arg;
+	int ret;
+
+	switch (cmd) {
+	case ALLOCINFO_IOC_CONTENT_ID:
+		ret = allocinfo_ioctl_get_content_id(file->private_data, arg);
+		break;
+	case ALLOCINFO_IOC_GET_AT:
+		ret = allocinfo_ioctl_get_at(file->private_data, arg);
+		break;
+	case ALLOCINFO_IOC_GET_NEXT:
+		ret = allocinfo_ioctl_get_next(file->private_data, arg);
+		break;
+	default:
+		ret = -ENOIOCTLCMD;
+		break;
+	}
+
+	return ret;
+}
+
+#ifdef CONFIG_COMPAT
+static long allocinfo_compat_ioctl(struct file *file, unsigned int cmd,
+				   unsigned long arg)
+{
+	return allocinfo_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
+}
+#endif
+
+static const struct proc_ops allocinfo_proc_ops = {
+	.proc_open		= allocinfo_open,
+	.proc_read_iter		= seq_read_iter,
+	.proc_lseek		= seq_lseek,
+	.proc_release		= allocinfo_release,
+	.proc_ioctl		= allocinfo_ioctl,
+#ifdef CONFIG_COMPAT
+	.proc_compat_ioctl	= allocinfo_compat_ioctl,
+#endif
+};
+
+size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)
+{
+	struct codetag_iterator iter;
+	struct codetag *ct;
+	struct codetag_bytes n;
+	unsigned int i, nr = 0;
+
+	if (IS_ERR_OR_NULL(alloc_tag_cttype))
+		return 0;
+
+	if (can_sleep)
+		codetag_lock_module_list(alloc_tag_cttype);
+	else if (!codetag_trylock_module_list(alloc_tag_cttype))
+		return 0;
+
+	iter = codetag_get_ct_iter(alloc_tag_cttype);
+	while ((ct = codetag_next_ct(&iter))) {
+		struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct));
+
+		n.ct	= ct;
+		n.bytes = counter.bytes;
+
+		for (i = 0; i < nr; i++)
+			if (n.bytes > tags[i].bytes)
+				break;
+
+		if (i < count) {
+			nr -= nr == count;
+			memmove(&tags[i + 1],
+				&tags[i],
+				sizeof(tags[0]) * (nr - i));
+			nr++;
+			tags[i] = n;
+		}
+	}
+
+	codetag_unlock_module_list(alloc_tag_cttype);
+
+	return nr;
+}
+
+void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
+{
+	int i;
+	struct alloc_tag *tag;
+	unsigned int nr_pages = 1 << new_order;
+
+	if (!mem_alloc_profiling_enabled())
+		return;
+
+	tag = __pgalloc_tag_get(&folio->page);
+	if (!tag)
+		return;
+
+	for (i = nr_pages; i < (1 << old_order); i += nr_pages) {
+		union pgtag_ref_handle handle;
+		union codetag_ref ref;
+
+		if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) {
+			/* Set new reference to point to the original tag */
+			alloc_tag_ref_set(&ref, tag);
+			update_page_tag_ref(handle, &ref);
+			put_page_tag_ref(handle);
+		}
+	}
+}
+
+void pgalloc_tag_swap(struct folio *new, struct folio *old)
+{
+	union pgtag_ref_handle handle_old, handle_new;
+	union codetag_ref ref_old, ref_new;
+	struct alloc_tag *tag_old, *tag_new;
+
+	if (!mem_alloc_profiling_enabled())
+		return;
+
+	tag_old = __pgalloc_tag_get(&old->page);
+	if (!tag_old)
+		return;
+	tag_new = __pgalloc_tag_get(&new->page);
+	if (!tag_new)
+		return;
+
+	if (!get_page_tag_ref(&old->page, &ref_old, &handle_old))
+		return;
+	if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) {
+		put_page_tag_ref(handle_old);
+		return;
+	}
+
+	/*
+	 * Clear tag references to avoid debug warning when using
+	 * __alloc_tag_ref_set() with non-empty reference.
+	 */
+	set_codetag_empty(&ref_old);
+	set_codetag_empty(&ref_new);
+
+	/* swap tags */
+	__alloc_tag_ref_set(&ref_old, tag_new);
+	update_page_tag_ref(handle_old, &ref_old);
+	__alloc_tag_ref_set(&ref_new, tag_old);
+	update_page_tag_ref(handle_new, &ref_new);
+
+	put_page_tag_ref(handle_old);
+	put_page_tag_ref(handle_new);
+}
+
+static void shutdown_mem_profiling(bool remove_file)
+{
+	if (mem_alloc_profiling_enabled())
+		static_branch_disable(&mem_alloc_profiling_key);
+
+	if (!mem_profiling_support)
+		return;
+
+	if (remove_file)
+		remove_proc_entry(ALLOCINFO_FILE_NAME, NULL);
+	mem_profiling_support = false;
+}
+
+void __init alloc_tag_sec_init(void)
+{
+	struct alloc_tag *last_codetag;
+
+	if (!mem_profiling_support)
+		return;
+
+	if (!static_key_enabled(&mem_profiling_compressed))
+		return;
+
+	kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name(
+					SECTION_START(ALLOC_TAG_SECTION_NAME));
+	last_codetag = (struct alloc_tag *)kallsyms_lookup_name(
+					SECTION_STOP(ALLOC_TAG_SECTION_NAME));
+	kernel_tags.count = last_codetag - kernel_tags.first_tag;
+
+	/* Check if kernel tags fit into page flags */
+	if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) {
+		shutdown_mem_profiling(false); /* allocinfo file does not exist yet */
+		pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n",
+			kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS);
+		return;
+	}
+
+	alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS);
+	alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1);
+	pr_debug("Memory allocation profiling compression is using %d page flag bits!\n",
+		 NR_UNUSED_PAGEFLAG_BITS);
+}
+
+#ifdef CONFIG_MODULES
+
+static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE);
+static struct vm_struct *vm_module_tags;
+/* A dummy object used to indicate an unloaded module */
+static struct module unloaded_mod;
+/* A dummy object used to indicate a module prepended area */
+static struct module prepend_mod;
+
+struct alloc_tag_module_section module_tags;
+
+static inline unsigned long alloc_tag_align(unsigned long val)
+{
+	if (!static_key_enabled(&mem_profiling_compressed)) {
+		/* No alignment requirements when we are not indexing the tags */
+		return val;
+	}
+
+	if (val % sizeof(struct alloc_tag) == 0)
+		return val;
+	return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag);
+}
+
+static bool ensure_alignment(unsigned long align, unsigned int *prepend)
+{
+	if (!static_key_enabled(&mem_profiling_compressed)) {
+		/* No alignment requirements when we are not indexing the tags */
+		return true;
+	}
+
+	/*
+	 * If alloc_tag size is not a multiple of required alignment, tag
+	 * indexing does not work.
+	 */
+	if (!IS_ALIGNED(sizeof(struct alloc_tag), align))
+		return false;
+
+	/* Ensure prepend consumes multiple of alloc_tag-sized blocks */
+	if (*prepend)
+		*prepend = alloc_tag_align(*prepend);
+
+	return true;
+}
+
+static inline bool tags_addressable(void)
+{
+	unsigned long tag_idx_count;
+
+	if (!static_key_enabled(&mem_profiling_compressed))
+		return true; /* with page_ext tags are always addressable */
+
+	tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count +
+			module_tags.size / sizeof(struct alloc_tag);
+
+	return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS);
+}
+
+static bool needs_section_mem(struct module *mod, unsigned long size)
+{
+	if (!mem_profiling_support)
+		return false;
+
+	return size >= sizeof(struct alloc_tag);
+}
+
+static bool clean_unused_counters(struct alloc_tag *start_tag,
+				  struct alloc_tag *end_tag)
+{
+	struct alloc_tag *tag;
+	bool ret = true;
+
+	for (tag = start_tag; tag <= end_tag; tag++) {
+		struct alloc_tag_counters counter;
+
+		if (!tag->counters)
+			continue;
+
+		counter = alloc_tag_read(tag);
+		if (!counter.bytes) {
+			free_percpu(tag->counters);
+			tag->counters = NULL;
+		} else {
+			ret = false;
+		}
+	}
+
+	return ret;
+}
+
+/* Called with mod_area_mt locked */
+static void clean_unused_module_areas_locked(void)
+{
+	MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
+	struct module *val;
+
+	mas_for_each(&mas, val, module_tags.size) {
+		struct alloc_tag *start_tag;
+		struct alloc_tag *end_tag;
+
+		if (val != &unloaded_mod)
+			continue;
+
+		/* Release area if all tags are unused */
+		start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
+		end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
+		if (clean_unused_counters(start_tag, end_tag))
+			mas_erase(&mas);
+	}
+}
+
+/* Called with mod_area_mt locked */
+static bool find_aligned_area(struct ma_state *mas, unsigned long section_size,
+			      unsigned long size, unsigned int prepend, unsigned long align)
+{
+	bool cleanup_done = false;
+
+repeat:
+	/* Try finding exact size and hope the start is aligned */
+	if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) {
+		if (IS_ALIGNED(mas->index + prepend, align))
+			return true;
+
+		/* Try finding larger area to align later */
+		mas_reset(mas);
+		if (!mas_empty_area(mas, 0, section_size - 1,
+				    size + prepend + align - 1))
+			return true;
+	}
+
+	/* No free area, try cleanup stale data and repeat the search once */
+	if (!cleanup_done) {
+		clean_unused_module_areas_locked();
+		cleanup_done = true;
+		mas_reset(mas);
+		goto repeat;
+	}
+
+	return false;
+}
+
+static int vm_module_tags_populate(void)
+{
+	unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) +
+				 (vm_module_tags->nr_pages << PAGE_SHIFT);
+	unsigned long new_end = module_tags.start_addr + module_tags.size;
+
+	if (phys_end < new_end) {
+		struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages;
+		unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN);
+		unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN);
+		unsigned long more_pages;
+		unsigned long nr = 0;
+
+		more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT;
+		while (nr < more_pages) {
+			unsigned long allocated;
+
+			allocated = alloc_pages_bulk_node(GFP_KERNEL | __GFP_NOWARN,
+				NUMA_NO_NODE, more_pages - nr, next_page + nr);
+
+			if (!allocated)
+				break;
+			nr += allocated;
+		}
+
+		if (nr < more_pages ||
+		    vmap_pages_range(phys_end, phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL,
+				     next_page, PAGE_SHIFT) < 0) {
+			release_pages_arg arg = { .pages = next_page };
+
+			/* Clean up and error out */
+			release_pages(arg, nr);
+			return -ENOMEM;
+		}
+
+		vm_module_tags->nr_pages += nr;
+
+		/*
+		 * Kasan allocates 1 byte of shadow for every 8 bytes of data.
+		 * When kasan_alloc_module_shadow allocates shadow memory,
+		 * its unit of allocation is a page.
+		 * Therefore, here we need to align to MODULE_ALIGN.
+		 */
+		if (old_shadow_end < new_shadow_end)
+			kasan_alloc_module_shadow((void *)old_shadow_end,
+						  new_shadow_end - old_shadow_end,
+						  GFP_KERNEL);
+	}
+
+	/*
+	 * Mark the pages as accessible, now that they are mapped.
+	 * With hardware tag-based KASAN, marking is skipped for
+	 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
+	 */
+	kasan_unpoison_vmalloc((void *)module_tags.start_addr,
+				new_end - module_tags.start_addr,
+				KASAN_VMALLOC_PROT_NORMAL);
+
+	return 0;
+}
+
+static void *reserve_module_tags(struct module *mod, unsigned long size,
+				 unsigned int prepend, unsigned long align)
+{
+	unsigned long section_size = module_tags.end_addr - module_tags.start_addr;
+	MA_STATE(mas, &mod_area_mt, 0, section_size - 1);
+	unsigned long offset;
+	void *ret = NULL;
+
+	/* If no tags return error */
+	if (size < sizeof(struct alloc_tag))
+		return ERR_PTR(-EINVAL);
+
+	/*
+	 * align is always power of 2, so we can use IS_ALIGNED and ALIGN.
+	 * align 0 or 1 means no alignment, to simplify set to 1.
+	 */
+	if (!align)
+		align = 1;
+
+	if (!ensure_alignment(align, &prepend)) {
+		shutdown_mem_profiling(true);
+		pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n",
+			mod->name, align);
+		return ERR_PTR(-EINVAL);
+	}
+
+	mas_lock(&mas);
+	if (!find_aligned_area(&mas, section_size, size, prepend, align)) {
+		ret = ERR_PTR(-ENOMEM);
+		goto unlock;
+	}
+
+	/* Mark found area as reserved */
+	offset = mas.index;
+	offset += prepend;
+	offset = ALIGN(offset, align);
+	if (offset != mas.index) {
+		unsigned long pad_start = mas.index;
+
+		mas.last = offset - 1;
+		mas_store(&mas, &prepend_mod);
+		if (mas_is_err(&mas)) {
+			ret = ERR_PTR(xa_err(mas.node));
+			goto unlock;
+		}
+		mas.index = offset;
+		mas.last = offset + size - 1;
+		mas_store(&mas, mod);
+		if (mas_is_err(&mas)) {
+			mas.index = pad_start;
+			mas_erase(&mas);
+			ret = ERR_PTR(xa_err(mas.node));
+		}
+	} else {
+		mas.last = offset + size - 1;
+		mas_store(&mas, mod);
+		if (mas_is_err(&mas))
+			ret = ERR_PTR(xa_err(mas.node));
+	}
+unlock:
+	mas_unlock(&mas);
+
+	if (IS_ERR(ret))
+		return ret;
+
+	if (module_tags.size < offset + size) {
+		int grow_res;
+
+		module_tags.size = offset + size;
+		if (mem_alloc_profiling_enabled() && !tags_addressable()) {
+			shutdown_mem_profiling(true);
+			pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n",
+				mod->name, NR_UNUSED_PAGEFLAG_BITS);
+		}
+
+		grow_res = vm_module_tags_populate();
+		if (grow_res) {
+			shutdown_mem_profiling(true);
+			pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n",
+			       mod->name);
+			return ERR_PTR(grow_res);
+		}
+	}
+
+	return (struct alloc_tag *)(module_tags.start_addr + offset);
+}
+
+static void release_module_tags(struct module *mod, bool used)
+{
+	MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size);
+	struct alloc_tag *start_tag;
+	struct alloc_tag *end_tag;
+	struct module *val;
+
+	mas_lock(&mas);
+	mas_for_each_rev(&mas, val, 0)
+		if (val == mod)
+			break;
+
+	if (!val) /* module not found */
+		goto out;
+
+	if (!used)
+		goto release_area;
+
+	start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
+	end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
+	if (!clean_unused_counters(start_tag, end_tag)) {
+		struct alloc_tag *tag;
+
+		for (tag = start_tag; tag <= end_tag; tag++) {
+			struct alloc_tag_counters counter;
+
+			if (!tag->counters)
+				continue;
+
+			counter = alloc_tag_read(tag);
+			pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n",
+				tag->ct.filename, tag->ct.lineno, tag->ct.modname,
+				tag->ct.function, counter.bytes);
+		}
+	} else {
+		used = false;
+	}
+release_area:
+	mas_store(&mas, used ? &unloaded_mod : NULL);
+	val = mas_prev_range(&mas, 0);
+	if (val == &prepend_mod)
+		mas_store(&mas, NULL);
+out:
+	mas_unlock(&mas);
+}
+
+static int load_module(struct module *mod, struct codetag *start, struct codetag *stop)
+{
+	/* Allocate module alloc_tag percpu counters */
+	struct alloc_tag *start_tag;
+	struct alloc_tag *stop_tag;
+	struct alloc_tag *tag;
+
+	/* percpu counters for core allocations are already statically allocated */
+	if (!mod)
+		return 0;
+
+	start_tag = ct_to_alloc_tag(start);
+	stop_tag = ct_to_alloc_tag(stop);
+	for (tag = start_tag; tag < stop_tag; tag++) {
+		WARN_ON(tag->counters);
+		tag->counters = alloc_percpu(struct alloc_tag_counters);
+		if (!tag->counters) {
+			while (--tag >= start_tag) {
+				free_percpu(tag->counters);
+				tag->counters = NULL;
+			}
+			pr_err("Failed to allocate memory for allocation tag percpu counters in the module %s\n",
+			       mod->name);
+			return -ENOMEM;
+		}
+
+		/*
+		 * Avoid a kmemleak false positive. The pointer to the counters is stored
+		 * in the alloc_tag section of the module and cannot be directly accessed.
+		 */
+		kmemleak_ignore_percpu(tag->counters);
+	}
+	return 0;
+}
+
+static void replace_module(struct module *mod, struct module *new_mod)
+{
+	MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
+	struct module *val;
+
+	mas_lock(&mas);
+	mas_for_each(&mas, val, module_tags.size) {
+		if (val != mod)
+			continue;
+
+		mas_store_gfp(&mas, new_mod, GFP_KERNEL);
+		break;
+	}
+	mas_unlock(&mas);
+}
+
+static int __init alloc_mod_tags_mem(void)
+{
+	/* Map space to copy allocation tags */
+	vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE);
+	if (!vm_module_tags) {
+		pr_err("Failed to map %lu bytes for module allocation tags\n",
+			MODULE_ALLOC_TAG_VMAP_SIZE);
+		module_tags.start_addr = 0;
+		return -ENOMEM;
+	}
+
+	vm_module_tags->pages = kmalloc_objs(struct page *,
+					     get_vm_area_size(vm_module_tags) >> PAGE_SHIFT,
+					     GFP_KERNEL | __GFP_ZERO);
+	if (!vm_module_tags->pages) {
+		free_vm_area(vm_module_tags);
+		return -ENOMEM;
+	}
+
+	module_tags.start_addr = (unsigned long)vm_module_tags->addr;
+	module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE;
+	/* Ensure the base is alloc_tag aligned when required for indexing */
+	module_tags.start_addr = alloc_tag_align(module_tags.start_addr);
+
+	return 0;
+}
+
+static void __init free_mod_tags_mem(void)
+{
+	release_pages_arg arg = { .pages = vm_module_tags->pages };
+
+	module_tags.start_addr = 0;
+	release_pages(arg, vm_module_tags->nr_pages);
+	kfree(vm_module_tags->pages);
+	free_vm_area(vm_module_tags);
+}
+
+#else /* CONFIG_MODULES */
+
+static inline int alloc_mod_tags_mem(void) { return 0; }
+static inline void free_mod_tags_mem(void) {}
+
+#endif /* CONFIG_MODULES */
+
+/* See: Documentation/mm/allocation-profiling.rst */
+static int __init setup_early_mem_profiling(char *str)
+{
+	bool compressed = false;
+	bool enable;
+
+	if (!str || !str[0])
+		return -EINVAL;
+
+	if (!strncmp(str, "never", 5)) {
+		enable = false;
+		mem_profiling_support = false;
+		pr_info("Memory allocation profiling is disabled!\n");
+	} else {
+		char *token = strsep(&str, ",");
+
+		if (kstrtobool(token, &enable))
+			return -EINVAL;
+
+		if (str) {
+
+			if (strcmp(str, "compressed"))
+				return -EINVAL;
+
+			compressed = true;
+		}
+		mem_profiling_support = true;
+		pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n",
+			compressed ? "with" : "without", str_on_off(enable));
+	}
+
+	if (enable != mem_alloc_profiling_enabled()) {
+		if (enable)
+			static_branch_enable(&mem_alloc_profiling_key);
+		else
+			static_branch_disable(&mem_alloc_profiling_key);
+	}
+	if (compressed != static_key_enabled(&mem_profiling_compressed)) {
+		if (compressed)
+			static_branch_enable(&mem_profiling_compressed);
+		else
+			static_branch_disable(&mem_profiling_compressed);
+	}
+
+	return 0;
+}
+early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling);
+
+static __init bool need_page_alloc_tagging(void)
+{
+	if (static_key_enabled(&mem_profiling_compressed))
+		return false;
+
+	return mem_profiling_support;
+}
+
+#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
+/*
+ * Track page allocations before page_ext is initialized.
+ * Some pages are allocated before page_ext becomes available, leaving
+ * their codetag uninitialized. Track these early PFNs so we can clear
+ * their codetag refs later to avoid warnings when they are freed.
+ *
+ * Each page is cast to a pfn_pool: the first few bytes hold metadata
+ * (next pointer and slot count), the remainder stores PFNs.
+ */
+struct pfn_pool {
+	struct pfn_pool *next;
+	atomic_t count;
+	unsigned long pfns[];
+};
+
+#define PFN_POOL_SIZE			((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \
+					 sizeof(unsigned long))
+static struct pfn_pool *current_pfn_pool __initdata;
+
+static void __init __alloc_tag_add_early_pfn(unsigned long pfn)
+{
+	struct pfn_pool *pool;
+	int idx;
+
+	do {
+		pool = READ_ONCE(current_pfn_pool);
+		if (!pool || atomic_read(&pool->count) >= PFN_POOL_SIZE) {
+			struct page *new_page = __alloc_pages(__GFP_HIGH, 0, numa_mem_id(),
+							      NULL, ALLOC_NO_CODETAG);
+			struct pfn_pool *new;
+
+			if (!new_page) {
+				pr_warn_once("early PFN tracking page allocation failed\n");
+				return;
+			}
+			new = page_address(new_page);
+			new->next = pool;
+			atomic_set(&new->count, 0);
+			if (cmpxchg(&current_pfn_pool, pool, new) != pool) {
+				clear_page_tag_ref(new_page);
+				__free_page(new_page);
+				continue;
+			}
+			pool = new;
+		}
+		idx = atomic_read(&pool->count);
+		if (idx >= PFN_POOL_SIZE)
+			continue;
+		if (atomic_cmpxchg(&pool->count, idx, idx + 1) == idx)
+			break;
+	} while (1);
+
+	pool->pfns[idx] = pfn;
+}
+
+typedef void alloc_tag_add_func(unsigned long pfn);
+static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata =
+	RCU_INITIALIZER(__alloc_tag_add_early_pfn);
+
+void alloc_tag_add_early_pfn(unsigned long pfn, unsigned int alloc_flags)
+{
+	alloc_tag_add_func *alloc_tag_add;
+
+	if (static_key_enabled(&mem_profiling_compressed))
+		return;
+
+	/* Skip allocations for the tracking list itself to avoid recursion. */
+	if (alloc_flags & ALLOC_NO_CODETAG)
+		return;
+
+	rcu_read_lock();
+	alloc_tag_add = rcu_dereference(alloc_tag_add_early_pfn_ptr);
+	if (alloc_tag_add)
+		alloc_tag_add(pfn);
+	rcu_read_unlock();
+}
+
+static void __init clear_early_alloc_pfn_tag_refs(void)
+{
+	struct pfn_pool *pool, *next;
+	struct page *page;
+	int i;
+
+	if (static_key_enabled(&mem_profiling_compressed))
+		return;
+
+	rcu_assign_pointer(alloc_tag_add_early_pfn_ptr, NULL);
+	/* Make sure we are not racing with __alloc_tag_add_early_pfn() */
+	synchronize_rcu();
+
+	for (pool = current_pfn_pool; pool; pool = next) {
+		int nr_pfns = atomic_read(&pool->count);
+
+		for (i = 0; i < nr_pfns; i++) {
+			unsigned long pfn = pool->pfns[i];
+
+			if (pfn_valid(pfn)) {
+				union pgtag_ref_handle handle;
+				union codetag_ref ref;
+
+				if (get_page_tag_ref(pfn_to_page(pfn), &ref, &handle)) {
+					/*
+					 * An early-allocated page could be freed and reallocated
+					 * after its page_ext is initialized but before we clear it.
+					 * In that case, it already has a valid tag set.
+					 * We should not overwrite that valid tag
+					 * with CODETAG_EMPTY.
+					 *
+					 * Note: there is still a small race window between checking
+					 * ref.ct and calling set_codetag_empty(). We accept this
+					 * race as it's unlikely and the extra complexity of atomic
+					 * cmpxchg is not worth it for this debug-only code path.
+					 */
+					if (ref.ct) {
+						put_page_tag_ref(handle);
+						continue;
+					}
+
+					set_codetag_empty(&ref);
+					update_page_tag_ref(handle, &ref);
+					put_page_tag_ref(handle);
+				}
+			}
+		}
+
+		next = pool->next;
+		page = virt_to_page(pool);
+		clear_page_tag_ref(page);
+		__free_page(page);
+	}
+}
+#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */
+static inline void __init clear_early_alloc_pfn_tag_refs(void) {}
+#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
+
+static __init void init_page_alloc_tagging(void)
+{
+	clear_early_alloc_pfn_tag_refs();
+}
+
+struct page_ext_operations page_alloc_tagging_ops = {
+	.size = sizeof(union codetag_ref),
+	.need = need_page_alloc_tagging,
+	.init = init_page_alloc_tagging,
+};
+EXPORT_SYMBOL(page_alloc_tagging_ops);
+
+#ifdef CONFIG_SYSCTL
+/*
+ * Not using proc_do_static_key() directly to prevent enabling profiling
+ * after it was shut down.
+ */
+static int proc_mem_profiling_handler(const struct ctl_table *table, int write,
+				      void *buffer, size_t *lenp, loff_t *ppos)
+{
+	if (write) {
+		/*
+		 * Call from do_sysctl_args() which is a no-op since the same
+		 * value was already set by setup_early_mem_profiling.
+		 * Return success to avoid warnings from do_sysctl_args().
+		 */
+		if (!current->mm)
+			return 0;
+
+#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
+		/* User can't toggle profiling while debugging */
+		return -EACCES;
+#endif
+		if (!mem_profiling_support)
+			return -EINVAL;
+	}
+
+	return proc_do_static_key(table, write, buffer, lenp, ppos);
+}
+
+
+static const struct ctl_table memory_allocation_profiling_sysctls[] = {
+	{
+		.procname	= "mem_profiling",
+		.data		= &mem_alloc_profiling_key,
+		.mode		= 0644,
+		.proc_handler	= proc_mem_profiling_handler,
+	},
+};
+
+static void __init sysctl_init(void)
+{
+	register_sysctl_init("vm", memory_allocation_profiling_sysctls);
+}
+#else /* CONFIG_SYSCTL */
+static inline void sysctl_init(void) {}
+#endif /* CONFIG_SYSCTL */
+
+static int __init alloc_tag_init(void)
+{
+	const struct codetag_type_desc desc = {
+		.section		= ALLOC_TAG_SECTION_NAME,
+		.tag_size		= sizeof(struct alloc_tag),
+#ifdef CONFIG_MODULES
+		.needs_section_mem	= needs_section_mem,
+		.alloc_section_mem	= reserve_module_tags,
+		.free_section_mem	= release_module_tags,
+		.module_load		= load_module,
+		.module_replaced	= replace_module,
+#endif
+	};
+	int res;
+
+	sysctl_init();
+
+	if (!mem_profiling_support) {
+		pr_info("Memory allocation profiling is not supported!\n");
+		return 0;
+	}
+
+	if (!proc_create(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_proc_ops)) {
+		pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME);
+		shutdown_mem_profiling(false);
+		return -ENOMEM;
+	}
+
+	res = alloc_mod_tags_mem();
+	if (res) {
+		pr_err("Failed to reserve address space for module tags, errno = %d\n", res);
+		shutdown_mem_profiling(true);
+		return res;
+	}
+
+	alloc_tag_cttype = codetag_register_type(&desc);
+	if (IS_ERR(alloc_tag_cttype)) {
+		pr_err("Allocation tags registration failed, errno = %pe\n", alloc_tag_cttype);
+		free_mod_tags_mem();
+		shutdown_mem_profiling(true);
+		return PTR_ERR(alloc_tag_cttype);
+	}
+
+	return 0;
+}
+module_init(alloc_tag_init);
diff --git a/mm/bootmem_info.c b/mm/bootmem_info.c
deleted file mode 100644
index 0fa78db7fbc0b..0000000000000
--- a/mm/bootmem_info.c
+++ /dev/null
@@ -1,72 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Bootmem core functions.
- *
- * Copyright (c) 2020, Bytedance.
- *
- *     Author: Muchun Song <songmuchun@bytedance.com>
- *
- */
-#include <linux/mm.h>
-#include <linux/compiler.h>
-#include <linux/memblock.h>
-#include <linux/bootmem_info.h>
-#include <linux/memory_hotplug.h>
-#include <linux/kmemleak.h>
-
-void get_page_bootmem(unsigned long info, struct page *page,
-		enum bootmem_type type)
-{
-	BUG_ON(type > 0xf);
-	BUG_ON(info > (ULONG_MAX >> 4));
-	set_page_private(page, info << 4 | type);
-	page_ref_inc(page);
-}
-
-void put_page_bootmem(struct page *page)
-{
-	enum bootmem_type type = bootmem_type(page);
-
-	BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
-	       type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
-
-	if (page_ref_dec_return(page) == 1) {
-		set_page_private(page, 0);
-		free_reserved_page(page);
-	}
-}
-
-static void __init register_page_bootmem_info_section(unsigned long start_pfn)
-{
-	unsigned long section_nr;
-	struct mem_section *ms;
-
-	start_pfn = SECTION_ALIGN_DOWN(start_pfn);
-	section_nr = pfn_to_section_nr(start_pfn);
-	ms = __nr_to_section(section_nr);
-
-	if (!preinited_vmemmap_section(ms))
-		register_page_bootmem_memmap(section_nr, pfn_to_page(start_pfn),
-					     PAGES_PER_SECTION);
-}
-
-void __init register_page_bootmem_info_node(struct pglist_data *pgdat)
-{
-	unsigned long pfn, end_pfn;
-	int node = pgdat->node_id;
-
-	pfn = pgdat->node_start_pfn;
-	end_pfn = pgdat_end_pfn(pgdat);
-
-	/* register section info */
-	for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
-		/*
-		 * Some platforms can assign the same pfn to multiple nodes - on
-		 * node0 as well as nodeN.  To avoid registering a pfn against
-		 * multiple nodes we check that this pfn does not already
-		 * reside in some other nodes.
-		 */
-		if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node))
-			register_page_bootmem_info_section(pfn);
-	}
-}
diff --git a/mm/cma.c b/mm/cma.c
index a13ce4999b39f..a7929c758df15 100644
--- a/mm/cma.c
+++ b/mm/cma.c
@@ -33,6 +33,7 @@
 
 #include "internal.h"
 #include "cma.h"
+#include "mm_init.h"
 
 struct cma cma_areas[MAX_CMA_AREAS];
 unsigned int cma_area_count;
@@ -126,7 +127,6 @@ bool cma_validate_zones(struct cma *cma)
 		 * to be in the same zone. Simplify by forcing the entire
 		 * CMA resv range to be in the same zone.
 		 */
-		WARN_ON_ONCE(!pfn_valid(base_pfn));
 		if (pfn_range_intersects_zones(cma->nid, base_pfn, cmr->count)) {
 			set_bit(CMA_ZONES_INVALID, &cma->flags);
 			return false;
@@ -165,6 +165,8 @@ static void __init cma_activate_area(struct cma *cma)
 			bitmap_set(cmr->bitmap, 0, bitmap_count);
 		}
 
+		WARN_ON_ONCE(!pfn_valid(cmr->base_pfn));
+
 		for (pfn = early_pfn[r]; pfn < cmr->base_pfn + cmr->count;
 		     pfn += pageblock_nr_pages)
 			init_cma_reserved_pageblock(pfn_to_page(pfn));
diff --git a/mm/compaction.c b/mm/compaction.c
index f08765ade014c..9f81055a358ed 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -24,6 +24,7 @@
 #include <linux/page_owner.h>
 #include <linux/psi.h>
 #include <linux/cpuset.h>
+#include "page_alloc.h"
 #include "internal.h"
 
 #ifdef CONFIG_COMPACTION
@@ -82,7 +83,7 @@ static inline bool is_via_compact_memory(int order) { return false; }
 
 static struct page *mark_allocated_noprof(struct page *page, unsigned int order, gfp_t gfp_flags)
 {
-	post_alloc_hook(page, order, __GFP_MOVABLE);
+	post_alloc_hook(page, order, __GFP_MOVABLE, ALLOC_DEFAULT);
 	set_page_refcounted(page);
 	return page;
 }
@@ -644,7 +645,6 @@ static unsigned long isolate_freepages_block(struct compact_control *cc,
 		isolated = __isolate_free_page(page, order);
 		if (!isolated)
 			break;
-		set_page_private(page, order);
 
 		nr_scanned += isolated - 1;
 		total_isolated += isolated;
@@ -1617,7 +1617,6 @@ static void fast_isolate_freepages(struct compact_control *cc)
 		/* Isolate the page if available */
 		if (page) {
 			if (__isolate_free_page(page, order)) {
-				set_page_private(page, order);
 				nr_isolated = 1 << order;
 				nr_scanned += nr_isolated - 1;
 				total_isolated += nr_isolated;
@@ -1846,11 +1845,10 @@ static struct folio *compaction_alloc_noprof(struct folio *src, unsigned long da
 		size >>= 1;
 
 		list_add(&freepage[size].lru, &cc->freepages[start_order]);
-		set_page_private(&freepage[size], start_order);
 	}
 	dst = (struct folio *)freepage;
 
-	post_alloc_hook(&dst->page, order, __GFP_MOVABLE);
+	post_alloc_hook(&dst->page, order, __GFP_MOVABLE, ALLOC_DEFAULT);
 	set_page_refcounted(&dst->page);
 	if (order)
 		prep_compound_page(&dst->page, order);
diff --git a/mm/damon/core.c b/mm/damon/core.c
index 7e4b9affc5b06..603b102ff80f9 100644
--- a/mm/damon/core.c
+++ b/mm/damon/core.c
@@ -1,8 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * Data Access Monitor
- *
- * Author: SeongJae Park <sj@kernel.org>
  */
 
 #define pr_fmt(fmt) "damon: " fmt
@@ -168,6 +166,7 @@ struct damon_probe *damon_new_probe(void)
 	p = kmalloc_obj(*p);
 	if (!p)
 		return NULL;
+	p->weight = 0;
 	INIT_LIST_HEAD(&p->filters);
 	INIT_LIST_HEAD(&p->list);
 	return p;
@@ -210,6 +209,106 @@ static struct damon_probe *damon_nth_probe(int n, struct damon_ctx *ctx)
 	return NULL;
 }
 
+static bool damon_has_probe_weights(struct damon_ctx *c)
+{
+	struct damon_probe *p;
+
+	damon_for_each_probe(p, c) {
+		if (p->weight)
+			return true;
+	}
+	return false;
+}
+
+/*
+ * damon_mvsum() - Returns pseudo moving sum value for a time window.
+ * @current_nr:		The value of the current aggregation window.
+ * @last_nr:		The value of the last aggregation window.
+ * @left_window_bp:	Left time of the current aggregation window.
+ *
+ * This function calculates a pseudo moving sum value of a counter that is
+ * aggregated for each time window.  @current_nr is the value of the counter
+ * that aggregated so far (maybe not yet complete), from the beginning of the
+ * current aggregation time window.  @last_nr is the value of the counter that
+ * has completely aggregated in the last aggregation time window.
+ * @left_window_bp represents how much time is left for the current aggregation
+ * time window in bp (1/10,000).  For example, the aggregation time window is
+ * for every 10 seconds and 7 seconds has passed since the beginning of the
+ * current window, this parameter will be 3000 ((10 - 7) / 10 * 10000).
+ *
+ * The logic assumes the aggregation in the last phase was made in a single
+ * speed.  Based on the assumption, the value from the last window that needs
+ * to be added to the current value is calculated as a portion of the last
+ * value based on the remaining time window.
+ */
+static unsigned long damon_mvsum(unsigned long current_nr,
+		unsigned long last_nr, unsigned long left_window_bp)
+{
+	return current_nr + mult_frac(last_nr, left_window_bp, 10000);
+}
+
+/**
+ * damon_nr_accesses_mvsum() - Returns moving sum access frequency score.
+ * @r:		Region to get the access frequency of.
+ * @ctx:	DAMON context of @r.
+ *
+ * This function returns for how many sampling iterations in the last
+ * aggregation interval (&damon_attrs->aggr_interval) the region was found to
+ * be accessed.  Hence the value can be interpreted as the relative access
+ * frequency score of the region (@r).  The value is calculated as a pseudo
+ * moving sum, and hence it is not an exact value but just a best-effort
+ * reasonable estimation.
+ *
+ * Return: the pseudo moving sum access frequency score.
+ */
+unsigned int damon_nr_accesses_mvsum(struct damon_region *r,
+		struct damon_ctx *ctx)
+{
+	unsigned long sample_interval, aggr_interval;
+	unsigned long window_len, left_window, left_window_bp;
+
+	sample_interval = ctx->attrs.sample_interval ? : 1;
+	aggr_interval = ctx->attrs.aggr_interval ? : 1;
+	window_len = aggr_interval / sample_interval;
+	if (time_after_eq(ctx->passed_sample_intervals,
+				ctx->next_aggregation_sis))
+		left_window = 0;
+	else
+		left_window = ctx->next_aggregation_sis -
+			ctx->passed_sample_intervals;
+	left_window_bp = mult_frac(left_window, 10000, window_len);
+
+	if (left_window_bp == 10000)
+		return r->last_nr_accesses;
+
+	return damon_mvsum(r->nr_accesses, r->last_nr_accesses,
+			left_window_bp);
+}
+
+unsigned char damon_probe_hits_mvsum(int probe_idx, struct damon_region *r,
+		struct damon_ctx *ctx)
+{
+	unsigned long sample_interval, aggr_interval;
+	unsigned long window_len, left_window, left_window_bp;
+
+	sample_interval = ctx->attrs.sample_interval ? : 1;
+	aggr_interval =  ctx->attrs.aggr_interval ? : 1;
+	window_len = aggr_interval / sample_interval;
+	if (time_after_eq(ctx->passed_sample_intervals,
+				ctx->next_aggregation_sis))
+		left_window = 0;
+	else
+		left_window = ctx->next_aggregation_sis -
+			ctx->passed_sample_intervals;
+	left_window_bp = mult_frac(left_window, 10000, window_len);
+
+	if (left_window_bp == 10000)
+		return r->last_probe_hits[probe_idx];
+
+	return damon_mvsum(r->probe_hits[probe_idx],
+			r->last_probe_hits[probe_idx], left_window_bp);
+}
+
 #ifdef CONFIG_DAMON_DEBUG_SANITY
 static void damon_verify_new_region(unsigned long start, unsigned long end)
 {
@@ -239,9 +338,10 @@ struct damon_region *damon_new_region(unsigned long start, unsigned long end)
 	region->ar.start = start;
 	region->ar.end = end;
 	region->nr_accesses = 0;
-	region->nr_accesses_bp = 0;
-	for (i = 0; i < DAMON_MAX_PROBES; i++)
+	for (i = 0; i < DAMON_MAX_PROBES; i++) {
 		region->probe_hits[i] = 0;
+		region->last_probe_hits[i] = 0;
+	}
 	INIT_LIST_HEAD(&region->list);
 
 	region->age = 0;
@@ -304,6 +404,30 @@ static bool damon_is_last_region(struct damon_region *r,
 	return list_is_last(&r->list, &t->regions_list);
 }
 
+/**
+ * damon_probe_hits_wsum() - Returns probe hits weighted sum of a region.
+ * @r:		region to get the weighted sum of.
+ * @last:	if the request is for last-window aggregated probe hits.
+ * @ctx:	context of &r.
+ *
+ * Return: the weighted sum of probe hits of the region.
+ */
+unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last,
+		struct damon_ctx *ctx)
+{
+	struct damon_probe *probe;
+	unsigned int sum = 0;
+	int i = 0;
+
+	damon_for_each_probe(probe, ctx) {
+		if (last)
+			sum += r->last_probe_hits[i++] * probe->weight;
+		else
+			sum += r->probe_hits[i++] * probe->weight;
+	}
+	return sum;
+}
+
 /*
  * Check whether a region is intersecting an address range
  *
@@ -356,8 +480,21 @@ int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges,
 {
 	struct damon_region *r, *next;
 	unsigned int i;
+	unsigned long last_end;
 	int err;
 
+	for (i = 0; i < nr_ranges; i++) {
+		unsigned long start, end;
+
+		start = ALIGN_DOWN(ranges[i].start, min_region_sz);
+		end = ALIGN(ranges[i].end, min_region_sz);
+		if (start >= end)
+			return -EINVAL;
+		if (i > 0 && last_end > start)
+			return -EINVAL;
+		last_end = end;
+	}
+
 	/* Remove regions which are not in the new ranges */
 	damon_for_each_region_safe(r, next, t) {
 		for (i = 0; i < nr_ranges; i++) {
@@ -778,55 +915,66 @@ static unsigned int damon_age_for_new_attrs(unsigned int age,
 	return age * old_attrs->aggr_interval / new_attrs->aggr_interval;
 }
 
-/* convert access ratio in bp (per 10,000) to nr_accesses */
-static unsigned int damon_accesses_bp_to_nr_accesses(
-		unsigned int accesses_bp, struct damon_attrs *attrs)
+/* convert sample ratio in bp (per 10,000) to count */
+static unsigned int damon_sample_bp_to_count(
+		unsigned int bp, struct damon_attrs *attrs)
 {
-	return accesses_bp * damon_max_nr_accesses(attrs) / 10000;
+	return bp * damon_nr_samples_per_aggr(attrs) / 10000;
 }
 
-/*
- * Convert nr_accesses to access ratio in bp (per 10,000).
- *
- * Callers should ensure attrs.aggr_interval is not zero, like
- * damon_update_monitoring_results() does .  Otherwise, divide-by-zero would
- * happen.
- */
-static unsigned int damon_nr_accesses_to_accesses_bp(
-		unsigned int nr_accesses, struct damon_attrs *attrs)
+/* convert sample count to ratio in bp (per 10,000) */
+static unsigned int damon_sample_count_to_bp(
+		unsigned int count, struct damon_attrs *attrs)
 {
-	return mult_frac(nr_accesses, 10000, damon_max_nr_accesses(attrs));
+	return mult_frac(count, 10000, damon_nr_samples_per_aggr(attrs));
 }
 
-static unsigned int damon_nr_accesses_for_new_attrs(unsigned int nr_accesses,
+static unsigned int damon_nr_samples_for_new_attrs(unsigned int nr,
 		struct damon_attrs *old_attrs, struct damon_attrs *new_attrs)
 {
-	return damon_accesses_bp_to_nr_accesses(
-			damon_nr_accesses_to_accesses_bp(
-				nr_accesses, old_attrs),
-			new_attrs);
+	return damon_sample_bp_to_count(
+			damon_sample_count_to_bp(nr, old_attrs), new_attrs);
+}
+
+static void damon_update_probe_hits(struct damon_region *r,
+		struct damon_attrs *old_attrs, struct damon_attrs *new_attrs,
+		bool aggregating, struct damon_ctx *ctx)
+{
+	struct damon_probe *p;
+	int i = 0;
+
+	damon_for_each_probe(p, ctx) {
+		r->last_probe_hits[i] = damon_nr_samples_for_new_attrs(
+				r->last_probe_hits[i], old_attrs, new_attrs);
+		if (!aggregating)
+			r->probe_hits[i] = damon_nr_samples_for_new_attrs(
+					r->probe_hits[i], old_attrs,
+					new_attrs);
+		else
+			r->probe_hits[i] = 0;
+		i++;
+	}
 }
 
 static void damon_update_monitoring_result(struct damon_region *r,
 		struct damon_attrs *old_attrs, struct damon_attrs *new_attrs,
-		bool aggregating)
+		bool aggregating, struct damon_ctx *ctx)
 {
-	if (!aggregating) {
-		r->nr_accesses = damon_nr_accesses_for_new_attrs(
+	damon_update_probe_hits(r, old_attrs, new_attrs, aggregating, ctx);
+
+	r->last_nr_accesses = damon_nr_samples_for_new_attrs(
+			r->last_nr_accesses, old_attrs, new_attrs);
+	if (!aggregating)
+		r->nr_accesses = damon_nr_samples_for_new_attrs(
 				r->nr_accesses, old_attrs, new_attrs);
-		r->nr_accesses_bp = r->nr_accesses * 10000;
-	} else {
+	else
 		/*
 		 * if this is called in the middle of the aggregation, reset
 		 * the aggregations we made so far for this aggregation
 		 * interval.  In other words, make the status like
 		 * kdamond_reset_aggregated() is called.
 		 */
-		r->last_nr_accesses = damon_nr_accesses_for_new_attrs(
-				r->last_nr_accesses, old_attrs, new_attrs);
-		r->nr_accesses_bp = r->last_nr_accesses * 10000;
 		r->nr_accesses = 0;
-	}
 	r->age = damon_age_for_new_attrs(r->age, old_attrs, new_attrs);
 }
 
@@ -853,8 +1001,8 @@ static void damon_update_monitoring_results(struct damon_ctx *ctx,
 
 	damon_for_each_target(t, ctx)
 		damon_for_each_region(r, t)
-			damon_update_monitoring_result(
-					r, old_attrs, new_attrs, aggregating);
+			damon_update_monitoring_result(r, old_attrs, new_attrs,
+					aggregating, ctx);
 }
 
 /*
@@ -1190,6 +1338,34 @@ static void damos_set_filters_default_reject(struct damos *s)
 		damos_filters_default_reject(&s->ops_filters);
 }
 
+static bool damon_valid_probe_params(struct damon_ctx *ctx)
+{
+	unsigned long sample_interval;
+	unsigned char max_probe_hits;
+	struct damon_probe *probe;
+	unsigned int wsum, wsum_to_add;
+
+	if (!damon_has_probe_weights(ctx))
+		return true;
+
+	sample_interval = ctx->attrs.sample_interval ? : 1;
+	if (ctx->attrs.aggr_interval / sample_interval > U8_MAX)
+		return false;
+
+	/* invalid if probe hits weighted sum can overflow */
+	max_probe_hits = damon_nr_samples_per_aggr(&ctx->attrs);
+	wsum = 0;
+	damon_for_each_probe(probe, ctx) {
+		if (probe->weight > UINT_MAX / max_probe_hits)
+			return false;
+		wsum_to_add = probe->weight * max_probe_hits;
+		if (UINT_MAX - wsum < wsum_to_add)
+			return false;
+		wsum += wsum_to_add;
+	}
+	return true;
+}
+
 /*
  * damos_commit_dests() - Copy migration destinations from @src to @dst.
  * @dst:	Destination structure to update.
@@ -1349,23 +1525,35 @@ static struct damon_target *damon_nth_target(int n, struct damon_ctx *ctx)
 static int damon_commit_target_regions(struct damon_target *dst,
 		struct damon_target *src, unsigned long src_min_region_sz)
 {
-	struct damon_region *src_region;
+	struct damon_region *src_region, *prev = NULL;
 	struct damon_addr_range *ranges;
 	int i = 0, err;
 
-	damon_for_each_region(src_region, src)
-		i++;
+	damon_for_each_region(src_region, src) {
+		if (!prev || prev->ar.end != src_region->ar.start)
+			i++;
+		prev = src_region;
+	}
 	if (!i)
 		return 0;
 
-	ranges = kmalloc_objs(*ranges, i, GFP_KERNEL | __GFP_NOWARN);
+	ranges = kvmalloc_objs(*ranges, i, GFP_KERNEL | __GFP_NOWARN);
 	if (!ranges)
 		return -ENOMEM;
+	prev = NULL;
 	i = 0;
-	damon_for_each_region(src_region, src)
-		ranges[i++] = src_region->ar;
+	damon_for_each_region(src_region, src) {
+		if (!prev) {
+			ranges[i].start = src_region->ar.start;
+		} else if (prev->ar.end != src_region->ar.start) {
+			ranges[i++].end = prev->ar.end;
+			ranges[i].start = src_region->ar.start;
+		}
+		prev = src_region;
+	}
+	ranges[i++].end = damon_last_region(src)->ar.end;
 	err = damon_set_regions(dst, ranges, i, src_min_region_sz);
-	kfree(ranges);
+	kvfree(ranges);
 	return err;
 }
 
@@ -1534,6 +1722,7 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src)
 	damon_for_each_probe_safe(dst_probe, next, dst) {
 		src_probe = damon_nth_probe(i++, src);
 		if (src_probe) {
+			dst_probe->weight = src_probe->weight;
 			err = damon_commit_filters(dst_probe, src_probe);
 			if (err)
 				return err;
@@ -1550,6 +1739,7 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src)
 		if (!new_probe)
 			return -ENOMEM;
 		damon_add_probe(dst, new_probe);
+		new_probe->weight = src_probe->weight;
 		err = damon_commit_filters(new_probe, src_probe);
 		if (err)
 			return err;
@@ -1557,20 +1747,7 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src)
 	return 0;
 }
 
-/**
- * damon_commit_ctx() - Commit parameters of a DAMON context to another.
- * @dst:	The commit destination DAMON context.
- * @src:	The commit source DAMON context.
- *
- * This function copies user-specified parameters from @src to @dst and update
- * the internal status and results accordingly.  Users should use this function
- * for context-level parameters update of running context, instead of manual
- * in-place updates.
- *
- * This function should be called from parameters-update safe context, like
- * damon_call().
- */
-int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src)
+static int __damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src)
 {
 	int err;
 	struct damos *scheme;
@@ -1593,6 +1770,9 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src)
 		}
 	}
 
+	if (!damon_valid_probe_params(src))
+		return -EINVAL;
+
 	err = damon_commit_schemes(dst, src);
 	if (err)
 		return err;
@@ -1625,6 +1805,52 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src)
 	return 0;
 }
 
+static struct damon_ctx *damon_new_test_ctx(struct damon_ctx *dst)
+{
+	struct damon_ctx *test_ctx;
+	int err;
+
+	test_ctx = damon_new_ctx();
+	if (!test_ctx)
+		return NULL;
+	err = __damon_commit_ctx(test_ctx, dst);
+	if (err) {
+		damon_destroy_ctx(test_ctx);
+		return NULL;
+	}
+	return test_ctx;
+}
+
+/**
+ * damon_commit_ctx() - Commit parameters of a DAMON context to another.
+ * @dst:	The commit destination DAMON context.
+ * @src:	The commit source DAMON context.
+ *
+ * This function copies user-specified parameters from @src to @dst and update
+ * the internal status and results accordingly.  Users should use this function
+ * for context-level parameters update of running context, instead of manual
+ * in-place updates.
+ *
+ * This function should be called from parameters-update safe context, like
+ * damon_call().
+ */
+int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src)
+{
+	struct damon_ctx *test_ctx;
+	int err;
+
+	test_ctx = damon_new_test_ctx(dst);
+	if (!test_ctx)
+		return -ENOMEM;
+	err = __damon_commit_ctx(test_ctx, src);
+	if (err)
+		goto out;
+	err = __damon_commit_ctx(dst, src);
+out:
+	damon_destroy_ctx(test_ctx);
+	return err;
+}
+
 /**
  * damon_nr_running_ctxs() - Return number of currently running contexts.
  */
@@ -1725,6 +1951,8 @@ static int __damon_start(struct damon_ctx *ctx)
 	return err;
 }
 
+static int __damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src);
+
 /**
  * damon_start() - Starts the monitorings for a given group of contexts.
  * @ctxs:	an array of the pointers for contexts to start monitoring
@@ -1746,8 +1974,16 @@ int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive)
 	int err = 0;
 
 	for (i = 0; i < nr_ctxs; i++) {
-		if (!is_power_of_2(ctxs[i]->min_region_sz))
-			return -EINVAL;
+		struct damon_ctx *test_ctx;
+
+		test_ctx = damon_new_ctx();
+		if (!test_ctx)
+			return -ENOMEM;
+
+		err = __damon_commit_ctx(test_ctx, ctxs[i]);
+		damon_destroy_ctx(test_ctx);
+		if (err)
+			return err;
 	}
 
 	mutex_lock(&damon_lock);
@@ -1767,16 +2003,17 @@ int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive)
 		running_exclusive_ctxs = true;
 	mutex_unlock(&damon_lock);
 
+	if (i != nr_ctxs)
+		damon_stop(ctxs, i);
+
 	return err;
 }
 
 /*
  * __damon_stop() - Stops monitoring of a given context.
  * @ctx:	monitoring context
- *
- * Return: 0 on success, negative error code otherwise.
  */
-static int __damon_stop(struct damon_ctx *ctx)
+static void __damon_stop(struct damon_ctx *ctx)
 {
 	struct task_struct *tsk;
 
@@ -1786,31 +2023,23 @@ static int __damon_stop(struct damon_ctx *ctx)
 		get_task_struct(tsk);
 		mutex_unlock(&ctx->kdamond_lock);
 		kthread_stop_put(tsk);
-		return 0;
+		return;
 	}
 	mutex_unlock(&ctx->kdamond_lock);
-
-	return -EPERM;
 }
 
 /**
  * damon_stop() - Stops the monitorings for a given group of contexts.
  * @ctxs:	an array of the pointers for contexts to stop monitoring
  * @nr_ctxs:	size of @ctxs
- *
- * Return: 0 on success, negative error code otherwise.
  */
-int damon_stop(struct damon_ctx **ctxs, int nr_ctxs)
+void damon_stop(struct damon_ctx **ctxs, int nr_ctxs)
 {
-	int i, err = 0;
+	int i;
 
-	for (i = 0; i < nr_ctxs; i++) {
+	for (i = 0; i < nr_ctxs; i++)
 		/* nr_running_ctxs is decremented in kdamond_fn */
-		err = __damon_stop(ctxs[i]);
-		if (err)
-			break;
-	}
-	return err;
+		__damon_stop(ctxs[i]);
 }
 
 /**
@@ -1867,6 +2096,8 @@ int damon_kdamond_pid(struct damon_ctx *ctx)
  * @ctx has succeeded.  Otherwise, this function could fall into an indefinite
  * wait.
  *
+ * When this function is failed, the @ctx is guaranteed to be stopped.
+ *
  * Return: 0 on success, negative error code otherwise.
  */
 int damon_call(struct damon_ctx *ctx, struct damon_call_control *control)
@@ -1879,7 +2110,7 @@ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control)
 	mutex_lock(&ctx->call_controls_lock);
 	if (ctx->call_controls_obsolete) {
 		mutex_unlock(&ctx->call_controls_lock);
-		return -ECANCELED;
+		goto canceled;
 	}
 	list_add_tail(&control->list, &ctx->call_controls);
 	mutex_unlock(&ctx->call_controls_lock);
@@ -1887,8 +2118,14 @@ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control)
 		return 0;
 	wait_for_completion(&control->completion);
 	if (control->canceled)
-		return -ECANCELED;
+		goto canceled;
 	return 0;
+
+canceled:
+	while (damon_is_running(ctx))
+		schedule_timeout_idle(msecs_to_jiffies(100));
+	return -ECANCELED;
+
 }
 
 /**
@@ -1937,36 +2174,6 @@ int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control)
 	return 0;
 }
 
-/*
- * Warn and fix corrupted ->nr_accesses[_bp] for investigations and preventing
- * the problem being propagated.
- */
-static void damon_warn_fix_nr_accesses_corruption(struct damon_region *r)
-{
-	if (r->nr_accesses_bp == r->nr_accesses * 10000)
-		return;
-	WARN_ONCE(true, "invalid nr_accesses_bp at reset: %u %u\n",
-			r->nr_accesses_bp, r->nr_accesses);
-	r->nr_accesses_bp = r->nr_accesses * 10000;
-}
-
-#ifdef CONFIG_DAMON_DEBUG_SANITY
-static void damon_verify_reset_aggregated(struct damon_region *r,
-		struct damon_ctx *c)
-{
-	WARN_ONCE(r->nr_accesses_bp != r->last_nr_accesses * 10000,
-			"nr_accesses_bp %u last_nr_accesses %u sis %lu %lu\n",
-			r->nr_accesses_bp, r->last_nr_accesses,
-			c->passed_sample_intervals, c->next_aggregation_sis);
-}
-#else
-static void damon_verify_reset_aggregated(struct damon_region *r,
-		struct damon_ctx *c)
-{
-}
-#endif
-
-
 /*
  * Reset the aggregated monitoring results ('nr_accesses' of each region).
  */
@@ -1991,12 +2198,12 @@ static void kdamond_reset_aggregated(struct damon_ctx *c)
 			trace_damon_aggregated(ti, r, damon_nr_regions(t));
 			trace_damon_region_aggregated(ti, r,
 					damon_nr_regions(t), nr_probes);
-			damon_warn_fix_nr_accesses_corruption(r);
 			r->last_nr_accesses = r->nr_accesses;
 			r->nr_accesses = 0;
-			for (i = 0; i < DAMON_MAX_PROBES; i++)
+			for (i = 0; i < DAMON_MAX_PROBES; i++) {
+				r->last_probe_hits[i] = r->probe_hits[i];
 				r->probe_hits[i] = 0;
-			damon_verify_reset_aggregated(r, c);
+			}
 		}
 		ti++;
 	}
@@ -2041,7 +2248,7 @@ static unsigned long damon_get_intervals_adaptation_bp(struct damon_ctx *c)
 	return adaptation_bp;
 }
 
-static void kdamond_tune_intervals(struct damon_ctx *c)
+static noinline_for_stack void kdamond_tune_intervals(struct damon_ctx *c)
 {
 	unsigned long adaptation_bp;
 	struct damon_attrs new_attrs;
@@ -2063,10 +2270,11 @@ static void kdamond_tune_intervals(struct damon_ctx *c)
 	damon_set_attrs(c, &new_attrs);
 }
 
-static bool __damos_valid_target(struct damon_region *r, struct damos *s)
+static bool __damos_valid_target(struct damon_region *r, struct damos *s,
+		struct damon_ctx *c)
 {
 	unsigned long sz;
-	unsigned int nr_accesses = r->nr_accesses_bp / 10000;
+	unsigned int nr_accesses = damon_nr_accesses_mvsum(r, c);
 
 	sz = damon_sz_region(r);
 	return s->pattern.min_sz_region <= sz &&
@@ -2092,7 +2300,7 @@ static bool damos_quota_is_set(struct damos_quota *quota)
 static bool damos_valid_target(struct damon_ctx *c, struct damon_region *r,
 		struct damos *s)
 {
-	bool ret = __damos_valid_target(r, s);
+	bool ret = __damos_valid_target(r, s, c);
 
 	if (!ret || !damos_quota_is_set(&s->quota) || !c->ops.get_scheme_score)
 		return ret;
@@ -2376,7 +2584,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t,
 	struct damos *siter;		/* schemes iterator */
 	unsigned int sidx = 0;
 	struct damon_target *titer;	/* targets iterator */
-	unsigned int tidx = 0;
+	unsigned int tidx = 0, nr_accesses = 0;
 	bool do_trace = false;
 
 	/* get indices for trace_damos_before_apply() */
@@ -2391,6 +2599,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t,
 				break;
 			tidx++;
 		}
+		nr_accesses = damon_nr_accesses_mvsum(r, c);
 		do_trace = true;
 	}
 
@@ -2406,7 +2615,7 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t,
 		if (damos_core_filter_out(c, t, r, s))
 			return;
 		ktime_get_coarse_ts64(&begin);
-		trace_damos_before_apply(cidx, sidx, tidx, r,
+		trace_damos_before_apply(cidx, sidx, tidx, r, nr_accesses,
 				damon_nr_regions(t), do_trace);
 		sz_applied = c->ops.apply_scheme(c, t, r, s,
 				&sz_ops_filter_passed);
@@ -2678,7 +2887,7 @@ static phys_addr_t damos_calc_eligible_bytes(struct damon_ctx *c,
 		damon_for_each_region(r, t) {
 			phys_addr_t addr, end_addr;
 
-			if (!__damos_valid_target(r, s))
+			if (!__damos_valid_target(r, s, c))
 				continue;
 
 			/* Convert from core address units to physical bytes */
@@ -2967,7 +3176,7 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s)
 			(DAMOS_MAX_SCORE + 1));
 	damon_for_each_target(t, c) {
 		damon_for_each_region(r, t) {
-			if (!__damos_valid_target(r, s))
+			if (!__damos_valid_target(r, s, c))
 				continue;
 			if (damos_core_filter_out(c, t, r, s))
 				continue;
@@ -3071,7 +3280,6 @@ static void damon_merge_two_regions(struct damon_target *t,
 
 	l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) /
 			(sz_l + sz_r);
-	l->nr_accesses_bp = l->nr_accesses * 10000;
 	l->age = (l->age * sz_l + r->age * sz_r) / (sz_l + sz_r);
 	l->ar.end = r->ar.end;
 	/* todo: do this for only installed probes */
@@ -3082,19 +3290,15 @@ static void damon_merge_two_regions(struct damon_target *t,
 	damon_destroy_region(r, t);
 }
 
-#ifdef CONFIG_DAMON_DEBUG_SANITY
-static void damon_verify_merge_regions_of(struct damon_region *r)
-{
-	WARN_ONCE(r->nr_accesses != r->nr_accesses_bp / 10000,
-			"nr_accesses (%u) != nr_accesses_bp (%u)\n",
-			r->nr_accesses, r->nr_accesses_bp);
-}
-#else
-static void damon_verify_merge_regions_of(struct damon_region *r)
+static unsigned int damon_merge_score(struct damon_region *r, bool last,
+		struct damon_ctx *ctx, bool use_probe_hits)
 {
+	if (use_probe_hits)
+		return damon_probe_hits_wsum(r, last, ctx);
+	if (last)
+		return r->last_nr_accesses;
+	return r->nr_accesses;
 }
-#endif
-
 
 /*
  * Merge adjacent regions having similar access frequencies
@@ -3104,25 +3308,40 @@ static void damon_verify_merge_regions_of(struct damon_region *r)
  * sz_limit	size upper limit of each region
  */
 static void damon_merge_regions_of(struct damon_target *t, unsigned int thres,
-				   unsigned long sz_limit)
+		unsigned long sz_limit, struct damon_ctx *ctx, bool count_age)
 {
 	struct damon_region *r, *prev = NULL, *next;
+	bool use_probe_hits = damon_has_probe_weights(ctx);
 
 	damon_for_each_region_safe(r, next, t) {
-		damon_verify_merge_regions_of(r);
-		if (abs(r->nr_accesses - r->last_nr_accesses) > thres)
-			r->age = 0;
-		else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0))
-			r->age = 0;
-		else
-			r->age++;
+		unsigned int score, last_score, diff;
 
-		if (prev && prev->ar.end == r->ar.start &&
-		    abs(prev->nr_accesses - r->nr_accesses) <= thres &&
-		    damon_sz_region(prev) + damon_sz_region(r) <= sz_limit)
-			damon_merge_two_regions(t, prev, r);
-		else
-			prev = r;
+		score = damon_merge_score(r, false, ctx, use_probe_hits);
+		last_score = damon_merge_score(r, true, ctx, use_probe_hits);
+
+		if (count_age) {
+			if (abs_diff(score, last_score) > thres)
+				r->age = 0;
+			else if ((score == 0) != (last_score == 0))
+				r->age = 0;
+			else
+				r->age++;
+		}
+
+		if (!prev)
+			goto set_prev_continue;
+		if (prev->ar.end != r->ar.start)
+			goto set_prev_continue;
+		diff = abs_diff(score, damon_merge_score(prev, false, ctx,
+					use_probe_hits));
+		if (diff > thres)
+			goto set_prev_continue;
+		if (damon_sz_region(prev) + damon_sz_region(r) > sz_limit)
+			goto set_prev_continue;
+		damon_merge_two_regions(t, prev, r);
+		continue;
+set_prev_continue:
+		prev = r;
 	}
 }
 
@@ -3149,18 +3368,26 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold,
 	struct damon_target *t;
 	unsigned int nr_regions;
 	unsigned int max_thres;
+	bool count_age = true;
 
 	max_thres = c->attrs.aggr_interval /
 		(c->attrs.sample_interval ?  c->attrs.sample_interval : 1);
-	do {
+	while (true) {
 		nr_regions = 0;
 		damon_for_each_target(t, c) {
-			damon_merge_regions_of(t, threshold, sz_limit);
+			damon_merge_regions_of(t, threshold, sz_limit, c,
+					count_age);
 			nr_regions += damon_nr_regions(t);
 		}
-		threshold = max(1, threshold * 2);
-	} while (nr_regions > c->attrs.max_nr_regions &&
-			threshold / 2 < max_thres);
+		count_age = false;
+		if (nr_regions <= c->attrs.max_nr_regions ||
+				max_thres <= threshold)
+			break;
+		if (threshold < max_thres / 2)
+			threshold = max(1, threshold * 2);
+		else
+			threshold = max_thres;
+	}
 }
 
 #ifdef CONFIG_DAMON_DEBUG_SANITY
@@ -3198,10 +3425,11 @@ static void damon_split_region_at(struct damon_target *t,
 
 	new->age = r->age;
 	new->last_nr_accesses = r->last_nr_accesses;
-	new->nr_accesses_bp = r->nr_accesses_bp;
 	new->nr_accesses = r->nr_accesses;
 	/* todo: do this for only installed probes */
 	memcpy(new->probe_hits, r->probe_hits, sizeof(r->probe_hits));
+	memcpy(new->last_probe_hits, r->last_probe_hits,
+			sizeof(r->last_probe_hits));
 
 	damon_insert_region(new, r, damon_next_region(r), t);
 }
@@ -3236,6 +3464,37 @@ static void damon_split_regions_of(struct damon_ctx *ctx,
 	}
 }
 
+/* Split one in every @split_step regions into two, from a rotating offset */
+static void damon_split_some_regions(struct damon_ctx *ctx,
+				     unsigned long split_step)
+{
+	static unsigned long rotation;
+	struct damon_target *t;
+	struct damon_region *r, *next;
+	unsigned long offset = rotation++ % split_step;
+	unsigned long idx = 0;
+
+	damon_for_each_target(t, ctx) {
+		damon_for_each_region_safe(r, next, t) {
+			unsigned long sz_region, sz_sub;
+
+			if (idx++ % split_step != offset)
+				continue;
+			sz_region = damon_sz_region(r);
+			if (sz_region < 2 * ctx->min_region_sz)
+				continue;
+
+			sz_sub = ALIGN_DOWN(damon_rand(ctx, 1, 10) *
+					sz_region / 10, ctx->min_region_sz);
+			/* Do not allow blank region */
+			if (sz_sub == 0 || sz_sub >= sz_region)
+				continue;
+
+			damon_split_region_at(t, r, sz_sub);
+		}
+	}
+}
+
 /*
  * Split every target region into randomly-sized small regions
  *
@@ -3249,25 +3508,33 @@ static void damon_split_regions_of(struct damon_ctx *ctx,
 static void kdamond_split_regions(struct damon_ctx *ctx)
 {
 	struct damon_target *t;
-	unsigned int nr_regions = 0;
-	static unsigned int last_nr_regions;
+	unsigned long nr_regions = 0;
+	unsigned long max_nr_regions = ctx->attrs.max_nr_regions;
+	static unsigned long last_nr_regions;
 	int nr_subregions = 2;
 
 	damon_for_each_target(t, ctx)
 		nr_regions += damon_nr_regions(t);
 
-	if (nr_regions > ctx->attrs.max_nr_regions / 2)
-		return;
+	if (nr_regions >= max_nr_regions)
+		goto done;
+
+	if (nr_regions > max_nr_regions / 2) {
+		damon_split_some_regions(ctx,
+			max_nr_regions / (max_nr_regions - nr_regions));
+		goto done;
+	}
 
 	/* Maybe the middle of the region has different access frequency */
 	if (last_nr_regions == nr_regions &&
-			nr_regions < ctx->attrs.max_nr_regions / 3)
+			nr_regions < max_nr_regions / 3)
 		nr_subregions = 3;
 
 	damon_for_each_target(t, ctx)
 		damon_split_regions_of(ctx, t, nr_subregions,
 				       ctx->min_region_sz);
 
+done:
 	last_nr_regions = nr_regions;
 }
 
@@ -3479,7 +3746,6 @@ static void kdamond_init_ctx(struct damon_ctx *ctx)
 static int kdamond_fn(void *data)
 {
 	struct damon_ctx *ctx = data;
-	unsigned int max_nr_accesses = 0;
 	unsigned long sz_limit = 0;
 
 	pr_debug("kdamond (%d) starts\n", current->pid);
@@ -3512,25 +3778,38 @@ static int kdamond_fn(void *data)
 		unsigned long next_aggregation_sis = ctx->next_aggregation_sis;
 		unsigned long next_ops_update_sis = ctx->next_ops_update_sis;
 		unsigned long sample_interval = ctx->attrs.sample_interval;
+		bool access_check_disabled = damon_has_probe_weights(ctx);
+		unsigned int max_merge_score = 0, max_wsum;
+		bool get_max_wsum;
 
 		if (kdamond_wait_activation(ctx))
 			break;
 
-		if (ctx->ops.prepare_access_checks)
+		if (!access_check_disabled && ctx->ops.prepare_access_checks)
 			ctx->ops.prepare_access_checks(ctx);
 
 		kdamond_usleep(sample_interval);
 		ctx->passed_sample_intervals++;
 
-		if (ctx->ops.check_accesses)
-			max_nr_accesses = ctx->ops.check_accesses(ctx);
-		if (ctx->ops.apply_probes)
-			ctx->ops.apply_probes(ctx);
+		if (!access_check_disabled && ctx->ops.check_accesses)
+			max_merge_score = ctx->ops.check_accesses(ctx);
+		if (ctx->ops.apply_probes) {
+			if (time_after_eq(ctx->passed_sample_intervals,
+						next_aggregation_sis) &&
+					access_check_disabled)
+				get_max_wsum = true;
+			else
+				get_max_wsum = false;
+			max_wsum = ctx->ops.apply_probes(ctx,
+					access_check_disabled, get_max_wsum);
+			if (get_max_wsum)
+				max_merge_score = max_wsum;
+		}
 
 		if (time_after_eq(ctx->passed_sample_intervals,
 					next_aggregation_sis)) {
 			kdamond_merge_regions(ctx,
-					max_nr_accesses / 10,
+					max_merge_score / 10,
 					sz_limit);
 			/* online updates might be made */
 			sz_limit = damon_apply_min_nr_regions(ctx);
@@ -3573,8 +3852,7 @@ static int kdamond_fn(void *data)
 				 * aggregation, and make aggregation
 				 * information reset for all regions.  Then,
 				 * following kdamond_reset_aggregated() call
-				 * will make the region information invalid,
-				 * particularly for ->nr_accesses_bp.
+				 * will make the region information invalid.
 				 *
 				 * Reset ->next_aggregation_sis to avoid that.
 				 * It will anyway correctly updated after this
@@ -3701,9 +3979,6 @@ int damon_set_region_system_rams_default(struct damon_target *t,
 {
 	struct damon_addr_range addr_range;
 
-	if (*start > *end)
-		return -EINVAL;
-
 	if (!*start && !*end &&
 		!damon_find_system_rams_range(start, end, addr_unit))
 		return -EINVAL;
@@ -3713,72 +3988,18 @@ int damon_set_region_system_rams_default(struct damon_target *t,
 	return damon_set_regions(t, &addr_range, 1, min_region_sz);
 }
 
-/*
- * damon_moving_sum() - Calculate an inferred moving sum value.
- * @mvsum:	Inferred sum of the last @len_window values.
- * @nomvsum:	Non-moving sum of the last discrete @len_window window values.
- * @len_window:	The number of last values to take care of.
- * @new_value:	New value that will be added to the pseudo moving sum.
- *
- * Moving sum (moving average * window size) is good for handling noise, but
- * the cost of keeping past values can be high for arbitrary window size.  This
- * function implements a lightweight pseudo moving sum function that doesn't
- * keep the past window values.
- *
- * It simply assumes there was no noise in the past, and get the no-noise
- * assumed past value to drop from @nomvsum and @len_window.  @nomvsum is a
- * non-moving sum of the last window.  For example, if @len_window is 10 and we
- * have 25 values, @nomvsum is the sum of the 11th to 20th values of the 25
- * values.  Hence, this function simply drops @nomvsum / @len_window from
- * given @mvsum and add @new_value.
- *
- * For example, if @len_window is 10 and @nomvsum is 50, the last 10 values for
- * the last window could be vary, e.g., 0, 10, 0, 10, 0, 10, 0, 0, 0, 20.  For
- * calculating next moving sum with a new value, we should drop 0 from 50 and
- * add the new value.  However, this function assumes it got value 5 for each
- * of the last ten times.  Based on the assumption, when the next value is
- * measured, it drops the assumed past value, 5 from the current sum, and add
- * the new value to get the updated pseduo-moving average.
- *
- * This means the value could have errors, but the errors will be disappeared
- * for every @len_window aligned calls.  For example, if @len_window is 10, the
- * pseudo moving sum with 11th value to 19th value would have an error.  But
- * the sum with 20th value will not have the error.
- *
- * Return: Pseudo-moving average after getting the @new_value.
- */
-static unsigned int damon_moving_sum(unsigned int mvsum, unsigned int nomvsum,
-		unsigned int len_window, unsigned int new_value)
-{
-	return mvsum - nomvsum / len_window + new_value;
-}
-
 /**
  * damon_update_region_access_rate() - Update the access rate of a region.
  * @r:		The DAMON region to update for its access check result.
  * @accessed:	Whether the region has accessed during last sampling interval.
- * @attrs:	The damon_attrs of the DAMON context.
  *
  * Update the access rate of a region with the region's last sampling interval
  * access check result.
  *
  * Usually this will be called by &damon_operations->check_accesses callback.
  */
-void damon_update_region_access_rate(struct damon_region *r, bool accessed,
-		struct damon_attrs *attrs)
+void damon_update_region_access_rate(struct damon_region *r, bool accessed)
 {
-	unsigned int len_window = 1;
-
-	/*
-	 * sample_interval can be zero, but cannot be larger than
-	 * aggr_interval, owing to validation of damon_set_attrs().
-	 */
-	if (attrs->sample_interval)
-		len_window = damon_max_nr_accesses(attrs);
-	r->nr_accesses_bp = damon_moving_sum(r->nr_accesses_bp,
-			r->last_nr_accesses * 10000, len_window,
-			accessed ? 10000 : 0);
-
 	if (accessed)
 		r->nr_accesses++;
 }
diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c
index 8298c6001fd09..7e077084cb03a 100644
--- a/mm/damon/lru_sort.c
+++ b/mm/damon/lru_sort.c
@@ -1,8 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * DAMON-based LRU-lists Sorting
- *
- * Author: SeongJae Park <sj@kernel.org>
  */
 
 #define pr_fmt(fmt) "damon-lru-sort: " fmt
@@ -286,11 +284,6 @@ static int damon_lru_sort_apply_parameters(void)
 	param_ctx->addr_unit = addr_unit;
 	param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1);
 
-	if (!is_power_of_2(param_ctx->min_region_sz)) {
-		err = -EINVAL;
-		goto out;
-	}
-
 	if (!damon_lru_sort_mon_attrs.sample_interval) {
 		err = -EINVAL;
 		goto out;
@@ -310,7 +303,7 @@ static int damon_lru_sort_apply_parameters(void)
 		goto out;
 
 	err = -ENOMEM;
-	hot_thres = damon_max_nr_accesses(&attrs) *
+	hot_thres = damon_nr_samples_per_aggr(&attrs) *
 		hot_thres_access_freq / 1000;
 	hot_scheme = damon_lru_sort_new_hot_scheme(hot_thres);
 	if (!hot_scheme)
@@ -416,8 +409,10 @@ static int damon_lru_sort_turn(bool on)
 {
 	int err;
 
-	if (!on)
-		return damon_stop(&ctx, 1);
+	if (!on) {
+		damon_stop(&ctx, 1);
+		return 0;
+	}
 
 	err = damon_lru_sort_apply_parameters();
 	if (err)
diff --git a/mm/damon/modules-common.c b/mm/damon/modules-common.c
index 86d58f8c4f635..f87fa46a95a08 100644
--- a/mm/damon/modules-common.c
+++ b/mm/damon/modules-common.c
@@ -1,8 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * Common Code for DAMON Modules
- *
- * Author: SeongJae Park <sj@kernel.org>
  */
 
 #include <linux/damon.h>
diff --git a/mm/damon/modules-common.h b/mm/damon/modules-common.h
index f103ad5563687..6fd45490e45b6 100644
--- a/mm/damon/modules-common.h
+++ b/mm/damon/modules-common.h
@@ -1,8 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Common Code for DAMON Modules
- *
- * Author: SeongJae Park <sj@kernel.org>
  */
 
 #include <linux/moduleparam.h>
diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c
index d1842e2b00ef8..89c694f6e4fa8 100644
--- a/mm/damon/ops-common.c
+++ b/mm/damon/ops-common.c
@@ -1,8 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * Common Code for Data Access Monitoring
- *
- * Author: SeongJae Park <sj@kernel.org>
  */
 
 #include <linux/migrate.h>
@@ -114,7 +112,7 @@ int damon_hot_score(struct damon_ctx *c, struct damon_region *r,
 	int hotness;
 
 	freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE /
-		damon_max_nr_accesses(&c->attrs);
+		damon_nr_samples_per_aggr(&c->attrs);
 
 	age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000;
 	if (age_in_sec)
diff --git a/mm/damon/ops-common.h b/mm/damon/ops-common.h
index 5efa5b5970def..38d295488fa18 100644
--- a/mm/damon/ops-common.h
+++ b/mm/damon/ops-common.h
@@ -1,8 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Common Code for Data Access Monitoring
- *
- * Author: SeongJae Park <sj@kernel.org>
  */
 
 #include <linux/damon.h>
diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c
index d0598f5f26882..e4f98d67461f5 100644
--- a/mm/damon/paddr.c
+++ b/mm/damon/paddr.c
@@ -1,8 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * DAMON Code for The Physical Address Space
- *
- * Author: SeongJae Park <sj@kernel.org>
  */
 
 #define pr_fmt(fmt) "damon-pa: " fmt
@@ -67,7 +65,7 @@ static void damon_pa_prepare_access_checks(struct damon_ctx *ctx)
 	}
 }
 
-static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz)
+static bool damon_pa_young(phys_addr_t paddr)
 {
 	struct folio *folio = damon_get_folio(PHYS_PFN(paddr));
 	bool accessed;
@@ -76,31 +74,19 @@ static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz)
 		return false;
 
 	accessed = damon_folio_young(folio);
-	*folio_sz = folio_size(folio);
 	folio_put(folio);
 	return accessed;
 }
 
 static void __damon_pa_check_access(struct damon_region *r,
-		struct damon_attrs *attrs, unsigned long addr_unit)
+		unsigned long addr_unit)
 {
-	static phys_addr_t last_addr;
-	static unsigned long last_folio_sz = PAGE_SIZE;
-	static bool last_accessed;
+	bool accessed;
 	phys_addr_t sampling_addr = damon_pa_phys_addr(
 			r->sampling_addr, addr_unit);
 
-	/* If the region is in the last checked page, reuse the result */
-	if (ALIGN_DOWN(last_addr, last_folio_sz) ==
-				ALIGN_DOWN(sampling_addr, last_folio_sz)) {
-		damon_update_region_access_rate(r, last_accessed, attrs);
-		return;
-	}
-
-	last_accessed = damon_pa_young(sampling_addr, &last_folio_sz);
-	damon_update_region_access_rate(r, last_accessed, attrs);
-
-	last_addr = sampling_addr;
+	accessed = damon_pa_young(sampling_addr);
+	damon_update_region_access_rate(r, accessed);
 }
 
 static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx)
@@ -111,8 +97,7 @@ static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx)
 
 	damon_for_each_target(t, ctx) {
 		damon_for_each_region(r, t) {
-			__damon_pa_check_access(
-					r, &ctx->attrs, ctx->addr_unit);
+			__damon_pa_check_access(r, ctx->addr_unit);
 			max_nr_accesses = max(r->nr_accesses, max_nr_accesses);
 		}
 	}
@@ -169,11 +154,13 @@ static bool damon_pa_filter_pass(phys_addr_t pa, struct folio *folio,
 	return pass;
 }
 
-static void damon_pa_apply_probes(struct damon_ctx *ctx)
+static unsigned int damon_pa_apply_probes(struct damon_ctx *ctx,
+		bool set_samples, bool return_max_wsum)
 {
 	struct damon_target *t;
 	struct damon_region *r;
 	struct damon_probe *p;
+	unsigned int max_wsum = 0;
 
 	damon_for_each_target(t, ctx) {
 		damon_for_each_region(r, t) {
@@ -181,6 +168,9 @@ static void damon_pa_apply_probes(struct damon_ctx *ctx)
 			phys_addr_t pa;
 			struct folio *folio;
 
+			if (set_samples)
+				r->sampling_addr = damon_rand(ctx, r->ar.start,
+						r->ar.end);
 			pa = damon_pa_phys_addr(r->sampling_addr,
 					ctx->addr_unit);
 			folio = damon_get_folio(PHYS_PFN(pa));
@@ -191,8 +181,12 @@ static void damon_pa_apply_probes(struct damon_ctx *ctx)
 			}
 			if (folio)
 				folio_put(folio);
+			if (return_max_wsum)
+				max_wsum = max(damon_probe_hits_wsum(r, false,
+							ctx), max_wsum);
 		}
 	}
+	return max_wsum;
 }
 
 /*
diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c
index ce4499cf4b8b0..09e941d75f67d 100644
--- a/mm/damon/reclaim.c
+++ b/mm/damon/reclaim.c
@@ -1,8 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * DAMON-based page reclamation
- *
- * Author: SeongJae Park <sj@kernel.org>
  */
 
 #define pr_fmt(fmt) "damon-reclaim: " fmt
@@ -210,11 +208,6 @@ static int damon_reclaim_apply_parameters(void)
 	param_ctx->addr_unit = addr_unit;
 	param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1);
 
-	if (!is_power_of_2(param_ctx->min_region_sz)) {
-		err = -EINVAL;
-		goto out;
-	}
-
 	if (!damon_reclaim_mon_attrs.aggr_interval) {
 		err = -EINVAL;
 		goto out;
@@ -339,8 +332,10 @@ static int damon_reclaim_turn(bool on)
 {
 	int err;
 
-	if (!on)
-		return damon_stop(&ctx, 1);
+	if (!on) {
+		damon_stop(&ctx, 1);
+		return 0;
+	}
 
 	err = damon_reclaim_apply_parameters();
 	if (err)
diff --git a/mm/damon/stat.c b/mm/damon/stat.c
index 0e14f5bb8f75d..b05b68f73e10d 100644
--- a/mm/damon/stat.c
+++ b/mm/damon/stat.c
@@ -138,7 +138,7 @@ static int damon_stat_damon_call_fn(void *data)
 
 	/* avoid unnecessarily frequent stat update */
 	if (time_before_eq(jiffies, damon_stat_last_refresh_jiffies +
-				msecs_to_jiffies(5 * MSEC_PER_SEC)))
+				secs_to_jiffies(5)))
 		return 0;
 	damon_stat_last_refresh_jiffies = jiffies;
 
diff --git a/mm/damon/sysfs-common.c b/mm/damon/sysfs-common.c
index bdc6ae2639e4f..c59d7bf7a73ae 100644
--- a/mm/damon/sysfs-common.c
+++ b/mm/damon/sysfs-common.c
@@ -1,8 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * Common Code for DAMON Sysfs Interface
- *
- * Author: SeongJae Park <sj@kernel.org>
  */
 
 #include <linux/slab.h>
diff --git a/mm/damon/sysfs-common.h b/mm/damon/sysfs-common.h
index 3079306966a91..733764716e8dd 100644
--- a/mm/damon/sysfs-common.h
+++ b/mm/damon/sysfs-common.h
@@ -1,8 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Common Code for DAMON Sysfs Interface
- *
- * Author: SeongJae Park <sj@kernel.org>
  */
 
 #include <linux/damon.h>
diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c
index 0134111c3c1ff..32f495a96b17a 100644
--- a/mm/damon/sysfs-schemes.c
+++ b/mm/damon/sysfs-schemes.c
@@ -1,8 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * DAMON sysfs Interface
- *
- * Copyright (c) 2022 SeongJae Park <sj@kernel.org>
  */
 
 #include <linux/slab.h>
@@ -112,7 +110,8 @@ static int damos_sysfs_probes_add_dirs(struct damos_sysfs_probes *probes,
 		struct damos_sysfs_probe *sys_probe;
 		int err;
 
-		sys_probe = damos_sysfs_probe_alloc(region->probe_hits[i]);
+		sys_probe = damos_sysfs_probe_alloc(
+				damon_probe_hits_mvsum(i, region, ctx));
 		if (!sys_probe) {
 			damos_sysfs_probes_rm_dirs(probes);
 			return -ENOMEM;
@@ -159,7 +158,7 @@ struct damon_sysfs_scheme_region {
 };
 
 static struct damon_sysfs_scheme_region *damon_sysfs_scheme_region_alloc(
-		struct damon_region *region)
+		struct damon_region *region, struct damon_ctx *ctx)
 {
 	struct damon_sysfs_scheme_region *sysfs_region = kmalloc_obj(*sysfs_region);
 
@@ -167,7 +166,7 @@ static struct damon_sysfs_scheme_region *damon_sysfs_scheme_region_alloc(
 		return NULL;
 	sysfs_region->kobj = (struct kobject){};
 	sysfs_region->ar = region->ar;
-	sysfs_region->nr_accesses = region->nr_accesses_bp / 10000;
+	sysfs_region->nr_accesses = damon_nr_accesses_mvsum(region, ctx);
 	sysfs_region->age = region->age;
 	sysfs_region->probes = NULL;
 	INIT_LIST_HEAD(&sysfs_region->list);
@@ -334,6 +333,7 @@ static void damon_sysfs_scheme_regions_rm_dirs(
 	list_for_each_entry_safe(r, next, &regions->regions_list, list) {
 		damos_sysfs_region_rm_dirs(r);
 		list_del(&r->list);
+		kobject_del(&r->kobj);
 		kobject_put(&r->kobj);
 		regions->nr_regions--;
 	}
@@ -913,8 +913,10 @@ static void damon_sysfs_scheme_filters_rm_dirs(
 	struct damon_sysfs_scheme_filter **filters_arr = filters->filters_arr;
 	int i;
 
-	for (i = 0; i < filters->nr; i++)
+	for (i = 0; i < filters->nr; i++) {
+		kobject_del(&filters_arr[i]->kobj);
 		kobject_put(&filters_arr[i]->kobj);
+	}
 	filters->nr = 0;
 	kfree(filters_arr);
 	filters->filters_arr = NULL;
@@ -1462,8 +1464,10 @@ static void damos_sysfs_quota_goals_rm_dirs(
 	struct damos_sysfs_quota_goal **goals_arr = goals->goals_arr;
 	int i;
 
-	for (i = 0; i < goals->nr; i++)
+	for (i = 0; i < goals->nr; i++) {
+		kobject_del(&goals_arr[i]->kobj);
 		kobject_put(&goals_arr[i]->kobj);
+	}
 	goals->nr = 0;
 	kfree(goals_arr);
 	goals->goals_arr = NULL;
@@ -2140,8 +2144,10 @@ static void damos_sysfs_dests_rm_dirs(
 	struct damos_sysfs_dest **dests_arr = dests->dests_arr;
 	int i;
 
-	for (i = 0; i < dests->nr; i++)
+	for (i = 0; i < dests->nr; i++) {
+		kobject_del(&dests_arr[i]->kobj);
 		kobject_put(&dests_arr[i]->kobj);
+	}
 	dests->nr = 0;
 	kfree(dests_arr);
 	dests->dests_arr = NULL;
@@ -2683,6 +2689,7 @@ void damon_sysfs_schemes_rm_dirs(struct damon_sysfs_schemes *schemes)
 
 	for (i = 0; i < schemes->nr; i++) {
 		damon_sysfs_scheme_rm_dirs(schemes_arr[i]);
+		kobject_del(&schemes_arr[i]->kobj);
 		kobject_put(&schemes_arr[i]->kobj);
 	}
 	schemes->nr = 0;
@@ -2724,13 +2731,15 @@ static int damon_sysfs_schemes_add_dirs(struct damon_sysfs_schemes *schemes,
 			goto out;
 		err = damon_sysfs_scheme_add_dirs(scheme);
 		if (err)
-			goto out;
+			goto del_out;
 
 		schemes_arr[i] = scheme;
 		schemes->nr++;
 	}
 	return 0;
 
+del_out:
+	kobject_del(&scheme->kobj);
 out:
 	damon_sysfs_schemes_rm_dirs(schemes);
 	kobject_put(&scheme->kobj);
@@ -3114,7 +3123,7 @@ void damos_sysfs_populate_region_dir(struct damon_sysfs_schemes *sysfs_schemes,
 	if (total_bytes_only)
 		return;
 
-	region = damon_sysfs_scheme_region_alloc(r);
+	region = damon_sysfs_scheme_region_alloc(r, ctx);
 	if (!region)
 		return;
 	region->sz_filter_passed = sz_filter_passed;
@@ -3124,12 +3133,14 @@ void damos_sysfs_populate_region_dir(struct damon_sysfs_schemes *sysfs_schemes,
 				sysfs_regions->nr_regions))
 		goto out;
 	if (damos_sysfs_region_add_dirs(region, ctx, r))
-		goto out;
+		goto del_out;
 
 	list_add_tail(&region->list, &sysfs_regions->regions_list);
 	sysfs_regions->nr_regions++;
 	return;
 
+del_out:
+	kobject_del(&region->kobj);
 out:
 	kobject_put(&region->kobj);
 }
diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c
index 2e95e3bac774d..e3858ffab4b22 100644
--- a/mm/damon/sysfs.c
+++ b/mm/damon/sysfs.c
@@ -1,8 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * DAMON sysfs Interface
- *
- * Copyright (c) 2022 SeongJae Park <sj@kernel.org>
  */
 
 #include <linux/pid.h>
@@ -107,8 +105,10 @@ static void damon_sysfs_regions_rm_dirs(struct damon_sysfs_regions *regions)
 	struct damon_sysfs_region **regions_arr = regions->regions_arr;
 	int i;
 
-	for (i = 0; i < regions->nr; i++)
+	for (i = 0; i < regions->nr; i++) {
+		kobject_del(&regions_arr[i]->kobj);
 		kobject_put(&regions_arr[i]->kobj);
+	}
 	regions->nr = 0;
 	kfree(regions_arr);
 	regions->regions_arr = NULL;
@@ -333,6 +333,7 @@ static void damon_sysfs_targets_rm_dirs(struct damon_sysfs_targets *targets)
 
 	for (i = 0; i < targets->nr; i++) {
 		damon_sysfs_target_rm_dirs(targets_arr[i]);
+		kobject_del(&targets_arr[i]->kobj);
 		kobject_put(&targets_arr[i]->kobj);
 	}
 	targets->nr = 0;
@@ -371,13 +372,15 @@ static int damon_sysfs_targets_add_dirs(struct damon_sysfs_targets *targets,
 
 		err = damon_sysfs_target_add_dirs(target);
 		if (err)
-			goto out;
+			goto del_out;
 
 		targets_arr[i] = target;
 		targets->nr++;
 	}
 	return 0;
 
+del_out:
+	kobject_del(&target->kobj);
 out:
 	damon_sysfs_targets_rm_dirs(targets);
 	kobject_put(&target->kobj);
@@ -957,8 +960,10 @@ static void damon_sysfs_filters_rm_dirs(struct damon_sysfs_filters *filters)
 	struct damon_sysfs_filter **filters_arr = filters->filters_arr;
 	int i;
 
-	for (i = 0; i < filters->nr; i++)
+	for (i = 0; i < filters->nr; i++) {
+		kobject_del(&filters_arr[i]->kobj);
 		kobject_put(&filters_arr[i]->kobj);
+	}
 	filters->nr = 0;
 	kfree(filters_arr);
 	filters->filters_arr = NULL;
@@ -1060,6 +1065,7 @@ static const struct kobj_type damon_sysfs_filters_ktype = {
 
 struct damon_sysfs_probe {
 	struct kobject kobj;
+	unsigned int weight;
 	struct damon_sysfs_filters *filters;
 };
 
@@ -1068,7 +1074,7 @@ static struct damon_sysfs_probe *damon_sysfs_probe_alloc(void)
 	return kzalloc_obj(struct damon_sysfs_probe);
 }
 
-static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *attr)
+static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *probe)
 {
 	struct damon_sysfs_filters *filters;
 	int err;
@@ -1076,31 +1082,54 @@ static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *attr)
 	filters = damon_sysfs_filters_alloc();
 	if (!filters)
 		return -ENOMEM;
-	attr->filters = filters;
+	probe->filters = filters;
 
 	err = kobject_init_and_add(&filters->kobj, &damon_sysfs_filters_ktype,
-			&attr->kobj, "filters");
+			&probe->kobj, "filters");
 	if (err) {
 		kobject_put(&filters->kobj);
-		attr->filters = NULL;
+		probe->filters = NULL;
 	}
 	return err;
 }
 
-static void damon_sysfs_probe_rm_dirs(struct damon_sysfs_probe *attr)
+static void damon_sysfs_probe_rm_dirs(struct damon_sysfs_probe *probe)
 {
-	if (attr->filters) {
-		damon_sysfs_filters_rm_dirs(attr->filters);
-		kobject_put(&attr->filters->kobj);
+	if (probe->filters) {
+		damon_sysfs_filters_rm_dirs(probe->filters);
+		kobject_put(&probe->filters->kobj);
 	}
 }
 
+static ssize_t weight_show(struct kobject *kobj, struct kobj_attribute *attr,
+		char *buf)
+{
+	struct damon_sysfs_probe *probe = container_of(kobj,
+			struct damon_sysfs_probe, kobj);
+
+	return sysfs_emit(buf, "%u\n", probe->weight);
+}
+
+static ssize_t weight_store(struct kobject *kobj,
+		struct kobj_attribute *attr, const char *buf, size_t count)
+{
+	struct damon_sysfs_probe *probe = container_of(kobj,
+			struct damon_sysfs_probe, kobj);
+	int err = kstrtouint(buf, 0, &probe->weight);
+
+	return err ? err : count;
+}
+
 static void damon_sysfs_probe_release(struct kobject *kobj)
 {
 	kfree(container_of(kobj, struct damon_sysfs_probe, kobj));
 }
 
+static struct kobj_attribute damon_sysfs_probe_weight_attr =
+		__ATTR_RW_MODE(weight, 0600);
+
 static struct attribute *damon_sysfs_probe_attrs[] = {
+	&damon_sysfs_probe_weight_attr.attr,
 	NULL,
 };
 ATTRIBUTE_GROUPS(damon_sysfs_probe);
@@ -1134,6 +1163,7 @@ static void damon_sysfs_probes_rm_dirs(
 
 	for (i = 0; i < probes->nr; i++) {
 		damon_sysfs_probe_rm_dirs(probes_arr[i]);
+		kobject_del(&probes_arr[i]->kobj);
 		kobject_put(&probes_arr[i]->kobj);
 	}
 	probes->nr = 0;
@@ -1175,6 +1205,7 @@ static int damon_sysfs_probes_add_dirs(
 
 		err = damon_sysfs_probe_add_dirs(probe);
 		if (err) {
+			kobject_del(&probe->kobj);
 			kobject_put(&probe->kobj);
 			damon_sysfs_probes_rm_dirs(probes);
 			return err;
@@ -1642,6 +1673,7 @@ static void damon_sysfs_contexts_rm_dirs(struct damon_sysfs_contexts *contexts)
 
 	for (i = 0; i < contexts->nr; i++) {
 		damon_sysfs_context_rm_dirs(contexts_arr[i]);
+		kobject_del(&contexts_arr[i]->kobj);
 		kobject_put(&contexts_arr[i]->kobj);
 	}
 	contexts->nr = 0;
@@ -1680,13 +1712,15 @@ static int damon_sysfs_contexts_add_dirs(struct damon_sysfs_contexts *contexts,
 
 		err = damon_sysfs_context_add_dirs(context);
 		if (err)
-			goto out;
+			goto del_out;
 
 		contexts_arr[i] = context;
 		contexts->nr++;
 	}
 	return 0;
 
+del_out:
+	kobject_del(&context->kobj);
 out:
 	damon_sysfs_contexts_rm_dirs(contexts);
 	kobject_put(&context->kobj);
@@ -1899,47 +1933,66 @@ static int damon_sysfs_set_attrs(struct damon_ctx *ctx,
 	return damon_set_attrs(ctx, &attrs);
 }
 
-static int damon_sysfs_set_probes(struct damon_ctx *ctx,
-		struct damon_sysfs_probes *sys_probes)
+static int damon_sysfs_set_filters(struct damon_probe *probe,
+		struct damon_sysfs_filters *sys_filters)
 {
 	int i;
 
-	for (i = 0; i < sys_probes->nr; i++) {
-		struct damon_sysfs_filters *sys_filters =
-			sys_probes->probes_arr[i]->filters;
-		struct damon_probe *c;
-		int j;
+	for (i = 0; i < sys_filters->nr; i++) {
+		struct damon_sysfs_filter *sys_filter =
+			sys_filters->filters_arr[i];
+		struct damon_filter *filter;
 
-		if (!sys_filters)
-			continue;
-		c = damon_new_probe();
-		if (!c)
+		filter = damon_new_filter(sys_filter->type,
+				sys_filter->matching,
+				sys_filter->allow);
+		if (!filter)
 			return -ENOMEM;
-		damon_add_probe(ctx, c);
-
-		for (j = 0; j < sys_filters->nr; j++) {
-			struct damon_sysfs_filter *sys_filter =
-				sys_filters->filters_arr[j];
-			struct damon_filter *filter;
-
-			filter = damon_new_filter(sys_filter->type,
-					sys_filter->matching,
-					sys_filter->allow);
-			if (!filter)
-				return -ENOMEM;
-			if (filter->type == DAMON_FILTER_TYPE_MEMCG) {
-				int err;
-
-				err = damon_sysfs_memcg_path_to_id(
-						sys_filter->path,
-						&filter->memcg_id);
-				if (err) {
-					damon_destroy_filter(filter);
-					return err;
-				}
+		if (filter->type == DAMON_FILTER_TYPE_MEMCG) {
+			int err;
+
+			err = damon_sysfs_memcg_path_to_id(
+					sys_filter->path,
+					&filter->memcg_id);
+			if (err) {
+				damon_destroy_filter(filter);
+				return err;
 			}
-			damon_add_filter(c, filter);
 		}
+		damon_add_filter(probe, filter);
+	}
+	return 0;
+}
+
+static int damon_sysfs_set_probe(struct damon_probe *probe,
+		struct damon_sysfs_probe *sys_probe)
+{
+	struct damon_sysfs_filters *sys_filters;
+
+	sys_filters = sys_probe->filters;
+	if (!sys_filters)
+		return 0;
+	return damon_sysfs_set_filters(probe, sys_filters);
+}
+
+static int damon_sysfs_set_probes(struct damon_ctx *ctx,
+		struct damon_sysfs_probes *sys_probes)
+{
+	int i, err;
+
+	for (i = 0; i < sys_probes->nr; i++) {
+		struct damon_sysfs_probe *sys_probe;
+		struct damon_probe *p;
+
+		p = damon_new_probe();
+		if (!p)
+			return -ENOMEM;
+		damon_add_probe(ctx, p);
+		sys_probe = sys_probes->probes_arr[i];
+		p->weight = sys_probe->weight;
+		err = damon_sysfs_set_probe(p, sys_probe);
+		if (err)
+			return err;
 	}
 	return 0;
 }
@@ -1959,9 +2012,6 @@ static int damon_sysfs_set_regions(struct damon_target *t,
 		struct damon_sysfs_region *sys_region =
 			sysfs_regions->regions_arr[i];
 
-		if (sys_region->ar.start > sys_region->ar.end)
-			goto out;
-
 		ranges[i].start = sys_region->ar.start;
 		ranges[i].end = sys_region->ar.end;
 		if (i == 0)
@@ -2001,7 +2051,7 @@ static int damon_sysfs_add_targets(struct damon_ctx *ctx,
 	int i, err;
 
 	/* Multiple physical address space monitoring targets makes no sense */
-	if (ctx->ops.id == DAMON_OPS_PADDR && sysfs_targets->nr > 1)
+	if (!damon_target_has_pid(ctx) && sysfs_targets->nr > 1)
 		return -EINVAL;
 
 	for (i = 0; i < sysfs_targets->nr; i++) {
@@ -2044,16 +2094,18 @@ static inline bool damon_sysfs_kdamond_running(
 static int damon_sysfs_apply_inputs(struct damon_ctx *ctx,
 		struct damon_sysfs_context *sys_ctx)
 {
+	enum damon_ops_id ops_id;
 	int err;
 
-	err = damon_select_ops(ctx, sys_ctx->ops_id);
+	ops_id = READ_ONCE(sys_ctx->ops_id);
+	err = damon_select_ops(ctx, ops_id);
 	if (err)
 		return err;
-	ctx->addr_unit = sys_ctx->addr_unit;
+	ctx->addr_unit = READ_ONCE(sys_ctx->addr_unit);
 	/* addr_unit is respected by only DAMON_OPS_PADDR */
-	if (sys_ctx->ops_id == DAMON_OPS_PADDR)
+	if (ops_id == DAMON_OPS_PADDR)
 		ctx->min_region_sz = max(
-				DAMON_MIN_REGION_SZ / sys_ctx->addr_unit, 1);
+				DAMON_MIN_REGION_SZ / ctx->addr_unit, 1);
 	ctx->pause = sys_ctx->pause;
 	err = damon_sysfs_set_attrs(ctx, sys_ctx->attrs);
 	if (err)
@@ -2070,26 +2122,6 @@ static int damon_sysfs_apply_inputs(struct damon_ctx *ctx,
 static struct damon_ctx *damon_sysfs_build_ctx(
 		struct damon_sysfs_context *sys_ctx);
 
-/*
- * Return a new damon_ctx for testing new parameters to commit.
- */
-static struct damon_ctx *damon_sysfs_new_test_ctx(
-		struct damon_ctx *running_ctx)
-{
-	struct damon_ctx *test_ctx;
-	int err;
-
-	test_ctx = damon_new_ctx();
-	if (!test_ctx)
-		return NULL;
-	err = damon_commit_ctx(test_ctx, running_ctx);
-	if (err) {
-		damon_destroy_ctx(test_ctx);
-		return NULL;
-	}
-	return test_ctx;
-}
-
 /*
  * damon_sysfs_commit_input() - Commit user inputs to a running kdamond.
  * @kdamond:	The kobject wrapper for the associated kdamond.
@@ -2099,7 +2131,7 @@ static struct damon_ctx *damon_sysfs_new_test_ctx(
 static int damon_sysfs_commit_input(void *data)
 {
 	struct damon_sysfs_kdamond *kdamond = data;
-	struct damon_ctx *param_ctx, *test_ctx;
+	struct damon_ctx *param_ctx;
 	int err;
 
 	if (!damon_sysfs_kdamond_running(kdamond))
@@ -2111,17 +2143,7 @@ static int damon_sysfs_commit_input(void *data)
 	param_ctx = damon_sysfs_build_ctx(kdamond->contexts->contexts_arr[0]);
 	if (IS_ERR(param_ctx))
 		return PTR_ERR(param_ctx);
-	test_ctx = damon_sysfs_new_test_ctx(kdamond->damon_ctx);
-	if (!test_ctx) {
-		damon_destroy_ctx(param_ctx);
-		return -ENOMEM;
-	}
-	err = damon_commit_ctx(test_ctx, param_ctx);
-	if (err)
-		goto out;
 	err = damon_commit_ctx(kdamond->damon_ctx, param_ctx);
-out:
-	damon_destroy_ctx(test_ctx);
 	damon_destroy_ctx(param_ctx);
 	return err;
 }
@@ -2267,12 +2289,13 @@ static int damon_sysfs_turn_damon_off(struct damon_sysfs_kdamond *kdamond)
 {
 	if (!kdamond->damon_ctx)
 		return -EINVAL;
-	return damon_stop(&kdamond->damon_ctx, 1);
+	damon_stop(&kdamond->damon_ctx, 1);
 	/*
 	 * To allow users show final monitoring results of already turned-off
 	 * DAMON, we free kdamond->damon_ctx in next
 	 * damon_sysfs_turn_damon_on(), or kdamonds_nr_store()
 	 */
+	return 0;
 }
 
 static int damon_sysfs_damon_call(int (*fn)(void *data),
@@ -2501,6 +2524,7 @@ static void damon_sysfs_kdamonds_rm_dirs(struct damon_sysfs_kdamonds *kdamonds)
 
 	for (i = 0; i < kdamonds->nr; i++) {
 		damon_sysfs_kdamond_rm_dirs(kdamonds_arr[i]);
+		kobject_del(&kdamonds_arr[i]->kobj);
 		kobject_put(&kdamonds_arr[i]->kobj);
 	}
 	kdamonds->nr = 0;
@@ -2555,13 +2579,15 @@ static int damon_sysfs_kdamonds_add_dirs(struct damon_sysfs_kdamonds *kdamonds,
 
 		err = damon_sysfs_kdamond_add_dirs(kdamond);
 		if (err)
-			goto out;
+			goto del_out;
 
 		kdamonds_arr[i] = kdamond;
 		kdamonds->nr++;
 	}
 	return 0;
 
+del_out:
+	kobject_del(&kdamond->kobj);
 out:
 	damon_sysfs_kdamonds_rm_dirs(kdamonds);
 	kobject_put(&kdamond->kobj);
diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h
index 1cfb8c176b873..eba643762132f 100644
--- a/mm/damon/tests/core-kunit.h
+++ b/mm/damon/tests/core-kunit.h
@@ -1,10 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Data Access Monitor Unit Tests
- *
- * Copyright 2019 Amazon.com, Inc. or its affiliates.  All rights reserved.
- *
- * Author: SeongJae Park <sj@kernel.org>
  */
 
 #ifdef CONFIG_DAMON_KUNIT_TEST
@@ -118,7 +114,6 @@ static void damon_test_aggregate(struct kunit *test)
 				kunit_skip(test, "region alloc fail");
 			}
 			r->nr_accesses = accesses[it][ir];
-			r->nr_accesses_bp = accesses[it][ir] * 10000;
 			damon_add_region(r, t);
 		}
 		it++;
@@ -155,7 +150,6 @@ static void damon_test_split_at(struct kunit *test)
 		damon_free_target(t);
 		kunit_skip(test, "region alloc fail");
 	}
-	r->nr_accesses_bp = 420000;
 	r->nr_accesses = 42;
 	r->last_nr_accesses = 15;
 	r->age = 10;
@@ -168,7 +162,6 @@ static void damon_test_split_at(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, r_new->ar.start, 25ul);
 	KUNIT_EXPECT_EQ(test, r_new->ar.end, 100ul);
 
-	KUNIT_EXPECT_EQ(test, r->nr_accesses_bp, r_new->nr_accesses_bp);
 	KUNIT_EXPECT_EQ(test, r->nr_accesses, r_new->nr_accesses);
 	KUNIT_EXPECT_EQ(test, r->last_nr_accesses, r_new->last_nr_accesses);
 	KUNIT_EXPECT_EQ(test, r->age, r_new->age);
@@ -191,7 +184,6 @@ static void damon_test_merge_two(struct kunit *test)
 		kunit_skip(test, "region alloc fail");
 	}
 	r->nr_accesses = 10;
-	r->nr_accesses_bp = 100000;
 	r->age = 9;
 	damon_add_region(r, t);
 	r2 = damon_new_region(100, 300);
@@ -200,7 +192,6 @@ static void damon_test_merge_two(struct kunit *test)
 		kunit_skip(test, "second region alloc fail");
 	}
 	r2->nr_accesses = 20;
-	r2->nr_accesses_bp = 200000;
 	r2->age = 21;
 	damon_add_region(r2, t);
 
@@ -208,7 +199,6 @@ static void damon_test_merge_two(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, r->ar.start, 0ul);
 	KUNIT_EXPECT_EQ(test, r->ar.end, 300ul);
 	KUNIT_EXPECT_EQ(test, r->nr_accesses, 16u);
-	KUNIT_EXPECT_EQ(test, r->nr_accesses_bp, 160000u);
 	KUNIT_EXPECT_EQ(test, r->age, 17u);
 
 	i = 0;
@@ -236,6 +226,7 @@ static struct damon_region *__nth_region_of(struct damon_target *t, int idx)
 
 static void damon_test_merge_regions_of(struct kunit *test)
 {
+	struct damon_ctx *ctx;
 	struct damon_target *t;
 	struct damon_region *r;
 	unsigned long sa[] = {0, 100, 114, 122, 130, 156, 170, 184, 230};
@@ -246,29 +237,39 @@ static void damon_test_merge_regions_of(struct kunit *test)
 	unsigned long eaddrs[] = {112, 130, 156, 170, 230, 10170};
 	int i;
 
+	ctx = damon_new_ctx();
+	if (!ctx)
+		kunit_skip(test, "ctx alloc fail");
+
 	t = damon_new_target();
-	if (!t)
+	if (!t) {
+		damon_destroy_ctx(ctx);
 		kunit_skip(test, "target alloc fail");
+	}
 	for (i = 0; i < ARRAY_SIZE(sa); i++) {
 		r = damon_new_region(sa[i], ea[i]);
 		if (!r) {
 			damon_free_target(t);
+			damon_destroy_ctx(ctx);
 			kunit_skip(test, "region alloc fail");
 		}
 		r->nr_accesses = nrs[i];
-		r->nr_accesses_bp = nrs[i] * 10000;
 		damon_add_region(r, t);
 	}
 
-	damon_merge_regions_of(t, 9, 9999);
+	damon_merge_regions_of(t, 9, 9999, ctx, true);
 	/* 0-112, 114-130, 130-156, 156-170, 170-230, 230-10170 */
 	KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 6u);
+	if (damon_nr_regions(t) != 6)
+		goto out;
 	for (i = 0; i < 6; i++) {
 		r = __nth_region_of(t, i);
 		KUNIT_EXPECT_EQ(test, r->ar.start, saddrs[i]);
 		KUNIT_EXPECT_EQ(test, r->ar.end, eaddrs[i]);
 	}
+out:
 	damon_free_target(t);
+	damon_destroy_ctx(ctx);
 }
 
 static void damon_test_split_regions_of(struct kunit *test)
@@ -339,6 +340,69 @@ static void damon_test_split_regions_of(struct kunit *test)
 	damon_destroy_ctx(c);
 }
 
+/*
+ * When the total region count is already above max_nr_regions / 2,
+ * kdamond_split_regions() must keep refining the resolution by splitting a
+ * fraction of the regions (making progress), without exceeding
+ * max_nr_regions.
+ */
+static void damon_test_split_above_half_progresses(struct kunit *test)
+{
+	struct damon_ctx *c;
+	struct damon_target *t;
+	struct damon_region *r;
+	unsigned long start;
+	unsigned int nr_before, nr_after, i;
+	const unsigned int nr_init = 760;
+	const unsigned long region_sz = 100;
+
+	c = damon_new_ctx();
+	if (!c)
+		kunit_skip(test, "ctx alloc fail");
+
+	/* Keep the split arithmetic independent of the page size */
+	c->min_region_sz = 1;
+	c->attrs.min_nr_regions = 10;
+	c->attrs.max_nr_regions = 1500;
+
+	t = damon_new_target();
+	if (!t) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "target alloc fail");
+	}
+
+	for (i = 0; i < nr_init; i++) {
+		start = i * region_sz;
+		r = damon_new_region(start, start + region_sz);
+		if (!r) {
+			damon_free_target(t);
+			damon_destroy_ctx(c);
+			kunit_skip(test, "region alloc fail");
+		}
+		r->nr_accesses = (i & 1) ? 0 : 100;
+		r->age = 5;
+		damon_add_region(r, t);
+	}
+
+	damon_add_target(c, t);
+
+	nr_before = damon_nr_regions(t);
+	/* Above max_nr_regions / 2, so the blanket-split path is skipped */
+	KUNIT_EXPECT_GT(test, (unsigned long)nr_before,
+			c->attrs.max_nr_regions / 2);
+
+	kdamond_split_regions(c);
+
+	nr_after = damon_nr_regions(t);
+	/* Still made progress ... */
+	KUNIT_EXPECT_GT(test, nr_after, nr_before);
+	/* ... but did not overshoot the configured maximum */
+	KUNIT_EXPECT_LE(test, (unsigned long)nr_after,
+			c->attrs.max_nr_regions);
+
+	damon_destroy_ctx(c);
+}
+
 static void damon_test_ops_registration(struct kunit *test)
 {
 	struct damon_ctx *c = damon_new_ctx();
@@ -523,61 +587,45 @@ static void damon_test_set_regions(struct kunit *test)
 			}, 3);
 }
 
-static void damon_test_nr_accesses_to_accesses_bp(struct kunit *test)
-{
-	struct damon_attrs attrs = {
-		.sample_interval = 10,
-		.aggr_interval = ((unsigned long)UINT_MAX + 1) * 10
-	};
-
-	/*
-	 * In some cases such as 32bit architectures where UINT_MAX is
-	 * ULONG_MAX, attrs.aggr_interval becomes zero.  Calling
-	 * damon_nr_accesses_to_accesses_bp() in the case will cause
-	 * divide-by-zero.  Such case is prohibited in normal execution since
-	 * the caution is documented on the comment for the function, and
-	 * damon_update_monitoring_results() does the check.  Skip the test in
-	 * the case.
-	 */
-	if (!attrs.aggr_interval)
-		kunit_skip(test, "aggr_interval is zero.");
-
-	KUNIT_EXPECT_EQ(test, damon_nr_accesses_to_accesses_bp(123, &attrs), 0);
-}
-
 static void damon_test_update_monitoring_result(struct kunit *test)
 {
 	struct damon_attrs old_attrs = {
 		.sample_interval = 10, .aggr_interval = 1000,};
 	struct damon_attrs new_attrs;
 	struct damon_region *r = damon_new_region(3, 7);
+	struct damon_ctx *ctx;
 
 	if (!r)
 		kunit_skip(test, "region alloc fail");
+	ctx = damon_new_ctx();
+	if (!ctx) {
+		damon_free_region(r);
+		kunit_skip(test, "ctx alloc fail");
+	}
 
 	r->nr_accesses = 15;
-	r->nr_accesses_bp = 150000;
 	r->age = 20;
 
 	new_attrs = (struct damon_attrs){
 		.sample_interval = 100, .aggr_interval = 10000,};
-	damon_update_monitoring_result(r, &old_attrs, &new_attrs, false);
+	damon_update_monitoring_result(r, &old_attrs, &new_attrs, false, ctx);
 	KUNIT_EXPECT_EQ(test, r->nr_accesses, 15);
 	KUNIT_EXPECT_EQ(test, r->age, 2);
 
 	new_attrs = (struct damon_attrs){
 		.sample_interval = 1, .aggr_interval = 1000};
-	damon_update_monitoring_result(r, &old_attrs, &new_attrs, false);
+	damon_update_monitoring_result(r, &old_attrs, &new_attrs, false, ctx);
 	KUNIT_EXPECT_EQ(test, r->nr_accesses, 150);
 	KUNIT_EXPECT_EQ(test, r->age, 2);
 
 	new_attrs = (struct damon_attrs){
 		.sample_interval = 1, .aggr_interval = 100};
-	damon_update_monitoring_result(r, &old_attrs, &new_attrs, false);
+	damon_update_monitoring_result(r, &old_attrs, &new_attrs, false, ctx);
 	KUNIT_EXPECT_EQ(test, r->nr_accesses, 150);
 	KUNIT_EXPECT_EQ(test, r->age, 20);
 
 	damon_free_region(r);
+	damon_destroy_ctx(ctx);
 }
 
 static void damon_test_set_attrs(struct kunit *test)
@@ -608,19 +656,76 @@ static void damon_test_set_attrs(struct kunit *test)
 	damon_destroy_ctx(c);
 }
 
-static void damon_test_moving_sum(struct kunit *test)
+static void damon_test_mvsum(struct kunit *test)
+{
+	unsigned long input_expects[] = {
+		/* current value, last value, remaining window (bp) */
+		0, 49, 10000, 49,	/* 0 + 49 * 1 */
+		3, 10, 7000, 10,	/* 3 + 10 * 0.7 */
+		3, 10, 5000, 8,		/* 3 + 10 * 0.5 */
+		32, 100, 1000, 42,	/* 32 + 100 * 0.1 */
+		42, 49, 0, 42,		/* 42 + 49 * 0 */
+	};
+
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(input_expects); i += 4) {
+		unsigned long current_nr = input_expects[i];
+		unsigned long last_nr = input_expects[i + 1];
+		unsigned long left_window_bp = input_expects[i + 2];
+		unsigned long expect = input_expects[i + 3];
+
+		KUNIT_EXPECT_EQ(test, damon_mvsum(current_nr, last_nr,
+					left_window_bp), expect);
+	}
+}
+
+/*
+ * Test damon_nr_accesses_mvsum(), which wraps damon_mvsum() with the
+ * monitoring intervals of the context.  With a sample interval of 1 and an
+ * aggregation interval of 10, an aggregation window is 10 sample intervals
+ * long.  Each row below specifies the passed sample intervals, the next
+ * aggregation time in sample intervals, the current and last nr_accesses of a
+ * region, and the expected return value.
+ */
+static void damon_test_nr_accesses_mvsum(struct kunit *test)
 {
-	unsigned int mvsum = 50000, nomvsum = 50000, len_window = 10;
-	unsigned int new_values[] = {10000, 0, 10000, 0, 0, 0, 10000, 0, 0, 0};
-	unsigned int expects[] = {55000, 50000, 55000, 50000, 45000, 40000,
-		45000, 40000, 35000, 30000};
+	unsigned long input_expects[] = {
+		/* passed, next_aggr, nr_accesses, last_nr_accesses, expect */
+		0, 10, 5, 3, 3,		/* full window left, unreset */
+		0, 10, 0, 7, 7,		/* full window left, reset */
+		5, 10, 3, 10, 8,	/* half window left */
+		8, 10, 3, 10, 5,	/* 20% window left */
+		10, 10, 42, 49, 42,	/* no window left */
+	};
+	struct damon_ctx *c = damon_new_ctx();
+	struct damon_region *r;
 	int i;
 
-	for (i = 0; i < ARRAY_SIZE(new_values); i++) {
-		mvsum = damon_moving_sum(mvsum, nomvsum, len_window,
-				new_values[i]);
-		KUNIT_EXPECT_EQ(test, mvsum, expects[i]);
+	if (!c)
+		kunit_skip(test, "ctx alloc fail");
+
+	r = damon_new_region(0, 4096);
+	if (!r) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "region alloc fail");
+	}
+
+	c->attrs.sample_interval = 1;
+	c->attrs.aggr_interval = 10;
+
+	for (i = 0; i < ARRAY_SIZE(input_expects); i += 5) {
+		c->passed_sample_intervals = input_expects[i];
+		c->next_aggregation_sis = input_expects[i + 1];
+		r->nr_accesses = input_expects[i + 2];
+		r->last_nr_accesses = input_expects[i + 3];
+
+		KUNIT_EXPECT_EQ(test, (unsigned int)input_expects[i + 4],
+				damon_nr_accesses_mvsum(r, c));
 	}
+
+	damon_free_region(r);
+	damon_destroy_ctx(c);
 }
 
 static void damos_test_new_filter(struct kunit *test)
@@ -1460,6 +1565,47 @@ static void damon_test_is_last_region(struct kunit *test)
 	damon_free_target(t);
 }
 
+/*
+ * Verify that damos_walk() rejects new requests when
+ * walk_control_obsolete is set.
+ *
+ * This tests the invariant introduced by:
+ * commit 33c3f6c2b48c ("mm/damon/core: fix damos_walk() vs kdamond_fn() exit race")
+ */
+static void damon_test_walk_control_obsolete(struct kunit *test)
+{
+	struct damon_ctx *ctx;
+	struct damos_walk_control control = {};
+	int ret;
+
+	ctx = damon_new_ctx();
+	if (!ctx)
+		kunit_skip(test, "ctx alloc fail");
+
+	/* Simulate shutdown phase */
+	ctx->walk_control_obsolete = true;
+
+	ret = damos_walk(ctx, &control);
+
+	KUNIT_EXPECT_EQ(test, ret, -ECANCELED);
+
+	damon_destroy_ctx(ctx);
+}
+
+static void damon_test_rand(struct kunit *test)
+{
+	struct damon_ctx ctx;
+	int i;
+
+	prandom_seed_state(&ctx.rnd_state, get_random_u64());
+	for (i = 0; i < 10000; i++) {
+		unsigned long rnd = damon_rand(&ctx, 0, 10);
+
+		KUNIT_EXPECT_GE(test, rnd, 0);
+		KUNIT_EXPECT_LE(test, rnd, 9);
+	}
+}
+
 static struct kunit_case damon_test_cases[] = {
 	KUNIT_CASE(damon_test_target),
 	KUNIT_CASE(damon_test_regions),
@@ -1468,12 +1614,13 @@ static struct kunit_case damon_test_cases[] = {
 	KUNIT_CASE(damon_test_merge_two),
 	KUNIT_CASE(damon_test_merge_regions_of),
 	KUNIT_CASE(damon_test_split_regions_of),
+	KUNIT_CASE(damon_test_split_above_half_progresses),
 	KUNIT_CASE(damon_test_ops_registration),
 	KUNIT_CASE(damon_test_set_regions),
-	KUNIT_CASE(damon_test_nr_accesses_to_accesses_bp),
 	KUNIT_CASE(damon_test_update_monitoring_result),
 	KUNIT_CASE(damon_test_set_attrs),
-	KUNIT_CASE(damon_test_moving_sum),
+	KUNIT_CASE(damon_test_mvsum),
+	KUNIT_CASE(damon_test_nr_accesses_mvsum),
 	KUNIT_CASE(damos_test_new_filter),
 	KUNIT_CASE(damos_test_commit_quota_goal),
 	KUNIT_CASE(damos_test_commit_quota_goals),
@@ -1489,6 +1636,8 @@ static struct kunit_case damon_test_cases[] = {
 	KUNIT_CASE(damon_test_set_filters_default_reject),
 	KUNIT_CASE(damon_test_apply_min_nr_regions),
 	KUNIT_CASE(damon_test_is_last_region),
+	KUNIT_CASE(damon_test_walk_control_obsolete),
+	KUNIT_CASE(damon_test_rand),
 	{},
 };
 
diff --git a/mm/damon/tests/sysfs-kunit.h b/mm/damon/tests/sysfs-kunit.h
index f9ec5e795b34b..138a4b8d14e73 100644
--- a/mm/damon/tests/sysfs-kunit.h
+++ b/mm/damon/tests/sysfs-kunit.h
@@ -1,8 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Data Access Monitor Unit Tests
- *
- * Author: SeongJae Park <sj@kernel.org>
  */
 
 #ifdef CONFIG_DAMON_SYSFS_KUNIT_TEST
diff --git a/mm/damon/tests/vaddr-kunit.h b/mm/damon/tests/vaddr-kunit.h
index 563fbc7e3f448..61f844336ffb5 100644
--- a/mm/damon/tests/vaddr-kunit.h
+++ b/mm/damon/tests/vaddr-kunit.h
@@ -1,10 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Data Access Monitor Unit Tests
- *
- * Copyright 2019 Amazon.com, Inc. or its affiliates.  All rights reserved.
- *
- * Author: SeongJae Park <sj@kernel.org>
  */
 
 #ifdef CONFIG_DAMON_VADDR_KUNIT_TEST
diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c
index d271476035641..d487b7a4a1042 100644
--- a/mm/damon/vaddr.c
+++ b/mm/damon/vaddr.c
@@ -1,8 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * DAMON Code for Virtual Address Spaces
- *
- * Author: SeongJae Park <sj@kernel.org>
  */
 
 #define pr_fmt(fmt) "damon-va: " fmt
@@ -12,6 +10,7 @@
 #include <linux/mman.h>
 #include <linux/mmu_notifier.h>
 #include <linux/page_idle.h>
+#include <linux/pagemap.h>
 #include <linux/pagewalk.h>
 #include <linux/sched/mm.h>
 
@@ -384,8 +383,6 @@ static void damon_va_prepare_access_checks(struct damon_ctx *ctx)
 }
 
 struct damon_young_walk_private {
-	/* size of the folio for the access checked virtual memory address */
-	unsigned long *folio_sz;
 	bool young;
 };
 
@@ -412,7 +409,6 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr,
 					mmu_notifier_test_young(walk->mm,
 						addr))
 			priv->young = true;
-		*priv->folio_sz = HPAGE_PMD_SIZE;
 huge_out:
 		spin_unlock(ptl);
 		return 0;
@@ -431,7 +427,6 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr,
 	if (pte_young(ptent) || !folio_test_idle(folio) ||
 			mmu_notifier_test_young(walk->mm, addr))
 		priv->young = true;
-	*priv->folio_sz = folio_size(folio);
 out:
 	pte_unmap_unlock(pte, ptl);
 	return 0;
@@ -459,7 +454,6 @@ static int damon_young_hugetlb_entry(pte_t *pte, unsigned long hmask,
 	if (pte_young(entry) || !folio_test_idle(folio) ||
 	    mmu_notifier_test_young(walk->mm, addr))
 		priv->young = true;
-	*priv->folio_sz = huge_page_size(h);
 
 	folio_put(folio);
 
@@ -471,11 +465,9 @@ static int damon_young_hugetlb_entry(pte_t *pte, unsigned long hmask,
 #define damon_young_hugetlb_entry NULL
 #endif /* CONFIG_HUGETLB_PAGE */
 
-static bool damon_va_young(struct mm_struct *mm, unsigned long addr,
-		unsigned long *folio_sz)
+static bool damon_va_young(struct mm_struct *mm, unsigned long addr)
 {
 	struct damon_young_walk_private arg = {
-		.folio_sz = folio_sz,
 		.young = false,
 	};
 
@@ -495,29 +487,17 @@ static bool damon_va_young(struct mm_struct *mm, unsigned long addr,
  * r	the region to be checked
  */
 static void __damon_va_check_access(struct mm_struct *mm,
-				struct damon_region *r, bool same_target,
-				struct damon_attrs *attrs)
+				struct damon_region *r)
 {
-	static unsigned long last_addr;
-	static unsigned long last_folio_sz = PAGE_SIZE;
-	static bool last_accessed;
+	bool accessed;
 
 	if (!mm) {
-		damon_update_region_access_rate(r, false, attrs);
+		damon_update_region_access_rate(r, false);
 		return;
 	}
 
-	/* If the region is in the last checked page, reuse the result */
-	if (same_target && (ALIGN_DOWN(last_addr, last_folio_sz) ==
-				ALIGN_DOWN(r->sampling_addr, last_folio_sz))) {
-		damon_update_region_access_rate(r, last_accessed, attrs);
-		return;
-	}
-
-	last_accessed = damon_va_young(mm, r->sampling_addr, &last_folio_sz);
-	damon_update_region_access_rate(r, last_accessed, attrs);
-
-	last_addr = r->sampling_addr;
+	accessed = damon_va_young(mm, r->sampling_addr);
+	damon_update_region_access_rate(r, accessed);
 }
 
 static unsigned int damon_va_check_accesses(struct damon_ctx *ctx)
@@ -526,16 +506,12 @@ static unsigned int damon_va_check_accesses(struct damon_ctx *ctx)
 	struct mm_struct *mm;
 	struct damon_region *r;
 	unsigned int max_nr_accesses = 0;
-	bool same_target;
 
 	damon_for_each_target(t, ctx) {
 		mm = damon_get_mm(t);
-		same_target = false;
 		damon_for_each_region(r, t) {
-			__damon_va_check_access(mm, r, same_target,
-					&ctx->attrs);
+			__damon_va_check_access(mm, r);
 			max_nr_accesses = max(r->nr_accesses, max_nr_accesses);
-			same_target = true;
 		}
 		if (mm)
 			mmput(mm);
@@ -627,8 +603,8 @@ static void damos_va_migrate_dests_add(struct folio *folio,
 	}
 
 	order = folio_order(folio);
-	ilx = vma->vm_pgoff >> order;
-	ilx += (addr - vma->vm_start) >> (PAGE_SHIFT + order);
+	ilx = vma_start_pgoff(vma) >> order;
+	ilx += linear_page_delta(vma, addr) >> order;
 
 	for (i = 0; i < dests->nr_dests; i++)
 		weight_total += dests->weight_arr[i];
diff --git a/mm/debug.c b/mm/debug.c
index 77fa8fe1d6414..9a0297b3988d8 100644
--- a/mm/debug.c
+++ b/mm/debug.c
@@ -163,7 +163,7 @@ void dump_vma(const struct vm_area_struct *vma)
 		"flags: %#lx(%pGv)\n",
 		vma, (void *)vma->vm_start, (void *)vma->vm_end, vma->vm_mm,
 		(unsigned long)pgprot_val(vma->vm_page_prot),
-		vma->anon_vma, vma->vm_ops, vma->vm_pgoff,
+		vma->anon_vma, vma->vm_ops, vma_start_pgoff(vma),
 		vma->vm_file, vma->vm_private_data,
 #ifdef CONFIG_PER_VMA_LOCK
 		refcount_read(&vma->vm_refcnt),
@@ -197,7 +197,7 @@ void dump_mm(const struct mm_struct *mm)
 		"numa_next_scan %lu numa_scan_offset %lu numa_scan_seq %d\n"
 #endif
 		"tlb_flush_pending %d\n"
-		"def_flags: %#lx(%pGv)\n",
+		"def_flags: %*pb(%pGv)\n",
 
 		mm, mm->task_size,
 		mm->mmap_base, mm->mmap_legacy_base,
@@ -226,7 +226,8 @@ void dump_mm(const struct mm_struct *mm)
 		mm->numa_next_scan, mm->numa_scan_offset, mm->numa_scan_seq,
 #endif
 		atomic_read(&mm->tlb_flush_pending),
-		mm->def_flags, &mm->def_flags
+		NUM_VMA_FLAG_BITS, mm->def_vma_flags.__vma_flags,
+		&mm->def_vma_flags
 	);
 }
 EXPORT_SYMBOL(dump_mm);
diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c
index 23dc3ee095619..2875fd22d7bb0 100644
--- a/mm/debug_vm_pgtable.c
+++ b/mm/debug_vm_pgtable.c
@@ -672,7 +672,7 @@ static void __init pte_protnone_tests(struct pgtable_debug_args *args)
 {
 	pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot_none);
 
-	if (!IS_ENABLED(CONFIG_NUMA_BALANCING))
+	if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE))
 		return;
 
 	pr_debug("Validating PTE protnone\n");
@@ -685,7 +685,7 @@ static void __init pmd_protnone_tests(struct pgtable_debug_args *args)
 {
 	pmd_t pmd;
 
-	if (!IS_ENABLED(CONFIG_NUMA_BALANCING))
+	if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE))
 		return;
 
 	if (!has_transparent_hugepage())
@@ -751,14 +751,14 @@ static void __init pmd_leaf_soft_dirty_tests(struct pgtable_debug_args *args)
 	pmd_t pmd;
 
 	if (!pgtable_supports_soft_dirty() ||
-	    !IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION))
+	    !IS_ENABLED(CONFIG_ARCH_HAS_PMD_SOFTLEAVES))
 		return;
 
 	if (!has_transparent_hugepage())
 		return;
 
 	pr_debug("Validating PMD swap soft dirty\n");
-	pmd = swp_entry_to_pmd(args->leaf_entry);
+	pmd = softleaf_to_pmd(args->leaf_entry);
 	WARN_ON(!pmd_is_huge(pmd));
 	WARN_ON(!pmd_is_valid_softleaf(pmd));
 
@@ -819,7 +819,7 @@ static void __init pte_swap_tests(struct pgtable_debug_args *args)
 	WARN_ON(memcmp(&pte1, &pte2, sizeof(pte1)));
 }
 
-#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
 static void __init pmd_softleaf_tests(struct pgtable_debug_args *args)
 {
 	swp_entry_t arch_entry;
@@ -829,7 +829,7 @@ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args)
 		return;
 
 	pr_debug("Validating PMD swap\n");
-	pmd1 = swp_entry_to_pmd(args->leaf_entry);
+	pmd1 = softleaf_to_pmd(args->leaf_entry);
 	WARN_ON(!pmd_is_huge(pmd1));
 	WARN_ON(!pmd_is_valid_softleaf(pmd1));
 
@@ -837,9 +837,9 @@ static void __init pmd_softleaf_tests(struct pgtable_debug_args *args)
 	pmd2 = __swp_entry_to_pmd(arch_entry);
 	WARN_ON(memcmp(&pmd1, &pmd2, sizeof(pmd1)));
 }
-#else  /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */
+#else  /* !CONFIG_ARCH_HAS_PMD_SOFTLEAVES */
 static void __init pmd_softleaf_tests(struct pgtable_debug_args *args) { }
-#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
+#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */
 
 static void __init swap_migration_tests(struct pgtable_debug_args *args)
 {
diff --git a/mm/execmem.c b/mm/execmem.c
index 084a207e4278b..74a178a87e758 100644
--- a/mm/execmem.c
+++ b/mm/execmem.c
@@ -20,6 +20,7 @@
 #include <asm/tlbflush.h>
 
 #include "internal.h"
+#include "vmalloc.h"
 
 static struct execmem_info *execmem_info __ro_after_init;
 static struct execmem_info default_execmem_info __ro_after_init;
diff --git a/mm/filemap.c b/mm/filemap.c
index 58eb9d2406434..1dbb4c6f824e9 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -2270,10 +2270,11 @@ unsigned filemap_get_folios_contig(struct address_space *mapping,
 	unsigned long nr;
 	struct folio *folio;
 
-	rcu_read_lock();
+	if (*start > end)
+		return 0;
 
-	for (folio = xas_load(&xas); folio && xas.xa_index <= end;
-			folio = xas_next(&xas)) {
+	rcu_read_lock();
+	for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) {
 		if (xas_retry(&xas, folio))
 			continue;
 		/*
@@ -2281,11 +2282,11 @@ unsigned filemap_get_folios_contig(struct address_space *mapping,
 		 * No current caller is looking for DAX entries.
 		 */
 		if (xa_is_value(folio))
-			goto update_start;
+			break;
 
 		/* If we landed in the middle of a THP, continue at its end. */
 		if (xa_is_sibling(folio))
-			goto update_start;
+			break;
 
 		if (!folio_try_get(folio))
 			goto retry;
@@ -2293,29 +2294,27 @@ unsigned filemap_get_folios_contig(struct address_space *mapping,
 		if (unlikely(folio != xas_reload(&xas)))
 			goto put_folio;
 
-		if (!folio_batch_add(fbatch, folio)) {
-			*start = folio_next_index(folio);
-			goto out;
-		}
+		if (!folio_batch_add(fbatch, folio))
+			break;
+
 		xas_advance(&xas, folio_next_index(folio) - 1);
+		if (xas.xa_index >= end)
+			break;
 		continue;
+
 put_folio:
 		folio_put(folio);
-
 retry:
 		xas_reset(&xas);
 	}
+	rcu_read_unlock();
 
-update_start:
 	nr = folio_batch_count(fbatch);
-
 	if (nr) {
 		folio = fbatch->folios[nr - 1];
 		*start = folio_next_index(folio);
 	}
-out:
-	rcu_read_unlock();
-	return folio_batch_count(fbatch);
+	return nr;
 }
 EXPORT_SYMBOL(filemap_get_folios_contig);
 
@@ -2467,11 +2466,14 @@ static void filemap_get_read_batch(struct address_space *mapping,
 	XA_STATE(xas, &mapping->i_pages, index);
 	struct folio *folio;
 
+	if (index > max)
+		return;
+
 	rcu_read_lock();
 	for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) {
 		if (xas_retry(&xas, folio))
 			continue;
-		if (xas.xa_index > max || xa_is_value(folio))
+		if (xa_is_value(folio))
 			break;
 		if (xa_is_sibling(folio))
 			break;
@@ -2488,6 +2490,8 @@ static void filemap_get_read_batch(struct address_space *mapping,
 		if (folio_test_readahead(folio))
 			break;
 		xas_advance(&xas, folio_next_index(folio) - 1);
+		if (xas.xa_index >= max)
+			break;
 		continue;
 put_folio:
 		folio_put(folio);
@@ -3225,6 +3229,7 @@ loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start,
 	while ((folio = find_get_entry(&xas, max, XA_PRESENT))) {
 		loff_t pos = (u64)xas.xa_index << PAGE_SHIFT;
 		size_t seek_size;
+		u64 next;
 
 		if (start < pos) {
 			if (!seek_data)
@@ -3233,7 +3238,11 @@ loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start,
 		}
 
 		seek_size = seek_folio_size(&xas, folio);
-		pos = round_up((u64)pos + 1, seek_size);
+		next = round_up((u64)pos + 1, seek_size);
+		if (next > (u64)end)
+			pos = end;
+		else
+			pos = next;
 		start = folio_seek_hole_data(&xas, mapping, folio, start, pos,
 				seek_data);
 		if (start < pos)
@@ -3402,8 +3411,8 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf)
 		 * of memory.
 		 */
 		struct vm_area_struct *vma = vmf->vma;
-		unsigned long start = vma->vm_pgoff;
-		unsigned long end = start + vma_pages(vma);
+		const unsigned long start = vma_start_pgoff(vma);
+		const unsigned long end = vma_end_pgoff(vma);
 		unsigned long ra_end;
 
 		ra->order = exec_folio_order();
@@ -3921,7 +3930,8 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf,
 		goto out;
 	}
 
-	addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT);
+	addr = vma->vm_start +
+		((start_pgoff - vma_start_pgoff(vma)) << PAGE_SHIFT);
 	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl);
 	if (!vmf->pte) {
 		folio_unlock(folio);
diff --git a/mm/folio.c b/mm/folio.c
new file mode 100644
index 0000000000000..d2937600cf726
--- /dev/null
+++ b/mm/folio.c
@@ -0,0 +1,1171 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ *  linux/mm/folio.c
+ *
+ *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
+ */
+
+/*
+ * Folio LRU helpers: add/remove folios from LRU lists, batching,
+ * activation/deactivation, and page cache release paths.
+ */
+
+#include <linux/mm.h>
+#include <linux/sched.h>
+#include <linux/kernel_stat.h>
+#include <linux/swap.h>
+#include <linux/mman.h>
+#include <linux/pagemap.h>
+#include <linux/folio_batch.h>
+#include <linux/init.h>
+#include <linux/export.h>
+#include <linux/mm_inline.h>
+#include <linux/percpu_counter.h>
+#include <linux/memremap.h>
+#include <linux/percpu.h>
+#include <linux/cpu.h>
+#include <linux/notifier.h>
+#include <linux/backing-dev.h>
+#include <linux/memcontrol.h>
+#include <linux/gfp.h>
+#include <linux/uio.h>
+#include <linux/hugetlb.h>
+#include <linux/page_idle.h>
+#include <linux/local_lock.h>
+#include <linux/buffer_head.h>
+
+#include "internal.h"
+#include "page_alloc.h"
+
+#define CREATE_TRACE_POINTS
+#include <trace/events/pagemap.h>
+
+struct cpu_fbatches {
+	/*
+	 * The following folio batches are grouped together because they are protected
+	 * by disabling preemption (and interrupts remain enabled).
+	 */
+	local_lock_t lock;
+	struct folio_batch lru_add;
+	struct folio_batch lru_deactivate_file;
+	struct folio_batch lru_deactivate;
+	struct folio_batch lru_lazyfree;
+#ifdef CONFIG_SMP
+	struct folio_batch lru_activate;
+#endif
+	/* Protecting the following batches which require disabling interrupts */
+	local_lock_t lock_irq;
+	struct folio_batch lru_move_tail;
+};
+
+static DEFINE_PER_CPU(struct cpu_fbatches, cpu_fbatches) = {
+	.lock = INIT_LOCAL_LOCK(lock),
+	.lock_irq = INIT_LOCAL_LOCK(lock_irq),
+};
+
+static void __page_cache_release(struct folio *folio, struct lruvec **lruvecp,
+		unsigned long *flagsp)
+{
+	if (folio_test_lru(folio)) {
+		folio_lruvec_relock_irqsave(folio, lruvecp, flagsp);
+		lruvec_del_folio(*lruvecp, folio);
+		__folio_clear_lru_flags(folio);
+	}
+}
+
+/*
+ * This path almost never happens for VM activity - pages are normally freed
+ * in batches.  But it gets used by networking - and for compound pages.
+ */
+static void page_cache_release(struct folio *folio)
+{
+	struct lruvec *lruvec = NULL;
+	unsigned long flags;
+
+	__page_cache_release(folio, &lruvec, &flags);
+	if (lruvec)
+		lruvec_unlock_irqrestore(lruvec, flags);
+}
+
+void __folio_put(struct folio *folio)
+{
+	if (unlikely(folio_is_zone_device(folio))) {
+		free_zone_device_folio(folio);
+		return;
+	}
+
+	if (folio_test_hugetlb(folio)) {
+		free_huge_folio(folio);
+		return;
+	}
+
+	page_cache_release(folio);
+	folio_unqueue_deferred_split(folio);
+	mem_cgroup_uncharge(folio);
+	free_frozen_pages(&folio->page, folio_order(folio));
+}
+EXPORT_SYMBOL(__folio_put);
+
+typedef void (*move_fn_t)(struct lruvec *lruvec, struct folio *folio);
+
+static void lru_add(struct lruvec *lruvec, struct folio *folio)
+{
+	int was_unevictable = folio_test_clear_unevictable(folio);
+	long nr_pages = folio_nr_pages(folio);
+
+	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
+
+	/*
+	 * Is an smp_mb__after_atomic() still required here, before
+	 * folio_evictable() tests the mlocked flag, to rule out the possibility
+	 * of stranding an evictable folio on an unevictable LRU?  I think
+	 * not, because __munlock_folio() only clears the mlocked flag
+	 * while the LRU lock is held.
+	 *
+	 * (That is not true of __page_cache_release(), and not necessarily
+	 * true of folios_put(): but those only clear the mlocked flag after
+	 * folio_put_testzero() has excluded any other users of the folio.)
+	 */
+	if (folio_evictable(folio)) {
+		if (was_unevictable)
+			__count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages);
+	} else {
+		folio_clear_active(folio);
+		folio_set_unevictable(folio);
+		/*
+		 * folio->mlock_count = !!folio_test_mlocked(folio)?
+		 * But that leaves __mlock_folio() in doubt whether another
+		 * actor has already counted the mlock or not.  Err on the
+		 * safe side, underestimate, let page reclaim fix it, rather
+		 * than leaving a page on the unevictable LRU indefinitely.
+		 */
+		folio->mlock_count = 0;
+		if (!was_unevictable)
+			__count_vm_events(UNEVICTABLE_PGCULLED, nr_pages);
+	}
+
+	lruvec_add_folio(lruvec, folio);
+	trace_mm_lru_insertion(folio);
+}
+
+static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
+{
+	int i;
+	struct lruvec *lruvec = NULL;
+	unsigned long flags = 0;
+	struct folio_batch free_fbatch;
+	bool is_lru_add = (move_fn == lru_add);
+
+	/*
+	 * If we're adding to the LRU, preemptively filter dead folios. Use
+	 * this dedicated folio batch for temp storage and deferred cleanup.
+	 */
+	if (is_lru_add)
+		folio_batch_init(&free_fbatch);
+
+	for (i = 0; i < folio_batch_count(fbatch); i++) {
+		struct folio *folio = fbatch->folios[i];
+
+		/* block memcg migration while the folio moves between lru */
+		if (!is_lru_add && !folio_test_clear_lru(folio))
+			continue;
+
+		/*
+		 * Filter dead folios by moving them from the add batch to the temp
+		 * batch for freeing after this loop.
+		 *
+		 * We're bypassing normal cleanup. Clear flags that are not
+		 * applicable to dead folios.
+		 *
+		 * Since the folio may be part of a huge page, unqueue from
+		 * deferred split list to avoid a dangling list entry.
+		 */
+		if (is_lru_add && folio_ref_freeze(folio, 1)) {
+			__folio_clear_active(folio);
+			__folio_clear_unevictable(folio);
+			folio_unqueue_deferred_split(folio);
+			fbatch->folios[i] = NULL;
+			folio_batch_add(&free_fbatch, folio);
+			continue;
+		}
+
+		folio_lruvec_relock_irqsave(folio, &lruvec, &flags);
+		move_fn(lruvec, folio);
+
+		folio_set_lru(folio);
+	}
+
+	if (lruvec)
+		lruvec_unlock_irqrestore(lruvec, flags);
+
+	/* Cleanup filtered dead folios. */
+	if (is_lru_add) {
+		mem_cgroup_uncharge_folios(&free_fbatch);
+		free_unref_folios(&free_fbatch);
+	}
+
+	folios_put(fbatch);
+}
+
+static void __folio_batch_add_and_move(struct folio_batch __percpu *fbatch,
+		struct folio *folio, move_fn_t move_fn, bool disable_irq)
+{
+	unsigned long flags;
+
+	folio_get(folio);
+
+	if (disable_irq)
+		local_lock_irqsave(&cpu_fbatches.lock_irq, flags);
+	else
+		local_lock(&cpu_fbatches.lock);
+
+	if (!folio_batch_add(this_cpu_ptr(fbatch), folio) ||
+			!folio_may_be_lru_cached(folio) || lru_cache_disabled())
+		folio_batch_move_lru(this_cpu_ptr(fbatch), move_fn);
+
+	if (disable_irq)
+		local_unlock_irqrestore(&cpu_fbatches.lock_irq, flags);
+	else
+		local_unlock(&cpu_fbatches.lock);
+}
+
+#define folio_batch_add_and_move(folio, op)		\
+	__folio_batch_add_and_move(			\
+		&cpu_fbatches.op,			\
+		folio,					\
+		op,					\
+		offsetof(struct cpu_fbatches, op) >=	\
+		offsetof(struct cpu_fbatches, lock_irq)	\
+	)
+
+static void lru_move_tail(struct lruvec *lruvec, struct folio *folio)
+{
+	if (folio_test_unevictable(folio))
+		return;
+
+	lruvec_del_folio(lruvec, folio);
+	folio_clear_active(folio);
+	lruvec_add_folio_tail(lruvec, folio);
+	__count_vm_events(PGROTATED, folio_nr_pages(folio));
+}
+
+/*
+ * Writeback is about to end against a folio which has been marked for
+ * immediate reclaim.  If it still appears to be reclaimable, move it
+ * to the tail of the inactive list.
+ *
+ * folio_rotate_reclaimable() must disable IRQs, to prevent nasty races.
+ */
+void folio_rotate_reclaimable(struct folio *folio)
+{
+	if (folio_test_locked(folio) || folio_test_dirty(folio) ||
+	    folio_test_unevictable(folio) || !folio_test_lru(folio))
+		return;
+
+	folio_batch_add_and_move(folio, lru_move_tail);
+}
+
+void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file,
+		unsigned int nr_io, unsigned int nr_rotated)
+		__releases(lruvec->lru_lock)
+		__releases(rcu)
+{
+	unsigned long cost;
+
+	/*
+	 * Reflect the relative cost of incurring IO and spending CPU
+	 * time on rotations. This doesn't attempt to make a precise
+	 * comparison, it just says: if reloads are about comparable
+	 * between the LRU lists, or rotations are overwhelmingly
+	 * different between them, adjust scan balance for CPU work.
+	 */
+	cost = nr_io * SWAP_CLUSTER_MAX + nr_rotated;
+	if (!cost) {
+		spin_unlock_irq(&lruvec->lru_lock);
+		rcu_read_unlock();
+		return;
+	}
+
+	for (;;) {
+		unsigned long lrusize;
+
+		/* Record cost event */
+		if (file)
+			lruvec->file_cost += cost;
+		else
+			lruvec->anon_cost += cost;
+
+		/*
+		 * Decay previous events
+		 *
+		 * Because workloads change over time (and to avoid
+		 * overflow) we keep these statistics as a floating
+		 * average, which ends up weighing recent refaults
+		 * more than old ones.
+		 */
+		lrusize = lruvec_page_state(lruvec, NR_INACTIVE_ANON) +
+			  lruvec_page_state(lruvec, NR_ACTIVE_ANON) +
+			  lruvec_page_state(lruvec, NR_INACTIVE_FILE) +
+			  lruvec_page_state(lruvec, NR_ACTIVE_FILE);
+
+		if (lruvec->file_cost + lruvec->anon_cost > lrusize / 4) {
+			lruvec->file_cost /= 2;
+			lruvec->anon_cost /= 2;
+		}
+
+		spin_unlock_irq(&lruvec->lru_lock);
+		lruvec = parent_lruvec(lruvec);
+		if (!lruvec) {
+			rcu_read_unlock();
+			break;
+		}
+		spin_lock_irq(&lruvec->lru_lock);
+	}
+}
+
+void lru_note_cost_refault(struct folio *folio)
+{
+	struct lruvec *lruvec;
+
+	lruvec = folio_lruvec_lock_irq(folio);
+	lru_note_cost_unlock_irq(lruvec, folio_is_file_lru(folio),
+				folio_nr_pages(folio), 0);
+}
+
+static void lru_activate(struct lruvec *lruvec, struct folio *folio)
+{
+	long nr_pages = folio_nr_pages(folio);
+
+	if (folio_test_active(folio) || folio_test_unevictable(folio))
+		return;
+
+
+	lruvec_del_folio(lruvec, folio);
+	folio_set_active(folio);
+	lruvec_add_folio(lruvec, folio);
+	trace_mm_lru_activate(folio);
+
+	__count_vm_events(PGACTIVATE, nr_pages);
+	count_memcg_events(lruvec_memcg(lruvec), PGACTIVATE, nr_pages);
+}
+
+#ifdef CONFIG_SMP
+static void folio_activate_drain(int cpu)
+{
+	struct folio_batch *fbatch = &per_cpu(cpu_fbatches.lru_activate, cpu);
+
+	if (folio_batch_count(fbatch))
+		folio_batch_move_lru(fbatch, lru_activate);
+}
+
+void folio_activate(struct folio *folio)
+{
+	if (folio_test_active(folio) || folio_test_unevictable(folio) ||
+	    !folio_test_lru(folio))
+		return;
+
+	folio_batch_add_and_move(folio, lru_activate);
+}
+
+#else
+static inline void folio_activate_drain(int cpu)
+{
+}
+
+void folio_activate(struct folio *folio)
+{
+	struct lruvec *lruvec;
+
+	if (!folio_test_clear_lru(folio))
+		return;
+
+	lruvec = folio_lruvec_lock_irq(folio);
+	lru_activate(lruvec, folio);
+	lruvec_unlock_irq(lruvec);
+	folio_set_lru(folio);
+}
+#endif
+
+static void __lru_cache_activate_folio(struct folio *folio)
+{
+	struct folio_batch *fbatch;
+	int i;
+
+	local_lock(&cpu_fbatches.lock);
+	fbatch = this_cpu_ptr(&cpu_fbatches.lru_add);
+
+	/*
+	 * Search backwards on the optimistic assumption that the folio being
+	 * activated has just been added to this batch. Note that only
+	 * the local batch is examined as a !LRU folio could be in the
+	 * process of being released, reclaimed, migrated or on a remote
+	 * batch that is currently being drained. Furthermore, marking
+	 * a remote batch's folio active potentially hits a race where
+	 * a folio is marked active just after it is added to the inactive
+	 * list causing accounting errors and BUG_ON checks to trigger.
+	 */
+	for (i = folio_batch_count(fbatch) - 1; i >= 0; i--) {
+		struct folio *batch_folio = fbatch->folios[i];
+
+		if (batch_folio == folio) {
+			folio_set_active(folio);
+			break;
+		}
+	}
+
+	local_unlock(&cpu_fbatches.lock);
+}
+
+#ifdef CONFIG_LRU_GEN
+
+static void lru_gen_inc_refs(struct folio *folio)
+{
+	unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
+
+	if (folio_test_unevictable(folio))
+		return;
+
+	/* see the comment on LRU_REFS_FLAGS */
+	if (!folio_test_referenced(folio)) {
+		set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
+		return;
+	}
+
+	do {
+		if ((old_flags & LRU_REFS_MASK) == LRU_REFS_MASK) {
+			if (!folio_test_workingset(folio))
+				folio_set_workingset(folio);
+			return;
+		}
+
+		new_flags = old_flags + BIT(LRU_REFS_PGOFF);
+	} while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
+}
+
+static bool lru_gen_clear_refs(struct folio *folio)
+{
+	int gen = folio_lru_gen(folio);
+	int type = folio_is_file_lru(folio);
+	unsigned long seq;
+
+	if (gen < 0)
+		return true;
+
+	set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS | BIT(PG_workingset), 0);
+
+	rcu_read_lock();
+	seq = READ_ONCE(folio_lruvec(folio)->lrugen.min_seq[type]);
+	rcu_read_unlock();
+	/* whether can do without shuffling under the LRU lock */
+	return gen == lru_gen_from_seq(seq);
+}
+
+#else /* !CONFIG_LRU_GEN */
+
+static void lru_gen_inc_refs(struct folio *folio)
+{
+}
+
+static bool lru_gen_clear_refs(struct folio *folio)
+{
+	return false;
+}
+
+#endif /* CONFIG_LRU_GEN */
+
+/**
+ * folio_mark_accessed - Mark a folio as having seen activity.
+ * @folio: The folio to mark.
+ *
+ * This function will perform one of the following transitions:
+ *
+ * * inactive,unreferenced	->	inactive,referenced
+ * * inactive,referenced	->	active,unreferenced
+ * * active,unreferenced	->	active,referenced
+ *
+ * When a newly allocated folio is not yet visible, so safe for non-atomic ops,
+ * __folio_set_referenced() may be substituted for folio_mark_accessed().
+ */
+void folio_mark_accessed(struct folio *folio)
+{
+	if (folio_test_dropbehind(folio))
+		return;
+	if (lru_gen_enabled()) {
+		lru_gen_inc_refs(folio);
+		return;
+	}
+
+	if (!folio_test_referenced(folio)) {
+		folio_set_referenced(folio);
+	} else if (folio_test_unevictable(folio)) {
+		/*
+		 * Unevictable pages are on the "LRU_UNEVICTABLE" list. But,
+		 * this list is never rotated or maintained, so marking an
+		 * unevictable page accessed has no effect.
+		 */
+	} else if (!folio_test_active(folio)) {
+		/*
+		 * If the folio is on the LRU, queue it for activation via
+		 * cpu_fbatches.lru_activate. Otherwise, assume the folio is in a
+		 * folio_batch, mark it active and it'll be moved to the active
+		 * LRU on the next drain.
+		 */
+		if (folio_test_lru(folio))
+			folio_activate(folio);
+		else
+			__lru_cache_activate_folio(folio);
+		folio_clear_referenced(folio);
+		workingset_activation(folio);
+	}
+	if (folio_test_idle(folio))
+		folio_clear_idle(folio);
+}
+EXPORT_SYMBOL(folio_mark_accessed);
+
+/**
+ * folio_add_lru - Add a folio to an LRU list.
+ * @folio: The folio to be added to the LRU.
+ *
+ * Queue the folio for addition to the LRU. The decision on whether
+ * to add the page to the [in]active [file|anon] list is deferred until the
+ * folio_batch is drained. This gives a chance for the caller of folio_add_lru()
+ * have the folio added to the active list using folio_mark_accessed().
+ */
+void folio_add_lru(struct folio *folio)
+{
+	VM_BUG_ON_FOLIO(folio_test_active(folio) &&
+			folio_test_unevictable(folio), folio);
+	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
+
+	/*
+	 * For refaulted workingset folios, set PG_active so they
+	 * can be added to active generations.
+	 * For prefaulted file folios, folio_mark_accessed() sets
+	 * PG_referenced so lru_gen_folio_seq() places them into
+	 * the second oldest generation.
+	 */
+	if (lru_gen_enabled() && !folio_test_unevictable(folio) &&
+	    lru_gen_in_fault() && !(current->flags & PF_MEMALLOC)) {
+		if (folio_test_workingset(folio))
+			folio_set_active(folio);
+		else if (!folio_test_referenced(folio))
+			folio_mark_accessed(folio);
+	}
+
+	folio_batch_add_and_move(folio, lru_add);
+}
+EXPORT_SYMBOL(folio_add_lru);
+
+/**
+ * folio_add_lru_vma() - Add a folio to the appropriate LRU list for this VMA.
+ * @folio: The folio to be added to the LRU.
+ * @vma: VMA in which the folio is mapped.
+ *
+ * If the VMA is mlocked, @folio is added to the unevictable list.
+ * Otherwise, it is treated the same way as folio_add_lru().
+ */
+void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma)
+{
+	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
+
+	if (unlikely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED))
+		mlock_new_folio(folio);
+	else
+		folio_add_lru(folio);
+}
+
+/*
+ * If the folio cannot be invalidated, it is moved to the
+ * inactive list to speed up its reclaim.  It is moved to the
+ * head of the list, rather than the tail, to give the flusher
+ * threads some time to write it out, as this is much more
+ * effective than the single-page writeout from reclaim.
+ *
+ * If the folio isn't mapped and dirty/writeback, the folio
+ * could be reclaimed asap using the reclaim flag.
+ *
+ * 1. active, mapped folio -> none
+ * 2. active, dirty/writeback folio -> inactive, head, reclaim
+ * 3. inactive, mapped folio -> none
+ * 4. inactive, dirty/writeback folio -> inactive, head, reclaim
+ * 5. inactive, clean -> inactive, tail
+ * 6. Others -> none
+ *
+ * In 4, it moves to the head of the inactive list so the folio is
+ * written out by flusher threads as this is much more efficient
+ * than the single-page writeout from reclaim.
+ */
+static void lru_deactivate_file(struct lruvec *lruvec, struct folio *folio)
+{
+	bool active = folio_test_active(folio) || lru_gen_enabled();
+	long nr_pages = folio_nr_pages(folio);
+
+	if (folio_test_unevictable(folio))
+		return;
+
+	/* Some processes are using the folio */
+	if (folio_mapped(folio))
+		return;
+
+	lruvec_del_folio(lruvec, folio);
+	folio_clear_active(folio);
+	folio_clear_referenced(folio);
+
+	if (folio_test_writeback(folio) || folio_test_dirty(folio)) {
+		/*
+		 * Setting the reclaim flag could race with
+		 * folio_end_writeback() and confuse readahead.  But the
+		 * race window is _really_ small and  it's not a critical
+		 * problem.
+		 */
+		lruvec_add_folio(lruvec, folio);
+		folio_set_reclaim(folio);
+	} else {
+		/*
+		 * The folio's writeback ended while it was in the batch.
+		 * We move that folio to the tail of the inactive list.
+		 */
+		lruvec_add_folio_tail(lruvec, folio);
+		__count_vm_events(PGROTATED, nr_pages);
+	}
+
+	if (active) {
+		__count_vm_events(PGDEACTIVATE, nr_pages);
+		count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE,
+				     nr_pages);
+	}
+}
+
+static void lru_deactivate(struct lruvec *lruvec, struct folio *folio)
+{
+	long nr_pages = folio_nr_pages(folio);
+
+	if (folio_test_unevictable(folio) || !(folio_test_active(folio) || lru_gen_enabled()))
+		return;
+
+	lruvec_del_folio(lruvec, folio);
+	folio_clear_active(folio);
+	folio_clear_referenced(folio);
+	lruvec_add_folio(lruvec, folio);
+
+	__count_vm_events(PGDEACTIVATE, nr_pages);
+	count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_pages);
+}
+
+static void lru_lazyfree(struct lruvec *lruvec, struct folio *folio)
+{
+	long nr_pages = folio_nr_pages(folio);
+
+	if (!folio_test_anon(folio) || !folio_test_swapbacked(folio) ||
+	    folio_test_swapcache(folio) || folio_test_unevictable(folio))
+		return;
+
+	lruvec_del_folio(lruvec, folio);
+	folio_clear_active(folio);
+	if (lru_gen_enabled())
+		lru_gen_clear_refs(folio);
+	else
+		folio_clear_referenced(folio);
+	/*
+	 * Lazyfree folios are clean anonymous folios.  They have
+	 * the swapbacked flag cleared, to distinguish them from normal
+	 * anonymous folios
+	 */
+	folio_clear_swapbacked(folio);
+	lruvec_add_folio(lruvec, folio);
+
+	__count_vm_events(PGLAZYFREE, nr_pages);
+	count_memcg_events(lruvec_memcg(lruvec), PGLAZYFREE, nr_pages);
+}
+
+/*
+ * Drain pages out of the cpu's folio_batch.
+ * Either "cpu" is the current CPU, and preemption has already been
+ * disabled; or "cpu" is being hot-unplugged, and is already dead.
+ */
+void lru_add_drain_cpu(int cpu)
+{
+	struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu);
+	struct folio_batch *fbatch = &fbatches->lru_add;
+	unsigned int nr_folios = folio_batch_count(fbatch);
+
+	if (nr_folios) {
+		folio_batch_move_lru(fbatch, lru_add);
+		trace_mm_lru_add_drain_tp(cpu, nr_folios);
+	}
+
+	fbatch = &fbatches->lru_move_tail;
+	/* Disabling interrupts below acts as a compiler barrier. */
+	if (data_race(folio_batch_count(fbatch))) {
+		unsigned long flags;
+
+		/* No harm done if a racing interrupt already did this */
+		local_lock_irqsave(&cpu_fbatches.lock_irq, flags);
+		folio_batch_move_lru(fbatch, lru_move_tail);
+		local_unlock_irqrestore(&cpu_fbatches.lock_irq, flags);
+	}
+
+	fbatch = &fbatches->lru_deactivate_file;
+	if (folio_batch_count(fbatch))
+		folio_batch_move_lru(fbatch, lru_deactivate_file);
+
+	fbatch = &fbatches->lru_deactivate;
+	if (folio_batch_count(fbatch))
+		folio_batch_move_lru(fbatch, lru_deactivate);
+
+	fbatch = &fbatches->lru_lazyfree;
+	if (folio_batch_count(fbatch))
+		folio_batch_move_lru(fbatch, lru_lazyfree);
+
+	folio_activate_drain(cpu);
+}
+
+/**
+ * deactivate_file_folio() - Deactivate a file folio.
+ * @folio: Folio to deactivate.
+ *
+ * This function hints to the VM that @folio is a good reclaim candidate,
+ * for example if its invalidation fails due to the folio being dirty
+ * or under writeback.
+ *
+ * Context: Caller holds a reference on the folio.
+ */
+void deactivate_file_folio(struct folio *folio)
+{
+	/* Deactivating an unevictable folio will not accelerate reclaim */
+	if (folio_test_unevictable(folio) || !folio_test_lru(folio))
+		return;
+
+	if (lru_gen_enabled() && lru_gen_clear_refs(folio))
+		return;
+
+	folio_batch_add_and_move(folio, lru_deactivate_file);
+}
+
+/*
+ * folio_deactivate - deactivate a folio
+ * @folio: folio to deactivate
+ *
+ * folio_deactivate() moves @folio to the inactive list if @folio was on the
+ * active list and was not unevictable. This is done to accelerate the
+ * reclaim of @folio.
+ */
+void folio_deactivate(struct folio *folio)
+{
+	if (folio_test_unevictable(folio) || !folio_test_lru(folio))
+		return;
+
+	if (lru_gen_enabled() ? lru_gen_clear_refs(folio) : !folio_test_active(folio))
+		return;
+
+	folio_batch_add_and_move(folio, lru_deactivate);
+}
+
+/**
+ * folio_mark_lazyfree - make an anon folio lazyfree
+ * @folio: folio to deactivate
+ *
+ * folio_mark_lazyfree() moves @folio to the inactive file list.
+ * This is done to accelerate the reclaim of @folio.
+ */
+void folio_mark_lazyfree(struct folio *folio)
+{
+	if (!folio_test_anon(folio) || !folio_test_swapbacked(folio) ||
+	    !folio_test_lru(folio) ||
+	    folio_test_swapcache(folio) || folio_test_unevictable(folio))
+		return;
+
+	folio_batch_add_and_move(folio, lru_lazyfree);
+}
+
+void lru_add_drain(void)
+{
+	local_lock(&cpu_fbatches.lock);
+	lru_add_drain_cpu(smp_processor_id());
+	local_unlock(&cpu_fbatches.lock);
+	mlock_drain_local();
+}
+
+/*
+ * It's called from per-cpu workqueue context in SMP case so
+ * lru_add_drain_cpu and invalidate_bh_lrus_cpu should run on
+ * the same cpu. It shouldn't be a problem in !SMP case since
+ * the core is only one and the locks will disable preemption.
+ */
+static void lru_add_and_bh_lrus_drain(void)
+{
+	local_lock(&cpu_fbatches.lock);
+	lru_add_drain_cpu(smp_processor_id());
+	local_unlock(&cpu_fbatches.lock);
+	invalidate_bh_lrus_cpu();
+	mlock_drain_local();
+}
+
+void lru_add_drain_cpu_zone(struct zone *zone)
+{
+	local_lock(&cpu_fbatches.lock);
+	lru_add_drain_cpu(smp_processor_id());
+	drain_local_pages(zone);
+	local_unlock(&cpu_fbatches.lock);
+	mlock_drain_local();
+}
+
+#ifdef CONFIG_SMP
+
+static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
+
+static void lru_add_drain_per_cpu(struct work_struct *dummy)
+{
+	lru_add_and_bh_lrus_drain();
+}
+
+static bool cpu_needs_drain(unsigned int cpu)
+{
+	struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu);
+
+	/* Check these in order of likelihood that they're not zero */
+	return data_race(folio_batch_count(&fbatches->lru_add) ||
+			 folio_batch_count(&fbatches->lru_move_tail) ||
+			 folio_batch_count(&fbatches->lru_deactivate_file) ||
+			 folio_batch_count(&fbatches->lru_deactivate) ||
+			 folio_batch_count(&fbatches->lru_lazyfree) ||
+			 folio_batch_count(&fbatches->lru_activate) ||
+			 need_mlock_drain(cpu)) ||
+		has_bh_in_lru(cpu, NULL);
+}
+
+/*
+ * Doesn't need any cpu hotplug locking because we do rely on per-cpu
+ * kworkers being shut down before our page_alloc_cpu_dead callback is
+ * executed on the offlined cpu.
+ * Calling this function with cpu hotplug locks held can actually lead
+ * to obscure indirect dependencies via WQ context.
+ */
+static inline void __lru_add_drain_all(bool force_all_cpus)
+{
+	/*
+	 * lru_drain_gen - Global pages generation number
+	 *
+	 * (A) Definition: global lru_drain_gen = x implies that all generations
+	 *     0 < n <= x are already *scheduled* for draining.
+	 *
+	 * This is an optimization for the highly-contended use case where a
+	 * user space workload keeps constantly generating a flow of pages for
+	 * each CPU.
+	 */
+	static unsigned int lru_drain_gen;
+	static struct cpumask has_work;
+	static DEFINE_MUTEX(lock);
+	unsigned cpu, this_gen;
+
+	/*
+	 * Make sure nobody triggers this path before mm_percpu_wq is fully
+	 * initialized.
+	 */
+	if (WARN_ON(!mm_percpu_wq))
+		return;
+
+	trace_mm_lru_add_drain_all_tp(force_all_cpus);
+
+	/*
+	 * Guarantee folio_batch counter stores visible by this CPU
+	 * are visible to other CPUs before loading the current drain
+	 * generation.
+	 */
+	smp_mb();
+
+	/*
+	 * (B) Locally cache global LRU draining generation number
+	 *
+	 * The read barrier ensures that the counter is loaded before the mutex
+	 * is taken. It pairs with smp_mb() inside the mutex critical section
+	 * at (D).
+	 */
+	this_gen = smp_load_acquire(&lru_drain_gen);
+
+	/* It helps everyone if we do our own local drain immediately. */
+	lru_add_drain();
+
+	mutex_lock(&lock);
+
+	/*
+	 * (C) Exit the draining operation if a newer generation, from another
+	 * lru_add_drain_all(), was already scheduled for draining. Check (A).
+	 */
+	if (unlikely(this_gen != lru_drain_gen && !force_all_cpus))
+		goto done;
+
+	/*
+	 * (D) Increment global generation number
+	 *
+	 * Pairs with smp_load_acquire() at (B), outside of the critical
+	 * section. Use a full memory barrier to guarantee that the
+	 * new global drain generation number is stored before loading
+	 * folio_batch counters.
+	 *
+	 * This pairing must be done here, before the for_each_online_cpu loop
+	 * below which drains the page vectors.
+	 *
+	 * Let x, y, and z represent some system CPU numbers, where x < y < z.
+	 * Assume CPU #z is in the middle of the for_each_online_cpu loop
+	 * below and has already reached CPU #y's per-cpu data. CPU #x comes
+	 * along, adds some pages to its per-cpu vectors, then calls
+	 * lru_add_drain_all().
+	 *
+	 * If the paired barrier is done at any later step, e.g. after the
+	 * loop, CPU #x will just exit at (C) and miss flushing out all of its
+	 * added pages.
+	 */
+	WRITE_ONCE(lru_drain_gen, lru_drain_gen + 1);
+	smp_mb();
+
+	cpumask_clear(&has_work);
+	for_each_online_cpu(cpu) {
+		struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
+
+		if (cpu_needs_drain(cpu)) {
+			INIT_WORK(work, lru_add_drain_per_cpu);
+			queue_work_on(cpu, mm_percpu_wq, work);
+			__cpumask_set_cpu(cpu, &has_work);
+		}
+	}
+
+	for_each_cpu(cpu, &has_work)
+		flush_work(&per_cpu(lru_add_drain_work, cpu));
+
+done:
+	mutex_unlock(&lock);
+}
+
+void lru_add_drain_all(void)
+{
+	__lru_add_drain_all(false);
+}
+#else
+void lru_add_drain_all(void)
+{
+	lru_add_drain();
+}
+#endif /* CONFIG_SMP */
+
+atomic_t lru_disable_count = ATOMIC_INIT(0);
+
+/*
+ * lru_cache_disable() needs to be called before we start compiling
+ * a list of folios to be migrated using folio_isolate_lru().
+ * It drains folios on LRU cache and then disable on all cpus until
+ * lru_cache_enable is called.
+ *
+ * Must be paired with a call to lru_cache_enable().
+ */
+void lru_cache_disable(void)
+{
+	atomic_inc(&lru_disable_count);
+	/*
+	 * Readers of lru_disable_count are protected by either disabling
+	 * preemption or rcu_read_lock:
+	 *
+	 * preempt_disable, local_irq_disable  [bh_lru_lock()]
+	 * rcu_read_lock		       [rt_spin_lock CONFIG_PREEMPT_RT]
+	 * preempt_disable		       [local_lock !CONFIG_PREEMPT_RT]
+	 *
+	 * Since v5.1 kernel, synchronize_rcu() is guaranteed to wait on
+	 * preempt_disable() regions of code. So any CPU which sees
+	 * lru_disable_count = 0 will have exited the critical
+	 * section when synchronize_rcu() returns.
+	 */
+	synchronize_rcu_expedited();
+#ifdef CONFIG_SMP
+	__lru_add_drain_all(true);
+#else
+	lru_add_and_bh_lrus_drain();
+#endif
+}
+
+/**
+ * folios_put_refs - Reduce the reference count on a batch of folios.
+ * @folios: The folios.
+ * @refs: The number of refs to subtract from each folio.
+ *
+ * Like folio_put(), but for a batch of folios.  This is more efficient
+ * than writing the loop yourself as it will optimise the locks which need
+ * to be taken if the folios are freed.  The folios batch is returned
+ * empty and ready to be reused for another batch; there is no need
+ * to reinitialise it.  If @refs is NULL, we subtract one from each
+ * folio refcount.
+ *
+ * Context: May be called in process or interrupt context, but not in NMI
+ * context.  May be called while holding a spinlock.
+ */
+void folios_put_refs(struct folio_batch *folios, unsigned int *refs)
+{
+	int i, j;
+	struct lruvec *lruvec = NULL;
+	unsigned long flags = 0;
+
+	for (i = 0, j = 0; i < folios->nr; i++) {
+		struct folio *folio = folios->folios[i];
+		unsigned int nr_refs = refs ? refs[i] : 1;
+
+		/* Folio batch entry may have been preemptively removed during drain. */
+		if (!folio)
+			continue;
+
+		if (is_huge_zero_folio(folio))
+			continue;
+
+		if (folio_is_zone_device(folio)) {
+			if (lruvec) {
+				lruvec_unlock_irqrestore(lruvec, flags);
+				lruvec = NULL;
+			}
+			if (folio_ref_sub_and_test(folio, nr_refs))
+				free_zone_device_folio(folio);
+			continue;
+		}
+
+		if (!folio_ref_sub_and_test(folio, nr_refs))
+			continue;
+
+		/* hugetlb has its own memcg */
+		if (folio_test_hugetlb(folio)) {
+			if (lruvec) {
+				lruvec_unlock_irqrestore(lruvec, flags);
+				lruvec = NULL;
+			}
+			free_huge_folio(folio);
+			continue;
+		}
+		folio_unqueue_deferred_split(folio);
+		__page_cache_release(folio, &lruvec, &flags);
+
+		if (j != i)
+			folios->folios[j] = folio;
+		j++;
+	}
+	if (lruvec)
+		lruvec_unlock_irqrestore(lruvec, flags);
+	if (!j) {
+		folio_batch_reinit(folios);
+		return;
+	}
+
+	folios->nr = j;
+	mem_cgroup_uncharge_folios(folios);
+	free_unref_folios(folios);
+}
+EXPORT_SYMBOL(folios_put_refs);
+
+/**
+ * release_pages - batched put_page()
+ * @arg: array of pages to release
+ * @nr: number of pages
+ *
+ * Decrement the reference count on all the pages in @arg.  If it
+ * fell to zero, remove the page from the LRU and free it.
+ *
+ * Note that the argument can be an array of pages, encoded pages,
+ * or folio pointers. We ignore any encoded bits, and turn any of
+ * them into just a folio that gets free'd.
+ */
+void release_pages(release_pages_arg arg, int nr)
+{
+	struct folio_batch fbatch;
+	int refs[FOLIO_BATCH_SIZE];
+	struct encoded_page **encoded = arg.encoded_pages;
+	int i;
+
+	folio_batch_init(&fbatch);
+	for (i = 0; i < nr; i++) {
+		/* Turn any of the argument types into a folio */
+		struct folio *folio = page_folio(encoded_page_ptr(encoded[i]));
+
+		/* Is our next entry actually "nr_pages" -> "nr_refs" ? */
+		refs[fbatch.nr] = 1;
+		if (unlikely(encoded_page_flags(encoded[i]) &
+			     ENCODED_PAGE_BIT_NR_PAGES_NEXT))
+			refs[fbatch.nr] = encoded_nr_pages(encoded[++i]);
+
+		if (folio_batch_add(&fbatch, folio) > 0)
+			continue;
+		folios_put_refs(&fbatch, refs);
+	}
+
+	if (fbatch.nr)
+		folios_put_refs(&fbatch, refs);
+}
+EXPORT_SYMBOL(release_pages);
+
+/*
+ * The folios which we're about to release may be in the deferred lru-addition
+ * queues.  That would prevent them from really being freed right now.  That's
+ * OK from a correctness point of view but is inefficient - those folios may be
+ * cache-warm and we want to give them back to the page allocator ASAP.
+ *
+ * So __folio_batch_release() will drain those queues here.
+ * folio_batch_move_lru() calls folios_put() directly to avoid
+ * mutual recursion.
+ */
+void __folio_batch_release(struct folio_batch *fbatch)
+{
+	if (!fbatch->percpu_pvec_drained) {
+		lru_add_drain();
+		fbatch->percpu_pvec_drained = true;
+	}
+	folios_put(fbatch);
+}
+EXPORT_SYMBOL(__folio_batch_release);
+
+/**
+ * folio_batch_remove_exceptionals() - Prune non-folios from a batch.
+ * @fbatch: The batch to prune
+ *
+ * find_get_entries() fills a batch with both folios and shadow/swap/DAX
+ * entries.  This function prunes all the non-folio entries from @fbatch
+ * without leaving holes, so that it can be passed on to folio-only batch
+ * operations.
+ */
+void folio_batch_remove_exceptionals(struct folio_batch *fbatch)
+{
+	unsigned int i, j;
+
+	for (i = 0, j = 0; i < folio_batch_count(fbatch); i++) {
+		struct folio *folio = fbatch->folios[i];
+		if (!xa_is_value(folio))
+			fbatch->folios[j++] = folio;
+	}
+	fbatch->nr = j;
+}
+
+#ifdef CONFIG_MEMCG
+static void lruvec_reparent_lru(struct lruvec *child_lruvec,
+				struct lruvec *parent_lruvec,
+				enum lru_list lru, int nid)
+{
+	int zid;
+	struct zone *zone;
+
+	if (lru != LRU_UNEVICTABLE)
+		list_splice_tail_init(&child_lruvec->lists[lru], &parent_lruvec->lists[lru]);
+
+	for_each_managed_zone_pgdat(zone, NODE_DATA(nid), zid, MAX_NR_ZONES - 1) {
+		unsigned long size = mem_cgroup_get_zone_lru_size(child_lruvec, lru, zid);
+
+		mem_cgroup_update_lru_size(parent_lruvec, lru, zid, size);
+	}
+}
+
+void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid)
+{
+	enum lru_list lru;
+	struct lruvec *child_lruvec, *parent_lruvec;
+
+	child_lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
+	parent_lruvec = mem_cgroup_lruvec(parent, NODE_DATA(nid));
+	parent_lruvec->anon_cost += child_lruvec->anon_cost;
+	parent_lruvec->file_cost += child_lruvec->file_cost;
+
+	for_each_lru(lru)
+		lruvec_reparent_lru(child_lruvec, parent_lruvec, lru, nid);
+}
+#endif
diff --git a/mm/gup.c b/mm/gup.c
index 0692119b79043..99902c15703b0 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -2784,12 +2784,17 @@ static bool gup_fast_folio_allowed(struct folio *folio, unsigned int flags)
 	mapping = READ_ONCE(folio->mapping);
 
 	/*
-	 * The mapping may have been truncated, in any case we cannot determine
-	 * if this mapping is safe - fall back to slow path to determine how to
-	 * proceed.
+	 * If the mapping is NULL (truncated, or never set), we cannot
+	 * determine whether the folio is file-backed, so a long-term writable
+	 * pin must fall back to the slow path.
+	 *
+	 * Otherwise, a NULL mapping proves this is not a secretmem folio
+	 * (secretmem folios always have a valid mapping to the secretmem
+	 * inode's address_space), so in that case, we can continue with the
+	 * fast path.
 	 */
 	if (!mapping)
-		return false;
+		return !reject_file_backed;
 
 	/* Anonymous folios pose no problem. */
 	mapping_flags = (unsigned long)mapping & FOLIO_MAPPING_FLAGS;
diff --git a/mm/hmm.c b/mm/hmm.c
index c72c9ddfdb2f6..fc2e1cd0cb22e 100644
--- a/mm/hmm.c
+++ b/mm/hmm.c
@@ -32,9 +32,27 @@
 
 struct hmm_vma_walk {
 	struct hmm_range	*range;
+	bool			*locked;
 	unsigned long		last;
+	unsigned long		end;
+	unsigned int		required_fault;
 };
 
+/*
+ * Internal sentinel returned by walk callbacks when they need a page fault.
+ * The callback stores end/required_fault in hmm_vma_walk; the outer loop
+ * consumes the sentinel and never propagates it to the caller.
+ */
+#define HMM_FAULT_PENDING	-EAGAIN
+
+/*
+ * Internal sentinel returned by hmm_do_fault() when handle_mm_fault()
+ * completes a page fault with the mmap lock dropped. hmm_do_fault() sets
+ * *locked = false; the outer loop consumes the sentinel and never propagates
+ * it to the caller.
+ */
+#define HMM_FAULT_UNLOCKED	-ENOLCK
+
 enum {
 	HMM_NEED_FAULT = 1 << 0,
 	HMM_NEED_WRITE_FAULT = 1 << 1,
@@ -60,37 +78,25 @@ static int hmm_pfns_fill(unsigned long addr, unsigned long end,
 }
 
 /*
- * hmm_vma_fault() - fault in a range lacking valid pmd or pte(s)
- * @addr: range virtual start address (inclusive)
- * @end: range virtual end address (exclusive)
- * @required_fault: HMM_NEED_* flags
- * @walk: mm_walk structure
- * Return: -EBUSY after page fault, or page fault error
+ * hmm_record_fault() - record a range that needs to be faulted in
  *
- * This function will be called whenever pmd_none() or pte_none() returns true,
- * or whenever there is no page directory covering the virtual address range.
+ * Called by the walk callbacks when they discover that part of the range
+ * needs a page fault.  The callback records what to fault and returns
+ * HMM_FAULT_PENDING; the outer loop in hmm_range_fault_locked() drops
+ * back out of walk_page_range() and invokes handle_mm_fault() from a context
+ * where no page-table or hugetlb_vma_lock is held.
  */
-static int hmm_vma_fault(unsigned long addr, unsigned long end,
-			 unsigned int required_fault, struct mm_walk *walk)
+static int hmm_record_fault(unsigned long addr, unsigned long end,
+			    unsigned int required_fault,
+			    struct mm_walk *walk)
 {
 	struct hmm_vma_walk *hmm_vma_walk = walk->private;
-	struct vm_area_struct *vma = walk->vma;
-	unsigned int fault_flags = FAULT_FLAG_REMOTE;
 
 	WARN_ON_ONCE(!required_fault);
 	hmm_vma_walk->last = addr;
-
-	if (required_fault & HMM_NEED_WRITE_FAULT) {
-		if (!(vma->vm_flags & VM_WRITE))
-			return -EPERM;
-		fault_flags |= FAULT_FLAG_WRITE;
-	}
-
-	for (; addr < end; addr += PAGE_SIZE)
-		if (handle_mm_fault(vma, addr, fault_flags, NULL) &
-		    VM_FAULT_ERROR)
-			return -EFAULT;
-	return -EBUSY;
+	hmm_vma_walk->end = end;
+	hmm_vma_walk->required_fault = required_fault;
+	return HMM_FAULT_PENDING;
 }
 
 static unsigned int hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
@@ -174,7 +180,7 @@ static int hmm_vma_walk_hole(unsigned long addr, unsigned long end,
 		return hmm_pfns_fill(addr, end, range, HMM_PFN_ERROR);
 	}
 	if (required_fault)
-		return hmm_vma_fault(addr, end, required_fault, walk);
+		return hmm_record_fault(addr, end, required_fault, walk);
 	return hmm_pfns_fill(addr, end, range, 0);
 }
 
@@ -209,7 +215,7 @@ static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,
 	required_fault =
 		hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, cpu_flags);
 	if (required_fault)
-		return hmm_vma_fault(addr, end, required_fault, walk);
+		return hmm_record_fault(addr, end, required_fault, walk);
 
 	pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
 	for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) {
@@ -328,10 +334,10 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
 fault:
 	pte_unmap(ptep);
 	/* Fault any virtual address we were asked to fault */
-	return hmm_vma_fault(addr, end, required_fault, walk);
+	return hmm_record_fault(addr, end, required_fault, walk);
 }
 
-#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
 static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start,
 				     unsigned long end, unsigned long *hmm_pfns,
 				     pmd_t pmd)
@@ -371,7 +377,7 @@ static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start,
 					      npages, 0);
 	if (required_fault) {
 		if (softleaf_is_device_private(entry))
-			return hmm_vma_fault(addr, end, required_fault, walk);
+			return hmm_record_fault(addr, end, required_fault, walk);
 		else
 			return -EFAULT;
 	}
@@ -391,7 +397,7 @@ static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start,
 		return -EFAULT;
 	return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);
 }
-#endif  /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
+#endif  /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */
 
 static int hmm_vma_walk_pmd(pmd_t *pmdp,
 			    unsigned long start,
@@ -517,7 +523,7 @@ static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end,
 						      npages, cpu_flags);
 		if (required_fault) {
 			spin_unlock(ptl);
-			return hmm_vma_fault(addr, end, required_fault, walk);
+			return hmm_record_fault(addr, end, required_fault, walk);
 		}
 
 		pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
@@ -564,21 +570,8 @@ static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask,
 	required_fault =
 		hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, cpu_flags);
 	if (required_fault) {
-		int ret;
-
 		spin_unlock(ptl);
-		hugetlb_vma_unlock_read(vma);
-		/*
-		 * Avoid deadlock: drop the vma lock before calling
-		 * hmm_vma_fault(), which will itself potentially take and
-		 * drop the vma lock. This is also correct from a
-		 * protection point of view, because there is no further
-		 * use here of either pte or ptl after dropping the vma
-		 * lock.
-		 */
-		ret = hmm_vma_fault(addr, end, required_fault, walk);
-		hugetlb_vma_lock_read(vma);
-		return ret;
+		return hmm_record_fault(addr, end, required_fault, walk);
 	}
 
 	pfn = pte_pfn(entry) + ((start & ~hmask) >> PAGE_SHIFT);
@@ -637,29 +630,67 @@ static const struct mm_walk_ops hmm_walk_ops = {
 	.walk_lock	= PGWALK_RDLOCK,
 };
 
-/**
- * hmm_range_fault - try to fault some address in a virtual address range
- * @range:	argument structure
- *
- * Returns 0 on success or one of the following error codes:
+/*
+ * hmm_do_fault - fault in a range recorded by a walk callback
  *
- * -EINVAL:	Invalid arguments or mm or virtual address is in an invalid vma
- *		(e.g., device file vma).
- * -ENOMEM:	Out of memory.
- * -EPERM:	Invalid permission (e.g., asking for write and range is read
- *		only).
- * -EBUSY:	The range has been invalidated and the caller needs to wait for
- *		the invalidation to finish.
- * -EFAULT:     A page was requested to be valid and could not be made valid
- *              ie it has no backing VMA or it is illegal to access
+ * Called from the outer loop in hmm_range_fault_locked() after a callback
+ * returned HMM_FAULT_PENDING.  At this point we hold only mmap_lock;
+ * the page-table spinlock and any hugetlb_vma_lock acquired by the walk
+ * framework have already been released by the unwind.
  *
- * This is similar to get_user_pages(), except that it can read the page tables
- * without mutating them (ie causing faults).
+ * Returns -EBUSY on success (all pages faulted, caller should re-walk).
+ * Returns a negative errno on failure.
  */
-int hmm_range_fault(struct hmm_range *range)
+static int hmm_do_fault(struct mm_struct *mm,
+			struct hmm_vma_walk *hmm_vma_walk)
+{
+	unsigned long addr = hmm_vma_walk->last;
+	unsigned long end = hmm_vma_walk->end;
+	unsigned int required_fault = hmm_vma_walk->required_fault;
+	unsigned int fault_flags = FAULT_FLAG_REMOTE;
+	struct vm_area_struct *vma;
+
+	if (hmm_vma_walk->locked)
+		fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
+
+	vma = vma_lookup(mm, addr);
+	if (!vma)
+		return -EFAULT;
+
+	if (required_fault & HMM_NEED_WRITE_FAULT) {
+		if (!(vma->vm_flags & VM_WRITE))
+			return -EPERM;
+		fault_flags |= FAULT_FLAG_WRITE;
+	}
+
+	for (; addr < end; addr += PAGE_SIZE) {
+		vm_fault_t ret;
+
+		ret = handle_mm_fault(vma, addr, fault_flags, NULL);
+
+		if (ret & (VM_FAULT_COMPLETED | VM_FAULT_RETRY)) {
+			*hmm_vma_walk->locked = false;
+			return HMM_FAULT_UNLOCKED;
+		}
+
+		if (ret & VM_FAULT_ERROR) {
+			int err = vm_fault_to_errno(ret, 0);
+
+			if (WARN_ON(!err))
+				err = -EINVAL;
+
+			return err;
+		}
+	}
+
+	return -EBUSY;
+}
+
+static int hmm_range_fault_locked(struct hmm_range *range, bool *locked)
 {
 	struct hmm_vma_walk hmm_vma_walk = {
 		.range = range,
+		.locked = locked,
 		.last = range->start,
 	};
 	struct mm_struct *mm = range->notifier->mm;
@@ -674,6 +705,22 @@ int hmm_range_fault(struct hmm_range *range)
 			return -EBUSY;
 		ret = walk_page_range(mm, hmm_vma_walk.last, range->end,
 				      &hmm_walk_ops, &hmm_vma_walk);
+		/*
+		 * When HMM_FAULT_PENDING is returned a walk callback
+		 * recorded a range that needs handle_mm_fault();
+		 * hmm_do_fault() runs the fault outside walk_page_range()
+		 * (so no page-table or hugetlb_vma_lock is held) and
+		 * returns -EBUSY so the loop re-walks and picks up the
+		 * now-present entries.
+		 */
+		if (ret == HMM_FAULT_PENDING) {
+			ret = hmm_do_fault(mm, &hmm_vma_walk);
+			if (ret == HMM_FAULT_UNLOCKED) {
+				if (fatal_signal_pending(current))
+					return -EINTR;
+				return -EBUSY;
+			}
+		}
 		/*
 		 * When -EBUSY is returned the loop restarts with
 		 * hmm_vma_walk.last set to an address that has not been stored
@@ -683,8 +730,100 @@ int hmm_range_fault(struct hmm_range *range)
 	} while (ret == -EBUSY);
 	return ret;
 }
+
+/**
+ * hmm_range_fault - try to fault some address in a virtual address range
+ * @range:	argument structure
+ *
+ * Returns 0 on success or one of the following error codes:
+ *
+ * -EINVAL:	Invalid arguments or mm or virtual address is in an invalid vma
+ *		(e.g., device file vma).
+ * -ENOMEM:	Out of memory.
+ * -EPERM:	Invalid permission (e.g., asking for write and range is read
+ *		only).
+ * -EBUSY:	The range has been invalidated and the caller needs to wait for
+ *		the invalidation to finish.
+ * -EFAULT:     A page was requested to be valid and could not be made valid
+ *              ie it has no backing VMA or it is illegal to access
+ *
+ * This is similar to get_user_pages(), except that it can read the page tables
+ * without mutating them (ie causing faults).
+ *
+ * The mmap lock must be held by the caller and will remain held on return.
+ * New users should prefer hmm_range_fault_unlocked_timeout() unless they
+ * specifically need to keep the mmap lock held across the call. This helper
+ * cannot support VMAs whose fault handlers need to drop the mmap lock.
+ */
+int hmm_range_fault(struct hmm_range *range)
+{
+	return hmm_range_fault_locked(range, NULL);
+}
 EXPORT_SYMBOL(hmm_range_fault);
 
+/**
+ * hmm_range_fault_unlocked_timeout - fault in a range with a retry timeout
+ * @range:	argument structure
+ * @timeout:	timeout in jiffies for internal -EBUSY retries, or 0 to retry
+ *		indefinitely
+ *
+ * The caller must not hold the mmap lock. The function takes the mmap read
+ * lock internally and allows handle_mm_fault() to drop it during faults. If
+ * the mmap lock is dropped or the range is invalidated, the function refreshes
+ * range->notifier_seq and restarts the walk internally.
+ *
+ * Passing 0 for @timeout retries indefinitely. A non-zero @timeout is a caller
+ * policy limit for repeated mmu-notifier invalidation retries. HMM does not
+ * interrupt page fault handling when the timeout expires, but returns -EBUSY
+ * if the retry budget is exhausted before a stable range is obtained.
+ *
+ * Returns 0 on success or one of the error codes documented for
+ * hmm_range_fault(). -EINTR is returned if mmap_lock acquisition is
+ * interrupted or a fatal signal is pending during retry handling.
+ */
+int hmm_range_fault_unlocked_timeout(struct hmm_range *range,
+				     unsigned long timeout)
+{
+	struct mm_struct *mm = range->notifier->mm;
+	unsigned long deadline = 0;
+	bool locked = false;
+	int ret;
+
+	do {
+		if (fatal_signal_pending(current))
+			return -EINTR;
+
+		if (timeout) {
+			/*
+			 * If the previous fault dropped mmap_lock, then the fault
+			 * handler made progress. Restart the retry timeout in that
+			 * case, but keep the existing deadline for ordinary -EBUSY
+			 * retries.
+			 */
+			if (!locked)
+				deadline = jiffies + timeout;
+
+			if (time_after(jiffies, deadline))
+				return -EBUSY;
+		}
+
+		range->notifier_seq =
+			mmu_interval_read_begin(range->notifier);
+
+		ret = mmap_read_lock_killable(mm);
+		if (ret)
+			return ret;
+
+		locked = true;
+		ret = hmm_range_fault_locked(range, &locked);
+		if (locked)
+			mmap_read_unlock(mm);
+	} while (ret == -EBUSY);
+
+	return ret;
+}
+EXPORT_SYMBOL(hmm_range_fault_unlocked_timeout);
+
 /**
  * hmm_dma_map_alloc - Allocate HMM map structure
  * @dev: device to allocate structure for
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 2bccb0a53a0a6..9b1f3b24f7e0d 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -180,7 +180,7 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
 	 */
 	if (!in_pf && shmem_file(vma->vm_file))
 		return orders & shmem_allowable_huge_orders(file_inode(vma->vm_file),
-						   vma, vma->vm_pgoff, 0,
+						   vma, vma_start_pgoff(vma), 0,
 						   forced_collapse);
 
 	if (!vma_is_anonymous(vma)) {
@@ -818,10 +818,8 @@ static struct thpsize *thpsize_create(int order, struct kobject *parent)
 
 	ret = kobject_init_and_add(&thpsize->kobj, &thpsize_ktype, parent,
 				   "hugepages-%lukB", size);
-	if (ret) {
-		kfree(thpsize);
-		goto err;
-	}
+	if (ret)
+		goto err_put;
 
 
 	ret = sysfs_add_group(&thpsize->kobj, &any_ctrl_attr_grp);
@@ -1196,7 +1194,7 @@ static inline bool is_transparent_hugepage(const struct folio *folio)
 static unsigned long __thp_get_unmapped_area(struct file *filp,
 		unsigned long addr, unsigned long len,
 		loff_t off, unsigned long flags, unsigned long size,
-		vm_flags_t vm_flags)
+		vma_flags_t vma_flags)
 {
 	loff_t off_end = off + len;
 	loff_t off_align = round_up(off, size);
@@ -1212,8 +1210,9 @@ static unsigned long __thp_get_unmapped_area(struct file *filp,
 	if (len_pad < len || (off + len_pad) < off)
 		return 0;
 
-	ret = mm_get_unmapped_area_vmflags(filp, addr, len_pad,
-					   off >> PAGE_SHIFT, flags, vm_flags);
+	ret = mm_get_unmapped_area_vmaflags(filp, addr, len_pad,
+					    off >> PAGE_SHIFT, flags,
+					    vma_flags);
 
 	/*
 	 * The failure might be due to length padding. The caller will retry
@@ -1238,25 +1237,27 @@ static unsigned long __thp_get_unmapped_area(struct file *filp,
 	return ret;
 }
 
-unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr,
+unsigned long thp_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr,
 		unsigned long len, unsigned long pgoff, unsigned long flags,
-		vm_flags_t vm_flags)
+		vma_flags_t vma_flags)
 {
 	unsigned long ret;
 	loff_t off = (loff_t)pgoff << PAGE_SHIFT;
 
-	ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, vm_flags);
+	ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE,
+				      vma_flags);
 	if (ret)
 		return ret;
 
-	return mm_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags,
-					    vm_flags);
+	return mm_get_unmapped_area_vmaflags(filp, addr, len, pgoff, flags,
+					     vma_flags);
 }
 
 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
 		unsigned long len, unsigned long pgoff, unsigned long flags)
 {
-	return thp_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 0);
+	return thp_get_unmapped_area_vmaflags(filp, addr, len, pgoff, flags,
+					      EMPTY_VMA_FLAGS);
 }
 EXPORT_SYMBOL_GPL(thp_get_unmapped_area);
 
@@ -1819,11 +1820,11 @@ static void copy_huge_non_present_pmd(
 	if (softleaf_is_migration_write(entry) ||
 	    softleaf_is_migration_read_exclusive(entry)) {
 		entry = make_readable_migration_entry(swp_offset(entry));
-		pmd = swp_entry_to_pmd(entry);
+		pmd = softleaf_to_pmd(entry);
 		if (pmd_swp_soft_dirty(*src_pmd))
 			pmd = pmd_swp_mksoft_dirty(pmd);
-		if (pmd_swp_uffd_wp(*src_pmd))
-			pmd = pmd_swp_mkuffd_wp(pmd);
+		if (pmd_swp_uffd(*src_pmd))
+			pmd = pmd_swp_mkuffd(pmd);
 		set_pmd_at(src_mm, addr, src_pmd, pmd);
 	} else if (softleaf_is_device_private(entry)) {
 		/*
@@ -1832,12 +1833,12 @@ static void copy_huge_non_present_pmd(
 		 */
 		if (softleaf_is_device_private_write(entry)) {
 			entry = make_readable_device_private_entry(swp_offset(entry));
-			pmd = swp_entry_to_pmd(entry);
+			pmd = softleaf_to_pmd(entry);
 
 			if (pmd_swp_soft_dirty(*src_pmd))
 				pmd = pmd_swp_mksoft_dirty(pmd);
-			if (pmd_swp_uffd_wp(*src_pmd))
-				pmd = pmd_swp_mkuffd_wp(pmd);
+			if (pmd_swp_uffd(*src_pmd))
+				pmd = pmd_swp_mkuffd(pmd);
 			set_pmd_at(src_mm, addr, src_pmd, pmd);
 		}
 
@@ -1856,8 +1857,8 @@ static void copy_huge_non_present_pmd(
 	add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
 	mm_inc_nr_ptes(dst_mm);
 	pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
-	if (!userfaultfd_wp(dst_vma))
-		pmd = pmd_swp_clear_uffd_wp(pmd);
+	if (!userfaultfd_protected(dst_vma))
+		pmd = pmd_swp_clear_uffd(pmd);
 	set_pmd_at(dst_mm, addr, dst_pmd, pmd);
 }
 
@@ -1951,9 +1952,15 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 out_zero_page:
 	mm_inc_nr_ptes(dst_mm);
 	pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
+
+	/* See __copy_present_ptes(): restore accessible protection. */
+	if (!userfaultfd_protected(dst_vma)) {
+		if (userfaultfd_rwp(src_vma) && pmd_uffd(pmd))
+			pmd = pmd_modify(pmd, dst_vma->vm_page_prot);
+		pmd = pmd_clear_uffd(pmd);
+	}
+
 	pmdp_set_wrprotect(src_mm, addr, src_pmd);
-	if (!userfaultfd_wp(dst_vma))
-		pmd = pmd_clear_uffd_wp(pmd);
 	pmd = pmd_wrprotect(pmd);
 set_pmd:
 	pmd = pmd_mkold(pmd);
@@ -2196,6 +2203,34 @@ static inline bool can_change_pmd_writable(struct vm_area_struct *vma,
 	return pmd_dirty(pmd);
 }
 
+vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	pmd_t pmd;
+
+	if (!userfaultfd_rwp_async(vma))
+		return handle_userfault(vmf, VM_UFFD_RWP);
+
+	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
+	if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) {
+		spin_unlock(vmf->ptl);
+		return 0;
+	}
+	pmd = pmd_modify(vmf->orig_pmd, vma->vm_page_prot);
+	/* pmd_modify() preserves _PAGE_UFFD; drop it on resolution */
+	pmd = pmd_clear_uffd(pmd);
+	pmd = pmd_mkyoung(pmd);
+	if (!pmd_write(pmd) &&
+	    vma_wants_manual_pte_write_upgrade(vma) &&
+	    can_change_pmd_writable(vma, vmf->address, pmd))
+		pmd = pmd_mkwrite(pmd, vma);
+	set_pmd_at(vma->vm_mm, vmf->address & HPAGE_PMD_MASK,
+		   vmf->pmd, pmd);
+	update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
+	spin_unlock(vmf->ptl);
+	return 0;
+}
+
 /* NUMA hinting page fault entry point for trans huge pmds */
 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf)
 {
@@ -2297,8 +2332,8 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		goto out;
 
 	if (unlikely(!pmd_present(orig_pmd))) {
-		VM_BUG_ON(thp_migration_supported() &&
-				  !pmd_is_migration_entry(orig_pmd));
+		VM_WARN_ON_ONCE(!pmd_is_migration_entry(orig_pmd) &&
+				!pmd_is_device_private_entry(orig_pmd));
 		goto out;
 	}
 
@@ -2494,9 +2529,9 @@ static pmd_t clear_uffd_wp_pmd(pmd_t pmd)
 	if (pmd_none(pmd))
 		return pmd;
 	if (pmd_present(pmd))
-		pmd = pmd_clear_uffd_wp(pmd);
+		pmd = pmd_clear_uffd(pmd);
 	else
-		pmd = pmd_swp_clear_uffd_wp(pmd);
+		pmd = pmd_swp_clear_uffd(pmd);
 
 	return pmd;
 }
@@ -2539,8 +2574,19 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
 			pgtable_trans_huge_deposit(mm, new_pmd, pgtable);
 		}
 		pmd = move_soft_dirty_pmd(pmd);
-		if (vma_has_uffd_without_event_remap(vma))
+		if (vma_has_uffd_without_event_remap(vma)) {
+			/*
+			 * See __copy_present_ptes(): normalise the RWP marker
+			 * so the destination starts accessible instead of
+			 * taking a numa-hinting fault on first access. Only the
+			 * marker (protnone + uffd) needs it; leave other present
+			 * PMDs in the VMA untouched.
+			 */
+			if (pmd_present(pmd) && userfaultfd_rwp(vma) &&
+			    pmd_uffd(pmd))
+				pmd = pmd_modify(pmd, vma->vm_page_prot);
 			pmd = clear_uffd_wp_pmd(pmd);
+		}
 		set_pmd_at(mm, new_addr, new_pmd, pmd);
 		if (force_flush)
 			flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
@@ -2553,15 +2599,16 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
 }
 
 static void change_non_present_huge_pmd(struct mm_struct *mm,
-		unsigned long addr, pmd_t *pmd, bool uffd_wp,
-		bool uffd_wp_resolve)
+		unsigned long addr, pmd_t *pmd, bool uffd_prot,
+		bool uffd_prot_resolve)
 {
 	softleaf_t entry = softleaf_from_pmd(*pmd);
-	const struct folio *folio = softleaf_to_folio(entry);
 	pmd_t newpmd;
 
 	VM_WARN_ON(!pmd_is_valid_softleaf(*pmd));
 	if (softleaf_is_migration_write(entry)) {
+		const struct folio *folio = softleaf_to_folio(entry);
+
 		/*
 		 * A protection check is difficult so
 		 * just be safe and disable write
@@ -2570,22 +2617,22 @@ static void change_non_present_huge_pmd(struct mm_struct *mm,
 			entry = make_readable_exclusive_migration_entry(swp_offset(entry));
 		else
 			entry = make_readable_migration_entry(swp_offset(entry));
-		newpmd = swp_entry_to_pmd(entry);
+		newpmd = softleaf_to_pmd(entry);
 		if (pmd_swp_soft_dirty(*pmd))
 			newpmd = pmd_swp_mksoft_dirty(newpmd);
 	} else if (softleaf_is_device_private_write(entry)) {
 		entry = make_readable_device_private_entry(swp_offset(entry));
-		newpmd = swp_entry_to_pmd(entry);
-		if (pmd_swp_uffd_wp(*pmd))
-			newpmd = pmd_swp_mkuffd_wp(newpmd);
+		newpmd = softleaf_to_pmd(entry);
+		if (pmd_swp_uffd(*pmd))
+			newpmd = pmd_swp_mkuffd(newpmd);
 	} else {
 		newpmd = *pmd;
 	}
 
-	if (uffd_wp)
-		newpmd = pmd_swp_mkuffd_wp(newpmd);
-	else if (uffd_wp_resolve)
-		newpmd = pmd_swp_clear_uffd_wp(newpmd);
+	if (uffd_prot)
+		newpmd = pmd_swp_mkuffd(newpmd);
+	else if (uffd_prot_resolve)
+		newpmd = pmd_swp_clear_uffd(newpmd);
 	if (!pmd_same(*pmd, newpmd))
 		set_pmd_at(mm, addr, pmd, newpmd);
 }
@@ -2605,8 +2652,9 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	spinlock_t *ptl;
 	pmd_t oldpmd, entry;
 	bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
-	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
-	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
+	bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP);
+	bool uffd_prot_resolve = cp_flags &
+		(MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE);
 	int ret = 1;
 
 	tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
@@ -2619,11 +2667,17 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		return 0;
 
 	if (thp_migration_supported() && pmd_is_valid_softleaf(*pmd)) {
-		change_non_present_huge_pmd(mm, addr, pmd, uffd_wp,
-					    uffd_wp_resolve);
+		change_non_present_huge_pmd(mm, addr, pmd, uffd_prot,
+					    uffd_prot_resolve);
 		goto unlock;
 	}
 
+	/* Already in the desired state */
+	if (prot_numa && pmd_protnone(*pmd))
+		goto unlock;
+	if ((cp_flags & MM_CP_UFFD_RWP) && pmd_protnone(*pmd) && pmd_uffd(*pmd))
+		goto unlock;
+
 	if (prot_numa) {
 
 		/*
@@ -2634,9 +2688,6 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		if (is_huge_zero_pmd(*pmd))
 			goto unlock;
 
-		if (pmd_protnone(*pmd))
-			goto unlock;
-
 		if (!folio_can_map_prot_numa(pmd_folio(*pmd), vma,
 					     vma_is_single_threaded_private(vma)))
 			goto unlock;
@@ -2665,15 +2716,19 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	oldpmd = pmdp_invalidate_ad(vma, addr, pmd);
 
 	entry = pmd_modify(oldpmd, newprot);
-	if (uffd_wp)
-		entry = pmd_mkuffd_wp(entry);
-	else if (uffd_wp_resolve)
+	if (uffd_prot)
+		entry = pmd_mkuffd(entry);
+	else if (uffd_prot_resolve)
 		/*
 		 * Leave the write bit to be handled by PF interrupt
 		 * handler, then things like COW could be properly
 		 * handled.
 		 */
-		entry = pmd_clear_uffd_wp(entry);
+		entry = pmd_clear_uffd(entry);
+
+	/* See change_pte_range(): preserve RWP protection across mprotect() */
+	if (userfaultfd_rwp(vma) && pmd_uffd(entry))
+		entry = pmd_modify(entry, PAGE_NONE);
 
 	/* See change_pte_range(). */
 	if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) &&
@@ -2713,10 +2768,10 @@ int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		return 1;
 
 	/*
-	 * Huge entries on userfault-wp only works with anonymous, while we
-	 * don't have anonymous PUDs yet.
+	 * Huge entries on userfault-wp or userfault-rwp only work with
+	 * anonymous, while we don't have anonymous PUDs yet.
 	 */
-	if (WARN_ON_ONCE(cp_flags & MM_CP_UFFD_WP_ALL))
+	if (WARN_ON_ONCE(cp_flags & (MM_CP_UFFD_WP_ALL | MM_CP_UFFD_RWP_ALL)))
 		return 1;
 
 	ptl = __pud_trans_huge_lock(pudp, vma);
@@ -2774,7 +2829,7 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm
 	if (!pmd_trans_huge(src_pmdval)) {
 		spin_unlock(src_ptl);
 		if (pmd_is_migration_entry(src_pmdval)) {
-			pmd_migration_entry_wait(mm, &src_pmdval);
+			pmd_migration_entry_wait(mm, src_pmd);
 			return -EAGAIN;
 		}
 		return -ENOENT;
@@ -2842,6 +2897,13 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm
 		_dst_pmd = move_soft_dirty_pmd(src_pmdval);
 		_dst_pmd = clear_uffd_wp_pmd(_dst_pmd);
 	}
+
+	/* Re-arm RWP on the moved PMD if dst_vma is RWP-registered. */
+	if (userfaultfd_rwp(dst_vma)) {
+		_dst_pmd = pmd_modify(_dst_pmd, PAGE_NONE);
+		_dst_pmd = pmd_mkuffd(_dst_pmd);
+	}
+
 	set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd);
 
 	src_pgtable = pgtable_trans_huge_withdraw(mm, src_pmd);
@@ -3016,8 +3078,13 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
 
 		entry = pfn_pte(zero_pfn(addr), vma->vm_page_prot);
 		entry = pte_mkspecial(entry);
-		if (pmd_uffd_wp(old_pmd))
-			entry = pte_mkuffd_wp(entry);
+		if (pmd_uffd(old_pmd))
+			entry = pte_mkuffd(entry);
+
+		/* Restore PAGE_NONE so an RWP marker keeps trapping */
+		if (userfaultfd_rwp(vma) && pmd_uffd(old_pmd))
+			entry = pte_modify(entry, PAGE_NONE);
+
 		VM_BUG_ON(!pte_none(ptep_get(pte)));
 		set_pte_at(mm, addr, pte, entry);
 		pte++;
@@ -3103,7 +3170,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
 		folio = page_folio(page);
 
 		soft_dirty = pmd_swp_soft_dirty(old_pmd);
-		uffd_wp = pmd_swp_uffd_wp(old_pmd);
+		uffd_wp = pmd_swp_uffd(old_pmd);
 
 		write = softleaf_is_migration_write(entry);
 		if (PageAnon(page))
@@ -3119,7 +3186,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
 		folio = page_folio(page);
 
 		soft_dirty = pmd_swp_soft_dirty(old_pmd);
-		uffd_wp = pmd_swp_uffd_wp(old_pmd);
+		uffd_wp = pmd_swp_uffd(old_pmd);
 
 		write = softleaf_is_device_private_write(entry);
 		anon_exclusive = PageAnonExclusive(page);
@@ -3176,7 +3243,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
 		write = pmd_write(old_pmd);
 		young = pmd_young(old_pmd);
 		soft_dirty = pmd_soft_dirty(old_pmd);
-		uffd_wp = pmd_uffd_wp(old_pmd);
+		uffd_wp = pmd_uffd(old_pmd);
 
 		VM_WARN_ON_FOLIO(!folio_ref_count(folio), folio);
 		VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
@@ -3247,7 +3314,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
 			if (soft_dirty)
 				entry = pte_swp_mksoft_dirty(entry);
 			if (uffd_wp)
-				entry = pte_swp_mkuffd_wp(entry);
+				entry = pte_swp_mkuffd(entry);
 			VM_WARN_ON(!pte_none(ptep_get(pte + i)));
 			set_pte_at(mm, addr, pte + i, entry);
 		}
@@ -3274,7 +3341,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
 			if (soft_dirty)
 				entry = pte_swp_mksoft_dirty(entry);
 			if (uffd_wp)
-				entry = pte_swp_mkuffd_wp(entry);
+				entry = pte_swp_mkuffd(entry);
 			VM_WARN_ON(!pte_none(ptep_get(pte + i)));
 			set_pte_at(mm, addr, pte + i, entry);
 		}
@@ -3292,7 +3359,11 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
 		if (soft_dirty)
 			entry = pte_mksoft_dirty(entry);
 		if (uffd_wp)
-			entry = pte_mkuffd_wp(entry);
+			entry = pte_mkuffd(entry);
+
+		/* Restore PAGE_NONE so an RWP marker keeps trapping */
+		if (userfaultfd_rwp(vma) && uffd_wp)
+			entry = pte_modify(entry, PAGE_NONE);
 
 		for (i = 0; i < HPAGE_PMD_NR; i++)
 			VM_WARN_ON(!pte_none(ptep_get(pte + i)));
@@ -3587,13 +3658,16 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
 				 (1L << PG_dropbehind) |
 				 LRU_GEN_MASK | LRU_REFS_MASK));
 
-		if (handle_hwpoison &&
-		    page_range_has_hwpoisoned(new_head, new_nr_pages))
-			folio_set_has_hwpoisoned(new_folio);
-
 		new_folio->mapping = folio->mapping;
 		new_folio->index = folio->index + i;
 
+		/*
+		 * page->private should not be set in tail pages. Warn once
+		 * if private is unexpectedly set. Do it before swap.val assignment
+		 * since private overlaps with swap.val.
+		 */
+		VM_WARN_ON_ONCE_PAGE(new_folio->private, new_head);
+
 		if (folio_test_swapcache(folio))
 			new_folio->swap.val = folio->swap.val + i;
 
@@ -3612,6 +3686,14 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
 			folio_set_large_rmappable(new_folio);
 		}
 
+		/*
+		 * PG_has_hwpoisoned is on the 2nd page, so set it after
+		 * the compound head is prepped.
+		 */
+		if (handle_hwpoison &&
+		    page_range_has_hwpoisoned(new_head, new_nr_pages))
+			folio_set_has_hwpoisoned(new_folio);
+
 		if (folio_test_young(folio))
 			folio_set_young(new_folio);
 		if (folio_test_idle(folio))
@@ -4863,7 +4945,7 @@ static int __init split_huge_pages_debugfs(void)
 late_initcall(split_huge_pages_debugfs);
 #endif
 
-#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
 int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
 		struct page *page)
 {
@@ -4887,7 +4969,7 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
 
 		writable = pmd_write(pmdval);
 		softdirty = pmd_soft_dirty(pmdval);
-		uffd_wp = pmd_uffd_wp(pmdval);
+		uffd_wp = pmd_uffd(pmdval);
 	} else {
 		softleaf_t old_entry;
 
@@ -4896,7 +4978,7 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
 
 		writable = softleaf_is_device_private_write(old_entry);
 		softdirty = pmd_swp_soft_dirty(pmdval);
-		uffd_wp = pmd_swp_uffd_wp(pmdval);
+		uffd_wp = pmd_swp_uffd(pmdval);
 	}
 
 	/* See folio_try_share_anon_rmap_pmd(): invalidate PMD first. */
@@ -4923,11 +5005,11 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
 	}
 
 	/* Set PMD. */
-	pmdswp = swp_entry_to_pmd(entry);
+	pmdswp = softleaf_to_pmd(entry);
 	if (softdirty)
 		pmdswp = pmd_swp_mksoft_dirty(pmdswp);
 	if (uffd_wp)
-		pmdswp = pmd_swp_mkuffd_wp(pmdswp);
+		pmdswp = pmd_swp_mkuffd(pmdswp);
 	set_pmd_at(mm, address, pvmw->pmd, pmdswp);
 
 	/* Migration entry installed: cleanup rmap, folio. */
@@ -4959,8 +5041,13 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)
 		pmde = pmd_mksoft_dirty(pmde);
 	if (softleaf_is_migration_write(entry))
 		pmde = pmd_mkwrite(pmde, vma);
-	if (pmd_swp_uffd_wp(*pvmw->pmd))
-		pmde = pmd_mkuffd_wp(pmde);
+	if (pmd_swp_uffd(*pvmw->pmd))
+		pmde = pmd_mkuffd(pmde);
+
+	/* See do_swap_page(): restore PAGE_NONE for RWP */
+	if (pmd_swp_uffd(*pvmw->pmd) && userfaultfd_rwp(vma))
+		pmde = pmd_modify(pmde, PAGE_NONE);
+
 	if (!softleaf_is_migration_young(entry))
 		pmde = pmd_mkold(pmde);
 	/* NOTE: this may contain setting soft-dirty on some archs */
@@ -4976,12 +5063,12 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)
 		else
 			entry = make_readable_device_private_entry(
 							page_to_pfn(new));
-		pmde = swp_entry_to_pmd(entry);
+		pmde = softleaf_to_pmd(entry);
 
 		if (pmd_swp_soft_dirty(*pvmw->pmd))
 			pmde = pmd_swp_mksoft_dirty(pmde);
-		if (pmd_swp_uffd_wp(*pvmw->pmd))
-			pmde = pmd_swp_mkuffd_wp(pmde);
+		if (pmd_swp_uffd(*pvmw->pmd))
+			pmde = pmd_swp_mkuffd(pmde);
 	}
 
 	if (folio_test_anon(folio)) {
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 571212b80835e..db09f0ab6d971 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -47,9 +47,11 @@
 #include <linux/node.h>
 #include <linux/page_owner.h>
 #include "internal.h"
+#include "page_alloc.h"
 #include "hugetlb_vmemmap.h"
 #include "hugetlb_cma.h"
 #include "hugetlb_internal.h"
+#include "mm_init.h"
 #include <linux/page-isolation.h>
 
 int hugetlb_max_hstate __read_mostly;
@@ -58,7 +60,6 @@ struct hstate hstates[HUGE_MAX_HSTATE];
 
 __initdata nodemask_t hugetlb_bootmem_nodes;
 __initdata struct list_head huge_boot_pages[MAX_NUMNODES];
-static unsigned long hstate_boot_nrinvalid[HUGE_MAX_HSTATE] __initdata;
 
 /*
  * Due to ordering constraints across the init code for various
@@ -693,7 +694,7 @@ static int allocate_file_region_entries(struct resv_map *resv,
 
 		spin_lock(&resv->lock);
 
-		list_splice(&allocated_regions, &resv->region_cache);
+		list_splice_init(&allocated_regions, &resv->region_cache);
 		resv->region_cache_count += to_allocate;
 	}
 
@@ -1011,8 +1012,7 @@ static long region_count(struct resv_map *resv, long f, long t)
 static pgoff_t vma_hugecache_offset(struct hstate *h,
 			struct vm_area_struct *vma, unsigned long address)
 {
-	return ((address - vma->vm_start) >> huge_page_shift(h)) +
-			(vma->vm_pgoff >> huge_page_order(h));
+	return linear_page_index(vma, address) >> huge_page_order(h);
 }
 
 /*
@@ -1317,43 +1317,27 @@ static unsigned long available_huge_pages(struct hstate *h)
 	return h->free_huge_pages - h->resv_huge_pages;
 }
 
-static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h,
-				struct vm_area_struct *vma,
-				unsigned long address, long gbl_chg)
+static struct folio *dequeue_hugetlb_folio(struct hstate *h, gfp_t gfp_mask,
+					   struct mempolicy_interpreted *mpoli)
 {
+	nodemask_t *nodemask = mpoli->nodemask;
 	struct folio *folio = NULL;
-	struct mempolicy *mpol;
-	gfp_t gfp_mask;
-	nodemask_t *nodemask;
-	int nid;
 
-	/*
-	 * gbl_chg==1 means the allocation requires a new page that was not
-	 * reserved before.  Making sure there's at least one free page.
-	 */
-	if (gbl_chg && !available_huge_pages(h))
-		goto err;
-
-	gfp_mask = htlb_alloc_mask(h);
-	nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
-
-	if (mpol_is_preferred_many(mpol)) {
+	if (mpoli->mode == MPOL_PREFERRED_MANY) {
 		folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
-							nid, nodemask);
+						       mpoli->nid,
+						       nodemask);
 
 		/* Fallback to all nodes if page==NULL */
 		nodemask = NULL;
 	}
 
-	if (!folio)
+	if (!folio) {
 		folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
-							nid, nodemask);
-
-	mpol_cond_put(mpol);
+						       mpoli->nid,
+						       nodemask);
+	}
 	return folio;
-
-err:
-	return NULL;
 }
 
 #if defined(CONFIG_ARCH_HAS_GIGANTIC_PAGE) && defined(CONFIG_CONTIG_ALLOC)
@@ -1806,7 +1790,8 @@ static struct folio *alloc_buddy_frozen_folio(int order, gfp_t gfp_mask,
 	if (alloc_try_hard)
 		gfp_mask |= __GFP_RETRY_MAYFAIL;
 
-	folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask);
+	folio = (struct folio *)__alloc_frozen_pages(gfp_mask, order, nid, nmask,
+						     ALLOC_DEFAULT);
 
 	/*
 	 * If we did not specify __GFP_RETRY_MAYFAIL, but still got a
@@ -1864,6 +1849,18 @@ static struct folio *alloc_fresh_hugetlb_folio(struct hstate *h,
 		gfp_t gfp_mask, int nid, nodemask_t *nmask)
 {
 	struct folio *folio;
+	nodemask_t local_node_mask;
+
+	if (!nmask) {
+		unsigned int cpuset_mems_cookie;
+
+		do {
+			cpuset_mems_cookie = read_mems_allowed_begin();
+			local_node_mask = cpuset_current_mems_allowed;
+		} while (read_mems_allowed_retry(cpuset_mems_cookie));
+
+		nmask = &local_node_mask;
+	}
 
 	folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask, NULL);
 	if (folio)
@@ -2148,32 +2145,28 @@ static struct folio *alloc_migrate_hugetlb_folio(struct hstate *h, gfp_t gfp_mas
 	return folio;
 }
 
-/*
- * Use the VMA's mpolicy to allocate a huge page from the buddy.
- */
 static
-struct folio *alloc_buddy_hugetlb_folio_with_mpol(struct hstate *h,
-		struct vm_area_struct *vma, unsigned long addr)
+struct folio *alloc_buddy_hugetlb_folio(struct hstate *h,
+		gfp_t gfp_mask, struct mempolicy_interpreted *mpoli)
 {
 	struct folio *folio = NULL;
-	struct mempolicy *mpol;
-	gfp_t gfp_mask = htlb_alloc_mask(h);
-	int nid;
-	nodemask_t *nodemask;
+	nodemask_t *nodemask = mpoli->nodemask;
 
-	nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask);
-	if (mpol_is_preferred_many(mpol)) {
+	if (mpoli->mode == MPOL_PREFERRED_MANY) {
 		gfp_t gfp = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
 
-		folio = alloc_surplus_hugetlb_folio(h, gfp, nid, nodemask);
+		folio = alloc_surplus_hugetlb_folio(h, gfp, mpoli->nid,
+						    nodemask);
 
 		/* Fallback to all nodes if page==NULL */
 		nodemask = NULL;
 	}
 
-	if (!folio)
-		folio = alloc_surplus_hugetlb_folio(h, gfp_mask, nid, nodemask);
-	mpol_cond_put(mpol);
+	if (!folio) {
+		folio = alloc_surplus_hugetlb_folio(h, gfp_mask, mpoli->nid,
+						    nodemask);
+	}
+
 	return folio;
 }
 
@@ -2826,6 +2819,104 @@ void wait_for_freed_hugetlb_folios(void)
 	flush_work(&free_hpage_work);
 }
 
+/**
+ * hugetlb_alloc_folio - Allocate a hugetlb folio.
+ * @h: Hugetlb state control block.
+ * @mpoli: Interpreted memory policy to use for allocation.
+ * @alloc_flags: Flags controlling the allocation behavior.
+ *
+ * Allocates a hugetlb folio and handles cgroup charging and global hstate
+ * reservations.
+ *
+ * Return: A pointer to the allocated folio, or an ERR_PTR on failure.
+ *         -ENOSPC if cgroup charging fails or no folio is available.
+ *         -ENOMEM if mem cgroup charging fails.
+ */
+struct folio *hugetlb_alloc_folio(struct hstate *h,
+		struct mempolicy_interpreted *mpoli, u8 alloc_flags)
+{
+	bool charge_hugetlb_cgroup_rsvd = alloc_flags &
+					  HUGETLB_ALLOC_CHARG_CGROUP_RSVD;
+	bool use_global_reservation = alloc_flags &
+				      HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS;
+	size_t nr_pages = pages_per_huge_page(h);
+	struct hugetlb_cgroup *h_cg_rsvd = NULL;
+	struct hugetlb_cgroup *h_cg = NULL;
+	gfp_t gfp = htlb_alloc_mask(h);
+	int idx = hstate_index(h);
+	struct folio *folio;
+	int ret;
+
+	if (charge_hugetlb_cgroup_rsvd &&
+	    hugetlb_cgroup_charge_cgroup_rsvd(idx, nr_pages, &h_cg_rsvd))
+		return ERR_PTR(-ENOSPC);
+
+	if (hugetlb_cgroup_charge_cgroup(idx, nr_pages, &h_cg)) {
+		ret = -ENOSPC;
+		goto err_uncharge_hugetlb_cgroup_rsvd;
+	}
+
+	spin_lock_irq(&hugetlb_lock);
+
+	folio = NULL;
+	if (use_global_reservation || available_huge_pages(h))
+		folio = dequeue_hugetlb_folio(h, gfp, mpoli);
+
+	if (!folio) {
+		spin_unlock_irq(&hugetlb_lock);
+		folio = alloc_buddy_hugetlb_folio(h, gfp, mpoli);
+		if (!folio) {
+			ret = -ENOSPC;
+			goto err_uncharge_hugetlb_cgroup;
+		}
+		spin_lock_irq(&hugetlb_lock);
+		list_add(&folio->lru, &h->hugepage_activelist);
+		folio_ref_unfreeze(folio, 1);
+	}
+
+	if (use_global_reservation) {
+		folio_set_hugetlb_restore_reserve(folio);
+		h->resv_huge_pages--;
+	}
+
+	hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, folio);
+
+	if (charge_hugetlb_cgroup_rsvd) {
+		hugetlb_cgroup_commit_charge_rsvd(idx, nr_pages, h_cg_rsvd,
+						  folio);
+	}
+
+	spin_unlock_irq(&hugetlb_lock);
+
+	ret = mem_cgroup_charge_hugetlb(folio, gfp | __GFP_RETRY_MAYFAIL);
+	/*
+	 * Unconditionally increment NR_HUGETLB here because if
+	 * mem_cgroup_charge_hugetlb failed, freeing the page will
+	 * decrement NR_HUGETLB.
+	 */
+	lruvec_stat_mod_folio(folio, NR_HUGETLB, nr_pages);
+
+	if (ret == -ENOMEM) {
+		free_huge_folio(folio);
+		/*
+		 * Skip uncharging hugetlb_cgroup since the charges
+		 * were committed to the folio and freeing the folio
+		 * would have cleared those up.
+		 */
+		return ERR_PTR(ret);
+	}
+
+	return folio;
+
+ err_uncharge_hugetlb_cgroup:
+	hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg);
+ err_uncharge_hugetlb_cgroup_rsvd:
+	if (charge_hugetlb_cgroup_rsvd)
+		hugetlb_cgroup_uncharge_cgroup_rsvd(idx, nr_pages, h_cg_rsvd);
+
+	return ERR_PTR(ret);
+}
+
 typedef enum {
 	/*
 	 * For either 0/1: we checked the per-vma resv map, and one resv
@@ -2860,12 +2951,13 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
 	struct folio *folio;
 	long retval, gbl_chg, gbl_reserve;
 	map_chg_state map_chg;
-	int ret, idx;
-	struct hugetlb_cgroup *h_cg = NULL;
-	struct hugetlb_cgroup *h_cg_rsvd = NULL;
-	gfp_t gfp = htlb_alloc_mask(h) | __GFP_RETRY_MAYFAIL;
-
-	idx = hstate_index(h);
+	struct mempolicy_interpreted mpoli;
+	gfp_t gfp = htlb_alloc_mask(h);
+	struct mempolicy *mpol;
+	nodemask_t *nodemask;
+	u8 alloc_flags = 0;
+	int nid;
+	int ret;
 
 	/* Whether we need a separate per-vma reservation? */
 	if (cow_from_owner) {
@@ -2897,8 +2989,10 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
 	 */
 	if (map_chg) {
 		gbl_chg = hugepage_subpool_get_pages(spool, 1);
-		if (gbl_chg < 0)
+		if (gbl_chg < 0) {
+			ret = -ENOSPC;
 			goto out_end_reservation;
+		}
 	} else {
 		/*
 		 * If we have the vma reservation ready, no need for extra
@@ -2908,57 +3002,39 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
 	}
 
 	/*
-	 * If this allocation is not consuming a per-vma reservation,
-	 * charge the hugetlb cgroup now.
+	 * If allocation doesn't reuse a reservation in the resv_map,
+	 * charge for the reservation.
 	 */
-	if (map_chg) {
-		ret = hugetlb_cgroup_charge_cgroup_rsvd(
-			idx, pages_per_huge_page(h), &h_cg_rsvd);
-		if (ret)
-			goto out_subpool_put;
-	}
+	if (map_chg != MAP_CHG_REUSE)
+		alloc_flags |= HUGETLB_ALLOC_CHARG_CGROUP_RSVD;
 
-	ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
-	if (ret)
-		goto out_uncharge_cgroup_reservation;
-
-	spin_lock_irq(&hugetlb_lock);
 	/*
-	 * glb_chg is passed to indicate whether or not a page must be taken
-	 * from the global free pool (global change).  gbl_chg == 0 indicates
-	 * a reservation exists for the allocation.
+	 * gbl_chg == 0 indicates a reservation exists for this
+	 * allocation, so try to use it.
 	 */
-	folio = dequeue_hugetlb_folio_vma(h, vma, addr, gbl_chg);
-	if (!folio) {
-		spin_unlock_irq(&hugetlb_lock);
-		folio = alloc_buddy_hugetlb_folio_with_mpol(h, vma, addr);
-		if (!folio)
-			goto out_uncharge_cgroup;
-		spin_lock_irq(&hugetlb_lock);
-		list_add(&folio->lru, &h->hugepage_activelist);
-		folio_ref_unfreeze(folio, 1);
-		/* Fall through */
-	}
+	if (gbl_chg == 0)
+		alloc_flags |= HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS;
 
-	/*
-	 * Either dequeued or buddy-allocated folio needs to add special
-	 * mark to the folio when it consumes a global reservation.
-	 */
-	if (!gbl_chg) {
-		folio_set_hugetlb_restore_reserve(folio);
-		h->resv_huge_pages--;
-	}
+	/* Takes reference on mpol. */
+	nid = huge_node(vma, addr, gfp, &mpol, &nodemask);
+	mpoli = (struct mempolicy_interpreted){
+		.nid = nid,
+#ifdef CONFIG_NUMA
+		.mode = mpol ? mpol->mode : MPOL_DEFAULT,
+#else
+		.mode = MPOL_DEFAULT,
+#endif
+		.nodemask = nodemask,
+	};
 
-	hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, folio);
-	/* If allocation is not consuming a reservation, also store the
-	 * hugetlb_cgroup pointer on the page.
-	 */
-	if (map_chg) {
-		hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h),
-						  h_cg_rsvd, folio);
-	}
+	folio = hugetlb_alloc_folio(h, &mpoli, alloc_flags);
 
-	spin_unlock_irq(&hugetlb_lock);
+	mpol_cond_put(mpol);
+
+	if (IS_ERR(folio)) {
+		ret = PTR_ERR(folio);
+		goto out_subpool_put;
+	}
 
 	hugetlb_set_folio_subpool(folio, spool);
 
@@ -2987,27 +3063,8 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
 		}
 	}
 
-	ret = mem_cgroup_charge_hugetlb(folio, gfp);
-	/*
-	 * Unconditionally increment NR_HUGETLB here. If it turns out that
-	 * mem_cgroup_charge_hugetlb failed, then immediately free the page and
-	 * decrement NR_HUGETLB.
-	 */
-	lruvec_stat_mod_folio(folio, NR_HUGETLB, pages_per_huge_page(h));
-
-	if (ret == -ENOMEM) {
-		free_huge_folio(folio);
-		return ERR_PTR(-ENOMEM);
-	}
-
 	return folio;
 
-out_uncharge_cgroup:
-	hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
-out_uncharge_cgroup_reservation:
-	if (map_chg)
-		hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h),
-						    h_cg_rsvd);
 out_subpool_put:
 	/*
 	 * put page to subpool iff the quota of subpool's rsv_hpages is used
@@ -3018,100 +3075,94 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
 		hugetlb_acct_memory(h, -gbl_reserve);
 	}
 
-
 out_end_reservation:
 	if (map_chg != MAP_CHG_ENFORCED)
 		vma_end_reservation(h, vma, addr);
-	return ERR_PTR(-ENOSPC);
+	return ERR_PTR(ret);
 }
 
 static __init void *alloc_bootmem(struct hstate *h, int nid, bool node_exact)
 {
-	struct huge_bootmem_page *m;
-	int listnode = nid;
-
 	if (hugetlb_early_cma(h))
-		m = hugetlb_cma_alloc_bootmem(h, &listnode, node_exact);
-	else {
-		if (node_exact)
-			m = memblock_alloc_exact_nid_raw(huge_page_size(h),
+		return hugetlb_cma_alloc_bootmem(h, nid, node_exact);
+
+	if (node_exact)
+		return memblock_alloc_exact_nid_raw(huge_page_size(h),
 				huge_page_size(h), 0,
 				MEMBLOCK_ALLOC_ACCESSIBLE, nid);
-		else {
-			m = memblock_alloc_try_nid_raw(huge_page_size(h),
+
+	return memblock_alloc_try_nid_raw(huge_page_size(h),
 				huge_page_size(h), 0,
 				MEMBLOCK_ALLOC_ACCESSIBLE, nid);
-			/*
-			 * For pre-HVO to work correctly, pages need to be on
-			 * the list for the node they were actually allocated
-			 * from. That node may be different in the case of
-			 * fallback by memblock_alloc_try_nid_raw. So,
-			 * extract the actual node first.
-			 */
-			if (m)
-				listnode = early_pfn_to_nid(PHYS_PFN(__pa(m)));
-		}
-
-		if (m) {
-			m->flags = 0;
-			m->cma = NULL;
-		}
-	}
-
-	if (m) {
-		/*
-		 * Use the beginning of the huge page to store the
-		 * huge_bootmem_page struct (until gather_bootmem
-		 * puts them into the mem_map).
-		 *
-		 * Put them into a private list first because mem_map
-		 * is not up yet.
-		 */
-		INIT_LIST_HEAD(&m->list);
-		list_add(&m->list, &huge_boot_pages[listnode]);
-		m->hstate = h;
-	}
-
-	return m;
 }
 
-int alloc_bootmem_huge_page(struct hstate *h, int nid)
+void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid)
 	__attribute__ ((weak, alias("__alloc_bootmem_huge_page")));
-int __alloc_bootmem_huge_page(struct hstate *h, int nid)
+void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid)
 {
-	struct huge_bootmem_page *m = NULL; /* initialize for clang */
 	int nr_nodes, node = nid;
 
 	/* do node specific alloc */
-	if (nid != NUMA_NO_NODE) {
-		m = alloc_bootmem(h, node, true);
-		if (!m)
-			return 0;
-		goto found;
-	}
+	if (nid != NUMA_NO_NODE)
+		return alloc_bootmem(h, node, true);
 
 	/* allocate from next node when distributing huge pages */
 	for_each_node_mask_to_alloc(&h->next_nid_to_alloc, nr_nodes, node,
-				    &hugetlb_bootmem_nodes) {
-		m = alloc_bootmem(h, node, false);
-		if (!m)
-			return 0;
-		goto found;
-	}
+				    &hugetlb_bootmem_nodes)
+		return alloc_bootmem(h, node, false);
 
-found:
+	return NULL;
+}
 
+static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid)
+{
+	unsigned long pfn;
+	unsigned int nid_request = nid;
+	struct huge_bootmem_page *m = arch_alloc_bootmem_huge_page(h, nid);
+
+	if (!m)
+		return false;
+
+	pfn = PHYS_PFN(__pa(m));
+	nid = early_pfn_to_nid(pfn);
 	/*
-	 * Only initialize the head struct page in memmap_init_reserved_pages,
-	 * rest of the struct pages will be initialized by the HugeTLB
-	 * subsystem itself.
-	 * The head struct page is used to get folio information by the HugeTLB
-	 * subsystem like zone id and node id.
+	 * Use the beginning of the huge page to store the huge_bootmem_page
+	 * struct (until gather_bootmem puts them into the mem_map).
+	 *
+	 * Put them into a private list first because mem_map is not up yet.
 	 */
-	memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE),
-		huge_page_size(h) - PAGE_SIZE);
+	INIT_LIST_HEAD(&m->list);
+	m->hstate = h;
+	m->flags = hugetlb_early_cma(h) ? HUGE_BOOTMEM_CMA : 0;
 
-	return 1;
+	/* CMA pages: zone-crossing is validated in hugetlb_cma_reserve(). */
+	if (!hugetlb_early_cma(h) &&
+	    pfn_range_intersects_zones(nid, pfn, pages_per_huge_page(h))) {
+		/*
+		 * If the allocated page is on a different node than requested
+		 * (e.g., on PowerPC LPARs), put it on the requested node's list,
+		 * because hugetlb_free_cross_zone_pages() only frees cross-zone
+		 * pages belonging to the requested node.
+		 */
+		if (WARN_ON_ONCE(nid_request != NUMA_NO_NODE && nid != nid_request))
+			list_add(&m->list, &huge_boot_pages[nid_request]);
+		else
+			list_add(&m->list, &huge_boot_pages[nid]);
+	} else {
+		list_add_tail(&m->list, &huge_boot_pages[nid]);
+		m->flags |= HUGE_BOOTMEM_ZONES_VALID;
+		/*
+		 * Only initialize the head struct page in memmap_init_reserved_pages,
+		 * rest of the struct pages will be initialized by the HugeTLB
+		 * subsystem itself.
+		 * The head struct page is used to get folio information by the HugeTLB
+		 * subsystem like zone id and node id.
+		 */
+		memblock_reserved_mark_noinit(__pa((void *)m + PAGE_SIZE),
+				huge_page_size(h) - PAGE_SIZE);
+	}
+
+	return true;
 }
 
 /* Initialize [start_page:end_page_number] tail struct pages of a hugepage */
@@ -3223,57 +3274,6 @@ static void __init prep_and_add_bootmem_folios(struct hstate *h,
 	}
 }
 
-bool __init hugetlb_bootmem_page_zones_valid(int nid,
-					     struct huge_bootmem_page *m)
-{
-	unsigned long start_pfn;
-	bool valid;
-
-	if (m->flags & HUGE_BOOTMEM_ZONES_VALID) {
-		/*
-		 * Already validated, skip check.
-		 */
-		return true;
-	}
-
-	if (hugetlb_bootmem_page_earlycma(m)) {
-		valid = cma_validate_zones(m->cma);
-		goto out;
-	}
-
-	start_pfn = virt_to_phys(m) >> PAGE_SHIFT;
-
-	valid = !pfn_range_intersects_zones(nid, start_pfn,
-			pages_per_huge_page(m->hstate));
-out:
-	if (!valid)
-		hstate_boot_nrinvalid[hstate_index(m->hstate)]++;
-
-	return valid;
-}
-
-/*
- * Free a bootmem page that was found to be invalid (intersecting with
- * multiple zones).
- *
- * Since it intersects with multiple zones, we can't just do a free
- * operation on all pages at once, but instead have to walk all
- * pages, freeing them one by one.
- */
-static void __init hugetlb_bootmem_free_invalid_page(int nid, struct page *page,
-					     struct hstate *h)
-{
-	unsigned long npages = pages_per_huge_page(h);
-	unsigned long pfn;
-
-	while (npages--) {
-		pfn = page_to_pfn(page);
-		__init_page_from_nid(pfn, nid);
-		free_reserved_page(page);
-		page++;
-	}
-}
-
 /*
  * Put bootmem huge pages into the standard lists after mem_map is up.
  * Note: This only applies to gigantic (order > MAX_PAGE_ORDER) pages.
@@ -3289,17 +3289,6 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid)
 		struct folio *folio = (void *)page;
 
 		h = m->hstate;
-		if (!hugetlb_bootmem_page_zones_valid(nid, m)) {
-			/*
-			 * Can't use this page. Initialize the
-			 * page structures if that hasn't already
-			 * been done, and give them to the page
-			 * allocator.
-			 */
-			hugetlb_bootmem_free_invalid_page(nid, page, h);
-			continue;
-		}
-
 		/*
 		 * It is possible to have multiple huge page sizes (hstates)
 		 * in this list.  If so, process each size separately.
@@ -3353,7 +3342,7 @@ static void __init gather_bootmem_prealloc_parallel(unsigned long start,
 		gather_bootmem_prealloc_node(nid);
 }
 
-static void __init gather_bootmem_prealloc(void)
+void __init hugetlb_bootmem_struct_page_init(void)
 {
 	struct padata_mt_job job = {
 		.thread_fn	= gather_bootmem_prealloc_parallel,
@@ -3365,10 +3354,63 @@ static void __init gather_bootmem_prealloc(void)
 		.max_threads	= num_node_state(N_MEMORY),
 		.numa_aware	= true,
 	};
+#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
+	struct zone *zone;
+
+	for_each_zone(zone) {
+		for (int i = 0; i < NR_VMEMMAP_TAILS; i++) {
+			struct page *tail, *p;
+			unsigned int order;
+
+			tail = zone->vmemmap_tails[i];
+			if (!tail)
+				continue;
+
+			order = i + VMEMMAP_TAIL_MIN_ORDER;
+			p = page_to_virt(tail);
+			/*
+			 * prep_and_add_bootmem_folios() can access pageblock
+			 * flags on bootmem HugeTLB pages, so initialize the
+			 * shared tail struct pages here before bootmem folios
+			 * start using them.
+			 */
+			for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++)
+				init_compound_tail(p + j, NULL, order, zone);
+		}
+	}
+#endif
 
 	padata_do_multithreaded(&job);
 }
 
+static unsigned long __init hugetlb_free_cross_zone_pages(struct hstate *h, int nid)
+{
+	unsigned long freed = 0;
+	struct huge_bootmem_page *m, *tmp;
+
+	if (!hstate_is_gigantic(h))
+		return freed;
+
+	list_for_each_entry_safe(m, tmp, &huge_boot_pages[nid], list) {
+		if (m->flags & HUGE_BOOTMEM_ZONES_VALID)
+			break;
+
+		list_del(&m->list);
+		memblock_free(m, huge_page_size(h));
+		freed++;
+	}
+
+	if (freed) {
+		char buf[32];
+
+		string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, sizeof(buf));
+		pr_warn("HugeTLB: freed %lu cross-zone hugepages of size %s on node %d.\n",
+			freed, buf, nid);
+	}
+
+	return freed;
+}
+
 static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid)
 {
 	unsigned long i;
@@ -3399,6 +3441,8 @@ static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid)
 		cond_resched();
 	}
 
+	i -= hugetlb_free_cross_zone_pages(h, nid);
+
 	if (!list_empty(&folio_list))
 		prep_and_add_allocated_folios(h, &folio_list);
 
@@ -3472,6 +3516,7 @@ static void __init hugetlb_pages_alloc_boot_node(unsigned long start, unsigned l
 
 static unsigned long __init hugetlb_gigantic_pages_alloc_boot(struct hstate *h)
 {
+	int nid;
 	unsigned long i;
 
 	for (i = 0; i < h->max_huge_pages; ++i) {
@@ -3480,6 +3525,9 @@ static unsigned long __init hugetlb_gigantic_pages_alloc_boot(struct hstate *h)
 		cond_resched();
 	}
 
+	for_each_node(nid)
+		i -= hugetlb_free_cross_zone_pages(h, nid);
+
 	return i;
 }
 
@@ -3557,7 +3605,7 @@ static unsigned long __init hugetlb_pages_alloc_boot(struct hstate *h)
  * - For gigantic pages, this is called early in the boot process and
  *   pages are allocated from memblock allocated or something similar.
  *   Gigantic pages are actually added to pools later with the routine
- *   gather_bootmem_prealloc.
+ *   hugetlb_bootmem_struct_page_init.
  * - For non-gigantic pages, this is called later in the boot process after
  *   all of mm is up and functional.  Pages are allocated from buddy and
  *   then added to hugetlb pools.
@@ -3635,20 +3683,13 @@ static void __init hugetlb_init_hstates(void)
 static void __init report_hugepages(void)
 {
 	struct hstate *h;
-	unsigned long nrinvalid;
 
 	for_each_hstate(h) {
 		char buf[32];
 
-		nrinvalid = hstate_boot_nrinvalid[hstate_index(h)];
-		h->max_huge_pages -= nrinvalid;
-
 		string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
 		pr_info("HugeTLB: registered %s page size, pre-allocated %ld pages\n",
 			buf, h->nr_huge_pages);
-		if (nrinvalid)
-			pr_info("HugeTLB: %s page size: %lu invalid page%s discarded\n",
-					buf, nrinvalid, str_plural(nrinvalid));
 		pr_info("HugeTLB: %d KiB vmemmap can be freed for a %s page\n",
 			hugetlb_vmemmap_optimizable_size(h) / SZ_1K, buf);
 	}
@@ -4127,7 +4168,6 @@ static int __init hugetlb_init(void)
 	}
 
 	hugetlb_init_hstates();
-	gather_bootmem_prealloc();
 	report_hugepages();
 
 	hugetlb_sysfs_init();
@@ -4444,15 +4484,12 @@ hugetlb_early_param("default_hugepagesz", default_hugepagesz_setup);
 void __init hugetlb_bootmem_set_nodes(void)
 {
 	int i, nid;
-	unsigned long start_pfn, end_pfn;
 
 	if (!nodes_empty(hugetlb_bootmem_nodes))
 		return;
 
-	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
-		if (end_pfn > start_pfn)
-			node_set(nid, hugetlb_bootmem_nodes);
-	}
+	for_each_mem_pfn_range(i, MAX_NUMNODES, NULL, NULL, &nid)
+		node_set(nid, hugetlb_bootmem_nodes);
 }
 
 void __init hugetlb_bootmem_alloc(void)
@@ -4843,8 +4880,16 @@ hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long add
 
 	__folio_mark_uptodate(new_folio);
 	hugetlb_add_new_anon_rmap(new_folio, vma, addr);
-	if (userfaultfd_wp(vma) && huge_pte_uffd_wp(old))
-		newpte = huge_pte_mkuffd_wp(newpte);
+	if (userfaultfd_protected(vma) && huge_pte_uffd(old)) {
+		newpte = huge_pte_mkuffd(newpte);
+		/* Restore PAGE_NONE so the RWP marker keeps trapping. */
+		if (userfaultfd_rwp(vma)) {
+			unsigned int shift = huge_page_shift(hstate_vma(vma));
+
+			newpte = huge_pte_modify(newpte, PAGE_NONE);
+			newpte = arch_make_huge_pte(newpte, shift, vma->vm_flags);
+		}
+	}
 	set_huge_pte_at(vma->vm_mm, addr, ptep, newpte, sz);
 	hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm);
 	folio_set_hugetlb_migratable(new_folio);
@@ -4917,11 +4962,15 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
 
 		softleaf = softleaf_from_pte(entry);
 		if (unlikely(softleaf_is_hwpoison(softleaf))) {
-			if (!userfaultfd_wp(dst_vma))
-				entry = huge_pte_clear_uffd_wp(entry);
+			/*
+			 * A hwpoison entry never carries the uffd-wp bit: it is
+			 * installed fresh by make_hwpoison_entry() and
+			 * hugetlb_change_protection() leaves it untouched, so
+			 * there is nothing to clear for the child.
+			 */
 			set_huge_pte_at(dst, addr, dst_pte, entry, sz);
 		} else if (unlikely(softleaf_is_migration(softleaf))) {
-			bool uffd_wp = pte_swp_uffd_wp(entry);
+			bool uffd = pte_swp_uffd(entry);
 
 			if (!softleaf_is_migration_read(softleaf) && cow) {
 				/*
@@ -4931,12 +4980,12 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
 				softleaf = make_readable_migration_entry(
 							swp_offset(softleaf));
 				entry = swp_entry_to_pte(softleaf);
-				if (userfaultfd_wp(src_vma) && uffd_wp)
-					entry = pte_swp_mkuffd_wp(entry);
+				if (userfaultfd_protected(src_vma) && uffd)
+					entry = pte_swp_mkuffd(entry);
 				set_huge_pte_at(src, addr, src_pte, entry, sz);
 			}
-			if (!userfaultfd_wp(dst_vma))
-				entry = huge_pte_clear_uffd_wp(entry);
+			if (!userfaultfd_protected(dst_vma))
+				entry = pte_swp_clear_uffd(entry);
 			set_huge_pte_at(dst, addr, dst_pte, entry, sz);
 		} else if (unlikely(pte_is_marker(entry))) {
 			const pte_marker marker = copy_pte_marker(softleaf, dst_vma);
@@ -5000,6 +5049,16 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
 				goto next;
 			}
 
+			/* See __copy_present_ptes(): restore accessible protection. */
+			if (!userfaultfd_protected(dst_vma)) {
+				if (userfaultfd_rwp(src_vma) && huge_pte_uffd(entry)) {
+					entry = huge_pte_modify(entry, dst_vma->vm_page_prot);
+					entry = arch_make_huge_pte(entry, huge_page_shift(h),
+								   dst_vma->vm_flags);
+				}
+				entry = huge_pte_clear_uffd(entry);
+			}
+
 			if (cow) {
 				/*
 				 * No need to notify as we are downgrading page
@@ -5012,9 +5071,6 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
 				entry = huge_pte_wrprotect(entry);
 			}
 
-			if (!userfaultfd_wp(dst_vma))
-				entry = huge_pte_clear_uffd_wp(entry);
-
 			set_huge_pte_at(dst, addr, dst_pte, entry, sz);
 			hugetlb_count_add(npages, dst);
 		}
@@ -5060,10 +5116,23 @@ static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr,
 		huge_pte_clear(mm, new_addr, dst_pte, sz);
 	} else {
 		if (need_clear_uffd_wp) {
-			if (pte_present(pte))
-				pte = huge_pte_clear_uffd_wp(pte);
-			else
-				pte = pte_swp_clear_uffd_wp(pte);
+			if (pte_present(pte)) {
+				/*
+				 * See __copy_present_ptes(): normalise the RWP
+				 * marker so the destination starts accessible
+				 * instead of taking a numa-hinting fault on
+				 * first access. Only the marker (protnone + uffd)
+				 * needs it; leave other present PTEs untouched.
+				 */
+				if (userfaultfd_rwp(vma) && huge_pte_uffd(pte)) {
+					pte = huge_pte_modify(pte, vma->vm_page_prot);
+					pte = arch_make_huge_pte(pte, huge_page_shift(h),
+								 vma->vm_flags);
+				}
+				pte = huge_pte_clear_uffd(pte);
+			} else {
+				pte = pte_swp_clear_uffd(pte);
+			}
 		}
 		set_huge_pte_at(mm, new_addr, dst_pte, pte, sz);
 	}
@@ -5197,7 +5266,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
 			 * drop the uffd-wp bit in this zap, then replace the
 			 * pte with a marker.
 			 */
-			if (pte_swp_uffd_wp_any(pte) &&
+			if (pte_swp_uffd_any(pte) &&
 			    !(zap_flags & ZAP_FLAG_DROP_MARKER))
 				set_huge_pte_at(mm, address, ptep,
 						make_pte_marker(PTE_MARKER_UFFD_WP),
@@ -5233,7 +5302,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		if (huge_pte_dirty(pte))
 			folio_mark_dirty(folio);
 		/* Leave a uffd-wp pte marker if needed */
-		if (huge_pte_uffd_wp(pte) &&
+		if (huge_pte_uffd(pte) &&
 		    !(zap_flags & ZAP_FLAG_DROP_MARKER))
 			set_huge_pte_at(mm, address, ptep,
 					make_pte_marker(PTE_MARKER_UFFD_WP),
@@ -5372,8 +5441,7 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
 	 * from page cache lookup which is in HPAGE_SIZE units.
 	 */
 	address = address & huge_page_mask(h);
-	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
-			vma->vm_pgoff;
+	pgoff = linear_page_index(vma, address);
 	mapping = vma->vm_file->f_mapping;
 
 	/*
@@ -5382,7 +5450,7 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
 	 * __unmap_hugepage_range() is called as the lock is already held
 	 */
 	i_mmap_lock_write(mapping);
-	vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
+	mapping_rmap_tree_foreach(iter_vma, mapping, pgoff, pgoff) {
 		/* Do not unmap the current VMA */
 		if (iter_vma == vma)
 			continue;
@@ -5437,7 +5505,7 @@ static vm_fault_t hugetlb_wp(struct vm_fault *vmf)
 	 * can trigger this, because hugetlb_fault() will always resolve
 	 * uffd-wp bit first.
 	 */
-	if (!unshare && huge_pte_uffd_wp(pte))
+	if (!unshare && huge_pte_uffd(pte))
 		return 0;
 
 	/* Let's take out MAP_SHARED mappings first. */
@@ -5581,8 +5649,8 @@ static vm_fault_t hugetlb_wp(struct vm_fault *vmf)
 		huge_ptep_clear_flush(vma, vmf->address, vmf->pte);
 		hugetlb_remove_rmap(old_folio);
 		hugetlb_add_new_anon_rmap(new_folio, vma, vmf->address);
-		if (huge_pte_uffd_wp(pte))
-			newpte = huge_pte_mkuffd_wp(newpte);
+		if (huge_pte_uffd(pte))
+			newpte = huge_pte_mkuffd(newpte);
 		set_huge_pte_at(mm, vmf->address, vmf->pte, newpte,
 				huge_page_size(h));
 		folio_set_hugetlb_migratable(new_folio);
@@ -5860,7 +5928,7 @@ static vm_fault_t hugetlb_no_page(struct address_space *mapping,
 	 * if populated.
 	 */
 	if (unlikely(pte_is_uffd_wp_marker(vmf->orig_pte)))
-		new_pte = huge_pte_mkuffd_wp(new_pte);
+		new_pte = huge_pte_mkuffd(new_pte);
 	set_huge_pte_at(mm, vmf->address, vmf->pte, new_pte, huge_page_size(h));
 
 	hugetlb_count_add(pages_per_huge_page(h), mm);
@@ -6035,6 +6103,47 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
 		goto out_mutex;
 	}
 
+	/*
+	 * Protnone hugetlb PTEs with the uffd bit are used by
+	 * userfaultfd RWP for access tracking. Plain PROT_NONE (without the
+	 * marker) is not an RWP fault and is not expected on hugetlb (no
+	 * NUMA hinting), so let normal hugetlb fault handling proceed.
+	 */
+	if (pte_protnone(vmf.orig_pte) && vma_is_accessible(vma) &&
+	    userfaultfd_rwp(vma) && huge_pte_uffd(vmf.orig_pte)) {
+		spinlock_t *ptl;
+		pte_t pte;
+
+		/* Sync: drop hugetlb locks before blocking in handle_userfault() */
+		if (!userfaultfd_rwp_async(vma))
+			return hugetlb_handle_userfault(&vmf, mapping, VM_UFFD_RWP);
+
+		ptl = huge_pte_lock(h, mm, vmf.pte);
+		pte = huge_ptep_get(mm, vmf.address, vmf.pte);
+		if (pte_protnone(pte) && huge_pte_uffd(pte)) {
+			unsigned int shift = huge_page_shift(h);
+
+			pte = huge_pte_modify(pte, vma->vm_page_prot);
+			pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
+			/* huge_pte_modify() preserves _PAGE_UFFD; drop it on resolution */
+			pte = huge_pte_clear_uffd(pte);
+			pte = pte_mkyoung(pte);
+			/*
+			 * Unlike do_uffd_rwp(), do not upgrade to writable
+			 * here. Hugetlb lacks a can_change_huge_pte_writable()
+			 * equivalent, so a write access will take a separate
+			 * COW fault — acceptable for the rare private hugetlb
+			 * case.
+			 */
+			set_huge_pte_at(mm, vmf.address, vmf.pte, pte,
+					huge_page_size(h));
+			update_mmu_cache(vma, vmf.address, vmf.pte);
+		}
+		spin_unlock(ptl);
+		ret = 0;
+		goto out_mutex;
+	}
+
 	/*
 	 * If we are going to COW/unshare the mapping later, we examine the
 	 * pending reservations for this page now. This will ensure that any
@@ -6058,7 +6167,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
 		goto out_ptl;
 
 	/* Handle userfault-wp first, before trying to lock more pages */
-	if (userfaultfd_wp(vma) && huge_pte_uffd_wp(huge_ptep_get(mm, vmf.address, vmf.pte)) &&
+	if (userfaultfd_wp(vma) && huge_pte_uffd(huge_ptep_get(mm, vmf.address, vmf.pte)) &&
 	    (flags & FAULT_FLAG_WRITE) && !huge_pte_write(vmf.orig_pte)) {
 		if (!userfaultfd_wp_async(vma)) {
 			spin_unlock(vmf.ptl);
@@ -6067,7 +6176,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
 			return handle_userfault(&vmf, VM_UFFD_WP);
 		}
 
-		vmf.orig_pte = huge_pte_clear_uffd_wp(vmf.orig_pte);
+		vmf.orig_pte = huge_pte_clear_uffd(vmf.orig_pte);
 		set_huge_pte_at(mm, vmf.address, vmf.pte, vmf.orig_pte,
 				huge_page_size(hstate_vma(vma)));
 		/* Fallthrough to CoW */
@@ -6352,7 +6461,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
 	_dst_pte = pte_mkyoung(_dst_pte);
 
 	if (wp_enabled)
-		_dst_pte = huge_pte_mkuffd_wp(_dst_pte);
+		_dst_pte = huge_pte_mkuffd(_dst_pte);
 
 	set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte, size);
 
@@ -6395,6 +6504,8 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
 	unsigned long last_addr_mask;
 	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
 	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
+	bool uffd_rwp = cp_flags & MM_CP_UFFD_RWP;
+	bool uffd_rwp_resolve = cp_flags & MM_CP_UFFD_RWP_RESOLVE;
 	struct mmu_gather tlb;
 
 	/*
@@ -6420,6 +6531,11 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
 
 		ptep = hugetlb_walk(vma, address, psize);
 		if (!ptep) {
+			/*
+			 * uffd_wp installs a pte marker on the unpopulated
+			 * entry; uffd_rwp does not install markers so the
+			 * allocation is unnecessary for it.
+			 */
 			if (!uffd_wp) {
 				address |= last_addr_mask;
 				continue;
@@ -6441,7 +6557,8 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
 			 * shouldn't happen at all.  Warn about it if it
 			 * happened due to some reason.
 			 */
-			WARN_ON_ONCE(uffd_wp || uffd_wp_resolve);
+			WARN_ON_ONCE(uffd_wp || uffd_wp_resolve ||
+				     uffd_rwp || uffd_rwp_resolve);
 			pages++;
 			spin_unlock(ptl);
 			address |= last_addr_mask;
@@ -6475,10 +6592,10 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
 				pages++;
 			}
 
-			if (uffd_wp)
-				newpte = pte_swp_mkuffd_wp(newpte);
-			else if (uffd_wp_resolve)
-				newpte = pte_swp_clear_uffd_wp(newpte);
+			if (uffd_wp || uffd_rwp)
+				newpte = pte_swp_mkuffd(newpte);
+			else if (uffd_wp_resolve || uffd_rwp_resolve)
+				newpte = pte_swp_clear_uffd(newpte);
 			if (!pte_same(pte, newpte))
 				set_huge_pte_at(mm, address, ptep, newpte, psize);
 		} else if (unlikely(pte_is_marker(pte))) {
@@ -6488,20 +6605,32 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
 			 * pte_marker_uffd_wp()==true implies !poison
 			 * because they're mutual exclusive.
 			 */
-			if (pte_is_uffd_wp_marker(pte) && uffd_wp_resolve)
+			if (pte_is_uffd_wp_marker(pte) &&
+			    (uffd_wp_resolve || uffd_rwp_resolve))
 				/* Safe to modify directly (non-present->none). */
 				huge_pte_clear(mm, address, ptep, psize);
 		} else {
 			pte_t old_pte;
 			unsigned int shift = huge_page_shift(hstate_vma(vma));
 
+			/* Already protnone with uffd bit set? Nothing to do. */
+			if (uffd_rwp && pte_protnone(pte) && huge_pte_uffd(pte))
+				goto next;
+
 			old_pte = huge_ptep_modify_prot_start(vma, address, ptep);
 			pte = huge_pte_modify(old_pte, newprot);
 			pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
-			if (uffd_wp)
-				pte = huge_pte_mkuffd_wp(pte);
-			else if (uffd_wp_resolve)
-				pte = huge_pte_clear_uffd_wp(pte);
+			if (uffd_wp || uffd_rwp)
+				pte = huge_pte_mkuffd(pte);
+			else if (uffd_wp_resolve || uffd_rwp_resolve)
+				pte = huge_pte_clear_uffd(pte);
+
+			/* Preserve RWP protection across mprotect() */
+			if (userfaultfd_rwp(vma) && huge_pte_uffd(pte)) {
+				pte = huge_pte_modify(pte, PAGE_NONE);
+				pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
+			}
+
 			huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte);
 			pages++;
 			tlb_remove_huge_tlb_entry(h, &tlb, ptep, address);
@@ -6771,7 +6900,7 @@ static unsigned long page_table_shareable(struct vm_area_struct *svma,
 				struct vm_area_struct *vma,
 				unsigned long addr, pgoff_t idx)
 {
-	unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
+	unsigned long saddr = ((idx - vma_start_pgoff(svma)) << PAGE_SHIFT) +
 				svma->vm_start;
 	unsigned long sbase = saddr & PUD_MASK;
 	unsigned long s_end = sbase + PUD_SIZE;
@@ -6856,15 +6985,14 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
 		      unsigned long addr, pud_t *pud)
 {
 	struct address_space *mapping = vma->vm_file->f_mapping;
-	pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
-			vma->vm_pgoff;
+	const pgoff_t idx = linear_page_index(vma, addr);
 	struct vm_area_struct *svma;
 	unsigned long saddr;
 	pte_t *spte = NULL;
 	pte_t *pte;
 
 	i_mmap_lock_read(mapping);
-	vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
+	mapping_rmap_tree_foreach(svma, mapping, idx, idx) {
 		if (svma == vma)
 			continue;
 
@@ -7182,7 +7310,8 @@ void folio_putback_hugetlb(struct folio *folio)
 	folio_put(folio);
 }
 
-void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason)
+void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio,
+			enum migrate_reason reason)
 {
 	struct hstate *h = folio_hstate(old_folio);
 
@@ -7211,14 +7340,14 @@ void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int re
 		 * There is no need to transfer the per-node surplus state
 		 * when we do not cross the node.
 		 */
-		if (new_nid == old_nid)
-			return;
-		spin_lock_irq(&hugetlb_lock);
-		if (h->surplus_huge_pages_node[old_nid]) {
-			h->surplus_huge_pages_node[old_nid]--;
-			h->surplus_huge_pages_node[new_nid]++;
+		if (new_nid != old_nid) {
+			spin_lock_irq(&hugetlb_lock);
+			if (h->surplus_huge_pages_node[old_nid]) {
+				h->surplus_huge_pages_node[old_nid]--;
+				h->surplus_huge_pages_node[new_nid]++;
+			}
+			spin_unlock_irq(&hugetlb_lock);
 		}
-		spin_unlock_irq(&hugetlb_lock);
 	}
 
 	/*
diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c
index 39344d6c78d84..07faf625675b7 100644
--- a/mm/hugetlb_cma.c
+++ b/mm/hugetlb_cma.c
@@ -9,6 +9,9 @@
 #include <asm/setup.h>
 
 #include <linux/hugetlb.h>
+#include <linux/memblock.h>
+#include <linux/math.h>
+#include <linux/math64.h>
 #include "internal.h"
 #include "hugetlb_cma.h"
 
@@ -18,6 +21,28 @@ static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata;
 static bool hugetlb_cma_only __ro_after_init;
 static unsigned long hugetlb_cma_size __ro_after_init;
 
+static unsigned int hugetlb_cma_percent __initdata;
+static unsigned int hugetlb_cma_percent_in_node[MAX_NUMNODES] __initdata;
+
+#ifdef CONFIG_NUMA
+static phys_addr_t __init memblock_node_memory_size(int nid)
+{
+	struct memblock_region *reg;
+	phys_addr_t size = 0;
+
+	for_each_mem_region(reg) {
+		if (reg->nid == nid)
+			size += reg->size;
+	}
+	return size;
+}
+#else
+static phys_addr_t __init memblock_node_memory_size(int nid)
+{
+	return memblock_phys_mem_size();
+}
+#endif
+
 void hugetlb_cma_free_frozen_folio(struct folio *folio)
 {
 	WARN_ON_ONCE(!cma_release_frozen(hugetlb_cma[folio_nid(folio)],
@@ -34,7 +59,7 @@ struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask,
 	if (!hugetlb_cma_size)
 		return NULL;
 
-	if (hugetlb_cma[nid])
+	if (hugetlb_cma[nid] && node_isset(nid, *nodemask))
 		page = cma_alloc_frozen_compound(hugetlb_cma[nid], order);
 
 	if (!page && !(gfp_mask & __GFP_THISNODE)) {
@@ -56,37 +81,27 @@ struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask,
 	return folio;
 }
 
-struct huge_bootmem_page * __init
-hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, bool node_exact)
+void * __init hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact)
 {
 	struct cma *cma;
-	struct huge_bootmem_page *m;
-	int node = *nid;
+	void *m;
+	int node;
 
-	cma = hugetlb_cma[*nid];
+	cma = hugetlb_cma[nid];
 	m = cma_reserve_early(cma, huge_page_size(h));
-	if (!m) {
-		if (node_exact)
-			return NULL;
-
-		for_each_node_mask(node, hugetlb_bootmem_nodes) {
-			cma = hugetlb_cma[node];
-			if (!cma || node == *nid)
-				continue;
-			m = cma_reserve_early(cma, huge_page_size(h));
-			if (m) {
-				*nid = node;
-				break;
-			}
-		}
-	}
+	if (m || node_exact)
+		return m;
 
-	if (m) {
-		m->flags = HUGE_BOOTMEM_CMA;
-		m->cma = cma;
+	for_each_node_mask(node, hugetlb_bootmem_nodes) {
+		cma = hugetlb_cma[node];
+		if (!cma || node == nid)
+			continue;
+		m = cma_reserve_early(cma, huge_page_size(h));
+		if (m)
+			return m;
 	}
 
-	return m;
+	return NULL;
 }
 
 static int __init cmdline_parse_hugetlb_cma(char *p)
@@ -100,14 +115,28 @@ static int __init cmdline_parse_hugetlb_cma(char *p)
 			break;
 
 		if (s[count] == ':') {
+			char *next;
+
 			if (tmp >= MAX_NUMNODES)
 				break;
 			nid = array_index_nospec(tmp, MAX_NUMNODES);
 
 			s += count + 1;
-			tmp = memparse(s, &s);
-			hugetlb_cma_size_in_node[nid] = tmp;
-			hugetlb_cma_size += tmp;
+			tmp = memparse(s, &next);
+			if (*next == '%') {
+				if (tmp > 100) {
+					pr_warn("hugetlb_cma: invalid percentage %lu for node %d\n",
+						tmp, nid);
+					break;
+				}
+				hugetlb_cma_percent_in_node[nid] = tmp;
+				hugetlb_cma_size_in_node[nid] = 0;
+				s = next + 1;
+			} else {
+				hugetlb_cma_size_in_node[nid] = tmp;
+				hugetlb_cma_percent_in_node[nid] = 0;
+				s = next;
+			}
 
 			/*
 			 * Skip the separator if have one, otherwise
@@ -118,7 +147,28 @@ static int __init cmdline_parse_hugetlb_cma(char *p)
 			else
 				break;
 		} else {
-			hugetlb_cma_size = memparse(p, &p);
+			char *next;
+
+			tmp = memparse(p, &next);
+			if (*next == '%') {
+				if (tmp > 100) {
+					pr_warn("hugetlb_cma: invalid percentage %lu\n", tmp);
+				} else {
+					hugetlb_cma_percent = tmp;
+					hugetlb_cma_size = 0;
+					for (nid = 0; nid < MAX_NUMNODES; nid++) {
+						hugetlb_cma_size_in_node[nid] = 0;
+						hugetlb_cma_percent_in_node[nid] = 0;
+					}
+				}
+			} else {
+				hugetlb_cma_size = tmp;
+				hugetlb_cma_percent = 0;
+				for (nid = 0; nid < MAX_NUMNODES; nid++) {
+					hugetlb_cma_size_in_node[nid] = 0;
+					hugetlb_cma_percent_in_node[nid] = 0;
+				}
+			}
 			break;
 		}
 	}
@@ -144,8 +194,36 @@ void __init hugetlb_cma_reserve(void)
 {
 	unsigned long size, reserved, per_node, order, gigantic_page_size;
 	bool node_specific_cma_alloc = false;
+	bool has_node_specific_param = false;
 	int nid;
 
+	for (nid = 0; nid < MAX_NUMNODES; nid++) {
+		if (hugetlb_cma_size_in_node[nid] || hugetlb_cma_percent_in_node[nid]) {
+			has_node_specific_param = true;
+			break;
+		}
+	}
+
+	if (has_node_specific_param) {
+		hugetlb_cma_size = 0;
+		for (nid = 0; nid < MAX_NUMNODES; nid++) {
+			if (hugetlb_cma_percent_in_node[nid]) {
+				phys_addr_t node_gfp_mem = memblock_node_memory_size(nid);
+				u64 s;
+
+				s = mul_u64_u32_div((u64)node_gfp_mem,
+						    hugetlb_cma_percent_in_node[nid],
+						    100);
+
+				hugetlb_cma_size_in_node[nid] = s;
+			}
+			hugetlb_cma_size += hugetlb_cma_size_in_node[nid];
+		}
+	} else if (hugetlb_cma_percent) {
+		hugetlb_cma_size = mul_u64_u32_div((u64)memblock_phys_mem_size(),
+						   hugetlb_cma_percent, 100);
+	}
+
 	if (!hugetlb_cma_size)
 		return;
 
@@ -231,9 +309,11 @@ void __init hugetlb_cma_reserve(void)
 		res = cma_declare_contiguous_multi(size, gigantic_page_size,
 					HUGETLB_PAGE_ORDER, name,
 					&hugetlb_cma[nid], nid);
-		if (res) {
-			pr_warn("hugetlb_cma: reservation failed: err %d, node %d",
+		if (res || !cma_validate_zones(hugetlb_cma[nid])) {
+			pr_warn("hugetlb_cma: %s: err %d, node %d\n",
+				res ? "reservation failed" : "reserved area spans zones",
 				res, nid);
+			hugetlb_cma[nid] = NULL;
 			continue;
 		}
 
diff --git a/mm/hugetlb_cma.h b/mm/hugetlb_cma.h
index c619c394b1ae2..3aa483573d170 100644
--- a/mm/hugetlb_cma.h
+++ b/mm/hugetlb_cma.h
@@ -6,8 +6,7 @@
 void hugetlb_cma_free_frozen_folio(struct folio *folio);
 struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask,
 				      int nid, nodemask_t *nodemask);
-struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid,
-						    bool node_exact);
+void *hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact);
 bool hugetlb_cma_exclusive_alloc(void);
 unsigned long hugetlb_cma_total_size(void);
 void hugetlb_cma_validate_params(void);
@@ -23,9 +22,8 @@ static inline struct folio *hugetlb_cma_alloc_frozen_folio(int order,
 	return NULL;
 }
 
-static inline
-struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid,
-						    bool node_exact)
+static inline void *hugetlb_cma_alloc_bootmem(struct hstate *h, int nid,
+					      bool node_exact)
 {
 	return NULL;
 }
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index 133b46dfb09f5..917db0984143c 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -12,7 +12,6 @@
 
 #include <linux/pgtable.h>
 #include <linux/moduleparam.h>
-#include <linux/bootmem_info.h>
 #include <linux/mmdebug.h>
 #include <linux/pagewalk.h>
 #include <linux/pgalloc.h>
@@ -177,13 +176,13 @@ static int vmemmap_remap_range(unsigned long start, unsigned long end,
  * Free a vmemmap page. A vmemmap page can be allocated from the memblock
  * allocator or buddy allocator. If the PG_reserved flag is set, it means
  * that it allocated from the memblock allocator, just free it via the
- * free_bootmem_page(). Otherwise, use __free_page().
+ * free_reserved_page(). Otherwise, use __free_page().
  */
 static inline void free_vmemmap_page(struct page *page)
 {
 	if (PageReserved(page)) {
 		memmap_boot_pages_add(-1);
-		free_bootmem_page(page);
+		free_reserved_page(page);
 	} else {
 		memmap_pages_add(-1);
 		__free_page(page);
@@ -624,6 +623,9 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h,
 	LIST_HEAD(vmemmap_pages);
 	unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH;
 
+	if (list_empty(folio_list))
+		return;
+
 	nr_to_optimize = 0;
 	list_for_each_entry(folio, folio_list, lru) {
 		int ret;
@@ -635,12 +637,9 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h,
 			 * mirrored tail page structs RO.
 			 */
 			spfn = (unsigned long)&folio->page;
-			epfn = spfn + pages_per_huge_page(h);
+			epfn = spfn + hugetlb_vmemmap_size(h);
 			vmemmap_wrprotect_hvo(spfn, epfn, folio_nid(folio),
 					HUGETLB_VMEMMAP_RESERVE_SIZE);
-			register_page_bootmem_memmap(pfn_to_section_nr(spfn),
-					&folio->page,
-					HUGETLB_VMEMMAP_RESERVE_SIZE);
 			continue;
 		}
 
@@ -745,6 +744,20 @@ static bool vmemmap_should_optimize_bootmem_page(struct huge_bootmem_page *m)
 	return true;
 }
 
+static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn)
+{
+	struct zone *zone;
+	enum zone_type zone_type;
+
+	for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
+		zone = &NODE_DATA(nid)->node_zones[zone_type];
+		if (zone_spans_pfn(zone, pfn))
+			return zone;
+	}
+
+	return NULL;
+}
+
 /*
  * Initialize memmap section for a gigantic page, HVO-style.
  */
@@ -752,6 +765,7 @@ void __init hugetlb_vmemmap_init_early(int nid)
 {
 	unsigned long psize, paddr, section_size;
 	unsigned long ns, i, pnum, pfn, nr_pages;
+	unsigned long start, end;
 	struct huge_bootmem_page *m = NULL;
 	void *map;
 
@@ -761,6 +775,8 @@ void __init hugetlb_vmemmap_init_early(int nid)
 	section_size = (1UL << PA_SECTION_SHIFT);
 
 	list_for_each_entry(m, &huge_boot_pages[nid], list) {
+		struct zone *zone;
+
 		if (!vmemmap_should_optimize_bootmem_page(m))
 			continue;
 
@@ -769,6 +785,14 @@ void __init hugetlb_vmemmap_init_early(int nid)
 		paddr = virt_to_phys(m);
 		pfn = PHYS_PFN(paddr);
 		map = pfn_to_page(pfn);
+		start = (unsigned long)map;
+		end = start + hugetlb_vmemmap_size(m->hstate);
+		zone = pfn_to_zone(nid, pfn);
+
+		if (vmemmap_populate_hvo(start, end, huge_page_order(m->hstate),
+					 zone, HUGETLB_VMEMMAP_RESERVE_SIZE))
+			panic("Failed to allocate memmap for HugeTLB page\n");
+		memmap_boot_pages_add(DIV_ROUND_UP(HUGETLB_VMEMMAP_RESERVE_SIZE, PAGE_SIZE));
 
 		pnum = pfn_to_section_nr(pfn);
 		ns = psize / section_size;
@@ -783,78 +807,6 @@ void __init hugetlb_vmemmap_init_early(int nid)
 		m->flags |= HUGE_BOOTMEM_HVO;
 	}
 }
-
-static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn)
-{
-	struct zone *zone;
-	enum zone_type zone_type;
-
-	for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
-		zone = &NODE_DATA(nid)->node_zones[zone_type];
-		if (zone_spans_pfn(zone, pfn))
-			return zone;
-	}
-
-	return NULL;
-}
-
-void __init hugetlb_vmemmap_init_late(int nid)
-{
-	struct huge_bootmem_page *m, *tm;
-	unsigned long phys, nr_pages, start, end;
-	unsigned long pfn, nr_mmap;
-	struct zone *zone = NULL;
-	struct hstate *h;
-	void *map;
-
-	if (!READ_ONCE(vmemmap_optimize_enabled))
-		return;
-
-	list_for_each_entry_safe(m, tm, &huge_boot_pages[nid], list) {
-		if (!(m->flags & HUGE_BOOTMEM_HVO))
-			continue;
-
-		phys = virt_to_phys(m);
-		h = m->hstate;
-		pfn = PHYS_PFN(phys);
-		nr_pages = pages_per_huge_page(h);
-		map = pfn_to_page(pfn);
-		start = (unsigned long)map;
-		end = start + nr_pages * sizeof(struct page);
-
-		if (!hugetlb_bootmem_page_zones_valid(nid, m)) {
-			/*
-			 * Oops, the hugetlb page spans multiple zones.
-			 * Remove it from the list, and populate it normally.
-			 */
-			list_del(&m->list);
-
-			vmemmap_populate(start, end, nid, NULL);
-			nr_mmap = end - start;
-			memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE));
-
-			memblock_phys_free(phys, huge_page_size(h));
-			continue;
-		}
-
-		if (!zone || !zone_spans_pfn(zone, pfn))
-			zone = pfn_to_zone(nid, pfn);
-		if (WARN_ON_ONCE(!zone))
-			continue;
-
-		if (vmemmap_populate_hvo(start, end, huge_page_order(h), zone,
-					 HUGETLB_VMEMMAP_RESERVE_SIZE) < 0) {
-			/* Fallback if HVO population fails */
-			vmemmap_populate(start, end, nid, NULL);
-			nr_mmap = end - start;
-		} else {
-			m->flags |= HUGE_BOOTMEM_ZONES_VALID;
-			nr_mmap = HUGETLB_VMEMMAP_RESERVE_SIZE;
-		}
-
-		memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE));
-	}
-}
 #endif
 
 static const struct ctl_table hugetlb_vmemmap_sysctls[] = {
@@ -870,27 +822,10 @@ static const struct ctl_table hugetlb_vmemmap_sysctls[] = {
 static int __init hugetlb_vmemmap_init(void)
 {
 	const struct hstate *h;
-	struct zone *zone;
 
 	/* HUGETLB_VMEMMAP_RESERVE_SIZE should cover all used struct pages */
 	BUILD_BUG_ON(__NR_USED_SUBPAGE > HUGETLB_VMEMMAP_RESERVE_PAGES);
 
-	for_each_zone(zone) {
-		for (int i = 0; i < NR_VMEMMAP_TAILS; i++) {
-			struct page *tail, *p;
-			unsigned int order;
-
-			tail = zone->vmemmap_tails[i];
-			if (!tail)
-				continue;
-
-			order = i + VMEMMAP_TAIL_MIN_ORDER;
-			p = page_to_virt(tail);
-			for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++)
-				init_compound_tail(p + j, NULL, order, zone);
-		}
-	}
-
 	for_each_hstate(h) {
 		if (hugetlb_vmemmap_optimizable(h)) {
 			register_sysctl_init("vm", hugetlb_vmemmap_sysctls);
diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h
index 18b4908252153..7ac49c52457dd 100644
--- a/mm/hugetlb_vmemmap.h
+++ b/mm/hugetlb_vmemmap.h
@@ -29,7 +29,6 @@ void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_l
 void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list);
 #ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT
 void hugetlb_vmemmap_init_early(int nid);
-void hugetlb_vmemmap_init_late(int nid);
 #endif
 
 
@@ -81,10 +80,6 @@ static inline void hugetlb_vmemmap_init_early(int nid)
 {
 }
 
-static inline void hugetlb_vmemmap_init_late(int nid)
-{
-}
-
 static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h)
 {
 	return 0;
diff --git a/mm/internal.h b/mm/internal.h
index 181e79f1d6a20..f26423de4ca28 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -23,6 +23,80 @@
 #include "vma.h"
 
 struct folio_batch;
+struct hstate;
+
+struct huge_bootmem_page {
+	struct list_head list;
+	struct hstate *hstate;
+	unsigned long flags;
+};
+
+/* mm/workingset.c */
+bool workingset_test_recent(void *shadow, bool file, bool *workingset,
+			    bool flush);
+void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages);
+void *workingset_eviction(struct folio *folio,
+			  struct mem_cgroup *target_memcg);
+void workingset_refault(struct folio *folio, void *shadow);
+void workingset_activation(struct folio *folio);
+
+/* mm/folio.c */
+void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file,
+		unsigned int nr_io, unsigned int nr_rotated);
+void lru_note_cost_refault(struct folio *folio);
+void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma);
+
+static inline bool folio_may_be_lru_cached(struct folio *folio)
+{
+	/*
+	 * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting.
+	 * Holding small numbers of low-order mTHP folios in per-CPU LRU cache
+	 * will be sensible, but nobody has implemented and tested that yet.
+	 */
+	return !folio_test_large(folio);
+}
+
+static inline void lru_cache_enable(void)
+{
+	atomic_dec(&lru_disable_count);
+}
+
+void lru_cache_disable(void);
+void lru_add_drain(void);
+void lru_add_drain_cpu(int cpu);
+void lru_add_drain_cpu_zone(struct zone *zone);
+void folio_deactivate(struct folio *folio);
+void folio_mark_lazyfree(struct folio *folio);
+
+/* mm/vmscan.c */
+unsigned long zone_reclaimable_pages(struct zone *zone);
+unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
+				gfp_t gfp_mask, const nodemask_t *mask);
+unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru,
+			      int zone_idx);
+
+#define MEMCG_RECLAIM_MAY_SWAP (1 << 1)
+#define MEMCG_RECLAIM_PROACTIVE (1 << 2)
+#define MIN_SWAPPINESS 0
+#define MAX_SWAPPINESS 200
+
+/* Just reclaim from anon folios in proactive memory reclaim */
+#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1)
+
+unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
+					   unsigned long nr_pages,
+					   gfp_t gfp_mask,
+					   unsigned int reclaim_options,
+					   int *swappiness);
+unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
+				     gfp_t gfp_mask, bool noswap,
+				     pg_data_t *pgdat,
+				     unsigned long *nr_scanned);
+
+#ifdef CONFIG_NUMA
+extern int sysctl_min_unmapped_ratio;
+extern int sysctl_min_slab_ratio;
+#endif
 
 /*
  * Maintains state across a page table move. The operation assumes both source
@@ -412,8 +486,8 @@ static inline pte_t pte_move_swp_offset(pte_t pte, long delta)
 		new = pte_swp_mksoft_dirty(new);
 	if (pte_swp_exclusive(pte))
 		new = pte_swp_mkexclusive(new);
-	if (pte_swp_uffd_wp(pte))
-		new = pte_swp_mkuffd_wp(new);
+	if (pte_swp_uffd(pte))
+		new = pte_swp_mkuffd(new);
 
 	return new;
 }
@@ -642,190 +716,16 @@ void set_recommended_min_free_kbytes(void);
 
 extern char * const zone_names[MAX_NR_ZONES];
 
-/* perform sanity checks on struct pages being allocated or freed */
-DECLARE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled);
-
 extern int min_free_kbytes;
 extern int defrag_mode;
 
 void setup_per_zone_wmarks(void);
 void calculate_min_free_kbytes(void);
 int __meminit init_per_zone_wmark_min(void);
-void page_alloc_sysctl_init(void);
-
-/*
- * Structure for holding the mostly immutable allocation parameters passed
- * between functions involved in allocations, including the alloc_pages*
- * family of functions.
- *
- * nodemask, migratetype and highest_zoneidx are initialized only once in
- * __alloc_pages() and then never change.
- *
- * zonelist, preferred_zone and highest_zoneidx are set first in
- * __alloc_pages() for the fast path, and might be later changed
- * in __alloc_pages_slowpath(). All other functions pass the whole structure
- * by a const pointer.
- */
-struct alloc_context {
-	struct zonelist *zonelist;
-	nodemask_t *nodemask;
-	struct zoneref *preferred_zoneref;
-	int migratetype;
-
-	/*
-	 * highest_zoneidx represents highest usable zone index of
-	 * the allocation request. Due to the nature of the zone,
-	 * memory on lower zone than the highest_zoneidx will be
-	 * protected by lowmem_reserve[highest_zoneidx].
-	 *
-	 * highest_zoneidx is also used by reclaim/compaction to limit
-	 * the target zone since higher zone than this index cannot be
-	 * usable for this allocation request.
-	 */
-	enum zone_type highest_zoneidx;
-	bool spread_dirty_pages;
-};
-
-/*
- * This function returns the order of a free page in the buddy system. In
- * general, page_zone(page)->lock must be held by the caller to prevent the
- * page from being allocated in parallel and returning garbage as the order.
- * If a caller does not hold page_zone(page)->lock, it must guarantee that the
- * page cannot be allocated or merged in parallel. Alternatively, it must
- * handle invalid values gracefully, and use buddy_order_unsafe() below.
- */
-static inline unsigned int buddy_order(struct page *page)
-{
-	/* PageBuddy() must be checked by the caller */
-	return page_private(page);
-}
-
-/*
- * Like buddy_order(), but for callers who cannot afford to hold the zone lock.
- * PageBuddy() should be checked first by the caller to minimize race window,
- * and invalid values must be handled gracefully.
- *
- * READ_ONCE is used so that if the caller assigns the result into a local
- * variable and e.g. tests it for valid range before using, the compiler cannot
- * decide to remove the variable and inline the page_private(page) multiple
- * times, potentially observing different values in the tests and the actual
- * use of the result.
- */
-#define buddy_order_unsafe(page)	READ_ONCE(page_private(page))
-
-/*
- * This function checks whether a page is free && is the buddy
- * we can coalesce a page and its buddy if
- * (a) the buddy is not in a hole (check before calling!) &&
- * (b) the buddy is in the buddy system &&
- * (c) a page and its buddy have the same order &&
- * (d) a page and its buddy are in the same zone.
- *
- * For recording whether a page is in the buddy system, we set PageBuddy.
- * Setting, clearing, and testing PageBuddy is serialized by zone->lock.
- *
- * For recording page's order, we use page_private(page).
- */
-static inline bool page_is_buddy(struct page *page, struct page *buddy,
-				 unsigned int order)
-{
-	if (!page_is_guard(buddy) && !PageBuddy(buddy))
-		return false;
-
-	if (buddy_order(buddy) != order)
-		return false;
-
-	/*
-	 * zone check is done late to avoid uselessly calculating
-	 * zone/node ids for pages that could never merge.
-	 */
-	if (page_zone_id(page) != page_zone_id(buddy))
-		return false;
-
-	VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
-
-	return true;
-}
-
-/*
- * Locate the struct page for both the matching buddy in our
- * pair (buddy1) and the combined O(n+1) page they form (page).
- *
- * 1) Any buddy B1 will have an order O twin B2 which satisfies
- * the following equation:
- *     B2 = B1 ^ (1 << O)
- * For example, if the starting buddy (buddy2) is #8 its order
- * 1 buddy is #10:
- *     B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
- *
- * 2) Any buddy B will have an order O+1 parent P which
- * satisfies the following equation:
- *     P = B & ~(1 << O)
- *
- * Assumption: *_mem_map is contiguous at least up to MAX_PAGE_ORDER
- */
-static inline unsigned long
-__find_buddy_pfn(unsigned long page_pfn, unsigned int order)
-{
-	return page_pfn ^ (1 << order);
-}
-
-/*
- * Find the buddy of @page and validate it.
- * @page: The input page
- * @pfn: The pfn of the page, it saves a call to page_to_pfn() when the
- *       function is used in the performance-critical __free_one_page().
- * @order: The order of the page
- * @buddy_pfn: The output pointer to the buddy pfn, it also saves a call to
- *             page_to_pfn().
- *
- * The found buddy can be a non PageBuddy, out of @page's zone, or its order is
- * not the same as @page. The validation is necessary before use it.
- *
- * Return: the found buddy page or NULL if not found.
- */
-static inline struct page *find_buddy_page_pfn(struct page *page,
-			unsigned long pfn, unsigned int order, unsigned long *buddy_pfn)
-{
-	unsigned long __buddy_pfn = __find_buddy_pfn(pfn, order);
-	struct page *buddy;
-
-	buddy = page + (__buddy_pfn - pfn);
-	if (buddy_pfn)
-		*buddy_pfn = __buddy_pfn;
-
-	if (page_is_buddy(page, buddy, order))
-		return buddy;
-	return NULL;
-}
-
-extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
-				unsigned long end_pfn, struct zone *zone);
-
-static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn,
-				unsigned long end_pfn, struct zone *zone)
-{
-	if (zone->contiguous)
-		return pfn_to_page(start_pfn);
-
-	return __pageblock_pfn_to_page(start_pfn, end_pfn, zone);
-}
-
-void set_zone_contiguous(struct zone *zone);
-bool pfn_range_intersects_zones(int nid, unsigned long start_pfn,
-			   unsigned long nr_pages);
-
-static inline void clear_zone_contiguous(struct zone *zone)
-{
-	zone->contiguous = false;
-}
 
 extern int __isolate_free_page(struct page *page, unsigned int order);
 extern void __putback_isolated_page(struct page *page, unsigned int order,
 				    int mt);
-extern void memblock_free_pages(unsigned long pfn, unsigned int order);
-extern void __free_pages_core(struct page *page, unsigned int order,
-		enum meminit_context context);
 
 /*
  * This will have no effect, other than possibly generating a warning, if the
@@ -895,7 +795,7 @@ static inline void prep_compound_tail(struct page *tail,
 {
 	tail->mapping = TAIL_MAPPING;
 	set_compound_head(tail, head, order);
-	set_page_private(tail, 0);
+	VM_WARN_ON_ONCE(tail->private);
 }
 
 static inline void init_compound_tail(struct page *tail,
@@ -907,101 +807,6 @@ static inline void init_compound_tail(struct page *tail,
 	prep_compound_tail(tail, head, order);
 }
 
-void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags);
-extern bool free_pages_prepare(struct page *page, unsigned int order);
-
-extern int user_min_free_kbytes;
-
-struct page *__alloc_frozen_pages_noprof(gfp_t, unsigned int order, int nid,
-		nodemask_t *);
-#define __alloc_frozen_pages(...) \
-	alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__))
-void free_frozen_pages(struct page *page, unsigned int order);
-void free_unref_folios(struct folio_batch *fbatch);
-
-#ifdef CONFIG_NUMA
-struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order);
-#else
-static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order)
-{
-	return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL);
-}
-#endif
-
-#define alloc_frozen_pages(...) \
-	alloc_hooks(alloc_frozen_pages_noprof(__VA_ARGS__))
-
-struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order);
-#define alloc_frozen_pages_nolock(...) \
-	alloc_hooks(alloc_frozen_pages_nolock_noprof(__VA_ARGS__))
-void free_frozen_pages_nolock(struct page *page, unsigned int order);
-
-extern void zone_pcp_reset(struct zone *zone);
-extern void zone_pcp_disable(struct zone *zone);
-extern void zone_pcp_enable(struct zone *zone);
-extern void zone_pcp_init(struct zone *zone);
-
-extern void *memmap_alloc(phys_addr_t size, phys_addr_t align,
-			  phys_addr_t min_addr,
-			  int nid, bool exact_nid);
-
-void memmap_init_range(unsigned long, int, unsigned long, unsigned long,
-		unsigned long, enum meminit_context, struct vmem_altmap *, int,
-		bool);
-
-/*
- * mm/sparse.c
- */
-#ifdef CONFIG_SPARSEMEM
-void sparse_init(void);
-int sparse_index_init(unsigned long section_nr, int nid);
-
-static inline void sparse_init_one_section(struct mem_section *ms,
-		unsigned long pnum, struct page *mem_map,
-		struct mem_section_usage *usage, unsigned long flags)
-{
-	unsigned long coded_mem_map;
-
-	BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT);
-
-	/*
-	 * We encode the start PFN of the section into the mem_map such that
-	 * page_to_pfn() on !CONFIG_SPARSEMEM_VMEMMAP can simply subtract it
-	 * from the page pointer to obtain the PFN.
-	 */
-	coded_mem_map = (unsigned long)(mem_map - section_nr_to_pfn(pnum));
-	VM_WARN_ON_ONCE(coded_mem_map & ~SECTION_MAP_MASK);
-
-	ms->section_mem_map &= ~SECTION_MAP_MASK;
-	ms->section_mem_map |= coded_mem_map;
-	ms->section_mem_map |= flags | SECTION_HAS_MEM_MAP;
-	ms->usage = usage;
-}
-
-static inline void __section_mark_present(struct mem_section *ms,
-		unsigned long section_nr)
-{
-	if (section_nr > __highest_present_section_nr)
-		__highest_present_section_nr = section_nr;
-
-	ms->section_mem_map |= SECTION_MARKED_PRESENT;
-}
-#else
-static inline void sparse_init(void) {}
-#endif /* CONFIG_SPARSEMEM */
-
-/*
- * mm/sparse-vmemmap.c
- */
-#ifdef CONFIG_SPARSEMEM_VMEMMAP
-void sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages);
-#else
-static inline void sparse_init_subsection_map(unsigned long pfn,
-		unsigned long nr_pages)
-{
-}
-#endif /* CONFIG_SPARSEMEM_VMEMMAP */
-
 #if defined CONFIG_COMPACTION || defined CONFIG_CMA
 
 /*
@@ -1070,9 +875,6 @@ int
 isolate_migratepages_range(struct compact_control *cc,
 			   unsigned long low_pfn, unsigned long end_pfn);
 
-/* Free whole pageblock and set its migration type to MIGRATE_CMA. */
-void init_cma_reserved_pageblock(struct page *page);
-
 #endif /* CONFIG_COMPACTION || CONFIG_CMA */
 
 struct cma;
@@ -1080,7 +882,6 @@ struct cma;
 #ifdef CONFIG_CMA
 bool cma_validate_zones(struct cma *cma);
 void *cma_reserve_early(struct cma *cma, unsigned long size);
-void init_cma_pageblock(struct page *page);
 #else
 static inline bool cma_validate_zones(struct cma *cma)
 {
@@ -1090,28 +891,8 @@ static inline void *cma_reserve_early(struct cma *cma, unsigned long size)
 {
 	return NULL;
 }
-static inline void init_cma_pageblock(struct page *page)
-{
-}
 #endif
 
-enum fallback_result {
-	/* Found suitable migratetype, *mt_out is valid. */
-	FALLBACK_FOUND,
-	/* No fallback found in requested order. */
-	FALLBACK_EMPTY,
-	/* Passed @claimable, but claiming whole block is a bad idea. */
-	FALLBACK_NOCLAIM,
-};
-enum fallback_result
-find_suitable_fallback(struct free_area *area, unsigned int order,
-		       int migratetype, bool claimable, int *mt_out);
-
-static inline bool free_area_empty(struct free_area *area, int migratetype)
-{
-	return list_empty(&area->free_list[migratetype]);
-}
-
 /* mm/util.c */
 struct anon_vma *folio_anon_vma(const struct folio *folio);
 
@@ -1143,26 +924,28 @@ static inline bool
 folio_within_range(struct folio *folio, struct vm_area_struct *vma,
 		unsigned long start, unsigned long end)
 {
-	pgoff_t pgoff, addr;
-	unsigned long vma_pglen = vma_pages(vma);
+	const unsigned long vma_pglen = vma_pages(vma);
+	pgoff_t pgoff_folio, pgoff_vma_start;
+	unsigned long addr;
 
 	VM_WARN_ON_FOLIO(folio_test_ksm(folio), folio);
 	if (start > end)
 		return false;
 
+	pgoff_folio = folio_pgoff(folio);
+	pgoff_vma_start = vma_start_pgoff(vma);
+
 	if (start < vma->vm_start)
 		start = vma->vm_start;
 
 	if (end > vma->vm_end)
 		end = vma->vm_end;
 
-	pgoff = folio_pgoff(folio);
-
 	/* if folio start address is not in vma range */
-	if (!in_range(pgoff, vma->vm_pgoff, vma_pglen))
+	if (!in_range(pgoff_folio, pgoff_vma_start, vma_pglen))
 		return false;
 
-	addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
+	addr = vma->vm_start + ((pgoff_folio - pgoff_vma_start) << PAGE_SHIFT);
 
 	return !(addr < start || end - addr < folio_size(folio));
 }
@@ -1234,15 +1017,16 @@ extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
 static inline unsigned long vma_address(const struct vm_area_struct *vma,
 		pgoff_t pgoff, unsigned long nr_pages)
 {
+	const pgoff_t pgoff_start = vma_start_pgoff(vma);
 	unsigned long address;
 
-	if (pgoff >= vma->vm_pgoff) {
+	if (pgoff >= pgoff_start) {
 		address = vma->vm_start +
-			((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
+			((pgoff - pgoff_start) << PAGE_SHIFT);
 		/* Check for address beyond vma (or wrapped through 0?) */
 		if (address < vma->vm_start || address >= vma->vm_end)
 			address = -EFAULT;
-	} else if (pgoff + nr_pages - 1 >= vma->vm_pgoff) {
+	} else if (pgoff + nr_pages - 1 >= pgoff_start) {
 		/* Test above avoids possibility of wrap to 0 on 32-bit */
 		address = vma->vm_start;
 	} else {
@@ -1266,7 +1050,8 @@ static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw)
 		return pvmw->address + PAGE_SIZE;
 
 	pgoff = pvmw->pgoff + pvmw->nr_pages;
-	address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
+	address = vma->vm_start +
+		((pgoff - vma_start_pgoff(vma)) << PAGE_SHIFT);
 	/* Check for address beyond vma (or wrapped through 0?) */
 	if (address < vma->vm_start || address > vma->vm_end)
 		address = vma->vm_end;
@@ -1311,85 +1096,21 @@ static inline void mlock_new_folio(struct folio *folio) { }
 static inline bool need_mlock_drain(int cpu) { return false; }
 static inline void mlock_drain_local(void) { }
 static inline void mlock_drain_remote(int cpu) { }
-static inline void vunmap_range_noflush(unsigned long start, unsigned long end)
-{
-}
 #endif /* !CONFIG_MMU */
 
-/* Memory initialisation debug and verification */
-#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
-DECLARE_STATIC_KEY_TRUE(deferred_pages);
-
-static inline bool deferred_pages_enabled(void)
-{
-	return static_branch_unlikely(&deferred_pages);
-}
-
-bool __init deferred_grow_zone(struct zone *zone, unsigned int order);
-#else
-static inline bool deferred_pages_enabled(void)
-{
-	return false;
-}
-#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
-
-void init_deferred_page(unsigned long pfn, int nid);
-
-enum mminit_level {
-	MMINIT_WARNING,
-	MMINIT_VERIFY,
-	MMINIT_TRACE
-};
-
-#ifdef CONFIG_DEBUG_MEMORY_INIT
-
-extern int mminit_loglevel;
-
-#define mminit_dprintk(level, prefix, fmt, arg...) \
-do { \
-	if (level < mminit_loglevel) { \
-		if (level <= MMINIT_WARNING) \
-			pr_warn("mminit::" prefix " " fmt, ##arg);	\
-		else \
-			printk(KERN_DEBUG "mminit::" prefix " " fmt, ##arg); \
-	} \
-} while (0)
-
-extern void mminit_verify_pageflags_layout(void);
-extern void mminit_verify_zonelist(void);
-#else
-
-static inline void mminit_dprintk(enum mminit_level level,
-				const char *prefix, const char *fmt, ...)
-{
-}
-
-static inline void mminit_verify_pageflags_layout(void)
-{
-}
-
-static inline void mminit_verify_zonelist(void)
-{
-}
-#endif /* CONFIG_DEBUG_MEMORY_INIT */
-
-#define NODE_RECLAIM_NOSCAN	-2
-#define NODE_RECLAIM_FULL	-1
-#define NODE_RECLAIM_SOME	0
-#define NODE_RECLAIM_SUCCESS	1
-
 #ifdef CONFIG_NUMA
 extern int node_reclaim_mode;
 
-extern int node_reclaim(struct pglist_data *, gfp_t, unsigned int);
+extern unsigned long node_reclaim(struct pglist_data *pgdat,
+				  gfp_t gfp_mask, unsigned int order);
 extern int find_next_best_node(int node, nodemask_t *used_node_mask);
 #else
 #define node_reclaim_mode 0
 
-static inline int node_reclaim(struct pglist_data *pgdat, gfp_t mask,
-				unsigned int order)
+static inline unsigned long node_reclaim(struct pglist_data *pgdat,
+					 gfp_t mask, unsigned int order)
 {
-	return NODE_RECLAIM_NOSCAN;
+	return 0;
 }
 static inline int find_next_best_node(int node, nodemask_t *used_node_mask)
 {
@@ -1436,50 +1157,9 @@ extern unsigned long  __must_check vm_mmap_pgoff(struct file *, unsigned long,
         unsigned long, unsigned long,
         unsigned long, unsigned long);
 
-extern void set_pageblock_order(void);
 unsigned long reclaim_pages(struct list_head *folio_list);
 unsigned int reclaim_clean_pages_from_list(struct zone *zone,
 					    struct list_head *folio_list);
-/* The ALLOC_WMARK bits are used as an index to zone->watermark */
-#define ALLOC_WMARK_MIN		WMARK_MIN
-#define ALLOC_WMARK_LOW		WMARK_LOW
-#define ALLOC_WMARK_HIGH	WMARK_HIGH
-#define ALLOC_NO_WATERMARKS	0x04 /* don't check watermarks at all */
-
-/* Mask to get the watermark bits */
-#define ALLOC_WMARK_MASK	(ALLOC_NO_WATERMARKS-1)
-
-/*
- * Only MMU archs have async oom victim reclaim - aka oom_reaper so we
- * cannot assume a reduced access to memory reserves is sufficient for
- * !MMU
- */
-#ifdef CONFIG_MMU
-#define ALLOC_OOM		0x08
-#else
-#define ALLOC_OOM		ALLOC_NO_WATERMARKS
-#endif
-
-#define ALLOC_NON_BLOCK		 0x10 /* Caller cannot block. Allow access
-				       * to 25% of the min watermark or
-				       * 62.5% if __GFP_HIGH is set.
-				       */
-#define ALLOC_MIN_RESERVE	 0x20 /* __GFP_HIGH set. Allow access to 50%
-				       * of the min watermark.
-				       */
-#define ALLOC_CPUSET		 0x40 /* check for correct cpuset */
-#define ALLOC_CMA		 0x80 /* allow allocations from CMA areas */
-#ifdef CONFIG_ZONE_DMA32
-#define ALLOC_NOFRAGMENT	0x100 /* avoid mixing pageblock types */
-#else
-#define ALLOC_NOFRAGMENT	  0x0
-#endif
-#define ALLOC_HIGHATOMIC	0x200 /* Allows access to MIGRATE_HIGHATOMIC */
-#define ALLOC_TRYLOCK		0x400 /* Only use spin_trylock in allocation path */
-#define ALLOC_KSWAPD		0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */
-
-/* Flags that allow allocations below the min watermark. */
-#define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM)
 
 enum ttu_flags;
 struct tlbflush_unmap_batch;
@@ -1526,37 +1206,6 @@ struct migration_target_control {
 size_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
 			      struct folio *folio, loff_t fpos, size_t size);
 
-/*
- * mm/vmalloc.c
- */
-#ifdef CONFIG_MMU
-void __init vmalloc_init(void);
-int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end,
-	pgprot_t prot, struct page **pages, unsigned int page_shift, gfp_t gfp_mask);
-unsigned int get_vm_area_page_order(struct vm_struct *vm);
-#else
-static inline void vmalloc_init(void)
-{
-}
-
-static inline
-int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end,
-	pgprot_t prot, struct page **pages, unsigned int page_shift, gfp_t gfp_mask)
-{
-	return -EINVAL;
-}
-#endif
-
-void clear_vm_uninitialized_flag(struct vm_struct *vm);
-
-int __must_check __vmap_pages_range_noflush(unsigned long addr,
-			       unsigned long end, pgprot_t prot,
-			       struct page **pages, unsigned int page_shift);
-
-void vunmap_range_noflush(unsigned long start, unsigned long end);
-
-void __vunmap_range_noflush(unsigned long start, unsigned long end);
-
 static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma)
 {
 	if (vma->vm_flags & VM_SHARED)
@@ -1584,12 +1233,6 @@ int numa_migrate_check(struct folio *folio, struct vm_fault *vmf,
 void free_zone_device_folio(struct folio *folio);
 int migrate_device_coherent_folio(struct folio *folio);
 
-struct vm_struct *__get_vm_area_node(unsigned long size,
-				     unsigned long align, unsigned long shift,
-				     unsigned long vm_flags, unsigned long start,
-				     unsigned long end, int node, gfp_t gfp_mask,
-				     const void *caller);
-
 /*
  * mm/gup.c
  */
@@ -1712,18 +1355,6 @@ static inline bool gup_must_unshare(struct vm_area_struct *vma,
 	return !PageAnonExclusive(page);
 }
 
-extern bool mirrored_kernelcore;
-bool memblock_has_mirror(void);
-void memblock_free_all(void);
-
-static __always_inline void vma_set_range(struct vm_area_struct *vma,
-					  unsigned long start, unsigned long end,
-					  pgoff_t pgoff)
-{
-	vma->vm_start = start;
-	vma->vm_end = end;
-	vma->vm_pgoff = pgoff;
-}
 
 static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma)
 {
@@ -1753,10 +1384,6 @@ static inline bool pte_needs_soft_dirty_wp(struct vm_area_struct *vma, pte_t pte
 	return vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte);
 }
 
-void __meminit __init_single_page(struct page *page, unsigned long pfn,
-				unsigned long zone, int nid);
-void __meminit __init_page_from_nid(unsigned long pfn, int nid);
-
 /* shrinker related functions */
 unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg,
 			  int priority);
@@ -1951,4 +1578,37 @@ static inline int get_sysctl_max_map_count(void)
 bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags,
 		   unsigned long npages);
 
+static inline void mm_prepare_for_swap_entries(struct mm_struct *mm)
+{
+	if (list_empty(&mm->mmlist)) {
+		spin_lock(&mmlist_lock);
+		if (list_empty(&mm->mmlist))
+			list_add(&mm->mmlist, &init_mm.mmlist);
+		spin_unlock(&mmlist_lock);
+	}
+}
+
+static inline bool can_spin_trylock(void)
+{
+	/*
+	 * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is
+	 * unsafe in NMI. If spin_trylock() is called from hard IRQ the current
+	 * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will
+	 * mark the task as the owner of another rt_spin_lock which will
+	 * confuse PI logic, so return immediately if called from hard IRQ or
+	 * NMI.
+	 *
+	 * Note, irqs_disabled() case is ok. spin_trylock() can be called
+	 * from raw_spin_lock_irqsave region.
+	 */
+	if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq()))
+		return false;
+
+	/* On UP, spin_trylock() always succeeds even when it is locked */
+	if (!IS_ENABLED(CONFIG_SMP) && in_nmi())
+		return false;
+
+	return true;
+}
+
 #endif	/* __MM_INTERNAL_H */
diff --git a/mm/interval_tree.c b/mm/interval_tree.c
index 32bcfbfcf15f1..3ae9e106d3afd 100644
--- a/mm/interval_tree.c
+++ b/mm/interval_tree.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * mm/interval_tree.c - interval tree for mapping->i_mmap
+ * mm/interval_tree.c - interval tree for address_space->i_mmap and
+ * anon_vma->rb_root
  *
  * Copyright (C) 2012, Michel Lespinasse <walken@google.com>
  */
@@ -10,25 +11,29 @@
 #include <linux/rmap.h>
 #include <linux/interval_tree_generic.h>
 
-static inline unsigned long vma_start_pgoff(struct vm_area_struct *v)
-{
-	return v->vm_pgoff;
-}
+/* File-backed interval tree (address_space->i_mmap) */
 
 INTERVAL_TREE_DEFINE(struct vm_area_struct, shared.rb,
-		     unsigned long, shared.rb_subtree_last,
-		     vma_start_pgoff, vma_last_pgoff, /* empty */, vma_interval_tree)
+		     pgoff_t, shared.rb_subtree_last,
+		     vma_start_pgoff, vma_last_pgoff, static,
+		     __mapping_rmap_tree)
+
+void mapping_rmap_tree_insert(struct vm_area_struct *vma,
+			      struct address_space *mapping)
+{
+	__mapping_rmap_tree_insert(vma, &mapping->i_mmap);
+}
 
-/* Insert node immediately after prev in the interval tree */
-void vma_interval_tree_insert_after(struct vm_area_struct *node,
+/* Insert vma immediately after prev in the interval tree */
+void mapping_rmap_tree_insert_after(struct vm_area_struct *vma,
 				    struct vm_area_struct *prev,
-				    struct rb_root_cached *root)
+				    struct address_space *mapping)
 {
 	struct rb_node **link;
 	struct vm_area_struct *parent;
-	unsigned long last = vma_last_pgoff(node);
+	const pgoff_t pgoff_last = vma_last_pgoff(vma);
 
-	VM_BUG_ON_VMA(vma_start_pgoff(node) != vma_start_pgoff(prev), node);
+	VM_WARN_ON_ONCE_VMA(vma_start_pgoff(vma) != vma_start_pgoff(prev), vma);
 
 	if (!prev->shared.rb.rb_right) {
 		parent = prev;
@@ -36,71 +41,95 @@ void vma_interval_tree_insert_after(struct vm_area_struct *node,
 	} else {
 		parent = rb_entry(prev->shared.rb.rb_right,
 				  struct vm_area_struct, shared.rb);
-		if (parent->shared.rb_subtree_last < last)
-			parent->shared.rb_subtree_last = last;
+		if (parent->shared.rb_subtree_last < pgoff_last)
+			parent->shared.rb_subtree_last = pgoff_last;
 		while (parent->shared.rb.rb_left) {
 			parent = rb_entry(parent->shared.rb.rb_left,
 				struct vm_area_struct, shared.rb);
-			if (parent->shared.rb_subtree_last < last)
-				parent->shared.rb_subtree_last = last;
+			if (parent->shared.rb_subtree_last < pgoff_last)
+				parent->shared.rb_subtree_last = pgoff_last;
 		}
 		link = &parent->shared.rb.rb_left;
 	}
 
-	node->shared.rb_subtree_last = last;
-	rb_link_node(&node->shared.rb, &parent->shared.rb, link);
-	rb_insert_augmented(&node->shared.rb, &root->rb_root,
-			    &vma_interval_tree_augment);
+	vma->shared.rb_subtree_last = pgoff_last;
+	rb_link_node(&vma->shared.rb, &parent->shared.rb, link);
+	rb_insert_augmented(&vma->shared.rb, &mapping->i_mmap.rb_root,
+			    &__mapping_rmap_tree_augment);
+}
+
+void mapping_rmap_tree_remove(struct vm_area_struct *vma,
+			      struct address_space *mapping)
+{
+	__mapping_rmap_tree_remove(vma, &mapping->i_mmap);
 }
 
-static inline unsigned long avc_start_pgoff(struct anon_vma_chain *avc)
+struct vm_area_struct *
+mapping_rmap_tree_iter_first(struct address_space *mapping,
+			     pgoff_t pgoff_start, pgoff_t pgoff_last)
+{
+	return __mapping_rmap_tree_iter_first(&mapping->i_mmap,
+					      pgoff_start, pgoff_last);
+}
+
+struct vm_area_struct *
+mapping_rmap_tree_iter_next(struct vm_area_struct *vma,
+			    pgoff_t pgoff_start, pgoff_t pgoff_last)
+{
+	return __mapping_rmap_tree_iter_next(vma, pgoff_start, pgoff_last);
+}
+
+/* Anonymous interval tree (anon_vma->rb_root) */
+
+static pgoff_t avc_start_pgoff(struct anon_vma_chain *avc)
 {
 	return vma_start_pgoff(avc->vma);
 }
 
-static inline unsigned long avc_last_pgoff(struct anon_vma_chain *avc)
+static pgoff_t avc_last_pgoff(struct anon_vma_chain *avc)
 {
 	return vma_last_pgoff(avc->vma);
 }
 
-INTERVAL_TREE_DEFINE(struct anon_vma_chain, rb, unsigned long, rb_subtree_last,
+INTERVAL_TREE_DEFINE(struct anon_vma_chain, rb, pgoff_t, rb_subtree_last,
 		     avc_start_pgoff, avc_last_pgoff,
-		     static inline, __anon_vma_interval_tree)
+		     static, __anon_rmap_tree)
 
-void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
-				   struct rb_root_cached *root)
+void anon_rmap_tree_insert(struct anon_vma_chain *avc,
+			   struct anon_vma *anon_vma)
 {
 #ifdef CONFIG_DEBUG_VM_RB
-	node->cached_vma_start = avc_start_pgoff(node);
-	node->cached_vma_last = avc_last_pgoff(node);
+	avc->cached_vma_start = avc_start_pgoff(avc);
+	avc->cached_vma_last = avc_last_pgoff(avc);
 #endif
-	__anon_vma_interval_tree_insert(node, root);
+	__anon_rmap_tree_insert(avc, &anon_vma->rb_root);
 }
 
-void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
-				   struct rb_root_cached *root)
+void anon_rmap_tree_remove(struct anon_vma_chain *avc,
+			   struct anon_vma *anon_vma)
 {
-	__anon_vma_interval_tree_remove(node, root);
+	__anon_rmap_tree_remove(avc, &anon_vma->rb_root);
 }
 
 struct anon_vma_chain *
-anon_vma_interval_tree_iter_first(struct rb_root_cached *root,
-				  unsigned long first, unsigned long last)
+anon_rmap_tree_iter_first(struct anon_vma *anon_vma,
+			  pgoff_t pgoff_start, pgoff_t pgoff_last)
 {
-	return __anon_vma_interval_tree_iter_first(root, first, last);
+	return __anon_rmap_tree_iter_first(&anon_vma->rb_root,
+					   pgoff_start, pgoff_last);
 }
 
 struct anon_vma_chain *
-anon_vma_interval_tree_iter_next(struct anon_vma_chain *node,
-				 unsigned long first, unsigned long last)
+anon_rmap_tree_iter_next(struct anon_vma_chain *avc,
+			 pgoff_t pgoff_start, pgoff_t pgoff_last)
 {
-	return __anon_vma_interval_tree_iter_next(node, first, last);
+	return __anon_rmap_tree_iter_next(avc, pgoff_start, pgoff_last);
 }
 
 #ifdef CONFIG_DEBUG_VM_RB
-void anon_vma_interval_tree_verify(struct anon_vma_chain *node)
+void anon_rmap_tree_verify(struct anon_vma_chain *avc)
 {
-	WARN_ON_ONCE(node->cached_vma_start != avc_start_pgoff(node));
-	WARN_ON_ONCE(node->cached_vma_last != avc_last_pgoff(node));
+	WARN_ON_ONCE(avc->cached_vma_start != avc_start_pgoff(avc));
+	WARN_ON_ONCE(avc->cached_vma_last != avc_last_pgoff(avc));
 }
 #endif
diff --git a/mm/kasan/hw_tags.c b/mm/kasan/hw_tags.c
index cbef5e450954e..a848eb2f9910c 100644
--- a/mm/kasan/hw_tags.c
+++ b/mm/kasan/hw_tags.c
@@ -61,7 +61,7 @@ DEFINE_STATIC_KEY_FALSE(kasan_flag_vmalloc);
 EXPORT_SYMBOL_GPL(kasan_flag_vmalloc);
 
 /* Whether to check write accesses only. */
-static bool kasan_flag_write_only = false;
+static bool kasan_flag_write_only;
 
 #define PAGE_ALLOC_SAMPLE_DEFAULT	1
 #define PAGE_ALLOC_SAMPLE_ORDER_DEFAULT	3
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 617bca76db49b..27e8f3077e80f 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -26,6 +26,7 @@
 
 #include <asm/tlb.h>
 #include "internal.h"
+#include "page_alloc.h"
 #include "mm_slot.h"
 
 enum scan_result {
@@ -37,7 +38,7 @@ enum scan_result {
 	SCAN_EXCEED_SWAP_PTE,
 	SCAN_EXCEED_SHARED_PTE,
 	SCAN_PTE_NON_PRESENT,
-	SCAN_PTE_UFFD_WP,
+	SCAN_PTE_UFFD,
 	SCAN_PTE_MAPPED_HUGEPAGE,
 	SCAN_LACK_REFERENCED_PAGE,
 	SCAN_PAGE_NULL,
@@ -605,8 +606,7 @@ void __khugepaged_exit(struct mm_struct *mm)
 	spin_lock(&khugepaged_mm_lock);
 	slot = mm_slot_lookup(mm_slots_hash, mm);
 	if (slot && khugepaged_scan.mm_slot != slot) {
-		hash_del(&slot->hash);
-		list_del(&slot->mm_node);
+		mm_slot_remove(slot);
 		free = 1;
 	}
 	spin_unlock(&khugepaged_mm_lock);
@@ -695,8 +695,8 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma,
 			result = SCAN_PTE_NON_PRESENT;
 			goto out;
 		}
-		if (pte_uffd_wp(pteval)) {
-			result = SCAN_PTE_UFFD_WP;
+		if (pte_uffd(pteval)) {
+			result = SCAN_PTE_UFFD;
 			goto out;
 		}
 		page = vm_normal_page(vma, addr, pteval);
@@ -1543,7 +1543,7 @@ static enum scan_result mthp_collapse(struct mm_struct *mm,
 			case SCAN_PAGE_NULL:
 			case SCAN_DEL_PAGE_LRU:
 			case SCAN_PTE_NON_PRESENT:
-			case SCAN_PTE_UFFD_WP:
+			case SCAN_PTE_UFFD:
 			case SCAN_PAGE_LAZYFREE:
 				last_result = ret;
 				goto next_order;
@@ -1664,15 +1664,15 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
 			/*
 			 * Always be strict with uffd-wp
 			 * enabled swap entries.  Please see
-			 * comment below for pte_uffd_wp().
+			 * comment below for pte_uffd().
 			 */
-			if (pte_swp_uffd_wp_any(pteval)) {
-				result = SCAN_PTE_UFFD_WP;
+			if (pte_swp_uffd_any(pteval)) {
+				result = SCAN_PTE_UFFD;
 				goto out_unmap;
 			}
 			continue;
 		}
-		if (pte_uffd_wp(pteval)) {
+		if (pte_uffd(pteval)) {
 			/*
 			 * Don't collapse the page if any of the small
 			 * PTEs are armed with uffd write protection.
@@ -1682,7 +1682,7 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
 			 * userfault messages that falls outside of
 			 * the registered range.  So, just be simple.
 			 */
-			result = SCAN_PTE_UFFD_WP;
+			result = SCAN_PTE_UFFD;
 			goto out_unmap;
 		}
 
@@ -1801,8 +1801,7 @@ static void collect_mm_slot(struct mm_slot *slot)
 
 	if (collapse_test_exit(mm)) {
 		/* free mm_slot */
-		hash_del(&slot->hash);
-		list_del(&slot->mm_node);
+		mm_slot_remove(slot);
 
 		/*
 		 * Not strictly needed because the mm exited already.
@@ -1892,9 +1891,12 @@ static enum scan_result try_collapse_pte_mapped_thp(struct mm_struct *mm, unsign
 	if (!thp_vma_allowable_order(vma, vma->vm_flags, TVA_FORCED_COLLAPSE, PMD_ORDER))
 		return SCAN_VMA_CHECK;
 
-	/* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */
-	if (userfaultfd_wp(vma))
-		return SCAN_PTE_UFFD_WP;
+	/*
+	 * Keep pmd pgtable while the uffd bit is in use; see comment in
+	 * retract_page_tables().
+	 */
+	if (userfaultfd_protected(vma))
+		return SCAN_PTE_UFFD;
 
 	folio = filemap_lock_folio(vma->vm_file->f_mapping,
 			       linear_page_index(vma, haddr));
@@ -2106,13 +2108,14 @@ static bool file_backed_vma_is_retractable(struct vm_area_struct *vma)
 		return false;
 
 	/*
-	 * When a vma is registered with uffd-wp, we cannot recycle
+	 * When a vma is registered with uffd-wp or RWP, we cannot recycle
 	 * the page table because there may be pte markers installed.
-	 * Other vmas can still have the same file mapped hugely, but
-	 * skip this one: it will always be mapped in small page size
-	 * for uffd-wp registered ranges.
+	 * VM_UFFD_RWP ranges similarly rely on per-PTE uffd state
+	 * and cannot be recycled to a shared PMD. Other vmas can still
+	 * have the same file mapped hugely, but skip this one: it will
+	 * always be mapped in small page size for these registrations.
 	 */
-	if (userfaultfd_wp(vma))
+	if (userfaultfd_protected(vma))
 		return false;
 
 	/*
@@ -2136,7 +2139,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
 	struct vm_area_struct *vma;
 
 	i_mmap_lock_read(mapping);
-	vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
+	mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) {
 		struct mmu_notifier_range range;
 		struct mm_struct *mm;
 		unsigned long addr;
@@ -2145,7 +2148,8 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
 		spinlock_t *ptl;
 		bool success = false;
 
-		addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
+		addr = vma->vm_start +
+			((pgoff - vma_start_pgoff(vma)) << PAGE_SHIFT);
 		if (addr & ~HPAGE_PMD_MASK ||
 		    vma->vm_end < addr + HPAGE_PMD_SIZE)
 			continue;
@@ -2568,7 +2572,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
 		 * not be able to observe any missing pages due to the
 		 * previously inserted retry entries.
 		 */
-		vma_interval_tree_foreach(vma, &mapping->i_mmap, start, end) {
+		mapping_rmap_tree_foreach(vma, mapping, start, end) {
 			if (userfaultfd_missing(vma)) {
 				result = SCAN_EXCEED_NONE_PTE;
 				goto immap_locked;
@@ -3241,7 +3245,7 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start,
 		/* Whitelisted set of results where continuing OK */
 		case SCAN_NO_PTE_TABLE:
 		case SCAN_PTE_NON_PRESENT:
-		case SCAN_PTE_UFFD_WP:
+		case SCAN_PTE_UFFD:
 		case SCAN_LACK_REFERENCED_PAGE:
 		case SCAN_PAGE_NULL:
 		case SCAN_PAGE_COUNT:
diff --git a/mm/kmemleak.c b/mm/kmemleak.c
index 7c7ba17ce7af0..85f18b17e79c4 100644
--- a/mm/kmemleak.c
+++ b/mm/kmemleak.c
@@ -151,6 +151,8 @@ struct kmemleak_object {
 	int min_count;
 	/* the total number of pointers found pointing to this object */
 	int count;
+	/* consecutive scans the object has been seen unreferenced */
+	unsigned int unref_scans;
 	/* checksum for detecting modified objects */
 	u32 checksum;
 	depot_stack_handle_t trace_handle;
@@ -175,6 +177,8 @@ struct kmemleak_object {
 #define OBJECT_PHYS		(1 << 4)
 /* flag set for per-CPU pointers */
 #define OBJECT_PERCPU		(1 << 5)
+/* flag set on an object left unreferenced by the full scan, pending confirmation */
+#define OBJECT_SUSPECT		(1 << 6)
 
 /* set when __remove_object() called */
 #define DELSTATE_REMOVED	(1 << 0)
@@ -232,9 +236,14 @@ static unsigned long max_percpu_addr;
 static struct task_struct *scan_thread;
 /* used to avoid reporting of recently allocated objects */
 static unsigned long jiffies_min_age;
+/* consecutive scans an object must stay unreferenced before reporting */
+static unsigned int min_unref_scans = 1;
+module_param(min_unref_scans, uint, 0644);
 static unsigned long jiffies_last_scan;
 /* delay between automatic memory scannings */
 static unsigned long jiffies_scan_wait;
+/* number of objects flagged OBJECT_SUSPECT during the current scan */
+static int nr_suspects;
 /* enables or disables the task stacks scanning */
 static int kmemleak_stack_scan = 1;
 /* protects the memory scanning, parameters and debug/kmemleak file access */
@@ -687,7 +696,8 @@ static struct kmemleak_object *__alloc_object(gfp_t gfp)
 	atomic_set(&object->use_count, 1);
 	object->excess_ref = 0;
 	object->count = 0;			/* white color initially */
-	object->checksum = 0;
+	object->checksum = ~0;
+	object->unref_scans = 0;
 	object->del_state = 0;
 
 	/* task information */
@@ -981,7 +991,7 @@ static void reset_checksum(unsigned long ptr)
 	}
 
 	raw_spin_lock_irqsave(&object->lock, flags);
-	object->checksum = 0;
+	object->checksum = ~0;
 	raw_spin_unlock_irqrestore(&object->lock, flags);
 	put_object(object);
 }
@@ -1410,7 +1420,8 @@ static bool update_checksum(struct kmemleak_object *object)
 		for_each_possible_cpu(cpu) {
 			void *ptr = per_cpu_ptr((void __percpu *)object->pointer, cpu);
 
-			object->checksum ^= crc32(0, kasan_reset_tag((void *)ptr), object->size);
+			object->checksum = crc32(object->checksum,
+						 kasan_reset_tag((void *)ptr), object->size);
 		}
 	} else {
 		object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size);
@@ -1439,6 +1450,11 @@ static void update_refs(struct kmemleak_object *object)
 	 */
 	object->count++;
 	if (color_gray(object)) {
+		/* referenced after all, no longer a suspect */
+		if (object->flags & OBJECT_SUSPECT) {
+			object->flags &= ~OBJECT_SUSPECT;
+			nr_suspects--;
+		}
 		/* put_object() called when removing from gray_list */
 		WARN_ON(!get_object(object));
 		list_add_tail(&object->gray_list, &gray_list);
@@ -1524,22 +1540,25 @@ static int scan_should_stop(void)
 
 /*
  * Scan a memory block (exclusive range) for valid pointers and add those
- * found to the gray list.
+ * found to the gray list. Return non-zero if the scan was interrupted.
  */
-static void scan_block(void *_start, void *_end,
-		       struct kmemleak_object *scanned)
+static int scan_block(void *_start, void *_end,
+		      struct kmemleak_object *scanned)
 {
 	unsigned long *ptr;
 	unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
 	unsigned long *end = _end - (BYTES_PER_POINTER - 1);
 	unsigned long flags;
+	int stop = 0;
 
 	raw_spin_lock_irqsave(&kmemleak_lock, flags);
 	for (ptr = start; ptr < end; ptr++) {
 		unsigned long pointer;
 
-		if (scan_should_stop())
+		if (scan_should_stop()) {
+			stop = 1;
 			break;
+		}
 
 		kasan_disable_current();
 		pointer = *(unsigned long *)kasan_reset_tag((void *)ptr);
@@ -1549,22 +1568,28 @@ static void scan_block(void *_start, void *_end,
 		pointer_update_refs(scanned, pointer, OBJECT_PERCPU);
 	}
 	raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
+
+	return stop;
 }
 
 /*
  * Scan a large memory block in MAX_SCAN_SIZE chunks to reduce the latency.
+ * Return non-zero if the scan was interrupted.
  */
 #ifdef CONFIG_SMP
-static void scan_large_block(void *start, void *end)
+static int scan_large_block(void *start, void *end)
 {
 	void *next;
 
 	while (start < end) {
 		next = min(start + MAX_SCAN_SIZE, end);
-		scan_block(start, next, NULL);
+		if (scan_block(start, next, NULL))
+			return 1;
 		start = next;
 		cond_resched();
 	}
+
+	return 0;
 }
 #endif
 
@@ -1695,6 +1720,43 @@ static void kmemleak_cond_resched(struct kmemleak_object *object)
 	put_object(object);
 }
 
+/*
+ * Scan all task kernel stacks, rescheduling between tasks. Each task is looked
+ * up and pinned within its own RCU read-side section, so no lock is held across
+ * the scan and the walk cannot trip the soft lockup watchdog.
+ */
+static void kmemleak_scan_task_stacks(void)
+{
+	struct pid *pid;
+	int nr = 1;
+	int stop = 0;
+
+	do {
+		struct task_struct *p = NULL;
+
+		rcu_read_lock();
+		pid = find_ge_pid(nr, &init_pid_ns);
+		if (pid) {
+			nr = pid_nr(pid) + 1;
+			p = pid_task(pid, PIDTYPE_PID);
+			if (p)
+				get_task_struct(p);
+		}
+		rcu_read_unlock();
+
+		if (p) {
+			void *stack = try_get_task_stack(p);
+
+			if (stack) {
+				stop = scan_block(stack, stack + THREAD_SIZE, NULL);
+				put_task_stack(p);
+			}
+			put_task_struct(p);
+		}
+		cond_resched();
+	} while (pid && !stop);
+}
+
 /*
  * Print one leak inline. The hex dump is gated on OBJECT_ALLOCATED so it
  * does not touch user memory that was freed concurrently; the rest of the
@@ -1797,15 +1859,16 @@ static void dedup_flush(struct xarray *dedup)
  * kernel's standard allocators. This function must be called with the
  * scan_mutex held.
  */
-static void kmemleak_scan(void)
+static int __kmemleak_scan(bool full)
 {
 	struct kmemleak_object *object;
 	struct zone *zone;
 	int __maybe_unused i;
-	struct xarray dedup;
-	int new_leaks = 0;
+	int stop = 0;
 
 	jiffies_last_scan = jiffies;
+	if (full)
+		nr_suspects = 0;
 
 	/* prepare the kmemleak_object's */
 	rcu_read_lock();
@@ -1833,8 +1896,13 @@ static void kmemleak_scan(void)
 				__paint_it(object, KMEMLEAK_BLACK);
 		}
 
+		/* referenced last scan: restart the unreferenced run */
+		if (!color_white(object))
+			object->unref_scans = 0;
 		/* reset the reference count (whiten the object) */
 		object->count = 0;
+		if (full)
+			object->flags &= ~OBJECT_SUSPECT;
 		if (color_gray(object) && get_object(object))
 			list_add_tail(&object->gray_list, &gray_list);
 
@@ -1847,9 +1915,11 @@ static void kmemleak_scan(void)
 
 #ifdef CONFIG_SMP
 	/* per-cpu sections scanning */
-	for_each_possible_cpu(i)
-		scan_large_block(__per_cpu_start + per_cpu_offset(i),
-				 __per_cpu_end + per_cpu_offset(i));
+	for_each_possible_cpu(i) {
+		if (scan_large_block(__per_cpu_start + per_cpu_offset(i),
+				     __per_cpu_end + per_cpu_offset(i)))
+			goto scan_gray;
+	}
 #endif
 
 	/*
@@ -1876,34 +1946,34 @@ static void kmemleak_scan(void)
 			/* only scan if page is in use */
 			if (page_count(page) == 0)
 				continue;
-			scan_block(page, page + 1, NULL);
+			stop = scan_block(page, page + 1, NULL);
+			if (stop)
+				break;
 		}
+		if (stop)
+			break;
 	}
 	put_online_mems();
+	if (stop)
+		goto scan_gray;
 
 	/*
 	 * Scanning the task stacks (may introduce false negatives).
 	 */
-	if (kmemleak_stack_scan) {
-		struct task_struct *p, *g;
-
-		rcu_read_lock();
-		for_each_process_thread(g, p) {
-			void *stack = try_get_task_stack(p);
-			if (stack) {
-				scan_block(stack, stack + THREAD_SIZE, NULL);
-				put_task_stack(p);
-			}
-		}
-		rcu_read_unlock();
-	}
+	if (kmemleak_stack_scan)
+		kmemleak_scan_task_stacks();
 
 	/*
 	 * Scan the objects already referenced from the sections scanned
 	 * above.
 	 */
+scan_gray:
 	scan_gray_list();
 
+	/* a confirmation scan does not look for modified objects */
+	if (!full)
+		return nr_suspects;
+
 	/*
 	 * Check for new or unreferenced objects modified since the previous
 	 * scan and color them gray until the next scan.
@@ -1926,6 +1996,11 @@ static void kmemleak_scan(void)
 			/* color it gray temporarily */
 			object->count = object->min_count;
 			list_add_tail(&object->gray_list, &gray_list);
+		} else if (unreferenced_object(object) &&
+			   !(object->flags & OBJECT_REPORTED)) {
+			/* flag the objects left unreferenced by this scan */
+			object->flags |= OBJECT_SUSPECT;
+			nr_suspects++;
 		}
 		raw_spin_unlock_irq(&object->lock);
 	}
@@ -1936,12 +2011,62 @@ static void kmemleak_scan(void)
 	 */
 	scan_gray_list();
 
+	return nr_suspects;
+}
+
+/*
+ * Promote a suspected object to a reported leak once it has stayed
+ * unreferenced for min_unref_scans consecutive scans. Called with
+ * object->lock held; returns true when the object is newly reported.
+ */
+static bool confirm_leak(struct kmemleak_object *object)
+{
+	if (!unreferenced_object(object) ||
+	    !(object->flags & OBJECT_SUSPECT) ||
+	    (object->flags & OBJECT_REPORTED))
+		return false;
+
+	object->unref_scans += 1;
+	if (object->unref_scans < min_unref_scans)
+		return false;
+
+	object->flags |= OBJECT_REPORTED;
+	return true;
+}
+
+/*
+ * Scan the memory and report the unreferenced objects as leaks. Must be
+ * called with the scan_mutex held.
+ */
+static void kmemleak_scan(void)
+{
+	struct kmemleak_object *object;
+	struct xarray dedup;
+	int new_leaks = 0;
+
+	/*
+	 * Full scan. Objects left unreferenced are flagged OBJECT_SUSPECT and
+	 * counted in the return value; nothing to confirm or report otherwise.
+	 */
+	if (!__kmemleak_scan(true))
+		return;
+
 	/*
 	 * If scanning was stopped do not report any new unreferenced objects.
 	 */
 	if (scan_should_stop())
 		return;
 
+	/*
+	 * A live object whose only reference is moved by, for example, a
+	 * concurrent RCU update can be missed for one scan and reported as a
+	 * transient false positive. Scan again and only report the objects
+	 * left unreferenced (still flagged OBJECT_SUSPECT) by both scans.
+	 */
+	__kmemleak_scan(false);
+	if (scan_should_stop())
+		return;
+
 	/*
 	 * Scanning result reporting. When verbose printing is enabled, dedupe
 	 * by stackdepot trace_handle so each unique backtrace is logged once
@@ -1968,9 +2093,8 @@ static void kmemleak_scan(void)
 		raw_spin_lock_irq(&object->lock);
 		trace_handle = 0;
 		dedup_print = false;
-		if (unreferenced_object(object) &&
-		    !(object->flags & OBJECT_REPORTED)) {
-			object->flags |= OBJECT_REPORTED;
+
+		if (confirm_leak(object)) {
 			if (kmemleak_verbose) {
 				trace_handle = object->trace_handle;
 				dedup_print = true;
diff --git a/mm/kmsan/hooks.c b/mm/kmsan/hooks.c
index 8f22d1f229813..5f1b8053f9fa8 100644
--- a/mm/kmsan/hooks.c
+++ b/mm/kmsan/hooks.c
@@ -21,6 +21,7 @@
 #include <linux/usb.h>
 
 #include "../internal.h"
+#include "../vmalloc.h"
 #include "../slab.h"
 #include "kmsan.h"
 
diff --git a/mm/kmsan/init.c b/mm/kmsan/init.c
index b14ce3417e65e..4983b6e9f7c99 100644
--- a/mm/kmsan/init.c
+++ b/mm/kmsan/init.c
@@ -13,7 +13,7 @@
 #include <linux/mm.h>
 #include <linux/memblock.h>
 
-#include "../internal.h"
+#include "../page_alloc.h"
 
 #define NUM_FUTURE_RANGES 128
 struct start_end_pair {
diff --git a/mm/kmsan/shadow.c b/mm/kmsan/shadow.c
index 8fde939784a7f..0c88d89bf0d68 100644
--- a/mm/kmsan/shadow.c
+++ b/mm/kmsan/shadow.c
@@ -17,6 +17,7 @@
 #include <linux/stddef.h>
 
 #include "../internal.h"
+#include "../vmalloc.h"
 #include "kmsan.h"
 
 #define shadow_page_for(page) ((page)->kmsan_shadow)
diff --git a/mm/ksm.c b/mm/ksm.c
index 7d5b76478f0b6..47006f494fcb1 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -195,22 +195,28 @@ struct ksm_stable_node {
  * @node: rb node of this rmap_item in the unstable tree
  * @head: pointer to stable_node heading this list in the stable tree
  * @hlist: link into hlist of rmap_items hanging off that stable_node
- * @age: number of scan iterations since creation
- * @remaining_skips: how many scans to skip
+ * @age: number of scan iterations since creation (unstable node)
+ * @remaining_skips: how many scans to skip (unstable node)
+ * @linear_page_index: the original page's index before merged by KSM (stable node)
  */
 struct ksm_rmap_item {
 	struct ksm_rmap_item *rmap_list;
 	union {
-		struct anon_vma *anon_vma;	/* when stable */
+		struct anon_vma *anon_vma;	/* for reverse mapping, when stable */
 #ifdef CONFIG_NUMA
 		int nid;		/* when node of unstable tree */
 #endif
 	};
 	struct mm_struct *mm;
 	unsigned long address;		/* + low bits used for flags below */
-	unsigned int oldchecksum;	/* when unstable */
-	rmap_age_t age;
-	rmap_age_t remaining_skips;
+	union {
+		struct {
+			unsigned int oldchecksum;
+			rmap_age_t age;
+			rmap_age_t remaining_skips;
+		};			/* when unstable */
+		unsigned long linear_page_index;    /* for reverse mapping, when stable */
+	};
 	union {
 		struct rb_node node;	/* when node of unstable tree */
 		struct {		/* when listed from stable tree */
@@ -776,6 +782,11 @@ static struct vm_area_struct *find_mergeable_vma(struct mm_struct *mm,
 	return vma;
 }
 
+/*
+ * break_cow: actively break COW, replacing the KSM page by a fresh anonymous
+ * page. This is called when rmap_item has not yet become stable, but page
+ * has been merged.
+ */
 static void break_cow(struct ksm_rmap_item *rmap_item)
 {
 	struct mm_struct *mm = rmap_item->mm;
@@ -787,6 +798,11 @@ static void break_cow(struct ksm_rmap_item *rmap_item)
 	 * to undo, we also need to drop a reference to the anon_vma.
 	 */
 	put_anon_vma(rmap_item->anon_vma);
+	/*
+	 * Reset linear_page_index that might overlay age-related
+	 * information. (it's still unstable node)
+	 */
+	rmap_item->linear_page_index = 0;
 
 	mmap_read_lock(mm);
 	vma = find_mergeable_vma(mm, addr);
@@ -899,6 +915,8 @@ static void remove_node_from_stable_tree(struct ksm_stable_node *stable_node)
 		VM_BUG_ON(stable_node->rmap_hlist_len <= 0);
 		stable_node->rmap_hlist_len--;
 		put_anon_vma(rmap_item->anon_vma);
+		/* Reset linear_page_index that might overlay age-related information. */
+		rmap_item->linear_page_index = 0;
 		rmap_item->address &= PAGE_MASK;
 		cond_resched();
 	}
@@ -1052,6 +1070,8 @@ static void remove_rmap_item_from_tree(struct ksm_rmap_item *rmap_item)
 		stable_node->rmap_hlist_len--;
 
 		put_anon_vma(rmap_item->anon_vma);
+		/* Reset linear_page_index that might overlay age-related information. */
+		rmap_item->linear_page_index = 0;
 		rmap_item->head = NULL;
 		rmap_item->address &= PAGE_MASK;
 
@@ -1237,8 +1257,7 @@ static int unmerge_and_remove_all_rmap_items(void)
 				  struct mm_slot, mm_node);
 		ksm_scan.mm_slot = mm_slot_entry(slot, struct ksm_mm_slot, slot);
 		if (ksm_test_exit(mm)) {
-			hash_del(&mm_slot->slot.hash);
-			list_del(&mm_slot->slot.mm_node);
+			mm_slot_remove(&mm_slot->slot);
 			spin_unlock(&ksm_mmlist_lock);
 
 			mm_slot_free(mm_slot_cache, mm_slot);
@@ -1598,8 +1617,15 @@ static int try_to_merge_with_ksm_page(struct ksm_rmap_item *rmap_item,
 	/* Unstable nid is in union with stable anon_vma: remove first */
 	remove_rmap_item_from_tree(rmap_item);
 
-	/* Must get reference to anon_vma while still holding mmap_lock */
+	/*
+	 * We can consider the VMA only while still holding the mmap lock,
+	 * so lock, so reference the anon_vma and calculate the linear
+	 * page index early, before stable_tree_append(). If anything goes
+	 * wrong that prevents the rmap_item from being added to the
+	 * stable_tree, break_cow() will clean it up.
+	 */
 	rmap_item->anon_vma = vma->anon_vma;
+	rmap_item->linear_page_index = linear_page_index(vma, rmap_item->address);
 	get_anon_vma(vma->anon_vma);
 out:
 	mmap_read_unlock(mm);
@@ -2327,23 +2353,24 @@ static void cmp_and_merge_page(struct page *page, struct ksm_rmap_item *rmap_ite
 	tree_rmap_item =
 		unstable_tree_search_insert(rmap_item, page, &tree_page);
 	if (tree_rmap_item) {
+		struct folio *tree_folio;
 		bool split;
 
 		kfolio = try_to_merge_two_pages(rmap_item, page,
 						tree_rmap_item, tree_page);
+		tree_folio = page_folio(tree_page);
 		/*
-		 * If both pages we tried to merge belong to the same compound
-		 * page, then we actually ended up increasing the reference
-		 * count of the same compound page twice, and split_huge_page
-		 * failed.
+		 * If both pages we tried to merge belong to the same (large)
+		 * folio, then we actually ended up increasing the reference
+		 * count of the same folio twice, and split_huge_page failed.
+		 *
 		 * Here we set a flag if that happened, and we use it later to
-		 * try split_huge_page again. Since we call put_page right
+		 * try split_huge_page again. Since we call folio_put() right
 		 * afterwards, the reference count will be correct and
 		 * split_huge_page should succeed.
 		 */
-		split = PageTransCompound(page)
-			&& compound_head(page) == compound_head(tree_page);
-		put_page(tree_page);
+		split = folio == tree_folio;
+		folio_put(tree_folio);
 		if (kfolio) {
 			/*
 			 * The pages were successfully merged: insert new
@@ -2458,6 +2485,13 @@ static bool should_skip_rmap_item(struct folio *folio,
 	if (folio_test_ksm(folio))
 		return false;
 
+	/*
+	 * There is no age information in stable-tree nodes. We might end up
+	 * here without a KSM page for example after COW.
+	 */
+	if (rmap_item->address & STABLE_FLAG)
+		return false;
+
 	age = rmap_item->age;
 	if (age != U8_MAX)
 		rmap_item->age++;
@@ -2737,8 +2771,7 @@ static struct ksm_rmap_item *scan_get_next_rmap_item(struct page **page)
 		 * or when all VM_MERGEABLE areas have been unmapped (and
 		 * mmap_lock then protects against race with MADV_MERGEABLE).
 		 */
-		hash_del(&mm_slot->slot.hash);
-		list_del(&mm_slot->slot.mm_node);
+		mm_slot_remove(&mm_slot->slot);
 		spin_unlock(&ksm_mmlist_lock);
 
 		mm_slot_free(mm_slot_cache, mm_slot);
@@ -3081,8 +3114,7 @@ void __ksm_exit(struct mm_struct *mm)
 	if (ksm_scan.mm_slot == mm_slot)
 		goto unlock;
 	if (!mm_slot->rmap_list) {
-		hash_del(&slot->hash);
-		list_del(&slot->mm_node);
+		mm_slot_remove(slot);
 		easy_to_free = 1;
 	} else {
 		list_move(&slot->mm_node,
@@ -3173,6 +3205,7 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc)
 	hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
 		/* Ignore the stable/unstable/sqnr flags */
 		const unsigned long addr = rmap_item->address & PAGE_MASK;
+		const unsigned long index = rmap_item->linear_page_index;
 		struct anon_vma *anon_vma = rmap_item->anon_vma;
 		struct anon_vma_chain *vmac;
 		struct vm_area_struct *vma;
@@ -3186,8 +3219,17 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc)
 			anon_vma_lock_read(anon_vma);
 		}
 
-		anon_vma_interval_tree_foreach(vmac, &anon_vma->rb_root,
-					       0, ULONG_MAX) {
+		/*
+		 * Currently, KSM folios are always small folios, so it's
+		 * sufficient to search for a single page. We can simply use
+		 * the linear_page_index of the original de-duplicate
+		 * anonymous page that we remembered in the rmap_item while
+		 * de-duplicating. Note that mremap() always de-duplicates KSM
+		 * folios: so if there was mremap() in our parent or our child,
+		 * we wouldn't have the KSM folio mapped in these processes
+		 * anymore.
+		 */
+		anon_rmap_tree_foreach(vmac, anon_vma, index, index) {
 
 			cond_resched();
 			vma = vmac->vma;
@@ -3243,17 +3285,16 @@ void collect_procs_ksm(const struct folio *folio, const struct page *page,
 		rcu_read_lock();
 		for_each_process(tsk) {
 			struct anon_vma_chain *vmac;
-			unsigned long addr;
+			const unsigned long addr = rmap_item->address & PAGE_MASK;
+			const unsigned long index = rmap_item->linear_page_index;
 			struct task_struct *t =
 				task_early_kill(tsk, force_early);
 			if (!t)
 				continue;
-			anon_vma_interval_tree_foreach(vmac, &av->rb_root, 0,
-						       ULONG_MAX)
+			anon_rmap_tree_foreach(vmac, av, index, index)
 			{
 				vma = vmac->vma;
 				if (vma->vm_mm == t->mm) {
-					addr = rmap_item->address & PAGE_MASK;
 					add_to_kill_ksm(t, page, vma, to_kill,
 							addr);
 				}
diff --git a/mm/madvise.c b/mm/madvise.c
index 77552b03d318d..07a21ca31bad4 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -188,7 +188,7 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
 		unsigned long end, struct mm_walk *walk)
 {
 	struct vm_area_struct *vma = walk->private;
-	struct swap_iocb *splug = NULL;
+	struct swap_io_ctx ctx = {};
 	pte_t *ptep = NULL;
 	spinlock_t *ptl;
 	unsigned long addr;
@@ -212,15 +212,15 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
 		pte_unmap_unlock(ptep, ptl);
 		ptep = NULL;
 
-		folio = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
-					     vma, addr, &splug);
+		folio = read_swap_cache_async(&ctx, entry, GFP_HIGHUSER_MOVABLE,
+					vma, addr);
 		if (folio)
 			folio_put(folio);
 	}
 
 	if (ptep)
 		pte_unmap_unlock(ptep, ptl);
-	swap_read_unplug(splug);
+	swap_read_submit(&ctx);
 	cond_resched();
 
 	return 0;
@@ -238,7 +238,7 @@ static void shmem_swapin_range(struct vm_area_struct *vma,
 	XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start));
 	pgoff_t end_index = linear_page_index(vma, end) - 1;
 	struct folio *folio;
-	struct swap_iocb *splug = NULL;
+	struct swap_io_ctx ctx = {};
 
 	rcu_read_lock();
 	xas_for_each(&xas, folio, end_index) {
@@ -253,19 +253,19 @@ static void shmem_swapin_range(struct vm_area_struct *vma,
 			continue;
 
 		addr = vma->vm_start +
-			((xas.xa_index - vma->vm_pgoff) << PAGE_SHIFT);
+			((xas.xa_index - vma_start_pgoff(vma)) << PAGE_SHIFT);
 		xas_pause(&xas);
 		rcu_read_unlock();
 
-		folio = read_swap_cache_async(entry, mapping_gfp_mask(mapping),
-					     vma, addr, &splug);
+		folio = read_swap_cache_async(&ctx, entry,
+				mapping_gfp_mask(mapping), vma, addr);
 		if (folio)
 			folio_put(folio);
 
 		rcu_read_lock();
 	}
 	rcu_read_unlock();
-	swap_read_unplug(splug);
+	swap_read_submit(&ctx);
 }
 #endif		/* CONFIG_SWAP */
 
@@ -318,7 +318,7 @@ static long madvise_willneed(struct madvise_behavior *madv_behavior)
 	mark_mmap_lock_dropped(madv_behavior);
 	get_file(file);
 	offset = (loff_t)(start - vma->vm_start)
-			+ ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
+			+ ((loff_t)vma_start_pgoff(vma) << PAGE_SHIFT);
 	mmap_read_unlock(mm);
 	vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED);
 	fput(file);
@@ -388,8 +388,8 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
 			goto huge_unlock;
 
 		if (unlikely(!pmd_present(orig_pmd))) {
-			VM_BUG_ON(thp_migration_supported() &&
-					!pmd_is_migration_entry(orig_pmd));
+			VM_WARN_ON_ONCE(!pmd_is_migration_entry(orig_pmd) &&
+					!pmd_is_device_private_entry(orig_pmd));
 			goto huge_unlock;
 		}
 
@@ -694,10 +694,10 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
 				nr = swap_pte_batch(pte, max_nr, ptent);
 				nr_swap -= nr;
 				swap_put_entries_direct(entry, nr);
-				clear_not_present_full_ptes(mm, addr, pte, nr, tlb->fullmm);
+				clear_nonpresent_ptes(mm, addr, pte, nr);
 			} else if (softleaf_is_hwpoison(entry) ||
 				   softleaf_is_poison_marker(entry)) {
-				pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
+				pte_clear(mm, addr, pte);
 			}
 			continue;
 		}
@@ -1022,7 +1022,7 @@ static long madvise_remove(struct madvise_behavior *madv_behavior)
 		return -EACCES;
 
 	offset = (loff_t)(start - vma->vm_start)
-			+ ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
+			+ ((loff_t)vma_start_pgoff(vma) << PAGE_SHIFT);
 
 	/*
 	 * Filesystem's fallocate may need to take i_rwsem.  We need to
@@ -1233,7 +1233,7 @@ static int guard_remove_pte_entry(pte_t *pte, unsigned long addr,
 
 	if (is_guard_pte_marker(ptent)) {
 		/* Simply clear the PTE marker. */
-		pte_clear_not_present_full(walk->mm, addr, pte, false);
+		pte_clear(walk->mm, addr, pte);
 		update_mmu_cache(walk->vma, addr, pte);
 	}
 
diff --git a/mm/mapping_dirty_helpers.c b/mm/mapping_dirty_helpers.c
index 737c407f40812..e0efa36e0a076 100644
--- a/mm/mapping_dirty_helpers.c
+++ b/mm/mapping_dirty_helpers.c
@@ -95,7 +95,7 @@ static int clean_record_pte(pte_t *pte, unsigned long addr,
 
 	if (pte_dirty(ptent)) {
 		pgoff_t pgoff = ((addr - walk->vma->vm_start) >> PAGE_SHIFT) +
-			walk->vma->vm_pgoff - cwalk->bitmap_pgoff;
+			vma_start_pgoff(walk->vma) - cwalk->bitmap_pgoff;
 		pte_t old_pte = ptep_modify_prot_start(walk->vma, addr, pte);
 
 		ptent = pte_mkclean(old_pte);
diff --git a/mm/memblock.c b/mm/memblock.c
index 6349c48154f4b..036316e9138ba 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -29,6 +29,7 @@
 #include <linux/io.h>
 
 #include "internal.h"
+#include "mm_init.h"
 
 #define INIT_MEMBLOCK_REGIONS			128
 #define INIT_PHYSMEM_REGIONS			4
@@ -2224,10 +2225,8 @@ static void __init free_unused_memmap(void)
 	}
 
 #ifdef CONFIG_SPARSEMEM
-	if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) {
-		prev_end = pageblock_align(end);
+	if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION))
 		free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION));
-	}
 #endif
 }
 
diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c
index 7650692115678..2dc599484d006 100644
--- a/mm/memcontrol-v1.c
+++ b/mm/memcontrol-v1.c
@@ -6,6 +6,7 @@
 #include <linux/pagewalk.h>
 #include <linux/backing-dev.h>
 #include <linux/eventfd.h>
+#include <linux/log2.h>
 #include <linux/poll.h>
 #include <linux/sort.h>
 #include <linux/file.h>
@@ -751,7 +752,7 @@ static int compare_thresholds(const void *a, const void *b)
 	return 0;
 }
 
-static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
+static void mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
 {
 	struct mem_cgroup_eventfd_list *ev;
 
@@ -761,7 +762,6 @@ static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
 		eventfd_signal(ev->eventfd);
 
 	spin_unlock(&memcg_oom_lock);
-	return 0;
 }
 
 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
@@ -1181,13 +1181,13 @@ static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
 		event->unregister_event = mem_cgroup_usage_unregister_event;
 	} else if (!strcmp(name, "memory.oom_control")) {
 		pr_warn_once("oom_control is deprecated and will be removed. "
-			     "Please report your usecase to linux-mm-@kvack.org"
+			     "Please report your usecase to linux-mm@kvack.org"
 			     " if you depend on this functionality.\n");
 		event->register_event = mem_cgroup_oom_register_event;
 		event->unregister_event = mem_cgroup_oom_unregister_event;
 	} else if (!strcmp(name, "memory.pressure_level")) {
 		pr_warn_once("pressure_level is deprecated and will be removed. "
-			     "Please report your usecase to linux-mm-@kvack.org "
+			     "Please report your usecase to linux-mm@kvack.org "
 			     "if you depend on this functionality.\n");
 		event->register_event = vmpressure_register_event;
 		event->unregister_event = vmpressure_unregister_event;
@@ -1476,6 +1476,297 @@ void memcg1_oom_finish(struct mem_cgroup *memcg, bool locked)
 		mem_cgroup_oom_unlock(memcg);
 }
 
+/*
+ * cgroup v1 userspace vmpressure interface (memory.pressure_level /
+ * cgroup.event_control). Kept here so v2-only kernels (CONFIG_MEMCG_V1=n)
+ * drop the whole eventfd accumulator, its work item, and the per-memcg
+ * state it requires.
+ *
+ * When there are too little pages left to scan, vmpressure() may miss the
+ * critical pressure as number of pages will be less than "window size".
+ * However, in that case the vmscan priority will raise fast as the
+ * reclaimer will try to scan LRUs more deeply.
+ *
+ * The vmscan logic considers these special priorities:
+ *
+ * prio == DEF_PRIORITY (12): reclaimer starts with that value
+ * prio <= DEF_PRIORITY - 2 : kswapd becomes somewhat overwhelmed
+ * prio == 0                : close to OOM, kernel scans every page in an lru
+ *
+ * Any value in this range is acceptable for this tunable (i.e. from 12 to
+ * 0). Current value for the vmpressure_level_critical_prio is chosen
+ * empirically, but the number, in essence, means that we consider
+ * critical level when scanning depth is ~10% of the lru size (vmscan
+ * scans 'lru_size >> prio' pages, so it is actually 12.5%, or one
+ * eights).
+ */
+static const unsigned int vmpressure_level_critical_prio = ilog2(100 / 10);
+
+enum vmpressure_modes {
+	VMPRESSURE_NO_PASSTHROUGH = 0,
+	VMPRESSURE_HIERARCHY,
+	VMPRESSURE_LOCAL,
+	VMPRESSURE_NUM_MODES,
+};
+
+static const char * const vmpressure_str_levels[] = {
+	[VMPRESSURE_LOW] = "low",
+	[VMPRESSURE_MEDIUM] = "medium",
+	[VMPRESSURE_CRITICAL] = "critical",
+};
+
+static const char * const vmpressure_str_modes[] = {
+	[VMPRESSURE_NO_PASSTHROUGH] = "default",
+	[VMPRESSURE_HIERARCHY] = "hierarchy",
+	[VMPRESSURE_LOCAL] = "local",
+};
+
+struct vmpressure_event {
+	struct eventfd_ctx *efd;
+	enum vmpressure_levels level;
+	enum vmpressure_modes mode;
+	struct list_head node;
+};
+
+static struct vmpressure *work_to_vmpressure(struct work_struct *work)
+{
+	return container_of(work, struct vmpressure, work);
+}
+
+static struct vmpressure *vmpressure_parent(struct vmpressure *vmpr)
+{
+	struct mem_cgroup *memcg = vmpressure_to_memcg(vmpr);
+
+	memcg = parent_mem_cgroup(memcg);
+	if (!memcg)
+		return NULL;
+	return memcg_to_vmpressure(memcg);
+}
+
+static bool vmpressure_event(struct vmpressure *vmpr,
+			     const enum vmpressure_levels level,
+			     bool ancestor, bool signalled)
+{
+	struct vmpressure_event *ev;
+	bool ret = false;
+
+	mutex_lock(&vmpr->events_lock);
+	list_for_each_entry(ev, &vmpr->events, node) {
+		if (ancestor && ev->mode == VMPRESSURE_LOCAL)
+			continue;
+		if (signalled && ev->mode == VMPRESSURE_NO_PASSTHROUGH)
+			continue;
+		if (level < ev->level)
+			continue;
+		eventfd_signal(ev->efd);
+		ret = true;
+	}
+	mutex_unlock(&vmpr->events_lock);
+
+	return ret;
+}
+
+static void vmpressure_work_fn(struct work_struct *work)
+{
+	struct vmpressure *vmpr = work_to_vmpressure(work);
+	unsigned long scanned;
+	unsigned long reclaimed;
+	enum vmpressure_levels level;
+	bool ancestor = false;
+	bool signalled = false;
+
+	spin_lock(&vmpr->sr_lock);
+	/*
+	 * Several contexts might be calling vmpressure(), so it is
+	 * possible that the work was rescheduled again before the old
+	 * work context cleared the counters. In that case we will run
+	 * just after the old work returns, but then scanned might be zero
+	 * here. No need for any locks here since we don't care if
+	 * vmpr->reclaimed is in sync.
+	 */
+	scanned = vmpr->tree_scanned;
+	if (!scanned) {
+		spin_unlock(&vmpr->sr_lock);
+		return;
+	}
+
+	reclaimed = vmpr->tree_reclaimed;
+	vmpr->tree_scanned = 0;
+	vmpr->tree_reclaimed = 0;
+	spin_unlock(&vmpr->sr_lock);
+
+	level = vmpressure_calc_level(scanned, reclaimed);
+
+	do {
+		if (vmpressure_event(vmpr, level, ancestor, signalled))
+			signalled = true;
+		ancestor = true;
+	} while ((vmpr = vmpressure_parent(vmpr)));
+}
+
+/*
+ * Tree-mode accumulator: accumulate per-memcg scanned/reclaimed and
+ * schedule the work that walks the parent chain and signals registered
+ * eventfd listeners once we cross the window threshold.
+ */
+void vmpressure_v1_account_tree(struct vmpressure *vmpr,
+				unsigned long scanned,
+				unsigned long reclaimed)
+{
+	spin_lock(&vmpr->sr_lock);
+	scanned = vmpr->tree_scanned += scanned;
+	vmpr->tree_reclaimed += reclaimed;
+	spin_unlock(&vmpr->sr_lock);
+
+	if (scanned < vmpressure_win)
+		return;
+	schedule_work(&vmpr->work);
+}
+
+void vmpressure_v1_init(struct vmpressure *vmpr)
+{
+	mutex_init(&vmpr->events_lock);
+	INIT_LIST_HEAD(&vmpr->events);
+	INIT_WORK(&vmpr->work, vmpressure_work_fn);
+}
+
+void vmpressure_v1_cleanup(struct vmpressure *vmpr)
+{
+	/*
+	 * Make sure there is no pending work before eventfd infrastructure
+	 * goes away.
+	 */
+	flush_work(&vmpr->work);
+}
+
+/**
+ * vmpressure_prio() - Account memory pressure through reclaimer priority level
+ * @gfp:	reclaimer's gfp mask
+ * @memcg:	cgroup memory controller handle
+ * @prio:	reclaimer's priority
+ *
+ * This function should be called from the reclaim path every time when
+ * the vmscan's reclaiming priority (scanning depth) changes.
+ *
+ * This function does not return any value.
+ */
+void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio)
+{
+	/*
+	 * We only use prio for accounting critical level. For more info
+	 * see comment for vmpressure_level_critical_prio variable above.
+	 */
+	if (prio > vmpressure_level_critical_prio)
+		return;
+
+	/*
+	 * OK, the prio is below the threshold, updating vmpressure
+	 * information before shrinker dives into long shrinking of long
+	 * range vmscan. Passing scanned = vmpressure_win, reclaimed = 0
+	 * to the vmpressure() basically means that we signal 'critical'
+	 * level.
+	 */
+	vmpressure(gfp, 0, memcg, true, vmpressure_win, 0);
+}
+
+#define MAX_VMPRESSURE_ARGS_LEN	(strlen("critical") + strlen("hierarchy") + 2)
+
+/**
+ * vmpressure_register_event() - Bind vmpressure notifications to an eventfd
+ * @memcg:	memcg that is interested in vmpressure notifications
+ * @eventfd:	eventfd context to link notifications with
+ * @args:	event arguments (pressure level threshold, optional mode)
+ *
+ * This function associates eventfd context with the vmpressure
+ * infrastructure, so that the notifications will be delivered to the
+ * @eventfd. The @args parameter is a comma-delimited string that denotes a
+ * pressure level threshold (one of vmpressure_str_levels, i.e. "low", "medium",
+ * or "critical") and an optional mode (one of vmpressure_str_modes, i.e.
+ * "hierarchy" or "local").
+ *
+ * To be used as memcg event method.
+ *
+ * Return: 0 on success, -ENOMEM on memory failure or -EINVAL if @args could
+ * not be parsed.
+ */
+int vmpressure_register_event(struct mem_cgroup *memcg,
+			      struct eventfd_ctx *eventfd, const char *args)
+{
+	struct vmpressure *vmpr = memcg_to_vmpressure(memcg);
+	struct vmpressure_event *ev;
+	enum vmpressure_modes mode = VMPRESSURE_NO_PASSTHROUGH;
+	enum vmpressure_levels level;
+	char *spec, *spec_orig;
+	char *token;
+	int ret = 0;
+
+	spec_orig = spec = kstrndup(args, MAX_VMPRESSURE_ARGS_LEN, GFP_KERNEL);
+	if (!spec)
+		return -ENOMEM;
+
+	/* Find required level */
+	token = strsep(&spec, ",");
+	ret = match_string(vmpressure_str_levels, VMPRESSURE_NUM_LEVELS, token);
+	if (ret < 0)
+		goto out;
+	level = ret;
+
+	/* Find optional mode */
+	token = strsep(&spec, ",");
+	if (token) {
+		ret = match_string(vmpressure_str_modes, VMPRESSURE_NUM_MODES, token);
+		if (ret < 0)
+			goto out;
+		mode = ret;
+	}
+
+	ev = kzalloc_obj(*ev, GFP_KERNEL_ACCOUNT);
+	if (!ev) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	ev->efd = eventfd;
+	ev->level = level;
+	ev->mode = mode;
+
+	mutex_lock(&vmpr->events_lock);
+	list_add(&ev->node, &vmpr->events);
+	mutex_unlock(&vmpr->events_lock);
+	ret = 0;
+out:
+	kfree(spec_orig);
+	return ret;
+}
+
+/**
+ * vmpressure_unregister_event() - Unbind eventfd from vmpressure
+ * @memcg:	memcg handle
+ * @eventfd:	eventfd context that was used to link vmpressure with the @cg
+ *
+ * This function does internal manipulations to detach the @eventfd from
+ * the vmpressure notifications, and then frees internal resources
+ * associated with the @eventfd (but the @eventfd itself is not freed).
+ *
+ * To be used as memcg event method.
+ */
+void vmpressure_unregister_event(struct mem_cgroup *memcg,
+				 struct eventfd_ctx *eventfd)
+{
+	struct vmpressure *vmpr = memcg_to_vmpressure(memcg);
+	struct vmpressure_event *ev;
+
+	mutex_lock(&vmpr->events_lock);
+	list_for_each_entry(ev, &vmpr->events, node) {
+		if (ev->efd != eventfd)
+			continue;
+		list_del(&ev->node);
+		kfree(ev);
+		break;
+	}
+	mutex_unlock(&vmpr->events_lock);
+}
+
 static DEFINE_MUTEX(memcg_max_mutex);
 
 static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
@@ -1513,6 +1804,10 @@ static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
 		if (!ret)
 			break;
 
+		/* cgroup_rmdir() waits for us with cgroup_mutex held. */
+		if (memcg_is_dying(memcg))
+			break;
+
 		if (!drained) {
 			drain_all_stock(memcg);
 			drained = true;
@@ -1551,6 +1846,10 @@ static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
 		if (signal_pending(current))
 			return -EINTR;
 
+		/* cgroup_rmdir() waits for us with cgroup_mutex held. */
+		if (memcg_is_dying(memcg))
+			break;
+
 		if (!try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL,
 						  MEMCG_RECLAIM_MAY_SWAP, NULL))
 			nr_retries--;
@@ -2040,7 +2339,7 @@ static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
 	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
 
 	pr_warn_once("oom_control is deprecated and will be removed. "
-		     "Please report your usecase to linux-mm-@kvack.org if you "
+		     "Please report your usecase to linux-mm@kvack.org if you "
 		     "depend on this functionality.\n");
 
 	/* cannot set to root cgroup and only 0 and 1 are allowed */
diff --git a/mm/memcontrol-v1.h b/mm/memcontrol-v1.h
index f92f81108d5ed..0f703f239c80f 100644
--- a/mm/memcontrol-v1.h
+++ b/mm/memcontrol-v1.h
@@ -17,14 +17,8 @@
 	     iter != NULL;				\
 	     iter = mem_cgroup_iter(root, iter, NULL))
 
-#define for_each_mem_cgroup(iter)			\
-	for (iter = mem_cgroup_iter(NULL, NULL, NULL);	\
-	     iter != NULL;				\
-	     iter = mem_cgroup_iter(NULL, iter, NULL))
-
 void drain_all_stock(struct mem_cgroup *root_memcg);
 
-unsigned long memcg_events(struct mem_cgroup *memcg, int event);
 int memory_stat_show(struct seq_file *m, void *v);
 
 struct mem_cgroup *mem_cgroup_private_id_get_online(struct mem_cgroup *memcg,
@@ -107,7 +101,11 @@ static inline void memcg1_remove_from_trees(struct mem_cgroup *memcg) {}
 static inline void memcg1_soft_limit_reset(struct mem_cgroup *memcg) {}
 static inline void memcg1_css_offline(struct mem_cgroup *memcg) {}
 
-static inline bool memcg1_oom_prepare(struct mem_cgroup *memcg, bool *locked) { return true; }
+static inline bool memcg1_oom_prepare(struct mem_cgroup *memcg, bool *locked)
+{
+	*locked = false;
+	return true;
+}
 static inline void memcg1_oom_finish(struct mem_cgroup *memcg, bool locked) {}
 static inline void memcg1_oom_recover(struct mem_cgroup *memcg) {}
 
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 6dc4888a90f3f..8319ad8c5c23a 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -137,6 +137,14 @@ bool mem_cgroup_kmem_disabled(void)
 
 static void memcg_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages);
 
+static void memcg_uncharge_kmem(struct mem_cgroup *memcg, unsigned int nr_pages)
+{
+	mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages);
+	memcg1_account_kmem(memcg, -nr_pages);
+	if (!mem_cgroup_is_root(memcg))
+		memcg_uncharge(memcg, nr_pages);
+}
+
 static void obj_cgroup_release(struct percpu_ref *ref)
 {
 	struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt);
@@ -172,10 +180,7 @@ static void obj_cgroup_release(struct percpu_ref *ref)
 		struct mem_cgroup *memcg;
 
 		memcg = get_mem_cgroup_from_objcg(objcg);
-		mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages);
-		memcg1_account_kmem(memcg, -nr_pages);
-		if (!mem_cgroup_is_root(memcg))
-			memcg_uncharge(memcg, nr_pages);
+		memcg_uncharge_kmem(memcg, nr_pages);
 		mem_cgroup_put(memcg);
 	}
 
@@ -2039,7 +2044,7 @@ struct obj_stock_pcp {
 	/*
 	 * On rare archs with 256KiB base page size (hexagon and powerpc 44x)
 	 * keep nr_bytes to unsigned int as uint16_t cannot represent the full
-e patches/memcg-uint16_t-for-nr_bytes-in-obj_stock_pcp.patch	 * sub-page remainder. Such archs are not cacheline optimization target.
+	 * sub-page remainder. Such archs are not cacheline optimization targets.
 	 */
 	unsigned int nr_bytes[NR_OBJ_STOCK];
 #else
@@ -3329,10 +3334,7 @@ static void drain_obj_stock_slot(struct obj_stock_pcp *stock, int i)
 
 			memcg = get_mem_cgroup_from_objcg(old);
 
-			mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages);
-			memcg1_account_kmem(memcg, -nr_pages);
-			if (!mem_cgroup_is_root(memcg))
-				memcg_uncharge(memcg, nr_pages);
+			memcg_uncharge_kmem(memcg, nr_pages);
 
 			css_put(&memcg->css);
 		}
@@ -4217,7 +4219,7 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
 	/*
 	 * A memcg must be visible for expand_shrinker_info()
 	 * by the time the maps are allocated. So, we allocate maps
-	 * here, when for_each_mem_cgroup() can't skip it.
+	 * here, when mem_cgroup_iter() can't skip it.
 	 */
 	if (alloc_shrinker_info(memcg))
 		goto offline_kmem;
@@ -4362,6 +4364,11 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
 
 	page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
 	page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
+	WRITE_ONCE(memcg->oom_group, false);
+#ifdef CONFIG_ZSWAP
+	WRITE_ONCE(memcg->zswap_max, PAGE_COUNTER_MAX);
+	WRITE_ONCE(memcg->zswap_writeback, true);
+#endif
 #ifdef CONFIG_MEMCG_V1
 	page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
 	page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
@@ -4428,8 +4435,7 @@ static void mem_cgroup_stat_aggregate(struct aggregate_control *ac)
 }
 
 #ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC
-static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent,
-			    int cpu)
+static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent)
 {
 	int nid;
 
@@ -4438,6 +4444,7 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent,
 		int index = memcg_stats_index(MEMCG_KMEM);
 
 		memcg->vmstats->state[index] += kmem;
+		memcg->vmstats->state_local[index] += kmem;
 		if (parent)
 			parent->vmstats->state_pending[index] += kmem;
 	}
@@ -4455,9 +4462,11 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent,
 			int index = memcg_stats_index(NR_SLAB_RECLAIMABLE_B);
 
 			lstats->state[index] += slab;
+			lstats->state_local[index] += slab;
 			if (plstats)
 				plstats->state_pending[index] += slab;
 			memcg->vmstats->state[index] += slab;
+			memcg->vmstats->state_local[index] += slab;
 			if (parent)
 				parent->vmstats->state_pending[index] += slab;
 		}
@@ -4466,17 +4475,18 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent,
 			int index = memcg_stats_index(NR_SLAB_UNRECLAIMABLE_B);
 
 			lstats->state[index] += slab;
+			lstats->state_local[index] += slab;
 			if (plstats)
 				plstats->state_pending[index] += slab;
 			memcg->vmstats->state[index] += slab;
+			memcg->vmstats->state_local[index] += slab;
 			if (parent)
 				parent->vmstats->state_pending[index] += slab;
 		}
 	}
 }
 #else
-static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent,
-			    int cpu)
+static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent)
 {}
 #endif
 
@@ -4488,7 +4498,7 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu)
 	struct aggregate_control ac;
 	int nid;
 
-	flush_nmi_stats(memcg, parent, cpu);
+	flush_nmi_stats(memcg, parent);
 
 	statc = per_cpu_ptr(memcg->vmstats_percpu, cpu);
 
@@ -4794,6 +4804,10 @@ static ssize_t memory_high_write(struct kernfs_open_file *of,
 		if (signal_pending(current))
 			break;
 
+		/* cgroup_rmdir() waits for us with cgroup_mutex held. */
+		if (memcg_is_dying(memcg))
+			break;
+
 		if (!drained) {
 			drain_all_stock(memcg);
 			drained = true;
@@ -4845,6 +4859,10 @@ static ssize_t memory_max_write(struct kernfs_open_file *of,
 		if (signal_pending(current))
 			break;
 
+		/* cgroup_rmdir() waits for us with cgroup_mutex held. */
+		if (memcg_is_dying(memcg))
+			break;
+
 		if (!drained) {
 			drain_all_stock(memcg);
 			drained = true;
diff --git a/mm/memfd.c b/mm/memfd.c
index abe13b291ddc8..c708d92533f4a 100644
--- a/mm/memfd.c
+++ b/mm/memfd.c
@@ -19,6 +19,7 @@
 #include <linux/memfd.h>
 #include <linux/pid_namespace.h>
 #include <uapi/linux/memfd.h>
+#include "internal.h"
 #include "swap.h"
 
 /*
@@ -369,39 +370,36 @@ static inline bool is_write_sealed(unsigned int seals)
 	return seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE);
 }
 
-static int check_write_seal(vm_flags_t *vm_flags_ptr)
+static int check_write_seal(vma_flags_t *vma_flags_ptr)
 {
-	vm_flags_t vm_flags = *vm_flags_ptr;
-	vm_flags_t mask = vm_flags & (VM_SHARED | VM_WRITE);
-
 	/* If a private mapping then writability is irrelevant. */
-	if (!(mask & VM_SHARED))
+	if (!vma_flags_test(vma_flags_ptr, VMA_SHARED_BIT))
 		return 0;
 
 	/*
 	 * New PROT_WRITE and MAP_SHARED mmaps are not allowed when
 	 * write seals are active.
 	 */
-	if (mask & VM_WRITE)
+	if (vma_flags_test(vma_flags_ptr, VMA_WRITE_BIT))
 		return -EPERM;
 
 	/*
 	 * This is a read-only mapping, disallow mprotect() from making a
 	 * write-sealed mapping writable in future.
 	 */
-	*vm_flags_ptr &= ~VM_MAYWRITE;
+	vma_flags_clear(vma_flags_ptr, VMA_MAYWRITE_BIT);
 
 	return 0;
 }
 
-int memfd_check_seals_mmap(struct file *file, vm_flags_t *vm_flags_ptr)
+int memfd_check_seals_mmap(struct file *file, vma_flags_t *vma_flags_ptr)
 {
 	int err = 0;
 	unsigned int *seals_ptr = memfd_file_seals_ptr(file);
 	unsigned int seals = seals_ptr ? *seals_ptr : 0;
 
 	if (is_write_sealed(seals))
-		err = check_write_seal(vm_flags_ptr);
+		err = check_write_seal(vma_flags_ptr);
 
 	return err;
 }
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index 51508a55c4055..aaf14608b30e2 100644
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -66,6 +66,7 @@
 #include <trace/events/memory-failure.h>
 
 #include "swap.h"
+#include "page_alloc.h"
 #include "internal.h"
 
 static int sysctl_memory_failure_early_kill __read_mostly;
@@ -74,9 +75,11 @@ static int sysctl_memory_failure_recovery __read_mostly = 1;
 
 static int sysctl_enable_soft_offline __read_mostly = 1;
 
+static int sysctl_panic_on_unrecoverable_mf __read_mostly;
+
 atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0);
 
-static bool hw_memory_failure __read_mostly = false;
+static bool hw_memory_failure __read_mostly;
 
 static DEFINE_MUTEX(mf_mutex);
 
@@ -155,6 +158,15 @@ static const struct ctl_table memory_failure_table[] = {
 		.proc_handler	= proc_dointvec_minmax,
 		.extra1		= SYSCTL_ZERO,
 		.extra2		= SYSCTL_ONE,
+	},
+	{
+		.procname	= "panic_on_unrecoverable_memory_failure",
+		.data		= &sysctl_panic_on_unrecoverable_mf,
+		.maxlen		= sizeof(sysctl_panic_on_unrecoverable_mf),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_ONE,
 	}
 };
 
@@ -552,8 +564,7 @@ static void collect_procs_anon(const struct folio *folio,
 
 		if (!t)
 			continue;
-		anon_vma_interval_tree_foreach(vmac, &av->rb_root,
-					       pgoff, pgoff) {
+		anon_rmap_tree_foreach(vmac, av, pgoff, pgoff) {
 			vma = vmac->vma;
 			if (vma->vm_mm != t->mm)
 				continue;
@@ -586,8 +597,7 @@ static void collect_procs_file(const struct folio *folio,
 
 		if (!t)
 			continue;
-		vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff,
-				      pgoff) {
+		mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) {
 			/*
 			 * Send early kill signal to tasks where a vma covers
 			 * the page but the corrupted page is not necessarily
@@ -638,7 +648,7 @@ static void collect_procs_fsdax(const struct page *page,
 			t = task_early_kill(tsk, true);
 		if (!t)
 			continue;
-		vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
+		mapping_rmap_tree_foreach(vma, mapping, pgoff, pgoff) {
 			if (vma->vm_mm == t->mm)
 				add_to_kill_fsdax(t, page, vma, to_kill, pgoff);
 		}
@@ -980,17 +990,6 @@ static bool has_extra_refcount(struct page_state *ps, struct page *p,
 	return false;
 }
 
-/*
- * Error hit kernel page.
- * Do nothing, try to be lucky and not touch this instead. For a few cases we
- * could be more sophisticated.
- */
-static int me_kernel(struct page_state *ps, struct page *p)
-{
-	unlock_page(p);
-	return MF_IGNORED;
-}
-
 /*
  * Page in unknown state. Do nothing.
  * This is a catch-all in case we fail to make sense of the page state.
@@ -1199,10 +1198,8 @@ static int me_huge_page(struct page_state *ps, struct page *p)
 #define mlock		(1UL << PG_mlocked)
 #define lru		(1UL << PG_lru)
 #define head		(1UL << PG_head)
-#define reserved	(1UL << PG_reserved)
 
 static struct page_state error_states[] = {
-	{ reserved,	reserved,	MF_MSG_KERNEL,	me_kernel },
 	/*
 	 * free pages are specially detected outside this table:
 	 * PG_buddy pages only make a small fraction of all free pages.
@@ -1234,7 +1231,6 @@ static struct page_state error_states[] = {
 #undef mlock
 #undef lru
 #undef head
-#undef reserved
 
 static void update_per_node_mf_stats(unsigned long pfn,
 				     enum mf_result result)
@@ -1269,6 +1265,15 @@ static void update_per_node_mf_stats(unsigned long pfn,
 	++mf_stats->total;
 }
 
+static bool panic_on_unrecoverable_mf(enum mf_action_page_type type,
+				      enum mf_result result)
+{
+	if (!sysctl_panic_on_unrecoverable_mf)
+		return false;
+
+	return type == MF_MSG_KERNEL && result == MF_IGNORED;
+}
+
 /*
  * "Dirty/Clean" indication is not 100% accurate due to the possibility of
  * setting PG_dirty outside page lock. See also comment above set_page_dirty().
@@ -1286,6 +1291,9 @@ static int action_result(unsigned long pfn, enum mf_action_page_type type,
 	pr_err("%#lx: recovery action for %s: %s\n",
 		pfn, action_page_types[type], action_name[result]);
 
+	if (panic_on_unrecoverable_mf(type, result))
+		panic("Memory failure: %#lx: unrecoverable page", pfn);
+
 	return (result == MF_RECOVERED || result == MF_DELAYED) ? 0 : -EBUSY;
 }
 
@@ -1339,6 +1347,38 @@ static inline bool HWPoisonHandlable(struct page *page, unsigned long flags)
 	return PageLRU(page) || is_free_buddy_page(page);
 }
 
+/*
+ * Positive identification of pages the hwpoison handler cannot recover:
+ * pages owned by kernel internals with no userspace mapping to unmap, no
+ * file mapping to invalidate, and no migration target.
+ */
+static inline bool is_kernel_owned_page(struct page *page)
+{
+	struct page *head;
+	bool kernel_owned;
+
+	/* PG_reserved is a per-page flag, never set on a compound page. */
+	if (PageReserved(page))
+		return true;
+
+	/*
+	 * Page-type bits live only on the head page, so resolve any tail
+	 * first.  The check takes no refcount; recheck the head afterwards
+	 * so a concurrent split or compound free cannot leave us trusting
+	 * a stale view.  A residual free->alloc->free cannot be closed here
+	 * (frozen slab and large-kmalloc pages cannot be pinned), but is
+	 * harmless: where a wrong verdict could panic, memory_failure() has
+	 * already set PageHWPoison, which bars the page from the allocator.
+	 */
+retry:
+	head = compound_head(page);
+	kernel_owned = PageSlab(head) || PageTable(head) ||
+		       PageLargeKmalloc(head);
+	if (head != compound_head(page))
+		goto retry;
+	return kernel_owned;
+}
+
 static int __get_hwpoison_page(struct page *page, unsigned long flags)
 {
 	struct folio *folio = page_folio(page);
@@ -1385,6 +1425,19 @@ static int get_any_page(struct page *p, unsigned long flags)
 	if (flags & MF_COUNT_INCREASED)
 		count_increased = true;
 
+	/*
+	 * Page types we know are kernel-owned and cannot be recovered.
+	 * Short-circuit before the shake_page() / retry loop, which
+	 * cannot turn any of these into something HWPoisonHandlable().
+	 * Drop the caller's reference if MF_COUNT_INCREASED took one.
+	 */
+	if (is_kernel_owned_page(p)) {
+		if (count_increased)
+			put_page(p);
+		ret = -ENOTRECOVERABLE;
+		goto out;
+	}
+
 try_again:
 	if (!count_increased) {
 		ret = __get_hwpoison_page(p, flags);
@@ -1432,7 +1485,7 @@ static int get_any_page(struct page *p, unsigned long flags)
 		ret = -EIO;
 	}
 out:
-	if (ret == -EIO)
+	if (ret == -EIO || ret == -ENOTRECOVERABLE)
 		pr_err("%#lx: unhandlable page.\n", page_to_pfn(p));
 
 	return ret;
@@ -1489,7 +1542,10 @@ static int __get_unpoison_page(struct page *page)
  *         -EIO for pages on which we can not handle memory errors,
  *         -EBUSY when get_hwpoison_page() has raced with page lifecycle
  *         operations like allocation and free,
- *         -EHWPOISON when the page is hwpoisoned and taken off from buddy.
+ *         -EHWPOISON when the page is hwpoisoned and taken off from buddy,
+ *         -ENOTRECOVERABLE for kernel-owned pages identified by
+ *         is_kernel_owned_page() (PG_reserved, slab,
+ *         page-table, large-kmalloc) that the handler cannot recover.
  */
 static int get_hwpoison_page(struct page *p, unsigned long flags)
 {
@@ -2239,7 +2295,7 @@ static void collect_procs_pfn(struct pfn_address_space *pfn_space,
 		t = task_early_kill(tsk, true);
 		if (!t)
 			continue;
-		vma_interval_tree_foreach(vma, &mapping->i_mmap, 0, ULONG_MAX) {
+		mapping_rmap_tree_foreach(vma, mapping, 0, ULONG_MAX) {
 			pgoff_t pgoff;
 
 			if (vma->vm_mm == t->mm &&
@@ -2402,7 +2458,8 @@ int memory_failure(unsigned long pfn, int flags)
 	 * that may make page_ref_freeze()/page_ref_unfreeze() mismatch.
 	 */
 	res = get_hwpoison_page(p, flags);
-	if (!res) {
+	switch (res) {
+	case 0:
 		if (is_free_buddy_page(p)) {
 			if (take_page_off_buddy(p)) {
 				page_ref_inc(p);
@@ -2421,7 +2478,19 @@ int memory_failure(unsigned long pfn, int flags)
 			res = action_result(pfn, MF_MSG_KERNEL_HIGH_ORDER, MF_IGNORED);
 		}
 		goto unlock_mutex;
-	} else if (res < 0) {
+	case 1:
+		/* Got a refcount on a handlable page. */
+		break;
+	case -ENOTRECOVERABLE:
+		/*
+		 * Stable unhandlable kernel-owned page (PG_reserved,
+		 * slab, page tables, large-kmalloc).
+		 * No recovery possible.
+		 */
+		res = action_result(pfn, MF_MSG_KERNEL, MF_IGNORED);
+		goto unlock_mutex;
+	default:
+		/* Transient lifecycle race with the page allocator. */
 		res = action_result(pfn, MF_MSG_GET_HWPOISON, MF_IGNORED);
 		goto unlock_mutex;
 	}
diff --git a/mm/memory.c b/mm/memory.c
index ff338c2abe923..d5e87624f6920 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -519,9 +519,52 @@ static bool is_bad_page_map_ratelimited(void)
 	return false;
 }
 
+static void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size)
+{
+	if (WARN_ON_ONCE(buf_size < entry_size * 2 + 1)) {
+		snprintf(buf, buf_size, "overflow");
+		return;
+	}
+
+	switch (entry_size) {
+	case sizeof(u32):
+		snprintf(buf, buf_size, "%08x", *(const u32 *)entry);
+		break;
+	case sizeof(u64):
+		snprintf(buf, buf_size, "%016llx", *(const u64 *)entry);
+		break;
+#if defined(__SIZEOF_INT128__)
+	case sizeof(u128):
+		snprintf(buf, buf_size, "%016llx%016llx",
+			 (unsigned long long)(*(const u128 *)entry >> 64),
+			 (unsigned long long)*(const u128 *)entry);
+		break;
+#endif
+	default:
+		snprintf(buf, buf_size, "unsupported");
+		break;
+	}
+}
+
+#define ptval_to_str(buf, val)								\
+	do {										\
+		auto __val = (val);							\
+											\
+		ptval_bytes_to_hex_str((buf), sizeof(buf), &__val, sizeof(__val));	\
+	} while (0)
+
+#if defined(__SIZEOF_INT128__)
+#define PTVAL_STR_MAX	(32 + 1) /* Max 128-bit value in hex + NUL */
+#else
+#define PTVAL_STR_MAX	(16 + 1) /* Max 64-bit value in hex + NUL */
+#endif
+
 static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long addr)
 {
-	unsigned long long pgdv, p4dv, pudv, pmdv;
+	char pgd_str[PTVAL_STR_MAX];
+	char p4d_str[PTVAL_STR_MAX];
+	char pud_str[PTVAL_STR_MAX];
+	char pmd_str[PTVAL_STR_MAX];
 	p4d_t p4d, *p4dp;
 	pud_t pud, *pudp;
 	pmd_t pmd, *pmdp;
@@ -532,34 +575,34 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add
 	 * see locking requirements for print_bad_page_map().
 	 */
 	pgdp = pgd_offset(mm, addr);
-	pgdv = pgd_val(*pgdp);
+	ptval_to_str(pgd_str, pgd_val(*pgdp));
 
 	if (!pgd_present(*pgdp) || pgd_leaf(*pgdp)) {
-		pr_alert("pgd:%08llx\n", pgdv);
+		pr_alert("pgd:%s\n", pgd_str);
 		return;
 	}
 
 	p4dp = p4d_offset(pgdp, addr);
 	p4d = p4dp_get(p4dp);
-	p4dv = p4d_val(p4d);
+	ptval_to_str(p4d_str, p4d_val(p4d));
 
 	if (!p4d_present(p4d) || p4d_leaf(p4d)) {
-		pr_alert("pgd:%08llx p4d:%08llx\n", pgdv, p4dv);
+		pr_alert("pgd:%s p4d:%s\n", pgd_str, p4d_str);
 		return;
 	}
 
 	pudp = pud_offset(p4dp, addr);
 	pud = pudp_get(pudp);
-	pudv = pud_val(pud);
+	ptval_to_str(pud_str, pud_val(pud));
 
 	if (!pud_present(pud) || pud_leaf(pud)) {
-		pr_alert("pgd:%08llx p4d:%08llx pud:%08llx\n", pgdv, p4dv, pudv);
+		pr_alert("pgd:%s p4d:%s pud:%s\n", pgd_str, p4d_str, pud_str);
 		return;
 	}
 
 	pmdp = pmd_offset(pudp, addr);
 	pmd = pmdp_get(pmdp);
-	pmdv = pmd_val(pmd);
+	ptval_to_str(pmd_str, pmd_val(pmd));
 
 	/*
 	 * Dumping the PTE would be nice, but it's tricky with CONFIG_HIGHPTE,
@@ -567,8 +610,7 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add
 	 * doing another map would be bad. print_bad_page_map() should
 	 * already take care of printing the PTE.
 	 */
-	pr_alert("pgd:%08llx p4d:%08llx pud:%08llx pmd:%08llx\n", pgdv,
-		 p4dv, pudv, pmdv);
+	pr_alert("pgd:%s p4d:%s pud:%s pmd:%s\n", pgd_str, p4d_str, pud_str, pmd_str);
 }
 
 /*
@@ -584,10 +626,11 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add
  * page table lock.
  */
 static void print_bad_page_map(struct vm_area_struct *vma,
-		unsigned long addr, unsigned long long entry, struct page *page,
-		enum pgtable_level level)
+		unsigned long addr, const void *entry, size_t entry_size,
+		struct page *page, enum pgtable_level level)
 {
 	struct address_space *mapping;
+	char entry_str[PTVAL_STR_MAX];
 	pgoff_t index;
 
 	if (is_bad_page_map_ratelimited())
@@ -596,8 +639,9 @@ static void print_bad_page_map(struct vm_area_struct *vma,
 	mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
 	index = linear_page_index(vma, addr);
 
-	pr_alert("BUG: Bad page map in process %s  %s:%08llx", current->comm,
-		 pgtable_level_to_str(level), entry);
+	ptval_bytes_to_hex_str(entry_str, sizeof(entry_str), entry, entry_size);
+	pr_alert("BUG: Bad page map in process %s  %s:%s", current->comm,
+		 pgtable_level_to_str(level), entry_str);
 	__print_bad_page_map_pgtable(vma->vm_mm, addr);
 	if (page)
 		dump_page(page, "bad page map");
@@ -627,8 +671,13 @@ static inline bool pgtable_level_has_pxx_special(enum pgtable_level level)
 	}
 }
 
-#define print_bad_pte(vma, addr, pte, page) \
-	print_bad_page_map(vma, addr, pte_val(pte), page, PGTABLE_LEVEL_PTE)
+static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
+			  pte_t pte, struct page *page)
+{
+	auto entry = pte_val(pte);
+
+	print_bad_page_map(vma, addr, &entry, sizeof(entry), page, PGTABLE_LEVEL_PTE);
+}
 
 /**
  * __vm_normal_page() - Get the "struct page" associated with a page table entry.
@@ -636,8 +685,9 @@ static inline bool pgtable_level_has_pxx_special(enum pgtable_level level)
  * @addr: The address where the page table entry is mapped.
  * @pfn: The PFN stored in the page table entry.
  * @special: Whether the page table entry is marked "special".
- * @level: The page table level for error reporting purposes only.
  * @entry: The page table entry value for error reporting purposes only.
+ * @entry_size: The size of @entry.
+ * @level: The page table level for error reporting purposes only.
  *
  * "Special" mappings do not wish to be associated with a "struct page" (either
  * it doesn't exist, or it exists but they don't want to touch it). In this
@@ -697,7 +747,7 @@ static inline bool pgtable_level_has_pxx_special(enum pgtable_level level)
  */
 static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
 		unsigned long addr, unsigned long pfn, bool special,
-		unsigned long long entry, enum pgtable_level level)
+		const void *entry, size_t entry_size, enum pgtable_level level)
 {
 	if (pgtable_level_has_pxx_special(level)) {
 		if (unlikely(special)) {
@@ -710,7 +760,7 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
 			if (is_zero_pfn(pfn) || is_huge_zero_pfn(pfn))
 				return NULL;
 
-			print_bad_page_map(vma, addr, entry, NULL, level);
+			print_bad_page_map(vma, addr, entry, entry_size, NULL, level);
 			return NULL;
 		}
 		/*
@@ -725,10 +775,10 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
 				if (!pfn_valid(pfn))
 					return NULL;
 			} else {
-				unsigned long off = (addr - vma->vm_start) >> PAGE_SHIFT;
+				const pgoff_t index = linear_page_index(vma, addr);
 
 				/* Only CoW'ed anon folios are "normal". */
-				if (pfn == vma->vm_pgoff + off)
+				if (pfn == index)
 					return NULL;
 				if (!is_cow_mapping(vma->vm_flags))
 					return NULL;
@@ -741,7 +791,7 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
 
 	if (unlikely(pfn > highest_memmap_pfn)) {
 		/* Corrupted page table entry. */
-		print_bad_page_map(vma, addr, entry, NULL, level);
+		print_bad_page_map(vma, addr, entry, entry_size, NULL, level);
 		return NULL;
 	}
 	/*
@@ -767,8 +817,10 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
 			    pte_t pte)
 {
+	auto entry = pte_val(pte);
+
 	return __vm_normal_page(vma, addr, pte_pfn(pte), pte_special(pte),
-				pte_val(pte), PGTABLE_LEVEL_PTE);
+				&entry, sizeof(entry), PGTABLE_LEVEL_PTE);
 }
 
 /**
@@ -809,8 +861,10 @@ struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr,
 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
 				pmd_t pmd)
 {
+	auto entry = pmd_val(pmd);
+
 	return __vm_normal_page(vma, addr, pmd_pfn(pmd), pmd_special(pmd),
-				pmd_val(pmd), PGTABLE_LEVEL_PMD);
+				&entry, sizeof(entry), PGTABLE_LEVEL_PMD);
 }
 
 /**
@@ -850,8 +904,10 @@ struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma,
 struct page *vm_normal_page_pud(struct vm_area_struct *vma,
 		unsigned long addr, pud_t pud)
 {
+	auto entry = pud_val(pud);
+
 	return __vm_normal_page(vma, addr, pud_pfn(pud), pud_special(pud),
-				pud_val(pud), PGTABLE_LEVEL_PUD);
+				&entry, sizeof(entry), PGTABLE_LEVEL_PUD);
 }
 #endif
 
@@ -893,8 +949,12 @@ static void restore_exclusive_pte(struct vm_area_struct *vma,
 	if (pte_swp_soft_dirty(orig_pte))
 		pte = pte_mksoft_dirty(pte);
 
-	if (pte_swp_uffd_wp(orig_pte))
-		pte = pte_mkuffd_wp(pte);
+	if (pte_swp_uffd(orig_pte))
+		pte = pte_mkuffd(pte);
+
+	/* See do_swap_page(): restore PAGE_NONE for RWP */
+	if (pte_swp_uffd(orig_pte) && userfaultfd_rwp(vma))
+		pte = pte_modify(pte, PAGE_NONE);
 
 	if ((vma->vm_flags & VM_WRITE) &&
 	    can_change_pte_writable(vma, address, pte)) {
@@ -953,14 +1013,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 		if (swap_dup_entry_direct(entry) < 0)
 			return -EIO;
 
-		/* make sure dst_mm is on swapoff's mmlist. */
-		if (unlikely(list_empty(&dst_mm->mmlist))) {
-			spin_lock(&mmlist_lock);
-			if (list_empty(&dst_mm->mmlist))
-				list_add(&dst_mm->mmlist,
-						&src_mm->mmlist);
-			spin_unlock(&mmlist_lock);
-		}
+		mm_prepare_for_swap_entries(dst_mm);
 		/* Mark the swap entry as shared. */
 		if (pte_swp_exclusive(orig_pte)) {
 			pte = pte_swp_clear_exclusive(orig_pte);
@@ -984,8 +1037,8 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 			pte = softleaf_to_pte(entry);
 			if (pte_swp_soft_dirty(orig_pte))
 				pte = pte_swp_mksoft_dirty(pte);
-			if (pte_swp_uffd_wp(orig_pte))
-				pte = pte_swp_mkuffd_wp(pte);
+			if (pte_swp_uffd(orig_pte))
+				pte = pte_swp_mkuffd(pte);
 			set_pte_at(src_mm, addr, src_pte, pte);
 		}
 	} else if (softleaf_is_device_private(entry)) {
@@ -1018,8 +1071,8 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 			entry = make_readable_device_private_entry(
 							swp_offset(entry));
 			pte = swp_entry_to_pte(entry);
-			if (pte_swp_uffd_wp(orig_pte))
-				pte = pte_swp_mkuffd_wp(pte);
+			if (pte_swp_uffd(orig_pte))
+				pte = pte_swp_mkuffd(pte);
 			set_pte_at(src_mm, addr, src_pte, pte);
 		}
 	} else if (softleaf_is_device_exclusive(entry)) {
@@ -1041,8 +1094,8 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 				   make_pte_marker(marker));
 		return 0;
 	}
-	if (!userfaultfd_wp(dst_vma))
-		pte = pte_swp_clear_uffd_wp(pte);
+	if (!userfaultfd_protected(dst_vma))
+		pte = pte_swp_clear_uffd(pte);
 	set_pte_at(dst_mm, addr, dst_pte, pte);
 	return 0;
 }
@@ -1088,9 +1141,13 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma
 	/* All done, just insert the new page copy in the child */
 	pte = folio_mk_pte(new_folio, dst_vma->vm_page_prot);
 	pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma);
-	if (userfaultfd_pte_wp(dst_vma, ptep_get(src_pte)))
-		/* Uffd-wp needs to be delivered to dest pte as well */
-		pte = pte_mkuffd_wp(pte);
+	if (userfaultfd_protected(dst_vma) && pte_uffd(ptep_get(src_pte))) {
+		/* The uffd bit needs to be delivered to the dest pte as well */
+		pte = pte_mkuffd(pte);
+		/* Restore PAGE_NONE so the RWP marker keeps trapping */
+		if (userfaultfd_rwp(dst_vma))
+			pte = pte_modify(pte, PAGE_NONE);
+	}
 	set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
 	return 0;
 }
@@ -1100,9 +1157,31 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma,
 		pte_t pte, unsigned long addr, int nr)
 {
 	struct mm_struct *src_mm = src_vma->vm_mm;
+	bool writable;
+
+	/*
+	 * Snapshot writability before the RWP-disarm rewrite below: when the
+	 * child is not RWP-armed, pte_modify(pte, dst_vma->vm_page_prot) can
+	 * silently drop _PAGE_RW from a resolved (no-marker) writable PTE,
+	 * so a later pte_write(pte) check would skip the COW wrprotect and
+	 * leave the parent writable over a folio shared with the child.
+	 */
+	writable = pte_write(pte);
+
+	/*
+	 * Child is not RWP-armed: restore accessible protection so the
+	 * inherited PAGE_NONE does not cost a fault on first read. Gate on
+	 * pte_uffd(pte) so unrelated PAGE_NONE markers (e.g. NUMA balancing)
+	 * are not normalised away.
+	 */
+	if (!userfaultfd_protected(dst_vma)) {
+		if (userfaultfd_rwp(src_vma) && pte_uffd(pte))
+			pte = pte_modify(pte, dst_vma->vm_page_prot);
+		pte = pte_clear_uffd(pte);
+	}
 
 	/* If it's a COW mapping, write protect it both processes. */
-	if (is_cow_mapping(src_vma->vm_flags) && pte_write(pte)) {
+	if (is_cow_mapping(src_vma->vm_flags) && writable) {
 		wrprotect_ptes(src_mm, addr, src_pte, nr);
 		pte = pte_wrprotect(pte);
 	}
@@ -1112,9 +1191,6 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma,
 		pte = pte_mkclean(pte);
 	pte = pte_mkold(pte);
 
-	if (!userfaultfd_wp(dst_vma))
-		pte = pte_clear_uffd_wp(pte);
-
 	set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr);
 }
 
@@ -1797,7 +1873,7 @@ static inline int zap_nonpresent_ptes(struct mmu_gather *tlb,
 		pr_alert("unrecognized swap entry 0x%lx\n", entry.val);
 		WARN_ON_ONCE(1);
 	}
-	clear_not_present_full_ptes(vma->vm_mm, addr, pte, nr, tlb->fullmm);
+	clear_nonpresent_ptes(vma->vm_mm, addr, pte, nr);
 	*any_skipped = zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details, ptent);
 
 	return nr;
@@ -2643,7 +2719,7 @@ static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages,
 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
 				unsigned long num)
 {
-	return __vm_map_pages(vma, pages, num, vma->vm_pgoff);
+	return __vm_map_pages(vma, pages, num, vma_start_pgoff(vma));
 }
 EXPORT_SYMBOL(vm_map_pages);
 
@@ -3298,7 +3374,8 @@ int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long
 	unsigned long pfn;
 	int err;
 
-	err = __simple_ioremap_prep(vm_len, vma->vm_pgoff, start, len, &pfn);
+	err = __simple_ioremap_prep(vm_len, vma_start_pgoff(vma), start, len,
+				    &pfn);
 	if (err)
 		return err;
 
@@ -3925,8 +4002,8 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
 		if (unlikely(unshare)) {
 			if (pte_soft_dirty(vmf->orig_pte))
 				entry = pte_mksoft_dirty(entry);
-			if (pte_uffd_wp(vmf->orig_pte))
-				entry = pte_mkuffd_wp(entry);
+			if (pte_uffd(vmf->orig_pte))
+				entry = pte_mkuffd(entry);
 		} else {
 			entry = maybe_mkwrite(pte_mkdirty(entry), vma);
 		}
@@ -4181,6 +4258,9 @@ static bool __wp_can_reuse_large_anon_folio(struct folio *folio,
 static bool wp_can_reuse_anon_folio(struct folio *folio,
 				    struct vm_area_struct *vma)
 {
+	const bool maybe_in_lru_cache = !folio_test_lru(folio);
+	const bool in_swapcache = folio_test_swapcache(folio);
+
 	if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && folio_test_large(folio))
 		return __wp_can_reuse_large_anon_folio(folio, vma);
 
@@ -4191,15 +4271,16 @@ static bool wp_can_reuse_anon_folio(struct folio *folio,
 	 *
 	 * KSM doesn't necessarily raise the folio refcount.
 	 */
-	if (folio_test_ksm(folio) || folio_ref_count(folio) > 3)
+	if (folio_test_ksm(folio) ||
+	    folio_ref_count(folio) > 1 + maybe_in_lru_cache + in_swapcache)
 		return false;
-	if (!folio_test_lru(folio))
+	if (maybe_in_lru_cache)
 		/*
 		 * We cannot easily detect+handle references from
 		 * remote LRU caches or references to LRU folios.
 		 */
 		lru_add_drain();
-	if (folio_ref_count(folio) > 1 + folio_test_swapcache(folio))
+	if (folio_ref_count(folio) > 1 + in_swapcache)
 		return false;
 	if (!folio_trylock(folio))
 		return false;
@@ -4261,7 +4342,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf)
 			 * etc.) because we're only removing the uffd-wp bit,
 			 * which is completely invisible to the user.
 			 */
-			pte = pte_clear_uffd_wp(ptep_get(vmf->pte));
+			pte = pte_clear_uffd(ptep_get(vmf->pte));
 
 			set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
 			/*
@@ -4336,21 +4417,21 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf)
 	return wp_page_copy(vmf);
 }
 
-static inline void unmap_mapping_range_tree(struct rb_root_cached *root,
+static inline void unmap_mapping_range_tree(struct address_space *mapping,
 					    pgoff_t first_index,
 					    pgoff_t last_index,
 					    struct zap_details *details)
 {
 	struct vm_area_struct *vma;
-	unsigned long start, size;
 	struct mmu_gather tlb;
 
-	vma_interval_tree_foreach(vma, root, first_index, last_index) {
-		const pgoff_t start_idx = max(first_index, vma->vm_pgoff);
+	mapping_rmap_tree_foreach(vma, mapping, first_index, last_index) {
+		const pgoff_t start_idx = max(first_index, vma_start_pgoff(vma));
 		const pgoff_t end_idx = min(last_index, vma_last_pgoff(vma)) + 1;
-
-		start = vma->vm_start + ((start_idx - vma->vm_pgoff) << PAGE_SHIFT);
-		size = (end_idx - start_idx) << PAGE_SHIFT;
+		const pgoff_t offset = start_idx - vma_start_pgoff(vma);
+		const unsigned long offset_bytes = offset << PAGE_SHIFT;
+		const unsigned long start = vma->vm_start + offset_bytes;
+		const unsigned long size = (end_idx - start_idx) << PAGE_SHIFT;
 
 		tlb_gather_mmu(&tlb, vma->vm_mm);
 		zap_vma_range_batched(&tlb, vma, start, size, details);
@@ -4387,7 +4468,7 @@ void unmap_mapping_folio(struct folio *folio)
 
 	i_mmap_lock_read(mapping);
 	if (unlikely(mapping_mapped(mapping)))
-		unmap_mapping_range_tree(&mapping->i_mmap, first_index,
+		unmap_mapping_range_tree(mapping, first_index,
 					 last_index, &details);
 	i_mmap_unlock_read(mapping);
 }
@@ -4417,7 +4498,7 @@ void unmap_mapping_pages(struct address_space *mapping, pgoff_t start,
 
 	i_mmap_lock_read(mapping);
 	if (unlikely(mapping_mapped(mapping)))
-		unmap_mapping_range_tree(&mapping->i_mmap, first_index,
+		unmap_mapping_range_tree(mapping, first_index,
 					 last_index, &details);
 	i_mmap_unlock_read(mapping);
 }
@@ -4512,7 +4593,7 @@ static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf)
 static inline bool should_try_to_free_swap(struct swap_info_struct *si,
 					   struct folio *folio,
 					   struct vm_area_struct *vma,
-					   unsigned int extra_refs,
+					   bool exclusive,
 					   unsigned int fault_flags)
 {
 	if (!folio_test_swapcache(folio))
@@ -4528,14 +4609,12 @@ static inline bool should_try_to_free_swap(struct swap_info_struct *si,
 	if (mem_cgroup_swap_full(folio) || (vma->vm_flags & VM_LOCKED) ||
 	    folio_test_mlocked(folio))
 		return true;
+
 	/*
-	 * If we want to map a page that's in the swapcache writable, we
-	 * have to detect via the refcount if we're really the exclusive
-	 * user. Try freeing the swapcache to get rid of the swapcache
-	 * reference only in case it's likely that we'll be the exclusive user.
+	 * Free the swapcache only if we are the exclusive user and
+	 * this is a write fault.
 	 */
-	return (fault_flags & FAULT_FLAG_WRITE) && !folio_test_ksm(folio) &&
-		folio_ref_count(folio) == (extra_refs + folio_nr_pages(folio));
+	return (fault_flags & FAULT_FLAG_WRITE) && exclusive;
 }
 
 static vm_fault_t pte_marker_clear(struct vm_fault *vmf)
@@ -4752,6 +4831,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
 	struct swap_info_struct *si = NULL;
 	rmap_t rmap_flags = RMAP_NONE;
 	bool exclusive = false;
+	bool rwp_restore = false;
 	softleaf_t entry;
 	pte_t pte;
 	vm_fault_t ret = 0;
@@ -4897,16 +4977,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
 	} else if (folio != swapcache)
 		page = folio_page(folio, 0);
 
-	/*
-	 * If we want to map a page that's in the swapcache writable, we
-	 * have to detect via the refcount if we're really the exclusive
-	 * owner. Try removing the extra reference from the local LRU
-	 * caches if required.
-	 */
-	if ((vmf->flags & FAULT_FLAG_WRITE) &&
-	    !folio_test_ksm(folio) && !folio_test_lru(folio))
-		lru_add_drain();
-
 	folio_throttle_swaprate(folio, GFP_KERNEL);
 
 	/*
@@ -5038,24 +5108,34 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
 	pte = mk_pte(page, vma->vm_page_prot);
 	if (pte_swp_soft_dirty(vmf->orig_pte))
 		pte = pte_mksoft_dirty(pte);
-	if (pte_swp_uffd_wp(vmf->orig_pte))
-		pte = pte_mkuffd_wp(pte);
+	if (pte_swp_uffd(vmf->orig_pte))
+		pte = pte_mkuffd(pte);
 
 	/*
-	 * Same logic as in do_wp_page(); however, optimize for pages that are
-	 * certainly not shared either because we just allocated them without
-	 * exposing them to the swapcache or because the swap entry indicates
-	 * exclusivity.
+	 * A page reclaimed while RWP-protected carries the uffd bit on
+	 * its swap entry. Re-apply PAGE_NONE on swap-in so the first access
+	 * still traps as an RWP fault. pte_modify() preserves _PAGE_UFFD.
+	 */
+	if (pte_swp_uffd(vmf->orig_pte) && userfaultfd_rwp(vma)) {
+		pte = pte_modify(pte, PAGE_NONE);
+		rwp_restore = true;
+	}
+
+	/*
+	 * Similar logic as in do_wp_page(); however, optimize for pages that
+	 * are certainly exclusive.
+	 *
+	 * Skip the write upgrade for an RWP-restored pte: it must stay
+	 * PROT_NONE so the access retries through the RWP fault path
+	 * (do_uffd_rwp()) rather than being made writable here.
 	 */
-	if (!folio_test_ksm(folio) &&
-	    (exclusive || folio_ref_count(folio) == 1)) {
-		if ((vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) &&
+	if (exclusive) {
+		if (!rwp_restore &&
+		    (vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) &&
 		    !pte_needs_soft_dirty_wp(vma, pte)) {
 			pte = pte_mkwrite(pte, vma);
-			if (vmf->flags & FAULT_FLAG_WRITE) {
+			if (vmf->flags & FAULT_FLAG_WRITE)
 				pte = pte_mkdirty(pte);
-				vmf->flags &= ~FAULT_FLAG_WRITE;
-			}
 		}
 		rmap_flags |= RMAP_EXCLUSIVE;
 	}
@@ -5095,7 +5175,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
 	 * Do it after mapping, so raced page faults will likely see the folio
 	 * in swap cache and wait on the folio lock.
 	 */
-	if (should_try_to_free_swap(si, folio, vma, nr_pages, vmf->flags))
+	if (should_try_to_free_swap(si, folio, vma, exclusive, vmf->flags))
 		folio_free_swap(folio);
 
 	folio_unlock(folio);
@@ -5112,7 +5192,12 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
 		folio_put(swapcache);
 	}
 
-	if (vmf->flags & FAULT_FLAG_WRITE) {
+	/*
+	 * For an RWP-restored pte, leave it PROT_NONE and let the write
+	 * retry through the RWP fault path; do not COW it here, which would
+	 * drop the marker for a non-exclusive page.
+	 */
+	if ((vmf->flags & FAULT_FLAG_WRITE) && !pte_write(pte) && !rwp_restore) {
 		ret |= do_wp_page(vmf);
 		if (ret & VM_FAULT_ERROR)
 			ret &= VM_FAULT_ERROR;
@@ -5259,7 +5344,7 @@ void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte,
 	if (vma->vm_flags & VM_WRITE)
 		entry = pte_mkwrite(pte_mkdirty(entry), vma);
 	if (uffd_wp)
-		entry = pte_mkuffd_wp(entry);
+		entry = pte_mkuffd(entry);
 
 	folio_ref_add(folio, nr_pages - 1);
 	folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE);
@@ -5326,7 +5411,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
 			return handle_userfault(vmf, VM_UFFD_MISSING);
 		}
 		if (vmf_orig_pte_uffd_wp(vmf))
-			entry = pte_mkuffd_wp(entry);
+			entry = pte_mkuffd(entry);
 		set_pte_at(vma->vm_mm, addr, vmf->pte, entry);
 
 		/* No need to invalidate - it was non-present before */
@@ -5576,7 +5661,7 @@ void set_pte_range(struct vm_fault *vmf, struct folio *folio,
 	else if (pte_write(entry) && folio_test_dirty(folio))
 		entry = pte_mkdirty(entry);
 	if (unlikely(vmf_orig_pte_uffd_wp(vmf)))
-		entry = pte_mkuffd_wp(entry);
+		entry = pte_mkuffd(entry);
 	/* copy-on-write page */
 	if (write && !(vma->vm_flags & VM_SHARED)) {
 		VM_BUG_ON_FOLIO(nr != 1, folio);
@@ -5684,7 +5769,7 @@ vm_fault_t finish_fault(struct vm_fault *vmf)
 	} else if (nr_pages > 1) {
 		pgoff_t idx = folio_page_idx(folio, page);
 		/* The page offset of vmf->address within the VMA. */
-		pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff;
+		pgoff_t vma_off = vmf->pgoff - vma_start_pgoff(vmf->vma);
 		/* The index of the entry in the pagetable for fault page. */
 		pgoff_t pte_off = pte_index(vmf->address);
 
@@ -5796,7 +5881,7 @@ static vm_fault_t do_fault_around(struct vm_fault *vmf)
 	pgoff_t nr_pages = READ_ONCE(fault_around_pages);
 	pgoff_t pte_off = pte_index(vmf->address);
 	/* The page offset of vmf->address within the VMA. */
-	pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff;
+	pgoff_t vma_off = vmf->pgoff - vma_start_pgoff(vmf->vma);
 	pgoff_t from_pte, to_pte;
 	vm_fault_t ret;
 
@@ -6091,6 +6176,16 @@ static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_stru
 		if (!pte_present(ptent) || !pte_protnone(ptent))
 			continue;
 
+		/*
+		 * RWP-armed PTEs are also protnone but carry _PAGE_UFFD as a
+		 * marker. Leave them alone -- rewriting to vm_page_prot would
+		 * stop the RWP trap. Gate on userfaultfd_rwp(vma) too:
+		 * NUMA balancing preserves _PAGE_UFFD on UFFD_WP-marked PTEs
+		 * when applying PROT_NONE, and those still need rebuilding.
+		 */
+		if (userfaultfd_rwp(vma) && pte_uffd(ptent))
+			continue;
+
 		if (pfn_folio(pte_pfn(ptent)) != folio)
 			continue;
 
@@ -6106,6 +6201,35 @@ static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_stru
 	}
 }
 
+static vm_fault_t do_uffd_rwp(struct vm_fault *vmf)
+{
+	pte_t pte;
+
+	if (!userfaultfd_rwp_async(vmf->vma)) {
+		/* Sync mode: unmap PTE and deliver to userfaultfd handler */
+		pte_unmap(vmf->pte);
+		return handle_userfault(vmf, VM_UFFD_RWP);
+	}
+
+	spin_lock(vmf->ptl);
+	if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
+		pte_unmap_unlock(vmf->pte, vmf->ptl);
+		return 0;
+	}
+	pte = pte_modify(vmf->orig_pte, vmf->vma->vm_page_prot);
+	/* pte_modify() preserves _PAGE_UFFD; drop it on resolution */
+	pte = pte_clear_uffd(pte);
+	pte = pte_mkyoung(pte);
+	if (!pte_write(pte) &&
+	    vma_wants_manual_pte_write_upgrade(vmf->vma) &&
+	    can_change_pte_writable(vmf->vma, vmf->address, pte))
+		pte = pte_mkwrite(pte, vmf->vma);
+	set_pte_at(vmf->vma->vm_mm, vmf->address, vmf->pte, pte);
+	update_mmu_cache(vmf->vma, vmf->address, vmf->pte);
+	pte_unmap_unlock(vmf->pte, vmf->ptl);
+	return 0;
+}
+
 static vm_fault_t do_numa_page(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
@@ -6381,8 +6505,16 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
 	if (!pte_present(vmf->orig_pte))
 		return do_swap_page(vmf);
 
-	if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
+	if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma)) {
+		/*
+		 * RWP-protected PTEs are protnone plus the uffd bit. On a
+		 * VM_UFFD_RWP VMA, a protnone PTE without the uffd bit is
+		 * NUMA hinting and must still fall through to do_numa_page().
+		 */
+		if (userfaultfd_pte_rwp(vmf->vma, vmf->orig_pte))
+			return do_uffd_rwp(vmf);
 		return do_numa_page(vmf);
+	}
 
 	spin_lock(vmf->ptl);
 	entry = vmf->orig_pte;
@@ -6496,8 +6628,11 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
 		return 0;
 	}
 	if (pmd_trans_huge(vmf.orig_pmd)) {
-		if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma))
+		if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma)) {
+			if (userfaultfd_huge_pmd_rwp(vma, vmf.orig_pmd))
+				return do_huge_pmd_uffd_rwp(&vmf);
 			return do_huge_pmd_numa_page(&vmf);
+		}
 
 		if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
 		    !pmd_write(vmf.orig_pmd)) {
@@ -7274,7 +7409,7 @@ void print_vma_addr(char *prefix, unsigned long ip)
 	if (vma && vma->vm_file) {
 		struct file *f = vma->vm_file;
 		ip -= vma->vm_start;
-		ip += vma->vm_pgoff << PAGE_SHIFT;
+		ip += vma_start_pgoff(vma) << PAGE_SHIFT;
 		printk("%s%pD[%lx,%lx+%lx]", prefix, f, ip,
 				vma->vm_start,
 				vma->vm_end - vma->vm_start);
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index 7ac19fab22632..226ab9cb078ad 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -40,6 +40,8 @@
 #include <asm/tlbflush.h>
 
 #include "internal.h"
+#include "mm_init.h"
+#include "page_alloc.h"
 #include "shuffle.h"
 
 enum {
@@ -239,6 +241,7 @@ enum mmop mhp_get_default_online_type(void)
 
 	return mhp_default_online_type;
 }
+EXPORT_SYMBOL_GPL(mhp_get_default_online_type);
 
 void mhp_set_default_online_type(enum mmop online_type)
 {
@@ -1263,7 +1266,8 @@ static pg_data_t *hotadd_init_pgdat(int nid)
 	pgdat = NODE_DATA(nid);
 
 	/* init node's zones as empty zones, we don't have any present pages.*/
-	free_area_init_core_hotplug(pgdat);
+	if (free_area_init_core_hotplug(pgdat))
+		return NULL;
 
 	/*
 	 * The node we allocated has no zone fallback lists. For avoiding
@@ -1337,7 +1341,9 @@ static int check_hotplug_memory_range(u64 start, u64 size)
 
 static int online_memory_block(struct memory_block *mem, void *arg)
 {
-	mem->online_type = mhp_get_default_online_type();
+	enum mmop *online_type = arg;
+
+	mem->online_type = *online_type;
 	return device_online(&mem->dev);
 }
 
@@ -1491,7 +1497,8 @@ static int create_altmaps_and_memory_blocks(int nid, struct memory_group *group,
  *
  * we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG
  */
-int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags)
+static int __add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags,
+				 enum mmop online_type)
 {
 	struct mhp_params params = { .pgprot = pgprot_mhp(PAGE_KERNEL) };
 	enum memblock_flags memblock_flags = MEMBLOCK_NONE;
@@ -1581,8 +1588,9 @@ int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags)
 		merge_system_ram_resource(res);
 
 	/* online pages if requested */
-	if (mhp_get_default_online_type() != MMOP_OFFLINE)
-		walk_memory_blocks(start, size, NULL, online_memory_block);
+	if (online_type != MMOP_OFFLINE)
+		walk_memory_blocks(start, size, &online_type,
+				   online_memory_block);
 
 	return ret;
 error:
@@ -1598,7 +1606,13 @@ int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags)
 	return ret;
 }
 
-/* requires device_hotplug_lock, see add_memory_resource() */
+int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags)
+{
+	return __add_memory_resource(nid, res, mhp_flags,
+				     mhp_get_default_online_type());
+}
+
+/* requires device_hotplug_lock, see __add_memory_resource() */
 int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags)
 {
 	struct resource *res;
@@ -1626,7 +1640,15 @@ int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags)
 }
 EXPORT_SYMBOL_GPL(add_memory);
 
-/*
+/**
+ * __add_memory_driver_managed - add driver-managed memory with explicit online_type
+ * @nid: NUMA node ID where the memory will be added
+ * @start: Start physical address of the memory range
+ * @size: Size of the memory range in bytes
+ * @resource_name: Resource name in format "System RAM ($DRIVER)"
+ * @mhp_flags: Memory hotplug flags
+ * @online_type: Auto-Online behavior (offline, online, kernel, movable)
+ *
  * Add special, driver-managed memory to the system as system RAM. Such
  * memory is not exposed via the raw firmware-provided memmap as system
  * RAM, instead, it is detected and added by a driver - during cold boot,
@@ -1634,6 +1656,7 @@ EXPORT_SYMBOL_GPL(add_memory);
  *
  * Reasons why this memory should not be used for the initial memmap of a
  * kexec kernel or for placing kexec images:
+ *
  * - The booting kernel is in charge of determining how this memory will be
  *   used (e.g., use persistent memory as system RAM)
  * - Coordination with a hypervisor is required before this memory
@@ -1646,9 +1669,12 @@ EXPORT_SYMBOL_GPL(add_memory);
  *
  * The resource_name (visible via /proc/iomem) has to have the format
  * "System RAM ($DRIVER)".
+ *
+ * Return: 0 on success, negative error code on failure.
  */
-int add_memory_driver_managed(int nid, u64 start, u64 size,
-			      const char *resource_name, mhp_t mhp_flags)
+int __add_memory_driver_managed(int nid, u64 start, u64 size,
+		const char *resource_name, mhp_t mhp_flags,
+		enum mmop online_type)
 {
 	struct resource *res;
 	int rc;
@@ -1658,6 +1684,9 @@ int add_memory_driver_managed(int nid, u64 start, u64 size,
 	    resource_name[strlen(resource_name) - 1] != ')')
 		return -EINVAL;
 
+	if (online_type < MMOP_OFFLINE || online_type > MMOP_ONLINE_MOVABLE)
+		return -EINVAL;
+
 	lock_device_hotplug();
 
 	res = register_memory_resource(start, size, resource_name);
@@ -1666,7 +1695,7 @@ int add_memory_driver_managed(int nid, u64 start, u64 size,
 		goto out_unlock;
 	}
 
-	rc = add_memory_resource(nid, res, mhp_flags);
+	rc = __add_memory_resource(nid, res, mhp_flags, online_type);
 	if (rc < 0)
 		release_memory_resource(res);
 
@@ -1674,6 +1703,30 @@ int add_memory_driver_managed(int nid, u64 start, u64 size,
 	unlock_device_hotplug();
 	return rc;
 }
+EXPORT_SYMBOL_FOR_MODULES(__add_memory_driver_managed, "kmem");
+
+/**
+ * add_memory_driver_managed - add driver-managed memory
+ * @nid: NUMA node ID where the memory will be added
+ * @start: Start physical address of the memory range
+ * @size: Size of the memory range in bytes
+ * @resource_name: Resource name in format "System RAM ($DRIVER)"
+ * @mhp_flags: Memory hotplug flags
+ *
+ * Add driver-managed memory with the system default online type set by
+ * build config or kernel boot parameter.
+ *
+ * See __add_memory_driver_managed for more details.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int add_memory_driver_managed(int nid, u64 start, u64 size,
+			      const char *resource_name, mhp_t mhp_flags)
+{
+	return __add_memory_driver_managed(nid, start, size, resource_name,
+			mhp_flags,
+			mhp_get_default_online_type());
+}
 EXPORT_SYMBOL_GPL(add_memory_driver_managed);
 
 /*
@@ -2379,58 +2432,98 @@ static int try_reonline_memory_block(struct memory_block *mem, void *arg)
  */
 int offline_and_remove_memory(u64 start, u64 size)
 {
-	const unsigned long mb_count = size / memory_block_size_bytes();
+	struct range range = {
+		.start = start,
+		.end = start + size - 1,
+	};
+
+	return offline_and_remove_memory_ranges(&range, 1);
+}
+EXPORT_SYMBOL_GPL(offline_and_remove_memory);
+
+/**
+ * offline_and_remove_memory_ranges - offline and remove multiple memory ranges
+ * @ranges: array of physical address ranges to offline and remove
+ * @nr_ranges: number of entries in @ranges
+ *
+ * Offline and remove several memory ranges as one operation, serialized
+ * against other hotplug operations by a single lock_device_hotplug().
+ *
+ * This offlines all ranges before removing any of them.  If offlining any
+ * range fails, the entire process is reverted and nothing is removed.
+ * This provides a fully atomic semantic for unplugging an entire device.
+ *
+ * Each range must be memory-block aligned in start and size.
+ *
+ * Return: 0 on success, negative errno on failure (never positive).  On
+ * failure no range has been removed.
+ */
+int offline_and_remove_memory_ranges(const struct range *ranges,
+		unsigned int nr_ranges)
+{
+	unsigned long mb_count = 0;
 	uint8_t *online_types, *tmp;
-	int rc;
+	unsigned int i;
+	int rc = 0;
 
-	if (!IS_ALIGNED(start, memory_block_size_bytes()) ||
-	    !IS_ALIGNED(size, memory_block_size_bytes()) || !size)
+	if (!ranges || !nr_ranges)
 		return -EINVAL;
 
+	for (i = 0; i < nr_ranges; i++) {
+		const u64 start = ranges[i].start;
+		const u64 size = range_len(&ranges[i]);
+
+		if (!IS_ALIGNED(start, memory_block_size_bytes()) ||
+		    !IS_ALIGNED(size, memory_block_size_bytes()) || !size)
+			return -EINVAL;
+		mb_count += size / memory_block_size_bytes();
+	}
+
 	/*
-	 * We'll remember the old online type of each memory block, so we can
-	 * try to revert whatever we did when offlining one memory block fails
-	 * after offlining some others succeeded.
+	 * Remember the old online type of every memory block across all ranges,
+	 * so we can revert if offlining a later block fails.  All entries start
+	 * as MMOP_OFFLINE so blocks we never touched are skipped on rollback.
 	 */
 	online_types = kmalloc_array(mb_count, sizeof(*online_types),
 				     GFP_KERNEL);
 	if (!online_types)
 		return -ENOMEM;
-	/*
-	 * Initialize all states to MMOP_OFFLINE, so when we abort processing in
-	 * try_offline_memory_block(), we'll skip all unprocessed blocks in
-	 * try_reonline_memory_block().
-	 */
 	memset(online_types, MMOP_OFFLINE, mb_count);
 
 	lock_device_hotplug();
 
-	tmp = online_types;
-	rc = walk_memory_blocks(start, size, &tmp, try_offline_memory_block);
-
 	/*
-	 * In case we succeeded to offline all memory, remove it.
-	 * This cannot fail as it cannot get onlined in the meantime.
+	 * Phase 1: offline every block in every range.  An already-offline
+	 * block folds to success, so out-of-band offlining never blocks unplug.
 	 */
-	if (!rc) {
-		rc = try_remove_memory(start, size);
+	tmp = online_types;
+	for (i = 0; i < nr_ranges; i++) {
+		rc = walk_memory_blocks(ranges[i].start, range_len(&ranges[i]),
+					&tmp, try_offline_memory_block);
 		if (rc)
-			pr_err("%s: Failed to remove memory: %d", __func__, rc);
+			break;
 	}
 
-	/*
-	 * Rollback what we did. While memory onlining might theoretically fail
-	 * (nacked by a notifier), it barely ever happens.
-	 */
+	/* If any failure occurred at all, rollback any changes and bail */
 	if (rc) {
 		tmp = online_types;
-		walk_memory_blocks(start, size, &tmp,
-				   try_reonline_memory_block);
+		for (i = 0; i < nr_ranges; i++)
+			walk_memory_blocks(ranges[i].start,
+					   range_len(&ranges[i]), &tmp,
+					   try_reonline_memory_block);
+		goto out_unlock;
 	}
+
+	/* Phase 2: Remove. This should never fail holding the hotplug lock */
+	for (i = 0; i < nr_ranges; i++)
+		WARN_ON_ONCE(try_remove_memory(ranges[i].start,
+					       range_len(&ranges[i])));
+
+out_unlock:
 	unlock_device_hotplug();
 
 	kfree(online_types);
 	return rc;
 }
-EXPORT_SYMBOL_GPL(offline_and_remove_memory);
+EXPORT_SYMBOL_GPL(offline_and_remove_memory_ranges);
 #endif /* CONFIG_MEMORY_HOTREMOVE */
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 36699fabd3c22..5720f7f54d942 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -119,6 +119,7 @@
 #include <linux/memory.h>
 
 #include "internal.h"
+#include "page_alloc.h"
 
 /* Internal flags */
 #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)	/* Skip checks for continuous vmas */
@@ -653,12 +654,14 @@ static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk)
 {
 	struct folio *folio;
 	struct queue_pages *qp = walk->private;
+	pmd_t pmdval = pmdp_get(pmd);
 
-	if (unlikely(pmd_is_migration_entry(*pmd))) {
-		qp->nr_failed++;
+	if (unlikely(!pmd_present(pmdval))) {
+		if (pmd_is_migration_entry(pmdval))
+			qp->nr_failed++;
 		return;
 	}
-	folio = pmd_folio(*pmd);
+	folio = pmd_folio(pmdval);
 	if (is_huge_zero_folio(folio)) {
 		walk->action = ACTION_CONTINUE;
 		return;
@@ -2048,8 +2051,8 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma,
 		pol = get_task_policy(current);
 	if (pol->mode == MPOL_INTERLEAVE ||
 	    pol->mode == MPOL_WEIGHTED_INTERLEAVE) {
-		*ilx += vma->vm_pgoff >> order;
-		*ilx += (addr - vma->vm_start) >> (PAGE_SHIFT + order);
+		*ilx += vma_start_pgoff(vma) >> order;
+		*ilx += linear_page_delta(vma, addr) >> order;
 	}
 	return pol;
 }
@@ -2057,24 +2060,15 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma,
 bool vma_policy_mof(struct vm_area_struct *vma)
 {
 	struct mempolicy *pol;
+	pgoff_t ilx;
+	bool mof;
 
-	if (vma->vm_ops && vma->vm_ops->get_policy) {
-		bool ret = false;
-		pgoff_t ilx;		/* ignored here */
-
-		pol = vma->vm_ops->get_policy(vma, vma->vm_start, &ilx);
-		if (pol && (pol->flags & MPOL_F_MOF))
-			ret = true;
-		mpol_cond_put(pol);
-
-		return ret;
-	}
-
-	pol = vma->vm_policy;
+	pol = __get_vma_policy(vma, vma->vm_start, &ilx);
 	if (!pol)
 		pol = get_task_policy(current);
-
-	return pol->flags & MPOL_F_MOF;
+	mof = pol->flags & MPOL_F_MOF;
+	mpol_cond_put(pol);
+	return mof;
 }
 
 bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone)
@@ -2425,9 +2419,11 @@ static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order,
 	 */
 	preferred_gfp = gfp | __GFP_NOWARN;
 	preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
-	page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask);
+	page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask,
+					   ALLOC_DEFAULT);
 	if (!page)
-		page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL);
+		page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL,
+						   ALLOC_DEFAULT);
 
 	return page;
 }
@@ -2475,7 +2471,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order,
 			 */
 			page = __alloc_frozen_pages_noprof(
 				gfp | __GFP_THISNODE | __GFP_NORETRY, order,
-				nid, NULL);
+				nid, NULL, ALLOC_DEFAULT);
 			if (page || !(gfp & __GFP_DIRECT_RECLAIM))
 				return page;
 			/*
@@ -2487,7 +2483,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order,
 		}
 	}
 
-	page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask);
+	page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask, ALLOC_DEFAULT);
 
 	if (unlikely(pol->mode == MPOL_INTERLEAVE ||
 		     pol->mode == MPOL_WEIGHTED_INTERLEAVE) && page) {
@@ -3250,16 +3246,17 @@ EXPORT_SYMBOL_FOR_MODULES(mpol_shared_policy_init, "kvm");
 int mpol_set_shared_policy(struct shared_policy *sp,
 			struct vm_area_struct *vma, struct mempolicy *pol)
 {
-	int err;
+	const pgoff_t pgoff = vma_start_pgoff(vma);
+	const pgoff_t pgoff_end = vma_end_pgoff(vma);
 	struct sp_node *new = NULL;
-	unsigned long sz = vma_pages(vma);
+	int err;
 
 	if (pol) {
-		new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, pol);
+		new = sp_alloc(pgoff, pgoff_end, pol);
 		if (!new)
 			return -ENOMEM;
 	}
-	err = shared_policy_replace(sp, vma->vm_pgoff, vma->vm_pgoff + sz, new);
+	err = shared_policy_replace(sp, pgoff, pgoff_end, new);
 	if (err && new)
 		sp_free(new);
 	return err;
diff --git a/mm/mempool.c b/mm/mempool.c
index 473a029fa31fd..cb74e718b2c65 100644
--- a/mm/mempool.c
+++ b/mm/mempool.c
@@ -16,11 +16,28 @@
 #include <linux/export.h>
 #include <linux/mempool.h>
 #include <linux/writeback.h>
+#include <linux/static_key.h>
+#include <linux/init.h>
 #include "slab.h"
 
 static DECLARE_FAULT_ATTR(fail_mempool_alloc);
 static DECLARE_FAULT_ATTR(fail_mempool_alloc_bulk);
 
+/*
+ * Debugging support for mempool using static key.
+ *
+ * This allows enabling mempool debug at boot time via:
+ *   mempool_debug
+ */
+static DEFINE_STATIC_KEY_FALSE(mempool_debug_enabled);
+
+static int __init mempool_debug_setup(char *str)
+{
+	static_branch_enable(&mempool_debug_enabled);
+	return 1;
+}
+__setup("mempool_debug", mempool_debug_setup);
+
 static int __init mempool_faul_inject_init(void)
 {
 	int error;
@@ -37,7 +54,6 @@ static int __init mempool_faul_inject_init(void)
 }
 late_initcall(mempool_faul_inject_init);
 
-#ifdef CONFIG_SLUB_DEBUG_ON
 static void poison_error(struct mempool *pool, void *element, size_t size,
 			 size_t byte)
 {
@@ -140,14 +156,6 @@ static void poison_element(struct mempool *pool, void *element)
 #endif
 	}
 }
-#else /* CONFIG_SLUB_DEBUG_ON */
-static inline void check_element(struct mempool *pool, void *element)
-{
-}
-static inline void poison_element(struct mempool *pool, void *element)
-{
-}
-#endif /* CONFIG_SLUB_DEBUG_ON */
 
 static __always_inline bool kasan_poison_element(struct mempool *pool,
 		void *element)
@@ -175,7 +183,10 @@ static void kasan_unpoison_element(struct mempool *pool, void *element)
 static __always_inline void add_element(struct mempool *pool, void *element)
 {
 	BUG_ON(pool->min_nr != 0 && pool->curr_nr >= pool->min_nr);
-	poison_element(pool, element);
+
+	if (static_branch_unlikely(&mempool_debug_enabled))
+		poison_element(pool, element);
+
 	if (kasan_poison_element(pool, element))
 		pool->elements[pool->curr_nr++] = element;
 }
@@ -186,7 +197,9 @@ static void *remove_element(struct mempool *pool)
 
 	BUG_ON(pool->curr_nr < 0);
 	kasan_unpoison_element(pool, element);
-	check_element(pool, element);
+
+	if (static_branch_unlikely(&mempool_debug_enabled))
+		check_element(pool, element);
 	return element;
 }
 
diff --git a/mm/memremap.c b/mm/memremap.c
index 81766d8224009..accba23aef28c 100644
--- a/mm/memremap.c
+++ b/mm/memremap.c
@@ -425,6 +425,7 @@ void free_zone_device_folio(struct folio *folio)
 	mem_cgroup_uncharge(folio);
 
 	if (folio_test_anon(folio)) {
+		mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, -1);
 		for (i = 0; i < nr; i++)
 			__ClearPageAnonExclusive(folio_page(folio, i));
 	}
diff --git a/mm/migrate.c b/mm/migrate.c
index d9b23909d716c..222c8c15f782f 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -49,6 +49,7 @@
 #include <trace/events/migrate.h>
 
 #include "internal.h"
+#include "page_alloc.h"
 #include "swap.h"
 
 static const struct movable_operations *offline_movable_ops;
@@ -326,8 +327,12 @@ static bool try_to_map_unused_to_zeropage(struct page_vma_mapped_walk *pvmw,
 
 	if (pte_swp_soft_dirty(old_pte))
 		newpte = pte_mksoft_dirty(newpte);
-	if (pte_swp_uffd_wp(old_pte))
-		newpte = pte_mkuffd_wp(newpte);
+	if (pte_swp_uffd(old_pte))
+		newpte = pte_mkuffd(newpte);
+
+	/* See remove_migration_pte(): restore PAGE_NONE for RWP */
+	if (pte_swp_uffd(old_pte) && userfaultfd_rwp(pvmw->vma))
+		newpte = pte_modify(newpte, PAGE_NONE);
 
 	set_pte_at(pvmw->vma->vm_mm, pvmw->address, pvmw->pte, newpte);
 
@@ -362,7 +367,7 @@ static bool remove_migration_pte(struct folio *folio,
 			idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff;
 		new = folio_page(folio, idx);
 
-#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
 		/* PMD-mapped THP migration entry */
 		if (!pvmw.pte) {
 			VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) ||
@@ -371,7 +376,11 @@ static bool remove_migration_pte(struct folio *folio,
 			continue;
 		}
 #endif
-		old_pte = ptep_get(pvmw.pte);
+		if (folio_test_hugetlb(folio))
+			old_pte = huge_ptep_get(vma->vm_mm, pvmw.address,
+						pvmw.pte);
+		else
+			old_pte = ptep_get(pvmw.pte);
 		if (rmap_walk_arg->map_unused_to_zeropage &&
 		    try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx))
 			continue;
@@ -391,8 +400,12 @@ static bool remove_migration_pte(struct folio *folio,
 
 		if (softleaf_is_migration_write(entry))
 			pte = pte_mkwrite(pte, vma);
-		else if (pte_swp_uffd_wp(old_pte))
-			pte = pte_mkuffd_wp(pte);
+		else if (pte_swp_uffd(old_pte))
+			pte = pte_mkuffd(pte);
+
+		/* See do_swap_page(): restore PAGE_NONE for RWP */
+		if (pte_swp_uffd(old_pte) && userfaultfd_rwp(vma))
+			pte = pte_modify(pte, PAGE_NONE);
 
 		if (folio_test_anon(folio) && !softleaf_is_migration_read(entry))
 			rmap_flags |= RMAP_EXCLUSIVE;
@@ -407,8 +420,8 @@ static bool remove_migration_pte(struct folio *folio,
 			pte = softleaf_to_pte(entry);
 			if (pte_swp_soft_dirty(old_pte))
 				pte = pte_swp_mksoft_dirty(pte);
-			if (pte_swp_uffd_wp(old_pte))
-				pte = pte_swp_mkuffd_wp(pte);
+			if (pte_swp_uffd(old_pte))
+				pte = pte_swp_mkuffd(pte);
 		}
 
 #ifdef CONFIG_HUGETLB_PAGE
@@ -545,7 +558,7 @@ void migration_entry_wait_huge(struct vm_area_struct *vma, unsigned long addr, p
 }
 #endif
 
-#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
 void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd)
 {
 	spinlock_t *ptl;
@@ -590,7 +603,8 @@ static int __folio_migrate_mapping(struct address_space *mapping,
 		/* No turning back from here */
 		newfolio->index = folio->index;
 		newfolio->mapping = folio->mapping;
-		if (folio_test_anon(folio) && folio_test_large(folio))
+		if (folio_test_anon(folio) && folio_test_large(folio) &&
+		    !folio_test_hugetlb(folio))
 			mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1);
 		if (folio_test_swapbacked(folio))
 			__folio_set_swapbacked(newfolio);
@@ -623,7 +637,8 @@ static int __folio_migrate_mapping(struct address_space *mapping,
 	 */
 	newfolio->index = folio->index;
 	newfolio->mapping = folio->mapping;
-	if (folio_test_anon(folio) && folio_test_large(folio))
+	if (folio_test_anon(folio) && folio_test_large(folio) &&
+	    !folio_test_hugetlb(folio))
 		mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1);
 	folio_ref_add(newfolio, nr); /* add cache reference */
 	if (folio_test_swapbacked(folio))
@@ -884,7 +899,7 @@ static int __migrate_folio(struct address_space *mapping, struct folio *dst,
  * @mapping: The address_space containing the folio.
  * @dst: The folio to migrate the data to.
  * @src: The folio containing the current data.
- * @mode: How to migrate the page.
+ * @mode: How to migrate the folio.
  *
  * Common logic to directly migrate a single LRU folio suitable for
  * folios that do not have private data.
@@ -1130,7 +1145,7 @@ static int move_to_new_folio(struct folio *dst, struct folio *src,
 }
 
 /*
- * To record some information during migration, we use unused private
+ * To record some information during migration, we use the migrate_info
  * field of struct folio of the newly allocated destination folio.
  * This is safe because nobody is using it except us.
  */
@@ -1143,27 +1158,24 @@ enum {
 static void __migrate_folio_record(struct folio *dst,
 		int old_folio_state, struct anon_vma *anon_vma)
 {
-	dst->private = (void *)anon_vma + old_folio_state;
+	dst->migrate_info = (unsigned long)anon_vma | old_folio_state;
 }
 
 static void __migrate_folio_extract(struct folio *dst,
 		int *old_folio_state, struct anon_vma **anon_vmap)
 {
-	unsigned long private = (unsigned long)dst->private;
+	unsigned long info = dst->migrate_info;
 
-	*anon_vmap = (struct anon_vma *)(private & ~FOLIO_OLD_STATES);
-	*old_folio_state = private & FOLIO_OLD_STATES;
-	dst->private = NULL;
+	*anon_vmap = (struct anon_vma *)(info & ~FOLIO_OLD_STATES);
+	*old_folio_state = info & FOLIO_OLD_STATES;
+	dst->migrate_info = 0;
 }
 
 /* Restore the source folio to the original state upon failure */
-static void migrate_folio_undo_src(struct folio *src,
-				   int page_was_mapped,
-				   struct anon_vma *anon_vma,
-				   bool locked,
-				   struct list_head *ret)
+static void migrate_folio_undo_src(struct folio *src, int was_mapped,
+		struct anon_vma *anon_vma, bool locked, struct list_head *ret)
 {
-	if (page_was_mapped)
+	if (was_mapped)
 		remove_migration_ptes(src, src, 0);
 	/* Drop an anon_vma reference if we took one */
 	if (anon_vma)
@@ -1217,7 +1229,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
 		return -ENOMEM;
 	*dstp = dst;
 
-	dst->private = NULL;
+	dst->migrate_info = 0;
 
 	if (!folio_trylock(src)) {
 		if (mode == MIGRATE_ASYNC)
@@ -1449,7 +1461,8 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private,
 }
 
 /*
- * Counterpart of unmap_and_move_page() for hugepage migration.
+ * Counterpart of migrate_folio_unmap() and migrate_folio_move() for hugetlb
+ * folio migration.
  *
  * This function doesn't wait the completion of hugepage I/O
  * because there is no race between I/O and migration for hugepage.
@@ -1466,20 +1479,20 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private,
  * because then pte is replaced with migration swap entry and direct I/O code
  * will wait in the page fault for migration to complete.
  */
-static int unmap_and_move_huge_page(new_folio_t get_new_folio,
+static int unmap_and_move_hugetlb_folio(new_folio_t get_new_folio,
 		free_folio_t put_new_folio, unsigned long private,
 		struct folio *src, int force, enum migrate_mode mode,
-		int reason, struct list_head *ret)
+		enum migrate_reason reason, struct list_head *ret)
 {
 	struct folio *dst;
 	int rc = -EAGAIN;
-	int page_was_mapped = 0;
+	int was_mapped = 0;
 	struct anon_vma *anon_vma = NULL;
 	struct address_space *mapping = NULL;
 	enum ttu_flags ttu = 0;
 
 	if (folio_ref_count(src) == 1) {
-		/* page was freed from under us. So we are done. */
+		/* folio was freed from under us. So we are done. */
 		folio_putback_hugetlb(src);
 		return 0;
 	}
@@ -1501,8 +1514,8 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio,
 	}
 
 	/*
-	 * Check for pages which are in the process of being freed.  Without
-	 * folio_mapping() set, hugetlbfs specific move page routine will not
+	 * Check for folios which are in the process of being freed.  Without
+	 * folio_mapping() set, hugetlbfs specific move folio routine will not
 	 * be called and we could leak usage counts for subpools.
 	 */
 	if (hugetlb_folio_subpool(src) && !folio_mapping(src)) {
@@ -1532,13 +1545,13 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio,
 		}
 
 		try_to_migrate(src, ttu);
-		page_was_mapped = 1;
+		was_mapped = 1;
 	}
 
 	if (!folio_mapped(src))
 		rc = move_to_new_folio(dst, src, mode);
 
-	if (page_was_mapped)
+	if (was_mapped)
 		remove_migration_ptes(src, !rc ? dst : src, ttu);
 
 	if (ttu & TTU_RMAP_LOCKED)
@@ -1626,7 +1639,7 @@ struct migrate_pages_stats {
  */
 static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio,
 			    free_folio_t put_new_folio, unsigned long private,
-			    enum migrate_mode mode, int reason,
+			    enum migrate_mode mode, enum migrate_reason reason,
 			    struct migrate_pages_stats *stats,
 			    struct list_head *ret_folios)
 {
@@ -1663,10 +1676,10 @@ static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio,
 				continue;
 			}
 
-			rc = unmap_and_move_huge_page(get_new_folio,
-						      put_new_folio, private,
-						      folio, pass > 2, mode,
-						      reason, ret_folios);
+			rc = unmap_and_move_hugetlb_folio(get_new_folio,
+							  put_new_folio, private,
+							  folio, pass > 2, mode,
+							  reason, ret_folios);
 			/*
 			 * The rules are:
 			 *	0: hugetlb folio will be put back
@@ -1716,7 +1729,7 @@ static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio,
 static void migrate_folios_move(struct list_head *src_folios,
 		struct list_head *dst_folios,
 		free_folio_t put_new_folio, unsigned long private,
-		enum migrate_mode mode, int reason,
+		enum migrate_mode mode, enum migrate_reason reason,
 		struct list_head *ret_folios,
 		struct migrate_pages_stats *stats,
 		int *retry, int *thp_retry, int *nr_failed,
@@ -1741,7 +1754,7 @@ static void migrate_folios_move(struct list_head *src_folios,
 		/*
 		 * The rules are:
 		 *	0: folio will be freed
-		 *	-EAGAIN: stay on the unmap_folios list
+		 *	-EAGAIN: stay on the src_folios list
 		 *	Other errno: put on ret_folios list
 		 */
 		switch (rc) {
@@ -1799,7 +1812,7 @@ static void migrate_folios_undo(struct list_head *src_folios,
  */
 static int migrate_pages_batch(struct list_head *from,
 		new_folio_t get_new_folio, free_folio_t put_new_folio,
-		unsigned long private, enum migrate_mode mode, int reason,
+		unsigned long private, enum migrate_mode mode, enum migrate_reason reason,
 		struct list_head *ret_folios, struct list_head *split_folios,
 		struct migrate_pages_stats *stats, int nr_pass)
 {
@@ -2011,7 +2024,7 @@ static int migrate_pages_batch(struct list_head *from,
 
 static int migrate_pages_sync(struct list_head *from, new_folio_t get_new_folio,
 		free_folio_t put_new_folio, unsigned long private,
-		enum migrate_mode mode, int reason,
+		enum migrate_mode mode, enum migrate_reason reason,
 		struct list_head *ret_folios, struct list_head *split_folios,
 		struct migrate_pages_stats *stats)
 {
@@ -2088,7 +2101,7 @@ static int migrate_pages_sync(struct list_head *from, new_folio_t get_new_folio,
  */
 int migrate_pages(struct list_head *from, new_folio_t get_new_folio,
 		free_folio_t put_new_folio, unsigned long private,
-		enum migrate_mode mode, int reason, unsigned int *ret_succeeded)
+		enum migrate_mode mode, enum migrate_reason reason, unsigned int *ret_succeeded)
 {
 	int rc, rc_gather;
 	int nr_pages;
diff --git a/mm/migrate_device.c b/mm/migrate_device.c
index 554754eb26ff2..18d097c388530 100644
--- a/mm/migrate_device.c
+++ b/mm/migrate_device.c
@@ -77,6 +77,9 @@ static int migrate_vma_collect_hole(unsigned long start,
  * @folio: the folio to split
  * @fault_page: struct page associated with the fault if any
  *
+ * If @folio is not the folio containing @fault_page, the caller must hold a
+ * reference on @folio. The helper consumes that reference.
+ *
  * Returns 0 on success
  */
 static int migrate_vma_split_folio(struct folio *folio,
@@ -86,10 +89,8 @@ static int migrate_vma_split_folio(struct folio *folio,
 	struct folio *fault_folio = fault_page ? page_folio(fault_page) : NULL;
 	struct folio *new_fault_folio = NULL;
 
-	if (folio != fault_folio) {
-		folio_get(folio);
+	if (folio != fault_folio)
 		folio_lock(folio);
-	}
 
 	ret = split_folio(folio);
 	if (ret) {
@@ -166,11 +167,14 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start,
 	} else if (!pmd_present(*pmdp)) {
 		const softleaf_t entry = softleaf_from_pmd(*pmdp);
 
-		folio = softleaf_to_folio(entry);
-
 		if (!softleaf_is_device_private(entry) ||
-			!(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE) ||
-			(folio->pgmap->owner != migrate->pgmap_owner)) {
+		    !(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE)) {
+			spin_unlock(ptl);
+			return migrate_vma_collect_skip(start, end, walk);
+		}
+
+		folio = softleaf_to_folio(entry);
+		if (folio->pgmap->owner != migrate->pgmap_owner) {
 			spin_unlock(ptl);
 			return migrate_vma_collect_skip(start, end, walk);
 		}
@@ -307,6 +311,9 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
 			if (folio_test_large(folio)) {
 				int ret;
 
+				/* migrate_vma_split_folio() consumes this reference */
+				if (folio != fault_folio)
+					folio_get(folio);
 				lazy_mmu_mode_disable();
 				pte_unmap_unlock(ptep, ptl);
 				ret = migrate_vma_split_folio(folio,
@@ -350,6 +357,9 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
 			if (folio && folio_test_large(folio)) {
 				int ret;
 
+				/* migrate_vma_split_folio() consumes this reference */
+				if (folio != fault_folio)
+					folio_get(folio);
 				lazy_mmu_mode_disable();
 				pte_unmap_unlock(ptep, ptl);
 				ret = migrate_vma_split_folio(folio,
@@ -401,7 +411,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
 			bool anon_exclusive;
 			pte_t swp_pte;
 
-			flush_cache_page(vma, addr, pte_pfn(pte));
+			if (pte_present(pte))
+				flush_cache_page(vma, addr, pte_pfn(pte));
 			anon_exclusive = folio_test_anon(folio) &&
 					  PageAnonExclusive(page);
 			if (anon_exclusive) {
@@ -422,7 +433,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
 			migrate->cpages++;
 
 			/* Set the dirty flag on the folio now the pte is gone. */
-			if (pte_dirty(pte))
+			if (pte_present(pte) && pte_dirty(pte))
 				folio_mark_dirty(folio);
 
 			/* Setup special migration page table entry */
@@ -445,13 +456,13 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
 			if (pte_present(pte)) {
 				if (pte_soft_dirty(pte))
 					swp_pte = pte_swp_mksoft_dirty(swp_pte);
-				if (pte_uffd_wp(pte))
-					swp_pte = pte_swp_mkuffd_wp(swp_pte);
+				if (pte_uffd(pte))
+					swp_pte = pte_swp_mkuffd(swp_pte);
 			} else {
 				if (pte_swp_soft_dirty(pte))
 					swp_pte = pte_swp_mksoft_dirty(swp_pte);
-				if (pte_swp_uffd_wp(pte))
-					swp_pte = pte_swp_mkuffd_wp(swp_pte);
+				if (pte_swp_uffd(pte))
+					swp_pte = pte_swp_mkuffd(swp_pte);
 			}
 			set_pte_at(mm, addr, ptep, swp_pte);
 
@@ -513,7 +524,7 @@ static void migrate_vma_collect(struct migrate_vma *migrate)
 		migrate->pgmap_owner);
 	mmu_notifier_invalidate_range_start(&range);
 
-	walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,
+	walk_page_range_vma(migrate->vma, migrate->start, migrate->end,
 			&migrate_vma_walk_ops, migrate);
 
 	mmu_notifier_invalidate_range_end(&range);
@@ -768,7 +779,7 @@ int migrate_vma_setup(struct migrate_vma *args)
 }
 EXPORT_SYMBOL(migrate_vma_setup);
 
-#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
 /**
  * migrate_vma_insert_huge_pmd_page: Insert a huge folio into @migrate->vma->vm_mm
  * at @addr. folio is already allocated as a part of the migration process with
@@ -835,7 +846,7 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate,
 		else
 			swp_entry = make_readable_device_private_entry(
 						page_to_pfn(page));
-		entry = swp_entry_to_pmd(swp_entry);
+		entry = softleaf_to_pmd(swp_entry);
 	} else {
 		if (folio_is_zone_device(folio) &&
 		    !folio_is_device_coherent(folio)) {
@@ -923,7 +934,7 @@ static int migrate_vma_split_unmapped_folio(struct migrate_vma *migrate,
 		migrate->src[i+idx] = migrate_pfn(pfn + i) | flags;
 	return ret;
 }
-#else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */
+#else /* !CONFIG_ARCH_HAS_PMD_SOFTLEAVES */
 static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate,
 					 unsigned long addr,
 					 struct page *page,
@@ -944,7 +955,7 @@ static int migrate_vma_split_unmapped_folio(struct migrate_vma *migrate,
 static unsigned long migrate_vma_nr_pages(unsigned long *src)
 {
 	unsigned long nr = 1;
-#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
 	if (*src & MIGRATE_PFN_COMPOUND)
 		nr = HPAGE_PMD_NR;
 #else
diff --git a/mm/mincore.c b/mm/mincore.c
index c8757c5085bf9..53b9828037713 100644
--- a/mm/mincore.c
+++ b/mm/mincore.c
@@ -258,7 +258,16 @@ static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *v
 		memset(vec, 1, pages);
 		return pages;
 	}
-	err = walk_page_range(vma->vm_mm, addr, end, &mincore_walk_ops, vec);
+
+	/*
+	 * mincore historically reports PFNMAP mappings as non-resident.
+	 */
+	if (vma->vm_flags & VM_PFNMAP) {
+		__mincore_unmapped_range(addr, end, vma, vec);
+		return (end - addr) >> PAGE_SHIFT;
+	}
+
+	err = walk_page_range_vma(vma, addr, end, &mincore_walk_ops, vec);
 	if (err < 0)
 		return err;
 	return (end - addr) >> PAGE_SHIFT;
diff --git a/mm/mlock.c b/mm/mlock.c
index 8c227fefa2df0..efa6716e4dfbd 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -329,7 +329,7 @@ static inline bool allow_mlock_munlock(struct folio *folio,
 	 * be split. And the pages are not in VM_LOCKed VMA
 	 * can be reclaimed.
 	 */
-	if (!(vma->vm_flags & VM_LOCKED))
+	if (!vma_test(vma, VMA_LOCKED_BIT))
 		return true;
 
 	/* folio_within_range() cannot take KSM, but any small folio is OK */
@@ -368,7 +368,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
 		folio = pmd_folio(*pmd);
 		if (folio_is_zone_device(folio))
 			goto out;
-		if (vma->vm_flags & VM_LOCKED)
+		if (vma_test(vma, VMA_LOCKED_BIT))
 			mlock_folio(folio);
 		else
 			munlock_folio(folio);
@@ -393,7 +393,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
 		if (!allow_mlock_munlock(folio, vma, start, end, step))
 			goto next_entry;
 
-		if (vma->vm_flags & VM_LOCKED)
+		if (vma_test(vma, VMA_LOCKED_BIT))
 			mlock_folio(folio);
 		else
 			munlock_folio(folio);
@@ -417,8 +417,8 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
  * @end - end of range in @vma
  * @new_vma_flags - the new set of flags for @vma.
  *
- * Called for mlock(), mlock2() and mlockall(), to set @vma VM_LOCKED;
- * called for munlock() and munlockall(), to clear VM_LOCKED from @vma.
+ * Called for mlock(), mlock2() and mlockall(), to set @vma VMA_LOCKED_BIT;
+ * called for munlock() and munlockall(), to clear VMA_LOCKED_BIT from @vma.
  */
 static void mlock_vma_pages_range(struct vm_area_struct *vma,
 	unsigned long start, unsigned long end,
@@ -431,14 +431,14 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma,
 
 	/*
 	 * There is a slight chance that concurrent page migration,
-	 * or page reclaim finding a page of this now-VM_LOCKED vma,
+	 * or page reclaim finding a page of this now-VMA_LOCKED_BIT vma,
 	 * will call mlock_vma_folio() and raise page's mlock_count:
 	 * double counting, leaving the page unevictable indefinitely.
-	 * Communicate this danger to mlock_vma_folio() with VM_IO,
-	 * which is a VM_SPECIAL flag not allowed on VM_LOCKED vmas.
+	 * Communicate this danger to mlock_vma_folio() with VMA_IO_BIT,
+	 * which is a VMA_SPECIAL_FLAGS flag not allowed on VMA_LOCKED_BIT vmas.
 	 * mmap_lock is held in write mode here, so this weird
 	 * combination should not be visible to other mmap_lock users;
-	 * but WRITE_ONCE so rmap walkers must see VM_IO if VM_LOCKED.
+	 * but WRITE_ONCE so rmap walkers must see VMA_IO_BIT if VMA_LOCKED_BIT.
 	 */
 	if (vma_flags_test(new_vma_flags, VMA_LOCKED_BIT))
 		vma_flags_set(new_vma_flags, VMA_IO_BIT);
@@ -446,7 +446,7 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma,
 	vma_flags_reset_once(vma, new_vma_flags);
 
 	lru_add_drain();
-	walk_page_range(vma->vm_mm, start, end, &mlock_walk_ops, NULL);
+	walk_page_range_vma(vma, start, end, &mlock_walk_ops, NULL);
 	lru_add_drain();
 
 	if (vma_flags_test(new_vma_flags, VMA_IO_BIT)) {
@@ -458,7 +458,7 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma,
 /*
  * mlock_fixup  - handle mlock[all]/munlock[all] requests.
  *
- * Filters out "special" vmas -- VM_LOCKED never gets set for these, and
+ * Filters out "special" vmas -- VMA_LOCKED_BIT never gets set for these, and
  * munlock is a no-op.  However, for some special vmas, we go ahead and
  * populate the ptes.
  *
@@ -466,24 +466,23 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma,
  */
 static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma,
 	       struct vm_area_struct **prev, unsigned long start,
-	       unsigned long end, vm_flags_t newflags)
+	       unsigned long end, vma_flags_t *new_vma_flags)
 {
-	vma_flags_t new_vma_flags = legacy_to_vma_flags(newflags);
 	const vma_flags_t old_vma_flags = vma->flags;
 	struct mm_struct *mm = vma->vm_mm;
 	int nr_pages;
 	int ret = 0;
 
-	if (vma_flags_same_pair(&old_vma_flags, &new_vma_flags) ||
+	if (vma_flags_same_pair(&old_vma_flags, new_vma_flags) ||
 	    vma_is_secretmem(vma) || !vma_supports_mlock(vma)) {
 		/*
-		 * Don't set VM_LOCKED or VM_LOCKONFAULT and don't count.
-		 * For secretmem, don't allow the memory to be unlocked.
+		 * Don't set VMA_LOCKED_BIT or VMA_LOCKONFAULT_BIT and don't
+		 * count.  For secretmem, don't allow the memory to be unlocked.
 		 */
 		goto out;
 	}
 
-	vma = vma_modify_flags(vmi, *prev, vma, start, end, &new_vma_flags);
+	vma = vma_modify_flags(vmi, *prev, vma, start, end, new_vma_flags);
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
 		goto out;
@@ -493,7 +492,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma,
 	 * Keep track of amount of locked VM.
 	 */
 	nr_pages = (end - start) >> PAGE_SHIFT;
-	if (!vma_flags_test(&new_vma_flags, VMA_LOCKED_BIT))
+	if (!vma_flags_test(new_vma_flags, VMA_LOCKED_BIT))
 		nr_pages = -nr_pages;
 	else if (vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT))
 		nr_pages = 0;
@@ -502,15 +501,15 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma,
 	/*
 	 * vm_flags is protected by the mmap_lock held in write mode.
 	 * It's okay if try_to_unmap_one unmaps a page just after we
-	 * set VM_LOCKED, populate_vma_page_range will bring it back.
+	 * set VMA_LOCKED_BIT, populate_vma_page_range will bring it back.
 	 */
-	if (vma_flags_test(&new_vma_flags, VMA_LOCKED_BIT) &&
+	if (vma_flags_test(new_vma_flags, VMA_LOCKED_BIT) &&
 	    vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT)) {
 		/* No work to do, and mlocking twice would be wrong */
 		vma_start_write(vma);
-		vma->flags = new_vma_flags;
+		vma->flags = *new_vma_flags;
 	} else {
-		mlock_vma_pages_range(vma, start, end, &new_vma_flags);
+		mlock_vma_pages_range(vma, start, end, new_vma_flags);
 	}
 out:
 	*prev = vma;
@@ -518,7 +517,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma,
 }
 
 static int apply_vma_lock_flags(unsigned long start, size_t len,
-				vm_flags_t flags)
+				const vma_flags_t *flags)
 {
 	unsigned long nstart, end, tmp;
 	struct vm_area_struct *vma, *prev;
@@ -543,18 +542,20 @@ static int apply_vma_lock_flags(unsigned long start, size_t len,
 	tmp = vma->vm_start;
 	for_each_vma_range(vmi, vma, end) {
 		int error;
-		vm_flags_t newflags;
+		vma_flags_t newflags;
 
 		if (vma->vm_start != tmp)
 			return -ENOMEM;
 
-		newflags = vma->vm_flags & ~VM_LOCKED_MASK;
-		newflags |= flags;
+		newflags = vma->flags;
+		vma_flags_clear_mask(&newflags, VMA_LOCKED_MASK);
+		vma_flags_set_mask(&newflags, *flags);
+
 		/* Here we know that  vma->vm_start <= nstart < vma->vm_end. */
 		tmp = vma->vm_end;
 		if (tmp > end)
 			tmp = end;
-		error = mlock_fixup(&vmi, vma, &prev, nstart, tmp, newflags);
+		error = mlock_fixup(&vmi, vma, &prev, nstart, tmp, &newflags);
 		if (error)
 			return error;
 		tmp = vma_iter_end(&vmi);
@@ -589,7 +590,7 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm,
 		end = start + len;
 
 	for_each_vma_range(vmi, vma, end) {
-		if (vma->vm_flags & VM_LOCKED) {
+		if (vma_test(vma, VMA_LOCKED_BIT)) {
 			if (start > vma->vm_start)
 				count -= (start - vma->vm_start);
 			if (end < vma->vm_end) {
@@ -615,7 +616,8 @@ static int __mlock_posix_error_return(long retval)
 	return retval;
 }
 
-static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t flags)
+static __must_check int do_mlock(unsigned long start, size_t len,
+				 vma_flags_t *flags)
 {
 	unsigned long locked;
 	unsigned long lock_limit;
@@ -664,24 +666,27 @@ static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t fla
 
 SYSCALL_DEFINE2(mlock, unsigned long, start, size_t, len)
 {
-	return do_mlock(start, len, VM_LOCKED);
+	vma_flags_t flags = mk_vma_flags(VMA_LOCKED_BIT);
+
+	return do_mlock(start, len, &flags);
 }
 
 SYSCALL_DEFINE3(mlock2, unsigned long, start, size_t, len, int, flags)
 {
-	vm_flags_t vm_flags = VM_LOCKED;
+	vma_flags_t vma_flags = mk_vma_flags(VMA_LOCKED_BIT);
 
 	if (flags & ~MLOCK_ONFAULT)
 		return -EINVAL;
 
 	if (flags & MLOCK_ONFAULT)
-		vm_flags |= VM_LOCKONFAULT;
+		vma_flags_set(&vma_flags, VMA_LOCKONFAULT_BIT);
 
-	return do_mlock(start, len, vm_flags);
+	return do_mlock(start, len, &vma_flags);
 }
 
 SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len)
 {
+	vma_flags_t flags = EMPTY_VMA_FLAGS;
 	int ret;
 
 	start = untagged_addr(start);
@@ -691,7 +696,7 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len)
 
 	if (mmap_write_lock_killable(current->mm))
 		return -EINTR;
-	ret = apply_vma_lock_flags(start, len, 0);
+	ret = apply_vma_lock_flags(start, len, &flags);
 	mmap_write_unlock(current->mm);
 
 	return ret;
@@ -699,46 +704,48 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len)
 
 /*
  * Take the MCL_* flags passed into mlockall (or 0 if called from munlockall)
- * and translate into the appropriate modifications to mm->def_flags and/or the
- * flags for all current VMAs.
+ * and translate into the appropriate modifications to mm->def_vma_flags and/or
+ * the flags for all current VMAs.
  *
  * There are a couple of subtleties with this.  If mlockall() is called multiple
  * times with different flags, the values do not necessarily stack.  If mlockall
  * is called once including the MCL_FUTURE flag and then a second time without
- * it, VM_LOCKED and VM_LOCKONFAULT will be cleared from mm->def_flags.
+ * it, VMA_LOCKED_BIT and VMA_LOCKONFAULT_BIT will be cleared from
+ * mm->def_vma_flags.
  */
 static int apply_mlockall_flags(int flags)
 {
 	VMA_ITERATOR(vmi, current->mm, 0);
+	struct mm_struct *mm = current->mm;
 	struct vm_area_struct *vma, *prev = NULL;
-	vm_flags_t to_add = 0;
+	vma_flags_t to_add = EMPTY_VMA_FLAGS;
 
-	current->mm->def_flags &= ~VM_LOCKED_MASK;
+	vma_flags_clear_mask(&mm->def_vma_flags, VMA_LOCKED_MASK);
 	if (flags & MCL_FUTURE) {
-		current->mm->def_flags |= VM_LOCKED;
+		vma_flags_set(&mm->def_vma_flags, VMA_LOCKED_BIT);
 
 		if (flags & MCL_ONFAULT)
-			current->mm->def_flags |= VM_LOCKONFAULT;
+			vma_flags_set(&mm->def_vma_flags, VMA_LOCKONFAULT_BIT);
 
 		if (!(flags & MCL_CURRENT))
 			goto out;
 	}
 
 	if (flags & MCL_CURRENT) {
-		to_add |= VM_LOCKED;
+		vma_flags_set(&to_add, VMA_LOCKED_BIT);
 		if (flags & MCL_ONFAULT)
-			to_add |= VM_LOCKONFAULT;
+			vma_flags_set(&to_add, VMA_LOCKONFAULT_BIT);
 	}
 
 	for_each_vma(vmi, vma) {
 		int error;
-		vm_flags_t newflags;
+		vma_flags_t newflags = vma->flags;
 
-		newflags = vma->vm_flags & ~VM_LOCKED_MASK;
-		newflags |= to_add;
+		vma_flags_clear_mask(&newflags, VMA_LOCKED_MASK);
+		vma_flags_set_mask(&newflags, to_add);
 
 		error = mlock_fixup(&vmi, vma, &prev, vma->vm_start, vma->vm_end,
-				    newflags);
+				    &newflags);
 		/* Ignore errors, but prev needs fixing up. */
 		if (error)
 			prev = vma;
diff --git a/mm/mm_init.c b/mm/mm_init.c
index 0f64909e8d20e..ddda9d6837f32 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -33,8 +33,12 @@
 #include <linux/kexec_handover.h>
 #include <linux/hugetlb.h>
 #include "internal.h"
+#include "mm_init.h"
+#include "page_alloc.h"
+#include "sparse.h"
 #include "slab.h"
 #include "shuffle.h"
+#include "vmalloc.h"
 
 #include <asm/setup.h>
 
@@ -332,8 +336,7 @@ static unsigned long __init early_calculate_totalpages(void)
 		unsigned long pages = end_pfn - start_pfn;
 
 		totalpages += pages;
-		if (pages)
-			node_set_state(nid, N_MEMORY);
+		node_set_state(nid, N_MEMORY);
 	}
 	return totalpages;
 }
@@ -674,45 +677,20 @@ static inline void fixup_hashdist(void)
 static inline void fixup_hashdist(void) {}
 #endif /* CONFIG_NUMA */
 
-#ifdef CONFIG_ZONE_DEVICE
+#if defined(CONFIG_ZONE_DEVICE) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT)
 static __meminit void pageblock_migratetype_init_range(unsigned long pfn,
-		unsigned long nr_pages, int migratetype)
+		unsigned long nr_pages, int migratetype, bool atomic)
 {
 	const unsigned long end = pfn + nr_pages;
 
 	for (pfn = pageblock_align(pfn); pfn < end; pfn += pageblock_nr_pages) {
 		init_pageblock_migratetype(pfn_to_page(pfn), migratetype, false);
-		if (IS_ALIGNED(pfn, PAGES_PER_SECTION))
+		if (!atomic && IS_ALIGNED(pfn, PAGES_PER_SECTION))
 			cond_resched();
 	}
 }
 #endif
 
-/*
- * Initialize a reserved page unconditionally, finding its zone first.
- */
-void __meminit __init_page_from_nid(unsigned long pfn, int nid)
-{
-	pg_data_t *pgdat;
-	int zid;
-
-	pgdat = NODE_DATA(nid);
-
-	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
-		struct zone *zone = &pgdat->node_zones[zid];
-
-		if (zone_spans_pfn(zone, pfn))
-			break;
-	}
-	__init_single_page(pfn_to_page(pfn), pfn, zid, nid);
-
-	if (pageblock_aligned(pfn)) {
-		enum migratetype mt =
-			kho_scratch_migratetype(pfn, MIGRATE_MOVABLE);
-		init_pageblock_migratetype(pfn_to_page(pfn), mt, false);
-	}
-}
-
 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
 static inline void pgdat_set_deferred_range(pg_data_t *pgdat)
 {
@@ -771,10 +749,25 @@ defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
 
 static void __meminit __init_deferred_page(unsigned long pfn, int nid)
 {
+	pg_data_t *pgdat = NODE_DATA(nid);
+	int zid;
+
 	if (early_page_initialised(pfn, nid))
 		return;
 
-	__init_page_from_nid(pfn, nid);
+	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
+		struct zone *zone = &pgdat->node_zones[zid];
+
+		if (zone_spans_pfn(zone, pfn))
+			break;
+	}
+	__init_single_page(pfn_to_page(pfn), pfn, zid, nid);
+
+	if (pageblock_aligned(pfn)) {
+		enum migratetype mt =
+			kho_scratch_migratetype(pfn, MIGRATE_MOVABLE);
+		init_pageblock_migratetype(pfn_to_page(pfn), mt, false);
+	}
 }
 #else
 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
@@ -799,28 +792,6 @@ void __meminit init_deferred_page(unsigned long pfn, int nid)
 	__init_deferred_page(pfn, nid);
 }
 
-/* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */
-static bool __meminit
-overlap_memmap_init(unsigned long zone, unsigned long *pfn)
-{
-	static struct memblock_region *r __meminitdata;
-
-	if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
-		if (!r || *pfn >= memblock_region_memory_end_pfn(r)) {
-			for_each_mem_region(r) {
-				if (*pfn < memblock_region_memory_end_pfn(r))
-					break;
-			}
-		}
-		if (*pfn >= memblock_region_memory_base_pfn(r) &&
-		    memblock_is_mirror(r)) {
-			*pfn = memblock_region_memory_end_pfn(r);
-			return true;
-		}
-	}
-	return false;
-}
-
 /*
  * Only struct pages that correspond to ranges defined by memblock.memory
  * are zeroed and initialized by going through __init_single_page() during
@@ -907,8 +878,6 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
 		 * function.  They do not exist on hotplugged memory.
 		 */
 		if (context == MEMINIT_EARLY) {
-			if (overlap_memmap_init(zone, &pfn))
-				continue;
 			if (defer_init(nid, pfn, zone_end_pfn)) {
 				deferred_struct_pages = true;
 				break;
@@ -1142,7 +1111,7 @@ void __ref memmap_init_zone_device(struct zone *zone,
 				     compound_nr_pages(pfn, altmap, pgmap));
 	}
 
-	pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE);
+	pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE, false);
 
 	pr_debug("%s initialised %lu pages in %ums\n", __func__,
 		nr_pages, jiffies_to_msecs(jiffies - start));
@@ -1174,9 +1143,8 @@ static void __init adjust_zone_range_for_zone_movable(int nid,
 				arch_zone_highest_possible_pfn[movable_zone]);
 
 		/* Adjust for ZONE_MOVABLE starting within this range */
-		} else if (!mirrored_kernelcore &&
-			*zone_start_pfn < zone_movable_pfn[nid] &&
-			*zone_end_pfn > zone_movable_pfn[nid]) {
+		} else if (*zone_start_pfn < zone_movable_pfn[nid] &&
+			   *zone_end_pfn > zone_movable_pfn[nid]) {
 			*zone_end_pfn = zone_movable_pfn[nid];
 
 		/* Check if this whole range is within ZONE_MOVABLE */
@@ -1224,40 +1192,11 @@ static unsigned long __init zone_absent_pages_in_node(int nid,
 					unsigned long zone_start_pfn,
 					unsigned long zone_end_pfn)
 {
-	unsigned long nr_absent;
-
 	/* zone is empty, we don't have any absent pages */
 	if (zone_start_pfn == zone_end_pfn)
 		return 0;
 
-	nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
-
-	/*
-	 * ZONE_MOVABLE handling.
-	 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages
-	 * and vice versa.
-	 */
-	if (mirrored_kernelcore && zone_movable_pfn[nid]) {
-		unsigned long start_pfn, end_pfn;
-		struct memblock_region *r;
-
-		for_each_mem_region(r) {
-			start_pfn = clamp(memblock_region_memory_base_pfn(r),
-					  zone_start_pfn, zone_end_pfn);
-			end_pfn = clamp(memblock_region_memory_end_pfn(r),
-					zone_start_pfn, zone_end_pfn);
-
-			if (zone_type == ZONE_MOVABLE &&
-			    memblock_is_mirror(r))
-				nr_absent += end_pfn - start_pfn;
-
-			if (zone_type == ZONE_NORMAL &&
-			    !memblock_is_mirror(r))
-				nr_absent += end_pfn - start_pfn;
-		}
-	}
-
-	return nr_absent;
+	return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
 }
 
 /*
@@ -1338,7 +1277,7 @@ static void __init calculate_node_totalpages(struct pglist_data *pgdat,
 						unsigned long node_start_pfn,
 						unsigned long node_end_pfn)
 {
-	unsigned long realtotalpages = 0, totalpages = 0;
+	unsigned long realtotalpages = 0;
 	enum zone_type i;
 
 	for (i = 0; i < MAX_NR_ZONES; i++) {
@@ -1368,11 +1307,10 @@ static void __init calculate_node_totalpages(struct pglist_data *pgdat,
 		zone->present_early_pages = real_size;
 #endif
 
-		totalpages += spanned;
 		realtotalpages += real_size;
 	}
 
-	pgdat->node_spanned_pages = totalpages;
+	pgdat->node_spanned_pages = node_end_pfn - node_start_pfn;
 	pgdat->node_present_pages = realtotalpages;
 	pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages);
 }
@@ -1496,7 +1434,7 @@ static inline void setup_usemap(struct zone *zone) {}
 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
 
 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
-void __init set_pageblock_order(void)
+static void __init set_pageblock_order(void)
 {
 	unsigned int order = PAGE_BLOCK_MAX_ORDER;
 
@@ -1522,7 +1460,7 @@ void __init set_pageblock_order(void)
  * include/linux/pageblock-flags.h for the values of pageblock_order based on
  * the kernel config
  */
-void __init set_pageblock_order(void)
+static inline void __init set_pageblock_order(void)
 {
 }
 
@@ -1536,16 +1474,22 @@ void __init set_pageblock_order(void)
  * NOTE: this function is only called during memory hotplug
  */
 #ifdef CONFIG_MEMORY_HOTPLUG
-void __ref free_area_init_core_hotplug(struct pglist_data *pgdat)
+int __ref free_area_init_core_hotplug(struct pglist_data *pgdat)
 {
 	int nid = pgdat->node_id;
 	enum zone_type z;
-	int cpu;
+	int cpu, i;
 
 	pgdat_init_internals(pgdat);
 
-	if (pgdat->per_cpu_nodestats == &boot_nodestats)
-		pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat);
+	if (pgdat->per_cpu_nodestats == &boot_nodestats) {
+		struct per_cpu_nodestat __percpu *p;
+
+		p = alloc_percpu(struct per_cpu_nodestat);
+		if (!p)
+			return -ENOMEM;
+		pgdat->per_cpu_nodestats = p;
+	}
 
 	/*
 	 * Reset the nr_zones, order and highest_zoneidx before reuse.
@@ -1558,10 +1502,17 @@ void __ref free_area_init_core_hotplug(struct pglist_data *pgdat)
 	pgdat->node_start_pfn = 0;
 	pgdat->node_present_pages = 0;
 
-	for_each_online_cpu(cpu) {
-		struct per_cpu_nodestat *p;
+	/*
+	 * Hot-unplug can leave per-cpu vmstat deltas unfolded (folders skip
+	 * offline nodes) - reconcile this at online. Foreign access to counters
+	 * is safe: the node is not online yet and we hold the hotplug lock.
+	 */
+	for_each_possible_cpu(cpu) {
+		struct per_cpu_nodestat *p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu);
 
-		p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu);
+		for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
+			if (p->vm_node_stat_diff[i])
+				node_page_state_add(p->vm_node_stat_diff[i], pgdat, i);
 		memset(p, 0, sizeof(*p));
 	}
 
@@ -1576,6 +1527,8 @@ void __ref free_area_init_core_hotplug(struct pglist_data *pgdat)
 		zone->present_pages = 0;
 		zone_init_internals(zone, z, nid, 0);
 	}
+
+	return 0;
 }
 #endif
 
@@ -1814,7 +1767,6 @@ static void __init free_area_init(void)
 	bool descending;
 
 	arch_zone_limits_init(max_zone_pfn);
-	sparse_init();
 
 	start_pfn = PHYS_PFN(memblock_start_of_DRAM());
 	descending = arch_has_descending_max_zone_pfns();
@@ -1864,18 +1816,12 @@ static void __init free_area_init(void)
 			       (u64)zone_movable_pfn[i] << PAGE_SHIFT);
 	}
 
-	/*
-	 * Print out the early node map, and initialize the
-	 * subsection-map relative to active online memory ranges to
-	 * enable future "sub-section" extensions of the memory map.
-	 */
+	/* Print out the early node map. */
 	pr_info("Early memory node ranges\n");
-	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
+	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
 		pr_info("  node %3d: [mem %#018Lx-%#018Lx]\n", nid,
 			(u64)start_pfn << PAGE_SHIFT,
 			((u64)end_pfn << PAGE_SHIFT) - 1);
-		sparse_init_subsection_map(start_pfn, end_pfn - start_pfn);
-	}
 
 	/* Initialise every node */
 	mminit_verify_pageflags_layout();
@@ -1909,11 +1855,7 @@ static void __init free_area_init(void)
 		}
 	}
 
-	for_each_node_state(nid, N_MEMORY)
-		sparse_vmemmap_init_nid_late(nid);
-
 	calc_nr_kernel_pages();
-	memmap_init();
 
 	/* disable hash distribution for systems with a single node */
 	fixup_hashdist();
@@ -1981,12 +1923,12 @@ static void __init deferred_free_pages(unsigned long pfn,
 	if (!nr_pages)
 		return;
 
+	pageblock_migratetype_init_range(pfn, nr_pages, mt, true);
+
 	page = pfn_to_page(pfn);
 
 	/* Free a large naturally-aligned chunk if possible */
 	if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) {
-		for (i = 0; i < nr_pages; i += pageblock_nr_pages)
-			init_pageblock_migratetype(page + i, mt, false);
 		__free_pages_core(page, MAX_PAGE_ORDER, MEMINIT_EARLY);
 		return;
 	}
@@ -1994,11 +1936,8 @@ static void __init deferred_free_pages(unsigned long pfn,
 	/* Accept chunks smaller than MAX_PAGE_ORDER upfront */
 	accept_memory(PFN_PHYS(pfn), nr_pages * PAGE_SIZE);
 
-	for (i = 0; i < nr_pages; i++, page++, pfn++) {
-		if (pageblock_aligned(pfn))
-			init_pageblock_migratetype(page, mt, false);
-		__free_pages_core(page, 0, MEMINIT_EARLY);
-	}
+	for (i = 0; i < nr_pages; i++)
+		__free_pages_core(page + i, 0, MEMINIT_EARLY);
 }
 
 /* Completion tracking for deferred_init_memmap() threads */
@@ -2323,6 +2262,7 @@ void __init page_alloc_init_late(void)
 	/* Reinit limits that are based on free pages after the kernel is up */
 	files_maxfiles_init();
 #endif
+	hugetlb_bootmem_struct_page_init();
 
 	/* Accounting of total+free memory is stable at this point. */
 	mem_init_print_info();
@@ -2687,10 +2627,13 @@ void __init __weak mem_init(void)
 
 void __init mm_core_init_early(void)
 {
+	free_area_init();
+
 	hugetlb_cma_reserve();
 	hugetlb_bootmem_alloc();
 
-	free_area_init();
+	sparse_init();
+	memmap_init();
 }
 
 /*
diff --git a/mm/mm_init.h b/mm/mm_init.h
new file mode 100644
index 0000000000000..39f75df9be1c2
--- /dev/null
+++ b/mm/mm_init.h
@@ -0,0 +1,120 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * mm_init.h:
+ *
+ * mm/ internal mm_init and memblock declarations
+ */
+
+#ifndef __MM_MM_INIT_H
+#define __MM_MM_INIT_H
+
+#include <linux/types.h>
+#include <linux/init.h>
+#include <linux/mmzone.h>
+#include <linux/jump_label.h>
+#include <linux/printk.h>
+
+struct page;
+struct vmem_altmap;
+
+/* perform sanity checks on struct pages being allocated or freed */
+DECLARE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled);
+
+void set_zone_contiguous(struct zone *zone);
+bool pfn_range_intersects_zones(int nid, unsigned long start_pfn,
+			   unsigned long nr_pages);
+
+static inline void clear_zone_contiguous(struct zone *zone)
+{
+	zone->contiguous = false;
+}
+
+void memblock_free_pages(unsigned long pfn, unsigned int order);
+
+void *memmap_alloc(phys_addr_t size, phys_addr_t align, phys_addr_t min_addr,
+		int nid, bool exact_nid);
+
+void memmap_init_range(unsigned long size, int nid, unsigned long zone,
+		unsigned long start_pfn, unsigned long zone_end_pfn,
+		enum meminit_context context,
+		struct vmem_altmap *altmap, int migratetype,
+		bool isolate_pageblock);
+
+#if defined CONFIG_COMPACTION || defined CONFIG_CMA
+/* Free whole pageblock and set its migration type to MIGRATE_CMA. */
+void init_cma_reserved_pageblock(struct page *page);
+#endif
+
+#ifdef CONFIG_CMA
+void init_cma_pageblock(struct page *page);
+#else
+static inline void init_cma_pageblock(struct page *page)
+{
+}
+#endif
+
+/* Memory initialisation debug and verification */
+#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
+DECLARE_STATIC_KEY_TRUE(deferred_pages);
+
+static inline bool deferred_pages_enabled(void)
+{
+	return static_branch_unlikely(&deferred_pages);
+}
+
+bool __init deferred_grow_zone(struct zone *zone, unsigned int order);
+#else
+static inline bool deferred_pages_enabled(void)
+{
+	return false;
+}
+#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
+
+void init_deferred_page(unsigned long pfn, int nid);
+
+enum mminit_level {
+	MMINIT_WARNING,
+	MMINIT_VERIFY,
+	MMINIT_TRACE
+};
+
+#ifdef CONFIG_DEBUG_MEMORY_INIT
+
+extern int mminit_loglevel;
+
+#define mminit_dprintk(level, prefix, fmt, arg...) \
+do { \
+	if (level < mminit_loglevel) { \
+		if (level <= MMINIT_WARNING) \
+			pr_warn("mminit::" prefix " " fmt, ##arg);	\
+		else \
+			printk(KERN_DEBUG "mminit::" prefix " " fmt, ##arg); \
+	} \
+} while (0)
+
+void mminit_verify_pageflags_layout(void);
+void mminit_verify_zonelist(void);
+#else
+
+static inline void mminit_dprintk(enum mminit_level level,
+				const char *prefix, const char *fmt, ...)
+{
+}
+
+static inline void mminit_verify_pageflags_layout(void)
+{
+}
+
+static inline void mminit_verify_zonelist(void)
+{
+}
+#endif /* CONFIG_DEBUG_MEMORY_INIT */
+
+extern bool mirrored_kernelcore;
+bool memblock_has_mirror(void);
+void memblock_free_all(void);
+
+void __meminit __init_single_page(struct page *page, unsigned long pfn,
+				unsigned long zone, int nid);
+
+#endif /* __MM_MM_INIT_H */
diff --git a/mm/mm_slot.h b/mm/mm_slot.h
index 83f18ed1c4bde..9b09b68e5742a 100644
--- a/mm/mm_slot.h
+++ b/mm/mm_slot.h
@@ -33,6 +33,12 @@ static inline void mm_slot_free(struct kmem_cache *cache, void *objp)
 	kmem_cache_free(cache, objp);
 }
 
+/*
+ * Note: mm_slot_lookup and mm_slot_insert cannot be converted to static inline
+ * functions because the hash helpers (hash_for_each_possible and hash_add) rely
+ * on the actual array argument 'hashtable' for sizeof() instead of pointers.
+ */
+
 #define mm_slot_lookup(_hashtable, _mm) 				       \
 ({									       \
 	struct mm_slot *tmp_slot, *mm_slot = NULL;			       \
@@ -52,4 +58,9 @@ static inline void mm_slot_free(struct kmem_cache *cache, void *objp)
 	hash_add(_hashtable, &_mm_slot->hash, (unsigned long)_mm);	       \
 })
 
+static inline void mm_slot_remove(struct mm_slot *slot)
+{
+	hash_del(&slot->hash);
+	list_del(&slot->mm_node);
+}
 #endif /* _LINUX_MM_SLOT_H */
diff --git a/mm/mmap.c b/mm/mmap.c
index 2311ae7c2ff45..4bf26b0f1e6e3 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -80,13 +80,13 @@ core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644);
 /* Update vma->vm_page_prot to reflect vma->vm_flags. */
 void vma_set_page_prot(struct vm_area_struct *vma)
 {
-	vm_flags_t vm_flags = vma->vm_flags;
+	vma_flags_t vma_flags = vma->flags;
 	pgprot_t vm_page_prot;
 
-	vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags);
+	vm_page_prot = vma_pgprot_modify(vma->vm_page_prot, vma_flags);
 	if (vma_wants_writenotify(vma, vm_page_prot)) {
-		vm_flags &= ~VM_SHARED;
-		vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags);
+		vma_flags_clear(&vma_flags, VMA_SHARED_BIT);
+		vm_page_prot = vma_pgprot_modify(vm_page_prot, vma_flags);
 	}
 	/* remove_protection_ptes reads vma->vm_page_prot without mmap_lock */
 	WRITE_ONCE(vma->vm_page_prot, vm_page_prot);
@@ -102,15 +102,16 @@ void vma_set_page_prot(struct vm_area_struct *vma)
  */
 static int check_brk_limits(unsigned long addr, unsigned long len)
 {
+	const struct mm_struct *mm = current->mm;
+	const bool is_def_locked =
+		vma_flags_test(&mm->def_vma_flags, VMA_LOCKED_BIT);
 	unsigned long mapped_addr;
 
 	mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
 	if (IS_ERR_VALUE(mapped_addr))
 		return mapped_addr;
 
-	return mlock_future_ok(current->mm,
-			      current->mm->def_flags & VM_LOCKED, len)
-		? 0 : -EAGAIN;
+	return mlock_future_ok(mm, is_def_locked, len) ? 0 : -EAGAIN;
 }
 
 SYSCALL_DEFINE1(brk, unsigned long, brk)
@@ -197,7 +198,7 @@ SYSCALL_DEFINE1(brk, unsigned long, brk)
 		goto out;
 
 	mm->brk = brk;
-	if (mm->def_flags & VM_LOCKED)
+	if (vma_flags_test(&mm->def_vma_flags, VMA_LOCKED_BIT))
 		populate = true;
 
 success:
@@ -280,7 +281,7 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode,
  * do_mmap() - Perform a userland memory mapping into the current process
  * address space of length @len with protection bits @prot, mmap flags @flags
  * (from which VMA flags will be inferred), and any additional VMA flags to
- * apply @vm_flags. If this is a file-backed mapping then the file is specified
+ * apply @vma_flags. If this is a file-backed mapping then the file is specified
  * in @file and page offset into the file via @pgoff.
  *
  * This function does not perform security checks on the file and assumes, if
@@ -320,7 +321,8 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode,
  * (2) for details.
  * @flags: Flags specifying how the mapping should be performed, see mmap (2)
  * for details.
- * @vm_flags: VMA flags which should be set by default, or 0 otherwise.
+ * @vma_flags: VMA flags which should be set by default, or EMPTY_VMA_FLAGS
+ * otherwise.
  * @pgoff: Page offset into the @file if file-backed, should be 0 otherwise.
  * @populate: A pointer to a value which will be set to 0 if no population of
  * the range is required, or the number of bytes to populate if it is. Must be
@@ -335,7 +337,7 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode,
  */
 unsigned long do_mmap(struct file *file, unsigned long addr,
 			unsigned long len, unsigned long prot,
-			unsigned long flags, vm_flags_t vm_flags,
+			unsigned long flags, vma_flags_t vma_flags,
 			unsigned long pgoff, unsigned long *populate,
 			struct list_head *uf)
 {
@@ -399,13 +401,18 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
 	 * to. we assume access permissions have been handled by the open
 	 * of the memory object, so we don't do any here.
 	 */
-	vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(file, flags) |
-			mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
+	vma_flags_set_mask(&vma_flags,
+			   legacy_to_vma_flags(calc_vm_prot_bits(prot, pkey)));
+	vma_flags_set_mask(&vma_flags,
+			   legacy_to_vma_flags(calc_vm_flag_bits(file, flags)));
+	vma_flags_set_mask(&vma_flags, mm->def_vma_flags);
+	vma_flags_set(&vma_flags, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT,
+		      VMA_MAYEXEC_BIT);
 
 	/* Obtain the address to map to. we verify (or select) it and ensure
 	 * that it represents a valid section of the address space.
 	 */
-	addr = __get_unmapped_area(file, addr, len, pgoff, flags, vm_flags);
+	addr = __get_unmapped_area(file, addr, len, pgoff, flags, vma_flags);
 	if (IS_ERR_VALUE(addr))
 		return addr;
 
@@ -418,7 +425,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
 		if (!can_do_mlock())
 			return -EPERM;
 
-	if (!mlock_future_ok(mm, vm_flags & VM_LOCKED, len))
+	if (!mlock_future_ok(mm, vma_flags_test(&vma_flags, VMA_LOCKED_BIT), len))
 		return -EAGAIN;
 
 	if (file) {
@@ -461,22 +468,23 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
 			if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
 				return -EACCES;
 
-			vm_flags |= VM_SHARED | VM_MAYSHARE;
+			vma_flags_set(&vma_flags, VMA_SHARED_BIT, VMA_MAYSHARE_BIT);
 			if (!(file->f_mode & FMODE_WRITE))
-				vm_flags &= ~(VM_MAYWRITE | VM_SHARED);
+				vma_flags_clear(&vma_flags, VMA_MAYWRITE_BIT,
+						VMA_SHARED_BIT);
 			fallthrough;
 		case MAP_PRIVATE:
 			if (!(file->f_mode & FMODE_READ))
 				return -EACCES;
 			if (path_noexec(&file->f_path)) {
-				if (vm_flags & VM_EXEC)
+				if (vma_flags_test(&vma_flags, VMA_EXEC_BIT))
 					return -EPERM;
-				vm_flags &= ~VM_MAYEXEC;
+				vma_flags_clear(&vma_flags, VMA_MAYEXEC_BIT);
 			}
 
 			if (!can_mmap_file(file))
 				return -ENODEV;
-			if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
+			if (vma_flags_can_grow(&vma_flags))
 				return -EINVAL;
 			break;
 
@@ -488,23 +496,27 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
 		 * Check to see if we are violating any seals and update VMA
 		 * flags if necessary to avoid future seal violations.
 		 */
-		err = memfd_check_seals_mmap(file, &vm_flags);
+		err = memfd_check_seals_mmap(file, &vma_flags);
 		if (err)
 			return (unsigned long)err;
 	} else {
 		switch (flags & MAP_TYPE) {
 		case MAP_SHARED:
-			if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
+			if (vma_flags_can_grow(&vma_flags))
 				return -EINVAL;
 			/*
 			 * Ignore pgoff.
 			 */
 			pgoff = 0;
-			vm_flags |= VM_SHARED | VM_MAYSHARE;
+			vma_flags_set(&vma_flags, VMA_SHARED_BIT, VMA_MAYSHARE_BIT);
 			break;
-		case MAP_DROPPABLE:
-			if (VM_DROPPABLE == VM_NONE)
+		case MAP_DROPPABLE: {
+			vma_flags_t droppable = VMA_DROPPABLE;
+
+			if (vma_flags_empty(&droppable))
 				return -EOPNOTSUPP;
+			vma_flags_set_mask(&vma_flags, droppable);
+
 			/*
 			 * A locked or stack area makes no sense to be droppable.
 			 *
@@ -515,23 +527,24 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
 			 */
 			if (flags & (MAP_LOCKED | MAP_HUGETLB))
 			        return -EINVAL;
-			if (vm_flags & (VM_GROWSDOWN | VM_GROWSUP))
+			if (vma_flags_can_grow(&vma_flags))
 			        return -EINVAL;
 
-			vm_flags |= VM_DROPPABLE;
-
 			/*
 			 * If the pages can be dropped, then it doesn't make
 			 * sense to reserve them.
 			 */
-			vm_flags |= VM_NORESERVE;
+			vma_flags_set(&vma_flags, VMA_NORESERVE_BIT);
 
 			/*
 			 * Likewise, they're volatile enough that they
 			 * shouldn't survive forks or coredumps.
 			 */
-			vm_flags |= VM_WIPEONFORK | VM_DONTDUMP;
+			vma_flags_set(&vma_flags, VMA_WIPEONFORK_BIT,
+				      VMA_DONTDUMP_BIT);
+
 			fallthrough;
+		}
 		case MAP_PRIVATE:
 			/*
 			 * Set pgoff according to addr for anon_vma.
@@ -544,22 +557,22 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
 	}
 
 	/*
-	 * Set 'VM_NORESERVE' if we should not account for the
-	 * memory use of this mapping.
+	 * Set VMA_NORESERVE_BIT if we should not account for the memory use
+	 * of this mapping.
 	 */
 	if (flags & MAP_NORESERVE) {
 		/* We honor MAP_NORESERVE if allowed to overcommit */
 		if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
-			vm_flags |= VM_NORESERVE;
+			vma_flags_set(&vma_flags, VMA_NORESERVE_BIT);
 
 		/* hugetlb applies strict overcommit unless MAP_NORESERVE */
 		if (file && is_file_hugepages(file))
-			vm_flags |= VM_NORESERVE;
+			vma_flags_set(&vma_flags, VMA_NORESERVE_BIT);
 	}
 
-	addr = mmap_region(file, addr, len, vm_flags, pgoff, uf);
+	addr = mmap_region(file, addr, len, vma_flags, pgoff, uf);
 	if (!IS_ERR_VALUE(addr) &&
-	    ((vm_flags & VM_LOCKED) ||
+	    (vma_flags_test(&vma_flags, VMA_LOCKED_BIT) ||
 	     (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE))
 		*populate = len;
 	return addr;
@@ -645,9 +658,9 @@ SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
  * Determine if the allocation needs to ensure that there is no
  * existing mapping within it's guard gaps, for use as start_gap.
  */
-static inline unsigned long stack_guard_placement(vm_flags_t vm_flags)
+static inline unsigned long stack_guard_placement(vma_flags_t vma_flags)
 {
-	if (vm_flags & VM_SHADOW_STACK)
+	if (vma_flags_test_single_mask(&vma_flags, VMA_SHADOW_STACK))
 		return PAGE_SIZE;
 
 	return 0;
@@ -689,7 +702,7 @@ unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info)
 unsigned long
 generic_get_unmapped_area(struct file *filp, unsigned long addr,
 			  unsigned long len, unsigned long pgoff,
-			  unsigned long flags, vm_flags_t vm_flags)
+			  unsigned long flags, vma_flags_t vma_flags)
 {
 	struct mm_struct *mm = current->mm;
 	struct vm_area_struct *vma, *prev;
@@ -714,7 +727,7 @@ generic_get_unmapped_area(struct file *filp, unsigned long addr,
 	info.length = len;
 	info.low_limit = mm->mmap_base;
 	info.high_limit = mmap_end;
-	info.start_gap = stack_guard_placement(vm_flags);
+	info.start_gap = stack_guard_placement(vma_flags);
 	if (filp && is_file_hugepages(filp))
 		info.align_mask = huge_page_mask_align(filp);
 	return vm_unmapped_area(&info);
@@ -727,7 +740,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr,
 		       unsigned long flags, vm_flags_t vm_flags)
 {
 	return generic_get_unmapped_area(filp, addr, len, pgoff, flags,
-					 vm_flags);
+					 legacy_to_vma_flags(vm_flags));
 }
 #endif
 
@@ -738,7 +751,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr,
 unsigned long
 generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
 				  unsigned long len, unsigned long pgoff,
-				  unsigned long flags, vm_flags_t vm_flags)
+				  unsigned long flags, vma_flags_t vma_flags)
 {
 	struct vm_area_struct *vma, *prev;
 	struct mm_struct *mm = current->mm;
@@ -766,7 +779,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
 	info.length = len;
 	info.low_limit = PAGE_SIZE;
 	info.high_limit = arch_get_mmap_base(addr, mm->mmap_base);
-	info.start_gap = stack_guard_placement(vm_flags);
+	info.start_gap = stack_guard_placement(vma_flags);
 	if (filp && is_file_hugepages(filp))
 		info.align_mask = huge_page_mask_align(filp);
 	addr = vm_unmapped_area(&info);
@@ -795,23 +808,24 @@ arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
 			       unsigned long flags, vm_flags_t vm_flags)
 {
 	return generic_get_unmapped_area_topdown(filp, addr, len, pgoff, flags,
-						 vm_flags);
+						 legacy_to_vma_flags(vm_flags));
 }
 #endif
 
-unsigned long mm_get_unmapped_area_vmflags(struct file *filp, unsigned long addr,
-					   unsigned long len, unsigned long pgoff,
-					   unsigned long flags, vm_flags_t vm_flags)
+unsigned long mm_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr,
+		unsigned long len, unsigned long pgoff, unsigned long flags,
+		vma_flags_t vma_flags)
 {
 	if (mm_flags_test(MMF_TOPDOWN, current->mm))
 		return arch_get_unmapped_area_topdown(filp, addr, len, pgoff,
-						      flags, vm_flags);
-	return arch_get_unmapped_area(filp, addr, len, pgoff, flags, vm_flags);
+				flags, vma_flags_to_legacy(vma_flags));
+	return arch_get_unmapped_area(filp, addr, len, pgoff, flags,
+			vma_flags_to_legacy(vma_flags));
 }
 
 unsigned long
 __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
-		unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags)
+		unsigned long pgoff, unsigned long flags, vma_flags_t vma_flags)
 {
 	unsigned long (*get_area)(struct file *, unsigned long,
 				  unsigned long, unsigned long, unsigned long)
@@ -846,11 +860,11 @@ __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
 		   && !addr /* no hint */
 		   && IS_ALIGNED(len, PMD_SIZE)) {
 		/* Ensures that larger anonymous mappings are THP aligned. */
-		addr = thp_get_unmapped_area_vmflags(file, addr, len,
-						     pgoff, flags, vm_flags);
+		addr = thp_get_unmapped_area_vmaflags(file, addr, len,
+						      pgoff, flags, vma_flags);
 	} else {
-		addr = mm_get_unmapped_area_vmflags(file, addr, len,
-						    pgoff, flags, vm_flags);
+		addr = mm_get_unmapped_area_vmaflags(file, addr, len,
+						     pgoff, flags, vma_flags);
 	}
 	if (IS_ERR_VALUE(addr))
 		return addr;
@@ -868,7 +882,8 @@ unsigned long
 mm_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
 		     unsigned long pgoff, unsigned long flags)
 {
-	return mm_get_unmapped_area_vmflags(file, addr, len, pgoff, flags, 0);
+	return mm_get_unmapped_area_vmaflags(file, addr, len, pgoff, flags,
+					     EMPTY_VMA_FLAGS);
 }
 EXPORT_SYMBOL(mm_get_unmapped_area);
 
@@ -970,7 +985,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon
 		return NULL;
 	if (expand_stack_locked(prev, addr))
 		return NULL;
-	if (prev->vm_flags & VM_LOCKED)
+	if (vma_test(prev, VMA_LOCKED_BIT))
 		populate_vma_page_range(prev, addr, prev->vm_end, NULL);
 	return prev;
 }
@@ -994,7 +1009,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon
 	start = vma->vm_start;
 	if (expand_stack_locked(vma, addr))
 		return NULL;
-	if (vma->vm_flags & VM_LOCKED)
+	if (vma_test(vma, VMA_LOCKED_BIT))
 		populate_vma_page_range(vma, addr, start, NULL);
 	return vma;
 }
@@ -1119,18 +1134,18 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
 	 */
 	vma = vma_lookup(mm, start);
 
-	if (!vma || !(vma->vm_flags & VM_SHARED)) {
+	if (!vma || !vma_test(vma, VMA_SHARED_BIT)) {
 		mmap_read_unlock(mm);
 		return -EINVAL;
 	}
 
-	prot |= vma->vm_flags & VM_READ ? PROT_READ : 0;
-	prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0;
-	prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0;
+	prot |= vma_test(vma, VMA_READ_BIT) ? PROT_READ : 0;
+	prot |= vma_test(vma, VMA_WRITE_BIT) ? PROT_WRITE : 0;
+	prot |= vma_test(vma, VMA_EXEC_BIT) ? PROT_EXEC : 0;
 
 	flags &= MAP_NONBLOCK;
 	flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE;
-	if (vma->vm_flags & VM_LOCKED)
+	if (vma_test(vma, VMA_LOCKED_BIT))
 		flags |= MAP_LOCKED;
 
 	/* Save vm_flags used to calculate prot and flags, and recheck later. */
@@ -1191,7 +1206,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
 	}
 
 	ret = do_mmap(vma->vm_file, start, size,
-			prot, flags, 0, pgoff, &populate, NULL);
+			prot, flags, EMPTY_VMA_FLAGS, pgoff, &populate, NULL);
 out:
 	mmap_write_unlock(mm);
 	fput(file);
@@ -1233,7 +1248,7 @@ int vm_brk_flags(unsigned long addr, unsigned long request, bool is_exec)
 
 	vma = vma_prev(&vmi);
 	ret = do_brk_flags(&vmi, vma, addr, len, vma_flags);
-	populate = ((mm->def_flags & VM_LOCKED) != 0);
+	populate = vma_flags_test(&mm->def_vma_flags, VMA_LOCKED_BIT);
 	mmap_write_unlock(mm);
 	userfaultfd_unmap_complete(mm, &uf);
 	if (populate && !ret)
@@ -1256,7 +1271,7 @@ unsigned long tear_down_vmas(struct mm_struct *mm, struct vma_iterator *vmi,
 	mmap_assert_write_locked(mm);
 	vma_iter_set(vmi, vma->vm_end);
 	do {
-		if (vma->vm_flags & VM_ACCOUNT)
+		if (vma_test(vma, VMA_ACCOUNT_BIT))
 			nr_accounted += vma_pages(vma);
 		vma_mark_detached(vma);
 		remove_vma(vma);
@@ -1405,7 +1420,7 @@ static int special_mapping_split(struct vm_area_struct *vma, unsigned long addr)
 {
 	/*
 	 * Forbid splitting special mappings - kernel has expectations over
-	 * the number of pages in mapping. Together with VM_DONTEXPAND
+	 * the number of pages in mapping. Together with VMA_DONTEXPAND_BIT
 	 * the size of vma should stay the same over the special mapping's
 	 * lifetime.
 	 */
@@ -1447,44 +1462,6 @@ static vm_fault_t special_mapping_fault(struct vm_fault *vmf)
 	return VM_FAULT_SIGBUS;
 }
 
-static struct vm_area_struct *__install_special_mapping(
-	struct mm_struct *mm,
-	unsigned long addr, unsigned long len,
-	vm_flags_t vm_flags, void *priv,
-	const struct vm_operations_struct *ops)
-{
-	int ret;
-	struct vm_area_struct *vma;
-
-	vma = vm_area_alloc(mm);
-	if (unlikely(vma == NULL))
-		return ERR_PTR(-ENOMEM);
-
-	vma_set_range(vma, addr, addr + len, 0);
-	vm_flags |= mm->def_flags | VM_DONTEXPAND;
-	if (pgtable_supports_soft_dirty())
-		vm_flags |= VM_SOFTDIRTY;
-	vm_flags_init(vma, vm_flags & ~VM_LOCKED_MASK);
-	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
-
-	vma->vm_ops = ops;
-	vma->vm_private_data = priv;
-
-	ret = insert_vm_struct(mm, vma);
-	if (ret)
-		goto out;
-
-	vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT);
-
-	perf_event_mmap(vma);
-
-	return vma;
-
-out:
-	vm_area_free(vma);
-	return ERR_PTR(ret);
-}
-
 bool vma_is_special_mapping(const struct vm_area_struct *vma,
 	const struct vm_special_mapping *sm)
 {
@@ -1715,7 +1692,7 @@ bool mmap_read_lock_maybe_expand(struct mm_struct *mm,
 		return true;
 	}
 
-	if (!(new_vma->vm_flags & VM_GROWSDOWN))
+	if (!vma_test(new_vma, VMA_GROWSDOWN_BIT))
 		return false;
 
 	mmap_write_lock(mm);
@@ -1765,7 +1742,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
 		retval = vma_start_write_killable(mpnt);
 		if (retval < 0)
 			goto loop_out;
-		if (mpnt->vm_flags & VM_DONTCOPY) {
+		if (vma_test(mpnt, VMA_DONTCOPY_BIT)) {
 			retval = vma_iter_clear_gfp(&vmi, mpnt->vm_start,
 						    mpnt->vm_end, GFP_KERNEL);
 			if (retval)
@@ -1775,7 +1752,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
 			continue;
 		}
 		charge = 0;
-		if (mpnt->vm_flags & VM_ACCOUNT) {
+		if (vma_test(mpnt, VMA_ACCOUNT_BIT)) {
 			unsigned long len = vma_pages(mpnt);
 
 			if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
@@ -1793,16 +1770,19 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
 		retval = dup_userfaultfd(tmp, &uf);
 		if (retval)
 			goto fail_nomem_anon_vma_fork;
-		if (tmp->vm_flags & VM_WIPEONFORK) {
+
+		if (vma_test(tmp, VMA_WIPEONFORK_BIT)) {
 			/*
-			 * VM_WIPEONFORK gets a clean slate in the child.
+			 * VMA_WIPEONFORK_BIT gets a clean slate in the child.
 			 * Don't prepare anon_vma until fault since we don't
 			 * copy page for current vma.
 			 */
 			tmp->anon_vma = NULL;
 		} else if (anon_vma_fork(tmp, mpnt))
 			goto fail_nomem_anon_vma_fork;
-		vm_flags_clear(tmp, VM_LOCKED_MASK);
+
+		vma_start_write(tmp);
+		vma_clear_flags_mask(tmp, VMA_LOCKED_MASK);
 		/*
 		 * Copy/update hugetlb private vma information.
 		 */
@@ -1830,13 +1810,12 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
 				mapping_allow_writable(mapping);
 			flush_dcache_mmap_lock(mapping);
 			/* insert tmp into the share list, just after mpnt */
-			vma_interval_tree_insert_after(tmp, mpnt,
-					&mapping->i_mmap);
+			mapping_rmap_tree_insert_after(tmp, mpnt, mapping);
 			flush_dcache_mmap_unlock(mapping);
 			i_mmap_unlock_write(mapping);
 		}
 
-		if (!(tmp->vm_flags & VM_WIPEONFORK))
+		if (!vma_test(tmp, VMA_WIPEONFORK_BIT))
 			retval = copy_page_range(tmp, mpnt);
 
 		if (retval) {
diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c
index 245b74f39f912..df69ba6e797f7 100644
--- a/mm/mmu_notifier.c
+++ b/mm/mmu_notifier.c
@@ -19,7 +19,7 @@
 #include <linux/sched/mm.h>
 #include <linux/slab.h>
 
-#include "vma.h"
+#include "internal.h"
 
 /* global SRCU for all MMs */
 DEFINE_STATIC_SRCU(srcu);
diff --git a/mm/mmzone.c b/mm/mmzone.c
index 0c8f181d9d500..59dc3f2076a6d 100644
--- a/mm/mmzone.c
+++ b/mm/mmzone.c
@@ -43,7 +43,8 @@ struct zone *next_zone(struct zone *zone)
 	return zone;
 }
 
-static inline int zref_in_nodemask(struct zoneref *zref, nodemask_t *nodes)
+static inline int zref_in_nodemask(struct zoneref *zref,
+				   const nodemask_t *nodes)
 {
 #ifdef CONFIG_NUMA
 	return node_isset(zonelist_node_idx(zref), *nodes);
@@ -55,7 +56,7 @@ static inline int zref_in_nodemask(struct zoneref *zref, nodemask_t *nodes)
 /* Returns the next zone at or below highest_zoneidx in a zonelist */
 struct zoneref *__next_zones_zonelist(struct zoneref *z,
 					enum zone_type highest_zoneidx,
-					nodemask_t *nodes)
+					const nodemask_t *nodes)
 {
 	/*
 	 * Find the next suitable zone to use for the allocation.
diff --git a/mm/mprotect.c b/mm/mprotect.c
index 9cbf932b028cf..2888ee638d872 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -40,7 +40,7 @@
 
 static bool maybe_change_pte_writable(struct vm_area_struct *vma, pte_t pte)
 {
-	if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))
+	if (WARN_ON_ONCE(!vma_test(vma, VMA_WRITE_BIT)))
 		return false;
 
 	/* Don't touch entries that are not even readable. */
@@ -97,7 +97,7 @@ static bool can_change_shared_pte_writable(struct vm_area_struct *vma,
 bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
 			     pte_t pte)
 {
-	if (!(vma->vm_flags & VM_SHARED))
+	if (!vma_test(vma, VMA_SHARED_BIT))
 		return can_change_private_pte_writable(vma, addr, pte);
 
 	return can_change_shared_pte_writable(vma, pte);
@@ -143,7 +143,7 @@ static __always_inline void prot_commit_flush_ptes(struct vm_area_struct *vma,
  * !PageAnonExclusive() pages, starting from start_idx. Caller must enforce
  * that the ptes point to consecutive pages of the same anon large folio.
  */
-static __always_inline int page_anon_exclusive_sub_batch(int start_idx, int max_len,
+static __always_inline int page_anon_exclusive_batch(int start_idx, int max_len,
 		struct page *first_page, bool expected_anon_exclusive)
 {
 	int idx;
@@ -174,16 +174,16 @@ static __always_inline void commit_anon_folio_batch(struct vm_area_struct *vma,
 		pte_t oldpte, pte_t ptent, int nr_ptes, struct mmu_gather *tlb)
 {
 	bool expected_anon_exclusive;
-	int sub_batch_idx = 0;
+	int batch_idx = 0;
 	int len;
 
 	while (nr_ptes) {
-		expected_anon_exclusive = PageAnonExclusive(first_page + sub_batch_idx);
-		len = page_anon_exclusive_sub_batch(sub_batch_idx, nr_ptes,
+		expected_anon_exclusive = PageAnonExclusive(first_page + batch_idx);
+		len = page_anon_exclusive_batch(batch_idx, nr_ptes,
 					first_page, expected_anon_exclusive);
 		prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, len,
-				       sub_batch_idx, expected_anon_exclusive, tlb);
-		sub_batch_idx += len;
+				       batch_idx, expected_anon_exclusive, tlb);
+		batch_idx += len;
 		nr_ptes -= len;
 	}
 }
@@ -194,7 +194,7 @@ static __always_inline void set_write_prot_commit_flush_ptes(struct vm_area_stru
 {
 	bool set_write;
 
-	if (vma->vm_flags & VM_SHARED) {
+	if (vma_test(vma, VMA_SHARED_BIT)) {
 		set_write = can_change_shared_pte_writable(vma, ptent);
 		prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes,
 				       /* idx = */ 0, set_write, tlb);
@@ -214,8 +214,9 @@ static __always_inline void set_write_prot_commit_flush_ptes(struct vm_area_stru
 static long change_softleaf_pte(struct vm_area_struct *vma,
 	unsigned long addr, pte_t *pte, pte_t oldpte, unsigned long cp_flags)
 {
-	const bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
-	const bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
+	const bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP);
+	const bool uffd_prot_resolve = cp_flags &
+		(MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE);
 	softleaf_t entry = softleaf_from_pte(oldpte);
 	pte_t newpte;
 
@@ -240,8 +241,8 @@ static long change_softleaf_pte(struct vm_area_struct *vma,
 		 */
 		entry = make_readable_device_private_entry(swp_offset(entry));
 		newpte = swp_entry_to_pte(entry);
-		if (pte_swp_uffd_wp(oldpte))
-			newpte = pte_swp_mkuffd_wp(newpte);
+		if (pte_swp_uffd(oldpte))
+			newpte = pte_swp_mkuffd(newpte);
 	} else if (softleaf_is_marker(entry)) {
 		/*
 		 * Ignore error swap entries unconditionally,
@@ -256,7 +257,7 @@ static long change_softleaf_pte(struct vm_area_struct *vma,
 		 * to unprotect it, drop it; the next page
 		 * fault will trigger without uffd trapping.
 		 */
-		if (uffd_wp_resolve) {
+		if (uffd_prot_resolve) {
 			pte_clear(vma->vm_mm, addr, pte);
 			return 1;
 		}
@@ -265,10 +266,10 @@ static long change_softleaf_pte(struct vm_area_struct *vma,
 		newpte = oldpte;
 	}
 
-	if (uffd_wp)
-		newpte = pte_swp_mkuffd_wp(newpte);
-	else if (uffd_wp_resolve)
-		newpte = pte_swp_clear_uffd_wp(newpte);
+	if (uffd_prot)
+		newpte = pte_swp_mkuffd(newpte);
+	else if (uffd_prot_resolve)
+		newpte = pte_swp_clear_uffd(newpte);
 
 	if (!pte_same(oldpte, newpte)) {
 		set_pte_at(vma->vm_mm, addr, pte, newpte);
@@ -282,17 +283,28 @@ static __always_inline void change_present_ptes(struct mmu_gather *tlb,
 		int nr_ptes, unsigned long end, pgprot_t newprot,
 		struct folio *folio, struct page *page, unsigned long cp_flags)
 {
-	const bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
-	const bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
+	const bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP);
+	const bool uffd_prot_resolve = cp_flags &
+		(MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE);
 	pte_t ptent, oldpte;
 
 	oldpte = modify_prot_start_ptes(vma, addr, ptep, nr_ptes);
 	ptent = pte_modify(oldpte, newprot);
 
-	if (uffd_wp)
-		ptent = pte_mkuffd_wp(ptent);
-	else if (uffd_wp_resolve)
-		ptent = pte_clear_uffd_wp(ptent);
+	if (uffd_prot)
+		ptent = pte_mkuffd(ptent);
+	else if (uffd_prot_resolve)
+		ptent = pte_clear_uffd(ptent);
+
+	/*
+	 * The uffd bit on a VM_UFFD_RWP VMA carries PROT_NONE
+	 * semantics. If mprotect() or NUMA hinting changed the
+	 * base protection, restore PAGE_NONE so the PTE still
+	 * traps on any access. pte_modify() preserves
+	 * _PAGE_UFFD.
+	 */
+	if (userfaultfd_rwp(vma) && pte_uffd(ptent))
+		ptent = pte_modify(ptent, PAGE_NONE);
 
 	/*
 	 * In some writable, shared mappings, we might want
@@ -325,6 +337,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 	long pages = 0;
 	bool is_private_single_threaded;
 	bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
+	bool uffd_rwp = cp_flags & MM_CP_UFFD_RWP;
 	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
 	int nr_ptes;
 
@@ -350,6 +363,14 @@ static long change_pte_range(struct mmu_gather *tlb,
 			/* Already in the desired state. */
 			if (prot_numa && pte_protnone(oldpte))
 				continue;
+			/*
+			 * RWP-protected PTEs carry _PAGE_UFFD as a marker on
+			 * top of PROT_NONE. Skip only entries already in that
+			 * exact state; plain PROT_NONE from mprotect() still needs
+			 * to be promoted so future faults can be distinguished.
+			 */
+			if (uffd_rwp && pte_protnone(oldpte) && pte_uffd(oldpte))
+				continue;
 
 			page = vm_normal_page(vma, addr, oldpte);
 			if (page)
@@ -358,6 +379,8 @@ static long change_pte_range(struct mmu_gather *tlb,
 			/*
 			 * Avoid trapping faults against the zero or KSM
 			 * pages. See similar comment in change_huge_pmd.
+			 * Skip this filter for uffd RWP which
+			 * must set protnone regardless of NUMA placement.
 			 */
 			if (prot_numa &&
 			    !folio_can_map_prot_numa(folio, vma,
@@ -428,7 +451,7 @@ pgtable_split_needed(struct vm_area_struct *vma, unsigned long cp_flags)
 	 * (e.g. 2M shmem) because file thp is handled differently when
 	 * split by erasing the pmd so far.
 	 */
-	return (cp_flags & MM_CP_UFFD_WP) && !vma_is_anonymous(vma);
+	return (cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP)) && !vma_is_anonymous(vma);
 }
 
 /*
@@ -667,7 +690,16 @@ long change_protection(struct mmu_gather *tlb,
 	pgprot_t newprot = vma->vm_page_prot;
 	long pages;
 
-	BUG_ON((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL);
+	/*
+	 * MM_CP_UFFD_{WP,RWP} and _RESOLVE are mutually exclusive within one
+	 * change, and WP and RWP cannot mix. Miswired callers get a warn and
+	 * a no-op; userspace cannot reach this state.
+	 */
+	if (WARN_ON_ONCE((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL ||
+			 (cp_flags & MM_CP_UFFD_RWP_ALL) == MM_CP_UFFD_RWP_ALL ||
+			 ((cp_flags & MM_CP_UFFD_WP_ALL) &&
+			  (cp_flags & MM_CP_UFFD_RWP_ALL))))
+		return 0;
 
 #ifdef CONFIG_NUMA_BALANCING
 	/*
@@ -681,6 +713,10 @@ long change_protection(struct mmu_gather *tlb,
 	WARN_ON_ONCE(cp_flags & MM_CP_PROT_NUMA);
 #endif
 
+	if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE) &&
+	    (cp_flags & MM_CP_UFFD_RWP))
+		newprot = PAGE_NONE;
+
 	if (is_vm_hugetlb_page(vma))
 		pages = hugetlb_change_protection(vma, start, end, newprot,
 						  cp_flags);
@@ -699,25 +735,24 @@ static int prot_none_pte_entry(pte_t *pte, unsigned long addr,
 		0 : -EACCES;
 }
 
+#ifdef CONFIG_HUGETLB_PAGE
 static int prot_none_hugetlb_entry(pte_t *pte, unsigned long hmask,
 				   unsigned long addr, unsigned long next,
 				   struct mm_walk *walk)
 {
-	return pfn_modify_allowed(pte_pfn(ptep_get(pte)),
-				  *(pgprot_t *)(walk->private)) ?
-		0 : -EACCES;
-}
+	const pte_t entry = huge_ptep_get(walk->mm, addr, pte);
 
-static int prot_none_test(unsigned long addr, unsigned long next,
-			  struct mm_walk *walk)
-{
-	return 0;
+	if (pfn_modify_allowed(pte_pfn(entry), *(pgprot_t *)(walk->private)))
+		return 0;
+	return -EACCES;
 }
+#else
+#define prot_none_hugetlb_entry	NULL
+#endif
 
 static const struct mm_walk_ops prot_none_walk_ops = {
 	.pte_entry		= prot_none_pte_entry,
 	.hugetlb_entry		= prot_none_hugetlb_entry,
-	.test_walk		= prot_none_test,
 	.walk_lock		= PGWALK_WRLOCK,
 };
 
@@ -753,7 +788,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
 	    !vma_flags_test_any_mask(&new_vma_flags, VMA_ACCESS_FLAGS)) {
 		pgprot_t new_pgprot = vm_get_page_prot(newflags);
 
-		error = walk_page_range(current->mm, start, end,
+		error = walk_page_range_vma(vma, start, end,
 				&prot_none_walk_ops, &new_pgprot);
 		if (error)
 			return error;
@@ -811,8 +846,8 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
 		vm_unacct_memory(nrpages);
 
 	/*
-	 * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major
-	 * fault on access.
+	 * Private VMA_LOCKED_BIT VMA becoming writable: trigger COW to avoid
+	 * major fault on access.
 	 */
 	if (vma_flags_test(&new_vma_flags, VMA_WRITE_BIT) &&
 	    vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT) &&
@@ -886,7 +921,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len,
 			goto out;
 		start = vma->vm_start;
 		error = -EINVAL;
-		if (!(vma->vm_flags & VM_GROWSDOWN))
+		if (!vma_test(vma, VMA_GROWSDOWN_BIT))
 			goto out;
 	} else {
 		if (vma->vm_start > start)
@@ -894,7 +929,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len,
 		if (unlikely(grows & PROT_GROWSUP)) {
 			end = vma->vm_end;
 			error = -EINVAL;
-			if (!(vma->vm_flags & VM_GROWSUP))
+			if (!vma_test_single_mask(vma, VMA_GROWSUP))
 				goto out;
 		}
 	}
@@ -918,7 +953,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len,
 		}
 
 		/* Does the application expect PROT_READ to imply PROT_EXEC */
-		if (rier && (vma->vm_flags & VM_MAYEXEC))
+		if (rier && vma_test(vma, VMA_MAYEXEC_BIT))
 			prot |= PROT_EXEC;
 
 		/*
diff --git a/mm/mremap.c b/mm/mremap.c
index e9c8b1d05832b..b64aa1f6e07ed 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -68,7 +68,7 @@ struct vma_remap_struct {
 	bool populate_expand;		/* mlock()'d expanded, must populate. */
 	enum mremap_type remap_type;	/* expand, shrink, etc. */
 	bool mmap_locked;		/* Is mm currently write-locked? */
-	unsigned long charged;		/* If VM_ACCOUNT, # pages to account. */
+	unsigned long charged;		/* If VMA_ACCOUNT_BIT, # pgs to account */
 	bool vmi_needs_invalidate;	/* Is the VMA iterator invalidated? */
 };
 
@@ -296,10 +296,19 @@ static int move_ptes(struct pagetable_move_control *pmc,
 			pte_clear(mm, new_addr, new_ptep);
 		else {
 			if (need_clear_uffd_wp) {
-				if (pte_present(pte))
-					pte = pte_clear_uffd_wp(pte);
-				else
-					pte = pte_swp_clear_uffd_wp(pte);
+				if (pte_present(pte)) {
+					/*
+					 * See __copy_present_ptes(): normalise
+					 * RWP PTEs so the destination starts
+					 * accessible instead of taking a
+					 * numa-hinting fault on first access.
+					 */
+					if (userfaultfd_rwp(vma) && pte_uffd(pte))
+						pte = pte_modify(pte, vma->vm_page_prot);
+					pte = pte_clear_uffd(pte);
+				} else {
+					pte = pte_swp_clear_uffd(pte);
+				}
 			}
 			set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);
 		}
@@ -948,14 +957,13 @@ static unsigned long vrm_set_new_addr(struct vma_remap_struct *vrm)
 	struct vm_area_struct *vma = vrm->vma;
 	unsigned long map_flags = 0;
 	/* Page Offset _into_ the VMA. */
-	pgoff_t internal_pgoff = (vrm->addr - vma->vm_start) >> PAGE_SHIFT;
-	pgoff_t pgoff = vma->vm_pgoff + internal_pgoff;
+	const pgoff_t pgoff = linear_page_index(vma, vrm->addr);
 	unsigned long new_addr = vrm_implies_new_addr(vrm) ? vrm->new_addr : 0;
 	unsigned long res;
 
 	if (vrm->flags & MREMAP_FIXED)
 		map_flags |= MAP_FIXED;
-	if (vma->vm_flags & VM_MAYSHARE)
+	if (vma_test(vma, VMA_MAYSHARE_BIT))
 		map_flags |= MAP_SHARED;
 
 	res = get_unmapped_area(vma->vm_file, new_addr, vrm->new_len, pgoff,
@@ -977,7 +985,7 @@ static bool vrm_calc_charge(struct vma_remap_struct *vrm)
 {
 	unsigned long charged;
 
-	if (!(vrm->vma->vm_flags & VM_ACCOUNT))
+	if (!vma_test(vrm->vma, VMA_ACCOUNT_BIT))
 		return true;
 
 	/*
@@ -1004,7 +1012,7 @@ static bool vrm_calc_charge(struct vma_remap_struct *vrm)
  */
 static void vrm_uncharge(struct vma_remap_struct *vrm)
 {
-	if (!(vrm->vma->vm_flags & VM_ACCOUNT))
+	if (!vma_test(vrm->vma, VMA_ACCOUNT_BIT))
 		return;
 
 	vm_unacct_memory(vrm->charged);
@@ -1024,7 +1032,7 @@ static void vrm_stat_account(struct vma_remap_struct *vrm,
 	struct vm_area_struct *vma = vrm->vma;
 
 	vm_stat_account(mm, vma->vm_flags, pages);
-	if (vma->vm_flags & VM_LOCKED)
+	if (vma_test(vma, VMA_LOCKED_BIT))
 		mm->locked_vm += pages;
 }
 
@@ -1168,7 +1176,7 @@ static void unmap_source_vma(struct vma_remap_struct *vrm)
 	 * arose, in which case we _do_ wish to unmap the _new_ VMA, which means
 	 * we actually _do_ want it be unaccounted.
 	 */
-	bool accountable_move = (vma->vm_flags & VM_ACCOUNT) &&
+	bool accountable_move = vma_test(vma, VMA_ACCOUNT_BIT) &&
 		!(vrm->flags & MREMAP_DONTUNMAP);
 
 	/*
@@ -1187,7 +1195,7 @@ static void unmap_source_vma(struct vma_remap_struct *vrm)
 	 * portions of the original VMA that remain.
 	 */
 	if (accountable_move) {
-		vm_flags_clear(vma, VM_ACCOUNT);
+		vma_clear_flags(vma, VMA_ACCOUNT_BIT);
 		/* We are about to split vma, so store the start/end. */
 		vm_start = vma->vm_start;
 		vm_end = vma->vm_end;
@@ -1212,8 +1220,8 @@ static void unmap_source_vma(struct vma_remap_struct *vrm)
 	 * |             |
 	 * |-------------|
 	 *
-	 * Having cleared VM_ACCOUNT from the whole VMA, after we unmap above
-	 * we'll end up with:
+	 * Having cleared VMA_ACCOUNT_BIT from the whole VMA, after we unmap
+	 * above we'll end up with:
 	 *
 	 *    addr  end
 	 *     |     |
@@ -1233,13 +1241,15 @@ static void unmap_source_vma(struct vma_remap_struct *vrm)
 		if (vm_start < addr) {
 			struct vm_area_struct *prev = vma_prev(&vmi);
 
-			vm_flags_set(prev, VM_ACCOUNT); /* Acquires VMA lock. */
+			vma_start_write(prev);
+			vma_set_flags(prev, VMA_ACCOUNT_BIT);
 		}
 
 		if (vm_end > end) {
 			struct vm_area_struct *next = vma_next(&vmi);
 
-			vm_flags_set(next, VM_ACCOUNT); /* Acquires VMA lock. */
+			vma_start_write(next);
+			vma_set_flags(next, VMA_ACCOUNT_BIT);
 		}
 	}
 }
@@ -1255,12 +1265,10 @@ static void unmap_source_vma(struct vma_remap_struct *vrm)
 static int copy_vma_and_data(struct vma_remap_struct *vrm,
 			     struct vm_area_struct **new_vma_ptr)
 {
-	unsigned long internal_offset = vrm->addr - vrm->vma->vm_start;
-	unsigned long internal_pgoff = internal_offset >> PAGE_SHIFT;
-	unsigned long new_pgoff = vrm->vma->vm_pgoff + internal_pgoff;
-	unsigned long moved_len;
+	const unsigned long new_pgoff = linear_page_index(vrm->vma, vrm->addr);
 	struct vm_area_struct *vma = vrm->vma;
 	struct vm_area_struct *new_vma;
+	unsigned long moved_len;
 	int err = 0;
 	PAGETABLE_MOVE(pmc, NULL, NULL, vrm->addr, vrm->new_addr, vrm->old_len);
 
@@ -1324,8 +1332,8 @@ static void dontunmap_complete(struct vma_remap_struct *vrm,
 	unsigned long old_start = vrm->vma->vm_start;
 	unsigned long old_end = vrm->vma->vm_end;
 
-	/* We always clear VM_LOCKED[ONFAULT] on the old VMA. */
-	vm_flags_clear(vrm->vma, VM_LOCKED_MASK);
+	/* We always clear VMA_LOCKED[ONFAULT]_BIT on the old VMA. */
+	vma_clear_flags_mask(vrm->vma, VMA_LOCKED_MASK);
 
 	/*
 	 * anon_vma links of the old vma is no longer needed after its page
@@ -1761,14 +1769,14 @@ static int check_prep_vma(struct vma_remap_struct *vrm)
 	 * based on the original.  There are no known use cases for this
 	 * behavior.  As a result, fail such attempts.
 	 */
-	if (!old_len && !(vma->vm_flags & (VM_SHARED | VM_MAYSHARE))) {
+	if (!old_len && !vma_test_any(vma, VMA_SHARED_BIT, VMA_MAYSHARE_BIT)) {
 		pr_warn_once("%s (%d): attempted to duplicate a private mapping with mremap.  This is not supported.\n",
 			     current->comm, current->pid);
 		return -EINVAL;
 	}
 
 	if ((vrm->flags & MREMAP_DONTUNMAP) &&
-			(vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)))
+	    vma_test_any(vma, VMA_DONTEXPAND_BIT, VMA_PFNMAP_BIT))
 		return -EINVAL;
 
 	/*
@@ -1798,19 +1806,18 @@ static int check_prep_vma(struct vma_remap_struct *vrm)
 		return 0;
 
 	/* We are expanding and the VMA is mlock()'d so we need to populate. */
-	if (vma->vm_flags & VM_LOCKED)
+	if (vma_test(vma, VMA_LOCKED_BIT))
 		vrm->populate_expand = true;
 
 	/* Need to be careful about a growing mapping */
-	pgoff = (addr - vma->vm_start) >> PAGE_SHIFT;
-	pgoff += vma->vm_pgoff;
+	pgoff = linear_page_index(vma, addr);
 	if (pgoff + (new_len >> PAGE_SHIFT) < pgoff)
 		return -EINVAL;
 
-	if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP))
+	if (vma_test_any(vma, VMA_DONTEXPAND_BIT, VMA_PFNMAP_BIT))
 		return -EFAULT;
 
-	if (!mlock_future_ok(mm, vma->vm_flags & VM_LOCKED, vrm->delta))
+	if (!mlock_future_ok(mm, vma_test(vma, VMA_LOCKED_BIT), vrm->delta))
 		return -EAGAIN;
 
 	if (!may_expand_vm(mm, &vma->flags, vrm->delta >> PAGE_SHIFT))
diff --git a/mm/msync.c b/mm/msync.c
index ac4c9bfea2e7f..90b491a27a14b 100644
--- a/mm/msync.c
+++ b/mm/msync.c
@@ -12,6 +12,7 @@
 #include <linux/mm.h>
 #include <linux/mman.h>
 #include <linux/file.h>
+#include <linux/pagemap.h>
 #include <linux/syscalls.h>
 #include <linux/sched.h>
 
@@ -85,8 +86,7 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags)
 			goto out_unlock;
 		}
 		file = vma->vm_file;
-		fstart = (start - vma->vm_start) +
-			 ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
+		fstart = (loff_t)linear_page_index(vma, start) << PAGE_SHIFT;
 		fend = fstart + (min(end, vma->vm_end) - start) - 1;
 		start = vma->vm_end;
 		if ((flags & MS_SYNC) && file &&
diff --git a/mm/nommu.c b/mm/nommu.c
index ed3934bc2de48..498e01ee40b05 100644
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -569,7 +569,7 @@ static void setup_vma_to_mm(struct vm_area_struct *vma, struct mm_struct *mm)
 
 		i_mmap_lock_write(mapping);
 		flush_dcache_mmap_lock(mapping);
-		vma_interval_tree_insert(vma, &mapping->i_mmap);
+		mapping_rmap_tree_insert(vma, mapping);
 		flush_dcache_mmap_unlock(mapping);
 		i_mmap_unlock_write(mapping);
 	}
@@ -585,7 +585,7 @@ static void cleanup_vma_from_mm(struct vm_area_struct *vma)
 
 		i_mmap_lock_write(mapping);
 		flush_dcache_mmap_lock(mapping);
-		vma_interval_tree_remove(vma, &mapping->i_mmap);
+		mapping_rmap_tree_remove(vma, mapping);
 		flush_dcache_mmap_unlock(mapping);
 		i_mmap_unlock_write(mapping);
 	}
@@ -975,7 +975,7 @@ static int do_mmap_private(struct vm_area_struct *vma,
 		/* read the contents of a file into the copy */
 		loff_t fpos;
 
-		fpos = vma->vm_pgoff;
+		fpos = vma_start_pgoff(vma);
 		fpos <<= PAGE_SHIFT;
 
 		ret = kernel_read(vma->vm_file, base, len, &fpos);
@@ -1014,11 +1014,12 @@ unsigned long do_mmap(struct file *file,
 			unsigned long len,
 			unsigned long prot,
 			unsigned long flags,
-			vm_flags_t vm_flags,
+			vma_flags_t vma_flags,
 			unsigned long pgoff,
 			unsigned long *populate,
 			struct list_head *uf)
 {
+	vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags);
 	struct vm_area_struct *vma;
 	struct vm_region *region;
 	struct rb_node *rb;
@@ -1035,6 +1036,9 @@ unsigned long do_mmap(struct file *file,
 	if (ret < 0)
 		return ret;
 
+	if (current->mm->map_count >= get_sysctl_max_map_count())
+		return -ENOMEM;
+
 	/* we ignore the address hint */
 	addr = 0;
 	len = PAGE_ALIGN(len);
@@ -1058,7 +1062,7 @@ unsigned long do_mmap(struct file *file,
 	region->vm_pgoff = pgoff;
 
 	vm_flags_init(vma, vm_flags);
-	vma->vm_pgoff = pgoff;
+	vma_set_pgoff(vma, pgoff);
 
 	if (file) {
 		region->vm_file = get_file(file);
@@ -1178,7 +1182,6 @@ unsigned long do_mmap(struct file *file,
 		ret = do_mmap_private(vma, region, len, capabilities);
 	if (ret < 0)
 		goto error_just_free;
-	add_nommu_region(region);
 
 	/* clear anonymous mappings that don't ask for uninitialized data */
 	if (!vma->vm_file &&
@@ -1196,7 +1199,9 @@ unsigned long do_mmap(struct file *file,
 	BUG_ON(!vma->vm_region);
 	vma_iter_config(&vmi, vma->vm_start, vma->vm_end);
 	if (vma_iter_prealloc(&vmi, vma))
-		goto error_just_free;
+		goto error_vma_iter_prealloc;
+
+	add_nommu_region(region);
 
 	setup_vma_to_mm(vma, current->mm);
 	current->mm->map_count++;
@@ -1215,22 +1220,41 @@ unsigned long do_mmap(struct file *file,
 	return result;
 
 error_just_free:
+	vma_close(vma);
+	/* if the error was from shared mapping/existing region, don't free the region.
+	 * this has to be before releasing semaphore.
+	 */
+	if (region->vm_usage == 1) {
+		if (region->vm_file)
+			fput(region->vm_file);
+		kmem_cache_free(vm_region_jar, region);
+
+	} else
+		region->vm_usage--;
+
 	up_write(&nommu_region_sem);
-error:
 	vma_iter_free(&vmi);
-	if (region->vm_file)
-		fput(region->vm_file);
-	kmem_cache_free(vm_region_jar, region);
+
 	if (vma->vm_file)
 		fput(vma->vm_file);
 	vm_area_free(vma);
 	return ret;
 
 sharing_violation:
-	up_write(&nommu_region_sem);
 	pr_warn("Attempt to share mismatched mappings\n");
 	ret = -EINVAL;
-	goto error;
+	goto error_just_free;
+
+error_vma_iter_prealloc:
+	pr_warn("Allocation of vma iterator for process %d failed\n", current->pid);
+	show_mem();
+	ret = -ENOMEM;
+
+	/* in case that the region is allocated via do_mmap_private() */
+	if ((region->vm_usage == 1) && (region->vm_flags & VM_MAPPED_COPY))
+		free_page_series(region->vm_start, region->vm_top);
+
+	goto error_just_free;
 
 error_getting_vma:
 	kmem_cache_free(vm_region_jar, region);
@@ -1332,13 +1356,14 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
 	*region = *vma->vm_region;
 	new->vm_region = region;
 
-	npages = (addr - vma->vm_start) >> PAGE_SHIFT;
+	npages = linear_page_delta(vma, addr);
 
 	if (new_below) {
 		region->vm_top = region->vm_end = new->vm_end = addr;
 	} else {
 		region->vm_start = new->vm_start = addr;
-		region->vm_pgoff = new->vm_pgoff += npages;
+		vma_add_pgoff(new, npages);
+		region->vm_pgoff = vma_start_pgoff(new);
 	}
 
 	vma_iter_config(vmi, new->vm_start, new->vm_end);
@@ -1355,7 +1380,8 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
 	delete_nommu_region(vma->vm_region);
 	if (new_below) {
 		vma->vm_region->vm_start = vma->vm_start = addr;
-		vma->vm_region->vm_pgoff = vma->vm_pgoff += npages;
+		vma_add_pgoff(vma, npages);
+		vma->vm_region->vm_pgoff = vma_start_pgoff(vma);
 	} else {
 		vma->vm_region->vm_end = vma->vm_end = addr;
 		vma->vm_region->vm_top = addr;
@@ -1367,6 +1393,10 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
 	setup_vma_to_mm(vma, mm);
 	setup_vma_to_mm(new, mm);
 	vma_iter_store_new(vmi, new);
+
+	/* vmi should point lower address */
+	if (new_below)
+		vma_next(vmi);
 	mm->map_count++;
 	return 0;
 
@@ -1603,7 +1633,7 @@ int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long
 	unsigned long pfn = start >> PAGE_SHIFT;
 	unsigned long vm_len = vma->vm_end - vma->vm_start;
 
-	pfn += vma->vm_pgoff;
+	pfn += vma_start_pgoff(vma);
 	return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
 }
 EXPORT_SYMBOL(vm_iomap_memory);
@@ -1816,7 +1846,7 @@ int nommu_shrink_inode_mappings(struct inode *inode, size_t size,
 	i_mmap_lock_read(inode->i_mapping);
 
 	/* search for VMAs that fall within the dead zone */
-	vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) {
+	mapping_rmap_tree_foreach(vma, inode->i_mapping, low, high) {
 		/* found one - only interested if it's shared out of the page
 		 * cache */
 		if (vma->vm_flags & VM_SHARED) {
@@ -1832,7 +1862,7 @@ int nommu_shrink_inode_mappings(struct inode *inode, size_t size,
 	 * we don't check for any regions that start beyond the EOF as there
 	 * shouldn't be any
 	 */
-	vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, 0, ULONG_MAX) {
+	mapping_rmap_tree_foreach(vma, inode->i_mapping, 0, ULONG_MAX) {
 		if (!(vma->vm_flags & VM_SHARED))
 			continue;
 
diff --git a/mm/numa_memblks.c b/mm/numa_memblks.c
index 3c3c4eac3514d..be6e7346f6c78 100644
--- a/mm/numa_memblks.c
+++ b/mm/numa_memblks.c
@@ -17,20 +17,6 @@ nodemask_t numa_nodes_parsed __initdata;
 static struct numa_meminfo numa_meminfo __initdata_or_meminfo;
 static struct numa_meminfo numa_reserved_meminfo __initdata_or_meminfo;
 
-/*
- * Set nodes, which have memory in @mi, in *@nodemask.
- */
-static void __init numa_nodemask_from_meminfo(nodemask_t *nodemask,
-					      const struct numa_meminfo *mi)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(mi->blk); i++)
-		if (mi->blk[i].start != mi->blk[i].end &&
-		    mi->blk[i].nid != NUMA_NO_NODE)
-			node_set(mi->blk[i].nid, *nodemask);
-}
-
 /**
  * numa_reset_distance - Reset NUMA distance table
  *
@@ -56,7 +42,6 @@ static int __init numa_alloc_distance(void)
 
 	/* size the new table and allocate it */
 	nodes_parsed = numa_nodes_parsed;
-	numa_nodemask_from_meminfo(&nodes_parsed, &numa_meminfo);
 
 	for_each_node_mask(i, nodes_parsed)
 		cnt = i;
@@ -135,13 +120,20 @@ EXPORT_SYMBOL(__node_distance);
 static int __init numa_add_memblk_to(int nid, u64 start, u64 end,
 				     struct numa_meminfo *mi)
 {
+	/* whine about and ignore invalid nid */
+	if (nid < 0 || nid >= MAX_NUMNODES) {
+		pr_warn("Warning: invalid memblk node id %d [mem %#010Lx-%#010Lx]\n",
+			nid, start, end - 1);
+		return -EINVAL;
+	}
+
 	/* ignore zero length blks */
 	if (start == end)
 		return 0;
 
-	/* whine about and ignore invalid blks */
-	if (start > end || nid < 0 || nid >= MAX_NUMNODES) {
-		pr_warn("Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n",
+	/* whine about and ignore invalid ranges */
+	if (start > end) {
+		pr_warn("Warning: invalid memblk range for node %d [mem %#010Lx-%#010Lx]\n",
 			nid, start, end - 1);
 		return 0;
 	}
@@ -193,13 +185,20 @@ static void __init numa_move_tail_memblk(struct numa_meminfo *dst, int idx,
  * @end: End address of the new memblk
  *
  * Add a new memblk to the default numa_meminfo.
+ * On success @nid is also set in numa_nodes_parsed.
  *
  * RETURNS:
  * 0 on success, -errno on failure.
  */
 int __init numa_add_memblk(int nid, u64 start, u64 end)
 {
-	return numa_add_memblk_to(nid, start, end, &numa_meminfo);
+	int ret;
+
+	ret = numa_add_memblk_to(nid, start, end, &numa_meminfo);
+	if (!ret)
+		node_set(nid, numa_nodes_parsed);
+
+	return ret;
 }
 
 /**
@@ -256,8 +255,7 @@ int __init numa_cleanup_meminfo(struct numa_meminfo *mi)
 
 		/* preserve info for non-RAM areas above 'max_pfn': */
 		if (bi->end > high) {
-			numa_add_memblk_to(bi->nid, high, bi->end,
-					   &numa_reserved_meminfo);
+			numa_add_reserved_memblk(bi->nid, high, bi->end);
 			bi->end = high;
 		}
 
@@ -401,7 +399,6 @@ static int __init numa_register_meminfo(struct numa_meminfo *mi)
 
 	/* Account for nodes with cpus and no memory */
 	node_possible_map = numa_nodes_parsed;
-	numa_nodemask_from_meminfo(&node_possible_map, mi);
 	if (WARN_ON(nodes_empty(node_possible_map)))
 		return -EINVAL;
 
diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index e98748112d1ed..47495be68598f 100644
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -584,16 +584,6 @@ static inline void __wb_writeout_add(struct bdi_writeback *wb, long nr)
 				       wb->bdi->max_prop_frac, nr);
 }
 
-void wb_writeout_inc(struct bdi_writeback *wb)
-{
-	unsigned long flags;
-
-	local_irq_save(flags);
-	__wb_writeout_add(wb, 1);
-	local_irq_restore(flags);
-}
-EXPORT_SYMBOL_GPL(wb_writeout_inc);
-
 /*
  * On idle system, we can be called long after we scheduled because we use
  * deferred timers so count with missed periods.
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index ee902a468c2f5..520ebd2fa40b5 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -56,6 +56,8 @@
 #include <linux/pgalloc_tag.h>
 #include <asm/div64.h>
 #include "internal.h"
+#include "mm_init.h"
+#include "page_alloc.h"
 #include "shuffle.h"
 #include "page_reporting.h"
 
@@ -165,7 +167,9 @@ DEFINE_PER_CPU(int, numa_node);
 EXPORT_PER_CPU_SYMBOL(numa_node);
 #endif
 
+#ifdef CONFIG_NUMA
 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);
+#endif
 
 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
 /*
@@ -1246,7 +1250,7 @@ void __clear_page_tag_ref(struct page *page)
 /* Should be called only if mem_alloc_profiling_enabled() */
 static noinline
 void __pgalloc_tag_add(struct page *page, struct task_struct *task,
-		       unsigned int nr, gfp_t gfp_flags)
+		       unsigned int nr, unsigned int alloc_flags)
 {
 	union pgtag_ref_handle handle;
 	union codetag_ref ref;
@@ -1260,17 +1264,17 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,
 		 * page_ext is not available yet, record the pfn so we can
 		 * clear the tag ref later when page_ext is initialized.
 		 */
-		alloc_tag_add_early_pfn(page_to_pfn(page), gfp_flags);
+		alloc_tag_add_early_pfn(page_to_pfn(page), alloc_flags);
 		if (task->alloc_tag)
 			alloc_tag_set_inaccurate(task->alloc_tag);
 	}
 }
 
 static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
-				   unsigned int nr, gfp_t gfp_flags)
+				   unsigned int nr, unsigned int alloc_flags)
 {
 	if (mem_alloc_profiling_enabled())
-		__pgalloc_tag_add(page, task, nr, gfp_flags);
+		__pgalloc_tag_add(page, task, nr, alloc_flags);
 }
 
 /* Should be called only if mem_alloc_profiling_enabled() */
@@ -1303,7 +1307,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)
 #else /* CONFIG_MEM_ALLOC_PROFILING */
 
 static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
-				   unsigned int nr, gfp_t gfp_flags) {}
+				   unsigned int nr, unsigned int alloc_flags) {}
 static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {}
 static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {}
 
@@ -1374,15 +1378,23 @@ static __always_inline bool __free_pages_prepare(struct page *page,
 #endif
 		}
 		for (i = 1; i < (1 << order); i++) {
+			struct page *tail_page = page + i;
+
 			if (compound)
-				bad += free_tail_page_prepare(page, page + i);
+				bad += free_tail_page_prepare(page, tail_page);
 			if (is_check_pages_enabled()) {
-				if (free_page_is_bad(page + i)) {
+				if (free_page_is_bad(tail_page)) {
+					bad++;
+					continue;
+				}
+
+				if (tail_page->private) {
+					bad_page(tail_page, "nonzero private");
 					bad++;
 					continue;
 				}
 			}
-			(page + i)->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP;
+			tail_page->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP;
 		}
 	}
 	if (folio_test_anon(folio)) {
@@ -1807,7 +1819,7 @@ static inline bool should_skip_init(gfp_t flags)
 }
 
 inline void post_alloc_hook(struct page *page, unsigned int order,
-				gfp_t gfp_flags)
+				gfp_t gfp_flags, unsigned int alloc_flags)
 {
 	const bool zero_tags = gfp_flags & __GFP_ZEROTAGS;
 	bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) &&
@@ -1858,13 +1870,13 @@ inline void post_alloc_hook(struct page *page, unsigned int order,
 
 	set_page_owner(page, order, gfp_flags);
 	page_table_check_alloc(page, order);
-	pgalloc_tag_add(page, current, 1 << order, gfp_flags);
+	pgalloc_tag_add(page, current, 1 << order, alloc_flags);
 }
 
 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
 							unsigned int alloc_flags)
 {
-	post_alloc_hook(page, order, gfp_flags);
+	post_alloc_hook(page, order, gfp_flags, alloc_flags);
 
 	if (order && (gfp_flags & __GFP_COMP))
 		prep_compound_page(page, order);
@@ -2528,7 +2540,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
 	unsigned long flags;
 	int i;
 
-	if (unlikely(alloc_flags & ALLOC_TRYLOCK)) {
+	if (unlikely(alloc_flags & ALLOC_NOLOCK)) {
 		if (!spin_trylock_irqsave(&zone->lock, flags))
 			return 0;
 	} else {
@@ -3216,7 +3228,7 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone,
 
 	do {
 		page = NULL;
-		if (unlikely(alloc_flags & ALLOC_TRYLOCK)) {
+		if (unlikely(alloc_flags & ALLOC_NOLOCK)) {
 			if (!spin_trylock_irqsave(&zone->lock, flags))
 				return NULL;
 		} else {
@@ -3247,10 +3259,11 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone,
 	} while (check_new_pages(page, order));
 
 	/*
-	 * If this is a high-order atomic allocation then check
-	 * if the pageblock should be reserved for the future
+	 * Slowpath (precarious) high-atomic allocations may reserve
+	 * a pageblock for future use.
 	 */
-	if (unlikely(alloc_flags & ALLOC_HIGHATOMIC))
+	if (unlikely((alloc_flags & ALLOC_HIGHATOMIC) &&
+			((alloc_flags & ALLOC_WMARK_MASK) == ALLOC_WMARK_MIN)))
 		reserve_highatomic_pageblock(page, order, zone);
 
 	__count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
@@ -3739,13 +3752,10 @@ static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
 static inline unsigned int
 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)
 {
-	unsigned int alloc_flags;
+	unsigned int alloc_flags = 0;
 
-	/*
-	 * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
-	 * to save a branch.
-	 */
-	alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM);
+	if (gfp_mask & __GFP_KSWAPD_RECLAIM)
+		alloc_flags |= ALLOC_KSWAPD;
 
 	if (defrag_mode) {
 		alloc_flags |= ALLOC_NOFRAGMENT;
@@ -3774,14 +3784,13 @@ alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)
 }
 
 /* Must be called after current_gfp_context() which can change gfp_mask */
-static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask,
-						  unsigned int alloc_flags)
+static inline unsigned int alloc_flags_cma(gfp_t gfp_mask)
 {
 #ifdef CONFIG_CMA
 	if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE)
-		alloc_flags |= ALLOC_CMA;
+		return ALLOC_CMA;
 #endif
-	return alloc_flags;
+	return ALLOC_DEFAULT;
 }
 
 /*
@@ -3899,8 +3908,6 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
 		if (!zone_watermark_fast(zone, order, mark,
 				       ac->highest_zoneidx, alloc_flags,
 				       gfp_mask)) {
-			int ret;
-
 			if (cond_accept_memory(zone, order, alloc_flags))
 				goto try_this_zone;
 
@@ -3921,22 +3928,13 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
 			    !zone_allows_reclaim(zonelist_zone(ac->preferred_zoneref), zone))
 				continue;
 
-			ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
-			switch (ret) {
-			case NODE_RECLAIM_NOSCAN:
-				/* did not scan */
+			if (!node_reclaim(zone->zone_pgdat, gfp_mask, order))
 				continue;
-			case NODE_RECLAIM_FULL:
-				/* scanned but unreclaimable */
-				continue;
-			default:
-				/* did we reclaim enough */
-				if (zone_watermark_ok(zone, order, mark,
-					ac->highest_zoneidx, alloc_flags))
-					goto try_this_zone;
 
+			/* did we reclaim enough */
+			if (!zone_watermark_ok(zone, order, mark,
+					       ac->highest_zoneidx, alloc_flags))
 				continue;
-			}
 		}
 
 try_this_zone:
@@ -3979,7 +3977,7 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
 	return NULL;
 }
 
-static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)
+static void warn_alloc_show_mem(gfp_t gfp_mask, const nodemask_t *nodemask)
 {
 	unsigned int filter = SHOW_MEM_FILTER_NODES;
 
@@ -3999,7 +3997,7 @@ static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)
 	mem_cgroup_show_protected_memory(NULL);
 }
 
-void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
+void warn_alloc(gfp_t gfp_mask, const nodemask_t *nodemask, const char *fmt, ...)
 {
 	struct va_format vaf;
 	va_list args;
@@ -4077,7 +4075,7 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
 	 */
 	page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) &
 				      ~__GFP_DIRECT_RECLAIM, order,
-				      ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
+				      ac->alloc_flags|ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
 	if (page)
 		goto out;
 
@@ -4123,7 +4121,7 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
 		 */
 		if (gfp_mask & __GFP_NOFAIL)
 			page = __alloc_pages_cpuset_fallback(gfp_mask, order,
-					ALLOC_NO_WATERMARKS, ac);
+					ac->alloc_flags|ALLOC_NO_WATERMARKS, ac);
 	}
 out:
 	mutex_unlock(&oom_lock);
@@ -4474,17 +4472,32 @@ static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
 }
 
 static inline unsigned int
-gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order)
+alloc_flags_nonblocking(gfp_t gfp_mask, unsigned int order)
 {
-	unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
+	unsigned int alloc_flags = 0;
+
+	if (gfp_mask & __GFP_DIRECT_RECLAIM)
+		return 0;
 
 	/*
-	 * __GFP_HIGH is assumed to be the same as ALLOC_MIN_RESERVE
-	 * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
-	 * to save two branches.
+	 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
+	 * if it can't schedule.
 	 */
-	BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_MIN_RESERVE);
-	BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD);
+	if (gfp_mask & __GFP_NOMEMALLOC)
+		return 0;
+
+	alloc_flags |= ALLOC_NON_BLOCK;
+
+	if (order > 0 && (gfp_mask & __GFP_HIGH))
+		alloc_flags |= ALLOC_HIGHATOMIC;
+
+	return alloc_flags;
+}
+
+static inline unsigned int
+alloc_flags_slowpath(gfp_t gfp_mask, unsigned int order)
+{
+	unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
 
 	/*
 	 * The caller may dip into page reserves a bit more if the caller
@@ -4492,21 +4505,14 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order)
 	 * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will
 	 * set both ALLOC_NON_BLOCK and ALLOC_MIN_RESERVE(__GFP_HIGH).
 	 */
-	alloc_flags |= (__force int)
-		(gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM));
-
-	if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {
-		/*
-		 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
-		 * if it can't schedule.
-		 */
-		if (!(gfp_mask & __GFP_NOMEMALLOC)) {
-			alloc_flags |= ALLOC_NON_BLOCK;
+	if (gfp_mask & __GFP_HIGH)
+		alloc_flags |= ALLOC_MIN_RESERVE;
+	if (gfp_mask & __GFP_KSWAPD_RECLAIM)
+		alloc_flags |= ALLOC_KSWAPD;
 
-			if (order > 0 && (alloc_flags & ALLOC_MIN_RESERVE))
-				alloc_flags |= ALLOC_HIGHATOMIC;
-		}
+	alloc_flags |= alloc_flags_nonblocking(gfp_mask, order);
 
+	if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {
 		/*
 		 * Ignore cpuset mems for non-blocking __GFP_HIGH (probably
 		 * GFP_ATOMIC) rather than fail, see the comment for
@@ -4517,7 +4523,7 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order)
 	} else if (unlikely(rt_or_dl_task(current)) && in_task())
 		alloc_flags |= ALLOC_MIN_RESERVE;
 
-	alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags);
+	alloc_flags |= alloc_flags_cma(gfp_mask);
 
 	if (defrag_mode)
 		alloc_flags |= ALLOC_NOFRAGMENT;
@@ -4687,7 +4693,7 @@ check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac)
 	return false;
 }
 
-static void check_alloc_stall_warn(gfp_t gfp_mask, nodemask_t *nodemask,
+static void check_alloc_stall_warn(gfp_t gfp_mask, const nodemask_t *nodemask,
 				unsigned int order, unsigned long alloc_start_time)
 {
 	static DEFINE_SPINLOCK(alloc_stall_lock);
@@ -4782,8 +4788,12 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
 	 * The fast path uses conservative alloc_flags to succeed only until
 	 * kswapd needs to be woken up, and to avoid the cost of setting up
 	 * alloc_flags precisely. So we do that now.
+	 *
+	 * Can't just or alloc_flags if it contains WMARK bits, but those flags
+	 * shouldn't be set in ac->alloc_flags.
 	 */
-	alloc_flags = gfp_to_alloc_flags(gfp_mask, order);
+	VM_WARN_ON(ac->alloc_flags & ALLOC_WMARK_MASK);
+	alloc_flags = ac->alloc_flags | alloc_flags_slowpath(gfp_mask, order);
 
 	/*
 	 * We need to recalculate the starting point for the zonelist iterator
@@ -4824,8 +4834,8 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
 
 	reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
 	if (reserve_flags)
-		alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags) |
-					  (alloc_flags & ALLOC_KSWAPD);
+		alloc_flags = alloc_flags_cma(gfp_mask) | reserve_flags |
+				ac->alloc_flags | (alloc_flags & ALLOC_KSWAPD);
 
 	/*
 	 * Reset the nodemask and zonelist iterators if memory policies can be
@@ -4994,6 +5004,8 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
 	 * we always retry
 	 */
 	if (unlikely(nofail)) {
+		unsigned int alloc_flags = ac->alloc_flags | ALLOC_MIN_RESERVE;
+
 		/*
 		 * Lacking direct_reclaim we can't do anything to reclaim memory,
 		 * we disregard these unreasonable nofail requests and still
@@ -5009,7 +5021,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
 		 * could deplete whole memory reserves which would just make
 		 * the situation worse.
 		 */
-		page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_MIN_RESERVE, ac);
+		page = __alloc_pages_cpuset_fallback(gfp_mask, order, alloc_flags, ac);
 		if (page)
 			goto got_pg;
 
@@ -5051,11 +5063,11 @@ static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
 	 * Don't invoke should_fail logic, since it may call
 	 * get_random_u32() and printk() which need to spin_lock.
 	 */
-	if (!(*alloc_flags & ALLOC_TRYLOCK) &&
+	if (!(*alloc_flags & ALLOC_NOLOCK) &&
 	    should_fail_alloc_page(gfp_mask, order))
 		return false;
 
-	*alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, *alloc_flags);
+	*alloc_flags |= alloc_flags_cma(gfp_mask);
 
 	/* Dirty zone balancing only done in the fast path */
 	ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);
@@ -5213,7 +5225,7 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,
 		}
 		nr_account++;
 
-		prep_new_page(page, 0, gfp, 0);
+		prep_new_page(page, 0, gfp, ALLOC_DEFAULT);
 		set_page_refcounted(page);
 		page_array[nr_populated++] = page;
 	}
@@ -5227,7 +5239,7 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,
 	return nr_populated;
 
 failed:
-	page = __alloc_pages_noprof(gfp, 0, preferred_nid, nodemask);
+	page = __alloc_pages_noprof(gfp, 0, preferred_nid, nodemask, ALLOC_DEFAULT);
 	if (page)
 		page_array[nr_populated++] = page;
 	goto out;
@@ -5262,24 +5274,87 @@ void free_pages_bulk(struct page **page_array, unsigned long nr_pages)
 	}
 }
 
+static inline bool alloc_order_allowed(gfp_t gfp, unsigned int order,
+				       unsigned int alloc_flags)
+{
+	if (alloc_flags & ALLOC_NOLOCK)
+		return pcp_allowed_order(order);
+
+	/*
+	 * There are several places where we assume that the order value is sane
+	 * so bail out early if the request is out of bound.
+	 */
+	return !(WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER, gfp));
+}
+
+static inline bool alloc_nolock_allowed(void)
+{
+	if (!can_spin_trylock())
+		return false;
+
+	/* Bailout, since _deferred_grow_zone() needs to take a lock */
+	if (deferred_pages_enabled())
+		return false;
+
+	return true;
+}
+
+/*
+ * GFP flags to set for ALLOC_NOLOCK i.e. alloc_pages_nolock().
+ *
+ * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed.
+ * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd
+ * is not safe in arbitrary context.
+ *
+ * These two are the conditions for gfpflags_allow_spinning() being true.
+ *
+ * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason
+ * to warn. Also warn would trigger printk() which is unsafe from
+ * various contexts. We cannot use printk_deferred_enter() to mitigate,
+ * since the running context is unknown.
+ *
+ * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below
+ * is safe in any context. Also zeroing the page is mandatory for
+ * BPF use cases.
+ *
+ * Though __GFP_NOMEMALLOC is not checked in the code path below,
+ * specify it here to highlight that alloc_pages_nolock()
+ * doesn't want to deplete reserves.
+ */
+static const gfp_t gfp_nolock = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC |
+				__GFP_COMP;
+
 /*
  * This is the 'heart' of the zoned buddy allocator.
  */
 struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
-		int preferred_nid, nodemask_t *nodemask)
+		int preferred_nid, nodemask_t *nodemask, unsigned int alloc_flags)
 {
 	struct page *page;
-	unsigned int alloc_flags = ALLOC_WMARK_LOW;
 	gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */
-	struct alloc_context ac = { };
+	struct alloc_context ac = {
+		.alloc_flags = alloc_flags,
+	};
+	unsigned int fastpath_alloc_flags = alloc_flags;
 
-	/*
-	 * There are several places where we assume that the order value is sane
-	 * so bail out early if the request is out of bound.
-	 */
-	if (WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER, gfp))
+	/* Other flags could be supported later if needed. */
+	if (WARN_ON(alloc_flags & ~(ALLOC_NOLOCK | ALLOC_NO_CODETAG)))
+		return NULL;
+
+	if (!alloc_order_allowed(gfp, order, alloc_flags))
 		return NULL;
 
+	if (alloc_flags & ALLOC_NOLOCK) {
+		/* Certain other flags could be supported later if needed. */
+		VM_WARN_ON_ONCE(gfp & ~(__GFP_ACCOUNT | gfp_nolock));
+		if (!alloc_nolock_allowed())
+			return NULL;
+		gfp |= gfp_nolock;
+		fastpath_alloc_flags |= ALLOC_WMARK_MIN;
+	} else {
+		fastpath_alloc_flags |= ALLOC_WMARK_LOW;
+	}
+
 	gfp &= gfp_allowed_mask;
 	/*
 	 * Apply scoped allocation constraints. This is mainly about GFP_NOFS
@@ -5291,18 +5366,22 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
 	gfp = current_gfp_context(gfp);
 	alloc_gfp = gfp;
 	if (!prepare_alloc_pages(gfp, order, preferred_nid, nodemask, &ac,
-			&alloc_gfp, &alloc_flags))
+			&alloc_gfp, &fastpath_alloc_flags))
 		return NULL;
 
-	/*
-	 * Forbid the first pass from falling back to types that fragment
-	 * memory until all local zones are considered.
-	 */
-	alloc_flags |= alloc_flags_nofragment(zonelist_zone(ac.preferred_zoneref), gfp);
+	if (!(alloc_flags & ALLOC_NOLOCK)) {
+		/*
+		 * Forbid the first pass from falling back to types that
+		 * fragment memory until all local zones are considered.
+		 */
+		fastpath_alloc_flags |= alloc_flags_nofragment(
+			zonelist_zone(ac.preferred_zoneref), gfp);
+	}
+	fastpath_alloc_flags |= alloc_flags_nonblocking(gfp, order) & ALLOC_HIGHATOMIC;
 
-	/* First allocation attempt */
-	page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac);
-	if (likely(page))
+	/* First allocation attempt (or, for nolock, only attempt) */
+	page = get_page_from_freelist(alloc_gfp, order, fastpath_alloc_flags, &ac);
+	if (likely(page) || (alloc_flags & ALLOC_NOLOCK))
 		goto out;
 
 	alloc_gfp = gfp;
@@ -5319,7 +5398,8 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
 out:
 	if (memcg_kmem_online() && (gfp & __GFP_ACCOUNT) && page &&
 	    unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) {
-		free_frozen_pages(page, order);
+		__free_frozen_pages(page, order,
+				    alloc_flags & ALLOC_NOLOCK ? FPI_TRYLOCK : 0);
 		page = NULL;
 	}
 
@@ -5331,22 +5411,34 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
 EXPORT_SYMBOL(__alloc_frozen_pages_noprof);
 
 struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order,
-		int preferred_nid, nodemask_t *nodemask)
+		int preferred_nid, nodemask_t *nodemask, unsigned int alloc_flags)
 {
 	struct page *page;
 
-	page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask);
+	page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask,
+					   alloc_flags);
 	if (page)
 		set_page_refcounted(page);
 	return page;
 }
-EXPORT_SYMBOL(__alloc_pages_noprof);
+
+struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order)
+{
+	if (nid == NUMA_NO_NODE)
+		nid = numa_mem_id();
+
+	VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES);
+	warn_if_node_offline(nid, gfp_mask);
+
+	return __alloc_pages_noprof(gfp_mask, order, nid, NULL, ALLOC_DEFAULT);
+}
+EXPORT_SYMBOL(alloc_pages_node_noprof);
 
 struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_nid,
 		nodemask_t *nodemask)
 {
 	struct page *page = __alloc_pages_noprof(gfp | __GFP_COMP, order,
-					preferred_nid, nodemask);
+					preferred_nid, nodemask, ALLOC_DEFAULT);
 	return page_rmappable_folio(page);
 }
 EXPORT_SYMBOL(__folio_alloc_noprof);
@@ -6287,15 +6379,34 @@ void adjust_managed_page_count(struct page *page, long count)
 }
 EXPORT_SYMBOL(adjust_managed_page_count);
 
-void free_reserved_page(struct page *page)
+/**
+ * free_reserved_pages - free reserved pages
+ * @page: First page to free.
+ * @order: The page order to free.
+ *
+ * Free pages allocated through memblock during boot, letting the buddy
+ * manage them from now on.
+ *
+ * @page must be naturally aligned to the order and the order must not
+ * exceed MAX_PAGE_ORDER. All pages must be reserved.
+ */
+void free_reserved_pages(struct page *page, unsigned int order)
 {
-	clear_page_tag_ref(page);
-	ClearPageReserved(page);
-	init_page_count(page);
-	__free_page(page);
-	adjust_managed_page_count(page, 1);
+	const unsigned long nr_pages = 1UL << order;
+	int i;
+
+	VM_WARN_ON_ONCE(!IS_ALIGNED(page_to_pfn(page), nr_pages));
+	VM_WARN_ON_ONCE(order > MAX_PAGE_ORDER);
+
+	for (i = 0; i < nr_pages; i++) {
+		clear_page_tag_ref(page + i);
+		set_page_count(page + i, 0);
+		ClearPageReserved(page + i);
+	}
+	adjust_managed_page_count(page, nr_pages);
+	__free_frozen_pages(page, order, FPI_NONE);
 }
-EXPORT_SYMBOL(free_reserved_page);
+EXPORT_SYMBOL(free_reserved_pages);
 
 static int page_alloc_cpu_dead(unsigned int cpu)
 {
@@ -6904,15 +7015,15 @@ static void __free_contig_range_common(unsigned long pfn, unsigned long nr_pages
 			continue;
 		}
 
-		if (start && memdesc_section(page->flags) != start_sec) {
+		if (start && memdesc_section(&page->flags) != start_sec) {
 			free_prepared_contig_range(start, i - nr_start);
 			start = page;
 			nr_start = i;
-			start_sec = memdesc_section(page->flags);
+			start_sec = memdesc_section(&page->flags);
 		} else if (!start) {
 			start = page;
 			nr_start = i;
-			start_sec = memdesc_section(page->flags);
+			start_sec = memdesc_section(&page->flags);
 		}
 	}
 
@@ -7028,7 +7139,7 @@ static void split_free_frozen_pages(struct list_head *list, gfp_t gfp_mask)
 		list_for_each_entry_safe(page, next, &list[order], lru) {
 			int i;
 
-			post_alloc_hook(page, order, gfp_mask);
+			post_alloc_hook(page, order, gfp_mask, ALLOC_DEFAULT);
 			if (!order)
 				continue;
 
@@ -7233,7 +7344,7 @@ int alloc_contig_frozen_range_noprof(unsigned long start, unsigned long end,
 		struct page *head = pfn_to_page(start);
 
 		check_new_pages(head, order);
-		prep_new_page(head, order, gfp_mask, 0);
+		prep_new_page(head, order, gfp_mask, ALLOC_DEFAULT);
 	} else {
 		ret = -EINVAL;
 		WARN(true, "PFN range: requested [%lu, %lu), allocated [%lu, %lu)\n",
@@ -7795,7 +7906,7 @@ static bool cond_accept_memory(struct zone *zone, unsigned int order,
 		return false;
 
 	/* Bailout, since try_to_accept_memory_one() needs to take a lock */
-	if (alloc_flags & ALLOC_TRYLOCK)
+	if (alloc_flags & ALLOC_NOLOCK)
 		return false;
 
 	wmark = promo_wmark_pages(zone);
@@ -7865,80 +7976,10 @@ static bool __free_unaccepted(struct page *page)
 
 struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order)
 {
-	/*
-	 * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed.
-	 * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd
-	 * is not safe in arbitrary context.
-	 *
-	 * These two are the conditions for gfpflags_allow_spinning() being true.
-	 *
-	 * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason
-	 * to warn. Also warn would trigger printk() which is unsafe from
-	 * various contexts. We cannot use printk_deferred_enter() to mitigate,
-	 * since the running context is unknown.
-	 *
-	 * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below
-	 * is safe in any context. Also zeroing the page is mandatory for
-	 * BPF use cases.
-	 *
-	 * Though __GFP_NOMEMALLOC is not checked in the code path below,
-	 * specify it here to highlight that alloc_pages_nolock()
-	 * doesn't want to deplete reserves.
-	 */
-	gfp_t alloc_gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC | __GFP_COMP
-			| gfp_flags;
-	unsigned int alloc_flags = ALLOC_TRYLOCK;
-	struct alloc_context ac = { };
-	struct page *page;
-
-	VM_WARN_ON_ONCE(gfp_flags & ~__GFP_ACCOUNT);
-	/*
-	 * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is
-	 * unsafe in NMI. If spin_trylock() is called from hard IRQ the current
-	 * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will
-	 * mark the task as the owner of another rt_spin_lock which will
-	 * confuse PI logic, so return immediately if called from hard IRQ or
-	 * NMI.
-	 *
-	 * Note, irqs_disabled() case is ok. This function can be called
-	 * from raw_spin_lock_irqsave region.
-	 */
-	if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq()))
-		return NULL;
-
-	/* On UP, spin_trylock() always succeeds even when it is locked */
-	if (!IS_ENABLED(CONFIG_SMP) && in_nmi())
-		return NULL;
-
-	if (!pcp_allowed_order(order))
-		return NULL;
-
-	/* Bailout, since _deferred_grow_zone() needs to take a lock */
-	if (deferred_pages_enabled())
-		return NULL;
-
 	if (nid == NUMA_NO_NODE)
 		nid = numa_node_id();
 
-	prepare_alloc_pages(alloc_gfp, order, nid, NULL, &ac,
-			    &alloc_gfp, &alloc_flags);
-
-	/*
-	 * Best effort allocation from percpu free list.
-	 * If it's empty attempt to spin_trylock zone->lock.
-	 */
-	page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac);
-
-	/* Unlike regular alloc_pages() there is no __alloc_pages_slowpath(). */
-
-	if (memcg_kmem_online() && page && (gfp_flags & __GFP_ACCOUNT) &&
-	    unlikely(__memcg_kmem_charge_page(page, alloc_gfp, order) != 0)) {
-		__free_frozen_pages(page, order, FPI_TRYLOCK);
-		page = NULL;
-	}
-	trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype);
-	kmsan_alloc_page(page, order, alloc_gfp);
-	return page;
+	return __alloc_frozen_pages_noprof(gfp_flags, order, nid, NULL, ALLOC_NOLOCK);
 }
 /**
  * alloc_pages_nolock - opportunistic reentrant allocation from any context
diff --git a/mm/page_alloc.h b/mm/page_alloc.h
new file mode 100644
index 0000000000000..b9259deddb59d
--- /dev/null
+++ b/mm/page_alloc.h
@@ -0,0 +1,312 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * mm-internal API for the page (buddy) allocator. Public API lives in
+ * include/linux/gfp.h.
+ */
+#ifndef __MM_PAGE_ALLOC_H
+#define __MM_PAGE_ALLOC_H
+
+#include <linux/mm.h>
+#include <linux/mmzone.h>
+#include <linux/nodemask.h>
+#include <linux/types.h>
+
+#define ALLOC_DEFAULT		0
+/* The ALLOC_WMARK bits are used as an index to zone->watermark */
+#define ALLOC_WMARK_MIN		WMARK_MIN
+#define ALLOC_WMARK_LOW		WMARK_LOW
+#define ALLOC_WMARK_HIGH	WMARK_HIGH
+#define ALLOC_NO_WATERMARKS	0x04 /* don't check watermarks at all */
+
+/* Mask to get the watermark bits */
+#define ALLOC_WMARK_MASK	(ALLOC_NO_WATERMARKS-1)
+
+/*
+ * Only MMU archs have async oom victim reclaim - aka oom_reaper so we
+ * cannot assume a reduced access to memory reserves is sufficient for
+ * !MMU
+ */
+#ifdef CONFIG_MMU
+#define ALLOC_OOM		0x08
+#else
+#define ALLOC_OOM		ALLOC_NO_WATERMARKS
+#endif
+
+#define ALLOC_NON_BLOCK		 0x10 /* Caller cannot block. Allow access
+				       * to 25% of the min watermark or
+				       * 62.5% if __GFP_HIGH is set.
+				       */
+#define ALLOC_MIN_RESERVE	 0x20 /* __GFP_HIGH set. Allow access to 50%
+				       * of the min watermark.
+				       */
+#define ALLOC_CPUSET		 0x40 /* check for correct cpuset */
+#define ALLOC_CMA		 0x80 /* allow allocations from CMA areas */
+#ifdef CONFIG_ZONE_DMA32
+#define ALLOC_NOFRAGMENT	0x100 /* avoid mixing pageblock types */
+#else
+#define ALLOC_NOFRAGMENT	  0x0
+#endif
+#define ALLOC_HIGHATOMIC	0x200 /* Allows access to MIGRATE_HIGHATOMIC */
+#define ALLOC_NOLOCK		0x400 /* Only use spin_trylock in allocation path */
+#define ALLOC_KSWAPD		0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */
+/*
+ * Avoid alloc_tag recursion for internal allocations.
+ *
+ * Callers must clear_page_tag_ref() before freeing to avoid warnings from
+ * alloc_tag_sub_check().
+ */
+#define ALLOC_NO_CODETAG       0x1000
+
+/* Flags that allow allocations below the min watermark. */
+#define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM)
+
+/*
+ * Structure for holding the mostly immutable allocation parameters passed
+ * between functions involved in allocations, including the alloc_pages*
+ * family of functions.
+ *
+ * nodemask, migratetype and highest_zoneidx are initialized only once in
+ * __alloc_pages() and then never change.
+ *
+ * zonelist, preferred_zone and highest_zoneidx are set first in
+ * __alloc_pages() for the fast path, and might be later changed
+ * in __alloc_pages_slowpath(). All other functions pass the whole structure
+ * by a const pointer.
+ */
+struct alloc_context {
+	struct zonelist *zonelist;
+	const nodemask_t *nodemask;
+	struct zoneref *preferred_zoneref;
+	int migratetype;
+
+	/*
+	 * highest_zoneidx represents highest usable zone index of
+	 * the allocation request. Due to the nature of the zone,
+	 * memory on lower zone than the highest_zoneidx will be
+	 * protected by lowmem_reserve[highest_zoneidx].
+	 *
+	 * highest_zoneidx is also used by reclaim/compaction to limit
+	 * the target zone since higher zone than this index cannot be
+	 * usable for this allocation request.
+	 */
+	enum zone_type highest_zoneidx;
+	bool spread_dirty_pages;
+	/* Only flags that are global to the whole allocation go here. */
+	unsigned int alloc_flags;
+};
+
+/*
+ * This function returns the order of a free page in the buddy system. In
+ * general, page_zone(page)->lock must be held by the caller to prevent the
+ * page from being allocated in parallel and returning garbage as the order.
+ * If a caller does not hold page_zone(page)->lock, it must guarantee that the
+ * page cannot be allocated or merged in parallel. Alternatively, it must
+ * handle invalid values gracefully, and use buddy_order_unsafe() below.
+ */
+static inline unsigned int buddy_order(struct page *page)
+{
+	/* PageBuddy() must be checked by the caller */
+	return page_private(page);
+}
+
+/*
+ * Like buddy_order(), but for callers who cannot afford to hold the zone lock.
+ * PageBuddy() should be checked first by the caller to minimize race window,
+ * and invalid values must be handled gracefully.
+ *
+ * READ_ONCE is used so that if the caller assigns the result into a local
+ * variable and e.g. tests it for valid range before using, the compiler cannot
+ * decide to remove the variable and inline the page_private(page) multiple
+ * times, potentially observing different values in the tests and the actual
+ * use of the result.
+ */
+#define buddy_order_unsafe(page)	READ_ONCE(page_private(page))
+
+/*
+ * This function checks whether a page is free && is the buddy
+ * we can coalesce a page and its buddy if
+ * (a) the buddy is not in a hole (check before calling!) &&
+ * (b) the buddy is in the buddy system &&
+ * (c) a page and its buddy have the same order &&
+ * (d) a page and its buddy are in the same zone.
+ *
+ * For recording whether a page is in the buddy system, we set PageBuddy.
+ * Setting, clearing, and testing PageBuddy is serialized by zone->lock.
+ *
+ * For recording page's order, we use page_private(page).
+ */
+static inline bool page_is_buddy(struct page *page, struct page *buddy,
+				 unsigned int order)
+{
+	if (!page_is_guard(buddy) && !PageBuddy(buddy))
+		return false;
+
+	if (buddy_order(buddy) != order)
+		return false;
+
+	/*
+	 * zone check is done late to avoid uselessly calculating
+	 * zone/node ids for pages that could never merge.
+	 */
+	if (page_zone_id(page) != page_zone_id(buddy))
+		return false;
+
+	VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
+
+	return true;
+}
+
+/*
+ * Locate the struct page for both the matching buddy in our
+ * pair (buddy1) and the combined O(n+1) page they form (page).
+ *
+ * 1) Any buddy B1 will have an order O twin B2 which satisfies
+ * the following equation:
+ *     B2 = B1 ^ (1 << O)
+ * For example, if the starting buddy (buddy2) is #8 its order
+ * 1 buddy is #10:
+ *     B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
+ *
+ * 2) Any buddy B will have an order O+1 parent P which
+ * satisfies the following equation:
+ *     P = B & ~(1 << O)
+ *
+ * Assumption: *_mem_map is contiguous at least up to MAX_PAGE_ORDER
+ */
+static inline unsigned long
+__find_buddy_pfn(unsigned long page_pfn, unsigned int order)
+{
+	return page_pfn ^ (1 << order);
+}
+
+/*
+ * Find the buddy of @page and validate it.
+ * @page: The input page
+ * @pfn: The pfn of the page, it saves a call to page_to_pfn() when the
+ *       function is used in the performance-critical __free_one_page().
+ * @order: The order of the page
+ * @buddy_pfn: The output pointer to the buddy pfn, it also saves a call to
+ *             page_to_pfn().
+ *
+ * The found buddy can be a non PageBuddy, out of @page's zone, or its order is
+ * not the same as @page. The validation is necessary before use it.
+ *
+ * Return: the found buddy page or NULL if not found.
+ */
+static inline struct page *find_buddy_page_pfn(struct page *page,
+			unsigned long pfn, unsigned int order, unsigned long *buddy_pfn)
+{
+	unsigned long __buddy_pfn = __find_buddy_pfn(pfn, order);
+	struct page *buddy;
+
+	buddy = page + (__buddy_pfn - pfn);
+	if (buddy_pfn)
+		*buddy_pfn = __buddy_pfn;
+
+	if (page_is_buddy(page, buddy, order))
+		return buddy;
+	return NULL;
+}
+
+extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
+				unsigned long end_pfn, struct zone *zone);
+
+static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn,
+				unsigned long end_pfn, struct zone *zone)
+{
+	if (zone->contiguous)
+		return pfn_to_page(start_pfn);
+
+	return __pageblock_pfn_to_page(start_pfn, end_pfn, zone);
+}
+
+extern void __free_pages_core(struct page *page, unsigned int order,
+		enum meminit_context context);
+
+void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags,
+		     unsigned int alloc_flags);
+extern bool free_pages_prepare(struct page *page, unsigned int order);
+
+extern int user_min_free_kbytes;
+
+struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, int nid,
+		nodemask_t *nodemask, unsigned int alloc_flags);
+#define __alloc_frozen_pages(...) \
+	alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__))
+void free_frozen_pages(struct page *page, unsigned int order);
+void free_unref_folios(struct folio_batch *fbatch);
+
+#ifdef CONFIG_NUMA
+struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order);
+#else
+static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order)
+{
+	return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL,
+					   ALLOC_DEFAULT);
+}
+#endif
+
+#define alloc_frozen_pages(...) \
+	alloc_hooks(alloc_frozen_pages_noprof(__VA_ARGS__))
+
+struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order);
+#define alloc_frozen_pages_nolock(...) \
+	alloc_hooks(alloc_frozen_pages_nolock_noprof(__VA_ARGS__))
+void free_frozen_pages_nolock(struct page *page, unsigned int order);
+
+struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, int preferred_nid,
+		nodemask_t *nodemask, unsigned int alloc_flags);
+#define __alloc_pages(...)			alloc_hooks(__alloc_pages_noprof(__VA_ARGS__))
+
+extern void zone_pcp_reset(struct zone *zone);
+extern void zone_pcp_disable(struct zone *zone);
+extern void zone_pcp_enable(struct zone *zone);
+extern void zone_pcp_init(struct zone *zone);
+
+enum fallback_result {
+	/* Found suitable migratetype, *mt_out is valid. */
+	FALLBACK_FOUND,
+	/* No fallback found in requested order. */
+	FALLBACK_EMPTY,
+	/* Passed @claimable, but claiming whole block is a bad idea. */
+	FALLBACK_NOCLAIM,
+};
+enum fallback_result
+find_suitable_fallback(struct free_area *area, unsigned int order,
+		       int migratetype, bool claimable, int *mt_out);
+
+static inline bool free_area_empty(struct free_area *area, int migratetype)
+{
+	return list_empty(&area->free_list[migratetype]);
+}
+
+/* Convert GFP flags to their corresponding migrate type */
+#define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE)
+#define GFP_MOVABLE_SHIFT 3
+
+static inline int gfp_migratetype(const gfp_t gfp_flags)
+{
+	VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK);
+	BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE);
+	BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE);
+	BUILD_BUG_ON((___GFP_RECLAIMABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_RECLAIMABLE);
+	BUILD_BUG_ON(((___GFP_MOVABLE | ___GFP_RECLAIMABLE) >>
+		      GFP_MOVABLE_SHIFT) != MIGRATE_HIGHATOMIC);
+
+	if (unlikely(page_group_by_mobility_disabled))
+		return MIGRATE_UNMOVABLE;
+
+	/* Group based on mobility */
+	return (__force unsigned long)(gfp_flags & GFP_MOVABLE_MASK) >> GFP_MOVABLE_SHIFT;
+}
+#undef GFP_MOVABLE_MASK
+#undef GFP_MOVABLE_SHIFT
+
+bool decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp);
+void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp);
+void drain_all_pages(struct zone *zone);
+
+void page_alloc_init_cpuhp(void);
+void page_alloc_sysctl_init(void);
+
+#endif /* __MM_PAGE_ALLOC_H */
diff --git a/mm/page_frag_cache.c b/mm/page_frag_cache.c
index d2423f30577e4..e63efe78b7d4b 100644
--- a/mm/page_frag_cache.c
+++ b/mm/page_frag_cache.c
@@ -18,7 +18,7 @@
 #include <linux/init.h>
 #include <linux/mm.h>
 #include <linux/page_frag_cache.h>
-#include "internal.h"
+#include "page_alloc.h"
 
 static unsigned long encoded_page_create(struct page *page, unsigned int order,
 					 bool pfmemalloc)
@@ -57,10 +57,10 @@ static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
 	gfp_mask = (gfp_mask & ~__GFP_DIRECT_RECLAIM) |  __GFP_COMP |
 		   __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC;
 	page = __alloc_pages(gfp_mask, PAGE_FRAG_CACHE_MAX_ORDER,
-			     numa_mem_id(), NULL);
+			     numa_mem_id(), NULL, ALLOC_DEFAULT);
 #endif
 	if (unlikely(!page)) {
-		page = __alloc_pages(gfp, 0, numa_mem_id(), NULL);
+		page = __alloc_pages(gfp, 0, numa_mem_id(), NULL, ALLOC_DEFAULT);
 		order = 0;
 	}
 
diff --git a/mm/page_io.c b/mm/page_io.c
index b23f494fcc83d..e4fa7ffffe8bb 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -28,54 +28,6 @@
 #include "swap.h"
 #include "swap_table.h"
 
-static void __end_swap_bio_write(struct bio *bio)
-{
-	struct folio *folio = bio_first_folio_all(bio);
-
-	if (bio->bi_status) {
-		/*
-		 * We failed to write the page out to swap-space.
-		 * Re-dirty the page in order to avoid it being reclaimed.
-		 * Also print a dire warning that things will go BAD (tm)
-		 * very quickly.
-		 *
-		 * Also clear PG_reclaim to avoid folio_rotate_reclaimable()
-		 */
-		folio_mark_dirty(folio);
-		pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n",
-				     MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
-				     (unsigned long long)bio->bi_iter.bi_sector);
-		folio_clear_reclaim(folio);
-	}
-	folio_end_writeback(folio);
-}
-
-static void end_swap_bio_write(struct bio *bio)
-{
-	__end_swap_bio_write(bio);
-	bio_put(bio);
-}
-
-static void __end_swap_bio_read(struct bio *bio)
-{
-	struct folio *folio = bio_first_folio_all(bio);
-
-	if (bio->bi_status) {
-		pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n",
-				     MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
-				     (unsigned long long)bio->bi_iter.bi_sector);
-	} else {
-		folio_mark_uptodate(folio);
-	}
-	folio_unlock(folio);
-}
-
-static void end_swap_bio_read(struct bio *bio)
-{
-	__end_swap_bio_read(bio);
-	bio_put(bio);
-}
-
 int generic_swapfile_activate(struct swap_info_struct *sis,
 				struct file *swap_file,
 				sector_t *span)
@@ -248,7 +200,7 @@ static void swap_zeromap_folio_clear(struct folio *folio)
  * We may have stale swap cache pages in memory: notice
  * them here and get rid of the unnecessary final write.
  */
-int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug)
+int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio)
 {
 	int ret = 0;
 
@@ -295,39 +247,44 @@ int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug)
 	}
 	rcu_read_unlock();
 
-	__swap_writepage(folio, swap_plug);
+	__swap_writepage(ctx, folio);
 	return 0;
 out_unlock:
 	folio_unlock(folio);
 	return ret;
 }
 
-static inline void count_swpout_vm_event(struct folio *folio)
+#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
+static struct cgroup_subsys_state *folio_memcg_blkg_css(struct folio *folio)
 {
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
-	if (unlikely(folio_test_pmd_mappable(folio))) {
-		count_memcg_folio_events(folio, THP_SWPOUT, 1);
-		count_vm_event(THP_SWPOUT);
+	return cgroup_e_css(folio_memcg(folio)->css.cgroup, &io_cgrp_subsys);
+}
+
+static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio)
+{
+	bool can_merge = true;
+
+	if (folio_memcg_charged(folio) != folio_memcg_charged(prev_folio))
+		return false;
+	if (folio_memcg_charged(folio)) {
+		rcu_read_lock();
+		if (folio_memcg_blkg_css(folio) !=
+		    folio_memcg_blkg_css(prev_folio))
+			can_merge = false;
+		rcu_read_unlock();
 	}
-#endif
-	count_mthp_stat(folio_order(folio), MTHP_STAT_SWPOUT);
-	count_memcg_folio_events(folio, PSWPOUT, folio_nr_pages(folio));
-	count_vm_events(PSWPOUT, folio_nr_pages(folio));
+	return can_merge;
 }
 
-#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
 static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio)
 {
 	struct cgroup_subsys_state *css;
-	struct mem_cgroup *memcg;
 
 	if (!folio_memcg_charged(folio))
 		return;
-
 	rcu_read_lock();
-	memcg = folio_memcg(folio);
-	css = cgroup_e_css(memcg->css.cgroup, &io_cgrp_subsys);
-	if (!css || !css_tryget(css))
+	css = folio_memcg_blkg_css(folio);
+	if (css && !css_tryget(css))
 		css = NULL;
 	rcu_read_unlock();
 
@@ -336,11 +293,18 @@ static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio)
 		css_put(css);
 }
 #else
+static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio)
+{
+	return true;
+}
 #define bio_associate_blkg_from_page(bio, folio)		do { } while (0)
 #endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */
 
 struct swap_iocb {
-	struct kiocb		iocb;
+	union {
+		struct kiocb	iocb;
+		struct bio	bio;
+	};
 	struct bio_vec		bvecs[SWAP_CLUSTER_MAX];
 	int			nr_bvecs;
 	int			len;
@@ -360,168 +324,65 @@ int sio_pool_init(void)
 	return 0;
 }
 
-static void sio_write_complete(struct kiocb *iocb, long ret)
+static bool swap_can_merge(struct swap_io_ctx *ctx, struct folio *folio,
+		int rw)
 {
-	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);
-	struct page *page = sio->bvecs[0].bv_page;
-	int p;
-
-	if (ret != sio->len) {
-		/*
-		 * In the case of swap-over-nfs, this can be a
-		 * temporary failure if the system has limited
-		 * memory for allocating transmit buffers.
-		 * Mark the page dirty and avoid
-		 * folio_rotate_reclaimable but rate-limit the
-		 * messages.
-		 */
-		pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n",
-				   ret, swap_dev_pos(page_swap_entry(page)));
-		for (p = 0; p < sio->nr_bvecs; p++) {
-			page = sio->bvecs[p].bv_page;
-			set_page_dirty(page);
-			ClearPageReclaim(page);
-		}
-	}
-
-	for (p = 0; p < sio->nr_bvecs; p++)
-		end_page_writeback(sio->bvecs[p].bv_page);
+	struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
+	struct bio_vec *last_bv = &ctx->sio->bvecs[ctx->sio->nr_bvecs - 1];
+	struct folio *prev_folio = bvec_folio(last_bv);
+	size_t prev_folio_size = folio_size(prev_folio);
 
-	mempool_free(sio, sio_pool);
+	if (ctx->sis != sis)
+		return false;
+	return sis->ops->can_merge(folio, prev_folio, prev_folio_size, rw);
 }
 
-static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug)
+static void swap_add_folio(struct swap_io_ctx *ctx, struct folio *folio, int rw)
 {
-	struct swap_iocb *sio = swap_plug ? *swap_plug : NULL;
 	struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
-	struct file *swap_file = sis->swap_file;
-	loff_t pos = swap_dev_pos(folio->swap);
-
-	count_swpout_vm_event(folio);
-	folio_start_writeback(folio);
-	folio_unlock(folio);
-	if (sio) {
-		if (sio->iocb.ki_filp != swap_file ||
-		    sio->iocb.ki_pos + sio->len != pos) {
-			swap_write_unplug(sio);
-			sio = NULL;
-		}
+	struct swap_iocb *sio = ctx->sio;
+
+	if (sio && !swap_can_merge(ctx, folio, rw)) {
+		if (rw == WRITE)
+			swap_write_submit(ctx);
+		else
+			swap_read_submit(ctx);
+		sio = ctx->sio;
 	}
+
 	if (!sio) {
-		sio = mempool_alloc(sio_pool, GFP_NOIO);
-		init_sync_kiocb(&sio->iocb, swap_file);
-		sio->iocb.ki_complete = sio_write_complete;
-		sio->iocb.ki_pos = pos;
+		ctx->sis = sis;
+		ctx->sio = sio = mempool_alloc(sio_pool, GFP_NOIO);
 		sio->nr_bvecs = 0;
 		sio->len = 0;
 	}
 	bvec_set_folio(&sio->bvecs[sio->nr_bvecs], folio, folio_size(folio), 0);
 	sio->len += folio_size(folio);
-	sio->nr_bvecs += 1;
-	if (sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) || !swap_plug) {
-		swap_write_unplug(sio);
-		sio = NULL;
+	if (++sio->nr_bvecs == ARRAY_SIZE(sio->bvecs)) {
+		if (rw == WRITE)
+			swap_write_submit(ctx);
+		else
+			swap_read_submit(ctx);
 	}
-	if (swap_plug)
-		*swap_plug = sio;
-}
-
-static void swap_writepage_bdev_sync(struct folio *folio,
-		struct swap_info_struct *sis)
-{
-	struct bio_vec bv;
-	struct bio bio;
-
-	bio_init(&bio, sis->bdev, &bv, 1, REQ_OP_WRITE | REQ_SWAP);
-	bio.bi_iter.bi_sector = swap_folio_sector(folio);
-	bio_add_folio_nofail(&bio, folio, folio_size(folio), 0);
-
-	bio_associate_blkg_from_page(&bio, folio);
-	count_swpout_vm_event(folio);
-
-	folio_start_writeback(folio);
-	folio_unlock(folio);
-
-	submit_bio_wait(&bio);
-	__end_swap_bio_write(&bio);
 }
 
-static void swap_writepage_bdev_async(struct folio *folio,
-		struct swap_info_struct *sis)
+void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio)
 {
-	struct bio *bio;
+	VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio);
 
-	bio = bio_alloc(sis->bdev, 1, REQ_OP_WRITE | REQ_SWAP, GFP_NOIO);
-	bio->bi_iter.bi_sector = swap_folio_sector(folio);
-	bio->bi_end_io = end_swap_bio_write;
-	bio_add_folio_nofail(bio, folio, folio_size(folio), 0);
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+	if (unlikely(folio_test_pmd_mappable(folio))) {
+		count_memcg_folio_events(folio, THP_SWPOUT, 1);
+		count_vm_event(THP_SWPOUT);
+	}
+#endif
+	count_mthp_stat(folio_order(folio), MTHP_STAT_SWPOUT);
+	count_memcg_folio_events(folio, PSWPOUT, folio_nr_pages(folio));
+	count_vm_events(PSWPOUT, folio_nr_pages(folio));
 
-	bio_associate_blkg_from_page(bio, folio);
-	count_swpout_vm_event(folio);
 	folio_start_writeback(folio);
 	folio_unlock(folio);
-	submit_bio(bio);
-}
-
-void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug)
-{
-	struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
-
-	VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio);
-	/*
-	 * ->flags can be updated non-atomically,
-	 * but that will never affect SWP_FS_OPS, so the data_race
-	 * is safe.
-	 */
-	if (data_race(sis->flags & SWP_FS_OPS))
-		swap_writepage_fs(folio, swap_plug);
-	/*
-	 * ->flags can be updated non-atomically,
-	 * but that will never affect SWP_SYNCHRONOUS_IO, so the data_race
-	 * is safe.
-	 */
-	else if (data_race(sis->flags & SWP_SYNCHRONOUS_IO))
-		swap_writepage_bdev_sync(folio, sis);
-	else
-		swap_writepage_bdev_async(folio, sis);
-}
-
-void swap_write_unplug(struct swap_iocb *sio)
-{
-	struct iov_iter from;
-	struct address_space *mapping = sio->iocb.ki_filp->f_mapping;
-	int ret;
-
-	iov_iter_bvec(&from, ITER_SOURCE, sio->bvecs, sio->nr_bvecs, sio->len);
-	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);
-	if (ret != -EIOCBQUEUED)
-		sio_write_complete(&sio->iocb, ret);
-}
-
-static void sio_read_complete(struct kiocb *iocb, long ret)
-{
-	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);
-	int p;
-
-	if (ret == sio->len) {
-		for (p = 0; p < sio->nr_bvecs; p++) {
-			struct folio *folio = bvec_folio(&sio->bvecs[p]);
-
-			count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN);
-			count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio));
-			folio_mark_uptodate(folio);
-			folio_unlock(folio);
-		}
-		count_vm_events(PSWPIN, sio->len >> PAGE_SHIFT);
-	} else {
-		for (p = 0; p < sio->nr_bvecs; p++) {
-			struct folio *folio = bvec_folio(&sio->bvecs[p]);
-
-			folio_unlock(folio);
-		}
-		pr_alert_ratelimited("Read-error on swap-device\n");
-	}
-	mempool_free(sio, sio_pool);
+	swap_add_folio(ctx, folio, WRITE);
 }
 
 /*
@@ -587,78 +448,7 @@ static bool swap_read_folio_zeromap(struct folio *folio)
 	return true;
 }
 
-static void swap_read_folio_fs(struct folio *folio, struct swap_iocb **plug)
-{
-	struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
-	struct swap_iocb *sio = NULL;
-	loff_t pos = swap_dev_pos(folio->swap);
-
-	if (plug)
-		sio = *plug;
-	if (sio) {
-		if (sio->iocb.ki_filp != sis->swap_file ||
-		    sio->iocb.ki_pos + sio->len != pos) {
-			swap_read_unplug(sio);
-			sio = NULL;
-		}
-	}
-	if (!sio) {
-		sio = mempool_alloc(sio_pool, GFP_KERNEL);
-		init_sync_kiocb(&sio->iocb, sis->swap_file);
-		sio->iocb.ki_pos = pos;
-		sio->iocb.ki_complete = sio_read_complete;
-		sio->nr_bvecs = 0;
-		sio->len = 0;
-	}
-	bvec_set_folio(&sio->bvecs[sio->nr_bvecs], folio, folio_size(folio), 0);
-	sio->len += folio_size(folio);
-	sio->nr_bvecs += 1;
-	if (sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) || !plug) {
-		swap_read_unplug(sio);
-		sio = NULL;
-	}
-	if (plug)
-		*plug = sio;
-}
-
-static void swap_read_folio_bdev_sync(struct folio *folio,
-		struct swap_info_struct *sis)
-{
-	struct bio_vec bv;
-	struct bio bio;
-
-	bio_init(&bio, sis->bdev, &bv, 1, REQ_OP_READ);
-	bio.bi_iter.bi_sector = swap_folio_sector(folio);
-	bio_add_folio_nofail(&bio, folio, folio_size(folio), 0);
-	/*
-	 * Keep this task valid during swap readpage because the oom killer may
-	 * attempt to access it in the page fault retry time check.
-	 */
-	get_task_struct(current);
-	count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN);
-	count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio));
-	count_vm_events(PSWPIN, folio_nr_pages(folio));
-	submit_bio_wait(&bio);
-	__end_swap_bio_read(&bio);
-	put_task_struct(current);
-}
-
-static void swap_read_folio_bdev_async(struct folio *folio,
-		struct swap_info_struct *sis)
-{
-	struct bio *bio;
-
-	bio = bio_alloc(sis->bdev, 1, REQ_OP_READ, GFP_KERNEL);
-	bio->bi_iter.bi_sector = swap_folio_sector(folio);
-	bio->bi_end_io = end_swap_bio_read;
-	bio_add_folio_nofail(bio, folio, folio_size(folio), 0);
-	count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN);
-	count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio));
-	count_vm_events(PSWPIN, folio_nr_pages(folio));
-	submit_bio(bio);
-}
-
-void swap_read_folio(struct folio *folio, struct swap_iocb **plug)
+void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio)
 {
 	struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
 	bool synchronous = sis->flags & SWP_SYNCHRONOUS_IO;
@@ -691,14 +481,7 @@ void swap_read_folio(struct folio *folio, struct swap_iocb **plug)
 
 	/* We have to read from slower devices. Increase zswap protection. */
 	zswap_folio_swapin(folio);
-
-	if (data_race(sis->flags & SWP_FS_OPS)) {
-		swap_read_folio_fs(folio, plug);
-	} else if (synchronous) {
-		swap_read_folio_bdev_sync(folio, sis);
-	} else {
-		swap_read_folio_bdev_async(folio, sis);
-	}
+	swap_add_folio(ctx, folio, READ);
 
 finish:
 	if (workingset) {
@@ -708,14 +491,231 @@ void swap_read_folio(struct folio *folio, struct swap_iocb **plug)
 	delayacct_swapin_end();
 }
 
-void __swap_read_unplug(struct swap_iocb *sio)
+static void swap_write_end(struct swap_iocb *sio, bool failed)
+{
+	int p;
+
+	for (p = 0; p < sio->nr_bvecs; p++) {
+		struct page *page = sio->bvecs[p].bv_page;
+
+		if (failed) {
+			set_page_dirty(page);
+			ClearPageReclaim(page);
+		}
+		end_page_writeback(page);
+	}
+	mempool_free(sio, sio_pool);
+}
+
+static void swap_fs_write_complete(struct kiocb *iocb, long ret)
+{
+	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);
+	bool failed = ret != sio->len;
+
+	if (failed) {
+		struct page *page = sio->bvecs[0].bv_page;
+
+		/*
+		 * In the case of swap-over-nfs, this can be a temporary failure
+		 * if the system has limited memory for allocating transmit
+		 * buffers.  Mark the page dirty and avoid
+		 * folio_rotate_reclaimable but rate-limit the messages.
+		 */
+		pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n",
+				   ret, swap_dev_pos(page_swap_entry(page)));
+	}
+
+	swap_write_end(sio, failed);
+}
+
+static void end_swap_bio_write(struct bio *bio)
+{
+	struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio);
+	bool failed = !!bio->bi_status;
+
+	if (failed)
+		pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n",
+				     MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
+				     (unsigned long long)bio->bi_iter.bi_sector);
+	bio_uninit(bio);
+	swap_write_end(sio, failed);
+}
+
+static void swap_read_end(struct swap_iocb *sio, bool failed)
+{
+	int p;
+
+	for (p = 0; p < sio->nr_bvecs; p++) {
+		struct folio *folio = bvec_folio(&sio->bvecs[p]);
+
+		if (!failed) {
+			count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN);
+			count_memcg_folio_events(folio, PSWPIN,
+					folio_nr_pages(folio));
+			folio_mark_uptodate(folio);
+		}
+		folio_unlock(folio);
+	}
+
+	if (!failed)
+		count_vm_events(PSWPIN, sio->len >> PAGE_SHIFT);
+
+	mempool_free(sio, sio_pool);
+}
+
+static void swap_fs_read_complete(struct kiocb *iocb, long ret)
+{
+	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);
+	bool failed = ret != sio->len;
+
+	if (failed)
+		pr_alert_ratelimited("Read-error on swap-device\n");
+	swap_read_end(sio, failed);
+}
+
+static void swap_bio_read_end_io(struct bio *bio)
+{
+	struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio);
+	bool failed = !!bio->bi_status;
+
+	if (failed)
+		pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n",
+				     MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
+				     (unsigned long long)bio->bi_iter.bi_sector);
+	bio_uninit(bio);
+	swap_read_end(sio, failed);
+}
+
+static void swap_bdev_submit_write(struct swap_io_ctx *ctx)
+{
+	struct swap_iocb *sio = ctx->sio;
+	struct bio *bio = &sio->bio;
+
+	bio_init(bio, ctx->sis->bdev, sio->bvecs, ARRAY_SIZE(sio->bvecs),
+			REQ_OP_WRITE | REQ_SWAP);
+	bio->bi_iter.bi_size = sio->len;
+	bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio));
+	bio_associate_blkg_from_page(bio, bio_first_folio_all(bio));
+
+	if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) {
+		submit_bio_wait(bio);
+		end_swap_bio_write(bio);
+	} else {
+		bio->bi_end_io = end_swap_bio_write;
+		submit_bio(bio);
+	}
+}
+
+static void swap_bdev_submit_read(struct swap_io_ctx *ctx)
+{
+	struct swap_iocb *sio = ctx->sio;
+	struct bio *bio = &sio->bio;
+
+	bio_init(bio, ctx->sis->bdev, sio->bvecs, ARRAY_SIZE(sio->bvecs),
+			REQ_OP_READ);
+	bio->bi_iter.bi_size = sio->len;
+	bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio));
+
+	if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) {
+		/*
+		 * Keep this task valid during swap readpage because the oom
+		 * killer may attempt to access it in the page fault retry
+		 * time check.
+		 */
+		get_task_struct(current);
+		submit_bio_wait(bio);
+		swap_bio_read_end_io(bio);
+		put_task_struct(current);
+	} else {
+		bio->bi_end_io = swap_bio_read_end_io;
+		submit_bio(bio);
+	}
+}
+
+static bool swap_bdev_can_merge(struct folio *folio, struct folio *prev_folio,
+		size_t prev_folio_size, int rw)
 {
-	struct iov_iter from;
-	struct address_space *mapping = sio->iocb.ki_filp->f_mapping;
+	if (swap_folio_sector(folio) !=
+	    swap_folio_sector(prev_folio) + (prev_folio_size >> SECTOR_SHIFT))
+		return false;
+	if (rw == WRITE && !folio_blkg_can_merge(folio, prev_folio))
+		return false;
+	return true;
+}
+
+const struct swap_ops swap_bdev_ops = {
+	.submit_write		= swap_bdev_submit_write,
+	.submit_read		= swap_bdev_submit_read,
+	.can_merge		= swap_bdev_can_merge,
+};
+
+static void swap_fs_submit(struct swap_io_ctx *ctx, int rw)
+{
+	struct swap_iocb *sio = ctx->sio;
+	struct iov_iter iter;
 	int ret;
 
-	iov_iter_bvec(&from, ITER_DEST, sio->bvecs, sio->nr_bvecs, sio->len);
-	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);
+	init_sync_kiocb(&sio->iocb, ctx->sis->swap_file);
+	sio->iocb.ki_pos = swap_dev_pos(bvec_folio(&sio->bvecs[0])->swap);
+	if (rw == WRITE)
+		sio->iocb.ki_complete = swap_fs_write_complete;
+	else
+		sio->iocb.ki_complete = swap_fs_read_complete;
+
+	iov_iter_bvec(&iter, rw == WRITE ? ITER_SOURCE : ITER_DEST,
+			sio->bvecs, sio->nr_bvecs, sio->len);
+	ret = sio->iocb.ki_filp->f_mapping->a_ops->swap_rw(&sio->iocb, &iter);
 	if (ret != -EIOCBQUEUED)
-		sio_read_complete(&sio->iocb, ret);
+		sio->iocb.ki_complete(&sio->iocb, ret);
+}
+
+static void swap_fs_submit_write(struct swap_io_ctx *ctx)
+{
+	swap_fs_submit(ctx, WRITE);
+}
+
+static void swap_fs_submit_read(struct swap_io_ctx *ctx)
+{
+	swap_fs_submit(ctx, READ);
+}
+
+static bool swap_fs_can_merge(struct folio *folio, struct folio *prev_folio,
+		size_t prev_folio_size, int rw)
+{
+	return swap_dev_pos(folio->swap) ==
+		swap_dev_pos(prev_folio->swap) + prev_folio_size;
+}
+
+static const struct swap_ops swap_fs_ops = {
+	.flags			= SWAP_OPS_F_REQUIRE_NOFS,
+	.submit_write		= swap_fs_submit_write,
+	.submit_read		= swap_fs_submit_read,
+	.can_merge		= swap_fs_can_merge,
+};
+
+int swap_fs_activate(struct swap_info_struct *sis)
+{
+	sis->ops = &swap_fs_ops;
+	return add_swap_extent(sis, 0, sis->max, 0);
+}
+EXPORT_SYMBOL_GPL(swap_fs_activate);
+
+void swap_write_submit(struct swap_io_ctx *ctx)
+{
+	if (!ctx->sio)
+		return;
+	count_vm_events(NRSWPOUT, 1);
+	ctx->sis->ops->submit_write(ctx);
+	ctx->sio = NULL;
+	ctx->sis = NULL;
+}
+
+void swap_read_submit(struct swap_io_ctx *ctx)
+{
+	if (!ctx->sio)
+		return;
+	count_vm_events(NRSWPIN, 1);
+	ctx->sis->ops->submit_read(ctx);
+	ctx->sio = NULL;
+	ctx->sis = NULL;
 }
diff --git a/mm/page_isolation.c b/mm/page_isolation.c
index 32ce8a7d9df35..e5dfc7bf49446 100644
--- a/mm/page_isolation.c
+++ b/mm/page_isolation.c
@@ -11,6 +11,7 @@
 #include <linux/page_owner.h>
 #include <linux/migrate.h>
 #include "internal.h"
+#include "page_alloc.h"
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/page_isolation.h>
diff --git a/mm/page_owner.c b/mm/page_owner.c
index 2dddcb6510aa1..fbbda7ba914ba 100644
--- a/mm/page_owner.c
+++ b/mm/page_owner.c
@@ -13,7 +13,7 @@
 #include <linux/memcontrol.h>
 #include <linux/sched/clock.h>
 
-#include "internal.h"
+#include "page_alloc.h"
 
 /*
  * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack)
@@ -54,6 +54,24 @@ struct stack_print_ctx {
 	u8 flags;
 };
 
+enum page_owner_print_mode {
+	PAGE_OWNER_PRINT_STACK,
+	PAGE_OWNER_PRINT_HANDLE,
+	PAGE_OWNER_PRINT_STACK_HANDLE,
+};
+
+static const char * const page_owner_print_mode_strings[] = {
+	[PAGE_OWNER_PRINT_STACK]	= "stack",
+	[PAGE_OWNER_PRINT_HANDLE]	= "handle",
+	[PAGE_OWNER_PRINT_STACK_HANDLE]	= "stack_handle",
+};
+
+struct page_owner_filter_state {
+	enum page_owner_print_mode print_mode;
+	nodemask_t nid_filter;
+	bool nid_filter_enabled;
+};
+
 static bool page_owner_enabled __initdata;
 DEFINE_STATIC_KEY_FALSE(page_owner_inited);
 
@@ -339,13 +357,13 @@ noinline void __set_page_owner(struct page *page, unsigned short order,
 	depot_stack_handle_t handle;
 
 	handle = save_stack(gfp_mask);
-	__update_page_owner_handle(page, handle, order, gfp_mask, -1,
+	__update_page_owner_handle(page, handle, order, gfp_mask, MR_NEVER,
 				   ts_nsec, current->pid, current->tgid,
 				   current->comm);
 	inc_stack_record_count(handle, gfp_mask, 1 << order);
 }
 
-void __folio_set_owner_migrate_reason(struct folio *folio, int reason)
+void __folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason)
 {
 	struct page_ext *page_ext = page_ext_get(&folio->page);
 	struct page_owner *page_owner;
@@ -422,6 +440,39 @@ void __folio_copy_owner(struct folio *newfolio, struct folio *old)
 	rcu_read_unlock();
 }
 
+/*
+ * Check if a page is a buddy page and advance @pfn past the entire buddy block.
+ * This safely reads the buddy order without the zone lock, which may cause us
+ * to skip less than the full buddy block, but that is acceptable for page owner
+ * iteration purposes.
+ *
+ * The lockless read of buddy_order_unsafe() can also return a garbage order if
+ * the page is concurrently allocated and PageBuddy is cleared between the check
+ * and the read. Clamp the advance at the next MAX_ORDER_NR_PAGES boundary so
+ * that a bogus order cannot carry @pfn into an unvalidated memory section,
+ * which would break callers that rely on boundary-aligned pfn_valid() checks.
+ *
+ * Return: true if the page was skipped (caller should continue its loop),
+ *         false if the page is not a buddy page and should be processed normally.
+ */
+static inline bool skip_buddy_pages(unsigned long *pfn, struct page *page)
+{
+	unsigned long order;
+
+	if (!PageBuddy(page))
+		return false;
+
+	order = buddy_order_unsafe(page);
+	if (order <= MAX_PAGE_ORDER) {
+		unsigned long new_pfn = *pfn + (1UL << order);
+		unsigned long boundary = ALIGN(*pfn + 1, MAX_ORDER_NR_PAGES);
+
+		*pfn = min(new_pfn, boundary) - 1;
+	}
+
+	return true;
+}
+
 void pagetypeinfo_showmixedcount_print(struct seq_file *m,
 				       pg_data_t *pgdat, struct zone *zone)
 {
@@ -461,14 +512,8 @@ void pagetypeinfo_showmixedcount_print(struct seq_file *m,
 			if (page_zone(page) != zone)
 				continue;
 
-			if (PageBuddy(page)) {
-				unsigned long freepage_order;
-
-				freepage_order = buddy_order_unsafe(page);
-				if (freepage_order <= MAX_PAGE_ORDER)
-					pfn += (1UL << freepage_order) - 1;
+			if (skip_buddy_pages(&pfn, page))
 				continue;
-			}
 
 			if (PageReserved(page))
 				continue;
@@ -505,14 +550,15 @@ void pagetypeinfo_showmixedcount_print(struct seq_file *m,
 	seq_putc(m, '\n');
 }
 
+#ifdef CONFIG_MEMCG
 /*
  * Looking for memcg information and print it out
  */
 static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret,
 					 struct page *page)
 {
-#ifdef CONFIG_MEMCG
 	unsigned long memcg_data;
+	struct obj_cgroup *objcg;
 	struct mem_cgroup *memcg;
 	bool online;
 	char name[80];
@@ -522,11 +568,14 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret,
 	if (!memcg_data || PageTail(page))
 		goto out_unlock;
 
-	if (memcg_data & MEMCG_DATA_OBJEXTS)
+	if (memcg_data & MEMCG_DATA_OBJEXTS) {
 		ret += scnprintf(kbuf + ret, count - ret,
 				"Slab cache page\n");
+		goto out_unlock;
+	}
 
-	memcg = page_memcg_check(page);
+	objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
+	memcg = objcg ? obj_cgroup_memcg(objcg) : NULL;
 	if (!memcg)
 		goto out_unlock;
 
@@ -534,29 +583,39 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret,
 	cgroup_name(memcg->css.cgroup, name, sizeof(name));
 	ret += scnprintf(kbuf + ret, count - ret,
 			"Charged %sto %smemcg %s\n",
-			PageMemcgKmem(page) ? "(via objcg) " : "",
+			(memcg_data & MEMCG_DATA_KMEM) ? "(via objcg) " : "",
 			online ? "" : "offline ",
 			name);
 out_unlock:
 	rcu_read_unlock();
-#endif /* CONFIG_MEMCG */
 
 	return ret;
 }
+#else
+static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret,
+					 struct page *page)
+{
+	return ret;
+}
+#endif
 
 static ssize_t
 print_page_owner(char __user *buf, size_t count, unsigned long pfn,
 		struct page *page, struct page_owner *page_owner,
-		depot_stack_handle_t handle)
+		depot_stack_handle_t handle,
+		struct page_owner_filter_state *state)
 {
 	int ret, pageblock_mt, page_mt;
 	char *kbuf;
+	enum page_owner_print_mode print_mode;
 
 	count = min_t(size_t, count, PAGE_SIZE);
 	kbuf = kmalloc(count, GFP_KERNEL);
 	if (!kbuf)
 		return -ENOMEM;
 
+	print_mode = state->print_mode;
+
 	ret = scnprintf(kbuf, count,
 			"Page allocated via order %u, mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu ns\n",
 			page_owner->order, page_owner->gfp_mask,
@@ -575,11 +634,20 @@ print_page_owner(char __user *buf, size_t count, unsigned long pfn,
 			migratetype_names[pageblock_mt],
 			&page->flags.f);
 
-	ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0);
-	if (ret >= count)
-		goto err;
+	if (print_mode != PAGE_OWNER_PRINT_HANDLE) {
+		ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0);
+		if (ret >= count)
+			goto err;
+	}
+
+	if (print_mode != PAGE_OWNER_PRINT_STACK) {
+		ret += scnprintf(kbuf + ret, count - ret, "handle: %u\n",
+				 handle);
+		if (ret >= count)
+			goto err;
+	}
 
-	if (page_owner->last_migrate_reason != -1) {
+	if (page_owner->last_migrate_reason != MR_NEVER) {
 		ret += scnprintf(kbuf + ret, count - ret,
 			"Page has been migrated, last migrate reason: %s\n",
 			migrate_reason_names[page_owner->last_migrate_reason]);
@@ -630,10 +698,10 @@ void __dump_page_owner(const struct page *page)
 	else
 		pr_alert("page_owner tracks the page as freed\n");
 
-	pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu, free_ts %llu\n",
+	pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu\n",
 		 page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask,
 		 page_owner->pid, page_owner->tgid, page_owner->comm,
-		 page_owner->ts_nsec, page_owner->free_ts_nsec);
+		 page_owner->ts_nsec);
 
 	handle = READ_ONCE(page_owner->handle);
 	if (!handle)
@@ -645,12 +713,13 @@ void __dump_page_owner(const struct page *page)
 	if (!handle) {
 		pr_alert("page_owner free stack trace missing\n");
 	} else {
-		pr_alert("page last free pid %d tgid %d stack trace:\n",
-			  page_owner->free_pid, page_owner->free_tgid);
+		pr_alert("page last free pid %d tgid %d ts %llu stack trace:\n",
+			  page_owner->free_pid, page_owner->free_tgid,
+			  page_owner->free_ts_nsec);
 		stack_depot_print(handle);
 	}
 
-	if (page_owner->last_migrate_reason != -1)
+	if (page_owner->last_migrate_reason != MR_NEVER)
 		pr_alert("page has been migrated, last migrate reason: %s\n",
 			migrate_reason_names[page_owner->last_migrate_reason]);
 	page_ext_put(page_ext);
@@ -664,6 +733,7 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos)
 	struct page_ext *page_ext;
 	struct page_owner *page_owner;
 	depot_stack_handle_t handle;
+	struct page_owner_filter_state *state = file->private_data;
 
 	if (!static_branch_unlikely(&page_owner_inited))
 		return -EINVAL;
@@ -697,13 +767,8 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos)
 		}
 
 		page = pfn_to_page(pfn);
-		if (PageBuddy(page)) {
-			unsigned long freepage_order = buddy_order_unsafe(page);
-
-			if (freepage_order <= MAX_PAGE_ORDER)
-				pfn += (1UL << freepage_order) - 1;
+		if (skip_buddy_pages(&pfn, page))
 			continue;
-		}
 
 		page_ext = page_ext_get(page);
 		if (unlikely(!page_ext))
@@ -740,15 +805,31 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos)
 		if (!handle)
 			goto ext_put_continue;
 
+		if (state->nid_filter_enabled) {
+			int nid;
+			memdesc_flags_t page_flags = READ_ONCE(page->flags);
+
+			/*
+			 * Bypass PF_POISONED_CHECK() in page_to_nid() to avoid
+			 * VM_BUG_ON when accessing poisoned pages.
+			 */
+			if (page_flags.f == PAGE_POISON_PATTERN)
+				goto ext_put_continue;
+			nid = memdesc_nid(&page_flags);
+			if (!node_isset(nid, state->nid_filter))
+				goto ext_put_continue;
+		}
+
 		/* Record the next PFN to read in the file offset */
 		*ppos = pfn + 1;
 
 		page_owner_tmp = *page_owner;
 		page_ext_put(page_ext);
 		return print_page_owner(buf, count, pfn, page,
-				&page_owner_tmp, handle);
+				&page_owner_tmp, handle, state);
 ext_put_continue:
 		page_ext_put(page_ext);
+		cond_resched();
 	}
 
 	return 0;
@@ -798,20 +879,8 @@ static void init_pages_in_zone(struct zone *zone)
 			if (page_zone(page) != zone)
 				continue;
 
-			/*
-			 * To avoid having to grab zone->lock, be a little
-			 * careful when reading buddy page order. The only
-			 * danger is that we skip too much and potentially miss
-			 * some early allocated pages, which is better than
-			 * heavy lock contention.
-			 */
-			if (PageBuddy(page)) {
-				unsigned long order = buddy_order_unsafe(page);
-
-				if (order > 0 && order <= MAX_PAGE_ORDER)
-					pfn += (1UL << order) - 1;
+			if (skip_buddy_pages(&pfn, page))
 				continue;
-			}
 
 			if (PageReserved(page))
 				continue;
@@ -826,7 +895,7 @@ static void init_pages_in_zone(struct zone *zone)
 
 			/* Found early allocated page */
 			__update_page_owner_handle(page, early_handle, 0, 0,
-						   -1, local_clock(), current->pid,
+						   MR_NEVER, local_clock(), current->pid,
 						   current->tgid, current->comm);
 			count++;
 ext_put_continue:
@@ -847,7 +916,113 @@ static void init_early_allocated_pages(void)
 		init_pages_in_zone(zone);
 }
 
+static int page_owner_open(struct inode *inode, struct file *file)
+{
+	struct page_owner_filter_state *state;
+
+	state = kzalloc_obj(*state);
+	if (!state)
+		return -ENOMEM;
+
+	state->print_mode = PAGE_OWNER_PRINT_STACK;
+	nodes_clear(state->nid_filter);
+	state->nid_filter_enabled = false;
+	file->private_data = state;
+	return 0;
+}
+
+static int page_owner_release(struct inode *inode, struct file *file)
+{
+	kfree(file->private_data);
+	return 0;
+}
+
+static ssize_t page_owner_write(struct file *file,
+				 const char __user *buf,
+				 size_t count, loff_t *ppos)
+{
+	char *kbuf;
+	char *orig;
+	char *token;
+	int ret;
+	struct page_owner_filter_state *state = file->private_data;
+	enum page_owner_print_mode new_print_mode;
+	nodemask_t new_nid_filter;
+	bool new_nid_filter_enabled;
+
+	/*
+	 * Maximum input length for filter commands:
+	 * - 32: print_mode command max length is 17 ("mode=stack_handle")
+	 *        with sufficient buffer
+	 * - 6 * MAX_NUMNODES: worst case for nid list
+	 *   Worst case per node: ",NNNNN" (comma + 5-digit node number) = 6 bytes
+	 */
+	if (count > 32 + 6 * MAX_NUMNODES)
+		return -EINVAL;
+
+	kbuf = memdup_user_nul(buf, count);
+	if (IS_ERR(kbuf))
+		return PTR_ERR(kbuf);
+
+	orig = kbuf;
+
+	new_print_mode = state->print_mode;
+	new_nid_filter = state->nid_filter;
+	new_nid_filter_enabled = state->nid_filter_enabled;
+
+	while ((token = strsep(&kbuf, " \t\n")) != NULL) {
+		if (*token == '\0')
+			continue;
+
+		if (!strncmp(token, "mode=", 5)) {
+			ret = sysfs_match_string(page_owner_print_mode_strings,
+						token + 5);
+			if (ret < 0)
+				goto out_free;
+			new_print_mode = ret;
+		} else if (!strncmp(token, "nid=", 4)) {
+			ret = nodelist_parse(token + 4, new_nid_filter);
+			if (ret < 0)
+				goto out_free;
+
+			if (nodes_empty(new_nid_filter)) {
+				ret = -EINVAL;
+				goto out_free;
+			}
+
+			/*
+			 * We want to filter memory allocations by numa nodes, so make sure
+			 * that the specified nodes have memory.
+			 */
+			if (!nodes_subset(new_nid_filter, node_states[N_MEMORY])) {
+				ret = -EINVAL;
+				goto out_free;
+			}
+
+			new_nid_filter_enabled = true;
+		} else {
+			ret = -EINVAL;
+			goto out_free;
+		}
+	}
+
+	/* Commit all filter changes */
+	state->print_mode = new_print_mode;
+	state->nid_filter = new_nid_filter;
+	state->nid_filter_enabled = new_nid_filter_enabled;
+
+	ret = count;
+
+out_free:
+	kfree(orig);
+	return ret;
+}
+
 static const struct file_operations page_owner_fops = {
+	.owner		= THIS_MODULE,
+	.open		= page_owner_open,
+	.release	= page_owner_release,
+	.write		= page_owner_write,
 	.read		= read_page_owner,
 	.llseek		= lseek_page_owner,
 };
@@ -887,7 +1062,7 @@ static void *stack_next(struct seq_file *m, void *v, loff_t *ppos)
 	return stack;
 }
 
-static unsigned long page_owner_pages_threshold;
+static unsigned long pages_threshold;
 
 static int stack_print(struct seq_file *m, void *v)
 {
@@ -904,7 +1079,7 @@ static int stack_print(struct seq_file *m, void *v)
 	nr_base_pages = refcount_read(&stack_record->count) - 1;
 
 	if (ctx->flags & STACK_PRINT_FLAG_PAGES &&
-	    (nr_base_pages < 1 || nr_base_pages < page_owner_pages_threshold))
+	    (nr_base_pages < 1 || nr_base_pages < pages_threshold))
 		return 0;
 
 	if (ctx->flags & STACK_PRINT_FLAG_STACK) {
@@ -926,16 +1101,16 @@ static void stack_stop(struct seq_file *m, void *v)
 {
 }
 
-static const struct seq_operations page_owner_stack_op = {
+static const struct seq_operations stack_op = {
 	.start	= stack_start,
 	.next	= stack_next,
 	.stop	= stack_stop,
 	.show	= stack_print
 };
 
-static int page_owner_stack_open(struct inode *inode, struct file *file)
+static int stack_open(struct inode *inode, struct file *file)
 {
-	int ret = seq_open_private(file, &page_owner_stack_op,
+	int ret = seq_open_private(file, &stack_op,
 				   sizeof(struct stack_print_ctx));
 
 	if (!ret) {
@@ -948,28 +1123,26 @@ static int page_owner_stack_open(struct inode *inode, struct file *file)
 	return ret;
 }
 
-static const struct file_operations page_owner_stack_fops = {
-	.open		= page_owner_stack_open,
+static const struct file_operations stack_fops = {
+	.open		= stack_open,
 	.read		= seq_read,
 	.llseek		= seq_lseek,
 	.release	= seq_release_private,
 };
 
-static int page_owner_threshold_get(void *data, u64 *val)
+static int threshold_get(void *data, u64 *val)
 {
-	*val = READ_ONCE(page_owner_pages_threshold);
+	*val = READ_ONCE(pages_threshold);
 	return 0;
 }
 
-static int page_owner_threshold_set(void *data, u64 val)
+static int threshold_set(void *data, u64 val)
 {
-	WRITE_ONCE(page_owner_pages_threshold, val);
+	WRITE_ONCE(pages_threshold, val);
 	return 0;
 }
 
-DEFINE_SIMPLE_ATTRIBUTE(page_owner_threshold_fops, &page_owner_threshold_get,
-			&page_owner_threshold_set, "%llu");
-
+DEFINE_SIMPLE_ATTRIBUTE(threshold_fops, &threshold_get, &threshold_set, "%llu\n");
 
 static int __init pageowner_init(void)
 {
@@ -980,22 +1153,22 @@ static int __init pageowner_init(void)
 		return 0;
 	}
 
-	debugfs_create_file("page_owner", 0400, NULL, NULL, &page_owner_fops);
+	debugfs_create_file("page_owner", 0600, NULL, NULL, &page_owner_fops);
 	dir = debugfs_create_dir("page_owner_stacks", NULL);
 	debugfs_create_file("show_stacks", 0400, dir,
 			    (void *)(STACK_PRINT_FLAG_STACK |
 				     STACK_PRINT_FLAG_PAGES),
-			     &page_owner_stack_fops);
+			     &stack_fops);
 	debugfs_create_file("show_handles", 0400, dir,
 			    (void *)(STACK_PRINT_FLAG_HANDLE |
 				     STACK_PRINT_FLAG_PAGES),
-			    &page_owner_stack_fops);
+			    &stack_fops);
 	debugfs_create_file("show_stacks_handles", 0400, dir,
 			    (void *)(STACK_PRINT_FLAG_STACK |
 				     STACK_PRINT_FLAG_HANDLE),
-			    &page_owner_stack_fops);
+			    &stack_fops);
 	debugfs_create_file("count_threshold", 0600, dir, NULL,
-			    &page_owner_threshold_fops);
+			    &threshold_fops);
 	return 0;
 }
 late_initcall(pageowner_init)
diff --git a/mm/page_reporting.c b/mm/page_reporting.c
index 7418f2e500bb4..9d21f9255779c 100644
--- a/mm/page_reporting.c
+++ b/mm/page_reporting.c
@@ -8,6 +8,7 @@
 #include <linux/delay.h>
 #include <linux/scatterlist.h>
 
+#include "page_alloc.h"
 #include "page_reporting.h"
 #include "internal.h"
 
@@ -173,11 +174,8 @@ page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone,
 	 * any pages that may have already been present from the previous
 	 * list processed. This should result in us reporting all pages on
 	 * an idle system in about 30 seconds.
-	 *
-	 * The division here should be cheap since PAGE_REPORTING_CAPACITY
-	 * should always be a power of 2.
 	 */
-	budget = DIV_ROUND_UP(area->nr_free, PAGE_REPORTING_CAPACITY * 16);
+	budget = DIV_ROUND_UP(area->nr_free, prdev->capacity * 16);
 
 	/* loop through free list adding unreported pages to sg list */
 	list_for_each_entry_safe(page, next, list, lru) {
@@ -222,10 +220,10 @@ page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone,
 		spin_unlock_irq(&zone->lock);
 
 		/* begin processing pages in local list */
-		err = prdev->report(prdev, sgl, PAGE_REPORTING_CAPACITY);
+		err = prdev->report(prdev, sgl, prdev->capacity);
 
 		/* reset offset since the full list was reported */
-		*offset = PAGE_REPORTING_CAPACITY;
+		*offset = prdev->capacity;
 
 		/* update budget to reflect call to report function */
 		budget--;
@@ -234,7 +232,7 @@ page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone,
 		spin_lock_irq(&zone->lock);
 
 		/* flush reported pages from the sg list */
-		page_reporting_drain(prdev, sgl, PAGE_REPORTING_CAPACITY, !err);
+		page_reporting_drain(prdev, sgl, prdev->capacity, !err);
 
 		/*
 		 * Reset next to first entry, the old next isn't valid
@@ -260,13 +258,13 @@ static int
 page_reporting_process_zone(struct page_reporting_dev_info *prdev,
 			    struct scatterlist *sgl, struct zone *zone)
 {
-	unsigned int order, mt, leftover, offset = PAGE_REPORTING_CAPACITY;
+	unsigned int order, mt, leftover, offset = prdev->capacity;
 	unsigned long watermark;
 	int err = 0;
 
 	/* Generate minimum watermark to be able to guarantee progress */
 	watermark = low_wmark_pages(zone) +
-		    (PAGE_REPORTING_CAPACITY << page_reporting_order);
+		    (prdev->capacity << page_reporting_order);
 
 	/*
 	 * Cancel request if insufficient free memory or if we failed
@@ -290,7 +288,7 @@ page_reporting_process_zone(struct page_reporting_dev_info *prdev,
 	}
 
 	/* report the leftover pages before going idle */
-	leftover = PAGE_REPORTING_CAPACITY - offset;
+	leftover = prdev->capacity - offset;
 	if (leftover) {
 		sgl = &sgl[offset];
 		err = prdev->report(prdev, sgl, leftover);
@@ -322,11 +320,11 @@ static void page_reporting_process(struct work_struct *work)
 	atomic_set(&prdev->state, state);
 
 	/* allocate scatterlist to store pages being reported on */
-	sgl = kmalloc_objs(*sgl, PAGE_REPORTING_CAPACITY);
+	sgl = kmalloc_objs(*sgl, prdev->capacity);
 	if (!sgl)
 		goto err_out;
 
-	sg_init_table(sgl, PAGE_REPORTING_CAPACITY);
+	sg_init_table(sgl, prdev->capacity);
 
 	for_each_zone(zone) {
 		err = page_reporting_process_zone(prdev, sgl, zone);
@@ -377,6 +375,9 @@ int page_reporting_register(struct page_reporting_dev_info *prdev)
 			page_reporting_order = pageblock_order;
 	}
 
+	if (!prdev->capacity || prdev->capacity > PAGE_REPORTING_CAPACITY)
+		prdev->capacity = PAGE_REPORTING_CAPACITY;
+
 	/* initialize state and work structures */
 	atomic_set(&prdev->state, PAGE_REPORTING_IDLE);
 	INIT_DELAYED_WORK(&prdev->work, &page_reporting_process);
diff --git a/mm/page_table_check.c b/mm/page_table_check.c
index 53a8997ec043e..3fb995e5d40d6 100644
--- a/mm/page_table_check.c
+++ b/mm/page_table_check.c
@@ -188,8 +188,8 @@ static inline bool softleaf_cached_writable(softleaf_t entry)
 static void page_table_check_pte_flags(pte_t pte)
 {
 	if (pte_present(pte)) {
-		WARN_ON_ONCE(pte_uffd_wp(pte) && pte_write(pte));
-	} else if (pte_swp_uffd_wp(pte)) {
+		WARN_ON_ONCE(pte_uffd(pte) && pte_write(pte));
+	} else if (pte_swp_uffd(pte)) {
 		const softleaf_t entry = softleaf_from_pte(pte);
 
 		WARN_ON_ONCE(softleaf_cached_writable(entry));
@@ -216,9 +216,9 @@ EXPORT_SYMBOL(__page_table_check_ptes_set);
 static inline void page_table_check_pmd_flags(pmd_t pmd)
 {
 	if (pmd_present(pmd)) {
-		if (pmd_uffd_wp(pmd))
+		if (pmd_uffd(pmd))
 			WARN_ON_ONCE(pmd_write(pmd));
-	} else if (pmd_swp_uffd_wp(pmd)) {
+	} else if (pmd_swp_uffd(pmd)) {
 		const softleaf_t entry = softleaf_from_pmd(pmd);
 
 		WARN_ON_ONCE(softleaf_cached_writable(entry));
diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c
index 2ccbabfb2cc17..d7670ba4147bf 100644
--- a/mm/page_vma_mapped.c
+++ b/mm/page_vma_mapped.c
@@ -107,7 +107,13 @@ static bool map_pte(struct page_vma_mapped_walk *pvmw, pmd_t *pmdvalp,
 static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr)
 {
 	unsigned long pfn;
-	pte_t ptent = ptep_get(pvmw->pte);
+	pte_t ptent;
+
+	if (is_vm_hugetlb_page(pvmw->vma))
+		ptent = huge_ptep_get(pvmw->vma->vm_mm, pvmw->address,
+				      pvmw->pte);
+	else
+		ptent = ptep_get(pvmw->pte);
 
 	if (pvmw->flags & PVMW_MIGRATION) {
 		const softleaf_t entry = softleaf_from_pte(ptent);
@@ -243,21 +249,31 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw)
 		 */
 		pmde = pmdp_get_lockless(pvmw->pmd);
 
-		if (pmd_trans_huge(pmde) || pmd_is_migration_entry(pmde)) {
+		if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
+		    (pmd_trans_huge(pmde) || pmd_is_migration_entry(pmde) ||
+		    pmd_is_device_private_entry(pmde))) {
 			pvmw->ptl = pmd_lock(mm, pvmw->pmd);
 			pmde = *pvmw->pmd;
-			if (!pmd_present(pmde)) {
+			if (pmd_is_migration_entry(pmde)) {
 				softleaf_t entry;
 
-				if (!thp_migration_supported() ||
-				    !(pvmw->flags & PVMW_MIGRATION))
+				if (!(pvmw->flags & PVMW_MIGRATION))
 					return not_found(pvmw);
 				entry = softleaf_from_pmd(pmde);
+				if (!check_pmd(softleaf_to_pfn(entry), pvmw))
+					return not_found(pvmw);
+				return true;
+			} else if (pmd_is_device_private_entry(pmde)) {
+				softleaf_t entry;
 
-				if (!softleaf_is_migration(entry) ||
-				    !check_pmd(softleaf_to_pfn(entry), pvmw))
+				if (pvmw->flags & PVMW_MIGRATION)
+					return not_found(pvmw);
+				entry = softleaf_from_pmd(pmde);
+				if (!check_pmd(softleaf_to_pfn(entry), pvmw))
 					return not_found(pvmw);
 				return true;
+			} else if (!pmd_present(pmde)) {
+				return not_found(pvmw);
 			}
 			if (likely(pmd_trans_huge(pmde))) {
 				if (pvmw->flags & PVMW_MIGRATION)
@@ -266,17 +282,10 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw)
 					return not_found(pvmw);
 				return true;
 			}
-			/* THP pmd was split under us: handle on pte level */
+			/* THP/device-private pmd was split under us: handle on pte level */
 			spin_unlock(pvmw->ptl);
 			pvmw->ptl = NULL;
 		} else if (!pmd_present(pmde)) {
-			const softleaf_t entry = softleaf_from_pmd(pmde);
-
-			if (softleaf_is_device_private(entry)) {
-				pvmw->ptl = pmd_lock(mm, pvmw->pmd);
-				return true;
-			}
-
 			if ((pvmw->flags & PVMW_SYNC) &&
 			    thp_vma_suitable_order(vma, pvmw->address,
 						   PMD_ORDER) &&
diff --git a/mm/pagewalk.c b/mm/pagewalk.c
index 3ae2586ff45b4..ed4860c01936c 100644
--- a/mm/pagewalk.c
+++ b/mm/pagewalk.c
@@ -678,6 +678,8 @@ int walk_kernel_page_table_range_lockless(unsigned long start, unsigned long end
  * will also not lock the PTEs for the pte_entry() callback.
  *
  * This is for debugging purposes ONLY.
+ *
+ * The mmap write lock must be held.
  */
 int walk_page_range_debug(struct mm_struct *mm, unsigned long start,
 			  unsigned long end, const struct mm_walk_ops *ops,
@@ -691,24 +693,28 @@ int walk_page_range_debug(struct mm_struct *mm, unsigned long start,
 		.no_vma		= true
 	};
 
-	/* For convenience, we allow traversal of kernel mappings. */
-	if (mm == &init_mm)
-		return walk_kernel_page_table_range(start, end, ops,
-						    pgd, private);
-	if (start >= end || !walk.mm)
-		return -EINVAL;
-	if (!check_ops_safe(ops))
-		return -EINVAL;
-
 	/*
-	 * The mmap lock protects the page walker from changes to the page
-	 * tables during the walk.  However a read lock is insufficient to
-	 * protect those areas which don't have a VMA as munmap() detaches
-	 * the VMAs before downgrading to a read lock and actually tearing
-	 * down PTEs/page tables. In which case, the mmap write lock should
-	 * be held.
+	 * When walking userland page tables, an mmap write lock must be held to
+	 * account for munmap() downgrading to an mmap read lock when tearing
+	 * down page tables.
+	 *
+	 * When walking kernel page tables, an mmap write lock must also be held
+	 * to account for page table freeing on vmap huge page mapping.
 	 */
 	mmap_assert_write_locked(mm);
+	/*
+	 * x86, arm64 ptdump allow walks of efi mm's and x86 ptdump allows walks
+	 * of arbitrary mm's.
+	 *
+	 * However, they both must also hold the init_mm lock to account for
+	 * concurrent kernel page table freeing.
+	 */
+	mmap_assert_write_locked(&init_mm);
+
+	if (start >= end)
+		return -EINVAL;
+	if (!check_ops_safe(ops))
+		return -EINVAL;
 
 	return walk_pgd_range(start, end, &walk);
 }
@@ -810,10 +816,10 @@ int walk_page_mapping(struct address_space *mapping, pgoff_t first_index,
 		return -EINVAL;
 
 	lockdep_assert_held(&mapping->i_mmap_rwsem);
-	vma_interval_tree_foreach(vma, &mapping->i_mmap, first_index,
+	mapping_rmap_tree_foreach(vma, mapping, first_index,
 				  first_index + nr - 1) {
 		/* Clip to the vma */
-		vba = vma->vm_pgoff;
+		vba = vma_start_pgoff(vma);
 		vea = vba + vma_pages(vma);
 		cba = first_index;
 		cba = max(cba, vba);
diff --git a/mm/percpu-km.c b/mm/percpu-km.c
index 4efa74a495cb6..65fd5580e4472 100644
--- a/mm/percpu-km.c
+++ b/mm/percpu-km.c
@@ -75,7 +75,7 @@ static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp)
 	chunk->base_addr = page_address(pages);
 
 	spin_lock_irqsave(&pcpu_lock, flags);
-	pcpu_chunk_populated(chunk, 0, nr_pages);
+	pcpu_chunk_populated(chunk, 0, chunk->nr_pages);
 	spin_unlock_irqrestore(&pcpu_lock, flags);
 
 	pcpu_stats_chunk_alloc();
@@ -94,8 +94,15 @@ static void pcpu_destroy_chunk(struct pcpu_chunk *chunk)
 	pcpu_stats_chunk_dealloc();
 	trace_percpu_destroy_chunk(chunk->base_addr);
 
-	if (chunk->data)
+	if (chunk->data) {
+		struct page *pages = (struct page *)chunk->data;
+		int i;
+
+		/* clear chunk info from each page before free them */
+		for (i = 0; i < nr_pages; i++)
+			pcpu_set_page_chunk(pages + i, NULL);
 		__free_pages(chunk->data, order_base_2(nr_pages));
+	}
 	pcpu_free_chunk(chunk);
 }
 
diff --git a/mm/percpu-vm.c b/mm/percpu-vm.c
index 4f5937090590d..509d8901835cd 100644
--- a/mm/percpu-vm.c
+++ b/mm/percpu-vm.c
@@ -9,6 +9,7 @@
  * This is the default chunk allocator.
  */
 #include "internal.h"
+#include "vmalloc.h"
 
 static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk,
 				    unsigned int cpu, int page_idx)
@@ -21,6 +22,8 @@ static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk,
 
 /**
  * pcpu_get_pages - get temp pages array
+ * @gfp: allocation flags passed to the underlying allocator, 0 to only
+ *	 return the cached array
  *
  * Returns pointer to array of pointers to struct page which can be indexed
  * with pcpu_page_idx().  Note that there is only one array and accesses
@@ -29,18 +32,23 @@ static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk,
  * RETURNS:
  * Pointer to temp pages array on success.
  */
-static struct page **pcpu_get_pages(void)
+static struct page **pcpu_get_pages(gfp_t gfp)
 {
 	static struct page **pages;
 	size_t pages_size = pcpu_nr_units * pcpu_unit_pages * sizeof(pages[0]);
 
 	lockdep_assert_held(&pcpu_alloc_mutex);
 
-	if (!pages)
-		pages = pcpu_mem_zalloc(pages_size, GFP_KERNEL);
+	if (!pages && gfp)
+		pages = pcpu_mem_zalloc(pages_size, gfp);
 	return pages;
 }
 
+static struct page **pcpu_get_pages_cached(void)
+{
+	return pcpu_get_pages(0);
+}
+
 /**
  * pcpu_free_pages - free pages which were allocated for @chunk
  * @chunk: chunk pages were allocated for
@@ -191,10 +199,22 @@ static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,
 }
 
 static int __pcpu_map_pages(unsigned long addr, struct page **pages,
-			    int nr_pages)
+			    int nr_pages, gfp_t gfp)
 {
-	return vmap_pages_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT),
-			PAGE_KERNEL, pages, PAGE_SHIFT, GFP_KERNEL);
+	unsigned int flags;
+	int ret;
+
+	/*
+	 * The vmalloc page table allocation path does not pass @gfp down
+	 * explicitly.  Apply the corresponding memalloc scope so implicit
+	 * page table allocations preserve NOFS/NOIO constraints.
+	 */
+	flags = memalloc_apply_gfp_scope(gfp);
+	ret = vmap_pages_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT),
+				       PAGE_KERNEL, pages, PAGE_SHIFT, gfp);
+	memalloc_restore_scope(flags);
+
+	return ret;
 }
 
 /**
@@ -203,6 +223,7 @@ static int __pcpu_map_pages(unsigned long addr, struct page **pages,
  * @pages: pages array containing pages to be mapped
  * @page_start: page index of the first page to map
  * @page_end: page index of the last page to map + 1
+ * @gfp: allocation flags passed to the underlying allocator
  *
  * For each cpu, map pages [@page_start,@page_end) into @chunk.  The
  * caller is responsible for calling pcpu_post_map_flush() after all
@@ -211,8 +232,8 @@ static int __pcpu_map_pages(unsigned long addr, struct page **pages,
  * This function is responsible for setting up whatever is necessary for
  * reverse lookup (addr -> chunk).
  */
-static int pcpu_map_pages(struct pcpu_chunk *chunk,
-			  struct page **pages, int page_start, int page_end)
+static int pcpu_map_pages(struct pcpu_chunk *chunk, struct page **pages,
+			  int page_start, int page_end, gfp_t gfp)
 {
 	unsigned int cpu, tcpu;
 	int i, err;
@@ -220,7 +241,7 @@ static int pcpu_map_pages(struct pcpu_chunk *chunk,
 	for_each_possible_cpu(cpu) {
 		err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start),
 				       &pages[pcpu_page_idx(cpu, page_start)],
-				       page_end - page_start);
+				       page_end - page_start, gfp);
 		if (err < 0)
 			goto err;
 
@@ -271,21 +292,21 @@ static void pcpu_post_map_flush(struct pcpu_chunk *chunk,
  * @chunk.
  *
  * CONTEXT:
- * pcpu_alloc_mutex, does GFP_KERNEL allocation.
+ * pcpu_alloc_mutex, does @gfp allocation.
  */
 static int pcpu_populate_chunk(struct pcpu_chunk *chunk,
 			       int page_start, int page_end, gfp_t gfp)
 {
 	struct page **pages;
 
-	pages = pcpu_get_pages();
+	pages = pcpu_get_pages(gfp);
 	if (!pages)
 		return -ENOMEM;
 
 	if (pcpu_alloc_pages(chunk, pages, page_start, page_end, gfp))
 		return -ENOMEM;
 
-	if (pcpu_map_pages(chunk, pages, page_start, page_end)) {
+	if (pcpu_map_pages(chunk, pages, page_start, page_end, gfp)) {
 		pcpu_free_pages(chunk, pages, page_start, page_end);
 		return -ENOMEM;
 	}
@@ -319,7 +340,7 @@ static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk,
 	 * successful population attempt so the temp pages array must
 	 * be available now.
 	 */
-	pages = pcpu_get_pages();
+	pages = pcpu_get_pages_cached();
 	BUG_ON(!pages);
 
 	/* unmap and free */
@@ -340,7 +361,7 @@ static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp)
 		return NULL;
 
 	vms = pcpu_get_vm_areas(pcpu_group_offsets, pcpu_group_sizes,
-				pcpu_nr_groups, pcpu_atom_size);
+				pcpu_nr_groups, pcpu_atom_size, gfp);
 	if (!vms) {
 		pcpu_free_chunk(chunk);
 		return NULL;
diff --git a/mm/percpu.c b/mm/percpu.c
index b0676b8054ed0..a802d72c116fb 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -1726,9 +1726,8 @@ static void pcpu_alloc_tag_free_hook(struct pcpu_chunk *chunk, int off, size_t s
  * @gfp: allocation flags
  *
  * Allocate percpu area of @size bytes aligned at @align.  If @gfp doesn't
- * contain %GFP_KERNEL, the allocation is atomic. If @gfp has __GFP_NOWARN
- * then no warning will be triggered on invalid or failed allocation
- * requests.
+ * allow blocking, the allocation is atomic. If @gfp has __GFP_NOWARN then no
+ * warning will be triggered on invalid or failed allocation requests.
  *
  * RETURNS:
  * Percpu pointer to the allocated area on success, NULL on failure.
@@ -1749,8 +1748,17 @@ void __percpu *pcpu_alloc_noprof(size_t size, size_t align, bool reserved,
 	size_t bits, bit_align;
 
 	gfp = current_gfp_context(gfp);
-	/* whitelisted flags that can be passed to the backing allocators */
-	pcpu_gfp = gfp & (GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
+	/*
+	 * Allowlisted flags that can be passed to the backing allocators.
+	 * Backing allocations under pcpu_alloc_mutex must not recurse into
+	 * IO/FS reclaim.  Otherwise a GFP_KERNEL caller holding the mutex can
+	 * block on reclaim while a GFP_NOIO/NOFS caller holding an IO/FS lock
+	 * waits for the same mutex.
+	 *
+	 * Do not pass __GFP_NOFAIL.  A small percpu allocation may need many
+	 * backing pages, making nofail reclaim too costly under NOIO/NOFS.
+	 */
+	pcpu_gfp = gfp & (GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN);
 	is_atomic = !gfpflags_allow_blocking(gfp);
 	do_warn = !(gfp & __GFP_NOWARN);
 
@@ -3256,7 +3264,7 @@ int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_cpu_to_node_fn_t
 
 		/* pte already populated, the following shouldn't fail */
 		rc = __pcpu_map_pages(unit_addr, &pages[unit * unit_pages],
-				      unit_pages);
+				      unit_pages, GFP_KERNEL);
 		if (rc < 0)
 			panic("failed to map percpu area, err=%d\n", rc);
 
diff --git a/mm/ptdump.c b/mm/ptdump.c
index 973020000096c..5851096e6f656 100644
--- a/mm/ptdump.c
+++ b/mm/ptdump.c
@@ -178,11 +178,18 @@ void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd)
 
 	get_online_mems();
 	mmap_write_lock(mm);
+	/* To stabilise kernel page tables we must hold the init_mm lock too. */
+	if (mm != &init_mm)
+		mmap_write_lock_nested(&init_mm, SINGLE_DEPTH_NESTING);
+
 	while (range->start != range->end) {
 		walk_page_range_debug(mm, range->start, range->end,
 				      &ptdump_ops, pgd, st);
 		range++;
 	}
+
+	if (mm != &init_mm)
+		mmap_write_unlock(&init_mm);
 	mmap_write_unlock(mm);
 	put_online_mems();
 
diff --git a/mm/rmap.c b/mm/rmap.c
index 1c77d5dc06e9f..ad820fe86f7d8 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -211,7 +211,7 @@ int __anon_vma_prepare(struct vm_area_struct *vma)
 	if (likely(!vma->anon_vma)) {
 		vma->anon_vma = anon_vma;
 		anon_vma_chain_assign(vma, avc, anon_vma);
-		anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);
+		anon_rmap_tree_insert(avc, anon_vma);
 		anon_vma->num_active_vmas++;
 		allocated = NULL;
 		avc = NULL;
@@ -354,7 +354,7 @@ int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src,
 	list_for_each_entry_reverse(avc, &dst->anon_vma_chain, same_vma) {
 		struct anon_vma *anon_vma = avc->anon_vma;
 
-		anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);
+		anon_rmap_tree_insert(avc, anon_vma);
 		if (operation == VMA_OP_FORK)
 			maybe_reuse_anon_vma(dst, anon_vma);
 	}
@@ -434,7 +434,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
 	anon_vma_chain_assign(vma, avc, anon_vma);
 	/* Now let rmap see it. */
 	anon_vma_lock_write(anon_vma);
-	anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);
+	anon_rmap_tree_insert(avc, anon_vma);
 	anon_vma->parent->num_children++;
 	anon_vma_unlock_write(anon_vma);
 
@@ -499,7 +499,7 @@ void unlink_anon_vmas(struct vm_area_struct *vma)
 	list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
 		struct anon_vma *anon_vma = avc->anon_vma;
 
-		anon_vma_interval_tree_remove(avc, &anon_vma->rb_root);
+		anon_rmap_tree_remove(avc, anon_vma);
 
 		/*
 		 * Leave empty anon_vmas on the list - we'll need
@@ -1965,7 +1965,7 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
 	if (pte_unused(pte))
 		return 1;
 
-	if (userfaultfd_wp(vma))
+	if (userfaultfd_protected(vma))
 		return 1;
 
 	/*
@@ -1978,6 +1978,220 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
 				     FPB_RESPECT_WRITE | FPB_RESPECT_SOFT_DIRTY);
 }
 
+static bool try_to_unmap_hugetlb_one(struct folio *folio,
+		struct vm_area_struct *vma, unsigned long address, void *arg)
+{
+	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
+	const unsigned long hsz = huge_page_size(hstate_vma(vma));
+	const enum ttu_flags flags = (enum ttu_flags)(long)arg;
+	struct mm_struct *mm = vma->vm_mm;
+	struct mmu_notifier_range range;
+	bool ret = true;
+	pte_t pteval;
+
+	/*
+	 * The try_to_unmap() is only passed a hugetlb folio in the case
+	 * where the hugetlb folio is poisoned.
+	 */
+	VM_WARN_ON_FOLIO(!folio_test_hwpoison(folio), folio);
+	VM_WARN_ON_ONCE(!(flags & TTU_HWPOISON));
+
+	range.end = vma_address_end(&pvmw);
+	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
+				address, range.end);
+	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
+	mmu_notifier_invalidate_range_start(&range);
+
+	/* There is only a single mapping in a VMA. */
+	if (!page_vma_mapped_walk(&pvmw))
+		goto range_end;
+
+	VM_WARN_ON_ONCE(address != pvmw.address);
+
+	pteval = huge_ptep_get(mm, address, pvmw.pte);
+	VM_WARN_ON_ONCE(!pte_present(pteval));
+	VM_WARN_ON_ONCE(pte_pfn(pteval) != folio_pfn(folio));
+
+	/*
+	 * huge_pmd_unshare may unmap an entire PMD page. There is no way of
+	 * knowing exactly which PMDs may be cached for this mm, so we must
+	 * flush them all. start/end were already adjusted above to cover this
+	 * range.
+	 */
+	flush_cache_range(vma, range.start, range.end);
+
+	/*
+	 * To call huge_pmd_unshare, i_mmap_rwsem must be held in write mode.
+	 * Caller needs to explicitly do this outside rmap routines.
+	 *
+	 * We also must hold hugetlb vma_lock in write mode. Lock order dictates
+	 * acquiring vma_lock BEFORE i_mmap_rwsem. We can only try lock here and
+	 * fail if unsuccessful.
+	 */
+	if (!folio_test_anon(folio)) {
+		struct mmu_gather tlb;
+
+		VM_WARN_ON(!(flags & TTU_RMAP_LOCKED));
+		if (!hugetlb_vma_trylock_write(vma)) {
+			ret = false;
+			goto walk_done;
+		}
+
+		tlb_gather_mmu_vma(&tlb, vma);
+		if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) {
+			hugetlb_vma_unlock_write(vma);
+			huge_pmd_unshare_flush(&tlb, vma);
+			tlb_finish_mmu(&tlb);
+			/*
+			 * The PMD table was unmapped, consequently unmapping
+			 * the folio.
+			 */
+			goto walk_done;
+		}
+		hugetlb_vma_unlock_write(vma);
+		tlb_finish_mmu(&tlb);
+	}
+	pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);
+	if (huge_pte_dirty(pteval))
+		folio_mark_dirty(folio);
+
+	pteval = swp_entry_to_pte(make_hwpoison_entry(folio_page(folio, 0)));
+	hugetlb_count_sub(folio_nr_pages(folio), mm);
+	set_huge_pte_at(mm, address, pvmw.pte, pteval, hsz);
+	hugetlb_remove_rmap(folio);
+	folio_put_refs(folio, 1);
+
+walk_done:
+	page_vma_mapped_walk_done(&pvmw);
+range_end:
+	mmu_notifier_invalidate_range_end(&range);
+	return ret;
+}
+
+static bool ttu_anon_lazyfree_folio(struct vm_area_struct *vma,
+		struct folio *folio, unsigned long nr_pages)
+{
+	int ref_count, map_count;
+
+	/*
+	 * Synchronize with gup_pte_range():
+	 * - clear PTE; barrier; read refcount
+	 * - inc refcount; barrier; read PTE
+	 */
+	smp_mb();
+
+	ref_count = folio_ref_count(folio);
+	map_count = folio_mapcount(folio);
+
+	/*
+	 * Order reads for page refcount and dirty flag
+	 * (see comments in __remove_mapping()).
+	 */
+	smp_rmb();
+
+	if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) {
+		/*
+		 * redirtied either using the page table or a previously
+		 * obtained GUP reference.
+		 */
+		folio_set_swapbacked(folio);
+		return false;
+	}
+
+	/*
+	 * Additional references could be due to GUP or speculative lookups.
+	 * GUP users must mark the folio dirty if there was a modification.
+	 * This folio cannot be reclaimed right now either way, so act just
+	 * like nothing happened. We'll come back here later and detect if the
+	 * folio was dirtied when the additional reference is gone.
+	 */
+	if (ref_count != 1 + map_count)
+		return false;
+
+	add_mm_counter(vma->vm_mm, MM_ANONPAGES, -nr_pages);
+	return true;
+}
+
+static pte_t swp_pte_prepare(swp_entry_t entry, pte_t old_pte,
+		bool anon_exclusive)
+{
+	pte_t swp_pte = swp_entry_to_pte(entry);
+
+	if (anon_exclusive)
+		swp_pte = pte_swp_mkexclusive(swp_pte);
+
+	if (likely(pte_present(old_pte))) {
+		if (pte_soft_dirty(old_pte))
+			swp_pte = pte_swp_mksoft_dirty(swp_pte);
+		if (pte_uffd(old_pte))
+			swp_pte = pte_swp_mkuffd(swp_pte);
+	} else {
+		/* Device-exclusive entry */
+		if (pte_swp_soft_dirty(old_pte))
+			swp_pte = pte_swp_mksoft_dirty(swp_pte);
+		if (pte_swp_uffd(old_pte))
+			swp_pte = pte_swp_mkuffd(swp_pte);
+	}
+
+	return swp_pte;
+}
+
+static bool ttu_anon_swapbacked_folio(struct vm_area_struct *vma,
+		struct folio *folio, struct page *page, unsigned long address,
+		pte_t *ptep, pte_t pteval)
+{
+	const bool anon_exclusive = folio_test_anon(folio) &&
+				    PageAnonExclusive(page);
+	swp_entry_t entry = page_swap_entry(page);
+	struct mm_struct *mm = vma->vm_mm;
+
+	if (folio_dup_swap(folio, page) < 0)
+		return false;
+
+	/*
+	 * arch_unmap_one() is expected to be a NOP on
+	 * architectures where we could have PFN swap PTEs,
+	 * so we'll not check/care.
+	 */
+	if (arch_unmap_one(mm, vma, address, pteval) < 0) {
+		folio_put_swap(folio, page);
+		return false;
+	}
+
+	/* See folio_try_share_anon_rmap(): clear PTE first. */
+	if (anon_exclusive && folio_try_share_anon_rmap_pte(folio, page)) {
+		folio_put_swap(folio, page);
+		return false;
+	}
+
+	mm_prepare_for_swap_entries(mm);
+	dec_mm_counter(mm, MM_ANONPAGES);
+	inc_mm_counter(mm, MM_SWAPENTS);
+	set_pte_at(mm, address, ptep,
+		   swp_pte_prepare(entry, pteval, anon_exclusive));
+	return true;
+}
+
+static bool ttu_anon_folio(struct vm_area_struct *vma, struct folio *folio,
+		struct page *page, unsigned long address, pte_t *ptep,
+		pte_t pteval, unsigned long nr_pages)
+{
+	/*
+	 * Store the swap location in the pte.
+	 * See handle_pte_fault() ...
+	 */
+	if (WARN_ON_ONCE(folio_test_swapbacked(folio) !=
+			 folio_test_swapcache(folio)))
+		return false;
+
+	if (!folio_test_swapbacked(folio))
+		return ttu_anon_lazyfree_folio(vma, folio, nr_pages);
+
+	/* nr_pages > 1 not supported yet */
+	return ttu_anon_swapbacked_folio(vma, folio, page, address, ptep,
+					 pteval);
+}
+
 /*
  * @arg: enum ttu_flags will be passed to this argument
  */
@@ -1986,14 +2200,13 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
 {
 	struct mm_struct *mm = vma->vm_mm;
 	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
-	bool anon_exclusive, ret = true;
+	bool ret = true;
 	pte_t pteval;
-	struct page *subpage;
+	struct page *page;
 	struct mmu_notifier_range range;
 	enum ttu_flags flags = (enum ttu_flags)(long)arg;
 	unsigned long nr_pages = 1, end_addr;
 	unsigned long pfn;
-	unsigned long hsz = 0;
 	int ptes = 0;
 
 	/*
@@ -2007,8 +2220,6 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
 
 	/*
 	 * For THP, we have to assume the worse case ie pmd for invalidation.
-	 * For hugetlb, it could be much worse if we need to do pud
-	 * invalidation in the case of pmd sharing.
 	 *
 	 * Note that the folio can not be freed in this function as call of
 	 * try_to_unmap() must hold a reference on the folio.
@@ -2016,17 +2227,6 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
 	range.end = vma_address_end(&pvmw);
 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
 				address, range.end);
-	if (folio_test_hugetlb(folio)) {
-		/*
-		 * If sharing is possible, start and end will be adjusted
-		 * accordingly.
-		 */
-		adjust_range_if_pmd_sharing_possible(vma, &range.start,
-						     &range.end);
-
-		/* We need the huge page size for set_huge_pte_at() */
-		hsz = huge_page_size(hstate_vma(vma));
-	}
 	mmu_notifier_invalidate_range_start(&range);
 
 	while (page_vma_mapped_walk(&pvmw)) {
@@ -2095,77 +2295,27 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
 		/* Unexpected PMD-mapped THP? */
 		VM_BUG_ON_FOLIO(!pvmw.pte, folio);
 
-		/*
-		 * Handle PFN swap PTEs, such as device-exclusive ones, that
-		 * actually map pages.
-		 */
-		pteval = ptep_get(pvmw.pte);
+		address = pvmw.address;
+		if (folio_test_hugetlb(folio)) {
+			pteval = huge_ptep_get(mm, address, pvmw.pte);
+		} else {
+			pteval = ptep_get(pvmw.pte);
+		}
 		if (likely(pte_present(pteval))) {
 			pfn = pte_pfn(pteval);
 		} else {
+			/*
+			 * Handle PFN swap PTEs, such as device-exclusive ones,
+			 * that actually map pages.
+			 */
 			const softleaf_t entry = softleaf_from_pte(pteval);
 
 			pfn = softleaf_to_pfn(entry);
-			VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio);
 		}
 
-		subpage = folio_page(folio, pfn - folio_pfn(folio));
-		address = pvmw.address;
-		anon_exclusive = folio_test_anon(folio) &&
-				 PageAnonExclusive(subpage);
-
-		if (folio_test_hugetlb(folio)) {
-			bool anon = folio_test_anon(folio);
-
-			/*
-			 * The try_to_unmap() is only passed a hugetlb page
-			 * in the case where the hugetlb page is poisoned.
-			 */
-			VM_BUG_ON_PAGE(!PageHWPoison(subpage), subpage);
-			/*
-			 * huge_pmd_unshare may unmap an entire PMD page.
-			 * There is no way of knowing exactly which PMDs may
-			 * be cached for this mm, so we must flush them all.
-			 * start/end were already adjusted above to cover this
-			 * range.
-			 */
-			flush_cache_range(vma, range.start, range.end);
-
-			/*
-			 * To call huge_pmd_unshare, i_mmap_rwsem must be
-			 * held in write mode.  Caller needs to explicitly
-			 * do this outside rmap routines.
-			 *
-			 * We also must hold hugetlb vma_lock in write mode.
-			 * Lock order dictates acquiring vma_lock BEFORE
-			 * i_mmap_rwsem.  We can only try lock here and fail
-			 * if unsuccessful.
-			 */
-			if (!anon) {
-				struct mmu_gather tlb;
-
-				VM_BUG_ON(!(flags & TTU_RMAP_LOCKED));
-				if (!hugetlb_vma_trylock_write(vma))
-					goto walk_abort;
+		page = folio_page(folio, pfn - folio_pfn(folio));
 
-				tlb_gather_mmu_vma(&tlb, vma);
-				if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) {
-					hugetlb_vma_unlock_write(vma);
-					huge_pmd_unshare_flush(&tlb, vma);
-					tlb_finish_mmu(&tlb);
-					/*
-					 * The PMD table was unmapped,
-					 * consequently unmapping the folio.
-					 */
-					goto walk_done;
-				}
-				hugetlb_vma_unlock_write(vma);
-				tlb_finish_mmu(&tlb);
-			}
-			pteval = huge_ptep_clear_flush(vma, address, pvmw.pte);
-			if (pte_dirty(pteval))
-				folio_mark_dirty(folio);
-		} else if (likely(pte_present(pteval))) {
+		if (likely(pte_present(pteval))) {
 			nr_pages = folio_unmap_pte_batch(folio, &pvmw, flags, pteval);
 			end_addr = address + nr_pages * PAGE_SIZE;
 			flush_cache_range(vma, address, end_addr);
@@ -2200,16 +2350,11 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
 		/* Update high watermark before we lower rss */
 		update_hiwater_rss(mm);
 
-		if (PageHWPoison(subpage) && (flags & TTU_HWPOISON)) {
-			pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
-			if (folio_test_hugetlb(folio)) {
-				hugetlb_count_sub(folio_nr_pages(folio), mm);
-				set_huge_pte_at(mm, address, pvmw.pte, pteval,
-						hsz);
-			} else {
-				dec_mm_counter(mm, mm_counter(folio));
-				set_pte_at(mm, address, pvmw.pte, pteval);
-			}
+		/* With TTU_HWPOISON, we only expect small folios here. */
+		if (folio_test_hwpoison(folio) && (flags & TTU_HWPOISON)) {
+			pteval = swp_entry_to_pte(make_hwpoison_entry(page));
+			dec_mm_counter(mm, mm_counter(folio));
+			set_pte_at(mm, address, pvmw.pte, pteval);
 		} else if (likely(pte_present(pteval)) && pte_unused(pteval) &&
 			   !userfaultfd_armed(vma)) {
 			/*
@@ -2224,109 +2369,13 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
 			 */
 			dec_mm_counter(mm, mm_counter(folio));
 		} else if (folio_test_anon(folio)) {
-			swp_entry_t entry = page_swap_entry(subpage);
-			pte_t swp_pte;
-			/*
-			 * Store the swap location in the pte.
-			 * See handle_pte_fault() ...
-			 */
-			if (unlikely(folio_test_swapbacked(folio) !=
-					folio_test_swapcache(folio))) {
-				WARN_ON_ONCE(1);
-				goto walk_abort;
-			}
-
-			/* MADV_FREE page check */
-			if (!folio_test_swapbacked(folio)) {
-				int ref_count, map_count;
-
-				/*
-				 * Synchronize with gup_pte_range():
-				 * - clear PTE; barrier; read refcount
-				 * - inc refcount; barrier; read PTE
-				 */
-				smp_mb();
-
-				ref_count = folio_ref_count(folio);
-				map_count = folio_mapcount(folio);
-
-				/*
-				 * Order reads for page refcount and dirty flag
-				 * (see comments in __remove_mapping()).
-				 */
-				smp_rmb();
-
-				if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) {
-					/*
-					 * redirtied either using the page table or a previously
-					 * obtained GUP reference.
-					 */
-					set_ptes(mm, address, pvmw.pte, pteval, nr_pages);
-					folio_set_swapbacked(folio);
-					goto walk_abort;
-				} else if (ref_count != 1 + map_count) {
-					/*
-					 * Additional reference. Could be a GUP reference or any
-					 * speculative reference. GUP users must mark the folio
-					 * dirty if there was a modification. This folio cannot be
-					 * reclaimed right now either way, so act just like nothing
-					 * happened.
-					 * We'll come back here later and detect if the folio was
-					 * dirtied when the additional reference is gone.
-					 */
-					set_ptes(mm, address, pvmw.pte, pteval, nr_pages);
-					goto walk_abort;
-				}
-				add_mm_counter(mm, MM_ANONPAGES, -nr_pages);
-				goto discard;
-			}
-
-			if (folio_dup_swap(folio, subpage) < 0) {
-				set_pte_at(mm, address, pvmw.pte, pteval);
-				goto walk_abort;
-			}
-
-			/*
-			 * arch_unmap_one() is expected to be a NOP on
-			 * architectures where we could have PFN swap PTEs,
-			 * so we'll not check/care.
-			 */
-			if (arch_unmap_one(mm, vma, address, pteval) < 0) {
-				folio_put_swap(folio, subpage);
-				set_pte_at(mm, address, pvmw.pte, pteval);
+			if (!ttu_anon_folio(vma, folio, page, address,
+					    pvmw.pte, pteval, nr_pages)) {
+				set_ptes(mm, address, pvmw.pte, pteval, nr_pages);
 				goto walk_abort;
 			}
 
-			/* See folio_try_share_anon_rmap(): clear PTE first. */
-			if (anon_exclusive &&
-			    folio_try_share_anon_rmap_pte(folio, subpage)) {
-				folio_put_swap(folio, subpage);
-				set_pte_at(mm, address, pvmw.pte, pteval);
-				goto walk_abort;
-			}
-			if (list_empty(&mm->mmlist)) {
-				spin_lock(&mmlist_lock);
-				if (list_empty(&mm->mmlist))
-					list_add(&mm->mmlist, &init_mm.mmlist);
-				spin_unlock(&mmlist_lock);
-			}
-			dec_mm_counter(mm, MM_ANONPAGES);
-			inc_mm_counter(mm, MM_SWAPENTS);
-			swp_pte = swp_entry_to_pte(entry);
-			if (anon_exclusive)
-				swp_pte = pte_swp_mkexclusive(swp_pte);
-			if (likely(pte_present(pteval))) {
-				if (pte_soft_dirty(pteval))
-					swp_pte = pte_swp_mksoft_dirty(swp_pte);
-				if (pte_uffd_wp(pteval))
-					swp_pte = pte_swp_mkuffd_wp(swp_pte);
-			} else {
-				if (pte_swp_soft_dirty(pteval))
-					swp_pte = pte_swp_mksoft_dirty(swp_pte);
-				if (pte_swp_uffd_wp(pteval))
-					swp_pte = pte_swp_mkuffd_wp(swp_pte);
-			}
-			set_pte_at(mm, address, pvmw.pte, swp_pte);
+			goto finish_unmap;
 		} else {
 			/*
 			 * This is a locked file-backed folio,
@@ -2341,12 +2390,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
 			 */
 			add_mm_counter(mm, mm_counter_file(folio), -nr_pages);
 		}
-discard:
-		if (unlikely(folio_test_hugetlb(folio))) {
-			hugetlb_remove_rmap(folio);
-		} else {
-			folio_remove_rmap_ptes(folio, subpage, nr_pages, vma);
-		}
+finish_unmap:
+		folio_remove_rmap_ptes(folio, page, nr_pages, vma);
 		if (vma->vm_flags & VM_LOCKED)
 			mlock_drain_local();
 		folio_put_refs(folio, nr_pages);
@@ -2394,7 +2439,8 @@ static int folio_not_mapped(struct folio *folio)
 void try_to_unmap(struct folio *folio, enum ttu_flags flags)
 {
 	struct rmap_walk_control rwc = {
-		.rmap_one = try_to_unmap_one,
+		.rmap_one = folio_test_hugetlb(folio) ?
+				try_to_unmap_hugetlb_one : try_to_unmap_one,
 		.arg = (void *)flags,
 		.done = folio_not_mapped,
 		.anon_lock = folio_lock_anon_vma_read,
@@ -2477,7 +2523,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
 				page_vma_mapped_walk_restart(&pvmw);
 				continue;
 			}
-#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
 			pmdval = pmdp_get(pvmw.pmd);
 			if (likely(pmd_present(pmdval)))
 				pfn = pmd_pfn(pmdval);
@@ -2501,14 +2547,18 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
 		/* Unexpected PMD-mapped THP? */
 		VM_BUG_ON_FOLIO(!pvmw.pte, folio);
 
-		/*
-		 * Handle PFN swap PTEs, such as device-exclusive ones, that
-		 * actually map pages.
-		 */
-		pteval = ptep_get(pvmw.pte);
+		address = pvmw.address;
+		if (folio_test_hugetlb(folio))
+			pteval = huge_ptep_get(mm, address, pvmw.pte);
+		else
+			pteval = ptep_get(pvmw.pte);
 		if (likely(pte_present(pteval))) {
 			pfn = pte_pfn(pteval);
 		} else {
+			/*
+			 * Handle PFN swap PTEs, such as device-exclusive ones,
+			 * that actually map pages.
+			 */
 			const softleaf_t entry = softleaf_from_pte(pteval);
 
 			pfn = softleaf_to_pfn(entry);
@@ -2516,7 +2566,6 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
 		}
 
 		subpage = folio_page(folio, pfn - folio_pfn(folio));
-		address = pvmw.address;
 		anon_exclusive = folio_test_anon(folio) &&
 				 PageAnonExclusive(subpage);
 
@@ -2692,14 +2741,14 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
 				swp_pte = swp_entry_to_pte(entry);
 				if (pte_soft_dirty(pteval))
 					swp_pte = pte_swp_mksoft_dirty(swp_pte);
-				if (pte_uffd_wp(pteval))
-					swp_pte = pte_swp_mkuffd_wp(swp_pte);
+				if (pte_uffd(pteval))
+					swp_pte = pte_swp_mkuffd(swp_pte);
 			} else {
 				swp_pte = swp_entry_to_pte(entry);
 				if (pte_swp_soft_dirty(pteval))
 					swp_pte = pte_swp_mksoft_dirty(swp_pte);
-				if (pte_swp_uffd_wp(pteval))
-					swp_pte = pte_swp_mkuffd_wp(swp_pte);
+				if (pte_swp_uffd(pteval))
+					swp_pte = pte_swp_mkuffd(swp_pte);
 			}
 			if (folio_test_hugetlb(folio))
 				set_huge_pte_at(mm, address, pvmw.pte, swp_pte,
@@ -2986,8 +3035,7 @@ static void rmap_walk_anon(struct folio *folio,
 
 	pgoff_start = folio_pgoff(folio);
 	pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
-	anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
-			pgoff_start, pgoff_end) {
+	anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_end) {
 		struct vm_area_struct *vma = avc->vma;
 		unsigned long address = vma_address(vma, pgoff_start,
 				folio_nr_pages(folio));
@@ -3051,8 +3099,7 @@ static void __rmap_walk_file(struct folio *folio, struct address_space *mapping,
 		i_mmap_lock_read(mapping);
 	}
 lookup:
-	vma_interval_tree_foreach(vma, &mapping->i_mmap,
-			pgoff_start, pgoff_end) {
+	mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_end) {
 		unsigned long address = vma_address(vma, pgoff_start, nr_pages);
 
 		VM_BUG_ON_VMA(address == -EFAULT, vma);
diff --git a/mm/shmem.c b/mm/shmem.c
index b51f83c970bb3..f189d1ac40db2 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -1032,6 +1032,8 @@ unsigned long shmem_swap_usage(struct vm_area_struct *vma)
 	struct inode *inode = file_inode(vma->vm_file);
 	struct shmem_inode_info *info = SHMEM_I(inode);
 	struct address_space *mapping = inode->i_mapping;
+	const pgoff_t pgoff = vma_start_pgoff(vma);
+	const pgoff_t pgoff_end = vma_end_pgoff(vma);
 	unsigned long swapped;
 
 	/* Be careful as we don't hold info->lock */
@@ -1045,12 +1047,11 @@ unsigned long shmem_swap_usage(struct vm_area_struct *vma)
 	if (!swapped)
 		return 0;
 
-	if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
+	if (!pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
 		return swapped << PAGE_SHIFT;
 
 	/* Here comes the more involved part */
-	return shmem_partial_swap_usage(mapping, vma->vm_pgoff,
-					vma->vm_pgoff + vma_pages(vma));
+	return shmem_partial_swap_usage(mapping, pgoff, pgoff_end);
 }
 
 /*
@@ -1287,7 +1288,8 @@ static int shmem_getattr(struct mnt_idmap *idmap,
 	struct inode *inode = path->dentry->d_inode;
 	struct shmem_inode_info *info = SHMEM_I(inode);
 
-	if (info->alloced - info->swapped != inode->i_mapping->nrpages)
+	/* Fast-path hint; recalc under info->lock corrects any stale read. */
+	if (data_race(info->alloced - info->swapped != inode->i_mapping->nrpages))
 		shmem_recalc_inode(inode, 0, 0);
 
 	if (info->fsflags & FS_APPEND_FL)
@@ -1582,13 +1584,13 @@ int shmem_unuse(unsigned int type)
 
 /**
  * shmem_writeout - Write the folio to swap
+ * @ctx: swap I/O context
  * @folio: The folio to write
- * @plug: swap plug
  * @folio_list: list to put back folios on split
  *
  * Move the folio from the page cache to the swap cache.
  */
-int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
+int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio,
 		struct list_head *folio_list)
 {
 	struct address_space *mapping = folio->mapping;
@@ -1657,7 +1659,7 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
 	 * reactivate the folio, and let shmem_fallocate() quit when too many.
 	 */
 	if (!folio_test_uptodate(folio)) {
-		if (inode->i_private) {
+		if (READ_ONCE(inode->i_private)) {
 			struct shmem_falloc *shmem_falloc;
 			spin_lock(&inode->i_lock);
 			shmem_falloc = inode->i_private;
@@ -1700,7 +1702,7 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
 		shmem_delete_from_page_cache(folio, swp_to_radix_entry(folio->swap));
 
 		BUG_ON(folio_mapped(folio));
-		error = swap_writeout(folio, plug);
+		error = swap_writeout(ctx, folio);
 		if (error != AOP_WRITEPAGE_ACTIVATE) {
 			/* folio has been unlocked */
 			return error;
@@ -1736,7 +1738,17 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
 	folio_mark_dirty(folio);
 	return AOP_WRITEPAGE_ACTIVATE;	/* Return with folio locked */
 }
-EXPORT_SYMBOL_GPL(shmem_writeout);
+
+int shmem_write_folio(struct folio *folio)
+{
+	struct swap_io_ctx ctx = {};
+	int err;
+
+	err = shmem_writeout(&ctx, folio, NULL);
+	swap_write_submit(&ctx);
+	return err;
+}
+EXPORT_SYMBOL_GPL(shmem_write_folio);
 
 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
@@ -2693,7 +2705,7 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf)
 	 * Trinity finds that probing a hole which tmpfs is punching can
 	 * prevent the hole-punch from ever completing: noted in i_private.
 	 */
-	if (unlikely(inode->i_private)) {
+	if (unlikely(READ_ONCE(inode->i_private))) {
 		ret = shmem_falloc_wait(vmf, inode);
 		if (ret)
 			return ret;
@@ -2839,7 +2851,7 @@ static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
 	 * by page order, as in shmem_get_pgoff_policy() and get_vma_policy()).
 	 */
 	*ilx = inode->i_ino;
-	index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
+	index = linear_page_index(vma, addr);
 	return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
 }
 
@@ -3630,7 +3642,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset,
 		shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT;
 		shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
 		spin_lock(&inode->i_lock);
-		inode->i_private = &shmem_falloc;
+		WRITE_ONCE(inode->i_private, &shmem_falloc);
 		spin_unlock(&inode->i_lock);
 
 		if ((u64)unmap_end > (u64)unmap_start)
@@ -3640,7 +3652,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset,
 		/* No need to unmap again: hole-punching leaves COWed pages */
 
 		spin_lock(&inode->i_lock);
-		inode->i_private = NULL;
+		WRITE_ONCE(inode->i_private, NULL);
 		wake_up_all(&shmem_falloc_waitq);
 		WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
 		spin_unlock(&inode->i_lock);
@@ -3672,7 +3684,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset,
 	shmem_falloc.nr_falloced = 0;
 	shmem_falloc.nr_unswapped = 0;
 	spin_lock(&inode->i_lock);
-	inode->i_private = &shmem_falloc;
+	WRITE_ONCE(inode->i_private, &shmem_falloc);
 	spin_unlock(&inode->i_lock);
 
 	/*
@@ -3747,7 +3759,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset,
 		i_size_write(inode, offset + len);
 undone:
 	spin_lock(&inode->i_lock);
-	inode->i_private = NULL;
+	WRITE_ONCE(inode->i_private, NULL);
 	spin_unlock(&inode->i_lock);
 out:
 	if (!error)
@@ -3864,7 +3876,7 @@ static struct dentry *shmem_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 }
 
 static int shmem_create(struct mnt_idmap *idmap, struct inode *dir,
-			struct dentry *dentry, umode_t mode, bool excl)
+			struct dentry *dentry, umode_t mode)
 {
 	return shmem_mknod(idmap, dir, dentry, mode | S_IFREG, 0);
 }
@@ -4057,6 +4069,7 @@ static int shmem_symlink(struct mnt_idmap *idmap, struct inode *dir,
 			goto out_remove_offset;
 		inode->i_op = &shmem_symlink_inode_operations;
 		memcpy(folio_address(folio), symname, len);
+		folio_zero_range(folio, len, folio_size(folio) - len);
 		folio_mark_uptodate(folio);
 		folio_mark_dirty(folio);
 		folio_unlock(folio);
diff --git a/mm/show_mem.c b/mm/show_mem.c
index 43aca5a2ac990..d1288b4c2b640 100644
--- a/mm/show_mem.c
+++ b/mm/show_mem.c
@@ -16,6 +16,7 @@
 #include <linux/vmstat.h>
 
 #include "internal.h"
+#include "page_alloc.h"
 #include "swap.h"
 
 atomic_long_t _totalram_pages __read_mostly;
@@ -116,7 +117,8 @@ void si_meminfo_node(struct sysinfo *val, int nid)
  * Determine whether the node should be displayed or not, depending on whether
  * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
  */
-static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask)
+static bool show_mem_node_skip(unsigned int flags, int nid,
+			       const nodemask_t *nodemask)
 {
 	if (!(flags & SHOW_MEM_FILTER_NODES))
 		return false;
@@ -177,7 +179,8 @@ static bool node_has_managed_zones(pg_data_t *pgdat, int max_zone_idx)
  * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's
  *   cpuset.
  */
-static void show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_zone_idx)
+static void show_free_areas(unsigned int filter, const nodemask_t *nodemask,
+			    int max_zone_idx)
 {
 	unsigned long free_pcp = 0;
 	int cpu, nid;
@@ -402,7 +405,8 @@ static void show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_z
 	show_swap_cache_info();
 }
 
-void __show_mem(unsigned int filter, nodemask_t *nodemask, int max_zone_idx)
+void __show_mem(unsigned int filter, const nodemask_t *nodemask,
+		int max_zone_idx)
 {
 	unsigned long total = 0, reserved = 0, highmem = 0;
 	struct zone *zone;
diff --git a/mm/shuffle.c b/mm/shuffle.c
index fb1393b8b3a9d..82a2c7725a08a 100644
--- a/mm/shuffle.c
+++ b/mm/shuffle.c
@@ -7,6 +7,7 @@
 #include <linux/random.h>
 #include <linux/moduleparam.h>
 #include "internal.h"
+#include "page_alloc.h"
 #include "shuffle.h"
 
 DEFINE_STATIC_KEY_FALSE(page_alloc_shuffle_key);
diff --git a/mm/slab.h b/mm/slab.h
index 281a65233795d..c24c3daaa8691 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -22,6 +22,7 @@
 #define SLAB_ALLOC_NOLOCK	0x01 /* a kmalloc_nolock() allocation */
 #define SLAB_ALLOC_NEW_SLAB	0x02 /* a flag for alloc_slab_obj_exts() */
 #define SLAB_ALLOC_NO_RECURSE	0x04 /* prevent kmalloc() recursion */
+#define SLAB_ALLOC_NO_OBJ_EXT	0x08 /* prevent obj_exts array allocation */
 
 static inline bool alloc_flags_allow_spinning(const unsigned int alloc_flags)
 {
@@ -179,7 +180,7 @@ static inline void *slab_address(const struct slab *slab)
 
 static inline int slab_nid(const struct slab *slab)
 {
-	return memdesc_nid(slab->flags);
+	return memdesc_nid(&slab->flags);
 }
 
 static inline pg_data_t *slab_pgdat(const struct slab *slab)
@@ -386,12 +387,17 @@ static inline unsigned int size_index_elem(unsigned int bytes)
  * KMALLOC_MAX_CACHE_SIZE and the caller must check that.
  */
 static inline struct kmem_cache *
-kmalloc_slab(size_t size, kmem_buckets *b, gfp_t flags, kmalloc_token_t token)
+kmalloc_slab(size_t size, kmem_buckets *b, gfp_t flags, kmalloc_token_t token,
+	     unsigned int alloc_flags)
 {
 	unsigned int index;
+	enum kmalloc_cache_type type = kmalloc_type(flags, token);
+
+	if (alloc_flags & SLAB_ALLOC_NO_OBJ_EXT)
+		type = KMALLOC_NO_OBJ_EXT;
 
 	if (!b)
-		b = &kmalloc_caches[kmalloc_type(flags, token)];
+		b = &kmalloc_caches[type];
 	if (size <= 192)
 		index = kmalloc_size_index[size_index_elem(size)];
 	else
@@ -426,7 +432,8 @@ static inline bool is_kmalloc_normal(struct kmem_cache *s)
 {
 	if (!is_kmalloc_cache(s))
 		return false;
-	return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT));
+
+	return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT|SLAB_NO_OBJ_EXT));
 }
 
 bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj);
@@ -529,6 +536,25 @@ static inline void metadata_access_disable(void)
 	kasan_enable_current();
 }
 
+/*
+ * Return true if KMALLOC_NORMAL caches may need obj_exts arrays.
+ *
+ * Memory allocation profiling requires obj_exts for all caches.
+ * Memcg usually doesn't need them for normal kmalloc caches, but kmalloc types
+ * with a priority higher than KMALLOC_CGROUP can be aliased with KMALLOC_NORMAL.
+ */
+static inline bool need_kmalloc_no_objext(void)
+{
+	if (!mem_alloc_profiling_permanently_disabled())
+		return true;
+
+	if (!mem_cgroup_kmem_disabled() &&
+			(KMALLOC_NORMAL == KMALLOC_RECLAIM))
+		return true;
+
+	return false;
+}
+
 #ifdef CONFIG_SLAB_OBJ_EXT
 
 /*
diff --git a/mm/slab_common.c b/mm/slab_common.c
index b6426d7ceec92..03ecac12cd865 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -783,11 +783,15 @@ u8 kmalloc_size_index[24] __ro_after_init = {
 size_t kmalloc_size_roundup(size_t size)
 {
 	if (size && size <= KMALLOC_MAX_CACHE_SIZE) {
+		struct kmem_cache *s;
+
 		/*
 		 * The flags don't matter since size_index is common to all.
 		 * Neither does the caller for just getting ->object_size.
 		 */
-		return kmalloc_slab(size, NULL, GFP_KERNEL, __kmalloc_token(0))->object_size;
+		s = kmalloc_slab(size, NULL, GFP_KERNEL, __kmalloc_token(0),
+				 SLAB_ALLOC_DEFAULT);
+		return s->object_size;
 	}
 
 	/* Above the smaller buckets, size is a multiple of page size. */
@@ -843,6 +847,12 @@ EXPORT_SYMBOL(kmalloc_size_roundup);
 #define KMALLOC_PARTITION_NAME(N, sz)
 #endif
 
+#ifdef CONFIG_SLAB_OBJ_EXT
+#define KMALLOC_NO_OBJ_EXT_NAME(sz) .name[KMALLOC_NO_OBJ_EXT] = "kmalloc-no-objext-" #sz,
+#else
+#define KMALLOC_NO_OBJ_EXT_NAME(sz)
+#endif
+
 #define INIT_KMALLOC_INFO(__size, __short_size)			\
 {								\
 	.name[KMALLOC_NORMAL]  = "kmalloc-" #__short_size,	\
@@ -850,6 +860,7 @@ EXPORT_SYMBOL(kmalloc_size_roundup);
 	KMALLOC_CGROUP_NAME(__short_size)			\
 	KMALLOC_DMA_NAME(__short_size)				\
 	KMALLOC_PARTITION_NAME(KMALLOC_PARTITION_CACHES_NR, __short_size)	\
+	KMALLOC_NO_OBJ_EXT_NAME(__short_size)			\
 	.size = __size,						\
 }
 
@@ -957,6 +968,12 @@ new_kmalloc_cache(int idx, enum kmalloc_cache_type type)
 			return;
 		}
 		flags |= SLAB_ACCOUNT;
+	} else if (IS_ENABLED(CONFIG_SLAB_OBJ_EXT) && type == KMALLOC_NO_OBJ_EXT) {
+		if (!need_kmalloc_no_objext()) {
+			kmalloc_caches[type][idx] = kmalloc_caches[KMALLOC_NORMAL][idx];
+			return;
+		}
+		flags |= SLAB_NO_OBJ_EXT | SLAB_NO_MERGE;
 	} else if (IS_ENABLED(CONFIG_ZONE_DMA) && (type == KMALLOC_DMA)) {
 		flags |= SLAB_CACHE_DMA;
 	}
diff --git a/mm/slub.c b/mm/slub.c
index 9ec774dc70096..53b4976d38318 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -53,6 +53,7 @@
 #include <trace/events/kmem.h>
 
 #include "internal.h"
+#include "page_alloc.h"
 
 /*
  * Lock order:
@@ -2123,42 +2124,6 @@ static inline void init_slab_obj_exts(struct slab *slab)
 	slab->obj_exts = 0;
 }
 
-/*
- * Calculate the allocation size for slabobj_ext array.
- *
- * When memory allocation profiling is enabled, the obj_exts array
- * could be allocated from the same slab cache it's being allocated for.
- * This would prevent the slab from ever being freed because it would
- * always contain at least one allocated object (its own obj_exts array).
- *
- * To avoid this, increase the allocation size when we detect the array
- * may come from the same cache, forcing it to use a different cache.
- */
-static inline size_t obj_exts_alloc_size(struct kmem_cache *s,
-					 struct slab *slab, gfp_t gfp)
-{
-	size_t sz = sizeof(struct slabobj_ext) * slab->objects;
-	struct kmem_cache *obj_exts_cache;
-
-	if (sz > KMALLOC_MAX_CACHE_SIZE)
-		return sz;
-
-	if (!is_kmalloc_normal(s))
-		return sz;
-
-	obj_exts_cache = kmalloc_slab(sz, NULL, gfp, __kmalloc_token(0));
-	/*
-	 * We can't simply compare s with obj_exts_cache, because partitioned kmalloc
-	 * caches have multiple caches per size, selected by caller address or type.
-	 * Since caller address or type may differ between kmalloc_slab() and actual
-	 * allocation, bump size when sizes are equal.
-	 */
-	if (s->object_size == obj_exts_cache->object_size)
-		return obj_exts_cache->object_size + 1;
-
-	return sz;
-}
-
 int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
 			gfp_t gfp, unsigned int alloc_flags)
 {
@@ -2168,14 +2133,18 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
 	unsigned long new_exts;
 	unsigned long old_exts;
 	struct slabobj_ext *vec;
-	size_t sz;
+	size_t sz = sizeof(struct slabobj_ext) * slab->objects;
 
 	gfp &= ~OBJCGS_CLEAR_MASK;
-	/* Prevent recursive extension vector allocation */
-	alloc_flags |= SLAB_ALLOC_NO_RECURSE;
-	alloc_flags &= ~SLAB_ALLOC_NEW_SLAB;
+	/*
+	 * In most cases, obj_exts arrays are allocated from normal kmalloc.
+	 * However, normal kmalloc caches must allocate them from
+	 * KMALLOC_NO_OBJ_EXT caches to prevent recursion.
+	 */
+	if (is_kmalloc_normal(s))
+		alloc_flags |= SLAB_ALLOC_NO_OBJ_EXT;
 
-	sz = obj_exts_alloc_size(s, slab, gfp);
+	alloc_flags &= ~SLAB_ALLOC_NEW_SLAB;
 
 	/* This will use kmalloc_nolock() if alloc_flags say so */
 	vec = kmalloc_flags(sz, gfp | __GFP_ZERO, alloc_flags, slab_nid(slab));
@@ -2193,8 +2162,21 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
 		return -ENOMEM;
 	}
 
-	VM_WARN_ON_ONCE(virt_to_slab(vec) != NULL &&
-			virt_to_slab(vec)->slab_cache == s);
+	if (IS_ENABLED(CONFIG_DEBUG_VM)) {
+		struct kmem_cache *exts_cache;
+		struct slab *exts_slab;
+
+		exts_slab = virt_to_slab(vec);
+		if (exts_slab) {
+			/*
+			 * The vector must be allocated from either normal or
+			 * KMALLOC_NO_OBJ_EXT kmalloc caches to avoid cycles.
+			 */
+			exts_cache = exts_slab->slab_cache;
+			WARN_ON_ONCE(!is_kmalloc_normal(exts_cache) &&
+					!(exts_cache->flags & SLAB_NO_OBJ_EXT));
+		}
+	}
 
 	new_exts = (unsigned long)vec;
 #ifdef CONFIG_MEMCG
@@ -2217,7 +2199,6 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
 		 * assign slabobj_exts in parallel. In this case the existing
 		 * objcg vector should be reused.
 		 */
-		mark_obj_codetag_empty(vec);
 		if (unlikely(!allow_spin))
 			kfree_nolock(vec);
 		else
@@ -2253,14 +2234,6 @@ static inline void free_slab_obj_exts(struct slab *slab, bool allow_spin)
 		return;
 	}
 
-	/*
-	 * obj_exts was created with SLAB_ALLOC_NO_RECURSE flag, therefore its
-	 * corresponding extension will be NULL. alloc_tag_sub() will throw a
-	 * warning if slab has extensions but the extension of an object is
-	 * NULL, therefore replace NULL with CODETAG_EMPTY to indicate that
-	 * the extension for obj_exts is expected to be NULL.
-	 */
-	mark_obj_codetag_empty(obj_exts);
 	if (allow_spin)
 		kfree(obj_exts);
 	else
@@ -3291,7 +3264,8 @@ static inline struct slab *alloc_slab_page(gfp_t flags, int node,
 	else if (node == NUMA_NO_NODE)
 		page = alloc_frozen_pages(flags, order);
 	else
-		page = __alloc_frozen_pages(flags, order, node, NULL);
+		page = __alloc_frozen_pages(flags, order, node, NULL,
+					    ALLOC_DEFAULT);
 
 	if (!page)
 		return NULL;
@@ -4526,11 +4500,8 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 	return object;
 }
 
-static void *__slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node,
-			       const struct slab_alloc_context *ac)
+static __always_inline int apply_strict_numa_policy(int node)
 {
-	void *object;
-
 #ifdef CONFIG_NUMA
 	if (static_branch_unlikely(&strict_numa) &&
 			node == NUMA_NO_NODE) {
@@ -4551,10 +4522,7 @@ static void *__slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node,
 		}
 	}
 #endif
-
-	object = ___slab_alloc(s, gfpflags, node, ac);
-
-	return object;
+	return node;
 }
 
 static __fastpath_inline
@@ -4759,28 +4727,6 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags,
 	bool node_requested;
 	void *object;
 
-#ifdef CONFIG_NUMA
-	if (static_branch_unlikely(&strict_numa) &&
-			 node == NUMA_NO_NODE) {
-
-		struct mempolicy *mpol = current->mempolicy;
-
-		if (mpol) {
-			/*
-			 * Special BIND rule support. If the local node
-			 * is in permitted set then do not redirect
-			 * to a particular node.
-			 * Otherwise we apply the memory policy to get
-			 * the node we need to allocate on.
-			 */
-			if (mpol->mode != MPOL_BIND ||
-					!node_isset(numa_mem_id(), mpol->nodes))
-
-				node = mempolicy_slab_node();
-		}
-	}
-#endif
-
 	node_requested = IS_ENABLED(CONFIG_NUMA) && node != NUMA_NO_NODE;
 
 	/*
@@ -4930,10 +4876,12 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s,
 	if (unlikely(object))
 		goto out;
 
+	node = apply_strict_numa_policy(node);
+
 	object = alloc_from_pcs(s, gfpflags, ac->alloc_flags, node);
 
 	if (unlikely(!object))
-		object = __slab_alloc_node(s, gfpflags, node, ac);
+		object = ___slab_alloc(s, gfpflags, node, ac);
 
 	maybe_wipe_obj_freeptr(s, object);
 
@@ -5301,7 +5249,8 @@ static void *___kmalloc_large_node(size_t size, gfp_t flags, int node)
 	if (node == NUMA_NO_NODE)
 		page = alloc_frozen_pages_noprof(flags, order);
 	else
-		page = __alloc_frozen_pages_noprof(flags, order, node, NULL);
+		page = __alloc_frozen_pages_noprof(flags, order, node, NULL,
+						   ALLOC_DEFAULT);
 
 	if (page) {
 		ptr = page_address(page);
@@ -5356,7 +5305,7 @@ void *__do_kmalloc_node(kmem_buckets *b, gfp_t flags, int node,
 	if (unlikely(!size))
 		return ZERO_SIZE_PTR;
 
-	s = kmalloc_slab(size, b, flags, token);
+	s = kmalloc_slab(size, b, flags, token, ac->alloc_flags);
 
 	ret = slab_alloc_node(s, flags, node, ac);
 	ret = kasan_kmalloc(s, ret, size, flags);
@@ -5405,21 +5354,17 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f
 	if (unlikely(!size))
 		return ZERO_SIZE_PTR;
 
-	/*
-	 * See the comment for the same check in
-	 * alloc_frozen_pages_nolock_noprof()
-	 */
-	if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq()))
+	if (!can_spin_trylock())
 		return NULL;
 
-	/* On UP, spin_trylock() always succeeds even when it is locked */
-	if (!IS_ENABLED(CONFIG_SMP) && in_nmi())
-		return NULL;
+	node = apply_strict_numa_policy(node);
 
 retry:
 	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
 		return NULL;
-	s = kmalloc_slab(size, NULL, gfp_flags, PASS_TOKEN_PARAM(token));
+
+	s = kmalloc_slab(size, NULL, gfp_flags, PASS_TOKEN_PARAM(token),
+			 ac->alloc_flags);
 
 	if (!(s->flags & __CMPXCHG_DOUBLE) && !kmem_cache_debug(s))
 		/*
@@ -5440,10 +5385,10 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f
 	/*
 	 * Do not call slab_alloc_node(), since trylock mode isn't
 	 * compatible with slab_pre_alloc_hook/should_failslab and
-	 * kfence_alloc. Hence call __slab_alloc_node() (at most twice)
+	 * kfence_alloc. Hence call ___slab_alloc() (at most twice)
 	 * and slab_post_alloc_hook() directly.
 	 */
-	ret = __slab_alloc_node(s, gfp_flags, node, ac);
+	ret = ___slab_alloc(s, gfp_flags, node, ac);
 
 	/*
 	 * It's possible we failed due to trylock as we preempted someone with
@@ -6203,49 +6148,21 @@ static __always_inline bool can_free_to_pcs(struct slab *slab)
 }
 
 /*
- * Bulk free objects to the percpu sheaves.
- * Unlike free_to_pcs() this includes the calls to all necessary hooks
- * and the fallback to freeing to slab pages.
+ * Try to free as many objects (already processed by free hooks) as possible to
+ * a single per-cpu sheaf.
+ *
+ * Returns how many objects were freed. Zero means failure and the caller should
+ * fall back to __kmem_cache_free_bulk().
  */
-static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
+static unsigned int __free_to_pcs_batch(struct kmem_cache *s, size_t size, void **p)
 {
 	struct slub_percpu_sheaves *pcs;
 	struct slab_sheaf *main, *empty;
-	bool init = slab_want_init_on_free(s);
-	unsigned int batch, i = 0;
 	struct node_barn *barn;
-	void *remote_objects[PCS_BATCH_MAX];
-	unsigned int remote_nr = 0;
-
-next_remote_batch:
-	while (i < size) {
-		struct slab *slab = virt_to_slab(p[i]);
-
-		memcg_slab_free_hook(s, slab, p + i, 1);
-		alloc_tagging_slab_free_hook(s, slab, p + i, 1);
-
-		if (unlikely(!slab_free_hook(s, p[i], init, false))) {
-			p[i] = p[--size];
-			continue;
-		}
-
-		if (unlikely(!can_free_to_pcs(slab))) {
-			remote_objects[remote_nr] = p[i];
-			p[i] = p[--size];
-			if (++remote_nr >= PCS_BATCH_MAX)
-				goto flush_remote;
-			continue;
-		}
-
-		i++;
-	}
-
-	if (!size)
-		goto flush_remote;
+	unsigned int batch;
 
-next_batch:
 	if (!local_trylock(&s->cpu_sheaves->lock))
-		goto fallback;
+		return 0;
 
 	pcs = this_cpu_ptr(s->cpu_sheaves);
 
@@ -6291,36 +6208,76 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
 
 	stat_add(s, FREE_FASTPATH, batch);
 
-	if (batch < size) {
-		p += batch;
-		size -= batch;
-		goto next_batch;
-	}
-
-	if (remote_nr)
-		goto flush_remote;
-
-	return;
+	return batch;
 
 no_empty:
 	local_unlock(&s->cpu_sheaves->lock);
 
+	return 0;
+}
+
+/*
+ * Bulk free objects to the percpu sheaves.
+ * Unlike free_to_pcs() this includes the calls to all necessary hooks
+ * and the fallback to freeing to slab pages.
+ */
+static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
+{
+	bool init = slab_want_init_on_free(s);
+	void **remote_objects = p;
+	unsigned int remote_nr = 0;
+
 	/*
-	 * if we depleted all empty sheaves in the barn or there are too
-	 * many full sheaves, free the rest to slab pages
+	 * Process the free hooks and separate out remote objects by
+	 * partitioning the 'p' array in place:
+	 *
+	 * [0, remote_nr) - processed remote objects
+	 * [remote_nr, i) - processed local objects
+	 * [i, size)      - unprocessed objects
 	 */
-fallback:
-	__kmem_cache_free_bulk(s, size, p);
-	stat_add(s, FREE_SLOWPATH, size);
+	for (unsigned int i = 0; i < size;) {
+		struct slab *slab = virt_to_slab(p[i]);
+
+		memcg_slab_free_hook(s, slab, p + i, 1);
+		alloc_tagging_slab_free_hook(s, slab, p + i, 1);
+
+		if (unlikely(!slab_free_hook(s, p[i], init, false))) {
+			p[i] = p[--size];
+			continue;
+		}
+
+		if (unlikely(!can_free_to_pcs(slab))) {
+			if (i != remote_nr)
+				swap(remote_objects[remote_nr], p[i]);
+			remote_nr++;
+		}
+
+		i++;
+	}
 
-flush_remote:
+	p += remote_nr;
+	size -= remote_nr;
+
+	while (size) {
+		unsigned int batch_freed = __free_to_pcs_batch(s, size, p);
+
+		if (!batch_freed) {
+			__kmem_cache_free_bulk(s, size, p);
+			stat_add(s, FREE_SLOWPATH, size);
+			break;
+		}
+
+		p += batch_freed;
+		size -= batch_freed;
+	}
+
+	/*
+	 * Processing remote objects last decreases the chances of cpu migration
+	 * while freeing to sheaves and compromising object locality
+	 */
 	if (remote_nr) {
-		__kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]);
+		__kmem_cache_free_bulk(s, remote_nr, remote_objects);
 		stat_add(s, FREE_SLOWPATH, remote_nr);
-		if (i < size) {
-			remote_nr = 0;
-			goto next_remote_batch;
-		}
 	}
 }
 
@@ -7801,12 +7758,12 @@ static unsigned int calculate_sheaf_capacity(struct kmem_cache *s,
 		return 0;
 
 	/*
-	 * Bootstrap caches can't have sheaves for now (SLAB_NO_OBJ_EXT).
+	 * Bootstrap caches can't have sheaves for now (SLAB_NO_SHEAVES).
 	 * SLAB_NOLEAKTRACE caches (e.g., kmemleak's object_cache) must not
 	 * have sheaves to avoid recursion when sheaf allocation triggers
 	 * kmemleak tracking.
 	 */
-	if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE))
+	if (s->flags & (SLAB_NO_SHEAVES | SLAB_NOLEAKTRACE))
 		return 0;
 
 	/*
@@ -7981,10 +7938,10 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s)
 		s->allocflags |= __GFP_RECLAIMABLE;
 
 	/*
-	 * For KMALLOC_NORMAL caches we enable sheaves later by
-	 * bootstrap_kmalloc_sheaves() to avoid recursion
+	 * For kmalloc caches we enable sheaves later by
+	 * bootstrap_kmalloc_sheaves() to avoid recursion.
 	 */
-	if (!is_kmalloc_normal(s))
+	if (!is_kmalloc_cache(s))
 		s->sheaf_capacity = calculate_sheaf_capacity(s, args);
 
 	/*
@@ -8499,6 +8456,8 @@ static void __init bootstrap_cache_sheaves(struct kmem_cache *s)
 	bool failed = false;
 	int node, cpu;
 
+	VM_WARN_ON_ONCE(cache_has_sheaves(s));
+
 	capacity = calculate_sheaf_capacity(s, &empty_args);
 
 	/* capacity can be 0 due to debugging or SLUB_TINY */
@@ -8548,10 +8507,13 @@ static void __init bootstrap_kmalloc_sheaves(void)
 {
 	enum kmalloc_cache_type type;
 
-	for (type = KMALLOC_NORMAL; type <= KMALLOC_PARTITION_END; type++) {
+	for (type = KMALLOC_NORMAL; type < NR_KMALLOC_TYPES; type++) {
 		for (int idx = 0; idx < KMALLOC_SHIFT_HIGH + 1; idx++) {
-			if (kmalloc_caches[type][idx])
-				bootstrap_cache_sheaves(kmalloc_caches[type][idx]);
+			struct kmem_cache *s = kmalloc_caches[type][idx];
+
+			/* Do not bootstrap twice when caches are aliased */
+			if (s && !cache_has_sheaves(s))
+				bootstrap_cache_sheaves(s);
 		}
 	}
 }
@@ -8583,7 +8545,8 @@ void __init kmem_cache_init(void)
 
 	create_boot_cache(kmem_cache_node, "kmem_cache_node",
 			sizeof(struct kmem_cache_node),
-			SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0);
+			SLAB_HWCACHE_ALIGN | SLAB_NO_SHEAVES | SLAB_NO_OBJ_EXT,
+			0, 0);
 
 	hotplug_node_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
 
@@ -8593,7 +8556,8 @@ void __init kmem_cache_init(void)
 	create_boot_cache(kmem_cache, "kmem_cache",
 			offsetof(struct kmem_cache, per_node) +
 				nr_node_ids * sizeof(struct kmem_cache_per_node_ptrs),
-			SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0);
+			SLAB_HWCACHE_ALIGN | SLAB_NO_SHEAVES | SLAB_NO_OBJ_EXT,
+			0, 0);
 
 	kmem_cache = bootstrap(&boot_kmem_cache);
 	kmem_cache_node = bootstrap(&boot_kmem_cache_node);
@@ -8992,14 +8956,12 @@ static void process_slab(struct loc_track *t, struct kmem_cache *s,
 enum slab_stat_type {
 	SL_ALL,			/* All slabs */
 	SL_PARTIAL,		/* Only partially allocated slabs */
-	SL_CPU,			/* Only slabs used for cpu caches */
 	SL_OBJECTS,		/* Determine allocated objects not slabs */
 	SL_TOTAL		/* Determine object capacity not slabs */
 };
 
 #define SO_ALL		(1 << SL_ALL)
 #define SO_PARTIAL	(1 << SL_PARTIAL)
-#define SO_CPU		(1 << SL_CPU)
 #define SO_OBJECTS	(1 << SL_OBJECTS)
 #define SO_TOTAL	(1 << SL_TOTAL)
 
@@ -9188,7 +9150,7 @@ SLAB_ATTR_RO(partial);
 
 static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
 {
-	return show_slab_objects(s, buf, SO_CPU);
+	return sysfs_emit(buf, "0\n");
 }
 SLAB_ATTR_RO(cpu_slabs);
 
@@ -9690,6 +9652,11 @@ static int sysfs_slab_add(struct kmem_cache *s)
 
 	s->kobj.kset = kset;
 	err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
+	/*
+	 * Intentionally skip kobject_put(). See commit 2420baa8e046
+	 * ("mm/slab: Allow cache creation to proceed even if sysfs
+	 * registration fails")
+	 */
 	if (err)
 		goto out;
 
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 99e2be39671b6..5a2469fb1838c 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -41,6 +41,8 @@
 #define VMEMMAP_POPULATE_PAGEREF	0x0001
 
 #include "internal.h"
+#include "mm_init.h"
+#include "sparse.h"
 
 /*
  * Allocate a block of memory to be used to back the virtual memory map
@@ -342,8 +344,8 @@ static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone *
 	 *
 	 * Any initialization done here will be overwritten by memmap_init().
 	 *
-	 * hugetlb_vmemmap_init() will take care of initialization after
-	 * memmap_init().
+	 * hugetlb_bootmem_struct_page_init() will take care of initialization
+	 * after memmap_init().
 	 */
 
 	p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
@@ -564,6 +566,8 @@ struct page * __meminit __populate_section_memmap(unsigned long pfn,
 	if (r < 0)
 		return NULL;
 
+	flush_cache_vmap(start, end);
+
 	return pfn_to_page(pfn);
 }
 
@@ -579,17 +583,6 @@ void __init sparse_vmemmap_init_nid_early(int nid)
 {
 	hugetlb_vmemmap_init_early(nid);
 }
-
-/*
- * This is called just before the initialization of page structures
- * through memmap_init. Zones are now initialized, so any work that
- * needs to be done that needs zone information can be done from
- * here.
- */
-void __init sparse_vmemmap_init_nid_late(int nid)
-{
-	hugetlb_vmemmap_init_late(nid);
-}
 #endif
 
 static void subsection_mask_set(unsigned long *map, unsigned long pfn,
@@ -601,7 +594,7 @@ static void subsection_mask_set(unsigned long *map, unsigned long pfn,
 	bitmap_set(map, idx, end - idx + 1);
 }
 
-void __init sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages)
+static void __init sparse_init_subsection_map_range(unsigned long pfn, unsigned long nr_pages)
 {
 	int end_sec_nr = pfn_to_section_nr(pfn + nr_pages - 1);
 	unsigned long nr, start_sec_nr = pfn_to_section_nr(pfn);
@@ -624,6 +617,15 @@ void __init sparse_init_subsection_map(unsigned long pfn, unsigned long nr_pages
 	}
 }
 
+void __init sparse_init_subsection_map(void)
+{
+	int i, nid;
+	unsigned long start, end;
+
+	for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid)
+		sparse_init_subsection_map_range(start, end - start);
+}
+
 #ifdef CONFIG_MEMORY_HOTPLUG
 
 /* Mark all memory sections within the pfn range as online */
diff --git a/mm/sparse.c b/mm/sparse.c
index 16ac6df3c89fa..704a9dec2b9af 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -13,9 +13,10 @@
 #include <linux/vmalloc.h>
 #include <linux/swap.h>
 #include <linux/swapops.h>
-#include <linux/bootmem_info.h>
 #include <linux/vmstat.h>
 #include "internal.h"
+#include "mm_init.h"
+#include "sparse.h"
 #include <asm/dma.h>
 
 /*
@@ -43,7 +44,7 @@ static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
 static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
 #endif
 
-int memdesc_nid(memdesc_flags_t mdf)
+int memdesc_nid(const memdesc_flags_t *mdf)
 {
 	return section_to_node_table[memdesc_section(mdf)];
 }
@@ -239,15 +240,8 @@ struct page __init *__populate_section_memmap(unsigned long pfn,
 		struct dev_pagemap *pgmap)
 {
 	unsigned long size = section_map_size();
-	struct page *map;
-	phys_addr_t addr = __pa(MAX_DMA_ADDRESS);
 
-	map = memmap_alloc(size, size, addr, nid, false);
-	if (!map)
-		panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n",
-		      __func__, size, PAGE_SIZE, nid, &addr);
-
-	return map;
+	return memmap_alloc(size, size, __pa(MAX_DMA_ADDRESS), nid, false);
 }
 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
 
@@ -300,17 +294,14 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
 				   unsigned long map_count)
 {
 	unsigned long pnum;
-	struct page *map;
-	struct mem_section *ms;
 
-	if (sparse_usage_init(nid, map_count)) {
-		pr_err("%s: node[%d] usemap allocation failed", __func__, nid);
-		goto failed;
-	}
+	if (sparse_usage_init(nid, map_count))
+		panic("Failed to allocate usemap for node %d\n", nid);
 
 	sparse_vmemmap_init_nid_early(nid);
 
 	for_each_present_section_nr(pnum_begin, pnum) {
+		struct mem_section *ms;
 		unsigned long pfn = section_nr_to_pfn(pnum);
 
 		if (pnum >= pnum_end)
@@ -318,34 +309,18 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
 
 		ms = __nr_to_section(pnum);
 		if (!preinited_vmemmap_section(ms)) {
+			struct page *map;
+
 			map = __populate_section_memmap(pfn, PAGES_PER_SECTION,
-					nid, NULL, NULL);
-			if (!map) {
-				pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.",
-				       __func__, nid);
-				pnum_begin = pnum;
-				sparse_usage_fini();
-				goto failed;
-			}
+							nid, NULL, NULL);
+			if (!map)
+				panic("Failed to allocate memmap for section %lu\n", pnum);
 			memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page),
 							   PAGE_SIZE));
 			sparse_init_early_section(nid, map, pnum, 0);
 		}
 	}
 	sparse_usage_fini();
-	return;
-failed:
-	/*
-	 * We failed to allocate, mark all the following pnums as not present,
-	 * except the ones already initialized earlier.
-	 */
-	for_each_present_section_nr(pnum_begin, pnum) {
-		if (pnum >= pnum_end)
-			break;
-		ms = __nr_to_section(pnum);
-		if (!preinited_vmemmap_section(ms))
-			ms->section_mem_map = 0;
-	}
 }
 
 /*
@@ -369,9 +344,6 @@ void __init sparse_init(void)
 	pnum_begin = first_present_section_nr();
 	nid_begin = sparse_early_nid(__nr_to_section(pnum_begin));
 
-	/* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */
-	set_pageblock_order();
-
 	for_each_present_section_nr(pnum_begin + 1, pnum_end) {
 		int nid = sparse_early_nid(__nr_to_section(pnum_end));
 
@@ -387,5 +359,6 @@ void __init sparse_init(void)
 	}
 	/* cover the last node */
 	sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count);
+	sparse_init_subsection_map();
 	vmemmap_populate_print_last();
 }
diff --git a/mm/sparse.h b/mm/sparse.h
new file mode 100644
index 0000000000000..95aa031213f22
--- /dev/null
+++ b/mm/sparse.h
@@ -0,0 +1,63 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * sparse.h:
+ *
+ * mm/ internal sparse and sparse-vmemmap declarations
+ */
+
+#ifndef __MM_SPARSE_H
+#define __MM_SPARSE_H
+
+#include <linux/mmzone.h>
+
+/*
+ * mm/sparse.c
+ */
+#ifdef CONFIG_SPARSEMEM
+void sparse_init(void);
+int sparse_index_init(unsigned long section_nr, int nid);
+
+static inline void sparse_init_one_section(struct mem_section *ms,
+		unsigned long pnum, struct page *mem_map,
+		struct mem_section_usage *usage, unsigned long flags)
+{
+	unsigned long coded_mem_map;
+
+	BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT);
+
+	/*
+	 * We encode the start PFN of the section into the mem_map such that
+	 * page_to_pfn() on !CONFIG_SPARSEMEM_VMEMMAP can simply subtract it
+	 * from the page pointer to obtain the PFN.
+	 */
+	coded_mem_map = (unsigned long)(mem_map - section_nr_to_pfn(pnum));
+	VM_WARN_ON_ONCE(coded_mem_map & ~SECTION_MAP_MASK);
+
+	ms->section_mem_map &= ~SECTION_MAP_MASK;
+	ms->section_mem_map |= coded_mem_map;
+	ms->section_mem_map |= flags | SECTION_HAS_MEM_MAP;
+	ms->usage = usage;
+}
+
+static inline void __section_mark_present(struct mem_section *ms,
+		unsigned long section_nr)
+{
+	if (section_nr > __highest_present_section_nr)
+		__highest_present_section_nr = section_nr;
+
+	ms->section_mem_map |= SECTION_MARKED_PRESENT;
+}
+#else
+static inline void sparse_init(void) {}
+#endif /* CONFIG_SPARSEMEM */
+
+/*
+ * mm/sparse-vmemmap.c
+ */
+#ifdef CONFIG_SPARSEMEM_VMEMMAP
+void sparse_init_subsection_map(void);
+#else
+static inline void sparse_init_subsection_map(void) {}
+#endif /* CONFIG_SPARSEMEM_VMEMMAP */
+
+#endif /* __MM_SPARSE_H */
diff --git a/mm/swap.c b/mm/swap.c
deleted file mode 100644
index 588f50d8f1a8c..0000000000000
--- a/mm/swap.c
+++ /dev/null
@@ -1,1205 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- *  linux/mm/swap.c
- *
- *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
- */
-
-/*
- * This file contains the default values for the operation of the
- * Linux VM subsystem. Fine-tuning documentation can be found in
- * Documentation/admin-guide/sysctl/vm.rst.
- * Started 18.12.91
- * Swap aging added 23.2.95, Stephen Tweedie.
- * Buffermem limits added 12.3.98, Rik van Riel.
- */
-
-#include <linux/mm.h>
-#include <linux/sched.h>
-#include <linux/kernel_stat.h>
-#include <linux/swap.h>
-#include <linux/mman.h>
-#include <linux/pagemap.h>
-#include <linux/folio_batch.h>
-#include <linux/init.h>
-#include <linux/export.h>
-#include <linux/mm_inline.h>
-#include <linux/percpu_counter.h>
-#include <linux/memremap.h>
-#include <linux/percpu.h>
-#include <linux/cpu.h>
-#include <linux/notifier.h>
-#include <linux/backing-dev.h>
-#include <linux/memcontrol.h>
-#include <linux/gfp.h>
-#include <linux/uio.h>
-#include <linux/hugetlb.h>
-#include <linux/page_idle.h>
-#include <linux/local_lock.h>
-#include <linux/buffer_head.h>
-
-#include "internal.h"
-
-#define CREATE_TRACE_POINTS
-#include <trace/events/pagemap.h>
-
-/* How many pages do we try to swap or page in/out together? As a power of 2 */
-int page_cluster;
-static const int page_cluster_max = 31;
-
-struct cpu_fbatches {
-	/*
-	 * The following folio batches are grouped together because they are protected
-	 * by disabling preemption (and interrupts remain enabled).
-	 */
-	local_lock_t lock;
-	struct folio_batch lru_add;
-	struct folio_batch lru_deactivate_file;
-	struct folio_batch lru_deactivate;
-	struct folio_batch lru_lazyfree;
-#ifdef CONFIG_SMP
-	struct folio_batch lru_activate;
-#endif
-	/* Protecting the following batches which require disabling interrupts */
-	local_lock_t lock_irq;
-	struct folio_batch lru_move_tail;
-};
-
-static DEFINE_PER_CPU(struct cpu_fbatches, cpu_fbatches) = {
-	.lock = INIT_LOCAL_LOCK(lock),
-	.lock_irq = INIT_LOCAL_LOCK(lock_irq),
-};
-
-static void __page_cache_release(struct folio *folio, struct lruvec **lruvecp,
-		unsigned long *flagsp)
-{
-	if (folio_test_lru(folio)) {
-		folio_lruvec_relock_irqsave(folio, lruvecp, flagsp);
-		lruvec_del_folio(*lruvecp, folio);
-		__folio_clear_lru_flags(folio);
-	}
-}
-
-/*
- * This path almost never happens for VM activity - pages are normally freed
- * in batches.  But it gets used by networking - and for compound pages.
- */
-static void page_cache_release(struct folio *folio)
-{
-	struct lruvec *lruvec = NULL;
-	unsigned long flags;
-
-	__page_cache_release(folio, &lruvec, &flags);
-	if (lruvec)
-		lruvec_unlock_irqrestore(lruvec, flags);
-}
-
-void __folio_put(struct folio *folio)
-{
-	if (unlikely(folio_is_zone_device(folio))) {
-		free_zone_device_folio(folio);
-		return;
-	}
-
-	if (folio_test_hugetlb(folio)) {
-		free_huge_folio(folio);
-		return;
-	}
-
-	page_cache_release(folio);
-	folio_unqueue_deferred_split(folio);
-	mem_cgroup_uncharge(folio);
-	free_frozen_pages(&folio->page, folio_order(folio));
-}
-EXPORT_SYMBOL(__folio_put);
-
-typedef void (*move_fn_t)(struct lruvec *lruvec, struct folio *folio);
-
-static void lru_add(struct lruvec *lruvec, struct folio *folio)
-{
-	int was_unevictable = folio_test_clear_unevictable(folio);
-	long nr_pages = folio_nr_pages(folio);
-
-	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
-
-	/*
-	 * Is an smp_mb__after_atomic() still required here, before
-	 * folio_evictable() tests the mlocked flag, to rule out the possibility
-	 * of stranding an evictable folio on an unevictable LRU?  I think
-	 * not, because __munlock_folio() only clears the mlocked flag
-	 * while the LRU lock is held.
-	 *
-	 * (That is not true of __page_cache_release(), and not necessarily
-	 * true of folios_put(): but those only clear the mlocked flag after
-	 * folio_put_testzero() has excluded any other users of the folio.)
-	 */
-	if (folio_evictable(folio)) {
-		if (was_unevictable)
-			__count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages);
-	} else {
-		folio_clear_active(folio);
-		folio_set_unevictable(folio);
-		/*
-		 * folio->mlock_count = !!folio_test_mlocked(folio)?
-		 * But that leaves __mlock_folio() in doubt whether another
-		 * actor has already counted the mlock or not.  Err on the
-		 * safe side, underestimate, let page reclaim fix it, rather
-		 * than leaving a page on the unevictable LRU indefinitely.
-		 */
-		folio->mlock_count = 0;
-		if (!was_unevictable)
-			__count_vm_events(UNEVICTABLE_PGCULLED, nr_pages);
-	}
-
-	lruvec_add_folio(lruvec, folio);
-	trace_mm_lru_insertion(folio);
-}
-
-static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
-{
-	int i;
-	struct lruvec *lruvec = NULL;
-	unsigned long flags = 0;
-	struct folio_batch free_fbatch;
-	bool is_lru_add = (move_fn == lru_add);
-
-	/*
-	 * If we're adding to the LRU, preemptively filter dead folios. Use
-	 * this dedicated folio batch for temp storage and deferred cleanup.
-	 */
-	if (is_lru_add)
-		folio_batch_init(&free_fbatch);
-
-	for (i = 0; i < folio_batch_count(fbatch); i++) {
-		struct folio *folio = fbatch->folios[i];
-
-		/* block memcg migration while the folio moves between lru */
-		if (!is_lru_add && !folio_test_clear_lru(folio))
-			continue;
-
-		/*
-		 * Filter dead folios by moving them from the add batch to the temp
-		 * batch for freeing after this loop.
-		 *
-		 * We're bypassing normal cleanup. Clear flags that are not
-		 * applicable to dead folios.
-		 *
-		 * Since the folio may be part of a huge page, unqueue from
-		 * deferred split list to avoid a dangling list entry.
-		 */
-		if (is_lru_add && folio_ref_freeze(folio, 1)) {
-			__folio_clear_active(folio);
-			__folio_clear_unevictable(folio);
-			folio_unqueue_deferred_split(folio);
-			fbatch->folios[i] = NULL;
-			folio_batch_add(&free_fbatch, folio);
-			continue;
-		}
-
-		folio_lruvec_relock_irqsave(folio, &lruvec, &flags);
-		move_fn(lruvec, folio);
-
-		folio_set_lru(folio);
-	}
-
-	if (lruvec)
-		lruvec_unlock_irqrestore(lruvec, flags);
-
-	/* Cleanup filtered dead folios. */
-	if (is_lru_add) {
-		mem_cgroup_uncharge_folios(&free_fbatch);
-		free_unref_folios(&free_fbatch);
-	}
-
-	folios_put(fbatch);
-}
-
-static void __folio_batch_add_and_move(struct folio_batch __percpu *fbatch,
-		struct folio *folio, move_fn_t move_fn, bool disable_irq)
-{
-	unsigned long flags;
-
-	folio_get(folio);
-
-	if (disable_irq)
-		local_lock_irqsave(&cpu_fbatches.lock_irq, flags);
-	else
-		local_lock(&cpu_fbatches.lock);
-
-	if (!folio_batch_add(this_cpu_ptr(fbatch), folio) ||
-			!folio_may_be_lru_cached(folio) || lru_cache_disabled())
-		folio_batch_move_lru(this_cpu_ptr(fbatch), move_fn);
-
-	if (disable_irq)
-		local_unlock_irqrestore(&cpu_fbatches.lock_irq, flags);
-	else
-		local_unlock(&cpu_fbatches.lock);
-}
-
-#define folio_batch_add_and_move(folio, op)		\
-	__folio_batch_add_and_move(			\
-		&cpu_fbatches.op,			\
-		folio,					\
-		op,					\
-		offsetof(struct cpu_fbatches, op) >=	\
-		offsetof(struct cpu_fbatches, lock_irq)	\
-	)
-
-static void lru_move_tail(struct lruvec *lruvec, struct folio *folio)
-{
-	if (folio_test_unevictable(folio))
-		return;
-
-	lruvec_del_folio(lruvec, folio);
-	folio_clear_active(folio);
-	lruvec_add_folio_tail(lruvec, folio);
-	__count_vm_events(PGROTATED, folio_nr_pages(folio));
-}
-
-/*
- * Writeback is about to end against a folio which has been marked for
- * immediate reclaim.  If it still appears to be reclaimable, move it
- * to the tail of the inactive list.
- *
- * folio_rotate_reclaimable() must disable IRQs, to prevent nasty races.
- */
-void folio_rotate_reclaimable(struct folio *folio)
-{
-	if (folio_test_locked(folio) || folio_test_dirty(folio) ||
-	    folio_test_unevictable(folio) || !folio_test_lru(folio))
-		return;
-
-	folio_batch_add_and_move(folio, lru_move_tail);
-}
-
-void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file,
-		unsigned int nr_io, unsigned int nr_rotated)
-		__releases(lruvec->lru_lock)
-		__releases(rcu)
-{
-	unsigned long cost;
-
-	/*
-	 * Reflect the relative cost of incurring IO and spending CPU
-	 * time on rotations. This doesn't attempt to make a precise
-	 * comparison, it just says: if reloads are about comparable
-	 * between the LRU lists, or rotations are overwhelmingly
-	 * different between them, adjust scan balance for CPU work.
-	 */
-	cost = nr_io * SWAP_CLUSTER_MAX + nr_rotated;
-	if (!cost) {
-		spin_unlock_irq(&lruvec->lru_lock);
-		rcu_read_unlock();
-		return;
-	}
-
-	for (;;) {
-		unsigned long lrusize;
-
-		/* Record cost event */
-		if (file)
-			lruvec->file_cost += cost;
-		else
-			lruvec->anon_cost += cost;
-
-		/*
-		 * Decay previous events
-		 *
-		 * Because workloads change over time (and to avoid
-		 * overflow) we keep these statistics as a floating
-		 * average, which ends up weighing recent refaults
-		 * more than old ones.
-		 */
-		lrusize = lruvec_page_state(lruvec, NR_INACTIVE_ANON) +
-			  lruvec_page_state(lruvec, NR_ACTIVE_ANON) +
-			  lruvec_page_state(lruvec, NR_INACTIVE_FILE) +
-			  lruvec_page_state(lruvec, NR_ACTIVE_FILE);
-
-		if (lruvec->file_cost + lruvec->anon_cost > lrusize / 4) {
-			lruvec->file_cost /= 2;
-			lruvec->anon_cost /= 2;
-		}
-
-		spin_unlock_irq(&lruvec->lru_lock);
-		lruvec = parent_lruvec(lruvec);
-		if (!lruvec) {
-			rcu_read_unlock();
-			break;
-		}
-		spin_lock_irq(&lruvec->lru_lock);
-	}
-}
-
-void lru_note_cost_refault(struct folio *folio)
-{
-	struct lruvec *lruvec;
-
-	lruvec = folio_lruvec_lock_irq(folio);
-	lru_note_cost_unlock_irq(lruvec, folio_is_file_lru(folio),
-				folio_nr_pages(folio), 0);
-}
-
-static void lru_activate(struct lruvec *lruvec, struct folio *folio)
-{
-	long nr_pages = folio_nr_pages(folio);
-
-	if (folio_test_active(folio) || folio_test_unevictable(folio))
-		return;
-
-
-	lruvec_del_folio(lruvec, folio);
-	folio_set_active(folio);
-	lruvec_add_folio(lruvec, folio);
-	trace_mm_lru_activate(folio);
-
-	__count_vm_events(PGACTIVATE, nr_pages);
-	count_memcg_events(lruvec_memcg(lruvec), PGACTIVATE, nr_pages);
-}
-
-#ifdef CONFIG_SMP
-static void folio_activate_drain(int cpu)
-{
-	struct folio_batch *fbatch = &per_cpu(cpu_fbatches.lru_activate, cpu);
-
-	if (folio_batch_count(fbatch))
-		folio_batch_move_lru(fbatch, lru_activate);
-}
-
-void folio_activate(struct folio *folio)
-{
-	if (folio_test_active(folio) || folio_test_unevictable(folio) ||
-	    !folio_test_lru(folio))
-		return;
-
-	folio_batch_add_and_move(folio, lru_activate);
-}
-
-#else
-static inline void folio_activate_drain(int cpu)
-{
-}
-
-void folio_activate(struct folio *folio)
-{
-	struct lruvec *lruvec;
-
-	if (!folio_test_clear_lru(folio))
-		return;
-
-	lruvec = folio_lruvec_lock_irq(folio);
-	lru_activate(lruvec, folio);
-	lruvec_unlock_irq(lruvec);
-	folio_set_lru(folio);
-}
-#endif
-
-static void __lru_cache_activate_folio(struct folio *folio)
-{
-	struct folio_batch *fbatch;
-	int i;
-
-	local_lock(&cpu_fbatches.lock);
-	fbatch = this_cpu_ptr(&cpu_fbatches.lru_add);
-
-	/*
-	 * Search backwards on the optimistic assumption that the folio being
-	 * activated has just been added to this batch. Note that only
-	 * the local batch is examined as a !LRU folio could be in the
-	 * process of being released, reclaimed, migrated or on a remote
-	 * batch that is currently being drained. Furthermore, marking
-	 * a remote batch's folio active potentially hits a race where
-	 * a folio is marked active just after it is added to the inactive
-	 * list causing accounting errors and BUG_ON checks to trigger.
-	 */
-	for (i = folio_batch_count(fbatch) - 1; i >= 0; i--) {
-		struct folio *batch_folio = fbatch->folios[i];
-
-		if (batch_folio == folio) {
-			folio_set_active(folio);
-			break;
-		}
-	}
-
-	local_unlock(&cpu_fbatches.lock);
-}
-
-#ifdef CONFIG_LRU_GEN
-
-static void lru_gen_inc_refs(struct folio *folio)
-{
-	unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
-
-	if (folio_test_unevictable(folio))
-		return;
-
-	/* see the comment on LRU_REFS_FLAGS */
-	if (!folio_test_referenced(folio)) {
-		set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
-		return;
-	}
-
-	do {
-		if ((old_flags & LRU_REFS_MASK) == LRU_REFS_MASK) {
-			if (!folio_test_workingset(folio))
-				folio_set_workingset(folio);
-			return;
-		}
-
-		new_flags = old_flags + BIT(LRU_REFS_PGOFF);
-	} while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
-}
-
-static bool lru_gen_clear_refs(struct folio *folio)
-{
-	int gen = folio_lru_gen(folio);
-	int type = folio_is_file_lru(folio);
-	unsigned long seq;
-
-	if (gen < 0)
-		return true;
-
-	set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS | BIT(PG_workingset), 0);
-
-	rcu_read_lock();
-	seq = READ_ONCE(folio_lruvec(folio)->lrugen.min_seq[type]);
-	rcu_read_unlock();
-	/* whether can do without shuffling under the LRU lock */
-	return gen == lru_gen_from_seq(seq);
-}
-
-#else /* !CONFIG_LRU_GEN */
-
-static void lru_gen_inc_refs(struct folio *folio)
-{
-}
-
-static bool lru_gen_clear_refs(struct folio *folio)
-{
-	return false;
-}
-
-#endif /* CONFIG_LRU_GEN */
-
-/**
- * folio_mark_accessed - Mark a folio as having seen activity.
- * @folio: The folio to mark.
- *
- * This function will perform one of the following transitions:
- *
- * * inactive,unreferenced	->	inactive,referenced
- * * inactive,referenced	->	active,unreferenced
- * * active,unreferenced	->	active,referenced
- *
- * When a newly allocated folio is not yet visible, so safe for non-atomic ops,
- * __folio_set_referenced() may be substituted for folio_mark_accessed().
- */
-void folio_mark_accessed(struct folio *folio)
-{
-	if (folio_test_dropbehind(folio))
-		return;
-	if (lru_gen_enabled()) {
-		lru_gen_inc_refs(folio);
-		return;
-	}
-
-	if (!folio_test_referenced(folio)) {
-		folio_set_referenced(folio);
-	} else if (folio_test_unevictable(folio)) {
-		/*
-		 * Unevictable pages are on the "LRU_UNEVICTABLE" list. But,
-		 * this list is never rotated or maintained, so marking an
-		 * unevictable page accessed has no effect.
-		 */
-	} else if (!folio_test_active(folio)) {
-		/*
-		 * If the folio is on the LRU, queue it for activation via
-		 * cpu_fbatches.lru_activate. Otherwise, assume the folio is in a
-		 * folio_batch, mark it active and it'll be moved to the active
-		 * LRU on the next drain.
-		 */
-		if (folio_test_lru(folio))
-			folio_activate(folio);
-		else
-			__lru_cache_activate_folio(folio);
-		folio_clear_referenced(folio);
-		workingset_activation(folio);
-	}
-	if (folio_test_idle(folio))
-		folio_clear_idle(folio);
-}
-EXPORT_SYMBOL(folio_mark_accessed);
-
-/**
- * folio_add_lru - Add a folio to an LRU list.
- * @folio: The folio to be added to the LRU.
- *
- * Queue the folio for addition to the LRU. The decision on whether
- * to add the page to the [in]active [file|anon] list is deferred until the
- * folio_batch is drained. This gives a chance for the caller of folio_add_lru()
- * have the folio added to the active list using folio_mark_accessed().
- */
-void folio_add_lru(struct folio *folio)
-{
-	VM_BUG_ON_FOLIO(folio_test_active(folio) &&
-			folio_test_unevictable(folio), folio);
-	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
-
-	/*
-	 * For refaulted workingset folios, set PG_active so they
-	 * can be added to active generations.
-	 * For prefaulted file folios, folio_mark_accessed() sets
-	 * PG_referenced so lru_gen_folio_seq() places them into
-	 * the second oldest generation.
-	 */
-	if (lru_gen_enabled() && !folio_test_unevictable(folio) &&
-	    lru_gen_in_fault() && !(current->flags & PF_MEMALLOC)) {
-		if (folio_test_workingset(folio))
-			folio_set_active(folio);
-		else if (!folio_test_referenced(folio))
-			folio_mark_accessed(folio);
-	}
-
-	folio_batch_add_and_move(folio, lru_add);
-}
-EXPORT_SYMBOL(folio_add_lru);
-
-/**
- * folio_add_lru_vma() - Add a folio to the appropriate LRU list for this VMA.
- * @folio: The folio to be added to the LRU.
- * @vma: VMA in which the folio is mapped.
- *
- * If the VMA is mlocked, @folio is added to the unevictable list.
- * Otherwise, it is treated the same way as folio_add_lru().
- */
-void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma)
-{
-	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
-
-	if (unlikely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED))
-		mlock_new_folio(folio);
-	else
-		folio_add_lru(folio);
-}
-
-/*
- * If the folio cannot be invalidated, it is moved to the
- * inactive list to speed up its reclaim.  It is moved to the
- * head of the list, rather than the tail, to give the flusher
- * threads some time to write it out, as this is much more
- * effective than the single-page writeout from reclaim.
- *
- * If the folio isn't mapped and dirty/writeback, the folio
- * could be reclaimed asap using the reclaim flag.
- *
- * 1. active, mapped folio -> none
- * 2. active, dirty/writeback folio -> inactive, head, reclaim
- * 3. inactive, mapped folio -> none
- * 4. inactive, dirty/writeback folio -> inactive, head, reclaim
- * 5. inactive, clean -> inactive, tail
- * 6. Others -> none
- *
- * In 4, it moves to the head of the inactive list so the folio is
- * written out by flusher threads as this is much more efficient
- * than the single-page writeout from reclaim.
- */
-static void lru_deactivate_file(struct lruvec *lruvec, struct folio *folio)
-{
-	bool active = folio_test_active(folio) || lru_gen_enabled();
-	long nr_pages = folio_nr_pages(folio);
-
-	if (folio_test_unevictable(folio))
-		return;
-
-	/* Some processes are using the folio */
-	if (folio_mapped(folio))
-		return;
-
-	lruvec_del_folio(lruvec, folio);
-	folio_clear_active(folio);
-	folio_clear_referenced(folio);
-
-	if (folio_test_writeback(folio) || folio_test_dirty(folio)) {
-		/*
-		 * Setting the reclaim flag could race with
-		 * folio_end_writeback() and confuse readahead.  But the
-		 * race window is _really_ small and  it's not a critical
-		 * problem.
-		 */
-		lruvec_add_folio(lruvec, folio);
-		folio_set_reclaim(folio);
-	} else {
-		/*
-		 * The folio's writeback ended while it was in the batch.
-		 * We move that folio to the tail of the inactive list.
-		 */
-		lruvec_add_folio_tail(lruvec, folio);
-		__count_vm_events(PGROTATED, nr_pages);
-	}
-
-	if (active) {
-		__count_vm_events(PGDEACTIVATE, nr_pages);
-		count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE,
-				     nr_pages);
-	}
-}
-
-static void lru_deactivate(struct lruvec *lruvec, struct folio *folio)
-{
-	long nr_pages = folio_nr_pages(folio);
-
-	if (folio_test_unevictable(folio) || !(folio_test_active(folio) || lru_gen_enabled()))
-		return;
-
-	lruvec_del_folio(lruvec, folio);
-	folio_clear_active(folio);
-	folio_clear_referenced(folio);
-	lruvec_add_folio(lruvec, folio);
-
-	__count_vm_events(PGDEACTIVATE, nr_pages);
-	count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_pages);
-}
-
-static void lru_lazyfree(struct lruvec *lruvec, struct folio *folio)
-{
-	long nr_pages = folio_nr_pages(folio);
-
-	if (!folio_test_anon(folio) || !folio_test_swapbacked(folio) ||
-	    folio_test_swapcache(folio) || folio_test_unevictable(folio))
-		return;
-
-	lruvec_del_folio(lruvec, folio);
-	folio_clear_active(folio);
-	if (lru_gen_enabled())
-		lru_gen_clear_refs(folio);
-	else
-		folio_clear_referenced(folio);
-	/*
-	 * Lazyfree folios are clean anonymous folios.  They have
-	 * the swapbacked flag cleared, to distinguish them from normal
-	 * anonymous folios
-	 */
-	folio_clear_swapbacked(folio);
-	lruvec_add_folio(lruvec, folio);
-
-	__count_vm_events(PGLAZYFREE, nr_pages);
-	count_memcg_events(lruvec_memcg(lruvec), PGLAZYFREE, nr_pages);
-}
-
-/*
- * Drain pages out of the cpu's folio_batch.
- * Either "cpu" is the current CPU, and preemption has already been
- * disabled; or "cpu" is being hot-unplugged, and is already dead.
- */
-void lru_add_drain_cpu(int cpu)
-{
-	struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu);
-	struct folio_batch *fbatch = &fbatches->lru_add;
-
-	if (folio_batch_count(fbatch))
-		folio_batch_move_lru(fbatch, lru_add);
-
-	fbatch = &fbatches->lru_move_tail;
-	/* Disabling interrupts below acts as a compiler barrier. */
-	if (data_race(folio_batch_count(fbatch))) {
-		unsigned long flags;
-
-		/* No harm done if a racing interrupt already did this */
-		local_lock_irqsave(&cpu_fbatches.lock_irq, flags);
-		folio_batch_move_lru(fbatch, lru_move_tail);
-		local_unlock_irqrestore(&cpu_fbatches.lock_irq, flags);
-	}
-
-	fbatch = &fbatches->lru_deactivate_file;
-	if (folio_batch_count(fbatch))
-		folio_batch_move_lru(fbatch, lru_deactivate_file);
-
-	fbatch = &fbatches->lru_deactivate;
-	if (folio_batch_count(fbatch))
-		folio_batch_move_lru(fbatch, lru_deactivate);
-
-	fbatch = &fbatches->lru_lazyfree;
-	if (folio_batch_count(fbatch))
-		folio_batch_move_lru(fbatch, lru_lazyfree);
-
-	folio_activate_drain(cpu);
-}
-
-/**
- * deactivate_file_folio() - Deactivate a file folio.
- * @folio: Folio to deactivate.
- *
- * This function hints to the VM that @folio is a good reclaim candidate,
- * for example if its invalidation fails due to the folio being dirty
- * or under writeback.
- *
- * Context: Caller holds a reference on the folio.
- */
-void deactivate_file_folio(struct folio *folio)
-{
-	/* Deactivating an unevictable folio will not accelerate reclaim */
-	if (folio_test_unevictable(folio) || !folio_test_lru(folio))
-		return;
-
-	if (lru_gen_enabled() && lru_gen_clear_refs(folio))
-		return;
-
-	folio_batch_add_and_move(folio, lru_deactivate_file);
-}
-
-/*
- * folio_deactivate - deactivate a folio
- * @folio: folio to deactivate
- *
- * folio_deactivate() moves @folio to the inactive list if @folio was on the
- * active list and was not unevictable. This is done to accelerate the
- * reclaim of @folio.
- */
-void folio_deactivate(struct folio *folio)
-{
-	if (folio_test_unevictable(folio) || !folio_test_lru(folio))
-		return;
-
-	if (lru_gen_enabled() ? lru_gen_clear_refs(folio) : !folio_test_active(folio))
-		return;
-
-	folio_batch_add_and_move(folio, lru_deactivate);
-}
-
-/**
- * folio_mark_lazyfree - make an anon folio lazyfree
- * @folio: folio to deactivate
- *
- * folio_mark_lazyfree() moves @folio to the inactive file list.
- * This is done to accelerate the reclaim of @folio.
- */
-void folio_mark_lazyfree(struct folio *folio)
-{
-	if (!folio_test_anon(folio) || !folio_test_swapbacked(folio) ||
-	    !folio_test_lru(folio) ||
-	    folio_test_swapcache(folio) || folio_test_unevictable(folio))
-		return;
-
-	folio_batch_add_and_move(folio, lru_lazyfree);
-}
-
-void lru_add_drain(void)
-{
-	local_lock(&cpu_fbatches.lock);
-	lru_add_drain_cpu(smp_processor_id());
-	local_unlock(&cpu_fbatches.lock);
-	mlock_drain_local();
-}
-
-/*
- * It's called from per-cpu workqueue context in SMP case so
- * lru_add_drain_cpu and invalidate_bh_lrus_cpu should run on
- * the same cpu. It shouldn't be a problem in !SMP case since
- * the core is only one and the locks will disable preemption.
- */
-static void lru_add_and_bh_lrus_drain(void)
-{
-	local_lock(&cpu_fbatches.lock);
-	lru_add_drain_cpu(smp_processor_id());
-	local_unlock(&cpu_fbatches.lock);
-	invalidate_bh_lrus_cpu();
-	mlock_drain_local();
-}
-
-void lru_add_drain_cpu_zone(struct zone *zone)
-{
-	local_lock(&cpu_fbatches.lock);
-	lru_add_drain_cpu(smp_processor_id());
-	drain_local_pages(zone);
-	local_unlock(&cpu_fbatches.lock);
-	mlock_drain_local();
-}
-
-#ifdef CONFIG_SMP
-
-static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
-
-static void lru_add_drain_per_cpu(struct work_struct *dummy)
-{
-	lru_add_and_bh_lrus_drain();
-}
-
-static bool cpu_needs_drain(unsigned int cpu)
-{
-	struct cpu_fbatches *fbatches = &per_cpu(cpu_fbatches, cpu);
-
-	/* Check these in order of likelihood that they're not zero */
-	return folio_batch_count(&fbatches->lru_add) ||
-		folio_batch_count(&fbatches->lru_move_tail) ||
-		folio_batch_count(&fbatches->lru_deactivate_file) ||
-		folio_batch_count(&fbatches->lru_deactivate) ||
-		folio_batch_count(&fbatches->lru_lazyfree) ||
-		folio_batch_count(&fbatches->lru_activate) ||
-		need_mlock_drain(cpu) ||
-		has_bh_in_lru(cpu, NULL);
-}
-
-/*
- * Doesn't need any cpu hotplug locking because we do rely on per-cpu
- * kworkers being shut down before our page_alloc_cpu_dead callback is
- * executed on the offlined cpu.
- * Calling this function with cpu hotplug locks held can actually lead
- * to obscure indirect dependencies via WQ context.
- */
-static inline void __lru_add_drain_all(bool force_all_cpus)
-{
-	/*
-	 * lru_drain_gen - Global pages generation number
-	 *
-	 * (A) Definition: global lru_drain_gen = x implies that all generations
-	 *     0 < n <= x are already *scheduled* for draining.
-	 *
-	 * This is an optimization for the highly-contended use case where a
-	 * user space workload keeps constantly generating a flow of pages for
-	 * each CPU.
-	 */
-	static unsigned int lru_drain_gen;
-	static struct cpumask has_work;
-	static DEFINE_MUTEX(lock);
-	unsigned cpu, this_gen;
-
-	/*
-	 * Make sure nobody triggers this path before mm_percpu_wq is fully
-	 * initialized.
-	 */
-	if (WARN_ON(!mm_percpu_wq))
-		return;
-
-	/*
-	 * Guarantee folio_batch counter stores visible by this CPU
-	 * are visible to other CPUs before loading the current drain
-	 * generation.
-	 */
-	smp_mb();
-
-	/*
-	 * (B) Locally cache global LRU draining generation number
-	 *
-	 * The read barrier ensures that the counter is loaded before the mutex
-	 * is taken. It pairs with smp_mb() inside the mutex critical section
-	 * at (D).
-	 */
-	this_gen = smp_load_acquire(&lru_drain_gen);
-
-	/* It helps everyone if we do our own local drain immediately. */
-	lru_add_drain();
-
-	mutex_lock(&lock);
-
-	/*
-	 * (C) Exit the draining operation if a newer generation, from another
-	 * lru_add_drain_all(), was already scheduled for draining. Check (A).
-	 */
-	if (unlikely(this_gen != lru_drain_gen && !force_all_cpus))
-		goto done;
-
-	/*
-	 * (D) Increment global generation number
-	 *
-	 * Pairs with smp_load_acquire() at (B), outside of the critical
-	 * section. Use a full memory barrier to guarantee that the
-	 * new global drain generation number is stored before loading
-	 * folio_batch counters.
-	 *
-	 * This pairing must be done here, before the for_each_online_cpu loop
-	 * below which drains the page vectors.
-	 *
-	 * Let x, y, and z represent some system CPU numbers, where x < y < z.
-	 * Assume CPU #z is in the middle of the for_each_online_cpu loop
-	 * below and has already reached CPU #y's per-cpu data. CPU #x comes
-	 * along, adds some pages to its per-cpu vectors, then calls
-	 * lru_add_drain_all().
-	 *
-	 * If the paired barrier is done at any later step, e.g. after the
-	 * loop, CPU #x will just exit at (C) and miss flushing out all of its
-	 * added pages.
-	 */
-	WRITE_ONCE(lru_drain_gen, lru_drain_gen + 1);
-	smp_mb();
-
-	cpumask_clear(&has_work);
-	for_each_online_cpu(cpu) {
-		struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
-
-		if (cpu_needs_drain(cpu)) {
-			INIT_WORK(work, lru_add_drain_per_cpu);
-			queue_work_on(cpu, mm_percpu_wq, work);
-			__cpumask_set_cpu(cpu, &has_work);
-		}
-	}
-
-	for_each_cpu(cpu, &has_work)
-		flush_work(&per_cpu(lru_add_drain_work, cpu));
-
-done:
-	mutex_unlock(&lock);
-}
-
-void lru_add_drain_all(void)
-{
-	__lru_add_drain_all(false);
-}
-#else
-void lru_add_drain_all(void)
-{
-	lru_add_drain();
-}
-#endif /* CONFIG_SMP */
-
-atomic_t lru_disable_count = ATOMIC_INIT(0);
-
-/*
- * lru_cache_disable() needs to be called before we start compiling
- * a list of folios to be migrated using folio_isolate_lru().
- * It drains folios on LRU cache and then disable on all cpus until
- * lru_cache_enable is called.
- *
- * Must be paired with a call to lru_cache_enable().
- */
-void lru_cache_disable(void)
-{
-	atomic_inc(&lru_disable_count);
-	/*
-	 * Readers of lru_disable_count are protected by either disabling
-	 * preemption or rcu_read_lock:
-	 *
-	 * preempt_disable, local_irq_disable  [bh_lru_lock()]
-	 * rcu_read_lock		       [rt_spin_lock CONFIG_PREEMPT_RT]
-	 * preempt_disable		       [local_lock !CONFIG_PREEMPT_RT]
-	 *
-	 * Since v5.1 kernel, synchronize_rcu() is guaranteed to wait on
-	 * preempt_disable() regions of code. So any CPU which sees
-	 * lru_disable_count = 0 will have exited the critical
-	 * section when synchronize_rcu() returns.
-	 */
-	synchronize_rcu_expedited();
-#ifdef CONFIG_SMP
-	__lru_add_drain_all(true);
-#else
-	lru_add_and_bh_lrus_drain();
-#endif
-}
-
-/**
- * folios_put_refs - Reduce the reference count on a batch of folios.
- * @folios: The folios.
- * @refs: The number of refs to subtract from each folio.
- *
- * Like folio_put(), but for a batch of folios.  This is more efficient
- * than writing the loop yourself as it will optimise the locks which need
- * to be taken if the folios are freed.  The folios batch is returned
- * empty and ready to be reused for another batch; there is no need
- * to reinitialise it.  If @refs is NULL, we subtract one from each
- * folio refcount.
- *
- * Context: May be called in process or interrupt context, but not in NMI
- * context.  May be called while holding a spinlock.
- */
-void folios_put_refs(struct folio_batch *folios, unsigned int *refs)
-{
-	int i, j;
-	struct lruvec *lruvec = NULL;
-	unsigned long flags = 0;
-
-	for (i = 0, j = 0; i < folios->nr; i++) {
-		struct folio *folio = folios->folios[i];
-		unsigned int nr_refs = refs ? refs[i] : 1;
-
-		/* Folio batch entry may have been preemptively removed during drain. */
-		if (!folio)
-			continue;
-
-		if (is_huge_zero_folio(folio))
-			continue;
-
-		if (folio_is_zone_device(folio)) {
-			if (lruvec) {
-				lruvec_unlock_irqrestore(lruvec, flags);
-				lruvec = NULL;
-			}
-			if (folio_ref_sub_and_test(folio, nr_refs))
-				free_zone_device_folio(folio);
-			continue;
-		}
-
-		if (!folio_ref_sub_and_test(folio, nr_refs))
-			continue;
-
-		/* hugetlb has its own memcg */
-		if (folio_test_hugetlb(folio)) {
-			if (lruvec) {
-				lruvec_unlock_irqrestore(lruvec, flags);
-				lruvec = NULL;
-			}
-			free_huge_folio(folio);
-			continue;
-		}
-		folio_unqueue_deferred_split(folio);
-		__page_cache_release(folio, &lruvec, &flags);
-
-		if (j != i)
-			folios->folios[j] = folio;
-		j++;
-	}
-	if (lruvec)
-		lruvec_unlock_irqrestore(lruvec, flags);
-	if (!j) {
-		folio_batch_reinit(folios);
-		return;
-	}
-
-	folios->nr = j;
-	mem_cgroup_uncharge_folios(folios);
-	free_unref_folios(folios);
-}
-EXPORT_SYMBOL(folios_put_refs);
-
-/**
- * release_pages - batched put_page()
- * @arg: array of pages to release
- * @nr: number of pages
- *
- * Decrement the reference count on all the pages in @arg.  If it
- * fell to zero, remove the page from the LRU and free it.
- *
- * Note that the argument can be an array of pages, encoded pages,
- * or folio pointers. We ignore any encoded bits, and turn any of
- * them into just a folio that gets free'd.
- */
-void release_pages(release_pages_arg arg, int nr)
-{
-	struct folio_batch fbatch;
-	int refs[FOLIO_BATCH_SIZE];
-	struct encoded_page **encoded = arg.encoded_pages;
-	int i;
-
-	folio_batch_init(&fbatch);
-	for (i = 0; i < nr; i++) {
-		/* Turn any of the argument types into a folio */
-		struct folio *folio = page_folio(encoded_page_ptr(encoded[i]));
-
-		/* Is our next entry actually "nr_pages" -> "nr_refs" ? */
-		refs[fbatch.nr] = 1;
-		if (unlikely(encoded_page_flags(encoded[i]) &
-			     ENCODED_PAGE_BIT_NR_PAGES_NEXT))
-			refs[fbatch.nr] = encoded_nr_pages(encoded[++i]);
-
-		if (folio_batch_add(&fbatch, folio) > 0)
-			continue;
-		folios_put_refs(&fbatch, refs);
-	}
-
-	if (fbatch.nr)
-		folios_put_refs(&fbatch, refs);
-}
-EXPORT_SYMBOL(release_pages);
-
-/*
- * The folios which we're about to release may be in the deferred lru-addition
- * queues.  That would prevent them from really being freed right now.  That's
- * OK from a correctness point of view but is inefficient - those folios may be
- * cache-warm and we want to give them back to the page allocator ASAP.
- *
- * So __folio_batch_release() will drain those queues here.
- * folio_batch_move_lru() calls folios_put() directly to avoid
- * mutual recursion.
- */
-void __folio_batch_release(struct folio_batch *fbatch)
-{
-	if (!fbatch->percpu_pvec_drained) {
-		lru_add_drain();
-		fbatch->percpu_pvec_drained = true;
-	}
-	folios_put(fbatch);
-}
-EXPORT_SYMBOL(__folio_batch_release);
-
-/**
- * folio_batch_remove_exceptionals() - Prune non-folios from a batch.
- * @fbatch: The batch to prune
- *
- * find_get_entries() fills a batch with both folios and shadow/swap/DAX
- * entries.  This function prunes all the non-folio entries from @fbatch
- * without leaving holes, so that it can be passed on to folio-only batch
- * operations.
- */
-void folio_batch_remove_exceptionals(struct folio_batch *fbatch)
-{
-	unsigned int i, j;
-
-	for (i = 0, j = 0; i < folio_batch_count(fbatch); i++) {
-		struct folio *folio = fbatch->folios[i];
-		if (!xa_is_value(folio))
-			fbatch->folios[j++] = folio;
-	}
-	fbatch->nr = j;
-}
-
-#ifdef CONFIG_MEMCG
-static void lruvec_reparent_lru(struct lruvec *child_lruvec,
-				struct lruvec *parent_lruvec,
-				enum lru_list lru, int nid)
-{
-	int zid;
-	struct zone *zone;
-
-	if (lru != LRU_UNEVICTABLE)
-		list_splice_tail_init(&child_lruvec->lists[lru], &parent_lruvec->lists[lru]);
-
-	for_each_managed_zone_pgdat(zone, NODE_DATA(nid), zid, MAX_NR_ZONES - 1) {
-		unsigned long size = mem_cgroup_get_zone_lru_size(child_lruvec, lru, zid);
-
-		mem_cgroup_update_lru_size(parent_lruvec, lru, zid, size);
-	}
-}
-
-void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid)
-{
-	enum lru_list lru;
-	struct lruvec *child_lruvec, *parent_lruvec;
-
-	child_lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
-	parent_lruvec = mem_cgroup_lruvec(parent, NODE_DATA(nid));
-	parent_lruvec->anon_cost += child_lruvec->anon_cost;
-	parent_lruvec->file_cost += child_lruvec->file_cost;
-
-	for_each_lru(lru)
-		lruvec_reparent_lru(child_lruvec, parent_lruvec, lru, nid);
-}
-#endif
-
-static const struct ctl_table swap_sysctl_table[] = {
-	{
-		.procname	= "page-cluster",
-		.data		= &page_cluster,
-		.maxlen		= sizeof(int),
-		.mode		= 0644,
-		.proc_handler	= proc_dointvec_minmax,
-		.extra1		= SYSCTL_ZERO,
-		.extra2		= (void *)&page_cluster_max,
-	}
-};
-
-/*
- * Perform any setup for the swap system
- */
-void __init swap_setup(void)
-{
-	unsigned long megs = PAGES_TO_MB(totalram_pages());
-
-	/* Use a smaller cluster for small-memory machines */
-	if (megs < 16)
-		page_cluster = 2;
-	else
-		page_cluster = 3;
-	/*
-	 * Right now other parts of the system means that we
-	 * _really_ don't want to cluster much more
-	 */
-
-	register_sysctl_init("vm", swap_sysctl_table);
-}
diff --git a/mm/swap.h b/mm/swap.h
index 77d2d14eda421..8fb32bf157392 100644
--- a/mm/swap.h
+++ b/mm/swap.h
@@ -4,12 +4,11 @@
 
 #include <linux/atomic.h> /* for atomic_long_t */
 #include <linux/mm.h> /* for PAGE_SHIFT */
+
 struct mempolicy;
 struct swap_iocb;
 struct swap_memcg_table;
 
-extern int page_cluster;
-
 #if defined(MAX_POSSIBLE_PHYSMEM_BITS)
 #define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT)
 #elif defined(MAX_PHYSMEM_BITS)
@@ -78,6 +77,28 @@ enum swap_cluster_flags {
 	CLUSTER_FLAG_MAX,
 };
 
+struct swap_io_ctx {
+	struct swap_iocb	*sio;
+	struct swap_info_struct	*sis;
+};
+
+/*
+ * SWAP_OPS_F_REQUIRE_NOFS:
+ *	When set, all reclaim operations must operated as GFS_NOFS and not
+ *	just GFP_NOIO, as GFP_NOIO allocations could recourse into the
+ *	file system backing this swap file.
+ */
+#define SWAP_OPS_F_REQUIRE_NOFS		(1U << 0)
+
+struct swap_ops {
+	unsigned int		flags;
+
+	bool (*can_merge)(struct folio *folio, struct folio *prev_folio,
+			size_t prev_folio_size, int rw);
+	void (*submit_write)(struct swap_io_ctx *ctx);
+	void (*submit_read)(struct swap_io_ctx *ctx);
+};
+
 #ifdef CONFIG_SWAP
 #include <linux/swapops.h> /* for swp_offset */
 #include <linux/blk_types.h> /* for bio_end_io_t */
@@ -230,8 +251,8 @@ extern int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp);
  * folio_put_swap(): does the opposite thing of folio_dup_swap().
  */
 int folio_alloc_swap(struct folio *folio);
-int folio_dup_swap(struct folio *folio, struct page *subpage);
-void folio_put_swap(struct folio *folio, struct page *subpage);
+int folio_dup_swap(struct folio *folio, struct page *page);
+void folio_put_swap(struct folio *folio, struct page *page);
 
 /* For internal use */
 extern void __swap_cluster_free_entries(struct swap_info_struct *si,
@@ -240,17 +261,11 @@ extern void __swap_cluster_free_entries(struct swap_info_struct *si,
 
 /* linux/mm/page_io.c */
 int sio_pool_init(void);
-struct swap_iocb;
-void swap_read_folio(struct folio *folio, struct swap_iocb **plug);
-void __swap_read_unplug(struct swap_iocb *plug);
-static inline void swap_read_unplug(struct swap_iocb *plug)
-{
-	if (unlikely(plug))
-		__swap_read_unplug(plug);
-}
-void swap_write_unplug(struct swap_iocb *sio);
-int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug);
-void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug);
+void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio);
+void swap_read_submit(struct swap_io_ctx *ctx);
+void swap_write_submit(struct swap_io_ctx *ctx);
+int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio);
+void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio);
 
 /* linux/mm/swap_state.c */
 extern struct address_space swap_space __read_mostly;
@@ -317,9 +332,8 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci,
 
 void show_swap_cache_info(void);
 void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr);
-struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
-		struct vm_area_struct *vma, unsigned long addr,
-		struct swap_iocb **plug);
+struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry,
+		gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr);
 struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag,
 		struct mempolicy *mpol, pgoff_t ilx);
 struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag,
@@ -329,13 +343,7 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t flag, unsigned long orders,
 void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma,
 			   unsigned long addr);
 
-static inline unsigned int folio_swap_flags(struct folio *folio)
-{
-	return __swap_entry_to_info(folio->swap)->flags;
-}
-
 #else /* CONFIG_SWAP */
-struct swap_iocb;
 static inline struct swap_cluster_info *swap_cluster_lock(
 	struct swap_info_struct *si, pgoff_t offset, bool irq)
 {
@@ -381,11 +389,11 @@ static inline void folio_put_swap(struct folio *folio, struct page *page)
 {
 }
 
-static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug)
+static inline void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio)
 {
 }
 
-static inline void swap_write_unplug(struct swap_iocb *sio)
+static inline void swap_write_submit(struct swap_io_ctx *ctx)
 {
 }
 
@@ -427,8 +435,7 @@ static inline void swap_update_readahead(struct folio *folio,
 {
 }
 
-static inline int swap_writeout(struct folio *folio,
-		struct swap_iocb **swap_plug)
+static inline int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio)
 {
 	return 0;
 }
@@ -466,11 +473,11 @@ static inline void __swap_cache_replace_folio(struct swap_cluster_info *ci,
 		struct folio *old, struct folio *new)
 {
 }
+#endif /* CONFIG_SWAP */
 
-static inline unsigned int folio_swap_flags(struct folio *folio)
-{
-	return 0;
-}
+extern const struct swap_ops swap_bdev_ops;
+
+int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio,
+		struct list_head *folio_list);
 
-#endif /* CONFIG_SWAP */
 #endif /* _MM_SWAP_H */
diff --git a/mm/swap_state.c b/mm/swap_state.c
index 9c3a5cf997786..5be825911e645 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -22,10 +22,15 @@
 #include <linux/vmalloc.h>
 #include <linux/huge_mm.h>
 #include <linux/shmem_fs.h>
+#include <linux/sysctl.h>
 #include "internal.h"
 #include "swap_table.h"
 #include "swap.h"
 
+/* Swap readahead cluster size, as a power of 2 pages. */
+static int page_cluster;
+static const int page_cluster_max = 31;
+
 /*
  * swapper_space is a fiction, retained to simplify the path through
  * vmscan's shrink_folio_list.
@@ -633,9 +638,9 @@ void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma,
 	}
 }
 
-static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp,
-					   struct mempolicy *mpol, pgoff_t ilx,
-					   struct swap_iocb **plug, bool readahead)
+static struct folio *swap_cache_read_folio(struct swap_io_ctx *ctx,
+		swp_entry_t entry, gfp_t gfp, struct mempolicy *mpol,
+		pgoff_t ilx, bool readahead)
 {
 	struct folio *folio;
 
@@ -649,7 +654,7 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp,
 	if (IS_ERR_OR_NULL(folio))
 		return NULL;
 
-	swap_read_folio(folio, plug);
+	swap_read_folio(ctx, folio);
 	if (readahead) {
 		folio_set_readahead(folio);
 		count_vm_event(SWAP_RA);
@@ -677,6 +682,7 @@ static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp,
 struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders,
 			   struct vm_fault *vmf, struct mempolicy *mpol, pgoff_t ilx)
 {
+	struct swap_io_ctx ctx = {};
 	struct folio *folio;
 
 	do {
@@ -689,7 +695,8 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders,
 	if (IS_ERR(folio))
 		return folio;
 
-	swap_read_folio(folio, NULL);
+	swap_read_folio(&ctx, folio);
+	swap_read_submit(&ctx);
 	return folio;
 }
 
@@ -699,9 +706,8 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders,
  * A failure return means that either the page allocation failed or that
  * the swap entry is no longer in use.
  */
-struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
-		struct vm_area_struct *vma, unsigned long addr,
-		struct swap_iocb **plug)
+struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry,
+		gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr)
 {
 	struct swap_info_struct *si;
 	struct mempolicy *mpol;
@@ -713,13 +719,24 @@ struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
 		return NULL;
 
 	mpol = get_vma_policy(vma, addr, 0, &ilx);
-	folio = swap_cache_read_folio(entry, gfp_mask, mpol, ilx, plug, false);
+	folio = swap_cache_read_folio(ctx, entry, gfp_mask, mpol, ilx, false);
 	mpol_cond_put(mpol);
 
 	put_swap_device(si);
 	return folio;
 }
 
+static struct folio *swap_cache_read_folio_sync(swp_entry_t entry, gfp_t gfp,
+		struct mempolicy *mpol, pgoff_t ilx)
+{
+	struct swap_io_ctx ctx = {};
+	struct folio *folio;
+
+	folio = swap_cache_read_folio(&ctx, entry, gfp, mpol, ilx, false);
+	swap_read_submit(&ctx);
+	return folio;
+}
+
 static unsigned int __swapin_nr_pages(unsigned long prev_offset,
 				      unsigned long offset,
 				      int hits,
@@ -808,8 +825,8 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,
 	unsigned long start_offset, end_offset;
 	unsigned long mask;
 	struct swap_info_struct *si = __swap_entry_to_info(entry);
+	struct swap_io_ctx ctx = {};
 	struct blk_plug plug;
-	struct swap_iocb *splug = NULL;
 	swp_entry_t ra_entry;
 
 	mask = swapin_nr_pages(offset) - 1;
@@ -828,18 +845,16 @@ struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,
 	for (offset = start_offset; offset <= end_offset ; offset++) {
 		/* Ok, do the async read-ahead now */
 		ra_entry = swp_entry(swp_type(entry), offset);
-		folio = swap_cache_read_folio(ra_entry, gfp_mask, mpol, ilx,
-					      &splug, offset != entry_offset);
+		folio = swap_cache_read_folio(&ctx, ra_entry, gfp_mask, mpol,
+				ilx, offset != entry_offset);
 		if (!folio)
 			continue;
 		folio_put(folio);
 	}
 	blk_finish_plug(&plug);
-	swap_read_unplug(splug);
-	lru_add_drain();	/* Push any new pages onto the LRU now */
+	swap_read_submit(&ctx);
 skip:
-	/* The page was likely read above, so no need for plugging here */
-	return swap_cache_read_folio(entry, gfp_mask, mpol, ilx, NULL, false);
+	return swap_cache_read_folio_sync(entry, gfp_mask, mpol, ilx);
 }
 
 static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start,
@@ -899,8 +914,8 @@ static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start,
 static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask,
 		struct mempolicy *mpol, pgoff_t targ_ilx, struct vm_fault *vmf)
 {
+	struct swap_io_ctx ctx = {};
 	struct blk_plug plug;
-	struct swap_iocb *splug = NULL;
 	struct folio *folio;
 	pte_t *pte = NULL, pentry;
 	int win;
@@ -939,8 +954,8 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask,
 			if (!si)
 				continue;
 		}
-		folio = swap_cache_read_folio(entry, gfp_mask, mpol, ilx,
-					      &splug, addr != vmf->address);
+		folio = swap_cache_read_folio(&ctx, entry, gfp_mask, mpol, ilx,
+					      addr != vmf->address);
 		if (si)
 			put_swap_device(si);
 		if (!folio)
@@ -950,13 +965,10 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask,
 	if (pte)
 		pte_unmap(pte);
 	blk_finish_plug(&plug);
-	swap_read_unplug(splug);
-	lru_add_drain();
+	swap_read_submit(&ctx);
 skip:
 	/* The folio was likely read above, so no need for plugging here */
-	folio = swap_cache_read_folio(targ_entry, gfp_mask, mpol, targ_ilx,
-				      NULL, false);
-	return folio;
+	return swap_cache_read_folio_sync(targ_entry, gfp_mask, mpol, targ_ilx);
 }
 
 /**
@@ -987,6 +999,35 @@ struct folio *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask,
 	return folio;
 }
 
+static const struct ctl_table swap_readahead_sysctl_table[] = {
+	{
+		.procname	= "page-cluster",
+		.data		= &page_cluster,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= (void *)&page_cluster_max,
+	}
+};
+
+static void __init swap_readahead_setup(void)
+{
+	unsigned long megs = PAGES_TO_MB(totalram_pages());
+
+	/* Use a smaller cluster for small-memory machines */
+	if (megs < 16)
+		page_cluster = 2;
+	else
+		page_cluster = 3;
+	/*
+	 * Right now other parts of the system means that we
+	 * _really_ don't want to cluster much more
+	 */
+
+	register_sysctl_init("vm", swap_readahead_sysctl_table);
+}
+
 #ifdef CONFIG_SYSFS
 static ssize_t vma_ra_enabled_show(struct kobject *kobj,
 				     struct kobj_attribute *attr, char *buf)
@@ -1016,7 +1057,7 @@ static const struct attribute_group swap_attr_group = {
 	.attrs = swap_attrs,
 };
 
-static int __init swap_init(void)
+static int __init swap_sysfs_init(void)
 {
 	int err;
 	struct kobject *swap_kobj;
@@ -1039,5 +1080,17 @@ static int __init swap_init(void)
 	kobject_put(swap_kobj);
 	return err;
 }
-subsys_initcall(swap_init);
+#else
+static int __init swap_sysfs_init(void)
+{
+	return 0;
+}
 #endif
+
+static int __init swap_init(void)
+{
+	swap_readahead_setup();
+
+	return swap_sysfs_init();
+}
+subsys_initcall(swap_init);
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 78b49b0658add..70b90fa9c2a04 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -1072,6 +1072,12 @@ static void swap_reclaim_full_clusters(struct swap_info_struct *si, bool force)
 		swap_cluster_unlock(ci);
 		if (to_scan <= 0)
 			break;
+
+		/*
+		 * When 'force' is false, 'to_scan' is initialized to 1.
+		 * The loop breaks above, making this cond_resched() unreachable
+		 * in atomic contexts.
+		 */
 		cond_resched();
 	}
 }
@@ -1781,7 +1787,7 @@ int folio_alloc_swap(struct folio *folio)
 /**
  * folio_dup_swap() - Increase swap count of swap entries of a folio.
  * @folio: folio with swap entries bounded.
- * @subpage: if not NULL, only increase the swap count of this subpage.
+ * @page: if not NULL, only increase the swap count of this page.
  *
  * Typically called when the folio is unmapped and have its swap entry to
  * take its place: Swap entries allocated to a folio has count == 0 and pinned
@@ -1795,7 +1801,7 @@ int folio_alloc_swap(struct folio *folio)
  * swap_put_entries_direct on its swap entry before this helper returns, or
  * the swap count may underflow.
  */
-int folio_dup_swap(struct folio *folio, struct page *subpage)
+int folio_dup_swap(struct folio *folio, struct page *page)
 {
 	swp_entry_t entry = folio->swap;
 	unsigned long nr_pages = folio_nr_pages(folio);
@@ -1803,8 +1809,8 @@ int folio_dup_swap(struct folio *folio, struct page *subpage)
 	VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
 	VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio);
 
-	if (subpage) {
-		entry.val += folio_page_idx(folio, subpage);
+	if (page) {
+		entry.val += folio_page_idx(folio, page);
 		nr_pages = 1;
 	}
 
@@ -1815,13 +1821,13 @@ int folio_dup_swap(struct folio *folio, struct page *subpage)
 /**
  * folio_put_swap() - Decrease swap count of swap entries of a folio.
  * @folio: folio with swap entries bounded, must be in swap cache and locked.
- * @subpage: if not NULL, only decrease the swap count of this subpage.
+ * @page: if not NULL, only decrease the swap count of this page.
  *
  * This won't free the swap slots even if swap count drops to zero, they are
  * still pinned by the swap cache. User may call folio_free_swap to free them.
  * Context: Caller must ensure the folio is locked and in the swap cache.
  */
-void folio_put_swap(struct folio *folio, struct page *subpage)
+void folio_put_swap(struct folio *folio, struct page *page)
 {
 	swp_entry_t entry = folio->swap;
 	unsigned long nr_pages = folio_nr_pages(folio);
@@ -1830,8 +1836,8 @@ void folio_put_swap(struct folio *folio, struct page *subpage)
 	VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
 	VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio);
 
-	if (subpage) {
-		entry.val += folio_page_idx(folio, subpage);
+	if (page) {
+		entry.val += folio_page_idx(folio, page);
 		nr_pages = 1;
 	}
 
@@ -2496,8 +2502,13 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
 	new_pte = pte_mkold(mk_pte(page, vma->vm_page_prot));
 	if (pte_swp_soft_dirty(old_pte))
 		new_pte = pte_mksoft_dirty(new_pte);
-	if (pte_swp_uffd_wp(old_pte))
-		new_pte = pte_mkuffd_wp(new_pte);
+	if (pte_swp_uffd(old_pte))
+		new_pte = pte_mkuffd(new_pte);
+
+	/* See do_swap_page(): restore PAGE_NONE for RWP */
+	if (pte_swp_uffd(old_pte) && userfaultfd_rwp(vma))
+		new_pte = pte_modify(new_pte, PAGE_NONE);
+
 setpte:
 	set_pte_at(vma->vm_mm, addr, pte, new_pte);
 	folio_put_swap(swapcache, folio_file_page(swapcache, swp_offset(entry)));
@@ -2941,6 +2952,12 @@ static int setup_swap_extents(struct swap_info_struct *sis,
 	struct inode *inode = mapping->host;
 	int ret;
 
+	ret = sio_pool_init();
+	if (ret)
+		return ret;
+
+	sis->ops = &swap_bdev_ops;
+
 	if (S_ISBLK(inode->i_mode)) {
 		ret = add_swap_extent(sis, 0, sis->max, 0);
 		*span = sis->pages;
@@ -2952,11 +2969,6 @@ static int setup_swap_extents(struct swap_info_struct *sis,
 		if (ret < 0)
 			return ret;
 		sis->flags |= SWP_ACTIVATED;
-		if ((sis->flags & SWP_FS_OPS) &&
-		    sio_pool_init() != 0) {
-			destroy_swap_extents(sis, swap_file);
-			return -ENOMEM;
-		}
 		return ret;
 	}
 
diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index 246af12bf8014..8fd24c8b428e1 100644
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -376,7 +376,7 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd,
 	if (writable)
 		_dst_pte = pte_mkwrite(_dst_pte, dst_vma);
 	if (flags & MFILL_ATOMIC_WP)
-		_dst_pte = pte_mkuffd_wp(_dst_pte);
+		_dst_pte = pte_mkuffd(_dst_pte);
 
 	ret = -EAGAIN;
 	dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
@@ -481,7 +481,7 @@ static void mfill_retry_state_save(struct mfill_retry_state *s,
 {
 	s->flags = vma_flags_and_mask(&vma->flags, MFILL_RETRY_STATE_VMA_FLAGS);
 	s->ops = vma_uffd_ops(vma);
-	s->pgoff = vma->vm_pgoff;
+	s->pgoff = vma_start_pgoff(vma);
 
 	if (vma->vm_file)
 		s->file = get_file(vma->vm_file);
@@ -507,7 +507,7 @@ static bool mfill_retry_state_changed(struct mfill_retry_state *state,
 
 	/* VMA was file backed, but file, inode or offset has changed */
 	if (!vma->vm_file || vma->vm_file->f_inode != state->file->f_inode ||
-	    state->file != vma->vm_file || vma->vm_pgoff != state->pgoff)
+	    state->file != vma->vm_file || vma_start_pgoff(vma) != state->pgoff)
 		return true;
 
 	return false;
@@ -1164,6 +1164,75 @@ static int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
 	return err;
 }
 
+int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
+		     unsigned long len, bool enable_rwp)
+{
+	struct mm_struct *dst_mm = ctx->mm;
+	unsigned long end = start + len;
+	struct vm_area_struct *dst_vma;
+	unsigned int mm_cp_flags;
+	struct mmu_gather tlb;
+	bool found = false;
+	VMA_ITERATOR(vmi, dst_mm, start);
+
+	VM_WARN_ON_ONCE(start & ~PAGE_MASK);
+	VM_WARN_ON_ONCE(len & ~PAGE_MASK);
+	VM_WARN_ON_ONCE(start + len <= start);
+
+	guard(mmap_read_lock)(dst_mm);
+	guard(rwsem_read)(&ctx->map_changing_lock);
+
+	if (atomic_read(&ctx->mmap_changing))
+		return -EAGAIN;
+
+	if (enable_rwp)
+		mm_cp_flags = MM_CP_UFFD_RWP;
+	else
+		mm_cp_flags = MM_CP_UFFD_RWP_RESOLVE;
+
+	/*
+	 * Pre-scan the range: validate every spanned VMA before applying
+	 * any change_protection() so a partial failure cannot leave the
+	 * process with only a prefix of the range re-protected.
+	 */
+	for_each_vma_range(vmi, dst_vma, end) {
+		if (!userfaultfd_rwp(dst_vma))
+			return -ENOENT;
+
+		if (is_vm_hugetlb_page(dst_vma)) {
+			unsigned long page_mask;
+
+			page_mask = vma_kernel_pagesize(dst_vma) - 1;
+			if ((start & page_mask) || (len & page_mask))
+				return -EINVAL;
+		}
+		found = true;
+	}
+	if (!found)
+		return -ENOENT;
+
+	vma_iter_set(&vmi, start);
+	tlb_gather_mmu(&tlb, dst_mm);
+	for_each_vma_range(vmi, dst_vma, end) {
+		unsigned long vma_start = max(dst_vma->vm_start, start);
+		unsigned long vma_end = min(dst_vma->vm_end, end);
+		unsigned int flags = mm_cp_flags;
+
+		/*
+		 * On resolve, try to upgrade writability per-VMA --
+		 * MM_CP_TRY_CHANGE_WRITABLE WARNs in
+		 * maybe_change_pte_writable() if the VMA is not VM_WRITE,
+		 * and RWP can be registered on PROT_READ-only mappings.
+		 */
+		if (!enable_rwp && vma_wants_manual_pte_write_upgrade(dst_vma))
+			flags |= MM_CP_TRY_CHANGE_WRITABLE;
+
+		change_protection(&tlb, dst_vma, vma_start, vma_end, flags);
+	}
+	tlb_finish_mmu(&tlb);
+
+	return 0;
+}
 
 void double_pt_lock(spinlock_t *ptl1,
 		    spinlock_t *ptl2)
@@ -1292,6 +1361,13 @@ static long move_present_ptes(struct mm_struct *mm,
 		if (pte_dirty(orig_src_pte))
 			orig_dst_pte = pte_mkdirty(orig_dst_pte);
 		orig_dst_pte = pte_mkwrite(orig_dst_pte, dst_vma);
+
+		/* Re-arm RWP on the moved PTE if dst_vma is RWP-registered. */
+		if (userfaultfd_rwp(dst_vma)) {
+			orig_dst_pte = pte_modify(orig_dst_pte, PAGE_NONE);
+			orig_dst_pte = pte_mkuffd(orig_dst_pte);
+		}
+
 		set_pte_at(mm, dst_addr, dst_pte, orig_dst_pte);
 
 		src_addr += PAGE_SIZE;
@@ -1373,6 +1449,9 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma,
 	orig_src_pte = ptep_get_and_clear(mm, src_addr, src_pte);
 	if (pgtable_supports_soft_dirty())
 		orig_src_pte = pte_swp_mksoft_dirty(orig_src_pte);
+	/* Re-arm RWP on the moved swap entry if dst_vma is RWP-registered. */
+	if (userfaultfd_rwp(dst_vma))
+		orig_src_pte = pte_swp_mkuffd(orig_src_pte);
 	set_pte_at(mm, dst_addr, dst_pte, orig_src_pte);
 	double_pt_unlock(dst_ptl, src_ptl);
 
@@ -1399,6 +1478,13 @@ static int move_zeropage_pte(struct mm_struct *mm,
 
 	zero_pte = pte_mkspecial(pfn_pte(zero_pfn(dst_addr),
 					 dst_vma->vm_page_prot));
+
+	/* Re-arm RWP on the moved PTE if dst_vma is RWP-registered. */
+	if (userfaultfd_rwp(dst_vma)) {
+		zero_pte = pte_modify(zero_pte, PAGE_NONE);
+		zero_pte = pte_mkuffd(zero_pte);
+	}
+
 	ptep_clear_flush(src_vma, src_addr, src_pte);
 	set_pte_at(mm, dst_addr, dst_pte, zero_pte);
 	double_pt_unlock(dst_ptl, src_ptl);
@@ -2111,7 +2197,10 @@ static bool vma_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags,
 {
 	const struct vm_uffd_ops *ops = vma_uffd_ops(vma);
 
-	if (vma->vm_flags & VM_DROPPABLE)
+	if (vma->vm_flags & (VM_DROPPABLE | VM_SHADOW_STACK))
+		return false;
+
+	if (!is_vm_hugetlb_page(vma) && (vma->vm_flags & VM_SPECIAL))
 		return false;
 
 	vm_flags &= __VM_UFFD_FLAGS;
@@ -2187,9 +2276,22 @@ static struct vm_area_struct *userfaultfd_clear_vma(struct vma_iterator *vmi,
 	if (start == vma->vm_start && end == vma->vm_end)
 		give_up_on_oom = true;
 
-	/* Reset ptes for the whole vma range if wr-protected */
-	if (userfaultfd_wp(vma))
-		uffd_wp_range(vma, start, end - start, false);
+	/* Clear the uffd bit and/or restore protnone PTEs */
+	if (userfaultfd_protected(vma)) {
+		unsigned int mm_cp_flags = 0;
+		struct mmu_gather tlb;
+
+		if (userfaultfd_wp(vma))
+			mm_cp_flags |= MM_CP_UFFD_WP_RESOLVE;
+		if (userfaultfd_rwp(vma))
+			mm_cp_flags |= MM_CP_UFFD_RWP_RESOLVE;
+		if (vma_wants_manual_pte_write_upgrade(vma))
+			mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE;
+
+		tlb_gather_mmu(&tlb, vma->vm_mm);
+		change_protection(&tlb, vma, start, end, mm_cp_flags);
+		tlb_finish_mmu(&tlb);
+	}
 
 	ret = vma_modify_flags_uffd(vmi, prev, vma, start, end,
 				    &new_vma_flags, NULL_VM_UFFD_CTX,
@@ -2235,13 +2337,29 @@ static int userfaultfd_register_range(struct userfaultfd_ctx *ctx,
 		 * userfaultfd and with the right tracking mode too.
 		 */
 		if (vma->vm_userfaultfd_ctx.ctx == ctx &&
-		    vma_test_all_mask(vma, vma_flags))
+		    (vma->vm_flags & __VM_UFFD_FLAGS) == vm_flags)
 			goto skip;
 
+		/*
+		 * Pre-scan in userfaultfd_register() already rejected mode
+		 * switches that would drop VM_UFFD_WP or VM_UFFD_RWP, so a
+		 * stray bit here is a bug.
+		 */
+		VM_WARN_ON_ONCE(vma->vm_userfaultfd_ctx.ctx == ctx &&
+				vma->vm_flags & (VM_UFFD_WP | VM_UFFD_RWP) & ~vm_flags);
+
 		if (vma->vm_start > start)
 			start = vma->vm_start;
 		vma_end = min(end, vma->vm_end);
 
+		/*
+		 * Re-registering into the same userfaultfd can remove WP mode.
+		 * Clear any per-PTE uffd-wp state before dropping VM_UFFD_WP,
+		 * matching the UFFDIO_UNREGISTER cleanup semantics.
+		 */
+		if (userfaultfd_wp(vma) && !(vm_flags & VM_UFFD_WP))
+			uffd_wp_range(vma, start, vma_end - start, false);
+
 		new_vma_flags = vma->flags;
 		vma_flags_clear_mask(&new_vma_flags, __VMA_UFFD_FLAGS);
 		vma_flags_set_mask(&new_vma_flags, vma_flags);
@@ -2368,14 +2486,29 @@ struct userfaultfd_wake_range {
 /* internal indication that UFFD_API ioctl was successfully executed */
 #define UFFD_FEATURE_INITIALIZED		(1u << 31)
 
+/*
+ * UFFDIO_SET_MODE updates ctx->features under mmap_write_lock with
+ * WRITE_ONCE; readers that run outside mmap_read_lock or the per-VMA
+ * lock (poll/read_iter/ioctl, fdinfo) must pair with READ_ONCE.
+ */
+static unsigned int userfaultfd_features(struct userfaultfd_ctx *ctx)
+{
+	return READ_ONCE(ctx->features);
+}
+
 static bool userfaultfd_is_initialized(struct userfaultfd_ctx *ctx)
 {
-	return ctx->features & UFFD_FEATURE_INITIALIZED;
+	return userfaultfd_features(ctx) & UFFD_FEATURE_INITIALIZED;
 }
 
 static bool userfaultfd_wp_async_ctx(struct userfaultfd_ctx *ctx)
 {
-	return ctx && (ctx->features & UFFD_FEATURE_WP_ASYNC);
+	return ctx && (userfaultfd_features(ctx) & UFFD_FEATURE_WP_ASYNC);
+}
+
+static bool userfaultfd_rwp_async_ctx(struct userfaultfd_ctx *ctx)
+{
+	return ctx && (userfaultfd_features(ctx) & UFFD_FEATURE_RWP_ASYNC);
 }
 
 /*
@@ -2390,7 +2523,7 @@ bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma)
 	if (!ctx)
 		return false;
 
-	return ctx->features & UFFD_FEATURE_WP_UNPOPULATED;
+	return userfaultfd_features(ctx) & UFFD_FEATURE_WP_UNPOPULATED;
 }
 
 static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode,
@@ -2504,6 +2637,8 @@ static inline struct uffd_msg userfault_msg(unsigned long address,
 		msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE;
 	if (reason & VM_UFFD_WP)
 		msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP;
+	if (reason & VM_UFFD_RWP)
+		msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_RWP;
 	if (reason & VM_UFFD_MINOR)
 		msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_MINOR;
 	if (features & UFFD_FEATURE_THREAD_ID)
@@ -2557,6 +2692,12 @@ static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
 	 */
 	if (!huge_pte_write(pte) && (reason & VM_UFFD_WP))
 		return true;
+	/*
+	 * PTE is still RW-protected (protnone with uffd bit), wait for
+	 * resolution. Plain PROT_NONE without the marker is not an RWP fault.
+	 */
+	if (pte_protnone(pte) && huge_pte_uffd(pte) && (reason & VM_UFFD_RWP))
+		return true;
 
 	return false;
 }
@@ -2617,8 +2758,14 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,
 	if (!pmd_present(_pmd))
 		return false;
 
-	if (pmd_trans_huge(_pmd))
-		return !pmd_write(_pmd) && (reason & VM_UFFD_WP);
+	if (pmd_trans_huge(_pmd)) {
+		if (!pmd_write(_pmd) && (reason & VM_UFFD_WP))
+			return true;
+		if (pmd_protnone(_pmd) && pmd_uffd(_pmd) &&
+		    (reason & VM_UFFD_RWP))
+			return true;
+		return false;
+	}
 
 	pte = pte_offset_map(pmd, address);
 	if (!pte)
@@ -2654,6 +2801,13 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,
 	 */
 	if (!pte_write(ptent) && (reason & VM_UFFD_WP))
 		goto out;
+	/*
+	 * PTE is still RW-protected (protnone with uffd bit), wait for
+	 * userspace to resolve. Plain PROT_NONE without the marker is not
+	 * an RWP fault.
+	 */
+	if (pte_protnone(ptent) && pte_uffd(ptent) && (reason & VM_UFFD_RWP))
+		goto out;
 
 	ret = false;
 out:
@@ -3598,11 +3752,27 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
 	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING)
 		vm_flags |= VM_UFFD_MISSING;
 	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) {
-		if (!pgtable_supports_uffd_wp())
+		if (!pgtable_supports_uffd())
 			goto out;
 
 		vm_flags |= VM_UFFD_WP;
 	}
+	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_RWP) {
+		if (!pgtable_supports_uffd() || VM_UFFD_RWP == VM_NONE)
+			goto out;
+		if (!(userfaultfd_features(ctx) & UFFD_FEATURE_RWP))
+			goto out;
+		vm_flags |= VM_UFFD_RWP;
+	}
+
+	/*
+	 * WP and RWP share the uffd PTE bit and
+	 * cannot coexist in the same VMA — the bit would carry ambiguous
+	 * semantics. Reject the combination up front.
+	 */
+	if ((vm_flags & VM_UFFD_WP) && (vm_flags & VM_UFFD_RWP))
+		goto out;
+
 	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) {
 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
 		goto out;
@@ -3657,6 +3827,17 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
 		if (!vma_can_userfault(cur, vm_flags, wp_async))
 			goto out_unlock;
 
+		/*
+		 * RWP uses protnone as an access-tracking marker. PROT_NONE
+		 * VMAs have vm_page_prot == PAGE_NONE, so RWP resolution
+		 * cannot make a page accessible again. Reject at register
+		 * time only: a VMA that later becomes inaccessible via
+		 * mprotect() must still be unregisterable, so this is not
+		 * part of vma_can_userfault().
+		 */
+		if ((vm_flags & VM_UFFD_RWP) && !vma_is_accessible(cur))
+			goto out_unlock;
+
 		/*
 		 * UFFDIO_COPY will fill file holes even without
 		 * PROT_WRITE. This check enforces that if this is a
@@ -3696,6 +3877,16 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
 		    cur->vm_userfaultfd_ctx.ctx != ctx)
 			goto out_unlock;
 
+		/*
+		 * Mode switches that drop VM_UFFD_WP or VM_UFFD_RWP would
+		 * leave PTE markers without the flag that describes them;
+		 * subsequent mprotect() would then promote stale markers
+		 * into the other mode. Require an unregister first.
+		 */
+		if (cur->vm_userfaultfd_ctx.ctx == ctx &&
+		    cur->vm_flags & (VM_UFFD_WP | VM_UFFD_RWP) & ~vm_flags)
+			goto out_unlock;
+
 		/*
 		 * Note vmas containing huge pages
 		 */
@@ -3729,6 +3920,10 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
 		if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR))
 			ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE);
 
+		/* RWPROTECT is only supported for RWP ranges */
+		if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_RWP))
+			ioctls_out &= ~((__u64)1 << _UFFDIO_RWPROTECT);
+
 		/*
 		 * Now that we scanned all vmas we can already tell
 		 * userland which ioctls methods are guaranteed to
@@ -4076,6 +4271,158 @@ static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx,
 	return ret;
 }
 
+static int userfaultfd_rwprotect(struct userfaultfd_ctx *ctx,
+				 unsigned long arg)
+{
+	int ret;
+	struct uffdio_rwprotect uffdio_rwp;
+	struct userfaultfd_wake_range range;
+	bool mode_rwp, mode_dontwake;
+
+	if (atomic_read(&ctx->mmap_changing))
+		return -EAGAIN;
+
+	if (copy_from_user(&uffdio_rwp, (void __user *)arg,
+			   sizeof(uffdio_rwp)))
+		return -EFAULT;
+
+	ret = validate_range(ctx->mm, uffdio_rwp.range.start,
+			     uffdio_rwp.range.len);
+	if (ret)
+		return ret;
+
+	if (uffdio_rwp.mode & ~(UFFDIO_RWPROTECT_MODE_DONTWAKE |
+				UFFDIO_RWPROTECT_MODE_RWP))
+		return -EINVAL;
+
+	mode_rwp = uffdio_rwp.mode & UFFDIO_RWPROTECT_MODE_RWP;
+	mode_dontwake = uffdio_rwp.mode & UFFDIO_RWPROTECT_MODE_DONTWAKE;
+
+	if (mode_rwp && mode_dontwake)
+		return -EINVAL;
+
+	if (mmget_not_zero(ctx->mm)) {
+		ret = mrwprotect_range(ctx, uffdio_rwp.range.start,
+				       uffdio_rwp.range.len, mode_rwp);
+		mmput(ctx->mm);
+	} else {
+		return -ESRCH;
+	}
+
+	if (ret)
+		return ret;
+
+	if (!mode_rwp && !mode_dontwake) {
+		range.start = uffdio_rwp.range.start;
+		range.len = uffdio_rwp.range.len;
+		wake_userfault(ctx, &range);
+	}
+	return ret;
+}
+
+/* Subset of UFFD_API_FEATURES actually supported by this kernel/arch */
+static __u64 uffd_api_available_features(void)
+{
+	__u64 f = UFFD_API_FEATURES;
+
+	if (!IS_ENABLED(CONFIG_HAVE_ARCH_USERFAULTFD_MINOR))
+		f &= ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM);
+	if (!pgtable_supports_uffd())
+		f &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP;
+	if (!uffd_supports_wp_marker())
+		f &= ~(UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
+		       UFFD_FEATURE_WP_UNPOPULATED |
+		       UFFD_FEATURE_WP_ASYNC);
+	/*
+	 * RWP needs both PROT_NONE support and the uffd PTE bit. The
+	 * VM_UFFD_RWP check covers compile-time unavailability; the
+	 * pgtable_supports_uffd() check covers runtime (e.g. riscv
+	 * without the SVRSW60T59B extension) where the PTE bit is declared
+	 * but not actually usable.
+	 */
+	if (VM_UFFD_RWP == VM_NONE || !pgtable_supports_uffd())
+		f &= ~(UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC);
+	return f;
+}
+
+/* Async features that can be toggled at runtime via UFFDIO_SET_MODE */
+#define UFFD_FEATURE_TOGGLEABLE	UFFD_FEATURE_RWP_ASYNC
+
+static int userfaultfd_set_mode(struct userfaultfd_ctx *ctx,
+				unsigned long arg)
+{
+	struct uffdio_set_mode mode;
+	struct mm_struct *mm = ctx->mm;
+
+	if (copy_from_user(&mode, (void __user *)arg, sizeof(mode)))
+		return -EFAULT;
+
+	/* enable and disable must not overlap */
+	if (mode.enable & mode.disable)
+		return -EINVAL;
+
+	/* only toggleable features that this kernel/arch actually supports */
+	if ((mode.enable | mode.disable) &
+	    ~(uffd_api_available_features() & UFFD_FEATURE_TOGGLEABLE))
+		return -EINVAL;
+
+	/* RWP_ASYNC can only be enabled on contexts that negotiated RWP */
+	if ((mode.enable & UFFD_FEATURE_RWP_ASYNC) &&
+	    !(userfaultfd_features(ctx) & UFFD_FEATURE_RWP))
+		return -EINVAL;
+
+	if (!mmget_not_zero(mm))
+		return -ESRCH;
+
+	/*
+	 * Drain in-flight faults before flipping features. mmap_write_lock()
+	 * blocks new mmap_read_lock() callers, but per-VMA locked faults
+	 * (lock_vma_under_rcu() + FAULT_FLAG_VMA_LOCK) that acquired before
+	 * this point keep running. Calling vma_start_write() on each UFFD-
+	 * armed VMA waits for those readers to drop, so no in-flight fault
+	 * can observe the old features after mmap_write_unlock().
+	 */
+	mmap_write_lock(mm);
+	{
+		struct vm_area_struct *vma;
+		VMA_ITERATOR(vmi, mm, 0);
+
+		for_each_vma(vmi, vma) {
+			if (vma->vm_userfaultfd_ctx.ctx == ctx)
+				vma_start_write(vma);
+		}
+	}
+	/*
+	 * Single WRITE_ONCE so lockless readers (fdinfo, poll/read_iter
+	 * via userfaultfd_is_initialized(), and the userfaultfd_features()
+	 * helper used elsewhere) can't observe a mid-RMW intermediate
+	 * value. Hot-path readers already serialise through the mmap lock
+	 * + vma_start_write() drain above, so their load doesn't need an
+	 * annotation.
+	 */
+	WRITE_ONCE(ctx->features,
+		   (ctx->features | mode.enable) & ~mode.disable);
+	mmap_write_unlock(mm);
+
+	/*
+	 * If switching to async, wake threads blocked in handle_userfault().
+	 * They will retry the fault and auto-resolve under the new mode.
+	 * len=0 means wake all pending faults on this context.
+	 */
+	if (mode.enable & UFFD_FEATURE_RWP_ASYNC) {
+		struct userfaultfd_wake_range range = { .len = 0 };
+
+		spin_lock_irq(&ctx->fault_pending_wqh.lock);
+		__wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL,
+				     &range);
+		__wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range);
+		spin_unlock_irq(&ctx->fault_pending_wqh.lock);
+	}
+
+	mmput(mm);
+	return 0;
+}
+
 static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg)
 {
 	__s64 ret;
@@ -4199,6 +4546,11 @@ bool userfaultfd_wp_async(struct vm_area_struct *vma)
 	return userfaultfd_wp_async_ctx(vma->vm_userfaultfd_ctx.ctx);
 }
 
+bool userfaultfd_rwp_async(struct vm_area_struct *vma)
+{
+	return userfaultfd_rwp_async_ctx(vma->vm_userfaultfd_ctx.ctx);
+}
+
 static inline unsigned int uffd_ctx_features(__u64 user_features)
 {
 	/*
@@ -4302,20 +4654,14 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx,
 	if (features & UFFD_FEATURE_WP_ASYNC)
 		features |= UFFD_FEATURE_WP_UNPOPULATED;
 
-	/* report all available features and ioctls to userland */
-	uffdio_api.features = UFFD_API_FEATURES;
-#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
-	uffdio_api.features &=
-		~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM);
-#endif
-	if (!pgtable_supports_uffd_wp())
-		uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP;
+	ret = -EINVAL;
+	/* RWP_ASYNC requires RWP */
+	if ((features & UFFD_FEATURE_RWP_ASYNC) &&
+	    !(features & UFFD_FEATURE_RWP))
+		goto err_out;
 
-	if (!uffd_supports_wp_marker()) {
-		uffdio_api.features &= ~UFFD_FEATURE_WP_HUGETLBFS_SHMEM;
-		uffdio_api.features &= ~UFFD_FEATURE_WP_UNPOPULATED;
-		uffdio_api.features &= ~UFFD_FEATURE_WP_ASYNC;
-	}
+	/* report all available features and ioctls to userland */
+	uffdio_api.features = uffd_api_available_features();
 
 	ret = -EINVAL;
 	if (features & ~uffdio_api.features)
@@ -4382,6 +4728,12 @@ static long userfaultfd_ioctl(struct file *file, unsigned cmd,
 	case UFFDIO_POISON:
 		ret = userfaultfd_poison(ctx, arg);
 		break;
+	case UFFDIO_RWPROTECT:
+		ret = userfaultfd_rwprotect(ctx, arg);
+		break;
+	case UFFDIO_SET_MODE:
+		ret = userfaultfd_set_mode(ctx, arg);
+		break;
 	}
 	return ret;
 }
@@ -4409,7 +4761,7 @@ static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f)
 	 *	protocols: aa:... bb:...
 	 */
 	seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n",
-		   pending, total, UFFD_API, ctx->features,
+		   pending, total, UFFD_API, userfaultfd_features(ctx),
 		   UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS);
 }
 #endif
diff --git a/mm/util.c b/mm/util.c
index af2c2103f0d95..bf0513d1d3d08 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -578,8 +578,8 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,
 	if (!ret) {
 		if (mmap_write_lock_killable(mm))
 			return -EINTR;
-		ret = do_mmap(file, addr, len, prot, flag, 0, pgoff, &populate,
-			      &uf);
+		ret = do_mmap(file, addr, len, prot, flag, EMPTY_VMA_FLAGS, pgoff,
+			      &populate, &uf);
 		mmap_write_unlock(mm);
 		userfaultfd_unmap_complete(mm, &uf);
 		if (populate)
@@ -627,20 +627,20 @@ EXPORT_SYMBOL(vm_mmap);
 unsigned long vm_mmap_shadow_stack(unsigned long addr, unsigned long len,
 		unsigned long flags)
 {
+	vma_flags_t vma_flags = VMA_SHADOW_STACK;
 	struct mm_struct *mm = current->mm;
 	unsigned long ret, unused;
-	vm_flags_t vm_flags = VM_SHADOW_STACK;
 
 	flags |= MAP_ANONYMOUS | MAP_PRIVATE;
 	if (addr)
 		flags |= MAP_FIXED_NOREPLACE;
 
 	if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
-		vm_flags |= VM_NOHUGEPAGE;
+		vma_flags_set(&vma_flags, VMA_NOHUGEPAGE_BIT);
 
 	mmap_write_lock(mm);
 	ret = do_mmap(NULL, addr, len, PROT_READ | PROT_WRITE, flags,
-		      vm_flags, 0, &unused, NULL);
+		      vma_flags, 0, &unused, NULL);
 	mmap_write_unlock(mm);
 
 	return ret;
@@ -1188,7 +1188,7 @@ void compat_set_desc_from_vma(struct vm_area_desc *desc,
 	desc->start = vma->vm_start;
 	desc->end = vma->vm_end;
 
-	desc->pgoff = vma->vm_pgoff;
+	desc->pgoff = vma_start_pgoff(vma);
 	desc->vm_file = vma->vm_file;
 	desc->vma_flags = vma->flags;
 	desc->page_prot = vma->vm_page_prot;
@@ -1353,7 +1353,7 @@ void snapshot_page(struct page_snapshot *ps, const struct page *page)
 	if (ps->idx < MAX_FOLIO_NR_PAGES) {
 		memcpy(&ps->folio_snapshot, foliop, 2 * sizeof(struct page));
 		nr_pages = folio_nr_pages(&ps->folio_snapshot);
-		if (nr_pages > 1)
+		if (nr_pages > 2)
 			memcpy(&ps->folio_snapshot.__page_2, &foliop->__page_2,
 			       sizeof(struct page));
 		set_ps_flags(ps, foliop, page);
@@ -1379,7 +1379,7 @@ static int call_vma_mapped(struct vm_area_struct *vma)
 	if (!vm_ops || !vm_ops->mapped)
 		return 0;
 
-	err = vm_ops->mapped(vma->vm_start, vma->vm_end, vma->vm_pgoff,
+	err = vm_ops->mapped(vma->vm_start, vma->vm_end, vma_start_pgoff(vma),
 			     vma->vm_file, &vm_private_data);
 	if (err)
 		return err;
diff --git a/mm/vma.c b/mm/vma.c
index 9eea2850818a8..b5bc3eec961c7 100644
--- a/mm/vma.c
+++ b/mm/vma.c
@@ -4,6 +4,10 @@
  * VMA-specific functions.
  */
 
+/*
+ * To allow for userland testing we place internal dependencies in
+ * vma_internal.h and external VMA API declarations in vma.h.
+ */
 #include "vma_internal.h"
 #include "vma.h"
 
@@ -52,7 +56,7 @@ struct mmap_state {
 		.pglen = PHYS_PFN(len_),				\
 		.vma_flags = vma_flags_,				\
 		.file = file_,						\
-		.page_prot = vma_get_page_prot(vma_flags_),		\
+		.page_prot = vma_flags_to_page_prot(vma_flags_),	\
 	}
 
 #define VMG_MMAP_STATE(name, map_, vma_)				\
@@ -70,6 +74,20 @@ struct mmap_state {
 		.state = VMA_MERGE_START,				\
 	}
 
+static void __vma_set_range(struct vm_area_struct *vma, unsigned long start,
+			    unsigned long end)
+{
+	vma->vm_start = start;
+	vma->vm_end = end;
+}
+
+static void vma_set_range(struct vm_area_struct *vma, unsigned long start,
+			  unsigned long end, pgoff_t pgoff)
+{
+	__vma_set_range(vma, start, end);
+	vma_set_pgoff(vma, pgoff);
+}
+
 /* Was this VMA ever forked from a parent, i.e. maybe contains CoW mappings? */
 static bool vma_is_fork_child(struct vm_area_struct *vma)
 {
@@ -197,15 +215,13 @@ static void init_multi_vma_prep(struct vma_prepare *vp,
  */
 static bool can_vma_merge_before(struct vma_merge_struct *vmg)
 {
-	pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start);
-
-	if (is_mergeable_vma(vmg, /* merge_next = */ true) &&
-	    is_mergeable_anon_vma(vmg, /* merge_next = */ true)) {
-		if (vmg->next->vm_pgoff == vmg->pgoff + pglen)
-			return true;
-	}
-
-	return false;
+	if (!is_mergeable_vma(vmg, /* merge_next = */ true))
+		return false;
+	if (!is_mergeable_anon_vma(vmg, /* merge_next = */ true))
+		return false;
+	if (vmg_end_pgoff(vmg) != vma_start_pgoff(vmg->next))
+		return false;
+	return true;
 }
 
 /*
@@ -219,12 +235,13 @@ static bool can_vma_merge_before(struct vma_merge_struct *vmg)
  */
 static bool can_vma_merge_after(struct vma_merge_struct *vmg)
 {
-	if (is_mergeable_vma(vmg, /* merge_next = */ false) &&
-	    is_mergeable_anon_vma(vmg, /* merge_next = */ false)) {
-		if (vmg->prev->vm_pgoff + vma_pages(vmg->prev) == vmg->pgoff)
-			return true;
-	}
-	return false;
+	if (!is_mergeable_vma(vmg, /* merge_next = */ false))
+		return false;
+	if (!is_mergeable_anon_vma(vmg, /* merge_next = */ false))
+		return false;
+	if (vma_end_pgoff(vmg->prev) != vmg_start_pgoff(vmg))
+		return false;
+	return true;
 }
 
 static void __vma_link_file(struct vm_area_struct *vma,
@@ -234,7 +251,7 @@ static void __vma_link_file(struct vm_area_struct *vma,
 		mapping_allow_writable(mapping);
 
 	flush_dcache_mmap_lock(mapping);
-	vma_interval_tree_insert(vma, &mapping->i_mmap);
+	mapping_rmap_tree_insert(vma, mapping);
 	flush_dcache_mmap_unlock(mapping);
 }
 
@@ -248,7 +265,7 @@ static void __remove_shared_vm_struct(struct vm_area_struct *vma,
 		mapping_unmap_writable(mapping);
 
 	flush_dcache_mmap_lock(mapping);
-	vma_interval_tree_remove(vma, &mapping->i_mmap);
+	mapping_rmap_tree_remove(vma, mapping);
 	flush_dcache_mmap_unlock(mapping);
 }
 
@@ -258,30 +275,30 @@ static void __remove_shared_vm_struct(struct vm_area_struct *vma,
  *
  * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
  * vma must be removed from the anon_vma's interval trees using
- * anon_vma_interval_tree_pre_update_vma().
+ * anon_rmap_tree_pre_update_vma().
  *
  * After the update, the vma will be reinserted using
- * anon_vma_interval_tree_post_update_vma().
+ * anon_rmap_tree_post_update_vma().
  *
  * The entire update must be protected by exclusive mmap_lock and by
  * the root anon_vma's mutex.
  */
 static void
-anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma)
+anon_rmap_tree_pre_update_vma(struct vm_area_struct *vma)
 {
 	struct anon_vma_chain *avc;
 
 	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
-		anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
+		anon_rmap_tree_remove(avc, avc->anon_vma);
 }
 
 static void
-anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma)
+anon_rmap_tree_post_update_vma(struct vm_area_struct *vma)
 {
 	struct anon_vma_chain *avc;
 
 	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
-		anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
+		anon_rmap_tree_insert(avc, avc->anon_vma);
 }
 
 /*
@@ -312,17 +329,16 @@ static void vma_prepare(struct vma_prepare *vp)
 
 	if (vp->anon_vma) {
 		anon_vma_lock_write(vp->anon_vma);
-		anon_vma_interval_tree_pre_update_vma(vp->vma);
+		anon_rmap_tree_pre_update_vma(vp->vma);
 		if (vp->adj_next)
-			anon_vma_interval_tree_pre_update_vma(vp->adj_next);
+			anon_rmap_tree_pre_update_vma(vp->adj_next);
 	}
 
 	if (vp->file) {
 		flush_dcache_mmap_lock(vp->mapping);
-		vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap);
+		mapping_rmap_tree_remove(vp->vma, vp->mapping);
 		if (vp->adj_next)
-			vma_interval_tree_remove(vp->adj_next,
-						 &vp->mapping->i_mmap);
+			mapping_rmap_tree_remove(vp->adj_next, vp->mapping);
 	}
 
 }
@@ -340,9 +356,8 @@ static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi,
 {
 	if (vp->file) {
 		if (vp->adj_next)
-			vma_interval_tree_insert(vp->adj_next,
-						 &vp->mapping->i_mmap);
-		vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap);
+			mapping_rmap_tree_insert(vp->adj_next, vp->mapping);
+		mapping_rmap_tree_insert(vp->vma, vp->mapping);
 		flush_dcache_mmap_unlock(vp->mapping);
 	}
 
@@ -361,9 +376,9 @@ static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi,
 	}
 
 	if (vp->anon_vma) {
-		anon_vma_interval_tree_post_update_vma(vp->vma);
+		anon_rmap_tree_post_update_vma(vp->vma);
 		if (vp->adj_next)
-			anon_vma_interval_tree_post_update_vma(vp->adj_next);
+			anon_rmap_tree_post_update_vma(vp->adj_next);
 		anon_vma_unlock_write(vp->anon_vma);
 	}
 
@@ -521,7 +536,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
 		new->vm_end = addr;
 	} else {
 		new->vm_start = addr;
-		new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
+		vma_add_pgoff(new, linear_page_delta(vma, addr));
 	}
 
 	err = -ENOMEM;
@@ -560,7 +575,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
 
 	if (new_below) {
 		vma->vm_start = addr;
-		vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT;
+		vma_add_pgoff(vma, (addr - new->vm_start) >> PAGE_SHIFT);
 	} else {
 		vma->vm_end = addr;
 	}
@@ -679,7 +694,7 @@ void validate_mm(struct mm_struct *mm)
 		if (anon_vma) {
 			anon_vma_lock_read(anon_vma);
 			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
-				anon_vma_interval_tree_verify(avc);
+				anon_rmap_tree_verify(avc);
 			anon_vma_unlock_read(anon_vma);
 		}
 #endif
@@ -704,20 +719,56 @@ void validate_mm(struct mm_struct *mm)
  */
 static void vmg_adjust_set_range(struct vma_merge_struct *vmg)
 {
-	struct vm_area_struct *adjust;
-	pgoff_t pgoff;
-
 	if (vmg->__adjust_middle_start) {
-		adjust = vmg->middle;
-		pgoff = adjust->vm_pgoff + PHYS_PFN(vmg->end - adjust->vm_start);
+		/*
+		 * vmg->start    vmg->end
+		 * |             |
+		 * v    merge    v
+		 * <------------->
+		 *         delta
+		 *        <------>
+		 * |------|----------------|
+		 * | prev |    middle      |
+		 * |------|----------------|
+		 *        ^
+		 *        |
+		 *        middle->vm_start
+		 */
+		struct vm_area_struct *middle = vmg->middle;
+		const unsigned long delta = vmg->end - middle->vm_start;
+
+		__vma_set_range(middle, vmg->end, middle->vm_end);
+		vma_add_pgoff(middle, delta >> PAGE_SHIFT);
 	} else if (vmg->__adjust_next_start) {
-		adjust = vmg->next;
-		pgoff = adjust->vm_pgoff - PHYS_PFN(adjust->vm_start - vmg->end);
-	} else {
-		return;
-	}
+		/*
+		 *                Originally:
+		 *
+		 *            vmg->start   vmg->end
+		 *            |            |
+		 *            v    merge   v
+		 *            <------------>
+		 *            .            .
+		 * merge_existing_range() updates to:
+		 *            .            .
+		 * vmg->start vmg->end     .
+		 * |          |            .
+		 * v  retain  v            .
+		 * <---------->            .
+		 *             delta       .
+		 *            <----->      .
+		 * |----------------|------|
+		 * |    middle      | next |
+		 * |----------------|------|
+		 *                  ^
+		 *                  |
+		 *                  next->vm_start
+		 */
+		struct vm_area_struct *next = vmg->next;
+		const unsigned long delta = next->vm_start - vmg->end;
 
-	vma_set_range(adjust, vmg->end, adjust->vm_end, pgoff);
+		__vma_set_range(next, vmg->end, next->vm_end);
+		vma_sub_pgoff(next, delta >> PAGE_SHIFT);
+	}
 }
 
 /*
@@ -761,7 +812,7 @@ static int commit_merge(struct vma_merge_struct *vmg)
 	 */
 	vma_adjust_trans_huge(vma, vmg->start, vmg->end,
 			      vmg->__adjust_middle_start ? vmg->middle : NULL);
-	vma_set_range(vma, vmg->start, vmg->end, vmg->pgoff);
+	vma_set_range(vma, vmg->start, vmg->end, vmg_start_pgoff(vmg));
 	vmg_adjust_set_range(vmg);
 	vma_iter_store_overwrite(vmg->vmi, vmg->target);
 
@@ -928,10 +979,9 @@ static __must_check struct vm_area_struct *vma_merge_existing_range(
 		 *   prev   middle   next
 		 *  extend  delete  delete
 		 */
-
 		vmg->start = prev->vm_start;
 		vmg->end = next->vm_end;
-		vmg->pgoff = prev->vm_pgoff;
+		vmg->pgoff = vma_start_pgoff(prev);
 
 		/*
 		 * We already ensured anon_vma compatibility above, so now it's
@@ -948,9 +998,8 @@ static __must_check struct vm_area_struct *vma_merge_existing_range(
 		 *   prev     middle
 		 *  extend shrink/delete
 		 */
-
 		vmg->start = prev->vm_start;
-		vmg->pgoff = prev->vm_pgoff;
+		vmg->pgoff = vma_start_pgoff(prev);
 
 		if (!vmg->__remove_middle)
 			vmg->__adjust_middle_start = true;
@@ -964,8 +1013,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range(
 		 *    middle     next
 		 * shrink/delete extend
 		 */
-
-		pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start);
+		const pgoff_t pglen = vmg_pages(vmg);
 
 		VM_WARN_ON_VMG(!merge_right, vmg);
 		/* If we are offset into a VMA, then prev must be middle. */
@@ -973,13 +1021,13 @@ static __must_check struct vm_area_struct *vma_merge_existing_range(
 
 		if (vmg->__remove_middle) {
 			vmg->end = next->vm_end;
-			vmg->pgoff = next->vm_pgoff - pglen;
+			vmg->pgoff = vma_start_pgoff(next) - pglen;
 		} else {
 			/* We shrink middle and expand next. */
 			vmg->__adjust_next_start = true;
 			vmg->start = middle->vm_start;
 			vmg->end = start;
-			vmg->pgoff = middle->vm_pgoff;
+			vmg->pgoff = vma_start_pgoff(middle);
 		}
 
 		err = dup_anon_vma(next, middle, &anon_dup);
@@ -1088,7 +1136,7 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg)
 	if (can_merge_left) {
 		vmg->start = prev->vm_start;
 		vmg->target = prev;
-		vmg->pgoff = prev->vm_pgoff;
+		vmg->pgoff = vma_start_pgoff(prev);
 
 		/*
 		 * If this merge would result in removal of the next VMA but we
@@ -1235,27 +1283,24 @@ int vma_expand(struct vma_merge_struct *vmg)
 	return -ENOMEM;
 }
 
-/*
- * vma_shrink() - Reduce an existing VMAs memory area
+/**
+ * vma_shrink() - Shrink the end of a VMA
  * @vmi: The vma iterator
  * @vma: The VMA to modify
- * @start: The new start
  * @end: The new end
  *
+ * Note that the caller may only shrink the end of the VMA.
+ *
  * Returns: 0 on success, -ENOMEM otherwise
  */
 int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
-	       unsigned long start, unsigned long end, pgoff_t pgoff)
+	       unsigned long end)
 {
 	struct vma_prepare vp;
 
-	WARN_ON((vma->vm_start != start) && (vma->vm_end != end));
-
-	if (vma->vm_start < start)
-		vma_iter_config(vmi, vma->vm_start, start);
-	else
-		vma_iter_config(vmi, end, vma->vm_end);
+	VM_WARN_ON_ONCE(end > vma->vm_end);
 
+	vma_iter_config(vmi, end, vma->vm_end);
 	if (vma_iter_prealloc(vmi, NULL))
 		return -ENOMEM;
 
@@ -1263,10 +1308,10 @@ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
 
 	init_vma_prep(&vp, vma);
 	vma_prepare(&vp);
-	vma_adjust_trans_huge(vma, start, end, NULL);
+	vma_adjust_trans_huge(vma, vma->vm_start, end, NULL);
 
 	vma_iter_clear(vmi);
-	vma_set_range(vma, start, end, pgoff);
+	__vma_set_range(vma, vma->vm_start, end);
 	vma_complete(&vp, vmi, vma->vm_mm);
 	validate_mm(vma->vm_mm);
 	return 0;
@@ -1919,7 +1964,8 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
 			VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma);
 			*vmap = vma = new_vma;
 		}
-		*need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff);
+		*need_rmap_locks =
+			(vma_start_pgoff(new_vma) <= vma_start_pgoff(vma));
 	} else {
 		new_vma = vm_area_dup(vma);
 		if (!new_vma)
@@ -1972,14 +2018,25 @@ static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *
 {
 	vma_flags_t diff = vma_flags_diff_pair(&a->flags, &b->flags);
 
+	/* Ignore flags that mprotect() can change. */
 	vma_flags_clear_mask(&diff, VMA_ACCESS_FLAGS);
+	/* Ignore flags that do not impact merging. */
 	vma_flags_clear_mask(&diff, VMA_IGNORE_MERGE_FLAGS);
 
-	return a->vm_end == b->vm_start &&
-		mpol_equal(vma_policy(a), vma_policy(b)) &&
-		a->vm_file == b->vm_file &&
-		vma_flags_empty(&diff) &&
-		b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
+	/* Must be adjacent. */
+	if (a->vm_end != b->vm_start)
+		return false;
+	/* Must have matching policy. */
+	if (!mpol_equal(vma_policy(a), vma_policy(b)))
+		return false;
+	/* Must both be anon or map the same file (MAP_PRIVATE case). */
+	if (a->vm_file != b->vm_file)
+		return false;
+	/* Flags must be equivalent modulo mprotect(). */
+	if (!vma_flags_empty(&diff))
+		return false;
+	/* Page offset must align. */
+	return vma_end_pgoff(a) == vma_start_pgoff(b);
 }
 
 /*
@@ -2119,7 +2176,7 @@ bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot)
 	/* The open routine did something to the protections that pgprot_modify
 	 * won't preserve? */
 	if (pgprot_val(vm_page_prot) !=
-	    pgprot_val(vm_pgprot_modify(vm_page_prot, vma->vm_flags)))
+	    pgprot_val(vma_pgprot_modify(vm_page_prot, vma->flags)))
 		return false;
 
 	/*
@@ -2535,6 +2592,8 @@ static int __mmap_new_file_vma(struct mmap_state *map,
 static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap,
 	struct mmap_action *action)
 {
+	const bool is_anon = !map->file &&
+		!vma_flags_test(&map->vma_flags, VMA_SHARED_BIT);
 	struct vma_iterator *vmi = map->vmi;
 	int error = 0;
 	struct vm_area_struct *vma;
@@ -2549,6 +2608,10 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap,
 		return -ENOMEM;
 
 	vma_iter_config(vmi, map->addr, map->end);
+
+	if (is_anon)
+		vma_set_anonymous(vma);
+
 	vma_set_range(vma, map->addr, map->end, map->pgoff);
 	vma->flags = map->vma_flags;
 	vma->vm_page_prot = map->page_prot;
@@ -2558,12 +2621,11 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap,
 		goto free_vma;
 	}
 
+	/* Invoke callbacks. */
 	if (map->file)
 		error = __mmap_new_file_vma(map, vma);
-	else if (vma_flags_test(&map->vma_flags, VMA_SHARED_BIT))
+	else if (!is_anon)
 		error = shmem_zero_setup(vma);
-	else
-		vma_set_anonymous(vma);
 
 	if (error)
 		goto free_iter_vma;
@@ -2821,7 +2883,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr,
  * file to be mapped, otherwise NULL.
  * @addr: The page-aligned address at which to perform the mapping.
  * @len: The page-aligned, non-zero, length of the mapping.
- * @vm_flags: The VMA flags which should be applied to the mapping.
+ * @vma_flags: The VMA flags which should be applied to the mapping.
  * @pgoff: If @file is specified, the page offset into the file, if not then
  * the virtual page offset in memory of the anonymous mapping.
  * @uf: Optionally, a pointer to a list head used for tracking userfaultfd unmap
@@ -2831,12 +2893,11 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr,
  * been performed.
  */
 unsigned long mmap_region(struct file *file, unsigned long addr,
-			  unsigned long len, vm_flags_t vm_flags,
+			  unsigned long len, vma_flags_t vma_flags,
 			  unsigned long pgoff, struct list_head *uf)
 {
 	unsigned long ret;
 	bool writable_file_mapping = false;
-	const vma_flags_t vma_flags = legacy_to_vma_flags(vm_flags);
 
 	mmap_assert_write_locked(current->mm);
 
@@ -2845,7 +2906,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
 		return -EACCES;
 
 	/* Allow architectures to sanity-check the vm_flags. */
-	if (!arch_validate_flags(vm_flags))
+	if (!arch_validate_flags(vma_flags_to_legacy(vma_flags)))
 		return -EINVAL;
 
 	/* Map writable and ensure this isn't a sealed memfd. */
@@ -3167,23 +3228,22 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address)
 
 	/* Somebody else might have raced and expanded it already */
 	if (address > vma->vm_end) {
-		unsigned long size, grow;
-
-		size = address - vma->vm_start;
-		grow = (address - vma->vm_end) >> PAGE_SHIFT;
+		const unsigned long size = address - vma->vm_start;
+		const unsigned long grow = (address - vma->vm_end) >> PAGE_SHIFT;
+		const pgoff_t pgoff = vma_start_pgoff(vma);
 
 		error = -ENOMEM;
-		if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) {
+		if (pgoff + (size >> PAGE_SHIFT) >= pgoff) {
 			error = acct_stack_growth(vma, size, grow);
 			if (!error) {
 				if (vma_test(vma, VMA_LOCKED_BIT))
 					mm->locked_vm += grow;
 				vm_stat_account(mm, vma->vm_flags, grow);
-				anon_vma_interval_tree_pre_update_vma(vma);
+				anon_rmap_tree_pre_update_vma(vma);
 				vma->vm_end = address;
 				/* Overwrite old entry in mtree. */
 				vma_iter_store_overwrite(&vmi, vma);
-				anon_vma_interval_tree_post_update_vma(vma);
+				anon_rmap_tree_post_update_vma(vma);
 
 				perf_event_mmap(vma);
 			}
@@ -3246,24 +3306,22 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address)
 
 	/* Somebody else might have raced and expanded it already */
 	if (address < vma->vm_start) {
-		unsigned long size, grow;
-
-		size = vma->vm_end - address;
-		grow = (vma->vm_start - address) >> PAGE_SHIFT;
+		const unsigned long size = vma->vm_end - address;
+		const unsigned long grow = (vma->vm_start - address) >> PAGE_SHIFT;
 
 		error = -ENOMEM;
-		if (grow <= vma->vm_pgoff) {
+		if (grow <= vma_start_pgoff(vma)) {
 			error = acct_stack_growth(vma, size, grow);
 			if (!error) {
 				if (vma_test(vma, VMA_LOCKED_BIT))
 					mm->locked_vm += grow;
 				vm_stat_account(mm, vma->vm_flags, grow);
-				anon_vma_interval_tree_pre_update_vma(vma);
+				anon_rmap_tree_pre_update_vma(vma);
 				vma->vm_start = address;
-				vma->vm_pgoff -= grow;
+				vma_sub_pgoff(vma, grow);
 				/* Overwrite old entry in mtree. */
 				vma_iter_store_overwrite(&vmi, vma);
-				anon_vma_interval_tree_post_update_vma(vma);
+				anon_rmap_tree_post_update_vma(vma);
 
 				perf_event_mmap(vma);
 			}
@@ -3293,9 +3351,9 @@ int __vm_munmap(unsigned long start, size_t len, bool unlock)
 	return ret;
 }
 
-/* Insert vm structure into process list sorted by address
- * and into the inode's i_mmap tree.  If vm_file is non-NULL
- * then i_mmap_rwsem is taken here.
+/*
+ * Insert vm structure into process list sorted by address
+ * and into the inode's i_mmap tree if file-backed.
  */
 int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
 {
@@ -3321,8 +3379,8 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
 	 * Similarly in do_mmap and in do_brk_flags.
 	 */
 	if (vma_is_anonymous(vma)) {
-		BUG_ON(vma->anon_vma);
-		vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
+		WARN_ON_ONCE(vma->anon_vma);
+		vma_set_pgoff(vma, vma->vm_start >> PAGE_SHIFT);
 	}
 
 	if (vma_link(mm, vma)) {
@@ -3354,3 +3412,44 @@ __weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
 {
 	return vma_kernel_pagesize(vma);
 }
+
+struct vm_area_struct *__install_special_mapping(
+	struct mm_struct *mm,
+	unsigned long addr, unsigned long len,
+	vm_flags_t vm_flags, void *priv,
+	const struct vm_operations_struct *ops)
+{
+	vma_flags_t vma_flags = legacy_to_vma_flags(vm_flags);
+	struct vm_area_struct *vma;
+	int ret;
+
+	vma = vm_area_alloc(mm);
+	if (unlikely(!vma))
+		return ERR_PTR(-ENOMEM);
+
+	vma_flags_set_mask(&vma_flags, mm->def_vma_flags);
+	vma_flags_set(&vma_flags, VMA_DONTEXPAND_BIT);
+	if (pgtable_supports_soft_dirty())
+		vma_flags_set(&vma_flags, VMA_SOFTDIRTY_BIT);
+	vma_flags_clear_mask(&vma_flags, VMA_LOCKED_MASK);
+	vma->flags = vma_flags;
+	vma->vm_page_prot = vma_get_page_prot(vma);
+
+	vma->vm_ops = ops;
+	vma->vm_private_data = priv;
+	vma_set_range(vma, addr, addr + len, 0);
+
+	ret = insert_vm_struct(mm, vma);
+	if (ret)
+		goto out;
+
+	vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT);
+
+	perf_event_mmap(vma);
+
+	return vma;
+
+out:
+	vm_area_free(vma);
+	return ERR_PTR(ret);
+}
diff --git a/mm/vma.h b/mm/vma.h
index 8e4b61a7304c6..0bc7d521e9767 100644
--- a/mm/vma.h
+++ b/mm/vma.h
@@ -2,7 +2,14 @@
 /*
  * vma.h
  *
- * Core VMA manipulation API implemented in vma.c.
+ * Core VMA manipulation API implemented in vma.c, vma_init.c and vma_exec.c.
+ *
+ * Note that, in order for VMA logic to be userland testable, this header
+ * intentionally includes no dependencies.
+ *
+ * This is specifically scoped to mm-only. Users of this functionality (other
+ * than the core VMA implementation itself) should not include this header
+ * directly, but rather include internal.h.
  */
 #ifndef __MM_VMA_H
 #define __MM_VMA_H
@@ -230,11 +237,62 @@ static inline bool vmg_nomem(struct vma_merge_struct *vmg)
 	return vmg->state == VMA_MERGE_ERROR_NOMEM;
 }
 
-/* Assumes addr >= vma->vm_start. */
-static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma,
-				       unsigned long addr)
+static inline pgoff_t vmg_start_pgoff(const struct vma_merge_struct *vmg)
 {
-	return vma->vm_pgoff + PHYS_PFN(addr - vma->vm_start);
+	return vmg->pgoff;
+}
+
+static inline pgoff_t vmg_pages(const struct vma_merge_struct *vmg)
+{
+	const unsigned long size = vmg->end - vmg->start;
+
+	return size >> PAGE_SHIFT;
+}
+
+static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg)
+{
+	return vmg_start_pgoff(vmg) + vmg_pages(vmg);
+}
+
+static inline void assert_sane_pgoff(struct vm_area_struct *vma, pgoff_t pgoff)
+{
+	/* nommu doesn't set a virtual pgoff for anon VMAs. */
+	if (!IS_ENABLED(CONFIG_MMU))
+		return;
+	/*
+	 * File-backed VMAs have arbitrary page offset (either page offset into
+	 * file or for pfnmap the PFN of the start of the range or drivers may
+	 * set arbitrary page offset).
+	 */
+	if (!vma_is_anonymous(vma))
+		return;
+	/* MAP_PRIVATE-/dev/zero is anon, non-NULL vm_file, but has file pgoff. */
+	if (vma->vm_file)
+		return;
+	/* If faulted in, could have been remapped. */
+	if (vma->anon_vma)
+		return;
+	/* OK this is really an anon VMA - expect virtual page offset. */
+	VM_WARN_ON_ONCE(pgoff != vma->vm_start >> PAGE_SHIFT);
+}
+
+static inline void vma_set_pgoff(struct vm_area_struct *vma, pgoff_t pgoff)
+{
+	vma_assert_can_modify(vma);
+	assert_sane_pgoff(vma, pgoff);
+	vma->vm_pgoff = pgoff;
+}
+
+static inline void vma_add_pgoff(struct vm_area_struct *vma, pgoff_t delta)
+{
+	vma_assert_can_modify(vma);
+	vma_set_pgoff(vma, vma_start_pgoff(vma) + delta);
+}
+
+static inline void vma_sub_pgoff(struct vm_area_struct *vma, pgoff_t delta)
+{
+	vma_assert_can_modify(vma);
+	vma_set_pgoff(vma, vma_start_pgoff(vma) - delta);
 }
 
 #define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_)	\
@@ -258,7 +316,7 @@ static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma,
 		.start = start_,				\
 		.end = end_,					\
 		.vm_flags = vma_->vm_flags,			\
-		.pgoff = vma_pgoff_offset(vma_, start_),	\
+		.pgoff = linear_page_index(vma_, start_),	\
 		.file = vma_->vm_file,				\
 		.anon_vma = vma_->anon_vma,			\
 		.policy = vma_policy(vma_),			\
@@ -275,8 +333,7 @@ void validate_mm(struct mm_struct *mm);
 
 __must_check int vma_expand(struct vma_merge_struct *vmg);
 __must_check int vma_shrink(struct vma_iterator *vmi,
-		struct vm_area_struct *vma,
-		unsigned long start, unsigned long end, pgoff_t pgoff);
+		struct vm_area_struct *vma, unsigned long end);
 
 static inline int vma_iter_store_gfp(struct vma_iterator *vmi,
 			struct vm_area_struct *vma, gfp_t gfp)
@@ -310,7 +367,7 @@ static inline void compat_set_vma_from_desc(struct vm_area_struct *vma,
 	 */
 
 	/* Mutable fields. Populated with initial state. */
-	vma->vm_pgoff = desc->pgoff;
+	vma_set_pgoff(vma, desc->pgoff);
 	if (desc->vm_file != vma->vm_file)
 		vma_set_file(vma, desc->vm_file);
 	vma->flags = desc->vma_flags;
@@ -460,7 +517,7 @@ int mm_take_all_locks(struct mm_struct *mm);
 void mm_drop_all_locks(struct mm_struct *mm);
 
 unsigned long mmap_region(struct file *file, unsigned long addr,
-		unsigned long len, vm_flags_t vm_flags, unsigned long pgoff,
+		unsigned long len, vma_flags_t vma_flags, unsigned long pgoff,
 		struct list_head *uf);
 
 int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma,
@@ -484,9 +541,11 @@ static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma
 }
 
 #ifdef CONFIG_MMU
-static inline pgprot_t vm_pgprot_modify(pgprot_t oldprot, vm_flags_t vm_flags)
+static inline pgprot_t vma_pgprot_modify(pgprot_t oldprot, vma_flags_t vma_flags)
 {
-	return pgprot_modify(oldprot, vm_get_page_prot(vm_flags));
+	const pgprot_t prot = vma_flags_to_page_prot(vma_flags);
+
+	return pgprot_modify(oldprot, prot);
 }
 #endif
 
@@ -753,4 +812,9 @@ static inline bool map_deny_write_exec(const vma_flags_t *old,
 }
 #endif
 
+struct vm_area_struct *__install_special_mapping(struct mm_struct *mm,
+		unsigned long addr, unsigned long len,
+		vm_flags_t vm_flags, void *priv,
+		const struct vm_operations_struct *ops);
+
 #endif	/* __MM_VMA_H */
diff --git a/mm/vma_exec.c b/mm/vma_exec.c
index 5cee8b7efa0f2..7af1260689b97 100644
--- a/mm/vma_exec.c
+++ b/mm/vma_exec.c
@@ -1,10 +1,14 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
 /*
- * Functions explicitly implemented for exec functionality which however are
- * explicitly VMA-only logic.
+ * Functions provided for exec functionality which however are
+ * specifically VMA-only logic.
  */
 
+/*
+ * To allow for userland testing we place internal dependencies in
+ * vma_internal.h and external VMA API declarations in vma.h.
+ */
 #include "vma_internal.h"
 #include "vma.h"
 
@@ -37,7 +41,7 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift)
 	unsigned long new_end = old_end - shift;
 	VMA_ITERATOR(vmi, mm, new_start);
 	VMG_STATE(vmg, mm, &vmi, new_start, old_end, EMPTY_VMA_FLAGS,
-		  vma->vm_pgoff);
+		  vma_start_pgoff(vma));
 	struct vm_area_struct *next;
 	struct mmu_gather tlb;
 	PAGETABLE_MOVE(pmc, vma, vma, old_start, new_start, length);
@@ -89,7 +93,7 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift)
 
 	vma_prev(&vmi);
 	/* Shrink the vma to just the new range */
-	return vma_shrink(&vmi, vma, new_start, new_end, vma->vm_pgoff);
+	return vma_shrink(&vmi, vma, new_end);
 }
 
 /*
@@ -108,15 +112,17 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift)
 int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap,
 			  unsigned long *top_mem_p)
 {
-	unsigned long flags = VM_STACK_FLAGS | VM_STACK_INCOMPLETE_SETUP;
+	vma_flags_t flags = VMA_STACK_INCOMPLETE_SETUP;
+	struct vm_area_struct *vma;
 	int err;
-	struct vm_area_struct *vma = vm_area_alloc(mm);
 
+	/* VMA_STACK_FLAGS and VMA_STACK_INCOMPLETE_SETUP must not overlap. */
+	VM_WARN_ON_ONCE(vma_flags_test_any_mask(&flags, VMA_STACK_FLAGS));
+
+	vma = vm_area_alloc(mm);
 	if (!vma)
 		return -ENOMEM;
 
-	vma_set_anonymous(vma);
-
 	if (mmap_write_lock_killable(mm)) {
 		err = -EINTR;
 		goto err_free;
@@ -130,19 +136,21 @@ int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap,
 	if (err)
 		goto err_ksm;
 
+	vma_flags_set_mask(&flags, VMA_STACK_FLAGS);
+	vma_set_anonymous(vma);
+
 	/*
 	 * Place the stack at the largest stack address the architecture
 	 * supports. Later, we'll move this to an appropriate place. We don't
 	 * use STACK_TOP because that can depend on attributes which aren't
 	 * configured yet.
 	 */
-	VM_WARN_ON_ONCE(VM_STACK_FLAGS & VM_STACK_INCOMPLETE_SETUP);
 	vma->vm_end = STACK_TOP_MAX;
 	vma->vm_start = vma->vm_end - PAGE_SIZE;
 	if (pgtable_supports_soft_dirty())
-		flags |= VM_SOFTDIRTY;
-	vm_flags_init(vma, flags);
-	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+		vma_flags_set(&flags, VMA_SOFTDIRTY_BIT);
+	vma->flags = flags;
+	vma->vm_page_prot = vma_get_page_prot(vma);
 
 	err = insert_vm_struct(mm, vma);
 	if (err)
diff --git a/mm/vma_init.c b/mm/vma_init.c
index 3c0b65950510e..715feee283f0f 100644
--- a/mm/vma_init.c
+++ b/mm/vma_init.c
@@ -5,6 +5,10 @@
  * between CONFIG_MMU and non-CONFIG_MMU kernel configurations.
  */
 
+/*
+ * To allow for userland testing we place internal dependencies in
+ * vma_internal.h and external VMA API declarations in vma.h.
+ */
 #include "vma_internal.h"
 #include "vma.h"
 
@@ -46,7 +50,7 @@ static void vm_area_init_from(const struct vm_area_struct *src,
 	dest->vm_start = src->vm_start;
 	dest->vm_end = src->vm_end;
 	dest->anon_vma = src->anon_vma;
-	dest->vm_pgoff = src->vm_pgoff;
+	dest->vm_pgoff = vma_start_pgoff(src);
 	dest->vm_file = src->vm_file;
 	dest->vm_private_data = src->vm_private_data;
 	vm_flags_init(dest, src->vm_flags);
diff --git a/mm/vma_internal.h b/mm/vma_internal.h
index 2da6d224c1a85..4d300e7bbaf4c 100644
--- a/mm/vma_internal.h
+++ b/mm/vma_internal.h
@@ -2,8 +2,8 @@
 /*
  * vma_internal.h
  *
- * Headers required by vma.c, which can be substituted accordingly when testing
- * VMA functionality.
+ * Headers required by vma.c, vma_init.c and vma_exec.c, which can be
+ * substituted accordingly when testing VMA functionality.
  */
 
 #ifndef __MM_VMA_INTERNAL_H
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 1afca3568b9b6..deda558b2f49a 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -43,12 +43,14 @@
 #include <asm/tlbflush.h>
 #include <asm/shmparam.h>
 #include <linux/page_owner.h>
+#include <linux/cleanup.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/vmalloc.h>
 
 #include "internal.h"
 #include "pgalloc-track.h"
+#include "vmalloc.h"
 
 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
 static unsigned int __ro_after_init ioremap_max_page_shift = BITS_PER_LONG - 1;
@@ -91,6 +93,33 @@ struct vfree_deferred {
 static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred);
 
 /*** Page table manipulation functions ***/
+
+/*
+ * Try contiguous mappings at the PTE level for arches which support them, and if
+ * requested by the caller. Fall back to PAGE_SIZE mappings otherwise.
+ *
+ * Return: mapping size.
+ */
+static __always_inline unsigned long vmap_set_ptes(pte_t *pte,
+		unsigned long addr, unsigned long end, u64 pfn,
+		pgprot_t prot, unsigned int max_page_shift)
+{
+#ifdef CONFIG_HUGETLB_PAGE
+	unsigned long size;
+
+	size = arch_vmap_pte_range_map_size(addr, end, pfn, max_page_shift);
+	if (size != PAGE_SIZE) {
+		pte_t entry = pfn_pte(pfn, prot);
+
+		entry = arch_make_huge_pte(entry, ilog2(size), 0);
+		set_huge_pte_at(&init_mm, addr, pte, entry, size);
+		return size;
+	}
+#endif
+	set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot));
+	return PAGE_SIZE;
+}
+
 static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
 			phys_addr_t phys_addr, pgprot_t prot,
 			unsigned int max_page_shift, pgtbl_mod_mask *mask)
@@ -98,7 +127,8 @@ static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
 	pte_t *pte;
 	u64 pfn;
 	struct page *page;
-	unsigned long size = PAGE_SIZE;
+	unsigned long size;
+	unsigned int steps;
 
 	if (WARN_ON_ONCE(!PAGE_ALIGNED(end - addr)))
 		return -EINVAL;
@@ -119,20 +149,9 @@ static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
 			BUG();
 		}
 
-#ifdef CONFIG_HUGETLB_PAGE
-		size = arch_vmap_pte_range_map_size(addr, end, pfn, max_page_shift);
-		if (size != PAGE_SIZE) {
-			pte_t entry = pfn_pte(pfn, prot);
-
-			entry = arch_make_huge_pte(entry, ilog2(size), 0);
-			set_huge_pte_at(&init_mm, addr, pte, entry, size);
-			pfn += PFN_DOWN(size);
-			continue;
-		}
-#endif
-		set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot));
-		pfn++;
-	} while (pte += PFN_DOWN(size), addr += size, addr != end);
+		size = vmap_set_ptes(pte, addr, end, pfn, prot, max_page_shift);
+		steps = PFN_DOWN(size);
+	} while (pte += steps, pfn += steps, addr += size, addr != end);
 
 	lazy_mmu_mode_disable();
 	*mask |= PGTBL_PTE_MODIFIED;
@@ -158,10 +177,21 @@ static int vmap_try_huge_pmd(pmd_t *pmd, unsigned long addr, unsigned long end,
 	if (!IS_ALIGNED(phys_addr, PMD_SIZE))
 		return 0;
 
-	if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr))
-		return 0;
+	if (!pmd_present(*pmd))
+		return pmd_set_huge(pmd, phys_addr, prot);
 
-	return pmd_set_huge(pmd, phys_addr, prot);
+	/*
+	 * Acquire the mmap read lock to exclude ptdump, which walks
+	 * kernel page tables it does not own under the mmap write lock.
+	 *
+	 * Concurrent read lock holders are safe: each exclusively owns
+	 * the range it operates on and cannot reach this page table.
+	 */
+	scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) {
+		if (!pmd_free_pte_page(pmd, addr))
+			return 0;
+		return pmd_set_huge(pmd, phys_addr, prot);
+	}
 }
 
 static int vmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
@@ -210,10 +240,15 @@ static int vmap_try_huge_pud(pud_t *pud, unsigned long addr, unsigned long end,
 	if (!IS_ALIGNED(phys_addr, PUD_SIZE))
 		return 0;
 
-	if (pud_present(*pud) && !pud_free_pmd_page(pud, addr))
-		return 0;
+	if (!pud_present(*pud))
+		return pud_set_huge(pud, phys_addr, prot);
 
-	return pud_set_huge(pud, phys_addr, prot);
+	/* See comment in vmap_try_huge_pmd(). */
+	scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) {
+		if (!pud_free_pmd_page(pud, addr))
+			return 0;
+		return pud_set_huge(pud, phys_addr, prot);
+	}
 }
 
 static int vmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
@@ -262,10 +297,15 @@ static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end,
 	if (!IS_ALIGNED(phys_addr, P4D_SIZE))
 		return 0;
 
-	if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr))
-		return 0;
+	if (!p4d_present(*p4d))
+		return p4d_set_huge(p4d, phys_addr, prot);
 
-	return p4d_set_huge(p4d, phys_addr, prot);
+	/* See comment in vmap_try_huge_pmd(). */
+	scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) {
+		if (!p4d_free_pud_page(p4d, addr))
+			return 0;
+		return p4d_set_huge(p4d, phys_addr, prot);
+	}
 }
 
 static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
@@ -524,8 +564,10 @@ void vunmap_range(unsigned long addr, unsigned long end)
 
 static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr,
 		unsigned long end, pgprot_t prot, struct page **pages, int *nr,
-		pgtbl_mod_mask *mask)
+		pgtbl_mod_mask *mask, unsigned int shift)
 {
+	unsigned long pfn, size;
+	unsigned int steps;
 	int err = 0;
 	pte_t *pte;
 
@@ -556,9 +598,10 @@ static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr,
 			break;
 		}
 
-		set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
-		(*nr)++;
-	} while (pte++, addr += PAGE_SIZE, addr != end);
+		pfn = page_to_pfn(page);
+		size = vmap_set_ptes(pte, addr, end, pfn, prot, shift);
+		steps = PFN_DOWN(size);
+	} while (pte += steps, *nr += steps, addr += size, addr != end);
 
 	lazy_mmu_mode_disable();
 	*mask |= PGTBL_PTE_MODIFIED;
@@ -568,60 +611,90 @@ static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr,
 
 static int vmap_pages_pmd_range(pud_t *pud, unsigned long addr,
 		unsigned long end, pgprot_t prot, struct page **pages, int *nr,
-		pgtbl_mod_mask *mask)
+		pgtbl_mod_mask *mask, unsigned int shift)
 {
 	pmd_t *pmd;
 	unsigned long next;
+	int err;
 
 	pmd = pmd_alloc_track(&init_mm, pud, addr, mask);
 	if (!pmd)
 		return -ENOMEM;
 	do {
 		next = pmd_addr_end(addr, end);
-		if (vmap_pages_pte_range(pmd, addr, next, prot, pages, nr, mask))
-			return -ENOMEM;
+
+		if (shift >= PMD_SHIFT) {
+			struct page *page = pages[*nr];
+			phys_addr_t phys_addr;
+
+			if (WARN_ON(!page))
+				return -ENOMEM;
+			if (WARN_ON(!pfn_valid(page_to_pfn(page))))
+				return -EINVAL;
+
+			phys_addr = page_to_phys(page);
+
+			if (vmap_try_huge_pmd(pmd, addr, next, phys_addr, prot,
+						shift)) {
+				*mask |= PGTBL_PMD_MODIFIED;
+				*nr += 1 << (PMD_SHIFT - PAGE_SHIFT);
+				continue;
+			}
+		}
+
+		err = vmap_pages_pte_range(pmd, addr, next, prot, pages, nr, mask, shift);
+		if (err)
+			return err;
 	} while (pmd++, addr = next, addr != end);
 	return 0;
 }
 
 static int vmap_pages_pud_range(p4d_t *p4d, unsigned long addr,
 		unsigned long end, pgprot_t prot, struct page **pages, int *nr,
-		pgtbl_mod_mask *mask)
+		pgtbl_mod_mask *mask, unsigned int shift)
 {
 	pud_t *pud;
 	unsigned long next;
+	int err;
 
 	pud = pud_alloc_track(&init_mm, p4d, addr, mask);
 	if (!pud)
 		return -ENOMEM;
 	do {
 		next = pud_addr_end(addr, end);
-		if (vmap_pages_pmd_range(pud, addr, next, prot, pages, nr, mask))
-			return -ENOMEM;
+		err = vmap_pages_pmd_range(pud, addr, next, prot, pages, nr, mask, shift);
+		if (err)
+			return err;
 	} while (pud++, addr = next, addr != end);
 	return 0;
 }
 
 static int vmap_pages_p4d_range(pgd_t *pgd, unsigned long addr,
 		unsigned long end, pgprot_t prot, struct page **pages, int *nr,
-		pgtbl_mod_mask *mask)
+		pgtbl_mod_mask *mask, unsigned int shift)
 {
 	p4d_t *p4d;
 	unsigned long next;
+	int err;
 
 	p4d = p4d_alloc_track(&init_mm, pgd, addr, mask);
 	if (!p4d)
 		return -ENOMEM;
 	do {
 		next = p4d_addr_end(addr, end);
-		if (vmap_pages_pud_range(p4d, addr, next, prot, pages, nr, mask))
-			return -ENOMEM;
+		err = vmap_pages_pud_range(p4d, addr, next, prot, pages, nr, mask, shift);
+		if (err)
+			return err;
 	} while (p4d++, addr = next, addr != end);
 	return 0;
 }
 
-static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end,
-		pgprot_t prot, struct page **pages)
+/*
+ * It can take an array of pages which are not all contiguous, but it
+ * may have contiguous chunks, as hinted by @shift.
+ */
+static int vmap_pages_range_noflush_walk(unsigned long addr, unsigned long end,
+		pgprot_t prot, struct page **pages, unsigned int shift)
 {
 	unsigned long start = addr;
 	pgd_t *pgd;
@@ -636,7 +709,7 @@ static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end,
 		next = pgd_addr_end(addr, end);
 		if (pgd_bad(*pgd))
 			mask |= PGTBL_PGD_MODIFIED;
-		err = vmap_pages_p4d_range(pgd, addr, next, prot, pages, &nr, &mask);
+		err = vmap_pages_p4d_range(pgd, addr, next, prot, pages, &nr, &mask, shift);
 		if (err)
 			break;
 	} while (pgd++, addr = next, addr != end);
@@ -659,27 +732,12 @@ static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end,
 int __vmap_pages_range_noflush(unsigned long addr, unsigned long end,
 		pgprot_t prot, struct page **pages, unsigned int page_shift)
 {
-	unsigned int i, nr = (end - addr) >> PAGE_SHIFT;
-
 	WARN_ON(page_shift < PAGE_SHIFT);
 
-	if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMALLOC) ||
-			page_shift == PAGE_SHIFT)
-		return vmap_small_pages_range_noflush(addr, end, prot, pages);
-
-	for (i = 0; i < nr; i += 1U << (page_shift - PAGE_SHIFT)) {
-		int err;
-
-		err = vmap_range_noflush(addr, addr + (1UL << page_shift),
-					page_to_phys(pages[i]), prot,
-					page_shift);
-		if (err)
-			return err;
-
-		addr += 1UL << page_shift;
-	}
+	if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMALLOC))
+		page_shift = PAGE_SHIFT;
 
-	return 0;
+	return vmap_pages_range_noflush_walk(addr, end, prot, pages, page_shift);
 }
 
 int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
@@ -3279,6 +3337,14 @@ struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
 				  NUMA_NO_NODE, GFP_KERNEL, caller);
 }
 
+static struct vm_struct *__get_vm_area_node_aligned_caller(unsigned long size,
+		unsigned long align, unsigned long flags, const void *caller)
+{
+	return __get_vm_area_node(size, align, PAGE_SHIFT, flags,
+				  VMALLOC_START, VMALLOC_END,
+				  NUMA_NO_NODE, GFP_KERNEL, caller);
+}
+
 /**
  * find_vm_area - find a continuous kernel virtual area
  * @addr:	  base address
@@ -3519,6 +3585,116 @@ void vunmap(const void *addr)
 }
 EXPORT_SYMBOL(vunmap);
 
+static inline unsigned int vm_shift(pgprot_t prot, unsigned long size)
+{
+	if (arch_vmap_pmd_supported(prot) && size >= PMD_SIZE)
+		return PMD_SHIFT;
+
+	return arch_vmap_pte_supported_shift(size);
+}
+
+static inline int get_vmap_batch_order(struct page **pages,
+		pgprot_t prot, unsigned int max_steps, unsigned int idx)
+{
+	unsigned long pfn;
+	unsigned int nr_contig;
+	int order;
+
+	if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMAP))
+		return 0;
+
+	nr_contig = num_pages_contiguous(&pages[idx], max_steps);
+	if (nr_contig < 2)
+		return 0;
+
+	order = ilog2(nr_contig);
+	pfn = page_to_pfn(pages[idx]);
+
+	/* Limit order by pfn alignment */
+	if (pfn > 0)
+		order = min_t(int, order, __ffs(pfn));
+
+	if (vm_shift(prot, PAGE_SIZE << order) == PAGE_SHIFT)
+		return 0;
+
+	return order;
+}
+
+static int vmap_pages_range_batched(unsigned long addr, unsigned long end,
+		pgprot_t prot, struct page **pages)
+{
+	unsigned int count = (end - addr) >> PAGE_SHIFT;
+	unsigned int prev_shift = 0, idx = 0;
+	unsigned long map_addr = addr, batch_end = addr;
+	int err;
+
+	err = kmsan_vmap_pages_range_noflush(addr, end, prot, pages,
+					     PAGE_SHIFT, GFP_KERNEL);
+	if (err)
+		goto out;
+
+	for (unsigned int i = 0; i < count; ) {
+		unsigned int shift = PAGE_SHIFT +
+			get_vmap_batch_order(pages, prot, count - i, i);
+
+		if (!i)
+			prev_shift = shift;
+
+		if (shift != prev_shift) {
+			err = vmap_pages_range_noflush_walk(map_addr, batch_end,
+					prot, pages + idx, prev_shift);
+			if (err)
+				goto out;
+			prev_shift = shift;
+			map_addr = batch_end;
+			idx = i;
+		}
+
+		/*
+		 * Once small pages are encountered, the remaining pages
+		 * are likely small as well.
+		 */
+		if (shift == PAGE_SHIFT)
+			break;
+
+		batch_end += 1UL << shift;
+		i += 1U << (shift - PAGE_SHIFT);
+	}
+
+	/* Remaining */
+	if (map_addr < end)
+		err = vmap_pages_range_noflush_walk(map_addr, end,
+				prot, pages + idx, prev_shift);
+
+out:
+	flush_cache_vmap(addr, end);
+	return err;
+}
+
+static struct vm_struct *vmap_get_aligned_vm_area(unsigned long size,
+		unsigned long flags, pgprot_t prot, const void *caller)
+{
+	struct vm_struct *vm_area;
+	unsigned int shift;
+
+	if (arch_vmap_pmd_supported(prot) && size >= PMD_SIZE) {
+		vm_area = __get_vm_area_node_aligned_caller(size, PMD_SIZE,
+				flags, caller);
+		if (vm_area)
+			return vm_area;
+	}
+
+	shift = arch_vmap_pte_supported_shift(size);
+	if (shift > PAGE_SHIFT) {
+		vm_area = __get_vm_area_node_aligned_caller(size, 1UL << shift,
+				flags, caller);
+		if (vm_area)
+			return vm_area;
+	}
+
+	return __get_vm_area_node_aligned_caller(size, PAGE_SIZE, flags, caller);
+}
+
 /**
  * vmap - map an array of pages into virtually contiguous space
  * @pages: array of page pointers
@@ -3557,13 +3733,14 @@ void *vmap(struct page **pages, unsigned int count,
 		return NULL;
 
 	size = (unsigned long)count << PAGE_SHIFT;
-	area = get_vm_area_caller(size, flags, __builtin_return_address(0));
+	area = vmap_get_aligned_vm_area(size, flags, prot,
+				__builtin_return_address(0));
 	if (!area)
 		return NULL;
 
 	addr = (unsigned long)area->addr;
-	if (vmap_pages_range(addr, addr + size, pgprot_nx(prot),
-				pages, PAGE_SHIFT) < 0) {
+	if (vmap_pages_range_batched(addr, addr + size, pgprot_nx(prot),
+				pages) < 0) {
 		vunmap(area->addr);
 		return NULL;
 	}
@@ -4030,11 +4207,7 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align,
 		 * supporting them.
 		 */
 
-		if (arch_vmap_pmd_supported(prot) && size >= PMD_SIZE)
-			shift = PMD_SHIFT;
-		else
-			shift = arch_vmap_pte_supported_shift(size);
-
+		shift = vm_shift(prot, size);
 		align = max(original_align, 1UL << shift);
 	}
 
@@ -4045,8 +4218,8 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align,
 	if (!area) {
 		bool nofail = gfp_mask & __GFP_NOFAIL;
 		warn_alloc(gfp_mask, NULL,
-			"vmalloc error: size %lu, vm_struct allocation failed%s",
-			size, (nofail) ? ". Retrying." : "");
+			"vmalloc error: size %lu, align 0x%lx, vm_struct allocation failed%s",
+			size, align, (nofail) ? ". Retrying." : "");
 		if (nofail) {
 			schedule_timeout_uninterruptible(1);
 			goto again;
@@ -4946,16 +5119,17 @@ pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align)
  * @sizes: array containing size of each area
  * @nr_vms: the number of areas to allocate
  * @align: alignment, all entries in @offsets and @sizes must be aligned to this
+ * @gfp: allocation flags passed to the underlying memory allocator
  *
  * Returns: kmalloc'd vm_struct pointer array pointing to allocated
  *	    vm_structs on success, %NULL on failure
  *
  * Percpu allocator wants to use congruent vm areas so that it can
  * maintain the offsets among percpu areas.  This function allocates
- * congruent vmalloc areas for it with GFP_KERNEL.  These areas tend to
- * be scattered pretty far, distance between two areas easily going up
- * to gigabytes.  To avoid interacting with regular vmallocs, these
- * areas are allocated from top.
+ * congruent vmalloc areas for it. These areas tend to be scattered
+ * pretty far, distance between two areas easily going up to gigabytes.
+ * To avoid interacting with regular vmallocs, these areas are allocated
+ * from top.
  *
  * Despite its complicated look, this allocator is rather simple. It
  * does everything top-down and scans free blocks from the end looking
@@ -4966,7 +5140,7 @@ pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align)
  */
 struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
 				     const size_t *sizes, int nr_vms,
-				     size_t align)
+				     size_t align, gfp_t gfp)
 {
 	const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
 	const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
@@ -5004,14 +5178,14 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
 		return NULL;
 	}
 
-	vms = kzalloc_objs(vms[0], nr_vms);
-	vas = kzalloc_objs(vas[0], nr_vms);
+	vms = kzalloc_objs(vms[0], nr_vms, gfp);
+	vas = kzalloc_objs(vas[0], nr_vms, gfp);
 	if (!vas || !vms)
 		goto err_free2;
 
 	for (area = 0; area < nr_vms; area++) {
-		vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL);
-		vms[area] = kzalloc_obj(struct vm_struct);
+		vas[area] = kmem_cache_zalloc(vmap_area_cachep, gfp);
+		vms[area] = kzalloc_obj(struct vm_struct, gfp);
 		if (!vas[area] || !vms[area])
 			goto err_free;
 	}
@@ -5101,7 +5275,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
 
 	/* populate the kasan shadow space */
 	for (area = 0; area < nr_vms; area++) {
-		if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area], GFP_KERNEL))
+		if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area], gfp))
 			goto err_free_shadow;
 	}
 
@@ -5158,7 +5332,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
 				continue;
 
 			vas[area] = kmem_cache_zalloc(
-				vmap_area_cachep, GFP_KERNEL);
+				vmap_area_cachep, gfp);
 			if (!vas[area])
 				goto err_free;
 		}
@@ -5198,9 +5372,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
 		kfree(vms[area]);
 	}
 	spin_unlock(&free_vmap_area_lock);
-	kfree(vas);
-	kfree(vms);
-	return NULL;
+	goto err_free2;
 }
 
 /**
diff --git a/mm/vmalloc.h b/mm/vmalloc.h
new file mode 100644
index 0000000000000..dcfe30eaa80c1
--- /dev/null
+++ b/mm/vmalloc.h
@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * mm-internal APIs for vmalloc
+ */
+#ifndef __MM_VMALLOC_H
+#define __MM_VMALLOC_H
+
+#ifdef CONFIG_MMU
+void __init vmalloc_init(void);
+int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end,
+		pgprot_t prot, struct page **pages,
+		unsigned int page_shift, gfp_t gfp_mask);
+unsigned int get_vm_area_page_order(struct vm_struct *vm);
+#else
+static inline void vmalloc_init(void) {}
+
+static inline
+int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end,
+		pgprot_t prot, struct page **pages,
+		unsigned int page_shift, gfp_t gfp_mask)
+{
+	return -EINVAL;
+}
+
+static inline void vunmap_range_noflush(unsigned long start, unsigned long end)
+{
+}
+#endif
+
+struct vm_struct *__get_vm_area_node(unsigned long size,
+		unsigned long align, unsigned long shift,
+		unsigned long vm_flags, unsigned long start,
+		unsigned long end, int node, gfp_t gfp_mask,
+		const void *caller);
+
+void clear_vm_uninitialized_flag(struct vm_struct *vm);
+
+int __must_check __vmap_pages_range_noflush(unsigned long addr,
+		unsigned long end, pgprot_t prot,
+		struct page **pages, unsigned int page_shift);
+
+void vunmap_range_noflush(unsigned long start, unsigned long end);
+
+void __vunmap_range_noflush(unsigned long start, unsigned long end);
+
+#endif /* __MM_VMALLOC_H */
diff --git a/mm/vmpressure.c b/mm/vmpressure.c
index f053554e58264..9629240d77adc 100644
--- a/mm/vmpressure.c
+++ b/mm/vmpressure.c
@@ -7,16 +7,15 @@
  *
  * Based on ideas from Andrew Morton, David Rientjes, KOSAKI Motohiro,
  * Leonid Moiseichuk, Mel Gorman, Minchan Kim and Pekka Enberg.
+ *
+ * Tree-mode (cgroup v1 userspace eventfd) bookkeeping lives in
+ * mm/memcontrol-v1.c; this file holds the shared code and the in-kernel
+ * (tree=false) socket-pressure path that runs on cgroup v2.
  */
 
 #include <linux/cgroup.h>
-#include <linux/fs.h>
 #include <linux/log2.h>
-#include <linux/sched.h>
 #include <linux/mm.h>
-#include <linux/vmstat.h>
-#include <linux/eventfd.h>
-#include <linux/slab.h>
 #include <linux/swap.h>
 #include <linux/printk.h>
 #include <linux/vmpressure.h>
@@ -35,7 +34,7 @@
  * TODO: Make the window size depend on machine size, as we do for vmstat
  * thresholds. Currently we set it to 512 pages (2MB for 4KB pages).
  */
-static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16;
+const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16;
 
 /*
  * These thresholds are used when we account memory pressure through
@@ -46,68 +45,6 @@ static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16;
 static const unsigned int vmpressure_level_med = 60;
 static const unsigned int vmpressure_level_critical = 95;
 
-/*
- * When there are too little pages left to scan, vmpressure() may miss the
- * critical pressure as number of pages will be less than "window size".
- * However, in that case the vmscan priority will raise fast as the
- * reclaimer will try to scan LRUs more deeply.
- *
- * The vmscan logic considers these special priorities:
- *
- * prio == DEF_PRIORITY (12): reclaimer starts with that value
- * prio <= DEF_PRIORITY - 2 : kswapd becomes somewhat overwhelmed
- * prio == 0                : close to OOM, kernel scans every page in an lru
- *
- * Any value in this range is acceptable for this tunable (i.e. from 12 to
- * 0). Current value for the vmpressure_level_critical_prio is chosen
- * empirically, but the number, in essence, means that we consider
- * critical level when scanning depth is ~10% of the lru size (vmscan
- * scans 'lru_size >> prio' pages, so it is actually 12.5%, or one
- * eights).
- */
-static const unsigned int vmpressure_level_critical_prio = ilog2(100 / 10);
-
-static struct vmpressure *work_to_vmpressure(struct work_struct *work)
-{
-	return container_of(work, struct vmpressure, work);
-}
-
-static struct vmpressure *vmpressure_parent(struct vmpressure *vmpr)
-{
-	struct mem_cgroup *memcg = vmpressure_to_memcg(vmpr);
-
-	memcg = parent_mem_cgroup(memcg);
-	if (!memcg)
-		return NULL;
-	return memcg_to_vmpressure(memcg);
-}
-
-enum vmpressure_levels {
-	VMPRESSURE_LOW = 0,
-	VMPRESSURE_MEDIUM,
-	VMPRESSURE_CRITICAL,
-	VMPRESSURE_NUM_LEVELS,
-};
-
-enum vmpressure_modes {
-	VMPRESSURE_NO_PASSTHROUGH = 0,
-	VMPRESSURE_HIERARCHY,
-	VMPRESSURE_LOCAL,
-	VMPRESSURE_NUM_MODES,
-};
-
-static const char * const vmpressure_str_levels[] = {
-	[VMPRESSURE_LOW] = "low",
-	[VMPRESSURE_MEDIUM] = "medium",
-	[VMPRESSURE_CRITICAL] = "critical",
-};
-
-static const char * const vmpressure_str_modes[] = {
-	[VMPRESSURE_NO_PASSTHROUGH] = "default",
-	[VMPRESSURE_HIERARCHY] = "hierarchy",
-	[VMPRESSURE_LOCAL] = "local",
-};
-
 static enum vmpressure_levels vmpressure_level(unsigned long pressure)
 {
 	if (pressure >= vmpressure_level_critical)
@@ -117,8 +54,8 @@ static enum vmpressure_levels vmpressure_level(unsigned long pressure)
 	return VMPRESSURE_LOW;
 }
 
-static enum vmpressure_levels vmpressure_calc_level(unsigned long scanned,
-						    unsigned long reclaimed)
+enum vmpressure_levels vmpressure_calc_level(unsigned long scanned,
+					     unsigned long reclaimed)
 {
 	unsigned long scale = scanned + reclaimed;
 	unsigned long pressure = 0;
@@ -147,74 +84,6 @@ static enum vmpressure_levels vmpressure_calc_level(unsigned long scanned,
 	return vmpressure_level(pressure);
 }
 
-struct vmpressure_event {
-	struct eventfd_ctx *efd;
-	enum vmpressure_levels level;
-	enum vmpressure_modes mode;
-	struct list_head node;
-};
-
-static bool vmpressure_event(struct vmpressure *vmpr,
-			     const enum vmpressure_levels level,
-			     bool ancestor, bool signalled)
-{
-	struct vmpressure_event *ev;
-	bool ret = false;
-
-	mutex_lock(&vmpr->events_lock);
-	list_for_each_entry(ev, &vmpr->events, node) {
-		if (ancestor && ev->mode == VMPRESSURE_LOCAL)
-			continue;
-		if (signalled && ev->mode == VMPRESSURE_NO_PASSTHROUGH)
-			continue;
-		if (level < ev->level)
-			continue;
-		eventfd_signal(ev->efd);
-		ret = true;
-	}
-	mutex_unlock(&vmpr->events_lock);
-
-	return ret;
-}
-
-static void vmpressure_work_fn(struct work_struct *work)
-{
-	struct vmpressure *vmpr = work_to_vmpressure(work);
-	unsigned long scanned;
-	unsigned long reclaimed;
-	enum vmpressure_levels level;
-	bool ancestor = false;
-	bool signalled = false;
-
-	spin_lock(&vmpr->sr_lock);
-	/*
-	 * Several contexts might be calling vmpressure(), so it is
-	 * possible that the work was rescheduled again before the old
-	 * work context cleared the counters. In that case we will run
-	 * just after the old work returns, but then scanned might be zero
-	 * here. No need for any locks here since we don't care if
-	 * vmpr->reclaimed is in sync.
-	 */
-	scanned = vmpr->tree_scanned;
-	if (!scanned) {
-		spin_unlock(&vmpr->sr_lock);
-		return;
-	}
-
-	reclaimed = vmpr->tree_reclaimed;
-	vmpr->tree_scanned = 0;
-	vmpr->tree_reclaimed = 0;
-	spin_unlock(&vmpr->sr_lock);
-
-	level = vmpressure_calc_level(scanned, reclaimed);
-
-	do {
-		if (vmpressure_event(vmpr, level, ancestor, signalled))
-			signalled = true;
-		ancestor = true;
-	} while ((vmpr = vmpressure_parent(vmpr)));
-}
-
 /**
  * vmpressure() - Account memory pressure through scanned/reclaimed ratio
  * @gfp:	reclaimer's gfp mask
@@ -246,11 +115,12 @@ void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree,
 		return;
 
 	/*
-	 * The in-kernel users only care about the reclaim efficiency
-	 * for this @memcg rather than the whole subtree, and there
-	 * isn't and won't be any in-kernel user in a legacy cgroup.
+	 * Only two combinations have a consumer:
+	 *   cgroup v2 + tree=false -> in-kernel socket pressure
+	 *   cgroup v1 + tree=true  -> userspace eventfds (memory.pressure_level)
+	 * Skip the other two: nothing consumes the result.
 	 */
-	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !tree)
+	if (cgroup_subsys_on_dfl(memory_cgrp_subsys) == tree)
 		return;
 
 	vmpr = memcg_to_vmpressure(memcg);
@@ -281,14 +151,7 @@ void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree,
 		return;
 
 	if (tree) {
-		spin_lock(&vmpr->sr_lock);
-		scanned = vmpr->tree_scanned += scanned;
-		vmpr->tree_reclaimed += reclaimed;
-		spin_unlock(&vmpr->sr_lock);
-
-		if (scanned < vmpressure_win)
-			return;
-		schedule_work(&vmpr->work);
+		vmpressure_v1_account_tree(vmpr, scanned, reclaimed);
 	} else {
 		enum vmpressure_levels level;
 
@@ -330,134 +193,6 @@ void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree,
 	}
 }
 
-/**
- * vmpressure_prio() - Account memory pressure through reclaimer priority level
- * @gfp:	reclaimer's gfp mask
- * @memcg:	cgroup memory controller handle
- * @prio:	reclaimer's priority
- *
- * This function should be called from the reclaim path every time when
- * the vmscan's reclaiming priority (scanning depth) changes.
- *
- * This function does not return any value.
- */
-void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio)
-{
-	/*
-	 * We only use prio for accounting critical level. For more info
-	 * see comment for vmpressure_level_critical_prio variable above.
-	 */
-	if (prio > vmpressure_level_critical_prio)
-		return;
-
-	/*
-	 * OK, the prio is below the threshold, updating vmpressure
-	 * information before shrinker dives into long shrinking of long
-	 * range vmscan. Passing scanned = vmpressure_win, reclaimed = 0
-	 * to the vmpressure() basically means that we signal 'critical'
-	 * level.
-	 */
-	vmpressure(gfp, 0, memcg, true, vmpressure_win, 0);
-}
-
-#define MAX_VMPRESSURE_ARGS_LEN	(strlen("critical") + strlen("hierarchy") + 2)
-
-/**
- * vmpressure_register_event() - Bind vmpressure notifications to an eventfd
- * @memcg:	memcg that is interested in vmpressure notifications
- * @eventfd:	eventfd context to link notifications with
- * @args:	event arguments (pressure level threshold, optional mode)
- *
- * This function associates eventfd context with the vmpressure
- * infrastructure, so that the notifications will be delivered to the
- * @eventfd. The @args parameter is a comma-delimited string that denotes a
- * pressure level threshold (one of vmpressure_str_levels, i.e. "low", "medium",
- * or "critical") and an optional mode (one of vmpressure_str_modes, i.e.
- * "hierarchy" or "local").
- *
- * To be used as memcg event method.
- *
- * Return: 0 on success, -ENOMEM on memory failure or -EINVAL if @args could
- * not be parsed.
- */
-int vmpressure_register_event(struct mem_cgroup *memcg,
-			      struct eventfd_ctx *eventfd, const char *args)
-{
-	struct vmpressure *vmpr = memcg_to_vmpressure(memcg);
-	struct vmpressure_event *ev;
-	enum vmpressure_modes mode = VMPRESSURE_NO_PASSTHROUGH;
-	enum vmpressure_levels level;
-	char *spec, *spec_orig;
-	char *token;
-	int ret = 0;
-
-	spec_orig = spec = kstrndup(args, MAX_VMPRESSURE_ARGS_LEN, GFP_KERNEL);
-	if (!spec)
-		return -ENOMEM;
-
-	/* Find required level */
-	token = strsep(&spec, ",");
-	ret = match_string(vmpressure_str_levels, VMPRESSURE_NUM_LEVELS, token);
-	if (ret < 0)
-		goto out;
-	level = ret;
-
-	/* Find optional mode */
-	token = strsep(&spec, ",");
-	if (token) {
-		ret = match_string(vmpressure_str_modes, VMPRESSURE_NUM_MODES, token);
-		if (ret < 0)
-			goto out;
-		mode = ret;
-	}
-
-	ev = kzalloc_obj(*ev);
-	if (!ev) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	ev->efd = eventfd;
-	ev->level = level;
-	ev->mode = mode;
-
-	mutex_lock(&vmpr->events_lock);
-	list_add(&ev->node, &vmpr->events);
-	mutex_unlock(&vmpr->events_lock);
-	ret = 0;
-out:
-	kfree(spec_orig);
-	return ret;
-}
-
-/**
- * vmpressure_unregister_event() - Unbind eventfd from vmpressure
- * @memcg:	memcg handle
- * @eventfd:	eventfd context that was used to link vmpressure with the @cg
- *
- * This function does internal manipulations to detach the @eventfd from
- * the vmpressure notifications, and then frees internal resources
- * associated with the @eventfd (but the @eventfd itself is not freed).
- *
- * To be used as memcg event method.
- */
-void vmpressure_unregister_event(struct mem_cgroup *memcg,
-				 struct eventfd_ctx *eventfd)
-{
-	struct vmpressure *vmpr = memcg_to_vmpressure(memcg);
-	struct vmpressure_event *ev;
-
-	mutex_lock(&vmpr->events_lock);
-	list_for_each_entry(ev, &vmpr->events, node) {
-		if (ev->efd != eventfd)
-			continue;
-		list_del(&ev->node);
-		kfree(ev);
-		break;
-	}
-	mutex_unlock(&vmpr->events_lock);
-}
-
 /**
  * vmpressure_init() - Initialize vmpressure control structure
  * @vmpr:	Structure to be initialized
@@ -468,9 +203,7 @@ void vmpressure_unregister_event(struct mem_cgroup *memcg,
 void vmpressure_init(struct vmpressure *vmpr)
 {
 	spin_lock_init(&vmpr->sr_lock);
-	mutex_init(&vmpr->events_lock);
-	INIT_LIST_HEAD(&vmpr->events);
-	INIT_WORK(&vmpr->work, vmpressure_work_fn);
+	vmpressure_v1_init(vmpr);
 }
 
 /**
@@ -482,9 +215,5 @@ void vmpressure_init(struct vmpressure *vmpr)
  */
 void vmpressure_cleanup(struct vmpressure *vmpr)
 {
-	/*
-	 * Make sure there is no pending work before eventfd infrastructure
-	 * goes away.
-	 */
-	flush_work(&vmpr->work);
+	vmpressure_v1_cleanup(vmpr);
 }
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 35c3bb15ae96a..b7ef032e78a0b 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -66,6 +66,7 @@
 #include <linux/sched/sysctl.h>
 
 #include "internal.h"
+#include "page_alloc.h"
 #include "swap.h"
 
 #define CREATE_TRACE_POINTS
@@ -79,7 +80,7 @@ struct scan_control {
 	 * Nodemask of nodes allowed by the caller. If NULL, all nodes
 	 * are scanned.
 	 */
-	nodemask_t	*nodemask;
+	const nodemask_t *nodemask;
 
 	/*
 	 * The memory cgroup that hit its limit and as a result is the
@@ -615,8 +616,8 @@ typedef enum {
 /*
  * pageout is called by shrink_folio_list() for each dirty folio.
  */
-static pageout_t pageout(struct folio *folio, struct address_space *mapping,
-			 struct swap_iocb **plug, struct list_head *folio_list)
+static pageout_t pageout(struct swap_io_ctx *ctx, struct address_space *mapping,
+		struct folio *folio, struct list_head *folio_list)
 {
 	int res;
 
@@ -652,9 +653,9 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping,
 	 * the split out folios get added back to folio_list.
 	 */
 	if (shmem_mapping(mapping))
-		res = shmem_writeout(folio, plug, folio_list);
+		res = shmem_writeout(ctx, folio, folio_list);
 	else
-		res = swap_writeout(folio, plug);
+		res = swap_writeout(ctx, folio);
 
 	if (res < 0)
 		handle_write_error(mapping, folio, res);
@@ -823,7 +824,6 @@ void folio_putback_lru(struct folio *folio)
 
 enum folio_references {
 	FOLIOREF_RECLAIM,
-	FOLIOREF_RECLAIM_CLEAN,
 	FOLIOREF_KEEP,
 	FOLIOREF_ACTIVATE,
 };
@@ -920,10 +920,6 @@ static enum folio_references folio_check_references(struct folio *folio,
 		return FOLIOREF_KEEP;
 	}
 
-	/* Reclaim if clean, defer dirty folios to writeback */
-	if (referenced_folio && folio_is_file_lru(folio))
-		return FOLIOREF_RECLAIM_CLEAN;
-
 	return FOLIOREF_RECLAIM;
 }
 
@@ -1037,16 +1033,15 @@ static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask)
 {
 	if (gfp_mask & __GFP_FS)
 		return true;
-	if (!folio_test_swapcache(folio) || !(gfp_mask & __GFP_IO))
-		return false;
 	/*
-	 * We can "enter_fs" for swap-cache with only __GFP_IO
-	 * providing this isn't SWP_FS_OPS.
-	 * ->flags can be updated non-atomically,
-	 * but that will never affect SWP_FS_OPS, so the data_race
-	 * is safe.
+	 * We can "enter_fs" for swap-cache with only __GFP_IO unless backed by
+	 * a swapfile that requires GFP_NOFS I/O.
 	 */
-	return !data_race(folio_swap_flags(folio) & SWP_FS_OPS);
+	if (folio_test_swapcache(folio) && (gfp_mask & __GFP_IO) &&
+	    !(__swap_entry_to_info(folio->swap)->ops->flags &
+			SWAP_OPS_F_REQUIRE_NOFS))
+		return true;
+	return false;
 }
 
 /*
@@ -1063,7 +1058,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
 	unsigned int nr_reclaimed = 0, nr_demoted = 0;
 	unsigned int pgactivate = 0;
 	bool do_demote_pass;
-	struct swap_iocb *plug = NULL;
+	struct swap_io_ctx ctx = {};
 
 	folio_batch_init(&free_folios);
 	memset(stat, 0, sizeof(*stat));
@@ -1235,7 +1230,6 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
 			stat->nr_ref_keep += nr_pages;
 			goto keep_locked;
 		case FOLIOREF_RECLAIM:
-		case FOLIOREF_RECLAIM_CLEAN:
 			; /* try to reclaim the folio below */
 		}
 
@@ -1381,8 +1375,6 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
 				goto activate_locked;
 			}
 
-			if (references == FOLIOREF_RECLAIM_CLEAN)
-				goto keep_locked;
 			if (!may_enter_fs(folio, sc->gfp_mask))
 				goto keep_locked;
 			if (!sc->may_writepage)
@@ -1394,7 +1386,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
 			 * starts and then write it out here.
 			 */
 			try_to_unmap_flush_dirty();
-			switch (pageout(folio, mapping, &plug, folio_list)) {
+			switch (pageout(&ctx, mapping, folio, folio_list)) {
 			case PAGE_KEEP:
 				goto keep_locked;
 			case PAGE_ACTIVATE:
@@ -1584,8 +1576,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
 	list_splice(&ret_folios, folio_list);
 	count_vm_events(PGACTIVATE, pgactivate);
 
-	if (plug)
-		swap_write_unplug(plug);
+	swap_write_submit(&ctx);
 	return nr_reclaimed;
 }
 
@@ -3265,7 +3256,7 @@ static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio,
 static void reset_batch_size(struct lru_gen_mm_walk *walk)
 {
 	int gen, type, zone;
-	struct lruvec *lruvec = walk->lruvec;
+	struct lruvec *lruvec = lruvec_live_lock_irq(walk->lruvec);
 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
 
 	walk->batched = 0;
@@ -3285,6 +3276,8 @@ static void reset_batch_size(struct lru_gen_mm_walk *walk)
 			lru += LRU_ACTIVE;
 		__update_lru_size(lruvec, lru, zone, delta);
 	}
+
+	lruvec_unlock_irq(lruvec);
 }
 
 static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args)
@@ -3779,11 +3772,8 @@ static void walk_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk)
 			mmap_read_unlock(mm);
 		}
 
-		if (walk->batched) {
-			lruvec_lock_irq(lruvec);
+		if (walk->batched)
 			reset_batch_size(walk);
-			lruvec_unlock_irq(lruvec);
-		}
 
 		cond_resched();
 	} while (err == -EAGAIN);
@@ -4867,9 +4857,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 	walk = current->reclaim_state->mm_walk;
 	if (walk && walk->batched) {
 		walk->lruvec = lruvec;
-		lruvec_lock_irq(lruvec);
 		reset_batch_size(walk);
-		lruvec_unlock_irq(lruvec);
 	}
 
 	mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc),
@@ -6594,7 +6582,7 @@ static bool allow_direct_reclaim(pg_data_t *pgdat)
  * happens, the page allocator should not consider triggering the OOM killer.
  */
 static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
-					nodemask_t *nodemask)
+				    const nodemask_t *nodemask)
 {
 	struct zoneref *z;
 	struct zone *zone;
@@ -6674,7 +6662,7 @@ static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
 }
 
 unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
-				gfp_t gfp_mask, nodemask_t *nodemask)
+				gfp_t gfp_mask, const nodemask_t *nodemask)
 {
 	unsigned long nr_reclaimed;
 	struct scan_control sc = {
@@ -7654,7 +7642,6 @@ static int __init kswapd_init(void)
 {
 	int nid;
 
-	swap_setup();
 	for_each_node_state(nid, N_MEMORY)
  		kswapd_run(nid);
 	register_sysctl_init("vm", vmscan_sysctl_table);
@@ -7781,9 +7768,9 @@ static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask,
 	return sc->nr_reclaimed;
 }
 
-int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
+unsigned long node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
 {
-	int ret;
+	unsigned long ret;
 	/* Minimum pages needed in order to stay on node */
 	const unsigned long nr_pages = 1 << order;
 	struct scan_control sc = {
@@ -7810,13 +7797,13 @@ int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
 	if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages &&
 	    node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <=
 	    pgdat->min_slab_pages)
-		return NODE_RECLAIM_FULL;
+		return 0;
 
 	/*
 	 * Do not scan if the allocation should not be delayed.
 	 */
 	if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC))
-		return NODE_RECLAIM_NOSCAN;
+		return 0;
 
 	/*
 	 * Only run node reclaim on the local node or on nodes that do not
@@ -7825,15 +7812,15 @@ int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
 	 * as wide as possible.
 	 */
 	if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id())
-		return NODE_RECLAIM_NOSCAN;
+		return 0;
 
 	if (test_and_set_bit_lock(PGDAT_RECLAIM_LOCKED, &pgdat->flags))
-		return NODE_RECLAIM_NOSCAN;
+		return 0;
 
-	ret = __node_reclaim(pgdat, gfp_mask, nr_pages, &sc) >= nr_pages;
+	ret = __node_reclaim(pgdat, gfp_mask, nr_pages, &sc);
 	clear_bit_unlock(PGDAT_RECLAIM_LOCKED, &pgdat->flags);
 
-	if (ret)
+	if (ret >= nr_pages)
 		count_vm_event(PGSCAN_ZONE_RECLAIM_SUCCESS);
 	else
 		count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED);
@@ -7912,6 +7899,10 @@ int user_proactive_reclaim(char *buf,
 		if (signal_pending(current))
 			return -EINTR;
 
+		/* cgroup_rmdir() waits for us with cgroup_mutex held. */
+		if (memcg && memcg_is_dying(memcg))
+			return -EAGAIN;
+
 		/*
 		 * This is the final attempt, drain percpu lru caches in the
 		 * hope of introducing more evictable pages.
diff --git a/mm/vmstat.c b/mm/vmstat.c
index f534972f517de..4e26e5fd66662 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -30,6 +30,7 @@
 #include <linux/sched/isolation.h>
 
 #include "internal.h"
+#include "page_alloc.h"
 
 #ifdef CONFIG_PROC_FS
 #ifdef CONFIG_NUMA
@@ -1488,7 +1489,11 @@ const char * const vmstat_text[] = {
 #if THREAD_SIZE > 65536
 	[I(KSTACK_REST)]			= "kstack_rest",
 #endif
-#endif
+#endif /* CONFIG_DEBUG_STACK_USAGE */
+#ifdef CONFIG_SWAP
+	[I(NRSWPIN)]				= "nrswpin",
+	[I(NRSWPOUT)]				= "nrswpout",
+#endif /* CONFIG_SWAP */
 #undef I
 #endif /* CONFIG_VM_EVENT_COUNTERS */
 };
@@ -1568,7 +1573,7 @@ static void frag_show_print(struct seq_file *m, pg_data_t *pgdat,
 static int frag_show(struct seq_file *m, void *arg)
 {
 	pg_data_t *pgdat = (pg_data_t *)arg;
-	walk_zones_in_node(m, pgdat, true, false, frag_show_print);
+	walk_zones_in_node(m, pgdat, true, true, frag_show_print);
 	return 0;
 }
 
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 83f5820c45f93..3d566a3ee96f1 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -21,6 +21,10 @@
  *	pool->lock
  *	class->lock
  *	zspage->lock
+ *
+ * When ZS_OBJ_CLASS_BITS > 0, zs_free() skips pool->lock; it picks
+ * the size_class from obj's encoded class_idx and serializes against
+ * page migration via class->lock.
  */
 
 #include <linux/module.h>
@@ -67,8 +71,8 @@
 #define MAX_POSSIBLE_PHYSMEM_BITS MAX_PHYSMEM_BITS
 #else
 /*
- * If this definition of MAX_PHYSMEM_BITS is used, OBJ_INDEX_BITS will just
- * be PAGE_SHIFT
+ * If this definition of MAX_PHYSMEM_BITS is used, ZS_OBJ_PFN_SHIFT will
+ * just be PAGE_SHIFT
  */
 #define MAX_POSSIBLE_PHYSMEM_BITS BITS_PER_LONG
 #endif
@@ -88,8 +92,23 @@
 #define OBJ_TAG_BITS	1
 #define OBJ_TAG_MASK	OBJ_ALLOCATED_TAG
 
-#define OBJ_INDEX_BITS	(BITS_PER_LONG - _PFN_BITS)
-#define OBJ_INDEX_MASK	((_AC(1, UL) << OBJ_INDEX_BITS) - 1)
+/*
+ * obj is encoded as [PFN | class_idx | obj_idx] within an unsigned long:
+ *
+ *   |<-- _PFN_BITS -->|<-- ZS_OBJ_CLASS_BITS -->|<-- ZS_OBJ_IDX_BITS -->|
+ *   +-----------------+-------------------------+-----------------------+
+ *   |       PFN       |        class_idx        |        obj_idx        |
+ *   +-----------------+-------------------------+-----------------------+
+ *  MSB                ^                                                LSB
+ *                     |
+ *                     +-- ZS_OBJ_PFN_SHIFT
+ *
+ * Encoding class_idx into obj lets zs_free() locate the size_class
+ * without holding pool->lock; class_idx is invariant across page
+ * migration (only PFN changes), so a lockless read of the obj value
+ * always yields a valid class_idx.
+ */
+#define ZS_OBJ_PFN_SHIFT	(BITS_PER_LONG - _PFN_BITS)
 
 #define HUGE_BITS	1
 #define FULLNESS_BITS	4
@@ -98,9 +117,61 @@
 
 #define ZS_MAX_PAGES_PER_ZSPAGE	(_AC(CONFIG_ZSMALLOC_CHAIN_SIZE, UL))
 
+/*
+ * Bits to index a page within a zspage = ceil(log2(ZS_MAX_PAGES_PER_ZSPAGE)).
+ * Computed at preprocessor time, for use in #if below.  Kconfig
+ * restricts ZSMALLOC_CHAIN_SIZE to [4, 16].
+ */
+#if ZS_MAX_PAGES_PER_ZSPAGE <= 4
+#define ZS_PAGES_PER_ZSPAGE_BITS	2
+#elif ZS_MAX_PAGES_PER_ZSPAGE <= 8
+#define ZS_PAGES_PER_ZSPAGE_BITS	3
+#elif ZS_MAX_PAGES_PER_ZSPAGE <= 16
+#define ZS_PAGES_PER_ZSPAGE_BITS	4
+#else
+#error "ZSMALLOC_CHAIN_SIZE out of expected range [4,16]"
+#endif
+
+/*
+ * Bits to index an object within a single PAGE_SIZE at the smallest
+ * possible object size: log2(PAGE_SIZE / 32) = PAGE_SHIFT - 5.
+ * 32 is the hard floor of ZS_MIN_ALLOC_SIZE.
+ */
+#define ZS_OBJS_PER_PAGE_BITS	(PAGE_SHIFT - 5)
+
+/*
+ * Bits to index any object in the densest possible zspage.  Below this,
+ * ZS_MIN_ALLOC_SIZE is auto-raised by the MAX(32, ...) formula -- still
+ * correct, but objects are coarser.
+ */
+#define ZS_OBJS_PER_ZSPAGE_BITS \
+	(ZS_PAGES_PER_ZSPAGE_BITS + ZS_OBJS_PER_PAGE_BITS)
+
+/*
+ * Encode class_idx only when obj has spare bits; otherwise
+ * ZS_OBJ_CLASS_BITS folds to 0 (32-bit, or 64-bit UML/fallback).
+ */
+#if BITS_PER_LONG >= 64 && \
+	ZS_OBJ_PFN_SHIFT >= (CLASS_BITS + 1) + ZS_OBJS_PER_ZSPAGE_BITS
+#define ZS_OBJ_CLASS_BITS	(CLASS_BITS + 1)
+#else
+#define ZS_OBJ_CLASS_BITS	0
+#endif
+#define ZS_OBJ_CLASS_MASK	((_AC(1, UL) << ZS_OBJ_CLASS_BITS) - 1)
+
+#define ZS_OBJ_IDX_BITS		(ZS_OBJ_PFN_SHIFT - ZS_OBJ_CLASS_BITS)
+#define ZS_OBJ_IDX_MASK		((_AC(1, UL) << ZS_OBJ_IDX_BITS) - 1)
+
+/*
+ * Belt-and-suspenders: the #if above already guarantees this when
+ * class_idx is enabled.  Catches future tweaks that bypass it.
+ */
+static_assert(ZS_OBJ_IDX_BITS >= ZS_PAGES_PER_ZSPAGE_BITS,
+	      "zsmalloc: ZS_MIN_ALLOC_SIZE would exceed ZS_MAX_ALLOC_SIZE");
+
 /* ZS_MIN_ALLOC_SIZE must be multiple of ZS_ALIGN */
 #define ZS_MIN_ALLOC_SIZE \
-	MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> OBJ_INDEX_BITS))
+	MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> ZS_OBJ_IDX_BITS))
 /* each chunk includes extra space to keep handle */
 #define ZS_MAX_ALLOC_SIZE	PAGE_SIZE
 
@@ -396,10 +467,13 @@ static void cache_free_zspage(struct zspage *zspage)
 	kmem_cache_free(zspage_cachep, zspage);
 }
 
-/* class->lock(which owns the handle) synchronizes races */
+/*
+ * Pairs with READ_ONCE() in handle_to_obj(): zs_free() may read the
+ * handle locklessly, so prevent store tearing here.
+ */
 static void record_obj(unsigned long handle, unsigned long obj)
 {
-	*(unsigned long *)handle = obj;
+	WRITE_ONCE(*(unsigned long *)handle, obj);
 }
 
 static inline bool __maybe_unused is_first_zpdesc(struct zpdesc *zpdesc)
@@ -720,33 +794,36 @@ static struct zpdesc *get_next_zpdesc(struct zpdesc *zpdesc)
 static void obj_to_location(unsigned long obj, struct zpdesc **zpdesc,
 				unsigned int *obj_idx)
 {
-	*zpdesc = pfn_zpdesc(obj >> OBJ_INDEX_BITS);
-	*obj_idx = (obj & OBJ_INDEX_MASK);
+	*zpdesc = pfn_zpdesc(obj >> ZS_OBJ_PFN_SHIFT);
+	*obj_idx = (obj & ZS_OBJ_IDX_MASK);
 }
 
 static void obj_to_zpdesc(unsigned long obj, struct zpdesc **zpdesc)
 {
-	*zpdesc = pfn_zpdesc(obj >> OBJ_INDEX_BITS);
+	*zpdesc = pfn_zpdesc(obj >> ZS_OBJ_PFN_SHIFT);
 }
 
 /**
- * location_to_obj - get obj value encoded from (<zpdesc>, <obj_idx>)
+ * location_to_obj - encode (<zpdesc>, <obj_idx>, <class_idx>) into obj value
  * @zpdesc: zpdesc object resides in zspage
  * @obj_idx: object index
+ * @class_idx: size class index; ignored when ZS_OBJ_CLASS_BITS == 0
  */
-static unsigned long location_to_obj(struct zpdesc *zpdesc, unsigned int obj_idx)
+static unsigned long location_to_obj(struct zpdesc *zpdesc, unsigned int obj_idx,
+				     unsigned int class_idx)
 {
 	unsigned long obj;
 
-	obj = zpdesc_pfn(zpdesc) << OBJ_INDEX_BITS;
-	obj |= obj_idx & OBJ_INDEX_MASK;
+	obj  = zpdesc_pfn(zpdesc) << ZS_OBJ_PFN_SHIFT;
+	obj |= (unsigned long)(class_idx & ZS_OBJ_CLASS_MASK) << ZS_OBJ_IDX_BITS;
+	obj |= obj_idx & ZS_OBJ_IDX_MASK;
 
 	return obj;
 }
 
 static unsigned long handle_to_obj(unsigned long handle)
 {
-	return *(unsigned long *)handle;
+	return READ_ONCE(*(unsigned long *)handle);
 }
 
 static inline bool obj_allocated(struct zpdesc *zpdesc, void *obj,
@@ -800,13 +877,26 @@ static int trylock_zspage(struct zspage *zspage)
 	return 0;
 }
 
-static void __free_zspage(struct zs_pool *pool, struct size_class *class,
-				struct zspage *zspage)
+/*
+ * Three free helpers, kept apart here:
+ *
+ * __free_zspage_lockless(): bare core; walks zpdescs and returns pages
+ *   to the buddy allocator.  Caller owns all zpdesc locks and has
+ *   removed the zspage from its class list.  Used by zs_free() outside
+ *   class->lock so the buddy-side work does not stall the class.
+ *
+ * __free_zspage(): __free_zspage_lockless() + per-class accounting,
+ *   under class->lock.  Used by async_free_zspage(), the worker for
+ *   zspages whose trylock_zspage() failed.
+ *
+ * free_zspage(): full wrapper - trylock zpdescs, remove from class
+ *   list, call __free_zspage(); kicks deferred free on contention.
+ *   Used by compaction.
+ */
+static inline void __free_zspage_lockless(struct zspage *zspage)
 {
 	struct zpdesc *zpdesc, *next;
 
-	assert_spin_locked(&class->lock);
-
 	VM_BUG_ON(get_zspage_inuse(zspage));
 	VM_BUG_ON(zspage->fullness != ZS_INUSE_RATIO_0);
 
@@ -822,7 +912,13 @@ static void __free_zspage(struct zs_pool *pool, struct size_class *class,
 	} while (zpdesc != NULL);
 
 	cache_free_zspage(zspage);
+}
 
+static void __free_zspage(struct zs_pool *pool, struct size_class *class,
+			  struct zspage *zspage)
+{
+	assert_spin_locked(&class->lock);
+	__free_zspage_lockless(zspage);
 	class_stat_sub(class, ZS_OBJS_ALLOCATED, class->objs_per_zspage);
 	atomic_long_sub(class->pages_per_zspage, &pool->pages_allocated);
 }
@@ -1275,7 +1371,7 @@ static unsigned long obj_malloc(struct zs_pool *pool,
 	kunmap_local(vaddr);
 	mod_zspage_inuse(zspage, 1);
 
-	obj = location_to_obj(m_zpdesc, obj);
+	obj = location_to_obj(m_zpdesc, obj, zspage->class);
 	record_obj(handle, obj);
 
 	return obj;
@@ -1380,37 +1476,97 @@ static void obj_free(int class_size, unsigned long obj)
 	mod_zspage_inuse(zspage, -1);
 }
 
+#if (ZS_OBJ_CLASS_BITS > 0) || defined(CONFIG_COMPACTION)
+/* Folds to 0 when ZS_OBJ_CLASS_BITS == 0; no ifdef needed at callers. */
+static unsigned int obj_to_class_idx(unsigned long obj)
+{
+	return (obj >> ZS_OBJ_IDX_BITS) & ZS_OBJ_CLASS_MASK;
+}
+#endif
+
+/*
+ * Resolve @handle to its zspage / size_class and acquire class->lock.
+ *
+ * When class_idx is encoded in obj (ZS_OBJ_CLASS_BITS > 0), it is
+ * invariant under page migration, so the handle can be read locklessly
+ * to pick the size_class.  Once class->lock is held migration is
+ * blocked and the handle is re-read to obtain a stable PFN.
+ *
+ * Otherwise (32-bit, or 64-bit fallback paths like UML where the
+ * encoding is disabled), fall back to pool->lock for the lookup.
+ */
+#if ZS_OBJ_CLASS_BITS > 0
+static inline void obj_class_get_and_lock(struct zs_pool *pool, unsigned long handle,
+					 unsigned long *objp, struct zspage **zspagep,
+					 struct size_class **classp)
+	__acquires(&(*classp)->lock)
+{
+	struct zpdesc *f_zpdesc;
+	unsigned long obj;
+
+	obj = handle_to_obj(handle);
+	*classp = pool->size_class[obj_to_class_idx(obj)];
+	spin_lock(&(*classp)->lock);
+	/* Re-read under class->lock: PFN is now stable vs migration. */
+	obj = handle_to_obj(handle);
+	obj_to_zpdesc(obj, &f_zpdesc);
+	*zspagep = get_zspage(f_zpdesc);
+	*objp = obj;
+}
+#else
+static inline void obj_class_get_and_lock(struct zs_pool *pool, unsigned long handle,
+					 unsigned long *objp, struct zspage **zspagep,
+					 struct size_class **classp)
+	__acquires(&(*classp)->lock)
+{
+	struct zpdesc *f_zpdesc;
+	unsigned long obj;
+
+	read_lock(&pool->lock);
+	obj = handle_to_obj(handle);
+	obj_to_zpdesc(obj, &f_zpdesc);
+	*zspagep = get_zspage(f_zpdesc);
+	*classp = zspage_class(pool, *zspagep);
+	spin_lock(&(*classp)->lock);
+	read_unlock(&pool->lock);
+	*objp = obj;
+}
+#endif
+
 void zs_free(struct zs_pool *pool, unsigned long handle)
 {
 	struct zspage *zspage;
-	struct zpdesc *f_zpdesc;
 	unsigned long obj;
 	struct size_class *class;
 	int fullness;
+	struct zspage *zspage_to_free = NULL;
 
 	if (IS_ERR_OR_NULL((void *)handle))
 		return;
 
-	/*
-	 * The pool->lock protects the race with zpage's migration
-	 * so it's safe to get the page from handle.
-	 */
-	read_lock(&pool->lock);
-	obj = handle_to_obj(handle);
-	obj_to_zpdesc(obj, &f_zpdesc);
-	zspage = get_zspage(f_zpdesc);
-	class = zspage_class(pool, zspage);
-	spin_lock(&class->lock);
-	read_unlock(&pool->lock);
+	obj_class_get_and_lock(pool, handle, &obj, &zspage, &class);
 
 	class_stat_sub(class, ZS_OBJS_INUSE, 1);
 	obj_free(class->size, obj);
 
 	fullness = fix_fullness_group(class, zspage);
-	if (fullness == ZS_INUSE_RATIO_0)
-		free_zspage(pool, class, zspage);
+	if (fullness == ZS_INUSE_RATIO_0) {
+		if (trylock_zspage(zspage)) {
+			remove_zspage(class, zspage);
+			class_stat_sub(class, ZS_OBJS_ALLOCATED,
+				       class->objs_per_zspage);
+			zspage_to_free = zspage;
+		} else {
+			kick_deferred_free(pool);
+		}
+	}
 
 	spin_unlock(&class->lock);
+
+	if (zspage_to_free) {
+		__free_zspage_lockless(zspage_to_free);
+		atomic_long_sub(class->pages_per_zspage, &pool->pages_allocated);
+	}
 	cache_free_handle(handle);
 }
 EXPORT_SYMBOL_GPL(zs_free);
@@ -1643,9 +1799,6 @@ static void lock_zspage(struct zspage *zspage)
 	}
 	zspage_read_unlock(zspage);
 }
-#endif /* CONFIG_COMPACTION */
-
-#ifdef CONFIG_COMPACTION
 
 static void replace_sub_page(struct size_class *class, struct zspage *zspage,
 				struct zpdesc *newzpdesc, struct zpdesc *oldzpdesc)
@@ -1761,7 +1914,8 @@ static int zs_page_migrate(struct page *newpage, struct page *page,
 
 			old_obj = handle_to_obj(handle);
 			obj_to_location(old_obj, &dummy, &obj_idx);
-			new_obj = (unsigned long)location_to_obj(newzpdesc, obj_idx);
+			new_obj = location_to_obj(newzpdesc, obj_idx,
+						  obj_to_class_idx(old_obj));
 			record_obj(handle, new_obj);
 		}
 	}
diff --git a/mm/zswap.c b/mm/zswap.c
index 761cd699e0a3e..4e76a4a87cdcc 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -992,6 +992,7 @@ static int zswap_writeback_entry(struct zswap_entry *entry,
 	struct folio *folio;
 	struct mempolicy *mpol;
 	struct swap_info_struct *si;
+	struct swap_io_ctx ctx = {};
 	int ret = 0;
 
 	/* try to allocate swap cache folio */
@@ -1049,7 +1050,8 @@ static int zswap_writeback_entry(struct zswap_entry *entry,
 	folio_set_reclaim(folio);
 
 	/* start writeback */
-	__swap_writepage(folio, NULL);
+	__swap_writepage(&ctx, folio);
+	swap_write_submit(&ctx);
 
 out:
 	if (ret) {
@@ -1217,7 +1219,7 @@ static unsigned long zswap_shrinker_count(struct shrinker *shrinker,
 	 * Without memcg, use the zswap pool-wide metrics.
 	 */
 	if (!mem_cgroup_disabled()) {
-		mem_cgroup_flush_stats(memcg);
+		mem_cgroup_flush_stats_ratelimited(memcg);
 		nr_backing = memcg_page_state(memcg, MEMCG_ZSWAP_B) >> PAGE_SHIFT;
 		nr_stored = memcg_page_state(memcg, MEMCG_ZSWAPPED);
 	} else {
diff --git a/net/batman-adv/bat_iv_ogm.c b/net/batman-adv/bat_iv_ogm.c
index bb2f012b454ea..22622283f59b0 100644
--- a/net/batman-adv/bat_iv_ogm.c
+++ b/net/batman-adv/bat_iv_ogm.c
@@ -404,23 +404,14 @@ static void batadv_iv_ogm_send_to_if(struct batadv_forw_packet *forw_packet,
 /* send a batman ogm packet */
 static void batadv_iv_ogm_emit(struct batadv_forw_packet *forw_packet)
 {
-	struct net_device *mesh_iface;
-
 	if (!forw_packet->if_incoming) {
 		pr_err("Error - can't forward packet: incoming iface not specified\n");
 		return;
 	}
 
-	mesh_iface = forw_packet->if_incoming->mesh_iface;
-
 	if (WARN_ON(!forw_packet->if_outgoing))
 		return;
 
-	if (forw_packet->if_outgoing->mesh_iface != mesh_iface) {
-		pr_warn("%s: mesh interface switch for queued OGM\n", __func__);
-		return;
-	}
-
 	if (forw_packet->if_incoming->if_status != BATADV_IF_ACTIVE)
 		return;
 
@@ -910,8 +901,7 @@ static void batadv_iv_ogm_schedule_buff(struct batadv_hard_iface *hard_iface)
 
 static void batadv_iv_ogm_schedule(struct batadv_hard_iface *hard_iface)
 {
-	if (hard_iface->if_status == BATADV_IF_NOT_IN_USE ||
-	    hard_iface->if_status == BATADV_IF_TO_BE_REMOVED)
+	if (hard_iface->if_status == BATADV_IF_TO_BE_REMOVED)
 		return;
 
 	mutex_lock(&hard_iface->bat_iv.ogm_buff_mutex);
diff --git a/net/batman-adv/bat_v_elp.c b/net/batman-adv/bat_v_elp.c
index 4841f0f1a9b13..262e40040007c 100644
--- a/net/batman-adv/bat_v_elp.c
+++ b/net/batman-adv/bat_v_elp.c
@@ -90,12 +90,6 @@ static bool batadv_v_elp_get_throughput(struct batadv_hardif_neigh_node *neigh,
 	u32 throughput;
 	int ret;
 
-	/* don't query throughput when no longer associated with any
-	 * batman-adv interface
-	 */
-	if (!mesh_iface)
-		return false;
-
 	/* if the user specified a customised value for this interface, then
 	 * return it directly
 	 */
@@ -311,8 +305,7 @@ static void batadv_v_elp_periodic_work(struct work_struct *work)
 		goto out;
 
 	/* we are in the process of shutting this interface down */
-	if (hard_iface->if_status == BATADV_IF_NOT_IN_USE ||
-	    hard_iface->if_status == BATADV_IF_TO_BE_REMOVED)
+	if (hard_iface->if_status == BATADV_IF_TO_BE_REMOVED)
 		goto out;
 
 	/* the interface was enabled but may not be ready yet */
diff --git a/net/batman-adv/bat_v_ogm.c b/net/batman-adv/bat_v_ogm.c
index 037921aad35d5..e921d49f7eced 100644
--- a/net/batman-adv/bat_v_ogm.c
+++ b/net/batman-adv/bat_v_ogm.c
@@ -115,14 +115,14 @@ static void batadv_v_ogm_start_timer(struct batadv_priv *bat_priv)
 
 /**
  * batadv_v_ogm_send_to_if() - send a batman ogm using a given interface
- * @bat_priv: the bat priv with all the mesh interface information
  * @skb: the OGM to send
  * @hard_iface: the interface to use to send the OGM
  */
-static void batadv_v_ogm_send_to_if(struct batadv_priv *bat_priv,
-				    struct sk_buff *skb,
+static void batadv_v_ogm_send_to_if(struct sk_buff *skb,
 				    struct batadv_hard_iface *hard_iface)
 {
+	struct batadv_priv *bat_priv = netdev_priv(hard_iface->mesh_iface);
+
 	if (hard_iface->if_status != BATADV_IF_ACTIVE) {
 		kfree_skb(skb);
 		return;
@@ -189,7 +189,6 @@ static void batadv_v_ogm_aggr_list_free(struct batadv_hard_iface *hard_iface)
 
 /**
  * batadv_v_ogm_aggr_send() - flush & send aggregation queue
- * @bat_priv: the bat priv with all the mesh interface information
  * @hard_iface: the interface with the aggregation queue to flush
  *
  * Aggregates all OGMv2 packets currently in the aggregation queue into a
@@ -199,8 +198,7 @@ static void batadv_v_ogm_aggr_list_free(struct batadv_hard_iface *hard_iface)
  *
  * Caller needs to hold the hard_iface->bat_v.aggr_list.lock.
  */
-static void batadv_v_ogm_aggr_send(struct batadv_priv *bat_priv,
-				   struct batadv_hard_iface *hard_iface)
+static void batadv_v_ogm_aggr_send(struct batadv_hard_iface *hard_iface)
 {
 	unsigned int aggr_len = hard_iface->bat_v.aggr_len;
 	struct sk_buff *skb_aggr;
@@ -230,26 +228,21 @@ static void batadv_v_ogm_aggr_send(struct batadv_priv *bat_priv,
 		consume_skb(skb);
 	}
 
-	batadv_v_ogm_send_to_if(bat_priv, skb_aggr, hard_iface);
+	batadv_v_ogm_send_to_if(skb_aggr, hard_iface);
 }
 
 /**
  * batadv_v_ogm_queue_on_if() - queue a batman ogm on a given interface
- * @bat_priv: the bat priv with all the mesh interface information
  * @skb: the OGM to queue
  * @hard_iface: the interface to queue the OGM on
  */
-static void batadv_v_ogm_queue_on_if(struct batadv_priv *bat_priv,
-				     struct sk_buff *skb,
+static void batadv_v_ogm_queue_on_if(struct sk_buff *skb,
 				     struct batadv_hard_iface *hard_iface)
 {
-	if (hard_iface->mesh_iface != bat_priv->mesh_iface) {
-		kfree_skb(skb);
-		return;
-	}
+	struct batadv_priv *bat_priv = netdev_priv(hard_iface->mesh_iface);
 
 	if (!READ_ONCE(bat_priv->aggregated_ogms)) {
-		batadv_v_ogm_send_to_if(bat_priv, skb, hard_iface);
+		batadv_v_ogm_send_to_if(skb, hard_iface);
 		return;
 	}
 
@@ -260,7 +253,7 @@ static void batadv_v_ogm_queue_on_if(struct batadv_priv *bat_priv,
 	}
 
 	if (!batadv_v_ogm_queue_left(skb, hard_iface))
-		batadv_v_ogm_aggr_send(bat_priv, hard_iface);
+		batadv_v_ogm_aggr_send(hard_iface);
 
 	hard_iface->bat_v.aggr_len += batadv_v_ogm_len(skb);
 	__skb_queue_tail(&hard_iface->bat_v.aggr_list, skb);
@@ -357,7 +350,7 @@ static void batadv_v_ogm_send_meshif(struct batadv_priv *bat_priv)
 			break;
 		}
 
-		batadv_v_ogm_queue_on_if(bat_priv, skb_tmp, hard_iface);
+		batadv_v_ogm_queue_on_if(skb_tmp, hard_iface);
 		batadv_hardif_put(hard_iface);
 	}
 	rcu_read_unlock();
@@ -397,14 +390,12 @@ void batadv_v_ogm_aggr_work(struct work_struct *work)
 {
 	struct batadv_hard_iface_bat_v *batv;
 	struct batadv_hard_iface *hard_iface;
-	struct batadv_priv *bat_priv;
 
 	batv = container_of(work, struct batadv_hard_iface_bat_v, aggr_wq.work);
 	hard_iface = container_of(batv, struct batadv_hard_iface, bat_v);
-	bat_priv = netdev_priv(hard_iface->mesh_iface);
 
 	spin_lock_bh(&hard_iface->bat_v.aggr_list.lock);
-	batadv_v_ogm_aggr_send(bat_priv, hard_iface);
+	batadv_v_ogm_aggr_send(hard_iface);
 	spin_unlock_bh(&hard_iface->bat_v.aggr_list.lock);
 
 	batadv_v_ogm_start_queue_timer(hard_iface);
@@ -601,7 +592,7 @@ static void batadv_v_ogm_forward(struct batadv_priv *bat_priv,
 		   if_outgoing->net_dev->name, ntohl(ogm_forward->throughput),
 		   ogm_forward->ttl, if_incoming->net_dev->name);
 
-	batadv_v_ogm_queue_on_if(bat_priv, skb, if_outgoing);
+	batadv_v_ogm_queue_on_if(skb, if_outgoing);
 
 out:
 	batadv_orig_ifinfo_put(orig_ifinfo);
diff --git a/net/batman-adv/bridge_loop_avoidance.c b/net/batman-adv/bridge_loop_avoidance.c
index 5c73f6ba16cff..f9a1fadf8de9e 100644
--- a/net/batman-adv/bridge_loop_avoidance.c
+++ b/net/batman-adv/bridge_loop_avoidance.c
@@ -344,7 +344,6 @@ static void batadv_bla_send_claim(struct batadv_priv *bat_priv, const u8 *mac,
 	struct sk_buff *skb;
 	struct ethhdr *ethhdr;
 	struct batadv_hard_iface *primary_if;
-	struct net_device *mesh_iface;
 	u8 *hw_src;
 	struct batadv_bla_claim_dst local_claim_dest;
 	__be32 zeroip = 0;
@@ -357,14 +356,10 @@ static void batadv_bla_send_claim(struct batadv_priv *bat_priv, const u8 *mac,
 	       sizeof(local_claim_dest));
 	local_claim_dest.type = claimtype;
 
-	mesh_iface = READ_ONCE(primary_if->mesh_iface);
-	if (!mesh_iface)
-		goto out;
-
 	skb = arp_create(ARPOP_REPLY, ETH_P_ARP,
 			 /* IP DST: 0.0.0.0 */
 			 zeroip,
-			 mesh_iface,
+			 primary_if->mesh_iface,
 			 /* IP SRC: 0.0.0.0 */
 			 zeroip,
 			 /* Ethernet DST: Broadcast */
@@ -442,7 +437,7 @@ static void batadv_bla_send_claim(struct batadv_priv *bat_priv, const u8 *mac,
 	}
 
 	skb_reset_mac_header(skb);
-	skb->protocol = eth_type_trans(skb, mesh_iface);
+	skb->protocol = eth_type_trans(skb, primary_if->mesh_iface);
 	batadv_inc_counter(bat_priv, BATADV_CNT_RX);
 	batadv_add_counter(bat_priv, BATADV_CNT_RX_BYTES,
 			   skb->len + ETH_HLEN);
diff --git a/net/batman-adv/hard-interface.c b/net/batman-adv/hard-interface.c
index 03d01c20a9548..b6867576bbafa 100644
--- a/net/batman-adv/hard-interface.c
+++ b/net/batman-adv/hard-interface.c
@@ -63,6 +63,7 @@ void batadv_hardif_release(struct kref *ref)
 	struct batadv_hard_iface *hard_iface;
 
 	hard_iface = container_of(ref, struct batadv_hard_iface, refcount);
+	netdev_put(hard_iface->mesh_iface, &hard_iface->meshif_dev_tracker);
 	netdev_put(hard_iface->net_dev, &hard_iface->dev_tracker);
 
 	kfree_rcu(hard_iface, rcu);
@@ -75,21 +76,21 @@ void batadv_hardif_release(struct kref *ref)
  * Return: batadv_hard_iface of net_dev (with increased refcnt), NULL on errors
  */
 struct batadv_hard_iface *
-batadv_hardif_get_by_netdev(const struct net_device *net_dev)
+batadv_hardif_get_by_netdev(struct net_device *net_dev)
 {
 	struct batadv_hard_iface *hard_iface;
+	struct net_device *mesh_iface;
 
-	rcu_read_lock();
-	list_for_each_entry_rcu(hard_iface, &batadv_hardif_list, list) {
-		if (hard_iface->net_dev == net_dev &&
-		    kref_get_unless_zero(&hard_iface->refcount))
-			goto out;
-	}
+	ASSERT_RTNL();
 
-	hard_iface = NULL;
+	mesh_iface = netdev_master_upper_dev_get(net_dev);
+	if (!mesh_iface || !batadv_meshif_is_valid(mesh_iface))
+		return NULL;
+
+	hard_iface = netdev_lower_dev_get_private(mesh_iface, net_dev);
+	if (!kref_get_unless_zero(&hard_iface->refcount))
+		return NULL;
 
-out:
-	rcu_read_unlock();
 	return hard_iface;
 }
 
@@ -245,7 +246,7 @@ struct net_device *__batadv_get_real_netdev(struct net_device *netdev)
 	}
 
 	hard_iface = batadv_hardif_get_by_netdev(netdev);
-	if (!hard_iface || !hard_iface->mesh_iface)
+	if (!hard_iface)
 		goto out;
 
 	net = dev_net(hard_iface->mesh_iface);
@@ -539,16 +540,10 @@ static void batadv_check_known_mac_addr(const struct batadv_hard_iface *hard_ifa
 	const struct batadv_hard_iface *tmp_hard_iface;
 	struct list_head *iter;
 
-	if (!mesh_iface)
-		return;
-
 	netdev_for_each_lower_private(mesh_iface, tmp_hard_iface, iter) {
 		if (tmp_hard_iface == hard_iface)
 			continue;
 
-		if (tmp_hard_iface->if_status == BATADV_IF_NOT_IN_USE)
-			continue;
-
 		if (!batadv_compare_eth(tmp_hard_iface->net_dev->dev_addr,
 					hard_iface->net_dev->dev_addr))
 			continue;
@@ -574,9 +569,6 @@ static void batadv_hardif_recalc_extra_skbroom(struct net_device *mesh_iface)
 
 	rcu_read_lock();
 	netdev_for_each_lower_private_rcu(mesh_iface, hard_iface, iter) {
-		if (hard_iface->if_status == BATADV_IF_NOT_IN_USE)
-			continue;
-
 		lower_header_len = max_t(unsigned short, lower_header_len,
 					 hard_iface->net_dev->hard_header_len);
 
@@ -723,38 +715,63 @@ batadv_hardif_deactivate_interface(struct batadv_hard_iface *hard_iface)
 }
 
 /**
- * batadv_hardif_enable_interface() - Enslave hard interface to mesh interface
- * @hard_iface: hard interface to add to mesh interface
+ * batadv_hardif_enable_interface() - Enslave interface to mesh interface
+ * @net_dev: netdev struct of the interface to add to mesh interface
  * @mesh_iface: netdev struct of the mesh interface
  *
  * Return: 0 on success or negative error number in case of failure
  */
-int batadv_hardif_enable_interface(struct batadv_hard_iface *hard_iface,
+int batadv_hardif_enable_interface(struct net_device *net_dev,
 				   struct net_device *mesh_iface)
 {
 	struct batadv_priv *bat_priv;
 	__be16 ethertype = htons(ETH_P_BATMAN);
 	int max_header_len = batadv_max_header_len();
+	struct batadv_hard_iface *hard_iface;
 	unsigned int required_mtu;
 	unsigned int hardif_mtu;
 	bool fragmentation;
 	int ret;
 
-	hardif_mtu = READ_ONCE(hard_iface->net_dev->mtu);
+	ASSERT_RTNL();
+
+	if (!batadv_is_valid_iface(net_dev))
+		return -EINVAL;
+
+	hardif_mtu = READ_ONCE(net_dev->mtu);
 	required_mtu = READ_ONCE(mesh_iface->mtu) + max_header_len;
 
 	if (hardif_mtu < ETH_MIN_MTU + max_header_len)
 		return -EINVAL;
 
-	if (hard_iface->if_status != BATADV_IF_NOT_IN_USE)
-		goto out;
+	hard_iface = kzalloc_obj(*hard_iface, GFP_ATOMIC);
+	if (!hard_iface)
+		return -ENOMEM;
 
-	kref_get(&hard_iface->refcount);
+	netdev_hold(net_dev, &hard_iface->dev_tracker, GFP_ATOMIC);
+	hard_iface->net_dev = net_dev;
+
+	hard_iface->if_status = BATADV_IF_INACTIVE;
+
+	INIT_HLIST_HEAD(&hard_iface->neigh_list);
+
+	mutex_init(&hard_iface->bat_iv.ogm_buff_mutex);
+	spin_lock_init(&hard_iface->neigh_list_lock);
+	kref_init(&hard_iface->refcount);
+
+	hard_iface->num_bcasts = BATADV_NUM_BCASTS_DEFAULT;
+	if (batadv_is_wifi_hardif(hard_iface))
+		hard_iface->num_bcasts = BATADV_NUM_BCASTS_WIRELESS;
+
+	WRITE_ONCE(hard_iface->hop_penalty, 0);
+
+	batadv_v_hardif_init(hard_iface);
 
 	netdev_hold(mesh_iface, &hard_iface->meshif_dev_tracker, GFP_ATOMIC);
 	hard_iface->mesh_iface = mesh_iface;
 	bat_priv = netdev_priv(hard_iface->mesh_iface);
 
+	bat_priv->hardif_generation++;
 	ret = netdev_master_upper_dev_link(hard_iface->net_dev,
 					   mesh_iface, hard_iface, NULL, NULL);
 	if (ret)
@@ -764,9 +781,6 @@ int batadv_hardif_enable_interface(struct batadv_hard_iface *hard_iface,
 	if (ret < 0)
 		goto err_upper;
 
-	hard_iface->if_status = BATADV_IF_INACTIVE;
-
-	kref_get(&hard_iface->refcount);
 	hard_iface->batman_adv_ptype.type = ethertype;
 	hard_iface->batman_adv_ptype.func = batadv_batman_skb_recv;
 	hard_iface->batman_adv_ptype.dev = hard_iface->net_dev;
@@ -802,14 +816,11 @@ int batadv_hardif_enable_interface(struct batadv_hard_iface *hard_iface,
 	if (bat_priv->algo_ops->iface.enabled)
 		bat_priv->algo_ops->iface.enabled(hard_iface);
 
-out:
 	return 0;
 
 err_upper:
 	netdev_upper_dev_unlink(hard_iface->net_dev, mesh_iface);
 err_dev:
-	hard_iface->mesh_iface = NULL;
-	netdev_put(mesh_iface, &hard_iface->meshif_dev_tracker);
 	batadv_hardif_put(hard_iface);
 	return ret;
 }
@@ -821,17 +832,18 @@ int batadv_hardif_enable_interface(struct batadv_hard_iface *hard_iface,
 void batadv_hardif_disable_interface(struct batadv_hard_iface *hard_iface)
 {
 	struct batadv_priv *bat_priv = netdev_priv(hard_iface->mesh_iface);
-	struct batadv_hard_iface *primary_if = NULL;
+	struct batadv_hard_iface *primary_if;
+
+	ASSERT_RTNL();
 
 	batadv_hardif_deactivate_interface(hard_iface);
 
 	if (hard_iface->if_status != BATADV_IF_INACTIVE)
-		goto out;
+		return;
 
 	batadv_info(hard_iface->mesh_iface, "Removing interface: %s\n",
 		    hard_iface->net_dev->name);
 	dev_remove_pack(&hard_iface->batman_adv_ptype);
-	batadv_hardif_put(hard_iface);
 
 	primary_if = batadv_primary_if_get_selected(bat_priv);
 	if (hard_iface == primary_if) {
@@ -842,15 +854,16 @@ void batadv_hardif_disable_interface(struct batadv_hard_iface *hard_iface)
 
 		batadv_hardif_put(new_if);
 	}
+	batadv_hardif_put(primary_if);
 
 	bat_priv->algo_ops->iface.disable(hard_iface);
-	hard_iface->if_status = BATADV_IF_NOT_IN_USE;
+	hard_iface->if_status = BATADV_IF_TO_BE_REMOVED;
 
 	/* delete all references to this hard_iface */
 	batadv_purge_orig_ref(bat_priv);
 	batadv_purge_outstanding_packets(bat_priv, hard_iface);
-	netdev_put(hard_iface->mesh_iface, &hard_iface->meshif_dev_tracker);
 
+	bat_priv->hardif_generation++;
 	netdev_upper_dev_unlink(hard_iface->net_dev, hard_iface->mesh_iface);
 	batadv_hardif_recalc_extra_skbroom(hard_iface->mesh_iface);
 
@@ -858,67 +871,6 @@ void batadv_hardif_disable_interface(struct batadv_hard_iface *hard_iface)
 	if (list_empty(&hard_iface->mesh_iface->adj_list.lower))
 		batadv_gw_check_client_stop(bat_priv);
 
-	hard_iface->mesh_iface = NULL;
-	batadv_hardif_put(hard_iface);
-
-out:
-	batadv_hardif_put(primary_if);
-}
-
-static struct batadv_hard_iface *
-batadv_hardif_add_interface(struct net_device *net_dev)
-{
-	struct batadv_hard_iface *hard_iface;
-
-	ASSERT_RTNL();
-
-	if (!batadv_is_valid_iface(net_dev))
-		return NULL;
-
-	hard_iface = kzalloc_obj(*hard_iface, GFP_ATOMIC);
-	if (!hard_iface)
-		return NULL;
-
-	netdev_hold(net_dev, &hard_iface->dev_tracker, GFP_ATOMIC);
-	hard_iface->net_dev = net_dev;
-
-	hard_iface->mesh_iface = NULL;
-	hard_iface->if_status = BATADV_IF_NOT_IN_USE;
-
-	INIT_LIST_HEAD(&hard_iface->list);
-	INIT_HLIST_HEAD(&hard_iface->neigh_list);
-
-	mutex_init(&hard_iface->bat_iv.ogm_buff_mutex);
-	spin_lock_init(&hard_iface->neigh_list_lock);
-	kref_init(&hard_iface->refcount);
-
-	hard_iface->num_bcasts = BATADV_NUM_BCASTS_DEFAULT;
-	if (batadv_is_wifi_hardif(hard_iface))
-		hard_iface->num_bcasts = BATADV_NUM_BCASTS_WIRELESS;
-
-	WRITE_ONCE(hard_iface->hop_penalty, 0);
-
-	batadv_v_hardif_init(hard_iface);
-
-	kref_get(&hard_iface->refcount);
-	list_add_tail_rcu(&hard_iface->list, &batadv_hardif_list);
-	batadv_hardif_generation++;
-
-	return hard_iface;
-}
-
-static void batadv_hardif_remove_interface(struct batadv_hard_iface *hard_iface)
-{
-	ASSERT_RTNL();
-
-	/* first deactivate interface */
-	if (hard_iface->if_status != BATADV_IF_NOT_IN_USE)
-		batadv_hardif_disable_interface(hard_iface);
-
-	if (hard_iface->if_status != BATADV_IF_NOT_IN_USE)
-		return;
-
-	hard_iface->if_status = BATADV_IF_TO_BE_REMOVED;
 	batadv_hardif_put(hard_iface);
 }
 
@@ -1082,10 +1034,6 @@ static int batadv_hard_if_event(struct notifier_block *this,
 	batadv_wifi_net_device_event(event, net_dev);
 
 	hard_iface = batadv_hardif_get_by_netdev(net_dev);
-	if (!hard_iface && (event == NETDEV_REGISTER ||
-			    event == NETDEV_POST_TYPE_CHANGE))
-		hard_iface = batadv_hardif_add_interface(net_dev);
-
 	if (!hard_iface)
 		goto out;
 
@@ -1099,19 +1047,12 @@ static int batadv_hard_if_event(struct notifier_block *this,
 		break;
 	case NETDEV_UNREGISTER:
 	case NETDEV_PRE_TYPE_CHANGE:
-		list_del_rcu(&hard_iface->list);
-		batadv_hardif_generation++;
-
-		batadv_hardif_remove_interface(hard_iface);
+		batadv_hardif_disable_interface(hard_iface);
 		break;
 	case NETDEV_CHANGEMTU:
-		if (hard_iface->mesh_iface)
-			batadv_update_min_mtu(hard_iface->mesh_iface);
+		batadv_update_min_mtu(hard_iface->mesh_iface);
 		break;
 	case NETDEV_CHANGEADDR:
-		if (hard_iface->if_status == BATADV_IF_NOT_IN_USE)
-			goto hardif_put;
-
 		batadv_check_known_mac_addr(hard_iface);
 
 		bat_priv = netdev_priv(hard_iface->mesh_iface);
diff --git a/net/batman-adv/hard-interface.h b/net/batman-adv/hard-interface.h
index af31696c39780..935f47ca9a48f 100644
--- a/net/batman-adv/hard-interface.h
+++ b/net/batman-adv/hard-interface.h
@@ -21,12 +21,6 @@
  * enum batadv_hard_if_state - State of a hard interface
  */
 enum batadv_hard_if_state {
-	/**
-	 * @BATADV_IF_NOT_IN_USE: interface is not used as slave interface of a
-	 * batman-adv mesh interface
-	 */
-	BATADV_IF_NOT_IN_USE,
-
 	/**
 	 * @BATADV_IF_TO_BE_REMOVED: interface will be removed from mesh
 	 * interface
@@ -74,8 +68,8 @@ u32 batadv_netdev_get_wifi_flags(struct net_device *net_dev);
 u32 batadv_hardif_get_wifi_flags(struct batadv_hard_iface *hard_iface);
 bool batadv_is_wifi_hardif(struct batadv_hard_iface *hard_iface);
 struct batadv_hard_iface*
-batadv_hardif_get_by_netdev(const struct net_device *net_dev);
-int batadv_hardif_enable_interface(struct batadv_hard_iface *hard_iface,
+batadv_hardif_get_by_netdev(struct net_device *net_dev);
+int batadv_hardif_enable_interface(struct net_device *net_dev,
 				   struct net_device *mesh_iface);
 void batadv_hardif_disable_interface(struct batadv_hard_iface *hard_iface);
 int batadv_hardif_min_mtu(struct net_device *mesh_iface);
diff --git a/net/batman-adv/main.c b/net/batman-adv/main.c
index 67bed3ee77e7e..73becb0549488 100644
--- a/net/batman-adv/main.c
+++ b/net/batman-adv/main.c
@@ -59,11 +59,6 @@
 #include "tp_meter.h"
 #include "translation-table.h"
 
-/* List manipulations on hardif_list have to be rtnl_lock()'ed,
- * list traversals just rcu-locked
- */
-struct list_head batadv_hardif_list;
-unsigned int batadv_hardif_generation;
 static int (*batadv_rx_handler[256])(struct sk_buff *skb,
 				     struct batadv_hard_iface *recv_if);
 
@@ -95,7 +90,6 @@ static int __init batadv_init(void)
 	if (ret < 0)
 		return ret;
 
-	INIT_LIST_HEAD(&batadv_hardif_list);
 	batadv_algo_init();
 
 	batadv_recv_handler_init();
@@ -458,9 +452,6 @@ int batadv_batman_skb_recv(struct sk_buff *skb, struct net_device *dev,
 	if (unlikely(skb->mac_len != ETH_HLEN || !skb_mac_header(skb)))
 		goto err_free;
 
-	if (!hard_iface->mesh_iface)
-		goto err_free;
-
 	bat_priv = netdev_priv(hard_iface->mesh_iface);
 
 	if (READ_ONCE(bat_priv->mesh_state) != BATADV_MESH_ACTIVE)
diff --git a/net/batman-adv/main.h b/net/batman-adv/main.h
index f68fc8b7239cd..e738758ee4a7a 100644
--- a/net/batman-adv/main.h
+++ b/net/batman-adv/main.h
@@ -226,9 +226,6 @@ static inline int batadv_print_vid(unsigned short vid)
 		return -1;
 }
 
-extern struct list_head batadv_hardif_list;
-extern unsigned int batadv_hardif_generation;
-
 extern struct workqueue_struct *batadv_event_workqueue;
 
 int batadv_mesh_init(struct net_device *mesh_iface);
diff --git a/net/batman-adv/mesh-interface.c b/net/batman-adv/mesh-interface.c
index fbfd99268de47..1dc6cfae9e9ae 100644
--- a/net/batman-adv/mesh-interface.c
+++ b/net/batman-adv/mesh-interface.c
@@ -840,18 +840,7 @@ static int batadv_meshif_slave_add(struct net_device *dev,
 				   struct net_device *slave_dev,
 				   struct netlink_ext_ack *extack)
 {
-	struct batadv_hard_iface *hard_iface;
-	int ret = -EINVAL;
-
-	hard_iface = batadv_hardif_get_by_netdev(slave_dev);
-	if (!hard_iface || hard_iface->mesh_iface)
-		goto out;
-
-	ret = batadv_hardif_enable_interface(hard_iface, dev);
-
-out:
-	batadv_hardif_put(hard_iface);
-	return ret;
+	return batadv_hardif_enable_interface(slave_dev, dev);
 }
 
 /**
diff --git a/net/batman-adv/netlink.c b/net/batman-adv/netlink.c
index 4cf9e3c54ad3b..d2bc48c707143 100644
--- a/net/batman-adv/netlink.c
+++ b/net/batman-adv/netlink.c
@@ -968,7 +968,7 @@ batadv_netlink_dump_hardif(struct sk_buff *msg, struct netlink_callback *cb)
 	bat_priv = netdev_priv(mesh_iface);
 
 	rtnl_lock();
-	cb->seq = batadv_hardif_generation << 1 | 1;
+	cb->seq = bat_priv->hardif_generation << 1 | 1;
 
 	netdev_for_each_lower_private(mesh_iface, hard_iface, iter) {
 		if (i++ < skip)
@@ -1211,7 +1211,9 @@ batadv_netlink_get_hardif_from_ifindex(struct batadv_priv *bat_priv,
 	if (!hard_dev)
 		return ERR_PTR(-ENODEV);
 
+	rtnl_lock();
 	hard_iface = batadv_hardif_get_by_netdev(hard_dev);
+	rtnl_unlock();
 	if (!hard_iface)
 		goto err_put_harddev;
 
diff --git a/net/batman-adv/originator.c b/net/batman-adv/originator.c
index 9b38bd9e8da7e..48f837cf665a2 100644
--- a/net/batman-adv/originator.c
+++ b/net/batman-adv/originator.c
@@ -1033,7 +1033,6 @@ batadv_purge_neigh_ifinfo(struct batadv_priv *bat_priv,
 
 		/* don't purge if the interface is not (going) down */
 		if (if_outgoing->if_status != BATADV_IF_INACTIVE &&
-		    if_outgoing->if_status != BATADV_IF_NOT_IN_USE &&
 		    if_outgoing->if_status != BATADV_IF_TO_BE_REMOVED)
 			continue;
 
@@ -1077,7 +1076,6 @@ batadv_purge_orig_ifinfo(struct batadv_priv *bat_priv,
 
 		/* don't purge if the interface is not (going) down */
 		if (if_outgoing->if_status != BATADV_IF_INACTIVE &&
-		    if_outgoing->if_status != BATADV_IF_NOT_IN_USE &&
 		    if_outgoing->if_status != BATADV_IF_TO_BE_REMOVED)
 			continue;
 
@@ -1127,10 +1125,8 @@ batadv_purge_orig_neighbors(struct batadv_priv *bat_priv,
 
 		if (batadv_has_timed_out(last_seen, BATADV_PURGE_TIMEOUT) ||
 		    if_incoming->if_status == BATADV_IF_INACTIVE ||
-		    if_incoming->if_status == BATADV_IF_NOT_IN_USE ||
 		    if_incoming->if_status == BATADV_IF_TO_BE_REMOVED) {
 			if (if_incoming->if_status == BATADV_IF_INACTIVE ||
-			    if_incoming->if_status == BATADV_IF_NOT_IN_USE ||
 			    if_incoming->if_status == BATADV_IF_TO_BE_REMOVED)
 				batadv_dbg(BATADV_DBG_BATMAN, bat_priv,
 					   "neighbor purge: originator %pM, neighbor: %pM, iface: %s\n",
diff --git a/net/batman-adv/tp_meter.c b/net/batman-adv/tp_meter.c
index c2eea7dbc4883..00467aa79de9d 100644
--- a/net/batman-adv/tp_meter.c
+++ b/net/batman-adv/tp_meter.c
@@ -358,28 +358,16 @@ batadv_tp_list_find_sender_session(struct batadv_priv *bat_priv, const u8 *dst,
 }
 
 /**
- * batadv_tp_vars_common_release() - release batadv_tp_vars_common from lists
+ * batadv_tp_sender_release() - release batadv_tp_sender
  *  and queue for free after rcu grace period
- * @ref: kref pointer of the batadv_tp_vars_common
+ * @ref: kref pointer of the batadv_tp_sender
  */
-static void batadv_tp_vars_common_release(struct kref *ref)
+static void batadv_tp_sender_release(struct kref *ref)
 {
-	struct batadv_tp_vars_common *tp_vars;
-	struct batadv_tp_unacked *un, *safe;
-
-	tp_vars = container_of(ref, struct batadv_tp_vars_common, refcount);
-
-	/* lock should not be needed because this object is now out of any
-	 * context!
-	 */
-	spin_lock_bh(&tp_vars->unacked_lock);
-	list_for_each_entry_safe(un, safe, &tp_vars->unacked_list, list) {
-		list_del(&un->list);
-		kfree(un);
-	}
-	spin_unlock_bh(&tp_vars->unacked_lock);
+	struct batadv_tp_sender *tp_vars;
 
-	kfree_rcu(tp_vars, rcu);
+	tp_vars = container_of(ref, struct batadv_tp_sender, common.refcount);
+	kfree_rcu(tp_vars, common.rcu);
 }
 
 /**
@@ -392,7 +380,7 @@ static void batadv_tp_sender_put(struct batadv_tp_sender *tp_vars)
 	if (!tp_vars)
 		return;
 
-	kref_put(&tp_vars->common.refcount, batadv_tp_vars_common_release);
+	kref_put(&tp_vars->common.refcount, batadv_tp_sender_release);
 }
 
 /**
@@ -1145,9 +1133,6 @@ void batadv_tp_start(struct batadv_priv *bat_priv, const u8 *dst,
 	init_waitqueue_head(&tp_vars->more_bytes);
 	init_completion(&tp_vars->finished);
 
-	spin_lock_init(&tp_vars->common.unacked_lock);
-	INIT_LIST_HEAD(&tp_vars->common.unacked_list);
-
 	spin_lock_init(&tp_vars->cc_lock);
 
 	tp_vars->prerandom_offset = 0;
@@ -1251,6 +1236,33 @@ batadv_tp_list_find_receiver_session(struct batadv_priv *bat_priv, const u8 *dst
 	return tp_vars;
 }
 
+/**
+ * batadv_tp_receiver_release() - release batadv_tp_receiver
+ *  and queue for free after rcu grace period
+ * @ref: kref pointer of the batadv_tp_receiver
+ */
+static void batadv_tp_receiver_release(struct kref *ref)
+{
+	struct batadv_tp_receiver *tp_vars;
+	struct batadv_tp_unacked *safe;
+	struct batadv_tp_unacked *un;
+
+	tp_vars = container_of(ref, struct batadv_tp_receiver, common.refcount);
+
+	/* lock should not be needed because this object is now out of any
+	 * context!
+	 */
+	spin_lock_bh(&tp_vars->ack_seqno_lock);
+	list_for_each_entry_safe(un, safe, &tp_vars->unacked_list, list) {
+		list_del(&un->list);
+		kfree(un);
+		tp_vars->unacked_count--;
+	}
+	spin_unlock_bh(&tp_vars->ack_seqno_lock);
+
+	kfree_rcu(tp_vars, common.rcu);
+}
+
 /**
  * batadv_tp_receiver_put() - decrement the batadv_tp_receiver
  *  refcounter and possibly release it
@@ -1261,7 +1273,7 @@ static void batadv_tp_receiver_put(struct batadv_tp_receiver *tp_vars)
 	if (!tp_vars)
 		return;
 
-	kref_put(&tp_vars->common.refcount, batadv_tp_vars_common_release);
+	kref_put(&tp_vars->common.refcount, batadv_tp_receiver_release);
 }
 
 /**
@@ -1304,13 +1316,13 @@ static void batadv_tp_receiver_shutdown(struct timer_list *t)
 	if (batadv_tp_list_detach(&tp_vars->common))
 		batadv_tp_receiver_put(tp_vars);
 
-	spin_lock_bh(&tp_vars->common.unacked_lock);
-	list_for_each_entry_safe(un, safe, &tp_vars->common.unacked_list, list) {
+	spin_lock_bh(&tp_vars->ack_seqno_lock);
+	list_for_each_entry_safe(un, safe, &tp_vars->unacked_list, list) {
 		list_del(&un->list);
 		kfree(un);
-		tp_vars->common.unacked_count--;
+		tp_vars->unacked_count--;
 	}
-	spin_unlock_bh(&tp_vars->common.unacked_lock);
+	spin_unlock_bh(&tp_vars->ack_seqno_lock);
 
 	/* drop reference of timer */
 	if (WARN_ON(atomic_xchg(&tp_vars->receiving, 0) != 1))
@@ -1403,69 +1415,106 @@ static int batadv_tp_send_ack(struct batadv_priv *bat_priv, const u8 *dst,
  */
 static bool batadv_tp_handle_out_of_order(struct batadv_tp_receiver *tp_vars,
 					  u32 seqno, u32 payload_len)
-	__must_hold(&tp_vars->common.unacked_lock)
+	__must_hold(&tp_vars->ack_seqno_lock)
 {
-	struct batadv_tp_unacked *un, *new;
-	bool added = false;
-
-	new = kmalloc_obj(*new, GFP_ATOMIC);
-	if (unlikely(!new))
-		return false;
-
-	new->seqno = seqno;
-	new->len = payload_len;
-
-	/* if the list is empty immediately attach this new object */
-	if (list_empty(&tp_vars->common.unacked_list)) {
-		list_add(&new->list, &tp_vars->common.unacked_list);
-		tp_vars->common.unacked_count++;
-		return true;
-	}
-
-	/* otherwise loop over the list and either drop the packet because this
-	 * is a duplicate or store it at the right position.
+	struct list_head *pos = &tp_vars->unacked_list;
+	struct batadv_tp_unacked *new = NULL;
+	u32 end_seqno = seqno + payload_len;
+	struct batadv_tp_unacked *safe;
+	struct batadv_tp_unacked *un;
+
+	/* loop over the list to find either an existing entry which the new
+	 * seqno range can be merged with or the position at which a new entry
+	 * has to be inserted.
 	 *
 	 * The iteration is done in the reverse way because it is likely that
 	 * the last received packet (the one being processed now) has a bigger
 	 * seqno than all the others already stored.
 	 */
-	list_for_each_entry_reverse(un, &tp_vars->common.unacked_list, list) {
-		/* check for duplicates */
-		if (new->seqno == un->seqno) {
-			if (new->len > un->len)
-				un->len = new->len;
-			kfree(new);
-			added = true;
-			break;
-		}
-
-		/* look for the right position */
-		if (batadv_seq_before(new->seqno, un->seqno))
+	list_for_each_entry_reverse(un, &tp_vars->unacked_list, list) {
+		/* look for the right position - an un which is smaller */
+		if (batadv_seq_before(seqno, un->seqno))
 			continue;
 
-		/* as soon as an entry having a bigger seqno is found, the new
-		 * one is attached _after_ it. In this way the list is kept in
-		 * ascending order
+		/* smaller/equal seqno was found but they might be directly
+		 * after another or overlapping. keep only a single entry
+		 *
+		 * It is already known that:
+		 *
+		 *	un->seqno <= seqno
+		 *
+		 * When establishing that:
+		 *
+		 *	seqno <= un->seqno + un->len
+		 *
+		 * Then it is not necessary to add a new entry because the
+		 * smaller/equal seqno of un might already contain the new
+		 * received packet or we only add new data directly after
+		 * the end of un. The latter can be identified using:
+		 *
+		 *	un->seqno + un->len <= end_seqno
 		 */
-		list_add(&new->list, &un->list);
-		added = true;
-		tp_vars->common.unacked_count++;
+		if (!batadv_seq_before(un->seqno + un->len, seqno)) {
+			/* new data directly after un? */
+			if (!batadv_seq_before(end_seqno, un->seqno + un->len))
+				un->len = end_seqno - un->seqno;
+
+			/* un now represents both old un + new range and has to
+			 * be used to check if the gap to the next seqno range
+			 * was closed
+			 */
+			new = un;
+		} else {
+			/* as soon as an entry having a smaller seqno is found,
+			 * the new one is attached _after_ it. In this way the
+			 * list is kept in ascending order
+			 */
+			pos = &un->list;
+		}
+
 		break;
 	}
 
-	/* received packet with smallest seqno out of order; add it to front */
-	if (!added) {
-		list_add(&new->list, &tp_vars->common.unacked_list);
-		tp_vars->common.unacked_count++;
+	/* no entry to merge with was found; insert a new one after the entry
+	 * with the next smaller seqno (or at the front of the list when the
+	 * new seqno is the smallest or the list is empty)
+	 */
+	if (!new) {
+		new = kmalloc_obj(*new, GFP_ATOMIC);
+		if (unlikely(!new))
+			return false;
+
+		new->seqno = seqno;
+		new->len = payload_len;
+
+		list_add(&new->list, pos);
+		tp_vars->unacked_count++;
+	}
+
+	/* check if new filled the gap to the next list entries */
+	un = new;
+	list_for_each_entry_safe_continue(un, safe, &tp_vars->unacked_list, list) {
+		if (batadv_seq_before(end_seqno, un->seqno))
+			break;
+
+		/* next entry is overlapping or adjacent - combine both */
+		if (batadv_seq_before(end_seqno, un->seqno + un->len)) {
+			end_seqno = un->seqno + un->len;
+			new->len = end_seqno - new->seqno;
+		}
+
+		list_del(&un->list);
+		kfree(un);
+		tp_vars->unacked_count--;
 	}
 
 	/* remove the last (biggest) unacked seqno when list is too large */
-	if (tp_vars->common.unacked_count > BATADV_TP_MAX_UNACKED) {
-		un = list_last_entry(&tp_vars->common.unacked_list,
+	if (tp_vars->unacked_count > BATADV_TP_MAX_UNACKED) {
+		un = list_last_entry(&tp_vars->unacked_list,
 				     struct batadv_tp_unacked, list);
 		list_del(&un->list);
 		kfree(un);
-		tp_vars->common.unacked_count--;
+		tp_vars->unacked_count--;
 	}
 
 	return true;
@@ -1477,7 +1526,7 @@ static bool batadv_tp_handle_out_of_order(struct batadv_tp_receiver *tp_vars,
  * @tp_vars: the private data of the current TP meter session
  */
 static void batadv_tp_ack_unordered(struct batadv_tp_receiver *tp_vars)
-	__must_hold(&tp_vars->common.unacked_lock)
+	__must_hold(&tp_vars->ack_seqno_lock)
 {
 	struct batadv_tp_unacked *un, *safe;
 	u32 to_ack;
@@ -1485,7 +1534,7 @@ static void batadv_tp_ack_unordered(struct batadv_tp_receiver *tp_vars)
 	/* go through the unacked packet list and possibly ACK them as
 	 * well
 	 */
-	list_for_each_entry_safe(un, safe, &tp_vars->common.unacked_list, list) {
+	list_for_each_entry_safe(un, safe, &tp_vars->unacked_list, list) {
 		/* the list is ordered, therefore it is possible to stop as soon
 		 * there is a gap between the last acked seqno and the seqno of
 		 * the packet under inspection
@@ -1493,14 +1542,14 @@ static void batadv_tp_ack_unordered(struct batadv_tp_receiver *tp_vars)
 		if (batadv_seq_before(tp_vars->last_recv, un->seqno))
 			break;
 
-		to_ack = un->seqno + un->len - tp_vars->last_recv;
+		to_ack = un->seqno + un->len;
 
-		if (batadv_seq_before(tp_vars->last_recv, un->seqno + un->len))
-			tp_vars->last_recv += to_ack;
+		if (batadv_seq_before(tp_vars->last_recv, to_ack))
+			tp_vars->last_recv = to_ack;
 
 		list_del(&un->list);
 		kfree(un);
-		tp_vars->common.unacked_count--;
+		tp_vars->unacked_count--;
 	}
 }
 
@@ -1547,9 +1596,9 @@ batadv_tp_init_recv(struct batadv_priv *bat_priv,
 	tp_vars->common.bat_priv = bat_priv;
 	kref_init(&tp_vars->common.refcount);
 
-	spin_lock_init(&tp_vars->common.unacked_lock);
-	INIT_LIST_HEAD(&tp_vars->common.unacked_list);
-	tp_vars->common.unacked_count = 0;
+	spin_lock_init(&tp_vars->ack_seqno_lock);
+	INIT_LIST_HEAD(&tp_vars->unacked_list);
+	tp_vars->unacked_count = 0;
 
 	kref_get(&tp_vars->common.refcount);
 	timer_setup(&tp_vars->common.timer, batadv_tp_receiver_shutdown, 0);
@@ -1609,7 +1658,7 @@ static void batadv_tp_recv_msg(struct batadv_priv *bat_priv,
 		WRITE_ONCE(tp_vars->last_recv_time, jiffies);
 	}
 
-	spin_lock_bh(&tp_vars->common.unacked_lock);
+	spin_lock_bh(&tp_vars->ack_seqno_lock);
 
 	/* if the packet is a duplicate, it may be the case that an ACK has been
 	 * lost. Resend the ACK
@@ -1625,7 +1674,7 @@ static void batadv_tp_recv_msg(struct batadv_priv *bat_priv,
 		 * not been enqueued correctly
 		 */
 		if (!batadv_tp_handle_out_of_order(tp_vars, seqno, payload_len)) {
-			spin_unlock_bh(&tp_vars->common.unacked_lock);
+			spin_unlock_bh(&tp_vars->ack_seqno_lock);
 			goto out;
 		}
 
@@ -1641,7 +1690,7 @@ static void batadv_tp_recv_msg(struct batadv_priv *bat_priv,
 
 send_ack:
 	to_ack = tp_vars->last_recv;
-	spin_unlock_bh(&tp_vars->common.unacked_lock);
+	spin_unlock_bh(&tp_vars->ack_seqno_lock);
 
 	/* send the ACK. If the received packet was out of order, the ACK that
 	 * is going to be sent is a duplicate (the sender will count them and
diff --git a/net/batman-adv/tvlv.c b/net/batman-adv/tvlv.c
index 1c9fb21985f6a..49bf2ed9ecdc3 100644
--- a/net/batman-adv/tvlv.c
+++ b/net/batman-adv/tvlv.c
@@ -442,6 +442,54 @@ static int batadv_tvlv_call_handler(struct batadv_priv *bat_priv,
 	return NET_RX_SUCCESS;
 }
 
+/**
+ * batadv_tvlv_hdr_next() - move a tvlv buffer cursor to the next container
+ * @tvlv_value: cursor into the tvlv buffer, advanced past the returned
+ *  container's content on success
+ * @tvlv_value_len: remaining length of the tvlv buffer, reduced by the returned
+ *  container's size on success
+ *
+ * Parses a single container header at the current cursor position and, if a
+ * complete container is available, advances the cursor and remaining length
+ * past it. The returned header stays valid; its content is located at
+ * (returned header + 1) and is ntohs(hdr->len) bytes long.
+ *
+ * Return: pointer to the next tvlv container header, or NULL if no further
+ * complete container is present in the buffer.
+ */
+static struct batadv_tvlv_hdr *batadv_tvlv_hdr_next(void **tvlv_value, u16 *tvlv_value_len)
+{
+	struct batadv_tvlv_hdr *tvlv_hdr;
+	u16 tvlv_value_cont_len;
+	void *tvlv_value_cont;
+	u16 tvlv_len;
+
+	tvlv_value_cont = *tvlv_value;
+	tvlv_len = *tvlv_value_len;
+
+	if (tvlv_len < sizeof(*tvlv_hdr))
+		return NULL;
+
+	tvlv_hdr = tvlv_value_cont;
+	tvlv_value_cont_len = ntohs(tvlv_hdr->len);
+	tvlv_value_cont = tvlv_hdr + 1;
+	tvlv_len -= sizeof(*tvlv_hdr);
+
+	if (tvlv_value_cont_len > tvlv_len)
+		return NULL;
+
+	/* the next tvlv header is accessed assuming (at least) 2-byte
+	 * alignment, so it must start at an even offset.
+	 */
+	if (tvlv_value_cont_len & 1)
+		return NULL;
+
+	*tvlv_value = (u8 *)tvlv_value_cont + tvlv_value_cont_len;
+	*tvlv_value_len = tvlv_len - tvlv_value_cont_len;
+
+	return tvlv_hdr;
+}
+
 /**
  * batadv_tvlv_containers_contain() - check if a tvlv buffer holds a container
  * @tvlv_value: tvlv content
@@ -457,28 +505,10 @@ static bool batadv_tvlv_containers_contain(void *tvlv_value,
 					   u8 version)
 {
 	struct batadv_tvlv_hdr *tvlv_hdr;
-	u16 tvlv_value_cont_len;
-
-	while (tvlv_value_len >= sizeof(*tvlv_hdr)) {
-		tvlv_hdr = tvlv_value;
-		tvlv_value_cont_len = ntohs(tvlv_hdr->len);
-		tvlv_value = tvlv_hdr + 1;
-		tvlv_value_len -= sizeof(*tvlv_hdr);
-
-		if (tvlv_value_cont_len > tvlv_value_len)
-			break;
-
-		/* the next tvlv header is accessed assuming (at least) 2-byte
-		 * alignment, so it must start at an even offset.
-		 */
-		if (tvlv_value_cont_len & 1)
-			break;
 
+	while ((tvlv_hdr = batadv_tvlv_hdr_next(&tvlv_value, &tvlv_value_len))) {
 		if (tvlv_hdr->type == type && tvlv_hdr->version == version)
 			return true;
-
-		tvlv_value = (u8 *)tvlv_value + tvlv_value_cont_len;
-		tvlv_value_len -= tvlv_value_cont_len;
 	}
 
 	return false;
@@ -511,20 +541,8 @@ int batadv_tvlv_containers_process(struct batadv_priv *bat_priv,
 	u8 cifnotfound = BATADV_TVLV_HANDLER_OGM_CIFNOTFND;
 	int ret = NET_RX_SUCCESS;
 
-	while (tvlv_value_len >= sizeof(*tvlv_hdr)) {
-		tvlv_hdr = tvlv_value;
+	while ((tvlv_hdr = batadv_tvlv_hdr_next(&tvlv_value, &tvlv_value_len))) {
 		tvlv_value_cont_len = ntohs(tvlv_hdr->len);
-		tvlv_value = tvlv_hdr + 1;
-		tvlv_value_len -= sizeof(*tvlv_hdr);
-
-		if (tvlv_value_cont_len > tvlv_value_len)
-			break;
-
-		/* the next tvlv header is accessed assuming (at least) 2-byte
-		 * alignment, so it must start at an even offset.
-		 */
-		if (tvlv_value_cont_len & 1)
-			break;
 
 		tvlv_handler = batadv_tvlv_handler_get(bat_priv,
 						       tvlv_hdr->type,
@@ -532,11 +550,9 @@ int batadv_tvlv_containers_process(struct batadv_priv *bat_priv,
 
 		ret |= batadv_tvlv_call_handler(bat_priv, tvlv_handler,
 						packet_type, orig_node, skb,
-						tvlv_value,
+						tvlv_hdr + 1,
 						tvlv_value_cont_len);
 		batadv_tvlv_handler_put(tvlv_handler);
-		tvlv_value = (u8 *)tvlv_value + tvlv_value_cont_len;
-		tvlv_value_len -= tvlv_value_cont_len;
 	}
 
 	if (packet_type != BATADV_IV_OGM &&
diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h
index b1f9f8964c3fd..cd12755d21f35 100644
--- a/net/batman-adv/types.h
+++ b/net/batman-adv/types.h
@@ -214,9 +214,6 @@ struct batadv_wifi_net_device_state {
  * struct batadv_hard_iface - network device known to batman-adv
  */
 struct batadv_hard_iface {
-	/** @list: list node for batadv_hardif_list */
-	struct list_head list;
-
 	/** @if_status: status of the interface for batman-adv */
 	char if_status;
 
@@ -1335,9 +1332,9 @@ struct batadv_tp_unacked {
 	u32 seqno;
 
 	/** @len: length of the packet */
-	u16 len;
+	u32 len;
 
-	/** @list: list node for &batadv_tp_vars_common.unacked_list */
+	/** @list: list node for &batadv_tp_receiver.unacked_list */
 	struct list_head list;
 };
 
@@ -1360,15 +1357,6 @@ struct batadv_tp_vars_common {
 	/** @session: TP session identifier */
 	u8 session[2];
 
-	/** @unacked_list: list of unacked packets (meta-info only) */
-	struct list_head unacked_list;
-
-	/** @unacked_lock: protect unacked_list + &batadv_tp_receiver.last_recv */
-	spinlock_t unacked_lock;
-
-	/** @unacked_count: number of unacked entries */
-	size_t unacked_count;
-
 	/** @refcount: number of context where the object is used */
 	struct kref refcount;
 
@@ -1482,6 +1470,15 @@ struct batadv_tp_receiver {
 
 	/** @last_recv_time: time (jiffies) a msg was received */
 	unsigned long last_recv_time;
+
+	/** @unacked_list: list of unacked packets (meta-info only) */
+	struct list_head unacked_list;
+
+	/** @ack_seqno_lock: protect unacked_list + &batadv_tp_receiver.last_recv */
+	spinlock_t ack_seqno_lock;
+
+	/** @unacked_count: number of unacked entries */
+	size_t unacked_count;
 };
 
 /**
@@ -1679,6 +1676,9 @@ struct batadv_priv {
 	/** @tp_num: number of currently active tp sessions */
 	atomic_t tp_num;
 
+	/** @hardif_generation: generation counter added to netlink hardif dumps */
+	unsigned int hardif_generation;
+
 	/** @orig_work: work queue callback item for orig node purging */
 	struct delayed_work orig_work;
 
diff --git a/net/bluetooth/Makefile b/net/bluetooth/Makefile
index 41049b280887d..ff466ea97436a 100644
--- a/net/bluetooth/Makefile
+++ b/net/bluetooth/Makefile
@@ -25,3 +25,5 @@ bluetooth-$(CONFIG_BT_MSFTEXT) += msft.o
 bluetooth-$(CONFIG_BT_AOSPEXT) += aosp.o
 bluetooth-$(CONFIG_BT_DEBUGFS) += hci_debugfs.o
 bluetooth-$(CONFIG_BT_SELFTEST) += selftest.o
+
+CONTEXT_ANALYSIS := y
diff --git a/net/bluetooth/af_bluetooth.c b/net/bluetooth/af_bluetooth.c
index a2290ffdc2c1a..411d66f24393d 100644
--- a/net/bluetooth/af_bluetooth.c
+++ b/net/bluetooth/af_bluetooth.c
@@ -209,6 +209,7 @@ bool bt_sock_linked(struct bt_sock_list *l, struct sock *s)
 EXPORT_SYMBOL(bt_sock_linked);
 
 void bt_accept_enqueue(struct sock *parent, struct sock *sk, bool bh)
+	__context_unsafe(/* conditional locking */)
 {
 	const struct cred *old_cred;
 	struct pid *old_pid;
@@ -815,7 +816,8 @@ EXPORT_SYMBOL(bt_sock_wait_ready);
 
 #ifdef CONFIG_PROC_FS
 static void *bt_seq_start(struct seq_file *seq, loff_t *pos)
-	__acquires(seq->private->l->lock)
+	__acquires_shared(&((struct bt_sock_list *)
+			    pde_data(file_inode(seq->file)))->lock)
 {
 	struct bt_sock_list *l = pde_data(file_inode(seq->file));
 
@@ -831,7 +833,8 @@ static void *bt_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 }
 
 static void bt_seq_stop(struct seq_file *seq, void *v)
-	__releases(seq->private->l->lock)
+	__releases_shared(&((struct bt_sock_list *)
+			    pde_data(file_inode(seq->file)))->lock)
 {
 	struct bt_sock_list *l = pde_data(file_inode(seq->file));
 
diff --git a/net/bluetooth/bnep/Makefile b/net/bluetooth/bnep/Makefile
index 8af9d56bb0125..f42015cc32451 100644
--- a/net/bluetooth/bnep/Makefile
+++ b/net/bluetooth/bnep/Makefile
@@ -6,3 +6,5 @@
 obj-$(CONFIG_BT_BNEP) += bnep.o
 
 bnep-objs := core.o sock.o netdev.o
+
+CONTEXT_ANALYSIS := y
diff --git a/net/bluetooth/coredump.c b/net/bluetooth/coredump.c
index 720cb79adf964..c0f027fab5835 100644
--- a/net/bluetooth/coredump.c
+++ b/net/bluetooth/coredump.c
@@ -390,7 +390,6 @@ void hci_devcd_rx(struct work_struct *work)
 		hci_dev_unlock(hdev);
 	}
 }
-EXPORT_SYMBOL(hci_devcd_rx);
 
 void hci_devcd_timeout(struct work_struct *work)
 {
@@ -416,7 +415,6 @@ void hci_devcd_timeout(struct work_struct *work)
 
 	hci_dev_unlock(hdev);
 }
-EXPORT_SYMBOL(hci_devcd_timeout);
 
 int hci_devcd_register(struct hci_dev *hdev, coredump_t coredump,
 		       dmp_hdr_t dmp_hdr, notify_change_t notify_change)
diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c
index 5ba9fe8261ec8..d1e78ae7728ee 100644
--- a/net/bluetooth/hci_core.c
+++ b/net/bluetooth/hci_core.c
@@ -62,6 +62,7 @@ static DEFINE_IDA(hci_index_ida);
 /* Get HCI device by index.
  * Device is held on return. */
 static struct hci_dev *__hci_dev_get(int index, int *srcu_index)
+	__context_unsafe(/* conditional locking */)
 {
 	struct hci_dev *hdev = NULL, *d;
 
@@ -89,11 +90,13 @@ struct hci_dev *hci_dev_get(int index)
 }
 
 static struct hci_dev *hci_dev_get_srcu(int index, int *srcu_index)
+	__context_unsafe(/* conditional locking vs return */)
 {
 	return __hci_dev_get(index, srcu_index);
 }
 
 static void hci_dev_put_srcu(struct hci_dev *hdev, int srcu_index)
+	__context_unsafe(/* conditional locking vs return */)
 {
 	srcu_read_unlock(&hdev->srcu, srcu_index);
 	hci_dev_put(hdev);
diff --git a/net/bluetooth/hci_debugfs.c b/net/bluetooth/hci_debugfs.c
index 0635e4641db4b..aadffaaff20e1 100644
--- a/net/bluetooth/hci_debugfs.c
+++ b/net/bluetooth/hci_debugfs.c
@@ -1161,16 +1161,12 @@ static ssize_t force_no_mitm_write(struct file *file,
 				   size_t count, loff_t *ppos)
 {
 	struct hci_dev *hdev = file->private_data;
-	char buf[32];
-	size_t buf_size = min(count, (sizeof(buf) - 1));
 	bool enable;
+	int err;
 
-	if (copy_from_user(buf, user_buf, buf_size))
-		return -EFAULT;
-
-	buf[buf_size] = '\0';
-	if (kstrtobool(buf, &enable))
-		return -EINVAL;
+	err = kstrtobool_from_user(user_buf, count, &enable);
+	if (err)
+		return err;
 
 	if (enable == hci_dev_test_flag(hdev, HCI_FORCE_NO_MITM))
 		return -EALREADY;
diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c
index b6d963ce26d0e..ea858391c789f 100644
--- a/net/bluetooth/hci_event.c
+++ b/net/bluetooth/hci_event.c
@@ -269,7 +269,10 @@ static u8 hci_cc_reset(struct hci_dev *hdev, void *data, struct sk_buff *skb)
 {
 	struct hci_ev_status *rp = data;
 
-	bt_dev_dbg(hdev, "status 0x%2.2x", rp->status);
+	if (rp->status)
+		bt_dev_err(hdev, "status 0x%2.2x", rp->status);
+	else
+		bt_dev_dbg(hdev, "status 0x%2.2x", rp->status);
 
 	clear_bit(HCI_RESET, &hdev->flags);
 
@@ -2763,7 +2766,7 @@ static void hci_cs_disconnect(struct hci_dev *hdev, u8 status)
 	}
 
 	mgmt_device_disconnected(hdev, &conn->dst, conn->type, conn->dst_type,
-				 cp->reason, mgmt_conn);
+				 hci_to_mgmt_reason(cp->reason), mgmt_conn);
 
 	hci_disconn_cfm(conn, cp->reason);
 
@@ -3381,22 +3384,6 @@ static void hci_conn_request_evt(struct hci_dev *hdev, void *data,
 	hci_dev_unlock(hdev);
 }
 
-static u8 hci_to_mgmt_reason(u8 err)
-{
-	switch (err) {
-	case HCI_ERROR_CONNECTION_TIMEOUT:
-		return MGMT_DEV_DISCONN_TIMEOUT;
-	case HCI_ERROR_REMOTE_USER_TERM:
-	case HCI_ERROR_REMOTE_LOW_RESOURCES:
-	case HCI_ERROR_REMOTE_POWER_OFF:
-		return MGMT_DEV_DISCONN_REMOTE;
-	case HCI_ERROR_LOCAL_HOST_TERM:
-		return MGMT_DEV_DISCONN_LOCAL_HOST;
-	default:
-		return MGMT_DEV_DISCONN_UNKNOWN;
-	}
-}
-
 static void hci_disconn_complete_evt(struct hci_dev *hdev, void *data,
 				     struct sk_buff *skb)
 {
diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c
index c896d4edd0133..b4ca34abe0329 100644
--- a/net/bluetooth/hci_sync.c
+++ b/net/bluetooth/hci_sync.c
@@ -860,32 +860,6 @@ void hci_cmd_sync_cancel_entry(struct hci_dev *hdev,
 }
 EXPORT_SYMBOL(hci_cmd_sync_cancel_entry);
 
-/* Dequeue one HCI command entry:
- *
- * - Lookup and cancel first entry that matches.
- */
-bool hci_cmd_sync_dequeue_once(struct hci_dev *hdev,
-			       hci_cmd_sync_work_func_t func,
-			       void *data, hci_cmd_sync_work_destroy_t destroy)
-{
-	struct hci_cmd_sync_work_entry *entry;
-
-	mutex_lock(&hdev->cmd_sync_work_lock);
-
-	entry = _hci_cmd_sync_lookup_entry(hdev, func, data, destroy);
-	if (!entry) {
-		mutex_unlock(&hdev->cmd_sync_work_lock);
-		return false;
-	}
-
-	_hci_cmd_sync_cancel_entry(hdev, entry, -ECANCELED);
-
-	mutex_unlock(&hdev->cmd_sync_work_lock);
-
-	return true;
-}
-EXPORT_SYMBOL(hci_cmd_sync_dequeue_once);
-
 /* Dequeue HCI command entry:
  *
  * - Lookup and cancel any entry that matches by function callback or data or
@@ -1054,14 +1028,19 @@ static int hci_set_random_addr_sync(struct hci_dev *hdev, bdaddr_t *rpa)
 	 * In this kind of scenario skip the update and let the random
 	 * address be updated at the next cycle.
 	 */
+	rcu_read_lock();
+
 	if (bacmp(&hdev->random_addr, BDADDR_ANY) &&
 	    (hci_dev_test_flag(hdev, HCI_LE_ADV) ||
 	    hci_lookup_le_connect(hdev))) {
 		bt_dev_dbg(hdev, "Deferring random address update");
 		hci_dev_set_flag(hdev, HCI_RPA_EXPIRED);
+		rcu_read_unlock();
 		return 0;
 	}
 
+	rcu_read_unlock();
+
 	return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_RANDOM_ADDR,
 				     6, rpa, HCI_CMD_TIMEOUT);
 }
@@ -2647,12 +2626,17 @@ static int hci_pause_addr_resolution(struct hci_dev *hdev)
 	/* Cannot disable addr resolution if scanning is enabled or
 	 * when initiating an LE connection.
 	 */
+	rcu_read_lock();
+
 	if (hci_dev_test_flag(hdev, HCI_LE_SCAN) ||
 	    hci_lookup_le_connect(hdev)) {
+		rcu_read_unlock();
 		bt_dev_err(hdev, "Command not allowed when scan/LE connect");
 		return -EPERM;
 	}
 
+	rcu_read_unlock();
+
 	/* Cannot disable addr resolution if advertising is enabled. */
 	err = hci_pause_advertising_sync(hdev);
 	if (err) {
@@ -2790,6 +2774,8 @@ static u8 hci_update_accept_list_sync(struct hci_dev *hdev)
 	if (hci_dev_test_flag(hdev, HCI_PA_SYNC)) {
 		struct hci_conn *conn;
 
+		rcu_read_lock();
+
 		conn = hci_conn_hash_lookup_create_pa_sync(hdev);
 		if (conn) {
 			struct conn_params pa;
@@ -2799,6 +2785,8 @@ static u8 hci_update_accept_list_sync(struct hci_dev *hdev)
 			bacpy(&pa.addr, &conn->dst);
 			pa.addr_type = conn->dst_type;
 
+			rcu_read_unlock();
+
 			/* Clear first since there could be addresses left
 			 * behind.
 			 */
@@ -2808,6 +2796,8 @@ static u8 hci_update_accept_list_sync(struct hci_dev *hdev)
 			err = hci_le_add_accept_list_sync(hdev, &pa,
 							  &num_entries);
 			goto done;
+		} else {
+			rcu_read_unlock();
 		}
 	}
 
@@ -2818,10 +2808,13 @@ static u8 hci_update_accept_list_sync(struct hci_dev *hdev)
 	 * the controller.
 	 */
 	list_for_each_entry_safe(b, t, &hdev->le_accept_list, list) {
-		if (hci_conn_hash_lookup_le(hdev, &b->bdaddr, b->bdaddr_type))
+		rcu_read_lock();
+
+		if (hci_conn_hash_lookup_le(hdev, &b->bdaddr, b->bdaddr_type)) {
+			rcu_read_unlock();
 			continue;
+		}
 
-		/* Pointers not dereferenced, no locks needed */
 		pend_conn = hci_pend_le_action_lookup(&hdev->pend_le_conns,
 						      &b->bdaddr,
 						      b->bdaddr_type);
@@ -2829,6 +2822,8 @@ static u8 hci_update_accept_list_sync(struct hci_dev *hdev)
 							&b->bdaddr,
 							b->bdaddr_type);
 
+		rcu_read_unlock();
+
 		/* If the device is not likely to connect or report,
 		 * remove it from the acceptlist.
 		 */
@@ -2955,6 +2950,8 @@ static int hci_le_set_ext_scan_param_sync(struct hci_dev *hdev, u8 type,
 		if (sent) {
 			struct hci_conn *conn;
 
+			rcu_read_lock();
+
 			conn = hci_conn_hash_lookup_ba(hdev, PA_LINK,
 						       &sent->bdaddr);
 			if (conn) {
@@ -2979,8 +2976,12 @@ static int hci_le_set_ext_scan_param_sync(struct hci_dev *hdev, u8 type,
 					phy++;
 				}
 
+				rcu_read_unlock();
+
 				if (num_phy)
 					goto done;
+			} else {
+				rcu_read_unlock();
 			}
 		}
 	}
@@ -3231,12 +3232,16 @@ int hci_update_passive_scan_sync(struct hci_dev *hdev)
 		/* If there is at least one pending LE connection, we should
 		 * keep the background scan running.
 		 */
+		bool exists;
 
 		/* If controller is connecting, we should not start scanning
 		 * since some controllers are not able to scan and connect at
 		 * the same time.
 		 */
-		if (hci_lookup_le_connect(hdev))
+		rcu_read_lock();
+		exists = hci_lookup_le_connect(hdev);
+		rcu_read_unlock();
+		if (exists)
 			return 0;
 
 		bt_dev_dbg(hdev, "start background scanning");
@@ -3454,6 +3459,7 @@ int hci_write_fast_connectable_sync(struct hci_dev *hdev, bool enable)
 }
 
 static bool disconnected_accept_list_entries(struct hci_dev *hdev)
+	__must_hold(&hdev->lock)
 {
 	struct bdaddr_list *b;
 
@@ -3494,12 +3500,16 @@ int hci_update_scan_sync(struct hci_dev *hdev)
 	if (hdev->scanning_paused)
 		return 0;
 
+	hci_dev_lock(hdev);
+
 	if (hci_dev_test_flag(hdev, HCI_CONNECTABLE) ||
 	    disconnected_accept_list_entries(hdev))
 		scan = SCAN_PAGE;
 	else
 		scan = SCAN_DISABLED;
 
+	hci_dev_unlock(hdev);
+
 	if (hci_dev_test_flag(hdev, HCI_DISCOVERABLE))
 		scan |= SCAN_INQUIRY;
 
@@ -3672,17 +3682,19 @@ static const struct hci_init_stage hci_init0[] = {
 
 int hci_reset_sync(struct hci_dev *hdev)
 {
-	int err;
-
 	set_bit(HCI_RESET, &hdev->flags);
 
-	err = __hci_cmd_sync_status(hdev, HCI_OP_RESET, 0, NULL,
-				    HCI_CMD_TIMEOUT);
-	if (err)
-		return err;
+	return __hci_cmd_sync_status(hdev, HCI_OP_RESET, 0, NULL,
+				     HCI_CMD_TIMEOUT);
+}
 
-	return 0;
+/* Send a raw HCI reset for use by vendor drivers */
+int __hci_reset_sync(struct hci_dev *hdev)
+{
+	return __hci_cmd_sync_status(hdev, HCI_OP_RESET, 0, NULL,
+				     HCI_INIT_TIMEOUT);
 }
+EXPORT_SYMBOL(__hci_reset_sync);
 
 static int hci_init0_sync(struct hci_dev *hdev)
 {
@@ -6665,6 +6677,8 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data)
 	if (!hci_dev_test_flag(hdev, HCI_LE_SIMULTANEOUS_ROLES))
 		hci_pause_advertising_sync(hdev);
 
+	hci_dev_lock(hdev);
+
 	params = hci_conn_params_lookup(hdev, &conn->dst, conn->dst_type);
 	if (params) {
 		conn->le_conn_min_interval = params->conn_min_interval;
@@ -6678,6 +6692,8 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data)
 		conn->le_supv_timeout = hdev->le_supv_timeout;
 	}
 
+	hci_dev_unlock(hdev);
+
 	/* If controller is scanning, we stop it since some controllers are
 	 * not able to scan and connect at the same time. Also set the
 	 * HCI_LE_SCAN_INTERRUPTED flag so that the command complete
@@ -7235,13 +7251,13 @@ static void create_pa_complete(struct hci_dev *hdev, void *data, int err)
 }
 
 static int hci_le_past_params_sync(struct hci_dev *hdev, struct hci_conn *conn,
-				   struct hci_conn *acl, struct bt_iso_qos *qos)
+				   u16 acl_handle, struct bt_iso_qos *qos)
 {
 	struct hci_cp_le_past_params cp;
 	int err;
 
 	memset(&cp, 0, sizeof(cp));
-	cp.handle = cpu_to_le16(acl->handle);
+	cp.handle = cpu_to_le16(acl_handle);
 	/* An HCI_LE_Periodic_Advertising_Sync_Transfer_Received event is sent
 	 * to the Host. HCI_LE_Periodic_Advertising_Report events will be
 	 * enabled with duplicate filtering enabled.
@@ -7306,16 +7322,28 @@ static int hci_le_pa_create_sync(struct hci_dev *hdev, void *data)
 	 * 2. Check if that HCI_CONN_FLAG_PAST has been set which indicates that
 	 *    user really intended to use PAST.
 	 */
+	hci_dev_lock(hdev);
+
 	le = hci_conn_hash_lookup_le(hdev, &conn->dst, conn->dst_type);
 	if (le) {
 		struct hci_conn_params *params;
+		hci_conn_flags_t flags = 0;
+		u16 le_handle = le->handle;
 
 		params = hci_conn_params_lookup(hdev, &le->dst, le->dst_type);
-		if (params && params->flags & HCI_CONN_FLAG_PAST) {
-			err = hci_le_past_params_sync(hdev, conn, le, qos);
+		if (params)
+			flags = params->flags;
+
+		hci_dev_unlock(hdev);
+
+		if (flags & HCI_CONN_FLAG_PAST) {
+			err = hci_le_past_params_sync(hdev, conn, le_handle,
+						      qos);
 			if (!err)
 				goto done;
 		}
+	} else {
+		hci_dev_unlock(hdev);
 	}
 
 	/* SID has not been set listen for HCI_EV_LE_EXT_ADV_REPORT to update
diff --git a/net/bluetooth/hidp/Makefile b/net/bluetooth/hidp/Makefile
index f41b0aa02b235..53e139e41bdc1 100644
--- a/net/bluetooth/hidp/Makefile
+++ b/net/bluetooth/hidp/Makefile
@@ -6,3 +6,5 @@
 obj-$(CONFIG_BT_HIDP) += hidp.o
 
 hidp-objs := core.o sock.o
+
+CONTEXT_ANALYSIS := y
diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c
index 4058ff50cc27a..735167f73f312 100644
--- a/net/bluetooth/l2cap_sock.c
+++ b/net/bluetooth/l2cap_sock.c
@@ -1365,6 +1365,7 @@ static int __l2cap_wait_ack(struct sock *sk, struct l2cap_chan *chan)
 }
 
 static int l2cap_sock_shutdown(struct socket *sock, int how)
+	__context_unsafe(/* complex chan->conn locking */)
 {
 	struct sock *sk = sock->sk;
 	struct l2cap_chan *chan;
diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c
index 733a4b70e10c5..1db10e0f617f9 100644
--- a/net/bluetooth/mgmt.c
+++ b/net/bluetooth/mgmt.c
@@ -3091,6 +3091,8 @@ static int unpair_device_sync(struct hci_dev *hdev, void *data)
 	struct mgmt_cp_unpair_device *cp = cmd->param;
 	struct hci_conn *conn;
 
+	hci_dev_lock(hdev);
+
 	if (cp->addr.type == BDADDR_BREDR)
 		conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK,
 					       &cp->addr.bdaddr);
@@ -3098,6 +3100,11 @@ static int unpair_device_sync(struct hci_dev *hdev, void *data)
 		conn = hci_conn_hash_lookup_le(hdev, &cp->addr.bdaddr,
 					       le_addr_type(cp->addr.type));
 
+	if (conn)
+		hci_conn_get(conn);
+
+	hci_dev_unlock(hdev);
+
 	if (!conn)
 		return 0;
 
@@ -3105,6 +3112,7 @@ static int unpair_device_sync(struct hci_dev *hdev, void *data)
 	 * will clean up the connection no matter the error.
 	 */
 	hci_abort_conn(conn, HCI_ERROR_REMOTE_USER_TERM);
+	hci_conn_put(conn);
 
 	return 0;
 }
@@ -3252,6 +3260,8 @@ static int disconnect_sync(struct hci_dev *hdev, void *data)
 	struct mgmt_cp_disconnect *cp = cmd->param;
 	struct hci_conn *conn;
 
+	hci_dev_lock(hdev);
+
 	if (cp->addr.type == BDADDR_BREDR)
 		conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK,
 					       &cp->addr.bdaddr);
@@ -3259,6 +3269,11 @@ static int disconnect_sync(struct hci_dev *hdev, void *data)
 		conn = hci_conn_hash_lookup_le(hdev, &cp->addr.bdaddr,
 					       le_addr_type(cp->addr.type));
 
+	if (conn)
+		hci_conn_get(conn);
+
+	hci_dev_unlock(hdev);
+
 	if (!conn)
 		return -ENOTCONN;
 
@@ -3266,6 +3281,7 @@ static int disconnect_sync(struct hci_dev *hdev, void *data)
 	 * will clean up the connection no matter the error.
 	 */
 	hci_abort_conn(conn, HCI_ERROR_REMOTE_USER_TERM);
+	hci_conn_put(conn);
 
 	return 0;
 }
@@ -7388,6 +7404,9 @@ static void get_conn_info_complete(struct hci_dev *hdev, void *data, int err)
 		rp.max_tx_power = HCI_TX_POWER_INVALID;
 	}
 
+	if (conn)
+		hci_conn_put(conn);
+
 	mgmt_cmd_complete(cmd->sk, cmd->hdev->id, MGMT_OP_GET_CONN_INFO, status,
 			  &rp, sizeof(rp));
 
@@ -7402,6 +7421,8 @@ static int get_conn_info_sync(struct hci_dev *hdev, void *data)
 	int err;
 	__le16   handle;
 
+	hci_dev_lock(hdev);
+
 	/* Make sure we are still connected */
 	if (cp->addr.type == BDADDR_BREDR)
 		conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK,
@@ -7409,12 +7430,16 @@ static int get_conn_info_sync(struct hci_dev *hdev, void *data)
 	else
 		conn = hci_conn_hash_lookup_ba(hdev, LE_LINK, &cp->addr.bdaddr);
 
-	if (!conn || conn->state != BT_CONNECTED)
+	if (!conn || conn->state != BT_CONNECTED) {
+		hci_dev_unlock(hdev);
 		return MGMT_STATUS_NOT_CONNECTED;
+	}
 
-	cmd->user_data = conn;
+	cmd->user_data = hci_conn_get(conn);
 	handle = cpu_to_le16(conn->handle);
 
+	hci_dev_unlock(hdev);
+
 	/* Refresh RSSI each time */
 	err = hci_read_rssi_sync(hdev, handle);
 
@@ -7548,6 +7573,9 @@ static void get_clock_info_complete(struct hci_dev *hdev, void *data, int err)
 	}
 
 complete:
+	if (conn)
+		hci_conn_put(conn);
+
 	mgmt_cmd_complete(cmd->sk, cmd->hdev->id, cmd->opcode, status, &rp,
 			  sizeof(rp));
 
@@ -7564,15 +7592,21 @@ static int get_clock_info_sync(struct hci_dev *hdev, void *data)
 	memset(&hci_cp, 0, sizeof(hci_cp));
 	hci_read_clock_sync(hdev, &hci_cp);
 
+	hci_dev_lock(hdev);
+
 	/* Make sure connection still exists */
 	conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, &cp->addr.bdaddr);
-	if (!conn || conn->state != BT_CONNECTED)
+	if (!conn || conn->state != BT_CONNECTED) {
+		hci_dev_unlock(hdev);
 		return MGMT_STATUS_NOT_CONNECTED;
+	}
 
-	cmd->user_data = conn;
+	cmd->user_data = hci_conn_get(conn);
 	hci_cp.handle = cpu_to_le16(conn->handle);
 	hci_cp.which = 0x01; /* Piconet clock */
 
+	hci_dev_unlock(hdev);
+
 	return hci_read_clock_sync(hdev, &hci_cp);
 }
 
@@ -7937,14 +7971,36 @@ static int remove_device(struct sock *sk, struct hci_dev *hdev,
 
 static int conn_update_sync(struct hci_dev *hdev, void *data)
 {
-	struct hci_conn_params *params = data;
-	struct hci_conn *conn;
+	struct hci_conn *conn = data;
+	struct hci_conn_params *params;
+	struct hci_conn_params local = {};
 
-	conn = hci_conn_hash_lookup_le(hdev, &params->addr, params->addr_type);
-	if (!conn)
-		return -ECANCELED;
+	hci_dev_lock(hdev);
+
+	if (!hci_conn_valid(hdev, conn) || conn->role != HCI_ROLE_MASTER)
+		goto cancel;
+
+	params = hci_conn_params_lookup(hdev, &conn->dst, conn->dst_type);
+	if (!params)
+		goto cancel;
+
+	local.conn_min_interval = params->conn_min_interval;
+	local.conn_max_interval = params->conn_max_interval;
+	local.conn_latency = params->conn_latency;
+	local.supervision_timeout = params->supervision_timeout;
+
+	hci_dev_unlock(hdev);
+
+	return hci_le_conn_update_sync(hdev, conn, &local);
 
-	return hci_le_conn_update_sync(hdev, conn, params);
+cancel:
+	hci_dev_unlock(hdev);
+	return -ECANCELED;
+}
+
+static void conn_update_sync_destroy(struct hci_dev *hdev, void *data, int err)
+{
+	hci_conn_put(data);
 }
 
 static int load_conn_param(struct sock *sk, struct hci_dev *hdev, void *data,
@@ -8054,9 +8110,13 @@ static int load_conn_param(struct sock *sk, struct hci_dev *hdev, void *data,
 			    (conn->le_conn_min_interval != min ||
 			     conn->le_conn_max_interval != max ||
 			     conn->le_conn_latency != latency ||
-			     conn->le_supv_timeout != timeout))
-				hci_cmd_sync_queue(hdev, conn_update_sync,
-						   hci_param, NULL);
+			     conn->le_supv_timeout != timeout)) {
+				hci_conn_get(conn);
+				if (hci_cmd_sync_queue(hdev, conn_update_sync,
+						       conn,
+						       conn_update_sync_destroy) < 0)
+					hci_conn_put(conn);
+			}
 		}
 	}
 
@@ -9848,6 +9908,22 @@ bool mgmt_powering_down(struct hci_dev *hdev)
 	return false;
 }
 
+u8 hci_to_mgmt_reason(u8 err)
+{
+	switch (err) {
+	case HCI_ERROR_CONNECTION_TIMEOUT:
+		return MGMT_DEV_DISCONN_TIMEOUT;
+	case HCI_ERROR_REMOTE_USER_TERM:
+	case HCI_ERROR_REMOTE_LOW_RESOURCES:
+	case HCI_ERROR_REMOTE_POWER_OFF:
+		return MGMT_DEV_DISCONN_REMOTE;
+	case HCI_ERROR_LOCAL_HOST_TERM:
+		return MGMT_DEV_DISCONN_LOCAL_HOST;
+	default:
+		return MGMT_DEV_DISCONN_UNKNOWN;
+	}
+}
+
 void mgmt_device_disconnected(struct hci_dev *hdev, bdaddr_t *bdaddr,
 			      u8 link_type, u8 addr_type, u8 reason,
 			      bool mgmt_connected)
@@ -9909,7 +9985,8 @@ void mgmt_connect_failed(struct hci_dev *hdev, struct hci_conn *conn, u8 status)
 
 	if (test_and_clear_bit(HCI_CONN_MGMT_CONNECTED, &conn->flags)) {
 		mgmt_device_disconnected(hdev, &conn->dst, conn->type,
-					 conn->dst_type, status, true);
+					 conn->dst_type,
+					 hci_to_mgmt_reason(status), true);
 		return;
 	}
 
diff --git a/net/bluetooth/rfcomm/Makefile b/net/bluetooth/rfcomm/Makefile
index 593e5c48c1315..15f909f40f25f 100644
--- a/net/bluetooth/rfcomm/Makefile
+++ b/net/bluetooth/rfcomm/Makefile
@@ -7,3 +7,5 @@ obj-$(CONFIG_BT_RFCOMM) += rfcomm.o
 
 rfcomm-y			:= core.o sock.o
 rfcomm-$(CONFIG_BT_RFCOMM_TTY)	+= tty.o
+
+CONTEXT_ANALYSIS := y
diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c
index feb302a491fa0..958081adb9b5c 100644
--- a/net/bluetooth/rfcomm/sock.c
+++ b/net/bluetooth/rfcomm/sock.c
@@ -60,6 +60,7 @@ static void rfcomm_sk_data_ready(struct rfcomm_dlc *d, struct sk_buff *skb)
 }
 
 static void rfcomm_sk_state_change(struct rfcomm_dlc *d, int err)
+	__must_hold(&d->lock)
 {
 	struct sock *sk = d->owner, *parent;
 
diff --git a/net/bridge/netfilter/ebtables.c b/net/bridge/netfilter/ebtables.c
index 96c9a8f57c870..0bf541936f601 100644
--- a/net/bridge/netfilter/ebtables.c
+++ b/net/bridge/netfilter/ebtables.c
@@ -39,6 +39,8 @@
 #define COUNTER_OFFSET(n) (SMP_ALIGN(n * sizeof(struct ebt_counter)))
 #define COUNTER_BASE(c, n, cpu) ((struct ebt_counter *)(((char *)c) + \
 				 COUNTER_OFFSET(n) * cpu))
+#define MAX_EBT_ENTRIES (((INT_MAX - sizeof(struct ebt_table_info)) / \
+			 NR_CPUS - SMP_CACHE_BYTES) / sizeof(struct ebt_counter))
 
 struct ebt_pernet {
 	struct list_head tables;
@@ -1126,10 +1128,9 @@ static int do_replace(struct net *net, sockptr_t arg, unsigned int len)
 		return -EINVAL;
 
 	/* overflow check */
-	if (tmp.nentries >= ((INT_MAX - sizeof(struct ebt_table_info)) /
-			NR_CPUS - SMP_CACHE_BYTES) / sizeof(struct ebt_counter))
+	if (tmp.nentries >= MAX_EBT_ENTRIES)
 		return -ENOMEM;
-	if (tmp.num_counters >= INT_MAX / sizeof(struct ebt_counter))
+	if (tmp.num_counters >= MAX_EBT_ENTRIES)
 		return -ENOMEM;
 
 	tmp.name[sizeof(tmp.name) - 1] = 0;
@@ -2269,10 +2270,9 @@ static int compat_copy_ebt_replace_from_user(struct ebt_replace *repl,
 	if (tmp.entries_size == 0)
 		return -EINVAL;
 
-	if (tmp.nentries >= ((INT_MAX - sizeof(struct ebt_table_info)) /
-			NR_CPUS - SMP_CACHE_BYTES) / sizeof(struct ebt_counter))
+	if (tmp.nentries >= MAX_EBT_ENTRIES)
 		return -ENOMEM;
-	if (tmp.num_counters >= INT_MAX / sizeof(struct ebt_counter))
+	if (tmp.num_counters >= MAX_EBT_ENTRIES)
 		return -ENOMEM;
 
 	memcpy(repl, &tmp, offsetof(struct ebt_replace, hook_entry));
diff --git a/net/can/bcm.c b/net/can/bcm.c
index a4bef2c48a559..3d637a1e0ac1a 100644
--- a/net/can/bcm.c
+++ b/net/can/bcm.c
@@ -58,6 +58,7 @@
 #include <linux/can/skb.h>
 #include <linux/can/bcm.h>
 #include <linux/slab.h>
+#include <linux/workqueue.h>
 #include <linux/spinlock.h>
 #include <net/can.h>
 #include <net/sock.h>
@@ -92,6 +93,8 @@ MODULE_ALIAS("can-proto-2");
 
 #define BCM_MIN_NAMELEN CAN_REQUIRED_SIZE(struct sockaddr_can, can_ifindex)
 
+static struct workqueue_struct *bcm_wq;
+
 /*
  * easy access to the first 64 bit of can(fd)_frame payload. cp->data is
  * 64 bit aligned so the offset has to be multiples of 8 which is ensured
@@ -105,14 +108,16 @@ static inline u64 get_u64(const struct canfd_frame *cp, int offset)
 struct bcm_op {
 	struct list_head list;
 	struct rcu_head rcu;
+	struct work_struct work;
 	int ifindex;
 	canid_t can_id;
 	u32 flags;
-	unsigned long frames_abs, frames_filtered;
+	atomic_long_t frames_abs, frames_filtered;
 	struct bcm_timeval ival1, ival2;
 	struct hrtimer timer, thrtimer;
 	ktime_t rx_stamp, kt_ival1, kt_ival2, kt_lastmsg;
 	int rx_ifindex;
+	int if_detected; /* first received ifindex in ANYDEV rx_op mode */
 	int cfsiz;
 	u32 count;
 	u32 nframes;
@@ -124,7 +129,9 @@ struct bcm_op {
 	struct canfd_frame last_sframe;
 	struct sock *sk;
 	struct net_device *rx_reg_dev;
-	spinlock_t bcm_tx_lock; /* protect currframe/count in runtime updates */
+	netdevice_tracker rx_reg_dev_tracker;
+	spinlock_t bcm_tx_lock; /* protect tx data and timer updates */
+	spinlock_t bcm_rx_update_lock; /* protect filter/timer data updates */
 };
 
 struct bcm_sock {
@@ -224,10 +231,13 @@ static int bcm_proc_show(struct seq_file *m, void *v)
 
 	list_for_each_entry_rcu(op, &bo->rx_ops, list) {
 
-		unsigned long reduction;
+		long reduction, frames_filtered, frames_abs;
+
+		frames_filtered = atomic_long_read(&op->frames_filtered);
+		frames_abs = atomic_long_read(&op->frames_abs);
 
 		/* print only active entries & prevent division by zero */
-		if (!op->frames_abs)
+		if (!frames_abs)
 			continue;
 
 		seq_printf(m, "rx_op: %03X %-5s ", op->can_id,
@@ -249,15 +259,15 @@ static int bcm_proc_show(struct seq_file *m, void *v)
 				   (long long)ktime_to_us(op->kt_ival2));
 
 		seq_printf(m, "# recv %ld (%ld) => reduction: ",
-			   op->frames_filtered, op->frames_abs);
+			   frames_filtered, frames_abs);
 
-		reduction = 100 - (op->frames_filtered * 100) / op->frames_abs;
+		reduction = 100 - (frames_filtered * 100) / frames_abs;
 
 		seq_printf(m, "%s%ld%%\n",
 			   (reduction == 100) ? "near " : "", reduction);
 	}
 
-	list_for_each_entry(op, &bo->tx_ops, list) {
+	list_for_each_entry_rcu(op, &bo->tx_ops, list) {
 
 		seq_printf(m, "tx_op: %03X %s ", op->can_id,
 			   bcm_proc_getifname(net, ifname, op->ifindex));
@@ -275,7 +285,8 @@ static int bcm_proc_show(struct seq_file *m, void *v)
 			seq_printf(m, "t2=%lld ",
 				   (long long)ktime_to_us(op->kt_ival2));
 
-		seq_printf(m, "# sent %ld\n", op->frames_abs);
+		seq_printf(m, "# sent %ld\n",
+			   atomic_long_read(&op->frames_abs));
 	}
 	seq_putc(m, '\n');
 
@@ -285,26 +296,50 @@ static int bcm_proc_show(struct seq_file *m, void *v)
 }
 #endif /* CONFIG_PROC_FS */
 
+static void bcm_update_rx_stats(struct bcm_op *op)
+{
+	/* prevent overflow of the reduction% calculation in bcm_proc_show() */
+	if (atomic_long_inc_return(&op->frames_abs) > LONG_MAX / 100) {
+		atomic_long_set(&op->frames_filtered, 0);
+		atomic_long_set(&op->frames_abs, 0);
+	}
+}
+
+static void bcm_update_tx_stats(struct bcm_op *op)
+{
+	/* tx_op has no reduction% calculation - use the full range and
+	 * just keep the displayed counter non-negative on overflow
+	 */
+	if (atomic_long_inc_return(&op->frames_abs) == LONG_MAX)
+		atomic_long_set(&op->frames_abs, 0);
+}
+
 /*
  * bcm_can_tx - send the (next) CAN frame to the appropriate CAN interface
  *              of the given bcm tx op
  */
-static void bcm_can_tx(struct bcm_op *op)
+static void bcm_can_tx(struct bcm_op *op, struct canfd_frame *cf)
 {
 	struct sk_buff *skb;
 	struct can_skb_ext *csx;
 	struct net_device *dev;
-	struct canfd_frame *cf;
+	struct canfd_frame cframe;
+	bool cyclic = !cf;
+	unsigned int idx = 0;
 	int err;
 
 	/* no target device? => exit */
 	if (!op->ifindex)
 		return;
 
-	/* read currframe under lock protection */
-	spin_lock_bh(&op->bcm_tx_lock);
-	cf = op->frames + op->cfsiz * op->currframe;
-	spin_unlock_bh(&op->bcm_tx_lock);
+	if (cyclic) {
+		/* read currframe under lock protection */
+		spin_lock_bh(&op->bcm_tx_lock);
+		idx = op->currframe;
+		memcpy(&cframe, op->frames + op->cfsiz * idx, op->cfsiz);
+		cf = &cframe;
+		spin_unlock_bh(&op->bcm_tx_lock);
+	}
 
 	dev = dev_get_by_index(sock_net(op->sk), op->ifindex);
 	if (!dev) {
@@ -335,16 +370,22 @@ static void bcm_can_tx(struct bcm_op *op)
 	spin_lock_bh(&op->bcm_tx_lock);
 
 	if (!err)
-		op->frames_abs++;
+		bcm_update_tx_stats(op);
 
-	op->currframe++;
+	/* only advance the cyclic sequence if nothing reset currframe while
+	 * we were sending - a concurrent TX_RESET_MULTI_IDX means this
+	 * frame's bookkeeping belongs to a sequence that no longer exists
+	 */
+	if (!cyclic || op->currframe == idx) {
+		op->currframe++;
 
-	/* reached last frame? */
-	if (op->currframe >= op->nframes)
-		op->currframe = 0;
+		/* reached last frame? */
+		if (op->currframe >= op->nframes)
+			op->currframe = 0;
 
-	if (op->count > 0)
-		op->count--;
+		if (op->count > 0)
+			op->count--;
+	}
 
 	spin_unlock_bh(&op->bcm_tx_lock);
 out:
@@ -433,12 +474,18 @@ static bool bcm_tx_set_expiry(struct bcm_op *op, struct hrtimer *hrt)
 {
 	ktime_t ival;
 
+	spin_lock_bh(&op->bcm_tx_lock);
+
 	if (op->kt_ival1 && op->count)
 		ival = op->kt_ival1;
-	else if (op->kt_ival2)
+	else if (op->kt_ival2) {
 		ival = op->kt_ival2;
-	else
+	} else {
+		spin_unlock_bh(&op->bcm_tx_lock);
 		return false;
+	}
+
+	spin_unlock_bh(&op->bcm_tx_lock);
 
 	hrtimer_set_expires(hrt, ktime_add(ktime_get(), ival));
 	return true;
@@ -455,26 +502,48 @@ static enum hrtimer_restart bcm_tx_timeout_handler(struct hrtimer *hrtimer)
 {
 	struct bcm_op *op = container_of(hrtimer, struct bcm_op, timer);
 	struct bcm_msg_head msg_head;
+	bool tx_ival1, tx_ival2;
+
+	/* snapshot kt_ival1/kt_ival2/count under lock to avoid torn
+	 * ktime_t reads racing with concurrent bcm_tx_setup() updates
+	 */
+	spin_lock_bh(&op->bcm_tx_lock);
+	tx_ival1 = op->kt_ival1 && (op->count > 0);
+	tx_ival2 = !!op->kt_ival2;
+	spin_unlock_bh(&op->bcm_tx_lock);
 
-	if (op->kt_ival1 && (op->count > 0)) {
-		bcm_can_tx(op);
-		if (!op->count && (op->flags & TX_COUNTEVT)) {
+	if (tx_ival1) {
+		u32 flags, count;
+		struct bcm_timeval ival1, ival2;
 
+		bcm_can_tx(op, NULL);
+
+		/* snapshot variables under lock to avoid torn reads racing
+		 * with concurrent bcm_tx_setup() updates
+		 */
+		spin_lock_bh(&op->bcm_tx_lock);
+		flags = op->flags;
+		count = op->count;
+		ival1 = op->ival1;
+		ival2 = op->ival2;
+		spin_unlock_bh(&op->bcm_tx_lock);
+
+		if (!count && (flags & TX_COUNTEVT)) {
 			/* create notification to user */
 			memset(&msg_head, 0, sizeof(msg_head));
 			msg_head.opcode  = TX_EXPIRED;
-			msg_head.flags   = op->flags;
-			msg_head.count   = op->count;
-			msg_head.ival1   = op->ival1;
-			msg_head.ival2   = op->ival2;
+			msg_head.flags   = flags;
+			msg_head.count   = count;
+			msg_head.ival1   = ival1;
+			msg_head.ival2   = ival2;
 			msg_head.can_id  = op->can_id;
 			msg_head.nframes = 0;
 
 			bcm_send_to_user(op, &msg_head, NULL, 0);
 		}
 
-	} else if (op->kt_ival2) {
-		bcm_can_tx(op);
+	} else if (tx_ival2) {
+		bcm_can_tx(op, NULL);
 	}
 
 	return bcm_tx_set_expiry(op, &op->timer) ?
@@ -488,12 +557,9 @@ static void bcm_rx_changed(struct bcm_op *op, struct canfd_frame *data)
 {
 	struct bcm_msg_head head;
 
-	/* update statistics */
-	op->frames_filtered++;
-
-	/* prevent statistics overflow */
-	if (op->frames_filtered > ULONG_MAX/100)
-		op->frames_filtered = op->frames_abs = 0;
+	/* update statistics (frames_filtered <= frames_abs) */
+	if (atomic_long_read(&op->frames_abs))
+		atomic_long_inc(&op->frames_filtered);
 
 	/* this element is not throttled anymore */
 	data->flags &= ~RX_THR;
@@ -618,6 +684,8 @@ static enum hrtimer_restart bcm_rx_timeout_handler(struct hrtimer *hrtimer)
 	struct bcm_op *op = container_of(hrtimer, struct bcm_op, timer);
 	struct bcm_msg_head msg_head;
 
+	spin_lock_bh(&op->bcm_rx_update_lock);
+
 	/* if user wants to be informed, when cyclic CAN-Messages come back */
 	if ((op->flags & RX_ANNOUNCE_RESUME) && op->last_frames) {
 		/* clear received CAN frames to indicate 'nothing received' */
@@ -634,6 +702,8 @@ static enum hrtimer_restart bcm_rx_timeout_handler(struct hrtimer *hrtimer)
 	msg_head.can_id  = op->can_id;
 	msg_head.nframes = 0;
 
+	spin_unlock_bh(&op->bcm_rx_update_lock);
+
 	bcm_send_to_user(op, &msg_head, NULL, 0);
 
 	return HRTIMER_NORESTART;
@@ -682,15 +752,26 @@ static int bcm_rx_thr_flush(struct bcm_op *op)
 static enum hrtimer_restart bcm_rx_thr_handler(struct hrtimer *hrtimer)
 {
 	struct bcm_op *op = container_of(hrtimer, struct bcm_op, thrtimer);
+	enum hrtimer_restart ret;
+
+	spin_lock_bh(&op->bcm_rx_update_lock);
 
-	if (bcm_rx_thr_flush(op)) {
+	/* kt_ival2 may have been concurrently cleared by bcm_rx_setup()
+	 * before it cancels this timer - never forward with a zero
+	 * interval in that case.
+	 */
+	if (bcm_rx_thr_flush(op) && op->kt_ival2) {
 		hrtimer_forward_now(hrtimer, op->kt_ival2);
-		return HRTIMER_RESTART;
+		ret = HRTIMER_RESTART;
 	} else {
 		/* rearm throttle handling */
 		op->kt_lastmsg = 0;
-		return HRTIMER_NORESTART;
+		ret = HRTIMER_NORESTART;
 	}
+
+	spin_unlock_bh(&op->bcm_rx_update_lock);
+
+	return ret;
 }
 
 /*
@@ -700,8 +781,10 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data)
 {
 	struct bcm_op *op = (struct bcm_op *)data;
 	const struct canfd_frame *rxframe = (struct canfd_frame *)skb->data;
+	struct canfd_frame rtrframe;
 	unsigned int i;
 	unsigned char traffic_flags;
+	bool rtr_frame;
 
 	if (op->can_id != rxframe->can_id)
 		return;
@@ -715,22 +798,59 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data)
 			return;
 	}
 
+	/* An ANYDEV op with an active RX timeout and/or throttle timer
+	 * tracks a single source interface: claim the first interface that
+	 * delivers a matching frame and reject frames from any other one,
+	 * before hrtimer_cancel() below can touch op->timer - this avoids
+	 * racing bcm_rx_timeout_handler() across concurrent interfaces.
+	 * RX_RTR_FRAME ops are excluded, as kt_ival1/kt_ival2 may briefly
+	 * hold a stale value from an earlier non-RTR configuration.
+	 */
+	if (!op->ifindex) {
+		spin_lock_bh(&op->bcm_rx_update_lock);
+
+		if (!(op->flags & RX_RTR_FRAME) &&
+		    (op->kt_ival1 || op->kt_ival2)) {
+			/* don't claim to vanishing interface */
+			if (!op->if_detected &&
+			    READ_ONCE(skb->dev->reg_state) == NETREG_REGISTERED)
+				op->if_detected = skb->dev->ifindex;
+
+			if (op->if_detected != skb->dev->ifindex) {
+				spin_unlock_bh(&op->bcm_rx_update_lock);
+				return;
+			}
+		}
+
+		spin_unlock_bh(&op->bcm_rx_update_lock);
+	}
+
 	/* disable timeout */
 	hrtimer_cancel(&op->timer);
 
-	/* save rx timestamp */
-	op->rx_stamp = skb->tstamp;
-	/* save originator for recvfrom() */
-	op->rx_ifindex = skb->dev->ifindex;
-	/* update statistics */
-	op->frames_abs++;
+	/* op->flags/op->frames may be updated concurrently by bcm_rx_setup() */
+	spin_lock_bh(&op->bcm_rx_update_lock);
+
+	rtr_frame = op->flags & RX_RTR_FRAME;
+	if (rtr_frame) {
+		bcm_update_rx_stats(op);
+		/* snapshot RTR content under lock */
+		memcpy(&rtrframe, op->frames, op->cfsiz);
+		spin_unlock_bh(&op->bcm_rx_update_lock);
 
-	if (op->flags & RX_RTR_FRAME) {
 		/* send reply for RTR-request (placed in op->frames[0]) */
-		bcm_can_tx(op);
+		bcm_can_tx(op, &rtrframe);
 		return;
 	}
 
+	/* update statistics in the same critical section as bcm_rx_changed()
+	 * below: frames_filtered must never be checked/incremented against a
+	 * frames_abs snapshot from a concurrent bcm_rx_handler() call on
+	 * another CPU for the same (wildcard) op, or frames_filtered can end
+	 * up larger than frames_abs.
+	 */
+	bcm_update_rx_stats(op);
+
 	/* compute flags to distinguish between own/local/remote CAN traffic */
 	traffic_flags = 0;
 	if (skb->sk) {
@@ -739,6 +859,13 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data)
 			traffic_flags |= RX_OWN;
 	}
 
+	/* save rx timestamp and originator for recvfrom() under lock: an
+	 * ANYDEV op without an active timer can still run concurrently on
+	 * different CPUs, so content and meta data must be bundled here.
+	 */
+	op->rx_stamp = skb->tstamp;
+	op->rx_ifindex = skb->dev->ifindex;
+
 	if (op->flags & RX_FILTER_ID) {
 		/* the easiest case */
 		bcm_rx_update_and_send(op, op->last_frames, rxframe,
@@ -774,6 +901,8 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data)
 
 rx_starttimer:
 	bcm_rx_starttimer(op);
+
+	spin_unlock_bh(&op->bcm_rx_update_lock);
 }
 
 /*
@@ -793,9 +922,12 @@ static struct bcm_op *bcm_find_op(struct list_head *ops,
 	return NULL;
 }
 
-static void bcm_free_op_rcu(struct rcu_head *rcu_head)
+static void bcm_free_op_work(struct work_struct *work)
 {
-	struct bcm_op *op = container_of(rcu_head, struct bcm_op, rcu);
+	struct bcm_op *op = container_of(work, struct bcm_op, work);
+
+	hrtimer_cancel(&op->timer);
+	hrtimer_cancel(&op->thrtimer);
 
 	if ((op->frames) && (op->frames != &op->sframe))
 		kfree(op->frames);
@@ -803,9 +935,23 @@ static void bcm_free_op_rcu(struct rcu_head *rcu_head)
 	if ((op->last_frames) && (op->last_frames != &op->last_sframe))
 		kfree(op->last_frames);
 
+	/* the last possible access to op->timer/op->thrtimer has now
+	 * happened above via hrtimer_cancel() - op->sk is no longer
+	 * needed by any pending timer callback, so drop our reference
+	 */
+	sock_put(op->sk);
+
 	kfree(op);
 }
 
+static void bcm_free_op_rcu(struct rcu_head *rcu_head)
+{
+	struct bcm_op *op = container_of(rcu_head, struct bcm_op, rcu);
+
+	INIT_WORK(&op->work, bcm_free_op_work);
+	queue_work(bcm_wq, &op->work);
+}
+
 static void bcm_remove_op(struct bcm_op *op)
 {
 	hrtimer_cancel(&op->timer);
@@ -822,6 +968,7 @@ static void bcm_rx_unreg(struct net_device *dev, struct bcm_op *op)
 
 		/* mark as removed subscription */
 		op->rx_reg_dev = NULL;
+		netdev_put(dev, &op->rx_reg_dev_tracker);
 	} else
 		printk(KERN_ERR "can-bcm: bcm_rx_unreg: registered device "
 		       "mismatch %p %p\n", op->rx_reg_dev, dev);
@@ -852,17 +999,14 @@ static int bcm_delete_rx_op(struct list_head *ops, struct bcm_msg_head *mh,
 				 * Only remove subscriptions that had not
 				 * been removed due to NETDEV_UNREGISTER
 				 * in bcm_notifier()
+				 *
+				 * op->rx_reg_dev is a tracked reference taken
+				 * when the subscription was registered, so it
+				 * stays valid here even if a concurrent
+				 * NETDEV_UNREGISTER already unlisted the dev.
 				 */
-				if (op->rx_reg_dev) {
-					struct net_device *dev;
-
-					dev = dev_get_by_index(sock_net(op->sk),
-							       op->ifindex);
-					if (dev) {
-						bcm_rx_unreg(dev, op);
-						dev_put(dev);
-					}
-				}
+				if (op->rx_reg_dev)
+					bcm_rx_unreg(op->rx_reg_dev, op);
 			} else
 				can_rx_unregister(sock_net(op->sk), NULL,
 						  op->can_id,
@@ -930,6 +1074,7 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 	struct bcm_sock *bo = bcm_sk(sk);
 	struct bcm_op *op;
 	struct canfd_frame *cf;
+	bool add_op_to_list = false;
 	unsigned int i;
 	int err;
 
@@ -948,6 +1093,8 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 	/* check the given can_id */
 	op = bcm_find_op(&bo->tx_ops, msg_head, ifindex);
 	if (op) {
+		void *new_frames;
+
 		/* update existing BCM operation */
 
 		/*
@@ -958,11 +1105,23 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 		if (msg_head->nframes > op->nframes)
 			return -E2BIG;
 
-		/* update CAN frames content */
+		/* get new CAN frames content into a staging buffer before
+		 * locking: validate and normalize the frames there so that
+		 * bcm_can_tx() / bcm_tx_timeout_handler() never observe a
+		 * partially updated or unvalidated frame in op->frames
+		 */
+		new_frames = kmalloc(msg_head->nframes * op->cfsiz, GFP_KERNEL);
+		if (!new_frames)
+			return -ENOMEM;
+
 		for (i = 0; i < msg_head->nframes; i++) {
 
-			cf = op->frames + op->cfsiz * i;
+			cf = new_frames + op->cfsiz * i;
 			err = memcpy_from_msg((u8 *)cf, msg, op->cfsiz);
+			if (err < 0) {
+				kfree(new_frames);
+				return err;
+			}
 
 			if (op->flags & CAN_FD_FRAME) {
 				if (cf->len > 64)
@@ -972,36 +1131,38 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 					err = -EINVAL;
 			}
 
-			if (err < 0)
+			if (err < 0) {
+				kfree(new_frames);
 				return err;
+			}
 
 			if (msg_head->flags & TX_CP_CAN_ID) {
 				/* copy can_id into frame */
 				cf->can_id = msg_head->can_id;
 			}
 		}
+
+		spin_lock_bh(&op->bcm_tx_lock);
+
+		/* update CAN frames content */
+		memcpy(op->frames, new_frames, msg_head->nframes * op->cfsiz);
+
 		op->flags = msg_head->flags;
 
-		/* only lock for unlikely count/nframes/currframe changes */
 		if (op->nframes != msg_head->nframes ||
-		    op->flags & TX_RESET_MULTI_IDX ||
-		    op->flags & SETTIMER) {
-
-			spin_lock_bh(&op->bcm_tx_lock);
+		    op->flags & TX_RESET_MULTI_IDX) {
+			/* potentially update changed nframes */
+			op->nframes = msg_head->nframes;
+			/* restart multiple frame transmission */
+			op->currframe = 0;
+		}
 
-			if (op->nframes != msg_head->nframes ||
-			    op->flags & TX_RESET_MULTI_IDX) {
-				/* potentially update changed nframes */
-				op->nframes = msg_head->nframes;
-				/* restart multiple frame transmission */
-				op->currframe = 0;
-			}
+		if (op->flags & SETTIMER)
+			op->count = msg_head->count;
 
-			if (op->flags & SETTIMER)
-				op->count = msg_head->count;
+		spin_unlock_bh(&op->bcm_tx_lock);
 
-			spin_unlock_bh(&op->bcm_tx_lock);
-		}
+		kfree(new_frames);
 
 	} else {
 		/* insert new BCM operation for the given can_id */
@@ -1060,6 +1221,7 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 
 		/* bcm_can_tx / bcm_tx_timeout_handler needs this */
 		op->sk = sk;
+		sock_hold(sk);
 		op->ifindex = ifindex;
 
 		/* initialize uninitialized (kzalloc) structure */
@@ -1070,17 +1232,18 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 		hrtimer_setup(&op->thrtimer, hrtimer_dummy_timeout, CLOCK_MONOTONIC,
 			      HRTIMER_MODE_REL_SOFT);
 
-		/* add this bcm_op to the list of the tx_ops */
-		list_add(&op->list, &bo->tx_ops);
+		add_op_to_list = true;
 
 	} /* if ((op = bcm_find_op(&bo->tx_ops, msg_head->can_id, ifindex))) */
 
 	if (op->flags & SETTIMER) {
 		/* set timer values */
+		spin_lock_bh(&op->bcm_tx_lock);
 		op->ival1 = msg_head->ival1;
 		op->ival2 = msg_head->ival2;
 		op->kt_ival1 = bcm_timeval_to_ktime(msg_head->ival1);
 		op->kt_ival2 = bcm_timeval_to_ktime(msg_head->ival2);
+		spin_unlock_bh(&op->bcm_tx_lock);
 
 		/* disable an active timer due to zero values? */
 		if (!op->kt_ival1 && !op->kt_ival2)
@@ -1093,8 +1256,12 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 		op->flags |= TX_ANNOUNCE;
 	}
 
+	/* add this bcm_op to the list of the tx_ops? */
+	if (add_op_to_list)
+		list_add_rcu(&op->list, &bo->tx_ops);
+
 	if (op->flags & TX_ANNOUNCE)
-		bcm_can_tx(op);
+		bcm_can_tx(op, NULL);
 
 	if (op->flags & STARTTIMER)
 		bcm_tx_start_timer(op);
@@ -1108,6 +1275,39 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 	return err;
 }
 
+static int bcm_rx_setup_rtr_check(struct bcm_msg_head *msg_head,
+				  struct bcm_op *op, void *new_frames)
+{
+	struct canfd_frame *frame0 = new_frames;
+
+	if (!(msg_head->flags & RX_RTR_FRAME))
+		return 0;
+
+	/* this frame is sent out as-is by bcm_can_tx() whenever a matching
+	 * remote request is received, so validate its length the same way
+	 * bcm_tx_setup() validates TX_SETUP frames before installing it
+	 */
+	if (msg_head->flags & CAN_FD_FRAME) {
+		if (frame0->len > 64)
+			return -EINVAL;
+	} else {
+		if (frame0->len > 8)
+			return -EINVAL;
+	}
+
+	/* funny feature in RX(!)_SETUP only for RTR-mode:
+	 * copy can_id into frame BUT without RTR-flag to
+	 * prevent a full-load-loopback-test ... ;-]
+	 * normalize this on the staged buffer, before it is
+	 * ever installed into op->frames.
+	 */
+	if ((msg_head->flags & TX_CP_CAN_ID) ||
+	    frame0->can_id == op->can_id)
+		frame0->can_id = op->can_id & ~CAN_RTR_FLAG;
+
+	return 0;
+}
+
 /*
  * bcm_rx_setup - create or update a bcm rx op (for bcm_sendmsg)
  */
@@ -1117,6 +1317,7 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 	struct bcm_sock *bo = bcm_sk(sk);
 	struct bcm_op *op;
 	int do_rx_register;
+	int new_op = 0;
 	int err = 0;
 
 	if ((msg_head->flags & RX_FILTER_ID) || (!(msg_head->nframes))) {
@@ -1142,6 +1343,8 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 	/* check the given can_id */
 	op = bcm_find_op(&bo->rx_ops, msg_head, ifindex);
 	if (op) {
+		void *new_frames = NULL;
+
 		/* update existing BCM operation */
 
 		/*
@@ -1153,21 +1356,62 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 			return -E2BIG;
 
 		if (msg_head->nframes) {
-			/* update CAN frames content */
-			err = memcpy_from_msg(op->frames, msg,
+			/* get new CAN frames content before locking */
+			new_frames = kmalloc(msg_head->nframes * op->cfsiz,
+					     GFP_KERNEL);
+			if (!new_frames)
+				return -ENOMEM;
+
+			err = memcpy_from_msg(new_frames, msg,
 					      msg_head->nframes * op->cfsiz);
-			if (err < 0)
+			if (err < 0) {
+				kfree(new_frames);
 				return err;
+			}
 
-			/* clear last_frames to indicate 'nothing received' */
-			memset(op->last_frames, 0, msg_head->nframes * op->cfsiz);
+			err = bcm_rx_setup_rtr_check(msg_head, op, new_frames);
+			if (err < 0) {
+				kfree(new_frames);
+				return err;
+			}
 		}
 
+		spin_lock_bh(&op->bcm_rx_update_lock);
 		op->nframes = msg_head->nframes;
 		op->flags = msg_head->flags;
 
-		/* Only an update -> do not call can_rx_register() */
-		do_rx_register = 0;
+		if (msg_head->nframes) {
+			/* update CAN frames content */
+			memcpy(op->frames, new_frames,
+			       msg_head->nframes * op->cfsiz);
+
+			/* clear last_frames to indicate 'nothing received' */
+			memset(op->last_frames, 0,
+			       msg_head->nframes * op->cfsiz);
+		}
+
+		if (msg_head->flags & SETTIMER) {
+			op->ival1 = msg_head->ival1;
+			op->ival2 = msg_head->ival2;
+			op->kt_ival1 = bcm_timeval_to_ktime(msg_head->ival1);
+			op->kt_ival2 = bcm_timeval_to_ktime(msg_head->ival2);
+			op->kt_lastmsg = 0;
+			op->if_detected = 0; /* reclaim ifindex in ANYDEV mode */
+		}
+		spin_unlock_bh(&op->bcm_rx_update_lock);
+
+		/* free temporary frames / kfree(NULL) is safe */
+		kfree(new_frames);
+
+		/* Don't register a new CAN filter for the rx_op update unless
+		 * a concurrent NETDEV_UNREGISTER notifier already tore down
+		 * the previous registration. In this case the receiver needs
+		 * to be re-registered here so that this update doesn't
+		 * silently stop delivering frames for the given ifindex.
+		 * Ops with ifindex = 0 (all CAN interfaces) never carry a
+		 * tracked rx_reg_dev and stay registered as-is.
+		 */
+		do_rx_register = (ifindex && !op->rx_reg_dev) ? 1 : 0;
 
 	} else {
 		/* insert new BCM operation for the given can_id */
@@ -1176,6 +1420,7 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 			return -ENOMEM;
 
 		spin_lock_init(&op->bcm_tx_lock);
+		spin_lock_init(&op->bcm_rx_update_lock);
 		op->can_id = msg_head->can_id;
 		op->nframes = msg_head->nframes;
 		op->cfsiz = CFSIZ(msg_head->flags);
@@ -1209,18 +1454,17 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 		if (msg_head->nframes) {
 			err = memcpy_from_msg(op->frames, msg,
 					      msg_head->nframes * op->cfsiz);
-			if (err < 0) {
-				if (op->frames != &op->sframe)
-					kfree(op->frames);
-				if (op->last_frames != &op->last_sframe)
-					kfree(op->last_frames);
-				kfree(op);
-				return err;
-			}
+			if (err < 0)
+				goto free_op;
+
+			err = bcm_rx_setup_rtr_check(msg_head, op, op->frames);
+			if (err < 0)
+				goto free_op;
 		}
 
 		/* bcm_can_tx / bcm_tx_timeout_handler needs this */
 		op->sk = sk;
+		sock_hold(sk);
 		op->ifindex = ifindex;
 
 		/* ifindex for timeout events w/o previous frame reception */
@@ -1232,40 +1476,31 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 		hrtimer_setup(&op->thrtimer, bcm_rx_thr_handler, CLOCK_MONOTONIC,
 			      HRTIMER_MODE_REL_SOFT);
 
-		/* add this bcm_op to the list of the rx_ops */
-		list_add(&op->list, &bo->rx_ops);
-
 		/* call can_rx_register() */
 		do_rx_register = 1;
+		new_op = 1;
 
 	} /* if ((op = bcm_find_op(&bo->rx_ops, msg_head->can_id, ifindex))) */
 
 	/* check flags */
 
 	if (op->flags & RX_RTR_FRAME) {
-		struct canfd_frame *frame0 = op->frames;
-
 		/* no timers in RTR-mode */
 		hrtimer_cancel(&op->thrtimer);
 		hrtimer_cancel(&op->timer);
-
-		/*
-		 * funny feature in RX(!)_SETUP only for RTR-mode:
-		 * copy can_id into frame BUT without RTR-flag to
-		 * prevent a full-load-loopback-test ... ;-]
-		 */
-		if ((op->flags & TX_CP_CAN_ID) ||
-		    (frame0->can_id == op->can_id))
-			frame0->can_id = op->can_id & ~CAN_RTR_FLAG;
-
 	} else {
 		if (op->flags & SETTIMER) {
 
-			/* set timer value */
-			op->ival1 = msg_head->ival1;
-			op->ival2 = msg_head->ival2;
-			op->kt_ival1 = bcm_timeval_to_ktime(msg_head->ival1);
-			op->kt_ival2 = bcm_timeval_to_ktime(msg_head->ival2);
+			/* set timers (locked) for newly created op */
+			if (new_op) {
+				spin_lock_bh(&op->bcm_rx_update_lock);
+				op->ival1 = msg_head->ival1;
+				op->ival2 = msg_head->ival2;
+				op->kt_ival1 = bcm_timeval_to_ktime(msg_head->ival1);
+				op->kt_ival2 = bcm_timeval_to_ktime(msg_head->ival2);
+				op->kt_lastmsg = 0;
+				spin_unlock_bh(&op->bcm_rx_update_lock);
+			}
 
 			/* disable an active timer due to zero value? */
 			if (!op->kt_ival1)
@@ -1275,9 +1510,11 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 			 * In any case cancel the throttle timer, flush
 			 * potentially blocked msgs and reset throttle handling
 			 */
-			op->kt_lastmsg = 0;
 			hrtimer_cancel(&op->thrtimer);
+
+			spin_lock_bh(&op->bcm_rx_update_lock);
 			bcm_rx_thr_flush(op);
+			spin_unlock_bh(&op->bcm_rx_update_lock);
 		}
 
 		if ((op->flags & STARTTIMER) && op->kt_ival1)
@@ -1285,7 +1522,10 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 				      HRTIMER_MODE_REL_SOFT);
 	}
 
-	/* now we can register for can_ids, if we added a new bcm_op */
+	/* now we can register for can_ids, if we added a new bcm_op
+	 * or need to re-register after a NETDEV_UNREGISTER tore down
+	 * the previous registration of an existing op
+	 */
 	if (do_rx_register) {
 		if (ifindex) {
 			struct net_device *dev;
@@ -1298,23 +1538,62 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg,
 						      bcm_rx_handler, op,
 						      "bcm", sk);
 
-				op->rx_reg_dev = dev;
+				/* keep a tracked reference so that a later
+				 * unregister can safely reach the device even
+				 * if a concurrent NETDEV_UNREGISTER has
+				 * already unlisted it by ifindex
+				 */
+				if (!err) {
+					op->rx_reg_dev = dev;
+					netdev_hold(dev,
+						    &op->rx_reg_dev_tracker,
+						    GFP_KERNEL);
+				}
 				dev_put(dev);
+			} else {
+				/* the requested device is gone - do not
+				 * silently succeed without registering
+				 */
+				err = -ENODEV;
 			}
 
-		} else
+		} else {
 			err = can_rx_register(sock_net(sk), NULL, op->can_id,
 					      REGMASK(op->can_id),
 					      bcm_rx_handler, op, "bcm", sk);
+		}
+
 		if (err) {
-			/* this bcm rx op is broken -> remove it */
-			list_del_rcu(&op->list);
-			bcm_remove_op(op);
+			/* newly created bcm rx op is broken -> remove it */
+			if (new_op) {
+				bcm_remove_op(op);
+				return err;
+			}
+
+			/* an existing op just stays unregistered.
+			 * Cancel op->timer and (defensively) op->thrtimer.
+			 * Other settings can't be reached until the next
+			 * successful RX_SETUP.
+			 */
+			hrtimer_cancel(&op->timer);
+			hrtimer_cancel(&op->thrtimer);
 			return err;
 		}
+
+		/* add a new bcm_op to the list of the rx_ops */
+		if (new_op)
+			list_add_rcu(&op->list, &bo->rx_ops);
 	}
 
 	return msg_head->nframes * op->cfsiz + MHSIZ;
+
+free_op:
+	if (op->frames != &op->sframe)
+		kfree(op->frames);
+	if (op->last_frames != &op->last_sframe)
+		kfree(op->last_frames);
+	kfree(op);
+	return err;
 }
 
 /*
@@ -1373,12 +1652,13 @@ static int bcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
 {
 	struct sock *sk = sock->sk;
 	struct bcm_sock *bo = bcm_sk(sk);
-	int ifindex = bo->ifindex; /* default ifindex for this bcm_op */
+	int ifindex;
 	struct bcm_msg_head msg_head;
 	int cfsiz;
 	int ret; /* read bytes or error codes as return value */
 
-	if (!bo->bound)
+	/* Lockless fast-path check for bound socket */
+	if (!READ_ONCE(bo->bound))
 		return -ENOTCONN;
 
 	/* check for valid message length from userspace */
@@ -1394,17 +1674,38 @@ static int bcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
 	if ((size - MHSIZ) % cfsiz)
 		return -EINVAL;
 
+	lock_sock(sk);
+
+	/* Re-validate under the socket lock: a concurrent bcm_notify()
+	 * may have unbound this socket (device removal) after the
+	 * lockless fast-path check above. bo->ifindex is only ever
+	 * mutated under lock_sock(), so reading it here - instead of
+	 * before taking the lock - guarantees it can't be observed
+	 * torn against bo->bound.
+	 */
+	if (!bo->bound) {
+		ret = -ENOTCONN;
+		goto out_release;
+	}
+
+	/* default ifindex for this bcm_op */
+	ifindex = bo->ifindex;
+
 	/* check for alternative ifindex for this bcm_op */
 
 	if (!ifindex && msg->msg_name) {
 		/* no bound device as default => check msg_name */
 		DECLARE_SOCKADDR(struct sockaddr_can *, addr, msg->msg_name);
 
-		if (msg->msg_namelen < BCM_MIN_NAMELEN)
-			return -EINVAL;
+		if (msg->msg_namelen < BCM_MIN_NAMELEN) {
+			ret = -EINVAL;
+			goto out_release;
+		}
 
-		if (addr->can_family != AF_CAN)
-			return -EINVAL;
+		if (addr->can_family != AF_CAN) {
+			ret = -EINVAL;
+			goto out_release;
+		}
 
 		/* ifindex from sendto() */
 		ifindex = addr->can_ifindex;
@@ -1413,20 +1714,21 @@ static int bcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
 			struct net_device *dev;
 
 			dev = dev_get_by_index(sock_net(sk), ifindex);
-			if (!dev)
-				return -ENODEV;
+			if (!dev) {
+				ret = -ENODEV;
+				goto out_release;
+			}
 
 			if (dev->type != ARPHRD_CAN) {
 				dev_put(dev);
-				return -ENODEV;
+				ret = -ENODEV;
+				goto out_release;
 			}
 
 			dev_put(dev);
 		}
 	}
 
-	lock_sock(sk);
-
 	switch (msg_head.opcode) {
 
 	case TX_SETUP:
@@ -1476,6 +1778,7 @@ static int bcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
 		break;
 	}
 
+out_release:
 	release_sock(sk);
 
 	return ret;
@@ -1499,11 +1802,30 @@ static void bcm_notify(struct bcm_sock *bo, unsigned long msg,
 	case NETDEV_UNREGISTER:
 		lock_sock(sk);
 
-		/* remove device specific receive entries */
-		list_for_each_entry(op, &bo->rx_ops, list)
+		/* rx_ops: remove device specific receive entries */
+		list_for_each_entry(op, &bo->rx_ops, list) {
 			if (op->rx_reg_dev == dev)
 				bcm_rx_unreg(dev, op);
 
+			/* release an ANYDEV op's claim (see bcm_rx_handler())
+			 * on this now confirmed-gone interface.
+			 */
+			if (!op->ifindex) {
+				spin_lock_bh(&op->bcm_rx_update_lock);
+				if (op->if_detected == dev->ifindex)
+					op->if_detected = 0;
+				spin_unlock_bh(&op->bcm_rx_update_lock);
+			}
+		}
+
+		/* tx_ops: stop device specific cyclic transmissions on the
+		 * vanishing ifindex. Cancelling the timer is enough to stop
+		 * cyclic bcm_can_tx() calls as there is no re-arming.
+		 */
+		list_for_each_entry(op, &bo->tx_ops, list)
+			if (op->ifindex == dev->ifindex)
+				hrtimer_cancel(&op->timer);
+
 		/* remove device reference, if this is our bound device */
 		if (bo->bound && bo->ifindex == dev->ifindex) {
 #if IS_ENABLED(CONFIG_PROC_FS)
@@ -1512,7 +1834,12 @@ static void bcm_notify(struct bcm_sock *bo, unsigned long msg,
 				bo->bcm_proc_read = NULL;
 			}
 #endif
-			bo->bound   = 0;
+			/* Paired with the lockless fast-path check in
+			 * bcm_sendmsg(); bo->ifindex itself is only ever
+			 * accessed under lock_sock() so it needs no
+			 * annotation.
+			 */
+			WRITE_ONCE(bo->bound, 0);
 			bo->ifindex = 0;
 			notify_enodev = 1;
 		}
@@ -1616,8 +1943,10 @@ static int bcm_release(struct socket *sock)
 		remove_proc_entry(bo->procname, net->can.bcmproc_dir);
 #endif /* CONFIG_PROC_FS */
 
-	list_for_each_entry_safe(op, next, &bo->tx_ops, list)
+	list_for_each_entry_safe(op, next, &bo->tx_ops, list) {
+		list_del_rcu(&op->list);
 		bcm_remove_op(op);
+	}
 
 	list_for_each_entry_safe(op, next, &bo->rx_ops, list) {
 		/*
@@ -1629,16 +1958,14 @@ static int bcm_release(struct socket *sock)
 			 * Only remove subscriptions that had not
 			 * been removed due to NETDEV_UNREGISTER
 			 * in bcm_notifier()
+			 *
+			 * op->rx_reg_dev is a tracked reference taken
+			 * when the subscription was registered, so it
+			 * stays valid here even if a concurrent
+			 * NETDEV_UNREGISTER already unlisted the device.
 			 */
-			if (op->rx_reg_dev) {
-				struct net_device *dev;
-
-				dev = dev_get_by_index(net, op->ifindex);
-				if (dev) {
-					bcm_rx_unreg(dev, op);
-					dev_put(dev);
-				}
-			}
+			if (op->rx_reg_dev)
+				bcm_rx_unreg(op->rx_reg_dev, op);
 		} else
 			can_rx_unregister(net, NULL, op->can_id,
 					  REGMASK(op->can_id),
@@ -1648,12 +1975,14 @@ static int bcm_release(struct socket *sock)
 
 	synchronize_rcu();
 
-	list_for_each_entry_safe(op, next, &bo->rx_ops, list)
+	list_for_each_entry_safe(op, next, &bo->rx_ops, list) {
+		list_del_rcu(&op->list);
 		bcm_remove_op(op);
+	}
 
 	/* remove device reference */
 	if (bo->bound) {
-		bo->bound   = 0;
+		WRITE_ONCE(bo->bound, 0);
 		bo->ifindex = 0;
 	}
 
@@ -1723,7 +2052,10 @@ static int bcm_connect(struct socket *sock, struct sockaddr_unsized *uaddr, int
 	}
 #endif /* CONFIG_PROC_FS */
 
-	bo->bound = 1;
+	/* bo->ifindex above is fully assigned before this point; pairs
+	 * with the lockless fast-path check in bcm_sendmsg()
+	 */
+	WRITE_ONCE(bo->bound, 1);
 
 fail:
 	release_sock(sk);
@@ -1839,11 +2171,15 @@ static int __init bcm_module_init(void)
 {
 	int err;
 
+	bcm_wq = alloc_workqueue("can-bcm-wq", WQ_UNBOUND, 0);
+	if (!bcm_wq)
+		return -ENOMEM;
+
 	pr_info("can: broadcast manager protocol\n");
 
 	err = register_pernet_subsys(&canbcm_pernet_ops);
 	if (err)
-		return err;
+		goto register_pernet_failed;
 
 	err = register_netdevice_notifier(&canbcm_notifier);
 	if (err)
@@ -1861,6 +2197,8 @@ static int __init bcm_module_init(void)
 	unregister_netdevice_notifier(&canbcm_notifier);
 register_notifier_failed:
 	unregister_pernet_subsys(&canbcm_pernet_ops);
+register_pernet_failed:
+	destroy_workqueue(bcm_wq);
 	return err;
 }
 
@@ -1869,6 +2207,8 @@ static void __exit bcm_module_exit(void)
 	can_proto_unregister(&bcm_can_proto);
 	unregister_netdevice_notifier(&canbcm_notifier);
 	unregister_pernet_subsys(&canbcm_pernet_ops);
+	rcu_barrier();
+	destroy_workqueue(bcm_wq);
 }
 
 module_init(bcm_module_init);
diff --git a/net/can/isotp.c b/net/can/isotp.c
index c48b4a818297e..54becaf6898f1 100644
--- a/net/can/isotp.c
+++ b/net/can/isotp.c
@@ -152,11 +152,13 @@ struct isotp_sock {
 	struct sock sk;
 	int bound;
 	int ifindex;
+	struct net_device *dev;
+	netdevice_tracker dev_tracker;
 	canid_t txid;
 	canid_t rxid;
 	ktime_t tx_gap;
 	ktime_t lastrxcf_tstamp;
-	struct hrtimer rxtimer, txtimer, txfrtimer;
+	struct hrtimer rxtimer, txtimer, txfrtimer, echotimer;
 	struct can_isotp_options opt;
 	struct can_isotp_fc_options rxfc, txfc;
 	struct can_isotp_ll_options ll;
@@ -164,6 +166,7 @@ struct isotp_sock {
 	u32 force_tx_stmin;
 	u32 force_rx_stmin;
 	u32 cfecho; /* consecutive frame echo tag */
+	u32 tx_gen; /* generation, bumped per new tx transfer */
 	struct tpcon rx, tx;
 	struct list_head notifier;
 	wait_queue_head_t wait;
@@ -376,6 +379,15 @@ static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae)
 
 	hrtimer_cancel(&so->txtimer);
 
+	/* isotp_tx_timeout() may have given up on this job while
+	 * hrtimer_cancel() above waited for it to finish; so->rx_lock
+	 * (held by our caller isotp_rcv()) rules out a concurrent claim,
+	 * so a plain recheck is enough here.
+	 */
+	if (so->tx.state != ISOTP_WAIT_FC &&
+	    so->tx.state != ISOTP_WAIT_FIRST_FC)
+		return 1;
+
 	if ((cf->len < ae + FC_CONTENT_SZ) ||
 	    ((so->opt.flags & ISOTP_CHECK_PADDING) &&
 	     check_pad(so, cf, ae + FC_CONTENT_SZ, so->opt.rxpad_content))) {
@@ -422,7 +434,7 @@ static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae)
 		so->tx.bs = 0;
 		so->tx.state = ISOTP_SENDING;
 		/* send CF frame and enable echo timeout handling */
-		hrtimer_start(&so->txtimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0),
+		hrtimer_start(&so->echotimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0),
 			      HRTIMER_MODE_REL_SOFT);
 		isotp_send_cframe(so);
 		break;
@@ -575,6 +587,14 @@ static int isotp_rcv_cf(struct sock *sk, struct canfd_frame *cf, int ae,
 
 	hrtimer_cancel(&so->rxtimer);
 
+	/* isotp_rx_timer_handler() may have raced us for so->rx.state
+	 * while hrtimer_cancel() above waited for it to finish, already
+	 * reporting ETIMEDOUT and resetting the reception; don't process
+	 * this CF into a reassembly that has already been given up on.
+	 */
+	if (so->rx.state != ISOTP_WAIT_DATA)
+		return 1;
+
 	/* CFs are never longer than the FF */
 	if (cf->len > so->rx.ll_dl)
 		return 1;
@@ -870,20 +890,36 @@ static void isotp_rcv_echo(struct sk_buff *skb, void *data)
 	struct canfd_frame *cf = (struct canfd_frame *)skb->data;
 
 	/* only handle my own local echo CF/SF skb's (no FF!) */
-	if (skb->sk != sk || so->cfecho != *(u32 *)cf->data)
+	if (skb->sk != sk)
 		return;
 
+	/* unlike isotp_rcv_fc()/isotp_rcv_cf(), not already under so->rx_lock
+	 * (no isotp_rcv() caller here), so take it ourselves
+	 */
+	spin_lock(&so->rx_lock);
+
+	/* so->cfecho may since belong to a new transfer; recheck under lock */
+	if (so->cfecho != *(u32 *)cf->data)
+		goto out_unlock;
+
 	/* cancel local echo timeout */
-	hrtimer_cancel(&so->txtimer);
+	hrtimer_cancel(&so->echotimer);
 
 	/* local echo skb with consecutive frame has been consumed */
 	so->cfecho = 0;
 
+	/* claiming a transfer also takes so->rx_lock, so a plain recheck
+	 * is enough: so->tx.state can't have flipped to ISOTP_SENDING for
+	 * a new claim while we're still in here
+	 */
+	if (so->tx.state != ISOTP_SENDING)
+		goto out_unlock;
+
 	if (so->tx.idx >= so->tx.len) {
 		/* we are done */
 		so->tx.state = ISOTP_IDLE;
 		wake_up_interruptible(&so->wait);
-		return;
+		goto out_unlock;
 	}
 
 	if (so->txfc.bs && so->tx.bs >= so->txfc.bs) {
@@ -891,53 +927,83 @@ static void isotp_rcv_echo(struct sk_buff *skb, void *data)
 		so->tx.state = ISOTP_WAIT_FC;
 		hrtimer_start(&so->txtimer, ktime_set(ISOTP_FC_TIMEOUT, 0),
 			      HRTIMER_MODE_REL_SOFT);
-		return;
+		goto out_unlock;
 	}
 
 	/* no gap between data frames needed => use burst mode */
 	if (!so->tx_gap) {
 		/* enable echo timeout handling */
-		hrtimer_start(&so->txtimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0),
+		hrtimer_start(&so->echotimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0),
 			      HRTIMER_MODE_REL_SOFT);
 		isotp_send_cframe(so);
-		return;
+		goto out_unlock;
 	}
 
 	/* start timer to send next consecutive frame with correct delay */
 	hrtimer_start(&so->txfrtimer, so->tx_gap, HRTIMER_MODE_REL_SOFT);
+
+out_unlock:
+	spin_unlock(&so->rx_lock);
 }
 
-static enum hrtimer_restart isotp_tx_timer_handler(struct hrtimer *hrtimer)
+/* shared by so->txtimer's and so->echotimer's callbacks. Both timers get
+ * cancelled under so->rx_lock elsewhere, so this must stay lock-free to
+ * avoid deadlocking with that; uses so->tx_gen instead to avoid tainting
+ * a new transfer with an error from the one that just timed out.
+ */
+static enum hrtimer_restart isotp_tx_timeout(struct isotp_sock *so)
 {
-	struct isotp_sock *so = container_of(hrtimer, struct isotp_sock,
-					     txtimer);
 	struct sock *sk = &so->sk;
+	u32 gen = READ_ONCE(so->tx_gen);
+	u32 old_state = READ_ONCE(so->tx.state);
 
 	/* don't handle timeouts in IDLE or SHUTDOWN state */
-	if (so->tx.state == ISOTP_IDLE || so->tx.state == ISOTP_SHUTDOWN)
+	if (old_state == ISOTP_IDLE || old_state == ISOTP_SHUTDOWN)
+		return HRTIMER_NORESTART;
+
+	/* only claim the timeout if the state is still unchanged */
+	if (cmpxchg(&so->tx.state, old_state, ISOTP_IDLE) != old_state)
 		return HRTIMER_NORESTART;
 
 	/* we did not get any flow control or echo frame in time */
 
-	/* report 'communication error on send' */
-	sk->sk_err = ECOMM;
-	if (!sock_flag(sk, SOCK_DEAD))
-		sk_error_report(sk);
+	if (READ_ONCE(so->tx_gen) == gen) {
+		/* report 'communication error on send' */
+		sk->sk_err = ECOMM;
+		if (!sock_flag(sk, SOCK_DEAD))
+			sk_error_report(sk);
+	}
 
-	/* reset tx state */
-	so->tx.state = ISOTP_IDLE;
 	wake_up_interruptible(&so->wait);
 
 	return HRTIMER_NORESTART;
 }
 
+/* so->txtimer: fires when a Flow Control frame does not arrive in time */
+static enum hrtimer_restart isotp_tx_timer_handler(struct hrtimer *hrtimer)
+{
+	struct isotp_sock *so = container_of(hrtimer, struct isotp_sock,
+					     txtimer);
+
+	return isotp_tx_timeout(so);
+}
+
+/* so->echotimer: fires when a sent CF/SF's local echo does not arrive */
+static enum hrtimer_restart isotp_echo_timer_handler(struct hrtimer *hrtimer)
+{
+	struct isotp_sock *so = container_of(hrtimer, struct isotp_sock,
+					     echotimer);
+
+	return isotp_tx_timeout(so);
+}
+
 static enum hrtimer_restart isotp_txfr_timer_handler(struct hrtimer *hrtimer)
 {
 	struct isotp_sock *so = container_of(hrtimer, struct isotp_sock,
 					     txfrtimer);
 
 	/* start echo timeout handling and cover below protocol error */
-	hrtimer_start(&so->txtimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0),
+	hrtimer_start(&so->echotimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0),
 		      HRTIMER_MODE_REL_SOFT);
 
 	/* cfecho should be consumed by isotp_rcv_echo() here */
@@ -958,13 +1024,24 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
 	int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR) ? 1 : 0;
 	int wait_tx_done = (so->opt.flags & CAN_ISOTP_WAIT_TX_DONE) ? 1 : 0;
 	s64 hrtimer_sec = ISOTP_ECHO_TIMEOUT;
+	struct hrtimer *tx_hrt = &so->echotimer;
+	u32 new_state = ISOTP_SENDING;
 	int off;
 	int err;
 
 	if (!so->bound || so->tx.state == ISOTP_SHUTDOWN)
 		return -EADDRNOTAVAIL;
 
-	while (cmpxchg(&so->tx.state, ISOTP_IDLE, ISOTP_SENDING) != ISOTP_IDLE) {
+	/* claim the socket under so->rx_lock: this serializes the claim
+	 * with the RX path and with sendmsg()'s own error paths below, so
+	 * none of them can ever see a transfer mid-claim
+	 */
+	for (;;) {
+		spin_lock_bh(&so->rx_lock);
+		if (READ_ONCE(so->tx.state) == ISOTP_IDLE)
+			break;
+		spin_unlock_bh(&so->rx_lock);
+
 		/* we do not support multiple buffers - for now */
 		if (msg->msg_flags & MSG_DONTWAIT)
 			return -EAGAIN;
@@ -973,9 +1050,29 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
 			return -EADDRNOTAVAIL;
 
 		/* wait for complete transmission of current pdu */
-		err = wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE);
+		err = wait_event_interruptible(so->wait,
+					       so->tx.state == ISOTP_IDLE);
 		if (err)
-			goto err_event_drop;
+			return err;
+	}
+
+	/* new transfer: bump so->tx_gen and drain the old one's timers,
+	 * still under the so->rx_lock we just claimed the socket with
+	 */
+	WRITE_ONCE(so->tx.state, ISOTP_SENDING);
+	WRITE_ONCE(so->tx_gen, READ_ONCE(so->tx_gen) + 1);
+	hrtimer_cancel(&so->txtimer);
+	hrtimer_cancel(&so->echotimer);
+	hrtimer_cancel(&so->txfrtimer);
+	so->cfecho = 0;
+	spin_unlock_bh(&so->rx_lock);
+
+	/* so->bound is only checked once above - a wakeup may have
+	 * unbound/rebound the socket meanwhile, so re-validate it
+	 */
+	if (!so->bound) {
+		err = -EADDRNOTAVAIL;
+		goto err_out_drop;
 	}
 
 	/* PDU size > default => try max_pdu_size */
@@ -1086,18 +1183,33 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
 			so->cfecho = *(u32 *)cf->data;
 		} else {
 			/* standard flow control check */
-			so->tx.state = ISOTP_WAIT_FIRST_FC;
+			new_state = ISOTP_WAIT_FIRST_FC;
 
 			/* start timeout for FC */
 			hrtimer_sec = ISOTP_FC_TIMEOUT;
+			tx_hrt = &so->txtimer;
 
 			/* no CF echo tag for isotp_rcv_echo() (FF-mode) */
 			so->cfecho = 0;
 		}
 	}
 
-	hrtimer_start(&so->txtimer, ktime_set(hrtimer_sec, 0),
+	spin_lock_bh(&so->rx_lock);
+	if (so->tx.state == ISOTP_SHUTDOWN) {
+		/* isotp_release() has since taken over and already drained
+		 * our timers - don't send into a socket that's going away
+		 */
+		spin_unlock_bh(&so->rx_lock);
+		kfree_skb(skb);
+		dev_put(dev);
+		wake_up_interruptible(&so->wait);
+		return -EADDRNOTAVAIL;
+	}
+	/* WAIT_FIRST_FC for standard FF, else stays ISOTP_SENDING */
+	so->tx.state = new_state;
+	hrtimer_start(tx_hrt, ktime_set(hrtimer_sec, 0),
 		      HRTIMER_MODE_REL_SOFT);
+	spin_unlock_bh(&so->rx_lock);
 
 	/* send the first or only CAN frame */
 	cf->flags = so->ll.tx_flags;
@@ -1110,13 +1222,10 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
 		pr_notice_once("can-isotp: %s: can_send_ret %pe\n",
 			       __func__, ERR_PTR(err));
 
+		spin_lock_bh(&so->rx_lock);
 		/* no transmission -> no timeout monitoring */
-		hrtimer_cancel(&so->txtimer);
-
-		/* reset consecutive frame echo tag */
-		so->cfecho = 0;
-
-		goto err_out_drop;
+		hrtimer_cancel(tx_hrt);
+		goto err_out_drop_locked;
 	}
 
 	if (wait_tx_done) {
@@ -1132,14 +1241,21 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
 
 	return size;
 
+err_out_drop:
+	/* claimed but nothing sent yet - no timer to cancel */
+	spin_lock_bh(&so->rx_lock);
+	goto err_out_drop_locked;
 err_event_drop:
-	/* got signal: force tx state machine to be idle */
-	so->tx.state = ISOTP_IDLE;
+	/* interrupted waiting on our own transfer - drain its timers */
+	spin_lock_bh(&so->rx_lock);
 	hrtimer_cancel(&so->txfrtimer);
 	hrtimer_cancel(&so->txtimer);
-err_out_drop:
-	/* drop this PDU and unlock a potential wait queue */
+	hrtimer_cancel(&so->echotimer);
+err_out_drop_locked:
+	/* release the claim; so->rx_lock still held from above */
+	so->cfecho = 0;
 	so->tx.state = ISOTP_IDLE;
+	spin_unlock_bh(&so->rx_lock);
 	wake_up_interruptible(&so->wait);
 
 	return err;
@@ -1201,13 +1317,20 @@ static int isotp_release(struct socket *sock)
 	so = isotp_sk(sk);
 	net = sock_net(sk);
 
-	/* wait for complete transmission of current pdu */
-	while (wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE) == 0 &&
-	       cmpxchg(&so->tx.state, ISOTP_IDLE, ISOTP_SHUTDOWN) != ISOTP_IDLE)
+	/* best-effort: wait for a running pdu to finish, but don't block on
+	 * it forever - give up after the first signal
+	 */
+	while (so->tx.state != ISOTP_IDLE &&
+	       wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE) == 0)
 		;
 
-	/* force state machines to be idle also when a signal occurred */
+	/* claim the socket under so->rx_lock like sendmsg() does, so its
+	 * claim can't race the forced ISOTP_SHUTDOWN below; force it
+	 * unconditionally, even when a signal cut the wait above short
+	 */
+	spin_lock_bh(&so->rx_lock);
 	so->tx.state = ISOTP_SHUTDOWN;
+	spin_unlock_bh(&so->rx_lock);
 	so->rx.state = ISOTP_IDLE;
 
 	spin_lock(&isotp_notifier_lock);
@@ -1219,36 +1342,43 @@ static int isotp_release(struct socket *sock)
 	list_del(&so->notifier);
 	spin_unlock(&isotp_notifier_lock);
 
+	rtnl_lock();
 	lock_sock(sk);
 
-	/* remove current filters & unregister */
-	if (so->bound) {
-		if (so->ifindex) {
-			struct net_device *dev;
-
-			dev = dev_get_by_index(net, so->ifindex);
-			if (dev) {
-				if (isotp_register_rxid(so))
-					can_rx_unregister(net, dev, so->rxid,
-							  SINGLE_MASK(so->rxid),
-							  isotp_rcv, sk);
-
-				can_rx_unregister(net, dev, so->txid,
-						  SINGLE_MASK(so->txid),
-						  isotp_rcv_echo, sk);
-				dev_put(dev);
-				synchronize_rcu();
-			}
-		}
+	/* remove current filters & unregister
+	 * tracked reference so->dev is taken at bind() time with rtnl_lock
+	 */
+	if (so->bound && so->dev) {
+		if (isotp_register_rxid(so))
+			can_rx_unregister(net, so->dev, so->rxid,
+					  SINGLE_MASK(so->rxid),
+					  isotp_rcv, sk);
+
+		can_rx_unregister(net, so->dev, so->txid,
+				  SINGLE_MASK(so->txid),
+				  isotp_rcv_echo, sk);
+		netdev_put(so->dev, &so->dev_tracker);
 	}
 
+	so->ifindex = 0;
+	so->bound = 0;
+	so->dev = NULL;
+
+	rtnl_unlock();
+
+	/* Always wait for a grace period before touching the timers below.
+	 * A concurrent NETDEV_UNREGISTER may have already unregistered our
+	 * filters and cleared so->bound in isotp_notify() without waiting
+	 * for in-flight isotp_rcv() callers to finish, so this call must not
+	 * be skipped just because so->bound is already 0 here.
+	 */
+	synchronize_rcu();
+
 	hrtimer_cancel(&so->txfrtimer);
 	hrtimer_cancel(&so->txtimer);
+	hrtimer_cancel(&so->echotimer);
 	hrtimer_cancel(&so->rxtimer);
 
-	so->ifindex = 0;
-	so->bound = 0;
-
 	sock_orphan(sk);
 	sock->sk = NULL;
 
@@ -1303,6 +1433,7 @@ static int isotp_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int l
 	if (!addr->can_ifindex)
 		return -ENODEV;
 
+	rtnl_lock();
 	lock_sock(sk);
 
 	if (so->bound) {
@@ -1310,6 +1441,17 @@ static int isotp_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int l
 		goto out;
 	}
 
+	/* A transmission or reception that outlived a previous binding
+	 * (unbound by NETDEV_UNREGISTER) may still be draining; the FC/echo
+	 * and RX watchdog timers bound how long this takes. Checked together
+	 * with so->bound in the same lock_sock() section above, so there is
+	 * no window in which a concurrent isotp_notify() could be missed.
+	 */
+	if (so->tx.state != ISOTP_IDLE || so->rx.state != ISOTP_IDLE) {
+		err = -EAGAIN;
+		goto out;
+	}
+
 	/* ensure different CAN IDs when the rx_id is to be registered */
 	if (isotp_register_rxid(so) && rx_id == tx_id) {
 		err = -EADDRNOTAVAIL;
@@ -1322,14 +1464,12 @@ static int isotp_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int l
 		goto out;
 	}
 	if (dev->type != ARPHRD_CAN) {
-		dev_put(dev);
 		err = -ENODEV;
-		goto out;
+		goto out_put_dev;
 	}
 	if (READ_ONCE(dev->mtu) < so->ll.mtu) {
-		dev_put(dev);
 		err = -EINVAL;
-		goto out;
+		goto out_put_dev;
 	}
 	if (!(dev->flags & IFF_UP))
 		notify_enetdown = 1;
@@ -1347,16 +1487,25 @@ static int isotp_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int l
 	can_rx_register(net, dev, tx_id, SINGLE_MASK(tx_id),
 			isotp_rcv_echo, sk, "isotpe", sk);
 
-	dev_put(dev);
-
 	/* switch to new settings */
 	so->ifindex = ifindex;
 	so->rxid = rx_id;
 	so->txid = tx_id;
 	so->bound = 1;
 
+	/* bind() ok -> hold a reference for so->dev so that isotp_release()
+	 * can safely reach the device later, even if a concurrent
+	 * NETDEV_UNREGISTER has already unlisted it by ifindex.
+	 */
+	so->dev = dev;
+	netdev_hold(so->dev, &so->dev_tracker, GFP_KERNEL);
+
+out_put_dev:
+	/* remove potential reference from dev_get_by_index() */
+	dev_put(dev);
 out:
 	release_sock(sk);
+	rtnl_unlock();
 
 	if (notify_enetdown) {
 		sk->sk_err = ENETDOWN;
@@ -1559,7 +1708,7 @@ static void isotp_notify(struct isotp_sock *so, unsigned long msg,
 	if (!net_eq(dev_net(dev), sock_net(sk)))
 		return;
 
-	if (so->ifindex != dev->ifindex)
+	if (so->dev != dev)
 		return;
 
 	switch (msg) {
@@ -1575,10 +1724,12 @@ static void isotp_notify(struct isotp_sock *so, unsigned long msg,
 			can_rx_unregister(dev_net(dev), dev, so->txid,
 					  SINGLE_MASK(so->txid),
 					  isotp_rcv_echo, sk);
+			netdev_put(so->dev, &so->dev_tracker);
 		}
 
 		so->ifindex = 0;
 		so->bound  = 0;
+		so->dev = NULL;
 		release_sock(sk);
 
 		sk->sk_err = ENODEV;
@@ -1638,6 +1789,7 @@ static int isotp_init(struct sock *sk)
 
 	so->ifindex = 0;
 	so->bound = 0;
+	so->dev = NULL;
 
 	so->opt.flags = CAN_ISOTP_DEFAULT_FLAGS;
 	so->opt.ext_address = CAN_ISOTP_DEFAULT_EXT_ADDRESS;
@@ -1664,10 +1816,14 @@ static int isotp_init(struct sock *sk)
 	so->rx.buflen = ARRAY_SIZE(so->rx.sbuf);
 	so->tx.buflen = ARRAY_SIZE(so->tx.sbuf);
 
-	hrtimer_setup(&so->rxtimer, isotp_rx_timer_handler, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT);
-	hrtimer_setup(&so->txtimer, isotp_tx_timer_handler, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT);
-	hrtimer_setup(&so->txfrtimer, isotp_txfr_timer_handler, CLOCK_MONOTONIC,
-		      HRTIMER_MODE_REL_SOFT);
+	hrtimer_setup(&so->rxtimer, isotp_rx_timer_handler,
+		      CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT);
+	hrtimer_setup(&so->txtimer, isotp_tx_timer_handler,
+		      CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT);
+	hrtimer_setup(&so->echotimer, isotp_echo_timer_handler,
+		      CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT);
+	hrtimer_setup(&so->txfrtimer, isotp_txfr_timer_handler,
+		      CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT);
 
 	init_waitqueue_head(&so->wait);
 	spin_lock_init(&so->rx_lock);
diff --git a/net/can/j1939/transport.c b/net/can/j1939/transport.c
index df93d57907da7..8a31cb23bc76d 100644
--- a/net/can/j1939/transport.c
+++ b/net/can/j1939/transport.c
@@ -351,6 +351,18 @@ static void j1939_session_skb_drop_old(struct j1939_session *session)
 	}
 }
 
+static bool j1939_address_is_local(struct j1939_priv *priv, u8 addr)
+{
+	bool local = false;
+
+	read_lock_bh(&priv->lock);
+	if (j1939_address_is_unicast(addr) && priv->ents[addr].nusers)
+		local = true;
+	read_unlock_bh(&priv->lock);
+
+	return local;
+}
+
 void j1939_session_skb_queue(struct j1939_session *session,
 			     struct sk_buff *skb)
 {
@@ -359,8 +371,7 @@ void j1939_session_skb_queue(struct j1939_session *session,
 
 	j1939_ac_fixup(priv, skb);
 
-	if (j1939_address_is_unicast(skcb->addr.da) &&
-	    priv->ents[skcb->addr.da].nusers)
+	if (j1939_address_is_local(priv, skcb->addr.da))
 		skcb->flags |= J1939_ECU_LOCAL_DST;
 
 	skcb->flags |= J1939_ECU_LOCAL_SRC;
@@ -2038,8 +2049,7 @@ struct j1939_session *j1939_tp_send(struct j1939_priv *priv,
 		return ERR_PTR(ret);
 
 	/* fix DST flags, it may be used there soon */
-	if (j1939_address_is_unicast(skcb->addr.da) &&
-	    priv->ents[skcb->addr.da].nusers)
+	if (j1939_address_is_local(priv, skcb->addr.da))
 		skcb->flags |= J1939_ECU_LOCAL_DST;
 
 	/* src is always local, I'm sending ... */
diff --git a/net/can/raw.c b/net/can/raw.c
index a26942e78e688..82d9c0499c95f 100644
--- a/net/can/raw.c
+++ b/net/can/raw.c
@@ -562,8 +562,8 @@ static int raw_getname(struct socket *sock, struct sockaddr *uaddr,
 	return RAW_MIN_NAMELEN;
 }
 
-static int raw_setsockopt(struct socket *sock, int level, int optname,
-			  sockptr_t optval, unsigned int optlen)
+static int raw_setsockopt_locked(struct socket *sock, int optname,
+				 sockptr_t optval, unsigned int optlen)
 {
 	struct sock *sk = sock->sk;
 	struct raw_sock *ro = raw_sk(sk);
@@ -575,9 +575,6 @@ static int raw_setsockopt(struct socket *sock, int level, int optname,
 	int flag;
 	int err = 0;
 
-	if (level != SOL_CAN_RAW)
-		return -EINVAL;
-
 	switch (optname) {
 	case CAN_RAW_FILTER:
 		if (optlen % sizeof(struct can_filter) != 0)
@@ -598,17 +595,11 @@ static int raw_setsockopt(struct socket *sock, int level, int optname,
 				return -EFAULT;
 		}
 
-		rtnl_lock();
-		lock_sock(sk);
-
 		dev = ro->dev;
-		if (ro->bound && dev) {
-			if (dev->reg_state != NETREG_REGISTERED) {
-				if (count > 1)
-					kfree(filter);
-				err = -ENODEV;
-				goto out_fil;
-			}
+		if (ro->bound && dev && dev->reg_state != NETREG_REGISTERED) {
+			if (count > 1)
+				kfree(filter);
+			return -ENODEV;
 		}
 
 		if (ro->bound) {
@@ -622,7 +613,7 @@ static int raw_setsockopt(struct socket *sock, int level, int optname,
 			if (err) {
 				if (count > 1)
 					kfree(filter);
-				goto out_fil;
+				return err;
 			}
 
 			/* remove old filter registrations */
@@ -642,11 +633,6 @@ static int raw_setsockopt(struct socket *sock, int level, int optname,
 		}
 		ro->filter = filter;
 		ro->count  = count;
-
- out_fil:
-		release_sock(sk);
-		rtnl_unlock();
-
 		break;
 
 	case CAN_RAW_ERR_FILTER:
@@ -658,16 +644,9 @@ static int raw_setsockopt(struct socket *sock, int level, int optname,
 
 		err_mask &= CAN_ERR_MASK;
 
-		rtnl_lock();
-		lock_sock(sk);
-
 		dev = ro->dev;
-		if (ro->bound && dev) {
-			if (dev->reg_state != NETREG_REGISTERED) {
-				err = -ENODEV;
-				goto out_err;
-			}
-		}
+		if (ro->bound && dev && dev->reg_state != NETREG_REGISTERED)
+			return -ENODEV;
 
 		/* remove current error mask */
 		if (ro->bound) {
@@ -676,7 +655,7 @@ static int raw_setsockopt(struct socket *sock, int level, int optname,
 						   err_mask);
 
 			if (err)
-				goto out_err;
+				return err;
 
 			/* remove old err_mask registration */
 			raw_disable_errfilter(sock_net(sk), dev, sk,
@@ -685,11 +664,6 @@ static int raw_setsockopt(struct socket *sock, int level, int optname,
 
 		/* link new err_mask to the socket */
 		ro->err_mask = err_mask;
-
- out_err:
-		release_sock(sk);
-		rtnl_unlock();
-
 		break;
 
 	case CAN_RAW_LOOPBACK:
@@ -769,6 +743,26 @@ static int raw_setsockopt(struct socket *sock, int level, int optname,
 	return err;
 }
 
+static int raw_setsockopt(struct socket *sock, int level, int optname,
+			  sockptr_t optval, unsigned int optlen)
+{
+	struct sock *sk = sock->sk;
+	int err;
+
+	if (level != SOL_CAN_RAW)
+		return -EINVAL;
+
+	rtnl_lock();
+	lock_sock(sk);
+
+	err = raw_setsockopt_locked(sock, optname, optval, optlen);
+
+	release_sock(sk);
+	rtnl_unlock();
+
+	return err;
+}
+
 static int raw_getsockopt(struct socket *sock, int level, int optname,
 			  sockopt_t *opt)
 {
diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c
index ecd659f79fd4a..1d295a8769fad 100644
--- a/net/core/bpf_sk_storage.c
+++ b/net/core/bpf_sk_storage.c
@@ -158,8 +158,6 @@ int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
 	struct bpf_local_storage_elem *selem;
 	int ret = 0;
 
-	RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
-
 	rcu_read_lock_dont_migrate();
 	sk_storage = rcu_dereference(sk->sk_bpf_storage);
 
diff --git a/net/core/dev.c b/net/core/dev.c
index 4b3d5cfdf6e00..c1c1be1a6962f 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -1802,6 +1802,7 @@ void netif_close_many(struct list_head *head, bool unlink)
 	__dev_close_many(head);
 
 	list_for_each_entry_safe(dev, tmp, head, close_list) {
+		netdev_assert_locked_ops_compat(dev);
 		rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL);
 		call_netdevice_notifiers(NETDEV_DOWN, dev);
 		if (unlink)
@@ -1912,9 +1913,11 @@ static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
 						struct net_device *dev)
 {
 	if (dev->flags & IFF_UP) {
+		netdev_lock_ops(dev);
 		call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
 					dev);
 		call_netdevice_notifier(nb, NETDEV_DOWN, dev);
+		netdev_unlock_ops(dev);
 	}
 	call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
 }
@@ -4018,6 +4021,9 @@ static struct sk_buff *validate_xmit_unreadable_skb(struct sk_buff *skb,
 	return NULL;
 }
 
+/* Returns the skb on success, NULL if dropped, or ERR_PTR(-EINPROGRESS)
+ * if stolen by async xfrm crypto (delivered via xfrm_dev_resume()).
+ */
 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
 {
 	netdev_features_t features;
@@ -4089,7 +4095,7 @@ struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *d
 		skb->prev = skb;
 
 		skb = validate_xmit_skb(skb, dev, again);
-		if (!skb)
+		if (IS_ERR_OR_NULL(skb))
 			continue;
 
 		if (!head)
@@ -4860,8 +4866,11 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
 			goto recursion_alert;
 
 		skb = validate_xmit_skb(skb, dev, &again);
-		if (!skb)
+		if (IS_ERR_OR_NULL(skb)) {
+			if (PTR_ERR(skb) == -EINPROGRESS)
+				rc = NET_XMIT_SUCCESS;
 			goto out;
+		}
 
 		HARD_TX_LOCK(dev, txq, cpu);
 
@@ -9785,6 +9794,8 @@ void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
 {
 	unsigned int changes = dev->flags ^ old_flags;
 
+	netdev_assert_locked_ops_compat(dev);
+
 	if (gchanges)
 		rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC, portid, nlh);
 
@@ -11608,8 +11619,13 @@ static struct net_device *netdev_wait_allrefs_any(struct list_head *list)
 			rtnl_lock();
 
 			/* Rebroadcast unregister notification */
-			list_for_each_entry(dev, list, todo_list)
+			list_for_each_entry(dev, list, todo_list) {
+				struct net *net = dev_net(dev);
+
+				__rtnl_net_lock(net);
 				call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
+				__rtnl_net_unlock(net);
+			}
 
 			__rtnl_unlock();
 			rcu_barrier();
@@ -12087,6 +12103,9 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
 
 	INIT_LIST_HEAD(&dev->napi_list);
 	INIT_LIST_HEAD(&dev->unreg_list);
+#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
+	INIT_LIST_HEAD(&dev->unreg_list_net);
+#endif
 	INIT_LIST_HEAD(&dev->close_list);
 	INIT_LIST_HEAD(&dev->link_watch_list);
 	INIT_LIST_HEAD(&dev->adj_list.upper);
@@ -12304,6 +12323,10 @@ void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
 {
 	ASSERT_RTNL();
 
+#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
+	DEBUG_NET_WARN_ON_ONCE(!list_empty(&dev->unreg_list_net));
+#endif
+
 	if (head) {
 		list_move_tail(&dev->unreg_list, head);
 	} else {
@@ -12480,6 +12503,16 @@ void unregister_netdevice_many_notify(struct list_head *head,
 	synchronize_net();
 
 	list_for_each_entry(dev, head, unreg_list) {
+#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
+		struct net *net = dev_net(dev);
+
+		/* spin_lock() can be moved outside of the loop
+		 * once the per-netns RTNL conversion completes.
+		 */
+		spin_lock(&net->dev_unreg_lock);
+		list_del(&dev->unreg_list_net);
+		spin_unlock(&net->dev_unreg_lock);
+#endif
 		netdev_put(dev, &dev->dev_registered_tracker);
 		net_set_todo(dev);
 		cnt++;
@@ -12502,6 +12535,96 @@ void unregister_netdevice_many(struct list_head *head)
 }
 EXPORT_SYMBOL(unregister_netdevice_many);
 
+#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
+void unregister_netdevice_queue_net(struct net *net, struct net_device *dev,
+				    struct list_head *head)
+{
+	netdev_lock(dev);
+
+	if (net_eq(dev_net(dev), net)) {
+		netdev_unlock(dev);
+		unregister_netdevice_queue(dev, head);
+		return;
+	}
+
+	net = dev_net(dev);
+
+	spin_lock(&net->dev_unreg_lock);
+
+	DEBUG_NET_WARN_ON_ONCE(!list_empty(&dev->unreg_list));
+	DEBUG_NET_WARN_ON_ONCE(!list_empty(&dev->unreg_list_net));
+
+	list_add_tail(&dev->unreg_list_net, &net->dev_unreg_head);
+	rtnl_net_queue_work(net);
+
+	spin_unlock(&net->dev_unreg_lock);
+
+	netdev_unlock(dev);
+}
+EXPORT_SYMBOL(unregister_netdevice_queue_net);
+
+void unregister_netdevice_queue_many_net(struct net *net, struct list_head *head)
+{
+	struct net_device *dev, *tmp;
+
+	spin_lock(&net->dev_unreg_lock);
+	list_for_each_entry_safe(dev, tmp, head, unreg_list) {
+		/* Once all cross-netns unregister_netdevice_queue() is
+		 * converted to _net() (or for debugging), remove this check.
+		 */
+		if (!net_eq(dev_net(dev), net))
+			continue;
+
+		DEBUG_NET_WARN_ONCE(!net_eq(dev_net(dev), net),
+				    "%s was unregistered from a different netns.\n",
+				    dev->name);
+
+		list_del_init(&dev->unreg_list);
+		list_move_tail(&dev->unreg_list_net, &net->dev_unreg_head);
+	}
+	spin_unlock(&net->dev_unreg_lock);
+}
+
+static void unregister_netdevice_move_net(struct net *net_old,
+					  struct net *net,
+					  struct net_device *dev)
+{
+	if (net_old > net) {
+		spin_lock(&net->dev_unreg_lock);
+		spin_lock_nested(&net_old->dev_unreg_lock, SINGLE_DEPTH_NESTING);
+	} else {
+		spin_lock(&net_old->dev_unreg_lock);
+		spin_lock_nested(&net->dev_unreg_lock, SINGLE_DEPTH_NESTING);
+	}
+
+	if (!list_empty(&dev->unreg_list_net)) {
+		list_del(&dev->unreg_list_net);
+		list_add_tail(&dev->unreg_list_net, &net->dev_unreg_head);
+	}
+
+	spin_unlock(&net_old->dev_unreg_lock);
+	spin_unlock(&net->dev_unreg_lock);
+}
+
+void unregister_netdevice_many_net(struct net *net)
+{
+	struct net_device *dev, *tmp;
+	LIST_HEAD(unreg_head_net);
+	LIST_HEAD(unreg_head);
+
+	spin_lock(&net->dev_unreg_lock);
+	list_splice_init(&net->dev_unreg_head, &unreg_head_net);
+	spin_unlock(&net->dev_unreg_lock);
+
+	list_for_each_entry_safe(dev, tmp, &unreg_head_net, unreg_list_net) {
+		list_del_init(&dev->unreg_list_net);
+		list_add_tail(&dev->unreg_list, &unreg_head);
+	}
+
+	unregister_netdevice_many(&unreg_head);
+}
+#endif
+
 /**
  *	unregister_netdev - remove device from the kernel
  *	@dev: device
@@ -12658,6 +12781,10 @@ int __dev_change_net_namespace(struct net_device *dev, struct net *net,
 	netdev_unlock(dev);
 	dev->ifindex = new_ifindex;
 
+#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
+	unregister_netdevice_move_net(net_old, net, dev);
+#endif
+
 	if (new_name[0]) {
 		/* Rename the netdev to prepared name */
 		write_seqlock_bh(&netdev_rename_lock);
@@ -13034,7 +13161,7 @@ static void __net_exit default_device_exit_net(struct net *net)
 	 * Push all migratable network devices back to the
 	 * initial network namespace
 	 */
-	ASSERT_RTNL();
+
 	for_each_netdev_safe(net, dev, aux) {
 		int err;
 		char fb_name[IFNAMSIZ];
@@ -13077,21 +13204,36 @@ static void __net_exit default_device_exit_batch(struct list_head *net_list)
 	LIST_HEAD(dev_kill_list);
 
 	rtnl_lock();
+
+	__rtnl_net_lock(&init_net);
+
 	list_for_each_entry(net, net_list, exit_list) {
+		__rtnl_net_lock(net);
 		default_device_exit_net(net);
+		__rtnl_net_unlock(net);
+
 		cond_resched();
 	}
 
+	__rtnl_net_unlock(&init_net);
+
 	list_for_each_entry(net, net_list, exit_list) {
+		__rtnl_net_lock(net);
+
 		for_each_netdev_reverse(net, dev) {
 			if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
 				dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
 			else
 				unregister_netdevice_queue(dev, &dev_kill_list);
 		}
+
+		unregister_netdevice_queue_many_net(net, &dev_kill_list);
+		__rtnl_net_unlock(net);
 	}
 	unregister_netdevice_many(&dev_kill_list);
 	rtnl_unlock();
+
+	rtnl_net_flush_workqueue();
 }
 
 static struct pernet_operations __net_initdata default_device_ops = {
diff --git a/net/core/fib_rules.c b/net/core/fib_rules.c
index cf374c2087325..7df216c17c67a 100644
--- a/net/core/fib_rules.c
+++ b/net/core/fib_rules.c
@@ -51,7 +51,6 @@ bool fib_rule_matchall(const struct fib_rule *rule)
 		return false;
 	return true;
 }
-EXPORT_SYMBOL_GPL(fib_rule_matchall);
 
 int fib_default_rule_add(struct fib_rules_ops *ops,
 			 u32 pref, u32 table)
@@ -78,7 +77,6 @@ int fib_default_rule_add(struct fib_rules_ops *ops,
 	list_add_tail(&r->list, &ops->rules_list);
 	return 0;
 }
-EXPORT_SYMBOL(fib_default_rule_add);
 
 static u32 fib_default_rule_pref(struct fib_rules_ops *ops)
 {
@@ -172,6 +170,7 @@ fib_rules_register(const struct fib_rules_ops *tmpl, struct net *net)
 		return ERR_PTR(-ENOMEM);
 
 	INIT_LIST_HEAD(&ops->rules_list);
+	mutex_init(&ops->lock);
 	ops->fro_net = net;
 
 	err = __fib_rules_register(ops);
@@ -182,7 +181,6 @@ fib_rules_register(const struct fib_rules_ops *tmpl, struct net *net)
 
 	return ops;
 }
-EXPORT_SYMBOL_GPL(fib_rules_register);
 
 static void fib_rules_cleanup_ops(struct fib_rules_ops *ops)
 {
@@ -205,9 +203,9 @@ void fib_rules_unregister(struct fib_rules_ops *ops)
 	spin_unlock(&net->rules_mod_lock);
 
 	fib_rules_cleanup_ops(ops);
+	mutex_destroy(&ops->lock);
 	kfree_rcu(ops, rcu);
 }
-EXPORT_SYMBOL_GPL(fib_rules_unregister);
 
 static int uid_range_set(struct fib_kuid_range *range)
 {
@@ -363,7 +361,6 @@ int fib_rules_lookup(struct fib_rules_ops *ops, struct flowi *fl,
 
 	return err;
 }
-EXPORT_SYMBOL_GPL(fib_rules_lookup);
 
 static int call_fib_rule_notifier(struct notifier_block *nb,
 				  enum fib_event_type event_type,
@@ -391,7 +388,7 @@ static int call_fib_rule_notifiers(struct net *net,
 		.rule = rule,
 	};
 
-	ASSERT_RTNL_NET(net);
+	lockdep_assert_held(&ops->lock);
 
 	/* Paired with READ_ONCE() in fib_rules_seq() */
 	WRITE_ONCE(ops->fib_rules_seq, ops->fib_rules_seq + 1);
@@ -423,7 +420,6 @@ int fib_rules_dump(struct net *net, struct notifier_block *nb, int family,
 
 	return err;
 }
-EXPORT_SYMBOL_GPL(fib_rules_dump);
 
 unsigned int fib_rules_seq_read(const struct net *net, int family)
 {
@@ -439,7 +435,6 @@ unsigned int fib_rules_seq_read(const struct net *net, int family)
 
 	return fib_rules_seq;
 }
-EXPORT_SYMBOL_GPL(fib_rules_seq_read);
 
 static struct fib_rule *rule_find(struct fib_rules_ops *ops,
 				  struct fib_rule_hdr *frh,
@@ -740,10 +735,10 @@ static int fib_nl2rule(struct net *net, struct nlmsghdr *nlh,
 	return err;
 }
 
-static int fib_nl2rule_rtnl(struct fib_rule *nlrule,
-			    struct fib_rules_ops *ops,
-			    struct nlattr *tb[],
-			    struct netlink_ext_ack *extack)
+static int fib_nl2rule_locked(struct fib_rule *nlrule,
+			      struct fib_rules_ops *ops,
+			      struct nlattr *tb[],
+			      struct netlink_ext_ack *extack)
 {
 	if (!tb[FRA_PRIORITY])
 		nlrule->pref = fib_default_rule_pref(ops);
@@ -754,12 +749,14 @@ static int fib_nl2rule_rtnl(struct fib_rule *nlrule,
 		return -EINVAL;
 	}
 
+	rcu_read_lock();
+
 	if (tb[FRA_IIFNAME]) {
 		struct net_device *dev;
 
-		dev = __dev_get_by_name(nlrule->fr_net, nlrule->iifname);
+		dev = dev_get_by_name_rcu(nlrule->fr_net, nlrule->iifname);
 		if (dev) {
-			nlrule->iifindex = dev->ifindex;
+			nlrule->iifindex = READ_ONCE(dev->ifindex);
 			nlrule->iif_is_l3_master = netif_is_l3_master(dev);
 		}
 	}
@@ -767,13 +764,15 @@ static int fib_nl2rule_rtnl(struct fib_rule *nlrule,
 	if (tb[FRA_OIFNAME]) {
 		struct net_device *dev;
 
-		dev = __dev_get_by_name(nlrule->fr_net, nlrule->oifname);
+		dev = dev_get_by_name_rcu(nlrule->fr_net, nlrule->oifname);
 		if (dev) {
-			nlrule->oifindex = dev->ifindex;
+			nlrule->oifindex = READ_ONCE(dev->ifindex);
 			nlrule->oif_is_l3_master = netif_is_l3_master(dev);
 		}
 	}
 
+	rcu_read_unlock();
+
 	return 0;
 }
 
@@ -883,6 +882,7 @@ int fib_newrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh,
 	struct nlattr *tb[FRA_MAX + 1];
 	bool user_priority = false;
 	struct fib_rule_hdr *frh;
+	bool unlock_rtnl = false;
 
 	frh = nlmsg_payload(nlh, sizeof(*frh));
 	if (!frh) {
@@ -908,10 +908,13 @@ int fib_newrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh,
 	if (err)
 		goto errout;
 
-	if (!rtnl_held)
+	if (!rtnl_held && ops->need_rtnl && ops->need_rtnl(net)) {
+		unlock_rtnl = true;
 		rtnl_net_lock(net);
+	}
+	mutex_lock(&ops->lock);
 
-	err = fib_nl2rule_rtnl(rule, ops, tb, extack);
+	err = fib_nl2rule_locked(rule, ops, tb, extack);
 	if (err)
 		goto errout_free;
 
@@ -959,7 +962,7 @@ int fib_newrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh,
 		list_for_each_entry(r, &ops->rules_list, list) {
 			if (r->action == FR_ACT_GOTO &&
 			    r->target == rule->pref &&
-			    rtnl_dereference(r->ctarget) == NULL) {
+			    !rcu_access_pointer(r->ctarget)) {
 				rcu_assign_pointer(r->ctarget, rule);
 				if (--ops->unresolved_rules == 0)
 					break;
@@ -978,7 +981,8 @@ int fib_newrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh,
 
 	fib_rule_get(rule);
 
-	if (!rtnl_held)
+	mutex_unlock(&ops->lock);
+	if (unlock_rtnl)
 		rtnl_net_unlock(net);
 
 	notify_rule_change(RTM_NEWRULE, rule, ops, nlh, NETLINK_CB(skb).portid);
@@ -988,7 +992,8 @@ int fib_newrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh,
 	return 0;
 
 errout_free:
-	if (!rtnl_held)
+	mutex_unlock(&ops->lock);
+	if (unlock_rtnl)
 		rtnl_net_unlock(net);
 	kfree(rule);
 errout:
@@ -1037,10 +1042,9 @@ int fib_delrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh,
 	if (err)
 		goto errout;
 
-	if (!rtnl_held)
-		rtnl_net_lock(net);
+	mutex_lock(&ops->lock);
 
-	err = fib_nl2rule_rtnl(nlrule, ops, tb, extack);
+	err = fib_nl2rule_locked(nlrule, ops, tb, extack);
 	if (err)
 		goto errout_free;
 
@@ -1055,11 +1059,8 @@ int fib_delrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh,
 		goto errout_free;
 	}
 
-	if (ops->delete) {
-		err = ops->delete(rule);
-		if (err)
-			goto errout_free;
-	}
+	if (ops->delete)
+		ops->delete(rule);
 
 	if (rule->tun_id)
 		ip_tunnel_unneed_metadata();
@@ -1068,7 +1069,7 @@ int fib_delrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh,
 
 	if (rule->action == FR_ACT_GOTO) {
 		ops->nr_goto_rules--;
-		if (rtnl_dereference(rule->ctarget) == NULL)
+		if (!rcu_access_pointer(rule->ctarget))
 			ops->unresolved_rules--;
 	}
 
@@ -1086,7 +1087,7 @@ int fib_delrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh,
 		if (&n->list == &ops->rules_list || n->pref != rule->pref)
 			n = NULL;
 		list_for_each_entry(r, &ops->rules_list, list) {
-			if (rtnl_dereference(r->ctarget) != rule)
+			if (rcu_access_pointer(r->ctarget) != rule)
 				continue;
 			rcu_assign_pointer(r->ctarget, n);
 			if (!n)
@@ -1096,8 +1097,7 @@ int fib_delrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh,
 
 	call_fib_rule_notifiers(net, FIB_EVENT_RULE_DEL, rule, ops, NULL);
 
-	if (!rtnl_held)
-		rtnl_net_unlock(net);
+	mutex_unlock(&ops->lock);
 
 	notify_rule_change(RTM_DELRULE, rule, ops, nlh, NETLINK_CB(skb).portid);
 	fib_rule_put(rule);
@@ -1107,8 +1107,7 @@ int fib_delrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh,
 	return 0;
 
 errout_free:
-	if (!rtnl_held)
-		rtnl_net_unlock(net);
+	mutex_unlock(&ops->lock);
 	kfree(nlrule);
 errout:
 	rules_ops_put(ops);
@@ -1402,24 +1401,30 @@ static int fib_rules_event(struct notifier_block *this, unsigned long event,
 	struct net *net = dev_net(dev);
 	struct fib_rules_ops *ops;
 
-	ASSERT_RTNL();
-
 	switch (event) {
 	case NETDEV_REGISTER:
-		list_for_each_entry(ops, &net->rules_ops, list)
+		list_for_each_entry(ops, &net->rules_ops, list) {
+			mutex_lock(&ops->lock);
 			attach_rules(&ops->rules_list, dev);
+			mutex_unlock(&ops->lock);
+		}
 		break;
 
 	case NETDEV_CHANGENAME:
 		list_for_each_entry(ops, &net->rules_ops, list) {
+			mutex_lock(&ops->lock);
 			detach_rules(&ops->rules_list, dev);
 			attach_rules(&ops->rules_list, dev);
+			mutex_unlock(&ops->lock);
 		}
 		break;
 
 	case NETDEV_UNREGISTER:
-		list_for_each_entry(ops, &net->rules_ops, list)
+		list_for_each_entry(ops, &net->rules_ops, list) {
+			mutex_lock(&ops->lock);
 			detach_rules(&ops->rules_list, dev);
+			mutex_unlock(&ops->lock);
+		}
 		break;
 	}
 
diff --git a/net/core/filter.c b/net/core/filter.c
index b446aa8be5c37..056deb9b3fc35 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -84,6 +84,9 @@
 #include <linux/un.h>
 #include <net/xdp_sock_drv.h>
 #include <net/inet_dscp.h>
+#include <linux/icmpv6.h>
+#include <net/icmp.h>
+#include <net/ip6_route.h>
 
 #include "dev.h"
 
@@ -2529,16 +2532,18 @@ int skb_do_redirect(struct sk_buff *skb)
 	if (unlikely(!dev))
 		goto out_drop;
 	if (flags & BPF_F_PEER) {
-		if (unlikely(!skb_at_tc_ingress(skb)))
-			goto out_drop;
 		dev = skb_get_peer_dev(dev);
 		if (unlikely(!dev ||
 			     !(dev->flags & IFF_UP) ||
 			     net_eq(net, dev_net(dev))))
 			goto out_drop;
+		skb_scrub_packet(skb, false);
+		if (flags & BPF_F_EGRESS)
+			return __bpf_redirect(skb, dev, 0);
+		if (unlikely(!skb_at_tc_ingress(skb)))
+			goto out_drop;
 		skb->dev = dev;
 		dev_sw_netstats_rx_add(dev, skb->len);
-		skb_scrub_packet(skb, false);
 		return -EAGAIN;
 	}
 	return flags & BPF_F_NEIGH ?
@@ -2575,10 +2580,10 @@ BPF_CALL_2(bpf_redirect_peer, u32, ifindex, u64, flags)
 {
 	struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
 
-	if (unlikely(flags))
+	if (unlikely(flags & ~BPF_F_EGRESS))
 		return TC_ACT_SHOT;
 
-	ri->flags = BPF_F_PEER;
+	ri->flags = BPF_F_PEER | flags;
 	ri->tgt_index = ifindex;
 
 	return TC_ACT_REDIRECT;
@@ -12546,6 +12551,87 @@ __bpf_kfunc int bpf_xdp_pull_data(struct xdp_md *x, u32 len)
 	return 0;
 }
 
+/**
+ * bpf_icmp_send - Send an ICMP control message
+ * @skb_ctx: Packet that triggered the control message
+ * @type: ICMP type (only ICMP_DEST_UNREACH/ICMPV6_DEST_UNREACH supported)
+ * @code: ICMP code (0-15 except ICMP_FRAG_NEEDED for IPv4, 0-6 for IPv6)
+ *
+ * Sends an ICMP control message in response to the packet. The original packet
+ * is cloned before sending the ICMP message, so the BPF program can still let
+ * the packet pass if desired.
+ *
+ * Currently only ICMP_DEST_UNREACH (IPv4) and ICMPV6_DEST_UNREACH (IPv6) are
+ * supported.
+ *
+ * Return: 0 on success (send attempt), negative error code on failure:
+ *         -EBUSY: Recursion detected
+ *         -EPROTONOSUPPORT: Non-IP protocol
+ *         -EOPNOTSUPP: Unsupported ICMP type
+ *         -EINVAL: Invalid code parameter
+ *         -ENETUNREACH: No usable route/dst for the ICMP reply
+ *         -ENOMEM: Memory allocation failed
+ */
+__bpf_kfunc int bpf_icmp_send(struct __sk_buff *skb_ctx, int type, int code)
+{
+	struct sk_buff *skb = (struct sk_buff *)skb_ctx;
+	struct sock *sk;
+
+	sk = skb_to_full_sk(skb);
+	if (sk && sk->sk_kern_sock &&
+	    (sk->sk_protocol == IPPROTO_ICMP || sk->sk_protocol == IPPROTO_ICMPV6))
+		return -EBUSY;
+
+	if (!skb_valid_dst(skb))
+		return -ENETUNREACH;
+
+	switch (skb->protocol) {
+#if IS_ENABLED(CONFIG_INET)
+	case htons(ETH_P_IP): {
+		struct sk_buff *nskb;
+
+		if (type != ICMP_DEST_UNREACH)
+			return -EOPNOTSUPP;
+		if (code < 0 || code > NR_ICMP_UNREACH ||
+		    code == ICMP_FRAG_NEEDED) /* needs a valid next-hop MTU */
+			return -EINVAL;
+
+		nskb = skb_clone(skb, GFP_ATOMIC);
+		if (!nskb)
+			return -ENOMEM;
+
+		memset(IPCB(nskb), 0, sizeof(*IPCB(nskb)));
+		icmp_send(nskb, type, code, 0);
+		consume_skb(nskb);
+		break;
+	}
+#endif
+#if IS_ENABLED(CONFIG_IPV6)
+	case htons(ETH_P_IPV6): {
+		struct sk_buff *nskb;
+
+		if (type != ICMPV6_DEST_UNREACH)
+			return -EOPNOTSUPP;
+		if (code < 0 || code > ICMPV6_REJECT_ROUTE)
+			return -EINVAL;
+
+		nskb = skb_clone(skb, GFP_ATOMIC);
+		if (!nskb)
+			return -ENOMEM;
+
+		memset(IP6CB(nskb), 0, sizeof(*IP6CB(nskb)));
+		icmpv6_send(nskb, type, code, 0);
+		consume_skb(nskb);
+		break;
+	}
+#endif
+	default:
+		return -EPROTONOSUPPORT;
+	}
+
+	return 0;
+}
+
 __bpf_kfunc_end_defs();
 
 int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags,
@@ -12588,6 +12674,10 @@ BTF_KFUNCS_START(bpf_kfunc_check_set_sock_ops)
 BTF_ID_FLAGS(func, bpf_sock_ops_enable_tx_tstamp)
 BTF_KFUNCS_END(bpf_kfunc_check_set_sock_ops)
 
+BTF_KFUNCS_START(bpf_kfunc_check_set_icmp_send)
+BTF_ID_FLAGS(func, bpf_icmp_send)
+BTF_KFUNCS_END(bpf_kfunc_check_set_icmp_send)
+
 static const struct btf_kfunc_id_set bpf_kfunc_set_skb = {
 	.owner = THIS_MODULE,
 	.set = &bpf_kfunc_check_set_skb,
@@ -12618,6 +12708,11 @@ static const struct btf_kfunc_id_set bpf_kfunc_set_sock_ops = {
 	.set = &bpf_kfunc_check_set_sock_ops,
 };
 
+static const struct btf_kfunc_id_set bpf_kfunc_set_icmp_send = {
+	.owner = THIS_MODULE,
+	.set = &bpf_kfunc_check_set_icmp_send,
+};
+
 static int __init bpf_kfunc_init(void)
 {
 	int ret;
@@ -12639,6 +12734,7 @@ static int __init bpf_kfunc_init(void)
 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
 					       &bpf_kfunc_set_sock_addr);
 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_kfunc_set_tcp_reqsk);
+	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SKB, &bpf_kfunc_set_icmp_send);
 	return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SOCK_OPS, &bpf_kfunc_set_sock_ops);
 }
 late_initcall(bpf_kfunc_init);
diff --git a/net/core/lock_debug.c b/net/core/lock_debug.c
index 8a81c54307053..abc4c00728b16 100644
--- a/net/core/lock_debug.c
+++ b/net/core/lock_debug.c
@@ -24,15 +24,15 @@ int netdev_debug_event(struct notifier_block *nb, unsigned long event,
 	case NETDEV_CHANGE:
 	case NETDEV_REGISTER:
 	case NETDEV_UP:
+	case NETDEV_DOWN:
+	case NETDEV_GOING_DOWN:
 		netdev_assert_locked_ops_compat(dev);
 		fallthrough;
-	case NETDEV_DOWN:
 	case NETDEV_REBOOT:
 	case NETDEV_UNREGISTER:
 	case NETDEV_CHANGEMTU:
 	case NETDEV_CHANGEADDR:
 	case NETDEV_PRE_CHANGEADDR:
-	case NETDEV_GOING_DOWN:
 	case NETDEV_FEAT_CHANGE:
 	case NETDEV_BONDING_FAILOVER:
 	case NETDEV_PRE_UP:
diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c
index d9dafe24f57e3..a91d2b58aaddf 100644
--- a/net/core/net_namespace.c
+++ b/net/core/net_namespace.c
@@ -181,6 +181,7 @@ static void ops_exit_rtnl_list(const struct list_head *ops_list,
 				ops->exit_rtnl(net, &dev_kill_list);
 		}
 
+		unregister_netdevice_queue_many_net(net, &dev_kill_list);
 		__rtnl_net_unlock(net);
 	}
 
@@ -422,6 +423,9 @@ static __net_init int preinit_net(struct net *net, struct user_namespace *user_n
 #ifdef CONFIG_DEBUG_NET_SMALL_RTNL
 	mutex_init(&net->rtnl_mutex);
 	lock_set_cmp_fn(&net->rtnl_mutex, rtnl_net_lock_cmp_fn, NULL);
+	INIT_WORK(&net->rtnl_work, rtnl_net_work_func);
+	INIT_LIST_HEAD(&net->dev_unreg_head);
+	spin_lock_init(&net->dev_unreg_lock);
 #endif
 
 	INIT_LIST_HEAD(&net->ptype_all);
diff --git a/net/core/netpoll.c b/net/core/netpoll.c
index 96d5945e6a30f..f8da1048ea3ab 100644
--- a/net/core/netpoll.c
+++ b/net/core/netpoll.c
@@ -36,12 +36,6 @@
 #include <trace/events/napi.h>
 #include <linux/kconfig.h>
 
-/*
- * We maintain a small pool of fully-sized skbs, to make sure the
- * message gets out even in extreme OOM situations.
- */
-
-#define MAX_SKBS 32
 #define USEC_PER_POLL	50
 
 static unsigned int carrier_timeout = 4;
@@ -213,22 +207,6 @@ void netpoll_poll_enable(struct net_device *dev)
 		up(&ni->dev_lock);
 }
 
-static void refill_skbs(struct netpoll *np)
-{
-	struct sk_buff_head *skb_pool;
-	struct sk_buff *skb;
-
-	skb_pool = &np->skb_pool;
-
-	while (READ_ONCE(skb_pool->qlen) < MAX_SKBS) {
-		skb = alloc_skb(MAX_SKB_SIZE, GFP_ATOMIC);
-		if (!skb)
-			break;
-
-		skb_queue_tail(skb_pool, skb);
-	}
-}
-
 void netpoll_zap_completion_queue(void)
 {
 	unsigned long flags;
@@ -351,32 +329,12 @@ netdev_tx_t netpoll_send_skb(struct netpoll *np, struct sk_buff *skb)
 }
 EXPORT_SYMBOL(netpoll_send_skb);
 
-static void skb_pool_flush(struct netpoll *np)
-{
-	struct sk_buff_head *skb_pool;
-
-	cancel_work_sync(&np->refill_wq);
-	skb_pool = &np->skb_pool;
-	skb_queue_purge_reason(skb_pool, SKB_CONSUMED);
-}
-
-static void refill_skbs_work_handler(struct work_struct *work)
-{
-	struct netpoll *np =
-		container_of(work, struct netpoll, refill_wq);
-
-	refill_skbs(np);
-}
-
 int __netpoll_setup(struct netpoll *np, struct net_device *ndev)
 {
 	struct netpoll_info *npinfo;
 	const struct net_device_ops *ops;
 	int err;
 
-	skb_queue_head_init(&np->skb_pool);
-	INIT_WORK(&np->refill_wq, refill_skbs_work_handler);
-
 	if (ndev->priv_flags & IFF_DISABLE_NETPOLL) {
 		np_err(np, "%s doesn't support polling, aborting\n",
 		       ndev->name);
@@ -411,9 +369,6 @@ int __netpoll_setup(struct netpoll *np, struct net_device *ndev)
 	np->dev = ndev;
 	strscpy(np->dev_name, ndev->name, IFNAMSIZ);
 
-	/* fill up the skb queue */
-	refill_skbs(np);
-
 	/* last thing to do is link it to the net device structure */
 	rcu_assign_pointer(ndev->npinfo, npinfo);
 
@@ -603,7 +558,7 @@ int netpoll_setup(struct netpoll *np)
 
 	err = __netpoll_setup(np, ndev);
 	if (err)
-		goto flush;
+		goto put;
 	rtnl_unlock();
 
 	/* Make sure all NAPI polls which started before dev->npinfo
@@ -614,8 +569,6 @@ int netpoll_setup(struct netpoll *np)
 
 	return 0;
 
-flush:
-	skb_pool_flush(np);
 put:
 	DEBUG_NET_WARN_ON_ONCE(np->dev);
 	if (ip_overwritten)
@@ -659,8 +612,6 @@ static void __netpoll_cleanup(struct netpoll *np)
 		disable_delayed_work_sync(&npinfo->tx_work);
 		call_rcu(&npinfo->rcu, rcu_cleanup_netpoll_info);
 	}
-
-	skb_pool_flush(np);
 }
 
 void __netpoll_free(struct netpoll *np)
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 12aa3aa1688b1..31c65a545a107 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -197,6 +197,7 @@ void __rtnl_net_unlock(struct net *net)
 {
 	ASSERT_RTNL();
 
+	unregister_netdevice_many_net(net);
 	mutex_unlock(&net->rtnl_mutex);
 }
 EXPORT_SYMBOL(__rtnl_net_unlock);
@@ -273,6 +274,29 @@ bool lockdep_rtnl_net_is_held(struct net *net)
 	return lockdep_rtnl_is_held() && lockdep_is_held(&net->rtnl_mutex);
 }
 EXPORT_SYMBOL(lockdep_rtnl_net_is_held);
+
+static struct workqueue_struct *rtnl_net_wq;
+
+void rtnl_net_queue_work(struct net *net)
+{
+	queue_work(rtnl_net_wq, &net->rtnl_work);
+}
+
+void rtnl_net_flush_workqueue(void)
+{
+	flush_workqueue(rtnl_net_wq);
+}
+
+void rtnl_net_work_func(struct work_struct *work)
+{
+	struct net *net = container_of(work, struct net, rtnl_work);
+
+	if (list_empty(&net->dev_unreg_head))
+		return;
+
+	rtnl_net_lock(net);
+	rtnl_net_unlock(net);
+}
 #else
 static int rtnl_net_cmp_locks(const struct net *net_a, const struct net *net_b)
 {
@@ -282,10 +306,11 @@ static int rtnl_net_cmp_locks(const struct net *net_a, const struct net *net_b)
 #endif
 
 struct rtnl_nets {
-	/* ->newlink() needs to freeze 3 netns at most;
-	 * 2 for the new device, 1 for its peer.
+	/* ->newlink() needs to freeze 4 netns at most;
+	 * 2 for the new device, 1 for its peer, 1 for
+	 * an existing device (do_setlink() path).
 	 */
-	struct net *net[3];
+	struct net *net[4];
 	unsigned char len;
 };
 
@@ -636,16 +661,15 @@ int rtnl_link_register(struct rtnl_link_ops *ops)
 }
 EXPORT_SYMBOL_GPL(rtnl_link_register);
 
-static void __rtnl_kill_links(struct net *net, struct rtnl_link_ops *ops)
+static void __rtnl_kill_links(struct net *net, struct rtnl_link_ops *ops,
+			      struct list_head *dev_kill_list)
 {
 	struct net_device *dev;
-	LIST_HEAD(list_kill);
 
 	for_each_netdev(net, dev) {
 		if (dev->rtnl_link_ops == ops)
-			ops->dellink(dev, &list_kill);
+			ops->dellink(dev, dev_kill_list);
 	}
-	unregister_netdevice_many(&list_kill);
 }
 
 /* Return with the rtnl_lock held when there are no network
@@ -676,6 +700,7 @@ static void rtnl_lock_unregistering_all(void)
  */
 void rtnl_link_unregister(struct rtnl_link_ops *ops)
 {
+	LIST_HEAD(dev_kill_list);
 	struct net *net;
 
 	mutex_lock(&link_ops_mutex);
@@ -689,8 +714,14 @@ void rtnl_link_unregister(struct rtnl_link_ops *ops)
 	down_write(&pernet_ops_rwsem);
 	rtnl_lock_unregistering_all();
 
-	for_each_net(net)
-		__rtnl_kill_links(net, ops);
+	for_each_net(net) {
+		__rtnl_net_lock(net);
+		__rtnl_kill_links(net, ops, &dev_kill_list);
+		unregister_netdevice_queue_many_net(net, &dev_kill_list);
+		__rtnl_net_unlock(net);
+	}
+
+	unregister_netdevice_many(&dev_kill_list);
 
 	rtnl_unlock();
 	up_write(&pernet_ops_rwsem);
@@ -3660,14 +3691,16 @@ int rtnl_configure_link(struct net_device *dev, const struct ifinfomsg *ifm,
 			u32 portid, const struct nlmsghdr *nlh)
 {
 	unsigned int old_flags, changed;
-	int err;
+	int err = 0;
+
+	netdev_lock_ops(dev);
 
 	old_flags = dev->flags;
 	if (ifm && (ifm->ifi_flags || ifm->ifi_change)) {
 		err = __dev_change_flags(dev, rtnl_dev_combine_flags(dev, ifm),
 					 NULL);
 		if (err < 0)
-			return err;
+			goto out;
 	}
 
 	changed = old_flags ^ dev->flags;
@@ -3677,7 +3710,10 @@ int rtnl_configure_link(struct net_device *dev, const struct ifinfomsg *ifm,
 	}
 
 	__dev_notify_flags(dev, old_flags, changed, portid, nlh);
-	return 0;
+
+out:
+	netdev_unlock_ops(dev);
+	return err;
 }
 EXPORT_SYMBOL(rtnl_configure_link);
 
@@ -3918,22 +3954,20 @@ static int rtnl_newlink_create(struct sk_buff *skb, struct ifinfomsg *ifm,
 		goto out;
 	}
 
-	netdev_lock_ops(dev);
-
 	err = rtnl_configure_link(dev, ifm, portid, nlh);
 	if (err < 0)
 		goto out_unregister;
 	if (tb[IFLA_MASTER]) {
+		netdev_lock_ops(dev);
 		err = do_set_master(dev, nla_get_u32(tb[IFLA_MASTER]), extack);
+		netdev_unlock_ops(dev);
 		if (err)
 			goto out_unregister;
 	}
 
-	netdev_unlock_ops(dev);
 out:
 	return err;
 out_unregister:
-	netdev_unlock_ops(dev);
 	if (ops->newlink) {
 		LIST_HEAD(list_kill);
 
@@ -4155,6 +4189,8 @@ static int rtnl_newlink(struct sk_buff *skb, struct nlmsghdr *nlh,
 		}
 	}
 
+	rtnl_nets_add(&rtnl_nets, get_net(sock_net(skb->sk)));
+
 	rtnl_nets_lock(&rtnl_nets);
 	ret = __rtnl_newlink(skb, nlh, ops, tgt_net, link_net, peer_net, tbs, data, extack);
 	rtnl_nets_unlock(&rtnl_nets);
@@ -7221,4 +7257,10 @@ void __init rtnetlink_init(void)
 	register_netdevice_notifier(&rtnetlink_dev_notifier);
 
 	rtnl_register_many(rtnetlink_rtnl_msg_handlers);
+
+#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
+	rtnl_net_wq = create_workqueue("rtnl_net");
+	if (!rtnl_net_wq)
+		panic("Could not create rtnl_net workq");
+#endif
 }
diff --git a/net/core/sock.c b/net/core/sock.c
index 8a59bfaa8096e..498a57f34f5b5 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -2492,6 +2492,9 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority,
 	sock_copy(newsk, sk);
 
 	newsk->sk_prot_creator = prot;
+#ifdef CONFIG_BPF_SYSCALL
+	RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
+#endif
 
 	/* SANITY */
 	if (likely(newsk->sk_net_refcnt)) {
diff --git a/net/core/sock_map.c b/net/core/sock_map.c
index c60ba6d292f9a..9efbd8ca7db83 100644
--- a/net/core/sock_map.c
+++ b/net/core/sock_map.c
@@ -542,6 +542,8 @@ static bool sock_map_sk_state_allowed(const struct sock *sk)
 {
 	if (sk_is_tcp(sk))
 		return (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_LISTEN);
+	if (sk_is_udp(sk))
+		return sk_hashed(sk);
 	if (sk_is_stream_unix(sk))
 		return (1 << READ_ONCE(sk->sk_state)) & TCPF_ESTABLISHED;
 	if (sk_is_vsock(sk) &&
diff --git a/net/devlink/core.c b/net/devlink/core.c
index fe9f6a0a67d5b..c53a42e17a589 100644
--- a/net/devlink/core.c
+++ b/net/devlink/core.c
@@ -67,6 +67,22 @@ static void __devlink_rel_put(struct devlink_rel *rel)
 		devlink_rel_free(rel);
 }
 
+struct devlink *__must_check devlink_nested_in_get_lock(struct devlink *devlink)
+{
+	devl_assert_locked(devlink);
+	if (!devlink->rel)
+		return NULL;
+	devlink = devlinks_xa_get(devlink->rel->nested_in.devlink_index);
+	if (!devlink)
+		return NULL;
+	devl_lock(devlink);
+	if (devl_is_registered(devlink))
+		return devlink;
+	devl_unlock(devlink);
+	devlink_put(devlink);
+	return NULL;
+}
+
 static void devlink_rel_nested_in_notify_work(struct work_struct *work)
 {
 	struct devlink_rel *rel = container_of(work, struct devlink_rel,
@@ -518,6 +534,9 @@ void devlink_free(struct devlink *devlink)
 {
 	ASSERT_DEVLINK_NOT_REGISTERED(devlink);
 
+	devl_lock(devlink);
+	WARN_ON(devlink_rates_check(devlink, NULL, NULL));
+	devl_unlock(devlink);
 	devlink_rel_put(devlink);
 
 	WARN_ON(!list_empty(&devlink->trap_policer_list));
@@ -528,7 +547,6 @@ void devlink_free(struct devlink *devlink)
 	WARN_ON(!list_empty(&devlink->resource_list));
 	WARN_ON(!list_empty(&devlink->dpipe_table_list));
 	WARN_ON(!list_empty(&devlink->sb_list));
-	WARN_ON(devlink_rates_check(devlink, NULL, NULL));
 	WARN_ON(!list_empty(&devlink->linecard_list));
 	WARN_ON(!xa_empty(&devlink->ports));
 
diff --git a/net/devlink/dev.c b/net/devlink/dev.c
index 57b2b8f035436..bcf001554e840 100644
--- a/net/devlink/dev.c
+++ b/net/devlink/dev.c
@@ -222,7 +222,7 @@ static void devlink_notify(struct devlink *devlink, enum devlink_command cmd)
 
 int devlink_nl_get_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct sk_buff *msg;
 	int err;
 
@@ -519,7 +519,7 @@ devlink_nl_reload_actions_performed_snd(struct devlink *devlink, u32 actions_per
 
 int devlink_nl_reload_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	enum devlink_reload_action action;
 	enum devlink_reload_limit limit;
 	struct net *dest_net = NULL;
@@ -683,7 +683,7 @@ static int devlink_nl_eswitch_fill(struct sk_buff *msg, struct devlink *devlink,
 
 int devlink_nl_eswitch_get_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct sk_buff *msg;
 	int err;
 
@@ -704,7 +704,7 @@ int devlink_nl_eswitch_get_doit(struct sk_buff *skb, struct genl_info *info)
 
 int devlink_nl_eswitch_set_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	const struct devlink_ops *ops = devlink->ops;
 	enum devlink_eswitch_encap_mode encap_mode;
 	u8 inline_mode;
@@ -906,7 +906,7 @@ devlink_nl_info_fill(struct sk_buff *msg, struct devlink *devlink,
 
 int devlink_nl_info_get_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct sk_buff *msg;
 	int err;
 
@@ -1134,7 +1134,7 @@ int devlink_nl_flash_update_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct nlattr *nla_overwrite_mask, *nla_file_name;
 	struct devlink_flash_update_params params = {};
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	const char *file_name;
 	u32 supported_params;
 	int ret;
@@ -1302,7 +1302,7 @@ devlink_nl_selftests_fill(struct sk_buff *msg, struct devlink *devlink,
 
 int devlink_nl_selftests_get_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct sk_buff *msg;
 	int err;
 
@@ -1372,7 +1372,7 @@ static const struct nla_policy devlink_selftest_nl_policy[DEVLINK_ATTR_SELFTEST_
 int devlink_nl_selftests_run_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct nlattr *tb[DEVLINK_ATTR_SELFTEST_ID_MAX + 1];
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct nlattr *attrs, *selftests;
 	struct sk_buff *msg;
 	void *hdr;
diff --git a/net/devlink/devl_internal.h b/net/devlink/devl_internal.h
index e4e48ee2da5ad..cdf894ba5a9d7 100644
--- a/net/devlink/devl_internal.h
+++ b/net/devlink/devl_internal.h
@@ -136,6 +136,10 @@ typedef void devlink_rel_notify_cb_t(struct devlink *devlink, u32 obj_index);
 typedef void devlink_rel_cleanup_cb_t(struct devlink *devlink, u32 obj_index,
 				      u32 rel_index);
 
+/* Returns the locked+referenced nested-in instance or NULL. */
+struct devlink *__must_check
+devlink_nested_in_get_lock(struct devlink *devlink);
+
 void devlink_rel_nested_in_clear(u32 rel_index);
 int devlink_rel_nested_in_add(u32 *rel_index, u32 devlink_index,
 			      u32 obj_index, devlink_rel_notify_cb_t *notify_cb,
@@ -147,6 +151,20 @@ int devlink_rel_devlink_handle_put(struct sk_buff *msg, struct devlink *devlink,
 				   bool *msg_updated);
 
 /* Netlink */
+struct devlink_nl_ctx {
+	struct devlink *devlink;
+	struct devlink_port *devlink_port;
+	struct devlink *parent_devlink;
+};
+
+static inline struct devlink_nl_ctx *
+devlink_nl_ctx(struct genl_info *info)
+{
+	BUILD_BUG_ON(sizeof(struct devlink_nl_ctx) >
+		     sizeof_field(struct genl_info, ctx));
+	return (struct devlink_nl_ctx *)info->ctx;
+}
+
 enum devlink_multicast_groups {
 	DEVLINK_MCGRP_CONFIG,
 };
@@ -180,6 +198,8 @@ typedef int devlink_nl_dump_one_func_t(struct sk_buff *msg,
 struct devlink *
 devlink_get_from_attrs_lock(struct net *net, struct nlattr **attrs,
 			    bool dev_lock);
+struct devlink *
+devlink_get_parent_from_attrs_lock(struct net *net, struct nlattr **attrs);
 
 int devlink_nl_dumpit(struct sk_buff *msg, struct netlink_callback *cb,
 		      devlink_nl_dump_one_func_t *dump_one);
diff --git a/net/devlink/dpipe.c b/net/devlink/dpipe.c
index c8d4a4374ae1f..08c7b66fc3e8f 100644
--- a/net/devlink/dpipe.c
+++ b/net/devlink/dpipe.c
@@ -213,7 +213,7 @@ static int devlink_dpipe_tables_fill(struct genl_info *info,
 				     struct list_head *dpipe_tables,
 				     const char *table_name)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_dpipe_table *table;
 	struct nlattr *tables_attr;
 	struct sk_buff *skb = NULL;
@@ -290,7 +290,7 @@ static int devlink_dpipe_tables_fill(struct genl_info *info,
 
 int devlink_nl_dpipe_table_get_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	const char *table_name =  NULL;
 
 	if (info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
@@ -478,7 +478,7 @@ int devlink_dpipe_entry_ctx_prepare(struct devlink_dpipe_dump_ctx *dump_ctx)
 	if (!dump_ctx->hdr)
 		goto nla_put_failure;
 
-	devlink = dump_ctx->info->user_ptr[0];
+	devlink = devlink_nl_ctx(dump_ctx->info)->devlink;
 	if (devlink_nl_put_handle(dump_ctx->skb, devlink))
 		goto nla_put_failure;
 	dump_ctx->nest = nla_nest_start_noflag(dump_ctx->skb,
@@ -563,7 +563,7 @@ static int devlink_dpipe_entries_fill(struct genl_info *info,
 int devlink_nl_dpipe_entries_get_doit(struct sk_buff *skb,
 				      struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_dpipe_table *table;
 	const char *table_name;
 
@@ -650,7 +650,7 @@ static int devlink_dpipe_headers_fill(struct genl_info *info,
 				      struct devlink_dpipe_headers *
 				      dpipe_headers)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct nlattr *headers_attr;
 	struct sk_buff *skb = NULL;
 	struct nlmsghdr *nlh;
@@ -713,7 +713,7 @@ static int devlink_dpipe_headers_fill(struct genl_info *info,
 int devlink_nl_dpipe_headers_get_doit(struct sk_buff *skb,
 				      struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 
 	if (!devlink->dpipe_headers)
 		return -EOPNOTSUPP;
@@ -747,7 +747,7 @@ static int devlink_dpipe_table_counters_set(struct devlink *devlink,
 int devlink_nl_dpipe_table_counters_set_doit(struct sk_buff *skb,
 					     struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	const char *table_name;
 	bool counters_enable;
 
diff --git a/net/devlink/health.c b/net/devlink/health.c
index ea7a334e939b5..8ce6cd399cb7d 100644
--- a/net/devlink/health.c
+++ b/net/devlink/health.c
@@ -358,7 +358,7 @@ devlink_health_reporter_get_from_info(struct devlink *devlink,
 int devlink_nl_health_reporter_get_doit(struct sk_buff *skb,
 					struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_health_reporter *reporter;
 	struct sk_buff *msg;
 	int err;
@@ -456,7 +456,7 @@ int devlink_nl_health_reporter_get_dumpit(struct sk_buff *skb,
 int devlink_nl_health_reporter_set_doit(struct sk_buff *skb,
 					struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_health_reporter *reporter;
 
 	reporter = devlink_health_reporter_get_from_info(devlink, info);
@@ -715,7 +715,7 @@ EXPORT_SYMBOL_GPL(devlink_health_reporter_state_update);
 int devlink_nl_health_reporter_recover_doit(struct sk_buff *skb,
 					    struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_health_reporter *reporter;
 
 	reporter = devlink_health_reporter_get_from_info(devlink, info);
@@ -1157,7 +1157,7 @@ static int devlink_fmsg_dumpit(struct devlink_fmsg *fmsg, struct sk_buff *skb,
 int devlink_nl_health_reporter_diagnose_doit(struct sk_buff *skb,
 					     struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_health_reporter *reporter;
 	struct devlink_fmsg *fmsg;
 	int err;
@@ -1252,7 +1252,7 @@ int devlink_nl_health_reporter_dump_get_dumpit(struct sk_buff *skb,
 int devlink_nl_health_reporter_dump_clear_doit(struct sk_buff *skb,
 					       struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_health_reporter *reporter;
 
 	reporter = devlink_health_reporter_get_from_info(devlink, info);
@@ -1269,7 +1269,7 @@ int devlink_nl_health_reporter_dump_clear_doit(struct sk_buff *skb,
 int devlink_nl_health_reporter_test_doit(struct sk_buff *skb,
 					 struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_health_reporter *reporter;
 
 	reporter = devlink_health_reporter_get_from_info(devlink, info);
diff --git a/net/devlink/linecard.c b/net/devlink/linecard.c
index 8315d35cb91db..fd18f2759770d 100644
--- a/net/devlink/linecard.c
+++ b/net/devlink/linecard.c
@@ -171,7 +171,7 @@ void devlink_linecards_notify_unregister(struct devlink *devlink)
 
 int devlink_nl_linecard_get_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_linecard *linecard;
 	struct sk_buff *msg;
 	int err;
@@ -371,7 +371,7 @@ static int devlink_linecard_type_unset(struct devlink_linecard *linecard,
 int devlink_nl_linecard_set_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct netlink_ext_ack *extack = info->extack;
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_linecard *linecard;
 	int err;
 
diff --git a/net/devlink/netlink.c b/net/devlink/netlink.c
index ae4afc7396786..300580c1a2174 100644
--- a/net/devlink/netlink.c
+++ b/net/devlink/netlink.c
@@ -12,6 +12,7 @@
 #define DEVLINK_NL_FLAG_NEED_PORT		BIT(0)
 #define DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT	BIT(1)
 #define DEVLINK_NL_FLAG_NEED_DEV_LOCK		BIT(2)
+#define DEVLINK_NL_FLAG_OPTIONAL_PARENT_DEV	BIT(3)
 
 static const struct genl_multicast_group devlink_nl_mcgrps[] = {
 	[DEVLINK_MCGRP_CONFIG] = { .name = DEVLINK_GENL_MCGRP_CONFIG_NAME },
@@ -239,37 +240,82 @@ devlink_get_from_attrs_lock(struct net *net, struct nlattr **attrs,
 	return ERR_PTR(-ENODEV);
 }
 
+struct devlink *
+devlink_get_parent_from_attrs_lock(struct net *net, struct nlattr **attrs)
+{
+	unsigned int maxtype = ARRAY_SIZE(devlink_dl_parent_dev_nl_policy) - 1;
+	struct devlink *devlink;
+	struct nlattr **tb;
+	int err;
+
+	if (!attrs[DEVLINK_ATTR_PARENT_DEV])
+		return ERR_PTR(-EINVAL);
+
+	tb = kcalloc(maxtype + 1, sizeof(*tb), GFP_KERNEL);
+	if (!tb)
+		return ERR_PTR(-ENOMEM);
+
+	err = nla_parse_nested(tb, maxtype, attrs[DEVLINK_ATTR_PARENT_DEV],
+			       devlink_dl_parent_dev_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	devlink = devlink_get_from_attrs_lock(net, tb, false);
+	kfree(tb);
+	return devlink;
+out:
+	kfree(tb);
+	return ERR_PTR(err);
+}
+
 static int __devlink_nl_pre_doit(struct sk_buff *skb, struct genl_info *info,
 				 u8 flags)
 {
+	bool parent_dev = flags & DEVLINK_NL_FLAG_OPTIONAL_PARENT_DEV;
 	bool dev_lock = flags & DEVLINK_NL_FLAG_NEED_DEV_LOCK;
+	struct devlink *devlink, *parent_devlink = NULL;
+	struct net *net = genl_info_net(info);
+	struct nlattr **attrs = info->attrs;
 	struct devlink_port *devlink_port;
-	struct devlink *devlink;
 	int err;
 
-	devlink = devlink_get_from_attrs_lock(genl_info_net(info), info->attrs,
-					      dev_lock);
-	if (IS_ERR(devlink))
-		return PTR_ERR(devlink);
+	if (parent_dev && attrs[DEVLINK_ATTR_PARENT_DEV]) {
+		parent_devlink = devlink_get_parent_from_attrs_lock(net, attrs);
+		if (IS_ERR(parent_devlink))
+			return PTR_ERR(parent_devlink);
+		devlink_nl_ctx(info)->parent_devlink = parent_devlink;
+		/* Drop the parent devlink lock but don't release the reference.
+		 * This will keep it alive until the end of the request.
+		 */
+		devl_unlock(parent_devlink);
+	}
 
-	info->user_ptr[0] = devlink;
+	devlink = devlink_get_from_attrs_lock(net, attrs, dev_lock);
+	if (IS_ERR(devlink)) {
+		err = PTR_ERR(devlink);
+		goto parent_put;
+	}
+	devlink_nl_ctx(info)->devlink = devlink;
 	if (flags & DEVLINK_NL_FLAG_NEED_PORT) {
 		devlink_port = devlink_port_get_from_info(devlink, info);
 		if (IS_ERR(devlink_port)) {
 			err = PTR_ERR(devlink_port);
 			goto unlock;
 		}
-		info->user_ptr[1] = devlink_port;
+		devlink_nl_ctx(info)->devlink_port = devlink_port;
 	} else if (flags & DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT) {
 		devlink_port = devlink_port_get_from_info(devlink, info);
 		if (!IS_ERR(devlink_port))
-			info->user_ptr[1] = devlink_port;
+			devlink_nl_ctx(info)->devlink_port = devlink_port;
 	}
 	return 0;
 
 unlock:
 	devl_dev_unlock(devlink, dev_lock);
 	devlink_put(devlink);
+parent_put:
+	if (parent_dev && parent_devlink)
+		devlink_put(parent_devlink);
 	return err;
 }
 
@@ -298,15 +344,25 @@ int devlink_nl_pre_doit_port_optional(const struct genl_split_ops *ops,
 	return __devlink_nl_pre_doit(skb, info, DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT);
 }
 
+int devlink_nl_pre_doit_parent_dev_optional(const struct genl_split_ops *ops,
+					    struct sk_buff *skb,
+					    struct genl_info *info)
+{
+	return __devlink_nl_pre_doit(skb, info,
+				     DEVLINK_NL_FLAG_OPTIONAL_PARENT_DEV);
+}
+
 static void __devlink_nl_post_doit(struct sk_buff *skb, struct genl_info *info,
 				   u8 flags)
 {
 	bool dev_lock = flags & DEVLINK_NL_FLAG_NEED_DEV_LOCK;
 	struct devlink *devlink;
 
-	devlink = info->user_ptr[0];
+	devlink = devlink_nl_ctx(info)->devlink;
 	devl_dev_unlock(devlink, dev_lock);
 	devlink_put(devlink);
+	if (devlink_nl_ctx(info)->parent_devlink)
+		devlink_put(devlink_nl_ctx(info)->parent_devlink);
 }
 
 void devlink_nl_post_doit(const struct genl_split_ops *ops,
@@ -322,6 +378,14 @@ devlink_nl_post_doit_dev_lock(const struct genl_split_ops *ops,
 	__devlink_nl_post_doit(skb, info, DEVLINK_NL_FLAG_NEED_DEV_LOCK);
 }
 
+void
+devlink_nl_post_doit_parent_dev_optional(const struct genl_split_ops *ops,
+					 struct sk_buff *skb,
+					 struct genl_info *info)
+{
+	__devlink_nl_post_doit(skb, info, DEVLINK_NL_FLAG_OPTIONAL_PARENT_DEV);
+}
+
 static int devlink_nl_inst_single_dumpit(struct sk_buff *msg,
 					 struct netlink_callback *cb, int flags,
 					 devlink_nl_dump_one_func_t *dump_one,
diff --git a/net/devlink/netlink_gen.c b/net/devlink/netlink_gen.c
index f52b0c2b19ed6..dec00133178d1 100644
--- a/net/devlink/netlink_gen.c
+++ b/net/devlink/netlink_gen.c
@@ -46,6 +46,12 @@ devlink_attr_param_type_validate(const struct nlattr *attr,
 }
 
 /* Common nested types */
+const struct nla_policy devlink_dl_parent_dev_nl_policy[DEVLINK_ATTR_INDEX + 1] = {
+	[DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING, },
+	[DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING, },
+	[DEVLINK_ATTR_INDEX] = NLA_POLICY_FULL_RANGE(NLA_UINT, &devlink_attr_index_range),
+};
+
 const struct nla_policy devlink_dl_port_function_nl_policy[DEVLINK_PORT_FN_ATTR_CAPS + 1] = {
 	[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR] = { .type = NLA_BINARY, },
 	[DEVLINK_PORT_FN_ATTR_STATE] = NLA_POLICY_MAX(NLA_U8, 1),
@@ -608,7 +614,7 @@ static const struct nla_policy devlink_rate_get_dump_nl_policy[DEVLINK_ATTR_INDE
 };
 
 /* DEVLINK_CMD_RATE_SET - do */
-static const struct nla_policy devlink_rate_set_nl_policy[DEVLINK_ATTR_INDEX + 1] = {
+static const struct nla_policy devlink_rate_set_nl_policy[DEVLINK_ATTR_PARENT_DEV + 1] = {
 	[DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING, },
 	[DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING, },
 	[DEVLINK_ATTR_INDEX] = NLA_POLICY_FULL_RANGE(NLA_UINT, &devlink_attr_index_range),
@@ -619,10 +625,11 @@ static const struct nla_policy devlink_rate_set_nl_policy[DEVLINK_ATTR_INDEX + 1
 	[DEVLINK_ATTR_RATE_TX_WEIGHT] = { .type = NLA_U32, },
 	[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] = { .type = NLA_NUL_STRING, },
 	[DEVLINK_ATTR_RATE_TC_BWS] = NLA_POLICY_NESTED(devlink_dl_rate_tc_bws_nl_policy),
+	[DEVLINK_ATTR_PARENT_DEV] = NLA_POLICY_NESTED(devlink_dl_parent_dev_nl_policy),
 };
 
 /* DEVLINK_CMD_RATE_NEW - do */
-static const struct nla_policy devlink_rate_new_nl_policy[DEVLINK_ATTR_INDEX + 1] = {
+static const struct nla_policy devlink_rate_new_nl_policy[DEVLINK_ATTR_PARENT_DEV + 1] = {
 	[DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING, },
 	[DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING, },
 	[DEVLINK_ATTR_INDEX] = NLA_POLICY_FULL_RANGE(NLA_UINT, &devlink_attr_index_range),
@@ -633,6 +640,7 @@ static const struct nla_policy devlink_rate_new_nl_policy[DEVLINK_ATTR_INDEX + 1
 	[DEVLINK_ATTR_RATE_TX_WEIGHT] = { .type = NLA_U32, },
 	[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] = { .type = NLA_NUL_STRING, },
 	[DEVLINK_ATTR_RATE_TC_BWS] = NLA_POLICY_NESTED(devlink_dl_rate_tc_bws_nl_policy),
+	[DEVLINK_ATTR_PARENT_DEV] = NLA_POLICY_NESTED(devlink_dl_parent_dev_nl_policy),
 };
 
 /* DEVLINK_CMD_RATE_DEL - do */
@@ -1290,21 +1298,21 @@ const struct genl_split_ops devlink_nl_ops[75] = {
 	{
 		.cmd		= DEVLINK_CMD_RATE_SET,
 		.validate	= GENL_DONT_VALIDATE_STRICT,
-		.pre_doit	= devlink_nl_pre_doit,
+		.pre_doit	= devlink_nl_pre_doit_parent_dev_optional,
 		.doit		= devlink_nl_rate_set_doit,
-		.post_doit	= devlink_nl_post_doit,
+		.post_doit	= devlink_nl_post_doit_parent_dev_optional,
 		.policy		= devlink_rate_set_nl_policy,
-		.maxattr	= DEVLINK_ATTR_INDEX,
+		.maxattr	= DEVLINK_ATTR_PARENT_DEV,
 		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
 	},
 	{
 		.cmd		= DEVLINK_CMD_RATE_NEW,
 		.validate	= GENL_DONT_VALIDATE_STRICT,
-		.pre_doit	= devlink_nl_pre_doit,
+		.pre_doit	= devlink_nl_pre_doit_parent_dev_optional,
 		.doit		= devlink_nl_rate_new_doit,
-		.post_doit	= devlink_nl_post_doit,
+		.post_doit	= devlink_nl_post_doit_parent_dev_optional,
 		.policy		= devlink_rate_new_nl_policy,
-		.maxattr	= DEVLINK_ATTR_INDEX,
+		.maxattr	= DEVLINK_ATTR_PARENT_DEV,
 		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
 	},
 	{
diff --git a/net/devlink/netlink_gen.h b/net/devlink/netlink_gen.h
index 20034b0929a82..a70e0e4769aa5 100644
--- a/net/devlink/netlink_gen.h
+++ b/net/devlink/netlink_gen.h
@@ -13,6 +13,7 @@
 #include <uapi/linux/devlink.h>
 
 /* Common nested types */
+extern const struct nla_policy devlink_dl_parent_dev_nl_policy[DEVLINK_ATTR_INDEX + 1];
 extern const struct nla_policy devlink_dl_port_function_nl_policy[DEVLINK_PORT_FN_ATTR_CAPS + 1];
 extern const struct nla_policy devlink_dl_rate_tc_bws_nl_policy[DEVLINK_RATE_TC_ATTR_BW + 1];
 extern const struct nla_policy devlink_dl_selftest_id_nl_policy[DEVLINK_ATTR_SELFTEST_ID_FLASH + 1];
@@ -29,12 +30,19 @@ int devlink_nl_pre_doit_port_optional(const struct genl_split_ops *ops,
 				      struct genl_info *info);
 int devlink_nl_pre_doit_dev_lock(const struct genl_split_ops *ops,
 				 struct sk_buff *skb, struct genl_info *info);
+int devlink_nl_pre_doit_parent_dev_optional(const struct genl_split_ops *ops,
+					    struct sk_buff *skb,
+					    struct genl_info *info);
 void
 devlink_nl_post_doit(const struct genl_split_ops *ops, struct sk_buff *skb,
 		     struct genl_info *info);
 void
 devlink_nl_post_doit_dev_lock(const struct genl_split_ops *ops,
 			      struct sk_buff *skb, struct genl_info *info);
+void
+devlink_nl_post_doit_parent_dev_optional(const struct genl_split_ops *ops,
+					 struct sk_buff *skb,
+					 struct genl_info *info);
 
 int devlink_nl_get_doit(struct sk_buff *skb, struct genl_info *info);
 int devlink_nl_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb);
diff --git a/net/devlink/param.c b/net/devlink/param.c
index 3e9d2e5750c23..1cc562a6ebfdf 100644
--- a/net/devlink/param.c
+++ b/net/devlink/param.c
@@ -627,7 +627,7 @@ devlink_param_get_from_info(struct xarray *params, struct genl_info *info)
 int devlink_nl_param_get_doit(struct sk_buff *skb,
 			      struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_param_item *param_item;
 	struct sk_buff *msg;
 	int err;
@@ -728,7 +728,7 @@ static int __devlink_nl_cmd_param_set_doit(struct devlink *devlink,
 
 int devlink_nl_param_set_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 
 	return __devlink_nl_cmd_param_set_doit(devlink, 0, &devlink->params,
 					       info, DEVLINK_CMD_PARAM_NEW);
diff --git a/net/devlink/port.c b/net/devlink/port.c
index 485029d434287..dc82cac68e7d0 100644
--- a/net/devlink/port.c
+++ b/net/devlink/port.c
@@ -594,7 +594,7 @@ void devlink_ports_notify_unregister(struct devlink *devlink)
 
 int devlink_nl_port_get_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink_port *devlink_port = info->user_ptr[1];
+	struct devlink_port *devlink_port = devlink_nl_ctx(info)->devlink_port;
 	struct sk_buff *msg;
 	int err;
 
@@ -830,7 +830,7 @@ static int devlink_port_function_set(struct devlink_port *port,
 
 int devlink_nl_port_set_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink_port *devlink_port = info->user_ptr[1];
+	struct devlink_port *devlink_port = devlink_nl_ctx(info)->devlink_port;
 	int err;
 
 	if (info->attrs[DEVLINK_ATTR_PORT_TYPE]) {
@@ -856,8 +856,8 @@ int devlink_nl_port_set_doit(struct sk_buff *skb, struct genl_info *info)
 
 int devlink_nl_port_split_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink_port *devlink_port = info->user_ptr[1];
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink_port *devlink_port = devlink_nl_ctx(info)->devlink_port;
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	u32 count;
 
 	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PORT_SPLIT_COUNT))
@@ -887,8 +887,8 @@ int devlink_nl_port_split_doit(struct sk_buff *skb, struct genl_info *info)
 
 int devlink_nl_port_unsplit_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink_port *devlink_port = info->user_ptr[1];
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink_port *devlink_port = devlink_nl_ctx(info)->devlink_port;
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 
 	if (!devlink_port->ops->port_unsplit)
 		return -EOPNOTSUPP;
@@ -899,7 +899,7 @@ int devlink_nl_port_new_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct netlink_ext_ack *extack = info->extack;
 	struct devlink_port_new_attrs new_attrs = {};
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_port *devlink_port;
 	struct sk_buff *msg;
 	int err;
@@ -961,9 +961,9 @@ int devlink_nl_port_new_doit(struct sk_buff *skb, struct genl_info *info)
 
 int devlink_nl_port_del_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink_port *devlink_port = info->user_ptr[1];
+	struct devlink_port *devlink_port = devlink_nl_ctx(info)->devlink_port;
 	struct netlink_ext_ack *extack = info->extack;
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 
 	if (!devlink_port->ops->port_del)
 		return -EOPNOTSUPP;
@@ -1529,7 +1529,7 @@ static int __devlink_port_phys_port_name_get(struct devlink_port *devlink_port,
 		WARN_ON(1);
 		return -EINVAL;
 	case DEVLINK_PORT_FLAVOUR_PCI_PF:
-		if (attrs->pci_pf.external) {
+		if (attrs->pci_pf.external || attrs->pci_pf.controller) {
 			n = snprintf(name, len, "c%u", attrs->pci_pf.controller);
 			if (n >= len)
 				return -EINVAL;
@@ -1539,7 +1539,7 @@ static int __devlink_port_phys_port_name_get(struct devlink_port *devlink_port,
 		n = snprintf(name, len, "pf%u", attrs->pci_pf.pf);
 		break;
 	case DEVLINK_PORT_FLAVOUR_PCI_VF:
-		if (attrs->pci_vf.external) {
+		if (attrs->pci_vf.external || attrs->pci_vf.controller) {
 			n = snprintf(name, len, "c%u", attrs->pci_vf.controller);
 			if (n >= len)
 				return -EINVAL;
@@ -1550,7 +1550,7 @@ static int __devlink_port_phys_port_name_get(struct devlink_port *devlink_port,
 			     attrs->pci_vf.pf, attrs->pci_vf.vf);
 		break;
 	case DEVLINK_PORT_FLAVOUR_PCI_SF:
-		if (attrs->pci_sf.external) {
+		if (attrs->pci_sf.external || attrs->pci_sf.controller) {
 			n = snprintf(name, len, "c%u", attrs->pci_sf.controller);
 			if (n >= len)
 				return -EINVAL;
diff --git a/net/devlink/rate.c b/net/devlink/rate.c
index 533d21b028a7b..e727c8b8b33ea 100644
--- a/net/devlink/rate.c
+++ b/net/devlink/rate.c
@@ -30,13 +30,53 @@ devlink_rate_leaf_get_from_info(struct devlink *devlink, struct genl_info *info)
 	return devlink_rate ?: ERR_PTR(-ENODEV);
 }
 
+/* Repeatedly walks the nested devlink chain while cross device rate nodes are
+ * supported and finds the topmost instance where rates should be stored.
+ * That instance is locked, referenced and returned.
+ * When cross device rate nodes aren't supported the original devlink instance
+ * is returned.
+ */
+static struct devlink *devl_rate_lock(struct devlink *devlink)
+{
+	struct devlink *rate_devlink = devlink, *parent;
+
+	devl_assert_locked(devlink);
+
+	while (rate_devlink->ops &&
+	       rate_devlink->ops->supported_cross_device_rate_nodes) {
+		parent = devlink_nested_in_get_lock(rate_devlink);
+		if (!parent)
+			break;
+		if (rate_devlink != devlink) {
+			/* Unlock intermediate instances. */
+			devl_unlock(rate_devlink);
+			devlink_put(rate_devlink);
+		}
+		rate_devlink = parent;
+	}
+	return rate_devlink;
+}
+
+/* Unlocks and puts 'rate devlink' if different than 'devlink'. */
+static void devl_rate_unlock(struct devlink *devlink,
+			     struct devlink *rate_devlink)
+{
+	if (devlink == rate_devlink)
+		return;
+
+	devl_unlock(rate_devlink);
+	devlink_put(rate_devlink);
+}
+
 static struct devlink_rate *
-devlink_rate_node_get_by_name(struct devlink *devlink, const char *node_name)
+devlink_rate_node_get_by_name(struct devlink *rate_devlink,
+			      struct devlink *devlink, const char *node_name)
 {
 	struct devlink_rate *devlink_rate;
 
-	list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
-		if (devlink_rate_is_node(devlink_rate) &&
+	list_for_each_entry(devlink_rate, &rate_devlink->rate_list, list) {
+		if (devlink_rate->devlink == devlink &&
+		    devlink_rate_is_node(devlink_rate) &&
 		    !strcmp(node_name, devlink_rate->name))
 			return devlink_rate;
 	}
@@ -44,7 +84,8 @@ devlink_rate_node_get_by_name(struct devlink *devlink, const char *node_name)
 }
 
 static struct devlink_rate *
-devlink_rate_node_get_from_attrs(struct devlink *devlink, struct nlattr **attrs)
+devlink_rate_node_get_from_attrs(struct devlink *rate_devlink,
+				 struct devlink *devlink, struct nlattr **attrs)
 {
 	const char *rate_node_name;
 	size_t len;
@@ -57,24 +98,30 @@ devlink_rate_node_get_from_attrs(struct devlink *devlink, struct nlattr **attrs)
 	if (!len || strspn(rate_node_name, "0123456789") == len)
 		return ERR_PTR(-EINVAL);
 
-	return devlink_rate_node_get_by_name(devlink, rate_node_name);
+	return devlink_rate_node_get_by_name(rate_devlink, devlink,
+					     rate_node_name);
 }
 
 static struct devlink_rate *
-devlink_rate_node_get_from_info(struct devlink *devlink, struct genl_info *info)
+devlink_rate_node_get_from_info(struct devlink *rate_devlink,
+				struct devlink *devlink,
+				struct genl_info *info)
 {
-	return devlink_rate_node_get_from_attrs(devlink, info->attrs);
+	return devlink_rate_node_get_from_attrs(rate_devlink, devlink,
+						info->attrs);
 }
 
 static struct devlink_rate *
-devlink_rate_get_from_info(struct devlink *devlink, struct genl_info *info)
+devlink_rate_get_from_info(struct devlink *rate_devlink,
+			   struct devlink *devlink, struct genl_info *info)
 {
 	struct nlattr **attrs = info->attrs;
 
 	if (attrs[DEVLINK_ATTR_PORT_INDEX])
 		return devlink_rate_leaf_get_from_info(devlink, info);
 	else if (attrs[DEVLINK_ATTR_RATE_NODE_NAME])
-		return devlink_rate_node_get_from_info(devlink, info);
+		return devlink_rate_node_get_from_info(rate_devlink, devlink,
+						       info);
 	else
 		return ERR_PTR(-EINVAL);
 }
@@ -102,6 +149,25 @@ static int devlink_rate_put_tc_bws(struct sk_buff *msg, u32 *tc_bw)
 	return -EMSGSIZE;
 }
 
+static int devlink_nl_rate_parent_fill(struct sk_buff *msg,
+				       struct devlink_rate *devlink_rate)
+{
+	struct devlink_rate *parent = devlink_rate->parent;
+	struct devlink *devlink = parent->devlink;
+
+	if (nla_put_string(msg, DEVLINK_ATTR_RATE_PARENT_NODE_NAME,
+			   parent->name))
+		return -EMSGSIZE;
+
+	if (devlink != devlink_rate->devlink &&
+	    devlink_nl_put_nested_handle(msg,
+					 devlink_net(devlink_rate->devlink),
+					 devlink, DEVLINK_ATTR_PARENT_DEV))
+		return -EMSGSIZE;
+
+	return 0;
+}
+
 static int devlink_nl_rate_fill(struct sk_buff *msg,
 				struct devlink_rate *devlink_rate,
 				enum devlink_command cmd, u32 portid, u32 seq,
@@ -146,10 +212,9 @@ static int devlink_nl_rate_fill(struct sk_buff *msg,
 			devlink_rate->tx_weight))
 		goto nla_put_failure;
 
-	if (devlink_rate->parent)
-		if (nla_put_string(msg, DEVLINK_ATTR_RATE_PARENT_NODE_NAME,
-				   devlink_rate->parent->name))
-			goto nla_put_failure;
+	if (devlink_rate->parent &&
+	    devlink_nl_rate_parent_fill(msg, devlink_rate))
+		goto nla_put_failure;
 
 	if (devlink_rate_put_tc_bws(msg, devlink_rate->tc_bw))
 		goto nla_put_failure;
@@ -190,17 +255,25 @@ static void devlink_rate_notify(struct devlink_rate *devlink_rate,
 void devlink_rates_notify_register(struct devlink *devlink)
 {
 	struct devlink_rate *rate_node;
+	struct devlink *rate_devlink;
 
-	list_for_each_entry(rate_node, &devlink->rate_list, list)
-		devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW);
+	rate_devlink = devl_rate_lock(devlink);
+	list_for_each_entry(rate_node, &rate_devlink->rate_list, list)
+		if (rate_node->devlink == devlink)
+			devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW);
+	devl_rate_unlock(devlink, rate_devlink);
 }
 
 void devlink_rates_notify_unregister(struct devlink *devlink)
 {
 	struct devlink_rate *rate_node;
+	struct devlink *rate_devlink;
 
-	list_for_each_entry_reverse(rate_node, &devlink->rate_list, list)
-		devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_DEL);
+	rate_devlink = devl_rate_lock(devlink);
+	list_for_each_entry_reverse(rate_node, &rate_devlink->rate_list, list)
+		if (rate_node->devlink == devlink)
+			devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_DEL);
+	devl_rate_unlock(devlink, rate_devlink);
 }
 
 static int
@@ -209,17 +282,20 @@ devlink_nl_rate_get_dump_one(struct sk_buff *msg, struct devlink *devlink,
 {
 	struct devlink_nl_dump_state *state = devlink_dump_state(cb);
 	struct devlink_rate *devlink_rate;
+	struct devlink *rate_devlink;
 	int idx = 0;
 	int err = 0;
 
-	list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
+	rate_devlink = devl_rate_lock(devlink);
+	list_for_each_entry(devlink_rate, &rate_devlink->rate_list, list) {
 		enum devlink_command cmd = DEVLINK_CMD_RATE_NEW;
 		u32 id = NETLINK_CB(cb->skb).portid;
 
-		if (idx < state->idx) {
+		if (idx < state->idx || devlink_rate->devlink != devlink) {
 			idx++;
 			continue;
 		}
+
 		err = devlink_nl_rate_fill(msg, devlink_rate, cmd, id,
 					   cb->nlh->nlmsg_seq, flags, NULL);
 		if (err) {
@@ -228,6 +304,7 @@ devlink_nl_rate_get_dump_one(struct sk_buff *msg, struct devlink *devlink,
 		}
 		idx++;
 	}
+	devl_rate_unlock(devlink, rate_devlink);
 
 	return err;
 }
@@ -239,28 +316,38 @@ int devlink_nl_rate_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
 
 int devlink_nl_rate_get_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *rate_devlink, *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_rate *devlink_rate;
 	struct sk_buff *msg;
 	int err;
 
-	devlink_rate = devlink_rate_get_from_info(devlink, info);
-	if (IS_ERR(devlink_rate))
-		return PTR_ERR(devlink_rate);
+	rate_devlink = devl_rate_lock(devlink);
+	devlink_rate = devlink_rate_get_from_info(rate_devlink, devlink, info);
+	if (IS_ERR(devlink_rate)) {
+		err = PTR_ERR(devlink_rate);
+		goto unlock;
+	}
 
 	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
-	if (!msg)
-		return -ENOMEM;
+	if (!msg) {
+		err = -ENOMEM;
+		goto unlock;
+	}
 
 	err = devlink_nl_rate_fill(msg, devlink_rate, DEVLINK_CMD_RATE_NEW,
 				   info->snd_portid, info->snd_seq, 0,
 				   info->extack);
-	if (err) {
-		nlmsg_free(msg);
-		return err;
-	}
+	if (err)
+		goto err_fill;
 
+	devl_rate_unlock(devlink, rate_devlink);
 	return genlmsg_reply(msg, info);
+
+err_fill:
+	nlmsg_free(msg);
+unlock:
+	devl_rate_unlock(devlink, rate_devlink);
+	return err;
 }
 
 static bool
@@ -277,16 +364,18 @@ devlink_rate_is_parent_node(struct devlink_rate *devlink_rate,
 
 static int
 devlink_nl_rate_parent_node_set(struct devlink_rate *devlink_rate,
+				struct devlink *rate_devlink,
 				struct genl_info *info,
 				struct nlattr *nla_parent)
 {
-	struct devlink *devlink = devlink_rate->devlink;
+	struct devlink *devlink = devlink_rate->devlink, *parent_devlink;
 	const char *parent_name = nla_data(nla_parent);
 	const struct devlink_ops *ops = devlink->ops;
 	size_t len = strlen(parent_name);
 	struct devlink_rate *parent;
 	int err = -EOPNOTSUPP;
 
+	parent_devlink = devlink_nl_ctx(info)->parent_devlink ? : devlink;
 	parent = devlink_rate->parent;
 
 	if (parent && !len) {
@@ -304,7 +393,14 @@ devlink_nl_rate_parent_node_set(struct devlink_rate *devlink_rate,
 		refcount_dec(&parent->refcnt);
 		devlink_rate->parent = NULL;
 	} else if (len) {
-		parent = devlink_rate_node_get_by_name(devlink, parent_name);
+		/* parent_devlink (when different than devlink) isn't locked,
+		 * but the rate node devlink instance is, so nobody from the
+		 * same group of devices sharing rates could change the used
+		 * fields or unregister the parent.
+		 */
+		parent = devlink_rate_node_get_by_name(rate_devlink,
+						       parent_devlink,
+						       parent_name);
 		if (IS_ERR(parent))
 			return -ENODEV;
 
@@ -423,6 +519,7 @@ static int devlink_nl_rate_tc_bw_set(struct devlink_rate *devlink_rate,
 }
 
 static int devlink_nl_rate_set(struct devlink_rate *devlink_rate,
+			       struct devlink *rate_devlink,
 			       const struct devlink_ops *ops,
 			       struct genl_info *info)
 {
@@ -497,7 +594,8 @@ static int devlink_nl_rate_set(struct devlink_rate *devlink_rate,
 	 */
 	nla_parent = attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME];
 	if (nla_parent) {
-		err = devlink_nl_rate_parent_node_set(devlink_rate, info,
+		err = devlink_nl_rate_parent_node_set(devlink_rate,
+						      rate_devlink, info,
 						      nla_parent);
 		if (err)
 			return err;
@@ -588,31 +686,51 @@ static bool devlink_rate_set_ops_supported(const struct devlink_ops *ops,
 
 int devlink_nl_rate_set_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink_nl_ctx *ctx = devlink_nl_ctx(info);
+	struct devlink *devlink = ctx->devlink;
 	struct devlink_rate *devlink_rate;
 	const struct devlink_ops *ops;
+	struct devlink *rate_devlink;
 	int err;
 
-	devlink_rate = devlink_rate_get_from_info(devlink, info);
-	if (IS_ERR(devlink_rate))
-		return PTR_ERR(devlink_rate);
+	rate_devlink = devl_rate_lock(devlink);
+	devlink_rate = devlink_rate_get_from_info(rate_devlink, devlink, info);
+	if (IS_ERR(devlink_rate)) {
+		err = PTR_ERR(devlink_rate);
+		goto unlock;
+	}
 
 	ops = devlink->ops;
-	if (!ops || !devlink_rate_set_ops_supported(ops, info, devlink_rate->type))
-		return -EOPNOTSUPP;
+	if (!ops ||
+	    !devlink_rate_set_ops_supported(ops, info, devlink_rate->type)) {
+		err = -EOPNOTSUPP;
+		goto unlock;
+	}
+
+	if (ctx->parent_devlink && ctx->parent_devlink != devlink &&
+	    !ops->supported_cross_device_rate_nodes) {
+		NL_SET_ERR_MSG(info->extack,
+			       "Cross-device rate parents aren't supported");
+		err = -EOPNOTSUPP;
+		goto unlock;
+	}
 
-	err = devlink_nl_rate_set(devlink_rate, ops, info);
+	err = devlink_nl_rate_set(devlink_rate, rate_devlink, ops, info);
 
 	if (!err)
 		devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_NEW);
+unlock:
+	devl_rate_unlock(devlink, rate_devlink);
 	return err;
 }
 
 int devlink_nl_rate_new_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink_nl_ctx *ctx = devlink_nl_ctx(info);
+	struct devlink *devlink = ctx->devlink;
 	struct devlink_rate *rate_node;
 	const struct devlink_ops *ops;
+	struct devlink *rate_devlink;
 	int err;
 
 	ops = devlink->ops;
@@ -624,15 +742,29 @@ int devlink_nl_rate_new_doit(struct sk_buff *skb, struct genl_info *info)
 	if (!devlink_rate_set_ops_supported(ops, info, DEVLINK_RATE_TYPE_NODE))
 		return -EOPNOTSUPP;
 
-	rate_node = devlink_rate_node_get_from_attrs(devlink, info->attrs);
-	if (!IS_ERR(rate_node))
-		return -EEXIST;
-	else if (rate_node == ERR_PTR(-EINVAL))
-		return -EINVAL;
+	if (ctx->parent_devlink && ctx->parent_devlink != devlink &&
+	    !ops->supported_cross_device_rate_nodes) {
+		NL_SET_ERR_MSG(info->extack,
+			       "Cross-device rate parents aren't supported");
+		return -EOPNOTSUPP;
+	}
+
+	rate_devlink = devl_rate_lock(devlink);
+	rate_node = devlink_rate_node_get_from_attrs(rate_devlink, devlink,
+						     info->attrs);
+	if (!IS_ERR(rate_node)) {
+		err = -EEXIST;
+		goto unlock;
+	} else if (rate_node == ERR_PTR(-EINVAL)) {
+		err = -EINVAL;
+		goto unlock;
+	}
 
 	rate_node = kzalloc_obj(*rate_node);
-	if (!rate_node)
-		return -ENOMEM;
+	if (!rate_node) {
+		err = -ENOMEM;
+		goto unlock;
+	}
 
 	rate_node->devlink = devlink;
 	rate_node->type = DEVLINK_RATE_TYPE_NODE;
@@ -646,13 +778,14 @@ int devlink_nl_rate_new_doit(struct sk_buff *skb, struct genl_info *info)
 	if (err)
 		goto err_node_new;
 
-	err = devlink_nl_rate_set(rate_node, ops, info);
+	err = devlink_nl_rate_set(rate_node, rate_devlink, ops, info);
 	if (err)
 		goto err_rate_set;
 
 	refcount_set(&rate_node->refcnt, 1);
-	list_add(&rate_node->list, &devlink->rate_list);
+	list_add(&rate_node->list, &rate_devlink->rate_list);
 	devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW);
+	devl_rate_unlock(devlink, rate_devlink);
 	return 0;
 
 err_rate_set:
@@ -661,22 +794,29 @@ int devlink_nl_rate_new_doit(struct sk_buff *skb, struct genl_info *info)
 	kfree(rate_node->name);
 err_strdup:
 	kfree(rate_node);
+unlock:
+	devl_rate_unlock(devlink, rate_devlink);
 	return err;
 }
 
 int devlink_nl_rate_del_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *rate_devlink, *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_rate *rate_node;
 	int err;
 
-	rate_node = devlink_rate_node_get_from_info(devlink, info);
-	if (IS_ERR(rate_node))
-		return PTR_ERR(rate_node);
+	rate_devlink = devl_rate_lock(devlink);
+	rate_node = devlink_rate_node_get_from_info(rate_devlink, devlink,
+						    info);
+	if (IS_ERR(rate_node)) {
+		err = PTR_ERR(rate_node);
+		goto unlock;
+	}
 
 	if (refcount_read(&rate_node->refcnt) > 1) {
 		NL_SET_ERR_MSG(info->extack, "Node has children. Cannot delete node.");
-		return -EBUSY;
+		err = -EBUSY;
+		goto unlock;
 	}
 
 	devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_DEL);
@@ -687,6 +827,8 @@ int devlink_nl_rate_del_doit(struct sk_buff *skb, struct genl_info *info)
 	list_del(&rate_node->list);
 	kfree(rate_node->name);
 	kfree(rate_node);
+unlock:
+	devl_rate_unlock(devlink, rate_devlink);
 	return err;
 }
 
@@ -695,14 +837,20 @@ int devlink_rates_check(struct devlink *devlink,
 			struct netlink_ext_ack *extack)
 {
 	struct devlink_rate *devlink_rate;
+	struct devlink *rate_devlink;
+	int err = 0;
 
-	list_for_each_entry(devlink_rate, &devlink->rate_list, list)
-		if (!rate_filter || rate_filter(devlink_rate)) {
+	rate_devlink = devl_rate_lock(devlink);
+	list_for_each_entry(devlink_rate, &rate_devlink->rate_list, list)
+		if (devlink_rate->devlink == devlink &&
+		    (!rate_filter || rate_filter(devlink_rate))) {
 			if (extack)
 				NL_SET_ERR_MSG(extack, "Rate node(s) exists.");
-			return -EBUSY;
+			err = -EBUSY;
+			break;
 		}
-	return 0;
+	devl_rate_unlock(devlink, rate_devlink);
+	return err;
 }
 
 /**
@@ -719,14 +867,21 @@ devl_rate_node_create(struct devlink *devlink, void *priv, char *node_name,
 		      struct devlink_rate *parent)
 {
 	struct devlink_rate *rate_node;
-
-	rate_node = devlink_rate_node_get_by_name(devlink, node_name);
-	if (!IS_ERR(rate_node))
-		return ERR_PTR(-EEXIST);
+	struct devlink *rate_devlink;
+
+	rate_devlink = devl_rate_lock(devlink);
+	rate_node = devlink_rate_node_get_by_name(rate_devlink, devlink,
+						  node_name);
+	if (!IS_ERR(rate_node)) {
+		rate_node = ERR_PTR(-EEXIST);
+		goto unlock;
+	}
 
 	rate_node = kzalloc_obj(*rate_node);
-	if (!rate_node)
-		return ERR_PTR(-ENOMEM);
+	if (!rate_node) {
+		rate_node = ERR_PTR(-ENOMEM);
+		goto unlock;
+	}
 
 	rate_node->type = DEVLINK_RATE_TYPE_NODE;
 	rate_node->devlink = devlink;
@@ -735,7 +890,8 @@ devl_rate_node_create(struct devlink *devlink, void *priv, char *node_name,
 	rate_node->name = kstrdup(node_name, GFP_KERNEL);
 	if (!rate_node->name) {
 		kfree(rate_node);
-		return ERR_PTR(-ENOMEM);
+		rate_node = ERR_PTR(-ENOMEM);
+		goto unlock;
 	}
 
 	if (parent) {
@@ -744,8 +900,10 @@ devl_rate_node_create(struct devlink *devlink, void *priv, char *node_name,
 	}
 
 	refcount_set(&rate_node->refcnt, 1);
-	list_add(&rate_node->list, &devlink->rate_list);
+	list_add(&rate_node->list, &rate_devlink->rate_list);
 	devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW);
+unlock:
+	devl_rate_unlock(devlink, rate_devlink);
 	return rate_node;
 }
 EXPORT_SYMBOL_GPL(devl_rate_node_create);
@@ -761,10 +919,10 @@ EXPORT_SYMBOL_GPL(devl_rate_node_create);
 int devl_rate_leaf_create(struct devlink_port *devlink_port, void *priv,
 			  struct devlink_rate *parent)
 {
-	struct devlink *devlink = devlink_port->devlink;
+	struct devlink *rate_devlink, *devlink = devlink_port->devlink;
 	struct devlink_rate *devlink_rate;
 
-	devl_assert_locked(devlink_port->devlink);
+	devl_assert_locked(devlink);
 
 	if (WARN_ON(devlink_port->devlink_rate))
 		return -EBUSY;
@@ -773,6 +931,7 @@ int devl_rate_leaf_create(struct devlink_port *devlink_port, void *priv,
 	if (!devlink_rate)
 		return -ENOMEM;
 
+	rate_devlink = devl_rate_lock(devlink);
 	if (parent) {
 		devlink_rate->parent = parent;
 		refcount_inc(&devlink_rate->parent->refcnt);
@@ -782,9 +941,10 @@ int devl_rate_leaf_create(struct devlink_port *devlink_port, void *priv,
 	devlink_rate->devlink = devlink;
 	devlink_rate->devlink_port = devlink_port;
 	devlink_rate->priv = priv;
-	list_add_tail(&devlink_rate->list, &devlink->rate_list);
+	list_add_tail(&devlink_rate->list, &rate_devlink->rate_list);
 	devlink_port->devlink_rate = devlink_rate;
 	devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_NEW);
+	devl_rate_unlock(devlink, rate_devlink);
 
 	return 0;
 }
@@ -800,16 +960,19 @@ EXPORT_SYMBOL_GPL(devl_rate_leaf_create);
 void devl_rate_leaf_destroy(struct devlink_port *devlink_port)
 {
 	struct devlink_rate *devlink_rate = devlink_port->devlink_rate;
+	struct devlink *rate_devlink, *devlink = devlink_port->devlink;
 
-	devl_assert_locked(devlink_port->devlink);
+	devl_assert_locked(devlink);
 	if (!devlink_rate)
 		return;
 
+	rate_devlink = devl_rate_lock(devlink);
 	devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_DEL);
 	if (devlink_rate->parent)
 		refcount_dec(&devlink_rate->parent->refcnt);
 	list_del(&devlink_rate->list);
 	devlink_port->devlink_rate = NULL;
+	devl_rate_unlock(devlink, rate_devlink);
 	kfree(devlink_rate);
 }
 EXPORT_SYMBOL_GPL(devl_rate_leaf_destroy);
@@ -818,20 +981,30 @@ EXPORT_SYMBOL_GPL(devl_rate_leaf_destroy);
  * devl_rate_nodes_destroy - destroy all devlink rate nodes on device
  * @devlink: devlink instance
  *
- * Unset parent for all rate objects and destroy all rate nodes
- * on specified device.
+ * Unset parent for all rate objects involving this device and destroy all rate
+ * nodes on it.
  */
 void devl_rate_nodes_destroy(struct devlink *devlink)
 {
-	const struct devlink_ops *ops = devlink->ops;
 	struct devlink_rate *devlink_rate, *tmp;
+	const struct devlink_ops *ops;
+	struct devlink *rate_devlink;
 
 	devl_assert_locked(devlink);
+	rate_devlink = devl_rate_lock(devlink);
 
-	list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
-		if (!devlink_rate->parent)
+	list_for_each_entry(devlink_rate, &rate_devlink->rate_list, list) {
+		if (!devlink_rate->parent ||
+		    (devlink_rate->devlink != devlink &&
+		     devlink_rate->parent->devlink != devlink))
 			continue;
 
+		/* This could destroy rate objects on other devlinks in the
+		 * same hierarchy under 'rate_devlink'. This is safe because
+		 * the shared common ancestor is locked so there can be no
+		 * other concurrent rate operations on devlink_rate->devlink.
+		 */
+		ops = devlink_rate->devlink->ops;
 		if (devlink_rate_is_leaf(devlink_rate))
 			ops->rate_leaf_parent_set(devlink_rate, NULL, devlink_rate->priv,
 						  NULL, NULL);
@@ -842,13 +1015,17 @@ void devl_rate_nodes_destroy(struct devlink *devlink)
 		refcount_dec(&devlink_rate->parent->refcnt);
 		devlink_rate->parent = NULL;
 	}
-	list_for_each_entry_safe(devlink_rate, tmp, &devlink->rate_list, list) {
-		if (devlink_rate_is_node(devlink_rate)) {
+	ops = devlink->ops;
+	list_for_each_entry_safe(devlink_rate, tmp, &rate_devlink->rate_list,
+				 list) {
+		if (devlink_rate->devlink == devlink &&
+		    devlink_rate_is_node(devlink_rate)) {
 			ops->rate_node_del(devlink_rate, devlink_rate->priv, NULL);
 			list_del(&devlink_rate->list);
 			kfree(devlink_rate->name);
 			kfree(devlink_rate);
 		}
 	}
+	devl_rate_unlock(devlink, rate_devlink);
 }
 EXPORT_SYMBOL_GPL(devl_rate_nodes_destroy);
diff --git a/net/devlink/region.c b/net/devlink/region.c
index 5588e3d560b92..537779bbff07a 100644
--- a/net/devlink/region.c
+++ b/net/devlink/region.c
@@ -469,7 +469,7 @@ static void devlink_region_snapshot_del(struct devlink_region *region,
 
 int devlink_nl_region_get_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_port *port = NULL;
 	struct devlink_region *region;
 	const char *region_name;
@@ -588,7 +588,7 @@ int devlink_nl_region_get_dumpit(struct sk_buff *skb,
 
 int devlink_nl_region_del_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_snapshot *snapshot;
 	struct devlink_port *port = NULL;
 	struct devlink_region *region;
@@ -633,7 +633,7 @@ int devlink_nl_region_del_doit(struct sk_buff *skb, struct genl_info *info)
 
 int devlink_nl_region_new_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_snapshot *snapshot;
 	struct devlink_port *port = NULL;
 	struct nlattr *snapshot_id_attr;
diff --git a/net/devlink/resource.c b/net/devlink/resource.c
index 574108ccfe5d2..c3cfda7ea0703 100644
--- a/net/devlink/resource.c
+++ b/net/devlink/resource.c
@@ -117,7 +117,7 @@ devlink_resource_validate_size(struct devlink_resource *resource, u64 size,
 
 int devlink_nl_resource_set_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_resource *resource;
 	u64 resource_id;
 	u64 size;
@@ -251,8 +251,9 @@ static int devlink_resource_list_fill(struct sk_buff *skb,
 static int devlink_resource_fill(struct genl_info *info,
 				 enum devlink_command cmd, int flags)
 {
-	struct devlink_port *devlink_port = info->user_ptr[1];
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink_nl_ctx *ctx = devlink_nl_ctx(info);
+	struct devlink *devlink = ctx->devlink;
+	struct devlink_port *devlink_port;
 	struct devlink_resource *resource;
 	struct list_head *resource_list;
 	struct nlattr *resources_attr;
@@ -263,6 +264,7 @@ static int devlink_resource_fill(struct genl_info *info,
 	int i;
 	int err;
 
+	devlink_port = ctx->devlink_port;
 	resource_list = devlink_port ?
 		&devlink_port->resource_list : &devlink->resource_list;
 	resource = list_first_entry(resource_list,
@@ -326,10 +328,12 @@ static int devlink_resource_fill(struct genl_info *info,
 
 int devlink_nl_resource_dump_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink_port *devlink_port = info->user_ptr[1];
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink_nl_ctx *ctx = devlink_nl_ctx(info);
+	struct devlink *devlink = ctx->devlink;
+	struct devlink_port *devlink_port;
 	struct list_head *resource_list;
 
+	devlink_port = ctx->devlink_port;
 	if (info->attrs[DEVLINK_ATTR_PORT_INDEX] && !devlink_port)
 		return -ENODEV;
 
diff --git a/net/devlink/sb.c b/net/devlink/sb.c
index 49fcbfe08f155..129bd016e3023 100644
--- a/net/devlink/sb.c
+++ b/net/devlink/sb.c
@@ -204,7 +204,7 @@ static int devlink_nl_sb_fill(struct sk_buff *msg, struct devlink *devlink,
 
 int devlink_nl_sb_get_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_sb *devlink_sb;
 	struct sk_buff *msg;
 	int err;
@@ -306,7 +306,7 @@ static int devlink_nl_sb_pool_fill(struct sk_buff *msg, struct devlink *devlink,
 
 int devlink_nl_sb_pool_get_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_sb *devlink_sb;
 	struct sk_buff *msg;
 	u16 pool_index;
@@ -415,7 +415,7 @@ static int devlink_sb_pool_set(struct devlink *devlink, unsigned int sb_index,
 
 int devlink_nl_sb_pool_set_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	enum devlink_sb_threshold_type threshold_type;
 	struct devlink_sb *devlink_sb;
 	u16 pool_index;
@@ -506,7 +506,7 @@ static int devlink_nl_sb_port_pool_fill(struct sk_buff *msg,
 int devlink_nl_sb_port_pool_get_doit(struct sk_buff *skb,
 				     struct genl_info *info)
 {
-	struct devlink_port *devlink_port = info->user_ptr[1];
+	struct devlink_port *devlink_port = devlink_nl_ctx(info)->devlink_port;
 	struct devlink *devlink = devlink_port->devlink;
 	struct devlink_sb *devlink_sb;
 	struct sk_buff *msg;
@@ -624,8 +624,8 @@ static int devlink_sb_port_pool_set(struct devlink_port *devlink_port,
 int devlink_nl_sb_port_pool_set_doit(struct sk_buff *skb,
 				     struct genl_info *info)
 {
-	struct devlink_port *devlink_port = info->user_ptr[1];
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink_port *devlink_port = devlink_nl_ctx(info)->devlink_port;
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_sb *devlink_sb;
 	u16 pool_index;
 	u32 threshold;
@@ -716,7 +716,7 @@ devlink_nl_sb_tc_pool_bind_fill(struct sk_buff *msg, struct devlink *devlink,
 int devlink_nl_sb_tc_pool_bind_get_doit(struct sk_buff *skb,
 					struct genl_info *info)
 {
-	struct devlink_port *devlink_port = info->user_ptr[1];
+	struct devlink_port *devlink_port = devlink_nl_ctx(info)->devlink_port;
 	struct devlink *devlink = devlink_port->devlink;
 	struct devlink_sb *devlink_sb;
 	struct sk_buff *msg;
@@ -864,8 +864,8 @@ static int devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port,
 int devlink_nl_sb_tc_pool_bind_set_doit(struct sk_buff *skb,
 					struct genl_info *info)
 {
-	struct devlink_port *devlink_port = info->user_ptr[1];
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink_port *devlink_port = devlink_nl_ctx(info)->devlink_port;
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	enum devlink_sb_pool_type pool_type;
 	struct devlink_sb *devlink_sb;
 	u16 tc_index;
@@ -902,7 +902,7 @@ int devlink_nl_sb_tc_pool_bind_set_doit(struct sk_buff *skb,
 
 int devlink_nl_sb_occ_snapshot_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	const struct devlink_ops *ops = devlink->ops;
 	struct devlink_sb *devlink_sb;
 
@@ -918,7 +918,7 @@ int devlink_nl_sb_occ_snapshot_doit(struct sk_buff *skb, struct genl_info *info)
 int devlink_nl_sb_occ_max_clear_doit(struct sk_buff *skb,
 				     struct genl_info *info)
 {
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	const struct devlink_ops *ops = devlink->ops;
 	struct devlink_sb *devlink_sb;
 
diff --git a/net/devlink/trap.c b/net/devlink/trap.c
index 8edb31654a68a..793ffc66dc110 100644
--- a/net/devlink/trap.c
+++ b/net/devlink/trap.c
@@ -302,7 +302,7 @@ static int devlink_nl_trap_fill(struct sk_buff *msg, struct devlink *devlink,
 int devlink_nl_trap_get_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct netlink_ext_ack *extack = info->extack;
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_trap_item *trap_item;
 	struct sk_buff *msg;
 	int err;
@@ -412,7 +412,7 @@ static int devlink_trap_action_set(struct devlink *devlink,
 int devlink_nl_trap_set_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct netlink_ext_ack *extack = info->extack;
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_trap_item *trap_item;
 
 	if (list_empty(&devlink->trap_list))
@@ -511,7 +511,7 @@ devlink_nl_trap_group_fill(struct sk_buff *msg, struct devlink *devlink,
 int devlink_nl_trap_group_get_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct netlink_ext_ack *extack = info->extack;
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_trap_group_item *group_item;
 	struct sk_buff *msg;
 	int err;
@@ -682,7 +682,7 @@ static int devlink_trap_group_set(struct devlink *devlink,
 int devlink_nl_trap_group_set_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct netlink_ext_ack *extack = info->extack;
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct devlink_trap_group_item *group_item;
 	bool modified = false;
 	int err;
@@ -804,7 +804,7 @@ int devlink_nl_trap_policer_get_doit(struct sk_buff *skb,
 {
 	struct devlink_trap_policer_item *policer_item;
 	struct netlink_ext_ack *extack = info->extack;
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 	struct sk_buff *msg;
 	int err;
 
@@ -924,7 +924,7 @@ int devlink_nl_trap_policer_set_doit(struct sk_buff *skb,
 {
 	struct devlink_trap_policer_item *policer_item;
 	struct netlink_ext_ack *extack = info->extack;
-	struct devlink *devlink = info->user_ptr[0];
+	struct devlink *devlink = devlink_nl_ctx(info)->devlink;
 
 	if (list_empty(&devlink->trap_policer_list))
 		return -EOPNOTSUPP;
diff --git a/net/dsa/dsa.c b/net/dsa/dsa.c
index 9cb732f6b1e3e..da53a666d4b82 100644
--- a/net/dsa/dsa.c
+++ b/net/dsa/dsa.c
@@ -18,6 +18,7 @@
 #include <linux/of.h>
 #include <linux/of_net.h>
 #include <net/dsa_stubs.h>
+#include <net/netdev_lock.h>
 #include <net/sch_generic.h>
 
 #include "conduit.h"
@@ -1620,10 +1621,23 @@ void dsa_switch_shutdown(struct dsa_switch *ds)
 
 	rtnl_lock();
 
-	dsa_switch_for_each_cpu_port(dp, ds)
-		list_add(&dp->conduit->close_list, &close_list);
+	dsa_switch_for_each_cpu_port(dp, ds) {
+		if (!(dp->conduit->flags & IFF_UP))
+			continue;
+		list_add_tail(&dp->conduit->close_list, &close_list);
+		netdev_lock_ops(dp->conduit);
+	}
+
+	netif_close_many(&close_list, false);
 
-	netif_close_many(&close_list, true);
+	while (!list_empty(&close_list)) {
+		struct net_device *conduit;
+
+		conduit = list_first_entry(&close_list, struct net_device,
+					   close_list);
+		netdev_unlock_ops(conduit);
+		list_del_init(&conduit->close_list);
+	}
 
 	dsa_switch_for_each_user_port(dp, ds) {
 		conduit = dsa_port_to_conduit(dp);
diff --git a/net/dsa/user.c b/net/dsa/user.c
index 072fa76972cc6..03c7af6abe18a 100644
--- a/net/dsa/user.c
+++ b/net/dsa/user.c
@@ -13,6 +13,7 @@
 #include <linux/of_net.h>
 #include <linux/of_mdio.h>
 #include <linux/mdio.h>
+#include <net/netdev_lock.h>
 #include <net/rtnetlink.h>
 #include <net/pkt_cls.h>
 #include <net/selftests.h>
@@ -3599,10 +3600,24 @@ static int dsa_user_netdevice_event(struct notifier_block *nb,
 			if (dp->cpu_dp != cpu_dp)
 				continue;
 
-			list_add(&dp->user->close_list, &close_list);
+			if (!(dp->user->flags & IFF_UP))
+				continue;
+
+			list_add_tail(&dp->user->close_list, &close_list);
+			netdev_lock_ops(dp->user);
 		}
 
-		netif_close_many(&close_list, true);
+		netif_close_many(&close_list, false);
+
+		while (!list_empty(&close_list)) {
+			struct net_device *user_dev;
+
+			user_dev = list_first_entry(&close_list,
+						    struct net_device,
+						    close_list);
+			netdev_unlock_ops(user_dev);
+			list_del_init(&user_dev->close_list);
+		}
 
 		return NOTIFY_OK;
 	}
diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c
index dfc81ee969ae0..fa1710e27e505 100644
--- a/net/ipv4/esp4.c
+++ b/net/ipv4/esp4.c
@@ -104,6 +104,13 @@ static void esp_ssg_unref(struct xfrm_state *x, void *tmp, struct sk_buff *skb,
 	struct aead_request *req;
 	struct scatterlist *sg;
 
+	/* Managed frags are owned by the zerocopy ubuf; the skb holds no
+	 * per-frag page reference, so we must not drop one here.  Mirrors
+	 * the SKBFL_MANAGED_FRAG_REFS handling in skb_release_data().
+	 */
+	if (skb_zcopy_managed(skb))
+		return;
+
 	if (x->props.flags & XFRM_STATE_ESN)
 		extralen += sizeof(struct esp_output_extra);
 
diff --git a/net/ipv4/fib_frontend.c b/net/ipv4/fib_frontend.c
index 42212970d7354..8a3dc04e8cac6 100644
--- a/net/ipv4/fib_frontend.c
+++ b/net/ipv4/fib_frontend.c
@@ -76,7 +76,7 @@ static int __net_init fib4_rules_init(struct net *net)
 
 struct fib_table *fib_new_table(struct net *net, u32 id)
 {
-	struct fib_table *tb, *alias = NULL;
+	struct fib_table *tb, *new_tb, *alias = NULL;
 	unsigned int h;
 
 	if (id == 0)
@@ -85,14 +85,27 @@ struct fib_table *fib_new_table(struct net *net, u32 id)
 	if (tb)
 		return tb;
 
+	if (!check_net(net))
+		return NULL;
+
 	if (id == RT_TABLE_LOCAL && !net->ipv4.fib_has_custom_rules)
 		alias = fib_new_table(net, RT_TABLE_MAIN);
 
-	if (check_net(net))
-		tb = fib_trie_table(id, alias);
-	if (!tb)
+	new_tb = fib_trie_table(id, alias);
+	if (!new_tb)
 		return NULL;
 
+	spin_lock(&net->ipv4.fib_table_hash_lock);
+
+	tb = fib_get_table(net, id);
+	if (tb) {
+		spin_unlock(&net->ipv4.fib_table_hash_lock);
+		fib_free_table(new_tb);
+		return tb;
+	}
+
+	tb = new_tb;
+
 	switch (id) {
 	case RT_TABLE_MAIN:
 		rcu_assign_pointer(net->ipv4.fib_main, tb);
@@ -106,28 +119,35 @@ struct fib_table *fib_new_table(struct net *net, u32 id)
 
 	h = id & (FIB_TABLE_HASHSZ - 1);
 	hlist_add_head_rcu(&tb->tb_hlist, &net->ipv4.fib_table_hash[h]);
+
+	spin_unlock(&net->ipv4.fib_table_hash_lock);
+
 	return tb;
 }
 EXPORT_SYMBOL_GPL(fib_new_table);
 
-/* caller must hold either rtnl or rcu read lock */
 struct fib_table *fib_get_table(struct net *net, u32 id)
 {
-	struct fib_table *tb;
+	struct fib_table *tb = NULL;
 	struct hlist_head *head;
 	unsigned int h;
 
 	if (id == 0)
 		id = RT_TABLE_MAIN;
 	h = id & (FIB_TABLE_HASHSZ - 1);
-
 	head = &net->ipv4.fib_table_hash[h];
-	hlist_for_each_entry_rcu(tb, head, tb_hlist,
-				 lockdep_rtnl_is_held()) {
+
+	/* fib_table is not destroyed until ip_fib_net_exit()
+	 * except for the merged main/local table.
+	 * fib_unmerge() is called under RTNL, so other readers
+	 * under RTNL (e.g. fib_flush(), fib_info_notify_update())
+	 * can safely traverse the list with rcu_dereference_raw().
+	 */
+	hlist_for_each_entry_rcu(tb, head, tb_hlist, true)
 		if (tb->tb_id == id)
-			return tb;
-	}
-	return NULL;
+			break;
+
+	return tb;
 }
 #endif /* CONFIG_IP_MULTIPLE_TABLES */
 
@@ -190,10 +210,9 @@ void fib_flush(struct net *net)
 
 	for (h = 0; h < FIB_TABLE_HASHSZ; h++) {
 		struct hlist_head *head = &net->ipv4.fib_table_hash[h];
-		struct hlist_node *tmp;
 		struct fib_table *tb;
 
-		hlist_for_each_entry_safe(tb, tmp, head, tb_hlist)
+		hlist_for_each_entry_rcu(tb, head, tb_hlist, true)
 			flushed += fib_table_flush(net, tb, false);
 	}
 
@@ -1019,10 +1038,11 @@ static int inet_dump_fib(struct sk_buff *skb, struct netlink_callback *cb)
 		.dump_routes = true,
 		.dump_exceptions = true,
 	};
-	unsigned int e = 0, s_e, h, s_h;
 	struct hlist_head *head;
 	int dumped = 0, err = 0;
 	struct fib_table *tb;
+	unsigned int h, s_h;
+	u32 s_id;
 
 	rcu_read_lock();
 	if (cb->strict_check) {
@@ -1054,29 +1074,28 @@ static int inet_dump_fib(struct sk_buff *skb, struct netlink_callback *cb)
 	}
 
 	s_h = cb->args[0];
-	s_e = cb->args[1];
+	s_id = cb->args[1];
 
 	err = 0;
-	for (h = s_h; h < FIB_TABLE_HASHSZ; h++, s_e = 0) {
-		e = 0;
+	for (h = s_h; h < FIB_TABLE_HASHSZ; h++, s_id = 0) {
 		head = &net->ipv4.fib_table_hash[h];
 		hlist_for_each_entry_rcu(tb, head, tb_hlist) {
-			if (e < s_e)
-				goto next;
+			if (s_id && tb->tb_id != s_id)
+				continue;
+
+			s_id = 0;
 			if (dumped)
 				memset(&cb->args[2], 0, sizeof(cb->args) -
 						 2 * sizeof(cb->args[0]));
+			cb->args[1] = tb->tb_id;
 			err = fib_table_dump(tb, skb, cb, &filter);
 			if (err < 0)
 				goto out;
 			dumped = 1;
-next:
-			e++;
 		}
 	}
 out:
 
-	cb->args[1] = e;
 	cb->args[0] = h;
 
 unlock:
@@ -1566,6 +1585,10 @@ static int __net_init ip_fib_net_init(struct net *net)
 		FIB_MULTIPATH_HASH_FIELD_DEFAULT_MASK;
 #endif
 
+#ifdef CONFIG_IP_MULTIPLE_TABLES
+	spin_lock_init(&net->ipv4.fib_table_hash_lock);
+#endif
+
 	/* Avoid false sharing : Use at least a full cache line */
 	size = max_t(size_t, size, L1_CACHE_BYTES);
 
diff --git a/net/ipv4/fib_rules.c b/net/ipv4/fib_rules.c
index e068a5bace73f..4edb0dca7be8a 100644
--- a/net/ipv4/fib_rules.c
+++ b/net/ipv4/fib_rules.c
@@ -301,10 +301,12 @@ static int fib4_rule_configure(struct fib_rule *rule, struct sk_buff *skb,
 	    fib4_nl2rule_dscp_mask(tb[FRA_DSCP_MASK], rule4, extack) < 0)
 		goto errout;
 
-	/* split local/main if they are not already split */
-	err = fib_unmerge(net);
-	if (err)
-		goto errout;
+	if (!net->ipv4.fib_has_custom_rules) {
+		/* split local/main if they are not already split */
+		err = fib_unmerge(net);
+		if (err)
+			goto errout;
+	}
 
 	if (rule->table == RT_TABLE_UNSPEC && !rule->l3mdev) {
 		if (rule->action == FR_ACT_TO_TBL) {
@@ -349,7 +351,7 @@ static int fib4_rule_configure(struct fib_rule *rule, struct sk_buff *skb,
 	return err;
 }
 
-static int fib4_rule_delete(struct fib_rule *rule)
+static void fib4_rule_delete(struct fib_rule *rule)
 {
 	struct net *net = rule->fr_net;
 
@@ -361,8 +363,6 @@ static int fib4_rule_delete(struct fib_rule *rule)
 	if (net->ipv4.fib_rules_require_fldissect &&
 	    fib_rule_requires_fldissect(rule))
 		net->ipv4.fib_rules_require_fldissect--;
-
-	return 0;
 }
 
 static int fib4_rule_compare(struct fib_rule *rule, struct fib_rule_hdr *frh,
@@ -460,6 +460,11 @@ static void fib4_rule_flush_cache(struct fib_rules_ops *ops)
 	rt_cache_flush(ops->fro_net);
 }
 
+static bool fib4_rule_need_rtnl(struct net *net)
+{
+	return !net->ipv4.fib_has_custom_rules;
+}
+
 static const struct fib_rules_ops __net_initconst fib4_rules_ops_template = {
 	.family		= AF_INET,
 	.rule_size	= sizeof(struct fib4_rule),
@@ -473,6 +478,7 @@ static const struct fib_rules_ops __net_initconst fib4_rules_ops_template = {
 	.fill		= fib4_rule_fill,
 	.nlmsg_payload	= fib4_rule_nlmsg_payload,
 	.flush_cache	= fib4_rule_flush_cache,
+	.need_rtnl	= fib4_rule_need_rtnl,
 	.nlgroup	= RTNLGRP_IPV4_RULE,
 	.owner		= THIS_MODULE,
 };
diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c
index e11dc86ceda0b..acb1e43859143 100644
--- a/net/ipv4/fib_trie.c
+++ b/net/ipv4/fib_trie.c
@@ -1385,7 +1385,7 @@ int fib_table_insert(struct net *net, struct fib_table *tb,
 out_remove_new_fa:
 	fib_remove_alias(t, tp, l, new_fa);
 out_free_new_fa:
-	kmem_cache_free(fn_alias_kmem, new_fa);
+	alias_free_mem_rcu(new_fa);
 out:
 	fib_release_info(fi);
 err:
@@ -2137,8 +2137,7 @@ void fib_info_notify_update(struct net *net, struct nl_info *info)
 		struct hlist_head *head = &net->ipv4.fib_table_hash[h];
 		struct fib_table *tb;
 
-		hlist_for_each_entry_rcu(tb, head, tb_hlist,
-					 lockdep_rtnl_is_held())
+		hlist_for_each_entry_rcu(tb, head, tb_hlist, true)
 			__fib_info_notify_update(net, tb, info);
 	}
 }
diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c
index bb2d4441a4929..b80b8a92f46e5 100644
--- a/net/ipv4/igmp.c
+++ b/net/ipv4/igmp.c
@@ -1487,6 +1487,7 @@ int inet_fill_ifmcaddr(struct sk_buff *skb, struct net_device *dev,
 	ci.ifa_valid = INFINITY_LIFE_TIME;
 
 	if (nla_put_in_addr(skb, IFA_MULTICAST, im->multiaddr) < 0 ||
+	    nla_put_u32(skb, IFA_MC_USERS, READ_ONCE(im->users)) < 0 ||
 	    nla_put(skb, IFA_CACHEINFO, sizeof(ci), &ci) < 0) {
 		nlmsg_cancel(skb, nlh);
 		return -EMSGSIZE;
@@ -1508,6 +1509,7 @@ static void inet_ifmcaddr_notify(struct net_device *dev,
 
 	skb = nlmsg_new(NLMSG_ALIGN(sizeof(struct ifaddrmsg)) +
 			nla_total_size(sizeof(__be32)) +
+			nla_total_size(sizeof(u32)) +
 			nla_total_size(sizeof(struct ifa_cacheinfo)),
 			GFP_KERNEL);
 	if (!skb)
diff --git a/net/ipv4/netfilter/nf_nat_snmp_basic_main.c b/net/ipv4/netfilter/nf_nat_snmp_basic_main.c
index 0ede138dfd29e..e540b86bd15b6 100644
--- a/net/ipv4/netfilter/nf_nat_snmp_basic_main.c
+++ b/net/ipv4/netfilter/nf_nat_snmp_basic_main.c
@@ -213,7 +213,7 @@ static int __init nf_nat_snmp_basic_init(void)
 	RCU_INIT_POINTER(nf_nat_snmp_hook, help);
 
 	nf_ct_helper_init(&snmp_trap_helper, AF_INET, IPPROTO_UDP,
-			  "snmp_trap", SNMP_TRAP_PORT, SNMP_TRAP_PORT, SNMP_TRAP_PORT,
+			  "snmp_trap",
 			  &snmp_exp_policy, 0, help, NULL, THIS_MODULE);
 
 	err = nf_conntrack_helper_register(&snmp_trap_helper, &snmp_trap_helper_ptr);
diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c
index e9fbab6ad9146..2aebaf8297e04 100644
--- a/net/ipv4/raw.c
+++ b/net/ipv4/raw.c
@@ -809,23 +809,18 @@ static int raw_seticmpfilter(struct sock *sk, sockptr_t optval, int optlen)
 	return 0;
 }
 
-static int raw_geticmpfilter(struct sock *sk, char __user *optval, int __user *optlen)
+static int raw_geticmpfilter(struct sock *sk, sockopt_t *opt)
 {
-	int len, ret = -EFAULT;
+	int len = opt->optlen;
 
-	if (get_user(len, optlen))
-		goto out;
-	ret = -EINVAL;
 	if (len < 0)
-		goto out;
+		return -EINVAL;
 	if (len > sizeof(struct icmp_filter))
 		len = sizeof(struct icmp_filter);
-	ret = -EFAULT;
-	if (put_user(len, optlen) ||
-	    copy_to_user(optval, &raw_sk(sk)->filter, len))
-		goto out;
-	ret = 0;
-out:	return ret;
+	opt->optlen = len;
+	if (copy_to_iter(&raw_sk(sk)->filter, len, &opt->iter_out) != len)
+		return -EFAULT;
+	return 0;
 }
 
 static int do_raw_setsockopt(struct sock *sk, int optname,
@@ -848,14 +843,13 @@ static int raw_setsockopt(struct sock *sk, int level, int optname,
 	return do_raw_setsockopt(sk, optname, optval, optlen);
 }
 
-static int do_raw_getsockopt(struct sock *sk, int optname,
-			     char __user *optval, int __user *optlen)
+static int do_raw_getsockopt(struct sock *sk, int optname, sockopt_t *opt)
 {
 	if (optname == ICMP_FILTER) {
 		if (inet_sk(sk)->inet_num != IPPROTO_ICMP)
 			return -EOPNOTSUPP;
 		else
-			return raw_geticmpfilter(sk, optval, optlen);
+			return raw_geticmpfilter(sk, opt);
 	}
 	return -ENOPROTOOPT;
 }
@@ -863,9 +857,24 @@ static int do_raw_getsockopt(struct sock *sk, int optname,
 static int raw_getsockopt(struct sock *sk, int level, int optname,
 			  char __user *optval, int __user *optlen)
 {
+	sockopt_t opt;
+	int err;
+
 	if (level != SOL_RAW)
 		return ip_getsockopt(sk, level, optname, optval, optlen);
-	return do_raw_getsockopt(sk, optname, optval, optlen);
+
+	err = sockopt_init_user(&opt, optval, optlen);
+	if (err)
+		return err;
+
+	err = do_raw_getsockopt(sk, optname, &opt);
+	if (err)
+		return err;
+
+	if (put_user(opt.optlen, optlen))
+		return -EFAULT;
+
+	return 0;
 }
 
 static int raw_ioctl(struct sock *sk, int cmd, int *karg)
diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c
index cc0bd73f36b6d..8e905b50deadb 100644
--- a/net/ipv4/tcp_bpf.c
+++ b/net/ipv4/tcp_bpf.c
@@ -334,6 +334,7 @@ static int tcp_bpf_recvmsg_parser(struct sock *sk,
 
 static int tcp_bpf_ioctl(struct sock *sk, int cmd, int *karg)
 {
+	struct sk_psock *psock;
 	bool slow;
 
 	if (cmd != SIOCINQ)
@@ -344,7 +345,21 @@ static int tcp_bpf_ioctl(struct sock *sk, int cmd, int *karg)
 		return -EINVAL;
 
 	slow = lock_sock_fast(sk);
-	*karg = sk_psock_msg_inq(sk);
+	psock = sk_psock_get(sk);
+	if (unlikely(!psock)) {
+		unlock_sock_fast(sk, slow);
+		return tcp_ioctl(sk, cmd, karg);
+	}
+	*karg = sk_psock_get_msg_len_nolock(psock);
+	/* Without a verdict program, ingress data is never diverted to
+	 * ingress_msg: it stays in sk_receive_queue and is read through
+	 * the fallback to tcp_recvmsg(), so account for it like
+	 * tcp_ioctl() does.
+	 */
+	if (!READ_ONCE(psock->progs.stream_verdict) &&
+	    !READ_ONCE(psock->progs.skb_verdict))
+		*karg += tcp_inq(sk);
+	sk_psock_put(sk, psock);
 	unlock_sock_fast(sk, slow);
 
 	return 0;
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 209ef7522508f..4a46da375043b 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -2318,8 +2318,10 @@ int tcp_v4_rcv(struct sk_buff *skb)
 		}
 
 		drop_reason = psp_twsk_rx_policy_check(inet_twsk(sk), skb);
-		if (drop_reason)
-			break;
+		if (drop_reason) {
+			inet_twsk_put(inet_twsk(sk));
+			goto discard_it;
+		}
 	}
 		/* to ACK */
 		fallthrough;
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 70f6cbd4ef73b..59248a59358ca 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -76,6 +76,7 @@
 
 #include <linux/bpf-cgroup.h>
 #include <linux/uaccess.h>
+#include <linux/uio.h>
 #include <asm/ioctls.h>
 #include <linux/memblock.h>
 #include <linux/highmem.h>
@@ -2995,14 +2996,13 @@ static int udp_setsockopt(struct sock *sk, int level, int optname, sockptr_t opt
 }
 
 int udp_lib_getsockopt(struct sock *sk, int level, int optname,
-		       char __user *optval, int __user *optlen)
+		       sockopt_t *opt)
 {
 	struct udp_sock *up = udp_sk(sk);
 	int val, len;
 
-	if (get_user(len, optlen))
-		return -EFAULT;
-
+	len = opt->optlen;
+	/* keep the check so direct sockopt_t callers stay covered. */
 	if (len < 0)
 		return -EINVAL;
 
@@ -3037,9 +3037,8 @@ int udp_lib_getsockopt(struct sock *sk, int level, int optname,
 		return -ENOPROTOOPT;
 	}
 
-	if (put_user(len, optlen))
-		return -EFAULT;
-	if (copy_to_user(optval, &val, len))
+	opt->optlen = len;
+	if (copy_to_iter(&val, len, &opt->iter_out) != len)
 		return -EFAULT;
 	return 0;
 }
@@ -3047,9 +3046,29 @@ int udp_lib_getsockopt(struct sock *sk, int level, int optname,
 static int udp_getsockopt(struct sock *sk, int level, int optname,
 			  char __user *optval, int __user *optlen)
 {
-	if (level == SOL_UDP)
-		return udp_lib_getsockopt(sk, level, optname, optval, optlen);
-	return ip_getsockopt(sk, level, optname, optval, optlen);
+	sockopt_t opt;
+	int err;
+
+	/*
+	 * keep the old __user pointers, until ip_getsockopt() moves
+	 * to sockopt_t
+	 */
+	if (level != SOL_UDP)
+		return ip_getsockopt(sk, level, optname, optval, optlen);
+
+	err = sockopt_init_user(&opt, optval, optlen);
+	if (err)
+		return err;
+
+	err = udp_lib_getsockopt(sk, level, optname, &opt);
+	if (err)
+		return err;
+
+	/* optval was written by copy_to_iter() in udp_lib_getsockopt() */
+	if (put_user(opt.optlen, optlen))
+		return -EFAULT;
+
+	return 0;
 }
 
 /**
diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c
index cbe681de38188..f1fe9ede1edb3 100644
--- a/net/ipv6/addrconf.c
+++ b/net/ipv6/addrconf.c
@@ -5264,6 +5264,7 @@ int inet6_fill_ifmcaddr(struct sk_buff *skb,
 
 	put_ifaddrmsg(nlh, 128, IFA_F_PERMANENT, scope, ifindex);
 	if (nla_put_in6_addr(skb, IFA_MULTICAST, &ifmca->mca_addr) < 0 ||
+	    nla_put_u32(skb, IFA_MC_USERS, READ_ONCE(ifmca->mca_users)) < 0 ||
 	    put_cacheinfo(skb, ifmca->mca_cstamp, READ_ONCE(ifmca->mca_tstamp),
 			  INFINITY_LIFE_TIME, INFINITY_LIFE_TIME) < 0) {
 		nlmsg_cancel(skb, nlh);
diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c
index 296b57926abb9..7d216b9c59f04 100644
--- a/net/ipv6/esp6.c
+++ b/net/ipv6/esp6.c
@@ -121,6 +121,13 @@ static void esp_ssg_unref(struct xfrm_state *x, void *tmp, struct sk_buff *skb,
 	struct aead_request *req;
 	struct scatterlist *sg;
 
+	/* Managed frags are owned by the zerocopy ubuf; the skb holds no
+	 * per-frag page reference, so we must not drop one here.  Mirrors
+	 * the SKBFL_MANAGED_FRAG_REFS handling in skb_release_data().
+	 */
+	if (skb_zcopy_managed(skb))
+		return;
+
 	if (x->props.flags & XFRM_STATE_ESN)
 		extralen += sizeof(struct esp_output_extra);
 
diff --git a/net/ipv6/fib6_rules.c b/net/ipv6/fib6_rules.c
index e1b2b4fa6e189..04dab9329d0cf 100644
--- a/net/ipv6/fib6_rules.c
+++ b/net/ipv6/fib6_rules.c
@@ -480,15 +480,13 @@ static int fib6_rule_configure(struct fib_rule *rule, struct sk_buff *skb,
 	return err;
 }
 
-static int fib6_rule_delete(struct fib_rule *rule)
+static void fib6_rule_delete(struct fib_rule *rule)
 {
 	struct net *net = rule->fr_net;
 
 	if (net->ipv6.fib6_rules_require_fldissect &&
 	    fib_rule_requires_fldissect(rule))
 		net->ipv6.fib6_rules_require_fldissect--;
-
-	return 0;
 }
 
 static int fib6_rule_compare(struct fib_rule *rule, struct fib_rule_hdr *frh,
@@ -637,21 +635,14 @@ static int __net_init fib6_rules_net_init(struct net *net)
 	goto out;
 }
 
-static void __net_exit fib6_rules_net_exit_batch(struct list_head *net_list)
+static void __net_exit fib6_rules_net_exit(struct net *net)
 {
-	struct net *net;
-
-	rtnl_lock();
-	list_for_each_entry(net, net_list, exit_list) {
-		fib_rules_unregister(net->ipv6.fib6_rules_ops);
-		cond_resched();
-	}
-	rtnl_unlock();
+	fib_rules_unregister(net->ipv6.fib6_rules_ops);
 }
 
 static struct pernet_operations fib6_rules_net_ops = {
 	.init = fib6_rules_net_init,
-	.exit_batch = fib6_rules_net_exit_batch,
+	.exit = fib6_rules_net_exit,
 };
 
 int __init fib6_rules_init(void)
diff --git a/net/ipv6/mcast.c b/net/ipv6/mcast.c
index 4d2b9377ba2de..aaba4c2aae23e 100644
--- a/net/ipv6/mcast.c
+++ b/net/ipv6/mcast.c
@@ -908,6 +908,7 @@ static void inet6_ifmcaddr_notify(struct net_device *dev,
 
 	skb = nlmsg_new(NLMSG_ALIGN(sizeof(struct ifaddrmsg)) +
 			nla_total_size(sizeof(struct in6_addr)) +
+			nla_total_size(sizeof(u32)) +
 			nla_total_size(sizeof(struct ifa_cacheinfo)),
 			GFP_KERNEL);
 	if (!skb)
diff --git a/net/ipv6/netfilter.c b/net/ipv6/netfilter.c
index 6d80f85e55fa7..a7025ec870359 100644
--- a/net/ipv6/netfilter.c
+++ b/net/ipv6/netfilter.c
@@ -120,7 +120,7 @@ int br_ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
 	ktime_t tstamp = skb->tstamp;
 	struct ip6_frag_state state;
 	u8 *prevhdr, nexthdr = 0;
-	unsigned int mtu, hlen;
+	unsigned int mtu, hlen, nexthdr_offset;
 	int hroom, err = 0;
 	__be32 frag_id;
 
@@ -129,6 +129,7 @@ int br_ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
 		goto blackhole;
 	hlen = err;
 	nexthdr = *prevhdr;
+	nexthdr_offset = prevhdr - skb_network_header(skb);
 
 	mtu = skb->dev->mtu;
 	if (frag_max_size > mtu ||
@@ -147,6 +148,7 @@ int br_ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
 	    (err = skb_checksum_help(skb)))
 		goto blackhole;
 
+	prevhdr = skb_network_header(skb) + nexthdr_offset;
 	hroom = LL_RESERVED_SPACE(skb->dev);
 	if (skb_has_frag_list(skb)) {
 		unsigned int first_len = skb_pagelen(skb);
diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index ebe161d72fbd0..522ba45ce9b75 100644
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -1977,8 +1977,10 @@ INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
 		}
 
 		drop_reason = psp_twsk_rx_policy_check(inet_twsk(sk), skb);
-		if (drop_reason)
-			break;
+		if (drop_reason) {
+			inet_twsk_put(inet_twsk(sk));
+			goto discard_it;
+		}
 	}
 		/* to ACK */
 		fallthrough;
diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 15e032194eccc..392e18b970454 100644
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -1826,9 +1826,22 @@ static int udpv6_setsockopt(struct sock *sk, int level, int optname,
 static int udpv6_getsockopt(struct sock *sk, int level, int optname,
 			    char __user *optval, int __user *optlen)
 {
-	if (level == SOL_UDP)
-		return udp_lib_getsockopt(sk, level, optname, optval, optlen);
-	return ipv6_getsockopt(sk, level, optname, optval, optlen);
+	sockopt_t opt;
+	int err;
+
+	if (level != SOL_UDP)
+		return ipv6_getsockopt(sk, level, optname, optval, optlen);
+
+	err = sockopt_init_user(&opt, optval, optlen);
+	if (err)
+		return err;
+
+	err = udp_lib_getsockopt(sk, level, optname, &opt);
+	if (err)
+		return err;
+	if (put_user(opt.optlen, optlen))
+		return -EFAULT;
+	return 0;
 }
 
 
diff --git a/net/ipv6/xfrm6_input.c b/net/ipv6/xfrm6_input.c
index 89d0443b53073..07edef2589844 100644
--- a/net/ipv6/xfrm6_input.c
+++ b/net/ipv6/xfrm6_input.c
@@ -247,7 +247,7 @@ int xfrm6_input_addr(struct sk_buff *skb, xfrm_address_t *daddr,
 		goto drop;
 	}
 
-	if (1 + sp->len == XFRM_MAX_DEPTH) {
+	if (sp->len >= XFRM_MAX_DEPTH) {
 		XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
 		goto drop;
 	}
diff --git a/net/ipv6/xfrm6_policy.c b/net/ipv6/xfrm6_policy.c
index 125ea9a5b8a08..3b749475f6ed6 100644
--- a/net/ipv6/xfrm6_policy.c
+++ b/net/ipv6/xfrm6_policy.c
@@ -88,6 +88,7 @@ static int xfrm6_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
 	xdst->u.rt6.rt6i_idev = in6_dev_get(dev);
 	if (!xdst->u.rt6.rt6i_idev) {
 		netdev_put(dev, &xdst->u.dst.dev_tracker);
+		xdst->u.dst.dev = NULL;
 		return -ENODEV;
 	}
 
diff --git a/net/iucv/af_iucv.c b/net/iucv/af_iucv.c
index fed240b453bd9..b85fb9767dec2 100644
--- a/net/iucv/af_iucv.c
+++ b/net/iucv/af_iucv.c
@@ -2089,6 +2089,8 @@ static int afiucv_hs_rcv(struct sk_buff *skb, struct net_device *dev,
 			}
 		}
 	}
+	if (sk)
+		sock_hold(sk);
 	read_unlock(&iucv_sk_list.lock);
 	if (!iucv)
 		sk = NULL;
@@ -2138,6 +2140,8 @@ static int afiucv_hs_rcv(struct sk_buff *skb, struct net_device *dev,
 		kfree_skb(skb);
 	}
 
+	if (sk)
+		sock_put(sk);
 	return err;
 }
 
diff --git a/net/llc/llc_conn.c b/net/llc/llc_conn.c
index e8f427375c689..260460d50f54c 100644
--- a/net/llc/llc_conn.c
+++ b/net/llc/llc_conn.c
@@ -767,7 +767,6 @@ static struct sock *llc_create_incoming_sock(struct sock *sk,
 	newllc->dev = dev;
 	dev_hold(dev);
 	llc_sap_add_socket(llc->sap, newsk);
-	llc_sap_hold(llc->sap);
 out:
 	return newsk;
 }
diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c
index 3b58af59f7e42..11943f14936ff 100644
--- a/net/mac80211/cfg.c
+++ b/net/mac80211/cfg.c
@@ -1146,9 +1146,6 @@ static int ieee80211_set_fils_discovery(struct ieee80211_sub_if_data *sdata,
 	fd->max_interval = params->max_interval;
 
 	old = sdata_dereference(link->u.ap.fils_discovery, sdata);
-	if (old)
-		kfree_rcu(old, rcu_head);
-
 	if (params->tmpl && params->tmpl_len) {
 		new = kzalloc(sizeof(*new) + params->tmpl_len, GFP_KERNEL);
 		if (!new)
@@ -1160,6 +1157,9 @@ static int ieee80211_set_fils_discovery(struct ieee80211_sub_if_data *sdata,
 		RCU_INIT_POINTER(link->u.ap.fils_discovery, NULL);
 	}
 
+	if (old)
+		kfree_rcu(old, rcu_head);
+
 	*changed |= BSS_CHANGED_FILS_DISCOVERY;
 	return 0;
 }
@@ -1179,8 +1179,6 @@ ieee80211_set_unsol_bcast_probe_resp(struct ieee80211_sub_if_data *sdata,
 	link_conf->unsol_bcast_probe_resp_interval = params->interval;
 
 	old = sdata_dereference(link->u.ap.unsol_bcast_probe_resp, sdata);
-	if (old)
-		kfree_rcu(old, rcu_head);
 
 	if (params->tmpl && params->tmpl_len) {
 		new = kzalloc(sizeof(*new) + params->tmpl_len, GFP_KERNEL);
@@ -1193,6 +1191,9 @@ ieee80211_set_unsol_bcast_probe_resp(struct ieee80211_sub_if_data *sdata,
 		RCU_INIT_POINTER(link->u.ap.unsol_bcast_probe_resp, NULL);
 	}
 
+	if (old)
+		kfree_rcu(old, rcu_head);
+
 	*changed |= BSS_CHANGED_UNSOL_BCAST_PROBE_RESP;
 	return 0;
 }
@@ -4949,8 +4950,8 @@ static int ieee80211_set_rekey_data(struct wiphy *wiphy,
 	return 0;
 }
 
-static int ieee80211_probe_client(struct wiphy *wiphy, struct net_device *dev,
-				  const u8 *peer, u64 *cookie)
+static int ieee80211_probe_peer(struct wiphy *wiphy, struct net_device *dev,
+				const u8 *peer, u64 *cookie)
 {
 	struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
 	struct ieee80211_local *local = sdata->local;
@@ -6060,7 +6061,7 @@ const struct cfg80211_ops mac80211_config_ops = {
 	.tdls_mgmt = ieee80211_tdls_mgmt,
 	.tdls_channel_switch = ieee80211_tdls_channel_switch,
 	.tdls_cancel_channel_switch = ieee80211_tdls_cancel_channel_switch,
-	.probe_client = ieee80211_probe_client,
+	.probe_peer = ieee80211_probe_peer,
 	.set_noack_map = ieee80211_set_noack_map,
 #ifdef CONFIG_PM
 	.set_wakeup = ieee80211_set_wakeup,
diff --git a/net/mac80211/ibss.c b/net/mac80211/ibss.c
index d0fd6054f1820..882f91abbb662 100644
--- a/net/mac80211/ibss.c
+++ b/net/mac80211/ibss.c
@@ -668,7 +668,9 @@ static void ieee80211_ibss_disconnect(struct ieee80211_sub_if_data *sdata)
 
 	ifibss->state = IEEE80211_IBSS_MLME_SEARCH;
 
-	sta_info_flush(sdata, -1);
+	netif_carrier_off(sdata->dev);
+	if (!sta_info_flush(sdata, -1))
+		synchronize_net();
 
 	spin_lock_bh(&ifibss->incomplete_lock);
 	while (!list_empty(&ifibss->incomplete_stations)) {
@@ -682,8 +684,6 @@ static void ieee80211_ibss_disconnect(struct ieee80211_sub_if_data *sdata)
 	}
 	spin_unlock_bh(&ifibss->incomplete_lock);
 
-	netif_carrier_off(sdata->dev);
-
 	sdata->vif.cfg.ibss_joined = false;
 	sdata->vif.cfg.ibss_creator = false;
 	sdata->vif.bss_conf.enable_beacon = false;
@@ -710,7 +710,6 @@ static void ieee80211_csa_connection_drop_work(struct wiphy *wiphy,
 			     u.ibss.csa_connection_drop_work);
 
 	ieee80211_ibss_disconnect(sdata);
-	synchronize_rcu();
 	skb_queue_purge(&sdata->skb_queue);
 
 	/* trigger a scan to find another IBSS network to join */
@@ -1029,8 +1028,8 @@ static void ieee80211_update_sta_info(struct ieee80211_sub_if_data *sdata,
 		u32 changed = IEEE80211_RC_SUPP_RATES_CHANGED;
 		u8 rx_nss = sta->sta.deflink.rx_nss;
 
-		/* Force rx_nss recalculation */
-		sta->sta.deflink.rx_nss = 0;
+		ieee80211_sta_init_nss_bw_capa(&sta->deflink,
+					       &sdata->deflink.conf->chanreq.oper);
 		rate_control_rate_init(&sta->deflink);
 		if (sta->sta.deflink.rx_nss != rx_nss)
 			changed |= IEEE80211_RC_NSS_CHANGED;
@@ -1797,8 +1796,6 @@ int ieee80211_ibss_leave(struct ieee80211_sub_if_data *sdata)
 	memset(&ifibss->ht_capa, 0, sizeof(ifibss->ht_capa));
 	memset(&ifibss->ht_capa_mask, 0, sizeof(ifibss->ht_capa_mask));
 
-	synchronize_rcu();
-
 	skb_queue_purge(&sdata->skb_queue);
 
 	timer_delete_sync(&sdata->u.ibss.timer);
diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h
index 34a9ea8b6f857..d585820245ddb 100644
--- a/net/mac80211/ieee80211_i.h
+++ b/net/mac80211/ieee80211_i.h
@@ -2470,9 +2470,7 @@ void __ieee80211_tx_skb_tid_band(struct ieee80211_sub_if_data *sdata,
 static inline bool ieee80211_require_encrypted_assoc(__le16 fc,
 						     struct sta_info *sta)
 {
-	return (sta && sta->sta.epp_peer &&
-		(ieee80211_is_assoc_req(fc) || ieee80211_is_reassoc_req(fc) ||
-		 ieee80211_is_assoc_resp(fc) || ieee80211_is_reassoc_resp(fc)));
+	return sta && sta->sta.epp_peer && ieee80211_is_assoc(fc);
 }
 
 /* sta_out needs to be checked for ERR_PTR() before using */
diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c
index 086272c3ec08c..43460a705a6bd 100644
--- a/net/mac80211/iface.c
+++ b/net/mac80211/iface.c
@@ -588,6 +588,7 @@ static void ieee80211_do_stop(struct ieee80211_sub_if_data *sdata, bool going_do
 		WARN_ON(!list_empty(&sdata->u.ap.vlans));
 	} else if (sdata->vif.type == NL80211_IFTYPE_AP_VLAN) {
 		/* remove all packets in parent bc_buf pointing to this dev */
+		__skb_queue_head_init(&freeq);
 		ps = &sdata->bss->ps;
 
 		spin_lock_irqsave(&ps->bc_buf.lock, flags);
@@ -595,10 +596,15 @@ static void ieee80211_do_stop(struct ieee80211_sub_if_data *sdata, bool going_do
 			if (skb->dev == sdata->dev) {
 				__skb_unlink(skb, &ps->bc_buf);
 				local->total_ps_buffered--;
-				ieee80211_free_txskb(&local->hw, skb);
+				__skb_queue_tail(&freeq, skb);
 			}
 		}
 		spin_unlock_irqrestore(&ps->bc_buf.lock, flags);
+
+		skb_queue_walk_safe(&freeq, skb, tmp) {
+			__skb_unlink(skb, &freeq);
+			ieee80211_free_txskb(&local->hw, skb);
+		}
 	}
 
 	if (going_down)
diff --git a/net/mac80211/main.c b/net/mac80211/main.c
index 90d295cc364fb..eb1eaaf34612e 100644
--- a/net/mac80211/main.c
+++ b/net/mac80211/main.c
@@ -1602,7 +1602,7 @@ int ieee80211_register_hw(struct ieee80211_hw *hw)
 		sband = kmemdup(sband, sizeof(*sband), GFP_KERNEL);
 		if (!sband) {
 			result = -ENOMEM;
-			goto fail_rate;
+			goto fail_band;
 		}
 
 		wiphy_dbg(hw->wiphy, "copying sband (band %d) due to VHT EXT NSS BW flag\n",
@@ -1678,6 +1678,7 @@ int ieee80211_register_hw(struct ieee80211_hw *hw)
 #endif
 	wiphy_unregister(local->hw.wiphy);
  fail_wiphy_register:
+ fail_band:
 	rtnl_lock();
 	rate_control_deinitialize(local);
 	ieee80211_remove_interfaces(local);
diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c
index 9e92337bb6f9e..fa773f3b0541a 100644
--- a/net/mac80211/mlme.c
+++ b/net/mac80211/mlme.c
@@ -5641,13 +5641,15 @@ static void ieee80211_rx_mgmt_deauth(struct ieee80211_sub_if_data *sdata,
 				     struct ieee80211_mgmt *mgmt, size_t len)
 {
 	struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
-	u16 reason_code = le16_to_cpu(mgmt->u.deauth.reason_code);
+	u16 reason_code;
 
 	lockdep_assert_wiphy(sdata->local->hw.wiphy);
 
-	if (len < 24 + 2)
+	if (len < offsetofend(struct ieee80211_mgmt, u.deauth.reason_code))
 		return;
 
+	reason_code = le16_to_cpu(mgmt->u.deauth.reason_code);
+
 	if (!ether_addr_equal(mgmt->bssid, mgmt->sa)) {
 		ieee80211_tdls_handle_disconnect(sdata, mgmt->sa, reason_code);
 		return;
@@ -7138,7 +7140,7 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata,
 {
 	struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
 	struct ieee80211_mgd_assoc_data *assoc_data = ifmgd->assoc_data;
-	u16 capab_info, status_code, aid;
+	u16 capab_info, status_code, aid = 0;
 	struct ieee80211_elems_parse_params parse_params = {
 		.bss = NULL,
 		.link_id = -1,
@@ -7217,8 +7219,10 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata,
 
 	if (elems->aid_resp)
 		aid = le16_to_cpu(elems->aid_resp->aid);
-	else
+	else if (!assoc_data->s1g)
 		aid = le16_to_cpu(mgmt->u.assoc_resp.aid);
+	else if (status_code == WLAN_STATUS_SUCCESS)
+		goto abandon_assoc;
 
 	/*
 	 * The 5 MSB of the AID field are reserved for a non-S1G STA. For
diff --git a/net/mac80211/nan.c b/net/mac80211/nan.c
index 1800bb96dd294..19e08661be43a 100644
--- a/net/mac80211/nan.c
+++ b/net/mac80211/nan.c
@@ -253,9 +253,12 @@ int ieee80211_nan_set_local_sched(struct ieee80211_sub_if_data *sdata,
 {
 	struct ieee80211_nan_channel *sched_idx_to_chan[IEEE80211_NAN_MAX_CHANNELS] = {};
 	struct ieee80211_nan_sched_cfg *sched_cfg = &sdata->vif.cfg.nan_sched;
-	struct ieee80211_nan_sched_cfg backup_sched;
+	struct ieee80211_nan_sched_cfg *backup_sched __free(kfree) = kmalloc_obj(*backup_sched);
 	int ret;
 
+	if (!backup_sched)
+		return -ENOMEM;
+
 	if (sched->n_channels > IEEE80211_NAN_MAX_CHANNELS)
 		return -EOPNOTSUPP;
 
@@ -275,13 +278,13 @@ int ieee80211_nan_set_local_sched(struct ieee80211_sub_if_data *sdata,
 
 	bitmap_zero(sdata->u.nan.removed_channels, IEEE80211_NAN_MAX_CHANNELS);
 
-	memcpy(backup_sched.schedule, sched_cfg->schedule,
-	       sizeof(backup_sched.schedule));
-	memcpy(backup_sched.channels, sched_cfg->channels,
-	       sizeof(backup_sched.channels));
-	memcpy(backup_sched.avail_blob, sched_cfg->avail_blob,
-	       sizeof(backup_sched.avail_blob));
-	backup_sched.avail_blob_len = sched_cfg->avail_blob_len;
+	memcpy(backup_sched->schedule, sched_cfg->schedule,
+	       sizeof(backup_sched->schedule));
+	memcpy(backup_sched->channels, sched_cfg->channels,
+	       sizeof(backup_sched->channels));
+	memcpy(backup_sched->avail_blob, sched_cfg->avail_blob,
+	       sizeof(backup_sched->avail_blob));
+	backup_sched->avail_blob_len = sched_cfg->avail_blob_len;
 
 	memcpy(sched_cfg->avail_blob, sched->nan_avail_blob,
 	       sched->nan_avail_blob_len);
@@ -380,17 +383,17 @@ int ieee80211_nan_set_local_sched(struct ieee80211_sub_if_data *sdata,
 		if (!chan_def->chan)
 			continue;
 
-		if (!cfg80211_chandef_identical(&backup_sched.channels[i].chanreq.oper,
+		if (!cfg80211_chandef_identical(&backup_sched->channels[i].chanreq.oper,
 						chan_def))
 			ieee80211_nan_remove_channel(sdata,
 						     &sched_cfg->channels[i]);
 	}
 
 	/* Re-add all backed up channels */
-	for (int i = 0; i < ARRAY_SIZE(backup_sched.channels); i++) {
+	for (int i = 0; i < ARRAY_SIZE(backup_sched->channels); i++) {
 		struct ieee80211_nan_channel *chan = &sched_cfg->channels[i];
 
-		*chan = backup_sched.channels[i];
+		*chan = backup_sched->channels[i];
 
 		/*
 		 * For deferred update, no channels were removed and the channel
@@ -421,11 +424,11 @@ int ieee80211_nan_set_local_sched(struct ieee80211_sub_if_data *sdata,
 		}
 	}
 
-	memcpy(sched_cfg->schedule, backup_sched.schedule,
-	       sizeof(backup_sched.schedule));
-	memcpy(sched_cfg->avail_blob, backup_sched.avail_blob,
-	       sizeof(backup_sched.avail_blob));
-	sched_cfg->avail_blob_len = backup_sched.avail_blob_len;
+	memcpy(sched_cfg->schedule, backup_sched->schedule,
+	       sizeof(backup_sched->schedule));
+	memcpy(sched_cfg->avail_blob, backup_sched->avail_blob,
+	       sizeof(backup_sched->avail_blob));
+	sched_cfg->avail_blob_len = backup_sched->avail_blob_len;
 	sched_cfg->deferred = false;
 	bitmap_zero(sdata->u.nan.removed_channels, IEEE80211_NAN_MAX_CHANNELS);
 
diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c
index fb9a3574afe99..d9ea19be075dc 100644
--- a/net/mac80211/rx.c
+++ b/net/mac80211/rx.c
@@ -1526,6 +1526,9 @@ ieee80211_rx_h_check_dup(struct ieee80211_rx_data *rx)
 	if (status->flag & RX_FLAG_DUP_VALIDATED)
 		return RX_CONTINUE;
 
+	if (ieee80211_is_ext(hdr->frame_control))
+		return RX_CONTINUE;
+
 	/*
 	 * Drop duplicate 802.11 retransmissions
 	 * (IEEE 802.11-2012: 9.3.2.10 "Duplicate detection and recovery")
@@ -4510,12 +4513,16 @@ static bool ieee80211_accept_frame(struct ieee80211_rx_data *rx)
 	struct ieee80211_hdr *hdr = (void *)skb->data;
 	struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
 	u8 *bssid = ieee80211_get_bssid(hdr, skb->len, sdata->vif.type);
-	bool multicast = is_multicast_ether_addr(hdr->addr1) ||
-			 ieee80211_is_s1g_beacon(hdr->frame_control);
+	bool multicast;
 	static const u8 nan_network_id[ETH_ALEN] __aligned(2) = {
 		0x51, 0x6F, 0x9A, 0x01, 0x00, 0x00
 	};
 
+	if (ieee80211_is_s1g_beacon(hdr->frame_control))
+		return sdata->vif.type == NL80211_IFTYPE_STATION && bssid;
+
+	multicast = is_multicast_ether_addr(hdr->addr1);
+
 	switch (sdata->vif.type) {
 	case NL80211_IFTYPE_STATION:
 		if (!bssid && !sdata->u.mgd.use_4addr)
@@ -5212,6 +5219,11 @@ static bool ieee80211_prepare_and_rx_handle(struct ieee80211_rx_data *rx,
 		hdr = (struct ieee80211_hdr *)rx->skb->data;
 	}
 
+	if (ieee80211_is_s1g_beacon(hdr->frame_control)) {
+		ieee80211_invoke_rx_handlers(rx);
+		return true;
+	}
+
 	/* Store a copy of the pre-translated link addresses for SW crypto */
 	if (unlikely(is_unicast_ether_addr(hdr->addr1) &&
 		     !ieee80211_is_data(hdr->frame_control)))
@@ -5301,6 +5313,13 @@ static bool ieee80211_rx_for_interface(struct ieee80211_rx_data *rx,
 	struct sta_info *sta;
 	int link_id = -1;
 
+	if (ieee80211_is_s1g_beacon(hdr->frame_control)) {
+		if (!ieee80211_rx_data_set_sta(rx, NULL, -1))
+			return false;
+
+		return ieee80211_prepare_and_rx_handle(rx, skb, consume);
+	}
+
 	/*
 	 * Look up link station first, in case there's a
 	 * chance that they might have a link address that
@@ -5376,6 +5395,17 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw,
 			err = -ENOBUFS;
 		else
 			err = skb_linearize(skb);
+	} else if (ieee80211_is_s1g_beacon(fc)) {
+		size_t s1g_hdr_len = offsetof(struct ieee80211_ext,
+					      u.s1g_beacon.variable) +
+				     ieee80211_s1g_optional_len(fc);
+
+		if (skb->len < s1g_hdr_len)
+			err = -ENOBUFS;
+		else
+			err = skb_linearize(skb);
+	} else if (ieee80211_is_ext(fc)) {
+		err = -EINVAL;
 	} else {
 		err = !pskb_may_pull(skb, ieee80211_hdrlen(fc));
 	}
diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c
index 02b587ff85046..22eba0e6e54c5 100644
--- a/net/mac80211/sta_info.c
+++ b/net/mac80211/sta_info.c
@@ -355,6 +355,15 @@ static void sta_info_free_link(struct link_sta_info *link_sta)
 	free_percpu(link_sta->pcpu_rx_stats);
 }
 
+static void sta_link_free_rcu(struct rcu_head *head)
+{
+	struct sta_link_alloc *alloc =
+		container_of(head, struct sta_link_alloc, rcu_head);
+
+	sta_info_free_link(&alloc->info);
+	kfree(alloc);
+}
+
 static void sta_accumulate_removed_link_stats(struct sta_info *sta, int link_id)
 {
 	struct link_sta_info *link_sta = wiphy_dereference(sta->local->hw.wiphy,
@@ -439,10 +448,8 @@ static void sta_remove_link(struct sta_info *sta, unsigned int link_id,
 
 	RCU_INIT_POINTER(sta->link[link_id], NULL);
 	RCU_INIT_POINTER(sta->sta.link[link_id], NULL);
-	if (alloc) {
-		sta_info_free_link(&alloc->info);
-		kfree_rcu(alloc, rcu_head);
-	}
+	if (alloc)
+		call_rcu(&alloc->rcu_head, sta_link_free_rcu);
 
 	ieee80211_sta_recalc_aggregates(&sta->sta);
 }
diff --git a/net/mac80211/status.c b/net/mac80211/status.c
index dd1dbba06838e..c3d29aed93fe8 100644
--- a/net/mac80211/status.c
+++ b/net/mac80211/status.c
@@ -655,7 +655,7 @@ static void ieee80211_report_ack_skb(struct ieee80211_local *local,
 								GFP_ATOMIC);
 			else if (ieee80211_is_any_nullfunc(hdr->frame_control))
 				cfg80211_probe_status(sdata->dev, hdr->addr1,
-						      cookie, acked,
+						      cookie, -1, acked,
 						      info->status.ack_signal,
 						      is_valid_ack_signal,
 						      GFP_ATOMIC);
diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c
index c13b209fad47a..ca62d7bc59a91 100644
--- a/net/mac80211/tx.c
+++ b/net/mac80211/tx.c
@@ -1309,10 +1309,17 @@ static struct txq_info *ieee80211_get_txq(struct ieee80211_local *local,
 	    (info->control.flags & IEEE80211_TX_CTRL_PS_RESPONSE))
 		return NULL;
 
+	/*
+	 * While (re)association request/response frames are not considered
+	 * bufferable MMPDUs, use the TXQ abstraction for the transmission of
+	 * these frames. This is specifically useful for drivers that might
+	 * associate other resources with the TXQ, e.g., encryption keys etc.
+	 */
 	if (!(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) &&
 	    unlikely(!ieee80211_is_data_present(hdr->frame_control))) {
 		if ((!ieee80211_is_mgmt(hdr->frame_control) ||
 		     ieee80211_is_bufferable_mmpdu(skb) ||
+		     ieee80211_is_assoc(hdr->frame_control) ||
 		     vif->type == NL80211_IFTYPE_STATION ||
 		     vif->type == NL80211_IFTYPE_NAN ||
 		     vif->type == NL80211_IFTYPE_NAN_DATA) &&
@@ -2607,6 +2614,18 @@ static u16 ieee80211_store_ack_skb(struct ieee80211_local *local,
 	return info_id;
 }
 
+static void ieee80211_remove_ack_skb(struct ieee80211_local *local, u16 info_id)
+{
+	struct sk_buff *ack_skb;
+	unsigned long flags;
+
+	spin_lock_irqsave(&local->ack_status_lock, flags);
+	ack_skb = idr_remove(&local->ack_status_frames, info_id);
+	spin_unlock_irqrestore(&local->ack_status_lock, flags);
+
+	kfree_skb(ack_skb);
+}
+
 /**
  * ieee80211_build_hdr - build 802.11 header in the given frame
  * @sdata: virtual interface to build the header for
@@ -2982,7 +3001,8 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata,
 		if (ieee80211_skb_resize(sdata, skb, head_need, ENCRYPT_DATA)) {
 			ieee80211_free_txskb(&local->hw, skb);
 			skb = NULL;
-			return ERR_PTR(-ENOMEM);
+			ret = -ENOMEM;
+			goto free;
 		}
 	}
 
@@ -3050,6 +3070,8 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata,
 
 	return skb;
  free:
+	if (info_id)
+		ieee80211_remove_ack_skb(local, info_id);
 	kfree_skb(skb);
 	return ERR_PTR(ret);
 }
diff --git a/net/mac80211/util.c b/net/mac80211/util.c
index f6d4ae4127c87..59f73dabe6e0a 100644
--- a/net/mac80211/util.c
+++ b/net/mac80211/util.c
@@ -73,6 +73,9 @@ u8 *ieee80211_get_bssid(struct ieee80211_hdr *hdr, size_t len,
 	if (ieee80211_is_s1g_beacon(fc)) {
 		struct ieee80211_ext *ext = (void *) hdr;
 
+		if (len < offsetofend(struct ieee80211_ext, u.s1g_beacon.sa))
+			return NULL;
+
 		return ext->u.s1g_beacon.sa;
 	}
 
diff --git a/net/netfilter/ipvs/ip_vs_app.c b/net/netfilter/ipvs/ip_vs_app.c
index d54d7da583346..b0e00be85cb12 100644
--- a/net/netfilter/ipvs/ip_vs_app.c
+++ b/net/netfilter/ipvs/ip_vs_app.c
@@ -361,14 +361,13 @@ static inline int app_tcp_pkt_out(struct ip_vs_conn *cp, struct sk_buff *skb,
 				  struct ip_vs_iphdr *ipvsh)
 {
 	int diff;
-	const unsigned int tcp_offset = ip_hdrlen(skb);
 	struct tcphdr *th;
 	__u32 seq;
 
-	if (skb_ensure_writable(skb, tcp_offset + sizeof(*th)))
+	if (skb_ensure_writable(skb, ipvsh->len + sizeof(*th)))
 		return 0;
 
-	th = (struct tcphdr *)(skb_network_header(skb) + tcp_offset);
+	th = (struct tcphdr *)(skb_network_header(skb) + ipvsh->len);
 
 	/*
 	 *	Remember seq number in case this pkt gets resized
@@ -438,14 +437,13 @@ static inline int app_tcp_pkt_in(struct ip_vs_conn *cp, struct sk_buff *skb,
 				 struct ip_vs_iphdr *ipvsh)
 {
 	int diff;
-	const unsigned int tcp_offset = ip_hdrlen(skb);
 	struct tcphdr *th;
 	__u32 seq;
 
-	if (skb_ensure_writable(skb, tcp_offset + sizeof(*th)))
+	if (skb_ensure_writable(skb, ipvsh->len + sizeof(*th)))
 		return 0;
 
-	th = (struct tcphdr *)(skb_network_header(skb) + tcp_offset);
+	th = (struct tcphdr *)(skb_network_header(skb) + ipvsh->len);
 
 	/*
 	 *	Remember seq number in case this pkt gets resized
diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c
index 35cbe821c2596..bafab93451d03 100644
--- a/net/netfilter/ipvs/ip_vs_core.c
+++ b/net/netfilter/ipvs/ip_vs_core.c
@@ -1219,8 +1219,7 @@ static int ip_vs_out_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb,
 	snet.in6 = ciph.saddr.in6;
 	offset = ciph.len;
 	return handle_response_icmp(AF_INET6, skb, &snet, ciph.protocol, cp,
-				    pp, offset, sizeof(struct ipv6hdr),
-				    hooknum);
+				    pp, offset, ipvsh->len, hooknum);
 }
 #endif
 
diff --git a/net/netfilter/ipvs/ip_vs_nfct.c b/net/netfilter/ipvs/ip_vs_nfct.c
index 81974f69e5bb4..347185fd0c8c6 100644
--- a/net/netfilter/ipvs/ip_vs_nfct.c
+++ b/net/netfilter/ipvs/ip_vs_nfct.c
@@ -208,7 +208,7 @@ static void ip_vs_nfct_expect_callback(struct nf_conn *ct,
  * Use port 0 to expect connection from any port.
  */
 void ip_vs_nfct_expect_related(struct sk_buff *skb, struct nf_conn *ct,
-			       struct ip_vs_conn *cp, u_int8_t proto,
+			       struct ip_vs_conn *cp, u8 proto,
 			       const __be16 port, int from_rs)
 {
 	struct nf_conntrack_expect *exp;
diff --git a/net/netfilter/ipvs/ip_vs_xmit.c b/net/netfilter/ipvs/ip_vs_xmit.c
index ce542ed4b013c..9fef4335da13f 100644
--- a/net/netfilter/ipvs/ip_vs_xmit.c
+++ b/net/netfilter/ipvs/ip_vs_xmit.c
@@ -736,13 +736,11 @@ int
 ip_vs_bypass_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
 		  struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
 {
-	struct iphdr  *iph = ip_hdr(skb);
-
-	if (__ip_vs_get_out_rt(cp->ipvs, cp->af, skb, NULL, iph->daddr,
+	if (__ip_vs_get_out_rt(cp->ipvs, cp->af, skb, NULL, ip_hdr(skb)->daddr,
 			       IP_VS_RT_MODE_NON_LOCAL, NULL, ipvsh) < 0)
 		goto tx_error;
 
-	ip_send_check(iph);
+	ip_send_check(ip_hdr(skb));
 
 	/* Another hack: avoid icmp_send in ip_fragment */
 	skb->ignore_df = 1;
diff --git a/net/netfilter/nf_conncount.c b/net/netfilter/nf_conncount.c
index 91582069f6d2e..e9ea6d9466e74 100644
--- a/net/netfilter/nf_conncount.c
+++ b/net/netfilter/nf_conncount.c
@@ -211,8 +211,8 @@ static int __nf_conncount_add(struct net *net,
 			/* Not found, but might be about to be confirmed */
 			if (PTR_ERR(found) == -EAGAIN) {
 				if (nf_ct_tuple_equal(&conn->tuple, &tuple) &&
-				    nf_ct_zone_id(&conn->zone, conn->zone.dir) ==
-				    nf_ct_zone_id(zone, zone->dir))
+				    nf_ct_zone_id(&conn->zone, IP_CT_DIR_ORIGINAL) ==
+				    nf_ct_zone_id(zone, IP_CT_DIR_ORIGINAL))
 					goto out_put; /* already exists */
 			} else {
 				collect++;
@@ -223,7 +223,7 @@ static int __nf_conncount_add(struct net *net,
 		found_ct = nf_ct_tuplehash_to_ctrack(found);
 
 		if (nf_ct_tuple_equal(&conn->tuple, &tuple) &&
-		    nf_ct_zone_equal(found_ct, zone, zone->dir)) {
+		    nf_ct_zone_equal(found_ct, zone, IP_CT_DIR_ORIGINAL)) {
 			/*
 			 * We should not see tuples twice unless someone hooks
 			 * this into a table without "-p tcp --syn".
diff --git a/net/netfilter/nf_conntrack_amanda.c b/net/netfilter/nf_conntrack_amanda.c
index ddafbdfc96dc3..06d6ec12c86df 100644
--- a/net/netfilter/nf_conntrack_amanda.c
+++ b/net/netfilter/nf_conntrack_amanda.c
@@ -89,7 +89,7 @@ static int amanda_help(struct sk_buff *skb,
 	struct nf_conntrack_tuple *tuple;
 	unsigned int dataoff, start, stop, off, i;
 	char pbuf[sizeof("65535")], *tmp;
-	u_int16_t len;
+	u16 len;
 	__be16 port;
 	int ret = NF_ACCEPT;
 	nf_nat_amanda_hook_fn *nf_nat_amanda;
@@ -199,10 +199,10 @@ static int __init nf_conntrack_amanda_init(void)
 	}
 
 	nf_ct_helper_init(&amanda_helper[0], AF_INET, IPPROTO_UDP,
-			  HELPER_NAME, 10080, 10080, 10080,
+			  HELPER_NAME,
 			  &amanda_exp_policy, 0, amanda_help, NULL, THIS_MODULE);
 	nf_ct_helper_init(&amanda_helper[1], AF_INET6, IPPROTO_UDP,
-			  HELPER_NAME, 10080, 10080, 10080,
+			  HELPER_NAME,
 			  &amanda_exp_policy, 0, amanda_help, NULL, THIS_MODULE);
 
 	ret = nf_conntrack_helpers_register(amanda_helper,
diff --git a/net/netfilter/nf_conntrack_broadcast.c b/net/netfilter/nf_conntrack_broadcast.c
index bf78828c7549d..6ff954f1bfb87 100644
--- a/net/netfilter/nf_conntrack_broadcast.c
+++ b/net/netfilter/nf_conntrack_broadcast.c
@@ -66,8 +66,6 @@ int nf_conntrack_broadcast_help(struct sk_buff *skb,
 	exp->tuple                = ct->tuplehash[IP_CT_DIR_REPLY].tuple;
 
 	helper = rcu_dereference(help->helper);
-	if (helper)
-		exp->tuple.src.u.udp.port = helper->tuple.src.u.udp.port;
 
 	exp->mask.src.u3.ip       = mask;
 	exp->mask.src.u.udp.port  = htons(0xFFFF);
diff --git a/net/netfilter/nf_conntrack_ecache.c b/net/netfilter/nf_conntrack_ecache.c
index 9df159448b897..cc8d8e85169fb 100644
--- a/net/netfilter/nf_conntrack_ecache.c
+++ b/net/netfilter/nf_conntrack_ecache.c
@@ -77,7 +77,7 @@ static enum retry_state ecache_work_evict_list(struct nf_conntrack_net *cnet)
 		hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 		hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode, &evicted_list);
 
-		if (time_after(stop, jiffies)) {
+		if (time_after(jiffies, stop)) {
 			ret = STATE_RESTART;
 			break;
 		}
diff --git a/net/netfilter/nf_conntrack_ftp.c b/net/netfilter/nf_conntrack_ftp.c
index 0847f845613d6..f3944598c1726 100644
--- a/net/netfilter/nf_conntrack_ftp.c
+++ b/net/netfilter/nf_conntrack_ftp.c
@@ -35,11 +35,6 @@ MODULE_ALIAS("ip_conntrack_ftp");
 MODULE_ALIAS_NFCT_HELPER(HELPER_NAME);
 static DEFINE_SPINLOCK(nf_ftp_lock);
 
-#define MAX_PORTS 8
-static u_int16_t ports[MAX_PORTS];
-static unsigned int ports_c;
-module_param_array(ports, ushort, &ports_c, 0400);
-
 static bool loose;
 module_param(loose, bool, 0600);
 
@@ -560,8 +555,8 @@ static int nf_ct_ftp_from_nlattr(struct nlattr *attr, struct nf_conn *ct)
 	return 0;
 }
 
-static struct nf_conntrack_helper ftp[MAX_PORTS * 2] __read_mostly;
-static struct nf_conntrack_helper *ftp_ptr[MAX_PORTS * 2] __read_mostly;
+static struct nf_conntrack_helper ftp __read_mostly;
+static struct nf_conntrack_helper *ftp_ptr __read_mostly;
 
 static const struct nf_conntrack_expect_policy ftp_exp_policy = {
 	.max_expected	= 1,
@@ -570,32 +565,23 @@ static const struct nf_conntrack_expect_policy ftp_exp_policy = {
 
 static void __exit nf_conntrack_ftp_fini(void)
 {
-	nf_conntrack_helpers_unregister(ftp_ptr, ports_c * 2);
+	nf_conntrack_helper_unregister(ftp_ptr);
 }
 
 static int __init nf_conntrack_ftp_init(void)
 {
-	int i, ret = 0;
+	int ret = 0;
 
 	NF_CT_HELPER_BUILD_BUG_ON(sizeof(struct nf_ct_ftp_master));
 
-	if (ports_c == 0)
-		ports[ports_c++] = FTP_PORT;
-
 	/* FIXME should be configurable whether IPv4 and IPv6 FTP connections
 		 are tracked or not - YK */
-	for (i = 0; i < ports_c; i++) {
-		nf_ct_helper_init(&ftp[2 * i], AF_INET, IPPROTO_TCP,
-				  HELPER_NAME, FTP_PORT, ports[i], ports[i],
-				  &ftp_exp_policy, 0, help,
-				  nf_ct_ftp_from_nlattr, THIS_MODULE);
-		nf_ct_helper_init(&ftp[2 * i + 1], AF_INET6, IPPROTO_TCP,
-				  HELPER_NAME, FTP_PORT, ports[i], ports[i],
-				  &ftp_exp_policy, 0, help,
-				  nf_ct_ftp_from_nlattr, THIS_MODULE);
-	}
+	nf_ct_helper_init(&ftp, NFPROTO_UNSPEC, IPPROTO_TCP,
+			  HELPER_NAME,
+			  &ftp_exp_policy, 0, help,
+			  nf_ct_ftp_from_nlattr, THIS_MODULE);
 
-	ret = nf_conntrack_helpers_register(ftp, ports_c * 2, ftp_ptr);
+	ret = nf_conntrack_helper_register(&ftp, &ftp_ptr);
 	if (ret < 0) {
 		pr_err("failed to register helpers\n");
 		return ret;
diff --git a/net/netfilter/nf_conntrack_h323_main.c b/net/netfilter/nf_conntrack_h323_main.c
index 24931e379985b..4cb1665bba021 100644
--- a/net/netfilter/nf_conntrack_h323_main.c
+++ b/net/netfilter/nf_conntrack_h323_main.c
@@ -671,7 +671,7 @@ static int expect_h245(struct sk_buff *skb, struct nf_conn *ct,
 static int callforward_do_filter(struct net *net,
 				 const union nf_inet_addr *src,
 				 const union nf_inet_addr *dst,
-				 u_int8_t family)
+				 u8 family)
 {
 	int ret = 0;
 
@@ -1713,19 +1713,19 @@ static int __init h323_helper_init(void)
 	int ret;
 
 	nf_ct_helper_init(&nf_conntrack_helper_ras[0], AF_INET, IPPROTO_UDP,
-			  "RAS", RAS_PORT, RAS_PORT, RAS_PORT,
+			  "RAS",
 			  &ras_exp_policy, 0, ras_help, NULL, THIS_MODULE);
 	nf_ct_helper_init(&nf_conntrack_helper_ras[1], AF_INET6, IPPROTO_UDP,
-			  "RAS", RAS_PORT, RAS_PORT, RAS_PORT,
+			  "RAS",
 			  &ras_exp_policy, 0, ras_help, NULL, THIS_MODULE);
 	nf_ct_helper_init(&nf_conntrack_helper_h245, AF_UNSPEC, IPPROTO_UDP,
-			  "H.245", 0, 0, 0,
+			  "H.245",
 			  &h245_exp_policy, 0, h245_help, NULL, THIS_MODULE);
 	nf_ct_helper_init(&nf_conntrack_helper_q931[0], AF_INET, IPPROTO_TCP,
-			  "Q.931", Q931_PORT, Q931_PORT, Q931_PORT,
+			  "Q.931",
 			  &q931_exp_policy, 0, q931_help, NULL, THIS_MODULE);
 	nf_ct_helper_init(&nf_conntrack_helper_q931[1], AF_INET6, IPPROTO_TCP,
-			  "Q.931", Q931_PORT, Q931_PORT, Q931_PORT,
+			  "Q.931",
 			  &q931_exp_policy, 0, q931_help, NULL, THIS_MODULE);
 
 	ret = nf_conntrack_helper_register(&nf_conntrack_helper_h245,
diff --git a/net/netfilter/nf_conntrack_helper.c b/net/netfilter/nf_conntrack_helper.c
index 500509b17663e..506c580347615 100644
--- a/net/netfilter/nf_conntrack_helper.c
+++ b/net/netfilter/nf_conntrack_helper.c
@@ -40,12 +40,16 @@ static unsigned int nf_ct_helper_count __read_mostly;
 static DEFINE_MUTEX(nf_ct_nat_helpers_mutex);
 static struct list_head nf_ct_nat_helpers __read_mostly;
 
-/* Stupid hash, but collision free for the default registrations of the
- * helpers currently in the kernel. */
-static unsigned int helper_hash(const struct nf_conntrack_tuple *tuple)
+static unsigned int helper_hash(const char *name, u8 protonum)
 {
-	return (((tuple->src.l3num << 8) | tuple->dst.protonum) ^
-		(__force __u16)tuple->src.u.all) % nf_ct_helper_hsize;
+	static u32 seed;
+	u32 initval;
+
+	get_random_once(&seed, sizeof(seed));
+
+	initval = seed ^ protonum;
+
+	return jhash(name, strlen(name), initval) % nf_ct_helper_hsize;
 }
 
 struct nf_conntrack_helper *
@@ -54,18 +58,18 @@ __nf_conntrack_helper_find(const char *name, u16 l3num, u8 protonum)
 	struct nf_conntrack_helper *h;
 	unsigned int i;
 
-	for (i = 0; i < nf_ct_helper_hsize; i++) {
-		hlist_for_each_entry_rcu(h, &nf_ct_helper_hash[i], hnode) {
-			if (strcmp(h->name, name))
-				continue;
+	if (!nf_ct_helper_hash)
+		return NULL;
 
-			if (h->tuple.src.l3num != NFPROTO_UNSPEC &&
-			    h->tuple.src.l3num != l3num)
-				continue;
+	i = helper_hash(name, protonum);
 
-			if (h->tuple.dst.protonum == protonum)
-				return h;
-		}
+	hlist_for_each_entry_rcu(h, &nf_ct_helper_hash[i], hnode) {
+		if (strcmp(h->name, name))
+			continue;
+		if (h->nfproto != NFPROTO_UNSPEC && h->nfproto != l3num)
+			continue;
+		if (h->l4proto == protonum)
+			return h;
 	}
 	return NULL;
 }
@@ -363,9 +367,8 @@ EXPORT_SYMBOL_GPL(nf_ct_helper_log);
 
 int __nf_conntrack_helper_register(struct nf_conntrack_helper *me)
 {
-	struct nf_conntrack_tuple_mask mask = { .src.u.all = htons(0xFFFF) };
-	unsigned int h = helper_hash(&me->tuple);
 	struct nf_conntrack_helper *cur;
+	unsigned int h;
 	int ret = 0, i;
 
 	BUG_ON(me->expect_class_max >= NF_CT_MAX_EXPECT_CLASSES);
@@ -382,29 +385,18 @@ int __nf_conntrack_helper_register(struct nf_conntrack_helper *me)
 			return -EINVAL;
 	}
 
+	h = helper_hash(me->name, me->l4proto);
 	mutex_lock(&nf_ct_helper_mutex);
-	for (i = 0; i < nf_ct_helper_hsize; i++) {
-		hlist_for_each_entry(cur, &nf_ct_helper_hash[i], hnode) {
-			if (!strcmp(cur->name, me->name) &&
-			    (cur->tuple.src.l3num == NFPROTO_UNSPEC ||
-			     cur->tuple.src.l3num == me->tuple.src.l3num) &&
-			    cur->tuple.dst.protonum == me->tuple.dst.protonum) {
-				ret = -EBUSY;
-				goto out;
-			}
+	hlist_for_each_entry(cur, &nf_ct_helper_hash[h], hnode) {
+		if (!strcmp(cur->name, me->name) &&
+		    (cur->nfproto == NFPROTO_UNSPEC ||
+		     cur->nfproto == me->nfproto) &&
+		    cur->l4proto == me->l4proto) {
+			ret = -EBUSY;
+			goto out;
 		}
 	}
 
-	/* avoid unpredictable behaviour for auto_assign_helper */
-	if (!(me->flags & NF_CT_HELPER_F_USERSPACE)) {
-		hlist_for_each_entry(cur, &nf_ct_helper_hash[h], hnode) {
-			if (nf_ct_tuple_src_mask_cmp(&cur->tuple, &me->tuple,
-						     &mask)) {
-				ret = -EBUSY;
-				goto out;
-			}
-		}
-	}
 	refcount_set(&me->ct_refcnt, 1);
 	hlist_add_head_rcu(&me->hnode, &nf_ct_helper_hash[h]);
 	nf_ct_helper_count++;
@@ -479,8 +471,7 @@ void nf_conntrack_helper_unregister(struct nf_conntrack_helper *me)
 EXPORT_SYMBOL_GPL(nf_conntrack_helper_unregister);
 
 void nf_ct_helper_init(struct nf_conntrack_helper *helper,
-		       u16 l3num, u16 protonum, const char *name,
-		       u16 default_port, u16 spec_port, u32 id,
+		       u8 l3num, u16 protonum, const char *name,
 		       const struct nf_conntrack_expect_policy *exp_pol,
 		       u32 expect_class_max,
 		       int (*help)(struct sk_buff *skb, unsigned int protoff,
@@ -492,9 +483,8 @@ void nf_ct_helper_init(struct nf_conntrack_helper *helper,
 {
 	memset(helper, 0, sizeof(*helper));
 
-	helper->tuple.src.l3num = l3num;
-	helper->tuple.dst.protonum = protonum;
-	helper->tuple.src.u.all = htons(spec_port);
+	helper->nfproto = l3num;
+	helper->l4proto = protonum;
 
 	rcu_assign_pointer(helper->help, help);
 	helper->from_nlattr = from_nlattr;
@@ -502,10 +492,7 @@ void nf_ct_helper_init(struct nf_conntrack_helper *helper,
 	snprintf(helper->nat_mod_name, sizeof(helper->nat_mod_name),
 		 NF_NAT_HELPER_PREFIX "%s", name);
 
-	if (spec_port == default_port)
-		snprintf(helper->name, sizeof(helper->name), "%s", name);
-	else
-		snprintf(helper->name, sizeof(helper->name), "%s-%u", name, id);
+	snprintf(helper->name, sizeof(helper->name), "%s", name);
 
 	if (WARN_ON_ONCE(expect_class_max >= NF_CT_MAX_EXPECT_CLASSES))
 		return;
diff --git a/net/netfilter/nf_conntrack_irc.c b/net/netfilter/nf_conntrack_irc.c
index 193ab34db795f..4e6bafe414375 100644
--- a/net/netfilter/nf_conntrack_irc.c
+++ b/net/netfilter/nf_conntrack_irc.c
@@ -21,9 +21,6 @@
 #include <net/netfilter/nf_conntrack_helper.h>
 #include <linux/netfilter/nf_conntrack_irc.h>
 
-#define MAX_PORTS 8
-static unsigned short ports[MAX_PORTS];
-static unsigned int ports_c;
 static unsigned int max_dcc_channels = 8;
 static unsigned int dcc_timeout __read_mostly = 300;
 /* This is slow, but it's simple. --RR */
@@ -42,8 +39,6 @@ MODULE_LICENSE("GPL");
 MODULE_ALIAS("ip_conntrack_irc");
 MODULE_ALIAS_NFCT_HELPER(HELPER_NAME);
 
-module_param_array(ports, ushort, &ports_c, 0400);
-MODULE_PARM_DESC(ports, "port numbers of IRC servers");
 module_param(max_dcc_channels, uint, 0400);
 MODULE_PARM_DESC(max_dcc_channels, "max number of expected DCC channels per "
 				   "IRC session");
@@ -254,13 +249,13 @@ static int help(struct sk_buff *skb, unsigned int protoff,
 	return ret;
 }
 
-static struct nf_conntrack_helper irc[MAX_PORTS] __read_mostly;
-static struct nf_conntrack_helper *irc_ptr[MAX_PORTS] __read_mostly;
+static struct nf_conntrack_helper irc __read_mostly;
+static struct nf_conntrack_helper *irc_ptr __read_mostly;
 static struct nf_conntrack_expect_policy irc_exp_policy;
 
 static int __init nf_conntrack_irc_init(void)
 {
-	int i, ret;
+	int ret;
 
 	nf_conntrack_helper_deprecated(HELPER_NAME);
 
@@ -282,17 +277,11 @@ static int __init nf_conntrack_irc_init(void)
 	if (!irc_buffer)
 		return -ENOMEM;
 
-	/* If no port given, default to standard irc port */
-	if (ports_c == 0)
-		ports[ports_c++] = IRC_PORT;
+	nf_ct_helper_init(&irc, AF_INET, IPPROTO_TCP, HELPER_NAME,
+			  &irc_exp_policy,
+			  0, help, NULL, THIS_MODULE);
 
-	for (i = 0; i < ports_c; i++) {
-		nf_ct_helper_init(&irc[i], AF_INET, IPPROTO_TCP, HELPER_NAME,
-				  IRC_PORT, ports[i], i, &irc_exp_policy,
-				  0, help, NULL, THIS_MODULE);
-	}
-
-	ret = nf_conntrack_helpers_register(&irc[0], ports_c, irc_ptr);
+	ret = nf_conntrack_helper_register(&irc, &irc_ptr);
 	if (ret) {
 		pr_err("failed to register helpers\n");
 		kfree(irc_buffer);
@@ -304,7 +293,7 @@ static int __init nf_conntrack_irc_init(void)
 
 static void __exit nf_conntrack_irc_fini(void)
 {
-	nf_conntrack_helpers_unregister(irc_ptr, ports_c);
+	nf_conntrack_helper_unregister(irc_ptr);
 	kfree(irc_buffer);
 }
 
diff --git a/net/netfilter/nf_conntrack_netbios_ns.c b/net/netfilter/nf_conntrack_netbios_ns.c
index 89d1cf7d6512e..caa2b101fa9e5 100644
--- a/net/netfilter/nf_conntrack_netbios_ns.c
+++ b/net/netfilter/nf_conntrack_netbios_ns.c
@@ -21,7 +21,6 @@
 #include <net/netfilter/nf_conntrack_expect.h>
 
 #define HELPER_NAME	"netbios-ns"
-#define NMBD_PORT	137
 
 MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
 MODULE_DESCRIPTION("NetBIOS name service broadcast connection tracking helper");
@@ -54,7 +53,6 @@ static int __init nf_conntrack_netbios_ns_init(void)
 	exp_policy.timeout = timeout;
 
 	nf_ct_helper_init(&helper, AF_INET, IPPROTO_UDP, HELPER_NAME,
-			  NMBD_PORT, NMBD_PORT, NMBD_PORT,
 			  &exp_policy, 0, netbios_ns_help, NULL, THIS_MODULE);
 
 	return nf_conntrack_helper_register(&helper, &helper_ptr);
diff --git a/net/netfilter/nf_conntrack_ovs.c b/net/netfilter/nf_conntrack_ovs.c
index 49d1511e99218..b4085af3ad1ce 100644
--- a/net/netfilter/nf_conntrack_ovs.c
+++ b/net/netfilter/nf_conntrack_ovs.c
@@ -31,8 +31,8 @@ int nf_ct_helper(struct sk_buff *skb, struct nf_conn *ct,
 	if (!helper)
 		return NF_ACCEPT;
 
-	if (helper->tuple.src.l3num != NFPROTO_UNSPEC &&
-	    helper->tuple.src.l3num != proto)
+	if (helper->nfproto != NFPROTO_UNSPEC &&
+	    helper->nfproto != proto)
 		return NF_ACCEPT;
 
 	switch (proto) {
@@ -60,7 +60,7 @@ int nf_ct_helper(struct sk_buff *skb, struct nf_conn *ct,
 		return NF_DROP;
 	}
 
-	if (helper->tuple.dst.protonum != proto)
+	if (helper->l4proto != proto)
 		return NF_ACCEPT;
 
 	helper_cb = rcu_dereference(helper->help);
diff --git a/net/netfilter/nf_conntrack_pptp.c b/net/netfilter/nf_conntrack_pptp.c
index 80fc14c87ddc4..cbf32a3cb1f61 100644
--- a/net/netfilter/nf_conntrack_pptp.c
+++ b/net/netfilter/nf_conntrack_pptp.c
@@ -540,7 +540,7 @@ static int __init nf_conntrack_pptp_init(void)
 	NF_CT_HELPER_BUILD_BUG_ON(sizeof(struct nf_ct_pptp_master));
 
 	nf_ct_helper_init(&pptp, AF_INET, IPPROTO_TCP,
-			  "pptp", PPTP_CONTROL_PORT, PPTP_CONTROL_PORT, PPTP_CONTROL_PORT,
+			  "pptp",
 			  &pptp_exp_policy, 0, conntrack_pptp_help, NULL, THIS_MODULE);
 
 	pptp.destroy = gre_pptp_destroy_siblings;
diff --git a/net/netfilter/nf_conntrack_sane.c b/net/netfilter/nf_conntrack_sane.c
index 39085acf7a718..a0658f69d78fb 100644
--- a/net/netfilter/nf_conntrack_sane.c
+++ b/net/netfilter/nf_conntrack_sane.c
@@ -34,11 +34,6 @@ MODULE_AUTHOR("Michal Schmidt <mschmidt@redhat.com>");
 MODULE_DESCRIPTION("SANE connection tracking helper");
 MODULE_ALIAS_NFCT_HELPER(HELPER_NAME);
 
-#define MAX_PORTS 8
-static u_int16_t ports[MAX_PORTS];
-static unsigned int ports_c;
-module_param_array(ports, ushort, &ports_c, 0400);
-
 struct sane_request {
 	__be32 RPC_code;
 #define SANE_NET_START      7   /* RPC code */
@@ -169,8 +164,8 @@ static int help(struct sk_buff *skb,
 	return ret;
 }
 
-static struct nf_conntrack_helper sane[MAX_PORTS * 2] __read_mostly;
-static struct nf_conntrack_helper *sane_ptr[MAX_PORTS * 2] __read_mostly;
+static struct nf_conntrack_helper sane __read_mostly;
+static struct nf_conntrack_helper *sane_ptr __read_mostly;
 
 static const struct nf_conntrack_expect_policy sane_exp_policy = {
 	.max_expected	= 1,
@@ -179,32 +174,21 @@ static const struct nf_conntrack_expect_policy sane_exp_policy = {
 
 static void __exit nf_conntrack_sane_fini(void)
 {
-	nf_conntrack_helpers_unregister(sane_ptr, ports_c * 2);
+	nf_conntrack_helper_unregister(sane_ptr);
 }
 
 static int __init nf_conntrack_sane_init(void)
 {
-	int i, ret = 0;
+	int ret = 0;
 
 	NF_CT_HELPER_BUILD_BUG_ON(sizeof(struct nf_ct_sane_master));
 
-	if (ports_c == 0)
-		ports[ports_c++] = SANE_PORT;
-
-	/* FIXME should be configurable whether IPv4 and IPv6 connections
-		 are tracked or not - YK */
-	for (i = 0; i < ports_c; i++) {
-		nf_ct_helper_init(&sane[2 * i], AF_INET, IPPROTO_TCP,
-				  HELPER_NAME, SANE_PORT, ports[i], ports[i],
-				  &sane_exp_policy, 0, help, NULL,
-				  THIS_MODULE);
-		nf_ct_helper_init(&sane[2 * i + 1], AF_INET6, IPPROTO_TCP,
-				  HELPER_NAME, SANE_PORT, ports[i], ports[i],
-				  &sane_exp_policy, 0, help, NULL,
-				  THIS_MODULE);
-	}
+	nf_ct_helper_init(&sane, NFPROTO_UNSPEC, IPPROTO_TCP,
+			  HELPER_NAME,
+			  &sane_exp_policy, 0, help, NULL,
+			  THIS_MODULE);
 
-	ret = nf_conntrack_helpers_register(sane, ports_c * 2, sane_ptr);
+	ret = nf_conntrack_helper_register(&sane, &sane_ptr);
 	if (ret < 0) {
 		pr_err("failed to register helpers\n");
 		return ret;
diff --git a/net/netfilter/nf_conntrack_sip.c b/net/netfilter/nf_conntrack_sip.c
index f3f90a8663389..0ff089e03891e 100644
--- a/net/netfilter/nf_conntrack_sip.c
+++ b/net/netfilter/nf_conntrack_sip.c
@@ -35,12 +35,6 @@ MODULE_DESCRIPTION("SIP connection tracking helper");
 MODULE_ALIAS("ip_conntrack_sip");
 MODULE_ALIAS_NFCT_HELPER(HELPER_NAME);
 
-#define MAX_PORTS	8
-static unsigned short ports[MAX_PORTS];
-static unsigned int ports_c;
-module_param_array(ports, ushort, &ports_c, 0400);
-MODULE_PARM_DESC(ports, "port numbers of SIP servers");
-
 static unsigned int sip_timeout __read_mostly = SIP_TIMEOUT;
 module_param(sip_timeout, uint, 0600);
 MODULE_PARM_DESC(sip_timeout, "timeout for the master SIP session");
@@ -1767,8 +1761,8 @@ static int sip_help_udp(struct sk_buff *skb, unsigned int protoff,
 	return process_sip_msg(skb, ct, protoff, dataoff, &dptr, &datalen);
 }
 
-static struct nf_conntrack_helper sip[MAX_PORTS * 4] __read_mostly;
-static struct nf_conntrack_helper *sip_ptr[MAX_PORTS * 4] __read_mostly;
+static struct nf_conntrack_helper sip[2] __read_mostly;
+static struct nf_conntrack_helper *sip_ptr[2] __read_mostly;
 
 static const struct nf_conntrack_expect_policy sip_exp_policy[SIP_EXPECT_MAX + 1] = {
 	[SIP_EXPECT_SIGNALLING] = {
@@ -1795,38 +1789,25 @@ static const struct nf_conntrack_expect_policy sip_exp_policy[SIP_EXPECT_MAX + 1
 
 static void __exit nf_conntrack_sip_fini(void)
 {
-	nf_conntrack_helpers_unregister(sip_ptr, ports_c * 4);
+	nf_conntrack_helpers_unregister(sip_ptr, 2);
 }
 
 static int __init nf_conntrack_sip_init(void)
 {
-	int i, ret;
+	int ret;
 
 	NF_CT_HELPER_BUILD_BUG_ON(sizeof(struct nf_ct_sip_master));
 
-	if (ports_c == 0)
-		ports[ports_c++] = SIP_PORT;
-
-	for (i = 0; i < ports_c; i++) {
-		nf_ct_helper_init(&sip[4 * i], AF_INET, IPPROTO_UDP,
-				  HELPER_NAME, SIP_PORT, ports[i], i,
-				  sip_exp_policy, SIP_EXPECT_MAX, sip_help_udp,
-				  NULL, THIS_MODULE);
-		nf_ct_helper_init(&sip[4 * i + 1], AF_INET, IPPROTO_TCP,
-				  HELPER_NAME, SIP_PORT, ports[i], i,
-				  sip_exp_policy, SIP_EXPECT_MAX, sip_help_tcp,
-				  NULL, THIS_MODULE);
-		nf_ct_helper_init(&sip[4 * i + 2], AF_INET6, IPPROTO_UDP,
-				  HELPER_NAME, SIP_PORT, ports[i], i,
-				  sip_exp_policy, SIP_EXPECT_MAX, sip_help_udp,
-				  NULL, THIS_MODULE);
-		nf_ct_helper_init(&sip[4 * i + 3], AF_INET6, IPPROTO_TCP,
-				  HELPER_NAME, SIP_PORT, ports[i], i,
-				  sip_exp_policy, SIP_EXPECT_MAX, sip_help_tcp,
-				  NULL, THIS_MODULE);
-	}
+	nf_ct_helper_init(&sip[0], NFPROTO_UNSPEC, IPPROTO_UDP,
+			  HELPER_NAME,
+			  sip_exp_policy, SIP_EXPECT_MAX, sip_help_udp,
+			  NULL, THIS_MODULE);
+	nf_ct_helper_init(&sip[1], NFPROTO_UNSPEC, IPPROTO_TCP,
+			  HELPER_NAME,
+			  sip_exp_policy, SIP_EXPECT_MAX, sip_help_tcp,
+			  NULL, THIS_MODULE);
 
-	ret = nf_conntrack_helpers_register(sip, ports_c * 4, sip_ptr);
+	ret = nf_conntrack_helpers_register(sip, 2, sip_ptr);
 	if (ret < 0) {
 		pr_err("failed to register helpers\n");
 		return ret;
diff --git a/net/netfilter/nf_conntrack_snmp.c b/net/netfilter/nf_conntrack_snmp.c
index b6fce5703fce7..109986d5d55e6 100644
--- a/net/netfilter/nf_conntrack_snmp.c
+++ b/net/netfilter/nf_conntrack_snmp.c
@@ -14,8 +14,6 @@
 #include <net/netfilter/nf_conntrack_expect.h>
 #include <linux/netfilter/nf_conntrack_snmp.h>
 
-#define SNMP_PORT	161
-
 MODULE_AUTHOR("Jiri Olsa <jolsa@redhat.com>");
 MODULE_DESCRIPTION("SNMP service broadcast connection tracking helper");
 MODULE_LICENSE("GPL");
@@ -55,7 +53,7 @@ static int __init nf_conntrack_snmp_init(void)
 	exp_policy.timeout = timeout;
 
 	nf_ct_helper_init(&helper, AF_INET, IPPROTO_UDP,
-			  "snmp", SNMP_PORT, SNMP_PORT, SNMP_PORT,
+			  "snmp",
 			  &exp_policy, 0, snmp_conntrack_help, NULL,
 			  THIS_MODULE);
 
diff --git a/net/netfilter/nf_conntrack_tftp.c b/net/netfilter/nf_conntrack_tftp.c
index 4393c435aa354..a69559edf9b36 100644
--- a/net/netfilter/nf_conntrack_tftp.c
+++ b/net/netfilter/nf_conntrack_tftp.c
@@ -26,12 +26,6 @@ MODULE_LICENSE("GPL");
 MODULE_ALIAS("ip_conntrack_tftp");
 MODULE_ALIAS_NFCT_HELPER(HELPER_NAME);
 
-#define MAX_PORTS 8
-static unsigned short ports[MAX_PORTS];
-static unsigned int ports_c;
-module_param_array(ports, ushort, &ports_c, 0400);
-MODULE_PARM_DESC(ports, "Port numbers of TFTP servers");
-
 nf_nat_tftp_hook_fn __rcu *nf_nat_tftp_hook __read_mostly;
 EXPORT_SYMBOL_GPL(nf_nat_tftp_hook);
 
@@ -95,8 +89,8 @@ static int tftp_help(struct sk_buff *skb,
 	return ret;
 }
 
-static struct nf_conntrack_helper tftp[MAX_PORTS * 2] __read_mostly;
-static struct nf_conntrack_helper *tftp_ptr[MAX_PORTS * 2] __read_mostly;
+static struct nf_conntrack_helper tftp __read_mostly;
+static struct nf_conntrack_helper *tftp_ptr __read_mostly;
 
 static const struct nf_conntrack_expect_policy tftp_exp_policy = {
 	.max_expected	= 1,
@@ -105,30 +99,21 @@ static const struct nf_conntrack_expect_policy tftp_exp_policy = {
 
 static void __exit nf_conntrack_tftp_fini(void)
 {
-	nf_conntrack_helpers_unregister(tftp_ptr, ports_c * 2);
+	nf_conntrack_helper_unregister(tftp_ptr);
 }
 
 static int __init nf_conntrack_tftp_init(void)
 {
-	int i, ret;
+	int ret;
 
 	NF_CT_HELPER_BUILD_BUG_ON(0);
 
-	if (ports_c == 0)
-		ports[ports_c++] = TFTP_PORT;
-
-	for (i = 0; i < ports_c; i++) {
-		nf_ct_helper_init(&tftp[2 * i], AF_INET, IPPROTO_UDP,
-				  HELPER_NAME, TFTP_PORT, ports[i], i,
-				  &tftp_exp_policy, 0, tftp_help, NULL,
-				  THIS_MODULE);
-		nf_ct_helper_init(&tftp[2 * i + 1], AF_INET6, IPPROTO_UDP,
-				  HELPER_NAME, TFTP_PORT, ports[i], i,
-				  &tftp_exp_policy, 0, tftp_help, NULL,
-				  THIS_MODULE);
-	}
+	nf_ct_helper_init(&tftp, NFPROTO_UNSPEC, IPPROTO_UDP,
+			  HELPER_NAME,
+			  &tftp_exp_policy, 0, tftp_help, NULL,
+			  THIS_MODULE);
 
-	ret = nf_conntrack_helpers_register(tftp, ports_c * 2, tftp_ptr);
+	ret = nf_conntrack_helper_register(&tftp, &tftp_ptr);
 	if (ret < 0) {
 		pr_err("failed to register helpers\n");
 		return ret;
diff --git a/net/netfilter/nf_flow_table_core.c b/net/netfilter/nf_flow_table_core.c
index 2a829b5e8240a..b66e654393411 100644
--- a/net/netfilter/nf_flow_table_core.c
+++ b/net/netfilter/nf_flow_table_core.c
@@ -127,18 +127,18 @@ static int flow_offload_fill_route(struct flow_offload *flow,
 
 	switch (route->tuple[dir].xmit_type) {
 	case FLOW_OFFLOAD_XMIT_DIRECT:
-		if (flow_tuple->tun_num) {
+		if (route->tuple[!dir].in.num_tuns) {
 			flow_tuple->dst_cache = dst;
 			flow_tuple->dst_cookie =
 				flow_offload_dst_cookie(flow_tuple);
+		} else {
+			dst_release(dst);
 		}
 		memcpy(flow_tuple->out.h_dest, route->tuple[dir].out.h_dest,
 		       ETH_ALEN);
 		memcpy(flow_tuple->out.h_source, route->tuple[dir].out.h_source,
 		       ETH_ALEN);
 		flow_tuple->out.ifidx = route->tuple[dir].out.ifindex;
-		if (!flow_tuple->tun_num)
-			dst_release(dst);
 		break;
 	case FLOW_OFFLOAD_XMIT_XFRM:
 	case FLOW_OFFLOAD_XMIT_NEIGH:
diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c
index 63ff6b4d5d214..8ac326e1eb5ba 100644
--- a/net/netfilter/nf_nat_core.c
+++ b/net/netfilter/nf_nat_core.c
@@ -39,12 +39,6 @@ static struct hlist_head *nf_nat_bysource __read_mostly;
 static unsigned int nf_nat_htable_size __read_mostly;
 static siphash_aligned_key_t nf_nat_hash_rnd;
 
-struct nf_nat_lookup_hook_priv {
-	struct nf_hook_entries __rcu *entries;
-
-	struct rcu_head rcu_head;
-};
-
 struct nf_nat_hooks_net {
 	struct nf_hook_ops *nat_hook_ops;
 	unsigned int users;
diff --git a/net/netfilter/nf_nat_proto.c b/net/netfilter/nf_nat_proto.c
index 07f51fe75fbe4..64b9bac228eae 100644
--- a/net/netfilter/nf_nat_proto.c
+++ b/net/netfilter/nf_nat_proto.c
@@ -770,6 +770,7 @@ static const struct nf_hook_ops nf_nat_ipv4_ops[] = {
 		.pf		= NFPROTO_IPV4,
 		.hooknum	= NF_INET_PRE_ROUTING,
 		.priority	= NF_IP_PRI_NAT_DST,
+		.hook_ops_type	= NF_HOOK_OP_NAT,
 	},
 	/* After packet filtering, change source */
 	{
@@ -777,6 +778,7 @@ static const struct nf_hook_ops nf_nat_ipv4_ops[] = {
 		.pf		= NFPROTO_IPV4,
 		.hooknum	= NF_INET_POST_ROUTING,
 		.priority	= NF_IP_PRI_NAT_SRC,
+		.hook_ops_type	= NF_HOOK_OP_NAT,
 	},
 	/* Before packet filtering, change destination */
 	{
@@ -784,6 +786,7 @@ static const struct nf_hook_ops nf_nat_ipv4_ops[] = {
 		.pf		= NFPROTO_IPV4,
 		.hooknum	= NF_INET_LOCAL_OUT,
 		.priority	= NF_IP_PRI_NAT_DST,
+		.hook_ops_type	= NF_HOOK_OP_NAT,
 	},
 	/* After packet filtering, change source */
 	{
@@ -791,6 +794,7 @@ static const struct nf_hook_ops nf_nat_ipv4_ops[] = {
 		.pf		= NFPROTO_IPV4,
 		.hooknum	= NF_INET_LOCAL_IN,
 		.priority	= NF_IP_PRI_NAT_SRC,
+		.hook_ops_type	= NF_HOOK_OP_NAT,
 	},
 };
 
@@ -1031,6 +1035,7 @@ static const struct nf_hook_ops nf_nat_ipv6_ops[] = {
 		.pf		= NFPROTO_IPV6,
 		.hooknum	= NF_INET_PRE_ROUTING,
 		.priority	= NF_IP6_PRI_NAT_DST,
+		.hook_ops_type	= NF_HOOK_OP_NAT,
 	},
 	/* After packet filtering, change source */
 	{
@@ -1038,6 +1043,7 @@ static const struct nf_hook_ops nf_nat_ipv6_ops[] = {
 		.pf		= NFPROTO_IPV6,
 		.hooknum	= NF_INET_POST_ROUTING,
 		.priority	= NF_IP6_PRI_NAT_SRC,
+		.hook_ops_type	= NF_HOOK_OP_NAT,
 	},
 	/* Before packet filtering, change destination */
 	{
@@ -1045,6 +1051,7 @@ static const struct nf_hook_ops nf_nat_ipv6_ops[] = {
 		.pf		= NFPROTO_IPV6,
 		.hooknum	= NF_INET_LOCAL_OUT,
 		.priority	= NF_IP6_PRI_NAT_DST,
+		.hook_ops_type	= NF_HOOK_OP_NAT,
 	},
 	/* After packet filtering, change source */
 	{
@@ -1052,6 +1059,7 @@ static const struct nf_hook_ops nf_nat_ipv6_ops[] = {
 		.pf		= NFPROTO_IPV6,
 		.hooknum	= NF_INET_LOCAL_IN,
 		.priority	= NF_IP6_PRI_NAT_SRC,
+		.hook_ops_type	= NF_HOOK_OP_NAT,
 	},
 };
 
diff --git a/net/netfilter/nfnetlink_cthelper.c b/net/netfilter/nfnetlink_cthelper.c
index f062ac2103432..5b6a09f46903f 100644
--- a/net/netfilter/nfnetlink_cthelper.c
+++ b/net/netfilter/nfnetlink_cthelper.c
@@ -67,7 +67,7 @@ nfnl_userspace_cthelper(struct sk_buff *skb, unsigned int protoff,
 }
 
 static const struct nla_policy nfnl_cthelper_tuple_pol[NFCTH_TUPLE_MAX+1] = {
-	[NFCTH_TUPLE_L3PROTONUM] = { .type = NLA_U16, },
+	[NFCTH_TUPLE_L3PROTONUM] = NLA_POLICY_MAX(NLA_BE16, NFPROTO_IPV6),
 	[NFCTH_TUPLE_L4PROTONUM] = { .type = NLA_U8, },
 };
 
@@ -256,7 +256,8 @@ nfnl_cthelper_create(const struct nlattr * const tb[],
 	helper->data_len = size;
 
 	helper->flags |= NF_CT_HELPER_F_USERSPACE;
-	memcpy(&helper->tuple, tuple, sizeof(struct nf_conntrack_tuple));
+	helper->nfproto = tuple->src.l3num;
+	helper->l4proto = tuple->dst.protonum;
 
 	helper->me = THIS_MODULE;
 	helper->help = nfnl_userspace_cthelper;
@@ -453,8 +454,8 @@ static int nfnl_cthelper_new(struct sk_buff *skb, const struct nfnl_info *info,
 		if (strncmp(cur->name, helper_name, NF_CT_HELPER_NAME_LEN))
 			continue;
 
-		if ((tuple.src.l3num != cur->tuple.src.l3num ||
-		     tuple.dst.protonum != cur->tuple.dst.protonum))
+		if ((tuple.src.l3num != cur->nfproto ||
+		     tuple.dst.protonum != cur->l4proto))
 			continue;
 
 		if (info->nlh->nlmsg_flags & NLM_F_EXCL)
@@ -483,10 +484,10 @@ nfnl_cthelper_dump_tuple(struct sk_buff *skb,
 		goto nla_put_failure;
 
 	if (nla_put_be16(skb, NFCTH_TUPLE_L3PROTONUM,
-			 htons(helper->tuple.src.l3num)))
+			 htons(helper->nfproto)))
 		goto nla_put_failure;
 
-	if (nla_put_u8(skb, NFCTH_TUPLE_L4PROTONUM, helper->tuple.dst.protonum))
+	if (nla_put_u8(skb, NFCTH_TUPLE_L4PROTONUM, helper->l4proto))
 		goto nla_put_failure;
 
 	nla_nest_end(skb, nest_parms);
@@ -665,8 +666,8 @@ static int nfnl_cthelper_get(struct sk_buff *skb, const struct nfnl_info *info,
 			continue;
 
 		if (tuple_set &&
-		    (tuple.src.l3num != cur->tuple.src.l3num ||
-		     tuple.dst.protonum != cur->tuple.dst.protonum))
+		    (tuple.src.l3num != cur->nfproto ||
+		     tuple.dst.protonum != cur->l4proto))
 			continue;
 
 		skb2 = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
@@ -725,8 +726,8 @@ static int nfnl_cthelper_del(struct sk_buff *skb, const struct nfnl_info *info,
 			continue;
 
 		if (tuple_set &&
-		    (tuple.src.l3num != cur->tuple.src.l3num ||
-		     tuple.dst.protonum != cur->tuple.dst.protonum))
+		    (tuple.src.l3num != cur->nfproto ||
+		     tuple.dst.protonum != cur->l4proto))
 			continue;
 
 		found = true;
diff --git a/net/netfilter/nfnetlink_cttimeout.c b/net/netfilter/nfnetlink_cttimeout.c
index 170d3db860c56..66c2016f6049d 100644
--- a/net/netfilter/nfnetlink_cttimeout.c
+++ b/net/netfilter/nfnetlink_cttimeout.c
@@ -168,7 +168,7 @@ static int cttimeout_new_timeout(struct sk_buff *skb,
 	if (ret < 0)
 		goto err_free_timeout_policy;
 
-	strcpy(timeout->name, nla_data(cda[CTA_TIMEOUT_NAME]));
+	nla_strscpy(timeout->name, cda[CTA_TIMEOUT_NAME], sizeof(timeout->name));
 	timeout->timeout->l3num = l3num;
 	timeout->timeout->l4proto = l4proto;
 	refcount_set(&timeout->timeout->refcnt, 1);
diff --git a/net/netfilter/nfnetlink_hook.c b/net/netfilter/nfnetlink_hook.c
index 5623c18fcd12f..95005e9a60668 100644
--- a/net/netfilter/nfnetlink_hook.c
+++ b/net/netfilter/nfnetlink_hook.c
@@ -190,7 +190,7 @@ static int nfnl_hook_put_nft_ft_info(struct sk_buff *nlskb,
 
 static int nfnl_hook_dump_one(struct sk_buff *nlskb,
 			      const struct nfnl_dump_hook_data *ctx,
-			      const struct nf_hook_ops *ops,
+			      const struct nf_hook_ops *ops, int priority,
 			      int family, unsigned int seq)
 {
 	u16 event = nfnl_msg_type(NFNL_SUBSYS_HOOK, NFNL_MSG_HOOK_GET);
@@ -244,7 +244,7 @@ static int nfnl_hook_dump_one(struct sk_buff *nlskb,
 	if (ret)
 		goto nla_put_failure;
 
-	ret = nla_put_be32(nlskb, NFNLA_HOOK_PRIORITY, htonl(ops->priority));
+	ret = nla_put_be32(nlskb, NFNLA_HOOK_PRIORITY, htonl(priority));
 	if (ret)
 		goto nla_put_failure;
 
@@ -337,6 +337,30 @@ nfnl_hook_entries_head(u8 pf, unsigned int hook, struct net *net, const char *de
 	return hook_head;
 }
 
+static int nfnl_hook_dump_nat(struct sk_buff *nlskb,
+			      const struct nfnl_dump_hook_data *ctx,
+			      const struct nf_hook_ops *ops,
+			      int family, unsigned int seq)
+{
+	struct nf_nat_lookup_hook_priv *priv = ops->priv;
+	struct nf_hook_entries *e = rcu_dereference(priv->entries);
+	struct nf_hook_ops **nat_ops;
+	int i, err;
+
+	if (!e)
+		return 0;
+
+	nat_ops = nf_hook_entries_get_hook_ops(e);
+
+	for (i = 0; i < e->num_hook_entries; i++) {
+		err = nfnl_hook_dump_one(nlskb, ctx, nat_ops[i],
+					 ops->priority, family, seq);
+		if (err)
+			return err;
+	}
+	return 0;
+}
+
 static int nfnl_hook_dump(struct sk_buff *nlskb,
 			  struct netlink_callback *cb)
 {
@@ -365,8 +389,13 @@ static int nfnl_hook_dump(struct sk_buff *nlskb,
 	ops = nf_hook_entries_get_hook_ops(e);
 
 	for (; i < e->num_hook_entries; i++) {
-		err = nfnl_hook_dump_one(nlskb, ctx, ops[i], family,
-					 cb->nlh->nlmsg_seq);
+		if (ops[i]->hook_ops_type == NF_HOOK_OP_NAT)
+			err = nfnl_hook_dump_nat(nlskb, ctx, ops[i], family,
+						 cb->nlh->nlmsg_seq);
+		else
+			err = nfnl_hook_dump_one(nlskb, ctx, ops[i],
+						 ops[i]->priority, family,
+						 cb->nlh->nlmsg_seq);
 		if (err)
 			break;
 	}
diff --git a/net/netfilter/nft_ct.c b/net/netfilter/nft_ct.c
index 03a88c77e0f02..358b9287e12e8 100644
--- a/net/netfilter/nft_ct.c
+++ b/net/netfilter/nft_ct.c
@@ -1297,6 +1297,17 @@ static int nft_ct_expect_obj_dump(struct sk_buff *skb,
 	return 0;
 }
 
+#if IS_ENABLED(CONFIG_NF_NAT)
+static void nft_ct_nat_follow_master(struct nf_conn *ct, struct nf_conntrack_expect *this)
+{
+	const struct nf_ct_helper_expectfn *expfn;
+
+	expfn = nf_ct_helper_expectfn_find_by_name("nat-follow-master");
+	if (expfn)
+		expfn->expectfn(ct, this);
+}
+#endif
+
 static void nft_ct_expect_obj_eval(struct nft_object *obj,
 				   struct nft_regs *regs,
 				   const struct nft_pktinfo *pkt)
@@ -1342,6 +1353,13 @@ static void nft_ct_expect_obj_eval(struct nft_object *obj,
 		          priv->l4proto, NULL, &priv->dport);
 	exp->timeout += priv->timeout;
 
+#if IS_ENABLED(CONFIG_NF_NAT)
+	if (ct->status & IPS_NAT_MASK) {
+		exp->saved_proto.tcp.port = priv->dport;
+		exp->dir = !dir;
+		exp->expectfn = nft_ct_nat_follow_master;
+	}
+#endif
 	if (nf_ct_expect_related(exp, 0) != 0)
 		regs->verdict.code = NF_DROP;
 
@@ -1375,6 +1393,13 @@ static struct nft_object_type nft_ct_expect_obj_type __read_mostly = {
 	.owner		= THIS_MODULE,
 };
 
+#if IS_ENABLED(CONFIG_NF_NAT)
+static struct nf_ct_helper_expectfn nft_ct_nat __read_mostly = {
+	.name = "nft_ct-follow-master",
+	.expectfn = nft_ct_nat_follow_master,
+};
+#endif
+
 static int __init nft_ct_module_init(void)
 {
 	int err;
@@ -1400,6 +1425,9 @@ static int __init nft_ct_module_init(void)
 	err = nft_register_obj(&nft_ct_timeout_obj_type);
 	if (err < 0)
 		goto err4;
+#endif
+#if IS_ENABLED(CONFIG_NF_NAT)
+	nf_ct_helper_expectfn_register(&nft_ct_nat);
 #endif
 	return 0;
 
@@ -1425,6 +1453,13 @@ static void __exit nft_ct_module_exit(void)
 	nft_unregister_obj(&nft_ct_helper_obj_type);
 	nft_unregister_expr(&nft_notrack_type);
 	nft_unregister_expr(&nft_ct_type);
+
+#if IS_ENABLED(CONFIG_NF_NAT)
+	nf_ct_helper_expectfn_unregister(&nft_ct_nat);
+	synchronize_rcu();
+	nf_ct_helper_expectfn_destroy(&nft_ct_nat);
+	synchronize_rcu();
+#endif
 }
 
 module_init(nft_ct_module_init);
diff --git a/net/netfilter/nft_set_rbtree.c b/net/netfilter/nft_set_rbtree.c
index 6222e9bb57bc9..9894832281c4b 100644
--- a/net/netfilter/nft_set_rbtree.c
+++ b/net/netfilter/nft_set_rbtree.c
@@ -548,8 +548,7 @@ static int nft_array_intervals_alloc(struct nft_array *array, u32 max_intervals)
 	if (!intervals)
 		return -ENOMEM;
 
-	if (array->intervals)
-		kvfree(array->intervals);
+	kvfree(array->intervals);
 
 	array->intervals = intervals;
 	array->max_intervals = max_intervals;
diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c
index 4e6708c23922b..e64116bf26378 100644
--- a/net/netfilter/x_tables.c
+++ b/net/netfilter/x_tables.c
@@ -1920,9 +1920,9 @@ static const struct seq_operations xt_target_seq_ops = {
 	.show	= xt_target_seq_show,
 };
 
-#define FORMAT_TABLES	"_tables_names"
-#define	FORMAT_MATCHES	"_tables_matches"
-#define FORMAT_TARGETS 	"_tables_targets"
+#define FORMAT_TABLES	"%s_tables_names"
+#define	FORMAT_MATCHES	"%s_tables_matches"
+#define FORMAT_TARGETS	"%s_tables_targets"
 
 #endif /* CONFIG_PROC_FS */
 
@@ -2033,8 +2033,7 @@ int xt_proto_init(struct net *net, u_int8_t af)
 	root_uid = make_kuid(net->user_ns, 0);
 	root_gid = make_kgid(net->user_ns, 0);
 
-	strscpy(buf, xt_prefix[af], sizeof(buf));
-	strlcat(buf, FORMAT_TABLES, sizeof(buf));
+	snprintf(buf, sizeof(buf), FORMAT_TABLES, xt_prefix[af]);
 	proc = proc_create_net_data(buf, 0440, net->proc_net, &xt_table_seq_ops,
 			sizeof(struct seq_net_private),
 			(void *)(unsigned long)af);
@@ -2043,8 +2042,7 @@ int xt_proto_init(struct net *net, u_int8_t af)
 	if (uid_valid(root_uid) && gid_valid(root_gid))
 		proc_set_user(proc, root_uid, root_gid);
 
-	strscpy(buf, xt_prefix[af], sizeof(buf));
-	strlcat(buf, FORMAT_MATCHES, sizeof(buf));
+	snprintf(buf, sizeof(buf), FORMAT_MATCHES, xt_prefix[af]);
 	proc = proc_create_seq_private(buf, 0440, net->proc_net,
 			&xt_match_seq_ops, sizeof(struct nf_mttg_trav),
 			(void *)(unsigned long)af);
@@ -2053,8 +2051,7 @@ int xt_proto_init(struct net *net, u_int8_t af)
 	if (uid_valid(root_uid) && gid_valid(root_gid))
 		proc_set_user(proc, root_uid, root_gid);
 
-	strscpy(buf, xt_prefix[af], sizeof(buf));
-	strlcat(buf, FORMAT_TARGETS, sizeof(buf));
+	snprintf(buf, sizeof(buf), FORMAT_TARGETS, xt_prefix[af]);
 	proc = proc_create_seq_private(buf, 0440, net->proc_net,
 			 &xt_target_seq_ops, sizeof(struct nf_mttg_trav),
 			 (void *)(unsigned long)af);
@@ -2068,13 +2065,11 @@ int xt_proto_init(struct net *net, u_int8_t af)
 
 #ifdef CONFIG_PROC_FS
 out_remove_matches:
-	strscpy(buf, xt_prefix[af], sizeof(buf));
-	strlcat(buf, FORMAT_MATCHES, sizeof(buf));
+	snprintf(buf, sizeof(buf), FORMAT_MATCHES, xt_prefix[af]);
 	remove_proc_entry(buf, net->proc_net);
 
 out_remove_tables:
-	strscpy(buf, xt_prefix[af], sizeof(buf));
-	strlcat(buf, FORMAT_TABLES, sizeof(buf));
+	snprintf(buf, sizeof(buf), FORMAT_TABLES, xt_prefix[af]);
 	remove_proc_entry(buf, net->proc_net);
 out:
 	return -1;
@@ -2087,16 +2082,13 @@ void xt_proto_fini(struct net *net, u_int8_t af)
 #ifdef CONFIG_PROC_FS
 	char buf[XT_FUNCTION_MAXNAMELEN];
 
-	strscpy(buf, xt_prefix[af], sizeof(buf));
-	strlcat(buf, FORMAT_TABLES, sizeof(buf));
+	snprintf(buf, sizeof(buf), FORMAT_TABLES, xt_prefix[af]);
 	remove_proc_entry(buf, net->proc_net);
 
-	strscpy(buf, xt_prefix[af], sizeof(buf));
-	strlcat(buf, FORMAT_TARGETS, sizeof(buf));
+	snprintf(buf, sizeof(buf), FORMAT_TARGETS, xt_prefix[af]);
 	remove_proc_entry(buf, net->proc_net);
 
-	strscpy(buf, xt_prefix[af], sizeof(buf));
-	strlcat(buf, FORMAT_MATCHES, sizeof(buf));
+	snprintf(buf, sizeof(buf), FORMAT_MATCHES, xt_prefix[af]);
 	remove_proc_entry(buf, net->proc_net);
 #endif /*CONFIG_PROC_FS*/
 }
diff --git a/net/netfilter/xt_TCPOPTSTRIP.c b/net/netfilter/xt_TCPOPTSTRIP.c
index 93f064306901c..265d21697847b 100644
--- a/net/netfilter/xt_TCPOPTSTRIP.c
+++ b/net/netfilter/xt_TCPOPTSTRIP.c
@@ -16,7 +16,7 @@
 #include <linux/netfilter/x_tables.h>
 #include <linux/netfilter/xt_TCPOPTSTRIP.h>
 
-static inline unsigned int optlen(const u_int8_t *opt, unsigned int offset)
+static inline unsigned int optlen(const u8 *opt, unsigned int offset)
 {
 	/* Beware zero-length options: make finite progress */
 	if (opt[offset] <= TCPOPT_NOP || opt[offset+1] == 0)
@@ -33,8 +33,8 @@ tcpoptstrip_mangle_packet(struct sk_buff *skb,
 	const struct xt_tcpoptstrip_target_info *info = par->targinfo;
 	struct tcphdr *tcph, _th;
 	unsigned int optl, i, j;
-	u_int16_t n, o;
-	u_int8_t *opt;
+	u16 n, o;
+	u8 *opt;
 	int tcp_hdrlen;
 
 	/* This is a fragment, no TCP header is available */
@@ -97,7 +97,7 @@ tcpoptstrip_tg6(struct sk_buff *skb, const struct xt_action_param *par)
 {
 	struct ipv6hdr *ipv6h = ipv6_hdr(skb);
 	int tcphoff;
-	u_int8_t nexthdr;
+	u8 nexthdr;
 	__be16 frag_off;
 
 	nexthdr = ipv6h->nexthdr;
diff --git a/net/netfilter/xt_dscp.c b/net/netfilter/xt_dscp.c
index fb0169a8f9bbd..878f27016e998 100644
--- a/net/netfilter/xt_dscp.c
+++ b/net/netfilter/xt_dscp.c
@@ -49,6 +49,16 @@ static int dscp_mt_check(const struct xt_mtchk_param *par)
 	return 0;
 }
 
+static int tos_mt_check(const struct xt_mtchk_param *par)
+{
+	const struct xt_tos_match_info *info = par->matchinfo;
+
+	if (info->invert > 1)
+		return -EINVAL;
+
+	return 0;
+}
+
 static bool tos_mt(const struct sk_buff *skb, struct xt_action_param *par)
 {
 	const struct xt_tos_match_info *info = par->matchinfo;
@@ -82,6 +92,7 @@ static struct xt_match dscp_mt_reg[] __read_mostly = {
 		.name		= "tos",
 		.revision	= 1,
 		.family		= NFPROTO_IPV4,
+		.checkentry	= tos_mt_check,
 		.match		= tos_mt,
 		.matchsize	= sizeof(struct xt_tos_match_info),
 		.me		= THIS_MODULE,
@@ -90,6 +101,7 @@ static struct xt_match dscp_mt_reg[] __read_mostly = {
 		.name		= "tos",
 		.revision	= 1,
 		.family		= NFPROTO_IPV6,
+		.checkentry	= tos_mt_check,
 		.match		= tos_mt,
 		.matchsize	= sizeof(struct xt_tos_match_info),
 		.me		= THIS_MODULE,
diff --git a/net/netfilter/xt_nat.c b/net/netfilter/xt_nat.c
index b4f7bbc3f3caf..51c7f7ce88d9a 100644
--- a/net/netfilter/xt_nat.c
+++ b/net/netfilter/xt_nat.c
@@ -26,6 +26,15 @@ static int xt_nat_checkentry_v0(const struct xt_tgchk_param *par)
 
 static int xt_nat_checkentry(const struct xt_tgchk_param *par)
 {
+	switch (par->family) {
+	case NFPROTO_IPV4:
+	case NFPROTO_IPV6:
+	case NFPROTO_INET:
+		break;
+	default:
+		return -EINVAL;
+	}
+
 	return nf_ct_netns_get(par->net, par->family);
 }
 
diff --git a/net/netfilter/xt_physdev.c b/net/netfilter/xt_physdev.c
index dd98f758176c2..a388881c68d42 100644
--- a/net/netfilter/xt_physdev.c
+++ b/net/netfilter/xt_physdev.c
@@ -130,11 +130,6 @@ static int physdev_mt_check(const struct xt_mtchk_param *par)
 		if (X(physoutdev))
 			return -ENAMETOOLONG;
 	}
-
-	if (X(in_mask))
-		return -ENAMETOOLONG;
-	if (X(out_mask))
-		return -ENAMETOOLONG;
 #undef X
 
 	if (!brnf_probed) {
diff --git a/net/netfilter/xt_recent.c b/net/netfilter/xt_recent.c
index f72752fa4374d..d34831ce3adfb 100644
--- a/net/netfilter/xt_recent.c
+++ b/net/netfilter/xt_recent.c
@@ -400,7 +400,7 @@ static int recent_mt_check(const struct xt_mtchk_param *par,
 	t->nstamps_max_mask = nstamp_mask;
 
 	memcpy(&t->mask, &info->mask, sizeof(t->mask));
-	strcpy(t->name, info->name);
+	strscpy(t->name, info->name);
 	INIT_LIST_HEAD(&t->lru_list);
 	for (i = 0; i < ip_list_hash_size; i++)
 		INIT_LIST_HEAD(&t->iphash[i]);
diff --git a/net/netfilter/xt_tcpmss.c b/net/netfilter/xt_tcpmss.c
index b9da8269161d8..b08b077d7f0ad 100644
--- a/net/netfilter/xt_tcpmss.c
+++ b/net/netfilter/xt_tcpmss.c
@@ -78,10 +78,23 @@ tcpmss_mt(const struct sk_buff *skb, struct xt_action_param *par)
 	return false;
 }
 
+static int tcpmss_mt_check(const struct xt_mtchk_param *par)
+{
+	const struct xt_tcpmss_match_info *info = par->matchinfo;
+
+	if (info->mss_min > info->mss_max)
+		return -EINVAL;
+	if (info->invert > 1)
+		return -EINVAL;
+
+	return 0;
+}
+
 static struct xt_match tcpmss_mt_reg[] __read_mostly = {
 	{
 		.name		= "tcpmss",
 		.family		= NFPROTO_IPV4,
+		.checkentry	= tcpmss_mt_check,
 		.match		= tcpmss_mt,
 		.matchsize	= sizeof(struct xt_tcpmss_match_info),
 		.proto		= IPPROTO_TCP,
diff --git a/net/nfc/digital_technology.c b/net/nfc/digital_technology.c
index ae63c5eb06faa..ae6487c10a254 100644
--- a/net/nfc/digital_technology.c
+++ b/net/nfc/digital_technology.c
@@ -778,6 +778,8 @@ static void digital_in_recv_sensf_res(struct nfc_digital_dev *ddev, void *arg,
 
 	sensf_res = (struct digital_sensf_res *)resp->data;
 
+	resp->len = min_t(unsigned int, resp->len, NFC_SENSF_RES_MAXSIZE);
+
 	memcpy(target.sensf_res, sensf_res, resp->len);
 	target.sensf_res_len = resp->len;
 
diff --git a/net/nfc/llcp_commands.c b/net/nfc/llcp_commands.c
index 291f26facbf3a..ca89fe967d6a2 100644
--- a/net/nfc/llcp_commands.c
+++ b/net/nfc/llcp_commands.c
@@ -193,7 +193,8 @@ int nfc_llcp_parse_gb_tlv(struct nfc_llcp_local *local,
 			  const u8 *tlv_array, u16 tlv_array_len)
 {
 	const u8 *tlv = tlv_array;
-	u8 type, length, offset = 0;
+	u8 type, length;
+	u16 offset = 0;
 
 	pr_debug("TLV array length %d\n", tlv_array_len);
 
@@ -201,9 +202,15 @@ int nfc_llcp_parse_gb_tlv(struct nfc_llcp_local *local,
 		return -ENODEV;
 
 	while (offset < tlv_array_len) {
+		if (offset + 2 > tlv_array_len)
+			return -EINVAL;
+
 		type = tlv[0];
 		length = tlv[1];
 
+		if (offset + 2 + length > tlv_array_len)
+			return -EINVAL;
+
 		pr_debug("type 0x%x length %d\n", type, length);
 
 		switch (type) {
@@ -243,7 +250,8 @@ int nfc_llcp_parse_connection_tlv(struct nfc_llcp_sock *sock,
 				  const u8 *tlv_array, u16 tlv_array_len)
 {
 	const u8 *tlv = tlv_array;
-	u8 type, length, offset = 0;
+	u8 type, length;
+	u16 offset = 0;
 
 	pr_debug("TLV array length %d\n", tlv_array_len);
 
@@ -251,9 +259,15 @@ int nfc_llcp_parse_connection_tlv(struct nfc_llcp_sock *sock,
 		return -ENOTCONN;
 
 	while (offset < tlv_array_len) {
+		if (offset + 2 > tlv_array_len)
+			return -EINVAL;
+
 		type = tlv[0];
 		length = tlv[1];
 
+		if (offset + 2 + length > tlv_array_len)
+			return -EINVAL;
+
 		pr_debug("type 0x%x length %d\n", type, length);
 
 		switch (type) {
diff --git a/net/nfc/llcp_core.c b/net/nfc/llcp_core.c
index dc65c719f35f2..aed5fe1afef01 100644
--- a/net/nfc/llcp_core.c
+++ b/net/nfc/llcp_core.c
@@ -1286,10 +1286,9 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local,
 {
 	struct nfc_llcp_sock *llcp_sock;
 	u8 dsap, ssap, type, length, tid, sap;
-	const u8 *tlv;
-	u16 tlv_len, offset;
+	const u8 *tlv, *tlv_end;
 	const char *service_name;
-	size_t service_name_len;
+	int service_name_len;
 	struct nfc_llcp_sdp_tlv *sdp;
 	HLIST_HEAD(llc_sdres_list);
 	size_t sdres_tlvs_len;
@@ -1305,22 +1304,34 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local,
 		return;
 	}
 
+	/*
+	 * Walk the SNL TLV list in the linear part of the skb only,
+	 * bounded by skb_tail_pointer(). Each TLV needs a two-byte
+	 * header (type, length) and its declared length must fit before
+	 * the end; this also keeps the walk safe for very short frames.
+	 */
 	tlv = &skb->data[LLCP_HEADER_SIZE];
-	tlv_len = skb->len - LLCP_HEADER_SIZE;
-	offset = 0;
+	tlv_end = skb_tail_pointer(skb);
 	sdres_tlvs_len = 0;
 
-	while (offset < tlv_len) {
+	while (tlv + 2 < tlv_end) {
 		type = tlv[0];
 		length = tlv[1];
 
+		if (tlv + 2 + length > tlv_end)
+			break;
+
 		switch (type) {
 		case LLCP_TLV_SDREQ:
+			if (length < 1)
+				break;
+
 			tid = tlv[2];
 			service_name = (char *) &tlv[3];
 			service_name_len = length - 1;
 
-			pr_debug("Looking for %.16s\n", service_name);
+			pr_debug("Looking for %.*s\n", service_name_len,
+				 service_name);
 
 			if (service_name_len == strlen("urn:nfc:sn:sdp") &&
 			    !strncmp(service_name, "urn:nfc:sn:sdp",
@@ -1380,6 +1391,9 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local,
 			break;
 
 		case LLCP_TLV_SDRES:
+			if (length != 2)
+				break;
+
 			mutex_lock(&local->sdreq_lock);
 
 			pr_debug("LLCP_TLV_SDRES: searching tid %d\n", tlv[2]);
@@ -1408,7 +1422,6 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local,
 			break;
 		}
 
-		offset += length + 2;
 		tlv += length + 2;
 	}
 
diff --git a/net/nfc/llcp_sock.c b/net/nfc/llcp_sock.c
index feab29fc62f44..5558d8a4d48b3 100644
--- a/net/nfc/llcp_sock.c
+++ b/net/nfc/llcp_sock.c
@@ -319,14 +319,22 @@ static int nfc_llcp_getsockopt(struct socket *sock, int level, int optname,
 	if (get_user(len, optlen))
 		return -EFAULT;
 
-	local = llcp_sock->local;
-	if (!local)
-		return -ENODEV;
+	if (len < 0)
+		return -EINVAL;
+
+	if (len < sizeof(u32))
+		return -EINVAL;
 
 	len = min_t(u32, len, sizeof(u32));
 
 	lock_sock(sk);
 
+	local = llcp_sock->local;
+	if (!local) {
+		release_sock(sk);
+		return -ENODEV;
+	}
+
 	switch (optname) {
 	case NFC_LLCP_RW:
 		rw = llcp_sock->rw > LLCP_MAX_RW ? local->rw : llcp_sock->rw;
diff --git a/net/nfc/nci/data.c b/net/nfc/nci/data.c
index 5f98c73db5afd..4253edea5c8db 100644
--- a/net/nfc/nci/data.c
+++ b/net/nfc/nci/data.c
@@ -46,11 +46,11 @@ void nci_data_exchange_complete(struct nci_dev *ndev, struct sk_buff *skb,
 	timer_delete_sync(&ndev->data_timer);
 	clear_bit(NCI_DATA_EXCHANGE_TO, &ndev->flags);
 
-	/* Mark the exchange as done before calling the callback.
-	 * The callback (e.g. rawsock_data_exchange_complete) may
-	 * want to immediately queue another data exchange.
-	 */
-	clear_bit(NCI_DATA_EXCHANGE, &ndev->flags);
+	/* Claim completion atomically -- both close and rx_work may race here */
+	if (!test_and_clear_bit(NCI_DATA_EXCHANGE, &ndev->flags)) {
+		kfree_skb(skb);
+		return;
+	}
 
 	if (cb) {
 		/* forward skb to nfc core */
diff --git a/net/nfc/nci/ntf.c b/net/nfc/nci/ntf.c
index c96512bb86531..87f76a29f0ece 100644
--- a/net/nfc/nci/ntf.c
+++ b/net/nfc/nci/ntf.c
@@ -525,15 +525,19 @@ static int nci_rf_discover_ntf_packet(struct nci_dev *ndev,
 
 static int nci_extract_activation_params_iso_dep(struct nci_dev *ndev,
 						 struct nci_rf_intf_activated_ntf *ntf,
-						 const __u8 *data)
+						 const __u8 *data, __u8 data_len)
 {
 	struct activation_params_nfca_poll_iso_dep *nfca_poll;
 	struct activation_params_nfcb_poll_iso_dep *nfcb_poll;
 
 	switch (ntf->activation_rf_tech_and_mode) {
 	case NCI_NFC_A_PASSIVE_POLL_MODE:
+		if (data_len < 1)
+			return NCI_STATUS_RF_PROTOCOL_ERROR;
 		nfca_poll = &ntf->activation_params.nfca_poll_iso_dep;
 		nfca_poll->rats_res_len = min_t(__u8, *data++, NFC_ATS_MAXSIZE);
+		data_len--;
+		nfca_poll->rats_res_len = min_t(__u8, nfca_poll->rats_res_len, data_len);
 		pr_debug("rats_res_len %d\n", nfca_poll->rats_res_len);
 		if (nfca_poll->rats_res_len > 0) {
 			memcpy(nfca_poll->rats_res,
@@ -542,8 +546,12 @@ static int nci_extract_activation_params_iso_dep(struct nci_dev *ndev,
 		break;
 
 	case NCI_NFC_B_PASSIVE_POLL_MODE:
+		if (data_len < 1)
+			return NCI_STATUS_RF_PROTOCOL_ERROR;
 		nfcb_poll = &ntf->activation_params.nfcb_poll_iso_dep;
 		nfcb_poll->attrib_res_len = min_t(__u8, *data++, 50);
+		data_len--;
+		nfcb_poll->attrib_res_len = min_t(__u8, nfcb_poll->attrib_res_len, data_len);
 		pr_debug("attrib_res_len %d\n", nfcb_poll->attrib_res_len);
 		if (nfcb_poll->attrib_res_len > 0) {
 			memcpy(nfcb_poll->attrib_res,
@@ -562,7 +570,7 @@ static int nci_extract_activation_params_iso_dep(struct nci_dev *ndev,
 
 static int nci_extract_activation_params_nfc_dep(struct nci_dev *ndev,
 						 struct nci_rf_intf_activated_ntf *ntf,
-						 const __u8 *data)
+						 const __u8 *data, __u8 data_len)
 {
 	struct activation_params_poll_nfc_dep *poll;
 	struct activation_params_listen_nfc_dep *listen;
@@ -570,9 +578,13 @@ static int nci_extract_activation_params_nfc_dep(struct nci_dev *ndev,
 	switch (ntf->activation_rf_tech_and_mode) {
 	case NCI_NFC_A_PASSIVE_POLL_MODE:
 	case NCI_NFC_F_PASSIVE_POLL_MODE:
+		if (data_len < 1)
+			return NCI_STATUS_RF_PROTOCOL_ERROR;
 		poll = &ntf->activation_params.poll_nfc_dep;
 		poll->atr_res_len = min_t(__u8, *data++,
 					  NFC_ATR_RES_MAXSIZE - 2);
+		data_len--;
+		poll->atr_res_len = min_t(__u8, poll->atr_res_len, data_len);
 		pr_debug("atr_res_len %d\n", poll->atr_res_len);
 		if (poll->atr_res_len > 0)
 			memcpy(poll->atr_res, data, poll->atr_res_len);
@@ -580,9 +592,13 @@ static int nci_extract_activation_params_nfc_dep(struct nci_dev *ndev,
 
 	case NCI_NFC_A_PASSIVE_LISTEN_MODE:
 	case NCI_NFC_F_PASSIVE_LISTEN_MODE:
+		if (data_len < 1)
+			return NCI_STATUS_RF_PROTOCOL_ERROR;
 		listen = &ntf->activation_params.listen_nfc_dep;
 		listen->atr_req_len = min_t(__u8, *data++,
 					    NFC_ATR_REQ_MAXSIZE - 2);
+		data_len--;
+		listen->atr_req_len = min_t(__u8, listen->atr_req_len, data_len);
 		pr_debug("atr_req_len %d\n", listen->atr_req_len);
 		if (listen->atr_req_len > 0)
 			memcpy(listen->atr_req, data, listen->atr_req_len);
@@ -603,6 +619,12 @@ static void nci_target_auto_activated(struct nci_dev *ndev,
 	struct nfc_target *target;
 	int rc;
 
+	/* This is a new target, check if we've enough room */
+	if (ndev->n_targets == NCI_MAX_DISCOVERED_TARGETS) {
+		pr_debug("not enough room, ignoring new target...\n");
+		return;
+	}
+
 	target = &ndev->targets[ndev->n_targets];
 
 	rc = nci_add_new_protocol(ndev, target, ntf->rf_protocol,
@@ -806,12 +828,14 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev,
 		switch (ntf.rf_interface) {
 		case NCI_RF_INTERFACE_ISO_DEP:
 			err = nci_extract_activation_params_iso_dep(ndev,
-								    &ntf, data);
+								    &ntf, data,
+								    ntf.activation_params_len);
 			break;
 
 		case NCI_RF_INTERFACE_NFC_DEP:
 			err = nci_extract_activation_params_nfc_dep(ndev,
-								    &ntf, data);
+								    &ntf, data,
+								    ntf.activation_params_len);
 			break;
 
 		case NCI_RF_INTERFACE_FRAME:
diff --git a/net/nfc/nci/rsp.c b/net/nfc/nci/rsp.c
index 9eeb862825c5f..dd6f16f49ceb9 100644
--- a/net/nfc/nci/rsp.c
+++ b/net/nfc/nci/rsp.c
@@ -50,11 +50,26 @@ static u8 nci_core_init_rsp_packet_v1(struct nci_dev *ndev,
 	const struct nci_core_init_rsp_1 *rsp_1 = (void *)skb->data;
 	const struct nci_core_init_rsp_2 *rsp_2;
 
+	/* Ensure that the status field can be accessed. */
+	if (skb->len < sizeof(*rsp_1))
+		return NCI_STATUS_SYNTAX_ERROR;
+
 	pr_debug("status 0x%x\n", rsp_1->status);
 
 	if (rsp_1->status != NCI_STATUS_OK)
 		return rsp_1->status;
 
+	/*
+	 * supported_rf_interfaces[] and the trailing nci_core_init_rsp_2 are
+	 * addressed using the on-wire (unclamped) interface count, so the
+	 * response must be long enough for both before any of it is parsed or
+	 * stored into @ndev - otherwise a truncated response would leave
+	 * ndev->num_supported_rf_interfaces holding the unclamped count.
+	 */
+	if (skb->len < sizeof(*rsp_1) +
+	    rsp_1->num_supported_rf_interfaces + sizeof(*rsp_2))
+		return NCI_STATUS_SYNTAX_ERROR;
+
 	ndev->nfcc_features = __le32_to_cpu(rsp_1->nfcc_features);
 	ndev->num_supported_rf_interfaces = rsp_1->num_supported_rf_interfaces;
 
@@ -88,14 +103,24 @@ static u8 nci_core_init_rsp_packet_v2(struct nci_dev *ndev,
 {
 	const struct nci_core_init_rsp_nci_ver2 *rsp = (void *)skb->data;
 	const u8 *supported_rf_interface = rsp->supported_rf_interfaces;
+	const u8 *end = skb->data + skb->len;
 	u8 rf_interface_idx = 0;
 	u8 rf_extension_cnt = 0;
 
+	if (skb->len < sizeof(*rsp))
+		return NCI_STATUS_SYNTAX_ERROR;
+
 	pr_debug("status %x\n", rsp->status);
 
 	if (rsp->status != NCI_STATUS_OK)
 		return rsp->status;
 
+	/* Success response must contain the full fixed-size header */
+	if (skb_headlen(skb) < sizeof(*rsp))
+		return NCI_STATUS_SYNTAX_ERROR;
+
+	supported_rf_interface = rsp->supported_rf_interfaces;
+
 	ndev->nfcc_features = __le32_to_cpu(rsp->nfcc_features);
 	ndev->num_supported_rf_interfaces = rsp->num_supported_rf_interfaces;
 
@@ -104,13 +129,21 @@ static u8 nci_core_init_rsp_packet_v2(struct nci_dev *ndev,
 		    NCI_MAX_SUPPORTED_RF_INTERFACES);
 
 	while (rf_interface_idx < ndev->num_supported_rf_interfaces) {
-		ndev->supported_rf_interfaces[rf_interface_idx++] = *supported_rf_interface++;
+		/* one interface byte + one extension-count byte must be present */
+		if (end - supported_rf_interface < 2)
+			return NCI_STATUS_SYNTAX_ERROR;
+		ndev->supported_rf_interfaces[rf_interface_idx++] =
+			*supported_rf_interface++;
 
-		/* skip rf extension parameters */
+		/* skip rf extension parameters, bounded by the packet */
 		rf_extension_cnt = *supported_rf_interface++;
+		if (rf_extension_cnt > end - supported_rf_interface)
+			return NCI_STATUS_SYNTAX_ERROR;
 		supported_rf_interface += rf_extension_cnt;
 	}
 
+	ndev->num_supported_rf_interfaces = rf_interface_idx;
+
 	ndev->max_logical_connections = rsp->max_logical_connections;
 	ndev->max_routing_table_size =
 			__le16_to_cpu(rsp->max_routing_table_size);
diff --git a/net/openvswitch/flow_netlink.c b/net/openvswitch/flow_netlink.c
index 13052408a132f..d8079dee700ed 100644
--- a/net/openvswitch/flow_netlink.c
+++ b/net/openvswitch/flow_netlink.c
@@ -2496,13 +2496,56 @@ static inline int add_nested_action_start(struct sw_flow_actions **sfa,
 	return used;
 }
 
-static inline void add_nested_action_end(struct sw_flow_actions *sfa,
-					 int st_offset)
+static inline int add_nested_action_end(struct sw_flow_actions *sfa,
+					int st_offset)
 {
-	struct nlattr *a = (struct nlattr *) ((unsigned char *)sfa->actions +
-							       st_offset);
+	struct nlattr *a;
+	u32 attr_len;
+
+	if (WARN_ON_ONCE(st_offset < 0 ||
+			 (u32)st_offset > sfa->actions_len))
+		return -EINVAL;
+
+	attr_len = sfa->actions_len - (u32)st_offset;
+	if (WARN_ON_ONCE(attr_len < NLA_HDRLEN))
+		return -EINVAL;
 
-	a->nla_len = sfa->actions_len - st_offset;
+	if (attr_len > U16_MAX)
+		return -EMSGSIZE;
+
+	a = (struct nlattr *)((u8 *)sfa->actions + st_offset);
+	a->nla_len = attr_len;
+	return 0;
+}
+
+/* Free the generated action-list tail at @start and truncate it.
+ * If @nested, @start points to its containing nlattr header.
+ */
+static void ovs_nla_trim(struct sw_flow_actions *sfa, int start, bool nested)
+{
+	const struct nlattr *actions;
+	u32 len;
+
+	if (start < 0)
+		return;
+
+	if (WARN_ON_ONCE((u32)start > sfa->actions_len))
+		return;
+
+	actions = (const struct nlattr *)((u8 *)sfa->actions + start);
+	len = sfa->actions_len - (u32)start;
+
+	if (nested) {
+		if (len < NLA_HDRLEN)
+			goto out;
+
+		actions = (const struct nlattr *)((u8 *)actions + NLA_HDRLEN);
+		len -= NLA_HDRLEN;
+	}
+
+	ovs_nla_free_nested_actions(actions, len);
+out:
+	sfa->actions_len = start;
 }
 
 static int __ovs_nla_copy_actions(struct net *net, const struct nlattr *attr,
@@ -2522,6 +2565,7 @@ static int validate_and_copy_sample(struct net *net, const struct nlattr *attr,
 	const struct nlattr *attrs[OVS_SAMPLE_ATTR_MAX + 1];
 	const struct nlattr *probability, *actions;
 	const struct nlattr *a;
+	int actions_start;
 	int rem, start, err;
 	struct sample_arg arg;
 
@@ -2565,18 +2609,27 @@ static int validate_and_copy_sample(struct net *net, const struct nlattr *attr,
 	err = ovs_nla_add_action(sfa, OVS_SAMPLE_ATTR_ARG, &arg, sizeof(arg),
 				 log);
 	if (err)
-		return err;
+		goto err;
 
+	actions_start = (*sfa)->actions_len;
 	err = __ovs_nla_copy_actions(net, actions, key, sfa,
 				     eth_type, vlan_tci, mpls_label_count, log,
 				     depth + 1);
 
 	if (err)
-		return err;
+		goto err_free;
 
-	add_nested_action_end(*sfa, start);
+	err = add_nested_action_end(*sfa, start);
+	if (err)
+		goto err_free;
 
 	return 0;
+
+err_free:
+	ovs_nla_trim(*sfa, actions_start, false);
+err:
+	(*sfa)->actions_len = start;
+	return err;
 }
 
 static int validate_and_copy_dec_ttl(struct net *net,
@@ -2624,18 +2677,31 @@ static int validate_and_copy_dec_ttl(struct net *net,
 		return start;
 
 	action_start = add_nested_action_start(sfa, OVS_DEC_TTL_ATTR_ACTION, log);
-	if (action_start < 0)
-		return action_start;
+	if (action_start < 0) {
+		err = action_start;
+		goto err;
+	}
 
 	err = __ovs_nla_copy_actions(net, actions, key, sfa, eth_type,
 				     vlan_tci, mpls_label_count, log,
 				     depth + 1);
 	if (err)
-		return err;
+		goto err_free;
+
+	err = add_nested_action_end(*sfa, action_start);
+	if (err)
+		goto err_free;
 
-	add_nested_action_end(*sfa, action_start);
-	add_nested_action_end(*sfa, start);
+	err = add_nested_action_end(*sfa, start);
+	if (err)
+		goto err_free;
 	return 0;
+
+err_free:
+	ovs_nla_trim(*sfa, action_start, true);
+err:
+	(*sfa)->actions_len = start;
+	return err;
 }
 
 static int validate_and_copy_clone(struct net *net,
@@ -2646,6 +2712,7 @@ static int validate_and_copy_clone(struct net *net,
 				   u32 mpls_label_count, bool log, bool last,
 				   u32 depth)
 {
+	int actions_start;
 	int start, err;
 	u32 exec;
 
@@ -2661,17 +2728,26 @@ static int validate_and_copy_clone(struct net *net,
 	err = ovs_nla_add_action(sfa, OVS_CLONE_ATTR_EXEC, &exec,
 				 sizeof(exec), log);
 	if (err)
-		return err;
+		goto err;
 
+	actions_start = (*sfa)->actions_len;
 	err = __ovs_nla_copy_actions(net, attr, key, sfa,
 				     eth_type, vlan_tci, mpls_label_count, log,
 				     depth + 1);
 	if (err)
-		return err;
+		goto err_free;
 
-	add_nested_action_end(*sfa, start);
+	err = add_nested_action_end(*sfa, start);
+	if (err)
+		goto err_free;
 
 	return 0;
+
+err_free:
+	ovs_nla_trim(*sfa, actions_start, false);
+err:
+	(*sfa)->actions_len = start;
+	return err;
 }
 
 void ovs_match_init(struct sw_flow_match *match,
@@ -2763,20 +2839,20 @@ static int validate_and_copy_set_tun(const struct nlattr *attr,
 	tun_dst = metadata_dst_alloc(key.tun_opts_len, METADATA_IP_TUNNEL,
 				     GFP_KERNEL);
 
-	if (!tun_dst)
-		return -ENOMEM;
+	if (!tun_dst) {
+		err = -ENOMEM;
+		goto err;
+	}
 
 	err = dst_cache_init(&tun_dst->u.tun_info.dst_cache, GFP_KERNEL);
-	if (err) {
-		dst_release((struct dst_entry *)tun_dst);
-		return err;
-	}
+	if (err)
+		goto err_free_tun_dst;
 
 	a = __add_action(sfa, OVS_KEY_ATTR_TUNNEL_INFO, NULL,
 			 sizeof(*ovs_tun), log);
 	if (IS_ERR(a)) {
-		dst_release((struct dst_entry *)tun_dst);
-		return PTR_ERR(a);
+		err = PTR_ERR(a);
+		goto err_free_tun_dst;
 	}
 
 	ovs_tun = nla_data(a);
@@ -2797,8 +2873,16 @@ static int validate_and_copy_set_tun(const struct nlattr *attr,
 	ip_tunnel_info_opts_set(tun_info,
 				TUN_METADATA_OPTS(&key, key.tun_opts_len),
 				key.tun_opts_len, dst_opt_type);
-	add_nested_action_end(*sfa, start);
+	err = add_nested_action_end(*sfa, start);
+	if (WARN_ON_ONCE(err))
+		goto err_free_tun_dst;
+
+	return 0;
 
+err_free_tun_dst:
+	dst_release((struct dst_entry *)tun_dst);
+err:
+	(*sfa)->actions_len = start;
 	return err;
 }
 
@@ -2971,7 +3055,7 @@ static int validate_set(const struct nlattr *a,
 
 	/* Convert non-masked non-tunnel set actions to masked set actions. */
 	if (!masked && key_type != OVS_KEY_ATTR_TUNNEL) {
-		int start, len = key_len * 2;
+		int err, start, len = key_len * 2;
 		struct nlattr *at;
 
 		*skip_copy = true;
@@ -2983,8 +3067,11 @@ static int validate_set(const struct nlattr *a,
 			return start;
 
 		at = __add_action(sfa, key_type, NULL, len, log);
-		if (IS_ERR(at))
-			return PTR_ERR(at);
+		if (IS_ERR(at)) {
+			err = PTR_ERR(at);
+			(*sfa)->actions_len = start;
+			return err;
+		}
 
 		memcpy(nla_data(at), nla_data(ovs_key), key_len); /* Key. */
 		memset(nla_data(at) + key_len, 0xff, key_len);    /* Mask. */
@@ -2994,7 +3081,11 @@ static int validate_set(const struct nlattr *a,
 
 			mask->ipv6_label &= htonl(0x000FFFFF);
 		}
-		add_nested_action_end(*sfa, start);
+		err = add_nested_action_end(*sfa, start);
+		if (WARN_ON_ONCE(err)) {
+			(*sfa)->actions_len = start;
+			return err;
+		}
 	}
 
 	return 0;
@@ -3040,7 +3131,8 @@ static int validate_and_copy_check_pkt_len(struct net *net,
 	const struct nlattr *acts_if_greater, *acts_if_lesser_eq;
 	struct nlattr *a[OVS_CHECK_PKT_LEN_ATTR_MAX + 1];
 	struct check_pkt_len_arg arg;
-	int nested_acts_start;
+	int greater_acts_start = -1;
+	int lesser_acts_start = -1;
 	int start, err;
 
 	err = nla_parse_deprecated_strict(a, OVS_CHECK_PKT_LEN_ATTR_MAX,
@@ -3075,37 +3167,58 @@ static int validate_and_copy_check_pkt_len(struct net *net,
 	err = ovs_nla_add_action(sfa, OVS_CHECK_PKT_LEN_ATTR_ARG, &arg,
 				 sizeof(arg), log);
 	if (err)
-		return err;
+		goto err_free;
 
-	nested_acts_start = add_nested_action_start(sfa,
-		OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL, log);
-	if (nested_acts_start < 0)
-		return nested_acts_start;
+	lesser_acts_start =
+		add_nested_action_start(sfa,
+					OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL,
+					log);
+	if (lesser_acts_start < 0) {
+		err = lesser_acts_start;
+		goto err_free;
+	}
 
 	err = __ovs_nla_copy_actions(net, acts_if_lesser_eq, key, sfa,
 				     eth_type, vlan_tci, mpls_label_count, log,
 				     depth + 1);
 
 	if (err)
-		return err;
+		goto err_free;
 
-	add_nested_action_end(*sfa, nested_acts_start);
+	err = add_nested_action_end(*sfa, lesser_acts_start);
+	if (err)
+		goto err_free;
 
-	nested_acts_start = add_nested_action_start(sfa,
-		OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER, log);
-	if (nested_acts_start < 0)
-		return nested_acts_start;
+	greater_acts_start =
+		add_nested_action_start(sfa,
+					OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER,
+					log);
+	if (greater_acts_start < 0) {
+		err = greater_acts_start;
+		goto err_free;
+	}
 
 	err = __ovs_nla_copy_actions(net, acts_if_greater, key, sfa,
 				     eth_type, vlan_tci, mpls_label_count, log,
 				     depth + 1);
 
 	if (err)
-		return err;
+		goto err_free;
+
+	err = add_nested_action_end(*sfa, greater_acts_start);
+	if (err)
+		goto err_free;
 
-	add_nested_action_end(*sfa, nested_acts_start);
-	add_nested_action_end(*sfa, start);
+	err = add_nested_action_end(*sfa, start);
+	if (err)
+		goto err_free;
 	return 0;
+
+err_free:
+	ovs_nla_trim(*sfa, greater_acts_start, true);
+	ovs_nla_trim(*sfa, lesser_acts_start, true);
+	ovs_nla_trim(*sfa, start, false);
+	return err;
 }
 
 static int validate_psample(const struct nlattr *attr)
diff --git a/net/rds/ib.c b/net/rds/ib.c
index 39f87272e071b..c62684d4259c4 100644
--- a/net/rds/ib.c
+++ b/net/rds/ib.c
@@ -162,12 +162,12 @@ static int rds_ib_add_one(struct ib_device *device)
 		   IB_ODP_SUPPORT_READ);
 
 	rds_ibdev->max_1m_mrs = device->attrs.max_mr ?
-		min_t(unsigned int, (device->attrs.max_mr / 2),
-		      rds_ib_mr_1m_pool_size) : rds_ib_mr_1m_pool_size;
+		min(device->attrs.max_mr / 2,
+		    rds_ib_mr_1m_pool_size) : rds_ib_mr_1m_pool_size;
 
 	rds_ibdev->max_8k_mrs = device->attrs.max_mr ?
-		min_t(unsigned int, ((device->attrs.max_mr / 2) * RDS_MR_8K_SCALE),
-		      rds_ib_mr_8k_pool_size) : rds_ib_mr_8k_pool_size;
+		min((device->attrs.max_mr / 2) * RDS_MR_8K_SCALE,
+		    rds_ib_mr_8k_pool_size) : rds_ib_mr_8k_pool_size;
 
 	rds_ibdev->max_initiator_depth = device->attrs.max_qp_init_rd_atom;
 	rds_ibdev->max_responder_resources = device->attrs.max_qp_rd_atom;
@@ -204,7 +204,7 @@ static int rds_ib_add_one(struct ib_device *device)
 		goto put_dev;
 	}
 
-	rdsdebug("RDS/IB: max_mr = %d, max_wrs = %d, max_sge = %d, max_1m_mrs = %d, max_8k_mrs = %d\n",
+	rdsdebug("RDS/IB: max_mr = %u, max_wrs = %d, max_sge = %d, max_1m_mrs = %d, max_8k_mrs = %d\n",
 		 device->attrs.max_mr, rds_ibdev->max_wrs, rds_ibdev->max_sge,
 		 rds_ibdev->max_1m_mrs, rds_ibdev->max_8k_mrs);
 
diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
index 5667f0173b47f..17e587c300766 100644
--- a/net/rds/ib_cm.c
+++ b/net/rds/ib_cm.c
@@ -173,11 +173,11 @@ static void rds_ib_cm_fill_conn_param(struct rds_connection *conn,
 
 	memset(conn_param, 0, sizeof(struct rdma_conn_param));
 
-	conn_param->responder_resources =
-		min_t(u32, rds_ibdev->max_responder_resources, max_responder_resources);
-	conn_param->initiator_depth =
-		min_t(u32, rds_ibdev->max_initiator_depth, max_initiator_depth);
-	conn_param->retry_count = min_t(unsigned int, rds_ib_retry_count, 7);
+	conn_param->responder_resources = min3(rds_ibdev->max_responder_resources,
+					       max_responder_resources, U8_MAX);
+	conn_param->initiator_depth = min3(rds_ibdev->max_initiator_depth,
+					   max_initiator_depth, U8_MAX);
+	conn_param->retry_count = min(rds_ib_retry_count, 7U);
 	conn_param->rnr_retry_count = 7;
 
 	if (dp) {
diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c
index be535a261fa04..4ca7964e83c85 100644
--- a/net/sched/act_ct.c
+++ b/net/sched/act_ct.c
@@ -1527,8 +1527,8 @@ static int tcf_ct_dump_helper(struct sk_buff *skb,
 		return 0;
 
 	if (nla_put_string(skb, TCA_CT_HELPER_NAME, helper->name) ||
-	    nla_put_u8(skb, TCA_CT_HELPER_FAMILY, helper->tuple.src.l3num) ||
-	    nla_put_u8(skb, TCA_CT_HELPER_PROTO, helper->tuple.dst.protonum))
+	    nla_put_u8(skb, TCA_CT_HELPER_FAMILY, helper->nfproto) ||
+	    nla_put_u8(skb, TCA_CT_HELPER_PROTO, helper->l4proto))
 		return -1;
 
 	return 0;
diff --git a/net/sched/act_tunnel_key.c b/net/sched/act_tunnel_key.c
index 876b30c5709e1..b14807761d829 100644
--- a/net/sched/act_tunnel_key.c
+++ b/net/sched/act_tunnel_key.c
@@ -342,14 +342,20 @@ static const struct nla_policy tunnel_key_policy[TCA_TUNNEL_KEY_MAX + 1] = {
 	[TCA_TUNNEL_KEY_ENC_TTL]      = { .type = NLA_U8 },
 };
 
-static void tunnel_key_release_params(struct tcf_tunnel_key_params *p)
+static void tunnel_key_release_params_rcu(struct rcu_head *head)
 {
-	if (!p)
-		return;
+	struct tcf_tunnel_key_params *p = container_of(head, typeof(*p), rcu);
+
 	if (p->tcft_action == TCA_TUNNEL_KEY_ACT_SET)
 		dst_release(&p->tcft_enc_metadata->dst);
+	kfree(p);
+}
 
-	kfree_rcu(p, rcu);
+static void tunnel_key_release_params(struct tcf_tunnel_key_params *p)
+{
+	if (!p)
+		return;
+	call_rcu(&p->rcu, tunnel_key_release_params_rcu);
 }
 
 static int tunnel_key_init(struct net *net, struct nlattr *nla,
diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c
index d23d935e128e9..3893b44448b38 100644
--- a/net/sctp/sm_statefuns.c
+++ b/net/sctp/sm_statefuns.c
@@ -74,7 +74,8 @@ static enum sctp_disposition sctp_sf_do_5_2_6_stale(
 					const struct sctp_association *asoc,
 					const union sctp_subtype type,
 					void *arg,
-					struct sctp_cmd_seq *commands);
+					struct sctp_cmd_seq *commands,
+					struct sctp_errhdr *err);
 static enum sctp_disposition sctp_sf_shut_8_4_5(
 					struct net *net,
 					const struct sctp_endpoint *ep,
@@ -2529,9 +2530,15 @@ enum sctp_disposition sctp_sf_cookie_echoed_err(
 	 * errors.
 	 */
 	sctp_walk_errors(err, chunk->chunk_hdr) {
-		if (SCTP_ERROR_STALE_COOKIE == err->cause)
-			return sctp_sf_do_5_2_6_stale(net, ep, asoc, type,
-							arg, commands);
+		if (err->cause != SCTP_ERROR_STALE_COOKIE)
+			continue;
+		/* The staleness is only meaningful if the cause is long
+		 * enough to hold it; a shorter one is malformed.
+		 */
+		if (ntohs(err->length) < sizeof(*err) + sizeof(__be32))
+			break;
+		return sctp_sf_do_5_2_6_stale(net, ep, asoc, type,
+					      arg, commands, err);
 	}
 
 	/* It is possible to have malformed error causes, and that
@@ -2573,13 +2580,13 @@ static enum sctp_disposition sctp_sf_do_5_2_6_stale(
 					const struct sctp_association *asoc,
 					const union sctp_subtype type,
 					void *arg,
-					struct sctp_cmd_seq *commands)
+					struct sctp_cmd_seq *commands,
+					struct sctp_errhdr *err)
 {
 	int attempts = asoc->init_err_counter + 1;
-	struct sctp_chunk *chunk = arg, *reply;
 	struct sctp_cookie_preserve_param bht;
 	struct sctp_bind_addr *bp;
-	struct sctp_errhdr *err;
+	struct sctp_chunk *reply;
 	u32 stale;
 
 	if (attempts > asoc->max_init_attempts) {
@@ -2590,8 +2597,6 @@ static enum sctp_disposition sctp_sf_do_5_2_6_stale(
 		return SCTP_DISPOSITION_DELETE_TCB;
 	}
 
-	err = (struct sctp_errhdr *)(chunk->skb->data);
-
 	/* When calculating the time extension, an implementation
 	 * SHOULD use the RTT information measured based on the
 	 * previous COOKIE ECHO / ERROR exchange, and should add no
diff --git a/net/socket.c b/net/socket.c
index 63c69a0fa74e1..b0256cd222f88 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -465,6 +465,31 @@ static const struct xattr_handler sockfs_user_xattr_handler = {
 	.set = sockfs_user_xattr_set,
 };
 
+/**
+ * sock_read_xattr - read a user.* xattr from a socket's sockfs inode
+ * @sock: socket whose inode holds the xattr
+ * @name: full xattr name, e.g. "user.bpf_test"
+ * @value: output buffer
+ * @size: size of @value in bytes
+ *
+ * SOCK_INODE() is valid only for sockfs sockets; sock_from_file() rejects
+ * anything else (e.g. tun, tap).
+ * Lockless: simple_xattr_get() looks up the value under RCU, no inode lock.
+ *
+ * Return: length of the value on success, a negative errno on error.
+ */
+int sock_read_xattr(struct socket *sock, const char *name, void *value, size_t size)
+{
+	struct file *file = sock->file;
+	struct sockfs_inode *si;
+
+	if (!file || sock_from_file(file) != sock)
+		return -EOPNOTSUPP;
+
+	si = SOCKFS_I(SOCK_INODE(sock));
+	return simple_xattr_get(&sockfs_xa_cache, &si->xattrs, name, value, size);
+}
+
 static const struct xattr_handler * const sockfs_xattr_handlers[] = {
 	&sockfs_xattr_handler,
 	&sockfs_security_xattr_handler,
diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c
index 9d3fb6848f400..8ddc65e894dab 100644
--- a/net/sunrpc/auth_gss/auth_gss.c
+++ b/net/sunrpc/auth_gss/auth_gss.c
@@ -2072,7 +2072,11 @@ gss_unwrap_resp_priv(struct rpc_task *task, struct rpc_cred *cred,
 		goto unwrap_failed;
 	opaque_len = be32_to_cpup(p++);
 	offset = (u8 *)(p) - (u8 *)head->iov_base;
-	if (offset + opaque_len > rcv_buf->len)
+	if (offset > rcv_buf->len)
+		goto unwrap_failed;
+	if (opaque_len > rcv_buf->len - offset)
+		goto unwrap_failed;
+	if (opaque_len <= GSS_KRB5_TOK_HDR_LEN)
 		goto unwrap_failed;
 
 	maj_stat = gss_unwrap(ctx->gc_gss_ctx, offset,
diff --git a/net/sunrpc/auth_gss/gss_krb5_unseal.c b/net/sunrpc/auth_gss/gss_krb5_unseal.c
index b5fb70419faaa..4d12d49434c28 100644
--- a/net/sunrpc/auth_gss/gss_krb5_unseal.c
+++ b/net/sunrpc/auth_gss/gss_krb5_unseal.c
@@ -89,6 +89,9 @@ gss_krb5_verify_mic_v2(struct krb5_ctx *ctx, struct xdr_buf *message_buffer,
 
 	dprintk("RPC:       %s\n", __func__);
 
+	if (read_token->len < GSS_KRB5_TOK_HDR_LEN + cksum_len)
+		return GSS_S_DEFECTIVE_TOKEN;
+
 	memcpy(&be16_ptr, (char *) ptr, 2);
 	if (be16_to_cpu(be16_ptr) != KG2_TOK_MIC)
 		return GSS_S_DEFECTIVE_TOKEN;
diff --git a/net/sunrpc/auth_gss/gss_krb5_wrap.c b/net/sunrpc/auth_gss/gss_krb5_wrap.c
index ac4b32df42b96..d3f61c4b5a137 100644
--- a/net/sunrpc/auth_gss/gss_krb5_wrap.c
+++ b/net/sunrpc/auth_gss/gss_krb5_wrap.c
@@ -73,6 +73,8 @@ static void _rotate_left(struct xdr_buf *buf, unsigned int shift)
 	int shifted = 0;
 	int this_shift;
 
+	if (!buf->len)
+		return;
 	shift %= buf->len;
 	while (shifted < shift) {
 		this_shift = min(shift - shifted, LOCAL_BUF_LEN);
@@ -85,6 +87,8 @@ static void rotate_left(u32 base, struct xdr_buf *buf, unsigned int shift)
 {
 	struct xdr_buf subbuf;
 
+	if (buf->len <= base)
+		return;
 	xdr_buf_subsegment(buf, &subbuf, base, buf->len - base);
 	_rotate_left(&subbuf, shift);
 }
@@ -154,6 +158,9 @@ gss_krb5_unwrap_v2(struct krb5_ctx *kctx, int offset, int len,
 
 	dprintk("RPC:       %s\n", __func__);
 
+	if (len - offset <= GSS_KRB5_TOK_HDR_LEN)
+		return GSS_S_DEFECTIVE_TOKEN;
+
 	ptr = buf->head[0].iov_base + offset;
 
 	if (be16_to_cpu(*((__be16 *)ptr)) != KG2_TOK_WRAP)
@@ -220,14 +227,16 @@ gss_krb5_unwrap_v2(struct krb5_ctx *kctx, int offset, int len,
 	 * head buffer space rather than that actually occupied.
 	 */
 	movelen = min_t(unsigned int, buf->head[0].iov_len, len);
+	if (movelen < offset + GSS_KRB5_TOK_HDR_LEN + headskip)
+		return GSS_S_DEFECTIVE_TOKEN;
 	movelen -= offset + GSS_KRB5_TOK_HDR_LEN + headskip;
-	BUG_ON(offset + GSS_KRB5_TOK_HDR_LEN + headskip + movelen >
-							buf->head[0].iov_len);
 	memmove(ptr, ptr + GSS_KRB5_TOK_HDR_LEN + headskip, movelen);
 	buf->head[0].iov_len -= GSS_KRB5_TOK_HDR_LEN + headskip;
 	buf->len = len - (GSS_KRB5_TOK_HDR_LEN + headskip);
 
 	/* Trim off the trailing "extra count" and checksum blob */
+	if (ec + GSS_KRB5_TOK_HDR_LEN + tailskip > buf->len - offset)
+		return GSS_S_DEFECTIVE_TOKEN;
 	xdr_buf_trim(buf, ec + GSS_KRB5_TOK_HDR_LEN + tailskip);
 
 	*align = XDR_QUADLEN(GSS_KRB5_TOK_HDR_LEN + headskip);
diff --git a/net/sunrpc/auth_gss/gss_rpc_upcall.c b/net/sunrpc/auth_gss/gss_rpc_upcall.c
index 0fa4778620d9a..b7f70b1adb18a 100644
--- a/net/sunrpc/auth_gss/gss_rpc_upcall.c
+++ b/net/sunrpc/auth_gss/gss_rpc_upcall.c
@@ -121,12 +121,6 @@ static int gssp_rpc_create(struct net *net, struct rpc_clnt **_clnt)
 	return result;
 }
 
-void init_gssp_clnt(struct sunrpc_net *sn)
-{
-	mutex_init(&sn->gssp_lock);
-	sn->gssp_clnt = NULL;
-}
-
 int set_gssp_clnt(struct net *net)
 {
 	struct sunrpc_net *sn = net_generic(net, sunrpc_net_id);
diff --git a/net/sunrpc/auth_gss/gss_rpc_upcall.h b/net/sunrpc/auth_gss/gss_rpc_upcall.h
index 31e96344167e3..b3c2b2b907985 100644
--- a/net/sunrpc/auth_gss/gss_rpc_upcall.h
+++ b/net/sunrpc/auth_gss/gss_rpc_upcall.h
@@ -29,7 +29,6 @@ int gssp_accept_sec_context_upcall(struct net *net,
 				struct gssp_upcall_data *data);
 void gssp_free_upcall_data(struct gssp_upcall_data *data);
 
-void init_gssp_clnt(struct sunrpc_net *);
 int set_gssp_clnt(struct net *);
 void clear_gssp_clnt(struct sunrpc_net *);
 
diff --git a/net/sunrpc/auth_gss/gss_rpc_xdr.c b/net/sunrpc/auth_gss/gss_rpc_xdr.c
index fceee648d545b..0549edae1ebee 100644
--- a/net/sunrpc/auth_gss/gss_rpc_xdr.c
+++ b/net/sunrpc/auth_gss/gss_rpc_xdr.c
@@ -222,7 +222,8 @@ static int gssx_dec_linux_creds(struct xdr_stream *xdr,
 
 	return 0;
 out_free_groups:
-	groups_free(creds->cr_group_info);
+	put_group_info(creds->cr_group_info);
+	creds->cr_group_info = NULL;
 	return err;
 }
 
@@ -230,6 +231,7 @@ static int gssx_dec_option_array(struct xdr_stream *xdr,
 				 struct gssx_option_array *oa)
 {
 	struct svc_cred *creds;
+	bool creds_decoded = false;
 	u32 count, i;
 	__be32 *p;
 	int err;
@@ -242,12 +244,12 @@ static int gssx_dec_option_array(struct xdr_stream *xdr,
 		return 0;
 
 	/* we recognize only 1 currently: CREDS_VALUE */
-	oa->count = 1;
-
 	oa->data = kmalloc_obj(struct gssx_option);
 	if (!oa->data)
 		return -ENOMEM;
 
+	oa->count = 1;
+
 	creds = kzalloc_obj(struct svc_cred);
 	if (!creds) {
 		err = -ENOMEM;
@@ -280,9 +282,14 @@ static int gssx_dec_option_array(struct xdr_stream *xdr,
 		if (length == sizeof(CREDS_VALUE) &&
 		    memcmp(p, CREDS_VALUE, sizeof(CREDS_VALUE)) == 0) {
 			/* We have creds here. parse them */
+			if (creds_decoded) {
+				err = -EINVAL;
+				goto free_creds;
+			}
 			err = gssx_dec_linux_creds(xdr, creds);
 			if (err)
 				goto free_creds;
+			creds_decoded = true;
 			oa->data[0].value.len = 1; /* presence */
 		} else {
 			/* consume uninteresting buffer */
@@ -294,8 +301,10 @@ static int gssx_dec_option_array(struct xdr_stream *xdr,
 	return 0;
 
 free_creds:
+	free_svc_cred(creds);
 	kfree(creds);
 free_oa:
+	oa->count = 0;
 	kfree(oa->data);
 	oa->data = NULL;
 	return err;
diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c
index d14209031e180..967e9d53080d7 100644
--- a/net/sunrpc/auth_gss/svcauth_gss.c
+++ b/net/sunrpc/auth_gss/svcauth_gss.c
@@ -949,6 +949,8 @@ svcauth_gss_unwrap_priv(struct svc_rqst *rqstp, u32 seq, struct gss_ctx *ctx)
 	}
 	if (len > xdr_stream_remaining(xdr))
 		goto unwrap_failed;
+	if (len <= GSS_KRB5_TOK_HDR_LEN)
+		goto unwrap_failed;
 	offset = xdr_stream_pos(xdr);
 
 	saved_len = buf->len;
@@ -1466,7 +1468,6 @@ static int create_use_gss_proxy_proc_entry(struct net *net)
 			      &use_gss_proxy_proc_ops, net);
 	if (!*p)
 		return -ENOMEM;
-	init_gssp_clnt(sn);
 	return 0;
 }
 
@@ -1573,6 +1574,9 @@ svcauth_gss_decode_credbody(struct xdr_stream *xdr,
 	u32 body_len;
 	__be32 *p;
 
+	/* Early-return paths leave deterministic state, not stale residue. */
+	memset(gc, 0, sizeof(*gc));
+
 	p = xdr_inline_decode(xdr, XDR_UNIT);
 	if (!p)
 		return false;
@@ -1942,6 +1946,8 @@ svcauth_gss_release(struct svc_rqst *rqstp)
 
 	if (!gsd)
 		goto out;
+	if (rqstp->rq_auth_stat != rpc_auth_ok)
+		goto out;
 	gc = &gsd->clcred;
 	if (gc->gc_proc != RPC_GSS_PROC_DATA)
 		goto out;
diff --git a/net/sunrpc/backchannel_rqst.c b/net/sunrpc/backchannel_rqst.c
index 0ffa4d01a9389..1482b06e0f38b 100644
--- a/net/sunrpc/backchannel_rqst.c
+++ b/net/sunrpc/backchannel_rqst.c
@@ -25,20 +25,39 @@ unsigned int xprt_bc_max_slots(struct rpc_xprt *xprt)
 }
 
 /*
- * Helper function to nullify backchannel server pointer in transport.
- * We need to synchronize setting the pointer to NULL (done so after
- * the backchannel server is shutdown) with the usage of that pointer
- * by the backchannel request processing routines
- * xprt_complete_bc_request() and rpcrdma_bc_receive_call().
+ * Close the backchannel producer side, drain any requests still
+ * queued on sv_cb_list, then destroy the callback service.
  */
 void xprt_svc_destroy_nullify_bc(struct rpc_xprt *xprt, struct svc_serv **serv)
 {
-	spin_lock(&xprt->bc_pa_lock);
+	struct svc_serv *bc_serv = *serv;
+	struct rpc_rqst *req;
+
+	xprt_svc_shutdown_bc(xprt);
+	while ((req = lwq_dequeue(&bc_serv->sv_cb_list, struct rpc_rqst,
+				  rq_bc_list)) != NULL) {
+		atomic_dec(&req->rq_xprt->bc_slot_count);
+		xprt_free_bc_request(req);
+	}
 	svc_destroy(serv);
+}
+EXPORT_SYMBOL_GPL(xprt_svc_destroy_nullify_bc);
+
+/*
+ * Clear the backchannel server pointer in the transport.  The NULL
+ * store is serialized under bc_pa_lock against readers of
+ * xprt->bc_serv in xprt_complete_bc_request() and
+ * rpcrdma_bc_receive_call().  Clearing it before the callback service
+ * is stopped prevents a producer from enqueueing onto a service that
+ * is being torn down.
+ */
+void xprt_svc_shutdown_bc(struct rpc_xprt *xprt)
+{
+	spin_lock(&xprt->bc_pa_lock);
 	xprt->bc_serv = NULL;
 	spin_unlock(&xprt->bc_pa_lock);
 }
-EXPORT_SYMBOL_GPL(xprt_svc_destroy_nullify_bc);
+EXPORT_SYMBOL_GPL(xprt_svc_shutdown_bc);
 
 /*
  * Helper routines that track the number of preallocation elements
@@ -393,7 +412,12 @@ void xprt_enqueue_bc_request(struct rpc_rqst *req)
 	if (bc_serv) {
 		lwq_enqueue(&req->rq_bc_list, &bc_serv->sv_cb_list);
 		svc_pool_wake_idle_thread(&bc_serv->sv_pools[0]);
+		spin_unlock(&xprt->bc_pa_lock);
+		return;
 	}
 	spin_unlock(&xprt->bc_pa_lock);
+
+	atomic_dec(&xprt->bc_slot_count);
+	xprt_free_bc_request(req);
 }
 EXPORT_SYMBOL_GPL(xprt_enqueue_bc_request);
diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c
index 391037f152928..1bc04109d213d 100644
--- a/net/sunrpc/cache.c
+++ b/net/sunrpc/cache.c
@@ -430,10 +430,9 @@ void sunrpc_destroy_cache_detail(struct cache_detail *cd)
 	list_del_init(&cd->others);
 	spin_unlock(&cd->hash_lock);
 	spin_unlock(&cache_list_lock);
-	if (list_empty(&cache_list)) {
-		/* module must be being unloaded so its safe to kill the worker */
-		cancel_delayed_work_sync(&cache_cleaner);
-	}
+	cancel_delayed_work_sync(&cache_cleaner);
+	if (!list_empty(&cache_list))
+		queue_delayed_work(system_power_efficient_wq, &cache_cleaner, 0);
 }
 EXPORT_SYMBOL_GPL(sunrpc_destroy_cache_detail);
 
diff --git a/net/sunrpc/sunrpc_syms.c b/net/sunrpc/sunrpc_syms.c
index ab88ce46afb55..1a3884a0376a4 100644
--- a/net/sunrpc/sunrpc_syms.c
+++ b/net/sunrpc/sunrpc_syms.c
@@ -57,6 +57,7 @@ static __net_init int sunrpc_init_net(struct net *net)
 	INIT_LIST_HEAD(&sn->all_clients);
 	spin_lock_init(&sn->rpc_client_lock);
 	spin_lock_init(&sn->rpcb_clnt_lock);
+	mutex_init(&sn->gssp_lock);
 	return 0;
 
 err_pipefs:
diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
index ae9ec4bf34f72..dd80a2eaaa742 100644
--- a/net/sunrpc/svc.c
+++ b/net/sunrpc/svc.c
@@ -476,6 +476,35 @@ __svc_init_bc(struct svc_serv *serv)
 }
 #endif
 
+static int svc_pool_init_counters(struct svc_pool *pool)
+{
+	int err;
+
+	err = percpu_counter_init(&pool->sp_messages_arrived, 0, GFP_KERNEL);
+	if (err)
+		return err;
+	err = percpu_counter_init(&pool->sp_sockets_queued, 0, GFP_KERNEL);
+	if (err)
+		goto err_sockets;
+	err = percpu_counter_init(&pool->sp_threads_woken, 0, GFP_KERNEL);
+	if (err)
+		goto err_threads;
+	return 0;
+
+err_threads:
+	percpu_counter_destroy(&pool->sp_sockets_queued);
+err_sockets:
+	percpu_counter_destroy(&pool->sp_messages_arrived);
+	return err;
+}
+
+static void svc_pool_destroy_counters(struct svc_pool *pool)
+{
+	percpu_counter_destroy(&pool->sp_messages_arrived);
+	percpu_counter_destroy(&pool->sp_sockets_queued);
+	percpu_counter_destroy(&pool->sp_threads_woken);
+}
+
 /*
  * Create an RPC service
  */
@@ -540,12 +569,18 @@ __svc_create(struct svc_program *prog, int nprogs, struct svc_stat *stats,
 		INIT_LIST_HEAD(&pool->sp_all_threads);
 		init_llist_head(&pool->sp_idle_threads);
 
-		percpu_counter_init(&pool->sp_messages_arrived, 0, GFP_KERNEL);
-		percpu_counter_init(&pool->sp_sockets_queued, 0, GFP_KERNEL);
-		percpu_counter_init(&pool->sp_threads_woken, 0, GFP_KERNEL);
+		if (svc_pool_init_counters(pool))
+			goto out_err;
 	}
 
 	return serv;
+
+out_err:
+	while (i--)
+		svc_pool_destroy_counters(&serv->sv_pools[i]);
+	kfree(serv->sv_pools);
+	kfree(serv);
+	return NULL;
 }
 
 /**
@@ -624,9 +659,7 @@ svc_destroy(struct svc_serv **servp)
 	for (i = 0; i < serv->sv_nrpools; i++) {
 		struct svc_pool *pool = &serv->sv_pools[i];
 
-		percpu_counter_destroy(&pool->sp_messages_arrived);
-		percpu_counter_destroy(&pool->sp_sockets_queued);
-		percpu_counter_destroy(&pool->sp_threads_woken);
+		svc_pool_destroy_counters(pool);
 	}
 	kfree(serv->sv_pools);
 	kfree(serv);
@@ -683,6 +716,15 @@ svc_release_buffer(struct svc_rqst *rqstp)
 	}
 }
 
+static void svc_rqst_free_rcu(struct rcu_head *head)
+{
+	struct svc_rqst *rqstp = container_of(head, struct svc_rqst, rq_rcu_head);
+
+	kfree(rqstp->rq_resp);
+	kfree(rqstp->rq_argp);
+	kfree(rqstp);
+}
+
 static void
 svc_rqst_free(struct svc_rqst *rqstp)
 {
@@ -691,10 +733,8 @@ svc_rqst_free(struct svc_rqst *rqstp)
 	svc_release_buffer(rqstp);
 	if (rqstp->rq_scratch_folio)
 		folio_put(rqstp->rq_scratch_folio);
-	kfree(rqstp->rq_resp);
-	kfree(rqstp->rq_argp);
 	kfree(rqstp->rq_auth_data);
-	kfree_rcu(rqstp, rq_rcu_head);
+	call_rcu(&rqstp->rq_rcu_head, svc_rqst_free_rcu);
 }
 
 static struct svc_rqst *
@@ -954,6 +994,29 @@ svc_set_num_threads(struct svc_serv *serv, unsigned int min_threads,
 }
 EXPORT_SYMBOL_GPL(svc_set_num_threads);
 
+/**
+ * svc_serv_maxthreads - report a service's configured thread ceiling
+ * @serv: RPC service to query
+ *
+ * A pooled service sizes its threads dynamically, so the number of
+ * threads running at any moment tracks recent load rather than the
+ * service's capacity. The per-pool maximum is the stable figure a
+ * consumer should size against.
+ *
+ * The caller must keep @serv valid for the duration of the call.
+ *
+ * Return: the sum of every pool's maximum thread count.
+ */
+unsigned int svc_serv_maxthreads(const struct svc_serv *serv)
+{
+	unsigned int i, max = 0;
+
+	for (i = 0; i < serv->sv_nrpools; i++)
+		max += data_race(serv->sv_pools[i].sp_nrthrmax);
+	return max;
+}
+EXPORT_SYMBOL_GPL(svc_serv_maxthreads);
+
 /**
  * svc_rqst_replace_page - Replace one page in rq_respages[]
  * @rqstp: svc_rqst with pages to replace
diff --git a/net/sunrpc/svcauth_unix.c b/net/sunrpc/svcauth_unix.c
index 64a2658faddbe..aebd97e7f66c7 100644
--- a/net/sunrpc/svcauth_unix.c
+++ b/net/sunrpc/svcauth_unix.c
@@ -133,7 +133,7 @@ static void ip_map_init(struct cache_head *cnew, struct cache_head *citem)
 	struct ip_map *new = container_of(cnew, struct ip_map, h);
 	struct ip_map *item = container_of(citem, struct ip_map, h);
 
-	strcpy(new->m_class, item->m_class);
+	strscpy(new->m_class, item->m_class);
 	new->m_addr = item->m_addr;
 }
 static void update(struct cache_head *cnew, struct cache_head *citem)
@@ -296,7 +296,7 @@ static struct ip_map *__ip_map_lookup(struct cache_detail *cd, char *class,
 	struct ip_map ip;
 	struct cache_head *ch;
 
-	strcpy(ip.m_class, class);
+	strscpy(ip.m_class, class);
 	ip.m_addr = *addr;
 	ch = sunrpc_cache_lookup_rcu(cd, &ip.h,
 				     hash_str(class, IP_HASHBITS) ^
diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c
index fa6a30b5f0463..cb2ef428651f6 100644
--- a/net/sunrpc/xdr.c
+++ b/net/sunrpc/xdr.c
@@ -2049,7 +2049,7 @@ void xdr_buf_trim(struct xdr_buf *buf, unsigned int len)
 		trim -= cur;
 	}
 fix_len:
-	buf->len -= (len - trim);
+	buf->len -= min_t(unsigned int, buf->len, len - trim);
 }
 EXPORT_SYMBOL_GPL(xdr_buf_trim);
 
diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c
index e5c71cf705a33..e83cef19e6569 100644
--- a/net/sunrpc/xprtrdma/frwr_ops.c
+++ b/net/sunrpc/xprtrdma/frwr_ops.c
@@ -172,8 +172,9 @@ int frwr_mr_init(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr)
 int frwr_query_device(struct rpcrdma_ep *ep, const struct ib_device *device)
 {
 	const struct ib_device_attr *attrs = &device->attrs;
-	int max_qp_wr, depth, delta;
 	unsigned int max_sge;
+	u32 max_qp_wr;
+	int depth, delta;
 
 	if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) ||
 	    attrs->max_fast_reg_page_list_len == 0) {
@@ -229,10 +230,10 @@ int frwr_query_device(struct rpcrdma_ep *ep, const struct ib_device *device)
 	}
 
 	max_qp_wr = attrs->max_qp_wr;
+	if (max_qp_wr < RPCRDMA_BACKWARD_WRS + 1 + RPCRDMA_MIN_SLOT_TABLE)
+		return -ENOMEM;
 	max_qp_wr -= RPCRDMA_BACKWARD_WRS;
 	max_qp_wr -= 1;
-	if (max_qp_wr < RPCRDMA_MIN_SLOT_TABLE)
-		return -ENOMEM;
 	if (ep->re_max_requests > max_qp_wr)
 		ep->re_max_requests = max_qp_wr;
 	ep->re_attr.cap.max_send_wr = ep->re_max_requests * depth;
diff --git a/net/sunrpc/xprtrdma/ib_client.c b/net/sunrpc/xprtrdma/ib_client.c
index de49ad02053d8..188f7a13397f2 100644
--- a/net/sunrpc/xprtrdma/ib_client.c
+++ b/net/sunrpc/xprtrdma/ib_client.c
@@ -51,7 +51,11 @@ static struct rpcrdma_device *rpcrdma_get_client_data(struct ib_device *device)
  * to be invoked when the device is removed, unless this notification
  * is unregistered first.
  *
- * On failure, a negative errno is returned.
+ * On failure, a negative errno is returned. rn->rn_done is left
+ * NULL on every failure path (it is armed before xa_alloc but
+ * cleared again if xa_alloc fails), so the @rn may safely be
+ * passed to rpcrdma_rn_unregister() without a separate
+ * registered/unregistered flag in the caller.
  */
 int rpcrdma_rn_register(struct ib_device *device,
 			struct rpcrdma_notification *rn,
@@ -62,10 +66,21 @@ int rpcrdma_rn_register(struct ib_device *device,
 	if (!rd || test_bit(RPCRDMA_RD_F_REMOVING, &rd->rd_flags))
 		return -ENETUNREACH;
 
-	if (xa_alloc(&rd->rd_xa, &rn->rn_index, rn, xa_limit_32b, GFP_KERNEL) < 0)
+	/*
+	 * Arm rn_done before xa_alloc() publishes @rn: once @rn is
+	 * visible in rd_xa, a concurrent rpcrdma_remove_one() can
+	 * call rn->rn_done(), so the pointer must already be set.
+	 *
+	 * Restore NULL if xa_alloc() fails. rn_done doubles as the
+	 * registration sentinel for rpcrdma_rn_unregister(); a stale
+	 * value would unregister an @rn that was never inserted.
+	 */
+	rn->rn_done = done;
+	if (xa_alloc(&rd->rd_xa, &rn->rn_index, rn, xa_limit_32b, GFP_KERNEL) < 0) {
+		rn->rn_done = NULL;
 		return -ENOMEM;
+	}
 	kref_get(&rd->rd_kref);
-	rn->rn_done = done;
 	trace_rpcrdma_client_register(device, rn);
 	return 0;
 }
@@ -83,6 +98,10 @@ static void rpcrdma_rn_release(struct kref *kref)
  * rpcrdma_rn_unregister - stop device removal notifications
  * @device: monitored device
  * @rn: notification object that no longer wishes to be notified
+ *
+ * It is safe to call this on an @rn whose registration never
+ * completed or failed; rn_done == NULL is treated as
+ * never-registered and the call is a no-op.
  */
 void rpcrdma_rn_unregister(struct ib_device *device,
 			   struct rpcrdma_notification *rn)
@@ -92,6 +111,21 @@ void rpcrdma_rn_unregister(struct ib_device *device,
 	if (!rd)
 		return;
 
+	/*
+	 * rn_done is the registration sentinel: rpcrdma_rn_register
+	 * leaves it NULL on every failure path, clearing it again if
+	 * xa_alloc fails, so a non-NULL rn_done marks a completed
+	 * registration. A NULL rn_done means this notification was
+	 * never registered (or its registration failed) or has
+	 * already been unregistered, and the call is a no-op.
+	 * Without this guard, rn_index == 0 from a kzalloc'd
+	 * parent would erase another caller's slot 0 and underflow
+	 * rd_kref.
+	 */
+	if (!rn->rn_done)
+		return;
+	rn->rn_done = NULL;
+
 	trace_rpcrdma_client_unregister(device, rn);
 	xa_erase(&rd->rd_xa, rn->rn_index);
 	kref_put(&rd->rd_kref, rpcrdma_rn_release);
diff --git a/net/sunrpc/xprtrdma/svc_rdma_pcl.c b/net/sunrpc/xprtrdma/svc_rdma_pcl.c
index 1f8f7dad8b6f8..8623722790f21 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_pcl.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_pcl.c
@@ -149,9 +149,6 @@ bool pcl_alloc_call(struct svc_rdma_recv_ctxt *rctxt, __be32 *p)
  *              cl_count is updated to be the number of chunks (ie.
  *              unique position values) in the Read list.
  *      %false: Memory allocation failed.
- *
- * TODO:
- * - Check for chunk range overlaps
  */
 bool pcl_alloc_read(struct svc_rdma_recv_ctxt *rctxt, __be32 *p)
 {
@@ -213,7 +210,6 @@ bool pcl_alloc_write(struct svc_rdma_recv_ctxt *rctxt,
 		chunk = pcl_alloc_chunk(segcount, 0);
 		if (!chunk)
 			return false;
-		list_add_tail(&chunk->ch_list, &pcl->cl_chunks);
 
 		for (j = 0; j < segcount; j++) {
 			segment = &chunk->ch_segments[j];
@@ -225,7 +221,66 @@ bool pcl_alloc_write(struct svc_rdma_recv_ctxt *rctxt,
 			chunk->ch_length += segment->rs_length;
 			chunk->ch_segcount++;
 		}
+		list_add_tail(&chunk->ch_list, &pcl->cl_chunks);
+	}
+	return true;
+}
+
+/**
+ * pcl_check_read_chunk_positions - Validate Read chunk positions
+ * @rctxt: Ingress receive context with populated chunk lists
+ * @inline_len: Length of the inline RPC body after the transport header
+ *
+ * Read chunk positions are offsets in the unreduced XDR stream
+ * (RFC 8166 Section 3.4.4), so each position includes the
+ * cumulative length of preceding Read chunks. This function
+ * subtracts those lengths to recover the inline-body offset
+ * before comparing against @inline_len or the Call chunk length.
+ *
+ * Rejects frames where a Read chunk's inline-body offset exceeds
+ * the bound, where adjacent Read chunks overlap, or where any
+ * single chunk length exceeds the page budget.
+ *
+ * Return values:
+ *       %true: Read chunk positions and lengths are valid
+ *      %false: Malformed chunk list detected
+ */
+bool pcl_check_read_chunk_positions(struct svc_rdma_recv_ctxt *rctxt,
+				    unsigned int inline_len)
+{
+	unsigned int max_len, bound, total_read;
+	struct svc_rdma_chunk *chunk, *next;
+
+	max_len = rctxt->rc_maxpages << PAGE_SHIFT;
+
+	if (!pcl_is_empty(&rctxt->rc_call_pcl)) {
+		chunk = pcl_first_chunk(&rctxt->rc_call_pcl);
+		if (chunk->ch_length > max_len)
+			return false;
+		bound = chunk->ch_length;
+	} else {
+		bound = inline_len;
+	}
+
+	if (pcl_is_empty(&rctxt->rc_read_pcl))
+		return true;
+
+	total_read = 0;
+	pcl_for_each_chunk(chunk, &rctxt->rc_read_pcl) {
+		if (chunk->ch_position - total_read > bound)
+			return false;
+		if (chunk->ch_length > max_len)
+			return false;
+
+		next = pcl_next_chunk(&rctxt->rc_read_pcl, chunk);
+		if (!next)
+			break;
+
+		if (chunk->ch_position + chunk->ch_length > next->ch_position)
+			return false;
+		total_read += chunk->ch_length;
 	}
+
 	return true;
 }
 
diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
index fe9bf0371b6ec..fdfed1be97da9 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
@@ -440,11 +440,14 @@ static void svc_rdma_build_arg_xdr(struct svc_rqst *rqstp,
  *	    to the first byte past the Read list. rc_read_pcl and
  *	    rc_call_pcl cl_count fields are set to the number of
  *	    Read segments in the list.
- *  %false: Read list is corrupt. @rctxt's xdr_stream is left in an
- *	    unknown state.
+ *  %false: Read list is corrupt or exceeds the page budget. @rctxt's
+ *	    xdr_stream is left in an unknown state.
  */
 static bool xdr_count_read_segments(struct svc_rdma_recv_ctxt *rctxt, __be32 *p)
 {
+	unsigned int maxlen = rctxt->rc_maxpages << PAGE_SHIFT;
+	unsigned int total_len = 0;
+
 	rctxt->rc_call_pcl.cl_count = 0;
 	rctxt->rc_read_pcl.cl_count = 0;
 	while (xdr_item_is_present(p)) {
@@ -458,6 +461,11 @@ static bool xdr_count_read_segments(struct svc_rdma_recv_ctxt *rctxt, __be32 *p)
 
 		xdr_decode_read_segment(p, &position, &handle,
 					    &length, &offset);
+		if (length > maxlen)
+			return false;
+		total_len += length;
+		if (PAGE_ALIGN(total_len) > maxlen)
+			return false;
 		if (position) {
 			if (position & 3)
 				return false;
@@ -508,10 +516,13 @@ static bool xdr_check_write_chunk(struct svc_rdma_recv_ctxt *rctxt)
 		return false;
 
 	/* Before trusting the segcount value enough to use it in
-	 * a computation, perform a simple range check. This is an
-	 * arbitrary but sensible limit (ie, not architectural).
+	 * a computation, perform a simple range check. A zero
+	 * segcount describes no remote buffer and is rejected so
+	 * downstream consumers never see a degenerate ch_segcount==0
+	 * chunk. The upper bound is an arbitrary but sensible limit
+	 * (ie, not architectural).
 	 */
-	if (unlikely(segcount > rctxt->rc_maxpages))
+	if (segcount == 0 || unlikely(segcount > rctxt->rc_maxpages))
 		return false;
 
 	p = xdr_inline_decode(&rctxt->rc_stream,
@@ -719,6 +730,9 @@ static int svc_rdma_xdr_decode_req(struct xdr_buf *rq_arg,
 
 	rq_arg->head[0].iov_base = rctxt->rc_stream.p;
 	hdr_len = xdr_stream_pos(&rctxt->rc_stream);
+	if (!pcl_check_read_chunk_positions(rctxt,
+					    rq_arg->head[0].iov_len - hdr_len))
+		goto out_inval;
 	rq_arg->head[0].iov_len -= hdr_len;
 	rq_arg->len -= hdr_len;
 	trace_svcrdma_decode_rqst(rctxt, rdma_argp, hdr_len);
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index f7fd22cc4a598..9aaaade99e6e4 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -795,7 +795,7 @@ static int svc_rdma_build_read_segment(struct svc_rqst *rqstp,
 		len -= seg_len;
 
 		if (len && ((head->rc_curpage + 1) > rqstp->rq_maxpages))
-			goto out_overrun;
+			goto out_put;
 	}
 
 	ret = svc_rdma_rw_ctx_init(rdma, ctxt, segment->rs_offset,
@@ -809,7 +809,8 @@ static int svc_rdma_build_read_segment(struct svc_rqst *rqstp,
 	cc->cc_sqecount += ret;
 	return 0;
 
-out_overrun:
+out_put:
+	svc_rdma_put_rw_ctxt(rdma, ctxt);
 	trace_svcrdma_page_overrun_err(&cc->cc_cid, head->rc_curpage);
 	return -EINVAL;
 }
@@ -847,7 +848,7 @@ static int svc_rdma_build_read_chunk(struct svc_rqst *rqstp,
  * svc_rdma_copy_inline_range - Copy part of the inline content into pages
  * @rqstp: RPC transaction context
  * @head: context for ongoing I/O
- * @offset: offset into the Receive buffer of region to copy
+ * @offset: offset into the inline content of region to copy
  * @remaining: length of region to copy
  *
  * Take a page at a time from rqstp->rq_pages and copy the inline
@@ -864,9 +865,13 @@ static int svc_rdma_copy_inline_range(struct svc_rqst *rqstp,
 				      unsigned int offset,
 				      unsigned int remaining)
 {
-	unsigned char *dst, *src = head->rc_recv_buf;
+	unsigned char *dst, *src = head->rc_saved_arg.head[0].iov_base;
+	unsigned int inline_len = head->rc_saved_arg.head[0].iov_len;
 	unsigned int page_no, numpages;
 
+	if (offset > inline_len || remaining > inline_len - offset)
+		return -EINVAL;
+
 	numpages = PAGE_ALIGN(head->rc_pageoff + remaining) >> PAGE_SHIFT;
 	for (page_no = 0; page_no < numpages; page_no++) {
 		unsigned int page_len;
@@ -917,9 +922,10 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp,
 {
 	const struct svc_rdma_pcl *pcl = &head->rc_read_pcl;
 	struct svc_rdma_chunk *chunk, *next;
-	unsigned int start, length;
+	unsigned int inline_len, start, length;
 	int ret;
 
+	inline_len = head->rc_saved_arg.head[0].iov_len;
 	start = 0;
 	chunk = pcl_first_chunk(pcl);
 	length = chunk->ch_position;
@@ -937,6 +943,8 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp,
 			break;
 
 		start += length;
+		if (head->rc_readbytes > next->ch_position)
+			return -EINVAL;
 		length = next->ch_position - head->rc_readbytes;
 		ret = svc_rdma_copy_inline_range(rqstp, head, start, length);
 		if (ret < 0)
@@ -944,7 +952,9 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp,
 	}
 
 	start += length;
-	length = head->rc_byte_len - start;
+	if (start > inline_len)
+		return -EINVAL;
+	length = inline_len - start;
 	return svc_rdma_copy_inline_range(rqstp, head, start, length);
 }
 
@@ -969,8 +979,12 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp,
 static int svc_rdma_read_data_item(struct svc_rqst *rqstp,
 				   struct svc_rdma_recv_ctxt *head)
 {
-	return svc_rdma_build_read_chunk(rqstp, head,
-					 pcl_first_chunk(&head->rc_read_pcl));
+	struct svc_rdma_chunk *chunk = pcl_first_chunk(&head->rc_read_pcl);
+
+	if (chunk->ch_position > head->rc_saved_arg.head[0].iov_len)
+		return -EINVAL;
+
+	return svc_rdma_build_read_chunk(rqstp, head, chunk);
 }
 
 /**
@@ -996,17 +1010,20 @@ static int svc_rdma_read_chunk_range(struct svc_rqst *rqstp,
 	const struct svc_rdma_segment *segment;
 	int ret;
 
+	if (!length)
+		return 0;
+
 	ret = -EINVAL;
 	pcl_for_each_segment(segment, chunk) {
 		struct svc_rdma_segment dummy;
 
-		if (offset > segment->rs_length) {
+		if (offset >= segment->rs_length) {
 			offset -= segment->rs_length;
 			continue;
 		}
 
 		dummy.rs_handle = segment->rs_handle;
-		dummy.rs_length = min_t(u32, length, segment->rs_length) - offset;
+		dummy.rs_length = min_t(u32, length, segment->rs_length - offset);
 		dummy.rs_offset = segment->rs_offset + offset;
 
 		ret = svc_rdma_build_read_segment(rqstp, head, &dummy);
@@ -1015,6 +1032,8 @@ static int svc_rdma_read_chunk_range(struct svc_rqst *rqstp,
 
 		head->rc_readbytes += dummy.rs_length;
 		length -= dummy.rs_length;
+		if (!length)
+			break;
 		offset = 0;
 	}
 	return ret;
@@ -1039,14 +1058,17 @@ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp,
 			pcl_first_chunk(&head->rc_call_pcl);
 	const struct svc_rdma_pcl *pcl = &head->rc_read_pcl;
 	struct svc_rdma_chunk *chunk, *next;
-	unsigned int start, length;
+	unsigned int call_len, start, length;
 	int ret;
 
 	if (pcl_is_empty(pcl))
 		return svc_rdma_build_read_chunk(rqstp, head, call_chunk);
 
+	call_len = call_chunk->ch_length;
 	start = 0;
 	chunk = pcl_first_chunk(pcl);
+	if (chunk->ch_position > call_len)
+		return -EINVAL;
 	length = chunk->ch_position;
 	ret = svc_rdma_read_chunk_range(rqstp, head, call_chunk,
 					start, length);
@@ -1063,6 +1085,10 @@ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp,
 			break;
 
 		start += length;
+		if (next->ch_position > call_len)
+			return -EINVAL;
+		if (head->rc_readbytes > next->ch_position)
+			return -EINVAL;
 		length = next->ch_position - head->rc_readbytes;
 		ret = svc_rdma_read_chunk_range(rqstp, head, call_chunk,
 						start, length);
@@ -1071,7 +1097,9 @@ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp,
 	}
 
 	start += length;
-	length = call_chunk->ch_length - start;
+	if (start > call_len)
+		return -EINVAL;
+	length = call_len - start;
 	return svc_rdma_read_chunk_range(rqstp, head, call_chunk,
 					 start, length);
 }
diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c
index 7ca71741106b1..927269598ac24 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_transport.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c
@@ -43,6 +43,7 @@
  */
 
 #include <linux/interrupt.h>
+#include <linux/module.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
@@ -245,12 +246,16 @@ svc_rdma_parse_connect_private(struct svcxprt_rdma *newxprt,
  * structure for the listening endpoint.
  *
  * This function creates a new xprt for the new connection and enqueues it on
- * the accept queue for the listent xprt. When the listen thread is kicked, it
+ * the accept queue for the listen xprt. When the listen thread is kicked, it
  * will call the recvfrom method on the listen xprt which will accept the new
  * connection.
+ *
+ * Return values:
+ *     %0: Do not destroy @new_cma_id
+ *     %1: Destroy @new_cma_id (allocation failure)
  */
-static void handle_connect_req(struct rdma_cm_id *new_cma_id,
-			       struct rdma_conn_param *param)
+static int handle_connect_req(struct rdma_cm_id *new_cma_id,
+			      struct rdma_conn_param *param)
 {
 	struct svcxprt_rdma *listen_xprt = new_cma_id->context;
 	struct svcxprt_rdma *newxprt;
@@ -260,7 +265,7 @@ static void handle_connect_req(struct rdma_cm_id *new_cma_id,
 				       listen_xprt->sc_xprt.xpt_net,
 				       ibdev_to_node(new_cma_id->device));
 	if (!newxprt)
-		return;
+		return 1;
 	newxprt->sc_cm_id = new_cma_id;
 	new_cma_id->context = newxprt;
 	svc_rdma_parse_connect_private(newxprt, param);
@@ -294,6 +299,7 @@ static void handle_connect_req(struct rdma_cm_id *new_cma_id,
 
 	set_bit(XPT_CONN, &listen_xprt->sc_xprt.xpt_flags);
 	svc_xprt_enqueue(&listen_xprt->sc_xprt);
+	return 0;
 }
 
 /**
@@ -317,14 +323,14 @@ static int svc_rdma_listen_handler(struct rdma_cm_id *cma_id,
 
 	switch (event->event) {
 	case RDMA_CM_EVENT_CONNECT_REQUEST:
-		handle_connect_req(cma_id, &event->param.conn);
-		break;
+		return handle_connect_req(cma_id, &event->param.conn);
 	case RDMA_CM_EVENT_ADDR_CHANGE:
 		listen_id = svc_rdma_create_listen_id(cma_rdma->xpt_net,
 						      sap, cma_xprt);
 		if (IS_ERR(listen_id)) {
 			pr_err("Listener dead, address change failed for device %s\n",
 				cma_id->device->name);
+			cma_xprt->sc_cm_id = NULL;
 		} else
 			cma_xprt->sc_cm_id = listen_id;
 		return 1;
@@ -387,7 +393,13 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv,
 
 	listen_id = svc_rdma_create_listen_id(net, sa, cma_xprt);
 	if (IS_ERR(listen_id)) {
-		kfree(cma_xprt);
+		/* _svc_xprt_create() acquired one module reference and
+		 * puts it on xpo_create failure.  svc_xprt_free() puts
+		 * a second one when the kref drops to zero.  Take a
+		 * compensating reference so both puts are balanced.
+		 */
+		__module_get(cma_xprt->sc_xprt.xpt_class->xcl_owner);
+		svc_xprt_put(&cma_xprt->sc_xprt);
 		return ERR_CAST(listen_id);
 	}
 	cma_xprt->sc_cm_id = listen_id;
@@ -562,8 +574,7 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt)
 	set_bit(RDMAXPRT_CONN_PENDING, &newxprt->sc_flags);
 	memset(&conn_param, 0, sizeof conn_param);
 	conn_param.responder_resources = 0;
-	conn_param.initiator_depth = min_t(int, newxprt->sc_ord,
-					   dev->attrs.max_qp_init_rd_atom);
+	conn_param.initiator_depth = min(newxprt->sc_ord, dev->attrs.max_qp_init_rd_atom);
 	if (!conn_param.initiator_depth) {
 		ret = -EINVAL;
 		trace_svcrdma_initdepth_err(newxprt, ret);
@@ -588,7 +599,7 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt)
 		dprintk("    local address   : %pIS:%u\n", sap, rpc_get_port(sap));
 		sap = (struct sockaddr *)&newxprt->sc_cm_id->route.addr.dst_addr;
 		dprintk("    remote address  : %pIS:%u\n", sap, rpc_get_port(sap));
-		dprintk("    max_sge         : %d\n", newxprt->sc_max_send_sges);
+		dprintk("    max_sge         : %u\n", newxprt->sc_max_send_sges);
 		dprintk("    sq_depth        : %d\n", newxprt->sc_sq_depth);
 		dprintk("    rdma_rw_ctxs    : %d\n", ctxts);
 		dprintk("    max_requests    : %d\n", newxprt->sc_max_requests);
@@ -598,13 +609,26 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt)
 	return &newxprt->sc_xprt;
 
  errout:
-	/* Take a reference in case the DTO handler runs */
-	svc_xprt_get(&newxprt->sc_xprt);
-	if (newxprt->sc_qp && !IS_ERR(newxprt->sc_qp))
-		ib_destroy_qp(newxprt->sc_qp);
-	rdma_destroy_id(newxprt->sc_cm_id);
-	rpcrdma_rn_unregister(dev, &newxprt->sc_rn);
-	/* This call to put will destroy the transport */
+	/*
+	 * Drop the kref_init birth reference. svc_xprt_free will
+	 * dispatch xpo_free = svc_rdma_free, which tears down sc_qp,
+	 * sc_sq_cq, sc_rq_cq, and sc_pd under existing IS_ERR/NULL
+	 * guards, and sc_rn under the rn_done sentinel guard inside
+	 * rpcrdma_rn_unregister.
+	 *
+	 * sc_cm_id is destroyed unconditionally by svc_rdma_free; that
+	 * is safe here because sc_cm_id is non-NULL by caller invariant
+	 * on every path that reaches this errout: handle_connect_req
+	 * installs newxprt->sc_cm_id before queueing the new xprt for
+	 * accept, and svc_rdma_accept has already dereferenced it above
+	 * the first goto errout.
+	 *
+	 * svc_handle_xprt() drops its pre-acquired module reference when
+	 * ->xpo_accept() returns NULL. Take a replacement reference before
+	 * freeing @newxprt, because svc_xprt_free() drops the module
+	 * reference associated with @newxprt.
+	 */
+	__module_get(newxprt->sc_xprt.xpt_class->xcl_owner);
 	svc_xprt_put(&newxprt->sc_xprt);
 	return NULL;
 }
@@ -614,7 +638,8 @@ static void svc_rdma_detach(struct svc_xprt *xprt)
 	struct svcxprt_rdma *rdma =
 		container_of(xprt, struct svcxprt_rdma, sc_xprt);
 
-	rdma_disconnect(rdma->sc_cm_id);
+	if (rdma->sc_cm_id)
+		rdma_disconnect(rdma->sc_cm_id);
 
 	/*
 	 * Most close paths go through svc_rdma_xprt_deferred_close(),
@@ -634,10 +659,15 @@ static void svc_rdma_free(struct svc_xprt *xprt)
 {
 	struct svcxprt_rdma *rdma =
 		container_of(xprt, struct svcxprt_rdma, sc_xprt);
-	struct ib_device *device = rdma->sc_cm_id->device;
+	struct ib_device *device;
 
 	might_sleep();
 
+	if (!rdma->sc_cm_id)
+		goto out_free;
+
+	device = rdma->sc_cm_id->device;
+
 	/* This blocks until the Completion Queues are empty */
 	if (rdma->sc_qp && !IS_ERR(rdma->sc_qp))
 		ib_drain_qp(rdma->sc_qp);
@@ -662,11 +692,13 @@ static void svc_rdma_free(struct svc_xprt *xprt)
 	if (rdma->sc_pd && !IS_ERR(rdma->sc_pd))
 		ib_dealloc_pd(rdma->sc_pd);
 
+	if (!test_bit(XPT_LISTENER, &rdma->sc_xprt.xpt_flags))
+		rpcrdma_rn_unregister(device, &rdma->sc_rn);
+
 	/* Destroy the CM ID */
 	rdma_destroy_id(rdma->sc_cm_id);
 
-	if (!test_bit(XPT_LISTENER, &rdma->sc_xprt.xpt_flags))
-		rpcrdma_rn_unregister(device, &rdma->sc_rn);
+out_free:
 	kfree(rdma);
 }
 
diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index 04b286223b24c..be335eed329d1 100644
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -465,7 +465,7 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt)
 	/* Client offers RDMA Read but does not initiate */
 	ep->re_remote_cma.initiator_depth = 0;
 	ep->re_remote_cma.responder_resources =
-		min_t(int, U8_MAX, device->attrs.max_qp_rd_atom);
+		min(device->attrs.max_qp_rd_atom, U8_MAX);
 
 	/* Limit transport retries so client can detect server
 	 * GID changes quickly. RPC layer handles re-establishing
diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c
index f7a9d55eee8a1..92a768cc9ecf9 100644
--- a/net/unix/af_unix.c
+++ b/net/unix/af_unix.c
@@ -1196,17 +1196,12 @@ static struct sock *unix_find_bsd(struct sockaddr_un *sunaddr, int addr_len,
 	unix_mkname_bsd(sunaddr, addr_len);
 
 	if (flags & SOCK_COREDUMP) {
-		struct path root;
-
-		task_lock(&init_task);
-		get_fs_root(init_task.fs, &root);
-		task_unlock(&init_task);
-
-		scoped_with_kernel_creds()
-			err = vfs_path_lookup(root.dentry, root.mnt, sunaddr->sun_path,
-					      LOOKUP_BENEATH | LOOKUP_NO_SYMLINKS |
-					      LOOKUP_NO_MAGICLINKS, &path);
-		path_put(&root);
+		scoped_with_init_fs() {
+			scoped_with_kernel_creds()
+				err = kern_path(sunaddr->sun_path,
+						LOOKUP_NO_SYMLINKS |
+						LOOKUP_NO_MAGICLINKS, &path);
+		}
 		if (err)
 			goto fail;
 	} else {
diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c
index 09475007165b3..41c2a0b82a8e8 100644
--- a/net/vmw_vsock/virtio_transport_common.c
+++ b/net/vmw_vsock/virtio_transport_common.c
@@ -328,38 +328,35 @@ static int virtio_transport_send_pkt_info(struct vsock_sock *vsk,
 	if (pkt_len == 0 && info->op == VIRTIO_VSOCK_OP_RW)
 		return pkt_len;
 
-	if (info->msg) {
-		/* If zerocopy is not enabled by 'setsockopt()', we behave as
-		 * there is no MSG_ZEROCOPY flag set.
+	if (info->msg && (info->msg->msg_flags & MSG_ZEROCOPY)) {
+		/* If 'info->msg' is not NULL, this is only VIRTIO_VSOCK_OP_RW.
+		 * 'MSG_ZEROCOPY' flag handling here is based on the same flag
+		 * handling from 'tcp_sendmsg_locked()'.
 		 */
-		if (!sock_flag(sk_vsock(vsk), SOCK_ZEROCOPY))
-			info->msg->msg_flags &= ~MSG_ZEROCOPY;
+		if (info->msg->msg_ubuf) {
+			uarg = info->msg->msg_ubuf;
+			can_zcopy = virtio_transport_can_zcopy(t_ops, info, pkt_len);
+		} else if (sock_flag(sk_vsock(vsk), SOCK_ZEROCOPY)) {
+			uarg = msg_zerocopy_realloc(sk_vsock(vsk), pkt_len,
+						    NULL, false);
+			if (!uarg) {
+				virtio_transport_put_credit(vvs, pkt_len);
+				return -ENOMEM;
+			}
 
-		if (info->msg->msg_flags & MSG_ZEROCOPY)
 			can_zcopy = virtio_transport_can_zcopy(t_ops, info, pkt_len);
+			if (!can_zcopy)
+				uarg_to_msgzc(uarg)->zerocopy = 0;
 
+			have_uref = true;
+		}
+
+		/* 'can_zcopy' means that this transmission will be
+		 * in zerocopy way (e.g. using 'frags' array).
+		 */
 		if (can_zcopy)
 			max_skb_len = min_t(u32, VIRTIO_VSOCK_MAX_PKT_BUF_SIZE,
 					    (MAX_SKB_FRAGS * PAGE_SIZE));
-
-		if (info->msg->msg_flags & MSG_ZEROCOPY &&
-		    info->op == VIRTIO_VSOCK_OP_RW) {
-			uarg = info->msg->msg_ubuf;
-
-			if (!uarg) {
-				uarg = msg_zerocopy_realloc(sk_vsock(vsk),
-							    pkt_len, NULL, false);
-				if (!uarg) {
-					virtio_transport_put_credit(vvs, pkt_len);
-					return -ENOMEM;
-				}
-
-				if (!can_zcopy)
-					uarg_to_msgzc(uarg)->zerocopy = 0;
-
-				have_uref = true;
-			}
-		}
 	}
 
 	rest_len = pkt_len;
diff --git a/net/wireless/core.c b/net/wireless/core.c
index 3dcf63b04c41d..610238d723fff 100644
--- a/net/wireless/core.c
+++ b/net/wireless/core.c
@@ -1335,6 +1335,7 @@ void wiphy_unregister(struct wiphy *wiphy)
 	/* this has nothing to do now but make sure it's gone */
 	cancel_work_sync(&rdev->wiphy_work);
 
+	cancel_work_sync(&rdev->sched_scan_res_wk);
 	cancel_work_sync(&rdev->rfkill_block);
 	cancel_work_sync(&rdev->conn_work);
 	flush_work(&rdev->event_work);
@@ -1424,6 +1425,7 @@ static void _cfg80211_unregister_wdev(struct wireless_dev *wdev,
 	list_del_rcu(&wdev->list);
 	synchronize_net();
 	rdev->devlist_generation++;
+	wiphy_work_cancel(wdev->wiphy, &wdev->disconnect_wk);
 
 	cfg80211_mlme_purge_registrations(wdev);
 
@@ -1613,7 +1615,7 @@ void cfg80211_init_wdev(struct wireless_dev *wdev)
 	INIT_LIST_HEAD(&wdev->mgmt_registrations);
 	INIT_LIST_HEAD(&wdev->pmsr_list);
 	spin_lock_init(&wdev->pmsr_lock);
-	INIT_WORK(&wdev->pmsr_free_wk, cfg80211_pmsr_free_wk);
+	wiphy_work_init(&wdev->pmsr_free_wk, cfg80211_pmsr_free_wk);
 
 #ifdef CONFIG_CFG80211_WEXT
 	wdev->wext.default_key = -1;
@@ -1637,7 +1639,7 @@ void cfg80211_init_wdev(struct wireless_dev *wdev)
 	     wdev->iftype == NL80211_IFTYPE_ADHOC) && !wdev->use_4addr)
 		wdev->netdev->priv_flags |= IFF_DONT_BRIDGE;
 
-	INIT_WORK(&wdev->disconnect_wk, cfg80211_autodisconnect_wk);
+	wiphy_work_init(&wdev->disconnect_wk, cfg80211_autodisconnect_wk);
 }
 
 void cfg80211_register_wdev(struct cfg80211_registered_device *rdev,
@@ -1743,11 +1745,11 @@ static int cfg80211_netdev_notifier_call(struct notifier_block *nb,
 		break;
 	case NETDEV_GOING_DOWN:
 		cfg80211_leave(rdev, wdev, -1);
-		scoped_guard(wiphy, &rdev->wiphy)
+		scoped_guard(wiphy, &rdev->wiphy) {
 			cfg80211_remove_links(wdev);
-		/* since we just did cfg80211_leave() nothing to do there */
-		cancel_work_sync(&wdev->disconnect_wk);
-		cancel_work_sync(&wdev->pmsr_free_wk);
+			/* since we just did cfg80211_leave() nothing to do there */
+			wiphy_work_cancel(wdev->wiphy, &wdev->disconnect_wk);
+		}
 		break;
 	case NETDEV_DOWN:
 		wiphy_lock(&rdev->wiphy);
diff --git a/net/wireless/core.h b/net/wireless/core.h
index df47ed6208a50..ac6ce9f967ec7 100644
--- a/net/wireless/core.h
+++ b/net/wireless/core.h
@@ -428,7 +428,7 @@ void __cfg80211_port_authorized(struct wireless_dev *wdev, const u8 *peer_addr,
 				const u8 *td_bitmap, u8 td_bitmap_len);
 int cfg80211_mgd_wext_connect(struct cfg80211_registered_device *rdev,
 			      struct wireless_dev *wdev);
-void cfg80211_autodisconnect_wk(struct work_struct *work);
+void cfg80211_autodisconnect_wk(struct wiphy *wiphy, struct wiphy_work *work);
 
 /* SME implementation */
 void cfg80211_conn_work(struct work_struct *work);
@@ -586,7 +586,7 @@ cfg80211_get_6ghz_power_type(const u8 *elems, size_t elems_len,
 
 void cfg80211_release_pmsr(struct wireless_dev *wdev, u32 portid);
 void cfg80211_pmsr_wdev_down(struct wireless_dev *wdev);
-void cfg80211_pmsr_free_wk(struct work_struct *work);
+void cfg80211_pmsr_free_wk(struct wiphy *wiphy, struct wiphy_work *work);
 
 void cfg80211_remove_link(struct wireless_dev *wdev, unsigned int link_id);
 void cfg80211_remove_links(struct wireless_dev *wdev);
diff --git a/net/wireless/mlme.c b/net/wireless/mlme.c
index 2a2c173058bac..7824b7ac2770f 100644
--- a/net/wireless/mlme.c
+++ b/net/wireless/mlme.c
@@ -32,15 +32,11 @@ void cfg80211_rx_assoc_resp(struct net_device *dev,
 		.timeout_reason = NL80211_TIMEOUT_UNSPECIFIED,
 		.req_ie = data->req_ies,
 		.req_ie_len = data->req_ies_len,
-		.resp_ie = mgmt->u.assoc_resp.variable,
-		.resp_ie_len = data->len -
-			       offsetof(struct ieee80211_mgmt,
-					u.assoc_resp.variable),
-		.status = le16_to_cpu(mgmt->u.assoc_resp.status_code),
 		.ap_mld_addr = data->ap_mld_addr,
 		.assoc_encrypted = data->assoc_encrypted,
 	};
 	unsigned int link_id;
+	bool is_s1g = false;
 
 	for (link_id = 0; link_id < ARRAY_SIZE(data->links); link_id++) {
 		cr.links[link_id].status = data->links[link_id].status;
@@ -61,16 +57,32 @@ void cfg80211_rx_assoc_resp(struct net_device *dev,
 
 		if (cr.links[link_id].bss->channel->band == NL80211_BAND_S1GHZ) {
 			WARN_ON(link_id);
-			cr.resp_ie = (u8 *)&mgmt->u.s1g_assoc_resp.variable;
-			cr.resp_ie_len = data->len -
-					 offsetof(struct ieee80211_mgmt,
-						  u.s1g_assoc_resp.variable);
+			is_s1g = true;
 		}
 
 		if (cr.ap_mld_addr)
 			cr.valid_links |= BIT(link_id);
 	}
 
+	if (is_s1g) {
+		if (data->len < offsetof(struct ieee80211_mgmt,
+					 u.s1g_assoc_resp.variable))
+			goto free_bss;
+		cr.resp_ie = (u8 *)&mgmt->u.s1g_assoc_resp.variable;
+		cr.resp_ie_len = data->len -
+				 offsetof(struct ieee80211_mgmt,
+					  u.s1g_assoc_resp.variable);
+	} else {
+		if (data->len < offsetof(struct ieee80211_mgmt,
+					 u.assoc_resp.variable))
+			goto free_bss;
+		cr.resp_ie = mgmt->u.assoc_resp.variable;
+		cr.resp_ie_len = data->len -
+				 offsetof(struct ieee80211_mgmt,
+					  u.assoc_resp.variable);
+	}
+	cr.status = le16_to_cpu(mgmt->u.assoc_resp.status_code);
+
 	trace_cfg80211_send_rx_assoc(dev, data);
 
 	/*
@@ -79,22 +91,24 @@ void cfg80211_rx_assoc_resp(struct net_device *dev,
 	 * and got a reject -- we only try again with an assoc
 	 * frame instead of reassoc.
 	 */
-	if (cfg80211_sme_rx_assoc_resp(wdev, cr.status)) {
-		for (link_id = 0; link_id < ARRAY_SIZE(data->links); link_id++) {
-			struct cfg80211_bss *bss = data->links[link_id].bss;
-
-			if (!bss)
-				continue;
-
-			cfg80211_unhold_bss(bss_from_pub(bss));
-			cfg80211_put_bss(wiphy, bss);
-		}
-		return;
-	}
+	if (cfg80211_sme_rx_assoc_resp(wdev, cr.status))
+		goto free_bss;
 
 	nl80211_send_rx_assoc(rdev, dev, data);
 	/* update current_bss etc., consumes the bss reference */
 	__cfg80211_connect_result(dev, &cr, cr.status == WLAN_STATUS_SUCCESS);
+	return;
+
+free_bss:
+	for (link_id = 0; link_id < ARRAY_SIZE(data->links); link_id++) {
+		struct cfg80211_bss *bss = data->links[link_id].bss;
+
+		if (!bss)
+			continue;
+
+		cfg80211_unhold_bss(bss_from_pub(bss));
+		cfg80211_put_bss(wiphy, bss);
+	}
 }
 EXPORT_SYMBOL(cfg80211_rx_assoc_resp);
 
@@ -151,19 +165,35 @@ void cfg80211_rx_mlme_mgmt(struct net_device *dev, const u8 *buf, size_t len)
 {
 	struct wireless_dev *wdev = dev->ieee80211_ptr;
 	struct ieee80211_mgmt *mgmt = (void *)buf;
+	__le16 fc;
 
 	lockdep_assert_wiphy(wdev->wiphy);
 
-	trace_cfg80211_rx_mlme_mgmt(dev, buf, len);
+	if (len < sizeof(fc))
+		return;
+
+	fc = mgmt->frame_control;
 
-	if (WARN_ON(len < 2))
+	if (ieee80211_is_auth(fc)) {
+		if (len < offsetofend(struct ieee80211_mgmt, u.auth.status_code))
+			return;
+	} else if (ieee80211_is_deauth(fc)) {
+		if (len < offsetofend(struct ieee80211_mgmt, u.deauth.reason_code))
+			return;
+	} else if (ieee80211_is_disassoc(fc)) {
+		if (len < offsetofend(struct ieee80211_mgmt, u.disassoc.reason_code))
+			return;
+	} else {
 		return;
+	}
+
+	trace_cfg80211_rx_mlme_mgmt(dev, buf, len);
 
-	if (ieee80211_is_auth(mgmt->frame_control))
+	if (ieee80211_is_auth(fc))
 		cfg80211_process_auth(wdev, buf, len);
-	else if (ieee80211_is_deauth(mgmt->frame_control))
+	else if (ieee80211_is_deauth(fc))
 		cfg80211_process_deauth(wdev, buf, len, false);
-	else if (ieee80211_is_disassoc(mgmt->frame_control))
+	else
 		cfg80211_process_disassoc(wdev, buf, len, false);
 }
 EXPORT_SYMBOL(cfg80211_rx_mlme_mgmt);
@@ -216,15 +246,28 @@ void cfg80211_tx_mlme_mgmt(struct net_device *dev, const u8 *buf, size_t len,
 {
 	struct wireless_dev *wdev = dev->ieee80211_ptr;
 	struct ieee80211_mgmt *mgmt = (void *)buf;
+	__le16 fc;
 
 	lockdep_assert_wiphy(wdev->wiphy);
 
-	trace_cfg80211_tx_mlme_mgmt(dev, buf, len, reconnect);
+	if (len < sizeof(fc))
+		return;
 
-	if (WARN_ON(len < 2))
+	fc = mgmt->frame_control;
+
+	if (ieee80211_is_deauth(fc)) {
+		if (len < offsetofend(struct ieee80211_mgmt, u.deauth.reason_code))
+			return;
+	} else if (ieee80211_is_disassoc(fc)) {
+		if (len < offsetofend(struct ieee80211_mgmt, u.disassoc.reason_code))
+			return;
+	} else {
 		return;
+	}
+
+	trace_cfg80211_tx_mlme_mgmt(dev, buf, len, reconnect);
 
-	if (ieee80211_is_deauth(mgmt->frame_control))
+	if (ieee80211_is_deauth(fc))
 		cfg80211_process_deauth(wdev, buf, len, reconnect);
 	else
 		cfg80211_process_disassoc(wdev, buf, len, reconnect);
diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c
index 53b4b3f766978..6d7d1405f4690 100644
--- a/net/wireless/nl80211.c
+++ b/net/wireless/nl80211.c
@@ -461,7 +461,9 @@ nl80211_ftm_responder_policy[NL80211_FTM_RESP_ATTR_MAX + 1] = {
 static const struct nla_policy
 nl80211_pmsr_ftm_req_attr_policy[NL80211_PMSR_FTM_REQ_ATTR_MAX + 1] = {
 	[NL80211_PMSR_FTM_REQ_ATTR_ASAP] = { .type = NLA_FLAG },
-	[NL80211_PMSR_FTM_REQ_ATTR_PREAMBLE] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_REQ_ATTR_PREAMBLE] =
+		NLA_POLICY_RANGE(NLA_U32, NL80211_PREAMBLE_LEGACY,
+				 NL80211_PREAMBLE_HE),
 	[NL80211_PMSR_FTM_REQ_ATTR_NUM_BURSTS_EXP] =
 		NLA_POLICY_MAX(NLA_U8, 15),
 	[NL80211_PMSR_FTM_REQ_ATTR_BURST_PERIOD] = { .type = NLA_U16 },
@@ -630,7 +632,7 @@ nl80211_mbssid_config_policy[NL80211_MBSSID_CONFIG_ATTR_MAX + 1] = {
 	[NL80211_MBSSID_CONFIG_ATTR_TX_IFINDEX] = { .type = NLA_U32 },
 	[NL80211_MBSSID_CONFIG_ATTR_EMA] = { .type = NLA_FLAG },
 	[NL80211_MBSSID_CONFIG_ATTR_TX_LINK_ID] =
-		NLA_POLICY_MAX(NLA_U8, IEEE80211_MLD_MAX_NUM_LINKS),
+		NLA_POLICY_RANGE(NLA_U8, 0, IEEE80211_MLD_MAX_NUM_LINKS - 1),
 };
 
 static const struct nla_policy
@@ -1093,6 +1095,7 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = {
 	[NL80211_ATTR_NPCA_PRIMARY_FREQ] = { .type = NLA_U32 },
 	[NL80211_ATTR_NPCA_PUNCT_BITMAP] =
 		NLA_POLICY_FULL_RANGE(NLA_U32, &nl80211_punct_bitmap_range),
+	[NL80211_ATTR_STA_DUMP_LINK_STATS] = { .type = NLA_FLAG },
 };
 
 /* policy for the key attributes */
@@ -2444,7 +2447,7 @@ static int nl80211_add_commands_unsplit(struct cfg80211_registered_device *rdev,
 	}
 	if (rdev->wiphy.max_sched_scan_reqs)
 		CMD(sched_scan_start, START_SCHED_SCAN);
-	CMD(probe_client, PROBE_CLIENT);
+	CMD(probe_peer, PROBE_PEER);
 	CMD(set_noack_map, SET_NOACK_MAP);
 	if (rdev->wiphy.flags & WIPHY_FLAG_REPORTS_OBSS) {
 		i++;
@@ -6510,7 +6513,8 @@ static int nl80211_parse_mbssid_config(struct wiphy *wiphy,
 }
 
 static struct cfg80211_mbssid_elems *
-nl80211_parse_mbssid_elems(struct wiphy *wiphy, struct nlattr *attrs)
+nl80211_parse_mbssid_elems(struct wiphy *wiphy, struct nlattr *attrs,
+			   struct netlink_ext_ack *extack)
 {
 	struct nlattr *nl_elems;
 	struct cfg80211_mbssid_elems *elems;
@@ -6521,6 +6525,12 @@ nl80211_parse_mbssid_elems(struct wiphy *wiphy, struct nlattr *attrs)
 		return ERR_PTR(-EINVAL);
 
 	nla_for_each_nested(nl_elems, attrs, rem_elems) {
+		int ret;
+
+		ret = validate_ie_attr(nl_elems, extack);
+		if (ret)
+			return ERR_PTR(ret);
+
 		if (num_elems >= 255)
 			return ERR_PTR(-EINVAL);
 		num_elems++;
@@ -6787,7 +6797,8 @@ static int nl80211_parse_beacon(struct cfg80211_registered_device *rdev,
 	if (attrs[NL80211_ATTR_MBSSID_ELEMS]) {
 		struct cfg80211_mbssid_elems *mbssid =
 			nl80211_parse_mbssid_elems(&rdev->wiphy,
-						   attrs[NL80211_ATTR_MBSSID_ELEMS]);
+						   attrs[NL80211_ATTR_MBSSID_ELEMS],
+						   extack);
 
 		if (IS_ERR(mbssid))
 			return PTR_ERR(mbssid);
@@ -6803,8 +6814,10 @@ static int nl80211_parse_beacon(struct cfg80211_registered_device *rdev,
 			if (IS_ERR(rnr))
 				return PTR_ERR(rnr);
 
-			if (rnr && rnr->cnt < bcn->mbssid_ies->cnt)
+			if (rnr && rnr->cnt < bcn->mbssid_ies->cnt) {
+				kfree(rnr);
 				return -EINVAL;
+			}
 
 			bcn->rnr_ies = rnr;
 		}
@@ -7870,7 +7883,7 @@ static int nl80211_fill_link_station(struct sk_buff *msg,
 		goto nla_put_failure;					\
 	} while (0)
 
-	link_sinfoattr = nla_nest_start_noflag(msg, NL80211_ATTR_STA_INFO);
+	link_sinfoattr = nla_nest_start(msg, NL80211_ATTR_STA_INFO);
 	if (!link_sinfoattr)
 		goto nla_put_failure;
 
@@ -7936,8 +7949,8 @@ static int nl80211_fill_link_station(struct sk_buff *msg,
 	PUT_LINK_SINFO(BEACON_LOSS, beacon_loss_count, u32);
 
 	if (link_sinfo->filled & BIT_ULL(NL80211_STA_INFO_BSS_PARAM)) {
-		bss_param = nla_nest_start_noflag(msg,
-						  NL80211_STA_INFO_BSS_PARAM);
+		bss_param = nla_nest_start(msg,
+					   NL80211_STA_INFO_BSS_PARAM);
 		if (!bss_param)
 			goto nla_put_failure;
 
@@ -7979,8 +7992,7 @@ static int nl80211_fill_link_station(struct sk_buff *msg,
 		struct nlattr *tidsattr;
 		int tid;
 
-		tidsattr = nla_nest_start_noflag(msg,
-						 NL80211_STA_INFO_TID_STATS);
+		tidsattr = nla_nest_start(msg, NL80211_STA_INFO_TID_STATS);
 		if (!tidsattr)
 			goto nla_put_failure;
 
@@ -7993,7 +8005,7 @@ static int nl80211_fill_link_station(struct sk_buff *msg,
 			if (!tidstats->filled)
 				continue;
 
-			tidattr = nla_nest_start_noflag(msg, tid + 1);
+			tidattr = nla_nest_start(msg, tid + 1);
 			if (!tidattr)
 				goto nla_put_failure;
 
@@ -8029,36 +8041,15 @@ static int nl80211_fill_link_station(struct sk_buff *msg,
 	return -EMSGSIZE;
 }
 
-static int nl80211_send_station(struct sk_buff *msg, u32 cmd, u32 portid,
-				u32 seq, int flags,
-				struct cfg80211_registered_device *rdev,
-				struct wireless_dev *wdev,
-				const u8 *mac_addr, struct station_info *sinfo,
-				bool link_stats)
+static int nl80211_put_sta_info_common(struct sk_buff *msg,
+				       struct cfg80211_registered_device *rdev,
+				       struct station_info *sinfo)
 {
-	void *hdr;
 	struct nlattr *sinfoattr, *bss_param;
-	struct link_station_info *link_sinfo;
-	struct nlattr *links, *link;
-	int link_id;
-
-	hdr = nl80211hdr_put(msg, portid, seq, flags, cmd);
-	if (!hdr) {
-		cfg80211_sinfo_release_content(sinfo);
-		return -1;
-	}
-
-	if ((wdev->netdev &&
-	     nla_put_u32(msg, NL80211_ATTR_IFINDEX, wdev->netdev->ifindex)) ||
-	    nla_put_u64_64bit(msg, NL80211_ATTR_WDEV, wdev_id(wdev),
-			      NL80211_ATTR_PAD) ||
-	    nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, mac_addr) ||
-	    nla_put_u32(msg, NL80211_ATTR_GENERATION, sinfo->generation))
-		goto nla_put_failure;
 
-	sinfoattr = nla_nest_start_noflag(msg, NL80211_ATTR_STA_INFO);
+	sinfoattr = nla_nest_start(msg, NL80211_ATTR_STA_INFO);
 	if (!sinfoattr)
-		goto nla_put_failure;
+		return -EMSGSIZE;
 
 #define PUT_SINFO(attr, memb, type) do {				\
 	BUILD_BUG_ON(sizeof(type) == sizeof(u64));			\
@@ -8149,8 +8140,7 @@ static int nl80211_send_station(struct sk_buff *msg, u32 cmd, u32 portid,
 	PUT_SINFO_U64(T_OFFSET, t_offset);
 
 	if (sinfo->filled & BIT_ULL(NL80211_STA_INFO_BSS_PARAM)) {
-		bss_param = nla_nest_start_noflag(msg,
-						  NL80211_STA_INFO_BSS_PARAM);
+		bss_param = nla_nest_start(msg, NL80211_STA_INFO_BSS_PARAM);
 		if (!bss_param)
 			goto nla_put_failure;
 
@@ -8192,8 +8182,7 @@ static int nl80211_send_station(struct sk_buff *msg, u32 cmd, u32 portid,
 		struct nlattr *tidsattr;
 		int tid;
 
-		tidsattr = nla_nest_start_noflag(msg,
-						 NL80211_STA_INFO_TID_STATS);
+		tidsattr = nla_nest_start(msg, NL80211_STA_INFO_TID_STATS);
 		if (!tidsattr)
 			goto nla_put_failure;
 
@@ -8206,7 +8195,7 @@ static int nl80211_send_station(struct sk_buff *msg, u32 cmd, u32 portid,
 			if (!tidstats->filled)
 				continue;
 
-			tidattr = nla_nest_start_noflag(msg, tid + 1);
+			tidattr = nla_nest_start(msg, tid + 1);
 			if (!tidattr)
 				goto nla_put_failure;
 
@@ -8236,6 +8225,37 @@ static int nl80211_send_station(struct sk_buff *msg, u32 cmd, u32 portid,
 	}
 
 	nla_nest_end(msg, sinfoattr);
+	return 0;
+
+nla_put_failure:
+	nla_nest_cancel(msg, sinfoattr);
+	return -EMSGSIZE;
+}
+
+static int nl80211_send_station(struct sk_buff *msg, u32 cmd, u32 portid,
+				u32 seq, int flags,
+				struct cfg80211_registered_device *rdev,
+				struct wireless_dev *wdev,
+				const u8 *mac_addr, struct station_info *sinfo)
+{
+	void *hdr;
+
+	hdr = nl80211hdr_put(msg, portid, seq, flags, cmd);
+	if (!hdr) {
+		cfg80211_sinfo_release_content(sinfo);
+		return -1;
+	}
+
+	if ((wdev->netdev &&
+	     nla_put_u32(msg, NL80211_ATTR_IFINDEX, wdev->netdev->ifindex)) ||
+	    nla_put_u64_64bit(msg, NL80211_ATTR_WDEV, wdev_id(wdev),
+			      NL80211_ATTR_PAD) ||
+	    nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, mac_addr) ||
+	    nla_put_u32(msg, NL80211_ATTR_GENERATION, sinfo->generation))
+		goto nla_put_failure;
+
+	if (nl80211_put_sta_info_common(msg, rdev, sinfo))
+		goto nla_put_failure;
 
 	if (sinfo->assoc_req_ies_len &&
 	    nla_put(msg, NL80211_ATTR_IE, sinfo->assoc_req_ies_len,
@@ -8258,45 +8278,11 @@ static int nl80211_send_station(struct sk_buff *msg, u32 cmd, u32 portid,
 			goto nla_put_failure;
 	}
 
-	if (link_stats && sinfo->valid_links) {
-		links = nla_nest_start(msg, NL80211_ATTR_MLO_LINKS);
-		if (!links)
-			goto nla_put_failure;
-
-		for_each_valid_link(sinfo, link_id) {
-			link_sinfo = sinfo->links[link_id];
-
-			if (WARN_ON_ONCE(!link_sinfo))
-				continue;
-
-			if (!is_valid_ether_addr(link_sinfo->addr))
-				continue;
-
-			link = nla_nest_start(msg, link_id + 1);
-			if (!link)
-				goto nla_put_failure;
-
-			if (nla_put_u8(msg, NL80211_ATTR_MLO_LINK_ID,
-				       link_id))
-				goto nla_put_failure;
-
-			if (nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN,
-				    link_sinfo->addr))
-				goto nla_put_failure;
-
-			if (nl80211_fill_link_station(msg, rdev, link_sinfo))
-				goto nla_put_failure;
-
-			nla_nest_end(msg, link);
-		}
-		nla_nest_end(msg, links);
-	}
-
 	cfg80211_sinfo_release_content(sinfo);
 	genlmsg_end(msg, hdr);
 	return 0;
 
- nla_put_failure:
+nla_put_failure:
 	cfg80211_sinfo_release_content(sinfo);
 	genlmsg_cancel(msg, hdr);
 	return -EMSGSIZE;
@@ -8490,82 +8476,261 @@ static void cfg80211_sta_set_mld_sinfo(struct station_info *sinfo)
 	sinfo->filled &= ~BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL_AVG);
 }
 
+enum nl80211_dump_station_phase {
+	NL80211_DUMP_STA_PHASE_AGGREGATED = 0,
+	NL80211_DUMP_STA_PHASE_PER_LINK   = 1,
+};
+
+struct nl80211_dump_station_ctx {
+	int sta_idx;
+	int link_idx;
+	enum nl80211_dump_station_phase phase;
+	bool dump_link_stats;
+	bool filter_mac;
+	u8 filter_mac_addr[ETH_ALEN];
+	u8 mac_addr[ETH_ALEN];
+	struct station_info sinfo;
+};
+
+static int nl80211_put_link_station_payload(struct sk_buff *msg,
+					    struct cfg80211_registered_device *rdev,
+					    struct station_info *sinfo,
+					    int link_idx)
+{
+	struct link_station_info *link_sinfo = sinfo->links[link_idx];
+	struct nlattr *links, *link;
+
+	if (WARN_ON_ONCE(!link_sinfo))
+		return -ENOENT;
+
+	if (!is_valid_ether_addr(link_sinfo->addr))
+		return -EADDRNOTAVAIL;
+
+	links = nla_nest_start(msg, NL80211_ATTR_MLO_LINKS);
+	if (!links)
+		return -EMSGSIZE;
+
+	link = nla_nest_start(msg, link_idx + 1);
+	if (!link)
+		goto nla_put_failure;
+
+	if (nla_put_u8(msg, NL80211_ATTR_MLO_LINK_ID, link_idx) ||
+	    nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, link_sinfo->addr))
+		goto nla_put_failure;
+
+	if (nl80211_fill_link_station(msg, rdev, link_sinfo))
+		goto nla_put_failure;
+
+	nla_nest_end(msg, link);
+	nla_nest_end(msg, links);
+	return 0;
+
+nla_put_failure:
+	nla_nest_cancel(msg, links);
+	return -EMSGSIZE;
+}
+
 static int nl80211_dump_station(struct sk_buff *skb,
 				struct netlink_callback *cb)
 {
-	struct station_info sinfo;
 	struct cfg80211_registered_device *rdev;
 	struct wireless_dev *wdev;
-	u8 mac_addr[ETH_ALEN];
-	int sta_idx = cb->args[2];
-	bool sinfo_alloc = false;
-	int err, i;
+	struct nl80211_dump_station_ctx *ctx = (void *)cb->args[2];
+	struct nlattr **attrbuf __free(kfree) = NULL;
+	int err;
 
-	err = nl80211_prepare_wdev_dump(cb, &rdev, &wdev, NULL);
+	if (!ctx) {
+		attrbuf = kzalloc_objs(*attrbuf, NUM_NL80211_ATTR);
+		if (!attrbuf)
+			return -ENOMEM;
+	}
+
+	err = nl80211_prepare_wdev_dump(cb, &rdev, &wdev, attrbuf);
 	if (err)
 		return err;
 	/* nl80211_prepare_wdev_dump acquired it in the successful case */
 	__acquire(&rdev->wiphy.mtx);
 
+	if (!ctx) {
+		ctx = kzalloc_obj(*ctx);
+		if (!ctx) {
+			err = -ENOMEM;
+			goto out_err;
+		}
+		cb->args[2] = (long)ctx;
+		ctx->phase = NL80211_DUMP_STA_PHASE_AGGREGATED;
+		ctx->dump_link_stats =
+			!!attrbuf[NL80211_ATTR_STA_DUMP_LINK_STATS];
+		if (attrbuf[NL80211_ATTR_MAC]) {
+			const u8 *mac = nla_data(attrbuf[NL80211_ATTR_MAC]);
+
+			if (!is_valid_ether_addr(mac)) {
+				kfree(ctx);
+				cb->args[2] = 0;
+				err = -EINVAL;
+				goto out_err;
+			}
+			ctx->filter_mac = true;
+			memcpy(ctx->filter_mac_addr, mac, ETH_ALEN);
+		}
+	}
+
 	if (!wdev->netdev && wdev->iftype != NL80211_IFTYPE_NAN) {
 		err = -EINVAL;
 		goto out_err;
 	}
 
-	if (!rdev->ops->dump_station) {
+	if (ctx->filter_mac) {
+		if (!rdev->ops->get_station) {
+			err = -EOPNOTSUPP;
+			goto out_err;
+		}
+	} else if (!rdev->ops->dump_station) {
 		err = -EOPNOTSUPP;
 		goto out_err;
 	}
 
-	while (1) {
-		memset(&sinfo, 0, sizeof(sinfo));
+	while (true) {
+		void *hdr;
+		int ret;
 
-		for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) {
-			sinfo.links[i] =
-				kzalloc_obj(*sinfo.links[0]);
-			if (!sinfo.links[i]) {
-				err = -ENOMEM;
-				goto out_err;
+		/* AGGREGATED phase: fetch sinfo from driver once per station */
+		if (ctx->phase == NL80211_DUMP_STA_PHASE_AGGREGATED) {
+			memset(&ctx->sinfo, 0, sizeof(ctx->sinfo));
+			for (int i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) {
+				ctx->sinfo.links[i] =
+					kzalloc_obj(*ctx->sinfo.links[0]);
+				if (!ctx->sinfo.links[i]) {
+					err = -ENOMEM;
+					goto out_err_release;
+				}
 			}
-			sinfo_alloc = true;
-		}
 
-		err = rdev_dump_station(rdev, wdev, sta_idx,
-					mac_addr, &sinfo);
-		if (err == -ENOENT)
-			break;
-		if (err)
-			goto out_err;
+			if (ctx->filter_mac) {
+				if (ctx->sta_idx > 0) {
+					err = skb->len;
+					goto out_err_release;
+				}
+				err = rdev_get_station(rdev, wdev,
+						       ctx->filter_mac_addr,
+						       &ctx->sinfo);
+				if (!err)
+					memcpy(ctx->mac_addr,
+					       ctx->filter_mac_addr, ETH_ALEN);
+			} else {
+				err = rdev_dump_station(rdev, wdev, ctx->sta_idx,
+							ctx->mac_addr,
+							&ctx->sinfo);
+			}
+			if (err == -ENOENT) {
+				err = skb->len;
+				goto out_err_release;
+			}
+			if (err)
+				goto out_err_release;
 
-		if (sinfo.valid_links)
-			cfg80211_sta_set_mld_sinfo(&sinfo);
+			if (ctx->sinfo.valid_links)
+				cfg80211_sta_set_mld_sinfo(&ctx->sinfo);
+		} else {
+			/* PER_LINK phase: advance to next valid link */
+			while (ctx->link_idx < IEEE80211_MLD_MAX_NUM_LINKS &&
+			       !(ctx->sinfo.valid_links & BIT(ctx->link_idx)))
+				ctx->link_idx++;
+
+			if (ctx->link_idx >= IEEE80211_MLD_MAX_NUM_LINKS) {
+				cfg80211_sinfo_release_content(&ctx->sinfo);
+				ctx->sta_idx++;
+				ctx->phase = NL80211_DUMP_STA_PHASE_AGGREGATED;
+				continue;
+			}
+		}
 
-		/* reset the sinfo_alloc flag as nl80211_send_station()
-		 * always releases sinfo
-		 */
-		sinfo_alloc = false;
+		/* Build common header for both phases */
+		hdr = nl80211hdr_put(skb, NETLINK_CB(cb->skb).portid,
+				     cb->nlh->nlmsg_seq, NLM_F_MULTI,
+				     NL80211_CMD_NEW_STATION);
+		if (!hdr) {
+			err = skb->len;
+			if (ctx->phase == NL80211_DUMP_STA_PHASE_PER_LINK)
+				goto out_err;
+			goto out_err_release;
+		}
 
-		if (nl80211_send_station(skb, NL80211_CMD_NEW_STATION,
-				NETLINK_CB(cb->skb).portid,
-				cb->nlh->nlmsg_seq, NLM_F_MULTI,
-				rdev, wdev, mac_addr,
-				&sinfo, false) < 0)
-			goto out;
+		if ((wdev->netdev &&
+		     nla_put_u32(skb, NL80211_ATTR_IFINDEX,
+				 wdev->netdev->ifindex)) ||
+		    nla_put_u64_64bit(skb, NL80211_ATTR_WDEV,
+				      wdev_id(wdev), NL80211_ATTR_PAD) ||
+		    nla_put(skb, NL80211_ATTR_MAC, ETH_ALEN, ctx->mac_addr) ||
+		    nla_put_u32(skb, NL80211_ATTR_GENERATION,
+				ctx->sinfo.generation)) {
+			genlmsg_cancel(skb, hdr);
+			err = skb->len;
+			if (ctx->phase == NL80211_DUMP_STA_PHASE_PER_LINK)
+				goto out_err;
+			goto out_err_release;
+		}
+
+		switch (ctx->phase) {
+		case NL80211_DUMP_STA_PHASE_AGGREGATED:
+			ret = nl80211_put_sta_info_common(skb, rdev, &ctx->sinfo);
+			if (ret) {
+				genlmsg_cancel(skb, hdr);
+				err = ret;
+				goto out_err_release;
+			}
+			genlmsg_end(skb, hdr);
+
+			if (ctx->dump_link_stats && ctx->sinfo.valid_links) {
+				ctx->phase = NL80211_DUMP_STA_PHASE_PER_LINK;
+				ctx->link_idx = 0;
+			} else {
+				cfg80211_sinfo_release_content(&ctx->sinfo);
+				ctx->sta_idx++;
+			}
+			break;
 
-		sta_idx++;
+		case NL80211_DUMP_STA_PHASE_PER_LINK:
+			ret = nl80211_put_link_station_payload(skb, rdev,
+							       &ctx->sinfo,
+							       ctx->link_idx);
+			if (ret == -EMSGSIZE) {
+				genlmsg_cancel(skb, hdr);
+				err = skb->len;
+				goto out_err;
+			}
+			if (ret) {
+				/* skip invalid link, do not abort the dump */
+				genlmsg_cancel(skb, hdr);
+				ctx->link_idx++;
+				continue;
+			}
+			genlmsg_end(skb, hdr);
+			ctx->link_idx++;
+			break;
+		}
 	}
 
- out:
-	cb->args[2] = sta_idx;
-	err = skb->len;
- out_err:
-	if (sinfo_alloc)
-		cfg80211_sinfo_release_content(&sinfo);
+out_err_release:
+	cfg80211_sinfo_release_content(&ctx->sinfo);
+	memset(&ctx->sinfo, 0, sizeof(ctx->sinfo));
+out_err:
 	wiphy_unlock(&rdev->wiphy);
 
 	return err;
 }
 
+static int nl80211_dump_station_done(struct netlink_callback *cb)
+{
+	struct nl80211_dump_station_ctx *ctx = (void *)cb->args[2];
+
+	if (ctx) {
+		cfg80211_sinfo_release_content(&ctx->sinfo);
+		kfree(ctx);
+	}
+	return 0;
+}
+
 static int nl80211_get_station(struct sk_buff *skb, struct genl_info *info)
 {
 	struct cfg80211_registered_device *rdev = info->user_ptr[0];
@@ -8613,7 +8778,7 @@ static int nl80211_get_station(struct sk_buff *skb, struct genl_info *info)
 
 	if (nl80211_send_station(msg, NL80211_CMD_NEW_STATION,
 				 info->snd_portid, info->snd_seq, 0,
-				 rdev, wdev, mac_addr, &sinfo, false) < 0) {
+				 rdev, wdev, mac_addr, &sinfo) < 0) {
 		nlmsg_free(msg);
 		return -ENOBUFS;
 	}
@@ -16150,26 +16315,41 @@ static int nl80211_register_unexpected_frame(struct sk_buff *skb,
 	return 0;
 }
 
-static int nl80211_probe_client(struct sk_buff *skb,
-				struct genl_info *info)
+static int nl80211_probe_peer(struct sk_buff *skb, struct genl_info *info)
 {
 	struct cfg80211_registered_device *rdev = info->user_ptr[0];
 	struct net_device *dev = info->user_ptr[1];
 	struct wireless_dev *wdev = dev->ieee80211_ptr;
 	struct sk_buff *msg;
 	void *hdr;
-	const u8 *addr;
+	const u8 *addr = NULL;
 	u64 cookie;
 	int err;
 
-	if (wdev->iftype != NL80211_IFTYPE_AP &&
-	    wdev->iftype != NL80211_IFTYPE_P2P_GO)
+	/* Allow in AP, STA, and their P2P counterparts */
+	switch (wdev->iftype) {
+	case NL80211_IFTYPE_AP:
+	case NL80211_IFTYPE_P2P_GO:
+		if (!info->attrs[NL80211_ATTR_MAC])
+			return -EINVAL;
+		addr = nla_data(info->attrs[NL80211_ATTR_MAC]);
+		break;
+	case NL80211_IFTYPE_STATION:
+	case NL80211_IFTYPE_P2P_CLIENT:
+		if (!wiphy_ext_feature_isset(&rdev->wiphy,
+					     NL80211_EXT_FEATURE_PROBE_AP))
+			return -EOPNOTSUPP;
+		if (!wdev->connected)
+			return -ENOLINK;
+		/* STA/P2P-client probes the currently associated AP/GO. */
+		if (info->attrs[NL80211_ATTR_MAC])
+			return -EINVAL;
+		break;
+	default:
 		return -EOPNOTSUPP;
+	}
 
-	if (!info->attrs[NL80211_ATTR_MAC])
-		return -EINVAL;
-
-	if (!rdev->ops->probe_client)
+	if (!rdev->ops->probe_peer)
 		return -EOPNOTSUPP;
 
 	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
@@ -16177,15 +16357,13 @@ static int nl80211_probe_client(struct sk_buff *skb,
 		return -ENOMEM;
 
 	hdr = nl80211hdr_put(msg, info->snd_portid, info->snd_seq, 0,
-			     NL80211_CMD_PROBE_CLIENT);
+			     NL80211_CMD_PROBE_PEER);
 	if (!hdr) {
 		err = -ENOBUFS;
 		goto free_msg;
 	}
 
-	addr = nla_data(info->attrs[NL80211_ATTR_MAC]);
-
-	err = rdev_probe_client(rdev, dev, addr, &cookie);
+	err = rdev_probe_peer(rdev, dev, addr, &cookie);
 	if (err)
 		goto free_msg;
 
@@ -19530,6 +19708,14 @@ static const struct genl_ops nl80211_ops[] = {
 		/* can be retrieved by unprivileged users */
 		.internal_flags = IFLAGS(NL80211_FLAG_NEED_WIPHY),
 	},
+	{
+		.cmd = NL80211_CMD_GET_STATION,
+		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+		.doit = nl80211_get_station,
+		.dumpit = nl80211_dump_station,
+		.done = nl80211_dump_station_done,
+		.internal_flags = IFLAGS(NL80211_FLAG_NEED_WDEV),
+	},
 };
 
 static const struct genl_small_ops nl80211_small_ops[] = {
@@ -19629,13 +19815,6 @@ static const struct genl_small_ops nl80211_small_ops[] = {
 		.internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP |
 					 NL80211_FLAG_MLO_VALID_LINK_ID),
 	},
-	{
-		.cmd = NL80211_CMD_GET_STATION,
-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
-		.doit = nl80211_get_station,
-		.dumpit = nl80211_dump_station,
-		.internal_flags = IFLAGS(NL80211_FLAG_NEED_WDEV),
-	},
 	{
 		.cmd = NL80211_CMD_SET_STATION,
 		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
@@ -20042,9 +20221,9 @@ static const struct genl_small_ops nl80211_small_ops[] = {
 		.internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV),
 	},
 	{
-		.cmd = NL80211_CMD_PROBE_CLIENT,
+		.cmd = NL80211_CMD_PROBE_PEER,
 		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
-		.doit = nl80211_probe_client,
+		.doit = nl80211_probe_peer,
 		.flags = GENL_UNS_ADMIN_PERM,
 		.internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP),
 	},
@@ -20890,6 +21069,9 @@ void cfg80211_rx_unprot_mlme_mgmt(struct net_device *dev, const u8 *buf,
 	} else if (ieee80211_is_disassoc(mgmt->frame_control)) {
 		event.cmd = NL80211_CMD_UNPROT_DISASSOCIATE;
 	} else if (ieee80211_is_beacon(mgmt->frame_control)) {
+		if (wdev->iftype == NL80211_IFTYPE_AP ||
+		    wdev->iftype == NL80211_IFTYPE_P2P_GO)
+			return;
 		if (wdev->unprot_beacon_reported &&
 		    elapsed_jiffies_msecs(wdev->unprot_beacon_reported) < 10000)
 			return;
@@ -21635,7 +21817,7 @@ void cfg80211_new_sta(struct wireless_dev *wdev, const u8 *mac_addr,
 		return;
 
 	if (nl80211_send_station(msg, NL80211_CMD_NEW_STATION, 0, 0, 0,
-				 rdev, wdev, mac_addr, sinfo, false) < 0) {
+				 rdev, wdev, mac_addr, sinfo) < 0) {
 		nlmsg_free(msg);
 		return;
 	}
@@ -21665,7 +21847,7 @@ void cfg80211_del_sta_sinfo(struct wireless_dev *wdev, const u8 *mac_addr,
 	}
 
 	if (nl80211_send_station(msg, NL80211_CMD_DEL_STATION, 0, 0, 0,
-				 rdev, wdev, mac_addr, sinfo, false) < 0) {
+				 rdev, wdev, mac_addr, sinfo) < 0) {
 		nlmsg_free(msg);
 		return;
 	}
@@ -22598,8 +22780,8 @@ void cfg80211_sta_opmode_change_notify(struct net_device *dev, const u8 *mac,
 }
 EXPORT_SYMBOL(cfg80211_sta_opmode_change_notify);
 
-void cfg80211_probe_status(struct net_device *dev, const u8 *addr,
-			   u64 cookie, bool acked, s32 ack_signal,
+void cfg80211_probe_status(struct net_device *dev, const u8 *peer, u64 cookie,
+			   int link_id, bool acked, s32 ack_signal,
 			   bool is_valid_ack_signal, gfp_t gfp)
 {
 	struct wireless_dev *wdev = dev->ieee80211_ptr;
@@ -22607,14 +22789,14 @@ void cfg80211_probe_status(struct net_device *dev, const u8 *addr,
 	struct sk_buff *msg;
 	void *hdr;
 
-	trace_cfg80211_probe_status(dev, addr, cookie, acked);
+	trace_cfg80211_probe_status(dev, peer, cookie, acked);
 
 	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, gfp);
 
 	if (!msg)
 		return;
 
-	hdr = nl80211hdr_put(msg, 0, 0, 0, NL80211_CMD_PROBE_CLIENT);
+	hdr = nl80211hdr_put(msg, 0, 0, 0, NL80211_CMD_PROBE_PEER);
 	if (!hdr) {
 		nlmsg_free(msg);
 		return;
@@ -22622,12 +22804,18 @@ void cfg80211_probe_status(struct net_device *dev, const u8 *addr,
 
 	if (nla_put_u32(msg, NL80211_ATTR_WIPHY, rdev->wiphy_idx) ||
 	    nla_put_u32(msg, NL80211_ATTR_IFINDEX, dev->ifindex) ||
-	    nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, addr) ||
+	    (peer && nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, peer)) ||
 	    nla_put_u64_64bit(msg, NL80211_ATTR_COOKIE, cookie,
-			      NL80211_ATTR_PAD) ||
-	    (acked && nla_put_flag(msg, NL80211_ATTR_ACK)) ||
-	    (is_valid_ack_signal && nla_put_s32(msg, NL80211_ATTR_ACK_SIGNAL,
-						ack_signal)))
+			      NL80211_ATTR_PAD))
+		goto nla_put_failure;
+
+	if (link_id >= 0 &&
+	    nla_put_u8(msg, NL80211_ATTR_MLO_LINK_ID, link_id))
+		goto nla_put_failure;
+
+	if ((acked && nla_put_flag(msg, NL80211_ATTR_ACK)) ||
+	    (is_valid_ack_signal &&
+	     nla_put_s32(msg, NL80211_ATTR_ACK_SIGNAL, ack_signal)))
 		goto nla_put_failure;
 
 	genlmsg_end(msg, hdr);
@@ -22942,7 +23130,8 @@ static int nl80211_netlink_notify(struct notifier_block * nb,
 				wdev->nl_owner_dead = true;
 				schedule_work(&rdev->destroy_work);
 			} else if (wdev->conn_owner_nlportid == notify->portid) {
-				schedule_work(&wdev->disconnect_wk);
+				wiphy_work_queue(wdev->wiphy,
+						 &wdev->disconnect_wk);
 			}
 
 			cfg80211_release_pmsr(wdev, notify->portid);
diff --git a/net/wireless/pmsr.c b/net/wireless/pmsr.c
index c8447448f3a50..34c3625f7fd5e 100644
--- a/net/wireless/pmsr.c
+++ b/net/wireless/pmsr.c
@@ -125,6 +125,7 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev,
 		NL_SET_ERR_MSG_ATTR(info->extack,
 				    tb[NL80211_PMSR_FTM_REQ_ATTR_REQUEST_LCI],
 				    "FTM: LCI request not supported");
+		return -EOPNOTSUPP;
 	}
 
 	out->ftm.request_civicloc =
@@ -133,6 +134,7 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev,
 		NL_SET_ERR_MSG_ATTR(info->extack,
 				    tb[NL80211_PMSR_FTM_REQ_ATTR_REQUEST_CIVICLOC],
 			    "FTM: civic location request not supported");
+		return -EOPNOTSUPP;
 	}
 
 	out->ftm.trigger_based =
@@ -310,6 +312,7 @@ static int pmsr_parse_peer(struct cfg80211_registered_device *rdev,
 {
 	struct nlattr *tb[NL80211_PMSR_PEER_ATTR_MAX + 1];
 	struct nlattr *req[NL80211_PMSR_REQ_ATTR_MAX + 1];
+	bool have_measurement_type = false;
 	struct nlattr *treq;
 	int err, rem;
 
@@ -376,6 +379,14 @@ static int pmsr_parse_peer(struct cfg80211_registered_device *rdev,
 	}
 
 	nla_for_each_nested(treq, req[NL80211_PMSR_REQ_ATTR_DATA], rem) {
+		if (have_measurement_type) {
+			NL_SET_ERR_MSG_ATTR(info->extack, treq,
+					    "multiple measurement types in request data");
+			return -EINVAL;
+		}
+
+		have_measurement_type = true;
+
 		switch (nla_type(treq)) {
 		case NL80211_PMSR_TYPE_FTM:
 			err = pmsr_parse_ftm(rdev, treq, out, info);
@@ -385,10 +396,16 @@ static int pmsr_parse_peer(struct cfg80211_registered_device *rdev,
 					    "unsupported measurement type");
 			err = -EINVAL;
 		}
+		if (err)
+			return err;
 	}
 
-	if (err)
-		return err;
+	if (!have_measurement_type) {
+		NL_SET_ERR_MSG_ATTR(info->extack,
+				    req[NL80211_PMSR_REQ_ATTR_DATA],
+				    "missing measurement type in request data");
+		return -EINVAL;
+	}
 
 	return 0;
 }
@@ -427,6 +444,11 @@ int nl80211_pmsr_start(struct sk_buff *skb, struct genl_info *info)
 		}
 	}
 
+	if (!count) {
+		NL_SET_ERR_MSG_ATTR(info->extack, peers, "No peers specified");
+		return -EINVAL;
+	}
+
 	req = kzalloc_flex(*req, peers, count);
 	if (!req)
 		return -ENOMEM;
@@ -807,13 +829,11 @@ static void cfg80211_pmsr_process_abort(struct wireless_dev *wdev)
 	}
 }
 
-void cfg80211_pmsr_free_wk(struct work_struct *work)
+void cfg80211_pmsr_free_wk(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct wireless_dev *wdev = container_of(work, struct wireless_dev,
 						 pmsr_free_wk);
 
-	guard(wiphy)(wdev->wiphy);
-
 	cfg80211_pmsr_process_abort(wdev);
 }
 
@@ -829,7 +849,7 @@ void cfg80211_pmsr_wdev_down(struct wireless_dev *wdev)
 	}
 	spin_unlock_bh(&wdev->pmsr_lock);
 
-	cancel_work_sync(&wdev->pmsr_free_wk);
+	wiphy_work_cancel(wdev->wiphy, &wdev->pmsr_free_wk);
 	if (found)
 		cfg80211_pmsr_process_abort(wdev);
 
@@ -844,7 +864,7 @@ void cfg80211_release_pmsr(struct wireless_dev *wdev, u32 portid)
 	list_for_each_entry(req, &wdev->pmsr_list, list) {
 		if (req->nl_portid == portid) {
 			req->nl_portid = 0;
-			schedule_work(&wdev->pmsr_free_wk);
+			wiphy_work_queue(wdev->wiphy, &wdev->pmsr_free_wk);
 		}
 	}
 	spin_unlock_bh(&wdev->pmsr_lock);
diff --git a/net/wireless/rdev-ops.h b/net/wireless/rdev-ops.h
index 63c26e8b11395..6c3bad8b2d6f6 100644
--- a/net/wireless/rdev-ops.h
+++ b/net/wireless/rdev-ops.h
@@ -948,13 +948,13 @@ static inline int rdev_tdls_oper(struct cfg80211_registered_device *rdev,
 	return ret;
 }
 
-static inline int rdev_probe_client(struct cfg80211_registered_device *rdev,
-				    struct net_device *dev, const u8 *peer,
-				    u64 *cookie)
+static inline int rdev_probe_peer(struct cfg80211_registered_device *rdev,
+				  struct net_device *dev, const u8 *peer,
+				  u64 *cookie)
 {
 	int ret;
-	trace_rdev_probe_client(&rdev->wiphy, dev, peer);
-	ret = rdev->ops->probe_client(&rdev->wiphy, dev, peer, cookie);
+	trace_rdev_probe_peer(&rdev->wiphy, dev, peer);
+	ret = rdev->ops->probe_peer(&rdev->wiphy, dev, peer, cookie);
 	trace_rdev_return_int_cookie(&rdev->wiphy, ret, *cookie);
 	return ret;
 }
diff --git a/net/wireless/scan.c b/net/wireless/scan.c
index 05b7dc6b766ce..071083cc33672 100644
--- a/net/wireless/scan.c
+++ b/net/wireless/scan.c
@@ -205,7 +205,7 @@ bool cfg80211_is_element_inherited(const struct element *elem,
 		return true;
 
 	if (elem->id == WLAN_EID_EXTENSION) {
-		if (!ext_id_len)
+		if (!ext_id_len || !elem->datalen)
 			return true;
 		loop_len = ext_id_len;
 		list = &non_inherit_elem->data[3 + id_len];
@@ -326,8 +326,11 @@ cfg80211_gen_new_ie(const u8 *ie, size_t ielen,
 		/* For ML probe response, match the MLE in the frame body with
 		 * MLD id being 'bssid_index'
 		 */
-		if (parent->id == WLAN_EID_EXTENSION && parent->datalen > 1 &&
+		if (parent->id == WLAN_EID_EXTENSION &&
 		    parent->data[0] == WLAN_EID_EXT_EHT_MULTI_LINK &&
+		    ieee80211_mle_type_ok(parent->data + 1,
+					  IEEE80211_ML_CONTROL_TYPE_BASIC,
+					  parent->datalen - 1) &&
 		    bssid_index == ieee80211_mle_get_mld_id(parent->data + 1)) {
 			if (!cfg80211_copy_elem_with_frags(parent,
 							   ie, ielen,
@@ -3311,14 +3314,15 @@ cfg80211_inform_bss_frame_data(struct wiphy *wiphy,
 		bssid = ext->u.s1g_beacon.sa;
 		capability = le16_to_cpu(compat->compat_info);
 		beacon_interval = le16_to_cpu(compat->beacon_int);
+		tsf = le32_to_cpu(ext->u.s1g_beacon.timestamp);
+		tsf |= (u64)le32_to_cpu(compat->tsf_completion) << 32;
 	} else {
 		bssid = mgmt->bssid;
 		beacon_interval = le16_to_cpu(mgmt->u.probe_resp.beacon_int);
 		capability = le16_to_cpu(mgmt->u.probe_resp.capab_info);
+		tsf = le64_to_cpu(mgmt->u.probe_resp.timestamp);
 	}
 
-	tsf = le64_to_cpu(mgmt->u.probe_resp.timestamp);
-
 	if (ieee80211_is_probe_resp(mgmt->frame_control))
 		ftype = CFG80211_BSS_FTYPE_PRESP;
 	else if (ext)
@@ -3612,8 +3616,10 @@ int cfg80211_wext_siwscan(struct net_device *dev,
 	/* translate "Scan for SSID" request */
 	if (wreq) {
 		if (wrqu->data.flags & IW_SCAN_THIS_ESSID) {
-			if (wreq->essid_len > IEEE80211_MAX_SSID_LEN)
-				return -EINVAL;
+			if (wreq->essid_len > IEEE80211_MAX_SSID_LEN) {
+				err = -EINVAL;
+				goto out;
+			}
 			memcpy(creq->req.ssids[0].ssid, wreq->essid,
 			       wreq->essid_len);
 			creq->req.ssids[0].ssid_len = wreq->essid_len;
diff --git a/net/wireless/sme.c b/net/wireless/sme.c
index b451df3096dd1..2a719b5c487e3 100644
--- a/net/wireless/sme.c
+++ b/net/wireless/sme.c
@@ -1578,13 +1578,11 @@ int cfg80211_disconnect(struct cfg80211_registered_device *rdev,
  * Used to clean up after the connection / connection attempt owner socket
  * disconnects
  */
-void cfg80211_autodisconnect_wk(struct work_struct *work)
+void cfg80211_autodisconnect_wk(struct wiphy *wiphy, struct wiphy_work *work)
 {
 	struct wireless_dev *wdev =
 		container_of(work, struct wireless_dev, disconnect_wk);
-	struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
-
-	guard(wiphy)(wdev->wiphy);
+	struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
 
 	if (wdev->conn_owner_nlportid) {
 		switch (wdev->iftype) {
diff --git a/net/wireless/trace.h b/net/wireless/trace.h
index 94944f2a39a4d..8c2a91b85c396 100644
--- a/net/wireless/trace.h
+++ b/net/wireless/trace.h
@@ -2132,7 +2132,7 @@ DECLARE_EVENT_CLASS(rdev_pmksa,
 		  WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->bssid)
 );
 
-TRACE_EVENT(rdev_probe_client,
+TRACE_EVENT(rdev_probe_peer,
 	TP_PROTO(struct wiphy *wiphy, struct net_device *netdev,
 		 const u8 *peer),
 	TP_ARGS(wiphy, netdev, peer),
diff --git a/net/wireless/wext-core.c b/net/wireless/wext-core.c
index c19dece2bc6ee..db77912b3994b 100644
--- a/net/wireless/wext-core.c
+++ b/net/wireless/wext-core.c
@@ -660,8 +660,7 @@ struct iw_statistics *get_wireless_stats(struct net_device *dev)
 	    dev->ieee80211_ptr->wiphy->wext &&
 	    dev->ieee80211_ptr->wiphy->wext->get_wireless_stats) {
 		wireless_warn_cfg80211_wext();
-		if (dev->ieee80211_ptr->wiphy->flags & (WIPHY_FLAG_SUPPORTS_MLO |
-							WIPHY_FLAG_DISABLE_WEXT))
+		if (dev->ieee80211_ptr->wiphy->flags & WIPHY_FLAG_SUPPORTS_MLO)
 			return NULL;
 		return dev->ieee80211_ptr->wiphy->wext->get_wireless_stats(dev);
 	}
@@ -703,8 +702,7 @@ static iw_handler get_handler(struct net_device *dev, unsigned int cmd)
 #ifdef CONFIG_CFG80211_WEXT
 	if (dev->ieee80211_ptr && dev->ieee80211_ptr->wiphy) {
 		wireless_warn_cfg80211_wext();
-		if (dev->ieee80211_ptr->wiphy->flags & (WIPHY_FLAG_SUPPORTS_MLO |
-							WIPHY_FLAG_DISABLE_WEXT))
+		if (dev->ieee80211_ptr->wiphy->flags & WIPHY_FLAG_SUPPORTS_MLO)
 			return NULL;
 		handlers = dev->ieee80211_ptr->wiphy->wext;
 	}
diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c
index 630f3dd31cc5c..f153bf695b9dd 100644
--- a/net/xfrm/xfrm_device.c
+++ b/net/xfrm/xfrm_device.c
@@ -182,7 +182,7 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur
 		err = x->type_offload->xmit(x, skb, esp_features);
 		if (err) {
 			if (err == -EINPROGRESS)
-				return NULL;
+				return ERR_PTR(-EINPROGRESS);
 
 			XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
 			kfree_skb(skb);
@@ -224,7 +224,15 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur
 		pskb = skb2;
 	}
 
-	return skb;
+	/* skb_gso_segment() set skb->prev to the last segment, but async
+	 * crypto may have stolen it above without updating ->prev.  Repoint
+	 * it at the last retained segment so validate_xmit_skb_list() does
+	 * not chain onto a segment now owned by the crypto engine.
+	 */
+	if (skb)
+		skb->prev = pskb;
+
+	return skb ? skb : ERR_PTR(-EINPROGRESS);
 }
 EXPORT_SYMBOL_GPL(validate_xmit_xfrm);
 
@@ -313,6 +321,7 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
 	}
 
 	xso->dev = dev;
+	xso->ifindex = dev->ifindex;
 	netdev_tracker_alloc(dev, &xso->dev_tracker, GFP_ATOMIC);
 
 	if (xuo->flags & XFRM_OFFLOAD_INBOUND)
diff --git a/net/xfrm/xfrm_iptfs.c b/net/xfrm/xfrm_iptfs.c
index ad810d1f97c06..597aedeac26eb 100644
--- a/net/xfrm/xfrm_iptfs.c
+++ b/net/xfrm/xfrm_iptfs.c
@@ -480,6 +480,7 @@ static int iptfs_skb_add_frags(struct sk_buff *skb,
 		}
 		__skb_frag_ref(tofrag);
 		shinfo->nr_frags++;
+		shinfo->flags |= SKBFL_SHARED_FRAG;
 
 		/* see if we are done */
 		fraglen = tofrag->len;
diff --git a/net/xfrm/xfrm_nat_keepalive.c b/net/xfrm/xfrm_nat_keepalive.c
index 458931062a04e..eb1b6f67739e1 100644
--- a/net/xfrm/xfrm_nat_keepalive.c
+++ b/net/xfrm/xfrm_nat_keepalive.c
@@ -55,8 +55,10 @@ static int nat_keepalive_send_ipv4(struct sk_buff *skb,
 			   ka->encap_sport, sock_net_uid(net, NULL));
 
 	rt = ip_route_output_key(net, &fl4);
-	if (IS_ERR(rt))
+	if (IS_ERR(rt)) {
+		kfree_skb(skb);
 		return PTR_ERR(rt);
+	}
 
 	skb_dst_set(skb, &rt->dst);
 
@@ -101,6 +103,7 @@ static int nat_keepalive_send_ipv6(struct sk_buff *skb,
 	dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL);
 	if (IS_ERR(dst)) {
 		local_unlock_nested_bh(&nat_keepalive_sk_ipv6.bh_lock);
+		kfree_skb(skb);
 		return PTR_ERR(dst);
 	}
 
@@ -118,7 +121,6 @@ static void nat_keepalive_send(struct nat_keepalive *ka)
 					sizeof(struct ipv6hdr)) +
 				    sizeof(struct udphdr);
 	const u8 nat_ka_payload = 0xFF;
-	int err = -EAFNOSUPPORT;
 	struct sk_buff *skb;
 	struct udphdr *uh;
 
@@ -140,16 +142,17 @@ static void nat_keepalive_send(struct nat_keepalive *ka)
 
 	switch (ka->family) {
 	case AF_INET:
-		err = nat_keepalive_send_ipv4(skb, ka);
+		nat_keepalive_send_ipv4(skb, ka);
 		break;
 #if IS_ENABLED(CONFIG_IPV6)
 	case AF_INET6:
-		err = nat_keepalive_send_ipv6(skb, ka, uh);
+		nat_keepalive_send_ipv6(skb, ka, uh);
 		break;
 #endif
-	}
-	if (err)
+	default:
 		kfree_skb(skb);
+		break;
+	}
 }
 
 struct nat_keepalive_work_ctx {
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index 7ef861a0e8231..932a313b9460a 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -1329,8 +1329,8 @@ static void xfrm_hash_rebuild(struct work_struct *work)
 			}
 		}
 
-		if (policy->selector.prefixlen_d < dbits ||
-		    policy->selector.prefixlen_s < sbits)
+		if (policy->selector.prefixlen_d >= dbits &&
+		    policy->selector.prefixlen_s >= sbits)
 			continue;
 
 		bin = xfrm_policy_inexact_alloc_bin(policy, dir);
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
index c58cd024e3c67..36a4f6793edef 100644
--- a/net/xfrm/xfrm_state.c
+++ b/net/xfrm/xfrm_state.c
@@ -1547,6 +1547,7 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
 			xso->type = XFRM_DEV_OFFLOAD_PACKET;
 			xso->dir = xdo->dir;
 			xso->dev = dev;
+			xso->ifindex = dev->ifindex;
 			xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ;
 			netdev_hold(dev, &xso->dev_tracker, GFP_ATOMIC);
 			error = dev->xfrmdev_ops->xdo_dev_state_add(dev, x,
@@ -2071,8 +2072,11 @@ static struct xfrm_state *xfrm_state_clone_and_setup(struct xfrm_state *orig,
 
 	x->mode_cbs = orig->mode_cbs;
 	if (x->mode_cbs && x->mode_cbs->clone_state) {
-		if (x->mode_cbs->clone_state(x, orig))
+		if (x->mode_cbs->clone_state(x, orig)) {
+			if (!x->mode_data)
+				x->mode_cbs = NULL;
 			goto error;
+		}
 	}
 
 	x->props.reqid = m->new_reqid;
@@ -3010,7 +3014,7 @@ int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen)
 	if (sockptr_is_null(optval) && !optlen) {
 		xfrm_sk_policy_insert(sk, XFRM_POLICY_IN, NULL);
 		xfrm_sk_policy_insert(sk, XFRM_POLICY_OUT, NULL);
-		__sk_dst_reset(sk);
+		sk_dst_reset(sk);
 		return 0;
 	}
 
@@ -3050,7 +3054,7 @@ int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen)
 	if (err >= 0) {
 		xfrm_sk_policy_insert(sk, err, pol);
 		xfrm_pol_put(pol);
-		__sk_dst_reset(sk);
+		sk_dst_reset(sk);
 		err = 0;
 	}
 
@@ -3291,6 +3295,8 @@ int __xfrm_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
 		if (x->mode_cbs->init_state)
 			err = x->mode_cbs->init_state(x);
 		module_put(x->mode_cbs->owner);
+		if (err && !x->mode_data)
+			x->mode_cbs = NULL;
 	}
 error:
 	return err;
diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c
index 6384795ee6b26..d6db63304ba6b 100644
--- a/net/xfrm/xfrm_user.c
+++ b/net/xfrm/xfrm_user.c
@@ -1201,17 +1201,26 @@ static int copy_sec_ctx(struct xfrm_sec_ctx *s, struct sk_buff *skb)
 	return 0;
 }
 
-static void xso_to_xuo(const struct xfrm_dev_offload *xso,
-		       struct xfrm_user_offload *xuo)
+static void xso_to_xuo_ifindex(const struct xfrm_dev_offload *xso, int ifindex,
+			       struct xfrm_user_offload *xuo)
 {
-	xuo->ifindex = xso->dev->ifindex;
+	xuo->ifindex = ifindex;
 	if (xso->dir == XFRM_DEV_OFFLOAD_IN)
 		xuo->flags = XFRM_OFFLOAD_INBOUND;
 	if (xso->type == XFRM_DEV_OFFLOAD_PACKET)
 		xuo->flags |= XFRM_OFFLOAD_PACKET;
 }
 
-static int copy_user_offload(struct xfrm_dev_offload *xso, struct sk_buff *skb)
+#ifdef CONFIG_XFRM_MIGRATE
+static void xso_to_xuo(const struct xfrm_dev_offload *xso,
+		       struct xfrm_user_offload *xuo)
+{
+	xso_to_xuo_ifindex(xso, xso->dev->ifindex, xuo);
+}
+#endif
+
+static int copy_user_offload_ifindex(const struct xfrm_dev_offload *xso,
+				     int ifindex, struct sk_buff *skb)
 {
 	struct xfrm_user_offload *xuo;
 	struct nlattr *attr;
@@ -1222,11 +1231,22 @@ static int copy_user_offload(struct xfrm_dev_offload *xso, struct sk_buff *skb)
 
 	xuo = nla_data(attr);
 	memset(xuo, 0, sizeof(*xuo));
-	xso_to_xuo(xso, xuo);
+	xso_to_xuo_ifindex(xso, ifindex, xuo);
 
 	return 0;
 }
 
+static int copy_user_offload(struct xfrm_dev_offload *xso, struct sk_buff *skb)
+{
+	return copy_user_offload_ifindex(xso, xso->dev->ifindex, skb);
+}
+
+static int copy_user_state_offload(const struct xfrm_dev_offload *xso,
+				   struct sk_buff *skb)
+{
+	return copy_user_offload_ifindex(xso, READ_ONCE(xso->ifindex), skb);
+}
+
 static bool xfrm_redact(void)
 {
 	return IS_ENABLED(CONFIG_SECURITY) &&
@@ -1433,8 +1453,8 @@ static int copy_to_user_state_extra(struct xfrm_state *x,
 			      &x->replay);
 	if (ret)
 		goto out;
-	if(x->xso.dev)
-		ret = copy_user_offload(&x->xso, skb);
+	if (READ_ONCE(x->xso.dev))
+		ret = copy_user_state_offload(&x->xso, skb);
 	if (ret)
 		goto out;
 	if (x->if_id) {
@@ -2104,13 +2124,12 @@ static int validate_tmpl(int nr, struct xfrm_user_tmpl *ut, u16 family,
 		switch (ut[i].mode) {
 		case XFRM_MODE_TUNNEL:
 		case XFRM_MODE_BEET:
+		case XFRM_MODE_IPTFS:
 			if (ut[i].optional && dir == XFRM_POLICY_OUT) {
 				NL_SET_ERR_MSG(extack, "Mode in optional template not allowed in outbound policy");
 				return -EINVAL;
 			}
 			break;
-		case XFRM_MODE_IPTFS:
-			break;
 		default:
 			if (ut[i].family != prev_family) {
 				NL_SET_ERR_MSG(extack, "Mode in template doesn't support a family change");
@@ -4046,8 +4065,8 @@ static inline unsigned int xfrm_sa_len(struct xfrm_state *x)
 		l += nla_total_size(sizeof(*x->coaddr));
 	if (x->props.extra_flags)
 		l += nla_total_size(sizeof(x->props.extra_flags));
-	if (x->xso.dev)
-		 l += nla_total_size(sizeof(struct xfrm_user_offload));
+	if (READ_ONCE(x->xso.dev))
+		l += nla_total_size(sizeof(struct xfrm_user_offload));
 	if (x->props.smark.v | x->props.smark.m) {
 		l += nla_total_size(sizeof(x->props.smark.v));
 		l += nla_total_size(sizeof(x->props.smark.m));
diff --git a/rust/Makefile b/rust/Makefile
index a870d1616c711..627ed79dc6f50 100644
--- a/rust/Makefile
+++ b/rust/Makefile
@@ -84,11 +84,15 @@ core-flags := \
     --edition=$(core-edition) \
     $(call cfgs-to-flags,$(core-cfgs))
 
+zerocopy-cfgs := \
+    no_fp_fmt_parse
+
 zerocopy-flags := \
-    --cap-lints=allow
+    --cap-lints=allow \
+    $(call cfgs-to-flags,$(zerocopy-cfgs))
 
 zerocopy-envs := \
-    CARGO_PKG_VERSION=0.8.50
+    CARGO_PKG_VERSION=0.8.54
 
 proc_macro2-cfgs := \
     feature="proc-macro" \
@@ -128,11 +132,15 @@ syn-flags := \
     --extern quote \
     $(call cfgs-to-flags,$(syn-cfgs))
 
+zerocopy_derive-cfgs := \
+    zerocopy_unstable_linux
+
 zerocopy_derive-flags := \
     --cap-lints=allow \
     --extern proc_macro2 \
     --extern quote \
-    --extern syn
+    --extern syn \
+    $(call cfgs-to-flags,$(zerocopy_derive-cfgs))
 
 pin_init_internal-cfgs := \
     kernel USE_RUSTC_FEATURES
@@ -644,9 +652,11 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
 rust-analyzer:
 	$(Q)MAKEFLAGS= $(srctree)/scripts/generate_rust_analyzer.py \
 		--cfgs='core=$(core-cfgs)' $(core-edition) \
+		--cfgs='zerocopy=$(zerocopy-cfgs)' \
 		--cfgs='proc_macro2=$(proc_macro2-cfgs)' \
 		--cfgs='quote=$(quote-cfgs)' \
 		--cfgs='syn=$(syn-cfgs)' \
+		--cfgs='zerocopy_derive=$(zerocopy_derive-cfgs)' \
 		--cfgs='pin_init_internal=$(pin_init_internal-cfgs)' \
 		--cfgs='pin_init=$(pin_init-cfgs)' \
 		--envs='zerocopy=$(zerocopy-envs)' \
diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs
index 854e7c471434c..812f8e5a08d53 100644
--- a/rust/bindings/lib.rs
+++ b/rust/bindings/lib.rs
@@ -27,6 +27,10 @@
 #[allow(clippy::ref_as_ptr)]
 #[allow(clippy::undocumented_unsafe_blocks)]
 #[cfg_attr(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES, allow(unnecessary_transmutes))]
+#[cfg_attr(
+    CONFIG_RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS,
+    allow(suspicious_runtime_symbol_definitions)
+)]
 mod bindings_raw {
     use pin_init::{MaybeZeroable, Zeroable};
 
diff --git a/rust/helpers/io.c b/rust/helpers/io.c
index 397810864a243..7ed9a4f77f1b5 100644
--- a/rust/helpers/io.c
+++ b/rust/helpers/io.c
@@ -19,6 +19,19 @@ __rust_helper void rust_helper_iounmap(void __iomem *addr)
 	iounmap(addr);
 }
 
+__rust_helper void rust_helper_memcpy_fromio(void *dst,
+					     const volatile void __iomem *src,
+					     size_t count)
+{
+	memcpy_fromio(dst, src, count);
+}
+
+__rust_helper void rust_helper_memcpy_toio(volatile void __iomem *dst,
+					   const void *src, size_t count)
+{
+	memcpy_toio(dst, src, count);
+}
+
 __rust_helper u8 rust_helper_readb(const void __iomem *addr)
 {
 	return readb(addr);
diff --git a/rust/helpers/task.c b/rust/helpers/task.c
index c0e1a06ede78c..b46b1433a67e8 100644
--- a/rust/helpers/task.c
+++ b/rust/helpers/task.c
@@ -28,11 +28,6 @@ __rust_helper kuid_t rust_helper_task_uid(struct task_struct *task)
 	return task_uid(task);
 }
 
-__rust_helper kuid_t rust_helper_task_euid(struct task_struct *task)
-{
-	return task_euid(task);
-}
-
 #ifndef CONFIG_USER_NS
 __rust_helper uid_t rust_helper_from_kuid(struct user_namespace *to, kuid_t uid)
 {
diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs
index c42928d5a2393..19a488700bb9d 100644
--- a/rust/kernel/auxiliary.rs
+++ b/rust/kernel/auxiliary.rs
@@ -20,6 +20,7 @@
     },
     prelude::*,
     types::{
+        CovariantForLt,
         ForLt,
         ForeignOwnable,
         Opaque, //
@@ -270,18 +271,15 @@ pub fn parent(&self) -> &device::Device<device::Bound> {
         unsafe { parent.as_bound() }
     }
 
-    /// Returns a pinned reference to the registration data set by the registering (parent) driver.
+    /// Returns the stored registration data as a pinned reference.
     ///
-    /// `F` is the [`ForLt`](trait@ForLt) encoding of the data type. The returned
-    /// reference has its lifetime shortened from `'static` to `&self`'s borrow lifetime via
-    /// [`ForLt::cast_ref`].
+    /// Performs null and [`TypeId`] checks, then borrows the stored [`KBox`].
     ///
-    /// Returns [`EINVAL`] if `F` does not match the type used by the parent driver when calling
-    /// [`Registration::new()`].
+    /// # Safety
     ///
-    /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was
-    /// registered by a C driver.
-    pub fn registration_data<F: ForLt + 'static>(&self) -> Result<Pin<&F::Of<'_>>> {
+    /// Callers must ensure that the lifetime shortening from the original `'static` storage to
+    /// `'_` is sound, e.g. via an HRTB closure or [`CovariantForLt`] guarantee.
+    unsafe fn registration_data_pinned<F: ForLt + 'static>(&self) -> Result<Pin<&F::Of<'_>>> {
         // SAFETY: By the type invariant, `self.as_raw()` is a valid `struct auxiliary_device`.
         let ptr = unsafe { (*self.as_raw()).registration_data_rust };
         if ptr.is_null() {
@@ -300,17 +298,59 @@ pub fn registration_data<F: ForLt + 'static>(&self) -> Result<Pin<&F::Of<'_>>> {
             return Err(EINVAL);
         }
 
-        // SAFETY: The `TypeId` check above confirms that the stored type matches
-        // `F::Of<'static>`; `ptr` remains valid until `Registration::drop()` calls
-        // `from_foreign()`.
-        let wrapper = unsafe { Pin::<KBox<RegistrationData<F::Of<'static>>>>::borrow(ptr) };
+        // SAFETY: The `TypeId` check above confirms that the stored type matches `F`'s
+        // encoding; lifetimes are erased at runtime, so borrowing as `F::Of<'_>` is
+        // layout-compatible with the stored `F::Of<'static>`. `ptr` remains valid until
+        // `Registration::drop()` calls `from_foreign()`.
+        let wrapper = unsafe { Pin::<KBox<RegistrationData<F::Of<'_>>>>::borrow(ptr) };
 
         // SAFETY: `data` is a structurally pinned field of `RegistrationData`.
-        let pinned: Pin<&F::Of<'_>> = unsafe { wrapper.map_unchecked(|w| &w.data) };
+        Ok(unsafe { wrapper.map_unchecked(|w| &w.data) })
+    }
 
-        // SAFETY: The data was pinned when stored; `cast_ref` only shortens
-        // the lifetime, so the pinning guarantee is preserved.
-        Ok(unsafe { Pin::new_unchecked(F::cast_ref(pinned.get_ref())) })
+    /// Access the registration data set by the registering (parent) driver through a closure.
+    ///
+    /// `F` is the [`ForLt`](trait@ForLt) encoding of the data type. The closure receives a pinned
+    /// reference to the registration data.
+    ///
+    /// For covariant types that implement [`trait@CovariantForLt`], prefer
+    /// [`registration_data`](Self::registration_data) which returns a direct reference.
+    ///
+    /// Returns [`EINVAL`] if `F` does not match the type used by the parent driver when calling
+    /// [`Registration::new()`].
+    ///
+    /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was
+    /// registered by a C driver.
+    #[inline]
+    pub fn registration_data_with<F: ForLt + 'static, R>(
+        &self,
+        f: impl for<'a> FnOnce(Pin<&'a F::Of<'a>>) -> R,
+    ) -> Result<R> {
+        // SAFETY: The HRTB closure prevents the caller from smuggling in references with a
+        // concrete short lifetime, making the round-trip from `'static` sound regardless of
+        // variance.
+        let pinned = unsafe { self.registration_data_pinned::<F>()? };
+
+        Ok(f(pinned))
+    }
+
+    /// Returns a pinned reference to the registration data set by the registering (parent) driver.
+    ///
+    /// This method is only available when `F` implements [`trait@CovariantForLt`], which guarantees
+    /// that the lifetime shortening is sound.
+    ///
+    /// For non-covariant types, use the closure-based [`Self::registration_data_with`].
+    ///
+    /// Returns [`EINVAL`] if `F` does not match the type used by the parent driver when calling
+    /// [`Registration::new()`].
+    ///
+    /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was
+    /// registered by a C driver.
+    #[inline]
+    pub fn registration_data<F: CovariantForLt + 'static>(&self) -> Result<Pin<&F::Of<'_>>> {
+        // SAFETY: `CovariantForLt` guarantees covariance, which makes the lifetime shortening
+        // from `'static` to `'_` performed by `registration_data_pinned` sound.
+        unsafe { self.registration_data_pinned::<F>() }
     }
 }
 
@@ -401,7 +441,9 @@ struct RegistrationData<T> {
 ///
 /// The type parameter `F` is a [`ForLt`](trait@ForLt) encoding of the registration
 /// data type. For non-lifetime-parameterized types, use [`ForLt!(T)`](macro@ForLt).
-/// The data can be accessed by the auxiliary driver through [`Device::registration_data()`].
+///
+/// The data can be accessed by the auxiliary driver through [`Device::registration_data()`] and
+/// [`Device::registration_data_with()`].
 ///
 /// # Invariants
 ///
diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs
index 58ac04c650a11..d624cccb65402 100644
--- a/rust/kernel/cpufreq.rs
+++ b/rust/kernel/cpufreq.rs
@@ -792,7 +792,13 @@ fn fast_switch(_policy: &mut Policy, _target_freq: u32) -> u32 {
     }
 
     /// Driver's `adjust_perf` callback.
-    fn adjust_perf(_policy: &mut Policy, _min_perf: usize, _target_perf: usize, _capacity: usize) {
+    fn adjust_perf(
+        _policy: &mut Policy,
+        _min_perf: usize,
+        _target_perf: usize,
+        _max_perf: usize,
+        _capacity: usize,
+    ) {
         build_error!(VTABLE_DEFAULT_ERROR)
     }
 
@@ -1263,12 +1269,13 @@ impl<T: Driver> Registration<T> {
         ptr: *mut bindings::cpufreq_policy,
         min_perf: c_ulong,
         target_perf: c_ulong,
+        max_perf: c_ulong,
         capacity: c_ulong,
     ) {
         // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the
         // lifetime of `policy`.
         let policy = unsafe { Policy::from_raw_mut(ptr) };
-        T::adjust_perf(policy, min_perf, target_perf, capacity);
+        T::adjust_perf(policy, min_perf, target_perf, max_perf, capacity);
     }
 
     /// Driver's `get_intermediate` callback.
diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs
index 645afc49a27d6..1a38b3bbdfb7d 100644
--- a/rust/kernel/device.rs
+++ b/rust/kernel/device.rs
@@ -708,9 +708,7 @@ macro_rules! impl_device_context_into_aref {
 #[macro_export]
 macro_rules! dev_printk {
     ($method:ident, $dev:expr, $($f:tt)*) => {
-        {
-            $crate::device::Device::$method($dev.as_ref(), $crate::prelude::fmt!($($f)*))
-        }
+        $crate::device::Device::$method($dev.as_ref(), $crate::prelude::fmt!($($f)*))
     }
 }
 
@@ -737,7 +735,7 @@ macro_rules! dev_printk {
 /// ```
 #[macro_export]
 macro_rules! dev_emerg {
-    ($($f:tt)*) => { $crate::dev_printk!(pr_emerg, $($f)*); }
+    ($($f:tt)*) => { $crate::dev_printk!(pr_emerg, $($f)*) }
 }
 
 /// Prints an alert-level message (level 1) prefixed with device information.
@@ -763,7 +761,7 @@ macro_rules! dev_emerg {
 /// ```
 #[macro_export]
 macro_rules! dev_alert {
-    ($($f:tt)*) => { $crate::dev_printk!(pr_alert, $($f)*); }
+    ($($f:tt)*) => { $crate::dev_printk!(pr_alert, $($f)*) }
 }
 
 /// Prints a critical-level message (level 2) prefixed with device information.
@@ -789,7 +787,7 @@ macro_rules! dev_alert {
 /// ```
 #[macro_export]
 macro_rules! dev_crit {
-    ($($f:tt)*) => { $crate::dev_printk!(pr_crit, $($f)*); }
+    ($($f:tt)*) => { $crate::dev_printk!(pr_crit, $($f)*) }
 }
 
 /// Prints an error-level message (level 3) prefixed with device information.
@@ -815,7 +813,7 @@ macro_rules! dev_crit {
 /// ```
 #[macro_export]
 macro_rules! dev_err {
-    ($($f:tt)*) => { $crate::dev_printk!(pr_err, $($f)*); }
+    ($($f:tt)*) => { $crate::dev_printk!(pr_err, $($f)*) }
 }
 
 /// Prints a warning-level message (level 4) prefixed with device information.
@@ -841,7 +839,7 @@ macro_rules! dev_err {
 /// ```
 #[macro_export]
 macro_rules! dev_warn {
-    ($($f:tt)*) => { $crate::dev_printk!(pr_warn, $($f)*); }
+    ($($f:tt)*) => { $crate::dev_printk!(pr_warn, $($f)*) }
 }
 
 /// Prints a notice-level message (level 5) prefixed with device information.
@@ -867,7 +865,7 @@ macro_rules! dev_warn {
 /// ```
 #[macro_export]
 macro_rules! dev_notice {
-    ($($f:tt)*) => { $crate::dev_printk!(pr_notice, $($f)*); }
+    ($($f:tt)*) => { $crate::dev_printk!(pr_notice, $($f)*) }
 }
 
 /// Prints an info-level message (level 6) prefixed with device information.
@@ -893,7 +891,7 @@ macro_rules! dev_notice {
 /// ```
 #[macro_export]
 macro_rules! dev_info {
-    ($($f:tt)*) => { $crate::dev_printk!(pr_info, $($f)*); }
+    ($($f:tt)*) => { $crate::dev_printk!(pr_info, $($f)*) }
 }
 
 /// Prints a debug-level message (level 7) prefixed with device information.
@@ -919,5 +917,5 @@ macro_rules! dev_info {
 /// ```
 #[macro_export]
 macro_rules! dev_dbg {
-    ($($f:tt)*) => { $crate::dev_printk!(pr_dbg, $($f)*); }
+    ($($f:tt)*) => { $crate::dev_printk!(pr_dbg, $($f)*) }
 }
diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs
index 11ce500e9b76f..ebb5c19e851e6 100644
--- a/rust/kernel/devres.rs
+++ b/rust/kernel/devres.rs
@@ -21,9 +21,12 @@
     sync::{
         aref::ARef,
         rcu,
-        Arc, //
+        Arc,
+        Completion, //
     },
     types::{
+        CovariantForLt,
+        ForLt,
         ForeignOwnable,
         Opaque, //
     },
@@ -37,6 +40,8 @@ struct Inner<T> {
     node: Opaque<bindings::devres_node>,
     #[pin]
     data: Revocable<T>,
+    #[pin]
+    revocation: Completion,
 }
 
 /// This abstraction is meant to be used by subsystems to containerize [`Device`] bound resources to
@@ -53,6 +58,10 @@ struct Inner<T> {
 /// After the [`Devres`] has been unbound it is not possible to access the encapsulated resource
 /// anymore.
 ///
+/// When a [`Devres`] is dropped, it is guaranteed that `T` has been fully dropped by the time
+/// [`Devres::drop`] returns, even if a concurrent revocation through the release callback is in
+/// progress.
+///
 /// [`Devres`] users should make sure to simply free the corresponding backing resource in `T`'s
 /// [`Drop`] implementation.
 ///
@@ -68,17 +77,19 @@ struct Inner<T> {
 ///     devres::Devres,
 ///     io::{
 ///         Io,
-///         IoKnownSize,
+///         IoBase,
 ///         Mmio,
 ///         MmioRaw,
-///         PhysAddr, //
+///         MmioBackend,
+///         PhysAddr,
+///         Region, //
 ///     },
 ///     prelude::*,
 /// };
 /// use core::ops::Deref;
 ///
 /// // See also [`pci::Bar`] for a real example.
-/// struct IoMem<const SIZE: usize>(MmioRaw<SIZE>);
+/// struct IoMem<const SIZE: usize>(MmioRaw<Region<SIZE>>);
 ///
 /// impl<const SIZE: usize> IoMem<SIZE> {
 ///     /// # Safety
@@ -93,7 +104,7 @@ struct Inner<T> {
 ///             return Err(ENOMEM);
 ///         }
 ///
-///         Ok(IoMem(MmioRaw::new(addr as usize, SIZE)?))
+///         Ok(IoMem(MmioRaw::new_region(addr as usize, SIZE)?))
 ///     }
 /// }
 ///
@@ -104,12 +115,13 @@ struct Inner<T> {
 ///     }
 /// }
 ///
-/// impl<const SIZE: usize> Deref for IoMem<SIZE> {
-///    type Target = Mmio<SIZE>;
+/// impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem<SIZE> {
+///    type Backend = MmioBackend;
+///    type Target = Region<SIZE>;
 ///
-///    fn deref(&self) -> &Self::Target {
+///    fn as_view(self) -> Mmio<'a, Region<SIZE>> {
 ///         // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`.
-///         unsafe { Mmio::from_raw(&self.0) }
+///         unsafe { Mmio::from_raw(self.0) }
 ///    }
 /// }
 /// # fn no_run(dev: &Device<Bound>) -> Result<(), Error> {
@@ -217,6 +229,7 @@ pub fn new<E>(dev: &Device<Bound>, data: impl PinInit<T, E>) -> Result<Self>
                     };
                 }),
                 data <- Revocable::new(data),
+                revocation <- Completion::new(),
             }),
             GFP_KERNEL,
         )?;
@@ -254,7 +267,14 @@ fn data(&self) -> &Revocable<T> {
         // SAFETY: `inner` is a valid `Inner<T>` pointer.
         let inner = unsafe { &*inner };
 
-        inner.data.revoke();
+        if inner.data.revoke() {
+            inner.revocation.complete_all();
+        } else {
+            // Devres::drop() is concurrently revoking; wait for it to finish `drop_in_place()`
+            // before returning to `devres_release_all()`, ensuring `T` is fully torn down before
+            // the device finishes unbinding.
+            inner.revocation.wait_for_completion();
+        }
     }
 
     #[allow(clippy::missing_safety_doc)]
@@ -297,10 +317,7 @@ pub fn device(&self) -> &Device {
     /// use kernel::{
     ///     device::Core,
     ///     devres::Devres,
-    ///     io::{
-    ///         Io,
-    ///         IoKnownSize, //
-    ///     },
+    ///     io::Io,
     ///     pci, //
     /// };
     ///
@@ -354,6 +371,8 @@ fn drop(&mut self) {
         // SAFETY: When `drop` runs, it is guaranteed that nobody is accessing the revocable data
         // anymore, hence it is safe not to wait for the grace period to finish.
         if unsafe { self.data().revoke_nosync() } {
+            self.inner.revocation.complete_all();
+
             // We revoked `self.data` before devres did, hence try to remove it.
             if self.remove_node() {
                 // SAFETY: In `Self::new` we have taken an additional reference count of `self.data`
@@ -361,10 +380,118 @@ fn drop(&mut self) {
                 // this additional reference count.
                 drop(unsafe { Arc::from_raw(Arc::as_ptr(&self.inner)) });
             }
+        } else {
+            // The release callback is concurrently revoking; wait for it to finish
+            // `drop_in_place()` of the wrapped object before returning.
+            self.inner.revocation.wait_for_completion();
         }
     }
 }
 
+/// Guard returned by [`DevresLt::try_access`].
+///
+/// Dereferences to `F::Of<'a>`, shortening the lifetime of the stored data to the guard's borrow
+/// lifetime.
+pub struct DevresGuard<'a, F: CovariantForLt>(RevocableGuard<'a, F::Of<'static>>);
+
+impl<'a, F: CovariantForLt> core::ops::Deref for DevresGuard<'a, F> {
+    type Target = F::Of<'a>;
+
+    #[inline]
+    fn deref(&self) -> &Self::Target {
+        F::cast_ref(&*self.0)
+    }
+}
+
+/// Device-managed resource with [`ForLt`](trait@ForLt)-aware access.
+///
+/// `DevresLt` wraps [`Devres`] and shortens the stored `'static` lifetime to the caller's borrow
+/// lifetime in all access methods.
+///
+/// Types that implement [`trait@CovariantForLt`] get direct-reference accessors ([`Self::access`],
+/// [`Self::try_access`]). Plain [`ForLt`](trait@ForLt) types use closure-based accessors
+/// ([`Self::access_with`], [`Self::try_access_with`]).
+pub struct DevresLt<F: ForLt>(Devres<F::Of<'static>>)
+where
+    for<'a> F::Of<'a>: Send;
+
+impl<F: ForLt> DevresLt<F>
+where
+    for<'a> F::Of<'a>: Send,
+{
+    /// Creates a new [`DevresLt`] instance of the given `data`.
+    ///
+    /// # Safety
+    ///
+    /// The data must remain valid for the device's full bound scope. [`DevresLt`] allows
+    /// access until the device is unbound, which may outlast `'a`.
+    pub unsafe fn new<'a, E>(
+        dev: &'a Device<Bound>,
+        data: impl PinInit<F::Of<'a>, E>,
+    ) -> Result<Self>
+    where
+        Error: From<E>,
+    {
+        // SAFETY: The caller guarantees the data is valid for the device's full bound scope.
+        // Lifetimes do not affect layout, so F::Of<'a> and F::Of<'static> have identical
+        // representation; casting the slot pointer is sound.
+        let data = unsafe {
+            pin_init::pin_init_from_closure::<F::Of<'static>, E>(move |slot| {
+                data.__pinned_init(slot.cast())
+            })
+        };
+
+        Ok(Self(Devres::new(dev, data)?))
+    }
+
+    /// Return a reference of the [`Device`] this [`DevresLt`] instance has been created with.
+    #[inline]
+    pub fn device(&self) -> &Device {
+        self.0.device()
+    }
+
+    /// Obtain `&F::Of<'_>`, bypassing the [`Revocable`], through a closure.
+    ///
+    /// This method works like [`DevresLt::access`](DevresLt::access) but accepts any
+    /// [`trait@ForLt`] type, not just [`trait@CovariantForLt`].
+    #[inline]
+    pub fn access_with<R, G>(&self, dev: &Device<Bound>, f: G) -> Result<R>
+    where
+        G: for<'a> FnOnce(&F::Of<'a>) -> R,
+    {
+        self.0.access(dev).map(f)
+    }
+
+    /// [`DevresLt`] accessor for [`Revocable::try_access_with`].
+    #[inline]
+    pub fn try_access_with<R, G>(&self, f: G) -> Option<R>
+    where
+        G: for<'a> FnOnce(&F::Of<'a>) -> R,
+    {
+        self.0.data().try_access_with(f)
+    }
+}
+
+impl<F: CovariantForLt> DevresLt<F>
+where
+    for<'a> F::Of<'a>: Send,
+{
+    /// Obtain `&'a F::Of<'a>`, bypassing the [`Revocable`].
+    ///
+    /// This method works like [`Devres::access`], but shortens the returned reference's lifetime
+    /// from `'static` to `'a` via [`CovariantForLt::cast_ref`].
+    #[inline]
+    pub fn access<'a>(&'a self, dev: &'a Device<Bound>) -> Result<&'a F::Of<'a>> {
+        self.0.access(dev).map(F::cast_ref)
+    }
+
+    /// [`DevresLt`] accessor for [`Revocable::try_access`].
+    #[inline]
+    pub fn try_access(&self) -> Option<DevresGuard<'_, F>> {
+        self.0.data().try_access().map(DevresGuard)
+    }
+}
+
 /// Consume `data` and [`Drop::drop`] `data` once `dev` is unbound.
 fn register_foreign<P>(dev: &Device<Bound>, data: P) -> Result
 where
diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs
index 200def84fb69e..e275f2562a5bb 100644
--- a/rust/kernel/dma.rs
+++ b/rust/kernel/dma.rs
@@ -14,14 +14,22 @@
     },
     error::to_result,
     fs::file,
+    io::{
+        IoBackend,
+        IoBase,
+        IoCapable,
+        IoCopyable,
+        SysMem,
+        SysMemBackend, //
+    },
     prelude::*,
     ptr::KnownSize,
     sync::aref::ARef,
     transmute::{
         AsBytes,
         FromBytes, //
-    }, //
-    uaccess::UserSliceWriter,
+    },
+    uaccess::UserSliceWriter, //
 };
 use core::{
     ops::{
@@ -654,52 +662,6 @@ pub unsafe fn as_mut(&self) -> &mut T {
         // SAFETY: per safety requirement.
         unsafe { &mut *self.as_mut_ptr() }
     }
-
-    /// Reads the value of `field` and ensures that its type is [`FromBytes`].
-    ///
-    /// # Safety
-    ///
-    /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is
-    /// validated beforehand.
-    ///
-    /// Public but hidden since it should only be used from [`dma_read`] macro.
-    #[doc(hidden)]
-    pub unsafe fn field_read<F: FromBytes>(&self, field: *const F) -> F {
-        // SAFETY:
-        // - By the safety requirements field is valid.
-        // - Using read_volatile() here is not sound as per the usual rules, the usage here is
-        // a special exception with the following notes in place. When dealing with a potential
-        // race from a hardware or code outside kernel (e.g. user-space program), we need that
-        // read on a valid memory is not UB. Currently read_volatile() is used for this, and the
-        // rationale behind is that it should generate the same code as READ_ONCE() which the
-        // kernel already relies on to avoid UB on data races. Note that the usage of
-        // read_volatile() is limited to this particular case, it cannot be used to prevent
-        // the UB caused by racing between two kernel functions nor do they provide atomicity.
-        unsafe { field.read_volatile() }
-    }
-
-    /// Writes a value to `field` and ensures that its type is [`AsBytes`].
-    ///
-    /// # Safety
-    ///
-    /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is
-    /// validated beforehand.
-    ///
-    /// Public but hidden since it should only be used from [`dma_write`] macro.
-    #[doc(hidden)]
-    pub unsafe fn field_write<F: AsBytes>(&self, field: *mut F, val: F) {
-        // SAFETY:
-        // - By the safety requirements field is valid.
-        // - Using write_volatile() here is not sound as per the usual rules, the usage here is
-        // a special exception with the following notes in place. When dealing with a potential
-        // race from a hardware or code outside kernel (e.g. user-space program), we need that
-        // write on a valid memory is not UB. Currently write_volatile() is used for this, and the
-        // rationale behind is that it should generate the same code as WRITE_ONCE() which the
-        // kernel already relies on to avoid UB on data races. Note that the usage of
-        // write_volatile() is limited to this particular case, it cannot be used to prevent
-        // the UB caused by racing between two kernel functions nor do they provide atomicity.
-        unsafe { field.write_volatile(val) }
-    }
 }
 
 impl<T: AsBytes + FromBytes> Coherent<T> {
@@ -1133,84 +1095,153 @@ unsafe impl Send for CoherentHandle {}
 // plain `Copy` values.
 unsafe impl Sync for CoherentHandle {}
 
-/// Reads a field of an item from an allocated region of structs.
+/// View type for `Coherent`.
 ///
-/// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating
-/// to a [`Coherent`] and `proj` is a [projection specification](kernel::ptr::project!).
-///
-/// # Examples
-///
-/// ```
-/// use kernel::device::Device;
-/// use kernel::dma::{attrs::*, Coherent};
-///
-/// struct MyStruct { field: u32, }
-///
-/// // SAFETY: All bit patterns are acceptable values for `MyStruct`.
-/// unsafe impl kernel::transmute::FromBytes for MyStruct{};
-/// // SAFETY: Instances of `MyStruct` have no uninitialized portions.
-/// unsafe impl kernel::transmute::AsBytes for MyStruct{};
-///
-/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result {
-/// let whole = kernel::dma_read!(alloc, [try: 2]);
-/// let field = kernel::dma_read!(alloc, [panic: 1].field);
-/// # Ok::<(), Error>(()) }
-/// ```
-#[macro_export]
-macro_rules! dma_read {
-    ($dma:expr, $($proj:tt)*) => {{
-        let dma = &$dma;
-        let ptr = $crate::ptr::project!(
-            $crate::dma::Coherent::as_ptr(dma), $($proj)*
-        );
-        // SAFETY: The pointer created by the projection is within the DMA region.
-        unsafe { $crate::dma::Coherent::field_read(dma, ptr) }
-    }};
+/// This is same as [`SysMem`] but with additional information that allows handing out a DMA handle.
+pub struct CoherentView<'a, T: ?Sized> {
+    cpu_addr: SysMem<'a, T>,
+    dma_handle: DmaAddress,
 }
 
-/// Writes to a field of an item from an allocated region of structs.
-///
-/// The syntax is of the form `kernel::dma_write!(dma, proj, val)` where `dma` is an expression
-/// evaluating to a [`Coherent`], `proj` is a
-/// [projection specification](kernel::ptr::project!), and `val` is the value to be written to the
-/// projected location.
-///
-/// # Examples
-///
-/// ```
-/// use kernel::device::Device;
-/// use kernel::dma::{attrs::*, Coherent};
-///
-/// struct MyStruct { member: u32, }
-///
-/// // SAFETY: All bit patterns are acceptable values for `MyStruct`.
-/// unsafe impl kernel::transmute::FromBytes for MyStruct{};
-/// // SAFETY: Instances of `MyStruct` have no uninitialized portions.
-/// unsafe impl kernel::transmute::AsBytes for MyStruct{};
-///
-/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result {
-/// kernel::dma_write!(alloc, [try: 2].member, 0xf);
-/// kernel::dma_write!(alloc, [panic: 1], MyStruct { member: 0xf });
-/// # Ok::<(), Error>(()) }
-/// ```
-#[macro_export]
-macro_rules! dma_write {
-    (@parse [$dma:expr] [$($proj:tt)*] [, $val:expr]) => {{
-        let dma = &$dma;
-        let ptr = $crate::ptr::project!(
-            mut $crate::dma::Coherent::as_mut_ptr(dma), $($proj)*
-        );
-        let val = $val;
-        // SAFETY: The pointer created by the projection is within the DMA region.
-        unsafe { $crate::dma::Coherent::field_write(dma, ptr, val) }
-    }};
-    (@parse [$dma:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => {
-        $crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*])
-    };
-    (@parse [$dma:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => {
-        $crate::dma_write!(@parse [$dma] [$($proj)* [$flavor: $index]] [$($rest)*])
-    };
-    ($dma:expr, $($rest:tt)*) => {
-        $crate::dma_write!(@parse [$dma] [] [$($rest)*])
-    };
+impl<T: ?Sized> Copy for CoherentView<'_, T> {}
+impl<T: ?Sized> Clone for CoherentView<'_, T> {
+    #[inline]
+    fn clone(&self) -> Self {
+        *self
+    }
+}
+
+impl<'a, T: ?Sized> CoherentView<'a, T> {
+    /// Erase the DMA handle information and obtain a [`SysMem`] view of the same memory region.
+    #[inline]
+    pub fn as_sys_mem(self) -> SysMem<'a, T> {
+        self.cpu_addr
+    }
+
+    /// Returns a DMA handle which may be given to the device as the DMA address base of the region.
+    #[inline]
+    pub fn dma_handle(self) -> DmaAddress {
+        self.dma_handle
+    }
+
+    /// Returns a reference to the data in the region.
+    ///
+    /// # Safety
+    ///
+    /// * Callers must ensure that the device does not read/write to/from memory while the returned
+    ///   reference is live.
+    /// * Callers must ensure that this call does not race with a write (including call to `as_mut`)
+    ///   to the same region while the returned reference is live.
+    #[inline]
+    pub unsafe fn as_ref(self) -> &'a T {
+        // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per
+        // safety requirement.
+        unsafe { &*self.cpu_addr.as_ptr() }
+    }
+
+    /// Returns a mutable reference to the data in the region.
+    ///
+    /// # Safety
+    ///
+    /// * Callers must ensure that the device does not read/write to/from memory while the returned
+    ///   reference is live.
+    /// * Callers must ensure that this call does not race with a read (including call to `as_ref`)
+    ///   or write (including call to `as_mut`) to the same region while the returned reference is
+    ///   live.
+    #[inline]
+    pub unsafe fn as_mut(self) -> &'a mut T {
+        // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per
+        // safety requirement.
+        unsafe { &mut *self.cpu_addr.as_ptr() }
+    }
+}
+
+/// `IoBackend` implementation for `Coherent`.
+pub struct CoherentIoBackend;
+
+impl IoBackend for CoherentIoBackend {
+    type View<'a, T: ?Sized + KnownSize> = CoherentView<'a, T>;
+
+    #[inline]
+    fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
+        SysMemBackend::as_ptr(view.cpu_addr)
+    }
+
+    #[inline]
+    unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
+        view: Self::View<'a, T>,
+        ptr: *mut U,
+    ) -> Self::View<'a, U> {
+        let offset = ptr.addr() - view.cpu_addr.as_ptr().addr();
+        // CAST: The offset DMA address can never overflow.
+        let dma_handle = view.dma_handle + offset as DmaAddress;
+        CoherentView {
+            dma_handle,
+            // SAFETY: Per safety requirement.
+            cpu_addr: unsafe { SysMemBackend::project_view(view.cpu_addr, ptr) },
+        }
+    }
+}
+
+impl<T> IoCapable<T> for CoherentIoBackend
+where
+    SysMemBackend: IoCapable<T>,
+{
+    #[inline]
+    fn io_read<'a>(view: Self::View<'a, T>) -> T {
+        SysMemBackend::io_read(view.cpu_addr)
+    }
+
+    #[inline]
+    fn io_write<'a>(view: Self::View<'a, T>, value: T) {
+        SysMemBackend::io_write(view.cpu_addr, value)
+    }
+}
+
+impl IoCopyable for CoherentIoBackend {
+    #[inline]
+    unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
+        // SAFETY: Per safety requirement.
+        unsafe { SysMemBackend::copy_from_io(view.cpu_addr, buffer) }
+    }
+
+    #[inline]
+    unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
+        // SAFETY: Per safety requirement.
+        unsafe { SysMemBackend::copy_to_io(view.cpu_addr, buffer) }
+    }
+
+    #[inline]
+    fn copy_read<T: zerocopy::FromBytes>(view: Self::View<'_, T>) -> T {
+        SysMemBackend::copy_read(view.cpu_addr)
+    }
+
+    #[inline]
+    fn copy_write<T: zerocopy::IntoBytes>(view: Self::View<'_, T>, value: T) {
+        SysMemBackend::copy_write(view.cpu_addr, value)
+    }
+}
+
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for CoherentView<'a, T> {
+    type Backend = CoherentIoBackend;
+    type Target = T;
+
+    #[inline]
+    fn as_view(self) -> CoherentView<'a, Self::Target> {
+        self
+    }
+}
+
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent<T> {
+    type Backend = CoherentIoBackend;
+    type Target = T;
+
+    #[inline]
+    fn as_view(self) -> CoherentView<'a, Self::Target> {
+        CoherentView {
+            // SAFETY: `cpu_addr` is valid and aligned kernel accessible memory.
+            cpu_addr: unsafe { SysMem::new(self.cpu_addr.as_ptr()) },
+            dma_handle: self.dma_handle,
+        }
+    }
 }
diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs
index 477cf771fb10e..f43c6887ad236 100644
--- a/rust/kernel/drm/device.rs
+++ b/rust/kernel/drm/device.rs
@@ -32,6 +32,7 @@
 };
 use core::{
     alloc::Layout,
+    cell::UnsafeCell,
     marker::PhantomData,
     mem,
     ops::Deref,
@@ -74,66 +75,59 @@ macro_rules! drm_legacy_fields {
 
 /// A trait implemented by all possible contexts a [`Device`] can be used in.
 ///
-/// Setting up a new [`Device`] is a multi-stage process. Each step of the process that a user
-/// interacts with in Rust has a respective [`DeviceContext`] typestate. For example,
-/// `Device<T, Registered>` would be a [`Device`] that reached the [`Registered`] [`DeviceContext`].
+/// A [`Device`] can be in one of the following contexts:
 ///
-/// Each stage of this process is described below:
+/// - [`Normal`]: The general-purpose, reference-counted context. A [`Device`] in this context may
+///   or may not be registered with userspace.
+/// - [`Ioctl`]: The device has been registered with userspace at some point; used in ioctl
+///   dispatch context.
+/// - [`Registered`]: The device is currently registered with userspace and the parent bus device
+///   is bound.
 ///
-/// ```text
-///        1                     2                         3
-/// +--------------+   +------------------+   +-----------------------+
-/// |Device created| → |Device initialized| → |Registered w/ userspace|
-/// +--------------+   +------------------+   +-----------------------+
-///    (Uninit)                                      (Registered)
-/// ```
-///
-/// 1. The [`Device`] is in the [`Uninit`] context and is not guaranteed to be initialized or
-///    registered with userspace. Only a limited subset of DRM core functionality is available.
-/// 2. The [`Device`] is guaranteed to be fully initialized, but is not guaranteed to be registered
-///    with userspace. All DRM core functionality which doesn't interact with userspace is
-///    available. We currently don't have a context for representing this.
-/// 3. The [`Device`] is guaranteed to be fully initialized, and is guaranteed to have been
-///    registered with userspace at some point - thus putting it in the [`Registered`] context.
+/// Both `Device<T, Ioctl>` and `Device<T, Registered>` dereference to `Device<T>` ([`Normal`]),
+/// so any method available on a [`Normal`] device is also available in the other contexts.
+pub trait DeviceContext: Sealed + Send + Sync + 'static {}
+
+/// The general-purpose, reference-counted [`DeviceContext`].
 ///
-/// An important caveat of [`DeviceContext`] which must be kept in mind: when used as a typestate
-/// for a reference type, it can only guarantee that a [`Device`] reached a particular stage in the
-/// initialization process _at the time the reference was taken_. No guarantee is made in regards to
-/// what stage of the process the [`Device`] is currently in. This means for instance that a
-/// `&Device<T, Uninit>` may actually be registered with userspace, it just wasn't known to be
-/// registered at the time the reference was taken.
-pub trait DeviceContext: Sealed + Send + Sync {}
-
-/// The [`DeviceContext`] of a [`Device`] that was registered with userspace at some point.
+/// A [`Device`] in this context may or may not be registered with userspace. This context is used
+/// for reference-counted device handles and during device setup via [`UnregisteredDevice`].
 ///
-/// This represents a [`Device`] which is guaranteed to have been registered with userspace at
-/// some point in time. Such a DRM device is guaranteed to have been fully-initialized.
+/// [`AlwaysRefCounted`] is only implemented for `Device<T, Normal>`, making this the required
+/// context for [`ARef`]-based device handles.
+pub struct Normal;
+
+impl Sealed for Normal {}
+impl DeviceContext for Normal {}
+
+/// The [`DeviceContext`] of a [`Device`] that is currently registered with userspace.
 ///
-/// Note: A device in this context is not guaranteed to remain registered with userspace for its
-/// entire lifetime, as this is impossible to guarantee at compile-time.
+/// A [`Device`] in this context is guaranteed to be registered and its parent bus device is
+/// guaranteed to be bound. This is enforced at runtime by [`RegistrationGuard`], which holds a
+/// `drm_dev_enter()` / `drm_dev_exit()` SRCU critical section.
 ///
 /// # Invariants
 ///
-/// A [`Device`] in this [`DeviceContext`] is guaranteed to have been registered with userspace
-/// at some point in time.
+/// The parent bus device is bound for the duration of any reference to a `Device<T, Registered>`.
 pub struct Registered;
 
 impl Sealed for Registered {}
 impl DeviceContext for Registered {}
 
-/// The [`DeviceContext`] of a [`Device`] that may be unregistered and partly uninitialized.
+/// The [`DeviceContext`] of a [`Device`] that has been registered with userspace previously.
 ///
-/// A [`Device`] in this context is only guaranteed to be partly initialized, and may or may not
-/// be registered with userspace. Thus operations which depend on the [`Device`] being fully
-/// initialized, or which depend on the [`Device`] being registered with userspace are not
-/// available through this [`DeviceContext`].
+/// A [`Device`] in this context has been registered at some point, but may be concurrently
+/// unregistering or already unregistered. `drm_dev_enter()` can guard against this, ensuring the
+/// device remains registered for the duration of the critical section.
 ///
-/// A [`Device`] in this context can be used to create a
-/// [`Registration`](drm::driver::Registration).
-pub struct Uninit;
+/// # Invariants
+///
+/// A [`Device`] in this context has been registered with userspace via `drm_dev_register()` at
+/// some point.
+pub struct Ioctl;
 
-impl Sealed for Uninit {}
-impl DeviceContext for Uninit {}
+impl Sealed for Ioctl {}
+impl DeviceContext for Ioctl {}
 
 /// A [`Device`] which is known at compile-time to be unregistered with userspace.
 ///
@@ -147,10 +141,10 @@ impl DeviceContext for Uninit {}
 ///
 /// The device in `self.0` is guaranteed to be a newly created [`Device`] that has not yet been
 /// registered with userspace until this type is dropped.
-pub struct UnregisteredDevice<T: drm::Driver>(ARef<Device<T, Uninit>>, NotThreadSafe);
+pub struct UnregisteredDevice<T: drm::Driver>(ARef<Device<T, Normal>>, NotThreadSafe);
 
 impl<T: drm::Driver> Deref for UnregisteredDevice<T> {
-    type Target = Device<T, Uninit>;
+    type Target = Device<T, Normal>;
 
     fn deref(&self) -> &Self::Target {
         &self.0
@@ -178,15 +172,13 @@ const fn compute_features() -> u32 {
         master_drop: None,
         debugfs_init: None,
 
-        // Ignore the Uninit DeviceContext below. It is only provided because it is required by the
-        // compiler, and it is not actually used by these functions.
-        gem_create_object: T::Object::<Uninit>::ALLOC_OPS.gem_create_object,
-        prime_handle_to_fd: T::Object::<Uninit>::ALLOC_OPS.prime_handle_to_fd,
-        prime_fd_to_handle: T::Object::<Uninit>::ALLOC_OPS.prime_fd_to_handle,
-        gem_prime_import: T::Object::<Uninit>::ALLOC_OPS.gem_prime_import,
-        gem_prime_import_sg_table: T::Object::<Uninit>::ALLOC_OPS.gem_prime_import_sg_table,
-        dumb_create: T::Object::<Uninit>::ALLOC_OPS.dumb_create,
-        dumb_map_offset: T::Object::<Uninit>::ALLOC_OPS.dumb_map_offset,
+        gem_create_object: T::Object::ALLOC_OPS.gem_create_object,
+        prime_handle_to_fd: T::Object::ALLOC_OPS.prime_handle_to_fd,
+        prime_fd_to_handle: T::Object::ALLOC_OPS.prime_fd_to_handle,
+        gem_prime_import: T::Object::ALLOC_OPS.gem_prime_import,
+        gem_prime_import_sg_table: T::Object::ALLOC_OPS.gem_prime_import_sg_table,
+        dumb_create: T::Object::ALLOC_OPS.dumb_create,
+        dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset,
 
         show_fdinfo: None,
         fbdev_probe: None,
@@ -208,10 +200,13 @@ const fn compute_features() -> u32 {
     /// Create a new `UnregisteredDevice` for a `drm::Driver`.
     ///
     /// This can be used to create a [`Registration`](kernel::drm::Registration).
-    pub fn new(dev: &device::Device, data: impl PinInit<T::Data, Error>) -> Result<Self> {
+    pub fn new(
+        dev: &T::ParentDevice<device::Bound>,
+        data: impl PinInit<T::Data, Error>,
+    ) -> Result<Self> {
         // `__drm_dev_alloc` uses `kmalloc()` to allocate memory, hence ensure a `kmalloc()`
         // compatible `Layout`.
-        let layout = Kmalloc::aligned_layout(Layout::new::<Device<T, Uninit>>());
+        let layout = Kmalloc::aligned_layout(Layout::new::<Device<T, Normal>>());
 
         // Use a temporary vtable without a `release` callback until `data` is initialized, so
         // init failure can release the DRM device without dropping uninitialized fields.
@@ -223,12 +218,12 @@ pub fn new(dev: &device::Device, data: impl PinInit<T::Data, Error>) -> Result<S
         // SAFETY:
         // - `alloc_vtable` reference remains valid until no longer used,
         // - `dev` is valid by its type invarants,
-        let raw_drm: *mut Device<T, Uninit> = unsafe {
+        let raw_drm: *mut Device<T, Normal> = unsafe {
             bindings::__drm_dev_alloc(
-                dev.as_raw(),
+                dev.as_ref().as_raw(),
                 &alloc_vtable,
                 layout.size(),
-                mem::offset_of!(Device<T, Uninit>, dev),
+                mem::offset_of!(Device<T, Normal>, dev),
             )
         }
         .cast();
@@ -253,6 +248,9 @@ pub fn new(dev: &device::Device, data: impl PinInit<T::Data, Error>) -> Result<S
         // SAFETY: `drm_dev` is still private to this function.
         unsafe { (*drm_dev).driver = const { &Self::VTABLE } };
 
+        // SAFETY: `raw_drm` is valid; no concurrent access before registration.
+        unsafe { (*raw_drm.as_ptr()).registration_data = UnsafeCell::new(NonNull::dangling()) };
+
         // SAFETY: The reference count is one, and now we take ownership of that reference as a
         // `drm::Device`.
         // INVARIANT: We just created the device above, but have yet to call `drm_dev_register`.
@@ -264,16 +262,8 @@ pub fn new(dev: &device::Device, data: impl PinInit<T::Data, Error>) -> Result<S
 
 /// A typed DRM device with a specific [`drm::Driver`] implementation and [`DeviceContext`].
 ///
-/// Since DRM devices can be used before being fully initialized and registered with userspace, `C`
-/// represents the furthest [`DeviceContext`] we can guarantee that this [`Device`] has reached.
-///
-/// Keep in mind: this means that an unregistered device can still have the registration state
-/// [`Registered`] as long as it was registered with userspace once in the past, and that the
-/// behavior of such a device is still well-defined. Additionally, a device with the registration
-/// state [`Uninit`] simply does not have a guaranteed registration state at compile time, and could
-/// be either registered or unregistered. Since there is no way to guarantee a long-lived reference
-/// to an unregistered device would remain unregistered, we do not provide a [`DeviceContext`] for
-/// this.
+/// A device in the [`Registered`] context is currently registered with userspace and its parent
+/// bus device is bound. The [`Normal`] context is the general-purpose, reference-counted context.
 ///
 /// # Invariants
 ///
@@ -281,9 +271,10 @@ pub fn new(dev: &device::Device, data: impl PinInit<T::Data, Error>) -> Result<S
 /// * The data layout of `Self` remains the same across all implementations of `C`.
 /// * Any invariants for `C` also apply.
 #[repr(C)]
-pub struct Device<T: drm::Driver, C: DeviceContext = Registered> {
+pub struct Device<T: drm::Driver, C: DeviceContext = Normal> {
     dev: Opaque<bindings::drm_device>,
     data: T::Data,
+    pub(super) registration_data: UnsafeCell<NonNull<T::RegistrationData<'static>>>,
     _ctx: PhantomData<C>,
 }
 
@@ -352,7 +343,111 @@ pub(crate) unsafe fn assume_ctx<NewCtx: DeviceContext>(&self) -> &Device<T, NewC
     }
 }
 
-impl<T: drm::Driver, C: DeviceContext> Deref for Device<T, C> {
+impl<T: drm::Driver> Device<T, Ioctl> {
+    /// Guard against the parent bus device being unbound.
+    ///
+    /// Returns a [`RegistrationGuard`] if the device has not been unplugged, [`None`] otherwise.
+    ///
+    /// While [`RegistrationGuard`] is held the parent device is guaranteed to be bound.
+    #[must_use]
+    pub fn registration_guard(&self) -> Option<RegistrationGuard<'_, T>> {
+        let mut idx: i32 = 0;
+        // SAFETY: `self.as_raw()` is a valid pointer to a `struct drm_device`.
+        if unsafe { bindings::drm_dev_enter(self.as_raw(), &mut idx) } {
+            // INVARIANT:
+            // - `idx` is the SRCU index from the successful `drm_dev_enter()` above.
+            // - The parent bus device is bound: `drm_dev_enter()` succeeded, meaning
+            //   `drm_dev_unplug()` has not completed; since it is only called from
+            //   `Registration::drop()` during parent unbind, the parent is still bound.
+            Some(RegistrationGuard {
+                // SAFETY: See INVARIANT above; the `Registered` context invariant holds.
+                dev: unsafe { self.assume_ctx() },
+                idx,
+                _not_send: NotThreadSafe,
+            })
+        } else {
+            None
+        }
+    }
+}
+
+/// A guard proving the DRM device is registered and the parent bus device is bound.
+///
+/// The guard dereferences to [`Device<T, Registered>`], providing access to the DRM device with
+/// the guarantee that the parent bus device is bound for the entire duration of the critical
+/// section.
+///
+/// Internally this is backed by a `drm_dev_enter()` / `drm_dev_exit()` SRCU critical section.
+///
+/// # Invariants
+///
+/// - `idx` is the SRCU read lock index returned by a successful `drm_dev_enter()` call.
+/// - The parent bus device of `dev` is bound for the lifetime of this guard.
+#[must_use]
+pub struct RegistrationGuard<'a, T: drm::Driver> {
+    dev: &'a Device<T, Registered>,
+    idx: i32,
+    _not_send: NotThreadSafe,
+}
+
+impl<T: drm::Driver> Device<T, Registered> {
+    /// Returns a reference to the registration data with lifetime shortened from `'static`.
+    ///
+    /// # Safety
+    ///
+    /// The returned reference must not be exposed to code that can choose a concrete lifetime for
+    /// it, as that would be unsound for types that are invariant over their lifetime parameter
+    /// (e.g. it must be passed through an HRTB-bounded closure).
+    #[inline]
+    unsafe fn registration_data_unchecked(&self) -> &T::RegistrationData<'_> {
+        // SAFETY:
+        // - `Registered` guarantees the parent bus device is bound, hence the pointer is valid.
+        // - The pointer cast from `Of<'static>` to `Of<'_>` is layout-compatible since lifetimes
+        //   are erased at runtime.
+        // - Caller guarantees the reference is only used behind an HRTB, making the lifetime
+        //   shortening sound regardless of variance.
+        unsafe { (*self.registration_data.get()).cast::<_>().as_ref() }
+    }
+
+    /// Access the registration data through a closure, with the lifetime tied to the closure
+    /// scope.
+    ///
+    /// The data is owned by [`Registration`](drm::Registration) and is guaranteed to remain valid
+    /// as long as the device is registered, since [`Registration`](drm::Registration)'s `drop`
+    /// calls `drm_dev_unplug()` which waits for all `drm_dev_enter()` critical sections to
+    /// complete.
+    #[inline]
+    pub fn registration_data_with<R, F>(&self, f: F) -> R
+    where
+        F: for<'a> FnOnce(&'a T::RegistrationData<'a>) -> R,
+    {
+        // SAFETY: `Registered` guarantees the device is registered and the parent bus device is
+        // bound. The closure's HRTB `for<'a>` prevents the caller from smuggling in references
+        // with a concrete short lifetime, satisfying the lifetime requirement of
+        // `registration_data_unchecked`.
+        f(unsafe { self.registration_data_unchecked() })
+    }
+}
+
+impl<T: drm::Driver> Deref for RegistrationGuard<'_, T> {
+    type Target = Device<T, Registered>;
+
+    #[inline]
+    fn deref(&self) -> &Self::Target {
+        self.dev
+    }
+}
+
+impl<T: drm::Driver> Drop for RegistrationGuard<'_, T> {
+    #[inline]
+    fn drop(&mut self) {
+        // SAFETY: `self.idx` was returned by a successful `drm_dev_enter()` call, as guaranteed
+        // by the type invariants of `RegistrationGuard`.
+        unsafe { bindings::drm_dev_exit(self.idx) };
+    }
+}
+
+impl<T: drm::Driver> Deref for Device<T> {
     type Target = T::Data;
 
     fn deref(&self) -> &Self::Target {
@@ -360,9 +455,31 @@ fn deref(&self) -> &Self::Target {
     }
 }
 
+impl<T: drm::Driver> Deref for Device<T, Registered> {
+    type Target = Device<T>;
+
+    #[inline]
+    fn deref(&self) -> &Self::Target {
+        // SAFETY: The caller holds a `Device<T, Registered>`, which guarantees all invariants
+        // of the weaker `Normal` context.
+        unsafe { self.assume_ctx() }
+    }
+}
+
+impl<T: drm::Driver> Deref for Device<T, Ioctl> {
+    type Target = Device<T>;
+
+    #[inline]
+    fn deref(&self) -> &Self::Target {
+        // SAFETY: The caller holds a `Device<T, Ioctl>`, which guarantees all invariants
+        // of the weaker `Normal` context.
+        unsafe { self.assume_ctx() }
+    }
+}
+
 // SAFETY: DRM device objects are always reference counted and the get/put functions
 // satisfy the requirements.
-unsafe impl<T: drm::Driver, C: DeviceContext> AlwaysRefCounted for Device<T, C> {
+unsafe impl<T: drm::Driver> AlwaysRefCounted for Device<T> {
     fn inc_ref(&self) {
         // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero.
         unsafe { bindings::drm_dev_get(self.as_raw()) };
@@ -377,11 +494,29 @@ unsafe fn dec_ref(obj: NonNull<Self>) {
     }
 }
 
-impl<T: drm::Driver, C: DeviceContext> AsRef<device::Device> for Device<T, C> {
-    fn as_ref(&self) -> &device::Device {
+impl<T: drm::Driver> AsRef<T::ParentDevice<device::Normal>> for Device<T> {
+    fn as_ref(&self) -> &T::ParentDevice<device::Normal> {
         // SAFETY: `bindings::drm_device::dev` is valid as long as the DRM device itself is valid,
         // which is guaranteed by the type invariant.
-        unsafe { device::Device::from_raw((*self.as_raw()).dev) }
+        let dev = unsafe { device::Device::from_raw((*self.as_raw()).dev) };
+
+        // SAFETY: The DRM device was constructed in `UnregisteredDevice::new()` with a parent
+        // device of type `T::ParentDevice`, hence `dev` is contained in a `T::ParentDevice`.
+        unsafe { device::AsBusDevice::from_device(dev) }
+    }
+}
+
+impl<T: drm::Driver> AsRef<T::ParentDevice<device::Bound>> for Device<T, Registered> {
+    #[inline]
+    fn as_ref(&self) -> &T::ParentDevice<device::Bound> {
+        let dev = (**self).as_ref().as_ref();
+
+        // SAFETY: A `Device<T, Registered>` guarantees that the parent device is bound.
+        let dev = unsafe { dev.as_bound() };
+
+        // SAFETY: The DRM device was constructed in `UnregisteredDevice::new()` with a parent
+        // device of type `T::ParentDevice`, hence `dev` is contained in a `T::ParentDevice`.
+        unsafe { device::AsBusDevice::from_device(dev) }
     }
 }
 
@@ -392,12 +527,10 @@ unsafe impl<T: drm::Driver, C: DeviceContext> Send for Device<T, C> {}
 // by the synchronization in `struct drm_device`.
 unsafe impl<T: drm::Driver, C: DeviceContext> Sync for Device<T, C> {}
 
-impl<T, C, const ID: u64> WorkItem<ID> for Device<T, C>
+impl<T: drm::Driver, const ID: u64> WorkItem<ID> for Device<T>
 where
-    T: drm::Driver,
     T::Data: WorkItem<ID, Pointer = ARef<Self>>,
     T::Data: HasWork<Self, ID>,
-    C: DeviceContext,
 {
     type Pointer = ARef<Self>;
 
diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs
index 25f7e233884db..74f6ed690d8bd 100644
--- a/rust/kernel/drm/driver.rs
+++ b/rust/kernel/drm/driver.rs
@@ -7,16 +7,12 @@
 use crate::{
     bindings,
     device,
-    devres,
     drm,
     error::to_result,
     prelude::*,
     sync::aref::ARef, //
 };
-use core::{
-    mem,
-    ptr::NonNull, //
-};
+use core::ptr::NonNull;
 
 /// Driver use the GEM memory manager. This should be set for all modern drivers.
 pub(crate) const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM;
@@ -110,12 +106,23 @@ pub trait Driver {
     /// Context data associated with the DRM driver
     type Data: Sync + Send;
 
+    /// Data owned by the [`Registration`] and accessible within a
+    /// [`RegistrationGuard`](drm::RegistrationGuard) critical section via
+    /// [`Device::registration_data_with()`](drm::Device::registration_data_with).
+    ///
+    /// The lifetime parameter is tied to the [`Registration`] scope, which is enclosed in the
+    /// parent bus device binding scope but may be shorter.
+    type RegistrationData<'a>: Send + Sync + 'a;
+
     /// The type used to manage memory for this driver.
-    type Object<Ctx: drm::DeviceContext>: AllocImpl;
+    type Object: AllocImpl;
 
     /// The type used to represent a DRM File (client)
     type File: drm::file::DriverFile;
 
+    /// The bus device type of the parent device that the DRM device is associated with.
+    type ParentDevice<Ctx: device::DeviceContext>: device::AsBusDevice<Ctx>;
+
     /// Driver metadata
     const INFO: DriverInfo;
 
@@ -125,7 +132,7 @@ pub trait Driver {
     /// Sets the `DRIVER_RENDER` feature for this driver.
     ///
     /// When enabled, the driver exposes `/dev/dri/renderDXX` render nodes to
-    /// userspace. The render node is an alternate low-priviledge way to access
+    /// userspace. The render node is an alternate low-privilege way to access
     /// the driver, which is enforced on a per-ioctl level. Userspace processes
     /// that open the render node can only invoke ioctls explicitly listed as
     /// usable from the render node (i.e. marked DRM_RENDER_ALLOW), whereas
@@ -136,68 +143,84 @@ pub trait Driver {
 /// The registration type of a `drm::Device`.
 ///
 /// Once the `Registration` structure is dropped, the device is unregistered.
-pub struct Registration<T: Driver>(ARef<drm::Device<T>>);
-
-impl<T: Driver> Registration<T> {
-    fn new(drm: drm::UnregisteredDevice<T>, flags: usize) -> Result<Self> {
-        // SAFETY: `drm.as_raw()` is valid by the invariants of `drm::Device`.
-        to_result(unsafe { bindings::drm_dev_register(drm.as_raw(), flags) })?;
-
-        // SAFETY: We just called `drm_dev_register` above
-        let new = NonNull::from(unsafe { drm.assume_ctx() });
-
-        // Leak the ARef from UnregisteredDevice in preparation for transferring its ownership.
-        mem::forget(drm);
-
-        // SAFETY: `drm`'s `Drop` constructor was never called, ensuring that there remains at least
-        // one reference to the device - which we take ownership over here.
-        let new = unsafe { ARef::from_raw(new) };
-
-        Ok(Self(new))
-    }
+pub struct Registration<'a, T: Driver> {
+    drm: ARef<drm::Device<T>>,
+    _reg_data: Pin<KBox<T::RegistrationData<'a>>>,
+}
 
-    /// Registers a new [`UnregisteredDevice`](drm::UnregisteredDevice) with userspace.
+impl<'a, T: Driver> Registration<'a, T> {
+    /// Register a new [`UnregisteredDevice`](drm::UnregisteredDevice) with userspace.
     ///
-    /// Ownership of the [`Registration`] object is passed to [`devres::register`].
-    pub fn new_foreign_owned<'a>(
-        drm: drm::UnregisteredDevice<T>,
+    /// # Safety
+    ///
+    /// The caller must not `mem::forget()` the returned [`Registration`] or otherwise prevent its
+    /// [`Drop`] implementation from running, since the registration data may contain borrowed
+    /// references that become invalid after `'a` ends.
+    pub unsafe fn new<E>(
         dev: &'a device::Device<device::Bound>,
+        drm: drm::UnregisteredDevice<T>,
+        reg_data: impl PinInit<T::RegistrationData<'a>, E>,
         flags: usize,
-    ) -> Result<&'a drm::Device<T>>
+    ) -> Result<Self>
     where
-        T: 'static,
+        Error: From<E>,
     {
-        if drm.as_ref().as_raw() != dev.as_raw() {
+        let parent = drm.as_ref();
+        if parent.as_ref().as_raw() != dev.as_raw() {
             return Err(EINVAL);
         }
 
-        let reg = Registration::<T>::new(drm, flags)?;
-        let drm = NonNull::from(reg.device());
+        let reg_data: Pin<KBox<T::RegistrationData<'a>>> = KBox::pin_init(reg_data, GFP_KERNEL)?;
+
+        // Store the registration data pointer in the device before registration, so that it is
+        // visible once ioctls can be called.
+        let ptr: NonNull<T::RegistrationData<'static>> =
+            NonNull::from(Pin::get_ref(reg_data.as_ref())).cast();
 
-        devres::register(dev, reg, GFP_KERNEL)?;
+        // SAFETY: No concurrent access; the device is not yet registered.
+        unsafe { *drm.registration_data.get() = ptr };
+
+        // SAFETY: `drm` is a valid, initialized but not yet registered DRM device.
+        let ret = unsafe { bindings::drm_dev_register(drm.as_raw(), flags) };
+        if let Err(e) = to_result(ret) {
+            // SAFETY: `drm_dev_register()` synchronizes SRCU on failure, so no concurrent
+            // access to `registration_data` is possible at this point.
+            unsafe { *drm.registration_data.get() = NonNull::dangling() };
+            return Err(e);
+        }
 
-        // SAFETY: Since `reg` was passed to devres::register(), the device now owns the lifetime
-        // of the DRM registration - ensuring that this references lives for at least as long as 'a.
-        Ok(unsafe { drm.as_ref() })
+        Ok(Self {
+            drm: (&*drm).into(),
+            _reg_data: reg_data,
+        })
     }
 
     /// Returns a reference to the `Device` instance for this registration.
     pub fn device(&self) -> &drm::Device<T> {
-        &self.0
+        &self.drm
     }
 }
 
 // SAFETY: `Registration` doesn't offer any methods or access to fields when shared between
 // threads, hence it's safe to share it.
-unsafe impl<T: Driver> Sync for Registration<T> {}
+unsafe impl<T: Driver> Sync for Registration<'_, T> {}
 
 // SAFETY: Registration with and unregistration from the DRM subsystem can happen from any thread.
-unsafe impl<T: Driver> Send for Registration<T> {}
+unsafe impl<T: Driver> Send for Registration<'_, T> {}
 
-impl<T: Driver> Drop for Registration<T> {
+impl<T: Driver> Drop for Registration<'_, T> {
     fn drop(&mut self) {
+        // Use `drm_dev_unplug` rather than `drm_dev_unregister` to ensure that existing
+        // `drm_dev_enter()` critical sections complete before unregistration proceeds. This
+        // is required for the safety of `RegistrationGuard`, which relies on the SRCU barrier in
+        // `drm_dev_unplug()` to guarantee that the parent device is still bound within the
+        // critical section.
+        //
         // SAFETY: Safe by the invariant of `ARef<drm::Device<T>>`. The existence of this
-        // `Registration` also guarantees the this `drm::Device` is actually registered.
-        unsafe { bindings::drm_dev_unregister(self.0.as_raw()) };
+        // `Registration` also guarantees that this `drm::Device` is actually registered.
+        unsafe { bindings::drm_dev_unplug(self.drm.as_raw()) };
+        // After drm_dev_unplug(), the SRCU barrier guarantees that all RegistrationGuard critical
+        // sections have completed, so no one holds a reference to reg_data anymore.
+        // reg_data is dropped here automatically.
     }
 }
diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs
index c8b66d8168719..80d8f524f9d57 100644
--- a/rust/kernel/drm/gem/mod.rs
+++ b/rust/kernel/drm/gem/mod.rs
@@ -10,7 +10,7 @@
         self,
         device::{
             DeviceContext,
-            Registered, //
+            Normal, //
         },
         driver::{
             AllocImpl,
@@ -81,11 +81,10 @@ unsafe fn dec_ref(obj: core::ptr::NonNull<Self>) {
 /// A type alias for retrieving the current [`AllocImpl`] for a given [`DriverObject`].
 ///
 /// [`Driver`]: drm::Driver
-pub type DriverAllocImpl<T, Ctx = Registered> =
-    <<T as DriverObject>::Driver as drm::Driver>::Object<Ctx>;
+pub type DriverAllocImpl<T> = <<T as DriverObject>::Driver as drm::Driver>::Object;
 
 /// GEM object functions, which must be implemented by drivers.
-pub trait DriverObject: Sync + Send + Sized {
+pub trait DriverObject: Sync + Send + Sized + 'static {
     /// Parent `Driver` for this object.
     type Driver: drm::Driver;
 
@@ -93,8 +92,8 @@ pub trait DriverObject: Sync + Send + Sized {
     type Args;
 
     /// Create a new driver data object for a GEM object of a given size.
-    fn new<Ctx: DeviceContext>(
-        dev: &drm::Device<Self::Driver, Ctx>,
+    fn new(
+        dev: &drm::Device<Self::Driver>,
         size: usize,
         args: Self::Args,
     ) -> impl PinInit<Self, Error>;
@@ -109,7 +108,7 @@ fn close(_obj: &DriverAllocImpl<Self>, _file: &DriverFile<Self>) {}
 }
 
 /// Trait that represents a GEM object subtype
-pub trait IntoGEMObject: Sized + super::private::Sealed + AlwaysRefCounted {
+pub trait IntoGEMObject: Sized + super::private::Sealed {
     /// Returns a reference to the raw `drm_gem_object` structure, which must be valid as long as
     /// this owning object is valid.
     fn as_raw(&self) -> *mut bindings::drm_gem_object;
@@ -118,7 +117,8 @@ pub trait IntoGEMObject: Sized + super::private::Sealed + AlwaysRefCounted {
     ///
     /// # Safety
     ///
-    /// - `self_ptr` must be a valid pointer to `Self`.
+    /// - `self_ptr` must be a valid pointer to the `struct drm_gem_object` embedded in a
+    ///   valid instance of `Self`.
     /// - The caller promises that holding the immutable reference returned by this function does
     ///   not violate rust's data aliasing rules and remains valid throughout the lifetime of `'a`.
     unsafe fn from_raw<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self;
@@ -183,7 +183,7 @@ fn size(&self) -> usize {
     fn create_handle<D, F>(&self, file: &drm::File<F>) -> Result<u32>
     where
         Self: AllocImpl<Driver = D>,
-        D: drm::Driver<Object<Registered> = Self, File = F>,
+        D: drm::Driver<Object = Self, File = F>,
         F: drm::file::DriverFile<Driver = D>,
     {
         let mut handle: u32 = 0;
@@ -197,8 +197,8 @@ fn create_handle<D, F>(&self, file: &drm::File<F>) -> Result<u32>
     /// Looks up an object by its handle for a given `File`.
     fn lookup_handle<D, F>(file: &drm::File<F>, handle: u32) -> Result<ARef<Self>>
     where
-        Self: AllocImpl<Driver = D>,
-        D: drm::Driver<Object<Registered> = Self, File = F>,
+        Self: AllocImpl<Driver = D> + AlwaysRefCounted,
+        D: drm::Driver<Object = Self, File = F>,
         F: drm::file::DriverFile<Driver = D>,
     {
         // SAFETY: The arguments are all valid per the type invariants.
@@ -254,7 +254,7 @@ impl<T: IntoGEMObject> BaseObjectPrivate for T {}
 /// * Any type invariants of `Ctx` apply to the parent DRM device for this GEM object.
 #[repr(C)]
 #[pin_data]
-pub struct Object<T: DriverObject + Send + Sync, Ctx: DeviceContext = Registered> {
+pub struct Object<T: DriverObject + Send + Sync, Ctx: DeviceContext = Normal> {
     obj: Opaque<bindings::drm_gem_object>,
     #[pin]
     data: T,
@@ -280,12 +280,43 @@ impl<T: DriverObject, Ctx: DeviceContext> Object<T, Ctx> {
         rss: None,
     };
 
+    /// Returns the `Device` that owns this GEM object.
+    pub fn dev(&self) -> &drm::Device<T::Driver, Ctx> {
+        // SAFETY:
+        // - `struct drm_gem_object.dev` is initialized and valid for as long as the GEM
+        //   object lives.
+        // - The device we used for creating the gem object is passed as &drm::Device<T::Driver> to
+        //   Object::<T>::new(), so we know that `T::Driver` is the right generic parameter to use
+        //   here.
+        // - Any type invariants of `Ctx` are upheld by using the same `Ctx` for the `Device` we
+        //   return.
+        unsafe { drm::Device::from_raw((*self.as_raw()).dev) }
+    }
+
+    fn as_raw(&self) -> *mut bindings::drm_gem_object {
+        self.obj.get()
+    }
+
+    extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) {
+        let ptr: *mut Opaque<bindings::drm_gem_object> = obj.cast();
+
+        // SAFETY: All of our objects are of type `Object<T>`.
+        let this = unsafe { crate::container_of!(ptr, Self, obj) };
+
+        // SAFETY: The C code only ever calls this callback with a valid pointer to a `struct
+        // drm_gem_object`.
+        unsafe { bindings::drm_gem_object_release(obj) };
+
+        // SAFETY: All of our objects are allocated via `KBox`, and we're in the
+        // free callback which guarantees this object has zero remaining references,
+        // so we can drop it.
+        let _ = unsafe { KBox::from_raw(this) };
+    }
+}
+
+impl<T: DriverObject> Object<T> {
     /// Create a new GEM object.
-    pub fn new(
-        dev: &drm::Device<T::Driver, Ctx>,
-        size: usize,
-        args: T::Args,
-    ) -> Result<ARef<Self>> {
+    pub fn new(dev: &drm::Device<T::Driver>, size: usize, args: T::Args) -> Result<ARef<Self>> {
         let obj: Pin<KBox<Self>> = KBox::pin_init(
             try_pin_init!(Self {
                 obj: Opaque::new(bindings::drm_gem_object::default()),
@@ -321,46 +352,12 @@ pub fn new(
         // SAFETY: We take over the initial reference count from `drm_gem_object_init()`.
         Ok(unsafe { ARef::from_raw(ptr) })
     }
-
-    /// Returns the `Device` that owns this GEM object.
-    pub fn dev(&self) -> &drm::Device<T::Driver, Ctx> {
-        // SAFETY:
-        // - `struct drm_gem_object.dev` is initialized and valid for as long as the GEM
-        //   object lives.
-        // - The device we used for creating the gem object is passed as &drm::Device<T::Driver> to
-        //   Object::<T>::new(), so we know that `T::Driver` is the right generic parameter to use
-        //   here.
-        // - Any type invariants of `Ctx` are upheld by using the same `Ctx` for the `Device` we
-        //   return.
-        unsafe { drm::Device::from_raw((*self.as_raw()).dev) }
-    }
-
-    fn as_raw(&self) -> *mut bindings::drm_gem_object {
-        self.obj.get()
-    }
-
-    extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) {
-        let ptr: *mut Opaque<bindings::drm_gem_object> = obj.cast();
-
-        // SAFETY: All of our objects are of type `Object<T>`.
-        let this = unsafe { crate::container_of!(ptr, Self, obj) };
-
-        // SAFETY: The C code only ever calls this callback with a valid pointer to a `struct
-        // drm_gem_object`.
-        unsafe { bindings::drm_gem_object_release(obj) };
-
-        // SAFETY: All of our objects are allocated via `KBox`, and we're in the
-        // free callback which guarantees this object has zero remaining references,
-        // so we can drop it.
-        let _ = unsafe { KBox::from_raw(this) };
-    }
 }
 
 impl_aref_for_gem_obj! {
-    impl<T, C> for Object<T, C>
+    impl<T> for Object<T>
     where
-        T: DriverObject,
-        C: DeviceContext
+        T: DriverObject
 }
 
 impl<T: DriverObject, Ctx: DeviceContext> super::private::Sealed for Object<T, Ctx> {}
diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs
index 34af402899a0e..580e0808d6c0b 100644
--- a/rust/kernel/drm/gem/shmem.rs
+++ b/rust/kernel/drm/gem/shmem.rs
@@ -11,28 +11,57 @@
 
 use crate::{
     container_of,
+    device::{
+        self,
+        Bound, //
+    },
+    devres::*,
     drm::{
         driver,
         gem,
         private::Sealed,
-        Device,
-        DeviceContext,
-        Registered, //
+        Device, //
+    },
+    error::{
+        from_err_ptr,
+        to_result, //
+    },
+    io::{
+        IoBase,
+        Region,
+        SysMem,
+        SysMemBackend, //
     },
-    error::to_result,
     prelude::*,
-    sync::aref::ARef,
-    types::Opaque, //
+    scatterlist,
+    sync::{
+        aref::ARef,
+        new_mutex,
+        Mutex,
+        SetOnce, //
+    },
+    types::{
+        NotThreadSafe,
+        Opaque, //
+    },
 };
 use core::{
-    marker::PhantomData,
+    ffi::c_void,
+    mem::{
+        ManuallyDrop,
+        MaybeUninit, //
+    },
     ops::{
         Deref,
         DerefMut, //
     },
-    ptr::NonNull, //
+    ptr::{
+        self,
+        NonNull, //
+    },
 };
 use gem::{
+    BaseObject,
     BaseObjectPrivate,
     DriverObject,
     IntoGEMObject, //
@@ -42,15 +71,24 @@
 ///
 /// This is used with [`Object::new()`] to control various properties that can only be set when
 /// initially creating a shmem-backed GEM object.
-#[derive(Default)]
-pub struct ObjectConfig<'a, T: DriverObject, C: DeviceContext = Registered> {
+pub struct ObjectConfig<'a, T: DriverObject> {
     /// Whether to set the write-combine map flag.
     pub map_wc: bool,
 
     /// Reuse the DMA reservation from another GEM object.
     ///
     /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified.
-    pub parent_resv_obj: Option<&'a Object<T, C>>,
+    pub parent_resv_obj: Option<&'a Object<T>>,
+}
+
+impl<'a, T: DriverObject> Default for ObjectConfig<'a, T> {
+    #[inline(always)]
+    fn default() -> Self {
+        Self {
+            map_wc: false,
+            parent_resv_obj: None,
+        }
+    }
 }
 
 /// A shmem-backed GEM object.
@@ -59,33 +97,35 @@ pub struct ObjectConfig<'a, T: DriverObject, C: DeviceContext = Registered> {
 ///
 /// - `obj` contains a valid initialized `struct drm_gem_shmem_object` for the lifetime of this
 ///   object.
-/// - Any type invariants of `C` apply to the parent DRM device for this GEM object.
 #[repr(C)]
 #[pin_data]
-pub struct Object<T: DriverObject, C: DeviceContext = Registered> {
+pub struct Object<T: DriverObject> {
     #[pin]
     obj: Opaque<bindings::drm_gem_shmem_object>,
     /// Parent object that owns this object's DMA reservation object.
-    parent_resv_obj: Option<ARef<Object<T, C>>>,
+    parent_resv_obj: Option<ARef<Object<T>>>,
+    /// Devres object for unmapping any SGTable on driver-unbind.
+    sgt_res: ManuallyDrop<SetOnce<Devres<SGTableMap<T>>>>,
+    #[pin]
+    /// Lock for protecting initialization of `sgt_res`.
+    sgt_lock: Mutex<()>,
     #[pin]
     inner: T,
-    _ctx: PhantomData<C>,
 }
 
 super::impl_aref_for_gem_obj! {
-    impl<T, C> for Object<T, C>
+    impl<T> for Object<T>
     where
-        T: DriverObject,
-        C: DeviceContext
+        T: DriverObject
 }
 
 // SAFETY: All GEM objects are thread-safe.
-unsafe impl<T: DriverObject, C: DeviceContext> Send for Object<T, C> {}
+unsafe impl<T: DriverObject> Send for Object<T> {}
 
 // SAFETY: All GEM objects are thread-safe.
-unsafe impl<T: DriverObject, C: DeviceContext> Sync for Object<T, C> {}
+unsafe impl<T: DriverObject> Sync for Object<T> {}
 
-impl<T: DriverObject, C: DeviceContext> Object<T, C> {
+impl<T: DriverObject> Object<T> {
     /// `drm_gem_object_funcs` vtable suitable for GEM shmem objects.
     const VTABLE: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs {
         free: Some(Self::free_callback),
@@ -112,21 +152,166 @@ fn as_raw_shmem(&self) -> *mut bindings::drm_gem_shmem_object {
         self.obj.get()
     }
 
+    /// Returns the `Device` that owns this GEM object.
+    pub fn dev(&self) -> &Device<T::Driver> {
+        // SAFETY: `dev` will have been initialized in `Self::new()` by `drm_gem_shmem_init()`.
+        unsafe { Device::from_raw((*self.as_raw()).dev) }
+    }
+
+    extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) {
+        // SAFETY:
+        // - DRM always passes a valid gem object here
+        // - We used drm_gem_shmem_create() in our create_gem_object callback, so we know that
+        //   `obj` is contained within a drm_gem_shmem_object
+        let base = unsafe { container_of!(obj, bindings::drm_gem_shmem_object, base) };
+
+        // SAFETY:
+        // - We verified above that `obj` is valid, which makes `this` valid
+        // - This function is set in AllocOps, so we know that `this` is contained within an
+        //   `Object<T>`
+        let this = unsafe { container_of!(Opaque::cast_from(base), Self, obj) }.cast_mut();
+
+        // We need to drop `sgt_res` first, since doing so requires that the GEM object is still
+        // alive.
+        // SAFETY:
+        // - We verified above that `this` is valid.
+        // - We are in free_callback, guaranteeing we have exclusive access to `this` and that
+        //   `sgt_res` will not be used after dropping it here.
+        unsafe { ManuallyDrop::drop(&mut (*this).sgt_res) };
+
+        // SAFETY:
+        // - We're in free_callback - so this function is safe to call.
+        // - We won't be using the gem resources on `this` after this call.
+        unsafe { bindings::drm_gem_shmem_release(base) };
+
+        // SAFETY: We're recovering the Kbox<> we created in gem_create_object()
+        let _ = unsafe { KBox::from_raw(this) };
+    }
+
+    /// Attempt to create a vmap from the gem object, and confirm the size of said vmap.
+    fn make_vmap<'a, R, const SIZE: usize>(&'a self) -> Result<VMap<T, R, SIZE>>
+    where
+        R: Deref<Target = Self> + From<&'a Self>,
+    {
+        // INVARIANT: We check here that the gem object is at least as large as `SIZE`.
+        if self.size() < SIZE {
+            return Err(ENOSPC);
+        }
+
+        let mut map: MaybeUninit<bindings::iosys_map> = MaybeUninit::uninit();
+        let guard = DmaResvGuard::new(self);
+
+        // SAFETY: `drm_gem_shmem_vmap()` can be called with the DMA reservation lock held.
+        to_result(unsafe {
+            bindings::drm_gem_shmem_vmap_locked(self.as_raw_shmem(), map.as_mut_ptr())
+        })?;
+
+        // Drop the guard explicitly here, since we may need to call `raw_vunmap()` (which
+        // re-acquires the lock).
+        drop(guard);
+
+        // SAFETY: The call to `drm_gem_shmem_vmap_locked()` succeeded above, so we are guaranteed
+        // that map is properly initialized.
+        let map = unsafe { map.assume_init() };
+
+        // XXX: We don't currently support iomem allocations
+        if map.is_iomem {
+            // SAFETY: The vmap operation above succeeded, guaranteeing that `map` points to a valid
+            // memory mapping.
+            unsafe { self.raw_vunmap(map) };
+
+            Err(ENOTSUPP)
+        } else {
+            Ok(VMap {
+                // INVARIANT: `addr` remains valid for as long as `owner` does, which extends to the
+                // lifetime of `VMap` itself.
+                // SAFETY: We checked that this is not an iomem allocation, making it safe to read
+                // vaddr.
+                addr: unsafe { map.__bindgen_anon_1.vaddr },
+                owner: self.into(),
+            })
+        }
+    }
+
+    /// Unmap a vmap from the gem object.
+    ///
+    /// # Safety
+    ///
+    /// - The caller promises that `map` is a valid vmap on this gem object.
+    /// - The caller promises that the memory pointed to by map will no longer be accesed through
+    ///   this instance.
+    unsafe fn raw_vunmap(&self, mut map: bindings::iosys_map) {
+        let _guard = DmaResvGuard::new(self);
+
+        // SAFETY:
+        // - This function is safe to call with the DMA reservation lock held.
+        // - The caller promises that `map` is a valid vmap on this gem object.
+        unsafe { bindings::drm_gem_shmem_vunmap_locked(self.as_raw_shmem(), &mut map) };
+    }
+
+    /// Creates and returns a virtual kernel memory mapping for this object.
+    #[inline]
+    pub fn vmap<const SIZE: usize>(&self) -> Result<VMapRef<'_, T, SIZE>> {
+        self.make_vmap()
+    }
+
+    /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA
+    /// pages for this object.
+    ///
+    /// This will pin the object in memory. It is expected that `dev` should be a pointer to the
+    /// same [`device::Device`] which `self` belongs to, otherwise this function will return
+    /// `Err(EINVAL)`.
+    pub fn sg_table<'a>(
+        &'a self,
+        dev: &'a device::Device<Bound>,
+    ) -> Result<&'a scatterlist::SGTable> {
+        let parent = self.dev().as_ref();
+        if dev.as_raw() != parent.as_ref().as_raw() {
+            return Err(EINVAL);
+        }
+
+        let sgt_res = 'out: {
+            // Fast path: sgt_res is already initialized
+            if let Some(sgt_res) = self.sgt_res.as_ref() {
+                break 'out sgt_res;
+            }
+
+            // Slow path: Grab the lock and see if we need to initialize sgt_res.
+            let _guard = self.sgt_lock.lock();
+
+            // If someone initialized it while we were waiting, we can exit early.
+            if let Some(sgt_res) = self.sgt_res.as_ref() {
+                break 'out sgt_res;
+            }
+
+            // If not, finish initializing and return. `populate()` cannot return false, as
+            // `sgt_res` must be unpopulated, and we must hold `sgt_lock` to reach this point.
+            self.sgt_res
+                .populate(Devres::new(dev, SGTableMap::new(self))?);
+
+            // SAFETY: We just populated sgt_res above.
+            unsafe { self.sgt_res.as_ref().unwrap_unchecked() }
+        };
+
+        Ok(sgt_res.access(dev)?)
+    }
+
     /// Create a new shmem-backed DRM object of the given size.
     ///
     /// Additional config options can be specified using `config`.
     pub fn new(
-        dev: &Device<T::Driver, C>,
+        dev: &Device<T::Driver>,
         size: usize,
-        config: ObjectConfig<'_, T, C>,
+        config: ObjectConfig<'_, T>,
         args: T::Args,
     ) -> Result<ARef<Self>> {
         let new: Pin<KBox<Self>> = KBox::try_pin_init(
             try_pin_init!(Self {
                 obj <- Opaque::init_zeroed(),
                 parent_resv_obj: config.parent_resv_obj.map(|p| p.into()),
+                sgt_res: ManuallyDrop::new(SetOnce::new()),
+                sgt_lock <- new_mutex!(()),
                 inner <- T::new(dev, size, args),
-                _ctx: PhantomData::<C>,
             }),
             GFP_KERNEL,
         )?;
@@ -158,36 +343,14 @@ pub fn new(
         Ok(obj)
     }
 
-    /// Returns the `Device` that owns this GEM object.
-    pub fn dev(&self) -> &Device<T::Driver, C> {
-        // SAFETY: `dev` will have been initialized in `Self::new()` by `drm_gem_shmem_init()`.
-        unsafe { Device::from_raw((*self.as_raw()).dev) }
-    }
-
-    extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) {
-        // SAFETY:
-        // - DRM always passes a valid gem object here
-        // - We used drm_gem_shmem_create() in our create_gem_object callback, so we know that
-        //   `obj` is contained within a drm_gem_shmem_object
-        let this = unsafe { container_of!(obj, bindings::drm_gem_shmem_object, base) };
-
-        // SAFETY:
-        // - We're in free_callback - so this function is safe to call.
-        // - We won't be using the gem resources on `this` after this call.
-        unsafe { bindings::drm_gem_shmem_release(this) };
-
-        // SAFETY:
-        // - We verified above that `obj` is valid, which makes `this` valid
-        // - This function is set in AllocOps, so we know that `this` is contained within a
-        //   `Object<T, C>`
-        let this = unsafe { container_of!(Opaque::cast_from(this), Self, obj) }.cast_mut();
-
-        // SAFETY: We're recovering the Kbox<> we created in gem_create_object()
-        let _ = unsafe { KBox::from_raw(this) };
+    /// Creates and returns an owned reference to a virtual kernel memory mapping for this object.
+    #[inline]
+    pub fn owned_vmap<const SIZE: usize>(&self) -> Result<VMapOwned<T, SIZE>> {
+        self.make_vmap()
     }
 }
 
-impl<T: DriverObject, C: DeviceContext> Deref for Object<T, C> {
+impl<T: DriverObject> Deref for Object<T> {
     type Target = T;
 
     fn deref(&self) -> &Self::Target {
@@ -195,15 +358,15 @@ fn deref(&self) -> &Self::Target {
     }
 }
 
-impl<T: DriverObject, C: DeviceContext> DerefMut for Object<T, C> {
+impl<T: DriverObject> DerefMut for Object<T> {
     fn deref_mut(&mut self) -> &mut Self::Target {
         &mut self.inner
     }
 }
 
-impl<T: DriverObject, C: DeviceContext> Sealed for Object<T, C> {}
+impl<T: DriverObject> Sealed for Object<T> {}
 
-impl<T: DriverObject, C: DeviceContext> gem::IntoGEMObject for Object<T, C> {
+impl<T: DriverObject> gem::IntoGEMObject for Object<T> {
     fn as_raw(&self) -> *mut bindings::drm_gem_object {
         // SAFETY:
         // - Our immutable reference is proof that this is safe to dereference.
@@ -222,7 +385,7 @@ unsafe fn from_raw<'a>(obj: *mut bindings::drm_gem_object) -> &'a Self {
     }
 }
 
-impl<T: DriverObject, C: DeviceContext> driver::AllocImpl for Object<T, C> {
+impl<T: DriverObject> driver::AllocImpl for Object<T> {
     type Driver = T::Driver;
 
     const ALLOC_OPS: driver::AllocOps = driver::AllocOps {
@@ -235,3 +398,324 @@ impl<T: DriverObject, C: DeviceContext> driver::AllocImpl for Object<T, C> {
         dumb_map_offset: None,
     };
 }
+
+/// Private helper-type for holding the `dma_resv` object for a GEM shmem object.
+///
+/// When this is dropped, the `dma_resv` lock is dropped as well.
+///
+// TODO: This should be replace with a WwMutex equivalent once we have such bindings in the kernel.
+struct DmaResvGuard<'a, T: DriverObject>(&'a Object<T>, NotThreadSafe);
+
+impl<'a, T: DriverObject> DmaResvGuard<'a, T> {
+    #[inline]
+    fn new(obj: &'a Object<T>) -> Self {
+        // SAFETY: This lock is initialized throughout the lifetime of `object`.
+        unsafe { bindings::dma_resv_lock(obj.raw_dma_resv(), ptr::null_mut()) };
+
+        Self(obj, NotThreadSafe)
+    }
+}
+
+impl<'a, T: DriverObject> Drop for DmaResvGuard<'a, T> {
+    #[inline]
+    fn drop(&mut self) {
+        // SAFETY: We are releasing the lock grabbed during the creation of this object.
+        unsafe { bindings::dma_resv_unlock(self.0.raw_dma_resv()) };
+    }
+}
+
+/// A reference to a virtual mapping for an shmem-based GEM object in kernel address space.
+///
+/// # Invariants
+///
+/// - The size of `owner` is >= SIZE.
+/// - The memory pointed to by `addr` remains valid at least until this object is dropped.
+pub struct VMap<D, R, const SIZE: usize = 0>
+where
+    D: DriverObject,
+    R: Deref<Target = Object<D>>,
+{
+    addr: *mut c_void,
+    owner: R,
+}
+
+/// An alias type for a reference to a shmem-based GEM object's VMap.
+pub type VMapRef<'a, D, const SIZE: usize = 0> = VMap<D, &'a Object<D>, SIZE>;
+
+/// An alias type for an owned reference to a shmem-based GEM object's VMap.
+pub type VMapOwned<D, const SIZE: usize = 0> = VMap<D, ARef<Object<D>>, SIZE>;
+
+impl<D, R, const SIZE: usize> VMap<D, R, SIZE>
+where
+    D: DriverObject,
+    R: Deref<Target = Object<D>>,
+{
+    /// Borrows a reference to the object that owns this virtual mapping.
+    #[inline]
+    pub fn owner(&self) -> &Object<D> {
+        &self.owner
+    }
+}
+
+impl<'a, D, R, const SIZE: usize> IoBase<'a> for &'a VMap<D, R, SIZE>
+where
+    D: DriverObject,
+    R: Deref<Target = Object<D>>,
+{
+    type Backend = SysMemBackend;
+    type Target = Region<SIZE>;
+
+    #[inline]
+    fn as_view(self) -> SysMem<'a, Region<SIZE>> {
+        let ptr = Region::ptr_from_raw_parts_mut(self.addr.cast(), self.owner.size());
+
+        // SAFETY: Per type invariants of `VMap`:
+        // - `addr .. addr + owner.size()` is a valid kernel accessible memory region.
+        // - `addr` is page-aligned, which satisfies `Region`'s 4-byte alignment requirement.
+        // - The memory remains valid until this `VMap` is dropped; since `self` is `&'a VMap`,
+        //   the borrow prevents the `VMap` from being dropped for the lifetime `'a`.
+        unsafe { SysMem::new(ptr) }
+    }
+}
+
+impl<D, R, const SIZE: usize> Drop for VMap<D, R, SIZE>
+where
+    D: DriverObject,
+    R: Deref<Target = Object<D>>,
+{
+    #[inline]
+    fn drop(&mut self) {
+        // SAFETY:
+        // - Our existence is proof that this map was previously created using self.owner.
+        // - Since we are in Drop, we are guaranteed that no one will access the memory
+        //   through this mapping after calling this.
+        unsafe {
+            self.owner.raw_vunmap(bindings::iosys_map {
+                is_iomem: false,
+                __bindgen_anon_1: bindings::iosys_map__bindgen_ty_1 { vaddr: self.addr },
+            })
+        };
+    }
+}
+
+// SAFETY: `addr` points to a valid memory address for as long as `owner` exists, meaning that so
+// long as `owner` is `Send` so is `VMap`.
+unsafe impl<D, R, const SIZE: usize> Send for VMap<D, R, SIZE>
+where
+    D: DriverObject,
+    R: Deref<Target = Object<D>> + Send,
+{
+}
+
+// SAFETY: `addr` points to a valid memory address for as long as `owner` exists, meaning that so
+// long as `owner` is `Sync` so is `VMap`.
+unsafe impl<D, R, const SIZE: usize> Sync for VMap<D, R, SIZE>
+where
+    D: DriverObject,
+    R: Deref<Target = Object<D>> + Sync,
+{
+}
+
+/// A reference to a GEM object that is known to have a mapped [`SGTable`].
+///
+/// This is used by the Rust bindings with [`Devres`] in order to ensure that mappings for SGTables
+/// on GEM shmem objects are revoked on driver-unbind.
+///
+/// # Invariants
+///
+/// - `self.obj` always points to a valid GEM object.
+/// - This object is proof that `self.obj.owner.sgt_res` has an initialized and valid pointer to an
+///   [`SGTable`].
+///
+/// [`SGTable`]: scatterlist::SGTable
+pub struct SGTableMap<T: DriverObject> {
+    obj: NonNull<Object<T>>,
+}
+
+impl<T: DriverObject> Deref for SGTableMap<T> {
+    type Target = scatterlist::SGTable;
+
+    fn deref(&self) -> &Self::Target {
+        // SAFETY:
+        // - The NonNull is guaranteed to be valid via our type invariants.
+        // - The sgt field is guaranteed to be initialized and valid via our type invariants.
+        unsafe { scatterlist::SGTable::from_raw((*self.obj.as_ref().as_raw_shmem()).sgt) }
+    }
+}
+
+impl<T: DriverObject> Drop for SGTableMap<T> {
+    fn drop(&mut self) {
+        // SAFETY: `obj` is always valid via our type invariants
+        let obj = unsafe { self.obj.as_ref() };
+        let _lock = DmaResvGuard::new(obj);
+
+        // SAFETY: We acquired the lock needed for calling this function above
+        unsafe { bindings::__drm_gem_shmem_free_sgt_locked(obj.as_raw_shmem()) };
+    }
+}
+
+impl<T: DriverObject> SGTableMap<T> {
+    fn new(obj: &Object<T>) -> impl Init<Self, Error> {
+        // INVARIANT:
+        // - We call drm_gem_shmem_get_pages_sgt below and check whether or not it succeeds,
+        //   fulfilling the invariant of SGTableMap that the object's `sgt` field is initialized.
+        // SAFETY:
+        // - `obj` is fully initialized, making this function safe to call.
+        from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(obj.as_raw_shmem()) })?;
+
+        Ok(Self { obj: obj.into() })
+    }
+}
+
+// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object
+// it points to is guaranteed to be thread-safe.
+unsafe impl<T: DriverObject> Send for SGTableMap<T> {}
+// SAFETY: The NonNull in SGTableMap is guaranteed valid by our type invariants, and the GEM object
+// it points to is guaranteed to be thread-safe.
+unsafe impl<T: DriverObject> Sync for SGTableMap<T> {}
+
+#[kunit_tests(rust_drm_gem_shmem)]
+mod tests {
+    use super::*;
+    use crate::{
+        drm::{
+            self,
+            UnregisteredDevice, //
+        },
+        faux,
+        io::Io,
+        page::PAGE_SIZE, //
+    };
+
+    // The bare minimum needed to create a fake drm driver for kunit
+
+    #[pin_data]
+    struct KunitData {}
+    struct KunitDriver;
+    struct KunitFile;
+    #[pin_data]
+    struct KunitObject {}
+
+    const INFO: drm::DriverInfo = drm::DriverInfo {
+        major: 0,
+        minor: 0,
+        patchlevel: 0,
+        name: c"kunit",
+        desc: c"Kunit",
+    };
+
+    impl drm::file::DriverFile for KunitFile {
+        type Driver = KunitDriver;
+
+        fn open(_dev: &drm::Device<KunitDriver>) -> Result<Pin<KBox<Self>>> {
+            Ok(KBox::new(Self, GFP_KERNEL)?.into())
+        }
+    }
+
+    impl gem::DriverObject for KunitObject {
+        type Driver = KunitDriver;
+        type Args = ();
+
+        fn new(
+            _dev: &drm::Device<KunitDriver>,
+            _size: usize,
+            _args: Self::Args,
+        ) -> impl PinInit<Self, Error> {
+            try_pin_init!(KunitObject {})
+        }
+    }
+
+    #[vtable]
+    impl drm::Driver for KunitDriver {
+        type Data = KunitData;
+        type RegistrationData<'a> = ();
+        type File = KunitFile;
+        type Object = Object<KunitObject>;
+        type ParentDevice<Ctx: device::DeviceContext> = faux::Device<Ctx>;
+
+        const INFO: drm::DriverInfo = INFO;
+        const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor] = &[];
+    }
+
+    fn create_drm_dev() -> Result<(faux::Registration, UnregisteredDevice<KunitDriver>)> {
+        // Create a faux DRM device so we can test gem object creation.
+        let data = try_pin_init!(KunitData {});
+        let reg = faux::Registration::new(c"Kunit", None)?;
+        let fdev = reg.as_ref();
+        let drm = UnregisteredDevice::new(fdev, data)?;
+
+        Ok((reg, drm))
+    }
+
+    #[test]
+    fn compile_time_vmap_sizes() -> Result {
+        let (_dev, drm) = create_drm_dev()?;
+
+        let obj = Object::<KunitObject>::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?;
+
+        // Try creating a normal vmap
+        obj.vmap::<PAGE_SIZE>()?;
+
+        // Try creating a vmap that's smaller then the size we specified
+        let vmap = obj.vmap::<{ PAGE_SIZE - 100 }>()?;
+
+        // Verify the owner matches
+        assert!(ptr::eq(vmap.owner(), obj.deref()));
+
+        // Verify the size matches the actual object size
+        assert_eq!(vmap.size(), PAGE_SIZE);
+
+        // Make sure creating a vmap that's too large fails
+        assert!(obj.vmap::<{ PAGE_SIZE + 200 }>().is_err());
+
+        Ok(())
+    }
+
+    #[test]
+    fn vmap_io() -> Result {
+        let (_dev, drm) = create_drm_dev()?;
+
+        let obj = Object::<KunitObject>::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?;
+
+        let vmap = obj.vmap::<PAGE_SIZE>()?;
+
+        vmap.write8(0xDE, 0x0);
+        assert_eq!(vmap.read8(0x0), 0xDE);
+        vmap.write32(0xFEDCBA98, 0x20);
+
+        assert_eq!(vmap.read32(0x20), 0xFEDCBA98);
+
+        // Ensure the ordering in memory is correct
+        let expected = 0xFEDCBA98_u32.to_ne_bytes().into_iter();
+        for (offset, expected) in (0x20..=0x23).zip(expected) {
+            assert_eq!(vmap.read8(offset), expected);
+        }
+
+        Ok(())
+    }
+
+    // TODO: I would love to actually test the success paths of sg_table(), but that would require
+    // also implementing dummy dma_ops so that trying to create a mapping doesn't explode. So, leave
+    // that for someone else.
+
+    // Ensures that passing the wrong device to sg_table() fails as we expect, and also ensure it
+    // skips initializing `sgt_res` since we could otherwise create `sgt_res` with the wrong device
+    // bound to it.
+    #[test]
+    fn fail_sg_table_on_wrong_dev() -> Result {
+        let (_dev, drm) = create_drm_dev()?;
+        let reg = faux::Registration::new(c"EvilKunit", None)?;
+        let wrong_dev = reg.as_ref();
+
+        let obj = Object::<KunitObject>::new(&drm, PAGE_SIZE, ObjectConfig::default(), ())?;
+
+        assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL);
+
+        // If sgt_res was not initialized mistakenly with the wrong device, this should still fail.
+        assert_eq!(obj.sg_table(wrong_dev.as_ref()).err().unwrap(), EINVAL);
+
+        // TODO: Someday, we should test that creating an sg_table here still succeeds.
+
+        Ok(())
+    }
+}
diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs
index ae58f6f667c1f..d9d43d7197613 100644
--- a/rust/kernel/drm/gpuvm/mod.rs
+++ b/rust/kernel/drm/gpuvm/mod.rs
@@ -72,10 +72,12 @@ pub struct GpuVm<T: DriverGpuVm> {
     data: UnsafeCell<T>,
 }
 
-// SAFETY: The GPUVM api does not assume that it is tied to a specific thread. The destructor will
-// drop the `data` field, which is okay because it is guaranteed `Send` by the `DriverGpuVm` trait.
+// SAFETY: It is safe to send a `GpuVm<T>` to another thread: all data reachable through it
+// (`T`, `T::VmBoData`, and the GEM `T::Object`) is `Send` by the `DriverGpuVm` bounds.
 unsafe impl<T: DriverGpuVm> Send for GpuVm<T> {}
-// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel.
+// SAFETY: It is safe to share a `&GpuVm<T>` between threads: `&self` methods only alias data
+// that is `Sync` by the `DriverGpuVm` bounds, and any thread may drop that data, or upgrade the
+// reference and ultimately drop `T`, which the same bounds make `Send`.
 unsafe impl<T: DriverGpuVm> Sync for GpuVm<T> {}
 
 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`.
@@ -116,9 +118,9 @@ const fn vtable() -> &'static bindings::drm_gpuvm_ops {
 
     /// Creates a GPUVM instance.
     #[expect(clippy::new_ret_no_self)]
-    pub fn new<E>(
+    pub fn new<E, Ctx: drm::DeviceContext>(
         name: &'static CStr,
-        dev: &drm::Device<T::Driver>,
+        dev: &drm::Device<T::Driver, Ctx>,
         r_obj: &T::Object,
         range: Range<u64>,
         reserve_range: Range<u64>,
@@ -250,21 +252,27 @@ fn raw_resv(&self) -> *mut bindings::dma_resv {
 }
 
 /// The manager for a GPUVM.
-pub trait DriverGpuVm: Sized + Send {
+pub trait DriverGpuVm: Sized + Send + Sync {
     /// Parent `Driver` for this object.
-    type Driver: drm::Driver<Object = Self::Object>;
+    type Driver: drm::Driver;
 
     /// The kind of GEM object stored in this GPUVM.
-    type Object: IntoGEMObject;
+    type Object: drm::driver::AllocImpl<Driver = Self::Driver> + Send + Sync;
 
     /// Data stored with each [`struct drm_gpuva`](struct@GpuVa).
-    type VaData;
+    ///
+    /// Only `Send` is required: the data has a single owner at all times, moving
+    /// between threads by value (handed back as a [`GpuVaRemoved`]) but never
+    /// accessed by two threads concurrently.
+    type VaData: Send;
 
     /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo).
-    type VmBoData;
+    type VmBoData: Send + Sync;
 
     /// The private data passed to callbacks.
-    type SmContext<'ctx>;
+    type SmContext<'ctx>
+    where
+        Self: 'ctx;
 
     /// Indicates that a new mapping should be created.
     fn sm_step_map<'op, 'ctx>(
@@ -296,12 +304,10 @@ fn sm_step_remap<'op, 'ctx>(
 /// # Invariants
 ///
 /// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference.
+// `Send`/`Sync` derive from `ARef<GpuVm<T>>`; the trait bounds make them correct for the unique
+// handle's `&mut T` access.
 pub struct UniqueRefGpuVm<T: DriverGpuVm>(ARef<GpuVm<T>>);
 
-// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel, and
-// concurrent access to `data` is safe due to the `T: Sync` requirement.
-unsafe impl<T: DriverGpuVm + Sync> Sync for UniqueRefGpuVm<T> {}
-
 impl<T: DriverGpuVm> UniqueRefGpuVm<T> {
     /// Access the data owned by this `UniqueRefGpuVm` immutably.
     #[inline]
diff --git a/rust/kernel/drm/gpuvm/sm_ops.rs b/rust/kernel/drm/gpuvm/sm_ops.rs
index 69a8e5ab2821f..742c151b2540e 100644
--- a/rust/kernel/drm/gpuvm/sm_ops.rs
+++ b/rust/kernel/drm/gpuvm/sm_ops.rs
@@ -3,7 +3,7 @@
 use super::*;
 
 /// The actual data that gets threaded through the callbacks.
-struct SmData<'a, 'ctx, T: DriverGpuVm> {
+struct SmData<'a, 'ctx, T: DriverGpuVm + 'ctx> {
     gpuvm: &'a mut UniqueRefGpuVm<T>,
     user_context: &'a mut T::SmContext<'ctx>,
 }
@@ -20,7 +20,7 @@ struct SmMapData<'a, 'ctx, T: DriverGpuVm> {
 }
 
 /// The argument for [`UniqueRefGpuVm::sm_map`].
-pub struct OpMapRequest<'a, 'ctx, T: DriverGpuVm> {
+pub struct OpMapRequest<'a, 'ctx, T: DriverGpuVm + 'ctx> {
     /// Address in GPU virtual address space.
     pub addr: u64,
     /// Length of mapping to create.
diff --git a/rust/kernel/drm/gpuvm/va.rs b/rust/kernel/drm/gpuvm/va.rs
index 0b09fe44ab397..b108ec7aa1bc1 100644
--- a/rust/kernel/drm/gpuvm/va.rs
+++ b/rust/kernel/drm/gpuvm/va.rs
@@ -104,6 +104,14 @@ pub fn vm_bo(&self) -> &GpuVmBo<T> {
 /// The memory is zeroed.
 pub struct GpuVaAlloc<T: DriverGpuVm>(KBox<MaybeUninit<GpuVa<T>>>);
 
+// SAFETY: A `GpuVaAlloc` is an owned, uninitialised allocation with no live `T::VaData` and no
+// thread-bound state.
+unsafe impl<T: DriverGpuVm> Send for GpuVaAlloc<T> {}
+
+// SAFETY: A `GpuVaAlloc` has no `&self` method that reaches its contents, so a shared
+// `&GpuVaAlloc` cannot access the allocation.
+unsafe impl<T: DriverGpuVm> Sync for GpuVaAlloc<T> {}
+
 impl<T: DriverGpuVm> GpuVaAlloc<T> {
     /// Pre-allocate a [`GpuVa`] object.
     pub fn new(flags: AllocFlags) -> Result<GpuVaAlloc<T>, AllocError> {
diff --git a/rust/kernel/drm/gpuvm/vm_bo.rs b/rust/kernel/drm/gpuvm/vm_bo.rs
index c064ac63897be..a30f838c11b8e 100644
--- a/rust/kernel/drm/gpuvm/vm_bo.rs
+++ b/rust/kernel/drm/gpuvm/vm_bo.rs
@@ -19,6 +19,15 @@ pub struct GpuVmBo<T: DriverGpuVm> {
     data: T::VmBoData,
 }
 
+// SAFETY: It is safe to send a `GpuVmBo<T>` to another thread: dropping it there drops
+// `T::VmBoData` and the GEM `T::Object`, both `Send` by the `DriverGpuVm` bounds.
+unsafe impl<T: DriverGpuVm> Send for GpuVmBo<T> {}
+
+// SAFETY: It is safe to share a `&GpuVmBo<T>` between threads: it effectively shares
+// `&T::VmBoData` and the GEM `&T::Object` (both `Sync`), and any thread may upgrade to an
+// `ARef` and ultimately drop them (both `Send`), per the `DriverGpuVm` bounds.
+unsafe impl<T: DriverGpuVm> Sync for GpuVmBo<T> {}
+
 // SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`.
 unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVmBo<T> {
     fn inc_ref(&self) {
diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs
index cf328101dde4e..64af9eacc3066 100644
--- a/rust/kernel/drm/ioctl.rs
+++ b/rust/kernel/drm/ioctl.rs
@@ -70,6 +70,18 @@ pub mod internal {
     pub use bindings::drm_device;
     pub use bindings::drm_file;
     pub use bindings::drm_ioctl_desc;
+
+    /// Cast an [`Ioctl`] DRM device pointer to [`Registered`], preserving the driver type
+    /// parameter `T`.
+    ///
+    /// Used by [`declare_drm_ioctls!`] to anchor type inference.
+    #[doc(hidden)]
+    #[inline]
+    pub const fn __dev_ctx_cast<T: crate::drm::Driver>(
+        ptr: *const crate::drm::Device<T, crate::drm::Ioctl>,
+    ) -> *const crate::drm::Device<T, crate::drm::Registered> {
+        ptr.cast()
+    }
 }
 
 /// Declare the DRM ioctls for a driver.
@@ -82,7 +94,8 @@ pub mod internal {
 /// `user_callback` should have the following prototype:
 ///
 /// ```ignore
-/// fn foo(device: &kernel::drm::Device<Self>,
+/// fn foo(device: &kernel::drm::Device<Self, kernel::drm::Registered>,
+///        reg_data: &Self::RegistrationData<'_>,
 ///        data: &mut uapi::argument_type,
 ///        file: &kernel::drm::File<Self::File>,
 /// ) -> Result<u32>
@@ -131,10 +144,45 @@ macro_rules! declare_drm_ioctls {
                             // - The DRM device must have been registered when we're called through
                             //   an IOCTL.
                             //
+                            // INVARIANT: The `Ioctl` context requires that the device has been
+                            // registered via `drm_dev_register()` at some point; the DRM core
+                            // guarantees this for ioctl dispatch callbacks.
+                            //
                             // FIXME: Currently there is nothing enforcing that the types of the
                             // dev/file match the current driver these ioctls are being declared
                             // for, and it's not clear how to enforce this within the type system.
-                            let dev = $crate::drm::device::Device::from_raw(raw_dev);
+                            let dev: &$crate::drm::device::Device<_, $crate::drm::Ioctl> =
+                                $crate::drm::device::Device::from_raw(raw_dev);
+
+                            // Type-inference anchor: the closure is never called but ties `dev`'s
+                            // type to `$func`'s first parameter, which the compiler cannot infer
+                            // through method resolution and associated-type projections alone.
+                            #[allow(unreachable_code)]
+                            let _ = || {
+                                let __ptr = $crate::drm::ioctl::internal::__dev_ctx_cast(
+                                    ::core::ptr::from_ref(dev),
+                                );
+
+                                $func(
+                                    // SAFETY: This closure is never executed; the dereference
+                                    // exists purely to unify the type parameter with `$func`.
+                                    // The pointer is valid regardless.
+                                    unsafe { &*__ptr },
+                                    unreachable!(),
+                                    unreachable!(),
+                                    unreachable!(),
+                                )
+                            };
+
+                            // Enforce that the handler accepts higher-ranked
+                            // lifetimes, preventing it from requiring 'static
+                            // references that could escape this scope.
+                            let _: for<'a> fn(&'a _, &'a _, &'a mut _, &'a _) -> _ = $func;
+
+                            let Some(guard) = dev.registration_guard() else {
+                                return $crate::error::code::ENODEV.to_errno();
+                            };
+
                             // SAFETY: The ioctl argument has size `_IOC_SIZE(cmd)`, which we
                             // asserted above matches the size of this type, and all bit patterns of
                             // UAPI structs must be valid.
@@ -147,7 +195,9 @@ macro_rules! declare_drm_ioctls {
                             // SAFETY: This is just the DRM file structure
                             let file = unsafe { $crate::drm::File::from_raw(raw_file) };
 
-                            match $func(dev, data, file) {
+                            match guard.registration_data_with(|reg_data| {
+                                $func(&*guard, reg_data, data, file)
+                            }) {
                                 Err(e) => e.to_errno(),
                                 Ok(i) => i.try_into()
                                             .unwrap_or($crate::error::code::ERANGE.to_errno()),
diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs
index a66e7166f66b5..fd6ed35bc35a9 100644
--- a/rust/kernel/drm/mod.rs
+++ b/rust/kernel/drm/mod.rs
@@ -11,8 +11,10 @@
 
 pub use self::device::Device;
 pub use self::device::DeviceContext;
+pub use self::device::Ioctl;
+pub use self::device::Normal;
 pub use self::device::Registered;
-pub use self::device::Uninit;
+pub use self::device::RegistrationGuard;
 pub use self::device::UnregisteredDevice;
 pub use self::driver::Driver;
 pub use self::driver::DriverInfo;
diff --git a/rust/kernel/faux.rs b/rust/kernel/faux.rs
index 43b4974f48cd2..cd4198fbb2322 100644
--- a/rust/kernel/faux.rs
+++ b/rust/kernel/faux.rs
@@ -9,15 +9,63 @@
 use crate::{
     bindings,
     device,
-    prelude::*, //
+    prelude::*,
+    types::Opaque, //
 };
-use core::ptr::{
-    addr_of_mut,
-    null,
-    null_mut,
-    NonNull, //
+use core::{
+    marker::PhantomData,
+    ptr::{
+        null,
+        null_mut,
+        NonNull, //
+    },
 };
 
+/// A faux device.
+///
+/// A faux device is a virtual device backed by the faux bus, primarily used for scenarios where a
+/// real hardware device is not available or for testing.
+///
+/// # Invariants
+///
+/// The underlying `struct faux_device` is valid.
+#[repr(transparent)]
+pub struct Device<Ctx: device::DeviceContext = device::Normal>(
+    Opaque<bindings::faux_device>,
+    PhantomData<Ctx>,
+);
+
+impl<Ctx: device::DeviceContext> Device<Ctx> {
+    #[inline]
+    fn as_raw(&self) -> *mut bindings::faux_device {
+        self.0.get()
+    }
+
+    /// # Safety
+    ///
+    /// `ptr` must be a valid pointer to a `struct faux_device`.
+    #[inline]
+    unsafe fn from_raw<'a>(ptr: *mut bindings::faux_device) -> &'a Self {
+        // SAFETY: `Device` is a transparent wrapper of `Opaque<bindings::faux_device>`.
+        unsafe { &*ptr.cast() }
+    }
+}
+
+impl<Ctx: device::DeviceContext> AsRef<device::Device<Ctx>> for Device<Ctx> {
+    #[inline]
+    fn as_ref(&self) -> &device::Device<Ctx> {
+        // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid
+        // `struct faux_device`. `dev` points to a valid `struct device`.
+        unsafe { device::Device::from_raw(&raw mut (*self.as_raw()).dev) }
+    }
+}
+
+// SAFETY: `faux::Device` is a transparent wrapper of `struct faux_device`.
+// The offset is guaranteed to point to a valid device field inside `faux::Device`.
+unsafe impl<Ctx: device::DeviceContext> device::AsBusDevice<Ctx> for Device<Ctx> {
+    const OFFSET: usize = core::mem::offset_of!(bindings::faux_device, dev);
+}
+
 /// The registration of a faux device.
 ///
 /// This type represents the registration of a [`struct faux_device`]. When an instance of this type
@@ -25,7 +73,8 @@
 ///
 /// # Invariants
 ///
-/// `self.0` always holds a valid pointer to an initialized and registered [`struct faux_device`].
+/// - `self.0` always holds a valid pointer to an initialized and registered [`struct faux_device`].
+/// - This object is proof that the object described by this `Registration` is bound to a device.
 ///
 /// [`struct faux_device`]: srctree/include/linux/device/faux.h
 pub struct Registration(NonNull<bindings::faux_device>);
@@ -59,11 +108,19 @@ fn as_raw(&self) -> *mut bindings::faux_device {
     }
 }
 
-impl AsRef<device::Device> for Registration {
-    fn as_ref(&self) -> &device::Device {
-        // SAFETY: The underlying `device` in `faux_device` is guaranteed by the C API to be
-        // a valid initialized `device`.
-        unsafe { device::Device::from_raw(addr_of_mut!((*self.as_raw()).dev)) }
+impl AsRef<Device<device::Bound>> for Registration {
+    #[inline]
+    fn as_ref(&self) -> &Device<device::Bound> {
+        // SAFETY:
+        // - The underlying `struct faux_device` is guaranteed by the C API to be a valid
+        //   initialized `device`.
+        // - `faux_match()` always returns 1, and probe runs synchronously
+        //   (PROBE_FORCE_SYNCHRONOUS).
+        // - `suppress_bind_attrs = true` on faux_driver prevents userspace-triggered unbind via
+        //   sysfs.
+        // - `mem::forget(Registration)` is not a problem; if the `Registration` is leaked, the faux
+        //   device stays bound forever.
+        unsafe { Device::from_raw(self.as_raw()) }
     }
 }
 
diff --git a/rust/kernel/i2c.rs b/rust/kernel/i2c.rs
index 624b971ca8b0b..7655d61daac8e 100644
--- a/rust/kernel/i2c.rs
+++ b/rust/kernel/i2c.rs
@@ -404,6 +404,7 @@ pub fn index(&self) -> i32 {
     }
 
     /// Gets pointer to an `i2c_adapter` by index.
+    #[inline]
     pub fn get(index: i32) -> Result<ARef<Self>> {
         // SAFETY: `index` must refer to a valid I2C adapter; the kernel
         // guarantees that `i2c_get_adapter(index)` returns either a valid
@@ -425,11 +426,13 @@ pub fn get(index: i32) -> Result<ARef<Self>> {
 
 // SAFETY: Instances of `I2cAdapter` are always reference-counted.
 unsafe impl AlwaysRefCounted for I2cAdapter {
+    #[inline]
     fn inc_ref(&self) {
         // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero.
         unsafe { bindings::i2c_get_adapter(self.index()) };
     }
 
+    #[inline]
     unsafe fn dec_ref(obj: NonNull<Self>) {
         // SAFETY: The safety requirements guarantee that the refcount is non-zero.
         unsafe { bindings::i2c_put_adapter(obj.as_ref().as_raw()) }
diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs
index fcc7678fd9e3c..95f46bb75f9e8 100644
--- a/rust/kernel/io.rs
+++ b/rust/kernel/io.rs
@@ -4,9 +4,18 @@
 //!
 //! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h)
 
+use core::{
+    marker::PhantomData,
+    mem::MaybeUninit, //
+};
+
 use crate::{
     bindings,
-    prelude::*, //
+    prelude::*,
+    ptr::{
+        Alignment,
+        KnownSize, //
+    }, //
 };
 
 pub mod mem;
@@ -31,128 +40,226 @@
 /// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures.
 pub type ResourceSize = bindings::resource_size_t;
 
+/// Untyped I/O region.
+///
+/// This type can be used when an I/O region without known type information has a compile-time known
+/// minimum size (and a runtime known actual size).
+///
+/// # Invariants
+///
+/// - Size of the region is at least as large as the `SIZE` generic parameter.
+/// - Size of the region is multiple of 4.
+#[repr(C, align(4))]
+#[derive(FromBytes)]
+pub struct Region<const SIZE: usize = 0> {
+    inner: [u8],
+}
+
+impl<const SIZE: usize> Region<SIZE> {
+    /// Create a raw mutable pointer from given base address and size.
+    ///
+    /// `size` should be at least as large as the minimum size `SIZE`, and `base` and `size` should
+    /// be 4-byte aligned to uphold the type invariant.
+    ///
+    /// Just like other methods on raw pointers, it is not unsafe to create a raw pointer
+    /// that does not uphold the type invariants. However such pointers are not valid.
+    #[inline]
+    pub fn ptr_from_raw_parts_mut(base: *mut u8, size: usize) -> *mut Self {
+        core::ptr::slice_from_raw_parts_mut(base, size) as *mut Region<SIZE>
+    }
+
+    /// Create a raw mutable pointer from given base address and size.
+    ///
+    /// The alignment of `base` is checked, and `size` is checked against the minimum size specified
+    /// via const generics.
+    #[inline]
+    pub fn ptr_try_from_raw_parts_mut(base: *mut u8, size: usize) -> Result<*mut Self> {
+        if size < SIZE || base.align_offset(4) != 0 || !size.is_multiple_of(4) {
+            return Err(EINVAL);
+        }
+
+        Ok(Self::ptr_from_raw_parts_mut(base, size))
+    }
+}
+
+impl<const SIZE: usize> KnownSize for Region<SIZE> {
+    const MIN_SIZE: usize = SIZE;
+    // Alignment of 4 is the most common; different base types can be added once required.
+    const MIN_ALIGN: Alignment = Alignment::new::<4>();
+
+    #[inline(always)]
+    fn size(p: *const Self) -> usize {
+        (p as *const [u8]).len()
+    }
+}
+
+// SAFETY:
+// - Values read from I/O are always treated as initialized.
+// - Per type invariant the size is multiple of 4 and the type is 4-byte aligned, so it is padding
+//   free.
+//
+// This cannot be derived as `derive(IntoBytes)` as the padding free property comes from type
+// invariant which the macro does not know.
+unsafe impl<const SIZE: usize> IntoBytes for Region<SIZE> {
+    #[inline]
+    #[allow(unused)] // Rust 1.87+ stops requiring this and will emit unused warnings.
+    fn only_derive_is_allowed_to_implement_this_trait() {}
+}
+
 /// Raw representation of an MMIO region.
 ///
+/// `MmioRaw<T>` is equivalent to `T __iomem *` in C.
+///
 /// By itself, the existence of an instance of this structure does not provide any guarantees that
 /// the represented MMIO region does exist or is properly mapped.
 ///
 /// Instead, the bus specific MMIO implementation must convert this raw representation into an
 /// `Mmio` instance providing the actual memory accessors. Only by the conversion into an `Mmio`
 /// structure any guarantees are given.
-pub struct MmioRaw<const SIZE: usize = 0> {
-    addr: usize,
-    maxsize: usize,
+pub struct MmioRaw<T: ?Sized> {
+    /// Pointer is in I/O address space.
+    ///
+    /// The provenance does not matter, only the address and metadata do.
+    ptr: *mut T,
 }
 
-impl<const SIZE: usize> MmioRaw<SIZE> {
-    /// Returns a new `MmioRaw` instance on success, an error otherwise.
-    pub fn new(addr: usize, maxsize: usize) -> Result<Self> {
-        if maxsize < SIZE {
-            return Err(EINVAL);
+impl<T: ?Sized> Copy for MmioRaw<T> {}
+impl<T: ?Sized> Clone for MmioRaw<T> {
+    #[inline]
+    fn clone(&self) -> Self {
+        *self
+    }
+}
+
+// SAFETY: `MmioRaw` is just an address, so is thread-safe.
+unsafe impl<T: ?Sized> Send for MmioRaw<T> {}
+// SAFETY: `MmioRaw` is just an address, so is thread-safe.
+unsafe impl<T: ?Sized> Sync for MmioRaw<T> {}
+
+impl<T> MmioRaw<T> {
+    /// Create a `MmioRaw` from address.
+    #[inline]
+    pub fn new(addr: usize) -> Self {
+        Self {
+            ptr: core::ptr::without_provenance_mut(addr),
         }
+    }
+}
 
-        Ok(Self { addr, maxsize })
+impl<const SIZE: usize> MmioRaw<Region<SIZE>> {
+    /// Create a `MmioRaw` representing a I/O region with given size.
+    ///
+    /// The size is checked against the minimum size specified via const generics.
+    #[inline]
+    pub fn new_region(addr: usize, size: usize) -> Result<Self> {
+        Ok(Self {
+            ptr: Region::ptr_try_from_raw_parts_mut(core::ptr::without_provenance_mut(addr), size)?,
+        })
     }
+}
 
+impl<T: ?Sized + KnownSize> MmioRaw<T> {
     /// Returns the base address of the MMIO region.
     #[inline]
     pub fn addr(&self) -> usize {
-        self.addr
+        self.ptr.addr()
     }
 
-    /// Returns the maximum size of the MMIO region.
+    /// Returns the size of the MMIO region.
     #[inline]
-    pub fn maxsize(&self) -> usize {
-        self.maxsize
+    pub fn size(&self) -> usize {
+        KnownSize::size(self.ptr)
     }
 }
 
-/// IO-mapped memory region.
-///
-/// The creator (usually a subsystem / bus such as PCI) is responsible for creating the
-/// mapping, performing an additional region request etc.
-///
-/// # Invariant
-///
-/// `addr` is the start and `maxsize` the length of valid I/O mapped memory region of size
-/// `maxsize`.
-///
-/// # Examples
-///
-/// ```no_run
-/// use kernel::{
-///     bindings,
-///     ffi::c_void,
-///     io::{
-///         Io,
-///         IoKnownSize,
-///         Mmio,
-///         MmioRaw,
-///         PhysAddr,
-///     },
-/// };
-/// use core::ops::Deref;
-///
-/// // See also `pci::Bar` for a real example.
-/// struct IoMem<const SIZE: usize>(MmioRaw<SIZE>);
-///
-/// impl<const SIZE: usize> IoMem<SIZE> {
-///     /// # Safety
-///     ///
-///     /// [`paddr`, `paddr` + `SIZE`) must be a valid MMIO region that is mappable into the CPUs
-///     /// virtual address space.
-///     unsafe fn new(paddr: usize) -> Result<Self>{
-///         // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is
-///         // valid for `ioremap`.
-///         let addr = unsafe { bindings::ioremap(paddr as PhysAddr, SIZE) };
-///         if addr.is_null() {
-///             return Err(ENOMEM);
-///         }
-///
-///         Ok(IoMem(MmioRaw::new(addr as usize, SIZE)?))
-///     }
-/// }
-///
-/// impl<const SIZE: usize> Drop for IoMem<SIZE> {
-///     fn drop(&mut self) {
-///         // SAFETY: `self.0.addr()` is guaranteed to be properly mapped by `Self::new`.
-///         unsafe { bindings::iounmap(self.0.addr() as *mut c_void); };
-///     }
-/// }
-///
-/// impl<const SIZE: usize> Deref for IoMem<SIZE> {
-///    type Target = Mmio<SIZE>;
-///
-///    fn deref(&self) -> &Self::Target {
-///         // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`.
-///         unsafe { Mmio::from_raw(&self.0) }
-///    }
-/// }
-///
-///# fn no_run() -> Result<(), Error> {
-/// // SAFETY: Invalid usage for example purposes.
-/// let iomem = unsafe { IoMem::<{ core::mem::size_of::<u32>() }>::new(0xBAAAAAAD)? };
-/// iomem.write32(0x42, 0x0);
-/// assert!(iomem.try_write32(0x42, 0x0).is_ok());
-/// assert!(iomem.try_write32(0x42, 0x4).is_err());
-/// # Ok(())
-/// # }
-/// ```
-#[repr(transparent)]
-pub struct Mmio<const SIZE: usize = 0>(MmioRaw<SIZE>);
-
-/// Checks whether an access of type `U` at the given `offset`
+/// Checks whether an access of type `U` at the given `base` and the given `offset`
 /// is valid within this region.
+///
+/// The `base` is used for alignment checking only. This can be set to 0 to skip the check.
 #[inline]
-const fn offset_valid<U>(offset: usize, size: usize) -> bool {
-    let type_size = core::mem::size_of::<U>();
-    if let Some(end) = offset.checked_add(type_size) {
-        end <= size && offset % type_size == 0
+const fn offset_valid<U>(base: usize, offset: usize, size: usize) -> bool {
+    if let Some(end) = offset.checked_add(size_of::<U>()) {
+        end <= size && (base.wrapping_add(offset) % align_of::<U>() == 0)
     } else {
         false
     }
 }
 
+/// Returns a view for a given `offset`, performing compile-time bound checks.
+// Always inline to optimize out error path of `build_assert`.
+#[inline(always)]
+fn io_view_assert<'a, IO: Io<'a>, U>(
+    this: IO,
+    offset: usize,
+) -> <IO::Backend as IoBackend>::View<'a, U> {
+    // We cannot check alignment with `offset_valid` using `ptr.addr()`. So set 0 for it and
+    // ensure alignment by checking that the alignment of `U` is smaller or equal to the
+    // alignment of `IO::Target`.
+    const_assert!(Alignment::of::<U>().as_usize() <= IO::Target::MIN_ALIGN.as_usize());
+    build_assert!(offset_valid::<U>(0, offset, IO::Target::MIN_SIZE));
+
+    let view = this.as_view();
+    let ptr = IO::Backend::as_ptr(view);
+    let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset);
+    // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a
+    // valid projection.
+    unsafe { IO::Backend::project_view(view, projected_ptr) }
+}
+
+/// Returns a view for a given `offset`, performing runtime bound checks.
+#[inline]
+fn io_view<'a, IO: Io<'a>, U>(
+    this: IO,
+    offset: usize,
+) -> Result<<IO::Backend as IoBackend>::View<'a, U>> {
+    let view = this.as_view();
+    let ptr = IO::Backend::as_ptr(view);
+
+    if !offset_valid::<U>(ptr.addr(), offset, KnownSize::size(ptr)) {
+        return Err(EINVAL);
+    }
+
+    let projected_ptr = ptr.cast::<U>().wrapping_byte_add(offset);
+    // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a
+    // valid projection.
+    Ok(unsafe { IO::Backend::project_view(view, projected_ptr) })
+}
+
+/// I/O backends.
+///
+/// This is an abstract representation to be implemented by arbitrary I/O
+/// backends (e.g. MMIO, PCI config space, etc.).
+///
+/// The base trait only defines the projection operations; which I/O methods are available depends
+/// on which [`IoCapable<T>`] traits are implemented for the type. For example, for MMIO regions,
+/// all widths (u8, u16, u32, and u64 on 64-bit systems) are typically supported. For PCI
+/// configuration space, u8, u16, and u32 are supported but u64 is not.
+///
+/// This trait is separate from the `Io` trait as multiple different I/O types may share the same
+/// operation.
+pub trait IoBackend {
+    /// View type for this I/O backend.
+    type View<'a, T: ?Sized + KnownSize>: IoBase<'a, Backend = Self, Target = T>;
+
+    /// Convert a `view` to a raw pointer for projection.
+    ///
+    /// The returned pointer is private implementation detail of the backend; it is likely not
+    /// valid. It should not be dereferenced.
+    fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T;
+
+    /// Project `view` to its subregion indicated by `ptr`.
+    ///
+    /// If input `view` is valid, returned view must also be valid.
+    ///
+    /// # Safety
+    ///
+    /// `ptr` must be a projection of `Self::as_ptr(view)`.
+    unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
+        view: Self::View<'a, T>,
+        ptr: *mut U,
+    ) -> Self::View<'a, U>;
+}
+
 /// Trait indicating that an I/O backend supports operations of a certain type and providing an
 /// implementation for these operations.
 ///
@@ -161,20 +268,75 @@ const fn offset_valid<U>(offset: usize, size: usize) -> bool {
 /// For example, a PCI configuration space may implement `IoCapable<u8>`, `IoCapable<u16>`,
 /// and `IoCapable<u32>`, but not `IoCapable<u64>`, while an MMIO region on a 64-bit
 /// system might implement all four.
-pub trait IoCapable<T> {
-    /// Performs an I/O read of type `T` at `address` and returns the result.
+pub trait IoCapable<T>: IoBackend {
+    /// Performs an I/O read of type `T` at `view` and returns the result.
+    fn io_read<'a>(view: Self::View<'a, T>) -> T;
+
+    /// Performs an I/O write of `value` at `view`.
+    fn io_write<'a>(view: Self::View<'a, T>, value: T);
+}
+
+/// Trait indicating that an I/O backend supports memory copy operations.
+pub trait IoCopyable: IoBackend {
+    /// Copy contents of `view` to `buffer`.
     ///
     /// # Safety
     ///
-    /// The range `[address..address + size_of::<T>()]` must be within the bounds of `Self`.
-    unsafe fn io_read(&self, address: usize) -> T;
+    /// - `buffer` is valid for volatile write for `view.size()` bytes.
+    /// - `buffer` should not overlap with `view`.
+    unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8);
 
-    /// Performs an I/O write of `value` at `address`.
+    /// Copy contents from `buffer` to `view`.
     ///
     /// # Safety
     ///
-    /// The range `[address..address + size_of::<T>()]` must be within the bounds of `Self`.
-    unsafe fn io_write(&self, value: T, address: usize);
+    /// - `buffer` is valid for volatile read for `view.size()` bytes.
+    /// - `buffer` should not overlap with `view`.
+    unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8);
+
+    /// Copy from `view` and return the value.
+    #[inline]
+    fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T {
+        // Project `self` to `[u8]`.
+        let ptr = Self::as_ptr(view);
+        // SAFETY: This is a identity projection.
+        let slice_view = unsafe {
+            Self::project_view(
+                view,
+                core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()),
+            )
+        };
+
+        let mut buf = MaybeUninit::<T>::uninit();
+        // SAFETY:
+        // - `buf.as_mut_ptr()` is valid for write for `size_of::<T>()` bytes.
+        // - `buf` is local so `buf.as_mut_ptr()` cannot overlap with `slice_view`.
+        unsafe { Self::copy_from_io(slice_view, buf.as_mut_ptr().cast()) };
+        // SAFETY: `T: FromBytes` guarantee that all bit patterns are valid.
+        unsafe { buf.assume_init() }
+    }
+
+    /// Copy `value` to `view`.
+    ///
+    /// Destructor of `value` will not be executed, consistent with [`zerocopy::transmute`].
+    #[inline]
+    fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) {
+        // Project `self` to `[u8]`.
+        let ptr = Self::as_ptr(view);
+        // SAFETY: This is a identity projection.
+        let slice_view = unsafe {
+            Self::project_view(
+                view,
+                core::ptr::slice_from_raw_parts_mut::<u8>(ptr.cast(), size_of::<T>()),
+            )
+        };
+
+        // SAFETY:
+        // - `&raw const value` is valid for read for `size_of::<T>()` bytes.
+        // - `value` is local so `&raw const value` cannot overlap with `slice_view`.
+        unsafe { Self::copy_to_io(slice_view, (&raw const value).cast()) };
+        core::mem::forget(value);
+    }
 }
 
 /// Describes a given I/O location: its offset, width, and type to convert the raw value from and
@@ -186,15 +348,16 @@ pub trait IoCapable<T> {
 /// (for primitive types like [`u32`]) and typed ones (like those generated by the [`register!`]
 /// macro).
 ///
-/// An `IoLoc<T>` carries three pieces of information:
+/// An `IoLoc<Base, T>` carries the following pieces of information:
 ///
+/// - The valid `Base` to operate on. For most registers, this should be [`Region`].
 /// - The offset to access (returned by [`IoLoc::offset`]),
 /// - The width of the access (determined by [`IoLoc::IoType`]),
 /// - The type `T` in which the raw data is returned or provided.
 ///
 /// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type
 /// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`).
-pub trait IoLoc<T> {
+pub trait IoLoc<Base: ?Sized, T> {
     /// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset).
     type IoType: Into<T> + From<T>;
 
@@ -202,12 +365,12 @@ pub trait IoLoc<T> {
     fn offset(self) -> usize;
 }
 
-/// Implements [`IoLoc<$ty>`] for [`usize`], allowing [`usize`] to be used as a parameter of
-/// [`Io::read`] and [`Io::write`].
+/// Implements [`IoLoc<Region<SIZE>, $ty>`] for [`usize`], allowing [`usize`] to be used as a
+/// parameter of [`Io::read`] and [`Io::write`].
 macro_rules! impl_usize_ioloc {
     ($($ty:ty),*) => {
         $(
-            impl IoLoc<$ty> for usize {
+            impl<const SIZE: usize> IoLoc<Region<SIZE>, $ty> for usize {
                 type IoType = $ty;
 
                 #[inline(always)]
@@ -225,181 +388,414 @@ fn offset(self) -> usize {
 /// Types implementing this trait (e.g. MMIO BARs or PCI config regions)
 /// can perform I/O operations on regions of memory.
 ///
-/// This is an abstract representation to be implemented by arbitrary I/O
-/// backends (e.g. MMIO, PCI config space, etc.).
+/// This trait defines which backend shall be used for I/O operations and provides a method to
+/// convert into [`IoBackend::View`]. Users should use the [`Io`] trait which provides the actual
+/// methods to perform I/O operations.
+///
+/// This should be implemented on cheaply copyable handles, such as references or view types.
+pub trait IoBase<'a>: Copy {
+    /// Type that defines all I/O operations.
+    type Backend: IoBackend;
+
+    /// Type of this I/O region. For untyped regions, [`Region`] can be used.
+    type Target: ?Sized + KnownSize;
+
+    /// Return a view that covers the full region.
+    fn as_view(self) -> <Self::Backend as IoBackend>::View<'a, Self::Target>;
+}
+
+/// Extension trait to provide I/O operation methods to types that implement [`IoBase`].
 ///
-/// The [`Io`] trait provides:
-/// - Base address and size information
+/// This trait provides:
 /// - Helper methods for offset validation and address calculation
 /// - Fallible (runtime checked) accessors for different data widths
 ///
-/// Which I/O methods are available depends on which [`IoCapable<T>`] traits
-/// are implemented for the type.
-///
-/// # Examples
-///
-/// For MMIO regions, all widths (u8, u16, u32, and u64 on 64-bit systems) are typically
-/// supported. For PCI configuration space, u8, u16, and u32 are supported but u64 is not.
-pub trait Io {
-    /// Returns the base address of this mapping.
-    fn addr(&self) -> usize;
+/// Which I/O methods are available depends on the associated [`IoBackend`] implementation.
+pub trait Io<'a>: IoBase<'a> {
+    /// Returns the size of this I/O region.
+    #[inline]
+    fn size(self) -> usize {
+        KnownSize::size(Self::Backend::as_ptr(self.as_view()))
+    }
 
-    /// Returns the maximum size of this mapping.
-    fn maxsize(&self) -> usize;
+    /// Returns the length of the slice in number of elements.
+    #[inline]
+    fn len<T>(self) -> usize
+    where
+        Self: Io<'a, Target = [T]>,
+    {
+        Self::Backend::as_ptr(self.as_view()).len()
+    }
+
+    /// Returns `true` if the slice has a length of 0.
+    #[inline]
+    fn is_empty<T>(self) -> bool
+    where
+        Self: Io<'a, Target = [T]>,
+    {
+        self.len() == 0
+    }
 
-    /// Returns the absolute I/O address for a given `offset`,
-    /// performing runtime bound checks.
+    /// Try to convert into a different typed I/O view.
+    ///
+    /// A runtime check is performed to ensure that the target type is of same or smaller size to
+    /// current type, and the current view is properly aligned for the target type. Returns
+    /// `Err(EINVAL)` if the runtime check fails.
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// use kernel::io::{
+    ///     io_project,
+    ///     Mmio,
+    ///     Io,
+    ///     Region,
+    /// };
+    /// #[derive(FromBytes, IntoBytes)]
+    /// #[repr(C)]
+    /// struct MyStruct { field: u32, }
+    ///
+    /// # fn test(mmio: &Mmio<'_, Region>) -> Result {
+    /// // let mmio: Mmio<'_, Region>;
+    /// let whole: Mmio<'_, MyStruct> = mmio.try_cast()?;
+    /// # Ok::<(), Error>(()) }
+    /// ```
     #[inline]
-    fn io_addr<U>(&self, offset: usize) -> Result<usize> {
-        if !offset_valid::<U>(offset, self.maxsize()) {
+    fn try_cast<U>(self) -> Result<<Self::Backend as IoBackend>::View<'a, U>>
+    where
+        Self::Target: FromBytes + IntoBytes,
+        U: FromBytes + IntoBytes,
+    {
+        let view = self.as_view();
+        let ptr = Self::Backend::as_ptr(view);
+
+        if size_of::<U>() > KnownSize::size(ptr) {
             return Err(EINVAL);
         }
 
-        // Probably no need to check, since the safety requirements of `Self::new` guarantee that
-        // this can't overflow.
-        self.addr().checked_add(offset).ok_or(EINVAL)
+        if ptr.addr() % align_of::<U>() != 0 {
+            return Err(EINVAL);
+        }
+
+        // SAFETY: We have checked bounds and alignment, so this is a valid projection.
+        Ok(unsafe { Self::Backend::project_view(view, ptr.cast()) })
+    }
+
+    /// Read a value from I/O.
+    ///
+    /// This only works for primitives supported by the I/O backend.
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// # use kernel::io::*;
+    /// # fn test_read_val(mmio: Mmio<'_, u32>) {
+    /// // let mmio: Mmio<'_, u32>;
+    /// let val: u32 = mmio.read_val();
+    /// # }
+    /// ```
+    #[inline]
+    fn read_val(self) -> Self::Target
+    where
+        Self::Backend: IoCapable<Self::Target>,
+        Self::Target: Sized,
+    {
+        Self::Backend::io_read(self.as_view())
+    }
+
+    /// Write a value to I/O.
+    ///
+    /// This only works for primitives supported by the I/O backend.
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// # use kernel::io::*;
+    /// # fn test_write_val(mmio: Mmio<'_, u32>) {
+    /// // let mmio: Mmio<'_, u32>;
+    /// mmio.write_val(1u32);
+    /// # }
+    /// ```
+    #[inline]
+    fn write_val(self, value: Self::Target)
+    where
+        Self::Backend: IoCapable<Self::Target>,
+        Self::Target: Sized,
+    {
+        Self::Backend::io_write(self.as_view(), value)
+    }
+
+    /// Copy-read from I/O memory.
+    ///
+    /// This is equivalent to reading from the I/O memory with byte-wise copy, although the actual
+    /// implementation might be more efficient. There is no atomicity guarantee. Note that for some
+    /// backends (e.g. `Mmio`), this can read different value compared to [`read_val`] as
+    /// byte-swapping is not performed.
+    ///
+    /// [`read_val`]: Io::read_val
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// # use kernel::io::*;
+    /// # fn test_copy_read(mmio: Mmio<'_, [u8; 6]>) {
+    /// // let mmio: Mmio<'_, [u8; 6]>;
+    /// let val: [u8; 6] = mmio.copy_read();
+    /// # }
+    /// ```
+    #[inline]
+    fn copy_read(self) -> Self::Target
+    where
+        Self::Backend: IoCopyable,
+        Self::Target: Sized + FromBytes,
+    {
+        Self::Backend::copy_read(self.as_view())
+    }
+
+    /// Copy-write to I/O memory.
+    ///
+    /// This is equivalent to writing to the I/O memory with byte-wise copy, although the actual
+    /// implementation might be more efficient. There is no atomicity guarantee. Note that for some
+    /// backends (e.g. `Mmio`), this can write different value compared to [`write_val`] as
+    /// byte-swapping is not performed.
+    ///
+    /// [`write_val`]: Io::write_val
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// # use kernel::io::*;
+    /// # fn test_copy_write(mmio: Mmio<'_, [u8; 6]>) {
+    /// // let mmio: Mmio<'_, [u8; 6]>;
+    /// mmio.copy_write([0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]);
+    /// # }
+    /// ```
+    #[inline]
+    fn copy_write(self, value: Self::Target)
+    where
+        Self::Backend: IoCopyable,
+        Self::Target: Sized + IntoBytes,
+    {
+        Self::Backend::copy_write(self.as_view(), value);
+    }
+
+    /// Copy bytes from `data` to I/O memory.
+    ///
+    /// # Panics
+    ///
+    /// This function will panic if the length of `self` differs from the length of `data`, similar
+    /// to [`[u8]::copy_from_slice`].
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// # use kernel::io::*;
+    /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) {
+    /// // let mmio: Mmio<'_, [u8]>;
+    /// mmio.copy_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]);
+    /// # }
+    /// ```
+    #[inline]
+    fn copy_from_slice(self, data: &[u8])
+    where
+        Self::Backend: IoCopyable,
+        Self: Io<'a, Target = [u8]>,
+    {
+        assert_eq!(self.len(), data.len());
+
+        // SAFETY: `data.as_ptr()` is valid for read for `self.size()` bytes.
+        unsafe {
+            Self::Backend::copy_to_io(self.as_view(), data.as_ptr());
+        }
+    }
+
+    /// Copy bytes from I/O memory to `data`.
+    ///
+    /// # Panics
+    ///
+    /// This function will panic if the length of `self` differs from the length of `data`, similar
+    /// to [`[u8]::copy_from_slice`].
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// # use kernel::io::*;
+    /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) {
+    /// // let mmio: Mmio<'_, [u8]>;
+    /// let mut buf = [0; 6];
+    /// mmio.copy_to_slice(&mut buf);
+    /// # }
+    /// ```
+    #[inline]
+    fn copy_to_slice(self, data: &mut [u8])
+    where
+        Self::Backend: IoCopyable,
+        Self: Io<'a, Target = [u8]>,
+    {
+        assert_eq!(self.len(), data.len());
+
+        // SAFETY: `data.as_mut_ptr()` is valid for write for `self.size()` bytes.
+        unsafe {
+            Self::Backend::copy_from_io(self.as_view(), data.as_mut_ptr());
+        }
     }
 
     /// Fallible 8-bit read with runtime bounds check.
     #[inline(always)]
-    fn try_read8(&self, offset: usize) -> Result<u8>
+    fn try_read8(self, offset: usize) -> Result<u8>
     where
-        Self: IoCapable<u8>,
+        usize: IoLoc<Self::Target, u8, IoType = u8>,
+        Self::Backend: IoCapable<u8>,
     {
         self.try_read(offset)
     }
 
     /// Fallible 16-bit read with runtime bounds check.
     #[inline(always)]
-    fn try_read16(&self, offset: usize) -> Result<u16>
+    fn try_read16(self, offset: usize) -> Result<u16>
     where
-        Self: IoCapable<u16>,
+        usize: IoLoc<Self::Target, u16, IoType = u16>,
+        Self::Backend: IoCapable<u16>,
     {
         self.try_read(offset)
     }
 
     /// Fallible 32-bit read with runtime bounds check.
     #[inline(always)]
-    fn try_read32(&self, offset: usize) -> Result<u32>
+    fn try_read32(self, offset: usize) -> Result<u32>
     where
-        Self: IoCapable<u32>,
+        usize: IoLoc<Self::Target, u32, IoType = u32>,
+        Self::Backend: IoCapable<u32>,
     {
         self.try_read(offset)
     }
 
     /// Fallible 64-bit read with runtime bounds check.
     #[inline(always)]
-    fn try_read64(&self, offset: usize) -> Result<u64>
+    fn try_read64(self, offset: usize) -> Result<u64>
     where
-        Self: IoCapable<u64>,
+        usize: IoLoc<Self::Target, u64, IoType = u64>,
+        Self::Backend: IoCapable<u64>,
     {
         self.try_read(offset)
     }
 
     /// Fallible 8-bit write with runtime bounds check.
     #[inline(always)]
-    fn try_write8(&self, value: u8, offset: usize) -> Result
+    fn try_write8(self, value: u8, offset: usize) -> Result
     where
-        Self: IoCapable<u8>,
+        usize: IoLoc<Self::Target, u8, IoType = u8>,
+        Self::Backend: IoCapable<u8>,
     {
         self.try_write(offset, value)
     }
 
     /// Fallible 16-bit write with runtime bounds check.
     #[inline(always)]
-    fn try_write16(&self, value: u16, offset: usize) -> Result
+    fn try_write16(self, value: u16, offset: usize) -> Result
     where
-        Self: IoCapable<u16>,
+        usize: IoLoc<Self::Target, u16, IoType = u16>,
+        Self::Backend: IoCapable<u16>,
     {
         self.try_write(offset, value)
     }
 
     /// Fallible 32-bit write with runtime bounds check.
     #[inline(always)]
-    fn try_write32(&self, value: u32, offset: usize) -> Result
+    fn try_write32(self, value: u32, offset: usize) -> Result
     where
-        Self: IoCapable<u32>,
+        usize: IoLoc<Self::Target, u32, IoType = u32>,
+        Self::Backend: IoCapable<u32>,
     {
         self.try_write(offset, value)
     }
 
     /// Fallible 64-bit write with runtime bounds check.
     #[inline(always)]
-    fn try_write64(&self, value: u64, offset: usize) -> Result
+    fn try_write64(self, value: u64, offset: usize) -> Result
     where
-        Self: IoCapable<u64>,
+        usize: IoLoc<Self::Target, u64, IoType = u64>,
+        Self::Backend: IoCapable<u64>,
     {
         self.try_write(offset, value)
     }
 
     /// Infallible 8-bit read with compile-time bounds check.
     #[inline(always)]
-    fn read8(&self, offset: usize) -> u8
+    fn read8(self, offset: usize) -> u8
     where
-        Self: IoKnownSize + IoCapable<u8>,
+        usize: IoLoc<Self::Target, u8, IoType = u8>,
+        Self::Backend: IoCapable<u8>,
     {
         self.read(offset)
     }
 
     /// Infallible 16-bit read with compile-time bounds check.
     #[inline(always)]
-    fn read16(&self, offset: usize) -> u16
+    fn read16(self, offset: usize) -> u16
     where
-        Self: IoKnownSize + IoCapable<u16>,
+        usize: IoLoc<Self::Target, u16, IoType = u16>,
+        Self::Backend: IoCapable<u16>,
     {
         self.read(offset)
     }
 
     /// Infallible 32-bit read with compile-time bounds check.
     #[inline(always)]
-    fn read32(&self, offset: usize) -> u32
+    fn read32(self, offset: usize) -> u32
     where
-        Self: IoKnownSize + IoCapable<u32>,
+        usize: IoLoc<Self::Target, u32, IoType = u32>,
+        Self::Backend: IoCapable<u32>,
     {
         self.read(offset)
     }
 
     /// Infallible 64-bit read with compile-time bounds check.
     #[inline(always)]
-    fn read64(&self, offset: usize) -> u64
+    fn read64(self, offset: usize) -> u64
     where
-        Self: IoKnownSize + IoCapable<u64>,
+        usize: IoLoc<Self::Target, u64, IoType = u64>,
+        Self::Backend: IoCapable<u64>,
     {
         self.read(offset)
     }
 
     /// Infallible 8-bit write with compile-time bounds check.
     #[inline(always)]
-    fn write8(&self, value: u8, offset: usize)
+    fn write8(self, value: u8, offset: usize)
     where
-        Self: IoKnownSize + IoCapable<u8>,
+        usize: IoLoc<Self::Target, u8, IoType = u8>,
+        Self::Backend: IoCapable<u8>,
     {
         self.write(offset, value)
     }
 
     /// Infallible 16-bit write with compile-time bounds check.
     #[inline(always)]
-    fn write16(&self, value: u16, offset: usize)
+    fn write16(self, value: u16, offset: usize)
     where
-        Self: IoKnownSize + IoCapable<u16>,
+        usize: IoLoc<Self::Target, u16, IoType = u16>,
+        Self::Backend: IoCapable<u16>,
     {
         self.write(offset, value)
     }
 
     /// Infallible 32-bit write with compile-time bounds check.
     #[inline(always)]
-    fn write32(&self, value: u32, offset: usize)
+    fn write32(self, value: u32, offset: usize)
     where
-        Self: IoKnownSize + IoCapable<u32>,
+        usize: IoLoc<Self::Target, u32, IoType = u32>,
+        Self::Backend: IoCapable<u32>,
     {
         self.write(offset, value)
     }
 
     /// Infallible 64-bit write with compile-time bounds check.
     #[inline(always)]
-    fn write64(&self, value: u64, offset: usize)
+    fn write64(self, value: u64, offset: usize)
     where
-        Self: IoKnownSize + IoCapable<u64>,
+        usize: IoLoc<Self::Target, u64, IoType = u64>,
+        Self::Backend: IoCapable<u64>,
     {
         self.write(offset, value)
     }
@@ -414,9 +810,10 @@ fn write64(&self, value: u64, offset: usize)
     /// use kernel::io::{
     ///     Io,
     ///     Mmio,
+    ///     Region,
     /// };
     ///
-    /// fn do_reads(io: &Mmio) -> Result {
+    /// fn do_reads(io: Mmio<'_, Region>) -> Result {
     ///     // 32-bit read from address `0x10`.
     ///     let v: u32 = io.try_read(0x10)?;
     ///
@@ -427,15 +824,13 @@ fn write64(&self, value: u64, offset: usize)
     /// }
     /// ```
     #[inline(always)]
-    fn try_read<T, L>(&self, location: L) -> Result<T>
+    fn try_read<T, L>(self, location: L) -> Result<T>
     where
-        L: IoLoc<T>,
-        Self: IoCapable<L::IoType>,
+        L: IoLoc<Self::Target, T>,
+        Self::Backend: IoCapable<L::IoType>,
     {
-        let address = self.io_addr::<L::IoType>(location.offset())?;
-
-        // SAFETY: `address` has been validated by `io_addr`.
-        Ok(unsafe { self.io_read(address) }.into())
+        let view = io_view::<Self, L::IoType>(self, location.offset())?;
+        Ok(Self::Backend::io_read(view).into())
     }
 
     /// Generic fallible write with runtime bounds check.
@@ -448,9 +843,10 @@ fn try_read<T, L>(&self, location: L) -> Result<T>
     /// use kernel::io::{
     ///     Io,
     ///     Mmio,
+    ///     Region,
     /// };
     ///
-    /// fn do_writes(io: &Mmio) -> Result {
+    /// fn do_writes(io: Mmio<'_, Region>) -> Result {
     ///     // 32-bit write of value `1` at address `0x10`.
     ///     io.try_write(0x10, 1u32)?;
     ///
@@ -461,17 +857,14 @@ fn try_read<T, L>(&self, location: L) -> Result<T>
     /// }
     /// ```
     #[inline(always)]
-    fn try_write<T, L>(&self, location: L, value: T) -> Result
+    fn try_write<T, L>(self, location: L, value: T) -> Result
     where
-        L: IoLoc<T>,
-        Self: IoCapable<L::IoType>,
+        L: IoLoc<Self::Target, T>,
+        Self::Backend: IoCapable<L::IoType>,
     {
-        let address = self.io_addr::<L::IoType>(location.offset())?;
+        let view = io_view::<Self, L::IoType>(self, location.offset())?;
         let io_value = value.into();
-
-        // SAFETY: `address` has been validated by `io_addr`.
-        unsafe { self.io_write(io_value, address) }
-
+        Self::Backend::io_write(view, io_value);
         Ok(())
     }
 
@@ -486,6 +879,7 @@ fn try_write<T, L>(&self, location: L, value: T) -> Result
     ///     register,
     ///     Io,
     ///     Mmio,
+    ///     Region,
     /// };
     ///
     /// register! {
@@ -501,17 +895,17 @@ fn try_write<T, L>(&self, location: L, value: T) -> Result
     ///     }
     /// }
     ///
-    /// fn do_write_reg(io: &Mmio) -> Result {
+    /// fn do_write_reg(io: Mmio<'_, Region>) -> Result {
     ///
     ///     io.try_write_reg(VERSION::new(1, 0))
     /// }
     /// ```
     #[inline(always)]
-    fn try_write_reg<T, L, V>(&self, value: V) -> Result
+    fn try_write_reg<T, L, V>(self, value: V) -> Result
     where
-        L: IoLoc<T>,
-        V: LocatedRegister<Location = L, Value = T>,
-        Self: IoCapable<L::IoType>,
+        L: IoLoc<Self::Target, T>,
+        V: LocatedRegister<Self::Target, Location = L, Value = T>,
+        Self::Backend: IoCapable<L::IoType>,
     {
         let (location, value) = value.into_io_op();
 
@@ -531,29 +925,27 @@ fn try_write_reg<T, L, V>(&self, value: V) -> Result
     /// use kernel::io::{
     ///     Io,
     ///     Mmio,
+    ///     Region,
     /// };
     ///
-    /// fn do_update(io: &Mmio<0x1000>) -> Result {
+    /// fn do_update(io: Mmio<'_, Region<0x1000>>) -> Result {
     ///     io.try_update(0x10, |v: u32| {
     ///         v + 1
     ///     })
     /// }
     /// ```
     #[inline(always)]
-    fn try_update<T, L, F>(&self, location: L, f: F) -> Result
+    fn try_update<T, L, F>(self, location: L, f: F) -> Result
     where
-        L: IoLoc<T>,
-        Self: IoCapable<L::IoType>,
+        L: IoLoc<Self::Target, T>,
+        Self::Backend: IoCapable<L::IoType>,
         F: FnOnce(T) -> T,
     {
-        let address = self.io_addr::<L::IoType>(location.offset())?;
+        let view = io_view::<Self, L::IoType>(self, location.offset())?;
 
-        // SAFETY: `address` has been validated by `io_addr`.
-        let value: T = unsafe { self.io_read(address) }.into();
+        let value: T = Self::Backend::io_read(view).into();
         let io_value = f(value).into();
-
-        // SAFETY: `address` has been validated by `io_addr`.
-        unsafe { self.io_write(io_value, address) }
+        Self::Backend::io_write(view, io_value);
 
         Ok(())
     }
@@ -568,9 +960,10 @@ fn try_update<T, L, F>(&self, location: L, f: F) -> Result
     /// use kernel::io::{
     ///     Io,
     ///     Mmio,
+    ///     Region,
     /// };
     ///
-    /// fn do_reads(io: &Mmio<0x1000>) {
+    /// fn do_reads(io: Mmio<'_, Region<0x1000>>) {
     ///     // 32-bit read from address `0x10`.
     ///     let v: u32 = io.read(0x10);
     ///
@@ -579,15 +972,13 @@ fn try_update<T, L, F>(&self, location: L, f: F) -> Result
     /// }
     /// ```
     #[inline(always)]
-    fn read<T, L>(&self, location: L) -> T
+    fn read<T, L>(self, location: L) -> T
     where
-        L: IoLoc<T>,
-        Self: IoKnownSize + IoCapable<L::IoType>,
+        L: IoLoc<Self::Target, T>,
+        Self::Backend: IoCapable<L::IoType>,
     {
-        let address = self.io_addr_assert::<L::IoType>(location.offset());
-
-        // SAFETY: `address` has been validated by `io_addr_assert`.
-        unsafe { self.io_read(address) }.into()
+        let view = io_view_assert::<Self, L::IoType>(self, location.offset());
+        Self::Backend::io_read(view).into()
     }
 
     /// Generic infallible write with compile-time bounds check.
@@ -600,9 +991,10 @@ fn read<T, L>(&self, location: L) -> T
     /// use kernel::io::{
     ///     Io,
     ///     Mmio,
+    ///     Region,
     /// };
     ///
-    /// fn do_writes(io: &Mmio<0x1000>) {
+    /// fn do_writes(io: Mmio<'_, Region<0x1000>>) {
     ///     // 32-bit write of value `1` at address `0x10`.
     ///     io.write(0x10, 1u32);
     ///
@@ -611,16 +1003,14 @@ fn read<T, L>(&self, location: L) -> T
     /// }
     /// ```
     #[inline(always)]
-    fn write<T, L>(&self, location: L, value: T)
+    fn write<T, L>(self, location: L, value: T)
     where
-        L: IoLoc<T>,
-        Self: IoKnownSize + IoCapable<L::IoType>,
+        L: IoLoc<Self::Target, T>,
+        Self::Backend: IoCapable<L::IoType>,
     {
-        let address = self.io_addr_assert::<L::IoType>(location.offset());
+        let view = io_view_assert::<Self, L::IoType>(self, location.offset());
         let io_value = value.into();
-
-        // SAFETY: `address` has been validated by `io_addr_assert`.
-        unsafe { self.io_write(io_value, address) }
+        Self::Backend::io_write(view, io_value);
     }
 
     /// Generic infallible write of a fully-located register value.
@@ -634,6 +1024,7 @@ fn write<T, L>(&self, location: L, value: T)
     ///     register,
     ///     Io,
     ///     Mmio,
+    ///     Region,
     /// };
     ///
     /// register! {
@@ -649,16 +1040,16 @@ fn write<T, L>(&self, location: L, value: T)
     ///     }
     /// }
     ///
-    /// fn do_write_reg(io: &Mmio<0x1000>) {
+    /// fn do_write_reg(io: Mmio<'_, Region<0x1000>>) {
     ///     io.write_reg(VERSION::new(1, 0));
     /// }
     /// ```
     #[inline(always)]
-    fn write_reg<T, L, V>(&self, value: V)
+    fn write_reg<T, L, V>(self, value: V)
     where
-        L: IoLoc<T>,
-        V: LocatedRegister<Location = L, Value = T>,
-        Self: IoKnownSize + IoCapable<L::IoType>,
+        L: IoLoc<Self::Target, T>,
+        V: LocatedRegister<Self::Target, Location = L, Value = T>,
+        Self::Backend: IoCapable<L::IoType>,
     {
         let (location, value) = value.into_io_op();
 
@@ -678,143 +1069,208 @@ fn write_reg<T, L, V>(&self, value: V)
     /// use kernel::io::{
     ///     Io,
     ///     Mmio,
+    ///     Region,
     /// };
     ///
-    /// fn do_update(io: &Mmio<0x1000>) {
+    /// fn do_update(io: Mmio<'_, Region<0x1000>>) {
     ///     io.update(0x10, |v: u32| {
     ///         v + 1
     ///     })
     /// }
     /// ```
     #[inline(always)]
-    fn update<T, L, F>(&self, location: L, f: F)
+    fn update<T, L, F>(self, location: L, f: F)
     where
-        L: IoLoc<T>,
-        Self: IoKnownSize + IoCapable<L::IoType> + Sized,
+        L: IoLoc<Self::Target, T>,
+        Self::Backend: IoCapable<L::IoType>,
         F: FnOnce(T) -> T,
     {
-        let address = self.io_addr_assert::<L::IoType>(location.offset());
-
-        // SAFETY: `address` has been validated by `io_addr_assert`.
-        let value: T = unsafe { self.io_read(address) }.into();
+        let view = io_view_assert::<Self, L::IoType>(self, location.offset());
+        let value: T = Self::Backend::io_read(view).into();
         let io_value = f(value).into();
-
-        // SAFETY: `address` has been validated by `io_addr_assert`.
-        unsafe { self.io_write(io_value, address) }
+        Self::Backend::io_write(view, io_value);
     }
 }
 
-/// Trait for types with a known size at compile time.
+// Blanket implementation ensures that provided methods cannot be arbitrarily overridden by
+// implementers, which is relied upon for correctness and soundness.
+impl<'a, T: IoBase<'a>> Io<'a> for T {}
+
+/// A view of memory-mapped I/O region.
 ///
-/// This trait is implemented by I/O backends that have a compile-time known size,
-/// enabling the use of infallible I/O accessors with compile-time bounds checking.
+/// # Invariant
 ///
-/// Types implementing this trait can use the infallible methods in [`Io`] trait
-/// (e.g., `read8`, `write32`), which require `Self: IoKnownSize` bound.
-pub trait IoKnownSize: Io {
-    /// Minimum usable size of this region.
-    const MIN_SIZE: usize;
+/// `ptr` points to a valid and aligned memory-mapped I/O region for the duration lifetime `'a`.
+pub struct Mmio<'a, T: ?Sized> {
+    ptr: *mut T,
+    phantom: PhantomData<&'a ()>,
+}
 
-    /// Returns the absolute I/O address for a given `offset`,
-    /// performing compile-time bound checks.
-    // Always inline to optimize out error path of `build_assert`.
-    #[inline(always)]
-    fn io_addr_assert<U>(&self, offset: usize) -> usize {
-        build_assert!(offset_valid::<U>(offset, Self::MIN_SIZE));
+impl<T: ?Sized> Copy for Mmio<'_, T> {}
+impl<T: ?Sized> Clone for Mmio<'_, T> {
+    #[inline]
+    fn clone(&self) -> Self {
+        *self
+    }
+}
+
+impl<'a, T: ?Sized> Mmio<'a, T> {
+    /// Create a `Mmio`, providing the accessors to the MMIO mapping.
+    ///
+    /// # Safety
+    ///
+    /// `raw` represents a valid and aligned memory-mapped I/O region while `'a` is alive.
+    #[inline]
+    pub unsafe fn from_raw(raw: MmioRaw<T>) -> Self {
+        // INVARIANT: Per safety requirement.
+        Self {
+            ptr: raw.ptr,
+            phantom: PhantomData,
+        }
+    }
+}
+
+// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory.
+unsafe impl<T: ?Sized + Sync> Send for Mmio<'_, T> {}
 
-        self.addr() + offset
+// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory.
+unsafe impl<T: ?Sized + Sync> Sync for Mmio<'_, T> {}
+
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for Mmio<'a, T> {
+    type Backend = MmioBackend;
+    type Target = T;
+
+    #[inline]
+    fn as_view(self) -> Mmio<'a, T> {
+        self
+    }
+}
+
+/// I/O Backend for memory-mapped I/O.
+pub struct MmioBackend;
+
+impl IoBackend for MmioBackend {
+    type View<'a, T: ?Sized + KnownSize> = Mmio<'a, T>;
+
+    #[inline]
+    fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
+        view.ptr
+    }
+
+    #[inline]
+    unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
+        _view: Self::View<'a, T>,
+        ptr: *mut U,
+    ) -> Self::View<'a, U> {
+        // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid
+        // memory-mapped I/O region.
+        Mmio {
+            ptr,
+            phantom: PhantomData,
+        }
     }
 }
 
-/// Implements [`IoCapable`] on `$mmio` for `$ty` using `$read_fn` and `$write_fn`.
+/// Implements [`IoCapable`] on `$backend` for `$ty` using `$read_fn` and `$write_fn`.
 macro_rules! impl_mmio_io_capable {
-    ($mmio:ident, $(#[$attr:meta])* $ty:ty, $read_fn:ident, $write_fn:ident) => {
-        $(#[$attr])*
-        impl<const SIZE: usize> IoCapable<$ty> for $mmio<SIZE> {
-            unsafe fn io_read(&self, address: usize) -> $ty {
-                // SAFETY: By the trait invariant `address` is a valid address for MMIO operations.
-                unsafe { bindings::$read_fn(address as *const c_void) }
+    ($backend: ident, $ty:ty, $read_fn:ident, $write_fn:ident) => {
+        impl IoCapable<$ty> for $backend {
+            #[inline]
+            fn io_read(view: <$backend as IoBackend>::View<'_, $ty>) -> $ty {
+                // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both
+                // `MmioBackend` and `RelaxedMmioBackend`.
+                unsafe { bindings::$read_fn($backend::as_ptr(view).cast_const().cast()) }
             }
 
-            unsafe fn io_write(&self, value: $ty, address: usize) {
-                // SAFETY: By the trait invariant `address` is a valid address for MMIO operations.
-                unsafe { bindings::$write_fn(value, address as *mut c_void) }
+            #[inline]
+            fn io_write(view: <$backend as IoBackend>::View<'_, $ty>, value: $ty) {
+                // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both
+                // `MmioBackend` and `RelaxedMmioBackend`.
+                unsafe { bindings::$write_fn(value, $backend::as_ptr(view).cast()) }
             }
         }
     };
 }
 
 // MMIO regions support 8, 16, and 32-bit accesses.
-impl_mmio_io_capable!(Mmio, u8, readb, writeb);
-impl_mmio_io_capable!(Mmio, u16, readw, writew);
-impl_mmio_io_capable!(Mmio, u32, readl, writel);
+impl_mmio_io_capable!(MmioBackend, u8, readb, writeb);
+impl_mmio_io_capable!(MmioBackend, u16, readw, writew);
+impl_mmio_io_capable!(MmioBackend, u32, readl, writel);
 // MMIO regions on 64-bit systems also support 64-bit accesses.
-impl_mmio_io_capable!(
-    Mmio,
-    #[cfg(CONFIG_64BIT)]
-    u64,
-    readq,
-    writeq
-);
+#[cfg(CONFIG_64BIT)]
+impl_mmio_io_capable!(MmioBackend, u64, readq, writeq);
 
-impl<const SIZE: usize> Io for Mmio<SIZE> {
-    /// Returns the base address of this mapping.
+impl IoCopyable for MmioBackend {
     #[inline]
-    fn addr(&self) -> usize {
-        self.0.addr()
+    unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
+        // SAFETY:
+        // - `view.ptr` is valid MMIO memory for `view.size()` bytes.
+        // - `buffer` is valid for write for `view.size()` bytes.
+        unsafe {
+            bindings::memcpy_fromio(buffer.cast(), view.ptr.cast(), view.size());
+        }
     }
 
-    /// Returns the maximum size of this mapping.
     #[inline]
-    fn maxsize(&self) -> usize {
-        self.0.maxsize()
-    }
-}
-
-impl<const SIZE: usize> IoKnownSize for Mmio<SIZE> {
-    const MIN_SIZE: usize = SIZE;
-}
-
-impl<const SIZE: usize> Mmio<SIZE> {
-    /// Converts an `MmioRaw` into an `Mmio` instance, providing the accessors to the MMIO mapping.
-    ///
-    /// # Safety
-    ///
-    /// Callers must ensure that `addr` is the start of a valid I/O mapped memory region of size
-    /// `maxsize`.
-    pub unsafe fn from_raw(raw: &MmioRaw<SIZE>) -> &Self {
-        // SAFETY: `Mmio` is a transparent wrapper around `MmioRaw`.
-        unsafe { &*core::ptr::from_ref(raw).cast() }
+    unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
+        // SAFETY:
+        // - `view.ptr` is valid MMIO memory for `view.size()` bytes.
+        // - `buffer` is valid for read for `view.size()` bytes.
+        unsafe {
+            bindings::memcpy_toio(view.ptr.cast(), buffer.cast(), view.size());
+        }
     }
 }
 
-/// [`Mmio`] wrapper using relaxed accessors.
+/// [`Mmio`] but using relaxed accessors.
 ///
 /// This type provides an implementation of [`Io`] that uses relaxed I/O MMIO operands instead of
 /// the regular ones.
 ///
 /// See [`Mmio::relaxed`] for a usage example.
-#[repr(transparent)]
-pub struct RelaxedMmio<const SIZE: usize = 0>(Mmio<SIZE>);
+pub struct RelaxedMmio<'a, T: ?Sized>(Mmio<'a, T>);
+
+impl<T: ?Sized> Copy for RelaxedMmio<'_, T> {}
+impl<T: ?Sized> Clone for RelaxedMmio<'_, T> {
+    #[inline]
+    fn clone(&self) -> Self {
+        *self
+    }
+}
+
+/// I/O Backend for memory-mapped I/O, with relaxed access semantics.
+pub struct RelaxedMmioBackend;
+
+impl IoBackend for RelaxedMmioBackend {
+    type View<'a, T: ?Sized + KnownSize> = RelaxedMmio<'a, T>;
 
-impl<const SIZE: usize> Io for RelaxedMmio<SIZE> {
     #[inline]
-    fn addr(&self) -> usize {
-        self.0.addr()
+    fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
+        MmioBackend::as_ptr(view.0)
     }
 
     #[inline]
-    fn maxsize(&self) -> usize {
-        self.0.maxsize()
+    unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
+        view: Self::View<'a, T>,
+        ptr: *mut U,
+    ) -> Self::View<'a, U> {
+        // SAFETY: Per safety requirement.
+        RelaxedMmio(unsafe { MmioBackend::project_view(view.0, ptr) })
     }
 }
 
-impl<const SIZE: usize> IoKnownSize for RelaxedMmio<SIZE> {
-    const MIN_SIZE: usize = SIZE;
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for RelaxedMmio<'a, T> {
+    type Backend = RelaxedMmioBackend;
+    type Target = T;
+
+    #[inline]
+    fn as_view(self) -> RelaxedMmio<'a, T> {
+        self
+    }
 }
 
-impl<const SIZE: usize> Mmio<SIZE> {
-    /// Returns a [`RelaxedMmio`] reference that performs relaxed I/O operations.
+impl<'a, T: ?Sized> Mmio<'a, T> {
+    /// Returns a [`RelaxedMmio`] that performs relaxed I/O operations.
     ///
     /// Relaxed accessors do not provide ordering guarantees with respect to DMA or memory accesses
     /// and can be used when such ordering is not required.
@@ -825,31 +1281,457 @@ impl<const SIZE: usize> Mmio<SIZE> {
     /// use kernel::io::{
     ///     Io,
     ///     Mmio,
+    ///     Region,
     ///     RelaxedMmio,
     /// };
     ///
-    /// fn do_io(io: &Mmio<0x100>) {
+    /// fn do_io(io: Mmio<'_, Region<0x100>>) {
     ///     // The access is performed using `readl_relaxed` instead of `readl`.
     ///     let v = io.relaxed().read32(0x10);
     /// }
     ///
     /// ```
-    pub fn relaxed(&self) -> &RelaxedMmio<SIZE> {
-        // SAFETY: `RelaxedMmio` is `#[repr(transparent)]` over `Mmio`, so `Mmio<SIZE>` and
-        // `RelaxedMmio<SIZE>` have identical layout.
-        unsafe { core::mem::transmute(self) }
+    #[inline]
+    pub fn relaxed(self) -> RelaxedMmio<'a, T> {
+        RelaxedMmio(self)
     }
 }
 
 // MMIO regions support 8, 16, and 32-bit accesses.
-impl_mmio_io_capable!(RelaxedMmio, u8, readb_relaxed, writeb_relaxed);
-impl_mmio_io_capable!(RelaxedMmio, u16, readw_relaxed, writew_relaxed);
-impl_mmio_io_capable!(RelaxedMmio, u32, readl_relaxed, writel_relaxed);
+impl_mmio_io_capable!(RelaxedMmioBackend, u8, readb_relaxed, writeb_relaxed);
+impl_mmio_io_capable!(RelaxedMmioBackend, u16, readw_relaxed, writew_relaxed);
+impl_mmio_io_capable!(RelaxedMmioBackend, u32, readl_relaxed, writel_relaxed);
 // MMIO regions on 64-bit systems also support 64-bit accesses.
-impl_mmio_io_capable!(
-    RelaxedMmio,
-    #[cfg(CONFIG_64BIT)]
-    u64,
-    readq_relaxed,
-    writeq_relaxed
-);
+#[cfg(CONFIG_64BIT)]
+impl_mmio_io_capable!(RelaxedMmioBackend, u64, readq_relaxed, writeq_relaxed);
+
+/// I/O Backend for system memory.
+pub struct SysMemBackend;
+
+impl IoBackend for SysMemBackend {
+    type View<'a, T: ?Sized + KnownSize> = SysMem<'a, T>;
+
+    #[inline]
+    fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
+        view.ptr
+    }
+
+    #[inline]
+    unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
+        _view: Self::View<'a, T>,
+        ptr: *mut U,
+    ) -> Self::View<'a, U> {
+        // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid
+        // kernel accessible memory region.
+        SysMem {
+            ptr,
+            phantom: PhantomData,
+        }
+    }
+}
+
+/// Implements [`IoCapable`] on `SysMemBackend` for `$ty` using `read_volatile` and
+/// `write_volatile`.
+macro_rules! impl_sysmem_io_capable {
+    ($ty:ty) => {
+        impl IoCapable<$ty> for SysMemBackend {
+            #[inline]
+            fn io_read(view: SysMem<'_, $ty>) -> $ty {
+                // SAFETY:
+                // - Per type invariant, `ptr` is valid and aligned.
+                // - Using read_volatile() here so that race with hardware is well-defined.
+                // - Using read_volatile() here is not sound if it races with other CPU per Rust
+                //   rules, but this is allowed per LKMM.
+                // - The macro is only used on primitives so all bit patterns are valid.
+                unsafe { view.ptr.read_volatile() }
+            }
+
+            #[inline]
+            fn io_write(view: SysMem<'_, $ty>, value: $ty) {
+                // SAFETY:
+                // - Per type invariant, `ptr` is valid and aligned.
+                // - Using write_volatile() here so that race with hardware is well-defined.
+                // - Using write_volatile() here is not sound if it races with other CPU per Rust
+                //   rules, but this is allowed per LKMM.
+                unsafe { view.ptr.write_volatile(value) }
+            }
+        }
+    };
+}
+
+impl_sysmem_io_capable!(u8);
+impl_sysmem_io_capable!(u16);
+impl_sysmem_io_capable!(u32);
+#[cfg(CONFIG_64BIT)]
+impl_sysmem_io_capable!(u64);
+
+impl IoCopyable for SysMemBackend {
+    #[inline]
+    unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
+        // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile.
+        // SAFETY:
+        // - `view.ptr` is in CPU address space and valid for read.
+        // - `buffer` is valid for write for `view.size()` bytes which is equal to `view.ptr.len()`.
+        unsafe { bindings::memcpy(buffer.cast(), view.ptr.cast(), view.ptr.len()) };
+    }
+
+    #[inline]
+    unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
+        // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile.
+        // SAFETY:
+        // - `view.ptr` is in CPU address space and valid for write.
+        // - `buffer` is valid for read for `view.size()` bytes which is equal to `view.ptr.len()`.
+        unsafe { bindings::memcpy(view.ptr.cast(), buffer.cast(), view.ptr.len()) };
+    }
+
+    #[inline]
+    fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T {
+        // SAFETY:
+        // - Per type invariant, `ptr` is valid and aligned.
+        // - Using read_volatile() here so that race with hardware is well-defined.
+        // - Using read_volatile() here is not sound if it races with other CPU per Rust
+        //   rules, but this is allowed per LKMM.
+        // - `T: FromBytes` so all bit patterns are valid.
+        unsafe { view.ptr.read_volatile() }
+    }
+
+    #[inline]
+    fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) {
+        // SAFETY:
+        // - Per type invariant, `ptr` is valid and aligned.
+        // - Using write_volatile() here so that race with hardware is well-defined.
+        // - Using write_volatile() here is not sound if it races with other CPU per Rust
+        //   rules, but this is allowed per LKMM.
+        unsafe { view.ptr.write_volatile(value) }
+    }
+}
+
+/// A view of a system memory region.
+///
+/// Provides `Io` trait implementation for kernel virtual address ranges,
+/// using volatile read/write to safely access shared memory that may be
+/// concurrently accessed by external hardware.
+///
+/// # Invariants
+///
+/// `self.ptr.addr() .. self.ptr.addr() + KnownSize::size(self.ptr)` is valid and aligned kernel
+/// accessible memory region for the lifetime `'a`.
+pub struct SysMem<'a, T: ?Sized> {
+    ptr: *mut T,
+    phantom: PhantomData<&'a ()>,
+}
+
+impl<T: ?Sized> Copy for SysMem<'_, T> {}
+impl<T: ?Sized> Clone for SysMem<'_, T> {
+    #[inline]
+    fn clone(&self) -> Self {
+        *self
+    }
+}
+
+// SAFETY: `SysMem<'_, T>` is conceptually `&T`.
+unsafe impl<T: ?Sized + Sync> Send for SysMem<'_, T> {}
+
+// SAFETY: `SysMem<'_, T>` is conceptually `&T`.
+unsafe impl<T: ?Sized + Sync> Sync for SysMem<'_, T> {}
+
+impl<'a, T: ?Sized> SysMem<'a, T> {
+    /// Create a `SysMem` from a raw pointer.
+    ///
+    /// # Safety
+    ///
+    /// `ptr.addr() .. ptr.addr() + KnownSize::size(ptr)` must be valid and aligned kernel
+    /// accessible memory region for the lifetime `'a`.
+    #[inline]
+    pub unsafe fn new(ptr: *mut T) -> Self {
+        // INVARIANT: Per safety requirement.
+        Self {
+            ptr,
+            phantom: PhantomData,
+        }
+    }
+
+    /// Obtain the raw pointer to the memory.
+    #[inline]
+    pub fn as_ptr(self) -> *mut T {
+        self.ptr
+    }
+}
+
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for SysMem<'a, T> {
+    type Backend = SysMemBackend;
+    type Target = T;
+
+    #[inline]
+    fn as_view(self) -> <Self::Backend as IoBackend>::View<'a, Self::Target> {
+        self
+    }
+}
+
+/// I/O Backend for [`IoSysMap`].
+pub struct IoSysMapBackend;
+
+/// Either [`Mmio`] or [`SysMem`].
+///
+/// This can be used when a piece of logic may wish to handle both MMIO or system memory but does
+/// not want or cannot be generic over I/O backends. This serves a similar purpose to
+/// [`include/linux/iosys-map.h`] in C.
+///
+/// This type can be used like any other types that implements [`Io`]; this also include
+/// [`io_project!`], [`io_read!`], [`io_write!`].
+///
+/// [`include/linux/iosys-map.h`]: srctree/include/linux/iosys-map.h
+pub enum IoSysMap<'a, T: ?Sized> {
+    /// The view is I/O memory.
+    Io(Mmio<'a, T>),
+    /// The view is system memory.
+    Sys(SysMem<'a, T>),
+}
+
+impl<T: ?Sized> Copy for IoSysMap<'_, T> {}
+impl<T: ?Sized> Clone for IoSysMap<'_, T> {
+    #[inline]
+    fn clone(&self) -> Self {
+        *self
+    }
+}
+
+impl<'a, T: ?Sized> From<Mmio<'a, T>> for IoSysMap<'a, T> {
+    #[inline]
+    fn from(value: Mmio<'a, T>) -> Self {
+        IoSysMap::Io(value)
+    }
+}
+
+impl<'a, T: ?Sized> From<SysMem<'a, T>> for IoSysMap<'a, T> {
+    #[inline]
+    fn from(value: SysMem<'a, T>) -> Self {
+        IoSysMap::Sys(value)
+    }
+}
+
+impl IoBackend for IoSysMapBackend {
+    type View<'a, T: ?Sized + KnownSize> = IoSysMap<'a, T>;
+
+    #[inline]
+    fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T {
+        match view {
+            IoSysMap::Io(l) => MmioBackend::as_ptr(l),
+            IoSysMap::Sys(r) => SysMemBackend::as_ptr(r),
+        }
+    }
+
+    #[inline]
+    unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
+        view: Self::View<'a, T>,
+        ptr: *mut U,
+    ) -> Self::View<'a, U> {
+        match view {
+            // SAFETY: Per safety requirement.
+            IoSysMap::Io(l) => IoSysMap::Io(unsafe { MmioBackend::project_view(l, ptr) }),
+            // SAFETY: Per safety requirement.
+            IoSysMap::Sys(r) => IoSysMap::Sys(unsafe { SysMemBackend::project_view(r, ptr) }),
+        }
+    }
+}
+
+impl<T> IoCapable<T> for IoSysMapBackend
+where
+    MmioBackend: IoCapable<T>,
+    SysMemBackend: IoCapable<T>,
+{
+    #[inline]
+    fn io_read(view: Self::View<'_, T>) -> T {
+        match view {
+            IoSysMap::Io(l) => MmioBackend::io_read(l),
+            IoSysMap::Sys(r) => SysMemBackend::io_read(r),
+        }
+    }
+
+    #[inline]
+    fn io_write<'a>(view: Self::View<'a, T>, value: T) {
+        match view {
+            IoSysMap::Io(l) => MmioBackend::io_write(l, value),
+            IoSysMap::Sys(r) => SysMemBackend::io_write(r, value),
+        }
+    }
+}
+
+impl IoCopyable for IoSysMapBackend {
+    #[inline]
+    unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) {
+        match view {
+            // SAFETY: Per safety requirement.
+            IoSysMap::Io(l) => unsafe { MmioBackend::copy_from_io(l, buffer) },
+            // SAFETY: Per safety requirement.
+            IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_from_io(r, buffer) },
+        }
+    }
+
+    #[inline]
+    unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) {
+        match view {
+            // SAFETY: Per safety requirement.
+            IoSysMap::Io(l) => unsafe { MmioBackend::copy_to_io(l, buffer) },
+            // SAFETY: Per safety requirement.
+            IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_to_io(r, buffer) },
+        }
+    }
+
+    #[inline]
+    fn copy_read<T: FromBytes>(view: Self::View<'_, T>) -> T {
+        match view {
+            IoSysMap::Io(l) => MmioBackend::copy_read(l),
+            IoSysMap::Sys(r) => SysMemBackend::copy_read(r),
+        }
+    }
+
+    #[inline]
+    fn copy_write<T: IntoBytes>(view: Self::View<'_, T>, value: T) {
+        match view {
+            IoSysMap::Io(l) => MmioBackend::copy_write(l, value),
+            IoSysMap::Sys(r) => SysMemBackend::copy_write(r, value),
+        }
+    }
+}
+
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for IoSysMap<'a, T> {
+    type Backend = IoSysMapBackend;
+    type Target = T;
+
+    #[inline]
+    fn as_view(self) -> IoSysMap<'a, T> {
+        self
+    }
+}
+
+// This helper turns associated functions to methods so it can be invoked in macro.
+// Used by `io_project!()` only.
+#[doc(hidden)]
+#[derive(Clone, Copy)]
+pub struct ProjectHelper<T>(pub T);
+
+impl<'a, T> ProjectHelper<T>
+where
+    T: Io<'a, Backend: IoBackend<View<'a, T::Target> = T>>,
+{
+    // These helper methods must not have symbols present in the binary to avoid confusion.
+    #[inline(always)]
+    pub fn as_ptr(self) -> *mut T::Target {
+        T::Backend::as_ptr(self.0)
+    }
+
+    /// # Safety
+    ///
+    /// Same as `IoBackend::project_view`
+    #[inline(always)]
+    pub unsafe fn project_view<U: ?Sized + KnownSize>(
+        self,
+        ptr: *mut U,
+    ) -> <T::Backend as IoBackend>::View<'a, U> {
+        // SAFETY: Per safety requirement.
+        unsafe { T::Backend::project_view::<T::Target, _>(self.0, ptr) }
+    }
+}
+
+/// Project an I/O type to a subview of it.
+///
+/// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that
+/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!).
+///
+/// # Examples
+///
+/// ```
+/// use kernel::io::{
+///     io_project,
+///     Mmio,
+/// };
+/// #[repr(C)]
+/// struct MyStruct { field: u32, }
+///
+/// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result {
+/// // let mmio: Mmio<[MyStruct]>;
+/// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field);
+/// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]);
+/// let nested: Mmio<'_, u32> = io_project!(whole, .field);
+/// # Ok::<(), Error>(()) }
+/// ```
+#[macro_export]
+#[doc(hidden)]
+macro_rules! io_project {
+    ($io:expr, $($proj:tt)*) => {{
+        #[allow(unused)]
+        use $crate::io::IoBase as _;
+        let view = $crate::io::ProjectHelper($io.as_view());
+        let ptr = $crate::ptr::project!(
+            mut view.as_ptr(), $($proj)*
+        );
+        #[allow(unused_unsafe)]
+        // SAFETY: `ptr` is a projection.
+        unsafe { view.project_view(ptr) }
+    }};
+}
+#[doc(inline)]
+pub use crate::io_project;
+
+/// Read from I/O memory.
+///
+/// The syntax is of form `io_read!(io, proj)` where `io` is an expression to a type that
+/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!).
+///
+/// # Examples
+///
+/// ```
+/// #[repr(C)]
+/// struct MyStruct { field: u32, }
+///
+/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result {
+/// // let mmio: Mmio<'_, [MyStruct]>;
+/// let field: u32 = kernel::io::io_read!(mmio, [try: 2].field);
+/// # Ok::<(), Error>(()) }
+/// ```
+#[macro_export]
+#[doc(hidden)]
+macro_rules! io_read {
+    ($io:expr, $($proj:tt)*) => {
+        $crate::io::Io::read_val($crate::io_project!($io, $($proj)*))
+    };
+}
+#[doc(inline)]
+pub use crate::io_read;
+
+/// Writes to I/O memory.
+///
+/// The syntax is of form `io_write!(io, proj, val)` where `io` is an expression to a type that
+/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!),
+/// and `val` is the value to be written to the projected location.
+///
+/// # Examples
+///
+/// ```
+/// #[repr(C)]
+/// struct MyStruct { field: u32, }
+///
+/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result {
+/// // let mmio: Mmio<'_, [MyStruct]>;
+/// kernel::io::io_write!(mmio, [try: 2].field, 10);
+/// # Ok::<(), Error>(()) }
+/// ```
+#[macro_export]
+#[doc(hidden)]
+macro_rules! io_write {
+    (@parse [$io:expr] [$($proj:tt)*] [, $val:expr]) => {
+        $crate::io::Io::write_val($crate::io_project!($io, $($proj)*), $val)
+    };
+    (@parse [$io:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => {
+        $crate::io_write!(@parse [$io] [$($proj)* .$field] [$($rest)*])
+    };
+    (@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => {
+        $crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*])
+    };
+    ($io:expr, $($rest:tt)*) => {
+        $crate::io_write!(@parse [$io] [] [$($rest)*])
+    };
+}
+#[doc(inline)]
+pub use crate::io_write;
diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs
index fc2a3e24f8d5d..32a919099dcda 100644
--- a/rust/kernel/io/mem.rs
+++ b/rust/kernel/io/mem.rs
@@ -2,24 +2,28 @@
 
 //! Generic memory-mapped IO.
 
-use core::ops::Deref;
-
 use crate::{
     device::{
         Bound,
         Device, //
     },
-    devres::Devres,
+    devres::DevresLt,
     io::{
         self,
         resource::{
             Region,
             Resource, //
         },
+        IoBase,
         Mmio,
+        MmioBackend,
         MmioRaw, //
     },
     prelude::*,
+    types::{
+        CovariantForLt,
+        ForLt, //
+    },
 };
 
 /// An IO request for a specific device and resource.
@@ -172,6 +176,19 @@ pub struct ExclusiveIoMem<'a, const SIZE: usize> {
     _region: Region,
 }
 
+impl<const SIZE: usize> ForLt for ExclusiveIoMem<'static, SIZE> {
+    type Of<'a> = ExclusiveIoMem<'a, SIZE>;
+}
+
+// SAFETY: `ExclusiveIoMem<'a, SIZE>` is covariant over `'a`; it holds an `IoMem<'a, SIZE>`,
+// which holds `&'a Device<Bound>`, which is covariant.
+unsafe impl<const SIZE: usize> CovariantForLt for ExclusiveIoMem<'static, SIZE> {}
+
+/// A device-managed exclusive I/O memory region.
+///
+/// See [`ExclusiveIoMem::into_devres`].
+pub type DevresExclusiveIoMem<const SIZE: usize> = DevresLt<ExclusiveIoMem<'static, SIZE>>;
+
 impl<'a, const SIZE: usize> ExclusiveIoMem<'a, SIZE> {
     /// Creates a new `ExclusiveIoMem` instance.
     fn ioremap(dev: &'a Device<Bound>, resource: &Resource) -> Result<Self> {
@@ -198,23 +215,23 @@ fn ioremap(dev: &'a Device<Bound>, resource: &Resource) -> Result<Self> {
 
     /// Consume the `ExclusiveIoMem` and register it as a device-managed resource.
     ///
-    /// The returned `Devres<ExclusiveIoMem<'static, SIZE>>` can outlive the original lifetime
-    /// `'a`. Access to the I/O memory is revoked when the device is unbound.
-    pub fn into_devres(self) -> Result<Devres<ExclusiveIoMem<'static, SIZE>>> {
-        // SAFETY: Casting to `'static` is sound because `Devres` guarantees the
-        // `ExclusiveIoMem` does not actually outlive the device -- access is revoked and the
-        // resource is released when the device is unbound.
-        let iomem: ExclusiveIoMem<'static, SIZE> = unsafe { core::mem::transmute(self) };
-        let dev = iomem.iomem.dev;
-        Devres::new(dev, iomem)
+    /// The returned [`DevresExclusiveIoMem`] can outlive the original borrow and be stored in
+    /// driver data. Access to the I/O memory is revoked automatically when the device is unbound.
+    pub fn into_devres(self) -> Result<DevresExclusiveIoMem<SIZE>> {
+        let dev = self.iomem.dev;
+        // SAFETY: `ExclusiveIoMem` only holds a device reference and an I/O mapping, both of
+        // which remain valid for the device's full bound scope, not just for `'a`.
+        unsafe { DevresLt::new(dev, self) }
     }
 }
 
-impl<const SIZE: usize> Deref for ExclusiveIoMem<'_, SIZE> {
-    type Target = Mmio<SIZE>;
+impl<'a, const SIZE: usize> IoBase<'a> for &'a ExclusiveIoMem<'_, SIZE> {
+    type Backend = MmioBackend;
+    type Target = super::Region<SIZE>;
 
-    fn deref(&self) -> &Self::Target {
-        &self.iomem
+    #[inline]
+    fn as_view(self) -> Mmio<'a, Self::Target> {
+        self.iomem.as_view()
     }
 }
 
@@ -229,9 +246,22 @@ fn deref(&self) -> &Self::Target {
 /// start of the I/O memory mapped region.
 pub struct IoMem<'a, const SIZE: usize = 0> {
     dev: &'a Device<Bound>,
-    io: MmioRaw<SIZE>,
+    io: MmioRaw<super::Region<SIZE>>,
 }
 
+impl<const SIZE: usize> ForLt for IoMem<'static, SIZE> {
+    type Of<'a> = IoMem<'a, SIZE>;
+}
+
+// SAFETY: `IoMem<'a, SIZE>` is covariant over `'a`; it holds `&'a Device<Bound>`,
+// which is covariant.
+unsafe impl<const SIZE: usize> CovariantForLt for IoMem<'static, SIZE> {}
+
+/// A device-managed I/O memory region.
+///
+/// See [`IoMem::into_devres`].
+pub type DevresIoMem<const SIZE: usize = 0> = DevresLt<IoMem<'static, SIZE>>;
+
 impl<'a, const SIZE: usize> IoMem<'a, SIZE> {
     fn ioremap(dev: &'a Device<Bound>, resource: &Resource) -> Result<Self> {
         // Note: Some ioremap() implementations use types that depend on the CPU
@@ -264,23 +294,19 @@ fn ioremap(dev: &'a Device<Bound>, resource: &Resource) -> Result<Self> {
             return Err(ENOMEM);
         }
 
-        let io = MmioRaw::new(addr as usize, size)?;
-
+        let io = MmioRaw::new_region(addr as usize, size)?;
         Ok(IoMem { dev, io })
     }
 
     /// Consume the `IoMem` and register it as a device-managed resource.
     ///
-    /// The returned `Devres<IoMem<'static, SIZE>>` can outlive the original
-    /// lifetime `'a`. Access to the I/O memory is revoked when the device
-    /// is unbound.
-    pub fn into_devres(self) -> Result<Devres<IoMem<'static, SIZE>>> {
-        // SAFETY: Casting to `'static` is sound because `Devres` guarantees the `IoMem` does not
-        // actually outlive the device -- access is revoked and the resource is released when the
-        // device is unbound.
-        let iomem: IoMem<'static, SIZE> = unsafe { core::mem::transmute(self) };
-        let dev = iomem.dev;
-        Devres::new(dev, iomem)
+    /// The returned [`DevresIoMem`] can outlive the original borrow and be stored in driver data.
+    /// Access to the I/O memory is revoked automatically when the device is unbound.
+    pub fn into_devres(self) -> Result<DevresIoMem<SIZE>> {
+        let dev = self.dev;
+        // SAFETY: `IoMem` only holds a device reference and an I/O mapping, both of which
+        // remain valid for the device's full bound scope, not just for `'a`.
+        unsafe { DevresLt::new(dev, self) }
     }
 }
 
@@ -291,11 +317,13 @@ fn drop(&mut self) {
     }
 }
 
-impl<const SIZE: usize> Deref for IoMem<'_, SIZE> {
-    type Target = Mmio<SIZE>;
+impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem<'_, SIZE> {
+    type Backend = MmioBackend;
+    type Target = super::Region<SIZE>;
 
-    fn deref(&self) -> &Self::Target {
+    #[inline]
+    fn as_view(self) -> Mmio<'a, Self::Target> {
         // SAFETY: Safe as by the invariant of `IoMem`.
-        unsafe { Mmio::from_raw(&self.io) }
+        unsafe { Mmio::from_raw(self.io) }
     }
 }
diff --git a/rust/kernel/io/poll.rs b/rust/kernel/io/poll.rs
index 75d1b3e8596c6..d75f2fcf46f2c 100644
--- a/rust/kernel/io/poll.rs
+++ b/rust/kernel/io/poll.rs
@@ -48,13 +48,14 @@
 /// use kernel::io::{
 ///     Io,
 ///     Mmio,
+///     Region,
 ///     poll::read_poll_timeout, //
 /// };
 /// use kernel::time::Delta;
 ///
 /// const HW_READY: u16 = 0x01;
 ///
-/// fn wait_for_hardware<const SIZE: usize>(io: &Mmio<SIZE>) -> Result {
+/// fn wait_for_hardware<const SIZE: usize>(io: Mmio<'_, Region<SIZE>>) -> Result {
 ///     read_poll_timeout(
 ///         // The `op` closure reads the value of a specific status register.
 ///         || io.try_read16(0x1000),
@@ -135,13 +136,14 @@ pub fn read_poll_timeout<Op, Cond, T>(
 /// use kernel::io::{
 ///     Io,
 ///     Mmio,
+///     Region,
 ///     poll::read_poll_timeout_atomic, //
 /// };
 /// use kernel::time::Delta;
 ///
 /// const HW_READY: u16 = 0x01;
 ///
-/// fn wait_for_hardware<const SIZE: usize>(io: &Mmio<SIZE>) -> Result {
+/// fn wait_for_hardware<const SIZE: usize>(io: Mmio<'_, Region<SIZE>>) -> Result {
 ///     read_poll_timeout_atomic(
 ///         // The `op` closure reads the value of a specific status register.
 ///         || io.try_read16(0x1000),
diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs
index f924c7c7c1dbb..6cb07fc92cc36 100644
--- a/rust/kernel/io/register.rs
+++ b/rust/kernel/io/register.rs
@@ -58,7 +58,7 @@
 //!     },
 //!     num::Bounded,
 //! };
-//! # use kernel::io::Mmio;
+//! # use kernel::io::{Mmio, Region};
 //! # register! {
 //! #     pub BOOT_0(u32) @ 0x00000100 {
 //! #         15:8 vendor_id;
@@ -66,7 +66,7 @@
 //! #         3:0 minor_revision;
 //! #     }
 //! # }
-//! # fn test(io: &Mmio<0x1000>) {
+//! # fn test(io: Mmio<'_, Region<0x1000>>) {
 //! # fn obtain_vendor_id() -> u8 { 0xff }
 //!
 //! // Read from the register's defined offset (0x100).
@@ -113,6 +113,8 @@
     io::IoLoc, //
 };
 
+use super::Region;
+
 /// Trait implemented by all registers.
 pub trait Register: Sized {
     /// Backing primitive type of the register.
@@ -129,7 +131,7 @@ pub trait FixedRegister: Register {}
 
 /// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when
 /// passing a [`FixedRegister`] value.
-impl<T> IoLoc<T> for ()
+impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for ()
 where
     T: FixedRegister,
 {
@@ -143,7 +145,7 @@ fn offset(self) -> usize {
 
 /// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used
 /// as an [`IoLoc`].
-impl<T> IoLoc<T> for T
+impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for T
 where
     T: FixedRegister,
 {
@@ -168,7 +170,7 @@ pub const fn new() -> Self {
     }
 }
 
-impl<T> IoLoc<T> for FixedRegisterLoc<T>
+impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for FixedRegisterLoc<T>
 where
     T: FixedRegister,
 {
@@ -239,7 +241,7 @@ const fn offset(self) -> usize {
     }
 }
 
-impl<T, B> IoLoc<T> for RelativeRegisterLoc<T, B>
+impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterLoc<T, B>
 where
     T: RelativeRegister,
     B: RegisterBase<T::BaseFamily> + ?Sized,
@@ -283,7 +285,7 @@ pub fn try_new(idx: usize) -> Option<Self> {
     }
 }
 
-impl<T> IoLoc<T> for RegisterArrayLoc<T>
+impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for RegisterArrayLoc<T>
 where
     T: RegisterArray,
 {
@@ -370,7 +372,7 @@ pub fn try_at(self, idx: usize) -> Option<RelativeRegisterArrayLoc<T, B>> {
     }
 }
 
-impl<T, B> IoLoc<T> for RelativeRegisterArrayLoc<T, B>
+impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterArrayLoc<T, B>
 where
     T: RelativeRegisterArray,
     B: RegisterBase<T::BaseFamily> + ?Sized,
@@ -387,18 +389,18 @@ fn offset(self) -> usize {
 /// which to write it.
 ///
 /// Implementors can be used with [`Io::write_reg`](super::Io::write_reg).
-pub trait LocatedRegister {
+pub trait LocatedRegister<Base: ?Sized> {
     /// Register value to write.
     type Value: Register;
     /// Full location information at which to write the value.
-    type Location: IoLoc<Self::Value>;
+    type Location: IoLoc<Base, Self::Value>;
 
     /// Consumes `self` and returns a `(location, value)` tuple describing a valid I/O write
     /// operation.
     fn into_io_op(self) -> (Self::Location, Self::Value);
 }
 
-impl<T> LocatedRegister for T
+impl<const SIZE: usize, T> LocatedRegister<Region<SIZE>> for T
 where
     T: FixedRegister,
 {
@@ -444,16 +446,16 @@ fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
 ///         Io,
 ///     },
 /// };
-/// # use kernel::io::Mmio;
+/// # use kernel::io::{Mmio, Region};
 ///
 /// register! {
 ///     FIXED_REG(u32) @ 0x100 {
-///         16:8 high_byte;
+///         15:8 high_byte;
 ///         7:0  low_byte;
 ///     }
 /// }
 ///
-/// # fn test(io: &Mmio<0x1000>) {
+/// # fn test(io: Mmio<'_, Region<0x1000>>) {
 /// let val = io.read(FIXED_REG);
 ///
 /// // Write from an already-existing value.
@@ -557,7 +559,7 @@ fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
 ///         Io,
 ///     },
 /// };
-/// # use kernel::io::Mmio;
+/// # use kernel::io::{Mmio, Region};
 ///
 /// // Type used to identify the base.
 /// pub struct CpuCtlBase;
@@ -582,7 +584,7 @@ fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
 ///     }
 /// }
 ///
-/// # fn test(io: Mmio<0x1000>) {
+/// # fn test(io: Mmio<'_, Region<0x1000>>) {
 /// // Read the status of `Cpu0`.
 /// let cpu0_started = io.read(CPU_CTL::of::<Cpu0>());
 ///
@@ -599,7 +601,7 @@ fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
 ///     }
 /// }
 ///
-/// # fn test2(io: Mmio<0x1000>) {
+/// # fn test2(io: Mmio<'_, Region<0x1000>>) {
 /// // Start the aliased `CPU0`, leaving its other fields untouched.
 /// io.update(CPU_CTL_ALIAS::of::<Cpu0>(), |r| r.with_alias_start(true));
 /// # }
@@ -636,7 +638,7 @@ fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
 ///         Io,
 ///     },
 /// };
-/// # use kernel::io::Mmio;
+/// # use kernel::io::{Mmio, Region};
 /// # fn get_scratch_idx() -> usize {
 /// #   0x15
 /// # }
@@ -649,7 +651,7 @@ fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
 ///     }
 /// }
 ///
-/// # fn test(io: &Mmio<0x1000>)
+/// # fn test(io: Mmio<'_, Region<0x1000>>)
 /// #     -> Result<(), Error>{
 /// // Read scratch register 0, i.e. I/O address `0x80`.
 /// let scratch_0 = io.read(SCRATCH::at(0)).value();
@@ -722,7 +724,7 @@ fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
 ///         Io,
 ///     },
 /// };
-/// # use kernel::io::Mmio;
+/// # use kernel::io::{Mmio, Region};
 /// # fn get_scratch_idx() -> usize {
 /// #   0x15
 /// # }
@@ -750,7 +752,7 @@ fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
 ///     }
 /// }
 ///
-/// # fn test(io: &Mmio<0x1000>) -> Result<(), Error> {
+/// # fn test(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> {
 /// // Read scratch register 0 of CPU0.
 /// let scratch = io.read(CPU_SCRATCH::of::<Cpu0>().at(0));
 ///
@@ -792,7 +794,7 @@ fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
 ///     }
 /// }
 ///
-/// # fn test2(io: &Mmio<0x1000>) -> Result<(), Error> {
+/// # fn test2(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> {
 /// let cpu0_status = io.read(CPU_FIRMWARE_STATUS::of::<Cpu0>()).status();
 /// # Ok(())
 /// # }
diff --git a/rust/kernel/iommu/pgtable.rs b/rust/kernel/iommu/pgtable.rs
index c88e38fd938ae..eddda8d9d62a0 100644
--- a/rust/kernel/iommu/pgtable.rs
+++ b/rust/kernel/iommu/pgtable.rs
@@ -102,8 +102,7 @@ pub unsafe fn new_raw(dev: &Device<Bound>, config: Config) -> Result<IoPageTable
             coherent_walk: config.coherent_walk,
             tlb: &raw const NOOP_FLUSH_OPS,
             iommu_dev: dev.as_raw(),
-            // SAFETY: All zeroes is a valid value for `struct io_pgtable_cfg`.
-            ..unsafe { core::mem::zeroed() }
+            ..Zeroable::zeroed()
         };
 
         // SAFETY:
diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs
index cdee5f27bd7fa..91eaff8c186af 100644
--- a/rust/kernel/kunit.rs
+++ b/rust/kernel/kunit.rs
@@ -288,6 +288,7 @@ macro_rules! kunit_unsafe_test_suite {
                     log: ::core::ptr::null_mut(),
                     suite_init_err: 0,
                     is_init: false,
+                    status: kernel::bindings::kunit_status_KUNIT_SUCCESS,
                 };
 
             #[used(compiler)]
diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
index 9512af7156df2..68f4d9a3425d6 100644
--- a/rust/kernel/lib.rs
+++ b/rust/kernel/lib.rs
@@ -16,6 +16,9 @@
 // Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on
 // the unstable features in use.
 //
+// Stable since Rust 1.87.0.
+#![feature(unsigned_is_multiple_of)]
+//
 // Stable since Rust 1.89.0.
 #![feature(generic_arg_infer)]
 //
diff --git a/rust/kernel/module_param.rs b/rust/kernel/module_param.rs
index 6541af218390c..f9a14765a9265 100644
--- a/rust/kernel/module_param.rs
+++ b/rust/kernel/module_param.rs
@@ -5,7 +5,7 @@
 //! C header: [`include/linux/moduleparam.h`](srctree/include/linux/moduleparam.h)
 
 use crate::prelude::*;
-use crate::str::BStr;
+use crate::str::{kstrtobool_bytes, BStr};
 use bindings;
 use kernel::sync::SetOnce;
 
@@ -105,6 +105,12 @@ fn try_from_param_arg(arg: &BStr) -> Result<Self> {
 impl_int_module_param!(isize);
 impl_int_module_param!(usize);
 
+impl ModuleParam for bool {
+    fn try_from_param_arg(arg: &BStr) -> Result<Self> {
+        kstrtobool_bytes(arg)
+    }
+}
+
 /// A wrapper for kernel parameters.
 ///
 /// This type is instantiated by the [`module!`] macro when module parameters are
@@ -130,10 +136,26 @@ pub const fn new(default: T) -> Self {
         }
     }
 
+    /// Get a copy of the parameter value.
+    ///
+    /// Returns the value supplied at module load time, or the default value
+    /// if the parameter has not been set.
+    #[inline]
+    pub fn value(&self) -> T
+    where
+        T: Copy,
+    {
+        self.value.copy().unwrap_or(self.default)
+    }
+
     /// Get a shared reference to the parameter value.
+    ///
+    /// Returns a reference to the value supplied at module load time, or a
+    /// reference to the default value if the parameter has not been set.
     // Note: When sysfs access to parameters are enabled, we have to pass in a
     // held lock guard here.
-    pub fn value(&self) -> &T {
+    #[inline]
+    pub fn value_ref(&self) -> &T {
         self.value.as_ref().unwrap_or(&self.default)
     }
 
@@ -179,3 +201,4 @@ macro_rules! make_param_ops {
 make_param_ops!(PARAM_OPS_U64, u64);
 make_param_ops!(PARAM_OPS_ISIZE, isize);
 make_param_ops!(PARAM_OPS_USIZE, usize);
+make_param_ops!(PARAM_OPS_BOOL, bool);
diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs
index 5071cae6543fd..ee25ae56bb92f 100644
--- a/rust/kernel/pci.rs
+++ b/rust/kernel/pci.rs
@@ -43,8 +43,8 @@
 pub use self::io::{
     Bar,
     ConfigSpace,
-    ConfigSpaceKind,
     ConfigSpaceSize,
+    DevresBar,
     Extended,
     Normal, //
 };
diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs
index 0461e01aaa20e..953e16735c6ed 100644
--- a/rust/kernel/pci/io.rs
+++ b/rust/kernel/pci/io.rs
@@ -6,19 +6,22 @@
 use crate::{
     bindings,
     device,
-    devres::Devres,
+    devres::DevresLt,
     io::{
-        Io,
+        IoBackend,
+        IoBase,
         IoCapable,
-        IoKnownSize,
         Mmio,
-        MmioRaw, //
+        MmioBackend,
+        MmioRaw,
+        Region, //
     },
-    prelude::*, //
-};
-use core::{
-    marker::PhantomData,
-    ops::Deref, //
+    prelude::*,
+    ptr::KnownSize,
+    types::{
+        CovariantForLt,
+        ForLt, //
+    }, //
 };
 
 /// Represents the size of a PCI configuration space.
@@ -46,68 +49,95 @@ pub const fn into_raw(self) -> usize {
     }
 }
 
-/// Marker type for normal (256-byte) PCI configuration space.
-pub struct Normal;
+/// Alias for normal (256-byte) PCI configuration space.
+pub type Normal = Region<256>;
 
-/// Marker type for extended (4096-byte) PCIe configuration space.
-pub struct Extended;
+/// Alias for extended (4096-byte) PCIe configuration space.
+pub type Extended = Region<4096>;
 
-/// Trait for PCI configuration space size markers.
+/// A view of PCI configuration space of a device.
+///
+/// Provides typed read and write accessors for configuration registers
+/// using the standard `pci_read_config_*` and `pci_write_config_*` helpers.
+///
+/// The generic parameter `T` is the type of the view. The full configuration space is also a
+/// special type of view; in such cases, `T` can be [`Normal`] for 256-byte legacy configuration
+/// space or [`Extended`] for 4096-byte PCIe extended configuration space (default).
 ///
-/// This trait is implemented by [`Normal`] and [`Extended`] to provide
-/// compile-time knowledge of the configuration space size.
-pub trait ConfigSpaceKind {
-    /// The size of this configuration space in bytes.
-    const SIZE: usize;
+/// # Invariants
+///
+/// `ptr` is aligned and range `ptr..ptr + KnownSize::size(ptr)` is within
+/// `0..pdev.cfg_size().into_raw()`.
+pub struct ConfigSpace<'a, T: ?Sized = Extended> {
+    pub(crate) pdev: &'a Device<device::Bound>,
+    ptr: *mut T,
 }
 
-impl ConfigSpaceKind for Normal {
-    const SIZE: usize = 256;
+impl<T: ?Sized> Copy for ConfigSpace<'_, T> {}
+impl<T: ?Sized> Clone for ConfigSpace<'_, T> {
+    #[inline]
+    fn clone(&self) -> Self {
+        *self
+    }
 }
 
-impl ConfigSpaceKind for Extended {
-    const SIZE: usize = 4096;
-}
+// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory.
+unsafe impl<T: ?Sized + Sync> Send for ConfigSpace<'_, T> {}
 
-/// The PCI configuration space of a device.
-///
-/// Provides typed read and write accessors for configuration registers
-/// using the standard `pci_read_config_*` and `pci_write_config_*` helpers.
-///
-/// The generic parameter `S` indicates the maximum size of the configuration space.
-/// Use [`Normal`] for 256-byte legacy configuration space or [`Extended`] for
-/// 4096-byte PCIe extended configuration space (default).
-pub struct ConfigSpace<'a, S: ConfigSpaceKind = Extended> {
-    pub(crate) pdev: &'a Device<device::Bound>,
-    _marker: PhantomData<S>,
+// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory.
+unsafe impl<T: ?Sized + Sync> Sync for ConfigSpace<'_, T> {}
+
+/// I/O Backend for PCI configuration space.
+pub struct ConfigSpaceBackend;
+
+impl IoBackend for ConfigSpaceBackend {
+    type View<'a, T: ?Sized + KnownSize> = ConfigSpace<'a, T>;
+
+    #[inline]
+    fn as_ptr<'a, T: ?Sized + KnownSize>(view: ConfigSpace<'a, T>) -> *mut T {
+        view.ptr
+    }
+
+    #[inline]
+    unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>(
+        view: Self::View<'a, T>,
+        ptr: *mut U,
+    ) -> Self::View<'a, U> {
+        // INVARIANT: Per safety requirement.
+        ConfigSpace {
+            pdev: view.pdev,
+            ptr,
+        }
+    }
 }
 
 /// Implements [`IoCapable`] on [`ConfigSpace`] for `$ty` using `$read_fn` and `$write_fn`.
 macro_rules! impl_config_space_io_capable {
     ($ty:ty, $read_fn:ident, $write_fn:ident) => {
-        impl<'a, S: ConfigSpaceKind> IoCapable<$ty> for ConfigSpace<'a, S> {
-            unsafe fn io_read(&self, address: usize) -> $ty {
+        impl IoCapable<$ty> for ConfigSpaceBackend {
+            fn io_read(view: ConfigSpace<'_, $ty>) -> $ty {
+                // CAST: The offset is cast to `i32` because the C functions expect a 32-bit
+                // signed offset parameter. PCI configuration space size is at most 4096 bytes,
+                // so the value always fits within `i32` without truncation or sign change.
+                let addr = view.ptr.addr() as i32;
+
                 let mut val: $ty = 0;
 
                 // Return value from C function is ignored in infallible accessors.
-                let _ret =
-                    // SAFETY: By the type invariant `self.pdev` is a valid address.
-                    // CAST: The offset is cast to `i32` because the C functions expect a 32-bit
-                    // signed offset parameter. PCI configuration space size is at most 4096 bytes,
-                    // so the value always fits within `i32` without truncation or sign change.
-                    unsafe { bindings::$read_fn(self.pdev.as_raw(), address as i32, &mut val) };
-
+                // SAFETY: By the type invariant `pdev` is a valid address.
+                let _ = unsafe { bindings::$read_fn(view.pdev.as_raw(), addr, &mut val) };
                 val
             }
 
-            unsafe fn io_write(&self, value: $ty, address: usize) {
+            fn io_write(view: ConfigSpace<'_, $ty>, value: $ty) {
+                // CAST: The offset is cast to `i32` because the C functions expect a 32-bit
+                // signed offset parameter. PCI configuration space size is at most 4096 bytes,
+                // so the value always fits within `i32` without truncation or sign change.
+                let addr = view.ptr.addr() as i32;
+
                 // Return value from C function is ignored in infallible accessors.
-                let _ret =
-                    // SAFETY: By the type invariant `self.pdev` is a valid address.
-                    // CAST: The offset is cast to `i32` because the C functions expect a 32-bit
-                    // signed offset parameter. PCI configuration space size is at most 4096 bytes,
-                    // so the value always fits within `i32` without truncation or sign change.
-                    unsafe { bindings::$write_fn(self.pdev.as_raw(), address as i32, value) };
+                // SAFETY: By the type invariant `pdev` is a valid address.
+                let _ = unsafe { bindings::$write_fn(view.pdev.as_raw(), addr, value) };
             }
         }
     };
@@ -118,24 +148,16 @@ unsafe fn io_write(&self, value: $ty, address: usize) {
 impl_config_space_io_capable!(u16, pci_read_config_word, pci_write_config_word);
 impl_config_space_io_capable!(u32, pci_read_config_dword, pci_write_config_dword);
 
-impl<'a, S: ConfigSpaceKind> Io for ConfigSpace<'a, S> {
-    /// Returns the base address of the I/O region. It is always 0 for configuration space.
-    #[inline]
-    fn addr(&self) -> usize {
-        0
-    }
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for ConfigSpace<'a, T> {
+    type Backend = ConfigSpaceBackend;
+    type Target = T;
 
-    /// Returns the maximum size of the configuration space.
     #[inline]
-    fn maxsize(&self) -> usize {
-        self.pdev.cfg_size().into_raw()
+    fn as_view(self) -> ConfigSpace<'a, T> {
+        self
     }
 }
 
-impl<'a, S: ConfigSpaceKind> IoKnownSize for ConfigSpace<'a, S> {
-    const MIN_SIZE: usize = S::SIZE;
-}
-
 /// A PCI BAR to perform I/O-Operations on.
 ///
 /// I/O backend assumes that the device is little-endian and will automatically
@@ -147,10 +169,23 @@ impl<'a, S: ConfigSpaceKind> IoKnownSize for ConfigSpace<'a, S> {
 /// memory mapped PCI BAR and its size.
 pub struct Bar<'a, const SIZE: usize = 0> {
     pdev: &'a Device<device::Bound>,
-    io: MmioRaw<SIZE>,
+    io: MmioRaw<crate::io::Region<SIZE>>,
     num: i32,
 }
 
+impl<const SIZE: usize> ForLt for Bar<'static, SIZE> {
+    type Of<'a> = Bar<'a, SIZE>;
+}
+
+// SAFETY: `Bar<'a, SIZE>` is covariant over `'a`; it holds `&'a Device<Bound>`,
+// which is covariant.
+unsafe impl<const SIZE: usize> CovariantForLt for Bar<'static, SIZE> {}
+
+/// A device-managed PCI BAR mapping.
+///
+/// See [`Bar::into_devres`].
+pub type DevresBar<const SIZE: usize = 0> = DevresLt<Bar<'static, SIZE>>;
+
 impl<'a, const SIZE: usize> Bar<'a, SIZE> {
     pub(super) fn new(
         pdev: &'a Device<device::Bound>,
@@ -187,7 +222,7 @@ pub(super) fn new(
             return Err(ENOMEM);
         }
 
-        let io = match MmioRaw::new(ioptr, len as usize) {
+        let io = match MmioRaw::new_region(ioptr, len as usize) {
             Ok(io) => io,
             Err(err) => {
                 // SAFETY:
@@ -223,15 +258,13 @@ fn release(&self) {
 
     /// Consume the `Bar` and register it as a device-managed resource.
     ///
-    /// The returned `Devres<Bar<'static, SIZE>>` can outlive the original lifetime `'a`. Access
-    /// to the BAR is revoked when the device is unbound.
-    pub fn into_devres(self) -> Result<Devres<Bar<'static, SIZE>>> {
-        // SAFETY: Casting to `'static` is sound because `Devres` guarantees the `Bar` does not
-        // actually outlive the device -- access is revoked and the resource is released when the
-        // device is unbound.
-        let bar: Bar<'static, SIZE> = unsafe { core::mem::transmute(self) };
-        let pdev = bar.pdev;
-        Devres::new(pdev.as_ref(), bar)
+    /// The returned [`DevresBar`] can outlive the original borrow and be stored in driver data.
+    /// Access to the BAR is revoked automatically when the device is unbound.
+    pub fn into_devres(self) -> Result<DevresBar<SIZE>> {
+        let pdev = self.pdev;
+        // SAFETY: `Bar` only holds a reference to the device and an I/O mapping, both of which
+        // remain valid for the device's full bound scope, not just for `'a`.
+        unsafe { DevresLt::new(pdev.as_ref(), self) }
     }
 }
 
@@ -249,12 +282,14 @@ fn drop(&mut self) {
     }
 }
 
-impl<const SIZE: usize> Deref for Bar<'_, SIZE> {
-    type Target = Mmio<SIZE>;
+impl<'a, const SIZE: usize> IoBase<'a> for &'a Bar<'_, SIZE> {
+    type Backend = MmioBackend;
+    type Target = crate::io::Region<SIZE>;
 
-    fn deref(&self) -> &Self::Target {
+    #[inline]
+    fn as_view(self) -> Mmio<'a, Self::Target> {
         // SAFETY: By the type invariant of `Self`, the MMIO range in `self.io` is properly mapped.
-        unsafe { Mmio::from_raw(&self.io) }
+        unsafe { Mmio::from_raw(self.io) }
     }
 }
 
@@ -289,23 +324,25 @@ pub fn cfg_size(&self) -> ConfigSpaceSize {
         }
     }
 
-    /// Return an initialized normal (256-byte) config space object.
+    /// Return a view of the normal (256-byte) config space.
     pub fn config_space<'a>(&'a self) -> ConfigSpace<'a, Normal> {
+        // INVARIANT: null is aligned and the range is within config space.
         ConfigSpace {
             pdev: self,
-            _marker: PhantomData,
+            ptr: Normal::ptr_from_raw_parts_mut(core::ptr::null_mut(), self.cfg_size().into_raw()),
         }
     }
 
-    /// Return an initialized extended (4096-byte) config space object.
+    /// Return a view of the extended (4096-byte) config space.
     pub fn config_space_extended<'a>(&'a self) -> Result<ConfigSpace<'a, Extended>> {
         if self.cfg_size() != ConfigSpaceSize::Extended {
             return Err(EINVAL);
         }
 
+        // INVARIANT: null is aligned and we just checked the `cfg_size`.
         Ok(ConfigSpace {
             pdev: self,
-            _marker: PhantomData,
+            ptr: Extended::ptr_from_raw_parts_mut(core::ptr::null_mut(), 4096),
         })
     }
 }
diff --git a/rust/kernel/ptr.rs b/rust/kernel/ptr.rs
index 3f3e529e9f582..82acb531b17bd 100644
--- a/rust/kernel/ptr.rs
+++ b/rust/kernel/ptr.rs
@@ -235,11 +235,20 @@ fn align_up(self, alignment: Alignment) -> Option<Self> {
 ///
 /// This is a generalization of [`size_of`] that works for dynamically sized types.
 pub trait KnownSize {
+    /// Minimum size of this type known at compile-time.
+    const MIN_SIZE: usize;
+
+    /// Minimum alignment of this type known at compile-time.
+    const MIN_ALIGN: Alignment;
+
     /// Get the size of an object of this type in bytes, with the metadata of the given pointer.
     fn size(p: *const Self) -> usize;
 }
 
 impl<T> KnownSize for T {
+    const MIN_SIZE: usize = size_of::<T>();
+    const MIN_ALIGN: Alignment = Alignment::of::<T>();
+
     #[inline(always)]
     fn size(_: *const Self) -> usize {
         size_of::<T>()
@@ -247,6 +256,9 @@ fn size(_: *const Self) -> usize {
 }
 
 impl<T> KnownSize for [T] {
+    const MIN_SIZE: usize = 0;
+    const MIN_ALIGN: Alignment = Alignment::of::<T>();
+
     #[inline(always)]
     fn size(p: *const Self) -> usize {
         p.len() * size_of::<T>()
diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs
index 38273f4eedb51..c2b3457b700c1 100644
--- a/rust/kernel/task.rs
+++ b/rust/kernel/task.rs
@@ -210,20 +210,13 @@ pub fn pid(&self) -> Pid {
         unsafe { *ptr::addr_of!((*self.as_ptr()).pid) }
     }
 
-    /// Returns the UID of the given task.
+    /// Returns the objective real UID of the given task.
     #[inline]
     pub fn uid(&self) -> Kuid {
         // SAFETY: It's always safe to call `task_uid` on a valid task.
         Kuid::from_raw(unsafe { bindings::task_uid(self.as_ptr()) })
     }
 
-    /// Returns the effective UID of the given task.
-    #[inline]
-    pub fn euid(&self) -> Kuid {
-        // SAFETY: It's always safe to call `task_euid` on a valid task.
-        Kuid::from_raw(unsafe { bindings::task_euid(self.as_ptr()) })
-    }
-
     /// Determines whether the given task has pending signals.
     #[inline]
     pub fn signal_pending(&self) -> bool {
@@ -371,7 +364,7 @@ fn eq(&self, other: &Self) -> bool {
 impl Eq for Task {}
 
 impl Kuid {
-    /// Get the current euid.
+    /// Get the current subjective effective UID.
     #[inline]
     pub fn current_euid() -> Kuid {
         // SAFETY: Just an FFI call.
diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs
index ac316fd7b538f..699aabe01ee50 100644
--- a/rust/kernel/types.rs
+++ b/rust/kernel/types.rs
@@ -13,7 +13,10 @@
 
 #[doc(hidden)]
 pub mod for_lt;
-pub use for_lt::ForLt;
+pub use for_lt::{
+    CovariantForLt,
+    ForLt, //
+};
 
 /// Used to transfer ownership to and from foreign (non-Rust) languages.
 ///
diff --git a/rust/kernel/types/for_lt.rs b/rust/kernel/types/for_lt.rs
index d44323c28e8d9..b8f422c802dca 100644
--- a/rust/kernel/types/for_lt.rs
+++ b/rust/kernel/types/for_lt.rs
@@ -1,22 +1,19 @@
 // SPDX-License-Identifier: Apache-2.0 OR MIT
 
-//! Provide implementation and test of the `ForLt` trait and macro.
+//! Provide implementation and test of the [`trait@ForLt`] and [`trait@CovariantForLt`] traits and
+//! macros.
 //!
-//! This module is hidden and user should just use `ForLt!` directly.
+//! This module is hidden and users should just use [`ForLt!`](macro@ForLt) /
+//! [`CovariantForLt!`](macro@CovariantForLt) directly.
 
 use core::marker::PhantomData;
 
 /// Representation of types generic over a lifetime.
 ///
-/// The type must be covariant over the generic lifetime, i.e. the lifetime parameter
-/// can be soundly shortened.
-///
-/// The lifetime involved must be covariant.
-///
 /// # Macro
 ///
-/// It is not recommended to implement this trait directly. `ForLt!` macro is provided to obtain a
-/// type that implements this trait.
+/// It is not recommended to implement this trait directly. [`ForLt!`](macro@ForLt) macro is
+/// provided to obtain a type that implements this trait.
 ///
 /// The full syntax is
 ///
@@ -49,16 +46,65 @@
 /// ForLt!(u32) // Equivalent to `ForLt!(for<'a> u32)`.
 /// # >();
 /// ```
+pub trait ForLt {
+    /// The type parameterized by the lifetime.
+    type Of<'a>: 'a;
+}
+pub use macros::ForLt;
+
+/// [`trait@ForLt`] subtrait for types that are covariant over their lifetime parameter.
+///
+/// Provides a safe [`cast_ref`](CovariantForLt::cast_ref) method for types that are proven to be
+/// covariant. The `CovariantForLt!` macro syntax is the same as `ForLt!`.
+///
+/// # Macro
+///
+/// It is not recommended to implement this trait directly.
+/// [`CovariantForLt!`](macro@CovariantForLt) macro is provided to obtain a type that implements
+/// this trait.
+///
+/// The full syntax is
+///
+/// ```
+/// # use kernel::types::CovariantForLt;
+/// # fn expect_lt<F: CovariantForLt>() {}
+/// # struct TypeThatUse<'a>(&'a ());
+/// # expect_lt::<
+/// CovariantForLt!(for<'a> TypeThatUse<'a>)
+/// # >();
+/// ```
+///
+/// which gives a type so that
+/// `<CovariantForLt!(for<'a> TypeThatUse<'a>) as CovariantForLt>::Of<'b>`
+/// is `TypeThatUse<'b>`.
+///
+/// You may also use a short-hand syntax which works similar to lifetime elision.
+/// The macro also accepts types that do not involve a lifetime at all.
+///
+/// ```
+/// # use kernel::types::CovariantForLt;
+/// # fn expect_lt<F: CovariantForLt>() {}
+/// # struct TypeThatUse<'a>(&'a ());
+/// # expect_lt::<
+/// CovariantForLt!(TypeThatUse<'_>) // Equivalent to `CovariantForLt!(for<'a> TypeThatUse<'a>)`.
+/// # >();
+/// # expect_lt::<
+/// CovariantForLt!(&u32) // Equivalent to `CovariantForLt!(for<'a> &'a u32)`.
+/// # >();
+/// # expect_lt::<
+/// CovariantForLt!(u32) // Equivalent to `CovariantForLt!(for<'a> u32)`.
+/// # >();
+/// ```
 ///
 /// The macro will attempt to prove that the type is indeed covariant over the lifetime supplied.
 /// When it cannot be syntactically proven, it will emit checks to ask the Rust compiler to prove
 /// it.
 ///
 /// ```ignore,compile_fail
-/// # use kernel::types::ForLt;
-/// # fn expect_lt<F: ForLt>() {}
+/// # use kernel::types::CovariantForLt;
+/// # fn expect_lt<F: CovariantForLt>() {}
 /// # expect_lt::<
-/// ForLt!(fn(&u32)) // Contravariant, will fail compilation.
+/// CovariantForLt!(fn(&u32)) // Contravariant, will fail compilation.
 /// # >();
 /// ```
 ///
@@ -67,26 +113,23 @@
 /// the generic parameter but is in a separate item.
 ///
 /// ```
-/// # use kernel::types::ForLt;
-/// fn expect_lt<F: ForLt>() {}
+/// # use kernel::types::CovariantForLt;
+/// fn expect_lt<F: CovariantForLt>() {}
 /// # #[allow(clippy::unnecessary_safety_comment, reason = "false positive")]
 /// fn generic_fn<T: 'static>() {
 ///     // Syntactically proven by the macro
-///     expect_lt::<ForLt!(&T)>();
+///     expect_lt::<CovariantForLt!(&T)>();
 ///     // Syntactically proven by the macro
-///     expect_lt::<ForLt!(&KBox<T>)>();
+///     expect_lt::<CovariantForLt!(&KBox<T>)>();
 ///     // Cannot be syntactically proven, need to check covariance of `KBox`
-///     // expect_lt::<ForLt!(&KBox<&T>)>();
+///     // expect_lt::<CovariantForLt!(&KBox<&T>)>();
 /// }
 /// ```
 ///
 /// # Safety
 ///
 /// `Self::Of<'a>` must be covariant over the lifetime `'a`.
-pub unsafe trait ForLt {
-    /// The type parameterized by the lifetime.
-    type Of<'a>: 'a;
-
+pub unsafe trait CovariantForLt: ForLt {
     /// Cast a reference to a shorter lifetime.
     #[inline(always)]
     fn cast_ref<'r, 'short: 'r, 'long: 'short>(long: &'r Self::Of<'long>) -> &'r Self::Of<'short> {
@@ -94,29 +137,33 @@ fn cast_ref<'r, 'short: 'r, 'long: 'short>(long: &'r Self::Of<'long>) -> &'r Sel
         unsafe { core::mem::transmute(long) }
     }
 }
-pub use macros::ForLt;
+pub use macros::CovariantForLt;
 
 /// This is intended to be an "unsafe-to-refer-to" type.
 ///
-/// Must only be used by the `ForLt!` macro.
+/// Must only be used by the [`ForLt!`](macro@ForLt) / [`CovariantForLt!`](macro@CovariantForLt)
+/// macros.
 ///
 /// `T` is the magic `dyn for<'a> WithLt<'a, TypeThatUse<'a>>` generated by macro.
 ///
 /// `WF` is a type that the macro can use to assert some specific type is well-formed.
 ///
 /// `N` is to provide the macro a place to emit arbitrary items, in case it needs to prove
-/// additional properties.
+/// additional properties. [`ForLt!`](macro@ForLt) emits `N = 0`;
+/// [`CovariantForLt!`](macro@CovariantForLt) emits `N = 1` after a covariance proof.
 #[doc(hidden)]
 pub struct UnsafeForLtImpl<T: ?Sized, WF, const N: usize>(PhantomData<(WF, T)>);
 
-// This is a helper trait for implementation `ForLt` to be able to use HRTB.
+// This is a helper trait for implementation of `ForLt` / `CovariantForLt` to be able to use HRTB.
 #[doc(hidden)]
 pub trait WithLt<'a> {
     type Of: 'a;
 }
 
-// SAFETY: In `ForLt!` macro, a covariance proof is generated when naming `UnsafeForLtImpl`
-// and it will fail to evaluate if the type is not covariant.
-unsafe impl<T: ?Sized + for<'a> WithLt<'a>, WF> ForLt for UnsafeForLtImpl<T, WF, 0> {
+impl<T: ?Sized + for<'a> WithLt<'a>, WF, const N: usize> ForLt for UnsafeForLtImpl<T, WF, N> {
     type Of<'a> = <T as WithLt<'a>>::Of;
 }
+
+// SAFETY: In `CovariantForLt!` macro, a covariance proof is generated in the `N` const generic
+// and it will fail to evaluate if the type is not covariant. Only `N = 1` gets this impl.
+unsafe impl<T: ?Sized + for<'a> WithLt<'a>, WF> CovariantForLt for UnsafeForLtImpl<T, WF, 1> {}
diff --git a/rust/kernel/usb.rs b/rust/kernel/usb.rs
index 7aff0c82d0afc..bbf85366d2c95 100644
--- a/rust/kernel/usb.rs
+++ b/rust/kernel/usb.rs
@@ -24,11 +24,8 @@
 };
 use core::{
     marker::PhantomData,
-    mem::{
-        offset_of,
-        MaybeUninit, //
-    },
-    ptr::NonNull,
+    mem::offset_of,
+    ptr::NonNull, //
 };
 
 /// An adapter for the registration of USB drivers.
@@ -130,8 +127,7 @@ pub const fn from_id(vendor: u16, product: u16) -> Self {
             match_flags: bindings::USB_DEVICE_ID_MATCH_DEVICE as u16,
             idVendor: vendor,
             idProduct: product,
-            // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`.
-            ..unsafe { MaybeUninit::zeroed().assume_init() }
+            ..pin_init::zeroed()
         })
     }
 
@@ -143,8 +139,7 @@ pub const fn from_device_ver(vendor: u16, product: u16, bcd_lo: u16, bcd_hi: u16
             idProduct: product,
             bcdDevice_lo: bcd_lo,
             bcdDevice_hi: bcd_hi,
-            // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`.
-            ..unsafe { MaybeUninit::zeroed().assume_init() }
+            ..pin_init::zeroed()
         })
     }
 
@@ -155,8 +150,7 @@ pub const fn from_device_info(class: u8, subclass: u8, protocol: u8) -> Self {
             bDeviceClass: class,
             bDeviceSubClass: subclass,
             bDeviceProtocol: protocol,
-            // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`.
-            ..unsafe { MaybeUninit::zeroed().assume_init() }
+            ..pin_init::zeroed()
         })
     }
 
@@ -167,8 +161,7 @@ pub const fn from_interface_info(class: u8, subclass: u8, protocol: u8) -> Self
             bInterfaceClass: class,
             bInterfaceSubClass: subclass,
             bInterfaceProtocol: protocol,
-            // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`.
-            ..unsafe { MaybeUninit::zeroed().assume_init() }
+            ..pin_init::zeroed()
         })
     }
 
@@ -180,8 +173,7 @@ pub const fn from_device_interface_class(vendor: u16, product: u16, class: u8) -
             idVendor: vendor,
             idProduct: product,
             bInterfaceClass: class,
-            // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`.
-            ..unsafe { MaybeUninit::zeroed().assume_init() }
+            ..pin_init::zeroed()
         })
     }
 
@@ -193,8 +185,7 @@ pub const fn from_device_interface_protocol(vendor: u16, product: u16, protocol:
             idVendor: vendor,
             idProduct: product,
             bInterfaceProtocol: protocol,
-            // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`.
-            ..unsafe { MaybeUninit::zeroed().assume_init() }
+            ..pin_init::zeroed()
         })
     }
 
@@ -206,8 +197,7 @@ pub const fn from_device_interface_number(vendor: u16, product: u16, number: u8)
             idVendor: vendor,
             idProduct: product,
             bInterfaceNumber: number,
-            // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`.
-            ..unsafe { MaybeUninit::zeroed().assume_init() }
+            ..pin_init::zeroed()
         })
     }
 
@@ -227,8 +217,7 @@ pub const fn from_device_and_interface_info(
             bInterfaceClass: class,
             bInterfaceSubClass: subclass,
             bInterfaceProtocol: protocol,
-            // SAFETY: It is safe to use all zeroes for the other fields of `usb_device_id`.
-            ..unsafe { MaybeUninit::zeroed().assume_init() }
+            ..pin_init::zeroed()
         })
     }
 }
diff --git a/rust/macros/for_lt.rs b/rust/macros/for_lt.rs
index 364d4113cd107..4372cbad3ec4d 100644
--- a/rust/macros/for_lt.rs
+++ b/rust/macros/for_lt.rs
@@ -154,8 +154,8 @@ fn prove(&mut self, ty: &'a Type) {
             // Note that if we encounter `&'other_lt T`, then we still need to make sure the type
             // is wellformed if `T` involves `&'lt`, so we defer to the compiler.
             //
-            // This is to block cases like `ForLt!(for<'a> &'static &'a u32)`, as the presence of
-            // the type implies `'a: 'static` but this is unsound.
+            // This is to block cases like `CovariantForLt!(for<'a> &'static &'a u32)`, as the
+            // presence of the type implies `'a: 'static` but this is unsound.
             Type::Reference(ty)
                 if ty.mutability.is_none() && ty.lifetime.as_ref() == Some(self.0) =>
             {
@@ -176,7 +176,12 @@ fn prove(&mut self, ty: &'a Type) {
     }
 }
 
-pub(crate) fn for_lt(input: HigherRankedType) -> TokenStream {
+/// Shared implementation for both `ForLt!` and `CovariantForLt!`.
+///
+/// Both macros run the prover and emit `ProveWf` structs to check well-formedness for all lifetime
+/// instances (workaround for <https://github.com/rust-lang/rust/issues/152489>). `CovariantForLt!`
+/// additionally emits covariance proof functions and sets `N = 1`.
+fn for_lt_inner(input: HigherRankedType, prove_covariance: bool) -> TokenStream {
     let (ty, lifetime) = match input {
         HigherRankedType::Explicit { lifetime, ty, .. } => (ty, lifetime),
         HigherRankedType::Implicit { ty } => {
@@ -211,14 +216,16 @@ struct #wf_proof_name<#lifetime>(
         ));
 
         // Insert a proof that the type is covariant.
-        let cov_proof_name = format_ident!("prove_covariant_{idx}");
-        proof.push(quote!(
-            fn #cov_proof_name<'__short, '__long: '__short>(
-                long: #wf_proof_name<'__long>
-            ) -> #wf_proof_name<'__short> {
-                long
-            }
-        ));
+        if prove_covariance {
+            let cov_proof_name = format_ident!("prove_covariant_{idx}");
+            proof.push(quote!(
+                fn #cov_proof_name<'__short, '__long: '__short>(
+                    long: #wf_proof_name<'__long>
+                ) -> #wf_proof_name<'__short> {
+                    long
+                }
+            ));
+        }
     }
 
     // Make sure that the type is wellformed when substituting lifetime with `'static`.
@@ -234,6 +241,8 @@ fn #cov_proof_name<'__short, '__long: '__short>(
         },
     );
 
+    let n: usize = prove_covariance.into();
+
     quote!(
         ::kernel::types::for_lt::UnsafeForLtImpl::<
             dyn for<#lifetime> ::kernel::types::for_lt::WithLt<#lifetime, Of = #ty>,
@@ -241,8 +250,16 @@ fn #cov_proof_name<'__short, '__long: '__short>(
             {
                 #(#proof)*
 
-                0
+                #n
             }
         >
     )
 }
+
+pub(crate) fn for_lt(input: HigherRankedType) -> TokenStream {
+    for_lt_inner(input, false)
+}
+
+pub(crate) fn covariant_for_lt(input: HigherRankedType) -> TokenStream {
+    for_lt_inner(input, true)
+}
diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs
index 4a48fabbc2682..0514fc7c0a55a 100644
--- a/rust/macros/lib.rs
+++ b/rust/macros/lib.rs
@@ -56,6 +56,7 @@
 /// - [`u64`]
 /// - [`isize`]
 /// - [`usize`]
+/// - [`bool`]
 ///
 /// C header: [`include/linux/moduleparam.h`](srctree/include/linux/moduleparam.h)
 ///
@@ -497,8 +498,24 @@ pub fn kunit_tests(attr: TokenStream, input: TokenStream) -> TokenStream {
 ///
 /// [`ForLt`]: trait.ForLt.html
 #[proc_macro]
-// The macro shares the name with the trait.
 #[allow(non_snake_case)]
 pub fn ForLt(input: TokenStream) -> TokenStream {
     for_lt::for_lt(parse_macro_input!(input)).into()
 }
+
+/// Obtain a type that implements [`CovariantForLt`] (and [`ForLt`]) for the given higher-ranked
+/// type.
+///
+/// Unlike [`ForLt!`], this macro additionally proves that the type is covariant over the lifetime,
+/// providing a safe [`CovariantForLt::cast_ref`] method.
+///
+/// Please refer to the documentation of the [`CovariantForLt`] trait.
+///
+/// [`CovariantForLt`]: trait.CovariantForLt.html
+/// [`CovariantForLt::cast_ref`]: trait.CovariantForLt.html#method.cast_ref
+/// [`ForLt`]: trait.ForLt.html
+#[proc_macro]
+#[allow(non_snake_case)]
+pub fn CovariantForLt(input: TokenStream) -> TokenStream {
+    for_lt::covariant_for_lt(parse_macro_input!(input)).into()
+}
diff --git a/rust/macros/module.rs b/rust/macros/module.rs
index 06c18e2075083..025323fb030d1 100644
--- a/rust/macros/module.rs
+++ b/rust/macros/module.rs
@@ -192,6 +192,7 @@ fn param_ops_path(param_type: &str) -> Path {
         "u64" => parse_quote!(::kernel::module_param::PARAM_OPS_U64),
         "isize" => parse_quote!(::kernel::module_param::PARAM_OPS_ISIZE),
         "usize" => parse_quote!(::kernel::module_param::PARAM_OPS_USIZE),
+        "bool" => parse_quote!(::kernel::module_param::PARAM_OPS_BOOL),
         t => panic!("Unsupported parameter type {}", t),
     }
 }
diff --git a/rust/pin-init/examples/mutex.rs b/rust/pin-init/examples/mutex.rs
index 35ecb5f68dc31..882f3e23f5ddd 100644
--- a/rust/pin-init/examples/mutex.rs
+++ b/rust/pin-init/examples/mutex.rs
@@ -91,7 +91,7 @@ pub fn new(val: impl PinInit<T>) -> impl PinInit<Self> {
     pub fn lock(&self) -> Pin<CMutexGuard<'_, T>> {
         let mut sguard = self.spin_lock.acquire();
         if self.locked.get() {
-            stack_pin_init!(let wait_entry = WaitEntry::insert_new(&self.wait_list));
+            stack_pin_init!(let _wait_entry = WaitEntry::insert_new(&self.wait_list));
             // println!("wait list length: {}", self.wait_list.size());
             while self.locked.get() {
                 drop(sguard);
@@ -99,9 +99,6 @@ pub fn lock(&self) -> Pin<CMutexGuard<'_, T>> {
                 thread::park();
                 sguard = self.spin_lock.acquire();
             }
-            // This does have an effect, as the ListHead inside wait_entry implements Drop!
-            #[expect(clippy::drop_non_drop)]
-            drop(wait_entry);
         }
         self.locked.set(true);
         unsafe {
diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs
index 28d30805d06b6..fd0b5ea4a0a30 100644
--- a/rust/pin-init/internal/src/init.rs
+++ b/rust/pin-init/internal/src/init.rs
@@ -233,10 +233,12 @@ fn init_fields(
             InitializerKind::Value { ident, .. } => ident,
             InitializerKind::Init { ident, .. } => ident,
             InitializerKind::Code { block, .. } => {
+                let stmt = &block.stmts;
                 res.extend(quote! {
                     #(#attrs)*
-                    #[allow(unused_braces)]
-                    #block
+                    {
+                        #(#stmt)*
+                    }
                 });
                 continue;
             }
@@ -334,7 +336,7 @@ fn make_field_check(
         }),
     };
     quote! {
-        #[allow(unreachable_code, clippy::diverging_sub_expression)]
+        #[allow(unreachable_code)]
         // We use unreachable code to perform field checks. They're still checked by the compiler.
         // SAFETY: this code is never executed.
         let _ = || unsafe {
diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs
index 9fbbd25bcaac5..4438107682e04 100644
--- a/rust/pin-init/internal/src/pin_data.rs
+++ b/rust/pin-init/internal/src/pin_data.rs
@@ -85,7 +85,10 @@ pub(crate) fn pin_data(
         .map(|field| {
             let len = field.attrs.len();
             field.attrs.retain(|a| !a.path().is_ident("pin"));
-            let pinned = len != field.attrs.len();
+            let pinned_count = len - field.attrs.len();
+            if pinned_count > 1 {
+                dcx.error(&field, "#[pin] attribute specified more than once");
+            }
 
             let cfg_attrs = field
                 .attrs
@@ -95,7 +98,7 @@ pub(crate) fn pin_data(
 
             FieldInfo {
                 field: &*field,
-                pinned,
+                pinned: pinned_count != 0,
                 cfg_attrs,
             }
         })
@@ -242,7 +245,6 @@ fn drop(&mut self) {
             // `Drop`. Additionally we will implement this trait for the struct leading to a conflict,
             // if it also implements `Drop`
             trait MustNotImplDrop {}
-            #[expect(drop_bounds)]
             impl<T: ::core::ops::Drop + ?::core::marker::Sized> MustNotImplDrop for T {}
             impl #impl_generics MustNotImplDrop for #ident #ty_generics
                 #whr
@@ -250,7 +252,6 @@ impl #impl_generics MustNotImplDrop for #ident #ty_generics
             // We also take care to prevent users from writing a useless `PinnedDrop` implementation.
             // They might implement `PinnedDrop` correctly for the struct, but forget to give
             // `PinnedDrop` as the parameter to `#[pin_data]`.
-            #[expect(non_camel_case_types)]
             trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
             impl<T: ::pin_init::PinnedDrop + ?::core::marker::Sized>
                 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
@@ -429,7 +430,6 @@ impl #impl_generics ::core::marker::Copy for __ThePinData #ty_generics
         {}
 
         #[allow(dead_code)] // Some functions might never be used and private.
-        #[expect(clippy::missing_safety_doc)]
         impl #impl_generics __ThePinData #ty_generics
             #whr
         {
diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs
index fd40c8f244a1e..ef9f20b110347 100644
--- a/rust/pin-init/src/lib.rs
+++ b/rust/pin-init/src/lib.rs
@@ -70,7 +70,6 @@
 //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
 //!
 //! ```rust
-//! # #![expect(clippy::disallowed_names)]
 //! # #![feature(allocator_api)]
 //! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
 //! # use core::pin::Pin;
@@ -94,7 +93,6 @@
 //! (or just the stack) to actually initialize a `Foo`:
 //!
 //! ```rust
-//! # #![expect(clippy::disallowed_names)]
 //! # #![feature(allocator_api)]
 //! # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
 //! # use core::{alloc::AllocError, pin::Pin};
@@ -456,7 +454,6 @@
 /// # Examples
 ///
 /// ```rust
-/// # #![expect(clippy::disallowed_names)]
 /// # #![feature(allocator_api)]
 /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
 /// # use pin_init::*;
@@ -508,7 +505,6 @@ macro_rules! stack_pin_init {
 /// # Examples
 ///
 /// ```rust
-/// # #![expect(clippy::disallowed_names)]
 /// # #![feature(allocator_api)]
 /// # #[path = "../examples/error.rs"] mod error; use error::Error;
 /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
@@ -535,7 +531,6 @@ macro_rules! stack_pin_init {
 /// ```
 ///
 /// ```rust
-/// # #![expect(clippy::disallowed_names)]
 /// # #![feature(allocator_api)]
 /// # #[path = "../examples/error.rs"] mod error; use error::Error;
 /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
@@ -658,7 +653,6 @@ macro_rules! stack_try_pin_init {
 /// Users of `Foo` can now create it like this:
 ///
 /// ```rust
-/// # #![expect(clippy::disallowed_names)]
 /// # use pin_init::*;
 /// # use core::pin::Pin;
 /// # #[pin_data]
@@ -965,13 +959,11 @@ unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainPinInit<I, F, T, E>
 {
     unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
         // SAFETY: All requirements fulfilled since this function is `__pinned_init`.
-        unsafe { self.0.__pinned_init(slot)? };
-        // SAFETY: The above call initialized `slot` and we still have unique access.
-        let val = unsafe { &mut *slot };
-        // SAFETY: `slot` is considered pinned.
-        let val = unsafe { Pin::new_unchecked(val) };
-        // SAFETY: `slot` was initialized above.
-        (self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) })
+        let slot = unsafe { __internal::Slot::<__internal::Pinned, _>::new(slot) };
+        let mut guard = slot.init(self.0)?;
+        (self.1)(guard.let_binding())?;
+        core::mem::forget(guard);
+        Ok(())
     }
 }
 
@@ -1031,7 +1023,6 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
     /// # Examples
     ///
     /// ```rust
-    /// # #![expect(clippy::disallowed_names)]
     /// use pin_init::{init, init_zeroed, Init};
     ///
     /// struct Foo {
@@ -1072,11 +1063,11 @@ unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E>
 {
     unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
         // SAFETY: All requirements fulfilled since this function is `__init`.
-        unsafe { self.0.__pinned_init(slot)? };
-        // SAFETY: The above call initialized `slot` and we still have unique access.
-        (self.1)(unsafe { &mut *slot }).inspect_err(|_|
-            // SAFETY: `slot` was initialized above.
-            unsafe { core::ptr::drop_in_place(slot) })
+        let slot = unsafe { __internal::Slot::<__internal::Unpinned, _>::new(slot) };
+        let mut guard = slot.init(self.0)?;
+        (self.1)(guard.let_binding())?;
+        core::mem::forget(guard);
+        Ok(())
     }
 }
 
@@ -1193,6 +1184,82 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
     unsafe { init_from_closure(|_| Ok(())) }
 }
 
+/// Array initializer from element initializer.
+struct ArrayInit<T: ?Sized, F>(F, __internal::PhantomInvariant<T>);
+
+// SAFETY: On success, all `N` elements of the array have been initialized. On error or panic, the
+// elements that have been initialized so far are dropped, thus leaving the array uninitialized and
+// ready to deallocate.
+unsafe impl<T, F, I, E, const N: usize> PinInit<[T; N], E> for ArrayInit<T, F>
+where
+    F: FnMut(usize) -> I,
+    I: PinInit<T, E>,
+{
+    unsafe fn __pinned_init(mut self, slot: *mut [T; N]) -> Result<(), E> {
+        /// # Invariants
+        ///
+        /// - `ptr[..num_init]` contains initialized elements of type `T`
+        /// - `ptr[num_init..N]` (where N is the size of the array) contains uninitialized memory
+        struct ArrayInitGuard<T> {
+            /// A pointer to the first element of the array.
+            ptr: *mut T,
+            /// The number of initialized elements in the array.
+            num_init: usize,
+        }
+
+        impl<T> Drop for ArrayInitGuard<T> {
+            #[inline]
+            fn drop(&mut self) {
+                // SAFETY: Per type invariant, `self.ptr[..self.num_init]` are initialized.
+                unsafe {
+                    core::ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut(
+                        self.ptr,
+                        self.num_init,
+                    ))
+                };
+            }
+        }
+
+        // INVARIANT: nothing is initialized yet.
+        let mut guard = ArrayInitGuard {
+            ptr: slot.cast::<T>(),
+            num_init: 0,
+        };
+
+        for i in 0..N {
+            // INVARIANT: Elements `self.ptr[..self.num_init]` have been initialized
+            // thus far. This holds true for every `self.num_init = i`.
+            guard.num_init = i;
+
+            let init = (self.0)(i);
+            // SAFETY:
+            // - The subslot is derived from `slot` with a valid offset.
+            // - If `Err` is touched, the subslot is not touched further, the guard will drop
+            //   previously initialized elements only.
+            // - `slot` is pinned so is the subslot.
+            unsafe { init.__pinned_init(&raw mut (*slot)[i]) }?;
+        }
+
+        // Dismiss the drop guard now that all elements are initialized.
+        core::mem::forget(guard);
+        Ok(())
+    }
+}
+
+// SAFETY: Follows the `PinInit` impl. `__init` executes the same code as `__pinned_init`.
+unsafe impl<T, F, I, E, const N: usize> Init<[T; N], E> for ArrayInit<T, F>
+where
+    F: FnMut(usize) -> I,
+    I: Init<T, E>,
+{
+    #[inline(always)]
+    unsafe fn __init(self, slot: *mut [T; N]) -> Result<(), E> {
+        // SAFETY: `I: Init` cancels out the pinning requirement on subslots. The other safety
+        // requirements follow that of `__init`.
+        unsafe { self.__pinned_init(slot) }
+    }
+}
+
 /// Initializes an array by initializing each element via the provided initializer.
 ///
 /// # Examples
@@ -1204,31 +1271,12 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
 /// assert_eq!(array.len(), 1_000);
 /// ```
 pub fn init_array_from_fn<I, const N: usize, T, E>(
-    mut make_init: impl FnMut(usize) -> I,
+    make_init: impl FnMut(usize) -> I,
 ) -> impl Init<[T; N], E>
 where
     I: Init<T, E>,
 {
-    let init = move |slot: *mut [T; N]| {
-        let slot = slot.cast::<T>();
-        for i in 0..N {
-            let init = make_init(i);
-            // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
-            let ptr = unsafe { slot.add(i) };
-            // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
-            // requirements.
-            if let Err(e) = unsafe { init.__init(ptr) } {
-                // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return
-                // `Err` below, `slot` will be considered uninitialized memory.
-                unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
-                return Err(e);
-            }
-        }
-        Ok(())
-    };
-    // SAFETY: The initializer above initializes every element of the array. On failure it drops
-    // any initialized elements and returns `Err`.
-    unsafe { init_from_closure(init) }
+    ArrayInit(make_init, __internal::PhantomInvariant::new())
 }
 
 /// Initializes an array by initializing each element via the provided initializer.
@@ -1247,31 +1295,12 @@ pub fn init_array_from_fn<I, const N: usize, T, E>(
 /// assert_eq!(array.len(), 1_000);
 /// ```
 pub fn pin_init_array_from_fn<I, const N: usize, T, E>(
-    mut make_init: impl FnMut(usize) -> I,
+    make_init: impl FnMut(usize) -> I,
 ) -> impl PinInit<[T; N], E>
 where
     I: PinInit<T, E>,
 {
-    let init = move |slot: *mut [T; N]| {
-        let slot = slot.cast::<T>();
-        for i in 0..N {
-            let init = make_init(i);
-            // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
-            let ptr = unsafe { slot.add(i) };
-            // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
-            // requirements.
-            if let Err(e) = unsafe { init.__pinned_init(ptr) } {
-                // SAFETY: The loop has initialized the elements `slot[0..i]` and since we return
-                // `Err` below, `slot` will be considered uninitialized memory.
-                unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
-                return Err(e);
-            }
-        }
-        Ok(())
-    };
-    // SAFETY: The initializer above initializes every element of the array. On failure it drops
-    // any initialized elements and returns `Err`.
-    unsafe { pin_init_from_closure(init) }
+    ArrayInit(make_init, __internal::PhantomInvariant::new())
 }
 
 /// Construct an initializer in a closure and run it.
diff --git a/rust/uapi/lib.rs b/rust/uapi/lib.rs
index b8a515de31cae..797ead5b5626c 100644
--- a/rust/uapi/lib.rs
+++ b/rust/uapi/lib.rs
@@ -24,6 +24,10 @@
     unsafe_op_in_unsafe_fn
 )]
 #![cfg_attr(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES, allow(unnecessary_transmutes))]
+#![cfg_attr(
+    CONFIG_RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS,
+    allow(suspicious_runtime_symbol_definitions)
+)]
 #![feature(cfi_encoding)]
 
 // Manual definition of blocklisted types.
diff --git a/rust/zerocopy-derive/README.md b/rust/zerocopy-derive/README.md
index 110f4a4017783..d62c798043424 100644
--- a/rust/zerocopy-derive/README.md
+++ b/rust/zerocopy-derive/README.md
@@ -1,14 +1,14 @@
 # `zerocopy-derive`
 
-These source files come from the Rust `zerocopy-derive` crate, version v0.8.50
-(released 2026-05-31), hosted in the <https://github.com/google/zerocopy>
+These source files come from the Rust `zerocopy-derive` crate, version v0.8.54
+(released 2026-07-08), hosted in the <https://github.com/google/zerocopy>
 repository, licensed under "BSD-2-Clause OR Apache-2.0 OR MIT" and only
-modified to add the SPDX license identifiers and to remove the generation of
+modified to tweak the SPDX license identifiers and to remove the generation of
 non-ASCII identifiers.
 
 For copyright details, please see:
 
-    https://github.com/google/zerocopy/blob/v0.8.50/README.md?plain=1
-    https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-BSD
-    https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-APACHE
-    https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-MIT
+    https://github.com/google/zerocopy/blob/v0.8.54/README.md?plain=1
+    https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-BSD
+    https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-APACHE
+    https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-MIT
diff --git a/rust/zerocopy-derive/derive/from_bytes.rs b/rust/zerocopy-derive/derive/from_bytes.rs
index d693a63b76451..66d820f6ad4cd 100644
--- a/rust/zerocopy-derive/derive/from_bytes.rs
+++ b/rust/zerocopy-derive/derive/from_bytes.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 use proc_macro2::{Span, TokenStream};
 use syn::{
     parse_quote, Data, DataEnum, DataStruct, DataUnion, Error, Expr, ExprLit, ExprUnary, Lit, UnOp,
diff --git a/rust/zerocopy-derive/derive/into_bytes.rs b/rust/zerocopy-derive/derive/into_bytes.rs
index ad52a6b45d288..0103a78d087f7 100644
--- a/rust/zerocopy-derive/derive/into_bytes.rs
+++ b/rust/zerocopy-derive/derive/into_bytes.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 use proc_macro2::{Span, TokenStream};
 use quote::quote;
 use syn::{Data, DataEnum, DataStruct, DataUnion, Error, Type};
diff --git a/rust/zerocopy-derive/derive/known_layout.rs b/rust/zerocopy-derive/derive/known_layout.rs
index fddffd167c827..d0c4cecfff151 100644
--- a/rust/zerocopy-derive/derive/known_layout.rs
+++ b/rust/zerocopy-derive/derive/known_layout.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 use proc_macro2::TokenStream;
 use quote::quote;
 use syn::{parse_quote, Data, Error, Type};
@@ -87,7 +87,6 @@ fn pointer_to_metadata(ptr: *mut Self) -> <Self as #zerocopy_crate::KnownLayout>
     };
 
     let inner_extras = {
-        let leading_fields_tys = leading_fields_tys.clone();
         let methods = make_methods(*trailing_field_ty);
         let (_, ty_generics, _) = ctx.ast.generics.split_for_impl();
 
diff --git a/rust/zerocopy-derive/derive/mod.rs b/rust/zerocopy-derive/derive/mod.rs
index 665ba7da55a80..b3839fcf73c97 100644
--- a/rust/zerocopy-derive/derive/mod.rs
+++ b/rust/zerocopy-derive/derive/mod.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 pub mod from_bytes;
 pub mod into_bytes;
 pub mod known_layout;
@@ -15,8 +15,8 @@
     util::{Ctx, DataExt, FieldBounds, ImplBlockBuilder, Trait},
 };
 
-pub(crate) fn derive_immutable(ctx: &Ctx, _top_level: Trait) -> TokenStream {
-    match &ctx.ast.data {
+pub(crate) fn derive_immutable(ctx: &Ctx, _top_level: Trait) -> Result<TokenStream, Error> {
+    Ok(match &ctx.ast.data {
         Data::Struct(strct) => {
             ImplBlockBuilder::new(ctx, strct, Trait::Immutable, FieldBounds::ALL_SELF).build()
         }
@@ -26,7 +26,7 @@ pub(crate) fn derive_immutable(ctx: &Ctx, _top_level: Trait) -> TokenStream {
         Data::Union(unn) => {
             ImplBlockBuilder::new(ctx, unn, Trait::Immutable, FieldBounds::ALL_SELF).build()
         }
-    }
+    })
 }
 
 pub(crate) fn derive_hash(ctx: &Ctx, _top_level: Trait) -> Result<TokenStream, Error> {
@@ -97,16 +97,20 @@ pub(crate) fn derive_split_at(ctx: &Ctx, _top_level: Trait) -> Result<TokenStrea
     match &ctx.ast.data {
         Data::Struct(_) => {}
         Data::Enum(_) | Data::Union(_) => {
-            return Err(Error::new(Span::call_site(), "can only be applied to structs"));
+            return ctx
+                .error_or_skip(Error::new(Span::call_site(), "can only be applied to structs"));
         }
     };
 
     if repr.get_packed().is_some() {
-        return Err(Error::new(Span::call_site(), "must not have #[repr(packed)] attribute"));
+        return ctx.error_or_skip(Error::new(
+            Span::call_site(),
+            "must not have #[repr(packed)] attribute",
+        ));
     }
 
     if !(repr.is_c() || repr.is_transparent()) {
-        return Err(Error::new(
+        return ctx.error_or_skip(Error::new(
             Span::call_site(),
             "must have #[repr(C)] or #[repr(transparent)] in order to guarantee this type's layout is splitable",
         ));
@@ -116,7 +120,7 @@ pub(crate) fn derive_split_at(ctx: &Ctx, _top_level: Trait) -> Result<TokenStrea
     let trailing_field = if let Some(((_, _, trailing_field), _)) = fields.split_last() {
         trailing_field
     } else {
-        return Err(Error::new(Span::call_site(), "must at least one field"));
+        return ctx.error_or_skip(Error::new(Span::call_site(), "must at least one field"));
     };
 
     let zerocopy_crate = &ctx.zerocopy_crate;
diff --git a/rust/zerocopy-derive/derive/try_from_bytes.rs b/rust/zerocopy-derive/derive/try_from_bytes.rs
index a3e4a75631a5c..44f0833287864 100644
--- a/rust/zerocopy-derive/derive/try_from_bytes.rs
+++ b/rust/zerocopy-derive/derive/try_from_bytes.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 use proc_macro2::TokenStream;
 use quote::quote;
 use syn::{
@@ -531,8 +531,8 @@ fn project(slf: #zerocopy_crate::pointer::PtrInner<'_, Self>) -> *mut <Self as #
                 data,
                 Trait::ProjectField {
                     variant_id: variant_id.clone(),
-                    field: field.clone(),
-                    field_id: field_id.clone(),
+                    field,
+                    field_id,
                     invariants: parse_quote!((Aliasing, Alignment, #zerocopy_crate::invariant::Initialized)),
                 },
                 FieldBounds::None,
diff --git a/rust/zerocopy-derive/derive/unaligned.rs b/rust/zerocopy-derive/derive/unaligned.rs
index d6dea0a11f1eb..7c97d62e2dcb5 100644
--- a/rust/zerocopy-derive/derive/unaligned.rs
+++ b/rust/zerocopy-derive/derive/unaligned.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 use proc_macro2::{Span, TokenStream};
 use syn::{Data, DataEnum, DataStruct, DataUnion, Error};
 
diff --git a/rust/zerocopy-derive/lib.rs b/rust/zerocopy-derive/lib.rs
index c517ea7db1eb1..d387de3683674 100644
--- a/rust/zerocopy-derive/lib.rs
+++ b/rust/zerocopy-derive/lib.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2019 The Fuchsia Authors
 //
 // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
@@ -129,6 +129,40 @@ fn into_ts(self) -> proc_macro2::TokenStream {
 derive!(ByteEq => derive_eq => crate::derive::derive_eq);
 derive!(SplitAt => derive_split_at => crate::derive::derive_split_at);
 
+#[cfg_attr(not(zerocopy_unstable_linux), doc(hidden))]
+#[proc_macro_derive(most_traits, attributes(zerocopy))]
+pub fn most_traits(ts: proc_macro::TokenStream) -> proc_macro::TokenStream {
+    let ast = syn::parse_macro_input!(ts as DeriveInput);
+    let ctx = match Ctx::try_from_derive_input(ast) {
+        Ok(ctx) => ctx,
+        Err(e) => return e.into_compile_error().into(),
+    }
+    .skip_on_error();
+
+    // top-level traits for which to attempt a derive
+    let derives: [(fn(&Ctx, Trait) -> _, _); 6] = [
+        (crate::derive::known_layout::derive, Trait::KnownLayout),
+        (crate::derive::derive_immutable, Trait::Immutable),
+        (crate::derive::from_bytes::derive_from_bytes, Trait::FromBytes),
+        (crate::derive::into_bytes::derive_into_bytes, Trait::IntoBytes),
+        (crate::derive::derive_split_at, Trait::SplitAt),
+        (crate::derive::unaligned::derive_unaligned, Trait::Unaligned),
+    ];
+
+    let mut tokens = proc_macro2::TokenStream::new();
+    for (derive, t) in derives {
+        tokens.extend(derive(&ctx, t))
+    }
+
+    // We wrap in `const_block` as a backstop in case any derive fails
+    // to wrap its output in `const_block` (and thus fails to annotate)
+    // with the full set of `#[allow(...)]` attributes).
+    let ts = const_block([Some(tokens)]);
+    #[cfg(test)]
+    crate::util::testutil::check_hygiene(ts.clone());
+    ts.into()
+}
+
 /// Deprecated: prefer [`FromZeros`] instead.
 #[deprecated(since = "0.8.0", note = "`FromZeroes` was renamed to `FromZeros`")]
 #[doc(hidden)]
diff --git a/rust/zerocopy-derive/repr.rs b/rust/zerocopy-derive/repr.rs
index 74fd376d9fdab..1525e94302d1b 100644
--- a/rust/zerocopy-derive/repr.rs
+++ b/rust/zerocopy-derive/repr.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2019 The Fuchsia Authors
 //
 // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
diff --git a/rust/zerocopy-derive/util.rs b/rust/zerocopy-derive/util.rs
index 5ba5228e2a44f..5c5e9d3bdcb8f 100644
--- a/rust/zerocopy-derive/util.rs
+++ b/rust/zerocopy-derive/util.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2019 The Fuchsia Authors
 //
 // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
@@ -98,6 +98,11 @@ pub(crate) fn with_input(&self, input: &DeriveInput) -> Self {
         }
     }
 
+    pub(crate) fn skip_on_error(mut self) -> Self {
+        self.skip_on_error = true;
+        self
+    }
+
     pub(crate) fn core_path(&self) -> TokenStream {
         let zerocopy_crate = &self.zerocopy_crate;
         quote!(#zerocopy_crate::util::macro_util::core_reexport)
@@ -106,20 +111,21 @@ pub(crate) fn core_path(&self) -> TokenStream {
     pub(crate) fn cfg_compile_error(&self) -> TokenStream {
         // By checking both during the compilation of the proc macro *and* in
         // the generated code, we ensure that `--cfg
-        // zerocopy_unstable_derive_on_error` need only be passed *either* when
+        // zerocopy_unstable_linux` need only be passed *either* when
         // compiling this crate *or* when compiling the user's crate. The former
         // is preferable, but in some situations (such as when cross-compiling
         // using `cargo build --target`), it doesn't get propagated to this
         // crate's build by default.
-        if cfg!(zerocopy_unstable_derive_on_error) {
+        if cfg!(zerocopy_unstable_linux) {
             quote!()
         } else if let Some(span) = self.on_error_span {
             let core = self.core_path();
-            let error_message = "`on_error` is experimental; pass '--cfg zerocopy_unstable_derive_on_error' to enable";
+            let error_message =
+                "`on_error` is experimental; pass '--cfg zerocopy_unstable_linux' to enable";
             quote::quote_spanned! {span=>
                 #[allow(unused_attributes, unexpected_cfgs)]
                 const _: () = {
-                    #[cfg(not(zerocopy_unstable_derive_on_error))]
+                    #[cfg(not(zerocopy_unstable_linux))]
                     #core::compile_error!(#error_message);
                 };
             }
@@ -612,6 +618,20 @@ fn bound_tt(ty: &Type, traits: impl Iterator<Item = Trait>, ctx: &Ctx) -> WhereP
             }
         };
 
+        let zerocopy_bounds =
+            field_type_bounds
+                .into_iter()
+                .chain(padding_check_bound)
+                .chain(self_bounds)
+                .map(|bound| {
+                    if self.ctx.skip_on_error {
+                        parse_quote!(for<'zc> #bound)
+                    } else {
+                        bound.clone()
+                    }
+                })
+                .collect::<Vec<_>>();
+
         let bounds = self
             .ctx
             .ast
@@ -621,9 +641,7 @@ fn bound_tt(ty: &Type, traits: impl Iterator<Item = Trait>, ctx: &Ctx) -> WhereP
             .map(|where_clause| where_clause.predicates.iter())
             .into_iter()
             .flatten()
-            .chain(field_type_bounds.iter())
-            .chain(padding_check_bound.iter())
-            .chain(self_bounds.iter());
+            .chain(zerocopy_bounds.iter());
 
         // The parameters with trait bounds, but without type defaults.
         let mut params: Vec<_> = self
diff --git a/rust/zerocopy/README.md b/rust/zerocopy/README.md
index 99e6cad0e26ca..3d11a6502cf0c 100644
--- a/rust/zerocopy/README.md
+++ b/rust/zerocopy/README.md
@@ -1,14 +1,13 @@
 # `zerocopy`
 
-These source files come from the Rust `zerocopy` crate, version v0.8.50
-(released 2026-05-31), hosted in the <https://github.com/google/zerocopy>
+These source files come from the Rust `zerocopy` crate, version v0.8.54
+(released 2026-07-08), hosted in the <https://github.com/google/zerocopy>
 repository, licensed under "BSD-2-Clause OR Apache-2.0 OR MIT" and only
-modified to add the SPDX license identifiers and to remove `Display`
-for `f32` and `f64`.
+modified to tweak the SPDX license identifiers.
 
 For copyright details, please see:
 
-    https://github.com/google/zerocopy/blob/v0.8.50/README.md?plain=1
-    https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-BSD
-    https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-APACHE
-    https://github.com/google/zerocopy/blob/v0.8.50/LICENSE-MIT
+    https://github.com/google/zerocopy/blob/v0.8.54/README.md?plain=1
+    https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-BSD
+    https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-APACHE
+    https://github.com/google/zerocopy/blob/v0.8.54/LICENSE-MIT
diff --git a/rust/zerocopy/rustdoc/style.css b/rust/zerocopy/rustdoc/style.css
index 414348964af27..d2d55ad2e6893 100644
--- a/rust/zerocopy/rustdoc/style.css
+++ b/rust/zerocopy/rustdoc/style.css
@@ -1,5 +1,4 @@
 /* SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT */
-
 /*
 Copyright 2026 The Fuchsia Authors
 
diff --git a/rust/zerocopy/src/byte_slice.rs b/rust/zerocopy/src/byte_slice.rs
index a5ded4a18b397..b7f85098dbc4c 100644
--- a/rust/zerocopy/src/byte_slice.rs
+++ b/rust/zerocopy/src/byte_slice.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2024 The Fuchsia Authors
 //
 // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
diff --git a/rust/zerocopy/src/byteorder.rs b/rust/zerocopy/src/byteorder.rs
index 8f70048f1eb03..c761d5728320b 100644
--- a/rust/zerocopy/src/byteorder.rs
+++ b/rust/zerocopy/src/byteorder.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2019 The Fuchsia Authors
 //
 // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
@@ -100,6 +100,7 @@ impl Sealed for super::LittleEndian {}
 
 #[allow(missing_copy_implementations, missing_debug_implementations)]
 #[doc(hidden)]
+#[derive(PartialEq)]
 pub enum Order {
     BigEndian,
     LittleEndian,
@@ -164,6 +165,42 @@ fn fmt(&self, _: &mut Formatter<'_>) -> fmt::Result {
 /// A type alias for [`LittleEndian`].
 pub type LE = LittleEndian;
 
+macro_rules! impl_dbg_trait {
+    ($name:ident, $native:ident) => {
+        impl<O: ByteOrder> Debug for $name<O> {
+            #[inline]
+            fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+                // This results in a format like "U16(42)".
+                f.debug_tuple(stringify!($name)).field(&self.get()).finish()
+            }
+        }
+    };
+}
+
+macro_rules! impl_dbg_traits {
+    ($name:ident, $native:ident, "floating point number") => {
+        #[cfg(not(no_fp_fmt_parse))]
+        impl_dbg_trait!($name, $native);
+
+        #[cfg(no_fp_fmt_parse)]
+        impl<O: ByteOrder> Debug for $name<O> {
+            #[inline]
+            fn fmt(&self, _f: &mut Formatter<'_>) -> fmt::Result {
+                panic!("floating point support is turned off");
+            }
+        }
+    };
+    ($name:ident, $native:ident, "unsigned integer") => {
+        impl_dbg_traits!($name, $native, @all_types);
+    };
+    ($name:ident, $native:ident, "signed integer") => {
+        impl_dbg_traits!($name, $native, @all_types);
+    };
+    ($name:ident, $native:ident, @all_types) => {
+        impl_dbg_trait!($name, $native);
+    };
+}
+
 macro_rules! impl_fmt_trait {
     ($name:ident, $native:ident, $trait:ident) => {
         impl<O: ByteOrder> $trait for $name<O> {
@@ -177,6 +214,8 @@ fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
 
 macro_rules! impl_fmt_traits {
     ($name:ident, $native:ident, "floating point number") => {
+        #[cfg(not(no_fp_fmt_parse))]
+        impl_fmt_trait!($name, $native, Display);
     };
     ($name:ident, $native:ident, "unsigned integer") => {
         impl_fmt_traits!($name, $native, @all_types);
@@ -687,16 +726,9 @@ fn eq(&self, other: &$native) -> bool {
             }
         }
 
+        impl_dbg_traits!($name, $native, $number_kind);
         impl_fmt_traits!($name, $native, $number_kind);
         impl_ops_traits!($name, $native, $number_kind);
-
-        impl<O: ByteOrder> Debug for $name<O> {
-            #[inline]
-            fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-                // This results in a format like "U16(42)".
-                f.debug_tuple(stringify!($name)).field(&self.get()).finish()
-            }
-        }
     };
 }
 
diff --git a/rust/zerocopy/src/deprecated.rs b/rust/zerocopy/src/deprecated.rs
index 24bafbf9adebe..59ddd35c77c6a 100644
--- a/rust/zerocopy/src/deprecated.rs
+++ b/rust/zerocopy/src/deprecated.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2024 The Fuchsia Authors
 //
 // Licensed under the 2-Clause BSD License <LICENSE-BSD or
diff --git a/rust/zerocopy/src/error.rs b/rust/zerocopy/src/error.rs
index 7cb08c31d452c..5eb30de934f38 100644
--- a/rust/zerocopy/src/error.rs
+++ b/rust/zerocopy/src/error.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2024 The Fuchsia Authors
 //
 // Licensed under the 2-Clause BSD License <LICENSE-BSD or
diff --git a/rust/zerocopy/src/impls.rs b/rust/zerocopy/src/impls.rs
index 22fd6c3d5d940..62e234c5202b5 100644
--- a/rust/zerocopy/src/impls.rs
+++ b/rust/zerocopy/src/impls.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2024 The Fuchsia Authors
 //
 // Licensed under the 2-Clause BSD License <LICENSE-BSD or
@@ -1359,7 +1359,7 @@ mod $mod {
         #[cfg(not(no_zerocopy_aarch64_simd_1_59_0))]
         simd_arch_mod!(
             #[cfg(all(
-                target_arch = "aarch64", 
+                target_arch = "aarch64",
                 any(
                     target_endian = "little",
                     not(no_zerocopy_aarch64_simd_be_1_87_0)
diff --git a/rust/zerocopy/src/layout.rs b/rust/zerocopy/src/layout.rs
index 6015d0f2de52a..b1fa0cd436db8 100644
--- a/rust/zerocopy/src/layout.rs
+++ b/rust/zerocopy/src/layout.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2024 The Fuchsia Authors
 //
 // Licensed under the 2-Clause BSD License <LICENSE-BSD or
@@ -71,6 +71,8 @@ impl SizeInfo {
     /// Attempts to create a `SizeInfo` from `Self` in which `elem_size` is a
     /// `NonZeroUsize`. If `elem_size` is 0, returns `None`.
     #[allow(unused)]
+    #[cfg_attr(not(zerocopy_inline_always), inline)]
+    #[cfg_attr(zerocopy_inline_always, inline(always))]
     const fn try_to_nonzero_elem_size(&self) -> Option<SizeInfo<NonZeroUsize>> {
         Some(match *self {
             SizeInfo::Sized { size } => SizeInfo::Sized { size },
diff --git a/rust/zerocopy/src/lib.rs b/rust/zerocopy/src/lib.rs
index 3302d67602abd..572f0563fe2f2 100644
--- a/rust/zerocopy/src/lib.rs
+++ b/rust/zerocopy/src/lib.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2018 The Fuchsia Authors
 //
 // Licensed under the 2-Clause BSD License <LICENSE-BSD or
@@ -12,7 +12,7 @@
 // After updating the following doc comment, make sure to run the following
 // command to update `README.md` based on its contents:
 //
-//   cargo -q run --manifest-path tools/Cargo.toml -p generate-readme > README.md
+//   (cd .. && cargo -q run --manifest-path tools/Cargo.toml -p generate-readme) > README.md
 
 //! ***<span style="font-size: 140%">Fast, safe, <span
 //! style="color:red;">compile error</span>. Pick two.</span>***
@@ -174,7 +174,7 @@
 //!
 //! [Miri]: https://github.com/rust-lang/miri
 //! [Kani]: https://github.com/model-checking/kani
-//! [soundness policy]: https://github.com/google/zerocopy/blob/main/POLICIES.md#soundness
+//! [soundness policy]: https://github.com/google/zerocopy/blob/main/zerocopy/POLICIES.md#soundness
 //!
 //! # Relationship to Project Safe Transmute
 //!
@@ -203,7 +203,7 @@
 //!
 //! See our [MSRV policy].
 //!
-//! [MSRV policy]: https://github.com/google/zerocopy/blob/main/POLICIES.md#msrv
+//! [MSRV policy]: https://github.com/google/zerocopy/blob/main/zerocopy/POLICIES.md#msrv
 //!
 //! # Changelog
 //!
@@ -435,6 +435,8 @@
     WARNING
 };
 
+#[cfg(all(any(feature = "derive", test), zerocopy_unstable_linux))]
+pub use zerocopy_derive::most_traits;
 /// Implements [`KnownLayout`].
 ///
 /// This derive analyzes various aspects of a type's layout that are needed for
@@ -2832,7 +2834,7 @@ fn try_ref_from_suffix_with_elems(
     /// ```
     ///
     /// [`try_mut_from_bytes`]: TryFromBytes::try_mut_from_bytes
-    ///  
+    ///
     #[doc = codegen_header!("h5", "try_mut_from_bytes_with_elems")]
     ///
     /// See [`TryFromBytes::try_ref_from_bytes_with_elems`](#method.try_ref_from_bytes_with_elems.codegen).
diff --git a/rust/zerocopy/src/macros.rs b/rust/zerocopy/src/macros.rs
index b801d86a8fa63..ec67c03a44fc4 100644
--- a/rust/zerocopy/src/macros.rs
+++ b/rust/zerocopy/src/macros.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2024 The Fuchsia Authors
 //
 // Licensed under the 2-Clause BSD License <LICENSE-BSD or
diff --git a/rust/zerocopy/src/pointer/inner.rs b/rust/zerocopy/src/pointer/inner.rs
index 5db08080141fa..949b60a3f83ea 100644
--- a/rust/zerocopy/src/pointer/inner.rs
+++ b/rust/zerocopy/src/pointer/inner.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2024 The Fuchsia Authors
 //
 // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
diff --git a/rust/zerocopy/src/pointer/invariant.rs b/rust/zerocopy/src/pointer/invariant.rs
index 1802d23563db8..7ff0d43dad5e8 100644
--- a/rust/zerocopy/src/pointer/invariant.rs
+++ b/rust/zerocopy/src/pointer/invariant.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2024 The Fuchsia Authors
 //
 // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
diff --git a/rust/zerocopy/src/pointer/mod.rs b/rust/zerocopy/src/pointer/mod.rs
index 3461f7f5ca80e..d6eacc52febe2 100644
--- a/rust/zerocopy/src/pointer/mod.rs
+++ b/rust/zerocopy/src/pointer/mod.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2023 The Fuchsia Authors
 //
 // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
diff --git a/rust/zerocopy/src/pointer/ptr.rs b/rust/zerocopy/src/pointer/ptr.rs
index b7c4ea56d2b2c..7213f6f4a04ee 100644
--- a/rust/zerocopy/src/pointer/ptr.rs
+++ b/rust/zerocopy/src/pointer/ptr.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2023 The Fuchsia Authors
 //
 // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
diff --git a/rust/zerocopy/src/pointer/transmute.rs b/rust/zerocopy/src/pointer/transmute.rs
index a534984b70d30..ef9836698203a 100644
--- a/rust/zerocopy/src/pointer/transmute.rs
+++ b/rust/zerocopy/src/pointer/transmute.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2025 The Fuchsia Authors
 //
 // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
diff --git a/rust/zerocopy/src/ref.rs b/rust/zerocopy/src/ref.rs
index 860066d751962..e49f2a887ffa7 100644
--- a/rust/zerocopy/src/ref.rs
+++ b/rust/zerocopy/src/ref.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2024 The Fuchsia Authors
 //
 // Licensed under the 2-Clause BSD License <LICENSE-BSD or
diff --git a/rust/zerocopy/src/split_at.rs b/rust/zerocopy/src/split_at.rs
index 9a67d5acbb0d7..d7778425a31d3 100644
--- a/rust/zerocopy/src/split_at.rs
+++ b/rust/zerocopy/src/split_at.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2025 The Fuchsia Authors
 //
 // Licensed under the 2-Clause BSD License <LICENSE-BSD or
diff --git a/rust/zerocopy/src/util/macro_util.rs b/rust/zerocopy/src/util/macro_util.rs
index 1abb0fbeb46e6..ceeb80432b0ba 100644
--- a/rust/zerocopy/src/util/macro_util.rs
+++ b/rust/zerocopy/src/util/macro_util.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2022 The Fuchsia Authors
 //
 // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
diff --git a/rust/zerocopy/src/util/macros.rs b/rust/zerocopy/src/util/macros.rs
index 43e4fd64ee158..7e63e3a54fc41 100644
--- a/rust/zerocopy/src/util/macros.rs
+++ b/rust/zerocopy/src/util/macros.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2023 The Fuchsia Authors
 //
 // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
diff --git a/rust/zerocopy/src/util/mod.rs b/rust/zerocopy/src/util/mod.rs
index d6d4c6c2fcd9a..02fd4ed62741b 100644
--- a/rust/zerocopy/src/util/mod.rs
+++ b/rust/zerocopy/src/util/mod.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2023 The Fuchsia Authors
 //
 // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
@@ -150,6 +150,8 @@ pub(crate) fn validate_aligned_to<T: AsAddress, U>(t: T) -> Result<(), Alignment
     // Ensures that we add the minimum required padding.
     kani::ensures(|&p| p < align.get()),
 )]
+#[cfg_attr(not(zerocopy_inline_always), inline)]
+#[cfg_attr(zerocopy_inline_always, inline(always))]
 pub(crate) const fn padding_needed_for(len: usize, align: NonZeroUsize) -> usize {
     #[cfg(kani)]
     #[kani::proof_for_contract(padding_needed_for)]
@@ -251,6 +253,8 @@ fn proof() {
     n & mask
 }
 
+#[cfg_attr(not(zerocopy_inline_always), inline)]
+#[cfg_attr(zerocopy_inline_always, inline(always))]
 pub(crate) const fn max(a: NonZeroUsize, b: NonZeroUsize) -> NonZeroUsize {
     if a.get() < b.get() {
         b
@@ -259,6 +263,8 @@ pub(crate) const fn max(a: NonZeroUsize, b: NonZeroUsize) -> NonZeroUsize {
     }
 }
 
+#[cfg_attr(not(zerocopy_inline_always), inline)]
+#[cfg_attr(zerocopy_inline_always, inline(always))]
 pub(crate) const fn min(a: NonZeroUsize, b: NonZeroUsize) -> NonZeroUsize {
     if a.get() > b.get() {
         b
diff --git a/rust/zerocopy/src/wrappers.rs b/rust/zerocopy/src/wrappers.rs
index 266aec25fa580..1a8cf2b41d555 100644
--- a/rust/zerocopy/src/wrappers.rs
+++ b/rust/zerocopy/src/wrappers.rs
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: (BSD-2-Clause OR Apache-2.0) OR MIT
-
+//
 // Copyright 2023 The Fuchsia Authors
 //
 // Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
diff --git a/samples/damon/Kconfig b/samples/damon/Kconfig
index cbf96fd8a8bf9..00be3e6bdd659 100644
--- a/samples/damon/Kconfig
+++ b/samples/damon/Kconfig
@@ -31,7 +31,7 @@ config SAMPLE_DAMON_MTIER
 	bool "DAMON sample module for memory tiering"
 	depends on DAMON && DAMON_PADDR
 	help
-	  Thps builds DAMON sample module for memory tierign.
+	  This builds DAMON sample module for memory tiering.
 
 	  The module assumes the system is constructed with two NUMA nodes,
 	  which seems as local and remote nodes to all CPUs.  For example,
diff --git a/samples/damon/mtier.c b/samples/damon/mtier.c
index 3785b0c7ffb17..ac9c24b92ead8 100644
--- a/samples/damon/mtier.c
+++ b/samples/damon/mtier.c
@@ -177,6 +177,7 @@ static struct damon_ctx *damon_sample_mtier_build_ctx(bool promote)
 static int damon_sample_mtier_start(void)
 {
 	struct damon_ctx *ctx;
+	int err;
 
 	ctx = damon_sample_mtier_build_ctx(true);
 	if (!ctx)
@@ -188,7 +189,13 @@ static int damon_sample_mtier_start(void)
 		return -ENOMEM;
 	}
 	ctxs[1] = ctx;
-	return damon_start(ctxs, 2, true);
+	err = damon_start(ctxs, 2, true);
+	if (!err)
+		return 0;
+
+	damon_destroy_ctx(ctxs[0]);
+	damon_destroy_ctx(ctxs[1]);
+	return err;
 }
 
 static void damon_sample_mtier_stop(void)
diff --git a/samples/damon/prcl.c b/samples/damon/prcl.c
index b7c50f2656ce7..842099bd62286 100644
--- a/samples/damon/prcl.c
+++ b/samples/damon/prcl.c
@@ -106,11 +106,16 @@ static int damon_sample_prcl_start(void)
 	damon_set_schemes(ctx, &scheme, 1);
 
 	err = damon_start(&ctx, 1, true);
-	if (err)
+	if (err) {
+		damon_destroy_ctx(ctx);
 		return err;
+	}
 
 	repeat_call_control.data = ctx;
-	return damon_call(ctx, &repeat_call_control);
+	err = damon_call(ctx, &repeat_call_control);
+	if (err)
+		damon_destroy_ctx(ctx);
+	return err;
 }
 
 static void damon_sample_prcl_stop(void)
diff --git a/samples/damon/wsse.c b/samples/damon/wsse.c
index 799ad44439434..37fd5da201588 100644
--- a/samples/damon/wsse.c
+++ b/samples/damon/wsse.c
@@ -87,10 +87,15 @@ static int damon_sample_wsse_start(void)
 	target->pid = target_pidp;
 
 	err = damon_start(&ctx, 1, true);
-	if (err)
+	if (err) {
+		damon_destroy_ctx(ctx);
 		return err;
+	}
 	repeat_call_control.data = ctx;
-	return damon_call(ctx, &repeat_call_control);
+	err = damon_call(ctx, &repeat_call_control);
+	if (err)
+		damon_destroy_ctx(ctx);
+	return err;
 }
 
 static void damon_sample_wsse_stop(void)
diff --git a/samples/kobject/kobject-example.c b/samples/kobject/kobject-example.c
index 36d87ca0bee24..36f3fbc47fe01 100644
--- a/samples/kobject/kobject-example.c
+++ b/samples/kobject/kobject-example.c
@@ -25,13 +25,13 @@ static int bar;
 /*
  * The "foo" file where a static variable is read from and written to.
  */
-static ssize_t foo_show(struct kobject *kobj, struct kobj_attribute *attr,
+static ssize_t foo_show(struct kobject *kobj, const struct kobj_attribute *attr,
 			char *buf)
 {
 	return sysfs_emit(buf, "%d\n", foo);
 }
 
-static ssize_t foo_store(struct kobject *kobj, struct kobj_attribute *attr,
+static ssize_t foo_store(struct kobject *kobj, const struct kobj_attribute *attr,
 			 const char *buf, size_t count)
 {
 	int ret;
@@ -44,14 +44,14 @@ static ssize_t foo_store(struct kobject *kobj, struct kobj_attribute *attr,
 }
 
 /* Sysfs attributes cannot be world-writable. */
-static struct kobj_attribute foo_attribute =
-	__ATTR(foo, 0664, foo_show, foo_store);
+static const struct kobj_attribute foo_attribute =
+	__KOBJ_ATTR(foo, 0664, foo_show, foo_store);
 
 /*
  * More complex function where we determine which variable is being accessed by
  * looking at the attribute for the "baz" and "bar" files.
  */
-static ssize_t b_show(struct kobject *kobj, struct kobj_attribute *attr,
+static ssize_t b_show(struct kobject *kobj, const struct kobj_attribute *attr,
 		      char *buf)
 {
 	int var;
@@ -63,7 +63,7 @@ static ssize_t b_show(struct kobject *kobj, struct kobj_attribute *attr,
 	return sysfs_emit(buf, "%d\n", var);
 }
 
-static ssize_t b_store(struct kobject *kobj, struct kobj_attribute *attr,
+static ssize_t b_store(struct kobject *kobj, const struct kobj_attribute *attr,
 		       const char *buf, size_t count)
 {
 	int var, ret;
@@ -79,17 +79,17 @@ static ssize_t b_store(struct kobject *kobj, struct kobj_attribute *attr,
 	return count;
 }
 
-static struct kobj_attribute baz_attribute =
-	__ATTR(baz, 0664, b_show, b_store);
-static struct kobj_attribute bar_attribute =
-	__ATTR(bar, 0664, b_show, b_store);
+static const struct kobj_attribute baz_attribute =
+	__KOBJ_ATTR(baz, 0664, b_show, b_store);
+static const struct kobj_attribute bar_attribute =
+	__KOBJ_ATTR(bar, 0664, b_show, b_store);
 
 
 /*
  * Create a group of attributes so that we can create and destroy them all
  * at once.
  */
-static struct attribute *attrs[] = {
+static const struct attribute *const attrs[] = {
 	&foo_attribute.attr,
 	&baz_attribute.attr,
 	&bar_attribute.attr,
@@ -103,7 +103,7 @@ static struct attribute *attrs[] = {
  * attribute group.
  */
 static const struct attribute_group attr_group = {
-	.attrs = attrs,
+	.attrs_const = attrs,
 };
 
 static struct kobject *example_kobj;
diff --git a/samples/rust/rust_dma.rs b/samples/rust/rust_dma.rs
index 5046b4628d0e2..b629acc6d9152 100644
--- a/samples/rust/rust_dma.rs
+++ b/samples/rust/rust_dma.rs
@@ -12,6 +12,11 @@
         Device,
         DmaMask, //
     },
+    io::{
+        io_project,
+        io_read,
+        Io, //
+    },
     page, pci,
     prelude::*,
     scatterlist::{Owned, SGTable},
@@ -34,6 +39,7 @@ struct DmaSampleDriver {
     (0xcd, 0xef),
 ];
 
+#[derive(FromBytes, IntoBytes)]
 struct MyStruct {
     h: u32,
     b: u32,
@@ -77,7 +83,7 @@ fn probe<'bound>(
                 Coherent::zeroed_slice(pdev.as_ref(), TEST_VALUES.len(), GFP_KERNEL)?;
 
             for (i, value) in TEST_VALUES.into_iter().enumerate() {
-                kernel::dma_write!(ca, [try: i], MyStruct::new(value.0, value.1));
+                io_project!(ca, [panic: i]).copy_write(MyStruct::new(value.0, value.1));
             }
 
             let size = 4 * page::PAGE_SIZE;
@@ -97,8 +103,8 @@ fn probe<'bound>(
 impl DmaSampleDriver {
     fn check_dma(&self) {
         for (i, value) in TEST_VALUES.into_iter().enumerate() {
-            let val0 = kernel::dma_read!(self.ca, [panic: i].h);
-            let val1 = kernel::dma_read!(self.ca, [panic: i].b);
+            let val0 = io_read!(self.ca, [panic: i].h);
+            let val1 = io_read!(self.ca, [panic: i].b);
 
             assert_eq!(val0, value.0);
             assert_eq!(val1, value.1);
diff --git a/samples/rust/rust_driver_auxiliary.rs b/samples/rust/rust_driver_auxiliary.rs
index 2c1351040e45e..e441ae81fa2cf 100644
--- a/samples/rust/rust_driver_auxiliary.rs
+++ b/samples/rust/rust_driver_auxiliary.rs
@@ -11,14 +11,21 @@
         Core, //
     },
     driver,
+    new_mutex,
     pci,
     prelude::*,
-    types::ForLt,
+    sync::Mutex,
+    types::{
+        CovariantForLt,
+        ForLt, //
+    },
     InPlaceModule, //
 };
 
 const MODULE_NAME: &CStr = <LocalModule as kernel::ModuleMetadata>::NAME;
 const AUXILIARY_NAME: &CStr = c"auxiliary";
+const COVARIANT_DEV_ID: u32 = 0;
+const INVARIANT_DEV_ID: u32 = 1;
 
 struct AuxiliaryDriver;
 
@@ -56,12 +63,26 @@ struct Data<'bound> {
     parent: &'bound pci::Device<Bound>,
 }
 
+/// Registration data with interior mutability.
+///
+/// `Mutex<&'bound T>` is invariant over `'bound`, so this type cannot implement
+/// [`CovariantForLt`](trait@CovariantForLt). Access must go through the closure-based
+/// [`auxiliary::Device::registration_data_with()`].
+#[pin_data]
+struct MutexData<'bound> {
+    #[pin]
+    parent: Mutex<&'bound pci::Device<Bound>>,
+    index: u32,
+}
+
 struct ParentDriver;
 
 #[allow(clippy::type_complexity)]
+#[pin_data]
 struct ParentData<'bound> {
-    _reg0: auxiliary::Registration<'bound, ForLt!(Data<'_>)>,
-    _reg1: auxiliary::Registration<'bound, ForLt!(Data<'_>)>,
+    _reg0: auxiliary::Registration<'bound, CovariantForLt!(Data<'_>)>,
+    #[pin]
+    _reg1: auxiliary::Registration<'bound, ForLt!(MutexData<'_>)>,
 }
 
 kernel::pci_device_table!(
@@ -81,17 +102,17 @@ fn probe<'bound>(
         pdev: &'bound pci::Device<Core<'_>>,
         _info: &'bound Self::IdInfo,
     ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound {
-        Ok(ParentData {
+        try_pin_init!(ParentData {
             // SAFETY: `ParentData` is the driver's private data, which is dropped when the
             // device is unbound; i.e. `mem::forget()` is never called on it.
             _reg0: unsafe {
                 auxiliary::Registration::new_with_lt(
                     pdev.as_ref(),
                     AUXILIARY_NAME,
-                    0,
+                    COVARIANT_DEV_ID,
                     MODULE_NAME,
                     Data {
-                        index: 0,
+                        index: COVARIANT_DEV_ID,
                         parent: pdev,
                     },
                 )?
@@ -101,12 +122,16 @@ fn probe<'bound>(
                 auxiliary::Registration::new_with_lt(
                     pdev.as_ref(),
                     AUXILIARY_NAME,
-                    1,
+                    INVARIANT_DEV_ID,
                     MODULE_NAME,
-                    Data {
-                        index: 1,
-                        parent: pdev,
-                    },
+                    pin_init!(MutexData {
+                        parent <- {
+                            let pdev: &pci::Device<Bound> = pdev;
+
+                            new_mutex!(pdev)
+                        },
+                        index: INVARIANT_DEV_ID,
+                    }),
                 )?
             },
         })
@@ -115,22 +140,39 @@ fn probe<'bound>(
 
 impl ParentDriver {
     fn connect(adev: &auxiliary::Device<Bound>) -> Result {
-        let data = adev.registration_data::<ForLt!(Data<'_>)>()?;
-        let pdev = data.parent;
-
-        dev_info!(
-            pdev,
-            "Connect auxiliary {} with parent: VendorID={}, DeviceID={:#x}\n",
-            adev.id(),
-            pdev.vendor_id(),
-            pdev.device_id()
-        );
-
-        dev_info!(
-            pdev,
-            "Connected to auxiliary device with index {}.\n",
-            data.index
-        );
+        match adev.id() {
+            // CovariantForLt types can use the direct-reference accessor.
+            COVARIANT_DEV_ID => {
+                let data = adev.registration_data::<CovariantForLt!(Data<'_>)>()?;
+                let pdev = data.parent;
+
+                dev_info!(
+                    pdev,
+                    "Connect auxiliary {} with parent: VendorID={}, DeviceID={:#x}\n",
+                    adev.id(),
+                    pdev.vendor_id(),
+                    pdev.device_id()
+                );
+
+                dev_info!(
+                    pdev,
+                    "Connected to auxiliary device with index {}.\n",
+                    data.index
+                );
+            }
+            // Invariant ForLt types (e.g. containing a Mutex) require the closure-based accessor.
+            INVARIANT_DEV_ID => {
+                adev.registration_data_with::<ForLt!(MutexData<'_>), _>(|data| {
+                    let pdev = *data.parent.lock();
+                    dev_info!(
+                        pdev,
+                        "Connected to auxiliary device with index {} (via Mutex).\n",
+                        data.index
+                    );
+                })?;
+            }
+            _ => return Err(EINVAL),
+        }
 
         Ok(())
     }
diff --git a/samples/rust/rust_driver_faux.rs b/samples/rust/rust_driver_faux.rs
index 99876c8e3743f..27b6d3e2bb446 100644
--- a/samples/rust/rust_driver_faux.rs
+++ b/samples/rust/rust_driver_faux.rs
@@ -25,8 +25,9 @@ fn init(_module: &'static ThisModule) -> Result<Self> {
         pr_info!("Initialising Rust Faux Device Sample\n");
 
         let reg = faux::Registration::new(c"rust-faux-sample-device", None)?;
+        let fdev = reg.as_ref();
 
-        dev_info!(reg, "Hello from faux device!\n");
+        dev_info!(fdev, "Hello from faux device!\n");
 
         Ok(Self { _reg: reg })
     }
diff --git a/samples/rust/rust_minimal.rs b/samples/rust/rust_minimal.rs
index 8eb9583571d72..8c910d314dfab 100644
--- a/samples/rust/rust_minimal.rs
+++ b/samples/rust/rust_minimal.rs
@@ -15,6 +15,10 @@
             default: 1,
             description: "This parameter has a default of 1",
         },
+        test_bool_parameter: bool {
+            default: false,
+            description: "This boolean parameter defaults to false",
+        },
     },
 }
 
@@ -28,7 +32,11 @@ fn init(_module: &'static ThisModule) -> Result<Self> {
         pr_info!("Am I built-in? {}\n", !cfg!(MODULE));
         pr_info!(
             "test_parameter: {}\n",
-            *module_parameters::test_parameter.value()
+            module_parameters::test_parameter.value()
+        );
+        pr_info!(
+            "test_bool_parameter: {}\n",
+            module_parameters::test_bool_parameter.value()
         );
 
         let mut numbers = KVec::new();
diff --git a/samples/trace_events/trace-events-sample.c b/samples/trace_events/trace-events-sample.c
index 0b7a6efdb247a..ca5d98c360cb1 100644
--- a/samples/trace_events/trace-events-sample.c
+++ b/samples/trace_events/trace-events-sample.c
@@ -94,6 +94,20 @@ static int simple_thread_fn(void *arg)
 static DEFINE_MUTEX(thread_mutex);
 static int simple_thread_cnt;
 
+static struct foo_timer_data *foo_timer_data;
+
+static void sample_timer_cb(struct timer_list *t)
+{
+	struct foo_timer_data *data = container_of(t, struct foo_timer_data, timer);
+
+	get_cpu();
+	trace_foo_timer_fn(data);
+	(*this_cpu_ptr(data->counter))++;
+	put_cpu();
+
+	mod_timer(t, jiffies + HZ);
+}
+
 int foo_bar_reg(void)
 {
 	mutex_lock(&thread_mutex);
@@ -132,9 +146,27 @@ void foo_bar_unreg(void)
 
 static int __init trace_event_init(void)
 {
+	foo_timer_data = kzalloc_obj(*foo_timer_data, GFP_KERNEL);
+	if (!foo_timer_data)
+		return -ENOMEM;
+
+	foo_timer_data->name = "sample_timer_counter";
+	foo_timer_data->counter = alloc_percpu(int);
+	if (!foo_timer_data->counter) {
+		kfree(foo_timer_data);
+		return -ENOMEM;
+	}
+
+	timer_setup(&foo_timer_data->timer, sample_timer_cb, 0);
+	mod_timer(&foo_timer_data->timer, jiffies + HZ);
+
 	simple_tsk = kthread_run(simple_thread, NULL, "event-sample");
-	if (IS_ERR(simple_tsk))
-		return -1;
+	if (IS_ERR(simple_tsk)) {
+		timer_shutdown_sync(&foo_timer_data->timer);
+		free_percpu(foo_timer_data->counter);
+		kfree(foo_timer_data);
+		return PTR_ERR(simple_tsk);
+	}
 
 	return 0;
 }
@@ -147,6 +179,10 @@ static void __exit trace_event_exit(void)
 		kthread_stop(simple_tsk_fn);
 	simple_tsk_fn = NULL;
 	mutex_unlock(&thread_mutex);
+
+	timer_shutdown_sync(&foo_timer_data->timer);
+	free_percpu(foo_timer_data->counter);
+	kfree(foo_timer_data);
 }
 
 module_init(trace_event_init);
diff --git a/samples/trace_events/trace-events-sample.h b/samples/trace_events/trace-events-sample.h
index 1a05fc1533531..816848a456a2b 100644
--- a/samples/trace_events/trace-events-sample.h
+++ b/samples/trace_events/trace-events-sample.h
@@ -247,12 +247,14 @@
  */
 
 /*
- * It is OK to have helper functions in the file, but they need to be protected
- * from being defined more than once. Remember, this file gets included more
- * than once.
+ * It is OK to have helper functions and data structures in the file, but they
+ * need to be protected from being defined more than once. Remember, this file
+ * gets included more than once.
  */
 #ifndef __TRACE_EVENT_SAMPLE_HELPER_FUNCTIONS
 #define __TRACE_EVENT_SAMPLE_HELPER_FUNCTIONS
+#include <linux/timer.h>
+
 static inline int __length_of(const int *list)
 {
 	int i;
@@ -270,6 +272,13 @@ enum {
 	TRACE_SAMPLE_BAR = 4,
 	TRACE_SAMPLE_ZOO = 8,
 };
+
+struct foo_timer_data {
+	const char		*name;
+	struct timer_list	timer;
+	int __percpu		*counter;
+};
+
 #endif
 
 /*
@@ -595,6 +604,25 @@ TRACE_EVENT(foo_rel_loc,
 		  __get_rel_bitmask(bitmask),
 		  __get_rel_cpumask(cpumask))
 );
+
+TRACE_EVENT(foo_timer_fn,
+
+	TP_PROTO(struct foo_timer_data *data),
+
+	TP_ARGS(data),
+
+	TP_STRUCT__entry(
+		__string(	name,			data->name	)
+		__field(	int,			count		)
+	),
+
+	TP_fast_assign(
+		__assign_str(name);
+		__entry->count	= *this_cpu_ptr(data->counter);
+	),
+
+	TP_printk("name=%s count=%d", __get_str(name), __entry->count)
+);
 #endif
 
 /***** NOTICE! The #if protection ends here. *****/
diff --git a/scripts/Makefile.build b/scripts/Makefile.build
index 9117457432463..d432693e53674 100644
--- a/scripts/Makefile.build
+++ b/scripts/Makefile.build
@@ -159,10 +159,10 @@ targets += $(targets-for-builtin) $(targets-for-modules)
 
 # Linus' kernel sanity checking tool
 ifeq ($(KBUILD_CHECKSRC),1)
-  quiet_cmd_checksrc       = CHECK   $<
+  quiet_cmd_checksrc       = CHECK   $(patsubst $(srctree)/%,%,$<)
         cmd_checksrc       = $(CHECK) $(CHECKFLAGS) $(c_flags) $<
 else ifeq ($(KBUILD_CHECKSRC),2)
-  quiet_cmd_force_checksrc = CHECK   $<
+  quiet_cmd_force_checksrc = CHECK   $(patsubst $(srctree)/%,%,$<)
         cmd_force_checksrc = $(CHECK) $(CHECKFLAGS) $(c_flags) $<
 endif
 
diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl
index 2b7a42bbdd94f..7a846a3ea1273 100755
--- a/scripts/checkpatch.pl
+++ b/scripts/checkpatch.pl
@@ -3781,6 +3781,12 @@ sub process {
 			my $vp_file = $dt_path . "vendor-prefixes.yaml";
 
 			foreach my $compat (@compats) {
+				# Skip ID-based PCI and USB compatible patterns.
+				# DT validation will check them properly.
+				next if $compat =~ /^pciclass,/;
+				next if $compat =~ /^pci[a-f0-9]{2,4},/;
+				next if $compat =~ /^usb(if)?[a-f0-9]{1,4},/;
+
 				my $compat2 = $compat;
 				$compat2 =~ s/\,[a-zA-Z0-9]*\-/\,<\.\*>\-/;
 				my $compat3 = $compat;
diff --git a/scripts/gdb/linux/page_owner.py b/scripts/gdb/linux/page_owner.py
index 8e713a09cfe78..eeabaeed438b2 100644
--- a/scripts/gdb/linux/page_owner.py
+++ b/scripts/gdb/linux/page_owner.py
@@ -34,6 +34,7 @@ class DumpPageOwner(gdb.Command):
     max_pfn = None
     p_ops = None
     migrate_reason_names = None
+    mr_never = None
 
     def __init__(self):
         super(DumpPageOwner, self).__init__("lx-dump-page-owner", gdb.COMMAND_SUPPORT)
@@ -65,6 +66,7 @@ class DumpPageOwner(gdb.Command):
         self.max_pfn = int(gdb.parse_and_eval("max_pfn"))
         self.page_ext_size = int(gdb.parse_and_eval("page_ext_size"))
         self.migrate_reason_names = gdb.parse_and_eval('migrate_reason_names')
+        self.mr_never = int(gdb.parse_and_eval('MR_NEVER'))
 
     def page_ext_invalid(self, page_ext):
         if page_ext == gdb.Value(0):
@@ -138,7 +140,7 @@ class DumpPageOwner(gdb.Command):
         else:
             gdb.write('page last free stack trace:\n')
             stackdepot.stack_depot_print(page_owner["free_handle"])
-        if page_owner['last_migrate_reason'] != -1:
+        if page_owner['last_migrate_reason'] != self.mr_never:
             gdb.write('page has been migrated, last migrate reason: %s\n' % self.migrate_reason_names[page_owner['last_migrate_reason']])
 
     def read_page_owner(self):
diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py
index dc1219736f77a..621895bae128f 100755
--- a/scripts/generate_rust_analyzer.py
+++ b/scripts/generate_rust_analyzer.py
@@ -250,6 +250,7 @@ def generate_crates(
         "macros",
         srctree / "rust" / "macros" / "lib.rs",
         [std, proc_macro, proc_macro2, quote, syn],
+        cfg=generated_cfg,
     )
 
     zerocopy_derive = append_proc_macro_crate(
diff --git a/scripts/headers_install.sh b/scripts/headers_install.sh
index 9c15e748761c4..83e4475968781 100755
--- a/scripts/headers_install.sh
+++ b/scripts/headers_install.sh
@@ -36,6 +36,7 @@ sed -E -e '
 	s/(^|[^a-zA-Z0-9])__packed([^a-zA-Z0-9_]|$)/\1__attribute__((packed))\2/g
 	s/(^|[[:space:](])(inline|asm|volatile)([[:space:](]|$)/\1__\2__\3/g
 	s@#(ifndef|define|endif[[:space:]]*/[*])[[:space:]]*_UAPI@#\1 @
+	s/__ASSEMBLER__/__ASSEMBLY__/g
 ' $INFILE > $TMPFILE || exit 1
 
 scripts/unifdef -U__KERNEL__ -D__EXPORTED_HEADERS__ $TMPFILE > $OUTFILE
diff --git a/scripts/kconfig/merge_config.sh b/scripts/kconfig/merge_config.sh
index f08e0863b7128..ec242e03f509f 100755
--- a/scripts/kconfig/merge_config.sh
+++ b/scripts/kconfig/merge_config.sh
@@ -122,7 +122,7 @@ fi
 
 MERGE_LIST=$*
 
-TMP_FILE=$(mktemp ./.tmp.config.XXXXXXXXXX)
+TMP_FILE=$(mktemp --tmpdir="$OUTPUT" .tmp.config.XXXXXXXXXX)
 
 echo "Using $INITFILE as base"
 
diff --git a/scripts/module.lds.S b/scripts/module.lds.S
index b62683061d79b..d0f2004289576 100644
--- a/scripts/module.lds.S
+++ b/scripts/module.lds.S
@@ -3,11 +3,6 @@
  * Archs are free to supply their own linker scripts.  ld will
  * combine them automatically.
  */
-#ifdef CONFIG_UNWIND_TABLES
-#define DISCARD_EH_FRAME
-#else
-#define DISCARD_EH_FRAME	*(.eh_frame)
-#endif
 
 #include <asm-generic/codetag.lds.h>
 
diff --git a/scripts/tags.sh b/scripts/tags.sh
index 243373683f98a..41e38df969840 100755
--- a/scripts/tags.sh
+++ b/scripts/tags.sh
@@ -46,13 +46,31 @@ elif [ "${ALLSOURCE_ARCHS}" = "all" ]; then
 	ALLSOURCE_ARCHS=$(find ${tree}arch/ -mindepth 1 -maxdepth 1 -type d -printf '%f ')
 fi
 
+setup_name_pattern()
+{
+	pattern=()
+	for ext; do
+		if [ ${#pattern[@]} -gt 0 ]; then
+			pattern+=("-o" "-name" "$ext")
+		else
+			pattern+=("(" "-name" "$ext")
+		fi
+	done
+	if [ ${#pattern[@]} -gt 0 ]; then
+		pattern+=(")")
+	fi
+}
+
 # find sources in arch/$1
 find_arch_sources()
 {
 	for i in $archincludedir; do
 		local prune="$prune ( -path $i ) -prune -o"
 	done
-	find ${tree}arch/$1 $ignore $prune -name "$2" -not -type l -print;
+	local src=${tree}arch/$1
+	shift
+	setup_name_pattern "$@"
+	find $src $ignore $prune "${pattern[@]}" -not -type l -print;
 }
 
 # find sources in arch/$1/include
@@ -61,14 +79,17 @@ find_arch_include_sources()
 	local include=$(find ${tree}arch/$1/ -name include -type d -print);
 	if [ -n "$include" ]; then
 		archincludedir="$archincludedir $include"
-		find $include $ignore -name "$2" -not -type l -print;
+		shift
+		setup_name_pattern "$@"
+		find $include $ignore "${pattern[@]}" -not -type l -print;
 	fi
 }
 
 # find sources in include/
 find_include_sources()
 {
-	find ${tree}include $ignore -name config -prune -o -name "$1" \
+	setup_name_pattern "$@"
+	find ${tree}include $ignore -name config -prune -o "${pattern[@]}" \
 		-not -type l -print;
 }
 
@@ -76,23 +97,24 @@ find_include_sources()
 # we could benefit from a list of dirs to search in here
 find_other_sources()
 {
+	setup_name_pattern "$@"
 	find ${tree}* $ignore \
 	     \( -path ${tree}include -o -path ${tree}arch -o -name '.tmp_*' \) -prune -o \
-	       -name "$1" -not -type l -print;
+	       "${pattern[@]}" -not -type l -print;
 }
 
 all_sources()
 {
-	find_arch_include_sources ${SRCARCH} '*.[chS]'
+	find_arch_include_sources ${SRCARCH} '*.[chS]' '*.rs'
 	if [ -n "$archinclude" ]; then
-		find_arch_include_sources $archinclude '*.[chS]'
+		find_arch_include_sources $archinclude '*.[chS]' '*.rs'
 	fi
-	find_include_sources '*.[chS]'
+	find_include_sources '*.[chS]' '*.rs'
 	for arch in $ALLSOURCE_ARCHS
 	do
-		find_arch_sources $arch '*.[chS]'
+		find_arch_sources $arch '*.[chS]' '*.rs'
 	done
-	find_other_sources '*.[chS]'
+	find_other_sources '*.[chS]' '*.rs'
 }
 
 all_compiled_sources()
@@ -100,7 +122,7 @@ all_compiled_sources()
 	{
 		echo include/generated/autoconf.h
 		find $ignore -name "*.cmd" -exec \
-			grep -Poh '(?<=^  )\S+|(?<== )\S+[^\\](?=$)' {} \+ |
+			grep -Poh '(?<=^  )\S+\.([chS]|rs)(?=\s)|(?<== )\S+\.(?1)(?=$)' {} \+ |
 		awk '!a[$0]++'
 	} | xargs realpath -esq $([ -z "$KBUILD_ABS_SRCTREE" ] && echo --relative-to=.) |
 	sort -u
diff --git a/security/apparmor/af_unix.c b/security/apparmor/af_unix.c
index b9b22edae2029..9ca9d1b890ba6 100644
--- a/security/apparmor/af_unix.c
+++ b/security/apparmor/af_unix.c
@@ -20,6 +20,7 @@
 #include "include/apparmor.h"
 #include "include/file.h"
 #include "include/label.h"
+#include "include/net.h"
 #include "include/path.h"
 #include "include/policy.h"
 #include "include/cred.h"
diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c
index 56155d7d5b2fc..152c7967ff1ba 100644
--- a/security/apparmor/apparmorfs.c
+++ b/security/apparmor/apparmorfs.c
@@ -23,6 +23,7 @@
 #include <linux/fs_context.h>
 #include <linux/poll.h>
 #include <linux/zstd.h>
+#include <linux/string.h>
 #include <uapi/linux/major.h>
 #include <uapi/linux/magic.h>
 
@@ -33,6 +34,7 @@
 #include "include/crypto.h"
 #include "include/ipc.h"
 #include "include/label.h"
+#include "include/net.h"
 #include "include/lib.h"
 #include "include/policy.h"
 #include "include/policy_ns.h"
@@ -482,6 +484,113 @@ static struct aa_loaddata *aa_simple_write_to_buffer(const char __user *userbuf,
 	return data;
 }
 
+static int decompress_zstd(char *src, size_t slen, char *dst, size_t dlen)
+{
+	if (slen < dlen) {
+		const size_t wksp_len = zstd_dctx_workspace_bound();
+		zstd_dctx *ctx;
+		void *wksp;
+		size_t out_len;
+		int ret = 0;
+
+		wksp = kvzalloc(wksp_len, GFP_KERNEL);
+		if (!wksp) {
+			ret = -ENOMEM;
+			goto cleanup;
+		}
+		ctx = zstd_init_dctx(wksp, wksp_len);
+		if (ctx == NULL) {
+			ret = -ENOMEM;
+			goto cleanup;
+		}
+		out_len = zstd_decompress_dctx(ctx, dst, dlen, src, slen);
+		if (zstd_is_error(out_len)) {
+			ret = -EINVAL;
+			goto cleanup;
+		}
+cleanup:
+		kvfree(wksp);
+		return ret;
+	}
+
+	if (dlen < slen)
+		return -EINVAL;
+	memcpy(dst, src, slen);
+	return 0;
+}
+
+/**
+ * aa_get_data_from_compressed - common routine for getting compressed policy
+ * from user and get both compressed and uncompressed version.
+ * @userbuf: user buffer to copy data from  (NOT NULL)
+ * @buffer_size: size of user buffer
+ * @pos: position write is at in the file (NOT NULL)
+ * @compressed_data Ptr on compressed data. *compressed_data is allocated there
+ *
+ * Returns: kernel buffer containing copy of user buffer data or an
+ *          ERR_PTR on failure.
+ */
+
+static struct aa_loaddata *aa_get_data_from_compressed(const char __user *userbuf,
+						  size_t buffer_size,
+						  loff_t *pos,
+						  char **compressed_data)
+{
+	struct aa_loaddata *data;
+	zstd_frame_header header;
+	int error;
+
+	if (!userbuf || !pos)
+		return ERR_PTR(-EINVAL);
+	if (*pos)
+		return ERR_PTR(-ESPIPE);
+
+	*compressed_data = kvmalloc(buffer_size, GFP_KERNEL);
+	if (!*compressed_data)
+		return ERR_PTR(-ENOMEM);
+	error = copy_from_user(*compressed_data, userbuf, buffer_size);
+	if (error)
+		goto fail;
+
+	error = zstd_get_frame_header(&header, *compressed_data, buffer_size);
+	if (error || header.frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN ||
+	    header.frameContentSize == ZSTD_CONTENTSIZE_ERROR) {
+		error = -EINVAL;
+		goto fail;
+	}
+
+	data = aa_loaddata_alloc(header.frameContentSize);
+	if (IS_ERR(data)) {
+		error = PTR_ERR(data);
+		goto fail;
+	}
+
+	// We then decompress the data
+	error = decompress_zstd(*compressed_data, buffer_size, data->data,
+				header.frameContentSize);
+	if (error)
+		goto fail_decompress;
+
+	data->size = header.frameContentSize;
+	return data;
+
+fail_decompress:
+	aa_put_i_loaddata(data);
+fail:
+	kvfree(*compressed_data);
+	return ERR_PTR(error);
+
+}
+
+struct aa_user_hdr {
+	uint8_t version;
+	uint8_t compress_level;
+	uint8_t padding[6];	/* force 8-byte alignment */
+} __packed __aligned(8);
+
+#define aa_hdr_magic "\x04\x08\x00\x76\x65\x72\x73\x69\x6f\x6e\x00\x02"
+#define aa_hdr_magic_size 12
+
 static ssize_t policy_update(u32 mask, const char __user *buf, size_t size,
 			     loff_t *pos, struct aa_ns *ns,
 			     const struct cred *ocred)
@@ -489,6 +598,10 @@ static ssize_t policy_update(u32 mask, const char __user *buf, size_t size,
 	struct aa_loaddata *data;
 	struct aa_label *label;
 	ssize_t error;
+	char *compressed_data = NULL;
+	__le32 magic_le;
+	bool is_compressed;
+	u8 aahdr[aa_hdr_magic_size];
 
 	label = begin_current_label_crit_section();
 
@@ -499,10 +612,52 @@ static ssize_t policy_update(u32 mask, const char __user *buf, size_t size,
 	if (error)
 		goto end_section;
 
-	data = aa_simple_write_to_buffer(buf, size, size, pos);
-	error = PTR_ERR(data);
+	/* If the policy is userspace compressed we start by decompressing it
+	 * to make the required checks (computing hash, verifying profile, ...)
+	 *
+	 * Getting a userspace-compressed version then decompressing it in the
+	 * kernel actually makes sense since zstd decompression is ~3.5x faster
+	 * than compression. This also allow to increase the compression level.
+	 */
+
+	if (size >= sizeof(__le32) &&
+	    !copy_from_user(&magic_le, buf, sizeof(magic_le)) &&
+	    le32_to_cpu(magic_le) == ZSTD_MAGICNUMBER) {
+		is_compressed = true;
+	} else if (size >= sizeof(struct aa_user_hdr) + sizeof(__le32) &&
+		   !copy_from_user(&magic_le,
+				   buf + sizeof(struct aa_user_hdr),
+				   sizeof(magic_le)) &&
+		   le32_to_cpu(magic_le) == ZSTD_MAGICNUMBER) {
+		is_compressed = true;
+		/* skip the userspace header if present */
+		buf += sizeof(struct aa_user_hdr);
+		size -= sizeof(struct aa_user_hdr);
+	} else if (size >= sizeof(struct aa_user_hdr) +
+		   aa_hdr_magic_size &&
+		   !copy_from_user(&aahdr,
+				   buf + sizeof(struct aa_user_hdr),
+				   aa_hdr_magic_size) &&
+		   memcmp(&aahdr, aa_hdr_magic, aa_hdr_magic_size) == 0) {
+		/* uncompressed blob with user hdr */
+		buf += sizeof(struct aa_user_hdr);
+		size -= sizeof(struct aa_user_hdr);
+		is_compressed = false;
+	} else {
+		is_compressed = false;
+	}
+	if (is_compressed) {
+
+		data = aa_get_data_from_compressed(buf, size, pos, &compressed_data);
+		error = PTR_ERR(data);
+	} else {
+		data = aa_simple_write_to_buffer(buf, size, size, pos);
+		error = PTR_ERR(data);
+	}
+
 	if (!IS_ERR(data)) {
-		error = aa_replace_profiles(ns, label, mask, data);
+		error = aa_replace_profiles(ns, label, mask, data,
+					    compressed_data, size);
 		/* put pcount, which will put count and free if no
 		 * profiles referencing it.
 		 */
@@ -549,6 +704,7 @@ static const struct file_operations aa_fs_profile_replace = {
 	.llseek = default_llseek,
 };
 
+
 /* .remove file hook fn to remove loaded policy */
 static ssize_t profile_remove(struct file *f, const char __user *buf,
 			      size_t size, loff_t *pos)
@@ -1396,41 +1552,6 @@ SEQ_RAWDATA_FOPS(revision);
 SEQ_RAWDATA_FOPS(hash);
 SEQ_RAWDATA_FOPS(compressed_size);
 
-static int decompress_zstd(char *src, size_t slen, char *dst, size_t dlen)
-{
-	if (slen < dlen) {
-		const size_t wksp_len = zstd_dctx_workspace_bound();
-		zstd_dctx *ctx;
-		void *wksp;
-		size_t out_len;
-		int ret = 0;
-
-		wksp = kvzalloc(wksp_len, GFP_KERNEL);
-		if (!wksp) {
-			ret = -ENOMEM;
-			goto cleanup;
-		}
-		ctx = zstd_init_dctx(wksp, wksp_len);
-		if (ctx == NULL) {
-			ret = -ENOMEM;
-			goto cleanup;
-		}
-		out_len = zstd_decompress_dctx(ctx, dst, dlen, src, slen);
-		if (zstd_is_error(out_len)) {
-			ret = -EINVAL;
-			goto cleanup;
-		}
-cleanup:
-		kvfree(wksp);
-		return ret;
-	}
-
-	if (dlen < slen)
-		return -EINVAL;
-	memcpy(dst, src, slen);
-	return 0;
-}
-
 static ssize_t rawdata_read(struct file *file, char __user *buf, size_t size,
 			    loff_t *ppos)
 {
@@ -2483,6 +2604,8 @@ static struct aa_sfs_entry aa_sfs_entry_policy[] = {
 	AA_SFS_FILE_STRING("permstable32", PERMS32STR),
 	AA_SFS_FILE_U64("state32",	1),
 	AA_SFS_DIR("unconfined_restrictions",   aa_sfs_entry_unconfined),
+	AA_SFS_FILE_BOOLEAN("compressed_load",	1),
+	AA_SFS_FILE_BOOLEAN("extended_policy_header",	1),
 	{ }
 };
 
diff --git a/security/apparmor/domain.c b/security/apparmor/domain.c
index d6958eb00e304..4154964cdac51 100644
--- a/security/apparmor/domain.c
+++ b/security/apparmor/domain.c
@@ -24,6 +24,7 @@
 #include "include/domain.h"
 #include "include/file.h"
 #include "include/ipc.h"
+#include "include/lib.h"
 #include "include/match.h"
 #include "include/path.h"
 #include "include/policy.h"
diff --git a/security/apparmor/include/apparmorfs.h b/security/apparmor/include/apparmorfs.h
index 33243d11fd106..c6c8fcde728fe 100644
--- a/security/apparmor/include/apparmorfs.h
+++ b/security/apparmor/include/apparmorfs.h
@@ -11,6 +11,9 @@
 #ifndef __AA_APPARMORFS_H
 #define __AA_APPARMORFS_H
 
+#include <linux/init.h>
+#include <linux/types.h>
+
 extern struct path aa_null;
 
 enum aa_sfs_type {
diff --git a/security/apparmor/include/capability.h b/security/apparmor/include/capability.h
index 1ddcec2d1160d..3fb8b8f2182a7 100644
--- a/security/apparmor/include/capability.h
+++ b/security/apparmor/include/capability.h
@@ -11,6 +11,7 @@
 #ifndef __AA_CAPABILITY_H
 #define __AA_CAPABILITY_H
 
+#include <linux/capability.h>
 #include <linux/sched.h>
 
 #include "apparmorfs.h"
diff --git a/security/apparmor/include/path.h b/security/apparmor/include/path.h
index 8bb915d48dc7f..250812a77e118 100644
--- a/security/apparmor/include/path.h
+++ b/security/apparmor/include/path.h
@@ -11,6 +11,9 @@
 #ifndef __AA_PATH_H
 #define __AA_PATH_H
 
+#include <linux/path.h>
+#include <linux/types.h>
+
 enum path_flags {
 	PATH_IS_DIR = 0x1,		/* path is a directory */
 	PATH_SOCK_COND = 0x2,
diff --git a/security/apparmor/include/policy.h b/security/apparmor/include/policy.h
index 3895f8774a3f9..240b2eba7687d 100644
--- a/security/apparmor/include/policy.h
+++ b/security/apparmor/include/policy.h
@@ -26,7 +26,6 @@
 #include "file.h"
 #include "lib.h"
 #include "label.h"
-#include "net.h"
 #include "perms.h"
 #include "resource.h"
 
@@ -305,7 +304,8 @@ struct aa_profile *aa_fqlookupn_profile(struct aa_label *base,
 					const char *fqname, size_t n);
 
 ssize_t aa_replace_profiles(struct aa_ns *view, struct aa_label *label,
-			    u32 mask, struct aa_loaddata *udata);
+			    u32 mask, struct aa_loaddata *udata,
+			    char *compressed_profile, size_t compressed_size);
 ssize_t aa_remove_profiles(struct aa_ns *view, struct aa_label *label,
 			   char *name, size_t size);
 void __aa_profile_list_release(struct list_head *head);
diff --git a/security/apparmor/include/policy_unpack.h b/security/apparmor/include/policy_unpack.h
index 4ea9b6479a3ed..c01f6885dbe3a 100644
--- a/security/apparmor/include/policy_unpack.h
+++ b/security/apparmor/include/policy_unpack.h
@@ -16,6 +16,7 @@
 #include <linux/dcache.h>
 #include <linux/workqueue.h>
 
+#include "lib.h"
 
 struct aa_load_ent {
 	struct list_head list;
@@ -128,7 +129,8 @@ struct aa_loaddata {
 	char *data;
 };
 
-int aa_unpack(struct aa_loaddata *udata, struct list_head *lh, const char **ns);
+int aa_unpack(struct aa_loaddata *udata, struct list_head *lh, const char **ns,
+	      char *compressed_data, size_t compressed_size);
 
 /**
  * aa_get_i_loaddata - get a reference count from a counted data reference
diff --git a/security/apparmor/include/procattr.h b/security/apparmor/include/procattr.h
index 03dbfdb2f2c09..56acd1bdb6340 100644
--- a/security/apparmor/include/procattr.h
+++ b/security/apparmor/include/procattr.h
@@ -11,6 +11,8 @@
 #ifndef __AA_PROCATTR_H
 #define __AA_PROCATTR_H
 
+#include "label.h"
+
 int aa_getprocattr(struct aa_label *label, char **string, bool newline);
 int aa_setprocattr_changehat(char *args, size_t size, int flags);
 
diff --git a/security/apparmor/include/task.h b/security/apparmor/include/task.h
index b1aaaf60fa8b8..017d8b06b8f23 100644
--- a/security/apparmor/include/task.h
+++ b/security/apparmor/include/task.h
@@ -10,6 +10,11 @@
 #ifndef __AA_TASK_H
 #define __AA_TASK_H
 
+#include <linux/sched.h>
+
+#include "audit.h"
+#include "label.h"
+
 static inline struct aa_task_ctx *task_ctx(struct task_struct *task)
 {
 	return task->security + apparmor_blob_sizes.lbs_task;
diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c
index 94b4a7e727cc1..f6f1b72d7c3df 100644
--- a/security/apparmor/policy.c
+++ b/security/apparmor/policy.c
@@ -81,6 +81,7 @@
 #include "include/file.h"
 #include "include/ipc.h"
 #include "include/match.h"
+#include "include/net.h"
 #include "include/path.h"
 #include "include/policy.h"
 #include "include/policy_ns.h"
@@ -1158,6 +1159,8 @@ static struct aa_profile *update_to_newest_parent(struct aa_profile *new)
  * @label: label that is attempting to load/replace policy
  * @mask: permission mask
  * @udata: serialized data stream  (NOT NULL)
+ * @compressed_profile: The userspace-provided compressed profile. May be NULL
+ * @compressed_size: If compressed_data is not NULL, the compressed data size
  *
  * unpack and replace a profile on the profile list and uses of that profile
  * by any task creds via invalidating the old version of the profile, which
@@ -1167,7 +1170,8 @@ static struct aa_profile *update_to_newest_parent(struct aa_profile *new)
  * Returns: size of data consumed else error code on failure.
  */
 ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label,
-			    u32 mask, struct aa_loaddata *udata)
+			    u32 mask, struct aa_loaddata *udata,
+			    char *compressed_profile, size_t compressed_size)
 {
 	const char *ns_name = NULL, *info = NULL;
 	struct aa_ns *ns = NULL;
@@ -1180,7 +1184,7 @@ ssize_t aa_replace_profiles(struct aa_ns *policy_ns, struct aa_label *label,
 	op = mask & AA_MAY_REPLACE_POLICY ? OP_PROF_REPL : OP_PROF_LOAD;
 	aa_get_profile_loaddata(udata);
 	/* released below */
-	error = aa_unpack(udata, &lh, &ns_name);
+	error = aa_unpack(udata, &lh, &ns_name, compressed_profile, compressed_size);
 	if (error)
 		goto out;
 
diff --git a/security/apparmor/policy_unpack.c b/security/apparmor/policy_unpack.c
index d9dcff167c480..5969d78f16af1 100644
--- a/security/apparmor/policy_unpack.c
+++ b/security/apparmor/policy_unpack.c
@@ -25,6 +25,7 @@
 #include "include/crypto.h"
 #include "include/file.h"
 #include "include/match.h"
+#include "include/net.h"
 #include "include/path.h"
 #include "include/policy.h"
 #include "include/policy_unpack.h"
@@ -1717,6 +1718,8 @@ static int compress_loaddata(struct aa_loaddata *data)
  * @udata: user data copied to kmem  (NOT NULL)
  * @lh: list to place unpacked profiles in a aa_repl_ws
  * @ns: Returns namespace profile is in if specified else NULL (NOT NULL)
+ * @compressed_data: The userspace-provided compressed data. May be NULL
+ * @compressed_size: If compressed_data is not NULL, the compressed data size
  *
  * Unpack user data and return refcounted allocated profile(s) stored in
  * @lh in order of discovery, with the list chain stored in base.list
@@ -1725,12 +1728,12 @@ static int compress_loaddata(struct aa_loaddata *data)
  * Returns: profile(s) on @lh else error pointer if fails to unpack
  */
 int aa_unpack(struct aa_loaddata *udata, struct list_head *lh,
-	      const char **ns)
+	      const char **ns, char *compressed_data, size_t compressed_size)
 {
 	struct aa_load_ent *tmp, *ent;
 	struct aa_profile *profile = NULL;
 	char *ns_name = NULL;
-	int error;
+	int error = 0;
 	struct aa_ext e = {
 		.start = udata->data,
 		.end = udata->data + udata->size,
@@ -1783,10 +1786,23 @@ int aa_unpack(struct aa_loaddata *udata, struct list_head *lh,
 	}
 
 	if (aa_g_export_binary) {
-		error = compress_loaddata(udata);
+		/* Do we have userspace-compressed data? */
+		if (compressed_data) {
+			kvfree(udata->data);
+			udata->data = compressed_data;
+			udata->compressed_size = compressed_size;
+			compressed_data = NULL; /* consumed */
+
+		} else
+			error = compress_loaddata(udata);
+
 		if (error)
 			goto fail;
+	} else if (compressed_data) {
+		kvfree(compressed_data);
+		compressed_data = NULL;
 	}
+
 	return 0;
 
 fail_profile:
@@ -1794,6 +1810,8 @@ int aa_unpack(struct aa_loaddata *udata, struct list_head *lh,
 	aa_put_profile(profile);
 
 fail:
+	if (compressed_data)
+		kvfree(compressed_data);
 	list_for_each_entry_safe(ent, tmp, lh, list) {
 		list_del_init(&ent->list);
 		aa_load_ent_free(ent);
diff --git a/security/integrity/ima/Kconfig b/security/integrity/ima/Kconfig
index f4d25e045808a..b3a9f86809b0c 100644
--- a/security/integrity/ima/Kconfig
+++ b/security/integrity/ima/Kconfig
@@ -347,4 +347,14 @@ config IMA_STAGING
 	  On kexec, staging is aborted and any staged measurement records are
 	  copied to the secondary kernel.
 
+config IMA_INIT_LATE_SYNC
+	bool "Initialise IMA at late_initcall_sync"
+	default n
+	help
+	  This option initialises IMA at late_initcall_sync for platforms
+	  where TPM device probing is deferred.
+	  When this option is enabled, modules that access files in the
+	  initramfs through usermode helper calls such as request_module()
+	  during initcall must not be built-in. Otherwise, IMA may miss
+	  file measurements for them.
 endif
diff --git a/security/integrity/ima/ima.h b/security/integrity/ima/ima.h
index caaedd4b58fd3..10214f73ca1ed 100644
--- a/security/integrity/ima/ima.h
+++ b/security/integrity/ima/ima.h
@@ -55,6 +55,8 @@ enum binary_lists {
 /* current content of the policy */
 extern int ima_policy_flag;
 
+extern struct mutex ima_write_mutex;
+
 /* bitset of digests algorithms allowed in the setxattr hook */
 extern atomic_t ima_setxattr_allowed_hash_algorithms;
 
@@ -75,6 +77,7 @@ extern struct ima_algo_desc *ima_algo_array __ro_after_init;
 extern int ima_appraise;
 extern struct tpm_chip *ima_tpm_chip;
 extern const char boot_aggregate_name[];
+extern const char boot_aggregate_late_name[];
 
 /* IMA event related data */
 struct ima_event_data {
@@ -466,6 +469,8 @@ void *ima_policy_start(struct seq_file *m, loff_t *pos);
 void *ima_policy_next(struct seq_file *m, void *v, loff_t *pos);
 void ima_policy_stop(struct seq_file *m, void *v);
 int ima_policy_show(struct seq_file *m, void *v);
+void ima_measure_loaded_policy(void);
+int ima_measure_raw_policy(const char *buf, size_t buf_len);
 
 /* Appraise integrity measurements */
 #define IMA_APPRAISE_ENFORCE	0x01
diff --git a/security/integrity/ima/ima_efi.c b/security/integrity/ima/ima_efi.c
index bca57d836cb96..be79110094546 100644
--- a/security/integrity/ima/ima_efi.c
+++ b/security/integrity/ima/ima_efi.c
@@ -17,6 +17,8 @@ static const char * const sb_arch_rules[] = {
 #endif
 #if IS_ENABLED(CONFIG_INTEGRITY_MACHINE_KEYRING) && IS_ENABLED(CONFIG_IMA_KEYRINGS_PERMIT_SIGNED_BY_BUILTIN_OR_SECONDARY)
 	"appraise func=POLICY_CHECK appraise_type=imasig",
+#else
+	"measure func=CRITICAL_DATA label=ima_policy",
 #endif
 	"measure func=MODULE_CHECK",
 	NULL
diff --git a/security/integrity/ima/ima_fs.c b/security/integrity/ima/ima_fs.c
index 174a94740da1a..2a0bca5543161 100644
--- a/security/integrity/ima/ima_fs.c
+++ b/security/integrity/ima/ima_fs.c
@@ -32,7 +32,9 @@
  */
 #define STAGED_REQ_LENGTH 21
 
-static DEFINE_MUTEX(ima_write_mutex);
+/* lock for protecting concurrent IMA policy updates */
+DEFINE_MUTEX(ima_write_mutex);
+
 static DEFINE_MUTEX(ima_measure_mutex);
 static long ima_measure_users;
 static struct task_struct *measure_writer;
@@ -596,6 +598,7 @@ static ssize_t ima_write_policy(struct file *file, const char __user *buf,
 				    1, 0);
 		result = -EACCES;
 	} else {
+		ima_measure_raw_policy(data, datalen);
 		result = ima_parse_add_rule(data);
 	}
 	mutex_unlock(&ima_write_mutex);
@@ -752,6 +755,10 @@ static int ima_release_policy(struct inode *inode, struct file *file)
 	}
 
 	ima_update_policy();
+
+	mutex_lock(&ima_write_mutex);
+	ima_measure_loaded_policy();
+	mutex_unlock(&ima_write_mutex);
 #if !defined(CONFIG_IMA_WRITE_POLICY) && !defined(CONFIG_IMA_READ_POLICY)
 	securityfs_remove(file->f_path.dentry);
 #elif defined(CONFIG_IMA_WRITE_POLICY)
diff --git a/security/integrity/ima/ima_init.c b/security/integrity/ima/ima_init.c
index 7e0aa09a12e64..d53f4d89a53e5 100644
--- a/security/integrity/ima/ima_init.c
+++ b/security/integrity/ima/ima_init.c
@@ -22,6 +22,7 @@
 
 /* name for boot aggregate entry */
 const char boot_aggregate_name[] = "boot_aggregate";
+const char boot_aggregate_late_name[] = "boot_aggregate_late";
 struct tpm_chip *ima_tpm_chip;
 
 /* Add the boot aggregate to the IMA measurement list and extend
@@ -45,11 +46,11 @@ static int __init ima_add_boot_aggregate(void)
 	const char *audit_cause = "ENOMEM";
 	struct ima_template_entry *entry;
 	struct ima_iint_cache tmp_iint, *iint = &tmp_iint;
-	struct ima_event_data event_data = { .iint = iint,
-					     .filename = boot_aggregate_name };
+	struct ima_event_data event_data = { .iint = iint };
 	struct ima_max_digest_data hash;
 	struct ima_digest_data *hash_hdr = container_of(&hash.hdr,
 						struct ima_digest_data, hdr);
+	const char *filename;
 	int result = -ENOMEM;
 	int violation = 0;
 
@@ -59,6 +60,12 @@ static int __init ima_add_boot_aggregate(void)
 	iint->ima_hash->algo = ima_hash_algo;
 	iint->ima_hash->length = hash_digest_size[ima_hash_algo];
 
+	if (IS_ENABLED(CONFIG_IMA_INIT_LATE_SYNC))
+		filename = boot_aggregate_late_name;
+	else
+		filename = boot_aggregate_name;
+	event_data.filename = filename;
+
 	/*
 	 * With TPM 2.0 hash agility, TPM chips could support multiple TPM
 	 * PCR banks, allowing firmware to configure and enable different
@@ -86,7 +93,7 @@ static int __init ima_add_boot_aggregate(void)
 	}
 
 	result = ima_store_template(entry, violation, NULL,
-				    boot_aggregate_name,
+				    filename,
 				    CONFIG_IMA_MEASURE_PCR_IDX);
 	if (result < 0) {
 		ima_free_template_entry(entry);
@@ -95,7 +102,7 @@ static int __init ima_add_boot_aggregate(void)
 	}
 	return 0;
 err_out:
-	integrity_audit_msg(AUDIT_INTEGRITY_PCR, NULL, boot_aggregate_name, op,
+	integrity_audit_msg(AUDIT_INTEGRITY_PCR, NULL, filename, op,
 			    audit_cause, result, 0);
 	return result;
 }
diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c
index 5cea53fc36dff..334e21be40754 100644
--- a/security/integrity/ima/ima_main.c
+++ b/security/integrity/ima/ima_main.c
@@ -1221,6 +1221,28 @@ int ima_measure_critical_data(const char *event_label,
 }
 EXPORT_SYMBOL_GPL(ima_measure_critical_data);
 
+/**
+ * ima_measure_raw_policy - Measure the raw policy write buffer
+ * @buf: pointer to the buffer containing the raw policy data
+ * @buf_len: size of the buffer
+ *
+ * Measure the raw policy buffer sent to the IMA policy securityfs file. The
+ * buffer is written from userspace, and the measurement is performed before
+ * parsing it. This measurement includes any data written on the policy file
+ * such as malformed policy rules and comments.
+ *
+ * Return 0 on success, a negative value otherwise.
+ */
+int ima_measure_raw_policy(const char *buf, size_t buf_len)
+{
+	if (!buf || !buf_len)
+		return -EINVAL;
+
+	return process_buffer_measurement(&nop_mnt_idmap, NULL, buf, buf_len,
+					  "ima_policy_written", POLICY_CHECK,
+					  0, NULL, false, NULL, 0);
+}
+
 #ifdef CONFIG_INTEGRITY_ASYMMETRIC_KEYS
 
 /**
@@ -1337,5 +1359,9 @@ DEFINE_LSM(ima) = {
 	.order = LSM_ORDER_LAST,
 	.blobs = &ima_blob_sizes,
 	/* Start IMA after the TPM is available */
+#ifndef CONFIG_IMA_INIT_LATE_SYNC
 	.initcall_late = init_ima,
+#else
+	.initcall_late_sync = init_ima,
+#endif
 };
diff --git a/security/integrity/ima/ima_policy.c b/security/integrity/ima/ima_policy.c
index b1c010e8eb138..f79d07bb63c6f 100644
--- a/security/integrity/ima/ima_policy.c
+++ b/security/integrity/ima/ima_policy.c
@@ -53,6 +53,8 @@
 #define INVALID_PCR(a) (((a) < 0) || \
 	(a) >= (sizeof_field(struct ima_iint_cache, measured_pcrs) * 8))
 
+static size_t max_rule_len;
+
 int ima_policy_flag;
 static int temp_ima_appraise;
 static int build_ima_appraise __ro_after_init;
@@ -540,6 +542,8 @@ static bool ima_match_rule_data(struct ima_rule_entry *rule,
 
 		opt_list = rule->label;
 		break;
+	case POLICY_CHECK:
+		return true;
 	default:
 		return false;
 	}
@@ -586,6 +590,10 @@ static bool ima_match_rules(struct ima_rule_entry *rule,
 		return false;
 
 	switch (func) {
+	case POLICY_CHECK:
+		if (inode)
+			break;
+		fallthrough;
 	case KEY_CHECK:
 	case CRITICAL_DATA:
 		return ((rule->func == func) &&
@@ -955,6 +963,8 @@ void __init ima_init_policy(void)
 {
 	int build_appraise_entries, arch_entries;
 
+	max_rule_len = 255;
+
 	/* if !ima_policy, we load NO default rules */
 	if (ima_policy)
 		add_rules(dont_measure_rules, ARRAY_SIZE(dont_measure_rules),
@@ -1994,6 +2004,9 @@ ssize_t ima_parse_add_rule(char *rule)
 
 	list_add_tail(&entry->list, &ima_temp_rules);
 
+	if (len > max_rule_len)
+		max_rule_len = len;
+
 	return len;
 }
 
@@ -2021,7 +2034,6 @@ const char *const func_tokens[] = {
 	__ima_hooks(__ima_hook_stringify)
 };
 
-#ifdef	CONFIG_IMA_READ_POLICY
 enum {
 	mask_exec = 0, mask_write, mask_read, mask_append
 };
@@ -2323,7 +2335,6 @@ int ima_policy_show(struct seq_file *m, void *v)
 	seq_puts(m, "\n");
 	return 0;
 }
-#endif	/* CONFIG_IMA_READ_POLICY */
 
 #if defined(CONFIG_IMA_APPRAISE) && defined(CONFIG_INTEGRITY_TRUSTED_KEYRING)
 /*
@@ -2380,3 +2391,83 @@ bool ima_appraise_signature(enum kernel_read_file_id id)
 	return found;
 }
 #endif /* CONFIG_IMA_APPRAISE && CONFIG_INTEGRITY_TRUSTED_KEYRING */
+
+/**
+ * ima_measure_loaded_policy - measure the active IMA policy ruleset
+ *
+ * Must be called with ima_write_mutex held, as it performs two
+ * separate RCU read passes over ima_rules and relies on the mutex
+ * to prevent concurrent policy updates between them.
+ */
+void ima_measure_loaded_policy(void)
+{
+	const char *event_name = "ima_policy_loaded";
+	const char *op = "measure_loaded_ima_policy";
+	size_t rule_len = max_rule_len + 2;
+	struct ima_rule_entry *rule_entry;
+	struct list_head *ima_rules_tmp;
+	struct seq_file file = { 0 };
+	int result = -ENOMEM;
+	size_t file_len = 0;
+	char *rule;
+
+	lockdep_assert_held(&ima_write_mutex);
+
+	rule = kmalloc(rule_len, GFP_KERNEL);
+	if (!rule) {
+		integrity_audit_msg(AUDIT_INTEGRITY_PCR, NULL, event_name,
+				    op, "ENOMEM", result, 0);
+		return;
+	}
+
+	/* calculate IMA policy rules memory size */
+	file.buf = rule;
+	file.read_pos = 0;
+	file.size = rule_len;
+	file.count = 0;
+
+	rcu_read_lock();
+	ima_rules_tmp = rcu_dereference(ima_rules);
+	list_for_each_entry_rcu(rule_entry, ima_rules_tmp, list) {
+		ima_policy_show(&file, rule_entry);
+
+		if (seq_has_overflowed(&file)) {
+			result = -E2BIG;
+			integrity_audit_msg(AUDIT_INTEGRITY_PCR, NULL,
+					    event_name, op, "rule_length",
+					    result, 0);
+			rcu_read_unlock();
+			goto free_rule;
+		}
+
+		file_len += file.count;
+		file.count = 0;
+	}
+	rcu_read_unlock();
+
+	/* copy IMA policy rules to a buffer for measuring */
+	file.buf = kmalloc(file_len, GFP_KERNEL);
+	if (!file.buf) {
+		integrity_audit_msg(AUDIT_INTEGRITY_PCR, NULL, event_name,
+				    op, "ENOMEM", result, 0);
+		goto free_rule;
+	}
+
+	file.read_pos = 0;
+	file.size = file_len;
+	file.count = 0;
+
+	rcu_read_lock();
+	ima_rules_tmp = rcu_dereference(ima_rules);
+	list_for_each_entry_rcu(rule_entry, ima_rules_tmp, list) {
+		ima_policy_show(&file, rule_entry);
+	}
+	rcu_read_unlock();
+
+	ima_measure_critical_data("ima_policy", event_name, file.buf,
+				  file.count, false, NULL, 0);
+
+	kfree(file.buf);
+free_rule:
+	kfree(rule);
+}
diff --git a/security/integrity/ima/ima_template_lib.c b/security/integrity/ima/ima_template_lib.c
index 0e627eac9c33b..8a89236f926c1 100644
--- a/security/integrity/ima/ima_template_lib.c
+++ b/security/integrity/ima/ima_template_lib.c
@@ -363,7 +363,8 @@ int ima_eventdigest_init(struct ima_event_data *event_data,
 		goto out;
 	}
 
-	if ((const char *)event_data->filename == boot_aggregate_name) {
+	if ((const char *)event_data->filename == boot_aggregate_name ||
+	    (const char *)event_data->filename == boot_aggregate_late_name) {
 		if (ima_tpm_chip) {
 			hash.hdr.algo = HASH_ALGO_SHA1;
 			result = ima_calc_boot_aggregate(hash_hdr);
diff --git a/security/landlock/net.c b/security/landlock/net.c
index cbff59ec3abaa..46c17116fcf42 100644
--- a/security/landlock/net.c
+++ b/security/landlock/net.c
@@ -351,6 +351,14 @@ static int hook_socket_sendmsg(struct socket *const sock,
 	access_mask_t access_request;
 	int ret = 0;
 
+	if ((msg->msg_flags & MSG_FASTOPEN) && address && sk_is_tcp(sock->sk)) {
+		ret = current_check_access_socket(
+			sock, address, addrlen, LANDLOCK_ACCESS_NET_CONNECT_TCP,
+			true);
+		if (ret != 0)
+			return ret;
+	}
+
 	if (sk_is_udp(sock->sk))
 		access_request = LANDLOCK_ACCESS_NET_CONNECT_SEND_UDP;
 	else
diff --git a/security/landlock/ruleset.h b/security/landlock/ruleset.h
index 61f3c253d5c90..0437adf174287 100644
--- a/security/landlock/ruleset.h
+++ b/security/landlock/ruleset.h
@@ -35,8 +35,8 @@ struct landlock_layer {
 	 */
 	struct {
 		/**
-		 * @quiet: Suppresses denial logs for the object covered by this
-		 * rule in this domain.  For filesystem rules, this inherits
+		 * @flags.quiet: Suppresses denial logs for the object covered by
+		 * this rule in this domain.  For filesystem rules, this inherits
 		 * down the file hierarchy.
 		 */
 		u8 quiet : 1;
diff --git a/security/landlock/task.c b/security/landlock/task.c
index 7ddf211f75c37..55522a601367e 100644
--- a/security/landlock/task.c
+++ b/security/landlock/task.c
@@ -95,8 +95,7 @@ static int hook_ptrace_access_check(struct task_struct *const child,
 	if (!parent_subject)
 		return 0;
 
-	scoped_guard(rcu)
-	{
+	scoped_guard(rcu) {
 		const struct landlock_ruleset *const child_dom =
 			landlock_get_task_domain(child);
 		err = domain_ptrace(parent_subject->domain, child_dom);
@@ -243,6 +242,17 @@ static bool sock_is_scoped(struct sock *const other,
 
 	/* The credentials will not change. */
 	lockdep_assert_held(&unix_sk(other)->lock);
+
+	/*
+	 * A live kernel socket (e.g. from sock_create_kern()) has no backing
+	 * file, hence no Landlock domain, so treat it as unscoped.  The
+	 * sk_socket check only guards that dereference; sk_socket is NULL
+	 * solely for a dead peer, which the caller already excludes under the
+	 * held lock, so no separate SOCK_DEAD check is needed.
+	 */
+	if (unlikely(!other->sk_socket || !other->sk_socket->file))
+		return false;
+
 	dom_other = landlock_cred(other->sk_socket->file->f_cred)->domain;
 	return domain_is_scoped(domain, dom_other,
 				LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET);
@@ -370,8 +380,7 @@ static int hook_task_kill(struct task_struct *const p,
 	if (!subject)
 		return 0;
 
-	scoped_guard(rcu)
-	{
+	scoped_guard(rcu) {
 		is_scoped = domain_is_scoped(subject->domain,
 					     landlock_get_task_domain(p),
 					     signal_scope.scope);
@@ -422,8 +431,7 @@ static int hook_file_send_sigiotask(struct task_struct *tsk,
 	if (task_tgid(tsk) == landlock_file(fown->file)->fown_tg)
 		return 0;
 
-	scoped_guard(rcu)
-	{
+	scoped_guard(rcu) {
 		is_scoped = domain_is_scoped(subject->domain,
 					     landlock_get_task_domain(tsk),
 					     signal_scope.scope);
diff --git a/security/lsm_init.c b/security/lsm_init.c
index 7c0fd17f1601f..a1ad641811de0 100644
--- a/security/lsm_init.c
+++ b/security/lsm_init.c
@@ -556,13 +556,22 @@ device_initcall(security_initcall_device);
  * security_initcall_late - Run the LSM late initcalls
  */
 static int __init security_initcall_late(void)
+{
+	return lsm_initcall(late);
+}
+late_initcall(security_initcall_late);
+
+/**
+ * security_initcall_late_sync - Run the LSM late initcalls sync
+ */
+static int __init security_initcall_late_sync(void)
 {
 	int rc;
 
-	rc = lsm_initcall(late);
+	rc = lsm_initcall(late_sync);
 	lsm_pr_dbg("all enabled LSMs fully activated\n");
 	call_blocking_lsm_notifier(LSM_STARTED_ALL, NULL);
 
 	return rc;
 }
-late_initcall(security_initcall_late);
+late_initcall_sync(security_initcall_late_sync);
diff --git a/security/security.c b/security/security.c
index 71aea8fdf0142..2ee276ab15c5b 100644
--- a/security/security.c
+++ b/security/security.c
@@ -3301,15 +3301,14 @@ int security_task_kill(struct task_struct *p, struct kernel_siginfo *info,
 }
 
 /**
- * security_task_prctl() - Check if a prctl op is allowed
+ * security_task_prctl() - Handle an LSM specific prctl() call
  * @option: operation
  * @arg2: argument
  * @arg3: argument
  * @arg4: argument
  * @arg5: argument
  *
- * Check permission before performing a process control operation on the
- * current process.
+ * Handle lsm specific prctl() operations.
  *
  * Return: Return -ENOSYS if no-one wanted to handle this op, any other value
  *         to cause prctl() to return immediately with that value.
diff --git a/security/selinux/avc.c b/security/selinux/avc.c
index 813e82bcfc278..a9401d6c2e5f1 100644
--- a/security/selinux/avc.c
+++ b/security/selinux/avc.c
@@ -497,7 +497,7 @@ static struct avc_node *avc_alloc_node(void)
 
 	node = kmem_cache_zalloc(avc_node_cachep, GFP_NOWAIT);
 	if (!node)
-		goto out;
+		return NULL;
 
 	INIT_HLIST_NODE(&node->list);
 	avc_cache_stats_incr(allocations);
@@ -506,7 +506,6 @@ static struct avc_node *avc_alloc_node(void)
 	    selinux_avc.avc_cache_threshold)
 		avc_reclaim_node();
 
-out:
 	return node;
 }
 
diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c
index 70a3388c047d9..59069e752644f 100644
--- a/security/selinux/hooks.c
+++ b/security/selinux/hooks.c
@@ -94,6 +94,7 @@
 #include <linux/io_uring/cmd.h>
 #include <uapi/linux/lsm.h>
 #include <linux/memfd.h>
+#include <uapi/linux/inet_diag.h>
 
 #include "initcalls.h"
 #include "avc.h"
@@ -106,6 +107,7 @@
 #include "netlabel.h"
 #include "audit.h"
 #include "avc_ss.h"
+#include "ima.h"
 
 #define SELINUX_INODE_INIT_XATTRS 1
 
@@ -1336,11 +1338,11 @@ static int selinux_genfs_get_sid(struct dentry *dentry,
 	struct super_block *sb = dentry->d_sb;
 	char *buffer, *path;
 
-	buffer = (char *)__get_free_page(GFP_KERNEL);
+	buffer = kmalloc(PATH_MAX, GFP_KERNEL);
 	if (!buffer)
 		return -ENOMEM;
 
-	path = dentry_path_raw(dentry, buffer, PAGE_SIZE);
+	path = dentry_path_raw(dentry, buffer, PATH_MAX);
 	if (IS_ERR(path))
 		rc = PTR_ERR(path);
 	else {
@@ -1361,7 +1363,7 @@ static int selinux_genfs_get_sid(struct dentry *dentry,
 			rc = 0;
 		}
 	}
-	free_page((unsigned long)buffer);
+	kfree(buffer);
 	return rc;
 }
 
@@ -3969,9 +3971,9 @@ static int selinux_file_ioctl_compat(struct file *file, unsigned int cmd,
 
 static int default_noexec __ro_after_init;
 
-static int __file_map_prot_check(const struct cred *cred,
-				 const struct file *file, unsigned long prot,
-				 bool shared, bool bf_user_file)
+static int __file_map_prot_check(const struct file *file, unsigned long prot,
+				 bool shared, bool mounter_check,
+				 bool bf_user_file)
 {
 	struct inode *inode = NULL;
 	bool prot_exec = prot & PROT_EXEC;
@@ -3984,10 +3986,10 @@ static int __file_map_prot_check(const struct cred *cred,
 			inode = file_inode(file);
 	}
 
-	if (default_noexec && prot_exec &&
+	if (!mounter_check && default_noexec && prot_exec &&
 	    (!file || IS_PRIVATE(inode) || (!shared && prot_write))) {
 		int rc;
-		u32 sid = cred_sid(cred);
+		u32 sid = current_sid();
 
 		/*
 		 * We are making executable an anonymous mapping or a private
@@ -4000,6 +4002,8 @@ static int __file_map_prot_check(const struct cred *cred,
 	}
 
 	if (file) {
+		const struct cred *cred = mounter_check ?
+				file->f_cred : current_cred();
 		/* "read" always possible, "write" only if shared */
 		u32 av = FILE__READ;
 		if (shared && prot_write)
@@ -4013,11 +4017,11 @@ static int __file_map_prot_check(const struct cred *cred,
 	return 0;
 }
 
-static inline int file_map_prot_check(const struct cred *cred,
-				      const struct file *file,
-				      unsigned long prot, bool shared)
+static inline int file_map_prot_check(const struct file *file,
+				      unsigned long prot, bool shared,
+				      bool mounter_check)
 {
-	return __file_map_prot_check(cred, file, prot, shared, false);
+	return __file_map_prot_check(file, prot, shared, mounter_check, false);
 }
 
 static int selinux_mmap_addr(unsigned long addr)
@@ -4033,12 +4037,14 @@ static int selinux_mmap_addr(unsigned long addr)
 	return rc;
 }
 
-static int selinux_mmap_file_common(const struct cred *cred, struct file *file,
-				    unsigned long prot, bool shared)
+static int selinux_mmap_file_common(struct file *file, unsigned long prot,
+				    bool shared, bool mounter_check)
 {
 	if (file) {
 		int rc;
 		struct common_audit_data ad;
+		const struct cred *cred = mounter_check ?
+				file->f_cred : current_cred();
 
 		ad.type = LSM_AUDIT_DATA_FILE;
 		ad.u.file = file;
@@ -4047,15 +4053,16 @@ static int selinux_mmap_file_common(const struct cred *cred, struct file *file,
 			return rc;
 	}
 
-	return file_map_prot_check(cred, file, prot, shared);
+	return file_map_prot_check(file, prot, shared, mounter_check);
 }
 
 static int selinux_mmap_file(struct file *file,
 			     unsigned long reqprot __always_unused,
 			     unsigned long prot, unsigned long flags)
 {
-	return selinux_mmap_file_common(current_cred(), file, prot,
-					(flags & MAP_TYPE) == MAP_SHARED);
+	return selinux_mmap_file_common(file, prot,
+					(flags & MAP_TYPE) == MAP_SHARED,
+					false);
 }
 
 /**
@@ -4087,8 +4094,9 @@ static int selinux_mmap_backing_file(struct vm_area_struct *vma,
 	if (vma->vm_flags & VM_EXEC)
 		prot |= PROT_EXEC;
 
-	return selinux_mmap_file_common(backing_file->f_cred, backing_file,
-					prot, vma->vm_flags & VM_SHARED);
+	return selinux_mmap_file_common(backing_file, prot,
+					vma->vm_flags & VM_SHARED,
+					true);
 }
 
 static int selinux_file_mprotect(struct vm_area_struct *vma,
@@ -4149,11 +4157,11 @@ static int selinux_file_mprotect(struct vm_area_struct *vma,
 		}
 	}
 
-	rc = __file_map_prot_check(cred, file, prot, shared, backing_file);
+	rc = __file_map_prot_check(file, prot, shared, false, backing_file);
 	if (rc)
 		return rc;
 	if (backing_file) {
-		rc = file_map_prot_check(file->f_cred, file, prot, shared);
+		rc = file_map_prot_check(file, prot, shared, true);
 		if (rc)
 			return rc;
 	}
@@ -6290,12 +6298,17 @@ static int selinux_netlink_send(struct sock *sk, struct sk_buff *skb)
 				return rc;
 		} else if (rc == -EINVAL) {
 			/* -EINVAL is a missing msg/perm mapping */
-			pr_warn_ratelimited("SELinux: unrecognized netlink"
-				" message: protocol=%hu nlmsg_type=%hu sclass=%s"
-				" pid=%d comm=%s\n",
-				sk->sk_protocol, nlh->nlmsg_type,
-				secclass_map[sclass - 1].name,
-				task_pid_nr(current), current->comm);
+			if (sclass == SECCLASS_NETLINK_TCPDIAG_SOCKET &&
+			    nlh->nlmsg_type == DCCPDIAG_GETSOCK)
+				pr_warn_once("SELinux: DCCP has been removed, pid=%d comm=%s\n",
+					     task_pid_nr(current), current->comm);
+			else
+				pr_warn_ratelimited("SELinux: unrecognized netlink"
+					" message: protocol=%hu nlmsg_type=%hu sclass=%s"
+					" pid=%d comm=%s\n",
+					sk->sk_protocol, nlh->nlmsg_type,
+					secclass_map[sclass - 1].name,
+					task_pid_nr(current), current->comm);
 			if (enforcing_enabled() &&
 			    !security_get_allow_unknown())
 				return rc;
@@ -7868,6 +7881,8 @@ static __init int selinux_init(void)
 
 	hashtab_cache_init();
 
+	selinux_ima_config_len_init();
+
 	security_add_hooks(selinux_hooks, ARRAY_SIZE(selinux_hooks),
 			   &selinux_lsmid);
 
diff --git a/security/selinux/ima.c b/security/selinux/ima.c
index aa34da9b0aebd..cda68122e032d 100644
--- a/security/selinux/ima.c
+++ b/security/selinux/ima.c
@@ -9,9 +9,31 @@
  */
 #include <linux/vmalloc.h>
 #include <linux/ima.h>
+#include <linux/seq_buf.h>
 #include "security.h"
 #include "ima.h"
 
+static int selinux_ima_config_len __ro_after_init;
+
+/*
+ * selinux_ima_config_len_init - Compute the configuration settings string length
+ *
+ * The string is fixed text plus one digit per setting, so its length
+ * is known at boot.
+ */
+void __init selinux_ima_config_len_init(void)
+{
+	int buf_len, suffix_len, i;
+
+	buf_len = strlen("initialized=0;enforcing=0;checkreqprot=0;") + 1;
+	suffix_len = strlen("=0;");
+
+	for (i = 0; i < __POLICYDB_CAP_MAX; i++)
+		buf_len += strlen(selinux_policycap_names[i]) + suffix_len;
+
+	selinux_ima_config_len = buf_len;
+}
+
 /*
  * selinux_ima_collect_state - Read selinux configuration settings
  *
@@ -20,46 +42,25 @@
  */
 static char *selinux_ima_collect_state(void)
 {
-	const char *on = "=1;", *off = "=0;";
+	struct seq_buf s;
 	char *buf;
-	int buf_len, len, i, rc;
-
-	buf_len = strlen("initialized=0;enforcing=0;checkreqprot=0;") + 1;
+	int i;
 
-	len = strlen(on);
-	for (i = 0; i < __POLICYDB_CAP_MAX; i++)
-		buf_len += strlen(selinux_policycap_names[i]) + len;
-
-	buf = kzalloc(buf_len, GFP_KERNEL);
+	buf = kzalloc(selinux_ima_config_len, GFP_KERNEL);
 	if (!buf)
 		return NULL;
 
-	rc = strscpy(buf, "initialized", buf_len);
-	WARN_ON(rc < 0);
-
-	rc = strlcat(buf, selinux_initialized() ? on : off, buf_len);
-	WARN_ON(rc >= buf_len);
+	seq_buf_init(&s, buf, selinux_ima_config_len);
 
-	rc = strlcat(buf, "enforcing", buf_len);
-	WARN_ON(rc >= buf_len);
+	seq_buf_printf(&s, "initialized=%d;enforcing=%d;checkreqprot=%d;",
+		       selinux_initialized(), enforcing_enabled(),
+		       checkreqprot_get());
 
-	rc = strlcat(buf, enforcing_enabled() ? on : off, buf_len);
-	WARN_ON(rc >= buf_len);
-
-	rc = strlcat(buf, "checkreqprot", buf_len);
-	WARN_ON(rc >= buf_len);
-
-	rc = strlcat(buf, checkreqprot_get() ? on : off, buf_len);
-	WARN_ON(rc >= buf_len);
-
-	for (i = 0; i < __POLICYDB_CAP_MAX; i++) {
-		rc = strlcat(buf, selinux_policycap_names[i], buf_len);
-		WARN_ON(rc >= buf_len);
+	for (i = 0; i < __POLICYDB_CAP_MAX; i++)
+		seq_buf_printf(&s, "%s=%d;", selinux_policycap_names[i],
+			       selinux_state.policycap[i]);
 
-		rc = strlcat(buf, selinux_state.policycap[i] ? on : off,
-			buf_len);
-		WARN_ON(rc >= buf_len);
-	}
+	WARN_ON(seq_buf_has_overflowed(&s));
 
 	return buf;
 }
diff --git a/security/selinux/include/ima.h b/security/selinux/include/ima.h
index 38ab302f59463..d7d18f030d3df 100644
--- a/security/selinux/include/ima.h
+++ b/security/selinux/include/ima.h
@@ -14,9 +14,13 @@
 #include "security.h"
 
 #ifdef CONFIG_IMA
+void __init selinux_ima_config_len_init(void);
 extern void selinux_ima_measure_state(void);
 extern void selinux_ima_measure_state_locked(void);
 #else
+static inline void selinux_ima_config_len_init(void)
+{
+}
 static inline void selinux_ima_measure_state(void)
 {
 }
diff --git a/security/selinux/selinuxfs.c b/security/selinux/selinuxfs.c
index 5aaaf69410bb0..c7d91476971cb 100644
--- a/security/selinux/selinuxfs.c
+++ b/security/selinux/selinuxfs.c
@@ -1984,17 +1984,15 @@ int __init init_sel_fs(void)
 		return err;
 
 	err = register_filesystem(&sel_fs_type);
-	if (err) {
-		sysfs_remove_mount_point(fs_kobj, "selinux");
-		return err;
-	}
+	if (err)
+		goto err_remove_mount_point;
 
 	selinux_null.mnt = kern_mount(&sel_fs_type);
 	if (IS_ERR(selinux_null.mnt)) {
 		pr_err("selinuxfs:  could not mount!\n");
 		err = PTR_ERR(selinux_null.mnt);
 		selinux_null.mnt = NULL;
-		return err;
+		goto err_unregister_fs;
 	}
 
 	selinux_null.dentry = try_lookup_noperm(&null_name,
@@ -2003,7 +2001,7 @@ int __init init_sel_fs(void)
 		pr_err("selinuxfs:  could not lookup null!\n");
 		err = PTR_ERR(selinux_null.dentry);
 		selinux_null.dentry = NULL;
-		return err;
+		goto err_unmount;
 	}
 
 	/*
@@ -2012,5 +2010,14 @@ int __init init_sel_fs(void)
 	 */
 	(void) selinux_kernel_status_page();
 
+	return 0;
+
+err_unmount:
+	kern_unmount(selinux_null.mnt);
+	selinux_null.mnt = NULL;
+err_unregister_fs:
+	unregister_filesystem(&sel_fs_type);
+err_remove_mount_point:
+	sysfs_remove_mount_point(fs_kobj, "selinux");
 	return err;
 }
diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c
index ead504a639e36..ba1c9bd9e29f8 100644
--- a/security/selinux/ss/policydb.c
+++ b/security/selinux/ss/policydb.c
@@ -604,10 +604,15 @@ static int type_index(void *key, void *datum, void *datap)
 	typdatum = datum;
 	p = datap;
 
+	if (!typdatum->value || typdatum->value > p->p_types.nprim ||
+		typdatum->bounds > p->p_types.nprim) {
+		pr_err("SELinux: type %s had value %u bounds %u nprim %u\n",
+			(char *)key, typdatum->value, typdatum->bounds,
+			p->p_types.nprim);
+		return -EINVAL;
+	}
+
 	if (typdatum->primary) {
-		if (!typdatum->value || typdatum->value > p->p_types.nprim ||
-		    typdatum->bounds > p->p_types.nprim)
-			return -EINVAL;
 		p->sym_val_to_name[SYM_TYPES][typdatum->value - 1] = key;
 		p->type_val_to_struct[typdatum->value - 1] = typdatum;
 	}
diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c
index 2d828548f3db6..c03d3acc0a7c8 100644
--- a/security/selinux/ss/services.c
+++ b/security/selinux/ss/services.c
@@ -1355,8 +1355,8 @@ const char *security_get_initial_sid_context(u32 sid)
 }
 
 static int security_sid_to_context_core(u32 sid, char **scontext,
-					u32 *scontext_len, int force,
-					int only_invalid)
+					u32 *scontext_len, bool force,
+					bool only_invalid)
 {
 	struct selinux_policy *policy;
 	struct policydb *policydb;
@@ -1439,14 +1439,14 @@ static int security_sid_to_context_core(u32 sid, char **scontext,
 int security_sid_to_context(u32 sid, char **scontext, u32 *scontext_len)
 {
 	return security_sid_to_context_core(sid, scontext,
-					    scontext_len, 0, 0);
+					    scontext_len, false, false);
 }
 
 int security_sid_to_context_force(u32 sid,
 				  char **scontext, u32 *scontext_len)
 {
 	return security_sid_to_context_core(sid, scontext,
-					    scontext_len, 1, 0);
+					    scontext_len, true, false);
 }
 
 /**
@@ -1466,7 +1466,7 @@ int security_sid_to_context_inval(u32 sid,
 				  char **scontext, u32 *scontext_len)
 {
 	return security_sid_to_context_core(sid, scontext,
-					    scontext_len, 1, 1);
+					    scontext_len, true, true);
 }
 
 /*
@@ -1552,7 +1552,7 @@ static int string_to_context_struct(struct policydb *pol,
 
 static int security_context_to_sid_core(const char *scontext, u32 scontext_len,
 					u32 *sid, u32 def_sid, gfp_t gfp_flags,
-					int force)
+					bool force)
 {
 	struct selinux_policy *policy;
 	struct policydb *policydb;
@@ -1641,7 +1641,7 @@ int security_context_to_sid(const char *scontext, u32 scontext_len, u32 *sid,
 			    gfp_t gfp)
 {
 	return security_context_to_sid_core(scontext, scontext_len,
-					    sid, SECSID_NULL, gfp, 0);
+					    sid, SECSID_NULL, gfp, false);
 }
 
 int security_context_str_to_sid(const char *scontext, u32 *sid, gfp_t gfp)
@@ -1673,14 +1673,14 @@ int security_context_to_sid_default(const char *scontext, u32 scontext_len,
 				    u32 *sid, u32 def_sid, gfp_t gfp_flags)
 {
 	return security_context_to_sid_core(scontext, scontext_len,
-					    sid, def_sid, gfp_flags, 1);
+					    sid, def_sid, gfp_flags, true);
 }
 
 int security_context_to_sid_force(const char *scontext, u32 scontext_len,
 				  u32 *sid)
 {
 	return security_context_to_sid_core(scontext, scontext_len,
-					    sid, SECSID_NULL, GFP_KERNEL, 1);
+					    sid, SECSID_NULL, GFP_KERNEL, true);
 }
 
 static int compute_sid_handle_invalid_context(
diff --git a/security/smack/smack.h b/security/smack/smack.h
index 9b9eb262fe33e..2a60a045b8cf4 100644
--- a/security/smack/smack.h
+++ b/security/smack/smack.h
@@ -261,6 +261,7 @@ struct smack_audit_data {
 	char *subject;
 	char *object;
 	char *request;
+	struct task_struct *subj_tsk;
 	int result;
 };
 
@@ -317,8 +318,9 @@ int smack_populate_secattr(struct smack_known *skp);
  * Shared data.
  */
 extern int smack_enabled __initdata;
-extern int smack_cipso_direct;
-extern int smack_cipso_mapped;
+extern u8  smack_cipso_auto_level[2];
+#define smack_cipso_direct (+smack_cipso_auto_level[0])
+#define smack_cipso_mapped (+smack_cipso_auto_level[1])
 extern struct smack_known *smack_net_ambient;
 extern struct smack_known *smack_syslog_label;
 #ifdef CONFIG_SECURITY_SMACK_BRINGUP
diff --git a/security/smack/smack_access.c b/security/smack/smack_access.c
index 350b88d582b39..fb85356266e53 100644
--- a/security/smack/smack_access.c
+++ b/security/smack/smack_access.c
@@ -331,6 +331,15 @@ static void smack_log_callback(struct audit_buffer *ab, void *a)
 		audit_log_format(ab, " labels_differ");
 	else
 		audit_log_format(ab, " requested=%s", sad->request);
+
+	if (sad->subj_tsk) {
+		char comm[TASK_COMM_LEN];
+
+		audit_log_format(ab, " subj_pid=%d subj_comm=",
+			task_tgid_nr(sad->subj_tsk));
+		audit_log_untrustedstring(ab,
+			get_task_comm(comm, sad->subj_tsk));
+	}
 }
 
 /**
diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c
index ff115068c5c06..df2693f117540 100644
--- a/security/smack/smack_lsm.c
+++ b/security/smack/smack_lsm.c
@@ -130,12 +130,13 @@ static int smk_bu_note(char *note, struct smack_known *sskp,
 #define smk_bu_note(note, sskp, oskp, mode, RC) (RC)
 #endif
 
-#ifdef CONFIG_SECURITY_SMACK_BRINGUP
-static int smk_bu_current(char *note, struct smack_known *oskp,
-			  int mode, int rc)
+static int
+smk_bu_tsk_to_obj(struct task_struct *tsk, const struct task_smack *tsp,
+		  char *note, struct smack_known *oskp, int mode, int rc)
 {
-	struct task_smack *tsp = smack_cred(current_cred());
+#ifdef CONFIG_SECURITY_SMACK_BRINGUP
 	char acc[SMK_NUM_ACCESS_TYPE + 1];
+	char comm[TASK_COMM_LEN];
 
 	if (rc <= 0)
 		return rc;
@@ -143,14 +144,22 @@ static int smk_bu_current(char *note, struct smack_known *oskp,
 		rc = 0;
 
 	smk_bu_mode(mode, acc);
+
 	pr_info("Smack %s: (%s %s %s) %s %s\n", smk_bu_mess[rc],
-		tsp->smk_task->smk_known, oskp->smk_known,
-		acc, current->comm, note);
+		smk_of_task(tsp)->smk_known, oskp->smk_known,
+		acc, get_task_comm(comm, tsk), note);
 	return 0;
-}
 #else
-#define smk_bu_current(note, oskp, mode, RC) (RC)
+	return rc;
 #endif
+}
+
+static int smk_bu_current(char *note, struct smack_known *oskp,
+			  int mode, int rc)
+{
+	return smk_bu_tsk_to_obj(current, smack_cred(current_cred()),
+				 note, oskp, mode, rc);
+}
 
 #ifdef CONFIG_SECURITY_SMACK_BRINGUP
 static int smk_bu_task(struct task_struct *otp, int mode, int rc)
@@ -1312,7 +1321,7 @@ static int smack_inode_getattr(const struct path *path)
  */
 static int smack_inode_xattr_skipcap(const char *name)
 {
-	if (strncmp(name, XATTR_SMACK_SUFFIX, strlen(XATTR_SMACK_SUFFIX)))
+	if (strncmp(name, XATTR_SMACK_SUFFIX, strlen(XATTR_SMACK_SUFFIX)) == 0)
 		return 0;
 
 	if (strcmp(name, XATTR_NAME_SMACK) == 0 ||
@@ -3348,14 +3357,20 @@ static int smack_sem_semop(struct kern_ipc_perm *isp, struct sembuf *sops,
 }
 
 /**
- * smk_curacc_msq : helper to check if current has access on msq
- * @isp : the msq
+ * smk_tskacc_msq : helper to check if tsk has access on msq
+ * @tsk: the task that requests access
+ * @isp : the sysv msg queue permissions
  * @access : access requested
  *
- * return 0 if current has access, error otherwise
+ * return 0 if tsk has access, error otherwise
  */
-static int smk_curacc_msq(struct kern_ipc_perm *isp, int access)
+static int
+smk_tskacc_msq(struct task_struct *tsk, struct kern_ipc_perm *isp, int access)
 {
+	const bool tsk_is_current = (tsk == current);
+	const struct cred * const tsk_cred =
+			(tsk_is_current ? current_cred() : get_task_cred(tsk));
+	struct task_smack * const tsp = smack_cred(tsk_cred);
 	struct smack_known *msp = smack_of_ipc(isp);
 	struct smk_audit_info ad;
 	int rc;
@@ -3363,12 +3378,28 @@ static int smk_curacc_msq(struct kern_ipc_perm *isp, int access)
 #ifdef CONFIG_AUDIT
 	smk_ad_init(&ad, __func__, LSM_AUDIT_DATA_IPC);
 	ad.a.u.ipc_id = isp->id;
+	if (!tsk_is_current)
+		ad.sad.subj_tsk = tsk;
 #endif
-	rc = smk_curacc(msp, access, &ad);
-	rc = smk_bu_current("msq", msp, access, rc);
+	rc = smk_tskacc(tsp, msp, access, &ad);
+	rc = smk_bu_tsk_to_obj(tsk, tsp, "msq", msp, access, rc);
+	if (!tsk_is_current)
+		put_cred(tsk_cred);
 	return rc;
 }
 
+/**
+ * smk_curacc_msq : helper to check if current has access on msq
+ * @isp : the sysv msg queue permissions
+ * @access : access requested
+ *
+ * return 0 if current has access, error otherwise
+ */
+static int smk_curacc_msq(struct kern_ipc_perm *isp, int access)
+{
+	return smk_tskacc_msq(current, isp, access);
+}
+
 /**
  * smack_msg_queue_associate - Smack access check for msg_queue
  * @isp: the object
@@ -3436,21 +3467,21 @@ static int smack_msg_queue_msgsnd(struct kern_ipc_perm *isp, struct msg_msg *msg
 }
 
 /**
- * smack_msg_queue_msgrcv - Smack access check for msg_queue
+ * smack_msg_queue_msgrcv - check it target has r/w access to msg_queue
  * @isp: the object
  * @msg: unused
- * @target: unused
+ * @target: the task that msgrcv() from the queue
  * @type: unused
  * @mode: unused
  *
- * Returns 0 if current has read and write access, error code otherwise
+ * Returns 0 if target has read and write access, error code otherwise
  */
 static int smack_msg_queue_msgrcv(struct kern_ipc_perm *isp,
 				  struct msg_msg *msg,
 				  struct task_struct *target, long type,
 				  int mode)
 {
-	return smk_curacc_msq(isp, MAY_READWRITE);
+	return smk_tskacc_msq(target, isp, MAY_READWRITE);
 }
 
 /**
diff --git a/security/smack/smackfs.c b/security/smack/smackfs.c
index 6e62dcb36f74f..a72bc7fabea99 100644
--- a/security/smack/smackfs.c
+++ b/security/smack/smackfs.c
@@ -83,18 +83,27 @@ static DEFINE_MUTEX(smk_net6addr_lock);
 struct smack_known *smack_net_ambient;
 
 /*
- * This is the level in a CIPSO header that indicates a
+ * Sensitivity levels for automatically created CIPSO labels.
+ * See smack_access.c`smack_populate_secattr()
+ *
+ * [0] "direct" labeling, label length < SMK_CIPSOLEN(24):
  * smack label is contained directly in the category set.
  * It can be reset via smackfs/direct
- */
-int smack_cipso_direct = SMACK_CIPSO_DIRECT_DEFAULT;
-
-/*
- * This is the level in a CIPSO header that indicates a
+ *
+ * [1] "mapped" labeling, label length >= SMK_CIPSOLEN(24):
  * secid is contained directly in the category set.
  * It can be reset via smackfs/mapped
  */
-int smack_cipso_mapped = SMACK_CIPSO_MAPPED_DEFAULT;
+u8 smack_cipso_auto_level[2] = {
+	SMACK_CIPSO_DIRECT_DEFAULT,
+	SMACK_CIPSO_MAPPED_DEFAULT,
+};
+
+static int
+smk_cipso_auto_level_idx(const struct file *file)
+{
+	return (file_inode(file)->i_ino != SMK_DIRECT);
+}
 
 #ifdef CONFIG_SECURITY_SMACK_BRINGUP
 /*
@@ -115,7 +124,7 @@ struct smack_known *smack_syslog_label;
 /*
  * Ptrace current rule
  * SMACK_PTRACE_DEFAULT    regular smack ptrace rules (/proc based)
- * SMACK_PTRACE_EXACT      labels must match, but can be overriden with
+ * SMACK_PTRACE_EXACT      labels must match, but can be overridden with
  *			   CAP_SYS_PTRACE
  * SMACK_PTRACE_DRACONIAN  labels must match, CAP_SYS_PTRACE has no effect
  */
@@ -1598,24 +1607,17 @@ static ssize_t smk_read_doi(struct file *filp, char __user *buf,
 static ssize_t smk_write_doi(struct file *file, const char __user *buf,
 			     size_t count, loff_t *ppos)
 {
-	char temp[80];
-	unsigned long u;
+	int ret;
+	u32 u;
 
 	if (!smack_privileged(CAP_MAC_ADMIN))
 		return -EPERM;
 
-	if (count >= sizeof(temp) || count == 0)
-		return -EINVAL;
-
-	if (copy_from_user(temp, buf, count) != 0)
-		return -EFAULT;
+	ret = kstrtou32_from_user(buf, count, 10, &u);
+	if (unlikely(ret))
+		return ret;
 
-	temp[count] = '\0';
-
-	if (kstrtoul(temp, 10, &u))
-		return -EINVAL;
-
-	if (u == CIPSO_V4_DOI_UNKNOWN || u > U32_MAX)
+	if (u == CIPSO_V4_DOI_UNKNOWN)
 		return -EINVAL;
 
 	return smk_cipso_doi(u, GFP_KERNEL) ? : count;
@@ -1628,158 +1630,80 @@ static const struct file_operations smk_doi_ops = {
 };
 
 /**
- * smk_read_direct - read() for /smack/direct
- * @filp: file pointer, not actually used
+ * smk_read_cipso_auto_level - read() for smackfs/direct and smackfs/mapped
+ * @filp: file pointer
  * @buf: where to put the result
  * @count: maximum to send along
  * @ppos: where to start
  *
  * Returns number of bytes read or error code, as appropriate
  */
-static ssize_t smk_read_direct(struct file *filp, char __user *buf,
+static ssize_t smk_read_cipso_auto_level(struct file *filp, char __user *buf,
 			       size_t count, loff_t *ppos)
 {
-	char temp[80];
-	ssize_t rc;
+	char temp[sizeof "255"];
+	int n;
 
 	if (*ppos != 0)
 		return 0;
 
-	sprintf(temp, "%d", smack_cipso_direct);
-	rc = simple_read_from_buffer(buf, count, ppos, temp, strlen(temp));
-
-	return rc;
+	n = sprintf(temp, "%u", (unsigned int)smack_cipso_auto_level[
+		smk_cipso_auto_level_idx(filp)]);
+	return simple_read_from_buffer(buf, count, ppos, temp, n);
 }
 
 /**
- * smk_write_direct - write() for /smack/direct
- * @file: file pointer, not actually used
+ * smk_write_cipso_auto_level - write() for smackfs/direct and smackfs/mapped
+ * @filp: file pointer
  * @buf: where to get the data from
  * @count: bytes sent
  * @ppos: where to start
  *
  * Returns number of bytes written or error code, as appropriate
  */
-static ssize_t smk_write_direct(struct file *file, const char __user *buf,
-				size_t count, loff_t *ppos)
+static ssize_t
+smk_write_cipso_auto_level(struct file *filp, const char __user *buf,
+			   size_t count, loff_t *ppos)
 {
-	struct smack_known *skp;
-	char temp[80];
-	int i;
+	int ret, idx;
+	u8  i, old_lvl;
 
 	if (!smack_privileged(CAP_MAC_ADMIN))
 		return -EPERM;
-
-	if (count >= sizeof(temp) || count == 0)
-		return -EINVAL;
-
-	if (copy_from_user(temp, buf, count) != 0)
-		return -EFAULT;
-
-	temp[count] = '\0';
-
-	if (sscanf(temp, "%d", &i) != 1)
-		return -EINVAL;
-
 	/*
-	 * Don't do anything if the value hasn't actually changed.
-	 * If it is changing reset the level on entries that were
-	 * set up to be direct when they were created.
+	 * draft-ietf-cipso-ipsecurity-01 (CIPSO 2.2), 3.4.2.4:
+	 * "Sensitivity Level is 1 octet in length. Its value is from 0 to 255"
 	 */
-	if (smack_cipso_direct != i) {
-		mutex_lock(&smack_known_lock);
-		list_for_each_entry_rcu(skp, &smack_known_list, list)
-			if (skp->smk_netlabel.attr.mls.lvl ==
-			    smack_cipso_direct)
-				skp->smk_netlabel.attr.mls.lvl = i;
-		smack_cipso_direct = i;
-		mutex_unlock(&smack_known_lock);
-	}
-
-	return count;
-}
-
-static const struct file_operations smk_direct_ops = {
-	.read		= smk_read_direct,
-	.write		= smk_write_direct,
-	.llseek		= default_llseek,
-};
-
-/**
- * smk_read_mapped - read() for /smack/mapped
- * @filp: file pointer, not actually used
- * @buf: where to put the result
- * @count: maximum to send along
- * @ppos: where to start
- *
- * Returns number of bytes read or error code, as appropriate
- */
-static ssize_t smk_read_mapped(struct file *filp, char __user *buf,
-			       size_t count, loff_t *ppos)
-{
-	char temp[80];
-	ssize_t rc;
-
-	if (*ppos != 0)
-		return 0;
-
-	sprintf(temp, "%d", smack_cipso_mapped);
-	rc = simple_read_from_buffer(buf, count, ppos, temp, strlen(temp));
-
-	return rc;
-}
-
-/**
- * smk_write_mapped - write() for /smack/mapped
- * @file: file pointer, not actually used
- * @buf: where to get the data from
- * @count: bytes sent
- * @ppos: where to start
- *
- * Returns number of bytes written or error code, as appropriate
- */
-static ssize_t smk_write_mapped(struct file *file, const char __user *buf,
-				size_t count, loff_t *ppos)
-{
-	struct smack_known *skp;
-	char temp[80];
-	int i;
-
-	if (!smack_privileged(CAP_MAC_ADMIN))
-		return -EPERM;
-
-	if (count >= sizeof(temp) || count == 0)
-		return -EINVAL;
-
-	if (copy_from_user(temp, buf, count) != 0)
-		return -EFAULT;
-
-	temp[count] = '\0';
-
-	if (sscanf(temp, "%d", &i) != 1)
-		return -EINVAL;
+	ret = kstrtou8_from_user(buf, count, 10, &i);
+	if (unlikely(ret))
+		return ret;
 
 	/*
 	 * Don't do anything if the value hasn't actually changed.
 	 * If it is changing reset the level on entries that were
-	 * set up to be mapped when they were created.
+	 * set up to be "auto" level when they were created.
 	 */
-	if (smack_cipso_mapped != i) {
+	idx = smk_cipso_auto_level_idx(filp);
+	old_lvl = smack_cipso_auto_level[idx];
+
+	if (old_lvl != i) {
+		struct smack_known *skp;
 		mutex_lock(&smack_known_lock);
 		list_for_each_entry_rcu(skp, &smack_known_list, list)
 			if (skp->smk_netlabel.attr.mls.lvl ==
-			    smack_cipso_mapped)
+			    old_lvl)
 				skp->smk_netlabel.attr.mls.lvl = i;
-		smack_cipso_mapped = i;
+		smack_cipso_auto_level[idx] = i;
 		mutex_unlock(&smack_known_lock);
 	}
 
 	return count;
 }
 
-static const struct file_operations smk_mapped_ops = {
-	.read		= smk_read_mapped,
-	.write		= smk_write_mapped,
+static const struct file_operations
+smk_cipso_auto_level_ops = {
+	.read		= smk_read_cipso_auto_level,
+	.write		= smk_write_cipso_auto_level,
 	.llseek		= default_llseek,
 };
 
@@ -2179,22 +2103,15 @@ static ssize_t smk_read_logging(struct file *filp, char __user *buf,
 static ssize_t smk_write_logging(struct file *file, const char __user *buf,
 				size_t count, loff_t *ppos)
 {
-	char temp[32];
-	int i;
+	int i, ret;
 
 	if (!smack_privileged(CAP_MAC_ADMIN))
 		return -EPERM;
 
-	if (count >= sizeof(temp) || count == 0)
-		return -EINVAL;
+	ret = kstrtos32_from_user(buf, count, 10, &i);
+	if (unlikely(ret))
+		return ret;
 
-	if (copy_from_user(temp, buf, count) != 0)
-		return -EFAULT;
-
-	temp[count] = '\0';
-
-	if (sscanf(temp, "%d", &i) != 1)
-		return -EINVAL;
 	if (i < 0 || i > 3)
 		return -EINVAL;
 	log_policy = i;
@@ -2838,22 +2755,15 @@ static ssize_t smk_read_ptrace(struct file *filp, char __user *buf,
 static ssize_t smk_write_ptrace(struct file *file, const char __user *buf,
 				size_t count, loff_t *ppos)
 {
-	char temp[32];
-	int i;
+	int i, ret;
 
 	if (!smack_privileged(CAP_MAC_ADMIN))
 		return -EPERM;
 
-	if (*ppos != 0 || count >= sizeof(temp) || count == 0)
-		return -EINVAL;
-
-	if (copy_from_user(temp, buf, count) != 0)
-		return -EFAULT;
-
-	temp[count] = '\0';
+	ret = kstrtos32_from_user(buf, count, 10, &i);
+	if (unlikely(ret))
+		return ret;
 
-	if (sscanf(temp, "%d", &i) != 1)
-		return -EINVAL;
 	if (i < SMACK_PTRACE_DEFAULT || i > SMACK_PTRACE_MAX)
 		return -EINVAL;
 	smack_ptrace_rule = i;
@@ -2888,7 +2798,7 @@ static int smk_fill_super(struct super_block *sb, struct fs_context *fc)
 		[SMK_DOI] = {
 			"doi", &smk_doi_ops, S_IRUGO|S_IWUSR},
 		[SMK_DIRECT] = {
-			"direct", &smk_direct_ops, S_IRUGO|S_IWUSR},
+			"direct", &smk_cipso_auto_level_ops, 0644},
 		[SMK_AMBIENT] = {
 			"ambient", &smk_ambient_ops, S_IRUGO|S_IWUSR},
 		[SMK_NET4ADDR] = {
@@ -2904,7 +2814,7 @@ static int smk_fill_super(struct super_block *sb, struct fs_context *fc)
 		[SMK_ACCESSES] = {
 			"access", &smk_access_ops, S_IRUGO|S_IWUGO},
 		[SMK_MAPPED] = {
-			"mapped", &smk_mapped_ops, S_IRUGO|S_IWUSR},
+			"mapped", &smk_cipso_auto_level_ops, 0644},
 		[SMK_LOAD2] = {
 			"load2", &smk_load2_ops, S_IRUGO|S_IWUSR},
 		[SMK_LOAD_SELF2] = {
diff --git a/sound/core/compress_offload.c b/sound/core/compress_offload.c
index ea699491f0c36..23d62fede06e8 100644
--- a/sound/core/compress_offload.c
+++ b/sound/core/compress_offload.c
@@ -1393,17 +1393,17 @@ static long snd_compr_ioctl_compat(struct file *file, unsigned int cmd,
 #endif
 
 static const struct file_operations snd_compr_file_ops = {
-		.owner =	THIS_MODULE,
-		.open =		snd_compr_open,
-		.release =	snd_compr_free,
-		.write =	snd_compr_write,
-		.read =		snd_compr_read,
-		.unlocked_ioctl = snd_compr_ioctl,
+		.owner		=	THIS_MODULE,
+		.open		=	snd_compr_open,
+		.release	=	snd_compr_free,
+		.write		=	snd_compr_write,
+		.read		=	snd_compr_read,
+		.unlocked_ioctl	=	snd_compr_ioctl,
 #ifdef CONFIG_COMPAT
-		.compat_ioctl = snd_compr_ioctl_compat,
+		.compat_ioctl	=	snd_compr_ioctl_compat,
 #endif
-		.mmap =		snd_compr_mmap,
-		.poll =		snd_compr_poll,
+		.mmap		=	snd_compr_mmap,
+		.poll		=	snd_compr_poll,
 };
 
 static int snd_compress_dev_register(struct snd_device *device)
diff --git a/sound/core/control.c b/sound/core/control.c
index 7a8dc506221e9..1116a40d11ae9 100644
--- a/sound/core/control.c
+++ b/sound/core/control.c
@@ -55,9 +55,7 @@ static int snd_ctl_open(struct inode *inode, struct file *file)
 	struct snd_ctl_file *ctl;
 	int i, err;
 
-	err = stream_open(inode, file);
-	if (err < 0)
-		return err;
+	stream_open(inode, file);
 
 	card = snd_lookup_minor_data(iminor(inode), SNDRV_DEVICE_TYPE_CONTROL);
 	if (!card) {
@@ -65,12 +63,10 @@ static int snd_ctl_open(struct inode *inode, struct file *file)
 		goto __error1;
 	}
 	err = snd_card_file_add(card, file);
-	if (err < 0) {
-		err = -ENODEV;
+	if (err < 0)
 		goto __error1;
-	}
 	if (!try_module_get(card->module)) {
-		err = -EFAULT;
+		err = -ENODEV;
 		goto __error2;
 	}
 	ctl = kzalloc(sizeof(*ctl), GFP_KERNEL);
@@ -2339,16 +2335,15 @@ EXPORT_SYMBOL_GPL(snd_ctl_disconnect_layer);
  *  INIT PART
  */
 
-static const struct file_operations snd_ctl_f_ops =
-{
-	.owner =	THIS_MODULE,
-	.read =		snd_ctl_read,
-	.open =		snd_ctl_open,
-	.release =	snd_ctl_release,
-	.poll =		snd_ctl_poll,
-	.unlocked_ioctl =	snd_ctl_ioctl,
-	.compat_ioctl =	snd_ctl_ioctl_compat,
-	.fasync =	snd_ctl_fasync,
+static const struct file_operations snd_ctl_f_ops = {
+	.owner		=	THIS_MODULE,
+	.read		=	snd_ctl_read,
+	.open		=	snd_ctl_open,
+	.release	=	snd_ctl_release,
+	.poll		=	snd_ctl_poll,
+	.unlocked_ioctl	=	snd_ctl_ioctl,
+	.compat_ioctl	=	snd_ctl_ioctl_compat,
+	.fasync		=	snd_ctl_fasync,
 };
 
 /* call lops under rwsems; called from snd_ctl_dev_*() below() */
diff --git a/sound/core/hwdep.c b/sound/core/hwdep.c
index 973d11732bfd0..c6de5345e946f 100644
--- a/sound/core/hwdep.c
+++ b/sound/core/hwdep.c
@@ -87,7 +87,7 @@ static int snd_hwdep_open(struct inode *inode, struct file * file)
 
 	if (!try_module_get(hw->card->module)) {
 		snd_card_unref(hw->card);
-		return -EFAULT;
+		return -ENODEV;
 	}
 
 	init_waitqueue_entry(&wait, current);
@@ -323,18 +323,17 @@ static int snd_hwdep_control_ioctl(struct snd_card *card,
 
  */
 
-static const struct file_operations snd_hwdep_f_ops =
-{
-	.owner = 	THIS_MODULE,
-	.llseek =	snd_hwdep_llseek,
-	.read = 	snd_hwdep_read,
-	.write =	snd_hwdep_write,
-	.open =		snd_hwdep_open,
-	.release =	snd_hwdep_release,
-	.poll =		snd_hwdep_poll,
-	.unlocked_ioctl =	snd_hwdep_ioctl,
-	.compat_ioctl =	snd_hwdep_ioctl_compat,
-	.mmap =		snd_hwdep_mmap,
+static const struct file_operations snd_hwdep_f_ops = {
+	.owner		=	THIS_MODULE,
+	.llseek		=	snd_hwdep_llseek,
+	.read		=	snd_hwdep_read,
+	.write		=	snd_hwdep_write,
+	.open		=	snd_hwdep_open,
+	.release	=	snd_hwdep_release,
+	.poll		=	snd_hwdep_poll,
+	.unlocked_ioctl	=	snd_hwdep_ioctl,
+	.compat_ioctl	=	snd_hwdep_ioctl_compat,
+	.mmap		=	snd_hwdep_mmap,
 };
 
 static void snd_hwdep_free(struct snd_hwdep *hwdep)
diff --git a/sound/core/info.c b/sound/core/info.c
index 54834dbe6b597..3b8ccda04e1a5 100644
--- a/sound/core/info.c
+++ b/sound/core/info.c
@@ -78,7 +78,7 @@ static int alloc_info_private(struct snd_info_entry *entry,
 	if (!entry || !entry->p)
 		return -ENODEV;
 	if (!try_module_get(entry->module))
-		return -EFAULT;
+		return -ENODEV;
 	data = kzalloc_obj(*data);
 	if (!data) {
 		module_put(entry->module);
diff --git a/sound/core/init.c b/sound/core/init.c
index 56dde5bd73c4e..8c5850ce08a0d 100644
--- a/sound/core/init.c
+++ b/sound/core/init.c
@@ -466,20 +466,19 @@ static int snd_disconnect_fasync(int fd, struct file *file, int on)
 	return -ENODEV;
 }
 
-static const struct file_operations snd_shutdown_f_ops =
-{
-	.owner = 	THIS_MODULE,
-	.llseek =	snd_disconnect_llseek,
-	.read = 	snd_disconnect_read,
-	.write =	snd_disconnect_write,
-	.release =	snd_disconnect_release,
-	.poll =		snd_disconnect_poll,
-	.unlocked_ioctl = snd_disconnect_ioctl,
+static const struct file_operations snd_shutdown_f_ops = {
+	.owner		=	THIS_MODULE,
+	.llseek		=	snd_disconnect_llseek,
+	.read		=	snd_disconnect_read,
+	.write		=	snd_disconnect_write,
+	.release	=	snd_disconnect_release,
+	.poll		=	snd_disconnect_poll,
+	.unlocked_ioctl	=	snd_disconnect_ioctl,
 #ifdef CONFIG_COMPAT
-	.compat_ioctl = snd_disconnect_ioctl,
+	.compat_ioctl	=	snd_disconnect_ioctl,
 #endif
-	.mmap =		snd_disconnect_mmap,
-	.fasync =	snd_disconnect_fasync
+	.mmap		=	snd_disconnect_mmap,
+	.fasync		=	snd_disconnect_fasync
 };
 
 /**
diff --git a/sound/core/jack.c b/sound/core/jack.c
index 96e0733ede778..0f04018181acb 100644
--- a/sound/core/jack.c
+++ b/sound/core/jack.c
@@ -303,41 +303,41 @@ static ssize_t jack_type_read(struct file *file,
 }
 
 static const struct file_operations jack_type_fops = {
-	.open = simple_open,
-	.read = jack_type_read,
-	.llseek = default_llseek,
+	.open	=	simple_open,
+	.read	=	jack_type_read,
+	.llseek	=	default_llseek,
 };
 #endif
 
 static const struct file_operations sw_inject_enable_fops = {
-	.open = simple_open,
-	.read = sw_inject_enable_read,
-	.write = sw_inject_enable_write,
-	.llseek = default_llseek,
+	.open	=	simple_open,
+	.read	=	sw_inject_enable_read,
+	.write	=	sw_inject_enable_write,
+	.llseek	=	default_llseek,
 };
 
 static const struct file_operations jackin_inject_fops = {
-	.open = simple_open,
-	.write = jackin_inject_write,
-	.llseek = default_llseek,
+	.open	=	simple_open,
+	.write	=	jackin_inject_write,
+	.llseek	=	default_llseek,
 };
 
 static const struct file_operations jack_kctl_id_fops = {
-	.open = simple_open,
-	.read = jack_kctl_id_read,
-	.llseek = default_llseek,
+	.open	=	simple_open,
+	.read	=	jack_kctl_id_read,
+	.llseek	=	default_llseek,
 };
 
 static const struct file_operations jack_kctl_mask_bits_fops = {
-	.open = simple_open,
-	.read = jack_kctl_mask_bits_read,
-	.llseek = default_llseek,
+	.open	=	simple_open,
+	.read	=	jack_kctl_mask_bits_read,
+	.llseek	=	default_llseek,
 };
 
 static const struct file_operations jack_kctl_status_fops = {
-	.open = simple_open,
-	.read = jack_kctl_status_read,
-	.llseek = default_llseek,
+	.open	=	simple_open,
+	.read	=	jack_kctl_status_read,
+	.llseek	=	default_llseek,
 };
 
 static int snd_jack_debugfs_add_inject_node(struct snd_jack *jack,
diff --git a/sound/core/memalloc.c b/sound/core/memalloc.c
index 9320671dfcc8b..5bc7e586b430f 100644
--- a/sound/core/memalloc.c
+++ b/sound/core/memalloc.c
@@ -851,7 +851,7 @@ static void snd_dma_noncoherent_free(struct snd_dma_buffer *dmab)
 static int snd_dma_noncoherent_mmap(struct snd_dma_buffer *dmab,
 				    struct vm_area_struct *area)
 {
-	area->vm_page_prot = vm_get_page_prot(area->vm_flags);
+	area->vm_page_prot = vma_get_page_prot(area);
 	return dma_mmap_pages(dmab->dev.dev, area,
 			      area->vm_end - area->vm_start,
 			      virt_to_page(dmab->area));
diff --git a/sound/core/oss/mixer_oss.c b/sound/core/oss/mixer_oss.c
index 591cff800329a..ff9d7fd60a7ea 100644
--- a/sound/core/oss/mixer_oss.c
+++ b/sound/core/oss/mixer_oss.c
@@ -30,9 +30,7 @@ static int snd_mixer_oss_open(struct inode *inode, struct file *file)
 	struct snd_mixer_oss_file *fmixer;
 	int err;
 
-	err = nonseekable_open(inode, file);
-	if (err < 0)
-		return err;
+	nonseekable_open(inode, file);
 
 	card = snd_lookup_oss_minor_data(iminor(inode),
 					 SNDRV_OSS_DEVICE_TYPE_MIXER);
@@ -60,7 +58,7 @@ static int snd_mixer_oss_open(struct inode *inode, struct file *file)
 		kfree(fmixer);
 		snd_card_file_remove(card, file);
 		snd_card_unref(card);
-		return -EFAULT;
+		return -ENODEV;
 	}
 	snd_card_unref(card);
 	return 0;
@@ -407,13 +405,12 @@ static long snd_mixer_oss_ioctl_compat(struct file *file, unsigned int cmd,
  *  REGISTRATION PART
  */
 
-static const struct file_operations snd_mixer_oss_f_ops =
-{
-	.owner =	THIS_MODULE,
-	.open =		snd_mixer_oss_open,
-	.release =	snd_mixer_oss_release,
-	.unlocked_ioctl =	snd_mixer_oss_ioctl,
-	.compat_ioctl =	snd_mixer_oss_ioctl_compat,
+static const struct file_operations snd_mixer_oss_f_ops = {
+	.owner		=	THIS_MODULE,
+	.open		=	snd_mixer_oss_open,
+	.release	=	snd_mixer_oss_release,
+	.unlocked_ioctl	=	snd_mixer_oss_ioctl,
+	.compat_ioctl	=	snd_mixer_oss_ioctl_compat,
 };
 
 /*
diff --git a/sound/core/oss/pcm_oss.c b/sound/core/oss/pcm_oss.c
index 9826d9a0be6d3..0924f1ff1ae79 100644
--- a/sound/core/oss/pcm_oss.c
+++ b/sound/core/oss/pcm_oss.c
@@ -2495,9 +2495,7 @@ static int snd_pcm_oss_open(struct inode *inode, struct file *file)
 	int nonblock;
 	wait_queue_entry_t wait;
 
-	err = nonseekable_open(inode, file);
-	if (err < 0)
-		return err;
+	nonseekable_open(inode, file);
 
 	pcm = snd_lookup_oss_minor_data(iminor(inode),
 					SNDRV_OSS_DEVICE_TYPE_PCM);
@@ -2509,7 +2507,7 @@ static int snd_pcm_oss_open(struct inode *inode, struct file *file)
 	if (err < 0)
 		goto __error1;
 	if (!try_module_get(pcm->card->module)) {
-		err = -EFAULT;
+		err = -ENODEV;
 		goto __error2;
 	}
 	if (snd_task_name(current, task_name, sizeof(task_name)) < 0) {
@@ -3129,17 +3127,16 @@ static inline void snd_pcm_oss_proc_done(struct snd_pcm *pcm)
  *  ENTRY functions
  */
 
-static const struct file_operations snd_pcm_oss_f_reg =
-{
-	.owner =	THIS_MODULE,
-	.read =		snd_pcm_oss_read,
-	.write =	snd_pcm_oss_write,
-	.open =		snd_pcm_oss_open,
-	.release =	snd_pcm_oss_release,
-	.poll =		snd_pcm_oss_poll,
-	.unlocked_ioctl =	snd_pcm_oss_ioctl,
-	.compat_ioctl =	snd_pcm_oss_ioctl_compat,
-	.mmap =		snd_pcm_oss_mmap,
+static const struct file_operations snd_pcm_oss_f_reg = {
+	.owner		=	THIS_MODULE,
+	.read		=	snd_pcm_oss_read,
+	.write		=	snd_pcm_oss_write,
+	.open		=	snd_pcm_oss_open,
+	.release	=	snd_pcm_oss_release,
+	.poll		=	snd_pcm_oss_poll,
+	.unlocked_ioctl	=	snd_pcm_oss_ioctl,
+	.compat_ioctl	=	snd_pcm_oss_ioctl_compat,
+	.mmap		=	snd_pcm_oss_mmap,
 };
 
 static void register_oss_dsp(struct snd_pcm *pcm, int index)
diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c
index 7dc0060617f1e..07f61e3386c70 100644
--- a/sound/core/pcm_native.c
+++ b/sound/core/pcm_native.c
@@ -2873,9 +2873,10 @@ static int snd_pcm_open_file(struct file *file,
 static int snd_pcm_playback_open(struct inode *inode, struct file *file)
 {
 	struct snd_pcm *pcm;
-	int err = nonseekable_open(inode, file);
-	if (err < 0)
-		return err;
+	int err;
+
+	nonseekable_open(inode, file);
+
 	pcm = snd_lookup_minor_data(iminor(inode),
 				    SNDRV_DEVICE_TYPE_PCM_PLAYBACK);
 	err = snd_pcm_open(file, pcm, SNDRV_PCM_STREAM_PLAYBACK);
@@ -2887,9 +2888,10 @@ static int snd_pcm_playback_open(struct inode *inode, struct file *file)
 static int snd_pcm_capture_open(struct inode *inode, struct file *file)
 {
 	struct snd_pcm *pcm;
-	int err = nonseekable_open(inode, file);
-	if (err < 0)
-		return err;
+	int err;
+
+	nonseekable_open(inode, file);
+
 	pcm = snd_lookup_minor_data(iminor(inode),
 				    SNDRV_DEVICE_TYPE_PCM_CAPTURE);
 	err = snd_pcm_open(file, pcm, SNDRV_PCM_STREAM_CAPTURE);
@@ -2911,7 +2913,7 @@ static int snd_pcm_open(struct file *file, struct snd_pcm *pcm, int stream)
 	if (err < 0)
 		goto __error1;
 	if (!try_module_get(pcm->card->module)) {
-		err = -EFAULT;
+		err = -ENODEV;
 		goto __error2;
 	}
 	init_waitqueue_entry(&wait, current);
@@ -3907,6 +3909,8 @@ static vm_fault_t snd_pcm_mmap_data_fault(struct vm_fault *vmf)
 	if (substream == NULL)
 		return VM_FAULT_SIGBUS;
 	runtime = substream->runtime;
+	if (runtime->state == SNDRV_PCM_STATE_DISCONNECTED)
+		return VM_FAULT_SIGBUS;
 	offset = vmf->pgoff << PAGE_SHIFT;
 	dma_bytes = PAGE_ALIGN(runtime->dma_bytes);
 	if (offset > dma_bytes - PAGE_SIZE)
@@ -4232,29 +4236,29 @@ static unsigned long snd_pcm_get_unmapped_area(struct file *file,
 
 const struct file_operations snd_pcm_f_ops[2] = {
 	{
-		.owner =		THIS_MODULE,
-		.write =		snd_pcm_write,
-		.write_iter =		snd_pcm_writev,
-		.open =			snd_pcm_playback_open,
-		.release =		snd_pcm_release,
-		.poll =			snd_pcm_poll,
-		.unlocked_ioctl =	snd_pcm_ioctl,
-		.compat_ioctl = 	snd_pcm_ioctl_compat,
-		.mmap =			snd_pcm_mmap,
-		.fasync =		snd_pcm_fasync,
-		.get_unmapped_area =	snd_pcm_get_unmapped_area,
+		.owner			=	THIS_MODULE,
+		.write			=	snd_pcm_write,
+		.write_iter		=	snd_pcm_writev,
+		.open			=	snd_pcm_playback_open,
+		.release		=	snd_pcm_release,
+		.poll			=	snd_pcm_poll,
+		.unlocked_ioctl		=	snd_pcm_ioctl,
+		.compat_ioctl		=	snd_pcm_ioctl_compat,
+		.mmap			=	snd_pcm_mmap,
+		.fasync			=	snd_pcm_fasync,
+		.get_unmapped_area	=	snd_pcm_get_unmapped_area,
 	},
 	{
-		.owner =		THIS_MODULE,
-		.read =			snd_pcm_read,
-		.read_iter =		snd_pcm_readv,
-		.open =			snd_pcm_capture_open,
-		.release =		snd_pcm_release,
-		.poll =			snd_pcm_poll,
-		.unlocked_ioctl =	snd_pcm_ioctl,
-		.compat_ioctl = 	snd_pcm_ioctl_compat,
-		.mmap =			snd_pcm_mmap,
-		.fasync =		snd_pcm_fasync,
-		.get_unmapped_area =	snd_pcm_get_unmapped_area,
+		.owner			=	THIS_MODULE,
+		.read			=	snd_pcm_read,
+		.read_iter		=	snd_pcm_readv,
+		.open			=	snd_pcm_capture_open,
+		.release		=	snd_pcm_release,
+		.poll			=	snd_pcm_poll,
+		.unlocked_ioctl		=	snd_pcm_ioctl,
+		.compat_ioctl		=	snd_pcm_ioctl_compat,
+		.mmap			=	snd_pcm_mmap,
+		.fasync			=	snd_pcm_fasync,
+		.get_unmapped_area	=	snd_pcm_get_unmapped_area,
 	}
 };
diff --git a/sound/core/rawmidi.c b/sound/core/rawmidi.c
index 4dfd9d53e6d3a..1d55da2dcb01f 100644
--- a/sound/core/rawmidi.c
+++ b/sound/core/rawmidi.c
@@ -441,9 +441,7 @@ static int snd_rawmidi_open(struct inode *inode, struct file *file)
 	if ((file->f_flags & O_APPEND) && !(file->f_flags & O_NONBLOCK))
 		return -EINVAL;		/* invalid combination */
 
-	err = stream_open(inode, file);
-	if (err < 0)
-		return err;
+	stream_open(inode, file);
 
 	if (maj == snd_major) {
 		rmidi = snd_lookup_minor_data(iminor(inode),
@@ -1784,14 +1782,14 @@ static void snd_rawmidi_proc_info_read(struct snd_info_entry *entry,
  */
 
 static const struct file_operations snd_rawmidi_f_ops = {
-	.owner =	THIS_MODULE,
-	.read =		snd_rawmidi_read,
-	.write =	snd_rawmidi_write,
-	.open =		snd_rawmidi_open,
-	.release =	snd_rawmidi_release,
-	.poll =		snd_rawmidi_poll,
-	.unlocked_ioctl =	snd_rawmidi_ioctl,
-	.compat_ioctl =	snd_rawmidi_ioctl_compat,
+	.owner		=	THIS_MODULE,
+	.read		=	snd_rawmidi_read,
+	.write		=	snd_rawmidi_write,
+	.open		=	snd_rawmidi_open,
+	.release	=	snd_rawmidi_release,
+	.poll		=	snd_rawmidi_poll,
+	.unlocked_ioctl	=	snd_rawmidi_ioctl,
+	.compat_ioctl	=	snd_rawmidi_ioctl_compat,
 };
 
 static int snd_rawmidi_alloc_substreams(struct snd_rawmidi *rmidi,
diff --git a/sound/core/seq/oss/seq_oss.c b/sound/core/seq/oss/seq_oss.c
index 021cd70f90db1..2835576040ed4 100644
--- a/sound/core/seq/oss/seq_oss.c
+++ b/sound/core/seq/oss/seq_oss.c
@@ -206,17 +206,16 @@ odev_poll(struct file *file, poll_table * wait)
  * registration of sequencer minor device
  */
 
-static const struct file_operations seq_oss_f_ops =
-{
-	.owner =	THIS_MODULE,
-	.read =		odev_read,
-	.write =	odev_write,
-	.open =		odev_open,
-	.release =	odev_release,
-	.poll =		odev_poll,
-	.unlocked_ioctl =	odev_ioctl,
-	.compat_ioctl =	odev_ioctl_compat,
-	.llseek =	noop_llseek,
+static const struct file_operations seq_oss_f_ops = {
+	.owner		=	THIS_MODULE,
+	.read		=	odev_read,
+	.write		=	odev_write,
+	.open		=	odev_open,
+	.release	=	odev_release,
+	.poll		=	odev_poll,
+	.unlocked_ioctl	=	odev_ioctl,
+	.compat_ioctl	=	odev_ioctl_compat,
+	.llseek		=	noop_llseek,
 };
 
 static int __init
diff --git a/sound/core/seq/seq_clientmgr.c b/sound/core/seq/seq_clientmgr.c
index 28782e1776fa8..23ec239640c33 100644
--- a/sound/core/seq/seq_clientmgr.c
+++ b/sound/core/seq/seq_clientmgr.c
@@ -296,11 +296,8 @@ static int snd_seq_open(struct inode *inode, struct file *file)
 	int c, mode;			/* client id */
 	struct snd_seq_client *client;
 	struct snd_seq_user_client *user;
-	int err;
 
-	err = stream_open(inode, file);
-	if (err < 0)
-		return err;
+	stream_open(inode, file);
 
 	scoped_guard(mutex, &register_mutex) {
 		client = seq_create_client1(-1, SNDRV_SEQ_DEFAULT_EVENTS);
@@ -2666,16 +2663,15 @@ void snd_seq_info_clients_read(struct snd_info_entry *entry,
  *  REGISTRATION PART
  */
 
-static const struct file_operations snd_seq_f_ops =
-{
-	.owner =	THIS_MODULE,
-	.read =		snd_seq_read,
-	.write =	snd_seq_write,
-	.open =		snd_seq_open,
-	.release =	snd_seq_release,
-	.poll =		snd_seq_poll,
-	.unlocked_ioctl =	snd_seq_ioctl,
-	.compat_ioctl =	snd_seq_ioctl_compat,
+static const struct file_operations snd_seq_f_ops = {
+	.owner		=	THIS_MODULE,
+	.read		=	snd_seq_read,
+	.write		=	snd_seq_write,
+	.open		=	snd_seq_open,
+	.release	=	snd_seq_release,
+	.poll		=	snd_seq_poll,
+	.unlocked_ioctl	=	snd_seq_ioctl,
+	.compat_ioctl	=	snd_seq_ioctl_compat,
 };
 
 static struct device *seq_dev;
diff --git a/sound/core/seq/seq_timer.c b/sound/core/seq/seq_timer.c
index 4cd7211ccf487..419288eec4bb3 100644
--- a/sound/core/seq/seq_timer.c
+++ b/sound/core/seq/seq_timer.c
@@ -61,12 +61,23 @@ struct snd_seq_timer *snd_seq_timer_new(void)
 void snd_seq_timer_delete(struct snd_seq_timer **tmr)
 {
 	struct snd_seq_timer *t = *tmr;
-	*tmr = NULL;
+	struct snd_timer_instance *ti;
 
 	if (t == NULL) {
 		pr_debug("ALSA: seq: snd_seq_timer_delete() called with NULL timer\n");
 		return;
 	}
+
+	scoped_guard(spinlock_irq, &t->lock) {
+		ti = t->timeri;
+		t->timeri = NULL;
+	}
+	if (ti) {
+		snd_timer_close(ti);
+		snd_timer_instance_free(ti);
+	}
+
+	*tmr = NULL;
 	t->running = 0;
 
 	/* reset time */
diff --git a/sound/core/sound.c b/sound/core/sound.c
index 8d05fe0d263b2..24f40763a20c3 100644
--- a/sound/core/sound.c
+++ b/sound/core/sound.c
@@ -167,11 +167,10 @@ static int snd_open(struct inode *inode, struct file *file)
 	return err;
 }
 
-static const struct file_operations snd_fops =
-{
-	.owner =	THIS_MODULE,
-	.open =		snd_open,
-	.llseek =	noop_llseek,
+static const struct file_operations snd_fops = {
+	.owner	=	THIS_MODULE,
+	.open	=	snd_open,
+	.llseek	=	noop_llseek,
 };
 
 #ifdef CONFIG_SND_DYNAMIC_MINORS
@@ -358,7 +357,7 @@ static void snd_minor_info_read(struct snd_info_entry *entry, struct snd_info_bu
 	struct snd_minor *mptr;
 
 	guard(mutex)(&sound_mutex);
-	for (minor = 0; minor < SNDRV_OS_MINORS; ++minor) {
+	for (minor = 0; minor < ARRAY_SIZE(snd_minors); ++minor) {
 		mptr = snd_minors[minor];
 		if (!mptr)
 			continue;
@@ -395,15 +394,21 @@ int __init snd_minor_info_init(void)
 
 static int __init alsa_sound_init(void)
 {
+	int err;
+
 	snd_major = major;
 	snd_ecards_limit = cards_limit;
-	if (register_chrdev(major, "alsa", &snd_fops)) {
+
+	err = register_chrdev(major, "alsa", &snd_fops);
+	if (err < 0) {
 		pr_err("ALSA core: unable to register native major device number %d\n", major);
-		return -EIO;
+		return err;
 	}
-	if (snd_info_init() < 0) {
+
+	err = snd_info_init();
+	if (err < 0) {
 		unregister_chrdev(major, "alsa");
-		return -ENOMEM;
+		return err;
 	}
 
 #ifdef CONFIG_SND_DEBUG
diff --git a/sound/core/timer.c b/sound/core/timer.c
index 51c6ac4df9f47..13d73788324ea 100644
--- a/sound/core/timer.c
+++ b/sound/core/timer.c
@@ -1531,11 +1531,8 @@ static int realloc_user_queue(struct snd_timer_user *tu, int size)
 static int snd_timer_user_open(struct inode *inode, struct file *file)
 {
 	struct snd_timer_user *tu;
-	int err;
 
-	err = stream_open(inode, file);
-	if (err < 0)
-		return err;
+	stream_open(inode, file);
 
 	tu = kzalloc_obj(*tu);
 	if (tu == NULL)
@@ -2161,9 +2158,9 @@ static long snd_utimer_ioctl(struct file *file, unsigned int ioctl, unsigned lon
 }
 
 static const struct file_operations snd_utimer_fops = {
-	.llseek = noop_llseek,
-	.release = snd_utimer_release,
-	.unlocked_ioctl = snd_utimer_ioctl,
+	.llseek		=	noop_llseek,
+	.release	=	snd_utimer_release,
+	.unlocked_ioctl	=	snd_utimer_ioctl,
 };
 
 static int snd_utimer_start(struct snd_timer *t)
@@ -2515,16 +2512,15 @@ static __poll_t snd_timer_user_poll(struct file *file, poll_table * wait)
 #define snd_timer_user_ioctl_compat	NULL
 #endif
 
-static const struct file_operations snd_timer_f_ops =
-{
-	.owner =	THIS_MODULE,
-	.read =		snd_timer_user_read,
-	.open =		snd_timer_user_open,
-	.release =	snd_timer_user_release,
-	.poll =		snd_timer_user_poll,
-	.unlocked_ioctl =	snd_timer_user_ioctl,
-	.compat_ioctl =	snd_timer_user_ioctl_compat,
-	.fasync = 	snd_timer_user_fasync,
+static const struct file_operations snd_timer_f_ops = {
+	.owner		=	THIS_MODULE,
+	.read		=	snd_timer_user_read,
+	.open		=	snd_timer_user_open,
+	.release	=	snd_timer_user_release,
+	.poll		=	snd_timer_user_poll,
+	.unlocked_ioctl	=	snd_timer_user_ioctl,
+	.compat_ioctl	=	snd_timer_user_ioctl_compat,
+	.fasync		=	snd_timer_user_fasync,
 };
 
 /* unregister the system timer */
diff --git a/sound/drivers/aloop.c b/sound/drivers/aloop.c
index 06bfe09eae1a2..7ebe8e5c93039 100644
--- a/sound/drivers/aloop.c
+++ b/sound/drivers/aloop.c
@@ -23,6 +23,7 @@
 #include <linux/wait.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/hrtimer.h>
 #include <sound/core.h>
 #include <sound/control.h>
 #include <sound/pcm.h>
@@ -55,7 +56,7 @@ MODULE_PARM_DESC(pcm_substreams, "PCM substreams # (1-8) for loopback driver.");
 module_param_array(pcm_notify, int, NULL, 0444);
 MODULE_PARM_DESC(pcm_notify, "Break capture when PCM format/rate/channels changes.");
 module_param_array(timer_source, charp, NULL, 0444);
-MODULE_PARM_DESC(timer_source, "Sound card name or number and device/subdevice number of timer to be used. Empty string for jiffies timer [default].");
+MODULE_PARM_DESC(timer_source, "Sound card name or number and device/subdevice number of timer to be used. Empty string for jiffies timer [default], 'hrtimer' for high-resolution timer.");
 
 #define NO_PITCH 100000
 
@@ -161,8 +162,11 @@ struct loopback_pcm {
 	unsigned int period_size_frac;	/* period size in jiffies ticks */
 	unsigned int last_drift;
 	unsigned long last_jiffies;
-	/* If jiffies timer is used */
+	/* If jiffies / hrtimer is used */
 	struct timer_list timer;
+#ifdef CONFIG_HIGH_RES_TIMERS
+	struct hrtimer hrtimer;
+#endif
 
 	/* size of per channel buffer in case of non-interleaved access */
 	unsigned int channel_buf_n;
@@ -232,6 +236,31 @@ static int loopback_jiffies_timer_start(struct loopback_pcm *dpcm)
 	return 0;
 }
 
+#ifdef CONFIG_HIGH_RES_TIMERS
+/* call in cable->lock */
+static int loopback_hrtimer_start(struct loopback_pcm *dpcm)
+{
+	unsigned long tick;
+	unsigned int rate_shift = get_rate_shift(dpcm);
+
+	if (rate_shift != dpcm->pcm_rate_shift) {
+		dpcm->pcm_rate_shift = rate_shift;
+		dpcm->period_size_frac = frac_pos(dpcm, dpcm->pcm_period_size);
+	}
+	if (dpcm->period_size_frac <= dpcm->irq_pos) {
+		dpcm->irq_pos %= dpcm->period_size_frac;
+		dpcm->period_update_pending = 1;
+	}
+	tick = dpcm->period_size_frac - dpcm->irq_pos;
+	tick = DIV_ROUND_UP(tick, dpcm->pcm_bps);
+	hrtimer_start(&dpcm->hrtimer,
+		      ns_to_ktime(div_u64((u64)tick * NSEC_PER_SEC, HZ)),
+		      HRTIMER_MODE_REL_SOFT);
+
+	return 0;
+}
+#endif
+
 /* call in cable->lock */
 static int loopback_snd_timer_start(struct loopback_pcm *dpcm)
 {
@@ -270,6 +299,16 @@ static inline int loopback_jiffies_timer_stop(struct loopback_pcm *dpcm)
 	return 0;
 }
 
+#ifdef CONFIG_HIGH_RES_TIMERS
+/* call in cable->lock */
+static inline int loopback_hrtimer_stop(struct loopback_pcm *dpcm)
+{
+	hrtimer_cancel(&dpcm->hrtimer);
+
+	return 0;
+}
+#endif
+
 /* call in cable->lock */
 static int loopback_snd_timer_stop(struct loopback_pcm *dpcm)
 {
@@ -300,6 +339,15 @@ static inline int loopback_jiffies_timer_stop_sync(struct loopback_pcm *dpcm)
 	return 0;
 }
 
+#ifdef CONFIG_HIGH_RES_TIMERS
+static inline int loopback_hrtimer_stop_sync(struct loopback_pcm *dpcm)
+{
+	hrtimer_cancel(&dpcm->hrtimer);
+
+	return 0;
+}
+#endif
+
 /* call in loopback->cable_lock */
 static int loopback_snd_timer_close_cable(struct loopback_pcm *dpcm)
 {
@@ -734,6 +782,30 @@ static void loopback_jiffies_timer_function(struct timer_list *t)
 		snd_pcm_period_elapsed(dpcm->substream);
 }
 
+#ifdef CONFIG_HIGH_RES_TIMERS
+static enum hrtimer_restart loopback_hrtimer_function(struct hrtimer *t)
+{
+	struct loopback_pcm *dpcm = container_of(t, struct loopback_pcm, hrtimer);
+	bool period_elapsed = false;
+
+	scoped_guard(spinlock_irqsave, &dpcm->cable->lock) {
+		if (loopback_jiffies_timer_pos_update(dpcm->cable) &
+		    (1 << dpcm->substream->stream)) {
+			loopback_hrtimer_start(dpcm);
+			if (dpcm->period_update_pending) {
+				dpcm->period_update_pending = 0;
+				period_elapsed = true;
+			}
+		}
+	}
+
+	if (period_elapsed)
+		snd_pcm_period_elapsed(dpcm->substream);
+
+	return HRTIMER_NORESTART;
+}
+#endif
+
 /* call in cable->lock */
 static int loopback_snd_timer_check_resolution(struct snd_pcm_runtime *runtime,
 					       unsigned long resolution)
@@ -907,6 +979,21 @@ static void loopback_jiffies_timer_dpcm_info(struct loopback_pcm *dpcm,
 	snd_iprintf(buffer, "    timer_expires:\t%lu\n", dpcm->timer.expires);
 }
 
+#ifdef CONFIG_HIGH_RES_TIMERS
+static void loopback_hrtimer_dpcm_info(struct loopback_pcm *dpcm,
+				       struct snd_info_buffer *buffer)
+{
+	snd_iprintf(buffer, "    update_pending:\t%u\n",
+		    dpcm->period_update_pending);
+	snd_iprintf(buffer, "    irq_pos:\t\t%u\n", dpcm->irq_pos);
+	snd_iprintf(buffer, "    period_frac:\t%u\n", dpcm->period_size_frac);
+	snd_iprintf(buffer, "    last_jiffies:\t%lu (%lu)\n",
+		    dpcm->last_jiffies, jiffies);
+	snd_iprintf(buffer, "    timer_expires:\t%llu\n",
+		    ktime_to_ns(hrtimer_get_expires(&dpcm->hrtimer)));
+}
+#endif
+
 static void loopback_snd_timer_dpcm_info(struct loopback_pcm *dpcm,
 					 struct snd_info_buffer *buffer)
 {
@@ -1097,6 +1184,26 @@ static const struct loopback_ops loopback_jiffies_timer_ops = {
 	.dpcm_info = loopback_jiffies_timer_dpcm_info,
 };
 
+#ifdef CONFIG_HIGH_RES_TIMERS
+static int loopback_hrtimer_open(struct loopback_pcm *dpcm)
+{
+	hrtimer_setup(&dpcm->hrtimer, loopback_hrtimer_function,
+		      CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT);
+
+	return 0;
+}
+
+static const struct loopback_ops loopback_hrtimer_ops = {
+	.open = loopback_hrtimer_open,
+	.start = loopback_hrtimer_start,
+	.stop = loopback_hrtimer_stop,
+	.stop_sync = loopback_hrtimer_stop_sync,
+	.close_substream = loopback_hrtimer_stop_sync,
+	.pos_update = loopback_jiffies_timer_pos_update,
+	.dpcm_info = loopback_hrtimer_dpcm_info,
+};
+#endif
+
 static int loopback_parse_timer_id(const char *str,
 				   struct snd_timer_id *tid)
 {
@@ -1274,7 +1381,12 @@ static int loopback_open(struct snd_pcm_substream *substream)
 		spin_lock_init(&cable->lock);
 		snd_refcount_init(&cable->stop_count);
 		cable->hw = loopback_pcm_hardware;
-		if (loopback->timer_source)
+#ifdef CONFIG_HIGH_RES_TIMERS
+		if (loopback->timer_source && !strcmp(loopback->timer_source, "hrtimer"))
+			cable->ops = &loopback_hrtimer_ops;
+		else
+#endif
+		if (loopback->timer_source && loopback->timer_source[0])
 			cable->ops = &loopback_snd_timer_ops;
 		else
 			cable->ops = &loopback_jiffies_timer_ops;
@@ -1556,7 +1668,7 @@ static int loopback_channels_get(struct snd_kcontrol *kcontrol,
 static int loopback_access_info(struct snd_kcontrol *kcontrol,
 				struct snd_ctl_elem_info *uinfo)
 {
-	const char * const texts[] = {"Interleaved", "Non-interleaved"};
+	static const char * const texts[] = {"Interleaved", "Non-interleaved"};
 
 	return snd_ctl_enum_info(uinfo, 1, ARRAY_SIZE(texts), texts);
 }
diff --git a/sound/drivers/dummy.c b/sound/drivers/dummy.c
index 7283f0f18813d..ce7ab986dee66 100644
--- a/sound/drivers/dummy.c
+++ b/sound/drivers/dummy.c
@@ -13,6 +13,7 @@
 #include <linux/time.h>
 #include <linux/wait.h>
 #include <linux/hrtimer.h>
+#include <linux/hrtimer_bases.h>
 #include <linux/math64.h>
 #include <linux/module.h>
 #include <sound/core.h>
diff --git a/sound/firewire/bebob/bebob.h b/sound/firewire/bebob/bebob.h
index 4d73ecb30d79b..41e019b9e24e5 100644
--- a/sound/firewire/bebob/bebob.h
+++ b/sound/firewire/bebob/bebob.h
@@ -13,7 +13,6 @@
 #include <linux/firewire.h>
 #include <linux/firewire-constants.h>
 #include <linux/module.h>
-#include <linux/mod_devicetable.h>
 #include <linux/delay.h>
 #include <linux/slab.h>
 #include <linux/sched/signal.h>
diff --git a/sound/firewire/dice/dice.h b/sound/firewire/dice/dice.h
index 7744ea6a0791d..5abae26d2bba2 100644
--- a/sound/firewire/dice/dice.h
+++ b/sound/firewire/dice/dice.h
@@ -17,7 +17,6 @@
 #include <linux/firewire-constants.h>
 #include <linux/jiffies.h>
 #include <linux/module.h>
-#include <linux/mod_devicetable.h>
 #include <linux/mutex.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
diff --git a/sound/firewire/digi00x/digi00x.h b/sound/firewire/digi00x/digi00x.h
index 82b647d383c5c..60d17a6fddd1b 100644
--- a/sound/firewire/digi00x/digi00x.h
+++ b/sound/firewire/digi00x/digi00x.h
@@ -12,7 +12,6 @@
 #include <linux/device.h>
 #include <linux/firewire.h>
 #include <linux/module.h>
-#include <linux/mod_devicetable.h>
 #include <linux/delay.h>
 #include <linux/slab.h>
 #include <linux/sched/signal.h>
diff --git a/sound/firewire/fireface/ff.h b/sound/firewire/fireface/ff.h
index 7e42f5778a8a3..c9c35c67db27d 100644
--- a/sound/firewire/fireface/ff.h
+++ b/sound/firewire/fireface/ff.h
@@ -12,7 +12,6 @@
 #include <linux/firewire.h>
 #include <linux/firewire-constants.h>
 #include <linux/module.h>
-#include <linux/mod_devicetable.h>
 #include <linux/mutex.h>
 #include <linux/slab.h>
 #include <linux/compat.h>
diff --git a/sound/firewire/fireworks/fireworks.h b/sound/firewire/fireworks/fireworks.h
index c8d5879efe286..a8aadf754e72f 100644
--- a/sound/firewire/fireworks/fireworks.h
+++ b/sound/firewire/fireworks/fireworks.h
@@ -13,7 +13,6 @@
 #include <linux/firewire.h>
 #include <linux/firewire-constants.h>
 #include <linux/module.h>
-#include <linux/mod_devicetable.h>
 #include <linux/delay.h>
 #include <linux/slab.h>
 #include <linux/sched/signal.h>
diff --git a/sound/firewire/motu/motu.h b/sound/firewire/motu/motu.h
index c66be0a89ccf1..7333a76b50aa1 100644
--- a/sound/firewire/motu/motu.h
+++ b/sound/firewire/motu/motu.h
@@ -12,7 +12,6 @@
 #include <linux/firewire.h>
 #include <linux/firewire-constants.h>
 #include <linux/module.h>
-#include <linux/mod_devicetable.h>
 #include <linux/mutex.h>
 #include <linux/slab.h>
 #include <linux/compat.h>
diff --git a/sound/firewire/oxfw/oxfw.h b/sound/firewire/oxfw/oxfw.h
index 39ea9a6dde33a..aeb83e1595b71 100644
--- a/sound/firewire/oxfw/oxfw.h
+++ b/sound/firewire/oxfw/oxfw.h
@@ -9,7 +9,6 @@
 #include <linux/firewire.h>
 #include <linux/firewire-constants.h>
 #include <linux/module.h>
-#include <linux/mod_devicetable.h>
 #include <linux/mutex.h>
 #include <linux/slab.h>
 #include <linux/compat.h>
diff --git a/sound/firewire/tascam/tascam.h b/sound/firewire/tascam/tascam.h
index d07ffcb27be6e..9c42c9b48f214 100644
--- a/sound/firewire/tascam/tascam.h
+++ b/sound/firewire/tascam/tascam.h
@@ -12,7 +12,6 @@
 #include <linux/firewire.h>
 #include <linux/firewire-constants.h>
 #include <linux/module.h>
-#include <linux/mod_devicetable.h>
 #include <linux/mutex.h>
 #include <linux/slab.h>
 #include <linux/compat.h>
diff --git a/sound/hda/codecs/cm9825.c b/sound/hda/codecs/cm9825.c
index eeb9ca38d2d8b..8a74862d8a813 100644
--- a/sound/hda/codecs/cm9825.c
+++ b/sound/hda/codecs/cm9825.c
@@ -15,7 +15,8 @@
 
 enum {
 	QUIRK_CM_STD = 0x0,
-	QUIRK_GENE_TWL7_SSID = 0x160dc000
+	QUIRK_GENE_TWL7_SSID = 0x160dc000,
+	QUIRK_IBP_SSID = 0x15bd3275
 };
 
 /* CM9825 Offset Definitions */
@@ -40,13 +41,19 @@ enum {
 #define CM9825_VERB_SET_GAD 0x7b4
 #define CM9825_VERB_SET_TMOD 0x7b5
 #define CM9825_VERB_SET_SNR 0x7b6
+#define CM9825_VERB_SET_OMTP 0x7ef
+#define CM9825_VERB_READ_OMTP 0xfec
 
 struct cmi_spec {
 	struct hda_gen_spec gen;
 	const struct hda_verb *chip_d0_verbs;
 	const struct hda_verb *chip_d3_verbs;
+	const struct hda_verb *chip_playback_start_verbs;
+	const struct hda_verb *chip_playback_stop_verbs;
 	const struct hda_verb *chip_hp_present_verbs;
 	const struct hda_verb *chip_hp_remove_verbs;
+	const struct hda_verb *chip_lineout_present_verbs;
+	const struct hda_verb *chip_lineout_remove_verbs;
 	struct hda_codec *codec;
 	struct delayed_work unsol_inputs_work;
 	struct delayed_work unsol_lineout_work;
@@ -193,6 +200,94 @@ static const struct hda_verb cm9825_gene_twl7_playback_stop_verbs[] = {
 	{}
 };
 
+/*
+ * To save power, AD/CLK is turned off.
+ */
+static const struct hda_verb cm9825_ibp_d3_verbs[] = {
+	{0x43, CM9825_VERB_SET_D2S, 0x62},
+	{0x43, CM9825_VERB_SET_PLL, 0x01},
+	{0x43, CM9825_VERB_SET_NEG, 0xc2},
+	{0x43, CM9825_VERB_SET_ADCL, 0x00},
+	{0x43, CM9825_VERB_SET_DACL, 0x02},
+	{0x43, CM9825_VERB_SET_VNEG, 0x50},
+	{0x43, CM9825_VERB_SET_MBIAS, 0x00},
+	{0x43, CM9825_VERB_SET_PDNEG, 0x04},
+	{0x43, CM9825_VERB_SET_CDALR, 0xf6},
+	{0x43, CM9825_VERB_SET_OTP, 0xcd},
+	{}
+};
+
+/*
+ * D0 configuration: enable PLL/CLK/ADC/DAC and optimize performance
+ */
+static const struct hda_verb cm9825_ibp_d0_verbs[] = {
+	{0x34, AC_VERB_SET_EAPD_BTLENABLE, 0x02},
+	{0x43, CM9825_VERB_SET_SNR, 0x38},
+	{0x43, CM9825_VERB_SET_PLL, 0x00},
+	{0x43, CM9825_VERB_SET_ADCL, 0x00},
+	{0x43, CM9825_VERB_SET_DACL, 0x02},
+	{0x43, CM9825_VERB_SET_MBIAS, 0x00},
+	{0x43, CM9825_VERB_SET_VNEG, 0x56},
+	{0x43, CM9825_VERB_SET_D2S, 0x62},
+	{0x43, CM9825_VERB_SET_DACTRL, 0x00},
+	{0x43, CM9825_VERB_SET_PDNEG, 0x0c},
+	{0x43, CM9825_VERB_SET_CDALR, 0xf4},
+	{0x43, CM9825_VERB_SET_OTP, 0xcd},
+	{0x43, CM9825_VERB_SET_MTCBA, 0x61},
+	{0x43, CM9825_VERB_SET_OCP, 0x33},
+	{0x43, CM9825_VERB_SET_GAD, 0x07},
+	{0x43, CM9825_VERB_SET_TMOD, 0x26},
+	{0x3c, AC_VERB_SET_AMP_GAIN_MUTE | 0xa0, 0x2d},
+	{0x3c, AC_VERB_SET_AMP_GAIN_MUTE | 0x90, 0x2d},
+	{0x43, CM9825_VERB_SET_HPF_1, 0x40},
+	{0x43, CM9825_VERB_SET_HPF_2, 0x40},
+	{}
+};
+
+/*
+ * Enable mbias, ADC.
+ */
+static const struct hda_verb cm9825_ibp_lineout_present_verbs[] = {
+	{0x43, CM9825_VERB_SET_ADCL, 0x8c},
+	{0x43, CM9825_VERB_SET_MBIAS, 0x10},
+	{}
+};
+
+/*
+ * Disable mbias, ADC.
+ */
+static const struct hda_verb cm9825_ibp_lineout_remove_verbs[] = {
+	{0x43, CM9825_VERB_SET_ADCL, 0x00},
+	{0x43, CM9825_VERB_SET_MBIAS, 0x00},
+	{}
+};
+
+/*
+ * Turn on the DAC (widget NID 0x43) in the playback path by writing
+ * a sequence of vendor-specific verbs.
+ */
+static const struct hda_verb cm9825_ibp_playback_start_verbs[] = {
+	{0x43, CM9825_VERB_SET_DACL, 0xaa},
+	{0x43, CM9825_VERB_SET_D2S, 0xf2},
+	{0x43, CM9825_VERB_SET_VDO, 0xc4},
+	{0x43, CM9825_VERB_SET_SNR, 0x30},
+	{}
+};
+
+/*
+ * Shut down the playback path. The order differs slightly from the
+ * enable sequence, likely to avoid audible pop noise when powering
+ * down the output stage.
+ */
+static const struct hda_verb cm9825_ibp_playback_stop_verbs[] = {
+	{0x43, CM9825_VERB_SET_VDO, 0xc0},
+	{0x43, CM9825_VERB_SET_DACL, 0x02},
+	{0x43, CM9825_VERB_SET_D2S, 0x62},
+	{0x43, CM9825_VERB_SET_VDO, 0x80},
+	{0x43, CM9825_VERB_SET_SNR, 0x38},
+	{}
+};
+
 static void cm9825_update_jk_plug_status(struct hda_codec *codec, hda_nid_t nid)
 {
 	struct cmi_spec *spec = codec->spec;
@@ -232,6 +327,23 @@ static void cm9825_unsol_lineout_delayed(struct work_struct *work)
 	struct cmi_spec *spec =
 	    container_of(to_delayed_work(work), struct cmi_spec,
 			 unsol_lineout_work);
+	hda_nid_t line_out_pin = spec->gen.autocfg.line_out_pins[0];
+	bool line_out_jack_plugin = false;
+
+	line_out_jack_plugin = snd_hda_jack_detect(spec->codec, line_out_pin);
+
+	codec_dbg(spec->codec, "lineout_jack_plugin %d, lineout_pin 0x%X\n",
+		  (int)line_out_jack_plugin, line_out_pin);
+
+	if (!line_out_jack_plugin) {
+		/* Jack plugout */
+		snd_hda_sequence_write(spec->codec,
+				       spec->chip_lineout_remove_verbs);
+	} else {
+		/* Jack plugin */
+		snd_hda_sequence_write(spec->codec,
+				       spec->chip_lineout_present_verbs);
+	}
 
 	cm9825_update_jk_plug_status(spec->codec,
 				     spec->gen.autocfg.line_out_pins[0]);
@@ -362,12 +474,10 @@ static void cm9825_playback_pcm_hook(struct hda_pcm_stream *hinfo,
 
 	switch (action) {
 	case HDA_GEN_PCM_ACT_PREPARE:
-		snd_hda_sequence_write(spec->codec,
-				       cm9825_gene_twl7_playback_start_verbs);
+		snd_hda_sequence_write(codec, spec->chip_playback_start_verbs);
 		break;
 	case HDA_GEN_PCM_ACT_CLEANUP:
-		snd_hda_sequence_write(spec->codec,
-				       cm9825_gene_twl7_playback_stop_verbs);
+		snd_hda_sequence_write(codec, spec->chip_playback_stop_verbs);
 		break;
 	default:
 		return;
@@ -471,7 +581,8 @@ static int cm9825_resume(struct hda_codec *codec)
 
 	if (codec->core.subsystem_id == QUIRK_CM_STD)
 		cm9825_cm_std_resume(codec);
-	else if (codec->core.subsystem_id == QUIRK_GENE_TWL7_SSID) {
+	else if (codec->core.subsystem_id == QUIRK_GENE_TWL7_SSID ||
+		 codec->core.subsystem_id == QUIRK_IBP_SSID) {
 		snd_hda_codec_init(codec);
 		snd_hda_sequence_write(codec, spec->chip_d0_verbs);
 	}
@@ -487,6 +598,7 @@ static int cm9825_probe(struct hda_codec *codec, const struct hda_device_id *id)
 	struct cmi_spec *spec;
 	struct auto_pin_cfg *cfg;
 	int err = 0;
+	unsigned int val;
 
 	spec = kzalloc_obj(*spec);
 	if (spec == NULL)
@@ -523,9 +635,33 @@ static int cm9825_probe(struct hda_codec *codec, const struct hda_device_id *id)
 		spec->chip_d0_verbs = cm9825_gene_twl7_d0_verbs;
 		spec->chip_d3_verbs = cm9825_gene_twl7_d3_verbs;
 		spec->gen.pcm_playback_hook = cm9825_playback_pcm_hook;
+		spec->chip_playback_start_verbs =
+		    cm9825_gene_twl7_playback_start_verbs;
+		spec->chip_playback_stop_verbs =
+		    cm9825_gene_twl7_playback_stop_verbs;
 		/* Internal fixed device, Rear, Mic-in, 3.5mm */
 		snd_hda_codec_set_pincfg(codec, 0x37, 0x24A70100);
 		break;
+	case QUIRK_IBP_SSID:
+		snd_hda_codec_set_name(codec, "CM9825 IBP");
+		spec->chip_d0_verbs = cm9825_ibp_d0_verbs;
+		spec->chip_d3_verbs = cm9825_ibp_d3_verbs;
+		spec->gen.pcm_playback_hook = cm9825_playback_pcm_hook;
+		spec->chip_lineout_present_verbs =
+		    cm9825_ibp_lineout_present_verbs;
+		spec->chip_lineout_remove_verbs =
+		    cm9825_ibp_lineout_remove_verbs;
+		spec->chip_playback_start_verbs =
+		    cm9825_ibp_playback_start_verbs;
+		spec->chip_playback_stop_verbs = cm9825_ibp_playback_stop_verbs;
+
+		/* OMTP */
+		val =
+		    snd_hda_codec_read(codec, 0x46, 0, CM9825_VERB_READ_OMTP,
+				       0x0);
+		snd_hda_codec_write(codec, 0x46, 0, CM9825_VERB_SET_OMTP,
+				    (val >> 24) & 0x7f);
+		break;
 	default:
 		err = -ENXIO;
 		break;
diff --git a/sound/hda/codecs/conexant.c b/sound/hda/codecs/conexant.c
index 3d92262763f62..40da2832ba664 100644
--- a/sound/hda/codecs/conexant.c
+++ b/sound/hda/codecs/conexant.c
@@ -162,9 +162,6 @@ static void cx_fixup_headset_recog(struct hda_codec *codec)
 {
 	unsigned int mic_present;
 
-	/* fix some headset type recognize fail issue, such as EDIFIER headset */
-	/* set micbias output current comparator threshold from 66% to 55%. */
-	snd_hda_codec_write(codec, 0x1c, 0, 0x320, 0x010);
 	/* set OFF voltage for DFET from -1.2V to -0.8V, set headset micbias register
 	 * value adjustment trim from 2.2K ohms to 2.0K ohms.
 	 */
diff --git a/sound/hda/codecs/hdmi/hdmi.c b/sound/hda/codecs/hdmi/hdmi.c
index 1f4d646724edc..053c41d98028a 100644
--- a/sound/hda/codecs/hdmi/hdmi.c
+++ b/sound/hda/codecs/hdmi/hdmi.c
@@ -1688,6 +1688,9 @@ int snd_hda_hdmi_generic_pcm_prepare(struct hda_pcm_stream *hinfo,
 		per_pin->channels = substream->runtime->channels;
 		per_pin->setup = true;
 
+		if (spec->ops.prepare)
+			spec->ops.prepare(codec, per_pin);
+
 		if (get_wcaps(codec, cvt_nid) & AC_WCAP_STRIPE) {
 			stripe = snd_hdac_get_stream_stripe_ctl(&codec->bus->core,
 								substream);
@@ -2233,8 +2236,10 @@ void snd_hda_hdmi_acomp_pin_eld_notify(void *audio_ptr, int port, int dev_id)
 	/* skip notification during system suspend (but not in runtime PM);
 	 * the state will be updated at resume
 	 */
-	if (codec->core.dev.power.power_state.event == PM_EVENT_SUSPEND)
+	if (codec->core.dev.power.power_state.event == PM_EVENT_SUSPEND) {
+		codec->acomp_requested_resume = 1;
 		return;
+	}
 
 	snd_hda_hdmi_check_presence_and_report(codec, pin_nid, dev_id);
 }
diff --git a/sound/hda/codecs/hdmi/hdmi_local.h b/sound/hda/codecs/hdmi/hdmi_local.h
index 548241ad3fa99..dce9a335cff28 100644
--- a/sound/hda/codecs/hdmi/hdmi_local.h
+++ b/sound/hda/codecs/hdmi/hdmi_local.h
@@ -74,6 +74,15 @@ struct hdmi_ops {
 			    hda_nid_t pin_nid, int dev_id, u32 stream_tag,
 			    int format);
 
+	/*
+	 * Optional hook invoked at the beginning of the PCM prepare
+	 * sequence, before the audio infoframe and stream format are
+	 * (re)programmed. Used to disable keep-alive / silent stream so
+	 * that the format change is not done while keep-alive is active.
+	 */
+	void (*prepare)(struct hda_codec *codec,
+			struct hdmi_spec_per_pin *per_pin);
+
 	void (*pin_cvt_fixup)(struct hda_codec *codec,
 			      struct hdmi_spec_per_pin *per_pin,
 			      hda_nid_t cvt_nid);
diff --git a/sound/hda/codecs/hdmi/intelhdmi.c b/sound/hda/codecs/hdmi/intelhdmi.c
index 6a7882544ab77..73141046d502b 100644
--- a/sound/hda/codecs/hdmi/intelhdmi.c
+++ b/sound/hda/codecs/hdmi/intelhdmi.c
@@ -418,6 +418,28 @@ static void intel_not_share_assigned_cvt_nid(struct hda_codec *codec,
 		intel_not_share_assigned_cvt(codec, pin_nid, dev_id, mux_idx);
 }
 
+/*
+ * prepare ops override for HSW+
+ *
+ * Disable keep-alive before the converter format and audio infoframe are
+ * reprogrammed by the PCM prepare sequence. Changing the audio format (e.g.
+ * the channel count when switching to multichannel PCM) while a keep-alive
+ * stream is active is not safe, so release keep-alive here, early in the
+ * sequence. It is re-enabled once the new stream has been set up, in
+ * i915_hsw_setup_stream().
+ */
+static void i915_hsw_prepare(struct hda_codec *codec,
+			     struct hdmi_spec_per_pin *per_pin)
+{
+	struct hdmi_spec *spec = codec->spec;
+
+	if (spec->silent_stream_type == SILENT_STREAM_KAE && per_pin->silent_stream) {
+		silent_stream_set_kae(codec, per_pin, false);
+		/* wait for pending transfers in codec to clear */
+		usleep_range(100, 200);
+	}
+}
+
 /* setup_stream ops override for HSW+ */
 static int i915_hsw_setup_stream(struct hda_codec *codec, hda_nid_t cvt_nid,
 				 hda_nid_t pin_nid, int dev_id, u32 stream_tag,
@@ -435,15 +457,16 @@ static int i915_hsw_setup_stream(struct hda_codec *codec, hda_nid_t cvt_nid,
 
 	haswell_verify_D0(codec, cvt_nid, pin_nid);
 
-	if (spec->silent_stream_type == SILENT_STREAM_KAE && per_pin && per_pin->silent_stream) {
-		silent_stream_set_kae(codec, per_pin, false);
-		/* wait for pending transfers in codec to clear */
-		usleep_range(100, 200);
-	}
-
 	res = snd_hda_hdmi_setup_stream(codec, cvt_nid, pin_nid, dev_id,
 					stream_tag, format);
 
+	/*
+	 * Keep-alive was disabled in i915_hsw_prepare(), re-enable it now.
+	 * The pin lookup above resolves to the same per_pin that prepare
+	 * used (pin_nid comes from that per_pin), so this stays balanced; a
+	 * NULL per_pin only occurs on a lookup failure that also implies no
+	 * active keep-alive stream to restore.
+	 */
 	if (spec->silent_stream_type == SILENT_STREAM_KAE && per_pin && per_pin->silent_stream) {
 		usleep_range(100, 200);
 		silent_stream_set_kae(codec, per_pin, true);
@@ -607,6 +630,7 @@ static int intel_hsw_common_init(struct hda_codec *codec, hda_nid_t vendor_nid,
 	codec->depop_delay = 0;
 	codec->auto_runtime_pm = 1;
 
+	spec->ops.prepare = i915_hsw_prepare;
 	spec->ops.setup_stream = i915_hsw_setup_stream;
 	spec->ops.pin_cvt_fixup = i915_pin_cvt_fixup;
 	spec->ops.silent_stream = i915_set_silent_stream;
diff --git a/sound/hda/codecs/helpers/razer_blade16_2025.c b/sound/hda/codecs/helpers/razer_blade16_2025.c
new file mode 100644
index 0000000000000..1f2e06d8fc131
--- /dev/null
+++ b/sound/hda/codecs/helpers/razer_blade16_2025.c
@@ -0,0 +1,820 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Razer Blade 16 (2025) external smart-amplifier programming and fixup,
+ * to be included from the codec driver.
+ *
+ * The internal speakers are driven by an external smart amplifier whose
+ * crossover, voicing and protection live in a DSP reached over a vendor
+ * coef mailbox on NID 0x20.  These tables hold the programming captured
+ * from the Windows driver, reduced to the NID-0x20 traffic this machine
+ * needs (everything else the HDA parser rebuilds).
+ *
+ * Each entry is either a plain codec coef write or a mailbox write:
+ *   strobe == 0:  coef[addr] = lo
+ *   strobe != 0:  amp[addr]  = (hi << 16) | lo, with the bank selected on
+ *                 port 0x89 and the write committed by the strobe opcode.
+ */
+
+struct alc298_razer_blade16_2025_op {
+	u16 bank;	/* mailbox bank (port 0x89); unused for a plain coef */
+	u16 addr;	/* mailbox register, or plain coef index */
+	u16 hi;		/* mailbox high word; unused for a plain coef */
+	u16 lo;		/* mailbox low word, or plain coef value */
+	u16 strobe;	/* mailbox commit opcode; 0 marks a plain coef write */
+};
+
+/* program the external DSP at init; the amp is woken/parked separately */
+static const struct alc298_razer_blade16_2025_op alc298_razer_blade16_2025_amp_init[] = {
+	{ 0x0000, 0xf102, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xf103, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xc000, 0x0000, 0x0001, 0xb031 },
+	{ 0x0000, 0xea00, 0x0000, 0x0047, 0xb031 },
+	{ 0x0000, 0xf20d, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xf212, 0x0000, 0x003e, 0xb031 },
+	{ 0x0000, 0xc001, 0x0000, 0x0002, 0xb031 },
+	{ 0x0000, 0xc003, 0x0000, 0x0022, 0xb031 },
+	{ 0x0000, 0xc004, 0x0000, 0x0044, 0xb031 },
+	{ 0x0000, 0xc005, 0x0000, 0x0044, 0xb031 },
+	{ 0x0000, 0xc007, 0x0000, 0x0064, 0xb031 },
+	{ 0x0000, 0xc00e, 0x0000, 0x00e7, 0xb031 },
+	{ 0x0000, 0xf223, 0x0000, 0x007f, 0xb031 },
+	{ 0x0000, 0xf224, 0x0000, 0x00db, 0xb031 },
+	{ 0x0000, 0xf225, 0x0000, 0x00ee, 0xb031 },
+	{ 0x0000, 0xf226, 0x0000, 0x003f, 0xb031 },
+	{ 0x0000, 0xf227, 0x0000, 0x000f, 0xb031 },
+	{ 0x0000, 0xf21a, 0x0000, 0x0078, 0xb031 },
+	{ 0x0000, 0xf242, 0x0000, 0x003c, 0xb031 },
+	{ 0x0000, 0xc120, 0x0000, 0x0040, 0xb031 },
+	{ 0x0000, 0xc125, 0x0000, 0x0003, 0xb031 },
+	{ 0x0000, 0xc321, 0x0000, 0x000b, 0xb031 },
+	{ 0x0000, 0xc200, 0x0000, 0x00d8, 0xb031 },
+	{ 0x0000, 0xc201, 0x0000, 0x0027, 0xb031 },
+	{ 0x0000, 0xc202, 0x0000, 0x000f, 0xb031 },
+	{ 0x0000, 0xc400, 0x0000, 0x000e, 0xb031 },
+	{ 0x0000, 0xc401, 0x0000, 0x0043, 0xb031 },
+	{ 0x0000, 0xc402, 0x0000, 0x00e0, 0xb031 },
+	{ 0x0000, 0xc403, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xc404, 0x0000, 0x004c, 0xb031 },
+	{ 0x0000, 0xc406, 0x0000, 0x0040, 0xb031 },
+	{ 0x0000, 0xc407, 0x0000, 0x0002, 0xb031 },
+	{ 0x0000, 0xc408, 0x0000, 0x003f, 0xb031 },
+	{ 0x0000, 0xc300, 0x0000, 0x0001, 0xb031 },
+	{ 0x0000, 0xc125, 0x0000, 0x0003, 0xb031 },
+	{ 0x0000, 0xdf00, 0x0000, 0x0010, 0xb031 },
+	{ 0x0000, 0xdf5f, 0x0000, 0x0001, 0xb031 },
+	{ 0x0000, 0xdf60, 0x0000, 0x00a7, 0xb031 },
+	{ 0x0000, 0xc203, 0x0000, 0x0084, 0xb031 },
+	{ 0x0000, 0xc206, 0x0000, 0x0078, 0xb031 },
+	{ 0x0000, 0xf10a, 0x0000, 0x000b, 0xb031 },
+	{ 0x0000, 0xf10b, 0x0000, 0x004c, 0xb031 },
+	{ 0x0000, 0xf104, 0x0000, 0x00f4, 0xb031 },
+	{ 0x0000, 0xf105, 0x0000, 0x0003, 0xb031 },
+	{ 0x0000, 0xf109, 0x0000, 0x00e0, 0xb031 },
+	{ 0x0000, 0xf10b, 0x0000, 0x005c, 0xb031 },
+	{ 0x0000, 0xf104, 0x0000, 0x00f4, 0xb031 },
+	{ 0x0000, 0xf105, 0x0000, 0x0004, 0xb031 },
+	{ 0x0000, 0xf109, 0x0000, 0x0065, 0xb031 },
+	{ 0x0000, 0xf10b, 0x0000, 0x005c, 0xb031 },
+	{ 0x0000, 0xf104, 0x0000, 0x00f7, 0xb031 },
+	{ 0x0000, 0xf105, 0x0000, 0x0030, 0xb031 },
+	{ 0x0000, 0xf109, 0x0000, 0x0006, 0xb031 },
+	{ 0x0000, 0xf10b, 0x0000, 0x005c, 0xb031 },
+	{ 0x0000, 0xf104, 0x0000, 0x00f7, 0xb031 },
+	{ 0x0000, 0xf105, 0x0000, 0x0031, 0xb031 },
+	{ 0x0000, 0xf109, 0x0000, 0x0003, 0xb031 },
+	{ 0x0000, 0xf10b, 0x0000, 0x005c, 0xb031 },
+	{ 0x0000, 0xe706, 0x0000, 0x000f, 0xb031 },
+	{ 0x0000, 0xe707, 0x0000, 0x0030, 0xb031 },
+	{ 0x0000, 0xe806, 0x0000, 0x000f, 0xb031 },
+	{ 0x0000, 0xe807, 0x0000, 0x0030, 0xb031 },
+	{ 0x0000, 0xce04, 0x0000, 0x0002, 0xb031 },
+	{ 0x0000, 0xce05, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xce06, 0x0000, 0x0031, 0xb031 },
+	{ 0x0000, 0xce07, 0x0000, 0x00b4, 0xb031 },
+	{ 0x0000, 0xcf04, 0x0000, 0x0002, 0xb031 },
+	{ 0x0000, 0xcf05, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xcf06, 0x0000, 0x0031, 0xb031 },
+	{ 0x0000, 0xcf07, 0x0000, 0x00b4, 0xb031 },
+	{ 0x0000, 0xce60, 0x0000, 0x00e3, 0xb031 },
+	{ 0x0000, 0xc130, 0x0000, 0x0051, 0xb031 },
+	{ 0x0000, 0xe000, 0x0000, 0x00a8, 0xb031 },
+	{ 0x4100, 0x1888, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xc121, 0x0000, 0x000b, 0xb031 },
+	{ 0x0000, 0xea00, 0x0000, 0x0040, 0xb031 },
+	{ 0x0000, 0xf800, 0x0000, 0x0020, 0xb031 },
+	{ 0x0000, 0xca00, 0x0000, 0x0080, 0xb031 },
+	{ 0x0000, 0xca10, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xca02, 0x0000, 0x0078, 0xb031 },
+	{ 0x0000, 0xca12, 0x0000, 0x0078, 0xb031 },
+	{ 0x0000, 0xed00, 0x0000, 0x0090, 0xb031 },
+	{ 0x0000, 0xcc2c, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcc2d, 0x0000, 0x000e, 0xb031 },
+	{ 0x0000, 0xcc2e, 0x0000, 0x0076, 0xb031 },
+	{ 0x0000, 0xcc2f, 0x0000, 0x0043, 0xb031 },
+	{ 0x0000, 0xcd2c, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcd2d, 0x0000, 0x000e, 0xb031 },
+	{ 0x0000, 0xcd2e, 0x0000, 0x0076, 0xb031 },
+	{ 0x0000, 0xcd2f, 0x0000, 0x0043, 0xb031 },
+	{ 0x0000, 0xcc24, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcc25, 0x0000, 0x0051, 0xb031 },
+	{ 0x0000, 0xcc26, 0x0000, 0x00eb, 0xb031 },
+	{ 0x0000, 0xcc27, 0x0000, 0x0085, 0xb031 },
+	{ 0x0000, 0xcd24, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcd25, 0x0000, 0x0051, 0xb031 },
+	{ 0x0000, 0xcd26, 0x0000, 0x00eb, 0xb031 },
+	{ 0x0000, 0xcd27, 0x0000, 0x0085, 0xb031 },
+	{ 0x0000, 0xcc20, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcc21, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcc22, 0x0000, 0x0043, 0xb031 },
+	{ 0x0000, 0xcd20, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcd21, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcd22, 0x0000, 0x0043, 0xb031 },
+	{ 0x0000, 0xcc16, 0x0000, 0x000f, 0xb031 },
+	{ 0x0000, 0xcc17, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcd16, 0x0000, 0x000f, 0xb031 },
+	{ 0x0000, 0xcd17, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcc29, 0x0000, 0x005d, 0xb031 },
+	{ 0x0000, 0xcc2a, 0x0000, 0x00c0, 0xb031 },
+	{ 0x0000, 0xcd29, 0x0000, 0x005d, 0xb031 },
+	{ 0x0000, 0xcd2a, 0x0000, 0x00c0, 0xb031 },
+	{ 0x0000, 0xcc31, 0x0000, 0x0020, 0xb031 },
+	{ 0x0000, 0xcc32, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcc33, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcc34, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcd31, 0x0000, 0x0020, 0xb031 },
+	{ 0x0000, 0xcd32, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcd33, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcd34, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcc36, 0x0000, 0x0079, 0xb031 },
+	{ 0x0000, 0xcc37, 0x0000, 0x0099, 0xb031 },
+	{ 0x0000, 0xcc38, 0x0000, 0x0099, 0xb031 },
+	{ 0x0000, 0xcc39, 0x0000, 0x0099, 0xb031 },
+	{ 0x0000, 0xcd36, 0x0000, 0x0079, 0xb031 },
+	{ 0x0000, 0xcd37, 0x0000, 0x0099, 0xb031 },
+	{ 0x0000, 0xcd38, 0x0000, 0x0099, 0xb031 },
+	{ 0x0000, 0xcd39, 0x0000, 0x0099, 0xb031 },
+	{ 0x0000, 0xcc09, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcc0a, 0x0000, 0x000a, 0xb031 },
+	{ 0x0000, 0xcc0b, 0x0000, 0x007c, 0xb031 },
+	{ 0x0000, 0xcc0c, 0x0000, 0x005b, 0xb031 },
+	{ 0x0000, 0xcd09, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcd0a, 0x0000, 0x000a, 0xb031 },
+	{ 0x0000, 0xcd0b, 0x0000, 0x007c, 0xb031 },
+	{ 0x0000, 0xcd0c, 0x0000, 0x005b, 0xb031 },
+	{ 0x0000, 0xcc0e, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcc0f, 0x0000, 0x0005, 0xb031 },
+	{ 0x0000, 0xcc10, 0x0000, 0x003e, 0xb031 },
+	{ 0x0000, 0xcc11, 0x0000, 0x002d, 0xb031 },
+	{ 0x0000, 0xcd0e, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcd0f, 0x0000, 0x0005, 0xb031 },
+	{ 0x0000, 0xcd10, 0x0000, 0x003e, 0xb031 },
+	{ 0x0000, 0xcd11, 0x0000, 0x002d, 0xb031 },
+	{ 0x0000, 0xccd6, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xccd7, 0x0000, 0x0003, 0xb031 },
+	{ 0x0000, 0xcdd6, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcdd7, 0x0000, 0x0003, 0xb031 },
+	{ 0x0000, 0xccd8, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xccd9, 0x0000, 0x0003, 0xb031 },
+	{ 0x0000, 0xcdd8, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcdd9, 0x0000, 0x0003, 0xb031 },
+	{ 0x0000, 0xccda, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xccdb, 0x0000, 0x0003, 0xb031 },
+	{ 0x0000, 0xcdda, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xcddb, 0x0000, 0x0003, 0xb031 },
+	{ 0x0000, 0xc320, 0x0000, 0x0020, 0xb031 },
+	{ 0x0000, 0xc321, 0x0000, 0x000a, 0xb031 },
+	{ 0x0000, 0xe604, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xdb00, 0x0000, 0x000c, 0xb031 },
+	{ 0x0000, 0xdd00, 0x0000, 0x000c, 0xb031 },
+	{ 0x0000, 0xdc19, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xdc1a, 0x0000, 0x006a, 0xb031 },
+	{ 0x0000, 0xdc1b, 0x0000, 0x00aa, 0xb031 },
+	{ 0x0000, 0xdc1c, 0x0000, 0x00ab, 0xb031 },
+	{ 0x0000, 0xdc1d, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xdc1e, 0x0000, 0x0027, 0xb031 },
+	{ 0x0000, 0xdc1f, 0x0000, 0x0062, 0xb031 },
+	{ 0x0000, 0xdc20, 0x0000, 0x0076, 0xb031 },
+	{ 0x0000, 0xde19, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xde1a, 0x0000, 0x006a, 0xb031 },
+	{ 0x0000, 0xde1b, 0x0000, 0x00aa, 0xb031 },
+	{ 0x0000, 0xde1c, 0x0000, 0x00ab, 0xb031 },
+	{ 0x0000, 0xde1d, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xde1e, 0x0000, 0x0027, 0xb031 },
+	{ 0x0000, 0xde1f, 0x0000, 0x0062, 0xb031 },
+	{ 0x0000, 0xde20, 0x0000, 0x0076, 0xb031 },
+	{ 0x0000, 0xdb32, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xdd32, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xdb33, 0x0000, 0x000a, 0xb031 },
+	{ 0x0000, 0xdd33, 0x0000, 0x000a, 0xb031 },
+	{ 0x0000, 0xdb34, 0x0000, 0x001a, 0xb031 },
+	{ 0x0000, 0xdd34, 0x0000, 0x001a, 0xb031 },
+	{ 0x0000, 0xdb15, 0x0000, 0x00ef, 0xb031 },
+	{ 0x0000, 0xdd15, 0x0000, 0x00ef, 0xb031 },
+	{ 0x0000, 0xdb17, 0x0000, 0x003f, 0xb031 },
+	{ 0x0000, 0xdd17, 0x0000, 0x003f, 0xb031 },
+	{ 0x0000, 0xdb94, 0x0000, 0x0070, 0xb031 },
+	{ 0x0000, 0xdd94, 0x0000, 0x0070, 0xb031 },
+	{ 0x0000, 0xdb19, 0x0000, 0x0040, 0xb031 },
+	{ 0x0000, 0xdd19, 0x0000, 0x0040, 0xb031 },
+	{ 0x0000, 0xc203, 0x0000, 0x001c, 0xb031 },
+	{ 0x0000, 0xdb00, 0x0000, 0x004c, 0xb031 },
+	{ 0x0000, 0xdb04, 0x0000, 0x0005, 0xb031 },
+	{ 0x0000, 0xdb05, 0x0000, 0x0003, 0xb031 },
+	{ 0x0000, 0xdd04, 0x0000, 0x0005, 0xb031 },
+	{ 0x0000, 0xdd05, 0x0000, 0x0003, 0xb031 },
+	{ 0x0000, 0xdbbb, 0x0000, 0x0009, 0xb031 },
+	{ 0x0000, 0xdbbc, 0x0000, 0x004c, 0xb031 },
+	{ 0x0000, 0xdbbd, 0x0000, 0x00f3, 0xb031 },
+	{ 0x0000, 0xdbbe, 0x0000, 0x00cf, 0xb031 },
+	{ 0x0000, 0xddbb, 0x0000, 0x0009, 0xb031 },
+	{ 0x0000, 0xddbc, 0x0000, 0x004c, 0xb031 },
+	{ 0x0000, 0xddbd, 0x0000, 0x00f3, 0xb031 },
+	{ 0x0000, 0xddbe, 0x0000, 0x00cf, 0xb031 },
+	{ 0x0000, 0xdb01, 0x0000, 0x0079, 0xb031 },
+	{ 0x0000, 0xdd01, 0x0000, 0x0079, 0xb031 },
+	{ 0x0000, 0xdb08, 0x0000, 0x0040, 0xb031 },
+	{ 0x0000, 0xdd08, 0x0000, 0x0040, 0xb031 },
+	{ 0x0000, 0xdc52, 0x0000, 0x00ef, 0xb031 },
+	{ 0x0000, 0xde52, 0x0000, 0x00ef, 0xb031 },
+	{ 0x0000, 0xdb00, 0x0000, 0x00cc, 0xb031 },
+	{ 0x0000, 0xc203, 0x0000, 0x009c, 0xb031 },
+	{ 0x0000, 0xdf0a, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xdf0b, 0x0000, 0x007f, 0xb031 },
+	{ 0x0000, 0xc851, 0x0000, 0x0080, 0xb031 },
+	{ 0x0000, 0xc951, 0x0000, 0x0080, 0xb031 },
+	{ 0x0000, 0xdf01, 0x0000, 0x0073, 0xb031 },
+	{ 0x0000, 0xc203, 0x0000, 0x009c, 0xb031 },
+	{ 0x0000, 0xf800, 0x0000, 0x0020, 0xb031 },
+	{ 0x0000, 0xebcb, 0x0000, 0x0080, 0xb031 },
+	{ 0x0000, 0xeb10, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xeb11, 0x0000, 0x00bb, 0xb031 },
+	{ 0x0000, 0xeb12, 0x0000, 0x00ab, 0xb031 },
+	{ 0x0000, 0xeb13, 0x0000, 0x0030, 0xb031 },
+	{ 0x0000, 0xeb14, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xeb15, 0x0000, 0x004c, 0xb031 },
+	{ 0x0000, 0xeb16, 0x0000, 0x0069, 0xb031 },
+	{ 0x0000, 0xeb21, 0x0000, 0x0011, 0xb031 },
+	{ 0x0000, 0xeb22, 0x0000, 0x00c2, 0xb031 },
+	{ 0x0000, 0xeb23, 0x0000, 0x0080, 0xb031 },
+	{ 0x0000, 0xeb24, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xeb25, 0x0000, 0x00bd, 0xb031 },
+	{ 0x0000, 0xeb26, 0x0000, 0x0055, 0xb031 },
+	{ 0x0000, 0xeb27, 0x0000, 0x00d0, 0xb031 },
+	{ 0x0000, 0xeb28, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xeb29, 0x0000, 0x003c, 0xb031 },
+	{ 0x0000, 0xeb2a, 0x0000, 0x0051, 0xb031 },
+	{ 0x0000, 0xeb2b, 0x0000, 0x0018, 0xb031 },
+	{ 0x0000, 0xeb30, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xeb31, 0x0000, 0x004f, 0xb031 },
+	{ 0x0000, 0xeb32, 0x0000, 0x004a, 0xb031 },
+	{ 0x0000, 0xeb33, 0x0000, 0x0040, 0xb031 },
+	{ 0x0000, 0xeb34, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xeb35, 0x0000, 0x00bb, 0xb031 },
+	{ 0x0000, 0xeb36, 0x0000, 0x0024, 0xb031 },
+	{ 0x0000, 0xeb37, 0x0000, 0x0068, 0xb031 },
+	{ 0x0000, 0xeb40, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xeb41, 0x0000, 0x00f5, 0xb031 },
+	{ 0x0000, 0xeb42, 0x0000, 0x00f1, 0xb031 },
+	{ 0x0000, 0xeb43, 0x0000, 0x00e8, 0xb031 },
+	{ 0x0000, 0xeb44, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xeb45, 0x0000, 0x004f, 0xb031 },
+	{ 0x0000, 0xeb46, 0x0000, 0x004a, 0xb031 },
+	{ 0x0000, 0xeb47, 0x0000, 0x0040, 0xb031 },
+	{ 0x0000, 0xeb48, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xeb49, 0x0000, 0x00c5, 0xb031 },
+	{ 0x0000, 0xeb4a, 0x0000, 0x0032, 0xb031 },
+	{ 0x0000, 0xeb4b, 0x0000, 0x0080, 0xb031 },
+	{ 0x0000, 0xeb50, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xeb51, 0x0000, 0x008a, 0xb031 },
+	{ 0x0000, 0xeb52, 0x0000, 0x00b4, 0xb031 },
+	{ 0x0000, 0xeb53, 0x0000, 0x0040, 0xb031 },
+	{ 0x0000, 0xeb54, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xeb55, 0x0000, 0x007d, 0xb031 },
+	{ 0x0000, 0xeb56, 0x0000, 0x00d9, 0xb031 },
+	{ 0x0000, 0xeb57, 0x0000, 0x0008, 0xb031 },
+	{ 0x0000, 0xeb60, 0x0000, 0x0002, 0xb031 },
+	{ 0x0000, 0xeb61, 0x0000, 0x00e7, 0xb031 },
+	{ 0x0000, 0xeb62, 0x0000, 0x00b5, 0xb031 },
+	{ 0x0000, 0xeb63, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xeb64, 0x0000, 0x00fa, 0xb031 },
+	{ 0x0000, 0xeb65, 0x0000, 0x0086, 0xb031 },
+	{ 0x0000, 0xeb66, 0x0000, 0x00a8, 0xb031 },
+	{ 0x0000, 0xeb67, 0x0000, 0x0040, 0xb031 },
+	{ 0x0000, 0xeb68, 0x0000, 0x0002, 0xb031 },
+	{ 0x0000, 0xeb69, 0x0000, 0x009a, 0xb031 },
+	{ 0x0000, 0xeb6a, 0x0000, 0x002f, 0xb031 },
+	{ 0x0000, 0xeb6b, 0x0000, 0x0018, 0xb031 },
+	{ 0x0000, 0xebb6, 0x0000, 0x000c, 0xb031 },
+	{ 0x0000, 0xeb70, 0x0000, 0x00f5, 0xb031 },
+	{ 0x0000, 0xeb71, 0x0000, 0x00e1, 0xb031 },
+	{ 0x0000, 0xeb72, 0x0000, 0x0009, 0xb031 },
+	{ 0x0000, 0xeb73, 0x0000, 0x00d0, 0xb031 },
+	{ 0x0000, 0xeb74, 0x0000, 0x0006, 0xb031 },
+	{ 0x0000, 0xeb75, 0x0000, 0x0050, 0xb031 },
+	{ 0x0000, 0xeb76, 0x0000, 0x002c, 0xb031 },
+	{ 0x0000, 0xeb77, 0x0000, 0x0050, 0xb031 },
+	{ 0x0000, 0xeb80, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xeb81, 0x0000, 0x0028, 0xb031 },
+	{ 0x0000, 0xeb82, 0x0000, 0x0016, 0xb031 },
+	{ 0x0000, 0xeb83, 0x0000, 0x0028, 0xb031 },
+	{ 0x0000, 0xeb84, 0x0000, 0x00f5, 0xb031 },
+	{ 0x0000, 0xeb85, 0x0000, 0x00e1, 0xb031 },
+	{ 0x0000, 0xeb86, 0x0000, 0x0009, 0xb031 },
+	{ 0x0000, 0xeb87, 0x0000, 0x00d0, 0xb031 },
+	{ 0x0000, 0xeb88, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xeb89, 0x0000, 0x0028, 0xb031 },
+	{ 0x0000, 0xeb8a, 0x0000, 0x0016, 0xb031 },
+	{ 0x0000, 0xeb8b, 0x0000, 0x0028, 0xb031 },
+	{ 0x0000, 0xebb8, 0x0000, 0x000c, 0xb031 },
+	{ 0x0000, 0xeb90, 0x0000, 0x00fc, 0xb031 },
+	{ 0x0000, 0xeb91, 0x0000, 0x008b, 0xb031 },
+	{ 0x0000, 0xeb92, 0x0000, 0x00bd, 0xb031 },
+	{ 0x0000, 0xeb93, 0x0000, 0x0064, 0xb031 },
+	{ 0x0000, 0xeb94, 0x0000, 0x0001, 0xb031 },
+	{ 0x0000, 0xeb95, 0x0000, 0x0099, 0xb031 },
+	{ 0x0000, 0xeb96, 0x0000, 0x00e7, 0xb031 },
+	{ 0x0000, 0xeb97, 0x0000, 0x0094, 0xb031 },
+	{ 0x0000, 0xeba0, 0x0000, 0x000c, 0xb031 },
+	{ 0x0000, 0xeba1, 0x0000, 0x0091, 0xb031 },
+	{ 0x0000, 0xeba2, 0x0000, 0x0078, 0xb031 },
+	{ 0x0000, 0xeba3, 0x0000, 0x0080, 0xb031 },
+	{ 0x0000, 0xeba4, 0x0000, 0x00f6, 0xb031 },
+	{ 0x0000, 0xeba5, 0x0000, 0x000c, 0xb031 },
+	{ 0x0000, 0xeba6, 0x0000, 0x0093, 0xb031 },
+	{ 0x0000, 0xeba7, 0x0000, 0x0010, 0xb031 },
+	{ 0x0000, 0xeba8, 0x0000, 0x0003, 0xb031 },
+	{ 0x0000, 0xeba9, 0x0000, 0x0087, 0xb031 },
+	{ 0x0000, 0xebaa, 0x0000, 0x0099, 0xb031 },
+	{ 0x0000, 0xebab, 0x0000, 0x006c, 0xb031 },
+	{ 0x0000, 0xebc3, 0x0000, 0x00ff, 0xb031 },
+	{ 0x0000, 0xebc4, 0x0000, 0x00fc, 0xb031 },
+	{ 0x0000, 0xeb00, 0x0000, 0x0005, 0xb031 },
+	{ 0x0000, 0xeb01, 0x0000, 0x00a9, 0xb031 },
+	{ 0x0000, 0xeb02, 0x0000, 0x00df, 0xb031 },
+	{ 0x0000, 0xeb03, 0x0000, 0x007b, 0xb031 },
+	{ 0x0000, 0xeb05, 0x0000, 0x0079, 0xb031 },
+	{ 0x0000, 0xeb06, 0x0000, 0x005a, 0xb031 },
+	{ 0x0000, 0xeb07, 0x0000, 0x0004, 0xb031 },
+	{ 0x0000, 0xebc0, 0x0000, 0x00e3, 0xb031 },
+	{ 0x0000, 0xd0cb, 0x0000, 0x0080, 0xb031 },
+	{ 0x0000, 0xd0b0, 0x0000, 0x000c, 0xb031 },
+	{ 0x0000, 0xd010, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xd011, 0x0000, 0x0009, 0xb031 },
+	{ 0x0000, 0xd012, 0x0000, 0x0092, 0xb031 },
+	{ 0x0000, 0xd013, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xd014, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xd015, 0x0000, 0x00f6, 0xb031 },
+	{ 0x0000, 0xd016, 0x0000, 0x0072, 0xb031 },
+	{ 0x0000, 0xd017, 0x0000, 0x00b0, 0xb031 },
+	{ 0x0000, 0xd020, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xd021, 0x0000, 0x00fb, 0xb031 },
+	{ 0x0000, 0xd022, 0x0000, 0x0037, 0xb031 },
+	{ 0x0000, 0xd023, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xd024, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xd025, 0x0000, 0x0009, 0xb031 },
+	{ 0x0000, 0xd026, 0x0000, 0x0090, 0xb031 },
+	{ 0x0000, 0xd027, 0x0000, 0x0020, 0xb031 },
+	{ 0x0000, 0xd028, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xd029, 0x0000, 0x00fb, 0xb031 },
+	{ 0x0000, 0xd02a, 0x0000, 0x0037, 0xb031 },
+	{ 0x0000, 0xd02b, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xd0c3, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xd0c4, 0x0000, 0x000c, 0xb031 },
+	{ 0x0000, 0xd0c0, 0x0000, 0x00e3, 0xb031 },
+	{ 0x0000, 0xce6a, 0x0000, 0x0080, 0xb031 },
+	{ 0x0000, 0xce63, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xce64, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xce60, 0x0000, 0x00e3, 0xb031 },
+	{ 0x0000, 0xdb00, 0x0000, 0x00cc, 0xb031 },
+	{ 0x0000, 0xdb04, 0x0000, 0x0008, 0xb031 },
+	{ 0x0000, 0xdb05, 0x0000, 0x0006, 0xb031 },
+	{ 0x0000, 0xcc00, 0x0000, 0x00c1, 0xb031 },
+	{ 0x0000, 0xca02, 0x0000, 0x000f, 0xb031 },
+	{ 0x0000, 0xca00, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xca62, 0x0000, 0x0077, 0xb031 },
+	{ 0x0000, 0xca65, 0x0000, 0x0082, 0xb031 },
+	{ 0x0000, 0xca68, 0x0000, 0x00c2, 0xb031 },
+	{ 0x0000, 0xca75, 0x0000, 0x003a, 0xb031 },
+	{ 0x0000, 0xca7b, 0x0000, 0x006f, 0xb031 },
+	{ 0x0000, 0xca00, 0x0000, 0x0080, 0xb031 },
+	{ 0x0000, 0xca02, 0x0000, 0x0078, 0xb031 },
+	{ 0x0000, 0xc860, 0x0000, 0x0078, 0xb031 },
+	{ 0x0000, 0xc861, 0x0000, 0x00e5, 0xb031 },
+	{ 0x0000, 0xc862, 0x0000, 0x00eb, 0xb031 },
+	{ 0x0000, 0xc863, 0x0000, 0x00c6, 0xb031 },
+	{ 0x0000, 0xc864, 0x0000, 0x00ba, 0xb031 },
+	{ 0x0000, 0xc865, 0x0000, 0x0061, 0xb031 },
+	{ 0x0000, 0xc867, 0x0000, 0x0017, 0xb031 },
+	{ 0x0000, 0xc868, 0x0000, 0x00ed, 0xb031 },
+	{ 0x0000, 0xc869, 0x0000, 0x003c, 0xb031 },
+	{ 0x0000, 0xc86a, 0x0000, 0x008a, 0xb031 },
+	{ 0x0000, 0xc86b, 0x0000, 0x00e2, 0xb031 },
+	{ 0x0000, 0xc875, 0x0000, 0x003a, 0xb031 },
+	{ 0x0000, 0xc87b, 0x0000, 0x006f, 0xb031 },
+	{ 0x0000, 0xc836, 0x0000, 0x00f9, 0xb031 },
+	{ 0x0000, 0xc8a0, 0x0000, 0x00f1, 0xb031 },
+	{ 0x0000, 0xc833, 0x0000, 0x000b, 0xb031 },
+	{ 0x0000, 0xc834, 0x0000, 0x0002, 0xb031 },
+	{ 0x0000, 0xc83f, 0x0000, 0x0014, 0xb031 },
+	{ 0x0000, 0xc83e, 0x0000, 0x0094, 0xb031 },
+	{ 0x0000, 0xc83a, 0x0000, 0x0076, 0xb031 },
+	{ 0x0000, 0xc83b, 0x0000, 0x0089, 0xb031 },
+	{ 0x0000, 0xc830, 0x0000, 0x0013, 0xb031 },
+	{ 0x0000, 0xc826, 0x0000, 0x00d1, 0xb031 },
+	{ 0x0000, 0xc890, 0x0000, 0x00f1, 0xb031 },
+	{ 0x0000, 0xc823, 0x0000, 0x0010, 0xb031 },
+	{ 0x0000, 0xc824, 0x0000, 0x0004, 0xb031 },
+	{ 0x0000, 0xc82f, 0x0000, 0x0018, 0xb031 },
+	{ 0x0000, 0xc82e, 0x0000, 0x0094, 0xb031 },
+	{ 0x0000, 0xc820, 0x0000, 0x0008, 0xb031 },
+	{ 0x0000, 0xc816, 0x0000, 0x00f9, 0xb031 },
+	{ 0x0000, 0xc880, 0x0000, 0x00e1, 0xb031 },
+	{ 0x0000, 0xc813, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xc814, 0x0000, 0x0001, 0xb031 },
+	{ 0x0000, 0xc81f, 0x0000, 0x0014, 0xb031 },
+	{ 0x0000, 0xc81e, 0x0000, 0x0094, 0xb031 },
+	{ 0x0000, 0xc81a, 0x0000, 0x0066, 0xb031 },
+	{ 0x0000, 0xc81b, 0x0000, 0x0078, 0xb031 },
+	{ 0x0000, 0xc810, 0x0000, 0x000d, 0xb031 },
+	{ 0x0000, 0xc846, 0x0000, 0x00f9, 0xb031 },
+	{ 0x0000, 0xc8d0, 0x0000, 0x00f1, 0xb031 },
+	{ 0x0000, 0xc844, 0x0000, 0x0001, 0xb031 },
+	{ 0x0000, 0xc84f, 0x0000, 0x0014, 0xb031 },
+	{ 0x0000, 0xc84e, 0x0000, 0x0094, 0xb031 },
+	{ 0x0000, 0xc84a, 0x0000, 0x0026, 0xb031 },
+	{ 0x0000, 0xc84b, 0x0000, 0x0056, 0xb031 },
+	{ 0x0000, 0xc800, 0x0000, 0x00b0, 0xb031 },
+	{ 0x0000, 0xc851, 0x0000, 0x0080, 0xb031 },
+	{ 0x0000, 0xc500, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xc501, 0x0000, 0x0026, 0xb031 },
+	{ 0x0000, 0xc502, 0x0000, 0x004a, 0xb031 },
+	{ 0x0000, 0xc503, 0x0000, 0x0010, 0xb031 },
+	{ 0x0000, 0xc504, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xc505, 0x0000, 0x00da, 0xb031 },
+	{ 0x0000, 0xc506, 0x0000, 0x000e, 0xb031 },
+	{ 0x0000, 0xc507, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xc510, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xc511, 0x0000, 0x00da, 0xb031 },
+	{ 0x0000, 0xc512, 0x0000, 0x000e, 0xb031 },
+	{ 0x0000, 0xc513, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xc514, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xc515, 0x0000, 0x0026, 0xb031 },
+	{ 0x0000, 0xc516, 0x0000, 0x004a, 0xb031 },
+	{ 0x0000, 0xc517, 0x0000, 0x0010, 0xb031 },
+	{ 0x0000, 0xc518, 0x0000, 0x0008, 0xb031 },
+	{ 0x0000, 0xc541, 0x0000, 0x0030, 0xb031 },
+	{ 0x0000, 0xd250, 0x0000, 0x0087, 0xb031 },
+	{ 0x0000, 0xd251, 0x0000, 0x0030, 0xb031 },
+	{ 0x0000, 0xd264, 0x0000, 0x00c0, 0xb031 },
+	{ 0x0000, 0xc203, 0x0000, 0x009c, 0xb031 },
+	{ 0x0000, 0xf800, 0x0000, 0x0020, 0xb031 },
+	{ 0x0000, 0xecb0, 0x0000, 0x0008, 0xb031 },
+	{ 0x0000, 0xec10, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xec11, 0x0000, 0x00bb, 0xb031 },
+	{ 0x0000, 0xec12, 0x0000, 0x00ab, 0xb031 },
+	{ 0x0000, 0xec13, 0x0000, 0x0030, 0xb031 },
+	{ 0x0000, 0xec14, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xec15, 0x0000, 0x004c, 0xb031 },
+	{ 0x0000, 0xec16, 0x0000, 0x0069, 0xb031 },
+	{ 0x0000, 0xec21, 0x0000, 0x0011, 0xb031 },
+	{ 0x0000, 0xec22, 0x0000, 0x00c2, 0xb031 },
+	{ 0x0000, 0xec23, 0x0000, 0x0080, 0xb031 },
+	{ 0x0000, 0xec24, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xec25, 0x0000, 0x00bd, 0xb031 },
+	{ 0x0000, 0xec26, 0x0000, 0x0055, 0xb031 },
+	{ 0x0000, 0xec27, 0x0000, 0x00d0, 0xb031 },
+	{ 0x0000, 0xec28, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xec29, 0x0000, 0x003c, 0xb031 },
+	{ 0x0000, 0xec2a, 0x0000, 0x0051, 0xb031 },
+	{ 0x0000, 0xec2b, 0x0000, 0x0018, 0xb031 },
+	{ 0x0000, 0xecb2, 0x0000, 0x0008, 0xb031 },
+	{ 0x0000, 0xec30, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xec31, 0x0000, 0x004f, 0xb031 },
+	{ 0x0000, 0xec32, 0x0000, 0x004a, 0xb031 },
+	{ 0x0000, 0xec33, 0x0000, 0x0040, 0xb031 },
+	{ 0x0000, 0xec34, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xec35, 0x0000, 0x00bb, 0xb031 },
+	{ 0x0000, 0xec36, 0x0000, 0x0024, 0xb031 },
+	{ 0x0000, 0xec37, 0x0000, 0x0068, 0xb031 },
+	{ 0x0000, 0xec40, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xec41, 0x0000, 0x00f5, 0xb031 },
+	{ 0x0000, 0xec42, 0x0000, 0x00f1, 0xb031 },
+	{ 0x0000, 0xec43, 0x0000, 0x00e8, 0xb031 },
+	{ 0x0000, 0xec44, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xec45, 0x0000, 0x004f, 0xb031 },
+	{ 0x0000, 0xec46, 0x0000, 0x004a, 0xb031 },
+	{ 0x0000, 0xec47, 0x0000, 0x0040, 0xb031 },
+	{ 0x0000, 0xec48, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xec49, 0x0000, 0x00c5, 0xb031 },
+	{ 0x0000, 0xec4a, 0x0000, 0x0032, 0xb031 },
+	{ 0x0000, 0xec4b, 0x0000, 0x0080, 0xb031 },
+	{ 0x0000, 0xecb4, 0x0000, 0x0008, 0xb031 },
+	{ 0x0000, 0xec50, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xec51, 0x0000, 0x008a, 0xb031 },
+	{ 0x0000, 0xec52, 0x0000, 0x00b4, 0xb031 },
+	{ 0x0000, 0xec53, 0x0000, 0x0040, 0xb031 },
+	{ 0x0000, 0xec54, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xec55, 0x0000, 0x007d, 0xb031 },
+	{ 0x0000, 0xec56, 0x0000, 0x00d9, 0xb031 },
+	{ 0x0000, 0xec57, 0x0000, 0x0008, 0xb031 },
+	{ 0x0000, 0xec60, 0x0000, 0x0002, 0xb031 },
+	{ 0x0000, 0xec61, 0x0000, 0x00e7, 0xb031 },
+	{ 0x0000, 0xec62, 0x0000, 0x00b5, 0xb031 },
+	{ 0x0000, 0xec63, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xec64, 0x0000, 0x00fa, 0xb031 },
+	{ 0x0000, 0xec65, 0x0000, 0x0086, 0xb031 },
+	{ 0x0000, 0xec66, 0x0000, 0x00a8, 0xb031 },
+	{ 0x0000, 0xec67, 0x0000, 0x0040, 0xb031 },
+	{ 0x0000, 0xec68, 0x0000, 0x0002, 0xb031 },
+	{ 0x0000, 0xec69, 0x0000, 0x009a, 0xb031 },
+	{ 0x0000, 0xec6a, 0x0000, 0x002f, 0xb031 },
+	{ 0x0000, 0xec6b, 0x0000, 0x0018, 0xb031 },
+	{ 0x0000, 0xecb6, 0x0000, 0x000c, 0xb031 },
+	{ 0x0000, 0xec70, 0x0000, 0x00f5, 0xb031 },
+	{ 0x0000, 0xec71, 0x0000, 0x00e1, 0xb031 },
+	{ 0x0000, 0xec72, 0x0000, 0x0009, 0xb031 },
+	{ 0x0000, 0xec73, 0x0000, 0x00d0, 0xb031 },
+	{ 0x0000, 0xec74, 0x0000, 0x0006, 0xb031 },
+	{ 0x0000, 0xec75, 0x0000, 0x0050, 0xb031 },
+	{ 0x0000, 0xec76, 0x0000, 0x002c, 0xb031 },
+	{ 0x0000, 0xec77, 0x0000, 0x0050, 0xb031 },
+	{ 0x0000, 0xec80, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xec81, 0x0000, 0x0028, 0xb031 },
+	{ 0x0000, 0xec82, 0x0000, 0x0016, 0xb031 },
+	{ 0x0000, 0xec83, 0x0000, 0x0028, 0xb031 },
+	{ 0x0000, 0xec84, 0x0000, 0x00f5, 0xb031 },
+	{ 0x0000, 0xec85, 0x0000, 0x00e1, 0xb031 },
+	{ 0x0000, 0xec86, 0x0000, 0x0009, 0xb031 },
+	{ 0x0000, 0xec87, 0x0000, 0x00d0, 0xb031 },
+	{ 0x0000, 0xec88, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xec89, 0x0000, 0x0028, 0xb031 },
+	{ 0x0000, 0xec8a, 0x0000, 0x0016, 0xb031 },
+	{ 0x0000, 0xec8b, 0x0000, 0x0028, 0xb031 },
+	{ 0x0000, 0xecb8, 0x0000, 0x000c, 0xb031 },
+	{ 0x0000, 0xec90, 0x0000, 0x00fc, 0xb031 },
+	{ 0x0000, 0xec91, 0x0000, 0x008b, 0xb031 },
+	{ 0x0000, 0xec92, 0x0000, 0x00bd, 0xb031 },
+	{ 0x0000, 0xec93, 0x0000, 0x0064, 0xb031 },
+	{ 0x0000, 0xec94, 0x0000, 0x0001, 0xb031 },
+	{ 0x0000, 0xec95, 0x0000, 0x0099, 0xb031 },
+	{ 0x0000, 0xec96, 0x0000, 0x00e7, 0xb031 },
+	{ 0x0000, 0xec97, 0x0000, 0x0094, 0xb031 },
+	{ 0x0000, 0xeca0, 0x0000, 0x000c, 0xb031 },
+	{ 0x0000, 0xeca1, 0x0000, 0x0091, 0xb031 },
+	{ 0x0000, 0xeca2, 0x0000, 0x0078, 0xb031 },
+	{ 0x0000, 0xeca3, 0x0000, 0x0080, 0xb031 },
+	{ 0x0000, 0xeca4, 0x0000, 0x00f6, 0xb031 },
+	{ 0x0000, 0xeca5, 0x0000, 0x000c, 0xb031 },
+	{ 0x0000, 0xeca6, 0x0000, 0x0093, 0xb031 },
+	{ 0x0000, 0xeca7, 0x0000, 0x0010, 0xb031 },
+	{ 0x0000, 0xeca8, 0x0000, 0x0003, 0xb031 },
+	{ 0x0000, 0xeca9, 0x0000, 0x0087, 0xb031 },
+	{ 0x0000, 0xecaa, 0x0000, 0x0099, 0xb031 },
+	{ 0x0000, 0xecab, 0x0000, 0x006c, 0xb031 },
+	{ 0x0000, 0xebc3, 0x0000, 0x00ff, 0xb031 },
+	{ 0x0000, 0xebc4, 0x0000, 0x00fc, 0xb031 },
+	{ 0x0000, 0xec00, 0x0000, 0x0005, 0xb031 },
+	{ 0x0000, 0xec01, 0x0000, 0x00a9, 0xb031 },
+	{ 0x0000, 0xec02, 0x0000, 0x00df, 0xb031 },
+	{ 0x0000, 0xec03, 0x0000, 0x007b, 0xb031 },
+	{ 0x0000, 0xec05, 0x0000, 0x0079, 0xb031 },
+	{ 0x0000, 0xec06, 0x0000, 0x005a, 0xb031 },
+	{ 0x0000, 0xec07, 0x0000, 0x0004, 0xb031 },
+	{ 0x0000, 0xebc0, 0x0000, 0x00e3, 0xb031 },
+	{ 0x0000, 0xd1b0, 0x0000, 0x000c, 0xb031 },
+	{ 0x0000, 0xd110, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xd111, 0x0000, 0x0009, 0xb031 },
+	{ 0x0000, 0xd112, 0x0000, 0x0092, 0xb031 },
+	{ 0x0000, 0xd113, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xd114, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xd115, 0x0000, 0x00f6, 0xb031 },
+	{ 0x0000, 0xd116, 0x0000, 0x0072, 0xb031 },
+	{ 0x0000, 0xd117, 0x0000, 0x00b0, 0xb031 },
+	{ 0x0000, 0xd120, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xd121, 0x0000, 0x00fb, 0xb031 },
+	{ 0x0000, 0xd122, 0x0000, 0x0037, 0xb031 },
+	{ 0x0000, 0xd123, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xd124, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xd125, 0x0000, 0x0009, 0xb031 },
+	{ 0x0000, 0xd126, 0x0000, 0x0090, 0xb031 },
+	{ 0x0000, 0xd127, 0x0000, 0x0020, 0xb031 },
+	{ 0x0000, 0xd128, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xd129, 0x0000, 0x00fb, 0xb031 },
+	{ 0x0000, 0xd12a, 0x0000, 0x0037, 0xb031 },
+	{ 0x0000, 0xd12b, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xd0c3, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xd0c4, 0x0000, 0x000c, 0xb031 },
+	{ 0x0000, 0xd0c0, 0x0000, 0x00e3, 0xb031 },
+	{ 0x0000, 0xce63, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xce64, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xce60, 0x0000, 0x00e3, 0xb031 },
+	{ 0x0000, 0xdb00, 0x0000, 0x00cc, 0xb031 },
+	{ 0x0000, 0xdd04, 0x0000, 0x0008, 0xb031 },
+	{ 0x0000, 0xdd05, 0x0000, 0x0006, 0xb031 },
+	{ 0x0000, 0xca12, 0x0000, 0x000f, 0xb031 },
+	{ 0x0000, 0xca10, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xca82, 0x0000, 0x0077, 0xb031 },
+	{ 0x0000, 0xca85, 0x0000, 0x0082, 0xb031 },
+	{ 0x0000, 0xca88, 0x0000, 0x00c2, 0xb031 },
+	{ 0x0000, 0xca95, 0x0000, 0x003a, 0xb031 },
+	{ 0x0000, 0xca9b, 0x0000, 0x006f, 0xb031 },
+	{ 0x0000, 0xca10, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xca12, 0x0000, 0x0078, 0xb031 },
+	{ 0x0000, 0xc960, 0x0000, 0x0078, 0xb031 },
+	{ 0x0000, 0xc961, 0x0000, 0x00e5, 0xb031 },
+	{ 0x0000, 0xc962, 0x0000, 0x00eb, 0xb031 },
+	{ 0x0000, 0xc963, 0x0000, 0x00c6, 0xb031 },
+	{ 0x0000, 0xc964, 0x0000, 0x00ba, 0xb031 },
+	{ 0x0000, 0xc965, 0x0000, 0x0061, 0xb031 },
+	{ 0x0000, 0xc967, 0x0000, 0x0017, 0xb031 },
+	{ 0x0000, 0xc968, 0x0000, 0x00ed, 0xb031 },
+	{ 0x0000, 0xc969, 0x0000, 0x003c, 0xb031 },
+	{ 0x0000, 0xc96a, 0x0000, 0x008a, 0xb031 },
+	{ 0x0000, 0xc96b, 0x0000, 0x00e2, 0xb031 },
+	{ 0x0000, 0xc975, 0x0000, 0x003a, 0xb031 },
+	{ 0x0000, 0xc97b, 0x0000, 0x006f, 0xb031 },
+	{ 0x0000, 0xc936, 0x0000, 0x00f9, 0xb031 },
+	{ 0x0000, 0xc9a0, 0x0000, 0x00f1, 0xb031 },
+	{ 0x0000, 0xc933, 0x0000, 0x000b, 0xb031 },
+	{ 0x0000, 0xc934, 0x0000, 0x0002, 0xb031 },
+	{ 0x0000, 0xc93f, 0x0000, 0x0014, 0xb031 },
+	{ 0x0000, 0xc93e, 0x0000, 0x0094, 0xb031 },
+	{ 0x0000, 0xc93a, 0x0000, 0x0076, 0xb031 },
+	{ 0x0000, 0xc93b, 0x0000, 0x0089, 0xb031 },
+	{ 0x0000, 0xc930, 0x0000, 0x0013, 0xb031 },
+	{ 0x0000, 0xc926, 0x0000, 0x00d1, 0xb031 },
+	{ 0x0000, 0xc990, 0x0000, 0x00f1, 0xb031 },
+	{ 0x0000, 0xc923, 0x0000, 0x0010, 0xb031 },
+	{ 0x0000, 0xc924, 0x0000, 0x0004, 0xb031 },
+	{ 0x0000, 0xc92f, 0x0000, 0x0018, 0xb031 },
+	{ 0x0000, 0xc92e, 0x0000, 0x0094, 0xb031 },
+	{ 0x0000, 0xc920, 0x0000, 0x0008, 0xb031 },
+	{ 0x0000, 0xc916, 0x0000, 0x00f9, 0xb031 },
+	{ 0x0000, 0xc980, 0x0000, 0x00e1, 0xb031 },
+	{ 0x0000, 0xc913, 0x0000, 0x0000, 0xb031 },
+	{ 0x0000, 0xc914, 0x0000, 0x0001, 0xb031 },
+	{ 0x0000, 0xc91f, 0x0000, 0x0014, 0xb031 },
+	{ 0x0000, 0xc91e, 0x0000, 0x0094, 0xb031 },
+	{ 0x0000, 0xc91a, 0x0000, 0x0066, 0xb031 },
+	{ 0x0000, 0xc91b, 0x0000, 0x0078, 0xb031 },
+	{ 0x0000, 0xc910, 0x0000, 0x000d, 0xb031 },
+	{ 0x0000, 0xc946, 0x0000, 0x00f9, 0xb031 },
+	{ 0x0000, 0xc9d0, 0x0000, 0x00f1, 0xb031 },
+	{ 0x0000, 0xc944, 0x0000, 0x0001, 0xb031 },
+	{ 0x0000, 0xc94f, 0x0000, 0x0014, 0xb031 },
+	{ 0x0000, 0xc94e, 0x0000, 0x0094, 0xb031 },
+	{ 0x0000, 0xc94a, 0x0000, 0x0026, 0xb031 },
+	{ 0x0000, 0xc94b, 0x0000, 0x0056, 0xb031 },
+	{ 0x0000, 0xc951, 0x0000, 0x0080, 0xb031 },
+	{ 0x0000, 0xc600, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xc601, 0x0000, 0x0026, 0xb031 },
+	{ 0x0000, 0xc602, 0x0000, 0x004a, 0xb031 },
+	{ 0x0000, 0xc603, 0x0000, 0x0010, 0xb031 },
+	{ 0x0000, 0xc604, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xc605, 0x0000, 0x00da, 0xb031 },
+	{ 0x0000, 0xc606, 0x0000, 0x000e, 0xb031 },
+	{ 0x0000, 0xc607, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xc610, 0x0000, 0x0007, 0xb031 },
+	{ 0x0000, 0xc611, 0x0000, 0x00da, 0xb031 },
+	{ 0x0000, 0xc612, 0x0000, 0x000e, 0xb031 },
+	{ 0x0000, 0xc613, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xc614, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xc615, 0x0000, 0x0026, 0xb031 },
+	{ 0x0000, 0xc616, 0x0000, 0x004a, 0xb031 },
+	{ 0x0000, 0xc617, 0x0000, 0x0010, 0xb031 },
+	{ 0x0000, 0xc618, 0x0000, 0x0008, 0xb031 },
+	{ 0x0000, 0xc541, 0x0000, 0x0030, 0xb031 },
+	{ 0x0000, 0xc802, 0x0000, 0x00d2, 0xb031 },
+	{ 0x0000, 0xc902, 0x0000, 0x00d2, 0xb031 },
+	{ 0x0000, 0xc800, 0x0000, 0x00f0, 0xb031 },
+	{ 0x0000, 0xd250, 0x0000, 0x0087, 0xb031 },
+	{ 0x0000, 0xd252, 0x0000, 0x0030, 0xb031 },
+	{ 0x0000, 0xd264, 0x0000, 0x00c0, 0xb031 },
+	{ 0x0000, 0xce60, 0x0000, 0x00e3, 0xb031 },
+	{ 0x0000, 0xdf00, 0x0000, 0x0010, 0xb031 },
+	{ 0x0000, 0xdbb5, 0x0041, 0xd27f, 0xb037 },
+	{ 0x0000, 0xddb5, 0x0040, 0x6e3b, 0xb037 },
+	{ 0x0000, 0xdb93, 0x0000, 0x009e, 0xb033 },
+	{ 0x0000, 0xdd93, 0x0000, 0x009e, 0xb033 },
+	{ 0x0000, 0xdb12, 0x0000, 0x00c0, 0xb031 },
+	{ 0x0000, 0xdd12, 0x0000, 0x00c0, 0xb031 },
+	{ 0x0000, 0xdb08, 0x0000, 0x0080, 0xb031 },
+	{ 0x0000, 0xdd08, 0x0000, 0x0080, 0xb031 },
+	{ 0x0000, 0xdb00, 0x0000, 0x00cc, 0xb031 },
+	{ 0x0000, 0xc203, 0x0000, 0x009c, 0xb031 },
+	{ 0x0000, 0x0010, 0x0000, 0x0f21, 0x0000 },
+};
+
+/* wake the amp (ea00=0x43, c121=0x0b, f109=0xe0) */
+static const struct alc298_razer_blade16_2025_op alc298_razer_blade16_2025_amp_wake[] = {
+	{ 0x0000, 0xea00, 0x0000, 0x0043, 0xb031 },
+	{ 0x0000, 0xc121, 0x0000, 0x000b, 0xb031 },
+	{ 0x0000, 0xf109, 0x0000, 0x00e0, 0xb031 },
+};
+
+/* park the amp (ea00=0x47, c121=0x0a, f109=0xa0) */
+static const struct alc298_razer_blade16_2025_op alc298_razer_blade16_2025_amp_sleep[] = {
+	{ 0x0000, 0xea00, 0x0000, 0x0047, 0xb031 },
+	{ 0x0000, 0xc121, 0x0000, 0x000a, 0xb031 },
+	{ 0x0000, 0xf109, 0x0000, 0x00a0, 0xb031 },
+};
+
+/*
+ * Razer Blade 16 (2025): Realtek ALC298 driving an external smart amp.
+ * Crossover, voicing and speaker protection live in an external DSP
+ * reached over a vendor-coef mailbox on NID 0x20; the captured DSP
+ * programming is in the tables above.  NID 0x14, the BIOS-disabled
+ * tweeter pin, is re-exposed via a pin-config override so the parser
+ * drives it as a normal internal speaker (shared volume/mute, auto-mute
+ * on headphone insertion).
+ *
+ * alc298_razer_blade16_2025_apply() runs one op-table: a plain coef write
+ * (strobe == 0) goes through alc_write_coef_idx(), while a mailbox op
+ * selects a bank and streams four PROC_COEF words (addr/hi/lo/commit)
+ * into a single coef index (0x23) on NID 0x20 to reach the external DSP
+ * -- that one-index/many-words form is why it cannot be expressed as an
+ * alc_process_coef_fw() table (one value per index).
+ */
+static void alc298_razer_blade16_2025_apply(struct hda_codec *codec,
+					    const struct alc298_razer_blade16_2025_op *seq,
+					    int num)
+{
+	int i;
+
+	for (i = 0; i < num; i++) {
+		if (!seq[i].strobe) {
+			alc_write_coef_idx(codec, seq[i].addr, seq[i].lo);
+			continue;
+		}
+		alc_write_coef_idx(codec, 0x89, seq[i].bank);
+		snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_COEF_INDEX, 0x23);
+		snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, seq[i].addr);
+		snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, seq[i].hi);
+		snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, seq[i].lo);
+		snd_hda_codec_write(codec, 0x20, 0, AC_VERB_SET_PROC_COEF, seq[i].strobe);
+	}
+}
+
+static void alc298_razer_blade16_2025_pcm_hook(struct hda_pcm_stream *hinfo,
+					       struct hda_codec *codec,
+					       struct snd_pcm_substream *substream,
+					       int action)
+{
+	/* power the external amp only while a stream is running */
+	switch (action) {
+	case HDA_GEN_PCM_ACT_PREPARE:
+		alc298_razer_blade16_2025_apply(codec, alc298_razer_blade16_2025_amp_wake,
+						ARRAY_SIZE(alc298_razer_blade16_2025_amp_wake));
+		break;
+	case HDA_GEN_PCM_ACT_CLEANUP:
+		alc298_razer_blade16_2025_apply(codec, alc298_razer_blade16_2025_amp_sleep,
+						ARRAY_SIZE(alc298_razer_blade16_2025_amp_sleep));
+		break;
+	}
+}
+
+static void alc298_fixup_razer_blade16_2025(struct hda_codec *codec,
+					    const struct hda_fixup *fix, int action)
+{
+	/*
+	 * Pin every output to a fixed converter so the routing does not
+	 * rely on the generic parser's DAC-allocation heuristics: without
+	 * a fixed assignment the parser brings up only one speaker pin and
+	 * leaves the other silent, and the allocation could change between
+	 * kernel versions.  Both speaker pins share DAC 0x03 through mixer
+	 * 0x0d: that is the only converter that drives the woofer (NID
+	 * 0x17) on this machine -- feeding it from DAC 0x02 or the digital
+	 * converter 0x06 leaves the woofer silent.  Both pins carry the
+	 * same stereo stream and the external amp does the crossover.
+	 * Headphone NID 0x21 keeps DAC 0x02.
+	 */
+	static const hda_nid_t preferred_pairs[] = {
+		0x21, 0x02,
+		0x14, 0x03,
+		0x17, 0x03,
+		0
+	};
+	struct alc_spec *spec = codec->spec;
+
+	switch (action) {
+	case HDA_FIXUP_ACT_PRE_PROBE:
+		spec->gen.preferred_dacs = preferred_pairs;
+		break;
+	case HDA_FIXUP_ACT_PROBE:
+		spec->gen.pcm_playback_hook = alc298_razer_blade16_2025_pcm_hook;
+		break;
+	case HDA_FIXUP_ACT_INIT:
+		/*
+		 * Wake the amp, program its DSP at boot and on resume, then
+		 * park it.  It is woken again for playback and parked
+		 * afterwards from the pcm_playback_hook.
+		 */
+		alc298_razer_blade16_2025_apply(codec, alc298_razer_blade16_2025_amp_wake,
+						ARRAY_SIZE(alc298_razer_blade16_2025_amp_wake));
+		alc298_razer_blade16_2025_apply(codec, alc298_razer_blade16_2025_amp_init,
+						ARRAY_SIZE(alc298_razer_blade16_2025_amp_init));
+		alc298_razer_blade16_2025_apply(codec, alc298_razer_blade16_2025_amp_sleep,
+						ARRAY_SIZE(alc298_razer_blade16_2025_amp_sleep));
+		break;
+	}
+}
diff --git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c
index f7700713dc628..b3562c821954e 100644
--- a/sound/hda/codecs/realtek/alc269.c
+++ b/sound/hda/codecs/realtek/alc269.c
@@ -3357,6 +3357,9 @@ static void alc287_fixup_acer_micmute_led(struct hda_codec *codec,
 /* for alc285_fixup_ideapad_s740_coef() */
 #include "../helpers/ideapad_s740.c"
 
+/* for alc298_fixup_razer_blade16_2025() */
+#include "../helpers/razer_blade16_2025.c"
+
 static const struct coef_fw alc256_fixup_set_coef_defaults_coefs[] = {
 	WRITE_COEF(0x10, 0x0020), WRITE_COEF(0x24, 0x0000),
 	WRITE_COEF(0x26, 0x0000), WRITE_COEF(0x29, 0x3000),
@@ -3773,6 +3776,15 @@ static void alc245_hp_spk_mute_led_update(void *private_data, int enabled)
 	alc_update_coef_idx(codec, 0x0b, 0x0c, val);
 }
 
+static void alc245_hp_spk_mute_led_update_inverted(void *private_data, int enabled)
+{
+	struct hda_codec *codec = private_data;
+	unsigned int val;
+
+	val = enabled ? 0x04 : 0x08; /* inverted: 0x04 led off, 0x08 led on */
+	alc_update_coef_idx(codec, 0x0b, 0x0c, val);
+}
+
 /* JD2: mute led GPIO3: micmute led */
 static void alc245_tas2781_i2c_hp_fixup_muteled(struct hda_codec *codec,
 					  const struct hda_fixup *fix, int action)
@@ -3801,6 +3813,33 @@ static void alc245_tas2781_i2c_hp_fixup_muteled(struct hda_codec *codec,
 		alc245_fixup_hp_mute_led_coefbit(codec, fix, action);
 	alc285_fixup_hp_coef_micmute_led(codec, fix, action);
 }
+
+/* Same as alc245_tas2781_i2c_hp_fixup_muteled(), but with inverted speaker
+ * mute LED polarity. The HP ZBook 8 G2a 14/16 (0x103c:0x8f94, 0x103c:0x8f95)
+ * are speaker-only models without an HP pin, so the LED is driven directly
+ * through the vmaster_mute hook.
+ */
+static void alc245_tas2781_i2c_hp_fixup_muteled_inverted(struct hda_codec *codec,
+					const struct hda_fixup *fix, int action)
+{
+	struct alc_spec *spec = codec->spec;
+	static const hda_nid_t conn[] = { 0x02 };
+
+	switch (action) {
+	case HDA_FIXUP_ACT_PRE_PROBE:
+		spec->gen.vmaster_mute.hook = alc245_hp_spk_mute_led_update_inverted;
+		spec->gen.vmaster_mute_led = 1;
+		spec->gen.auto_mute_via_amp = 1;
+		snd_hda_override_conn_list(codec, 0x17, ARRAY_SIZE(conn), conn);
+		break;
+	case HDA_FIXUP_ACT_INIT:
+		alc245_hp_spk_mute_led_update_inverted(codec, !spec->gen.master_mute);
+		break;
+	}
+
+	tas2781_fixup_txnw_i2c(codec, fix, action);
+	alc285_fixup_hp_coef_micmute_led(codec, fix, action);
+}
 /*
  * Clear COEF 0x0d (PCBEEP passthrough) bit 0x40 where BIOS sets it wrongly
  * at PM resume
@@ -3942,6 +3981,7 @@ enum {
 	ALC275_FIXUP_DELL_XPS,
 	ALC293_FIXUP_LENOVO_SPK_NOISE,
 	ALC233_FIXUP_LENOVO_LINE2_MIC_HOTKEY,
+	ALC233_FIXUP_WUJIE_SPEAKERS,
 	ALC233_FIXUP_LENOVO_L2MH_LOW_ENLED,
 	ALC255_FIXUP_DELL_SPK_NOISE,
 	ALC225_FIXUP_DISABLE_MIC_VREF,
@@ -4041,6 +4081,8 @@ enum {
 	ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS,
 	ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS,
 	ALC298_FIXUP_LG_GRAM_STYLE_14,
+	ALC298_FIXUP_RAZER_BLADE16_2025_PINS,
+	ALC298_FIXUP_RAZER_BLADE16_2025,
 	ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET,
 	ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET,
 	ALC295_FIXUP_ASUS_MIC_NO_PRESENCE,
@@ -4117,6 +4159,9 @@ enum {
 	ALC236_FIXUP_DELL_DUAL_CODECS,
 	ALC287_FIXUP_CS35L41_I2C_2_THINKPAD_ACPI,
 	ALC287_FIXUP_TAS2781_I2C,
+	ALC287_FIXUP_ASUS_ALLY_X,
+	ALC287_FIXUP_ASUS_ALLY_X_SPEAKER,
+	ALC287_FIXUP_ASUS_ALLY_X_I2C,
 	ALC295_FIXUP_DELL_TAS2781_I2C,
 	ALC245_FIXUP_TAS2781_SPI_2,
 	ALC287_FIXUP_TXNW2781_I2C,
@@ -4159,6 +4204,7 @@ enum {
 	ALC256_FIXUP_VAIO_RPL_MIC_NO_PRESENCE,
 	ALC245_FIXUP_HP_TAS2781_SPI_MUTE_LED,
 	ALC245_FIXUP_HP_TAS2781_I2C_MUTE_LED,
+	ALC245_FIXUP_HP_TAS2781_I2C_MUTE_LED_INVERTED,
 	ALC288_FIXUP_SURFACE_SWAP_DACS,
 	ALC236_FIXUP_HP_MUTE_LED_MICMUTE_GPIO,
 	ALC233_FIXUP_LENOVO_GPIO2_MIC_HOTKEY,
@@ -4169,6 +4215,7 @@ enum {
 	ALC256_FIXUP_HONOR_MRB_XXX_M1020_AUDIO,
 	ALC245_FIXUP_HP_ENVY_X360_15_FH0XXX,
 	ALC287_FIXUP_ACER_MICMUTE_LED,
+	ALC236_FIXUP_DELL_HP_POP_NOISE,
 };
 
 /* A special fixup for Lenovo C940 and Yoga Duet 7;
@@ -4206,6 +4253,27 @@ static void alc287_fixup_lenovo_yoga_book_9i(struct hda_codec *codec,
 }
 
 static const struct hda_fixup alc269_fixups[] = {
+	[ALC298_FIXUP_RAZER_BLADE16_2025_PINS] = {
+		.type = HDA_FIXUP_PINS,
+		.v.pins = (const struct hda_pintbl[]) {
+			{ 0x14, 0x90170121 }, /* tweeter as internal speaker, seq 1 */
+			{ 0x17, 0x90170120 }, /* woofer  as internal speaker, seq 0 */
+			{ }
+		},
+		.chained = true,
+		.chain_id = ALC298_FIXUP_RAZER_BLADE16_2025,
+	},
+	[ALC298_FIXUP_RAZER_BLADE16_2025] = {
+		.type = HDA_FIXUP_FUNC,
+		.v.func = alc298_fixup_razer_blade16_2025,
+	},
+	[ALC233_FIXUP_WUJIE_SPEAKERS] = {
+		.type = HDA_FIXUP_PINS,
+		.v.pins = (const struct hda_pintbl[]) {
+			{ 0x1b, 0x90170150 }, /* internal speaker */
+			{ }
+		},
+	},
 	[ALC269_FIXUP_GPIO2] = {
 		.type = HDA_FIXUP_FUNC,
 		.v.func = alc_fixup_gpio2,
@@ -6476,6 +6544,27 @@ static const struct hda_fixup alc269_fixups[] = {
 		.chained = true,
 		.chain_id = ALC285_FIXUP_THINKPAD_HEADSET_JACK,
 	},
+	[ALC287_FIXUP_ASUS_ALLY_X] = {
+		.type = HDA_FIXUP_PINS,
+		.v.pins = (const struct hda_pintbl[]) {
+			{ 0x19, 0x03a11050 }, /* headset mic */
+			{ }
+		},
+		.chained = true,
+		.chain_id = ALC287_FIXUP_ASUS_ALLY_X_SPEAKER,
+	},
+	[ALC287_FIXUP_ASUS_ALLY_X_SPEAKER] = {
+		.type = HDA_FIXUP_FUNC,
+		.v.func = alc285_fixup_speaker2_to_dac1,
+		.chained = true,
+		.chain_id = ALC287_FIXUP_ASUS_ALLY_X_I2C,
+	},
+	[ALC287_FIXUP_ASUS_ALLY_X_I2C] = {
+		.type = HDA_FIXUP_FUNC,
+		.v.func = tas2781_fixup_tias_i2c,
+		.chained = true,
+		.chain_id = ALC225_FIXUP_HEADSET_JACK,
+	},
 	[ALC245_FIXUP_TAS2781_SPI_2] = {
 		.type = HDA_FIXUP_FUNC,
 		.v.func = tas2781_fixup_spi,
@@ -6701,6 +6790,10 @@ static const struct hda_fixup alc269_fixups[] = {
 		.type = HDA_FIXUP_FUNC,
 		.v.func = alc245_tas2781_i2c_hp_fixup_muteled,
 	},
+	[ALC245_FIXUP_HP_TAS2781_I2C_MUTE_LED_INVERTED] = {
+		.type = HDA_FIXUP_FUNC,
+		.v.func = alc245_tas2781_i2c_hp_fixup_muteled_inverted,
+	},
 	[ALC288_FIXUP_SURFACE_SWAP_DACS] = {
 		.type = HDA_FIXUP_FUNC,
 		.v.func = alc288_fixup_surface_swap_dacs,
@@ -6755,6 +6848,10 @@ static const struct hda_fixup alc269_fixups[] = {
 		.chained = true,
 		.chain_id = ALC2XX_FIXUP_HEADSET_MIC,
 	},
+	[ALC236_FIXUP_DELL_HP_POP_NOISE] = {
+		.type = HDA_FIXUP_FUNC,
+		.v.func = alc285_fixup_invalidate_dacs,
+	},
 };
 
 static const struct hda_quirk alc269_fixup_tbl[] = {
@@ -6895,6 +6992,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1028, 0x0c1e, "Dell Precision 3540", ALC236_FIXUP_DELL_DUAL_CODECS),
 	SND_PCI_QUIRK(0x1028, 0x0c28, "Dell Inspiron 16 Plus 7630", ALC295_FIXUP_DELL_INSPIRON_TOP_SPEAKERS),
 	SND_PCI_QUIRK(0x1028, 0x0c4d, "Dell", ALC287_FIXUP_CS35L41_I2C_4),
+	SND_PCI_QUIRK(0x1028, 0x0c90, "Alienware x16 R2", ALC289_FIXUP_DUAL_SPK),
 	SND_PCI_QUIRK(0x1028, 0x0c94, "Dell Polaris 3 metal", ALC295_FIXUP_DELL_TAS2781_I2C),
 	SND_PCI_QUIRK(0x1028, 0x0c96, "Dell Polaris 2in1", ALC295_FIXUP_DELL_TAS2781_I2C),
 	SND_PCI_QUIRK(0x1028, 0x0cbd, "Dell Oasis 13 CS MTL-U", ALC289_FIXUP_DELL_CS35L41_SPI_2),
@@ -6906,6 +7004,8 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1028, 0x0cc3, "Dell Oasis 14 Low Weight MTL-U", ALC289_FIXUP_DELL_CS35L41_SPI_2),
 	SND_PCI_QUIRK(0x1028, 0x0cc4, "Dell Oasis 16 MTL-H/U", ALC289_FIXUP_DELL_CS35L41_SPI_2),
 	SND_PCI_QUIRK(0x1028, 0x0cc5, "Dell Oasis 14", ALC289_FIXUP_RTK_AMP_DUAL_SPK),
+	SND_PCI_QUIRK(0x1028, 0x0e6b, "Dell Pro QCM1255", ALC236_FIXUP_DELL_HP_POP_NOISE),
+	SND_PCI_QUIRK(0x1028, 0x0e6d, "Dell Pro Micro QCM1255", ALC236_FIXUP_DELL_HP_POP_NOISE),
 	SND_PCI_QUIRK(0x1028, 0x164a, "Dell", ALC293_FIXUP_DELL1_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0x1028, 0x164b, "Dell", ALC293_FIXUP_DELL1_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0x103c, 0x1586, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC2),
@@ -6992,6 +7092,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x856a, "HP Pavilion 15-cs1xxx", ALC295_FIXUP_HP_PAVILION_MUTE_LED_1B),
 	SND_PCI_QUIRK(0x103c, 0x85c6, "HP Pavilion x360 Convertible 14-dy1xxx", ALC295_FIXUP_HP_MUTE_LED_COEFBIT11),
 	SND_PCI_QUIRK(0x103c, 0x85de, "HP Envy x360 13-ar0xxx", ALC285_FIXUP_HP_ENVY_X360),
+	SND_PCI_QUIRK(0x103c, 0x85f0, "HP Laptop 15-dw0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2),
 	SND_PCI_QUIRK(0x103c, 0x8603, "HP Omen 17-cb0xxx", ALC285_FIXUP_HP_MUTE_LED),
 	SND_PCI_QUIRK(0x103c, 0x860c, "HP ZBook 17 G6", ALC285_FIXUP_HP_GPIO_AMP_INIT),
 	SND_PCI_QUIRK(0x103c, 0x860f, "HP ZBook 15 G6", ALC285_FIXUP_HP_GPIO_AMP_INIT),
@@ -7121,6 +7222,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x8a36, "HP Pavilion Plus 14-eh0xxx", ALC245_FIXUP_HP_MUTE_LED_COEFBIT),
 	SND_PCI_QUIRK(0x103c, 0x8a3d, "HP Victus 15-fb0xxx (MB 8A3D)", ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT),
 	SND_PCI_QUIRK(0x103c, 0x8a4f, "HP Victus 15-fa0xxx (MB 8A4F)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT),
+	SND_PCI_QUIRK(0x103c, 0x8a50, "HP Victus 15-fa0xxx (MB 8A50)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT),
 	SND_PCI_QUIRK(0x103c, 0x8a6e, "HP EDNA 360", ALC287_FIXUP_CS35L41_I2C_4),
 	SND_PCI_QUIRK(0x103c, 0x8a74, "HP ProBook 440 G8 Notebook PC", ALC236_FIXUP_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x8a75, "HP ProBook 450 G8 Notebook PC", ALC236_FIXUP_HP_GPIO_LED),
@@ -7131,6 +7233,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x8aa3, "HP ProBook 450 G9 (MB 8AA1)", ALC236_FIXUP_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x8aa8, "HP EliteBook 640 G9 (MB 8AA6)", ALC236_FIXUP_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x8aab, "HP EliteBook 650 G9 (MB 8AA9)", ALC236_FIXUP_HP_GPIO_LED),
+	SND_PCI_QUIRK(0x103c, 0x8ab8, "HP EliteBook 830 G8 Notebook PC (MB 8AB8)", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x8ab9, "HP EliteBook 840 G8 (MB 8AB8)", ALC285_FIXUP_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x8abb, "HP ZBook Firefly 14 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x8ad1, "HP EliteBook 840 14 inch G9 Notebook PC", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED),
@@ -7268,7 +7371,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x8da8, "HP 16 Piston OmniBook X", ALC245_FIXUP_HP_ENVY_X360_15_FH0XXX),
 	SND_PCI_QUIRK(0x103c, 0x8dc9, "HP Laptop 15-fc0xxx", ALC236_FIXUP_HP_DMIC),
 	SND_PCI_QUIRK(0x103c, 0x8dd4, "HP EliteStudio 8 AIO", ALC274_FIXUP_HP_AIO_BIND_DACS),
-	SND_PCI_QUIRK(0x103c, 0x8dd7, "HP Laptop 15-fd0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2),
+	SND_PCI_QUIRK(0x103c, 0x8dd7, "HP Laptop 15-fd0xxx", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_GPIO),
 	SND_PCI_QUIRK(0x103c, 0x8de8, "HP Gemtree", ALC245_FIXUP_TAS2781_SPI_2),
 	SND_PCI_QUIRK(0x103c, 0x8de9, "HP Gemtree", ALC245_FIXUP_TAS2781_SPI_2),
 	SND_PCI_QUIRK(0x103c, 0x8dec, "HP EliteBook 640 G12", ALC236_FIXUP_HP_GPIO_LED),
@@ -7341,6 +7444,8 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x8f42, "HP ZBook 8 G2a 14W", ALC245_FIXUP_HP_TAS2781_I2C_MUTE_LED),
 	SND_PCI_QUIRK(0x103c, 0x8f57, "HP Trekker G7JC", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x103c, 0x8f62, "HP ZBook 8 G2a 16W", ALC245_FIXUP_HP_TAS2781_I2C_MUTE_LED),
+	SND_PCI_QUIRK(0x103c, 0x8f94, "HP ZBook 8 G2a 14", ALC245_FIXUP_HP_TAS2781_I2C_MUTE_LED_INVERTED),
+	SND_PCI_QUIRK(0x103c, 0x8f95, "HP ZBook 8 G2a 16", ALC245_FIXUP_HP_TAS2781_I2C_MUTE_LED_INVERTED),
 	SND_PCI_QUIRK(0x1043, 0x1024, "ASUS Zephyrus G14 2025", ALC285_FIXUP_ASUS_GA403U_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1043, 0x1032, "ASUS VivoBook X513EA", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0x1043, 0x1034, "ASUS GU605C", ALC285_FIXUP_ASUS_GU605_SPI_SPEAKER2_TO_DAC1),
@@ -7460,7 +7565,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1043, 0x1e83, "ASUS GA605W", ALC285_FIXUP_ASUS_GU605_SPI_SPEAKER2_TO_DAC1),
 	SND_PCI_QUIRK(0x1043, 0x1e8e, "ASUS Zephyrus G15", ALC289_FIXUP_ASUS_GA401),
 	SND_PCI_QUIRK(0x1043, 0x1e93, "ASUS ExpertBook B9403CVAR", ALC294_FIXUP_ASUS_HPE),
-	SND_PCI_QUIRK(0x1043, 0x1eb3, "ASUS Ally RCLA72", ALC287_FIXUP_TAS2781_I2C),
+	SND_PCI_QUIRK(0x1043, 0x1eb3, "ASUS Ally RC72LA", ALC287_FIXUP_ASUS_ALLY_X),
 	SND_PCI_QUIRK(0x1043, 0x1ed3, "ASUS HN7306W", ALC287_FIXUP_CS35L41_I2C_2),
 	HDA_CODEC_QUIRK(0x1043, 0x1ee2, "ASUS UM6702RA/RC", ALC285_FIXUP_ASUS_I2C_SPEAKER2_TO_DAC1),
 	SND_PCI_QUIRK(0x1043, 0x1ee2, "ASUS UM6702RA/RC", ALC287_FIXUP_CS35L41_I2C_2),
@@ -7766,6 +7871,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	HDA_CODEC_QUIRK(0x17aa, 0x386e, "Legion Y9000X 2022 IAH7", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x17aa, 0x386e, "Yoga Pro 7 14ARP8", ALC285_FIXUP_SPEAKER2_TO_DAC1),
 	HDA_CODEC_QUIRK(0x17aa, 0x38a8, "Legion Pro 7 16ARX8H", ALC287_FIXUP_TAS2781_I2C), /* this must match before PCI SSID 17aa:386f below */
+	HDA_CODEC_QUIRK(0x17aa, 0x38a7, "Legion Pro 7 16ARX8H", ALC287_FIXUP_TAS2781_I2C), /* this must match before PCI SSID 17aa:386f below */
 	SND_PCI_QUIRK(0x17aa, 0x386f, "Legion Pro 7i 16IAX7", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x17aa, 0x3870, "Lenovo Yoga 7 14ARB7", ALC287_FIXUP_YOGA7_14ARB7_I2C),
 	SND_PCI_QUIRK(0x17aa, 0x3874, "Legion 7i 16IAX7", ALC287_FIXUP_CS35L41_I2C_2),
@@ -7863,6 +7969,8 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x19e5, 0x3204, "Huawei MACH-WX9", ALC256_FIXUP_HUAWEI_MACH_WX9_PINS),
 	SND_PCI_QUIRK(0x19e5, 0x320f, "Huawei WRT-WX9 ", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0x19e5, 0x3212, "Huawei KLV-WX9 ", ALC256_FIXUP_ACER_HEADSET_MIC),
+	SND_PCI_QUIRK(0x1a58, 0x300e, "Razer Blade 16 (2025)",
+		      ALC298_FIXUP_RAZER_BLADE16_2025_PINS),
 	SND_PCI_QUIRK(0x1b35, 0x1235, "CZC B20", ALC269_FIXUP_CZC_B20),
 	SND_PCI_QUIRK(0x1b35, 0x1236, "CZC TMI", ALC269_FIXUP_CZC_TMI),
 	SND_PCI_QUIRK(0x1b35, 0x1237, "CZC L101", ALC269_FIXUP_CZC_L101),
@@ -7885,6 +7993,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1d05, 0x300f, "TongFang X6AR5xxY", ALC2XX_FIXUP_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1d05, 0x3019, "TongFang X6FR5xxY", ALC2XX_FIXUP_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1d05, 0x3031, "TongFang X6AR55xU", ALC2XX_FIXUP_HEADSET_MIC),
+	SND_PCI_QUIRK(0x1d05, 0x3034, "TongFang X6xx45xU", ALC2XX_FIXUP_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1d17, 0x3288, "Haier Boyue G42", ALC269VC_FIXUP_ACER_VCOPPERBOX_PINS),
 	SND_PCI_QUIRK(0x1d72, 0x1602, "RedmiBook", ALC255_FIXUP_XIAOMI_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1d72, 0x1701, "XiaomiNotebook Pro", ALC298_FIXUP_DELL1_MIC_NO_PRESENCE),
@@ -7915,6 +8024,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x8086, 0x2080, "Intel NUC 8 Rugged", ALC256_FIXUP_INTEL_NUC8_RUGGED),
 	SND_PCI_QUIRK(0x8086, 0x2081, "Intel NUC 10", ALC256_FIXUP_INTEL_NUC10),
 	SND_PCI_QUIRK(0x8086, 0x3038, "Intel NUC 13", ALC295_FIXUP_CHROME_BOOK),
+	SND_PCI_QUIRK(0xc011, 0x1d05, "MECHREVO WUJIE Series", ALC233_FIXUP_WUJIE_SPEAKERS),
 	SND_PCI_QUIRK(0xf111, 0x0001, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0xf111, 0x0006, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0xf111, 0x0009, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE),
@@ -7922,6 +8032,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0xf111, 0x000c, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0xf111, 0x000f, "Framework Laptop 13 Pro PTL", ALC295_FIXUP_FRAMEWORK_LAPTOP_LIMIT_INT_MIC_BOOST),
 	SND_PCI_QUIRK(0xf111, 0x010f, "Framework Laptop 13 PTL", ALC295_FIXUP_FRAMEWORK_LAPTOP_LIMIT_INT_MIC_BOOST),
+	SND_PCI_QUIRK(0xf111, 0x0010, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_LIMIT_INT_MIC_BOOST),
 
 #if 0
 	/* Below is a quirk table taken from the old code.
diff --git a/sound/hda/codecs/side-codecs/Kconfig b/sound/hda/codecs/side-codecs/Kconfig
index 2a2e8804bf9e4..1cfd83e251e4b 100644
--- a/sound/hda/codecs/side-codecs/Kconfig
+++ b/sound/hda/codecs/side-codecs/Kconfig
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config SND_HDA_CIRRUS_SCODEC
-	tristate
+	tristate "Cirrus side-codec library" if KUNIT
 
 config SND_HDA_CIRRUS_SCODEC_KUNIT_TEST
 	tristate "KUnit test for Cirrus side-codec library" if !KUNIT_ALL_TESTS
diff --git a/sound/hda/codecs/side-codecs/cs35l41_hda.c b/sound/hda/codecs/side-codecs/cs35l41_hda.c
index 64a5bd895fd1e..237059ef22f5c 100644
--- a/sound/hda/codecs/side-codecs/cs35l41_hda.c
+++ b/sound/hda/codecs/side-codecs/cs35l41_hda.c
@@ -1434,10 +1434,19 @@ static int cs35l41_get_acpi_mute_state(struct cs35l41_hda *cs35l41, acpi_handle
 	guid_parse(CS35L41_UUID, &guid);
 
 	if (cs35l41_dsm_supported(handle, CS35L41_DSM_GET_MUTE)) {
-		ret = acpi_evaluate_dsm(handle, &guid, 0, CS35L41_DSM_GET_MUTE, NULL);
+		ret = acpi_evaluate_dsm_typed(handle, &guid, 0,
+			      CS35L41_DSM_GET_MUTE, NULL,
+			      ACPI_TYPE_BUFFER);
+
 		if (!ret)
 			return -EINVAL;
+		if (!ret->buffer.length || !ret->buffer.pointer) {
+			ACPI_FREE(ret);
+			return -EINVAL;
+		}
+
 		mute = *ret->buffer.pointer;
+		ACPI_FREE(ret);
 		dev_dbg(cs35l41->dev, "CS35L41_DSM_GET_MUTE: %d\n", mute);
 	}
 
diff --git a/sound/hda/codecs/side-codecs/cs35l56_hda.c b/sound/hda/codecs/side-codecs/cs35l56_hda.c
index a0ea08eb96a93..78c2cf387a001 100644
--- a/sound/hda/codecs/side-codecs/cs35l56_hda.c
+++ b/sound/hda/codecs/side-codecs/cs35l56_hda.c
@@ -512,20 +512,6 @@ static void cs35l56_hda_request_firmware_files(struct cs35l56_hda *cs35l56,
 								  NULL, "bin");
 			return;
 		}
-
-		/*
-		 * Check for system-specific bin files without wmfw before
-		 * falling back to generic firmware
-		 */
-		if (amp_name)
-			cs35l56_hda_request_firmware_file(cs35l56, coeff_firmware, coeff_filename,
-							  base_name, system_name, amp_name, "bin");
-		if (!*coeff_firmware)
-			cs35l56_hda_request_firmware_file(cs35l56, coeff_firmware, coeff_filename,
-							  base_name, system_name, NULL, "bin");
-
-		if (*coeff_firmware)
-			return;
 	}
 
 	ret = cs35l56_hda_request_firmware_file(cs35l56, wmfw_firmware, wmfw_filename,
@@ -616,13 +602,15 @@ static void cs35l56_hda_fw_load(struct cs35l56_hda *cs35l56)
 					   &wmfw_firmware, &wmfw_filename,
 					   &coeff_firmware, &coeff_filename);
 
-	/*
-	 * If the BIOS didn't patch the firmware a bin file is mandatory to
-	 * enable the ASP·
-	 */
-	if (!coeff_firmware && firmware_missing) {
-		dev_err(cs35l56->base.dev, ".bin file required but not found\n");
-		goto err_fw_release;
+	/* If the BIOS didn't patch the firmware a wmfw and bin file are mandatory */
+	if (firmware_missing) {
+		if (!wmfw_firmware) {
+			dev_err(cs35l56->base.dev, ".%s file required but not found\n", "wmfw");
+			goto err_fw_release;
+		} else if (!coeff_firmware) {
+			dev_err(cs35l56->base.dev, ".%s file required but not found\n", "bin");
+			goto err_fw_release;
+		}
 	}
 
 	mutex_lock(&cs35l56->base.irq_lock);
diff --git a/sound/hda/common/codec.c b/sound/hda/common/codec.c
index ef533770179b4..b9ded149ea11e 100644
--- a/sound/hda/common/codec.c
+++ b/sound/hda/common/codec.c
@@ -2967,8 +2967,11 @@ static void hda_codec_pm_complete(struct device *dev)
 		dev->power.power_state = PMSG_RESUME;
 
 	if (pm_runtime_suspended(dev) && (codec->jackpoll_interval ||
-	    hda_codec_need_resume(codec) || codec->forced_resume))
+	    hda_codec_need_resume(codec) || codec->forced_resume ||
+	    codec->acomp_requested_resume)) {
+		codec->acomp_requested_resume = 0;
 		pm_request_resume(dev);
+	}
 }
 
 static int hda_codec_pm_suspend(struct device *dev)
diff --git a/sound/hda/controllers/intel.c b/sound/hda/controllers/intel.c
index 4b03c64e72ab4..192f5c044b7a2 100644
--- a/sound/hda/controllers/intel.c
+++ b/sound/hda/controllers/intel.c
@@ -100,6 +100,8 @@ enum {
 #define ATIHDMI_NUM_CAPTURE	0
 #define ATIHDMI_NUM_PLAYBACK	8
 
+/* Hygon HD Audio controller */
+#define PCI_DEVICE_ID_HYGON_18H_M05H_HDA	0x14a9
 
 static int index[SNDRV_CARDS] = SNDRV_DEFAULT_IDX;
 static char *id[SNDRV_CARDS] = SNDRV_DEFAULT_STR;
@@ -241,6 +243,7 @@ enum {
 	AZX_DRIVER_ZHAOXIN,
 	AZX_DRIVER_ZHAOXINHDMI,
 	AZX_DRIVER_LOONGSON,
+	AZX_DRIVER_HYGON,
 	AZX_DRIVER_GENERIC,
 	AZX_NUM_DRIVERS, /* keep this as last entry */
 };
@@ -360,6 +363,7 @@ static const char * const driver_short_names[] = {
 	[AZX_DRIVER_ZHAOXIN] = "HDA Zhaoxin",
 	[AZX_DRIVER_ZHAOXINHDMI] = "HDA Zhaoxin HDMI",
 	[AZX_DRIVER_LOONGSON] = "HDA Loongson",
+	[AZX_DRIVER_HYGON] = "HDA Hygon",
 	[AZX_DRIVER_GENERIC] = "HD-Audio Generic",
 };
 
@@ -1926,6 +1930,10 @@ static int azx_first_init(struct azx *chip)
 	if (chip->driver_type == AZX_DRIVER_ZHAOXINHDMI)
 		bus->polling_mode = 1;
 
+	if (chip->driver_type == AZX_DRIVER_HYGON &&
+	    chip->pci->device == PCI_DEVICE_ID_HYGON_18H_M05H_HDA)
+		bus->access_sdnctl_in_dword = 1;
+
 	bus->remap_addr = pcim_iomap_region(pci, 0, "ICH HD audio");
 	if (IS_ERR(bus->remap_addr))
 		return PTR_ERR(bus->remap_addr);
@@ -2836,6 +2844,9 @@ static const struct pci_device_id azx_ids[] = {
 	  .driver_data = AZX_DRIVER_LOONGSON | AZX_DCAPS_NO_TCSEL },
 	{ PCI_VDEVICE(LOONGSON, PCI_DEVICE_ID_LOONGSON_HDMI),
 	  .driver_data = AZX_DRIVER_LOONGSON | AZX_DCAPS_NO_TCSEL },
+	/* Hygon HDAudio */
+	{ PCI_VDEVICE(HYGON, PCI_DEVICE_ID_HYGON_18H_M05H_HDA),
+	  .driver_data = AZX_DRIVER_HYGON | AZX_DCAPS_POSFIX_LPIB | AZX_DCAPS_NO_MSI },
 	{ 0, }
 };
 MODULE_DEVICE_TABLE(pci, azx_ids);
diff --git a/sound/hda/core/controller.c b/sound/hda/core/controller.c
index 69e11d62bbfa7..32048ef32d966 100644
--- a/sound/hda/core/controller.c
+++ b/sound/hda/core/controller.c
@@ -511,7 +511,10 @@ void snd_hdac_bus_exit_link_reset(struct hdac_bus *bus)
 {
 	unsigned long timeout;
 
-	snd_hdac_chip_updateb(bus, GCTL, AZX_GCTL_RESET, AZX_GCTL_RESET);
+	if (bus->access_sdnctl_in_dword)
+		snd_hdac_chip_updatel(bus, GCTL, AZX_GCTL_RESET, AZX_GCTL_RESET);
+	else
+		snd_hdac_chip_updateb(bus, GCTL, AZX_GCTL_RESET, AZX_GCTL_RESET);
 
 	timeout = jiffies + msecs_to_jiffies(100);
 	while (!snd_hdac_chip_readb(bus, GCTL) && time_before(jiffies, timeout))
@@ -576,7 +579,10 @@ static void azx_int_disable(struct hdac_bus *bus)
 
 	/* disable interrupts in stream descriptor */
 	list_for_each_entry(azx_dev, &bus->stream_list, list)
-		snd_hdac_stream_updateb(azx_dev, SD_CTL, SD_INT_MASK, 0);
+		if (bus->access_sdnctl_in_dword)
+			snd_hdac_stream_updatel(azx_dev, SD_CTL, SD_INT_MASK, 0);
+		else
+			snd_hdac_stream_updateb(azx_dev, SD_CTL, SD_INT_MASK, 0);
 
 	/* disable SIE for all streams & disable controller CIE and GIE */
 	snd_hdac_chip_writel(bus, INTCTL, 0);
@@ -713,7 +719,7 @@ int snd_hdac_bus_alloc_stream_pages(struct hdac_bus *bus)
 					  BDL_SIZE, &s->bdl);
 		num_streams++;
 		if (err < 0)
-			return -ENOMEM;
+			goto error_bdl;
 	}
 
 	if (WARN_ON(!num_streams))
@@ -722,12 +728,24 @@ int snd_hdac_bus_alloc_stream_pages(struct hdac_bus *bus)
 	err = snd_dma_alloc_pages(dma_type, bus->dev,
 				  num_streams * 8, &bus->posbuf);
 	if (err < 0)
-		return -ENOMEM;
+		goto error_bdl;
 	list_for_each_entry(s, &bus->stream_list, list)
 		s->posbuf = (__le32 *)(bus->posbuf.area + s->index * 8);
 
 	/* single page (at least 4096 bytes) must suffice for both ringbuffes */
-	return snd_dma_alloc_pages(dma_type, bus->dev, PAGE_SIZE, &bus->rb);
+	err = snd_dma_alloc_pages(dma_type, bus->dev, PAGE_SIZE, &bus->rb);
+	if (err < 0)
+		goto error_posbuf;
+	return 0;
+
+error_posbuf:
+	snd_dma_free_pages(&bus->posbuf);
+error_bdl:
+	list_for_each_entry(s, &bus->stream_list, list) {
+		if (s->bdl.area)
+			snd_dma_free_pages(&s->bdl);
+	}
+	return -ENOMEM;
 }
 EXPORT_SYMBOL_GPL(snd_hdac_bus_alloc_stream_pages);
 
diff --git a/sound/hda/core/regmap.c b/sound/hda/core/regmap.c
index e7b866fc52c15..d6eb17aa9e086 100644
--- a/sound/hda/core/regmap.c
+++ b/sound/hda/core/regmap.c
@@ -214,7 +214,7 @@ static int hda_reg_read_coef(struct hdac_device *codec, unsigned int reg,
 	err = snd_hdac_exec_verb(codec, verb, 0, NULL);
 	if (err < 0)
 		return err;
-	verb = (reg & ~0xfffff) | (AC_VERB_GET_COEF_INDEX << 8);
+	verb = (reg & ~0xfffff) | (AC_VERB_GET_PROC_COEF << 8);
 	return snd_hdac_exec_verb(codec, verb, 0, val);
 }
 
@@ -232,7 +232,7 @@ static int hda_reg_write_coef(struct hdac_device *codec, unsigned int reg,
 	err = snd_hdac_exec_verb(codec, verb, 0, NULL);
 	if (err < 0)
 		return err;
-	verb = (reg & ~0xfffff) | (AC_VERB_GET_COEF_INDEX << 8) |
+	verb = (reg & ~0xfffff) | (AC_VERB_SET_PROC_COEF << 8) |
 		(val & 0xffff);
 	return snd_hdac_exec_verb(codec, verb, 0, NULL);
 }
diff --git a/sound/hda/core/stream.c b/sound/hda/core/stream.c
index b471a038b3140..0091c4ef64868 100644
--- a/sound/hda/core/stream.c
+++ b/sound/hda/core/stream.c
@@ -146,8 +146,12 @@ void snd_hdac_stream_start(struct hdac_stream *azx_dev)
 			stripe_ctl = snd_hdac_get_stream_stripe_ctl(bus, azx_dev->substream);
 		else
 			stripe_ctl = 0;
-		snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK,
-					stripe_ctl);
+		if (bus->access_sdnctl_in_dword)
+			snd_hdac_stream_updatel(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK,
+						stripe_ctl);
+		else
+			snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK,
+						stripe_ctl);
 	}
 	/* set DMA start and interrupt mask */
 	if (bus->access_sdnctl_in_dword)
@@ -166,11 +170,22 @@ EXPORT_SYMBOL_GPL(snd_hdac_stream_start);
  */
 static void snd_hdac_stream_clear(struct hdac_stream *azx_dev)
 {
-	snd_hdac_stream_updateb(azx_dev, SD_CTL,
-				SD_CTL_DMA_START | SD_INT_MASK, 0);
-	snd_hdac_stream_writeb(azx_dev, SD_STS, SD_INT_MASK); /* to be sure */
-	if (azx_dev->stripe)
-		snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, 0);
+	struct hdac_bus *bus = azx_dev->bus;
+
+	if (bus->access_sdnctl_in_dword) {
+		snd_hdac_stream_updatel(azx_dev, SD_CTL,
+					SD_CTL_DMA_START | SD_INT_MASK, 0);
+		snd_hdac_stream_writeb(azx_dev, SD_STS, SD_INT_MASK); /* to be sure */
+		if (azx_dev->stripe)
+			snd_hdac_stream_updatel(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, 0);
+	} else {
+		snd_hdac_stream_updateb(azx_dev, SD_CTL,
+					SD_CTL_DMA_START | SD_INT_MASK, 0);
+		snd_hdac_stream_writeb(azx_dev, SD_STS, SD_INT_MASK); /* to be sure */
+		if (azx_dev->stripe)
+			snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, 0);
+	}
+
 	azx_dev->running = false;
 }
 
@@ -225,12 +240,16 @@ void snd_hdac_stream_reset(struct hdac_stream *azx_dev)
 {
 	unsigned char val;
 	int dma_run_state;
+	struct hdac_bus *bus = azx_dev->bus;
 
 	snd_hdac_stream_clear(azx_dev);
 
 	dma_run_state = snd_hdac_stream_readb(azx_dev, SD_CTL) & SD_CTL_DMA_START;
 
-	snd_hdac_stream_updateb(azx_dev, SD_CTL, 0, SD_CTL_STREAM_RESET);
+	if (bus->access_sdnctl_in_dword)
+		snd_hdac_stream_updatel(azx_dev, SD_CTL, 0, SD_CTL_STREAM_RESET);
+	else
+		snd_hdac_stream_updateb(azx_dev, SD_CTL, 0, SD_CTL_STREAM_RESET);
 
 	/* wait for hardware to report that the stream entered reset */
 	snd_hdac_stream_readb_poll(azx_dev, SD_CTL, val, (val & SD_CTL_STREAM_RESET), 3, 300);
@@ -238,7 +257,10 @@ void snd_hdac_stream_reset(struct hdac_stream *azx_dev)
 	if (azx_dev->bus->dma_stop_delay && dma_run_state)
 		udelay(azx_dev->bus->dma_stop_delay);
 
-	snd_hdac_stream_updateb(azx_dev, SD_CTL, SD_CTL_STREAM_RESET, 0);
+	if (bus->access_sdnctl_in_dword)
+		snd_hdac_stream_updatel(azx_dev, SD_CTL, SD_CTL_STREAM_RESET, 0);
+	else
+		snd_hdac_stream_updateb(azx_dev, SD_CTL, SD_CTL_STREAM_RESET, 0);
 
 	/* wait for hardware to report that the stream is out of reset */
 	snd_hdac_stream_readb_poll(azx_dev, SD_CTL, val, !(val & SD_CTL_STREAM_RESET), 3, 300);
diff --git a/sound/pci/asihpi/hpi6000.c b/sound/pci/asihpi/hpi6000.c
index c8d1518ee3e74..fd7fe9dba0b80 100644
--- a/sound/pci/asihpi/hpi6000.c
+++ b/sound/pci/asihpi/hpi6000.c
@@ -537,6 +537,11 @@ static short create_adapter_obj(struct hpi_adapter_obj *pao,
 		hr1.size = sizeof(hr1);
 
 		error = hpi6000_message_response_sequence(pao, 0, &hm, &hr0);
+		if (error) {
+			HPI_DEBUG_LOG(ERROR, "message transport error %d\n",
+						  error);
+			return error;
+		}
 		if (hr0.error) {
 			HPI_DEBUG_LOG(DEBUG, "message error %d\n", hr0.error);
 			return hr0.error;
diff --git a/sound/pci/via82xx_modem.c b/sound/pci/via82xx_modem.c
index 9b84d3fb9eaf5..b32f84ac17cc1 100644
--- a/sound/pci/via82xx_modem.c
+++ b/sound/pci/via82xx_modem.c
@@ -573,24 +573,18 @@ static inline unsigned int calc_linear_pos(struct via82xx_modem *chip,
 		       viadev->bufsize2, viadev->idx_table[idx].offset,
 		       viadev->idx_table[idx].size, count);
 #endif
-		if (count && size < count) {
+		if (! count)
+			/* bogus count 0 on the DMA boundary? */
+			res = viadev->idx_table[idx].offset;
+		else
+			/* count register returns full size
+			 * when end of buffer is reached
+			 */
+			res = viadev->idx_table[idx].offset + size;
+		if (check_invalid_pos(viadev, res)) {
 			dev_dbg(chip->card->dev,
-				"invalid via82xx_cur_ptr, using last valid pointer\n");
+				"invalid via82xx_cur_ptr (2), using last valid pointer\n");
 			res = viadev->lastpos;
-		} else {
-			if (! count)
-				/* bogus count 0 on the DMA boundary? */
-				res = viadev->idx_table[idx].offset;
-			else
-				/* count register returns full size
-				 * when end of buffer is reached
-				 */
-				res = viadev->idx_table[idx].offset + size;
-			if (check_invalid_pos(viadev, res)) {
-				dev_dbg(chip->card->dev,
-					"invalid via82xx_cur_ptr (2), using last valid pointer\n");
-				res = viadev->lastpos;
-			}
 		}
 	}
 	viadev->lastpos = res; /* remember the last position */
diff --git a/sound/soc/amd/acp-config.c b/sound/soc/amd/acp-config.c
index 0d977f4f758d9..f4c1bae878c1a 100644
--- a/sound/soc/amd/acp-config.c
+++ b/sound/soc/amd/acp-config.c
@@ -44,6 +44,18 @@ static const struct dmi_system_id acp70_acpi_flag_override_table[] = {
 			DMI_MATCH(DMI_PRODUCT_NAME, "83W5"),
 		},
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "TUF Gaming A14 FA401EA"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Vivobook 18 M1807GA"),
+		},
+	},
 	{}
 };
 
@@ -360,5 +372,17 @@ struct snd_soc_acpi_mach snd_soc_acpi_amd_acp70_sof_machines[] = {
 };
 EXPORT_SYMBOL(snd_soc_acpi_amd_acp70_sof_machines);
 
+struct snd_soc_acpi_mach snd_soc_acpi_amd_acp7x_sof_machines[] = {
+	{
+		.id = "AMDI1010",
+		.drv_name = "acp7x-dsp",
+		.pdata = &acp_quirk_data,
+		.fw_filename = "sof-acp7x.ri",
+		.sof_tplg_filename = "sof-acp7x.tplg",
+	},
+	{},
+};
+EXPORT_SYMBOL(snd_soc_acpi_amd_acp7x_sof_machines);
+
 MODULE_DESCRIPTION("AMD ACP Machine Configuration Module");
 MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sound/soc/amd/acp/Kconfig b/sound/soc/amd/acp/Kconfig
index 977e4f2a7a701..a04c301df4d61 100644
--- a/sound/soc/amd/acp/Kconfig
+++ b/sound/soc/amd/acp/Kconfig
@@ -15,13 +15,17 @@ config SND_SOC_AMD_ACP_COMMON
 
 config SND_SOC_ACPI_AMD_MATCH
 	tristate
-        select SND_SOC_ACPI_AMD_SDCA_QUIRKS if SND_SOC_SDCA
         select SND_SOC_ACPI if ACPI
 
 config SND_SOC_ACPI_AMD_SDCA_QUIRKS
-	tristate
-	depends on ACPI
+	bool "AMD ACPI SDCA quirks"
+	depends on SND_SOC_ACPI_AMD_MATCH
 	depends on SND_SOC_SDCA
+	depends on SND_SOC_ACPI_AMD_MATCH = m || SND_SOC_SDCA = y
+	default y
+	help
+	  Enable SDCA quirk support for AMD ACPI match tables.
+	  This is compiled into the snd-soc-acpi-amd-match module.
 
 if SND_SOC_AMD_ACP_COMMON
 
diff --git a/sound/soc/amd/acp/Makefile b/sound/soc/amd/acp/Makefile
index 81d23aded348d..ab5f9dc871a67 100644
--- a/sound/soc/amd/acp/Makefile
+++ b/sound/soc/amd/acp/Makefile
@@ -24,10 +24,10 @@ snd-acp-mach-y     := acp-mach-common.o
 snd-acp-legacy-mach-y     := acp-legacy-mach.o acp3x-es83xx/acp3x-es83xx.o
 snd-acp-sof-mach-y     := acp-sof-mach.o
 snd-soc-acpi-amd-match-y := amd-acp63-acpi-match.o amd-acp70-acpi-match.o
+snd-soc-acpi-amd-match-$(CONFIG_SND_SOC_ACPI_AMD_SDCA_QUIRKS) += soc-acpi-amd-sdca-quirks.o
 snd-acp-sdw-mach-y     := acp-sdw-mach-common.o
 snd-acp-sdw-sof-mach-y += acp-sdw-sof-mach.o
 snd-acp-sdw-legacy-mach-y += acp-sdw-legacy-mach.o
-snd-soc-acpi-amd-sdca-quirks-y += soc-acpi-amd-sdca-quirks.o
 
 obj-$(CONFIG_SND_SOC_AMD_ACP_PCM) += snd-acp-pcm.o
 obj-$(CONFIG_SND_SOC_AMD_ACP_I2S) += snd-acp-i2s.o
@@ -41,7 +41,6 @@ obj-$(CONFIG_SND_AMD_ASOC_REMBRANDT) += snd-acp-rembrandt.o
 obj-$(CONFIG_SND_AMD_ASOC_ACP63) += snd-acp63.o
 obj-$(CONFIG_SND_AMD_ASOC_ACP70) += snd-acp70.o
 
-obj-$(CONFIG_SND_SOC_ACPI_AMD_SDCA_QUIRKS) += snd-soc-acpi-amd-sdca-quirks.o
 obj-$(CONFIG_SND_AMD_SOUNDWIRE_ACPI) += snd-amd-sdw-acpi.o
 obj-$(CONFIG_SND_SOC_AMD_MACH_COMMON) += snd-acp-mach.o
 obj-$(CONFIG_SND_SOC_AMD_LEGACY_MACH) += snd-acp-legacy-mach.o
diff --git a/sound/soc/amd/acp/acp-i2s.c b/sound/soc/amd/acp/acp-i2s.c
index 283a674c7e2c3..bb58a9d34993b 100644
--- a/sound/soc/amd/acp/acp-i2s.c
+++ b/sound/soc/amd/acp/acp-i2s.c
@@ -686,6 +686,10 @@ static int acp_i2s_startup(struct snd_pcm_substream *substream, struct snd_soc_d
 	return 0;
 }
 
+static const u64 acp_i2s_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A;
+
 const struct snd_soc_dai_ops asoc_acp_cpu_dai_ops = {
 	.startup	= acp_i2s_startup,
 	.hw_params	= acp_i2s_hwparams,
@@ -693,6 +697,8 @@ const struct snd_soc_dai_ops asoc_acp_cpu_dai_ops = {
 	.trigger	= acp_i2s_trigger,
 	.set_fmt	= acp_i2s_set_fmt,
 	.set_tdm_slot	= acp_i2s_set_tdm_slot,
+	.auto_selectable_formats	= &acp_i2s_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 };
 EXPORT_SYMBOL_NS_GPL(asoc_acp_cpu_dai_ops, "SND_SOC_ACP_COMMON");
 
diff --git a/sound/soc/amd/acp/acp-sdw-legacy-mach.c b/sound/soc/amd/acp/acp-sdw-legacy-mach.c
index e8b6819cc4b45..9726a9d33ec6e 100644
--- a/sound/soc/amd/acp/acp-sdw-legacy-mach.c
+++ b/sound/soc/amd/acp/acp-sdw-legacy-mach.c
@@ -432,7 +432,7 @@ static int soc_card_dai_links_create(struct snd_soc_card *card)
 	if (!soc_aux)
 		return -ENOMEM;
 
-	ret = asoc_sdw_parse_sdw_endpoints(card, soc_aux, soc_dais, soc_ends, &num_confs);
+	ret = asoc_sdw_parse_sdw_endpoints(dev, ctx, soc_aux, soc_dais, soc_ends, &num_confs);
 	if (ret < 0)
 		return ret;
 
diff --git a/sound/soc/amd/acp/acp-sdw-sof-mach.c b/sound/soc/amd/acp/acp-sdw-sof-mach.c
index a423853f3a97d..963ce6fd40122 100644
--- a/sound/soc/amd/acp/acp-sdw-sof-mach.c
+++ b/sound/soc/amd/acp/acp-sdw-sof-mach.c
@@ -303,7 +303,7 @@ static int sof_card_dai_links_create(struct snd_soc_card *card)
 	if (!sof_aux)
 		return -ENOMEM;
 
-	ret = asoc_sdw_parse_sdw_endpoints(card, sof_aux, sof_dais, sof_ends, &num_devs);
+	ret = asoc_sdw_parse_sdw_endpoints(dev, ctx, sof_aux, sof_dais, sof_ends, &num_devs);
 	if (ret < 0)
 		return ret;
 
diff --git a/sound/soc/amd/acp/amd-acp70-acpi-match.c b/sound/soc/amd/acp/amd-acp70-acpi-match.c
index 18f2918d4ada4..ccd01152c87d0 100644
--- a/sound/soc/amd/acp/amd-acp70-acpi-match.c
+++ b/sound/soc/amd/acp/amd-acp70-acpi-match.c
@@ -553,6 +553,46 @@ static const struct snd_soc_acpi_link_adr acp70_rt722_l0_rt1320_l1[] = {
 	{}
 };
 
+static const struct snd_soc_acpi_adr_device tas2783_2_adr[] = {
+	{
+		/* left */
+		.adr = 0x00003c0102000001ull,
+		.num_endpoints = 1,
+		.endpoints = &spk_l_endpoint,
+		.name_prefix = "tas2783-1",
+	},
+	{
+		/* right */
+		.adr = 0x00003d0102000001ull,
+		.num_endpoints = 1,
+		.endpoints = &spk_l_endpoint,
+		.name_prefix = "tas2783-2",
+	},
+	{
+		/* left */
+		.adr = 0x0000390102000001ull,
+		.num_endpoints = 1,
+		.endpoints = &spk_r_endpoint,
+		.name_prefix = "tas2783-3",
+	},
+	{
+		/* right */
+		.adr = 0x00003a0102000001ull,
+		.num_endpoints = 1,
+		.endpoints = &spk_r_endpoint,
+		.name_prefix = "tas2783-4",
+	},
+};
+
+static const struct snd_soc_acpi_link_adr acp70_tas2783_2[] = {
+	{
+		.mask = BIT(0),
+		.num_adr = ARRAY_SIZE(tas2783_2_adr),
+		.adr_d = tas2783_2_adr,
+	},
+	{}
+};
+
 static const struct snd_soc_acpi_adr_device rt1320_0_single_adr[] = {
 	{
 		.adr = 0x000030025D132001ull,
@@ -659,6 +699,11 @@ static const struct snd_soc_acpi_link_adr acp70_rt721_only[] = {
 };
 
 struct snd_soc_acpi_mach snd_soc_acpi_amd_acp70_sdw_machines[] = {
+	{
+		.link_mask = BIT(0),
+		.links = acp70_tas2783_2,
+		.drv_name = "amd_sdw",
+	},
 	{
 		.link_mask = BIT(0) | BIT(1),
 		.links = acp70_rt1320_l0_rt722_l1,
@@ -775,5 +820,5 @@ MODULE_DESCRIPTION("AMD ACP7.0 & ACP7.1 tables and support for ACPI enumeration"
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Vijendar.Mukunda@amd.com");
 #if IS_ENABLED(CONFIG_SND_SOC_ACPI_AMD_SDCA_QUIRKS)
-MODULE_IMPORT_NS("SND_SOC_ACPI_AMD_SDCA_QUIRKS");
+MODULE_IMPORT_NS("SND_SOC_SDCA");
 #endif
diff --git a/sound/soc/amd/acp/soc-acpi-amd-sdca-quirks.c b/sound/soc/amd/acp/soc-acpi-amd-sdca-quirks.c
index 63bf9e3c0ae18..9248b6d09e76f 100644
--- a/sound/soc/amd/acp/soc-acpi-amd-sdca-quirks.c
+++ b/sound/soc/amd/acp/soc-acpi-amd-sdca-quirks.c
@@ -35,8 +35,4 @@ bool snd_soc_acpi_amd_sdca_is_device_rt712_vb(void *arg)
 
 	return false;
 }
-EXPORT_SYMBOL_NS(snd_soc_acpi_amd_sdca_is_device_rt712_vb, "SND_SOC_ACPI_AMD_SDCA_QUIRKS");
 
-MODULE_DESCRIPTION("ASoC ACPI AMD SDCA quirks");
-MODULE_LICENSE("GPL");
-MODULE_IMPORT_NS("SND_SOC_SDCA");
diff --git a/sound/soc/amd/mach-config.h b/sound/soc/amd/mach-config.h
index 5b6362103ca0a..b602a983feb7f 100644
--- a/sound/soc/amd/mach-config.h
+++ b/sound/soc/amd/mach-config.h
@@ -28,6 +28,7 @@ extern struct snd_soc_acpi_mach snd_soc_acpi_amd_acp63_sof_sdw_machines[];
 extern struct snd_soc_acpi_mach snd_soc_acpi_amd_acp70_sof_machines[];
 extern struct snd_soc_acpi_mach snd_soc_acpi_amd_acp70_sdw_machines[];
 extern struct snd_soc_acpi_mach snd_soc_acpi_amd_acp70_sof_sdw_machines[];
+extern struct snd_soc_acpi_mach snd_soc_acpi_amd_acp7x_sof_machines[];
 
 struct config_entry {
 	u32 flags;
diff --git a/sound/soc/amd/ps/pci-ps.c b/sound/soc/amd/ps/pci-ps.c
index 4ecda224157b5..729f9aaba69e7 100644
--- a/sound/soc/amd/ps/pci-ps.c
+++ b/sound/soc/amd/ps/pci-ps.c
@@ -248,7 +248,7 @@ static irqreturn_t acp63_irq_handler(int irq, void *dev_id)
 	if (sdw_dma_irq_flag)
 		return IRQ_WAKE_THREAD;
 
-	if (irq_flag | wake_irq_flag)
+	if (irq_flag || wake_irq_flag)
 		return IRQ_HANDLED;
 	else
 		return IRQ_NONE;
@@ -602,7 +602,7 @@ static int snd_acp63_probe(struct pci_dev *pci,
 		return -ENODEV;
 	}
 
-	ret = pci_request_regions(pci, "AMD ACP6.2 audio");
+	ret = pci_request_regions(pci, "AMD ACP6.3 audio");
 	if (ret < 0) {
 		dev_err(&pci->dev, "pci_request_regions failed\n");
 		goto disable_pci;
@@ -693,8 +693,37 @@ static int snd_acp_runtime_resume(struct device *dev)
 	return acp_hw_runtime_resume(dev);
 }
 
+static void acp_disable_msi_on_resume(struct pci_dev *pdev)
+{
+	u16 control;
+
+	if (!pdev->msi_cap)
+		return;
+
+	pci_read_config_word(pdev, pdev->msi_cap + PCI_MSI_FLAGS, &control);
+	if (control & PCI_MSI_FLAGS_ENABLE) {
+		dev_warn(&pdev->dev,
+			 "ACP: MSI unexpectedly enabled after resume (flags=0x%04x), disabling\n",
+			 control);
+		control &= ~PCI_MSI_FLAGS_ENABLE;
+		pci_write_config_word(pdev, pdev->msi_cap + PCI_MSI_FLAGS, control);
+	}
+}
+
 static int snd_acp_resume(struct device *dev)
 {
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	/*
+	 * BIOS/firmware may re-enable MSI in PCI config space during
+	 * system resume even though this driver only uses legacy INTx
+	 * interrupts. If MSI is left enabled with stale address/data
+	 * registers, the device will write interrupts to a bogus address
+	 * causing IOMMU IO_PAGE_FAULT and interrupt delivery failure.
+	 * Explicitly clear the MSI Enable bit before reinitializing
+	 * the ACP hardware.
+	 */
+	acp_disable_msi_on_resume(pdev);
 	return acp_hw_resume(dev);
 }
 
diff --git a/sound/soc/amd/raven/acp3x-i2s.c b/sound/soc/amd/raven/acp3x-i2s.c
index 352485dd98b14..b0147e88ba54c 100644
--- a/sound/soc/amd/raven/acp3x-i2s.c
+++ b/sound/soc/amd/raven/acp3x-i2s.c
@@ -250,11 +250,17 @@ static int acp3x_i2s_trigger(struct snd_pcm_substream *substream,
 	return ret;
 }
 
+static const u64 acp3x_i2s_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A;
+
 static const struct snd_soc_dai_ops acp3x_i2s_dai_ops = {
 	.hw_params = acp3x_i2s_hwparams,
 	.trigger = acp3x_i2s_trigger,
 	.set_fmt = acp3x_i2s_set_fmt,
 	.set_tdm_slot = acp3x_i2s_set_tdm_slot,
+	.auto_selectable_formats = &acp3x_i2s_selectable_formats,
+	.num_auto_selectable_formats = 1,
 };
 
 static const struct snd_soc_component_driver acp3x_dai_component = {
diff --git a/sound/soc/amd/vangogh/acp5x-i2s.c b/sound/soc/amd/vangogh/acp5x-i2s.c
index bf719f6286174..dbfb87e2fe929 100644
--- a/sound/soc/amd/vangogh/acp5x-i2s.c
+++ b/sound/soc/amd/vangogh/acp5x-i2s.c
@@ -337,11 +337,17 @@ static int acp5x_i2s_trigger(struct snd_pcm_substream *substream,
 	return ret;
 }
 
+static const u64 acp5x_i2s_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A;
+
 static const struct snd_soc_dai_ops acp5x_i2s_dai_ops = {
 	.hw_params = acp5x_i2s_hwparams,
 	.trigger = acp5x_i2s_trigger,
 	.set_fmt = acp5x_i2s_set_fmt,
 	.set_tdm_slot = acp5x_i2s_set_tdm_slot,
+	.auto_selectable_formats = &acp5x_i2s_selectable_formats,
+	.num_auto_selectable_formats = 1,
 };
 
 static const struct snd_soc_component_driver acp5x_dai_component = {
diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c
index d6df7de70b275..6f2d6ef570d2c 100644
--- a/sound/soc/amd/yc/acp6x-mach.c
+++ b/sound/soc/amd/yc/acp6x-mach.c
@@ -521,6 +521,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = {
 			DMI_MATCH(DMI_PRODUCT_NAME, "Vector A16 HX A8WHG"),
 		}
 	},
+	{
+		.driver_data = &acp6x_card,
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "Micro-Star International Co., Ltd."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Vector A16 HX A8WIG"),
+		}
+	},
 	{
 		.driver_data = &acp6x_card,
 		.matches = {
@@ -753,12 +760,12 @@ static const struct dmi_system_id yc_acp_quirk_table[] = {
 		}
 	},
 	{
-			.driver_data = &acp6x_card,
-			.matches = {
-				DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."),
-				DMI_MATCH(DMI_PRODUCT_NAME, "Vivobook_ASUSLaptop M6501RR_M6501RR"),
-			}
-		},
+		.driver_data = &acp6x_card,
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Vivobook_ASUSLaptop M6501RR_M6501RR"),
+		}
+	},
 	{
 		.driver_data = &acp6x_card,
 		.matches = {
diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c
index 492165c0e1ea9..afb3754a88835 100644
--- a/sound/soc/apple/mca.c
+++ b/sound/soc/apple/mca.c
@@ -133,12 +133,17 @@ struct mca_cluster {
 	struct clk *clk_parent;
 	struct dma_chan *dma_chans[SNDRV_PCM_STREAM_LAST + 1];
 
-	bool port_started[SNDRV_PCM_STREAM_LAST + 1];
-	int port_driver; /* The cluster driving this cluster's port */
+	bool clk_provider;
+
+	bool port_clk_started[SNDRV_PCM_STREAM_LAST + 1];
+	int port_clk_driver; /* The cluster driving this cluster's port */
 
 	bool clocks_in_use[SNDRV_PCM_STREAM_LAST + 1];
 	struct device_link *pd_link;
 
+	/* In case of clock consumer FE */
+	int syncgen_in_use;
+
 	unsigned int bclk_ratio;
 
 	/* Masks etc. picked up via the set_tdm_slot method */
@@ -157,7 +162,7 @@ struct mca_data {
 	struct reset_control *rstc;
 	struct device_link *pd_link;
 
-	/* Mutex for accessing port_driver of foreign clusters */
+	/* Mutex for accessing port_clk_driver of foreign clusters */
 	struct mutex port_mutex;
 
 	int nclusters;
@@ -210,16 +215,22 @@ static void mca_fe_early_trigger(struct snd_pcm_substream *substream, int cmd,
 			   SERDES_STATUS_EN | SERDES_STATUS_RST,
 			   SERDES_STATUS_RST);
 		/*
-		 * Experiments suggest that it takes at most ~1 us
-		 * for the bit to clear, so wait 2 us for good measure.
+		 * The SERDES cluster needs a bit of time to reset itself
+		 * and settle before we start poking it. This is... slow...
 		 */
-		udelay(2);
+		udelay(25);
 		WARN_ON(readl_relaxed(cl->base + serdes_unit + REG_SERDES_STATUS) &
 			SERDES_STATUS_RST);
 		mca_modify(cl, serdes_conf, SERDES_CONF_SYNC_SEL,
 			   FIELD_PREP(SERDES_CONF_SYNC_SEL, 0));
 		mca_modify(cl, serdes_conf, SERDES_CONF_SYNC_SEL,
 			   FIELD_PREP(SERDES_CONF_SYNC_SEL, cl->no + 1));
+		/*
+		 * ADMAC gets started right after this. This delay seems
+		 * to be needed for that to be reliable, e.g. ensure the
+		 * clock is stable?
+		 */
+		udelay(100);
 		break;
 	default:
 		break;
@@ -256,11 +267,29 @@ static int mca_fe_trigger(struct snd_pcm_substream *substream, int cmd,
 	return 0;
 }
 
+static int mca_fe_get_portmask(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *fe = snd_soc_substream_to_rtd(substream);
+	struct snd_soc_dpcm *dpcm;
+	int mask = 0;
+
+	for_each_dpcm_be(fe, substream->stream, dpcm) {
+		int no = mca_dai_to_cluster(snd_soc_rtd_to_cpu(dpcm->be, 0))->no;
+
+		mask |= 1 << no;
+	}
+
+	return mask;
+}
+
 static int mca_fe_enable_clocks(struct mca_cluster *cl)
 {
 	struct mca_data *mca = cl->host;
 	int ret;
 
+	if (!cl->clk_provider)
+		return -EINVAL;
+
 	ret = clk_prepare_enable(cl->clk_parent);
 	if (ret) {
 		dev_err(mca->dev,
@@ -311,7 +340,7 @@ static bool mca_fe_clocks_in_use(struct mca_cluster *cl)
 	for (i = 0; i < mca->nclusters; i++) {
 		be_cl = &mca->clusters[i];
 
-		if (be_cl->port_driver != cl->no)
+		if (be_cl->port_clk_driver != cl->no)
 			continue;
 
 		for_each_pcm_streams(stream) {
@@ -323,59 +352,79 @@ static bool mca_fe_clocks_in_use(struct mca_cluster *cl)
 	return false;
 }
 
-static int mca_be_prepare(struct snd_pcm_substream *substream,
+static int mca_be_hw_free(struct snd_pcm_substream *substream,
 			  struct snd_soc_dai *dai)
 {
 	struct mca_cluster *cl = mca_dai_to_cluster(dai);
 	struct mca_data *mca = cl->host;
 	struct mca_cluster *fe_cl;
-	int ret;
 
-	if (cl->port_driver < 0)
+	if (cl->port_clk_driver < 0)
 		return -EINVAL;
 
-	fe_cl = &mca->clusters[cl->port_driver];
-
 	/*
-	 * Typically the CODECs we are paired with will require clocks
-	 * to be present at time of unmute with the 'mute_stream' op
-	 * or at time of DAPM widget power-up. We need to enable clocks
-	 * here at the latest (frontend prepare would be too late).
+	 * We are operating on a foreign cluster here, but since we
+	 * belong to the same PCM, accesses should have been
+	 * synchronized at ASoC level.
 	 */
-	if (!mca_fe_clocks_in_use(fe_cl)) {
-		ret = mca_fe_enable_clocks(fe_cl);
-		if (ret < 0)
-			return ret;
-	}
+	fe_cl = &mca->clusters[cl->port_clk_driver];
+	if (!mca_fe_clocks_in_use(fe_cl))
+		return 0; /* Nothing to do */
 
-	cl->clocks_in_use[substream->stream] = true;
+	cl->clocks_in_use[substream->stream] = false;
+
+	if (!mca_fe_clocks_in_use(fe_cl))
+		mca_fe_disable_clocks(fe_cl);
 
 	return 0;
 }
 
-static int mca_be_hw_free(struct snd_pcm_substream *substream,
+static int mca_fe_prepare(struct snd_pcm_substream *substream,
 			  struct snd_soc_dai *dai)
 {
 	struct mca_cluster *cl = mca_dai_to_cluster(dai);
 	struct mca_data *mca = cl->host;
-	struct mca_cluster *fe_cl;
 
-	if (cl->port_driver < 0)
-		return -EINVAL;
+	if (cl->clk_provider)
+		return 0;
 
-	/*
-	 * We are operating on a foreign cluster here, but since we
-	 * belong to the same PCM, accesses should have been
-	 * synchronized at ASoC level.
-	 */
-	fe_cl = &mca->clusters[cl->port_driver];
-	if (!mca_fe_clocks_in_use(fe_cl))
-		return 0; /* Nothing to do */
+	/* Turn on the cluster power domain if not already in use */
+	if (!cl->syncgen_in_use) {
+		int port = ffs(mca_fe_get_portmask(substream));
 
-	cl->clocks_in_use[substream->stream] = false;
+		cl->pd_link = device_link_add(mca->dev, cl->pd_dev,
+					      DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME |
+						DL_FLAG_RPM_ACTIVE);
+		if (!cl->pd_link) {
+			dev_err(mca->dev,
+				"cluster %d: unable to prop-up power domain\n", cl->no);
+			return -EINVAL;
+		}
 
-	if (!mca_fe_clocks_in_use(fe_cl))
-		mca_fe_disable_clocks(fe_cl);
+		mca_modify(cl, REG_SYNCGEN_MCLK_SEL, SYNCGEN_MCLK_SEL, BIT(port));
+		mca_modify(cl, REG_SYNCGEN_STATUS, SYNCGEN_STATUS_EN,
+			   SYNCGEN_STATUS_EN);
+	}
+	cl->syncgen_in_use |= 1 << substream->stream;
+
+	return 0;
+}
+
+static int mca_fe_hw_free(struct snd_pcm_substream *substream,
+			  struct snd_soc_dai *dai)
+{
+	struct mca_cluster *cl = mca_dai_to_cluster(dai);
+
+	if (cl->clk_provider)
+		return 0;
+
+	cl->syncgen_in_use &= ~(1 << substream->stream);
+	if (cl->syncgen_in_use)
+		return 0;
+
+	mca_modify(cl, REG_SYNCGEN_STATUS, SYNCGEN_STATUS_EN, 0);
+	if (cl->pd_link)
+		device_link_del(cl->pd_link);
 
 	return 0;
 }
@@ -390,7 +439,7 @@ static unsigned int mca_crop_mask(unsigned int mask, int nchans)
 
 static int mca_configure_serdes(struct mca_cluster *cl, int serdes_unit,
 				unsigned int mask, int slots, int nchans,
-				int slot_width, bool is_tx, int port)
+				int slot_width, bool is_tx, int portmask)
 {
 	__iomem void *serdes_base = cl->base + serdes_unit;
 	u32 serdes_conf, serdes_conf_mask;
@@ -449,7 +498,7 @@ static int mca_configure_serdes(struct mca_cluster *cl, int serdes_unit,
 			       serdes_base + REG_RX_SERDES_SLOTMASK);
 		writel_relaxed(~((u32)mca_crop_mask(mask, nchans)),
 			       serdes_base + REG_RX_SERDES_SLOTMASK + 0x4);
-		writel_relaxed(1 << port,
+		writel_relaxed(portmask,
 			       serdes_base + REG_RX_SERDES_PORT);
 	}
 
@@ -505,9 +554,18 @@ static int mca_fe_set_fmt(struct snd_soc_dai *dai, unsigned int fmt)
 	u32 serdes_conf = 0;
 	u32 bitstart;
 
-	if ((fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) !=
-	    SND_SOC_DAIFMT_BP_FP)
+	switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) {
+	case SND_SOC_DAIFMT_BP_FP:
+		cl->clk_provider = true;
+		break;
+
+	case SND_SOC_DAIFMT_BC_FC:
+		cl->clk_provider = false;
+		break;
+
+	default:
 		goto err;
+	}
 
 	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
 	case SND_SOC_DAIFMT_I2S:
@@ -564,24 +622,6 @@ static int mca_set_bclk_ratio(struct snd_soc_dai *dai, unsigned int ratio)
 	return 0;
 }
 
-static int mca_fe_get_port(struct snd_pcm_substream *substream)
-{
-	struct snd_soc_pcm_runtime *fe = snd_soc_substream_to_rtd(substream);
-	struct snd_soc_pcm_runtime *be;
-	struct snd_soc_dpcm *dpcm;
-
-	be = NULL;
-	for_each_dpcm_be(fe, substream->stream, dpcm) {
-		be = dpcm->be;
-		break;
-	}
-
-	if (!be)
-		return -EINVAL;
-
-	return mca_dai_to_cluster(snd_soc_rtd_to_cpu(be, 0))->no;
-}
-
 static int mca_fe_hw_params(struct snd_pcm_substream *substream,
 			    struct snd_pcm_hw_params *params,
 			    struct snd_soc_dai *dai)
@@ -595,7 +635,7 @@ static int mca_fe_hw_params(struct snd_pcm_substream *substream,
 	unsigned long bclk_ratio;
 	unsigned int tdm_slots, tdm_slot_width, tdm_mask;
 	u32 regval, pad;
-	int ret, port, nchans_ceiled;
+	int ret, portmask, nchans_ceiled;
 
 	if (!cl->tdm_slot_width) {
 		/*
@@ -644,13 +684,13 @@ static int mca_fe_hw_params(struct snd_pcm_substream *substream,
 		tdm_mask = (1 << tdm_slots) - 1;
 	}
 
-	port = mca_fe_get_port(substream);
-	if (port < 0)
-		return port;
+	portmask = mca_fe_get_portmask(substream);
+	if (!portmask)
+		return -EINVAL;
 
 	ret = mca_configure_serdes(cl, is_tx ? CLUSTER_TX_OFF : CLUSTER_RX_OFF,
 				   tdm_mask, tdm_slots, params_channels(params),
-				   tdm_slot_width, is_tx, port);
+				   tdm_slot_width, is_tx, portmask);
 	if (ret)
 		return ret;
 
@@ -706,66 +746,126 @@ static const struct snd_soc_dai_ops mca_fe_ops = {
 	.set_tdm_slot = mca_fe_set_tdm_slot,
 	.hw_params = mca_fe_hw_params,
 	.trigger = mca_fe_trigger,
+	.prepare = mca_fe_prepare,
+	.hw_free = mca_fe_hw_free,
 };
 
-static bool mca_be_started(struct mca_cluster *cl)
+/*
+ * Is there a FE attached which will be feeding this port's clocks?
+ */
+static bool mca_be_clk_started(struct mca_cluster *cl)
 {
 	int stream;
 
 	for_each_pcm_streams(stream)
-		if (cl->port_started[stream])
+		if (cl->port_clk_started[stream])
 			return true;
 	return false;
 }
 
-static int mca_be_startup(struct snd_pcm_substream *substream,
+static struct snd_soc_pcm_runtime *mca_be_get_fe(struct snd_soc_pcm_runtime *be,
+						 int stream)
+{
+	struct snd_soc_pcm_runtime *fe = NULL;
+	struct snd_soc_dpcm *dpcm;
+
+	for_each_dpcm_fe(be, stream, dpcm) {
+		if (fe && dpcm->fe != fe) {
+			dev_err(be->dev, "many FE per one BE unsupported\n");
+			return NULL;
+		}
+
+		fe = dpcm->fe;
+	}
+
+	return fe;
+}
+
+static int mca_be_prepare(struct snd_pcm_substream *substream,
 			  struct snd_soc_dai *dai)
 {
 	struct snd_soc_pcm_runtime *be = snd_soc_substream_to_rtd(substream);
-	struct snd_soc_pcm_runtime *fe;
+	struct snd_soc_pcm_runtime *fe = mca_be_get_fe(be, substream->stream);
 	struct mca_cluster *cl = mca_dai_to_cluster(dai);
-	struct mca_cluster *fe_cl;
 	struct mca_data *mca = cl->host;
-	struct snd_soc_dpcm *dpcm;
+	struct mca_cluster *fe_cl, *fe_clk_cl;
+	int ret;
 
-	fe = NULL;
+	fe_cl = mca_dai_to_cluster(snd_soc_rtd_to_cpu(fe, 0));
 
-	for_each_dpcm_fe(be, substream->stream, dpcm) {
-		if (fe && dpcm->fe != fe) {
-			dev_err(mca->dev, "many FE per one BE unsupported\n");
-			return -EINVAL;
-		}
+	if (!fe_cl->clk_provider)
+		return 0;
 
-		fe = dpcm->fe;
+	if (cl->port_clk_driver < 0)
+		return 0;
+
+	fe_clk_cl = &mca->clusters[cl->port_clk_driver];
+
+	/*
+	 * Typically the CODECs we are paired with will require clocks
+	 * to be present at time of unmute with the 'mute_stream' op
+	 * or at time of DAPM widget power-up. We need to enable clocks
+	 * here at the latest (frontend prepare would be too late).
+	 */
+	if (!mca_fe_clocks_in_use(fe_clk_cl)) {
+		ret = mca_fe_enable_clocks(fe_clk_cl);
+		if (ret < 0)
+			return ret;
 	}
 
+	cl->clocks_in_use[substream->stream] = true;
+
+	return 0;
+}
+
+static int mca_be_startup(struct snd_pcm_substream *substream,
+			  struct snd_soc_dai *dai)
+{
+	struct snd_soc_pcm_runtime *be = snd_soc_substream_to_rtd(substream);
+	struct snd_soc_pcm_runtime *fe = mca_be_get_fe(be, substream->stream);
+	struct mca_cluster *cl = mca_dai_to_cluster(dai);
+	struct mca_cluster *fe_cl;
+	struct mca_data *mca = cl->host;
+
 	if (!fe)
 		return -EINVAL;
-
 	fe_cl = mca_dai_to_cluster(snd_soc_rtd_to_cpu(fe, 0));
 
-	if (mca_be_started(cl)) {
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		writel_relaxed(PORT_DATA_SEL_TXA(fe_cl->no),
+			       cl->base + REG_PORT_DATA_SEL);
+		mca_modify(cl, REG_PORT_ENABLES, PORT_ENABLES_TX_DATA,
+			   PORT_ENABLES_TX_DATA);
+	}
+
+	if (!fe_cl->clk_provider)
+		return 0;
+
+	if (mca_be_clk_started(cl)) {
 		/*
 		 * Port is already started in the other direction.
 		 * Make sure there isn't a conflict with another cluster
-		 * driving the port.
+		 * driving the port clocks.
 		 */
-		if (cl->port_driver != fe_cl->no)
+		if (cl->port_clk_driver != fe_cl->no)
 			return -EINVAL;
 
-		cl->port_started[substream->stream] = true;
+		cl->port_clk_started[substream->stream] = true;
 		return 0;
 	}
 
-	writel_relaxed(PORT_ENABLES_CLOCKS | PORT_ENABLES_TX_DATA,
-		       cl->base + REG_PORT_ENABLES);
 	writel_relaxed(FIELD_PREP(PORT_CLOCK_SEL, fe_cl->no + 1),
 		       cl->base + REG_PORT_CLOCK_SEL);
+
 	writel_relaxed(PORT_DATA_SEL_TXA(fe_cl->no),
 		       cl->base + REG_PORT_DATA_SEL);
+
+	mca_modify(cl, REG_PORT_ENABLES, PORT_ENABLES_CLOCKS,
+		   PORT_ENABLES_CLOCKS);
+
 	scoped_guard(mutex, &mca->port_mutex)
-		cl->port_driver = fe_cl->no;
-	cl->port_started[substream->stream] = true;
+		cl->port_clk_driver = fe_cl->no;
+	cl->port_clk_started[substream->stream] = true;
 
 	return 0;
 }
@@ -773,20 +873,34 @@ static int mca_be_startup(struct snd_pcm_substream *substream,
 static void mca_be_shutdown(struct snd_pcm_substream *substream,
 			    struct snd_soc_dai *dai)
 {
+	struct snd_soc_pcm_runtime *be = snd_soc_substream_to_rtd(substream);
+	struct snd_soc_pcm_runtime *fe = mca_be_get_fe(be, substream->stream);
 	struct mca_cluster *cl = mca_dai_to_cluster(dai);
+	struct mca_cluster *fe_cl;
 	struct mca_data *mca = cl->host;
 
-	cl->port_started[substream->stream] = false;
+	if (!fe)
+		return;
+	fe_cl = mca_dai_to_cluster(snd_soc_rtd_to_cpu(fe, 0));
 
-	if (!mca_be_started(cl)) {
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		mca_modify(cl, REG_PORT_ENABLES, PORT_ENABLES_TX_DATA, 0);
+		writel_relaxed(0, cl->base + REG_PORT_DATA_SEL);
+	}
+
+	if (!fe_cl->clk_provider)
+		return;
+
+	cl->port_clk_started[substream->stream] = false;
+	if (!mca_be_clk_started(cl)) {
 		/*
 		 * Were we the last direction to shutdown?
-		 * Turn off the lights.
+		 * Turn off the lights (clocks).
 		 */
-		writel_relaxed(0, cl->base + REG_PORT_ENABLES);
-		writel_relaxed(0, cl->base + REG_PORT_DATA_SEL);
+		mca_modify(cl, REG_PORT_ENABLES, PORT_ENABLES_CLOCKS, 0);
+		writel_relaxed(0, cl->base + REG_PORT_CLOCK_SEL);
 		scoped_guard(mutex, &mca->port_mutex)
-			cl->port_driver = -1;
+			cl->port_clk_driver = -1;
 	}
 }
 
@@ -1092,7 +1206,7 @@ static int apple_mca_probe(struct platform_device *pdev)
 		cl->host = mca;
 		cl->no = i;
 		cl->base = base + CLUSTER_STRIDE * i;
-		cl->port_driver = -1;
+		cl->port_clk_driver = -1;
 		cl->clk_parent = of_clk_get(pdev->dev.of_node, i);
 		if (IS_ERR(cl->clk_parent)) {
 			dev_err(&pdev->dev, "unable to obtain clock %d: %ld\n",
diff --git a/sound/soc/au1x/psc-ac97.c b/sound/soc/au1x/psc-ac97.c
index 94698e08a5134..0f9d808831167 100644
--- a/sound/soc/au1x/psc-ac97.c
+++ b/sound/soc/au1x/psc-ac97.c
@@ -210,9 +210,7 @@ static int au1xpsc_ac97_hw_params(struct snd_pcm_substream *substream,
 {
 	struct au1xpsc_audio_data *pscdata = snd_soc_dai_get_drvdata(dai);
 	unsigned long r, ro, stat;
-	int chans, t, stype = substream->stream;
-
-	chans = params_channels(params);
+	int t, stype = substream->stream;
 
 	r = ro = __raw_readl(AC97_CFG(pscdata));
 	stat = __raw_readl(AC97_STAT(pscdata));
diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig
index 252f683be3c18..76e90144ea91e 100644
--- a/sound/soc/codecs/Kconfig
+++ b/sound/soc/codecs/Kconfig
@@ -615,8 +615,8 @@ config SND_SOC_AK4613
 	depends on I2C
 
 config SND_SOC_AK4619
-        tristate "AKM AK4619 CODEC"
-        depends on I2C
+	tristate "AKM AK4619 CODEC"
+	depends on I2C
 
 config SND_SOC_AK4642
 	tristate "AKM AK4642 CODEC"
@@ -1201,16 +1201,16 @@ config SND_SOC_JZ4725B_CODEC
 	  will be called snd-soc-jz4725b-codec.
 
 config SND_SOC_JZ4760_CODEC
-        depends on MACH_INGENIC || COMPILE_TEST
-        depends on OF
-        select REGMAP
-        tristate "Ingenic JZ4760 internal CODEC"
-        help
-          Enable support for the internal CODEC found in the JZ4760 SoC
-          from Ingenic.
+	depends on MACH_INGENIC || COMPILE_TEST
+	depends on OF
+	select REGMAP
+	tristate "Ingenic JZ4760 internal CODEC"
+	help
+	  Enable support for the internal CODEC found in the JZ4760 SoC
+	  from Ingenic.
 
-          This driver can also be built as a module. If so, the module
-          will be called snd-soc-jz4760-codec.
+	  This driver can also be built as a module. If so, the module
+	  will be called snd-soc-jz4760-codec.
 
 config SND_SOC_JZ4770_CODEC
 	depends on MACH_INGENIC || COMPILE_TEST
@@ -1888,7 +1888,7 @@ config SND_SOC_RT5670
 	depends on I2C
 
 config SND_SOC_RT5677
-	tristate
+	tristate "Realtek RT5677 Codec"
 	depends on I2C
 	select REGMAP_I2C
 	select REGMAP_IRQ
@@ -2296,8 +2296,8 @@ config SND_SOC_TLV320ADC3XXX
 	depends on I2C
 	depends on GPIOLIB
 	help
-	 Enable support for Texas Instruments TLV320ADC3001 and TLV320ADC3101
-	 ADCs.
+	  Enable support for Texas Instruments TLV320ADC3001 and TLV320ADC3101
+	  ADCs.
 
 config SND_SOC_TLV320AIC23
 	tristate
@@ -2893,28 +2893,32 @@ config SND_SOC_TPA6130A2
 	depends on I2C
 
 config SND_SOC_LPASS_MACRO_COMMON
-        tristate
+	tristate
 
 config SND_SOC_LPASS_WSA_MACRO
 	depends on COMMON_CLK
+	depends on PM_CLK
 	select REGMAP_MMIO
 	select SND_SOC_LPASS_MACRO_COMMON
 	tristate "Qualcomm WSA Macro in LPASS(Low Power Audio SubSystem)"
 
 config SND_SOC_LPASS_VA_MACRO
 	depends on COMMON_CLK
+	depends on PM_CLK
 	select REGMAP_MMIO
 	select SND_SOC_LPASS_MACRO_COMMON
 	tristate "Qualcomm VA Macro in LPASS(Low Power Audio SubSystem)"
 
 config SND_SOC_LPASS_RX_MACRO
 	depends on COMMON_CLK
+	depends on PM_CLK
 	select REGMAP_MMIO
 	select SND_SOC_LPASS_MACRO_COMMON
 	tristate "Qualcomm RX Macro in LPASS(Low Power Audio SubSystem)"
 
 config SND_SOC_LPASS_TX_MACRO
 	depends on COMMON_CLK
+	depends on PM_CLK
 	select REGMAP_MMIO
 	select SND_SOC_LPASS_MACRO_COMMON
 	tristate "Qualcomm TX Macro in LPASS(Low Power Audio SubSystem)"
diff --git a/sound/soc/codecs/ab8500-codec.c b/sound/soc/codecs/ab8500-codec.c
index 6e8ef9cd1b31a..2cf96cbdd294b 100644
--- a/sound/soc/codecs/ab8500-codec.c
+++ b/sound/soc/codecs/ab8500-codec.c
@@ -14,6 +14,7 @@
  *         for ST-Ericsson.
  */
 
+#include <linux/cleanup.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/device.h>
@@ -989,9 +990,8 @@ static int sid_status_control_get(struct snd_kcontrol *kcontrol,
 	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
 	struct ab8500_codec_drvdata *drvdata = dev_get_drvdata(component->dev);
 
-	mutex_lock(&drvdata->ctrl_lock);
+	guard(mutex)(&drvdata->ctrl_lock);
 	ucontrol->value.enumerated.item[0] = drvdata->sid_status;
-	mutex_unlock(&drvdata->ctrl_lock);
 
 	return 0;
 }
@@ -1014,7 +1014,7 @@ static int sid_status_control_put(struct snd_kcontrol *kcontrol,
 		return -EIO;
 	}
 
-	mutex_lock(&drvdata->ctrl_lock);
+	guard(mutex)(&drvdata->ctrl_lock);
 
 	sidconf = snd_soc_component_read(component, AB8500_SIDFIRCONF);
 	if (((sidconf & BIT(AB8500_SIDFIRCONF_FIRSIDBUSY)) != 0)) {
@@ -1025,7 +1025,8 @@ static int sid_status_control_put(struct snd_kcontrol *kcontrol,
 		} else {
 			status = -EBUSY;
 		}
-		goto out;
+		dev_dbg(component->dev, "%s: Exit\n", __func__);
+		return status;
 	}
 
 	snd_soc_component_write(component, AB8500_SIDFIRADR, 0);
@@ -1043,9 +1044,6 @@ static int sid_status_control_put(struct snd_kcontrol *kcontrol,
 
 	drvdata->sid_status = SID_FIR_CONFIGURED;
 
-out:
-	mutex_unlock(&drvdata->ctrl_lock);
-
 	dev_dbg(component->dev, "%s: Exit\n", __func__);
 
 	return status;
diff --git a/sound/soc/codecs/ak4613.c b/sound/soc/codecs/ak4613.c
index 3e0696b5abf52..be5306d07d406 100644
--- a/sound/soc/codecs/ak4613.c
+++ b/sound/soc/codecs/ak4613.c
@@ -95,6 +95,7 @@
  * see
  *	AK4613_ENABLE_TDM_TEST
  */
+#include <linux/cleanup.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/i2c.h>
@@ -384,7 +385,7 @@ static void ak4613_dai_shutdown(struct snd_pcm_substream *substream,
 	struct ak4613_priv *priv = snd_soc_component_get_drvdata(component);
 	struct device *dev = component->dev;
 
-	mutex_lock(&priv->lock);
+	guard(mutex)(&priv->lock);
 	priv->cnt--;
 	if (priv->cnt < 0) {
 		dev_err(dev, "unexpected counter error\n");
@@ -392,7 +393,6 @@ static void ak4613_dai_shutdown(struct snd_pcm_substream *substream,
 	}
 	if (!priv->cnt)
 		priv->ctrl1 = 0;
-	mutex_unlock(&priv->lock);
 }
 
 static void ak4613_hw_constraints(struct ak4613_priv *priv,
@@ -507,10 +507,9 @@ static int ak4613_dai_startup(struct snd_pcm_substream *substream,
 	struct snd_soc_component *component = dai->component;
 	struct ak4613_priv *priv = snd_soc_component_get_drvdata(component);
 
-	mutex_lock(&priv->lock);
+	guard(mutex)(&priv->lock);
 	ak4613_hw_constraints(priv, substream);
 	priv->cnt++;
-	mutex_unlock(&priv->lock);
 
 	return 0;
 }
@@ -599,45 +598,47 @@ static int ak4613_dai_hw_params(struct snd_pcm_substream *substream,
 	 */
 	ret = -EINVAL;
 
-	mutex_lock(&priv->lock);
-	if (priv->cnt > 1) {
-		/*
-		 * If it was already working, use current priv->ctrl1
-		 */
-		ret = 0;
-	} else {
-		/*
-		 * It is not yet working,
-		 */
-		unsigned int channel = params_channels(params);
-		u8 tdm;
-
-		/* STEREO or TDM */
-		if (channel == 2)
-			tdm = AK4613_CONFIG_MODE_STEREO;
-		else
-			tdm = AK4613_CONFIG_GET(priv, MODE);
-
-		for (i = ARRAY_SIZE(ak4613_iface) - 1; i >= 0; i--) {
-			const struct ak4613_interface *iface = ak4613_iface + i;
-
-			if ((iface->fmt == fmt) && (iface->width == width)) {
-				/*
-				 * Ctrl1
-				 * | D7 | D6 | D5 | D4 | D3 | D2 | D1 | D0  |
-				 * |TDM1|TDM0|DIF2|DIF1|DIF0|ATS1|ATS0|SMUTE|
-				 *  <  tdm  > < iface->dif >
-				 */
-				priv->ctrl1 = (tdm << 6) | (iface->dif << 3);
-				ret = 0;
-				break;
+	scoped_guard(mutex, &priv->lock) {
+		if (priv->cnt > 1) {
+			/*
+			 * If it was already working, use current priv->ctrl1
+			 */
+			ret = 0;
+		} else {
+			/*
+			 * It is not yet working,
+			 */
+			unsigned int channel = params_channels(params);
+			u8 tdm;
+
+			/* STEREO or TDM */
+			if (channel == 2)
+				tdm = AK4613_CONFIG_MODE_STEREO;
+			else
+				tdm = AK4613_CONFIG_GET(priv, MODE);
+
+			for (i = ARRAY_SIZE(ak4613_iface) - 1; i >= 0; i--) {
+				const struct ak4613_interface *iface = ak4613_iface + i;
+
+				if (iface->fmt == fmt && iface->width == width) {
+					/*
+					 * Ctrl1
+					 * | D7 | D6 | D5 | D4 | D3 | D2 | D1 | D0  |
+					 * |TDM1|TDM0|DIF2|DIF1|DIF0|ATS1|ATS0|SMUTE|
+					 *  <  tdm  > < iface->dif >
+					 */
+					priv->ctrl1 = (tdm << 6) | (iface->dif << 3);
+					ret = 0;
+					break;
+				}
 			}
 		}
 	}
-	mutex_unlock(&priv->lock);
 
-	if (ret < 0)
-		goto hw_params_end;
+	if (ret < 0) {
+		dev_warn(dev, "unsupported data width/format combination\n");
+		return ret;
+	}
 
 	snd_soc_component_update_bits(component, CTRL1, FMT_MASK, priv->ctrl1);
 	snd_soc_component_update_bits(component, CTRL2, DFS_MASK, ctrl2);
@@ -645,10 +646,6 @@ static int ak4613_dai_hw_params(struct snd_pcm_substream *substream,
 	snd_soc_component_update_bits(component, ICTRL, ICTRL_MASK, priv->ic);
 	snd_soc_component_update_bits(component, OCTRL, OCTRL_MASK, priv->oc);
 
-hw_params_end:
-	if (ret < 0)
-		dev_warn(dev, "unsupported data width/format combination\n");
-
 	return ret;
 }
 
diff --git a/sound/soc/codecs/arizona-jack.c b/sound/soc/codecs/arizona-jack.c
index a9063bac2752b..819d080b11889 100644
--- a/sound/soc/codecs/arizona-jack.c
+++ b/sound/soc/codecs/arizona-jack.c
@@ -5,6 +5,7 @@
  *  Copyright (C) 2012-2014 Wolfson Microelectronics plc
  */
 
+#include <linux/cleanup.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/slab.h>
@@ -707,15 +708,13 @@ static void arizona_micd_timeout_work(struct work_struct *work)
 						struct arizona_priv,
 						micd_timeout_work.work);
 
-	mutex_lock(&info->lock);
+	guard(mutex)(&info->lock);
 
 	dev_dbg(info->arizona->dev, "MICD timed out, reporting HP\n");
 
 	info->detecting = false;
 
 	arizona_identify_headphone(info);
-
-	mutex_unlock(&info->lock);
 }
 
 static int arizona_micd_adc_read(struct arizona_priv *info)
@@ -921,12 +920,11 @@ static void arizona_micd_detect(struct work_struct *work)
 
 	cancel_delayed_work_sync(&info->micd_timeout_work);
 
-	mutex_lock(&info->lock);
+	guard(mutex)(&info->lock);
 
 	/* If the cable was removed while measuring ignore the result */
 	if (!(info->jack->status & SND_JACK_MECHANICAL)) {
 		dev_dbg(arizona->dev, "Ignoring MICDET for removed cable\n");
-		mutex_unlock(&info->lock);
 		return;
 	}
 
@@ -936,7 +934,6 @@ static void arizona_micd_detect(struct work_struct *work)
 		arizona_button_reading(info);
 
 	pm_runtime_mark_last_busy(arizona->dev);
-	mutex_unlock(&info->lock);
 }
 
 static irqreturn_t arizona_micdet(int irq, void *data)
@@ -948,10 +945,10 @@ static irqreturn_t arizona_micdet(int irq, void *data)
 	cancel_delayed_work_sync(&info->micd_detect_work);
 	cancel_delayed_work_sync(&info->micd_timeout_work);
 
-	mutex_lock(&info->lock);
-	if (!info->detecting)
-		debounce = 0;
-	mutex_unlock(&info->lock);
+	scoped_guard(mutex, &info->lock) {
+		if (!info->detecting)
+			debounce = 0;
+	}
 
 	if (debounce)
 		queue_delayed_work(system_power_efficient_wq,
@@ -969,9 +966,8 @@ static void arizona_hpdet_work(struct work_struct *work)
 						struct arizona_priv,
 						hpdet_work.work);
 
-	mutex_lock(&info->lock);
+	guard(mutex)(&info->lock);
 	arizona_start_hpdet_acc_id(info);
-	mutex_unlock(&info->lock);
 }
 
 static int arizona_hpdet_wait(struct arizona_priv *info)
@@ -1018,9 +1014,9 @@ static irqreturn_t arizona_jackdet(int irq, void *data)
 	cancelled_hp = cancel_delayed_work_sync(&info->hpdet_work);
 	cancelled_mic = cancel_delayed_work_sync(&info->micd_timeout_work);
 
-	pm_runtime_get_sync(arizona->dev);
+	guard(pm_runtime_active_auto)(arizona->dev);
 
-	mutex_lock(&info->lock);
+	guard(mutex)(&info->lock);
 
 	if (info->micd_clamp) {
 		mask = ARIZONA_MICD_CLAMP_STS;
@@ -1036,8 +1032,6 @@ static irqreturn_t arizona_jackdet(int irq, void *data)
 	ret = regmap_read(arizona->regmap, ARIZONA_AOD_IRQ_RAW_STATUS, &val);
 	if (ret) {
 		dev_err(arizona->dev, "Failed to read jackdet status: %d\n", ret);
-		mutex_unlock(&info->lock);
-		pm_runtime_put_autosuspend(arizona->dev);
 		return IRQ_NONE;
 	}
 
@@ -1056,62 +1050,61 @@ static irqreturn_t arizona_jackdet(int irq, void *data)
 					   &info->micd_timeout_work,
 					   msecs_to_jiffies(micd_timeout));
 		}
+	} else {
+		info->last_jackdet = val;
 
-		goto out;
-	}
-	info->last_jackdet = val;
+		if (info->last_jackdet == present) {
+			dev_dbg(arizona->dev, "Detected jack\n");
+			snd_soc_jack_report(info->jack, SND_JACK_MECHANICAL, SND_JACK_MECHANICAL);
 
-	if (info->last_jackdet == present) {
-		dev_dbg(arizona->dev, "Detected jack\n");
-		snd_soc_jack_report(info->jack, SND_JACK_MECHANICAL, SND_JACK_MECHANICAL);
+			info->detecting = true;
+			info->mic = false;
+			info->jack_flips = 0;
 
-		info->detecting = true;
-		info->mic = false;
-		info->jack_flips = 0;
+			if (!arizona->pdata.hpdet_acc_id) {
+				arizona_start_mic(info);
+			} else {
+				queue_delayed_work(system_power_efficient_wq,
+						   &info->hpdet_work,
+						   msecs_to_jiffies(HPDET_DEBOUNCE));
+			}
 
-		if (!arizona->pdata.hpdet_acc_id) {
-			arizona_start_mic(info);
+			if (info->micd_clamp || !arizona->pdata.jd_invert)
+				regmap_update_bits(arizona->regmap,
+						   ARIZONA_JACK_DETECT_DEBOUNCE,
+						   ARIZONA_MICD_CLAMP_DB |
+						   ARIZONA_JD1_DB, 0);
 		} else {
-			queue_delayed_work(system_power_efficient_wq,
-					   &info->hpdet_work,
-					   msecs_to_jiffies(HPDET_DEBOUNCE));
-		}
+			dev_dbg(arizona->dev, "Detected jack removal\n");
 
-		if (info->micd_clamp || !arizona->pdata.jd_invert)
-			regmap_update_bits(arizona->regmap,
-					   ARIZONA_JACK_DETECT_DEBOUNCE,
-					   ARIZONA_MICD_CLAMP_DB |
-					   ARIZONA_JD1_DB, 0);
-	} else {
-		dev_dbg(arizona->dev, "Detected jack removal\n");
-
-		arizona_stop_mic(info);
+			arizona_stop_mic(info);
 
-		info->num_hpdet_res = 0;
-		for (i = 0; i < ARRAY_SIZE(info->hpdet_res); i++)
-			info->hpdet_res[i] = 0;
-		info->mic = false;
-		info->hpdet_done = false;
-		info->hpdet_retried = false;
+			info->num_hpdet_res = 0;
+			for (i = 0; i < ARRAY_SIZE(info->hpdet_res); i++)
+				info->hpdet_res[i] = 0;
+			info->mic = false;
+			info->hpdet_done = false;
+			info->hpdet_retried = false;
 
-		snd_soc_jack_report(info->jack, 0, ARIZONA_JACK_MASK | info->micd_button_mask);
+			snd_soc_jack_report(info->jack, 0,
+					    ARIZONA_JACK_MASK | info->micd_button_mask);
 
-		/*
-		 * If the jack was removed during a headphone detection we
-		 * need to wait for the headphone detection to finish, as
-		 * it can not be aborted. We don't want to be able to start
-		 * a new headphone detection from a fresh insert until this
-		 * one is finished.
-		 */
-		arizona_hpdet_wait(info);
+			/*
+			 * If the jack was removed during a headphone detection we
+			 * need to wait for the headphone detection to finish, as
+			 * it can not be aborted. We don't want to be able to start
+			 * a new headphone detection from a fresh insert until this
+			 * one is finished.
+			 */
+			arizona_hpdet_wait(info);
 
-		regmap_update_bits(arizona->regmap,
-				   ARIZONA_JACK_DETECT_DEBOUNCE,
-				   ARIZONA_MICD_CLAMP_DB | ARIZONA_JD1_DB,
-				   ARIZONA_MICD_CLAMP_DB | ARIZONA_JD1_DB);
+			regmap_update_bits(arizona->regmap,
+					   ARIZONA_JACK_DETECT_DEBOUNCE,
+					   ARIZONA_MICD_CLAMP_DB | ARIZONA_JD1_DB,
+					   ARIZONA_MICD_CLAMP_DB | ARIZONA_JD1_DB);
+		}
 	}
 
-out:
 	/* Clear trig_sts to make sure DCVDD is not forced up */
 	regmap_write(arizona->regmap, ARIZONA_AOD_WKUP_AND_TRIG,
 		     ARIZONA_MICD_CLAMP_FALL_TRIG_STS |
@@ -1119,10 +1112,6 @@ static irqreturn_t arizona_jackdet(int irq, void *data)
 		     ARIZONA_JD1_FALL_TRIG_STS |
 		     ARIZONA_JD1_RISE_TRIG_STS);
 
-	mutex_unlock(&info->lock);
-
-	pm_runtime_put_autosuspend(arizona->dev);
-
 	return IRQ_HANDLED;
 }
 
diff --git a/sound/soc/codecs/arizona.c b/sound/soc/codecs/arizona.c
index 8c683b0bb74c4..4411165c37345 100644
--- a/sound/soc/codecs/arizona.c
+++ b/sound/soc/codecs/arizona.c
@@ -7,6 +7,7 @@
  * Author: Mark Brown <broonie@opensource.wolfsonmicro.com>
  */
 
+#include <linux/cleanup.h>
 #include <linux/delay.h>
 #include <linux/gcd.h>
 #include <linux/module.h>
@@ -1158,17 +1159,16 @@ int arizona_dvfs_up(struct snd_soc_component *component, unsigned int flags)
 	struct arizona_priv *priv = snd_soc_component_get_drvdata(component);
 	int ret = 0;
 
-	mutex_lock(&priv->dvfs_lock);
+	guard(mutex)(&priv->dvfs_lock);
 
 	if (!priv->dvfs_cached && !priv->dvfs_reqs) {
 		ret = arizona_dvfs_enable(component);
 		if (ret)
-			goto err;
+			return ret;
 	}
 
 	priv->dvfs_reqs |= flags;
-err:
-	mutex_unlock(&priv->dvfs_lock);
+
 	return ret;
 }
 EXPORT_SYMBOL_GPL(arizona_dvfs_up);
@@ -1179,7 +1179,7 @@ int arizona_dvfs_down(struct snd_soc_component *component, unsigned int flags)
 	unsigned int old_reqs;
 	int ret = 0;
 
-	mutex_lock(&priv->dvfs_lock);
+	guard(mutex)(&priv->dvfs_lock);
 
 	old_reqs = priv->dvfs_reqs;
 	priv->dvfs_reqs &= ~flags;
@@ -1187,7 +1187,6 @@ int arizona_dvfs_down(struct snd_soc_component *component, unsigned int flags)
 	if (!priv->dvfs_cached && old_reqs && !priv->dvfs_reqs)
 		ret = arizona_dvfs_disable(component);
 
-	mutex_unlock(&priv->dvfs_lock);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(arizona_dvfs_down);
@@ -1199,7 +1198,7 @@ int arizona_dvfs_sysclk_ev(struct snd_soc_dapm_widget *w,
 	struct arizona_priv *priv = snd_soc_component_get_drvdata(component);
 	int ret = 0;
 
-	mutex_lock(&priv->dvfs_lock);
+	guard(mutex)(&priv->dvfs_lock);
 
 	switch (event) {
 	case SND_SOC_DAPM_POST_PMU:
@@ -1222,7 +1221,6 @@ int arizona_dvfs_sysclk_ev(struct snd_soc_dapm_widget *w,
 		break;
 	}
 
-	mutex_unlock(&priv->dvfs_lock);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(arizona_dvfs_sysclk_ev);
@@ -1657,13 +1655,11 @@ static void arizona_wm5102_set_dac_comp(struct snd_soc_component *component,
 		{ 0x80, 0x0 },
 	};
 
-	mutex_lock(&arizona->dac_comp_lock);
-
-	dac_comp[1].def = arizona->dac_comp_coeff;
-	if (rate >= 176400)
-		dac_comp[2].def = arizona->dac_comp_enabled;
-
-	mutex_unlock(&arizona->dac_comp_lock);
+	scoped_guard(mutex, &arizona->dac_comp_lock) {
+		dac_comp[1].def = arizona->dac_comp_coeff;
+		if (rate >= 176400)
+			dac_comp[2].def = arizona->dac_comp_enabled;
+	}
 
 	regmap_multi_reg_write(arizona->regmap,
 			       dac_comp,
diff --git a/sound/soc/codecs/audio-iio-aux.c b/sound/soc/codecs/audio-iio-aux.c
index 964b9a5b29904..eef0af63e3d61 100644
--- a/sound/soc/codecs/audio-iio-aux.c
+++ b/sound/soc/codecs/audio-iio-aux.c
@@ -311,3 +311,4 @@ module_platform_driver(audio_iio_aux_driver);
 MODULE_AUTHOR("Herve Codina <herve.codina@bootlin.com>");
 MODULE_DESCRIPTION("IIO ALSA SoC aux driver");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/sound/soc/codecs/aw87390.c b/sound/soc/codecs/aw87390.c
index 020213e0ca4b5..8150670fde2d8 100644
--- a/sound/soc/codecs/aw87390.c
+++ b/sound/soc/codecs/aw87390.c
@@ -7,6 +7,7 @@
 // Author: Weidong Wang <wangweidong.a@awinic.com>
 //
 
+#include <linux/cleanup.h>
 #include <linux/i2c.h>
 #include <linux/firmware.h>
 #include <linux/regmap.h>
@@ -225,11 +226,10 @@ static int aw87390_profile_set(struct snd_kcontrol *kcontrol,
 	struct aw87390 *aw87390 = snd_soc_component_get_drvdata(codec);
 	int ret;
 
-	mutex_lock(&aw87390->lock);
+	guard(mutex)(&aw87390->lock);
 	ret = aw87390_dev_set_profile_index(aw87390->aw_pa, ucontrol->value.integer.value[0]);
 	if (ret) {
 		dev_dbg(codec->dev, "profile index does not change\n");
-		mutex_unlock(&aw87390->lock);
 		return 0;
 	}
 
@@ -238,8 +238,6 @@ static int aw87390_profile_set(struct snd_kcontrol *kcontrol,
 		aw87390_power_on(aw87390->aw_pa);
 	}
 
-	mutex_unlock(&aw87390->lock);
-
 	return 1;
 }
 
@@ -280,14 +278,12 @@ static int aw87390_request_firmware_file(struct aw87390 *aw87390)
 		return ret;
 	}
 
-	mutex_lock(&aw87390->lock);
+	guard(mutex)(&aw87390->lock);
 
 	ret = aw88395_dev_cfg_load(aw87390->aw_pa, aw87390->aw_cfg);
 	if (ret)
 		dev_err(aw87390->aw_pa->dev, "aw_dev acf parse failed\n");
 
-	mutex_unlock(&aw87390->lock);
-
 	return ret;
 }
 
diff --git a/sound/soc/codecs/aw88081.c b/sound/soc/codecs/aw88081.c
index d5e886a8f1067..a3cc027de6065 100644
--- a/sound/soc/codecs/aw88081.c
+++ b/sound/soc/codecs/aw88081.c
@@ -7,6 +7,7 @@
 // Author: Weidong Wang <wangweidong.a@awinic.com>
 //
 
+#include <linux/cleanup.h>
 #include <linux/firmware.h>
 #include <linux/i2c.h>
 #include <linux/regmap.h>
@@ -763,9 +764,8 @@ static void aw88081_startup_work(struct work_struct *work)
 	struct aw88081 *aw88081 =
 		container_of(work, struct aw88081, start_work.work);
 
-	mutex_lock(&aw88081->lock);
+	guard(mutex)(&aw88081->lock);
 	aw88081_start_pa(aw88081);
-	mutex_unlock(&aw88081->lock);
 }
 
 static void aw88081_start(struct aw88081 *aw88081, bool sync_start)
@@ -942,11 +942,10 @@ static int aw88081_profile_set(struct snd_kcontrol *kcontrol,
 	int ret;
 
 	/* pa stop or stopping just set profile */
-	mutex_lock(&aw88081->lock);
+	guard(mutex)(&aw88081->lock);
 	ret = aw88081_dev_set_profile_index(aw88081->aw_pa, ucontrol->value.integer.value[0]);
 	if (ret) {
 		dev_dbg(codec->dev, "profile index does not change");
-		mutex_unlock(&aw88081->lock);
 		return 0;
 	}
 
@@ -955,8 +954,6 @@ static int aw88081_profile_set(struct snd_kcontrol *kcontrol,
 		aw88081_start(aw88081, AW88081_SYNC_START);
 	}
 
-	mutex_unlock(&aw88081->lock);
-
 	return 1;
 }
 
@@ -1165,11 +1162,9 @@ static int aw88081_request_firmware_file(struct aw88081 *aw88081)
 	if (ret)
 		return ret;
 
-	mutex_lock(&aw88081->lock);
-	ret = aw88081_dev_init(aw88081, aw88081->aw_cfg);
-	mutex_unlock(&aw88081->lock);
+	guard(mutex)(&aw88081->lock);
 
-	return ret;
+	return aw88081_dev_init(aw88081, aw88081->aw_cfg);
 }
 
 static int aw88081_playback_event(struct snd_soc_dapm_widget *w,
@@ -1178,7 +1173,7 @@ static int aw88081_playback_event(struct snd_soc_dapm_widget *w,
 	struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm);
 	struct aw88081 *aw88081 = snd_soc_component_get_drvdata(component);
 
-	mutex_lock(&aw88081->lock);
+	guard(mutex)(&aw88081->lock);
 	switch (event) {
 	case SND_SOC_DAPM_PRE_PMU:
 		aw88081_start(aw88081, AW88081_ASYNC_START);
@@ -1189,7 +1184,6 @@ static int aw88081_playback_event(struct snd_soc_dapm_widget *w,
 	default:
 		break;
 	}
-	mutex_unlock(&aw88081->lock);
 
 	return 0;
 }
diff --git a/sound/soc/codecs/aw88166.c b/sound/soc/codecs/aw88166.c
index 3f15f4ac51f7a..b72f87f677ddf 100644
--- a/sound/soc/codecs/aw88166.c
+++ b/sound/soc/codecs/aw88166.c
@@ -7,6 +7,7 @@
 // Author: Weidong Wang <wangweidong.a@awinic.com>
 //
 
+#include <linux/cleanup.h>
 #include <linux/crc32.h>
 #include <linux/firmware.h>
 #include <linux/gpio/consumer.h>
@@ -1173,9 +1174,8 @@ static void aw88166_startup_work(struct work_struct *work)
 	struct aw88166 *aw88166 =
 		container_of(work, struct aw88166, start_work.work);
 
-	mutex_lock(&aw88166->lock);
+	guard(mutex)(&aw88166->lock);
 	aw88166_start_pa(aw88166);
-	mutex_unlock(&aw88166->lock);
 }
 
 static void aw88166_start(struct aw88166 *aw88166, bool sync_start)
@@ -1413,11 +1413,10 @@ static int aw88166_profile_set(struct snd_kcontrol *kcontrol,
 	struct aw88166 *aw88166 = snd_soc_component_get_drvdata(codec);
 	int ret;
 
-	mutex_lock(&aw88166->lock);
+	guard(mutex)(&aw88166->lock);
 	ret = aw88166_dev_set_profile_index(aw88166->aw_pa, ucontrol->value.integer.value[0]);
 	if (ret) {
 		dev_dbg(codec->dev, "profile index does not change");
-		mutex_unlock(&aw88166->lock);
 		return 0;
 	}
 
@@ -1426,8 +1425,6 @@ static int aw88166_profile_set(struct snd_kcontrol *kcontrol,
 		aw88166_start(aw88166, AW88166_SYNC_START);
 	}
 
-	mutex_unlock(&aw88166->lock);
-
 	return 1;
 }
 
@@ -1607,12 +1604,12 @@ static int aw88166_request_firmware_file(struct aw88166 *aw88166)
 		return ret;
 	}
 
-	mutex_lock(&aw88166->lock);
-	/* aw device init */
-	ret = aw88166_dev_init(aw88166, aw88166->aw_cfg);
-	if (ret)
-		dev_err(aw88166->aw_pa->dev, "dev init failed\n");
-	mutex_unlock(&aw88166->lock);
+	scoped_guard(mutex, &aw88166->lock) {
+		/* aw device init */
+		ret = aw88166_dev_init(aw88166, aw88166->aw_cfg);
+		if (ret)
+			dev_err(aw88166->aw_pa->dev, "dev init failed\n");
+	}
 
 	return ret;
 }
@@ -1639,7 +1636,7 @@ static int aw88166_playback_event(struct snd_soc_dapm_widget *w,
 	struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm);
 	struct aw88166 *aw88166 = snd_soc_component_get_drvdata(component);
 
-	mutex_lock(&aw88166->lock);
+	guard(mutex)(&aw88166->lock);
 	switch (event) {
 	case SND_SOC_DAPM_PRE_PMU:
 		aw88166_start(aw88166, AW88166_ASYNC_START);
@@ -1650,7 +1647,6 @@ static int aw88166_playback_event(struct snd_soc_dapm_widget *w,
 	default:
 		break;
 	}
-	mutex_unlock(&aw88166->lock);
 
 	return 0;
 }
diff --git a/sound/soc/codecs/aw88261.c b/sound/soc/codecs/aw88261.c
index 549783d3e75ef..acbd7de2e40e9 100644
--- a/sound/soc/codecs/aw88261.c
+++ b/sound/soc/codecs/aw88261.c
@@ -8,6 +8,7 @@
 // Author: Weidong Wang <wangweidong.a@awinic.com>
 //
 
+#include <linux/cleanup.h>
 #include <linux/i2c.h>
 #include <linux/firmware.h>
 #include <linux/bitops.h>
@@ -206,10 +207,10 @@ static int aw88261_dev_check_sysst(struct aw_device *aw_dev)
 			return ret;
 
 		check_val = reg_val & (~AW88261_BIT_SYSST_CHECK_MASK)
-							& AW88261_BIT_SYSST_CHECK;
-		if (check_val != AW88261_BIT_SYSST_CHECK) {
+							& AW88261_BIT_PLL_CHECK;
+		if (check_val != AW88261_BIT_PLL_CHECK) {
 			dev_dbg(aw_dev->dev, "check sysst fail, reg_val=0x%04x, check:0x%x",
-				reg_val, AW88261_BIT_SYSST_CHECK);
+				reg_val, AW88261_BIT_PLL_CHECK);
 			usleep_range(AW88261_2000_US, AW88261_2000_US + 10);
 		} else {
 			return 0;
@@ -960,11 +961,10 @@ static int aw88261_profile_set(struct snd_kcontrol *kcontrol,
 	int ret;
 
 	/* pa stop or stopping just set profile */
-	mutex_lock(&aw88261->lock);
+	guard(mutex)(&aw88261->lock);
 	ret = aw88261_dev_set_profile_index(aw88261->aw_pa, ucontrol->value.integer.value[0]);
 	if (ret) {
 		dev_dbg(codec->dev, "profile index does not change");
-		mutex_unlock(&aw88261->lock);
 		return 0;
 	}
 
@@ -973,8 +973,6 @@ static int aw88261_profile_set(struct snd_kcontrol *kcontrol,
 		aw88261_start(aw88261);
 	}
 
-	mutex_unlock(&aw88261->lock);
-
 	return 1;
 }
 
@@ -1038,7 +1036,7 @@ static int aw88261_playback_event(struct snd_soc_dapm_widget *w,
 	struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm);
 	struct aw88261 *aw88261 = snd_soc_component_get_drvdata(component);
 
-	mutex_lock(&aw88261->lock);
+	guard(mutex)(&aw88261->lock);
 	switch (event) {
 	case SND_SOC_DAPM_PRE_PMU:
 		aw88261_start(aw88261);
@@ -1049,7 +1047,6 @@ static int aw88261_playback_event(struct snd_soc_dapm_widget *w,
 	default:
 		break;
 	}
-	mutex_unlock(&aw88261->lock);
 
 	return 0;
 }
@@ -1188,12 +1185,12 @@ static int aw88261_request_firmware_file(struct aw88261 *aw88261)
 		return ret;
 	}
 
-	mutex_lock(&aw88261->lock);
-	/* aw device init */
-	ret = aw88261_dev_init(aw88261, aw88261->aw_cfg);
-	if (ret)
-		dev_err(aw88261->aw_pa->dev, "dev init failed");
-	mutex_unlock(&aw88261->lock);
+	scoped_guard(mutex, &aw88261->lock) {
+		/* aw device init */
+		ret = aw88261_dev_init(aw88261, aw88261->aw_cfg);
+		if (ret)
+			dev_err(aw88261->aw_pa->dev, "dev init failed");
+	}
 
 	return ret;
 }
diff --git a/sound/soc/codecs/aw88261.h b/sound/soc/codecs/aw88261.h
index 270ccf375f367..f16f9f8c40a2f 100644
--- a/sound/soc/codecs/aw88261.h
+++ b/sound/soc/codecs/aw88261.h
@@ -194,12 +194,6 @@
 		AW88261_OTHS_OT_VALUE | \
 		AW88261_PLLS_LOCKED_VALUE))
 
-#define AW88261_BIT_SYSST_CHECK \
-		(AW88261_BSTS_FINISHED_VALUE | \
-		AW88261_SWS_SWITCHING_VALUE | \
-		AW88261_CLKS_STABLE_VALUE | \
-		AW88261_PLLS_LOCKED_VALUE)
-
 #define AW88261_ULS_HMUTE_START_BIT	(14)
 #define AW88261_ULS_HMUTE_BITS_LEN	(1)
 #define AW88261_ULS_HMUTE_MASK		\
diff --git a/sound/soc/codecs/aw88395/aw88395.c b/sound/soc/codecs/aw88395/aw88395.c
index ee0e8bd8c54cc..e9ff2c79ac152 100644
--- a/sound/soc/codecs/aw88395/aw88395.c
+++ b/sound/soc/codecs/aw88395/aw88395.c
@@ -8,6 +8,7 @@
 // Author: Weidong Wang <wangweidong.a@awinic.com>
 //
 
+#include <linux/cleanup.h>
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/firmware.h>
@@ -51,9 +52,8 @@ static void aw88395_startup_work(struct work_struct *work)
 	struct aw88395 *aw88395 =
 		container_of(work, struct aw88395, start_work.work);
 
-	mutex_lock(&aw88395->lock);
+	guard(mutex)(&aw88395->lock);
 	aw88395_start_pa(aw88395);
-	mutex_unlock(&aw88395->lock);
 }
 
 static void aw88395_start(struct aw88395 *aw88395, bool sync_start)
@@ -224,11 +224,10 @@ static int aw88395_profile_set(struct snd_kcontrol *kcontrol,
 	int ret;
 
 	/* pa stop or stopping just set profile */
-	mutex_lock(&aw88395->lock);
+	guard(mutex)(&aw88395->lock);
 	ret = aw88395_dev_set_profile_index(aw88395->aw_pa, ucontrol->value.integer.value[0]);
 	if (ret < 0) {
 		dev_dbg(codec->dev, "profile index does not change");
-		mutex_unlock(&aw88395->lock);
 		return 0;
 	}
 
@@ -237,8 +236,6 @@ static int aw88395_profile_set(struct snd_kcontrol *kcontrol,
 		aw88395_start(aw88395, AW88395_SYNC_START);
 	}
 
-	mutex_unlock(&aw88395->lock);
-
 	return 1;
 }
 
@@ -366,7 +363,7 @@ static int aw88395_playback_event(struct snd_soc_dapm_widget *w,
 	struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm);
 	struct aw88395 *aw88395 = snd_soc_component_get_drvdata(component);
 
-	mutex_lock(&aw88395->lock);
+	guard(mutex)(&aw88395->lock);
 	switch (event) {
 	case SND_SOC_DAPM_PRE_PMU:
 		aw88395_start(aw88395, AW88395_ASYNC_START);
@@ -377,7 +374,6 @@ static int aw88395_playback_event(struct snd_soc_dapm_widget *w,
 	default:
 		break;
 	}
-	mutex_unlock(&aw88395->lock);
 
 	return 0;
 }
@@ -495,12 +491,12 @@ static int aw88395_request_firmware_file(struct aw88395 *aw88395)
 
 	dev_dbg(aw88395->aw_pa->dev, "%s : bin load success\n", __func__);
 
-	mutex_lock(&aw88395->lock);
-	/* aw device init */
-	ret = aw88395_dev_init(aw88395->aw_pa, aw88395->aw_cfg);
-	if (ret < 0)
-		dev_err(aw88395->aw_pa->dev, "dev init failed");
-	mutex_unlock(&aw88395->lock);
+	scoped_guard(mutex, &aw88395->lock) {
+		/* aw device init */
+		ret = aw88395_dev_init(aw88395->aw_pa, aw88395->aw_cfg);
+		if (ret < 0)
+			dev_err(aw88395->aw_pa->dev, "dev init failed");
+	}
 
 	return ret;
 }
diff --git a/sound/soc/codecs/aw88395/aw88395_device.c b/sound/soc/codecs/aw88395/aw88395_device.c
index 79c3135a41103..12d42d27560db 100644
--- a/sound/soc/codecs/aw88395/aw88395_device.c
+++ b/sound/soc/codecs/aw88395/aw88395_device.c
@@ -8,6 +8,7 @@
 // Author: Ben Yi <yijiangtao@awinic.com>
 //
 
+#include <linux/cleanup.h>
 #include <linux/crc32.h>
 #include <linux/i2c.h>
 #include <linux/minmax.h>
@@ -70,7 +71,7 @@ int aw_dev_dsp_write(struct aw_device *aw_dev,
 	u32 reg_value;
 	int ret;
 
-	mutex_lock(&aw_dev->dsp_lock);
+	guard(mutex)(&aw_dev->dsp_lock);
 	switch (data_type) {
 	case AW_DSP_16_DATA:
 		ret = aw_dev_dsp_write_16bit(aw_dev, dsp_addr, dsp_data);
@@ -93,7 +94,6 @@ int aw_dev_dsp_write(struct aw_device *aw_dev,
 	/* clear dsp chip select state*/
 	if (regmap_read(aw_dev->regmap, AW88395_ID_REG, &reg_value))
 		dev_err(aw_dev->dev, "%s fail to clear chip state. Err=%d\n", __func__, ret);
-	mutex_unlock(&aw_dev->dsp_lock);
 
 	return ret;
 }
@@ -156,7 +156,7 @@ int aw_dev_dsp_read(struct aw_device *aw_dev,
 	u32 reg_value;
 	int ret;
 
-	mutex_lock(&aw_dev->dsp_lock);
+	guard(mutex)(&aw_dev->dsp_lock);
 	switch (data_type) {
 	case AW_DSP_16_DATA:
 		ret = aw_dev_dsp_read_16bit(aw_dev, dsp_addr, dsp_data);
@@ -179,7 +179,6 @@ int aw_dev_dsp_read(struct aw_device *aw_dev,
 	/* clear dsp chip select state*/
 	if (regmap_read(aw_dev->regmap, AW88395_ID_REG, &reg_value))
 		dev_err(aw_dev->dev, "%s fail to clear chip state. Err=%d\n", __func__, ret);
-	mutex_unlock(&aw_dev->dsp_lock);
 
 	return ret;
 }
@@ -1110,42 +1109,36 @@ static int aw_dev_dsp_update_container(struct aw_device *aw_dev,
 #ifdef AW88395_DSP_I2C_WRITES
 	u32 tmp_len;
 
-	mutex_lock(&aw_dev->dsp_lock);
+	guard(mutex)(&aw_dev->dsp_lock);
 	ret = regmap_write(aw_dev->regmap, AW88395_DSPMADD_REG, base);
 	if (ret)
-		goto error_operation;
+		return ret;
 
 	for (i = 0; i < len; i += AW88395_MAX_RAM_WRITE_BYTE_SIZE) {
 		tmp_len = min(len - i, AW88395_MAX_RAM_WRITE_BYTE_SIZE);
 		ret = regmap_raw_write(aw_dev->regmap, AW88395_DSPMDAT_REG,
 					&data[i], tmp_len);
 		if (ret)
-			goto error_operation;
+			return ret;
 	}
-	mutex_unlock(&aw_dev->dsp_lock);
 #else
 	__be16 reg_val;
 
-	mutex_lock(&aw_dev->dsp_lock);
+	guard(mutex)(&aw_dev->dsp_lock);
 	/* i2c write */
 	ret = regmap_write(aw_dev->regmap, AW88395_DSPMADD_REG, base);
 	if (ret)
-		goto error_operation;
+		return ret;
 	for (i = 0; i < len; i += 2) {
 		reg_val = cpu_to_be16p((u16 *)(data + i));
 		ret = regmap_write(aw_dev->regmap, AW88395_DSPMDAT_REG,
 					(u16)reg_val);
 		if (ret)
-			goto error_operation;
+			return ret;
 	}
-	mutex_unlock(&aw_dev->dsp_lock);
 #endif
 
 	return 0;
-
-error_operation:
-	mutex_unlock(&aw_dev->dsp_lock);
-	return ret;
 }
 
 static int aw_dev_dsp_update_fw(struct aw_device *aw_dev,
@@ -1231,14 +1224,14 @@ static int aw_dev_check_sram(struct aw_device *aw_dev)
 {
 	unsigned int reg_val;
 
-	mutex_lock(&aw_dev->dsp_lock);
+	guard(mutex)(&aw_dev->dsp_lock);
 	/* check the odd bits of reg 0x40 */
 	regmap_write(aw_dev->regmap, AW88395_DSPMADD_REG, AW88395_DSP_ODD_NUM_BIT_TEST);
 	regmap_read(aw_dev->regmap, AW88395_DSPMADD_REG, &reg_val);
 	if (reg_val != AW88395_DSP_ODD_NUM_BIT_TEST) {
 		dev_err(aw_dev->dev, "check reg 0x40 odd bit failed, read[0x%x] != write[0x%x]",
 				reg_val, AW88395_DSP_ODD_NUM_BIT_TEST);
-		goto error;
+		return -EPERM;
 	}
 
 	/* check the even bits of reg 0x40 */
@@ -1247,7 +1240,7 @@ static int aw_dev_check_sram(struct aw_device *aw_dev)
 	if (reg_val != AW88395_DSP_EVEN_NUM_BIT_TEST) {
 		dev_err(aw_dev->dev, "check reg 0x40 even bit failed, read[0x%x] != write[0x%x]",
 				reg_val, AW88395_DSP_EVEN_NUM_BIT_TEST);
-		goto error;
+		return -EPERM;
 	}
 
 	/* check dsp_fw_base_addr */
@@ -1256,7 +1249,7 @@ static int aw_dev_check_sram(struct aw_device *aw_dev)
 	if (reg_val != AW88395_DSP_EVEN_NUM_BIT_TEST) {
 		dev_err(aw_dev->dev, "check dsp fw addr failed, read[0x%x] != write[0x%x]",
 						reg_val, AW88395_DSP_EVEN_NUM_BIT_TEST);
-		goto error;
+		return -EPERM;
 	}
 
 	/* check dsp_cfg_base_addr */
@@ -1265,15 +1258,10 @@ static int aw_dev_check_sram(struct aw_device *aw_dev)
 	if (reg_val != AW88395_DSP_ODD_NUM_BIT_TEST) {
 		dev_err(aw_dev->dev, "check dsp cfg failed, read[0x%x] != write[0x%x]",
 						reg_val, AW88395_DSP_ODD_NUM_BIT_TEST);
-		goto error;
+		return -EPERM;
 	}
-	mutex_unlock(&aw_dev->dsp_lock);
 
 	return 0;
-
-error:
-	mutex_unlock(&aw_dev->dsp_lock);
-	return -EPERM;
 }
 
 int aw88395_dev_fw_update(struct aw_device *aw_dev, bool up_dsp_fw_en, bool force_up_en)
diff --git a/sound/soc/codecs/aw88399.c b/sound/soc/codecs/aw88399.c
index b2ec3503f7e23..67ea073abdb7f 100644
--- a/sound/soc/codecs/aw88399.c
+++ b/sound/soc/codecs/aw88399.c
@@ -7,6 +7,7 @@
 // Author: Weidong Wang <wangweidong.a@awinic.com>
 //
 
+#include <linux/cleanup.h>
 #include <linux/crc32.h>
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
@@ -1140,9 +1141,8 @@ static void aw88399_startup_work(struct work_struct *work)
 	struct aw88399 *aw88399 =
 		container_of(work, struct aw88399, start_work.work);
 
-	mutex_lock(&aw88399->lock);
+	guard(mutex)(&aw88399->lock);
 	aw88399_start_pa(aw88399);
-	mutex_unlock(&aw88399->lock);
 }
 
 static void aw88399_start(struct aw88399 *aw88399, bool sync_start)
@@ -1702,11 +1702,10 @@ static int aw88399_profile_set(struct snd_kcontrol *kcontrol,
 	struct aw88399 *aw88399 = snd_soc_component_get_drvdata(codec);
 	int ret;
 
-	mutex_lock(&aw88399->lock);
+	guard(mutex)(&aw88399->lock);
 	ret = aw88399_dev_set_profile_index(aw88399->aw_pa, ucontrol->value.integer.value[0]);
 	if (ret) {
 		dev_dbg(codec->dev, "profile index does not change");
-		mutex_unlock(&aw88399->lock);
 		return 0;
 	}
 
@@ -1715,8 +1714,6 @@ static int aw88399_profile_set(struct snd_kcontrol *kcontrol,
 		aw88399_start(aw88399, AW88399_SYNC_START);
 	}
 
-	mutex_unlock(&aw88399->lock);
-
 	return 1;
 }
 
@@ -1939,12 +1936,11 @@ static int aw88399_request_firmware_file(struct aw88399 *aw88399)
 		return ret;
 	}
 
-	mutex_lock(&aw88399->lock);
+	guard(mutex)(&aw88399->lock);
 	/* aw device init */
 	ret = aw88399_dev_init(aw88399, aw88399->aw_cfg);
 	if (ret)
 		dev_err(aw88399->aw_pa->dev, "dev init failed");
-	mutex_unlock(&aw88399->lock);
 
 	return ret;
 }
@@ -1975,7 +1971,7 @@ static int aw88399_playback_event(struct snd_soc_dapm_widget *w,
 	struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm);
 	struct aw88399 *aw88399 = snd_soc_component_get_drvdata(component);
 
-	mutex_lock(&aw88399->lock);
+	guard(mutex)(&aw88399->lock);
 	switch (event) {
 	case SND_SOC_DAPM_PRE_PMU:
 		aw88399_start(aw88399, AW88399_ASYNC_START);
@@ -1986,7 +1982,6 @@ static int aw88399_playback_event(struct snd_soc_dapm_widget *w,
 	default:
 		break;
 	}
-	mutex_unlock(&aw88399->lock);
 
 	return 0;
 }
diff --git a/sound/soc/codecs/bt-sco.c b/sound/soc/codecs/bt-sco.c
index 3afcef2dfa352..c0bf45b76cb8c 100644
--- a/sound/soc/codecs/bt-sco.c
+++ b/sound/soc/codecs/bt-sco.c
@@ -17,11 +17,17 @@ static const struct snd_soc_dapm_widget bt_sco_widgets[] = {
 			    SND_SOC_NOPM, 0, 0),
 	SND_SOC_DAPM_AIF_OUT("BT_SCO_TX", "Capture", 0,
 			     SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_IN("BT_SCO_RX_WB", "WB Playback", 0,
+			    SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("BT_SCO_TX_WB", "WB Capture", 0,
+			     SND_SOC_NOPM, 0, 0),
 };
 
 static const struct snd_soc_dapm_route bt_sco_routes[] = {
 	{ "BT_SCO_TX", NULL, "RX" },
 	{ "TX", NULL, "BT_SCO_RX" },
+	{ "BT_SCO_TX_WB", NULL, "RX" },
+	{ "TX", NULL, "BT_SCO_RX_WB" },
 };
 
 static struct snd_soc_dai_driver bt_sco_dai[] = {
@@ -45,14 +51,14 @@ static struct snd_soc_dai_driver bt_sco_dai[] = {
 	{
 		.name = "bt-sco-pcm-wb",
 		.playback = {
-			.stream_name = "Playback",
+			.stream_name = "WB Playback",
 			.channels_min = 1,
 			.channels_max = 1,
 			.rates = SNDRV_PCM_RATE_8000 | SNDRV_PCM_RATE_16000,
 			.formats = SNDRV_PCM_FMTBIT_S16_LE,
 		},
 		.capture = {
-			 .stream_name = "Capture",
+			 .stream_name = "WB Capture",
 			.channels_min = 1,
 			.channels_max = 1,
 			.rates = SNDRV_PCM_RATE_8000 | SNDRV_PCM_RATE_16000,
diff --git a/sound/soc/codecs/cros_ec_codec.c b/sound/soc/codecs/cros_ec_codec.c
index 7dc5a7c3ca96d..fd2d5d7f92769 100644
--- a/sound/soc/codecs/cros_ec_codec.c
+++ b/sound/soc/codecs/cros_ec_codec.c
@@ -12,6 +12,7 @@
 #include <linux/acpi.h>
 #include <linux/delay.h>
 #include <linux/device.h>
+#include <linux/cleanup.h>
 #include <linux/io.h>
 #include <linux/jiffies.h>
 #include <linux/kernel.h>
@@ -608,10 +609,10 @@ static void wov_copy_work(struct work_struct *w)
 		container_of(w, struct cros_ec_codec_priv, wov_copy_work.work);
 	int ret;
 
-	mutex_lock(&priv->wov_dma_lock);
+	guard(mutex)(&priv->wov_dma_lock);
 	if (!priv->wov_substream) {
 		dev_warn(priv->dev, "no pcm substream\n");
-		goto leave;
+		return;
 	}
 
 	if (ec_codec_capable(priv, EC_CODEC_CAP_WOV_AUDIO_SHM))
@@ -624,8 +625,6 @@ static void wov_copy_work(struct work_struct *w)
 				      msecs_to_jiffies(10));
 	else if (ret)
 		dev_err(priv->dev, "failed to read audio data\n");
-leave:
-	mutex_unlock(&priv->wov_dma_lock);
 }
 
 static int wov_enable_get(struct snd_kcontrol *kcontrol,
@@ -895,12 +894,11 @@ static int wov_pcm_hw_params(struct snd_soc_component *component,
 	struct cros_ec_codec_priv *priv =
 		snd_soc_component_get_drvdata(component);
 
-	mutex_lock(&priv->wov_dma_lock);
+	guard(mutex)(&priv->wov_dma_lock);
 	priv->wov_substream = substream;
 	priv->wov_rp = priv->wov_wp = 0;
 	priv->wov_dma_offset = 0;
 	priv->wov_burst_read = true;
-	mutex_unlock(&priv->wov_dma_lock);
 
 	return 0;
 }
@@ -911,10 +909,10 @@ static int wov_pcm_hw_free(struct snd_soc_component *component,
 	struct cros_ec_codec_priv *priv =
 		snd_soc_component_get_drvdata(component);
 
-	mutex_lock(&priv->wov_dma_lock);
-	wov_queue_dequeue(priv, wov_queue_size(priv));
-	priv->wov_substream = NULL;
-	mutex_unlock(&priv->wov_dma_lock);
+	scoped_guard(mutex, &priv->wov_dma_lock) {
+		wov_queue_dequeue(priv, wov_queue_size(priv));
+		priv->wov_substream = NULL;
+	}
 
 	cancel_delayed_work_sync(&priv->wov_copy_work);
 
diff --git a/sound/soc/codecs/cs-amp-lib.c b/sound/soc/codecs/cs-amp-lib.c
index 371e99205b58e..41a9a5b005c63 100644
--- a/sound/soc/codecs/cs-amp-lib.c
+++ b/sound/soc/codecs/cs-amp-lib.c
@@ -7,6 +7,7 @@
 
 #include <asm/byteorder.h>
 #include <kunit/static_stub.h>
+#include <linux/cleanup.h>
 #include <linux/debugfs.h>
 #include <linux/dev_printk.h>
 #include <linux/efi.h>
@@ -83,10 +84,12 @@ static int cs_amp_write_cal_coeff(struct cs_dsp *dsp,
 	KUNIT_STATIC_STUB_REDIRECT(cs_amp_write_cal_coeff, dsp, controls, ctl_name, val);
 
 	if (IS_REACHABLE(CONFIG_FW_CS_DSP)) {
-		mutex_lock(&dsp->pwr_lock);
-		cs_ctl = cs_dsp_get_ctl(dsp, ctl_name, controls->mem_region, controls->alg_id);
-		ret = cs_dsp_coeff_write_ctrl(cs_ctl, 0, &beval, sizeof(beval));
-		mutex_unlock(&dsp->pwr_lock);
+		scoped_guard(mutex, &dsp->pwr_lock) {
+			cs_ctl = cs_dsp_get_ctl(dsp, ctl_name,
+						controls->mem_region,
+						controls->alg_id);
+			ret = cs_dsp_coeff_write_ctrl(cs_ctl, 0, &beval, sizeof(beval));
+		}
 
 		if (ret < 0) {
 			dev_err(dsp->dev, "Failed to write to '%s': %d\n", ctl_name, ret);
diff --git a/sound/soc/codecs/cs35l32.c b/sound/soc/codecs/cs35l32.c
index c835088de578a..8288fb6d50be1 100644
--- a/sound/soc/codecs/cs35l32.c
+++ b/sound/soc/codecs/cs35l32.c
@@ -538,7 +538,15 @@ static int cs35l32_runtime_resume(struct device *dev)
 	gpiod_set_value_cansleep(cs35l32->reset_gpio, 1);
 
 	regcache_cache_only(cs35l32->regmap, false);
-	regcache_sync(cs35l32->regmap);
+	ret = regcache_sync(cs35l32->regmap);
+	if (ret) {
+		regcache_cache_only(cs35l32->regmap, true);
+		regcache_mark_dirty(cs35l32->regmap);
+		gpiod_set_value_cansleep(cs35l32->reset_gpio, 0);
+		regulator_bulk_disable(ARRAY_SIZE(cs35l32->supplies),
+				       cs35l32->supplies);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/cs35l33.c b/sound/soc/codecs/cs35l33.c
index f49edb9ea1f2c..e217da124549a 100644
--- a/sound/soc/codecs/cs35l33.c
+++ b/sound/soc/codecs/cs35l33.c
@@ -40,6 +40,7 @@ struct cs35l33_private {
 	struct regmap *regmap;
 	struct gpio_desc *reset_gpio;
 	bool amp_cal;
+	bool irq_requested;
 	int mclk_int;
 	struct regulator_bulk_data core_supplies[2];
 	int num_core_supplies;
@@ -881,6 +882,9 @@ static int cs35l33_runtime_resume(struct device *dev)
 		goto err;
 	}
 
+	if (cs35l33->irq_requested)
+		enable_irq(to_i2c_client(dev)->irq);
+
 	return 0;
 
 err:
@@ -900,6 +904,10 @@ static int cs35l33_runtime_suspend(struct device *dev)
 	/* redo the calibration in next power up */
 	cs35l33->amp_cal = false;
 
+	/* Drain and block the threaded IRQ before cache_only/power-off. */
+	if (cs35l33->irq_requested)
+		disable_irq(to_i2c_client(dev)->irq);
+
 	regcache_cache_only(cs35l33->regmap, true);
 	regcache_mark_dirty(cs35l33->regmap);
 	regulator_bulk_disable(cs35l33->num_core_supplies,
@@ -1154,10 +1162,12 @@ static int cs35l33_i2c_probe(struct i2c_client *i2c_client)
 	}
 
 	ret = devm_request_threaded_irq(&i2c_client->dev, i2c_client->irq, NULL,
-			cs35l33_irq_thread, IRQF_ONESHOT | IRQF_TRIGGER_LOW,
-			"cs35l33", cs35l33);
+				cs35l33_irq_thread, IRQF_ONESHOT | IRQF_TRIGGER_LOW,
+				"cs35l33", cs35l33);
 	if (ret != 0)
 		dev_warn(&i2c_client->dev, "Failed to request IRQ: %d\n", ret);
+	else
+		cs35l33->irq_requested = true;
 
 	/* We could issue !RST or skip it based on AMP topology */
 	cs35l33->reset_gpio = devm_gpiod_get_optional(&i2c_client->dev,
diff --git a/sound/soc/codecs/cs35l34.c b/sound/soc/codecs/cs35l34.c
index e80984b221593..3f265bd760f7c 100644
--- a/sound/soc/codecs/cs35l34.c
+++ b/sound/soc/codecs/cs35l34.c
@@ -45,6 +45,7 @@ struct  cs35l34_private {
 	int num_core_supplies;
 	int mclk_int;
 	bool tdm_mode;
+	bool irq_requested;
 	struct gpio_desc *reset_gpio;	/* Active-low reset GPIO */
 };
 
@@ -1032,10 +1033,12 @@ static int cs35l34_i2c_probe(struct i2c_client *i2c_client)
 	}
 
 	ret = devm_request_threaded_irq(&i2c_client->dev, i2c_client->irq, NULL,
-			cs35l34_irq_thread, IRQF_ONESHOT | IRQF_TRIGGER_LOW,
-			"cs35l34", cs35l34);
+				cs35l34_irq_thread, IRQF_ONESHOT | IRQF_TRIGGER_LOW,
+				"cs35l34", cs35l34);
 	if (ret != 0)
 		dev_err(&i2c_client->dev, "Failed to request IRQ: %d\n", ret);
+	else
+		cs35l34->irq_requested = true;
 
 	cs35l34->reset_gpio = devm_gpiod_get_optional(&i2c_client->dev,
 				"reset", GPIOD_OUT_LOW);
@@ -1140,6 +1143,9 @@ static int cs35l34_runtime_resume(struct device *dev)
 		dev_err(dev, "Failed to restore register cache\n");
 		goto err;
 	}
+
+	if (cs35l34->irq_requested)
+		enable_irq(to_i2c_client(dev)->irq);
 	return 0;
 err:
 	regcache_cache_only(cs35l34->regmap, true);
@@ -1153,6 +1159,10 @@ static int cs35l34_runtime_suspend(struct device *dev)
 {
 	struct cs35l34_private *cs35l34 = dev_get_drvdata(dev);
 
+	/* Drain and block the threaded IRQ before cache_only/power-off. */
+	if (cs35l34->irq_requested)
+		disable_irq(to_i2c_client(dev)->irq);
+
 	regcache_cache_only(cs35l34->regmap, true);
 	regcache_mark_dirty(cs35l34->regmap);
 
diff --git a/sound/soc/codecs/cs35l56-i2c.c b/sound/soc/codecs/cs35l56-i2c.c
index 0f64ab628b03b..4f6ddf1c5a3f6 100644
--- a/sound/soc/codecs/cs35l56-i2c.c
+++ b/sound/soc/codecs/cs35l56-i2c.c
@@ -55,9 +55,7 @@ static int cs35l56_i2c_probe(struct i2c_client *client)
 	if (ret != 0)
 		return ret;
 
-	ret = cs35l56_init(cs35l56);
-	if (ret == 0)
-		ret = cs35l56_irq_request(&cs35l56->base, client->irq);
+	ret = cs35l56_irq_request(&cs35l56->base, client->irq);
 	if (ret < 0)
 		cs35l56_remove(cs35l56);
 
diff --git a/sound/soc/codecs/cs35l56-sdw.c b/sound/soc/codecs/cs35l56-sdw.c
index 0a55b93b96f96..1e442f43b306b 100644
--- a/sound/soc/codecs/cs35l56-sdw.c
+++ b/sound/soc/codecs/cs35l56-sdw.c
@@ -457,6 +457,7 @@ static int cs35l56_sdw_probe(struct sdw_slave *peripheral, const struct sdw_devi
 		regmap_config = &cs35l56_regmap_sdw;
 		break;
 	case 0x3563:
+	case 0x3562:
 		regmap_config = &cs35l63_regmap_sdw;
 		break;
 	default:
@@ -510,6 +511,7 @@ static const struct sdw_device_id cs35l56_sdw_id[] = {
 	SDW_SLAVE_ENTRY(0x01FA, 0x3556, 0x3556),
 	SDW_SLAVE_ENTRY(0x01FA, 0x3557, 0x3557),
 	SDW_SLAVE_ENTRY(0x01FA, 0x3563, 0x3563),
+	SDW_SLAVE_ENTRY(0x01FA, 0x3562, 0x3562),
 	{},
 };
 MODULE_DEVICE_TABLE(sdw, cs35l56_sdw_id);
diff --git a/sound/soc/codecs/cs35l56-shared.c b/sound/soc/codecs/cs35l56-shared.c
index f14e2eaaa4ee4..f20bcdfd35af0 100644
--- a/sound/soc/codecs/cs35l56-shared.c
+++ b/sound/soc/codecs/cs35l56-shared.c
@@ -96,6 +96,7 @@ int cs35l56_set_patch(struct cs35l56_base *cs35l56_base)
 					    ARRAY_SIZE(cs35l56_patch_fw));
 		break;
 	case 0x63:
+	case 0x62:
 		ret = regmap_register_patch(cs35l56_base->regmap, cs35l63_patch_fw,
 					    ARRAY_SIZE(cs35l63_patch_fw));
 		break;
@@ -389,6 +390,7 @@ static void cs35l56_set_fw_reg_table(struct cs35l56_base *cs35l56_base)
 		}
 		break;
 	case 0x63:
+	case 0x62:
 		cs35l56_base->fw_reg = &cs35l63_fw_reg;
 		break;
 	}
@@ -595,6 +597,7 @@ void cs35l56_system_reset(struct cs35l56_base *cs35l56_base, bool is_soundwire)
 		}
 		break;
 	case 0x63:
+	case 0x62:
 		regmap_multi_reg_write_bypassed(cs35l56_base->regmap,
 						cs35l63_system_reset_seq,
 						ARRAY_SIZE(cs35l63_system_reset_seq));
@@ -640,23 +643,22 @@ irqreturn_t cs35l56_irq(int irq, void *data)
 	unsigned int val;
 	int rv;
 
-	irqreturn_t ret = IRQ_NONE;
-
 	if (!cs35l56_base->init_done)
 		return IRQ_NONE;
 
-	mutex_lock(&cs35l56_base->irq_lock);
+	guard(mutex)(&cs35l56_base->irq_lock);
 
-	rv = pm_runtime_resume_and_get(cs35l56_base->dev);
+	PM_RUNTIME_ACQUIRE_IF_ENABLED(cs35l56_base->dev, pm);
+	rv = PM_RUNTIME_ACQUIRE_ERR(&pm);
 	if (rv < 0) {
 		dev_err(cs35l56_base->dev, "irq: failed to get pm_runtime: %d\n", rv);
-		goto err_unlock;
+		return IRQ_NONE;
 	}
 
 	regmap_read(cs35l56_base->regmap, CS35L56_IRQ1_STATUS, &val);
 	if ((val & CS35L56_IRQ1_STS_MASK) == 0) {
 		dev_dbg(cs35l56_base->dev, "Spurious IRQ: no pending interrupt\n");
-		goto err;
+		return IRQ_NONE;
 	}
 
 	/* Ack interrupts */
@@ -680,7 +682,7 @@ irqreturn_t cs35l56_irq(int irq, void *data)
 
 	/* Check to see if unmasked bits are active */
 	if (!status1 && !status8 && !status20)
-		goto err;
+		return IRQ_NONE;
 
 	if (status1 & CS35L56_AMP_SHORT_ERR_EINT1_MASK)
 		dev_crit(cs35l56_base->dev, "Amp short error\n");
@@ -688,14 +690,7 @@ irqreturn_t cs35l56_irq(int irq, void *data)
 	if (status8 & CS35L56_TEMP_ERR_EINT1_MASK)
 		dev_crit(cs35l56_base->dev, "Overtemp error\n");
 
-	ret = IRQ_HANDLED;
-
-err:
-	pm_runtime_put(cs35l56_base->dev);
-err_unlock:
-	mutex_unlock(&cs35l56_base->irq_lock);
-
-	return ret;
+	return IRQ_HANDLED;
 }
 EXPORT_SYMBOL_NS_GPL(cs35l56_irq, "SND_SOC_CS35L56_SHARED");
 
@@ -1470,6 +1465,7 @@ int cs35l56_hw_init(struct cs35l56_base *cs35l56_base)
 		cs35l56_base->calibration_controls = &cs35l56_calibration_controls;
 		break;
 	case 0x35A630:
+	case 0x35A620:
 		cs35l56_base->calibration_controls = &cs35l63_calibration_controls;
 		devid = devid >> 4;
 		break;
diff --git a/sound/soc/codecs/cs35l56-spi.c b/sound/soc/codecs/cs35l56-spi.c
index 9bc9b7c98390d..b1eb924a5b6cc 100644
--- a/sound/soc/codecs/cs35l56-spi.c
+++ b/sound/soc/codecs/cs35l56-spi.c
@@ -44,9 +44,7 @@ static int cs35l56_spi_probe(struct spi_device *spi)
 	if (ret != 0)
 		return ret;
 
-	ret = cs35l56_init(cs35l56);
-	if (ret == 0)
-		ret = cs35l56_irq_request(&cs35l56->base, spi->irq);
+	ret = cs35l56_irq_request(&cs35l56->base, spi->irq);
 	if (ret < 0)
 		cs35l56_remove(cs35l56);
 
diff --git a/sound/soc/codecs/cs35l56.c b/sound/soc/codecs/cs35l56.c
index 570a68829ccd0..0b7b080939a18 100644
--- a/sound/soc/codecs/cs35l56.c
+++ b/sound/soc/codecs/cs35l56.c
@@ -1402,6 +1402,7 @@ static int _cs35l56_component_probe(struct snd_soc_component *component)
 						     ARRAY_SIZE(cs35l56_controls));
 		break;
 	case 0x63:
+	case 0x62:
 		ret = snd_soc_add_component_controls(component, cs35l63_controls,
 						     ARRAY_SIZE(cs35l63_controls));
 		break;
@@ -2008,6 +2009,16 @@ int cs35l56_common_probe(struct cs35l56_private *cs35l56)
 		goto err;
 	}
 
+	/*
+	 * On SoundWire the cs35l56_init() cannot be run until after the
+	 * device has been enumerated by the SoundWire core.
+	 */
+	if (!cs35l56->sdw_peripheral) {
+		ret = cs35l56_init(cs35l56);
+		if (ret)
+			goto err_remove_wm_adsp;
+	}
+
 	ret = snd_soc_register_component(cs35l56->base.dev,
 					 &soc_component_dev_cs35l56,
 					 cs35l56_dai, ARRAY_SIZE(cs35l56_dai));
@@ -2022,6 +2033,11 @@ int cs35l56_common_probe(struct cs35l56_private *cs35l56)
 	wm_adsp2_remove(&cs35l56->dsp);
 
 err:
+	if (pm_runtime_enabled(cs35l56->base.dev)) {
+		pm_runtime_dont_use_autosuspend(cs35l56->base.dev);
+		pm_runtime_disable(cs35l56->base.dev);
+	}
+
 	gpiod_set_value_cansleep(cs35l56->base.reset_gpio, 0);
 	regulator_bulk_disable(ARRAY_SIZE(cs35l56->supplies), cs35l56->supplies);
 
@@ -2108,7 +2124,7 @@ int cs35l56_init(struct cs35l56_private *cs35l56)
 		return dev_err_probe(cs35l56->base.dev, ret, "Failed to write ASP1_CONTROL3\n");
 
 	cs35l56->base.init_done = true;
-	complete(&cs35l56->init_completion);
+	complete_all(&cs35l56->init_completion);
 
 	return 0;
 }
diff --git a/sound/soc/codecs/cs42l42-sdw.c b/sound/soc/codecs/cs42l42-sdw.c
index ad1256910a180..b61ca74dcf0b6 100644
--- a/sound/soc/codecs/cs42l42-sdw.c
+++ b/sound/soc/codecs/cs42l42-sdw.c
@@ -481,10 +481,20 @@ static int cs42l42_sdw_runtime_resume(struct device *dev)
 	regcache_cache_only(cs42l42->regmap, false);
 
 	/* Sync LATCH_TO_VP first so the VP domain registers sync correctly */
-	regcache_sync_region(cs42l42->regmap, CS42L42_MIC_DET_CTL1, CS42L42_MIC_DET_CTL1);
-	regcache_sync(cs42l42->regmap);
+	ret = regcache_sync_region(cs42l42->regmap, CS42L42_MIC_DET_CTL1, CS42L42_MIC_DET_CTL1);
+	if (ret)
+		goto err_sync;
+
+	ret = regcache_sync(cs42l42->regmap);
+	if (ret)
+		goto err_sync;
 
 	return 0;
+
+err_sync:
+	regcache_cache_only(cs42l42->regmap, true);
+	regcache_mark_dirty(cs42l42->regmap);
+	return ret;
 }
 
 static int cs42l42_sdw_resume(struct device *dev)
diff --git a/sound/soc/codecs/cs42l42.c b/sound/soc/codecs/cs42l42.c
index 2652a639a79ad..fadf688606014 100644
--- a/sound/soc/codecs/cs42l42.c
+++ b/sound/soc/codecs/cs42l42.c
@@ -9,6 +9,7 @@
  * Author: Michael White <michael.white@cirrus.com>
  */
 
+#include <linux/cleanup.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/types.h>
@@ -565,7 +566,7 @@ static int cs42l42_set_jack(struct snd_soc_component *component, struct snd_soc_
 	struct cs42l42_private *cs42l42 = snd_soc_component_get_drvdata(component);
 
 	/* Prevent race with interrupt handler */
-	mutex_lock(&cs42l42->irq_lock);
+	guard(mutex)(&cs42l42->irq_lock);
 	cs42l42->jack = jk;
 
 	if (jk) {
@@ -581,7 +582,6 @@ static int cs42l42_set_jack(struct snd_soc_component *component, struct snd_soc_
 			break;
 		}
 	}
-	mutex_unlock(&cs42l42->irq_lock);
 
 	return 0;
 }
@@ -1668,13 +1668,10 @@ irqreturn_t cs42l42_irq_thread(int irq, void *data)
 	unsigned int current_button_status;
 	unsigned int i;
 
-	pm_runtime_get_sync(cs42l42->dev);
-	mutex_lock(&cs42l42->irq_lock);
-	if (cs42l42->suspended || !cs42l42->init_done) {
-		mutex_unlock(&cs42l42->irq_lock);
-		pm_runtime_put_autosuspend(cs42l42->dev);
+	guard(pm_runtime_active_auto)(cs42l42->dev);
+	guard(mutex)(&cs42l42->irq_lock);
+	if (cs42l42->suspended || !cs42l42->init_done)
 		return IRQ_NONE;
-	}
 
 	/* Read sticky registers to clear interurpt */
 	for (i = 0; i < ARRAY_SIZE(stickies); i++) {
@@ -1774,9 +1771,6 @@ irqreturn_t cs42l42_irq_thread(int irq, void *data)
 		}
 	}
 
-	mutex_unlock(&cs42l42->irq_lock);
-	pm_runtime_put_autosuspend(cs42l42->dev);
-
 	return IRQ_HANDLED;
 }
 EXPORT_SYMBOL_NS_GPL(cs42l42_irq_thread, "SND_SOC_CS42L42_CORE");
@@ -2163,22 +2157,22 @@ int cs42l42_suspend(struct device *dev)
 	 * future interrupts. This ensures a safe disable if the interrupt
 	 * is shared.
 	 */
-	mutex_lock(&cs42l42->irq_lock);
-	cs42l42->suspended = true;
+	scoped_guard(mutex, &cs42l42->irq_lock) {
+		cs42l42->suspended = true;
 
-	/* Save register values that will be overwritten by shutdown sequence */
-	for (i = 0; i < ARRAY_SIZE(cs42l42_shutdown_seq); ++i) {
-		regmap_read(cs42l42->regmap, cs42l42_shutdown_seq[i].reg, &reg);
-		save_regs[i] = (u8)reg;
-	}
+		/* Save register values that will be overwritten by shutdown sequence */
+		for (i = 0; i < ARRAY_SIZE(cs42l42_shutdown_seq); ++i) {
+			regmap_read(cs42l42->regmap, cs42l42_shutdown_seq[i].reg, &reg);
+			save_regs[i] = (u8)reg;
+		}
 
-	/* Shutdown codec */
-	regmap_multi_reg_write(cs42l42->regmap,
-			       cs42l42_shutdown_seq,
-			       ARRAY_SIZE(cs42l42_shutdown_seq));
+		/* Shutdown codec */
+		regmap_multi_reg_write(cs42l42->regmap,
+				       cs42l42_shutdown_seq,
+				       ARRAY_SIZE(cs42l42_shutdown_seq));
 
-	/* All interrupt sources are now disabled */
-	mutex_unlock(&cs42l42->irq_lock);
+		/* All interrupt sources are now disabled */
+	}
 
 	/* Wait for power-down complete */
 	msleep(CS42L42_PDN_DONE_TIME_MS);
@@ -2250,13 +2244,13 @@ void cs42l42_resume_restore(struct device *dev)
 	regcache_cache_only(cs42l42->regmap, false);
 	regcache_mark_dirty(cs42l42->regmap);
 
-	mutex_lock(&cs42l42->irq_lock);
-	/* Sync LATCH_TO_VP first so the VP domain registers sync correctly */
-	regcache_sync_region(cs42l42->regmap, CS42L42_MIC_DET_CTL1, CS42L42_MIC_DET_CTL1);
-	regcache_sync(cs42l42->regmap);
+	scoped_guard(mutex, &cs42l42->irq_lock) {
+		/* Sync LATCH_TO_VP first so the VP domain registers sync correctly */
+		regcache_sync_region(cs42l42->regmap, CS42L42_MIC_DET_CTL1, CS42L42_MIC_DET_CTL1);
+		regcache_sync(cs42l42->regmap);
 
-	cs42l42->suspended = false;
-	mutex_unlock(&cs42l42->irq_lock);
+		cs42l42->suspended = false;
+	}
 
 	dev_dbg(dev, "System resumed\n");
 }
diff --git a/sound/soc/codecs/cs42l43-jack.c b/sound/soc/codecs/cs42l43-jack.c
index 934666295ee30..a2555dd9229e3 100644
--- a/sound/soc/codecs/cs42l43-jack.c
+++ b/sound/soc/codecs/cs42l43-jack.c
@@ -6,6 +6,7 @@
 //                         Cirrus Logic International Semiconductor Ltd.
 
 #include <linux/build_bug.h>
+#include <linux/cleanup.h>
 #include <linux/completion.h>
 #include <linux/delay.h>
 #include <linux/errno.h>
@@ -67,6 +68,21 @@ static int cs42l43_find_index(struct cs42l43_codec *priv, const char * const pro
 	return -EINVAL;
 }
 
+static void cs42l43_apply_accdet_config(struct cs42l43_codec *priv,
+					unsigned int autocontrol,
+					unsigned int pdncntl)
+{
+	struct cs42l43 *cs42l43 = priv->core;
+
+	regmap_update_bits(cs42l43->regmap, CS42L43_HS_BIAS_SENSE_AND_CLAMP_AUTOCONTROL,
+			   CS42L43_JACKDET_MODE_MASK | CS42L43_S0_AUTO_ADCMUTE_DISABLE_MASK |
+			   CS42L43_HSBIAS_SENSE_TRIP_MASK, autocontrol);
+	regmap_update_bits(cs42l43->regmap, CS42L43_PDNCNTL,
+			   CS42L43_RING_SENSE_EN_MASK, pdncntl);
+
+	dev_dbg(priv->dev, "Successfully configured accessory detect\n");
+}
+
 int cs42l43_set_jack(struct snd_soc_component *component,
 		     struct snd_soc_jack *jack, void *d)
 {
@@ -80,31 +96,34 @@ int cs42l43_set_jack(struct snd_soc_component *component,
 
 	dev_dbg(priv->dev, "Configure accessory detect\n");
 
-	ret = pm_runtime_resume_and_get(priv->dev);
+	PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(priv->dev, pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
 	if (ret) {
 		dev_err(priv->dev, "Failed to resume for jack config: %d\n", ret);
 		return ret;
 	}
 
-	mutex_lock(&priv->jack_lock);
+	guard(mutex)(&priv->jack_lock);
 
 	priv->jack_hp = jack;
 
-	if (!jack)
-		goto done;
+	if (!jack) {
+		cs42l43_apply_accdet_config(priv, autocontrol, pdncntl);
+		return 0;
+	}
 
 	ret = device_property_count_u32(cs42l43->dev, "cirrus,buttons-ohms");
 	if (ret != -EINVAL) {
 		if (ret < 0) {
 			dev_err(priv->dev, "Property cirrus,buttons-ohms malformed: %d\n",
 				ret);
-			goto error;
+			return ret;
 		}
 
 		if (ret > CS42L43_N_BUTTONS) {
 			ret = -EINVAL;
 			dev_err(priv->dev, "Property cirrus,buttons-ohms too many entries\n");
-			goto error;
+			return ret;
 		}
 
 		ret = device_property_read_u32_array(cs42l43->dev, "cirrus,buttons-ohms",
@@ -112,7 +131,7 @@ int cs42l43_set_jack(struct snd_soc_component *component,
 		if (ret < 0) {
 			dev_err(priv->dev, "Property cirrus,button-ohms malformed: %d\n",
 				ret);
-			goto error;
+			return ret;
 		}
 	} else {
 		priv->buttons[0] = 70;
@@ -124,7 +143,7 @@ int cs42l43_set_jack(struct snd_soc_component *component,
 	ret = cs42l43_find_index(priv, "cirrus,detect-us", 50000, &priv->detect_us,
 				 cs42l43_accdet_us, ARRAY_SIZE(cs42l43_accdet_us));
 	if (ret < 0)
-		goto error;
+		return ret;
 
 	hs2 |= ret << CS42L43_AUTO_HSDET_TIME_SHIFT;
 
@@ -134,7 +153,7 @@ int cs42l43_set_jack(struct snd_soc_component *component,
 				 &priv->bias_ramp_ms, cs42l43_accdet_ramp_ms,
 				 ARRAY_SIZE(cs42l43_accdet_ramp_ms));
 	if (ret < 0)
-		goto error;
+		return ret;
 
 	hs2 |= ret << CS42L43_HSBIAS_RAMP_SHIFT;
 
@@ -142,7 +161,7 @@ int cs42l43_set_jack(struct snd_soc_component *component,
 				 &priv->bias_sense_ua, cs42l43_accdet_bias_sense,
 				 ARRAY_SIZE(cs42l43_accdet_bias_sense));
 	if (ret < 0)
-		goto error;
+		return ret;
 
 	if (priv->bias_sense_ua)
 		autocontrol |= ret << CS42L43_HSBIAS_SENSE_TRIP_SHIFT;
@@ -154,7 +173,7 @@ int cs42l43_set_jack(struct snd_soc_component *component,
 				       &priv->tip_debounce_ms);
 	if (ret < 0 && ret != -EINVAL) {
 		dev_err(priv->dev, "Property cirrus,tip-debounce-ms malformed: %d\n", ret);
-		goto error;
+		return ret;
 	}
 
 	/* This tip sense invert is set normally, as TIPSENSE_INV already inverted */
@@ -170,7 +189,7 @@ int cs42l43_set_jack(struct snd_soc_component *component,
 				 &priv->tip_fall_db_ms, cs42l43_accdet_db_ms,
 				 ARRAY_SIZE(cs42l43_accdet_db_ms));
 	if (ret < 0)
-		goto error;
+		return ret;
 
 	tip_deb |= ret << CS42L43_TIPSENSE_FALLING_DB_TIME_SHIFT;
 
@@ -178,7 +197,7 @@ int cs42l43_set_jack(struct snd_soc_component *component,
 				 &priv->tip_rise_db_ms, cs42l43_accdet_db_ms,
 				 ARRAY_SIZE(cs42l43_accdet_db_ms));
 	if (ret < 0)
-		goto error;
+		return ret;
 
 	tip_deb |= ret << CS42L43_TIPSENSE_RISING_DB_TIME_SHIFT;
 
@@ -199,7 +218,7 @@ int cs42l43_set_jack(struct snd_soc_component *component,
 					 NULL, cs42l43_accdet_db_ms,
 					 ARRAY_SIZE(cs42l43_accdet_db_ms));
 		if (ret < 0)
-			goto error;
+			return ret;
 
 		ring_deb |= ret << CS42L43_RINGSENSE_FALLING_DB_TIME_SHIFT;
 
@@ -207,7 +226,7 @@ int cs42l43_set_jack(struct snd_soc_component *component,
 					 NULL, cs42l43_accdet_db_ms,
 					 ARRAY_SIZE(cs42l43_accdet_db_ms));
 		if (ret < 0)
-			goto error;
+			return ret;
 
 		ring_deb |= ret << CS42L43_RINGSENSE_RISING_DB_TIME_SHIFT;
 		pdncntl |= CS42L43_RING_SENSE_EN_MASK;
@@ -228,23 +247,9 @@ int cs42l43_set_jack(struct snd_soc_component *component,
 			   CS42L43_HSBIAS_RAMP_MASK | CS42L43_HSDET_MODE_MASK |
 			   CS42L43_AUTO_HSDET_TIME_MASK, hs2);
 
-done:
-	ret = 0;
-
-	regmap_update_bits(cs42l43->regmap, CS42L43_HS_BIAS_SENSE_AND_CLAMP_AUTOCONTROL,
-			   CS42L43_JACKDET_MODE_MASK | CS42L43_S0_AUTO_ADCMUTE_DISABLE_MASK |
-			   CS42L43_HSBIAS_SENSE_TRIP_MASK, autocontrol);
-	regmap_update_bits(cs42l43->regmap, CS42L43_PDNCNTL,
-			   CS42L43_RING_SENSE_EN_MASK, pdncntl);
-
-	dev_dbg(priv->dev, "Successfully configured accessory detect\n");
+	cs42l43_apply_accdet_config(priv, autocontrol, pdncntl);
 
-error:
-	mutex_unlock(&priv->jack_lock);
-
-	pm_runtime_put_autosuspend(priv->dev);
-
-	return ret;
+	return 0;
 }
 
 static void cs42l43_start_hs_bias(struct cs42l43_codec *priv, bool type_detect)
@@ -310,6 +315,7 @@ irqreturn_t cs42l43_bias_detect_clamp(int irq, void *data)
 #define CS42L43_JACK_ABSENT 0x0
 
 #define CS42L43_JACK_OPTICAL (SND_JACK_MECHANICAL | SND_JACK_AVOUT)
+#define CS42L43_JACK_MICROPHONE (SND_JACK_MECHANICAL | SND_JACK_MICROPHONE)
 #define CS42L43_JACK_HEADPHONE (SND_JACK_MECHANICAL | SND_JACK_HEADPHONE)
 #define CS42L43_JACK_HEADSET (SND_JACK_MECHANICAL | SND_JACK_HEADSET)
 #define CS42L43_JACK_LINEOUT (SND_JACK_MECHANICAL | SND_JACK_LINEOUT)
@@ -369,22 +375,22 @@ irqreturn_t cs42l43_button_press(int irq, void *data)
 {
 	struct cs42l43_codec *priv = data;
 	struct cs42l43 *cs42l43 = priv->core;
-	irqreturn_t iret = IRQ_NONE;
 	unsigned int buttons = 0;
 	unsigned int val = 0;
 	int i, ret;
 
-	ret = pm_runtime_resume_and_get(priv->dev);
+	PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(priv->dev, pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
 	if (ret) {
 		dev_err(priv->dev, "Failed to resume for button press: %d\n", ret);
-		return iret;
+		return IRQ_NONE;
 	}
 
-	mutex_lock(&priv->jack_lock);
+	guard(mutex)(&priv->jack_lock);
 
 	if (!priv->button_detect_running) {
 		dev_dbg(priv->dev, "Spurious button press IRQ\n");
-		goto error;
+		return IRQ_NONE;
 	}
 
 	// Wait for 2 full cycles of comb filter to ensure good reading
@@ -395,12 +401,12 @@ irqreturn_t cs42l43_button_press(int irq, void *data)
 	/* Bail if jack removed, the button is irrelevant and likely invalid */
 	if (!cs42l43_jack_present(priv)) {
 		dev_dbg(priv->dev, "Button ignored due to removal\n");
-		goto error;
+		return IRQ_NONE;
 	}
 
 	if (val & CS42L43_HSBIAS_CLAMP_STS_MASK) {
 		dev_dbg(priv->dev, "Button ignored due to bias sense\n");
-		goto error;
+		return IRQ_NONE;
 	}
 
 	val = (val & CS42L43_HSDET_DC_STS_MASK) >> CS42L43_HSDET_DC_STS_SHIFT;
@@ -423,45 +429,32 @@ irqreturn_t cs42l43_button_press(int irq, void *data)
 
 	snd_soc_jack_report(priv->jack_hp, buttons, CS42L43_JACK_BUTTONS);
 
-	iret = IRQ_HANDLED;
-
-error:
-	mutex_unlock(&priv->jack_lock);
-
-	pm_runtime_put_autosuspend(priv->dev);
-
-	return iret;
+	return IRQ_HANDLED;
 }
 
 irqreturn_t cs42l43_button_release(int irq, void *data)
 {
 	struct cs42l43_codec *priv = data;
-	irqreturn_t iret = IRQ_NONE;
 	int ret;
 
-	ret = pm_runtime_resume_and_get(priv->dev);
+	PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(priv->dev, pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
 	if (ret) {
 		dev_err(priv->dev, "Failed to resume for button release: %d\n", ret);
-		return iret;
+		return IRQ_NONE;
 	}
 
-	mutex_lock(&priv->jack_lock);
-
-	if (priv->button_detect_running) {
-		dev_dbg(priv->dev, "Button release IRQ\n");
+	guard(mutex)(&priv->jack_lock);
 
-		snd_soc_jack_report(priv->jack_hp, 0, CS42L43_JACK_BUTTONS);
-
-		iret = IRQ_HANDLED;
-	} else {
+	if (!priv->button_detect_running) {
 		dev_dbg(priv->dev, "Spurious button release IRQ\n");
+		return IRQ_NONE;
 	}
 
-	mutex_unlock(&priv->jack_lock);
-
-	pm_runtime_put_autosuspend(priv->dev);
+	dev_dbg(priv->dev, "Button release IRQ\n");
+	snd_soc_jack_report(priv->jack_hp, 0, CS42L43_JACK_BUTTONS);
 
-	return iret;
+	return IRQ_HANDLED;
 }
 
 void cs42l43_bias_sense_timeout(struct work_struct *work)
@@ -471,13 +464,14 @@ void cs42l43_bias_sense_timeout(struct work_struct *work)
 	struct cs42l43 *cs42l43 = priv->core;
 	int ret;
 
-	ret = pm_runtime_resume_and_get(priv->dev);
+	PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(priv->dev, pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
 	if (ret) {
 		dev_err(priv->dev, "Failed to resume for bias sense: %d\n", ret);
 		return;
 	}
 
-	mutex_lock(&priv->jack_lock);
+	guard(mutex)(&priv->jack_lock);
 
 	if (cs42l43_jack_present(priv) && priv->button_detect_running) {
 		dev_dbg(priv->dev, "Bias sense timeout out, restore bias\n");
@@ -490,10 +484,6 @@ void cs42l43_bias_sense_timeout(struct work_struct *work)
 				   CS42L43_AUTO_HSBIAS_CLAMP_EN_MASK,
 				   CS42L43_AUTO_HSBIAS_CLAMP_EN_MASK);
 	}
-
-	mutex_unlock(&priv->jack_lock);
-
-	pm_runtime_put_autosuspend(priv->dev);
 }
 
 static const struct reg_sequence cs42l43_3pole_patch[] = {
@@ -871,7 +861,7 @@ static const struct cs42l43_jack_override_mode {
 		.hsdet_mode = CS42L43_JACK_3_POLE_SWITCHES,
 		.mic_ctrl = (0x3 << CS42L43_JACK_STEREO_CONFIG_SHIFT) |
 			    CS42L43_HS1_BIAS_EN_MASK | CS42L43_HS2_BIAS_EN_MASK,
-		.report = CS42L43_JACK_LINEIN,
+		.report = CS42L43_JACK_MICROPHONE,
 	},
 	[CS42L43_JACK_RAW_OPTICAL] = {
 		.hsdet_mode = CS42L43_JACK_3_POLE_SWITCHES,
@@ -895,9 +885,8 @@ int cs42l43_jack_get(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *u
 	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
 	struct cs42l43_codec *priv = snd_soc_component_get_drvdata(component);
 
-	mutex_lock(&priv->jack_lock);
+	guard(mutex)(&priv->jack_lock);
 	ucontrol->value.integer.value[0] = priv->jack_override;
-	mutex_unlock(&priv->jack_lock);
 
 	return 0;
 }
@@ -913,17 +902,13 @@ int cs42l43_jack_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *u
 	if (override >= e->items)
 		return -EINVAL;
 
-	mutex_lock(&priv->jack_lock);
+	guard(mutex)(&priv->jack_lock);
 
-	if (!cs42l43_jack_present(priv)) {
-		mutex_unlock(&priv->jack_lock);
+	if (!cs42l43_jack_present(priv))
 		return -EBUSY;
-	}
 
-	if (override == priv->jack_override) {
-		mutex_unlock(&priv->jack_lock);
+	if (override == priv->jack_override)
 		return 0;
-	}
 
 	priv->jack_override = override;
 
@@ -983,7 +968,5 @@ int cs42l43_jack_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *u
 				    cs42l43_jack_override_modes[override].report);
 	}
 
-	mutex_unlock(&priv->jack_lock);
-
 	return 1;
 }
diff --git a/sound/soc/codecs/cs42l43.c b/sound/soc/codecs/cs42l43.c
index 1d133577702ec..47308cd870fb5 100644
--- a/sound/soc/codecs/cs42l43.c
+++ b/sound/soc/codecs/cs42l43.c
@@ -8,6 +8,7 @@
 #include <linux/bitops.h>
 #include <linux/bits.h>
 #include <linux/build_bug.h>
+#include <linux/cleanup.h>
 #include <linux/clk.h>
 #include <linux/device.h>
 #include <linux/err.h>
@@ -1207,14 +1208,12 @@ static void cs42l43_spk_vu_sync(struct cs42l43_codec *priv)
 {
 	struct cs42l43 *cs42l43 = priv->core;
 
-	mutex_lock(&priv->spk_vu_lock);
+	guard(mutex)(&priv->spk_vu_lock);
 
 	regmap_update_bits(cs42l43->regmap, CS42L43_INTP_VOLUME_CTRL1,
 			   CS42L43_AMP1_2_VU_MASK, CS42L43_AMP1_2_VU_MASK);
 	regmap_update_bits(cs42l43->regmap, CS42L43_INTP_VOLUME_CTRL1,
 			   CS42L43_AMP1_2_VU_MASK, 0);
-
-	mutex_unlock(&priv->spk_vu_lock);
 }
 
 static int cs42l43_shutter_get(struct cs42l43_codec *priv, unsigned int shift)
@@ -1601,7 +1600,7 @@ static int cs42l43_pll_ev(struct snd_soc_dapm_widget *w,
 	struct cs42l43 *cs42l43 = priv->core;
 	int ret;
 
-	mutex_lock(&cs42l43->pll_lock);
+	guard(mutex)(&cs42l43->pll_lock);
 
 	switch (event) {
 	case SND_SOC_DAPM_PRE_PMU:
@@ -1626,8 +1625,6 @@ static int cs42l43_pll_ev(struct snd_soc_dapm_widget *w,
 		break;
 	}
 
-	mutex_unlock(&cs42l43->pll_lock);
-
 	return ret;
 }
 
@@ -2565,13 +2562,10 @@ static int cs42l43_set_sysclk(struct snd_soc_component *component, int clk_id,
 {
 	struct cs42l43_codec *priv = snd_soc_component_get_drvdata(component);
 	struct cs42l43 *cs42l43 = priv->core;
-	int ret;
 
-	mutex_lock(&cs42l43->pll_lock);
-	ret = cs42l43_set_pll(priv, src, freq);
-	mutex_unlock(&cs42l43->pll_lock);
+	guard(mutex)(&cs42l43->pll_lock);
 
-	return ret;
+	return cs42l43_set_pll(priv, src, freq);
 }
 
 static int cs42l43_component_probe(struct snd_soc_component *component)
diff --git a/sound/soc/codecs/cs42l84.c b/sound/soc/codecs/cs42l84.c
index f2a58163de0ed..36533a13319d4 100644
--- a/sound/soc/codecs/cs42l84.c
+++ b/sound/soc/codecs/cs42l84.c
@@ -10,6 +10,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/bits.h>
+#include <linux/cleanup.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/kernel.h>
@@ -281,10 +282,9 @@ static int cs42l84_set_jack(struct snd_soc_component *component, struct snd_soc_
 	struct cs42l84_private *cs42l84 = snd_soc_component_get_drvdata(component);
 
 	/* Prevent race with interrupt handler */
-	mutex_lock(&cs42l84->irq_lock);
+	guard(mutex)(&cs42l84->irq_lock);
 	cs42l84->jack = jk;
 	snd_soc_jack_report(jk, cs42l84->hs_type, SND_JACK_HEADSET);
-	mutex_unlock(&cs42l84->irq_lock);
 
 	return 0;
 }
@@ -831,7 +831,7 @@ static irqreturn_t cs42l84_irq_thread(int irq, void *data)
 	u8 current_ring_state;
 	int i;
 
-	mutex_lock(&cs42l84->irq_lock);
+	guard(mutex)(&cs42l84->irq_lock);
 	/* Read sticky registers to clear interrupt */
 	for (i = 0; i < ARRAY_SIZE(stickies); i++) {
 		regmap_read(cs42l84->regmap, irq_params_table[i].status_addr,
@@ -902,8 +902,6 @@ static irqreturn_t cs42l84_irq_thread(int irq, void *data)
 				break;
 			}
 
-			mutex_unlock(&cs42l84->irq_lock);
-
 			return IRQ_HANDLED;
 		}
 
@@ -919,8 +917,6 @@ static irqreturn_t cs42l84_irq_thread(int irq, void *data)
 		}
 	}
 
-	mutex_unlock(&cs42l84->irq_lock);
-
 	return IRQ_HANDLED;
 }
 
diff --git a/sound/soc/codecs/cs43130.c b/sound/soc/codecs/cs43130.c
index e7b06f962790d..48d88dad4ceb0 100644
--- a/sound/soc/codecs/cs43130.c
+++ b/sound/soc/codecs/cs43130.c
@@ -6,6 +6,7 @@
  *
  * Authors: Li Xu <li.xu@cirrus.com>
  */
+#include <linux/cleanup.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/kernel.h>
@@ -818,26 +819,26 @@ static int cs43130_dsd_hw_params(struct snd_pcm_substream *substream,
 	unsigned int required_clk;
 	u8 dsd_speed;
 
-	mutex_lock(&cs43130->clk_mutex);
-	if (!cs43130->clk_req) {
-		/* no DAI is currently using clk */
-		if (!(CS43130_MCLK_22M % params_rate(params)))
-			required_clk = CS43130_MCLK_22M;
-		else
-			required_clk = CS43130_MCLK_24M;
+	scoped_guard(mutex, &cs43130->clk_mutex) {
+		if (!cs43130->clk_req) {
+			/* no DAI is currently using clk */
+			if (!(CS43130_MCLK_22M % params_rate(params)))
+				required_clk = CS43130_MCLK_22M;
+			else
+				required_clk = CS43130_MCLK_24M;
+
+			cs43130_set_pll(component, 0, 0, cs43130->mclk, required_clk);
+			if (cs43130->pll_bypass)
+				cs43130_change_clksrc(component, CS43130_MCLK_SRC_EXT);
+			else
+				cs43130_change_clksrc(component, CS43130_MCLK_SRC_PLL);
+		}
 
-		cs43130_set_pll(component, 0, 0, cs43130->mclk, required_clk);
-		if (cs43130->pll_bypass)
-			cs43130_change_clksrc(component, CS43130_MCLK_SRC_EXT);
-		else
-			cs43130_change_clksrc(component, CS43130_MCLK_SRC_PLL);
+		cs43130->clk_req++;
+		if (cs43130->clk_req == 2)
+			cs43130_pcm_dsd_mix(true, cs43130->regmap);
 	}
 
-	cs43130->clk_req++;
-	if (cs43130->clk_req == 2)
-		cs43130_pcm_dsd_mix(true, cs43130->regmap);
-	mutex_unlock(&cs43130->clk_mutex);
-
 	switch (params_rate(params)) {
 	case 176400:
 		dsd_speed = 0;
@@ -881,26 +882,26 @@ static int cs43130_hw_params(struct snd_pcm_substream *substream,
 	unsigned int required_clk;
 	u8 dsd_speed;
 
-	mutex_lock(&cs43130->clk_mutex);
-	if (!cs43130->clk_req) {
-		/* no DAI is currently using clk */
-		if (!(CS43130_MCLK_22M % params_rate(params)))
-			required_clk = CS43130_MCLK_22M;
-		else
-			required_clk = CS43130_MCLK_24M;
+	scoped_guard(mutex, &cs43130->clk_mutex) {
+		if (!cs43130->clk_req) {
+			/* no DAI is currently using clk */
+			if (!(CS43130_MCLK_22M % params_rate(params)))
+				required_clk = CS43130_MCLK_22M;
+			else
+				required_clk = CS43130_MCLK_24M;
+
+			cs43130_set_pll(component, 0, 0, cs43130->mclk, required_clk);
+			if (cs43130->pll_bypass)
+				cs43130_change_clksrc(component, CS43130_MCLK_SRC_EXT);
+			else
+				cs43130_change_clksrc(component, CS43130_MCLK_SRC_PLL);
+		}
 
-		cs43130_set_pll(component, 0, 0, cs43130->mclk, required_clk);
-		if (cs43130->pll_bypass)
-			cs43130_change_clksrc(component, CS43130_MCLK_SRC_EXT);
-		else
-			cs43130_change_clksrc(component, CS43130_MCLK_SRC_PLL);
+		cs43130->clk_req++;
+		if (cs43130->clk_req == 2)
+			cs43130_pcm_dsd_mix(true, cs43130->regmap);
 	}
 
-	cs43130->clk_req++;
-	if (cs43130->clk_req == 2)
-		cs43130_pcm_dsd_mix(true, cs43130->regmap);
-	mutex_unlock(&cs43130->clk_mutex);
-
 	switch (dai->id) {
 	case CS43130_ASP_DOP_DAI:
 	case CS43130_XSP_DOP_DAI:
@@ -988,14 +989,13 @@ static int cs43130_hw_free(struct snd_pcm_substream *substream,
 	struct snd_soc_component *component = dai->component;
 	struct cs43130_private *cs43130 = snd_soc_component_get_drvdata(component);
 
-	mutex_lock(&cs43130->clk_mutex);
+	guard(mutex)(&cs43130->clk_mutex);
 	cs43130->clk_req--;
 	if (!cs43130->clk_req) {
 		/* no DAI is currently using clk */
 		cs43130_change_clksrc(component, CS43130_MCLK_SRC_RCO);
 		cs43130_pcm_dsd_mix(false, cs43130->regmap);
 	}
-	mutex_unlock(&cs43130->clk_mutex);
 
 	return 0;
 }
diff --git a/sound/soc/codecs/cs4349.c b/sound/soc/codecs/cs4349.c
index 6ac6d306b0541..9a7f701757174 100644
--- a/sound/soc/codecs/cs4349.c
+++ b/sound/soc/codecs/cs4349.c
@@ -340,7 +340,13 @@ static int cs4349_runtime_resume(struct device *dev)
 	gpiod_set_value_cansleep(cs4349->reset_gpio, 1);
 
 	regcache_cache_only(cs4349->regmap, false);
-	regcache_sync(cs4349->regmap);
+	ret = regcache_sync(cs4349->regmap);
+	if (ret) {
+		regcache_cache_only(cs4349->regmap, true);
+		regcache_mark_dirty(cs4349->regmap);
+		gpiod_set_value_cansleep(cs4349->reset_gpio, 0);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/cs47l15.c b/sound/soc/codecs/cs47l15.c
index da64e0a1db28e..2f30dd8347095 100644
--- a/sound/soc/codecs/cs47l15.c
+++ b/sound/soc/codecs/cs47l15.c
@@ -6,6 +6,7 @@
 //                         Cirrus Logic International Semiconductor Ltd.
 //
 
+#include <linux/cleanup.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/device.h>
@@ -1285,9 +1286,8 @@ static int cs47l15_component_probe(struct snd_soc_component *component)
 
 	snd_soc_component_init_regmap(component, madera->regmap);
 
-	mutex_lock(&madera->dapm_ptr_lock);
-	madera->dapm = snd_soc_component_to_dapm(component);
-	mutex_unlock(&madera->dapm_ptr_lock);
+	scoped_guard(mutex, &madera->dapm_ptr_lock)
+		madera->dapm = snd_soc_component_to_dapm(component);
 
 	ret = madera_init_inputs(component);
 	if (ret)
@@ -1317,9 +1317,8 @@ static void cs47l15_component_remove(struct snd_soc_component *component)
 	struct cs47l15 *cs47l15 = snd_soc_component_get_drvdata(component);
 	struct madera *madera = cs47l15->core.madera;
 
-	mutex_lock(&madera->dapm_ptr_lock);
-	madera->dapm = NULL;
-	mutex_unlock(&madera->dapm_ptr_lock);
+	scoped_guard(mutex, &madera->dapm_ptr_lock)
+		madera->dapm = NULL;
 
 	wm_adsp2_component_remove(&cs47l15->core.adsp[0], component);
 }
diff --git a/sound/soc/codecs/cs47l35.c b/sound/soc/codecs/cs47l35.c
index a8fe5a99a8bb5..e77ac7bc84665 100644
--- a/sound/soc/codecs/cs47l35.c
+++ b/sound/soc/codecs/cs47l35.c
@@ -6,6 +6,7 @@
 //                         Cirrus Logic International Semiconductor Ltd.
 //
 
+#include <linux/cleanup.h>
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/init.h>
@@ -1566,9 +1567,8 @@ static int cs47l35_component_probe(struct snd_soc_component *component)
 
 	snd_soc_component_init_regmap(component, madera->regmap);
 
-	mutex_lock(&madera->dapm_ptr_lock);
-	madera->dapm = snd_soc_component_to_dapm(component);
-	mutex_unlock(&madera->dapm_ptr_lock);
+	scoped_guard(mutex, &madera->dapm_ptr_lock)
+		madera->dapm = snd_soc_component_to_dapm(component);
 
 	ret = madera_init_inputs(component);
 	if (ret)
@@ -1600,9 +1600,8 @@ static void cs47l35_component_remove(struct snd_soc_component *component)
 	struct madera *madera = cs47l35->core.madera;
 	int i;
 
-	mutex_lock(&madera->dapm_ptr_lock);
-	madera->dapm = NULL;
-	mutex_unlock(&madera->dapm_ptr_lock);
+	scoped_guard(mutex, &madera->dapm_ptr_lock)
+		madera->dapm = NULL;
 
 	for (i = 0; i < CS47L35_NUM_ADSP; i++)
 		wm_adsp2_component_remove(&cs47l35->core.adsp[i], component);
diff --git a/sound/soc/codecs/cs47l85.c b/sound/soc/codecs/cs47l85.c
index 42fafb0b392c3..9ee60616c04c3 100644
--- a/sound/soc/codecs/cs47l85.c
+++ b/sound/soc/codecs/cs47l85.c
@@ -6,6 +6,7 @@
 //                         Cirrus Logic International Semiconductor Ltd.
 //
 
+#include <linux/cleanup.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/device.h>
@@ -2504,9 +2505,8 @@ static int cs47l85_component_probe(struct snd_soc_component *component)
 
 	snd_soc_component_init_regmap(component, madera->regmap);
 
-	mutex_lock(&madera->dapm_ptr_lock);
-	madera->dapm = snd_soc_component_to_dapm(component);
-	mutex_unlock(&madera->dapm_ptr_lock);
+	scoped_guard(mutex, &madera->dapm_ptr_lock)
+		madera->dapm = snd_soc_component_to_dapm(component);
 
 	ret = madera_init_inputs(component);
 	if (ret)
@@ -2537,9 +2537,8 @@ static void cs47l85_component_remove(struct snd_soc_component *component)
 	struct madera *madera = cs47l85->core.madera;
 	int i;
 
-	mutex_lock(&madera->dapm_ptr_lock);
-	madera->dapm = NULL;
-	mutex_unlock(&madera->dapm_ptr_lock);
+	scoped_guard(mutex, &madera->dapm_ptr_lock)
+		madera->dapm = NULL;
 
 	for (i = 0; i < CS47L85_NUM_ADSP; i++)
 		wm_adsp2_component_remove(&cs47l85->core.adsp[i], component);
diff --git a/sound/soc/codecs/cs47l90.c b/sound/soc/codecs/cs47l90.c
index 77e8aabb241a1..6ce603bfd5004 100644
--- a/sound/soc/codecs/cs47l90.c
+++ b/sound/soc/codecs/cs47l90.c
@@ -6,6 +6,7 @@
 //                         Cirrus Logic International Semiconductor Ltd.
 //
 
+#include <linux/cleanup.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/device.h>
@@ -2423,9 +2424,8 @@ static int cs47l90_component_probe(struct snd_soc_component *component)
 
 	snd_soc_component_init_regmap(component, madera->regmap);
 
-	mutex_lock(&madera->dapm_ptr_lock);
-	madera->dapm = snd_soc_component_to_dapm(component);
-	mutex_unlock(&madera->dapm_ptr_lock);
+	scoped_guard(mutex, &madera->dapm_ptr_lock)
+		madera->dapm = snd_soc_component_to_dapm(component);
 
 	ret = madera_init_inputs(component);
 	if (ret)
@@ -2456,9 +2456,8 @@ static void cs47l90_component_remove(struct snd_soc_component *component)
 	struct madera *madera = cs47l90->core.madera;
 	int i;
 
-	mutex_lock(&madera->dapm_ptr_lock);
-	madera->dapm = NULL;
-	mutex_unlock(&madera->dapm_ptr_lock);
+	scoped_guard(mutex, &madera->dapm_ptr_lock)
+		madera->dapm = NULL;
 
 	for (i = 0; i < CS47L90_NUM_ADSP; i++)
 		wm_adsp2_component_remove(&cs47l90->core.adsp[i], component);
diff --git a/sound/soc/codecs/cs47l92.c b/sound/soc/codecs/cs47l92.c
index 868237bd6d910..07bc303d9797a 100644
--- a/sound/soc/codecs/cs47l92.c
+++ b/sound/soc/codecs/cs47l92.c
@@ -6,6 +6,7 @@
 //                         Cirrus Logic International Semiconductor Ltd.
 //
 
+#include <linux/cleanup.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/device.h>
@@ -1892,9 +1893,8 @@ static int cs47l92_component_probe(struct snd_soc_component *component)
 
 	snd_soc_component_init_regmap(component, madera->regmap);
 
-	mutex_lock(&madera->dapm_ptr_lock);
-	madera->dapm = snd_soc_component_to_dapm(component);
-	mutex_unlock(&madera->dapm_ptr_lock);
+	scoped_guard(mutex, &madera->dapm_ptr_lock)
+		madera->dapm = snd_soc_component_to_dapm(component);
 
 	ret = madera_init_inputs(component);
 	if (ret)
@@ -1922,9 +1922,8 @@ static void cs47l92_component_remove(struct snd_soc_component *component)
 	struct cs47l92 *cs47l92 = snd_soc_component_get_drvdata(component);
 	struct madera *madera = cs47l92->core.madera;
 
-	mutex_lock(&madera->dapm_ptr_lock);
-	madera->dapm = NULL;
-	mutex_unlock(&madera->dapm_ptr_lock);
+	scoped_guard(mutex, &madera->dapm_ptr_lock)
+		madera->dapm = NULL;
 
 	wm_adsp2_component_remove(&cs47l92->core.adsp[0], component);
 }
diff --git a/sound/soc/codecs/cs48l32.c b/sound/soc/codecs/cs48l32.c
index 086ed0f57a85e..a9967771124e5 100644
--- a/sound/soc/codecs/cs48l32.c
+++ b/sound/soc/codecs/cs48l32.c
@@ -8,6 +8,7 @@
 #include <dt-bindings/sound/cs48l32.h>
 #include <linux/array_size.h>
 #include <linux/build_bug.h>
+#include <linux/cleanup.h>
 #include <linux/clk.h>
 #include <linux/container_of.h>
 #include <linux/delay.h>
@@ -236,15 +237,13 @@ static int cs48l32_rate_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_v
 	int ret;
 
 	/* Prevent any mixer mux changes while we do this */
-	mutex_lock(&cs48l32_codec->rate_lock);
+	guard(mutex)(&cs48l32_codec->rate_lock);
 
 	/* The write must be guarded by a number of SYSCLK cycles */
 	cs48l32_spin_sysclk(cs48l32_codec);
 	ret = snd_soc_put_enum_double(kcontrol, ucontrol);
 	cs48l32_spin_sysclk(cs48l32_codec);
 
-	mutex_unlock(&cs48l32_codec->rate_lock);
-
 	return ret;
 }
 
@@ -2242,7 +2241,6 @@ static int cs48l32_dai_set_sysclk(struct snd_soc_dai *dai,
 	struct cs48l32_dai_priv *dai_priv = &cs48l32_codec->dai[dai->id - 1];
 	unsigned int base = dai->driver->base;
 	unsigned int current_asp_rate, target_asp_rate;
-	bool change_rate_domain = false;
 	int ret;
 
 	if (clk_id == dai_priv->clk)
@@ -2284,19 +2282,15 @@ static int cs48l32_dai_set_sysclk(struct snd_soc_dai *dai,
 
 		if ((current_asp_rate & CS48L32_ASP_RATE_MASK) !=
 		    (target_asp_rate & CS48L32_ASP_RATE_MASK)) {
-			change_rate_domain = true;
-
-			mutex_lock(&cs48l32_codec->rate_lock);
 			/* Guard the rate change with SYSCLK cycles */
-			cs48l32_spin_sysclk(cs48l32_codec);
-		}
-
-		snd_soc_component_update_bits(component, base + CS48L32_ASP_CONTROL1,
-					      CS48L32_ASP_RATE_MASK, target_asp_rate);
-
-		if (change_rate_domain) {
-			cs48l32_spin_sysclk(cs48l32_codec);
-			mutex_unlock(&cs48l32_codec->rate_lock);
+			scoped_guard(mutex, &cs48l32_codec->rate_lock) {
+				cs48l32_spin_sysclk(cs48l32_codec);
+				snd_soc_component_update_bits(component,
+							      base + CS48L32_ASP_CONTROL1,
+							      CS48L32_ASP_RATE_MASK,
+							      target_asp_rate);
+				cs48l32_spin_sysclk(cs48l32_codec);
+			}
 		}
 	}
 
diff --git a/sound/soc/codecs/cx2072x.c b/sound/soc/codecs/cx2072x.c
index 83c6cbd40804f..908344ee58920 100644
--- a/sound/soc/codecs/cx2072x.c
+++ b/sound/soc/codecs/cx2072x.c
@@ -9,6 +9,7 @@
 //
 
 #include <linux/acpi.h>
+#include <linux/cleanup.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/gpio.h>
@@ -1408,7 +1409,7 @@ static int cx2072x_jack_status_check(void *data)
 	unsigned int type = 0;
 	int state = 0;
 
-	mutex_lock(&cx2072x->lock);
+	guard(mutex)(&cx2072x->lock);
 
 	regmap_read(cx2072x->regmap, CX2072X_PORTA_PIN_SENSE, &jack);
 	jack = jack >> 24;
@@ -1434,8 +1435,6 @@ static int cx2072x_jack_status_check(void *data)
 	/* clear interrupt */
 	regmap_write(cx2072x->regmap, CX2072X_UM_INTERRUPT_CRTL_E, 0x12 << 24);
 
-	mutex_unlock(&cx2072x->lock);
-
 	dev_dbg(codec->dev, "CX2072X_HSDETECT type=0x%X,Jack state = %x\n",
 		type, state);
 	return state;
diff --git a/sound/soc/codecs/es8328.c b/sound/soc/codecs/es8328.c
index 9838fe42cb6fc..aaa6646ad4c5e 100644
--- a/sound/soc/codecs/es8328.c
+++ b/sound/soc/codecs/es8328.c
@@ -405,6 +405,11 @@ static const struct snd_soc_dapm_route es8328_dapm_routes[] = {
 
 	{ "Mic Bias", NULL, "Mic Bias Gen" },
 
+	{ "LINPUT1", NULL, "Mic Bias" },
+	{ "RINPUT1", NULL, "Mic Bias" },
+	{ "LINPUT2", NULL, "Mic Bias" },
+	{ "RINPUT2", NULL, "Mic Bias" },
+
 	{ "Left Mixer", NULL, "Left DAC" },
 	{ "Left Mixer", "Left Bypass Switch", "Left Line Mux" },
 	{ "Left Mixer", "Right Playback Switch", "Right DAC" },
diff --git a/sound/soc/codecs/es8389.c b/sound/soc/codecs/es8389.c
index 3484c87853cb4..0c7567e2ffc2e 100644
--- a/sound/soc/codecs/es8389.c
+++ b/sound/soc/codecs/es8389.c
@@ -36,6 +36,10 @@ struct	es8389_private {
 	unsigned int sysclk;
 	int mastermode;
 
+	u8 hpfl;
+	u8 hpfr;
+	u32 hpf_freq;
+	u32 capture_rate;
 	u8 mclk_src;
 	u8 vddd;
 	int version;
@@ -50,10 +54,29 @@ static const char * const es8389_core_supplies[] = {
 static bool es8389_volatile_register(struct device *dev,
 			unsigned int reg)
 {
-	if ((reg  <= 0xff))
-		return true;
-	else
+	switch (reg) {
+	case ES8389_ADCL_VOL:
+	case ES8389_ADCR_VOL:
+	case ES8389_MIC1_GAIN:
+	case ES8389_MIC2_GAIN:
+	case ES8389_DACL_VOL:
+	case ES8389_DACR_VOL:
+	case ES8389_ALC_ON:
+	case ES8389_ALC_CTL:
+	case ES8389_ALC_TARGET:
+	case ES8389_ALC_GAIN:
+	case ES8389_ADC_MUTE:
+	case ES8389_OSR_VOL:
+	case ES8389_DAC_INV:
+	case ES8389_MIX_VOL:
+	case ES8389_DAC_MIX:
+	case ES8389_ADC_RESET:
+	case ES8389_ADC_MODE:
+	case ES8389_DMIC_EN:
 		return false;
+	default:
+		return true;
+	}
 }
 
 static const DECLARE_TLV_DB_SCALE(dac_vol_tlv, -9550, 50, 0);
@@ -63,6 +86,92 @@ static const DECLARE_TLV_DB_SCALE(mix_vol_tlv, -9500, 100, 0);
 static const DECLARE_TLV_DB_SCALE(alc_target_tlv, -3200, 200, 0);
 static const DECLARE_TLV_DB_SCALE(alc_max_level, -3200, 200, 0);
 
+static const u32 hpf_table[10][10] = {
+	{1020, 754, 624, 559, 527, 511, 502, 498, 497, 496},
+	{754, 495, 368, 306, 274, 259, 251, 247, 246, 244},
+	{624, 368, 243, 182, 151, 136, 128, 124, 123, 121},
+	{559, 306, 182, 120, 90, 75, 68, 63, 62, 60},
+	{527, 274, 151, 90, 60, 45, 38, 33, 32, 31},
+	{511, 259, 136, 75, 45, 30, 23, 19, 18, 17},
+	{502, 251, 128, 68, 38, 23, 16, 13, 11, 11},
+	{498, 247, 124, 63, 33, 19, 13, 10, 8, 8},
+	{497, 246, 123, 62, 32, 18, 11, 8, 8, 0},
+	{496, 244, 121, 60, 31, 17, 11, 8, 0, 0}
+};
+
+static bool find_best_hpf_freq(u32 target_hz, u8 *hpf1, u8 *hpf2, u32 *out)
+{
+	int best_row = -1, best_col = -1;
+	u32 min_diff = U32_MAX;
+	u32 f, diff;
+	int i, j;
+
+	if ((target_hz > 1020) | (target_hz < 0))
+		return false;
+
+	for (i = 0; i < 10; i++) {
+		for (j = i; j < 10; j++) {
+			f = hpf_table[i][j];
+
+			diff = (target_hz > f) ? (target_hz - f) : (f - target_hz);
+			if (diff < min_diff) {
+				min_diff = diff;
+				best_row = i;
+				best_col = j;
+				*out = f;
+			}
+		}
+	}
+
+	*hpf1 = best_col + ES8389_HPF_OFFSET;
+	*hpf2 = best_row + ES8389_HPF_OFFSET;
+
+	return true;
+}
+
+static int es8389_hpf_get(struct snd_kcontrol *kcontrol,
+	struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct es8389_private *es8389 = snd_soc_component_get_drvdata(component);
+
+	ucontrol->value.integer.value[0] = es8389->hpf_freq;
+	return 0;
+}
+
+static int es8389_hpf_set(struct snd_kcontrol *kcontrol,
+	struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct es8389_private *es8389 = snd_soc_component_get_drvdata(component);
+	u32 freq;
+	bool hpf;
+
+	if (es8389->hpf_freq == ucontrol->value.integer.value[0])
+		return 0;
+
+	if (es8389->capture_rate) {
+		freq = (ucontrol->value.integer.value[0] * 48000) / es8389->capture_rate;
+
+		hpf = find_best_hpf_freq(freq, &es8389->hpfl, &es8389->hpfr, &es8389->hpf_freq);
+		if (!hpf)
+			return -EBUSY;
+
+		if (es8389->hpf_freq != ucontrol->value.integer.value[0])
+			dev_dbg(component->dev, "At the %u Hz sampling rate, %ld Hz could not be obtained."
+				"the frequency has been set to the closest value, %u Hz\n",
+				es8389->capture_rate, ucontrol->value.integer.value[0], es8389->hpf_freq);
+
+		regmap_update_bits(es8389->regmap, ES8389_ADC_HPF1, 0x0f, es8389->hpfl);
+		regmap_update_bits(es8389->regmap, ES8389_ADC_HPF2, 0x0f, es8389->hpfr);
+	} else {
+		es8389->hpf_freq = ucontrol->value.integer.value[0];
+		dev_dbg(component->dev, "PCM_STREAM_CAPTURE is not active.retain the input frequency\n");
+	}
+
+	return 1;
+}
+
 static int es8389_dmic_set(struct snd_kcontrol *kcontrol,
 	struct snd_ctl_elem_value *ucontrol)
 {
@@ -143,6 +252,16 @@ static const struct soc_enum alc_ramprate =
 static const struct soc_enum alc_winsize =
 	SOC_ENUM_SINGLE(ES8389_ALC_CTL, 0, 16, winsize);
 
+static const char *const es8389_adcl_mux_txt[] = {
+	"Normal",
+	"ADC2 channel to ADC1 channel",
+};
+
+static const char *const es8389_adcr_mux_txt[] = {
+	"Normal",
+	"ADC1 channel to ADC2 channel",
+};
+
 static const char *const es8389_outl_mux_txt[] = {
 	"Normal",
 	"DAC2 channel to DAC1 channel",
@@ -170,6 +289,20 @@ static const unsigned int es8389_pga_values[] = {
 	1, 5, 6
 };
 
+static const struct soc_enum es8389_adcl_mux_enum =
+	SOC_ENUM_SINGLE(ES8389_ADC_MODE, 5,
+			ARRAY_SIZE(es8389_adcl_mux_txt), es8389_adcl_mux_txt);
+
+static const struct snd_kcontrol_new es8389_adcl_mux_controls =
+	SOC_DAPM_ENUM("INPUTL MUX", es8389_adcl_mux_enum);
+
+static const struct soc_enum es8389_adcr_mux_enum =
+	SOC_ENUM_SINGLE(ES8389_ADC_MODE, 4,
+			ARRAY_SIZE(es8389_adcr_mux_txt), es8389_adcr_mux_txt);
+
+static const struct snd_kcontrol_new es8389_adcr_mux_controls =
+	SOC_DAPM_ENUM("INPUTR MUX", es8389_adcr_mux_enum);
+
 static const struct soc_enum es8389_outl_mux_enum =
 	SOC_ENUM_SINGLE(ES8389_DAC_MIX, 5,
 			ARRAY_SIZE(es8389_outl_mux_txt), es8389_outl_mux_txt);
@@ -240,6 +373,8 @@ static const struct snd_kcontrol_new es8389_snd_controls[] = {
 	SOC_DOUBLE("ADC OSR Volume ON Switch", ES8389_ADC_MUTE, 6, 7, 1, 0),
 	SOC_SINGLE_TLV("ADC OSR Volume", ES8389_OSR_VOL, 0, 0xFF, 0, adc_vol_tlv),
 	SOC_DOUBLE("ADC OUTPUT Invert Switch", ES8389_ADC_HPF2, 5, 6, 1, 0),
+	SOC_SINGLE_EXT("ADC HPF Freq Select", SND_SOC_NOPM, 0, 1020, 0,
+				es8389_hpf_get, es8389_hpf_set),
 
 	SOC_SINGLE_TLV("DACL Playback Volume", ES8389_DACL_VOL, 0, 0xFF, 0, dac_vol_tlv),
 	SOC_SINGLE_TLV("DACR Playback Volume", ES8389_DACR_VOL, 0, 0xFF, 0, dac_vol_tlv),
@@ -298,6 +433,8 @@ static const struct snd_soc_dapm_widget es8389_dapm_widgets[] = {
 			   &es8389_adc_mixer_controls[0],
 			   ARRAY_SIZE(es8389_adc_mixer_controls)),
 	SND_SOC_DAPM_MUX("ADC MUX", SND_SOC_NOPM, 0, 0, &es8389_dmic_mux_controls),
+	SND_SOC_DAPM_MUX("INPUTL MUX", SND_SOC_NOPM, 0, 0, &es8389_adcl_mux_controls),
+	SND_SOC_DAPM_MUX("INPUTR MUX", SND_SOC_NOPM, 0, 0, &es8389_adcr_mux_controls),
 
 	SND_SOC_DAPM_MUX("OUTL MUX", SND_SOC_NOPM, 0, 0, &es8389_outl_mux_controls),
 	SND_SOC_DAPM_MUX("OUTR MUX", SND_SOC_NOPM, 0, 0, &es8389_outr_mux_controls),
@@ -311,10 +448,15 @@ static const struct snd_soc_dapm_route es8389_dapm_routes[] = {
 	{"ADCL", NULL, "PGAL"},
 	{"ADCR", NULL, "PGAR"},
 
+	{"INPUTL MUX", "Normal", "ADCL"},
+	{"INPUTL MUX", "ADC2 channel to ADC1 channel", "ADCR"},
+	{"INPUTR MUX", "Normal", "ADCR"},
+	{"INPUTR MUX", "ADC1 channel to ADC2 channel", "ADCL"},
+
 	{"ADC Mixer", "DACL ADCL Mixer", "DACL"},
 	{"ADC Mixer", "DACR ADCR Mixer", "DACR"},
-	{"ADC Mixer", NULL, "ADCL"},
-	{"ADC Mixer", NULL, "ADCR"},
+	{"ADC Mixer", NULL, "INPUTL MUX"},
+	{"ADC Mixer", NULL, "INPUTR MUX"},
 
 	{"ADC MUX", "AMIC", "ADC Mixer"},
 	{"ADC MUX", "DMIC", "DMIC"},
@@ -415,52 +557,54 @@ static const struct _coeff_div  coeff_div[] = {
 	{36, 576000, 16000, 0x00, 0x55, 0x84, 0xD0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x23, 0x8F, 0xBF, 0xC0, 0x1F, 0x8F, 0x01, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
 	{48, 768000, 16000, 0x02, 0x57, 0x04, 0xC0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
 	{50, 800000, 16000, 0x00, 0x7E, 0x01, 0xD9, 0x00, 0xC2, 0x80, 0x00, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0xC7, 0x95, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
-	{64, 1024000, 16000, 0x00, 0x45, 0x24, 0xC0, 0x01, 0xD1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
-	{72, 1152000, 16000, 0x00, 0x45, 0x24, 0xC0, 0x01, 0xD1, 0x90, 0x00, 0x00, 0x23, 0x8F, 0xBF, 0xC0, 0x1F, 0x8F, 0x01, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
+	{64, 1024000, 16000, 0x00, 0x45, 0x24, 0xC0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
+	{72, 1152000, 16000, 0x00, 0x45, 0x24, 0xC0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x23, 0x8F, 0xBF, 0xC0, 0x1F, 0x8F, 0x01, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
 	{96, 1536000, 16000, 0x02, 0x55, 0x84, 0xD0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
 	{128, 2048000, 16000, 0x00, 0x51, 0x04, 0xD0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
 	{144, 2304000, 16000, 0x00, 0x51, 0x00, 0xC0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x23, 0x8F, 0xBF, 0xC0, 0x1F, 0x8F, 0x01, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
+	{150, 2400000, 16000, 0x02, 0x7E, 0x01, 0xC9, 0x00, 0xC2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0xC7, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
 	{192, 3072000, 16000, 0x02, 0x65, 0x25, 0xE0, 0x00, 0xE1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
-	{256, 4096000, 16000, 0x00, 0x41, 0x04, 0xC0, 0x01, 0xD1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
+	{256, 4096000, 16000, 0x00, 0x41, 0x04, 0xC0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
 	{300, 4800000, 16000, 0x02, 0x66, 0x01, 0xD9, 0x00, 0xC2, 0x80, 0x00, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0xC7, 0x95, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
 	{384, 6144000, 16000, 0x02, 0x51, 0x04, 0xD0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
-	{512, 8192000, 16000, 0x01, 0x41, 0x04, 0xC0, 0x01, 0xD1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
+	{512, 8192000, 16000, 0x01, 0x41, 0x04, 0xC0, 0x01, 0xC1, 0x90, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
 	{750, 12000000, 16000, 0x0E, 0x7E, 0x01, 0xC9, 0x00, 0xC2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0xC7, 0x95, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
-	{768, 12288000, 16000, 0x02, 0x41, 0x04, 0xC0, 0x01, 0xD1, 0x90, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
-	{1024, 16384000, 16000, 0x03, 0x41, 0x04, 0xC0, 0x01, 0xD1, 0x90, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
+	{768, 12288000, 16000, 0x02, 0x41, 0x04, 0xC0, 0x01, 0xC1, 0x90, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
+	{1024, 16384000, 16000, 0x03, 0x41, 0x04, 0xC0, 0x01, 0xC1, 0x90, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
 	{1152, 18432000, 16000, 0x08, 0x51, 0x04, 0xD0, 0x01, 0xC1, 0x90, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
 	{1200, 19200000, 16000, 0x0B, 0x66, 0x01, 0xD9, 0x00, 0xC2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0xC7, 0x95, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
 	{1500, 24000000, 16000, 0x0E, 0x26, 0x01, 0xD9, 0x00, 0xC2, 0x80, 0xC0, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0xC7, 0x95, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
-	{1536, 24576000, 16000, 0x05, 0x41, 0x04, 0xC0, 0x01, 0xD1, 0x90, 0xC0, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
+	{1536, 24576000, 16000, 0x05, 0x41, 0x04, 0xC0, 0x01, 0xC1, 0x90, 0xC0, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0xFF, 0x7F, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
 	{1625, 26000000, 16000, 0x40, 0x6E, 0x05, 0xC8, 0x01, 0xC2, 0x90, 0xC0, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x12, 0x31, 0x0E, 2, 2},
 	{800, 19200000, 24000, 0x07, 0x66, 0x01, 0xD9, 0x00, 0xC2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0xC7, 0x95, 0x00, 0x12, 0x00, 0x1A, 0x49, 0x14, 2, 2},
 	{375, 12000000, 32000, 0x0E, 0x2E, 0x05, 0xC8, 0x00, 0xC2, 0x80, 0x40, 0x01, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x23, 0x61, 0x1B, 2, 0},
-	{600, 19200000, 32000, 0x05, 0x46, 0x01, 0xD8, 0x10, 0xD2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x23, 0x61, 0x1B, 2, 2},
-	{32, 1411200, 44100, 0x00, 0x45, 0xA4, 0xD0, 0x10, 0xD1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
+	{600, 19200000, 32000, 0x05, 0x46, 0x01, 0xD8, 0x10, 0xC2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x23, 0x61, 0x1B, 2, 2},
+	{32, 1411200, 44100, 0x00, 0x45, 0xA4, 0xD0, 0x10, 0xC1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
 	{64, 2822400, 44100, 0x00, 0x51, 0x00, 0xC0, 0x10, 0xC1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
-	{128, 5644800, 44100, 0x00, 0x41, 0x04, 0xD0, 0x10, 0xD1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
-	{256, 11289600, 44100, 0x01, 0x41, 0x04, 0xD0, 0x10, 0xD1, 0x80, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
-	{512, 22579200, 44100, 0x03, 0x41, 0x04, 0xD0, 0x10, 0xD1, 0x80, 0xC0, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
-	{32, 1536000, 48000, 0x00, 0x45, 0xA4, 0xD0, 0x10, 0xD1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
+	{128, 5644800, 44100, 0x00, 0x41, 0x04, 0xD0, 0x10, 0xC1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
+	{256, 11289600, 44100, 0x01, 0x41, 0x04, 0xD0, 0x10, 0xC1, 0x80, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
+	{512, 22579200, 44100, 0x03, 0x41, 0x04, 0xD0, 0x10, 0xC1, 0x80, 0xC0, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
+	{32, 1536000, 48000, 0x00, 0x45, 0xA4, 0xD0, 0x10, 0xC1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
 	{48, 2304000, 48000, 0x02, 0x55, 0x04, 0xC0, 0x10, 0xC1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
 	{50, 2400000, 48000, 0x00, 0x76, 0x01, 0xC8, 0x10, 0xC2, 0x80, 0x00, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
 	{64, 3072000, 48000, 0x00, 0x51, 0x04, 0xC0, 0x10, 0xC1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
-	{100, 4800000, 48000, 0x00, 0x46, 0x01, 0xD8, 0x10, 0xD2, 0x80, 0x00, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
+	{100, 4800000, 48000, 0x00, 0x46, 0x01, 0xD8, 0x10, 0xC2, 0x80, 0x00, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
 	{125, 6000000, 48000, 0x04, 0x6E, 0x05, 0xC8, 0x10, 0xC2, 0x80, 0x00, 0x01, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
-	{128, 6144000, 48000, 0x00, 0x41, 0x04, 0xD0, 0x10, 0xD1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
-	{200, 9600000, 48000, 0x01, 0x46, 0x01, 0xD8, 0x10, 0xD2, 0x80, 0x00, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
+	{128, 6144000, 48000, 0x00, 0x41, 0x04, 0xD0, 0x10, 0xC1, 0x80, 0x00, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
+	{200, 9600000, 48000, 0x01, 0x46, 0x01, 0xD8, 0x10, 0xC2, 0x80, 0x00, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
 	{250, 12000000, 48000, 0x04, 0x76, 0x01, 0xC8, 0x10, 0xC2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
-	{256, 12288000, 48000, 0x01, 0x41, 0x04, 0xD0, 0x10, 0xD1, 0x80, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
-	{384, 18432000, 48000, 0x02, 0x41, 0x04, 0xD0, 0x10, 0xD1, 0x80, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
-	{400, 19200000, 48000, 0x03, 0x46, 0x01, 0xD8, 0x10, 0xD2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
-	{500, 24000000, 48000, 0x04, 0x46, 0x01, 0xD8, 0x10, 0xD2, 0x80, 0xC0, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
-	{512, 24576000, 48000, 0x03, 0x41, 0x04, 0xD0, 0x10, 0xD1, 0x80, 0xC0, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
+	{256, 12288000, 48000, 0x01, 0x41, 0x04, 0xD0, 0x10, 0xC1, 0x80, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
+	{384, 18432000, 48000, 0x02, 0x41, 0x04, 0xD0, 0x10, 0xC1, 0x80, 0x40, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
+	{400, 19200000, 48000, 0x03, 0x46, 0x01, 0xD8, 0x10, 0xC2, 0x80, 0x40, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
+	{500, 24000000, 48000, 0x04, 0x46, 0x01, 0xD8, 0x10, 0xC2, 0x80, 0xC0, 0x00, 0x18, 0x95, 0xD0, 0xC0, 0x63, 0x95, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
+	{512, 24576000, 48000, 0x03, 0x41, 0x04, 0xD0, 0x10, 0xC1, 0x80, 0xC0, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
 	{800, 38400000, 48000, 0x18, 0x45, 0x04, 0xC0, 0x10, 0xC1, 0x80, 0xC0, 0x00, 0x1F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x00, 0x12, 0x00, 0x35, 0x91, 0x28, 2, 2},
 	{128, 11289600, 88200, 0x00, 0x50, 0x00, 0xC0, 0x10, 0xC1, 0x80, 0x40, 0x00, 0x9F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x80, 0x12, 0xC0, 0x32, 0x89, 0x25, 2, 2},
-	{64, 6144000, 96000, 0x00, 0x41, 0x00, 0xD0, 0x10, 0xD1, 0x80, 0x00, 0x00, 0x9F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x80, 0x12, 0xC0, 0x35, 0x91, 0x28, 2, 2},
+	{64, 6144000, 96000, 0x00, 0x41, 0x00, 0xD0, 0x10, 0xC1, 0x80, 0x00, 0x00, 0x9F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x80, 0x12, 0xC0, 0x35, 0x91, 0x28, 2, 2},
 	{96, 9216000, 96000, 0x02, 0x43, 0x00, 0xC0, 0x10, 0xC0, 0x80, 0x00, 0x00, 0x9F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x80, 0x12, 0xC0, 0x35, 0x91, 0x28, 2, 2},
 	{256, 24576000, 96000, 0x00, 0x40, 0x00, 0xC0, 0x10, 0xC1, 0x80, 0xC0, 0x00, 0x9F, 0x7F, 0xBF, 0xC0, 0x7F, 0x7F, 0x80, 0x12, 0xC0, 0x35, 0x91, 0x28, 2, 2},
 	{128, 24576000, 192000, 0x00, 0x50, 0x00, 0xC0, 0x18, 0xC1, 0x81, 0xC0, 0x00, 0x8F, 0x7F, 0xBF, 0xC0, 0x3F, 0x7F, 0x80, 0x12, 0xC0, 0x3F, 0xF9, 0x3F, 2, 2},
+	{64, 12288000, 192000, 0x00, 0x41, 0x00, 0xC0, 0x18, 0xC1, 0x80, 0x00, 0x00, 0x8F, 0x7F, 0xEF, 0xC0, 0x7F, 0x7F, 0x80, 0x12, 0xC0, 0x3F, 0xF9, 0x3F, 1, 0},
 };
 
 static inline int get_coeff(u8 vddd, u8 dmic, int mclk, int rate)
@@ -564,6 +708,8 @@ static int es8389_pcm_hw_params(struct snd_pcm_substream *substream,
 	int coeff, ret;
 	u8 dmic_enable, state = 0;
 	unsigned int regv;
+	u32 freq;
+	bool hpf;
 
 	switch (params_format(params)) {
 	case SNDRV_PCM_FORMAT_S16_LE:
@@ -590,7 +736,7 @@ static int es8389_pcm_hw_params(struct snd_pcm_substream *substream,
 
 	if (es8389->mclk_src == ES8389_SCLK_PIN) {
 		regmap_update_bits(es8389->regmap, ES8389_MASTER_CLK,
-					ES8389_MCLK_SOURCE, es8389->mclk_src);
+					ES8389_MCLK_MASK, ES8389_MCLK_FROM_SCLK);
 		es8389->sysclk = params_channels(params) * params_width(params) * params_rate(params);
 	}
 
@@ -641,6 +787,28 @@ static int es8389_pcm_hw_params(struct snd_pcm_substream *substream,
 		dev_warn(component->dev, "Clock coefficients do not match");
 	}
 
+	if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) {
+		es8389->capture_rate = params_rate(params);
+		freq = (es8389->hpf_freq * 48000) / params_rate(params);
+		hpf = find_best_hpf_freq(freq, &es8389->hpfl, &es8389->hpfr, &es8389->hpf_freq);
+		if (!hpf) {
+			dev_err(component->dev, "The HPF frequency is invalid\n");
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int es8389_pcm_hw_free(struct snd_pcm_substream *substream,
+		struct snd_soc_dai *dai)
+{
+	struct snd_soc_component *component = dai->component;
+	struct es8389_private *es8389 = snd_soc_component_get_drvdata(component);
+
+	if (substream->stream == SNDRV_PCM_STREAM_CAPTURE)
+		es8389->capture_rate = 0;
+
 	return 0;
 }
 
@@ -721,8 +889,8 @@ static int es8389_mute(struct snd_soc_dai *dai, int mute, int direction)
 			regmap_update_bits(es8389->regmap, ES8389_DAC_FORMAT_MUTE,
 						0x03, 0x00);
 		} else {
-			regmap_update_bits(es8389->regmap, ES8389_ADC_HPF1, 0x0f, 0x0a);
-			regmap_update_bits(es8389->regmap, ES8389_ADC_HPF2, 0x0f, 0x0a);
+			regmap_update_bits(es8389->regmap, ES8389_ADC_HPF1, 0x0f, es8389->hpfl);
+			regmap_update_bits(es8389->regmap, ES8389_ADC_HPF2, 0x0f, es8389->hpfr);
 			regmap_update_bits(es8389->regmap, ES8389_ADC_FORMAT_MUTE,
 						0x03, 0x00);
 		}
@@ -738,6 +906,7 @@ static int es8389_mute(struct snd_soc_dai *dai, int mute, int direction)
 
 static const struct snd_soc_dai_ops es8389_ops = {
 	.hw_params = es8389_pcm_hw_params,
+	.hw_free = es8389_pcm_hw_free,
 	.set_fmt = es8389_set_dai_fmt,
 	.set_sysclk = es8389_set_dai_sysclk,
 	.set_tdm_slot = es8389_set_tdm_slot,
@@ -771,7 +940,7 @@ static void es8389_init(struct snd_soc_component *component)
 
 	regmap_read(es8389->regmap, ES8389_MAX_REGISTER, &reg);
 	es8389->version = reg;
-	regmap_write(es8389->regmap, ES8389_ISO_CTL, 0x00);
+	regmap_write(es8389->regmap, ES8389_ISO_CTL, 0x56);
 	regmap_write(es8389->regmap, ES8389_RESET, 0x7E);
 	regmap_write(es8389->regmap, ES8389_ISO_CTL, 0x38);
 	regmap_write(es8389->regmap, ES8389_ADC_HPF1, 0x64);
@@ -823,7 +992,7 @@ static void es8389_init(struct snd_soc_component *component)
 	regmap_write(es8389->regmap, ES8389_SCLK_DIV, 0x04);
 	regmap_write(es8389->regmap, ES8389_LRCK_DIV1, 0x01);
 	regmap_write(es8389->regmap, ES8389_LRCK_DIV2, 0x00);
-	regmap_write(es8389->regmap, ES8389_OSC_CLK, 0x00);
+	regmap_write(es8389->regmap, ES8389_OSC_CLK, 0x10);
 	regmap_write(es8389->regmap, ES8389_ADC_OSR, 0x1F);
 	regmap_write(es8389->regmap, ES8389_ADC_DSP, 0x7F);
 	regmap_write(es8389->regmap, ES8389_ADC_MUTE, 0xC0);
@@ -861,13 +1030,13 @@ static int es8389_resume(struct snd_soc_component *component)
 	regcache_cache_only(es8389->regmap, false);
 	regcache_cache_bypass(es8389->regmap, true);
 	regmap_read(es8389->regmap, ES8389_RESET, &regv);
-	regcache_cache_bypass(es8389->regmap, false);
 
 	if (regv == 0xff)
 		es8389_init(component);
 	else
 		es8389_set_bias_level(component, SND_SOC_BIAS_ON);
 
+	regcache_cache_bypass(es8389->regmap, false);
 	regcache_sync(es8389->regmap);
 
 	return 0;
@@ -913,6 +1082,7 @@ static int es8389_probe(struct snd_soc_component *component)
 		return ret;
 	}
 
+	es8389->hpf_freq = ES8389_HPF_DEFAULT;
 	es8389_init(component);
 	es8389_set_bias_level(component, SND_SOC_BIAS_STANDBY);
 
diff --git a/sound/soc/codecs/es8389.h b/sound/soc/codecs/es8389.h
index d21e72f876a60..de39e9bf96e9a 100644
--- a/sound/soc/codecs/es8389.h
+++ b/sound/soc/codecs/es8389.h
@@ -106,6 +106,9 @@
 #define ES8389_MIC_SEL_MASK           (7 << 4)
 #define ES8389_MIC_DEFAULT            (1 << 4)
 
+#define ES8389_HPF_DEFAULT            16
+#define ES8389_HPF_OFFSET             4
+
 #define ES8389_MASTER_MODE_EN         (1 << 0)
 
 #define ES8389_TDM_OFF                (0 << 0)
@@ -116,9 +119,11 @@
 #define ES8389_TDM_SLOT               (0x70 << 0)
 #define ES8389_TDM_SHIFT              4
 
-#define ES8389_MCLK_SOURCE            (1 << 6)
-#define ES8389_MCLK_PIN               (1 << 6)
-#define ES8389_SCLK_PIN               (0 << 6)
+#define ES8389_MCLK_MASK              (3 << 6)
+#define ES8389_MCLK_FROM_SCLK         (1 << 6)
+#define ES8389_MCLK_SOURCE            ES8389_MCLK_PIN
+#define ES8389_MCLK_PIN               0
+#define ES8389_SCLK_PIN               1
 
 /* ES8389_FMT */
 #define ES8389_S24_LE                 (0 << 5)
diff --git a/sound/soc/codecs/es9356.c b/sound/soc/codecs/es9356.c
index 1122455aab77f..f574b3d6cb3cf 100644
--- a/sound/soc/codecs/es9356.c
+++ b/sound/soc/codecs/es9356.c
@@ -671,7 +671,7 @@ static int es9356_sdca_mbq_size(struct device *dev, unsigned int reg)
 	}
 }
 
-static struct regmap_sdw_mbq_cfg es9356_mbq_config = {
+static const struct regmap_sdw_mbq_cfg es9356_mbq_config = {
 	.mbq_size = es9356_sdca_mbq_size,
 };
 
diff --git a/sound/soc/codecs/fs210x.c b/sound/soc/codecs/fs210x.c
index 5f381fe063e81..d794ff918b6f0 100644
--- a/sound/soc/codecs/fs210x.c
+++ b/sound/soc/codecs/fs210x.c
@@ -607,6 +607,7 @@ static int fs210x_init_chip(struct fs210x_priv *fs210x)
 
 	regcache_cache_bypass(fs210x->regmap, false);
 	if (!ret) {
+		regcache_cache_only(fs210x->regmap, false);
 		regcache_mark_dirty(fs210x->regmap);
 		regcache_sync(fs210x->regmap);
 		fs210x->is_inited = true;
diff --git a/sound/soc/codecs/hdac_hda.c b/sound/soc/codecs/hdac_hda.c
index 680e341aa7f1b..1ab5f8a26e03b 100644
--- a/sound/soc/codecs/hdac_hda.c
+++ b/sound/soc/codecs/hdac_hda.c
@@ -642,10 +642,8 @@ static int hdac_hda_dev_probe(struct hdac_device *hdev)
 						&hdac_hda_codec, hdac_hda_dais,
 						ARRAY_SIZE(hdac_hda_dais));
 
-	if (ret < 0) {
+	if (ret < 0)
 		dev_err(&hdev->dev, "%s: failed to register HDA codec %d\n", __func__, ret);
-		return ret;
-	}
 
 	snd_hdac_ext_bus_link_put(hdev->bus, hlink);
 
diff --git a/sound/soc/codecs/hdac_hdmi.c b/sound/soc/codecs/hdac_hdmi.c
index 3220f9226e0b2..38073a70fa618 100644
--- a/sound/soc/codecs/hdac_hdmi.c
+++ b/sound/soc/codecs/hdac_hdmi.c
@@ -1866,8 +1866,10 @@ static int hdac_hdmi_dev_probe(struct hdac_device *hdev)
 	snd_hdac_ext_bus_link_get(hdev->bus, hlink);
 
 	hdmi_priv = devm_kzalloc(&hdev->dev, sizeof(*hdmi_priv), GFP_KERNEL);
-	if (hdmi_priv == NULL)
+	if (hdmi_priv == NULL) {
+		snd_hdac_ext_bus_link_put(hdev->bus, hlink);
 		return -ENOMEM;
+	}
 
 	snd_hdac_register_chmap_ops(hdev, &hdmi_priv->chmap);
 	hdmi_priv->chmap.ops.get_chmap = hdac_hdmi_get_chmap;
@@ -1876,8 +1878,10 @@ static int hdac_hdmi_dev_probe(struct hdac_device *hdev)
 	hdmi_priv->chmap.ops.get_spk_alloc = hdac_hdmi_get_spk_alloc;
 	hdmi_priv->hdev = hdev;
 
-	if (!hdac_id)
+	if (!hdac_id) {
+		snd_hdac_ext_bus_link_put(hdev->bus, hlink);
 		return -ENODEV;
+	}
 
 	if (hdac_id->driver_data)
 		hdmi_priv->drv_data =
@@ -1902,6 +1906,8 @@ static int hdac_hdmi_dev_probe(struct hdac_device *hdev)
 	if (ret < 0) {
 		dev_err(&hdev->dev,
 			"Failed in parse and map nid with err: %d\n", ret);
+		snd_hdac_ext_bus_link_put(hdev->bus, hlink);
+		snd_hdac_display_power(hdev->bus, hdev->addr, false);
 		return ret;
 	}
 	snd_hdac_refresh_widgets(hdev);
diff --git a/sound/soc/codecs/lpass-rx-macro.c b/sound/soc/codecs/lpass-rx-macro.c
index 6233aa9f5bc6d..282a6aaa99865 100644
--- a/sound/soc/codecs/lpass-rx-macro.c
+++ b/sound/soc/codecs/lpass-rx-macro.c
@@ -6,6 +6,7 @@
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/platform_device.h>
+#include <linux/pm_clock.h>
 #include <linux/pm_runtime.h>
 #include <linux/clk.h>
 #include <sound/soc.h>
@@ -2030,9 +2031,10 @@ static struct snd_soc_dai_driver rx_macro_dai[] = {
 	},
 };
 
-static void rx_macro_mclk_enable(struct rx_macro *rx, bool mclk_enable)
+static int rx_macro_mclk_enable(struct rx_macro *rx, bool mclk_enable)
 {
 	struct regmap *regmap = rx->regmap;
+	int ret;
 
 	if (mclk_enable) {
 		if (rx->rx_mclk_users == 0) {
@@ -2047,14 +2049,16 @@ static void rx_macro_mclk_enable(struct rx_macro *rx, bool mclk_enable)
 					   CDC_RX_FS_MCLK_CNT_EN_MASK,
 					   CDC_RX_FS_MCLK_CNT_ENABLE);
 			regcache_mark_dirty(regmap);
-			regcache_sync(regmap);
+			ret = regcache_sync(regmap);
+			if (ret)
+				return ret;
 		}
 		rx->rx_mclk_users++;
 	} else {
 		if (rx->rx_mclk_users <= 0) {
 			dev_err(rx->dev, "%s: clock already disabled\n", __func__);
 			rx->rx_mclk_users = 0;
-			return;
+			return 0;
 		}
 		rx->rx_mclk_users--;
 		if (rx->rx_mclk_users == 0) {
@@ -2068,6 +2072,8 @@ static void rx_macro_mclk_enable(struct rx_macro *rx, bool mclk_enable)
 					   CDC_RX_CLK_MCLK2_EN_MASK, 0x0);
 		}
 	}
+
+	return 0;
 }
 
 static int rx_macro_mclk_event(struct snd_soc_dapm_widget *w,
@@ -2079,11 +2085,9 @@ static int rx_macro_mclk_event(struct snd_soc_dapm_widget *w,
 
 	switch (event) {
 	case SND_SOC_DAPM_PRE_PMU:
-		rx_macro_mclk_enable(rx, true);
-		break;
+		return rx_macro_mclk_enable(rx, true);
 	case SND_SOC_DAPM_POST_PMD:
-		rx_macro_mclk_enable(rx, false);
-		break;
+		return rx_macro_mclk_enable(rx, false);
 	default:
 		dev_err(component->dev, "%s: invalid DAPM event %d\n", __func__, event);
 		ret = -EINVAL;
@@ -3671,14 +3675,16 @@ static int swclk_gate_enable(struct clk_hw *hw)
 	struct rx_macro *rx = to_rx_macro(hw);
 	int ret;
 
-	ret = clk_prepare_enable(rx->mclk);
+	ret = pm_runtime_resume_and_get(rx->dev);
+	if (ret < 0)
+		return ret;
+
+	ret = rx_macro_mclk_enable(rx, true);
 	if (ret) {
-		dev_err(rx->dev, "unable to prepare mclk\n");
+		clk_disable_unprepare(rx->mclk);
 		return ret;
 	}
 
-	rx_macro_mclk_enable(rx, true);
-
 	regmap_update_bits(rx->regmap, CDC_RX_CLK_RST_CTRL_SWR_CONTROL,
 			   CDC_RX_SWR_CLK_EN_MASK, 1);
 
@@ -3693,7 +3699,7 @@ static void swclk_gate_disable(struct clk_hw *hw)
 			   CDC_RX_SWR_CLK_EN_MASK, 0);
 
 	rx_macro_mclk_enable(rx, false);
-	clk_disable_unprepare(rx->mclk);
+	pm_runtime_put_autosuspend(rx->dev);
 }
 
 static int swclk_gate_is_enabled(struct clk_hw *hw)
@@ -3864,28 +3870,31 @@ static int rx_macro_probe(struct platform_device *pdev)
 	rx->dev = dev;
 
 	/* set MCLK and NPL rates */
-	clk_set_rate(rx->mclk, MCLK_FREQ);
-	clk_set_rate(rx->npl, MCLK_FREQ);
-
-	ret = clk_prepare_enable(rx->macro);
+	ret = clk_set_rate(rx->mclk, MCLK_FREQ);
 	if (ret)
 		return ret;
 
-	ret = clk_prepare_enable(rx->dcodec);
+	ret = clk_set_rate(rx->npl, MCLK_FREQ);
 	if (ret)
-		goto err_dcodec;
+		return ret;
 
-	ret = clk_prepare_enable(rx->mclk);
+	ret = devm_pm_clk_create(dev);
 	if (ret)
-		goto err_mclk;
+		return ret;
 
-	ret = clk_prepare_enable(rx->npl);
+	ret = of_pm_clk_add_clks(dev);
+	if (ret < 0)
+		return ret;
+
+	pm_runtime_set_autosuspend_delay(dev, 100);
+	pm_runtime_use_autosuspend(dev);
+	ret = devm_pm_runtime_enable(dev);
 	if (ret)
-		goto err_npl;
+		return ret;
 
-	ret = clk_prepare_enable(rx->fsgen);
+	ret = pm_runtime_resume_and_get(dev);
 	if (ret)
-		goto err_fsgen;
+		return ret;
 
 	/* reset swr block  */
 	regmap_update_bits(rx->regmap, CDC_RX_CLK_RST_CTRL_SWR_CONTROL,
@@ -3902,46 +3911,25 @@ static int rx_macro_probe(struct platform_device *pdev)
 					      rx_macro_dai,
 					      ARRAY_SIZE(rx_macro_dai));
 	if (ret)
-		goto err_clkout;
-
-
-	pm_runtime_set_autosuspend_delay(dev, 3000);
-	pm_runtime_use_autosuspend(dev);
-	pm_runtime_mark_last_busy(dev);
-	pm_runtime_set_active(dev);
-	pm_runtime_enable(dev);
+		goto err_rpm_put;
 
 	ret = rx_macro_register_mclk_output(rx);
 	if (ret)
-		goto err_clkout;
+		goto err_rpm_put;
+
+	ret = pm_runtime_put_autosuspend(dev);
+	if (ret < 0)
+		dev_warn(dev, "runtime PM put failed after probe: %d\n", ret);
 
 	return 0;
 
-err_clkout:
-	clk_disable_unprepare(rx->fsgen);
-err_fsgen:
-	clk_disable_unprepare(rx->npl);
-err_npl:
-	clk_disable_unprepare(rx->mclk);
-err_mclk:
-	clk_disable_unprepare(rx->dcodec);
-err_dcodec:
-	clk_disable_unprepare(rx->macro);
+err_rpm_put:
+	if (pm_runtime_put_sync_suspend(dev) < 0)
+		dev_warn(dev, "runtime PM sync suspend failed in probe unwind\n");
 
 	return ret;
 }
 
-static void rx_macro_remove(struct platform_device *pdev)
-{
-	struct rx_macro *rx = dev_get_drvdata(&pdev->dev);
-
-	clk_disable_unprepare(rx->mclk);
-	clk_disable_unprepare(rx->npl);
-	clk_disable_unprepare(rx->fsgen);
-	clk_disable_unprepare(rx->macro);
-	clk_disable_unprepare(rx->dcodec);
-}
-
 static const struct of_device_id rx_macro_dt_match[] = {
 	{
 		.compatible = "qcom,sc7280-lpass-rx-macro",
@@ -3969,13 +3957,17 @@ MODULE_DEVICE_TABLE(of, rx_macro_dt_match);
 static int rx_macro_runtime_suspend(struct device *dev)
 {
 	struct rx_macro *rx = dev_get_drvdata(dev);
+	int ret;
 
 	regcache_cache_only(rx->regmap, true);
-	regcache_mark_dirty(rx->regmap);
 
-	clk_disable_unprepare(rx->fsgen);
-	clk_disable_unprepare(rx->npl);
-	clk_disable_unprepare(rx->mclk);
+	ret = pm_clk_suspend(dev);
+	if (ret) {
+		regcache_cache_only(rx->regmap, false);
+		return ret;
+	}
+
+	regcache_mark_dirty(rx->regmap);
 
 	return 0;
 }
@@ -3985,33 +3977,23 @@ static int rx_macro_runtime_resume(struct device *dev)
 	struct rx_macro *rx = dev_get_drvdata(dev);
 	int ret;
 
-	ret = clk_prepare_enable(rx->mclk);
+	ret = pm_clk_resume(dev);
 	if (ret) {
-		dev_err(dev, "unable to prepare mclk\n");
+		regcache_cache_only(rx->regmap, true);
+		regcache_mark_dirty(rx->regmap);
 		return ret;
 	}
 
-	ret = clk_prepare_enable(rx->npl);
-	if (ret) {
-		dev_err(dev, "unable to prepare mclkx2\n");
-		goto err_npl;
-	}
-
-	ret = clk_prepare_enable(rx->fsgen);
+	regcache_cache_only(rx->regmap, false);
+	ret = regcache_sync(rx->regmap);
 	if (ret) {
-		dev_err(dev, "unable to prepare fsgen\n");
-		goto err_fsgen;
+		regcache_cache_only(rx->regmap, true);
+		regcache_mark_dirty(rx->regmap);
+		pm_clk_suspend(dev);
+		return ret;
 	}
-	regcache_cache_only(rx->regmap, false);
-	regcache_sync(rx->regmap);
 
 	return 0;
-err_fsgen:
-	clk_disable_unprepare(rx->npl);
-err_npl:
-	clk_disable_unprepare(rx->mclk);
-
-	return ret;
 }
 
 static const struct dev_pm_ops rx_macro_pm_ops = {
@@ -4026,7 +4008,6 @@ static struct platform_driver rx_macro_driver = {
 		.pm = pm_ptr(&rx_macro_pm_ops),
 	},
 	.probe = rx_macro_probe,
-	.remove = rx_macro_remove,
 };
 
 module_platform_driver(rx_macro_driver);
diff --git a/sound/soc/codecs/lpass-tx-macro.c b/sound/soc/codecs/lpass-tx-macro.c
index f7d168f557dd8..9c5c95fda2b12 100644
--- a/sound/soc/codecs/lpass-tx-macro.c
+++ b/sound/soc/codecs/lpass-tx-macro.c
@@ -6,6 +6,7 @@
 #include <linux/clk.h>
 #include <linux/io.h>
 #include <linux/platform_device.h>
+#include <linux/pm_clock.h>
 #include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 #include <sound/soc.h>
@@ -614,6 +615,7 @@ static int tx_macro_mclk_enable(struct tx_macro *tx,
 				bool mclk_enable)
 {
 	struct regmap *regmap = tx->regmap;
+	int ret;
 
 	if (mclk_enable) {
 		if (tx->tx_mclk_users == 0) {
@@ -626,7 +628,9 @@ static int tx_macro_mclk_enable(struct tx_macro *tx,
 					   CDC_TX_FS_CNT_EN_MASK,
 					   CDC_TX_FS_CNT_ENABLE);
 			regcache_mark_dirty(regmap);
-			regcache_sync(regmap);
+			ret = regcache_sync(regmap);
+			if (ret)
+				return ret;
 		}
 		tx->tx_mclk_users++;
 	} else {
@@ -739,11 +743,9 @@ static int tx_macro_mclk_event(struct snd_soc_dapm_widget *w,
 
 	switch (event) {
 	case SND_SOC_DAPM_PRE_PMU:
-		tx_macro_mclk_enable(tx, true);
-		break;
+		return tx_macro_mclk_enable(tx, true);
 	case SND_SOC_DAPM_POST_PMD:
-		tx_macro_mclk_enable(tx, false);
-		break;
+		return tx_macro_mclk_enable(tx, false);
 	default:
 		break;
 	}
@@ -2149,17 +2151,20 @@ static int swclk_gate_enable(struct clk_hw *hw)
 	struct regmap *regmap = tx->regmap;
 	int ret;
 
-	ret = clk_prepare_enable(tx->mclk);
+	ret = pm_runtime_resume_and_get(tx->dev);
+	if (ret < 0)
+		return ret;
+
+	ret = tx_macro_mclk_enable(tx, true);
 	if (ret) {
-		dev_err(tx->dev, "failed to enable mclk\n");
+		pm_runtime_put_autosuspend(tx->dev);
 		return ret;
 	}
 
-	tx_macro_mclk_enable(tx, true);
-
 	regmap_update_bits(regmap, CDC_TX_CLK_RST_CTRL_SWR_CONTROL,
 			   CDC_TX_SWR_CLK_EN_MASK,
 			   CDC_TX_SWR_CLK_ENABLE);
+
 	return 0;
 }
 
@@ -2172,7 +2177,7 @@ static void swclk_gate_disable(struct clk_hw *hw)
 			   CDC_TX_SWR_CLK_EN_MASK, 0x0);
 
 	tx_macro_mclk_enable(tx, false);
-	clk_disable_unprepare(tx->mclk);
+	pm_runtime_put_autosuspend(tx->dev);
 }
 
 static int swclk_gate_is_enabled(struct clk_hw *hw)
@@ -2315,28 +2320,31 @@ static int tx_macro_probe(struct platform_device *pdev)
 	tx->active_decimator[TX_MACRO_AIF3_CAP] = -1;
 
 	/* set MCLK and NPL rates */
-	clk_set_rate(tx->mclk, MCLK_FREQ);
-	clk_set_rate(tx->npl, MCLK_FREQ);
-
-	ret = clk_prepare_enable(tx->macro);
+	ret = clk_set_rate(tx->mclk, MCLK_FREQ);
 	if (ret)
 		goto err;
 
-	ret = clk_prepare_enable(tx->dcodec);
+	ret = clk_set_rate(tx->npl, MCLK_FREQ);
 	if (ret)
-		goto err_dcodec;
+		goto err;
 
-	ret = clk_prepare_enable(tx->mclk);
+	ret = devm_pm_clk_create(dev);
 	if (ret)
-		goto err_mclk;
+		goto err;
 
-	ret = clk_prepare_enable(tx->npl);
-	if (ret)
-		goto err_npl;
+	ret = of_pm_clk_add_clks(dev);
+	if (ret < 0)
+		goto err;
 
-	ret = clk_prepare_enable(tx->fsgen);
+	pm_runtime_set_autosuspend_delay(dev, 100);
+	pm_runtime_use_autosuspend(dev);
+	ret = devm_pm_runtime_enable(dev);
 	if (ret)
-		goto err_fsgen;
+		goto err;
+
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret < 0)
+		goto err;
 
 
 	/* reset soundwire block */
@@ -2356,30 +2364,21 @@ static int tx_macro_probe(struct platform_device *pdev)
 					      tx_macro_dai,
 					      ARRAY_SIZE(tx_macro_dai));
 	if (ret)
-		goto err_clkout;
-
-	pm_runtime_set_autosuspend_delay(dev, 3000);
-	pm_runtime_use_autosuspend(dev);
-	pm_runtime_mark_last_busy(dev);
-	pm_runtime_set_active(dev);
-	pm_runtime_enable(dev);
+		goto err_rpm_put;
 
 	ret = tx_macro_register_mclk_output(tx);
 	if (ret)
-		goto err_clkout;
+		goto err_rpm_put;
+
+	ret = pm_runtime_put_autosuspend(dev);
+	if (ret < 0)
+		dev_warn(dev, "runtime PM put failed after probe: %d\n", ret);
 
 	return 0;
 
-err_clkout:
-	clk_disable_unprepare(tx->fsgen);
-err_fsgen:
-	clk_disable_unprepare(tx->npl);
-err_npl:
-	clk_disable_unprepare(tx->mclk);
-err_mclk:
-	clk_disable_unprepare(tx->dcodec);
-err_dcodec:
-	clk_disable_unprepare(tx->macro);
+err_rpm_put:
+	if (pm_runtime_put_sync_suspend(dev) < 0)
+		dev_warn(dev, "runtime PM sync suspend failed in probe unwind\n");
 err:
 	lpass_macro_pds_exit(tx->pds);
 
@@ -2390,25 +2389,23 @@ static void tx_macro_remove(struct platform_device *pdev)
 {
 	struct tx_macro *tx = dev_get_drvdata(&pdev->dev);
 
-	clk_disable_unprepare(tx->macro);
-	clk_disable_unprepare(tx->dcodec);
-	clk_disable_unprepare(tx->mclk);
-	clk_disable_unprepare(tx->npl);
-	clk_disable_unprepare(tx->fsgen);
-
 	lpass_macro_pds_exit(tx->pds);
 }
 
 static int tx_macro_runtime_suspend(struct device *dev)
 {
 	struct tx_macro *tx = dev_get_drvdata(dev);
+	int ret;
 
 	regcache_cache_only(tx->regmap, true);
-	regcache_mark_dirty(tx->regmap);
 
-	clk_disable_unprepare(tx->fsgen);
-	clk_disable_unprepare(tx->npl);
-	clk_disable_unprepare(tx->mclk);
+	ret = pm_clk_suspend(dev);
+	if (ret) {
+		regcache_cache_only(tx->regmap, false);
+		return ret;
+	}
+
+	regcache_mark_dirty(tx->regmap);
 
 	return 0;
 }
@@ -2418,34 +2415,23 @@ static int tx_macro_runtime_resume(struct device *dev)
 	struct tx_macro *tx = dev_get_drvdata(dev);
 	int ret;
 
-	ret = clk_prepare_enable(tx->mclk);
+	ret = pm_clk_resume(dev);
 	if (ret) {
-		dev_err(dev, "unable to prepare mclk\n");
+		regcache_cache_only(tx->regmap, true);
+		regcache_mark_dirty(tx->regmap);
 		return ret;
 	}
 
-	ret = clk_prepare_enable(tx->npl);
-	if (ret) {
-		dev_err(dev, "unable to prepare npl\n");
-		goto err_npl;
-	}
-
-	ret = clk_prepare_enable(tx->fsgen);
+	regcache_cache_only(tx->regmap, false);
+	ret = regcache_sync(tx->regmap);
 	if (ret) {
-		dev_err(dev, "unable to prepare fsgen\n");
-		goto err_fsgen;
+		regcache_cache_only(tx->regmap, true);
+		regcache_mark_dirty(tx->regmap);
+		pm_clk_suspend(dev);
+		return ret;
 	}
 
-	regcache_cache_only(tx->regmap, false);
-	regcache_sync(tx->regmap);
-
 	return 0;
-err_fsgen:
-	clk_disable_unprepare(tx->npl);
-err_npl:
-	clk_disable_unprepare(tx->mclk);
-
-	return ret;
 }
 
 static const struct dev_pm_ops tx_macro_pm_ops = {
diff --git a/sound/soc/codecs/lpass-va-macro.c b/sound/soc/codecs/lpass-va-macro.c
index 58a5798823d7f..dbc5795b9273f 100644
--- a/sound/soc/codecs/lpass-va-macro.c
+++ b/sound/soc/codecs/lpass-va-macro.c
@@ -10,6 +10,7 @@
 #include <linux/of_clk.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
+#include <linux/pm_clock.h>
 #include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
@@ -1351,34 +1352,43 @@ static int fsgen_gate_enable(struct clk_hw *hw)
 {
 	struct va_macro *va = to_va_macro(hw);
 	struct regmap *regmap = va->regmap;
-	int ret;
+	int ret, rpm_ret;
 
-	if (va->has_swr_master) {
-		ret = clk_prepare_enable(va->mclk);
-		if (ret)
-			return ret;
-	}
+	ret = pm_runtime_resume_and_get(va->dev);
+	if (ret < 0)
+		return ret;
 
 	ret = va_macro_mclk_enable(va, true);
+	if (ret) {
+		rpm_ret = pm_runtime_put_autosuspend(va->dev);
+		if (rpm_ret < 0)
+			dev_warn(va->dev,
+				 "runtime PM put failed in fsgen enable unwind: %d\n",
+				 rpm_ret);
+		return ret;
+	}
 	if (va->has_swr_master)
 		regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_SWR_CONTROL,
 				   CDC_VA_SWR_CLK_EN_MASK, CDC_VA_SWR_CLK_ENABLE);
 
-	return ret;
+	return 0;
 }
 
 static void fsgen_gate_disable(struct clk_hw *hw)
 {
 	struct va_macro *va = to_va_macro(hw);
 	struct regmap *regmap = va->regmap;
+	int ret;
 
 	if (va->has_swr_master)
 		regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_SWR_CONTROL,
 			   CDC_VA_SWR_CLK_EN_MASK, 0x0);
 
 	va_macro_mclk_enable(va, false);
-	if (va->has_swr_master)
-		clk_disable_unprepare(va->mclk);
+
+	ret = pm_runtime_put_autosuspend(va->dev);
+	if (ret < 0)
+		dev_warn(va->dev, "runtime PM put failed in fsgen disable: %d\n", ret);
 }
 
 static int fsgen_gate_is_enabled(struct clk_hw *hw)
@@ -1539,6 +1549,7 @@ static int va_macro_probe(struct platform_device *pdev)
 	void __iomem *base;
 	u32 sample_rate = 0;
 	int ret;
+	int rpm_ret;
 
 	va = devm_kzalloc(dev, sizeof(*va), GFP_KERNEL);
 	if (!va)
@@ -1594,7 +1605,9 @@ static int va_macro_probe(struct platform_device *pdev)
 	va->has_npl_clk = data->has_npl_clk;
 
 	/* mclk rate */
-	clk_set_rate(va->mclk, 2 * VA_MACRO_MCLK_FREQ);
+	ret = clk_set_rate(va->mclk, 2 * VA_MACRO_MCLK_FREQ);
+	if (ret)
+		goto err;
 
 	if (va->has_npl_clk) {
 		va->npl = devm_clk_get(dev, "npl");
@@ -1603,25 +1616,29 @@ static int va_macro_probe(struct platform_device *pdev)
 			goto err;
 		}
 
-		clk_set_rate(va->npl, 2 * VA_MACRO_MCLK_FREQ);
+		ret = clk_set_rate(va->npl, 2 * VA_MACRO_MCLK_FREQ);
+		if (ret)
+			goto err;
 	}
 
-	ret = clk_prepare_enable(va->macro);
+	ret = devm_pm_clk_create(dev);
 	if (ret)
 		goto err;
 
-	ret = clk_prepare_enable(va->dcodec);
-	if (ret)
-		goto err_dcodec;
+	ret = of_pm_clk_add_clks(dev);
+	if (ret < 0)
+		goto err;
 
-	ret = clk_prepare_enable(va->mclk);
+	pm_runtime_set_autosuspend_delay(dev, 100);
+	pm_runtime_use_autosuspend(dev);
+	ret = devm_pm_runtime_enable(dev);
 	if (ret)
-		goto err_mclk;
+		goto err;
 
-	if (va->has_npl_clk) {
-		ret = clk_prepare_enable(va->npl);
-		if (ret)
-			goto err_npl;
+	rpm_ret = pm_runtime_resume_and_get(dev);
+	if (rpm_ret < 0) {
+		ret = rpm_ret;
+		goto err;
 	}
 
 	/**
@@ -1634,7 +1651,7 @@ static int va_macro_probe(struct platform_device *pdev)
 		/* read version from register */
 		ret = va_macro_set_lpass_codec_version(va);
 		if (ret)
-			goto err_clkout;
+			goto err_rpm_put;
 	}
 
 	if (va->has_swr_master) {
@@ -1664,35 +1681,28 @@ static int va_macro_probe(struct platform_device *pdev)
 					      va_macro_dais,
 					      ARRAY_SIZE(va_macro_dais));
 	if (ret)
-		goto err_clkout;
-
-	pm_runtime_set_autosuspend_delay(dev, 3000);
-	pm_runtime_use_autosuspend(dev);
-	pm_runtime_mark_last_busy(dev);
-	pm_runtime_set_active(dev);
-	pm_runtime_enable(dev);
+		goto err_rpm_put;
 
 	ret = va_macro_register_fsgen_output(va);
 	if (ret)
-		goto err_clkout;
+		goto err_rpm_put;
 
 	va->fsgen = devm_clk_hw_get_clk(dev, &va->hw, "fsgen");
 	if (IS_ERR(va->fsgen)) {
 		ret = PTR_ERR(va->fsgen);
-		goto err_clkout;
+		goto err_rpm_put;
 	}
 
+	rpm_ret = pm_runtime_put_autosuspend(dev);
+	if (rpm_ret < 0)
+		dev_warn(dev, "runtime PM put failed after probe: %d\n", rpm_ret);
+
 	return 0;
 
-err_clkout:
-	if (va->has_npl_clk)
-		clk_disable_unprepare(va->npl);
-err_npl:
-	clk_disable_unprepare(va->mclk);
-err_mclk:
-	clk_disable_unprepare(va->dcodec);
-err_dcodec:
-	clk_disable_unprepare(va->macro);
+err_rpm_put:
+	rpm_ret = pm_runtime_put_sync_suspend(dev);
+	if (rpm_ret < 0)
+		dev_warn(dev, "runtime PM sync suspend failed in probe unwind: %d\n", rpm_ret);
 err:
 	lpass_macro_pds_exit(va->pds);
 
@@ -1703,27 +1713,23 @@ static void va_macro_remove(struct platform_device *pdev)
 {
 	struct va_macro *va = dev_get_drvdata(&pdev->dev);
 
-	if (va->has_npl_clk)
-		clk_disable_unprepare(va->npl);
-
-	clk_disable_unprepare(va->mclk);
-	clk_disable_unprepare(va->dcodec);
-	clk_disable_unprepare(va->macro);
-
 	lpass_macro_pds_exit(va->pds);
 }
 
 static int va_macro_runtime_suspend(struct device *dev)
 {
 	struct va_macro *va = dev_get_drvdata(dev);
+	int ret;
 
 	regcache_cache_only(va->regmap, true);
-	regcache_mark_dirty(va->regmap);
 
-	if (va->has_npl_clk)
-		clk_disable_unprepare(va->npl);
+	ret = pm_clk_suspend(dev);
+	if (ret) {
+		regcache_cache_only(va->regmap, false);
+		return ret;
+	}
 
-	clk_disable_unprepare(va->mclk);
+	regcache_mark_dirty(va->regmap);
 
 	return 0;
 }
@@ -1731,25 +1737,28 @@ static int va_macro_runtime_suspend(struct device *dev)
 static int va_macro_runtime_resume(struct device *dev)
 {
 	struct va_macro *va = dev_get_drvdata(dev);
-	int ret;
+	int ret, sret;
 
-	ret = clk_prepare_enable(va->mclk);
+	ret = pm_clk_resume(dev);
 	if (ret) {
-		dev_err(va->dev, "unable to prepare mclk\n");
+		regcache_cache_only(va->regmap, true);
+		regcache_mark_dirty(va->regmap);
 		return ret;
 	}
 
-	if (va->has_npl_clk) {
-		ret = clk_prepare_enable(va->npl);
-		if (ret) {
-			clk_disable_unprepare(va->mclk);
-			dev_err(va->dev, "unable to prepare npl\n");
-			return ret;
-		}
-	}
-
 	regcache_cache_only(va->regmap, false);
-	regcache_sync(va->regmap);
+
+	ret = regcache_sync(va->regmap);
+	if (ret) {
+		regcache_cache_only(va->regmap, true);
+		regcache_mark_dirty(va->regmap);
+		sret = pm_clk_suspend(dev);
+		if (sret)
+			dev_err(va->dev,
+				"failed to suspend clocks after regcache sync failure: %d\n",
+				sret);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/lpass-wsa-macro.c b/sound/soc/codecs/lpass-wsa-macro.c
index 5ad0448af649d..fc9e0a37c0423 100644
--- a/sound/soc/codecs/lpass-wsa-macro.c
+++ b/sound/soc/codecs/lpass-wsa-macro.c
@@ -12,6 +12,7 @@
 #include <linux/clk-provider.h>
 #include <sound/soc.h>
 #include <sound/soc-dapm.h>
+#include <linux/pm_clock.h>
 #include <linux/pm_runtime.h>
 #include <linux/of_platform.h>
 #include <sound/tlv.h>
@@ -2529,15 +2530,13 @@ static const struct snd_soc_dapm_route wsa_audio_map[] = {
 static int wsa_swrm_clock(struct wsa_macro *wsa, bool enable)
 {
 	struct regmap *regmap = wsa->regmap;
+	int ret;
 
 	if (enable) {
-		int ret;
-
-		ret = clk_prepare_enable(wsa->mclk);
-		if (ret) {
-			dev_err(wsa->dev, "failed to enable mclk\n");
+		ret = pm_runtime_resume_and_get(wsa->dev);
+		if (ret < 0)
 			return ret;
-		}
+
 		wsa_macro_mclk_enable(wsa, true);
 
 		regmap_update_bits(regmap, CDC_WSA_CLK_RST_CTRL_SWR_CONTROL,
@@ -2548,7 +2547,10 @@ static int wsa_swrm_clock(struct wsa_macro *wsa, bool enable)
 		regmap_update_bits(regmap, CDC_WSA_CLK_RST_CTRL_SWR_CONTROL,
 				   CDC_WSA_SWR_CLK_EN_MASK, 0);
 		wsa_macro_mclk_enable(wsa, false);
-		clk_disable_unprepare(wsa->mclk);
+
+		ret = pm_runtime_put_autosuspend(wsa->dev);
+		if (ret < 0)
+			dev_warn(wsa->dev, "runtime PM put failed: %d\n", ret);
 	}
 
 	return 0;
@@ -2656,7 +2658,7 @@ static int wsa_macro_register_mclk_output(struct wsa_macro *wsa)
 	init.num_parents = 1;
 	wsa->hw.init = &init;
 	hw = &wsa->hw;
-	ret = clk_hw_register(wsa->dev, hw);
+	ret = devm_clk_hw_register(wsa->dev, hw);
 	if (ret)
 		return ret;
 
@@ -2771,28 +2773,31 @@ static int wsa_macro_probe(struct platform_device *pdev)
 	wsa->dev = dev;
 
 	/* set MCLK and NPL rates */
-	clk_set_rate(wsa->mclk, WSA_MACRO_MCLK_FREQ);
-	clk_set_rate(wsa->npl, WSA_MACRO_MCLK_FREQ);
-
-	ret = clk_prepare_enable(wsa->macro);
+	ret = clk_set_rate(wsa->mclk, WSA_MACRO_MCLK_FREQ);
 	if (ret)
-		goto err;
+		return ret;
 
-	ret = clk_prepare_enable(wsa->dcodec);
+	ret = clk_set_rate(wsa->npl, WSA_MACRO_MCLK_FREQ);
 	if (ret)
-		goto err_dcodec;
+		return ret;
 
-	ret = clk_prepare_enable(wsa->mclk);
+	ret = devm_pm_clk_create(dev);
 	if (ret)
-		goto err_mclk;
+		return ret;
 
-	ret = clk_prepare_enable(wsa->npl);
-	if (ret)
-		goto err_npl;
+	ret = of_pm_clk_add_clks(dev);
+	if (ret < 0)
+		return ret;
 
-	ret = clk_prepare_enable(wsa->fsgen);
+	pm_runtime_set_autosuspend_delay(dev, 100);
+	pm_runtime_use_autosuspend(dev);
+	ret = devm_pm_runtime_enable(dev);
 	if (ret)
-		goto err_fsgen;
+		return ret;
+
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret < 0)
+		return ret;
 
 	/* reset swr ip */
 	regmap_update_bits(wsa->regmap, CDC_WSA_CLK_RST_CTRL_SWR_CONTROL,
@@ -2809,56 +2814,37 @@ static int wsa_macro_probe(struct platform_device *pdev)
 					      wsa_macro_dai,
 					      ARRAY_SIZE(wsa_macro_dai));
 	if (ret)
-		goto err_clkout;
-
-	pm_runtime_set_autosuspend_delay(dev, 3000);
-	pm_runtime_use_autosuspend(dev);
-	pm_runtime_mark_last_busy(dev);
-	pm_runtime_set_active(dev);
-	pm_runtime_enable(dev);
+		goto err_rpm_put;
 
 	ret = wsa_macro_register_mclk_output(wsa);
 	if (ret)
-		goto err_clkout;
+		goto err_rpm_put;
 
-	return 0;
+	ret = pm_runtime_put_autosuspend(dev);
+	if (ret < 0)
+		dev_warn(dev, "runtime PM put failed after probe: %d\n", ret);
 
-err_clkout:
-	clk_disable_unprepare(wsa->fsgen);
-err_fsgen:
-	clk_disable_unprepare(wsa->npl);
-err_npl:
-	clk_disable_unprepare(wsa->mclk);
-err_mclk:
-	clk_disable_unprepare(wsa->dcodec);
-err_dcodec:
-	clk_disable_unprepare(wsa->macro);
-err:
+	return 0;
+err_rpm_put:
+	if (pm_runtime_put_sync_suspend(dev) < 0)
+		dev_warn(dev, "runtime PM sync suspend failed in probe unwind\n");
 	return ret;
-
-}
-
-static void wsa_macro_remove(struct platform_device *pdev)
-{
-	struct wsa_macro *wsa = dev_get_drvdata(&pdev->dev);
-
-	clk_disable_unprepare(wsa->macro);
-	clk_disable_unprepare(wsa->dcodec);
-	clk_disable_unprepare(wsa->mclk);
-	clk_disable_unprepare(wsa->npl);
-	clk_disable_unprepare(wsa->fsgen);
 }
 
 static int wsa_macro_runtime_suspend(struct device *dev)
 {
 	struct wsa_macro *wsa = dev_get_drvdata(dev);
+	int ret;
 
 	regcache_cache_only(wsa->regmap, true);
-	regcache_mark_dirty(wsa->regmap);
 
-	clk_disable_unprepare(wsa->fsgen);
-	clk_disable_unprepare(wsa->npl);
-	clk_disable_unprepare(wsa->mclk);
+	ret = pm_clk_suspend(dev);
+	if (ret) {
+		regcache_cache_only(wsa->regmap, false);
+		return ret;
+	}
+
+	regcache_mark_dirty(wsa->regmap);
 
 	return 0;
 }
@@ -2866,36 +2852,29 @@ static int wsa_macro_runtime_suspend(struct device *dev)
 static int wsa_macro_runtime_resume(struct device *dev)
 {
 	struct wsa_macro *wsa = dev_get_drvdata(dev);
-	int ret;
+	int ret, sret;
 
-	ret = clk_prepare_enable(wsa->mclk);
+	ret = pm_clk_resume(dev);
 	if (ret) {
-		dev_err(dev, "unable to prepare mclk\n");
+		regcache_cache_only(wsa->regmap, true);
+		regcache_mark_dirty(wsa->regmap);
 		return ret;
 	}
+	regcache_cache_only(wsa->regmap, false);
 
-	ret = clk_prepare_enable(wsa->npl);
-	if (ret) {
-		dev_err(dev, "unable to prepare mclkx2\n");
-		goto err_npl;
-	}
-
-	ret = clk_prepare_enable(wsa->fsgen);
+	ret = regcache_sync(wsa->regmap);
 	if (ret) {
-		dev_err(dev, "unable to prepare fsgen\n");
-		goto err_fsgen;
+		regcache_cache_only(wsa->regmap, true);
+		regcache_mark_dirty(wsa->regmap);
+		sret = pm_clk_suspend(dev);
+		if (sret)
+			dev_err(dev,
+				"failed to suspend clocks after regcache sync failure: %d\n",
+				sret);
+		return ret;
 	}
 
-	regcache_cache_only(wsa->regmap, false);
-	regcache_sync(wsa->regmap);
-
 	return 0;
-err_fsgen:
-	clk_disable_unprepare(wsa->npl);
-err_npl:
-	clk_disable_unprepare(wsa->mclk);
-
-	return ret;
 }
 
 static const struct dev_pm_ops wsa_macro_pm_ops = {
@@ -2929,7 +2908,6 @@ static struct platform_driver wsa_macro_driver = {
 		.pm = pm_ptr(&wsa_macro_pm_ops),
 	},
 	.probe = wsa_macro_probe,
-	.remove = wsa_macro_remove,
 };
 
 module_platform_driver(wsa_macro_driver);
diff --git a/sound/soc/codecs/max98090.c b/sound/soc/codecs/max98090.c
index da416329b0384..60da518b0e57c 100644
--- a/sound/soc/codecs/max98090.c
+++ b/sound/soc/codecs/max98090.c
@@ -2341,6 +2341,7 @@ static irqreturn_t max98090_interrupt(int irq, void *data)
  *
  * @component:  MAX98090 component
  * @jack:   jack to report detection events on
+ * @data: can be used if codec driver need extra data for configuring jack
  *
  * Enable microphone detection via IRQ on the MAX98090.  If GPIOs are
  * being used to bring out signals to the processor then only platform
@@ -2645,12 +2646,18 @@ static void max98090_i2c_remove(struct i2c_client *client)
 static int max98090_runtime_resume(struct device *dev)
 {
 	struct max98090_priv *max98090 = dev_get_drvdata(dev);
+	int ret;
 
 	regcache_cache_only(max98090->regmap, false);
 
 	max98090_reset(max98090);
 
-	regcache_sync(max98090->regmap);
+	ret = regcache_sync(max98090->regmap);
+	if (ret < 0) {
+		regcache_cache_only(max98090->regmap, true);
+		regcache_mark_dirty(max98090->regmap);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/max98363.c b/sound/soc/codecs/max98363.c
index 099dc5bf6195f..0e32b7a94cb05 100644
--- a/sound/soc/codecs/max98363.c
+++ b/sound/soc/codecs/max98363.c
@@ -100,7 +100,12 @@ static int max98363_resume(struct device *dev)
 		return ret;
 
 	regcache_cache_only(max98363->regmap, false);
-	regcache_sync(max98363->regmap);
+	ret = regcache_sync(max98363->regmap);
+	if (ret) {
+		regcache_cache_only(max98363->regmap, true);
+		regcache_mark_dirty(max98363->regmap);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/max98373-i2c.c b/sound/soc/codecs/max98373-i2c.c
index 8805bd01153c6..cbd7ebffa6d61 100644
--- a/sound/soc/codecs/max98373-i2c.c
+++ b/sound/soc/codecs/max98373-i2c.c
@@ -488,10 +488,17 @@ static int max98373_suspend(struct device *dev)
 static int max98373_resume(struct device *dev)
 {
 	struct max98373_priv *max98373 = dev_get_drvdata(dev);
+	int ret;
 
 	regcache_cache_only(max98373->regmap, false);
 	max98373_reset(max98373, dev);
-	regcache_sync(max98373->regmap);
+	ret = regcache_sync(max98373->regmap);
+	if (ret) {
+		regcache_cache_only(max98373->regmap, true);
+		regcache_mark_dirty(max98373->regmap);
+		return ret;
+	}
+
 	return 0;
 }
 
diff --git a/sound/soc/codecs/max98373-sdw.c b/sound/soc/codecs/max98373-sdw.c
index 8fe9c58e1a620..638ef343813bd 100644
--- a/sound/soc/codecs/max98373-sdw.c
+++ b/sound/soc/codecs/max98373-sdw.c
@@ -277,7 +277,12 @@ static int max98373_resume(struct device *dev)
 	}
 
 	regcache_cache_only(max98373->regmap, false);
-	regcache_sync(max98373->regmap);
+	ret = regcache_sync(max98373->regmap);
+	if (ret) {
+		regcache_cache_only(max98373->regmap, true);
+		regcache_mark_dirty(max98373->regmap);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/max98388.c b/sound/soc/codecs/max98388.c
index a4c57152d25fc..d97dab41bf0f5 100644
--- a/sound/soc/codecs/max98388.c
+++ b/sound/soc/codecs/max98388.c
@@ -863,10 +863,16 @@ static int max98388_suspend(struct device *dev)
 static int max98388_resume(struct device *dev)
 {
 	struct max98388_priv *max98388 = dev_get_drvdata(dev);
+	int ret;
 
 	regcache_cache_only(max98388->regmap, false);
 	max98388_reset(max98388, dev);
-	regcache_sync(max98388->regmap);
+	ret = regcache_sync(max98388->regmap);
+	if (ret) {
+		regcache_cache_only(max98388->regmap, true);
+		regcache_mark_dirty(max98388->regmap);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/max98390.c b/sound/soc/codecs/max98390.c
index 2bbedf84ee5d0..66309e87fdbd8 100644
--- a/sound/soc/codecs/max98390.c
+++ b/sound/soc/codecs/max98390.c
@@ -952,11 +952,17 @@ static int max98390_suspend(struct device *dev)
 static int max98390_resume(struct device *dev)
 {
 	struct max98390_priv *max98390 = dev_get_drvdata(dev);
+	int ret;
 
 	dev_dbg(dev, "%s:Enter\n", __func__);
 
 	regcache_cache_only(max98390->regmap, false);
-	regcache_sync(max98390->regmap);
+	ret = regcache_sync(max98390->regmap);
+	if (ret) {
+		regcache_cache_only(max98390->regmap, true);
+		regcache_mark_dirty(max98390->regmap);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/max98396.c b/sound/soc/codecs/max98396.c
index 9c1d7213410c2..e5a2fe824c03d 100644
--- a/sound/soc/codecs/max98396.c
+++ b/sound/soc/codecs/max98396.c
@@ -1600,19 +1600,37 @@ static int max98396_resume(struct device *dev)
 	if (max98396->pvdd) {
 		ret = regulator_enable(max98396->pvdd);
 		if (ret < 0)
-			return ret;
+			goto err_core_supplies;
 	}
 
 	if (max98396->vbat) {
 		ret = regulator_enable(max98396->vbat);
 		if (ret < 0)
-			return ret;
+			goto err_pvdd;
 	}
 
 	regcache_cache_only(max98396->regmap, false);
 	max98396_reset(max98396, dev);
-	regcache_sync(max98396->regmap);
+	ret = regcache_sync(max98396->regmap);
+	if (ret < 0) {
+		regcache_cache_only(max98396->regmap, true);
+		regcache_mark_dirty(max98396->regmap);
+		goto err_vbat;
+	}
+
 	return 0;
+
+err_vbat:
+	if (max98396->vbat)
+		regulator_disable(max98396->vbat);
+err_pvdd:
+	if (max98396->pvdd)
+		regulator_disable(max98396->pvdd);
+err_core_supplies:
+	regulator_bulk_disable(MAX98396_NUM_CORE_SUPPLIES,
+			       max98396->core_supplies);
+
+	return ret;
 }
 
 static const struct dev_pm_ops max98396_pm = {
diff --git a/sound/soc/codecs/max98927.c b/sound/soc/codecs/max98927.c
index 65e6fdb30eec8..524300d8fb0bc 100644
--- a/sound/soc/codecs/max98927.c
+++ b/sound/soc/codecs/max98927.c
@@ -742,11 +742,17 @@ static int max98927_suspend(struct device *dev)
 static int max98927_resume(struct device *dev)
 {
 	struct max98927_priv *max98927 = dev_get_drvdata(dev);
+	int ret;
 
 	regmap_write(max98927->regmap, MAX98927_R0100_SOFT_RESET,
 		     MAX98927_SOFT_RESET);
 	regcache_cache_only(max98927->regmap, false);
-	regcache_sync(max98927->regmap);
+	ret = regcache_sync(max98927->regmap);
+	if (ret) {
+		regcache_cache_only(max98927->regmap, true);
+		regcache_mark_dirty(max98927->regmap);
+		return ret;
+	}
 	return 0;
 }
 
diff --git a/sound/soc/codecs/pcm1681.c b/sound/soc/codecs/pcm1681.c
index cb923cecb47fc..60fdbe5c4e054 100644
--- a/sound/soc/codecs/pcm1681.c
+++ b/sound/soc/codecs/pcm1681.c
@@ -199,10 +199,17 @@ static int pcm1681_hw_params(struct snd_pcm_substream *substream,
 	return pcm1681_set_deemph(component);
 }
 
+static const u64 pcm1681_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J;
+
 static const struct snd_soc_dai_ops pcm1681_dai_ops = {
 	.set_fmt	= pcm1681_set_dai_fmt,
 	.hw_params	= pcm1681_hw_params,
 	.mute_stream	= pcm1681_mute,
+	.auto_selectable_formats	= &pcm1681_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
diff --git a/sound/soc/codecs/pcm1754.c b/sound/soc/codecs/pcm1754.c
index b68a528000be8..bb15da1c24aa2 100644
--- a/sound/soc/codecs/pcm1754.c
+++ b/sound/soc/codecs/pcm1754.c
@@ -78,10 +78,16 @@ static int pcm1754_mute_stream(struct snd_soc_dai *dai, int mute, int stream)
 	return 0;
 }
 
+static const u64 pcm1754_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J;
+
 static const struct snd_soc_dai_ops pcm1754_dai_ops = {
 	.set_fmt = pcm1754_set_dai_fmt,
 	.hw_params = pcm1754_hw_params,
 	.mute_stream = pcm1754_mute_stream,
+	.auto_selectable_formats = &pcm1754_selectable_formats,
+	.num_auto_selectable_formats = 1,
 };
 
 static const struct snd_soc_dai_driver pcm1754_dai = {
diff --git a/sound/soc/codecs/pcm1789.c b/sound/soc/codecs/pcm1789.c
index 3ab381e9a8566..f06e353743aaa 100644
--- a/sound/soc/codecs/pcm1789.c
+++ b/sound/soc/codecs/pcm1789.c
@@ -164,11 +164,18 @@ static int pcm1789_trigger(struct snd_pcm_substream *substream, int cmd,
 	return ret;
 }
 
+static const u64 pcm1789_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J;
+
 static const struct snd_soc_dai_ops pcm1789_dai_ops = {
 	.set_fmt	= pcm1789_set_dai_fmt,
 	.hw_params	= pcm1789_hw_params,
 	.mute_stream	= pcm1789_mute,
 	.trigger	= pcm1789_trigger,
+	.auto_selectable_formats	= &pcm1789_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
diff --git a/sound/soc/codecs/pcm179x.c b/sound/soc/codecs/pcm179x.c
index f52ff66b6e644..cb70927872aa0 100644
--- a/sound/soc/codecs/pcm179x.c
+++ b/sound/soc/codecs/pcm179x.c
@@ -142,10 +142,16 @@ static int pcm179x_hw_params(struct snd_pcm_substream *substream,
 	return 0;
 }
 
+static const u64 pcm179x_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J;
+
 static const struct snd_soc_dai_ops pcm179x_dai_ops = {
 	.set_fmt	= pcm179x_set_dai_fmt,
 	.hw_params	= pcm179x_hw_params,
 	.mute_stream	= pcm179x_mute,
+	.auto_selectable_formats	= &pcm179x_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
diff --git a/sound/soc/codecs/pcm186x.c b/sound/soc/codecs/pcm186x.c
index 0d1103fe4e04d..461ec74e87521 100644
--- a/sound/soc/codecs/pcm186x.c
+++ b/sound/soc/codecs/pcm186x.c
@@ -473,11 +473,20 @@ static int pcm186x_set_dai_sysclk(struct snd_soc_dai *dai, int clk_id,
 	return 0;
 }
 
+static const u64 pcm186x_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF;
+
 static const struct snd_soc_dai_ops pcm186x_dai_ops = {
 	.set_sysclk = pcm186x_set_dai_sysclk,
 	.set_tdm_slot = pcm186x_set_tdm_slot,
 	.set_fmt = pcm186x_set_fmt,
 	.hw_params = pcm186x_hw_params,
+	.auto_selectable_formats = &pcm186x_selectable_formats,
+	.num_auto_selectable_formats = 1,
 };
 
 static struct snd_soc_dai_driver pcm1863_dai = {
diff --git a/sound/soc/codecs/pcm3060.c b/sound/soc/codecs/pcm3060.c
index 8974200652e7e..f01c46583e897 100644
--- a/sound/soc/codecs/pcm3060.c
+++ b/sound/soc/codecs/pcm3060.c
@@ -164,10 +164,18 @@ static int pcm3060_hw_params(struct snd_pcm_substream *substream,
 	return 0;
 }
 
+static const u64 pcm3060_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF;
+
 static const struct snd_soc_dai_ops pcm3060_dai_ops = {
 	.set_sysclk = pcm3060_set_sysclk,
 	.set_fmt = pcm3060_set_fmt,
 	.hw_params = pcm3060_hw_params,
+	.auto_selectable_formats = &pcm3060_selectable_formats,
+	.num_auto_selectable_formats = 1,
 };
 
 #define PCM3060_DAI_RATES_ADC	(SNDRV_PCM_RATE_16000 | SNDRV_PCM_RATE_32000 | \
diff --git a/sound/soc/codecs/pcm512x.c b/sound/soc/codecs/pcm512x.c
index fe3b5011fa167..10868df3c75d5 100644
--- a/sound/soc/codecs/pcm512x.c
+++ b/sound/soc/codecs/pcm512x.c
@@ -1505,12 +1505,21 @@ static int pcm512x_mute(struct snd_soc_dai *dai, int mute, int direction)
 	return ret;
 }
 
+static const u64 pcm512x_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B;
+
 static const struct snd_soc_dai_ops pcm512x_dai_ops = {
 	.startup = pcm512x_dai_startup,
 	.hw_params = pcm512x_hw_params,
 	.set_fmt = pcm512x_set_fmt,
 	.mute_stream = pcm512x_mute,
 	.set_bclk_ratio = pcm512x_set_bclk_ratio,
+	.auto_selectable_formats = &pcm512x_selectable_formats,
+	.num_auto_selectable_formats = 1,
 	.no_capture_mute = 1,
 };
 
diff --git a/sound/soc/codecs/pm4125-sdw.c b/sound/soc/codecs/pm4125-sdw.c
index 1c612ae4a4b27..307b66025426e 100644
--- a/sound/soc/codecs/pm4125-sdw.c
+++ b/sound/soc/codecs/pm4125-sdw.c
@@ -464,10 +464,16 @@ static int __maybe_unused pm4125_sdw_runtime_suspend(struct device *dev)
 static int __maybe_unused pm4125_sdw_runtime_resume(struct device *dev)
 {
 	struct pm4125_sdw_priv *priv = dev_get_drvdata(dev);
+	int ret;
 
 	if (priv->regmap) {
 		regcache_cache_only(priv->regmap, false);
-		regcache_sync(priv->regmap);
+		ret = regcache_sync(priv->regmap);
+		if (ret) {
+			regcache_cache_only(priv->regmap, true);
+			regcache_mark_dirty(priv->regmap);
+			return ret;
+		}
 	}
 
 	return 0;
diff --git a/sound/soc/codecs/pm4125.c b/sound/soc/codecs/pm4125.c
index 29655175ea289..674a2fbc4edae 100644
--- a/sound/soc/codecs/pm4125.c
+++ b/sound/soc/codecs/pm4125.c
@@ -5,6 +5,7 @@
 #include <linux/component.h>
 #include <linux/delay.h>
 #include <linux/device.h>
+#include <linux/irqdomain.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/of.h>
@@ -1295,7 +1296,7 @@ static const struct irq_domain_ops pm4125_domain_ops = {
 
 static int pm4125_irq_init(struct pm4125_priv *pm4125, struct device *dev)
 {
-	pm4125->virq = irq_domain_add_linear(NULL, 1, &pm4125_domain_ops, NULL);
+	pm4125->virq = irq_domain_create_linear(NULL, 1, &pm4125_domain_ops, NULL);
 	if (!(pm4125->virq)) {
 		dev_err(dev, "%s: Failed to add IRQ domain\n", __func__);
 		return -EINVAL;
diff --git a/sound/soc/codecs/rt1017-sdca-sdw.c b/sound/soc/codecs/rt1017-sdca-sdw.c
index 95405cea81433..caf75e5657ef1 100644
--- a/sound/soc/codecs/rt1017-sdca-sdw.c
+++ b/sound/soc/codecs/rt1017-sdca-sdw.c
@@ -784,7 +784,12 @@ static int rt1017_sdca_dev_resume(struct device *dev)
 	}
 
 	regcache_cache_only(rt1017->regmap, false);
-	regcache_sync(rt1017->regmap);
+	ret = regcache_sync(rt1017->regmap);
+	if (ret) {
+		regcache_cache_only(rt1017->regmap, true);
+		regcache_mark_dirty(rt1017->regmap);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/rt1316-sdw.c b/sound/soc/codecs/rt1316-sdw.c
index ca318dbd946ee..359b52ba1ef93 100644
--- a/sound/soc/codecs/rt1316-sdw.c
+++ b/sound/soc/codecs/rt1316-sdw.c
@@ -756,7 +756,12 @@ static int rt1316_dev_resume(struct device *dev)
 	}
 
 	regcache_cache_only(rt1316->regmap, false);
-	regcache_sync(rt1316->regmap);
+	ret = regcache_sync(rt1316->regmap);
+	if (ret) {
+		regcache_cache_only(rt1316->regmap, true);
+		regcache_mark_dirty(rt1316->regmap);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/rt1318-sdw.c b/sound/soc/codecs/rt1318-sdw.c
index c038ac0e3b769..ef18cd63d461b 100644
--- a/sound/soc/codecs/rt1318-sdw.c
+++ b/sound/soc/codecs/rt1318-sdw.c
@@ -830,7 +830,12 @@ static int rt1318_dev_resume(struct device *dev)
 		return ret;
 
 	regcache_cache_only(rt1318->regmap, false);
-	regcache_sync(rt1318->regmap);
+	ret = regcache_sync(rt1318->regmap);
+	if (ret) {
+		regcache_cache_only(rt1318->regmap, true);
+		regcache_mark_dirty(rt1318->regmap);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/rt1320-sdw.c b/sound/soc/codecs/rt1320-sdw.c
index 1e930b27c67a8..a30fb575918d1 100644
--- a/sound/soc/codecs/rt1320-sdw.c
+++ b/sound/soc/codecs/rt1320-sdw.c
@@ -410,6 +410,402 @@ static const struct reg_sequence rt1321_blind_write[] = {
 	{ 0x41001988, 0x03 },
 };
 
+static const struct reg_sequence rt1321_va1_blind_write[] = {
+	{ 0x0000c003, 0xf0 },
+	{ 0x0000c01b, 0xfc },
+	{ 0x0000c5c3, 0xf2 },
+	{ 0x0000c5c2, 0x00 },
+	{ 0x0000c5c1, 0x10 },
+	{ 0x0000c5c0, 0x04 },
+	{ 0x0000c5c7, 0x03 },
+	{ 0x0000c5c6, 0x10 },
+	{ 0x0000c526, 0x47 },
+	{ 0x0000c5c4, 0x12 },
+	{ 0x0000c5c5, 0x60 },
+	{ 0x0000c520, 0x10 },
+	{ 0x0000c521, 0x32 },
+	{ 0x0000c5c7, 0x00 },
+	{ 0x0000c5c8, 0x03 },
+	{ 0x0000c5d3, 0x08 },
+	{ 0x0000c5d2, 0x0a },
+	{ 0x0000c5d1, 0x49 },
+	{ 0x0000c5d0, 0x0f },
+	{ 0x0000c580, 0x10 },
+	{ 0x0000c581, 0x32 },
+	{ 0x0000c582, 0x01 },
+	{ 0x0000c682, 0x60 },
+	{ 0x0000c019, 0x10 },
+	{ 0x0000c5f0, 0x01 },
+	{ 0x0000c5f7, 0x22 },
+	{ 0x0000c5f6, 0x22 },
+	{ 0x0000c057, 0x51 },
+	{ 0x0000c054, 0x55 },
+	{ 0x0000c053, 0x55 },
+	{ 0x0000c052, 0x55 },
+	{ 0x0000c051, 0x01 },
+	{ 0x0000c050, 0x15 },
+	{ 0x0000c060, 0x99 },
+	{ 0x0000c030, 0x55 },
+	{ 0x0000c061, 0x55 },
+	{ 0x0000c063, 0x55 },
+	{ 0x0000c065, 0xa5 },
+	{ 0x0000c06b, 0x0a },
+	{ 0x0000ca05, 0xd6 },
+	{ 0x0000ca06, 0x11 },
+	{ 0x0000ca07, 0x1e },
+	{ 0x0000ca25, 0xd6 },
+	{ 0x0000ca26, 0x11 },
+	{ 0x0000ca27, 0x1e },
+	{ 0x0000cd00, 0x05 },
+	{ 0x0000cd81, 0x49 },
+	{ 0x0000cd82, 0x49 },
+	{ 0x0000c604, 0x40 },
+	{ 0x0000c609, 0x40 },
+	{ 0x0000c046, 0xf7 },
+	{ 0x0000c045, 0xff },
+	{ 0x0000c044, 0xff },
+	{ 0x0000c043, 0xff },
+	{ 0x0000c042, 0xff },
+	{ 0x0000c041, 0xff },
+	{ 0x0000c040, 0xff },
+	{ 0x0000c049, 0xff },
+	{ 0x0000c028, 0x3f },
+	{ 0x0000c020, 0x3f },
+	{ 0x0000c032, 0x13 },
+	{ 0x0000c033, 0x01 },
+	{ 0x0000cc10, 0x01 },
+	{ 0x0000dc20, 0x03 },
+	{ 0x0000de03, 0x05 },
+	{ 0x0000dc00, 0x00 },
+	{ 0x0000c700, 0xf0 },
+	{ 0x0000c701, 0x13 },
+	{ 0x0000c900, 0xc3 },
+	{ 0x0000c570, 0x08 },
+	{ 0x0000c086, 0x02 },
+	{ 0x0000c085, 0x7f },
+	{ 0x0000c084, 0x00 },
+	{ 0x0000c081, 0xff },
+	{ 0x0000f084, 0x0f },
+	{ 0x0000f083, 0xff },
+	{ 0x0000f082, 0xff },
+	{ 0x0000f081, 0xff },
+	{ 0x0000f080, 0xff },
+	{ 0x20003003, 0x3f },
+	{ 0x20005818, 0x81 },
+	{ 0x20009018, 0x81 },
+	{ 0x2000301c, 0x81 },
+	{ 0x0000c003, 0xc0 },
+	{ 0x0000c047, 0x80 },
+	{ 0x0000d541, 0x80 },
+	{ 0x0000d487, 0x0b },
+	{ 0x0000d487, 0x3b },
+	{ 0x0000d486, 0xc3 },
+	{ 0x0000d470, 0x89 },
+	{ 0x0000d471, 0x3a },
+	{ 0x0000d472, 0x3d },
+	{ 0x0000d474, 0x11 },
+	{ 0x0000d475, 0x32 },
+	{ 0x0000d476, 0x64 },
+	{ 0x0000d477, 0x10 },
+	{ 0x0000d478, 0xff },
+	{ 0x0000d479, 0x20 },
+	{ 0x0000d47a, 0x10 },
+	{ 0x0000d73c, 0xb7 },
+	{ 0x0000d73d, 0xd7 },
+	{ 0x0000d73e, 0x00 },
+	{ 0x0000d73f, 0x10 },
+	{ 0x1000cd56, 0x00 },
+	{ 0x3fc2dfc0, 0x03 },
+	{ 0x3fc2dfc1, 0x00 },
+	{ 0x3fc2dfc2, 0x00 },
+	{ 0x3fc2dfc3, 0x00 },
+	{ 0x3fc2dfc4, 0x01 },
+	{ 0x3fc2dfc5, 0x00 },
+	{ 0x3fc2dfc6, 0x00 },
+	{ 0x3fc2dfc7, 0x00 },
+	{ 0x3fc2df80, 0x00 },
+	{ 0x3fc2df81, 0x00 },
+	{ 0x3fc2df82, 0x00 },
+	{ 0x3fc2df83, 0x00 },
+	{ 0x0000d541, 0x40 },
+	{ 0x0000d486, 0x43 },
+	{ 0x1000db00, 0x04 },
+	{ 0x1000db01, 0x00 },
+	{ 0x1000db02, 0x10 },
+	{ 0x1000db03, 0x00 },
+	{ 0x1000db04, 0x00 },
+	{ 0x1000db05, 0x45 },
+	{ 0x1000db06, 0x0d },
+	{ 0x1000db07, 0x01 },
+	{ 0x1000db08, 0x00 },
+	{ 0x1000db09, 0x00 },
+	{ 0x1000db0a, 0xbf },
+	{ 0x1000db0b, 0x0b },
+	{ 0x1000db0c, 0x11 },
+	{ 0x1000db0d, 0x00 },
+	{ 0x1000db0e, 0x00 },
+	{ 0x1000db0f, 0x00 },
+	{ 0x1000db10, 0x2c },
+	{ 0x1000db11, 0xfa },
+	{ 0x1000db12, 0x00 },
+	{ 0x1000db13, 0x00 },
+	{ 0x1000db14, 0x09 },
+	{ 0x0000d540, 0x21 },
+	{ 0x0000c570, 0x08 },
+	{ 0x0000d714, 0x17 },
+	{ 0x0000c5c3, 0xf2 },
+	{ 0x0000c5c8, 0x03 },
+	{ 0x20009012, 0x00 },
+	{ 0x0000dd08, 0x17 },
+	{ 0x0000dd09, 0x0e },
+	{ 0x0000dd0a, 0x17 },
+	{ 0x0000dd0b, 0x0e },
+	{ 0x0000c570, 0x08 },
+	{ 0x0000d471, 0x3a },
+	{ 0x0000db00, 0x00 },
+	{ 0x0000db01, 0x00 },
+	{ 0x0000db02, 0x73 },
+	{ 0x0000db03, 0x00 },
+	{ 0x0000db04, 0x00 },
+	{ 0x0000db05, 0x00 },
+	{ 0x0000db06, 0x00 },
+	{ 0x0000db07, 0x00 },
+	{ 0x0000db08, 0x7f },
+	{ 0x0000db09, 0x00 },
+	{ 0x0000db1a, 0x00 },
+	{ 0x0000db1b, 0x00 },
+	{ 0x0000db19, 0x00 },
+};
+
+static const struct reg_sequence rt1321_va2_blind_write[] = {
+	{ 0x0000c003, 0xf0 },
+	{ 0x0000c01b, 0xfc },
+	{ 0x0000c5c3, 0xf2 },
+	{ 0x0000c5c2, 0x00 },
+	{ 0x0000c5c1, 0x10 },
+	{ 0x0000c5c0, 0x04 },
+	{ 0x0000c5c7, 0x03 },
+	{ 0x0000c5c6, 0x10 },
+	{ 0x0000c526, 0x47 },
+	{ 0x0000c5c4, 0x12 },
+	{ 0x0000c5c5, 0x60 },
+	{ 0x0000c520, 0x10 },
+	{ 0x0000c521, 0x32 },
+	{ 0x0000c5c7, 0x00 },
+	{ 0x0000c5c8, 0x03 },
+	{ 0x0000c5d3, 0x08 },
+	{ 0x0000c5d2, 0x0a },
+	{ 0x0000c5d1, 0x49 },
+	{ 0x0000c5d0, 0x0f },
+	{ 0x0000c580, 0x10 },
+	{ 0x0000c581, 0x32 },
+	{ 0x0000c582, 0x01 },
+	{ 0x0000c682, 0x60 },
+	{ 0x0000c019, 0x10 },
+	{ 0x0000c5f0, 0x01 },
+	{ 0x0000c5f7, 0x22 },
+	{ 0x0000c5f6, 0x22 },
+	{ 0x0000c057, 0x51 },
+	{ 0x0000c054, 0x55 },
+	{ 0x0000c053, 0x55 },
+	{ 0x0000c052, 0x55 },
+	{ 0x0000c051, 0x01 },
+	{ 0x0000c050, 0x15 },
+	{ 0x0000c060, 0x99 },
+	{ 0x0000c030, 0x55 },
+	{ 0x0000c061, 0x55 },
+	{ 0x0000c063, 0x55 },
+	{ 0x0000c065, 0xa5 },
+	{ 0x0000c06b, 0x0a },
+	{ 0x0000ca05, 0xd6 },
+	{ 0x0000ca06, 0x11 },
+	{ 0x0000ca07, 0x1e },
+	{ 0x0000ca25, 0xd6 },
+	{ 0x0000ca26, 0x11 },
+	{ 0x0000ca27, 0x1e },
+	{ 0x0000cd00, 0x05 },
+	{ 0x0000cd81, 0x49 },
+	{ 0x0000cd82, 0x49 },
+	{ 0x0000c604, 0x40 },
+	{ 0x0000c609, 0x40 },
+	{ 0x0000c046, 0xf7 },
+	{ 0x0000c045, 0xff },
+	{ 0x0000c044, 0xff },
+	{ 0x0000c043, 0xff },
+	{ 0x0000c042, 0xff },
+	{ 0x0000c041, 0xff },
+	{ 0x0000c040, 0xff },
+	{ 0x0000c049, 0xff },
+	{ 0x0000c028, 0x3f },
+	{ 0x0000c020, 0x3f },
+	{ 0x0000c032, 0x13 },
+	{ 0x0000c033, 0x01 },
+	{ 0x0000cc10, 0x01 },
+	{ 0x0000dc20, 0x03 },
+	{ 0x0000de03, 0x05 },
+	{ 0x0000dc00, 0x00 },
+	{ 0x0000c700, 0xf0 },
+	{ 0x0000c701, 0x13 },
+	{ 0x0000c900, 0xc3 },
+	{ 0x0000c570, 0x08 },
+	{ 0x0000c086, 0x02 },
+	{ 0x0000c085, 0x7f },
+	{ 0x0000c084, 0x00 },
+	{ 0x0000c081, 0xff },
+	{ 0x0000f084, 0x0f },
+	{ 0x0000f083, 0xff },
+	{ 0x0000f082, 0xff },
+	{ 0x0000f081, 0xff },
+	{ 0x0000f080, 0xff },
+	{ 0x20003003, 0x3f },
+	{ 0x20005818, 0x81 },
+	{ 0x20009018, 0x81 },
+	{ 0x2000301c, 0x81 },
+	{ 0x0000c003, 0xc0 },
+	{ 0x0000c047, 0x80 },
+	{ 0x0000d541, 0x80 },
+	{ 0x0000d487, 0x0b },
+	{ 0x0000d487, 0x3b },
+	{ 0x0000d486, 0xc3 },
+	{ 0x0000d470, 0x89 },
+	{ 0x0000d471, 0x3a },
+	{ 0x0000d472, 0x3d },
+	{ 0x0000d474, 0x11 },
+	{ 0x0000d475, 0x32 },
+	{ 0x0000d476, 0x64 },
+	{ 0x0000d477, 0x10 },
+	{ 0x0000d478, 0xff },
+	{ 0x0000d479, 0x20 },
+	{ 0x0000d47a, 0x10 },
+	{ 0x10008000, 0x67 },
+	{ 0x10008001, 0x80 },
+	{ 0x10008002, 0x00 },
+	{ 0x10008003, 0x00 },
+	{ 0x1000cd56, 0x00 },
+	{ 0x0000d486, 0x43 },
+	{ 0x3fc2dfc3, 0x00 },
+	{ 0x3fc2dfc2, 0x00 },
+	{ 0x3fc2dfc1, 0x00 },
+	{ 0x3fc2dfc0, 0x03 },
+	{ 0x3fc2dfc7, 0x00 },
+	{ 0x3fc2dfc6, 0x00 },
+	{ 0x3fc2dfc5, 0x00 },
+	{ 0x3fc2dfc4, 0x01 },
+	{ 0x3fc2dfa3, 0x00 },
+	{ 0x3fc2dfa2, 0x00 },
+	{ 0x3fc2dfa1, 0x00 },
+	{ 0x3fc2dfa0, 0x00 },
+	{ 0x3fc2df80, 0x10 },
+	{ 0x3fc2df81, 0x20 },
+	{ 0x3fc2df82, 0x00 },
+	{ 0x3fc2df83, 0x00 },
+	{ 0x3fc2df84, 0x50 },
+	{ 0x3fc2df85, 0x19 },
+	{ 0x3fc2df86, 0x00 },
+	{ 0x3fc2df87, 0x00 },
+	{ 0x3fc2df88, 0x52 },
+	{ 0x3fc2df89, 0x23 },
+	{ 0x3fc2df8a, 0x00 },
+	{ 0x3fc2df8b, 0x00 },
+	{ 0x3fc2df8c, 0xe0 },
+	{ 0x3fc2df8d, 0x2e },
+	{ 0x3fc2df8e, 0x00 },
+	{ 0x3fc2df8f, 0x00 },
+	{ 0x3fc2df90, 0xe0 },
+	{ 0x3fc2df91, 0x2e },
+	{ 0x3fc2df92, 0x00 },
+	{ 0x3fc2df93, 0x00 },
+	{ 0x3fc2df94, 0x01 },
+	{ 0x3fc2df95, 0x08 },
+	{ 0x3fc2df96, 0x00 },
+	{ 0x3fc2df97, 0x00 },
+	{ 0x3fc2df40, 0x80 },
+	{ 0x3fc2df41, 0xbb },
+	{ 0x3fc2df42, 0x00 },
+	{ 0x3fc2df43, 0x00 },
+	{ 0x3fc2df44, 0xc0 },
+	{ 0x3fc2df45, 0x99 },
+	{ 0x3fc2df46, 0x01 },
+	{ 0x3fc2df47, 0x00 },
+	{ 0x3fc2df48, 0x00 },
+	{ 0x3fc2df49, 0x00 },
+	{ 0x3fc2df4a, 0x00 },
+	{ 0x3fc2df4b, 0x00 },
+	{ 0x3fc2df4c, 0x00 },
+	{ 0x3fc2df4d, 0x00 },
+	{ 0x3fc2df4e, 0x00 },
+	{ 0x3fc2df4f, 0x00 },
+	{ 0x3fc2df50, 0x01 },
+	{ 0x3fc2df51, 0x00 },
+	{ 0x3fc2df52, 0x00 },
+	{ 0x3fc2df53, 0x00 },
+	{ 0x3fc2df54, 0x01 },
+	{ 0x3fc2df55, 0x00 },
+	{ 0x3fc2df56, 0x00 },
+	{ 0x3fc2df57, 0x00 },
+	{ 0x3fc2df58, 0x00 },
+	{ 0x3fc2df59, 0x00 },
+	{ 0x3fc2df5a, 0x00 },
+	{ 0x3fc2df5b, 0x00 },
+	{ 0x3fc2df5c, 0x01 },
+	{ 0x3fc2df5d, 0x00 },
+	{ 0x3fc2df5e, 0x00 },
+	{ 0x3fc2df5f, 0x00 },
+	{ 0x3fc2df60, 0x00 },
+	{ 0x3fc2df61, 0x00 },
+	{ 0x3fc2df62, 0x00 },
+	{ 0x3fc2df63, 0x00 },
+	{ 0x3fc2df64, 0x00 },
+	{ 0x3fc2df65, 0x00 },
+	{ 0x3fc2df66, 0x00 },
+	{ 0x3fc2df67, 0x10 },
+	{ 0x3fc2df68, 0x01 },
+	{ 0x3fc2df69, 0x00 },
+	{ 0x3fc2df6a, 0x00 },
+	{ 0x3fc2df6b, 0x00 },
+	{ 0x3fc2df6c, 0x01 },
+	{ 0x3fc2df6d, 0x00 },
+	{ 0x3fc2df6e, 0x00 },
+	{ 0x3fc2df6f, 0x00 },
+	{ 0x3fc2df70, 0x04 },
+	{ 0x3fc2df71, 0x00 },
+	{ 0x3fc2df72, 0x00 },
+	{ 0x3fc2df73, 0x00 },
+	{ 0x3fc2df74, 0x01 },
+	{ 0x3fc2df75, 0x00 },
+	{ 0x3fc2df76, 0x00 },
+	{ 0x3fc2df77, 0x00 },
+	{ 0x1000db00, 0x04 },
+	{ 0x1000db01, 0x00 },
+	{ 0x1000db02, 0x10 },
+	{ 0x1000db03, 0x00 },
+	{ 0x1000db04, 0x00 },
+	{ 0x1000db05, 0x45 },
+	{ 0x1000db06, 0x0d },
+	{ 0x1000db07, 0x01 },
+	{ 0x1000db08, 0x00 },
+	{ 0x1000db09, 0x00 },
+	{ 0x1000db0a, 0xbf },
+	{ 0x1000db0b, 0x0b },
+	{ 0x1000db0c, 0x11 },
+	{ 0x1000db0d, 0x00 },
+	{ 0x1000db0e, 0x00 },
+	{ 0x1000db0f, 0x00 },
+	{ 0x1000db10, 0x2c },
+	{ 0x1000db11, 0xfa },
+	{ 0x1000db12, 0x00 },
+	{ 0x1000db13, 0x00 },
+	{ 0x1000db14, 0x09 },
+	{ 0x0000d540, 0x21 },
+	{ 0x0000d714, 0x17 },
+	{ 0x0000dd0b, 0x0d },
+	{ 0x0000dd0a, 0xff },
+	{ 0x0000dd09, 0x0d },
+	{ 0x0000dd08, 0xff },
+	{ 0x0000c5fb, 0x12 },
+	{ 0x0000c570, 0x08 },
+};
+
 static const struct reg_default rt1320_reg_defaults[] = {
 	{ SDW_SDCA_CTL(FUNC_NUM_MIC, RT1320_SDCA_ENT_PDE11, RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x03 },
 	{ SDW_SDCA_CTL(FUNC_NUM_MIC, RT1320_SDCA_ENT_FU113, RT1320_SDCA_CTL_FU_MUTE, CH_01), 0x01 },
@@ -497,10 +893,15 @@ static bool rt1320_readable_register(struct device *dev, unsigned int reg)
 	case RT1321_PATCH_MAIN_VER ... RT1321_PATCH_BETA_VER:
 	case 0x1000f008:
 	case 0x1000f021:
-	case 0x2000300f:
+	case 0x20003000 ... 0x2000300f:
 	case 0x2000301c:
-	case 0x2000900f:
+	case 0x20003040 ... 0x20003059:
+	case 0x20005800 ... 0x2000580f:
+	case 0x20005818:
+	case 0x20005840 ... 0x20005859:
+	case 0x20009000 ... 0x2000900f:
 	case 0x20009018:
+	case 0x20009040 ... 0x20009059:
 	case 0x3fc000c0 ... 0x3fc2dfc8:
 	case 0x3fe00000 ... 0x3fe36fff:
 	/* 0x40801508/0x40801809/0x4080180a/0x40801909/0x4080190a */
@@ -596,15 +997,20 @@ static bool rt1320_volatile_register(struct device *dev, unsigned int reg)
 	case 0x1000c000 ... 0x1000dfff:
 	case 0x1000f008:
 	case 0x1000f021:
-	case 0x2000300f:
+	case 0x2000300e ... 0x2000300f:
 	case 0x2000301c:
-	case 0x2000900f:
+	case 0x20003040 ... 0x20003059:
+	case 0x2000580e ... 0x2000580f:
+	case 0x20005818:
+	case 0x20005840 ... 0x20005859:
+	case 0x2000900e ... 0x2000900f:
 	case 0x20009018:
+	case 0x20009040 ... 0x20009059:
 	case 0x3fc2ab80 ... 0x3fc2ac4c:
 	case 0x3fc2b780:
 	case 0x3fc2bf80 ... 0x3fc2bf83:
 	case 0x3fc2bfc0 ... 0x3fc2bfc8:
-	case 0x3fc2d300 ... 0x3fc2d354:
+	case 0x3fc2d300 ... 0x3fc2d3cc:
 	case 0x3fc2dfc0 ... 0x3fc2dfc8:
 	case 0x3fe2e000 ... 0x3fe2e003:
 	case SDW_SDCA_CTL(FUNC_NUM_MIC, RT1320_SDCA_ENT_PDE11, RT1320_SDCA_CTL_ACTUAL_POWER_STATE, 0):
@@ -892,6 +1298,42 @@ static int rt1320_check_fw_ready(struct rt1320_sdw_priv *rt1320)
 	return 0;
 }
 
+static int rt1320_dspfw_status(struct rt1320_sdw_priv *rt1320)
+{
+	struct device *dev = &rt1320->sdw_slave->dev;
+	unsigned int fw_status_addr, fw_ready;
+	unsigned int dspfw_run;
+
+	switch (rt1320->dev_id) {
+	case RT1320_DEV_ID:
+		fw_status_addr = RT1320_DSPFW_STATUS_ADDR;
+		break;
+	case RT1321_DEV_ID:
+		fw_status_addr = RT1321_DSPFW_STATUS_ADDR;
+		break;
+	default:
+		dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id);
+		return -EINVAL;
+	}
+
+	regmap_read(rt1320->regmap, fw_status_addr, &fw_ready);
+	fw_ready &= 0x1;
+
+	if (rt1320->dev_id == RT1321_DEV_ID) {
+		regmap_read(rt1320->regmap, 0xf01e, &dspfw_run);
+		dspfw_run &= 0x1;
+		fw_ready = (!dspfw_run && fw_ready);
+	}
+
+	if (fw_ready) {
+		dev_dbg(dev, "%s, DSP FW was already\n", __func__);
+		return 1;
+	}
+
+	dev_dbg(dev, "%s, DSP FW is NOT ready. Please load DSP FW first\n", __func__);
+	return 0;
+}
+
 static int rt1320_check_power_state_ready(struct rt1320_sdw_priv *rt1320, enum rt1320_power_state ps)
 {
 	struct device *dev = &rt1320->sdw_slave->dev;
@@ -982,6 +1424,9 @@ static int rt1320_fw_param_protocol(struct rt1320_sdw_priv *rt1320, enum rt1320_
 	if (!tempbuf)
 		return -ENOMEM;
 
+	if (rt1320->dev_id == RT1321_DEV_ID && rt1320->version_id == RT1321_VA2)
+		paramid += 0x0bff0000;
+
 	paramhr.moudleid = 1;
 	paramhr.commandtype = cmdid;
 	/* 8 is "sizeof(paramid) + sizeof(paramlength)" */
@@ -1120,45 +1565,73 @@ static void rt1320_calc_r0(struct rt1320_sdw_priv *rt1320)
 		l_calir0, l_calir0_lo, r_calir0, r_calir0_lo);
 }
 
+static int rt1320_pilot_tone_output(struct rt1320_sdw_priv *rt1320)
+{
+	struct device *dev = &rt1320->sdw_slave->dev;
+	int l_targetpostgain, r_targetpostgain;
+	unsigned long long factor = (1 << 12);
+	int l_pilotgain[9], r_pilotgain[9];
+	const int postgain_step = 234;
+	int targetGain;
+
+	switch (rt1320->dev_id) {
+	case RT1320_DEV_ID:
+		targetGain = -320000;
+		break;
+	case RT1321_DEV_ID:
+		targetGain = -420000;
+		if (rt1320->version_id == RT1321_VA0)
+			targetGain = -320000;
+		break;
+	default:
+		dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id);
+		return -EINVAL;
+	}
+
+	rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 70, &l_pilotgain[0], sizeof(l_pilotgain));
+	rt1320_fw_param_protocol(rt1320, RT1320_GET_PARAM, 71, &r_pilotgain[0], sizeof(r_pilotgain));
+	dev_dbg(dev, "%s, LR pilotgain %d, %d\n", __func__, l_pilotgain[2], r_pilotgain[2]);
+
+	/* calculate pilot tone gain */
+	l_pilotgain[2] = (l_pilotgain[2] * 10000) / factor;
+	r_pilotgain[2] = (r_pilotgain[2] * 10000) / factor;
+
+	/* calculate post gain to meet target gain */
+	l_targetpostgain = abs(targetGain - l_pilotgain[2]) / postgain_step;
+	r_targetpostgain = abs(targetGain - r_pilotgain[2]) / postgain_step;
+	l_targetpostgain = 0xfff - l_targetpostgain;
+	r_targetpostgain = 0xfff - r_targetpostgain;
+	dev_dbg(dev, "%s, LR targetpostgain=0x%x, 0x%x\n", __func__, l_targetpostgain, r_targetpostgain);
+
+	regmap_write(rt1320->regmap, 0xdd0b, (l_targetpostgain & 0xf00) >> 8);
+	regmap_write(rt1320->regmap, 0xdd0a, l_targetpostgain & 0xff);
+	regmap_write(rt1320->regmap, 0xdd09, (r_targetpostgain & 0xf00) >> 8);
+	regmap_write(rt1320->regmap, 0xdd08, r_targetpostgain & 0xff);
+
+	return 0;
+}
+
 static void rt1320_calibrate(struct rt1320_sdw_priv *rt1320)
 {
 	struct device *dev = &rt1320->sdw_slave->dev;
 	struct rt1320_datafixpoint audfixpoint[2];
 	unsigned int reg_c5fb, reg_c570, reg_cd00;
-	unsigned int vol_reg[4], fw_ready;
+	unsigned int vol_reg[4];
 	unsigned long long l_meanr0, r_meanr0;
-	unsigned int fw_status_addr;
 	int l_re[5], r_re[5];
 	int ret, tmp;
 	unsigned long long factor = (1 << 27);
 	unsigned short l_advancegain, r_advancegain;
 	unsigned int delay_s = 7; /* delay seconds for the calibration */
+	int dspfw_status;
 
 	if (!rt1320->component)
 		return;
 
-	switch (rt1320->dev_id) {
-	case RT1320_DEV_ID:
-		fw_status_addr = RT1320_DSPFW_STATUS_ADDR;
-		break;
-	case RT1321_DEV_ID:
-		fw_status_addr = RT1321_DSPFW_STATUS_ADDR;
-		break;
-	default:
-		dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id);
-		return;
-	}
-
-	/* set volume 0dB */
 	regmap_read(rt1320->regmap, 0xdd0b, &vol_reg[3]);
 	regmap_read(rt1320->regmap, 0xdd0a, &vol_reg[2]);
 	regmap_read(rt1320->regmap, 0xdd09, &vol_reg[1]);
 	regmap_read(rt1320->regmap, 0xdd08, &vol_reg[0]);
-	regmap_write(rt1320->regmap, 0xdd0b, 0x0f);
-	regmap_write(rt1320->regmap, 0xdd0a, 0xff);
-	regmap_write(rt1320->regmap, 0xdd09, 0x0f);
-	regmap_write(rt1320->regmap, 0xdd08, 0xff);
-
 	regmap_read(rt1320->regmap, 0xc5fb, &reg_c5fb);
 	regmap_read(rt1320->regmap, 0xc570, &reg_c570);
 	regmap_read(rt1320->regmap, 0xcd00, &reg_cd00);
@@ -1171,9 +1644,8 @@ static void rt1320_calibrate(struct rt1320_sdw_priv *rt1320)
 		goto _finish_;
 	}
 
-	regmap_read(rt1320->regmap, fw_status_addr, &fw_ready);
-	fw_ready &= 0x1;
-	if (!fw_ready) {
+	dspfw_status = rt1320_dspfw_status(rt1320);
+	if (dspfw_status <= 0) {
 		dev_dbg(dev, "%s, DSP FW is NOT ready. Please load DSP FW first\n", __func__);
 		goto _finish_;
 	}
@@ -1184,8 +1656,19 @@ static void rt1320_calibrate(struct rt1320_sdw_priv *rt1320)
 		goto _finish_;
 	}
 
-	if (rt1320->dev_id == RT1320_DEV_ID)
-		regmap_write(rt1320->regmap, 0xc5fb, 0x00);
+	/* fine tune pilot tone output */
+	ret = rt1320_pilot_tone_output(rt1320);
+	if (ret < 0) {
+		dev_dbg(dev, "%s, Failed to tune pilot tone output\n", __func__);
+		goto _finish_;
+	}
+
+	if (rt1320->dev_id == RT1321_DEV_ID) {
+		regmap_update_bits(rt1320->regmap, 0xc047, 0x80, 0x00);
+		regmap_write(rt1320->regmap, 0xc5c4, 0x12);
+	}
+
+	regmap_write(rt1320->regmap, 0xc5fb, 0x00);
 	regmap_write(rt1320->regmap, 0xc570, 0x0b);
 	regmap_write(rt1320->regmap, 0xcd00, 0xc5);
 
@@ -1246,6 +1729,11 @@ static void rt1320_calibrate(struct rt1320_sdw_priv *rt1320)
 		SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x03);
 	rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x03);
 
+	if (rt1320->dev_id == RT1321_DEV_ID) {
+		regmap_update_bits(rt1320->regmap, 0xc047, 0x80, 0x80);
+		regmap_write(rt1320->regmap, 0xc5c4, 0x10);
+	}
+
 	/* advance gain will be set when R0 load, not here */
 	regmap_write(rt1320->regmap, 0xdd0b, vol_reg[3]);
 	regmap_write(rt1320->regmap, 0xdd0a, vol_reg[2]);
@@ -1312,6 +1800,9 @@ static void rt1320_load_mcu_patch(struct rt1320_sdw_priv *rt1320)
 		max_addr = 0x10007fff;
 		break;
 	case RT1321_DEV_ID:
+		if (rt1320->version_id == RT1321_VA2)
+			return;
+
 		filename = RT1321_VA_MCU_PATCH;
 		min_addr = 0x10008000;
 		max_addr = 0x10008fff;
@@ -1382,30 +1873,17 @@ static void rt1320_vab_preset(struct rt1320_sdw_priv *rt1320)
 static void rt1320_t0_load(struct rt1320_sdw_priv *rt1320, unsigned int l_t0, unsigned int r_t0)
 {
 	struct device *dev = &rt1320->sdw_slave->dev;
-	unsigned int factor = (1 << 22), fw_ready;
+	unsigned int factor = (1 << 22);
 	int l_t0_data[38], r_t0_data[38];
-	unsigned int fw_status_addr;
-
-	switch (rt1320->dev_id) {
-	case RT1320_DEV_ID:
-		fw_status_addr = RT1320_DSPFW_STATUS_ADDR;
-		break;
-	case RT1321_DEV_ID:
-		fw_status_addr = RT1321_DSPFW_STATUS_ADDR;
-		break;
-	default:
-		dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id);
-		return;
-	}
+	int dspfw_status;
 
 	regmap_write(rt1320->regmap,
 			SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23,
 				RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x00);
 	rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x00);
 
-	regmap_read(rt1320->regmap, fw_status_addr, &fw_ready);
-	fw_ready &= 0x1;
-	if (!fw_ready) {
+	dspfw_status = rt1320_dspfw_status(rt1320);
+	if (dspfw_status <= 0) {
 		dev_warn(dev, "%s, DSP FW is NOT ready\n", __func__);
 		goto _exit_;
 	}
@@ -1483,22 +1961,48 @@ static int rt1320_rae_load(struct rt1320_sdw_priv *rt1320)
 	request_firmware(&rae_fw, rae_filename, dev);
 	if (rae_fw) {
 
-		/* RAE CRC clear */
-		regmap_write(rt1320->regmap, 0xe80b, 0x0f);
-
-		/* RAE stop & CRC disable */
-		regmap_update_bits(rt1320->regmap, 0xe803, 0xbc, 0x00);
-
-		while (--retry) {
-			regmap_read(rt1320->regmap, 0xe83f, &value);
-			if (value & 0x40)
-				break;
-			usleep_range(1000, 1100);
-		}
-		if (!retry && !(value & 0x40)) {
-			dev_err(dev, "%s: RAE is not ready to load\n", __func__);
-			release_firmware(rae_fw);
-			return -ETIMEDOUT;
+		switch (rt1320->dev_id) {
+		case RT1320_DEV_ID:
+			/* RAE CRC clear */
+			regmap_write(rt1320->regmap, 0xe80b, 0x0f);
+			/* RAE stop & CRC disable */
+			regmap_update_bits(rt1320->regmap, 0xe803, 0xbc, 0x00);
+			while (--retry) {
+				regmap_read(rt1320->regmap, 0xe83f, &value);
+				if (value & 0x40)
+					break;
+				usleep_range(1000, 1100);
+			}
+			if (!retry && !(value & 0x40)) {
+				dev_err(dev, "%s: RAE is not ready to load\n", __func__);
+				release_firmware(rae_fw);
+				return -ETIMEDOUT;
+			}
+			break;
+		case RT1321_DEV_ID:
+			/* RAE CRC clear */
+			regmap_write(rt1320->regmap, 0x2000300e, 0xc0);
+			regmap_write(rt1320->regmap, 0x2000300f, 0x0f);
+			/* RAE stop & Phase sync & CRC disable */
+			regmap_update_bits(rt1320->regmap, 0x20003003, 0xfe, 0x00);
+			regmap_update_bits(rt1320->regmap, 0xc047, 0x80, 0x00);
+			regmap_update_bits(rt1320->regmap, 0x2000301c, 0x01, 0x00);
+			/* check whether write state is ready */
+			while (--retry) {
+				regmap_read(rt1320->regmap, 0x20003043, &value);
+				if (value & 0x40)
+					break;
+				usleep_range(1000, 1100);
+			}
+			if (!retry && !(value & 0x40)) {
+				dev_err(dev, "%s: RAE is not ready to load\n", __func__);
+				release_firmware(rae_fw);
+				return -ETIMEDOUT;
+			}
+			break;
+		default:
+			dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id);
+			return -EINVAL;
 		}
 
 		dev_dbg(dev, "%s, rae_fw size=0x%zx\n", __func__, rae_fw->size);
@@ -1559,18 +2063,34 @@ static int rt1320_rae_load(struct rt1320_sdw_priv *rt1320)
 		goto _exit_;
 	}
 
-	/* RAE CRC enable */
-	regmap_update_bits(rt1320->regmap, 0xe803, 0x0c, 0x0c);
-
-	/* RAE update */
-	regmap_update_bits(rt1320->regmap, 0xe80b, 0x80, 0x00);
-	regmap_update_bits(rt1320->regmap, 0xe80b, 0x80, 0x80);
-
-	/* RAE run */
-	regmap_update_bits(rt1320->regmap, 0xe803, 0x80, 0x80);
-
-	regmap_read(rt1320->regmap, 0xe80b, &value);
-	dev_dbg(dev, "%s: CAE run => 0xe80b reg = 0x%x\n", __func__, value);
+	switch (rt1320->dev_id) {
+	case RT1320_DEV_ID:
+		/* RAE CRC enable */
+		regmap_update_bits(rt1320->regmap, 0xe803, 0x0c, 0x0c);
+		/* RAE update */
+		regmap_update_bits(rt1320->regmap, 0xe80b, 0x80, 0x00);
+		regmap_update_bits(rt1320->regmap, 0xe80b, 0x80, 0x80);
+		/* RAE run */
+		regmap_update_bits(rt1320->regmap, 0xe803, 0x80, 0x80);
+		regmap_read(rt1320->regmap, 0xe80b, &value);
+		dev_dbg(dev, "%s: CAE run => 0xe80b reg = 0x%x\n", __func__, value);
+		break;
+	case RT1321_DEV_ID:
+		/* RAE CRC enable */
+		regmap_update_bits(rt1320->regmap, 0x20003003, 0x30, 0x30);
+		/* RAE update */
+		regmap_update_bits(rt1320->regmap, 0x2000301c, 0x80, 0x00);
+		regmap_update_bits(rt1320->regmap, 0x2000301c, 0x80, 0x80);
+		regmap_update_bits(rt1320->regmap, 0x20009018, 0x80, 0x00);
+		regmap_update_bits(rt1320->regmap, 0x20009018, 0x80, 0x80);
+		regmap_update_bits(rt1320->regmap, 0x20005818, 0x80, 0x00);
+		regmap_update_bits(rt1320->regmap, 0x20005818, 0x80, 0x80);
+		/* RAE run */
+		regmap_update_bits(rt1320->regmap, 0x2000301c, 0x01, 0x01);
+		/* Phase sync eanble */
+		regmap_update_bits(rt1320->regmap, 0xc047, 0x80, 0x80);
+		break;
+	}
 
 	rt1320->rae_update_done = true;
 
@@ -1598,8 +2118,7 @@ struct rt1320_dspfwheader {
 
 	struct snd_soc_dapm_context *dapm = snd_soc_component_to_dapm(rt1320->component);
 	struct device *dev = &rt1320->sdw_slave->dev;
-	unsigned int val, i, fw_offset, fw_ready;
-	unsigned int fw_status_addr;
+	unsigned int val, i, fw_offset;
 	struct rt1320_dspfwheader *fwheader;
 	struct rt1320_imageinfo *ptr_img;
 	struct sdw_bpt_section sec[10];
@@ -1610,20 +2129,10 @@ struct rt1320_dspfwheader {
 	unsigned int hdr_size = 0;
 	const char *dmi_vendor, *dmi_product, *dmi_sku;
 	int len_vendor, len_product, len_sku;
+	unsigned char boot_mode = 0; /* 0: from RAM; 1: from ROM */
+	unsigned char has_0x3fc00000 = 0;
 	char filename[512];
 
-	switch (rt1320->dev_id) {
-	case RT1320_DEV_ID:
-		fw_status_addr = RT1320_DSPFW_STATUS_ADDR;
-		break;
-	case RT1321_DEV_ID:
-		fw_status_addr = RT1321_DSPFW_STATUS_ADDR;
-		break;
-	default:
-		dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id);
-		return;
-	}
-
 	dmi_vendor = dmi_get_system_info(DMI_SYS_VENDOR);
 	dmi_product = dmi_get_system_info(DMI_PRODUCT_NAME);
 	dmi_sku = dmi_get_system_info(DMI_PRODUCT_SKU);
@@ -1653,17 +2162,12 @@ struct rt1320_dspfwheader {
 				RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x00);
 	rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x00);
 
-	regmap_read(rt1320->regmap, fw_status_addr, &fw_ready);
-	fw_ready &= 0x1;
-	if (fw_ready) {
+	if (rt1320_dspfw_status(rt1320)) {
 		dev_dbg(dev, "%s, DSP FW was already\n", __func__);
 		rt1320->fw_load_done = true;
 		goto _exit_;
 	}
 
-	/* change to IRAM */
-	regmap_update_bits(rt1320->regmap, 0xf01e, 0x80, 0x00);
-
 	request_firmware(&fw, filename, dev);
 	if (fw) {
 		fwheader = (struct rt1320_dspfwheader *)fw->data;
@@ -1709,9 +2213,11 @@ struct rt1320_dspfwheader {
 						dev_fw_match = true;
 				break;
 			case RT1321_DEV_ID:
-				if (ptr_img->addr == 0x3fc00000)
-					if (fw_data[9] == '1')
+				if (ptr_img->addr == 0x3fc00000) {
+					if (fw_data[7] == '1')
 						dev_fw_match = true;
+					has_0x3fc00000 = 1;
+				}
 				break;
 			default:
 				dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id);
@@ -1721,6 +2227,22 @@ struct rt1320_dspfwheader {
 			fw_offset += ptr_img->size;
 		}
 
+		if (rt1320->dev_id == RT1321_DEV_ID && rt1320->version_id == RT1321_VA2) {
+			/*
+			 * For 1321 VA2, if the FW doesn't include the section for address 0x3fc00000,
+			 * it means the FW will boot from ROM and force dev_fw_match to true to download FW by BRA.
+			 */
+			if (!has_0x3fc00000) {
+				boot_mode = 1;
+				dev_fw_match = true;
+			}
+			dev_dbg(dev, "%s: Boot from %s for VA2\n", __func__, (boot_mode ? "ROM" : "RAM"));
+		}
+
+		/* change to IRAM */
+		if (!boot_mode)
+			regmap_update_bits(rt1320->regmap, 0xf01e, 0x80, 0x00);
+
 		if (dev_fw_match) {
 			dev_dbg(dev, "%s, starting BRA downloading FW..\n", __func__);
 			rt1320->bra_msg.dev_num = rt1320->sdw_slave->dev_num;
@@ -1743,24 +2265,39 @@ struct rt1320_dspfwheader {
 		goto _exit_;
 	}
 
-	/* run RAM code */
-	regmap_read(rt1320->regmap, 0x3fc2bfc0, &val);
-	val |= 0x8;
-	regmap_write(rt1320->regmap, 0x3fc2bfc0, val);
-
-	/* clear frame counter */
 	switch (rt1320->dev_id) {
 	case RT1320_DEV_ID:
+		/* run RAM code */
+		regmap_read(rt1320->regmap, 0x3fc2bfc0, &val);
+		val |= 0x8;
+		regmap_write(rt1320->regmap, 0x3fc2bfc0, val);
+		/* clear frame counter */
 		regmap_write(rt1320->regmap, 0x3fc2bfcb, 0x00);
 		regmap_write(rt1320->regmap, 0x3fc2bfca, 0x00);
 		regmap_write(rt1320->regmap, 0x3fc2bfc9, 0x00);
 		regmap_write(rt1320->regmap, 0x3fc2bfc8, 0x00);
 		break;
 	case RT1321_DEV_ID:
+		if (!boot_mode) {
+			/* run RAM code */
+			regmap_read(rt1320->regmap, 0x3fc2dfc0, &val);
+			val |= 0x8;
+			regmap_write(rt1320->regmap, 0x3fc2dfc0, val);
+		}
+		/* clear frame counter */
 		regmap_write(rt1320->regmap, 0x3fc2dfcb, 0x00);
 		regmap_write(rt1320->regmap, 0x3fc2dfca, 0x00);
 		regmap_write(rt1320->regmap, 0x3fc2dfc9, 0x00);
 		regmap_write(rt1320->regmap, 0x3fc2dfc8, 0x00);
+		/* enable handshake */
+		regmap_write(rt1320->regmap, 0x3fc2dfc4, 0x00);
+		regmap_write(rt1320->regmap, 0xd470, 0xad);
+		/* minimum phase settings */
+		regmap_write(rt1320->regmap, 0xc5c4, 0x10);
+		regmap_write(rt1320->regmap, 0x20003003, 0x31);
+		regmap_update_bits(rt1320->regmap, 0x20003002, 0x40, 0x00);
+		regmap_write(rt1320->regmap, 0xc5b3, 0x01);
+		regmap_write(rt1320->regmap, 0xc052, 0x11);
 		break;
 	}
 
@@ -1843,14 +2380,35 @@ static void rt1320_vc_preset(struct rt1320_sdw_priv *rt1320)
 
 static void rt1321_preset(struct rt1320_sdw_priv *rt1320)
 {
+	const struct reg_sequence *blindwrite;
 	unsigned int i, reg, val, delay;
+	unsigned int array_size;
 
-	for (i = 0; i < ARRAY_SIZE(rt1321_blind_write); i++) {
-		reg = rt1321_blind_write[i].reg;
-		val = rt1321_blind_write[i].def;
-		delay = rt1321_blind_write[i].delay_us;
+	switch (rt1320->version_id) {
+	case RT1321_VA0:
+		blindwrite = rt1321_blind_write;
+		array_size = ARRAY_SIZE(rt1321_blind_write);
+		break;
+	case RT1321_VA1:
+		blindwrite = rt1321_va1_blind_write;
+		array_size = ARRAY_SIZE(rt1321_va1_blind_write);
+		break;
+	case RT1321_VA2:
+		blindwrite = rt1321_va2_blind_write;
+		array_size = ARRAY_SIZE(rt1321_va2_blind_write);
+		break;
+	default:
+		dev_err(&rt1320->sdw_slave->dev, "%s: Unknown version ID %d\n",
+			__func__, rt1320->version_id);
+		return;
+	}
 
-		if (reg == 0x3fc2dfc3)
+	for (i = 0; i < array_size; i++) {
+		reg = blindwrite[i].reg;
+		val = blindwrite[i].def;
+		delay = blindwrite[i].delay_us;
+
+		if (reg == 0x1000cd56)
 			rt1320_load_mcu_patch(rt1320);
 
 		regmap_write(rt1320->regmap, reg, val);
@@ -1892,6 +2450,24 @@ static int rt1320_io_init(struct device *dev, struct sdw_slave *slave)
 		regmap_read(rt1320->regmap, RT1320_DEV_ID_0, &val);
 		regmap_read(rt1320->regmap, RT1320_DEV_ID_1, &tmp);
 		rt1320->dev_id = (val << 8) | tmp;
+
+		/* This is a workaround that reads the value twice to obtain the correct result. */
+		rt1320_pr_read(rt1320, RT1320_HV_DEV_ID_0, &val);
+		rt1320_pr_read(rt1320, RT1320_HV_DEV_ID_1, &tmp);
+		rt1320_pr_read(rt1320, RT1320_HV_DEV_ID_0, &val);
+		rt1320_pr_read(rt1320, RT1320_HV_DEV_ID_1, &tmp);
+		val = (val << 8) | tmp;
+
+		if (rt1320->dev_id == RT1321_DEV_ID) {
+			if (rt1320->version_id == 0x01)
+				rt1320->version_id = RT1321_VA2;
+			else if (val == RT1321_DEV_HV_VA0_ID)
+				rt1320->version_id = RT1321_VA0;
+			else if (val == RT1321_DEV_HV_VA1_ID)
+				rt1320->version_id = RT1321_VA1;
+			else
+				dev_err(dev, "%s: Unknown version ID 0x%x for RT1321\n", __func__, rt1320->version_id);
+		}
 	}
 
 	regmap_read(rt1320->regmap,
@@ -2326,25 +2902,12 @@ static const DECLARE_TLV_DB_SCALE(in_vol_tlv, -1725, 75, 0);
 static int rt1320_r0_load(struct rt1320_sdw_priv *rt1320)
 {
 	struct device *dev = regmap_get_device(rt1320->regmap);
-	unsigned int fw_status_addr;
-	unsigned int fw_ready;
+	int dspfw_status;
 	int ret = 0;
 
 	if (!rt1320->r0_l_reg || !rt1320->r0_r_reg)
 		return -EINVAL;
 
-	switch (rt1320->dev_id) {
-	case RT1320_DEV_ID:
-		fw_status_addr = RT1320_DSPFW_STATUS_ADDR;
-		break;
-	case RT1321_DEV_ID:
-		fw_status_addr = RT1321_DSPFW_STATUS_ADDR;
-		break;
-	default:
-		dev_err(dev, "%s: Unknown device ID %d\n", __func__, rt1320->dev_id);
-		return -EINVAL;
-	}
-
 	regmap_write(rt1320->regmap,
 		SDW_SDCA_CTL(FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, RT1320_SDCA_CTL_REQ_POWER_STATE, 0), 0x00);
 	ret = rt1320_pde_transition_delay(rt1320, FUNC_NUM_AMP, RT1320_SDCA_ENT_PDE23, 0x00);
@@ -2353,9 +2916,8 @@ static int rt1320_r0_load(struct rt1320_sdw_priv *rt1320)
 		goto _timeout_;
 	}
 
-	regmap_read(rt1320->regmap, fw_status_addr, &fw_ready);
-	fw_ready &= 0x1;
-	if (!fw_ready) {
+	dspfw_status = rt1320_dspfw_status(rt1320);
+	if (dspfw_status <= 0) {
 		dev_dbg(dev, "%s, DSP FW is NOT ready\n", __func__);
 		goto _timeout_;
 	}
@@ -2458,7 +3020,7 @@ static int rt1320_dspfw_load_put(struct snd_kcontrol *kcontrol,
 	if (!rt1320->hw_init)
 		return 0;
 
-	ret = pm_runtime_resume(component->dev);
+	ret = pm_runtime_resume_and_get(component->dev);
 	if (ret < 0 && ret != -EACCES)
 		return ret;
 
@@ -2469,6 +3031,8 @@ static int rt1320_dspfw_load_put(struct snd_kcontrol *kcontrol,
 	if (!ucontrol->value.integer.value[0])
 		rt1320->fw_load_done = false;
 
+	pm_runtime_mark_last_busy(component->dev);
+	pm_runtime_put_autosuspend(component->dev);
 	return 0;
 }
 
@@ -3062,10 +3626,23 @@ static int rt1320_dev_resume(struct device *dev)
 		return ret;
 
 	regcache_cache_only(rt1320->regmap, false);
-	regcache_sync(rt1320->regmap);
+	ret = regcache_sync(rt1320->regmap);
+	if (ret)
+		goto err_sync;
+
 	regcache_cache_only(rt1320->mbq_regmap, false);
-	regcache_sync(rt1320->mbq_regmap);
+	ret = regcache_sync(rt1320->mbq_regmap);
+	if (ret)
+		goto err_sync;
+
 	return 0;
+
+err_sync:
+	regcache_cache_only(rt1320->regmap, true);
+	regcache_cache_only(rt1320->mbq_regmap, true);
+	regcache_mark_dirty(rt1320->regmap);
+	regcache_mark_dirty(rt1320->mbq_regmap);
+	return ret;
 }
 
 static const struct dev_pm_ops rt1320_pm = {
diff --git a/sound/soc/codecs/rt1320-sdw.h b/sound/soc/codecs/rt1320-sdw.h
index a7b573883dd04..bc0f78e03529b 100644
--- a/sound/soc/codecs/rt1320-sdw.h
+++ b/sound/soc/codecs/rt1320-sdw.h
@@ -17,12 +17,17 @@
 
 #define RT1320_DEV_ID 0x6981
 #define RT1321_DEV_ID 0x7045
+#define RT1321_DEV_HV_VA0_ID 0x6997
+#define RT1321_DEV_HV_VA1_ID 0x7071
 
 /* imp-defined registers */
 #define RT1320_DEV_VERSION_ID_1 0xc404
 #define RT1320_DEV_ID_1 0xc405
 #define RT1320_DEV_ID_0 0xc406
 
+#define RT1320_HV_DEV_ID_0 0xf622
+#define RT1320_HV_DEV_ID_1 0xf623
+
 #define RT1320_POWER_STATE 0xc560
 
 #define RT1321_PATCH_MAIN_VER 0x1000cffe
@@ -94,6 +99,12 @@ enum rt1320_version_id {
 	RT1320_VC,
 };
 
+enum rt1321_version_id {
+	RT1321_VA0,
+	RT1321_VA1,
+	RT1321_VA2,
+};
+
 #define RT1320_VER_B_ID 0x07392238
 #define RT1320_VAB_MCU_PATCH "realtek/rt1320/rt1320-patch-code-vab.bin"
 #define RT1320_VC_MCU_PATCH "realtek/rt1320/rt1320-patch-code-vc.bin"
@@ -121,6 +132,7 @@ struct rt1320_datafixpoint {
 	int invrs;
 };
 
+/* FW parameter id 1300 */
 typedef struct FwPara_HwSwGain {
 	unsigned int SwAdvGain;
 	unsigned int SwBasGain;
diff --git a/sound/soc/codecs/rt5514.c b/sound/soc/codecs/rt5514.c
index 00a4a208d2fa7..58aecd4c83a60 100644
--- a/sound/soc/codecs/rt5514.c
+++ b/sound/soc/codecs/rt5514.c
@@ -1073,12 +1073,18 @@ static int rt5514_set_bias_level(struct snd_soc_component *component,
 			 * settings to make sure recording properly.
 			 */
 			if (rt5514->dsp_enabled) {
-				rt5514->dsp_enabled = 0;
-				regmap_multi_reg_write(rt5514->i2c_regmap,
-					rt5514_i2c_patch,
-					ARRAY_SIZE(rt5514_i2c_patch));
+				ret = regmap_multi_reg_write(rt5514->i2c_regmap,
+							     rt5514_i2c_patch,
+							     ARRAY_SIZE(rt5514_i2c_patch));
+				if (ret)
+					return ret;
+
 				regcache_mark_dirty(rt5514->regmap);
-				regcache_sync(rt5514->regmap);
+				ret = regcache_sync(rt5514->regmap);
+				if (ret)
+					return ret;
+
+				rt5514->dsp_enabled = 0;
 			}
 		}
 		break;
diff --git a/sound/soc/codecs/rt5645.c b/sound/soc/codecs/rt5645.c
index e9819653b30d6..93a4148ccccd6 100644
--- a/sound/soc/codecs/rt5645.c
+++ b/sound/soc/codecs/rt5645.c
@@ -3518,9 +3518,15 @@ static int rt5645_suspend(struct snd_soc_component *component)
 static int rt5645_resume(struct snd_soc_component *component)
 {
 	struct rt5645_priv *rt5645 = snd_soc_component_get_drvdata(component);
+	int ret;
 
 	regcache_cache_only(rt5645->regmap, false);
-	regcache_sync(rt5645->regmap);
+	ret = regcache_sync(rt5645->regmap);
+	if (ret) {
+		regcache_cache_only(rt5645->regmap, true);
+		regcache_mark_dirty(rt5645->regmap);
+		return ret;
+	}
 
 	return 0;
 }
@@ -4331,9 +4337,15 @@ static int rt5645_sys_suspend(struct device *dev)
 static int rt5645_sys_resume(struct device *dev)
 {
 	struct rt5645_priv *rt5645 = dev_get_drvdata(dev);
+	int ret;
 
 	regcache_cache_only(rt5645->regmap, false);
-	regcache_sync(rt5645->regmap);
+	ret = regcache_sync(rt5645->regmap);
+	if (ret) {
+		regcache_cache_only(rt5645->regmap, true);
+		regcache_mark_dirty(rt5645->regmap);
+		return ret;
+	}
 
 	if (rt5645->hp_jack) {
 		rt5645->jack_type = 0;
diff --git a/sound/soc/codecs/rt5677.c b/sound/soc/codecs/rt5677.c
index ac084ca008f3c..73fc008d558ad 100644
--- a/sound/soc/codecs/rt5677.c
+++ b/sound/soc/codecs/rt5677.c
@@ -6,6 +6,7 @@
  * Author: Oder Chiou <oder_chiou@realtek.com>
  */
 
+#include <linux/bits.h>
 #include <linux/delay.h>
 #include <linux/firmware.h>
 #include <linux/fs.h>
@@ -4767,6 +4768,21 @@ static int rt5677_gpio_direction_in(struct gpio_chip *chip, unsigned offset)
 	return rt5677_update_gpio_bits(rt5677, offset, m, v);
 }
 
+static int rt5677_gpio_get_direction(struct gpio_chip *chip, unsigned int offset)
+{
+	struct rt5677_priv *rt5677 = gpiochip_get_data(chip);
+	unsigned int shift = RT5677_GPIOx_DIR_SFT + (offset % 5) * 3;
+	unsigned int bank = offset / 5;
+	unsigned int reg = bank ? RT5677_GPIO_CTRL3 : RT5677_GPIO_CTRL2;
+	int ret;
+
+	ret = regmap_test_bits(rt5677->regmap, reg, BIT(shift));
+	if (ret < 0)
+		return ret;
+
+	return ret ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN;
+}
+
 /*
  * Configures the GPIO as
  *   0 - floating
@@ -4834,6 +4850,7 @@ static int rt5677_to_irq(struct gpio_chip *chip, unsigned offset)
 static const struct gpio_chip rt5677_template_chip = {
 	.label			= RT5677_DRV_NAME,
 	.owner			= THIS_MODULE,
+	.get_direction		= rt5677_gpio_get_direction,
 	.direction_output	= rt5677_gpio_direction_out,
 	.set			= rt5677_gpio_set,
 	.direction_input	= rt5677_gpio_direction_in,
diff --git a/sound/soc/codecs/rt5682-sdw.c b/sound/soc/codecs/rt5682-sdw.c
index dec8c2147d684..e49ec28b61030 100644
--- a/sound/soc/codecs/rt5682-sdw.c
+++ b/sound/soc/codecs/rt5682-sdw.c
@@ -410,7 +410,9 @@ static int rt5682_io_init(struct device *dev, struct sdw_slave *slave)
 	if (rt5682->first_hw_init) {
 		regcache_cache_bypass(rt5682->regmap, false);
 		regcache_mark_dirty(rt5682->regmap);
-		regcache_sync(rt5682->regmap);
+		ret = regcache_sync(rt5682->regmap);
+		if (ret)
+			goto err_sync;
 
 		/* volatile registers */
 		regmap_update_bits(rt5682->regmap, RT5682_CBJ_CTRL_2,
@@ -472,8 +474,16 @@ static int rt5682_io_init(struct device *dev, struct sdw_slave *slave)
 	/* Mark Slave initialization complete */
 	rt5682->hw_init = true;
 	rt5682->first_hw_init = true;
+	goto out;
+
+err_sync:
+	regcache_cache_bypass(rt5682->regmap, false);
+	regcache_cache_only(rt5682->sdw_regmap, true);
+	regcache_cache_only(rt5682->regmap, true);
+	regcache_mark_dirty(rt5682->regmap);
 
 err_nodev:
+out:
 	pm_runtime_put_autosuspend(&slave->dev);
 
 	dev_dbg(&slave->dev, "%s hw_init complete: %d\n", __func__, ret);
@@ -776,7 +786,13 @@ static int rt5682_dev_resume(struct device *dev)
 
 	regcache_cache_only(rt5682->sdw_regmap, false);
 	regcache_cache_only(rt5682->regmap, false);
-	regcache_sync(rt5682->regmap);
+	ret = regcache_sync(rt5682->regmap);
+	if (ret) {
+		regcache_cache_only(rt5682->sdw_regmap, true);
+		regcache_cache_only(rt5682->regmap, true);
+		regcache_mark_dirty(rt5682->regmap);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/rt700-sdw.c b/sound/soc/codecs/rt700-sdw.c
index a451d5d1f8ab5..bb449f08e30cf 100644
--- a/sound/soc/codecs/rt700-sdw.c
+++ b/sound/soc/codecs/rt700-sdw.c
@@ -458,10 +458,8 @@ static void rt700_sdw_remove(struct sdw_slave *slave)
 {
 	struct rt700_priv *rt700 = dev_get_drvdata(&slave->dev);
 
-	if (rt700->hw_init) {
-		cancel_delayed_work_sync(&rt700->jack_detect_work);
-		cancel_delayed_work_sync(&rt700->jack_btn_check_work);
-	}
+	cancel_delayed_work_sync(&rt700->jack_detect_work);
+	cancel_delayed_work_sync(&rt700->jack_btn_check_work);
 
 	pm_runtime_disable(&slave->dev);
 }
diff --git a/sound/soc/codecs/rt712-sdca-dmic.c b/sound/soc/codecs/rt712-sdca-dmic.c
index 85779547653e1..8860d81134e70 100644
--- a/sound/soc/codecs/rt712-sdca-dmic.c
+++ b/sound/soc/codecs/rt712-sdca-dmic.c
@@ -916,10 +916,23 @@ static int rt712_sdca_dmic_dev_resume(struct device *dev)
 	}
 
 	regcache_cache_only(rt712->regmap, false);
-	regcache_sync(rt712->regmap);
+	ret = regcache_sync(rt712->regmap);
+	if (ret)
+		goto err_sync;
+
 	regcache_cache_only(rt712->mbq_regmap, false);
-	regcache_sync(rt712->mbq_regmap);
+	ret = regcache_sync(rt712->mbq_regmap);
+	if (ret)
+		goto err_sync;
+
 	return 0;
+
+err_sync:
+	regcache_cache_only(rt712->regmap, true);
+	regcache_cache_only(rt712->mbq_regmap, true);
+	regcache_mark_dirty(rt712->regmap);
+	regcache_mark_dirty(rt712->mbq_regmap);
+	return ret;
 }
 
 static const struct dev_pm_ops rt712_sdca_dmic_pm = {
diff --git a/sound/soc/codecs/rt712-sdca-sdw.c b/sound/soc/codecs/rt712-sdca-sdw.c
index 70d661ce2ef2e..ebdf7b3083310 100644
--- a/sound/soc/codecs/rt712-sdca-sdw.c
+++ b/sound/soc/codecs/rt712-sdca-sdw.c
@@ -472,10 +472,23 @@ static int rt712_sdca_dev_resume(struct device *dev)
 	}
 
 	regcache_cache_only(rt712->regmap, false);
-	regcache_sync(rt712->regmap);
+	ret = regcache_sync(rt712->regmap);
+	if (ret)
+		goto err_sync;
+
 	regcache_cache_only(rt712->mbq_regmap, false);
-	regcache_sync(rt712->mbq_regmap);
+	ret = regcache_sync(rt712->mbq_regmap);
+	if (ret)
+		goto err_sync;
+
 	return 0;
+
+err_sync:
+	regcache_cache_only(rt712->regmap, true);
+	regcache_cache_only(rt712->mbq_regmap, true);
+	regcache_mark_dirty(rt712->regmap);
+	regcache_mark_dirty(rt712->mbq_regmap);
+	return ret;
 }
 
 static const struct dev_pm_ops rt712_sdca_pm = {
diff --git a/sound/soc/codecs/rt712-sdca.c b/sound/soc/codecs/rt712-sdca.c
index 4796fce084ff4..d6353af07380c 100644
--- a/sound/soc/codecs/rt712-sdca.c
+++ b/sound/soc/codecs/rt712-sdca.c
@@ -1843,6 +1843,15 @@ static void rt712_sdca_vb_io_init(struct rt712_sdca_priv *rt712)
 	}
 }
 
+static void rt712_sdca_reset(struct rt712_sdca_priv *rt712)
+{
+	rt712_sdca_index_update_bits(rt712, RT712_VENDOR_REG,
+		RT712_PARA_VERB_CTL, RT712_HIDDEN_REG_SW_RESET,
+		RT712_HIDDEN_REG_SW_RESET);
+	rt712_sdca_index_update_bits(rt712, RT712_VENDOR_HDA_CTL,
+		RT712_HDA_LEGACY_RESET_CTL, 0x1, 0x1);
+}
+
 int rt712_sdca_io_init(struct device *dev, struct sdw_slave *slave)
 {
 	struct rt712_sdca_priv *rt712 = dev_get_drvdata(dev);
@@ -1870,6 +1879,8 @@ int rt712_sdca_io_init(struct device *dev, struct sdw_slave *slave)
 
 	pm_runtime_get_noresume(&slave->dev);
 
+	rt712_sdca_reset(rt712);
+
 	rt712_sdca_index_read(rt712, RT712_VENDOR_REG, RT712_JD_PRODUCT_NUM, &val);
 	rt712->hw_id = (val & 0xf000) >> 12;
 	rt712->version_id = (val & 0x0f00) >> 8;
diff --git a/sound/soc/codecs/rt721-sdca-sdw.c b/sound/soc/codecs/rt721-sdca-sdw.c
index 041b381e582b4..6e27e761afb63 100644
--- a/sound/soc/codecs/rt721-sdca-sdw.c
+++ b/sound/soc/codecs/rt721-sdca-sdw.c
@@ -510,10 +510,23 @@ static int rt721_sdca_dev_resume(struct device *dev)
 	}
 
 	regcache_cache_only(rt721->regmap, false);
-	regcache_sync(rt721->regmap);
+	ret = regcache_sync(rt721->regmap);
+	if (ret)
+		goto err_sync;
+
 	regcache_cache_only(rt721->mbq_regmap, false);
-	regcache_sync(rt721->mbq_regmap);
+	ret = regcache_sync(rt721->mbq_regmap);
+	if (ret)
+		goto err_sync;
+
 	return 0;
+
+err_sync:
+	regcache_cache_only(rt721->regmap, true);
+	regcache_cache_only(rt721->mbq_regmap, true);
+	regcache_mark_dirty(rt721->regmap);
+	regcache_mark_dirty(rt721->mbq_regmap);
+	return ret;
 }
 
 static const struct dev_pm_ops rt721_sdca_pm = {
diff --git a/sound/soc/codecs/rt722-sdca-sdw.c b/sound/soc/codecs/rt722-sdca-sdw.c
index e68aa0350a5b9..d2db33b4f6843 100644
--- a/sound/soc/codecs/rt722-sdca-sdw.c
+++ b/sound/soc/codecs/rt722-sdca-sdw.c
@@ -557,7 +557,13 @@ static int rt722_sdca_dev_resume(struct device *dev)
 	}
 
 	regcache_cache_only(rt722->regmap, false);
-	regcache_sync(rt722->regmap);
+	ret = regcache_sync(rt722->regmap);
+	if (ret) {
+		regcache_cache_only(rt722->regmap, true);
+		regcache_mark_dirty(rt722->regmap);
+		return ret;
+	}
+
 	return 0;
 }
 
diff --git a/sound/soc/codecs/rt9120.c b/sound/soc/codecs/rt9120.c
index 97f56af25577d..e51cd4a318948 100644
--- a/sound/soc/codecs/rt9120.c
+++ b/sound/soc/codecs/rt9120.c
@@ -606,12 +606,19 @@ static int rt9120_runtime_suspend(struct device *dev)
 static int rt9120_runtime_resume(struct device *dev)
 {
 	struct rt9120_data *data = dev_get_drvdata(dev);
+	int ret;
 
 	if (data->pwdnn_gpio) {
 		gpiod_set_value(data->pwdnn_gpio, 1);
 		msleep(RT9120_CHIPON_WAITMS);
 		regcache_cache_only(data->regmap, false);
-		regcache_sync(data->regmap);
+		ret = regcache_sync(data->regmap);
+		if (ret) {
+			regcache_cache_only(data->regmap, true);
+			regcache_mark_dirty(data->regmap);
+			gpiod_set_value(data->pwdnn_gpio, 0);
+			return ret;
+		}
 	}
 
 	return 0;
diff --git a/sound/soc/codecs/sma1303.c b/sound/soc/codecs/sma1303.c
index 7fce60de5e5fc..15758eb13f8c5 100644
--- a/sound/soc/codecs/sma1303.c
+++ b/sound/soc/codecs/sma1303.c
@@ -1777,6 +1777,8 @@ static void sma1303_i2c_remove(struct i2c_client *client)
 	struct sma1303_priv *sma1303 =
 		(struct sma1303_priv *) i2c_get_clientdata(client);
 
+	if (sma1303->attr_grp)
+		sysfs_remove_group(sma1303->kobj, sma1303->attr_grp);
 	cancel_delayed_work_sync(&sma1303->check_fault_work);
 }
 
diff --git a/sound/soc/codecs/tas2552.c b/sound/soc/codecs/tas2552.c
index 1a7650b9b2a7b..22f93e29454fe 100644
--- a/sound/soc/codecs/tas2552.c
+++ b/sound/soc/codecs/tas2552.c
@@ -495,13 +495,21 @@ static int tas2552_runtime_suspend(struct device *dev)
 static int tas2552_runtime_resume(struct device *dev)
 {
 	struct tas2552_data *tas2552 = dev_get_drvdata(dev);
+	int ret;
 
 	gpiod_set_value_cansleep(tas2552->enable_gpio, 1);
 
 	tas2552_sw_shutdown(tas2552, 0);
 
 	regcache_cache_only(tas2552->regmap, false);
-	regcache_sync(tas2552->regmap);
+	ret = regcache_sync(tas2552->regmap);
+	if (ret) {
+		regcache_cache_only(tas2552->regmap, true);
+		regcache_mark_dirty(tas2552->regmap);
+		tas2552_sw_shutdown(tas2552, 1);
+		gpiod_set_value_cansleep(tas2552->enable_gpio, 0);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/tas2562.c b/sound/soc/codecs/tas2562.c
index 2f7cfc2be970b..e1d62f30418ac 100644
--- a/sound/soc/codecs/tas2562.c
+++ b/sound/soc/codecs/tas2562.c
@@ -675,11 +675,12 @@ static int tas2562_parse_dt(struct tas2562_data *tas2562)
 	if (tas2562->sdz_gpio == NULL) {
 		tas2562->sdz_gpio = devm_gpiod_get_optional(dev, "shut-down",
 							      GPIOD_OUT_HIGH);
-		if (IS_ERR(tas2562->sdz_gpio))
+		if (IS_ERR(tas2562->sdz_gpio)) {
 			if (PTR_ERR(tas2562->sdz_gpio) == -EPROBE_DEFER)
 				return -EPROBE_DEFER;
 
-		tas2562->sdz_gpio = NULL;
+			tas2562->sdz_gpio = NULL;
+		}
 	}
 
 	if (tas2562->model_id == TAS2110)
diff --git a/sound/soc/codecs/tas2781-fmwlib.c b/sound/soc/codecs/tas2781-fmwlib.c
index bd16d5326a23d..dcbeb9618195b 100644
--- a/sound/soc/codecs/tas2781-fmwlib.c
+++ b/sound/soc/codecs/tas2781-fmwlib.c
@@ -12,6 +12,7 @@
 #include <linux/i2c.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
+#include <linux/limits.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_irq.h>
@@ -1100,13 +1101,42 @@ static int tasdevice_load_block_kernel(
 	return 0;
 }
 
+static int tasdevice_fw_strnlen(const struct firmware *fmw, int offset)
+{
+	const u8 *start;
+	const u8 *nul;
+	size_t remaining;
+	size_t len;
+
+	if (offset < 0 || offset >= fmw->size)
+		return -EINVAL;
+
+	start = fmw->data + offset;
+	remaining = fmw->size - offset;
+	nul = memchr(start, '\0', remaining);
+	if (!nul)
+		return -EINVAL;
+
+	len = nul - start;
+	if (len > INT_MAX)
+		return -EOVERFLOW;
+
+	return len;
+}
+
 static int fw_parse_variable_hdr(struct tasdevice_priv
 	*tas_priv, struct tasdevice_dspfw_hdr *fw_hdr,
 	const struct firmware *fmw, int offset)
 {
 	const unsigned char *buf = fmw->data;
-	int len = strlen((char *)&buf[offset]);
+	int len;
 
+	len = tasdevice_fw_strnlen(fmw, offset);
+	if (len < 0) {
+		dev_err(tas_priv->dev, "%s: Description error\n", __func__);
+		offset = len;
+		goto out;
+	}
 	len++;
 
 	if (offset + len + 8 > fmw->size) {
@@ -1238,7 +1268,12 @@ static int fw_parse_data(struct tasdevice_fw *tas_fmw,
 	memcpy(img_data->name, &data[offset], 64);
 	offset += 64;
 
-	n = strlen((char *)&data[offset]);
+	n = tasdevice_fw_strnlen(fmw, offset);
+	if (n < 0) {
+		dev_err(tas_fmw->dev, "%s: Description error\n", __func__);
+		offset = n;
+		goto out;
+	}
 	n++;
 	if (offset + n + 2 > fmw->size) {
 		dev_err(tas_fmw->dev, "%s: Description error\n", __func__);
@@ -1309,7 +1344,12 @@ static int fw_parse_program_data(struct tasdevice_priv *tas_priv,
 		}
 		offset += 64;
 
-		n = strlen((char *)&buf[offset]);
+		n = tasdevice_fw_strnlen(fmw, offset);
+		if (n < 0) {
+			dev_err(tas_priv->dev, "Description err\n");
+			offset = n;
+			goto out;
+		}
 		/* skip '\0' and 5 unused bytes */
 		n += 6;
 		if (offset + n > fmw->size) {
@@ -1372,7 +1412,12 @@ static int fw_parse_configuration_data(
 		memcpy(config->name, &data[offset], 64);
 		offset += 64;
 
-		n = strlen((char *)&data[offset]);
+		n = tasdevice_fw_strnlen(fmw, offset);
+		if (n < 0) {
+			dev_err(tas_priv->dev, "Description err\n");
+			offset = n;
+			goto out;
+		}
 		n += 15;
 		if (offset + n > fmw->size) {
 			dev_err(tas_priv->dev, "Description err\n");
@@ -2134,7 +2179,8 @@ static int fw_parse_calibration_data(struct tasdevice_priv *tas_priv,
 {
 	struct tasdevice_calibration *calibration;
 	unsigned char *data = (unsigned char *)fmw->data;
-	unsigned int i, n;
+	unsigned int i;
+	int n;
 
 	if (offset + 2 > fmw->size) {
 		dev_err(tas_priv->dev, "%s: Calibrations error\n", __func__);
@@ -2166,7 +2212,12 @@ static int fw_parse_calibration_data(struct tasdevice_priv *tas_priv,
 		calibration = &(tas_fmw->calibrations[i]);
 		offset += 64;
 
-		n = strlen((char *)&data[offset]);
+		n = tasdevice_fw_strnlen(fmw, offset);
+		if (n < 0) {
+			dev_err(tas_priv->dev, "Description err\n");
+			offset = n;
+			goto out;
+		}
 		/* skip '\0' and 2 unused bytes */
 		n += 3;
 		if (offset + n > fmw->size) {
diff --git a/sound/soc/codecs/tas2783-sdw.c b/sound/soc/codecs/tas2783-sdw.c
index 3d0b116544cc4..db58c50e8a83d 100644
--- a/sound/soc/codecs/tas2783-sdw.c
+++ b/sound/soc/codecs/tas2783-sdw.c
@@ -1099,7 +1099,13 @@ static s32 tas2783_sdca_dev_resume(struct device *dev)
 	}
 
 	regcache_cache_only(tas_dev->regmap, false);
-	regcache_sync(tas_dev->regmap);
+	ret = regcache_sync(tas_dev->regmap);
+	if (ret) {
+		regcache_cache_only(tas_dev->regmap, true);
+		regcache_mark_dirty(tas_dev->regmap);
+		return ret;
+	}
+
 	return 0;
 }
 
@@ -1210,6 +1216,7 @@ static s32 tas_update_status(struct sdw_slave *slave,
 {
 	struct tas2783_prv *tas_dev = dev_get_drvdata(&slave->dev);
 	struct device *dev = &slave->dev;
+	int ret;
 
 	dev_dbg(dev, "Peripheral status = %s",
 		status == SDW_SLAVE_UNATTACHED ? "unattached" :
@@ -1227,7 +1234,12 @@ static s32 tas_update_status(struct sdw_slave *slave,
 
 	/* updated the cache data to device */
 	regcache_cache_only(tas_dev->regmap, false);
-	regcache_sync(tas_dev->regmap);
+	ret = regcache_sync(tas_dev->regmap);
+	if (ret) {
+		regcache_cache_only(tas_dev->regmap, true);
+		regcache_mark_dirty(tas_dev->regmap);
+		return ret;
+	}
 
 	/* perform I/O transfers required for Slave initialization */
 	return tas_io_init(&slave->dev, slave);
diff --git a/sound/soc/codecs/tlv320aic26.c b/sound/soc/codecs/tlv320aic26.c
index e5dfb3d752a3a..84e954311c1b2 100644
--- a/sound/soc/codecs/tlv320aic26.c
+++ b/sound/soc/codecs/tlv320aic26.c
@@ -320,8 +320,14 @@ static int aic26_probe(struct snd_soc_component *component)
 	return 0;
 }
 
+static void aic26_remove(struct snd_soc_component *component)
+{
+	device_remove_file(component->dev, &dev_attr_keyclick);
+}
+
 static const struct snd_soc_component_driver aic26_soc_component_dev = {
 	.probe			= aic26_probe,
+	.remove			= aic26_remove,
 	.controls		= aic26_snd_controls,
 	.num_controls		= ARRAY_SIZE(aic26_snd_controls),
 	.dapm_widgets		= tlv320aic26_dapm_widgets,
diff --git a/sound/soc/codecs/uda1342.c b/sound/soc/codecs/uda1342.c
index 12f5757f4210d..4e2d6240a8695 100644
--- a/sound/soc/codecs/uda1342.c
+++ b/sound/soc/codecs/uda1342.c
@@ -308,9 +308,16 @@ static int uda1342_suspend(struct device *dev)
 static int uda1342_resume(struct device *dev)
 {
 	struct uda1342_priv *uda1342 = dev_get_drvdata(dev);
+	int ret;
 
+	regcache_cache_only(uda1342->regmap, false);
 	regcache_mark_dirty(uda1342->regmap);
-	regcache_sync(uda1342->regmap);
+	ret = regcache_sync(uda1342->regmap);
+	if (ret) {
+		regcache_cache_only(uda1342->regmap, true);
+		regcache_mark_dirty(uda1342->regmap);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/wcd937x-sdw.c b/sound/soc/codecs/wcd937x-sdw.c
index 7a18bed7f3471..1ad75c226060e 100644
--- a/sound/soc/codecs/wcd937x-sdw.c
+++ b/sound/soc/codecs/wcd937x-sdw.c
@@ -1084,10 +1084,16 @@ static int wcd937x_sdw_runtime_suspend(struct device *dev)
 static int wcd937x_sdw_runtime_resume(struct device *dev)
 {
 	struct wcd937x_sdw_priv *wcd = dev_get_drvdata(dev);
+	int ret;
 
 	if (wcd->regmap) {
 		regcache_cache_only(wcd->regmap, false);
-		regcache_sync(wcd->regmap);
+		ret = regcache_sync(wcd->regmap);
+		if (ret) {
+			regcache_cache_only(wcd->regmap, true);
+			regcache_mark_dirty(wcd->regmap);
+			return ret;
+		}
 	}
 
 	return 0;
diff --git a/sound/soc/codecs/wcd938x-sdw.c b/sound/soc/codecs/wcd938x-sdw.c
index 0f0cc0ac3056e..e5ad368418a2c 100644
--- a/sound/soc/codecs/wcd938x-sdw.c
+++ b/sound/soc/codecs/wcd938x-sdw.c
@@ -1245,10 +1245,16 @@ static int wcd938x_sdw_runtime_suspend(struct device *dev)
 static int wcd938x_sdw_runtime_resume(struct device *dev)
 {
 	struct wcd938x_sdw_priv *wcd = dev_get_drvdata(dev);
+	int ret;
 
 	if (wcd->regmap) {
 		regcache_cache_only(wcd->regmap, false);
-		regcache_sync(wcd->regmap);
+		ret = regcache_sync(wcd->regmap);
+		if (ret) {
+			regcache_cache_only(wcd->regmap, true);
+			regcache_mark_dirty(wcd->regmap);
+			return ret;
+		}
 	}
 
 	pm_runtime_mark_last_busy(dev);
diff --git a/sound/soc/codecs/wcd939x-sdw.c b/sound/soc/codecs/wcd939x-sdw.c
index 95f4be287e79b..69b9b08118f0e 100644
--- a/sound/soc/codecs/wcd939x-sdw.c
+++ b/sound/soc/codecs/wcd939x-sdw.c
@@ -1431,10 +1431,16 @@ static int wcd939x_sdw_runtime_suspend(struct device *dev)
 static int wcd939x_sdw_runtime_resume(struct device *dev)
 {
 	struct wcd939x_sdw_priv *wcd = dev_get_drvdata(dev);
+	int ret;
 
 	if (wcd->regmap) {
 		regcache_cache_only(wcd->regmap, false);
-		regcache_sync(wcd->regmap);
+		ret = regcache_sync(wcd->regmap);
+		if (ret) {
+			regcache_cache_only(wcd->regmap, true);
+			regcache_mark_dirty(wcd->regmap);
+			return ret;
+		}
 	}
 
 	return 0;
diff --git a/sound/soc/codecs/wm2200.c b/sound/soc/codecs/wm2200.c
index ba8ce2e6e6150..bc78b553b6cbe 100644
--- a/sound/soc/codecs/wm2200.c
+++ b/sound/soc/codecs/wm2200.c
@@ -2461,7 +2461,15 @@ static int wm2200_runtime_resume(struct device *dev)
 	}
 
 	regcache_cache_only(wm2200->regmap, false);
-	regcache_sync(wm2200->regmap);
+	ret = regcache_sync(wm2200->regmap);
+	if (ret) {
+		regcache_cache_only(wm2200->regmap, true);
+		regcache_mark_dirty(wm2200->regmap);
+		gpiod_set_value_cansleep(wm2200->ldo_ena, 0);
+		regulator_bulk_disable(ARRAY_SIZE(wm2200->core_supplies),
+				       wm2200->core_supplies);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/wm5100.c b/sound/soc/codecs/wm5100.c
index bd94fa53c3624..2fe937a93026c 100644
--- a/sound/soc/codecs/wm5100.c
+++ b/sound/soc/codecs/wm5100.c
@@ -2659,7 +2659,15 @@ static int wm5100_runtime_resume(struct device *dev)
 	}
 
 	regcache_cache_only(wm5100->regmap, false);
-	regcache_sync(wm5100->regmap);
+	ret = regcache_sync(wm5100->regmap);
+	if (ret) {
+		regcache_cache_only(wm5100->regmap, true);
+		regcache_mark_dirty(wm5100->regmap);
+		gpiod_set_value_cansleep(wm5100->ldo_ena, 0);
+		regulator_bulk_disable(ARRAY_SIZE(wm5100->core_supplies),
+				       wm5100->core_supplies);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/wm8962.c b/sound/soc/codecs/wm8962.c
index de18b1f85a32c..2db822fc1de7e 100644
--- a/sound/soc/codecs/wm8962.c
+++ b/sound/soc/codecs/wm8962.c
@@ -3937,7 +3937,9 @@ static int wm8962_runtime_resume(struct device *dev)
 			   WM8962_OSC_ENA | WM8962_PLL2_ENA | WM8962_PLL3_ENA,
 			   0);
 
-	regcache_sync(wm8962->regmap);
+	ret = regcache_sync(wm8962->regmap);
+	if (ret)
+		goto cache_sync_err;
 
 	regmap_update_bits(wm8962->regmap, WM8962_ANTI_POP,
 			   WM8962_STARTUP_BIAS_ENA | WM8962_VMID_BUF_ENA,
@@ -3955,6 +3957,11 @@ static int wm8962_runtime_resume(struct device *dev)
 
 	return 0;
 
+cache_sync_err:
+	regcache_cache_only(wm8962->regmap, true);
+	regcache_mark_dirty(wm8962->regmap);
+	regulator_bulk_disable(ARRAY_SIZE(wm8962->supplies),
+			       wm8962->supplies);
 disable_clock:
 	clk_disable_unprepare(wm8962->pdata.mclk);
 	return ret;
diff --git a/sound/soc/codecs/wsa881x.c b/sound/soc/codecs/wsa881x.c
index 5174614c3e837..47ac3e4985a56 100644
--- a/sound/soc/codecs/wsa881x.c
+++ b/sound/soc/codecs/wsa881x.c
@@ -1178,7 +1178,13 @@ static int wsa881x_runtime_resume(struct device *dev)
 	}
 
 	regcache_cache_only(regmap, false);
-	regcache_sync(regmap);
+	ret = regcache_sync(regmap);
+	if (ret) {
+		regcache_cache_only(regmap, true);
+		regcache_mark_dirty(regmap);
+		gpiod_direction_output(wsa881x->sd_n, 1);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/wsa883x.c b/sound/soc/codecs/wsa883x.c
index 468d2b38a22ab..24a5904d8e6c1 100644
--- a/sound/soc/codecs/wsa883x.c
+++ b/sound/soc/codecs/wsa883x.c
@@ -1696,9 +1696,15 @@ static int wsa883x_runtime_suspend(struct device *dev)
 static int wsa883x_runtime_resume(struct device *dev)
 {
 	struct regmap *regmap = dev_get_regmap(dev, NULL);
+	int ret;
 
 	regcache_cache_only(regmap, false);
-	regcache_sync(regmap);
+	ret = regcache_sync(regmap);
+	if (ret) {
+		regcache_cache_only(regmap, true);
+		regcache_mark_dirty(regmap);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/codecs/wsa884x.c b/sound/soc/codecs/wsa884x.c
index 6c6b497657d0c..567861dd42ad6 100644
--- a/sound/soc/codecs/wsa884x.c
+++ b/sound/soc/codecs/wsa884x.c
@@ -2147,9 +2147,15 @@ static int wsa884x_runtime_suspend(struct device *dev)
 static int wsa884x_runtime_resume(struct device *dev)
 {
 	struct regmap *regmap = dev_get_regmap(dev, NULL);
+	int ret;
 
 	regcache_cache_only(regmap, false);
-	regcache_sync(regmap);
+	ret = regcache_sync(regmap);
+	if (ret) {
+		regcache_cache_only(regmap, true);
+		regcache_mark_dirty(regmap);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/sound/soc/fsl/fsl_asrc.c b/sound/soc/fsl/fsl_asrc.c
index 5fda9b647c704..0b28bcfa47fed 100644
--- a/sound/soc/fsl/fsl_asrc.c
+++ b/sound/soc/fsl/fsl_asrc.c
@@ -222,10 +222,9 @@ static int fsl_asrc_request_pair(int channels, struct fsl_asrc_pair *pair)
 	enum asrc_pair_index index = ASRC_INVALID_PAIR;
 	struct fsl_asrc *asrc = pair->asrc;
 	struct device *dev = &asrc->pdev->dev;
-	unsigned long lock_flags;
 	int i, ret = 0;
 
-	spin_lock_irqsave(&asrc->lock, lock_flags);
+	guard(spinlock_irqsave)(&asrc->lock);
 
 	for (i = ASRC_PAIR_A; i < ASRC_PAIR_MAX_NUM; i++) {
 		if (asrc->pair[i] != NULL)
@@ -250,8 +249,6 @@ static int fsl_asrc_request_pair(int channels, struct fsl_asrc_pair *pair)
 		pair->index = index;
 	}
 
-	spin_unlock_irqrestore(&asrc->lock, lock_flags);
-
 	return ret;
 }
 
@@ -265,19 +262,16 @@ static void fsl_asrc_release_pair(struct fsl_asrc_pair *pair)
 {
 	struct fsl_asrc *asrc = pair->asrc;
 	enum asrc_pair_index index = pair->index;
-	unsigned long lock_flags;
 
 	/* Make sure the pair is disabled */
 	regmap_update_bits(asrc->regmap, REG_ASRCTR,
 			   ASRCTR_ASRCEi_MASK(index), 0);
 
-	spin_lock_irqsave(&asrc->lock, lock_flags);
+	guard(spinlock_irqsave)(&asrc->lock);
 
 	asrc->channel_avail += pair->channels;
 	asrc->pair[index] = NULL;
 	pair->error = 0;
-
-	spin_unlock_irqrestore(&asrc->lock, lock_flags);
 }
 
 /**
diff --git a/sound/soc/fsl/fsl_audmix.c b/sound/soc/fsl/fsl_audmix.c
index f819f33ec46b8..d4140e170edf8 100644
--- a/sound/soc/fsl/fsl_audmix.c
+++ b/sound/soc/fsl/fsl_audmix.c
@@ -286,7 +286,6 @@ static int fsl_audmix_dai_trigger(struct snd_pcm_substream *substream, int cmd,
 				  struct snd_soc_dai *dai)
 {
 	struct fsl_audmix *priv = snd_soc_dai_get_drvdata(dai);
-	unsigned long lock_flags;
 
 	/* Capture stream shall not be handled */
 	if (substream->stream == SNDRV_PCM_STREAM_CAPTURE)
@@ -296,16 +295,14 @@ static int fsl_audmix_dai_trigger(struct snd_pcm_substream *substream, int cmd,
 	case SNDRV_PCM_TRIGGER_START:
 	case SNDRV_PCM_TRIGGER_RESUME:
 	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
-		spin_lock_irqsave(&priv->lock, lock_flags);
-		priv->tdms |= BIT(dai->driver->id);
-		spin_unlock_irqrestore(&priv->lock, lock_flags);
+		scoped_guard(spinlock_irqsave, &priv->lock)
+			priv->tdms |= BIT(dai->driver->id);
 		break;
 	case SNDRV_PCM_TRIGGER_STOP:
 	case SNDRV_PCM_TRIGGER_SUSPEND:
 	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
-		spin_lock_irqsave(&priv->lock, lock_flags);
-		priv->tdms &= ~BIT(dai->driver->id);
-		spin_unlock_irqrestore(&priv->lock, lock_flags);
+		scoped_guard(spinlock_irqsave, &priv->lock)
+			priv->tdms &= ~BIT(dai->driver->id);
 		break;
 	default:
 		return -EINVAL;
@@ -457,6 +454,9 @@ static const struct of_device_id fsl_audmix_ids[] = {
 };
 MODULE_DEVICE_TABLE(of, fsl_audmix_ids);
 
+static int fsl_audmix_runtime_resume(struct device *dev);
+static int fsl_audmix_runtime_suspend(struct device *dev);
+
 static int fsl_audmix_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -488,13 +488,25 @@ static int fsl_audmix_probe(struct platform_device *pdev)
 	spin_lock_init(&priv->lock);
 	platform_set_drvdata(pdev, priv);
 	pm_runtime_enable(dev);
+	if (!pm_runtime_enabled(dev)) {
+		ret = fsl_audmix_runtime_resume(dev);
+		if (ret)
+			goto err_disable_pm;
+	}
+
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret < 0)
+		goto err_pm_get_sync;
+
+	/* To enable regmap cache only when runtime PM enabled */
+	pm_runtime_put(dev);
 
 	ret = devm_snd_soc_register_component(dev, &fsl_audmix_component,
 					      fsl_audmix_dai,
 					      ARRAY_SIZE(fsl_audmix_dai));
 	if (ret) {
 		dev_err(dev, "failed to register ASoC DAI\n");
-		goto err_disable_pm;
+		goto err_pm_get_sync;
 	}
 
 	/*
@@ -506,12 +518,15 @@ static int fsl_audmix_probe(struct platform_device *pdev)
 		if (IS_ERR(priv->pdev)) {
 			ret = PTR_ERR(priv->pdev);
 			dev_err(dev, "failed to register platform: %d\n", ret);
-			goto err_disable_pm;
+			goto err_pm_get_sync;
 		}
 	}
 
 	return 0;
 
+err_pm_get_sync:
+	if (!pm_runtime_status_suspended(dev))
+		fsl_audmix_runtime_suspend(dev);
 err_disable_pm:
 	pm_runtime_disable(dev);
 	return ret;
@@ -522,6 +537,8 @@ static void fsl_audmix_remove(struct platform_device *pdev)
 	struct fsl_audmix *priv = dev_get_drvdata(&pdev->dev);
 
 	pm_runtime_disable(&pdev->dev);
+	if (!pm_runtime_status_suspended(&pdev->dev))
+		fsl_audmix_runtime_suspend(&pdev->dev);
 
 	if (priv->pdev)
 		platform_device_unregister(priv->pdev);
diff --git a/sound/soc/fsl/fsl_easrc.c b/sound/soc/fsl/fsl_easrc.c
index 114a6c0b6b730..edfd943197a00 100644
--- a/sound/soc/fsl/fsl_easrc.c
+++ b/sound/soc/fsl/fsl_easrc.c
@@ -1025,7 +1025,6 @@ static int fsl_easrc_config_context(struct fsl_asrc *easrc, unsigned int ctx_id)
 	struct fsl_easrc_ctx_priv *ctx_priv;
 	struct fsl_asrc_pair *ctx;
 	struct device *dev;
-	unsigned long lock_flags;
 	int ret;
 
 	if (!easrc)
@@ -1053,9 +1052,8 @@ static int fsl_easrc_config_context(struct fsl_asrc *easrc, unsigned int ctx_id)
 	if (ret)
 		return ret;
 
-	spin_lock_irqsave(&easrc->lock, lock_flags);
-	ret = fsl_easrc_config_slot(easrc, ctx->index);
-	spin_unlock_irqrestore(&easrc->lock, lock_flags);
+	scoped_guard(spinlock_irqsave, &easrc->lock)
+		ret = fsl_easrc_config_slot(easrc, ctx->index);
 	if (ret)
 		return ret;
 
@@ -1301,13 +1299,12 @@ static int fsl_easrc_request_context(int channels, struct fsl_asrc_pair *ctx)
 	enum asrc_pair_index index = ASRC_INVALID_PAIR;
 	struct fsl_asrc *easrc = ctx->asrc;
 	struct device *dev;
-	unsigned long lock_flags;
 	int ret = 0;
 	int i;
 
 	dev = &easrc->pdev->dev;
 
-	spin_lock_irqsave(&easrc->lock, lock_flags);
+	guard(spinlock_irqsave)(&easrc->lock);
 
 	for (i = ASRC_PAIR_A; i < EASRC_CTX_MAX_NUM; i++) {
 		if (easrc->pair[i])
@@ -1331,8 +1328,6 @@ static int fsl_easrc_request_context(int channels, struct fsl_asrc_pair *ctx)
 		easrc->channel_avail -= channels;
 	}
 
-	spin_unlock_irqrestore(&easrc->lock, lock_flags);
-
 	return ret;
 }
 
@@ -1343,7 +1338,6 @@ static int fsl_easrc_request_context(int channels, struct fsl_asrc_pair *ctx)
  */
 static void fsl_easrc_release_context(struct fsl_asrc_pair *ctx)
 {
-	unsigned long lock_flags;
 	struct fsl_asrc *easrc;
 
 	if (!ctx)
@@ -1351,14 +1345,12 @@ static void fsl_easrc_release_context(struct fsl_asrc_pair *ctx)
 
 	easrc = ctx->asrc;
 
-	spin_lock_irqsave(&easrc->lock, lock_flags);
+	guard(spinlock_irqsave)(&easrc->lock);
 
 	fsl_easrc_release_slot(easrc, ctx->index);
 
 	easrc->channel_avail += ctx->channels;
 	easrc->pair[ctx->index] = NULL;
-
-	spin_unlock_irqrestore(&easrc->lock, lock_flags);
 }
 
 /*
@@ -2292,15 +2284,13 @@ static int fsl_easrc_runtime_suspend(struct device *dev)
 {
 	struct fsl_asrc *easrc = dev_get_drvdata(dev);
 	struct fsl_easrc_priv *easrc_priv = easrc->private;
-	unsigned long lock_flags;
 
 	regcache_cache_only(easrc->regmap, true);
 
 	clk_disable_unprepare(easrc->mem_clk);
 
-	spin_lock_irqsave(&easrc->lock, lock_flags);
-	easrc_priv->firmware_loaded = 0;
-	spin_unlock_irqrestore(&easrc->lock, lock_flags);
+	scoped_guard(spinlock_irqsave, &easrc->lock)
+		easrc_priv->firmware_loaded = 0;
 
 	return 0;
 }
@@ -2311,7 +2301,6 @@ static int fsl_easrc_runtime_resume(struct device *dev)
 	struct fsl_easrc_priv *easrc_priv = easrc->private;
 	struct fsl_easrc_ctx_priv *ctx_priv;
 	struct fsl_asrc_pair *ctx;
-	unsigned long lock_flags;
 	int ret;
 	int i;
 
@@ -2323,13 +2312,11 @@ static int fsl_easrc_runtime_resume(struct device *dev)
 	regcache_mark_dirty(easrc->regmap);
 	regcache_sync(easrc->regmap);
 
-	spin_lock_irqsave(&easrc->lock, lock_flags);
-	if (easrc_priv->firmware_loaded) {
-		spin_unlock_irqrestore(&easrc->lock, lock_flags);
-		goto skip_load;
+	scoped_guard(spinlock_irqsave, &easrc->lock) {
+		if (easrc_priv->firmware_loaded)
+			return 0;
+		easrc_priv->firmware_loaded = 1;
 	}
-	easrc_priv->firmware_loaded = 1;
-	spin_unlock_irqrestore(&easrc->lock, lock_flags);
 
 	ret = fsl_easrc_get_firmware(easrc);
 	if (ret) {
@@ -2377,9 +2364,6 @@ static int fsl_easrc_runtime_resume(struct device *dev)
 			goto disable_mem_clk;
 	}
 
-skip_load:
-	return 0;
-
 disable_mem_clk:
 	clk_disable_unprepare(easrc->mem_clk);
 	return ret;
diff --git a/sound/soc/fsl/fsl_esai.c b/sound/soc/fsl/fsl_esai.c
index cde0b0c6c1ef9..4a530a6c33f07 100644
--- a/sound/soc/fsl/fsl_esai.c
+++ b/sound/soc/fsl/fsl_esai.c
@@ -709,10 +709,9 @@ static void fsl_esai_hw_reset(struct work_struct *work)
 {
 	struct fsl_esai *esai_priv = container_of(work, struct fsl_esai, work);
 	bool tx = true, rx = false, enabled[2];
-	unsigned long lock_flags;
 	u32 tfcr, rfcr;
 
-	spin_lock_irqsave(&esai_priv->lock, lock_flags);
+	guard(spinlock_irqsave)(&esai_priv->lock);
 	/* Save the registers */
 	regmap_read(esai_priv->regmap, REG_ESAI_TFCR, &tfcr);
 	regmap_read(esai_priv->regmap, REG_ESAI_RFCR, &rfcr);
@@ -750,8 +749,6 @@ static void fsl_esai_hw_reset(struct work_struct *work)
 		fsl_esai_trigger_start(esai_priv, tx);
 	if (enabled[rx])
 		fsl_esai_trigger_start(esai_priv, rx);
-
-	spin_unlock_irqrestore(&esai_priv->lock, lock_flags);
 }
 
 static int fsl_esai_trigger(struct snd_pcm_substream *substream, int cmd,
@@ -759,7 +756,6 @@ static int fsl_esai_trigger(struct snd_pcm_substream *substream, int cmd,
 {
 	struct fsl_esai *esai_priv = snd_soc_dai_get_drvdata(dai);
 	bool tx = substream->stream == SNDRV_PCM_STREAM_PLAYBACK;
-	unsigned long lock_flags;
 
 	esai_priv->channels[tx] = substream->runtime->channels;
 
@@ -767,16 +763,14 @@ static int fsl_esai_trigger(struct snd_pcm_substream *substream, int cmd,
 	case SNDRV_PCM_TRIGGER_START:
 	case SNDRV_PCM_TRIGGER_RESUME:
 	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
-		spin_lock_irqsave(&esai_priv->lock, lock_flags);
-		fsl_esai_trigger_start(esai_priv, tx);
-		spin_unlock_irqrestore(&esai_priv->lock, lock_flags);
+		scoped_guard(spinlock_irqsave, &esai_priv->lock)
+			fsl_esai_trigger_start(esai_priv, tx);
 		break;
 	case SNDRV_PCM_TRIGGER_SUSPEND:
 	case SNDRV_PCM_TRIGGER_STOP:
 	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
-		spin_lock_irqsave(&esai_priv->lock, lock_flags);
-		fsl_esai_trigger_stop(esai_priv, tx);
-		spin_unlock_irqrestore(&esai_priv->lock, lock_flags);
+		scoped_guard(spinlock_irqsave, &esai_priv->lock)
+			fsl_esai_trigger_stop(esai_priv, tx);
 		break;
 	default:
 		return -EINVAL;
diff --git a/sound/soc/fsl/fsl_sai.c b/sound/soc/fsl/fsl_sai.c
index 9661602b53c5b..d232c8f530615 100644
--- a/sound/soc/fsl/fsl_sai.c
+++ b/sound/soc/fsl/fsl_sai.c
@@ -808,6 +808,8 @@ static int fsl_sai_hw_free(struct snd_pcm_substream *substream,
 	struct fsl_sai *sai = snd_soc_dai_get_drvdata(cpu_dai);
 	bool tx = substream->stream == SNDRV_PCM_STREAM_PLAYBACK;
 	unsigned int ofs = sai->soc_data->reg_offset;
+	int adir = tx ? RX : TX;
+	int dir  = tx ? TX : RX;
 
 	/* Clear xMR to avoid channel swap with mclk_with_tere enabled case */
 	regmap_write(sai->regmap, FSL_SAI_xMR(tx), 0);
@@ -815,10 +817,29 @@ static int fsl_sai_hw_free(struct snd_pcm_substream *substream,
 	regmap_update_bits(sai->regmap, FSL_SAI_xCR3(tx, ofs),
 			   FSL_SAI_CR3_TRCE_MASK, 0);
 
-	if (!sai->is_consumer_mode[tx] &&
-	    sai->mclk_streams & BIT(substream->stream)) {
-		clk_disable_unprepare(sai->mclk_clk[sai->mclk_id[tx]]);
-		sai->mclk_streams &= ~BIT(substream->stream);
+	if (!sai->is_consumer_mode[tx]) {
+		bool adir_active = !!(sai->mclk_streams & BIT(!substream->stream));
+		/*
+		 * If opposite stream provides clocks for synchronous mode and
+		 * it is inactive, Clear BYP and BCI
+		 */
+		if (fsl_sai_dir_is_synced(sai, adir) && !adir_active)
+			regmap_update_bits(sai->regmap, FSL_SAI_xCR2(!tx, ofs),
+					   FSL_SAI_CR2_BCI | FSL_SAI_CR2_BYP, 0);
+		/*
+		 * Clear BYP and BCI of current stream if either of:
+		 * 1. current stream doesn't provide clocks for synchronous mode
+		 * 2. current stream provides clocks for synchronous mode but no
+		 *    more stream is active.
+		 */
+		if (!fsl_sai_dir_is_synced(sai, dir) || !adir_active)
+			regmap_update_bits(sai->regmap, FSL_SAI_xCR2(tx, ofs),
+					   FSL_SAI_CR2_BCI | FSL_SAI_CR2_BYP, 0);
+
+		if (sai->mclk_streams & BIT(substream->stream)) {
+			clk_disable_unprepare(sai->mclk_clk[sai->mclk_id[tx]]);
+			sai->mclk_streams &= ~BIT(substream->stream);
+		}
 	}
 
 	return 0;
diff --git a/sound/soc/fsl/fsl_spdif.c b/sound/soc/fsl/fsl_spdif.c
index 1b9be85b34c24..ad1206ed98828 100644
--- a/sound/soc/fsl/fsl_spdif.c
+++ b/sound/soc/fsl/fsl_spdif.c
@@ -853,17 +853,15 @@ static int fsl_spdif_subcode_get(struct snd_kcontrol *kcontrol,
 	struct snd_soc_dai *cpu_dai = snd_kcontrol_chip(kcontrol);
 	struct fsl_spdif_priv *spdif_priv = snd_soc_dai_get_drvdata(cpu_dai);
 	struct spdif_mixer_control *ctrl = &spdif_priv->fsl_spdif_control;
-	unsigned long flags;
 	int ret = -EAGAIN;
 
-	spin_lock_irqsave(&ctrl->ctl_lock, flags);
+	guard(spinlock_irqsave)(&ctrl->ctl_lock);
 	if (ctrl->ready_buf) {
 		int idx = (ctrl->ready_buf - 1) * SPDIF_UBITS_SIZE;
 		memcpy(&ucontrol->value.iec958.subcode[0],
 				&ctrl->subcode[idx], SPDIF_UBITS_SIZE);
 		ret = 0;
 	}
-	spin_unlock_irqrestore(&ctrl->ctl_lock, flags);
 
 	return ret;
 }
@@ -885,17 +883,15 @@ static int fsl_spdif_qget(struct snd_kcontrol *kcontrol,
 	struct snd_soc_dai *cpu_dai = snd_kcontrol_chip(kcontrol);
 	struct fsl_spdif_priv *spdif_priv = snd_soc_dai_get_drvdata(cpu_dai);
 	struct spdif_mixer_control *ctrl = &spdif_priv->fsl_spdif_control;
-	unsigned long flags;
 	int ret = -EAGAIN;
 
-	spin_lock_irqsave(&ctrl->ctl_lock, flags);
+	guard(spinlock_irqsave)(&ctrl->ctl_lock);
 	if (ctrl->ready_buf) {
 		int idx = (ctrl->ready_buf - 1) * SPDIF_QSUB_SIZE;
 		memcpy(&ucontrol->value.bytes.data[0],
 				&ctrl->qsub[idx], SPDIF_QSUB_SIZE);
 		ret = 0;
 	}
-	spin_unlock_irqrestore(&ctrl->ctl_lock, flags);
 
 	return ret;
 }
diff --git a/sound/soc/fsl/fsl_ssi.c b/sound/soc/fsl/fsl_ssi.c
index b2e1da1781ae8..dc022976c9823 100644
--- a/sound/soc/fsl/fsl_ssi.c
+++ b/sound/soc/fsl/fsl_ssi.c
@@ -1218,13 +1218,13 @@ static void fsl_ssi_ac97_write(struct snd_ac97 *ac97, unsigned short reg,
 	if (reg > 0x7f)
 		return;
 
-	mutex_lock(&fsl_ac97_data->ac97_reg_lock);
+	guard(mutex)(&fsl_ac97_data->ac97_reg_lock);
 
 	ret = clk_prepare_enable(fsl_ac97_data->clk);
 	if (ret) {
 		pr_err("ac97 write clk_prepare_enable failed: %d\n",
 			ret);
-		goto ret_unlock;
+		return;
 	}
 
 	lreg = reg <<  12;
@@ -1238,9 +1238,6 @@ static void fsl_ssi_ac97_write(struct snd_ac97 *ac97, unsigned short reg,
 	udelay(100);
 
 	clk_disable_unprepare(fsl_ac97_data->clk);
-
-ret_unlock:
-	mutex_unlock(&fsl_ac97_data->ac97_reg_lock);
 }
 
 static unsigned short fsl_ssi_ac97_read(struct snd_ac97 *ac97,
@@ -1252,12 +1249,12 @@ static unsigned short fsl_ssi_ac97_read(struct snd_ac97 *ac97,
 	unsigned int lreg;
 	int ret;
 
-	mutex_lock(&fsl_ac97_data->ac97_reg_lock);
+	guard(mutex)(&fsl_ac97_data->ac97_reg_lock);
 
 	ret = clk_prepare_enable(fsl_ac97_data->clk);
 	if (ret) {
 		pr_err("ac97 read clk_prepare_enable failed: %d\n", ret);
-		goto ret_unlock;
+		return val;
 	}
 
 	lreg = (reg & 0x7f) <<  12;
@@ -1272,8 +1269,6 @@ static unsigned short fsl_ssi_ac97_read(struct snd_ac97 *ac97,
 
 	clk_disable_unprepare(fsl_ac97_data->clk);
 
-ret_unlock:
-	mutex_unlock(&fsl_ac97_data->ac97_reg_lock);
 	return val;
 }
 
diff --git a/sound/soc/fsl/fsl_xcvr.c b/sound/soc/fsl/fsl_xcvr.c
index 6677d3bf36ec2..41d100500534f 100644
--- a/sound/soc/fsl/fsl_xcvr.c
+++ b/sound/soc/fsl/fsl_xcvr.c
@@ -797,10 +797,9 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd,
 {
 	struct fsl_xcvr *xcvr = snd_soc_dai_get_drvdata(dai);
 	bool tx = substream->stream == SNDRV_PCM_STREAM_PLAYBACK;
-	unsigned long lock_flags;
 	int ret = 0;
 
-	spin_lock_irqsave(&xcvr->lock, lock_flags);
+	guard(spinlock_irqsave)(&xcvr->lock);
 
 	switch (cmd) {
 	case SNDRV_PCM_TRIGGER_START:
@@ -812,7 +811,7 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd,
 					 FSL_XCVR_EXT_CTRL_DPTH_RESET(tx));
 		if (ret < 0) {
 			dev_err(dai->dev, "Failed to set DPATH RESET: %d\n", ret);
-			goto release_lock;
+			return ret;
 		}
 
 		if (tx) {
@@ -824,7 +823,7 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd,
 						   FSL_XCVR_ISR_CMDC_TX_EN);
 				if (ret < 0) {
 					dev_err(dai->dev, "err updating isr %d\n", ret);
-					goto release_lock;
+					return ret;
 				}
 				fallthrough;
 			case FSL_XCVR_MODE_SPDIF:
@@ -833,7 +832,7 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd,
 						      FSL_XCVR_TX_DPTH_CTRL_STRT_DATA_TX);
 				if (ret < 0) {
 					dev_err(dai->dev, "Failed to start DATA_TX: %d\n", ret);
-					goto release_lock;
+					return ret;
 				}
 				break;
 			}
@@ -844,14 +843,14 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd,
 					 FSL_XCVR_EXT_CTRL_DMA_DIS(tx), 0);
 		if (ret < 0) {
 			dev_err(dai->dev, "Failed to enable DMA: %d\n", ret);
-			goto release_lock;
+			return ret;
 		}
 
 		ret = regmap_update_bits(xcvr->regmap, FSL_XCVR_EXT_IER0,
 					 FSL_XCVR_IRQ_EARC_ALL, FSL_XCVR_IRQ_EARC_ALL);
 		if (ret < 0) {
 			dev_err(dai->dev, "Error while setting IER0: %d\n", ret);
-			goto release_lock;
+			return ret;
 		}
 
 		/* clear DPATH RESET */
@@ -860,7 +859,7 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd,
 					 0);
 		if (ret < 0) {
 			dev_err(dai->dev, "Failed to clear DPATH RESET: %d\n", ret);
-			goto release_lock;
+			return ret;
 		}
 
 		break;
@@ -873,14 +872,14 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd,
 					 FSL_XCVR_EXT_CTRL_DMA_DIS(tx));
 		if (ret < 0) {
 			dev_err(dai->dev, "Failed to disable DMA: %d\n", ret);
-			goto release_lock;
+			return ret;
 		}
 
 		ret = regmap_update_bits(xcvr->regmap, FSL_XCVR_EXT_IER0,
 					 FSL_XCVR_IRQ_EARC_ALL, 0);
 		if (ret < 0) {
 			dev_err(dai->dev, "Failed to clear IER0: %d\n", ret);
-			goto release_lock;
+			return ret;
 		}
 
 		if (tx) {
@@ -891,7 +890,7 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd,
 							FSL_XCVR_TX_DPTH_CTRL_STRT_DATA_TX);
 				if (ret < 0) {
 					dev_err(dai->dev, "Failed to stop DATA_TX: %d\n", ret);
-					goto release_lock;
+					return ret;
 				}
 				if (xcvr->soc_data->spdif_only)
 					break;
@@ -905,7 +904,7 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd,
 				if (ret < 0) {
 					dev_err(dai->dev,
 						"Err updating ISR %d\n", ret);
-					goto release_lock;
+					return ret;
 				}
 				break;
 			}
@@ -916,8 +915,6 @@ static int fsl_xcvr_trigger(struct snd_pcm_substream *substream, int cmd,
 		break;
 	}
 
-release_lock:
-	spin_unlock_irqrestore(&xcvr->lock, lock_flags);
 	return ret;
 }
 
@@ -1448,11 +1445,10 @@ static void reset_rx_work(struct work_struct *work)
 {
 	struct fsl_xcvr *xcvr = container_of(work, struct fsl_xcvr, work_rst);
 	struct device *dev = &xcvr->pdev->dev;
-	unsigned long lock_flags;
 	u32 ext_ctrl;
 
 	dev_dbg(dev, "reset rx path\n");
-	spin_lock_irqsave(&xcvr->lock, lock_flags);
+	guard(spinlock_irqsave)(&xcvr->lock);
 	regmap_read(xcvr->regmap, FSL_XCVR_EXT_CTRL, &ext_ctrl);
 
 	if (!(ext_ctrl & FSL_XCVR_EXT_CTRL_DMA_RD_DIS)) {
@@ -1469,7 +1465,6 @@ static void reset_rx_work(struct work_struct *work)
 				   FSL_XCVR_EXT_CTRL_RX_DPTH_RESET,
 				   0);
 	}
-	spin_unlock_irqrestore(&xcvr->lock, lock_flags);
 }
 
 static irqreturn_t irq0_isr(int irq, void *devid)
diff --git a/sound/soc/fsl/imx-audio-rpmsg.c b/sound/soc/fsl/imx-audio-rpmsg.c
index 38aafb8954c76..b55dfbdb45026 100644
--- a/sound/soc/fsl/imx-audio-rpmsg.c
+++ b/sound/soc/fsl/imx-audio-rpmsg.c
@@ -22,7 +22,6 @@ static int imx_audio_rpmsg_cb(struct rpmsg_device *rpdev, void *data, int len,
 	struct rpmsg_r_msg *r_msg = (struct rpmsg_r_msg *)data;
 	struct rpmsg_info *info;
 	struct rpmsg_msg *msg;
-	unsigned long flags;
 
 	if (!rpmsg->rpmsg_pdev)
 		return 0;
@@ -37,21 +36,21 @@ static int imx_audio_rpmsg_cb(struct rpmsg_device *rpdev, void *data, int len,
 		/* TYPE C is notification from M core */
 		switch (r_msg->header.cmd) {
 		case TX_PERIOD_DONE:
-			spin_lock_irqsave(&info->lock[TX], flags);
-			msg = &info->msg[TX_PERIOD_DONE + MSG_TYPE_A_NUM];
-			msg->r_msg.param.buffer_tail =
-						r_msg->param.buffer_tail;
-			msg->r_msg.param.buffer_tail %= info->num_period[TX];
-			spin_unlock_irqrestore(&info->lock[TX], flags);
+			scoped_guard(spinlock_irqsave, &info->lock[TX]) {
+				msg = &info->msg[TX_PERIOD_DONE + MSG_TYPE_A_NUM];
+				msg->r_msg.param.buffer_tail =
+							r_msg->param.buffer_tail;
+				msg->r_msg.param.buffer_tail %= info->num_period[TX];
+			}
 			info->callback[TX](info->callback_param[TX]);
 			break;
 		case RX_PERIOD_DONE:
-			spin_lock_irqsave(&info->lock[RX], flags);
-			msg = &info->msg[RX_PERIOD_DONE + MSG_TYPE_A_NUM];
-			msg->r_msg.param.buffer_tail =
-						r_msg->param.buffer_tail;
-			msg->r_msg.param.buffer_tail %= info->num_period[1];
-			spin_unlock_irqrestore(&info->lock[RX], flags);
+			scoped_guard(spinlock_irqsave, &info->lock[RX]) {
+				msg = &info->msg[RX_PERIOD_DONE + MSG_TYPE_A_NUM];
+				msg->r_msg.param.buffer_tail =
+							r_msg->param.buffer_tail;
+				msg->r_msg.param.buffer_tail %= info->num_period[1];
+			}
 			info->callback[RX](info->callback_param[RX]);
 			break;
 		default:
diff --git a/sound/soc/fsl/imx-card.c b/sound/soc/fsl/imx-card.c
index a4518fefad690..43438af1e1c68 100644
--- a/sound/soc/fsl/imx-card.c
+++ b/sound/soc/fsl/imx-card.c
@@ -496,11 +496,15 @@ static void imx_aif_shutdown(struct snd_pcm_substream *substream)
 	struct snd_soc_dai *codec_dai;
 	int i;
 
-	for_each_rtd_cpu_dais(rtd, i, cpu_dai)
-		snd_soc_dai_set_sysclk(cpu_dai, 0, 0, SND_SOC_CLOCK_OUT);
+	for_each_rtd_cpu_dais(rtd, i, cpu_dai) {
+		if (!snd_soc_dai_active(cpu_dai))
+			snd_soc_dai_set_sysclk(cpu_dai, 0, 0, SND_SOC_CLOCK_OUT);
+	}
 
-	for_each_rtd_codec_dais(rtd, i, codec_dai)
-		snd_soc_dai_set_sysclk(codec_dai, 0, 0, SND_SOC_CLOCK_IN);
+	for_each_rtd_codec_dais(rtd, i, codec_dai) {
+		if (!snd_soc_dai_active(codec_dai))
+			snd_soc_dai_set_sysclk(codec_dai, 0, 0, SND_SOC_CLOCK_IN);
+	}
 }
 
 static const struct snd_soc_ops imx_aif_ops = {
diff --git a/sound/soc/fsl/imx-pcm-rpmsg.c b/sound/soc/fsl/imx-pcm-rpmsg.c
index 2a4813c6cda96..ee741f3d79bd2 100644
--- a/sound/soc/fsl/imx-pcm-rpmsg.c
+++ b/sound/soc/fsl/imx-pcm-rpmsg.c
@@ -39,10 +39,9 @@ static int imx_rpmsg_pcm_send_message(struct rpmsg_msg *msg,
 	struct rpmsg_device *rpdev = info->rpdev;
 	int ret = 0;
 
-	mutex_lock(&info->msg_lock);
+	guard(mutex)(&info->msg_lock);
 	if (!rpdev) {
 		dev_err(info->dev, "rpmsg channel not ready\n");
-		mutex_unlock(&info->msg_lock);
 		return -EINVAL;
 	}
 
@@ -55,15 +54,12 @@ static int imx_rpmsg_pcm_send_message(struct rpmsg_msg *msg,
 			 sizeof(struct rpmsg_s_msg));
 	if (ret) {
 		dev_err(&rpdev->dev, "rpmsg_send failed: %d\n", ret);
-		mutex_unlock(&info->msg_lock);
 		return ret;
 	}
 
 	/* No receive msg for TYPE_C command */
-	if (msg->s_msg.header.type == MSG_TYPE_C) {
-		mutex_unlock(&info->msg_lock);
+	if (msg->s_msg.header.type == MSG_TYPE_C)
 		return 0;
-	}
 
 	/* wait response from rpmsg */
 	ret = wait_for_completion_timeout(&info->cmd_complete,
@@ -71,7 +67,6 @@ static int imx_rpmsg_pcm_send_message(struct rpmsg_msg *msg,
 	if (!ret) {
 		dev_err(&rpdev->dev, "rpmsg_send cmd %d timeout!\n",
 			msg->s_msg.header.cmd);
-		mutex_unlock(&info->msg_lock);
 		return -ETIMEDOUT;
 	}
 
@@ -100,8 +95,6 @@ static int imx_rpmsg_pcm_send_message(struct rpmsg_msg *msg,
 	dev_dbg(&rpdev->dev, "cmd:%d, resp %d\n", msg->s_msg.header.cmd,
 		info->r_msg.param.resp);
 
-	mutex_unlock(&info->msg_lock);
-
 	return 0;
 }
 
@@ -109,14 +102,13 @@ static int imx_rpmsg_insert_workqueue(struct snd_pcm_substream *substream,
 				      struct rpmsg_msg *msg,
 				      struct rpmsg_info *info)
 {
-	unsigned long flags;
 	int ret = 0;
 
 	/*
 	 * Queue the work to workqueue.
 	 * If the queue is full, drop the message.
 	 */
-	spin_lock_irqsave(&info->wq_lock, flags);
+	guard(spinlock_irqsave)(&info->wq_lock);
 	if (info->work_write_index != info->work_read_index) {
 		int index = info->work_write_index;
 
@@ -130,7 +122,6 @@ static int imx_rpmsg_insert_workqueue(struct snd_pcm_substream *substream,
 		info->msg_drop_count[substream->stream]++;
 		ret = -EPIPE;
 	}
-	spin_unlock_irqrestore(&info->wq_lock, flags);
 
 	return ret;
 }
@@ -523,7 +514,6 @@ static int imx_rpmsg_pcm_ack(struct snd_soc_component *component,
 	snd_pcm_sframes_t avail;
 	struct timer_list *timer;
 	struct rpmsg_msg *msg;
-	unsigned long flags;
 	int buffer_tail = 0;
 	int written_num;
 
@@ -553,11 +543,11 @@ static int imx_rpmsg_pcm_ack(struct snd_soc_component *component,
 		msg->s_msg.param.buffer_tail = buffer_tail;
 
 		/* The notification message is updated to latest */
-		spin_lock_irqsave(&info->lock[substream->stream], flags);
-		memcpy(&info->notify[substream->stream], msg,
-		       sizeof(struct rpmsg_s_msg));
-		info->notify_updated[substream->stream] = true;
-		spin_unlock_irqrestore(&info->lock[substream->stream], flags);
+		scoped_guard(spinlock_irqsave, &info->lock[substream->stream]) {
+			memcpy(&info->notify[substream->stream], msg,
+			       sizeof(struct rpmsg_s_msg));
+			info->notify_updated[substream->stream] = true;
+		}
 
 		if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
 			avail = snd_pcm_playback_hw_avail(runtime);
@@ -641,7 +631,7 @@ static void imx_rpmsg_pcm_work(struct work_struct *work)
 	bool is_notification = false;
 	struct rpmsg_info *info;
 	struct rpmsg_msg msg;
-	unsigned long flags;
+	bool updated;
 
 	work_of_rpmsg = container_of(work, struct work_of_rpmsg, work);
 	info = work_of_rpmsg->info;
@@ -652,25 +642,26 @@ static void imx_rpmsg_pcm_work(struct work_struct *work)
 	 * enough data in M core side, need to let M core know
 	 * data is updated immediately.
 	 */
-	spin_lock_irqsave(&info->lock[TX], flags);
-	if (info->notify_updated[TX]) {
-		memcpy(&msg, &info->notify[TX], sizeof(struct rpmsg_s_msg));
-		info->notify_updated[TX] = false;
-		spin_unlock_irqrestore(&info->lock[TX], flags);
-		info->send_message(&msg, info);
-	} else {
-		spin_unlock_irqrestore(&info->lock[TX], flags);
+	scoped_guard(spinlock_irqsave, &info->lock[TX]) {
+		updated = info->notify_updated[TX];
+		if (updated) {
+			memcpy(&msg, &info->notify[TX], sizeof(struct rpmsg_s_msg));
+			info->notify_updated[TX] = false;
+		}
 	}
-
-	spin_lock_irqsave(&info->lock[RX], flags);
-	if (info->notify_updated[RX]) {
-		memcpy(&msg, &info->notify[RX], sizeof(struct rpmsg_s_msg));
-		info->notify_updated[RX] = false;
-		spin_unlock_irqrestore(&info->lock[RX], flags);
+	if (updated)
 		info->send_message(&msg, info);
-	} else {
-		spin_unlock_irqrestore(&info->lock[RX], flags);
+
+	scoped_guard(spinlock_irqsave, &info->lock[RX]) {
+		updated = info->notify_updated[RX];
+		if (updated) {
+			memcpy(&msg, &info->notify[RX], sizeof(struct rpmsg_s_msg));
+			info->notify_updated[RX] = false;
+		}
 	}
+	if (updated)
+		info->send_message(&msg, info);
+
 
 	/* Skip the notification message for it has been processed above */
 	if (work_of_rpmsg->msg.s_msg.header.type == MSG_TYPE_C &&
@@ -682,10 +673,10 @@ static void imx_rpmsg_pcm_work(struct work_struct *work)
 		info->send_message(&work_of_rpmsg->msg, info);
 
 	/* update read index */
-	spin_lock_irqsave(&info->wq_lock, flags);
-	info->work_read_index++;
-	info->work_read_index %= WORK_MAX_NUM;
-	spin_unlock_irqrestore(&info->wq_lock, flags);
+	scoped_guard(spinlock_irqsave, &info->wq_lock) {
+		info->work_read_index++;
+		info->work_read_index %= WORK_MAX_NUM;
+	}
 }
 
 static int imx_rpmsg_pcm_probe(struct platform_device *pdev)
diff --git a/sound/soc/fsl/mpc5200_dma.c b/sound/soc/fsl/mpc5200_dma.c
index 56e2cf2f727ba..bfedb2dea0b32 100644
--- a/sound/soc/fsl/mpc5200_dma.c
+++ b/sound/soc/fsl/mpc5200_dma.c
@@ -77,18 +77,20 @@ static irqreturn_t psc_dma_bcom_irq(int irq, void *_psc_dma_stream)
 {
 	struct psc_dma_stream *s = _psc_dma_stream;
 
-	spin_lock(&s->psc_dma->lock);
-	/* For each finished period, dequeue the completed period buffer
-	 * and enqueue a new one in it's place. */
-	while (bcom_buffer_done(s->bcom_task)) {
-		bcom_retrieve_buffer(s->bcom_task, NULL, NULL);
+	scoped_guard(spinlock, &s->psc_dma->lock) {
+		/*
+		 * For each finished period, dequeue the completed period buffer
+		 * and enqueue a new one in its place
+		 */
+		while (bcom_buffer_done(s->bcom_task)) {
+			bcom_retrieve_buffer(s->bcom_task, NULL, NULL);
 
-		s->period_current = (s->period_current+1) % s->runtime->periods;
-		s->period_count++;
+			s->period_current = (s->period_current+1) % s->runtime->periods;
+			s->period_count++;
 
-		psc_dma_bcom_enqueue_next_buffer(s);
+			psc_dma_bcom_enqueue_next_buffer(s);
+		}
 	}
-	spin_unlock(&s->psc_dma->lock);
 
 	/* If the stream is active, then also inform the PCM middle layer
 	 * of the period finished event. */
@@ -116,7 +118,6 @@ static int psc_dma_trigger(struct snd_soc_component *component,
 	struct psc_dma_stream *s = to_psc_dma_stream(substream, psc_dma);
 	struct mpc52xx_psc __iomem *regs = psc_dma->psc_regs;
 	u16 imr;
-	unsigned long flags;
 	int i;
 
 	switch (cmd) {
@@ -135,19 +136,18 @@ static int psc_dma_trigger(struct snd_soc_component *component,
 		/* Fill up the bestcomm bd queue and enable DMA.
 		 * This will begin filling the PSC's fifo.
 		 */
-		spin_lock_irqsave(&psc_dma->lock, flags);
-
-		if (substream->pstr->stream == SNDRV_PCM_STREAM_CAPTURE)
-			bcom_gen_bd_rx_reset(s->bcom_task);
-		else
-			bcom_gen_bd_tx_reset(s->bcom_task);
+		scoped_guard(spinlock_irqsave, &psc_dma->lock) {
+			if (substream->pstr->stream == SNDRV_PCM_STREAM_CAPTURE)
+				bcom_gen_bd_rx_reset(s->bcom_task);
+			else
+				bcom_gen_bd_tx_reset(s->bcom_task);
 
-		for (i = 0; i < runtime->periods; i++)
-			if (!bcom_queue_full(s->bcom_task))
-				psc_dma_bcom_enqueue_next_buffer(s);
+			for (i = 0; i < runtime->periods; i++)
+				if (!bcom_queue_full(s->bcom_task))
+					psc_dma_bcom_enqueue_next_buffer(s);
 
-		bcom_enable(s->bcom_task);
-		spin_unlock_irqrestore(&psc_dma->lock, flags);
+			bcom_enable(s->bcom_task);
+		}
 
 		out_8(&regs->command, MPC52xx_PSC_RST_ERR_STAT);
 
@@ -158,13 +158,13 @@ static int psc_dma_trigger(struct snd_soc_component *component,
 			substream->pstr->stream, s->period_count);
 		s->active = 0;
 
-		spin_lock_irqsave(&psc_dma->lock, flags);
-		bcom_disable(s->bcom_task);
-		if (substream->pstr->stream == SNDRV_PCM_STREAM_CAPTURE)
-			bcom_gen_bd_rx_reset(s->bcom_task);
-		else
-			bcom_gen_bd_tx_reset(s->bcom_task);
-		spin_unlock_irqrestore(&psc_dma->lock, flags);
+		scoped_guard(spinlock_irqsave, &psc_dma->lock) {
+			bcom_disable(s->bcom_task);
+			if (substream->pstr->stream == SNDRV_PCM_STREAM_CAPTURE)
+				bcom_gen_bd_rx_reset(s->bcom_task);
+			else
+				bcom_gen_bd_tx_reset(s->bcom_task);
+		}
 
 		break;
 
diff --git a/sound/soc/fsl/mpc5200_psc_ac97.c b/sound/soc/fsl/mpc5200_psc_ac97.c
index c21104355aa0d..2aefd6414acef 100644
--- a/sound/soc/fsl/mpc5200_psc_ac97.c
+++ b/sound/soc/fsl/mpc5200_psc_ac97.c
@@ -30,14 +30,13 @@ static unsigned short psc_ac97_read(struct snd_ac97 *ac97, unsigned short reg)
 	int status;
 	unsigned int val;
 
-	mutex_lock(&psc_dma->mutex);
+	guard(mutex)(&psc_dma->mutex);
 
 	/* Wait for command send status zero = ready */
 	status = spin_event_timeout(!(in_be16(&psc_dma->psc_regs->sr_csr.status) &
 				MPC52xx_PSC_SR_CMDSEND), 100, 0);
 	if (status == 0) {
 		pr_err("timeout on ac97 bus (rdy)\n");
-		mutex_unlock(&psc_dma->mutex);
 		return -ENODEV;
 	}
 
@@ -53,19 +52,16 @@ static unsigned short psc_ac97_read(struct snd_ac97 *ac97, unsigned short reg)
 	if (status == 0) {
 		pr_err("timeout on ac97 read (val) %x\n",
 				in_be16(&psc_dma->psc_regs->sr_csr.status));
-		mutex_unlock(&psc_dma->mutex);
 		return -ENODEV;
 	}
 	/* Get the data */
 	val = in_be32(&psc_dma->psc_regs->ac97_data);
 	if (((val >> 24) & 0x7f) != reg) {
 		pr_err("reg echo error on ac97 read\n");
-		mutex_unlock(&psc_dma->mutex);
 		return -ENODEV;
 	}
 	val = (val >> 8) & 0xffff;
 
-	mutex_unlock(&psc_dma->mutex);
 	return (unsigned short) val;
 }
 
@@ -74,52 +70,46 @@ static void psc_ac97_write(struct snd_ac97 *ac97,
 {
 	int status;
 
-	mutex_lock(&psc_dma->mutex);
+	guard(mutex)(&psc_dma->mutex);
 
 	/* Wait for command status zero = ready */
 	status = spin_event_timeout(!(in_be16(&psc_dma->psc_regs->sr_csr.status) &
 				MPC52xx_PSC_SR_CMDSEND), 100, 0);
 	if (status == 0) {
 		pr_err("timeout on ac97 bus (write)\n");
-		goto out;
+		return;
 	}
 	/* Write data */
 	out_be32(&psc_dma->psc_regs->ac97_cmd,
 			((reg & 0x7f) << 24) | (val << 8));
-
- out:
-	mutex_unlock(&psc_dma->mutex);
 }
 
 static void psc_ac97_warm_reset(struct snd_ac97 *ac97)
 {
 	struct mpc52xx_psc __iomem *regs = psc_dma->psc_regs;
 
-	mutex_lock(&psc_dma->mutex);
+	guard(mutex)(&psc_dma->mutex);
 
 	out_be32(&regs->sicr, psc_dma->sicr | MPC52xx_PSC_SICR_AWR);
 	udelay(3);
 	out_be32(&regs->sicr, psc_dma->sicr);
-
-	mutex_unlock(&psc_dma->mutex);
 }
 
 static void psc_ac97_cold_reset(struct snd_ac97 *ac97)
 {
 	struct mpc52xx_psc __iomem *regs = psc_dma->psc_regs;
 
-	mutex_lock(&psc_dma->mutex);
-	dev_dbg(psc_dma->dev, "cold reset\n");
+	scoped_guard(mutex, &psc_dma->mutex) {
+		dev_dbg(psc_dma->dev, "cold reset\n");
 
-	mpc5200_psc_ac97_gpio_reset(psc_dma->id);
+		mpc5200_psc_ac97_gpio_reset(psc_dma->id);
 
-	/* Notify the PSC that a reset has occurred */
-	out_be32(&regs->sicr, psc_dma->sicr | MPC52xx_PSC_SICR_ACRB);
+		/* Notify the PSC that a reset has occurred */
+		out_be32(&regs->sicr, psc_dma->sicr | MPC52xx_PSC_SICR_ACRB);
 
-	/* Re-enable RX and TX */
-	out_8(&regs->command, MPC52xx_PSC_TX_ENABLE | MPC52xx_PSC_RX_ENABLE);
-
-	mutex_unlock(&psc_dma->mutex);
+		/* Re-enable RX and TX */
+		out_8(&regs->command, MPC52xx_PSC_TX_ENABLE | MPC52xx_PSC_RX_ENABLE);
+	}
 
 	usleep_range(1000, 2000);
 	psc_ac97_warm_reset(ac97);
diff --git a/sound/soc/fsl/mpc5200_psc_i2s.c b/sound/soc/fsl/mpc5200_psc_i2s.c
index 9ad44eeed6ad8..7831136f4f129 100644
--- a/sound/soc/fsl/mpc5200_psc_i2s.c
+++ b/sound/soc/fsl/mpc5200_psc_i2s.c
@@ -170,6 +170,7 @@ static int psc_i2s_of_probe(struct platform_device *op)
 					psc_i2s_dai, ARRAY_SIZE(psc_i2s_dai));
 	if (rc != 0) {
 		pr_err("Failed to register DAI\n");
+		mpc5200_audio_dma_destroy(op);
 		return rc;
 	}
 
diff --git a/sound/soc/generic/audio-graph-card.c b/sound/soc/generic/audio-graph-card.c
index 18ce4ee063509..0a8a6d891d1f0 100644
--- a/sound/soc/generic/audio-graph-card.c
+++ b/sound/soc/generic/audio-graph-card.c
@@ -579,11 +579,11 @@ int audio_graph_parse_of(struct simple_util_priv *priv, struct device *dev)
 		goto end;
 	}
 
-	ret = simple_util_parse_widgets(card, NULL);
+	ret = simple_util_parse_widgets(priv, NULL);
 	if (ret < 0)
 		goto end;
 
-	ret = simple_util_parse_routing(card, NULL);
+	ret = simple_util_parse_routing(priv, NULL);
 	if (ret < 0)
 		goto end;
 
@@ -594,6 +594,7 @@ int audio_graph_parse_of(struct simple_util_priv *priv, struct device *dev)
 	if (ret < 0)
 		goto err;
 
+	/* Card name should be set after graph_for_each_link() */
 	ret = simple_util_parse_card_name(priv, NULL);
 	if (ret < 0)
 		goto err;
@@ -603,14 +604,13 @@ int audio_graph_parse_of(struct simple_util_priv *priv, struct device *dev)
 	simple_util_debug_info(priv);
 
 	ret = devm_snd_soc_register_card(dev, card);
-	if (ret < 0)
-		goto err;
-
-	return 0;
 err:
-	simple_util_clean_reference(card);
+	if (ret < 0) {
+		simple_util_clean_reference(priv);
+		return dev_err_probe(dev, ret, "parse error\n");
+	}
 end:
-	return dev_err_probe(dev, ret, "parse error\n");
+	return graph_ret(priv, ret);
 }
 EXPORT_SYMBOL_GPL(audio_graph_parse_of);
 
diff --git a/sound/soc/generic/audio-graph-card2.c b/sound/soc/generic/audio-graph-card2.c
index 0202ed0ee78e8..9fb3d3df5cf68 100644
--- a/sound/soc/generic/audio-graph-card2.c
+++ b/sound/soc/generic/audio-graph-card2.c
@@ -1303,11 +1303,11 @@ int audio_graph2_parse_of(struct simple_util_priv *priv, struct device *dev,
 			  struct graph2_custom_hooks *hooks)
 {
 	struct snd_soc_card *card = simple_priv_to_card(priv);
-	int ret;
+	int ret = -ENOMEM;
 
 	struct link_info *li __free(kfree) = kzalloc_obj(*li);
 	if (!li)
-		return -ENOMEM;
+		goto end;
 
 	card->probe	= graph_util_card_probe;
 	card->owner	= THIS_MODULE;
@@ -1316,31 +1316,35 @@ int audio_graph2_parse_of(struct simple_util_priv *priv, struct device *dev,
 	if ((hooks) && (hooks)->hook_pre) {
 		ret = (hooks)->hook_pre(priv);
 		if (ret < 0)
-			goto err;
+			goto end;
 	}
 
 	ret = graph_for_each_link(priv, hooks, li, graph_count);
 	if (!li->link)
 		ret = -EINVAL;
 	if (ret < 0)
-		goto err;
+		goto end;
 
 	ret = simple_util_init_priv(priv, li);
 	if (ret < 0)
-		goto err;
+		goto end;
 
 	priv->pa_gpio = devm_gpiod_get_optional(dev, "pa", GPIOD_OUT_LOW);
 	if (IS_ERR(priv->pa_gpio)) {
 		ret = PTR_ERR(priv->pa_gpio);
 		dev_err(dev, "failed to get amplifier gpio: %d\n", ret);
-		goto err;
+		goto end;
 	}
 
-	ret = simple_util_parse_widgets(card, NULL);
+	ret = simple_util_parse_widgets(priv, NULL);
 	if (ret < 0)
-		goto err;
+		goto end;
+
+	ret = simple_util_parse_routing(priv, NULL);
+	if (ret < 0)
+		goto end;
 
-	ret = simple_util_parse_routing(card, NULL);
+	ret = simple_util_parse_aux_devs(priv, NULL);
 	if (ret < 0)
 		goto err;
 
@@ -1349,6 +1353,7 @@ int audio_graph2_parse_of(struct simple_util_priv *priv, struct device *dev,
 	if (ret < 0)
 		goto err;
 
+	/* Card name should be set after graph_for_each_link() */
 	ret = simple_util_parse_card_name(priv, NULL);
 	if (ret < 0)
 		goto err;
@@ -1363,15 +1368,13 @@ int audio_graph2_parse_of(struct simple_util_priv *priv, struct device *dev,
 
 	simple_util_debug_info(priv);
 
-	ret = snd_soc_of_parse_aux_devs(card, "aux-devs");
-	if (ret < 0)
-		goto err;
-
 	ret = devm_snd_soc_register_card(dev, card);
 err:
-	if (ret < 0)
-		dev_err_probe(dev, ret, "parse error\n");
-
+	if (ret < 0) {
+		simple_util_clean_reference(priv);
+		return dev_err_probe(dev, ret, "parse error\n");
+	}
+end:
 	return graph_ret(priv, ret);
 }
 EXPORT_SYMBOL_GPL(audio_graph2_parse_of);
diff --git a/sound/soc/generic/simple-card-utils.c b/sound/soc/generic/simple-card-utils.c
index e5cb602fd2487..42019daa5e04f 100644
--- a/sound/soc/generic/simple-card-utils.c
+++ b/sound/soc/generic/simple-card-utils.c
@@ -216,6 +216,27 @@ int simple_util_set_dailink_name(struct simple_util_priv *priv,
 }
 EXPORT_SYMBOL_GPL(simple_util_set_dailink_name);
 
+int simple_util_parse_property(struct simple_util_priv *priv,
+			       int (*func)(struct snd_soc_card *card, const char *propname),
+			       char *prefix, char *property)
+{
+	struct snd_soc_card *card = simple_priv_to_card(priv);
+	struct device_node *node = card->dev->of_node;
+	char prop[128];
+
+	if (!prefix)
+		prefix = "";
+
+	snprintf(prop, sizeof(prop), "%s%s", prefix, property);
+
+	/* no property is not error */
+	if (!of_property_present(node, prop))
+		return 0;
+
+	return func(card, prop);
+}
+EXPORT_SYMBOL_GPL(simple_util_parse_property);
+
 int simple_util_parse_card_name(struct simple_util_priv *priv,
 				char *prefix)
 {
@@ -731,11 +752,12 @@ void simple_util_canonicalize_cpu(struct snd_soc_dai_link_component *cpus,
 }
 EXPORT_SYMBOL_GPL(simple_util_canonicalize_cpu);
 
-void simple_util_clean_reference(struct snd_soc_card *card)
+void simple_util_clean_reference(struct simple_util_priv *priv)
 {
 	struct snd_soc_dai_link *dai_link;
 	struct snd_soc_dai_link_component *cpu;
 	struct snd_soc_dai_link_component *codec;
+	struct snd_soc_card *card = simple_priv_to_card(priv);
 	int i, j;
 
 	for_each_card_prelinks(card, i, dai_link) {
@@ -747,57 +769,6 @@ void simple_util_clean_reference(struct snd_soc_card *card)
 }
 EXPORT_SYMBOL_GPL(simple_util_clean_reference);
 
-int simple_util_parse_routing(struct snd_soc_card *card,
-			      char *prefix)
-{
-	struct device_node *node = card->dev->of_node;
-	char prop[128];
-
-	if (!prefix)
-		prefix = "";
-
-	snprintf(prop, sizeof(prop), "%s%s", prefix, "routing");
-
-	if (!of_property_present(node, prop))
-		return 0;
-
-	return snd_soc_of_parse_audio_routing(card, prop);
-}
-EXPORT_SYMBOL_GPL(simple_util_parse_routing);
-
-int simple_util_parse_widgets(struct snd_soc_card *card,
-			      char *prefix)
-{
-	struct device_node *node = card->dev->of_node;
-	char prop[128];
-
-	if (!prefix)
-		prefix = "";
-
-	snprintf(prop, sizeof(prop), "%s%s", prefix, "widgets");
-
-	if (of_property_present(node, prop))
-		return snd_soc_of_parse_audio_simple_widgets(card, prop);
-
-	/* no widgets is not error */
-	return 0;
-}
-EXPORT_SYMBOL_GPL(simple_util_parse_widgets);
-
-int simple_util_parse_pin_switches(struct snd_soc_card *card,
-				   char *prefix)
-{
-	char prop[128];
-
-	if (!prefix)
-		prefix = "";
-
-	snprintf(prop, sizeof(prop), "%s%s", prefix, "pin-switches");
-
-	return snd_soc_of_parse_pin_switches(card, prop);
-}
-EXPORT_SYMBOL_GPL(simple_util_parse_pin_switches);
-
 int simple_util_init_jack(struct snd_soc_card *card,
 			  struct simple_util_jack *sjack,
 			  int is_hp, char *prefix,
@@ -854,9 +825,9 @@ int simple_util_init_jack(struct snd_soc_card *card,
 }
 EXPORT_SYMBOL_GPL(simple_util_init_jack);
 
-int simple_util_init_aux_jacks(struct simple_util_priv *priv, char *prefix)
+int simple_util_init_aux_jacks(struct snd_soc_card *card, char *prefix)
 {
-	struct snd_soc_card *card = simple_priv_to_card(priv);
+	struct simple_util_priv *priv = snd_soc_card_get_drvdata(card);
 	struct snd_soc_component *component;
 	int found_jack_index = 0;
 	int type = 0;
@@ -1026,8 +997,9 @@ EXPORT_SYMBOL_GPL(simple_util_init_priv);
 void simple_util_remove(struct platform_device *pdev)
 {
 	struct snd_soc_card *card = platform_get_drvdata(pdev);
+	struct simple_util_priv *priv = snd_soc_card_get_drvdata(card);
 
-	simple_util_clean_reference(card);
+	simple_util_clean_reference(priv);
 }
 EXPORT_SYMBOL_GPL(simple_util_remove);
 
diff --git a/sound/soc/generic/simple-card.c b/sound/soc/generic/simple-card.c
index b4957e0252113..e7ad7af04714d 100644
--- a/sound/soc/generic/simple-card.c
+++ b/sound/soc/generic/simple-card.c
@@ -517,44 +517,6 @@ static int simple_populate_aux(struct simple_util_priv *priv)
 	return simple_ret(priv, ret);
 }
 
-static int simple_parse_of(struct simple_util_priv *priv, struct link_info *li)
-{
-	struct snd_soc_card *card = simple_priv_to_card(priv);
-	int ret;
-
-	ret = simple_util_parse_widgets(card, PREFIX);
-	if (ret < 0)
-		goto end;
-
-	ret = simple_util_parse_routing(card, PREFIX);
-	if (ret < 0)
-		goto end;
-
-	ret = simple_util_parse_pin_switches(card, PREFIX);
-	if (ret < 0)
-		goto end;
-
-	/* Single/Muti DAI link(s) & New style of DT node */
-	memset(li, 0, sizeof(*li));
-	ret = simple_for_each_link(priv, li,
-				   simple_dai_link_of,
-				   simple_dai_link_of_dpcm);
-	if (ret < 0)
-		goto end;
-
-	ret = simple_util_parse_card_name(priv, PREFIX);
-	if (ret < 0)
-		goto end;
-
-	ret = simple_populate_aux(priv);
-	if (ret < 0)
-		goto end;
-
-	ret = snd_soc_of_parse_aux_devs(card, PREFIX "aux-devs");
-end:
-	return simple_ret(priv, ret);
-}
-
 static int simple_count_noml(struct simple_util_priv *priv,
 			     struct device_node *np,
 			     struct device_node *codec,
@@ -699,28 +661,19 @@ static int simple_soc_probe(struct snd_soc_card *card)
 	if (ret < 0)
 		goto end;
 
-	ret = simple_util_init_aux_jacks(priv, PREFIX);
+	ret = simple_util_init_aux_jacks(card, PREFIX);
 end:
 	return simple_ret(priv, ret);
 }
 
-static int simple_probe(struct platform_device *pdev)
+static int simple_parse_of(struct simple_util_priv *priv)
 {
-	struct simple_util_priv *priv;
-	struct device *dev = &pdev->dev;
-	struct snd_soc_card *card;
-	int ret;
-
-	/* Allocate the private data and the DAI link array */
-	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
+	struct snd_soc_card *card = simple_priv_to_card(priv);
+	struct device *dev = card->dev;
+	int ret = -EINVAL;
 
-	card = simple_priv_to_card(priv);
-	card->owner		= THIS_MODULE;
-	card->dev		= dev;
-	card->probe		= simple_soc_probe;
-	card->driver_name       = "simple-card";
+	if (!dev)
+		return simple_ret(priv, ret);
 
 	ret = -ENOMEM;
 	struct link_info *li __free(kfree) = kzalloc_obj(*li);
@@ -739,25 +692,71 @@ static int simple_probe(struct platform_device *pdev)
 	if (ret < 0)
 		goto end;
 
-	ret = simple_parse_of(priv, li);
-	if (ret < 0) {
-		dev_err_probe(dev, ret, "parse error\n");
+	ret = simple_util_parse_widgets(priv, PREFIX);
+	if (ret < 0)
+		goto end;
+
+	ret = simple_util_parse_routing(priv, PREFIX);
+	if (ret < 0)
+		goto end;
+
+	ret = simple_util_parse_pin_switches(priv, PREFIX);
+	if (ret < 0)
+		goto end;
+
+	ret = simple_util_parse_aux_devs(priv, PREFIX);
+	if (ret < 0)
 		goto err;
-	}
 
-	snd_soc_card_set_drvdata(card, priv);
+	/* Single/Muti DAI link(s) & New style of DT node */
+	memset(li, 0, sizeof(*li));
+	ret = simple_for_each_link(priv, li,
+				   simple_dai_link_of,
+				   simple_dai_link_of_dpcm);
+	if (ret < 0)
+		goto err;
 
-	simple_util_debug_info(priv);
+	/* Card name should be set after simple_for_each_link() */
+	ret = simple_util_parse_card_name(priv, PREFIX);
+	if (ret < 0)
+		goto err;
 
-	ret = devm_snd_soc_register_card(dev, card);
+	ret = simple_populate_aux(priv);
 	if (ret < 0)
 		goto err;
 
-	return 0;
+	snd_soc_card_set_drvdata(card, priv);
+
+	simple_util_debug_info(priv);
+
+	ret = devm_snd_soc_register_card(dev, card);
 err:
-	simple_util_clean_reference(card);
+	if (ret < 0) {
+		simple_util_clean_reference(priv);
+		return dev_err_probe(dev, ret, "parse error\n");
+	}
 end:
-	return dev_err_probe(dev, ret, "parse error\n");
+	return simple_ret(priv, ret);
+}
+
+static int simple_probe(struct platform_device *pdev)
+{
+	struct simple_util_priv *priv;
+	struct device *dev = &pdev->dev;
+	struct snd_soc_card *card;
+
+	/* Allocate the private data and the DAI link array */
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	card = simple_priv_to_card(priv);
+	card->owner		= THIS_MODULE;
+	card->dev		= dev;
+	card->probe		= simple_soc_probe;
+	card->driver_name       = "simple-card";
+
+	return simple_parse_of(priv);
 }
 
 static const struct of_device_id simple_of_match[] = {
diff --git a/sound/soc/intel/atom/sst-atom-controls.c b/sound/soc/intel/atom/sst-atom-controls.c
index 3629ceaaac17c..82df398237da2 100644
--- a/sound/soc/intel/atom/sst-atom-controls.c
+++ b/sound/soc/intel/atom/sst-atom-controls.c
@@ -14,6 +14,7 @@
  */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <linux/cleanup.h>
 #include <linux/slab.h>
 #include <sound/soc.h>
 #include <sound/tlv.h>
@@ -73,14 +74,10 @@ static int sst_fill_and_send_cmd(struct sst_data *drv,
 				 u8 ipc_msg, u8 block, u8 task_id, u8 pipe_id,
 				 void *cmd_data, u16 len)
 {
-	int ret;
+	guard(mutex)(&drv->lock);
 
-	mutex_lock(&drv->lock);
-	ret = sst_fill_and_send_cmd_unlocked(drv, ipc_msg, block,
-					task_id, pipe_id, cmd_data, len);
-	mutex_unlock(&drv->lock);
-
-	return ret;
+	return sst_fill_and_send_cmd_unlocked(drv, ipc_msg, block,
+					      task_id, pipe_id, cmd_data, len);
 }
 
 /*
@@ -167,7 +164,7 @@ static int sst_slot_get(struct snd_kcontrol *kcontrol,
 	unsigned int val, mux;
 	u8 *map = is_tx ? sst_ssp_rx_map : sst_ssp_tx_map;
 
-	mutex_lock(&drv->lock);
+	guard(mutex)(&drv->lock);
 	val = 1 << ctl_no;
 	/* search which slot/channel has this bit set - there should be only one */
 	for (mux = e->max; mux > 0;  mux--)
@@ -175,7 +172,6 @@ static int sst_slot_get(struct snd_kcontrol *kcontrol,
 			break;
 
 	ucontrol->value.enumerated.item[0] = mux;
-	mutex_unlock(&drv->lock);
 
 	dev_dbg(c->dev, "%s - %s map = %#x\n",
 			is_tx ? "tx channel" : "rx slot",
@@ -235,7 +231,7 @@ static int sst_slot_put(struct snd_kcontrol *kcontrol,
 	if (mux > e->max - 1)
 		return -EINVAL;
 
-	mutex_lock(&drv->lock);
+	guard(mutex)(&drv->lock);
 	/* first clear all registers of this bit */
 	for (i = 0; i < e->max; i++)
 		map[i] &= ~val;
@@ -244,7 +240,6 @@ static int sst_slot_put(struct snd_kcontrol *kcontrol,
 		/* kctl set to 'none' and we reset the bits so send IPC */
 		ret = sst_check_and_send_slot_map(drv, kcontrol);
 
-		mutex_unlock(&drv->lock);
 		return ret;
 	}
 
@@ -258,7 +253,6 @@ static int sst_slot_put(struct snd_kcontrol *kcontrol,
 
 	ret = sst_check_and_send_slot_map(drv, kcontrol);
 
-	mutex_unlock(&drv->lock);
 	return ret;
 }
 
@@ -354,13 +348,12 @@ static int sst_algo_control_set(struct snd_kcontrol *kcontrol,
 	struct sst_algo_control *bc = (void *)kcontrol->private_value;
 
 	dev_dbg(cmpnt->dev, "control_name=%s\n", kcontrol->id.name);
-	mutex_lock(&drv->lock);
+	guard(mutex)(&drv->lock);
 	switch (bc->type) {
 	case SST_ALGO_PARAMS:
 		memcpy(bc->params, ucontrol->value.bytes.data, bc->max);
 		break;
 	default:
-		mutex_unlock(&drv->lock);
 		dev_err(cmpnt->dev, "Invalid Input- algo type:%d\n",
 				bc->type);
 		return -EINVAL;
@@ -368,7 +361,6 @@ static int sst_algo_control_set(struct snd_kcontrol *kcontrol,
 	/*if pipe is enabled, need to send the algo params from here*/
 	if (bc->w && bc->w->power)
 		ret = sst_send_algo_cmd(drv, bc);
-	mutex_unlock(&drv->lock);
 
 	return ret;
 }
@@ -475,7 +467,7 @@ static int sst_gain_put(struct snd_kcontrol *kcontrol,
 	struct sst_gain_mixer_control *mc = (void *)kcontrol->private_value;
 	struct sst_gain_value *gv = mc->gain_val;
 
-	mutex_lock(&drv->lock);
+	guard(mutex)(&drv->lock);
 
 	switch (mc->type) {
 	case SST_GAIN_TLV:
@@ -497,7 +489,6 @@ static int sst_gain_put(struct snd_kcontrol *kcontrol,
 		break;
 
 	default:
-		mutex_unlock(&drv->lock);
 		dev_err(cmpnt->dev, "Invalid Input- gain type:%d\n",
 				mc->type);
 		return -EINVAL;
@@ -506,7 +497,6 @@ static int sst_gain_put(struct snd_kcontrol *kcontrol,
 	if (mc->w && mc->w->power)
 		ret = sst_send_gain_cmd(drv, gv, mc->task_id,
 			mc->pipe_id | mc->instance_id, mc->module_id, 0);
-	mutex_unlock(&drv->lock);
 
 	return ret;
 }
@@ -521,10 +511,9 @@ static int sst_send_pipe_module_params(struct snd_soc_dapm_widget *w,
 	struct sst_data *drv = snd_soc_component_get_drvdata(c);
 	struct sst_ids *ids = w->priv;
 
-	mutex_lock(&drv->lock);
+	guard(mutex)(&drv->lock);
 	sst_find_and_send_pipe_algo(drv, w->name, ids);
 	sst_set_pipe_gain(ids, drv, 0);
-	mutex_unlock(&drv->lock);
 
 	return 0;
 }
@@ -761,27 +750,29 @@ int sst_handle_vb_timer(struct snd_soc_dai *dai, bool enable)
 			return ret;
 	}
 
-	mutex_lock(&drv->lock);
-	if (enable)
-		timer_usage++;
-	else
-		timer_usage--;
-
-	/*
-	 * Send the command only if this call is the first enable or last
-	 * disable
-	 */
-	if ((enable && (timer_usage == 1)) ||
-	    (!enable && (timer_usage == 0))) {
-		ret = sst_fill_and_send_cmd_unlocked(drv, SST_IPC_IA_CMD,
-				SST_FLAG_BLOCKED, SST_TASK_SBA, 0, &cmd,
-				sizeof(cmd.header) + cmd.header.length);
-		if (ret && enable) {
+	scoped_guard(mutex, &drv->lock) {
+		if (enable)
+			timer_usage++;
+		else
 			timer_usage--;
-			enable  = false;
+
+		/*
+		 * Send the command only if this call is the first enable or last
+		 * disable
+		 */
+		if ((enable && timer_usage == 1) ||
+		    (!enable && timer_usage == 0)) {
+			ret = sst_fill_and_send_cmd_unlocked(drv, SST_IPC_IA_CMD,
+							     SST_FLAG_BLOCKED,
+							     SST_TASK_SBA, 0, &cmd,
+							     sizeof(cmd.header) +
+							     cmd.header.length);
+			if (ret && enable) {
+				timer_usage--;
+				enable  = false;
+			}
 		}
 	}
-	mutex_unlock(&drv->lock);
 
 	if (!enable)
 		sst->ops->power(sst->dev, false);
diff --git a/sound/soc/intel/atom/sst-mfld-platform-pcm.c b/sound/soc/intel/atom/sst-mfld-platform-pcm.c
index f074af2499c83..9ee4d9926e06a 100644
--- a/sound/soc/intel/atom/sst-mfld-platform-pcm.c
+++ b/sound/soc/intel/atom/sst-mfld-platform-pcm.c
@@ -11,6 +11,7 @@
  */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <linux/cleanup.h>
 #include <linux/slab.h>
 #include <linux/io.h>
 #include <linux/module.h>
@@ -32,7 +33,7 @@ int sst_register_dsp(struct sst_device *dev)
 		return -EINVAL;
 	if (!try_module_get(dev->dev->driver->owner))
 		return -ENODEV;
-	mutex_lock(&sst_lock);
+	guard(mutex)(&sst_lock);
 	if (sst) {
 		dev_err(dev->dev, "we already have a device %s\n", sst->name);
 		module_put(dev->dev->driver->owner);
@@ -41,7 +42,7 @@ int sst_register_dsp(struct sst_device *dev)
 	}
 	dev_dbg(dev->dev, "registering device %s\n", dev->name);
 	sst = dev;
-	mutex_unlock(&sst_lock);
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(sst_register_dsp);
@@ -53,17 +54,15 @@ int sst_unregister_dsp(struct sst_device *dev)
 	if (dev != sst)
 		return -EINVAL;
 
-	mutex_lock(&sst_lock);
+	guard(mutex)(&sst_lock);
 
-	if (!sst) {
-		mutex_unlock(&sst_lock);
+	if (!sst)
 		return -EIO;
-	}
 
 	module_put(sst->dev->driver->owner);
 	dev_dbg(dev->dev, "unreg %s\n", sst->name);
 	sst = NULL;
-	mutex_unlock(&sst_lock);
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(sst_unregister_dsp);
@@ -103,21 +102,14 @@ static int sst_media_digital_mute(struct snd_soc_dai *dai, int mute, int stream)
 void sst_set_stream_status(struct sst_runtime_stream *stream,
 					int state)
 {
-	unsigned long flags;
-	spin_lock_irqsave(&stream->status_lock, flags);
+	guard(spinlock_irqsave)(&stream->status_lock);
 	stream->stream_status = state;
-	spin_unlock_irqrestore(&stream->status_lock, flags);
 }
 
 static inline int sst_get_stream_status(struct sst_runtime_stream *stream)
 {
-	int state;
-	unsigned long flags;
-
-	spin_lock_irqsave(&stream->status_lock, flags);
-	state = stream->stream_status;
-	spin_unlock_irqrestore(&stream->status_lock, flags);
-	return state;
+	guard(spinlock_irqsave)(&stream->status_lock);
+	return stream->stream_status;
 }
 
 static void sst_fill_alloc_params(struct snd_pcm_substream *substream,
@@ -304,7 +296,7 @@ static int sst_media_open(struct snd_pcm_substream *substream,
 {
 	int ret_val = 0;
 	struct snd_pcm_runtime *runtime = substream->runtime;
-	struct sst_runtime_stream *stream;
+	struct sst_runtime_stream *stream __free(kfree) = NULL;
 
 	stream = kzalloc_obj(*stream);
 	if (!stream)
@@ -312,15 +304,14 @@ static int sst_media_open(struct snd_pcm_substream *substream,
 	spin_lock_init(&stream->status_lock);
 
 	/* get the sst ops */
-	mutex_lock(&sst_lock);
-	if (!sst ||
-	    !try_module_get(sst->dev->driver->owner)) {
-		dev_err(dai->dev, "no device available to run\n");
-		ret_val = -ENODEV;
-		goto out_ops;
+	scoped_guard(mutex, &sst_lock) {
+		if (!sst ||
+		    !try_module_get(sst->dev->driver->owner)) {
+			dev_err(dai->dev, "no device available to run\n");
+			return -ENODEV;
+		}
+		stream->ops = sst->ops;
 	}
-	stream->ops = sst->ops;
-	mutex_unlock(&sst_lock);
 
 	stream->stream_info.str_id = 0;
 
@@ -330,7 +321,7 @@ static int sst_media_open(struct snd_pcm_substream *substream,
 
 	ret_val = power_up_sst(stream);
 	if (ret_val < 0)
-		goto out_power_up;
+		return ret_val;
 
 	/*
 	 * Make sure the period to be multiple of 1ms to align the
@@ -347,12 +338,14 @@ static int sst_media_open(struct snd_pcm_substream *substream,
 	snd_pcm_hw_constraint_step(substream->runtime, 0,
 			   SNDRV_PCM_HW_PARAM_PERIODS, 2);
 
-	return snd_pcm_hw_constraint_integer(runtime,
-			 SNDRV_PCM_HW_PARAM_PERIODS);
-out_ops:
-	mutex_unlock(&sst_lock);
-out_power_up:
-	kfree(stream);
+	ret_val = snd_pcm_hw_constraint_integer(runtime,
+						SNDRV_PCM_HW_PARAM_PERIODS);
+
+	if (ret_val < 0)
+		return ret_val;
+
+	stream = NULL;
+
 	return ret_val;
 }
 
diff --git a/sound/soc/intel/atom/sst/sst_ipc.c b/sound/soc/intel/atom/sst/sst_ipc.c
index 0d5e71e8a5b50..6c19ab63aa4f8 100644
--- a/sound/soc/intel/atom/sst/sst_ipc.c
+++ b/sound/soc/intel/atom/sst/sst_ipc.c
@@ -11,6 +11,7 @@
  *
  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  */
+#include <linux/cleanup.h>
 #include <linux/pci.h>
 #include <linux/firmware.h>
 #include <linux/sched.h>
@@ -180,9 +181,8 @@ void intel_sst_clear_intr_mrfld(struct intel_sst_drv *sst_drv_ctx)
 	union interrupt_reg_mrfld isr;
 	union interrupt_reg_mrfld imr;
 	union ipc_header_mrfld clear_ipc;
-	unsigned long irq_flags;
 
-	spin_lock_irqsave(&sst_drv_ctx->ipc_spin_lock, irq_flags);
+	guard(spinlock_irqsave)(&sst_drv_ctx->ipc_spin_lock);
 	imr.full = sst_shim_read64(sst_drv_ctx->shim, SST_IMRX);
 	isr.full = sst_shim_read64(sst_drv_ctx->shim, SST_ISRX);
 
@@ -200,7 +200,6 @@ void intel_sst_clear_intr_mrfld(struct intel_sst_drv *sst_drv_ctx)
 	/* un mask busy interrupt */
 	imr.part.busy_interrupt = 0;
 	sst_shim_write64(sst_drv_ctx->shim, SST_IMRX, imr.full);
-	spin_unlock_irqrestore(&sst_drv_ctx->ipc_spin_lock, irq_flags);
 }
 
 
diff --git a/sound/soc/intel/atom/sst/sst_pci.c b/sound/soc/intel/atom/sst/sst_pci.c
index 22ae2d22f121a..44bb11c694905 100644
--- a/sound/soc/intel/atom/sst/sst_pci.c
+++ b/sound/soc/intel/atom/sst/sst_pci.c
@@ -130,13 +130,15 @@ static int intel_sst_probe(struct pci_dev *pci,
 	sst_drv_ctx->pci = pci_dev_get(pci);
 	ret = sst_platform_get_resources(sst_drv_ctx);
 	if (ret < 0)
-		goto do_free_drv_ctx;
+		goto do_put_pci;
 
 	pci_set_drvdata(pci, sst_drv_ctx);
 	sst_configure_runtime_pm(sst_drv_ctx);
 
 	return ret;
 
+do_put_pci:
+	pci_dev_put(sst_drv_ctx->pci);
 do_free_drv_ctx:
 	sst_context_cleanup(sst_drv_ctx);
 	dev_err(sst_drv_ctx->dev, "Probe failed with %d\n", ret);
diff --git a/sound/soc/intel/atom/sst/sst_pvt.c b/sound/soc/intel/atom/sst/sst_pvt.c
index 67b1ab14239f8..0b0cfd70efbc2 100644
--- a/sound/soc/intel/atom/sst/sst_pvt.c
+++ b/sound/soc/intel/atom/sst/sst_pvt.c
@@ -11,6 +11,7 @@
  *
  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  */
+#include <linux/cleanup.h>
 #include <linux/kobject.h>
 #include <linux/pci.h>
 #include <linux/fs.h>
@@ -64,9 +65,8 @@ u64 sst_shim_read64(void __iomem *addr, int offset)
 void sst_set_fw_state_locked(
 		struct intel_sst_drv *sst_drv_ctx, int sst_state)
 {
-	mutex_lock(&sst_drv_ctx->sst_lock);
+	guard(mutex)(&sst_drv_ctx->sst_lock);
 	sst_drv_ctx->sst_state = sst_state;
-	mutex_unlock(&sst_drv_ctx->sst_lock);
 }
 
 /*
@@ -302,18 +302,17 @@ int sst_assign_pvt_id(struct intel_sst_drv *drv)
 {
 	int local;
 
-	spin_lock(&drv->block_lock);
+	guard(spinlock)(&drv->block_lock);
 	/* find first zero index from lsb */
 	local = ffz(drv->pvt_id);
 	dev_dbg(drv->dev, "pvt_id assigned --> %d\n", local);
 	if (local >= SST_MAX_BLOCKS){
-		spin_unlock(&drv->block_lock);
 		dev_err(drv->dev, "PVT _ID error: no free id blocks ");
 		return -EINVAL;
 	}
 	/* toggle the index */
 	change_bit(local, &drv->pvt_id);
-	spin_unlock(&drv->block_lock);
+
 	return local;
 }
 
diff --git a/sound/soc/intel/avs/apl.c b/sound/soc/intel/avs/apl.c
index b922eeaba843b..cf600e1c986e5 100644
--- a/sound/soc/intel/avs/apl.c
+++ b/sound/soc/intel/avs/apl.c
@@ -6,6 +6,7 @@
 //          Amadeusz Slawinski <amadeuszx.slawinski@linux.intel.com>
 //
 
+#include <linux/cleanup.h>
 #include <linux/devcoredump.h>
 #include <linux/slab.h>
 #include <sound/hdaudio_ext.h>
@@ -190,7 +191,7 @@ static bool avs_apl_lp_streaming(struct avs_dev *adev)
 {
 	struct avs_path *path;
 
-	spin_lock(&adev->path_list_lock);
+	guard(spinlock)(&adev->path_list_lock);
 	/* Any gateway without buffer allocated in LP area disqualifies D0IX. */
 	list_for_each_entry(path, &adev->path_list, node) {
 		struct avs_path_pipeline *ppl;
@@ -210,14 +211,11 @@ static bool avs_apl_lp_streaming(struct avs_dev *adev)
 				if (cfg->copier.dma_type == INVALID_OBJECT_ID)
 					continue;
 
-				if (!mod->gtw_attrs.lp_buffer_alloc) {
-					spin_unlock(&adev->path_list_lock);
+				if (!mod->gtw_attrs.lp_buffer_alloc)
 					return false;
-				}
 			}
 		}
 	}
-	spin_unlock(&adev->path_list_lock);
 
 	return true;
 }
diff --git a/sound/soc/intel/avs/control.c b/sound/soc/intel/avs/control.c
index a8f05de338e0e..370069247a7de 100644
--- a/sound/soc/intel/avs/control.c
+++ b/sound/soc/intel/avs/control.c
@@ -27,7 +27,7 @@ static struct avs_path_module *avs_get_volume_module(struct avs_dev *adev, u32 i
 	struct avs_path_pipeline *ppl;
 	struct avs_path_module *mod;
 
-	spin_lock(&adev->path_list_lock);
+	guard(spinlock)(&adev->path_list_lock);
 	list_for_each_entry(path, &adev->path_list, node) {
 		list_for_each_entry(ppl, &path->ppl_list, node) {
 			list_for_each_entry(mod, &ppl->mod_list, node) {
@@ -35,14 +35,11 @@ static struct avs_path_module *avs_get_volume_module(struct avs_dev *adev, u32 i
 
 				if ((guid_equal(type, &AVS_PEAKVOL_MOD_UUID) ||
 				     guid_equal(type, &AVS_GAIN_MOD_UUID)) &&
-				    mod->template->ctl_id == id) {
-					spin_unlock(&adev->path_list_lock);
+				    mod->template->ctl_id == id)
 					return mod;
-				}
 			}
 		}
 	}
-	spin_unlock(&adev->path_list_lock);
 
 	return NULL;
 }
diff --git a/sound/soc/intel/avs/core.c b/sound/soc/intel/avs/core.c
index 1a53856c2ffbe..2afe596468967 100644
--- a/sound/soc/intel/avs/core.c
+++ b/sound/soc/intel/avs/core.c
@@ -14,6 +14,7 @@
 // foundation of this driver
 //
 
+#include <linux/cleanup.h>
 #include <linux/acpi.h>
 #include <linux/module.h>
 #include <linux/pci.h>
@@ -273,7 +274,7 @@ static irqreturn_t avs_hda_interrupt(struct hdac_bus *bus)
 	if (snd_hdac_bus_handle_stream_irq(bus, status, hdac_update_stream))
 		ret = IRQ_HANDLED;
 
-	spin_lock_irq(&bus->reg_lock);
+	guard(spinlock_irq)(&bus->reg_lock);
 	/* Clear RIRB interrupt. */
 	status = snd_hdac_chip_readb(bus, RIRBSTS);
 	if (status & RIRB_INT_MASK) {
@@ -283,7 +284,6 @@ static irqreturn_t avs_hda_interrupt(struct hdac_bus *bus)
 		ret = IRQ_HANDLED;
 	}
 
-	spin_unlock_irq(&bus->reg_lock);
 	return ret;
 }
 
diff --git a/sound/soc/intel/avs/debug.h b/sound/soc/intel/avs/debug.h
index 94fe8729a5c1b..c47fc4e8b02b3 100644
--- a/sound/soc/intel/avs/debug.h
+++ b/sound/soc/intel/avs/debug.h
@@ -9,6 +9,7 @@
 #ifndef __SOUND_SOC_INTEL_AVS_DEBUG_H
 #define __SOUND_SOC_INTEL_AVS_DEBUG_H
 
+#include <linux/cleanup.h>
 #include "messages.h"
 #include "registers.h"
 
@@ -26,14 +27,9 @@ struct avs_dev;
 
 static inline int avs_log_buffer_status_locked(struct avs_dev *adev, union avs_notify_msg *msg)
 {
-	unsigned long flags;
-	int ret;
+	guard(spinlock_irqsave)(&adev->trace_lock);
 
-	spin_lock_irqsave(&adev->trace_lock, flags);
-	ret = avs_dsp_op(adev, log_buffer_status, msg);
-	spin_unlock_irqrestore(&adev->trace_lock, flags);
-
-	return ret;
+	return avs_dsp_op(adev, log_buffer_status, msg);
 }
 
 struct avs_apl_log_buffer_layout {
diff --git a/sound/soc/intel/avs/debugfs.c b/sound/soc/intel/avs/debugfs.c
index 701c247227bf9..9ab503da3b75d 100644
--- a/sound/soc/intel/avs/debugfs.c
+++ b/sound/soc/intel/avs/debugfs.c
@@ -6,6 +6,7 @@
 //          Amadeusz Slawinski <amadeuszx.slawinski@linux.intel.com>
 //
 
+#include <linux/cleanup.h>
 #include <linux/debugfs.h>
 #include <linux/kfifo.h>
 #include <linux/wait.h>
@@ -251,24 +252,22 @@ static int strace_release(struct inode *inode, struct file *file)
 	union avs_notify_msg msg = AVS_NOTIFICATION(LOG_BUFFER_STATUS);
 	struct avs_dev *adev = file->private_data;
 	unsigned long resource_mask;
-	unsigned long flags, i;
+	unsigned long i;
 	u32 num_cores;
 
 	resource_mask = adev->logged_resources;
 	num_cores = adev->hw_cfg.dsp_cores;
 
-	spin_lock_irqsave(&adev->trace_lock, flags);
+	scoped_guard(spinlock_irqsave, &adev->trace_lock) {
+		/* Gather any remaining logs. */
+		for_each_set_bit(i, &resource_mask, num_cores) {
+			msg.log.core = i;
+			avs_dsp_op(adev, log_buffer_status, &msg);
+		}
 
-	/* Gather any remaining logs. */
-	for_each_set_bit(i, &resource_mask, num_cores) {
-		msg.log.core = i;
-		avs_dsp_op(adev, log_buffer_status, &msg);
+		kfifo_free(&adev->trace_fifo);
 	}
 
-	kfifo_free(&adev->trace_fifo);
-
-	spin_unlock_irqrestore(&adev->trace_lock, flags);
-
 	module_put(adev->dev->driver->owner);
 	return 0;
 }
diff --git a/sound/soc/intel/avs/ipc.c b/sound/soc/intel/avs/ipc.c
index c0feb9edd7f6b..71e7997e52c2f 100644
--- a/sound/soc/intel/avs/ipc.c
+++ b/sound/soc/intel/avs/ipc.c
@@ -6,6 +6,7 @@
 //          Amadeusz Slawinski <amadeuszx.slawinski@linux.intel.com>
 //
 
+#include <linux/cleanup.h>
 #include <linux/io-64-nonatomic-lo-hi.h>
 #include <linux/slab.h>
 #include <sound/hdaudio_ext.h>
@@ -99,39 +100,39 @@ static void avs_dsp_recovery(struct avs_dev *adev)
 	unsigned int core_mask;
 	int ret;
 
-	mutex_lock(&adev->comp_list_mutex);
-	/* disconnect all running streams */
-	list_for_each_entry(acomp, &adev->comp_list, node) {
-		struct snd_soc_pcm_runtime *rtd;
-		struct snd_soc_card *card;
+	scoped_guard(mutex, &adev->comp_list_mutex) {
+		/* disconnect all running streams */
+		list_for_each_entry(acomp, &adev->comp_list, node) {
+			struct snd_soc_pcm_runtime *rtd;
+			struct snd_soc_card *card;
 
-		card = acomp->base.card;
-		if (!card)
-			continue;
-
-		for_each_card_rtds(card, rtd) {
-			struct snd_pcm *pcm;
-			int dir;
-
-			pcm = rtd->pcm;
-			if (!pcm || rtd->dai_link->no_pcm)
+			card = acomp->base.card;
+			if (!card)
 				continue;
 
-			for_each_pcm_streams(dir) {
-				struct snd_pcm_substream *substream;
+			for_each_card_rtds(card, rtd) {
+				struct snd_pcm *pcm;
+				int dir;
 
-				substream = pcm->streams[dir].substream;
-				if (!substream || !substream->runtime)
+				pcm = rtd->pcm;
+				if (!pcm || rtd->dai_link->no_pcm)
 					continue;
 
-				/* No need for _irq() as we are in nonatomic context. */
-				snd_pcm_stream_lock(substream);
-				snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED);
-				snd_pcm_stream_unlock(substream);
+				for_each_pcm_streams(dir) {
+					struct snd_pcm_substream *substream;
+
+					substream = pcm->streams[dir].substream;
+					if (!substream || !substream->runtime)
+						continue;
+
+					/* No need for _irq() as we are in nonatomic context. */
+					snd_pcm_stream_lock(substream);
+					snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED);
+					snd_pcm_stream_unlock(substream);
+				}
 			}
 		}
 	}
-	mutex_unlock(&adev->comp_list_mutex);
 
 	/* forcibly shutdown all cores */
 	core_mask = GENMASK(adev->hw_cfg.dsp_cores - 1, 0);
@@ -397,7 +398,7 @@ static int avs_dsp_do_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request
 	if (!ipc->ready)
 		return -EPERM;
 
-	mutex_lock(&ipc->msg_mutex);
+	guard(mutex)(&ipc->msg_mutex);
 
 	spin_lock(&ipc->rx_lock);
 	avs_ipc_msg_init(ipc, reply);
@@ -412,7 +413,7 @@ static int avs_dsp_do_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request
 			/* Same treatment as on exception, just stack_dump=0. */
 			avs_dsp_exception_caught(adev, &msg);
 		}
-		goto exit;
+		return ret;
 	}
 
 	ret = ipc->rx.rsp.status;
@@ -436,8 +437,6 @@ static int avs_dsp_do_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request
 			memcpy(reply->data, ipc->rx.data, reply->size);
 	}
 
-exit:
-	mutex_unlock(&ipc->msg_mutex);
 	return ret;
 }
 
@@ -501,7 +500,7 @@ static int avs_dsp_do_send_rom_msg(struct avs_dev *adev, struct avs_ipc_msg *req
 	struct avs_ipc *ipc = adev->ipc;
 	int ret;
 
-	mutex_lock(&ipc->msg_mutex);
+	guard(mutex)(&ipc->msg_mutex);
 
 	spin_lock(&ipc->rx_lock);
 	avs_ipc_msg_init(ipc, NULL);
@@ -522,8 +521,6 @@ static int avs_dsp_do_send_rom_msg(struct avs_dev *adev, struct avs_ipc_msg *req
 		dev_err(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n",
 			name, request->glb.primary, request->glb.ext.val, ret);
 
-	mutex_unlock(&ipc->msg_mutex);
-
 	return ret;
 }
 
diff --git a/sound/soc/intel/avs/loader.c b/sound/soc/intel/avs/loader.c
index 353e343b1d288..bebdc79ec88e6 100644
--- a/sound/soc/intel/avs/loader.c
+++ b/sound/soc/intel/avs/loader.c
@@ -6,6 +6,7 @@
 //          Amadeusz Slawinski <amadeuszx.slawinski@linux.intel.com>
 //
 
+#include <linux/cleanup.h>
 #include <linux/firmware.h>
 #include <linux/module.h>
 #include <linux/slab.h>
@@ -630,15 +631,15 @@ static int avs_load_firmware(struct avs_dev *adev, bool purge)
 	if (ret)
 		goto reenable_gating;
 
-	mutex_lock(&adev->comp_list_mutex);
-	list_for_each_entry(acomp, &adev->comp_list, node) {
-		struct avs_tplg *tplg = acomp->tplg;
+	scoped_guard(mutex, &adev->comp_list_mutex) {
+		list_for_each_entry(acomp, &adev->comp_list, node) {
+			struct avs_tplg *tplg = acomp->tplg;
 
-		ret = avs_dsp_load_libraries(adev, tplg->libs, tplg->num_libs);
-		if (ret < 0)
-			break;
+			ret = avs_dsp_load_libraries(adev, tplg->libs, tplg->num_libs);
+			if (ret < 0)
+				break;
+		}
 	}
-	mutex_unlock(&adev->comp_list_mutex);
 
 reenable_gating:
 	avs_hda_l1sen_enable(adev, true);
diff --git a/sound/soc/intel/avs/path.c b/sound/soc/intel/avs/path.c
index 2291f9728a54f..213d6ecdd7ccd 100644
--- a/sound/soc/intel/avs/path.c
+++ b/sound/soc/intel/avs/path.c
@@ -6,6 +6,7 @@
 //          Amadeusz Slawinski <amadeuszx.slawinski@linux.intel.com>
 //
 
+#include <linux/cleanup.h>
 #include <linux/acpi.h>
 #include <acpi/nhlt.h>
 #include <sound/pcm_params.h>
@@ -69,16 +70,13 @@ avs_path_find_path(struct avs_dev *adev, const char *name, u32 template_id)
 	if (!template)
 		return NULL;
 
-	spin_lock(&adev->path_list_lock);
+	guard(spinlock)(&adev->path_list_lock);
 	/* Only one variant of given path template may be instantiated at a time. */
 	list_for_each_entry(path, &adev->path_list, node) {
-		if (path->template->owner == template) {
-			spin_unlock(&adev->path_list_lock);
+		if (path->template->owner == template)
 			return path;
-		}
 	}
 
-	spin_unlock(&adev->path_list_lock);
 	return NULL;
 }
 
@@ -1305,7 +1303,7 @@ void avs_path_free(struct avs_path *path)
 	struct avs_path *cpath, *csave;
 	struct avs_dev *adev = path->owner;
 
-	mutex_lock(&adev->path_mutex);
+	guard(mutex)(&adev->path_mutex);
 
 	/* Free all condpaths this path spawned. */
 	list_for_each_entry_safe(cpath, csave, &path->source_list, source_node)
@@ -1314,8 +1312,6 @@ void avs_path_free(struct avs_path *path)
 		avs_condpath_free(path->owner, cpath);
 
 	avs_path_free_unlocked(path);
-
-	mutex_unlock(&adev->path_mutex);
 }
 
 struct avs_path *avs_path_create(struct avs_dev *adev, u32 dma_id,
@@ -1334,13 +1330,13 @@ struct avs_path *avs_path_create(struct avs_dev *adev, u32 dma_id,
 	}
 
 	/* Serialize path and its components creation. */
-	mutex_lock(&adev->path_mutex);
+	guard(mutex)(&adev->path_mutex);
 	/* Satisfy needs of avs_path_find_tplg(). */
-	mutex_lock(&adev->comp_list_mutex);
+	guard(mutex)(&adev->comp_list_mutex);
 
 	path = avs_path_create_unlocked(adev, dma_id, variant);
 	if (IS_ERR(path))
-		goto exit;
+		return path;
 
 	ret = avs_condpaths_walk_all(adev, path);
 	if (ret) {
@@ -1348,10 +1344,6 @@ struct avs_path *avs_path_create(struct avs_dev *adev, u32 dma_id,
 		path = ERR_PTR(ret);
 	}
 
-exit:
-	mutex_unlock(&adev->comp_list_mutex);
-	mutex_unlock(&adev->path_mutex);
-
 	return path;
 }
 
@@ -1496,15 +1488,13 @@ static void avs_condpaths_pause(struct avs_dev *adev, struct avs_path *path)
 {
 	struct avs_path *cpath;
 
-	mutex_lock(&adev->path_mutex);
+	guard(mutex)(&adev->path_mutex);
 
 	/* If either source or sink stops, so do the attached conditional paths. */
 	list_for_each_entry(cpath, &path->source_list, source_node)
 		avs_condpath_pause(adev, cpath);
 	list_for_each_entry(cpath, &path->sink_list, sink_node)
 		avs_condpath_pause(adev, cpath);
-
-	mutex_unlock(&adev->path_mutex);
 }
 
 int avs_path_pause(struct avs_path *path)
@@ -1560,7 +1550,7 @@ static void avs_condpaths_run(struct avs_dev *adev, struct avs_path *path, int t
 {
 	struct avs_path *cpath;
 
-	mutex_lock(&adev->path_mutex);
+	guard(mutex)(&adev->path_mutex);
 
 	/* Run conditional paths only if source and sink are both running. */
 	list_for_each_entry(cpath, &path->source_list, source_node)
@@ -1572,8 +1562,6 @@ static void avs_condpaths_run(struct avs_dev *adev, struct avs_path *path, int t
 		if (cpath->source->state == AVS_PPL_STATE_RUNNING &&
 		    cpath->sink->state == AVS_PPL_STATE_RUNNING)
 			avs_condpath_run(adev, cpath, trigger);
-
-	mutex_unlock(&adev->path_mutex);
 }
 
 int avs_path_run(struct avs_path *path, int trigger)
diff --git a/sound/soc/intel/avs/utils.c b/sound/soc/intel/avs/utils.c
index ee36725ac7318..ea14ec1738552 100644
--- a/sound/soc/intel/avs/utils.c
+++ b/sound/soc/intel/avs/utils.c
@@ -6,6 +6,7 @@
 //          Amadeusz Slawinski <amadeuszx.slawinski@linux.intel.com>
 //
 
+#include <linux/cleanup.h>
 #include <linux/firmware.h>
 #include <linux/kfifo.h>
 #include <linux/slab.h>
@@ -48,13 +49,12 @@ int avs_get_module_entry(struct avs_dev *adev, const guid_t *uuid, struct avs_mo
 {
 	int idx;
 
-	mutex_lock(&adev->modres_mutex);
+	guard(mutex)(&adev->modres_mutex);
 
 	idx = avs_module_entry_index(adev, uuid);
 	if (idx >= 0)
 		memcpy(entry, &adev->mods_info->entries[idx], sizeof(*entry));
 
-	mutex_unlock(&adev->modres_mutex);
 	return (idx < 0) ? idx : 0;
 }
 
@@ -62,13 +62,12 @@ int avs_get_module_id_entry(struct avs_dev *adev, u32 module_id, struct avs_modu
 {
 	int idx;
 
-	mutex_lock(&adev->modres_mutex);
+	guard(mutex)(&adev->modres_mutex);
 
 	idx = avs_module_id_entry_index(adev, module_id);
 	if (idx >= 0)
 		memcpy(entry, &adev->mods_info->entries[idx], sizeof(*entry));
 
-	mutex_unlock(&adev->modres_mutex);
 	return (idx < 0) ? idx : 0;
 }
 
@@ -86,13 +85,12 @@ bool avs_is_module_ida_empty(struct avs_dev *adev, u32 module_id)
 	bool ret = false;
 	int idx;
 
-	mutex_lock(&adev->modres_mutex);
+	guard(mutex)(&adev->modres_mutex);
 
 	idx = avs_module_id_entry_index(adev, module_id);
 	if (idx >= 0)
 		ret = ida_is_empty(adev->mod_idas[idx]);
 
-	mutex_unlock(&adev->modres_mutex);
 	return ret;
 }
 
@@ -163,68 +161,57 @@ int avs_module_info_init(struct avs_dev *adev, bool purge)
 	if (ret)
 		return AVS_IPC_RET(ret);
 
-	mutex_lock(&adev->modres_mutex);
+	guard(mutex)(&adev->modres_mutex);
 
 	ret = avs_module_ida_alloc(adev, info, purge);
 	if (ret < 0) {
 		dev_err(adev->dev, "initialize module idas failed: %d\n", ret);
-		goto exit;
+		return ret;
 	}
 
 	/* Refresh current information with newly received table. */
 	kfree(adev->mods_info);
 	adev->mods_info = info;
 
-exit:
-	mutex_unlock(&adev->modres_mutex);
 	return ret;
 }
 
 void avs_module_info_free(struct avs_dev *adev)
 {
-	mutex_lock(&adev->modres_mutex);
+	guard(mutex)(&adev->modres_mutex);
 
 	avs_module_ida_destroy(adev);
 	kfree(adev->mods_info);
 	adev->mods_info = NULL;
-
-	mutex_unlock(&adev->modres_mutex);
 }
 
 int avs_module_id_alloc(struct avs_dev *adev, u16 module_id)
 {
-	int ret, idx, max_id;
+	int idx, max_id;
 
-	mutex_lock(&adev->modres_mutex);
+	guard(mutex)(&adev->modres_mutex);
 
 	idx = avs_module_id_entry_index(adev, module_id);
 	if (idx == -ENOENT) {
 		dev_err(adev->dev, "invalid module id: %d", module_id);
-		ret = -EINVAL;
-		goto exit;
+		return -EINVAL;
 	}
 	max_id = adev->mods_info->entries[idx].instance_max_count - 1;
-	ret = ida_alloc_max(adev->mod_idas[idx], max_id, GFP_KERNEL);
-exit:
-	mutex_unlock(&adev->modres_mutex);
-	return ret;
+
+	return ida_alloc_max(adev->mod_idas[idx], max_id, GFP_KERNEL);
 }
 
 void avs_module_id_free(struct avs_dev *adev, u16 module_id, u8 instance_id)
 {
 	int idx;
 
-	mutex_lock(&adev->modres_mutex);
+	guard(mutex)(&adev->modres_mutex);
 
 	idx = avs_module_id_entry_index(adev, module_id);
-	if (idx == -ENOENT) {
+	if (idx == -ENOENT)
 		dev_err(adev->dev, "invalid module id: %d", module_id);
-		goto exit;
-	}
-
-	ida_free(adev->mod_idas[idx], instance_id);
-exit:
-	mutex_unlock(&adev->modres_mutex);
+	else
+		ida_free(adev->mod_idas[idx], instance_id);
 }
 
 /*
diff --git a/sound/soc/intel/boards/sof_sdw.c b/sound/soc/intel/boards/sof_sdw.c
index d43daf9b025df..1013e07052a37 100644
--- a/sound/soc/intel/boards/sof_sdw.c
+++ b/sound/soc/intel/boards/sof_sdw.c
@@ -849,12 +849,15 @@ static const struct dmi_system_id sof_sdw_quirk_table[] = {
 };
 
 static const struct snd_pci_quirk sof_sdw_ssid_quirk_table[] = {
+	SND_PCI_QUIRK(0x1028, 0x0e53, "Dell XPS WCL", SOC_SDW_SIDECAR_AMPS),
+	SND_PCI_QUIRK(0x1028, 0x0e54, "Dell XPS PTL", SOC_SDW_SIDECAR_AMPS),
 	SND_PCI_QUIRK(0x1043, 0x1e13, "ASUS Zenbook S14", SOC_SDW_CODEC_MIC),
 	SND_PCI_QUIRK(0x1043, 0x1f43, "ASUS Zenbook S16", SOC_SDW_CODEC_MIC),
 	SND_PCI_QUIRK(0x17aa, 0x2347, "Lenovo P16", SOC_SDW_CODEC_MIC),
 	SND_PCI_QUIRK(0x17aa, 0x2348, "Lenovo P16", SOC_SDW_CODEC_MIC),
 	SND_PCI_QUIRK(0x17aa, 0x2349, "Lenovo P1", SOC_SDW_CODEC_MIC),
 	SND_PCI_QUIRK(0x17aa, 0x3821, "Lenovo 0x3821", SOC_SDW_SIDECAR_AMPS),
+	SND_PCI_QUIRK(0x17aa, 0x383c, "Lenovo 0x383c", SOC_SDW_SIDECAR_AMPS),
 	{}
 };
 
@@ -1285,7 +1288,7 @@ static int sof_card_dai_links_create(struct snd_soc_card *card)
 		goto err_dai;
 	}
 
-	ret = asoc_sdw_parse_sdw_endpoints(card, sof_aux, sof_dais, sof_ends, &num_confs);
+	ret = asoc_sdw_parse_sdw_endpoints(dev, ctx, sof_aux, sof_dais, sof_ends, &num_confs);
 	if (ret < 0)
 		goto err_end;
 
diff --git a/sound/soc/loongson/loongson_card.c b/sound/soc/loongson/loongson_card.c
index 7910d5d9ac4ff..25cd12eab4b1f 100644
--- a/sound/soc/loongson/loongson_card.c
+++ b/sound/soc/loongson/loongson_card.c
@@ -2,24 +2,131 @@
 //
 // Loongson ASoC Audio Machine driver
 //
-// Copyright (C) 2023 Loongson Technology Corporation Limited
+// Copyright (C) 2023-2026 Loongson Technology Corporation Limited
 // Author: Yingkun Meng <mengyingkun@loongson.cn>
+//         Binbin Zhou <zhoubinbin@loongson.cn>
 //
 
-#include <linux/module.h>
-#include <sound/soc.h>
-#include <sound/soc-acpi.h>
 #include <linux/acpi.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
 #include <linux/pci.h>
+#include <sound/jack.h>
 #include <sound/pcm_params.h>
+#include <sound/soc.h>
+#include <sound/soc-acpi.h>
 
 static char codec_name[SND_ACPI_I2C_ID_LEN];
 
 struct loongson_card_data {
 	struct snd_soc_card snd_card;
 	unsigned int mclk_fs;
+	struct gpio_desc *gpiod_hp_det;
+	struct gpio_desc *gpiod_hp_ctl;
+	struct gpio_desc *gpiod_spkr_en;
+	const struct loongson_card_config *cfg;
+};
+
+struct loongson_card_config {
+	unsigned int fmt;
+	bool add_hp_jack;
+	bool add_dapm_widgets;
+	bool add_dapm_routes;
+};
+
+static const struct loongson_card_config ls2k1000_card_config = {
+	.fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_IB_NF | SND_SOC_DAIFMT_CBC_CFC,
+	.add_hp_jack = false,
+	.add_dapm_widgets = false,
+	.add_dapm_routes = false,
+};
+
+static const struct loongson_card_config ls2k0300_forever_pi_card_config = {
+	.fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF | SND_SOC_DAIFMT_CBC_CFC,
+	.add_hp_jack = false,
+	.add_dapm_widgets = false,
+	.add_dapm_routes = false,
+};
+
+static const struct loongson_card_config ls2k0300_dl2k0300b_card_config = {
+	.fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF | SND_SOC_DAIFMT_CBC_CFC,
+	.add_hp_jack = true,
+	.add_dapm_widgets = true,
+	.add_dapm_routes = true,
+};
+
+static int tegra_machine_event(struct snd_soc_dapm_widget *w,
+			       struct snd_kcontrol *k, int event)
+{
+	struct snd_soc_card *card = snd_soc_dapm_to_card(w->dapm);
+	struct loongson_card_data *priv = snd_soc_card_get_drvdata(card);
+
+	if (!snd_soc_dapm_widget_name_cmp(w, "Speaker"))
+		gpiod_set_value_cansleep(priv->gpiod_spkr_en,
+					 SND_SOC_DAPM_EVENT_ON(event));
+
+	if (!snd_soc_dapm_widget_name_cmp(w, "Headphone"))
+		gpiod_set_value_cansleep(priv->gpiod_hp_ctl,
+					 SND_SOC_DAPM_EVENT_ON(event));
+
+	return 0;
+}
+
+static const struct snd_soc_dapm_widget loongson_aosc_dapm_widgets[] = {
+	SND_SOC_DAPM_HP("Headphone", tegra_machine_event),
+	SND_SOC_DAPM_SPK("Speaker", tegra_machine_event),
 };
 
+/* Headphones Jack */
+
+static struct snd_soc_jack loongson_asoc_hp_jack;
+
+static struct snd_soc_jack_pin loongson_asoc_hp_jack_pins[] = {
+	{
+		.pin = "Headphone",
+		.mask = SND_JACK_HEADPHONE
+	},
+	{
+		.pin = "Speaker",
+		.mask = SND_JACK_HEADPHONE,
+		.invert = 1
+	},
+};
+
+static struct snd_soc_jack_gpio loongson_asoc_hp_jack_gpio = {
+	.name = "Headphones detection",
+	.report = SND_JACK_HEADPHONE,
+	.debounce_time = 150,
+};
+
+static int loongson_asoc_machine_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_card *card = rtd->card;
+	struct loongson_card_data *ls_priv = snd_soc_card_get_drvdata(card);
+	int ret = 0;
+
+	if (!ls_priv->cfg->add_hp_jack || !ls_priv->gpiod_hp_det)
+		return 0;
+
+	ret = snd_soc_card_jack_new_pins(card, "Headphones Jack",
+					 SND_JACK_HEADPHONE,
+					 &loongson_asoc_hp_jack,
+					 loongson_asoc_hp_jack_pins,
+					 ARRAY_SIZE(loongson_asoc_hp_jack_pins));
+	if (ret) {
+		dev_err(rtd->dev, "Headphones Jack creation failed: %d\n", ret);
+		return ret;
+	}
+
+	loongson_asoc_hp_jack_gpio.desc = ls_priv->gpiod_hp_det;
+
+	ret = snd_soc_jack_add_gpios(&loongson_asoc_hp_jack, 1, &loongson_asoc_hp_jack_gpio);
+	if (ret)
+		dev_err(rtd->dev, "Headphone GPIO not added: %d\n", ret);
+
+	return ret;
+}
+
 static int loongson_card_hw_params(struct snd_pcm_substream *substream,
 				   struct snd_pcm_hw_params *params)
 {
@@ -45,7 +152,7 @@ static int loongson_card_hw_params(struct snd_pcm_substream *substream,
 		return ret;
 	}
 
-	return 0;
+	return snd_soc_runtime_set_dai_fmt(rtd, ls_card->cfg->fmt);
 }
 
 static const struct snd_soc_ops loongson_ops = {
@@ -61,8 +168,7 @@ static struct snd_soc_dai_link loongson_dai_links[] = {
 	{
 		.name = "Loongson Audio Port",
 		.stream_name = "Loongson Audio",
-		.dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_IB_NF
-			| SND_SOC_DAIFMT_CBC_CFC,
+		.init = loongson_asoc_machine_init,
 		SND_SOC_DAILINK_REG(analog),
 		.ops = &loongson_ops,
 	},
@@ -91,7 +197,7 @@ static int loongson_card_parse_acpi(struct loongson_card_data *data)
 	const char *codec_dai_name;
 	struct acpi_device *adev;
 	struct device *phy_dev;
-	int i;
+	int i, ret;
 
 	/* fixup platform name based on reference node */
 	adev = loongson_card_acpi_find_device(card, "cpu");
@@ -108,7 +214,9 @@ static int loongson_card_parse_acpi(struct loongson_card_data *data)
 		return -ENOENT;
 	snprintf(codec_name, sizeof(codec_name), "i2c-%s", acpi_dev_name(adev));
 
-	device_property_read_string(card->dev, "codec-dai-name", &codec_dai_name);
+	ret = device_property_read_string(card->dev, "codec-dai-name", &codec_dai_name);
+	if (ret)
+		return ret;
 
 	for (i = 0; i < card->num_links; i++) {
 		loongson_dai_links[i].platforms->name = dev_name(phy_dev);
@@ -121,16 +229,35 @@ static int loongson_card_parse_acpi(struct loongson_card_data *data)
 
 static int loongson_card_parse_of(struct loongson_card_data *data)
 {
-	struct device_node *cpu, *codec;
 	struct snd_soc_card *card = &data->snd_card;
+	struct device_node *cpu, *codec;
 	struct device *dev = card->dev;
 	int ret, i;
 
+	data->gpiod_hp_det = devm_gpiod_get_optional(dev, "hp-det", GPIOD_IN);
+	if (IS_ERR(data->gpiod_hp_det))
+		return PTR_ERR(data->gpiod_hp_det);
+
+	data->gpiod_hp_ctl = devm_gpiod_get_optional(dev, "hp-ctl", GPIOD_OUT_LOW);
+	if (IS_ERR(data->gpiod_hp_ctl))
+		return PTR_ERR(data->gpiod_hp_ctl);
+
+	data->gpiod_spkr_en = devm_gpiod_get_optional(dev, "spkr-en", GPIOD_OUT_LOW);
+	if (IS_ERR(data->gpiod_spkr_en))
+		return PTR_ERR(data->gpiod_spkr_en);
+
+	if (data->cfg->add_dapm_routes) {
+		ret = snd_soc_of_parse_audio_routing(card, "audio-routing");
+		if (ret)
+			return ret;
+	}
+
 	cpu = of_get_child_by_name(dev->of_node, "cpu");
 	if (!cpu) {
 		dev_err(dev, "platform property missing or invalid\n");
 		return -EINVAL;
 	}
+
 	codec = of_get_child_by_name(dev->of_node, "codec");
 	if (!codec) {
 		dev_err(dev, "audio-codec property missing or invalid\n");
@@ -175,12 +302,22 @@ static int loongson_asoc_card_probe(struct platform_device *pdev)
 	if (!ls_priv)
 		return -ENOMEM;
 
+	ls_priv->cfg = (const struct loongson_card_config *)device_get_match_data(dev);
+	if (!ls_priv->cfg)
+		return -EINVAL;
+
 	card = &ls_priv->snd_card;
 
 	card->dev = dev;
 	card->owner = THIS_MODULE;
 	card->dai_link = loongson_dai_links;
 	card->num_links = ARRAY_SIZE(loongson_dai_links);
+
+	if (ls_priv->cfg->add_dapm_widgets) {
+		card->dapm_widgets = loongson_aosc_dapm_widgets;
+		card->num_dapm_widgets = ARRAY_SIZE(loongson_aosc_dapm_widgets);
+	}
+
 	snd_soc_card_set_drvdata(card, ls_priv);
 
 	ret = device_property_read_string(dev, "model", &card->name);
@@ -200,7 +337,19 @@ static int loongson_asoc_card_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id loongson_asoc_dt_ids[] = {
-	{ .compatible = "loongson,ls-audio-card" },
+	/* Loongson-2K1000/Loongson-2K2000/LS7A */
+	{
+		.compatible = "loongson,ls-audio-card",
+		.data = &ls2k1000_card_config
+	},
+	{
+		.compatible = "loongson,ls2k0300-forever-pi-audio-card",
+		.data = &ls2k0300_forever_pi_card_config
+	},
+	{
+		.compatible = "loongson,ls2k0300-dl2k0300b-audio-card",
+		.data = &ls2k0300_dl2k0300b_card_config
+	},
 	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, loongson_asoc_dt_ids);
diff --git a/sound/soc/loongson/loongson_i2s_plat.c b/sound/soc/loongson/loongson_i2s_plat.c
index ac054b6ce6322..82d95c6644ef0 100644
--- a/sound/soc/loongson/loongson_i2s_plat.c
+++ b/sound/soc/loongson/loongson_i2s_plat.c
@@ -2,7 +2,7 @@
 //
 // Loongson I2S controller master mode dirver(platform device)
 //
-// Copyright (C) 2023-2024 Loongson Technology Corporation Limited
+// Copyright (C) 2023-2026 Loongson Technology Corporation Limited
 //
 // Author: Yingkun Meng <mengyingkun@loongson.cn>
 //         Binbin Zhou <zhoubinbin@loongson.cn>
@@ -21,6 +21,7 @@
 #include "loongson_i2s.h"
 #include "loongson_dma.h"
 
+/* Loongson-2K1000 APBDMA routing */
 #define LOONGSON_I2S_RX_DMA_OFFSET	21
 #define LOONGSON_I2S_TX_DMA_OFFSET	18
 
@@ -30,6 +31,11 @@
 #define LOONGSON_DMA3_CONF	0x3
 #define LOONGSON_DMA4_CONF	0x4
 
+struct loongson_i2s_plat_config {
+	int rev_id;
+	int (*i2s_dma_config)(struct platform_device *pdev);
+};
+
 static int loongson_i2s_apbdma_config(struct platform_device *pdev)
 {
 	int val;
@@ -47,8 +53,18 @@ static int loongson_i2s_apbdma_config(struct platform_device *pdev)
 	return 0;
 }
 
+static const struct loongson_i2s_plat_config ls2k0300_i2s_plat_config = {
+	.rev_id = 1,
+};
+
+static const struct loongson_i2s_plat_config ls2k1000_i2s_plat_config = {
+	.rev_id = 0,
+	.i2s_dma_config = loongson_i2s_apbdma_config,
+};
+
 static int loongson_i2s_plat_probe(struct platform_device *pdev)
 {
+	const struct loongson_i2s_plat_config *plat_config;
 	struct device *dev = &pdev->dev;
 	struct loongson_i2s *i2s;
 	struct resource *res;
@@ -59,12 +75,17 @@ static int loongson_i2s_plat_probe(struct platform_device *pdev)
 	if (!i2s)
 		return -ENOMEM;
 
-	ret = loongson_i2s_apbdma_config(pdev);
-	if (ret)
-		return ret;
+	plat_config = device_get_match_data(dev);
+	if (!plat_config)
+		return -EINVAL;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	i2s->reg_base = devm_ioremap_resource(&pdev->dev, res);
+	if (plat_config->i2s_dma_config) {
+		ret = plat_config->i2s_dma_config(pdev);
+		if (ret)
+			return ret;
+	}
+
+	i2s->reg_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(i2s->reg_base))
 		return dev_err_probe(dev, PTR_ERR(i2s->reg_base),
 				     "devm_ioremap_resource failed\n");
@@ -87,11 +108,17 @@ static int loongson_i2s_plat_probe(struct platform_device *pdev)
 	if (IS_ERR(i2s_clk))
 		return dev_err_probe(dev, PTR_ERR(i2s_clk), "clock property invalid\n");
 	i2s->clk_rate = clk_get_rate(i2s_clk);
+	i2s->rev_id = plat_config->rev_id;
 
 	dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
 	dev_set_name(dev, LS_I2S_DRVNAME);
 	dev_set_drvdata(dev, i2s);
 
+	if (i2s->rev_id == 1) {
+		regmap_update_bits(i2s->regmap, LS_I2S_CTRL, I2S_CTRL_RESET, I2S_CTRL_RESET);
+		fsleep(200);
+	}
+
 	ret = devm_snd_soc_register_component(dev, &loongson_i2s_edma_component,
 					      &loongson_i2s_dai, 1);
 	if (ret)
@@ -102,7 +129,8 @@ static int loongson_i2s_plat_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id loongson_i2s_ids[] = {
-	{ .compatible = "loongson,ls2k1000-i2s" },
+	{ .compatible = "loongson,ls2k0300-i2s", .data = &ls2k0300_i2s_plat_config },
+	{ .compatible = "loongson,ls2k1000-i2s", .data = &ls2k1000_i2s_plat_config },
 	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, loongson_i2s_ids);
diff --git a/sound/soc/mediatek/Kconfig b/sound/soc/mediatek/Kconfig
index 4af7bbb58010d..224746e7664fc 100644
--- a/sound/soc/mediatek/Kconfig
+++ b/sound/soc/mediatek/Kconfig
@@ -7,7 +7,7 @@ config SND_SOC_MEDIATEK
 
 config SND_SOC_MT2701
 	tristate "ASoC support for Mediatek MT2701 chip"
-	depends on ARCH_MEDIATEK
+	depends on ARCH_MEDIATEK || COMPILE_TEST
 	select SND_SOC_MEDIATEK
 	help
 	  This adds ASoC driver for Mediatek MT2701 boards
diff --git a/sound/soc/mediatek/mt8173/mt8173-rt5650.c b/sound/soc/mediatek/mt8173/mt8173-rt5650.c
index 3d6d7bc05b872..8c5096482520a 100644
--- a/sound/soc/mediatek/mt8173/mt8173-rt5650.c
+++ b/sound/soc/mediatek/mt8173/mt8173-rt5650.c
@@ -315,7 +315,7 @@ static int mt8173_rt5650_dev_probe(struct platform_device *pdev)
 					       &mt8173_rt5650_priv.pll_from);
 		if (ret) {
 			dev_err(&pdev->dev,
-				"%s snd_soc_register_card fail %d\n",
+				"%s device_property_read_u32() fail %d\n",
 				__func__, ret);
 		}
 	}
diff --git a/sound/soc/mediatek/mt8365/mt8365-mt6357.c b/sound/soc/mediatek/mt8365/mt8365-mt6357.c
index 90448df6c0b22..4339f45ee115a 100644
--- a/sound/soc/mediatek/mt8365/mt8365-mt6357.c
+++ b/sound/soc/mediatek/mt8365/mt8365-mt6357.c
@@ -70,7 +70,8 @@ static const struct snd_soc_dapm_route mt8365_mt6357_routes[] = {
 static int mt8365_mt6357_int_adda_startup(struct snd_pcm_substream *substream)
 {
 	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct mt8365_mt6357_priv *priv = snd_soc_card_get_drvdata(rtd->card);
+	struct mtk_soc_card_data *soc_card_data = snd_soc_card_get_drvdata(rtd->card);
+	struct mt8365_mt6357_priv *priv = soc_card_data->mach_priv;
 	int ret = 0;
 
 	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
@@ -101,7 +102,8 @@ static int mt8365_mt6357_int_adda_startup(struct snd_pcm_substream *substream)
 static void mt8365_mt6357_int_adda_shutdown(struct snd_pcm_substream *substream)
 {
 	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct mt8365_mt6357_priv *priv = snd_soc_card_get_drvdata(rtd->card);
+	struct mtk_soc_card_data *soc_card_data = snd_soc_card_get_drvdata(rtd->card);
+	struct mt8365_mt6357_priv *priv = soc_card_data->mach_priv;
 	int ret = 0;
 
 	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
@@ -245,7 +247,8 @@ static struct snd_soc_dai_link mt8365_mt6357_dais[] = {
 
 static int mt8365_mt6357_gpio_probe(struct snd_soc_card *card)
 {
-	struct mt8365_mt6357_priv *priv = snd_soc_card_get_drvdata(card);
+	struct mtk_soc_card_data *soc_card_data = snd_soc_card_get_drvdata(card);
+	struct mt8365_mt6357_priv *priv = soc_card_data->mach_priv;
 	struct device *dev = card->dev;
 	int ret, i;
 
diff --git a/sound/soc/meson/Makefile b/sound/soc/meson/Makefile
index 24078e4396b02..f9ec0ebb01f04 100644
--- a/sound/soc/meson/Makefile
+++ b/sound/soc/meson/Makefile
@@ -4,6 +4,8 @@ snd-soc-meson-aiu-y := aiu.o
 snd-soc-meson-aiu-y += aiu-acodec-ctrl.o
 snd-soc-meson-aiu-y += aiu-codec-ctrl.o
 snd-soc-meson-aiu-y += aiu-encoder-i2s.o
+snd-soc-meson-aiu-y += gx-formatter.o
+snd-soc-meson-aiu-y += aiu-formatter-i2s.o
 snd-soc-meson-aiu-y += aiu-encoder-spdif.o
 snd-soc-meson-aiu-y += aiu-fifo.o
 snd-soc-meson-aiu-y += aiu-fifo-i2s.o
diff --git a/sound/soc/meson/aiu-encoder-i2s.c b/sound/soc/meson/aiu-encoder-i2s.c
index 3b4061508c180..83b579e98f1c7 100644
--- a/sound/soc/meson/aiu-encoder-i2s.c
+++ b/sound/soc/meson/aiu-encoder-i2s.c
@@ -10,14 +10,11 @@
 #include <sound/soc-dai.h>
 
 #include "aiu.h"
+#include "gx-formatter.h"
+#include "gx-interface.h"
 
-#define AIU_I2S_SOURCE_DESC_MODE_8CH	BIT(0)
-#define AIU_I2S_SOURCE_DESC_MODE_24BIT	BIT(5)
-#define AIU_I2S_SOURCE_DESC_MODE_32BIT	BIT(9)
 #define AIU_I2S_SOURCE_DESC_MODE_SPLIT	BIT(11)
-#define AIU_RST_SOFT_I2S_FAST		BIT(0)
 
-#define AIU_I2S_DAC_CFG_MSB_FIRST	BIT(2)
 #define AIU_CLK_CTRL_I2S_DIV_EN		BIT(0)
 #define AIU_CLK_CTRL_I2S_DIV		GENMASK(3, 2)
 #define AIU_CLK_CTRL_AOCLK_INVERT	BIT(6)
@@ -35,49 +32,6 @@ static void aiu_encoder_i2s_divider_enable(struct snd_soc_component *component,
 				      enable ? AIU_CLK_CTRL_I2S_DIV_EN : 0);
 }
 
-static int aiu_encoder_i2s_setup_desc(struct snd_soc_component *component,
-				      struct snd_pcm_hw_params *params)
-{
-	/* Always operate in split (classic interleaved) mode */
-	unsigned int desc = AIU_I2S_SOURCE_DESC_MODE_SPLIT;
-
-	/* Reset required to update the pipeline */
-	snd_soc_component_write(component, AIU_RST_SOFT, AIU_RST_SOFT_I2S_FAST);
-	snd_soc_component_read(component, AIU_I2S_SYNC);
-
-	switch (params_physical_width(params)) {
-	case 16: /* Nothing to do */
-		break;
-
-	case 32:
-		desc |= (AIU_I2S_SOURCE_DESC_MODE_24BIT |
-			 AIU_I2S_SOURCE_DESC_MODE_32BIT);
-		break;
-
-	default:
-		return -EINVAL;
-	}
-
-	switch (params_channels(params)) {
-	case 2: /* Nothing to do */
-		break;
-	case 8:
-		desc |= AIU_I2S_SOURCE_DESC_MODE_8CH;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	snd_soc_component_update_bits(component, AIU_I2S_SOURCE_DESC,
-				      AIU_I2S_SOURCE_DESC_MODE_8CH |
-				      AIU_I2S_SOURCE_DESC_MODE_24BIT |
-				      AIU_I2S_SOURCE_DESC_MODE_32BIT |
-				      AIU_I2S_SOURCE_DESC_MODE_SPLIT,
-				      desc);
-
-	return 0;
-}
-
 static int aiu_encoder_i2s_set_legacy_div(struct snd_soc_component *component,
 					  struct snd_pcm_hw_params *params,
 					  unsigned int bs)
@@ -112,6 +66,9 @@ static int aiu_encoder_i2s_set_more_div(struct snd_soc_component *component,
 					struct snd_pcm_hw_params *params,
 					unsigned int bs)
 {
+	struct aiu *aiu = snd_soc_component_get_drvdata(component);
+	struct gx_iface *iface = &aiu->i2s.iface;
+
 	/*
 	 * NOTE: this HW is odd.
 	 * In most configuration, the i2s divider is 'mclk / blck'.
@@ -126,6 +83,18 @@ static int aiu_encoder_i2s_set_more_div(struct snd_soc_component *component,
 			return -EINVAL;
 		}
 		bs += bs / 2;
+		iface->bs_quirk = true;
+	} else {
+		/*
+		 * If the bs quirk is currently applied for one stream and another
+		 * ones tries to setup a configuration for which the quirk is
+		 * not required, then fail.
+		 */
+		if (iface->bs_quirk) {
+			dev_err(component->dev,
+				"bclk requirements are incompatible with active stream\n");
+			return -EINVAL;
+		}
 	}
 
 	/* Use CLK_MORE for mclk to bclk divider */
@@ -145,21 +114,17 @@ static int aiu_encoder_i2s_set_clocks(struct snd_soc_component *component,
 				      struct snd_pcm_hw_params *params)
 {
 	struct aiu *aiu = snd_soc_component_get_drvdata(component);
+	struct gx_iface *iface = &aiu->i2s.iface;
 	unsigned int srate = params_rate(params);
 	unsigned int fs, bs;
 	int ret;
 
 	/* Get the oversampling factor */
-	fs = DIV_ROUND_CLOSEST(clk_get_rate(aiu->i2s.clks[MCLK].clk), srate);
+	fs = DIV_ROUND_CLOSEST(iface->mclk_rate, srate);
 
-	if (fs % 64)
+	if ((fs % 64) || (fs == 0))
 		return -EINVAL;
 
-	/* Send data MSB first */
-	snd_soc_component_update_bits(component, AIU_I2S_DAC_CFG,
-				      AIU_I2S_DAC_CFG_MSB_FIRST,
-				      AIU_I2S_DAC_CFG_MSB_FIRST);
-
 	/* Set bclk to lrlck ratio */
 	snd_soc_component_update_bits(component, AIU_CODEC_DAC_LRCLK_CTRL,
 				      AIU_CODEC_DAC_LRCLK_CTRL_DIV,
@@ -188,24 +153,53 @@ static int aiu_encoder_i2s_hw_params(struct snd_pcm_substream *substream,
 				     struct snd_pcm_hw_params *params,
 				     struct snd_soc_dai *dai)
 {
+	struct gx_stream *ts = snd_soc_dai_get_dma_data(dai, substream);
+	struct gx_iface *iface = ts->iface;
 	struct snd_soc_component *component = dai->component;
 	int ret;
 
-	/* Disable the clock while changing the settings */
-	aiu_encoder_i2s_divider_enable(component, false);
-
-	ret = aiu_encoder_i2s_setup_desc(component, params);
-	if (ret) {
-		dev_err(dai->dev, "setting i2s desc failed\n");
-		return ret;
+	/*
+	 * Enforce interface wide rate symmetry only if there is more than
+	 * 1 stream active.
+	 */
+	if (snd_soc_dai_active(dai) > 1) {
+		if (iface->rate && iface->rate != params_rate(params)) {
+			dev_err(dai->dev, "can't set iface rate (%d != %d)\n",
+				iface->rate, params_rate(params));
+			return -EINVAL;
+		}
 	}
 
 	ret = aiu_encoder_i2s_set_clocks(component, params);
 	if (ret) {
-		dev_err(dai->dev, "setting i2s clocks failed\n");
+		dev_err(dai->dev, "setting i2s clocks failed: %d\n", ret);
 		return ret;
 	}
 
+	iface->rate = params_rate(params);
+	ts->physical_width = params_physical_width(params);
+	ts->width = params_width(params);
+	ts->channels = params_channels(params);
+
+	return 0;
+}
+
+static int aiu_encoder_i2s_prepare(struct snd_pcm_substream *substream,
+				   struct snd_soc_dai *dai)
+{
+	struct gx_stream *ts = snd_soc_dai_get_dma_data(dai, substream);
+	struct snd_soc_component *component = dai->component;
+	int ret;
+
+	if (ts->clk_enabled)
+		return 0;
+
+	ret = clk_prepare_enable(ts->iface->mclk);
+	if (ret)
+		return ret;
+
+	ts->clk_enabled = true;
+
 	aiu_encoder_i2s_divider_enable(component, true);
 
 	return 0;
@@ -214,9 +208,24 @@ static int aiu_encoder_i2s_hw_params(struct snd_pcm_substream *substream,
 static int aiu_encoder_i2s_hw_free(struct snd_pcm_substream *substream,
 				   struct snd_soc_dai *dai)
 {
+	struct gx_stream *ts = snd_soc_dai_get_dma_data(dai, substream);
+	struct gx_iface *iface = ts->iface;
 	struct snd_soc_component *component = dai->component;
 
-	aiu_encoder_i2s_divider_enable(component, false);
+	/*
+	 * If this is the last substream being closed then disable the i2s
+	 * clock divider and clear 'iface->rate'.
+	 */
+	if (snd_soc_dai_active(dai) <= 1) {
+		aiu_encoder_i2s_divider_enable(component, 0);
+		iface->rate = 0;
+		iface->bs_quirk = false;
+	}
+
+	if (ts->clk_enabled) {
+		clk_disable_unprepare(ts->iface->mclk);
+		ts->clk_enabled = false;
+	}
 
 	return 0;
 }
@@ -224,6 +233,8 @@ static int aiu_encoder_i2s_hw_free(struct snd_pcm_substream *substream,
 static int aiu_encoder_i2s_set_fmt(struct snd_soc_dai *dai, unsigned int fmt)
 {
 	struct snd_soc_component *component = dai->component;
+	struct aiu *aiu = snd_soc_component_get_drvdata(component);
+	struct gx_iface *iface = &aiu->i2s.iface;
 	unsigned int inv = fmt & SND_SOC_DAIFMT_INV_MASK;
 	unsigned int val = 0;
 	unsigned int skew;
@@ -255,9 +266,12 @@ static int aiu_encoder_i2s_set_fmt(struct snd_soc_dai *dai, unsigned int fmt)
 		skew = 0;
 		break;
 	default:
+		dev_err(dai->dev, "unsupported dai format\n");
 		return -EINVAL;
 	}
 
+	iface->fmt = fmt;
+
 	val |= FIELD_PREP(AIU_CLK_CTRL_LRCLK_SKEW, skew);
 	snd_soc_component_update_bits(component, AIU_CLK_CTRL,
 				      AIU_CLK_CTRL_LRCLK_INVERT |
@@ -272,6 +286,7 @@ static int aiu_encoder_i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id,
 				      unsigned int freq, int dir)
 {
 	struct aiu *aiu = snd_soc_component_get_drvdata(dai->component);
+	struct gx_iface *iface = &aiu->i2s.iface;
 	int ret;
 
 	if (WARN_ON(clk_id != 0))
@@ -280,11 +295,15 @@ static int aiu_encoder_i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id,
 	if (dir == SND_SOC_CLOCK_IN)
 		return 0;
 
-	ret = clk_set_rate(aiu->i2s.clks[MCLK].clk, freq);
-	if (ret)
-		dev_err(dai->dev, "Failed to set sysclk to %uHz", freq);
+	ret = clk_set_rate(iface->mclk, freq);
+	if (ret) {
+		dev_err(dai->dev, "Failed to set sysclk to %uHz: %d", freq, ret);
+		return ret;
+	}
 
-	return ret;
+	iface->mclk_rate = freq;
+
+	return 0;
 }
 
 static const unsigned int hw_channels[] = {2, 8};
@@ -305,15 +324,54 @@ static int aiu_encoder_i2s_startup(struct snd_pcm_substream *substream,
 					 SNDRV_PCM_HW_PARAM_CHANNELS,
 					 &hw_channel_constraints);
 	if (ret) {
-		dev_err(dai->dev, "adding channels constraints failed\n");
+		dev_err(dai->dev, "adding channels constraints failed: %d\n", ret);
 		return ret;
 	}
 
-	ret = clk_bulk_prepare_enable(aiu->i2s.clk_num, aiu->i2s.clks);
-	if (ret)
-		dev_err(dai->dev, "failed to enable i2s clocks\n");
+	/*
+	 * Enable only clocks which are required for the interface internal
+	 * logic. MCLK is enabled/disabled from the formatter and the I2S
+	 * divider is enabled/disabled in "hw_params"/"hw_free", respectively.
+	 */
+	ret = clk_prepare_enable(aiu->i2s.clks[PCLK].clk);
+	if (ret) {
+		dev_err(dai->dev, "failed to enable PCLK: %d\n", ret);
+		return ret;
+	}
+	ret = clk_prepare_enable(aiu->i2s.clks[MIXER].clk);
+	if (ret) {
+		dev_err(dai->dev, "failed to enable MIXER: %d\n", ret);
+		clk_disable_unprepare(aiu->i2s.clks[PCLK].clk);
+		return ret;
+	}
+	ret = clk_prepare_enable(aiu->i2s.clks[AOCLK].clk);
+	if (ret) {
+		dev_err(dai->dev, "failed to enable AOCLK: %d\n", ret);
+		clk_disable_unprepare(aiu->i2s.clks[MIXER].clk);
+		clk_disable_unprepare(aiu->i2s.clks[PCLK].clk);
+		return ret;
+	}
 
-	return ret;
+	/*
+	 * We're always operating in split mode for the playback stream.
+	 *
+	 * This setting arguably belong to the 'aiu-formatter', but it's kept
+	 * here for backward compatibility reason. At reset the I2S encoder
+	 * operates in normal mode which would only support 8ch, but by default
+	 * only 2ch are enabled. If a playback stream is started without
+	 * changing to split mode, then the I2S encoder doesn't consume audio
+	 * samples and the playback fails.
+	 * Moving this to 'aiu-formatter' would cause the split mode to be set
+	 * only when the formatter is enabled, which doesn't happen at boot as
+	 * the default value for "HDMI CTRL SRC" is "DISABLED".
+	 */
+	ret = snd_soc_component_update_bits(dai->component, AIU_I2S_SOURCE_DESC,
+					    AIU_I2S_SOURCE_DESC_MODE_SPLIT,
+					    AIU_I2S_SOURCE_DESC_MODE_SPLIT);
+	if (ret < 0)
+		dev_err(dai->dev, "failed to update AIU_I2S_SOURCE_DESC: %d", ret);
+
+	return 0;
 }
 
 static void aiu_encoder_i2s_shutdown(struct snd_pcm_substream *substream,
@@ -321,14 +379,89 @@ static void aiu_encoder_i2s_shutdown(struct snd_pcm_substream *substream,
 {
 	struct aiu *aiu = snd_soc_component_get_drvdata(dai->component);
 
-	clk_bulk_disable_unprepare(aiu->i2s.clk_num, aiu->i2s.clks);
+	clk_disable_unprepare(aiu->i2s.clks[AOCLK].clk);
+	clk_disable_unprepare(aiu->i2s.clks[MIXER].clk);
+	clk_disable_unprepare(aiu->i2s.clks[PCLK].clk);
+}
+
+static int aiu_encoder_i2s_trigger(struct snd_pcm_substream *substream,
+				   int cmd,
+				   struct snd_soc_dai *dai)
+{
+	struct gx_stream *ts = snd_soc_dai_get_dma_data(dai, substream);
+	int ret;
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_RESUME:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		ret = gx_stream_start(ts);
+		break;
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+	case SNDRV_PCM_TRIGGER_STOP:
+		gx_stream_stop(ts);
+		ret = 0;
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static int aiu_encoder_i2s_remove_dai(struct snd_soc_dai *dai)
+{
+	int stream;
+
+	for_each_pcm_streams(stream) {
+		struct gx_stream *ts;
+
+		ts = snd_soc_dai_dma_data_get(dai, stream);
+		if (ts)
+			gx_stream_free(ts);
+
+		snd_soc_dai_dma_data_set(dai, stream, NULL);
+	}
+
+	return 0;
+}
+
+static int aiu_encoder_i2s_probe_dai(struct snd_soc_dai *dai)
+{
+	struct aiu *aiu = snd_soc_dai_get_drvdata(dai);
+	struct gx_iface *iface = &aiu->i2s.iface;
+	int stream;
+
+	for_each_pcm_streams(stream) {
+		struct gx_stream *ts;
+
+		if (!snd_soc_dai_get_widget(dai, stream))
+			continue;
+
+		ts = gx_stream_alloc(iface);
+		if (!ts) {
+			aiu_encoder_i2s_remove_dai(dai);
+			return -ENOMEM;
+		}
+		snd_soc_dai_dma_data_set(dai, stream, ts);
+	}
+
+	iface->mclk = aiu->i2s.clks[MCLK].clk;
+	iface->mclk_rate = clk_get_rate(iface->mclk);
+
+	return 0;
 }
 
 const struct snd_soc_dai_ops aiu_encoder_i2s_dai_ops = {
+	.probe		= aiu_encoder_i2s_probe_dai,
+	.remove		= aiu_encoder_i2s_remove_dai,
 	.hw_params	= aiu_encoder_i2s_hw_params,
+	.prepare	= aiu_encoder_i2s_prepare,
 	.hw_free	= aiu_encoder_i2s_hw_free,
 	.set_fmt	= aiu_encoder_i2s_set_fmt,
 	.set_sysclk	= aiu_encoder_i2s_set_sysclk,
 	.startup	= aiu_encoder_i2s_startup,
 	.shutdown	= aiu_encoder_i2s_shutdown,
+	.trigger	= aiu_encoder_i2s_trigger,
 };
diff --git a/sound/soc/meson/aiu-fifo-spdif.c b/sound/soc/meson/aiu-fifo-spdif.c
index e0e00ec026dcc..a9861c5d66374 100644
--- a/sound/soc/meson/aiu-fifo-spdif.c
+++ b/sound/soc/meson/aiu-fifo-spdif.c
@@ -24,6 +24,7 @@
 #define AIU_MEM_IEC958_CONTROL_MODE_16BIT	BIT(7)
 #define AIU_MEM_IEC958_CONTROL_MODE_LINEAR	BIT(8)
 #define AIU_MEM_IEC958_BUF_CNTL_INIT		BIT(0)
+#define AIU_RST_SOFT_958_FAST			BIT(2)
 
 #define AIU_FIFO_SPDIF_BLOCK			8
 
@@ -68,11 +69,15 @@ static int fifo_spdif_trigger(struct snd_pcm_substream *substream, int cmd,
 	case SNDRV_PCM_TRIGGER_START:
 	case SNDRV_PCM_TRIGGER_RESUME:
 	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		snd_soc_component_write(component, AIU_RST_SOFT,
+					AIU_RST_SOFT_958_FAST);
 		fifo_spdif_dcu_enable(component, true);
 		break;
 	case SNDRV_PCM_TRIGGER_SUSPEND:
 	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
 	case SNDRV_PCM_TRIGGER_STOP:
+		snd_soc_component_write(component, AIU_RST_SOFT,
+					AIU_RST_SOFT_958_FAST);
 		fifo_spdif_dcu_enable(component, false);
 		break;
 	default:
diff --git a/sound/soc/meson/aiu-formatter-i2s.c b/sound/soc/meson/aiu-formatter-i2s.c
new file mode 100644
index 0000000000000..cb554c2e7ce4a
--- /dev/null
+++ b/sound/soc/meson/aiu-formatter-i2s.c
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2026 BayLibre, SAS.
+// Author: Valerio Setti <vsetti@baylibre.com>
+
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+#include <sound/soc-dai.h>
+
+#include "aiu.h"
+#include "gx-formatter.h"
+
+#define AIU_I2S_SOURCE_DESC_MODE_8CH	BIT(0)
+#define AIU_I2S_SOURCE_DESC_MODE_24BIT	BIT(5)
+#define AIU_I2S_SOURCE_DESC_MODE_32BIT	BIT(9)
+
+#define AIU_I2S_DAC_CFG_MSB_FIRST	BIT(2)
+
+static struct snd_soc_dai *
+aiu_formatter_i2s_get_be(struct snd_soc_dapm_widget *w)
+{
+	struct snd_soc_dapm_path *p;
+	struct snd_soc_dai *be;
+
+	snd_soc_dapm_widget_for_each_sink_path(w, p) {
+		if (!p->connect)
+			continue;
+
+		if (p->sink->id == snd_soc_dapm_dai_in)
+			return (struct snd_soc_dai *)p->sink->priv;
+
+		be = aiu_formatter_i2s_get_be(p->sink);
+		if (be)
+			return be;
+	}
+
+	return NULL;
+}
+
+static struct gx_stream *
+aiu_formatter_i2s_get_stream(struct snd_soc_dapm_widget *w)
+{
+	struct snd_soc_dai *be = aiu_formatter_i2s_get_be(w);
+
+	if (!be)
+		return NULL;
+
+	return snd_soc_dai_dma_data_get_playback(be);
+}
+
+static int aiu_formatter_i2s_prepare(struct regmap *map,
+				 const struct gx_formatter_hw *quirks,
+				 struct gx_stream *ts)
+{
+	/* Always operate in split (classic interleaved) mode */
+	unsigned int desc = 0;
+
+	/*
+	 * Pipeline reset is already implemented in aiu_fifo_i2s_trigger() at
+	 * trigger time.
+	 */
+
+	switch (ts->physical_width) {
+	case 16: /* Nothing to do */
+		break;
+
+	case 32:
+		desc |= (AIU_I2S_SOURCE_DESC_MODE_24BIT |
+			 AIU_I2S_SOURCE_DESC_MODE_32BIT);
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	switch (ts->channels) {
+	case 2: /* Nothing to do */
+		break;
+	case 8:
+		desc |= AIU_I2S_SOURCE_DESC_MODE_8CH;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	regmap_update_bits(map, AIU_I2S_SOURCE_DESC,
+				AIU_I2S_SOURCE_DESC_MODE_8CH |
+				AIU_I2S_SOURCE_DESC_MODE_24BIT |
+				AIU_I2S_SOURCE_DESC_MODE_32BIT,
+				desc);
+
+	/* Send data MSB first */
+	regmap_update_bits(map, AIU_I2S_DAC_CFG,
+				AIU_I2S_DAC_CFG_MSB_FIRST,
+				AIU_I2S_DAC_CFG_MSB_FIRST);
+
+	return 0;
+}
+
+const struct gx_formatter_ops aiu_formatter_i2s_ops = {
+	.get_stream	= aiu_formatter_i2s_get_stream,
+	.prepare	= aiu_formatter_i2s_prepare,
+};
diff --git a/sound/soc/meson/aiu.c b/sound/soc/meson/aiu.c
index f2890111c1d2c..64ace4d25d92c 100644
--- a/sound/soc/meson/aiu.c
+++ b/sound/soc/meson/aiu.c
@@ -29,13 +29,22 @@ static SOC_ENUM_SINGLE_DECL(aiu_spdif_encode_sel_enum, AIU_I2S_MISC,
 static const struct snd_kcontrol_new aiu_spdif_encode_mux =
 	SOC_DAPM_ENUM("SPDIF Buffer Src", aiu_spdif_encode_sel_enum);
 
-static const struct snd_soc_dapm_widget aiu_cpu_dapm_widgets[] = {
-	SND_SOC_DAPM_MUX("SPDIF SRC SEL", SND_SOC_NOPM, 0, 0,
-			 &aiu_spdif_encode_mux),
+#define AIU_WIDGET_SPDIF_SRC_SEL	0
+#define AIU_WIDGET_I2S_FORMATTER	1
+
+static struct snd_soc_dapm_widget aiu_cpu_dapm_widgets[] = {
+	[AIU_WIDGET_SPDIF_SRC_SEL] =
+		SND_SOC_DAPM_MUX("SPDIF SRC SEL", SND_SOC_NOPM, 0, 0,
+				 &aiu_spdif_encode_mux),
+	[AIU_WIDGET_I2S_FORMATTER] =
+		SND_SOC_DAPM_PGA_E("I2S Formatter", SND_SOC_NOPM, 0, 0, NULL, 0,
+				   gx_formatter_event,
+				   (SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_PRE_PMD)),
 };
 
 static const struct snd_soc_dapm_route aiu_cpu_dapm_routes[] = {
-	{ "I2S Encoder Playback", NULL, "I2S FIFO Playback" },
+	{ "I2S Formatter", NULL, "I2S FIFO Playback" },
+	{ "I2S Encoder Playback", NULL, "I2S Formatter" },
 	{ "SPDIF SRC SEL", "SPDIF", "SPDIF FIFO Playback" },
 	{ "SPDIF SRC SEL", "I2S", "I2S FIFO Playback" },
 	{ "SPDIF Encoder Playback", NULL, "SPDIF SRC SEL" },
@@ -172,6 +181,11 @@ static const struct regmap_config aiu_regmap_cfg = {
 	.max_register	= 0x2ac,
 };
 
+const struct gx_formatter_driver aiu_formatter_i2s_drv = {
+	.regmap_cfg	= &aiu_regmap_cfg,
+	.ops		= &aiu_formatter_i2s_ops,
+};
+
 static int aiu_clk_bulk_get(struct device *dev,
 			    const char * const *ids,
 			    unsigned int num,
@@ -282,6 +296,14 @@ static int aiu_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	/* Allocate the aiu-formatter into its widget */
+	ret = gx_formatter_create(dev, &aiu_cpu_dapm_widgets[AIU_WIDGET_I2S_FORMATTER],
+				  &aiu_formatter_i2s_drv, map);
+	if (ret) {
+		dev_err(dev, "Failed to allocate aiu formatter\n");
+		goto err;
+	}
+
 	/* Register the cpu component of the aiu */
 	ret = snd_soc_register_component(dev, &aiu_cpu_component,
 					 aiu_cpu_dai_drv,
@@ -310,12 +332,14 @@ static int aiu_probe(struct platform_device *pdev)
 
 	return 0;
 err:
+	gx_formatter_free(&aiu_cpu_dapm_widgets[AIU_WIDGET_I2S_FORMATTER]);
 	snd_soc_unregister_component(dev);
 	return ret;
 }
 
 static void aiu_remove(struct platform_device *pdev)
 {
+	gx_formatter_free(&aiu_cpu_dapm_widgets[AIU_WIDGET_I2S_FORMATTER]);
 	snd_soc_unregister_component(&pdev->dev);
 }
 
diff --git a/sound/soc/meson/aiu.h b/sound/soc/meson/aiu.h
index 0f94c8bf60818..7d0b98c1f351b 100644
--- a/sound/soc/meson/aiu.h
+++ b/sound/soc/meson/aiu.h
@@ -7,6 +7,8 @@
 #ifndef _MESON_AIU_H
 #define _MESON_AIU_H
 
+#include "gx-formatter.h"
+
 struct clk;
 struct clk_bulk_data;
 struct device;
@@ -25,6 +27,7 @@ struct aiu_interface {
 	struct clk_bulk_data *clks;
 	unsigned int clk_num;
 	int irq;
+	struct gx_iface iface;
 };
 
 struct aiu_platform_data {
@@ -58,6 +61,7 @@ extern const struct snd_soc_dai_ops aiu_fifo_i2s_dai_ops;
 extern const struct snd_soc_dai_ops aiu_fifo_spdif_dai_ops;
 extern const struct snd_soc_dai_ops aiu_encoder_i2s_dai_ops;
 extern const struct snd_soc_dai_ops aiu_encoder_spdif_dai_ops;
+extern const struct gx_formatter_ops aiu_formatter_i2s_ops;
 
 #define AIU_IEC958_BPF			0x000
 #define AIU_958_MISC			0x010
diff --git a/sound/soc/meson/g12a-toacodec.c b/sound/soc/meson/g12a-toacodec.c
index a95375b53f0a3..21941ee552c5c 100644
--- a/sound/soc/meson/g12a-toacodec.c
+++ b/sound/soc/meson/g12a-toacodec.c
@@ -312,7 +312,7 @@ static int g12a_toacodec_probe(struct platform_device *pdev)
 
 	ret = device_reset(dev);
 	if (ret)
-		return ret;
+		return dev_err_probe(dev, ret, "failed to reset device\n");
 
 	regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(regs))
diff --git a/sound/soc/meson/g12a-tohdmitx.c b/sound/soc/meson/g12a-tohdmitx.c
index d541ca4acfaf3..967109ca2b57c 100644
--- a/sound/soc/meson/g12a-tohdmitx.c
+++ b/sound/soc/meson/g12a-tohdmitx.c
@@ -251,7 +251,7 @@ static int g12a_tohdmitx_probe(struct platform_device *pdev)
 
 	ret = device_reset(dev);
 	if (ret)
-		return ret;
+		return dev_err_probe(dev, ret, "failed to reset device\n");
 
 	regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(regs))
diff --git a/sound/soc/meson/gx-formatter.c b/sound/soc/meson/gx-formatter.c
new file mode 100644
index 0000000000000..311e63affb239
--- /dev/null
+++ b/sound/soc/meson/gx-formatter.c
@@ -0,0 +1,282 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+//
+// Copyright (c) 2026 BayLibre, SAS.
+// Author: Valerio Setti <vsetti@baylibre.com>
+
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/regmap.h>
+#include <sound/soc.h>
+
+#include "gx-formatter.h"
+
+struct gx_formatter {
+	struct list_head list;
+	struct gx_stream *stream;
+	const struct gx_formatter_driver *drv;
+	bool enabled;
+	struct regmap *map;
+};
+
+static int gx_formatter_enable(struct gx_formatter *formatter)
+{
+	int ret;
+
+	/* Do nothing if the formatter is already enabled */
+	if (formatter->enabled)
+		return 0;
+
+	/* Setup the stream parameter in the formatter */
+	if (formatter->drv->ops->prepare) {
+		ret = formatter->drv->ops->prepare(formatter->map,
+					   formatter->drv->quirks,
+					   formatter->stream);
+		if (ret)
+			return ret;
+	}
+
+	/* Finally, actually enable the formatter */
+	if (formatter->drv->ops->enable)
+		formatter->drv->ops->enable(formatter->map);
+
+	formatter->enabled = true;
+
+	return 0;
+}
+
+static void gx_formatter_disable(struct gx_formatter *formatter)
+{
+	/* Do nothing if the formatter is already disabled */
+	if (!formatter->enabled)
+		return;
+
+	if (formatter->drv->ops->disable)
+		formatter->drv->ops->disable(formatter->map);
+
+	formatter->enabled = false;
+}
+
+static int gx_formatter_attach(struct gx_formatter *formatter)
+{
+	struct gx_stream *ts = formatter->stream;
+	int ret = 0;
+
+	mutex_lock(&ts->lock);
+
+	/* Catch up if the stream is already running when we attach */
+	if (ts->ready) {
+		ret = gx_formatter_enable(formatter);
+		if (ret) {
+			pr_err("failed to enable formatter\n");
+			goto out;
+		}
+	}
+
+	list_add_tail(&formatter->list, &ts->formatter_list);
+out:
+	mutex_unlock(&ts->lock);
+	return ret;
+}
+
+static void gx_formatter_detach(struct gx_formatter *formatter)
+{
+	struct gx_stream *ts = formatter->stream;
+
+	if (!ts)
+		return;
+
+	mutex_lock(&ts->lock);
+	list_del(&formatter->list);
+	mutex_unlock(&ts->lock);
+
+	gx_formatter_disable(formatter);
+}
+
+static int gx_formatter_power_up(struct gx_formatter *formatter,
+				      struct snd_soc_dapm_widget *w)
+{
+	struct gx_stream *ts = formatter->drv->ops->get_stream(w);
+	int ret;
+
+	/*
+	 * If we don't get a stream at this stage, it would mean that the
+	 * widget is powering up but is not attached to any backend DAI.
+	 * It should not happen, ever !
+	 */
+	if (WARN_ON(!ts))
+		return -ENODEV;
+
+	formatter->stream = ts;
+	INIT_LIST_HEAD(&formatter->list);
+	ret = gx_formatter_attach(formatter);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void gx_formatter_power_down(struct gx_formatter *formatter)
+{
+	gx_formatter_detach(formatter);
+	formatter->stream = NULL;
+}
+
+int gx_formatter_event(struct snd_soc_dapm_widget *w,
+		       struct snd_kcontrol *control,
+		       int event)
+{
+	struct snd_soc_component *c;
+	struct gx_formatter *formatter;
+	int ret = 0;
+
+	c = snd_soc_dapm_to_component(w->dapm);
+
+	if (w->priv)
+		formatter = w->priv;
+	else
+		formatter = snd_soc_component_get_drvdata(c);
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		ret = gx_formatter_power_up(formatter, w);
+		break;
+
+	case SND_SOC_DAPM_PRE_PMD:
+		gx_formatter_power_down(formatter);
+		break;
+
+	default:
+		dev_err(c->dev, "Unexpected event %d\n", event);
+		return -EINVAL;
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gx_formatter_event);
+
+int gx_formatter_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	const struct gx_formatter_driver *drv;
+	struct gx_formatter *formatter;
+	void __iomem *regs;
+
+	drv = of_device_get_match_data(dev);
+	if (!drv) {
+		dev_err(dev, "failed to match device\n");
+		return -ENODEV;
+	}
+
+	formatter = devm_kzalloc(dev, sizeof(*formatter), GFP_KERNEL);
+	if (!formatter)
+		return -ENOMEM;
+	platform_set_drvdata(pdev, formatter);
+	formatter->drv = drv;
+
+	regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
+
+	formatter->map = devm_regmap_init_mmio(dev, regs, drv->regmap_cfg);
+	if (IS_ERR(formatter->map)) {
+		dev_err(dev, "failed to init regmap: %ld\n",
+			PTR_ERR(formatter->map));
+		return PTR_ERR(formatter->map);
+	}
+
+	return devm_snd_soc_register_component(dev, drv->component_drv,
+					       NULL, 0);
+}
+EXPORT_SYMBOL_GPL(gx_formatter_probe);
+
+int gx_formatter_create(struct device *dev,
+			struct snd_soc_dapm_widget *w,
+			const struct gx_formatter_driver *drv,
+			struct regmap *regmap)
+{
+	struct gx_formatter *formatter;
+
+	formatter = devm_kzalloc(dev, sizeof(*formatter), GFP_KERNEL);
+	if (!formatter)
+		return -ENOMEM;
+
+	formatter->drv = drv;
+	formatter->map = regmap;
+
+	w->priv = formatter;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(gx_formatter_create);
+
+int gx_stream_start(struct gx_stream *ts)
+{
+	struct gx_formatter *formatter;
+	int ret = 0;
+
+	mutex_lock(&ts->lock);
+
+	/* Start all the formatters attached to the stream */
+	list_for_each_entry(formatter, &ts->formatter_list, list) {
+		ret = gx_formatter_enable(formatter);
+		if (ret) {
+			pr_err("failed to enable formatter\n");
+			goto out;
+		}
+	}
+
+	ts->ready = true;
+
+out:
+	mutex_unlock(&ts->lock);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gx_stream_start);
+
+void gx_stream_stop(struct gx_stream *ts)
+{
+	struct gx_formatter *formatter;
+
+	mutex_lock(&ts->lock);
+	ts->ready = false;
+
+	/* Stop all the formatters attached to the stream */
+	list_for_each_entry(formatter, &ts->formatter_list, list) {
+		gx_formatter_disable(formatter);
+	}
+
+	mutex_unlock(&ts->lock);
+}
+EXPORT_SYMBOL_GPL(gx_stream_stop);
+
+struct gx_stream *gx_stream_alloc(struct gx_iface *iface)
+{
+	struct gx_stream *ts;
+
+	ts = kzalloc(sizeof(*ts), GFP_KERNEL);
+	if (ts) {
+		INIT_LIST_HEAD(&ts->formatter_list);
+		mutex_init(&ts->lock);
+		ts->iface = iface;
+	}
+
+	return ts;
+}
+EXPORT_SYMBOL_GPL(gx_stream_alloc);
+
+void gx_stream_free(struct gx_stream *ts)
+{
+	/*
+	 * If the list is not empty, it would mean that one of the formatter
+	 * widget is still powered and attached to the interface while we
+	 * are removing the TDM DAI. It should not be possible
+	 */
+	WARN_ON(!list_empty(&ts->formatter_list));
+	mutex_destroy(&ts->lock);
+	kfree(ts);
+}
+EXPORT_SYMBOL_GPL(gx_stream_free);
+
+MODULE_DESCRIPTION("Amlogic GX formatter driver");
+MODULE_AUTHOR("Valerio Setti <vsetti@baylibre.com>");
+MODULE_LICENSE("GPL");
diff --git a/sound/soc/meson/gx-formatter.h b/sound/soc/meson/gx-formatter.h
new file mode 100644
index 0000000000000..b90b1814d79b4
--- /dev/null
+++ b/sound/soc/meson/gx-formatter.h
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
+/*
+ * Copyright (c) 2026 Baylibre SAS.
+ * Author: Valerio Setti <vsetti@baylibre.com>
+ */
+
+#ifndef _MESON_GX_FORMATTER_H
+#define _MESON_GX_FORMATTER_H
+
+#include "gx-interface.h"
+
+struct platform_device;
+struct regmap;
+struct snd_soc_dapm_widget;
+struct snd_kcontrol;
+
+struct gx_formatter_hw {
+	unsigned int skew_offset;
+};
+
+struct gx_formatter_ops {
+	struct gx_stream *(*get_stream)(struct snd_soc_dapm_widget *w);
+	void (*enable)(struct regmap *map);
+	void (*disable)(struct regmap *map);
+	int (*prepare)(struct regmap *map,
+		       const struct gx_formatter_hw *quirks,
+		       struct gx_stream *ts);
+};
+
+struct gx_formatter_driver {
+	const struct snd_soc_component_driver *component_drv;
+	const struct regmap_config *regmap_cfg;
+	const struct gx_formatter_ops *ops;
+	const struct gx_formatter_hw *quirks;
+};
+
+int gx_formatter_event(struct snd_soc_dapm_widget *w,
+		       struct snd_kcontrol *control,
+		       int event);
+int gx_formatter_probe(struct platform_device *pdev);
+
+int gx_formatter_create(struct device *dev,
+			struct snd_soc_dapm_widget *w,
+			const struct gx_formatter_driver *drv,
+			struct regmap *regmap);
+
+/*
+ * Formatter data is already freed when the associated device is removed,
+ * so we just need to remove the pointer from the widget.
+ */
+static inline void gx_formatter_free(struct snd_soc_dapm_widget *w)
+{
+	w->priv = NULL;
+}
+
+#endif /* _MESON_GX_FORMATTER_H */
diff --git a/sound/soc/meson/gx-interface.h b/sound/soc/meson/gx-interface.h
new file mode 100644
index 0000000000000..65c46dcce32a8
--- /dev/null
+++ b/sound/soc/meson/gx-interface.h
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
+/*
+ * Copyright (c) 2026 Baylibre SAS.
+ * Author: Valerio Setti <vsetti@baylibre.com>
+ */
+
+#ifndef _MESON_GX_INTERFACE_H
+#define _MESON_GX_INTERFACE_H
+
+#include <linux/clk.h>
+#include <linux/regmap.h>
+#include <sound/pcm.h>
+#include <sound/soc.h>
+#include <sound/soc-dai.h>
+
+struct gx_iface {
+	struct clk *mclk;
+	unsigned long mclk_rate;
+
+	/* format is common to all the DAIs of the iface */
+	unsigned int fmt;
+
+	/* For component wide symmetry */
+	int rate;
+
+	/* Only for GX platform */
+	int bs_quirk;
+};
+
+struct gx_stream {
+	struct gx_iface *iface;
+	struct list_head formatter_list;
+	struct mutex lock;
+	unsigned int channels;
+	unsigned int width;
+	unsigned int physical_width;
+	bool ready;
+
+	/* For continuous clock tracking */
+	bool clk_enabled;
+};
+
+struct gx_stream *gx_stream_alloc(struct gx_iface *iface);
+void gx_stream_free(struct gx_stream *ts);
+int gx_stream_start(struct gx_stream *ts);
+void gx_stream_stop(struct gx_stream *ts);
+
+#endif /* _MESON_GX_INTERFACE_H */
diff --git a/sound/soc/meson/t9015.c b/sound/soc/meson/t9015.c
index da1a93946d679..68b5323ca9c89 100644
--- a/sound/soc/meson/t9015.c
+++ b/sound/soc/meson/t9015.c
@@ -78,8 +78,14 @@ static int t9015_dai_set_fmt(struct snd_soc_dai *dai, unsigned int fmt)
 	return 0;
 }
 
+static const u64 t9015_dai_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J;
+
 static const struct snd_soc_dai_ops t9015_dai_ops = {
 	.set_fmt = t9015_dai_set_fmt,
+	.auto_selectable_formats = &t9015_dai_selectable_formats,
+	.num_auto_selectable_formats = 1,
 };
 
 static struct snd_soc_dai_driver t9015_dai = {
@@ -265,10 +271,8 @@ static int t9015_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, PTR_ERR(priv->avdd), "failed to AVDD\n");
 
 	ret = device_reset(dev);
-	if (ret) {
-		dev_err(dev, "reset failed\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to reset device\n");
 
 	regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(regs)) {
diff --git a/sound/soc/qcom/sc8280xp.c b/sound/soc/qcom/sc8280xp.c
index 7925aa3f63ba0..98b15a527e37e 100644
--- a/sound/soc/qcom/sc8280xp.c
+++ b/sound/soc/qcom/sc8280xp.c
@@ -164,6 +164,7 @@ static int sc8280xp_platform_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id snd_sc8280xp_dt_match[] = {
+	{.compatible = "qcom,eliza-sndcard", "eliza"},
 	{.compatible = "qcom,kaanapali-sndcard", "kaanapali"},
 	{.compatible = "qcom,qcm6490-idp-sndcard", "qcm6490"},
 	{.compatible = "qcom,qcs615-sndcard", "qcs615"},
diff --git a/sound/soc/renesas/fsi.c b/sound/soc/renesas/fsi.c
index ae86014c38197..6be6587e10957 100644
--- a/sound/soc/renesas/fsi.c
+++ b/sound/soc/renesas/fsi.c
@@ -1992,7 +1992,7 @@ static int fsi_probe(struct platform_device *pdev)
 	const struct fsi_core *core;
 	struct fsi_priv *fsi;
 	struct resource *res;
-	unsigned int irq;
+	int irq;
 	int ret;
 
 	memset(&info, 0, sizeof(info));
@@ -2007,12 +2007,15 @@ static int fsi_probe(struct platform_device *pdev)
 	}
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	irq = platform_get_irq(pdev, 0);
-	if (!res || (int)irq <= 0) {
+	if (!res) {
 		dev_err(&pdev->dev, "Not enough FSI platform resources.\n");
 		return -ENODEV;
 	}
 
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
 	master = devm_kzalloc(&pdev->dev, sizeof(*master), GFP_KERNEL);
 	if (!master)
 		return -ENOMEM;
diff --git a/sound/soc/renesas/rcar/adg.c b/sound/soc/renesas/rcar/adg.c
index 53efd1be51392..2a74d89ba39df 100644
--- a/sound/soc/renesas/rcar/adg.c
+++ b/sound/soc/renesas/rcar/adg.c
@@ -565,7 +565,7 @@ static struct clk *rsnd_adg_create_null_clk(struct rsnd_priv *priv,
 	struct clk *clk;
 
 	clk = clk_register_fixed_rate(dev, name, parent, 0, 0);
-	if (IS_ERR_OR_NULL(clk)) {
+	if (IS_ERR(clk)) {
 		dev_err(dev, "create null clk error\n");
 		return ERR_CAST(clk);
 	}
@@ -618,7 +618,7 @@ static int rsnd_adg_get_clkin(struct rsnd_priv *priv)
 	 * No "adg" is not error
 	 */
 	clk = devm_clk_get(dev, "adg");
-	if (IS_ERR_OR_NULL(clk))
+	if (IS_ERR(clk))
 		clk = rsnd_adg_null_clk_get(priv);
 	adg->adg = clk;
 
@@ -626,9 +626,9 @@ static int rsnd_adg_get_clkin(struct rsnd_priv *priv)
 	for (i = 0; i < clkin_size; i++) {
 		clk = devm_clk_get(dev, clkin_name[i]);
 
-		if (IS_ERR_OR_NULL(clk))
+		if (IS_ERR(clk))
 			clk = rsnd_adg_null_clk_get(priv);
-		if (IS_ERR_OR_NULL(clk))
+		if (IS_ERR(clk))
 			goto err;
 
 		adg->clkin[i] = clk;
diff --git a/sound/soc/samsung/aries_wm8994.c b/sound/soc/samsung/aries_wm8994.c
index 48ccc1d1854bc..d8659973540f2 100644
--- a/sound/soc/samsung/aries_wm8994.c
+++ b/sound/soc/samsung/aries_wm8994.c
@@ -658,6 +658,7 @@ static int aries_audio_probe(struct platform_device *pdev)
 		goto out;
 	}
 
+	of_node_get(aries_dai[0].cpus->of_node);
 	aries_dai[0].platforms->of_node = aries_dai[0].cpus->of_node;
 
 	/* Set CPU of_node for BT DAI */
@@ -700,3 +701,4 @@ module_platform_driver(aries_audio_driver);
 MODULE_DESCRIPTION("ALSA SoC ARIES WM8994");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("platform:aries-audio-wm8994");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/sound/soc/samsung/i2s.c b/sound/soc/samsung/i2s.c
index f80f697a5d55d..56d11741dabdd 100644
--- a/sound/soc/samsung/i2s.c
+++ b/sound/soc/samsung/i2s.c
@@ -8,6 +8,7 @@
 #include <dt-bindings/sound/samsung-i2s.h>
 #include <linux/delay.h>
 #include <linux/slab.h>
+#include <linux/cleanup.h>
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
 #include <linux/io.h>
@@ -512,7 +513,7 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs,
 	u32 mod, mask, val = 0;
 	int ret = 0;
 
-	pm_runtime_get_sync(dai->dev);
+	guard(pm_runtime_active)(dai->dev);
 
 	scoped_guard(spinlock_irqsave, &priv->lock)
 		mod = readl(priv->addr + I2SMOD);
@@ -537,8 +538,7 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs,
 					&& (mod & cdcon_mask))))) {
 			dev_err(&i2s->pdev->dev,
 				"%s:%d Other DAI busy\n", __func__, __LINE__);
-			ret = -EAGAIN;
-			goto err;
+			return -EAGAIN;
 		}
 
 		if (dir == SND_SOC_CLOCK_IN)
@@ -566,7 +566,7 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs,
 				} else {
 					priv->rclk_srcrate =
 						clk_get_rate(priv->op_clk);
-					goto done;
+					return 0;
 				}
 			}
 
@@ -580,14 +580,14 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs,
 			if (WARN_ON(IS_ERR(priv->op_clk))) {
 				ret = PTR_ERR(priv->op_clk);
 				priv->op_clk = NULL;
-				goto err;
+				return ret;
 			}
 
 			ret = clk_prepare_enable(priv->op_clk);
 			if (ret) {
 				clk_put(priv->op_clk);
 				priv->op_clk = NULL;
-				goto err;
+				return ret;
 			}
 			priv->rclk_srcrate = clk_get_rate(priv->op_clk);
 
@@ -595,11 +595,10 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs,
 				|| (clk_id && !(mod & rsrc_mask))) {
 			dev_err(&i2s->pdev->dev,
 				"%s:%d Other DAI busy\n", __func__, __LINE__);
-			ret = -EAGAIN;
-			goto err;
+			return -EAGAIN;
 		} else {
 			/* Call can't be on the active DAI */
-			goto done;
+			return 0;
 		}
 
 		if (clk_id == 1)
@@ -607,8 +606,7 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs,
 		break;
 	default:
 		dev_err(&i2s->pdev->dev, "We don't serve that!\n");
-		ret = -EINVAL;
-		goto err;
+		return -EINVAL;
 	}
 
 	scoped_guard(spinlock_irqsave, &priv->lock) {
@@ -616,13 +614,8 @@ static int i2s_set_sysclk(struct snd_soc_dai *dai, int clk_id, unsigned int rfs,
 		mod = (mod & ~mask) | val;
 		writel(mod, priv->addr + I2SMOD);
 	}
-done:
-	pm_runtime_put(dai->dev);
 
 	return 0;
-err:
-	pm_runtime_put(dai->dev);
-	return ret;
 }
 
 static int i2s_set_fmt(struct snd_soc_dai *dai, unsigned int fmt)
@@ -1448,10 +1441,10 @@ static int samsung_i2s_probe(struct platform_device *pdev)
 	regs_base = res->start;
 
 	priv->clk = devm_clk_get(&pdev->dev, "iis");
-	if (IS_ERR(priv->clk)) {
-		dev_err(&pdev->dev, "Failed to get iis clock\n");
-		return PTR_ERR(priv->clk);
-	}
+	if (IS_ERR(priv->clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(priv->clk),
+				     "Failed to get iis clock\n");
+
 
 	ret = clk_prepare_enable(priv->clk);
 	if (ret != 0) {
diff --git a/sound/soc/samsung/midas_wm1811.c b/sound/soc/samsung/midas_wm1811.c
index bc34dbbb50c91..d43af5069baf9 100644
--- a/sound/soc/samsung/midas_wm1811.c
+++ b/sound/soc/samsung/midas_wm1811.c
@@ -773,3 +773,4 @@ module_platform_driver(midas_driver);
 MODULE_AUTHOR("Simon Shields <simon@lineageos.org>");
 MODULE_DESCRIPTION("ASoC support for Midas");
 MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/sound/soc/samsung/spdif.c b/sound/soc/samsung/spdif.c
index 7fc46d55c522a..53eaabaf89560 100644
--- a/sound/soc/samsung/spdif.c
+++ b/sound/soc/samsung/spdif.c
@@ -380,8 +380,8 @@ static int spdif_probe(struct platform_device *pdev)
 
 	spdif->pclk = devm_clk_get(&pdev->dev, "spdif");
 	if (IS_ERR(spdif->pclk)) {
-		dev_err(&pdev->dev, "failed to get peri-clock\n");
-		ret = -ENOENT;
+		ret = dev_err_probe(&pdev->dev, PTR_ERR(spdif->pclk),
+				    "failed to get peri-clock\n");
 		goto err0;
 	}
 	ret = clk_prepare_enable(spdif->pclk);
@@ -390,8 +390,8 @@ static int spdif_probe(struct platform_device *pdev)
 
 	spdif->sclk = devm_clk_get(&pdev->dev, "sclk_spdif");
 	if (IS_ERR(spdif->sclk)) {
-		dev_err(&pdev->dev, "failed to get internal source clock\n");
-		ret = -ENOENT;
+		ret = dev_err_probe(&pdev->dev, PTR_ERR(spdif->sclk),
+				    "failed to get internal source clock\n");
 		goto err1;
 	}
 	ret = clk_prepare_enable(spdif->sclk);
diff --git a/sound/soc/sdw_utils/soc_sdw_bridge_cs35l56.c b/sound/soc/sdw_utils/soc_sdw_bridge_cs35l56.c
index e0e32a279787c..129a437ae397e 100644
--- a/sound/soc/sdw_utils/soc_sdw_bridge_cs35l56.c
+++ b/sound/soc/sdw_utils/soc_sdw_bridge_cs35l56.c
@@ -99,11 +99,9 @@ static const struct snd_soc_dai_link bridge_dai_template = {
 	SND_SOC_DAILINK_REG(asoc_sdw_bridge_dai),
 };
 
-int asoc_sdw_bridge_cs35l56_count_sidecar(struct snd_soc_card *card,
+int asoc_sdw_bridge_cs35l56_count_sidecar(struct asoc_sdw_mc_private *ctx,
 					  int *num_dais, int *num_devs)
 {
-	struct asoc_sdw_mc_private *ctx = snd_soc_card_get_drvdata(card);
-
 	if (ctx->mc_quirk & SOC_SDW_SIDECAR_AMPS) {
 		(*num_dais)++;
 		(*num_devs) += ARRAY_SIZE(bridge_cs35l56_name_prefixes);
diff --git a/sound/soc/sdw_utils/soc_sdw_utils.c b/sound/soc/sdw_utils/soc_sdw_utils.c
index d8db8fc5313ee..d4074309fa81c 100644
--- a/sound/soc/sdw_utils/soc_sdw_utils.c
+++ b/sound/soc/sdw_utils/soc_sdw_utils.c
@@ -834,6 +834,35 @@ struct asoc_sdw_codec_info codec_info_list[] = {
 		},
 		.dai_num = 2,
 	},
+	{
+		.vendor_id = 0x01fa,
+		.part_id = 0x3562,
+		.name_prefix = "AMP",
+		.is_amp = true,
+		.dais = {
+			{
+				.direction = {true, false},
+				.dai_name = "cs35l56-sdw1",
+				.component_name = "cs35l56",
+				.dai_type = SOC_SDW_DAI_TYPE_AMP,
+				.dailink = {SOC_SDW_AMP_OUT_DAI_ID, SOC_SDW_UNUSED_DAI_ID},
+				.init = asoc_sdw_cs_amp_init,
+				.rtd_init = asoc_sdw_cs_spk_rtd_init,
+				.controls = generic_spk_controls,
+				.num_controls = ARRAY_SIZE(generic_spk_controls),
+				.widgets = generic_spk_widgets,
+				.num_widgets = ARRAY_SIZE(generic_spk_widgets),
+			},
+			{
+				.direction = {false, true},
+				.dai_name = "cs35l56-sdw1c",
+				.dai_type = SOC_SDW_DAI_TYPE_AMP,
+				.dailink = {SOC_SDW_UNUSED_DAI_ID, SOC_SDW_AMP_IN_DAI_ID},
+				.rtd_init = asoc_sdw_cs_spk_feedback_rtd_init,
+			},
+		},
+		.dai_num = 2,
+	},
 	{
 		.vendor_id = 0x01fa,
 		.part_id = 0x3563,
@@ -1976,14 +2005,13 @@ static int is_sdca_endpoint_present(struct device *dev,
 	return ret;
 }
 
-int asoc_sdw_parse_sdw_endpoints(struct snd_soc_card *card,
+int asoc_sdw_parse_sdw_endpoints(struct device *dev,
+				 struct asoc_sdw_mc_private *ctx,
 				 struct snd_soc_aux_dev *soc_aux,
 				 struct asoc_sdw_dailink *soc_dais,
 				 struct asoc_sdw_endpoint *soc_ends,
 				 int *num_devs)
 {
-	struct device *dev = card->dev;
-	struct asoc_sdw_mc_private *ctx = snd_soc_card_get_drvdata(card);
 	struct snd_soc_acpi_mach *mach = dev_get_platdata(dev);
 	struct snd_soc_acpi_mach_params *mach_params = &mach->mach_params;
 	const struct snd_soc_acpi_link_adr *adr_link;
@@ -2045,7 +2073,7 @@ int asoc_sdw_parse_sdw_endpoints(struct snd_soc_card *card,
 			ctx->ignore_internal_dmic |= codec_info->ignore_internal_dmic;
 
 			if (codec_info->count_sidecar && codec_info->add_sidecar) {
-				ret = codec_info->count_sidecar(card, &num_dais, num_devs);
+				ret = codec_info->count_sidecar(ctx, &num_dais, num_devs);
 				if (ret)
 					return ret;
 
diff --git a/sound/soc/sof/amd/Kconfig b/sound/soc/sof/amd/Kconfig
index 05faf1c6d6fc3..903e7ec3b3ba4 100644
--- a/sound/soc/sof/amd/Kconfig
+++ b/sound/soc/sof/amd/Kconfig
@@ -104,4 +104,15 @@ config SND_SOC_SOF_AMD_ACP70
 	  AMD ACP7.0/ACP7.1 version based platforms.
 	  Say Y if you want to enable SOF on ACP7.0/ACP7.1 based platforms.
 
+config SND_SOC_SOF_AMD_ACP7X
+	tristate "SOF support for ACP7.B/7.F platforms"
+	depends on SND_SOC_SOF_PCI
+	depends on AMD_NODE
+	select SND_SOC_SOF_AMD_COMMON
+	help
+	  Select this option for SOF support on AMD ACP7.B and ACP7.F PCI
+	  revision based platforms.
+	  Say Y if you want to enable SOF on ACP7.B/7.F based platforms.
+	  If unsure select "N".
+
 endif
diff --git a/sound/soc/sof/amd/Makefile b/sound/soc/sof/amd/Makefile
index 6ae39fd5a8362..bb1c0ddef69d7 100644
--- a/sound/soc/sof/amd/Makefile
+++ b/sound/soc/sof/amd/Makefile
@@ -11,6 +11,7 @@ snd-sof-amd-rembrandt-y := pci-rmb.o rembrandt.o
 snd-sof-amd-vangogh-y := pci-vangogh.o vangogh.o
 snd-sof-amd-acp63-y := pci-acp63.o acp63.o
 snd-sof-amd-acp70-y := pci-acp70.o acp70.o
+snd-sof-amd-acp7x-y := pci-acp7x.o acp7x.o
 
 obj-$(CONFIG_SND_SOC_SOF_AMD_COMMON) += snd-sof-amd-acp.o
 obj-$(CONFIG_SND_SOC_SOF_AMD_RENOIR) += snd-sof-amd-renoir.o
@@ -18,3 +19,4 @@ obj-$(CONFIG_SND_SOC_SOF_AMD_REMBRANDT) += snd-sof-amd-rembrandt.o
 obj-$(CONFIG_SND_SOC_SOF_AMD_VANGOGH) += snd-sof-amd-vangogh.o
 obj-$(CONFIG_SND_SOC_SOF_AMD_ACP63) += snd-sof-amd-acp63.o
 obj-$(CONFIG_SND_SOC_SOF_AMD_ACP70) += snd-sof-amd-acp70.o
+obj-$(CONFIG_SND_SOC_SOF_AMD_ACP7X) += snd-sof-amd-acp7x.o
diff --git a/sound/soc/sof/amd/acp-dsp-offset.h b/sound/soc/sof/amd/acp-dsp-offset.h
index 08583a91afbc4..bea1bd3afa70d 100644
--- a/sound/soc/sof/amd/acp-dsp-offset.h
+++ b/sound/soc/sof/amd/acp-dsp-offset.h
@@ -68,12 +68,14 @@
 #define ACP5X_PGFSM_BASE			0x1424
 #define ACP6X_PGFSM_BASE                        0x1024
 #define ACP70_PGFSM_BASE                        ACP6X_PGFSM_BASE
+#define ACP7X_PGFSM_BASE                        ACP6X_PGFSM_BASE
 #define PGFSM_CONTROL_OFFSET			0x0
 #define PGFSM_STATUS_OFFSET			0x4
 #define ACP3X_CLKMUX_SEL			0x1424
 #define ACP5X_CLKMUX_SEL			0x142C
 #define ACP6X_CLKMUX_SEL			0x102C
 #define ACP70_CLKMUX_SEL			ACP6X_CLKMUX_SEL
+#define ACP7X_CLKMUX_SEL			ACP6X_CLKMUX_SEL
 
 /* Registers from ACP_INTR block */
 #define ACP3X_EXT_INTR_STAT			0x1808
@@ -86,21 +88,31 @@
 #define ACP70_EXTERNAL_INTR_CNTL		ACP6X_EXTERNAL_INTR_CNTL
 #define ACP70_EXT_INTR_STAT			ACP6X_EXT_INTR_STAT
 #define ACP70_EXT_INTR_STAT1			ACP6X_EXT_INTR_STAT1
+#define ACP7X_EXTERNAL_INTR_ENB			ACP6X_EXTERNAL_INTR_ENB
+#define ACP7X_EXTERNAL_INTR_CNTL		0x1A04
+#define ACP7X_EXT_INTR_STAT			0x1A1C
+#define ACP7X_EXTERNAL_INTR_CNTL1		0x1A08
+#define ACP7X_EXT_INTR_STAT1			0x1A20
 
 #define ACP3X_DSP_SW_INTR_BASE			0x1814
 #define ACP5X_DSP_SW_INTR_BASE			0x1814
 #define ACP6X_DSP_SW_INTR_BASE                  0x1808
 #define ACP70_DSP_SW_INTR_BASE			ACP6X_DSP_SW_INTR_BASE
+#define ACP7X_DSP_SW_INTR_BASE			0x1860
 #define DSP_SW_INTR_CNTL_OFFSET			0x0
 #define DSP_SW_INTR_STAT_OFFSET			0x4
+#define ACP7X_DSP_SW_INTR_STAT			(ACP7X_DSP_SW_INTR_BASE + DSP_SW_INTR_STAT_OFFSET)
 #define DSP_SW_INTR_TRIG_OFFSET			0x8
+#define ACP7X_DSP_SW_INTR_TRIG_OFFSET		0x30
 #define ACP3X_ERROR_STATUS			0x18C4
 #define ACP6X_ERROR_STATUS			0x1A4C
 #define ACP70_ERROR_STATUS			ACP6X_ERROR_STATUS
+#define ACP7X_ERROR_STATUS			0x1A88
 #define ACP3X_AXI2DAGB_SEM_0			0x1880
 #define ACP5X_AXI2DAGB_SEM_0			0x1884
 #define ACP6X_AXI2DAGB_SEM_0			0x1874
 #define ACP70_AXI2DAGB_SEM_0			ACP6X_AXI2DAGB_SEM_0
+#define ACP7X_AXI2DAGB_SEM_0			0x18F4
 
 /* ACP common registers to report errors related to I2S & SoundWire interfaces */
 #define ACP3X_SW_I2S_ERROR_REASON		0x18C8
@@ -123,6 +135,7 @@
 #define ACP_SCRATCH_REG_0			0x10000
 #define ACP6X_DSP_FUSION_RUNSTALL		0x0644
 #define ACP70_DSP_FUSION_RUNSTALL		ACP6X_DSP_FUSION_RUNSTALL
+#define ACP7X_DSP_FUSION_RUNSTALL		ACP6X_DSP_FUSION_RUNSTALL
 
 /* Cache window registers */
 #define ACP_DSP0_CACHE_OFFSET0			0x0420
@@ -139,5 +152,9 @@
 #define ACP70_SDW1_HOST_WAKE_STAT		BIT(25)
 #define ACP70_SDW0_PME_STAT			BIT(26)
 #define ACP70_SDW1_PME_STAT			BIT(27)
+#define ACP7X_DSP0_IDMA_ERROR_MASK		0x4B0
+#define ACP7X_IDMA_ERROR_MASK			0x1FF9FF
+#define ACP7X_ZSC_DSP_CTRL			0x001014
+#define ACP7X_PME_EN				ACP70_PME_EN
 
 #endif
diff --git a/sound/soc/sof/amd/acp-ipc.c b/sound/soc/sof/amd/acp-ipc.c
index 94025bc799ea4..494f9cb0c06fc 100644
--- a/sound/soc/sof/amd/acp-ipc.c
+++ b/sound/soc/sof/amd/acp-ipc.c
@@ -32,11 +32,20 @@ static void acpbus_trigger_host_to_dsp_swintr(struct acp_dev_data *adata)
 	struct snd_sof_dev *sdev = adata->dev;
 	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
 	u32 swintr_trigger;
-
+	unsigned int swintr_trigger_reg_offset;
+
+	switch (adata->pci_rev) {
+	case ACP7B_PCI_ID:
+	case ACP7F_PCI_ID:
+		swintr_trigger_reg_offset = ACP7X_DSP_SW_INTR_TRIG_OFFSET;
+		break;
+	default:
+		swintr_trigger_reg_offset = DSP_SW_INTR_TRIG_OFFSET;
+	}
 	swintr_trigger = snd_sof_dsp_read(sdev, ACP_DSP_BAR, desc->dsp_intr_base +
-						DSP_SW_INTR_TRIG_OFFSET);
+						swintr_trigger_reg_offset);
 	swintr_trigger |= 0x01;
-	snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->dsp_intr_base + DSP_SW_INTR_TRIG_OFFSET,
+	snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->dsp_intr_base + swintr_trigger_reg_offset,
 			  swintr_trigger);
 }
 
diff --git a/sound/soc/sof/amd/acp-loader.c b/sound/soc/sof/amd/acp-loader.c
index 98324bbade151..f1ec856943238 100644
--- a/sound/soc/sof/amd/acp-loader.c
+++ b/sound/soc/sof/amd/acp-loader.c
@@ -14,6 +14,7 @@
 #include <linux/firmware.h>
 #include <linux/module.h>
 #include <linux/pci.h>
+#include <linux/unaligned.h>
 
 #include "../ops.h"
 #include "acp-dsp-offset.h"
@@ -173,10 +174,31 @@ int acp_dsp_pre_fw_run(struct snd_sof_dev *sdev)
 
 	adata = sdev->pdata->hw_pdata;
 
-	if (adata->quirks && adata->quirks->signed_fw_image)
+	if (adata->pci_rev >= ACP7B_PCI_ID) {
+		if (adata->acp_sof_signed_firmware_image) {
+			if (adata->fw_bin_size <= ACP_IMAGE_HEADER_SIZE) {
+				dev_err(sdev->dev, "Invalid signed firmware size %u\n",
+					adata->fw_bin_size);
+				return -EINVAL;
+			}
+			size_fw = get_unaligned_le32(adata->bin_buf +
+						     ACP_IMAGE_HDR_SIZE_FW_SIGNED_OFF);
+			if (!size_fw ||
+			    size_fw > adata->fw_bin_size - ACP_IMAGE_HEADER_SIZE) {
+				dev_err(sdev->dev,
+					"Invalid signed firmware payload size %u (max %u)\n",
+					size_fw, adata->fw_bin_size - ACP_IMAGE_HEADER_SIZE);
+				return -EINVAL;
+			}
+			size_fw += ACP_IMAGE_HEADER_SIZE;
+		} else {
+			size_fw = adata->fw_bin_size;
+		}
+	} else if (adata->quirks && adata->quirks->signed_fw_image) {
 		size_fw = adata->fw_bin_size - ACP_FIRMWARE_SIGNATURE;
-	else
+	} else {
 		size_fw = adata->fw_bin_size;
+	}
 
 	page_count = PAGE_ALIGN(size_fw) >> PAGE_SHIFT;
 	adata->fw_bin_page_count = page_count;
@@ -312,9 +334,14 @@ int acp_sof_load_signed_firmware(struct snd_sof_dev *sdev)
 	}
 	kfree(fw_filename);
 
-	ret = snd_sof_dsp_block_write(sdev, SOF_FW_BLK_TYPE_DRAM, 0,
-				      (void *)adata->fw_dbin->data,
-				      adata->fw_dbin->size);
+	if (adata->pci_rev >= ACP7B_PCI_ID)
+		ret = snd_sof_dsp_block_write(sdev, SOF_FW_BLK_TYPE_SRAM, 0,
+					      (void *)adata->fw_dbin->data,
+					      adata->fw_dbin->size);
+	else
+		ret = snd_sof_dsp_block_write(sdev, SOF_FW_BLK_TYPE_DRAM, 0,
+					      (void *)adata->fw_dbin->data,
+					      adata->fw_dbin->size);
 	return ret;
 }
 EXPORT_SYMBOL_NS(acp_sof_load_signed_firmware, "SND_SOC_SOF_AMD_COMMON");
diff --git a/sound/soc/sof/amd/acp.c b/sound/soc/sof/amd/acp.c
index e6af8927baa0e..f89ad86260b4e 100644
--- a/sound/soc/sof/amd/acp.c
+++ b/sound/soc/sof/amd/acp.c
@@ -12,6 +12,7 @@
  * Hardware interface for generic AMD ACP processor
  */
 
+#include <linux/acpi.h>
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/pci.h>
@@ -60,6 +61,8 @@ static void init_dma_descriptor(struct acp_dev_data *adata)
 	case ACP70_PCI_ID:
 	case ACP71_PCI_ID:
 	case ACP72_PCI_ID:
+	case ACP7B_PCI_ID:
+	case ACP7F_PCI_ID:
 		acp_dma_desc_base_addr = ACP70_DMA_DESC_BASE_ADDR;
 		acp_dma_desc_max_num_dscr = ACP70_DMA_DESC_MAX_NUM_DSCR;
 		break;
@@ -101,6 +104,8 @@ static int config_dma_channel(struct acp_dev_data *adata, unsigned int ch,
 	case ACP70_PCI_ID:
 	case ACP71_PCI_ID:
 	case ACP72_PCI_ID:
+	case ACP7B_PCI_ID:
+	case ACP7F_PCI_ID:
 		acp_dma_cntl_0 = ACP70_DMA_CNTL_0;
 		acp_dma_ch_rst_sts = ACP70_DMA_CH_RST_STS;
 		acp_dma_dscr_err_sts_0 = ACP70_DMA_ERR_STS_0;
@@ -283,7 +288,8 @@ int configure_and_run_sha_dma(struct acp_dev_data *adata, void *image_addr,
 		}
 	}
 
-	if (adata->quirks && adata->quirks->signed_fw_image)
+	if ((adata->quirks && adata->quirks->signed_fw_image) ||
+	    adata->acp_sof_signed_firmware_image)
 		snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_SHA_DMA_INCLUDE_HDR, ACP_SHA_HEADER);
 
 	snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_SHA_DMA_STRT_ADDR, start_addr);
@@ -342,6 +348,8 @@ int acp_dma_status(struct acp_dev_data *adata, unsigned char ch)
 	case ACP70_PCI_ID:
 	case ACP71_PCI_ID:
 	case ACP72_PCI_ID:
+	case ACP7B_PCI_ID:
+	case ACP7F_PCI_ID:
 		acp_dma_ch_sts = ACP70_DMA_CH_STS;
 		break;
 	default:
@@ -534,8 +542,17 @@ static irqreturn_t acp_irq_handler(int irq, void *dev_id)
 		snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->ext_intr_stat, ACP_ERROR_IRQ_MASK);
 		snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->acp_sw0_i2s_err_reason, 0);
 		/* ACP_SW1_I2S_ERROR_REASON is newly added register from rmb platform onwards */
-		if (adata->pci_rev >= ACP_RMB_PCI_ID)
+		switch (adata->pci_rev) {
+		case ACP_RMB_PCI_ID:
+		case ACP63_PCI_ID:
+		case ACP70_PCI_ID:
+		case ACP71_PCI_ID:
+		case ACP72_PCI_ID:
 			snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_SW1_I2S_ERROR_REASON, 0);
+			break;
+		default:
+			break;
+		}
 		snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->acp_error_stat, 0);
 		irq_flag = 1;
 	}
@@ -564,6 +581,35 @@ static irqreturn_t acp_irq_handler(int irq, void *dev_id)
 		return IRQ_NONE;
 }
 
+static irqreturn_t acp7x_irq_handler(int irq, void *dev_id)
+{
+	struct snd_sof_dev *sdev = dev_id;
+	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
+	unsigned int base = desc->dsp_intr_base;
+	unsigned int val;
+	unsigned int ext_intr_stat;
+	int irq_flag = 0;
+
+	val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, base + DSP_SW_INTR_STAT_OFFSET);
+	if (val & ACP_DSP_TO_HOST_IRQ) {
+		snd_sof_dsp_write(sdev, ACP_DSP_BAR, base + DSP_SW_INTR_STAT_OFFSET,
+				  ACP_DSP_TO_HOST_IRQ);
+		return IRQ_WAKE_THREAD;
+	}
+
+	ext_intr_stat = snd_sof_dsp_read(sdev, ACP_DSP_BAR, desc->ext_intr_stat);
+	if (ext_intr_stat & ACP_ERROR_IRQ_MASK) {
+		snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->ext_intr_stat, ACP_ERROR_IRQ_MASK);
+		snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->acp_error_stat, 0);
+		irq_flag = 1;
+	}
+
+	if (irq_flag)
+		return IRQ_HANDLED;
+
+	return IRQ_NONE;
+}
+
 static int acp_power_on(struct snd_sof_dev *sdev)
 {
 	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
@@ -571,6 +617,7 @@ static int acp_power_on(struct snd_sof_dev *sdev)
 	unsigned int base = desc->pgfsm_base;
 	unsigned int val;
 	unsigned int acp_pgfsm_status_mask, acp_pgfsm_cntl_mask;
+	bool use_masked_status = false;
 	int ret;
 
 	val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, base + PGFSM_STATUS_OFFSET);
@@ -595,6 +642,12 @@ static int acp_power_on(struct snd_sof_dev *sdev)
 		acp_pgfsm_status_mask = ACP70_PGFSM_STATUS_MASK;
 		acp_pgfsm_cntl_mask = ACP70_PGFSM_CNTL_POWER_ON_MASK;
 		break;
+	case ACP7B_PCI_ID:
+	case ACP7F_PCI_ID:
+		acp_pgfsm_status_mask = ACP7X_PGFSM_STATUS_MASK;
+		acp_pgfsm_cntl_mask = ACP7X_PGFSM_CNTL_POWER_ON_MASK;
+		use_masked_status = true;
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -603,8 +656,17 @@ static int acp_power_on(struct snd_sof_dev *sdev)
 		snd_sof_dsp_write(sdev, ACP_DSP_BAR, base + PGFSM_CONTROL_OFFSET,
 				  acp_pgfsm_cntl_mask);
 
-	ret = snd_sof_dsp_read_poll_timeout(sdev, ACP_DSP_BAR, base + PGFSM_STATUS_OFFSET, val,
-					    !val, ACP_REG_POLL_INTERVAL, ACP_REG_POLL_TIMEOUT_US);
+	if (use_masked_status)
+		ret = snd_sof_dsp_read_poll_timeout(sdev, ACP_DSP_BAR,
+						    base + PGFSM_STATUS_OFFSET, val,
+						    !(val & acp_pgfsm_status_mask),
+						    ACP_REG_POLL_INTERVAL,
+						    ACP_REG_POLL_TIMEOUT_US);
+	else
+		ret = snd_sof_dsp_read_poll_timeout(sdev, ACP_DSP_BAR,
+						    base + PGFSM_STATUS_OFFSET, val,
+						    !val, ACP_REG_POLL_INTERVAL,
+						    ACP_REG_POLL_TIMEOUT_US);
 	if (ret < 0)
 		dev_err(sdev->dev, "timeout in ACP_PGFSM_STATUS read\n");
 
@@ -703,6 +765,13 @@ static int acp_init(struct snd_sof_dev *sdev)
 
 		snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP70_PME_EN, 1);
 		break;
+	case ACP7B_PCI_ID:
+	case ACP7F_PCI_ID:
+		snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP7X_ZSC_DSP_CTRL, 0);
+		snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP7X_PME_EN, 1);
+		snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP7X_DSP0_IDMA_ERROR_MASK,
+				  ACP7X_IDMA_ERROR_MASK);
+		break;
 	}
 	return 0;
 }
@@ -749,6 +818,8 @@ int amd_sof_acp_suspend(struct snd_sof_dev *sdev, u32 target_state)
 	case ACP72_PCI_ID:
 		enable = true;
 		break;
+	default:
+		break;
 	}
 	snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_CONTROL, enable);
 
@@ -990,6 +1061,194 @@ void amd_sof_acp_remove(struct snd_sof_dev *sdev)
 }
 EXPORT_SYMBOL_NS(amd_sof_acp_remove, "SND_SOC_SOF_AMD_COMMON");
 
+int amd_sof_acp7x_probe(struct snd_sof_dev *sdev)
+{
+	struct pci_dev *pci = to_pci_dev(sdev->dev);
+	struct acp_dev_data *adata;
+	const struct sof_amd_acp_desc *chip;
+	const union acpi_object *obj;
+	struct acpi_device *adev;
+	unsigned int addr;
+	unsigned int irqflags;
+	int ret;
+
+	chip = get_chip_info(sdev->pdata);
+	if (!chip) {
+		dev_err(sdev->dev, "no such device supported, chip id:%x\n", pci->device);
+		return -EIO;
+	}
+	adata = devm_kzalloc(sdev->dev, sizeof(struct acp_dev_data), GFP_KERNEL);
+	if (!adata)
+		return -ENOMEM;
+
+	adata->dev = sdev;
+	adata->dmic_dev = platform_device_register_data(sdev->dev, "dmic-codec",
+							PLATFORM_DEVID_NONE, NULL, 0);
+	if (IS_ERR(adata->dmic_dev)) {
+		dev_err(sdev->dev, "failed to register platform for dmic codec\n");
+		return PTR_ERR(adata->dmic_dev);
+	}
+
+	addr = pci_resource_start(pci, ACP_DSP_BAR);
+	sdev->bar[ACP_DSP_BAR] = devm_ioremap(sdev->dev, addr, pci_resource_len(pci, ACP_DSP_BAR));
+	if (!sdev->bar[ACP_DSP_BAR]) {
+		dev_err(sdev->dev, "ioremap error\n");
+		ret = -ENXIO;
+		goto unregister_dev;
+	}
+
+	pci_set_master(pci);
+	adata->addr = addr;
+	adata->reg_range = chip->reg_end_addr - chip->reg_start_addr;
+	adata->pci_rev = pci->revision;
+	mutex_init(&adata->acp_lock);
+	sdev->pdata->hw_pdata = adata;
+
+	ret = acp_init(sdev);
+	if (ret < 0)
+		goto unregister_dev;
+
+	adev = ACPI_COMPANION(&pci->dev);
+
+	sdev->ipc_irq = pci->irq;
+	irqflags = IRQF_SHARED;
+
+	ret = request_threaded_irq(pci->irq, acp7x_irq_handler, acp_irq_thread,
+				   irqflags, "AudioDSP", sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "failed to register IRQ %d\n", sdev->ipc_irq);
+		goto unregister_dev;
+	}
+
+	if (adev) {
+		if (!acpi_dev_get_property(adev, "acp-sof-signed-firmware-image",
+					   ACPI_TYPE_INTEGER, &obj))
+			adata->acp_sof_signed_firmware_image = obj->integer.value;
+	}
+
+	sdev->dsp_box.offset = 0;
+	sdev->dsp_box.size = BOX_SIZE_512;
+
+	sdev->host_box.offset = sdev->dsp_box.offset + sdev->dsp_box.size;
+	sdev->host_box.size = BOX_SIZE_512;
+
+	sdev->debug_box.offset = sdev->host_box.offset + sdev->host_box.size;
+	sdev->debug_box.size = BOX_SIZE_1024;
+
+	if (adata->acp_sof_signed_firmware_image) {
+		adata->fw_code_bin = devm_kasprintf(sdev->dev, GFP_KERNEL,
+						    "sof-%s-code.bin", chip->name);
+		if (!adata->fw_code_bin) {
+			ret = -ENOMEM;
+			goto free_ipc_irq;
+		}
+		adata->fw_data_bin = devm_kasprintf(sdev->dev, GFP_KERNEL,
+						    "sof-%s-data.bin", chip->name);
+		if (!adata->fw_data_bin) {
+			ret = -ENOMEM;
+			goto free_ipc_irq;
+		}
+	}
+
+	adata->enable_fw_debug = enable_fw_debug;
+	acp_memory_init(sdev);
+	acp_dsp_stream_init(sdev);
+
+	return 0;
+
+free_ipc_irq:
+	free_irq(sdev->ipc_irq, sdev);
+unregister_dev:
+	platform_device_unregister(adata->dmic_dev);
+	return ret;
+}
+EXPORT_SYMBOL_NS(amd_sof_acp7x_probe, "SND_SOC_SOF_AMD_COMMON");
+
+void amd_sof_acp7x_remove(struct snd_sof_dev *sdev)
+{
+	struct acp_dev_data *adata = sdev->pdata->hw_pdata;
+
+	if (sdev->ipc_irq)
+		free_irq(sdev->ipc_irq, sdev);
+
+	if (adata->dmic_dev)
+		platform_device_unregister(adata->dmic_dev);
+
+	acp_reset(sdev);
+}
+EXPORT_SYMBOL_NS(amd_sof_acp7x_remove, "SND_SOC_SOF_AMD_COMMON");
+
+int amd_sof_acp7x_suspend(struct snd_sof_dev *sdev, u32 target_state)
+{
+	struct acp_dev_data *acp_data;
+	int ret;
+	bool enable = false;
+
+	acp_data = sdev->pdata->hw_pdata;
+
+	ret = acp_reset(sdev);
+	if (ret) {
+		dev_err(sdev->dev, "ACP Reset failed\n");
+		return ret;
+	}
+	switch (acp_data->pci_rev) {
+	case ACP7B_PCI_ID:
+	case ACP7F_PCI_ID:
+		enable = true;
+		break;
+	default:
+		break;
+	}
+	snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_CONTROL, enable);
+	snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP7X_ZSC_DSP_CTRL, 1);
+
+	return 0;
+}
+EXPORT_SYMBOL_NS(amd_sof_acp7x_suspend, "SND_SOC_SOF_AMD_COMMON");
+
+int amd_sof_acp7x_resume(struct snd_sof_dev *sdev)
+{
+	struct acp_dev_data *acp_data;
+	int ret;
+
+	acp_data = sdev->pdata->hw_pdata;
+
+	ret = acp_init(sdev);
+	if (ret) {
+		dev_err(sdev->dev, "ACP Init failed\n");
+		return ret;
+	}
+	ret = acp_memory_init(sdev);
+	if (ret) {
+		dev_err(sdev->dev, "ACP Memory init failed\n");
+		return ret;
+	}
+
+	switch (acp_data->pci_rev) {
+	case ACP7B_PCI_ID:
+	case ACP7F_PCI_ID:
+		snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP7X_PME_EN, 1);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_NS(amd_sof_acp7x_resume, "SND_SOC_SOF_AMD_COMMON");
+
+int amd_sof_acp7x_suspend_runtime(struct snd_sof_dev *sdev)
+{
+	return amd_sof_acp7x_suspend(sdev, 0);
+}
+EXPORT_SYMBOL_NS(amd_sof_acp7x_suspend_runtime, "SND_SOC_SOF_AMD_COMMON");
+
+int amd_sof_acp7x_resume_runtime(struct snd_sof_dev *sdev)
+{
+	return amd_sof_acp7x_resume(sdev);
+}
+EXPORT_SYMBOL_NS(amd_sof_acp7x_resume_runtime, "SND_SOC_SOF_AMD_COMMON");
+
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_DESCRIPTION("AMD ACP sof driver");
 MODULE_IMPORT_NS("SOUNDWIRE_AMD_INIT");
diff --git a/sound/soc/sof/amd/acp.h b/sound/soc/sof/amd/acp.h
index 7bcb76676a984..16d66b2eaa70b 100644
--- a/sound/soc/sof/amd/acp.h
+++ b/sound/soc/sof/amd/acp.h
@@ -76,6 +76,12 @@
 #define ACP70_PCI_ID				0x70
 #define ACP71_PCI_ID				0x71
 #define ACP72_PCI_ID				0x72
+#define ACP7B_PCI_ID				0x7B
+#define ACP7F_PCI_ID				0x7F
+
+#define ACP7X_PGFSM_CNTL_POWER_ON_MASK		0x7F
+#define ACP7X_PGFSM_STATUS_MASK			0xFFF
+#define ACP7X_SRAM_PTE_OFFSET			ACP6X_SRAM_PTE_OFFSET
 
 #define HOST_BRIDGE_CZN				0x1630
 #define HOST_BRIDGE_VGH				0x1645
@@ -107,6 +113,9 @@
 #define PROBE_STATUS_BIT			BIT(31)
 
 #define ACP_FIRMWARE_SIGNATURE			0x100
+#define ACP_IMAGE_HEADER_SIZE			ACP_FIRMWARE_SIGNATURE
+#define ACP_IMAGE_HDR_SIZE_FW_SIGNED_OFF	0x14
+
 #define ACP_ERROR_IRQ_MASK			BIT(29)
 #define ACP_SDW0_IRQ_MASK			BIT(21)
 #define ACP_SDW1_IRQ_MASK			BIT(2)
@@ -271,6 +280,7 @@ struct acp_dev_data {
 	/* acp70_sdw1_wake_event flag set to true when wake irq asserted for SW1 instance */
 	bool acp70_sdw1_wake_event;
 	unsigned int pci_rev;
+	int acp_sof_signed_firmware_image;
 };
 
 void memcpy_to_scratch(struct snd_sof_dev *sdev, u32 offset, unsigned int *src, size_t bytes);
@@ -344,6 +354,16 @@ int sof_acp63_ops_init(struct snd_sof_dev *sdev);
 extern struct snd_sof_dsp_ops sof_acp70_ops;
 int sof_acp70_ops_init(struct snd_sof_dev *sdev);
 
+extern struct snd_sof_dsp_ops sof_acp7x_ops;
+int sof_acp7x_ops_init(struct snd_sof_dev *sdev);
+
+int amd_sof_acp7x_probe(struct snd_sof_dev *sdev);
+void amd_sof_acp7x_remove(struct snd_sof_dev *sdev);
+int amd_sof_acp7x_suspend(struct snd_sof_dev *sdev, u32 target_state);
+int amd_sof_acp7x_resume(struct snd_sof_dev *sdev);
+int amd_sof_acp7x_suspend_runtime(struct snd_sof_dev *sdev);
+int amd_sof_acp7x_resume_runtime(struct snd_sof_dev *sdev);
+
 struct snd_soc_acpi_mach *amd_sof_machine_select(struct snd_sof_dev *sdev);
 /* Machine configuration */
 int snd_amd_acp_find_config(struct pci_dev *pci);
diff --git a/sound/soc/sof/amd/acp7x.c b/sound/soc/sof/amd/acp7x.c
new file mode 100644
index 0000000000000..b6722d11168cb
--- /dev/null
+++ b/sound/soc/sof/amd/acp7x.c
@@ -0,0 +1,185 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license. When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2025 Advanced Micro Devices, Inc.
+//
+// Authors: Vijendar Mukunda <Vijendar.Mukunda@amd.com>
+
+/*
+ * Hardware interface for Audio DSP on ACP7.B/7.F platforms
+ */
+
+#include <linux/acpi.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pci.h>
+#include <sound/soc-acpi.h>
+
+#include "../ops.h"
+#include "../sof-audio.h"
+#include "acp.h"
+#include "acp-dsp-offset.h"
+
+#define I2S_TDM0_INSTANCE	0
+#define I2S_TDM1_INSTANCE	1
+#define I2S_TDM2_INSTANCE	2
+#define PDM0_DMIC_INSTANCE	3
+#define PDM1_DMIC_INSTANCE	4
+
+static struct snd_soc_dai_driver acp7x_sof_dai[] = {
+	[I2S_TDM0_INSTANCE] = {
+		.id = I2S_TDM0_INSTANCE,
+		.name = "acp-sof-i2s0",
+		.playback = {
+			.rates = SNDRV_PCM_RATE_8000_96000,
+			.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
+				   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+			.channels_min = 2,
+			.channels_max = 8,
+			.rate_min = 8000,
+			.rate_max = 96000,
+		},
+		.capture = {
+			.rates = SNDRV_PCM_RATE_8000_48000,
+			.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
+				   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+			/* Supporting only stereo for I2S HS controller capture */
+			.channels_min = 2,
+			.channels_max = 2,
+			.rate_min = 8000,
+			.rate_max = 48000,
+		},
+	},
+
+	[I2S_TDM1_INSTANCE] = {
+		.id = I2S_TDM1_INSTANCE,
+		.name = "acp-sof-i2s1",
+		.playback = {
+			.rates = SNDRV_PCM_RATE_8000_96000,
+			.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
+				   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+			.channels_min = 2,
+			.channels_max = 8,
+			.rate_min = 8000,
+			.rate_max = 96000,
+		},
+		.capture = {
+			.rates = SNDRV_PCM_RATE_8000_48000,
+			.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
+				   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+			/* Supporting only stereo for I2S BT controller capture */
+			.channels_min = 2,
+			.channels_max = 2,
+			.rate_min = 8000,
+			.rate_max = 48000,
+		},
+	},
+
+	[I2S_TDM2_INSTANCE] = {
+		.id = I2S_TDM2_INSTANCE,
+		.name = "acp-sof-i2s2",
+		.playback = {
+			.rates = SNDRV_PCM_RATE_8000_96000,
+			.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
+				   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+			.channels_min = 2,
+			.channels_max = 8,
+			.rate_min = 8000,
+			.rate_max = 96000,
+		},
+		.capture = {
+			.rates = SNDRV_PCM_RATE_8000_48000,
+			.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
+				   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+			/* Supporting only stereo for I2S SP controller capture */
+			.channels_min = 2,
+			.channels_max = 2,
+			.rate_min = 8000,
+			.rate_max = 48000,
+		},
+	},
+
+	[PDM0_DMIC_INSTANCE] = {
+		.id = PDM0_DMIC_INSTANCE,
+		.name = "acp-sof-dmic0",
+		.capture = {
+			.rates = SNDRV_PCM_RATE_8000_48000,
+			.formats = SNDRV_PCM_FMTBIT_S32_LE,
+			.channels_min = 2,
+			.channels_max = 4,
+			.rate_min = 8000,
+			.rate_max = 48000,
+		},
+	},
+
+	[PDM1_DMIC_INSTANCE] = {
+		.id = PDM1_DMIC_INSTANCE,
+		.name = "acp-sof-dmic1",
+		.capture = {
+			.rates = SNDRV_PCM_RATE_8000_96000,
+			.formats = SNDRV_PCM_FMTBIT_S32_LE,
+			.channels_min = 2,
+			.channels_max = 4,
+			.rate_min = 8000,
+			.rate_max = 96000,
+		},
+	},
+};
+
+static int sof_acp7x_post_fw_run_delay(struct snd_sof_dev *sdev)
+{
+	/*
+	 * Resuming from suspend in some cases may cause the DSP firmware
+	 * to enter an unrecoverable faulty state. Delaying a bit any host
+	 * to DSP transmission right after firmware boot completion seems
+	 * to resolve the issue.
+	 */
+	if (!sdev->first_boot)
+		usleep_range(100, 150);
+
+	return 0;
+}
+
+struct snd_sof_dsp_ops sof_acp7x_ops;
+EXPORT_SYMBOL_NS(sof_acp7x_ops, "SND_SOC_SOF_AMD_COMMON");
+
+int sof_acp7x_ops_init(struct snd_sof_dev *sdev)
+{
+	struct acpi_device *adev = ACPI_COMPANION(&to_pci_dev(sdev->dev)->dev);
+	const union acpi_object *obj;
+	int acp_sof_signed_firmware_image = 0;
+	int acp_sof_post_fw_run_delay = 0;
+
+	/* common defaults */
+	memcpy(&sof_acp7x_ops, &sof_acp_common_ops, sizeof(struct snd_sof_dsp_ops));
+
+	sof_acp7x_ops.drv = acp7x_sof_dai;
+	sof_acp7x_ops.num_drv = ARRAY_SIZE(acp7x_sof_dai);
+	sof_acp7x_ops.probe = amd_sof_acp7x_probe;
+	sof_acp7x_ops.remove = amd_sof_acp7x_remove;
+
+	if (adev) {
+		if (!acpi_dev_get_property(adev, "acp-sof-signed-firmware-image",
+					   ACPI_TYPE_INTEGER, &obj))
+			acp_sof_signed_firmware_image = obj->integer.value;
+
+		if (!acpi_dev_get_property(adev, "acp-sof-post_fw_run_delay",
+					   ACPI_TYPE_INTEGER, &obj))
+			acp_sof_post_fw_run_delay = obj->integer.value;
+	}
+
+	if (acp_sof_signed_firmware_image)
+		sof_acp7x_ops.load_firmware = acp_sof_load_signed_firmware;
+
+	if (acp_sof_post_fw_run_delay)
+		sof_acp7x_ops.post_fw_run = sof_acp7x_post_fw_run_delay;
+
+	sof_acp7x_ops.suspend = amd_sof_acp7x_suspend;
+	sof_acp7x_ops.resume = amd_sof_acp7x_resume;
+	sof_acp7x_ops.runtime_suspend = amd_sof_acp7x_suspend_runtime;
+	sof_acp7x_ops.runtime_resume = amd_sof_acp7x_resume_runtime;
+
+	return 0;
+}
diff --git a/sound/soc/sof/amd/pci-acp7x.c b/sound/soc/sof/amd/pci-acp7x.c
new file mode 100644
index 0000000000000..532e153137959
--- /dev/null
+++ b/sound/soc/sof/amd/pci-acp7x.c
@@ -0,0 +1,116 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license. When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2025 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Authors: Vijendar Mukunda <Vijendar.Mukunda@amd.com>
+
+/*
+ * PCI interface for ACP7.B/7.F devices
+ */
+
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <sound/sof.h>
+#include <sound/soc-acpi.h>
+
+#include "../ops.h"
+#include "../sof-pci-dev.h"
+#include "../../amd/mach-config.h"
+#include "acp.h"
+#include "acp-dsp-offset.h"
+
+#define ACP7X_FUTURE_REG_ACLK_0		0x18e0
+#define ACP7X_REG_START			0x1240000
+#define ACP7X_REG_END			0x125C000
+
+static const struct sof_amd_acp_desc acp7x_chip_info = {
+	.name		= "acp7x",
+	.pgfsm_base	= ACP7X_PGFSM_BASE,
+	.ext_intr_enb	= ACP6X_EXTERNAL_INTR_ENB,
+	.ext_intr_cntl	= ACP7X_EXTERNAL_INTR_CNTL,
+	.ext_intr_stat	= ACP7X_EXT_INTR_STAT,
+	.ext_intr_stat1	= ACP7X_EXT_INTR_STAT1,
+	.dsp_intr_base	= ACP7X_DSP_SW_INTR_BASE,
+	.acp_error_stat	= ACP7X_ERROR_STATUS,
+	.sram_pte_offset = ACP7X_SRAM_PTE_OFFSET,
+	.hw_semaphore_offset = ACP7X_AXI2DAGB_SEM_0,
+	.fusion_dsp_offset = ACP7X_DSP_FUSION_RUNSTALL,
+	.probe_reg_offset = ACP7X_FUTURE_REG_ACLK_0,
+	.reg_start_addr	= ACP7X_REG_START,
+	.reg_end_addr	= ACP7X_REG_END,
+};
+
+static const struct sof_dev_desc acp7x_desc = {
+	.machines		= snd_soc_acpi_amd_acp7x_sof_machines,
+	.resindex_lpe_base	= 0,
+	.resindex_pcicfg_base	= -1,
+	.resindex_imr_base	= -1,
+	.irqindex_host_ipc	= -1,
+	.chip_info		= &acp7x_chip_info,
+	.ipc_supported_mask	= BIT(SOF_IPC_TYPE_3),
+	.ipc_default		= SOF_IPC_TYPE_3,
+	.default_fw_path	= {
+		[SOF_IPC_TYPE_3] = "amd/sof",
+	},
+	.default_tplg_path	= {
+		[SOF_IPC_TYPE_3] = "amd/sof-tplg",
+	},
+	.default_fw_filename	= {
+		[SOF_IPC_TYPE_3] = "sof-acp7x.ri",
+	},
+	.nocodec_tplg_filename	= "sof-acp.tplg",
+	.ops			= &sof_acp7x_ops,
+	.ops_init		= sof_acp7x_ops_init,
+};
+
+static int acp7x_pci_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
+{
+	unsigned int flag;
+
+	switch (pci->revision) {
+	case ACP7B_PCI_ID:
+	case ACP7F_PCI_ID:
+		break;
+	default:
+		return -ENODEV;
+	}
+
+	flag = snd_amd_acp_find_config(pci);
+	if (flag != FLAG_AMD_SOF && flag != FLAG_AMD_SOF_ONLY_DMIC)
+		return -ENODEV;
+
+	return sof_pci_probe(pci, pci_id);
+}
+
+static void acp7x_pci_remove(struct pci_dev *pci)
+{
+	sof_pci_remove(pci);
+}
+
+/* PCI IDs */
+static const struct pci_device_id acp7x_pci_ids[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, ACP_PCI_DEV_ID),
+	.driver_data = (unsigned long)&acp7x_desc},
+	{ 0, }
+};
+MODULE_DEVICE_TABLE(pci, acp7x_pci_ids);
+
+/* pci_driver definition */
+static struct pci_driver snd_sof_pci_amd_acp7x_driver = {
+	.name = KBUILD_MODNAME,
+	.id_table = acp7x_pci_ids,
+	.probe = acp7x_pci_probe,
+	.remove = acp7x_pci_remove,
+	.driver = {
+		.pm = pm_ptr(&sof_pci_pm),
+	},
+};
+module_pci_driver(snd_sof_pci_amd_acp7x_driver);
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION("ACP7X SOF Driver");
+MODULE_IMPORT_NS("SND_SOC_SOF_AMD_COMMON");
+MODULE_IMPORT_NS("SND_SOC_SOF_PCI_DEV");
diff --git a/sound/soc/sof/intel/hda-loader.c b/sound/soc/sof/intel/hda-loader.c
index 2b3abcf75d558..4347e71337f93 100644
--- a/sound/soc/sof/intel/hda-loader.c
+++ b/sound/soc/sof/intel/hda-loader.c
@@ -216,9 +216,10 @@ int hda_cl_trigger(struct device *dev, struct hdac_ext_stream *hext_stream, int
 EXPORT_SYMBOL_NS(hda_cl_trigger, "SND_SOC_SOF_INTEL_HDA_COMMON");
 
 int hda_cl_cleanup(struct device *dev, struct snd_dma_buffer *dmab,
-			  bool persistent_buffer, struct hdac_ext_stream *hext_stream)
+		   bool persistent_buffer, struct hdac_ext_stream *hext_stream, bool is_iccmax)
 {
-	return hda_data_stream_cleanup(dev, dmab, persistent_buffer, hext_stream, false);
+	return hda_data_stream_cleanup(dev, dmab, persistent_buffer, hext_stream,
+				       is_iccmax, false);
 }
 EXPORT_SYMBOL_NS(hda_cl_cleanup, "SND_SOC_SOF_INTEL_HDA_COMMON");
 
@@ -302,7 +303,7 @@ int hda_dsp_cl_boot_firmware_iccmax(struct snd_sof_dev *sdev)
 	 * If the cleanup also fails, we return the initial error
 	 */
 	ret1 = hda_cl_cleanup(sdev->dev, &hda->iccmax_dmab,
-			      persistent_cl_buffer, iccmax_stream);
+			      persistent_cl_buffer, iccmax_stream, true);
 	if (ret1 < 0) {
 		dev_err(sdev->dev, "error: ICCMAX stream cleanup failed\n");
 
@@ -458,7 +459,7 @@ int hda_dsp_cl_boot_firmware(struct snd_sof_dev *sdev)
 	 * If the cleanup also fails, we return the initial error
 	 */
 	ret1 = hda_cl_cleanup(sdev->dev, &hda->cl_dmab,
-			      persistent_cl_buffer, hext_stream);
+			      persistent_cl_buffer, hext_stream, false);
 	if (ret1 < 0) {
 		dev_err(sdev->dev, "error: Code loader DSP cleanup failed\n");
 
@@ -587,7 +588,7 @@ int hda_dsp_ipc4_load_library(struct snd_sof_dev *sdev,
 cleanup:
 	/* clean up even in case of error and return the first error */
 	ret1 = hda_cl_cleanup(sdev->dev, &hda->cl_dmab, persistent_cl_buffer,
-			      hext_stream);
+			      hext_stream, false);
 	if (ret1 < 0) {
 		dev_err(sdev->dev, "%s: Code loader DSP cleanup failed\n", __func__);
 
diff --git a/sound/soc/sof/intel/hda-sdw-bpt.c b/sound/soc/sof/intel/hda-sdw-bpt.c
index 728ffe7ae54d8..c10648f56f239 100644
--- a/sound/soc/sof/intel/hda-sdw-bpt.c
+++ b/sound/soc/sof/intel/hda-sdw-bpt.c
@@ -163,7 +163,7 @@ static int hda_sdw_bpt_dma_deprepare(struct device *dev, struct hdac_ext_stream
 	u32 mask;
 	int ret;
 
-	ret = hda_data_stream_cleanup(sdev->dev, dmab_bdl, false, sdw_bpt_stream, true);
+	ret = hda_data_stream_cleanup(sdev->dev, dmab_bdl, false, sdw_bpt_stream, false, true);
 	if (ret < 0) {
 		dev_err(sdev->dev, "%s: SDW BPT DMA cleanup failed\n",
 			__func__);
diff --git a/sound/soc/sof/intel/hda-stream.c b/sound/soc/sof/intel/hda-stream.c
index 5c1f3b427cdb8..0778002e2bd6b 100644
--- a/sound/soc/sof/intel/hda-stream.c
+++ b/sound/soc/sof/intel/hda-stream.c
@@ -198,13 +198,18 @@ int hda_dsp_stream_spib_config(struct snd_sof_dev *sdev,
 
 	mask = (1 << hstream->index);
 
+	/* Reset the spib_addr before disabling SPIB */
+	if (!enable)
+		sof_io_write(sdev, hstream->spib_addr, 0);
+
 	/* enable/disable SPIB for the stream */
 	snd_sof_dsp_update_bits(sdev, HDA_DSP_SPIB_BAR,
 				SOF_HDA_ADSP_REG_CL_SPBFIFO_SPBFCCTL, mask,
 				enable << hstream->index);
 
 	/* set the SPIB value */
-	sof_io_write(sdev, hstream->spib_addr, size);
+	if (enable)
+		sof_io_write(sdev, hstream->spib_addr, size);
 
 	return 0;
 }
@@ -1323,16 +1328,18 @@ hda_data_stream_prepare(struct device *dev, unsigned int format, unsigned int si
 EXPORT_SYMBOL_NS(hda_data_stream_prepare, "SND_SOC_SOF_INTEL_HDA_COMMON");
 
 int hda_data_stream_cleanup(struct device *dev, struct snd_dma_buffer *dmab,
-			    bool persistent_buffer, struct hdac_ext_stream *hext_stream, bool pair)
+			    bool persistent_buffer, struct hdac_ext_stream *hext_stream,
+			    bool is_iccmax, bool pair)
 {
 	struct snd_sof_dev *sdev =  dev_get_drvdata(dev);
 	struct hdac_stream *hstream = hdac_stream(hext_stream);
 	int sd_offset = SOF_STREAM_SD_OFFSET(hstream);
 	int ret = 0;
 
-	if (hstream->direction == SNDRV_PCM_STREAM_PLAYBACK)
+	if (!is_iccmax)
 		ret = hda_dsp_stream_spib_config(sdev, hext_stream, HDA_DSP_SPIB_DISABLE, 0);
-	else
+
+	if (hstream->direction == SNDRV_PCM_STREAM_CAPTURE)
 		snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, sd_offset,
 					SOF_HDA_SD_CTL_DMA_START, 0);
 
diff --git a/sound/soc/sof/intel/hda.h b/sound/soc/sof/intel/hda.h
index 3f0966477ace2..799e49539b4a6 100644
--- a/sound/soc/sof/intel/hda.h
+++ b/sound/soc/sof/intel/hda.h
@@ -739,7 +739,7 @@ struct hdac_ext_stream *hda_cl_prepare(struct device *dev, unsigned int format,
 int hda_cl_trigger(struct device *dev, struct hdac_ext_stream *hext_stream, int cmd);
 
 int hda_cl_cleanup(struct device *dev, struct snd_dma_buffer *dmab,
-		   bool persistent_buffer, struct hdac_ext_stream *hext_stream);
+		   bool persistent_buffer, struct hdac_ext_stream *hext_stream, bool is_iccmax);
 int cl_dsp_init(struct snd_sof_dev *sdev, int stream_tag, bool imr_boot);
 #define HDA_CL_STREAM_FORMAT 0x40
 
@@ -911,7 +911,8 @@ hda_data_stream_prepare(struct device *dev, unsigned int format, unsigned int si
 			bool is_iccmax, bool pair);
 
 int hda_data_stream_cleanup(struct device *dev, struct snd_dma_buffer *dmab,
-			    bool persistent_buffer, struct hdac_ext_stream *hext_stream, bool pair);
+			    bool persistent_buffer, struct hdac_ext_stream *hext_stream,
+			    bool is_iccmax, bool pair);
 
 /* common dai driver */
 extern struct snd_soc_dai_driver skl_dai[];
diff --git a/sound/soc/sof/ipc3-pcm.c b/sound/soc/sof/ipc3-pcm.c
index 90ef5d99f626a..143bf0fe8dd9d 100644
--- a/sound/soc/sof/ipc3-pcm.c
+++ b/sound/soc/sof/ipc3-pcm.c
@@ -421,6 +421,12 @@ static int sof_ipc3_pcm_dai_link_fixup(struct snd_soc_pcm_runtime *rtd,
 		dev_dbg(component->dev, "AMD_SDW channels_min: %d channels_max: %d\n",
 			channels->min, channels->max);
 		break;
+	case SOF_DAI_AMD_I2S:
+		rate->min = private->dai_config->acp_i2s.fsync_rate;
+		rate->max = private->dai_config->acp_i2s.fsync_rate;
+		channels->min = private->dai_config->acp_i2s.tdm_slots;
+		channels->max = private->dai_config->acp_i2s.tdm_slots;
+		break;
 	default:
 		dev_err(component->dev, "Invalid DAI type %d\n", private->dai_config->type);
 		break;
diff --git a/sound/soc/sof/ipc3-topology.c b/sound/soc/sof/ipc3-topology.c
index 4e066bbded917..26d85ca9be26a 100644
--- a/sound/soc/sof/ipc3-topology.c
+++ b/sound/soc/sof/ipc3-topology.c
@@ -281,7 +281,10 @@ static const struct sof_topology_token acpdmic_tokens[] = {
 		offsetof(struct sof_ipc_dai_acpdmic_params, pdm_ch)},
 };
 
-/* ACPI2S */
+/*
+ * ACPI2S tokens fill struct sof_ipc_dai_acp_params; SOF_DAI_AMD_I2S (ACPTDM
+ * on ACP7.B/7.F) reuses this tuple group rather than defining a parallel set.
+ */
 static const struct sof_topology_token acpi2s_tokens[] = {
 	{SOF_TKN_AMD_ACPI2S_RATE, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
 		offsetof(struct sof_ipc_dai_acp_params, fsync_rate)},
@@ -289,6 +292,8 @@ static const struct sof_topology_token acpi2s_tokens[] = {
 		offsetof(struct sof_ipc_dai_acp_params, tdm_slots)},
 	{SOF_TKN_AMD_ACPI2S_TDM_MODE, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
 		offsetof(struct sof_ipc_dai_acp_params, tdm_mode)},
+	{SOF_TKN_AMD_ACPI2S_FORMAT, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_dai_acp_params, format)},
 };
 
 /* MICFIL PDM */
@@ -1363,6 +1368,35 @@ static int sof_link_acp_sdw_load(struct snd_soc_component *scomp, struct snd_sof
 	return 0;
 }
 
+static int sof_link_acp_i2s_load(struct snd_soc_component *scomp, struct snd_sof_dai_link *slink,
+				 struct sof_ipc_dai_config *config, struct snd_sof_dai *dai)
+{
+	struct snd_soc_tplg_hw_config *hw_config = slink->hw_configs;
+	struct sof_dai_private_data *private = dai->private;
+	u32 size = sizeof(*config);
+	int ret;
+
+	/* handle master/slave and inverted clocks */
+	sof_dai_set_format(hw_config, config);
+
+	/* init IPC */
+	memset(&config->acp_i2s, 0, sizeof(config->acp_i2s));
+	config->hdr.size = size;
+
+	ret = sof_update_ipc_object(scomp, &config->acp_i2s, SOF_ACPI2S_TOKENS, slink->tuples,
+				    slink->num_tuples, size, slink->num_hw_configs);
+	if (ret < 0)
+		return ret;
+
+	dai->number_configs = 1;
+	dai->current_config = 0;
+	private->dai_config = kmemdup(config, size, GFP_KERNEL);
+	if (!private->dai_config)
+		return -ENOMEM;
+
+	return 0;
+}
+
 static int sof_link_afe_load(struct snd_soc_component *scomp, struct snd_sof_dai_link *slink,
 			     struct sof_ipc_dai_config *config, struct snd_sof_dai *dai)
 {
@@ -1692,6 +1726,9 @@ static int sof_ipc3_widget_setup_comp_dai(struct snd_sof_widget *swidget)
 		case SOF_DAI_AMD_SDW:
 			ret = sof_link_acp_sdw_load(scomp, slink, config, dai);
 			break;
+		case SOF_DAI_AMD_I2S:
+			ret = sof_link_acp_i2s_load(scomp, slink, config, dai);
+			break;
 		default:
 			break;
 		}
diff --git a/sound/soc/sof/ipc4-topology.c b/sound/soc/sof/ipc4-topology.c
index 95ad5266b0c66..8ac7dde32f77a 100644
--- a/sound/soc/sof/ipc4-topology.c
+++ b/sound/soc/sof/ipc4-topology.c
@@ -1127,6 +1127,7 @@ static int sof_ipc4_widget_setup_comp_src(struct snd_sof_widget *swidget)
 			"Invalid number of formats: input: %d, output: %d\n",
 			src->available_fmt.num_input_formats,
 			src->available_fmt.num_output_formats);
+		ret = -EINVAL;
 		goto err;
 	}
 
@@ -1179,6 +1180,7 @@ static int sof_ipc4_widget_setup_comp_asrc(struct snd_sof_widget *swidget)
 			"Invalid number of formats: input: %d, output: %d\n",
 			asrc->available_fmt.num_input_formats,
 			asrc->available_fmt.num_output_formats);
+		ret = -EINVAL;
 		goto err;
 	}
 
diff --git a/sound/soc/sof/nocodec.c b/sound/soc/sof/nocodec.c
index 11a95dba3c9cf..34744f8ae34a7 100644
--- a/sound/soc/sof/nocodec.c
+++ b/sound/soc/sof/nocodec.c
@@ -21,12 +21,13 @@ static struct snd_soc_card sof_nocodec_card = {
 static int sof_nocodec_bes_setup(struct device *dev,
 				 struct snd_soc_dai_driver *drv,
 				 struct snd_soc_dai_link *links,
-				 int link_num, struct snd_soc_card *card)
+				 int link_num)
 {
+	struct snd_soc_card *card = &sof_nocodec_card;
 	struct snd_soc_dai_link_component *dlc;
 	int i;
 
-	if (!drv || !links || !card)
+	if (!drv || !links)
 		return -EINVAL;
 
 	/* set up BE dai_links */
@@ -67,10 +68,10 @@ static int sof_nocodec_bes_setup(struct device *dev,
 	return 0;
 }
 
-static int sof_nocodec_setup(struct device *dev,
-			     u32 num_dai_drivers,
-			     struct snd_soc_dai_driver *dai_drivers)
+static int sof_nocodec_setup(struct device *dev, struct snd_soc_acpi_mach *mach)
 {
+	u32 num_dai_drivers = mach->mach_params.num_dai_drivers;
+	struct snd_soc_dai_driver *dai_drivers = mach->mach_params.dai_drivers;
 	struct snd_soc_dai_link *links;
 
 	/* create dummy BE dai_links */
@@ -78,7 +79,7 @@ static int sof_nocodec_setup(struct device *dev,
 	if (!links)
 		return -ENOMEM;
 
-	return sof_nocodec_bes_setup(dev, dai_drivers, links, num_dai_drivers, &sof_nocodec_card);
+	return sof_nocodec_bes_setup(dev, dai_drivers, links, num_dai_drivers);
 }
 
 static int sof_nocodec_probe(struct platform_device *pdev)
@@ -92,8 +93,7 @@ static int sof_nocodec_probe(struct platform_device *pdev)
 
 	snd_soc_card_set_topology_name(card, "sof");
 
-	ret = sof_nocodec_setup(card->dev, mach->mach_params.num_dai_drivers,
-				mach->mach_params.dai_drivers);
+	ret = sof_nocodec_setup(card->dev, mach);
 	if (ret < 0)
 		return ret;
 
diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c
index 42a2d90bb705d..6fd69ba11c410 100644
--- a/sound/soc/sof/topology.c
+++ b/sound/soc/sof/topology.c
@@ -309,7 +309,7 @@ static const struct sof_dai_types sof_dais[] = {
 	{"ACPHS_VIRTUAL", SOF_DAI_AMD_HS_VIRTUAL},
 	{"MICFIL", SOF_DAI_IMX_MICFIL},
 	{"ACP_SDW", SOF_DAI_AMD_SDW},
-
+	{"ACPTDM", SOF_DAI_AMD_I2S},
 };
 
 static enum sof_ipc_dai_type find_dai(const char *name)
@@ -1102,7 +1102,7 @@ static int sof_connect_dai_widget(struct snd_soc_component *scomp,
 
 	full = NULL;
 	partial = NULL;
-	list_for_each_entry(rtd, &card->rtd_list, list) {
+	for_each_card_rtds(card, rtd) {
 		/* does stream match DAI link ? */
 		if (rtd->dai_link->stream_name) {
 			if (!strcmp(rtd->dai_link->stream_name, w->sname)) {
@@ -1167,7 +1167,7 @@ static void sof_disconnect_dai_widget(struct snd_soc_component *scomp,
 	else
 		return;
 
-	list_for_each_entry(rtd, &card->rtd_list, list) {
+	for_each_card_rtds(card, rtd) {
 		/* does stream match DAI link ? */
 		if (!rtd->dai_link->stream_name ||
 		    !strstr(rtd->dai_link->stream_name, sname))
@@ -1943,8 +1943,7 @@ static int sof_link_load(struct snd_soc_component *scomp, int index, struct snd_
 			       private->array, le32_to_cpu(private->size));
 	if (ret < 0) {
 		dev_err(scomp->dev, "Failed tp parse common DAI link tokens\n");
-		kfree(slink);
-		return ret;
+		goto free_slink;
 	}
 
 	token_list = tplg_ops ? tplg_ops->token_list : NULL;
@@ -1995,6 +1994,7 @@ static int sof_link_load(struct snd_soc_component *scomp, int index, struct snd_
 	case SOF_DAI_AMD_HS:
 	case SOF_DAI_AMD_SP_VIRTUAL:
 	case SOF_DAI_AMD_HS_VIRTUAL:
+	case SOF_DAI_AMD_I2S:
 		token_id = SOF_ACPI2S_TOKENS;
 		num_tuples += token_list[SOF_ACPI2S_TOKENS].count;
 		break;
@@ -2013,8 +2013,8 @@ static int sof_link_load(struct snd_soc_component *scomp, int index, struct snd_
 	/* allocate memory for tuples array */
 	slink->tuples = kzalloc_objs(*slink->tuples, num_tuples);
 	if (!slink->tuples) {
-		kfree(slink);
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto free_slink;
 	}
 
 	if (token_list[SOF_DAI_LINK_TOKENS].tokens) {
@@ -2070,6 +2070,7 @@ static int sof_link_load(struct snd_soc_component *scomp, int index, struct snd_
 
 err:
 	kfree(slink->tuples);
+free_slink:
 	kfree(slink);
 
 	return ret;
diff --git a/sound/soc/spacemit/k1_i2s.c b/sound/soc/spacemit/k1_i2s.c
index 8871fc15b29cc..cd461f2aa7567 100644
--- a/sound/soc/spacemit/k1_i2s.c
+++ b/sound/soc/spacemit/k1_i2s.c
@@ -335,6 +335,11 @@ static int spacemit_i2s_dai_remove(struct snd_soc_dai *dai)
 	return 0;
 }
 
+static const u64 spacemit_i2s_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B;
+
 static const struct snd_soc_dai_ops spacemit_i2s_dai_ops = {
 	.probe = spacemit_i2s_dai_probe,
 	.remove = spacemit_i2s_dai_remove,
@@ -343,6 +348,8 @@ static const struct snd_soc_dai_ops spacemit_i2s_dai_ops = {
 	.set_sysclk = spacemit_i2s_set_sysclk,
 	.set_fmt = spacemit_i2s_set_fmt,
 	.trigger = spacemit_i2s_trigger,
+	.auto_selectable_formats = &spacemit_i2s_selectable_formats,
+	.num_auto_selectable_formats = 1,
 };
 
 static struct snd_soc_dai_driver spacemit_i2s_dai = {
diff --git a/sound/soc/stm/stm32_adfsdm.c b/sound/soc/stm/stm32_adfsdm.c
index 66efb9a0acb9e..c8022f6efd8a6 100644
--- a/sound/soc/stm/stm32_adfsdm.c
+++ b/sound/soc/stm/stm32_adfsdm.c
@@ -392,3 +392,4 @@ MODULE_DESCRIPTION("stm32 DFSDM DAI driver");
 MODULE_AUTHOR("Arnaud Pouliquen <arnaud.pouliquen@st.com>");
 MODULE_LICENSE("GPL v2");
 MODULE_ALIAS("platform:" STM32_ADFSDM_DRV_NAME);
+MODULE_IMPORT_NS("IIO_CONSUMER");
diff --git a/sound/soc/sunxi/sun4i-codec.c b/sound/soc/sunxi/sun4i-codec.c
index f4e22af594fa3..2c5cf12f1a6c4 100644
--- a/sound/soc/sunxi/sun4i-codec.c
+++ b/sound/soc/sunxi/sun4i-codec.c
@@ -22,13 +22,13 @@
 #include <linux/gpio/consumer.h>
 
 #include <sound/core.h>
+#include <sound/dmaengine_pcm.h>
+#include <sound/initval.h>
 #include <sound/jack.h>
 #include <sound/pcm.h>
 #include <sound/pcm_params.h>
 #include <sound/soc.h>
 #include <sound/tlv.h>
-#include <sound/initval.h>
-#include <sound/dmaengine_pcm.h>
 
 /* Codec DAC digital controls and FIFO registers */
 #define SUN4I_CODEC_DAC_DPC			(0x00)
@@ -2237,6 +2237,7 @@ static const struct sun4i_codec_quirks sun50i_h616_codec_quirks = {
 	.reg_dac_fifoc	= REG_FIELD(SUN50I_H616_CODEC_DAC_FIFOC, 0, 31),
 	.reg_dac_txdata	= SUN8I_H3_CODEC_DAC_TXDATA,
 	.has_reset	= true,
+	.playback_only	= true,
 	.dma_max_burst	= SUN4I_DMA_MAX_BURST,
 };
 
diff --git a/sound/soc/tegra/tegra20_das.c b/sound/soc/tegra/tegra20_das.c
index b48cc4a6967b8..ea8bc9bcfa673 100644
--- a/sound/soc/tegra/tegra20_das.c
+++ b/sound/soc/tegra/tegra20_das.c
@@ -167,10 +167,9 @@ static int tegra20_das_probe(struct platform_device *pdev)
 
 	das->regmap = devm_regmap_init_mmio(&pdev->dev, regs,
 					    &tegra20_das_regmap_config);
-	if (IS_ERR(das->regmap)) {
-		dev_err(&pdev->dev, "regmap init failed\n");
-		return PTR_ERR(das->regmap);
-	}
+	if (IS_ERR(das->regmap))
+		return dev_err_probe(&pdev->dev, PTR_ERR(das->regmap),
+				     "regmap init failed\n");
 
 	tegra20_das_connect_dap_to_dac(das, TEGRA20_DAS_DAP_ID_1,
 				       TEGRA20_DAS_DAP_SEL_DAC1);
diff --git a/sound/soc/tegra/tegra210_adx.c b/sound/soc/tegra/tegra210_adx.c
index 15a94196ee1ac..e75bf50f8fb02 100644
--- a/sound/soc/tegra/tegra210_adx.c
+++ b/sound/soc/tegra/tegra210_adx.c
@@ -496,16 +496,12 @@ static struct snd_kcontrol_new tegra264_adx_controls[] = {
 static int tegra210_adx_component_probe(struct snd_soc_component *component)
 {
 	struct tegra210_adx *adx = snd_soc_component_get_drvdata(component);
-	int err = 0;
 
-	if (adx->soc_data->num_controls) {
-		err = snd_soc_add_component_controls(component, adx->soc_data->controls,
-						     adx->soc_data->num_controls);
-		if (err)
-			dev_err(component->dev, "can't add ADX controls, err: %d\n", err);
-	}
+	if (adx->soc_data->num_controls)
+		return snd_soc_add_component_controls(component, adx->soc_data->controls,
+						      adx->soc_data->num_controls);
 
-	return err;
+	return 0;
 }
 
 static const struct snd_soc_component_driver tegra210_adx_cmpnt = {
diff --git a/sound/soc/tegra/tegra210_amx.c b/sound/soc/tegra/tegra210_amx.c
index cc1f9c1581911..6a67ce6128b4f 100644
--- a/sound/soc/tegra/tegra210_amx.c
+++ b/sound/soc/tegra/tegra210_amx.c
@@ -504,16 +504,12 @@ static struct snd_kcontrol_new tegra264_amx_controls[] = {
 static int tegra210_amx_component_probe(struct snd_soc_component *component)
 {
 	struct tegra210_amx *amx = snd_soc_component_get_drvdata(component);
-	int err = 0;
 
-	if (amx->soc_data->num_controls) {
-		err = snd_soc_add_component_controls(component, amx->soc_data->controls,
-						     amx->soc_data->num_controls);
-		if (err)
-			dev_err(component->dev, "can't add AMX controls, err: %d\n", err);
-	}
+	if (amx->soc_data->num_controls)
+		return snd_soc_add_component_controls(component, amx->soc_data->controls,
+						      amx->soc_data->num_controls);
 
-	return err;
+	return 0;
 }
 
 static const struct snd_soc_component_driver tegra210_amx_cmpnt = {
diff --git a/sound/soc/tegra/tegra210_i2s.c b/sound/soc/tegra/tegra210_i2s.c
index ff8c72fc38c5c..e7e26e9172916 100644
--- a/sound/soc/tegra/tegra210_i2s.c
+++ b/sound/soc/tegra/tegra210_i2s.c
@@ -1078,13 +1078,13 @@ static int tegra210_i2s_probe(struct platform_device *pdev)
 				     "can't retrieve I2S bit clock\n");
 
 	/*
-	 * Not an error, as this clock is needed only when some other I/O
-	 * requires input clock from current I2S instance, which is
-	 * configurable from DT.
+	 * This clock is optional and is only needed when another I/O uses
+	 * the current I2S instance as its input clock, as configured in DT.
 	 */
-	i2s->clk_sync_input = devm_clk_get(dev, "sync_input");
+	i2s->clk_sync_input = devm_clk_get_optional(dev, "sync_input");
 	if (IS_ERR(i2s->clk_sync_input))
-		dev_dbg(dev, "can't retrieve I2S sync input clock\n");
+		return dev_err_probe(dev, PTR_ERR(i2s->clk_sync_input),
+				     "can't retrieve I2S sync input clock\n");
 
 	regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(regs))
diff --git a/sound/soc/ti/j721e-evm.c b/sound/soc/ti/j721e-evm.c
index c214ae0d7b95e..312298e0b0049 100644
--- a/sound/soc/ti/j721e-evm.c
+++ b/sound/soc/ti/j721e-evm.c
@@ -4,6 +4,7 @@
  *  Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
  */
 
+#include <linux/cleanup.h>
 #include <linux/clk.h>
 #include <linux/module.h>
 #include <linux/of.h>
@@ -263,7 +264,7 @@ static int j721e_audio_startup(struct snd_pcm_substream *substream)
 	int ret = 0;
 	int i;
 
-	guard(mutex)(&priv->mutex);
+	mutex_lock(&priv->mutex);
 
 	domain->active++;
 
@@ -303,6 +304,7 @@ static int j721e_audio_startup(struct snd_pcm_substream *substream)
 out:
 	if (ret)
 		domain->active--;
+	mutex_unlock(&priv->mutex);
 
 	return ret;
 }
diff --git a/sound/soc/ux500/ux500_msp_dai.c b/sound/soc/ux500/ux500_msp_dai.c
index 7798957c6504d..499e826d71208 100644
--- a/sound/soc/ux500/ux500_msp_dai.c
+++ b/sound/soc/ux500/ux500_msp_dai.c
@@ -14,7 +14,7 @@
 #include <linux/clk.h>
 #include <linux/of.h>
 #include <linux/regulator/consumer.h>
-#include <linux/mfd/dbx500-prcmu.h>
+#include <linux/mfd/db8500-prcmu.h>
 
 #include <sound/soc.h>
 #include <sound/soc-dai.h>
diff --git a/sound/soc/xilinx/xlnx_i2s.c b/sound/soc/xilinx/xlnx_i2s.c
index ca915a001ad5f..0676da122eddf 100644
--- a/sound/soc/xilinx/xlnx_i2s.c
+++ b/sound/soc/xilinx/xlnx_i2s.c
@@ -185,17 +185,15 @@ static int xlnx_i2s_probe(struct platform_device *pdev)
 		return PTR_ERR(drv_data->base);
 
 	ret = of_property_read_u32(node, "xlnx,num-channels", &drv_data->channels);
-	if (ret < 0) {
-		dev_err(dev, "cannot get supported channels\n");
-		return ret;
-	}
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "cannot get supported channels\n");
+
 	drv_data->channels *= 2;
 
 	ret = of_property_read_u32(node, "xlnx,dwidth", &drv_data->data_width);
-	if (ret < 0) {
-		dev_err(dev, "cannot get data width\n");
-		return ret;
-	}
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "cannot get data width\n");
+
 	switch (drv_data->data_width) {
 	case 16:
 		format = SNDRV_PCM_FMTBIT_S16_LE;
@@ -233,10 +231,8 @@ static int xlnx_i2s_probe(struct platform_device *pdev)
 
 	ret = devm_snd_soc_register_component(&pdev->dev, &xlnx_i2s_component,
 					      &drv_data->dai_drv, 1);
-	if (ret) {
-		dev_err(&pdev->dev, "i2s component registration failed\n");
+	if (ret)
 		return ret;
-	}
 
 	dev_info(&pdev->dev, "%s DAI registered\n", drv_data->dai_drv.name);
 
diff --git a/sound/soc/xilinx/xlnx_spdif.c b/sound/soc/xilinx/xlnx_spdif.c
index 017a64ab9f1eb..ae05818ba0643 100644
--- a/sound/soc/xilinx/xlnx_spdif.c
+++ b/sound/soc/xilinx/xlnx_spdif.c
@@ -249,21 +249,18 @@ static int xlnx_spdif_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	ctx->axi_clk = devm_clk_get_enabled(dev, "s_axi_aclk");
-	if (IS_ERR(ctx->axi_clk)) {
-		ret = PTR_ERR(ctx->axi_clk);
-		dev_err(dev, "failed to get s_axi_aclk(%d)\n", ret);
-		return ret;
-	}
+	if (IS_ERR(ctx->axi_clk))
+		return dev_err_probe(dev, PTR_ERR(ctx->axi_clk),
+				     "failed to get s_axi_aclk\n");
 
 	ctx->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(ctx->base))
 		return PTR_ERR(ctx->base);
 
 	ret = of_property_read_u32(node, "xlnx,spdif-mode", &ctx->mode);
-	if (ret < 0) {
-		dev_err(dev, "cannot get SPDIF mode\n");
-		return ret;
-	}
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "cannot get SPDIF mode\n");
+
 	if (ctx->mode) {
 		dai_drv = &xlnx_spdif_tx_dai;
 	} else {
@@ -274,29 +271,23 @@ static int xlnx_spdif_probe(struct platform_device *pdev)
 		ret = devm_request_irq(dev, ret,
 				       xlnx_spdifrx_irq_handler,
 				       0, "XLNX_SPDIF_RX", ctx);
-		if (ret) {
-			dev_err(dev, "spdif rx irq request failed\n");
-			return -ENODEV;
-		}
+		if (ret)
+			return ret;
 
 		init_waitqueue_head(&ctx->chsts_q);
 		dai_drv = &xlnx_spdif_rx_dai;
 	}
 
 	ret = of_property_read_u32(node, "xlnx,aud_clk_i", &ctx->aclk);
-	if (ret < 0) {
-		dev_err(dev, "cannot get aud_clk_i value\n");
-		return ret;
-	}
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "cannot get aud_clk_i value\n");
 
 	dev_set_drvdata(dev, ctx);
 
 	ret = devm_snd_soc_register_component(dev, &xlnx_spdif_component,
 					      dai_drv, 1);
-	if (ret) {
-		dev_err(dev, "SPDIF component registration failed\n");
+	if (ret)
 		return ret;
-	}
 
 	writel(XSPDIF_SOFT_RESET_VALUE, ctx->base + XSPDIF_SOFT_RESET_REG);
 	dev_info(dev, "%s DAI registered\n", dai_drv->name);
diff --git a/sound/soc/xtensa/xtfpga-i2s.c b/sound/soc/xtensa/xtfpga-i2s.c
index 9ad86c54e3ea1..fd0990201b40c 100644
--- a/sound/soc/xtensa/xtfpga-i2s.c
+++ b/sound/soc/xtensa/xtfpga-i2s.c
@@ -533,34 +533,25 @@ static int xtfpga_i2s_probe(struct platform_device *pdev)
 	int err, irq;
 
 	i2s = devm_kzalloc(&pdev->dev, sizeof(*i2s), GFP_KERNEL);
-	if (!i2s) {
-		err = -ENOMEM;
-		goto err;
-	}
+	if (!i2s)
+		return -ENOMEM;
+
 	platform_set_drvdata(pdev, i2s);
 	i2s->dev = &pdev->dev;
 	dev_dbg(&pdev->dev, "dev: %p, i2s: %p\n", &pdev->dev, i2s);
 
 	i2s->regs = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(i2s->regs)) {
-		err = PTR_ERR(i2s->regs);
-		goto err;
-	}
+	if (IS_ERR(i2s->regs))
+		return PTR_ERR(i2s->regs);
 
 	i2s->regmap = devm_regmap_init_mmio(&pdev->dev, i2s->regs,
 					    &xtfpga_i2s_regmap_config);
-	if (IS_ERR(i2s->regmap)) {
-		dev_err(&pdev->dev, "regmap init failed\n");
-		err = PTR_ERR(i2s->regmap);
-		goto err;
-	}
+	if (IS_ERR(i2s->regmap))
+		return dev_err_probe(&pdev->dev, PTR_ERR(i2s->regmap), "regmap init failed\n");
 
 	i2s->clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(i2s->clk)) {
-		dev_err(&pdev->dev, "couldn't get clock\n");
-		err = PTR_ERR(i2s->clk);
-		goto err;
-	}
+	if (IS_ERR(i2s->clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(i2s->clk), "couldn't get clock\n");
 
 	regmap_write(i2s->regmap, XTFPGA_I2S_CONFIG,
 		     (0x1 << XTFPGA_I2S_CONFIG_CHANNEL_BASE));
@@ -568,41 +559,34 @@ static int xtfpga_i2s_probe(struct platform_device *pdev)
 	regmap_write(i2s->regmap, XTFPGA_I2S_INT_MASK, XTFPGA_I2S_INT_UNDERRUN);
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		err = irq;
-		goto err;
-	}
+	if (irq < 0)
+		return irq;
+
 	err = devm_request_threaded_irq(&pdev->dev, irq, NULL,
 					xtfpga_i2s_threaded_irq_handler,
 					IRQF_SHARED | IRQF_ONESHOT,
 					pdev->name, i2s);
-	if (err < 0) {
-		dev_err(&pdev->dev, "request_irq failed\n");
-		goto err;
-	}
+	if (err < 0)
+		return err;
 
 	err = devm_snd_soc_register_component(&pdev->dev,
 					      &xtfpga_i2s_component,
 					      xtfpga_i2s_dai,
 					      ARRAY_SIZE(xtfpga_i2s_dai));
-	if (err < 0) {
-		dev_err(&pdev->dev, "couldn't register component\n");
-		goto err;
-	}
+	if (err < 0)
+		return err;
+
 
 	pm_runtime_enable(&pdev->dev);
 	if (!pm_runtime_enabled(&pdev->dev)) {
 		err = xtfpga_i2s_runtime_resume(&pdev->dev);
-		if (err)
-			goto err_pm_disable;
+		if (err) {
+			pm_runtime_disable(&pdev->dev);
+			return err;
+		}
 	}
-	return 0;
 
-err_pm_disable:
-	pm_runtime_disable(&pdev->dev);
-err:
-	dev_err(&pdev->dev, "%s: err = %d\n", __func__, err);
-	return err;
+	return 0;
 }
 
 static void xtfpga_i2s_remove(struct platform_device *pdev)
diff --git a/sound/usb/caiaq/device.c b/sound/usb/caiaq/device.c
index b20aae0caf60a..a16e592484803 100644
--- a/sound/usb/caiaq/device.c
+++ b/sound/usb/caiaq/device.c
@@ -134,14 +134,22 @@ static void usb_ep1_command_reply_dispatch (struct urb* urb)
 	struct device *dev = &urb->dev->dev;
 	struct snd_usb_caiaqdev *cdev = urb->context;
 	unsigned char *buf = urb->transfer_buffer;
+	unsigned int payload_len;
+	unsigned int copy_len;
 
 	if (urb->status || !cdev) {
 		dev_warn(dev, "received EP1 urb->status = %i\n", urb->status);
 		return;
 	}
+	if (urb->actual_length < 1)
+		return;
+
+	payload_len = urb->actual_length - 1;
 
 	switch(buf[0]) {
 	case EP1_CMD_GET_DEVICE_INFO:
+		if (payload_len < sizeof(struct caiaq_device_spec))
+			break;
 	 	memcpy(&cdev->spec, buf+1, sizeof(struct caiaq_device_spec));
 		cdev->spec.fw_version = le16_to_cpu(cdev->spec.fw_version);
 		dev_dbg(dev, "device spec (firmware %d): audio: %d in, %d out, "
@@ -157,18 +165,21 @@ static void usb_ep1_command_reply_dispatch (struct urb* urb)
 		wake_up(&cdev->ep1_wait_queue);
 		break;
 	case EP1_CMD_AUDIO_PARAMS:
+		if (payload_len < 1)
+			break;
 		cdev->audio_parm_answer = buf[1];
 		wake_up(&cdev->ep1_wait_queue);
 		break;
 	case EP1_CMD_MIDI_READ:
+		if (urb->actual_length < 3 || urb->actual_length - 3 < buf[2])
+			break;
 		snd_usb_caiaq_midi_handle_input(cdev, buf[1], buf + 3, buf[2]);
 		break;
 	case EP1_CMD_READ_IO:
 		if (cdev->chip.usb_id ==
 			USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_AUDIO8DJ)) {
-			if (urb->actual_length > sizeof(cdev->control_state))
-				urb->actual_length = sizeof(cdev->control_state);
-			memcpy(cdev->control_state, buf + 1, urb->actual_length);
+			copy_len = min_t(unsigned int, payload_len, sizeof(cdev->control_state));
+			memcpy(cdev->control_state, buf + 1, copy_len);
 			wake_up(&cdev->ep1_wait_queue);
 			break;
 		}
diff --git a/sound/usb/caiaq/input.c b/sound/usb/caiaq/input.c
index eabbf41fdfb2b..8d924330c54c8 100644
--- a/sound/usb/caiaq/input.c
+++ b/sound/usb/caiaq/input.c
@@ -203,6 +203,8 @@ static void snd_caiaq_input_read_analog(struct snd_usb_caiaqdev *cdev,
 
 	switch (cdev->chip.usb_id) {
 	case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_RIGKONTROL2):
+		if (len < 6)
+			return;
 		snd_caiaq_input_report_abs(cdev, ABS_X, buf, 2);
 		snd_caiaq_input_report_abs(cdev, ABS_Y, buf, 0);
 		snd_caiaq_input_report_abs(cdev, ABS_Z, buf, 1);
@@ -210,11 +212,15 @@ static void snd_caiaq_input_read_analog(struct snd_usb_caiaqdev *cdev,
 	case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_RIGKONTROL3):
 	case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_KORECONTROLLER):
 	case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_KORECONTROLLER2):
+		if (len < 6)
+			return;
 		snd_caiaq_input_report_abs(cdev, ABS_X, buf, 0);
 		snd_caiaq_input_report_abs(cdev, ABS_Y, buf, 1);
 		snd_caiaq_input_report_abs(cdev, ABS_Z, buf, 2);
 		break;
 	case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_TRAKTORKONTROLX1):
+		if (len < 16)
+			return;
 		snd_caiaq_input_report_abs(cdev, ABS_HAT0X, buf, 4);
 		snd_caiaq_input_report_abs(cdev, ABS_HAT0Y, buf, 2);
 		snd_caiaq_input_report_abs(cdev, ABS_HAT1X, buf, 6);
diff --git a/sound/usb/card.c b/sound/usb/card.c
index 6a3b576fb0679..b36f513dccb9a 100644
--- a/sound/usb/card.c
+++ b/sound/usb/card.c
@@ -905,6 +905,40 @@ static int try_to_register_card(struct snd_usb_audio *chip, int ifnum)
 	return 0;
 }
 
+static void usb_audio_disconnect_components(struct snd_usb_audio *chip)
+{
+	struct snd_usb_stream *as;
+	struct snd_usb_endpoint *ep;
+	struct usb_mixer_interface *mixer;
+	struct list_head *p;
+
+	/* release the pcm resources */
+	list_for_each_entry(as, &chip->pcm_list, list) {
+		snd_usb_stream_disconnect(as);
+	}
+	/* release the endpoint resources */
+	list_for_each_entry(ep, &chip->ep_list, list) {
+		snd_usb_endpoint_release(ep);
+	}
+	/* release the midi resources */
+	list_for_each(p, &chip->midi_list) {
+		snd_usbmidi_disconnect(p);
+	}
+	snd_usb_midi_v2_disconnect_all(chip);
+	/*
+	 * Nice to check quirk && quirk->shares_media_device and
+	 * then call the snd_media_device_delete(). Don't have
+	 * access to the quirk here. snd_media_device_delete()
+	 * accesses mixer_list
+	 */
+	snd_media_device_delete(chip);
+
+	/* release mixer resources */
+	list_for_each_entry(mixer, &chip->mixer_list, list) {
+		snd_usb_mixer_disconnect(mixer);
+	}
+}
+
 /*
  * probe the active usb device
  *
@@ -1077,8 +1111,10 @@ static int usb_audio_probe(struct usb_interface *intf,
 		 * decrement before memory is possibly returned.
 		 */
 		atomic_dec(&chip->active);
-		if (!chip->num_interfaces)
+		if (!chip->num_interfaces) {
+			usb_audio_disconnect_components(chip);
 			snd_card_free(chip->card);
+		}
 	}
 	return err;
 }
@@ -1091,48 +1127,18 @@ static bool __usb_audio_disconnect(struct usb_interface *intf,
 				   struct snd_usb_audio *chip,
 				   struct snd_card *card)
 {
-	struct list_head *p;
-
 	guard(mutex)(&register_mutex);
 
 	if (platform_ops && platform_ops->disconnect_cb)
 		platform_ops->disconnect_cb(chip);
 
 	if (atomic_inc_return(&chip->shutdown) == 1) {
-		struct snd_usb_stream *as;
-		struct snd_usb_endpoint *ep;
-		struct usb_mixer_interface *mixer;
-
 		/* wait until all pending tasks done;
 		 * they are protected by snd_usb_lock_shutdown()
 		 */
 		snd_refcount_sync(&chip->usage_count);
 		snd_card_disconnect(card);
-		/* release the pcm resources */
-		list_for_each_entry(as, &chip->pcm_list, list) {
-			snd_usb_stream_disconnect(as);
-		}
-		/* release the endpoint resources */
-		list_for_each_entry(ep, &chip->ep_list, list) {
-			snd_usb_endpoint_release(ep);
-		}
-		/* release the midi resources */
-		list_for_each(p, &chip->midi_list) {
-			snd_usbmidi_disconnect(p);
-		}
-		snd_usb_midi_v2_disconnect_all(chip);
-		/*
-		 * Nice to check quirk && quirk->shares_media_device and
-		 * then call the snd_media_device_delete(). Don't have
-		 * access to the quirk here. snd_media_device_delete()
-		 * accesses mixer_list
-		 */
-		snd_media_device_delete(chip);
-
-		/* release mixer resources */
-		list_for_each_entry(mixer, &chip->mixer_list, list) {
-			snd_usb_mixer_disconnect(mixer);
-		}
+		usb_audio_disconnect_components(chip);
 	}
 
 	if (chip->quirk_flags & QUIRK_FLAG_DISABLE_AUTOSUSPEND)
diff --git a/sound/usb/mixer.c b/sound/usb/mixer.c
index b4c855c25eef4..703c118f9d4ed 100644
--- a/sound/usb/mixer.c
+++ b/sound/usb/mixer.c
@@ -1473,8 +1473,8 @@ static int get_min_max_with_quirks(struct usb_mixer_elem_info *cval,
 	if (!cval->cmask) {
 		snd_usb_set_cur_mix_value(cval, 0, 0, cval->max);
 	} else {
+		idx = 0;
 		for (i = 0; i < MAX_CHANNELS; i++) {
-			idx = 0;
 			if (cval->cmask & BIT(i)) {
 				snd_usb_set_cur_mix_value(cval, i + 1, idx, cval->max);
 				idx++;
diff --git a/sound/usb/mixer_quirks.c b/sound/usb/mixer_quirks.c
index 10792d26fa94a..fb3bd360e5a94 100644
--- a/sound/usb/mixer_quirks.c
+++ b/sound/usb/mixer_quirks.c
@@ -3920,6 +3920,7 @@ static int snd_rme_digiface_controls_create(struct usb_mixer_interface *mixer)
 #define SND_DJM_450_IDX		0x5
 #define SND_DJM_A9_IDX		0x6
 #define SND_DJM_V10_IDX	0x7
+#define SND_DJM_S11_IDX	0x8
 
 #define SND_DJM_CTL(_name, suffix, _default_value, _windex) { \
 	.name = _name, \
@@ -4260,6 +4261,21 @@ static const struct snd_djm_ctl snd_djm_ctls_v10[] = {
 	// playback channels are fixed and controlled by hardware knobs on the mixer
 };
 
+// DJM-S11
+static const u16 snd_djm_opts_s11_cap1[] = {
+	0x0100, 0x0103, 0x0106, 0x0107, 0x0108, 0x0109, 0x010d };
+static const u16 snd_djm_opts_s11_cap2[] = {
+	0x0200, 0x0203, 0x0206, 0x0207, 0x0208, 0x0209, 0x020d };
+static const u16 snd_djm_opts_s11_cap3[] = {
+	0x0307, 0x0308, 0x0309, 0x030a, 0x030d, 0x0311, 0x0312 };
+
+static const struct snd_djm_ctl snd_djm_ctls_s11[] = {
+	SND_DJM_CTL("Master Input Level Capture Switch", cap_level, 0, SND_DJM_WINDEX_CAPLVL),
+	SND_DJM_CTL("Input 1 Capture Switch", s11_cap1, 1, SND_DJM_WINDEX_CAP),
+	SND_DJM_CTL("Input 2 Capture Switch", s11_cap2, 1, SND_DJM_WINDEX_CAP),
+	SND_DJM_CTL("Input 3 Capture Switch", s11_cap3, 3, SND_DJM_WINDEX_CAP)
+};
+
 static const struct snd_djm_device snd_djm_devices[] = {
 	[SND_DJM_250MK2_IDX] = SND_DJM_DEVICE(250mk2),
 	[SND_DJM_750_IDX] = SND_DJM_DEVICE(750),
@@ -4269,6 +4285,7 @@ static const struct snd_djm_device snd_djm_devices[] = {
 	[SND_DJM_450_IDX] = SND_DJM_DEVICE(450),
 	[SND_DJM_A9_IDX] = SND_DJM_DEVICE(a9),
 	[SND_DJM_V10_IDX] = SND_DJM_DEVICE(v10),
+	[SND_DJM_S11_IDX] = SND_DJM_DEVICE(s11),
 };
 
 static int snd_djm_controls_info(struct snd_kcontrol *kctl,
@@ -4573,6 +4590,9 @@ int snd_usb_mixer_apply_create_quirk(struct usb_mixer_interface *mixer)
 	case USB_ID(0x2b73, 0x0034): /* Pioneer DJ DJM-V10 */
 		err = snd_djm_controls_create(mixer, SND_DJM_V10_IDX);
 		break;
+	case USB_ID(0x2b73, 0x0037): /* Pioneer DJ DJM-S11 */
+		err = snd_djm_controls_create(mixer, SND_DJM_S11_IDX);
+		break;
 	case USB_ID(0x03f0, 0x0269): /* HP TB Dock G2 */
 		err = hp_dock_mixer_create(mixer);
 		break;
diff --git a/sound/usb/quirks-table.h b/sound/usb/quirks-table.h
index 71444c2898b4b..938908671d934 100644
--- a/sound/usb/quirks-table.h
+++ b/sound/usb/quirks-table.h
@@ -3310,6 +3310,73 @@ YAMAHA_DEVICE(0x7010, "UB99"),
 		}
 	}
 },
+{
+	/*
+	 * Pioneer DJ / AlphaTheta DJM-S11
+	 */
+	USB_DEVICE(0x2b73, 0x0037),
+	QUIRK_DRIVER_INFO {
+		QUIRK_DATA_COMPOSITE {
+			{
+				QUIRK_DATA_STANDARD_MIXER(0)
+			},
+			{
+				QUIRK_DATA_AUDIOFORMAT(1) {
+					.formats = SNDRV_PCM_FMTBIT_S24_3LE,
+					.channels = 14,
+					.iface = 1,
+					.altsetting = 1,
+					.altset_idx = 1,
+					.endpoint = 0x01,
+					.ep_attr = USB_ENDPOINT_XFER_ISOC |
+						   USB_ENDPOINT_SYNC_ASYNC,
+					.rates = SNDRV_PCM_RATE_48000,
+					.rate_min = 48000,
+					.rate_max = 48000,
+					.nr_rates = 1,
+					.rate_table = (unsigned int[]) { 48000 },
+					.clock = 1,
+					.fmt_type = UAC_FORMAT_TYPE_I
+				}
+			},
+			{
+				QUIRK_DATA_AUDIOFORMAT(2) {
+					.formats = SNDRV_PCM_FMTBIT_S24_3LE,
+					.channels = 10,
+					.iface = 2,
+					.altsetting = 1,
+					.altset_idx = 1,
+					.endpoint = 0x82,
+					.ep_attr = USB_ENDPOINT_XFER_ISOC |
+						   USB_ENDPOINT_SYNC_ASYNC |
+						   USB_ENDPOINT_USAGE_IMPLICIT_FB,
+					.rates = SNDRV_PCM_RATE_48000,
+					.rate_min = 48000,
+					.rate_max = 48000,
+					.nr_rates = 1,
+					.rate_table = (unsigned int[]) { 48000 },
+					.clock = 1,
+					.fmt_type = UAC_FORMAT_TYPE_I
+				}
+			},
+			{
+				/* Audio Control (unknown purpose) */
+				.ifnum = 3,
+				.type = QUIRK_IGNORE_INTERFACE
+			},
+			{
+				.ifnum = 4,
+				.type = QUIRK_MIDI_STANDARD_INTERFACE
+			},
+			{
+				/* HID */
+				.ifnum = 5,
+				.type = QUIRK_IGNORE_INTERFACE
+			},
+			QUIRK_COMPOSITE_END
+		}
+	}
+},
 
 /*
  * MacroSilicon MS2100/MS2106 based AV capture cards
diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c
index 1cb588691e16d..41149561aa066 100644
--- a/sound/usb/quirks.c
+++ b/sound/usb/quirks.c
@@ -2210,6 +2210,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = {
 		   QUIRK_FLAG_FORCE_IFACE_RESET | QUIRK_FLAG_IFACE_DELAY),
 	DEVICE_FLG(0x03f0, 0x654a, /* HP 320 FHD Webcam */
 		   QUIRK_FLAG_GET_SAMPLE_RATE | QUIRK_FLAG_MIC_RES_16),
+	DEVICE_FLG(0x040b, 0x0897, /* Weltrend Semiconductor, sold as Redragon H510-PRO Wireless headset */
+		   QUIRK_FLAG_MIXER_GET_CUR_BROKEN),
 	DEVICE_FLG(0x041e, 0x3000, /* Creative SB Extigy */
 		   QUIRK_FLAG_IGNORE_CTL_ERROR),
 	DEVICE_FLG(0x041e, 0x4080, /* Creative Live Cam VF0610 */
@@ -2461,6 +2463,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = {
 		   QUIRK_FLAG_DSD_RAW),
 	DEVICE_FLG(0x2708, 0x0002, /* Audient iD14 */
 		   QUIRK_FLAG_IGNORE_CTL_ERROR),
+	DEVICE_FLG(0x2772, 0x0502, /* Musical Fidelity M6s DAC */
+		   0), /* for avoiding QUIRK_FLAG_DSD_RAW with vendor match */
 	DEVICE_FLG(0x2912, 0x30c8, /* Audioengine D1 */
 		   QUIRK_FLAG_GET_SAMPLE_RATE),
 	DEVICE_FLG(0x2a70, 0x1881, /* OnePlus Technology (Shenzhen) Co., Ltd. BE02T */
@@ -2483,6 +2487,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = {
 		   QUIRK_FLAG_MIXER_GET_CUR_BROKEN),
 	DEVICE_FLG(0x2fc6, 0xf06b, /* MOONDROP Moonriver2 Ti */
 		   QUIRK_FLAG_CTL_MSG_DELAY),
+	DEVICE_FLG(0x2fc6, 0xf0b5, /* iBasso DC-Elite */
+		   QUIRK_FLAG_CTL_MSG_DELAY_1M),
 	DEVICE_FLG(0x2fc6, 0xf0b7, /* iBasso DC07 Pro */
 		   QUIRK_FLAG_CTL_MSG_DELAY_1M),
 	DEVICE_FLG(0x30be, 0x0101, /* Schiit Hel */
diff --git a/sound/usb/usx2y/us144mkii_pcm.h b/sound/usb/usx2y/us144mkii_pcm.h
index 74da8564431b9..a130678405e3b 100644
--- a/sound/usb/usx2y/us144mkii_pcm.h
+++ b/sound/usb/usx2y/us144mkii_pcm.h
@@ -7,7 +7,7 @@
 #include "us144mkii.h"
 
 /**
- * tascam_pcm_hw - Hardware capabilities for TASCAM US-144MKII PCM.
+ * var tascam_pcm_hw - Hardware capabilities for TASCAM US-144MKII PCM.
  *
  * Defines the supported PCM formats, rates, channels, and buffer/period sizes
  * for the TASCAM US-144MKII audio interface.
@@ -15,14 +15,14 @@
 extern const struct snd_pcm_hardware tascam_pcm_hw;
 
 /**
- * tascam_playback_ops - ALSA PCM operations for playback.
+ * var tascam_playback_ops - ALSA PCM operations for playback.
  *
  * This structure defines the callback functions for playback stream operations.
  */
 extern const struct snd_pcm_ops tascam_playback_ops;
 
 /**
- * tascam_capture_ops - ALSA PCM operations for capture.
+ * var tascam_capture_ops - ALSA PCM operations for capture.
  *
  * This structure defines the callback functions for capture stream operations.
  */
diff --git a/tools/accounting/delaytop.c b/tools/accounting/delaytop.c
index 72cc500b44b15..af8a1e42a055a 100644
--- a/tools/accounting/delaytop.c
+++ b/tools/accounting/delaytop.c
@@ -75,13 +75,23 @@
 	{#name, #cmd, \
 	offsetof(struct task_info, name##_delay_total), \
 	offsetof(struct task_info, name##_count), \
+	offsetof(struct task_info, name##_delay_max), \
+	offsetof(struct task_info, name##_delay_max_ts), \
 	modes}
-#define END_FIELD {NULL, 0, 0}
+#define SORT_FIELD_NO_MAX(name, cmd, modes) \
+	{#name, #cmd, \
+	offsetof(struct task_info, name##_delay_total), \
+	offsetof(struct task_info, name##_count), \
+	0, \
+	0, \
+	modes}
+#define END_FIELD {NULL, 0, 0, 0, 0, 0, 0}
 
 /* Display mode types */
 #define MODE_TYPE_ALL	(0xFFFFFFFF)
 #define MODE_DEFAULT	(1 << 0)
 #define MODE_MEMVERBOSE	(1 << 1)
+#define MODE_TYPE	(1 << 2)	/* Display specific type with avg/max */
 
 /* PSI statistics structure */
 struct psi_stats {
@@ -108,20 +118,36 @@ struct task_info {
 	char command[TASK_COMM_LEN];
 	unsigned long long cpu_count;
 	unsigned long long cpu_delay_total;
+	unsigned long long cpu_delay_max;
+	struct __kernel_timespec cpu_delay_max_ts;
 	unsigned long long blkio_count;
 	unsigned long long blkio_delay_total;
+	unsigned long long blkio_delay_max;
+	struct __kernel_timespec blkio_delay_max_ts;
 	unsigned long long swapin_count;
 	unsigned long long swapin_delay_total;
+	unsigned long long swapin_delay_max;
+	struct __kernel_timespec swapin_delay_max_ts;
 	unsigned long long freepages_count;
 	unsigned long long freepages_delay_total;
+	unsigned long long freepages_delay_max;
+	struct __kernel_timespec freepages_delay_max_ts;
 	unsigned long long thrashing_count;
 	unsigned long long thrashing_delay_total;
+	unsigned long long thrashing_delay_max;
+	struct __kernel_timespec thrashing_delay_max_ts;
 	unsigned long long compact_count;
 	unsigned long long compact_delay_total;
+	unsigned long long compact_delay_max;
+	struct __kernel_timespec compact_delay_max_ts;
 	unsigned long long wpcopy_count;
 	unsigned long long wpcopy_delay_total;
+	unsigned long long wpcopy_delay_max;
+	struct __kernel_timespec wpcopy_delay_max_ts;
 	unsigned long long irq_count;
 	unsigned long long irq_delay_total;
+	unsigned long long irq_delay_max;
+	struct __kernel_timespec irq_delay_max_ts;
 	unsigned long long mem_count;
 	unsigned long long mem_delay_total;
 };
@@ -141,6 +167,8 @@ struct field_desc {
 	const char *cmd_char;	/* Interactive command */
 	unsigned long total_offset; /* Offset of total delay in task_info */
 	unsigned long count_offset; /* Offset of count in task_info */
+	unsigned long max_offset;  /* Offset of max delay in task_info */
+	unsigned long max_ts_offset; /* Offset of max delay timestamp in task_info */
 	size_t supported_modes; /* Supported display modes */
 };
 
@@ -153,6 +181,7 @@ struct config {
 	int monitor_pid;		/* Monitor specific PID */
 	char *container_path;	/* Path to container cgroup */
 	const struct field_desc *sort_field;	/* Current sort field */
+	const struct field_desc *type_field;	/* Type field for -t option */
 	size_t display_mode;	/* Current display mode */
 };
 
@@ -164,15 +193,15 @@ static int task_count;
 static int running = 1;
 static struct container_stats container_stats;
 static const struct field_desc sort_fields[] = {
-	SORT_FIELD(cpu,		c,	MODE_DEFAULT),
-	SORT_FIELD(blkio,	i,	MODE_DEFAULT),
-	SORT_FIELD(irq,		q,	MODE_DEFAULT),
-	SORT_FIELD(mem,		m,	MODE_DEFAULT | MODE_MEMVERBOSE),
-	SORT_FIELD(swapin,	s,	MODE_MEMVERBOSE),
-	SORT_FIELD(freepages,	r,	MODE_MEMVERBOSE),
-	SORT_FIELD(thrashing,	t,	MODE_MEMVERBOSE),
-	SORT_FIELD(compact,	p,	MODE_MEMVERBOSE),
-	SORT_FIELD(wpcopy,	w,	MODE_MEMVERBOSE),
+	SORT_FIELD(cpu,		c,	MODE_DEFAULT | MODE_TYPE),
+	SORT_FIELD(blkio,	i,	MODE_DEFAULT | MODE_TYPE),
+	SORT_FIELD(irq,		q,	MODE_DEFAULT | MODE_TYPE),
+	SORT_FIELD_NO_MAX(mem,	m,	MODE_DEFAULT | MODE_MEMVERBOSE),
+	SORT_FIELD(swapin,	s,	MODE_MEMVERBOSE | MODE_TYPE),
+	SORT_FIELD(freepages,	r,	MODE_MEMVERBOSE | MODE_TYPE),
+	SORT_FIELD(thrashing,	t,	MODE_MEMVERBOSE | MODE_TYPE),
+	SORT_FIELD(compact,	p,	MODE_MEMVERBOSE | MODE_TYPE),
+	SORT_FIELD(wpcopy,	w,	MODE_MEMVERBOSE | MODE_TYPE),
 	END_FIELD
 };
 static int sort_selected;
@@ -265,6 +294,8 @@ static void usage(void)
 	"  -p, --pid=PID            Monitor only the specified PID\n"
 	"  -C, --container=PATH     Monitor the container at specified cgroup path\n"
 	"  -s, --sort=FIELD         Sort by delay field (default: cpu)\n"
+	"  -t, --type=FIELD         Display only specified delay type with avg/max/timestamp\n"
+	"                           (rows sorted by MAX for that type, largest first)\n"
 	"  -M, --memverbose         Display memory detailed information\n");
 	exit(0);
 }
@@ -283,6 +314,7 @@ static void parse_args(int argc, char **argv)
 		{"processes", required_argument, 0, 'P'},
 		{"sort", required_argument, 0, 's'},
 		{"container", required_argument, 0, 'C'},
+		{"type", required_argument, 0, 't'},
 		{"memverbose", no_argument, 0, 'M'},
 		{0, 0, 0, 0}
 	};
@@ -292,6 +324,7 @@ static void parse_args(int argc, char **argv)
 	cfg.iterations = 0;
 	cfg.max_processes = 20;
 	cfg.sort_field = &sort_fields[0];	/* Default sorted by CPU delay */
+	cfg.type_field = NULL;				/* No type field by default */
 	cfg.output_one_time = 0;
 	cfg.monitor_pid = 0;	/* 0 means monitor all PIDs */
 	cfg.container_path = NULL;
@@ -300,7 +333,7 @@ static void parse_args(int argc, char **argv)
 	while (1) {
 		int option_index = 0;
 
-		c = getopt_long(argc, argv, "hd:n:p:oP:C:s:M", long_options, &option_index);
+		c = getopt_long(argc, argv, "hd:n:p:oP:C:s:t:M", long_options, &option_index);
 		if (c == -1)
 			break;
 
@@ -363,9 +396,32 @@ static void parse_args(int argc, char **argv)
 
 			cfg.sort_field = field;
 			break;
+		case 't':
+			if (strlen(optarg) == 0) {
+				fprintf(stderr, "Error: empty type field\n");
+				exit(1);
+			}
+
+			field = get_field_by_name(optarg);
+			/* Show available fields if invalid option provided */
+			if (!field || !(field->supported_modes & MODE_TYPE)) {
+				fprintf(stderr, "Error: invalid type field '%s'\n", optarg);
+				display_available_fields(MODE_TYPE);
+				exit(1);
+			}
+
+			cfg.type_field = field;
+			cfg.display_mode = MODE_TYPE;
+			break;
 		case 'M':
 			cfg.display_mode = MODE_MEMVERBOSE;
-			cfg.sort_field = get_field_by_name("mem");
+			/* Find first field supporting MODE_MEMVERBOSE for sorting */
+			for (field = sort_fields; field->name != NULL; field++) {
+				if (field->supported_modes & MODE_MEMVERBOSE) {
+					cfg.sort_field = field;
+					break;
+				}
+			}
 			break;
 		default:
 			fprintf(stderr, "Try 'delaytop --help' for more information.\n");
@@ -690,7 +746,12 @@ static void fetch_and_fill_task_info(int pid, const char *comm)
 			nested_len = NLA_PAYLOAD(na->nla_len);
 			while (nested_len > 0) {
 				if (nested->nla_type == TASKSTATS_TYPE_STATS) {
-					memcpy(&stats, NLA_DATA(nested), sizeof(stats));
+					size_t payload_len = NLA_PAYLOAD(nested->nla_len);
+
+					memset(&stats, 0, sizeof(stats));
+					if (payload_len > sizeof(stats))
+						payload_len = sizeof(stats);
+					memcpy(&stats, NLA_DATA(nested), payload_len);
 					if (task_count < MAX_TASKS) {
 						tasks[task_count].pid = pid;
 						tasks[task_count].tgid = pid;
@@ -699,20 +760,36 @@ static void fetch_and_fill_task_info(int pid, const char *comm)
 						tasks[task_count].command[TASK_COMM_LEN - 1] = '\0';
 						SET_TASK_STAT(task_count, cpu_count);
 						SET_TASK_STAT(task_count, cpu_delay_total);
+						SET_TASK_STAT(task_count, cpu_delay_max);
+						SET_TASK_STAT(task_count, cpu_delay_max_ts);
 						SET_TASK_STAT(task_count, blkio_count);
 						SET_TASK_STAT(task_count, blkio_delay_total);
+						SET_TASK_STAT(task_count, blkio_delay_max);
+						SET_TASK_STAT(task_count, blkio_delay_max_ts);
 						SET_TASK_STAT(task_count, swapin_count);
 						SET_TASK_STAT(task_count, swapin_delay_total);
+						SET_TASK_STAT(task_count, swapin_delay_max);
+						SET_TASK_STAT(task_count, swapin_delay_max_ts);
 						SET_TASK_STAT(task_count, freepages_count);
 						SET_TASK_STAT(task_count, freepages_delay_total);
+						SET_TASK_STAT(task_count, freepages_delay_max);
+						SET_TASK_STAT(task_count, freepages_delay_max_ts);
 						SET_TASK_STAT(task_count, thrashing_count);
 						SET_TASK_STAT(task_count, thrashing_delay_total);
+						SET_TASK_STAT(task_count, thrashing_delay_max);
+						SET_TASK_STAT(task_count, thrashing_delay_max_ts);
 						SET_TASK_STAT(task_count, compact_count);
 						SET_TASK_STAT(task_count, compact_delay_total);
+						SET_TASK_STAT(task_count, compact_delay_max);
+						SET_TASK_STAT(task_count, compact_delay_max_ts);
 						SET_TASK_STAT(task_count, wpcopy_count);
 						SET_TASK_STAT(task_count, wpcopy_delay_total);
+						SET_TASK_STAT(task_count, wpcopy_delay_max);
+						SET_TASK_STAT(task_count, wpcopy_delay_max_ts);
 						SET_TASK_STAT(task_count, irq_count);
 						SET_TASK_STAT(task_count, irq_delay_total);
+						SET_TASK_STAT(task_count, irq_delay_max);
+						SET_TASK_STAT(task_count, irq_delay_max_ts);
 						set_mem_count(&tasks[task_count]);
 						set_mem_delay_total(&tasks[task_count]);
 						task_count++;
@@ -762,6 +839,15 @@ static void get_task_delays(void)
 	closedir(dir);
 }
 
+static void field_delay_max_and_ts(const struct task_info *task,
+				     const struct field_desc *field,
+				     unsigned long long *max_ns,
+				     struct __kernel_timespec *max_ts);
+static void get_field_delay_values(const struct task_info *task,
+				   const struct field_desc *field,
+				   double *avg_ms, double *max_ms,
+				   struct __kernel_timespec *max_ts);
+
 /* Calculate average delay in milliseconds */
 static double average_ms(unsigned long long total, unsigned long long count)
 {
@@ -770,6 +856,41 @@ static double average_ms(unsigned long long total, unsigned long long count)
 	return (double)total / 1000000.0 / count;
 }
 
+/*
+ * Format __kernel_timespec to human readable string (YYYY-MM-DDTHH:MM:SS)
+ * Returns formatted string or "N/A" if timestamp is zero
+ */
+static const char *format_kernel_timespec(struct __kernel_timespec *ts)
+{
+	static char buffer[32];
+	time_t time_sec;
+	struct tm tm_info;
+
+	/* Check if timestamp is zero (not set) */
+	if (ts->tv_sec == 0 && ts->tv_nsec == 0)
+		return "N/A";
+
+	/* Avoid Y2038 truncation: check if timestamp fits in time_t on 32-bit platforms */
+	if (sizeof(time_t) < sizeof(ts->tv_sec) &&
+	    ts->tv_sec > (__u64)((1ULL << (sizeof(time_t) * 8 - 1)) - 1))
+		return "N/A";
+
+	time_sec = (time_t)ts->tv_sec;
+
+	if (localtime_r(&time_sec, &tm_info) == NULL)
+		return "N/A";
+
+	snprintf(buffer, sizeof(buffer), "%04d-%02d-%02dT%02d:%02d:%02d",
+		tm_info.tm_year + 1900,
+		tm_info.tm_mon + 1,
+		tm_info.tm_mday,
+		tm_info.tm_hour,
+		tm_info.tm_min,
+		tm_info.tm_sec);
+
+	return buffer;
+}
+
 /* Comparison function for sorting tasks */
 static int compare_tasks(const void *a, const void *b)
 {
@@ -777,14 +898,24 @@ static int compare_tasks(const void *a, const void *b)
 	const struct task_info *t2 = (const struct task_info *)b;
 	unsigned long long total1;
 	unsigned long long total2;
-	unsigned long count1;
-	unsigned long count2;
+	unsigned long long count1;
+	unsigned long long count2;
 	double avg1, avg2;
+	unsigned long long max1, max2;
+
+	/* -t/--type: default sort by MAX column for the selected type (descending) */
+	if (cfg.display_mode == MODE_TYPE && cfg.type_field) {
+		field_delay_max_and_ts(t1, cfg.type_field, &max1, NULL);
+		field_delay_max_and_ts(t2, cfg.type_field, &max2, NULL);
+		if (max1 != max2)
+			return max2 > max1 ? 1 : -1;
+		return 0;
+	}
 
 	total1 = *(unsigned long long *)((char *)t1 + cfg.sort_field->total_offset);
 	total2 = *(unsigned long long *)((char *)t2 + cfg.sort_field->total_offset);
-	count1 = *(unsigned long *)((char *)t1 + cfg.sort_field->count_offset);
-	count2 = *(unsigned long *)((char *)t2 + cfg.sort_field->count_offset);
+	count1 = *(unsigned long long *)((char *)t1 + cfg.sort_field->count_offset);
+	count2 = *(unsigned long long *)((char *)t2 + cfg.sort_field->count_offset);
 
 	avg1 = average_ms(total1, count1);
 	avg2 = average_ms(total2, count2);
@@ -794,6 +925,50 @@ static int compare_tasks(const void *a, const void *b)
 	return 0;
 }
 
+/* Max delay (ns) and timestamp for field (shared by display and sort) */
+static void field_delay_max_and_ts(const struct task_info *task, const struct field_desc *field,
+				     unsigned long long *max_ns, struct __kernel_timespec *max_ts)
+{
+	if (!field || !field->max_offset) {
+		*max_ns = 0;
+		if (max_ts)
+			memset(max_ts, 0, sizeof(*max_ts));
+		return;
+	}
+
+	*max_ns = *(unsigned long long *)((char *)task + field->max_offset);
+
+	if (max_ts) {
+		if (field->max_ts_offset)
+			*max_ts = *(struct __kernel_timespec *)((char *)task +
+							       field->max_ts_offset);
+		else
+			memset(max_ts, 0, sizeof(*max_ts));
+	}
+}
+
+/* Get delay values for a specific field */
+static void get_field_delay_values(const struct task_info *task, const struct field_desc *field,
+				   double *avg_ms, double *max_ms,
+				   struct __kernel_timespec *max_ts)
+{
+	unsigned long long total, count, max;
+
+	if (!field || !field->max_offset) {
+		*avg_ms = 0;
+		*max_ms = 0;
+		memset(max_ts, 0, sizeof(*max_ts));
+		return;
+	}
+
+	total = *(unsigned long long *)((char *)task + field->total_offset);
+	count = *(unsigned long long *)((char *)task + field->count_offset);
+	*avg_ms = average_ms(total, count);
+
+	field_delay_max_and_ts(task, field, &max, max_ts);
+	*max_ms = (double)max / 1000000.0;  /* Convert nanoseconds to milliseconds */
+}
+
 /* Sort tasks by selected field */
 static void sort_tasks(void)
 {
@@ -847,7 +1022,12 @@ static void get_container_stats(void)
 	while (nl_len > 0) {
 		if (na->nla_type == CGROUPSTATS_TYPE_CGROUP_STATS) {
 			/* Get the cgroupstats structure */
-			memcpy(&stats, NLA_DATA(na), sizeof(stats));
+			size_t payload_len = NLA_PAYLOAD(na->nla_len);
+
+			memset(&stats, 0, sizeof(stats));
+			if (payload_len > sizeof(stats))
+				payload_len = sizeof(stats);
+			memcpy(&stats, NLA_DATA(na), payload_len);
 
 			/* Fill container stats */
 			container_stats.nr_sleeping = stats.nr_sleeping;
@@ -878,7 +1058,7 @@ static void display_results(int psi_ret)
 	suc &= BOOL_FPRINT(out, "\033[H\033[J");
 
 	/* PSI output (one-line, no cat style) */
-	suc &= BOOL_FPRINT(out, "System Pressure Information: (avg10/avg60vg300/total)\n");
+	suc &= BOOL_FPRINT(out, "System Pressure Information: (avg10/avg60/avg300/total)\n");
 	if (psi_ret) {
 		suc &= BOOL_FPRINT(out, "  PSI not found: check if psi=1 enabled in cmdline\n");
 	} else {
@@ -936,7 +1116,10 @@ static void display_results(int psi_ret)
 	}
 
 	/* Interacive command */
-	suc &= BOOL_FPRINT(out, "[o]sort [M]memverbose [q]quit\n");
+	if (cfg.display_mode == MODE_TYPE && cfg.type_field)
+		suc &= BOOL_FPRINT(out, "[q]quit\n");
+	else
+		suc &= BOOL_FPRINT(out, "[o]sort [M]memverbose [q]quit\n");
 	if (sort_selected) {
 		if (cfg.display_mode == MODE_MEMVERBOSE)
 			suc &= BOOL_FPRINT(out,
@@ -947,32 +1130,55 @@ static void display_results(int psi_ret)
 	}
 
 	/* Task delay output */
-	suc &= BOOL_FPRINT(out, "Top %d processes (sorted by %s delay):\n",
-			cfg.max_processes, get_name_by_field(cfg.sort_field));
-
-	suc &= BOOL_FPRINT(out, "%8s  %8s  %-17s", "PID", "TGID", "COMMAND");
-	if (cfg.display_mode == MODE_MEMVERBOSE) {
-		suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s %8s %8s\n",
-			"MEM(ms)", "SWAP(ms)", "RCL(ms)",
-			"THR(ms)", "CMP(ms)", "WP(ms)");
-		suc &= BOOL_FPRINT(out, "-----------------------");
-		suc &= BOOL_FPRINT(out, "-----------------------");
-		suc &= BOOL_FPRINT(out, "-----------------------");
-		suc &= BOOL_FPRINT(out, "---------------------\n");
+	if (cfg.display_mode == MODE_TYPE && cfg.type_field)
+		suc &= BOOL_FPRINT(out,
+			"Top %d processes (sorted by %s MAX delay, largest first):\n",
+			cfg.max_processes, get_name_by_field(cfg.type_field));
+	else
+		suc &= BOOL_FPRINT(out, "Top %d processes (sorted by %s delay):\n",
+				cfg.max_processes, get_name_by_field(cfg.sort_field));
+
+	if (cfg.display_mode == MODE_TYPE && cfg.type_field) {
+		/* Display mode for -t option: show only specified type with avg/max/timestamp */
+		suc &= BOOL_FPRINT(out, "%8s  %8s  %-17s %12s %12s %20s\n",
+			"PID", "TGID", "COMMAND",
+			"AVG(ms)", "MAX(ms)", "MAX_TIMESTAMP");
+		suc &= BOOL_FPRINT(out, "--------------------------------------------------------");
+		suc &= BOOL_FPRINT(out, "----------------------------------------\n");
 	} else {
-		suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s\n",
-			"CPU(ms)", "IO(ms)", "IRQ(ms)", "MEM(ms)");
-		suc &= BOOL_FPRINT(out, "-----------------------");
-		suc &= BOOL_FPRINT(out, "-----------------------");
-		suc &= BOOL_FPRINT(out, "--------------------------\n");
+		suc &= BOOL_FPRINT(out, "%8s  %8s  %-17s", "PID", "TGID", "COMMAND");
+		if (cfg.display_mode == MODE_MEMVERBOSE) {
+			suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s %8s %8s\n",
+				"MEM(ms)", "SWAP(ms)", "RCL(ms)",
+				"THR(ms)", "CMP(ms)", "WP(ms)");
+			suc &= BOOL_FPRINT(out, "-----------------------");
+			suc &= BOOL_FPRINT(out, "-----------------------");
+			suc &= BOOL_FPRINT(out, "-----------------------");
+			suc &= BOOL_FPRINT(out, "---------------------\n");
+		} else {
+			suc &= BOOL_FPRINT(out, "%8s %8s %8s %8s\n",
+				"CPU(ms)", "IO(ms)", "IRQ(ms)", "MEM(ms)");
+			suc &= BOOL_FPRINT(out, "-----------------------");
+			suc &= BOOL_FPRINT(out, "-----------------------");
+			suc &= BOOL_FPRINT(out, "--------------------------\n");
+		}
 	}
 
 	count = task_count < cfg.max_processes ? task_count : cfg.max_processes;
 
 	for (i = 0; i < count; i++) {
-		suc &= BOOL_FPRINT(out, "%8d  %8d  %-15s",
+		suc &= BOOL_FPRINT(out, "%8d  %8d  %-17s",
 			tasks[i].pid, tasks[i].tgid, tasks[i].command);
-		if (cfg.display_mode == MODE_MEMVERBOSE) {
+		if (cfg.display_mode == MODE_TYPE && cfg.type_field) {
+			double avg_ms, max_ms;
+			struct __kernel_timespec max_ts;
+
+			get_field_delay_values(&tasks[i], cfg.type_field, &avg_ms,
+					&max_ms, &max_ts);
+
+			suc &= BOOL_FPRINT(out, "%12.2f %12.2f %20s\n",
+				avg_ms, max_ms, format_kernel_timespec(&max_ts));
+		} else if (cfg.display_mode == MODE_MEMVERBOSE) {
 			suc &= BOOL_FPRINT(out, DELAY_FMT_MEMVERBOSE,
 				TASK_AVG(tasks[i], mem),
 				TASK_AVG(tasks[i], swapin),
@@ -1040,9 +1246,13 @@ static void handle_keypress(char ch, int *running)
 	} else {
 		switch (ch) {
 		case 'o':
+			if (cfg.display_mode == MODE_TYPE)
+				break;
 			sort_selected = 1;
 			break;
 		case 'M':
+			if (cfg.display_mode == MODE_TYPE)
+				break;
 			toggle_display_mode();
 			for (field = sort_fields; field->name != NULL; field++) {
 				if (field->supported_modes & cfg.display_mode) {
diff --git a/tools/accounting/getdelays.c b/tools/accounting/getdelays.c
index caa5fe9dd5734..263ded4567fd2 100644
--- a/tools/accounting/getdelays.c
+++ b/tools/accounting/getdelays.c
@@ -235,6 +235,11 @@ static const char *format_timespec(struct __kernel_timespec *ts)
 	if (ts->tv_sec == 0 && ts->tv_nsec == 0)
 		return "N/A";
 
+	/* Avoid Y2038 truncation on 32-bit platforms */
+	if (sizeof(time_sec) < sizeof(ts->tv_sec) &&
+	    ts->tv_sec > (__u64)((1ULL << (sizeof(time_sec) * 8 - 1)) - 1))
+		return "N/A";
+
 	time_sec = ts->tv_sec;
 
 	/* Use thread-safe localtime_r */
@@ -302,7 +307,7 @@ static const char *format_timespec(struct __kernel_timespec *ts)
 				average_ms((double)(t)->cpu_delay_total, (t)->cpu_count)); \
 		} \
 	} while (0)
-#define PRINT_FILED_DELAY(name, version, t, count, total, max, min) \
+#define PRINT_FIELD_DELAY(name, version, t, count, total, max, min) \
 	do { \
 		if (version >= 16) { \
 			printf("%-10s%15s%15s%15s%15s%15s\n", \
@@ -324,7 +329,7 @@ static const char *format_timespec(struct __kernel_timespec *ts)
 		} \
 	} while (0)
 
-#define PRINT_FILED_DELAY_WITH_TS(name, version, t, count, total, max, min, max_ts) \
+#define PRINT_FIELD_DELAY_WITH_TS(name, version, t, count, total, max, min, max_ts) \
 	do { \
 		if (version >= 17) { \
 			printf("%-10s%15s%15s%15s%15s%15s%25s\n", \
@@ -365,71 +370,71 @@ static void print_delayacct(struct taskstats *t)
 
 	/* Use new macro with timestamp support for version >= 17 */
 	if (t->version >= 17) {
-		PRINT_FILED_DELAY_WITH_TS("IO", t->version, t,
+		PRINT_FIELD_DELAY_WITH_TS("IO", t->version, t,
 			blkio_count, blkio_delay_total,
 			blkio_delay_max, blkio_delay_min, blkio_delay_max_ts);
 
-		PRINT_FILED_DELAY_WITH_TS("SWAP", t->version, t,
+		PRINT_FIELD_DELAY_WITH_TS("SWAP", t->version, t,
 			swapin_count, swapin_delay_total,
 			swapin_delay_max, swapin_delay_min, swapin_delay_max_ts);
 
-		PRINT_FILED_DELAY_WITH_TS("RECLAIM", t->version, t,
+		PRINT_FIELD_DELAY_WITH_TS("RECLAIM", t->version, t,
 			freepages_count, freepages_delay_total,
 			freepages_delay_max, freepages_delay_min, freepages_delay_max_ts);
 
-		PRINT_FILED_DELAY_WITH_TS("THRASHING", t->version, t,
+		PRINT_FIELD_DELAY_WITH_TS("THRASHING", t->version, t,
 			thrashing_count, thrashing_delay_total,
 			thrashing_delay_max, thrashing_delay_min, thrashing_delay_max_ts);
 
 		if (t->version >= 11) {
-			PRINT_FILED_DELAY_WITH_TS("COMPACT", t->version, t,
+			PRINT_FIELD_DELAY_WITH_TS("COMPACT", t->version, t,
 				compact_count, compact_delay_total,
 				compact_delay_max, compact_delay_min, compact_delay_max_ts);
 		}
 
 		if (t->version >= 13) {
-			PRINT_FILED_DELAY_WITH_TS("WPCOPY", t->version, t,
+			PRINT_FIELD_DELAY_WITH_TS("WPCOPY", t->version, t,
 				wpcopy_count, wpcopy_delay_total,
 				wpcopy_delay_max, wpcopy_delay_min, wpcopy_delay_max_ts);
 		}
 
 		if (t->version >= 14) {
-			PRINT_FILED_DELAY_WITH_TS("IRQ", t->version, t,
+			PRINT_FIELD_DELAY_WITH_TS("IRQ", t->version, t,
 				irq_count, irq_delay_total,
 				irq_delay_max, irq_delay_min, irq_delay_max_ts);
 		}
 	} else {
 		/* Use original macro for older versions */
-		PRINT_FILED_DELAY("IO", t->version, t,
+		PRINT_FIELD_DELAY("IO", t->version, t,
 			blkio_count, blkio_delay_total,
 			blkio_delay_max, blkio_delay_min);
 
-		PRINT_FILED_DELAY("SWAP", t->version, t,
+		PRINT_FIELD_DELAY("SWAP", t->version, t,
 			swapin_count, swapin_delay_total,
 			swapin_delay_max, swapin_delay_min);
 
-		PRINT_FILED_DELAY("RECLAIM", t->version, t,
+		PRINT_FIELD_DELAY("RECLAIM", t->version, t,
 			freepages_count, freepages_delay_total,
 			freepages_delay_max, freepages_delay_min);
 
-		PRINT_FILED_DELAY("THRASHING", t->version, t,
+		PRINT_FIELD_DELAY("THRASHING", t->version, t,
 			thrashing_count, thrashing_delay_total,
 			thrashing_delay_max, thrashing_delay_min);
 
 		if (t->version >= 11) {
-			PRINT_FILED_DELAY("COMPACT", t->version, t,
+			PRINT_FIELD_DELAY("COMPACT", t->version, t,
 				compact_count, compact_delay_total,
 				compact_delay_max, compact_delay_min);
 		}
 
 		if (t->version >= 13) {
-			PRINT_FILED_DELAY("WPCOPY", t->version, t,
+			PRINT_FIELD_DELAY("WPCOPY", t->version, t,
 				wpcopy_count, wpcopy_delay_total,
 				wpcopy_delay_max, wpcopy_delay_min);
 		}
 
 		if (t->version >= 14) {
-			PRINT_FILED_DELAY("IRQ", t->version, t,
+			PRINT_FIELD_DELAY("IRQ", t->version, t,
 				irq_count, irq_delay_total,
 				irq_delay_max, irq_delay_min);
 		}
diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h
index 86d17b195e794..8860778305ca1 100644
--- a/tools/arch/x86/include/asm/cpufeatures.h
+++ b/tools/arch/x86/include/asm/cpufeatures.h
@@ -78,7 +78,7 @@
 #define X86_FEATURE_ZEN6		( 3*32+ 6) /* CPU based on Zen6 microarchitecture */
 /* Free                                 ( 3*32+ 7) */
 #define X86_FEATURE_CONSTANT_TSC	( 3*32+ 8) /* "constant_tsc" TSC ticks at a constant rate */
-#define X86_FEATURE_UP			( 3*32+ 9) /* "up" SMP kernel running on UP */
+/* free: was #define X86_FEATURE_UP	( 3*32+ 9) * "up" SMP kernel running on UP */
 #define X86_FEATURE_ART			( 3*32+10) /* "art" Always running timer (ART) */
 #define X86_FEATURE_ARCH_PERFMON	( 3*32+11) /* "arch_perfmon" Intel Architectural PerfMon */
 #define X86_FEATURE_PEBS		( 3*32+12) /* "pebs" Precise-Event Based Sampling */
diff --git a/tools/bootconfig/Makefile b/tools/bootconfig/Makefile
index 90eb47c9d8de6..3cb8066d5141b 100644
--- a/tools/bootconfig/Makefile
+++ b/tools/bootconfig/Makefile
@@ -15,7 +15,7 @@ override CFLAGS += -Wall -g -I$(CURDIR)/include
 ALL_TARGETS := bootconfig
 ALL_PROGRAMS := $(patsubst %,$(OUTPUT)%,$(ALL_TARGETS))
 
-all: $(ALL_PROGRAMS) test
+all: $(ALL_PROGRAMS)
 
 $(OUTPUT)bootconfig: main.c include/linux/bootconfig.h $(LIBSRC)
 	$(CC) $(filter %.c,$^) $(CFLAGS) $(LDFLAGS) -o $@
@@ -27,4 +27,4 @@ install: $(ALL_PROGRAMS)
 	install $(OUTPUT)bootconfig $(DESTDIR)$(bindir)
 
 clean:
-	$(RM) -f $(OUTPUT)*.o $(ALL_PROGRAMS)
+	rm -f $(OUTPUT)*.o $(ALL_PROGRAMS)
diff --git a/tools/bootconfig/scripts/ftrace2bconf.sh b/tools/bootconfig/scripts/ftrace2bconf.sh
index 1603801cf1264..8eed445c295e3 100755
--- a/tools/bootconfig/scripts/ftrace2bconf.sh
+++ b/tools/bootconfig/scripts/ftrace2bconf.sh
@@ -57,6 +57,8 @@ EOF
 kprobe_event_options() {
 	cat $TRACEFS/kprobe_events | while read p args; do
 		case $p in
+		\#*)
+		continue;;
 		r*)
 		cat 1>&2 << EOF
 # WARN: A return probe found but it is not supported by bootconfig. Skip it.
diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile
index 271a7dc772730..b0f7168e79432 100644
--- a/tools/bpf/bpftool/Makefile
+++ b/tools/bpf/bpftool/Makefile
@@ -99,7 +99,7 @@ endif
 HOST_LDFLAGS := $(LDFLAGS)
 
 # Remove warnings for libbpf bootstrap build
-LIBBPF_BOOTSTRAP_CFLAGS := $(filter-out -W -Wall -Wextra -Wformat -Wformat-signedness,$(HOST_CFLAGS))
+LIBBPF_BOOTSTRAP_CFLAGS := $(filter-out -W -Wall -Wextra -Wformat%,$(HOST_CFLAGS))
 
 INSTALL ?= install
 RM ?= rm -f
diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c
index 6ef908adf3a4a..c9589026da8d6 100644
--- a/tools/bpf/bpftool/btf.c
+++ b/tools/bpf/bpftool/btf.c
@@ -179,7 +179,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id,
 	case BTF_KIND_STRUCT:
 	case BTF_KIND_UNION: {
 		const struct btf_member *m = (const void *)(t + 1);
-		__u32 i, vlen = BTF_INFO_VLEN(t->info);
+		__u32 i, vlen = btf_vlen(t);
 
 		if (json_output) {
 			jsonw_uint_field(w, "size", t->size);
@@ -193,7 +193,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id,
 			const char *name = btf_str(btf, m->name_off);
 			__u32 bit_off, bit_sz;
 
-			if (BTF_INFO_KFLAG(t->info)) {
+			if (btf_kflag(t)) {
 				bit_off = BTF_MEMBER_BIT_OFFSET(m->offset);
 				bit_sz = BTF_MEMBER_BITFIELD_SIZE(m->offset);
 			} else {
@@ -224,7 +224,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id,
 	}
 	case BTF_KIND_ENUM: {
 		const struct btf_enum *v = (const void *)(t + 1);
-		__u32 i, vlen = BTF_INFO_VLEN(t->info);
+		__u32 i, vlen = btf_vlen(t);
 		const char *encoding;
 
 		encoding = btf_kflag(t) ? "SIGNED" : "UNSIGNED";
@@ -300,8 +300,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id,
 		break;
 	}
 	case BTF_KIND_FWD: {
-		const char *fwd_kind = BTF_INFO_KFLAG(t->info) ? "union"
-							       : "struct";
+		const char *fwd_kind = btf_kflag(t) ? "union" : "struct";
 
 		if (json_output)
 			jsonw_string_field(w, "fwd_kind", fwd_kind);
@@ -322,7 +321,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id,
 	}
 	case BTF_KIND_FUNC_PROTO: {
 		const struct btf_param *p = (const void *)(t + 1);
-		__u32 i, vlen = BTF_INFO_VLEN(t->info);
+		__u32 i, vlen = btf_vlen(t);
 
 		if (json_output) {
 			jsonw_uint_field(w, "ret_type_id", t->type);
@@ -365,7 +364,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id,
 	case BTF_KIND_DATASEC: {
 		const struct btf_var_secinfo *v = (const void *)(t + 1);
 		const struct btf_type *vt;
-		__u32 i, vlen = BTF_INFO_VLEN(t->info);
+		__u32 i, vlen = btf_vlen(t);
 
 		if (json_output) {
 			jsonw_uint_field(w, "size", t->size);
diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c
index 9dc8425b17899..e4075824343fd 100644
--- a/tools/bpf/bpftool/btf_dumper.c
+++ b/tools/bpf/bpftool/btf_dumper.c
@@ -476,8 +476,8 @@ static int btf_dumper_struct(const struct btf_dumper *d, __u32 type_id,
 	if (!t)
 		return -EINVAL;
 
-	kind_flag = BTF_INFO_KFLAG(t->info);
-	vlen = BTF_INFO_VLEN(t->info);
+	kind_flag = btf_kflag(t);
+	vlen = btf_vlen(t);
 	jsonw_start_object(d->jw);
 	m = (struct btf_member *)(t + 1);
 
@@ -535,7 +535,7 @@ static int btf_dumper_datasec(const struct btf_dumper *d, __u32 type_id,
 	if (!t)
 		return -EINVAL;
 
-	vlen = BTF_INFO_VLEN(t->info);
+	vlen = btf_vlen(t);
 	vsi = (struct btf_var_secinfo *)(t + 1);
 
 	jsonw_start_object(d->jw);
@@ -557,7 +557,7 @@ static int btf_dumper_do_type(const struct btf_dumper *d, __u32 type_id,
 {
 	const struct btf_type *t = btf__type_by_id(d->btf, type_id);
 
-	switch (BTF_INFO_KIND(t->info)) {
+	switch (btf_kind(t)) {
 	case BTF_KIND_INT:
 		return btf_dumper_int(t, bit_offset, data, d->jw,
 				     d->is_plain_text);
@@ -631,7 +631,7 @@ static int __btf_dumper_type_only(const struct btf *btf, __u32 type_id,
 
 	t = btf__type_by_id(btf, type_id);
 
-	switch (BTF_INFO_KIND(t->info)) {
+	switch (btf_kind(t)) {
 	case BTF_KIND_INT:
 	case BTF_KIND_TYPEDEF:
 	case BTF_KIND_FLOAT:
@@ -661,7 +661,7 @@ static int __btf_dumper_type_only(const struct btf *btf, __u32 type_id,
 		break;
 	case BTF_KIND_FWD:
 		BTF_PRINT_ARG("%s %s ",
-			      BTF_INFO_KFLAG(t->info) ? "union" : "struct",
+			      btf_kflag(t) ? "union" : "struct",
 			      btf__name_by_offset(btf, t->name_off));
 		break;
 	case BTF_KIND_VOLATILE:
@@ -718,7 +718,7 @@ static int btf_dump_func(const struct btf *btf, char *func_sig,
 		BTF_PRINT_ARG("%s(", btf__name_by_offset(btf, func->name_off));
 	else
 		BTF_PRINT_ARG("(");
-	vlen = BTF_INFO_VLEN(func_proto->info);
+	vlen = btf_vlen(func_proto);
 	for (i = 0; i < vlen; i++) {
 		struct btf_param *arg = &((struct btf_param *)(func_proto + 1))[i];
 
diff --git a/tools/bpf/bpftool/gen.c b/tools/bpf/bpftool/gen.c
index 6ae7262ebe0c1..a01d06d22d1a3 100644
--- a/tools/bpf/bpftool/gen.c
+++ b/tools/bpf/bpftool/gen.c
@@ -793,6 +793,8 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h
 	if (sign_progs) {
 		sopts.insns = opts.insns;
 		sopts.insns_sz = opts.insns_sz;
+		sopts.data = opts.data;
+		sopts.data_sz = opts.data_sz;
 		sopts.excl_prog_hash = prog_sha;
 		sopts.excl_prog_hash_sz = sizeof(prog_sha);
 		sopts.signature = sig_buf;
diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
index bdcd717b0348f..088d1d206065f 100644
--- a/tools/bpf/bpftool/link.c
+++ b/tools/bpf/bpftool/link.c
@@ -377,6 +377,25 @@ static __u64 *u64_to_arr(__u64 val)
 	return (__u64 *) u64_to_ptr(val);
 }
 
+static __u32 *u64_to_u32_arr(__u64 val)
+{
+	return (__u32 *)u64_to_ptr(val);
+}
+
+static struct kernel_sym *find_kernel_sym_by_addr(__u64 addr, bool is_ibt_enabled)
+{
+	struct kernel_sym *sym;
+
+	if (!addr)
+		return NULL;
+
+	sym = kernel_syms_search(&dd, addr);
+	if (!sym && is_ibt_enabled && addr >= 4)
+		sym = kernel_syms_search(&dd, addr - 4);
+
+	return sym;
+}
+
 static void
 show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
 {
@@ -403,6 +422,52 @@ show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
 	jsonw_end_array(json_wtr);
 }
 
+static void
+show_tracing_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
+{
+	bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol;
+	__u64 *addrs, *cookies;
+	__u32 i, *ids;
+
+	if (!dd.sym_count)
+		kernel_syms_load(&dd);
+	show_symbol = !!dd.sym_count;
+
+	show_link_attach_type_json(info->tracing_multi.attach_type, wtr);
+	jsonw_uint_field(wtr, "func_cnt", info->tracing_multi.count);
+	jsonw_uint_field(wtr, "btf_obj_id", info->tracing_multi.btf_obj_id);
+	jsonw_name(wtr, "funcs");
+
+	jsonw_start_array(wtr);
+
+	ids = u64_to_u32_arr(info->tracing_multi.ids);
+	addrs = u64_to_arr(info->tracing_multi.addrs);
+	cookies = u64_to_arr(info->tracing_multi.cookies);
+
+	for (i = 0; i < info->tracing_multi.count; i++) {
+		struct kernel_sym *sym;
+		__u64 addr = addrs[i];
+
+		sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL;
+
+		jsonw_start_object(wtr);
+		jsonw_uint_field(wtr, "id", ids[i]);
+		jsonw_uint_field(wtr, "addr", addr);
+		if (sym) {
+			jsonw_string_field(wtr, "func", sym->name);
+			if (sym->module[0] == '\0') {
+				jsonw_name(wtr, "module");
+				jsonw_null(wtr);
+			} else {
+				jsonw_string_field(wtr, "module", sym->module);
+			}
+		}
+		jsonw_uint_field(wtr, "cookie", cookies[i]);
+		jsonw_end_object(wtr);
+	}
+	jsonw_end_array(wtr);
+}
+
 static void
 show_perf_event_kprobe_json(struct bpf_link_info *info, json_writer_t *wtr)
 {
@@ -589,6 +654,9 @@ static int show_link_close_json(int fd, struct bpf_link_info *info)
 	case BPF_LINK_TYPE_UPROBE_MULTI:
 		show_uprobe_multi_json(info, json_wtr);
 		break;
+	case BPF_LINK_TYPE_TRACING_MULTI:
+		show_tracing_multi_json(info, json_wtr);
+		break;
 	case BPF_LINK_TYPE_PERF_EVENT:
 		switch (info->perf_event.type) {
 		case BPF_PERF_EVENT_EVENT:
@@ -833,6 +901,46 @@ static void show_uprobe_multi_plain(struct bpf_link_info *info)
 	}
 }
 
+static void show_tracing_multi_plain(struct bpf_link_info *info)
+{
+	bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol;
+	__u64 *addrs, *cookies;
+	__u32 i, *ids;
+
+	if (!info->tracing_multi.count)
+		return;
+
+	if (!dd.sym_count)
+		kernel_syms_load(&dd);
+	show_symbol = !!dd.sym_count;
+
+	printf("\n\t");
+	show_link_attach_type_plain(info->tracing_multi.attach_type);
+	printf("btf_obj_id %u  ", info->tracing_multi.btf_obj_id);
+	printf("count %u  ", info->tracing_multi.count);
+
+	printf("\n\t%-16s %-16s %-16s %s",
+	       "btf_id", "addr", "cookie", "func [module]");
+
+	ids = u64_to_u32_arr(info->tracing_multi.ids);
+	addrs = u64_to_arr(info->tracing_multi.addrs);
+	cookies = u64_to_arr(info->tracing_multi.cookies);
+
+	for (i = 0; i < info->tracing_multi.count; i++) {
+		__u64 addr = addrs[i];
+		struct kernel_sym *sym;
+
+		sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL;
+
+		printf("\n\t%-16u %016llx %-16llu", ids[i], addr, cookies[i]);
+		if (sym) {
+			printf(" %s", sym->name);
+			if (sym->module[0] != '\0')
+				printf(" [%s]", sym->module);
+		}
+	}
+}
+
 static void show_perf_event_kprobe_plain(struct bpf_link_info *info)
 {
 	const char *buf;
@@ -989,6 +1097,9 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info)
 	case BPF_LINK_TYPE_UPROBE_MULTI:
 		show_uprobe_multi_plain(info);
 		break;
+	case BPF_LINK_TYPE_TRACING_MULTI:
+		show_tracing_multi_plain(info);
+		break;
 	case BPF_LINK_TYPE_PERF_EVENT:
 		switch (info->perf_event.type) {
 		case BPF_PERF_EVENT_EVENT:
@@ -1029,6 +1140,7 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info)
 static int do_show_link(int fd)
 {
 	__u64 *ref_ctr_offsets = NULL, *offsets = NULL, *cookies = NULL;
+	__u32 *ids = NULL;
 	struct bpf_link_info info;
 	__u32 len = sizeof(info);
 	char path_buf[PATH_MAX];
@@ -1114,6 +1226,26 @@ static int do_show_link(int fd)
 			goto again;
 		}
 	}
+	if (info.type == BPF_LINK_TYPE_TRACING_MULTI && !info.tracing_multi.ids) {
+		count = info.tracing_multi.count;
+		if (count) {
+			ids = calloc(count, sizeof(__u32));
+			addrs = calloc(count, sizeof(__u64));
+			cookies = calloc(count, sizeof(__u64));
+			if (!ids || !addrs || !cookies) {
+				p_err("mem alloc failed");
+				close(fd);
+				free(cookies);
+				free(addrs);
+				free(ids);
+				return -ENOMEM;
+			}
+			info.tracing_multi.ids = ptr_to_u64(ids);
+			info.tracing_multi.addrs = ptr_to_u64(addrs);
+			info.tracing_multi.cookies = ptr_to_u64(cookies);
+			goto again;
+		}
+	}
 	if (info.type == BPF_LINK_TYPE_PERF_EVENT) {
 		switch (info.perf_event.type) {
 		case BPF_PERF_EVENT_TRACEPOINT:
@@ -1153,6 +1285,7 @@ static int do_show_link(int fd)
 	free(cookies);
 	free(offsets);
 	free(addrs);
+	free(ids);
 	close(fd);
 	return 0;
 }
diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c
index 71a45d96617ed..6b9649294ca1a 100644
--- a/tools/bpf/bpftool/map.c
+++ b/tools/bpf/bpftool/map.c
@@ -790,6 +790,12 @@ static int maps_have_btf(int *fds, int nb_fds)
 
 static struct btf *btf_vmlinux;
 
+static void free_btf_vmlinux(void)
+{
+	btf__free(btf_vmlinux);
+	btf_vmlinux = NULL;
+}
+
 static int get_map_kv_btf(const struct bpf_map_info *info, struct btf **btf)
 {
 	int err = 0;
@@ -958,7 +964,7 @@ static int do_dump(int argc, char **argv)
 		close(fds[i]);
 exit_free:
 	free(fds);
-	btf__free(btf_vmlinux);
+	free_btf_vmlinux();
 	return err;
 }
 
@@ -1049,7 +1055,7 @@ static void print_key_value(struct bpf_map_info *info, void *key,
 		btf_wtr = get_btf_writer();
 		if (!btf_wtr) {
 			p_info("failed to create json writer for btf. falling back to plain output");
-			btf__free(btf);
+			free_map_kv_btf(btf);
 			btf = NULL;
 			print_entry_plain(info, key, value);
 		} else {
@@ -1065,7 +1071,7 @@ static void print_key_value(struct bpf_map_info *info, void *key,
 	} else {
 		print_entry_plain(info, key, value);
 	}
-	btf__free(btf);
+	free_map_kv_btf(btf);
 }
 
 static int do_lookup(int argc, char **argv)
diff --git a/tools/bpf/bpftool/sign.c b/tools/bpf/bpftool/sign.c
index f9b742f4bb104..88726a6db6d0e 100644
--- a/tools/bpf/bpftool/sign.c
+++ b/tools/bpf/bpftool/sign.c
@@ -135,9 +135,21 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts)
 	CMS_ContentInfo *cms = NULL;
 	long actual_sig_len = 0;
 	X509 *x509 = NULL;
+	void *data = NULL;
+	size_t data_sz;
 	int err = 0;
 
-	bd_in = BIO_new_mem_buf(opts->insns, opts->insns_sz);
+	data_sz = (size_t)opts->insns_sz + opts->data_sz;
+	data = malloc(data_sz);
+	if (!data) {
+		err = -ENOMEM;
+		goto cleanup;
+	}
+	memcpy(data, opts->insns, opts->insns_sz);
+	if (opts->data_sz)
+		memcpy((char *)data + opts->insns_sz, opts->data, opts->data_sz);
+
+	bd_in = BIO_new_mem_buf(data, data_sz);
 	if (!bd_in) {
 		err = -ENOMEM;
 		goto cleanup;
@@ -175,10 +187,13 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts)
 		goto cleanup;
 	}
 
-	EVP_Digest(opts->insns, opts->insns_sz, opts->excl_prog_hash,
-		   &opts->excl_prog_hash_sz, EVP_sha256(), NULL);
+	if (EVP_Digest(opts->insns, opts->insns_sz, opts->excl_prog_hash,
+		       &opts->excl_prog_hash_sz, EVP_sha256(), NULL) != 1) {
+		err = -EIO;
+		goto cleanup;
+	}
 
-		bd_out = BIO_new(BIO_s_mem());
+	bd_out = BIO_new(BIO_s_mem());
 	if (!bd_out) {
 		err = -ENOMEM;
 		goto cleanup;
@@ -212,6 +227,7 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts)
 	X509_free(x509);
 	EVP_PKEY_free(private_key);
 	BIO_free(bd_in);
+	free(data);
 	DISPLAY_OSSL_ERR(err < 0);
 	return err;
 }
diff --git a/tools/bpf/bpftool/struct_ops.c b/tools/bpf/bpftool/struct_ops.c
index aa43dead249cb..835e5e561f7fc 100644
--- a/tools/bpf/bpftool/struct_ops.c
+++ b/tools/bpf/bpftool/struct_ops.c
@@ -643,6 +643,10 @@ int do_struct_ops(int argc, char **argv)
 	err = cmd_select(cmds, argc, argv, do_help);
 
 	btf__free(btf_vmlinux);
+	btf_vmlinux = NULL;
+	map_info_type = NULL;
+	map_info_alloc_len = 0;
+	map_info_type_id = 0;
 
 	return err;
 }
diff --git a/tools/bpf/resolve_btfids/Makefile b/tools/bpf/resolve_btfids/Makefile
index 7672208f65e4b..bb0463b380af9 100644
--- a/tools/bpf/resolve_btfids/Makefile
+++ b/tools/bpf/resolve_btfids/Makefile
@@ -40,7 +40,6 @@ LIBBPF_DESTDIR := $(LIBBPF_OUT)
 LIBBPF_INCLUDE := $(LIBBPF_DESTDIR)include
 
 SUBCMD_DESTDIR := $(SUBCMD_OUT)
-SUBCMD_INCLUDE := $(SUBCMD_DESTDIR)include
 
 BINARY     := $(OUTPUT)/resolve_btfids
 BINARY_IN  := $(BINARY)-in.o
@@ -56,7 +55,7 @@ $(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT):
 $(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
 	$(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(SUBCMD_OUT) \
 		    DESTDIR=$(SUBCMD_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
-		    $(abspath $@) install_headers
+		    $(abspath $@)
 
 $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT)
 	$(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT)    \
@@ -77,7 +76,7 @@ HOSTCFLAGS_resolve_btfids += -g \
           -I$(srctree)/tools/include \
           -I$(srctree)/tools/include/uapi \
           -I$(LIBBPF_INCLUDE) \
-          -I$(SUBCMD_INCLUDE) \
+          -I$(srctree)/tools/lib \
           $(LIBELF_FLAGS) \
           -Wall -Werror
 
diff --git a/tools/cgroup/iocost_monitor.py b/tools/cgroup/iocost_monitor.py
index 933c750b319b7..bdd78ba271b04 100644
--- a/tools/cgroup/iocost_monitor.py
+++ b/tools/cgroup/iocost_monitor.py
@@ -15,11 +15,6 @@ import time
 import json
 import math
 
-import drgn
-from drgn import container_of
-from drgn.helpers.linux.list import list_for_each_entry,list_empty
-from drgn.helpers.linux.radixtree import radix_tree_for_each,radix_tree_lookup
-
 import argparse
 parser = argparse.ArgumentParser(description=desc,
                                  formatter_class=argparse.RawTextHelpFormatter)
@@ -34,6 +29,11 @@ parser.add_argument('--json', action='store_true',
                     help='Output in json')
 args = parser.parse_args()
 
+import drgn
+from drgn import container_of
+from drgn.helpers.linux.list import list_for_each_entry,list_empty
+from drgn.helpers.linux.radixtree import radix_tree_for_each,radix_tree_lookup
+
 def err(s):
     print(s, file=sys.stderr, flush=True)
     sys.exit(1)
diff --git a/tools/include/linux/bits.h b/tools/include/linux/bits.h
index a40cc861b3a7c..b7509f15718e6 100644
--- a/tools/include/linux/bits.h
+++ b/tools/include/linux/bits.h
@@ -17,7 +17,7 @@
  * position @h. For example
  * GENMASK_ULL(39, 21) gives us the 64bit vector 0x000000ffffe00000.
  */
-#if !defined(__ASSEMBLY__)
+#if !defined(__ASSEMBLER__)
 
 /*
  * Missing asm support
@@ -75,7 +75,7 @@
 #define BIT_U32(nr)	BIT_TYPE(u32, nr)
 #define BIT_U64(nr)	BIT_TYPE(u64, nr)
 
-#else /* defined(__ASSEMBLY__) */
+#else /* defined(__ASSEMBLER__) */
 
 /*
  * BUILD_BUG_ON_ZERO is not available in h files included from asm files,
@@ -84,6 +84,6 @@
 #define GENMASK(h, l)		__GENMASK(h, l)
 #define GENMASK_ULL(h, l)	__GENMASK_ULL(h, l)
 
-#endif /* !defined(__ASSEMBLY__) */
+#endif /* !defined(__ASSEMBLER__) */
 
 #endif	/* __LINUX_BITS_H */
diff --git a/tools/include/linux/btf_ids.h b/tools/include/linux/btf_ids.h
index 72ea363d434db..4fe5c5f1558ca 100644
--- a/tools/include/linux/btf_ids.h
+++ b/tools/include/linux/btf_ids.h
@@ -10,6 +10,9 @@ struct btf_id_set {
 	u32 ids[];
 };
 
+/* This flag implies BTF_SET8 holds kfunc(s) */
+#define BTF_SET8_KFUNCS		(1 << 0)
+
 struct btf_id_set8 {
 	u32 cnt;
 	u32 flags;
@@ -22,6 +25,7 @@ struct btf_id_set8 {
 #ifdef CONFIG_DEBUG_INFO_BTF
 
 #include <linux/compiler.h> /* for __PASTE */
+#include <linux/stringify.h>
 
 /*
  * Following macros help to define lists of BTF IDs placed
@@ -35,7 +39,7 @@ struct btf_id_set8 {
 
 #define BTF_IDS_SECTION ".BTF_ids"
 
-#define ____BTF_ID(symbol)				\
+#define ____BTF_ID(symbol, word)			\
 asm(							\
 ".pushsection " BTF_IDS_SECTION ",\"a\";       \n"	\
 ".local " #symbol " ;                          \n"	\
@@ -43,10 +47,11 @@ asm(							\
 ".size  " #symbol ", 4;                        \n"	\
 #symbol ":                                     \n"	\
 ".zero 4                                       \n"	\
+word							\
 ".popsection;                                  \n");
 
-#define __BTF_ID(symbol) \
-	____BTF_ID(symbol)
+#define __BTF_ID(symbol, word) \
+	____BTF_ID(symbol, word)
 
 #define __ID(prefix) \
 	__PASTE(__PASTE(prefix, __COUNTER__), __LINE__)
@@ -56,7 +61,14 @@ asm(							\
  * to 4 zero bytes.
  */
 #define BTF_ID(prefix, name) \
-	__BTF_ID(__ID(__BTF_ID__##prefix##__##name##__))
+	__BTF_ID(__ID(__BTF_ID__##prefix##__##name##__), "")
+
+#define ____BTF_ID_FLAGS(prefix, name, flags) \
+	__BTF_ID(__ID(__BTF_ID__##prefix##__##name##__), ".long " #flags "\n")
+#define __BTF_ID_FLAGS(prefix, name, flags, ...) \
+	____BTF_ID_FLAGS(prefix, name, flags)
+#define BTF_ID_FLAGS(prefix, name, ...) \
+	__BTF_ID_FLAGS(prefix, name, ##__VA_ARGS__, 0)
 
 /*
  * The BTF_ID_LIST macro defines pure (unsorted) list
@@ -155,10 +167,58 @@ asm(							\
 ".popsection;                                 \n");	\
 extern struct btf_id_set name;
 
+/*
+ * The BTF_SET8_START/END macros pair defines sorted list of
+ * BTF IDs and their flags plus its members count, with the
+ * following layout:
+ *
+ * BTF_SET8_START(list)
+ * BTF_ID_FLAGS(type1, name1, flags)
+ * BTF_ID_FLAGS(type2, name2, flags)
+ * BTF_SET8_END(list)
+ *
+ * __BTF_ID__set8__list:
+ * .zero 8
+ * list:
+ * __BTF_ID__type1__name1__3:
+ * .zero 4
+ * .word (1 << 0) | (1 << 2)
+ * __BTF_ID__type2__name2__5:
+ * .zero 4
+ * .word (1 << 3) | (1 << 1) | (1 << 2)
+ *
+ */
+#define __BTF_SET8_START(name, scope, flags)		\
+__BTF_ID_LIST(name, local)				\
+asm(							\
+".pushsection " BTF_IDS_SECTION ",\"a\";       \n"	\
+"." #scope " __BTF_ID__set8__" #name ";        \n"	\
+"__BTF_ID__set8__" #name ":;                   \n"	\
+".zero 4                                       \n"	\
+".long " __stringify(flags)                   "\n"	\
+".popsection;                                  \n");
+
+#define BTF_SET8_START(name)				\
+__BTF_SET8_START(name, local, 0)
+
+#define BTF_SET8_END(name)				\
+asm(							\
+".pushsection " BTF_IDS_SECTION ",\"a\";      \n"	\
+".size __BTF_ID__set8__" #name ", .-" #name "  \n"	\
+".popsection;                                 \n");	\
+extern struct btf_id_set8 name;
+
+#define BTF_KFUNCS_START(name)				\
+__BTF_SET8_START(name, local, BTF_SET8_KFUNCS)
+
+#define BTF_KFUNCS_END(name)				\
+BTF_SET8_END(name)
+
 #else
 
-#define BTF_ID_LIST(name) static u32 __maybe_unused name[5];
+#define BTF_ID_LIST(name) static u32 __maybe_unused name[128];
 #define BTF_ID(prefix, name)
+#define BTF_ID_FLAGS(prefix, name, ...)
 #define BTF_ID_UNUSED
 #define BTF_ID_LIST_GLOBAL(name, n) u32 __maybe_unused name[n];
 #define BTF_ID_LIST_SINGLE(name, prefix, typename) static u32 __maybe_unused name[1];
@@ -166,6 +226,10 @@ extern struct btf_id_set name;
 #define BTF_SET_START(name) static struct btf_id_set __maybe_unused name = { 0 };
 #define BTF_SET_START_GLOBAL(name) static struct btf_id_set __maybe_unused name = { 0 };
 #define BTF_SET_END(name)
+#define BTF_SET8_START(name) static struct btf_id_set8 __maybe_unused name = { 0 };
+#define BTF_SET8_END(name)
+#define BTF_KFUNCS_START(name) static struct btf_id_set8 __maybe_unused name = { .flags = BTF_SET8_KFUNCS };
+#define BTF_KFUNCS_END(name)
 
 #endif /* CONFIG_DEBUG_INFO_BTF */
 
@@ -215,5 +279,9 @@ MAX_BTF_TRACING_TYPE,
 };
 
 extern u32 btf_tracing_ids[];
+extern u32 bpf_cgroup_btf_id[];
+extern u32 bpf_local_storage_map_btf_id[];
+extern u32 btf_bpf_map_id[];
+extern u32 bpf_kmem_cache_btf_id[];
 
 #endif
diff --git a/tools/include/linux/compiler.h b/tools/include/linux/compiler.h
index f40bd2b04c298..f2f54b0381680 100644
--- a/tools/include/linux/compiler.h
+++ b/tools/include/linux/compiler.h
@@ -119,7 +119,7 @@
 #define __read_mostly
 
 #ifndef __attribute_const__
-# define __attribute_const__
+# define __attribute_const__ __attribute__ ((__const__))
 #endif
 
 #ifndef __maybe_unused
diff --git a/tools/include/linux/gfp_types.h b/tools/include/linux/gfp_types.h
index 6c75df30a281d..a93b8bd200b76 100644
--- a/tools/include/linux/gfp_types.h
+++ b/tools/include/linux/gfp_types.h
@@ -55,7 +55,6 @@ enum {
 #ifdef CONFIG_LOCKDEP
 	___GFP_NOLOCKDEP_BIT,
 #endif
-	___GFP_NO_OBJ_EXT_BIT,
 	___GFP_LAST_BIT
 };
 
@@ -96,7 +95,6 @@ enum {
 #else
 #define ___GFP_NOLOCKDEP	0
 #endif
-#define ___GFP_NO_OBJ_EXT       BIT(___GFP_NO_OBJ_EXT_BIT)
 
 /*
  * Physical address zone modifiers (see linux/mmzone.h - low four bits)
@@ -137,17 +135,12 @@ enum {
  * node with no fallbacks or placement policy enforcements.
  *
  * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg.
- *
- * %__GFP_NO_OBJ_EXT causes slab allocation to have no object extension.
- * mark_obj_codetag_empty() should be called upon freeing for objects allocated
- * with this flag to indicate that their NULL tags are expected and normal.
  */
 #define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE)
 #define __GFP_WRITE	((__force gfp_t)___GFP_WRITE)
 #define __GFP_HARDWALL   ((__force gfp_t)___GFP_HARDWALL)
 #define __GFP_THISNODE	((__force gfp_t)___GFP_THISNODE)
 #define __GFP_ACCOUNT	((__force gfp_t)___GFP_ACCOUNT)
-#define __GFP_NO_OBJ_EXT   ((__force gfp_t)___GFP_NO_OBJ_EXT)
 
 /**
  * DOC: Watermark modifiers
diff --git a/tools/include/nolibc/arch-sparc.h b/tools/include/nolibc/arch-sparc.h
index ddae9bc10dfe3..23fab40accfad 100644
--- a/tools/include/nolibc/arch-sparc.h
+++ b/tools/include/nolibc/arch-sparc.h
@@ -45,7 +45,7 @@
 									      \
 	__asm__ volatile (                                                    \
 		_NOLIBC_SYSCALL                                               \
-		: "+r"(_arg1)                                                 \
+		: "=r"(_arg1)                                                 \
 		: "r"(_num)                                                   \
 		: "memory", "cc"                                              \
 	);                                                                    \
diff --git a/tools/include/nolibc/unistd.h b/tools/include/nolibc/unistd.h
index 79599ceef45d9..a264a20da13dd 100644
--- a/tools/include/nolibc/unistd.h
+++ b/tools/include/nolibc/unistd.h
@@ -73,6 +73,48 @@ int ftruncate(int fd, off_t length)
 	return __sysret(_sys_ftruncate(fd, length));
 }
 
+/*
+ * char *getcwd(char *buf, size_t size);
+ */
+
+static __attribute__((unused))
+int _sys_getcwd(char *buf, size_t size)
+{
+	return __nolibc_syscall2(__NR_getcwd, buf, size);
+}
+
+static __attribute__((unused))
+char *getcwd(char *buf, size_t size)
+{
+	int ret;
+
+	/* Unlike other libc's we don't handle passing NULL for buf */
+	if (!buf || !size) {
+		SET_ERRNO(EINVAL);
+		return NULL;
+	}
+
+	ret = __sysret(_sys_getcwd(buf, size));
+
+	/* On error return NULL, __sysret() above will have set errno */
+	if (ret < 0)
+		return NULL;
+
+	/* Handle no path being written or the kernel putting
+	 * "(unreachable)" into the buffer instead of a path.
+	 * This matches what musl is doing.
+	 */
+	if (ret == 0 || buf[0] != '/') {
+		SET_ERRNO(ENOENT);
+		return NULL;
+	}
+
+	/* ret must be the number of bytes written at this point,
+	 * so return the pointer to buf.
+	 */
+	return buf;
+}
+
 static __attribute__((unused))
 int msleep(unsigned int msecs)
 {
@@ -86,6 +128,22 @@ int msleep(unsigned int msecs)
 		return 0;
 }
 
+/*
+ * ssize_t readlink(const char *path, char *buf, size_t bufsiz);
+ */
+
+static __attribute__((unused))
+ssize_t _sys_readlink(const char *path, char *buf, size_t bufsiz)
+{
+	return __nolibc_syscall4(__NR_readlinkat, AT_FDCWD, path, buf, bufsiz);
+}
+
+static __attribute__((unused))
+ssize_t readlink(const char *path, char *buf, size_t bufsiz)
+{
+	return __sysret(_sys_readlink(path, buf, bufsiz));
+}
+
 static __attribute__((unused))
 unsigned int sleep(unsigned int seconds)
 {
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 89b36de5fdbb6..2f1d24fef8573 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -5079,17 +5079,19 @@ union bpf_attr {
  * 	Description
  * 		Redirect the packet to another net device of index *ifindex*.
  * 		This helper is somewhat similar to **bpf_redirect**\ (), except
- * 		that the redirection happens to the *ifindex*' peer device and
- * 		the netns switch takes place from ingress to ingress without
- * 		going through the CPU's backlog queue.
+ * 		that the redirection happens to the *ifindex*' peer device. If
+ * 		*flags* is 0, the netns switch takes place from ingress to
+ * 		ingress without going through the CPU's backlog queue. If the
+ * 		**BPF_F_EGRESS** flag is provided then redirection happens in
+ * 		the egress direction of the peer device.
  *
  * 		*skb*\ **->mark** and *skb*\ **->tstamp** are not cleared during
  * 		the netns switch.
  *
- * 		The *flags* argument is reserved and must be 0. The helper is
- * 		currently only supported for tc BPF program types at the
- * 		ingress hook and for veth and netkit target device types. The
- * 		peer device must reside in a different network namespace.
+ * 		If the *flags* argument is 0, the helper is currently only
+ * 		supported for tc BPF program types at the ingress hook and for
+ * 		veth and netkit target device types. The peer device must reside
+ * 		in a different network namespace.
  * 	Return
  * 		The helper returns **TC_ACT_REDIRECT** on success or
  * 		**TC_ACT_SHOT** on error.
@@ -6336,9 +6338,10 @@ enum {
 /* Flags for bpf_redirect and bpf_redirect_map helpers */
 enum {
 	BPF_F_INGRESS		= (1ULL << 0), /* used for skb path */
+	BPF_F_EGRESS		= (1ULL << 1), /* used for skb path */
 	BPF_F_BROADCAST		= (1ULL << 3), /* used for XDP path */
 	BPF_F_EXCLUDE_INGRESS	= (1ULL << 4), /* used for XDP path */
-#define BPF_F_REDIRECT_FLAGS (BPF_F_INGRESS | BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS)
+#define BPF_F_REDIRECT_FLAGS (BPF_F_INGRESS | BPF_F_EGRESS | BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS)
 };
 
 #define __bpf_md_ptr(type, name)	\
@@ -6839,6 +6842,15 @@ struct bpf_link_info {
 			__u32 flags;
 			__u32 pid;
 		} uprobe_multi;
+		struct {
+			__u32 attach_type;
+			__u32 count; /* in/out: tracing_multi target count */
+			__u32 btf_obj_id;
+			__u32 :32;
+			__aligned_u64 ids;
+			__aligned_u64 addrs;
+			__aligned_u64 cookies;
+		} tracing_multi;
 		struct {
 			__u32 type; /* enum bpf_perf_event_type */
 			__u32 :32;
diff --git a/tools/include/uapi/linux/fs.h b/tools/include/uapi/linux/fs.h
index 24ddf7bc4f25f..f0a26309b6d54 100644
--- a/tools/include/uapi/linux/fs.h
+++ b/tools/include/uapi/linux/fs.h
@@ -364,6 +364,7 @@ typedef int __bitwise __kernel_rwf_t;
 #define PAGE_IS_HUGE		(1 << 6)
 #define PAGE_IS_SOFT_DIRTY	(1 << 7)
 #define PAGE_IS_GUARD		(1 << 8)
+#define PAGE_IS_ACCESSED	(1 << 9)
 
 /*
  * struct page_region - Page region with flags
diff --git a/tools/include/uapi/linux/fscrypt.h b/tools/include/uapi/linux/fscrypt.h
index 3aff99f2696a0..84507280b3ead 100644
--- a/tools/include/uapi/linux/fscrypt.h
+++ b/tools/include/uapi/linux/fscrypt.h
@@ -30,7 +30,6 @@
 #define FSCRYPT_MODE_SM4_CTS			8
 #define FSCRYPT_MODE_ADIANTUM			9
 #define FSCRYPT_MODE_AES_256_HCTR2		10
-/* If adding a mode number > 10, update FSCRYPT_MODE_MAX in fscrypt_private.h */
 
 /*
  * Legacy policy version; ad-hoc KDF and no key verification.
diff --git a/tools/include/uapi/linux/userfaultfd.h b/tools/include/uapi/linux/userfaultfd.h
index 4283de22d5b65..c8e99df95219f 100644
--- a/tools/include/uapi/linux/userfaultfd.h
+++ b/tools/include/uapi/linux/userfaultfd.h
@@ -25,7 +25,8 @@
 #define UFFD_API ((__u64)0xAA)
 #define UFFD_API_REGISTER_MODES (UFFDIO_REGISTER_MODE_MISSING |	\
 				 UFFDIO_REGISTER_MODE_WP |	\
-				 UFFDIO_REGISTER_MODE_MINOR)
+				 UFFDIO_REGISTER_MODE_MINOR |	\
+				 UFFDIO_REGISTER_MODE_RWP)
 #define UFFD_API_FEATURES (UFFD_FEATURE_PAGEFAULT_FLAG_WP |	\
 			   UFFD_FEATURE_EVENT_FORK |		\
 			   UFFD_FEATURE_EVENT_REMAP |		\
@@ -42,10 +43,13 @@
 			   UFFD_FEATURE_WP_UNPOPULATED |	\
 			   UFFD_FEATURE_POISON |		\
 			   UFFD_FEATURE_WP_ASYNC |		\
-			   UFFD_FEATURE_MOVE)
+			   UFFD_FEATURE_MOVE |			\
+			   UFFD_FEATURE_RWP |			\
+			   UFFD_FEATURE_RWP_ASYNC)
 #define UFFD_API_IOCTLS				\
 	((__u64)1 << _UFFDIO_REGISTER |		\
 	 (__u64)1 << _UFFDIO_UNREGISTER |	\
+	 (__u64)1 << _UFFDIO_SET_MODE |		\
 	 (__u64)1 << _UFFDIO_API)
 #define UFFD_API_RANGE_IOCTLS			\
 	((__u64)1 << _UFFDIO_WAKE |		\
@@ -54,13 +58,15 @@
 	 (__u64)1 << _UFFDIO_MOVE |		\
 	 (__u64)1 << _UFFDIO_WRITEPROTECT |	\
 	 (__u64)1 << _UFFDIO_CONTINUE |		\
-	 (__u64)1 << _UFFDIO_POISON)
+	 (__u64)1 << _UFFDIO_POISON |		\
+	 (__u64)1 << _UFFDIO_RWPROTECT)
 #define UFFD_API_RANGE_IOCTLS_BASIC		\
 	((__u64)1 << _UFFDIO_WAKE |		\
 	 (__u64)1 << _UFFDIO_COPY |		\
 	 (__u64)1 << _UFFDIO_WRITEPROTECT |	\
 	 (__u64)1 << _UFFDIO_CONTINUE |		\
-	 (__u64)1 << _UFFDIO_POISON)
+	 (__u64)1 << _UFFDIO_POISON |		\
+	 (__u64)1 << _UFFDIO_RWPROTECT)
 
 /*
  * Valid ioctl command number range with this API is from 0x00 to
@@ -79,6 +85,8 @@
 #define _UFFDIO_WRITEPROTECT		(0x06)
 #define _UFFDIO_CONTINUE		(0x07)
 #define _UFFDIO_POISON			(0x08)
+#define _UFFDIO_RWPROTECT		(0x09)
+#define _UFFDIO_SET_MODE		(0x0A)
 #define _UFFDIO_API			(0x3F)
 
 /* userfaultfd ioctl ids */
@@ -103,6 +111,10 @@
 				      struct uffdio_continue)
 #define UFFDIO_POISON		_IOWR(UFFDIO, _UFFDIO_POISON, \
 				      struct uffdio_poison)
+#define UFFDIO_RWPROTECT	_IOWR(UFFDIO, _UFFDIO_RWPROTECT,	\
+				      struct uffdio_rwprotect)
+#define UFFDIO_SET_MODE		_IOW(UFFDIO, _UFFDIO_SET_MODE,	\
+				     struct uffdio_set_mode)
 
 /* read() structure */
 struct uffd_msg {
@@ -158,6 +170,7 @@ struct uffd_msg {
 #define UFFD_PAGEFAULT_FLAG_WRITE	(1<<0)	/* If this was a write fault */
 #define UFFD_PAGEFAULT_FLAG_WP		(1<<1)	/* If reason is VM_UFFD_WP */
 #define UFFD_PAGEFAULT_FLAG_MINOR	(1<<2)	/* If reason is VM_UFFD_MINOR */
+#define UFFD_PAGEFAULT_FLAG_RWP		(1<<3)	/* If reason is VM_UFFD_RWP */
 
 struct uffdio_api {
 	/* userland asks for an API number and the features to enable */
@@ -230,6 +243,18 @@ struct uffdio_api {
 	 *
 	 * UFFD_FEATURE_MOVE indicates that the kernel supports moving an
 	 * existing page contents from userspace.
+	 *
+	 * UFFD_FEATURE_RWP indicates that the kernel supports
+	 * UFFDIO_REGISTER_MODE_RWP for read-write protection tracking.
+	 * Pages are made inaccessible via UFFDIO_RWPROTECT and faults
+	 * are delivered when the pages are re-accessed.
+	 *
+	 * UFFD_FEATURE_RWP_ASYNC indicates asynchronous mode for
+	 * UFFDIO_REGISTER_MODE_RWP.  When set, faults on read-write
+	 * protected pages are auto-resolved by the kernel (PTE
+	 * permissions restored immediately) without delivering a message
+	 * to the userfaultfd handler.  Use PAGEMAP_SCAN with inverted
+	 * PAGE_IS_ACCESSED to find pages that were not re-accessed.
 	 */
 #define UFFD_FEATURE_PAGEFAULT_FLAG_WP		(1<<0)
 #define UFFD_FEATURE_EVENT_FORK			(1<<1)
@@ -248,6 +273,8 @@ struct uffdio_api {
 #define UFFD_FEATURE_POISON			(1<<14)
 #define UFFD_FEATURE_WP_ASYNC			(1<<15)
 #define UFFD_FEATURE_MOVE			(1<<16)
+#define UFFD_FEATURE_RWP			(1<<17)
+#define UFFD_FEATURE_RWP_ASYNC			(1<<18)
 	__u64 features;
 
 	__u64 ioctls;
@@ -263,6 +290,7 @@ struct uffdio_register {
 #define UFFDIO_REGISTER_MODE_MISSING	((__u64)1<<0)
 #define UFFDIO_REGISTER_MODE_WP		((__u64)1<<1)
 #define UFFDIO_REGISTER_MODE_MINOR	((__u64)1<<2)
+#define UFFDIO_REGISTER_MODE_RWP	((__u64)1<<3)
 	__u64 mode;
 
 	/*
@@ -356,6 +384,14 @@ struct uffdio_poison {
 	__s64 updated;
 };
 
+struct uffdio_rwprotect {
+	struct uffdio_range range;
+	/* !RWP means undo RWP-protection */
+#define UFFDIO_RWPROTECT_MODE_RWP		((__u64)1<<0)
+#define UFFDIO_RWPROTECT_MODE_DONTWAKE		((__u64)1<<1)
+	__u64 mode;
+};
+
 struct uffdio_move {
 	__u64 dst;
 	__u64 src;
@@ -374,6 +410,16 @@ struct uffdio_move {
 	__s64 move;
 };
 
+struct uffdio_set_mode {
+	/*
+	 * Toggle async mode for features at runtime.
+	 * Supported: UFFD_FEATURE_RWP_ASYNC.
+	 * Setting a bit in both enable and disable is invalid.
+	 */
+	__u64 enable;
+	__u64 disable;
+};
+
 /*
  * Flags for the userfaultfd(2) system call itself.
  */
diff --git a/tools/lib/api/fd/array.c b/tools/lib/api/fd/array.c
index f0f195207fca9..67b73481df27e 100644
--- a/tools/lib/api/fd/array.c
+++ b/tools/lib/api/fd/array.c
@@ -31,7 +31,8 @@ int fdarray__grow(struct fdarray *fda, int nr)
 
 	priv = realloc(fda->priv, psize);
 	if (priv == NULL) {
-		free(entries);
+		/* this will be freed by fdarray__exit() */
+		fda->entries = entries;
 		return -ENOMEM;
 	}
 
@@ -50,7 +51,7 @@ struct fdarray *fdarray__new(int nr_alloc, int nr_autogrow)
 
 	if (fda != NULL) {
 		if (fdarray__grow(fda, nr_alloc)) {
-			free(fda);
+			fdarray__delete(fda);
 			fda = NULL;
 		} else {
 			fda->nr_autogrow = nr_autogrow;
@@ -115,6 +116,9 @@ int fdarray__filter(struct fdarray *fda, short revents,
 		return 0;
 
 	for (fd = 0; fd < fda->nr; ++fd) {
+		if (fda->priv[fd].flags & fdarray_flag__nonfilterable)
+			continue;
+
 		if (!fda->entries[fd].events)
 			continue;
 
@@ -122,12 +126,17 @@ int fdarray__filter(struct fdarray *fda, short revents,
 			if (entry_destructor)
 				entry_destructor(fda, fd, arg);
 
+			/*
+			 * Set fd to -1 so poll() ignores this entry; otherwise
+			 * POLLHUP/POLLERR are still reported for events=0 fds
+			 * (POSIX: always checked), causing a poll storm.
+			 */
+			fda->entries[fd].fd = -1;
 			fda->entries[fd].revents = fda->entries[fd].events = 0;
 			continue;
 		}
 
-		if (!(fda->priv[fd].flags & fdarray_flag__nonfilterable))
-			++nr;
+		++nr;
 	}
 
 	return nr;
diff --git a/tools/lib/bpf/bpf_gen_internal.h b/tools/lib/bpf/bpf_gen_internal.h
index 49af4260b8e6b..0425691877521 100644
--- a/tools/lib/bpf/bpf_gen_internal.h
+++ b/tools/lib/bpf/bpf_gen_internal.h
@@ -51,7 +51,6 @@ struct bpf_gen {
 	__u32 nr_ksyms;
 	int fd_array;
 	int nr_fd_array;
-	int hash_insn_offset[SHA256_DWORD_SIZE];
 };
 
 void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps);
diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c
index 823bce895178d..8417de92d0282 100644
--- a/tools/lib/bpf/btf.c
+++ b/tools/lib/bpf/btf.c
@@ -589,7 +589,7 @@ static int btf_parse_type_sec(struct btf *btf)
 		if (type_size < 0)
 			return type_size;
 		if (next_type + type_size > end_type) {
-			pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types);
+			pr_warn("BTF type [%u] is malformed\n", btf->start_id + btf->nr_types);
 			return -EINVAL;
 		}
 
@@ -1424,7 +1424,7 @@ static int btf_find_elf_sections(Elf *elf, const char *path, struct btf_elf_secs
 			continue;
 
 		if (sh.sh_type != SHT_PROGBITS) {
-			pr_warn("unexpected section type (%d) of section(%d, %s) from %s\n",
+			pr_warn("unexpected section type (%u) of section(%d, %s) from %s\n",
 				sh.sh_type, idx, name, path);
 			goto err;
 		}
@@ -1506,9 +1506,6 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf,
 		dist_base_btf = NULL;
 	}
 
-	if (dist_base_btf)
-		btf->owns_base = true;
-
 	switch (gelf_getclass(elf)) {
 	case ELFCLASS32:
 		btf__set_pointer_size(btf, 4);
@@ -1523,13 +1520,16 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf,
 
 	if (btf_ext && secs.btf_ext_data) {
 		*btf_ext = btf_ext__new(secs.btf_ext_data->d_buf, secs.btf_ext_data->d_size);
-		if (IS_ERR(*btf_ext)) {
-			err = PTR_ERR(*btf_ext);
+		if (!*btf_ext) {
+			err = -errno;
 			goto done;
 		}
 	} else if (btf_ext) {
 		*btf_ext = NULL;
 	}
+
+	if (dist_base_btf)
+		btf->owns_base = true;
 done:
 	if (elf)
 		elf_end(elf);
@@ -4854,7 +4854,7 @@ static bool btf_dedup_identical_types(struct btf_dedup *d, __u32 id1, __u32 id2,
 				continue;
 			if (!btf_dedup_identical_types(d, m1->type, m2->type, depth - 1)) {
 				if (t1->name_off) {
-					pr_debug("%s '%s' size=%d vlen=%d id1[%u] id2[%u] shallow-equal but not identical for field#%d '%s'\n",
+					pr_debug("%s '%s' size=%u vlen=%u id1[%u] id2[%u] shallow-equal but not identical for field#%d '%s'\n",
 						 k1 == BTF_KIND_STRUCT ? "STRUCT" : "UNION",
 						 btf__name_by_offset(d->btf, t1->name_off),
 						 t1->size, btf_vlen(t1), id1, id2, i,
@@ -5104,7 +5104,7 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
 			eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type);
 			if (eq <= 0) {
 				if (cand_type->name_off) {
-					pr_debug("%s '%s' size=%d vlen=%d cand_id[%u] canon_id[%u] shallow-equal but not equiv for field#%d '%s': %d\n",
+					pr_debug("%s '%s' size=%u vlen=%u cand_id[%u] canon_id[%u] shallow-equal but not equiv for field#%d '%s': %d\n",
 						 cand_kind == BTF_KIND_STRUCT ? "STRUCT" : "UNION",
 						 btf__name_by_offset(d->btf, cand_type->name_off),
 						 cand_type->size, vlen, cand_id, canon_id, i,
@@ -6069,7 +6069,7 @@ static int btf_add_distilled_types(struct btf_distill *dist)
 			err = btf_add_type(&dist->pipe, t);
 			break;
 		default:
-			pr_warn("unexpected kind when adding base type '%s'[%u] of kind [%u] to distilled base BTF.\n",
+			pr_warn("unexpected kind when adding base type '%s'[%d] of kind [%d] to distilled base BTF.\n",
 				name, i, kind);
 			return -EINVAL;
 
diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c
index cc1ba65bb6c5a..123c448f20c74 100644
--- a/tools/lib/bpf/btf_dump.c
+++ b/tools/lib/bpf/btf_dump.c
@@ -1776,7 +1776,7 @@ static int btf_dump_get_bitfield_value(struct btf_dump *d,
 
 	/* Maximum supported bitfield size is 64 bits */
 	if (t->size > 8) {
-		pr_warn("unexpected bitfield size %d\n", t->size);
+		pr_warn("unexpected bitfield size %u\n", t->size);
 		return -EINVAL;
 	}
 
@@ -2251,7 +2251,7 @@ static int btf_dump_get_enum_value(struct btf_dump *d,
 		*value = is_signed ? *(__s8 *)data : *(__u8 *)data;
 		return 0;
 	default:
-		pr_warn("unexpected size %d for enum, id:[%u]\n", t->size, id);
+		pr_warn("unexpected size %u for enum, id:[%u]\n", t->size, id);
 		return -EINVAL;
 	}
 }
diff --git a/tools/lib/bpf/btf_relocate.c b/tools/lib/bpf/btf_relocate.c
index 53d1f3541bce6..df5fa4bd87d69 100644
--- a/tools/lib/bpf/btf_relocate.c
+++ b/tools/lib/bpf/btf_relocate.c
@@ -280,7 +280,7 @@ static int btf_relocate_map_distilled_base(struct btf_relocate *r)
 		     cmp_btf_name_size(&base_info, dist_info) == 0;
 		     dist_info++) {
 			if (!dist_info->id || dist_info->id >= r->nr_dist_base_types) {
-				pr_warn("base BTF id [%d] maps to invalid distilled base BTF id [%d]\n",
+				pr_warn("base BTF id [%u] maps to invalid distilled base BTF id [%u]\n",
 					id, dist_info->id);
 				err = -EINVAL;
 				goto done;
@@ -368,7 +368,7 @@ static int btf_relocate_map_distilled_base(struct btf_relocate *r)
 			continue;
 		dist_t = btf_type_by_id(r->dist_base_btf, id);
 		name = btf__name_by_offset(r->dist_base_btf, dist_t->name_off);
-		pr_warn("distilled base BTF type '%s' [%d] is not mapped to base BTF id\n",
+		pr_warn("distilled base BTF type '%s' [%u] is not mapped to base BTF id\n",
 			name, id);
 		err = -EINVAL;
 		break;
@@ -397,11 +397,11 @@ static int btf_relocate_validate_distilled_base(struct btf_relocate *r)
 		case BTF_KIND_FWD:
 			if (t->name_off)
 				break;
-			pr_warn("type [%d], kind [%d] is invalid for distilled base BTF; it is anonymous\n",
+			pr_warn("type [%u], kind [%d] is invalid for distilled base BTF; it is anonymous\n",
 				i, kind);
 			return -EINVAL;
 		default:
-			pr_warn("type [%d] in distilled based BTF has unexpected kind [%d]\n",
+			pr_warn("type [%u] in distilled based BTF has unexpected kind [%d]\n",
 				i, kind);
 			return -EINVAL;
 		}
diff --git a/tools/lib/bpf/elf.c b/tools/lib/bpf/elf.c
index 295dbda245807..fe136d0259679 100644
--- a/tools/lib/bpf/elf.c
+++ b/tools/lib/bpf/elf.c
@@ -354,7 +354,7 @@ long elf_find_func_offset(Elf *elf, const char *binary_path, const char *name)
 
 	if (ret > 0) {
 		pr_debug("elf: symbol address match for '%s' in '%s': 0x%lx\n", name, binary_path,
-			 ret);
+			 (unsigned long)ret);
 	} else {
 		if (ret == 0) {
 			pr_warn("elf: '%s' is 0 in symtab for '%s': %s\n", name, binary_path,
diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c
index b7e388f99d0bb..e5641fa601637 100644
--- a/tools/lib/bpf/features.c
+++ b/tools/lib/bpf/features.c
@@ -577,10 +577,10 @@ static int probe_ldimm64_full_range_off(int token_fd)
 static int probe_uprobe_syscall(int token_fd)
 {
 	/*
-	 * If kernel supports uprobe() syscall, it will return -ENXIO when called
+	 * If kernel supports uprobe() syscall, it will return -EPROTO when called
 	 * from the outside of a kernel-generated uprobe trampoline.
 	 */
-	return syscall(__NR_uprobe) < 0 && errno == ENXIO;
+	return syscall(__NR_uprobe) < 0 && errno == EPROTO;
 }
 #else
 static int probe_uprobe_syscall(int token_fd)
diff --git a/tools/lib/bpf/gen_loader.c b/tools/lib/bpf/gen_loader.c
index d79695f01c87f..6e3dd52427618 100644
--- a/tools/lib/bpf/gen_loader.c
+++ b/tools/lib/bpf/gen_loader.c
@@ -111,7 +111,6 @@ static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn in
 
 static int add_data(struct bpf_gen *gen, const void *data, __u32 size);
 static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off);
-static void emit_signature_match(struct bpf_gen *gen);
 
 void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps)
 {
@@ -154,8 +153,6 @@ void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps
 	/* R7 contains the error code from sys_bpf. Copy it into R0 and exit. */
 	emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7));
 	emit(gen, BPF_EXIT_INSN());
-	if (OPTS_GET(gen->opts, gen_hash, false))
-		emit_signature_match(gen);
 }
 
 static int add_data(struct bpf_gen *gen, const void *data, __u32 size)
@@ -377,14 +374,12 @@ static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off)
 	__emit_sys_close(gen);
 }
 
-static void compute_sha_update_offsets(struct bpf_gen *gen);
-
 int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps)
 {
 	int i;
 
 	if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) {
-		pr_warn("nr_progs %d/%d nr_maps %d/%d mismatch\n",
+		pr_warn("nr_progs %d/%u nr_maps %d/%u mismatch\n",
 			nr_progs, gen->nr_progs, nr_maps, gen->nr_maps);
 		gen->error = -EFAULT;
 		return gen->error;
@@ -408,9 +403,6 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps)
 	if (!gen->error) {
 		struct gen_loader_opts *opts = gen->opts;
 
-		if (OPTS_GET(opts, gen_hash, false))
-			compute_sha_update_offsets(gen);
-
 		opts->insns = gen->insn_start;
 		opts->insns_sz = gen->insn_cur - gen->insn_start;
 		opts->data = gen->data_start;
@@ -460,22 +452,6 @@ void bpf_gen__free(struct bpf_gen *gen)
 	_val;							\
 })
 
-static void compute_sha_update_offsets(struct bpf_gen *gen)
-{
-	__u64 sha[SHA256_DWORD_SIZE];
-	__u64 sha_dw;
-	int i;
-
-	libbpf_sha256(gen->data_start, gen->data_cur - gen->data_start, (__u8 *)sha);
-	for (i = 0; i < SHA256_DWORD_SIZE; i++) {
-		struct bpf_insn *insn =
-			(struct bpf_insn *)(gen->insn_start + gen->hash_insn_offset[i]);
-		sha_dw = tgt_endian(sha[i]);
-		insn[0].imm = (__u32)sha_dw;
-		insn[1].imm = sha_dw >> 32;
-	}
-}
-
 void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data,
 		       __u32 btf_raw_size)
 {
@@ -488,7 +464,7 @@ void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data,
 
 	attr.btf_size = tgt_endian(btf_raw_size);
 	btf_load_attr = add_data(gen, &attr, attr_size);
-	pr_debug("gen: load_btf: off %d size %d, attr: off %d size %d\n",
+	pr_debug("gen: load_btf: off %d size %u, attr: off %d size %d\n",
 		 btf_data, btf_raw_size, btf_load_attr, attr_size);
 
 	/* populate union bpf_attr with user provided log details */
@@ -534,7 +510,7 @@ void bpf_gen__map_create(struct bpf_gen *gen,
 	attr.btf_value_type_id = tgt_endian(map_attr->btf_value_type_id);
 
 	map_create_attr = add_data(gen, &attr, attr_size);
-	pr_debug("gen: map_create: %s idx %d type %d value_type_id %d, attr: off %d size %d\n",
+	pr_debug("gen: map_create: %s idx %d type %u value_type_id %u, attr: off %d size %d\n",
 		 map_name, map_idx, map_type, map_attr->btf_value_type_id,
 		 map_create_attr, attr_size);
 
@@ -557,8 +533,9 @@ void bpf_gen__map_create(struct bpf_gen *gen,
 	 * Conditionally update max_entries from the host-supplied loader
 	 * ctx. This sizes the map at runtime, but for a signed loader
 	 * (gen_hash) it would let an untrusted host re-dimension the
-	 * program's maps after emit_signature_match(), outside what the
-	 * signature attests to. Keep the signer-provided max_entries
+	 * program's maps, outside what the signature attests to: the
+	 * metadata blob is covered by the program signature and verified
+	 * by the kernel at load time. Keep the signer-provided max_entries
 	 * baked into the blob in that case.
 	 */
 	if (map_idx >= 0 && !OPTS_GET(gen->opts, gen_hash, false))
@@ -596,45 +573,6 @@ void bpf_gen__map_create(struct bpf_gen *gen,
 		emit_sys_close_stack(gen, stack_off(inner_map_fd));
 }
 
-static void emit_signature_match(struct bpf_gen *gen)
-{
-	__s64 off;
-	int i;
-
-	/*
-	 * Reject if the metadata map is not exclusive. Without exclusivity
-	 * the cached map->sha[] verified above can be stale: another BPF
-	 * program with map access could have mutated the contents between
-	 * BPF_OBJ_GET_INFO_BY_FD and loader execution.
-	 */
-	emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX,
-					 0, 0, 0, 0));
-	emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, SHA256_DIGEST_LENGTH));
-	off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 2;
-	if (is_simm16(off)) {
-		emit(gen, BPF_MOV64_IMM(BPF_REG_7, -EINVAL));
-		emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_2, 1, off));
-	} else {
-		gen->error = -ERANGE;
-	}
-
-	for (i = 0; i < SHA256_DWORD_SIZE; i++) {
-		emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX,
-						 0, 0, 0, 0));
-		emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, i * sizeof(__u64)));
-		gen->hash_insn_offset[i] = gen->insn_cur - gen->insn_start;
-		emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_3, 0, 0, 0, 0, 0));
-
-		off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 2;
-		if (is_simm16(off)) {
-			emit(gen, BPF_MOV64_IMM(BPF_REG_7, -EINVAL));
-			emit(gen, BPF_JMP_REG(BPF_JNE, BPF_REG_2, BPF_REG_3, off));
-		} else {
-			gen->error = -ERANGE;
-		}
-	}
-}
-
 void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name,
 				   enum bpf_attach_type type)
 {
@@ -1082,7 +1020,7 @@ void bpf_gen__prog_load(struct bpf_gen *gen,
 	license_off = add_data(gen, license, strlen(license) + 1);
 	/* add insns to blob of bytes */
 	insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn));
-	pr_debug("gen: prog_load: prog_idx %d type %d insn off %d insns_cnt %zd license off %d\n",
+	pr_debug("gen: prog_load: prog_idx %d type %u insn off %d insns_cnt %zu license off %d\n",
 		 prog_idx, prog_type, insns_off, insn_cnt, license_off);
 
 	/* convert blob insns to target endianness */
@@ -1105,21 +1043,21 @@ void bpf_gen__prog_load(struct bpf_gen *gen,
 	attr.func_info_rec_size = tgt_endian(load_attr->func_info_rec_size);
 	attr.func_info_cnt = tgt_endian(load_attr->func_info_cnt);
 	func_info = add_data(gen, load_attr->func_info, func_info_tot_sz);
-	pr_debug("gen: prog_load: func_info: off %d cnt %d rec size %d\n",
+	pr_debug("gen: prog_load: func_info: off %d cnt %u rec size %u\n",
 		 func_info, load_attr->func_info_cnt,
 		 load_attr->func_info_rec_size);
 
 	attr.line_info_rec_size = tgt_endian(load_attr->line_info_rec_size);
 	attr.line_info_cnt = tgt_endian(load_attr->line_info_cnt);
 	line_info = add_data(gen, load_attr->line_info, line_info_tot_sz);
-	pr_debug("gen: prog_load: line_info: off %d cnt %d rec size %d\n",
+	pr_debug("gen: prog_load: line_info: off %d cnt %u rec size %u\n",
 		 line_info, load_attr->line_info_cnt,
 		 load_attr->line_info_rec_size);
 
 	attr.core_relo_rec_size = tgt_endian((__u32)sizeof(struct bpf_core_relo));
 	attr.core_relo_cnt = tgt_endian(gen->core_relo_cnt);
 	core_relos = add_data(gen, gen->core_relos, core_relo_tot_sz);
-	pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zd\n",
+	pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zu\n",
 		 core_relos, gen->core_relo_cnt,
 		 sizeof(struct bpf_core_relo));
 
@@ -1211,10 +1149,10 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue,
 	 * }
 	 *
 	 * The runtime initial_value comes from the host-supplied loader
-	 * ctx and would overwrite the blob value after emit_signature_match()
-	 * has already validated map->sha[]. For a signed loader (gen_hash)
-	 * the attested blob value must be authoritative, so skip the override
-	 * and leave the hashed value in place.
+	 * ctx and would overwrite the blob value that the program signature
+	 * covers and the kernel verifies at load time. For a signed loader
+	 * (gen_hash) the attested blob value must be authoritative, so skip
+	 * the override and leave the signed value in place.
 	 */
 	if (!OPTS_GET(gen->opts, gen_hash, false)) {
 		emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6,
@@ -1234,7 +1172,7 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue,
 	}
 
 	map_update_attr = add_data(gen, &attr, attr_size);
-	pr_debug("gen: map_update_elem: idx %d, value: off %d size %d, attr: off %d size %d\n",
+	pr_debug("gen: map_update_elem: idx %d, value: off %d size %u, attr: off %d size %d\n",
 		 map_idx, value, value_size, map_update_attr, attr_size);
 	move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
 		       blob_fd_array_off(gen, map_idx));
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 1368752aa13c3..f88b7c8de3046 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -1486,7 +1486,7 @@ static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name,
 		       type->size);
 		st_ops->type_id = type_id;
 
-		pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
+		pr_debug("struct_ops init: struct %s(type_id=%d) %s found at offset %u\n",
 			 tname, type_id, var_name, vsi->offset);
 	}
 
@@ -2626,7 +2626,7 @@ int parse_btf_map_def(const char *map_name, struct btf *btf,
 
 			t = btf__type_by_id(btf, m->type);
 			if (!t) {
-				pr_warn("map '%s': key type [%d] not found.\n",
+				pr_warn("map '%s': key type [%u] not found.\n",
 					map_name, m->type);
 				return -EINVAL;
 			}
@@ -2666,7 +2666,7 @@ int parse_btf_map_def(const char *map_name, struct btf *btf,
 
 			t = btf__type_by_id(btf, m->type);
 			if (!t) {
-				pr_warn("map '%s': value type [%d] not found.\n",
+				pr_warn("map '%s': value type [%u] not found.\n",
 					map_name, m->type);
 				return -EINVAL;
 			}
@@ -2720,7 +2720,7 @@ int parse_btf_map_def(const char *map_name, struct btf *btf,
 			map_def->value_size = 4;
 			t = btf__type_by_id(btf, m->type);
 			if (!t) {
-				pr_warn("map '%s': %s type [%d] not found.\n",
+				pr_warn("map '%s': %s type [%u] not found.\n",
 					map_name, desc, m->type);
 				return -EINVAL;
 			}
@@ -3476,7 +3476,7 @@ static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf,
 
 		var_name = btf__name_by_offset(btf, t_var->name_off);
 		if (!var_name) {
-			pr_debug("sec '%s': failed to find name of DATASEC's member #%d\n",
+			pr_debug("sec '%s': failed to find name of DATASEC's member #%u\n",
 				 sec_name, i);
 			return -ENOENT;
 		}
@@ -3971,7 +3971,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj)
 		if (!data)
 			return -LIBBPF_ERRNO__FORMAT;
 
-		pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
+		pr_debug("elf: section(%d) %s, size %lu, link %d, flags %lx, type=%d\n",
 			 idx, name, (unsigned long)data->d_size,
 			 (int)sh->sh_link, (unsigned long)sh->sh_flags,
 			 (int)sh->sh_type);
@@ -4494,7 +4494,7 @@ static int bpf_object__collect_externs(struct bpf_object *obj)
 
 			ext->kcfg.data_off = roundup(off, ext->kcfg.align);
 			off = ext->kcfg.data_off + ext->kcfg.sz;
-			pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n",
+			pr_debug("extern (kcfg) #%d: symbol %d, off %d, name %s\n",
 				 i, ext->sym_idx, ext->kcfg.data_off, ext->name);
 		}
 		sec->size = off;
@@ -4626,7 +4626,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog,
 	struct bpf_map *map;
 
 	if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) {
-		pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
+		pr_warn("prog '%s': invalid relo against '%s' for insns[%u].code 0x%x\n",
 			prog->name, sym_name, insn_idx, insn->code);
 		return -LIBBPF_ERRNO__RELOC;
 	}
@@ -4749,7 +4749,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog,
 			    map->sec_idx != sym->st_shndx ||
 			    map->sec_offset != sym->st_value)
 				continue;
-			pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
+			pr_debug("prog '%s': found map %zu (%s, sec %d, off %zu) for insn #%u\n",
 				 prog->name, map_idx, map->name, map->sec_idx,
 				 map->sec_offset, insn_idx);
 			break;
@@ -4776,7 +4776,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog,
 		map = &obj->maps[map_idx];
 		if (map->libbpf_type != type || map->sec_idx != sym->st_shndx)
 			continue;
-		pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
+		pr_debug("prog '%s': found data map %zu (%s, sec %d, off %zu) for insn %u\n",
 			 prog->name, map_idx, map->name, map->sec_idx,
 			 map->sec_offset, insn_idx);
 		break;
@@ -4985,7 +4985,7 @@ static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info)
 			info->value_size = val;
 		else if (sscanf(buff, "max_entries:\t%u", &val) == 1)
 			info->max_entries = val;
-		else if (sscanf(buff, "map_flags:\t%i", &val) == 1)
+		else if (sscanf(buff, "map_flags:\t%x", &val) == 1)
 			info->map_flags = val;
 	}
 
@@ -5172,12 +5172,8 @@ bpf_object__probe_loading(struct bpf_object *obj)
 		BPF_EXIT_INSN(),
 	};
 	int ret, insn_cnt = ARRAY_SIZE(insns);
-	LIBBPF_OPTS(bpf_prog_load_opts, opts,
-		.token_fd = obj->token_fd,
-		.prog_flags = obj->token_fd ? BPF_F_TOKEN_FD : 0,
-	);
 
-	if (obj->gen_loader)
+	if (obj->gen_loader || obj->token_fd)
 		return 0;
 
 	ret = bump_rlimit_memlock();
@@ -5186,9 +5182,9 @@ bpf_object__probe_loading(struct bpf_object *obj)
 			errstr(ret));
 
 	/* make sure basic loading works */
-	ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &opts);
+	ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
 	if (ret < 0)
-		ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts);
+		ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL);
 	if (ret < 0) {
 		ret = errno;
 		pr_warn("Error in %s(): %s. Couldn't load trivial BPF program. Make sure your kernel supports BPF (CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is set to big enough value.\n",
@@ -5521,11 +5517,11 @@ static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map)
 		}
 		if (err) {
 			err = -errno;
-			pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %s\n",
+			pr_warn("map '%s': failed to initialize slot [%u] to map '%s' fd=%d: %s\n",
 				map->name, i, targ_map->name, fd, errstr(err));
 			return err;
 		}
-		pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
+		pr_debug("map '%s': slot [%u] set to map '%s' fd=%d\n",
 			 map->name, i, targ_map->name, fd);
 	}
 
@@ -5554,11 +5550,11 @@ static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map)
 		err = bpf_map_update_elem(map->fd, &i, &fd, 0);
 		if (err) {
 			err = -errno;
-			pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %s\n",
+			pr_warn("map '%s': failed to initialize slot [%u] to prog '%s' fd=%d: %s\n",
 				map->name, i, targ_prog->name, fd, errstr(err));
 			return err;
 		}
-		pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n",
+		pr_debug("map '%s': slot [%u] set to prog '%s' fd=%d\n",
 			 map->name, i, targ_prog->name, fd);
 	}
 
@@ -5788,7 +5784,7 @@ int bpf_core_add_cands(struct bpf_core_cand *local_cand,
 		if (strncmp(local_name, targ_name, local_essent_len) != 0)
 			continue;
 
-		pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
+		pr_debug("CO-RE relocating [%u] %s %s: found target candidate [%d] %s %s in [%s]\n",
 			 local_cand->id, btf_kind_str(local_t),
 			 local_name, i, btf_kind_str(t), targ_name,
 			 targ_btf_name);
@@ -5848,7 +5844,7 @@ static int load_module_btfs(struct bpf_object *obj)
 			if (errno == ENOENT)
 				continue; /* expected race: BTF was unloaded */
 			err = -errno;
-			pr_warn("failed to get BTF object #%d FD: %s\n", id, errstr(err));
+			pr_warn("failed to get BTF object #%u FD: %s\n", id, errstr(err));
 			return err;
 		}
 
@@ -5861,7 +5857,7 @@ static int load_module_btfs(struct bpf_object *obj)
 		err = bpf_btf_get_info_by_fd(fd, &info, &len);
 		if (err) {
 			err = -errno;
-			pr_warn("failed to get BTF object #%d info: %s\n", id, errstr(err));
+			pr_warn("failed to get BTF object #%u info: %s\n", id, errstr(err));
 			break;
 		}
 
@@ -5874,7 +5870,7 @@ static int load_module_btfs(struct bpf_object *obj)
 		btf = btf_get_from_fd(fd, obj->btf_vmlinux);
 		err = libbpf_get_error(btf);
 		if (err) {
-			pr_warn("failed to load module [%s]'s BTF object #%d: %s\n",
+			pr_warn("failed to load module [%s]'s BTF object #%u: %s\n",
 				name, id, errstr(err));
 			break;
 		}
@@ -6067,7 +6063,7 @@ static int bpf_core_resolve_relo(struct bpf_program *prog,
 	    !hashmap__find(cand_cache, local_id, &cands)) {
 		cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
 		if (IS_ERR(cands)) {
-			pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
+			pr_warn("prog '%s': relo #%d: target candidate search failed for [%u] %s %s: %ld\n",
 				prog_name, relo_idx, local_id, btf_kind_str(local_type),
 				local_name, PTR_ERR(cands));
 			return PTR_ERR(cands);
@@ -6127,7 +6123,7 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
 			goto out;
 		}
 
-		pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info);
+		pr_debug("sec '%s': found %u CO-RE relocations\n", sec_name, sec->num_info);
 
 		for_each_btf_ext_rec(seg, sec, i, rec) {
 			if (rec->insn_off % BPF_INSN_SZ)
@@ -6181,7 +6177,7 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
 
 			err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
 			if (err) {
-				pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %s\n",
+				pr_warn("prog '%s': relo #%d: failed to patch insn #%d: %s\n",
 					prog->name, i, insn_idx, errstr(err));
 				goto out;
 			}
@@ -6346,7 +6342,7 @@ static int create_jt_map(struct bpf_object *obj, struct bpf_program *prog, struc
 		goto err_close;
 	}
 	if (sym_off + jt_size > obj->jumptables_data_sz) {
-		pr_warn("map '.jumptables': jumptables_data size is %zd, trying to access %d\n",
+		pr_warn("map '.jumptables': jumptables_data size is %zu, trying to access %u\n",
 			obj->jumptables_data_sz, sym_off + jt_size);
 		err = -EINVAL;
 		goto err_close;
@@ -6381,7 +6377,7 @@ static int create_jt_map(struct bpf_object *obj, struct bpf_program *prog, struc
 		 */
 		if (insn_off > UINT32_MAX) {
 			pr_warn("map '.jumptables': invalid jump table value 0x%llx at offset %u\n",
-				(long long)jt[i], sym_off + i * jt_entry_size);
+				(unsigned long long)jt[i], sym_off + i * jt_entry_size);
 			err = -EINVAL;
 			goto err_close;
 		}
@@ -6517,7 +6513,7 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
 		}
 			break;
 		default:
-			pr_warn("prog '%s': relo #%d: bad relo type %d\n",
+			pr_warn("prog '%s': relo #%d: bad relo type %u\n",
 				prog->name, i, relo->type);
 			return -EINVAL;
 		}
@@ -6797,7 +6793,7 @@ bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog,
 			 */
 			continue;
 		if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {
-			pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",
+			pr_warn("prog '%s': unexpected relo for insn #%zu, type %u\n",
 				prog->name, insn_idx, relo->type);
 			return -LIBBPF_ERRNO__RELOC;
 		}
@@ -7587,7 +7583,7 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj,
 		}
 		name = elf_sym_str(obj, sym->st_name) ?: "<?>";
 
-		pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n",
+		pr_debug(".maps relo #%d: for %zd value %zu rel->r_offset %zu name %u ('%s')\n",
 			 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value,
 			 (size_t)rel->r_offset, sym->st_name, name);
 
@@ -7678,7 +7674,7 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj,
 		}
 		map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog;
 
-		pr_debug(".maps relo #%d: map '%s' slot [%d] points to %s '%s'\n",
+		pr_debug(".maps relo #%d: map '%s' slot [%u] points to %s '%s'\n",
 			 i, map->name, moff, type, name);
 	}
 
@@ -8738,7 +8734,7 @@ static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj,
 		local_name = btf__name_by_offset(obj->btf, local_type->name_off);
 		targ_name = btf__name_by_offset(btf, targ_type->name_off);
 
-		pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
+		pr_warn("extern (var ksym) '%s': incompatible types, expected [%u] %s %s, but kernel has [%u] %s %s\n",
 			ext->name, local_type_id,
 			btf_kind_str(local_type), local_name, targ_type_id,
 			btf_kind_str(targ_type), targ_name);
@@ -8915,7 +8911,7 @@ static int bpf_object__resolve_externs(struct bpf_object *obj,
 			if (err)
 				return err;
 			pr_debug("extern (kcfg) '%s': set to 0x%llx\n",
-				 ext->name, (long long)value);
+				 ext->name, (unsigned long long)value);
 		} else {
 			pr_warn("extern '%s': unrecognized extern kind\n", ext->name);
 			return -EINVAL;
@@ -10494,7 +10490,7 @@ static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
 		moff = rel->r_offset - map->sec_offset;
 		shdr_idx = sym->st_shndx;
 		st_ops = map->st_ops;
-		pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
+		pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %u (\'%s\')\n",
 			 map->name,
 			 (long long)(rel->r_info >> 32),
 			 (long long)sym->st_value,
@@ -10643,7 +10639,7 @@ static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int t
 	memset(&info, 0, info_len);
 	err = bpf_prog_get_info_by_fd(attach_prog_fd, &info, &info_len);
 	if (err) {
-		pr_warn("failed bpf_prog_get_info_by_fd for FD %d: %s\n",
+		pr_warn("failed bpf_prog_get_info_by_fd for FD %u: %s\n",
 			attach_prog_fd, errstr(err));
 		return err;
 	}
@@ -10656,7 +10652,7 @@ static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int t
 	btf = btf_load_from_kernel(info.btf_id, NULL, token_fd);
 	err = libbpf_get_error(btf);
 	if (err) {
-		pr_warn("Failed to get BTF %d of the program: %s\n", info.btf_id, errstr(err));
+		pr_warn("Failed to get BTF %u of the program: %s\n", info.btf_id, errstr(err));
 		goto out;
 	}
 	err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
@@ -10738,7 +10734,7 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attac
 		}
 		err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd, prog->obj->token_fd);
 		if (err < 0) {
-			pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %s\n",
+			pr_warn("prog '%s': failed to find BPF program (FD %u) BTF ID for '%s': %s\n",
 				prog->name, attach_prog_fd, attach_name, errstr(err));
 			return err;
 		}
@@ -11233,7 +11229,7 @@ static int validate_map_op(const struct bpf_map *map, size_t key_sz,
 		}
 
 		if (value_sz != num_cpu * elem_sz) {
-			pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n",
+			pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zu\n",
 				map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);
 			return -EINVAL;
 		}
@@ -11774,7 +11770,7 @@ static void gen_probe_legacy_event_name(char *buf, size_t buf_sz,
 	static int index = 0;
 	int i;
 
-	snprintf(buf, buf_sz, "libbpf_%u_%d_%s_0x%zx", getpid(),
+	snprintf(buf, buf_sz, "libbpf_%d_%d_%s_0x%zx", getpid(),
 		 __sync_fetch_and_add(&index, 1), name, offset);
 
 	/* sanitize name in the probe name */
@@ -12924,8 +12920,8 @@ static long elf_find_func_offset_from_archive(const char *archive_path, const ch
 	ret = elf_find_func_offset(elf, file_name, func_name);
 	if (ret > 0) {
 		pr_debug("elf: symbol address match for %s of %s in %s: 0x%x + 0x%lx = 0x%lx\n",
-			 func_name, file_name, archive_path, entry.data_offset, ret,
-			 ret + entry.data_offset);
+			 func_name, file_name, archive_path, entry.data_offset, (unsigned long)ret,
+			 (unsigned long)(ret + entry.data_offset));
 		ret += entry.data_offset;
 	}
 	elf_end(elf);
@@ -14570,7 +14566,7 @@ perf_buffer__process_record(struct perf_event_header *e, void *ctx)
 		break;
 	}
 	default:
-		pr_warn("unknown perf sample type %d\n", e->type);
+		pr_warn("unknown perf sample type %u\n", e->type);
 		return LIBBPF_PERF_EVENT_ERROR;
 	}
 	return LIBBPF_PERF_EVENT_CONT;
diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h
index 04cd303fb5a87..d5b7db703b3fe 100644
--- a/tools/lib/bpf/libbpf_internal.h
+++ b/tools/lib/bpf/libbpf_internal.h
@@ -768,7 +768,6 @@ int elf_resolve_pattern_offsets(const char *binary_path, const char *pattern,
 int probe_fd(int fd);
 
 #define SHA256_DIGEST_LENGTH 32
-#define SHA256_DWORD_SIZE SHA256_DIGEST_LENGTH / sizeof(__u64)
 
 void libbpf_sha256(const void *data, size_t len, __u8 out[SHA256_DIGEST_LENGTH]);
 int probe_sys_bpf_ext(void);
diff --git a/tools/lib/bpf/nlattr.c b/tools/lib/bpf/nlattr.c
index 06663f9ea581f..007fe17d17b42 100644
--- a/tools/lib/bpf/nlattr.c
+++ b/tools/lib/bpf/nlattr.c
@@ -123,7 +123,7 @@ int libbpf_nla_parse(struct nlattr *tb[], int maxtype, struct nlattr *head,
 
 		if (tb[type]) {
 			pr_warn("Attribute of type %#x found multiple times in message, "
-				"previous attribute is being ignored.\n", type);
+				"previous attribute is being ignored.\n", (unsigned)type);
 		}
 
 		tb[type] = nla;
diff --git a/tools/lib/bpf/relo_core.c b/tools/lib/bpf/relo_core.c
index 6ae3f2a15ad0c..8ad2715721cfd 100644
--- a/tools/lib/bpf/relo_core.c
+++ b/tools/lib/bpf/relo_core.c
@@ -216,7 +216,7 @@ int __bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
 		goto recur;
 	}
 	default:
-		pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n",
+		pr_warn("unexpected kind %s relocated, local [%u], target [%u]\n",
 			btf_kind_str(local_type), local_id, targ_id);
 		return 0;
 	}
@@ -384,7 +384,7 @@ int bpf_core_parse_spec(const char *prog_name, const struct btf *btf,
 				return sz;
 			spec->bit_offset += access_idx * sz * 8;
 		} else {
-			pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n",
+			pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%u] of unexpected kind %s\n",
 				prog_name, relo->type_id, spec_str, i, id, btf_kind_str(t));
 			return -EINVAL;
 		}
@@ -725,7 +725,7 @@ static int bpf_core_calc_field_relo(const char *prog_name,
 				return -EINVAL;
 			*val = sz;
 		} else {
-			pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n",
+			pr_warn("prog '%s': relo %u at insn #%u can't be applied to array access\n",
 				prog_name, relo->kind, relo->insn_off / 8);
 			return -EINVAL;
 		}
@@ -747,7 +747,7 @@ static int bpf_core_calc_field_relo(const char *prog_name,
 		while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) {
 			if (byte_sz >= 8) {
 				/* bitfield can't be read with 64-bit read */
-				pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n",
+				pr_warn("prog '%s': relo %u at insn #%u can't be satisfied for bitfield\n",
 					prog_name, relo->kind, relo->insn_off / 8);
 				return -E2BIG;
 			}
@@ -971,7 +971,7 @@ static int bpf_core_calc_relo(const char *prog_name,
 		err = 0;
 	} else if (err == -EOPNOTSUPP) {
 		/* EOPNOTSUPP means unknown/unsupported relocation */
-		pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%d) at insn #%d\n",
+		pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%u) at insn #%u\n",
 			prog_name, relo_idx, core_relo_kind_str(relo->kind),
 			relo->kind, relo->insn_off / 8);
 	}
@@ -1067,7 +1067,7 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn,
 		if (BPF_SRC(insn->code) != BPF_K)
 			return -EINVAL;
 		if (res->validate && insn->imm != orig_val) {
-			pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %llu -> %llu\n",
+			pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %d, exp %llu -> %llu\n",
 				prog_name, relo_idx,
 				insn_idx, insn->imm, (unsigned long long)orig_val,
 				(unsigned long long)new_val);
@@ -1083,7 +1083,7 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn,
 	case BPF_ST:
 	case BPF_STX:
 		if (res->validate && insn->off != orig_val) {
-			pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %llu -> %llu\n",
+			pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %d, exp %llu -> %llu\n",
 				prog_name, relo_idx, insn_idx, insn->off, (unsigned long long)orig_val,
 				(unsigned long long)new_val);
 			return -EINVAL;
@@ -1159,7 +1159,7 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn,
 	default:
 		pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n",
 			prog_name, relo_idx, insn_idx, insn->code,
-			insn->src_reg, insn->dst_reg, insn->off, insn->imm);
+			(unsigned)insn->src_reg, (unsigned)insn->dst_reg, (unsigned)insn->off, (unsigned)insn->imm);
 		return -EINVAL;
 	}
 
@@ -1323,7 +1323,7 @@ int bpf_core_calc_relo_insn(const char *prog_name,
 		const char *spec_str;
 
 		spec_str = btf__name_by_offset(local_btf, relo->access_str_off);
-		pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n",
+		pr_warn("prog '%s': relo #%d: parsing [%u] %s %s + %s failed: %d\n",
 			prog_name, relo_idx, local_id, btf_kind_str(local_type),
 			str_is_empty(local_name) ? "<anon>" : local_name,
 			spec_str ?: "<?>", err);
@@ -1346,7 +1346,7 @@ int bpf_core_calc_relo_insn(const char *prog_name,
 
 	/* libbpf doesn't support candidate search for anonymous types */
 	if (str_is_empty(local_name)) {
-		pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n",
+		pr_warn("prog '%s': relo #%d: <%s> (%u) relocation doesn't support anonymous types\n",
 			prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind);
 		return -EOPNOTSUPP;
 	}
@@ -1697,7 +1697,7 @@ int __bpf_core_types_match(const struct btf *local_btf, __u32 local_id, const st
 		goto recur;
 	}
 	default:
-		pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n",
+		pr_warn("unexpected kind %s relocated, local [%u], target [%u]\n",
 			btf_kind_str(local_t), local_id, targ_id);
 		return 0;
 	}
diff --git a/tools/lib/bpf/skel_internal.h b/tools/lib/bpf/skel_internal.h
index 74503d358bc8a..53fee53d36d51 100644
--- a/tools/lib/bpf/skel_internal.h
+++ b/tools/lib/bpf/skel_internal.h
@@ -18,10 +18,6 @@
 #include "bpf.h"
 #endif
 
-#ifndef SHA256_DIGEST_LENGTH
-#define SHA256_DIGEST_LENGTH 32
-#endif
-
 #ifndef __NR_bpf
 # if defined(__mips__) && defined(_ABIO32)
 #  define __NR_bpf 4355
@@ -320,25 +316,6 @@ static inline int skel_link_create(int prog_fd, int target_fd,
 	return skel_sys_bpf(BPF_LINK_CREATE, &attr, attr_sz);
 }
 
-static inline int skel_obj_get_info_by_fd(int fd)
-{
-	const size_t attr_sz = offsetofend(union bpf_attr, info);
-	__u8 sha[SHA256_DIGEST_LENGTH];
-	struct bpf_map_info info;
-	__u32 info_len = sizeof(info);
-	union bpf_attr attr;
-
-	memset(&info, 0, sizeof(info));
-	info.hash = (long) &sha;
-	info.hash_size = SHA256_DIGEST_LENGTH;
-
-	memset(&attr, 0, attr_sz);
-	attr.info.bpf_fd = fd;
-	attr.info.info = (long) &info;
-	attr.info.info_len = info_len;
-	return skel_sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, attr_sz);
-}
-
 static inline int skel_map_freeze(int fd)
 {
 	const size_t attr_sz = offsetofend(union bpf_attr, map_fd);
@@ -384,12 +361,6 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts)
 		set_err;
 		goto out;
 	}
-	err = skel_obj_get_info_by_fd(map_fd);
-	if (err < 0) {
-		opts->errstr = "failed to fetch obj info";
-		set_err;
-		goto out;
-	}
 #endif
 
 	memset(&attr, 0, prog_load_attr_sz);
@@ -400,6 +371,8 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts)
 #ifndef __KERNEL__
 	attr.signature = (long) opts->signature;
 	attr.signature_size = opts->signature_sz;
+	if (opts->signature)
+		attr.fd_array_cnt = 1;
 #else
 	if (opts->signature || opts->signature_sz)
 		pr_warn("signatures are not supported from bpf_preload\n");
diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c
index 57fb82bb81b58..2e56e3ab5b6cc 100644
--- a/tools/lib/bpf/usdt.c
+++ b/tools/lib/bpf/usdt.c
@@ -305,7 +305,7 @@ struct usdt_manager *usdt_manager_new(struct bpf_object *obj)
 
 	/*
 	 * Detect kernel support for uprobe() syscall, it's presence means we can
-	 * take advantage of faster nop5 uprobe handling.
+	 * take advantage of faster nop10 uprobe handling.
 	 * Added in: 56101b69c919 ("uprobes/x86: Add uprobe syscall to speed up uprobe")
 	 */
 	man->has_uprobe_syscall = kernel_supports(obj, FEAT_UPROBE_SYSCALL);
@@ -327,7 +327,7 @@ static int sanity_check_usdt_elf(Elf *elf, const char *path)
 	int endianness;
 
 	if (elf_kind(elf) != ELF_K_ELF) {
-		pr_warn("usdt: unrecognized ELF kind %d for '%s'\n", elf_kind(elf), path);
+		pr_warn("usdt: unrecognized ELF kind %u for '%s'\n", elf_kind(elf), path);
 		return -EBADF;
 	}
 
@@ -438,8 +438,9 @@ static int parse_elf_segs(Elf *elf, const char *path, struct elf_seg **segs, siz
 		}
 
 		pr_debug("usdt: discovered PHDR #%d in '%s': vaddr 0x%lx memsz 0x%lx offset 0x%lx type 0x%lx flags 0x%lx\n",
-			 i, path, (long)phdr.p_vaddr, (long)phdr.p_memsz, (long)phdr.p_offset,
-			 (long)phdr.p_type, (long)phdr.p_flags);
+			 i, path,
+			 (unsigned long)phdr.p_vaddr, (unsigned long)phdr.p_memsz, (unsigned long)phdr.p_offset,
+			 (unsigned long)phdr.p_type, (unsigned long)phdr.p_flags);
 		if (phdr.p_type != PT_LOAD)
 			continue;
 
@@ -604,14 +605,14 @@ static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note,
 #if defined(__x86_64__)
 static bool has_nop_combo(int fd, long off)
 {
-	unsigned char nop_combo[6] = {
-		0x90, 0x0f, 0x1f, 0x44, 0x00, 0x00 /* nop,nop5 */
+	unsigned char nop_combo[11] = {
+		0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00,
 	};
-	unsigned char buf[6];
+	unsigned char buf[11];
 
-	if (pread(fd, buf, 6, off) != 6)
+	if (pread(fd, buf, 11, off) != 11)
 		return false;
-	return memcmp(buf, nop_combo, 6) == 0;
+	return memcmp(buf, nop_combo, 11) == 0;
 }
 #else
 static bool has_nop_combo(int fd, long off)
@@ -719,14 +720,14 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd,
 		if (!seg) {
 			err = -ESRCH;
 			pr_warn("usdt: failed to find ELF program segment for '%s:%s' in '%s' at IP 0x%lx\n",
-				usdt_provider, usdt_name, path, usdt_abs_ip);
+				usdt_provider, usdt_name, path, (unsigned long)usdt_abs_ip);
 			goto err_out;
 		}
 		if (!seg->is_exec) {
 			err = -ESRCH;
 			pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx) for '%s:%s' at IP 0x%lx is not executable\n",
-				path, seg->start, seg->end, usdt_provider, usdt_name,
-				usdt_abs_ip);
+				path, (unsigned long)seg->start, (unsigned long)seg->end, usdt_provider, usdt_name,
+				(unsigned long)usdt_abs_ip);
 			goto err_out;
 		}
 		/* translate from virtual address to file offset */
@@ -766,7 +767,7 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd,
 			if (!seg) {
 				err = -ESRCH;
 				pr_warn("usdt: failed to find shared lib memory segment for '%s:%s' in '%s' at relative IP 0x%lx\n",
-					usdt_provider, usdt_name, path, usdt_rel_ip);
+					usdt_provider, usdt_name, path, (unsigned long)usdt_rel_ip);
 				goto err_out;
 			}
 
@@ -775,8 +776,10 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd,
 
 		pr_debug("usdt: probe for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved abs_ip 0x%lx rel_ip 0x%lx) args '%s' in segment [0x%lx, 0x%lx) at offset 0x%lx\n",
 			 usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", path,
-			 note.loc_addr, note.base_addr, usdt_abs_ip, usdt_rel_ip, note.args,
-			 seg ? seg->start : 0, seg ? seg->end : 0, seg ? seg->offset : 0);
+			 (unsigned long)note.loc_addr, (unsigned long)note.base_addr,
+			 (unsigned long)usdt_abs_ip, (unsigned long)usdt_rel_ip, note.args,
+			 (unsigned long)(seg ? seg->start : 0), (unsigned long)(seg ? seg->end : 0),
+			 (unsigned long)(seg ? seg->offset : 0));
 
 		/* Adjust semaphore address to be a file offset */
 		if (note.sema_addr) {
@@ -791,14 +794,14 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd,
 			if (!seg) {
 				err = -ESRCH;
 				pr_warn("usdt: failed to find ELF loadable segment with semaphore of '%s:%s' in '%s' at 0x%lx\n",
-					usdt_provider, usdt_name, path, note.sema_addr);
+					usdt_provider, usdt_name, path, (unsigned long)note.sema_addr);
 				goto err_out;
 			}
 			if (seg->is_exec) {
 				err = -ESRCH;
 				pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx] for semaphore of '%s:%s' at 0x%lx is executable\n",
-					path, seg->start, seg->end, usdt_provider, usdt_name,
-					note.sema_addr);
+					path, (unsigned long)seg->start, (unsigned long)seg->end, usdt_provider, usdt_name,
+					(unsigned long)note.sema_addr);
 				goto err_out;
 			}
 
@@ -806,8 +809,8 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd,
 
 			pr_debug("usdt: sema  for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved 0x%lx) in segment [0x%lx, 0x%lx] at offset 0x%lx\n",
 				 usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ",
-				 path, note.sema_addr, note.base_addr, usdt_sema_off,
-				 seg->start, seg->end, seg->offset);
+				 path, (unsigned long)note.sema_addr, (unsigned long)note.base_addr, (unsigned long)usdt_sema_off,
+				 (unsigned long)seg->start, (unsigned long)seg->end, (unsigned long)seg->offset);
 		}
 
 		/* Record adjusted addresses and offsets and parse USDT spec */
@@ -822,8 +825,8 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd,
 		memset(target, 0, sizeof(*target));
 
 		/*
-		 * We have uprobe syscall and usdt with nop,nop5 instructions combo,
-		 * so we can place the uprobe directly on nop5 (+1) and get this probe
+		 * We have uprobe syscall and usdt with nop,nop10 instructions combo,
+		 * so we can place the uprobe directly on nop10 (+1) and get this probe
 		 * optimized.
 		 */
 		if (man->has_uprobe_syscall && has_nop_combo(elf_fd->fd, usdt_rel_ip)) {
@@ -1117,7 +1120,7 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
 				        spec_id, usdt_provider, usdt_name, path);
 			} else {
 				pr_warn("usdt: failed to map IP 0x%lx to spec #%d for '%s:%s' in '%s': %s\n",
-					target->abs_ip, spec_id, usdt_provider, usdt_name,
+					(unsigned long)target->abs_ip, spec_id, usdt_provider, usdt_name,
 					path, errstr(err));
 			}
 			goto err_out;
diff --git a/tools/lib/python/kdoc/kdoc_output.py b/tools/lib/python/kdoc/kdoc_output.py
index de107ab4a2811..618b0d765ef5c 100644
--- a/tools/lib/python/kdoc/kdoc_output.py
+++ b/tools/lib/python/kdoc/kdoc_output.py
@@ -624,7 +624,7 @@ class ManFormat(OutputFormat):
     ``manual``
         Defaults to ``Kernel API Manual``.
 
-    The above controls the output of teh corresponding fields on troff
+    The above controls the output of the corresponding fields on troff
     title headers, which will be filled like this::
 
         .TH "{name}" {section} "{date}" "{modulename}" "{manual}"
diff --git a/tools/mm/.gitignore b/tools/mm/.gitignore
index 922879f93fc84..1446a659e5408 100644
--- a/tools/mm/.gitignore
+++ b/tools/mm/.gitignore
@@ -2,3 +2,4 @@
 slabinfo
 page-types
 page_owner_sort
+thp_swap_allocator_test
diff --git a/tools/mm/Makefile b/tools/mm/Makefile
index f5725b5c23aa2..858186a6eefdb 100644
--- a/tools/mm/Makefile
+++ b/tools/mm/Makefile
@@ -3,7 +3,7 @@
 #
 include ../scripts/Makefile.include
 
-BUILD_TARGETS=page-types slabinfo page_owner_sort thp_swap_allocator_test
+BUILD_TARGETS=page-types slabinfo page_owner_sort page_owner_filter thp_swap_allocator_test
 INSTALL_TARGETS = $(BUILD_TARGETS) thpmaps
 
 LIB_DIR = ../lib/api
@@ -23,7 +23,7 @@ $(LIBS):
 	$(CC) $(CFLAGS) -o $@ $< $(LDFLAGS)
 
 clean:
-	$(RM) page-types slabinfo page_owner_sort thp_swap_allocator_test
+	$(RM) page-types slabinfo page_owner_sort page_owner_filter thp_swap_allocator_test
 	make -C $(LIB_DIR) clean
 
 sbindir ?= /usr/sbin
diff --git a/tools/mm/page_owner_filter.c b/tools/mm/page_owner_filter.c
new file mode 100644
index 0000000000000..1d1f0a38678a2
--- /dev/null
+++ b/tools/mm/page_owner_filter.c
@@ -0,0 +1,310 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * User-space helper to filter page_owner output per-fd
+ *
+ * Example use:
+ *   ./page_owner_filter -m handle
+ *   ./page_owner_filter -m stack_handle
+ *   ./page_owner_filter -n 0,1,2
+ *
+ * See Documentation/mm/page_owner.rst
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <ctype.h>
+#include <getopt.h>
+#include <signal.h>
+
+#define MAX_CMD_LEN	512
+
+static void usage(const char *prog)
+{
+	fprintf(stderr, "Usage: %s [OPTIONS]\n", prog);
+	fprintf(stderr, "\nOptions:\n");
+	fprintf(stderr, "  -m, --mode MODE      : print_mode (stack, handle, or stack_handle)\n");
+	fprintf(stderr, "  -n, --nid NID_LIST   : NUMA node IDs (comma-separated or ranges)\n");
+	fprintf(stderr, "  -o, --output FILE    : output file (default: stdout)\n");
+	fprintf(stderr, "  -h, --help           : show this help message\n");
+	fprintf(stderr, "\nExamples:\n");
+	fprintf(stderr, "  %s -m stack\n", prog);
+	fprintf(stderr, "  %s -m handle\n", prog);
+	fprintf(stderr, "  %s -m stack_handle\n", prog);
+	fprintf(stderr, "  %s -m stack -o output.txt\n", prog);
+	fprintf(stderr, "  %s -n 0,1,2\n", prog);
+	fprintf(stderr, "  %s -m stack -n 0\n", prog);
+}
+
+static int validate_mode(const char *mode)
+{
+	if (strcmp(mode, "stack") == 0 ||
+	    strcmp(mode, "handle") == 0 ||
+	    strcmp(mode, "stack_handle") == 0)
+		return 0;
+
+	fprintf(stderr, "Error: Invalid mode '%s'\n", mode);
+	fprintf(stderr, "Valid modes: stack, handle, stack_handle\n");
+	return -1;
+}
+
+static int validate_nid_list(const char *nid_list)
+{
+	const char *p;
+	int i = 0;
+	int has_digit = 0;
+	int in_range = 0;
+	int prev_num = 0;
+	int curr_num = 0;
+
+	if (!nid_list || strlen(nid_list) == 0)
+		return 0;
+
+	for (p = nid_list; *p; p++) {
+		if (*p == ',') {
+			if (!has_digit) {
+				fprintf(stderr, "Error: Invalid nid_list format\n");
+				return -1;
+			}
+			if (in_range && prev_num > curr_num) {
+				fprintf(stderr,
+					"Error: Invalid range %d-%d (start must be <= end)\n",
+					prev_num, curr_num);
+				return -1;
+			}
+			i = 0;
+			has_digit = 0;
+			in_range = 0;
+			prev_num = 0;
+			curr_num = 0;
+			continue;
+		}
+
+		if (*p == '-') {
+			if (!has_digit) {
+				fprintf(stderr,
+					"Error: Invalid nid_list format ");
+				fprintf(stderr,
+					"(dash without preceding number)\n");
+				return -1;
+			}
+			if (in_range) {
+				fprintf(stderr, "Error: Multiple dashes in nid_list\n");
+				return -1;
+			}
+			prev_num = curr_num;
+			curr_num = 0;
+			i = 0;
+			has_digit = 0;
+			in_range = 1;
+			continue;
+		}
+
+		if (!isdigit((unsigned char)*p)) {
+			fprintf(stderr, "Error: Invalid character '%c' in nid_list\n", *p);
+			return -1;
+		}
+
+		if (i > 5) {
+			fprintf(stderr, "Error: NID too long (max 65536)\n");
+			return -1;
+		}
+		curr_num = curr_num * 10 + (*p - '0');
+		i++;
+		has_digit = 1;
+	}
+
+	if (!has_digit) {
+		fprintf(stderr, "Error: Invalid nid_list format\n");
+		return -1;
+	}
+
+	if (in_range && prev_num > curr_num) {
+		fprintf(stderr,
+			"Error: Invalid range %d-%d (start must be <= end)\n",
+			prev_num, curr_num);
+		return -1;
+	}
+
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	const char *output_file = NULL;
+	char filter_cmd[MAX_CMD_LEN];
+	FILE *output = NULL;
+	int fd = -1;
+	ssize_t ret;
+	char buf[4096];
+	int opt;
+	size_t cmd_len = 0;
+
+	signal(SIGPIPE, SIG_IGN);
+
+	static struct option long_options[] = {
+		{"mode",	required_argument, 0, 'm'},
+		{"nid",		required_argument, 0, 'n'},
+		{"output",	required_argument, 0, 'o'},
+		{"help",	no_argument,	   0, 'h'},
+		{0, 0, 0, 0}
+	};
+
+	filter_cmd[0] = '\0';
+
+	if (argc > 1) {
+		for (int i = 1; i < argc; i++) {
+			if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
+				usage(argv[0]);
+				return 0;
+			}
+		}
+	}
+
+	/* Check if page_owner exists and is readable */
+	if (access("/sys/kernel/debug/page_owner", F_OK) != 0) {
+		if (errno == ENOENT)
+			fprintf(stderr, "Error: /sys/kernel/debug/page_owner does not exist\n");
+		else
+			perror("Error accessing /sys/kernel/debug/page_owner");
+		fprintf(stderr, "Make sure page_owner is enabled in kernel\n");
+		return 1;
+	}
+
+	while ((opt = getopt_long(argc, argv, "m:n:o:h", long_options, NULL)) != -1) {
+		int len;
+
+		switch (opt) {
+		case 'm': {
+			const char *mode = optarg;
+
+			if (validate_mode(mode) < 0)
+				return 1;
+			len = snprintf(filter_cmd + cmd_len, MAX_CMD_LEN - cmd_len,
+				       "%smode=%s", cmd_len > 0 ? " " : "", mode);
+			if (len < 0 || cmd_len + len >= MAX_CMD_LEN) {
+				fprintf(stderr, "Error: Command too long\n");
+				return 1;
+			}
+			cmd_len += len;
+			break;
+		}
+		case 'n': {
+			const char *nid_list = optarg;
+
+			if (validate_nid_list(nid_list) < 0)
+				return 1;
+			len = snprintf(filter_cmd + cmd_len, MAX_CMD_LEN - cmd_len,
+				       "%snid=%s", cmd_len > 0 ? " " : "", nid_list);
+			if (len < 0 || cmd_len + len >= MAX_CMD_LEN) {
+				fprintf(stderr, "Error: Command too long\n");
+				return 1;
+			}
+			cmd_len += len;
+			break;
+		}
+		case 'o':
+			output_file = optarg;
+			break;
+		case 'h':
+			/* Already handled above */
+			break;
+		default:
+			usage(argv[0]);
+			return 1;
+		}
+	}
+
+	/* At least one filter must be specified */
+	if (cmd_len == 0) {
+		fprintf(stderr, "Error: At least one filter (-m or -n) must be specified\n\n");
+		usage(argv[0]);
+		return 1;
+	}
+
+	/* Open page_owner for read-write - this will fail if kernel doesn't support write */
+	fd = open("/sys/kernel/debug/page_owner", O_RDWR);
+	if (fd < 0) {
+		if (errno == EACCES || errno == EPERM) {
+			fprintf(stderr, "Error: /sys/kernel/debug/page_owner ");
+			fprintf(stderr, "does not support write access\n");
+			fprintf(stderr, "This kernel does not support ");
+			fprintf(stderr, "per-fd filtering.\n");
+			fprintf(stderr, "Please ensure you have a kernel with ");
+			fprintf(stderr, "per-fd filtering support.\n");
+		} else {
+			perror("Error opening /sys/kernel/debug/page_owner");
+		}
+		return 1;
+	}
+
+	if (output_file) {
+		output = fopen(output_file, "w");
+		if (!output) {
+			perror("open output file");
+			close(fd);
+			return 1;
+		}
+	} else {
+		output = stdout;
+	}
+
+	ret = write(fd, filter_cmd, strlen(filter_cmd));
+
+	if (ret < 0) {
+		if (errno == EINVAL) {
+			fprintf(stderr, "Error: Kernel rejected the filter command.\n");
+			fprintf(stderr, "Possible causes:\n");
+			fprintf(stderr, "  - Kernel does not support per-fd filtering\n");
+			fprintf(stderr, "  - NUMA node has no memory\n");
+			fprintf(stderr, "  - Unknown reason\n");
+		} else {
+			perror("write filter command");
+		}
+		goto out;
+	}
+
+	if ((size_t)ret != strlen(filter_cmd))
+		fprintf(stderr, "Warning: Partial write (%zd/%zu)\n", ret, strlen(filter_cmd));
+
+	/* Read and display filtered output */
+	ret = 0;
+	while ((ret = read(fd, buf, sizeof(buf))) > 0) {
+		size_t written = fwrite(buf, 1, ret, output);
+
+		if (written != (size_t)ret) {
+			if (errno == EPIPE) {
+				/* Pipe closed, treat as success */
+				ret = 0;
+				goto out;
+			}
+			perror("write output");
+			ret = -1;
+			goto out;
+		}
+	}
+
+	if (ret < 0) {
+		perror("read page_owner");
+		goto out;
+	}
+
+	if (fflush(output)) {
+		if (errno == EPIPE) {
+			/* Pipe closed, treat as success */
+			ret = 0;
+		} else {
+			perror("flush output");
+			ret = -1;
+		}
+	}
+
+out:
+	close(fd);
+	if (output != stdout)
+		fclose(output);
+	return ret < 0 ? 1 : 0;
+}
diff --git a/tools/mm/page_owner_sort.c b/tools/mm/page_owner_sort.c
index e6954909401c8..35d3d254941c9 100644
--- a/tools/mm/page_owner_sort.c
+++ b/tools/mm/page_owner_sort.c
@@ -43,6 +43,13 @@ enum FILTER_BIT {
 	FILTER_TGID = 1<<2,
 	FILTER_COMM = 1<<3
 };
+
+enum FILTER_RESULT {
+	FILTER_ERROR,
+	FILTER_SKIP,
+	FILTER_MATCH
+};
+
 enum CULL_BIT {
 	CULL_PID = 1<<1,
 	CULL_TGID = 1<<2,
@@ -230,7 +237,8 @@ static int remove_pattern(regex_t *pattern, char *buf, int len)
 	return len - (pmatch[1].rm_eo - pmatch[1].rm_so);
 }
 
-static int search_pattern(regex_t *pattern, char *pattern_str, char *buf)
+static int search_pattern(regex_t *pattern, char *pattern_str,
+			  size_t pattern_str_size, char *buf)
 {
 	int err, val_len;
 	regmatch_t pmatch[2];
@@ -242,8 +250,14 @@ static int search_pattern(regex_t *pattern, char *pattern_str, char *buf)
 		return -1;
 	}
 	val_len = pmatch[1].rm_eo - pmatch[1].rm_so;
+	if ((size_t)val_len >= pattern_str_size) {
+		if (debug_on)
+			fprintf(stderr, "pattern too long in %s\n", buf);
+		return -1;
+	}
 
 	memcpy(pattern_str, buf + pmatch[1].rm_so, val_len);
+	pattern_str[val_len] = '\0';
 
 	return 0;
 }
@@ -300,7 +314,8 @@ static int get_page_num(char *buf)
 	char order_str[FIELD_BUFF] = {0};
 	char *endptr;
 
-	search_pattern(&order_pattern, order_str, buf);
+	if (search_pattern(&order_pattern, order_str, sizeof(order_str), buf) < 0)
+		return 0;
 	errno = 0;
 	order_val = strtol(order_str, &endptr, 10);
 	if (order_val > 64 || errno != 0 || endptr == order_str || *endptr != '\0') {
@@ -318,7 +333,8 @@ static pid_t get_pid(char *buf)
 	char pid_str[FIELD_BUFF] = {0};
 	char *endptr;
 
-	search_pattern(&pid_pattern, pid_str, buf);
+	if (search_pattern(&pid_pattern, pid_str, sizeof(pid_str), buf) < 0)
+		return -1;
 	errno = 0;
 	pid = strtol(pid_str, &endptr, 10);
 	if (errno != 0 || endptr == pid_str || *endptr != '\0') {
@@ -337,7 +353,8 @@ static pid_t get_tgid(char *buf)
 	char tgid_str[FIELD_BUFF] = {0};
 	char *endptr;
 
-	search_pattern(&tgid_pattern, tgid_str, buf);
+	if (search_pattern(&tgid_pattern, tgid_str, sizeof(tgid_str), buf) < 0)
+		return -1;
 	errno = 0;
 	tgid = strtol(tgid_str, &endptr, 10);
 	if (errno != 0 || endptr == tgid_str || *endptr != '\0') {
@@ -356,7 +373,9 @@ static __u64 get_ts_nsec(char *buf)
 	char ts_nsec_str[FIELD_BUFF] = {0};
 	char *endptr;
 
-	search_pattern(&ts_nsec_pattern, ts_nsec_str, buf);
+	if (search_pattern(&ts_nsec_pattern, ts_nsec_str,
+			   sizeof(ts_nsec_str), buf) < 0)
+		return -1;
 	errno = 0;
 	ts_nsec = strtoull(ts_nsec_str, &endptr, 10);
 	if (errno != 0 || endptr == ts_nsec_str || *endptr != '\0') {
@@ -372,9 +391,15 @@ static char *get_comm(char *buf)
 {
 	char *comm_str = malloc(TASK_COMM_LEN);
 
+	if (!comm_str)
+		return NULL;
+
 	memset(comm_str, 0, TASK_COMM_LEN);
 
-	search_pattern(&comm_pattern, comm_str, buf);
+	if (search_pattern(&comm_pattern, comm_str, TASK_COMM_LEN, buf) < 0) {
+		free(comm_str);
+		return NULL;
+	}
 	errno = 0;
 	if (errno != 0) {
 		if (debug_on)
@@ -386,6 +411,12 @@ static char *get_comm(char *buf)
 	return comm_str;
 }
 
+static void free_block_list(struct block_list *block)
+{
+	free(block->comm);
+	free(block->txt);
+}
+
 static int get_arg_type(const char *arg)
 {
 	if (!strcmp(arg, "pid") || !strcmp(arg, "p"))
@@ -450,39 +481,56 @@ static bool match_str_list(const char *str, char **list, int list_size)
 	return false;
 }
 
-static bool is_need(char *buf)
+static enum FILTER_RESULT filter_record(char *buf)
 {
+	char *comm;
+
 	if ((filter & FILTER_PID) && !match_num_list(get_pid(buf), fc.pids, fc.pids_size))
-		return false;
+		return FILTER_SKIP;
 	if ((filter & FILTER_TGID) &&
 		!match_num_list(get_tgid(buf), fc.tgids, fc.tgids_size))
-		return false;
+		return FILTER_SKIP;
+	if (!(filter & FILTER_COMM))
+		return FILTER_MATCH;
 
-	char *comm = get_comm(buf);
+	comm = get_comm(buf);
+	if (!comm)
+		return FILTER_ERROR;
 
-	if ((filter & FILTER_COMM) &&
-	!match_str_list(comm, fc.comms, fc.comms_size)) {
+	if (!match_str_list(comm, fc.comms, fc.comms_size)) {
 		free(comm);
-		return false;
+		return FILTER_SKIP;
 	}
 	free(comm);
-	return true;
+	return FILTER_MATCH;
 }
 
 static bool add_list(char *buf, int len, char *ext_buf)
 {
+	enum FILTER_RESULT filter_result;
+
 	if (list_size == max_size) {
 		fprintf(stderr, "max_size too small??\n");
 		return false;
 	}
-	if (!is_need(buf))
+	filter_result = filter_record(buf);
+	if (filter_result == FILTER_ERROR) {
+		fprintf(stderr, "Out of memory\n");
+		return false;
+	}
+	if (filter_result == FILTER_SKIP)
 		return true;
 	list[list_size].pid = get_pid(buf);
 	list[list_size].tgid = get_tgid(buf);
 	list[list_size].comm = get_comm(buf);
-	list[list_size].txt = malloc(len+1);
+	if (!list[list_size].comm) {
+		fprintf(stderr, "Out of memory\n");
+		return false;
+	}
+	list[list_size].txt = malloc(len + 1);
 	if (!list[list_size].txt) {
 		fprintf(stderr, "Out of memory\n");
+		free(list[list_size].comm);
 		return false;
 	}
 	memcpy(list[list_size].txt, buf, len);
@@ -841,8 +889,10 @@ int main(int argc, char **argv)
 		} else {
 			list[count-1].num += list[i].num;
 			list[count-1].page_num += list[i].page_num;
+			free_block_list(&list[i]);
 		}
 	}
+	list_size = count;
 
 	qsort(list, count, sizeof(list[0]), compare_sort_condition);
 
@@ -876,8 +926,11 @@ int main(int argc, char **argv)
 		free(ext_buf);
 	if (buf)
 		free(buf);
-	if (list)
+	if (list) {
+		for (i = 0; i < list_size; i++)
+			free_block_list(&list[i]);
 		free(list);
+	}
 out_ts:
 	regfree(&ts_nsec_pattern);
 out_comm:
diff --git a/tools/net/ynl/pyynl/__init__.py b/tools/net/ynl/pyynl/__init__.py
index e69de29bb2d1d..d8f59c132ab7e 100644
--- a/tools/net/ynl/pyynl/__init__.py
+++ b/tools/net/ynl/pyynl/__init__.py
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+
+""" Python YNL (YAML Netlink) library. """
+
+# Re-export the public library API so it can be imported straight from the
+# package, e.g. `from pyynl import YnlFamily`.
+# pylint: disable=wildcard-import,unused-wildcard-import
+from .lib import *
+from .lib import __all__
diff --git a/tools/net/ynl/pyynl/cli.py b/tools/net/ynl/pyynl/cli.py
index 8275a806cf73d..b6a6ce12b4a7f 100755
--- a/tools/net/ynl/pyynl/cli.py
+++ b/tools/net/ynl/pyynl/cli.py
@@ -17,9 +17,7 @@ import textwrap
 # pylint: disable=no-name-in-module,wrong-import-position
 sys.path.append(pathlib.Path(__file__).resolve().parent.as_posix())
 from lib import YnlFamily, Netlink, NlError, SpecFamily, SpecException, YnlException
-
-SYS_SCHEMA_DIR='/usr/share/ynl'
-RELATIVE_SCHEMA_DIR='../../../../Documentation/netlink'
+from lib import list_families
 
 # pylint: disable=too-few-public-methods,too-many-locals
 class Colors:
@@ -48,30 +46,6 @@ def term_width():
     """ Get terminal width in columns (80 if stdout is not a terminal) """
     return shutil.get_terminal_size().columns
 
-def schema_dir():
-    """
-    Return the effective schema directory, preferring in-tree before
-    system schema directory.
-    """
-    script_dir = os.path.dirname(os.path.abspath(__file__))
-    schema_dir_ = os.path.abspath(f"{script_dir}/{RELATIVE_SCHEMA_DIR}")
-    if not os.path.isdir(schema_dir_):
-        schema_dir_ = SYS_SCHEMA_DIR
-    if not os.path.isdir(schema_dir_):
-        raise YnlException(f"Schema directory {schema_dir_} does not exist")
-    return schema_dir_
-
-def spec_dir():
-    """
-    Return the effective spec directory, relative to the effective
-    schema directory.
-    """
-    spec_dir_ = schema_dir() + '/specs'
-    if not os.path.isdir(spec_dir_):
-        raise YnlException(f"Spec directory {spec_dir_} does not exist")
-    return spec_dir_
-
-
 class YnlEncoder(json.JSONEncoder):
     """A custom encoder for emitting JSON with ynl-specific instance types"""
     def default(self, o):
@@ -272,9 +246,8 @@ def main():
             pprint.pprint(msg, width=term_width(), compact=True)
 
     if args.list_families:
-        for filename in sorted(os.listdir(spec_dir())):
-            if filename.endswith('.yaml'):
-                print(filename.removesuffix('.yaml'))
+        for family in list_families():
+            print(family)
         return
 
     if args.no_schema:
@@ -284,28 +257,23 @@ def main():
     if args.json_text:
         attrs = json.loads(args.json_text)
 
-    if args.family:
-        spec = f"{spec_dir()}/{args.family}.yaml"
-    else:
-        spec = args.spec
-    if not os.path.isfile(spec):
-        raise YnlException(f"Spec file {spec} does not exist")
+    if args.spec and not os.path.isfile(args.spec):
+        raise YnlException(f"Spec file {args.spec} does not exist")
 
+    # Spec/YnlFamily will raise if both or neither spec and family are given
     if args.validate:
+        # Force validation even for installed specs (schema=True), unless the
+        # user explicitly picked a schema or opted out with --no-schema.
+        schema = True if args.schema is None else args.schema
         try:
-            SpecFamily(spec, args.schema)
+            SpecFamily(args.spec, schema_path=schema, family=args.family)
         except SpecException as error:
             print(error)
             sys.exit(1)
         return
 
-    if args.family: # set behaviour when using installed specs
-        if args.schema is None and spec.startswith(SYS_SCHEMA_DIR):
-            args.schema = '' # disable schema validation when installed
-        if args.process_unknown is None:
-            args.process_unknown = True
-
-    ynl = YnlFamily(spec, args.schema, args.process_unknown,
+    ynl = YnlFamily(args.spec, schema=args.schema, family=args.family,
+                    process_unknown=args.process_unknown,
                     recv_size=args.dbg_small_recv)
     if args.dbg_small_recv:
         ynl.set_recv_dbg(True)
diff --git a/tools/net/ynl/pyynl/lib/__init__.py b/tools/net/ynl/pyynl/lib/__init__.py
index be741985ae4ee..aa4263c8cba9e 100644
--- a/tools/net/ynl/pyynl/lib/__init__.py
+++ b/tools/net/ynl/pyynl/lib/__init__.py
@@ -5,12 +5,13 @@
 from .nlspec import SpecAttr, SpecAttrSet, SpecEnumEntry, SpecEnumSet, \
     SpecFamily, SpecOperation, SpecSubMessage, SpecSubMessageFormat, \
     SpecException
+from .specdir import list_families
 from .ynl import YnlFamily, Netlink, NlError, NlPolicy, YnlException
 
 from .doc_generator import YnlDocGenerator
 
 __all__ = ["SpecAttr", "SpecAttrSet", "SpecEnumEntry", "SpecEnumSet",
            "SpecFamily", "SpecOperation", "SpecSubMessage", "SpecSubMessageFormat",
-           "SpecException",
+           "SpecException", "list_families",
            "YnlFamily", "Netlink", "NlError", "NlPolicy", "YnlException",
            "YnlDocGenerator"]
diff --git a/tools/net/ynl/pyynl/lib/nlspec.py b/tools/net/ynl/pyynl/lib/nlspec.py
index 0469a0e270d05..b4ec59814ab1b 100644
--- a/tools/net/ynl/pyynl/lib/nlspec.py
+++ b/tools/net/ynl/pyynl/lib/nlspec.py
@@ -12,6 +12,8 @@ import importlib
 import os
 import yaml as pyyaml
 
+from .specdir import find_spec, SYS_SCHEMA_DIR
+
 
 class SpecException(Exception):
     """Netlink spec exception.
@@ -444,7 +446,23 @@ class SpecFamily(SpecElement):
     except AttributeError:
         _yaml_loader = pyyaml.SafeLoader
 
-    def __init__(self, spec_path, schema_path=None, exclude_ops=None):
+    def __init__(self, spec_path=None, schema_path=None, exclude_ops=None,
+                 family=None):
+        # schema_path selects how the spec is validated:
+        #   None  -- no preference: validate against the default schema,
+        #            but trust (skip) installed specs selected by family=
+        #   True  -- always validate against the default schema
+        #   path  -- validate against this schema
+        #   ''    -- do not validate
+        if (spec_path is None) == (family is None):
+            raise ValueError("Specify exactly one of spec path or family name")
+        if family is not None:
+            spec_path = find_spec(family)
+            # Installed specs are assumed correct, so skip schema validation
+            # to save cycles unless the caller asked to validate.
+            if schema_path is None and spec_path.startswith(SYS_SCHEMA_DIR):
+                schema_path = ''
+
         with open(spec_path, "r", encoding='utf-8') as stream:
             prefix = '# SPDX-License-Identifier: '
             first = stream.readline().strip()
@@ -465,7 +483,7 @@ class SpecFamily(SpecElement):
         self.proto = self.yaml.get('protocol', 'genetlink')
         self.msg_id_model = self.yaml['operations'].get('enum-model', 'unified')
 
-        if schema_path is None:
+        if schema_path is None or schema_path is True:
             schema_path = os.path.dirname(os.path.dirname(spec_path)) + f'/{self.proto}.yaml'
         if schema_path:
             with open(schema_path, "r", encoding='utf-8') as stream:
diff --git a/tools/net/ynl/pyynl/lib/specdir.py b/tools/net/ynl/pyynl/lib/specdir.py
new file mode 100644
index 0000000000000..fcea9b9fb7b04
--- /dev/null
+++ b/tools/net/ynl/pyynl/lib/specdir.py
@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+
+"""
+Locating YNL spec and schema files on disk.
+
+Resolves the directory holding the YAML specs (preferring an in-tree copy
+over the installed system path) and maps family names to spec files.
+"""
+
+import os
+
+SYS_SCHEMA_DIR='/usr/share/ynl'
+RELATIVE_SCHEMA_DIR='../../../../../Documentation/netlink'
+
+
+def schema_dir():
+    """
+    Return the effective schema directory, preferring in-tree before
+    system schema directory.
+    """
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    schema_dir_ = os.path.abspath(f"{script_dir}/{RELATIVE_SCHEMA_DIR}")
+    if not os.path.isdir(schema_dir_):
+        schema_dir_ = SYS_SCHEMA_DIR
+    if not os.path.isdir(schema_dir_):
+        raise FileNotFoundError(f"Schema directory {schema_dir_} does not exist")
+    return schema_dir_
+
+def spec_dir():
+    """
+    Return the effective spec directory, relative to the effective
+    schema directory.
+    """
+    spec_dir_ = schema_dir() + '/specs'
+    if not os.path.isdir(spec_dir_):
+        raise FileNotFoundError(f"Spec directory {spec_dir_} does not exist")
+    return spec_dir_
+
+
+def find_spec(family):
+    """ Return the path to the YAML spec file for a family by name. """
+    spec = f"{spec_dir()}/{family}.yaml"
+    if not os.path.isfile(spec):
+        raise FileNotFoundError(f"Spec for family '{family}' not found at {spec}")
+    return spec
+
+
+def list_families():
+    """ Return the sorted names of all families with an installed spec. """
+    return sorted(f.removesuffix('.yaml')
+                  for f in os.listdir(spec_dir()) if f.endswith('.yaml'))
diff --git a/tools/net/ynl/pyynl/lib/ynl.py b/tools/net/ynl/pyynl/lib/ynl.py
index 092d132edec12..8682bf588e1fa 100644
--- a/tools/net/ynl/pyynl/lib/ynl.py
+++ b/tools/net/ynl/pyynl/lib/ynl.py
@@ -661,6 +661,14 @@ class YnlFamily(SpecFamily):
     """
     YNL family -- a Netlink interface built from a YAML spec.
 
+    The spec can be selected either by file path (def_path=) or, when it
+    ships in a well-known location, by family name (family="xyz"); exactly
+    one of the two must be given. For example:
+
+      from pyynl import YnlFamily
+
+      ynl = YnlFamily(family="netdev")
+
     Primary use of the class is to execute Netlink commands:
 
       ynl.<op_name>(attrs, ...)
@@ -691,11 +699,16 @@ class YnlFamily(SpecFamily):
 
       ynl.get_policy(op_name, mode)      -- query kernel policy for an op
     """
-    def __init__(self, def_path, schema=None, process_unknown=False,
-                 recv_size=0):
-        super().__init__(def_path, schema)
+    def __init__(self, def_path=None, schema=None, process_unknown=None,
+                 recv_size=0, family=None):
+        super().__init__(def_path, schema, family=family)
 
         self.include_raw = False
+        # Specs from /usr (selected by family=) have a higher chance of being
+        # stale, default to ignoring unknown attrs. In-tree users, and users
+        # who bundle the spec need to make a conscious decision.
+        if process_unknown is None:
+            process_unknown = family is not None
         self.process_unknown = process_unknown
 
         try:
diff --git a/tools/net/ynl/tests/ethtool.py b/tools/net/ynl/tests/ethtool.py
index db3b62c652e77..0ee0c8e87686b 100755
--- a/tools/net/ynl/tests/ethtool.py
+++ b/tools/net/ynl/tests/ethtool.py
@@ -11,12 +11,10 @@ import pathlib
 import pprint
 import sys
 import re
-import os
 
 # pylint: disable=no-name-in-module,wrong-import-position
 sys.path.append(pathlib.Path(__file__).resolve().parent.parent.joinpath('pyynl').as_posix())
 # pylint: disable=import-error
-from cli import schema_dir, spec_dir
 from lib import YnlFamily
 
 
@@ -173,10 +171,7 @@ def main():
 
     args = parser.parse_args()
 
-    spec = os.path.join(spec_dir(), 'ethtool.yaml')
-    schema = os.path.join(schema_dir(), 'genetlink-legacy.yaml')
-
-    ynl = YnlFamily(spec, schema, recv_size=args.dbg_small_recv)
+    ynl = YnlFamily(family='ethtool', recv_size=args.dbg_small_recv)
     if args.dbg_small_recv:
         ynl.set_recv_dbg(True)
 
diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index 10b18cf9c3608..1b6b9924d4e45 100644
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -206,6 +206,7 @@ static bool is_rust_noreturn(const struct symbol *func)
 	       str_ends_with(func->name, "_4core9panicking18panic_nounwind_fmt")			||
 	       str_ends_with(func->name, "_4core9panicking19assert_failed_inner")			||
 	       str_ends_with(func->name, "_4core9panicking30panic_null_pointer_dereference")		||
+	       str_ends_with(func->name, "_4core9panicking32panic_null_reference_constructed")		||
 	       str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference")	||
 	       str_ends_with(func->name, "_7___rustc17rust_begin_unwind")				||
 	       strstr(func->name, "_4core9panicking13assert_failed")					||
@@ -4644,7 +4645,6 @@ static int validate_ibt(struct objtool_file *file)
 		    !strcmp(sec->name, ".kcfi_traps")			||
 		    !strcmp(sec->name, ".orc_unwind_ip")		||
 		    !strcmp(sec->name, ".retpoline_sites")		||
-		    !strcmp(sec->name, ".smp_locks")			||
 		    !strcmp(sec->name, ".static_call_sites")		||
 		    !strcmp(sec->name, "_error_injection_whitelist")	||
 		    !strcmp(sec->name, "_kprobe_blacklist")		||
diff --git a/tools/objtool/klp-diff.c b/tools/objtool/klp-diff.c
index f8787d7d14547..07b7ef34cc92b 100644
--- a/tools/objtool/klp-diff.c
+++ b/tools/objtool/klp-diff.c
@@ -297,7 +297,6 @@ static bool is_special_section(struct section *sec)
 	static const char * const specials[] = {
 		".altinstructions",
 		".kcfi_traps",
-		".smp_locks",
 		"__bug_table",
 		"__ex_table",
 		"__jump_table",
diff --git a/tools/objtool/noreturns.h b/tools/objtool/noreturns.h
index 14f8ab653449c..0e9dfd0a2446c 100644
--- a/tools/objtool/noreturns.h
+++ b/tools/objtool/noreturns.h
@@ -26,6 +26,7 @@ NORETURN(cpu_startup_entry)
 NORETURN(do_exit)
 NORETURN(do_group_exit)
 NORETURN(do_task_dead)
+NORETURN(efi_rts_park_worker)
 NORETURN(ex_handler_msr_mce)
 NORETURN(hlt_play_dead)
 NORETURN(hv_ghcb_terminate)
diff --git a/tools/perf/.clang-format b/tools/perf/.clang-format
index 902b2f7456f61..e4cff363a6038 100644
--- a/tools/perf/.clang-format
+++ b/tools/perf/.clang-format
@@ -2,15 +2,18 @@ BasedOnStyle: InheritParentConfig
 SortIncludes: true
 IncludeBlocks: Regroup
 IncludeCategories:
+  # Python.h must be first
+  - Regex:           '^<Python\.h>'
+    Priority:        -1
   # Implicitly the corresponding header for the C file has Priority 0
   # C Standard Library Headers
-  - Regex:           '^<(assert|complex|ctype|errno|fenv|float|inttypes|iso646|limits|locale|math|setjmp|signal|stdalign|stdarg|stdatomic|stdbool|stddef|stdint|stdio|stdlib|stdnoreturn|string|tgmath|threads|time|uchar|wchar|wctype)\.h>'
+  - Regex:           '^<(assert|complex|ctype|errno|fenv|float|inttypes|iso646|limits|locale|math|setjmp|signal|stdalign|stdarg|stdatomic|stdbit|stdbool|stdckdint|stddef|stdint|stdio|stdlib|stdnoreturn|string|tgmath|threads|time|uchar|wchar|wctype)\.h>'
     Priority:        1
   # OS/System-Specific Headers (directories)
-  - Regex:           '^<(sys|linux|asm|arpa|net|netinet|x86_64|machine)/.*>'
+  - Regex:           '^<(arpa|asm|asm-generic|linux|machine|net|netinet|sys|uapi|x86_64)/.*>'
     Priority:        2
   # OS/System-Specific Headers (POSIX/System flat headers)
-  - Regex:           '^<(unistd|pthread|fcntl|dirent|dlfcn|poll|sched|semaphore|spawn|syslog|termios|pwd|grp|netdb|sysexits|err|paths|pty|utmp|resolv|ifaddrs|elf|libelf|gelf)\.h>'
+  - Regex:           '^<(aio|byteswap|dirent|dlfcn|elf|endian|err|execinfo|fcntl|features|fnmatch|ftw|gelf|getopt|grp|ifaddrs|libelf|libgen|malloc|memory|mntent|netdb|numa|numaif|paths|poll|pthread|pty|pwd|regex|resolv|sched|semaphore|spawn|strings|syscall|sysexits|syslog|termios|unistd|utmp|x86intrin)\.h>'
     Priority:        2
   # Third-Party Library Headers
   - Regex:           '^<.*>'
diff --git a/tools/perf/Documentation/perf-test.txt b/tools/perf/Documentation/perf-test.txt
index 81c8525f59468..859df74e62efc 100644
--- a/tools/perf/Documentation/perf-test.txt
+++ b/tools/perf/Documentation/perf-test.txt
@@ -57,7 +57,8 @@ OPTIONS
 --workload=::
 	Run a built-in workload, to list them use '--list-workloads', current
 	ones include: noploop, thloop, leafloop, sqrtloop, brstack, datasym,
-	context_switch_loop, deterministic, named_threads and landlock.
+	context_switch_loop, deterministic, named_threads, landlock and
+	callchain.
 
 	Used with the shell script regression tests.
 
@@ -69,7 +70,8 @@ OPTIONS
 	'named_threads' accepts the number of threads and the number of loops to
 	do in each thread.
 
-	The datasym, landlock and deterministic workloads don't accept any.
+	The datasym, landlock, deterministic and callchain workloads don't accept
+	any.
 
 --list-workloads::
 	List the available workloads to use with -w/--workload.
diff --git a/tools/perf/Documentation/topdown.txt b/tools/perf/Documentation/topdown.txt
index 5c17fff694ee7..fd4214f9a7b08 100644
--- a/tools/perf/Documentation/topdown.txt
+++ b/tools/perf/Documentation/topdown.txt
@@ -87,7 +87,7 @@ if (slots_fd < 0)
 
 /* Memory mapping the fd permits _rdpmc calls from userspace */
 void *slots_p = mmap(0, getpagesize(), PROT_READ, MAP_SHARED, slots_fd, 0);
-if (!slot_p)
+if (slots_p == MAP_FAILED)
 	.... error ...
 
 /*
@@ -107,7 +107,7 @@ if (metrics_fd < 0)
 
 /* Memory mapping the fd permits _rdpmc calls from userspace */
 void *metrics_p = mmap(0, getpagesize(), PROT_READ, MAP_SHARED, metrics_fd, 0);
-if (!metrics_p)
+if (metrics_p == MAP_FAILED)
 	... error ...
 
 Note: the file descriptors returned by the perf_event_open calls must be memory
diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf
index 476b8dcaef58c..0031112c036e8 100644
--- a/tools/perf/Makefile.perf
+++ b/tools/perf/Makefile.perf
@@ -517,12 +517,15 @@ all: shell_compatibility_test $(ALL_PROGRAMS) $(LANG_BINDINGS) $(OTHER_PROGRAMS)
 # Create python binding output directory if not already present
 $(shell [ -d '$(OUTPUT)python' ] || mkdir -p '$(OUTPUT)python')
 
-$(OUTPUT)python/perf$(PYTHON_EXTENSION_SUFFIX): util/python.c util/setup.py $(PERFLIBS_PY)
+$(OUTPUT)python/perf$(PYTHON_EXTENSION_SUFFIX): util/python.c util/setup.py python/perf.pyi $(PERFLIBS_PY)
 	$(QUIET_GEN)LDSHARED="$(CC) -pthread -shared" \
         CFLAGS='$(CFLAGS)' LDFLAGS='$(LDFLAGS) $(LIBS_PY)' \
 	  $(PYTHON_WORD) util/setup.py \
 	  $(python_setup_quiet) build_ext; \
-	cp $(PYTHON_EXTBUILD_LIB)perf*.so $(OUTPUT)python/
+	cp $(PYTHON_EXTBUILD_LIB)perf*.so $(OUTPUT)python/; \
+	if [ "$(abspath python/perf.pyi)" != "$(abspath $(OUTPUT)python/perf.pyi)" ]; then \
+		cp python/perf.pyi $(OUTPUT)python/; \
+	fi
 
 python_perf_target:
 	@echo "Target is: $(OUTPUT)python/perf$(PYTHON_EXTENSION_SUFFIX)"
@@ -677,7 +680,7 @@ $(OUTPUT)dlfilters/%.o: dlfilters/%.c include/perf/perf_dlfilter.h
 .SECONDARY: $(DLFILTERS:.so=.o)
 
 $(OUTPUT)dlfilters/%.so: $(OUTPUT)dlfilters/%.o
-	$(QUIET_LINK)$(CC) $(EXTRA_CFLAGS) -shared -o $@ $<
+	$(QUIET_LINK)$(CC) $(filter-out -static,$(LDFLAGS)) $(EXTRA_CFLAGS) -shared -o $@ $<
 
 ifndef NO_JVMTI
 LIBJVMTI_IN := $(OUTPUT)jvmti/jvmti-in.o
diff --git a/tools/perf/arch/arm/util/cs-etm.c b/tools/perf/arch/arm/util/cs-etm.c
index cdf8e3e606067..d2861d66a6612 100644
--- a/tools/perf/arch/arm/util/cs-etm.c
+++ b/tools/perf/arch/arm/util/cs-etm.c
@@ -201,7 +201,7 @@ static int cs_etm_validate_config(struct perf_pmu *cs_etm_pmu,
 {
 	unsigned int idx;
 	int err = 0;
-	struct perf_cpu_map *event_cpus = evsel->evlist->core.user_requested_cpus;
+	struct perf_cpu_map *event_cpus = evlist__core(evsel->evlist)->user_requested_cpus;
 	struct perf_cpu_map *intersect_cpus;
 	struct perf_cpu cpu;
 
@@ -325,7 +325,7 @@ static int cs_etm_recording_options(struct auxtrace_record *itr,
 				container_of(itr, struct cs_etm_recording, itr);
 	struct perf_pmu *cs_etm_pmu = ptr->cs_etm_pmu;
 	struct evsel *evsel, *cs_etm_evsel = NULL;
-	struct perf_cpu_map *cpus = evlist->core.user_requested_cpus;
+	struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus;
 	bool privileged = perf_event_paranoid_check(-1);
 	int err = 0;
 
@@ -551,7 +551,7 @@ cs_etm_info_priv_size(struct auxtrace_record *itr,
 {
 	unsigned int idx;
 	int etmv3 = 0, etmv4 = 0, ete = 0;
-	struct perf_cpu_map *event_cpus = evlist->core.user_requested_cpus;
+	struct perf_cpu_map *event_cpus = evlist__core(evlist)->user_requested_cpus;
 	struct perf_cpu_map *intersect_cpus;
 	struct perf_cpu cpu;
 	struct perf_pmu *cs_etm_pmu = cs_etm_get_pmu(itr);
@@ -790,7 +790,7 @@ static int cs_etm_info_fill(struct auxtrace_record *itr,
 	u32 offset;
 	u64 nr_cpu, type;
 	struct perf_cpu_map *cpu_map;
-	struct perf_cpu_map *event_cpus = session->evlist->core.user_requested_cpus;
+	struct perf_cpu_map *event_cpus = evlist__core(session->evlist)->user_requested_cpus;
 	struct perf_cpu_map *online_cpus = perf_cpu_map__new_online_cpus();
 	struct cs_etm_recording *ptr =
 			container_of(itr, struct cs_etm_recording, itr);
@@ -800,7 +800,7 @@ static int cs_etm_info_fill(struct auxtrace_record *itr,
 	if (priv_size != cs_etm_info_priv_size(itr, session->evlist))
 		return -EINVAL;
 
-	if (!session->evlist->core.nr_mmaps)
+	if (!evlist__core(session->evlist)->nr_mmaps)
 		return -EINVAL;
 
 	/* If the cpu_map has the "any" CPU all online CPUs are involved */
diff --git a/tools/perf/arch/arm64/util/arm-spe.c b/tools/perf/arch/arm64/util/arm-spe.c
index 91bb28cad79a5..1ba803a8d9b45 100644
--- a/tools/perf/arch/arm64/util/arm-spe.c
+++ b/tools/perf/arch/arm64/util/arm-spe.c
@@ -60,7 +60,7 @@ static bool arm_spe_is_set_freq(struct evsel *evsel)
  */
 static struct perf_cpu_map *arm_spe_find_cpus(struct evlist *evlist)
 {
-	struct perf_cpu_map *event_cpus = evlist->core.user_requested_cpus;
+	struct perf_cpu_map *event_cpus = evlist__core(evlist)->user_requested_cpus;
 	struct perf_cpu_map *online_cpus = perf_cpu_map__new_online_cpus();
 	struct perf_cpu_map *intersect_cpus;
 
@@ -157,7 +157,7 @@ static int arm_spe_info_fill(struct auxtrace_record *itr,
 	if (priv_size != arm_spe_info_priv_size(itr, session->evlist))
 		return -EINVAL;
 
-	if (!session->evlist->core.nr_mmaps)
+	if (!evlist__core(session->evlist)->nr_mmaps)
 		return -EINVAL;
 
 	cpu_map = arm_spe_find_cpus(session->evlist);
@@ -363,7 +363,7 @@ static int arm_spe_setup_tracking_event(struct evlist *evlist,
 {
 	int err;
 	struct evsel *tracking_evsel;
-	struct perf_cpu_map *cpus = evlist->core.user_requested_cpus;
+	struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus;
 
 	/* Add dummy event to keep tracking */
 	err = parse_event(evlist, "dummy:u");
@@ -396,7 +396,7 @@ static int arm_spe_recording_options(struct auxtrace_record *itr,
 	struct arm_spe_recording *sper =
 			container_of(itr, struct arm_spe_recording, itr);
 	struct evsel *evsel, *tmp;
-	struct perf_cpu_map *cpus = evlist->core.user_requested_cpus;
+	struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus;
 	bool discard = false;
 	int err;
 	u64 discard_bit;
diff --git a/tools/perf/arch/arm64/util/hisi-ptt.c b/tools/perf/arch/arm64/util/hisi-ptt.c
index fe457fd58c9e8..52257715d2b74 100644
--- a/tools/perf/arch/arm64/util/hisi-ptt.c
+++ b/tools/perf/arch/arm64/util/hisi-ptt.c
@@ -53,7 +53,7 @@ static int hisi_ptt_info_fill(struct auxtrace_record *itr,
 	if (priv_size != HISI_PTT_AUXTRACE_PRIV_SIZE)
 		return -EINVAL;
 
-	if (!session->evlist->core.nr_mmaps)
+	if (!evlist__core(session->evlist)->nr_mmaps)
 		return -EINVAL;
 
 	auxtrace_info->type = PERF_AUXTRACE_HISI_PTT;
diff --git a/tools/perf/arch/x86/tests/amd-ibs-period.c b/tools/perf/arch/x86/tests/amd-ibs-period.c
index 6a92b3a23ed7a..32713f8fcd5c8 100644
--- a/tools/perf/arch/x86/tests/amd-ibs-period.c
+++ b/tools/perf/arch/x86/tests/amd-ibs-period.c
@@ -46,7 +46,6 @@ static int dummy_workload_1(unsigned long count)
 {
 	int (*func)(void);
 	int ret = 0;
-	char *p;
 	char insn1[] = {
 		0xb8, 0x01, 0x00, 0x00, 0x00, /* mov 1,%eax */
 		0xc3, /* ret */
@@ -59,18 +58,11 @@ static int dummy_workload_1(unsigned long count)
 		0xcc, /* int 3 */
 	};
 
-	p = calloc(2, page_size);
-	if (!p) {
-		printf("malloc() failed. %m");
-		return 1;
-	}
-
-	func = (void *)((unsigned long)(p + page_size - 1) & ~(page_size - 1));
-
-	ret = mprotect(func, page_size, PROT_READ | PROT_WRITE | PROT_EXEC);
-	if (ret) {
-		printf("mprotect() failed. %m");
-		goto out;
+	func = mmap(NULL, page_size, PROT_READ | PROT_WRITE | PROT_EXEC,
+		    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+	if (func == MAP_FAILED) {
+		pr_debug("mmap() failed. %m\n");
+		return -1;
 	}
 
 	if (count < 100000)
@@ -93,7 +85,7 @@ static int dummy_workload_1(unsigned long count)
 	}
 
 out:
-	free(p);
+	munmap(func, page_size);
 	return ret;
 }
 
diff --git a/tools/perf/arch/x86/tests/hybrid.c b/tools/perf/arch/x86/tests/hybrid.c
index e221ea1041740..0477e17b8e53d 100644
--- a/tools/perf/arch/x86/tests/hybrid.c
+++ b/tools/perf/arch/x86/tests/hybrid.c
@@ -26,7 +26,7 @@ static int test__hybrid_hw_event_with_pmu(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_VAL("wrong number of entries", 1 == evlist->core.nr_entries);
+	TEST_ASSERT_VAL("wrong number of entries", 1 == evlist__nr_entries(evlist));
 	TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type);
 	TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW));
 	TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES));
@@ -38,7 +38,7 @@ static int test__hybrid_hw_group_event(struct evlist *evlist)
 	struct evsel *evsel, *leader;
 
 	evsel = leader = evlist__first(evlist);
-	TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries);
+	TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist));
 	TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type);
 	TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW));
 	TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES));
@@ -57,7 +57,7 @@ static int test__hybrid_sw_hw_group_event(struct evlist *evlist)
 	struct evsel *evsel, *leader;
 
 	evsel = leader = evlist__first(evlist);
-	TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries);
+	TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist));
 	TEST_ASSERT_VAL("wrong type", PERF_TYPE_SOFTWARE == evsel->core.attr.type);
 	TEST_ASSERT_VAL("wrong leader", evsel__has_leader(evsel, leader));
 
@@ -74,7 +74,7 @@ static int test__hybrid_hw_sw_group_event(struct evlist *evlist)
 	struct evsel *evsel, *leader;
 
 	evsel = leader = evlist__first(evlist);
-	TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries);
+	TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist));
 	TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type);
 	TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW));
 	TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES));
@@ -91,7 +91,7 @@ static int test__hybrid_group_modifier1(struct evlist *evlist)
 	struct evsel *evsel, *leader;
 
 	evsel = leader = evlist__first(evlist);
-	TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries);
+	TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist));
 	TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type);
 	TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW));
 	TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES));
@@ -113,7 +113,7 @@ static int test__hybrid_raw1(struct evlist *evlist)
 {
 	struct perf_evsel *evsel;
 
-	perf_evlist__for_each_evsel(&evlist->core, evsel) {
+	perf_evlist__for_each_evsel(evlist__core(evlist), evsel) {
 		struct perf_pmu *pmu = perf_pmus__find_by_type(evsel->attr.type);
 
 		TEST_ASSERT_VAL("missing pmu", pmu);
@@ -127,7 +127,7 @@ static int test__hybrid_raw2(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_VAL("wrong number of entries", 1 == evlist->core.nr_entries);
+	TEST_ASSERT_VAL("wrong number of entries", 1 == evlist__nr_entries(evlist));
 	TEST_ASSERT_VAL("wrong type", PERF_TYPE_RAW == evsel->core.attr.type);
 	TEST_ASSERT_VAL("wrong config", test_config(evsel, 0x1a));
 	return TEST_OK;
@@ -137,7 +137,7 @@ static int test__hybrid_cache_event(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_VAL("wrong number of entries", 1 == evlist->core.nr_entries);
+	TEST_ASSERT_VAL("wrong number of entries", 1 == evlist__nr_entries(evlist));
 	TEST_ASSERT_VAL("wrong type", PERF_TYPE_HW_CACHE == evsel->core.attr.type);
 	TEST_ASSERT_VAL("wrong config", 0x2 == (evsel->core.attr.config & 0xffffffff));
 	return TEST_OK;
@@ -148,7 +148,7 @@ static int test__checkevent_pmu(struct evlist *evlist)
 
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_VAL("wrong number of entries", 1 == evlist->core.nr_entries);
+	TEST_ASSERT_VAL("wrong number of entries", 1 == evlist__nr_entries(evlist));
 	TEST_ASSERT_VAL("wrong type", PERF_TYPE_RAW == evsel->core.attr.type);
 	TEST_ASSERT_VAL("wrong config",    10 == evsel->core.attr.config);
 	TEST_ASSERT_VAL("wrong config1",    1 == evsel->core.attr.config1);
@@ -168,7 +168,7 @@ static int test__hybrid_hw_group_event_2(struct evlist *evlist)
 	struct evsel *evsel, *leader;
 
 	evsel = leader = evlist__first(evlist);
-	TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->core.nr_entries);
+	TEST_ASSERT_VAL("wrong number of entries", 2 == evlist__nr_entries(evlist));
 	TEST_ASSERT_VAL("wrong type", PERF_TYPE_HARDWARE == evsel->core.attr.type);
 	TEST_ASSERT_VAL("wrong hybrid type", test_hybrid_type(evsel, PERF_TYPE_RAW));
 	TEST_ASSERT_VAL("wrong config", test_config(evsel, PERF_COUNT_HW_CPU_CYCLES));
@@ -268,7 +268,7 @@ static int test_event(const struct evlist_test *e)
 		ret = e->check(evlist);
 	}
 	parse_events_error__exit(&err);
-	evlist__delete(evlist);
+	evlist__put(evlist);
 
 	return ret;
 }
diff --git a/tools/perf/arch/x86/tests/topdown.c b/tools/perf/arch/x86/tests/topdown.c
index 221f2c4bbb615..2b6f47ce49324 100644
--- a/tools/perf/arch/x86/tests/topdown.c
+++ b/tools/perf/arch/x86/tests/topdown.c
@@ -56,7 +56,7 @@ static int event_cb(void *state, struct pmu_event_info *info)
 			*ret = TEST_FAIL;
 		}
 	}
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	return 0;
 }
 
@@ -174,7 +174,7 @@ static int test_sort(const char *str, int expected_slots_group_size,
 	CHECK_COND(slots_seen, "slots seen");
 	ret = TEST_OK;
 out_err:
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	parse_events_error__exit(&err);
 	return ret;
 }
diff --git a/tools/perf/arch/x86/util/auxtrace.c b/tools/perf/arch/x86/util/auxtrace.c
index ecbf61a7eb3a3..84fce0b51ccf7 100644
--- a/tools/perf/arch/x86/util/auxtrace.c
+++ b/tools/perf/arch/x86/util/auxtrace.c
@@ -55,7 +55,7 @@ struct auxtrace_record *auxtrace_record__init(struct evlist *evlist,
 					      int *err)
 {
 	char buffer[64];
-	struct perf_cpu cpu = perf_cpu_map__min(evlist->core.all_cpus);
+	struct perf_cpu cpu = perf_cpu_map__min(evlist__core(evlist)->all_cpus);
 	int ret;
 
 	*err = 0;
diff --git a/tools/perf/arch/x86/util/intel-bts.c b/tools/perf/arch/x86/util/intel-bts.c
index 100a23d27998f..d44d568a6d210 100644
--- a/tools/perf/arch/x86/util/intel-bts.c
+++ b/tools/perf/arch/x86/util/intel-bts.c
@@ -79,10 +79,10 @@ static int intel_bts_info_fill(struct auxtrace_record *itr,
 	if (priv_size != INTEL_BTS_AUXTRACE_PRIV_SIZE)
 		return -EINVAL;
 
-	if (!session->evlist->core.nr_mmaps)
+	if (!evlist__core(session->evlist)->nr_mmaps)
 		return -EINVAL;
 
-	pc = session->evlist->mmap[0].core.base;
+	pc = evlist__mmap(session->evlist)[0].core.base;
 	if (pc) {
 		err = perf_read_tsc_conversion(pc, &tc);
 		if (err) {
@@ -114,7 +114,7 @@ static int intel_bts_recording_options(struct auxtrace_record *itr,
 			container_of(itr, struct intel_bts_recording, itr);
 	struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu;
 	struct evsel *evsel, *intel_bts_evsel = NULL;
-	const struct perf_cpu_map *cpus = evlist->core.user_requested_cpus;
+	const struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus;
 	bool privileged = perf_event_paranoid_check(-1);
 
 	if (opts->auxtrace_sample_mode) {
diff --git a/tools/perf/arch/x86/util/intel-pt.c b/tools/perf/arch/x86/util/intel-pt.c
index 0307ff15d9fc9..a533114c0048c 100644
--- a/tools/perf/arch/x86/util/intel-pt.c
+++ b/tools/perf/arch/x86/util/intel-pt.c
@@ -360,10 +360,10 @@ static int intel_pt_info_fill(struct auxtrace_record *itr,
 	filter = intel_pt_find_filter(session->evlist, ptr->intel_pt_pmu);
 	filter_str_len = filter ? strlen(filter) : 0;
 
-	if (!session->evlist->core.nr_mmaps)
+	if (!evlist__core(session->evlist)->nr_mmaps)
 		return -EINVAL;
 
-	pc = session->evlist->mmap[0].core.base;
+	pc = evlist__mmap(session->evlist)[0].core.base;
 	if (pc) {
 		err = perf_read_tsc_conversion(pc, &tc);
 		if (err) {
@@ -376,7 +376,8 @@ static int intel_pt_info_fill(struct auxtrace_record *itr,
 			ui__warning("Intel Processor Trace: TSC not available\n");
 	}
 
-	per_cpu_mmaps = !perf_cpu_map__is_any_cpu_or_is_empty(session->evlist->core.user_requested_cpus);
+	per_cpu_mmaps = !perf_cpu_map__is_any_cpu_or_is_empty(
+		evlist__core(session->evlist)->user_requested_cpus);
 
 	auxtrace_info->type = PERF_AUXTRACE_INTEL_PT;
 	auxtrace_info->priv[INTEL_PT_PMU_TYPE] = intel_pt_pmu->type;
@@ -621,7 +622,7 @@ static int intel_pt_recording_options(struct auxtrace_record *itr,
 	struct perf_pmu *intel_pt_pmu = ptr->intel_pt_pmu;
 	bool have_timing_info, need_immediate = false;
 	struct evsel *evsel, *intel_pt_evsel = NULL;
-	const struct perf_cpu_map *cpus = evlist->core.user_requested_cpus;
+	const struct perf_cpu_map *cpus = evlist__core(evlist)->user_requested_cpus;
 	bool privileged = perf_event_paranoid_check(-1);
 	u64 tsc_bit;
 	int err;
diff --git a/tools/perf/arch/x86/util/iostat.c b/tools/perf/arch/x86/util/iostat.c
index 7442a2cd87eda..b13abea3a6f48 100644
--- a/tools/perf/arch/x86/util/iostat.c
+++ b/tools/perf/arch/x86/util/iostat.c
@@ -332,13 +332,15 @@ static int iostat_event_group(struct evlist *evl,
 	return ret;
 }
 
-int iostat_prepare(struct evlist *evlist, struct perf_stat_config *config)
+int iostat_prepare(struct evlist **evlist_ptr, struct perf_stat_config *config)
 {
-	if (evlist->core.nr_entries > 0) {
+	struct evlist *evlist = *evlist_ptr;
+
+	if (evlist__nr_entries(evlist) > 0) {
 		pr_warning("The -e and -M options are not supported."
 			   "All chosen events/metrics will be dropped\n");
-		evlist__delete(evlist);
-		evlist = evlist__new();
+		evlist__put(evlist);
+		*evlist_ptr = evlist = evlist__new();
 		if (!evlist)
 			return -ENOMEM;
 	}
@@ -400,7 +402,7 @@ void iostat_prefix(struct evlist *evlist,
 		   struct perf_stat_config *config,
 		   char *prefix, struct timespec *ts)
 {
-	struct iio_root_port *rp = evlist->selected->priv;
+	struct iio_root_port *rp = evlist__selected(evlist)->priv;
 
 	if (rp) {
 		/*
@@ -463,7 +465,7 @@ void iostat_print_counters(struct evlist *evlist,
 	iostat_prefix(evlist, config, prefix, ts);
 	fprintf(config->output, "%s", prefix);
 	evlist__for_each_entry(evlist, counter) {
-		perf_device = evlist->selected->priv;
+		perf_device = evlist__selected(evlist)->priv;
 		if (perf_device && perf_device != counter->priv) {
 			evlist__set_selected(evlist, counter);
 			iostat_prefix(evlist, config, prefix, ts);
diff --git a/tools/perf/bench/evlist-open-close.c b/tools/perf/bench/evlist-open-close.c
index faf9c34b4a5dc..748ebbe458f49 100644
--- a/tools/perf/bench/evlist-open-close.c
+++ b/tools/perf/bench/evlist-open-close.c
@@ -76,7 +76,7 @@ static struct evlist *bench__create_evlist(char *evstr, const char *uid_str)
 		parse_events_error__exit(&err);
 		pr_err("Run 'perf list' for a list of valid events\n");
 		ret = 1;
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 	parse_events_error__exit(&err);
 	if (uid_str) {
@@ -85,24 +85,24 @@ static struct evlist *bench__create_evlist(char *evstr, const char *uid_str)
 		if (uid == UINT_MAX) {
 			pr_err("Invalid User: %s", uid_str);
 			ret = -EINVAL;
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 		ret = parse_uid_filter(evlist, uid);
 		if (ret)
-			goto out_delete_evlist;
+			goto out_put_evlist;
 	}
 	ret = evlist__create_maps(evlist, &opts.target);
 	if (ret < 0) {
 		pr_err("Not enough memory to create thread/cpu maps\n");
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	evlist__config(evlist, &opts, NULL);
 
 	return evlist;
 
-out_delete_evlist:
-	evlist__delete(evlist);
+out_put_evlist:
+	evlist__put(evlist);
 	return NULL;
 }
 
@@ -116,7 +116,7 @@ static int bench__do_evlist_open_close(struct evlist *evlist)
 		return err;
 	}
 
-	err = evlist__mmap(evlist, opts.mmap_pages);
+	err = evlist__do_mmap(evlist, opts.mmap_pages);
 	if (err < 0) {
 		pr_err("evlist__mmap: %s\n", str_error_r(errno, sbuf, sizeof(sbuf)));
 		return err;
@@ -124,7 +124,7 @@ static int bench__do_evlist_open_close(struct evlist *evlist)
 
 	evlist__enable(evlist);
 	evlist__disable(evlist);
-	evlist__munmap(evlist);
+	evlist__do_munmap(evlist);
 	evlist__close(evlist);
 
 	return 0;
@@ -145,13 +145,14 @@ static int bench_evlist_open_close__run(char *evstr, const char *uid_str)
 
 	init_stats(&time_stats);
 
-	printf("  Number of cpus:\t%d\n", perf_cpu_map__nr(evlist->core.user_requested_cpus));
-	printf("  Number of threads:\t%d\n", evlist->core.threads->nr);
+	printf("  Number of cpus:\t%d\n",
+	       perf_cpu_map__nr(evlist__core(evlist)->user_requested_cpus));
+	printf("  Number of threads:\t%d\n", evlist__core(evlist)->threads->nr);
 	printf("  Number of events:\t%d (%d fds)\n",
-		evlist->core.nr_entries, evlist__count_evsel_fds(evlist));
+		evlist__nr_entries(evlist), evlist__count_evsel_fds(evlist));
 	printf("  Number of iterations:\t%d\n", iterations);
 
-	evlist__delete(evlist);
+	evlist__put(evlist);
 
 	for (i = 0; i < iterations; i++) {
 		pr_debug("Started iteration %d\n", i);
@@ -162,7 +163,7 @@ static int bench_evlist_open_close__run(char *evstr, const char *uid_str)
 		gettimeofday(&start, NULL);
 		err = bench__do_evlist_open_close(evlist);
 		if (err) {
-			evlist__delete(evlist);
+			evlist__put(evlist);
 			return err;
 		}
 
@@ -171,7 +172,7 @@ static int bench_evlist_open_close__run(char *evstr, const char *uid_str)
 		runtime_us = timeval2usec(&diff);
 		update_stats(&time_stats, runtime_us);
 
-		evlist__delete(evlist);
+		evlist__put(evlist);
 		pr_debug("Iteration %d took:\t%" PRIu64 "us\n", i, runtime_us);
 	}
 
diff --git a/tools/perf/builtin-annotate.c b/tools/perf/builtin-annotate.c
index 8a0eb30eac24f..69cb72b2082a6 100644
--- a/tools/perf/builtin-annotate.c
+++ b/tools/perf/builtin-annotate.c
@@ -562,7 +562,7 @@ static int __cmd_annotate(struct perf_annotate *ann)
 		goto out;
 
 	if ((use_browser == 1 || ann->use_stdio2) && ann->has_br_stack)
-		if (session->evlist->nr_br_cntr > 0)
+		if (evlist__nr_br_cntr(session->evlist) > 0)
 			annotate_opts.show_br_cntr = true;
 
 	if (dump_trace) {
@@ -928,8 +928,11 @@ int cmd_annotate(int argc, const char **argv)
 	 * branch counters, if the corresponding branch info is available
 	 * in the perf data in the TUI mode.
 	 */
-	if ((use_browser == 1 || annotate.use_stdio2) && annotate.has_br_stack)
+	if ((use_browser == 1 || annotate.use_stdio2) && annotate.has_br_stack) {
 		sort__mode = SORT_MODE__BRANCH;
+		if (evlist__nr_br_cntr(annotate.session->evlist) > 0)
+			annotate_opts.show_br_cntr = true;
+	}
 
 	if (setup_sorting(/*evlist=*/NULL, perf_session__env(annotate.session)) < 0)
 		usage_with_options(annotate_usage, options);
diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index 8a7dbfb14535e..9e4c5220d43c2 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -377,9 +377,9 @@ static int set_tracing_pid(struct perf_ftrace *ftrace)
 	if (target__has_cpu(&ftrace->target))
 		return 0;
 
-	for (i = 0; i < perf_thread_map__nr(ftrace->evlist->core.threads); i++) {
+	for (i = 0; i < perf_thread_map__nr(evlist__core(ftrace->evlist)->threads); i++) {
 		scnprintf(buf, sizeof(buf), "%d",
-			  perf_thread_map__pid(ftrace->evlist->core.threads, i));
+			  perf_thread_map__pid(evlist__core(ftrace->evlist)->threads, i));
 		if (append_tracing_file("set_ftrace_pid", buf) < 0)
 			return -1;
 	}
@@ -413,7 +413,7 @@ static int set_tracing_cpumask(struct perf_cpu_map *cpumap)
 
 static int set_tracing_cpu(struct perf_ftrace *ftrace)
 {
-	struct perf_cpu_map *cpumap = ftrace->evlist->core.user_requested_cpus;
+	struct perf_cpu_map *cpumap = evlist__core(ftrace->evlist)->user_requested_cpus;
 
 	if (!target__has_cpu(&ftrace->target))
 		return 0;
@@ -1999,20 +1999,20 @@ int cmd_ftrace(int argc, const char **argv)
 
 	ret = evlist__create_maps(ftrace.evlist, &ftrace.target);
 	if (ret < 0)
-		goto out_delete_evlist;
+		goto out_put_evlist;
 
 	if (argc) {
 		ret = evlist__prepare_workload(ftrace.evlist, &ftrace.target,
 					       argv, false,
 					       ftrace__workload_exec_failed_signal);
 		if (ret < 0)
-			goto out_delete_evlist;
+			goto out_put_evlist;
 	}
 
 	ret = cmd_func(&ftrace);
 
-out_delete_evlist:
-	evlist__delete(ftrace.evlist);
+out_put_evlist:
+	evlist__put(ftrace.evlist);
 
 out_delete_filters:
 	delete_filter_func(&ftrace.filters);
diff --git a/tools/perf/builtin-inject.c b/tools/perf/builtin-inject.c
index 6d6cce4765a7f..70bbfad5653e1 100644
--- a/tools/perf/builtin-inject.c
+++ b/tools/perf/builtin-inject.c
@@ -726,8 +726,11 @@ static int perf_event__repipe_common_mmap(const struct perf_tool *tool,
 		}
 
 		if (dso && !dso__hit(dso)) {
-			if (!sample->evsel)
+			if (!sample->evsel) {
 				sample->evsel = evlist__event2evsel(inject->session->evlist, event);
+				if (sample->evsel)
+					evsel__get(sample->evsel);
+			}
 
 			if (sample->evsel) {
 				dso__set_hit(dso);
@@ -1520,7 +1523,7 @@ static int synthesize_id_index(struct perf_inject *inject, size_t new_cnt)
 	struct perf_session *session = inject->session;
 	struct evlist *evlist = session->evlist;
 	struct machine *machine = &session->machines.host;
-	size_t from = evlist->core.nr_entries - new_cnt;
+	size_t from = evlist__nr_entries(evlist) - new_cnt;
 
 	return __perf_event__synthesize_id_index(&inject->tool, perf_event__repipe,
 						 evlist, machine, from);
@@ -2055,7 +2058,7 @@ static int host__finished_init(const struct perf_tool *tool, struct perf_session
 	if (ret)
 		return ret;
 
-	ret = synthesize_id_index(inject, gs->session->evlist->core.nr_entries);
+	ret = synthesize_id_index(inject, evlist__nr_entries(gs->session->evlist));
 	if (ret) {
 		pr_err("Failed to synthesize id_index\n");
 		return ret;
diff --git a/tools/perf/builtin-kvm.c b/tools/perf/builtin-kvm.c
index 394302ebdb161..00baafb056032 100644
--- a/tools/perf/builtin-kvm.c
+++ b/tools/perf/builtin-kvm.c
@@ -51,6 +51,7 @@
 #include <termios.h>
 #include <semaphore.h>
 #include <signal.h>
+#include <stdlib.h>
 #include <math.h>
 #include <perf/mmap.h>
 
@@ -606,11 +607,11 @@ static const char *get_filename_for_perf_kvm(void)
 	const char *filename;
 
 	if (perf_host && !perf_guest)
-		filename = strdup("perf.data.host");
+		filename = "perf.data.host";
 	else if (!perf_host && perf_guest)
-		filename = strdup("perf.data.guest");
+		filename = "perf.data.guest";
 	else
-		filename = strdup("perf.data.kvm");
+		filename = "perf.data.kvm";
 
 	return filename;
 }
@@ -1221,7 +1222,7 @@ static s64 perf_kvm__mmap_read_idx(struct perf_kvm_stat *kvm, int idx,
 	int err;
 
 	*mmap_time = ULLONG_MAX;
-	md = &evlist->mmap[idx];
+	md = &evlist__mmap(evlist)[idx];
 	err = perf_mmap__read_init(&md->core);
 	if (err < 0)
 		return (err == -EAGAIN) ? 0 : -1;
@@ -1266,7 +1267,7 @@ static int perf_kvm__mmap_read(struct perf_kvm_stat *kvm)
 	s64 n, ntotal = 0;
 	u64 flush_time = ULLONG_MAX, mmap_time;
 
-	for (i = 0; i < kvm->evlist->core.nr_mmaps; i++) {
+	for (i = 0; i < evlist__core(kvm->evlist)->nr_mmaps; i++) {
 		n = perf_kvm__mmap_read_idx(kvm, i, &mmap_time);
 		if (n < 0)
 			return -1;
@@ -1449,7 +1450,7 @@ static int kvm_events_live_report(struct perf_kvm_stat *kvm)
 	evlist__enable(kvm->evlist);
 
 	while (!done) {
-		struct fdarray *fda = &kvm->evlist->core.pollfd;
+		struct fdarray *fda = &evlist__core(kvm->evlist)->pollfd;
 		int rc;
 
 		rc = perf_kvm__mmap_read(kvm);
@@ -1531,7 +1532,7 @@ static int kvm_live_open_events(struct perf_kvm_stat *kvm)
 		goto out;
 	}
 
-	if (evlist__mmap(evlist, kvm->opts.mmap_pages) < 0) {
+	if (evlist__do_mmap(evlist, kvm->opts.mmap_pages) < 0) {
 		ui__error("Failed to mmap the events: %s\n",
 			  str_error_r(errno, sbuf, sizeof(sbuf)));
 		evlist__close(evlist);
@@ -1681,18 +1682,18 @@ kvm_events_record(struct perf_kvm_stat *kvm, int argc, const char **argv)
 		return -ENOMEM;
 
 	for (i = 0; i < ARRAY_SIZE(record_args); i++)
-		rec_argv[i] = STRDUP_FAIL_EXIT(record_args[i]);
+		rec_argv[i] = record_args[i];
 
 	for (j = 0; j < events_tp_size; j++) {
-		rec_argv[i++] = STRDUP_FAIL_EXIT("-e");
-		rec_argv[i++] = STRDUP_FAIL_EXIT(kvm_events_tp(e_machine)[j]);
+		rec_argv[i++] = "-e";
+		rec_argv[i++] = kvm_events_tp(e_machine)[j];
 	}
 
-	rec_argv[i++] = STRDUP_FAIL_EXIT("-o");
-	rec_argv[i++] = STRDUP_FAIL_EXIT(kvm->file_name);
+	rec_argv[i++] = "-o";
+	rec_argv[i++] = kvm->file_name;
 
 	for (j = 1; j < (unsigned int)argc; j++, i++)
-		rec_argv[i] = STRDUP_FAIL_EXIT(argv[j]);
+		rec_argv[i] = argv[j];
 
 	set_option_flag(record_options, 'e', "event", PARSE_OPT_HIDDEN);
 	set_option_flag(record_options, 0, "filter", PARSE_OPT_HIDDEN);
@@ -1716,10 +1717,6 @@ kvm_events_record(struct perf_kvm_stat *kvm, int argc, const char **argv)
 
 	record_usage = kvm_stat_record_usage;
 	ret = cmd_record(i, rec_argv);
-
-EXIT:
-	for (i = 0; i < rec_argc; i++)
-		free((void *)rec_argv[i]);
 	free(rec_argv);
 	return ret;
 }
@@ -1810,7 +1807,7 @@ static struct evlist *kvm_live_event_list(void)
 
 out:
 	if (err) {
-		evlist__delete(evlist);
+		evlist__put(evlist);
 		evlist = NULL;
 	}
 
@@ -1931,7 +1928,7 @@ static int kvm_events_live(struct perf_kvm_stat *kvm,
 	perf_session__set_id_hdr_size(kvm->session);
 	ordered_events__set_copy_on_queue(&kvm->session->ordered_events, true);
 	machine__synthesize_threads(&kvm->session->machines.host, &kvm->opts.target,
-				    kvm->evlist->core.threads, true, false, 1);
+				    evlist__core(kvm->evlist)->threads, true, false, 1);
 	err = kvm_live_open_events(kvm);
 	if (err)
 		goto out;
@@ -1941,7 +1938,7 @@ static int kvm_events_live(struct perf_kvm_stat *kvm,
 out:
 	perf_session__delete(kvm->session);
 	kvm->session = NULL;
-	evlist__delete(kvm->evlist);
+	evlist__put(kvm->evlist);
 
 	return err;
 }
@@ -1995,34 +1992,35 @@ static int __cmd_record(const char *file_name, int argc, const char **argv)
 {
 	int rec_argc, i = 0, j, ret;
 	const char **rec_argv;
+	int need_arch_event = !!kvm_need_default_arch_event(EM_HOST, argc, argv);
 
 	/*
 	 * Besides the 2 more options "-o" and "filename",
 	 * kvm_add_default_arch_event() may add 2 extra options,
-	 * so allocate 4 more items.
+	 * so allocate more items conditionally.
 	 */
-	rec_argc = argc + 2 + 2;
+	rec_argc = argc + 2 + (2 * need_arch_event);
 	rec_argv = calloc(rec_argc + 1, sizeof(char *));
 	if (!rec_argv)
 		return -ENOMEM;
 
-	rec_argv[i++] = STRDUP_FAIL_EXIT("record");
-	rec_argv[i++] = STRDUP_FAIL_EXIT("-o");
-	rec_argv[i++] = STRDUP_FAIL_EXIT(file_name);
-	for (j = 1; j < argc; j++, i++)
-		rec_argv[i] = STRDUP_FAIL_EXIT(argv[j]);
+	rec_argv[i++] = "record";
+	rec_argv[i++] = "-o";
+	rec_argv[i++] = file_name;
+	if (need_arch_event) {
+		ret = kvm_add_default_arch_event(EM_HOST, &i, rec_argv);
+		if (ret)
+			goto EXIT;
+	}
 
-	BUG_ON(i + 2 != rec_argc);
+	for (j = 1; j < argc; j++, i++)
+		rec_argv[i] = argv[j];
 
-	ret = kvm_add_default_arch_event(EM_HOST, &i, rec_argv);
-	if (ret)
-		goto EXIT;
+	BUG_ON(i != rec_argc);
 
 	ret = cmd_record(i, rec_argv);
 
 EXIT:
-	for (i = 0; i < rec_argc; i++)
-		free((void *)rec_argv[i]);
 	free(rec_argv);
 	return ret;
 }
@@ -2037,19 +2035,15 @@ static int __cmd_report(const char *file_name, int argc, const char **argv)
 	if (!rec_argv)
 		return -ENOMEM;
 
-	rec_argv[i++] = STRDUP_FAIL_EXIT("report");
-	rec_argv[i++] = STRDUP_FAIL_EXIT("-i");
-	rec_argv[i++] = STRDUP_FAIL_EXIT(file_name);
+	rec_argv[i++] = "report";
+	rec_argv[i++] = "-i";
+	rec_argv[i++] = file_name;
 	for (j = 1; j < argc; j++, i++)
-		rec_argv[i] = STRDUP_FAIL_EXIT(argv[j]);
+		rec_argv[i] = argv[j];
 
 	BUG_ON(i != rec_argc);
 
 	ret = cmd_report(i, rec_argv);
-
-EXIT:
-	for (i = 0; i < rec_argc; i++)
-		free((void *)rec_argv[i]);
 	free(rec_argv);
 	return ret;
 }
@@ -2065,19 +2059,15 @@ __cmd_buildid_list(const char *file_name, int argc, const char **argv)
 	if (!rec_argv)
 		return -ENOMEM;
 
-	rec_argv[i++] = STRDUP_FAIL_EXIT("buildid-list");
-	rec_argv[i++] = STRDUP_FAIL_EXIT("-i");
-	rec_argv[i++] = STRDUP_FAIL_EXIT(file_name);
+	rec_argv[i++] = "buildid-list";
+	rec_argv[i++] = "-i";
+	rec_argv[i++] = file_name;
 	for (j = 1; j < argc; j++, i++)
-		rec_argv[i] = STRDUP_FAIL_EXIT(argv[j]);
+		rec_argv[i] = argv[j];
 
 	BUG_ON(i != rec_argc);
 
 	ret = cmd_buildid_list(i, rec_argv);
-
-EXIT:
-	for (i = 0; i < rec_argc; i++)
-		free((void *)rec_argv[i]);
 	free(rec_argv);
 	return ret;
 }
@@ -2097,19 +2087,19 @@ static int __cmd_top(int argc, const char **argv)
 		return -ENOMEM;
 
 	for (i = 0; i < argc; i++)
-		rec_argv[i] = STRDUP_FAIL_EXIT(argv[i]);
+		rec_argv[i] = argv[i];
 
 	BUG_ON(i != argc);
 
-	ret = kvm_add_default_arch_event(EM_HOST, &i, rec_argv);
-	if (ret)
-		goto EXIT;
+	if (kvm_need_default_arch_event(EM_HOST, argc, argv)) {
+		ret = kvm_add_default_arch_event(EM_HOST, &i, rec_argv);
+		if (ret)
+			goto EXIT;
+	}
 
 	ret = cmd_top(i, rec_argv);
 
 EXIT:
-	for (i = 0; i < rec_argc; i++)
-		free((void *)rec_argv[i]);
 	free(rec_argv);
 	return ret;
 }
@@ -2145,6 +2135,7 @@ int cmd_kvm(int argc, const char **argv)
 	const char *const kvm_subcommands[] = { "top", "record", "report", "diff",
 						"buildid-list", "stat", NULL };
 	const char *kvm_usage[] = { NULL, NULL };
+	int ret;
 
 	exclude_GH_default = true;
 	perf_host  = 0;
@@ -2158,28 +2149,22 @@ int cmd_kvm(int argc, const char **argv)
 	if (!perf_host)
 		perf_guest = 1;
 
-	if (!file_name) {
+	if (!file_name)
 		file_name = get_filename_for_perf_kvm();
 
-		if (!file_name) {
-			pr_err("Failed to allocate memory for filename\n");
-			return -ENOMEM;
-		}
-	}
-
 	if (strlen(argv[0]) > 2 && strstarts("record", argv[0]))
-		return __cmd_record(file_name, argc, argv);
+		ret = __cmd_record(file_name, argc, argv);
 	else if (strlen(argv[0]) > 2 && strstarts("report", argv[0]))
-		return __cmd_report(file_name, argc, argv);
+		ret = __cmd_report(file_name, argc, argv);
 	else if (strlen(argv[0]) > 2 && strstarts("diff", argv[0]))
-		return cmd_diff(argc, argv);
+		ret = cmd_diff(argc, argv);
 	else if (!strcmp(argv[0], "top"))
-		return __cmd_top(argc, argv);
+		ret = __cmd_top(argc, argv);
 	else if (strlen(argv[0]) > 2 && strstarts("buildid-list", argv[0]))
-		return __cmd_buildid_list(file_name, argc, argv);
+		ret = __cmd_buildid_list(file_name, argc, argv);
 #if defined(HAVE_LIBTRACEEVENT)
 	else if (strlen(argv[0]) > 2 && strstarts("stat", argv[0]))
-		return kvm_cmd_stat(file_name, argc, argv);
+		ret = kvm_cmd_stat(file_name, argc, argv);
 #endif
 	else
 		usage_with_options(kvm_usage, kvm_options);
@@ -2187,5 +2172,5 @@ int cmd_kvm(int argc, const char **argv)
 	/* free usage string allocated by parse_options_subcommand */
 	free((void *)kvm_usage[0]);
 
-	return 0;
+	return ret;
 }
diff --git a/tools/perf/builtin-kwork.c b/tools/perf/builtin-kwork.c
index 7b61168e01e9d..fce588441e499 100644
--- a/tools/perf/builtin-kwork.c
+++ b/tools/perf/builtin-kwork.c
@@ -1814,7 +1814,7 @@ static int perf_kwork__check_config(struct perf_kwork *kwork,
 		}
 	}
 
-	list_for_each_entry(evsel, &session->evlist->core.entries, core.node) {
+	list_for_each_entry(evsel, &evlist__core(session->evlist)->entries, core.node) {
 		if (kwork->show_callchain && !evsel__has_callchain(evsel)) {
 			pr_debug("Samples do not have callchains\n");
 			kwork->show_callchain = 0;
@@ -1864,9 +1864,9 @@ static int perf_kwork__read_events(struct perf_kwork *kwork)
 		goto out_delete;
 	}
 
-	kwork->nr_events      = session->evlist->stats.nr_events[0];
-	kwork->nr_lost_events = session->evlist->stats.total_lost;
-	kwork->nr_lost_chunks = session->evlist->stats.nr_events[PERF_RECORD_LOST];
+	kwork->nr_events      = evlist__stats(session->evlist)->nr_events[0];
+	kwork->nr_lost_events = evlist__stats(session->evlist)->total_lost;
+	kwork->nr_lost_chunks = evlist__stats(session->evlist)->nr_events[PERF_RECORD_LOST];
 
 out_delete:
 	perf_session__delete(session);
diff --git a/tools/perf/builtin-lock.c b/tools/perf/builtin-lock.c
index 5841d43be9718..d5c0d55cb82d9 100644
--- a/tools/perf/builtin-lock.c
+++ b/tools/perf/builtin-lock.c
@@ -2129,7 +2129,7 @@ static int __cmd_contention(int argc, const char **argv)
 			evlist__start_workload(con.evlist);
 
 		while (!done) {
-			if (argc && waitpid(con.evlist->workload.pid, NULL, WNOHANG) > 0)
+			if (argc && waitpid(evlist__workload_pid(con.evlist), NULL, WNOHANG) > 0)
 				break;
 			sleep(1);
 		}
@@ -2149,7 +2149,7 @@ static int __cmd_contention(int argc, const char **argv)
 
 out_delete:
 	lock_filter_finish();
-	evlist__delete(con.evlist);
+	evlist__put(con.evlist);
 	lock_contention_finish(&con);
 	perf_session__delete(session);
 	perf_env__exit(&host_env);
diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c
index e915390556752..f58d7e3c7879e 100644
--- a/tools/perf/builtin-record.c
+++ b/tools/perf/builtin-record.c
@@ -64,6 +64,7 @@
 #include <poll.h>
 #include <pthread.h>
 #include <unistd.h>
+#include <string.h>
 #ifndef HAVE_GETTID
 #include <syscall.h>
 #endif
@@ -502,12 +503,12 @@ static void record__aio_mmap_read_sync(struct record *rec)
 {
 	int i;
 	struct evlist *evlist = rec->evlist;
-	struct mmap *maps = evlist->mmap;
+	struct mmap *maps = evlist__mmap(evlist);
 
 	if (!record__aio_enabled(rec))
 		return;
 
-	for (i = 0; i < evlist->core.nr_mmaps; i++) {
+	for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) {
 		struct mmap *map = &maps[i];
 
 		if (map->core.base)
@@ -653,27 +654,14 @@ static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size)
 	struct record *rec = to;
 
 	if (record__comp_enabled(rec)) {
-		struct perf_record_compressed2 *event = map->data;
-		size_t padding = 0;
-		u8 pad[8] = {0};
 		ssize_t compressed = zstd_compress(rec->session, map, map->data,
 						   mmap__mmap_len(map), bf, size);
 
 		if (compressed < 0)
 			return (int)compressed;
 
-		bf = event;
 		thread->samples++;
-
-		/*
-		 * The record from `zstd_compress` is not 8 bytes aligned, which would cause asan
-		 * error. We make it aligned here.
-		 */
-		event->data_size = compressed - sizeof(struct perf_record_compressed2);
-		event->header.size = PERF_ALIGN(compressed, sizeof(u64));
-		padding = event->header.size - compressed;
-		return record__write(rec, map, bf, compressed) ||
-		       record__write(rec, map, &pad, padding);
+		return record__write(rec, map, map->data, compressed);
 	}
 
 	thread->samples++;
@@ -811,8 +799,8 @@ static int record__auxtrace_read_snapshot_all(struct record *rec)
 	int i;
 	int rc = 0;
 
-	for (i = 0; i < rec->evlist->core.nr_mmaps; i++) {
-		struct mmap *map = &rec->evlist->mmap[i];
+	for (i = 0; i < evlist__core(rec->evlist)->nr_mmaps; i++) {
+		struct mmap *map = &evlist__mmap(rec->evlist)[i];
 
 		if (!map->auxtrace_mmap.base)
 			continue;
@@ -1055,15 +1043,15 @@ static void record__thread_data_close_pipes(struct record_thread *thread_data)
 
 static bool evlist__per_thread(struct evlist *evlist)
 {
-	return cpu_map__is_dummy(evlist->core.user_requested_cpus);
+	return cpu_map__is_dummy(evlist__core(evlist)->user_requested_cpus);
 }
 
 static int record__thread_data_init_maps(struct record_thread *thread_data, struct evlist *evlist)
 {
-	int m, tm, nr_mmaps = evlist->core.nr_mmaps;
-	struct mmap *mmap = evlist->mmap;
-	struct mmap *overwrite_mmap = evlist->overwrite_mmap;
-	struct perf_cpu_map *cpus = evlist->core.all_cpus;
+	int m, tm, nr_mmaps = evlist__core(evlist)->nr_mmaps;
+	struct mmap *mmap = evlist__mmap(evlist);
+	struct mmap *overwrite_mmap = evlist__overwrite_mmap(evlist);
+	struct perf_cpu_map *cpus = evlist__core(evlist)->all_cpus;
 	bool per_thread = evlist__per_thread(evlist);
 
 	if (per_thread)
@@ -1118,16 +1106,17 @@ static int record__thread_data_init_pollfd(struct record_thread *thread_data, st
 		overwrite_map = thread_data->overwrite_maps ?
 				thread_data->overwrite_maps[tm] : NULL;
 
-		for (f = 0; f < evlist->core.pollfd.nr; f++) {
-			void *ptr = evlist->core.pollfd.priv[f].ptr;
+		for (f = 0; f < evlist__core(evlist)->pollfd.nr; f++) {
+			void *ptr = evlist__core(evlist)->pollfd.priv[f].ptr;
 
 			if ((map && ptr == map) || (overwrite_map && ptr == overwrite_map)) {
 				pos = fdarray__dup_entry_from(&thread_data->pollfd, f,
-							      &evlist->core.pollfd);
+							      &evlist__core(evlist)->pollfd);
 				if (pos < 0)
 					return pos;
 				pr_debug2("thread_data[%p]: pollfd[%d] <- event_fd=%d\n",
-					 thread_data, pos, evlist->core.pollfd.entries[f].fd);
+					 thread_data, pos,
+					 evlist__core(evlist)->pollfd.entries[f].fd);
 			}
 		}
 	}
@@ -1171,7 +1160,7 @@ static int record__update_evlist_pollfd_from_thread(struct record *rec,
 						    struct evlist *evlist,
 						    struct record_thread *thread_data)
 {
-	struct pollfd *e_entries = evlist->core.pollfd.entries;
+	struct pollfd *e_entries = evlist__core(evlist)->pollfd.entries;
 	struct pollfd *t_entries = thread_data->pollfd.entries;
 	int err = 0;
 	size_t i;
@@ -1195,7 +1184,7 @@ static int record__dup_non_perf_events(struct record *rec,
 				       struct evlist *evlist,
 				       struct record_thread *thread_data)
 {
-	struct fdarray *fda = &evlist->core.pollfd;
+	struct fdarray *fda = &evlist__core(evlist)->pollfd;
 	int i, ret;
 
 	for (i = 0; i < fda->nr; i++) {
@@ -1322,17 +1311,17 @@ static int record__mmap_evlist(struct record *rec,
 		return ret;
 
 	if (record__threads_enabled(rec)) {
-		ret = perf_data__create_dir(&rec->data, evlist->core.nr_mmaps);
+		ret = perf_data__create_dir(&rec->data, evlist__core(evlist)->nr_mmaps);
 		if (ret) {
 			errno = -ret;
 			pr_err("Failed to create data directory: %m\n");
 			return ret;
 		}
-		for (i = 0; i < evlist->core.nr_mmaps; i++) {
-			if (evlist->mmap)
-				evlist->mmap[i].file = &rec->data.dir.files[i];
-			if (evlist->overwrite_mmap)
-				evlist->overwrite_mmap[i].file = &rec->data.dir.files[i];
+		for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) {
+			if (evlist__mmap(evlist))
+				evlist__mmap(evlist)[i].file = &rec->data.dir.files[i];
+			if (evlist__overwrite_mmap(evlist))
+				evlist__overwrite_mmap(evlist)[i].file = &rec->data.dir.files[i];
 		}
 	}
 
@@ -1481,11 +1470,11 @@ static int record__open(struct record *rec)
 
 static void set_timestamp_boundary(struct record *rec, u64 sample_time)
 {
-	if (rec->evlist->first_sample_time == 0)
-		rec->evlist->first_sample_time = sample_time;
+	if (evlist__first_sample_time(rec->evlist) == 0)
+		evlist__set_first_sample_time(rec->evlist, sample_time);
 
 	if (sample_time)
-		rec->evlist->last_sample_time = sample_time;
+		evlist__set_last_sample_time(rec->evlist, sample_time);
 }
 
 static int process_sample_event(const struct perf_tool *tool,
@@ -1591,18 +1580,36 @@ static void record__adjust_affinity(struct record *rec, struct mmap *map)
 	}
 }
 
-static size_t process_comp_header(void *record, size_t increment)
+/*
+ * Called once with data_size == 0 to start a record, then once with
+ * data_size == compressed payload size to finalize and 8-byte-pad it
+ * (unaligned records trip ASan in the reader).
+ * Returns the bytes written, or -1 if it won't fit.
+ */
+static ssize_t process_comp_header(void *record, size_t dst_size,
+				   size_t data_size)
 {
 	struct perf_record_compressed2 *event = record;
 	size_t size = sizeof(*event);
 
-	if (increment) {
-		event->header.size += increment;
-		return increment;
+	if (data_size) {
+		size_t padding;
+
+		event->data_size = data_size;
+		event->header.size = PERF_ALIGN(size + data_size, sizeof(u64));
+		padding = event->header.size - size - data_size;
+		if (padding > dst_size)
+			return -1;
+		memset(record + size + data_size, 0, padding);
+		return padding;
 	}
 
+	if (size > dst_size)
+		return -1;
+
 	event->header.type = PERF_RECORD_COMPRESSED2;
 	event->header.size = size;
+	event->data_size = 0;
 
 	return size;
 }
@@ -1611,7 +1618,12 @@ static ssize_t zstd_compress(struct perf_session *session, struct mmap *map,
 			    void *dst, size_t dst_size, void *src, size_t src_size)
 {
 	ssize_t compressed;
-	size_t max_record_size = PERF_SAMPLE_MAX_SIZE - sizeof(struct perf_record_compressed2) - 1;
+	/*
+	 * Reserve space so per-record PERF_ALIGN() padding keeps header.size
+	 * within u16.
+	 */
+	size_t max_record_size = PERF_SAMPLE_MAX_SIZE
+		- sizeof(struct perf_record_compressed2) - sizeof(u64);
 	struct zstd_data *zstd_data = &session->zstd_data;
 
 	if (map && map->file)
@@ -1653,7 +1665,7 @@ static int record__mmap_read_evlist(struct record *rec, struct evlist *evlist,
 	if (!maps)
 		return 0;
 
-	if (overwrite && evlist->bkw_mmap_state != BKW_MMAP_DATA_PENDING)
+	if (overwrite && evlist__bkw_mmap_state(evlist) != BKW_MMAP_DATA_PENDING)
 		return 0;
 
 	if (record__aio_enabled(rec))
@@ -1808,7 +1820,7 @@ static void record__init_features(struct record *rec)
 	if (rec->no_buildid)
 		perf_header__clear_feat(&session->header, HEADER_BUILD_ID);
 
-	if (!have_tracepoints(&rec->evlist->core.entries))
+	if (!have_tracepoints(&evlist__core(rec->evlist)->entries))
 		perf_header__clear_feat(&session->header, HEADER_TRACING_DATA);
 
 	if (!rec->opts.branch_stack)
@@ -1874,7 +1886,7 @@ static int record__synthesize_workload(struct record *rec, bool tail)
 	if (rec->opts.tail_synthesize != tail)
 		return 0;
 
-	thread_map = thread_map__new_by_tid(rec->evlist->workload.pid);
+	thread_map = thread_map__new_by_tid(evlist__workload_pid(rec->evlist));
 	if (thread_map == NULL)
 		return -1;
 
@@ -2067,10 +2079,10 @@ static void alarm_sig_handler(int sig);
 static const struct perf_event_mmap_page *evlist__pick_pc(struct evlist *evlist)
 {
 	if (evlist) {
-		if (evlist->mmap && evlist->mmap[0].core.base)
-			return evlist->mmap[0].core.base;
-		if (evlist->overwrite_mmap && evlist->overwrite_mmap[0].core.base)
-			return evlist->overwrite_mmap[0].core.base;
+		if (evlist__mmap(evlist) && evlist__mmap(evlist)[0].core.base)
+			return evlist__mmap(evlist)[0].core.base;
+		if (evlist__overwrite_mmap(evlist) && evlist__overwrite_mmap(evlist)[0].core.base)
+			return evlist__overwrite_mmap(evlist)[0].core.base;
 	}
 	return NULL;
 }
@@ -2150,7 +2162,7 @@ static int record__synthesize(struct record *rec, bool tail)
 	if (err)
 		goto out;
 
-	err = perf_event__synthesize_thread_map2(&rec->tool, rec->evlist->core.threads,
+	err = perf_event__synthesize_thread_map2(&rec->tool, evlist__core(rec->evlist)->threads,
 						 process_synthesized_event,
 						NULL);
 	if (err < 0) {
@@ -2158,7 +2170,7 @@ static int record__synthesize(struct record *rec, bool tail)
 		return err;
 	}
 
-	err = perf_event__synthesize_cpu_map(&rec->tool, rec->evlist->core.all_cpus,
+	err = perf_event__synthesize_cpu_map(&rec->tool, evlist__core(rec->evlist)->all_cpus,
 					     process_synthesized_event, NULL);
 	if (err < 0) {
 		pr_err("Couldn't synthesize cpu map.\n");
@@ -2191,7 +2203,7 @@ static int record__synthesize(struct record *rec, bool tail)
 		bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP;
 
 		err = __machine__synthesize_threads(machine, tool, &opts->target,
-						    rec->evlist->core.threads,
+						    evlist__core(rec->evlist)->threads,
 						    f, needs_mmap, opts->record_data_mmap,
 						    rec->opts.nr_threads_synthesize);
 	}
@@ -2544,7 +2556,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv)
 	 * because we synthesize event name through the pipe
 	 * and need the id for that.
 	 */
-	if (data->is_pipe && rec->evlist->core.nr_entries == 1)
+	if (data->is_pipe && evlist__nr_entries(rec->evlist) == 1)
 		rec->opts.sample_id = true;
 
 	if (rec->timestamp_filename && perf_data__is_pipe(data)) {
@@ -2568,7 +2580,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv)
 	}
 	/* Debug message used by test scripts */
 	pr_debug3("perf record done opening and mmapping events\n");
-	env->comp_mmap_len = session->evlist->core.mmap_len;
+	env->comp_mmap_len = evlist__core(session->evlist)->mmap_len;
 
 	if (rec->opts.kcore) {
 		err = record__kcore_copy(&session->machines.host, data);
@@ -2669,7 +2681,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv)
 		 * Synthesize COMM event to prevent it.
 		 */
 		tgid = perf_event__synthesize_comm(tool, event,
-						   rec->evlist->workload.pid,
+						   evlist__workload_pid(rec->evlist),
 						   process_synthesized_event,
 						   machine);
 		free(event);
@@ -2689,7 +2701,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv)
 		 * Synthesize NAMESPACES event for the command specified.
 		 */
 		perf_event__synthesize_namespaces(tool, event,
-						  rec->evlist->workload.pid,
+						  evlist__workload_pid(rec->evlist),
 						  tgid, process_synthesized_event,
 						  machine);
 		free(event);
@@ -2706,7 +2718,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv)
 		}
 	}
 
-	err = event_enable_timer__start(rec->evlist->eet);
+	err = event_enable_timer__start(evlist__event_enable_timer(rec->evlist));
 	if (err)
 		goto out_child;
 
@@ -2768,7 +2780,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv)
 			 * record__mmap_read_all() didn't collect data from
 			 * overwritable ring buffer. Read again.
 			 */
-			if (rec->evlist->bkw_mmap_state == BKW_MMAP_RUNNING)
+			if (evlist__bkw_mmap_state(rec->evlist) == BKW_MMAP_RUNNING)
 				continue;
 			trigger_ready(&switch_output_trigger);
 
@@ -2837,7 +2849,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv)
 			}
 		}
 
-		err = event_enable_timer__process(rec->evlist->eet);
+		err = event_enable_timer__process(evlist__event_enable_timer(rec->evlist));
 		if (err < 0)
 			goto out_child;
 		if (err) {
@@ -2889,11 +2901,13 @@ static int __cmd_record(struct record *rec, int argc, const char **argv)
 		record__synthesize_workload(rec, true);
 
 out_child:
+	evlist__disable(rec->evlist);
 	record__stop_threads(rec);
 	record__mmap_read_all(rec, true);
 	goto out_free_threads;
 out_child_no_flush:
 	/* mmap read already failed — retrying would just fail again */
+	evlist__disable(rec->evlist);
 	record__stop_threads(rec);
 out_free_threads:
 	record__free_thread_data(rec);
@@ -2909,7 +2923,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv)
 		int exit_status;
 
 		if (!child_finished)
-			kill(rec->evlist->workload.pid, SIGTERM);
+			kill(evlist__workload_pid(rec->evlist), SIGTERM);
 
 		wait(&exit_status);
 
@@ -4032,7 +4046,7 @@ static int record__init_thread_default_masks(struct record *rec, struct perf_cpu
 static int record__init_thread_masks(struct record *rec)
 {
 	int ret = 0;
-	struct perf_cpu_map *cpus = rec->evlist->core.all_cpus;
+	struct perf_cpu_map *cpus = evlist__core(rec->evlist)->all_cpus;
 
 	if (!record__threads_enabled(rec))
 		return record__init_thread_default_masks(rec, cpus);
@@ -4283,15 +4297,15 @@ int cmd_record(int argc, const char **argv)
 	if (record.opts.overwrite)
 		record.opts.tail_synthesize = true;
 
-	if (rec->evlist->core.nr_entries == 0) {
+	if (evlist__nr_entries(rec->evlist) == 0) {
 		struct evlist *def_evlist = evlist__new_default(&rec->opts.target,
 								callchain_param.enabled);
 
 		if (!def_evlist)
 			goto out;
 
-		evlist__splice_list_tail(rec->evlist, &def_evlist->core.entries);
-		evlist__delete(def_evlist);
+		evlist__splice_list_tail(rec->evlist, &evlist__core(def_evlist)->entries);
+		evlist__put(def_evlist);
 	}
 
 	if (rec->opts.target.tid && !rec->opts.no_inherit_set)
@@ -4401,7 +4415,7 @@ int cmd_record(int argc, const char **argv)
 	auxtrace_record__free(rec->itr);
 out_opts:
 	evlist__close_control(rec->opts.ctl_fd, rec->opts.ctl_fd_ack, &rec->opts.ctl_fd_close);
-	evlist__delete(rec->evlist);
+	evlist__put(rec->evlist);
 	return err;
 }
 
diff --git a/tools/perf/builtin-report.c b/tools/perf/builtin-report.c
index dd1309c320943..10db1e5f1e6c4 100644
--- a/tools/perf/builtin-report.c
+++ b/tools/perf/builtin-report.c
@@ -561,7 +561,7 @@ static int evlist__tty_browse_hists(struct evlist *evlist, struct report *rep, c
 
 	if (!quiet) {
 		fprintf(stdout, "#\n# Total Lost Samples: %" PRIu64 "\n#\n",
-			evlist->stats.total_lost_samples);
+			evlist__stats(evlist)->total_lost_samples);
 	}
 
 	evlist__for_each_entry(evlist, pos) {
@@ -1156,7 +1156,7 @@ static int __cmd_report(struct report *rep)
 			PERF_HPP_REPORT__BLOCK_AVG_CYCLES,
 		};
 
-		if (session->evlist->nr_br_cntr > 0)
+		if (evlist__nr_br_cntr(session->evlist) > 0)
 			block_hpps[nr_hpps++] = PERF_HPP_REPORT__BLOCK_BRANCH_COUNTER;
 
 		block_hpps[nr_hpps++] = PERF_HPP_REPORT__BLOCK_RANGE;
@@ -1291,7 +1291,7 @@ static int process_attr(const struct perf_tool *tool __maybe_unused,
 	 * on events sample_type.
 	 */
 	sample_type = evlist__combined_sample_type(*pevlist);
-	session = (*pevlist)->session;
+	session = evlist__session(*pevlist);
 	callchain_param_setup(sample_type, perf_session__e_machine(session, /*e_flags=*/NULL));
 	return 0;
 }
diff --git a/tools/perf/builtin-sched.c b/tools/perf/builtin-sched.c
index 7fd63a9db4574..acc8d81a20d3b 100644
--- a/tools/perf/builtin-sched.c
+++ b/tools/perf/builtin-sched.c
@@ -2025,9 +2025,9 @@ static int perf_sched__read_events(struct perf_sched *sched)
 			goto out_delete;
 		}
 
-		sched->nr_events      = session->evlist->stats.nr_events[0];
-		sched->nr_lost_events = session->evlist->stats.total_lost;
-		sched->nr_lost_chunks = session->evlist->stats.nr_events[PERF_RECORD_LOST];
+		sched->nr_events      = evlist__stats(session->evlist)->nr_events[0];
+		sched->nr_lost_events = evlist__stats(session->evlist)->total_lost;
+		sched->nr_lost_chunks = evlist__stats(session->evlist)->nr_events[PERF_RECORD_LOST];
 	}
 
 	rc = 0;
@@ -2270,12 +2270,6 @@ static void timehist_print_sample(struct perf_sched *sched,
 		printf(" ");
 	}
 
-	if (!thread__comm_set(thread)) {
-		const char *prev_comm = perf_sample__strval(sample, "prev_comm");
-
-		thread__set_comm(thread, prev_comm, sample->time);
-        }
-
 	printf(" %-*s ", comm_width, timehist_get_commstr(thread));
 
 	if (sched->show_prio)
@@ -2981,6 +2975,16 @@ static int timehist_sched_change_event(const struct perf_tool *tool,
 			thread__zput(itr->last_thread);
 		}
 
+		/*
+		 * If the process name is not set for the thread, use "prev_comm"
+		 * to set it. Otherwise the sched summary will have just pid information
+		 */
+		if (!thread__comm_set(thread)) {
+			const char *prev_comm = perf_sample__strval(sample, "prev_comm");
+
+			thread__set_comm(thread, prev_comm, sample->time);
+		}
+
 		if (!sched->summary_only)
 			timehist_print_sample(sched, sample, &al, thread, t, state);
 	}
@@ -3303,7 +3307,7 @@ static int timehist_check_attr(struct perf_sched *sched,
 	struct evsel *evsel;
 	struct evsel_runtime *er;
 
-	list_for_each_entry(evsel, &evlist->core.entries, core.node) {
+	list_for_each_entry(evsel, &evlist__core(evlist)->entries, core.node) {
 		er = evsel__get_runtime(evsel);
 		if (er == NULL) {
 			pr_err("Failed to allocate memory for evsel runtime data\n");
@@ -3475,9 +3479,9 @@ static int perf_sched__timehist(struct perf_sched *sched)
 		goto out;
 	}
 
-	sched->nr_events      = evlist->stats.nr_events[0];
-	sched->nr_lost_events = evlist->stats.total_lost;
-	sched->nr_lost_chunks = evlist->stats.nr_events[PERF_RECORD_LOST];
+	sched->nr_events      = evlist__stats(evlist)->nr_events[0];
+	sched->nr_lost_events = evlist__stats(evlist)->total_lost;
+	sched->nr_lost_chunks = evlist__stats(evlist)->nr_events[PERF_RECORD_LOST];
 
 	if (sched->summary)
 		timehist_print_summary(sched, session);
@@ -3924,7 +3928,7 @@ static int perf_sched__schedstat_record(struct perf_sched *sched,
 	session = perf_session__new(&data, &sched->tool);
 	if (IS_ERR(session)) {
 		pr_err("Perf session creation failed.\n");
-		evlist__delete(evlist);
+		evlist__put(evlist);
 		return PTR_ERR(session);
 	}
 
@@ -3982,7 +3986,7 @@ static int perf_sched__schedstat_record(struct perf_sched *sched,
 	if (err < 0)
 		goto out;
 
-	user_requested_cpus = evlist->core.user_requested_cpus;
+	user_requested_cpus = evlist__core(evlist)->user_requested_cpus;
 
 	err = perf_event__synthesize_schedstat(&(sched->tool),
 					       process_synthesized_schedstat_event,
@@ -3998,7 +4002,7 @@ static int perf_sched__schedstat_record(struct perf_sched *sched,
 		evlist__start_workload(evlist);
 
 	while (!done) {
-		if (argc && waitpid(evlist->workload.pid, NULL, WNOHANG) > 0)
+		if (argc && waitpid(evlist__workload_pid(evlist), NULL, WNOHANG) > 0)
 			break;
 		sleep(1);
 	}
@@ -4023,8 +4027,8 @@ static int perf_sched__schedstat_record(struct perf_sched *sched,
 	else
 		fprintf(stderr, "[ perf sched stats: Failed !! ]\n");
 
-	evlist__delete(evlist);
-	close(fd);
+	perf_session__delete(session);
+	evlist__put(evlist);
 	return err;
 }
 
@@ -4627,6 +4631,8 @@ static int perf_sched__process_schedstat(const struct perf_tool *tool __maybe_un
 			domain_second_pass = list_first_entry(&cpu_second_pass->domain_head,
 							      struct schedstat_domain, domain_list);
 			store_schedstat_cpu_diff(temp);
+			free(temp->cpu_data);
+			free(temp);
 		}
 	} else if (event->header.type == PERF_RECORD_SCHEDSTAT_DOMAIN) {
 		struct schedstat_cpu *cpu_tail;
@@ -4647,6 +4653,8 @@ static int perf_sched__process_schedstat(const struct perf_tool *tool __maybe_un
 		} else {
 			store_schedstat_domain_diff(temp);
 			domain_second_pass = list_next_entry(domain_second_pass, domain_list);
+			free(temp->domain_data);
+			free(temp);
 		}
 	}
 
@@ -4699,7 +4707,7 @@ static int perf_sched__schedstat_report(struct perf_sched *sched)
 	if (err < 0)
 		goto out;
 
-	user_requested_cpus = session->evlist->core.user_requested_cpus;
+	user_requested_cpus = evlist__core(session->evlist)->user_requested_cpus;
 
 	err = perf_session__process_events(session);
 
@@ -4875,7 +4883,7 @@ static int perf_sched__schedstat_live(struct perf_sched *sched,
 	if (err < 0)
 		goto out;
 
-	user_requested_cpus = evlist->core.user_requested_cpus;
+	user_requested_cpus = evlist__core(evlist)->user_requested_cpus;
 
 	err = perf_event__synthesize_schedstat(&(sched->tool),
 					       process_synthesized_event_live,
@@ -4891,7 +4899,7 @@ static int perf_sched__schedstat_live(struct perf_sched *sched,
 		evlist__start_workload(evlist);
 
 	while (!done) {
-		if (argc && waitpid(evlist->workload.pid, NULL, WNOHANG) > 0)
+		if (argc && waitpid(evlist__workload_pid(evlist), NULL, WNOHANG) > 0)
 			break;
 		sleep(1);
 	}
@@ -4927,7 +4935,7 @@ static int perf_sched__schedstat_live(struct perf_sched *sched,
 	free_cpu_domain_info(cd_map, sv, nr);
 out:
 	free_schedstat(&cpu_head);
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	return err;
 }
 
@@ -5251,19 +5259,20 @@ int cmd_sched(int argc, const char **argv)
 			if (argc)
 				argc = parse_options(argc, argv, stats_options,
 						     stats_usage, 0);
-			return perf_sched__schedstat_record(&sched, argc, argv);
+			ret = perf_sched__schedstat_record(&sched, argc, argv);
 		} else if (argv[0] && !strcmp(argv[0], "report")) {
 			if (argc)
 				argc = parse_options(argc, argv, stats_options,
 						     stats_usage, 0);
-			return perf_sched__schedstat_report(&sched);
+			ret = perf_sched__schedstat_report(&sched);
 		} else if (argv[0] && !strcmp(argv[0], "diff")) {
 			if (argc)
 				argc = parse_options(argc, argv, stats_options,
 						     stats_usage, 0);
-			return perf_sched__schedstat_diff(&sched, argc, argv);
+			ret = perf_sched__schedstat_diff(&sched, argc, argv);
+		} else {
+			ret = perf_sched__schedstat_live(&sched, argc, argv);
 		}
-		return perf_sched__schedstat_live(&sched, argc, argv);
 	} else {
 		usage_with_options(sched_usage, sched_options);
 	}
diff --git a/tools/perf/builtin-script.c b/tools/perf/builtin-script.c
index 9ac29bdc3cd54..f91d8b1fbd011 100644
--- a/tools/perf/builtin-script.c
+++ b/tools/perf/builtin-script.c
@@ -2174,6 +2174,7 @@ static int script_find_metrics(const struct pmu_metric *pm,
 	struct evsel *metric_evsel;
 	int ret = metricgroup__parse_groups(metric_evlist,
 					/*pmu=*/"all",
+					/*cputype_filter=*/false,
 					pm->metric_name,
 					/*metric_no_group=*/false,
 					/*metric_no_merge=*/false,
@@ -2229,9 +2230,10 @@ static int script_find_metrics(const struct pmu_metric *pm,
 	evlist__for_each_entry(metric_evlist, metric_evsel) {
 		struct evsel *script_evsel =
 			map_metric_evsel_to_script_evsel(script_evlist, metric_evsel);
-		struct metric_event *metric_me = metricgroup__lookup(&metric_evlist->metric_events,
-								     metric_evsel,
-								     /*create=*/false);
+		struct metric_event *metric_me =
+			metricgroup__lookup(evlist__metric_events(metric_evlist),
+					    metric_evsel,
+					    /*create=*/false);
 
 		if (script_evsel->metric_id == NULL) {
 			script_evsel->metric_id = metric_evsel->metric_id;
@@ -2251,7 +2253,7 @@ static int script_find_metrics(const struct pmu_metric *pm,
 		if (metric_me) {
 			struct metric_expr *expr;
 			struct metric_event *script_me =
-				metricgroup__lookup(&script_evlist->metric_events,
+				metricgroup__lookup(evlist__metric_events(script_evlist),
 						    script_evsel,
 						    /*create=*/true);
 
@@ -2274,7 +2276,7 @@ static int script_find_metrics(const struct pmu_metric *pm,
 	}
 	pr_debug("Found metric '%s' whose evsels match those of in the perf data\n",
 		 pm->metric_name);
-	evlist__delete(metric_evlist);
+	evlist__put(metric_evlist);
 out:
 	return 0;
 }
@@ -2321,7 +2323,7 @@ static void perf_sample__fprint_metric(struct thread *thread,
 			assert(stat_config.aggr_mode == AGGR_GLOBAL);
 			stat_config.aggr_get_id = script_aggr_cpu_id_get;
 			stat_config.aggr_map =
-				cpu_aggr_map__new(evsel->evlist->core.user_requested_cpus,
+				cpu_aggr_map__new(evlist__core(evsel->evlist)->user_requested_cpus,
 						  aggr_cpu_id__global, /*data=*/NULL,
 						  /*needs_sort=*/false);
 		}
@@ -3909,7 +3911,7 @@ static int set_maps(struct perf_script *script)
 	if (WARN_ONCE(script->allocated, "stats double allocation\n"))
 		return -EINVAL;
 
-	perf_evlist__set_maps(&evlist->core, script->cpus, script->threads);
+	perf_evlist__set_maps(evlist__core(evlist), script->cpus, script->threads);
 
 	if (evlist__alloc_stats(&stat_config, evlist, /*alloc_raw=*/true))
 		return -ENOMEM;
diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c
index a04466ea3b0a0..a85a4f30221a8 100644
--- a/tools/perf/builtin-stat.c
+++ b/tools/perf/builtin-stat.c
@@ -321,7 +321,7 @@ static int read_single_counter(struct evsel *counter, int cpu_map_idx, int threa
  */
 static int read_counter_cpu(struct evsel *counter, int cpu_map_idx)
 {
-	int nthreads = perf_thread_map__nr(evsel_list->core.threads);
+	int nthreads = perf_thread_map__nr(evlist__core(evsel_list)->threads);
 	int thread;
 
 	if (!counter->supported)
@@ -628,11 +628,12 @@ static int dispatch_events(bool forks, int timeout, int interval, int *times)
 	time_to_sleep = sleep_time;
 
 	while (!done) {
-		if (forks)
+		if (forks) {
 			child_exited = waitpid(child_pid, &status, WNOHANG);
-		else
-			child_exited = !is_target_alive(&target, evsel_list->core.threads) ? 1 : 0;
-
+		} else {
+			child_exited = !is_target_alive(&target,
+							evlist__core(evsel_list)->threads) ? 1 : 0;
+		}
 		if (child_exited)
 			break;
 
@@ -681,14 +682,15 @@ static enum counter_recovery stat_handle_error(struct evsel *counter, int err)
 		return COUNTER_RETRY;
 	}
 	if (target__has_per_thread(&target) && err != EOPNOTSUPP &&
-	    evsel_list->core.threads && evsel_list->core.threads->err_thread != -1) {
+	    evlist__core(evsel_list)->threads &&
+	    evlist__core(evsel_list)->threads->err_thread != -1) {
 		/*
 		 * For global --per-thread case, skip current
 		 * error thread.
 		 */
-		if (!thread_map__remove(evsel_list->core.threads,
-					evsel_list->core.threads->err_thread)) {
-			evsel_list->core.threads->err_thread = -1;
+		if (!thread_map__remove(evlist__core(evsel_list)->threads,
+					evlist__core(evsel_list)->threads->err_thread)) {
+			evlist__core(evsel_list)->threads->err_thread = -1;
 			counter->supported = true;
 			return COUNTER_RETRY;
 		}
@@ -787,11 +789,12 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx)
 	bool second_pass = false, has_supported_counters;
 
 	if (forks) {
-		if (evlist__prepare_workload(evsel_list, &target, argv, is_pipe, workload_exec_failed_signal) < 0) {
+		if (evlist__prepare_workload(evsel_list, &target, argv, is_pipe,
+					     workload_exec_failed_signal) < 0) {
 			perror("failed to prepare workload");
 			return -1;
 		}
-		child_pid = evsel_list->workload.pid;
+		child_pid = evlist__workload_pid(evsel_list);
 	}
 
 	evlist__for_each_entry(evsel_list, counter) {
@@ -1199,7 +1202,7 @@ static int parse_cputype(const struct option *opt,
 	const struct perf_pmu *pmu;
 	struct evlist *evlist = *(struct evlist **)opt->value;
 
-	if (!list_empty(&evlist->core.entries)) {
+	if (!list_empty(&evlist__core(evlist)->entries)) {
 		fprintf(stderr, "Must define cputype before events/metrics\n");
 		return -1;
 	}
@@ -1210,6 +1213,7 @@ static int parse_cputype(const struct option *opt,
 		return -1;
 	}
 	parse_events_option_args.pmu_filter = pmu->name;
+	parse_events_option_args.cputype_filter = true;
 
 	return 0;
 }
@@ -1220,12 +1224,13 @@ static int parse_pmu_filter(const struct option *opt,
 {
 	struct evlist *evlist = *(struct evlist **)opt->value;
 
-	if (!list_empty(&evlist->core.entries)) {
+	if (!list_empty(&evlist__core(evlist)->entries)) {
 		fprintf(stderr, "Must define pmu-filter before events/metrics\n");
 		return -1;
 	}
 
 	parse_events_option_args.pmu_filter = str;
+	parse_events_option_args.cputype_filter = false;
 	return 0;
 }
 
@@ -1586,8 +1591,9 @@ static int perf_stat_init_aggr_mode(void)
 
 	if (get_id) {
 		bool needs_sort = stat_config.aggr_mode != AGGR_NONE;
-		stat_config.aggr_map = cpu_aggr_map__new(evsel_list->core.user_requested_cpus,
-							 get_id, /*data=*/NULL, needs_sort);
+		stat_config.aggr_map = cpu_aggr_map__new(
+			evlist__core(evsel_list)->user_requested_cpus,
+			get_id, /*data=*/NULL, needs_sort);
 		if (!stat_config.aggr_map) {
 			pr_err("cannot build %s map\n", aggr_mode__string[stat_config.aggr_mode]);
 			return -1;
@@ -1596,7 +1602,7 @@ static int perf_stat_init_aggr_mode(void)
 	}
 
 	if (stat_config.aggr_mode == AGGR_THREAD) {
-		nr = perf_thread_map__nr(evsel_list->core.threads);
+		nr = perf_thread_map__nr(evlist__core(evsel_list)->threads);
 		stat_config.aggr_map = cpu_aggr_map__empty_new(nr);
 		if (stat_config.aggr_map == NULL)
 			return -ENOMEM;
@@ -1615,7 +1621,7 @@ static int perf_stat_init_aggr_mode(void)
 	 * taking the highest cpu number to be the size of
 	 * the aggregation translate cpumap.
 	 */
-	nr = perf_cpu_map__max(evsel_list->core.all_cpus).cpu + 1;
+	nr = perf_cpu_map__max(evlist__core(evsel_list)->all_cpus).cpu + 1;
 	stat_config.cpus_aggr_map = cpu_aggr_map__empty_new(nr);
 	return stat_config.cpus_aggr_map ? 0 : -ENOMEM;
 }
@@ -1902,7 +1908,7 @@ static int perf_stat_init_aggr_mode_file(struct perf_stat *st)
 	bool needs_sort = stat_config.aggr_mode != AGGR_NONE;
 
 	if (stat_config.aggr_mode == AGGR_THREAD) {
-		int nr = perf_thread_map__nr(evsel_list->core.threads);
+		int nr = perf_thread_map__nr(evlist__core(evsel_list)->threads);
 
 		stat_config.aggr_map = cpu_aggr_map__empty_new(nr);
 		if (stat_config.aggr_map == NULL)
@@ -1920,7 +1926,7 @@ static int perf_stat_init_aggr_mode_file(struct perf_stat *st)
 	if (!get_id)
 		return 0;
 
-	stat_config.aggr_map = cpu_aggr_map__new(evsel_list->core.user_requested_cpus,
+	stat_config.aggr_map = cpu_aggr_map__new(evlist__core(evsel_list)->user_requested_cpus,
 						 get_id, env, needs_sort);
 	if (!stat_config.aggr_map) {
 		pr_err("cannot build %s map\n", aggr_mode__string[stat_config.aggr_mode]);
@@ -1999,7 +2005,9 @@ static int add_default_events(void)
 			ret = -1;
 			goto out;
 		}
-		ret = metricgroup__parse_groups(evlist, pmu, "transaction",
+		ret = metricgroup__parse_groups(evlist, pmu,
+						parse_events_option_args.cputype_filter,
+						"transaction",
 						stat_config.metric_no_group,
 						stat_config.metric_no_merge,
 						stat_config.metric_no_threshold,
@@ -2036,7 +2044,9 @@ static int add_default_events(void)
 		if (!force_metric_only)
 			stat_config.metric_only = true;
 
-		ret = metricgroup__parse_groups(evlist, pmu, "smi",
+		ret = metricgroup__parse_groups(evlist, pmu,
+						parse_events_option_args.cputype_filter,
+						"smi",
 						stat_config.metric_no_group,
 						stat_config.metric_no_merge,
 						stat_config.metric_no_threshold,
@@ -2073,7 +2083,7 @@ static int add_default_events(void)
 		}
 		str[8] = stat_config.topdown_level + '0';
 		if (metricgroup__parse_groups(evlist,
-						pmu, str,
+					      pmu, parse_events_option_args.cputype_filter, str,
 						/*metric_no_group=*/false,
 						/*metric_no_merge=*/false,
 						/*metric_no_threshold=*/true,
@@ -2088,7 +2098,7 @@ static int add_default_events(void)
 	if (!stat_config.topdown_level)
 		stat_config.topdown_level = 1;
 
-	if (!evlist->core.nr_entries && !evsel_list->core.nr_entries) {
+	if (!evlist__nr_entries(evlist) && !evlist__nr_entries(evsel_list)) {
 		/*
 		 * Add Default metrics. To minimize multiplexing, don't request
 		 * threshold computation, but it will be computed if the events
@@ -2112,14 +2122,16 @@ static int add_default_events(void)
 				ret = -ENOMEM;
 				break;
 			}
-			if (metricgroup__parse_groups(metric_evlist, pmu, default_metricgroup_names[i],
+			if (metricgroup__parse_groups(metric_evlist, pmu,
+						      parse_events_option_args.cputype_filter,
+						      default_metricgroup_names[i],
 							/*metric_no_group=*/false,
 							/*metric_no_merge=*/false,
 							/*metric_no_threshold=*/true,
 							stat_config.user_requested_cpu_list,
 							stat_config.system_wide,
 							stat_config.hardware_aware_grouping) < 0) {
-				evlist__delete(metric_evlist);
+				evlist__put(metric_evlist);
 				ret = -1;
 				break;
 			}
@@ -2127,13 +2139,13 @@ static int add_default_events(void)
 			evlist__for_each_entry(metric_evlist, evsel)
 				evsel->default_metricgroup = true;
 
-			evlist__splice_list_tail(evlist, &metric_evlist->core.entries);
+			evlist__splice_list_tail(evlist, &evlist__core(metric_evlist)->entries);
 			metricgroup__copy_metric_events(evlist, /*cgrp=*/NULL,
-							&evlist->metric_events,
-							&metric_evlist->metric_events);
-			evlist__delete(metric_evlist);
+							evlist__metric_events(evlist),
+							evlist__metric_events(metric_evlist));
+			evlist__put(metric_evlist);
 		}
-		list_sort(/*priv=*/NULL, &evlist->core.entries, default_evlist_evsel_cmp);
+		list_sort(/*priv=*/NULL, &evlist__core(evlist)->entries, default_evlist_evsel_cmp);
 
 	}
 out:
@@ -2148,11 +2160,11 @@ static int add_default_events(void)
 		}
 	}
 	parse_events_error__exit(&err);
-	evlist__splice_list_tail(evsel_list, &evlist->core.entries);
+	evlist__splice_list_tail(evsel_list, &evlist__core(evlist)->entries);
 	metricgroup__copy_metric_events(evsel_list, /*cgrp=*/NULL,
-					&evsel_list->metric_events,
-					&evlist->metric_events);
-	evlist__delete(evlist);
+					evlist__metric_events(evsel_list),
+					evlist__metric_events(evlist));
+	evlist__put(evlist);
 	return ret;
 }
 
@@ -2272,7 +2284,7 @@ static int set_maps(struct perf_stat *st)
 	if (WARN_ONCE(st->maps_allocated, "stats double allocation\n"))
 		return -EINVAL;
 
-	perf_evlist__set_maps(&evsel_list->core, st->cpus, st->threads);
+	perf_evlist__set_maps(evlist__core(evsel_list), st->cpus, st->threads);
 
 	if (evlist__alloc_stats(&stat_config, evsel_list, /*alloc_raw=*/true))
 		return -ENOMEM;
@@ -2387,7 +2399,7 @@ static int __cmd_report(int argc, const char **argv)
 
 	perf_stat.session  = session;
 	stat_config.output = stderr;
-	evlist__delete(evsel_list);
+	evlist__put(evsel_list);
 	evsel_list         = session->evlist;
 
 	ret = perf_session__process_events(session);
@@ -2424,7 +2436,7 @@ static void setup_system_wide(int forks)
 			}
 		}
 
-		if (evsel_list->core.nr_entries)
+		if (evlist__nr_entries(evsel_list))
 			target.system_wide = true;
 	}
 }
@@ -2643,6 +2655,13 @@ int cmd_stat(int argc, const char **argv)
 
 	stat_config.aggr_mode = opt_aggr_mode_to_aggr_mode(&opt_mode);
 
+	if (stat_config.csv_sep && stat_config.json_output) {
+		fprintf(stderr, "cannot use both --field-separator and --json-output\n");
+		parse_options_usage(stat_usage, stat_options, "x", 1);
+		parse_options_usage(NULL, stat_options, "j", 1);
+		goto out;
+	}
+
 	if (stat_config.csv_sep) {
 		stat_config.csv_output = true;
 		if (!strcmp(stat_config.csv_sep, "\\t"))
@@ -2651,7 +2670,7 @@ int cmd_stat(int argc, const char **argv)
 		stat_config.csv_sep = DEFAULT_SEPARATOR;
 
 	if (affinity_set)
-		evsel_list->no_affinity = !affinity;
+		evlist__set_no_affinity(evsel_list, !affinity);
 
 	if (argc && strlen(argv[0]) > 2 && strstarts("record", argv[0])) {
 		argc = __cmd_record(stat_options, &opt_mode, argc, argv);
@@ -2818,7 +2837,7 @@ int cmd_stat(int argc, const char **argv)
 	}
 
 	if (stat_config.iostat_run) {
-		status = iostat_prepare(evsel_list, &stat_config);
+		status = iostat_prepare(&evsel_list, &stat_config);
 		if (status)
 			goto out;
 		if (iostat_mode == IOSTAT_LIST) {
@@ -2848,7 +2867,9 @@ int cmd_stat(int argc, const char **argv)
 	 */
 	if (metrics) {
 		const char *pmu = parse_events_option_args.pmu_filter ?: "all";
-		int ret = metricgroup__parse_groups(evsel_list, pmu, metrics,
+		int ret = metricgroup__parse_groups(evsel_list, pmu,
+						    parse_events_option_args.cputype_filter,
+						    metrics,
 						stat_config.metric_no_group,
 						stat_config.metric_no_merge,
 						stat_config.metric_no_threshold,
@@ -2874,7 +2895,8 @@ int cmd_stat(int argc, const char **argv)
 			goto out;
 		}
 
-		if (evlist__expand_cgroup(evsel_list, stat_config.cgroup_list, true) < 0) {
+		if (evlist__expand_cgroup(evsel_list, stat_config.cgroup_list,
+					  !target.use_bpf) < 0) {
 			parse_options_usage(stat_usage, stat_options,
 					    "for-each-cgroup", 0);
 			goto out;
@@ -2882,9 +2904,10 @@ int cmd_stat(int argc, const char **argv)
 	}
 #ifdef HAVE_BPF_SKEL
 	if (target.use_bpf && nr_cgroups &&
-	    (evsel_list->core.nr_entries / nr_cgroups) > BPERF_CGROUP__MAX_EVENTS) {
+	    (evlist__nr_entries(evsel_list) / nr_cgroups) > BPERF_CGROUP__MAX_EVENTS) {
 		pr_warning("Disabling BPF counters due to more events (%d) than the max (%d)\n",
-			   evsel_list->core.nr_entries / nr_cgroups, BPERF_CGROUP__MAX_EVENTS);
+			   evlist__nr_entries(evsel_list) / nr_cgroups,
+			   BPERF_CGROUP__MAX_EVENTS);
 		target.use_bpf = false;
 	}
 #endif // HAVE_BPF_SKEL
@@ -2922,7 +2945,7 @@ int cmd_stat(int argc, const char **argv)
 	 * so we could print it out on output.
 	 */
 	if (stat_config.aggr_mode == AGGR_THREAD) {
-		thread_map__read_comms(evsel_list->core.threads);
+		thread_map__read_comms(evlist__core(evsel_list)->threads);
 	}
 
 	if (stat_config.aggr_mode == AGGR_NODE)
@@ -3066,7 +3089,7 @@ int cmd_stat(int argc, const char **argv)
 	if (smi_cost && smi_reset)
 		sysfs__write_int(FREEZE_ON_SMI_PATH, 0);
 
-	evlist__delete(evsel_list);
+	evlist__put(evsel_list);
 
 	evlist__close_control(stat_config.ctl_fd, stat_config.ctl_fd_ack, &stat_config.ctl_fd_close);
 
diff --git a/tools/perf/builtin-timechart.c b/tools/perf/builtin-timechart.c
index 85a9ad0455aec..3f9153d5ecfb7 100644
--- a/tools/perf/builtin-timechart.c
+++ b/tools/perf/builtin-timechart.c
@@ -56,6 +56,7 @@ struct timechart {
 	struct per_pid		*all_data;
 	struct power_event	*power_events;
 	struct wake_event	*wake_events;
+	struct perf_session	*session;
 	int			proc_num;
 	unsigned int		numcpus;
 	u64			min_freq,	/* Lowest CPU frequency seen */
@@ -299,7 +300,7 @@ static void pid_put_sample(struct timechart *tchart, int pid, int type,
 	sample->type = type;
 	sample->next = c->samples;
 	sample->cpu = cpu;
-	sample->backtrace = backtrace ? strdup(backtrace) : NULL;
+	sample->backtrace = backtrace;
 	c->samples = sample;
 
 	if (sample->type == TYPE_RUNNING && end > start && start > 0) {
@@ -428,12 +429,14 @@ static void sched_wakeup(struct timechart *tchart, int cpu, u64 timestamp,
 	struct per_pid *p;
 	struct wake_event *we = zalloc(sizeof(*we));
 
-	if (!we)
+	if (!we) {
+		free((char *)backtrace);
 		return;
+	}
 
 	we->time = timestamp;
 	we->waker = waker;
-	we->backtrace = backtrace ? strdup(backtrace) : NULL;
+	we->backtrace = backtrace;
 
 	if ((flags & TRACE_FLAG_HARDIRQ) || (flags & TRACE_FLAG_SOFTIRQ))
 		we->waker = -1;
@@ -460,20 +463,28 @@ static void sched_switch(struct timechart *tchart, int cpu, u64 timestamp,
 			 const char *backtrace)
 {
 	struct per_pid *p = NULL, *prev_p;
+	bool backtrace_used = false;
 
 	prev_p = find_create_pid(tchart, prev_pid);
 
 	p = find_create_pid(tchart, next_pid);
 
-	if (prev_p->current && prev_p->current->state != TYPE_NONE)
+	if (prev_p->current && prev_p->current->state != TYPE_NONE) {
 		pid_put_sample(tchart, prev_pid, TYPE_RUNNING, cpu,
 			       prev_p->current->state_since, timestamp,
 			       backtrace);
+		backtrace_used = true;
+	}
 	if (p && p->current) {
-		if (p->current->state != TYPE_NONE)
+		if (p->current->state != TYPE_NONE) {
+			if (backtrace && backtrace_used)
+				backtrace = strdup(backtrace);
+
 			pid_put_sample(tchart, next_pid, p->current->state, cpu,
 				       p->current->state_since, timestamp,
 				       backtrace);
+			backtrace_used = true;
+		}
 
 		p->current->state_since = timestamp;
 		p->current->state = TYPE_RUNNING;
@@ -487,14 +498,16 @@ static void sched_switch(struct timechart *tchart, int cpu, u64 timestamp,
 		if (prev_state == 0)
 			prev_p->current->state = TYPE_WAITING;
 	}
+
+	if (!backtrace_used)
+		free((char *)backtrace);
 }
 
 /*
  * Returns a malloc'd backtrace string built via open_memstream, or NULL
  * on error.  Caller must free() the returned pointer.
  */
-static char *cat_backtrace(union perf_event *event,
-			   struct perf_sample *sample,
+static char *cat_backtrace(struct perf_sample *sample,
 			   struct machine *machine)
 {
 	struct addr_location al;
@@ -516,9 +529,8 @@ static char *cat_backtrace(union perf_event *event,
 		goto exit;
 
 	if (machine__resolve(machine, &al, sample) < 0) {
-		pr_err("problem processing %s (%u) event at offset %#" PRIx64 ", skipping it.\n",
-		       perf_event__name(event->header.type), event->header.type,
-		       sample->file_offset);
+		pr_err("problem processing SAMPLE (%u) event at offset %#" PRIx64 ", skipping it.\n",
+		       PERF_RECORD_SAMPLE, sample->file_offset);
 		goto exit;
 	}
 
@@ -574,13 +586,12 @@ static char *cat_backtrace(union perf_event *event,
 }
 
 typedef int (*tracepoint_handler)(struct timechart *tchart,
-				  struct perf_sample *sample,
-				  const char *backtrace);
+				  struct perf_sample *sample);
 
 static int process_sample_event(const struct perf_tool *tool,
-				union perf_event *event,
+				union perf_event *event __maybe_unused,
 				struct perf_sample *sample,
-				struct machine *machine)
+				struct machine *machine __maybe_unused)
 {
 	struct timechart *tchart = container_of(tool, struct timechart, tool);
 	struct evsel *evsel = sample->evsel;
@@ -595,10 +606,8 @@ static int process_sample_event(const struct perf_tool *tool,
 
 	if (evsel->handler != NULL) {
 		tracepoint_handler f = evsel->handler;
-		char *backtrace = cat_backtrace(event, sample, machine);
 
-		ret = f(tchart, sample, backtrace);
-		free(backtrace);
+		ret = f(tchart, sample);
 	}
 
 	return ret;
@@ -606,8 +615,7 @@ static int process_sample_event(const struct perf_tool *tool,
 
 static int
 process_sample_cpu_idle(struct timechart *tchart __maybe_unused,
-			struct perf_sample *sample,
-			const char *backtrace __maybe_unused)
+			struct perf_sample *sample)
 {
 	u32 state  = perf_sample__intval(sample, "state");
 	u32 cpu_id = perf_sample__intval(sample, "cpu_id");
@@ -627,8 +635,7 @@ process_sample_cpu_idle(struct timechart *tchart __maybe_unused,
 
 static int
 process_sample_cpu_frequency(struct timechart *tchart,
-			     struct perf_sample *sample,
-			     const char *backtrace __maybe_unused)
+			     struct perf_sample *sample)
 {
 	u32 state  = perf_sample__intval(sample, "state");
 	u32 cpu_id = perf_sample__intval(sample, "cpu_id");
@@ -645,12 +652,12 @@ process_sample_cpu_frequency(struct timechart *tchart,
 
 static int
 process_sample_sched_wakeup(struct timechart *tchart,
-			    struct perf_sample *sample,
-			    const char *backtrace)
+			    struct perf_sample *sample)
 {
 	u8 flags  = perf_sample__intval(sample, "common_flags");
 	int waker = perf_sample__intval(sample, "common_pid");
 	int wakee = perf_sample__intval(sample, "pid");
+	char *backtrace;
 
 	/* perf.data is untrusted input — CPU may be absent or corrupted */
 	if (sample->cpu >= MAX_CPUS) {
@@ -658,18 +665,20 @@ process_sample_sched_wakeup(struct timechart *tchart,
 			 sample->file_offset, sample->cpu);
 		return -1;
 	}
+
+	backtrace = cat_backtrace(sample, &tchart->session->machines.host);
 	sched_wakeup(tchart, sample->cpu, sample->time, waker, wakee, flags, backtrace);
 	return 0;
 }
 
 static int
 process_sample_sched_switch(struct timechart *tchart,
-			    struct perf_sample *sample,
-			    const char *backtrace)
+			    struct perf_sample *sample)
 {
 	int prev_pid   = perf_sample__intval(sample, "prev_pid");
 	int next_pid   = perf_sample__intval(sample, "next_pid");
 	u64 prev_state = perf_sample__intval(sample, "prev_state");
+	char *backtrace;
 
 	/* perf.data is untrusted input — CPU may be absent or corrupted */
 	if (sample->cpu >= MAX_CPUS) {
@@ -677,6 +686,8 @@ process_sample_sched_switch(struct timechart *tchart,
 			 sample->file_offset, sample->cpu);
 		return -1;
 	}
+
+	backtrace = cat_backtrace(sample, &tchart->session->machines.host);
 	sched_switch(tchart, sample->cpu, sample->time, prev_pid, next_pid,
 		     prev_state, backtrace);
 	return 0;
@@ -685,8 +696,7 @@ process_sample_sched_switch(struct timechart *tchart,
 #ifdef SUPPORT_OLD_POWER_EVENTS
 static int
 process_sample_power_start(struct timechart *tchart __maybe_unused,
-			   struct perf_sample *sample,
-			   const char *backtrace __maybe_unused)
+			   struct perf_sample *sample)
 {
 	u64 cpu_id = perf_sample__intval(sample, "cpu_id");
 	u64 value  = perf_sample__intval(sample, "value");
@@ -703,8 +713,7 @@ process_sample_power_start(struct timechart *tchart __maybe_unused,
 
 static int
 process_sample_power_end(struct timechart *tchart,
-			 struct perf_sample *sample,
-			 const char *backtrace __maybe_unused)
+			 struct perf_sample *sample)
 {
 	/* perf.data is untrusted input — CPU may be absent or corrupted */
 	if (sample->cpu >= MAX_CPUS) {
@@ -718,8 +727,7 @@ process_sample_power_end(struct timechart *tchart,
 
 static int
 process_sample_power_frequency(struct timechart *tchart,
-			       struct perf_sample *sample,
-			       const char *backtrace __maybe_unused)
+			       struct perf_sample *sample)
 {
 	u64 cpu_id = perf_sample__intval(sample, "cpu_id");
 	u64 value  = perf_sample__intval(sample, "value");
@@ -892,8 +900,7 @@ static int pid_end_io_sample(struct timechart *tchart, int pid, int type,
 
 static int
 process_enter_read(struct timechart *tchart,
-		   struct perf_sample *sample,
-		   const char *backtrace __maybe_unused)
+		   struct perf_sample *sample)
 {
 	long fd = perf_sample__intval(sample, "fd");
 	return pid_begin_io_sample(tchart, sample->tid, IOTYPE_READ,
@@ -902,8 +909,7 @@ process_enter_read(struct timechart *tchart,
 
 static int
 process_exit_read(struct timechart *tchart,
-		  struct perf_sample *sample,
-		  const char *backtrace __maybe_unused)
+		  struct perf_sample *sample)
 {
 	long ret = perf_sample__intval(sample, "ret");
 	return pid_end_io_sample(tchart, sample->tid, IOTYPE_READ,
@@ -912,8 +918,7 @@ process_exit_read(struct timechart *tchart,
 
 static int
 process_enter_write(struct timechart *tchart,
-		    struct perf_sample *sample,
-		    const char *backtrace __maybe_unused)
+		    struct perf_sample *sample)
 {
 	long fd = perf_sample__intval(sample, "fd");
 	return pid_begin_io_sample(tchart, sample->tid, IOTYPE_WRITE,
@@ -922,8 +927,7 @@ process_enter_write(struct timechart *tchart,
 
 static int
 process_exit_write(struct timechart *tchart,
-		   struct perf_sample *sample,
-		   const char *backtrace __maybe_unused)
+		   struct perf_sample *sample)
 {
 	long ret = perf_sample__intval(sample, "ret");
 	return pid_end_io_sample(tchart, sample->tid, IOTYPE_WRITE,
@@ -932,8 +936,7 @@ process_exit_write(struct timechart *tchart,
 
 static int
 process_enter_sync(struct timechart *tchart,
-		   struct perf_sample *sample,
-		   const char *backtrace __maybe_unused)
+		   struct perf_sample *sample)
 {
 	long fd = perf_sample__intval(sample, "fd");
 	return pid_begin_io_sample(tchart, sample->tid, IOTYPE_SYNC,
@@ -942,8 +945,7 @@ process_enter_sync(struct timechart *tchart,
 
 static int
 process_exit_sync(struct timechart *tchart,
-		  struct perf_sample *sample,
-		  const char *backtrace __maybe_unused)
+		  struct perf_sample *sample)
 {
 	long ret = perf_sample__intval(sample, "ret");
 	return pid_end_io_sample(tchart, sample->tid, IOTYPE_SYNC,
@@ -952,8 +954,7 @@ process_exit_sync(struct timechart *tchart,
 
 static int
 process_enter_tx(struct timechart *tchart,
-		 struct perf_sample *sample,
-		 const char *backtrace __maybe_unused)
+		 struct perf_sample *sample)
 {
 	long fd = perf_sample__intval(sample, "fd");
 	return pid_begin_io_sample(tchart, sample->tid, IOTYPE_TX,
@@ -962,8 +963,7 @@ process_enter_tx(struct timechart *tchart,
 
 static int
 process_exit_tx(struct timechart *tchart,
-		struct perf_sample *sample,
-		const char *backtrace __maybe_unused)
+		struct perf_sample *sample)
 {
 	long ret = perf_sample__intval(sample, "ret");
 	return pid_end_io_sample(tchart, sample->tid, IOTYPE_TX,
@@ -972,8 +972,7 @@ process_exit_tx(struct timechart *tchart,
 
 static int
 process_enter_rx(struct timechart *tchart,
-		 struct perf_sample *sample,
-		 const char *backtrace __maybe_unused)
+		 struct perf_sample *sample)
 {
 	long fd = perf_sample__intval(sample, "fd");
 	return pid_begin_io_sample(tchart, sample->tid, IOTYPE_RX,
@@ -982,8 +981,7 @@ process_enter_rx(struct timechart *tchart,
 
 static int
 process_exit_rx(struct timechart *tchart,
-		struct perf_sample *sample,
-		const char *backtrace __maybe_unused)
+		struct perf_sample *sample)
 {
 	long ret = perf_sample__intval(sample, "ret");
 	return pid_end_io_sample(tchart, sample->tid, IOTYPE_RX,
@@ -992,8 +990,7 @@ process_exit_rx(struct timechart *tchart,
 
 static int
 process_enter_poll(struct timechart *tchart,
-		   struct perf_sample *sample,
-		   const char *backtrace __maybe_unused)
+		   struct perf_sample *sample)
 {
 	long fd = perf_sample__intval(sample, "fd");
 	return pid_begin_io_sample(tchart, sample->tid, IOTYPE_POLL,
@@ -1002,8 +999,7 @@ process_enter_poll(struct timechart *tchart,
 
 static int
 process_exit_poll(struct timechart *tchart,
-		  struct perf_sample *sample,
-		  const char *backtrace __maybe_unused)
+		  struct perf_sample *sample)
 {
 	long ret = perf_sample__intval(sample, "ret");
 	return pid_end_io_sample(tchart, sample->tid, IOTYPE_POLL,
@@ -1114,12 +1110,12 @@ static void draw_wakeups(struct timechart *tchart)
 					c = c->next;
 				}
 				c = p->all;
-				while (c) {
-					if (p->pid == we->waker && !from) {
+				while (c && (!from || !to)) {
+					if (c->Y && p->pid == we->waker && !from) {
 						from = c->Y;
 						task_from = strdup(c->comm);
 					}
-					if (p->pid == we->wakee && !to) {
+					if (c->Y && p->pid == we->wakee && !to) {
 						to = c->Y;
 						task_to = strdup(c->comm);
 					}
@@ -1552,6 +1548,73 @@ static void write_svg_file(struct timechart *tchart, const char *filename)
 	svg_close();
 }
 
+static void timechart__release(struct timechart *tchart)
+{
+	struct per_pid *p = tchart->all_data;
+	struct power_event *pwr = tchart->power_events;
+	struct wake_event *we = tchart->wake_events;
+
+	while (p) {
+		struct per_pid *next_pid = p->next;
+		struct per_pidcomm *c = p->all;
+
+		while (c) {
+			struct per_pidcomm *next_comm = c->next;
+			struct cpu_sample *cs = c->samples;
+			struct io_sample *ios = c->io_samples;
+
+			while (cs) {
+				struct cpu_sample *next = cs->next;
+
+				zfree(&cs->backtrace);
+				cs->next = NULL;
+				free(cs);
+
+				cs = next;
+			}
+
+			while (ios) {
+				struct io_sample *next = ios->next;
+
+				ios->next = NULL;
+				free(ios);
+
+				ios = next;
+			}
+
+			zfree(&c->comm);
+			c->next = NULL;
+			free(c);
+
+			c = next_comm;
+		}
+
+		p->next = NULL;
+		free(p);
+
+		p = next_pid;
+	}
+
+	while (pwr) {
+		struct power_event *next = pwr->next;
+
+		pwr->next = NULL;
+		free(pwr);
+
+		pwr = next;
+	}
+
+	while (we) {
+		struct wake_event *next = we->next;
+
+		zfree(&we->backtrace);
+		we->next = NULL;
+		free(we);
+
+		we = next;
+	}
+}
+
 static int process_header(struct perf_file_section *section __maybe_unused,
 			  struct perf_header *ph,
 			  int feat,
@@ -1663,6 +1726,7 @@ static int __cmd_timechart(struct timechart *tchart, const char *output_name)
 	if (IS_ERR(session))
 		return PTR_ERR(session);
 
+	tchart->session = session;
 	symbol__init(perf_session__env(session));
 
 	(void)perf_header__process_sections(&session->header,
@@ -1770,6 +1834,7 @@ static int timechart__io_record(int argc, const char **argv, const char *output_
 		"syscalls:sys_exit_select",
 	};
 	unsigned int poll_events_nr = ARRAY_SIZE(poll_events);
+	int ret;
 
 	rec_argc = common_args_nr +
 		disk_events_nr * 4 +
@@ -1788,7 +1853,7 @@ static int timechart__io_record(int argc, const char **argv, const char *output_
 
 	p = rec_argv;
 	for (i = 0; i < common_args_nr; i++)
-		*p++ = strdup(common_args[i]);
+		*p++ = common_args[i];
 
 	for (i = 0; i < disk_events_nr; i++) {
 		if (!is_valid_tracepoint(disk_events[i])) {
@@ -1797,7 +1862,7 @@ static int timechart__io_record(int argc, const char **argv, const char *output_
 		}
 
 		*p++ = "-e";
-		*p++ = strdup(disk_events[i]);
+		*p++ = disk_events[i];
 		*p++ = "--filter";
 		*p++ = filter;
 	}
@@ -1808,7 +1873,7 @@ static int timechart__io_record(int argc, const char **argv, const char *output_
 		}
 
 		*p++ = "-e";
-		*p++ = strdup(net_events[i]);
+		*p++ = net_events[i];
 		*p++ = "--filter";
 		*p++ = filter;
 	}
@@ -1819,7 +1884,7 @@ static int timechart__io_record(int argc, const char **argv, const char *output_
 		}
 
 		*p++ = "-e";
-		*p++ = strdup(poll_events[i]);
+		*p++ = poll_events[i];
 		*p++ = "--filter";
 		*p++ = filter;
 	}
@@ -1827,7 +1892,11 @@ static int timechart__io_record(int argc, const char **argv, const char *output_
 	for (i = 0; i < (unsigned int)argc; i++)
 		*p++ = argv[i];
 
-	return cmd_record(rec_argc, rec_argv);
+	ret = cmd_record(rec_argc, rec_argv);
+
+	free(rec_argv);
+	free(filter);
+	return ret;
 }
 
 
@@ -1838,6 +1907,7 @@ static int timechart__record(struct timechart *tchart, int argc, const char **ar
 	const char **rec_argv;
 	const char **p;
 	unsigned int record_elems;
+	int ret;
 
 	const char * const common_args[] = {
 		"record", "-a", "-R", "-c", "1", "-o", output_data,
@@ -1902,24 +1972,27 @@ static int timechart__record(struct timechart *tchart, int argc, const char **ar
 
 	p = rec_argv;
 	for (i = 0; i < common_args_nr; i++)
-		*p++ = strdup(common_args[i]);
+		*p++ = common_args[i];
 
 	for (i = 0; i < backtrace_args_no; i++)
-		*p++ = strdup(backtrace_args[i]);
+		*p++ = backtrace_args[i];
 
 	for (i = 0; i < tasks_args_nr; i++)
-		*p++ = strdup(tasks_args[i]);
+		*p++ = tasks_args[i];
 
 	for (i = 0; i < power_args_nr; i++)
-		*p++ = strdup(power_args[i]);
+		*p++ = power_args[i];
 
 	for (i = 0; i < old_power_args_nr; i++)
-		*p++ = strdup(old_power_args[i]);
+		*p++ = old_power_args[i];
 
 	for (j = 0; j < (unsigned int)argc; j++)
 		*p++ = argv[j];
 
-	return cmd_record(rec_argc, rec_argv);
+	ret = cmd_record(rec_argc, rec_argv);
+
+	free(rec_argv);
+	return ret;
 }
 
 static int
@@ -2082,6 +2155,7 @@ int cmd_timechart(int argc, const char **argv)
 
 	ret = __cmd_timechart(&tchart, output_name);
 out:
+	timechart__release(&tchart);
 	zfree(&cpus_cstate_start_times);
 	zfree(&cpus_cstate_state);
 	zfree(&cpus_pstate_start_times);
diff --git a/tools/perf/builtin-top.c b/tools/perf/builtin-top.c
index 1211401616ee3..5933c46ee137e 100644
--- a/tools/perf/builtin-top.c
+++ b/tools/perf/builtin-top.c
@@ -141,7 +141,7 @@ static int perf_top__parse_source(struct perf_top *top, struct hist_entry *he)
 	notes = symbol__annotation(sym);
 	annotation__lock(notes);
 
-	if (!symbol__hists(sym, top->evlist->core.nr_entries)) {
+	if (!symbol__hists(sym, evlist__nr_entries(top->evlist))) {
 		annotation__unlock(notes);
 		pr_err("Not enough memory for annotating '%s' symbol!\n",
 		       sym->name);
@@ -267,7 +267,7 @@ static void perf_top__show_details(struct perf_top *top)
 
 	more = hist_entry__annotate_printf(he, top->sym_evsel);
 
-	if (top->evlist->enabled) {
+	if (evlist__enabled(top->evlist)) {
 		if (top->zero)
 			symbol__annotate_zero_histogram(symbol, top->sym_evsel);
 		else
@@ -293,7 +293,7 @@ static void perf_top__resort_hists(struct perf_top *t)
 		 */
 		hists__unlink(hists);
 
-		if (evlist->enabled) {
+		if (evlist__enabled(evlist)) {
 			if (t->zero) {
 				hists__delete_entries(hists);
 			} else {
@@ -334,13 +334,13 @@ static void perf_top__print_sym_table(struct perf_top *top)
 	printf("%-*.*s\n", win_width, win_width, graph_dotted_line);
 
 	if (!top->record_opts.overwrite &&
-	    (top->evlist->stats.nr_lost_warned !=
-	     top->evlist->stats.nr_events[PERF_RECORD_LOST])) {
-		top->evlist->stats.nr_lost_warned =
-			      top->evlist->stats.nr_events[PERF_RECORD_LOST];
+	    (evlist__stats(top->evlist)->nr_lost_warned !=
+	     evlist__stats(top->evlist)->nr_events[PERF_RECORD_LOST])) {
+		evlist__stats(top->evlist)->nr_lost_warned =
+			      evlist__stats(top->evlist)->nr_events[PERF_RECORD_LOST];
 		color_fprintf(stdout, PERF_COLOR_RED,
 			      "WARNING: LOST %d chunks, Check IO/CPU overload",
-			      top->evlist->stats.nr_lost_warned);
+			      evlist__stats(top->evlist)->nr_lost_warned);
 		++printed;
 	}
 
@@ -447,7 +447,7 @@ static void perf_top__print_mapped_keys(struct perf_top *top)
 	fprintf(stdout, "\t[d]     display refresh delay.             \t(%d)\n", top->delay_secs);
 	fprintf(stdout, "\t[e]     display entries (lines).           \t(%d)\n", top->print_entries);
 
-	if (top->evlist->core.nr_entries > 1)
+	if (evlist__nr_entries(top->evlist) > 1)
 		fprintf(stdout, "\t[E]     active event counter.              \t(%s)\n", evsel__name(top->sym_evsel));
 
 	fprintf(stdout, "\t[f]     profile display filter (count).    \t(%d)\n", top->count_filter);
@@ -482,7 +482,7 @@ static int perf_top__key_mapped(struct perf_top *top, int c)
 		case 'S':
 			return 1;
 		case 'E':
-			return top->evlist->core.nr_entries > 1 ? 1 : 0;
+			return evlist__nr_entries(top->evlist) > 1 ? 1 : 0;
 		default:
 			break;
 	}
@@ -528,7 +528,7 @@ static bool perf_top__handle_keypress(struct perf_top *top, int c)
 			}
 			break;
 		case 'E':
-			if (top->evlist->core.nr_entries > 1) {
+			if (evlist__nr_entries(top->evlist) > 1) {
 				/* Select 0 as the default event: */
 				int counter = 0;
 
@@ -539,7 +539,7 @@ static bool perf_top__handle_keypress(struct perf_top *top, int c)
 
 				prompt_integer(&counter, "Enter details event counter");
 
-				if (counter >= top->evlist->core.nr_entries) {
+				if (counter >= evlist__nr_entries(top->evlist)) {
 					top->sym_evsel = evlist__first(top->evlist);
 					fprintf(stderr, "Sorry, no such event, using %s.\n", evsel__name(top->sym_evsel));
 					sleep(1);
@@ -598,8 +598,8 @@ static void perf_top__sort_new_samples(void *arg)
 {
 	struct perf_top *t = arg;
 
-	if (t->evlist->selected != NULL)
-		t->sym_evsel = t->evlist->selected;
+	if (evlist__selected(t->evlist) != NULL)
+		t->sym_evsel = evlist__selected(t->evlist);
 
 	perf_top__resort_hists(t);
 
@@ -766,7 +766,7 @@ static void perf_event__process_sample(const struct perf_tool *tool,
 
 	if (!machine) {
 		pr_err("%u unprocessable samples recorded.\r",
-		       top->session->evlist->stats.nr_unprocessable_samples++);
+		       evlist__stats(top->session->evlist)->nr_unprocessable_samples++);
 		return;
 	}
 
@@ -859,7 +859,7 @@ perf_top__process_lost(struct perf_top *top, union perf_event *event,
 {
 	top->lost += event->lost.lost;
 	top->lost_total += event->lost.lost;
-	evsel->evlist->stats.total_lost += event->lost.lost;
+	evlist__stats(evsel->evlist)->total_lost += event->lost.lost;
 }
 
 static void
@@ -869,7 +869,7 @@ perf_top__process_lost_samples(struct perf_top *top,
 {
 	top->lost += event->lost_samples.lost;
 	top->lost_total += event->lost_samples.lost;
-	evsel->evlist->stats.total_lost_samples += event->lost_samples.lost;
+	evlist__stats(evsel->evlist)->total_lost_samples += event->lost_samples.lost;
 }
 
 static u64 last_timestamp;
@@ -881,7 +881,7 @@ static void perf_top__mmap_read_idx(struct perf_top *top, int idx)
 	struct mmap *md;
 	union perf_event *event;
 
-	md = opts->overwrite ? &evlist->overwrite_mmap[idx] : &evlist->mmap[idx];
+	md = opts->overwrite ? &evlist__overwrite_mmap(evlist)[idx] : &evlist__mmap(evlist)[idx];
 	if (perf_mmap__read_init(&md->core) < 0)
 		return;
 
@@ -918,7 +918,7 @@ static void perf_top__mmap_read(struct perf_top *top)
 	if (overwrite)
 		evlist__toggle_bkw_mmap(evlist, BKW_MMAP_DATA_PENDING);
 
-	for (i = 0; i < top->evlist->core.nr_mmaps; i++)
+	for (i = 0; i < evlist__core(top->evlist)->nr_mmaps; i++)
 		perf_top__mmap_read_idx(top, i);
 
 	if (overwrite) {
@@ -1063,7 +1063,7 @@ static int perf_top__start_counters(struct perf_top *top)
 		goto out_err;
 	}
 
-	if (evlist__mmap(evlist, opts->mmap_pages) < 0) {
+	if (evlist__do_mmap(evlist, opts->mmap_pages) < 0) {
 		ui__error("Failed to mmap with %d (%s)\n",
 			    errno, str_error_r(errno, msg, sizeof(msg)));
 		goto out_err;
@@ -1218,10 +1218,10 @@ static int deliver_event(struct ordered_events *qe,
 	} else if (event->header.type == PERF_RECORD_LOST_SAMPLES) {
 		perf_top__process_lost_samples(top, event, evsel);
 	} else if (event->header.type < PERF_RECORD_MAX) {
-		events_stats__inc(&session->evlist->stats, event->header.type);
+		events_stats__inc(evlist__stats(session->evlist), event->header.type);
 		machine__process_event(machine, event, &sample);
 	} else
-		++session->evlist->stats.nr_unknown_events;
+		++evlist__stats(session->evlist)->nr_unknown_events;
 
 	ret = 0;
 next_event:
@@ -1296,7 +1296,7 @@ static int __cmd_top(struct perf_top *top)
 		pr_debug("Couldn't synthesize cgroup events.\n");
 
 	machine__synthesize_threads(&top->session->machines.host, &opts->target,
-				    top->evlist->core.threads, true, false,
+				    evlist__core(top->evlist)->threads, true, false,
 				    top->nr_threads_synthesize);
 
 	perf_set_multithreaded();
@@ -1652,14 +1652,14 @@ int cmd_top(int argc, const char **argv)
 	perf_env__init(&host_env);
 	status = perf_config(perf_top_config, &top);
 	if (status)
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	/*
 	 * Since the per arch annotation init routine may need the cpuid, read
 	 * it here, since we are not getting this from the perf.data header.
 	 */
 	status = perf_env__set_cmdline(&host_env, argc, argv);
 	if (status)
-		goto out_delete_evlist;
+		goto out_put_evlist;
 
 	status = perf_env__read_cpuid(&host_env);
 	if (status) {
@@ -1680,30 +1680,30 @@ int cmd_top(int argc, const char **argv)
 		annotate_opts.disassembler_style = strdup(disassembler_style);
 		if (!annotate_opts.disassembler_style) {
 			status = -ENOMEM;
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 	}
 	if (objdump_path) {
 		annotate_opts.objdump_path = strdup(objdump_path);
 		if (!annotate_opts.objdump_path) {
 			status = -ENOMEM;
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 	}
 	if (addr2line_path) {
 		symbol_conf.addr2line_path = strdup(addr2line_path);
 		if (!symbol_conf.addr2line_path) {
 			status = -ENOMEM;
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 	}
 
 	status = symbol__validate_sym_arguments();
 	if (status)
-		goto out_delete_evlist;
+		goto out_put_evlist;
 
 	if (annotate_check_args() < 0)
-		goto out_delete_evlist;
+		goto out_put_evlist;
 
 	status = target__validate(target);
 	if (status) {
@@ -1714,19 +1714,19 @@ int cmd_top(int argc, const char **argv)
 	if (target__none(target))
 		target->system_wide = true;
 
-	if (!top.evlist->core.nr_entries) {
+	if (!evlist__nr_entries(top.evlist)) {
 		struct evlist *def_evlist = evlist__new_default(target, callchain_param.enabled);
 
 		if (!def_evlist)
-			goto out_delete_evlist;
+			goto out_put_evlist;
 
-		evlist__splice_list_tail(top.evlist, &def_evlist->core.entries);
-		evlist__delete(def_evlist);
+		evlist__splice_list_tail(top.evlist, &evlist__core(def_evlist)->entries);
+		evlist__put(def_evlist);
 	}
 
 	status = evswitch__init(&top.evswitch, top.evlist, stderr);
 	if (status)
-		goto out_delete_evlist;
+		goto out_put_evlist;
 
 	if (symbol_conf.report_hierarchy) {
 		/* disable incompatible options */
@@ -1737,18 +1737,18 @@ int cmd_top(int argc, const char **argv)
 			pr_err("Error: --hierarchy and --fields options cannot be used together\n");
 			parse_options_usage(top_usage, options, "fields", 0);
 			parse_options_usage(NULL, options, "hierarchy", 0);
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 	}
 
 	if (top.stitch_lbr && !(callchain_param.record_mode == CALLCHAIN_LBR)) {
 		pr_err("Error: --stitch-lbr must be used with --call-graph lbr\n");
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	if (nr_cgroups > 0 && opts->record_cgroup) {
 		pr_err("--cgroup and --all-cgroups cannot be used together\n");
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	if (branch_call_mode) {
@@ -1772,7 +1772,7 @@ int cmd_top(int argc, const char **argv)
 		status = perf_env__read_core_pmu_caps(&host_env);
 		if (status) {
 			pr_err("PMU capability data is not available\n");
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 	}
 
@@ -1795,9 +1795,9 @@ int cmd_top(int argc, const char **argv)
 	if (IS_ERR(top.session)) {
 		status = PTR_ERR(top.session);
 		top.session = NULL;
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
-	top.evlist->session = top.session;
+	evlist__set_session(top.evlist, top.session);
 
 	if (setup_sorting(top.evlist, perf_session__env(top.session)) < 0) {
 		if (sort_order)
@@ -1805,7 +1805,7 @@ int cmd_top(int argc, const char **argv)
 		if (field_order)
 			parse_options_usage(sort_order ? NULL : top_usage,
 					    options, "fields", 0);
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	if (top.uid_str) {
@@ -1814,18 +1814,18 @@ int cmd_top(int argc, const char **argv)
 		if (uid == UINT_MAX) {
 			ui__error("Invalid User: %s", top.uid_str);
 			status = -EINVAL;
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 		status = parse_uid_filter(top.evlist, uid);
 		if (status)
-			goto out_delete_evlist;
+			goto out_put_evlist;
 	}
 
 	if (evlist__create_maps(top.evlist, target) < 0) {
 		ui__error("Couldn't create thread/CPU maps: %s\n",
 			  errno == ENOENT ? "No such process" : str_error_r(errno, errbuf, sizeof(errbuf)));
 		status = -errno;
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	if (top.delay_secs < 1)
@@ -1833,7 +1833,7 @@ int cmd_top(int argc, const char **argv)
 
 	if (record_opts__config(opts)) {
 		status = -EINVAL;
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	top.sym_evsel = evlist__first(top.evlist);
@@ -1848,14 +1848,14 @@ int cmd_top(int argc, const char **argv)
 
 	status = symbol__annotation_init();
 	if (status < 0)
-		goto out_delete_evlist;
+		goto out_put_evlist;
 
 	annotation_config__init();
 
 	symbol_conf.try_vmlinux_path = (symbol_conf.vmlinux_name == NULL);
 	status = symbol__init(NULL);
 	if (status < 0)
-		goto out_delete_evlist;
+		goto out_put_evlist;
 
 	sort__setup_elide(stdout);
 
@@ -1875,13 +1875,13 @@ int cmd_top(int argc, const char **argv)
 		if (top.sb_evlist == NULL) {
 			pr_err("Couldn't create side band evlist.\n.");
 			status = -EINVAL;
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 
 		if (evlist__add_bpf_sb_event(top.sb_evlist, &host_env)) {
 			pr_err("Couldn't ask for PERF_RECORD_BPF_EVENT side band events.\n.");
 			status = -EINVAL;
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 	}
 #endif
@@ -1896,8 +1896,8 @@ int cmd_top(int argc, const char **argv)
 	if (!opts->no_bpf_event)
 		evlist__stop_sb_thread(top.sb_evlist);
 
-out_delete_evlist:
-	evlist__delete(top.evlist);
+out_put_evlist:
+	evlist__put(top.evlist);
 	perf_session__delete(top.session);
 	annotation_options__exit();
 	perf_env__exit(&host_env);
diff --git a/tools/perf/builtin-trace.c b/tools/perf/builtin-trace.c
index ba0f8749fc7d7..845a9fe47f0b2 100644
--- a/tools/perf/builtin-trace.c
+++ b/tools/perf/builtin-trace.c
@@ -460,10 +460,10 @@ static int evsel__init_tp_ptr_field(struct evsel *evsel, struct tp_field *field,
 	({ struct syscall_tp *sc = __evsel__syscall_tp(evsel);\
 	   evsel__init_tp_ptr_field(evsel, &sc->name, #name); })
 
-static void evsel__delete_priv(struct evsel *evsel)
+static void evsel__put_and_free_priv(struct evsel *evsel)
 {
 	zfree(&evsel->priv);
-	evsel__delete(evsel);
+	evsel__put(evsel);
 }
 
 static int evsel__init_syscall_tp(struct evsel *evsel)
@@ -543,7 +543,7 @@ static struct evsel *perf_evsel__raw_syscall_newtp(const char *direction, void *
 	return evsel;
 
 out_delete:
-	evsel__delete_priv(evsel);
+	evsel__put_and_free_priv(evsel);
 	return NULL;
 }
 
@@ -2023,7 +2023,7 @@ static int trace__symbols_init(struct trace *trace, int argc, const char **argv,
 		goto out;
 
 	err = __machine__synthesize_threads(trace->host, &trace->tool, &trace->opts.target,
-					    evlist->core.threads, trace__tool_process,
+					    evlist__core(evlist)->threads, trace__tool_process,
 					    /*needs_mmap=*/callchain_param.enabled &&
 							   !trace->summary_only,
 					    /*mmap_data=*/false,
@@ -3633,7 +3633,7 @@ static bool evlist__add_vfs_getname(struct evlist *evlist)
 
 		list_del_init(&evsel->core.node);
 		evsel->evlist = NULL;
-		evsel__delete(evsel);
+		evsel__put(evsel);
 	}
 
 	return found;
@@ -3678,8 +3678,11 @@ static void trace__handle_event(struct trace *trace, union perf_event *event, st
 		return;
 	}
 
-	if (sample->evsel == NULL)
+	if (sample->evsel == NULL) {
 		sample->evsel = evlist__id2evsel(trace->evlist, sample->id);
+		if (sample->evsel)
+			evsel__get(sample->evsel);
+	}
 
 	if (sample->evsel == NULL) {
 		fprintf(trace->output, "Unknown tp ID %" PRIu64 ", skipping...\n", sample->id);
@@ -3749,9 +3752,9 @@ static int trace__add_syscall_newtp(struct trace *trace)
 	return ret;
 
 out_delete_sys_exit:
-	evsel__delete_priv(sys_exit);
+	evsel__put_and_free_priv(sys_exit);
 out_delete_sys_enter:
-	evsel__delete_priv(sys_enter);
+	evsel__put_and_free_priv(sys_enter);
 	goto out;
 }
 
@@ -4216,7 +4219,7 @@ static int trace__set_filter_pids(struct trace *trace)
 			err = augmented_syscalls__set_filter_pids(trace->filter_pids.nr,
 						       trace->filter_pids.entries);
 		}
-	} else if (perf_thread_map__pid(trace->evlist->core.threads, 0) == -1) {
+	} else if (perf_thread_map__pid(evlist__core(trace->evlist)->threads, 0) == -1) {
 		err = trace__set_filter_loop_pids(trace);
 	}
 
@@ -4439,7 +4442,7 @@ static int trace__run(struct trace *trace, int argc, const char **argv)
 
 	if (trace->summary_bpf) {
 		if (trace_prepare_bpf_summary(trace->summary_mode) < 0)
-			goto out_delete_evlist;
+			goto out_put_evlist;
 
 		if (trace->summary_only)
 			goto create_maps;
@@ -4507,19 +4510,19 @@ static int trace__run(struct trace *trace, int argc, const char **argv)
 	err = evlist__create_maps(evlist, &trace->opts.target);
 	if (err < 0) {
 		fprintf(trace->output, "Problems parsing the target to trace, check your options!\n");
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	err = trace__symbols_init(trace, argc, argv, evlist);
 	if (err < 0) {
 		fprintf(trace->output, "Problems initializing symbol libraries!\n");
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	if (trace->summary_mode == SUMMARY__BY_TOTAL && !trace->summary_bpf) {
 		trace->syscall_stats = alloc_syscall_stats();
 		if (!trace->syscall_stats)
-			goto out_delete_evlist;
+			goto out_put_evlist;
 	}
 
 	evlist__config(evlist, &trace->opts, &callchain_param);
@@ -4528,9 +4531,9 @@ static int trace__run(struct trace *trace, int argc, const char **argv)
 		err = evlist__prepare_workload(evlist, &trace->opts.target, argv, false, NULL);
 		if (err < 0) {
 			fprintf(trace->output, "Couldn't run the workload!\n");
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
-		workload_pid = evlist->workload.pid;
+		workload_pid = evlist__workload_pid(evlist);
 	}
 
 	err = evlist__open(evlist);
@@ -4576,13 +4579,13 @@ static int trace__run(struct trace *trace, int argc, const char **argv)
 
 	err = trace__expand_filters(trace, &evsel);
 	if (err)
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	err = evlist__apply_filters(evlist, &evsel, &trace->opts.target);
 	if (err < 0)
 		goto out_error_apply_filters;
 
 	if (!trace->summary_only || !trace->summary_bpf) {
-		err = evlist__mmap(evlist, trace->opts.mmap_pages);
+		err = evlist__do_mmap(evlist, trace->opts.mmap_pages);
 		if (err < 0)
 			goto out_error_mmap;
 	}
@@ -4601,8 +4604,8 @@ static int trace__run(struct trace *trace, int argc, const char **argv)
 	if (trace->summary_bpf)
 		trace_start_bpf_summary();
 
-	trace->multiple_threads = perf_thread_map__pid(evlist->core.threads, 0) == -1 ||
-		perf_thread_map__nr(evlist->core.threads) > 1 ||
+	trace->multiple_threads = perf_thread_map__pid(evlist__core(evlist)->threads, 0) == -1 ||
+		perf_thread_map__nr(evlist__core(evlist)->threads) > 1 ||
 		evlist__first(evlist)->core.attr.inherit;
 
 	/*
@@ -4619,11 +4622,11 @@ static int trace__run(struct trace *trace, int argc, const char **argv)
 again:
 	before = trace->nr_events;
 
-	for (i = 0; i < evlist->core.nr_mmaps; i++) {
+	for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) {
 		union perf_event *event;
 		struct mmap *md;
 
-		md = &evlist->mmap[i];
+		md = &evlist__mmap(evlist)[i];
 		if (perf_mmap__read_init(&md->core) < 0)
 			continue;
 
@@ -4693,12 +4696,12 @@ static int trace__run(struct trace *trace, int argc, const char **argv)
 		}
 	}
 
-out_delete_evlist:
+out_put_evlist:
 	trace_cleanup_bpf_summary();
 	delete_syscall_stats(trace->syscall_stats);
 	trace__symbols__exit(trace);
 	evlist__free_syscall_tp_fields(evlist);
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	cgroup__put(trace->cgroup);
 	trace->evlist = NULL;
 	trace->live = false;
@@ -4723,21 +4726,21 @@ static int trace__run(struct trace *trace, int argc, const char **argv)
 
 out_error:
 	fprintf(trace->output, "%s\n", errbuf);
-	goto out_delete_evlist;
+	goto out_put_evlist;
 
 out_error_apply_filters:
 	fprintf(trace->output,
 		"Failed to set filter \"%s\" on event %s: %m\n",
 		evsel->filter, evsel__name(evsel));
-	goto out_delete_evlist;
+	goto out_put_evlist;
 }
 out_error_mem:
 	fprintf(trace->output, "Not enough memory to run!\n");
-	goto out_delete_evlist;
+	goto out_put_evlist;
 
 out_errno:
 	fprintf(trace->output, "%m\n");
-	goto out_delete_evlist;
+	goto out_put_evlist;
 }
 
 static int trace__replay(struct trace *trace)
@@ -5325,7 +5328,7 @@ static int trace__parse_cgroups(const struct option *opt, const char *str, int u
 {
 	struct trace *trace = opt->value;
 
-	if (!list_empty(&trace->evlist->core.entries)) {
+	if (!list_empty(&evlist__core(trace->evlist)->entries)) {
 		struct option o = {
 			.value = &trace->evlist,
 		};
@@ -5417,7 +5420,7 @@ static void trace__exit(struct trace *trace)
 		zfree(&trace->syscalls.table);
 	}
 	zfree(&trace->perfconfig_events);
-	evlist__delete(trace->evlist);
+	evlist__put(trace->evlist);
 	trace->evlist = NULL;
 	ordered_events__free(&trace->oe.data);
 #ifdef HAVE_LIBBPF_SUPPORT
@@ -5599,7 +5602,7 @@ int cmd_trace(int argc, const char **argv)
 	 * .perfconfig trace.add_events, and filter those out.
 	 */
 	if (!trace.trace_syscalls && !trace.trace_pgfaults &&
-	    trace.evlist->core.nr_entries == 0 /* Was --events used? */) {
+	    evlist__nr_entries(trace.evlist) == 0 /* Was --events used? */) {
 		trace.trace_syscalls = true;
 	}
 	/*
@@ -5685,7 +5688,7 @@ int cmd_trace(int argc, const char **argv)
 		symbol_conf.use_callchain = true;
 	}
 
-	if (trace.evlist->core.nr_entries > 0) {
+	if (evlist__nr_entries(trace.evlist) > 0) {
 		bool use_btf = false;
 
 		evlist__set_default_evsel_handler(trace.evlist, trace__event_handler);
diff --git a/tools/perf/pmu-events/arch/x86/alderlake/adl-metrics.json b/tools/perf/pmu-events/arch/x86/alderlake/adl-metrics.json
index cae7c0cf02f22..e76d679a784b0 100644
--- a/tools/perf/pmu-events/arch/x86/alderlake/adl-metrics.json
+++ b/tools/perf/pmu-events/arch/x86/alderlake/adl-metrics.json
@@ -800,16 +800,16 @@
     },
     {
         "BriefDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + tma_memory_bound * (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_fb_full / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_miss_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + tma_memory_bound * (tma_l3_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_fb_full / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)))",
         "MetricGroup": "BvMB;Mem;MemoryBW;Offcore;tma_issueBW",
         "MetricName": "tma_bottleneck_data_cache_memory_bandwidth",
         "MetricThreshold": "tma_bottleneck_data_cache_memory_bandwidth > 20",
-        "PublicDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks. Related metrics: tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full",
+        "PublicDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks. Related metrics: tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_uc_bound",
         "Unit": "cpu_core"
     },
     {
         "BriefDescription": "Total pipeline cost of external Memory- or Cache-Latency related bottlenecks",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + tma_memory_bound * (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l1_latency_dependency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_lock_latency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_split_loads / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_split_stores / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)) + tma_memory_bound * (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_store_latency / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_miss_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + tma_memory_bound * (tma_l3_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * tma_l2_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_l1_latency_dependency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_lock_latency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_split_loads / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_split_stores / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)) + tma_memory_bound * (tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_store_latency / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
         "MetricGroup": "BvML;Mem;MemoryLat;Offcore;tma_issueLat",
         "MetricName": "tma_bottleneck_data_cache_memory_latency",
         "MetricThreshold": "tma_bottleneck_data_cache_memory_latency > 20",
@@ -835,7 +835,7 @@
     },
     {
         "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l1_bound / max(tma_memory_bound, tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
         "MetricGroup": "BvMT;Mem;MemoryTLB;Offcore;tma_issueTLB",
         "MetricName": "tma_bottleneck_memory_data_tlbs",
         "MetricThreshold": "tma_bottleneck_memory_data_tlbs > 20",
@@ -844,7 +844,7 @@
     },
     {
         "BriefDescription": "Total pipeline cost of Memory Synchronization related bottlenecks (data transfers and coherency updates across processors)",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) * (tma_contested_accesses + tma_data_sharing) / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full) + tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) * tma_false_sharing / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores - tma_store_latency)) + tma_machine_clears * (1 - tma_other_nukes / tma_other_nukes))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) * (tma_contested_accesses + tma_data_sharing) / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full) + tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) * tma_false_sharing / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores - tma_store_latency)) + tma_machine_clears * (1 - tma_other_nukes / tma_other_nukes))",
         "MetricGroup": "BvMS;LockCont;Mem;Offcore;tma_issueSyncxn",
         "MetricName": "tma_bottleneck_memory_synchronization",
         "MetricThreshold": "tma_bottleneck_memory_synchronization > 10",
@@ -1012,13 +1012,13 @@
         "Unit": "cpu_core"
     },
     {
-        "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+        "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to L3 data-sharing accesses",
         "MetricConstraint": "NO_GROUP_EVENTS",
         "MetricExpr": "24 * tma_info_system_core_frequency * (cpu_core@MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD@ + cpu_core@MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD@ * (1 - cpu_core@OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM@ / (cpu_core@OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM@ + cpu_core@OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD@))) * (1 + cpu_core@MEM_LOAD_RETIRED.FB_HIT@ / cpu_core@MEM_LOAD_RETIRED.L1_MISS@ / 2) / tma_info_thread_clks",
         "MetricGroup": "BvMS;Offcore;Snoop;TopdownL4;tma_L4_group;tma_issueSyncxn;tma_l3_bound_group",
         "MetricName": "tma_data_sharing",
         "MetricThreshold": "tma_data_sharing > 0.05 & (tma_l3_bound > 0.05 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD. Related metrics: tma_bottleneck_memory_synchronization, tma_contested_accesses, tma_false_sharing, tma_machine_clears, tma_remote_cache",
+        "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to L3 data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD. Related metrics: tma_bottleneck_memory_synchronization, tma_contested_accesses, tma_false_sharing, tma_machine_clears, tma_remote_cache",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -1042,16 +1042,6 @@
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
-    {
-        "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
-        "MetricExpr": "cpu_core@MEMORY_ACTIVITY.STALLS_L3_MISS@ / tma_info_thread_clks",
-        "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_L3_group;tma_memory_bound_group",
-        "MetricName": "tma_dram_bound",
-        "MetricThreshold": "tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)",
-        "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS",
-        "ScaleUnit": "100%",
-        "Unit": "cpu_core"
-    },
     {
         "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
         "MetricExpr": "(cpu_core@IDQ.DSB_CYCLES_ANY@ - cpu_core@IDQ.DSB_CYCLES_OK@) / tma_info_core_core_clks / 2",
@@ -1108,7 +1098,7 @@
         "MetricGroup": "BvMB;MemoryBW;TopdownL4;tma_L4_group;tma_issueBW;tma_issueSL;tma_issueSmSt;tma_l1_bound_group",
         "MetricName": "tma_fb_full",
         "MetricThreshold": "tma_fb_full > 0.3",
-        "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_store_latency, tma_streaming_stores",
+        "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_store_latency, tma_streaming_stores, tma_uc_bound",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -1426,7 +1416,7 @@
     },
     {
         "BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)",
-        "MetricExpr": "cpu_core@IDQ.DSB_UOPS@ / cpu_core@UOPS_ISSUED.ANY@",
+        "MetricExpr": "cpu_core@IDQ.DSB_UOPS@ / (cpu_core@IDQ.DSB_UOPS@ + cpu_core@LSD.UOPS@ + cpu_core@IDQ.MITE_UOPS@ + cpu_core@IDQ.MS_UOPS@)",
         "MetricGroup": "DSB;Fed;FetchBW;tma_issueFB",
         "MetricName": "tma_info_frontend_dsb_coverage",
         "MetricThreshold": "tma_info_frontend_dsb_coverage < 0.7 & tma_info_thread_ipc / 6 > 0.35",
@@ -1485,7 +1475,7 @@
     },
     {
         "BriefDescription": "Fraction of Uops delivered by the LSD (Loop Stream Detector; aka Loop Cache)",
-        "MetricExpr": "cpu_core@LSD.UOPS@ / cpu_core@UOPS_ISSUED.ANY@",
+        "MetricExpr": "cpu_core@LSD.UOPS@ / (cpu_core@IDQ.DSB_UOPS@ + cpu_core@LSD.UOPS@ + cpu_core@IDQ.MITE_UOPS@ + cpu_core@IDQ.MS_UOPS@)",
         "MetricGroup": "Fed;LSD",
         "MetricName": "tma_info_frontend_lsd_coverage",
         "Unit": "cpu_core"
@@ -1629,6 +1619,15 @@
         "PublicDescription": "Instructions per taken branch. Related metrics: tma_dsb_switches, tma_fetch_bandwidth, tma_info_botlnk_l2_dsb_bandwidth, tma_info_botlnk_l2_dsb_misses, tma_info_frontend_dsb_coverage, tma_lcp",
         "Unit": "cpu_core"
     },
+    {
+        "BriefDescription": "AVX preserve/restore assists per kilo instruction",
+        "MetricExpr": "1e3 * cpu_core@ASSISTS.SSE_AVX_MIX@ / cpu_core@INST_RETIRED.ANY@",
+        "MetricGroup": "tma_issueMV",
+        "MetricName": "tma_info_inst_mix_vectormixpki",
+        "MetricThreshold": "tma_info_inst_mix_vectormixpki > 0.05",
+        "PublicDescription": "AVX preserve/restore assists per kilo instruction. Related metrics: tma_mixing_vectors, tma_ms_switches",
+        "Unit": "cpu_core"
+    },
     {
         "BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]",
         "MetricExpr": "tma_info_memory_l1d_cache_fill_bw",
@@ -1850,7 +1849,7 @@
         "Unit": "cpu_core"
     },
     {
-        "BriefDescription": "Mem;Backend;CacheHits",
+        "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per physical core",
         "MetricExpr": "cpu_core@UOPS_EXECUTED.THREAD@ / (cpu_core@UOPS_EXECUTED.CORE_CYCLES_GE_1@ / 2 if #SMT_on else cpu_core@UOPS_EXECUTED.THREAD\\,cmask\\=1@)",
         "MetricGroup": "Cor;Pipeline;PortsUtil;SMT",
         "MetricName": "tma_info_pipeline_execute",
@@ -1935,7 +1934,7 @@
         "MetricExpr": "64 * (UNC_ARB_TRK_REQUESTS.ALL + UNC_ARB_COH_TRK_REQUESTS.ALL) / 1e6 / tma_info_system_time / 1e3",
         "MetricGroup": "HPC;MemOffcore;MemoryBW;SoC;tma_issueBW",
         "MetricName": "tma_info_system_dram_bw_use",
-        "PublicDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]. Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_mem_bandwidth, tma_sq_full",
+        "PublicDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]. Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_mem_bandwidth, tma_sq_full, tma_uc_bound",
         "Unit": "cpu_core"
     },
     {
@@ -2029,13 +2028,6 @@
         "MetricName": "tma_info_system_turbo_utilization",
         "Unit": "cpu_core"
     },
-    {
-        "BriefDescription": "Measured Average Uncore Frequency for the SoC [GHz]",
-        "MetricExpr": "tma_info_system_socket_clks / 1e9 / tma_info_system_time",
-        "MetricGroup": "SoC",
-        "MetricName": "tma_info_system_uncore_frequency",
-        "Unit": "cpu_core"
-    },
     {
         "BriefDescription": "Per-Logical Processor actual clocks when the Logical Processor is active.",
         "MetricExpr": "cpu_core@CPU_CLK_UNHALTED.THREAD@",
@@ -2204,6 +2196,16 @@
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
+    {
+        "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+        "MetricExpr": "cpu_core@MEMORY_ACTIVITY.STALLS_L3_MISS@ / tma_info_thread_clks",
+        "MetricGroup": "MemoryBound;Offcore;TmaL3mem;TopdownL3;tma_L3_group;tma_memory_bound_group",
+        "MetricName": "tma_l3_miss_bound",
+        "MetricThreshold": "tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)",
+        "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS",
+        "ScaleUnit": "100%",
+        "Unit": "cpu_core"
+    },
     {
         "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
         "MetricExpr": "cpu_core@DECODE.LCP@ / tma_info_thread_clks",
@@ -2283,7 +2285,7 @@
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations",
         "MetricConstraint": "NO_GROUP_EVENTS",
-        "MetricExpr": "(16 * max(0, cpu_core@MEM_INST_RETIRED.LOCK_LOADS@ - cpu_core@L2_RQSTS.ALL_RFO@) + cpu_core@MEM_INST_RETIRED.LOCK_LOADS@ / cpu_core@MEM_INST_RETIRED.ALL_STORES@ * (10 * cpu_core@L2_RQSTS.RFO_HIT@ + min(cpu_core@CPU_CLK_UNHALTED.THREAD@, cpu_core@OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO@))) / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@LOCK_CYCLES.CACHE_LOCK_DURATION@ / tma_info_thread_clks",
         "MetricGroup": "LockCont;Offcore;TopdownL4;tma_L4_group;tma_issueRFO;tma_l1_bound_group",
         "MetricName": "tma_lock_latency",
         "MetricThreshold": "tma_lock_latency > 0.2 & (tma_l1_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
@@ -2315,19 +2317,19 @@
     {
         "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM)",
         "MetricExpr": "min(cpu_core@CPU_CLK_UNHALTED.THREAD@, cpu_core@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / tma_info_thread_clks",
-        "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_dram_bound_group;tma_issueBW",
+        "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_issueBW;tma_l3_miss_bound_group",
         "MetricName": "tma_mem_bandwidth",
-        "MetricThreshold": "tma_mem_bandwidth > 0.2 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM).  The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_sq_full",
+        "MetricThreshold": "tma_mem_bandwidth > 0.2 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
+        "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM).  The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_sq_full, tma_uc_bound",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
     {
         "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory - DRAM ([SPR-HBM] and/or HBM)",
         "MetricExpr": "min(cpu_core@CPU_CLK_UNHALTED.THREAD@, cpu_core@OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD@) / tma_info_thread_clks - tma_mem_bandwidth",
-        "MetricGroup": "BvML;MemoryLat;Offcore;TopdownL4;tma_L4_group;tma_dram_bound_group;tma_issueLat",
+        "MetricGroup": "BvML;MemoryLat;Offcore;TopdownL4;tma_L4_group;tma_issueLat;tma_l3_miss_bound_group",
         "MetricName": "tma_mem_latency",
-        "MetricThreshold": "tma_mem_latency > 0.1 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
+        "MetricThreshold": "tma_mem_latency > 0.1 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
         "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory - DRAM ([SPR-HBM] and/or HBM).  This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_latency, tma_l3_hit_latency",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
@@ -2397,7 +2399,7 @@
         "MetricGroup": "TopdownL5;tma_L5_group;tma_issueMV;tma_ports_utilized_0_group",
         "MetricName": "tma_mixing_vectors",
         "MetricThreshold": "tma_mixing_vectors > 0.05",
-        "PublicDescription": "This metric estimates penalty in terms of percentage of([SKL+] injected blend uops out of all Uops Issued -- the Count Domain; [ADL+] cycles). Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic. Related metrics: tma_ms_switches",
+        "PublicDescription": "This metric estimates penalty in terms of percentage of([SKL+] injected blend uops out of all Uops Issued -- the Count Domain; [ADL+] cycles). Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic. Related metrics: tma_info_inst_mix_vectormixpki, tma_ms_switches",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -2416,7 +2418,7 @@
         "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_L3_group;tma_fetch_latency_group;tma_issueMC;tma_issueMS;tma_issueMV;tma_issueSO",
         "MetricName": "tma_ms_switches",
         "MetricThreshold": "tma_ms_switches > 0.05 & (tma_fetch_latency > 0.1 & tma_frontend_bound > 0.15)",
-        "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: FRONTEND_RETIRED.MS_FLOWS. Related metrics: tma_bottleneck_irregular_overhead, tma_clears_resteers, tma_l1_bound, tma_machine_clears, tma_microcode_sequencer, tma_mixing_vectors, tma_serializing_operation",
+        "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: FRONTEND_RETIRED.MS_FLOWS. Related metrics: tma_bottleneck_irregular_overhead, tma_clears_resteers, tma_info_inst_mix_vectormixpki, tma_l1_bound, tma_machine_clears, tma_microcode_sequencer, tma_mixing_vectors, tma_serializing_operation",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -2606,7 +2608,7 @@
     },
     {
         "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary",
-        "MetricExpr": "tma_info_memory_load_miss_real_latency * cpu_core@LD_BLOCKS.NO_SR@ / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@MEM_INST_RETIRED.SPLIT_LOADS@ * tma_info_memory_load_miss_real_latency / tma_info_thread_clks",
         "MetricGroup": "TopdownL4;tma_L4_group;tma_l1_bound_group",
         "MetricName": "tma_split_loads",
         "MetricThreshold": "tma_split_loads > 0.3",
@@ -2630,7 +2632,7 @@
         "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_issueBW;tma_l3_bound_group",
         "MetricName": "tma_sq_full",
         "MetricThreshold": "tma_sq_full > 0.3 & (tma_l3_bound > 0.05 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth",
+        "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_uc_bound",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
diff --git a/tools/perf/pmu-events/arch/x86/alderlake/cache.json b/tools/perf/pmu-events/arch/x86/alderlake/cache.json
index e44e6b651d55e..6c09da7b69e8e 100644
--- a/tools/perf/pmu-events/arch/x86/alderlake/cache.json
+++ b/tools/perf/pmu-events/arch/x86/alderlake/cache.json
@@ -1,4 +1,13 @@
 [
+    {
+        "BriefDescription": "Counts the number of core requests (demand and L1 prefetchers) rejected by the L2 queue (L2Q) due to a full condition.",
+        "Counter": "0,1,2,3,4,5",
+        "EventCode": "0x31",
+        "EventName": "CORE_REJECT_L2Q.ANY",
+        "PublicDescription": "Counts the number of (demand and L1 prefetchers) core requests rejected by the L2 queue (L2Q) due to a full or nearly full condition, which likely indicates back pressure from L2Q.  It also counts requests that would have gone directly to the External Queue (XQ), but are rejected due to a full or nearly full condition, indicating back pressure from the IDI link.  The L2Q may also reject transactions  from a core to ensure fairness between cores, or to delay a cores dirty eviction when the address conflicts incoming external snoops.  (Note that L2 prefetcher requests that are dropped are not counted by this event).  Counts on a per core basis.",
+        "SampleAfterValue": "200003",
+        "Unit": "cpu_atom"
+    },
     {
         "BriefDescription": "L1D.HWPF_MISS",
         "Counter": "0,1,2,3",
@@ -121,6 +130,15 @@
         "UMask": "0x4",
         "Unit": "cpu_core"
     },
+    {
+        "BriefDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition.",
+        "Counter": "0,1,2,3,4,5",
+        "EventCode": "0x30",
+        "EventName": "L2_REJECT_XQ.ANY",
+        "PublicDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition which likely indicates back pressure from the IDI link.  The XQ may reject transactions from the L2Q (non-cacheable requests), BBL (L2 misses) and WOB (L2 write-back victims).",
+        "SampleAfterValue": "200003",
+        "Unit": "cpu_atom"
+    },
     {
         "BriefDescription": "Counts the total number of L2 Cache accesses. Counts on a per core basis.",
         "Counter": "0,1,2,3,4,5",
@@ -438,6 +456,15 @@
         "UMask": "0x8",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of cycles the core is stalled due to an instruction cache or TLB miss which missed in the L2 cache.",
+        "Counter": "0,1,2,3,4,5",
+        "EventCode": "0x34",
+        "EventName": "MEM_BOUND_STALLS.IFETCH_L2_MISS",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x30",
+        "Unit": "cpu_atom"
+    },
     {
         "BriefDescription": "Counts the number of cycles the core is stalled due to an instruction cache or TLB miss which hit in the LLC or other core with HITE/F/M.",
         "Counter": "0,1,2,3,4,5",
@@ -475,6 +502,15 @@
         "UMask": "0x1",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of cycles the core is stalled due to a demand load which missed in the L2 cache.",
+        "Counter": "0,1,2,3,4,5",
+        "EventCode": "0x34",
+        "EventName": "MEM_BOUND_STALLS.LOAD_L2_MISS",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x6",
+        "Unit": "cpu_atom"
+    },
     {
         "BriefDescription": "Counts the number of cycles the core is stalled due to a demand load which hit in the LLC or other core with HITE/F/M.",
         "Counter": "0,1,2,3,4,5",
diff --git a/tools/perf/pmu-events/arch/x86/alderlake/metricgroups.json b/tools/perf/pmu-events/arch/x86/alderlake/metricgroups.json
index 855585fe6fae1..9acbda612c25c 100644
--- a/tools/perf/pmu-events/arch/x86/alderlake/metricgroups.json
+++ b/tools/perf/pmu-events/arch/x86/alderlake/metricgroups.json
@@ -93,7 +93,6 @@
     "tma_code_stlb_miss_group": "Metrics contributing to tma_code_stlb_miss category",
     "tma_core_bound_group": "Metrics contributing to tma_core_bound category",
     "tma_divider_group": "Metrics contributing to tma_divider category",
-    "tma_dram_bound_group": "Metrics contributing to tma_dram_bound category",
     "tma_dtlb_load_group": "Metrics contributing to tma_dtlb_load category",
     "tma_dtlb_store_group": "Metrics contributing to tma_dtlb_store category",
     "tma_fetch_bandwidth_group": "Metrics contributing to tma_fetch_bandwidth category",
@@ -129,6 +128,7 @@
     "tma_l1_bound_group": "Metrics contributing to tma_l1_bound category",
     "tma_l2_bound_group": "Metrics contributing to tma_l2_bound category",
     "tma_l3_bound_group": "Metrics contributing to tma_l3_bound category",
+    "tma_l3_miss_bound_group": "Metrics contributing to tma_l3_miss_bound category",
     "tma_light_operations_group": "Metrics contributing to tma_light_operations category",
     "tma_load_op_utilization_group": "Metrics contributing to tma_load_op_utilization category",
     "tma_load_stlb_miss_group": "Metrics contributing to tma_load_stlb_miss category",
diff --git a/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json b/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json
index 1c292f29b0aa0..eb840066a5a75 100644
--- a/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json
+++ b/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json
@@ -550,11 +550,11 @@
         "Unit": "cpu_atom"
     },
     {
-        "BriefDescription": "This event counts the number of mispredicted ret instructions retired. Non PEBS",
+        "BriefDescription": "This event counts the number of mispredicted ret instructions retired.",
         "Counter": "0,1,2,3,4,5,6,7",
         "EventCode": "0xc5",
         "EventName": "BR_MISP_RETIRED.RET",
-        "PublicDescription": "This is a non-precise version (that is, does not use PEBS) of the event that counts mispredicted return instructions retired. Available PDIST counters: 0",
+        "PublicDescription": "This event counts the number of mispredicted ret instructions retired. Available PDIST counters: 0",
         "SampleAfterValue": "100007",
         "UMask": "0x8",
         "Unit": "cpu_core"
@@ -988,7 +988,7 @@
         "Counter": "0,1,2,3,4,5,6,7",
         "EventCode": "0xc0",
         "EventName": "INST_RETIRED.REP_ITERATION",
-        "PublicDescription": "Number of iterations of Repeat (REP) string retired instructions such as MOVS, CMPS, and SCAS. Each has a byte, word, and doubleword version and string instructions can be repeated using a repetition prefix, REP, that allows their architectural execution to be repeated a number of times as specified by the RCX register. Note the number of iterations is implementation-dependent.",
+        "PublicDescription": "Number of iterations of Repeat (REP) string retired instructions such as MOVS, CMPS, and SCAS. Each has a byte, word, and doubleword version and string instructions can be repeated using a repetition prefix, REP, that allows their architectural execution to be repeated a number of times as specified by the RCX register. Note: Since the number of iterations within a REP instruction can be significantly affected by fast strings, this event may vary run to run and not match the architectural number of iterations (specified by RCX)",
         "SampleAfterValue": "2000003",
         "UMask": "0x8",
         "Unit": "cpu_core"
diff --git a/tools/perf/pmu-events/arch/x86/alderlaken/cache.json b/tools/perf/pmu-events/arch/x86/alderlaken/cache.json
index 0ffad953e7524..b39c73614b86b 100644
--- a/tools/perf/pmu-events/arch/x86/alderlaken/cache.json
+++ b/tools/perf/pmu-events/arch/x86/alderlaken/cache.json
@@ -1,4 +1,20 @@
 [
+    {
+        "BriefDescription": "Counts the number of core requests (demand and L1 prefetchers) rejected by the L2 queue (L2Q) due to a full condition.",
+        "Counter": "0,1,2,3,4,5",
+        "EventCode": "0x31",
+        "EventName": "CORE_REJECT_L2Q.ANY",
+        "PublicDescription": "Counts the number of (demand and L1 prefetchers) core requests rejected by the L2 queue (L2Q) due to a full or nearly full condition, which likely indicates back pressure from L2Q.  It also counts requests that would have gone directly to the External Queue (XQ), but are rejected due to a full or nearly full condition, indicating back pressure from the IDI link.  The L2Q may also reject transactions  from a core to ensure fairness between cores, or to delay a cores dirty eviction when the address conflicts incoming external snoops.  (Note that L2 prefetcher requests that are dropped are not counted by this event).  Counts on a per core basis.",
+        "SampleAfterValue": "200003"
+    },
+    {
+        "BriefDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition.",
+        "Counter": "0,1,2,3,4,5",
+        "EventCode": "0x30",
+        "EventName": "L2_REJECT_XQ.ANY",
+        "PublicDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition which likely indicates back pressure from the IDI link.  The XQ may reject transactions from the L2Q (non-cacheable requests), BBL (L2 misses) and WOB (L2 write-back victims).",
+        "SampleAfterValue": "200003"
+    },
     {
         "BriefDescription": "Counts the total number of L2 Cache accesses. Counts on a per core basis.",
         "Counter": "0,1,2,3,4,5",
@@ -70,6 +86,14 @@
         "SampleAfterValue": "200003",
         "UMask": "0x8"
     },
+    {
+        "BriefDescription": "Counts the number of cycles the core is stalled due to an instruction cache or TLB miss which missed in the L2 cache.",
+        "Counter": "0,1,2,3,4,5",
+        "EventCode": "0x34",
+        "EventName": "MEM_BOUND_STALLS.IFETCH_L2_MISS",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x30"
+    },
     {
         "BriefDescription": "Counts the number of cycles the core is stalled due to an instruction cache or TLB miss which hit in the LLC or other core with HITE/F/M.",
         "Counter": "0,1,2,3,4,5",
@@ -103,6 +127,14 @@
         "SampleAfterValue": "200003",
         "UMask": "0x1"
     },
+    {
+        "BriefDescription": "Counts the number of cycles the core is stalled due to a demand load which missed in the L2 cache.",
+        "Counter": "0,1,2,3,4,5",
+        "EventCode": "0x34",
+        "EventName": "MEM_BOUND_STALLS.LOAD_L2_MISS",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x6"
+    },
     {
         "BriefDescription": "Counts the number of cycles the core is stalled due to a demand load which hit in the LLC or other core with HITE/F/M.",
         "Counter": "0,1,2,3,4,5",
diff --git a/tools/perf/pmu-events/arch/x86/amdzen5/floating-point.json b/tools/perf/pmu-events/arch/x86/amdzen5/floating-point.json
index 9204bfb1d69e0..50d38434f8d3d 100644
--- a/tools/perf/pmu-events/arch/x86/amdzen5/floating-point.json
+++ b/tools/perf/pmu-events/arch/x86/amdzen5/floating-point.json
@@ -179,6 +179,30 @@
     "BriefDescription": "Retired scalar floating-point blend ops.",
     "UMask": "0x09"
   },
+  {
+    "EventName": "fp_ops_retired_by_type.scalar_mov",
+    "EventCode": "0x0a",
+    "BriefDescription": "Retired scalar floating-point MOV ops.",
+    "UMask": "0x0a"
+  },
+  {
+    "EventName": "fp_ops_retired_by_type.scalar_shuffle",
+    "EventCode": "0x0a",
+    "BriefDescription": "Retired scalar floating-point shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).",
+    "UMask": "0x0b"
+  },
+  {
+    "EventName": "fp_ops_retired_by_type.scalar_bfloat",
+    "EventCode": "0x0a",
+    "BriefDescription": "Retired scalar floating-point bfloat ops.",
+    "UMask": "0x0c"
+  },
+  {
+    "EventName": "fp_ops_retired_by_type.scalar_logical",
+    "EventCode": "0x0a",
+    "BriefDescription": "Retired scalar floating-point logical ops.",
+    "UMask": "0x0d"
+  },
   {
     "EventName": "fp_ops_retired_by_type.scalar_other",
     "EventCode": "0x0a",
@@ -245,12 +269,24 @@
     "BriefDescription": "Retired vector floating-point blend ops.",
     "UMask": "0x90"
   },
+  {
+    "EventName": "fp_ops_retired_by_type.vector_mov",
+    "EventCode": "0x0a",
+    "BriefDescription": "Retired vector floating-point MOV ops.",
+    "UMask": "0xa0"
+  },
   {
     "EventName": "fp_ops_retired_by_type.vector_shuffle",
     "EventCode": "0x0a",
     "BriefDescription": "Retired vector floating-point shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).",
     "UMask": "0xb0"
   },
+  {
+    "EventName": "fp_ops_retired_by_type.vector_bfloat",
+    "EventCode": "0x0a",
+    "BriefDescription": "Retired vector floating-point bfloat ops.",
+    "UMask": "0xc0"
+  },
   {
     "EventName": "fp_ops_retired_by_type.vector_logical",
     "EventCode": "0x0a",
@@ -278,7 +314,7 @@
   {
     "EventName": "sse_avx_ops_retired.mmx_add",
     "EventCode": "0x0b",
-    "BriefDescription": "Retired MMX integer add.",
+    "BriefDescription": "Retired MMX integer add ops.",
     "UMask": "0x01"
   },
   {
@@ -299,16 +335,34 @@
     "BriefDescription": "Retired MMX integer multiply-accumulate ops.",
     "UMask": "0x04"
   },
+  {
+    "EventName": "sse_avx_ops_retired.mmx_aes",
+    "EventCode": "0x0b",
+    "BriefDescription": "Retired MMX integer AES ops.",
+    "UMask": "0x05"
+  },
+  {
+    "EventName": "sse_avx_ops_retired.mmx_sha",
+    "EventCode": "0x0b",
+    "BriefDescription": "Retired MMX integer SHA ops.",
+    "UMask": "0x06"
+  },
   {
     "EventName": "sse_avx_ops_retired.mmx_cmp",
     "EventCode": "0x0b",
     "BriefDescription": "Retired MMX integer compare ops.",
     "UMask": "0x07"
   },
+  {
+    "EventName": "sse_avx_ops_retired.mmx_cvt",
+    "EventCode": "0x0b",
+    "BriefDescription": "Retired MMX integer convert or pack ops.",
+    "UMask": "0x08"
+  },
   {
     "EventName": "sse_avx_ops_retired.mmx_shift",
     "EventCode": "0x0b",
-    "BriefDescription": "Retired MMX integer shift ops.",
+    "BriefDescription": "Retired MMX integer shift or rotate ops.",
     "UMask": "0x09"
   },
   {
@@ -323,10 +377,17 @@
     "BriefDescription": "Retired MMX integer shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).",
     "UMask": "0x0b"
   },
+  {
+    "EventName": "sse_avx_ops_retired.mmx_vnni",
+    "EventCode": "0x0b",
+    "BriefDescription": "Retired MMX integer VNNI ops.",
+    "UMask": "0x0c"
+  },
   {
     "EventName": "sse_avx_ops_retired.mmx_pack",
     "EventCode": "0x0b",
-    "BriefDescription": "Retired MMX integer pack ops.",
+    "BriefDescription": "This event is deprecated. Refer to new event sse_avx_ops_retired.mmx_vnni",
+    "Deprecated": "1",
     "UMask": "0x0c"
   },
   {
@@ -389,16 +450,23 @@
     "BriefDescription": "Retired SSE and AVX integer compare ops.",
     "UMask": "0x70"
   },
+  {
+    "EventName": "sse_avx_ops_retired.sse_avx_cvt",
+    "EventCode": "0x0b",
+    "BriefDescription": "Retired SSE and AVX integer convert or pack ops.",
+    "UMask": "0x80"
+  },
   {
     "EventName": "sse_avx_ops_retired.sse_avx_clm",
     "EventCode": "0x0b",
-    "BriefDescription": "Retired SSE and AVX integer CLM ops.",
+    "BriefDescription": "This event is deprecated. Refer to new event sse_avx_ops_retired.sse_avx_cvt",
+    "Deprecated": "1",
     "UMask": "0x80"
   },
   {
     "EventName": "sse_avx_ops_retired.sse_avx_shift",
     "EventCode": "0x0b",
-    "BriefDescription": "Retired SSE and AVX integer shift ops.",
+    "BriefDescription": "Retired SSE and AVX integer shift or rotate ops.",
     "UMask": "0x90"
   },
   {
@@ -413,10 +481,17 @@
     "BriefDescription": "Retired SSE and AVX integer shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).",
     "UMask": "0xb0"
   },
+  {
+    "EventName": "sse_avx_ops_retired.sse_avx_vnni",
+    "EventCode": "0x0b",
+    "BriefDescription": "Retired SSE and AVX integer VNNI ops.",
+    "UMask": "0xc0"
+  },
   {
     "EventName": "sse_avx_ops_retired.sse_avx_pack",
     "EventCode": "0x0b",
-    "BriefDescription": "Retired SSE and AVX integer pack ops.",
+    "BriefDescription": "This event is deprecated. Refer to new event sse_avx_ops_retired.sse_avx_vnni",
+    "Deprecated": "1",
     "UMask": "0xc0"
   },
   {
@@ -497,12 +572,24 @@
     "BriefDescription": "Retired 128-bit packed floating-point blend ops.",
     "UMask": "0x09"
   },
+  {
+    "EventName": "fp_pack_ops_retired.fp128_mov",
+    "EventCode": "0x0c",
+    "BriefDescription": "Retired 128-bit packed floating-point MOV ops.",
+    "UMask": "0x0a"
+  },
   {
     "EventName": "fp_pack_ops_retired.fp128_shuffle",
     "EventCode": "0x0c",
     "BriefDescription": "Retired 128-bit packed floating-point shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).",
     "UMask": "0x0b"
   },
+  {
+    "EventName": "fp_pack_ops_retired.fp128_bfloat",
+    "EventCode": "0x0c",
+    "BriefDescription": "Retired 128-bit packed floating-point bfloat ops.",
+    "UMask": "0x0c"
+  },
   {
     "EventName": "fp_pack_ops_retired.fp128_logical",
     "EventCode": "0x0c",
@@ -575,12 +662,24 @@
     "BriefDescription": "Retired 256-bit packed floating-point blend ops.",
     "UMask": "0x90"
   },
+  {
+    "EventName": "fp_pack_ops_retired.fp256_mov",
+    "EventCode": "0x0c",
+    "BriefDescription": "Retired 256-bit packed floating-point MOV ops.",
+    "UMask": "0xa0"
+  },
   {
     "EventName": "fp_pack_ops_retired.fp256_shuffle",
     "EventCode": "0x0c",
     "BriefDescription": "Retired 256-bit packed floating-point shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).",
     "UMask": "0xb0"
   },
+  {
+    "EventName": "fp_pack_ops_retired.fp256_bfloat",
+    "EventCode": "0x0c",
+    "BriefDescription": "Retired 256-bit packed floating-point bfloat ops.",
+    "UMask": "0xc0"
+  },
   {
     "EventName": "fp_pack_ops_retired.fp256_logical",
     "EventCode": "0x0c",
@@ -647,16 +746,23 @@
     "BriefDescription": "Retired 128-bit packed integer compare ops.",
     "UMask": "0x07"
   },
+  {
+    "EventName": "packed_int_op_type.int128_cvt",
+    "EventCode": "0x0d",
+    "BriefDescription": "Retired 128-bit packed integer convert or pack ops.",
+    "UMask": "0x08"
+  },
   {
     "EventName": "packed_int_op_type.int128_clm",
     "EventCode": "0x0d",
-    "BriefDescription": "Retired 128-bit packed integer CLM ops.",
+    "BriefDescription": "This event is deprecated. Refer to new event packed_int_op_type.int128_cvt",
+    "Deprecated": "1",
     "UMask": "0x08"
   },
   {
     "EventName": "packed_int_op_type.int128_shift",
     "EventCode": "0x0d",
-    "BriefDescription": "Retired 128-bit packed integer shift ops.",
+    "BriefDescription": "Retired 128-bit packed integer shift or rotate ops.",
     "UMask": "0x09"
   },
   {
@@ -671,10 +777,17 @@
     "BriefDescription": "Retired 128-bit packed integer shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).",
     "UMask": "0x0b"
   },
+  {
+    "EventName": "packed_int_op_type.int128_vnni",
+    "EventCode": "0x0d",
+    "BriefDescription": "Retired 128-bit packed integer VNNI ops.",
+    "UMask": "0x0c"
+  },
   {
     "EventName": "packed_int_op_type.int128_pack",
     "EventCode": "0x0d",
-    "BriefDescription": "Retired 128-bit packed integer pack ops.",
+    "BriefDescription": "This event is deprecated. Refer to new event packed_int_op_type.int128_vnni",
+    "Deprecated": "1",
     "UMask": "0x0c"
   },
   {
@@ -719,16 +832,34 @@
     "BriefDescription": "Retired 256-bit packed integer multiply-accumulate ops.",
     "UMask": "0x40"
   },
+  {
+    "EventName": "packed_int_op_type.int256_aes",
+    "EventCode": "0x0d",
+    "BriefDescription": "Retired 256-bit packed integer AES ops.",
+    "UMask": "0x50"
+  },
+  {
+    "EventName": "packed_int_op_type.int256_sha",
+    "EventCode": "0x0d",
+    "BriefDescription": "Retired 256-bit packed integer SHA ops.",
+    "UMask": "0x60"
+  },
   {
     "EventName": "packed_int_op_type.int256_cmp",
     "EventCode": "0x0d",
     "BriefDescription": "Retired 256-bit packed integer compare ops.",
     "UMask": "0x70"
   },
+  {
+    "EventName": "packed_int_op_type.int256_cvt",
+    "EventCode": "0x0d",
+    "BriefDescription": "Retired 256-bit packed integer convert or pack ops.",
+    "UMask": "0x80"
+  },
   {
     "EventName": "packed_int_op_type.int256_shift",
     "EventCode": "0x0d",
-    "BriefDescription": "Retired 256-bit packed integer shift ops.",
+    "BriefDescription": "Retired 256-bit packed integer shift or rotate ops.",
     "UMask": "0x90"
   },
   {
@@ -743,10 +874,17 @@
     "BriefDescription": "Retired 256-bit packed integer shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).",
     "UMask": "0xb0"
   },
+  {
+    "EventName": "packed_int_op_type.int256_vnni",
+    "EventCode": "0x0d",
+    "BriefDescription": "Retired 256-bit packed integer VNNI ops.",
+    "UMask": "0xc0"
+  },
   {
     "EventName": "packed_int_op_type.int256_pack",
     "EventCode": "0x0d",
-    "BriefDescription": "Retired 256-bit packed integer pack ops.",
+    "BriefDescription": "This event is deprecated. Refer to new event packed_int_op_type.int256_vnni",
+    "Deprecated": "1",
     "UMask": "0xc0"
   },
   {
diff --git a/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json b/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json
index 06bbaea159259..b1994539ece82 100644
--- a/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json
+++ b/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json
@@ -8,9 +8,15 @@
   {
     "EventName": "ls_locks.bus_lock",
     "EventCode": "0x25",
-    "BriefDescription": "Retired Lock instructions which caused a bus lock.",
+    "BriefDescription": "Retired lock instructions which caused a bus lock.",
     "UMask": "0x01"
   },
+  {
+    "EventName": "ls_locks.all",
+    "EventCode": "0x25",
+    "BriefDescription": "Retired lock instructions of all types.",
+    "UMask": "0x1f"
+  },
   {
     "EventName": "ls_ret_cl_flush",
     "EventCode": "0x26",
diff --git a/tools/perf/pmu-events/arch/x86/amdzen6/floating-point.json b/tools/perf/pmu-events/arch/x86/amdzen6/floating-point.json
index 03cb039434dee..71b883776f37b 100644
--- a/tools/perf/pmu-events/arch/x86/amdzen6/floating-point.json
+++ b/tools/perf/pmu-events/arch/x86/amdzen6/floating-point.json
@@ -212,7 +212,7 @@
   {
     "EventName": "fp_ops_ret_by_type.scalar_logical",
     "EventCode": "0x0a",
-    "BriefDescription": "Retired scalar floating-point move uops.",
+    "BriefDescription": "Retired scalar floating-point logical uops.",
     "UMask": "0x0d"
   },
   {
@@ -665,6 +665,12 @@
     "BriefDescription": "Retired 256-bit packed floating-point shuffle uops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).",
     "UMask": "0xb0"
   },
+  {
+    "EventName": "fp_pack_ops_ret.fp256_bfloat",
+    "EventCode": "0x0c",
+    "BriefDescription": "Retired 256-bit packed floating-point bfloat uops.",
+    "UMask": "0xc0"
+  },
   {
     "EventName": "fp_pack_ops_ret.fp256_logical",
     "EventCode": "0x0c",
@@ -758,7 +764,7 @@
   {
     "EventName": "fp_pack_int_ops_ret.int128_vnni",
     "EventCode": "0x0d",
-    "BriefDescription": "Retired 128-bit packed integer VNNI ops.",
+    "BriefDescription": "Retired 128-bit packed integer VNNI uops.",
     "UMask": "0x0c"
   },
   {
@@ -803,12 +809,30 @@
     "BriefDescription": "Retired 256-bit packed integer multiply-accumulate uops.",
     "UMask": "0x40"
   },
+  {
+    "EventName": "fp_pack_int_ops_ret.int256_aes",
+    "EventCode": "0x0d",
+    "BriefDescription": "Retired 256-bit packed integer AES uops.",
+    "UMask": "0x50"
+  },
+  {
+    "EventName": "fp_pack_int_ops_ret.int256_sha",
+    "EventCode": "0x0d",
+    "BriefDescription": "Retired 256-bit packed integer SHA uops.",
+    "UMask": "0x60"
+  },
   {
     "EventName": "fp_pack_int_ops_ret.int256_cmp",
     "EventCode": "0x0d",
     "BriefDescription": "Retired 256-bit packed integer compare uops.",
     "UMask": "0x70"
   },
+  {
+    "EventName": "fp_pack_int_ops_ret.int256_cvt",
+    "EventCode": "0x0d",
+    "BriefDescription": "Retired 256-bit packed integer convert or pack uops.",
+    "UMask": "0x80"
+  },
   {
     "EventName": "fp_pack_int_ops_ret.int256_shift",
     "EventCode": "0x0d",
@@ -1083,19 +1107,19 @@
     "EventName": "fp_nsq_read_stalls.fp_prf",
     "EventCode": "0x13",
     "BriefDescription": "Cycles when reads of the NSQ and writes to the floating-point or SIMD schedulers are stalled due to insufficient free physical register file (FP-PRF) entries.",
-    "UMask": "0x0e"
+    "UMask": "0x02"
   },
   {
     "EventName": "fp_nsq_read_stalls.k_prf",
     "EventCode": "0x13",
     "BriefDescription": "Cycles when reads of the NSQ and writes to the floating-point or SIMD schedulers are stalled due to insufficient free mask physical register file (K-PRF) entries.",
-    "UMask": "0x0e"
+    "UMask": "0x04"
   },
   {
     "EventName": "fp_nsq_read_stalls.fp_sq",
     "EventCode": "0x13",
     "BriefDescription": "Cycles when reads of the NSQ and writes to the floating-point or SIMD schedulers are stalled due to insufficient free scheduler entries.",
-    "UMask": "0x0e"
+    "UMask": "0x08"
   },
   {
     "EventName": "fp_nsq_read_stalls.all",
diff --git a/tools/perf/pmu-events/arch/x86/arrowlake/arl-metrics.json b/tools/perf/pmu-events/arch/x86/arrowlake/arl-metrics.json
index 4f1f77404943f..52db2caafc0aa 100644
--- a/tools/perf/pmu-events/arch/x86/arrowlake/arl-metrics.json
+++ b/tools/perf/pmu-events/arch/x86/arrowlake/arl-metrics.json
@@ -823,16 +823,16 @@
     },
     {
         "BriefDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + tma_memory_bound * (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_fb_full / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_miss_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency + tma_uc_bound)) + tma_memory_bound * (tma_l3_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_fb_full / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)))",
         "MetricGroup": "BvMB;Mem;MemoryBW;Offcore;tma_issueBW",
         "MetricName": "tma_bottleneck_data_cache_memory_bandwidth",
         "MetricThreshold": "tma_bottleneck_data_cache_memory_bandwidth > 20",
-        "PublicDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks. Related metrics: tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full",
+        "PublicDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks. Related metrics: tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_uc_bound",
         "Unit": "cpu_core"
     },
     {
         "BriefDescription": "Total pipeline cost of external Memory- or Cache-Latency related bottlenecks",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + tma_memory_bound * (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l1_latency_dependency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l1_latency_capacity / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_lock_latency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_split_loads / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_split_stores / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)) + tma_memory_bound * (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_store_latency / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_miss_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency + tma_uc_bound)) + tma_memory_bound * (tma_l3_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * tma_l2_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_l1_latency_capacity / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_lock_latency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_split_loads / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_split_stores / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)) + tma_memory_bound * (tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_store_latency / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
         "MetricGroup": "BvML;Mem;MemoryLat;Offcore;tma_issueLat",
         "MetricName": "tma_bottleneck_data_cache_memory_latency",
         "MetricThreshold": "tma_bottleneck_data_cache_memory_latency > 20",
@@ -858,7 +858,7 @@
     },
     {
         "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_load / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_dtlb_load / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
         "MetricGroup": "BvMT;Mem;MemoryTLB;Offcore;tma_issueTLB",
         "MetricName": "tma_bottleneck_memory_data_tlbs",
         "MetricThreshold": "tma_bottleneck_memory_data_tlbs > 20",
@@ -867,7 +867,7 @@
     },
     {
         "BriefDescription": "Total pipeline cost of Memory Synchronization related bottlenecks (data transfers and coherency updates across processors)",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) * (tma_contested_accesses + tma_data_sharing) / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full) + tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) * tma_false_sharing / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores - tma_store_latency)) + tma_machine_clears * (1 - tma_other_nukes / tma_other_nukes))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) * (tma_contested_accesses + tma_data_sharing) / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full) + tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) * tma_false_sharing / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores - tma_store_latency)) + tma_machine_clears * (1 - tma_other_nukes / tma_other_nukes))",
         "MetricGroup": "BvMS;LockCont;Mem;Offcore;tma_issueSyncxn",
         "MetricName": "tma_bottleneck_memory_synchronization",
         "MetricThreshold": "tma_bottleneck_memory_synchronization > 10",
@@ -1015,7 +1015,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to retired misprediction by non-taken conditional branches.",
-        "MetricExpr": "cpu_core@BR_MISP_RETIRED.COND_NTAKEN_COST@ * cpu_core@BR_MISP_RETIRED.COND_NTAKEN_COST@R / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@BR_MISP_RETIRED.COND_NTAKEN@ / cpu_core@BR_MISP_RETIRED.ALL_BRANCHES@ * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_cond_nt_mispredicts",
         "MetricThreshold": "tma_cond_nt_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -1024,7 +1024,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to misprediction by backward-taken conditional branches.",
-        "MetricExpr": "cpu_core@BR_MISP_RETIRED.COND_TAKEN_BWD_COST@ * cpu_core@BR_MISP_RETIRED.COND_TAKEN_BWD_COST@R / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@BR_MISP_RETIRED.COND_TAKEN_BWD@ / cpu_core@BR_MISP_RETIRED.ALL_BRANCHES@ * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_cond_tk_bwd_mispredicts",
         "MetricThreshold": "tma_cond_tk_bwd_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -1033,7 +1033,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to misprediction by forward-taken conditional branches.",
-        "MetricExpr": "cpu_core@BR_MISP_RETIRED.COND_TAKEN_FWD_COST@ * cpu_core@BR_MISP_RETIRED.COND_TAKEN_FWD_COST@R / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@BR_MISP_RETIRED.COND_TAKEN_FWD@ / cpu_core@BR_MISP_RETIRED.ALL_BRANCHES@ * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_cond_tk_fwd_mispredicts",
         "MetricThreshold": "tma_cond_tk_fwd_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -1046,7 +1046,7 @@
         "MetricGroup": "BvMS;DataSharing;LockCont;Offcore;Snoop;TopdownL4;tma_L4_group;tma_issueSyncxn;tma_l3_bound_group",
         "MetricName": "tma_contested_accesses",
         "MetricThreshold": "tma_contested_accesses > 0.05 & (tma_l3_bound > 0.05 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS. Related metrics: tma_bottleneck_memory_synchronization, tma_data_sharing, tma_false_sharing, tma_machine_clears, tma_remote_cache",
+        "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L2_MISS_RETIRED.L3.SAME_CBB.HIT_ESF.SNP_MISS;MEM_LOAD_L2_MISS_RETIRED.OTHER_MODULE_L2.SAME_CBB.HIT_M.SNP_FWD. Related metrics: tma_bottleneck_memory_synchronization, tma_data_sharing, tma_false_sharing, tma_machine_clears, tma_remote_cache",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -1062,13 +1062,13 @@
         "Unit": "cpu_core"
     },
     {
-        "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+        "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to L3 data-sharing accesses",
         "MetricConstraint": "NO_GROUP_EVENTS",
         "MetricExpr": "(cpu_core@MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD@ * min(cpu_core@MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD@R, 24 * tma_info_system_core_frequency) + cpu_core@MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD@ * min(cpu_core@MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD@R, 25 * tma_info_system_core_frequency)) * (1 + cpu_core@MEM_LOAD_RETIRED.FB_HIT@ / cpu_core@MEM_LOAD_RETIRED.L1_MISS@ / 2) / tma_info_thread_clks",
         "MetricGroup": "BvMS;Offcore;Snoop;TopdownL4;tma_L4_group;tma_issueSyncxn;tma_l3_bound_group",
         "MetricName": "tma_data_sharing",
         "MetricThreshold": "tma_data_sharing > 0.05 & (tma_l3_bound > 0.05 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD. Related metrics: tma_bottleneck_memory_synchronization, tma_contested_accesses, tma_false_sharing, tma_machine_clears, tma_remote_cache",
+        "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to L3 data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L2_MISS_RETIRED.L3.SAME_CBB.HIT_ESF.SNP_HIT_NO_FWD. Related metrics: tma_bottleneck_memory_synchronization, tma_contested_accesses, tma_false_sharing, tma_machine_clears, tma_remote_cache",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -1082,19 +1082,9 @@
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
-    {
-        "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
-        "MetricExpr": "cpu_core@MEMORY_STALLS.MEM@ / tma_info_thread_clks",
-        "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_L3_group;tma_memory_bound_group",
-        "MetricName": "tma_dram_bound",
-        "MetricThreshold": "tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)",
-        "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS",
-        "ScaleUnit": "100%",
-        "Unit": "cpu_core"
-    },
     {
         "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
-        "MetricExpr": "(cpu_core@IDQ.DSB_UOPS\\,cmask\\=0x8\\,inv\\=0x1@ / 2 + cpu_core@IDQ.DSB_UOPS@ / (cpu_core@IDQ.DSB_UOPS@ + cpu_core@IDQ.MITE_UOPS@) * (cpu_core@IDQ_BUBBLES.STARVATION_CYCLES@ - cpu_core@IDQ_BUBBLES.FETCH_LATENCY@)) / tma_info_thread_clks",
+        "MetricExpr": "(cpu_core@IDQ.DSB_UOPS\\,cmask\\=0x8\\,inv\\=0x1@ * max(0, 8 - tma_info_pipeline_fetch_dsb) / 8 + cpu_core@IDQ.DSB_UOPS@ / (cpu_core@IDQ.DSB_UOPS@ + cpu_core@IDQ.MITE_UOPS@) * (cpu_core@IDQ_BUBBLES.STARVATION_CYCLES@ - cpu_core@IDQ_BUBBLES.FETCH_LATENCY@)) / tma_info_thread_clks",
         "MetricGroup": "DSB;FetchBW;TopdownL3;tma_L3_group;tma_fetch_bandwidth_group",
         "MetricName": "tma_dsb",
         "MetricThreshold": "tma_dsb > 0.15 & tma_fetch_bandwidth > 0.2",
@@ -1138,7 +1128,7 @@
         "MetricGroup": "BvMS;DataSharing;LockCont;Offcore;Snoop;TopdownL4;tma_L4_group;tma_issueSyncxn;tma_store_bound_group",
         "MetricName": "tma_false_sharing",
         "MetricThreshold": "tma_false_sharing > 0.05 & (tma_store_bound > 0.2 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM. Related metrics: tma_bottleneck_memory_synchronization, tma_contested_accesses, tma_data_sharing, tma_machine_clears, tma_remote_cache",
+        "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: L2_REQUEST.RFO_HIT_XSNP_HIT_FWD. Related metrics: tma_bottleneck_memory_synchronization, tma_contested_accesses, tma_data_sharing, tma_machine_clears, tma_remote_cache",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -1148,7 +1138,7 @@
         "MetricGroup": "BvMB;MemoryBW;TopdownL4;tma_L4_group;tma_issueBW;tma_issueSL;tma_issueSmSt;tma_l1_bound_group",
         "MetricName": "tma_fb_full",
         "MetricThreshold": "tma_fb_full > 0.3",
-        "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_store_latency, tma_streaming_stores",
+        "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_store_latency, tma_streaming_stores, tma_uc_bound",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -1245,7 +1235,7 @@
     },
     {
         "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors",
-        "MetricExpr": "cpu_core@FP_ARITH_OPS_RETIRED.VECTOR\\,umask\\=0x30@ / (tma_retiring * tma_info_thread_slots)",
+        "MetricExpr": "cpu_core@FP_ARITH_OPS_RETIRED.VECTOR_256B@ / (tma_retiring * tma_info_thread_slots)",
         "MetricGroup": "Compute;Flops;TopdownL5;tma_L5_group;tma_fp_vector_group;tma_issue2P",
         "MetricName": "tma_fp_vector_256b",
         "MetricThreshold": "tma_fp_vector_256b > 0.1 & (tma_fp_vector > 0.1 & (tma_fp_arith > 0.2 & tma_light_operations > 0.6))",
@@ -1288,7 +1278,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses",
-        "MetricExpr": "cpu_core@ICACHE_DATA.STALLS@ / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@ICACHE_DATA.STALLS@ / tma_info_thread_clks + tma_isb_full",
         "MetricGroup": "BigFootprint;BvBC;FetchLat;IcMiss;TopdownL3;tma_L3_group;tma_fetch_latency_group",
         "MetricName": "tma_icache_misses",
         "MetricThreshold": "tma_icache_misses > 0.05 & (tma_fetch_latency > 0.1 & tma_frontend_bound > 0.15)",
@@ -1298,7 +1288,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to retired misprediction by indirect CALL instructions.",
-        "MetricExpr": "cpu_core@BR_MISP_RETIRED.INDIRECT_CALL_COST@ * cpu_core@BR_MISP_RETIRED.INDIRECT_CALL_COST@R / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@BR_MISP_RETIRED.INDIRECT_CALL@ / cpu_core@BR_MISP_RETIRED.ALL_BRANCHES@ * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_ind_call_mispredicts",
         "MetricThreshold": "tma_ind_call_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -1307,7 +1297,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to retired misprediction by indirect JMP instructions.",
-        "MetricExpr": "max((cpu_core@BR_MISP_RETIRED.INDIRECT_COST@ * cpu_core@BR_MISP_RETIRED.INDIRECT_COST@R - cpu_core@BR_MISP_RETIRED.INDIRECT_CALL_COST@ * cpu_core@BR_MISP_RETIRED.INDIRECT_CALL_COST@R) / tma_info_thread_clks, 0)",
+        "MetricExpr": "max((cpu_core@BR_MISP_RETIRED.INDIRECT@ - cpu_core@BR_MISP_RETIRED.INDIRECT_CALL@) / cpu_core@BR_MISP_RETIRED.ALL_BRANCHES@ * tma_branch_mispredicts, 0)",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_ind_jump_mispredicts",
         "MetricThreshold": "tma_ind_jump_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -1477,7 +1467,7 @@
     },
     {
         "BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)",
-        "MetricExpr": "cpu_core@IDQ.DSB_UOPS@ / cpu_core@UOPS_ISSUED.ANY@",
+        "MetricExpr": "cpu_core@IDQ.DSB_UOPS@ / (cpu_core@IDQ.DSB_UOPS@ + cpu_core@LSD.UOPS@ + cpu_core@IDQ.MITE_UOPS@ + cpu_core@IDQ.MS_UOPS@)",
         "MetricGroup": "DSB;Fed;FetchBW;tma_issueFB",
         "MetricName": "tma_info_frontend_dsb_coverage",
         "MetricThreshold": "tma_info_frontend_dsb_coverage < 0.7 & tma_info_thread_ipc / 8 > 0.35",
@@ -1544,7 +1534,7 @@
     },
     {
         "BriefDescription": "Fraction of Uops delivered by the LSD (Loop Stream Detector; aka Loop Cache)",
-        "MetricExpr": "cpu_core@LSD.UOPS@ / cpu_core@UOPS_ISSUED.ANY@",
+        "MetricExpr": "cpu_core@LSD.UOPS@ / (cpu_core@IDQ.DSB_UOPS@ + cpu_core@LSD.UOPS@ + cpu_core@IDQ.MITE_UOPS@ + cpu_core@IDQ.MS_UOPS@)",
         "MetricGroup": "Fed;LSD",
         "MetricName": "tma_info_frontend_lsd_coverage",
         "Unit": "cpu_core"
@@ -1703,6 +1693,15 @@
         "PublicDescription": "Instructions per taken branch. Related metrics: tma_dsb_switches, tma_fetch_bandwidth, tma_info_botlnk_l2_dsb_bandwidth, tma_info_botlnk_l2_dsb_misses, tma_info_frontend_dsb_coverage, tma_lcp",
         "Unit": "cpu_core"
     },
+    {
+        "BriefDescription": "AVX preserve/restore assists per kilo instruction",
+        "MetricExpr": "1e3 * cpu_core@ASSISTS.SSE_AVX_MIX@ / cpu_core@INST_RETIRED.ANY@",
+        "MetricGroup": "tma_issueMV",
+        "MetricName": "tma_info_inst_mix_vectormixpki",
+        "MetricThreshold": "tma_info_inst_mix_vectormixpki > 0.05",
+        "PublicDescription": "AVX preserve/restore assists per kilo instruction. Related metrics: tma_mixing_vectors, tma_ms_switches",
+        "Unit": "cpu_core"
+    },
     {
         "BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)",
         "MetricExpr": "1e3 * cpu_core@MEM_LOAD_RETIRED.FB_HIT@ / cpu_core@INST_RETIRED.ANY@",
@@ -1969,6 +1968,13 @@
         "MetricName": "tma_info_pipeline_retire",
         "Unit": "cpu_core"
     },
+    {
+        "BriefDescription": "Total percent of cycles without actually retired uops",
+        "MetricExpr": "cpu_core@UOPS_RETIRED.STALLS@ / tma_info_thread_clks",
+        "MetricGroup": "Pipeline;Ret",
+        "MetricName": "tma_info_pipeline_retire_stall",
+        "Unit": "cpu_core"
+    },
     {
         "BriefDescription": "Estimated fraction of retirement-cycles dealing with repeat instructions",
         "MetricExpr": "cpu_core@INST_RETIRED.REP_ITERATION@ / cpu_core@UOPS_RETIRED.SLOTS\\,cmask\\=1@",
@@ -2011,7 +2017,7 @@
         "MetricExpr": "32 * UNC_M_TOTAL_DATA / 1e9 / tma_info_system_time",
         "MetricGroup": "HPC;MemOffcore;MemoryBW;SoC;tma_issueBW",
         "MetricName": "tma_info_system_dram_bw_use",
-        "PublicDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]. Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_mem_bandwidth, tma_sq_full",
+        "PublicDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]. Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_mem_bandwidth, tma_sq_full, tma_uc_bound",
         "Unit": "cpu_core"
     },
     {
@@ -2082,13 +2088,6 @@
         "MetricName": "tma_info_system_turbo_utilization",
         "Unit": "cpu_core"
     },
-    {
-        "BriefDescription": "Measured Average Uncore Frequency for the SoC [GHz]",
-        "MetricExpr": "tma_info_system_socket_clks / 1e9 / tma_info_system_time",
-        "MetricGroup": "SoC",
-        "MetricName": "tma_info_system_uncore_frequency",
-        "Unit": "cpu_core"
-    },
     {
         "BriefDescription": "Per-Logical Processor actual clocks when the Logical Processor is active.",
         "MetricExpr": "cpu_core@CPU_CLK_UNHALTED.THREAD@",
@@ -2180,9 +2179,18 @@
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
+    {
+        "BriefDescription": "This metric represents fraction of cycles Icache tag lookup was stalled due to Instruction Stream Buffer (ISB) being full",
+        "MetricExpr": "(cpu_core@ICACHE_TAG.STALLS_ISB@ + cpu_core@ICACHE_TAG.STALLS_INUSE@) / tma_info_thread_clks",
+        "MetricGroup": "TopdownL4;tma_L4_group;tma_icache_misses_group",
+        "MetricName": "tma_isb_full",
+        "MetricThreshold": "tma_isb_full > 0.05 & (tma_icache_misses > 0.05 & (tma_fetch_latency > 0.1 & tma_frontend_bound > 0.15))",
+        "ScaleUnit": "100%",
+        "Unit": "cpu_core"
+    },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
-        "MetricExpr": "cpu_core@ICACHE_TAG.STALLS@ / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@ICACHE_TAG.STALLS@ / tma_info_thread_clks - tma_isb_full",
         "MetricGroup": "BigFootprint;BvBC;FetchLat;MemoryTLB;TopdownL3;tma_L3_group;tma_fetch_latency_group",
         "MetricName": "tma_itlb_misses",
         "MetricThreshold": "tma_itlb_misses > 0.05 & (tma_fetch_latency > 0.1 & tma_frontend_bound > 0.15)",
@@ -2209,19 +2217,9 @@
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
-    {
-        "BriefDescription": "This metric ([SKL+] roughly; [LNL]) estimates fraction of cycles with demand load accesses that hit the L1D cache",
-        "MetricExpr": "4 * cpu_core@DEPENDENT_LOADS.ANY\\,cmask\\=1@ / tma_info_thread_clks",
-        "MetricGroup": "BvML;MemoryLat;TopdownL4;tma_L4_group;tma_l1_bound_group",
-        "MetricName": "tma_l1_latency_dependency",
-        "MetricThreshold": "tma_l1_latency_dependency > 0.1 & (tma_l1_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric ([SKL+] roughly; [LNL]) estimates fraction of cycles with demand load accesses that hit the L1D cache. The short latency of the L1D cache may be exposed in pointer-chasing memory access patterns as an example. Sample with: MEM_LOAD_RETIRED.L1_HIT",
-        "ScaleUnit": "100%",
-        "Unit": "cpu_core"
-    },
     {
         "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads",
-        "MetricExpr": "cpu_core@MEMORY_STALLS.L2@ / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@MEM_LOAD_RETIRED.L2_HIT@ / (cpu_core@MEM_LOAD_RETIRED.L2_HIT@ + cpu_core@MEM_LOAD_MISC_RETIRED.UC@) * cpu_core@MEMORY_STALLS.L2@ / tma_info_thread_clks",
         "MetricGroup": "BvML;CacheHits;MemoryBound;TmaL3mem;TopdownL3;tma_L3_group;tma_memory_bound_group",
         "MetricName": "tma_l2_bound",
         "MetricThreshold": "tma_l2_bound > 0.05 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)",
@@ -2245,7 +2243,7 @@
         "MetricGroup": "CacheHits;MemoryBound;TmaL3mem;TopdownL3;tma_L3_group;tma_memory_bound_group",
         "MetricName": "tma_l3_bound",
         "MetricThreshold": "tma_l3_bound > 0.05 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)",
-        "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core.  Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS",
+        "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core.  Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_L2_MISS_RETIRED.L3_HIT_SAME_CBB",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -2255,7 +2253,17 @@
         "MetricGroup": "BvML;MemoryLat;TopdownL4;tma_L4_group;tma_issueLat;tma_l3_bound_group",
         "MetricName": "tma_l3_hit_latency",
         "MetricThreshold": "tma_l3_hit_latency > 0.1 & (tma_l3_bound > 0.05 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric estimates fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited).  Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance.  Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS. Related metrics: tma_bottleneck_data_cache_memory_latency, tma_mem_latency",
+        "PublicDescription": "This metric estimates fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited).  Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance.  Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_L2_MISS_RETIRED.L3_HIT_SAME_CBB. Related metrics: tma_bottleneck_data_cache_memory_latency, tma_mem_latency",
+        "ScaleUnit": "100%",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+        "MetricExpr": "cpu_core@MEMORY_STALLS.MEM@ / tma_info_thread_clks",
+        "MetricGroup": "MemoryBound;Offcore;TmaL3mem;TopdownL3;tma_L3_group;tma_memory_bound_group",
+        "MetricName": "tma_l3_miss_bound",
+        "MetricThreshold": "tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)",
+        "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_L2_MISS_RETIRED.L3_MISS",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -2348,7 +2356,7 @@
     },
     {
         "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to LSD (Loop Stream Detector) unit",
-        "MetricExpr": "cpu_core@LSD.UOPS\\,cmask\\=0x8\\,inv\\=0x1@ / tma_info_thread_clks / 2",
+        "MetricExpr": "cpu_core@LSD.UOPS\\,cmask\\=0x8\\,inv\\=0x1@ * max(0, 8 - tma_info_pipeline_fetch_lsd) / 8 / tma_info_thread_clks",
         "MetricGroup": "FetchBW;LSD;TopdownL3;tma_L3_group;tma_fetch_bandwidth_group",
         "MetricName": "tma_lsd",
         "MetricThreshold": "tma_lsd > 0.15 & tma_fetch_bandwidth > 0.2",
@@ -2370,19 +2378,19 @@
     {
         "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM)",
         "MetricExpr": "min(cpu_core@CPU_CLK_UNHALTED.THREAD@, cpu_core@OFFCORE_REQUESTS_OUTSTANDING.DATA_RD\\,cmask\\=4@) / tma_info_thread_clks",
-        "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_dram_bound_group;tma_issueBW",
+        "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_issueBW;tma_l3_miss_bound_group",
         "MetricName": "tma_mem_bandwidth",
-        "MetricThreshold": "tma_mem_bandwidth > 0.2 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM).  The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_sq_full",
+        "MetricThreshold": "tma_mem_bandwidth > 0.2 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
+        "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM).  The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_sq_full, tma_uc_bound",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
     {
         "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory - DRAM ([SPR-HBM] and/or HBM)",
         "MetricExpr": "min(cpu_core@CPU_CLK_UNHALTED.THREAD@, cpu_core@OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD@) / tma_info_thread_clks - tma_mem_bandwidth",
-        "MetricGroup": "BvML;MemoryLat;Offcore;TopdownL4;tma_L4_group;tma_dram_bound_group;tma_issueLat",
+        "MetricGroup": "BvML;MemoryLat;Offcore;TopdownL4;tma_L4_group;tma_issueLat;tma_l3_miss_bound_group",
         "MetricName": "tma_mem_latency",
-        "MetricThreshold": "tma_mem_latency > 0.1 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
+        "MetricThreshold": "tma_mem_latency > 0.1 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
         "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory - DRAM ([SPR-HBM] and/or HBM).  This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_latency, tma_l3_hit_latency",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
@@ -2438,7 +2446,7 @@
     },
     {
         "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)",
-        "MetricExpr": "(cpu_core@IDQ.MITE_UOPS\\,cmask\\=0x8\\,inv\\=0x1@ / 2 + cpu_core@IDQ.MITE_UOPS@ / (cpu_core@IDQ.DSB_UOPS@ + cpu_core@IDQ.MITE_UOPS@) * (cpu_core@IDQ_BUBBLES.STARVATION_CYCLES@ - cpu_core@IDQ_BUBBLES.FETCH_LATENCY@)) / tma_info_thread_clks",
+        "MetricExpr": "(cpu_core@IDQ.MITE_UOPS\\,cmask\\=0x8\\,inv\\=0x1@ * max(0, 8 - tma_info_pipeline_fetch_mite) / 8 + cpu_core@IDQ.MITE_UOPS@ / (cpu_core@IDQ.DSB_UOPS@ + cpu_core@IDQ.MITE_UOPS@) * (cpu_core@IDQ_BUBBLES.STARVATION_CYCLES@ - cpu_core@IDQ_BUBBLES.FETCH_LATENCY@)) / tma_info_thread_clks",
         "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_L3_group;tma_fetch_bandwidth_group",
         "MetricName": "tma_mite",
         "MetricThreshold": "tma_mite > 0.1 & tma_fetch_bandwidth > 0.2",
@@ -2452,13 +2460,13 @@
         "MetricGroup": "TopdownL5;tma_L5_group;tma_issueMV;tma_ports_utilized_0_group",
         "MetricName": "tma_mixing_vectors",
         "MetricThreshold": "tma_mixing_vectors > 0.05",
-        "PublicDescription": "This metric estimates penalty in terms of percentage of([SKL+] injected blend uops out of all Uops Issued -- the Count Domain; [ADL+] cycles). Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic. Related metrics: tma_ms_switches",
+        "PublicDescription": "This metric estimates penalty in terms of percentage of([SKL+] injected blend uops out of all Uops Issued -- the Count Domain; [ADL+] cycles). Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic. Related metrics: tma_info_inst_mix_vectormixpki, tma_ms_switches",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
     {
         "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the Microcode Sequencer (MS) unit - see Microcode_Sequencer node for details.",
-        "MetricExpr": "cpu_core@IDQ.MS_CYCLES_ANY@ / tma_info_thread_clks / 1.8",
+        "MetricExpr": "cpu_core@IDQ.MS_CYCLES_ANY@ * (8 - cpu_core@IDQ.MS_UOPS@ / cpu_core@IDQ.MS_CYCLES_ANY@) / tma_info_thread_slots",
         "MetricGroup": "MicroSeq;TopdownL3;tma_L3_group;tma_fetch_bandwidth_group",
         "MetricName": "tma_ms",
         "MetricThreshold": "tma_ms > 0.05 & tma_fetch_bandwidth > 0.2",
@@ -2471,7 +2479,7 @@
         "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_L3_group;tma_fetch_latency_group;tma_issueMC;tma_issueMS;tma_issueMV;tma_issueSO",
         "MetricName": "tma_ms_switches",
         "MetricThreshold": "tma_ms_switches > 0.05 & (tma_fetch_latency > 0.1 & tma_frontend_bound > 0.15)",
-        "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES. Related metrics: tma_bottleneck_irregular_overhead, tma_clears_resteers, tma_l1_bound, tma_machine_clears, tma_microcode_sequencer, tma_mixing_vectors, tma_serializing_operation",
+        "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES. Related metrics: tma_bottleneck_irregular_overhead, tma_clears_resteers, tma_info_inst_mix_vectormixpki, tma_l1_bound, tma_machine_clears, tma_microcode_sequencer, tma_mixing_vectors, tma_serializing_operation",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -2589,7 +2597,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to retired misprediction by (indirect) RET instructions.",
-        "MetricExpr": "cpu_core@BR_MISP_RETIRED.RET_COST@ * cpu_core@BR_MISP_RETIRED.RET_COST@R / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@BR_MISP_RETIRED.RET@ / cpu_core@BR_MISP_RETIRED.ALL_BRANCHES@ * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_ret_mispredicts",
         "MetricThreshold": "tma_ret_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -2664,7 +2672,7 @@
         "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_issueBW;tma_l3_bound_group",
         "MetricName": "tma_sq_full",
         "MetricThreshold": "tma_sq_full > 0.3 & (tma_l3_bound > 0.05 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth",
+        "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_uc_bound",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -2679,8 +2687,8 @@
         "Unit": "cpu_core"
     },
     {
-        "BriefDescription": "This metric estimates clocks wasted due to loads blocked due to unknown store address (did not do memory disambiguation) or due to unknown store data",
-        "MetricExpr": "7 * cpu_core@LD_BLOCKS.STORE_EARLY\\,cmask\\=1@ / tma_info_thread_clks",
+        "BriefDescription": "This metric roughly estimates clocks wasted due to loads blocked due to unknown store address (did not do memory disambiguation) or due to unknown store data",
+        "MetricExpr": "cpu_core@LD_BLOCKS.STORE_EARLY\\,cmask\\=1@ / tma_info_thread_clks",
         "MetricGroup": "TopdownL4;tma_L4_group;tma_l1_bound_group",
         "MetricName": "tma_store_early_blk",
         "MetricThreshold": "tma_store_early_blk > 0.2",
@@ -2768,7 +2776,17 @@
         "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_issueSmSt;tma_store_bound_group",
         "MetricName": "tma_streaming_stores",
         "MetricThreshold": "tma_streaming_stores > 0.2 & (tma_store_bound > 0.2 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric estimates how often CPU was stalled  due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should Streaming stores be a bottleneck. Sample with: OCR.STREAMING_WR.ANY_RESPONSE. Related metrics: tma_fb_full",
+        "PublicDescription": "This metric estimates how often CPU was stalled  due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should Streaming stores be a bottleneck. Sample with: OMR.STREAMING_WR.ANY_RESPONSE. Related metrics: tma_fb_full",
+        "ScaleUnit": "100%",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "This metric roughly estimates fraction of cycles where the core's performance was likely hurt due to un-chacheable memory",
+        "MetricExpr": "cpu_core@MEM_LOAD_MISC_RETIRED.UC@ / (cpu_core@MEM_LOAD_RETIRED.L2_HIT@ + cpu_core@MEM_LOAD_MISC_RETIRED.UC@) * cpu_core@MEMORY_STALLS.L2@ / tma_info_thread_clks",
+        "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_issueBW;tma_l3_miss_bound_group",
+        "MetricName": "tma_uc_bound",
+        "MetricThreshold": "tma_uc_bound > 0.2 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
+        "PublicDescription": "This metric roughly estimates fraction of cycles where the core's performance was likely hurt due to un-chacheable memory. Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
diff --git a/tools/perf/pmu-events/arch/x86/arrowlake/cache.json b/tools/perf/pmu-events/arch/x86/arrowlake/cache.json
index fe6b9ad68f87f..142f62c595319 100644
--- a/tools/perf/pmu-events/arch/x86/arrowlake/cache.json
+++ b/tools/perf/pmu-events/arch/x86/arrowlake/cache.json
@@ -1,6 +1,6 @@
 [
     {
-        "BriefDescription": "Counts the number of request that were not accepted into the L2Q because the L2Q is FULL.",
+        "BriefDescription": "Counts the number of requests that were not accepted into the L2Q because the L2Q is FULL.",
         "Counter": "0,1,2,3,4,5,6,7",
         "EventCode": "0x31",
         "EventName": "CORE_REJECT_L2Q.ANY",
@@ -8,6 +8,15 @@
         "SampleAfterValue": "1000003",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of requests that were not accepted into the L2Q because the L2Q is FULL.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x31",
+        "EventName": "CORE_REJECT_L2Q.ANY",
+        "PublicDescription": "Counts the number of (demand and L1 prefetchers) core requests rejected by the L2Q due to a full or nearly full w condition which likely indicates back pressure from L2Q.  It also counts requests that would have gone directly to the XQ, but are rejected due to a full or nearly full condition, indicating back pressure from the IDI link.  The L2Q may also reject transactions  from a core to insure fairness between cores, or to delay a cores dirty eviction when the address conflicts incoming external snoops.  (Note that L2 prefetcher requests that are dropped are not counted by this event.) Counts on a per core basis.",
+        "SampleAfterValue": "200003",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of L1D cacheline (dirty) evictions caused by load misses, stores, and prefetches.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -310,6 +319,15 @@
         "SampleAfterValue": "1000003",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x30",
+        "EventName": "L2_REJECT_XQ.ANY",
+        "PublicDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition which likely indicates back pressure from the IDI link.  The XQ may reject transactions from the L2Q (non-cacheable requests), BBL (L2 misses) and WOB (L2 write-back victims).",
+        "SampleAfterValue": "200003",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of L2 Cache Accesses Counts the total number of L2 Cache Accesses - sum of hits, misses, rejects  front door requests for CRd/DRd/RFO/ItoM/L2 Prefetches only, per core event",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -1382,6 +1400,16 @@
         "UMask": "0x83",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of memory uops retired.  A single uop that performs both a load AND a store will be counted as 1, not 2 (e.g. ADD [mem], CONST)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xd0",
+        "EventName": "MEM_UOPS_RETIRED.ALL",
+        "SampleAfterValue": "200003",
+        "UMask": "0x83",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of load uops retired.",
         "Counter": "0,1,2,3,4,5,6,7",
diff --git a/tools/perf/pmu-events/arch/x86/arrowlake/floating-point.json b/tools/perf/pmu-events/arch/x86/arrowlake/floating-point.json
index c54fc201a6cae..8dc3a11350c54 100644
--- a/tools/perf/pmu-events/arch/x86/arrowlake/floating-point.json
+++ b/tools/perf/pmu-events/arch/x86/arrowlake/floating-point.json
@@ -510,6 +510,15 @@
         "UMask": "0x1f",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of uops executed on all floating point ports.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "FP_VINT_UOPS_EXECUTED.ALL",
+        "SampleAfterValue": "1000003",
+        "UMask": "0xf",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 0.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -519,6 +528,15 @@
         "UMask": "0x2",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 0.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "FP_VINT_UOPS_EXECUTED.P0",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 1.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -528,6 +546,15 @@
         "UMask": "0x4",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 1.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "FP_VINT_UOPS_EXECUTED.P1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 2.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -537,6 +564,15 @@
         "UMask": "0x8",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 2.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "FP_VINT_UOPS_EXECUTED.P2",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 3.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -555,6 +591,15 @@
         "UMask": "0x1e",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 0, 1, 2.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "FP_VINT_UOPS_EXECUTED.PRIMARY",
+        "SampleAfterValue": "1000003",
+        "UMask": "0xe",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of uops executed on floating point and vector integer store data port.",
         "Counter": "0,1,2,3,4,5,6,7",
diff --git a/tools/perf/pmu-events/arch/x86/arrowlake/memory.json b/tools/perf/pmu-events/arch/x86/arrowlake/memory.json
index 05cc46518232e..44922186c2b06 100644
--- a/tools/perf/pmu-events/arch/x86/arrowlake/memory.json
+++ b/tools/perf/pmu-events/arch/x86/arrowlake/memory.json
@@ -173,6 +173,15 @@
         "UMask": "0x2",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to request buffers full or lock in progress.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.WCB_FULL",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to request buffers full or lock in progress.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -182,6 +191,15 @@
         "UMask": "0x82",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to request buffers full or lock in progress.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.WCB_FULL_AT_RET",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x82",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of memory ordering machine clears triggered due to a snoop from an external agent. Does not count internally generated machine clears such as those due to disambiguations.",
         "Counter": "0,1,2,3,4,5,6,7",
diff --git a/tools/perf/pmu-events/arch/x86/arrowlake/metricgroups.json b/tools/perf/pmu-events/arch/x86/arrowlake/metricgroups.json
index 855585fe6fae1..9acbda612c25c 100644
--- a/tools/perf/pmu-events/arch/x86/arrowlake/metricgroups.json
+++ b/tools/perf/pmu-events/arch/x86/arrowlake/metricgroups.json
@@ -93,7 +93,6 @@
     "tma_code_stlb_miss_group": "Metrics contributing to tma_code_stlb_miss category",
     "tma_core_bound_group": "Metrics contributing to tma_core_bound category",
     "tma_divider_group": "Metrics contributing to tma_divider category",
-    "tma_dram_bound_group": "Metrics contributing to tma_dram_bound category",
     "tma_dtlb_load_group": "Metrics contributing to tma_dtlb_load category",
     "tma_dtlb_store_group": "Metrics contributing to tma_dtlb_store category",
     "tma_fetch_bandwidth_group": "Metrics contributing to tma_fetch_bandwidth category",
@@ -129,6 +128,7 @@
     "tma_l1_bound_group": "Metrics contributing to tma_l1_bound category",
     "tma_l2_bound_group": "Metrics contributing to tma_l2_bound category",
     "tma_l3_bound_group": "Metrics contributing to tma_l3_bound category",
+    "tma_l3_miss_bound_group": "Metrics contributing to tma_l3_miss_bound category",
     "tma_light_operations_group": "Metrics contributing to tma_light_operations category",
     "tma_load_op_utilization_group": "Metrics contributing to tma_load_op_utilization category",
     "tma_load_stlb_miss_group": "Metrics contributing to tma_load_stlb_miss category",
diff --git a/tools/perf/pmu-events/arch/x86/arrowlake/pipeline.json b/tools/perf/pmu-events/arch/x86/arrowlake/pipeline.json
index a0fd63cace220..1f1b5a88216a3 100644
--- a/tools/perf/pmu-events/arch/x86/arrowlake/pipeline.json
+++ b/tools/perf/pmu-events/arch/x86/arrowlake/pipeline.json
@@ -209,7 +209,6 @@
         "EventName": "BR_INST_RETIRED.COND_TAKEN_FWD",
         "PublicDescription": "Counts taken forward conditional branch instructions retired. Available PDIST counters: 0,1",
         "SampleAfterValue": "400009",
-        "UMask": "0x102",
         "Unit": "cpu_core"
     },
     {
@@ -608,7 +607,7 @@
         "EventName": "BR_MISP_RETIRED.COND_TAKEN_BWD_COST",
         "PublicDescription": "number of branch instructions retired that were mispredicted and taken backward. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch. Available PDIST counters: 0,1",
         "SampleAfterValue": "400009",
-        "UMask": "0x8001",
+        "UMask": "0x41",
         "Unit": "cpu_core"
     },
     {
@@ -637,7 +636,7 @@
         "EventName": "BR_MISP_RETIRED.COND_TAKEN_FWD_COST",
         "PublicDescription": "number of branch instructions retired that were mispredicted and taken forward. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch. Available PDIST counters: 0,1",
         "SampleAfterValue": "400009",
-        "UMask": "0x8002",
+        "UMask": "0x140",
         "Unit": "cpu_core"
     },
     {
@@ -773,11 +772,11 @@
         "Unit": "cpu_core"
     },
     {
-        "BriefDescription": "This event counts the number of mispredicted ret instructions retired. Non PEBS",
+        "BriefDescription": "This event counts the number of mispredicted ret instructions retired.",
         "Counter": "0,1,2,3,4,5,6,7,8,9",
         "EventCode": "0xc5",
         "EventName": "BR_MISP_RETIRED.RET",
-        "PublicDescription": "This is a non-precise version (that is, does not use PEBS) of the event that counts mispredicted return instructions retired. Available PDIST counters: 0,1",
+        "PublicDescription": "This event counts the number of mispredicted ret instructions retired. Available PDIST counters: 0,1",
         "SampleAfterValue": "100007",
         "UMask": "0x8",
         "Unit": "cpu_core"
@@ -1239,7 +1238,7 @@
         "Counter": "0,1,2,3,4,5,6,7,8,9",
         "EventCode": "0xc0",
         "EventName": "INST_RETIRED.REP_ITERATION",
-        "PublicDescription": "Number of iterations of Repeat (REP) string retired instructions such as MOVS, CMPS, and SCAS. Each has a byte, word, and doubleword version and string instructions can be repeated using a repetition prefix, REP, that allows their architectural execution to be repeated a number of times as specified by the RCX register. Note the number of iterations is implementation-dependent. Available PDIST counters: 0,1",
+        "PublicDescription": "Number of iterations of Repeat (REP) string retired instructions such as MOVS, CMPS, and SCAS. Each has a byte, word, and doubleword version and string instructions can be repeated using a repetition prefix, REP, that allows their architectural execution to be repeated a number of times as specified by the RCX register. Note: Since the number of iterations within a REP instruction can be significantly affected by fast strings, this event may vary run to run and not match the architectural number of iterations (specified by RCX) Available PDIST counters: 0,1",
         "SampleAfterValue": "2000003",
         "UMask": "0x8",
         "Unit": "cpu_core"
@@ -1326,6 +1325,15 @@
         "UMask": "0xff",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of uops executed on all Integer ports.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.ALL",
+        "SampleAfterValue": "1000003",
+        "UMask": "0xff",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of uops executed on a load port.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -1336,6 +1344,16 @@
         "UMask": "0x1",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of uops executed on a load port.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.LD",
+        "PublicDescription": "Counts the number of uops executed on a load port.  This event counts for integer uops even if the destination is FP/vector",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of uops executed on integer port 0.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -1345,6 +1363,15 @@
         "UMask": "0x8",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of uops executed on integer port 0.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.P0",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of uops executed on integer port 1.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -1354,6 +1381,15 @@
         "UMask": "0x10",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of uops executed on integer port 1.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.P1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x10",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of uops executed on integer port 2.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -1363,6 +1399,15 @@
         "UMask": "0x20",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of uops executed on integer port 2.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.P2",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x20",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of uops executed on integer port 3.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -1372,6 +1417,15 @@
         "UMask": "0x40",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of uops executed on integer port 3.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.P3",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x40",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of uops executed on integer port  0,1, 2, 3.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -1381,6 +1435,15 @@
         "UMask": "0x78",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of uops executed on integer port  0,1, 2, 3.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.PRIMARY",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x78",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of uops executed on a Store address port.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -1391,6 +1454,16 @@
         "UMask": "0x2",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of uops executed on a Store address port.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.STA",
+        "PublicDescription": "Counts the number of uops executed on a Store address port. This event counts integer uops even if the data source is FP/vector",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of uops executed on an integer store data and jump port.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -1400,6 +1473,15 @@
         "UMask": "0x4",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of uops executed on an integer store data and jump port.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.STD_JMP",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Number of vector integer instructions retired of 128-bit vector-width.",
         "Counter": "0,1,2,3,4,5,6,7,8,9",
@@ -1691,6 +1773,15 @@
         "UMask": "0x88",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.FAST",
+        "SampleAfterValue": "20003",
+        "UMask": "0x10",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts number of virtual trap actually taken (e.g. highest priority event during retirement). It can count virtual trap from FPC port 0 or port 1 (x87/SSE) equally in a single counter.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -1700,6 +1791,15 @@
         "UMask": "0x40",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of virtual traps taken.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.FPC_VIRTUAL_TRAP",
+        "SampleAfterValue": "20003",
+        "UMask": "0x40",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of nukes due to memory renaming",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -2015,6 +2115,15 @@
         "UMask": "0x8",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of issue slots not consumed due to a color request for an FCW or MXCSR control register when all 4 colors (copies) are already in use.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x75",
+        "EventName": "SERIALIZATION.COLOR_STALLS",
+        "SampleAfterValue": "200003",
+        "UMask": "0x8",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "Counts the number of issue slots where no uop could issue due to an IQ scoreboard that stalls allocation until a specified older uop retires or (in the case of jump scoreboard) executes. Commonly executed instructions with IQ scoreboards include LFENCE and MFENCE.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -2034,6 +2143,16 @@
         "UMask": "0x2",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of issue slots not consumed by the backend due to a micro-sequencer (MS) scoreboard, which stalls the front-end from issuing from the UROM until a specified older uop retires.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x75",
+        "EventName": "SERIALIZATION.NON_C01_MS_SCB",
+        "PublicDescription": "Counts the number of issue slots not consumed by the backend due to a micro-sequencer (MS) scoreboard, which stalls the front-end from issuing from the UROM until a specified older uop retires. The most commonly executed instruction with an MS scoreboard is PAUSE.",
+        "SampleAfterValue": "200003",
+        "UMask": "0x2",
+        "Unit": "cpu_lowpower"
+    },
     {
         "BriefDescription": "This event counts a subset of the Topdown Slots event that were not consumed by the back-end pipeline due to lack of back-end resources, as a result of memory subsystem delays, execution units limitations, or other conditions.",
         "Counter": "0,1,2,3,4,5,6,7,8,9",
diff --git a/tools/perf/pmu-events/arch/x86/arrowlake/uncore-memory.json b/tools/perf/pmu-events/arch/x86/arrowlake/uncore-memory.json
index ceb8839f07675..60bf390198d6b 100644
--- a/tools/perf/pmu-events/arch/x86/arrowlake/uncore-memory.json
+++ b/tools/perf/pmu-events/arch/x86/arrowlake/uncore-memory.json
@@ -1,61 +1,61 @@
 [
     {
-        "BriefDescription": "Counts every CAS read command sent from the Memory Controller 0 to DRAM (sum of all channels).",
+        "BriefDescription": "This event is deprecated. [This event is alias to UNC_M_MC0_RDCAS_COUNT_FREERUN]",
         "Counter": "0",
+        "Deprecated": "1",
         "EventCode": "0xff",
         "EventName": "UNC_MC0_RDCAS_COUNT_FREERUN",
         "PerPkg": "1",
-        "PublicDescription": "Counts every CAS read command sent from the Memory Controller 0 to DRAM (sum of all channels). Each CAS commands can be for 32B or 64B of data.",
         "UMask": "0x20",
         "Unit": "imc_free_running_0"
     },
     {
-        "BriefDescription": "Counts every read and write request entering the Memory Controller 0.",
+        "BriefDescription": "This event is deprecated. [This event is alias to UNC_M_MC0_TOTAL_REQCOUNT_FREERUN]",
         "Counter": "2",
+        "Deprecated": "1",
         "EventCode": "0xff",
         "EventName": "UNC_MC0_TOTAL_REQCOUNT_FREERUN",
         "PerPkg": "1",
-        "PublicDescription": "Counts every read and write request entering the Memory Controller 0 (sum of all channels). All requests are counted as one, whether they are 32B or 64B Read/Write or partial/full line writes. Some write requests to the same address may merge to a single write command to DRAM. Therefore, the total request count may be higher than total DRAM BW.",
         "UMask": "0x10",
         "Unit": "imc_free_running_0"
     },
     {
-        "BriefDescription": "Counts every CAS write command sent from the Memory Controller 0 to DRAM (sum of all channels).",
+        "BriefDescription": "This event is deprecated. [This event is alias to UNC_M_MC0_WRCAS_COUNT_FREERUN]",
         "Counter": "1",
+        "Deprecated": "1",
         "EventCode": "0xff",
         "EventName": "UNC_MC0_WRCAS_COUNT_FREERUN",
         "PerPkg": "1",
-        "PublicDescription": "Counts every CAS write command sent from the Memory Controller 0 to DRAM (sum of all channels).  Each CAS commands can be for 32B or 64B of data.",
         "UMask": "0x30",
         "Unit": "imc_free_running_0"
     },
     {
-        "BriefDescription": "Counts every CAS read command sent from the Memory Controller 1 to DRAM (sum of all channels).",
+        "BriefDescription": "This event is deprecated. [This event is alias to UNC_M_MC1_RDCAS_COUNT_FREERUN]",
         "Counter": "3",
+        "Deprecated": "1",
         "EventCode": "0xff",
         "EventName": "UNC_MC1_RDCAS_COUNT_FREERUN",
         "PerPkg": "1",
-        "PublicDescription": "Counts every CAS read command sent from the Memory Controller 1 to DRAM (sum of all channels). Each CAS commands can be for 32B or 64B of data.",
         "UMask": "0x20",
         "Unit": "imc_free_running_1"
     },
     {
-        "BriefDescription": "Counts every read and write request entering the Memory Controller 1.",
+        "BriefDescription": "This event is deprecated. [This event is alias to UNC_M_MC1_TOTAL_REQCOUNT_FREERUN]",
         "Counter": "5",
+        "Deprecated": "1",
         "EventCode": "0xff",
         "EventName": "UNC_MC1_TOTAL_REQCOUNT_FREERUN",
         "PerPkg": "1",
-        "PublicDescription": "Counts every read and write request entering the Memory Controller 1 (sum of all channels). All requests are counted as one, whether they are 32B or 64B Read/Write or partial/full line writes. Some write requests to the same address may merge to a single write command to DRAM. Therefore, the total request count may be higher than total DRAM BW.",
         "UMask": "0x10",
         "Unit": "imc_free_running_1"
     },
     {
-        "BriefDescription": "Counts every CAS write command sent from the Memory Controller 1 to DRAM (sum of all channels).",
+        "BriefDescription": "This event is deprecated. [This event is alias to UNC_M_MC1_WRCAS_COUNT_FREERUN]",
         "Counter": "4",
+        "Deprecated": "1",
         "EventCode": "0xff",
         "EventName": "UNC_MC1_WRCAS_COUNT_FREERUN",
         "PerPkg": "1",
-        "PublicDescription": "Counts every CAS write command sent from the Memory Controller 1 to DRAM (sum of all channels).  Each CAS commands can be for 32B or 64B of data.",
         "UMask": "0x30",
         "Unit": "imc_free_running_1"
     },
@@ -117,6 +117,66 @@
         "PerPkg": "1",
         "Unit": "iMC"
     },
+    {
+        "BriefDescription": "Counts every CAS read command sent from the Memory Controller 0 to DRAM (sum of all channels). [This event is alias to UNC_MC0_RDCAS_COUNT_FREERUN]",
+        "Counter": "0",
+        "EventCode": "0xff",
+        "EventName": "UNC_M_MC0_RDCAS_COUNT_FREERUN",
+        "PerPkg": "1",
+        "PublicDescription": "Counts every CAS read command sent from the Memory Controller 0 to DRAM (sum of all channels). Each CAS commands can be for 32B or 64B of data. [This event is alias to UNC_MC0_RDCAS_COUNT_FREERUN]",
+        "UMask": "0x20",
+        "Unit": "imc_free_running_0"
+    },
+    {
+        "BriefDescription": "Counts every read and write request entering the Memory Controller 0. [This event is alias to UNC_MC0_TOTAL_REQCOUNT_FREERUN]",
+        "Counter": "2",
+        "EventCode": "0xff",
+        "EventName": "UNC_M_MC0_TOTAL_REQCOUNT_FREERUN",
+        "PerPkg": "1",
+        "PublicDescription": "Counts every read and write request entering the Memory Controller 0 (sum of all channels). All requests are counted as one, whether they are 32B or 64B Read/Write or partial/full line writes. Some write requests to the same address may merge to a single write command to DRAM. Therefore, the total request count may be higher than total DRAM BW. [This event is alias to UNC_MC0_TOTAL_REQCOUNT_FREERUN]",
+        "UMask": "0x10",
+        "Unit": "imc_free_running_0"
+    },
+    {
+        "BriefDescription": "Counts every CAS write command sent from the Memory Controller 0 to DRAM (sum of all channels). [This event is alias to UNC_MC0_WRCAS_COUNT_FREERUN]",
+        "Counter": "1",
+        "EventCode": "0xff",
+        "EventName": "UNC_M_MC0_WRCAS_COUNT_FREERUN",
+        "PerPkg": "1",
+        "PublicDescription": "Counts every CAS write command sent from the Memory Controller 0 to DRAM (sum of all channels).  Each CAS commands can be for 32B or 64B of data. [This event is alias to UNC_MC0_WRCAS_COUNT_FREERUN]",
+        "UMask": "0x30",
+        "Unit": "imc_free_running_0"
+    },
+    {
+        "BriefDescription": "Counts every CAS read command sent from the Memory Controller 1 to DRAM (sum of all channels). [This event is alias to UNC_MC1_RDCAS_COUNT_FREERUN]",
+        "Counter": "3",
+        "EventCode": "0xff",
+        "EventName": "UNC_M_MC1_RDCAS_COUNT_FREERUN",
+        "PerPkg": "1",
+        "PublicDescription": "Counts every CAS read command sent from the Memory Controller 1 to DRAM (sum of all channels). Each CAS commands can be for 32B or 64B of data. [This event is alias to UNC_MC1_RDCAS_COUNT_FREERUN]",
+        "UMask": "0x20",
+        "Unit": "imc_free_running_1"
+    },
+    {
+        "BriefDescription": "Counts every read and write request entering the Memory Controller 1. [This event is alias to UNC_MC1_TOTAL_REQCOUNT_FREERUN]",
+        "Counter": "5",
+        "EventCode": "0xff",
+        "EventName": "UNC_M_MC1_TOTAL_REQCOUNT_FREERUN",
+        "PerPkg": "1",
+        "PublicDescription": "Counts every read and write request entering the Memory Controller 1 (sum of all channels). All requests are counted as one, whether they are 32B or 64B Read/Write or partial/full line writes. Some write requests to the same address may merge to a single write command to DRAM. Therefore, the total request count may be higher than total DRAM BW. [This event is alias to UNC_MC1_TOTAL_REQCOUNT_FREERUN]",
+        "UMask": "0x10",
+        "Unit": "imc_free_running_1"
+    },
+    {
+        "BriefDescription": "Counts every CAS write command sent from the Memory Controller 1 to DRAM (sum of all channels). [This event is alias to UNC_MC1_WRCAS_COUNT_FREERUN]",
+        "Counter": "4",
+        "EventCode": "0xff",
+        "EventName": "UNC_M_MC1_WRCAS_COUNT_FREERUN",
+        "PerPkg": "1",
+        "PublicDescription": "Counts every CAS write command sent from the Memory Controller 1 to DRAM (sum of all channels).  Each CAS commands can be for 32B or 64B of data. [This event is alias to UNC_MC1_WRCAS_COUNT_FREERUN]",
+        "UMask": "0x30",
+        "Unit": "imc_free_running_1"
+    },
     {
         "BriefDescription": "PRE command sent to DRAM due to page table idle timer expiration",
         "Counter": "0,1,2,3,4",
diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/memory.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/memory.json
index ae9095a090e51..477c06a561bbf 100644
--- a/tools/perf/pmu-events/arch/x86/clearwaterforest/memory.json
+++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/memory.json
@@ -91,6 +91,17 @@
         "SampleAfterValue": "1000003",
         "UMask": "0x4"
     },
+    {
+        "BriefDescription": "Counts demand data reads that were supplied by DRAM.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xB7",
+        "EventName": "OCR.DEMAND_DATA_RD.DRAM",
+        "MSRIndex": "0x1a6,0x1a7",
+        "MSRValue": "0x7BC000001",
+        "PublicDescription": "Counts demand data reads that were supplied by DRAM. Available PDIST counters: 0",
+        "SampleAfterValue": "100003",
+        "UMask": "0x1"
+    },
     {
         "BriefDescription": "Counts demand data reads that were not supplied by the L3 cache.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -102,6 +113,28 @@
         "SampleAfterValue": "100003",
         "UMask": "0x1"
     },
+    {
+        "BriefDescription": "Counts demand data reads that were supplied by DRAM attached to this socket.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xB7",
+        "EventName": "OCR.DEMAND_DATA_RD.LOCAL_DRAM",
+        "MSRIndex": "0x1a6,0x1a7",
+        "MSRValue": "0x784000001",
+        "PublicDescription": "Counts demand data reads that were supplied by DRAM attached to this socket. Available PDIST counters: 0",
+        "SampleAfterValue": "100003",
+        "UMask": "0x1"
+    },
+    {
+        "BriefDescription": "Counts demand data reads that were supplied by DRAM attached to another socket.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xB7",
+        "EventName": "OCR.DEMAND_DATA_RD.REMOTE_DRAM",
+        "MSRIndex": "0x1a6,0x1a7",
+        "MSRValue": "0x730000001",
+        "PublicDescription": "Counts demand data reads that were supplied by DRAM attached to another socket. Available PDIST counters: 0",
+        "SampleAfterValue": "100003",
+        "UMask": "0x1"
+    },
     {
         "BriefDescription": "Counts demand read for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that were not supplied by the L3 cache.",
         "Counter": "0,1,2,3,4,5,6,7",
diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/pipeline.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/pipeline.json
index 3d10dcffb8fcb..f73686174af90 100644
--- a/tools/perf/pmu-events/arch/x86/clearwaterforest/pipeline.json
+++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/pipeline.json
@@ -291,6 +291,25 @@
         "SampleAfterValue": "1000003",
         "UMask": "0x50"
     },
+    {
+        "BriefDescription": "This event is deprecated. [This event is alias to BR_MISP_RETIRED.NEAR_RETURN]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "Deprecated": "1",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.NEAR_RET",
+        "PublicDescription": "This event is deprecated. [This event is alias to BR_MISP_RETIRED.NEAR_RETURN] Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8"
+    },
+    {
+        "BriefDescription": "Counts the number of mispredicted near RET branch instructions retired. [This event is alias to BR_MISP_RETIRED.NEAR_RET]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.NEAR_RETURN",
+        "PublicDescription": "Counts the number of mispredicted near RET branch instructions retired. [This event is alias to BR_MISP_RETIRED.NEAR_RET] Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8"
+    },
     {
         "BriefDescription": "Fixed Counter: Counts the number of unhalted core clock cycles. [This event is alias to CPU_CLK_UNHALTED.THREAD]",
         "Counter": "Fixed counter 1",
diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-interconnect.json
index 251e5d20fefec..1f6e0970141a1 100644
--- a/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-interconnect.json
+++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-interconnect.json
@@ -814,6 +814,16 @@
         "PerPkg": "1",
         "Unit": "IRP"
     },
+    {
+        "BriefDescription": "Counts Timeouts - Set 0 : Cache Inserts of Write Transactions as Secondary",
+        "Counter": "0,1,2,3",
+        "EventCode": "0x1E",
+        "EventName": "UNC_I_MISC0.2ND_WR_INSERT",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "UMask": "0x8",
+        "Unit": "IRP"
+    },
     {
         "BriefDescription": "Counts Timeouts - Set 0 : Fastpath Rejects",
         "Counter": "0,1,2,3",
diff --git a/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-memory.json b/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-memory.json
index 8184c2b6b861c..cae7d0f6c34d2 100644
--- a/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-memory.json
+++ b/tools/perf/pmu-events/arch/x86/clearwaterforest/uncore-memory.json
@@ -520,7 +520,7 @@
         "Unit": "IMC"
     },
     {
-        "BriefDescription": "DRAM Precharge commands. : Precharge due to (?) : Counts the number of DRAM Precharge commands sent on this channel.",
+        "BriefDescription": "DRAM Precharge commands. : Precharge due to page table : Counts the number of DRAM Precharge commands sent on this channel.",
         "Counter": "0,1,2,3",
         "EventCode": "0x03",
         "EventName": "UNC_M_PRE_COUNT.PGT",
diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/cache.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/cache.json
index ff6071d7728e0..a44e1f027c1db 100644
--- a/tools/perf/pmu-events/arch/x86/emeraldrapids/cache.json
+++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/cache.json
@@ -368,6 +368,15 @@
         "SampleAfterValue": "200003",
         "UMask": "0x40"
     },
+    {
+        "BriefDescription": "Cycles when L1D is locked",
+        "Counter": "0,1,2,3",
+        "EventCode": "0x42",
+        "EventName": "LOCK_CYCLES.CACHE_LOCK_DURATION",
+        "PublicDescription": "This event counts the number of cycles when the L1D is locked. It is a superset of the 0x1 mask (BUS_LOCK_CLOCKS.BUS_LOCK_DURATION).",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x2"
+    },
     {
         "BriefDescription": "Core-originated cacheable requests that missed L3  (Except hardware prefetches to the L3)",
         "Counter": "0,1,2,3,4,5,6,7",
diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/emr-metrics.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/emr-metrics.json
index 433ae5f507040..e00e32e71691d 100644
--- a/tools/perf/pmu-events/arch/x86/emeraldrapids/emr-metrics.json
+++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/emr-metrics.json
@@ -84,6 +84,18 @@
         "PublicDescription": "Ratio of number of completed page walks (for all page sizes) caused by demand data stores to the total number of completed instructions. This implies it missed in the DTLB and further levels of TLB",
         "ScaleUnit": "1per_instr"
     },
+    {
+        "BriefDescription": "Bandwidth observed by the integrated I/O traffic controller (IIO) of IO reads that are initiated by end device controllers that are requesting memory from the CPU",
+        "MetricExpr": "UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.ALL_PARTS * 4 / 1e6 / duration_time",
+        "MetricName": "iio_bandwidth_read",
+        "ScaleUnit": "1MB/s"
+    },
+    {
+        "BriefDescription": "Bandwidth observed by the integrated I/O traffic controller (IIO) of IO writes that are initiated by end device controllers that are writing memory to the CPU",
+        "MetricExpr": "UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.ALL_PARTS * 4 / 1e6 / duration_time",
+        "MetricName": "iio_bandwidth_write",
+        "ScaleUnit": "1MB/s"
+    },
     {
         "BriefDescription": "Bandwidth of IO reads that are initiated by end device controllers that are requesting memory from the CPU",
         "MetricExpr": "UNC_CHA_TOR_INSERTS.IO_PCIRDCUR * 64 / 1e6 / duration_time",
@@ -242,6 +254,30 @@
         "MetricName": "llc_demand_data_read_miss_to_dram_latency",
         "ScaleUnit": "1ns"
     },
+    {
+        "BriefDescription": "Bandwidth (MB/sec) of read requests that miss the last level cache (LLC) and go to local memory",
+        "MetricExpr": "UNC_CHA_REQUESTS.READS_LOCAL * 64 / 1e6 / duration_time",
+        "MetricName": "llc_miss_local_memory_bandwidth_read",
+        "ScaleUnit": "1MB/s"
+    },
+    {
+        "BriefDescription": "Bandwidth (MB/sec) of write requests that miss the last level cache (LLC) and go to local memory",
+        "MetricExpr": "UNC_CHA_REQUESTS.WRITES_LOCAL * 64 / 1e6 / duration_time",
+        "MetricName": "llc_miss_local_memory_bandwidth_write",
+        "ScaleUnit": "1MB/s"
+    },
+    {
+        "BriefDescription": "Bandwidth (MB/sec) of read requests that miss the last level cache (LLC) and go to remote memory",
+        "MetricExpr": "UNC_CHA_REQUESTS.READS_REMOTE * 64 / 1e6 / duration_time",
+        "MetricName": "llc_miss_remote_memory_bandwidth_read",
+        "ScaleUnit": "1MB/s"
+    },
+    {
+        "BriefDescription": "Bandwidth (MB/sec) of write requests that miss the last level cache (LLC) and go to remote memory",
+        "MetricExpr": "UNC_CHA_REQUESTS.WRITES_REMOTE * 64 / 1e6 / duration_time",
+        "MetricName": "llc_miss_remote_memory_bandwidth_write",
+        "ScaleUnit": "1MB/s"
+    },
     {
         "BriefDescription": "The ratio of number of completed memory load instructions to the total number completed instructions",
         "MetricExpr": "MEM_INST_RETIRED.ALL_LOADS / INST_RETIRED.ANY",
@@ -403,15 +439,15 @@
     },
     {
         "BriefDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + 0 / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) * tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency) + tma_memory_bound * (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_fb_full / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_miss_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + 0 / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) * tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency) + tma_memory_bound * (tma_l3_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_fb_full / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)))",
         "MetricGroup": "BvMB;Mem;MemoryBW;Offcore;tma_issueBW",
         "MetricName": "tma_bottleneck_data_cache_memory_bandwidth",
         "MetricThreshold": "tma_bottleneck_data_cache_memory_bandwidth > 20",
-        "PublicDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks. Related metrics: tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full"
+        "PublicDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks. Related metrics: tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_uc_bound"
     },
     {
         "BriefDescription": "Total pipeline cost of external Memory- or Cache-Latency related bottlenecks",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + 0 / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) * tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency) + tma_memory_bound * (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l1_latency_dependency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_lock_latency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_split_loads / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_split_stores / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)) + tma_memory_bound * (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_store_latency / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_miss_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + 0 / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) * tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency) + tma_memory_bound * (tma_l3_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * tma_l2_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_l1_latency_dependency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_lock_latency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_split_loads / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_split_stores / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)) + tma_memory_bound * (tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_store_latency / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
         "MetricGroup": "BvML;Mem;MemoryLat;Offcore;tma_issueLat",
         "MetricName": "tma_bottleneck_data_cache_memory_latency",
         "MetricThreshold": "tma_bottleneck_data_cache_memory_latency > 20",
@@ -434,7 +470,7 @@
     },
     {
         "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l1_bound / max(tma_memory_bound, tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
         "MetricGroup": "BvMT;Mem;MemoryTLB;Offcore;tma_issueTLB",
         "MetricName": "tma_bottleneck_memory_data_tlbs",
         "MetricThreshold": "tma_bottleneck_memory_data_tlbs > 20",
@@ -442,7 +478,7 @@
     },
     {
         "BriefDescription": "Total pipeline cost of Memory Synchronization related bottlenecks (data transfers and coherency updates across processors)",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) * tma_remote_cache / (tma_local_mem + tma_remote_cache + tma_remote_mem) + tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) * (tma_contested_accesses + tma_data_sharing) / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full) + tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) * tma_false_sharing / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores - tma_store_latency)) + tma_machine_clears * (1 - tma_other_nukes / tma_other_nukes))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_miss_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) * tma_remote_cache / (tma_local_mem + tma_remote_cache + tma_remote_mem) + tma_l3_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) * (tma_contested_accesses + tma_data_sharing) / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full) + tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) * tma_false_sharing / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores - tma_store_latency)) + tma_machine_clears * (1 - tma_other_nukes / tma_other_nukes))",
         "MetricGroup": "BvMS;LockCont;Mem;Offcore;tma_issueSyncxn",
         "MetricName": "tma_bottleneck_memory_synchronization",
         "MetricThreshold": "tma_bottleneck_memory_synchronization > 10",
@@ -594,13 +630,13 @@
         "ScaleUnit": "100%"
     },
     {
-        "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+        "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to L3 data-sharing accesses",
         "MetricConstraint": "NO_GROUP_EVENTS",
         "MetricExpr": "74.6 * tma_info_system_core_frequency * (MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD + MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD * (1 - OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD))) * (1 + MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS / 2) / tma_info_thread_clks",
         "MetricGroup": "BvMS;Offcore;Snoop;TopdownL4;tma_L4_group;tma_issueSyncxn;tma_l3_bound_group",
         "MetricName": "tma_data_sharing",
         "MetricThreshold": "tma_data_sharing > 0.05 & (tma_l3_bound > 0.05 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD. Related metrics: tma_bottleneck_memory_synchronization, tma_contested_accesses, tma_false_sharing, tma_machine_clears, tma_remote_cache",
+        "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to L3 data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD. Related metrics: tma_bottleneck_memory_synchronization, tma_contested_accesses, tma_false_sharing, tma_machine_clears, tma_remote_cache",
         "ScaleUnit": "100%"
     },
     {
@@ -621,15 +657,6 @@
         "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIV_ACTIVE",
         "ScaleUnit": "100%"
     },
-    {
-        "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
-        "MetricExpr": "MEMORY_ACTIVITY.STALLS_L3_MISS / tma_info_thread_clks",
-        "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_L3_group;tma_memory_bound_group",
-        "MetricName": "tma_dram_bound",
-        "MetricThreshold": "tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)",
-        "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS",
-        "ScaleUnit": "100%"
-    },
     {
         "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
         "MetricExpr": "(IDQ.DSB_CYCLES_ANY - IDQ.DSB_CYCLES_OK) / tma_info_core_core_clks / 2",
@@ -681,7 +708,7 @@
         "MetricGroup": "BvMB;MemoryBW;TopdownL4;tma_L4_group;tma_issueBW;tma_issueSL;tma_issueSmSt;tma_l1_bound_group",
         "MetricName": "tma_fb_full",
         "MetricThreshold": "tma_fb_full > 0.3",
-        "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_store_latency, tma_streaming_stores",
+        "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_store_latency, tma_streaming_stores, tma_uc_bound",
         "ScaleUnit": "100%"
     },
     {
@@ -974,7 +1001,7 @@
     },
     {
         "BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)",
-        "MetricExpr": "IDQ.DSB_UOPS / UOPS_ISSUED.ANY",
+        "MetricExpr": "IDQ.DSB_UOPS / (IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS)",
         "MetricGroup": "DSB;Fed;FetchBW;tma_issueFB",
         "MetricName": "tma_info_frontend_dsb_coverage",
         "MetricThreshold": "tma_info_frontend_dsb_coverage < 0.7 & tma_info_thread_ipc / 6 > 0.35",
@@ -1092,7 +1119,7 @@
     {
         "BriefDescription": "Instructions per FP Arithmetic Scalar Half-Precision instruction (lower number means higher occurrence rate)",
         "MetricExpr": "INST_RETIRED.ANY / FP_ARITH_INST_RETIRED2.SCALAR",
-        "MetricGroup": "Flops;FpScalar;InsType;Server",
+        "MetricGroup": "Flops;FpScalar;InsType",
         "MetricName": "tma_info_inst_mix_iparith_scalar_hp",
         "MetricThreshold": "tma_info_inst_mix_iparith_scalar_hp < 10",
         "PublicDescription": "Instructions per FP Arithmetic Scalar Half-Precision instruction (lower number means higher occurrence rate). Values < 1 are possible due to intentional FMA double counting."
@@ -1161,6 +1188,14 @@
         "MetricThreshold": "tma_info_inst_mix_iptb < 13",
         "PublicDescription": "Instructions per taken branch. Related metrics: tma_dsb_switches, tma_fetch_bandwidth, tma_info_botlnk_l2_dsb_bandwidth, tma_info_botlnk_l2_dsb_misses, tma_info_frontend_dsb_coverage, tma_lcp"
     },
+    {
+        "BriefDescription": "AVX preserve/restore assists per kilo instruction",
+        "MetricExpr": "1e3 * ASSISTS.SSE_AVX_MIX / INST_RETIRED.ANY",
+        "MetricGroup": "tma_issueMV",
+        "MetricName": "tma_info_inst_mix_vectormixpki",
+        "MetricThreshold": "tma_info_inst_mix_vectormixpki > 0.05",
+        "PublicDescription": "AVX preserve/restore assists per kilo instruction. Related metrics: tma_mixing_vectors, tma_ms_switches"
+    },
     {
         "BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]",
         "MetricExpr": "tma_info_memory_l1d_cache_fill_bw",
@@ -1402,7 +1437,7 @@
         "MetricName": "tma_info_memory_tlb_store_stlb_mpki"
     },
     {
-        "BriefDescription": "Mem;Backend;CacheHits",
+        "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per physical core",
         "MetricExpr": "UOPS_EXECUTED.THREAD / (UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 if #SMT_on else cpu@UOPS_EXECUTED.THREAD\\,cmask\\=1@)",
         "MetricGroup": "Cor;Pipeline;PortsUtil;SMT",
         "MetricName": "tma_info_pipeline_execute"
@@ -1482,7 +1517,7 @@
         "MetricExpr": "64 * (UNC_M_CAS_COUNT.RD + UNC_M_CAS_COUNT.WR) / 1e9 / tma_info_system_time",
         "MetricGroup": "HPC;MemOffcore;MemoryBW;SoC;tma_issueBW",
         "MetricName": "tma_info_system_dram_bw_use",
-        "PublicDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]. Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_mem_bandwidth, tma_sq_full"
+        "PublicDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]. Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_mem_bandwidth, tma_sq_full, tma_uc_bound"
     },
     {
         "BriefDescription": "Giga Floating Point Operations Per Second",
@@ -1532,13 +1567,6 @@
         "MetricName": "tma_info_system_mem_dram_read_latency",
         "PublicDescription": "Average latency of data read request to external DRAM memory [in nanoseconds]. Accounts for demand loads and L1/L2 data-read prefetches"
     },
-    {
-        "BriefDescription": "Fraction of Uncore cycles where requests got rejected due to duplicate address already in IRQ ingress queue in the cache homing agent",
-        "MetricExpr": "UNC_CHA_RxC_IRQ1_REJECT.PA_MATCH / UNC_CHA_CLOCKTICKS",
-        "MetricGroup": "LockCont;MemOffcore;Server;SoC",
-        "MetricName": "tma_info_system_mem_irq_duplicate_address",
-        "MetricThreshold": "tma_info_system_mem_irq_duplicate_address > 0.1"
-    },
     {
         "BriefDescription": "Average number of parallel data read requests to external memory",
         "MetricExpr": "UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD / UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD@thresh\\=1@",
@@ -1591,12 +1619,6 @@
         "MetricGroup": "Power",
         "MetricName": "tma_info_system_turbo_utilization"
     },
-    {
-        "BriefDescription": "Measured Average Uncore Frequency for the SoC [GHz]",
-        "MetricExpr": "tma_info_system_socket_clks / 1e9 / tma_info_system_time",
-        "MetricGroup": "SoC",
-        "MetricName": "tma_info_system_uncore_frequency"
-    },
     {
         "BriefDescription": "Cross-socket Ultra Path Interconnect (UPI) data transmit bandwidth for data only [MB / sec]",
         "MetricExpr": "UNC_UPI_TxL_FLITS.ALL_DATA * 64 / 9 / 1e6",
@@ -1752,6 +1774,15 @@
         "PublicDescription": "This metric estimates fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited).  Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance.  Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS. Related metrics: tma_bottleneck_data_cache_memory_latency, tma_mem_latency",
         "ScaleUnit": "100%"
     },
+    {
+        "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+        "MetricExpr": "MEMORY_ACTIVITY.STALLS_L3_MISS / tma_info_thread_clks",
+        "MetricGroup": "MemoryBound;Offcore;TmaL3mem;TopdownL3;tma_L3_group;tma_memory_bound_group",
+        "MetricName": "tma_l3_miss_bound",
+        "MetricThreshold": "tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)",
+        "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS",
+        "ScaleUnit": "100%"
+    },
     {
         "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
         "MetricExpr": "DECODE.LCP / tma_info_thread_clks",
@@ -1826,14 +1857,14 @@
         "MetricExpr": "72 * tma_info_system_core_frequency * MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS / 2) / tma_info_thread_clks",
         "MetricGroup": "Server;TopdownL5;tma_L5_group;tma_mem_latency_group",
         "MetricName": "tma_local_mem",
-        "MetricThreshold": "tma_local_mem > 0.1 & (tma_mem_latency > 0.1 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)))",
+        "MetricThreshold": "tma_local_mem > 0.1 & (tma_mem_latency > 0.1 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)))",
         "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory. Caching will improve the latency and increase performance. Sample with: MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM",
         "ScaleUnit": "100%"
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations",
         "MetricConstraint": "NO_GROUP_EVENTS",
-        "MetricExpr": "(16 * max(0, MEM_INST_RETIRED.LOCK_LOADS - L2_RQSTS.ALL_RFO) + MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES * (10 * L2_RQSTS.RFO_HIT + min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO))) / tma_info_thread_clks",
+        "MetricExpr": "LOCK_CYCLES.CACHE_LOCK_DURATION / tma_info_thread_clks",
         "MetricGroup": "LockCont;Offcore;TopdownL4;tma_L4_group;tma_issueRFO;tma_l1_bound_group",
         "MetricName": "tma_lock_latency",
         "MetricThreshold": "tma_lock_latency > 0.2 & (tma_l1_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
@@ -1856,24 +1887,24 @@
         "MetricExpr": "INT_MISC.MBA_STALLS / tma_info_thread_clks",
         "MetricGroup": "MemoryBW;Offcore;Server;TopdownL5;tma_L5_group;tma_mem_bandwidth_group",
         "MetricName": "tma_mba_stalls",
-        "MetricThreshold": "tma_mba_stalls > 0.1 & (tma_mem_bandwidth > 0.2 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)))",
+        "MetricThreshold": "tma_mba_stalls > 0.1 & (tma_mem_bandwidth > 0.2 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)))",
         "ScaleUnit": "100%"
     },
     {
         "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM)",
-        "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / tma_info_thread_clks",
-        "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_dram_bound_group;tma_issueBW",
+        "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=12@) / tma_info_thread_clks",
+        "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_issueBW;tma_l3_miss_bound_group",
         "MetricName": "tma_mem_bandwidth",
-        "MetricThreshold": "tma_mem_bandwidth > 0.2 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM).  The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_sq_full",
+        "MetricThreshold": "tma_mem_bandwidth > 0.2 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
+        "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM).  The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_sq_full, tma_uc_bound",
         "ScaleUnit": "100%"
     },
     {
         "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory - DRAM ([SPR-HBM] and/or HBM)",
         "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / tma_info_thread_clks - tma_mem_bandwidth",
-        "MetricGroup": "BvML;MemoryLat;Offcore;TopdownL4;tma_L4_group;tma_dram_bound_group;tma_issueLat",
+        "MetricGroup": "BvML;MemoryLat;Offcore;TopdownL4;tma_L4_group;tma_issueLat;tma_l3_miss_bound_group",
         "MetricName": "tma_mem_latency",
-        "MetricThreshold": "tma_mem_latency > 0.1 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
+        "MetricThreshold": "tma_mem_latency > 0.1 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
         "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory - DRAM ([SPR-HBM] and/or HBM).  This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_latency, tma_l3_hit_latency",
         "ScaleUnit": "100%"
     },
@@ -1937,7 +1968,7 @@
         "MetricGroup": "TopdownL5;tma_L5_group;tma_issueMV;tma_ports_utilized_0_group",
         "MetricName": "tma_mixing_vectors",
         "MetricThreshold": "tma_mixing_vectors > 0.05",
-        "PublicDescription": "This metric estimates penalty in terms of percentage of([SKL+] injected blend uops out of all Uops Issued -- the Count Domain; [ADL+] cycles). Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic. Related metrics: tma_ms_switches",
+        "PublicDescription": "This metric estimates penalty in terms of percentage of([SKL+] injected blend uops out of all Uops Issued -- the Count Domain; [ADL+] cycles). Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic. Related metrics: tma_info_inst_mix_vectormixpki, tma_ms_switches",
         "ScaleUnit": "100%"
     },
     {
@@ -1954,7 +1985,7 @@
         "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_L3_group;tma_fetch_latency_group;tma_issueMC;tma_issueMS;tma_issueMV;tma_issueSO",
         "MetricName": "tma_ms_switches",
         "MetricThreshold": "tma_ms_switches > 0.05 & (tma_fetch_latency > 0.1 & tma_frontend_bound > 0.15)",
-        "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES. Related metrics: tma_bottleneck_irregular_overhead, tma_clears_resteers, tma_l1_bound, tma_machine_clears, tma_microcode_sequencer, tma_mixing_vectors, tma_serializing_operation",
+        "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES. Related metrics: tma_bottleneck_irregular_overhead, tma_clears_resteers, tma_info_inst_mix_vectormixpki, tma_l1_bound, tma_machine_clears, tma_microcode_sequencer, tma_mixing_vectors, tma_serializing_operation",
         "ScaleUnit": "100%"
     },
     {
@@ -2090,7 +2121,7 @@
         "MetricExpr": "(133 * tma_info_system_core_frequency * MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM + 133 * tma_info_system_core_frequency * MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD) * (1 + MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS / 2) / tma_info_thread_clks",
         "MetricGroup": "Offcore;Server;Snoop;TopdownL5;tma_L5_group;tma_issueSyncxn;tma_mem_latency_group",
         "MetricName": "tma_remote_cache",
-        "MetricThreshold": "tma_remote_cache > 0.05 & (tma_mem_latency > 0.1 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)))",
+        "MetricThreshold": "tma_remote_cache > 0.05 & (tma_mem_latency > 0.1 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)))",
         "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues. This is caused often due to non-optimal NUMA allocations. #link to NUMA article. Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM_PS;MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD_PS. Related metrics: tma_bottleneck_memory_synchronization, tma_contested_accesses, tma_data_sharing, tma_false_sharing, tma_machine_clears",
         "ScaleUnit": "100%"
     },
@@ -2099,7 +2130,7 @@
         "MetricExpr": "153 * tma_info_system_core_frequency * MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS / 2) / tma_info_thread_clks",
         "MetricGroup": "Server;Snoop;TopdownL5;tma_L5_group;tma_mem_latency_group",
         "MetricName": "tma_remote_mem",
-        "MetricThreshold": "tma_remote_mem > 0.1 & (tma_mem_latency > 0.1 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)))",
+        "MetricThreshold": "tma_remote_mem > 0.1 & (tma_mem_latency > 0.1 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)))",
         "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory. This is caused often due to non-optimal NUMA allocations. #link to NUMA article. Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM_PS",
         "ScaleUnit": "100%"
     },
@@ -2143,7 +2174,7 @@
     },
     {
         "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary",
-        "MetricExpr": "tma_info_memory_load_miss_real_latency * LD_BLOCKS.NO_SR / tma_info_thread_clks",
+        "MetricExpr": "MEM_INST_RETIRED.SPLIT_LOADS * tma_info_memory_load_miss_real_latency / tma_info_thread_clks",
         "MetricGroup": "TopdownL4;tma_L4_group;tma_l1_bound_group",
         "MetricName": "tma_split_loads",
         "MetricThreshold": "tma_split_loads > 0.3",
@@ -2165,7 +2196,7 @@
         "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_issueBW;tma_l3_bound_group",
         "MetricName": "tma_sq_full",
         "MetricThreshold": "tma_sq_full > 0.3 & (tma_l3_bound > 0.05 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth",
+        "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_uc_bound",
         "ScaleUnit": "100%"
     },
     {
@@ -2277,6 +2308,12 @@
         "MetricName": "uncore_frequency",
         "ScaleUnit": "1GHz"
     },
+    {
+        "BriefDescription": "Intel(R) Ultra Path Interconnect (UPI) data receive bandwidth (MB/sec)",
+        "MetricExpr": "UNC_UPI_RxL_FLITS.ALL_DATA * 7.111111111111111 / 1e6 / duration_time",
+        "MetricName": "upi_data_receive_bw",
+        "ScaleUnit": "1MB/s"
+    },
     {
         "BriefDescription": "Intel(R) Ultra Path Interconnect (UPI) data transmit bandwidth (MB/sec)",
         "MetricExpr": "UNC_UPI_TxL_FLITS.ALL_DATA * 7.111111111111111 / 1e6 / duration_time",
diff --git a/tools/perf/pmu-events/arch/x86/emeraldrapids/metricgroups.json b/tools/perf/pmu-events/arch/x86/emeraldrapids/metricgroups.json
index 9129fb7b7ce49..b5f601991e9d4 100644
--- a/tools/perf/pmu-events/arch/x86/emeraldrapids/metricgroups.json
+++ b/tools/perf/pmu-events/arch/x86/emeraldrapids/metricgroups.json
@@ -90,7 +90,6 @@
     "tma_code_stlb_miss_group": "Metrics contributing to tma_code_stlb_miss category",
     "tma_core_bound_group": "Metrics contributing to tma_core_bound category",
     "tma_divider_group": "Metrics contributing to tma_divider category",
-    "tma_dram_bound_group": "Metrics contributing to tma_dram_bound category",
     "tma_dtlb_load_group": "Metrics contributing to tma_dtlb_load category",
     "tma_dtlb_store_group": "Metrics contributing to tma_dtlb_store category",
     "tma_fetch_bandwidth_group": "Metrics contributing to tma_fetch_bandwidth category",
@@ -124,6 +123,7 @@
     "tma_l1_bound_group": "Metrics contributing to tma_l1_bound category",
     "tma_l2_bound_group": "Metrics contributing to tma_l2_bound category",
     "tma_l3_bound_group": "Metrics contributing to tma_l3_bound category",
+    "tma_l3_miss_bound_group": "Metrics contributing to tma_l3_miss_bound category",
     "tma_light_operations_group": "Metrics contributing to tma_light_operations category",
     "tma_load_op_utilization_group": "Metrics contributing to tma_load_op_utilization category",
     "tma_load_stlb_miss_group": "Metrics contributing to tma_load_stlb_miss category",
diff --git a/tools/perf/pmu-events/arch/x86/grandridge/cache.json b/tools/perf/pmu-events/arch/x86/grandridge/cache.json
index 393ce9421c125..4c7727df33e4e 100644
--- a/tools/perf/pmu-events/arch/x86/grandridge/cache.json
+++ b/tools/perf/pmu-events/arch/x86/grandridge/cache.json
@@ -1,4 +1,12 @@
 [
+    {
+        "BriefDescription": "Counts the number of requests that were not accepted into the L2Q because the L2Q is FULL.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x31",
+        "EventName": "CORE_REJECT_L2Q.ANY",
+        "PublicDescription": "Counts the number of (demand and L1 prefetchers) core requests rejected by the L2Q due to a full or nearly full w condition which likely indicates back pressure from L2Q.  It also counts requests that would have gone directly to the XQ, but are rejected due to a full or nearly full condition, indicating back pressure from the IDI link.  The L2Q may also reject transactions  from a core to insure fairness between cores, or to delay a cores dirty eviction when the address conflicts incoming external snoops.  (Note that L2 prefetcher requests that are dropped are not counted by this event.) Counts on a per core basis.",
+        "SampleAfterValue": "200003"
+    },
     {
         "BriefDescription": "Counts the number of L1D cacheline (dirty) evictions caused by load misses, stores, and prefetches.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -62,6 +70,14 @@
         "SampleAfterValue": "1000003",
         "UMask": "0x1"
     },
+    {
+        "BriefDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x30",
+        "EventName": "L2_REJECT_XQ.ANY",
+        "PublicDescription": "Counts the number of demand and prefetch transactions that the External Queue (XQ) rejects due to a full or near full condition which likely indicates back pressure from the IDI link.  The XQ may reject transactions from the L2Q (non-cacheable requests), BBL (L2 misses) and WOB (L2 write-back victims).",
+        "SampleAfterValue": "200003"
+    },
     {
         "BriefDescription": "Counts the number of L2 Cache Accesses that resulted in a Hit from a front door request only (does not include rejects or recycles), per core event",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -322,6 +338,15 @@
         "SampleAfterValue": "20003",
         "UMask": "0x1"
     },
+    {
+        "BriefDescription": "Counts the number of memory uops retired.  A single uop that performs both a load AND a store will be counted as 1, not 2 (e.g. ADD [mem], CONST)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xd0",
+        "EventName": "MEM_UOPS_RETIRED.ALL",
+        "SampleAfterValue": "200003",
+        "UMask": "0x83"
+    },
     {
         "BriefDescription": "Counts the number of load ops retired.",
         "Counter": "0,1,2,3,4,5,6,7",
diff --git a/tools/perf/pmu-events/arch/x86/grandridge/counter.json b/tools/perf/pmu-events/arch/x86/grandridge/counter.json
deleted file mode 100644
index d9ac3aca5bd50..0000000000000
--- a/tools/perf/pmu-events/arch/x86/grandridge/counter.json
+++ /dev/null
@@ -1,42 +0,0 @@
-[
-    {
-        "Unit": "core",
-        "CountersNumFixed": "3",
-        "CountersNumGeneric": "8"
-    },
-    {
-        "Unit": "B2CMI",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": "4"
-    },
-    {
-        "Unit": "CHA",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": "4"
-    },
-    {
-        "Unit": "IMC",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": "4"
-    },
-    {
-        "Unit": "IIO",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": "4"
-    },
-    {
-        "Unit": "IRP",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": "4"
-    },
-    {
-        "Unit": "PCU",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": 4
-    },
-    {
-        "Unit": "CHACMS",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": "4"
-    }
-]
\ No newline at end of file
diff --git a/tools/perf/pmu-events/arch/x86/grandridge/floating-point.json b/tools/perf/pmu-events/arch/x86/grandridge/floating-point.json
index c567f073713c3..67d52477aad6c 100644
--- a/tools/perf/pmu-events/arch/x86/grandridge/floating-point.json
+++ b/tools/perf/pmu-events/arch/x86/grandridge/floating-point.json
@@ -8,6 +8,22 @@
         "SampleAfterValue": "1000003",
         "UMask": "0x2"
     },
+    {
+        "BriefDescription": "Counts the number of active floating point dividers per cycle in the loop stage.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xcd",
+        "EventName": "ARITH.FPDIV_OCCUPANCY",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2"
+    },
+    {
+        "BriefDescription": "Counts the number of floating point divider uops executed per cycle.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xcd",
+        "EventName": "ARITH.FPDIV_UOPS",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8"
+    },
     {
         "BriefDescription": "Counts the number of all types of floating point operations per uop with all default weighting",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -90,6 +106,46 @@
         "SampleAfterValue": "1000003",
         "UMask": "0x2"
     },
+    {
+        "BriefDescription": "Counts the number of uops executed on all floating point ports.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "FP_VINT_UOPS_EXECUTED.ALL",
+        "SampleAfterValue": "1000003",
+        "UMask": "0xf"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 0.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "FP_VINT_UOPS_EXECUTED.P0",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 1.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "FP_VINT_UOPS_EXECUTED.P1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 2.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "FP_VINT_UOPS_EXECUTED.P2",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 0, 1, 2.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "FP_VINT_UOPS_EXECUTED.PRIMARY",
+        "SampleAfterValue": "1000003",
+        "UMask": "0xe"
+    },
     {
         "BriefDescription": "Counts the number of uops executed on floating point and vector integer store data port.",
         "Counter": "0,1,2,3,4,5,6,7",
diff --git a/tools/perf/pmu-events/arch/x86/grandridge/memory.json b/tools/perf/pmu-events/arch/x86/grandridge/memory.json
index 48b6301e7696d..3cbd86976e51e 100644
--- a/tools/perf/pmu-events/arch/x86/grandridge/memory.json
+++ b/tools/perf/pmu-events/arch/x86/grandridge/memory.json
@@ -1,4 +1,12 @@
 [
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to any number of reasons, including an L1 miss, WCB full, pagewalk, store address block or store data block.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.ANY",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x7f"
+    },
     {
         "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to any number of reasons, including an L1 miss, WCB full, pagewalk, store address block or store data block, on a load that retires.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -15,6 +23,14 @@
         "SampleAfterValue": "1000003",
         "UMask": "0xf4"
     },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a DL1 miss.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.L1_MISS",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1"
+    },
     {
         "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a DL1 miss.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -23,6 +39,15 @@
         "SampleAfterValue": "1000003",
         "UMask": "0x81"
     },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to other block cases.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.OTHER",
+        "PublicDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to other block cases such as pipeline conflicts, fences, etc.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x40"
+    },
     {
         "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to other block cases.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -32,6 +57,14 @@
         "SampleAfterValue": "1000003",
         "UMask": "0xc0"
     },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a pagewalk.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.PGWALK",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x20"
+    },
     {
         "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a pagewalk.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -40,6 +73,14 @@
         "SampleAfterValue": "1000003",
         "UMask": "0xa0"
     },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a store address match.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.ST_ADDR",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4"
+    },
     {
         "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a store address match.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -48,6 +89,22 @@
         "SampleAfterValue": "1000003",
         "UMask": "0x84"
     },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to request buffers full or lock in progress.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.WCB_FULL",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2"
+    },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to request buffers full or lock in progress.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.WCB_FULL_AT_RET",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x82"
+    },
     {
         "BriefDescription": "Counts the number of machine clears due to memory ordering caused by a snoop from an external agent. Does not count internally generated machine clears such as those due to memory disambiguation.",
         "Counter": "0,1,2,3,4,5,6,7",
diff --git a/tools/perf/pmu-events/arch/x86/grandridge/pipeline.json b/tools/perf/pmu-events/arch/x86/grandridge/pipeline.json
index 0a8f7d327150d..01cf2d81757f3 100644
--- a/tools/perf/pmu-events/arch/x86/grandridge/pipeline.json
+++ b/tools/perf/pmu-events/arch/x86/grandridge/pipeline.json
@@ -8,6 +8,31 @@
         "SampleAfterValue": "1000003",
         "UMask": "0x3"
     },
+    {
+        "BriefDescription": "Counts the number of cycles when any of the integer dividers are active.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EventCode": "0xcd",
+        "EventName": "ARITH.IDIV_ACTIVE",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1"
+    },
+    {
+        "BriefDescription": "Counts the number of active integer dividers per cycle.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xcd",
+        "EventName": "ARITH.IDIV_OCCUPANCY",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1"
+    },
+    {
+        "BriefDescription": "Counts the number of integer divider uops executed per cycle.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xcd",
+        "EventName": "ARITH.IDIV_UOPS",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4"
+    },
     {
         "BriefDescription": "Counts the total number of branch instructions retired for all branch types.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -113,6 +138,14 @@
         "SampleAfterValue": "200003",
         "UMask": "0xdf"
     },
+    {
+        "BriefDescription": "Counts the number of taken branch instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.TAKEN",
+        "SampleAfterValue": "200003",
+        "UMask": "0x80"
+    },
     {
         "BriefDescription": "Counts the total number of mispredicted branch instructions retired for all branch types.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -236,6 +269,80 @@
         "EventName": "INST_RETIRED.ANY_P",
         "SampleAfterValue": "2000003"
     },
+    {
+        "BriefDescription": "Counts the number of uops executed on all Integer ports.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.ALL",
+        "SampleAfterValue": "1000003",
+        "UMask": "0xff"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on a load port.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.LD",
+        "PublicDescription": "Counts the number of uops executed on a load port.  This event counts for integer uops even if the destination is FP/vector",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on integer port 0.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.P0",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on integer port 1.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.P1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x10"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on integer port 2.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.P2",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x20"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on integer port 3.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.P3",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x40"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on integer port  0,1, 2, 3.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.PRIMARY",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x78"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on a Store address port.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.STA",
+        "PublicDescription": "Counts the number of uops executed on a Store address port. This event counts integer uops even if the data source is FP/vector",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on an integer store data and jump port.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.STD_JMP",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4"
+    },
     {
         "BriefDescription": "Counts the number of retired loads that are blocked because it initially appears to be store forward blocked, but subsequently is shown not to be blocked based on 4K alias check.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -275,6 +382,22 @@
         "SampleAfterValue": "20003",
         "UMask": "0x8"
     },
+    {
+        "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.FAST",
+        "SampleAfterValue": "20003",
+        "UMask": "0x10"
+    },
+    {
+        "BriefDescription": "Counts the number of virtual traps taken.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.FPC_VIRTUAL_TRAP",
+        "SampleAfterValue": "20003",
+        "UMask": "0x40"
+    },
     {
         "BriefDescription": "Counts the number of machines clears due to memory renaming.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -356,6 +479,23 @@
         "SampleAfterValue": "200003",
         "UMask": "0x4"
     },
+    {
+        "BriefDescription": "Counts the number of issue slots not consumed due to a color request for an FCW or MXCSR control register when all 4 colors (copies) are already in use.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x75",
+        "EventName": "SERIALIZATION.COLOR_STALLS",
+        "SampleAfterValue": "200003",
+        "UMask": "0x8"
+    },
+    {
+        "BriefDescription": "Counts the number of issue slots not consumed by the backend due to a micro-sequencer (MS) scoreboard, which stalls the front-end from issuing from the UROM until a specified older uop retires.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x75",
+        "EventName": "SERIALIZATION.NON_C01_MS_SCB",
+        "PublicDescription": "Counts the number of issue slots not consumed by the backend due to a micro-sequencer (MS) scoreboard, which stalls the front-end from issuing from the UROM until a specified older uop retires. The most commonly executed instruction with an MS scoreboard is PAUSE.",
+        "SampleAfterValue": "200003",
+        "UMask": "0x2"
+    },
     {
         "BriefDescription": "Counts the number of issue slots that were not consumed by the backend because allocation is stalled due to a mispredicted jump or a machine clear. [This event is alias to TOPDOWN_BAD_SPECULATION.ALL_P]",
         "Counter": "0,1,2,3,4,5,6,7",
diff --git a/tools/perf/pmu-events/arch/x86/grandridge/uncore-memory.json b/tools/perf/pmu-events/arch/x86/grandridge/uncore-memory.json
index 8413391d752c0..cec6e0a3147ad 100644
--- a/tools/perf/pmu-events/arch/x86/grandridge/uncore-memory.json
+++ b/tools/perf/pmu-events/arch/x86/grandridge/uncore-memory.json
@@ -448,7 +448,7 @@
         "Unit": "IMC"
     },
     {
-        "BriefDescription": "DRAM Precharge commands. : Precharge due to (?) : Counts the number of DRAM Precharge commands sent on this channel.",
+        "BriefDescription": "DRAM Precharge commands. : Precharge due to page table : Counts the number of DRAM Precharge commands sent on this channel.",
         "Counter": "0,1,2,3",
         "EventCode": "0x03",
         "EventName": "UNC_M_PRE_COUNT.PGT",
diff --git a/tools/perf/pmu-events/arch/x86/grandridge/virtual-memory.json b/tools/perf/pmu-events/arch/x86/grandridge/virtual-memory.json
index 35cc5b6d41f20..88996fdac15f8 100644
--- a/tools/perf/pmu-events/arch/x86/grandridge/virtual-memory.json
+++ b/tools/perf/pmu-events/arch/x86/grandridge/virtual-memory.json
@@ -137,6 +137,14 @@
         "SampleAfterValue": "200003",
         "UMask": "0x10"
     },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a DTLB miss",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.DTLB_MISS",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x10"
+    },
     {
         "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a DTLB miss.",
         "Counter": "0,1,2,3,4,5,6,7",
diff --git a/tools/perf/pmu-events/arch/x86/graniterapids/gnr-metrics.json b/tools/perf/pmu-events/arch/x86/graniterapids/gnr-metrics.json
index 299631fb8d533..95e05812eb3b0 100644
--- a/tools/perf/pmu-events/arch/x86/graniterapids/gnr-metrics.json
+++ b/tools/perf/pmu-events/arch/x86/graniterapids/gnr-metrics.json
@@ -452,15 +452,15 @@
     },
     {
         "BriefDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_dram_bound / (tma_cxl_mem_bound + tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + tma_memory_bound * (tma_l3_bound / (tma_cxl_mem_bound + tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * (tma_l1_bound / (tma_cxl_mem_bound + tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_fb_full / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_miss_bound / (tma_cxl_mem_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + tma_memory_bound * (tma_l3_bound / (tma_cxl_mem_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * (tma_l1_bound / (tma_cxl_mem_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_fb_full / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)))",
         "MetricGroup": "BvMB;Mem;MemoryBW;Offcore;tma_issueBW",
         "MetricName": "tma_bottleneck_data_cache_memory_bandwidth",
         "MetricThreshold": "tma_bottleneck_data_cache_memory_bandwidth > 20",
-        "PublicDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks. Related metrics: tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full"
+        "PublicDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks. Related metrics: tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_uc_bound"
     },
     {
         "BriefDescription": "Total pipeline cost of external Memory- or Cache-Latency related bottlenecks",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_dram_bound / (tma_cxl_mem_bound + tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + 0 / (tma_cxl_mem_bound + tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) * tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency) + tma_memory_bound * (tma_l3_bound / (tma_cxl_mem_bound + tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * tma_l2_bound / (tma_cxl_mem_bound + tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) + tma_memory_bound * (tma_l1_bound / (tma_cxl_mem_bound + tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l1_latency_dependency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_cxl_mem_bound + tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_lock_latency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_cxl_mem_bound + tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_split_loads / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_cxl_mem_bound + tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_split_stores / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)) + tma_memory_bound * (tma_store_bound / (tma_cxl_mem_bound + tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_store_latency / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_miss_bound / (tma_cxl_mem_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + tma_memory_bound * (tma_l3_bound / (tma_cxl_mem_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * tma_l2_bound / (tma_cxl_mem_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) + tma_memory_bound * (tma_l1_bound / (tma_cxl_mem_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_l1_latency_dependency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_cxl_mem_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_lock_latency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_cxl_mem_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_split_loads / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_cxl_mem_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_split_stores / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)) + tma_memory_bound * (tma_store_bound / (tma_cxl_mem_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_store_latency / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
         "MetricGroup": "BvML;Mem;MemoryLat;Offcore;tma_issueLat",
         "MetricName": "tma_bottleneck_data_cache_memory_latency",
         "MetricThreshold": "tma_bottleneck_data_cache_memory_latency > 20",
@@ -483,7 +483,7 @@
     },
     {
         "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_l1_bound / (tma_cxl_mem_bound + tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_load / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_cxl_mem_bound + tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l1_bound / (tma_cxl_mem_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_dtlb_load / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_cxl_mem_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
         "MetricGroup": "BvMT;Mem;MemoryTLB;Offcore;tma_issueTLB",
         "MetricName": "tma_bottleneck_memory_data_tlbs",
         "MetricThreshold": "tma_bottleneck_memory_data_tlbs > 20",
@@ -491,7 +491,7 @@
     },
     {
         "BriefDescription": "Total pipeline cost of Memory Synchronization related bottlenecks (data transfers and coherency updates across processors)",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_dram_bound / (tma_cxl_mem_bound + tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) * tma_remote_cache / (tma_local_mem + tma_remote_cache + tma_remote_mem) + tma_l3_bound / (tma_cxl_mem_bound + tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) * (tma_contested_accesses + tma_data_sharing) / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full) + tma_store_bound / (tma_cxl_mem_bound + tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) * tma_false_sharing / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores - tma_store_latency)) + tma_machine_clears * (1 - tma_other_nukes / tma_other_nukes))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_miss_bound / (tma_cxl_mem_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) * tma_remote_cache / (tma_local_mem + tma_remote_cache + tma_remote_mem) + tma_l3_bound / (tma_cxl_mem_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) * (tma_contested_accesses + tma_data_sharing) / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full) + tma_store_bound / (tma_cxl_mem_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) * tma_false_sharing / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores - tma_store_latency)) + tma_machine_clears * (1 - tma_other_nukes / tma_other_nukes))",
         "MetricGroup": "BvMS;LockCont;Mem;Offcore;tma_issueSyncxn",
         "MetricName": "tma_bottleneck_memory_synchronization",
         "MetricThreshold": "tma_bottleneck_memory_synchronization > 10",
@@ -624,7 +624,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to retired misprediction by non-taken conditional branches.",
-        "MetricExpr": "BR_MISP_RETIRED.COND_NTAKEN_COST * BR_MISP_RETIRED.COND_NTAKEN_COST:R / tma_info_thread_clks",
+        "MetricExpr": "BR_MISP_RETIRED.COND_NTAKEN / BR_MISP_RETIRED.ALL_BRANCHES * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_cond_nt_mispredicts",
         "MetricThreshold": "tma_cond_nt_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -632,7 +632,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to misprediction by taken conditional branches.",
-        "MetricExpr": "BR_MISP_RETIRED.COND_TAKEN_COST * BR_MISP_RETIRED.COND_TAKEN_COST:R / tma_info_thread_clks",
+        "MetricExpr": "BR_MISP_RETIRED.COND_TAKEN / BR_MISP_RETIRED.ALL_BRANCHES * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_cond_tk_mispredicts",
         "MetricThreshold": "tma_cond_tk_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -668,13 +668,13 @@
         "ScaleUnit": "100%"
     },
     {
-        "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+        "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to L3 data-sharing accesses",
         "MetricConstraint": "NO_GROUP_EVENTS",
         "MetricExpr": "(MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD * min(MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD:R, 74.6 * tma_info_system_core_frequency) + MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD * min(MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD:R, 74.6 * tma_info_system_core_frequency) * (1 - OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD))) * (1 + MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS / 2) / tma_info_thread_clks",
         "MetricGroup": "BvMS;Offcore;Snoop;TopdownL4;tma_L4_group;tma_issueSyncxn;tma_l3_bound_group",
         "MetricName": "tma_data_sharing",
         "MetricThreshold": "tma_data_sharing > 0.05 & (tma_l3_bound > 0.05 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD. Related metrics: tma_bottleneck_memory_synchronization, tma_contested_accesses, tma_false_sharing, tma_machine_clears, tma_remote_cache",
+        "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to L3 data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD. Related metrics: tma_bottleneck_memory_synchronization, tma_contested_accesses, tma_false_sharing, tma_machine_clears, tma_remote_cache",
         "ScaleUnit": "100%"
     },
     {
@@ -695,15 +695,6 @@
         "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIV_ACTIVE",
         "ScaleUnit": "100%"
     },
-    {
-        "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
-        "MetricExpr": "(MEMORY_ACTIVITY.STALLS_L3_MISS / tma_info_thread_clks - tma_cxl_mem_bound if #has_pmem > 0 else MEMORY_ACTIVITY.STALLS_L3_MISS / tma_info_thread_clks)",
-        "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_L3_group;tma_memory_bound_group",
-        "MetricName": "tma_dram_bound",
-        "MetricThreshold": "tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)",
-        "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS",
-        "ScaleUnit": "100%"
-    },
     {
         "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
         "MetricExpr": "(IDQ.DSB_CYCLES_ANY - IDQ.DSB_CYCLES_OK) / tma_info_core_core_clks / 2",
@@ -755,7 +746,7 @@
         "MetricGroup": "BvMB;MemoryBW;TopdownL4;tma_L4_group;tma_issueBW;tma_issueSL;tma_issueSmSt;tma_l1_bound_group",
         "MetricName": "tma_fb_full",
         "MetricThreshold": "tma_fb_full > 0.3",
-        "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_store_latency, tma_streaming_stores",
+        "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_store_latency, tma_streaming_stores, tma_uc_bound",
         "ScaleUnit": "100%"
     },
     {
@@ -902,7 +893,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to retired misprediction by indirect CALL instructions.",
-        "MetricExpr": "BR_MISP_RETIRED.INDIRECT_CALL_COST * BR_MISP_RETIRED.INDIRECT_CALL_COST:R / tma_info_thread_clks",
+        "MetricExpr": "BR_MISP_RETIRED.INDIRECT_CALL / BR_MISP_RETIRED.ALL_BRANCHES * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_ind_call_mispredicts",
         "MetricThreshold": "tma_ind_call_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -910,7 +901,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to retired misprediction by indirect JMP instructions.",
-        "MetricExpr": "max((BR_MISP_RETIRED.INDIRECT_COST * BR_MISP_RETIRED.INDIRECT_COST:R - BR_MISP_RETIRED.INDIRECT_CALL_COST * BR_MISP_RETIRED.INDIRECT_CALL_COST:R) / tma_info_thread_clks, 0)",
+        "MetricExpr": "max((BR_MISP_RETIRED.INDIRECT - BR_MISP_RETIRED.INDIRECT_CALL) / BR_MISP_RETIRED.ALL_BRANCHES * tma_branch_mispredicts, 0)",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_ind_jump_mispredicts",
         "MetricThreshold": "tma_ind_jump_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -1045,7 +1036,7 @@
     },
     {
         "BriefDescription": "Floating Point Operations Per Cycle",
-        "MetricExpr": "(cpu@FP_ARITH_INST_RETIRED.SCALAR_SINGLE\\,umask\\=0x03@ + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * cpu@FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE\\,umask\\=0x18@ + 8 * cpu@FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE\\,umask\\=0x60@ + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / tma_info_core_core_clks",
+        "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * FP_ARITH_INST_RETIRED.4_FLOPS + 8 * FP_ARITH_INST_RETIRED.8_FLOPS + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / tma_info_core_core_clks",
         "MetricGroup": "Flops;Ret",
         "MetricName": "tma_info_core_flopc"
     },
@@ -1064,7 +1055,7 @@
     },
     {
         "BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)",
-        "MetricExpr": "IDQ.DSB_UOPS / UOPS_ISSUED.ANY",
+        "MetricExpr": "IDQ.DSB_UOPS / (IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS)",
         "MetricGroup": "DSB;Fed;FetchBW;tma_issueFB",
         "MetricName": "tma_info_frontend_dsb_coverage",
         "MetricThreshold": "tma_info_frontend_dsb_coverage < 0.7 & tma_info_thread_ipc / 6 > 0.35",
@@ -1202,7 +1193,7 @@
     {
         "BriefDescription": "Instructions per FP Arithmetic Scalar Half-Precision instruction (lower number means higher occurrence rate)",
         "MetricExpr": "INST_RETIRED.ANY / FP_ARITH_INST_RETIRED2.SCALAR",
-        "MetricGroup": "Flops;FpScalar;InsType;Server",
+        "MetricGroup": "Flops;FpScalar;InsType",
         "MetricName": "tma_info_inst_mix_iparith_scalar_hp",
         "MetricThreshold": "tma_info_inst_mix_iparith_scalar_hp < 10",
         "PublicDescription": "Instructions per FP Arithmetic Scalar Half-Precision instruction (lower number means higher occurrence rate). Values < 1 are possible due to intentional FMA double counting."
@@ -1231,7 +1222,7 @@
     },
     {
         "BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)",
-        "MetricExpr": "INST_RETIRED.ANY / (cpu@FP_ARITH_INST_RETIRED.SCALAR_SINGLE\\,umask\\=0x03@ + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * cpu@FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE\\,umask\\=0x18@ + 8 * cpu@FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE\\,umask\\=0x60@ + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)",
+        "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.SCALAR + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * FP_ARITH_INST_RETIRED.4_FLOPS + 8 * FP_ARITH_INST_RETIRED.8_FLOPS + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)",
         "MetricGroup": "Flops;InsType",
         "MetricName": "tma_info_inst_mix_ipflop",
         "MetricThreshold": "tma_info_inst_mix_ipflop < 10"
@@ -1271,6 +1262,14 @@
         "MetricThreshold": "tma_info_inst_mix_iptb < 13",
         "PublicDescription": "Instructions per taken branch. Related metrics: tma_dsb_switches, tma_fetch_bandwidth, tma_info_botlnk_l2_dsb_bandwidth, tma_info_botlnk_l2_dsb_misses, tma_info_frontend_dsb_coverage, tma_lcp"
     },
+    {
+        "BriefDescription": "AVX preserve/restore assists per kilo instruction",
+        "MetricExpr": "1e3 * ASSISTS.SSE_AVX_MIX / INST_RETIRED.ANY",
+        "MetricGroup": "tma_issueMV",
+        "MetricName": "tma_info_inst_mix_vectormixpki",
+        "MetricThreshold": "tma_info_inst_mix_vectormixpki > 0.05",
+        "PublicDescription": "AVX preserve/restore assists per kilo instruction. Related metrics: tma_mixing_vectors, tma_ms_switches"
+    },
     {
         "BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]",
         "MetricExpr": "tma_info_memory_l1d_cache_fill_bw",
@@ -1526,7 +1525,7 @@
         "MetricName": "tma_info_memory_tlb_store_stlb_mpki"
     },
     {
-        "BriefDescription": "Mem;Backend;CacheHits",
+        "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per physical core",
         "MetricExpr": "UOPS_EXECUTED.THREAD / (UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 if #SMT_on else cpu@UOPS_EXECUTED.THREAD\\,cmask\\=1@)",
         "MetricGroup": "Cor;Pipeline;PortsUtil;SMT",
         "MetricName": "tma_info_pipeline_execute"
@@ -1606,11 +1605,11 @@
         "MetricExpr": "64 * (UNC_M_CAS_COUNT_SCH0.RD + UNC_M_CAS_COUNT_SCH1.RD + UNC_M_CAS_COUNT_SCH0.WR + UNC_M_CAS_COUNT_SCH1.WR) / 1e9 / tma_info_system_time",
         "MetricGroup": "HPC;MemOffcore;MemoryBW;SoC;tma_issueBW",
         "MetricName": "tma_info_system_dram_bw_use",
-        "PublicDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]. Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_mem_bandwidth, tma_sq_full"
+        "PublicDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]. Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_mem_bandwidth, tma_sq_full, tma_uc_bound"
     },
     {
         "BriefDescription": "Giga Floating Point Operations Per Second",
-        "MetricExpr": "(cpu@FP_ARITH_INST_RETIRED.SCALAR_SINGLE\\,umask\\=0x03@ + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * cpu@FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE\\,umask\\=0x18@ + 8 * cpu@FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE\\,umask\\=0x60@ + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / 1e9 / tma_info_system_time",
+        "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * FP_ARITH_INST_RETIRED.4_FLOPS + 8 * FP_ARITH_INST_RETIRED.8_FLOPS + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / 1e9 / tma_info_system_time",
         "MetricGroup": "Cor;Flops;HPC",
         "MetricName": "tma_info_system_gflops",
         "PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width"
@@ -1701,12 +1700,6 @@
         "MetricGroup": "Power",
         "MetricName": "tma_info_system_turbo_utilization"
     },
-    {
-        "BriefDescription": "Measured Average Uncore Frequency for the SoC [GHz]",
-        "MetricExpr": "tma_info_system_socket_clks / 1e9 / tma_info_system_time",
-        "MetricGroup": "SoC",
-        "MetricName": "tma_info_system_uncore_frequency"
-    },
     {
         "BriefDescription": "Cross-socket Ultra Path Interconnect (UPI) data transmit bandwidth for data only [MB / sec]",
         "MetricExpr": "UNC_UPI_TxL_FLITS.ALL_DATA * 64 / 9 / 1e6",
@@ -1862,6 +1855,15 @@
         "PublicDescription": "This metric estimates fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited).  Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance.  Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS. Related metrics: tma_bottleneck_data_cache_memory_latency, tma_mem_latency",
         "ScaleUnit": "100%"
     },
+    {
+        "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+        "MetricExpr": "(MEMORY_ACTIVITY.STALLS_L3_MISS / tma_info_thread_clks - tma_cxl_mem_bound if #has_pmem > 0 else MEMORY_ACTIVITY.STALLS_L3_MISS / tma_info_thread_clks)",
+        "MetricGroup": "MemoryBound;Offcore;TmaL3mem;TopdownL3;tma_L3_group;tma_memory_bound_group",
+        "MetricName": "tma_l3_miss_bound",
+        "MetricThreshold": "tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)",
+        "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS",
+        "ScaleUnit": "100%"
+    },
     {
         "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
         "MetricExpr": "DECODE.LCP / tma_info_thread_clks",
@@ -1936,7 +1938,7 @@
         "MetricExpr": "MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM:R * (1 + MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS / 2) / tma_info_thread_clks",
         "MetricGroup": "Server;TopdownL5;tma_L5_group;tma_mem_latency_group",
         "MetricName": "tma_local_mem",
-        "MetricThreshold": "tma_local_mem > 0.1 & (tma_mem_latency > 0.1 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)))",
+        "MetricThreshold": "tma_local_mem > 0.1 & (tma_mem_latency > 0.1 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)))",
         "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory. Caching will improve the latency and increase performance. Sample with: MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM",
         "ScaleUnit": "100%"
     },
@@ -1966,24 +1968,24 @@
         "MetricExpr": "INT_MISC.MBA_STALLS / tma_info_thread_clks",
         "MetricGroup": "MemoryBW;Offcore;Server;TopdownL5;tma_L5_group;tma_mem_bandwidth_group",
         "MetricName": "tma_mba_stalls",
-        "MetricThreshold": "tma_mba_stalls > 0.1 & (tma_mem_bandwidth > 0.2 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)))",
+        "MetricThreshold": "tma_mba_stalls > 0.1 & (tma_mem_bandwidth > 0.2 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)))",
         "ScaleUnit": "100%"
     },
     {
         "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM)",
-        "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.DATA_RD\\,cmask\\=4@) / tma_info_thread_clks",
-        "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_dram_bound_group;tma_issueBW",
+        "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.DATA_RD\\,cmask\\=12@) / tma_info_thread_clks",
+        "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_issueBW;tma_l3_miss_bound_group",
         "MetricName": "tma_mem_bandwidth",
-        "MetricThreshold": "tma_mem_bandwidth > 0.2 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM).  The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_sq_full",
+        "MetricThreshold": "tma_mem_bandwidth > 0.2 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
+        "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM).  The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_sq_full, tma_uc_bound",
         "ScaleUnit": "100%"
     },
     {
         "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory - DRAM ([SPR-HBM] and/or HBM)",
         "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / tma_info_thread_clks - tma_mem_bandwidth",
-        "MetricGroup": "BvML;MemoryLat;Offcore;TopdownL4;tma_L4_group;tma_dram_bound_group;tma_issueLat",
+        "MetricGroup": "BvML;MemoryLat;Offcore;TopdownL4;tma_L4_group;tma_issueLat;tma_l3_miss_bound_group",
         "MetricName": "tma_mem_latency",
-        "MetricThreshold": "tma_mem_latency > 0.1 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
+        "MetricThreshold": "tma_mem_latency > 0.1 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
         "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory - DRAM ([SPR-HBM] and/or HBM).  This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_latency, tma_l3_hit_latency",
         "ScaleUnit": "100%"
     },
@@ -2047,7 +2049,7 @@
         "MetricGroup": "TopdownL5;tma_L5_group;tma_issueMV;tma_ports_utilized_0_group",
         "MetricName": "tma_mixing_vectors",
         "MetricThreshold": "tma_mixing_vectors > 0.05",
-        "PublicDescription": "This metric estimates penalty in terms of percentage of([SKL+] injected blend uops out of all Uops Issued -- the Count Domain; [ADL+] cycles). Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic. Related metrics: tma_ms_switches",
+        "PublicDescription": "This metric estimates penalty in terms of percentage of([SKL+] injected blend uops out of all Uops Issued -- the Count Domain; [ADL+] cycles). Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic. Related metrics: tma_info_inst_mix_vectormixpki, tma_ms_switches",
         "ScaleUnit": "100%"
     },
     {
@@ -2064,7 +2066,7 @@
         "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_L3_group;tma_fetch_latency_group;tma_issueMC;tma_issueMS;tma_issueMV;tma_issueSO",
         "MetricName": "tma_ms_switches",
         "MetricThreshold": "tma_ms_switches > 0.05 & (tma_fetch_latency > 0.1 & tma_frontend_bound > 0.15)",
-        "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES. Related metrics: tma_bottleneck_irregular_overhead, tma_clears_resteers, tma_l1_bound, tma_machine_clears, tma_microcode_sequencer, tma_mixing_vectors, tma_serializing_operation",
+        "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES. Related metrics: tma_bottleneck_irregular_overhead, tma_clears_resteers, tma_info_inst_mix_vectormixpki, tma_l1_bound, tma_machine_clears, tma_microcode_sequencer, tma_mixing_vectors, tma_serializing_operation",
         "ScaleUnit": "100%"
     },
     {
@@ -2197,25 +2199,25 @@
     },
     {
         "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues",
-        "MetricExpr": "(MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * PEBS + MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * PEBS) * (1 + MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS / 2) / tma_info_thread_clks",
+        "MetricExpr": "(MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM:R + MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD:R) * (1 + MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS / 2) / tma_info_thread_clks",
         "MetricGroup": "Offcore;Server;Snoop;TopdownL5;tma_L5_group;tma_issueSyncxn;tma_mem_latency_group",
         "MetricName": "tma_remote_cache",
-        "MetricThreshold": "tma_remote_cache > 0.05 & (tma_mem_latency > 0.1 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)))",
+        "MetricThreshold": "tma_remote_cache > 0.05 & (tma_mem_latency > 0.1 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)))",
         "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues. This is caused often due to non-optimal NUMA allocations. #link to NUMA article. Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM_PS;MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD_PS. Related metrics: tma_bottleneck_memory_synchronization, tma_contested_accesses, tma_data_sharing, tma_false_sharing, tma_machine_clears",
         "ScaleUnit": "100%"
     },
     {
         "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory",
-        "MetricExpr": "MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * PEBS * (1 + MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS / 2) / tma_info_thread_clks",
+        "MetricExpr": "MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM:R * (1 + MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS / 2) / tma_info_thread_clks",
         "MetricGroup": "Server;Snoop;TopdownL5;tma_L5_group;tma_mem_latency_group",
         "MetricName": "tma_remote_mem",
-        "MetricThreshold": "tma_remote_mem > 0.1 & (tma_mem_latency > 0.1 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)))",
+        "MetricThreshold": "tma_remote_mem > 0.1 & (tma_mem_latency > 0.1 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)))",
         "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory. This is caused often due to non-optimal NUMA allocations. #link to NUMA article. Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM_PS",
         "ScaleUnit": "100%"
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to retired misprediction by (indirect) RET instructions.",
-        "MetricExpr": "BR_MISP_RETIRED.RET_COST * BR_MISP_RETIRED.RET_COST:R / tma_info_thread_clks",
+        "MetricExpr": "BR_MISP_RETIRED.RET / BR_MISP_RETIRED.ALL_BRANCHES * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_ret_mispredicts",
         "MetricThreshold": "tma_ret_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -2283,7 +2285,7 @@
         "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_issueBW;tma_l3_bound_group",
         "MetricName": "tma_sq_full",
         "MetricThreshold": "tma_sq_full > 0.3 & (tma_l3_bound > 0.05 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth",
+        "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_uc_bound",
         "ScaleUnit": "100%"
     },
     {
diff --git a/tools/perf/pmu-events/arch/x86/graniterapids/metricgroups.json b/tools/perf/pmu-events/arch/x86/graniterapids/metricgroups.json
index 9129fb7b7ce49..b5f601991e9d4 100644
--- a/tools/perf/pmu-events/arch/x86/graniterapids/metricgroups.json
+++ b/tools/perf/pmu-events/arch/x86/graniterapids/metricgroups.json
@@ -90,7 +90,6 @@
     "tma_code_stlb_miss_group": "Metrics contributing to tma_code_stlb_miss category",
     "tma_core_bound_group": "Metrics contributing to tma_core_bound category",
     "tma_divider_group": "Metrics contributing to tma_divider category",
-    "tma_dram_bound_group": "Metrics contributing to tma_dram_bound category",
     "tma_dtlb_load_group": "Metrics contributing to tma_dtlb_load category",
     "tma_dtlb_store_group": "Metrics contributing to tma_dtlb_store category",
     "tma_fetch_bandwidth_group": "Metrics contributing to tma_fetch_bandwidth category",
@@ -124,6 +123,7 @@
     "tma_l1_bound_group": "Metrics contributing to tma_l1_bound category",
     "tma_l2_bound_group": "Metrics contributing to tma_l2_bound category",
     "tma_l3_bound_group": "Metrics contributing to tma_l3_bound category",
+    "tma_l3_miss_bound_group": "Metrics contributing to tma_l3_miss_bound category",
     "tma_light_operations_group": "Metrics contributing to tma_light_operations category",
     "tma_load_op_utilization_group": "Metrics contributing to tma_load_op_utilization category",
     "tma_load_stlb_miss_group": "Metrics contributing to tma_load_stlb_miss category",
diff --git a/tools/perf/pmu-events/arch/x86/graniterapids/pipeline.json b/tools/perf/pmu-events/arch/x86/graniterapids/pipeline.json
index 0fef8fd61974f..afc77bdeb600c 100644
--- a/tools/perf/pmu-events/arch/x86/graniterapids/pipeline.json
+++ b/tools/perf/pmu-events/arch/x86/graniterapids/pipeline.json
@@ -559,7 +559,7 @@
         "Counter": "0,1,2,3,4,5,6,7",
         "EventCode": "0xc0",
         "EventName": "INST_RETIRED.REP_ITERATION",
-        "PublicDescription": "Number of iterations of Repeat (REP) string retired instructions such as MOVS, CMPS, and SCAS. Each has a byte, word, and doubleword version and string instructions can be repeated using a repetition prefix, REP, that allows their architectural execution to be repeated a number of times as specified by the RCX register. Note the number of iterations is implementation-dependent.",
+        "PublicDescription": "Number of iterations of Repeat (REP) string retired instructions such as MOVS, CMPS, and SCAS. Each has a byte, word, and doubleword version and string instructions can be repeated using a repetition prefix, REP, that allows their architectural execution to be repeated a number of times as specified by the RCX register. Note: Since the number of iterations within a REP instruction can be significantly affected by fast strings, this event may vary run to run and not match the architectural number of iterations (specified by RCX)",
         "SampleAfterValue": "2000003",
         "UMask": "0x8"
     },
diff --git a/tools/perf/pmu-events/arch/x86/graniterapids/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/graniterapids/uncore-interconnect.json
index 5eb1145f204f5..9f0c4c7198b0d 100644
--- a/tools/perf/pmu-events/arch/x86/graniterapids/uncore-interconnect.json
+++ b/tools/perf/pmu-events/arch/x86/graniterapids/uncore-interconnect.json
@@ -808,6 +808,16 @@
         "PerPkg": "1",
         "Unit": "IRP"
     },
+    {
+        "BriefDescription": "Counts Timeouts - Set 0 : Cache Inserts of Write Transactions as Secondary",
+        "Counter": "0,1,2,3",
+        "EventCode": "0x1E",
+        "EventName": "UNC_I_MISC0.2ND_WR_INSERT",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "UMask": "0x8",
+        "Unit": "IRP"
+    },
     {
         "BriefDescription": "Counts Timeouts - Set 0 : Fastpath Rejects",
         "Counter": "0,1,2,3",
diff --git a/tools/perf/pmu-events/arch/x86/graniterapids/uncore-memory.json b/tools/perf/pmu-events/arch/x86/graniterapids/uncore-memory.json
index f559e27e2815b..9cd2905726fdb 100644
--- a/tools/perf/pmu-events/arch/x86/graniterapids/uncore-memory.json
+++ b/tools/perf/pmu-events/arch/x86/graniterapids/uncore-memory.json
@@ -539,7 +539,7 @@
         "Unit": "IMC"
     },
     {
-        "BriefDescription": "DRAM Precharge commands. : Precharge due to (?) : Counts the number of DRAM Precharge commands sent on this channel.",
+        "BriefDescription": "DRAM Precharge commands. : Precharge due to page table : Counts the number of DRAM Precharge commands sent on this channel.",
         "Counter": "0,1,2,3",
         "EventCode": "0x03",
         "EventName": "UNC_M_PRE_COUNT.PGT",
diff --git a/tools/perf/pmu-events/arch/x86/lunarlake/cache.json b/tools/perf/pmu-events/arch/x86/lunarlake/cache.json
index 92a3667b4520c..5b350233a5e18 100644
--- a/tools/perf/pmu-events/arch/x86/lunarlake/cache.json
+++ b/tools/perf/pmu-events/arch/x86/lunarlake/cache.json
@@ -1,6 +1,6 @@
 [
     {
-        "BriefDescription": "Counts the number of request that were not accepted into the L2Q because the L2Q is FULL.",
+        "BriefDescription": "Counts the number of requests that were not accepted into the L2Q because the L2Q is FULL.",
         "Counter": "0,1,2,3,4,5,6,7",
         "EventCode": "0x31",
         "EventName": "CORE_REJECT_L2Q.ANY",
diff --git a/tools/perf/pmu-events/arch/x86/lunarlake/lnl-metrics.json b/tools/perf/pmu-events/arch/x86/lunarlake/lnl-metrics.json
index 06390a72110de..6bc62297ca6cd 100644
--- a/tools/perf/pmu-events/arch/x86/lunarlake/lnl-metrics.json
+++ b/tools/perf/pmu-events/arch/x86/lunarlake/lnl-metrics.json
@@ -783,16 +783,16 @@
     },
     {
         "BriefDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + tma_memory_bound * (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_fb_full / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_miss_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency + tma_uc_bound)) + tma_memory_bound * (tma_l3_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_fb_full / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)))",
         "MetricGroup": "BvMB;Mem;MemoryBW;Offcore;tma_issueBW",
         "MetricName": "tma_bottleneck_data_cache_memory_bandwidth",
         "MetricThreshold": "tma_bottleneck_data_cache_memory_bandwidth > 20",
-        "PublicDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks. Related metrics: tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full",
+        "PublicDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks. Related metrics: tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_uc_bound",
         "Unit": "cpu_core"
     },
     {
         "BriefDescription": "Total pipeline cost of external Memory- or Cache-Latency related bottlenecks",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + tma_memory_bound * (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l1_latency_dependency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l1_latency_capacity / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_lock_latency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_split_loads / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_split_stores / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)) + tma_memory_bound * (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_store_latency / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_miss_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency + tma_uc_bound)) + tma_memory_bound * (tma_l3_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * tma_l2_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_l1_latency_capacity / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_lock_latency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_split_loads / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_split_stores / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)) + tma_memory_bound * (tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_store_latency / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
         "MetricGroup": "BvML;Mem;MemoryLat;Offcore;tma_issueLat",
         "MetricName": "tma_bottleneck_data_cache_memory_latency",
         "MetricThreshold": "tma_bottleneck_data_cache_memory_latency > 20",
@@ -818,7 +818,7 @@
     },
     {
         "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_load / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_dtlb_load / (tma_dtlb_load + tma_fb_full + tma_l1_latency_capacity + tma_lock_latency + tma_split_loads + tma_store_early_blk + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
         "MetricGroup": "BvMT;Mem;MemoryTLB;Offcore;tma_issueTLB",
         "MetricName": "tma_bottleneck_memory_data_tlbs",
         "MetricThreshold": "tma_bottleneck_memory_data_tlbs > 20",
@@ -827,7 +827,7 @@
     },
     {
         "BriefDescription": "Total pipeline cost of Memory Synchronization related bottlenecks (data transfers and coherency updates across processors)",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) * (tma_contested_accesses + tma_data_sharing) / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full) + tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) * tma_false_sharing / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores - tma_store_latency)) + tma_machine_clears * (1 - tma_other_nukes / tma_other_nukes))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) * (tma_contested_accesses + tma_data_sharing) / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full) + tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) * tma_false_sharing / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores - tma_store_latency)) + tma_machine_clears * (1 - tma_other_nukes / tma_other_nukes))",
         "MetricGroup": "BvMS;LockCont;Mem;Offcore;tma_issueSyncxn",
         "MetricName": "tma_bottleneck_memory_synchronization",
         "MetricThreshold": "tma_bottleneck_memory_synchronization > 10",
@@ -975,7 +975,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to retired misprediction by non-taken conditional branches.",
-        "MetricExpr": "cpu_core@BR_MISP_RETIRED.COND_NTAKEN_COST@ * cpu_core@BR_MISP_RETIRED.COND_NTAKEN_COST@R / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@BR_MISP_RETIRED.COND_NTAKEN@ / cpu_core@BR_MISP_RETIRED.ALL_BRANCHES@ * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_cond_nt_mispredicts",
         "MetricThreshold": "tma_cond_nt_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -984,7 +984,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to misprediction by backward-taken conditional branches.",
-        "MetricExpr": "cpu_core@BR_MISP_RETIRED.COND_TAKEN_BWD_COST@ * cpu_core@BR_MISP_RETIRED.COND_TAKEN_BWD_COST@R / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@BR_MISP_RETIRED.COND_TAKEN_BWD@ / cpu_core@BR_MISP_RETIRED.ALL_BRANCHES@ * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_cond_tk_bwd_mispredicts",
         "MetricThreshold": "tma_cond_tk_bwd_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -993,7 +993,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to misprediction by forward-taken conditional branches.",
-        "MetricExpr": "cpu_core@BR_MISP_RETIRED.COND_TAKEN_FWD_COST@ * cpu_core@BR_MISP_RETIRED.COND_TAKEN_FWD_COST@R / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@BR_MISP_RETIRED.COND_TAKEN_FWD@ / cpu_core@BR_MISP_RETIRED.ALL_BRANCHES@ * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_cond_tk_fwd_mispredicts",
         "MetricThreshold": "tma_cond_tk_fwd_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -1022,13 +1022,13 @@
         "Unit": "cpu_core"
     },
     {
-        "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+        "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to L3 data-sharing accesses",
         "MetricConstraint": "NO_GROUP_EVENTS",
         "MetricExpr": "(cpu_core@MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD@ * min(cpu_core@MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD@R, 24 * tma_info_system_core_frequency) + cpu_core@MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD@ * min(cpu_core@MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD@R, 25 * tma_info_system_core_frequency)) * (1 + cpu_core@MEM_LOAD_RETIRED.FB_HIT@ / cpu_core@MEM_LOAD_RETIRED.L1_MISS@ / 2) / tma_info_thread_clks",
         "MetricGroup": "BvMS;Offcore;Snoop;TopdownL4;tma_L4_group;tma_issueSyncxn;tma_l3_bound_group",
         "MetricName": "tma_data_sharing",
         "MetricThreshold": "tma_data_sharing > 0.05 & (tma_l3_bound > 0.05 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD. Related metrics: tma_bottleneck_memory_synchronization, tma_contested_accesses, tma_false_sharing, tma_machine_clears, tma_remote_cache",
+        "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to L3 data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD. Related metrics: tma_bottleneck_memory_synchronization, tma_contested_accesses, tma_false_sharing, tma_machine_clears, tma_remote_cache",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -1042,19 +1042,9 @@
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
-    {
-        "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
-        "MetricExpr": "cpu_core@MEMORY_STALLS.MEM@ / tma_info_thread_clks",
-        "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_L3_group;tma_memory_bound_group",
-        "MetricName": "tma_dram_bound",
-        "MetricThreshold": "tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)",
-        "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS",
-        "ScaleUnit": "100%",
-        "Unit": "cpu_core"
-    },
     {
         "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
-        "MetricExpr": "(cpu_core@IDQ.DSB_UOPS\\,cmask\\=0x8\\,inv\\=0x1@ / 2 + cpu_core@IDQ.DSB_UOPS@ / (cpu_core@IDQ.DSB_UOPS@ + cpu_core@IDQ.MITE_UOPS@) * (cpu_core@IDQ_BUBBLES.STARVATION_CYCLES@ - cpu_core@IDQ_BUBBLES.FETCH_LATENCY@)) / tma_info_thread_clks",
+        "MetricExpr": "(cpu_core@IDQ.DSB_UOPS\\,cmask\\=0x8\\,inv\\=0x1@ * max(0, 8 - tma_info_pipeline_fetch_dsb) / 8 + cpu_core@IDQ.DSB_UOPS@ / (cpu_core@IDQ.DSB_UOPS@ + cpu_core@IDQ.MITE_UOPS@) * (cpu_core@IDQ_BUBBLES.STARVATION_CYCLES@ - cpu_core@IDQ_BUBBLES.FETCH_LATENCY@)) / tma_info_thread_clks",
         "MetricGroup": "DSB;FetchBW;TopdownL3;tma_L3_group;tma_fetch_bandwidth_group",
         "MetricName": "tma_dsb",
         "MetricThreshold": "tma_dsb > 0.15 & tma_fetch_bandwidth > 0.2",
@@ -1108,7 +1098,7 @@
         "MetricGroup": "BvMB;MemoryBW;TopdownL4;tma_L4_group;tma_issueBW;tma_issueSL;tma_issueSmSt;tma_l1_bound_group",
         "MetricName": "tma_fb_full",
         "MetricThreshold": "tma_fb_full > 0.3",
-        "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_store_latency, tma_streaming_stores",
+        "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_store_latency, tma_streaming_stores, tma_uc_bound",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -1205,7 +1195,7 @@
     },
     {
         "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors",
-        "MetricExpr": "cpu_core@FP_ARITH_OPS_RETIRED.VECTOR\\,umask\\=0x30@ / (tma_retiring * tma_info_thread_slots)",
+        "MetricExpr": "cpu_core@FP_ARITH_OPS_RETIRED.VECTOR_256B@ / (tma_retiring * tma_info_thread_slots)",
         "MetricGroup": "Compute;Flops;TopdownL5;tma_L5_group;tma_fp_vector_group;tma_issue2P",
         "MetricName": "tma_fp_vector_256b",
         "MetricThreshold": "tma_fp_vector_256b > 0.1 & (tma_fp_vector > 0.1 & (tma_fp_arith > 0.2 & tma_light_operations > 0.6))",
@@ -1248,7 +1238,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses",
-        "MetricExpr": "cpu_core@ICACHE_DATA.STALLS@ / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@ICACHE_DATA.STALLS@ / tma_info_thread_clks + tma_isb_full",
         "MetricGroup": "BigFootprint;BvBC;FetchLat;IcMiss;TopdownL3;tma_L3_group;tma_fetch_latency_group",
         "MetricName": "tma_icache_misses",
         "MetricThreshold": "tma_icache_misses > 0.05 & (tma_fetch_latency > 0.1 & tma_frontend_bound > 0.15)",
@@ -1258,7 +1248,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to retired misprediction by indirect CALL instructions.",
-        "MetricExpr": "cpu_core@BR_MISP_RETIRED.INDIRECT_CALL_COST@ * cpu_core@BR_MISP_RETIRED.INDIRECT_CALL_COST@R / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@BR_MISP_RETIRED.INDIRECT_CALL@ / cpu_core@BR_MISP_RETIRED.ALL_BRANCHES@ * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_ind_call_mispredicts",
         "MetricThreshold": "tma_ind_call_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -1267,7 +1257,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to retired misprediction by indirect JMP instructions.",
-        "MetricExpr": "max((cpu_core@BR_MISP_RETIRED.INDIRECT_COST@ * cpu_core@BR_MISP_RETIRED.INDIRECT_COST@R - cpu_core@BR_MISP_RETIRED.INDIRECT_CALL_COST@ * cpu_core@BR_MISP_RETIRED.INDIRECT_CALL_COST@R) / tma_info_thread_clks, 0)",
+        "MetricExpr": "max((cpu_core@BR_MISP_RETIRED.INDIRECT@ - cpu_core@BR_MISP_RETIRED.INDIRECT_CALL@) / cpu_core@BR_MISP_RETIRED.ALL_BRANCHES@ * tma_branch_mispredicts, 0)",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_ind_jump_mispredicts",
         "MetricThreshold": "tma_ind_jump_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -1437,7 +1427,7 @@
     },
     {
         "BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)",
-        "MetricExpr": "cpu_core@IDQ.DSB_UOPS@ / cpu_core@UOPS_ISSUED.ANY@",
+        "MetricExpr": "cpu_core@IDQ.DSB_UOPS@ / (cpu_core@IDQ.DSB_UOPS@ + cpu_core@LSD.UOPS@ + cpu_core@IDQ.MITE_UOPS@ + cpu_core@IDQ.MS_UOPS@)",
         "MetricGroup": "DSB;Fed;FetchBW;tma_issueFB",
         "MetricName": "tma_info_frontend_dsb_coverage",
         "MetricThreshold": "tma_info_frontend_dsb_coverage < 0.7 & tma_info_thread_ipc / 8 > 0.35",
@@ -1504,7 +1494,7 @@
     },
     {
         "BriefDescription": "Fraction of Uops delivered by the LSD (Loop Stream Detector; aka Loop Cache)",
-        "MetricExpr": "cpu_core@LSD.UOPS@ / cpu_core@UOPS_ISSUED.ANY@",
+        "MetricExpr": "cpu_core@LSD.UOPS@ / (cpu_core@IDQ.DSB_UOPS@ + cpu_core@LSD.UOPS@ + cpu_core@IDQ.MITE_UOPS@ + cpu_core@IDQ.MS_UOPS@)",
         "MetricGroup": "Fed;LSD",
         "MetricName": "tma_info_frontend_lsd_coverage",
         "Unit": "cpu_core"
@@ -1663,6 +1653,15 @@
         "PublicDescription": "Instructions per taken branch. Related metrics: tma_dsb_switches, tma_fetch_bandwidth, tma_info_botlnk_l2_dsb_bandwidth, tma_info_botlnk_l2_dsb_misses, tma_info_frontend_dsb_coverage, tma_lcp",
         "Unit": "cpu_core"
     },
+    {
+        "BriefDescription": "AVX preserve/restore assists per kilo instruction",
+        "MetricExpr": "1e3 * cpu_core@ASSISTS.SSE_AVX_MIX@ / cpu_core@INST_RETIRED.ANY@",
+        "MetricGroup": "tma_issueMV",
+        "MetricName": "tma_info_inst_mix_vectormixpki",
+        "MetricThreshold": "tma_info_inst_mix_vectormixpki > 0.05",
+        "PublicDescription": "AVX preserve/restore assists per kilo instruction. Related metrics: tma_mixing_vectors, tma_ms_switches",
+        "Unit": "cpu_core"
+    },
     {
         "BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)",
         "MetricExpr": "1e3 * cpu_core@MEM_LOAD_RETIRED.FB_HIT@ / cpu_core@INST_RETIRED.ANY@",
@@ -1929,6 +1928,13 @@
         "MetricName": "tma_info_pipeline_retire",
         "Unit": "cpu_core"
     },
+    {
+        "BriefDescription": "Total percent of cycles without actually retired uops",
+        "MetricExpr": "cpu_core@UOPS_RETIRED.STALLS@ / tma_info_thread_clks",
+        "MetricGroup": "Pipeline;Ret",
+        "MetricName": "tma_info_pipeline_retire_stall",
+        "Unit": "cpu_core"
+    },
     {
         "BriefDescription": "Estimated fraction of retirement-cycles dealing with repeat instructions",
         "MetricExpr": "cpu_core@INST_RETIRED.REP_ITERATION@ / cpu_core@UOPS_RETIRED.SLOTS\\,cmask\\=1@",
@@ -1971,7 +1977,7 @@
         "MetricExpr": "32 * UNC_M_TOTAL_DATA / 1e9 / tma_info_system_time",
         "MetricGroup": "HPC;MemOffcore;MemoryBW;SoC;tma_issueBW",
         "MetricName": "tma_info_system_dram_bw_use",
-        "PublicDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]. Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_mem_bandwidth, tma_sq_full",
+        "PublicDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]. Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_mem_bandwidth, tma_sq_full, tma_uc_bound",
         "Unit": "cpu_core"
     },
     {
@@ -2042,13 +2048,6 @@
         "MetricName": "tma_info_system_turbo_utilization",
         "Unit": "cpu_core"
     },
-    {
-        "BriefDescription": "Measured Average Uncore Frequency for the SoC [GHz]",
-        "MetricExpr": "tma_info_system_socket_clks / 1e9 / tma_info_system_time",
-        "MetricGroup": "SoC",
-        "MetricName": "tma_info_system_uncore_frequency",
-        "Unit": "cpu_core"
-    },
     {
         "BriefDescription": "Per-Logical Processor actual clocks when the Logical Processor is active.",
         "MetricExpr": "cpu_core@CPU_CLK_UNHALTED.THREAD@",
@@ -2140,9 +2139,18 @@
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
+    {
+        "BriefDescription": "This metric represents fraction of cycles Icache tag lookup was stalled due to Instruction Stream Buffer (ISB) being full",
+        "MetricExpr": "(cpu_core@ICACHE_TAG.STALLS_ISB@ + cpu_core@ICACHE_TAG.STALLS_INUSE@) / tma_info_thread_clks",
+        "MetricGroup": "TopdownL4;tma_L4_group;tma_icache_misses_group",
+        "MetricName": "tma_isb_full",
+        "MetricThreshold": "tma_isb_full > 0.05 & (tma_icache_misses > 0.05 & (tma_fetch_latency > 0.1 & tma_frontend_bound > 0.15))",
+        "ScaleUnit": "100%",
+        "Unit": "cpu_core"
+    },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses",
-        "MetricExpr": "cpu_core@ICACHE_TAG.STALLS@ / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@ICACHE_TAG.STALLS@ / tma_info_thread_clks - tma_isb_full",
         "MetricGroup": "BigFootprint;BvBC;FetchLat;MemoryTLB;TopdownL3;tma_L3_group;tma_fetch_latency_group",
         "MetricName": "tma_itlb_misses",
         "MetricThreshold": "tma_itlb_misses > 0.05 & (tma_fetch_latency > 0.1 & tma_frontend_bound > 0.15)",
@@ -2169,19 +2177,9 @@
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
-    {
-        "BriefDescription": "This metric ([SKL+] roughly; [LNL]) estimates fraction of cycles with demand load accesses that hit the L1D cache",
-        "MetricExpr": "4 * cpu_core@DEPENDENT_LOADS.ANY\\,cmask\\=1@ / tma_info_thread_clks",
-        "MetricGroup": "BvML;MemoryLat;TopdownL4;tma_L4_group;tma_l1_bound_group",
-        "MetricName": "tma_l1_latency_dependency",
-        "MetricThreshold": "tma_l1_latency_dependency > 0.1 & (tma_l1_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric ([SKL+] roughly; [LNL]) estimates fraction of cycles with demand load accesses that hit the L1D cache. The short latency of the L1D cache may be exposed in pointer-chasing memory access patterns as an example. Sample with: MEM_LOAD_UOPS_RETIRED.L1_HIT_PS",
-        "ScaleUnit": "100%",
-        "Unit": "cpu_core"
-    },
     {
         "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads",
-        "MetricExpr": "cpu_core@MEMORY_STALLS.L2@ / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@MEM_LOAD_RETIRED.L2_HIT@ / (cpu_core@MEM_LOAD_RETIRED.L2_HIT@ + cpu_core@MEM_LOAD_MISC_RETIRED.UC@) * cpu_core@MEMORY_STALLS.L2@ / tma_info_thread_clks",
         "MetricGroup": "BvML;CacheHits;MemoryBound;TmaL3mem;TopdownL3;tma_L3_group;tma_memory_bound_group",
         "MetricName": "tma_l2_bound",
         "MetricThreshold": "tma_l2_bound > 0.05 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)",
@@ -2219,6 +2217,16 @@
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
+    {
+        "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+        "MetricExpr": "cpu_core@MEMORY_STALLS.MEM@ / tma_info_thread_clks",
+        "MetricGroup": "MemoryBound;Offcore;TmaL3mem;TopdownL3;tma_L3_group;tma_memory_bound_group",
+        "MetricName": "tma_l3_miss_bound",
+        "MetricThreshold": "tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)",
+        "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS",
+        "ScaleUnit": "100%",
+        "Unit": "cpu_core"
+    },
     {
         "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
         "MetricExpr": "cpu_core@DECODE.LCP@ / tma_info_thread_clks",
@@ -2308,7 +2316,7 @@
     },
     {
         "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to LSD (Loop Stream Detector) unit",
-        "MetricExpr": "cpu_core@LSD.UOPS\\,cmask\\=0x8\\,inv\\=0x1@ / tma_info_thread_clks / 2",
+        "MetricExpr": "cpu_core@LSD.UOPS\\,cmask\\=0x8\\,inv\\=0x1@ * max(0, 8 - tma_info_pipeline_fetch_lsd) / 8 / tma_info_thread_clks",
         "MetricGroup": "FetchBW;LSD;TopdownL3;tma_L3_group;tma_fetch_bandwidth_group",
         "MetricName": "tma_lsd",
         "MetricThreshold": "tma_lsd > 0.15 & tma_fetch_bandwidth > 0.2",
@@ -2330,19 +2338,19 @@
     {
         "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM)",
         "MetricExpr": "min(cpu_core@CPU_CLK_UNHALTED.THREAD@, cpu_core@OFFCORE_REQUESTS_OUTSTANDING.DATA_RD\\,cmask\\=4@) / tma_info_thread_clks",
-        "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_dram_bound_group;tma_issueBW",
+        "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_issueBW;tma_l3_miss_bound_group",
         "MetricName": "tma_mem_bandwidth",
-        "MetricThreshold": "tma_mem_bandwidth > 0.2 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM).  The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_sq_full",
+        "MetricThreshold": "tma_mem_bandwidth > 0.2 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
+        "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM).  The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_sq_full, tma_uc_bound",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
     {
         "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory - DRAM ([SPR-HBM] and/or HBM)",
         "MetricExpr": "min(cpu_core@CPU_CLK_UNHALTED.THREAD@, cpu_core@OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD@) / tma_info_thread_clks - tma_mem_bandwidth",
-        "MetricGroup": "BvML;MemoryLat;Offcore;TopdownL4;tma_L4_group;tma_dram_bound_group;tma_issueLat",
+        "MetricGroup": "BvML;MemoryLat;Offcore;TopdownL4;tma_L4_group;tma_issueLat;tma_l3_miss_bound_group",
         "MetricName": "tma_mem_latency",
-        "MetricThreshold": "tma_mem_latency > 0.1 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
+        "MetricThreshold": "tma_mem_latency > 0.1 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
         "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory - DRAM ([SPR-HBM] and/or HBM).  This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_latency, tma_l3_hit_latency",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
@@ -2398,7 +2406,7 @@
     },
     {
         "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)",
-        "MetricExpr": "(cpu_core@IDQ.MITE_UOPS\\,cmask\\=0x8\\,inv\\=0x1@ / 2 + cpu_core@IDQ.MITE_UOPS@ / (cpu_core@IDQ.DSB_UOPS@ + cpu_core@IDQ.MITE_UOPS@) * (cpu_core@IDQ_BUBBLES.STARVATION_CYCLES@ - cpu_core@IDQ_BUBBLES.FETCH_LATENCY@)) / tma_info_thread_clks",
+        "MetricExpr": "(cpu_core@IDQ.MITE_UOPS\\,cmask\\=0x8\\,inv\\=0x1@ * max(0, 8 - tma_info_pipeline_fetch_mite) / 8 + cpu_core@IDQ.MITE_UOPS@ / (cpu_core@IDQ.DSB_UOPS@ + cpu_core@IDQ.MITE_UOPS@) * (cpu_core@IDQ_BUBBLES.STARVATION_CYCLES@ - cpu_core@IDQ_BUBBLES.FETCH_LATENCY@)) / tma_info_thread_clks",
         "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_L3_group;tma_fetch_bandwidth_group",
         "MetricName": "tma_mite",
         "MetricThreshold": "tma_mite > 0.1 & tma_fetch_bandwidth > 0.2",
@@ -2412,13 +2420,13 @@
         "MetricGroup": "TopdownL5;tma_L5_group;tma_issueMV;tma_ports_utilized_0_group",
         "MetricName": "tma_mixing_vectors",
         "MetricThreshold": "tma_mixing_vectors > 0.05",
-        "PublicDescription": "This metric estimates penalty in terms of percentage of([SKL+] injected blend uops out of all Uops Issued -- the Count Domain; [ADL+] cycles). Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic. Related metrics: tma_ms_switches",
+        "PublicDescription": "This metric estimates penalty in terms of percentage of([SKL+] injected blend uops out of all Uops Issued -- the Count Domain; [ADL+] cycles). Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic. Related metrics: tma_info_inst_mix_vectormixpki, tma_ms_switches",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
     {
         "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the Microcode Sequencer (MS) unit - see Microcode_Sequencer node for details.",
-        "MetricExpr": "cpu_core@IDQ.MS_CYCLES_ANY@ / tma_info_thread_clks / 1.8",
+        "MetricExpr": "cpu_core@IDQ.MS_CYCLES_ANY@ * (8 - cpu_core@IDQ.MS_UOPS@ / cpu_core@IDQ.MS_CYCLES_ANY@) / tma_info_thread_slots",
         "MetricGroup": "MicroSeq;TopdownL3;tma_L3_group;tma_fetch_bandwidth_group",
         "MetricName": "tma_ms",
         "MetricThreshold": "tma_ms > 0.05 & tma_fetch_bandwidth > 0.2",
@@ -2431,7 +2439,7 @@
         "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_L3_group;tma_fetch_latency_group;tma_issueMC;tma_issueMS;tma_issueMV;tma_issueSO",
         "MetricName": "tma_ms_switches",
         "MetricThreshold": "tma_ms_switches > 0.05 & (tma_fetch_latency > 0.1 & tma_frontend_bound > 0.15)",
-        "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES. Related metrics: tma_bottleneck_irregular_overhead, tma_clears_resteers, tma_l1_bound, tma_machine_clears, tma_microcode_sequencer, tma_mixing_vectors, tma_serializing_operation",
+        "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES. Related metrics: tma_bottleneck_irregular_overhead, tma_clears_resteers, tma_info_inst_mix_vectormixpki, tma_l1_bound, tma_machine_clears, tma_microcode_sequencer, tma_mixing_vectors, tma_serializing_operation",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -2549,7 +2557,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to retired misprediction by (indirect) RET instructions.",
-        "MetricExpr": "cpu_core@BR_MISP_RETIRED.RET_COST@ * cpu_core@BR_MISP_RETIRED.RET_COST@R / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@BR_MISP_RETIRED.RET@ / cpu_core@BR_MISP_RETIRED.ALL_BRANCHES@ * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_ret_mispredicts",
         "MetricThreshold": "tma_ret_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -2624,7 +2632,7 @@
         "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_issueBW;tma_l3_bound_group",
         "MetricName": "tma_sq_full",
         "MetricThreshold": "tma_sq_full > 0.3 & (tma_l3_bound > 0.05 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth",
+        "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_uc_bound",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -2639,8 +2647,8 @@
         "Unit": "cpu_core"
     },
     {
-        "BriefDescription": "This metric estimates clocks wasted due to loads blocked due to unknown store address (did not do memory disambiguation) or due to unknown store data",
-        "MetricExpr": "7 * cpu_core@LD_BLOCKS.STORE_EARLY\\,cmask\\=1@ / tma_info_thread_clks",
+        "BriefDescription": "This metric roughly estimates clocks wasted due to loads blocked due to unknown store address (did not do memory disambiguation) or due to unknown store data",
+        "MetricExpr": "cpu_core@LD_BLOCKS.STORE_EARLY\\,cmask\\=1@ / tma_info_thread_clks",
         "MetricGroup": "TopdownL4;tma_L4_group;tma_l1_bound_group",
         "MetricName": "tma_store_early_blk",
         "MetricThreshold": "tma_store_early_blk > 0.2",
@@ -2731,6 +2739,16 @@
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
+    {
+        "BriefDescription": "This metric roughly estimates fraction of cycles where the core's performance was likely hurt due to un-chacheable memory",
+        "MetricExpr": "cpu_core@MEM_LOAD_MISC_RETIRED.UC@ / (cpu_core@MEM_LOAD_RETIRED.L2_HIT@ + cpu_core@MEM_LOAD_MISC_RETIRED.UC@) * cpu_core@MEMORY_STALLS.L2@ / tma_info_thread_clks",
+        "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_issueBW;tma_l3_miss_bound_group",
+        "MetricName": "tma_uc_bound",
+        "MetricThreshold": "tma_uc_bound > 0.2 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
+        "PublicDescription": "This metric roughly estimates fraction of cycles where the core's performance was likely hurt due to un-chacheable memory. Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full",
+        "ScaleUnit": "100%",
+        "Unit": "cpu_core"
+    },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears",
         "MetricExpr": "cpu_core@INT_MISC.UNKNOWN_BRANCH_CYCLES@ / tma_info_thread_clks",
diff --git a/tools/perf/pmu-events/arch/x86/lunarlake/metricgroups.json b/tools/perf/pmu-events/arch/x86/lunarlake/metricgroups.json
index 855585fe6fae1..9acbda612c25c 100644
--- a/tools/perf/pmu-events/arch/x86/lunarlake/metricgroups.json
+++ b/tools/perf/pmu-events/arch/x86/lunarlake/metricgroups.json
@@ -93,7 +93,6 @@
     "tma_code_stlb_miss_group": "Metrics contributing to tma_code_stlb_miss category",
     "tma_core_bound_group": "Metrics contributing to tma_core_bound category",
     "tma_divider_group": "Metrics contributing to tma_divider category",
-    "tma_dram_bound_group": "Metrics contributing to tma_dram_bound category",
     "tma_dtlb_load_group": "Metrics contributing to tma_dtlb_load category",
     "tma_dtlb_store_group": "Metrics contributing to tma_dtlb_store category",
     "tma_fetch_bandwidth_group": "Metrics contributing to tma_fetch_bandwidth category",
@@ -129,6 +128,7 @@
     "tma_l1_bound_group": "Metrics contributing to tma_l1_bound category",
     "tma_l2_bound_group": "Metrics contributing to tma_l2_bound category",
     "tma_l3_bound_group": "Metrics contributing to tma_l3_bound category",
+    "tma_l3_miss_bound_group": "Metrics contributing to tma_l3_miss_bound category",
     "tma_light_operations_group": "Metrics contributing to tma_light_operations category",
     "tma_load_op_utilization_group": "Metrics contributing to tma_load_op_utilization category",
     "tma_load_stlb_miss_group": "Metrics contributing to tma_load_stlb_miss category",
diff --git a/tools/perf/pmu-events/arch/x86/lunarlake/pipeline.json b/tools/perf/pmu-events/arch/x86/lunarlake/pipeline.json
index d66eafccebbb6..8be3ea336bb8a 100644
--- a/tools/perf/pmu-events/arch/x86/lunarlake/pipeline.json
+++ b/tools/perf/pmu-events/arch/x86/lunarlake/pipeline.json
@@ -190,7 +190,6 @@
         "EventName": "BR_INST_RETIRED.COND_TAKEN_FWD",
         "PublicDescription": "Counts taken forward conditional branch instructions retired. Available PDIST counters: 0,1",
         "SampleAfterValue": "400009",
-        "UMask": "0x102",
         "Unit": "cpu_core"
     },
     {
@@ -324,6 +323,15 @@
         "UMask": "0xdf",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of taken branch instructions retired",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.TAKEN",
+        "SampleAfterValue": "200003",
+        "UMask": "0x80",
+        "Unit": "cpu_atom"
+    },
     {
         "BriefDescription": "Counts the total number of mispredicted branch instructions retired for all branch types.",
         "Counter": "0,1,2,3,4,5,6,7",
@@ -446,7 +454,7 @@
         "EventName": "BR_MISP_RETIRED.COND_TAKEN_BWD_COST",
         "PublicDescription": "number of branch instructions retired that were mispredicted and taken backward. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch. Available PDIST counters: 0,1",
         "SampleAfterValue": "400009",
-        "UMask": "0x8001",
+        "UMask": "0x41",
         "Unit": "cpu_core"
     },
     {
@@ -475,7 +483,7 @@
         "EventName": "BR_MISP_RETIRED.COND_TAKEN_FWD_COST",
         "PublicDescription": "number of branch instructions retired that were mispredicted and taken forward. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch. Available PDIST counters: 0,1",
         "SampleAfterValue": "400009",
-        "UMask": "0x8002",
+        "UMask": "0x140",
         "Unit": "cpu_core"
     },
     {
@@ -575,11 +583,11 @@
         "Unit": "cpu_core"
     },
     {
-        "BriefDescription": "This event counts the number of mispredicted ret instructions retired. Non PEBS",
+        "BriefDescription": "This event counts the number of mispredicted ret instructions retired.",
         "Counter": "0,1,2,3,4,5,6,7,8,9",
         "EventCode": "0xc5",
         "EventName": "BR_MISP_RETIRED.RET",
-        "PublicDescription": "This is a non-precise version (that is, does not use PEBS) of the event that counts mispredicted return instructions retired. Available PDIST counters: 0,1",
+        "PublicDescription": "This event counts the number of mispredicted ret instructions retired. Available PDIST counters: 0,1",
         "SampleAfterValue": "100007",
         "UMask": "0x8",
         "Unit": "cpu_core"
@@ -965,7 +973,7 @@
         "Counter": "0,1,2,3,4,5,6,7,8,9",
         "EventCode": "0xc0",
         "EventName": "INST_RETIRED.REP_ITERATION",
-        "PublicDescription": "Number of iterations of Repeat (REP) string retired instructions such as MOVS, CMPS, and SCAS. Each has a byte, word, and doubleword version and string instructions can be repeated using a repetition prefix, REP, that allows their architectural execution to be repeated a number of times as specified by the RCX register. Note the number of iterations is implementation-dependent. Available PDIST counters: 0,1",
+        "PublicDescription": "Number of iterations of Repeat (REP) string retired instructions such as MOVS, CMPS, and SCAS. Each has a byte, word, and doubleword version and string instructions can be repeated using a repetition prefix, REP, that allows their architectural execution to be repeated a number of times as specified by the RCX register. Note: Since the number of iterations within a REP instruction can be significantly affected by fast strings, this event may vary run to run and not match the architectural number of iterations (specified by RCX) Available PDIST counters: 0,1",
         "SampleAfterValue": "2000003",
         "UMask": "0x8",
         "Unit": "cpu_core"
@@ -1373,6 +1381,15 @@
         "UMask": "0x88",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts number of virtual trap actually taken (e.g. highest priority event during retirement). It can count virtual trap from FPC port 0 or port 1 (x87/SSE) equally in a single counter.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.FPC_VIRTUAL_TRAP",
+        "SampleAfterValue": "20003",
+        "UMask": "0x40",
+        "Unit": "cpu_atom"
+    },
     {
         "BriefDescription": "Counts the number of nukes due to memory renaming",
         "Counter": "0,1,2,3,4,5,6,7",
diff --git a/tools/perf/pmu-events/arch/x86/lunarlake/uncore-memory.json b/tools/perf/pmu-events/arch/x86/lunarlake/uncore-memory.json
index 63c4aa2791e45..a1e79f06645ac 100644
--- a/tools/perf/pmu-events/arch/x86/lunarlake/uncore-memory.json
+++ b/tools/perf/pmu-events/arch/x86/lunarlake/uncore-memory.json
@@ -1,6 +1,30 @@
 [
     {
-        "BriefDescription": "Read CAS command sent to DRAM",
+        "BriefDescription": "ACT command for a read request sent to DRAM.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x24",
+        "EventName": "UNC_M_ACT_COUNT_RD",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "ACT command sent to DRAM.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x26",
+        "EventName": "UNC_M_ACT_COUNT_TOTAL",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "ACT command for a write request sent to DRAM.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x25",
+        "EventName": "UNC_M_ACT_COUNT_WR",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Read CAS command sent to DRAM.",
         "Counter": "0,1,2,3,4",
         "EventCode": "0x22",
         "EventName": "UNC_M_CAS_COUNT_RD",
@@ -8,7 +32,7 @@
         "Unit": "iMC"
     },
     {
-        "BriefDescription": "Write CAS command sent to DRAM",
+        "BriefDescription": "Write CAS command sent to DRAM.",
         "Counter": "0,1,2,3,4",
         "EventCode": "0x23",
         "EventName": "UNC_M_CAS_COUNT_WR",
@@ -16,7 +40,94 @@
         "Unit": "iMC"
     },
     {
-        "BriefDescription": "Any Rank at Hot state",
+        "BriefDescription": "Counting the number of clocks.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x01",
+        "EventName": "UNC_M_CLOCKTICKS",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "CKE in DRAM is low.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x29",
+        "EventName": "UNC_M_DRAM_CKE_OFF_CYCLES",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming read request page status is Page Empty.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x1D",
+        "EventName": "UNC_M_DRAM_PAGE_EMPTY_RD",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "PublicDescription": "incoming read request page status is Page Empty",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming write request page status is Page Empty.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x20",
+        "EventName": "UNC_M_DRAM_PAGE_EMPTY_WR",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "PublicDescription": "incoming write request page status is Page Empty",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming read request page status is Page Hit.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x1C",
+        "EventName": "UNC_M_DRAM_PAGE_HIT_RD",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "PublicDescription": "incoming read request page status is Page Hit",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming write request page status is Page Hit.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x1F",
+        "EventName": "UNC_M_DRAM_PAGE_HIT_WR",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "PublicDescription": "incoming write request page status is Page Hit",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming read request page status is Page Miss.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x1E",
+        "EventName": "UNC_M_DRAM_PAGE_MISS_RD",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "PublicDescription": "incoming read request page status is Page Miss",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming write request page status is Page Miss.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x21",
+        "EventName": "UNC_M_DRAM_PAGE_MISS_WR",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "PublicDescription": "incoming write request page status is Page Miss",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "DRAM in Self-refresh (all channels).",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x12",
+        "EventName": "UNC_M_DRAM_SELF_REFRESH",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Any Rank at Hot state.",
         "Counter": "0,1,2,3,4",
         "EventCode": "0x19",
         "EventName": "UNC_M_DRAM_THERMAL_HOT",
@@ -25,7 +136,7 @@
         "Unit": "iMC"
     },
     {
-        "BriefDescription": "Any Rank at Warm state",
+        "BriefDescription": "Any Rank at Warm state.",
         "Counter": "0,1,2,3,4",
         "EventCode": "0x1A",
         "EventName": "UNC_M_DRAM_THERMAL_WARM",
@@ -33,6 +144,42 @@
         "PerPkg": "1",
         "Unit": "iMC"
     },
+    {
+        "BriefDescription": "PRE command sent to DRAM for a read/write request.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x27",
+        "EventName": "UNC_M_PRE_COUNT_PAGE_MISS",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Counts number of bytes read, in 32B chunk, per DDR channel. Counter increments by 1 after receiving 32B chunk data.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x3A",
+        "EventName": "UNC_M_RD_DATA",
+        "PerPkg": "1",
+        "PublicDescription": "This counter counts number of bytes read, in 32B chunk, per DDR channel. Counter increments by 1 after receiving 32B chunk data.",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Number of VC0 read in channel0  - this event can increment by more than 1 (per channel/sub-ch).",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x13",
+        "EventName": "UNC_M_RD_OCCUPANCY_CH0",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "PublicDescription": "Number of VC0 read in channel0  - this event can  increment by more than 1 (per channel/sub-ch)",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Number of VC0 read in channel1 - this event can increment by more than 1 (per channel/sub-ch).",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x14",
+        "EventName": "UNC_M_RD_OCCUPANCY_CH1",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
     {
         "BriefDescription": "Total number of read and write byte transfers to/from DRAM, in 32B chunk, per DDR channel. Counter increments by 1 after sending  or receiving 32B chunk data.",
         "Counter": "0,1,2,3,4",
@@ -40,5 +187,58 @@
         "EventName": "UNC_M_TOTAL_DATA",
         "PerPkg": "1",
         "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Total number of requests entering MC, this is the sum of all RD + WR requests for all VCs.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x39",
+        "EventName": "UNC_M_TOTAL_REQUESTS",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming VC0 read request.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x02",
+        "EventName": "UNC_M_VC0_REQUESTS_RD",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming VC0 write request.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x03",
+        "EventName": "UNC_M_VC0_REQUESTS_WR",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming VC1 read request.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x04",
+        "EventName": "UNC_M_VC1_REQUESTS_RD",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming VC1 write request.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x05",
+        "EventName": "UNC_M_VC1_REQUESTS_WR",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Counts number of bytes written, in 32B chunk, per DDR channel. Counter increments by 1 after sending 32B chunk data.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x3B",
+        "EventName": "UNC_M_WR_DATA",
+        "PerPkg": "1",
+        "Unit": "iMC"
     }
 ]
diff --git a/tools/perf/pmu-events/arch/x86/mapfile.csv b/tools/perf/pmu-events/arch/x86/mapfile.csv
index 50a403b429b14..8e4ee898942cf 100644
--- a/tools/perf/pmu-events/arch/x86/mapfile.csv
+++ b/tools/perf/pmu-events/arch/x86/mapfile.csv
@@ -1,19 +1,19 @@
 Family-model,Version,Filename,EventType
-GenuineIntel-6-(97|9A|B7|BA|BF),v1.39,alderlake,core
-GenuineIntel-6-BE,v1.39,alderlaken,core
-GenuineIntel-6-C[56],v1.17,arrowlake,core
+GenuineIntel-6-(97|9A|B7|BA|BF),v1.40,alderlake,core
+GenuineIntel-6-BE,v1.40,alderlaken,core
+GenuineIntel-6-C[56],v1.20,arrowlake,core
 GenuineIntel-6-(1C|26|27|35|36),v5,bonnell,core
 GenuineIntel-6-(3D|47),v30,broadwell,core
 GenuineIntel-6-56,v12,broadwellde,core
 GenuineIntel-6-4F,v23,broadwellx,core
 GenuineIntel-6-55-[56789ABCDEF],v1.25,cascadelakex,core
-GenuineIntel-6-DD,v1.02,clearwaterforest,core
+GenuineIntel-6-DD,v1.04,clearwaterforest,core
 GenuineIntel-6-9[6C],v1.05,elkhartlake,core
-GenuineIntel-6-CF,v1.23,emeraldrapids,core
+GenuineIntel-6-CF,v1.24,emeraldrapids,core
 GenuineIntel-6-5[CF],v13,goldmont,core
 GenuineIntel-6-7A,v1.01,goldmontplus,core
-GenuineIntel-6-B6,v1.12,grandridge,core
-GenuineIntel-6-A[DE],v1.18,graniterapids,core
+GenuineIntel-6-B6,v1.13,grandridge,core
+GenuineIntel-6-A[DE],v1.20,graniterapids,core
 GenuineIntel-6-(3C|45|46),v36,haswell,core
 GenuineIntel-6-3F,v29,haswellx,core
 GenuineIntel-6-7[DE],v1.24,icelake,core
@@ -22,23 +22,24 @@ GenuineIntel-6-3A,v24,ivybridge,core
 GenuineIntel-6-3E,v24,ivytown,core
 GenuineIntel-6-2D,v24,jaketown,core
 GenuineIntel-6-(57|85),v16,knightslanding,core
-GenuineIntel-6-BD,v1.22,lunarlake,core
-GenuineIntel-6-(AA|AC|B5),v1.21,meteorlake,core
+GenuineIntel-6-BD,v1.26,lunarlake,core
+GenuineIntel-6-(AA|AC|B5),v1.22,meteorlake,core
 GenuineIntel-6-1[AEF],v4,nehalemep,core
 GenuineIntel-6-2E,v4,nehalemex,core
-GenuineIntel-6-(CC|D5),v1.05,pantherlake,core
+GenuineIntel-6-(CC|D5|E5),v1.07,pantherlake,core
 GenuineIntel-6-A7,v1.04,rocketlake,core
 GenuineIntel-6-2A,v19,sandybridge,core
 GenuineIntel-6-8F,v1.39,sapphirerapids,core
-GenuineIntel-6-AF,v1.17,sierraforest,core
+GenuineIntel-6-AF,v1.18,sierraforest,core
 GenuineIntel-6-(37|4A|4C|4D|5A),v15,silvermont,core
 GenuineIntel-6-(4E|5E|8E|9E|A5|A6),v59,skylake,core
 GenuineIntel-6-55-[01234],v1.37,skylakex,core
 GenuineIntel-6-86,v1.23,snowridgex,core
-GenuineIntel-6-8[CD],v1.18,tigerlake,core
+GenuineIntel-6-8[CD],v1.19,tigerlake,core
 GenuineIntel-6-2C,v5,westmereep-dp,core
 GenuineIntel-6-25,v4,westmereep-sp,core
 GenuineIntel-6-2F,v4,westmereex,core
+GenuineIntel-18-0[13],v1.00,novalake,core
 AuthenticAMD-23-([12][0-9A-F]|[0-9A-F]),v2,amdzen1,core
 AuthenticAMD-23-[[:xdigit:]]+,v1,amdzen2,core
 AuthenticAMD-25-([245][[:xdigit:]]|[[:xdigit:]]),v1,amdzen3,core
diff --git a/tools/perf/pmu-events/arch/x86/meteorlake/cache.json b/tools/perf/pmu-events/arch/x86/meteorlake/cache.json
index 6419bc36f2493..c2d351efa9d64 100644
--- a/tools/perf/pmu-events/arch/x86/meteorlake/cache.json
+++ b/tools/perf/pmu-events/arch/x86/meteorlake/cache.json
@@ -1,6 +1,6 @@
 [
     {
-        "BriefDescription": "Counts the number of request that were not accepted into the L2Q because the L2Q is FULL.",
+        "BriefDescription": "Counts the number of requests that were not accepted into the L2Q because the L2Q is FULL.",
         "Counter": "0,1,2,3,4,5,6,7",
         "EventCode": "0x31",
         "EventName": "CORE_REJECT_L2Q.ANY",
@@ -631,6 +631,15 @@
         "UMask": "0x50",
         "Unit": "cpu_atom"
     },
+    {
+        "BriefDescription": "Counts the number of unhalted cycles when the core is stalled due to a demand load miss which missed all the caches, a snoop was required, and hits in other core or module on the same die. Another core provides the data with a FWD, NO_FWD, or HITM. If the core has access to an L3 cache, an LLC miss refers to an L3 cache miss, otherwise it is an L2 cache miss.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x34",
+        "EventName": "MEM_BOUND_STALLS_LOAD.LLC_MISS_OTHERMOD",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8",
+        "Unit": "cpu_atom"
+    },
     {
         "BriefDescription": "Counts the number of unhalted cycles when the core is stalled to a store buffer full condition",
         "Counter": "0,1,2,3,4,5,6,7",
diff --git a/tools/perf/pmu-events/arch/x86/meteorlake/metricgroups.json b/tools/perf/pmu-events/arch/x86/meteorlake/metricgroups.json
index 855585fe6fae1..9acbda612c25c 100644
--- a/tools/perf/pmu-events/arch/x86/meteorlake/metricgroups.json
+++ b/tools/perf/pmu-events/arch/x86/meteorlake/metricgroups.json
@@ -93,7 +93,6 @@
     "tma_code_stlb_miss_group": "Metrics contributing to tma_code_stlb_miss category",
     "tma_core_bound_group": "Metrics contributing to tma_core_bound category",
     "tma_divider_group": "Metrics contributing to tma_divider category",
-    "tma_dram_bound_group": "Metrics contributing to tma_dram_bound category",
     "tma_dtlb_load_group": "Metrics contributing to tma_dtlb_load category",
     "tma_dtlb_store_group": "Metrics contributing to tma_dtlb_store category",
     "tma_fetch_bandwidth_group": "Metrics contributing to tma_fetch_bandwidth category",
@@ -129,6 +128,7 @@
     "tma_l1_bound_group": "Metrics contributing to tma_l1_bound category",
     "tma_l2_bound_group": "Metrics contributing to tma_l2_bound category",
     "tma_l3_bound_group": "Metrics contributing to tma_l3_bound category",
+    "tma_l3_miss_bound_group": "Metrics contributing to tma_l3_miss_bound category",
     "tma_light_operations_group": "Metrics contributing to tma_light_operations category",
     "tma_load_op_utilization_group": "Metrics contributing to tma_load_op_utilization category",
     "tma_load_stlb_miss_group": "Metrics contributing to tma_load_stlb_miss category",
diff --git a/tools/perf/pmu-events/arch/x86/meteorlake/mtl-metrics.json b/tools/perf/pmu-events/arch/x86/meteorlake/mtl-metrics.json
index 948c16a1f95ba..a23691052237e 100644
--- a/tools/perf/pmu-events/arch/x86/meteorlake/mtl-metrics.json
+++ b/tools/perf/pmu-events/arch/x86/meteorlake/mtl-metrics.json
@@ -797,16 +797,16 @@
     },
     {
         "BriefDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + tma_memory_bound * (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_fb_full / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_miss_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + tma_memory_bound * (tma_l3_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_fb_full / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)))",
         "MetricGroup": "BvMB;Mem;MemoryBW;Offcore;tma_issueBW",
         "MetricName": "tma_bottleneck_data_cache_memory_bandwidth",
         "MetricThreshold": "tma_bottleneck_data_cache_memory_bandwidth > 20",
-        "PublicDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks. Related metrics: tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full",
+        "PublicDescription": "Total pipeline cost of external Memory- or Cache-Bandwidth related bottlenecks. Related metrics: tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_uc_bound",
         "Unit": "cpu_core"
     },
     {
         "BriefDescription": "Total pipeline cost of external Memory- or Cache-Latency related bottlenecks",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + tma_memory_bound * (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l1_latency_dependency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_lock_latency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_split_loads / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_split_stores / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)) + tma_memory_bound * (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_store_latency / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_miss_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + tma_memory_bound * (tma_l3_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + tma_memory_bound * tma_l2_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_l1_latency_dependency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_lock_latency / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_split_loads / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_split_stores / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)) + tma_memory_bound * (tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_store_latency / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
         "MetricGroup": "BvML;Mem;MemoryLat;Offcore;tma_issueLat",
         "MetricName": "tma_bottleneck_data_cache_memory_latency",
         "MetricThreshold": "tma_bottleneck_data_cache_memory_latency > 20",
@@ -832,7 +832,7 @@
     },
     {
         "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_load / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l1_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_dtlb_load / (tma_dtlb_load + tma_fb_full + tma_l1_latency_dependency + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + tma_memory_bound * (tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores)))",
         "MetricGroup": "BvMT;Mem;MemoryTLB;Offcore;tma_issueTLB",
         "MetricName": "tma_bottleneck_memory_data_tlbs",
         "MetricThreshold": "tma_bottleneck_memory_data_tlbs > 20",
@@ -841,7 +841,7 @@
     },
     {
         "BriefDescription": "Total pipeline cost of Memory Synchronization related bottlenecks (data transfers and coherency updates across processors)",
-        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) * (tma_contested_accesses + tma_data_sharing) / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full) + tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound) * tma_false_sharing / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores - tma_store_latency)) + tma_machine_clears * (1 - tma_other_nukes / tma_other_nukes))",
+        "MetricExpr": "100 * (tma_memory_bound * (tma_l3_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) * (tma_contested_accesses + tma_data_sharing) / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full) + tma_store_bound / (tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_l3_miss_bound + tma_store_bound) * tma_false_sharing / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores - tma_store_latency)) + tma_machine_clears * (1 - tma_other_nukes / tma_other_nukes))",
         "MetricGroup": "BvMS;LockCont;Mem;Offcore;tma_issueSyncxn",
         "MetricName": "tma_bottleneck_memory_synchronization",
         "MetricThreshold": "tma_bottleneck_memory_synchronization > 10",
@@ -989,7 +989,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to retired misprediction by non-taken conditional branches.",
-        "MetricExpr": "cpu_core@BR_MISP_RETIRED.COND_NTAKEN_COST@ * cpu_core@BR_MISP_RETIRED.COND_NTAKEN_COST@R / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@BR_MISP_RETIRED.COND_NTAKEN@ / cpu_core@BR_MISP_RETIRED.ALL_BRANCHES@ * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_cond_nt_mispredicts",
         "MetricThreshold": "tma_cond_nt_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -998,7 +998,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to misprediction by taken conditional branches.",
-        "MetricExpr": "cpu_core@BR_MISP_RETIRED.COND_TAKEN_COST@ * cpu_core@BR_MISP_RETIRED.COND_TAKEN_COST@R / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@BR_MISP_RETIRED.COND_TAKEN@ / cpu_core@BR_MISP_RETIRED.ALL_BRANCHES@ * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_cond_tk_mispredicts",
         "MetricThreshold": "tma_cond_tk_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -1027,13 +1027,13 @@
         "Unit": "cpu_core"
     },
     {
-        "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses",
+        "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to L3 data-sharing accesses",
         "MetricConstraint": "NO_GROUP_EVENTS",
         "MetricExpr": "(cpu_core@MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD@ * min(cpu_core@MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD@R, 24 * tma_info_system_core_frequency) + cpu_core@MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD@ * min(cpu_core@MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD@R, 24 * tma_info_system_core_frequency) * (1 - cpu_core@OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM@ / (cpu_core@OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM@ + cpu_core@OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD@))) * (1 + cpu_core@MEM_LOAD_RETIRED.FB_HIT@ / cpu_core@MEM_LOAD_RETIRED.L1_MISS@ / 2) / tma_info_thread_clks",
         "MetricGroup": "BvMS;Offcore;Snoop;TopdownL4;tma_L4_group;tma_issueSyncxn;tma_l3_bound_group",
         "MetricName": "tma_data_sharing",
         "MetricThreshold": "tma_data_sharing > 0.05 & (tma_l3_bound > 0.05 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD. Related metrics: tma_bottleneck_memory_synchronization, tma_contested_accesses, tma_false_sharing, tma_machine_clears, tma_remote_cache",
+        "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to L3 data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD. Related metrics: tma_bottleneck_memory_synchronization, tma_contested_accesses, tma_false_sharing, tma_machine_clears, tma_remote_cache",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -1057,16 +1057,6 @@
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
-    {
-        "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
-        "MetricExpr": "cpu_core@MEMORY_ACTIVITY.STALLS_L3_MISS@ / tma_info_thread_clks",
-        "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_L3_group;tma_memory_bound_group",
-        "MetricName": "tma_dram_bound",
-        "MetricThreshold": "tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)",
-        "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS",
-        "ScaleUnit": "100%",
-        "Unit": "cpu_core"
-    },
     {
         "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline",
         "MetricExpr": "(cpu_core@IDQ.DSB_CYCLES_ANY@ - cpu_core@IDQ.DSB_CYCLES_OK@) / tma_info_core_core_clks / 2",
@@ -1123,7 +1113,7 @@
         "MetricGroup": "BvMB;MemoryBW;TopdownL4;tma_L4_group;tma_issueBW;tma_issueSL;tma_issueSmSt;tma_l1_bound_group",
         "MetricName": "tma_fb_full",
         "MetricThreshold": "tma_fb_full > 0.3",
-        "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_store_latency, tma_streaming_stores",
+        "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_sq_full, tma_store_latency, tma_streaming_stores, tma_uc_bound",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -1273,7 +1263,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to retired misprediction by indirect CALL instructions.",
-        "MetricExpr": "cpu_core@BR_MISP_RETIRED.INDIRECT_CALL_COST@ * cpu_core@BR_MISP_RETIRED.INDIRECT_CALL_COST@R / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@BR_MISP_RETIRED.INDIRECT_CALL@ / cpu_core@BR_MISP_RETIRED.ALL_BRANCHES@ * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_ind_call_mispredicts",
         "MetricThreshold": "tma_ind_call_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -1282,7 +1272,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to retired misprediction by indirect JMP instructions.",
-        "MetricExpr": "max((cpu_core@BR_MISP_RETIRED.INDIRECT_COST@ * cpu_core@BR_MISP_RETIRED.INDIRECT_COST@R - cpu_core@BR_MISP_RETIRED.INDIRECT_CALL_COST@ * cpu_core@BR_MISP_RETIRED.INDIRECT_CALL_COST@R) / tma_info_thread_clks, 0)",
+        "MetricExpr": "max((cpu_core@BR_MISP_RETIRED.INDIRECT@ - cpu_core@BR_MISP_RETIRED.INDIRECT_CALL@) / cpu_core@BR_MISP_RETIRED.ALL_BRANCHES@ * tma_branch_mispredicts, 0)",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_ind_jump_mispredicts",
         "MetricThreshold": "tma_ind_jump_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -1459,7 +1449,7 @@
     },
     {
         "BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)",
-        "MetricExpr": "cpu_core@IDQ.DSB_UOPS@ / cpu_core@UOPS_ISSUED.ANY@",
+        "MetricExpr": "cpu_core@IDQ.DSB_UOPS@ / (cpu_core@IDQ.DSB_UOPS@ + cpu_core@LSD.UOPS@ + cpu_core@IDQ.MITE_UOPS@ + cpu_core@IDQ.MS_UOPS@)",
         "MetricGroup": "DSB;Fed;FetchBW;tma_issueFB",
         "MetricName": "tma_info_frontend_dsb_coverage",
         "MetricThreshold": "tma_info_frontend_dsb_coverage < 0.7 & tma_info_thread_ipc / 6 > 0.35",
@@ -1526,7 +1516,7 @@
     },
     {
         "BriefDescription": "Fraction of Uops delivered by the LSD (Loop Stream Detector; aka Loop Cache)",
-        "MetricExpr": "cpu_core@LSD.UOPS@ / cpu_core@UOPS_ISSUED.ANY@",
+        "MetricExpr": "cpu_core@LSD.UOPS@ / (cpu_core@IDQ.DSB_UOPS@ + cpu_core@LSD.UOPS@ + cpu_core@IDQ.MITE_UOPS@ + cpu_core@IDQ.MS_UOPS@)",
         "MetricGroup": "Fed;LSD",
         "MetricName": "tma_info_frontend_lsd_coverage",
         "Unit": "cpu_core"
@@ -1685,6 +1675,15 @@
         "PublicDescription": "Instructions per taken branch. Related metrics: tma_dsb_switches, tma_fetch_bandwidth, tma_info_botlnk_l2_dsb_bandwidth, tma_info_botlnk_l2_dsb_misses, tma_info_frontend_dsb_coverage, tma_lcp",
         "Unit": "cpu_core"
     },
+    {
+        "BriefDescription": "AVX preserve/restore assists per kilo instruction",
+        "MetricExpr": "1e3 * cpu_core@ASSISTS.SSE_AVX_MIX@ / cpu_core@INST_RETIRED.ANY@",
+        "MetricGroup": "tma_issueMV",
+        "MetricName": "tma_info_inst_mix_vectormixpki",
+        "MetricThreshold": "tma_info_inst_mix_vectormixpki > 0.05",
+        "PublicDescription": "AVX preserve/restore assists per kilo instruction. Related metrics: tma_mixing_vectors, tma_ms_switches",
+        "Unit": "cpu_core"
+    },
     {
         "BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]",
         "MetricExpr": "tma_info_memory_l1d_cache_fill_bw",
@@ -1922,7 +1921,7 @@
         "Unit": "cpu_core"
     },
     {
-        "BriefDescription": "Mem;Backend;CacheHits",
+        "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per physical core",
         "MetricExpr": "cpu_core@UOPS_EXECUTED.THREAD@ / (cpu_core@UOPS_EXECUTED.CORE_CYCLES_GE_1@ / 2 if #SMT_on else cpu_core@UOPS_EXECUTED.THREAD\\,cmask\\=1@)",
         "MetricGroup": "Cor;Pipeline;PortsUtil;SMT",
         "MetricName": "tma_info_pipeline_execute",
@@ -2004,10 +2003,10 @@
     },
     {
         "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]",
-        "MetricExpr": "64 * (UNC_HAC_ARB_TRK_REQUESTS.ALL + UNC_HAC_ARB_COH_TRK_REQUESTS.ALL) / 1e9 / tma_info_system_time",
+        "MetricExpr": "32 * UNC_M_TOTAL_DATA / 1e9 / tma_info_system_time",
         "MetricGroup": "HPC;MemOffcore;MemoryBW;SoC;tma_issueBW",
         "MetricName": "tma_info_system_dram_bw_use",
-        "PublicDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]. Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_mem_bandwidth, tma_sq_full",
+        "PublicDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]. Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_mem_bandwidth, tma_sq_full, tma_uc_bound",
         "Unit": "cpu_core"
     },
     {
@@ -2093,13 +2092,6 @@
         "MetricName": "tma_info_system_turbo_utilization",
         "Unit": "cpu_core"
     },
-    {
-        "BriefDescription": "Measured Average Uncore Frequency for the SoC [GHz]",
-        "MetricExpr": "tma_info_system_socket_clks / 1e9 / tma_info_system_time",
-        "MetricGroup": "SoC",
-        "MetricName": "tma_info_system_uncore_frequency",
-        "Unit": "cpu_core"
-    },
     {
         "BriefDescription": "Per-Logical Processor actual clocks when the Logical Processor is active.",
         "MetricExpr": "cpu_core@CPU_CLK_UNHALTED.THREAD@",
@@ -2268,6 +2260,16 @@
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
+    {
+        "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads",
+        "MetricExpr": "cpu_core@MEMORY_ACTIVITY.STALLS_L3_MISS@ / tma_info_thread_clks",
+        "MetricGroup": "MemoryBound;Offcore;TmaL3mem;TopdownL3;tma_L3_group;tma_memory_bound_group",
+        "MetricName": "tma_l3_miss_bound",
+        "MetricThreshold": "tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2)",
+        "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS",
+        "ScaleUnit": "100%",
+        "Unit": "cpu_core"
+    },
     {
         "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)",
         "MetricExpr": "cpu_core@DECODE.LCP@ / tma_info_thread_clks",
@@ -2379,19 +2381,19 @@
     {
         "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM)",
         "MetricExpr": "min(cpu_core@CPU_CLK_UNHALTED.THREAD@, cpu_core@OFFCORE_REQUESTS_OUTSTANDING.DATA_RD\\,cmask\\=4@) / tma_info_thread_clks",
-        "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_dram_bound_group;tma_issueBW",
+        "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_issueBW;tma_l3_miss_bound_group",
         "MetricName": "tma_mem_bandwidth",
-        "MetricThreshold": "tma_mem_bandwidth > 0.2 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM).  The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_sq_full",
+        "MetricThreshold": "tma_mem_bandwidth > 0.2 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
+        "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory - DRAM ([SPR-HBM] and/or HBM).  The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_sq_full, tma_uc_bound",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
     {
         "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory - DRAM ([SPR-HBM] and/or HBM)",
         "MetricExpr": "min(cpu_core@CPU_CLK_UNHALTED.THREAD@, cpu_core@OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD@) / tma_info_thread_clks - tma_mem_bandwidth",
-        "MetricGroup": "BvML;MemoryLat;Offcore;TopdownL4;tma_L4_group;tma_dram_bound_group;tma_issueLat",
+        "MetricGroup": "BvML;MemoryLat;Offcore;TopdownL4;tma_L4_group;tma_issueLat;tma_l3_miss_bound_group",
         "MetricName": "tma_mem_latency",
-        "MetricThreshold": "tma_mem_latency > 0.1 & (tma_dram_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
+        "MetricThreshold": "tma_mem_latency > 0.1 & (tma_l3_miss_bound > 0.1 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
         "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory - DRAM ([SPR-HBM] and/or HBM).  This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that). Related metrics: tma_bottleneck_data_cache_memory_latency, tma_l3_hit_latency",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
@@ -2461,7 +2463,7 @@
         "MetricGroup": "TopdownL5;tma_L5_group;tma_issueMV;tma_ports_utilized_0_group",
         "MetricName": "tma_mixing_vectors",
         "MetricThreshold": "tma_mixing_vectors > 0.05",
-        "PublicDescription": "This metric estimates penalty in terms of percentage of([SKL+] injected blend uops out of all Uops Issued -- the Count Domain; [ADL+] cycles). Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic. Related metrics: tma_ms_switches",
+        "PublicDescription": "This metric estimates penalty in terms of percentage of([SKL+] injected blend uops out of all Uops Issued -- the Count Domain; [ADL+] cycles). Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic. Related metrics: tma_info_inst_mix_vectormixpki, tma_ms_switches",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -2480,7 +2482,7 @@
         "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_L3_group;tma_fetch_latency_group;tma_issueMC;tma_issueMS;tma_issueMV;tma_issueSO",
         "MetricName": "tma_ms_switches",
         "MetricThreshold": "tma_ms_switches > 0.05 & (tma_fetch_latency > 0.1 & tma_frontend_bound > 0.15)",
-        "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: FRONTEND_RETIRED.MS_FLOWS. Related metrics: tma_bottleneck_irregular_overhead, tma_clears_resteers, tma_l1_bound, tma_machine_clears, tma_microcode_sequencer, tma_mixing_vectors, tma_serializing_operation",
+        "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: FRONTEND_RETIRED.MS_FLOWS. Related metrics: tma_bottleneck_irregular_overhead, tma_clears_resteers, tma_info_inst_mix_vectormixpki, tma_l1_bound, tma_machine_clears, tma_microcode_sequencer, tma_mixing_vectors, tma_serializing_operation",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
@@ -2628,7 +2630,7 @@
     },
     {
         "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to retired misprediction by (indirect) RET instructions.",
-        "MetricExpr": "cpu_core@BR_MISP_RETIRED.RET_COST@ * cpu_core@BR_MISP_RETIRED.RET_COST@R / tma_info_thread_clks",
+        "MetricExpr": "cpu_core@BR_MISP_RETIRED.RET@ / cpu_core@BR_MISP_RETIRED.ALL_BRANCHES@ * tma_branch_mispredicts",
         "MetricGroup": "BrMispredicts;TopdownL3;tma_L3_group;tma_branch_mispredicts_group",
         "MetricName": "tma_ret_mispredicts",
         "MetricThreshold": "tma_ret_mispredicts > 0.05 & (tma_branch_mispredicts > 0.1 & tma_bad_speculation > 0.15)",
@@ -2703,7 +2705,7 @@
         "MetricGroup": "BvMB;MemoryBW;Offcore;TopdownL4;tma_L4_group;tma_issueBW;tma_l3_bound_group",
         "MetricName": "tma_sq_full",
         "MetricThreshold": "tma_sq_full > 0.3 & (tma_l3_bound > 0.05 & (tma_memory_bound > 0.2 & tma_backend_bound > 0.2))",
-        "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth",
+        "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). Related metrics: tma_bottleneck_data_cache_memory_bandwidth, tma_fb_full, tma_info_system_dram_bw_use, tma_mem_bandwidth, tma_uc_bound",
         "ScaleUnit": "100%",
         "Unit": "cpu_core"
     },
diff --git a/tools/perf/pmu-events/arch/x86/meteorlake/pipeline.json b/tools/perf/pmu-events/arch/x86/meteorlake/pipeline.json
index 09e1147c4733e..6cd1b1659ec7f 100644
--- a/tools/perf/pmu-events/arch/x86/meteorlake/pipeline.json
+++ b/tools/perf/pmu-events/arch/x86/meteorlake/pipeline.json
@@ -909,7 +909,7 @@
         "Counter": "0,1,2,3,4,5,6,7",
         "EventCode": "0xc0",
         "EventName": "INST_RETIRED.REP_ITERATION",
-        "PublicDescription": "Number of iterations of Repeat (REP) string retired instructions such as MOVS, CMPS, and SCAS. Each has a byte, word, and doubleword version and string instructions can be repeated using a repetition prefix, REP, that allows their architectural execution to be repeated a number of times as specified by the RCX register. Note the number of iterations is implementation-dependent.",
+        "PublicDescription": "Number of iterations of Repeat (REP) string retired instructions such as MOVS, CMPS, and SCAS. Each has a byte, word, and doubleword version and string instructions can be repeated using a repetition prefix, REP, that allows their architectural execution to be repeated a number of times as specified by the RCX register. Note: Since the number of iterations within a REP instruction can be significantly affected by fast strings, this event may vary run to run and not match the architectural number of iterations (specified by RCX)",
         "SampleAfterValue": "2000003",
         "UMask": "0x8",
         "Unit": "cpu_core"
@@ -1443,7 +1443,7 @@
         "Unit": "cpu_atom"
     },
     {
-        "BriefDescription": "Counts the number issue slots not consumed  due to a  color request for an FCW or MXCSR control register when all 4 colors (copies) are already in use.",
+        "BriefDescription": "Counts the number of issue slots not consumed due to a color request for an FCW or MXCSR control register when all 4 colors (copies) are already in use.",
         "Counter": "0,1,2,3,4,5,6,7",
         "EventCode": "0x75",
         "EventName": "SERIALIZATION.COLOR_STALLS",
diff --git a/tools/perf/pmu-events/arch/x86/meteorlake/uncore-memory.json b/tools/perf/pmu-events/arch/x86/meteorlake/uncore-memory.json
index ceb8839f07675..60bf390198d6b 100644
--- a/tools/perf/pmu-events/arch/x86/meteorlake/uncore-memory.json
+++ b/tools/perf/pmu-events/arch/x86/meteorlake/uncore-memory.json
@@ -1,61 +1,61 @@
 [
     {
-        "BriefDescription": "Counts every CAS read command sent from the Memory Controller 0 to DRAM (sum of all channels).",
+        "BriefDescription": "This event is deprecated. [This event is alias to UNC_M_MC0_RDCAS_COUNT_FREERUN]",
         "Counter": "0",
+        "Deprecated": "1",
         "EventCode": "0xff",
         "EventName": "UNC_MC0_RDCAS_COUNT_FREERUN",
         "PerPkg": "1",
-        "PublicDescription": "Counts every CAS read command sent from the Memory Controller 0 to DRAM (sum of all channels). Each CAS commands can be for 32B or 64B of data.",
         "UMask": "0x20",
         "Unit": "imc_free_running_0"
     },
     {
-        "BriefDescription": "Counts every read and write request entering the Memory Controller 0.",
+        "BriefDescription": "This event is deprecated. [This event is alias to UNC_M_MC0_TOTAL_REQCOUNT_FREERUN]",
         "Counter": "2",
+        "Deprecated": "1",
         "EventCode": "0xff",
         "EventName": "UNC_MC0_TOTAL_REQCOUNT_FREERUN",
         "PerPkg": "1",
-        "PublicDescription": "Counts every read and write request entering the Memory Controller 0 (sum of all channels). All requests are counted as one, whether they are 32B or 64B Read/Write or partial/full line writes. Some write requests to the same address may merge to a single write command to DRAM. Therefore, the total request count may be higher than total DRAM BW.",
         "UMask": "0x10",
         "Unit": "imc_free_running_0"
     },
     {
-        "BriefDescription": "Counts every CAS write command sent from the Memory Controller 0 to DRAM (sum of all channels).",
+        "BriefDescription": "This event is deprecated. [This event is alias to UNC_M_MC0_WRCAS_COUNT_FREERUN]",
         "Counter": "1",
+        "Deprecated": "1",
         "EventCode": "0xff",
         "EventName": "UNC_MC0_WRCAS_COUNT_FREERUN",
         "PerPkg": "1",
-        "PublicDescription": "Counts every CAS write command sent from the Memory Controller 0 to DRAM (sum of all channels).  Each CAS commands can be for 32B or 64B of data.",
         "UMask": "0x30",
         "Unit": "imc_free_running_0"
     },
     {
-        "BriefDescription": "Counts every CAS read command sent from the Memory Controller 1 to DRAM (sum of all channels).",
+        "BriefDescription": "This event is deprecated. [This event is alias to UNC_M_MC1_RDCAS_COUNT_FREERUN]",
         "Counter": "3",
+        "Deprecated": "1",
         "EventCode": "0xff",
         "EventName": "UNC_MC1_RDCAS_COUNT_FREERUN",
         "PerPkg": "1",
-        "PublicDescription": "Counts every CAS read command sent from the Memory Controller 1 to DRAM (sum of all channels). Each CAS commands can be for 32B or 64B of data.",
         "UMask": "0x20",
         "Unit": "imc_free_running_1"
     },
     {
-        "BriefDescription": "Counts every read and write request entering the Memory Controller 1.",
+        "BriefDescription": "This event is deprecated. [This event is alias to UNC_M_MC1_TOTAL_REQCOUNT_FREERUN]",
         "Counter": "5",
+        "Deprecated": "1",
         "EventCode": "0xff",
         "EventName": "UNC_MC1_TOTAL_REQCOUNT_FREERUN",
         "PerPkg": "1",
-        "PublicDescription": "Counts every read and write request entering the Memory Controller 1 (sum of all channels). All requests are counted as one, whether they are 32B or 64B Read/Write or partial/full line writes. Some write requests to the same address may merge to a single write command to DRAM. Therefore, the total request count may be higher than total DRAM BW.",
         "UMask": "0x10",
         "Unit": "imc_free_running_1"
     },
     {
-        "BriefDescription": "Counts every CAS write command sent from the Memory Controller 1 to DRAM (sum of all channels).",
+        "BriefDescription": "This event is deprecated. [This event is alias to UNC_M_MC1_WRCAS_COUNT_FREERUN]",
         "Counter": "4",
+        "Deprecated": "1",
         "EventCode": "0xff",
         "EventName": "UNC_MC1_WRCAS_COUNT_FREERUN",
         "PerPkg": "1",
-        "PublicDescription": "Counts every CAS write command sent from the Memory Controller 1 to DRAM (sum of all channels).  Each CAS commands can be for 32B or 64B of data.",
         "UMask": "0x30",
         "Unit": "imc_free_running_1"
     },
@@ -117,6 +117,66 @@
         "PerPkg": "1",
         "Unit": "iMC"
     },
+    {
+        "BriefDescription": "Counts every CAS read command sent from the Memory Controller 0 to DRAM (sum of all channels). [This event is alias to UNC_MC0_RDCAS_COUNT_FREERUN]",
+        "Counter": "0",
+        "EventCode": "0xff",
+        "EventName": "UNC_M_MC0_RDCAS_COUNT_FREERUN",
+        "PerPkg": "1",
+        "PublicDescription": "Counts every CAS read command sent from the Memory Controller 0 to DRAM (sum of all channels). Each CAS commands can be for 32B or 64B of data. [This event is alias to UNC_MC0_RDCAS_COUNT_FREERUN]",
+        "UMask": "0x20",
+        "Unit": "imc_free_running_0"
+    },
+    {
+        "BriefDescription": "Counts every read and write request entering the Memory Controller 0. [This event is alias to UNC_MC0_TOTAL_REQCOUNT_FREERUN]",
+        "Counter": "2",
+        "EventCode": "0xff",
+        "EventName": "UNC_M_MC0_TOTAL_REQCOUNT_FREERUN",
+        "PerPkg": "1",
+        "PublicDescription": "Counts every read and write request entering the Memory Controller 0 (sum of all channels). All requests are counted as one, whether they are 32B or 64B Read/Write or partial/full line writes. Some write requests to the same address may merge to a single write command to DRAM. Therefore, the total request count may be higher than total DRAM BW. [This event is alias to UNC_MC0_TOTAL_REQCOUNT_FREERUN]",
+        "UMask": "0x10",
+        "Unit": "imc_free_running_0"
+    },
+    {
+        "BriefDescription": "Counts every CAS write command sent from the Memory Controller 0 to DRAM (sum of all channels). [This event is alias to UNC_MC0_WRCAS_COUNT_FREERUN]",
+        "Counter": "1",
+        "EventCode": "0xff",
+        "EventName": "UNC_M_MC0_WRCAS_COUNT_FREERUN",
+        "PerPkg": "1",
+        "PublicDescription": "Counts every CAS write command sent from the Memory Controller 0 to DRAM (sum of all channels).  Each CAS commands can be for 32B or 64B of data. [This event is alias to UNC_MC0_WRCAS_COUNT_FREERUN]",
+        "UMask": "0x30",
+        "Unit": "imc_free_running_0"
+    },
+    {
+        "BriefDescription": "Counts every CAS read command sent from the Memory Controller 1 to DRAM (sum of all channels). [This event is alias to UNC_MC1_RDCAS_COUNT_FREERUN]",
+        "Counter": "3",
+        "EventCode": "0xff",
+        "EventName": "UNC_M_MC1_RDCAS_COUNT_FREERUN",
+        "PerPkg": "1",
+        "PublicDescription": "Counts every CAS read command sent from the Memory Controller 1 to DRAM (sum of all channels). Each CAS commands can be for 32B or 64B of data. [This event is alias to UNC_MC1_RDCAS_COUNT_FREERUN]",
+        "UMask": "0x20",
+        "Unit": "imc_free_running_1"
+    },
+    {
+        "BriefDescription": "Counts every read and write request entering the Memory Controller 1. [This event is alias to UNC_MC1_TOTAL_REQCOUNT_FREERUN]",
+        "Counter": "5",
+        "EventCode": "0xff",
+        "EventName": "UNC_M_MC1_TOTAL_REQCOUNT_FREERUN",
+        "PerPkg": "1",
+        "PublicDescription": "Counts every read and write request entering the Memory Controller 1 (sum of all channels). All requests are counted as one, whether they are 32B or 64B Read/Write or partial/full line writes. Some write requests to the same address may merge to a single write command to DRAM. Therefore, the total request count may be higher than total DRAM BW. [This event is alias to UNC_MC1_TOTAL_REQCOUNT_FREERUN]",
+        "UMask": "0x10",
+        "Unit": "imc_free_running_1"
+    },
+    {
+        "BriefDescription": "Counts every CAS write command sent from the Memory Controller 1 to DRAM (sum of all channels). [This event is alias to UNC_MC1_WRCAS_COUNT_FREERUN]",
+        "Counter": "4",
+        "EventCode": "0xff",
+        "EventName": "UNC_M_MC1_WRCAS_COUNT_FREERUN",
+        "PerPkg": "1",
+        "PublicDescription": "Counts every CAS write command sent from the Memory Controller 1 to DRAM (sum of all channels).  Each CAS commands can be for 32B or 64B of data. [This event is alias to UNC_MC1_WRCAS_COUNT_FREERUN]",
+        "UMask": "0x30",
+        "Unit": "imc_free_running_1"
+    },
     {
         "BriefDescription": "PRE command sent to DRAM due to page table idle timer expiration",
         "Counter": "0,1,2,3,4",
diff --git a/tools/perf/pmu-events/arch/x86/novalake/cache.json b/tools/perf/pmu-events/arch/x86/novalake/cache.json
new file mode 100644
index 0000000000000..106c50fa6eb48
--- /dev/null
+++ b/tools/perf/pmu-events/arch/x86/novalake/cache.json
@@ -0,0 +1,749 @@
+[
+    {
+        "BriefDescription": "Counts the number of requests that were not accepted into the L2Q because the L2Q is FULL.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x31",
+        "EventName": "CORE_REJECT_L2Q.ANY",
+        "PublicDescription": "Counts the number of (demand and L1 prefetchers) core requests rejected by the L2Q due to a full or nearly full w condition which likely indicates back pressure from L2Q.  It also counts requests that would have gone directly to the XQ, but are rejected due to a full or nearly full condition, indicating back pressure from the IDI link.  The L2Q may also reject transactions  from a core to insure fairness between cores, or to delay a core?s dirty eviction when the address conflicts incoming external snoops.  (Note that L2 prefetcher requests that are dropped are not counted by this event.) Counts on a per core basis.",
+        "SampleAfterValue": "1000003",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of cache lines replaced in L0 data cache.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x51",
+        "EventName": "L1D.L0_REPLACEMENT",
+        "PublicDescription": "Counts L0 data line replacements including opportunistic replacements, and replacements that require stall-for-replace or block-for-replace.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cachelines replaced into the L1 d-cache. Successful replacements only (not blocked) and exclude WB-miss case",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x51",
+        "EventName": "L1D.L1_REPLACEMENT",
+        "PublicDescription": "Counts cachelines replaced into the L1 d-cache.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cachelines replaced into the L0 and L1 d-cache. Successful replacements only (not blocked) and exclude WB-miss case",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x51",
+        "EventName": "L1D.REPLACEMENT",
+        "PublicDescription": "Counts cachelines replaced into the L0 and L1 d-cache.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x5",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of cycles a demand request has waited due to L1D Fill Buffer (FB) unavailability.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x49",
+        "EventName": "L1D_MISS.FB_FULL",
+        "PublicDescription": "Counts number of cycles a demand request has waited due to L1D Fill Buffer (FB) unavailability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of cycles a demand request has waited due to L1D due to lack of L2 resources.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x49",
+        "EventName": "L1D_MISS.L2_STALLS",
+        "PublicDescription": "Counts number of cycles a demand request has waited due to L1D due to lack of L2 resources. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of demand requests that missed L1D cache",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x49",
+        "EventName": "L1D_MISS.LOAD",
+        "PublicDescription": "Count occurrences (rising-edge) of DCACHE_PENDING sub-event0. Impl. sends per-port binary inc-bit the occupancy increases* (at FB alloc or promotion).",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of L1D misses that are outstanding",
+        "Counter": "0,1,2,3",
+        "EventCode": "0x48",
+        "EventName": "L1D_PENDING.LOAD",
+        "PublicDescription": "Counts number of L1D misses that are outstanding in each cycle, that is each cycle the number of Fill Buffers (FB) outstanding required by Demand Reads. FB either is held by demand loads, or it is held by non-demand loads and gets hit at least once by demand. The valid outstanding interval is defined until the FB deallocation by one of the following ways: from FB allocation, if FB is allocated by demand from the demand Hit FB, if it is allocated by hardware or software prefetch. Note: In the L1D, a Demand Read contains cacheable or noncacheable demand loads, including ones causing cache-line splits and reads due to page walks resulted from any request type.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles with L1D load Misses outstanding.",
+        "Counter": "0,1,2,3",
+        "CounterMask": "1",
+        "EventCode": "0x48",
+        "EventName": "L1D_PENDING.LOAD_CYCLES",
+        "PublicDescription": "Counts duration of L1D miss outstanding in cycles.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "L2 cache lines filling L2",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x25",
+        "EventName": "L2_LINES_IN.ALL",
+        "PublicDescription": "Counts the number of L2 cache lines filling the L2. Counting does not cover rejects.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x3f",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Modified cache lines that are evicted by L2 cache when triggered by an L2 cache fill.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x26",
+        "EventName": "L2_LINES_OUT.NON_SILENT",
+        "PublicDescription": "Counts the number of lines that are evicted by L2 cache when triggered by an L2 cache fill. Those lines are in Modified state. Modified lines are written back to L3",
+        "SampleAfterValue": "200003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Non-modified cache lines that are silently dropped by L2 cache.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x26",
+        "EventName": "L2_LINES_OUT.SILENT",
+        "PublicDescription": "Counts the number of lines that are silently dropped by L2 cache. These lines are typically in Shared or Exclusive state. A non-threaded event.",
+        "SampleAfterValue": "200003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of L2 cache lines that have been L2 hardware prefetched but not used by demand accesses.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x26",
+        "EventName": "L2_LINES_OUT.USELESS_HWPF",
+        "PublicDescription": "Counts the number of L2 cache lines that have been L2 hardware prefetched but not used by demand accesses.  Increments on the core that brought the line in originally.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Cache lines that have been L2 hardware prefetched but not used by demand accesses",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x26",
+        "EventName": "L2_LINES_OUT.USELESS_HWPF",
+        "PublicDescription": "Counts the number of cache lines that have been prefetched by the L2 hardware prefetcher but not used by demand access when evicted from the L2 cache",
+        "SampleAfterValue": "200003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "All accesses to L2 cache [This event is alias to L2_RQSTS.REFERENCES, L2_RQSTS.ANY]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_REQUEST.ALL",
+        "PublicDescription": "Counts all requests that were hit or true misses in L2 cache. True-miss excludes misses that were merged with ongoing L2 misses. [This event is alias to L2_RQSTS.REFERENCES, L2_RQSTS.ANY]",
+        "SampleAfterValue": "200003",
+        "UMask": "0xff",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "L2 code requests [This event is alias to L2_RQSTS.ALL_CODE_RD]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_REQUEST.ALL_CODE_RD",
+        "PublicDescription": "Counts the total number of L2 code requests. [This event is alias to L2_RQSTS.ALL_CODE_RD]",
+        "SampleAfterValue": "200003",
+        "UMask": "0xe4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Demand Data Read access L2 cache [This event is alias to L2_RQSTS.ALL_DEMAND_DATA_RD]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_REQUEST.ALL_DEMAND_DATA_RD",
+        "PublicDescription": "Counts Demand Data Read requests accessing the L2 cache. These requests may hit or miss L2 cache. True-miss exclude misses that were merged with ongoing L2 misses. An access is counted once. [This event is alias to L2_RQSTS.ALL_DEMAND_DATA_RD]",
+        "SampleAfterValue": "200003",
+        "UMask": "0xe1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "L2 cache hits when fetching instructions, code reads. [This event is alias to L2_RQSTS.CODE_RD_HIT]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_REQUEST.CODE_RD_HIT",
+        "PublicDescription": "Counts L2 cache hits when fetching instructions, code reads. [This event is alias to L2_RQSTS.CODE_RD_HIT]",
+        "SampleAfterValue": "200003",
+        "UMask": "0x44",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "L2 cache misses when fetching instructions [This event is alias to L2_RQSTS.CODE_RD_MISS]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_REQUEST.CODE_RD_MISS",
+        "PublicDescription": "Counts L2 cache misses when fetching instructions. [This event is alias to L2_RQSTS.CODE_RD_MISS]",
+        "SampleAfterValue": "200003",
+        "UMask": "0x24",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Demand Data Read requests that hit L2 cache [This event is alias to L2_RQSTS.DEMAND_DATA_RD_HIT]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_REQUEST.DEMAND_DATA_RD_HIT",
+        "PublicDescription": "Counts the number of demand Data Read requests initiated by load instructions that hit L2 cache. [This event is alias to L2_RQSTS.DEMAND_DATA_RD_HIT]",
+        "SampleAfterValue": "200003",
+        "UMask": "0x41",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Demand Data Read miss L2 cache [This event is alias to L2_RQSTS.DEMAND_DATA_RD_MISS]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_REQUEST.DEMAND_DATA_RD_MISS",
+        "PublicDescription": "Counts demand Data Read requests with true-miss in the L2 cache. True-miss excludes misses that were merged with ongoing L2 misses. An access is counted once. [This event is alias to L2_RQSTS.DEMAND_DATA_RD_MISS]",
+        "SampleAfterValue": "200003",
+        "UMask": "0x21",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "All requests that hit L2 cache. [This event is alias to L2_RQSTS.HIT]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_REQUEST.HIT",
+        "PublicDescription": "Counts all requests that hit L2 cache. [This event is alias to L2_RQSTS.HIT]",
+        "SampleAfterValue": "200003",
+        "UMask": "0x5f",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Read requests with true-miss in L2 cache [This event is alias to L2_RQSTS.MISS]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_REQUEST.MISS",
+        "PublicDescription": "Counts read requests of any type with true-miss in the L2 cache. True-miss excludes L2 misses that were merged with ongoing L2 misses. [This event is alias to L2_RQSTS.MISS]",
+        "SampleAfterValue": "200003",
+        "UMask": "0x3f",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "RFO requests that hit L2 cache [This event is alias to L2_RQSTS.RFO_HIT]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_REQUEST.RFO_HIT",
+        "PublicDescription": "Counts the RFO (Read-for-Ownership) requests that hit L2 cache. [This event is alias to L2_RQSTS.RFO_HIT]",
+        "SampleAfterValue": "200003",
+        "UMask": "0x42",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "RFO requests that miss L2 cache [This event is alias to L2_RQSTS.RFO_MISS]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_REQUEST.RFO_MISS",
+        "PublicDescription": "Counts the RFO (Read-for-Ownership) requests that miss L2 cache. [This event is alias to L2_RQSTS.RFO_MISS]",
+        "SampleAfterValue": "200003",
+        "UMask": "0x22",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "L2 code requests [This event is alias to L2_REQUEST.ALL_CODE_RD]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_RQSTS.ALL_CODE_RD",
+        "PublicDescription": "Counts the total number of L2 code requests. [This event is alias to L2_REQUEST.ALL_CODE_RD]",
+        "SampleAfterValue": "200003",
+        "UMask": "0xe4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Demand Data Read access L2 cache [This event is alias to L2_REQUEST.ALL_DEMAND_DATA_RD]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_RQSTS.ALL_DEMAND_DATA_RD",
+        "PublicDescription": "Counts Demand Data Read requests accessing the L2 cache. These requests may hit or miss L2 cache. True-miss exclude misses that were merged with ongoing L2 misses. An access is counted once. [This event is alias to L2_REQUEST.ALL_DEMAND_DATA_RD]",
+        "SampleAfterValue": "200003",
+        "UMask": "0xe1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "All accesses to L2 cache [This event is alias to L2_RQSTS.REFERENCES, L2_REQUEST.ALL]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_RQSTS.ANY",
+        "PublicDescription": "Counts all requests that were hit or true misses in L2 cache. True-miss excludes misses that were merged with ongoing L2 misses. [This event is alias to L2_RQSTS.REFERENCES, L2_REQUEST.ALL]",
+        "SampleAfterValue": "200003",
+        "UMask": "0xff",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "L2 cache hits when fetching instructions, code reads. [This event is alias to L2_REQUEST.CODE_RD_HIT]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_RQSTS.CODE_RD_HIT",
+        "PublicDescription": "Counts L2 cache hits when fetching instructions, code reads. [This event is alias to L2_REQUEST.CODE_RD_HIT]",
+        "SampleAfterValue": "200003",
+        "UMask": "0x44",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "L2 cache misses when fetching instructions [This event is alias to L2_REQUEST.CODE_RD_MISS]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_RQSTS.CODE_RD_MISS",
+        "PublicDescription": "Counts L2 cache misses when fetching instructions. [This event is alias to L2_REQUEST.CODE_RD_MISS]",
+        "SampleAfterValue": "200003",
+        "UMask": "0x24",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Demand Data Read requests that hit L2 cache [This event is alias to L2_REQUEST.DEMAND_DATA_RD_HIT]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_RQSTS.DEMAND_DATA_RD_HIT",
+        "PublicDescription": "Counts the number of demand Data Read requests initiated by load instructions that hit L2 cache. [This event is alias to L2_REQUEST.DEMAND_DATA_RD_HIT]",
+        "SampleAfterValue": "200003",
+        "UMask": "0x41",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Demand Data Read miss L2 cache [This event is alias to L2_REQUEST.DEMAND_DATA_RD_MISS]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_RQSTS.DEMAND_DATA_RD_MISS",
+        "PublicDescription": "Counts demand Data Read requests with true-miss in the L2 cache. True-miss excludes misses that were merged with ongoing L2 misses. An access is counted once. [This event is alias to L2_REQUEST.DEMAND_DATA_RD_MISS]",
+        "SampleAfterValue": "200003",
+        "UMask": "0x21",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "All requests that hit L2 cache. [This event is alias to L2_REQUEST.HIT]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_RQSTS.HIT",
+        "PublicDescription": "Counts all requests that hit L2 cache. [This event is alias to L2_REQUEST.HIT]",
+        "SampleAfterValue": "200003",
+        "UMask": "0x5f",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Read requests with true-miss in L2 cache [This event is alias to L2_REQUEST.MISS]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_RQSTS.MISS",
+        "PublicDescription": "Counts read requests of any type with true-miss in the L2 cache. True-miss excludes L2 misses that were merged with ongoing L2 misses. [This event is alias to L2_REQUEST.MISS]",
+        "SampleAfterValue": "200003",
+        "UMask": "0x3f",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "All accesses to L2 cache [This event is alias to L2_REQUEST.ALL,L2_RQSTS.ANY]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_RQSTS.REFERENCES",
+        "PublicDescription": "Counts all requests that were hit or true misses in L2 cache. True-miss excludes misses that were merged with ongoing L2 misses. [This event is alias to L2_REQUEST.ALL,L2_RQSTS.ANY]",
+        "SampleAfterValue": "200003",
+        "UMask": "0xff",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "RFO requests that hit L2 cache [This event is alias to L2_REQUEST.RFO_HIT]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_RQSTS.RFO_HIT",
+        "PublicDescription": "Counts the RFO (Read-for-Ownership) requests that hit L2 cache. [This event is alias to L2_REQUEST.RFO_HIT]",
+        "SampleAfterValue": "200003",
+        "UMask": "0x42",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "RFO requests that miss L2 cache [This event is alias to L2_REQUEST.RFO_MISS]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x24",
+        "EventName": "L2_RQSTS.RFO_MISS",
+        "PublicDescription": "Counts the RFO (Read-for-Ownership) requests that miss L2 cache. [This event is alias to L2_REQUEST.RFO_MISS]",
+        "SampleAfterValue": "200003",
+        "UMask": "0x22",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "L1D writebacks that access L2 cache",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x23",
+        "EventName": "L2_TRANS.L1D_WB",
+        "PublicDescription": "This event counts L1D writebacks that access L2 cache.",
+        "SampleAfterValue": "200003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of cacheable memory requests that miss in the LLC. Counts on a per core basis.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x2e",
+        "EventName": "LONGEST_LAT_CACHE.MISS",
+        "PublicDescription": "Counts the number of cacheable memory requests that miss in the Last Level Cache (LLC). Requests include demand loads, reads for ownership (RFO), instruction fetches and L1 HW prefetches. If the core has access to an L3 cache, the LLC is the L3 cache, otherwise it is the L2 cache. Counts on a per core basis.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x41",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Core-originated cacheable requests that missed L3  (Except hardware prefetches to the L3)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x2e",
+        "EventName": "LONGEST_LAT_CACHE.MISS",
+        "PublicDescription": "Counts core-originated cacheable requests that miss the L3 cache (Longest Latency cache). Requests include data and code reads, Reads-for-Ownership (RFOs), speculative accesses and hardware prefetches to the L1 and L2.  It does not include hardware prefetches to the L3, and may not count other types of requests to the L3.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x41",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of cacheable memory requests that access the LLC. Counts on a per core basis.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x2e",
+        "EventName": "LONGEST_LAT_CACHE.REFERENCE",
+        "PublicDescription": "Counts the number of cacheable memory requests that access the Last Level Cache (LLC). Requests include demand loads, reads for ownership (RFO), instruction fetches and L1 HW prefetches. If the core has access to an L3 cache, the LLC is the L3 cache, otherwise it is the L2 cache. Counts on a per core basis.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4f",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts all retired load instructions.",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xd0",
+        "EventName": "MEM_INST_RETIRED.ALL_LOADS",
+        "PublicDescription": "Counts Instructions with at least one architecturally visible load retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x81",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired store instructions.",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xd0",
+        "EventName": "MEM_INST_RETIRED.ALL_STORES",
+        "PublicDescription": "Counts all retired store instructions. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x82",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired software prefetch instructions.",
+        "Counter": "0,1,2,3",
+        "EventCode": "0xd0",
+        "EventName": "MEM_INST_RETIRED.ALL_SWPF",
+        "PublicDescription": "Counts all retired software prefetch instructions. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x84",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "All retired memory instructions.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xd0",
+        "EventName": "MEM_INST_RETIRED.ANY",
+        "PublicDescription": "Counts all retired memory instructions - loads and stores. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x87",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired load instructions with locked access.",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xd0",
+        "EventName": "MEM_INST_RETIRED.LOCK_LOADS",
+        "PublicDescription": "Counts retired load instructions with locked access. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x21",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired load instructions that split across a cacheline boundary.",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xd0",
+        "EventName": "MEM_INST_RETIRED.SPLIT_LOADS",
+        "PublicDescription": "Counts retired load instructions that split across a cacheline boundary. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100003",
+        "UMask": "0x41",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired store instructions that split across a cacheline boundary.",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xd0",
+        "EventName": "MEM_INST_RETIRED.SPLIT_STORES",
+        "PublicDescription": "Counts retired store instructions that split across a cacheline boundary. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100003",
+        "UMask": "0x42",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired load instructions that hit the STLB.",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xd0",
+        "EventName": "MEM_INST_RETIRED.STLB_HIT_LOADS",
+        "PublicDescription": "Number of retired load instructions with a clean hit in the 2nd-level TLB (STLB). Available PDIST counters: 0,1",
+        "SampleAfterValue": "100003",
+        "UMask": "0x9",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired store instructions that hit the STLB.",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xd0",
+        "EventName": "MEM_INST_RETIRED.STLB_HIT_STORES",
+        "PublicDescription": "Number of retired store instructions that hit in the 2nd-level TLB (STLB). Available PDIST counters: 0,1",
+        "SampleAfterValue": "100003",
+        "UMask": "0xa",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired load instructions that miss the STLB.",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xd0",
+        "EventName": "MEM_INST_RETIRED.STLB_MISS_LOADS",
+        "PublicDescription": "Number of retired load instructions that (start a) miss in the 2nd-level TLB (STLB). Available PDIST counters: 0,1",
+        "SampleAfterValue": "100003",
+        "UMask": "0x11",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired store instructions that miss the STLB.",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xd0",
+        "EventName": "MEM_INST_RETIRED.STLB_MISS_STORES",
+        "PublicDescription": "Number of retired store instructions that (start a) miss in the 2nd-level TLB (STLB). Available PDIST counters: 0,1",
+        "SampleAfterValue": "100003",
+        "UMask": "0x12",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "MEM_LOAD_L2_MISS_RETIRED.L3_HIT_SAME_CBB",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xD6",
+        "EventName": "MEM_LOAD_L2_MISS_RETIRED.L3_HIT_SAME_CBB",
+        "MSRIndex": "0x3E0,0x3E1,0x3E2,0x3E3",
+        "MSRValue": "0xED000400000001",
+        "PublicDescription": "MEM_LOAD_L2_MISS_RETIRED.L3_HIT_SAME_CBB Available PDIST counters: 0,1",
+        "SampleAfterValue": "100021",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "MEM_LOAD_L2_MISS_RETIRED.L3_HIT_SAME_CBB_SNP_HIT_NO_FWD",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xD6",
+        "EventName": "MEM_LOAD_L2_MISS_RETIRED.L3_HIT_SAME_CBB_SNP_HIT_NO_FWD",
+        "MSRIndex": "0x3E0,0x3E1,0x3E2,0x3E3",
+        "MSRValue": "0x4D000400000001",
+        "PublicDescription": "MEM_LOAD_L2_MISS_RETIRED.L3_HIT_SAME_CBB_SNP_HIT_NO_FWD Available PDIST counters: 0,1",
+        "SampleAfterValue": "100021",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "MEM_LOAD_L2_MISS_RETIRED.L3_MISS",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xD6",
+        "EventName": "MEM_LOAD_L2_MISS_RETIRED.L3_MISS",
+        "MSRIndex": "0x3E0,0x3E1,0x3E2,0x3E3",
+        "MSRValue": "0xFF03F000000001",
+        "PublicDescription": "MEM_LOAD_L2_MISS_RETIRED.L3_MISS Available PDIST counters: 0,1",
+        "SampleAfterValue": "100021",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "MEM_LOAD_L2_MISS_RETIRED.MEM_REGION_1",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xD6",
+        "EventName": "MEM_LOAD_L2_MISS_RETIRED.MEM_REGION_1",
+        "MSRIndex": "0x3E0,0x3E1,0x3E2,0x3E3",
+        "MSRValue": "0xF5020000000001",
+        "PublicDescription": "MEM_LOAD_L2_MISS_RETIRED.MEM_REGION_1 Available PDIST counters: 0,1",
+        "SampleAfterValue": "100021",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired instructions with at least 1 uncacheable load or lock.",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xd4",
+        "EventName": "MEM_LOAD_MISC_RETIRED.UC",
+        "PublicDescription": "Retired instructions with at least one load to uncacheable memory-type, or at least one cache-line split locked access (Bus Lock). Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of completed demand load requests that missed the L1, but hit the FB(fill buffer), because a preceding miss to the same cacheline initiated the line to be brought into L1, but data is not yet ready in L1.",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xd1",
+        "EventName": "MEM_LOAD_RETIRED.FB_HIT",
+        "PublicDescription": "Counts retired load instructions with at least one uop was load missed in L1 but hit FB (Fill Buffers) due to preceding miss to the same cache line with data not ready. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired load instructions with L1 cache hits as data sources",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xd1",
+        "EventName": "MEM_LOAD_RETIRED.L1_HIT",
+        "PublicDescription": "Counts retired load instructions with at least one uop that hit in the L1 data cache. This event includes all SW prefetches and lock instructions regardless of the data source. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x6",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts retired load instructions with at least one uop that hit in the Level 0 of the L1 data cache. This event includes all SW prefetches and lock instructions regardless of the data source.",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xd1",
+        "EventName": "MEM_LOAD_RETIRED.L1_HIT_L0",
+        "PublicDescription": "Counts retired load instructions with at least one uop that hit in the Level 0 of the L1 data cache. This event includes all SW prefetches and lock instructions regardless of the data source. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts retired load instructions with at least one uop that hit in the Level 1 of the L1 data cache.",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xd1",
+        "EventName": "MEM_LOAD_RETIRED.L1_HIT_L1",
+        "PublicDescription": "Counts retired load instructions with at least one uop that hit in the Level 1 of the L1 data cache. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired load instructions missed L1 cache as data sources",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xd1",
+        "EventName": "MEM_LOAD_RETIRED.L1_MISS",
+        "PublicDescription": "Counts retired load instructions with at least one uop that missed in the L1 cache. Available PDIST counters: 0,1",
+        "SampleAfterValue": "200003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired instruction with at least 1 load uop whose Data Source was: Hit in L2 cache in a local slice of MLC of DCM.",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xd1",
+        "EventName": "MEM_LOAD_RETIRED.L2_HIT_LOCAL",
+        "PublicDescription": "Retired instruction with at least 1 load uop whose Data Source was: Hit in L2 cache in a local slice of MLC of DCM. Available PDIST counters: 0,1",
+        "SampleAfterValue": "50021",
+        "UMask": "0x20",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired load instructions missed L2 cache as data sources",
+        "Counter": "0,1,2,3",
+        "Data_LA": "1",
+        "EventCode": "0xd1",
+        "EventName": "MEM_LOAD_RETIRED.L2_MISS",
+        "PublicDescription": "Counts retired load instructions missed L2 cache as data sources. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100021",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "MEM_STORE_RETIRED.L2_HIT_LOCAL",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x44",
+        "EventName": "MEM_STORE_RETIRED.L2_HIT_LOCAL",
+        "SampleAfterValue": "200003",
+        "UMask": "0x20",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired memory uops for any access",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe5",
+        "EventName": "MEM_UOP_RETIRED.ANY",
+        "PublicDescription": "Number of retired micro-operations (uops) for load or store memory accesses",
+        "SampleAfterValue": "1000003",
+        "UMask": "0xf",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts bus locks, accounts for cache line split locks and UC locks.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x2c",
+        "EventName": "SQ_MISC.BUS_LOCK",
+        "PublicDescription": "Counts the more expensive bus lock needed to enforce cache coherency for certain memory accesses that need to be done atomically.  Can be created by issuing an atomic instruction (via the LOCK prefix) which causes a cache line split or accesses uncacheable memory.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of PREFETCHNTA, PREFETCHW, PREFETCHT0, PREFETCHT1 or PREFETCHT2 instructions executed.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x40",
+        "EventName": "SW_PREFETCH_ACCESS.ANY",
+        "SampleAfterValue": "100003",
+        "UMask": "0xf",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of PREFETCHNTA instructions executed.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x40",
+        "EventName": "SW_PREFETCH_ACCESS.NTA",
+        "PublicDescription": "Counts the number of PREFETCHNTA instructions executed.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of PREFETCHW instructions executed.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x40",
+        "EventName": "SW_PREFETCH_ACCESS.PREFETCHW",
+        "PublicDescription": "Counts the number of PREFETCHW instructions executed.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of PREFETCHT0 instructions executed.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x40",
+        "EventName": "SW_PREFETCH_ACCESS.T0",
+        "PublicDescription": "Counts the number of PREFETCHT0 instructions executed.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of PREFETCHT1 or PREFETCHT2 instructions executed.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x40",
+        "EventName": "SW_PREFETCH_ACCESS.T1_T2",
+        "PublicDescription": "Counts the number of PREFETCHT1 or PREFETCHT2 instructions executed.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    }
+]
diff --git a/tools/perf/pmu-events/arch/x86/novalake/counter.json b/tools/perf/pmu-events/arch/x86/novalake/counter.json
new file mode 100644
index 0000000000000..6cdc63552c67c
--- /dev/null
+++ b/tools/perf/pmu-events/arch/x86/novalake/counter.json
@@ -0,0 +1,17 @@
+[
+    {
+        "Unit": "cpu_atom",
+        "CountersNumFixed": "7",
+        "CountersNumGeneric": "8"
+    },
+    {
+        "Unit": "cpu_core",
+        "CountersNumFixed": "4",
+        "CountersNumGeneric": "8"
+    },
+    {
+        "Unit": "iMC",
+        "CountersNumFixed": "0",
+        "CountersNumGeneric": "5"
+    }
+]
\ No newline at end of file
diff --git a/tools/perf/pmu-events/arch/x86/novalake/floating-point.json b/tools/perf/pmu-events/arch/x86/novalake/floating-point.json
new file mode 100644
index 0000000000000..471a90f3670c4
--- /dev/null
+++ b/tools/perf/pmu-events/arch/x86/novalake/floating-point.json
@@ -0,0 +1,452 @@
+[
+    {
+        "BriefDescription": "Counts the number of cycles when any of the floating point dividers are active.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EventCode": "0xcd",
+        "EventName": "ARITH.FPDIV_ACTIVE",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Cycles when floating-point divide unit is busy executing divide or square root operations.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EventCode": "0xb0",
+        "EventName": "ARITH.FPDIV_ACTIVE",
+        "PublicDescription": "Counts cycles when divide unit is busy executing divide or square root operations. Accounts for floating-point operations only.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of active floating point dividers per cycle in the loop stage.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xcd",
+        "EventName": "ARITH.FPDIV_OCCUPANCY",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of floating point divider uops executed per cycle.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xcd",
+        "EventName": "ARITH.FPDIV_UOPS",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts all microcode FP assists.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc1",
+        "EventName": "ASSISTS.FP",
+        "PublicDescription": "Counts all microcode Floating Point assists.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "ASSISTS.SSE_AVX_MIX",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc1",
+        "EventName": "ASSISTS.SSE_AVX_MIX",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "FP_ARITH_DISPATCHED.V0",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "FP_ARITH_DISPATCHED.V0",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "FP_ARITH_DISPATCHED.V1",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "FP_ARITH_DISPATCHED.V1",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "FP_ARITH_DISPATCHED.V2",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "FP_ARITH_DISPATCHED.V2",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "FP_ARITH_DISPATCHED.V3",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "FP_ARITH_DISPATCHED.V3",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "FP_ARITH_DISPATCHED.V4",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "FP_ARITH_DISPATCHED.V4",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "FP_ARITH_DISPATCHED.V5",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "FP_ARITH_DISPATCHED.V5",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x20",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts number of SSE/AVX computational 128-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 2 computation operations, one for each element.  Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT DPP FM(N)ADD/SUB.  DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc8",
+        "EventName": "FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE",
+        "PublicDescription": "Number of SSE/AVX computational 128-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 2 computation operations, one for each element.  Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT DPP FM(N)ADD/SUB.  DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of SSE/AVX computational 128-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 4 computation operations, one for each element.  Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB MUL DIV MIN MAX RCP14 RSQRT14 SQRT DPP FM(N)ADD/SUB.  DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc8",
+        "EventName": "FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE",
+        "PublicDescription": "Number of SSE/AVX computational 128-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 4 computation operations, one for each element.  Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT RSQRT RCP DPP FM(N)ADD/SUB.  DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts number of SSE/AVX computational 256-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 4 computation operations, one for each element.  Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT FM(N)ADD/SUB.  FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc8",
+        "EventName": "FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE",
+        "PublicDescription": "Number of SSE/AVX computational 256-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 4 computation operations, one for each element.  Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT FM(N)ADD/SUB.  FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts number of SSE/AVX computational 256-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 8 computation operations, one for each element.  Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT RSQRT RCP DPP FM(N)ADD/SUB.  DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc8",
+        "EventName": "FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE",
+        "PublicDescription": "Number of SSE/AVX computational 256-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 8 computation operations, one for each element.  Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT RSQRT RCP DPP FM(N)ADD/SUB.  DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x20",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts number of SSE/AVX computational 512-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 8 computation operations, one for each element.  Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT14 RCP14 FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc8",
+        "EventName": "FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE",
+        "PublicDescription": "Number of SSE/AVX computational 512-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 8 computation operations, one for each element.  Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT14 RCP14 FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x40",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts number of SSE/AVX computational 512-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 16 computation operations, one for each element.  Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT14 RCP14 FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc8",
+        "EventName": "FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE",
+        "PublicDescription": "Number of SSE/AVX computational 512-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 16 computation operations, one for each element.  Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT14 RCP14 FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x80",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts number of SSE/AVX computational scalar double precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 1 computational operation. Applies to SSE* and AVX* scalar double precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT FM(N)ADD/SUB.  FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc8",
+        "EventName": "FP_ARITH_INST_RETIRED.SCALAR_DOUBLE",
+        "PublicDescription": "Number of SSE/AVX computational scalar double precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 1 computational operation. Applies to SSE* and AVX* scalar double precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT FM(N)ADD/SUB.  FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts number of SSE/AVX computational scalar single precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 1 computational operation. Applies to SSE* and AVX* scalar single precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT RCP FM(N)ADD/SUB.  FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc8",
+        "EventName": "FP_ARITH_INST_RETIRED.SCALAR_SINGLE",
+        "PublicDescription": "Number of SSE/AVX computational scalar single precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 1 computational operation. Applies to SSE* and AVX* scalar single precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT RCP FM(N)ADD/SUB.  FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "FP_ARITH_INST_RETIRED.VECTOR_128B",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc8",
+        "EventName": "FP_ARITH_INST_RETIRED.VECTOR_128B",
+        "SampleAfterValue": "100003",
+        "UMask": "0x40c",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "FP_ARITH_INST_RETIRED.VECTOR_256B",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc8",
+        "EventName": "FP_ARITH_INST_RETIRED.VECTOR_256B",
+        "SampleAfterValue": "100003",
+        "UMask": "0x830",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "FP_ARITH_INST_RETIRED.VECTOR_512B",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc8",
+        "EventName": "FP_ARITH_INST_RETIRED.VECTOR_512B",
+        "SampleAfterValue": "100003",
+        "UMask": "0x10c0",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts number of SSE/AVX computational 128-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 2 computation operations, one for each element.  Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT DPP FM(N)ADD/SUB.  DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc7",
+        "EventName": "FP_ARITH_OPS_RETIRED.128B_PACKED_DOUBLE",
+        "PublicDescription": "Number of SSE/AVX computational 128-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 2 computation operations, one for each element.  Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT DPP FM(N)ADD/SUB.  DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of SSE/AVX computational 128-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 4 computation operations, one for each element.  Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB MUL DIV MIN MAX RCP14 RSQRT14 SQRT DPP FM(N)ADD/SUB.  DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc7",
+        "EventName": "FP_ARITH_OPS_RETIRED.128B_PACKED_SINGLE",
+        "PublicDescription": "Number of SSE/AVX computational 128-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 4 computation operations, one for each element.  Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT RSQRT RCP DPP FM(N)ADD/SUB.  DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts number of SSE/AVX computational 256-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 4 computation operations, one for each element.  Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT FM(N)ADD/SUB.  FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc7",
+        "EventName": "FP_ARITH_OPS_RETIRED.256B_PACKED_DOUBLE",
+        "PublicDescription": "Number of SSE/AVX computational 256-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 4 computation operations, one for each element.  Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT FM(N)ADD/SUB.  FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts number of SSE/AVX computational 256-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 8 computation operations, one for each element.  Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT RSQRT RCP DPP FM(N)ADD/SUB.  DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc7",
+        "EventName": "FP_ARITH_OPS_RETIRED.256B_PACKED_SINGLE",
+        "PublicDescription": "Number of SSE/AVX computational 256-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 8 computation operations, one for each element.  Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT RSQRT RCP DPP FM(N)ADD/SUB.  DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x20",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of SSE/AVX computational 128-bit packed single and 256-bit packed double precision FP instructions retired; some instructions will count twice as noted below.  Each count represents 2 or/and 4 computation operations, 1 for each element.  Applies to SSE* and AVX* packed single precision and packed double precision FP instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX RCP14 RSQRT14 SQRT DPP FM(N)ADD/SUB.  DPP and FM(N)ADD/SUB count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc7",
+        "EventName": "FP_ARITH_OPS_RETIRED.4_FLOPS",
+        "PublicDescription": "Number of SSE/AVX computational 128-bit packed single precision and 256-bit packed double precision  floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 2 or/and 4 computation operations, one for each element.  Applies to SSE* and AVX* packed single precision floating-point and packed double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX RCP14 RSQRT14 SQRT DPP FM(N)ADD/SUB.  DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x18",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts number of SSE/AVX computational 512-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 8 computation operations, one for each element.  Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT14 RCP14 FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc7",
+        "EventName": "FP_ARITH_OPS_RETIRED.512B_PACKED_DOUBLE",
+        "PublicDescription": "Number of SSE/AVX computational 512-bit packed double precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 8 computation operations, one for each element.  Applies to SSE* and AVX* packed double precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT14 RCP14 FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x40",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts number of SSE/AVX computational 512-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 16 computation operations, one for each element.  Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT14 RCP14 FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc7",
+        "EventName": "FP_ARITH_OPS_RETIRED.512B_PACKED_SINGLE",
+        "PublicDescription": "Number of SSE/AVX computational 512-bit packed single precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 16 computation operations, one for each element.  Applies to SSE* and AVX* packed single precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT14 RCP14 FM(N)ADD/SUB. FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x80",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of SSE/AVX computational 256-bit packed single precision and 512-bit packed double precision  FP instructions retired; some instructions will count twice as noted below.  Each count represents 8 computation operations, 1 for each element.  Applies to SSE* and AVX* packed single precision and double precision FP instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT RSQRT RSQRT14 RCP RCP14 DPP FM(N)ADD/SUB.  DPP and FM(N)ADD/SUB count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc7",
+        "EventName": "FP_ARITH_OPS_RETIRED.8_FLOPS",
+        "PublicDescription": "Number of SSE/AVX computational 256-bit packed single precision and 512-bit packed double precision  floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 8 computation operations, one for each element.  Applies to SSE* and AVX* packed single precision and double precision floating-point instructions: ADD SUB HADD HSUB SUBADD MUL DIV MIN MAX SQRT RSQRT RSQRT14 RCP RCP14 DPP FM(N)ADD/SUB.  DPP and FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x460",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of SSE/AVX computational scalar floating-point instructions retired; some instructions will count twice as noted below.  Applies to SSE* and AVX* scalar, double and single precision floating-point: ADD SUB MUL DIV MIN MAX RCP14 RSQRT14 RANGE SQRT DPP FM(N)ADD/SUB.  DPP and FM(N)ADD/SUB instructions count twice as they perform multiple calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc7",
+        "EventName": "FP_ARITH_OPS_RETIRED.SCALAR",
+        "PublicDescription": "Number of SSE/AVX computational scalar single precision and double precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 1 computational operation. Applies to SSE* and AVX* scalar single precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT RCP FM(N)ADD/SUB.  FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts number of SSE/AVX computational scalar double precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 1 computational operation. Applies to SSE* and AVX* scalar double precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT FM(N)ADD/SUB.  FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc7",
+        "EventName": "FP_ARITH_OPS_RETIRED.SCALAR_DOUBLE",
+        "PublicDescription": "Number of SSE/AVX computational scalar double precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 1 computational operation. Applies to SSE* and AVX* scalar double precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT FM(N)ADD/SUB.  FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts number of SSE/AVX computational scalar single precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 1 computational operation. Applies to SSE* and AVX* scalar single precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT RCP FM(N)ADD/SUB.  FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc7",
+        "EventName": "FP_ARITH_OPS_RETIRED.SCALAR_SINGLE",
+        "PublicDescription": "Number of SSE/AVX computational scalar single precision floating-point instructions retired; some instructions will count twice as noted below.  Each count represents 1 computational operation. Applies to SSE* and AVX* scalar single precision floating-point instructions: ADD SUB MUL DIV MIN MAX SQRT RSQRT RCP FM(N)ADD/SUB.  FM(N)ADD/SUB instructions count twice as they perform 2 calculations per element. The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of any Vector retired FP arithmetic instructions",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc7",
+        "EventName": "FP_ARITH_OPS_RETIRED.VECTOR",
+        "PublicDescription": "Number of any Vector retired FP arithmetic instructions.  The DAZ and FTZ flags in the MXCSR register need to be set when using these events.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x3c",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "FP_ARITH_OPS_RETIRED.VECTOR_128B",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc7",
+        "EventName": "FP_ARITH_OPS_RETIRED.VECTOR_128B",
+        "SampleAfterValue": "100003",
+        "UMask": "0x40c",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "FP_ARITH_OPS_RETIRED.VECTOR_256B",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc7",
+        "EventName": "FP_ARITH_OPS_RETIRED.VECTOR_256B",
+        "SampleAfterValue": "100003",
+        "UMask": "0x830",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "FP_ARITH_OPS_RETIRED.VECTOR_512B",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc7",
+        "EventName": "FP_ARITH_OPS_RETIRED.VECTOR_512B",
+        "SampleAfterValue": "100003",
+        "UMask": "0x10c0",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on all floating point ports.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "FP_VINT_UOPS_EXECUTED.ALL",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1f",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 0.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "FP_VINT_UOPS_EXECUTED.P0",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 1.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "FP_VINT_UOPS_EXECUTED.P1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 2.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "FP_VINT_UOPS_EXECUTED.P2",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 3.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "FP_VINT_UOPS_EXECUTED.P3",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x10",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on floating point and vector integer port 0, 1, 2, 3.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "FP_VINT_UOPS_EXECUTED.PRIMARY",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1e",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on floating point and vector integer store data port.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "FP_VINT_UOPS_EXECUTED.STD",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of floating point operations retired that required microcode assist.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.FP_ASSIST",
+        "PublicDescription": "Counts the number of floating point operations retired that required microcode assist, which is not a reflection of the number of FP operations, instructions or uops.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of floating point divide uops retired (x87 and sse, including x87 sqrt).",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xc2",
+        "EventName": "UOPS_RETIRED.FPDIV",
+        "PublicDescription": "Counts the number of floating point divide uops retired (x87 and sse, including x87 sqrt). Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x40",
+        "Unit": "cpu_atom"
+    }
+]
diff --git a/tools/perf/pmu-events/arch/x86/novalake/frontend.json b/tools/perf/pmu-events/arch/x86/novalake/frontend.json
new file mode 100644
index 0000000000000..7d1e62bb88039
--- /dev/null
+++ b/tools/perf/pmu-events/arch/x86/novalake/frontend.json
@@ -0,0 +1,600 @@
+[
+    {
+        "BriefDescription": "Counts the total number of BACLEARS due to all branch types including conditional and unconditional jumps, returns, and indirect branches.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe6",
+        "EventName": "BACLEARS.ANY",
+        "PublicDescription": "Counts the total number of BACLEARS, which occur when the Branch Target Buffer (BTB) prediction or lack thereof, was corrected by a later branch predictor in the frontend.  Includes BACLEARS due to all branch types including conditional and unconditional jumps, returns, and indirect branches.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Clears due to Unknown Branches.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x60",
+        "EventName": "BACLEARS.ANY",
+        "PublicDescription": "Number of times the front-end is resteered when it finds a branch instruction in a fetch line. This is called Unknown Branch which occurs for the first time a branch instruction is fetched or when the branch is not tracked by the BPU (Branch Prediction Unit) anymore.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of BACLEARS due to a conditional jump.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe6",
+        "EventName": "BACLEARS.COND",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x10",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of BACLEARS due to an indirect branch.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe6",
+        "EventName": "BACLEARS.INDIRECT",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of BACLEARS due to a return branch.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe6",
+        "EventName": "BACLEARS.RETURN",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of BACLEARS due to a direct, unconditional jump.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe6",
+        "EventName": "BACLEARS.UNCOND",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Stalls caused by changing prefix length of the instruction.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x87",
+        "EventName": "DECODE.LCP",
+        "PublicDescription": "Counts cycles that the Instruction Length decoder (ILD) stalls occurred due to dynamically changing prefix length of the decoded instruction (by operand size prefix instruction 0x66, address size prefix instruction 0x67 or REX.W for Intel64). Count is proportional to the number of prefixes in a 16B-line. This may result in a three-cycle penalty for each LCP (Length changing prefix) in a 16-byte chunk.",
+        "SampleAfterValue": "500009",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles the Microcode Sequencer is busy.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x87",
+        "EventName": "DECODE.MS_BUSY",
+        "SampleAfterValue": "500009",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "DSB-to-MITE switch true penalty cycles.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x61",
+        "EventName": "DSB2MITE_SWITCHES.PENALTY_CYCLES",
+        "PublicDescription": "Decode Stream Buffer (DSB) is a Uop-cache that holds translations of previously fetched instructions that were decoded by the legacy x86 decode pipeline (MITE). This event counts fetch penalty cycles when a transition occurs from DSB to MITE.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired ANT branches",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.ANY_ANT",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x9",
+        "PublicDescription": "Always Not Taken (ANT) conditional retired branches (no BTB entry and not mispredicted) Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired Instructions who experienced DSB miss.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.ANY_DSB_MISS",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x1",
+        "PublicDescription": "Counts retired Instructions that experienced DSB (Decode stream buffer i.e. the decoded instruction-cache) miss. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired Instructions who experienced a critical DSB miss.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.DSB_MISS",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x11",
+        "PublicDescription": "Number of retired Instructions that experienced a critical DSB (Decode stream buffer i.e. the decoded instruction-cache) miss. Critical means stalls were exposed to the back-end as a result of the DSB miss. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to icache miss.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.ICACHE",
+        "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to icache miss Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x20",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to ITLB miss.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.ITLB_MISS",
+        "PublicDescription": "Counts the number of instructions retired that were tagged because empty issue slots were seen before the uop due to ITLB miss Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x10",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Retired Instructions who experienced iTLB true miss.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.ITLB_MISS",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x14",
+        "PublicDescription": "Counts retired Instructions that experienced iTLB (Instruction TLB) true miss. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired Instructions who experienced Instruction L1 Cache true miss.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.L1I_MISS",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x12",
+        "PublicDescription": "Counts retired Instructions who experienced Instruction L1 Cache true miss. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired Instructions who experienced Instruction L2 Cache true miss.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.L2_MISS",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x13",
+        "PublicDescription": "Counts retired Instructions who experienced Instruction L2 Cache true miss. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 128 cycles which was not interrupted by a back-end stall.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.LATENCY_GE_128",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x608006",
+        "PublicDescription": "Counts retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 128 cycles which was not interrupted by a back-end stall. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 16 cycles which was not interrupted by a back-end stall.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.LATENCY_GE_16",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x601006",
+        "PublicDescription": "Counts retired instructions that are delivered to the back-end after a front-end stall of at least 16 cycles. During this period the front-end delivered no uops. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired instructions after front-end starvation of at least 2 cycles",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.LATENCY_GE_2",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x600206",
+        "PublicDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of at least 2 cycles which was not interrupted by a back-end stall. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 256 cycles which was not interrupted by a back-end stall.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.LATENCY_GE_256",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x610006",
+        "PublicDescription": "Counts retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 256 cycles which was not interrupted by a back-end stall. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired instructions that are fetched after an interval where the front-end had at least 1 bubble-slot for a period of 2 cycles which was not interrupted by a back-end stall.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x100206",
+        "PublicDescription": "Counts retired instructions that are delivered to the back-end after the front-end had at least 1 bubble-slot for a period of 2 cycles. A bubble-slot is an empty issue-pipeline slot while there was no RAT stall. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 32 cycles which was not interrupted by a back-end stall.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.LATENCY_GE_32",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x602006",
+        "PublicDescription": "Counts retired instructions that are delivered to the back-end after a front-end stall of at least 32 cycles. During this period the front-end delivered no uops. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 4 cycles which was not interrupted by a back-end stall.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.LATENCY_GE_4",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x600406",
+        "PublicDescription": "Counts retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 4 cycles which was not interrupted by a back-end stall. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 512 cycles which was not interrupted by a back-end stall.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.LATENCY_GE_512",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x620006",
+        "PublicDescription": "Counts retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 512 cycles which was not interrupted by a back-end stall. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 64 cycles which was not interrupted by a back-end stall.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.LATENCY_GE_64",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x604006",
+        "PublicDescription": "Counts retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 64 cycles which was not interrupted by a back-end stall. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired instructions that are fetched after an interval where the front-end delivered no uops for a period of 8 cycles which was not interrupted by a back-end stall.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.LATENCY_GE_8",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x600806",
+        "PublicDescription": "Counts retired instructions that are delivered to the back-end after a front-end stall of at least 8 cycles. During this period the front-end delivered no uops. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Mispredicted Retired ANT branches",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.MISP_ANT",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x9",
+        "PublicDescription": "ANT retired branches that got just mispredicted Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts flows delivered by the Microcode Sequencer",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.MS_FLOWS",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x8",
+        "PublicDescription": "Counts flows delivered by the Microcode Sequencer Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired Instructions who experienced STLB (2nd level TLB) true miss.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.STLB_MISS",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x15",
+        "PublicDescription": "Counts retired Instructions that experienced STLB (2nd level TLB) true miss. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired instructions that caused clears due to being Unknown Branches.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc6",
+        "EventName": "FRONTEND_RETIRED.UNKNOWN_BRANCH",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x17",
+        "PublicDescription": "Number retired branch instructions that caused the front-end to be resteered when it finds the instruction in a fetch line. This is called Unknown Branch which occurs for the first time a branch instruction is fetched or when the branch is not tracked by the BPU (Branch Prediction Unit) anymore. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts every time the code stream enters into a new cache line by walking sequential from the previous line or being redirected by a jump.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x80",
+        "EventName": "ICACHE.ACCESSES",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x3",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts every time the code stream enters into a new cache line by walking sequential from the previous line or being redirected by a jump and the instruction cache registers bytes are present.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x80",
+        "EventName": "ICACHE.HIT",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts every time the code stream enters into a new cache line by walking sequential from the previous line or being redirected by a jump and the instruction cache registers bytes are not present.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x80",
+        "EventName": "ICACHE.MISSES",
+        "PublicDescription": "Counts every time the code stream enters into a new cache line by walking sequential from the previous line or being redirected by a jump and the instruction cache registers bytes are not present. -",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Cycles where a code fetch is stalled due to L1 instruction cache miss.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x80",
+        "EventName": "ICACHE_DATA.STALLS",
+        "PublicDescription": "Counts cycles where a code line fetch is stalled due to an L1 instruction cache miss. The decode pipeline works at a 32 Byte granularity.",
+        "SampleAfterValue": "500009",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "ICACHE_DATA.STALL_PERIODS",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EdgeDetect": "1",
+        "EventCode": "0x80",
+        "EventName": "ICACHE_DATA.STALL_PERIODS",
+        "SampleAfterValue": "500009",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles where a code fetch is stalled due to L1 instruction cache tag miss.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x83",
+        "EventName": "ICACHE_TAG.STALLS",
+        "PublicDescription": "Counts cycles where a code fetch is stalled due to L1 instruction cache tag miss.",
+        "SampleAfterValue": "200003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles where a code fetch is stalled due to L1 instruction cache In use-full",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x83",
+        "EventName": "ICACHE_TAG.STALLS_INUSE",
+        "SampleAfterValue": "200003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles where a code fetch is stalled due to L1 instruction cache ISB-full",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x83",
+        "EventName": "ICACHE_TAG.STALLS_ISB",
+        "SampleAfterValue": "200003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles Decode Stream Buffer (DSB) is delivering any Uop",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EventCode": "0x79",
+        "EventName": "IDQ.DSB_CYCLES_ANY",
+        "PublicDescription": "Counts the number of cycles uops were delivered to Instruction Decode Queue (IDQ) from the Decode Stream Buffer (DSB) path.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles DSB is delivering optimal number of Uops",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "8",
+        "EventCode": "0x79",
+        "EventName": "IDQ.DSB_CYCLES_OK",
+        "PublicDescription": "Counts the number of cycles where optimal number of uops was delivered to the Instruction Decode Queue (IDQ) from the DSB (Decode Stream Buffer) path. Count includes uops that may 'bypass' the IDQ.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) from the Decode Stream Buffer (DSB) path",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x79",
+        "EventName": "IDQ.DSB_UOPS",
+        "PublicDescription": "Counts the number of uops delivered to Instruction Decode Queue (IDQ) from the Decode Stream Buffer (DSB) path.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles MITE is delivering any Uop",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EventCode": "0x79",
+        "EventName": "IDQ.MITE_CYCLES_ANY",
+        "PublicDescription": "Counts the number of cycles uops were delivered to the Instruction Decode Queue (IDQ) from the MITE (legacy decode pipeline) path. During these cycles uops are not being delivered from the Decode Stream Buffer (DSB).",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles MITE is delivering optimal number of Uops",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "8",
+        "EventCode": "0x79",
+        "EventName": "IDQ.MITE_CYCLES_OK",
+        "PublicDescription": "Counts the number of cycles where optimal number of uops was delivered to the Instruction Decode Queue (IDQ) from the MITE (legacy decode pipeline) path. During these cycles uops are not being delivered from the Decode Stream Buffer (DSB).",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) from MITE path",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x79",
+        "EventName": "IDQ.MITE_UOPS",
+        "PublicDescription": "Counts the number of uops delivered to Instruction Decode Queue (IDQ) from the MITE path. This also means that uops are not being delivered from the Decode Stream Buffer (DSB).",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles when uops are being delivered to IDQ while MS is busy",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EventCode": "0x79",
+        "EventName": "IDQ.MS_CYCLES_ANY",
+        "PublicDescription": "Counts cycles during which uops are being delivered to Instruction Decode Queue (IDQ) while the Microcode Sequencer (MS) is busy. Uops maybe initiated by Decode Stream Buffer (DSB) or MITE.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x20",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of switches from DSB or MITE to the MS",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EdgeDetect": "1",
+        "EventCode": "0x79",
+        "EventName": "IDQ.MS_SWITCHES",
+        "PublicDescription": "Number of switches from DSB (Decode Stream Buffer) or MITE (legacy decode pipeline) to the Microcode Sequencer.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x20",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Uops initiated by MITE or Decode Stream Buffer (DSB) and delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x79",
+        "EventName": "IDQ.MS_UOPS",
+        "PublicDescription": "Counts the number of uops initiated by MITE or Decode Stream Buffer (DSB) and delivered to Instruction Decode Queue (IDQ) while the Microcode Sequencer (MS) is busy. Counting includes uops that may 'bypass' the IDQ.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x20",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "count cycles when IDQ didn't deliver any uop for 1 cycle, counts 1-cycle starvation windows.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x9c",
+        "EventName": "IDQ_BUBBLES.BW_STARVATION",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "This event counts a subset of the Topdown Slots event that when no operation was delivered to the back-end pipeline due to instruction fetch limitations when the back-end could have accepted more operations. Common examples include instruction cache misses or x86 instruction decode limitations.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x9c",
+        "EventName": "IDQ_BUBBLES.CORE",
+        "PublicDescription": "This event counts a subset of the Topdown Slots event that when no operation was delivered to the back-end pipeline due to instruction fetch limitations when the back-end could have accepted more operations. Common examples include instruction cache misses or x86 instruction decode limitations. Software can use this event as the numerator for the Frontend Bound metric (or top-level category) of the Top-down Microarchitecture Analysis method.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles when optimal number of uops was delivered to the back-end when the back-end is not stalled",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EventCode": "0x9c",
+        "EventName": "IDQ_BUBBLES.CYCLES_FE_WAS_OK",
+        "Invert": "1",
+        "PublicDescription": "Counts the number of cycles when the optimal number of uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles when no uops are delivered by the IDQ for 2 or more cycles when backend of the machine is not stalled - normally indicating a Fetch Latency issue",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x9c",
+        "EventName": "IDQ_BUBBLES.FETCH_LATENCY",
+        "PublicDescription": "Counts the number of cycles when no uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls for 2 or more cycles - normally indicating a Fetch Latency issue.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles when no uops are not delivered by the IDQ when backend of the machine is not stalled",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "8",
+        "EventCode": "0x9c",
+        "EventName": "IDQ_BUBBLES.STARVATION_CYCLES",
+        "PublicDescription": "Counts the number of cycles when no uops were delivered by the Instruction Decode Queue (IDQ) to the back-end of the pipeline when there was no back-end stalls.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of cycles that the MS is busy.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe7",
+        "EventName": "MS_DECODED.MS_BUSY",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of times entered into a ucode flow in the FEC.  Includes inserted flows due to front-end detected faults or assists.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe7",
+        "EventName": "MS_DECODED.MS_ENTRY",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of times nanocode flow is executed.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe7",
+        "EventName": "MS_DECODED.NANO_CODE",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    }
+]
diff --git a/tools/perf/pmu-events/arch/x86/novalake/memory.json b/tools/perf/pmu-events/arch/x86/novalake/memory.json
new file mode 100644
index 0000000000000..459f0b7a026f2
--- /dev/null
+++ b/tools/perf/pmu-events/arch/x86/novalake/memory.json
@@ -0,0 +1,299 @@
+[
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to any number of reasons, including an L1 miss, WCB full, pagewalk, store address block or store data block.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.ANY",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x7f",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to any number of reasons, including an L1 miss, WCB full, pagewalk, store address block or store data block, on a load that retires.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.ANY_AT_RET",
+        "SampleAfterValue": "1000003",
+        "UMask": "0xff",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a core bound stall including a store address match, a DTLB miss or a page walk that detains the load from retiring.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.L1_BOUND_AT_RET",
+        "SampleAfterValue": "1000003",
+        "UMask": "0xf4",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a DL1 miss.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.L1_MISS",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a DL1 miss.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.L1_MISS_AT_RET",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x81",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to other block cases.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.OTHER",
+        "PublicDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to other block cases such as pipeline conflicts, fences, etc.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x40",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to other block cases.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.OTHER_AT_RET",
+        "PublicDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to other block cases such as pipeline conflicts, fences, etc.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0xc0",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a pagewalk.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.PGWALK",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x20",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a pagewalk.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.PGWALK_AT_RET",
+        "SampleAfterValue": "1000003",
+        "UMask": "0xa0",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a store address match.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.ST_ADDR",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a store address match.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.ST_ADDR_AT_RET",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x84",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to request buffers full or lock in progress.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.WCB_FULL",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to request buffers full or lock in progress.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.WCB_FULL_AT_RET",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x82",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of machine clears due to memory ordering caused by a snoop from an external agent. Does not count internally generated machine clears such as those due to memory disambiguation.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.MEMORY_ORDERING",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Number of machine clears due to memory ordering conflicts.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.MEMORY_ORDERING",
+        "PublicDescription": "Counts the number of Machine Clears detected dye to memory ordering. Memory Ordering Machine Clears may apply when a memory read may not conform to the memory ordering rules of the x86 architecture",
+        "SampleAfterValue": "100003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.MEMORY_ORDERING_FAST",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8002",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Execution stalls while L3 cache miss demand load is outstanding.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x47",
+        "EventName": "MEMORY_ACTIVITY.STALLS_L3_MISS",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x48",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 1024 cycles.",
+        "Counter": "2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xcd",
+        "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_1024",
+        "MSRIndex": "0x3F6",
+        "MSRValue": "0x400",
+        "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 1024 cycles.  Reported latency may be longer than just the memory latency. Available PDIST counters: 0,1",
+        "SampleAfterValue": "53",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 128 cycles.",
+        "Counter": "2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xcd",
+        "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_128",
+        "MSRIndex": "0x3F6",
+        "MSRValue": "0x80",
+        "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 128 cycles.  Reported latency may be longer than just the memory latency. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1009",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 16 cycles.",
+        "Counter": "2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xcd",
+        "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_16",
+        "MSRIndex": "0x3F6",
+        "MSRValue": "0x10",
+        "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 16 cycles.  Reported latency may be longer than just the memory latency. Available PDIST counters: 0,1",
+        "SampleAfterValue": "20011",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 2048 cycles.",
+        "Counter": "2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xcd",
+        "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_2048",
+        "MSRIndex": "0x3F6",
+        "MSRValue": "0x800",
+        "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 2048 cycles.  Reported latency may be longer than just the memory latency. Available PDIST counters: 0,1",
+        "SampleAfterValue": "23",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 256 cycles.",
+        "Counter": "2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xcd",
+        "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_256",
+        "MSRIndex": "0x3F6",
+        "MSRValue": "0x100",
+        "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 256 cycles.  Reported latency may be longer than just the memory latency. Available PDIST counters: 0,1",
+        "SampleAfterValue": "503",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 32 cycles.",
+        "Counter": "2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xcd",
+        "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_32",
+        "MSRIndex": "0x3F6",
+        "MSRValue": "0x20",
+        "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 32 cycles.  Reported latency may be longer than just the memory latency. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 4 cycles.",
+        "Counter": "2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xcd",
+        "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_4",
+        "MSRIndex": "0x3F6",
+        "MSRValue": "0x4",
+        "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 4 cycles.  Reported latency may be longer than just the memory latency. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 512 cycles.",
+        "Counter": "2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xcd",
+        "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_512",
+        "MSRIndex": "0x3F6",
+        "MSRValue": "0x200",
+        "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 512 cycles.  Reported latency may be longer than just the memory latency. Available PDIST counters: 0,1",
+        "SampleAfterValue": "101",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 64 cycles.",
+        "Counter": "2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xcd",
+        "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_64",
+        "MSRIndex": "0x3F6",
+        "MSRValue": "0x40",
+        "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 64 cycles.  Reported latency may be longer than just the memory latency. Available PDIST counters: 0,1",
+        "SampleAfterValue": "2003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 8 cycles.",
+        "Counter": "2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xcd",
+        "EventName": "MEM_TRANS_RETIRED.LOAD_LATENCY_GT_8",
+        "MSRIndex": "0x3F6",
+        "MSRValue": "0x8",
+        "PublicDescription": "Counts randomly selected loads when the latency from first dispatch to completion is greater than 8 cycles.  Reported latency may be longer than just the memory latency. Available PDIST counters: 0,1",
+        "SampleAfterValue": "50021",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired memory store access operations. A PDist event for PEBS Store Latency Facility.",
+        "Counter": "0,1",
+        "Data_LA": "1",
+        "EventCode": "0xcd",
+        "EventName": "MEM_TRANS_RETIRED.STORE_SAMPLE",
+        "PublicDescription": "Counts Retired memory accesses with at least 1 store operation. This PEBS event is the precisely-distributed (PDist) trigger covering all stores uops for sampling by the PEBS Store Latency Facility. The facility is described in Intel SDM Volume 3 section 19.9.8 Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    }
+]
diff --git a/tools/perf/pmu-events/arch/x86/novalake/other.json b/tools/perf/pmu-events/arch/x86/novalake/other.json
new file mode 100644
index 0000000000000..1c6ae1c9d4ff5
--- /dev/null
+++ b/tools/perf/pmu-events/arch/x86/novalake/other.json
@@ -0,0 +1,144 @@
+[
+    {
+        "BriefDescription": "Count all other hardware assists or traps that are not necessarily architecturally exposed (through a software handler) beyond FP; SSE-AVX mix and A/D assists who are counted by dedicated sub-events.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc1",
+        "EventName": "ASSISTS.HARDWARE",
+        "PublicDescription": "Count all other hardware assists or traps that are not necessarily architecturally exposed (through a software handler) beyond FP; SSE-AVX mix and A/D assists who are counted by dedicated sub-events.  This includes, but not limited to, assists at EXE or MEM uop writeback like AVX* load/store/gather/scatter (non-FP GSSE-assist ) , assists generated by ROB like PEBS and RTIT, Uncore trap, RAR (Remote Action Request) and CET (Control flow Enforcement Technology) assists.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "ASSISTS.PAGE_FAULT",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc1",
+        "EventName": "ASSISTS.PAGE_FAULT",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Core cycles during which interrupts are masked.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xcb",
+        "EventName": "HW_INTERRUPTS.MASKED",
+        "PublicDescription": "Core cycles during which interrupts are masked (disabled, when RFLAGS.IF is clear) while thread is Unhalted.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Core cycles during which there are pending interrupts but interrupts are masked.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xcb",
+        "EventName": "HW_INTERRUPTS.PENDING_AND_MASKED",
+        "PublicDescription": "Core cycles during which there are pending interrupts but interrupts are masked (disabled, when RFLAGS.IF is clear and there is pending interrupt from PIC) while thread is Unhalted.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of hardware interrupts received by the processor.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xcb",
+        "EventName": "HW_INTERRUPTS.RECEIVED",
+        "PublicDescription": "Counts the number of hardware interruptions received by the processor.",
+        "SampleAfterValue": "203",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of LBR entries recorded. Requires LBRs to be enabled in IA32_LBR_CTL. [This event is alias to MISC_RETIRED.LBR_INSERTS]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe4",
+        "EventName": "LBR_INSERTS.ANY",
+        "PublicDescription": "Counts the number of LBR entries recorded. Requires LBRs to be enabled in IA32_LBR_CTL. [This event is alias to MISC_RETIRED.LBR_INSERTS] Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "number of port pushouts of integer ports (ready uop was delayed due to port being used by another uop) including integer STD.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xab",
+        "EventName": "PORT_PUSHOUT.INTEGER",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "number of port pushouts of vector ports (ready uop was delayed due to port being used by another uop)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xab",
+        "EventName": "PORT_PUSHOUT.VECTOR",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of vector uops scheduled for execution (no cancel) including vec/fp STD uops- ports with ready uops",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xaa",
+        "EventName": "PORT_READY.VECTOR",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Alloc Stall cycles when stall is due to general shared resource (e.g. ROB) and is not MEMORY_BOUND stall",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa9",
+        "EventName": "STALL_CORE_BOUND.GENERAL",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x20",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Alloc Stall cycles when stall is due to MEM RS resource and is not MEMORY_BOUND stall",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa9",
+        "EventName": "STALL_CORE_BOUND.MEM_ADDRESS",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Alloc Stall cycles when the stall is due to MEM RS resource or other memory resource and is not MEMORY_BOUND",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa9",
+        "EventName": "STALL_CORE_BOUND.MEM_RESOURCE",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x14",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Alloc Stall cycles when stall is due to serialization and is not MEMORY_BOUND stall",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa9",
+        "EventName": "STALL_CORE_BOUND.SERIALIZATION",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Alloc Stall cycles when stall is due to vector stack resource and is not MEMORY_BOUND stall",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa9",
+        "EventName": "STALL_CORE_BOUND.VECTOR",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles the uncore cannot take further requests",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EventCode": "0x2d",
+        "EventName": "XQ.FULL",
+        "PublicDescription": "number of cycles when the thread is active and the uncore cannot take any further requests (for example prefetches, loads or stores initiated by the Core that miss the L2 cache).",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    }
+]
diff --git a/tools/perf/pmu-events/arch/x86/novalake/pipeline.json b/tools/perf/pmu-events/arch/x86/novalake/pipeline.json
new file mode 100644
index 0000000000000..998dd4cbc66a7
--- /dev/null
+++ b/tools/perf/pmu-events/arch/x86/novalake/pipeline.json
@@ -0,0 +1,1976 @@
+[
+    {
+        "BriefDescription": "Cycles when divide unit is busy executing divide or square root operations.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EventCode": "0xb0",
+        "EventName": "ARITH.DIV_ACTIVE",
+        "PublicDescription": "Counts cycles when divide unit is busy executing divide or square root operations. Accounts for integer and floating-point operations.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x9",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles when integer divide unit is busy executing divide or square root operations.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EventCode": "0xb0",
+        "EventName": "ARITH.IDIV_ACTIVE",
+        "PublicDescription": "Counts cycles when divide unit is busy executing divide or square root operations. Accounts for integer operations only.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of active integer dividers per cycle.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xcd",
+        "EventName": "ARITH.IDIV_OCCUPANCY",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of integer divider uops executed per cycle.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xcd",
+        "EventName": "ARITH.IDIV_UOPS",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Number of occurrences where a microcode assist is invoked by hardware.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc1",
+        "EventName": "ASSISTS.ANY",
+        "PublicDescription": "Counts the number of occurrences where a microcode assist is invoked by hardware. Examples include AD (page Access Dirty), FP and AVX related assists.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x1f",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles there was microcode scoreboard stall",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa2",
+        "EventName": "BE_STALLS.SCOREBOARD",
+        "SampleAfterValue": "100003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the total number of branch instructions retired for all branch types.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.ALL_BRANCHES",
+        "PublicDescription": "Counts the total number of instructions in which the instruction pointer (IP) of the processor is resteered due to a branch instruction and the branch instruction successfully retires.  All branch type instructions are accounted for. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "All branch instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.ALL_BRANCHES",
+        "PublicDescription": "Counts all branch instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Conditional branch instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.COND",
+        "PublicDescription": "Counts conditional branch instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0x7",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of not taken conditional branch instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.COND_NTAKEN",
+        "PublicDescription": "Counts the number of not taken conditional branch instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Not taken branch instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.COND_NTAKEN",
+        "PublicDescription": "Counts not taken branch instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of taken conditional branch instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.COND_TAKEN",
+        "PublicDescription": "Counts the number of taken conditional branch instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x3",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Taken conditional branch instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.COND_TAKEN",
+        "PublicDescription": "Counts taken conditional branch instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Taken backward conditional branch instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.COND_TAKEN_BWD",
+        "PublicDescription": "Counts taken backward conditional branch instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Taken forward conditional branch instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.COND_TAKEN_FWD",
+        "PublicDescription": "Counts taken forward conditional branch instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Far branch instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.FAR_BRANCH",
+        "PublicDescription": "Counts far branch instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Direct and indirect near call instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.NEAR_CALL",
+        "PublicDescription": "Counts both direct and indirect near call instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x30",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "near direct call instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.NEAR_DIRECT_CALL",
+        "PublicDescription": "Counts near direct call instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x20",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "near direct jump instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.NEAR_DIRECT_JMP",
+        "PublicDescription": "Counts near direct jump instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x80",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Indirect near branch instructions retired (excluding returns)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.NEAR_INDIRECT",
+        "PublicDescription": "Counts near indirect branch instructions retired excluding returns. TSX abort is an indirect branch. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100003",
+        "UMask": "0x50",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Indirect near call instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.NEAR_INDIRECT_CALL",
+        "PublicDescription": "Counts indirect near call instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Indirect near jump instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.NEAR_INDIRECT_JMP",
+        "PublicDescription": "Counts indirect near jump instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x40",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Indirect and Direct Relative near jump instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.NEAR_JMP",
+        "PublicDescription": "Counts near jump instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0xc0",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Return instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.NEAR_RETURN",
+        "PublicDescription": "Counts return instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Taken branch instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc4",
+        "EventName": "BR_INST_RETIRED.NEAR_TAKEN",
+        "PublicDescription": "Counts taken branch instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0xfb",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the total number of mispredicted branch instructions retired for all branch types.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.ALL_BRANCHES",
+        "PublicDescription": "Counts the total number of mispredicted branch instructions retired.  All branch type instructions are accounted for.  Prediction of the branch target address enables the processor to begin executing instructions before the non-speculative execution path is known. The branch prediction unit (BPU) predicts the target address based on the instruction pointer (IP) of the branch and on the execution path through which execution reached this IP.    A branch misprediction occurs when the prediction is wrong, and results in discarding all instructions executed in the speculative path and re-fetching from the correct path. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "All mispredicted branch instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.ALL_BRANCHES",
+        "PublicDescription": "Counts all the retired branch instructions that were mispredicted by the processor. A branch misprediction occurs when the processor incorrectly predicts the destination of the branch.  When the misprediction is discovered at execution, all the instructions executed in the wrong (speculative) path must be discarded, and the processor must start fetching from the correct path. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "All mispredicted branch instructions retired. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.ALL_BRANCHES_TPEBS",
+        "PublicDescription": "All mispredicted branch instructions retired. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Mispredicted conditional branch instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.COND",
+        "PublicDescription": "Counts mispredicted conditional branch instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0x7",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Mispredicted non-taken conditional branch instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.COND_NTAKEN",
+        "PublicDescription": "Counts the number of conditional branch instructions retired that were mispredicted and the branch direction was not taken. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Mispredicted non-taken conditional branch instructions retired. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.COND_NTAKEN_TPEBS",
+        "PublicDescription": "Mispredicted non-taken conditional branch instructions retired. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0x8004",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "number of branch instructions retired that were mispredicted and taken.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.COND_TAKEN",
+        "PublicDescription": "Counts taken conditional mispredicted branch instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "number of branch instructions retired that were mispredicted and taken forward.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.COND_TAKEN_FWD",
+        "PublicDescription": "Counts taken forward conditional mispredicted branch instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "number of branch instructions retired that were mispredicted and taken forward. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.COND_TAKEN_FWD_TPEBS",
+        "PublicDescription": "number of branch instructions retired that were mispredicted and taken forward. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0x8002",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Mispredicted taken conditional branch instructions retired. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.COND_TAKEN_TPEBS",
+        "PublicDescription": "Mispredicted taken conditional branch instructions retired. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0x8003",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Mispredicted conditional branch instructions retired. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.COND_TPEBS",
+        "PublicDescription": "Mispredicted conditional branch instructions retired. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0x8007",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Miss-predicted near indirect branch instructions retired (excluding returns)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.NEAR_INDIRECT",
+        "PublicDescription": "Counts miss-predicted near indirect branch instructions retired excluding returns. TSX abort is an indirect branch. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100003",
+        "UMask": "0x50",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Mispredicted indirect CALL retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.NEAR_INDIRECT_CALL",
+        "PublicDescription": "Counts retired mispredicted indirect (near taken) CALL instructions, including both register and memory indirect. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Mispredicted indirect CALL retired. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.NEAR_INDIRECT_CALL_TPEBS",
+        "PublicDescription": "Mispredicted indirect CALL retired. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0x8010",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Miss-predicted near indirect jump instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.NEAR_INDIRECT_JMP",
+        "PublicDescription": "Miss-predicted near indirect jump instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100003",
+        "UMask": "0x40",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Miss-predicted near indirect jump instructions retired. Precise cost.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.NEAR_INDIRECT_JMP_TPEBS",
+        "PublicDescription": "Miss-predicted near indirect jump instructions retired. Precise cost. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100003",
+        "UMask": "0x8040",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Miss-predicted near indirect branch instructions retired (excluding returns)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.NEAR_INDIRECT_TPEBS",
+        "PublicDescription": "Counts miss-predicted near indirect branch instructions retired excluding returns. TSX abort is an indirect branch. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100003",
+        "UMask": "0x8050",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "This event counts the number of mispredicted ret instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.NEAR_RETURN",
+        "PublicDescription": "This event counts the number of mispredicted ret instructions retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Mispredicted ret instructions retired. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.NEAR_RETURN_TPEBS",
+        "PublicDescription": "Mispredicted ret instructions retired. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch. Available PDIST counters: 0,1",
+        "SampleAfterValue": "100007",
+        "UMask": "0x8008",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of near branch instructions retired that were mispredicted and taken.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.NEAR_TAKEN",
+        "PublicDescription": "Counts number of near branch instructions retired that were mispredicted and taken. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0xfb",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Mispredicted taken near branch instructions retired. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.NEAR_TAKEN_TPEBS",
+        "PublicDescription": "Mispredicted taken near branch instructions retired. This precise event may be used to get the misprediction cost via the Retire_Latency field of PEBS. It fires on the instruction that immediately follows the mispredicted branch. Available PDIST counters: 0,1",
+        "SampleAfterValue": "400009",
+        "UMask": "0x80fb",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Core clocks when the thread is in the C0.1 light-weight slower wakeup time but more power saving optimized state.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xec",
+        "EventName": "CPU_CLK_UNHALTED.C01",
+        "PublicDescription": "Counts core clocks when the thread is in the C0.1 light-weight slower wakeup time but more power saving optimized state.  This state can be entered via the TPAUSE or UMWAIT instructions.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Core clocks when the thread is in the C0.2 light-weight faster wakeup time but less power saving optimized state.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xec",
+        "EventName": "CPU_CLK_UNHALTED.C02",
+        "PublicDescription": "Counts core clocks when the thread is in the C0.2 light-weight faster wakeup time but less power saving optimized state.  This state can be entered via the TPAUSE or UMWAIT instructions.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x20",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Core clocks when the thread is in the C0.1 or C0.2 or running a PAUSE in C0 ACPI state.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xec",
+        "EventName": "CPU_CLK_UNHALTED.C0_WAIT",
+        "PublicDescription": "Counts core clocks when the thread is in the C0.1 or C0.2 power saving optimized states (TPAUSE or UMWAIT instructions) or running the PAUSE instruction.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x70",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Fixed Counter: Counts the number of unhalted core clock cycles. [This event is alias to CPU_CLK_UNHALTED.THREAD]",
+        "Counter": "Fixed counter 1",
+        "EventName": "CPU_CLK_UNHALTED.CORE",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Core cycles when the core is not in a halt state.",
+        "Counter": "Fixed counter 1",
+        "EventName": "CPU_CLK_UNHALTED.CORE",
+        "PublicDescription": "Counts the number of core cycles while the core is not in a halt state. The core enters the halt state when it is running the HLT instruction. This event is a component in many key event ratios. The core frequency may change from time to time due to transitions associated with Enhanced Intel SpeedStep Technology or TM2. For this reason this event may have a changing ratio with regards to time. When the core frequency is constant, this event can approximate elapsed time while the core was not in the halt state. It is counted on a dedicated fixed counter, leaving the programmable counters available for other events.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of unhalted core clock cycles. [This event is alias to CPU_CLK_UNHALTED.THREAD_P]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x3c",
+        "EventName": "CPU_CLK_UNHALTED.CORE_P",
+        "PublicDescription": "Counts the number of unhalted core clock cycles [This event is alias to CPU_CLK_UNHALTED.THREAD_P]",
+        "SampleAfterValue": "2000003",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Thread cycles when thread is not in halt state [This event is alias to CPU_CLK_UNHALTED.THREAD_P]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x3c",
+        "EventName": "CPU_CLK_UNHALTED.CORE_P",
+        "PublicDescription": "This is an architectural event that counts the number of thread cycles while the thread is not in a halt state. The thread enters the halt state when it is running the HLT instruction. The core frequency may change from time to time due to power or thermal throttling. For this reason, this event may have a changing ratio with regards to wall clock time. [This event is alias to CPU_CLK_UNHALTED.THREAD_P]",
+        "SampleAfterValue": "2000003",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Core clocks when a PAUSE is pending.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xec",
+        "EventName": "CPU_CLK_UNHALTED.PAUSE",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x40",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of Pause instructions",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EdgeDetect": "1",
+        "EventCode": "0xec",
+        "EventName": "CPU_CLK_UNHALTED.PAUSE_INST",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x40",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Fixed Counter: Counts the number of unhalted reference clock cycles.",
+        "Counter": "Fixed counter 2",
+        "EventName": "CPU_CLK_UNHALTED.REF_TSC",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x3",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Reference cycles when the core is not in halt state.",
+        "Counter": "Fixed counter 2",
+        "EventName": "CPU_CLK_UNHALTED.REF_TSC",
+        "PublicDescription": "Counts the number of reference cycles when the core is not in a halt state. The core enters the halt state when it is running the HLT instruction or the MWAIT instruction. This event is not affected by core frequency changes (for example, P states, TM2 transitions) but has the same incrementing frequency as the time stamp counter. This event can approximate elapsed time while the core was not in a halt state. Note: On all current platforms this event stops counting during 'throttling (TM)' states duty off periods the processor is 'halted'.  The counter update is done at a lower clock rate then the core clock the overflow status bit for this counter may appear 'sticky'.  After the counter has overflowed and software clears the overflow status bit and resets the counter to less than MAX. The reset value to the counter is not clocked immediately so the overflow status bit will flip 'high (1)' and generate another PMI (if enabled) after which the reset value gets clocked into the counter. Therefore, software will get the interrupt, read the overflow status bit '1 for bit 34 while the counter value is less than MAX. Software should ignore this case.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of unhalted reference clock cycles at TSC frequency.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x3c",
+        "EventName": "CPU_CLK_UNHALTED.REF_TSC_P",
+        "PublicDescription": "Counts the number of reference cycles that the core is not in a halt state. The core enters the halt state when it is running the HLT instruction. This event is not affected by core frequency changes and increments at a fixed frequency that is also used for the Time Stamp Counter (TSC). This event uses a programmable general purpose performance counter.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Reference cycles when the core is not in halt state.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x3c",
+        "EventName": "CPU_CLK_UNHALTED.REF_TSC_P",
+        "PublicDescription": "Counts the number of reference cycles when the core is not in a halt state. The core enters the halt state when it is running the HLT instruction or the MWAIT instruction. This event is not affected by core frequency changes (for example, P states, TM2 transitions) but has the same incrementing frequency as the time stamp counter. This event can approximate elapsed time while the core was not in a halt state. Note: On all current platforms this event stops counting during 'throttling (TM)' states duty off periods the processor is 'halted'.  The counter update is done at a lower clock rate then the core clock the overflow status bit for this counter may appear 'sticky'.  After the counter has overflowed and software clears the overflow status bit and resets the counter to less than MAX. The reset value to the counter is not clocked immediately so the overflow status bit will flip 'high (1)' and generate another PMI (if enabled) after which the reset value gets clocked into the counter. Therefore, software will get the interrupt, read the overflow status bit '1 for bit 34 while the counter value is less than MAX. Software should ignore this case.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Fixed Counter: Counts the number of unhalted core clock cycles. [This event is alias to CPU_CLK_UNHALTED.CORE]",
+        "Counter": "Fixed counter 1",
+        "EventName": "CPU_CLK_UNHALTED.THREAD",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Core cycles when the thread is not in a halt state.",
+        "Counter": "Fixed counter 1",
+        "EventName": "CPU_CLK_UNHALTED.THREAD",
+        "PublicDescription": "Counts the number of core cycles while the thread is not in a halt state. The thread enters the halt state when it is running the HLT instruction. This event is a component in many key event ratios. The core frequency may change from time to time due to transitions associated with Enhanced Intel SpeedStep Technology or TM2. For this reason this event may have a changing ratio with regards to time. When the core frequency is constant, this event can approximate elapsed time while the core was not in the halt state. It is counted on a dedicated fixed counter, leaving the programmable counters available for other events.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of unhalted core clock cycles. [This event is alias to CPU_CLK_UNHALTED.CORE_P]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x3c",
+        "EventName": "CPU_CLK_UNHALTED.THREAD_P",
+        "PublicDescription": "Counts the number of unhalted core clock cycles [This event is alias to CPU_CLK_UNHALTED.CORE_P]",
+        "SampleAfterValue": "2000003",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Thread cycles when thread is not in halt state [This event is alias to CPU_CLK_UNHALTED.CORE_P]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x3c",
+        "EventName": "CPU_CLK_UNHALTED.THREAD_P",
+        "PublicDescription": "This is an architectural event that counts the number of thread cycles while the thread is not in a halt state. The thread enters the halt state when it is running the HLT instruction. The core frequency may change from time to time due to power or thermal throttling. For this reason, this event may have a changing ratio with regards to wall clock time. [This event is alias to CPU_CLK_UNHALTED.CORE_P]",
+        "SampleAfterValue": "2000003",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Total execution stalls.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "4",
+        "EventCode": "0xa3",
+        "EventName": "CYCLE_ACTIVITY.STALLS_TOTAL",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Count number of times a load is depending on another load that had just write back its data or in previous or 2 cycles back. This event supports in-direct dependency through a single uop.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x02",
+        "EventName": "DEPENDENT_LOADS.ANY",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x7",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles total of 1 uop is executed on all ports and Reservation Station was not empty.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa6",
+        "EventName": "EXE_ACTIVITY.1_PORTS_UTIL",
+        "PublicDescription": "Counts cycles during which a total of 1 uop was executed on all ports and Reservation Station (RS) was not empty.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles total of 2 or 3 uops are executed on all ports and Reservation Station (RS) was not empty.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa6",
+        "EventName": "EXE_ACTIVITY.2_3_PORTS_UTIL",
+        "SampleAfterValue": "2000003",
+        "UMask": "0xc",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles total of 2 uops are executed on all ports and Reservation Station was not empty.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa6",
+        "EventName": "EXE_ACTIVITY.2_PORTS_UTIL",
+        "PublicDescription": "Counts cycles during which a total of 2 uops were executed on all ports and Reservation Station (RS) was not empty.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles total of 3 uops are executed on all ports and Reservation Station was not empty.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa6",
+        "EventName": "EXE_ACTIVITY.3_PORTS_UTIL",
+        "PublicDescription": "Cycles total of 3 uops are executed on all ports and Reservation Station (RS) was not empty.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles total of 4 uops are executed on all ports and Reservation Station was not empty.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa6",
+        "EventName": "EXE_ACTIVITY.4_PORTS_UTIL",
+        "PublicDescription": "Cycles total of 4 uops are executed on all ports and Reservation Station (RS) was not empty.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Execution stalls while memory subsystem has an outstanding load.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "5",
+        "EventCode": "0xa6",
+        "EventName": "EXE_ACTIVITY.BOUND_ON_LOADS",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x21",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles where the Store Buffer was full and no loads caused an execution stall.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "2",
+        "EventCode": "0xa6",
+        "EventName": "EXE_ACTIVITY.BOUND_ON_STORES",
+        "PublicDescription": "Counts cycles where the Store Buffer was full and no loads caused an execution stall.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x40",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles no uop executed while RS was not empty, the SB was not full and there was no outstanding load.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa6",
+        "EventName": "EXE_ACTIVITY.EXE_BOUND_0_PORTS",
+        "PublicDescription": "Number of cycles total of 0 uops executed on all ports, Reservation Station (RS) was not empty, the Store Buffer (SB) was not full and there was no outstanding load.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x80",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "EXE_RETIRED.ANY",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xdf",
+        "EventName": "EXE_RETIRED.ANY",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired conditional instructions (e.g. CMOV) where condition was false",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xdf",
+        "EventName": "EXE_RETIRED.CMOV_NTAKEN",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired conditional instructions (e.g. CMOV) where condition was true",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xdf",
+        "EventName": "EXE_RETIRED.CMOV_TAKEN",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "INST_DECODED.DEC_BW_3MU",
+        "Counter": "0,1,2,3",
+        "EventCode": "0x75",
+        "EventName": "INST_DECODED.DEC_BW_3MU",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Fixed Counter: Counts the number of instructions retired.",
+        "Counter": "Fixed counter 0",
+        "EventName": "INST_RETIRED.ANY",
+        "PublicDescription": "Fixed Counter: Counts the number of instructions retired Available PDIST counters: 32",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Number of instructions retired. Fixed Counter - architectural event",
+        "Counter": "Fixed counter 0",
+        "EventName": "INST_RETIRED.ANY",
+        "PublicDescription": "Counts the number of X86 instructions retired - an Architectural PerfMon event. Counting continues during hardware interrupts, traps, and inside interrupt handlers. Notes: INST_RETIRED.ANY is counted by a designated fixed counter freeing up programmable counters to count other events. INST_RETIRED.ANY_P is counted by a programmable counter. Available PDIST counters: 32",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of instructions retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc0",
+        "EventName": "INST_RETIRED.ANY_P",
+        "PublicDescription": "Counts the number of instructions retired Available PDIST counters: 0,1",
+        "SampleAfterValue": "2000003",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Number of instructions retired. General Counter - architectural event",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc0",
+        "EventName": "INST_RETIRED.ANY_P",
+        "PublicDescription": "Counts the number of X86 instructions retired - an Architectural PerfMon event. Counting continues during hardware interrupts, traps, and inside interrupt handlers. Notes: INST_RETIRED.ANY is counted by a designated fixed counter freeing up programmable counters to count other events. INST_RETIRED.ANY_P is counted by a programmable counter. Available PDIST counters: 0,1",
+        "SampleAfterValue": "2000003",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "retired macro-fused uops when there is a branch in the macro-fused pair (the two instructions that got macro-fused count once in this pmon)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc0",
+        "EventName": "INST_RETIRED.BR_FUSED",
+        "PublicDescription": "retired macro-fused uops when there is a branch in the macro-fused pair (the two instructions that got macro-fused count once in this pmon) Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "INST_RETIRED.MACRO_FUSED",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc0",
+        "EventName": "INST_RETIRED.MACRO_FUSED",
+        "PublicDescription": "INST_RETIRED.MACRO_FUSED Available PDIST counters: 0,1",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x30",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Retired NOP instructions.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc0",
+        "EventName": "INST_RETIRED.NOP",
+        "PublicDescription": "Counts all retired NOP or ENDBR32/64 or PREFETCHIT0/1 instructions Available PDIST counters: 0,1",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Precise instruction retired with PEBS precise-distribution",
+        "Counter": "Fixed counter 0",
+        "EventName": "INST_RETIRED.PREC_DIST",
+        "PublicDescription": "A version of INST_RETIRED that allows for a precise distribution of samples across instructions retired. It utilizes the Precise Distribution of Instructions Retired (PDIR++) feature to fix bias in how retired instructions get sampled. Use on Fixed Counter 0. Available PDIST counters: 32",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Iterations of Repeat string retired instructions.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc0",
+        "EventName": "INST_RETIRED.REP_ITERATION",
+        "PublicDescription": "Number of iterations of Repeat (REP) string retired instructions such as MOVS, CMPS, and SCAS. Each has a byte, word, and doubleword version and string instructions can be repeated using a repetition prefix, REP, that allows their architectural execution to be repeated a number of times as specified by the RCX register. Note: Since the number of iterations within a REP instruction can be significantly affected by fast strings, this event may vary run to run and not match the architectural number of iterations (specified by RCX) Available PDIST counters: 0,1",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Bubble cycles of BPClear. [This event is alias to INT_MISC.BPCLEAR_CYCLES]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xad",
+        "EventName": "INT_MISC.BPCLEAR_BUBBLES",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0xB",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x40",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Bubble cycles of BPClear. [This event is alias to INT_MISC.BPCLEAR_BUBBLES]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xad",
+        "EventName": "INT_MISC.BPCLEAR_CYCLES",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0xB",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x40",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Clears speculative count",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EdgeDetect": "1",
+        "EventCode": "0xad",
+        "EventName": "INT_MISC.CLEARS_COUNT",
+        "PublicDescription": "Counts the number of speculative clears due to any type of branch misprediction or machine clears",
+        "SampleAfterValue": "500009",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts cycles after recovery from a branch misprediction or machine clear till the first uop is issued from the resteered path.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xad",
+        "EventName": "INT_MISC.CLEAR_RESTEER_CYCLES",
+        "PublicDescription": "Cycles after recovery from a branch misprediction or machine clear till the first uop is issued from the resteered path.",
+        "SampleAfterValue": "500009",
+        "UMask": "0x80",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Core cycles the allocator was stalled due to recovery from earlier clear event for this thread",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xad",
+        "EventName": "INT_MISC.RECOVERY_CYCLES",
+        "PublicDescription": "Counts core cycles when the Resource allocator was stalled due to recovery from an earlier branch misprediction or machine clear event.",
+        "SampleAfterValue": "500009",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "INT_MISC.UNKNOWN_BRANCH",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EdgeDetect": "1",
+        "EventCode": "0xad",
+        "EventName": "INT_MISC.UNKNOWN_BRANCH",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x7",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x40",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Bubble cycles of BAClear (Unknown Branch). [This event is alias to INT_MISC.UNKNOWN_BRANCH_CYCLES]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xad",
+        "EventName": "INT_MISC.UNKNOWN_BRANCH_BUBBLES",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x7",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x40",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Bubble cycles of BAClear (Unknown Branch). [This event is alias to INT_MISC.UNKNOWN_BRANCH_BUBBLES]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xad",
+        "EventName": "INT_MISC.UNKNOWN_BRANCH_CYCLES",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x7",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x40",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "TMA slots where uops got dropped",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xad",
+        "EventName": "INT_MISC.UOP_DROPPING",
+        "PublicDescription": "Estimated number of Top-down Microarchitecture Analysis slots that got dropped due to non front-end reasons",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on secondary integer ports 0,1,2,3.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.2ND",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x80",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on all Integer ports.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.ALL",
+        "SampleAfterValue": "1000003",
+        "UMask": "0xff",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on a load port.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.LD",
+        "PublicDescription": "Counts the number of uops executed on a load port.  This event counts for integer uops even if the destination is FP/vector",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on integer port 0.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.P0",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on integer port 1.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.P1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x10",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on integer port 2.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.P2",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x20",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on integer port 3.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.P3",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x40",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on integer port  0,1, 2, 3.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.PRIMARY",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x78",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on a Store address port.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.STA",
+        "PublicDescription": "Counts the number of uops executed on a Store address port. This event counts integer uops even if the data source is FP/vector",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops executed on an integer store data and jump port.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb3",
+        "EventName": "INT_UOPS_EXECUTED.STD_JMP",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Number of vector integer instructions retired of 128-bit vector-width.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe7",
+        "EventName": "INT_VEC_RETIRED.128BIT",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x13",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of vector integer instructions retired of 256-bit vector-width.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe7",
+        "EventName": "INT_VEC_RETIRED.256BIT",
+        "SampleAfterValue": "1000003",
+        "UMask": "0xac",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "integer ADD, SUB, SAD 128-bit vector instructions.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe7",
+        "EventName": "INT_VEC_RETIRED.ADD_128",
+        "PublicDescription": "Number of retired integer ADD/SUB (regular or horizontal), SAD 128-bit vector instructions.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x3",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "integer ADD, SUB, SAD 256-bit vector instructions.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe7",
+        "EventName": "INT_VEC_RETIRED.ADD_256",
+        "PublicDescription": "Number of retired integer ADD/SUB (regular or horizontal), SAD 256-bit vector instructions.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0xc",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "INT_VEC_RETIRED.MUL_256",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe7",
+        "EventName": "INT_VEC_RETIRED.MUL_256",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x80",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "INT_VEC_RETIRED.SHUFFLES",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe7",
+        "EventName": "INT_VEC_RETIRED.SHUFFLES",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x40",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "INT_VEC_RETIRED.VNNI_128",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe7",
+        "EventName": "INT_VEC_RETIRED.VNNI_128",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "INT_VEC_RETIRED.VNNI_256",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe7",
+        "EventName": "INT_VEC_RETIRED.VNNI_256",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x20",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of retired loads that are blocked because it initially appears to be store forward blocked, but subsequently is shown not to be blocked based on 4K alias check.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x03",
+        "EventName": "LD_BLOCKS.ADDRESS_ALIAS",
+        "PublicDescription": "Counts the number of retired loads that are blocked because it initially appears to be store forward blocked, but subsequently is shown not to be blocked based on 4K alias check. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "False dependencies in MOB due to partial compare on address.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x03",
+        "EventName": "LD_BLOCKS.ADDRESS_ALIAS",
+        "PublicDescription": "Counts the number of times a load got blocked due to false dependencies in MOB due to partial compare on address.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of retired loads that are blocked for any of the following reasons:  DTLB miss, address alias, store forward or data unknown (includes memory disambiguation blocks and ESP consuming load blocks), and others.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x03",
+        "EventName": "LD_BLOCKS.ALL",
+        "PublicDescription": "Counts the number of retired loads that are blocked for any of the following reasons:  DTLB miss, address alias, store forward or data unknown (includes memory disambiguation blocks and ESP consuming load blocks), and others. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1f",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Bank conflicts in DCU due to limited lookup ports.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x03",
+        "EventName": "LD_BLOCKS.BANK_CONFLICT",
+        "PublicDescription": "Counts the number of times a load got blocked due to bank conflicts in DCU",
+        "SampleAfterValue": "100003",
+        "UMask": "0x40",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of retired loads that are blocked because its address exactly matches an older store whose data is not ready.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x03",
+        "EventName": "LD_BLOCKS.DATA_UNKNOWN",
+        "PublicDescription": "Counts the number of retired loads that are blocked because its address exactly matches an older store whose data is not ready. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "The number of times that split load operations are temporarily blocked because all resources for handling the split accesses are in use.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x03",
+        "EventName": "LD_BLOCKS.NO_SR",
+        "PublicDescription": "Counts the number of times that split load operations are temporarily blocked because all resources for handling the split accesses are in use.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x88",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of times a load got early blocked due to preceding store operation with unknown address or unknown data. Excluding in-line (immediate) wakeups",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x03",
+        "EventName": "LD_BLOCKS.STORE_EARLY",
+        "SampleAfterValue": "100003",
+        "UMask": "0xa1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of retired loads that are blocked because its address partially overlapped with an older store.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x03",
+        "EventName": "LD_BLOCKS.STORE_FORWARD",
+        "PublicDescription": "Counts the number of retired loads that are blocked because its address partially overlapped with an older store. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Loads blocked due to overlapping with a preceding store that cannot be forwarded.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x03",
+        "EventName": "LD_BLOCKS.STORE_FORWARD",
+        "PublicDescription": "Counts the number of times where store forwarding was prevented for a load operation. The most common case is a load blocked due to the address of memory access (partially) overlapping with a preceding uncompleted store. Note: See the table of not supported store forwards in the Optimization Guide.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x82",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles Uops delivered by the LSD, but didn't come from the decoder.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EventCode": "0xa8",
+        "EventName": "LSD.CYCLES_ACTIVE",
+        "PublicDescription": "Counts the cycles when at least one uop is delivered by the LSD (Loop-stream detector).",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles optimal number of Uops delivered by the LSD, but did not come from the decoder.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "8",
+        "EventCode": "0xa8",
+        "EventName": "LSD.CYCLES_OK",
+        "PublicDescription": "Counts the cycles when optimal number of uops is delivered by the LSD (Loop-stream detector).",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of Uops delivered by the LSD.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa8",
+        "EventName": "LSD.UOPS",
+        "PublicDescription": "Counts the number of uops delivered to the back-end by the LSD(Loop Stream Detector).",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the total number of machine clears for any reason including, but not limited to, memory ordering, memory disambiguation, SMC, and FP assist.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.ANY",
+        "SampleAfterValue": "1000003",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.ANY_FAST",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x80ff",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Number of machine clears (nukes) of any type.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EdgeDetect": "1",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.COUNT",
+        "PublicDescription": "Counts the number of machine clears (nukes) of any type.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of machine clears due to memory ordering in which an internal load passes an older store within the same CPU.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.DISAMBIGUATION",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.DISAMBIGUATION_FAST",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8008",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of virtual traps taken.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.FPC_VIRTUAL_TRAP",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x40",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of machines clears due to memory renaming.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.MRN_NUKE",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x10",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of machine clears that flush the pipeline and restart the machine without the use of microcode.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.MRN_NUKE_FAST",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8010",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of machine clears due to a page fault.  Counts both I-Side and D-Side (Loads/Stores) page faults.  A page fault occurs when either the page is not present, or an access violation occurs.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.PAGE_FAULT",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x20",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of machine clears due to program modifying data (self modifying code) within 1K of a recently fetched code page.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.SMC",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Self-modifying code (SMC) detected.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.SMC",
+        "PublicDescription": "Counts self-modifying code (SMC) detected, which causes a machine clear.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of machine clears due to program modifying data (cross-core modifying code) within 1K of a recently fetched code page.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc3",
+        "EventName": "MACHINE_CLEARS.XMC",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x80",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts cycles where no execution is happening due to loads waiting for L1 cache (that is: no execution & load in flight & no load missed L1 cache)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x46",
+        "EventName": "MEMORY_STALLS.L1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts cycles where no execution is happening due to loads waiting for L2 cache (that is: no execution & load in flight & load missed L1 & no load missed L2 cache)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x46",
+        "EventName": "MEMORY_STALLS.L2",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts cycles where no execution is happening due to loads waiting for L3 cache (that is: no execution & load in flight & load missed L1 & load missed L2 cache & no load missed L3 Cache)",
+        "Counter": "0,1,2,3",
+        "EventCode": "0x46",
+        "EventName": "MEMORY_STALLS.L3",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts cycles where no execution is happening due to loads waiting for Memory (that is: no execution & load in flight & a load missed L3 cache)",
+        "Counter": "0,1,2,3",
+        "EventCode": "0x46",
+        "EventName": "MEMORY_STALLS.MEM",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of CLFLUSH, CLWB, and CLDEMOTE instructions retired. [This event is alias to MISC_RETIRED1.CL_INST]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe0",
+        "EventName": "MICROCODE_RETIRED.CL_INST",
+        "PublicDescription": "Counts the number of CLFLUSH, CLWB, and CLDEMOTE instructions retired. [This event is alias to MISC_RETIRED1.CL_INST] Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0xff",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of LFENCE instructions retired. [This event is alias to MISC_RETIRED1.LFENCE]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe0",
+        "EventName": "MICROCODE_RETIRED.LFENCE",
+        "PublicDescription": "Counts the number of LFENCE instructions retired. [This event is alias to MISC_RETIRED1.LFENCE] Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Count the number of WRMSR instructions retired. [This event is alias to MISC_RETIRED1.WRMSR]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe0",
+        "EventName": "MICROCODE_RETIRED.WRMSR",
+        "PublicDescription": "Count the number of WRMSR instructions retired. [This event is alias to MISC_RETIRED1.WRMSR] Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "LFENCE instructions retired",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe0",
+        "EventName": "MISC2_RETIRED.LFENCE",
+        "PublicDescription": "number of LFENCE retired instructions",
+        "SampleAfterValue": "400009",
+        "UMask": "0x20",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "This event is deprecated. [This event is alias to LBR_INSERTS.ANY]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "Deprecated": "1",
+        "EventCode": "0xe4",
+        "EventName": "MISC_RETIRED.LBR_INSERTS",
+        "PublicDescription": "This event is deprecated. [This event is alias to LBR_INSERTS.ANY] Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "LBR record is inserted",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe4",
+        "EventName": "MISC_RETIRED.LBR_INSERTS",
+        "PublicDescription": "LBR record is inserted Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of CLFLUSH, CLWB, and CLDEMOTE instructions retired. [This event is alias to MICROCODE_RETIRED.CL_INST]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe0",
+        "EventName": "MISC_RETIRED1.CL_INST",
+        "PublicDescription": "Counts the number of CLFLUSH, CLWB, and CLDEMOTE instructions retired. [This event is alias to MICROCODE_RETIRED.CL_INST] Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0xff",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of LFENCE instructions retired. [This event is alias to MICROCODE_RETIRED.LFENCE]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe0",
+        "EventName": "MISC_RETIRED1.LFENCE",
+        "PublicDescription": "Counts the number of LFENCE instructions retired. [This event is alias to MICROCODE_RETIRED.LFENCE] Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Count the number of WRMSR instructions retired. [This event is alias to MICROCODE_RETIRED.WRMSR]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xe0",
+        "EventName": "MISC_RETIRED1.WRMSR",
+        "PublicDescription": "Count the number of WRMSR instructions retired. [This event is alias to MICROCODE_RETIRED.WRMSR] Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Cycles when Reservation Station (RS) is empty for the thread.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa5",
+        "EventName": "RS.EMPTY",
+        "PublicDescription": "Counts cycles during which the reservation station (RS) is empty for this logical processor. This is usually caused when the front-end pipeline runs into starvation periods (e.g. branch mispredictions or i-cache misses)",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x7",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts end of periods where the Reservation Station (RS) was empty.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EdgeDetect": "1",
+        "EventCode": "0xa5",
+        "EventName": "RS.EMPTY_COUNT",
+        "Invert": "1",
+        "PublicDescription": "Counts end of periods where the Reservation Station (RS) was empty. Could be useful to closely sample on front-end latency issues (see the FRONTEND_RETIRED event of designated precise events)",
+        "SampleAfterValue": "100003",
+        "UMask": "0x7",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles when RS was empty and a resource allocation stall is asserted",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa5",
+        "EventName": "RS.EMPTY_RESOURCE",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of issue slots in a UMWAIT or TPAUSE instruction where no uop issues due to the instruction putting the CPU into the C0.1 activity state. For Tremont, UMWAIT and TPAUSE will only put the CPU into C0.1 activity state (not C0.2 activity state).",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x75",
+        "EventName": "SERIALIZATION.C01_MS_SCB",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number issue slots not consumed  due to a  color request for an FCW or MXCSR control register when all 4 colors (copies) are already in use.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x75",
+        "EventName": "SERIALIZATION.COLOR_STALLS",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of issue slots where no uop could issue due to an IQ scoreboard that stalls allocation until a specified older uop retires or (in the case of jump scoreboard) executes. Commonly executed instructions with IQ scoreboards include LFENCE and MFENCE.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x75",
+        "EventName": "SERIALIZATION.IQ_JEU_SCB",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of issue slots not consumed by the backend due to a micro-sequencer (MS) scoreboard, which stalls the front-end from issuing from the UROM until a specified older uop retires.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x75",
+        "EventName": "SERIALIZATION.NON_C01_MS_SCB",
+        "PublicDescription": "Counts the number of issue slots not consumed by the backend due to a micro-sequencer (MS) scoreboard, which stalls the front-end from issuing from the UROM until a specified older uop retires. The most commonly executed instruction with an MS scoreboard is PAUSE.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "This event counts a subset of the Topdown Slots event that were not consumed by the back-end pipeline due to lack of back-end resources, as a result of memory subsystem delays, execution units limitations, or other conditions.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa4",
+        "EventName": "TOPDOWN.BACKEND_BOUND_SLOTS",
+        "PublicDescription": "This event counts a subset of the Topdown Slots event that were not consumed by the back-end pipeline due to lack of back-end resources, as a result of memory subsystem delays, execution units limitations, or other conditions. Software can use this event as the numerator for the Backend Bound metric (or top-level category) of the Top-down Microarchitecture Analysis method.",
+        "SampleAfterValue": "10000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "TMA slots wasted due to incorrect speculations.",
+        "Counter": "0",
+        "EventCode": "0xa4",
+        "EventName": "TOPDOWN.BAD_SPEC_SLOTS",
+        "PublicDescription": "Number of slots of TMA method that were wasted due to incorrect speculation. It covers all types of control-flow or data-related mis-speculations.",
+        "SampleAfterValue": "10000003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "TMA slots wasted due to incorrect speculation by branch mispredictions",
+        "Counter": "0",
+        "EventCode": "0xa4",
+        "EventName": "TOPDOWN.BR_MISPREDICT_SLOTS",
+        "PublicDescription": "Number of TMA slots that were wasted due to incorrect speculation by (any type of) branch mispredictions. This event estimates number of speculative operations that were issued but not retired as well as the out-of-order engine recovery past a branch misprediction.",
+        "SampleAfterValue": "10000003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "TOPDOWN.MEMORY_BOUND_SLOTS",
+        "Counter": "3",
+        "EventCode": "0xa4",
+        "EventName": "TOPDOWN.MEMORY_BOUND_SLOTS",
+        "SampleAfterValue": "10000003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "TMA slots available for an unhalted logical processor. Fixed counter - architectural event",
+        "Counter": "Fixed counter 3",
+        "EventName": "TOPDOWN.SLOTS",
+        "PublicDescription": "Number of available slots for an unhalted logical processor. The event increments by machine-width of the narrowest pipeline as employed by the Top-down Microarchitecture Analysis method (TMA). Software can use this event as the denominator for the top-level metrics of the TMA method. This architectural event is counted on a designated fixed counter (Fixed Counter 3).",
+        "SampleAfterValue": "10000003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "TMA slots available for an unhalted logical processor. General counter - architectural event",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa4",
+        "EventName": "TOPDOWN.SLOTS_P",
+        "PublicDescription": "Counts the number of available slots for an unhalted logical processor. The event increments by machine-width of the narrowest pipeline as employed by the Top-down Microarchitecture Analysis method.",
+        "SampleAfterValue": "10000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Fixed Counter: Counts the number of issue slots that were not consumed by the backend because allocation is stalled due to a mispredicted jump or a machine clear.",
+        "Counter": "Fixed counter 4",
+        "EventName": "TOPDOWN_BAD_SPECULATION.ALL",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x5",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of issue slots that were not consumed by the backend because allocation is stalled due to a mispredicted jump or a machine clear.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x73",
+        "EventName": "TOPDOWN_BAD_SPECULATION.ALL_P",
+        "PublicDescription": "Counts the total number of issue slots that were not consumed by the backend because allocation is stalled due to a mispredicted jump or a machine clear. Only issue slots wasted due to fast nukes such as memory ordering nukes are counted. Other nukes are not accounted for. Counts all issue slots blocked during this recovery window, including relevant microcode flows, and while uops are not yet available in the instruction queue (IQ) or until an FE_BOUND event occurs besides OTHER and CISC. Also includes the issue slots that were consumed by the backend but were thrown away because they were younger than the mispredict or machine clear.",
+        "SampleAfterValue": "1000003",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of retirement slots not consumed due to backend stalls. [This event is alias to TOPDOWN_BE_BOUND.ALL_P]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa4",
+        "EventName": "TOPDOWN_BE_BOUND.ALL",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of retirement slots not consumed due to backend stalls. [This event is alias to TOPDOWN_BE_BOUND.ALL]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xa4",
+        "EventName": "TOPDOWN_BE_BOUND.ALL_P",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Fixed Counter: Counts the number of retirement slots not consumed due to front end stalls.",
+        "Counter": "Fixed counter 5",
+        "EventName": "TOPDOWN_FE_BOUND.ALL",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x6",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of retirement slots not consumed due to front end stalls.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x9c",
+        "EventName": "TOPDOWN_FE_BOUND.ALL_P",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Fixed Counter: Counts the number of consumed retirement slots.",
+        "Counter": "Fixed counter 6",
+        "EventName": "TOPDOWN_RETIRING.ALL",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x7",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of consumed retirement slots.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc2",
+        "EventName": "TOPDOWN_RETIRING.ALL_P",
+        "PublicDescription": "Counts the number of consumed retirement slots. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "This event counts the number of not dec-by-all uops decoded (any non dec-by-all uops) e.g. when a 2 uop instruction is decoded this events counts 2.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x76",
+        "EventName": "UOPS_DECODED.HEAVY_INST_UOPS",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Uops executed on INT EU ALU ports.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "UOPS_DISPATCHED.ALU",
+        "PublicDescription": "Number of ALU integer uops dispatch to execution.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Uops executed on any INT EU ports",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "UOPS_DISPATCHED.INT_EU_ALL",
+        "PublicDescription": "Number of integer uops dispatched to execution.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of Uops dispatched/executed by any of the 3 JEUs (all ups that hold the JEU including macro; micro jumps; fetch-from-eip)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "UOPS_DISPATCHED.JMP",
+        "PublicDescription": "Number of jump uops dispatch to execution",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x40",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Uops executed on Load ports",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "UOPS_DISPATCHED.LOAD",
+        "PublicDescription": "Number of Load uops dispatched to execution.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of (shift) 1-cycle Uops dispatched/executed by any of the Shift Eus",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "UOPS_DISPATCHED.SHIFT",
+        "PublicDescription": "Number of SHIFT integer uops dispatch to execution",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x20",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of Uops dispatched/executed by Slow EU (e.g. 3+ cycles LEA, >1 cycles shift, iDIVs, CR; *H operation)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "UOPS_DISPATCHED.SLOW",
+        "PublicDescription": "Number of Slow integer uops dispatch to execution.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of Uops dispatched on STA ports",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "UOPS_DISPATCHED.STA",
+        "PublicDescription": "Number of STA (Store Address) uops dispatch to execution",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x80",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Uops executed on STD ports",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb2",
+        "EventName": "UOPS_DISPATCHED.STD",
+        "PublicDescription": "Number of STD (Store Data) uops dispatch to execution",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles where at least 1 uop was executed per-thread",
+        "Counter": "3",
+        "CounterMask": "1",
+        "EventCode": "0xb1",
+        "EventName": "UOPS_EXECUTED.CYCLES_GE_1",
+        "PublicDescription": "Cycles where at least 1 uop was executed per-thread.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles where at least 2 uops were executed per-thread",
+        "Counter": "3",
+        "CounterMask": "2",
+        "EventCode": "0xb1",
+        "EventName": "UOPS_EXECUTED.CYCLES_GE_2",
+        "PublicDescription": "Cycles where at least 2 uops were executed per-thread.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles where at least 3 uops were executed per-thread",
+        "Counter": "3",
+        "CounterMask": "3",
+        "EventCode": "0xb1",
+        "EventName": "UOPS_EXECUTED.CYCLES_GE_3",
+        "PublicDescription": "Cycles where at least 3 uops were executed per-thread.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles where at least 4 uops were executed per-thread",
+        "Counter": "3",
+        "CounterMask": "4",
+        "EventCode": "0xb1",
+        "EventName": "UOPS_EXECUTED.CYCLES_GE_4",
+        "PublicDescription": "Cycles where at least 4 uops were executed per-thread.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts number of cycles no uops were dispatched to be executed on this thread.",
+        "Counter": "3",
+        "CounterMask": "1",
+        "EventCode": "0xb1",
+        "EventName": "UOPS_EXECUTED.STALLS",
+        "Invert": "1",
+        "PublicDescription": "Counts cycles during which no uops were dispatched from the Reservation Station (RS) per thread.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of uops to be executed per-thread each cycle.",
+        "Counter": "3",
+        "EventCode": "0xb1",
+        "EventName": "UOPS_EXECUTED.THREAD",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of x87 uops dispatched.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xb1",
+        "EventName": "UOPS_EXECUTED.X87",
+        "PublicDescription": "Counts the number of x87 uops executed.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of uops issued by the front end every cycle.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x0e",
+        "EventName": "UOPS_ISSUED.ANY",
+        "PublicDescription": "Counts the number of uops issued by the front end every cycle. When 4-uops are requested and only 2-uops are delivered, the event counts 2.  Uops_issued correlates to the number of ROB entries.  If uop takes 2 ROB slots it counts as 2 uops_issued.",
+        "SampleAfterValue": "1000003",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Uops that RAT issues to RS",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xae",
+        "EventName": "UOPS_ISSUED.ANY",
+        "PublicDescription": "Counts the number of uops that the Resource Allocation Table (RAT) issues to the Reservation Station (RS).",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "UOPS_ISSUED.CYCLES",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EventCode": "0xae",
+        "EventName": "UOPS_ISSUED.CYCLES",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles with retired uop(s).",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EventCode": "0xc2",
+        "EventName": "UOPS_RETIRED.CYCLES",
+        "PublicDescription": "Counts cycles where at least one uop has retired.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of uops retired that were dual destination.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xc2",
+        "EventName": "UOPS_RETIRED.DUAL_DEST",
+        "PublicDescription": "Counts the number of uops retired that were dual destination.  Dual destination (dual dest) micro-operations require writing to two separate destination registers or memory addresses upon execution. These uops consume two entries in the ROB and tracked separately because they involve more complex operations that necessitate multiple results. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x10",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops retired that are the last uop of a macro-instruction.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xc2",
+        "EventName": "UOPS_RETIRED.EOM",
+        "PublicDescription": "Counts the number of uops retired that are the last uop of a macro-instruction.   EOM uops indicate the 'end of a macro-instruction' and play a crucial role in the processor's control flow and recovery mechanisms. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Retired uops except the last uop of each instruction.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc2",
+        "EventName": "UOPS_RETIRED.HEAVY",
+        "PublicDescription": "Counts the number of retired micro-operations (uops) except the last uop of each instruction. An instruction that is decoded into less than two uops does not contribute to the count.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x1",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of integer divide uops retired.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xc2",
+        "EventName": "UOPS_RETIRED.IDIV",
+        "PublicDescription": "Counts the number of integer divide uops retired. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x80",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops retired that originated from a loop stream detector.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xc2",
+        "EventName": "UOPS_RETIRED.LSD",
+        "PublicDescription": "Counts the number of uops retired that originated from a loop stream detector. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x20",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of uops that are from the complex flows issued by the micro-sequencer (MS).  This includes uops from flows due to complex instructions, faults, assists, and inserted flows.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xc2",
+        "EventName": "UOPS_RETIRED.MS",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x4",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "UOPS_RETIRED.MS",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc2",
+        "EventName": "UOPS_RETIRED.MS",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x8",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of non-speculative switches to the Microcode Sequencer (MS)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EdgeDetect": "1",
+        "EventCode": "0xc2",
+        "EventName": "UOPS_RETIRED.MS_SWITCHES",
+        "MSRIndex": "0x3F7",
+        "MSRValue": "0x8",
+        "PublicDescription": "Switches to the Microcode Sequencer",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "This event counts a subset of the Topdown Slots event that are utilized by operations that eventually get retired (committed) by the processor pipeline. Usually, this event positively correlates with higher performance  for example, as measured by the instructions-per-cycle metric.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc2",
+        "EventName": "UOPS_RETIRED.SLOTS",
+        "PublicDescription": "This event counts a subset of the Topdown Slots event that are utilized by operations that eventually get retired (committed) by the processor pipeline. Usually, this event positively correlates with higher performance  for example, as measured by the instructions-per-cycle metric. Software can use this event as the numerator for the Retiring metric (or top-level category) of the Top-down Microarchitecture Analysis method.",
+        "SampleAfterValue": "2000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles without actually retired uops.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EventCode": "0xc2",
+        "EventName": "UOPS_RETIRED.STALLS",
+        "Invert": "1",
+        "PublicDescription": "This event counts cycles without actually retired uops.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of x87 uops retired, includes those in MS flows.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "Data_LA": "1",
+        "EventCode": "0xc2",
+        "EventName": "UOPS_RETIRED.X87",
+        "PublicDescription": "Counts the number of x87 uops retired, includes those in ms flows Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "Unit": "cpu_atom"
+    }
+]
diff --git a/tools/perf/pmu-events/arch/x86/novalake/uncore-memory.json b/tools/perf/pmu-events/arch/x86/novalake/uncore-memory.json
new file mode 100644
index 0000000000000..6f809cc34966f
--- /dev/null
+++ b/tools/perf/pmu-events/arch/x86/novalake/uncore-memory.json
@@ -0,0 +1,42 @@
+[
+    {
+        "BriefDescription": "Read CAS command sent to DRAM (including Read CAS with auto precharge)",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x22",
+        "EventName": "UNC_M_CAS_COUNT_RD",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Write CAS command sent to DRAM (including Write CAS with auto precharge)",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x23",
+        "EventName": "UNC_M_CAS_COUNT_WR",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "This counter counts number of bytes read, in 32B chunk, per DDR channel. Counter increments by 1 after receiving 32B chunk data. Note this includes data from mode register accesses",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x3A",
+        "EventName": "UNC_M_RD_DATA",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Total number of read and write byte transfers to/from DRAM, in 32B chunk, per DDR channel. Counter increments by 1 after sending or receiving 32B chunk data. Note this includes data from mode register accesses. Does not support channel masking",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x3C",
+        "EventName": "UNC_M_TOTAL_DATA",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "This counter counts number of bytes written, in 32B chunk, per DDR channel. Counter increments by 1 after sending 32B chunk data.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x3B",
+        "EventName": "UNC_M_WR_DATA",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    }
+]
diff --git a/tools/perf/pmu-events/arch/x86/novalake/virtual-memory.json b/tools/perf/pmu-events/arch/x86/novalake/virtual-memory.json
new file mode 100644
index 0000000000000..356a74434288f
--- /dev/null
+++ b/tools/perf/pmu-events/arch/x86/novalake/virtual-memory.json
@@ -0,0 +1,338 @@
+[
+    {
+        "BriefDescription": "Counts the number of page walks initiated by a demand load that missed the first and second level TLBs.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x08",
+        "EventName": "DTLB_LOAD_MISSES.MISS_CAUSED_WALK",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of first level TLB misses but second level hits due to a demand load that did not start a page walk. Accounts for all page sizes. Will result in a DTLB write from STLB.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x08",
+        "EventName": "DTLB_LOAD_MISSES.STLB_HIT",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x60",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Loads that miss the DTLB and hit the STLB.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x12",
+        "EventName": "DTLB_LOAD_MISSES.STLB_HIT",
+        "PublicDescription": "Counts loads that miss the DTLB (Data TLB) and hit the STLB (Second level TLB).",
+        "SampleAfterValue": "100003",
+        "UMask": "0x320",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles when at least one PMH is busy with a page walk for a demand load.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EventCode": "0x12",
+        "EventName": "DTLB_LOAD_MISSES.WALK_ACTIVE",
+        "PublicDescription": "Counts cycles when at least one PMH (Page Miss Handler) is busy with a page walk for a demand load.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Load miss in all TLB levels causes a page walk that completes. (All page sizes)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x12",
+        "EventName": "DTLB_LOAD_MISSES.WALK_COMPLETED",
+        "PublicDescription": "Counts completed page walks  (all page sizes) caused by demand data loads. This implies it missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.",
+        "SampleAfterValue": "100003",
+        "UMask": "0xe",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Page walks completed due to a demand data load to a 1G page.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x12",
+        "EventName": "DTLB_LOAD_MISSES.WALK_COMPLETED_1G",
+        "PublicDescription": "Counts completed page walks  (1G sizes) caused by demand data loads. This implies address translations missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Page walks completed due to a demand data load to a 2M/4M page.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x12",
+        "EventName": "DTLB_LOAD_MISSES.WALK_COMPLETED_2M_4M",
+        "PublicDescription": "Counts completed page walks  (2M/4M sizes) caused by demand data loads. This implies address translations missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of page walks completed due to load DTLB misses to a 4K page.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x08",
+        "EventName": "DTLB_LOAD_MISSES.WALK_COMPLETED_4K",
+        "PublicDescription": "Counts the number of page walks completed due to loads (including SW prefetches) whose address translations missed in all Translation Lookaside Buffer (TLB) levels and were mapped to 4K pages. Includes page walks that page fault.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Page walks completed due to a demand data load to a 4K page.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x12",
+        "EventName": "DTLB_LOAD_MISSES.WALK_COMPLETED_4K",
+        "PublicDescription": "Counts completed page walks  (4K sizes) caused by demand data loads. This implies address translations missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of page walks outstanding for Loads (demand or SW prefetch) in PMH every cycle.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x08",
+        "EventName": "DTLB_LOAD_MISSES.WALK_PENDING",
+        "PublicDescription": "Counts the number of page walks outstanding for Loads (demand or SW prefetch) in PMH every cycle.  A PMH page walk is outstanding from page walk start till PMH becomes idle again (ready to serve next walk). Includes EPT-walk intervals.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x10",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Number of page walks outstanding for a demand load in the PMH each cycle.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x12",
+        "EventName": "DTLB_LOAD_MISSES.WALK_PENDING",
+        "PublicDescription": "Counts the number of page walks outstanding for a demand load in the PMH (Page Miss Handler) each cycle.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of page walks initiated by a store that missed the first and second level TLBs.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x49",
+        "EventName": "DTLB_STORE_MISSES.MISS_CAUSED_WALK",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of first level TLB misses but second level hits due to stores that did not start a page walk. Accounts for all page sizes. Will result in a DTLB write from STLB.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x49",
+        "EventName": "DTLB_STORE_MISSES.STLB_HIT",
+        "PublicDescription": "Counts the number of first level TLB misses but second level hits due to a demand load that did not start a page walk. Accounts for all page sizes. Will result in a DTLB write from STLB.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x60",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Stores that miss the DTLB and hit the STLB.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x13",
+        "EventName": "DTLB_STORE_MISSES.STLB_HIT",
+        "PublicDescription": "Counts stores that miss the DTLB (Data TLB) and hit the STLB (2nd Level TLB).",
+        "SampleAfterValue": "100003",
+        "UMask": "0x320",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles when at least one PMH is busy with a page walk for a store.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EventCode": "0x13",
+        "EventName": "DTLB_STORE_MISSES.WALK_ACTIVE",
+        "PublicDescription": "Counts cycles when at least one PMH (Page Miss Handler) is busy with a page walk for a store.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Store misses in all TLB levels causes a page walk that completes. (All page sizes)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x13",
+        "EventName": "DTLB_STORE_MISSES.WALK_COMPLETED",
+        "PublicDescription": "Counts completed page walks  (all page sizes) caused by demand data stores. This implies it missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.",
+        "SampleAfterValue": "100003",
+        "UMask": "0xe",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Page walks completed due to a demand data store to a 1G page.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x13",
+        "EventName": "DTLB_STORE_MISSES.WALK_COMPLETED_1G",
+        "PublicDescription": "Counts completed page walks  (1G sizes) caused by demand data stores. This implies address translations missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x8",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Page walks completed due to a demand data store to a 2M/4M page.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x13",
+        "EventName": "DTLB_STORE_MISSES.WALK_COMPLETED_2M_4M",
+        "PublicDescription": "Counts completed page walks  (2M/4M sizes) caused by demand data stores. This implies address translations missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of page walks completed due to store DTLB misses to a 4K page.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x49",
+        "EventName": "DTLB_STORE_MISSES.WALK_COMPLETED_4K",
+        "PublicDescription": "Counts the number of page walks completed due to stores whose address translations missed in all Translation Lookaside Buffer (TLB) levels and were mapped to 4K pages.  Includes page walks that page fault.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Page walks completed due to a demand data store to a 4K page.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x13",
+        "EventName": "DTLB_STORE_MISSES.WALK_COMPLETED_4K",
+        "PublicDescription": "Counts completed page walks  (4K sizes) caused by demand data stores. This implies address translations missed in the DTLB and further levels of TLB. The page walk can end with or without a fault.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of page walks outstanding in the page miss handler (PMH) for stores every cycle.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x49",
+        "EventName": "DTLB_STORE_MISSES.WALK_PENDING",
+        "PublicDescription": "Counts the number of page walks outstanding in the page miss handler (PMH) for stores every cycle. A PMH page walk is outstanding from page walk start till PMH becomes idle again (ready to serve next walk). Includes EPT-walk intervals.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x10",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Number of page walks outstanding for a store in the PMH each cycle.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x13",
+        "EventName": "DTLB_STORE_MISSES.WALK_PENDING",
+        "PublicDescription": "Counts the number of page walks outstanding for a store in the PMH (Page Miss Handler) each cycle.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of page walks initiated by a instruction fetch that missed the first and second level TLBs.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x85",
+        "EventName": "ITLB_MISSES.MISS_CAUSED_WALK",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x1",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of first level TLB misses but second level hits due to an instruction fetch that did not start a page walk. Account for all pages sizes. Will result in an ITLB write from STLB.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x85",
+        "EventName": "ITLB_MISSES.STLB_HIT",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x20",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Instruction fetch requests that miss the ITLB and hit the STLB.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x11",
+        "EventName": "ITLB_MISSES.STLB_HIT",
+        "PublicDescription": "Counts instruction fetch requests that miss the ITLB (Instruction TLB) and hit the STLB (Second-level TLB).",
+        "SampleAfterValue": "100003",
+        "UMask": "0x120",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Cycles when at least one PMH is busy with a page walk for code (instruction fetch) request.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "CounterMask": "1",
+        "EventCode": "0x11",
+        "EventName": "ITLB_MISSES.WALK_ACTIVE",
+        "PublicDescription": "Counts cycles when at least one PMH (Page Miss Handler) is busy with a page walk for a code (instruction fetch) request.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Code miss in all TLB levels causes a page walk that completes. (All page sizes)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x11",
+        "EventName": "ITLB_MISSES.WALK_COMPLETED",
+        "PublicDescription": "Counts completed page walks (all page sizes) caused by a code fetch. This implies it missed in the ITLB (Instruction TLB) and further levels of TLB. The page walk can end with or without a fault.",
+        "SampleAfterValue": "100003",
+        "UMask": "0xe",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Code miss in all TLB levels causes a page walk that completes. (2M/4M)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x11",
+        "EventName": "ITLB_MISSES.WALK_COMPLETED_2M_4M",
+        "PublicDescription": "Counts completed page walks (2M/4M page sizes) caused by a code fetch. This implies it missed in the ITLB (Instruction TLB) and further levels of TLB. The page walk can end with or without a fault.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x4",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of page walks completed due to instruction fetch misses to a 4K page.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x85",
+        "EventName": "ITLB_MISSES.WALK_COMPLETED_4K",
+        "PublicDescription": "Counts the number of page walks completed due to instruction fetches whose address translations missed in all Translation Lookaside Buffer (TLB) levels and were mapped to 4K pages.  Includes page walks that page fault.",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x2",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Code miss in all TLB levels causes a page walk that completes. (4K)",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x11",
+        "EventName": "ITLB_MISSES.WALK_COMPLETED_4K",
+        "PublicDescription": "Counts completed page walks (4K page sizes) caused by a code fetch. This implies it missed in the ITLB (Instruction TLB) and further levels of TLB. The page walk can end with or without a fault.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x2",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Number of page walks outstanding for an outstanding code request in the PMH each cycle.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x11",
+        "EventName": "ITLB_MISSES.WALK_PENDING",
+        "PublicDescription": "Counts the number of page walks outstanding for an outstanding code (instruction fetch) request in the PMH (Page Miss Handler) each cycle.",
+        "SampleAfterValue": "100003",
+        "UMask": "0x10",
+        "Unit": "cpu_core"
+    },
+    {
+        "BriefDescription": "Counts the number of retired loads that are blocked due to a first level TLB miss.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x03",
+        "EventName": "LD_BLOCKS.DTLB_MISS",
+        "PublicDescription": "Counts the number of retired loads that are blocked due to a first level TLB miss. Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer is stalled due to a DTLB miss.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.DTLB_MISS",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x10",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of cycles that the head (oldest load) of the load buffer and retirement are both stalled due to a DTLB miss.",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0x05",
+        "EventName": "LD_HEAD.DTLB_MISS_AT_RET",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x90",
+        "Unit": "cpu_atom"
+    }
+]
diff --git a/tools/perf/pmu-events/arch/x86/pantherlake/counter.json b/tools/perf/pmu-events/arch/x86/pantherlake/counter.json
index 432b6946ccbc3..9794b435f6503 100644
--- a/tools/perf/pmu-events/arch/x86/pantherlake/counter.json
+++ b/tools/perf/pmu-events/arch/x86/pantherlake/counter.json
@@ -13,5 +13,10 @@
         "Unit": "iMC",
         "CountersNumFixed": "0",
         "CountersNumGeneric": "5"
+    },
+    {
+        "Unit": "SANTA",
+        "CountersNumFixed": 1,
+        "CountersNumGeneric": "0"
     }
 ]
\ No newline at end of file
diff --git a/tools/perf/pmu-events/arch/x86/pantherlake/pipeline.json b/tools/perf/pmu-events/arch/x86/pantherlake/pipeline.json
index d476bad5e2a7b..74209da5bc27b 100644
--- a/tools/perf/pmu-events/arch/x86/pantherlake/pipeline.json
+++ b/tools/perf/pmu-events/arch/x86/pantherlake/pipeline.json
@@ -887,11 +887,32 @@
         "Unit": "cpu_core"
     },
     {
-        "BriefDescription": "This event counts the number of mispredicted ret instructions retired. Non PEBS [This event is alias to BR_MISP_RETIRED.RET]",
+        "BriefDescription": "This event is deprecated. [This event is alias to BR_MISP_RETIRED.NEAR_RETURN]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "Deprecated": "1",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.NEAR_RET",
+        "PublicDescription": "This event is deprecated. [This event is alias to BR_MISP_RETIRED.NEAR_RETURN] Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "Counts the number of mispredicted near RET branch instructions retired. [This event is alias to BR_MISP_RETIRED.NEAR_RET]",
+        "Counter": "0,1,2,3,4,5,6,7",
+        "EventCode": "0xc5",
+        "EventName": "BR_MISP_RETIRED.NEAR_RETURN",
+        "PublicDescription": "Counts the number of mispredicted near RET branch instructions retired. [This event is alias to BR_MISP_RETIRED.NEAR_RET] Available PDIST counters: 0,1",
+        "SampleAfterValue": "1000003",
+        "UMask": "0x8",
+        "Unit": "cpu_atom"
+    },
+    {
+        "BriefDescription": "This event counts the number of mispredicted ret instructions retired [This event is alias to BR_MISP_RETIRED.RET]",
         "Counter": "0,1,2,3,4,5,6,7,8,9",
         "EventCode": "0xc5",
         "EventName": "BR_MISP_RETIRED.NEAR_RETURN",
-        "PublicDescription": "This is a non-precise version (that is, does not use PEBS) of the event that counts mispredicted return instructions retired. [This event is alias to BR_MISP_RETIRED.RET] Available PDIST counters: 0,1",
+        "PublicDescription": "This event counts the number of mispredicted ret instructions retired [This event is alias to BR_MISP_RETIRED.RET] Available PDIST counters: 0,1",
         "SampleAfterValue": "100007",
         "UMask": "0x8",
         "Unit": "cpu_core"
@@ -1313,7 +1334,7 @@
         "Counter": "0,1,2,3,4,5,6,7,8,9",
         "EventCode": "0xc0",
         "EventName": "INST_RETIRED.REP_ITERATION",
-        "PublicDescription": "Number of iterations of Repeat (REP) string retired instructions such as MOVS, CMPS, and SCAS. Each has a byte, word, and doubleword version and string instructions can be repeated using a repetition prefix, REP, that allows their architectural execution to be repeated a number of times as specified by the RCX register. Note the number of iterations is implementation-dependent. Available PDIST counters: 0,1",
+        "PublicDescription": "Number of iterations of Repeat (REP) string retired instructions such as MOVS, CMPS, and SCAS. Each has a byte, word, and doubleword version and string instructions can be repeated using a repetition prefix, REP, that allows their architectural execution to be repeated a number of times as specified by the RCX register. Note: Since the number of iterations within a REP instruction can be significantly affected by fast strings, this event may vary run to run and not match the architectural number of iterations (specified by RCX) Available PDIST counters: 0,1",
         "SampleAfterValue": "2000003",
         "UMask": "0x8",
         "Unit": "cpu_core"
@@ -1726,7 +1747,7 @@
         "Unit": "cpu_atom"
     },
     {
-        "BriefDescription": "Counts the number of machines clears due to memory renaming.",
+        "BriefDescription": "Counts the number of machine clears due to memory renaming.",
         "Counter": "0,1,2,3,4,5,6,7",
         "EventCode": "0xc3",
         "EventName": "MACHINE_CLEARS.MRN_NUKE",
@@ -1930,7 +1951,7 @@
         "Unit": "cpu_atom"
     },
     {
-        "BriefDescription": "Counts the number issue slots not consumed  due to a  color request for an FCW or MXCSR control register when all 4 colors (copies) are already in use.",
+        "BriefDescription": "Counts the number of issue slots not consumed due to a color request for an FCW or MXCSR control register when all 4 colors (copies) are already in use.",
         "Counter": "0,1,2,3,4,5,6,7",
         "EventCode": "0x75",
         "EventName": "SERIALIZATION.COLOR_STALLS",
diff --git a/tools/perf/pmu-events/arch/x86/pantherlake/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/pantherlake/uncore-interconnect.json
new file mode 100644
index 0000000000000..69ef928d57f6c
--- /dev/null
+++ b/tools/perf/pmu-events/arch/x86/pantherlake/uncore-interconnect.json
@@ -0,0 +1,10 @@
+[
+    {
+        "BriefDescription": "This 48-bit fixed counter counts the UCLK cycles.",
+        "Counter": "FIXED",
+        "EventCode": "0xff",
+        "EventName": "UNC_CLOCK.SOCKET",
+        "PerPkg": "1",
+        "Unit": "SANTA"
+    }
+]
diff --git a/tools/perf/pmu-events/arch/x86/pantherlake/uncore-memory.json b/tools/perf/pmu-events/arch/x86/pantherlake/uncore-memory.json
index a881b99be5f34..8faa03e1c6d01 100644
--- a/tools/perf/pmu-events/arch/x86/pantherlake/uncore-memory.json
+++ b/tools/perf/pmu-events/arch/x86/pantherlake/uncore-memory.json
@@ -1,6 +1,30 @@
 [
     {
-        "BriefDescription": "Read CAS command sent to DRAM",
+        "BriefDescription": "ACT command for a read request sent to DRAM.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x24",
+        "EventName": "UNC_M_ACT_COUNT_RD",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "ACT command sent to DRAM.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x26",
+        "EventName": "UNC_M_ACT_COUNT_TOTAL",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "ACT command for a write request sent to DRAM.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x25",
+        "EventName": "UNC_M_ACT_COUNT_WR",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Read CAS command sent to DRAM.",
         "Counter": "0,1,2,3,4",
         "EventCode": "0x22",
         "EventName": "UNC_M_CAS_COUNT_RD",
@@ -8,13 +32,153 @@
         "Unit": "iMC"
     },
     {
-        "BriefDescription": "Write CAS command sent to DRAM",
+        "BriefDescription": "Write CAS command sent to DRAM.",
         "Counter": "0,1,2,3,4",
         "EventCode": "0x23",
         "EventName": "UNC_M_CAS_COUNT_WR",
         "PerPkg": "1",
         "Unit": "iMC"
     },
+    {
+        "BriefDescription": "Counting the number of clocks.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x01",
+        "EventName": "UNC_M_CLOCKTICKS",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "CKE in DRAM is low.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x29",
+        "EventName": "UNC_M_DRAM_CKE_OFF_CYCLES",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming read request page status is Page Empty.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x1D",
+        "EventName": "UNC_M_DRAM_PAGE_EMPTY_RD",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "PublicDescription": "incoming read request page status is Page Empty",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming write request page status is Page Empty.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x20",
+        "EventName": "UNC_M_DRAM_PAGE_EMPTY_WR",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "PublicDescription": "incoming write request page status is Page Empty",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming read request page status is Page Hit.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x1C",
+        "EventName": "UNC_M_DRAM_PAGE_HIT_RD",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "PublicDescription": "incoming read request page status is Page Hit",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming write request page status is Page Hit.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x1F",
+        "EventName": "UNC_M_DRAM_PAGE_HIT_WR",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "PublicDescription": "incoming write request page status is Page Hit",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming read request page status is Page Miss.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x1E",
+        "EventName": "UNC_M_DRAM_PAGE_MISS_RD",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "PublicDescription": "incoming read request page status is Page Miss",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming write request page status is Page Miss.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x21",
+        "EventName": "UNC_M_DRAM_PAGE_MISS_WR",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "PublicDescription": "incoming write request page status is Page Miss",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "DRAM in Self-refresh (all channels).",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x12",
+        "EventName": "UNC_M_DRAM_SELF_REFRESH",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Any Rank at Hot state.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x19",
+        "EventName": "UNC_M_DRAM_THERMAL_HOT",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Any Rank at Warm state.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x1A",
+        "EventName": "UNC_M_DRAM_THERMAL_WARM",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "PRE command sent to DRAM for a read/write request.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x27",
+        "EventName": "UNC_M_PRE_COUNT_PAGE_MISS",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Counts number of bytes read, in 32B chunk, per DDR channel. Counter increments by 1 after receiving 32B chunk data.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x3A",
+        "EventName": "UNC_M_RD_DATA",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Number of VC0 read in channel0  - this event can increment by more than 1 (per channel/sub-ch).",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x13",
+        "EventName": "UNC_M_RD_OCCUPANCY_CH0",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "PublicDescription": "Number of VC0 read in channel0  - this event can  increment by more than 1 (per channel/sub-ch)",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Number of VC0 read in channel1 - this event can increment by more than 1 (per channel/sub-ch).",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x14",
+        "EventName": "UNC_M_RD_OCCUPANCY_CH1",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
     {
         "BriefDescription": "Total number of read and write byte transfers to/from DRAM, in 32B chunk, per DDR channel. Counter increments by 1 after sending  or receiving 32B chunk data.",
         "Counter": "0,1,2,3,4",
@@ -22,5 +186,58 @@
         "EventName": "UNC_M_TOTAL_DATA",
         "PerPkg": "1",
         "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Total number of requests entering MC, this is the sum of all RD + WR requests for all VCs.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x39",
+        "EventName": "UNC_M_TOTAL_REQUESTS",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming VC0 read request.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x02",
+        "EventName": "UNC_M_VC0_REQUESTS_RD",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming VC0 write request.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x03",
+        "EventName": "UNC_M_VC0_REQUESTS_WR",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming VC1 read request.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x04",
+        "EventName": "UNC_M_VC1_REQUESTS_RD",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Incoming VC1 write request.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x05",
+        "EventName": "UNC_M_VC1_REQUESTS_WR",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "Unit": "iMC"
+    },
+    {
+        "BriefDescription": "Counts number of bytes written, in 32B chunk, per DDR channel. Counter increments by 1 after sending 32B chunk data.",
+        "Counter": "0,1,2,3,4",
+        "EventCode": "0x3B",
+        "EventName": "UNC_M_WR_DATA",
+        "PerPkg": "1",
+        "Unit": "iMC"
     }
 ]
diff --git a/tools/perf/pmu-events/arch/x86/sierraforest/counter.json b/tools/perf/pmu-events/arch/x86/sierraforest/counter.json
deleted file mode 100644
index a92c75109e6ad..0000000000000
--- a/tools/perf/pmu-events/arch/x86/sierraforest/counter.json
+++ /dev/null
@@ -1,77 +0,0 @@
-[
-    {
-        "Unit": "core",
-        "CountersNumFixed": "3",
-        "CountersNumGeneric": "8"
-    },
-    {
-        "Unit": "B2CMI",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": "4"
-    },
-    {
-        "Unit": "CHA",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": "4"
-    },
-    {
-        "Unit": "IMC",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": "4"
-    },
-    {
-        "Unit": "CXLCM",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": 8
-    },
-    {
-        "Unit": "CXLDP",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": 4
-    },
-    {
-        "Unit": "B2HOT",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": 4
-    },
-    {
-        "Unit": "IIO",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": "4"
-    },
-    {
-        "Unit": "IRP",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": "4"
-    },
-    {
-        "Unit": "UPI",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": "4"
-    },
-    {
-        "Unit": "B2UPI",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": 4
-    },
-    {
-        "Unit": "B2CXL",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": 4
-    },
-    {
-        "Unit": "PCU",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": "4"
-    },
-    {
-        "Unit": "CHACMS",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": "4"
-    },
-    {
-        "Unit": "MDF",
-        "CountersNumFixed": "0",
-        "CountersNumGeneric": 4
-    }
-]
\ No newline at end of file
diff --git a/tools/perf/pmu-events/arch/x86/sierraforest/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/sierraforest/uncore-interconnect.json
index 251e5d20fefec..1f6e0970141a1 100644
--- a/tools/perf/pmu-events/arch/x86/sierraforest/uncore-interconnect.json
+++ b/tools/perf/pmu-events/arch/x86/sierraforest/uncore-interconnect.json
@@ -814,6 +814,16 @@
         "PerPkg": "1",
         "Unit": "IRP"
     },
+    {
+        "BriefDescription": "Counts Timeouts - Set 0 : Cache Inserts of Write Transactions as Secondary",
+        "Counter": "0,1,2,3",
+        "EventCode": "0x1E",
+        "EventName": "UNC_I_MISC0.2ND_WR_INSERT",
+        "Experimental": "1",
+        "PerPkg": "1",
+        "UMask": "0x8",
+        "Unit": "IRP"
+    },
     {
         "BriefDescription": "Counts Timeouts - Set 0 : Fastpath Rejects",
         "Counter": "0,1,2,3",
diff --git a/tools/perf/pmu-events/arch/x86/sierraforest/uncore-memory.json b/tools/perf/pmu-events/arch/x86/sierraforest/uncore-memory.json
index a9fd7a34b24b1..8b7222a3708bc 100644
--- a/tools/perf/pmu-events/arch/x86/sierraforest/uncore-memory.json
+++ b/tools/perf/pmu-events/arch/x86/sierraforest/uncore-memory.json
@@ -499,7 +499,7 @@
         "Unit": "IMC"
     },
     {
-        "BriefDescription": "DRAM Precharge commands. : Precharge due to (?) : Counts the number of DRAM Precharge commands sent on this channel.",
+        "BriefDescription": "DRAM Precharge commands. : Precharge due to page table : Counts the number of DRAM Precharge commands sent on this channel.",
         "Counter": "0,1,2,3",
         "EventCode": "0x03",
         "EventName": "UNC_M_PRE_COUNT.PGT",
diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py
index 376dc2d241621..e86989c1389d8 100755
--- a/tools/perf/pmu-events/jevents.py
+++ b/tools/perf/pmu-events/jevents.py
@@ -249,6 +249,10 @@ class JsonEvent:
           0x3F6: 'ldlat=',
           0x1A6: 'offcore_rsp=',
           0x1A7: 'offcore_rsp=',
+          0x3E0: 'offcore_rsp=',
+          0x3E1: 'offcore_rsp=',
+          0x3E2: 'offcore_rsp=',
+          0x3E3: 'offcore_rsp=',
           0x3F7: 'frontend=',
       }
       return msrmap[int(num.split(',', 1)[0], 0)]
@@ -570,13 +574,14 @@ static const struct pmu_table_entry {_pending_events_tblname}[] = {{
 def print_pending_metrics() -> None:
   """Optionally close metrics table."""
 
-  def metric_cmp_key(j: JsonEvent) -> Tuple[bool, str, str]:
+  def metric_cmp_key(j: JsonEvent) -> Tuple[str, str, str, str]:
     def fix_none(s: Optional[str]) -> str:
       if s is None:
         return ''
       return s
 
-    return (j.desc is not None, fix_none(j.pmu), fix_none(j.metric_name))
+    return (fix_none(j.pmu), fix_none(j.metric_name), j.metric_expr.ToPerfJson(),
+            fix_none(j.desc))
 
   global _pending_metrics
   if not _pending_metrics:
diff --git a/tools/perf/pmu-events/metric.py b/tools/perf/pmu-events/metric.py
index a91ccb5977f08..11c7162825f4b 100644
--- a/tools/perf/pmu-events/metric.py
+++ b/tools/perf/pmu-events/metric.py
@@ -623,7 +623,11 @@ class Metric:
 
   def __lt__(self, other):
     """Sort order."""
-    return self.name < other.name
+    if self.name != other.name:
+      return self.name < other.name
+    if not self.expr.Equals(other.expr):
+      return self.expr.ToPerfJson() < other.expr.ToPerfJson()
+    return self.description < other.description
 
   def AddToMetricGroup(self, group):
     """Callback used when being added to a MetricGroup."""
diff --git a/tools/perf/python/perf.pyi b/tools/perf/python/perf.pyi
new file mode 100644
index 0000000000000..f266c9baffd01
--- /dev/null
+++ b/tools/perf/python/perf.pyi
@@ -0,0 +1,672 @@
+"""Type stubs for the perf Python module."""
+from typing import Callable, Dict, List, Optional, Any, Iterator
+
+def config_get(name: str) -> Optional[str]:
+    """Get a configuration value from perf config.
+
+    Args:
+        name: The configuration variable name (e.g., 'colors.top').
+
+    Returns:
+        The configuration value as a string, or None if not set.
+    """
+    ...
+
+def metrics() -> List[Dict[str, str]]:
+    """Get a list of available metrics.
+
+    Returns:
+        A list of dictionaries, each describing a metric.
+    """
+    ...
+
+def syscall_name(id: int, *, elf_machine: Optional[int] = None) -> str:
+    """Convert a syscall number to its name.
+
+    Args:
+        sc_id: The syscall number.
+        elf_machine: Optional ELF machine type.
+
+    Returns:
+        The name of the syscall.
+    """
+    ...
+
+def syscall_id(name: str, *, elf_machine: Optional[int] = None) -> int:
+    """Convert a syscall name to its number.
+
+    Args:
+        name: The syscall name.
+        elf_machine: Optional ELF machine type.
+
+    Returns:
+        The number of the syscall.
+    """
+    ...
+
+def parse_events(
+    event_string: str,
+    cpus: Optional[cpu_map] = None,
+    threads: Optional['thread_map'] = None
+) -> 'evlist':
+    """Parse an event string and return an evlist.
+
+    Args:
+        event_string: The event string (e.g., 'cycles,instructions').
+        cpus: Optional CPU map to bind events to.
+        threads: Optional thread map to bind events to.
+
+    Returns:
+        An evlist containing the parsed events.
+    """
+    ...
+
+def parse_metrics(
+    metrics_string: str,
+    pmu: Optional[str] = None,
+    cpus: Optional[cpu_map] = None,
+    threads: Optional['thread_map'] = None
+) -> 'evlist':
+    """Parse a string of metrics or metric groups and return an evlist."""
+    ...
+
+def tracepoint(sys: str, name: str) -> int:
+    """Returns the tracepoint ID for a given system and name."""
+    ...
+
+def pmus() -> Iterator[Any]:
+    """Returns a sequence of pmus."""
+    ...
+
+class data:
+    """Represents a perf data file."""
+    def __init__(self, path: str = ..., fd: int = ...) -> None: ...
+
+class thread:
+    """Represents a thread in the system."""
+    def comm(self) -> str:
+        """Get the command name of the thread."""
+        ...
+    pid: int
+    tid: int
+    ppid: int
+    cpu: int
+
+class counts_values:
+    """Raw counter values."""
+    id: int
+    val: int
+    ena: int
+    run: int
+    lost: int
+    values: List[int]
+
+class thread_map:
+    """Map of threads being monitored."""
+    def __init__(self, pid: int = -1, tid: int = -1) -> None:
+        """Initialize a thread map.
+
+        Args:
+            pid: Process ID to monitor (-1 for all).
+            tid: Thread ID to monitor (-1 for all).
+        """
+        ...
+    def __len__(self) -> int: ...
+    def __getitem__(self, index: int) -> int: ...
+    def __iter__(self) -> Iterator[int]: ...
+
+class evsel:
+    """Event selector, represents a single event being monitored."""
+    def __init__(
+        self,
+        type: int = ...,
+        config: int = ...,
+        sample_freq: int = ...,
+        sample_period: int = ...,
+        sample_type: int = ...,
+        read_format: int = ...,
+        disabled: bool = ...,
+        inherit: bool = ...,
+        pinned: bool = ...,
+        exclusive: bool = ...,
+        exclude_user: bool = ...,
+        exclude_kernel: bool = ...,
+        exclude_hv: bool = ...,
+        exclude_idle: bool = ...,
+        mmap: bool = ...,
+        context_switch: bool = ...,
+        comm: bool = ...,
+        freq: bool = ...,
+        inherit_stat: bool = ...,
+        enable_on_exec: bool = ...,
+        task: bool = ...,
+        watermark: int = ...,
+        precise_ip: int = ...,
+        mmap_data: bool = ...,
+        sample_id_all: bool = ...,
+        wakeup_events: int = ...,
+        bp_type: int = ...,
+        bp_addr: int = ...,
+        bp_len: int = ...,
+        idx: int = ...,
+    ) -> None: ...
+    def __str__(self) -> str:
+        """Return string representation of the event."""
+        ...
+    def open(self) -> None:
+        """Open the event selector file descriptor table."""
+        ...
+    def read(self, cpu: int, thread: int) -> counts_values:
+        """Read counter values for a specific CPU and thread."""
+        ...
+    ids: List[int]
+    def cpus(self) -> cpu_map:
+        """Get CPU map for this event."""
+        ...
+    def threads(self) -> thread_map:
+        """Get thread map for this event."""
+        ...
+    tracking: bool
+    config: int
+    read_format: int
+    sample_period: int
+    sample_type: int
+    size: int
+    type: int
+    wakeup_events: int
+
+
+class _sample_members:
+    sample_pid: int
+    sample_tid: int
+    sample_time: int
+    sample_id: int
+    sample_stream_id: int
+    sample_period: int
+    sample_cpu: int
+
+class sample_event(_sample_members):
+    """Represents a sample event from perf."""
+    evsel: evsel
+    sample_ip: int
+    sample_addr: int
+    sample_phys_addr: int
+    sample_weight: int
+    sample_data_src: int
+    sample_insn_count: int
+    sample_cyc_count: int
+    type: int
+    raw_buf: bytes
+    dso: str
+    dso_long_name: str
+    dso_bid: Optional[bytes]
+    map_start: int
+    map_end: int
+    map_pgoff: int
+    symbol: str
+    sym_start: int
+    sym_end: int
+    brstack: Optional['branch_stack']
+    callchain: Optional['callchain']
+    def srccode(self) -> str: ...
+    def insn(self) -> str: ...
+    def __getattr__(self, name: str) -> Any: ...
+
+class mmap_event(_sample_members):
+    """Represents a mmap event from perf."""
+    type: int
+    misc: int
+    pid: int
+    tid: int
+    start: int
+    len: int
+    pgoff: int
+    filename: str
+    evsel: Optional['evsel']
+
+class mmap2_event(_sample_members):
+    """Represents a mmap2 event from perf."""
+    type: int
+    misc: int
+    pid: int
+    tid: int
+    start: int
+    len: int
+    pgoff: int
+    prot: int
+    flags: int
+    filename: str
+    maj: Optional[int]
+    min: Optional[int]
+    ino: Optional[int]
+    ino_generation: Optional[int]
+    build_id: Optional[bytes]
+    evsel: Optional['evsel']
+
+class lost_event(_sample_members):
+    """Represents a lost events record."""
+    type: int
+    id: int
+    lost: int
+    evsel: Optional['evsel']
+
+class comm_event(_sample_members):
+    """Represents a COMM record."""
+    type: int
+    pid: int
+    tid: int
+    comm: str
+    evsel: Optional['evsel']
+
+class task_event(_sample_members):
+    """Represents an EXIT or FORK record."""
+    type: int
+    pid: int
+    ppid: int
+    tid: int
+    ptid: int
+    time: int
+    evsel: Optional['evsel']
+
+class throttle_event(_sample_members):
+    """Represents a THROTTLE or UNTHROTTLE record."""
+    type: int
+    time: int
+    id: int
+    stream_id: int
+    evsel: Optional['evsel']
+
+class read_event(_sample_members):
+    """Represents a READ record."""
+    type: int
+    pid: int
+    tid: int
+    evsel: Optional['evsel']
+
+class switch_event(_sample_members):
+    """Represents a SWITCH or SWITCH_CPU_WIDE record."""
+    type: int
+    next_prev_pid: int
+    next_prev_tid: int
+    evsel: Optional['evsel']
+
+class branch_entry:
+    """Represents a branch entry in the branch stack.
+
+    Attributes:
+        from_ip: Source address of the branch (corresponds to 'from' keyword in C).
+        to_ip: Destination address of the branch.
+        mispred: True if the branch was mispredicted.
+        predicted: True if the branch was predicted.
+        in_tx: True if the branch was in a transaction.
+        abort: True if the branch was an abort.
+        cycles: Number of cycles since the last branch.
+        type: Type of branch.
+    """
+    from_ip: int
+    to_ip: int
+    mispred: bool
+    predicted: bool
+    in_tx: bool
+    abort: bool
+    cycles: int
+    type: int
+
+class branch_stack:
+    """Sequence of branch entries in the branch stack."""
+    def __len__(self) -> int: ...
+    def __getitem__(self, index: int) -> branch_entry: ...
+
+class callchain_node:
+    """Represents a frame in the callchain."""
+    ip: int
+    symbol: Optional[str]
+    dso: Optional[str]
+
+class callchain:
+    """Sequence of callchain frames."""
+    def __len__(self) -> int: ...
+    def __getitem__(self, index: int) -> callchain_node: ...
+
+class stat_event(_sample_members):
+    """Represents a stat event from perf."""
+    type: int
+    id: int
+    cpu: int
+    thread: int
+    val: int
+    ena: int
+    run: int
+    evsel: Optional['evsel']
+
+class stat_round_event(_sample_members):
+    """Represents a stat round event from perf."""
+    type: int
+    time: int
+    stat_round_type: int
+    evsel: Optional['evsel']
+
+class cpu_map:
+    """Map of CPUs being monitored."""
+    def __init__(self, cpustr: Optional[str] = None) -> None: ...
+    def __len__(self) -> int: ...
+    def __getitem__(self, index: int) -> int: ...
+    def __iter__(self) -> Iterator[int]: ...
+
+
+class evlist:
+    def __init__(self, cpus: cpu_map, threads: thread_map) -> None: ...
+    def open(self) -> None:
+        """Open the events in the list."""
+        ...
+    def close(self) -> None:
+        """Close the events in the list."""
+        ...
+    def mmap(self) -> None:
+        """Memory map the event buffers."""
+        ...
+    def poll(self, timeout: int) -> int:
+        """Poll for events.
+
+        Args:
+            timeout: Timeout in milliseconds.
+
+        Returns:
+            Number of events ready.
+        """
+        ...
+    def read_on_cpu(self, cpu: int) -> Optional[Any]:
+        """Read a sample event from a specific CPU.
+
+        Args:
+            cpu: The CPU number.
+
+        Returns:
+            A sample_event or other event type if available, or None.
+        """
+        ...
+    def all_cpus(self) -> cpu_map:
+        """Get a cpu_map of all CPUs in the system."""
+        ...
+    def metrics(self) -> List[str]:
+        """Get a list of metric names within the evlist."""
+        ...
+    def compute_metric(self, metric: str, cpu: int, thread: int) -> float:
+        """Compute metric for given name, cpu and thread.
+
+        Args:
+            metric: The metric name.
+            cpu: The CPU number.
+            thread: The thread ID.
+
+        Returns:
+            The computed metric value.
+        """
+        ...
+    def config(self) -> None:
+        """Configure the events in the list."""
+        ...
+    def disable(self) -> None:
+        """Disable all events in the list."""
+        ...
+    def enable(self) -> None:
+        """Enable all events in the list."""
+        ...
+    def get_pollfd(self) -> List[int]:
+        """Get a list of file descriptors for polling."""
+        ...
+    def add(self, evsel: evsel) -> int:
+        """Add an event to the list."""
+        ...
+    def __iter__(self) -> Iterator[evsel]:
+        """Iterate over the events (evsel) in the list."""
+        ...
+
+
+class session:
+    def __init__(
+        self,
+        data: data,
+        sample: Optional[Callable[[sample_event], None]] = None,
+        stat: Optional[Callable[[Any, Optional[str]], None]] = None
+    ) -> None:
+        """Initialize a perf session.
+
+        Args:
+            data: The perf data file to read.
+            sample: Callback for sample events.
+            stat: Callback for stat events.
+        """
+        ...
+    def process_events(self) -> None:
+        """Process all events in the session."""
+        ...
+    def find_thread(self, pid: int) -> thread:
+        """Returns the thread associated with a pid."""
+        ...
+
+# Event Types
+TYPE_HARDWARE: int
+"""Hardware event."""
+
+TYPE_SOFTWARE: int
+"""Software event."""
+
+TYPE_TRACEPOINT: int
+"""Tracepoint event."""
+
+TYPE_HW_CACHE: int
+"""Hardware cache event."""
+
+TYPE_RAW: int
+"""Raw hardware event."""
+
+TYPE_BREAKPOINT: int
+"""Breakpoint event."""
+
+
+# Hardware Counters
+COUNT_HW_CPU_CYCLES: int
+"""Total cycles. Be wary of what happens during CPU frequency scaling."""
+
+COUNT_HW_INSTRUCTIONS: int
+"""Retired instructions. Be careful, these can be affected by various issues,
+most notably hardware interrupt counts."""
+
+COUNT_HW_CACHE_REFERENCES: int
+"""Cache accesses. Usually this indicates Last Level Cache accesses but this
+may vary depending on your CPU."""
+
+COUNT_HW_CACHE_MISSES: int
+"""Cache misses. Usually this indicates Last Level Cache misses."""
+
+COUNT_HW_BRANCH_INSTRUCTIONS: int
+"""Retired branch instructions."""
+
+COUNT_HW_BRANCH_MISSES: int
+"""Mispredicted branch instructions."""
+
+COUNT_HW_BUS_CYCLES: int
+"""Bus cycles, which can be different from total cycles."""
+
+COUNT_HW_STALLED_CYCLES_FRONTEND: int
+"""Stalled cycles during issue [This event is an alias of idle-cycles-frontend]."""
+
+COUNT_HW_STALLED_CYCLES_BACKEND: int
+"""Stalled cycles during retirement [This event is an alias of idle-cycles-backend]."""
+
+COUNT_HW_REF_CPU_CYCLES: int
+"""Total cycles; not affected by CPU frequency scaling."""
+
+
+# Cache Counters
+COUNT_HW_CACHE_L1D: int
+"""Level 1 data cache."""
+
+COUNT_HW_CACHE_L1I: int
+"""Level 1 instruction cache."""
+
+COUNT_HW_CACHE_LL: int
+"""Last Level Cache."""
+
+COUNT_HW_CACHE_DTLB: int
+"""Data TLB."""
+
+COUNT_HW_CACHE_ITLB: int
+"""Instruction TLB."""
+
+COUNT_HW_CACHE_BPU: int
+"""Branch Processing Unit."""
+
+COUNT_HW_CACHE_OP_READ: int
+"""Read accesses."""
+
+COUNT_HW_CACHE_OP_WRITE: int
+"""Write accesses."""
+
+COUNT_HW_CACHE_OP_PREFETCH: int
+"""Prefetch accesses."""
+
+COUNT_HW_CACHE_RESULT_ACCESS: int
+"""Accesses."""
+
+COUNT_HW_CACHE_RESULT_MISS: int
+"""Misses."""
+
+
+# Software Counters
+COUNT_SW_CPU_CLOCK: int
+"""CPU clock event."""
+
+COUNT_SW_TASK_CLOCK: int
+"""Task clock event."""
+
+COUNT_SW_PAGE_FAULTS: int
+"""Page faults."""
+
+COUNT_SW_CONTEXT_SWITCHES: int
+"""Context switches."""
+
+COUNT_SW_CPU_MIGRATIONS: int
+"""CPU migrations."""
+
+COUNT_SW_PAGE_FAULTS_MIN: int
+"""Minor page faults."""
+
+COUNT_SW_PAGE_FAULTS_MAJ: int
+"""Major page faults."""
+
+COUNT_SW_ALIGNMENT_FAULTS: int
+"""Alignment faults."""
+
+COUNT_SW_EMULATION_FAULTS: int
+"""Emulation faults."""
+
+COUNT_SW_DUMMY: int
+"""Dummy event."""
+
+
+# Sample Fields
+SAMPLE_IP: int
+"""Instruction pointer."""
+
+SAMPLE_TID: int
+"""Process and thread ID."""
+
+SAMPLE_TIME: int
+"""Timestamp."""
+
+SAMPLE_ADDR: int
+"""Sampled address."""
+
+SAMPLE_READ: int
+"""Read barcode."""
+
+SAMPLE_CALLCHAIN: int
+"""Call chain."""
+
+SAMPLE_ID: int
+"""Unique ID."""
+
+SAMPLE_CPU: int
+"""CPU number."""
+
+SAMPLE_PERIOD: int
+"""Sample period."""
+
+SAMPLE_STREAM_ID: int
+"""Stream ID."""
+
+SAMPLE_RAW: int
+"""Raw sample."""
+
+
+# Format Fields
+FORMAT_TOTAL_TIME_ENABLED: int
+"""Total time enabled."""
+
+FORMAT_TOTAL_TIME_RUNNING: int
+"""Total time running."""
+
+FORMAT_ID: int
+"""Event ID."""
+
+FORMAT_GROUP: int
+"""Event group."""
+
+
+# Record Types
+RECORD_MMAP: int
+"""MMAP record. Contains header, pid, tid, addr, len, pgoff, filename, and sample_id."""
+
+RECORD_LOST: int
+"""Lost events record. Contains header, id, lost count, and sample_id."""
+
+RECORD_COMM: int
+"""COMM record. Contains header, pid, tid, comm, and sample_id."""
+
+RECORD_EXIT: int
+"""EXIT record. Contains header, pid, ppid, tid, ptid, time, and sample_id."""
+
+RECORD_THROTTLE: int
+"""THROTTLE record. Contains header, time, id, stream_id, and sample_id."""
+
+RECORD_UNTHROTTLE: int
+"""UNTHROTTLE record. Contains header, time, id, stream_id, and sample_id."""
+
+RECORD_FORK: int
+"""FORK record. Contains header, pid, ppid, tid, ptid, time, and sample_id."""
+
+RECORD_READ: int
+"""READ record. Contains header, and read values."""
+
+RECORD_SAMPLE: int
+"""SAMPLE record. Contains header, and sample data requested by sample_type."""
+
+RECORD_MMAP2: int
+"""MMAP2 record. Contains header, pid, tid, addr, len, pgoff, maj, min, ino,
+ino_generation, prot, flags, filename, and sample_id."""
+
+RECORD_AUX: int
+"""AUX record. Contains header, aux_offset, aux_size, flags, and sample_id."""
+
+RECORD_ITRACE_START: int
+"""ITRACE_START record. Contains header, pid, tid, and sample_id."""
+
+RECORD_LOST_SAMPLES: int
+"""LOST_SAMPLES record. Contains header, lost count, and sample_id."""
+
+RECORD_SWITCH: int
+"""SWITCH record. Contains header, and sample_id."""
+
+RECORD_SWITCH_CPU_WIDE: int
+"""SWITCH_CPU_WIDE record. Contains header, and sample_id."""
+
+RECORD_STAT: int
+"""STAT record."""
+
+RECORD_STAT_ROUND: int
+"""STAT_ROUND record."""
+
+RECORD_MISC_SWITCH_OUT: int
+"""MISC_SWITCH_OUT record."""
diff --git a/tools/perf/python/perf_live.py b/tools/perf/python/perf_live.py
new file mode 100755
index 0000000000000..616b5f6574633
--- /dev/null
+++ b/tools/perf/python/perf_live.py
@@ -0,0 +1,59 @@
+# SPDX-License-Identifier: GPL-2.0
+"""
+Live event session helper using perf.evlist.
+
+This module provides a LiveSession class that allows running a callback
+for each event collected live from the system, similar to perf.session
+but without requiring a perf.data file.
+"""
+
+import perf
+
+
+class LiveSession:
+    """Represents a live event collection session."""
+
+    def __init__(self, event_string: str, sample_callback):
+        self.event_string = event_string
+        self.sample_callback = sample_callback
+        # Create a cpu map for all online CPUs
+        self.cpus = perf.cpu_map()
+        # Parse events and set maps
+        self.evlist = perf.parse_events(self.event_string, self.cpus)
+        self.evlist.config()
+
+    def run(self):
+        """Run the live session."""
+        try:
+            self.evlist.open()
+            self.evlist.mmap()
+            self.evlist.enable()
+
+            while True:
+                # Poll for events with 100ms timeout
+                try:
+                    self.evlist.poll(100)
+                except InterruptedError:
+                    continue
+                for cpu in self.cpus:
+                    for _ in range(1000): # Limit to 1000 events per CPU per poll to prevent starvation
+                        try:
+                            event = self.evlist.read_on_cpu(cpu)
+                        except TypeError as e:
+                            if "Unknown CPU" in str(e):
+                                # CPU might be unmapped or offline, wait for mmap event
+                                break
+                            if "Unexpected header type" in str(e):
+                                # Ignore valid but unsupported event types
+                                continue
+                            raise
+
+                        if event is None:
+                            break
+
+                        if event.type == perf.RECORD_SAMPLE:
+                            self.sample_callback(event)
+        except KeyboardInterrupt:
+            pass
+        finally:
+            self.evlist.close()
diff --git a/tools/perf/scripts/python/arm-cs-trace-disasm.py b/tools/perf/scripts/python/arm-cs-trace-disasm.py
index 8f6fa4a007b42..42579f8586842 100755
--- a/tools/perf/scripts/python/arm-cs-trace-disasm.py
+++ b/tools/perf/scripts/python/arm-cs-trace-disasm.py
@@ -31,18 +31,19 @@ from perf_trace_context import perf_sample_srccode, perf_config_get
 #
 # Output disassembly with objdump and auto detect vmlinux
 # (when running on same machine.):
-#  perf script -s scripts/python/arm-cs-trace-disasm.py -d
+#  perf script --itrace=b -s scripts/python/arm-cs-trace-disasm.py \
+#       -- -d
 #
 # Output disassembly with llvm-objdump:
-#  perf script -s scripts/python/arm-cs-trace-disasm.py \
+#  perf script --itrace=b -s scripts/python/arm-cs-trace-disasm.py \
 #		-- -d llvm-objdump-11 -k path/to/vmlinux
 #
 # Output accurate disassembly by passing kcore to script:
-#  perf script -s scripts/python/arm-cs-trace-disasm.py \
+#  perf script --itrace=b -s scripts/python/arm-cs-trace-disasm.py \
 #		-- -d -k perf.data/kcore_dir/kcore
 #
 # Output only source line and symbols:
-#  perf script -s scripts/python/arm-cs-trace-disasm.py
+#  perf script --itrace=b -s scripts/python/arm-cs-trace-disasm.py
 
 def default_objdump():
 	config = perf_config_get("annotate.objdump")
diff --git a/tools/perf/scripts/python/powerpc-hcalls.py b/tools/perf/scripts/python/powerpc-hcalls.py
index 8b78dc790adb1..fedce1b68cad1 100644
--- a/tools/perf/scripts/python/powerpc-hcalls.py
+++ b/tools/perf/scripts/python/powerpc-hcalls.py
@@ -25,6 +25,7 @@ from Util import *
 #	} ...
 # }
 output = {}
+sort_key = 'count'
 
 # d_enter: {
 #	cpu: {
@@ -112,15 +113,17 @@ hcall_table = {
 	420: 'H_CREATE_RPT',
 	424: 'H_REMOVE_RPT',
 	428: 'H_REGISTER_RPAGES',
-	432: 'H_DISABLE_AND_GETC',
+	432: 'H_DISABLE_AND_GET',
 	436: 'H_ERROR_DATA',
 	440: 'H_GET_HCA_INFO',
 	444: 'H_GET_PERF_COUNT',
 	448: 'H_MANAGE_TRACE',
+	456: 'H_GET_CPU_CHARACTERISTICS',
 	468: 'H_FREE_LOGICAL_LAN_BUFFER',
 	472: 'H_POLL_PENDING',
 	484: 'H_QUERY_INT_STATE',
 	580: 'H_ILLAN_ATTRIBUTES',
+	584: 'H_ADD_LOGICAL_LAN_BUFFERS',
 	592: 'H_MODIFY_HEA_QP',
 	596: 'H_QUERY_HEA_QP',
 	600: 'H_QUERY_HEA',
@@ -135,11 +138,17 @@ hcall_table = {
 	644: 'H_ADD_CONN',
 	648: 'H_DEL_CONN',
 	664: 'H_JOIN',
+	672: 'H_VASI_SIGNAL',
 	676: 'H_VASI_STATE',
+	680: 'H_VIOCTL',
 	688: 'H_ENABLE_CRQ',
 	696: 'H_GET_EM_PARMS',
 	720: 'H_SET_MPP',
 	724: 'H_GET_MPP',
+	732: 'H_REG_SUB_CRQ',
+	736: 'H_FREE_SUB_CRQ',
+	740: 'H_SEND_SUB_CRQ',
+	744: 'H_SEND_SUB_CRQ_INDIRECT',
 	748: 'H_HOME_NODE_ASSOCIATIVITY',
 	756: 'H_BEST_ENERGY',
 	764: 'H_XIRR_X',
@@ -147,7 +156,88 @@ hcall_table = {
 	772: 'H_COP',
 	788: 'H_GET_MPP_X',
 	796: 'H_SET_MODE',
+	808: 'H_BLOCK_REMOVE',
+	856: 'H_CLEAR_HPT',
+	864: 'H_REQUEST_VMC',
+	876: 'H_RESIZE_HPT_PREPARE',
+	880: 'H_RESIZE_HPT_COMMIT',
+	892: 'H_REGISTER_PROC_TBL',
+	896: 'H_SIGNAL_SYS_RESET',
+	904: 'H_ALLOCATE_VAS_WINDOW',
+	908: 'H_MODIFY_VAS_WINDOW',
+	912: 'H_DEALLOCATE_VAS_WINDOW',
+	916: 'H_QUERY_VAS_WINDOW',
+	920: 'H_QUERY_VAS_CAPABILITIES',
+	924: 'H_QUERY_NX_CAPABILITIES',
+	928: 'H_GET_NX_FAULT',
+	936: 'H_INT_GET_SOURCE_INFO',
+	940: 'H_INT_SET_SOURCE_CONFIG',
+	944: 'H_INT_GET_SOURCE_CONFIG',
+	948: 'H_INT_GET_QUEUE_INFO',
+	952: 'H_INT_SET_QUEUE_CONFIG',
+	956: 'H_INT_GET_QUEUE_CONFIG',
+	960: 'H_INT_SET_OS_REPORTING_LINE',
+	964: 'H_INT_GET_OS_REPORTING_LINE',
+	968: 'H_INT_ESB',
+	972: 'H_INT_SYNC',
+	976: 'H_INT_RESET',
+	996: 'H_SCM_READ_METADATA',
+	1000: 'H_SCM_WRITE_METADATA',
+	1004: 'H_SCM_BIND_MEM',
+	1008: 'H_SCM_UNBIND_MEM',
+	1012: 'H_SCM_QUERY_BLOCK_MEM_BINDING',
+	1016: 'H_SCM_QUERY_LOGICAL_MEM_BINDING',
+	1020: 'H_SCM_UNBIND_ALL',
+	1024: 'H_SCM_HEALTH',
+	1048: 'H_SCM_PERFORMANCE_STATS',
+	1052: 'H_PKS_GET_CONFIG',
+	1056: 'H_PKS_SET_PASSWORD',
+	1060: 'H_PKS_GEN_PASSWORD',
+	1068: 'H_PKS_WRITE_OBJECT',
+	1072: 'H_PKS_GEN_KEY',
+	1076: 'H_PKS_READ_OBJECT',
+	1080: 'H_PKS_REMOVE_OBJECT',
+	1084: 'H_PKS_CONFIRM_OBJECT_FLUSHED',
+	1096: 'H_RPT_INVALIDATE',
+	1100: 'H_SCM_FLUSH',
+	1104: 'H_GET_ENERGY_SCALE_INFO',
+	1108: 'H_PKS_SIGNED_UPDATE',
+	1112: 'H_HTM',
+	1116: 'H_WATCHDOG',
+	# Platform specific hcalls used by KVM on PowerVM
+	1120: 'H_GUEST_GET_CAPABILITIES',
+	1124: 'H_GUEST_SET_CAPABILITIES',
+	1136: 'H_GUEST_CREATE',
+	1140: 'H_GUEST_CREATE_VCPU',
+	1144: 'H_GUEST_GET_STATE',
+	1148: 'H_GUEST_SET_STATE',
+	1152: 'H_GUEST_RUN_VCPU',
+	1156: 'H_GUEST_COPY_MEMORY',
+	1160: 'H_GUEST_DELETE',
+	# Key wrapping hcalls
+	1168: 'H_PKS_WRAP_OBJECT',
+	1172: 'H_PKS_UNWRAP_OBJECT',
+	# Platform-specific hcalls used by the Ultravisor
+	61184: 'H_SVM_PAGE_IN',
+	61188: 'H_SVM_PAGE_OUT',
+	61192: 'H_SVM_INIT_START',
+	61196: 'H_SVM_INIT_DONE',
+	61204: 'H_SVM_INIT_ABORT',
+	# Platform specific hcalls used by KVM
 	61440: 'H_RTAS',
+	# Platform specific hcalls used by QEMU/SLOF
+	61441: 'H_LOGICAL_MEMOP',
+	61442: 'H_CAS',
+	61443: 'H_UPDATE_DT',
+	# Platform specific hcalls provided by PHYP
+	61560: 'H_GET_24X7_CATALOG_PAGE',
+	61564: 'H_GET_24X7_DATA',
+	61568: 'H_GET_PERF_COUNTER_INFO',
+	# Platform-specific hcalls used for nested HV KVM
+	63488: 'H_SET_PARTITION_TABLE',
+	63492: 'H_ENTER_NESTED',
+	63496: 'H_TLB_INVALIDATE',
+	63500: 'H_COPY_TOFROM_GUEST',
 }
 
 def hcall_table_lookup(opcode):
@@ -158,10 +248,53 @@ def hcall_table_lookup(opcode):
 
 print_ptrn = '%-28s%10s%10s%10s%10s'
 
+def sort_output(opcode):
+	stats = output[opcode]
+
+	if sort_key == 'min':
+		return stats['min']
+	if sort_key == 'max':
+		return stats['max']
+	if sort_key == 'avg':
+		return stats['time'] // stats['cnt']
+
+	return stats['cnt']
+
+def trace_begin():
+	global sort_key
+
+	valid_sort_keys = ['count', 'min', 'max', 'avg']
+
+	i = 1
+	while i < len(sys.argv):
+		arg = sys.argv[i]
+
+		if arg == '-s' or arg == '--sort':
+			if i + 1 >= len(sys.argv):
+				print("Error: -s/--sort requires a sort key argument")
+				sys.exit(1)
+			sort_key = sys.argv[i + 1]
+			i += 2
+			continue
+
+		if arg.startswith('--sort='):
+			sort_key = arg.split('=', 1)[1]
+			i += 1
+			continue
+
+		i += 1
+
+	if sort_key not in valid_sort_keys:
+		print(f"Error: Invalid sort key '{sort_key}'. Valid options are: {', '.join(valid_sort_keys)}")
+		sys.exit(1)
+
+	print("SORT KEY =", sort_key)
+
 def trace_end():
 	print(print_ptrn % ('hcall', 'count', 'min(ns)', 'max(ns)', 'avg(ns)'))
 	print('-' * 68)
-	for opcode in output:
+	for opcode in sorted(output, key = sort_output,
+                            reverse=True):
 		h_name = hcall_table_lookup(opcode)
 		time = output[opcode]['time']
 		cnt = output[opcode]['cnt']
diff --git a/tools/perf/tests/backward-ring-buffer.c b/tools/perf/tests/backward-ring-buffer.c
index c5e7999f28177..2735cc26d7eec 100644
--- a/tools/perf/tests/backward-ring-buffer.c
+++ b/tools/perf/tests/backward-ring-buffer.c
@@ -34,8 +34,8 @@ static int count_samples(struct evlist *evlist, int *sample_count,
 {
 	int i;
 
-	for (i = 0; i < evlist->core.nr_mmaps; i++) {
-		struct mmap *map = &evlist->overwrite_mmap[i];
+	for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) {
+		struct mmap *map = &evlist__overwrite_mmap(evlist)[i];
 		union perf_event *event;
 
 		perf_mmap__read_init(&map->core);
@@ -65,7 +65,7 @@ static int do_test(struct evlist *evlist, int mmap_pages,
 	int err;
 	char sbuf[STRERR_BUFSIZE];
 
-	err = evlist__mmap(evlist, mmap_pages);
+	err = evlist__do_mmap(evlist, mmap_pages);
 	if (err < 0) {
 		pr_debug("evlist__mmap: %s\n",
 			 str_error_r(errno, sbuf, sizeof(sbuf)));
@@ -77,7 +77,7 @@ static int do_test(struct evlist *evlist, int mmap_pages,
 	evlist__disable(evlist);
 
 	err = count_samples(evlist, sample_count, comm_count);
-	evlist__munmap(evlist);
+	evlist__do_munmap(evlist);
 	return err;
 }
 
@@ -111,7 +111,7 @@ static int test__backward_ring_buffer(struct test_suite *test __maybe_unused, in
 	err = evlist__create_maps(evlist, &opts.target);
 	if (err < 0) {
 		pr_debug("Not enough memory to create thread/cpu maps\n");
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	parse_events_error__init(&parse_error);
@@ -124,7 +124,7 @@ static int test__backward_ring_buffer(struct test_suite *test __maybe_unused, in
 	if (err) {
 		pr_debug("Failed to parse tracepoint event, try use root\n");
 		ret = TEST_SKIP;
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	evlist__config(evlist, &opts, NULL);
@@ -133,19 +133,19 @@ static int test__backward_ring_buffer(struct test_suite *test __maybe_unused, in
 	if (err < 0) {
 		pr_debug("perf_evlist__open: %s\n",
 			 str_error_r(errno, sbuf, sizeof(sbuf)));
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	ret = TEST_FAIL;
 	err = do_test(evlist, opts.mmap_pages, &sample_count,
 		      &comm_count);
 	if (err != TEST_OK)
-		goto out_delete_evlist;
+		goto out_put_evlist;
 
 	if ((sample_count != NR_ITERS) || (comm_count != NR_ITERS)) {
 		pr_err("Unexpected counter: sample_count=%d, comm_count=%d\n",
 		       sample_count, comm_count);
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	evlist__close(evlist);
@@ -154,16 +154,16 @@ static int test__backward_ring_buffer(struct test_suite *test __maybe_unused, in
 	if (err < 0) {
 		pr_debug("perf_evlist__open: %s\n",
 			 str_error_r(errno, sbuf, sizeof(sbuf)));
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	err = do_test(evlist, 1, &sample_count, &comm_count);
 	if (err != TEST_OK)
-		goto out_delete_evlist;
+		goto out_put_evlist;
 
 	ret = TEST_OK;
-out_delete_evlist:
-	evlist__delete(evlist);
+out_put_evlist:
+	evlist__put(evlist);
 	return ret;
 }
 
diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c
index 7e75f590f225e..41dcfbf874f0a 100644
--- a/tools/perf/tests/builtin-test.c
+++ b/tools/perf/tests/builtin-test.c
@@ -10,37 +10,39 @@
 #ifdef HAVE_BACKTRACE_SUPPORT
 #include <execinfo.h>
 #endif
-#include <poll.h>
-#include <unistd.h>
 #include <setjmp.h>
-#include <string.h>
 #include <stdlib.h>
-#include <sys/types.h>
+#include <string.h>
+
 #include <dirent.h>
-#include <sys/wait.h>
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <linux/zalloc.h>
+#include <poll.h>
+#include <sys/ioctl.h>
 #include <sys/stat.h>
 #include <sys/time.h>
-#include <sys/ioctl.h>
-#include "util/term.h"
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include <subcmd/exec-cmd.h>
+#include <subcmd/parse-options.h>
+#include <subcmd/run-command.h>
+
 #include "builtin.h"
+#include "color.h"
 #include "config.h"
+#include "debug.h"
 #include "hist.h"
 #include "intlist.h"
-#include "tests.h"
-#include "debug.h"
-#include "color.h"
-#include <subcmd/parse-options.h>
-#include <subcmd/run-command.h>
 #include "string2.h"
 #include "symbol.h"
+#include "tests-scripts.h"
+#include "tests.h"
 #include "util/rlimit.h"
 #include "util/strbuf.h"
-#include <linux/kernel.h>
-#include <linux/string.h>
-#include <subcmd/exec-cmd.h>
-#include <linux/zalloc.h>
-
-#include "tests-scripts.h"
+#include "util/term.h"
 
 static const char *junit_filename;
 static struct strbuf junit_xml_buf = STRBUF_INIT;
@@ -168,6 +170,7 @@ static struct test_workload *workloads[] = {
 	&workload__jitdump,
 	&workload__context_switch_loop,
 	&workload__deterministic,
+	&workload__callchain,
 
 #ifdef HAVE_RUST_SUPPORT
 	&workload__code_with_type,
@@ -415,73 +418,73 @@ static char *xml_escape(const char *str)
 	return res ? res : strdup("");
 }
 
-static const char *format_test_description(const char *desc, int max_desc_width,
-					  char *buf, size_t buf_sz)
+static int get_term_width(void)
 {
-	int len = strlen(desc);
+	struct winsize ws;
+	int cols = 80;
+	int term_width;
 
 	/*
-	 * Clamp to buf_sz to prevent GCC format-truncation warnings
-	 * when terminal width is very large.
+	 * If output is redirected to a file or piped, we don't need to wrap
+	 * or truncate at all. Use a massive virtually infinite terminal width
+	 * so descriptions are printed in full.
 	 */
-	if (max_desc_width >= (int)buf_sz)
-		max_desc_width = buf_sz - 1;
+	if (!isatty(fileno(debug_file())))
+		return 10000;
 
-	if (len > max_desc_width) {
-		snprintf(buf, buf_sz, "%.*s...", max_desc_width - 3, desc);
-		return buf;
-	}
-	return desc;
+	get_term_dimensions(&ws);
+	if (ws.ws_col > 0)
+		cols = ws.ws_col;
+
+	/*
+	 * Limit description width to fit on a single line. We subtract 35
+	 * columns of headroom to allocate space for:
+	 * - The suite index prefix: e.g. " 10.100:" (8 characters) plus 1 space separator.
+	 * - The trailing colon (1 character) and space before status (1 character).
+	 * - The longest status results: e.g. "Skip (some metrics failed)" (26 characters)
+	 *   or "Running (XX active)" (20 characters).
+	 *
+	 * A minimum description width of 10 is enforced to ensure names are
+	 * legible even on very narrow consoles.
+	 */
+	term_width = cols - 35;
+	if (term_width < 10)
+		term_width = 10;
+
+	return term_width;
+}
+
+static int get_max_desc_width(int width)
+{
+	int term_width = get_term_width();
+
+	return width > term_width ? term_width : width;
 }
 
 static int print_test_result(struct test_suite *t, int curr_suite, int curr_test_case,
 			     int result, int width, int running,
 			     const char *err_output, double elapsed)
 {
-	char desc_buf[256];
-	const char *desc = test_description(t, curr_test_case);
-	struct winsize ws;
-	int max_desc_area_width;
-	int target_desc_area_width;
-	int desc_padding;
-
-	get_term_dimensions(&ws);
-	/*
-	 * Total terminal columns minus space for status e.g. " Running (12 active)"
-	 * which is 20 chars, plus a margin of 3 chars = 23 chars.
-	 */
-	max_desc_area_width = ws.ws_col - 23;
-	if (max_desc_area_width < 40)
-		max_desc_area_width = 40;
-
-	/* Standard test has prefix "%3d: " which is 5 chars */
-	target_desc_area_width = width + 5;
-	if (target_desc_area_width > max_desc_area_width)
-		target_desc_area_width = max_desc_area_width;
+	int pad_width = get_max_desc_width(width);
+	int term_width = get_term_width();
 
 	if (test_suite__num_test_cases(t) > 1) {
 		char prefix[32];
 		int len = snprintf(prefix, sizeof(prefix), "%3d.%1d:",
 				   curr_suite + 1, curr_test_case + 1);
+		int pad = len >= 4 ? pad_width + 4 - len : pad_width;
+		int trunc = len >= 4 ? term_width + 4 - len : term_width;
 
-		desc_padding = target_desc_area_width - (len + 1);
-		if (desc_padding < 20)
-			desc_padding = 20;
-
-		desc = format_test_description(desc, desc_padding, desc_buf, sizeof(desc_buf));
-		pr_info("%s %-*s:", prefix, desc_padding, desc);
+		pr_info("%s %-*.*s:", prefix, pad, trunc,
+			test_description(t, curr_test_case));
 	} else {
-		desc_padding = target_desc_area_width - 5;
-		if (desc_padding < 20)
-			desc_padding = 20;
-
-		desc = format_test_description(desc, desc_padding, desc_buf, sizeof(desc_buf));
-		pr_info("%3d: %-*s:", curr_suite + 1, desc_padding, desc);
+		pr_info("%3d: %-*.*s:", curr_suite + 1, pad_width, term_width,
+			test_description(t, curr_test_case));
 	}
 
 	switch (result) {
 	case TEST_RUNNING:
-		color_fprintf(stderr, PERF_COLOR_YELLOW, " Running (%d active)\n", running);
+		color_fprintf(debug_file(), PERF_COLOR_YELLOW, " Running (%d active)\n", running);
 		break;
 	case TEST_OK:
 		if (test_suite__num_test_cases(t) > 1)
@@ -495,9 +498,9 @@ static int print_test_result(struct test_suite *t, int curr_suite, int curr_test
 
 		summary_tests_skipped++;
 		if (reason)
-			color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (%s)\n", reason);
+			color_fprintf(debug_file(), PERF_COLOR_YELLOW, " Skip (%s)\n", reason);
 		else
-			color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip\n");
+			color_fprintf(debug_file(), PERF_COLOR_YELLOW, " Skip\n");
 	}
 		break;
 	case TEST_FAIL:
@@ -511,7 +514,7 @@ static int print_test_result(struct test_suite *t, int curr_suite, int curr_test
 			strbuf_addf_safe(&summary_failed_tests_buf, "  %3d: %s\n",
 				    curr_suite + 1,
 				    test_description(t, curr_test_case));
-		color_fprintf(stderr, PERF_COLOR_RED, " FAILED!\n");
+		color_fprintf(debug_file(), PERF_COLOR_RED, " FAILED!\n");
 		break;
 	}
 
@@ -533,8 +536,14 @@ static int print_test_result(struct test_suite *t, int curr_suite, int curr_test
 			const char *reason = skip_reason(t, curr_test_case);
 			char *escaped_reason = xml_escape(reason ? reason : "Skip");
 
-			strbuf_addf(&junit_xml_buf, "      <skipped message=\"%s\"/>\n",
-				    escaped_reason);
+			if (err_output && *err_output) {
+				strbuf_addf(&junit_xml_buf,
+					    "      <skipped message=\"%s\">\n%s\n      </skipped>\n",
+					    escaped_reason, escaped_err);
+			} else {
+				strbuf_addf(&junit_xml_buf, "      <skipped message=\"%s\"/>\n",
+					    escaped_reason);
+			}
 			free(escaped_reason);
 		}
 		strbuf_addstr(&junit_xml_buf, "    </testcase>\n");
@@ -747,6 +756,7 @@ static void finish_test(struct child_test **child_tests, int running_test, int c
 	int ret;
 	struct timespec end_time;
 	double elapsed;
+	width = get_max_desc_width(width);
 
 	if (child_test == NULL) {
 		/* Test wasn't started. */
@@ -761,7 +771,8 @@ static void finish_test(struct child_test **child_tests, int running_test, int c
 	 * sub test names.
 	 */
 	if (test_suite__num_test_cases(t) > 1 && curr_test_case == 0)
-		pr_info("%3d: %s:\n", curr_suite + 1, test_description(t, -1));
+		pr_info("%3d: %-*.*s:\n", curr_suite + 1, width, width,
+			test_description(t, -1));
 
 	/*
 	 * Busy loop reading from the child's stdout/stderr that are set to be
@@ -969,6 +980,8 @@ static int finish_tests_parallel(struct child_test **child_tests, size_t num_tes
 	int last_suite_printed = -1;
 	sigset_t set, oldset;
 
+	width = get_max_desc_width(width);
+
 	sigemptyset(&set);
 	sigaddset(&set, SIGINT);
 	sigaddset(&set, SIGTERM);
@@ -1037,8 +1050,11 @@ static int finish_tests_parallel(struct child_test **child_tests, size_t num_tes
 			if (next_child) {
 				if (test_suite__num_test_cases(next_child->test) > 1 &&
 				    last_suite_printed != next_child->suite_num) {
-					pr_info("%3d: %s:\n", next_child->suite_num + 1,
-						test_description(next_child->test, -1));
+					pr_info("%3d: %-*.*s:\n",
+						next_child->suite_num + 1,
+						width, width,
+						test_description(
+							next_child->test, -1));
 					last_suite_printed = next_child->suite_num;
 				}
 				print_test_result(next_child->test, next_child->suite_num,
@@ -1101,7 +1117,8 @@ static int finish_tests_parallel(struct child_test **child_tests, size_t num_tes
 
 			if (test_suite__num_test_cases(child->test) > 1 &&
 			    last_suite_printed != child->suite_num) {
-				pr_info("%3d: %s:\n", child->suite_num + 1,
+				pr_info("%3d: %-*.*s:\n", child->suite_num + 1,
+					width, width,
 					test_description(child->test, -1));
 				last_suite_printed = child->suite_num;
 			}
@@ -1225,12 +1242,12 @@ static void print_tests_summary(void)
 	pr_info("Passed subtests   : %u\n", summary_subtests_passed);
 	pr_info("Skipped tests     : %u\n", summary_tests_skipped);
 	if (summary_tests_failed > 0) {
-		color_fprintf(stderr, PERF_COLOR_RED, "Failed tests      : %u\n",
+		color_fprintf(debug_file(), PERF_COLOR_RED, "Failed tests      : %u\n",
 			      summary_tests_failed);
 		pr_info("List of failed tests:\n");
 		pr_info("%s", summary_failed_tests_buf.buf);
 	} else {
-		color_fprintf(stderr, PERF_COLOR_GREEN, "Failed tests      : 0\n");
+		color_fprintf(debug_file(), PERF_COLOR_GREEN, "Failed tests      : 0\n");
 	}
 
 	if (junit_filename) {
@@ -1348,9 +1365,13 @@ static int __cmd_test(struct test_suite **suites, int argc, const char *argv[],
 
 			if (intlist__find(skiplist, curr_suite + 1)) {
 				if (pass == 1) {
-					pr_info("%3d: %-*s:", curr_suite + 1, width,
+					int pad_width = get_max_desc_width(width);
+					int term_width = get_term_width();
+
+					pr_info("%3d: %-*.*s:", curr_suite + 1,
+						pad_width, term_width,
 						test_description(*t, -1));
-					color_fprintf(stderr, PERF_COLOR_YELLOW,
+					color_fprintf(debug_file(), PERF_COLOR_YELLOW,
 						      " Skip (user override)\n");
 					summary_tests_skipped++;
 					if (junit_filename) {
diff --git a/tools/perf/tests/code-reading.c b/tools/perf/tests/code-reading.c
index e82ecdc957778..f0e8ea8754ef5 100644
--- a/tools/perf/tests/code-reading.c
+++ b/tools/perf/tests/code-reading.c
@@ -592,8 +592,8 @@ static int process_events(struct machine *machine, struct evlist *evlist,
 	struct mmap *md;
 	int i, ret;
 
-	for (i = 0; i < evlist->core.nr_mmaps; i++) {
-		md = &evlist->mmap[i];
+	for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) {
+		md = &evlist__mmap(evlist)[i];
 		if (perf_mmap__read_init(&md->core) < 0)
 			continue;
 
@@ -781,7 +781,7 @@ static int do_test_code_reading(bool try_kcore)
 			goto out_put;
 		}
 
-		perf_evlist__set_maps(&evlist->core, cpus, threads);
+		perf_evlist__set_maps(evlist__core(evlist), cpus, threads);
 
 		str = events[evidx];
 		pr_debug("Parsing event '%s'\n", str);
@@ -809,8 +809,8 @@ static int do_test_code_reading(bool try_kcore)
 				pr_debug("perf_evlist__open() failed!\n%s\n", errbuf);
 			}
 
-			perf_evlist__set_maps(&evlist->core, NULL, NULL);
-			evlist__delete(evlist);
+			perf_evlist__set_maps(evlist__core(evlist), NULL, NULL);
+			evlist__put(evlist);
 			evlist = NULL;
 			continue;
 		}
@@ -820,7 +820,7 @@ static int do_test_code_reading(bool try_kcore)
 	if (events[evidx] == NULL)
 		goto out_put;
 
-	ret = evlist__mmap(evlist, UINT_MAX);
+	ret = evlist__do_mmap(evlist, UINT_MAX);
 	if (ret < 0) {
 		pr_debug("evlist__mmap failed\n");
 		goto out_put;
@@ -847,7 +847,7 @@ static int do_test_code_reading(bool try_kcore)
 out_put:
 	thread__put(thread);
 out_err:
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	perf_cpu_map__put(cpus);
 	perf_thread_map__put(threads);
 	machine__delete(machine);
diff --git a/tools/perf/tests/event-times.c b/tools/perf/tests/event-times.c
index ae3b98bb42cf8..56dd37ca760ec 100644
--- a/tools/perf/tests/event-times.c
+++ b/tools/perf/tests/event-times.c
@@ -50,7 +50,7 @@ static int attach__enable_on_exec(struct evlist *evlist)
 
 static int detach__enable_on_exec(struct evlist *evlist)
 {
-	waitpid(evlist->workload.pid, NULL, 0);
+	waitpid(evlist__workload_pid(evlist), NULL, 0);
 	return 0;
 }
 
@@ -186,7 +186,7 @@ static int test_times(int (attach)(struct evlist *),
 	err = attach(evlist);
 	if (err == TEST_SKIP) {
 		pr_debug("  SKIP  : not enough rights\n");
-		evlist__delete(evlist);
+		evlist__put(evlist);
 		return err;
 	}
 
@@ -205,7 +205,7 @@ static int test_times(int (attach)(struct evlist *),
 		 count.ena, count.run);
 
 out_err:
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	return !err ? TEST_OK : TEST_FAIL;
 }
 
diff --git a/tools/perf/tests/event_update.c b/tools/perf/tests/event_update.c
index facc65e29f20c..220cc0347747d 100644
--- a/tools/perf/tests/event_update.c
+++ b/tools/perf/tests/event_update.c
@@ -92,7 +92,7 @@ static int test__event_update(struct test_suite *test __maybe_unused, int subtes
 	TEST_ASSERT_VAL("failed to allocate ids",
 			!perf_evsel__alloc_id(&evsel->core, 1, 1));
 
-	perf_evlist__id_add(&evlist->core, &evsel->core, 0, 0, 123);
+	perf_evlist__id_add(evlist__core(evlist), &evsel->core, 0, 0, 123);
 
 	free((char *)evsel->unit);
 	evsel->unit = strdup("KRAVA");
@@ -117,7 +117,7 @@ static int test__event_update(struct test_suite *test __maybe_unused, int subtes
 	TEST_ASSERT_VAL("failed to synthesize attr update cpus",
 			!perf_event__synthesize_event_update_cpus(&tmp.tool, evsel, process_event_cpus));
 
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	return 0;
 }
 
diff --git a/tools/perf/tests/evsel-roundtrip-name.c b/tools/perf/tests/evsel-roundtrip-name.c
index 1922cac13a245..6a220634c52f6 100644
--- a/tools/perf/tests/evsel-roundtrip-name.c
+++ b/tools/perf/tests/evsel-roundtrip-name.c
@@ -33,7 +33,7 @@ static int perf_evsel__roundtrip_cache_name_test(void)
 				if (err) {
 					pr_debug("Failure to parse cache event '%s' possibly as PMUs don't support it",
 						name);
-					evlist__delete(evlist);
+					evlist__put(evlist);
 					continue;
 				}
 				evlist__for_each_entry(evlist, evsel) {
@@ -42,7 +42,7 @@ static int perf_evsel__roundtrip_cache_name_test(void)
 						ret = TEST_FAIL;
 					}
 				}
-				evlist__delete(evlist);
+				evlist__put(evlist);
 			}
 		}
 	}
@@ -66,7 +66,7 @@ static int perf_evsel__name_array_test(const char *const names[], int nr_names)
 		if (err) {
 			pr_debug("failed to parse event '%s', err %d\n",
 				 names[i], err);
-			evlist__delete(evlist);
+			evlist__put(evlist);
 			ret = TEST_FAIL;
 			continue;
 		}
@@ -76,7 +76,7 @@ static int perf_evsel__name_array_test(const char *const names[], int nr_names)
 				ret = TEST_FAIL;
 			}
 		}
-		evlist__delete(evlist);
+		evlist__put(evlist);
 	}
 	return ret;
 }
diff --git a/tools/perf/tests/evsel-tp-sched.c b/tools/perf/tests/evsel-tp-sched.c
index 226196fb96779..9e456f88a13ac 100644
--- a/tools/perf/tests/evsel-tp-sched.c
+++ b/tools/perf/tests/evsel-tp-sched.c
@@ -64,7 +64,7 @@ static int test__perf_evsel__tp_sched_test(struct test_suite *test __maybe_unuse
 	if (evsel__test_field(evsel, "next_prio", 4, true))
 		ret = TEST_FAIL;
 
-	evsel__delete(evsel);
+	evsel__put(evsel);
 
 	evsel = evsel__newtp("sched", "sched_wakeup");
 
@@ -85,7 +85,7 @@ static int test__perf_evsel__tp_sched_test(struct test_suite *test __maybe_unuse
 	if (evsel__test_field(evsel, "target_cpu", 4, true))
 		ret = TEST_FAIL;
 
-	evsel__delete(evsel);
+	evsel__put(evsel);
 	return ret;
 }
 
diff --git a/tools/perf/tests/expand-cgroup.c b/tools/perf/tests/expand-cgroup.c
index dd547f2f77cc8..04d62611766ad 100644
--- a/tools/perf/tests/expand-cgroup.c
+++ b/tools/perf/tests/expand-cgroup.c
@@ -28,7 +28,7 @@ static int test_expand_events(struct evlist *evlist)
 
 	TEST_ASSERT_VAL("evlist is empty", !evlist__empty(evlist));
 
-	nr_events = evlist->core.nr_entries;
+	nr_events = evlist__nr_entries(evlist);
 	ev_name = calloc(nr_events, sizeof(*ev_name));
 	if (ev_name == NULL) {
 		pr_debug("memory allocation failure\n");
@@ -54,7 +54,7 @@ static int test_expand_events(struct evlist *evlist)
 	}
 
 	ret = TEST_FAIL;
-	if (evlist->core.nr_entries != nr_events * nr_cgrps) {
+	if (evlist__nr_entries(evlist) != nr_events * nr_cgrps) {
 		pr_debug("event count doesn't match\n");
 		goto out;
 	}
@@ -106,7 +106,7 @@ static int expand_default_events(void)
 	TEST_ASSERT_VAL("failed to get evlist", evlist);
 
 	ret = test_expand_events(evlist);
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	return ret;
 }
 
@@ -133,7 +133,7 @@ static int expand_group_events(void)
 	ret = test_expand_events(evlist);
 out:
 	parse_events_error__exit(&err);
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	return ret;
 }
 
@@ -164,7 +164,7 @@ static int expand_libpfm_events(void)
 
 	ret = test_expand_events(evlist);
 out:
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	return ret;
 }
 
@@ -179,7 +179,8 @@ static int expand_metric_events(void)
 	TEST_ASSERT_VAL("failed to get evlist", evlist);
 
 	pme_test = find_core_metrics_table("testarch", "testcpu");
-	ret = metricgroup__parse_groups_test(evlist, pme_test, metric_str);
+	ret = metricgroup__parse_groups_test(evlist, pme_test, metric_str,
+					     /*cputype_filter=*/false);
 	if (ret < 0) {
 		pr_debug("failed to parse '%s' metric\n", metric_str);
 		goto out;
@@ -188,7 +189,7 @@ static int expand_metric_events(void)
 	ret = test_expand_events(evlist);
 
 out:
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	return ret;
 }
 
diff --git a/tools/perf/tests/hists_cumulate.c b/tools/perf/tests/hists_cumulate.c
index 09ee08085b06b..9356451a172ea 100644
--- a/tools/perf/tests/hists_cumulate.c
+++ b/tools/perf/tests/hists_cumulate.c
@@ -744,7 +744,7 @@ static int test__hists_cumulate(struct test_suite *test __maybe_unused, int subt
 
 out:
 	/* tear down everything */
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	machines__exit(&machines);
 	put_fake_samples();
 
diff --git a/tools/perf/tests/hists_filter.c b/tools/perf/tests/hists_filter.c
index ac5affb7afff1..f9eaa511487b4 100644
--- a/tools/perf/tests/hists_filter.c
+++ b/tools/perf/tests/hists_filter.c
@@ -332,7 +332,7 @@ static int test__hists_filter(struct test_suite *test __maybe_unused, int subtes
 
 out:
 	/* tear down everything */
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	reset_output_field();
 	machines__exit(&machines);
 	put_fake_samples();
diff --git a/tools/perf/tests/hists_link.c b/tools/perf/tests/hists_link.c
index e55990163865e..d88591bcbe508 100644
--- a/tools/perf/tests/hists_link.c
+++ b/tools/perf/tests/hists_link.c
@@ -353,7 +353,7 @@ static int test__hists_link(struct test_suite *test __maybe_unused, int subtest
 
 out:
 	/* tear down everything */
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	reset_output_field();
 	machines__exit(&machines);
 	put_fake_samples();
diff --git a/tools/perf/tests/hists_output.c b/tools/perf/tests/hists_output.c
index 5e59dba92e813..f58c8d18fe330 100644
--- a/tools/perf/tests/hists_output.c
+++ b/tools/perf/tests/hists_output.c
@@ -631,7 +631,7 @@ static int test__hists_output(struct test_suite *test __maybe_unused, int subtes
 
 out:
 	/* tear down everything */
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	machines__exit(&machines);
 	put_fake_samples();
 
diff --git a/tools/perf/tests/hwmon_pmu.c b/tools/perf/tests/hwmon_pmu.c
index 62e0841a6c310..e26b3fe3fab15 100644
--- a/tools/perf/tests/hwmon_pmu.c
+++ b/tools/perf/tests/hwmon_pmu.c
@@ -184,9 +184,10 @@ static int do_test(size_t i, bool with_pmu, bool with_alias)
 	}
 
 	ret = TEST_OK;
-	if (with_pmu ? (evlist->core.nr_entries != 1) : (evlist->core.nr_entries < 1)) {
+	if (with_pmu ? (evlist__nr_entries(evlist) != 1)
+		     : (evlist__nr_entries(evlist) < 1)) {
 		pr_debug("FAILED %s:%d Unexpected number of events for '%s' of %d\n",
-			 __FILE__, __LINE__, str, evlist->core.nr_entries);
+			 __FILE__, __LINE__, str, evlist__nr_entries(evlist));
 		ret = TEST_FAIL;
 		goto out;
 	}
@@ -215,7 +216,7 @@ static int do_test(size_t i, bool with_pmu, bool with_alias)
 
 out:
 	parse_events_error__exit(&err);
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	return ret;
 }
 
diff --git a/tools/perf/tests/keep-tracking.c b/tools/perf/tests/keep-tracking.c
index 729cc9cc1cb77..b760041bed307 100644
--- a/tools/perf/tests/keep-tracking.c
+++ b/tools/perf/tests/keep-tracking.c
@@ -37,8 +37,8 @@ static int find_comm(struct evlist *evlist, const char *comm)
 	int i, found;
 
 	found = 0;
-	for (i = 0; i < evlist->core.nr_mmaps; i++) {
-		md = &evlist->mmap[i];
+	for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) {
+		md = &evlist__mmap(evlist)[i];
 		if (perf_mmap__read_init(&md->core) < 0)
 			continue;
 		while ((event = perf_mmap__read_event(&md->core)) != NULL) {
@@ -87,7 +87,7 @@ static int test__keep_tracking(struct test_suite *test __maybe_unused, int subte
 	evlist = evlist__new();
 	CHECK_NOT_NULL__(evlist);
 
-	perf_evlist__set_maps(&evlist->core, cpus, threads);
+	perf_evlist__set_maps(evlist__core(evlist), cpus, threads);
 
 	CHECK__(parse_event(evlist, "dummy:u"));
 	CHECK__(parse_event(evlist, "cpu-cycles:u"));
@@ -106,7 +106,7 @@ static int test__keep_tracking(struct test_suite *test __maybe_unused, int subte
 		goto out_err;
 	}
 
-	CHECK__(evlist__mmap(evlist, UINT_MAX));
+	CHECK__(evlist__do_mmap(evlist, UINT_MAX));
 
 	/*
 	 * First, test that a 'comm' event can be found when the event is
@@ -153,7 +153,7 @@ static int test__keep_tracking(struct test_suite *test __maybe_unused, int subte
 out_err:
 	if (evlist) {
 		evlist__disable(evlist);
-		evlist__delete(evlist);
+		evlist__put(evlist);
 	}
 	perf_cpu_map__put(cpus);
 	perf_thread_map__put(threads);
diff --git a/tools/perf/tests/mmap-basic.c b/tools/perf/tests/mmap-basic.c
index a69cd1046e9aa..5cec7644952c7 100644
--- a/tools/perf/tests/mmap-basic.c
+++ b/tools/perf/tests/mmap-basic.c
@@ -81,7 +81,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest
 		goto out_free_cpus;
 	}
 
-	perf_evlist__set_maps(&evlist->core, cpus, threads);
+	perf_evlist__set_maps(evlist__core(evlist), cpus, threads);
 
 	for (i = 0; i < nsyscalls; ++i) {
 		char name[64];
@@ -94,7 +94,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest
 				/* Permissions failure, flag the failure as a skip. */
 				err = TEST_SKIP;
 			}
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 
 		evsels[i]->core.attr.wakeup_events = 1;
@@ -106,17 +106,17 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest
 			pr_debug("failed to open counter: %s, "
 				 "tweak /proc/sys/kernel/perf_event_paranoid?\n",
 				 str_error_r(errno, sbuf, sizeof(sbuf)));
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 
 		nr_events[i] = 0;
 		expected_nr_events[i] = 1 + rand() % 127;
 	}
 
-	if (evlist__mmap(evlist, 128) < 0) {
+	if (evlist__do_mmap(evlist, 128) < 0) {
 		pr_debug("failed to mmap events: %d (%s)\n", errno,
 			 str_error_r(errno, sbuf, sizeof(sbuf)));
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	for (i = 0; i < nsyscalls; ++i)
@@ -124,7 +124,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest
 			syscalls[i]();
 		}
 
-	md = &evlist->mmap[0];
+	md = &evlist__mmap(evlist)[0];
 	if (perf_mmap__read_init(&md->core) < 0)
 		goto out_init;
 
@@ -134,7 +134,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest
 		if (event->header.type != PERF_RECORD_SAMPLE) {
 			pr_debug("unexpected %s event\n",
 				 perf_event__name(event->header.type));
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 
 		perf_sample__init(&sample, /*all=*/false);
@@ -142,7 +142,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest
 		if (err) {
 			pr_err("Can't parse sample, err = %d\n", err);
 			perf_sample__exit(&sample);
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 
 		err = -1;
@@ -153,7 +153,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest
 		if (evsel == NULL) {
 			pr_debug("event with id %" PRIu64
 				 " doesn't map to an evsel\n", sample.id);
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 		nr_events[evsel->core.idx]++;
 		perf_mmap__consume(&md->core);
@@ -168,12 +168,12 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest
 				 expected_nr_events[evsel->core.idx],
 				 evsel__name(evsel), nr_events[evsel->core.idx]);
 			err = -1;
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 	}
 
-out_delete_evlist:
-	evlist__delete(evlist);
+out_put_evlist:
+	evlist__put(evlist);
 out_free_cpus:
 	perf_cpu_map__put(cpus);
 out_free_threads:
diff --git a/tools/perf/tests/openat-syscall-all-cpus.c b/tools/perf/tests/openat-syscall-all-cpus.c
index 0be43f8db3bda..cc63df2b3bc53 100644
--- a/tools/perf/tests/openat-syscall-all-cpus.c
+++ b/tools/perf/tests/openat-syscall-all-cpus.c
@@ -59,7 +59,7 @@ static int test__openat_syscall_event_on_all_cpus(struct test_suite *test __mayb
 			 "tweak /proc/sys/kernel/perf_event_paranoid?\n",
 			 str_error_r(errno, sbuf, sizeof(sbuf)));
 		err = TEST_SKIP;
-		goto out_evsel_delete;
+		goto out_evsel_put;
 	}
 
 	perf_cpu_map__for_each_cpu(cpu, idx, cpus) {
@@ -116,8 +116,8 @@ static int test__openat_syscall_event_on_all_cpus(struct test_suite *test __mayb
 	evsel__free_counts(evsel);
 out_close_fd:
 	perf_evsel__close_fd(&evsel->core);
-out_evsel_delete:
-	evsel__delete(evsel);
+out_evsel_put:
+	evsel__put(evsel);
 out_cpu_map_delete:
 	perf_cpu_map__put(cpus);
 out_thread_map_delete:
diff --git a/tools/perf/tests/openat-syscall-tp-fields.c b/tools/perf/tests/openat-syscall-tp-fields.c
index 9ff8caff98c3a..5365889d326f8 100644
--- a/tools/perf/tests/openat-syscall-tp-fields.c
+++ b/tools/perf/tests/openat-syscall-tp-fields.c
@@ -51,7 +51,7 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused
 	if (IS_ERR(evsel)) {
 		pr_debug("%s: evsel__newtp\n", __func__);
 		ret = PTR_ERR(evsel) == -EACCES ? TEST_SKIP : TEST_FAIL;
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	evlist__add(evlist, evsel);
@@ -59,25 +59,25 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused
 	err = evlist__create_maps(evlist, &opts.target);
 	if (err < 0) {
 		pr_debug("%s: evlist__create_maps\n", __func__);
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	evsel__config(evsel, &opts, NULL);
 
-	perf_thread_map__set_pid(evlist->core.threads, 0, getpid());
+	perf_thread_map__set_pid(evlist__core(evlist)->threads, 0, getpid());
 
 	err = evlist__open(evlist);
 	if (err < 0) {
 		pr_debug("perf_evlist__open: %s\n",
 			 str_error_r(errno, sbuf, sizeof(sbuf)));
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
-	err = evlist__mmap(evlist, UINT_MAX);
+	err = evlist__do_mmap(evlist, UINT_MAX);
 	if (err < 0) {
 		pr_debug("evlist__mmap: %s\n",
 			 str_error_r(errno, sbuf, sizeof(sbuf)));
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	evlist__enable(evlist);
@@ -90,11 +90,11 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused
 	while (1) {
 		int before = nr_events;
 
-		for (i = 0; i < evlist->core.nr_mmaps; i++) {
+		for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) {
 			union perf_event *event;
 			struct mmap *md;
 
-			md = &evlist->mmap[i];
+			md = &evlist__mmap(evlist)[i];
 			if (perf_mmap__read_init(&md->core) < 0)
 				continue;
 
@@ -115,7 +115,7 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused
 				if (err) {
 					pr_debug("Can't parse sample, err = %d\n", err);
 					perf_sample__exit(&sample);
-					goto out_delete_evlist;
+					goto out_put_evlist;
 				}
 
 				tp_flags = perf_sample__intval(&sample, "flags");
@@ -126,7 +126,7 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused
 				    (tp_flags & flags) != flags) {
 					pr_debug("%s: Expected flags=%#x, got %#x\n",
 						 __func__, flags, tp_flags);
-					goto out_delete_evlist;
+					goto out_put_evlist;
 				}
 
 				goto out_ok;
@@ -139,13 +139,13 @@ static int test__syscall_openat_tp_fields(struct test_suite *test __maybe_unused
 
 		if (++nr_polls > 5) {
 			pr_debug("%s: no events!\n", __func__);
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 	}
 out_ok:
 	ret = TEST_OK;
-out_delete_evlist:
-	evlist__delete(evlist);
+out_put_evlist:
+	evlist__put(evlist);
 out:
 	return ret;
 }
diff --git a/tools/perf/tests/openat-syscall.c b/tools/perf/tests/openat-syscall.c
index b54cbe5f18085..9f16f0dd3a295 100644
--- a/tools/perf/tests/openat-syscall.c
+++ b/tools/perf/tests/openat-syscall.c
@@ -42,7 +42,7 @@ static int test__openat_syscall_event(struct test_suite *test __maybe_unused,
 			 "tweak /proc/sys/kernel/perf_event_paranoid?\n",
 			 str_error_r(errno, sbuf, sizeof(sbuf)));
 		err = TEST_SKIP;
-		goto out_evsel_delete;
+		goto out_evsel_put;
 	}
 
 	for (i = 0; i < nr_openat_calls; ++i) {
@@ -64,8 +64,8 @@ static int test__openat_syscall_event(struct test_suite *test __maybe_unused,
 	err = TEST_OK;
 out_close_fd:
 	perf_evsel__close_fd(&evsel->core);
-out_evsel_delete:
-	evsel__delete(evsel);
+out_evsel_put:
+	evsel__put(evsel);
 out_thread_map_delete:
 	perf_thread_map__put(threads);
 	return err;
diff --git a/tools/perf/tests/parse-events.c b/tools/perf/tests/parse-events.c
index 05c3e899b4251..5f6f95c8a5b40 100644
--- a/tools/perf/tests/parse-events.c
+++ b/tools/perf/tests/parse-events.c
@@ -109,7 +109,7 @@ static int test__checkevent_tracepoint(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVLIST("wrong number of groups", 0 == evlist__nr_groups(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_TRACEPOINT == evsel->core.attr.type, evsel);
 	TEST_ASSERT_EVSEL("wrong sample_type",
@@ -122,7 +122,7 @@ static int test__checkevent_tracepoint_multi(struct evlist *evlist)
 {
 	struct evsel *evsel;
 
-	TEST_ASSERT_EVLIST("wrong number of entries", evlist->core.nr_entries > 1, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", evlist__nr_entries(evlist) > 1, evlist);
 	TEST_ASSERT_EVLIST("wrong number of groups", 0 == evlist__nr_groups(evlist), evlist);
 
 	evlist__for_each_entry(evlist, evsel) {
@@ -144,7 +144,7 @@ static int test__checkevent_raw(struct evlist *evlist)
 	struct evsel *evsel;
 	bool raw_type_match = false;
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist__nr_entries(evlist), evlist);
 
 	evlist__for_each_entry(evlist, evsel) {
 		struct perf_pmu *pmu __maybe_unused = NULL;
@@ -182,7 +182,7 @@ static int test__checkevent_numeric(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong type", 1 == evsel->core.attr.type, evsel);
 	TEST_ASSERT_EVSEL("wrong config", 1 == evsel->core.attr.config, evsel);
 	return TEST_OK;
@@ -193,7 +193,7 @@ static int test__checkevent_symbolic_name(struct evlist *evlist)
 {
 	struct evsel *evsel;
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist__nr_entries(evlist), evlist);
 
 	evlist__for_each_entry(evlist, evsel) {
 		TEST_ASSERT_EVSEL("unexpected event",
@@ -207,7 +207,7 @@ static int test__checkevent_symbolic_name_config(struct evlist *evlist)
 {
 	struct evsel *evsel;
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist__nr_entries(evlist), evlist);
 
 	evlist__for_each_entry(evlist, evsel) {
 		TEST_ASSERT_EVSEL("unexpected event",
@@ -228,7 +228,7 @@ static int test__checkevent_symbolic_alias(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong type/config", evsel__match(evsel, SOFTWARE, SW_PAGE_FAULTS),
 			  evsel);
 	return TEST_OK;
@@ -238,7 +238,7 @@ static int test__checkevent_genhw(struct evlist *evlist)
 {
 	struct evsel *evsel;
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 0 != evlist__nr_entries(evlist), evlist);
 
 	evlist__for_each_entry(evlist, evsel) {
 		TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_HW_CACHE == evsel->core.attr.type, evsel);
@@ -251,7 +251,7 @@ static int test__checkevent_breakpoint(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel);
 	TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel);
 	TEST_ASSERT_EVSEL("wrong bp_type",
@@ -265,7 +265,7 @@ static int test__checkevent_breakpoint_x(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel);
 	TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel);
 	TEST_ASSERT_EVSEL("wrong bp_type", HW_BREAKPOINT_X == evsel->core.attr.bp_type, evsel);
@@ -278,7 +278,7 @@ static int test__checkevent_breakpoint_r(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel);
 	TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel);
 	TEST_ASSERT_EVSEL("wrong bp_type", HW_BREAKPOINT_R == evsel->core.attr.bp_type, evsel);
@@ -290,7 +290,7 @@ static int test__checkevent_breakpoint_w(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel);
 	TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel);
 	TEST_ASSERT_EVSEL("wrong bp_type", HW_BREAKPOINT_W == evsel->core.attr.bp_type, evsel);
@@ -302,7 +302,7 @@ static int test__checkevent_breakpoint_rw(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel);
 	TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel);
 	TEST_ASSERT_EVSEL("wrong bp_type",
@@ -316,7 +316,7 @@ static int test__checkevent_tracepoint_modifier(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong exclude_user", evsel->core.attr.exclude_user, evsel);
 	TEST_ASSERT_EVSEL("wrong exclude_kernel", !evsel->core.attr.exclude_kernel, evsel);
 	TEST_ASSERT_EVSEL("wrong exclude_hv", evsel->core.attr.exclude_hv, evsel);
@@ -330,7 +330,7 @@ test__checkevent_tracepoint_multi_modifier(struct evlist *evlist)
 {
 	struct evsel *evsel;
 
-	TEST_ASSERT_EVLIST("wrong number of entries", evlist->core.nr_entries > 1, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", evlist__nr_entries(evlist) > 1, evlist);
 
 	evlist__for_each_entry(evlist, evsel) {
 		TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel);
@@ -346,7 +346,7 @@ static int test__checkevent_raw_modifier(struct evlist *evlist)
 {
 	struct evsel *evsel;
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 
 	evlist__for_each_entry(evlist, evsel) {
 		TEST_ASSERT_EVSEL("wrong exclude_user", evsel->core.attr.exclude_user, evsel);
@@ -361,7 +361,7 @@ static int test__checkevent_numeric_modifier(struct evlist *evlist)
 {
 	struct evsel *evsel;
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 
 	evlist__for_each_entry(evlist, evsel) {
 		TEST_ASSERT_EVSEL("wrong exclude_user", evsel->core.attr.exclude_user, evsel);
@@ -377,7 +377,7 @@ static int test__checkevent_symbolic_name_modifier(struct evlist *evlist)
 	struct evsel *evsel;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == num_core_entries(evlist),
+			   evlist__nr_entries(evlist) == num_core_entries(evlist),
 			   evlist);
 
 	evlist__for_each_entry(evlist, evsel) {
@@ -394,7 +394,7 @@ static int test__checkevent_exclude_host_modifier(struct evlist *evlist)
 	struct evsel *evsel;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == num_core_entries(evlist),
+			   evlist__nr_entries(evlist) == num_core_entries(evlist),
 			   evlist);
 
 	evlist__for_each_entry(evlist, evsel) {
@@ -409,7 +409,7 @@ static int test__checkevent_exclude_guest_modifier(struct evlist *evlist)
 	struct evsel *evsel;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == num_core_entries(evlist),
+			   evlist__nr_entries(evlist) == num_core_entries(evlist),
 			   evlist);
 
 	evlist__for_each_entry(evlist, evsel) {
@@ -423,7 +423,8 @@ static int test__checkevent_symbolic_alias_modifier(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries",
+			   1 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel);
 	TEST_ASSERT_EVSEL("wrong exclude_kernel", evsel->core.attr.exclude_kernel, evsel);
 	TEST_ASSERT_EVSEL("wrong exclude_hv", evsel->core.attr.exclude_hv, evsel);
@@ -437,7 +438,7 @@ static int test__checkevent_genhw_modifier(struct evlist *evlist)
 	struct evsel *evsel;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == num_core_entries(evlist),
+			   evlist__nr_entries(evlist) == num_core_entries(evlist),
 			   evlist);
 
 	evlist__for_each_entry(evlist, evsel) {
@@ -454,7 +455,7 @@ static int test__checkevent_exclude_idle_modifier(struct evlist *evlist)
 	struct evsel *evsel = evlist__first(evlist);
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == num_core_entries(evlist),
+			   evlist__nr_entries(evlist) == num_core_entries(evlist),
 			   evlist);
 
 	TEST_ASSERT_EVSEL("wrong exclude idle", evsel->core.attr.exclude_idle, evsel);
@@ -473,7 +474,7 @@ static int test__checkevent_exclude_idle_modifier_1(struct evlist *evlist)
 	struct evsel *evsel = evlist__first(evlist);
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == num_core_entries(evlist),
+			   evlist__nr_entries(evlist) == num_core_entries(evlist),
 			   evlist);
 
 	TEST_ASSERT_EVSEL("wrong exclude idle", evsel->core.attr.exclude_idle, evsel);
@@ -622,7 +623,7 @@ static int test__checkevent_breakpoint_2_events(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVSEL("wrong number of entries", 2 == evlist->core.nr_entries, evsel);
+	TEST_ASSERT_EVSEL("wrong number of entries", 2 == evlist__nr_entries(evlist), evsel);
 
 	TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel);
 	TEST_ASSERT_EVSEL("wrong name", evsel__name_is(evsel, "breakpoint1"), evsel);
@@ -641,7 +642,7 @@ static int test__checkevent_pmu(struct evlist *evlist)
 	struct evsel *evsel = evlist__first(evlist);
 	struct perf_pmu *core_pmu = perf_pmus__find_core_pmu();
 
-	TEST_ASSERT_EVSEL("wrong number of entries", 1 == evlist->core.nr_entries, evsel);
+	TEST_ASSERT_EVSEL("wrong number of entries", 1 == evlist__nr_entries(evlist), evsel);
 	TEST_ASSERT_EVSEL("wrong type", core_pmu->type == evsel->core.attr.type, evsel);
 	TEST_ASSERT_EVSEL("wrong config",    test_hw_config(evsel, 10), evsel);
 	TEST_ASSERT_EVSEL("wrong config1",    1 == evsel->core.attr.config1, evsel);
@@ -661,7 +662,7 @@ static int test__checkevent_list(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVSEL("wrong number of entries", 3 <= evlist->core.nr_entries, evsel);
+	TEST_ASSERT_EVSEL("wrong number of entries", 3 <= evlist__nr_entries(evlist), evsel);
 
 	/* r1 */
 	TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_TRACEPOINT != evsel->core.attr.type, evsel);
@@ -707,14 +708,15 @@ static int test__checkevent_pmu_name(struct evlist *evlist)
 	char buf[256];
 
 	/* default_core/config=1,name=krava/u */
-	TEST_ASSERT_EVLIST("wrong number of entries", 2 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries",
+			   2 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong type", core_pmu->type == evsel->core.attr.type, evsel);
 	TEST_ASSERT_EVSEL("wrong config", 1 == evsel->core.attr.config, evsel);
 	TEST_ASSERT_EVSEL("wrong name", evsel__name_is(evsel, "krava"), evsel);
 
 	/* default_core/config=2/u" */
 	evsel = evsel__next(evsel);
-	TEST_ASSERT_EVSEL("wrong number of entries", 2 == evlist->core.nr_entries, evsel);
+	TEST_ASSERT_EVSEL("wrong number of entries", 2 == evlist__nr_entries(evlist), evsel);
 	TEST_ASSERT_EVSEL("wrong type", core_pmu->type == evsel->core.attr.type, evsel);
 	TEST_ASSERT_EVSEL("wrong config", 2 == evsel->core.attr.config, evsel);
 	snprintf(buf, sizeof(buf), "%s/config=2/u", core_pmu->name);
@@ -729,7 +731,8 @@ static int test__checkevent_pmu_partial_time_callgraph(struct evlist *evlist)
 	struct perf_pmu *core_pmu = perf_pmus__find_core_pmu();
 
 	/* default_core/config=1,call-graph=fp,time,period=100000/ */
-	TEST_ASSERT_EVLIST("wrong number of entries", 2 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries",
+			   2 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong type", core_pmu->type == evsel->core.attr.type, evsel);
 	TEST_ASSERT_EVSEL("wrong config", 1 == evsel->core.attr.config, evsel);
 	/*
@@ -760,7 +763,7 @@ static int test__checkevent_pmu_events(struct evlist *evlist)
 	struct evsel *evsel;
 	struct perf_pmu *core_pmu = perf_pmus__find_core_pmu();
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 <= evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 <= evlist__nr_entries(evlist), evlist);
 
 	evlist__for_each_entry(evlist, evsel) {
 		TEST_ASSERT_EVSEL("wrong type",
@@ -787,8 +790,9 @@ static int test__checkevent_pmu_events_mix(struct evlist *evlist)
 	 * The wild card event will be opened at least once, but it may be
 	 * opened on each core PMU.
 	 */
-	TEST_ASSERT_EVLIST("wrong number of entries", evlist->core.nr_entries >= 2, evlist);
-	for (int i = 0; i < evlist->core.nr_entries - 1; i++) {
+	TEST_ASSERT_EVLIST("wrong number of entries",
+			   evlist__nr_entries(evlist) >= 2, evlist);
+	for (int i = 0; i < evlist__nr_entries(evlist) - 1; i++) {
 		evsel = (i == 0 ? evlist__first(evlist) : evsel__next(evsel));
 		/* pmu-event:u */
 		TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel);
@@ -905,7 +909,7 @@ static int test__group1(struct evlist *evlist)
 	struct evsel *evsel = NULL, *leader;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == (num_core_entries(evlist) * 2),
+			   evlist__nr_entries(evlist) == (num_core_entries(evlist) * 2),
 			   evlist);
 	TEST_ASSERT_EVLIST("wrong number of groups",
 			   evlist__nr_groups(evlist) == num_core_entries(evlist),
@@ -950,7 +954,7 @@ static int test__group2(struct evlist *evlist)
 	struct evsel *evsel, *leader = NULL;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == (2 * num_core_entries(evlist) + 1),
+			   evlist__nr_entries(evlist) == (2 * num_core_entries(evlist) + 1),
 			   evlist);
 	/*
 	 * TODO: Currently the software event won't be grouped with the hardware
@@ -1018,7 +1022,7 @@ static int test__group3(struct evlist *evlist __maybe_unused)
 	struct evsel *evsel, *group1_leader = NULL, *group2_leader = NULL;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == (3 * perf_pmus__num_core_pmus() + 2),
+			   evlist__nr_entries(evlist) == (3 * perf_pmus__num_core_pmus() + 2),
 			   evlist);
 	/*
 	 * Currently the software event won't be grouped with the hardware event
@@ -1144,7 +1148,7 @@ static int test__group4(struct evlist *evlist __maybe_unused)
 	struct evsel *evsel = NULL, *leader;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == (num_core_entries(evlist) * 2),
+			   evlist__nr_entries(evlist) == (num_core_entries(evlist) * 2),
 			   evlist);
 	TEST_ASSERT_EVLIST("wrong number of groups",
 			   num_core_entries(evlist) == evlist__nr_groups(evlist),
@@ -1191,7 +1195,7 @@ static int test__group5(struct evlist *evlist __maybe_unused)
 	struct evsel *evsel = NULL, *leader;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == (5 * num_core_entries(evlist)),
+			   evlist__nr_entries(evlist) == (5 * num_core_entries(evlist)),
 			   evlist);
 	TEST_ASSERT_EVLIST("wrong number of groups",
 			   evlist__nr_groups(evlist) == (2 * num_core_entries(evlist)),
@@ -1284,7 +1288,7 @@ static int test__group_gh1(struct evlist *evlist)
 	struct evsel *evsel = NULL, *leader;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == (2 * num_core_entries(evlist)),
+			   evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)),
 			   evlist);
 	TEST_ASSERT_EVLIST("wrong number of groups",
 			   evlist__nr_groups(evlist) == num_core_entries(evlist),
@@ -1329,7 +1333,7 @@ static int test__group_gh2(struct evlist *evlist)
 	struct evsel *evsel = NULL, *leader;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == (2 * num_core_entries(evlist)),
+			   evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)),
 			   evlist);
 	TEST_ASSERT_EVLIST("wrong number of groups",
 			   evlist__nr_groups(evlist) == num_core_entries(evlist),
@@ -1374,7 +1378,7 @@ static int test__group_gh3(struct evlist *evlist)
 	struct evsel *evsel = NULL, *leader;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == (2 * num_core_entries(evlist)),
+			   evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)),
 			   evlist);
 	TEST_ASSERT_EVLIST("wrong number of groups",
 			   evlist__nr_groups(evlist) == num_core_entries(evlist),
@@ -1419,7 +1423,7 @@ static int test__group_gh4(struct evlist *evlist)
 	struct evsel *evsel = NULL, *leader;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == (2 * num_core_entries(evlist)),
+			   evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)),
 			   evlist);
 	TEST_ASSERT_EVLIST("wrong number of groups",
 			   evlist__nr_groups(evlist) == num_core_entries(evlist),
@@ -1464,7 +1468,7 @@ static int test__leader_sample1(struct evlist *evlist)
 	struct evsel *evsel = NULL, *leader;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == (3 * num_core_entries(evlist)),
+			   evlist__nr_entries(evlist) == (3 * num_core_entries(evlist)),
 			   evlist);
 
 	for (int i = 0; i < num_core_entries(evlist); i++) {
@@ -1520,7 +1524,7 @@ static int test__leader_sample2(struct evlist *evlist __maybe_unused)
 	struct evsel *evsel = NULL, *leader;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == (2 * num_core_entries(evlist)),
+			   evlist__nr_entries(evlist) == (2 * num_core_entries(evlist)),
 			   evlist);
 
 	for (int i = 0; i < num_core_entries(evlist); i++) {
@@ -1562,7 +1566,7 @@ static int test__checkevent_pinned_modifier(struct evlist *evlist)
 	struct evsel *evsel = NULL;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == num_core_entries(evlist),
+			   evlist__nr_entries(evlist) == num_core_entries(evlist),
 			   evlist);
 
 	for (int i = 0; i < num_core_entries(evlist); i++) {
@@ -1581,7 +1585,7 @@ static int test__pinned_group(struct evlist *evlist)
 	struct evsel *evsel = NULL, *leader;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == (3 * num_core_entries(evlist)),
+			   evlist__nr_entries(evlist) == (3 * num_core_entries(evlist)),
 			   evlist);
 
 	for (int i = 0; i < num_core_entries(evlist); i++) {
@@ -1618,7 +1622,7 @@ static int test__checkevent_exclusive_modifier(struct evlist *evlist)
 	struct evsel *evsel = evlist__first(evlist);
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == num_core_entries(evlist),
+			   evlist__nr_entries(evlist) == num_core_entries(evlist),
 			   evlist);
 	TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel);
 	TEST_ASSERT_EVSEL("wrong exclude_kernel", evsel->core.attr.exclude_kernel, evsel);
@@ -1634,7 +1638,7 @@ static int test__exclusive_group(struct evlist *evlist)
 	struct evsel *evsel = NULL, *leader;
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == 3 * num_core_entries(evlist),
+			   evlist__nr_entries(evlist) == 3 * num_core_entries(evlist),
 			   evlist);
 
 	for (int i = 0; i < num_core_entries(evlist); i++) {
@@ -1669,7 +1673,7 @@ static int test__checkevent_breakpoint_len(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel);
 	TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel);
 	TEST_ASSERT_EVSEL("wrong bp_type",
@@ -1684,7 +1688,7 @@ static int test__checkevent_breakpoint_len_w(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_BREAKPOINT == evsel->core.attr.type, evsel);
 	TEST_ASSERT_EVSEL("wrong config", 0 == evsel->core.attr.config, evsel);
 	TEST_ASSERT_EVSEL("wrong bp_type", HW_BREAKPOINT_W == evsel->core.attr.bp_type, evsel);
@@ -1698,7 +1702,7 @@ test__checkevent_breakpoint_len_rw_modifier(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong exclude_user", !evsel->core.attr.exclude_user, evsel);
 	TEST_ASSERT_EVSEL("wrong exclude_kernel", evsel->core.attr.exclude_kernel, evsel);
 	TEST_ASSERT_EVSEL("wrong exclude_hv", evsel->core.attr.exclude_hv, evsel);
@@ -1712,7 +1716,7 @@ static int test__checkevent_precise_max_modifier(struct evlist *evlist)
 	struct evsel *evsel = evlist__first(evlist);
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == 1 + num_core_entries(evlist),
+			   evlist__nr_entries(evlist) == 1 + num_core_entries(evlist),
 			   evlist);
 	TEST_ASSERT_EVSEL("wrong type/config", evsel__match(evsel, SOFTWARE, SW_TASK_CLOCK), evsel);
 	return TEST_OK;
@@ -1723,7 +1727,7 @@ static int test__checkevent_config_symbol(struct evlist *evlist)
 	struct evsel *evsel = evlist__first(evlist);
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == num_core_entries(evlist),
+			   evlist__nr_entries(evlist) == num_core_entries(evlist),
 			   evlist);
 	TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "insn"), evsel);
 	return TEST_OK;
@@ -1733,7 +1737,7 @@ static int test__checkevent_config_raw(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "rawpmu"), evsel);
 	return TEST_OK;
 }
@@ -1742,7 +1746,7 @@ static int test__checkevent_config_num(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "numpmu"), evsel);
 	return TEST_OK;
 }
@@ -1752,7 +1756,7 @@ static int test__checkevent_config_cache(struct evlist *evlist)
 	struct evsel *evsel = evlist__first(evlist);
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == num_core_entries(evlist),
+			   evlist__nr_entries(evlist) == num_core_entries(evlist),
 			   evlist);
 	TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "cachepmu"), evsel);
 	return test__checkevent_genhw(evlist);
@@ -1777,7 +1781,7 @@ static int test__intel_pt(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong name setting", evsel__name_is(evsel, "intel_pt//u"), evsel);
 	return TEST_OK;
 }
@@ -1798,7 +1802,8 @@ static int test__ratio_to_prev(struct evlist *evlist)
 {
 	struct evsel *evsel, *leader;
 
-	TEST_ASSERT_VAL("wrong number of entries", 2 * perf_pmus__num_core_pmus() == evlist->core.nr_entries);
+	TEST_ASSERT_VAL("wrong number of entries",
+			2 * perf_pmus__num_core_pmus() == evlist__nr_entries(evlist));
 
 	evlist__for_each_entry(evlist, evsel) {
 		if (evsel != evsel__leader(evsel) ||
@@ -1842,7 +1847,7 @@ static int test__checkevent_complex_name(struct evlist *evlist)
 	struct evsel *evsel = evlist__first(evlist);
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == num_core_entries(evlist),
+			   evlist__nr_entries(evlist) == num_core_entries(evlist),
 			   evlist);
 	TEST_ASSERT_EVSEL("wrong complex name parsing",
 			  evsel__name_is(evsel,
@@ -1855,7 +1860,7 @@ static int test__checkevent_raw_pmu(struct evlist *evlist)
 {
 	struct evsel *evsel = evlist__first(evlist);
 
-	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist->core.nr_entries, evlist);
+	TEST_ASSERT_EVLIST("wrong number of entries", 1 == evlist__nr_entries(evlist), evlist);
 	TEST_ASSERT_EVSEL("wrong type", PERF_TYPE_SOFTWARE == evsel->core.attr.type, evsel);
 	TEST_ASSERT_EVSEL("wrong config", 0x1a == evsel->core.attr.config, evsel);
 	return TEST_OK;
@@ -1866,7 +1871,7 @@ static int test__sym_event_slash(struct evlist *evlist)
 	struct evsel *evsel = evlist__first(evlist);
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == num_core_entries(evlist),
+			   evlist__nr_entries(evlist) == num_core_entries(evlist),
 			   evlist);
 	TEST_ASSERT_EVSEL("unexpected event", evsel__match(evsel, HARDWARE, HW_CPU_CYCLES), evsel);
 	TEST_ASSERT_EVSEL("wrong exclude_kernel", evsel->core.attr.exclude_kernel, evsel);
@@ -1878,7 +1883,7 @@ static int test__sym_event_dc(struct evlist *evlist)
 	struct evsel *evsel = evlist__first(evlist);
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == num_core_entries(evlist),
+			   evlist__nr_entries(evlist) == num_core_entries(evlist),
 			   evlist);
 	TEST_ASSERT_EVSEL("unexpected event", evsel__match(evsel, HARDWARE, HW_CPU_CYCLES), evsel);
 	TEST_ASSERT_EVSEL("wrong exclude_user", evsel->core.attr.exclude_user, evsel);
@@ -1890,7 +1895,7 @@ static int test__term_equal_term(struct evlist *evlist)
 	struct evsel *evsel = evlist__first(evlist);
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == num_core_entries(evlist),
+			   evlist__nr_entries(evlist) == num_core_entries(evlist),
 			   evlist);
 	TEST_ASSERT_EVSEL("unexpected event", evsel__match(evsel, HARDWARE, HW_CPU_CYCLES), evsel);
 	TEST_ASSERT_EVSEL("wrong name setting", strcmp(evsel->name, "name") == 0, evsel);
@@ -1902,7 +1907,7 @@ static int test__term_equal_legacy(struct evlist *evlist)
 	struct evsel *evsel = evlist__first(evlist);
 
 	TEST_ASSERT_EVLIST("wrong number of entries",
-			   evlist->core.nr_entries == num_core_entries(evlist),
+			   evlist__nr_entries(evlist) == num_core_entries(evlist),
 			   evlist);
 	TEST_ASSERT_EVSEL("unexpected event", evsel__match(evsel, HARDWARE, HW_CPU_CYCLES), evsel);
 	TEST_ASSERT_EVSEL("wrong name setting", strcmp(evsel->name, "l1d") == 0, evsel);
@@ -1958,7 +1963,7 @@ static int count_tracepoints(void)
 static int test__all_tracepoints(struct evlist *evlist)
 {
 	TEST_ASSERT_VAL("wrong events count",
-			count_tracepoints() == evlist->core.nr_entries);
+			count_tracepoints() == evlist__nr_entries(evlist));
 
 	return test__checkevent_tracepoint_multi(evlist);
 }
@@ -2556,8 +2561,10 @@ static int test_event(const struct evlist_test *e)
 		return TEST_FAIL;
 	}
 	parse_events_error__init(&err);
-	ret = __parse_events(evlist, e->name, /*pmu_filter=*/NULL, &err, /*fake_pmu=*/false,
-			     /*warn_if_reordered=*/true, /*fake_tp=*/true);
+	ret = __parse_events(evlist, e->name, /*pmu_filter=*/NULL,
+			     /*cputype_filter=*/false, &err, /*fake_pmu=*/false,
+			     /*warn_if_reordered=*/true,
+			     /*fake_tp=*/true);
 	if (ret) {
 		pr_debug("failed to parse event '%s', err %d\n", e->name, ret);
 		parse_events_error__print(&err, e->name);
@@ -2568,7 +2575,7 @@ static int test_event(const struct evlist_test *e)
 		ret = e->check(evlist);
 	}
 	parse_events_error__exit(&err);
-	evlist__delete(evlist);
+	evlist__put(evlist);
 
 	return ret;
 }
@@ -2584,8 +2591,9 @@ static int test_event_fake_pmu(const char *str)
 		return -ENOMEM;
 
 	parse_events_error__init(&err);
-	ret = __parse_events(evlist, str, /*pmu_filter=*/NULL, &err,
-			     /*fake_pmu=*/true, /*warn_if_reordered=*/true,
+	ret = __parse_events(evlist, str, /*pmu_filter=*/NULL,
+			     /*cputype_filter=*/false, &err, /*fake_pmu=*/true,
+			     /*warn_if_reordered=*/true,
 			     /*fake_tp=*/true);
 	if (ret) {
 		pr_debug("failed to parse event '%s', err %d\n",
@@ -2594,7 +2602,7 @@ static int test_event_fake_pmu(const char *str)
 	}
 
 	parse_events_error__exit(&err);
-	evlist__delete(evlist);
+	evlist__put(evlist);
 
 	return ret;
 }
diff --git a/tools/perf/tests/parse-metric.c b/tools/perf/tests/parse-metric.c
index 7c7f489a5eb0a..872d9a7aa72f6 100644
--- a/tools/perf/tests/parse-metric.c
+++ b/tools/perf/tests/parse-metric.c
@@ -53,7 +53,7 @@ static double compute_single(struct evlist *evlist, const char *name)
 	struct evsel *evsel;
 
 	evlist__for_each_entry(evlist, evsel) {
-		me = metricgroup__lookup(&evlist->metric_events, evsel, false);
+		me = metricgroup__lookup(evlist__metric_events(evlist), evsel, false);
 		if (me != NULL) {
 			list_for_each_entry (mexp, &me->head, nd) {
 				if (strcmp(mexp->metric_name, name))
@@ -84,15 +84,16 @@ static int __compute_metric(const char *name, struct value *vals,
 
 	cpus = perf_cpu_map__new("0");
 	if (!cpus) {
-		evlist__delete(evlist);
+		evlist__put(evlist);
 		return -ENOMEM;
 	}
 
-	perf_evlist__set_maps(&evlist->core, cpus, NULL);
+	perf_evlist__set_maps(evlist__core(evlist), cpus, NULL);
 
 	/* Parse the metric into metric_events list. */
 	pme_test = find_core_metrics_table("testarch", "testcpu");
-	err = metricgroup__parse_groups_test(evlist, pme_test, name);
+	err = metricgroup__parse_groups_test(evlist, pme_test, name,
+					     /*cputype_filter=*/false);
 	if (err)
 		goto out;
 
@@ -113,7 +114,7 @@ static int __compute_metric(const char *name, struct value *vals,
 	/* ... cleanup. */
 	evlist__free_stats(evlist);
 	perf_cpu_map__put(cpus);
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	return err;
 }
 
diff --git a/tools/perf/tests/parse-no-sample-id-all.c b/tools/perf/tests/parse-no-sample-id-all.c
index 8ac862c94879f..78bb8db192404 100644
--- a/tools/perf/tests/parse-no-sample-id-all.c
+++ b/tools/perf/tests/parse-no-sample-id-all.c
@@ -49,7 +49,7 @@ static int process_events(union perf_event **events, size_t count)
 	for (i = 0; i < count && !err; i++)
 		err = process_event(&evlist, events[i]);
 
-	evlist__delete(evlist);
+	evlist__put(evlist);
 
 	return err;
 }
diff --git a/tools/perf/tests/perf-record.c b/tools/perf/tests/perf-record.c
index ad44cc68820b3..0cac6ae1a1fc0 100644
--- a/tools/perf/tests/perf-record.c
+++ b/tools/perf/tests/perf-record.c
@@ -105,7 +105,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest
 	err = evlist__create_maps(evlist, &opts.target);
 	if (err < 0) {
 		pr_debug("Not enough memory to create thread/cpu maps\n");
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	/*
@@ -117,7 +117,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest
 	err = evlist__prepare_workload(evlist, &opts.target, argv, false, NULL);
 	if (err < 0) {
 		pr_debug("Couldn't run the workload!\n");
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	/*
@@ -129,12 +129,12 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest
 	evsel__set_sample_bit(evsel, TIME);
 	evlist__config(evlist, &opts, NULL);
 
-	err = sched__get_first_possible_cpu(evlist->workload.pid, cpu_mask);
+	err = sched__get_first_possible_cpu(evlist__workload_pid(evlist), cpu_mask);
 	if (err < 0) {
 		pr_debug("sched__get_first_possible_cpu: %s\n",
 			 str_error_r(errno, sbuf, sizeof(sbuf)));
 		evlist__cancel_workload(evlist);
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	cpu = err;
@@ -142,11 +142,11 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest
 	/*
 	 * So that we can check perf_sample.cpu on all the samples.
 	 */
-	if (sched_setaffinity(evlist->workload.pid, cpu_mask_size, cpu_mask) < 0) {
+	if (sched_setaffinity(evlist__workload_pid(evlist), cpu_mask_size, cpu_mask) < 0) {
 		pr_debug("sched_setaffinity: %s\n",
 			 str_error_r(errno, sbuf, sizeof(sbuf)));
 		evlist__cancel_workload(evlist);
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	/*
@@ -158,7 +158,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest
 		pr_debug("perf_evlist__open: %s\n",
 			 str_error_r(errno, sbuf, sizeof(sbuf)));
 		evlist__cancel_workload(evlist);
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	/*
@@ -166,12 +166,12 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest
 	 * fds in the same CPU to be injected in the same mmap ring buffer
 	 * (using ioctl(PERF_EVENT_IOC_SET_OUTPUT)).
 	 */
-	err = evlist__mmap(evlist, opts.mmap_pages);
+	err = evlist__do_mmap(evlist, opts.mmap_pages);
 	if (err < 0) {
 		pr_debug("evlist__mmap: %s\n",
 			 str_error_r(errno, sbuf, sizeof(sbuf)));
 		evlist__cancel_workload(evlist);
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	/*
@@ -188,11 +188,11 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest
 	while (1) {
 		int before = total_events;
 
-		for (i = 0; i < evlist->core.nr_mmaps; i++) {
+		for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) {
 			union perf_event *event;
 			struct mmap *md;
 
-			md = &evlist->mmap[i];
+			md = &evlist__mmap(evlist)[i];
 			if (perf_mmap__read_init(&md->core) < 0)
 				continue;
 
@@ -209,7 +209,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest
 					if (verbose > 0)
 						perf_event__fprintf(event, NULL, stderr);
 					pr_debug("Couldn't parse sample\n");
-					goto out_delete_evlist;
+					goto out_put_evlist;
 				}
 
 				if (verbose > 0) {
@@ -231,15 +231,15 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest
 					++errs;
 				}
 
-				if ((pid_t)sample.pid != evlist->workload.pid) {
+				if ((pid_t)sample.pid != evlist__workload_pid(evlist)) {
 					pr_debug("%s with unexpected pid, expected %d, got %d\n",
-						 name, evlist->workload.pid, sample.pid);
+						 name, evlist__workload_pid(evlist), sample.pid);
 					++errs;
 				}
 
-				if ((pid_t)sample.tid != evlist->workload.pid) {
+				if ((pid_t)sample.tid != evlist__workload_pid(evlist)) {
 					pr_debug("%s with unexpected tid, expected %d, got %d\n",
-						 name, evlist->workload.pid, sample.tid);
+						 name, evlist__workload_pid(evlist), sample.tid);
 					++errs;
 				}
 
@@ -248,7 +248,7 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest
 				     type == PERF_RECORD_MMAP2 ||
 				     type == PERF_RECORD_FORK ||
 				     type == PERF_RECORD_EXIT) &&
-				     (pid_t)event->comm.pid != evlist->workload.pid) {
+				     (pid_t)event->comm.pid != evlist__workload_pid(evlist)) {
 					pr_debug("%s with unexpected pid/tid\n", name);
 					++errs;
 				}
@@ -350,11 +350,11 @@ static int test__PERF_RECORD(struct test_suite *test __maybe_unused, int subtest
 		pr_debug("PERF_RECORD_MMAP for %s missing!\n", "[vdso]");
 		++errs;
 	}
-out_delete_evlist:
+out_put_evlist:
 	CPU_FREE(cpu_mask);
-	evlist__delete(evlist);
 out:
 	perf_sample__exit(&sample);
+	evlist__put(evlist);
 	if (err == -EACCES)
 		return TEST_SKIP;
 	if (err < 0 || errs != 0)
diff --git a/tools/perf/tests/perf-time-to-tsc.c b/tools/perf/tests/perf-time-to-tsc.c
index cca41bd37ae3c..f8f71fdd32b1b 100644
--- a/tools/perf/tests/perf-time-to-tsc.c
+++ b/tools/perf/tests/perf-time-to-tsc.c
@@ -99,7 +99,7 @@ static int test__perf_time_to_tsc(struct test_suite *test __maybe_unused, int su
 	evlist = evlist__new();
 	CHECK_NOT_NULL__(evlist);
 
-	perf_evlist__set_maps(&evlist->core, cpus, threads);
+	perf_evlist__set_maps(evlist__core(evlist), cpus, threads);
 
 	CHECK__(parse_event(evlist, "cpu-cycles:u"));
 
@@ -121,9 +121,9 @@ static int test__perf_time_to_tsc(struct test_suite *test __maybe_unused, int su
 		goto out_err;
 	}
 
-	CHECK__(evlist__mmap(evlist, UINT_MAX));
+	CHECK__(evlist__do_mmap(evlist, UINT_MAX));
 
-	pc = evlist->mmap[0].core.base;
+	pc = evlist__mmap(evlist)[0].core.base;
 	ret = perf_read_tsc_conversion(pc, &tc);
 	if (ret) {
 		if (ret == -EOPNOTSUPP) {
@@ -145,8 +145,8 @@ static int test__perf_time_to_tsc(struct test_suite *test __maybe_unused, int su
 
 	evlist__disable(evlist);
 
-	for (i = 0; i < evlist->core.nr_mmaps; i++) {
-		md = &evlist->mmap[i];
+	for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) {
+		md = &evlist__mmap(evlist)[i];
 		if (perf_mmap__read_init(&md->core) < 0)
 			continue;
 
@@ -201,7 +201,7 @@ static int test__perf_time_to_tsc(struct test_suite *test __maybe_unused, int su
 	err = TEST_OK;
 
 out_err:
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	perf_cpu_map__put(cpus);
 	perf_thread_map__put(threads);
 	return err;
diff --git a/tools/perf/tests/pfm.c b/tools/perf/tests/pfm.c
index fca4a86452df6..f7bf55be5e6ee 100644
--- a/tools/perf/tests/pfm.c
+++ b/tools/perf/tests/pfm.c
@@ -69,18 +69,18 @@ static int test__pfm_events(struct test_suite *test __maybe_unused,
 		if (evlist == NULL)
 			return -ENOMEM;
 
-		opt.value = evlist;
+		opt.value = &evlist;
 		parse_libpfm_events_option(&opt,
 					table[i].events,
 					0);
 		TEST_ASSERT_EQUAL(table[i].events,
-				count_pfm_events(&evlist->core),
+				count_pfm_events(evlist__core(evlist)),
 				table[i].nr_events);
 		TEST_ASSERT_EQUAL(table[i].events,
 				evlist__nr_groups(evlist),
 				0);
 
-		evlist__delete(evlist);
+		evlist__put(evlist);
 	}
 	return 0;
 }
@@ -154,18 +154,18 @@ static int test__pfm_group(struct test_suite *test __maybe_unused,
 		if (evlist == NULL)
 			return -ENOMEM;
 
-		opt.value = evlist;
+		opt.value = &evlist;
 		parse_libpfm_events_option(&opt,
 					table[i].events,
 					0);
 		TEST_ASSERT_EQUAL(table[i].events,
-				count_pfm_events(&evlist->core),
+				count_pfm_events(evlist__core(evlist)),
 				table[i].nr_events);
 		TEST_ASSERT_EQUAL(table[i].events,
 				evlist__nr_groups(evlist),
 				table[i].nr_groups);
 
-		evlist__delete(evlist);
+		evlist__put(evlist);
 	}
 	return 0;
 }
diff --git a/tools/perf/tests/pmu-events.c b/tools/perf/tests/pmu-events.c
index fd5630f0a13c0..f507ce39439ad 100644
--- a/tools/perf/tests/pmu-events.c
+++ b/tools/perf/tests/pmu-events.c
@@ -794,11 +794,13 @@ static int check_parse_id(const char *id, struct parse_events_error *error)
 	for (cur = strchr(dup, '@') ; cur; cur = strchr(++cur, '@'))
 		*cur = '/';
 
-	ret = __parse_events(evlist, dup, /*pmu_filter=*/NULL, error, /*fake_pmu=*/true,
-			     /*warn_if_reordered=*/true, /*fake_tp=*/false);
+	ret = __parse_events(evlist, dup, /*pmu_filter=*/NULL,
+			     /*cputype_filter=*/false, error, /*fake_pmu=*/true,
+			     /*warn_if_reordered=*/true,
+			     /*fake_tp=*/false);
 	free(dup);
 
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	return ret;
 }
 
@@ -865,13 +867,15 @@ static int test__parsing_callback(const struct pmu_metric *pm,
 
 	cpus = perf_cpu_map__new("0");
 	if (!cpus) {
-		evlist__delete(evlist);
+		evlist__put(evlist);
 		return -ENOMEM;
 	}
 
-	perf_evlist__set_maps(&evlist->core, cpus, NULL);
+	perf_evlist__set_maps(evlist__core(evlist), cpus, NULL);
 
-	err = metricgroup__parse_groups_test(evlist, table, pm->metric_name);
+	err = metricgroup__parse_groups_test(evlist, table,
+					     pm->metric_name,
+					     /*cputype_filter=*/false);
 	if (err) {
 		if (is_expected_broken_metric(pm)) {
 			(*failures)--;
@@ -895,7 +899,8 @@ static int test__parsing_callback(const struct pmu_metric *pm,
 		k++;
 	}
 	evlist__for_each_entry(evlist, evsel) {
-		struct metric_event *me = metricgroup__lookup(&evlist->metric_events, evsel, false);
+		struct metric_event *me = metricgroup__lookup(evlist__metric_events(evlist),
+							      evsel, false);
 
 		if (me != NULL) {
 			struct metric_expr *mexp;
@@ -919,7 +924,7 @@ static int test__parsing_callback(const struct pmu_metric *pm,
 	/* ... cleanup. */
 	evlist__free_stats(evlist);
 	perf_cpu_map__put(cpus);
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	return err;
 }
 
diff --git a/tools/perf/tests/pmu.c b/tools/perf/tests/pmu.c
index d7be9d1c6f52b..13e8d7fa80af2 100644
--- a/tools/perf/tests/pmu.c
+++ b/tools/perf/tests/pmu.c
@@ -294,7 +294,7 @@ static int test__pmu_config_helpers(struct test_suite *test __maybe_unused,
 	ret = TEST_OK;
 err_out:
 	parse_events_terms__exit(&terms);
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	test_pmu_put(dir, pmu);
 	return ret;
 }
@@ -346,7 +346,7 @@ static int test__pmu_events(struct test_suite *test __maybe_unused, int subtest
 	ret = TEST_OK;
 err_out:
 	parse_events_error__exit(&err);
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	test_pmu_put(dir, pmu);
 	return ret;
 }
diff --git a/tools/perf/tests/sample-parsing.c b/tools/perf/tests/sample-parsing.c
index 55f0b73ca20e0..20cab91ceaeb2 100644
--- a/tools/perf/tests/sample-parsing.c
+++ b/tools/perf/tests/sample-parsing.c
@@ -205,15 +205,11 @@ static bool samples_same(struct perf_sample *s1,
 
 static int do_test(u64 sample_type, u64 sample_regs, u64 read_format)
 {
-	struct evsel evsel = {
-		.needs_swap = false,
-		.core = {
-			. attr = {
-				.sample_type = sample_type,
-				.read_format = read_format,
-			},
-		},
+	struct perf_event_attr attr = {
+		.sample_type = sample_type,
+		.read_format = read_format,
 	};
+	struct evsel *evsel;
 	union perf_event *event;
 	union {
 		struct ip_callchain callchain;
@@ -287,16 +283,21 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format)
 	size_t i, sz, bufsz;
 	int err, ret = -1;
 
+	evsel = evsel__new(&attr);
+	if (!evsel) {
+		pr_debug("evsel__new failed\n");
+		return -1;
+	}
 	perf_sample__init(&sample_out, /*all=*/false);
 	perf_sample__init(&sample_out_endian, /*all=*/false);
 	if (sample_type & PERF_SAMPLE_REGS_USER)
-		evsel.core.attr.sample_regs_user = sample_regs;
+		evsel->core.attr.sample_regs_user = sample_regs;
 
 	if (sample_type & PERF_SAMPLE_REGS_INTR)
-		evsel.core.attr.sample_regs_intr = sample_regs;
+		evsel->core.attr.sample_regs_intr = sample_regs;
 
 	if (sample_type & PERF_SAMPLE_BRANCH_STACK)
-		evsel.core.attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX;
+		evsel->core.attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX;
 
 	for (i = 0; i < sizeof(regs); i++)
 		*(i + (u8 *)regs) = i & 0xfe;
@@ -311,12 +312,12 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format)
 	}
 
 	sz = perf_event__sample_event_size(&sample, sample_type, read_format,
-					   evsel.core.attr.branch_sample_type);
+					   evsel->core.attr.branch_sample_type);
 	bufsz = sz + 4096; /* Add a bit for overrun checking */
 	event = malloc(bufsz);
 	if (!event) {
 		pr_debug("malloc failed\n");
-		return -1;
+		goto out_free;
 	}
 
 	memset(event, 0xff, bufsz);
@@ -325,7 +326,7 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format)
 	event->header.size = sz;
 
 	err = perf_event__synthesize_sample(event, sample_type, read_format,
-					    evsel.core.attr.branch_sample_type, &sample);
+					    evsel->core.attr.branch_sample_type, &sample);
 	if (err) {
 		pr_debug("%s failed for sample_type %#"PRIx64", error %d\n",
 			 "perf_event__synthesize_sample", sample_type, err);
@@ -343,32 +344,33 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format)
 		goto out_free;
 	}
 
-	evsel.sample_size = __evsel__sample_size(sample_type);
+	evsel->sample_size = __evsel__sample_size(sample_type);
 
-	err = evsel__parse_sample(&evsel, event, &sample_out);
+	err = evsel__parse_sample(evsel, event, &sample_out);
 	if (err) {
 		pr_debug("%s failed for sample_type %#"PRIx64", error %d\n",
 			 "evsel__parse_sample", sample_type, err);
 		goto out_free;
 	}
 
-	if (!samples_same(&sample, &sample_out, sample_type, read_format, evsel.needs_swap)) {
+	if (!samples_same(&sample, &sample_out, sample_type, read_format, evsel->needs_swap)) {
 		pr_debug("parsing failed for sample_type %#"PRIx64"\n",
 			 sample_type);
 		goto out_free;
 	}
 
 	if (sample_type == PERF_SAMPLE_BRANCH_STACK) {
-		evsel.needs_swap = true;
-		evsel.sample_size = __evsel__sample_size(sample_type);
-		err = evsel__parse_sample(&evsel, event, &sample_out_endian);
+		evsel->needs_swap = true;
+		evsel->sample_size = __evsel__sample_size(sample_type);
+		err = evsel__parse_sample(evsel, event, &sample_out_endian);
 		if (err) {
 			pr_debug("%s failed for sample_type %#"PRIx64", error %d\n",
 				 "evsel__parse_sample", sample_type, err);
 			goto out_free;
 		}
 
-		if (!samples_same(&sample, &sample_out_endian, sample_type, read_format, evsel.needs_swap)) {
+		if (!samples_same(&sample, &sample_out_endian, sample_type,
+				  read_format, evsel->needs_swap)) {
 			pr_debug("parsing failed for sample_type %#"PRIx64"\n",
 				 sample_type);
 			goto out_free;
@@ -380,6 +382,7 @@ static int do_test(u64 sample_type, u64 sample_regs, u64 read_format)
 	free(event);
 	perf_sample__exit(&sample_out_endian);
 	perf_sample__exit(&sample_out);
+	evsel__put(evsel);
 	if (ret && read_format)
 		pr_debug("read_format %#"PRIx64"\n", read_format);
 	return ret;
diff --git a/tools/perf/tests/shell/coresight/callchain.sh b/tools/perf/tests/shell/coresight/callchain.sh
new file mode 100755
index 0000000000000..13cca7dc11184
--- /dev/null
+++ b/tools/perf/tests/shell/coresight/callchain.sh
@@ -0,0 +1,172 @@
+#!/bin/bash
+# CoreSight synthesized callchain (exclusive)
+# SPDX-License-Identifier: GPL-2.0
+
+glb_err=1
+
+if ! tmpdir=$(mktemp -d /tmp/perf-cs-callchain-test.XXXXXX); then
+	echo "mktemp failed"
+	exit 1
+fi
+
+cleanup_files()
+{
+	rm -rf "$tmpdir"
+}
+
+trap cleanup_files EXIT
+trap 'cleanup_files; exit $glb_err' TERM INT
+
+skip_if_system_is_not_ready()
+{
+	perf list | grep -Pzq 'cs_etm//' || {
+		echo "[Skip] cs_etm event is not available" >&2
+		return 2
+	}
+
+	# Requires root for trace in kernel
+	[ "$(id -u)" = 0 ] || {
+		echo "[Skip] No root permission" >&2
+		return 2
+	}
+
+	return 0
+}
+
+record_trace()
+{
+	local data=$1
+	local script=$2
+
+	local cf="$tmpdir/ctl"
+	local af="$tmpdir/ack"
+
+	mkfifo "$cf" "$af"
+
+	perf record -o "$data" -e cs_etm// --per-thread -D -1 --control fifo:"$cf","$af" -- \
+		perf test --record-ctl fifo:"$cf","$af" -w callchain >/dev/null 2>&1 &&
+
+	# It is safe to use 'i3i' with a three-instruction interval, since the
+	# workload is compiled with -O0.
+	perf script --itrace=g16i3il64 -i "$data" > "$script"
+}
+
+callchain_regex_1()
+{
+	printf '%s' \
+'perf[[:space:]]+[0-9]+[[:space:]]+\[[0-9]+\][[:space:]]+([0-9.]+:[[:space:]]+)?[0-9]+ instructions:[[:space:]]*\n'\
+'[[:space:]]+[[:xdigit:]]+ callchain_foo\+0x[[:xdigit:]]+ \(.*/perf\)\n'\
+'[[:space:]]+[[:xdigit:]]+ callchain\+0x[[:xdigit:]]+ \(.*/perf\)\n'\
+'([[:space:]]+[[:xdigit:]]+ .*\n)*'
+}
+
+callchain_regex_2()
+{
+	printf '%s' \
+'perf[[:space:]]+[0-9]+[[:space:]]+\[[0-9]+\][[:space:]]+([0-9.]+:[[:space:]]+)?[0-9]+ instructions:[[:space:]]*\n'\
+'[[:space:]]+[[:xdigit:]]+ callchain_do_syscall\+0x[[:xdigit:]]+ \(.*/perf\)\n'\
+'[[:space:]]+[[:xdigit:]]+ callchain_foo\+0x[[:xdigit:]]+ \(.*/perf\)\n'\
+'[[:space:]]+[[:xdigit:]]+ callchain\+0x[[:xdigit:]]+ \(.*/perf\)\n'\
+'([[:space:]]+[[:xdigit:]]+ .*\n)*'
+}
+
+callchain_regex_3()
+{
+	printf '%s' \
+'perf[[:space:]]+[0-9]+[[:space:]]+\[[0-9]+\][[:space:]]+([0-9.]+:[[:space:]]+)?[0-9]+ instructions:[[:space:]]*\n'\
+'[[:space:]]+[[:xdigit:]]+ syscall(@plt)?\+0x[[:xdigit:]]+ \(.*\)\n'\
+'[[:space:]]+[[:xdigit:]]+ callchain_do_syscall\+0x[[:xdigit:]]+ \(.*/perf\)\n'\
+'[[:space:]]+[[:xdigit:]]+ callchain_foo\+0x[[:xdigit:]]+ \(.*/perf\)\n'\
+'[[:space:]]+[[:xdigit:]]+ callchain\+0x[[:xdigit:]]+ \(.*/perf\)\n'\
+'([[:space:]]+[[:xdigit:]]+ .*\n)*'
+}
+
+callchain_regex_4()
+{
+	printf '%s' \
+'perf[[:space:]]+[0-9]+[[:space:]]+\[[0-9]+\][[:space:]]+([0-9.]+:[[:space:]]+)?[0-9]+ instructions:[[:space:]]*\n'\
+'[[:space:]]+[[:xdigit:]]+ .*\+0x[[:xdigit:]]+ \(\[kernel\.kallsyms\]\)\n'\
+'[[:space:]]+[[:xdigit:]]+ syscall(@plt)?\+0x[[:xdigit:]]+ \(.*\)\n'\
+'[[:space:]]+[[:xdigit:]]+ callchain_do_syscall\+0x[[:xdigit:]]+ \(.*/perf\)\n'\
+'[[:space:]]+[[:xdigit:]]+ callchain_foo\+0x[[:xdigit:]]+ \(.*/perf\)\n'\
+'[[:space:]]+[[:xdigit:]]+ callchain\+0x[[:xdigit:]]+ \(.*/perf\)\n'\
+'([[:space:]]+[[:xdigit:]]+ .*\n)*'
+}
+
+find_after_line()
+{
+	local regex="$1"
+	local file="$2"
+	local start="$3"
+	local offset
+	local line
+
+	# Search in byte offset
+	offset=$(
+		tail -n +"$start" "$file" |
+		grep -Pzob -m1 "$regex" |
+		tr '\0' '\n' |
+		sed -n 's/^\([0-9][0-9]*\):.*/\1/p;q'
+	)
+
+	if [ -z "$offset" ]; then
+		echo "Failed to match regex after line $start" >&2
+		echo "Regex:" >&2
+		printf '%s\n' "$regex" >&2
+		echo "Context from line $start:" >&2
+		sed -n "${start},$((start + 100))p" "$file" >&2
+		return 1
+	fi
+
+	# Convert from offset to line
+	line=$(
+		tail -n +"$start" "$file" |
+		head -c "$offset" |
+		wc -l
+	)
+
+	echo "$((start + line))"
+}
+
+check_callchain_flow()
+{
+	local file="$1"
+	local l1 l2 l3 l4 l5 l6 l7
+
+	# Callchain push
+	l1=$(find_after_line "$(callchain_regex_1)" "$file" 1) || return 1
+	l2=$(find_after_line "$(callchain_regex_2)" "$file" "$((l1 + 1))") || return 1
+	l3=$(find_after_line "$(callchain_regex_3)" "$file" "$((l2 + 1))") || return 1
+	l4=$(find_after_line "$(callchain_regex_4)" "$file" "$((l3 + 1))") || return 1
+
+	# Callchain pop
+	l5=$(find_after_line "$(callchain_regex_3)" "$file" "$((l4 + 1))") || return 1
+	l6=$(find_after_line "$(callchain_regex_2)" "$file" "$((l5 + 1))") || return 1
+	l7=$(find_after_line "$(callchain_regex_1)" "$file" "$((l6 + 1))") || return 1
+
+	echo "Callchain flow matched:"
+	echo "  l1=$l1 l2=$l2 l3=$l3 l4=$l4 l5=$l5 l6=$l6 l7=$l7"
+
+	return 0
+}
+
+run_test()
+{
+	local data=$tmpdir/perf.data
+	local script=$tmpdir/perf.script
+
+	if ! record_trace "$data" "$script"; then
+		echo "perf record/script failed"
+		return
+	fi
+
+	check_callchain_flow "$script" || return
+
+	glb_err=0
+}
+
+skip_if_system_is_not_ready || exit 2
+
+run_test
+
+exit $glb_err
diff --git a/tools/perf/tests/shell/coresight/test_arm_coresight_disasm.sh b/tools/perf/tests/shell/coresight/test_arm_coresight_disasm.sh
index ccb90dda24758..f3ebad5963783 100755
--- a/tools/perf/tests/shell/coresight/test_arm_coresight_disasm.sh
+++ b/tools/perf/tests/shell/coresight/test_arm_coresight_disasm.sh
@@ -44,7 +44,7 @@ branch_search='[[:space:]](bl|b(\.(eq|ne|cs|cc|mi|pl|vs|vc|hi|ls|ge|lt|gt|le|al)
 if [ "$(id -u)" == 0 ] && [ -e /proc/kcore ]; then
 	echo "Testing kernel disassembly"
 	perf record -o ${perfdata} -e cs_etm//k --kcore -Se -m,64K -- touch $file > /dev/null 2>&1
-	perf script -i ${perfdata} -s python:${script_path} -- \
+	perf script -i ${perfdata} --itrace=b -s python:${script_path} -- \
 		-d --stop-sample=2 -k ${perfdata}/kcore_dir/kcore 2> /dev/null > ${file}
 	grep -q -E ${branch_search} ${file}
 	echo "Found kernel branches"
@@ -56,7 +56,7 @@ fi
 ## Test user ##
 echo "Testing userspace disassembly"
 perf record -o ${perfdata} -e cs_etm//u -Se -m,64K -- touch $file > /dev/null 2>&1
-perf script -i ${perfdata} -s python:${script_path} -- \
+perf script -i ${perfdata} --itrace=b -s python:${script_path} -- \
 	-d --stop-sample=2 2> /dev/null > ${file}
 grep -q -E ${branch_search} ${file}
 echo "Found userspace branches"
diff --git a/tools/perf/tests/shell/inject_aslr.sh b/tools/perf/tests/shell/inject_aslr.sh
index c00461828ea79..d83c2591db8f6 100755
--- a/tools/perf/tests/shell/inject_aslr.sh
+++ b/tools/perf/tests/shell/inject_aslr.sh
@@ -135,8 +135,14 @@ test_callchain_aslr() {
     echo "Callchain ASLR test [Failed - no noploop samples in original file]"
     err=1
   elif [ -z "$new_addr" ]; then
-    echo "Callchain ASLR test [Failed - could not find remapped address]"
-    err=1
+    if perf evlist -v -i "${data}" | grep -q 'sample_type:.*STACK_USER'; then
+      echo "Dropping stack user sample as possible ASLR leak"
+      echo "Call-graph dwarf not supported with 'perf inject --aslr'"
+      echo "Callchain ASLR test [Skip]"
+    else
+      echo "Callchain ASLR test [Failed - could not find remapped address]"
+      err=1
+    fi
   elif [ "$orig_addr" = "$new_addr" ]; then
     echo "Callchain ASLR test [Failed - addresses are not remapped]"
     err=1
diff --git a/tools/perf/tests/shell/jitdump-python.sh b/tools/perf/tests/shell/jitdump-python.sh
index ae86203b14a22..05aaa3bd900b5 100755
--- a/tools/perf/tests/shell/jitdump-python.sh
+++ b/tools/perf/tests/shell/jitdump-python.sh
@@ -16,11 +16,15 @@ if [ "${HAS_PERF_JIT}" != "True" ]; then
     exit 2
 fi
 
-PERF_DATA=$(mktemp /tmp/__perf_test.perf.data.XXXXXX)
+PERF_DATA_DIR=$(mktemp -d /tmp/__perf_test.perf.data.dir.XXXXXX)
+PERF_DATA="${PERF_DATA_DIR}/perf.data"
 
 cleanup() {
     echo "Cleaning up files..."
-    rm -f ${PERF_DATA} ${PERF_DATA}.jit /tmp/jit-${PID}.dump /tmp/jitted-${PID}-*.so 2> /dev/null
+    rm -rf ${PERF_DATA_DIR} 2> /dev/null
+    for p in ${ALL_PIDS}; do
+        rm -f /tmp/jit-${p}.dump /tmp/jitted-${p}-*.so 2> /dev/null
+    done
 
     trap - EXIT TERM INT
 }
@@ -33,9 +37,16 @@ trap_cleanup() {
 
 trap trap_cleanup EXIT TERM INT
 
-echo "Run python with -Xperf_jit"
-cat <<EOF | perf record -k 1 -g --call-graph dwarf -o "${PERF_DATA}" \
-		 -- ${PYTHON} -Xperf_jit
+ALL_PIDS=""
+NUM=0
+for iterations in 1000000 10000000 50000000 100000000; do
+    echo "Running with $iterations iterations..."
+    rm -f "${PERF_DATA}.pid"
+    cat <<EOF | perf record -k 1 -g --call-graph dwarf -o "${PERF_DATA}" -- ${PYTHON} -Xperf_jit
+import os
+with open("${PERF_DATA}.pid", "w") as f:
+    f.write(str(os.getpid()))
+
 def foo(n):
     result = 0
     for _ in range(n):
@@ -49,29 +60,45 @@ def baz(n):
     bar(n)
 
 if __name__ == "__main__":
-    baz(1000000)
+    baz($iterations)
 EOF
 
-# extract PID of the target process from the data
-_PID=$(perf report -i "${PERF_DATA}" -F pid -q -g none | cut -d: -f1 -s)
-PID=$(echo -n $_PID)  # remove newlines
-
-echo "Generate JIT-ed DSOs using perf inject"
-DEBUGINFOD_URLS='' perf inject -i "${PERF_DATA}" -j -o "${PERF_DATA}.jit"
-
-echo "Add JIT-ed DSOs to the build-ID cache"
-for F in /tmp/jitted-${PID}-*.so; do
-  perf buildid-cache -a "${F}"
-done
-
-echo "Check the symbol containing the function/module name"
-NUM=$(perf report -i "${PERF_DATA}.jit" -s sym | grep -cE 'py::(foo|bar|baz):<stdin>')
-
-echo "Found ${NUM} matching lines"
-
-echo "Remove JIT-ed DSOs from the build-ID cache"
-for F in /tmp/jitted-${PID}-*.so; do
-  perf buildid-cache -r "${F}"
+    if [ -f "${PERF_DATA}.pid" ]; then
+        REAL_PID=$(cat "${PERF_DATA}.pid")
+        ALL_PIDS="${ALL_PIDS} ${REAL_PID}"
+    fi
+
+    # extract PID of the target process from the data
+    PID=$(perf report -i "${PERF_DATA}" --stdio -F pid -q -g none | \
+          cut -d: -f1 -s | sort -u | head -n 1 | tr -d ' ')
+    if [ -z "${PID}" ]; then
+        echo "Failed to get PID, retrying..."
+        continue
+    fi
+    ALL_PIDS="${ALL_PIDS} ${PID}"
+
+    echo "Generate JIT-ed DSOs using perf inject"
+    DEBUGINFOD_URLS='' perf inject -i "${PERF_DATA}" -j -o "${PERF_DATA}.jit"
+
+    echo "Add JIT-ed DSOs to the build-ID cache"
+    for F in /tmp/jitted-${PID}-*.so; do
+        perf buildid-cache -a "${F}"
+    done
+
+    echo "Check the symbol containing the function/module name"
+    NUM=$(perf report -i "${PERF_DATA}.jit" -s sym --stdio | grep -cE 'py::(foo|bar|baz):<stdin>')
+
+    echo "Remove JIT-ed DSOs from the build-ID cache"
+    for F in /tmp/jitted-${PID}-*.so; do
+        perf buildid-cache -r "${F}"
+    done
+    rm -f /tmp/jitted-${PID}-*.so /tmp/jit-${PID}.dump 2>/dev/null
+
+    if [ "${NUM}" -gt 0 ]; then
+        echo "Success: found ${NUM} matching lines"
+        break
+    fi
+    echo "No matching lines found, retrying with more iterations..."
 done
 
 cleanup
diff --git a/tools/perf/tests/shell/kvm.sh b/tools/perf/tests/shell/kvm.sh
index f88e859025c42..7b26368051ffd 100755
--- a/tools/perf/tests/shell/kvm.sh
+++ b/tools/perf/tests/shell/kvm.sh
@@ -39,17 +39,28 @@ skip() {
 test_kvm_stat() {
 	echo "Testing perf kvm stat"
 
-	echo "Recording kvm events for pid ${qemu_pid}..."
-	if ! perf kvm stat record -p "${qemu_pid}" -o "${perfdata}" sleep 1; then
-		echo "Failed to record kvm events"
-		err=1
-		return
-	fi
+	local duration
+	local success=false
+	for duration in 1 2 4 8; do
+		echo "Recording kvm events for pid ${qemu_pid} (duration ${duration}s)..."
+		rm -f "${perfdata}" "${perfdata}".old
+		if ! perf kvm stat record -p "${qemu_pid}" -o "${perfdata}" \
+			sleep ${duration} >/dev/null 2>&1; then
+			echo "perf kvm stat record failed, retrying..."
+			continue
+		fi
 
-	echo "Reporting kvm events..."
-	if ! perf kvm -i "${perfdata}" stat report 2>&1 | grep -q "VM-EXIT"; then
+		if [ -e "${perfdata}" ] && \
+			perf kvm -i "${perfdata}" stat report 2>&1 | grep -q "VM-EXIT"; then
+			success=true
+			break
+		fi
+		echo "No VM-EXIT events found, retrying..."
+	done
+
+	if [ "$success" = false ]; then
 		echo "Failed to find VM-EXIT in report"
-		perf kvm -i "${perfdata}" stat report 2>&1
+		perf kvm -i "${perfdata}" stat report 2>&1 || true
 		err=1
 		return
 	fi
@@ -60,22 +71,26 @@ test_kvm_stat() {
 test_kvm_record_report() {
 	echo "Testing perf kvm record/report"
 
-	echo "Recording kvm profile for pid ${qemu_pid}..."
-	# Use --host to avoid needing guest symbols/mounts for this simple test
-	# We just want to verify the command runs and produces data
-	# We run in background and kill it because 'perf kvm record' appends options
-	# after the command, which breaks 'sleep' (e.g. it gets '-e cycles').
-	perf kvm --host record -p "${qemu_pid}" -o "${perfdata}" &
-	rec_pid=$!
-	sleep 1
-	kill -INT "${rec_pid}"
-	wait "${rec_pid}" || true
+	local duration
+	local success=false
+	for duration in 1 2 4 8; do
+		echo "Recording kvm profile for pid ${qemu_pid} (duration ${duration}s)..."
+		rm -f "${perfdata}" "${perfdata}".old
 
-	echo "Reporting kvm profile..."
-	# Check for some standard output from report
-	if ! perf kvm -i "${perfdata}" report --stdio 2>&1 | grep -q "Event count"; then
+		perf kvm --host record -p "${qemu_pid}" -o "${perfdata}" \
+			-e cpu-clock sleep ${duration}
+
+		if [ -e "${perfdata}" ] && \
+			perf kvm -i "${perfdata}" report --stdio 2>&1 | grep -q "Event count"; then
+			success=true
+			break
+		fi
+		echo "No samples or report failed, retrying..."
+	done
+
+	if [ "$success" = false ]; then
 		echo "Failed to report kvm profile"
-		perf kvm -i "${perfdata}" report --stdio 2>&1
+		perf kvm -i "${perfdata}" report --stdio 2>&1 || true
 		err=1
 		return
 	fi
@@ -123,6 +138,15 @@ test_kvm_stat_live() {
 	echo "perf kvm stat live test [Success]"
 }
 
+test_kvm_default_event() {
+	echo "Testing perf kvm record default event with command line"
+
+	# Check if kvm record with default events handle command line arguments
+	perf kvm record -p "${qemu_pid}" -o /dev/null sleep 1
+
+	echo "perf kvm record default event [Success]"
+}
+
 setup_qemu() {
 	# Find qemu
 	if [ "$(uname -m)" = "x86_64" ]; then
@@ -176,6 +200,7 @@ if [ $err -eq 0 ]; then
 	test_kvm_record_report
 	test_kvm_buildid_list
 	test_kvm_stat_live
+	test_kvm_default_event
 fi
 
 cleanup
diff --git a/tools/perf/tests/shell/lib/perf_metric_validation.py b/tools/perf/tests/shell/lib/perf_metric_validation.py
index dea8ef1977bf6..3d52f94f22b91 100644
--- a/tools/perf/tests/shell/lib/perf_metric_validation.py
+++ b/tools/perf/tests/shell/lib/perf_metric_validation.py
@@ -383,10 +383,13 @@ class Validator:
         wl = workload.split()
         command.extend(wl)
         print(" ".join(command))
-        cmd = subprocess.run(command, stderr=subprocess.PIPE, encoding='utf-8')
-        data = [x+'}' for x in cmd.stderr.split('}\n') if x]
-        if data[0][0] != '{':
-            data[0] = data[0][data[0].find('{'):]
+        cmd = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf-8')
+        lines = cmd.stderr.splitlines() + cmd.stdout.splitlines()
+        data = []
+        for line in lines:
+            line = line.strip()
+            if line.startswith('{') and line.endswith('}'):
+                data.append(line)
         return data
 
     def collect_perf(self, workload: str):
diff --git a/tools/perf/tests/shell/lib/perf_record.sh b/tools/perf/tests/shell/lib/perf_record.sh
new file mode 100644
index 0000000000000..2b9e11b66dc7a
--- /dev/null
+++ b/tools/perf/tests/shell/lib/perf_record.sh
@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: GPL-2.0
+
+PERF_RECORD_LOGS=()
+
+perf_record_with_retry() {
+  local perfdata="$1"
+  local check_cmd="$2"
+  local testprog_base="$3"
+  shift 3
+
+  local logfile
+  logfile=$(mktemp /tmp/__perf_record_retry.XXXXXX)
+  PERF_RECORD_LOGS+=("$logfile")
+
+  # Save the e flag state and disable it
+  local save_e
+  if [[ $- == *e* ]]; then
+    save_e="set -e"
+  else
+    save_e="set +e"
+  fi
+  set +e
+
+  local duration
+  local first_run=true
+  local ret=1
+  local cmd_prefix="perf record"
+  if [ -n "${PERF_RECORD_CMD}" ]; then
+    cmd_prefix="${PERF_RECORD_CMD}"
+  fi
+
+  for duration in 0.01 0.1 0.3 1.0 2.0; do
+    rm -f "${perfdata}".old
+    ${cmd_prefix} "$@" -o "${perfdata}" ${testprog_base} ${duration} > "$logfile" 2>&1
+    local record_exit=$?
+
+    if [ "$first_run" = true ] && [ $record_exit -ne 0 ]; then
+      ret=2
+      break
+    fi
+    first_run=false
+
+    if [ -e "${perfdata}" ] && eval "${check_cmd}"; then
+      ret=0
+      break
+    fi
+  done
+
+  eval "$save_e"
+  return $ret
+}
+
+perf_record_cleanup() {
+  for logfile in "${PERF_RECORD_LOGS[@]}"; do
+    rm -f "$logfile"
+  done
+  PERF_RECORD_LOGS=()
+}
diff --git a/tools/perf/tests/shell/lib/setup_python.sh b/tools/perf/tests/shell/lib/setup_python.sh
index a58e5536f2edd..2173215a05174 100644
--- a/tools/perf/tests/shell/lib/setup_python.sh
+++ b/tools/perf/tests/shell/lib/setup_python.sh
@@ -14,3 +14,16 @@ then
   echo Skipping test, python not detected please set environment variable PYTHON.
   exit 2
 fi
+
+# Set PYTHONPATH to find the in-tree built perf.so first, avoiding system-wide perf.so
+if [ -n "$PERF_EXEC_PATH" ] && [ -d "$PERF_EXEC_PATH/python" ]; then
+  PYTHONPATH_DIR="$PERF_EXEC_PATH/python"
+elif [ -d "$(dirname "$0")/../../python" ]; then
+  PYTHONPATH_DIR="$(dirname "$0")/../../python"
+elif [ -d "$(dirname "$0")/../python" ]; then
+  PYTHONPATH_DIR="$(dirname "$0")/../python"
+fi
+
+if [ -n "$PYTHONPATH_DIR" ]; then
+  export PYTHONPATH="$PYTHONPATH_DIR${PYTHONPATH:+:$PYTHONPATH}"
+fi
diff --git a/tools/perf/tests/shell/lock_contention.sh b/tools/perf/tests/shell/lock_contention.sh
index 52e8b9db9fbd8..5df9a0cd48a34 100755
--- a/tools/perf/tests/shell/lock_contention.sh
+++ b/tools/perf/tests/shell/lock_contention.sh
@@ -9,6 +9,11 @@ perfdata=$(mktemp /tmp/__perf_test.perf.data.XXXXX)
 result=$(mktemp /tmp/__perf_test.result.XXXXX)
 errout=$(mktemp /tmp/__perf_test.errout.XXXXX)
 
+# Workload to generate lock contention.
+# Using 1 group (-g 1) keeps runtime low while generating sufficient lock events.
+# We include -p (pipes) because socketpairs don't generate enough lock events on s390.
+msg_workload="perf bench sched messaging -g 1 -p"
+
 cleanup() {
 	rm -f ${perfdata}
 	rm -f ${result}
@@ -50,7 +55,7 @@ check() {
 test_record()
 {
 	echo "Testing perf lock record and perf lock contention"
-	perf lock record -o ${perfdata} -- perf bench sched messaging -p > /dev/null 2>&1
+	perf lock record -o ${perfdata} -- ${msg_workload} > /dev/null 2>&1
 	# the output goes to the stderr and we expect only 1 output (-E 1)
 	perf lock contention -i ${perfdata} -E 1 -q 2> ${result}
 	if [ "$(cat "${result}" | wc -l)" != "1" ]; then
@@ -70,7 +75,7 @@ test_bpf()
 	fi
 
 	# the perf lock contention output goes to the stderr
-	perf lock con -a -b -E 1 -q -- perf bench sched messaging -p > /dev/null 2> ${result}
+	perf lock con -a -b -E 1 -q -- ${msg_workload} > /dev/null 2> ${result}
 	if [ "$(cat "${result}" | wc -l)" != "1" ]; then
 		echo "[Fail] BPF result count is not 1:" "$(cat "${result}" | wc -l)"
 		err=1
@@ -81,7 +86,7 @@ test_bpf()
 test_record_concurrent()
 {
 	echo "Testing perf lock record and perf lock contention at the same time"
-	perf lock record -o- -- perf bench sched messaging -p 2> ${errout} | \
+	perf lock record -o- -- ${msg_workload} 2> ${errout} | \
 	perf lock contention -i- -E 1 -q 2> ${result}
 	if [ "$(cat "${result}" | wc -l)" != "1" ]; then
 		echo "[Fail] Recorded result count is not 1:" "$(cat "${result}" | wc -l)"
@@ -107,7 +112,7 @@ test_aggr_task()
 	fi
 
 	# the perf lock contention output goes to the stderr
-	perf lock con -a -b -t -E 1 -q -- perf bench sched messaging -p > /dev/null 2> ${result}
+	perf lock con -a -b -t -E 1 -q -- ${msg_workload} > /dev/null 2> ${result}
 	if [ "$(cat "${result}" | wc -l)" != "1" ]; then
 		echo "[Fail] BPF result count is not 1:" "$(cat "${result}" | wc -l)"
 		err=1
@@ -130,7 +135,7 @@ test_aggr_addr()
 	fi
 
 	# the perf lock contention output goes to the stderr
-	perf lock con -a -b -l -E 1 -q -- perf bench sched messaging -p > /dev/null 2> ${result}
+	perf lock con -a -b -l -E 1 -q -- ${msg_workload} > /dev/null 2> ${result}
 	if [ "$(cat "${result}" | wc -l)" != "1" ]; then
 		echo "[Fail] BPF result count is not 1:" "$(cat "${result}" | wc -l)"
 		err=1
@@ -148,7 +153,7 @@ test_aggr_cgroup()
 	fi
 
 	# the perf lock contention output goes to the stderr
-	perf lock con -a -b --lock-cgroup -E 1 -q -- perf bench sched messaging -p > /dev/null 2> ${result}
+	perf lock con -a -b --lock-cgroup -E 1 -q -- ${msg_workload} > /dev/null 2> ${result}
 	if [ "$(cat "${result}" | wc -l)" != "1" ]; then
 		echo "[Fail] BPF result count is not 1:" "$(cat "${result}" | wc -l)"
 		err=1
@@ -170,7 +175,7 @@ test_type_filter()
 		return
 	fi
 
-	perf lock con -a -b -Y spinlock -q -- perf bench sched messaging -p > /dev/null 2> ${result}
+	perf lock con -a -b -Y spinlock -q -- ${msg_workload} > /dev/null 2> ${result}
 	if [ "$(grep -c -v spinlock "${result}")" != "0" ]; then
 		echo "[Fail] BPF result should not have non-spinlocks:" "$(cat "${result}")"
 		err=1
@@ -202,7 +207,7 @@ test_lock_filter()
 		return
 	fi
 
-	perf lock con -a -b -L tasklist_lock -q -- perf bench sched messaging -p > /dev/null 2> ${result}
+	perf lock con -a -b -L tasklist_lock -q -- ${msg_workload} > /dev/null 2> ${result}
 	if [ "$(grep -c -v "${test_lock_filter_type}" "${result}")" != "0" ]; then
 		echo "[Fail] BPF result should not have non-${test_lock_filter_type} locks:" "$(cat "${result}")"
 		err=1
@@ -241,7 +246,7 @@ test_stack_filter()
 		return
 	fi
 
-	perf lock con -a -b -S unix_stream -E 1 -q -- perf bench sched messaging -p > /dev/null 2> ${result}
+	perf lock con -a -b -S unix_stream -E 1 -q -- ${msg_workload} > /dev/null 2> ${result}
 	if [ "$(cat "${result}" | wc -l)" != "1" ]; then
 		echo "[Fail] BPF result should have a lock from unix_stream:" "$(cat "${result}")"
 		err=1
@@ -269,7 +274,7 @@ test_aggr_task_stack_filter()
 		return
 	fi
 
-	perf lock con -a -b -t -S unix_stream -E 1 -q -- perf bench sched messaging -p > /dev/null 2> ${result}
+	perf lock con -a -b -t -S unix_stream -E 1 -q -- ${msg_workload} > /dev/null 2> ${result}
 	if [ "$(cat "${result}" | wc -l)" != "1" ]; then
 		echo "[Fail] BPF result should have a task from unix_stream:" "$(cat "${result}")"
 		err=1
@@ -285,7 +290,8 @@ test_cgroup_filter()
 		return
 	fi
 
-	perf lock con -a -b --lock-cgroup -E 1 -F wait_total -q -- perf bench sched messaging -p > /dev/null 2> ${result}
+	perf lock con -a -b --lock-cgroup -E 1 -F wait_total -q \
+		-- ${msg_workload} > /dev/null 2> ${result}
 	if [ "$(cat "${result}" | wc -l)" != "1" ]; then
 		echo "[Fail] BPF result should have a cgroup result:" "$(cat "${result}")"
 		err=1
@@ -293,7 +299,8 @@ test_cgroup_filter()
 	fi
 
 	cgroup=$(cat "${result}" | awk '{ print $3 }')
-	perf lock con -a -b --lock-cgroup -E 1 -G "${cgroup}" -q -- perf bench sched messaging -p > /dev/null 2> ${result}
+	perf lock con -a -b --lock-cgroup -E 1 -G "${cgroup}" -q \
+		-- ${msg_workload} > /dev/null 2> ${result}
 	if [ "$(cat "${result}" | wc -l)" != "1" ]; then
 		echo "[Fail] BPF result should have a result with cgroup filter:" "$(cat "${cgroup}")"
 		err=1
@@ -328,7 +335,7 @@ test_csv_output()
 	fi
 
 	# the perf lock contention output goes to the stderr
-	perf lock con -a -b -E 1 -x , --output ${result} -- perf bench sched messaging -p > /dev/null 2>&1
+	perf lock con -a -b -E 1 -x , --output ${result} -- ${msg_workload} > /dev/null 2>&1
 	output=$(grep -v "^#" ${result} | tr -d -c , | wc -c)
 	if [ "${header}" != "${output}" ]; then
 		echo "[Fail] BPF result does not match the number of commas: ${header} != ${output}"
diff --git a/tools/perf/tests/shell/pipe_test.sh b/tools/perf/tests/shell/pipe_test.sh
index e459aa99a9515..ce68d850c9838 100755
--- a/tools/perf/tests/shell/pipe_test.sh
+++ b/tools/perf/tests/shell/pipe_test.sh
@@ -12,8 +12,8 @@ skip_test_missing_symbol ${sym}
 
 data=$(mktemp /tmp/perf.data.XXXXXX)
 data2=$(mktemp /tmp/perf.data2.XXXXXX)
-prog="perf test -w noploop"
-[ "$(uname -m)" = "s390x" ] && prog="$prog 3"
+prog="perf test -w noploop 0.1"
+[ "$(uname -m)" = "s390x" ] && prog="perf test -w noploop 3"
 err=0
 
 set -e
diff --git a/tools/perf/tests/shell/record+zstd_comp_decomp_multi_record.sh b/tools/perf/tests/shell/record+zstd_comp_decomp_multi_record.sh
new file mode 100755
index 0000000000000..c05ace8214ca3
--- /dev/null
+++ b/tools/perf/tests/shell/record+zstd_comp_decomp_multi_record.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+# Zstd perf.data compression/decompression of multi-record data
+# SPDX-License-Identifier: GPL-2.0
+
+perfdata=$(mktemp /tmp/__perf_test.perf.data.XXXXX)
+recout=$(mktemp /tmp/__perf_test.zstd.rec.XXXXX)
+injout=$(mktemp /tmp/__perf_test.zstd.inj.XXXXX)
+perf_tool=perf
+
+cleanup() {
+	rm -f "${perfdata}" "${perfdata}".old "${perfdata}".decomp "${recout}" "${injout}"
+}
+trap cleanup EXIT TERM INT
+
+skip_if_no_z_record() {
+	$perf_tool record -h 2>&1 | grep -q -- '-z, --compression-level'
+}
+
+collect_z_record() {
+	echo "Collecting compressed record file:"
+	[ "$(uname -m)" != s390x ] && gflag='-g'
+	$perf_tool record -o "${perfdata}" $gflag -z -F max -m 32M --per-thread -- \
+		$perf_tool test -w thloop 5 1 \
+		>/dev/null 2>"${recout}"
+}
+
+check_record() {
+	echo "Checking record did not fail to write data:"
+	if grep -q "failed to write perf data" "${recout}"; then
+		cat "${recout}"
+		return 1
+	fi
+}
+
+check_decompress() {
+	echo "Checking compressed file decompresses cleanly:"
+	if ! $perf_tool inject -i "${perfdata}" -o "${perfdata}".decomp 2>"${injout}"; then
+		cat "${injout}"
+		return 1
+	fi
+	if grep -Eqi "decompress|corrupt|failed to process type" "${injout}"; then
+		cat "${injout}"
+		return 1
+	fi
+}
+
+skip_if_no_z_record || exit 2
+collect_z_record
+check_record || exit 1
+
+# Need >1 record, else the multi-record path wasn't exercised.
+# Skip rather than pass/fail spuriously.
+nr=$($perf_tool report -i "${perfdata}" --stats 2>/dev/null |
+	awk '/COMPRESSED2 events:/ { print $3 }')
+if [ -z "${nr}" ] || [ "${nr}" -lt 2 ]; then
+	echo "less than two compressed records (${nr:-0}), skipping"
+	exit 2
+fi
+echo "Produced ${nr} compressed records"
+
+check_decompress
+err=$?
+exit $err
diff --git a/tools/perf/tests/shell/record.sh b/tools/perf/tests/shell/record.sh
index 7cb81cf3444a7..625240809fce3 100755
--- a/tools/perf/tests/shell/record.sh
+++ b/tools/perf/tests/shell/record.sh
@@ -1,10 +1,13 @@
 #!/bin/bash
-# perf record tests (exclusive)
 # SPDX-License-Identifier: GPL-2.0
+# perf record tests
 
 set -e
 
 shelldir=$(dirname "$0")
+. "${shelldir}"/lib/perf_record.sh
+
+
 # shellcheck source=lib/waiting.sh
 . "${shelldir}"/lib/waiting.sh
 
@@ -39,6 +42,7 @@ cleanup() {
   rm -f "${perfdata}"
   rm -f "${perfdata}".old
   rm -f "${script_output}"
+  perf_record_cleanup
 
   trap - EXIT TERM INT
 }
@@ -50,22 +54,20 @@ trap_cleanup() {
 }
 trap trap_cleanup EXIT TERM INT
 
+check_per_thread() {
+  perf report -i "${perfdata}" -q | grep -q "${testsym}"
+}
+
 test_per_thread() {
   echo "Basic --per-thread mode test"
-  if ! perf record -o /dev/null --quiet ${testprog} 2> /dev/null
-  then
+  local ret=0
+  perf_record_with_retry "${perfdata}" "check_per_thread" "perf test -w thloop" \
+    --per-thread || ret=$?
+  if [ $ret -eq 2 ]; then
     echo "Per-thread record [Skipped event not supported]"
     return
-  fi
-  if ! perf record --per-thread -o "${perfdata}" ${testprog} 2> /dev/null
-  then
-    echo "Per-thread record [Failed record]"
-    err=1
-    return
-  fi
-  if ! perf report -i "${perfdata}" -q | grep -q "${testsym}"
-  then
-    echo "Per-thread record [Failed missing output]"
+  elif [ $ret -eq 1 ]; then
+    echo "Per-thread record [Failed record or missing output]"
     err=1
     return
   fi
@@ -96,6 +98,10 @@ test_per_thread() {
   echo "Basic --per-thread mode test [Success]"
 }
 
+check_register_capture() {
+  perf script -F ip,sym,iregs -i "${perfdata}" 2>/dev/null | grep -q "DI:"
+}
+
 test_register_capture() {
   echo "Register capture test"
   if ! perf list pmu | grep -q 'br_inst_retired.near_call'
@@ -108,11 +114,12 @@ test_register_capture() {
     echo "Register capture test [Skipped missing registers]"
     return
   fi
-  if ! perf record -o - --intr-regs=di,r8,dx,cx -e br_inst_retired.near_call \
-    -c 1000 --per-thread ${testprog} 2> /dev/null \
-    | perf script -F ip,sym,iregs -i - 2> /dev/null \
-    | grep -q "DI:"
-  then
+
+  local ret=0
+  perf_record_with_retry "${perfdata}" "check_register_capture" "perf test -w thloop" \
+    --intr-regs=di,r8,dx,cx -e br_inst_retired.near_call -c 1000 --per-thread || ret=$?
+
+  if [ $ret -ne 0 ]; then
     echo "Register capture test [Failed missing output]"
     err=1
     return
@@ -120,65 +127,66 @@ test_register_capture() {
   echo "Register capture test [Success]"
 }
 
+check_system_wide() {
+  perf report -i "${perfdata}" -q | grep -q "${testsym}"
+}
+
 test_system_wide() {
   echo "Basic --system-wide mode test"
-  if ! perf record -aB --synth=no -o "${perfdata}" ${testprog} 2> /dev/null
-  then
+  local ret=0
+  perf_record_with_retry "${perfdata}" "check_system_wide" "perf test -w thloop" \
+    -aB --synth=no || ret=$?
+  if [ $ret -eq 2 ]; then
     echo "System-wide record [Skipped not supported]"
     return
-  fi
-  if ! perf report -i "${perfdata}" -q | grep -q "${testsym}"
-  then
+  elif [ $ret -eq 1 ]; then
     echo "System-wide record [Failed missing output]"
     err=1
     return
   fi
-  if ! perf record -aB --synth=no -e cpu-clock,cs --threads=cpu \
-    -o "${perfdata}" ${testprog} 2> /dev/null
-  then
-    echo "System-wide record [Failed record --threads option]"
-    err=1
-    return
-  fi
-  if ! perf report -i "${perfdata}" -q | grep -q "${testsym}"
-  then
-    echo "System-wide record [Failed --threads missing output]"
+
+  ret=0
+  perf_record_with_retry "${perfdata}" "check_system_wide" "perf test -w thloop" \
+    -aB --synth=no -e cpu-clock,cs --threads=cpu || ret=$?
+  if [ $ret -ne 0 ]; then
+    echo "System-wide record [Failed record --threads option or missing output]"
     err=1
     return
   fi
   echo "Basic --system-wide mode test [Success]"
 }
 
+check_workload() {
+  perf report -i "${perfdata}" -q | grep -q "${testsym}"
+}
+
 test_workload() {
   echo "Basic target workload test"
-  if ! perf record -o "${perfdata}" ${testprog} 2> /dev/null
-  then
-    echo "Workload record [Failed record]"
-    err=1
-    return
-  fi
-  if ! perf report -i "${perfdata}" -q | grep -q "${testsym}"
-  then
-    echo "Workload record [Failed missing output]"
-    err=1
-    return
-  fi
-  if ! perf record -e cpu-clock,cs --threads=package \
-    -o "${perfdata}" ${testprog} 2> /dev/null
-  then
-    echo "Workload record [Failed record --threads option]"
+  local ret=0
+  perf_record_with_retry "${perfdata}" "check_workload" "perf test -w thloop" || ret=$?
+  if [ $ret -ne 0 ]; then
+    echo "Workload record [Failed record or missing output]"
     err=1
     return
   fi
-  if ! perf report -i "${perfdata}" -q | grep -q "${testsym}"
-  then
-    echo "Workload record [Failed --threads missing output]"
+
+  ret=0
+  perf_record_with_retry "${perfdata}" "check_workload" "perf test -w thloop" \
+    -e cpu-clock,cs --threads=package || ret=$?
+  if [ $ret -ne 0 ]; then
+    echo "Workload record [Failed record --threads option or missing output]"
     err=1
     return
   fi
   echo "Basic target workload test [Success]"
 }
 
+check_branch_counter() {
+  perf report -i "${perfdata}" -D -q 2>/dev/null | grep -q "$br_cntr_output" && \
+  perf script -i "${perfdata}" -F +brstackinsn,+brcntr 2>/dev/null | \
+    grep -q "$br_cntr_script_output"
+}
+
 test_branch_counter() {
   echo "Branch counter test"
   # Check if the branch counter feature is supported
@@ -190,67 +198,60 @@ test_branch_counter() {
       return
     fi
   done
-  if ! perf record -o "${perfdata}" -e "{branches:p,instructions}" -j any,counter ${testprog} 2> /dev/null
-  then
-    echo "Branch counter record test [Failed record]"
-    err=1
-    return
-  fi
-  if ! perf report -i "${perfdata}" -D -q | grep -q "$br_cntr_output"
-  then
-    echo "Branch counter report test [Failed missing output]"
-    err=1
-    return
-  fi
-  if ! perf script -i "${perfdata}" -F +brstackinsn,+brcntr | grep -q "$br_cntr_script_output"
-  then
-    echo " Branch counter script test [Failed missing output]"
+  local ret=0
+  perf_record_with_retry "${perfdata}" "check_branch_counter" "perf test -w thloop" \
+    -e "{branches:p,instructions}" -j any,counter || ret=$?
+  if [ $ret -ne 0 ]; then
+    echo "Branch counter test [Failed record or missing output]"
     err=1
     return
   fi
   echo "Branch counter test [Success]"
 }
 
+check_cgroup() {
+  perf report -i "${perfdata}" -D 2>/dev/null | grep -q "CGROUP" && \
+  perf script -i "${perfdata}" -F cgroup 2>/dev/null | grep -q -v "unknown"
+}
+
 test_cgroup() {
   echo "Cgroup sampling test"
-  if ! perf record -aB --synth=cgroup --all-cgroups -o "${perfdata}" ${testprog} 2> /dev/null
-  then
+  local ret=0
+  perf_record_with_retry "${perfdata}" "check_cgroup" "perf test -w thloop" \
+    -aB --synth=cgroup --all-cgroups || ret=$?
+  if [ $ret -eq 2 ]; then
     echo "Cgroup sampling [Skipped not supported]"
     return
-  fi
-  if ! perf report -i "${perfdata}" -D | grep -q "CGROUP"
-  then
+  elif [ $ret -eq 1 ]; then
     echo "Cgroup sampling [Failed missing output]"
     err=1
     return
   fi
-  if ! perf script -i "${perfdata}" -F cgroup | grep -q -v "unknown"
-  then
-    echo "Cgroup sampling [Failed cannot resolve cgroup names]"
-    err=1
-    return
-  fi
   echo "Cgroup sampling test [Success]"
 }
 
+check_uid() {
+  perf report -i "${perfdata}" -q | grep -q "${testsym}"
+}
+
 test_uid() {
   echo "Uid sampling test"
-  if ! perf record -aB --synth=no --uid "$(id -u)" -o "${perfdata}" ${testprog} \
-    > "${script_output}" 2>&1
-  then
-    if grep -q "libbpf.*EPERM" "${script_output}"
+  local ret=0
+  perf_record_with_retry "${perfdata}" "check_uid" "perf test -w thloop" \
+    -aB --synth=no --uid "$(id -u)" || ret=$?
+  if [ $ret -eq 2 ]; then
+    local logfile="${PERF_RECORD_LOGS[${#PERF_RECORD_LOGS[@]}-1]}"
+    if grep -q -E "libbpf.*EPERM|Access to performance monitoring" "$logfile" || \
+       grep -q -E "Permission denied|Failure to open any events" "$logfile"
     then
       echo "Uid sampling [Skipped permissions]"
       return
     else
       echo "Uid sampling [Failed to record]"
       err=1
-      # cat "${script_output}"
       return
     fi
-  fi
-  if ! perf report -i "${perfdata}" -q | grep -q "${testsym}"
-  then
+  elif [ $ret -eq 1 ]; then
     echo "Uid sampling [Failed missing output]"
     err=1
     return
@@ -402,6 +403,48 @@ test_callgraph() {
   echo "Callgraph test [Success]"
 }
 
+test_acr_sampling() {
+  events="{instructions/period=40000,acr_mask=0x2/u,cycles/period=20000,acr_mask=0x3/u}"
+  pebs_events="{instructions/period=40000,acr_mask=0x2/pu,cycles/period=20000,acr_mask=0x3/u}"
+  echo "Auto counter reload (ACR) sampling test"
+  if ! perf record -o "${perfdata}" -e "${events}" ${testprog} 2> /dev/null
+  then
+    echo "Auto counter reload sampling [Skipped not supported]"
+    return
+  fi
+  if ! perf script -i "${perfdata}" -F event | grep -q "instructions"
+  then
+    echo "Auto counter reload sampling [Failed missing instructions event]"
+    err=1
+    return
+  fi
+  if perf script -i "${perfdata}" -F event | grep -q "cycles"
+  then
+    echo "Auto counter reload sampling [Failed cycles event shouldn't be sampled]"
+    err=1
+    return
+  fi
+  if ! perf record -o "${perfdata}" -e "${pebs_events}" ${testprog} 2> /dev/null
+  then
+    echo "Auto counter reload PEBS sampling [Skipped not supported]"
+    echo "Auto counter reload sampling [Success]"
+    return
+  fi
+  if ! perf script -i "${perfdata}" -F event | grep -q "instructions"
+  then
+    echo "Auto counter reload PEBS sampling [Failed missing instructions event]"
+    err=1
+    return
+  fi
+  if perf script -i "${perfdata}" -F event | grep -q "cycles"
+  then
+    echo "Auto counter reload PEBS sampling [Failed cycles event shouldn't be sampled]"
+    err=1
+    return
+  fi
+  echo "Auto counter reload sampling [Success]"
+}
+
 test_ratio_to_prev() {
   echo "ratio-to-prev test"
   if ! perf record -o /dev/null -e "{instructions, cycles/period=100000,ratio-to-prev=0.5/}" \
@@ -457,6 +500,7 @@ test_leader_sampling
 test_topdown_leader_sampling
 test_precise_max
 test_callgraph
+test_acr_sampling
 test_ratio_to_prev
 
 # restore the default value
diff --git a/tools/perf/tests/shell/record_lbr.sh b/tools/perf/tests/shell/record_lbr.sh
index 78a02e90ece1e..8d51afeb437ba 100755
--- a/tools/perf/tests/shell/record_lbr.sh
+++ b/tools/perf/tests/shell/record_lbr.sh
@@ -1,9 +1,12 @@
 #!/bin/bash
-# perf record LBR tests (exclusive)
 # SPDX-License-Identifier: GPL-2.0
+# perf record LBR tests
 
 set -e
 
+shelldir=$(dirname "$0")
+. "${shelldir}"/lib/perf_record.sh
+
 ParanoidAndNotRoot() {
   [ "$(id -u)" != 0 ] && [ "$(cat /proc/sys/kernel/perf_event_paranoid)" -gt $1 ]
 }
@@ -22,6 +25,7 @@ cleanup() {
   rm -rf "${perfdata}"
   rm -rf "${perfdata}".old
   rm -rf "${perfdata}".txt
+  perf_record_cleanup
 
   trap - EXIT TERM INT
 }
@@ -34,22 +38,28 @@ trap_cleanup() {
 trap trap_cleanup EXIT TERM INT
 
 
+check_lbr_callgraph() {
+  perf report --stitch-lbr -i "${perfdata}" > "${perfdata}".txt 2>&1
+}
+
 lbr_callgraph_test() {
   test="LBR callgraph"
 
   echo "$test"
-  if ! perf record -e cycles --call-graph lbr -o "${perfdata}" perf test -w thloop
-  then
+  set +e
+  perf_record_with_retry "${perfdata}" "check_lbr_callgraph" "perf test -w thloop" \
+    -e cycles --call-graph lbr
+  local ret=$?
+  set -e
+
+  if [ $ret -eq 2 ]; then
     echo "$test [Failed support missing]"
     if [ $err -eq 0 ]
     then
       err=2
     fi
     return
-  fi
-
-  if ! perf report --stitch-lbr -i "${perfdata}" > "${perfdata}".txt
-  then
+  elif [ $ret -eq 1 ]; then
     cat "${perfdata}".txt
     echo "$test [Failed in perf report]"
     err=1
@@ -59,6 +69,12 @@ lbr_callgraph_test() {
   echo "$test [Success]"
 }
 
+check_lbr_samples() {
+  local out
+  out=$(perf report -D -i "${perfdata}" 2> /dev/null | grep -A1 'PERF_RECORD_SAMPLE')
+  [ "$(echo "$out" | grep -c 'PERF_RECORD_SAMPLE' || true)" -gt 0 ]
+}
+
 lbr_test() {
   local branch_flags=$1
   local test="LBR $2 test"
@@ -70,25 +86,27 @@ lbr_test() {
   local r
 
   echo "$test"
-  if ! perf record -e cycles $branch_flags -o "${perfdata}" perf test -w thloop
-  then
+  set +e
+  perf_record_with_retry "${perfdata}" "check_lbr_samples" "perf test -w thloop" \
+    -e cycles $branch_flags
+  local ret=$?
+  set -e
+
+  if [ $ret -eq 2 ]; then
     echo "$test [Failed support missing]"
-    perf record -e cycles $branch_flags -o "${perfdata}" perf test -w thloop || true
     if [ $err -eq 0 ]
     then
       err=2
     fi
     return
-  fi
-
-  out=$(perf report -D -i "${perfdata}" 2> /dev/null | grep -A1 'PERF_RECORD_SAMPLE')
-  sam_nr=$(echo "$out" | grep -c 'PERF_RECORD_SAMPLE' || true)
-  if [ $sam_nr -eq 0 ]
-  then
+  elif [ $ret -eq 1 ]; then
     echo "$test [Failed no samples captured]"
     err=1
     return
   fi
+
+  out=$(perf report -D -i "${perfdata}" 2> /dev/null | grep -A1 'PERF_RECORD_SAMPLE')
+  sam_nr=$(echo "$out" | grep -c 'PERF_RECORD_SAMPLE' || true)
   echo "$test: $sam_nr samples"
 
   bs_nr=$(echo "$out" | grep -c 'branch stack: nr:' || true)
diff --git a/tools/perf/tests/shell/record_offcpu.sh b/tools/perf/tests/shell/record_offcpu.sh
index 860a2d6f4b75f..ee095df47358b 100755
--- a/tools/perf/tests/shell/record_offcpu.sh
+++ b/tools/perf/tests/shell/record_offcpu.sh
@@ -45,7 +45,7 @@ test_offcpu_priv() {
 test_offcpu_basic() {
   echo "Basic off-cpu test"
 
-  if ! perf record --off-cpu -e dummy -o ${perfdata} sleep 1 2> /dev/null
+  if ! perf record --off-cpu -e dummy -o ${perfdata} sleep 0.3 2> /dev/null
   then
     echo "Basic off-cpu test [Failed record]"
     err=1
@@ -57,7 +57,7 @@ test_offcpu_basic() {
     err=1
     return
   fi
-  if ! perf report -i ${perfdata} -q --percent-limit=90 | grep -E -q sleep
+  if ! perf report -i ${perfdata} -q --percent-limit=70 | grep -E -q sleep
   then
     echo "Basic off-cpu test [Failed missing output]"
     err=1
@@ -98,8 +98,8 @@ test_offcpu_child() {
 test_offcpu_above_thresh() {
   echo "${test_above_thresh}"
 
-  # collect direct off-cpu samples for tasks blocked for more than 999ms
-  if ! perf record -e dummy --off-cpu --off-cpu-thresh 999 -o ${perfdata} -- sleep 1 2> /dev/null
+  # collect direct off-cpu samples for tasks blocked for more than 50ms
+  if ! perf record -e dummy --off-cpu --off-cpu-thresh 50 -o ${perfdata} -- sleep 0.1 2> /dev/null
   then
     echo "${test_above_thresh} [Failed record]"
     err=1
@@ -115,7 +115,7 @@ test_offcpu_above_thresh() {
   fi
   # there should only be one direct sample, and its period should be higher than off-cpu-thresh
   if ! perf script --time "0, ${dummy_timestamp}" -i ${perfdata} -F period | \
-       awk '{ if (int($1) > 999000000) exit 0; else exit 1; }'
+       awk '{ if (int($1) > 50000000) exit 0; else exit 1; }'
   then
     echo "${test_above_thresh} [Failed off-cpu time too short]"
     err=1
@@ -128,8 +128,8 @@ test_offcpu_above_thresh() {
 test_offcpu_below_thresh() {
   echo "${test_below_thresh}"
 
-  # collect direct off-cpu samples for tasks blocked for more than 1.2s
-  if ! perf record -e dummy --off-cpu --off-cpu-thresh 1200 -o ${perfdata} -- sleep 1 2> /dev/null
+  # collect direct off-cpu samples for tasks blocked for more than 500ms
+  if ! perf record -e dummy --off-cpu --off-cpu-thresh 500 -o ${perfdata} -- sleep 0.1 2> /dev/null
   then
     echo "${test_below_thresh} [Failed record]"
     err=1
diff --git a/tools/perf/tests/shell/stat.sh b/tools/perf/tests/shell/stat.sh
index 1e17bee026bd3..649de1166fede 100755
--- a/tools/perf/tests/shell/stat.sh
+++ b/tools/perf/tests/shell/stat.sh
@@ -535,6 +535,17 @@ test_stat_delay() {
   echo "stat -D test [Success]"
 }
 
+test_csv_json_fail() {
+  echo "stat -x <sep> -j test"
+  if perf stat -x , -j true > /dev/null 2>&1
+  then
+    echo "stat -x <sep> -j test [Failed - command should have errored]"
+    err=1
+  else
+    echo "stat -x <sep> -j test [Success]"
+  fi
+}
+
 test_default_stat
 test_null_stat
 test_offline_cpu_stat
@@ -551,6 +562,7 @@ test_stat_detailed
 test_stat_repeat
 test_stat_pid
 test_stat_delay
+test_csv_json_fail
 
 cleanup
 exit $err
diff --git a/tools/perf/tests/shell/stat_all_metrics.sh b/tools/perf/tests/shell/stat_all_metrics.sh
index b582d23f28c9e..df5fa27b21ae2 100755
--- a/tools/perf/tests/shell/stat_all_metrics.sh
+++ b/tools/perf/tests/shell/stat_all_metrics.sh
@@ -7,43 +7,70 @@ ParanoidAndNotRoot()
   [ "$(id -u)" != 0 ] && [ "$(cat /proc/sys/kernel/perf_event_paranoid)" -gt $1 ]
 }
 
-test_prog="sleep 0.01"
+test_prog="true"
 system_wide_flag="-a"
 if ParanoidAndNotRoot 0
 then
   system_wide_flag=""
-  test_prog="perf test -w noploop"
+  test_prog="perf test -w noploop 0.01"
 fi
 
+check_metric() {
+  local output="$1"
+  local status="$2"
+  local metric="$3"
+
+  if [[ $status -ne 0 || ! "$output" =~ ${metric:0:50} ]]; then
+    return 1
+  fi
+
+  if [[ "$output" =~ "<not counted>" || "$output" =~ "<not supported>" ]]; then
+    return 1
+  fi
+
+  return 0
+}
+
 skip=0
 err=3
 for m in $(perf list --raw-dump metrics); do
   echo "Testing $m"
   result=$(perf stat -M "$m" $system_wide_flag -- $test_prog 2>&1)
   result_err=$?
-  if [[ $result_err -eq 0 && "$result" =~ ${m:0:50} ]]
-  then
-    # No error result and metric shown.
+
+  if check_metric "$result" $result_err "$m"; then
     if [[ "$err" -ne 1 ]]
     then
       err=0
     fi
     continue
   fi
-  if [[ "$result" =~ "Cannot resolve IDs for" || "$result" =~ "No supported events found" ]]
+
+  if [[ "$result" =~ "Access to performance monitoring and observability operations is limited" || \
+        "$result" =~ "in per-thread mode, enable system wide" || \
+        "$result" =~ "<not supported>" || \
+        "$result" =~ "Cannot resolve IDs for" || \
+        "$result" =~ "No supported events found" || \
+        "$result" =~ "FP_ARITH" || \
+        "$result" =~ "AMX" || \
+        "$result" =~ "PMM" ]]
   then
-    if [[ $(perf list --raw-dump $m) == "Default"* ]]
-    then
-      echo "[Ignored $m] failed but as a Default metric this can be expected"
-      echo $result
+    true
+  else
+    result=$(perf stat -M "$m" $system_wide_flag -- perf test -w noploop 0.1 2>&1)
+    result_err=$?
+
+    if check_metric "$result" $result_err "$m"; then
+      if [[ "$err" -ne 1 ]]
+      then
+        err=0
+      fi
       continue
     fi
-    echo "[Failed $m] Metric contains missing events"
-    echo $result
-    err=1 # Fail
-    continue
-  elif [[ "$result" =~ \
-        "Access to performance monitoring and observability operations is limited" ]]
+  fi
+
+  # If retry also failed, determine if we skip, ignore, or fail
+  if [[ "$result" =~ "Access to performance monitoring and observability operations is limited" ]]
   then
     echo "[Skipped $m] Permission failure"
     echo $result
@@ -61,7 +88,9 @@ for m in $(perf list --raw-dump metrics); do
       skip=1
     fi
     continue
-  elif [[ "$result" =~ "<not supported>" ]]
+  elif [[ "$result" =~ "<not supported>" || \
+          "$result" =~ "Cannot resolve IDs for" || \
+          "$result" =~ "No supported events found" ]]
   then
     if [[ $(perf list --raw-dump $m) == "Default"* ]]
     then
@@ -105,19 +134,7 @@ for m in $(perf list --raw-dump metrics); do
     continue
   fi
 
-  # Failed, possibly the workload was too small so retry with something longer.
-  result=$(perf stat -M "$m" $system_wide_flag -- perf bench internals synthesize 2>&1)
-  result_err=$?
-  if [[ $result_err -eq 0 && "$result" =~ ${m:0:50} ]]
-  then
-    # No error result and metric shown.
-    if [[ "$err" -ne 1 ]]
-    then
-      err=0
-    fi
-    continue
-  fi
-  echo "[Failed $m] has non-zero error '$result_err' or not printed in:"
+  echo "[Failed $m] has non-zero error '$result_err' or not printed/counted in:"
   echo "$result"
   err=1
 done
diff --git a/tools/perf/tests/shell/stat_bpf_counters.sh b/tools/perf/tests/shell/stat_bpf_counters.sh
index 35463358b273c..11de77ee38ad4 100755
--- a/tools/perf/tests/shell/stat_bpf_counters.sh
+++ b/tools/perf/tests/shell/stat_bpf_counters.sh
@@ -4,21 +4,26 @@
 
 set -e
 
-workload="perf test -w sqrtloop"
+# Get the first allowed CPU
+CPU=$(taskset -c -p $$ | awk -F': ' '{print $2}' | awk -F'[,-]' '{print $1}')
+if [ -z "$CPU" ]; then
+	CPU=0
+fi
+workload=(taskset -c "$CPU" awk 'BEGIN { for (i=0; i<10000000; i++) sum+=i }')
 
-# check whether $2 is within +/- 20% of $1
+# check whether $2 is within +/- 15% of $1
 compare_number()
 {
 	first_num=$1
 	second_num=$2
 
-	# upper bound is first_num * 120%
-	upper=$(expr $first_num + $first_num / 5 )
-	# lower bound is first_num * 80%
-	lower=$(expr $first_num - $first_num / 5 )
+	# upper bound is first_num * 115%
+	upper=$(expr $first_num + $first_num / 20 \* 3 )
+	# lower bound is first_num * 85%
+	lower=$(expr $first_num - $first_num / 20 \* 3 )
 
 	if [ $second_num -gt $upper ] || [ $second_num -lt $lower ]; then
-		echo "The difference between $first_num and $second_num are greater than 20%."
+		echo "The difference between $first_num and $second_num are greater than 15%."
 		exit 1
 	fi
 }
@@ -41,11 +46,12 @@ check_counts()
 test_bpf_counters()
 {
 	printf "Testing --bpf-counters "
-	base_instructions=$(perf stat --no-big-num -e instructions -- $workload 2>&1 | \
+	base_instructions=$(perf stat --no-big-num -e instructions:u -- "${workload[@]}" 2>&1 | \
 				awk -v i=0 -v c=0 '/instructions/ { \
 					if ($1 != "<not") { i++; c += $1 } \
 				} END { if (i > 0) printf "%.0f", c; else print "<not" }')
-	bpf_instructions=$(perf stat --no-big-num --bpf-counters -e instructions -- $workload  2>&1 | \
+	bpf_instructions=$(perf stat --no-big-num --bpf-counters -e instructions:u \
+				-- "${workload[@]}"  2>&1 | \
 				awk -v i=0 -v c=0 '/instructions/ { \
 					if ($1 != "<not") { i++; c += $1 } \
 				} END { if (i > 0) printf "%.0f", c; else print "<not" }')
@@ -57,7 +63,9 @@ test_bpf_counters()
 test_bpf_modifier()
 {
 	printf "Testing bpf event modifier "
-	stat_output=$(perf stat --no-big-num -e instructions/name=base_instructions/,instructions/name=bpf_instructions/b -- $workload 2>&1)
+	stat_output=$(perf stat --no-big-num \
+		-e instructions/name=base_instructions/u,instructions/name=bpf_instructions/bu \
+		-- "${workload[@]}" 2>&1)
 	base_instructions=$(echo "$stat_output"| \
 				awk -v i=0 -v c=0 '/base_instructions/ { \
 					if ($1 != "<not") { i++; c += $1 } \
diff --git a/tools/perf/tests/shell/stat_metrics_values.sh b/tools/perf/tests/shell/stat_metrics_values.sh
index 30566f0b54279..86c5c7f709331 100755
--- a/tools/perf/tests/shell/stat_metrics_values.sh
+++ b/tools/perf/tests/shell/stat_metrics_values.sh
@@ -8,10 +8,17 @@ shelldir=$(dirname "$0")
 
 grep -q GenuineIntel /proc/cpuinfo || { echo Skipping non-Intel; exit 2; }
 
+# Skip if no permission to record system-wide events
+if ! perf stat -a -e instructions sleep 0.01 >/dev/null 2>&1; then
+  echo "Skipping: no permission to record system-wide events (-a)"
+  exit 2
+fi
+
+
 pythonvalidator=$(dirname $0)/lib/perf_metric_validation.py
 rulefile=$(dirname $0)/lib/perf_metric_validation_rules.json
 tmpdir=$(mktemp -d /tmp/__perf_test.program.XXXXX)
-workload="perf bench futex hash -r 2 -s"
+workload="perf bench futex hash -r 1 -s"
 
 # Add -debug, save data file and full rule file
 echo "Launch python validation script $pythonvalidator"
diff --git a/tools/perf/tests/shell/test_brstack.sh b/tools/perf/tests/shell/test_brstack.sh
index eb5837f82e390..71550e0b37baa 100755
--- a/tools/perf/tests/shell/test_brstack.sh
+++ b/tools/perf/tests/shell/test_brstack.sh
@@ -110,20 +110,29 @@ test_trap_eret_branches() {
 		return
 	fi
 	start_err=$err
-	err=0
-	perf record -o $TMPDIR/perf.data --branch-filter any,save_type,u,k -- \
-		perf test -w traploop 1000 > "$TMPDIR/record.txt" 2>&1
-	perf script -i $TMPDIR/perf.data --fields brstacksym | \
-		tr ' ' '\n' > $TMPDIR/perf.script
-
-	# BRBINF<n>.TYPE == TRAP are mapped to PERF_BR_IRQ by the BRBE driver
-	check_branches "^trap_bench\+[^ ]+/[^ ]/IRQ/"
-	check_branches "^[^ ]+/trap_bench\+[^ ]+/ERET/"
-	if [ $err -eq 0 ]; then
+	local ret=1
+	for loops in 1000 10000 100000; do
+		err=0
+		perf record -o $TMPDIR/perf.data --branch-filter any,save_type,u,k -- \
+			perf test -w traploop $loops > "$TMPDIR/record.txt" 2>&1
+		perf script -i $TMPDIR/perf.data --fields brstacksym | \
+			tr ' ' '\n' > $TMPDIR/perf.script
+
+		# BRBINF<n>.TYPE == TRAP are mapped to PERF_BR_IRQ by the BRBE driver
+		check_branches "^trap_bench\+[^ ]+/[^ ]/IRQ/"
+		check_branches "^[^ ]+/trap_bench\+[^ ]+/ERET/"
+		if [ $err -eq 0 ]; then
+			ret=0
+			break
+		fi
+	done
+
+	if [ $ret -eq 0 ]; then
 		echo "Testing trap & eret branches [Passed]"
 		err=$start_err
 	else
 		echo "Testing trap & eret branches [Failed]"
+		err=1
 	fi
 }
 
@@ -135,32 +144,40 @@ test_kernel_branches() {
 		return
 	fi
 	start_err=$err
-	err=0
-	perf record -o $TMPDIR/perf.data --branch-filter any,k -- \
-		perf bench syscall basic --loop 1000 > "$TMPDIR/record.txt" 2>&1
-	perf script -i $TMPDIR/perf.data --fields brstack | \
-		tr ' ' '\n' > $TMPDIR/perf.script
-
-	# Example of branch entries:
-	#       "0xffffffff93bda241/0xffffffff93bda20f/M/-/-/..."
-	# Source addresses come first in user or kernel code. Next is the target
-	# address that must be in the kernel.
-
-	# Look for source addresses with top bit set
-	if ! grep -q -E -m1 "^0x[89a-f][0-9a-f]{15}" $TMPDIR/perf.script; then
-		echo "Testing kernel branch sampling [Failed kernel branches missing]"
-		err=1
-	fi
-	# Look for no target addresses without top bit set
-	if grep -q -E -m1 "^0x[0-9a-f]{0,16}/0x[0-7][0-9a-f]{1,15}/" $TMPDIR/perf.script; then
-		echo "Testing kernel branch sampling [Failed user branches found]"
-		err=1
-	fi
-	if [ $err -eq 0 ]; then
+	local ret=1
+	for loops in 1000 10000 100000; do
+		err=0
+		perf record -o $TMPDIR/perf.data --branch-filter any,k -- \
+			perf bench syscall basic --loop $loops > "$TMPDIR/record.txt" 2>&1
+		perf script -i $TMPDIR/perf.data --fields brstack | \
+			tr ' ' '\n' > $TMPDIR/perf.script
+
+		# Example of branch entries:
+		#       "0xffffffff93bda241/0xffffffff93bda20f/M/-/-/..."
+		# Source addresses come first in user or kernel code. Next is the target
+		# address that must be in the kernel.
+
+		# Look for source addresses with top bit set
+		if ! grep -q -E -m1 "^0x[89a-f][0-9a-f]{15}" $TMPDIR/perf.script; then
+			err=1
+		fi
+		# Look for no target addresses without top bit set
+		if grep -q -E -m1 "^0x[0-9a-f]{0,16}/0x[0-7][0-9a-f]{1,15}/" \
+			$TMPDIR/perf.script; then
+			err=1
+		fi
+		if [ $err -eq 0 ]; then
+			ret=0
+			break
+		fi
+	done
+
+	if [ $ret -eq 0 ]; then
 		echo "Testing kernel branch sampling [Passed]"
 		err=$start_err
 	else
 		echo "Testing kernel branch sampling [Failed]"
+		err=1
 	fi
 }
 
@@ -206,20 +223,28 @@ test_syscall() {
 		return
 	fi
 	start_err=$err
-	err=0
-	perf record -o $TMPDIR/perf.data --branch-filter \
-		any_call,save_type,u,k -c 10007 -- \
-		perf bench syscall basic --loop 8000  > "$TMPDIR/record.txt" 2>&1
-	perf script -i $TMPDIR/perf.data --fields brstacksym | \
-		tr ' ' '\n' > $TMPDIR/perf.script
-
-	check_branches "getppid[^ ]*/SYSCALL/"
+	local ret=1
+	for loops in 8000 30000 100000; do
+		err=0
+		perf record -o $TMPDIR/perf.data --branch-filter \
+			any_call,save_type,u,k -c 10007 -- \
+			perf bench syscall basic --loop $loops  > "$TMPDIR/record.txt" 2>&1
+		perf script -i $TMPDIR/perf.data --fields brstacksym | \
+			tr ' ' '\n' > $TMPDIR/perf.script
+
+		check_branches "getppid[^ ]*/SYSCALL/"
+		if [ $err -eq 0 ]; then
+			ret=0
+			break
+		fi
+	done
 
-	if [ $err -eq 0 ]; then
+	if [ $ret -eq 0 ]; then
 		echo "Testing syscalls [Passed]"
 		err=$start_err
 	else
 		echo "Testing syscalls [Failed]"
+		err=1
 	fi
 }
 set -e
diff --git a/tools/perf/tests/shell/timechart.sh b/tools/perf/tests/shell/timechart.sh
index b14b3472c2846..40072b32171a9 100755
--- a/tools/perf/tests/shell/timechart.sh
+++ b/tools/perf/tests/shell/timechart.sh
@@ -9,7 +9,7 @@ perfdata=$(mktemp /tmp/__perf_timechart_test.perf.data.XXXXX)
 output=$(mktemp /tmp/__perf_timechart_test.output.XXXXX.svg)
 
 cleanup() {
-	rm -f "${perfdata}"
+	rm -f "${perfdata}"*
 	rm -f "${output}"
 	trap - EXIT TERM INT
 }
@@ -22,38 +22,41 @@ trap_cleanup() {
 trap trap_cleanup EXIT TERM INT
 
 test_timechart() {
-	echo "Basic perf timechart test"
+	NAME=$1
+	OPTION=$2
+
+	echo "perf timechart ${NAME} test"
 
 	# Try to record timechart data.
 	# perf timechart record uses system-wide recording and specific tracepoints.
 	# If it fails (e.g. permissions, missing tracepoints), skip the test.
-	if ! perf timechart record -o "${perfdata}" true > /dev/null 2>&1; then
-		echo "Basic perf timechart test [Skipped: perf timechart record failed (permissions/events?)]"
+	if ! perf timechart record -o "${perfdata}" ${OPTION} true > /dev/null 2>&1; then
+		echo "perf timechart ${NAME} test [Skipped: perf timechart record failed (permissions/events?)]"
 		return
 	fi
 
 	# Generate the timechart
 	if ! perf timechart -i "${perfdata}" -o "${output}" > /dev/null 2>&1; then
-		echo "Basic perf timechart test [Failed: perf timechart command failed]"
+		echo "perf timechart ${NAME} test [Failed: perf timechart command failed]"
 		err=1
 		return
 	fi
 
 	# Check if output file exists and is not empty
 	if [ ! -s "${output}" ]; then
-		echo "Basic perf timechart test [Failed: output file is empty or missing]"
+		echo "perf timechart ${NAME} test [Failed: output file is empty or missing]"
 		err=1
 		return
 	fi
 
 	# Check if it looks like an SVG
 	if ! grep -q "svg" "${output}"; then
-		echo "Basic perf timechart test [Failed: output doesn't look like SVG]"
+		echo "perf timechart ${NAME} test [Failed: output doesn't look like SVG]"
 		err=1
 		return
 	fi
 
-	echo "Basic perf timechart test [Success]"
+	echo "perf timechart ${NAME} test [Success]"
 }
 
 if ! perf check feature -q libtraceevent ; then
@@ -62,6 +65,9 @@ if ! perf check feature -q libtraceevent ; then
 	exit 2
 fi
 
-test_timechart
+test_timechart "Basic" ""
+test_timechart "IO-only" "-I"
+test_timechart "Backtrace" "-g"
+
 cleanup
 exit $err
diff --git a/tools/perf/tests/shell/trace_record_replay.sh b/tools/perf/tests/shell/trace_record_replay.sh
index 88d30a03dcecb..38fcafcdfb91c 100755
--- a/tools/perf/tests/shell/trace_record_replay.sh
+++ b/tools/perf/tests/shell/trace_record_replay.sh
@@ -6,16 +6,46 @@
 
 # shellcheck source=lib/probe.sh
 . "$(dirname $0)"/lib/probe.sh
+# shellcheck source=lib/perf_record.sh
+. "$(dirname $0)"/lib/perf_record.sh
 
 skip_if_no_perf_trace || exit 2
 [ "$(id -u)" = 0 ] || exit 2
 
 file=$(mktemp /tmp/temporary_file.XXXXX)
+err=0
 
-perf trace record -o ${file} sleep 1 || exit 1
-if ! perf trace -i ${file} 2>&1 | grep nanosleep; then
-	echo "Failed: cannot find *nanosleep syscall"
+cleanup() {
+	rm -f ${file}
+	perf_record_cleanup
+	trap - EXIT INT TERM
+}
+
+trap_cleanup() {
+	echo "Unexpected signal in ${FUNCNAME[1]}"
+	cleanup
+	exit 1
+}
+trap trap_cleanup EXIT INT TERM
+
+check_nanosleep() {
+  perf trace -i "${file}" 2>&1 | grep -q nanosleep
+}
+
+PERF_RECORD_CMD="perf trace record" perf_record_with_retry "${file}" "check_nanosleep" "sleep"
+err=$?
+
+if [ $err -ne 0 ]; then
+	if [ $err -eq 2 ]; then
+		logfile="${PERF_RECORD_LOGS[${#PERF_RECORD_LOGS[@]}-1]}"
+		echo "perf trace record failed. Log output:"
+		cat "$logfile"
+	else
+		echo "Failed: cannot find *nanosleep syscall"
+	fi
+	cleanup
 	exit 1
 fi
 
-rm -f ${file}
+cleanup
+exit 0
diff --git a/tools/perf/tests/sw-clock.c b/tools/perf/tests/sw-clock.c
index b6e46975379cd..d181858816352 100644
--- a/tools/perf/tests/sw-clock.c
+++ b/tools/perf/tests/sw-clock.c
@@ -59,7 +59,7 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id)
 	evsel = evsel__new(&attr);
 	if (evsel == NULL) {
 		pr_debug("evsel__new\n");
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 	evlist__add(evlist, evsel);
 
@@ -68,10 +68,10 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id)
 	if (!cpus || !threads) {
 		err = -ENOMEM;
 		pr_debug("Not enough memory to create thread/cpu maps\n");
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
-	perf_evlist__set_maps(&evlist->core, cpus, threads);
+	perf_evlist__set_maps(evlist__core(evlist), cpus, threads);
 
 	if (evlist__open(evlist)) {
 		const char *knob = "/proc/sys/kernel/perf_event_max_sample_rate";
@@ -80,14 +80,14 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id)
 		pr_debug("Couldn't open evlist: %s\nHint: check %s, using %" PRIu64 " in this test.\n",
 			 str_error_r(errno, sbuf, sizeof(sbuf)),
 			 knob, (u64)attr.sample_freq);
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
-	err = evlist__mmap(evlist, 128);
+	err = evlist__do_mmap(evlist, 128);
 	if (err < 0) {
 		pr_debug("failed to mmap event: %d (%s)\n", errno,
 			 str_error_r(errno, sbuf, sizeof(sbuf)));
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	evlist__enable(evlist);
@@ -98,7 +98,7 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id)
 
 	evlist__disable(evlist);
 
-	md = &evlist->mmap[0];
+	md = &evlist__mmap(evlist)[0];
 	if (perf_mmap__read_init(&md->core) < 0)
 		goto out_init;
 
@@ -113,7 +113,7 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id)
 		if (err < 0) {
 			pr_debug("Error during parse sample\n");
 			perf_sample__exit(&sample);
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 
 		total_periods += sample.period;
@@ -131,10 +131,10 @@ static int __test__sw_clock_freq(enum perf_sw_ids clock_id)
 		err = -1;
 	}
 
-out_delete_evlist:
+out_put_evlist:
 	perf_cpu_map__put(cpus);
 	perf_thread_map__put(threads);
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	return err;
 }
 
diff --git a/tools/perf/tests/switch-tracking.c b/tools/perf/tests/switch-tracking.c
index e32331fee2778..73568c782d72b 100644
--- a/tools/perf/tests/switch-tracking.c
+++ b/tools/perf/tests/switch-tracking.c
@@ -237,6 +237,7 @@ static int add_event(struct evlist *evlist, struct list_head *events,
 
 	if (evlist__parse_sample(evlist, event, &sample)) {
 		pr_debug("evlist__parse_sample failed\n");
+		perf_sample__exit(&sample);
 		return -1;
 	}
 
@@ -282,8 +283,8 @@ static int process_events(struct evlist *evlist,
 	struct mmap *md;
 	int i, ret;
 
-	for (i = 0; i < evlist->core.nr_mmaps; i++) {
-		md = &evlist->mmap[i];
+	for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) {
+		md = &evlist__mmap(evlist)[i];
 		if (perf_mmap__read_init(&md->core) < 0)
 			continue;
 
@@ -374,7 +375,7 @@ static int test__switch_tracking(struct test_suite *test __maybe_unused, int sub
 		goto out_err;
 	}
 
-	perf_evlist__set_maps(&evlist->core, cpus, threads);
+	perf_evlist__set_maps(evlist__core(evlist), cpus, threads);
 
 	/* First event */
 	err = parse_event(evlist, "cpu-clock:u");
@@ -471,7 +472,7 @@ static int test__switch_tracking(struct test_suite *test __maybe_unused, int sub
 		goto out;
 	}
 
-	err = evlist__mmap(evlist, UINT_MAX);
+	err = evlist__do_mmap(evlist, UINT_MAX);
 	if (err) {
 		pr_debug("evlist__mmap failed!\n");
 		goto out_err;
@@ -582,7 +583,7 @@ static int test__switch_tracking(struct test_suite *test __maybe_unused, int sub
 out:
 	if (evlist) {
 		evlist__disable(evlist);
-		evlist__delete(evlist);
+		evlist__put(evlist);
 	}
 	perf_cpu_map__put(cpus);
 	perf_thread_map__put(threads);
diff --git a/tools/perf/tests/task-exit.c b/tools/perf/tests/task-exit.c
index 4053ff2813bb7..95393edbfe363 100644
--- a/tools/perf/tests/task-exit.c
+++ b/tools/perf/tests/task-exit.c
@@ -74,15 +74,15 @@ static int test__task_exit(struct test_suite *test __maybe_unused, int subtest _
 	if (!cpus || !threads) {
 		err = -ENOMEM;
 		pr_debug("Not enough memory to create thread/cpu maps\n");
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
-	perf_evlist__set_maps(&evlist->core, cpus, threads);
+	perf_evlist__set_maps(evlist__core(evlist), cpus, threads);
 
 	err = evlist__prepare_workload(evlist, &target, argv, false, workload_exec_failed_signal);
 	if (err < 0) {
 		pr_debug("Couldn't run the workload!\n");
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	evsel = evlist__first(evlist);
@@ -101,20 +101,20 @@ static int test__task_exit(struct test_suite *test __maybe_unused, int subtest _
 	if (err < 0) {
 		pr_debug("Couldn't open the evlist: %s\n",
 			 str_error_r(-err, sbuf, sizeof(sbuf)));
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
-	if (evlist__mmap(evlist, 128) < 0) {
+	if (evlist__do_mmap(evlist, 128) < 0) {
 		pr_debug("failed to mmap events: %d (%s)\n", errno,
 			 str_error_r(errno, sbuf, sizeof(sbuf)));
 		err = -1;
-		goto out_delete_evlist;
+		goto out_put_evlist;
 	}
 
 	evlist__start_workload(evlist);
 
 retry:
-	md = &evlist->mmap[0];
+	md = &evlist__mmap(evlist)[0];
 	if (perf_mmap__read_init(&md->core) < 0)
 		goto out_init;
 
@@ -133,7 +133,7 @@ static int test__task_exit(struct test_suite *test __maybe_unused, int subtest _
 		if (retry_count++ > 1000) {
 			pr_debug("Failed after retrying 1000 times\n");
 			err = -1;
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 
 		goto retry;
@@ -144,10 +144,10 @@ static int test__task_exit(struct test_suite *test __maybe_unused, int subtest _
 		err = -1;
 	}
 
-out_delete_evlist:
+out_put_evlist:
 	perf_cpu_map__put(cpus);
 	perf_thread_map__put(threads);
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	return err;
 }
 
diff --git a/tools/perf/tests/tests.h b/tools/perf/tests/tests.h
index 7cedf05be544a..cee9e6b62dcc8 100644
--- a/tools/perf/tests/tests.h
+++ b/tools/perf/tests/tests.h
@@ -248,6 +248,7 @@ DECLARE_WORKLOAD(inlineloop);
 DECLARE_WORKLOAD(jitdump);
 DECLARE_WORKLOAD(context_switch_loop);
 DECLARE_WORKLOAD(deterministic);
+DECLARE_WORKLOAD(callchain);
 
 #ifdef HAVE_RUST_SUPPORT
 DECLARE_WORKLOAD(code_with_type);
diff --git a/tools/perf/tests/time-utils-test.c b/tools/perf/tests/time-utils-test.c
index 38df10373c1e1..90a9a4b4f1789 100644
--- a/tools/perf/tests/time-utils-test.c
+++ b/tools/perf/tests/time-utils-test.c
@@ -69,16 +69,19 @@ struct test_data {
 
 static bool test__perf_time__parse_for_ranges(struct test_data *d)
 {
-	struct evlist evlist = {
-		.first_sample_time = d->first,
-		.last_sample_time = d->last,
-	};
-	struct perf_session session = { .evlist = &evlist };
+	struct evlist *evlist = evlist__new();
+	struct perf_session session = { .evlist = evlist };
 	struct perf_time_interval *ptime = NULL;
 	int range_size, range_num;
 	bool pass = false;
 	int i, err;
 
+	if (!evlist) {
+		pr_debug("Missing evlist\n");
+		return false;
+	}
+	evlist__set_first_sample_time(evlist, d->first);
+	evlist__set_last_sample_time(evlist, d->last);
 	pr_debug("\nperf_time__parse_for_ranges(\"%s\")\n", d->str);
 
 	if (strchr(d->str, '%'))
@@ -127,6 +130,7 @@ static bool test__perf_time__parse_for_ranges(struct test_data *d)
 
 	pass = true;
 out:
+	evlist__put(evlist);
 	free(ptime);
 	return pass;
 }
diff --git a/tools/perf/tests/tool_pmu.c b/tools/perf/tests/tool_pmu.c
index 1e900ef92e378..c6c5ebf0e935f 100644
--- a/tools/perf/tests/tool_pmu.c
+++ b/tools/perf/tests/tool_pmu.c
@@ -40,9 +40,10 @@ static int do_test(enum tool_pmu_event ev, bool with_pmu)
 	}
 
 	ret = TEST_OK;
-	if (with_pmu ? (evlist->core.nr_entries != 1) : (evlist->core.nr_entries < 1)) {
+	if (with_pmu ? (evlist__nr_entries(evlist) != 1)
+		     : (evlist__nr_entries(evlist) < 1)) {
 		pr_debug("FAILED %s:%d Unexpected number of events for '%s' of %d\n",
-			 __FILE__, __LINE__, str, evlist->core.nr_entries);
+			 __FILE__, __LINE__, str, evlist__nr_entries(evlist));
 		ret = TEST_FAIL;
 		goto out;
 	}
@@ -67,7 +68,7 @@ static int do_test(enum tool_pmu_event ev, bool with_pmu)
 
 out:
 	parse_events_error__exit(&err);
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	return ret;
 }
 
diff --git a/tools/perf/tests/topology.c b/tools/perf/tests/topology.c
index bd7b859dea66a..77cb8318c0b18 100644
--- a/tools/perf/tests/topology.c
+++ b/tools/perf/tests/topology.c
@@ -46,7 +46,7 @@ static int session_write_header(char *path)
 
 	session->evlist = evlist__new_default(&target, /*sample_callchains=*/false);
 	TEST_ASSERT_VAL("can't get evlist", session->evlist);
-	session->evlist->session = session;
+	evlist__set_session(session->evlist, session);
 
 	perf_header__set_feat(&session->header, HEADER_CPU_TOPOLOGY);
 	perf_header__set_feat(&session->header, HEADER_NRCPUS);
@@ -58,7 +58,7 @@ static int session_write_header(char *path)
 			!perf_session__write_header(session, session->evlist,
 						    perf_data__fd(&data), true));
 
-	evlist__delete(session->evlist);
+	evlist__put(session->evlist);
 	perf_session__delete(session);
 
 	return 0;
diff --git a/tools/perf/tests/uncore-event-sorting.c b/tools/perf/tests/uncore-event-sorting.c
index 7d2fc304e21fe..7756777c54c2e 100644
--- a/tools/perf/tests/uncore-event-sorting.c
+++ b/tools/perf/tests/uncore-event-sorting.c
@@ -147,8 +147,8 @@ static int test__uncore_event_sorting(struct test_suite *test __maybe_unused,
 		goto out_err;
 	}
 
-	CHECK_COND(evlist->core.nr_entries >= 4, "Number of events is >= 4");
-	CHECK_EQUAL(evlist->core.nr_entries % 2, 0, "Number of events is a multiple of 2");
+	CHECK_COND(evlist__nr_entries(evlist) >= 4, "Number of events is >= 4");
+	CHECK_EQUAL(evlist__nr_entries(evlist) % 2, 0, "Number of events is a multiple of 2");
 
 	evlist__for_each_entry(evlist, evsel) {
 		struct evsel *next;
@@ -165,7 +165,7 @@ static int test__uncore_event_sorting(struct test_suite *test __maybe_unused,
 	ret = TEST_OK;
 
 out_err:
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	parse_events_error__exit(&err);
 	zfree(&pmu_prefix);
 	zfree(&m.event1);
diff --git a/tools/perf/tests/workloads/Build b/tools/perf/tests/workloads/Build
index 7bb4b9829ba24..048e371eb63e3 100644
--- a/tools/perf/tests/workloads/Build
+++ b/tools/perf/tests/workloads/Build
@@ -13,6 +13,7 @@ perf-test-y += inlineloop.o
 perf-test-y += jitdump.o
 perf-test-y += context_switch_loop.o
 perf-test-y += deterministic.o
+perf-test-y += callchain.o
 
 ifeq ($(CONFIG_RUST_SUPPORT),y)
     perf-test-y += code_with_type.o
@@ -27,3 +28,4 @@ CFLAGS_traploop.o         = -g -O0 -fno-inline -U_FORTIFY_SOURCE
 CFLAGS_inlineloop.o       = -g -O2
 CFLAGS_deterministic.o    = -g -O0 -fno-inline -U_FORTIFY_SOURCE
 CFLAGS_named_threads.o    = -g -O0 -fno-inline -U_FORTIFY_SOURCE
+CFLAGS_callchain.o        = -g -O0 -fno-inline -U_FORTIFY_SOURCE
diff --git a/tools/perf/tests/workloads/callchain.c b/tools/perf/tests/workloads/callchain.c
new file mode 100644
index 0000000000000..abbb406ba90b3
--- /dev/null
+++ b/tools/perf/tests/workloads/callchain.c
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/compiler.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#include "../tests.h"
+
+/*
+ * Mark as noinline to establish the call chain, and avoid the static
+ * annotation to prevent LTO from renaming the functions.
+ */
+noinline void callchain_do_syscall(void);
+noinline void callchain_foo(void);
+noinline int callchain(int argc, const char **argv);
+
+noinline void callchain_do_syscall(void)
+{
+	syscall(SYS_gettid);
+}
+
+noinline void callchain_foo(void)
+{
+	callchain_do_syscall();
+}
+
+noinline int callchain(int argc __maybe_unused,
+		       const char **argv __maybe_unused)
+{
+	callchain_foo();
+
+	return 0;
+}
+
+DEFINE_WORKLOAD(callchain);
diff --git a/tools/perf/tests/workloads/noploop.c b/tools/perf/tests/workloads/noploop.c
index 656e472e61882..ca9f871e136f5 100644
--- a/tools/perf/tests/workloads/noploop.c
+++ b/tools/perf/tests/workloads/noploop.c
@@ -15,15 +15,26 @@ static void sighandler(int sig __maybe_unused)
 
 static int noploop(int argc, const char **argv)
 {
-	int sec = 1;
+	double sec = 1.0;
 
 	pthread_setname_np(pthread_self(), "perf-noploop");
 	if (argc > 0)
-		sec = atoi(argv[0]);
+		sec = atof(argv[0]);
+
+	if (!(sec > 0.0)) {
+		fprintf(stderr, "Error: seconds (%f) must be > 0\n", sec);
+		return 1;
+	}
 
 	signal(SIGINT, sighandler);
 	signal(SIGALRM, sighandler);
-	alarm(sec);
+
+	if (sec < 1.0) {
+		useconds_t usecs = (useconds_t)(sec * 1000000.0);
+
+		ualarm(usecs > 0 ? usecs : 1, 0);
+	} else
+		alarm((unsigned int)sec);
 
 	while (!done)
 		continue;
diff --git a/tools/perf/tests/workloads/thloop.c b/tools/perf/tests/workloads/thloop.c
index bd8168f883fb6..c830d739489fe 100644
--- a/tools/perf/tests/workloads/thloop.c
+++ b/tools/perf/tests/workloads/thloop.c
@@ -31,14 +31,15 @@ static void *thfunc(void *arg)
 
 static int thloop(int argc, const char **argv)
 {
-	int nt = 2, sec = 1, err = 1;
+	int nt = 2, err = 1;
+	double sec = 1.0;
 	pthread_t *thread_list = NULL;
 
 	if (argc > 0)
-		sec = atoi(argv[0]);
+		sec = atof(argv[0]);
 
-	if (sec <= 0) {
-		fprintf(stderr, "Error: seconds (%d) must be >= 1\n", sec);
+	if (!(sec > 0.0)) {
+		fprintf(stderr, "Error: seconds (%f) must be > 0\n", sec);
 		return 1;
 	}
 
@@ -67,7 +68,12 @@ static int thloop(int argc, const char **argv)
 			goto out;
 		}
 	}
-	alarm(sec);
+	if (sec < 1.0) {
+		useconds_t usecs = (useconds_t)(sec * 1000000.0);
+
+		ualarm(usecs > 0 ? usecs : 1, 0);
+	} else
+		alarm((unsigned int)sec);
 	test_loop();
 	err = 0;
 out:
diff --git a/tools/perf/ui/browsers/annotate.c b/tools/perf/ui/browsers/annotate.c
index 97ae4c86bebbe..d25761a8d25eb 100644
--- a/tools/perf/ui/browsers/annotate.c
+++ b/tools/perf/ui/browsers/annotate.c
@@ -597,7 +597,7 @@ static bool annotate_browser__callq(struct annotate_browser *browser,
 	notes = symbol__annotation(dl->ops.target.sym);
 	annotation__lock(notes);
 
-	if (!symbol__hists(dl->ops.target.sym, evsel->evlist->core.nr_entries)) {
+	if (!symbol__hists(dl->ops.target.sym, evlist__nr_entries(evsel->evlist))) {
 		annotation__unlock(notes);
 		ui__warning("Not enough memory for annotating '%s' symbol!\n",
 			    dl->ops.target.sym->name);
diff --git a/tools/perf/ui/browsers/hists.c b/tools/perf/ui/browsers/hists.c
index cfa6386e6e1da..da7cc195b9f41 100644
--- a/tools/perf/ui/browsers/hists.c
+++ b/tools/perf/ui/browsers/hists.c
@@ -688,10 +688,10 @@ static int hist_browser__handle_hotkey(struct hist_browser *browser, bool warn_l
 		ui_browser__update_nr_entries(&browser->b, nr_entries);
 
 		if (warn_lost_event &&
-		    (evsel->evlist->stats.nr_lost_warned !=
-		     evsel->evlist->stats.nr_events[PERF_RECORD_LOST])) {
-			evsel->evlist->stats.nr_lost_warned =
-				evsel->evlist->stats.nr_events[PERF_RECORD_LOST];
+		    (evlist__stats(evsel->evlist)->nr_lost_warned !=
+		     evlist__stats(evsel->evlist)->nr_events[PERF_RECORD_LOST])) {
+			evlist__stats(evsel->evlist)->nr_lost_warned =
+				evlist__stats(evsel->evlist)->nr_events[PERF_RECORD_LOST];
 			ui_browser__warn_lost_events(&browser->b);
 		}
 
@@ -3321,7 +3321,7 @@ static int evsel__hists_browse(struct evsel *evsel, int nr_events, const char *h
 				 * No need to refresh, resort/decay histogram
 				 * entries if we are not collecting samples:
 				 */
-				if (top->evlist->enabled) {
+				if (evlist__enabled(top->evlist)) {
 					helpline = "Press 'f' to disable the events or 'h' to see other hotkeys";
 					hbt->refresh = delay_secs;
 				} else {
@@ -3493,7 +3493,7 @@ static void perf_evsel_menu__write(struct ui_browser *browser,
 			   unit, unit == ' ' ? "" : " ", ev_name);
 	ui_browser__printf(browser, "%s", bf);
 
-	nr_events = evsel->evlist->stats.nr_events[PERF_RECORD_LOST];
+	nr_events = evlist__stats(evsel->evlist)->nr_events[PERF_RECORD_LOST];
 	if (nr_events != 0) {
 		menu->lost_events = true;
 		if (!current_entry)
@@ -3559,13 +3559,13 @@ static int perf_evsel_menu__run(struct evsel_menu *menu,
 			ui_browser__show_title(&menu->b, title);
 			switch (key) {
 			case K_TAB:
-				if (pos->core.node.next == &evlist->core.entries)
+				if (pos->core.node.next == &evlist__core(evlist)->entries)
 					pos = evlist__first(evlist);
 				else
 					pos = evsel__next(pos);
 				goto browse_hists;
 			case K_UNTAB:
-				if (pos->core.node.prev == &evlist->core.entries)
+				if (pos->core.node.prev == &evlist__core(evlist)->entries)
 					pos = evlist__last(evlist);
 				else
 					pos = evsel__prev(pos);
@@ -3618,7 +3618,7 @@ static int __evlist__tui_browse_hists(struct evlist *evlist, int nr_entries, con
 	struct evsel *pos;
 	struct evsel_menu menu = {
 		.b = {
-			.entries    = &evlist->core.entries,
+			.entries    = &evlist__core(evlist)->entries,
 			.refresh    = ui_browser__list_head_refresh,
 			.seek	    = ui_browser__list_head_seek,
 			.write	    = perf_evsel_menu__write,
@@ -3646,7 +3646,7 @@ static int __evlist__tui_browse_hists(struct evlist *evlist, int nr_entries, con
 
 static bool evlist__single_entry(struct evlist *evlist)
 {
-	int nr_entries = evlist->core.nr_entries;
+	int nr_entries = evlist__nr_entries(evlist);
 
 	if (nr_entries == 1)
 	       return true;
@@ -3664,7 +3664,7 @@ static bool evlist__single_entry(struct evlist *evlist)
 int evlist__tui_browse_hists(struct evlist *evlist, const char *help, struct hist_browser_timer *hbt,
 			     float min_pcnt, struct perf_env *env, bool warn_lost_event)
 {
-	int nr_entries = evlist->core.nr_entries;
+	int nr_entries = evlist__nr_entries(evlist);
 
 	if (evlist__single_entry(evlist)) {
 single_entry: {
diff --git a/tools/perf/util/Build b/tools/perf/util/Build
index 5e2265018826f..330311cac5505 100644
--- a/tools/perf/util/Build
+++ b/tools/perf/util/Build
@@ -76,7 +76,6 @@ perf-util-y += sample.o
 perf-util-y += sample-raw.o
 perf-util-y += s390-sample-raw.o
 perf-util-y += amd-sample-raw.o
-
 perf-util-y += ordered-events.o
 perf-util-y += namespaces.o
 perf-util-y += comm.o
diff --git a/tools/perf/util/amd-sample-raw.c b/tools/perf/util/amd-sample-raw.c
index 394c061fbeb38..cda3836329c3a 100644
--- a/tools/perf/util/amd-sample-raw.c
+++ b/tools/perf/util/amd-sample-raw.c
@@ -421,7 +421,7 @@ static void parse_cpuid(struct perf_env *env)
  */
 bool evlist__has_amd_ibs(struct evlist *evlist)
 {
-	struct perf_env *env = perf_session__env(evlist->session);
+	struct perf_env *env = perf_session__env(evlist__session(evlist));
 	int ret, nr_pmu_mappings = perf_env__nr_pmu_mappings(env);
 	const char *pmu_mapping = perf_env__pmu_mappings(env);
 	char name[sizeof("ibs_fetch")];
diff --git a/tools/perf/util/annotate-data.c b/tools/perf/util/annotate-data.c
index 63e3c54fab421..4e4c587640823 100644
--- a/tools/perf/util/annotate-data.c
+++ b/tools/perf/util/annotate-data.c
@@ -1829,7 +1829,7 @@ int annotated_data_type__update_samples(struct annotated_data_type *adt,
 		return 0;
 
 	if (adt->histograms == NULL) {
-		int nr = evsel->evlist->core.nr_entries;
+		int nr = evlist__nr_entries(evsel->evlist);
 
 		if (alloc_data_type_histograms(adt, nr) < 0)
 			return -1;
diff --git a/tools/perf/util/annotate.c b/tools/perf/util/annotate.c
index 02505222d8c2d..53b2a224b21df 100644
--- a/tools/perf/util/annotate.c
+++ b/tools/perf/util/annotate.c
@@ -328,7 +328,7 @@ static int symbol__inc_addr_samples(struct map_symbol *ms,
 
 	if (sym == NULL)
 		return 0;
-	src = symbol__hists(sym, sample->evsel->evlist->core.nr_entries);
+	src = symbol__hists(sym, evlist__nr_entries(sample->evsel->evlist));
 	return src ? __symbol__inc_addr_samples(ms, src, addr, sample) : 0;
 }
 
@@ -339,7 +339,7 @@ static int symbol__account_br_cntr(struct annotated_branch *branch,
 {
 	unsigned int br_cntr_nr = evsel__leader(evsel)->br_cntr_nr;
 	unsigned int base = evsel__leader(evsel)->br_cntr_idx;
-	unsigned int off = offset * evsel->evlist->nr_br_cntr;
+	unsigned int off = offset * evlist__nr_br_cntr(evsel->evlist);
 	u64 *branch_br_cntr = branch->br_cntr;
 	unsigned int i, mask, width;
 
@@ -369,7 +369,7 @@ static int symbol__account_cycles(u64 addr, u64 start, struct symbol *sym,
 
 	if (sym == NULL)
 		return 0;
-	branch = symbol__find_branch_hist(sym, evsel->evlist->nr_br_cntr);
+	branch = symbol__find_branch_hist(sym, evlist__nr_br_cntr(evsel->evlist));
 	if (!branch)
 		return -ENOMEM;
 	if (addr < sym->start || addr >= sym->end)
@@ -511,7 +511,7 @@ static void annotation__count_and_fill(struct annotation *notes, u64 start, u64
 static int annotation__compute_ipc(struct annotation *notes, size_t size,
 				   struct evsel *evsel)
 {
-	unsigned int br_cntr_nr = evsel->evlist->nr_br_cntr;
+	unsigned int br_cntr_nr = evlist__nr_br_cntr(evsel->evlist);
 	int err = 0;
 	s64 offset;
 
@@ -1813,7 +1813,7 @@ int annotation_br_cntr_abbr_list(char **str, struct evsel *evsel, bool header)
 	struct evsel *pos;
 	struct strbuf sb;
 
-	if (evsel->evlist->nr_br_cntr <= 0)
+	if (evlist__nr_br_cntr(evsel->evlist) <= 0)
 		return -ENOTSUP;
 
 	strbuf_init(&sb, /*hint=*/ 0);
diff --git a/tools/perf/util/aslr.c b/tools/perf/util/aslr.c
index 6a7542e7db827..027695d967798 100644
--- a/tools/perf/util/aslr.c
+++ b/tools/perf/util/aslr.c
@@ -13,6 +13,7 @@
 #include "session.h"
 #include "data.h"
 #include "dso.h"
+#include "pmu.h"
 #include "pmus.h"
 
 #include <internal/lib.h>  /* page_size */
diff --git a/tools/perf/util/auxtrace.c b/tools/perf/util/auxtrace.c
index 4cd2caf540152..0b851f32e98c8 100644
--- a/tools/perf/util/auxtrace.c
+++ b/tools/perf/util/auxtrace.c
@@ -191,7 +191,7 @@ void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp,
 				   struct evlist *evlist,
 				   struct evsel *evsel, int idx)
 {
-	bool per_cpu = !perf_cpu_map__has_any_cpu(evlist->core.user_requested_cpus);
+	bool per_cpu = !perf_cpu_map__has_any_cpu(evlist__core(evlist)->user_requested_cpus);
 
 	mp->mmap_needed = evsel->needs_auxtrace_mmap;
 
@@ -201,11 +201,11 @@ void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp,
 	mp->idx = idx;
 
 	if (per_cpu) {
-		mp->cpu = perf_cpu_map__cpu(evlist->core.all_cpus, idx);
-		mp->tid = perf_thread_map__pid(evlist->core.threads, 0);
+		mp->cpu = perf_cpu_map__cpu(evlist__core(evlist)->all_cpus, idx);
+		mp->tid = perf_thread_map__pid(evlist__core(evlist)->threads, 0);
 	} else {
 		mp->cpu.cpu = -1;
-		mp->tid = perf_thread_map__pid(evlist->core.threads, idx);
+		mp->tid = perf_thread_map__pid(evlist__core(evlist)->threads, idx);
 	}
 }
 
@@ -668,10 +668,10 @@ int auxtrace_parse_snapshot_options(struct auxtrace_record *itr,
 
 static int evlist__enable_event_idx(struct evlist *evlist, struct evsel *evsel, int idx)
 {
-	bool per_cpu_mmaps = !perf_cpu_map__has_any_cpu(evlist->core.user_requested_cpus);
+	bool per_cpu_mmaps = !perf_cpu_map__has_any_cpu(evlist__core(evlist)->user_requested_cpus);
 
 	if (per_cpu_mmaps) {
-		struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist->core.all_cpus, idx);
+		struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist__core(evlist)->all_cpus, idx);
 		int cpu_map_idx = perf_cpu_map__idx(evsel->core.cpus, evlist_cpu);
 
 		if (cpu_map_idx == -1)
@@ -1838,7 +1838,7 @@ void perf_session__auxtrace_error_inc(struct perf_session *session,
 	struct perf_record_auxtrace_error *e = &event->auxtrace_error;
 
 	if (e->type < PERF_AUXTRACE_ERROR_MAX)
-		session->evlist->stats.nr_auxtrace_errors[e->type] += 1;
+		evlist__stats(session->evlist)->nr_auxtrace_errors[e->type] += 1;
 }
 
 void events_stats__auxtrace_error_warn(const struct events_stats *stats)
diff --git a/tools/perf/util/block-info.c b/tools/perf/util/block-info.c
index 8d3a9a661f267..1135e54f4c7fc 100644
--- a/tools/perf/util/block-info.c
+++ b/tools/perf/util/block-info.c
@@ -472,7 +472,7 @@ struct block_report *block_info__create_report(struct evlist *evlist,
 					       int *nr_reps)
 {
 	struct block_report *block_reports;
-	int nr_hists = evlist->core.nr_entries, i = 0;
+	int nr_hists = evlist__nr_entries(evlist), i = 0;
 	struct evsel *pos;
 
 	block_reports = calloc(nr_hists, sizeof(struct block_report));
@@ -483,7 +483,7 @@ struct block_report *block_info__create_report(struct evlist *evlist,
 		struct hists *hists = evsel__hists(pos);
 
 		process_block_report(hists, &block_reports[i], total_cycles,
-				     block_hpps, nr_hpps, evlist->nr_br_cntr);
+				     block_hpps, nr_hpps, evlist__nr_br_cntr(evlist));
 		i++;
 	}
 
diff --git a/tools/perf/util/bpf_counter.c b/tools/perf/util/bpf_counter.c
index 34b6b0da18b73..9362e45e17ceb 100644
--- a/tools/perf/util/bpf_counter.c
+++ b/tools/perf/util/bpf_counter.c
@@ -443,7 +443,7 @@ static int bperf_check_target(struct evsel *evsel,
 	} else if (target->tid) {
 		*filter_type = BPERF_FILTER_PID;
 		*filter_entry_cnt = perf_thread_map__nr(evsel->core.threads);
-	} else if (target->pid || evsel->evlist->workload.pid != -1) {
+	} else if (target->pid || evlist__workload_pid(evsel->evlist) != -1) {
 		*filter_type = BPERF_FILTER_TGID;
 		*filter_entry_cnt = perf_thread_map__nr(evsel->core.threads);
 	} else {
diff --git a/tools/perf/util/bpf_counter_cgroup.c b/tools/perf/util/bpf_counter_cgroup.c
index e1ce5aa3b9578..4e5f4b9dd4428 100644
--- a/tools/perf/util/bpf_counter_cgroup.c
+++ b/tools/perf/util/bpf_counter_cgroup.c
@@ -104,7 +104,7 @@ static int bperf_load_program(struct evlist *evlist)
 
 	set_max_rlimit();
 
-	if (nr_cgroups == 0 || evlist->core.nr_entries % nr_cgroups != 0) {
+	if (nr_cgroups == 0 || evlist__nr_entries(evlist) % nr_cgroups != 0) {
 		pr_err("Invalid cgroup or event count\n");
 		return -EINVAL;
 	}
@@ -116,7 +116,7 @@ static int bperf_load_program(struct evlist *evlist)
 		pr_err("Failed to open cgroup skeleton\n");
 		return -1;
 	}
-	setup_rodata(skel, evlist->core.nr_entries);
+	setup_rodata(skel, evlist__nr_entries(evlist));
 
 	err = bperf_cgroup_bpf__load(skel);
 	if (err) {
@@ -127,12 +127,12 @@ static int bperf_load_program(struct evlist *evlist)
 	err = -1;
 
 	cgrp_switch = evsel__new(&cgrp_switch_attr);
-	if (evsel__open_per_cpu(cgrp_switch, evlist->core.all_cpus, -1) < 0) {
+	if (evsel__open_per_cpu(cgrp_switch, evlist__core(evlist)->all_cpus, -1) < 0) {
 		pr_err("Failed to open cgroup switches event\n");
 		goto out;
 	}
 
-	perf_cpu_map__for_each_cpu(cpu, i, evlist->core.all_cpus) {
+	perf_cpu_map__for_each_cpu(cpu, i, evlist__core(evlist)->all_cpus) {
 		link = bpf_program__attach_perf_event(skel->progs.on_cgrp_switch,
 						      FD(cgrp_switch, i));
 		if (IS_ERR(link)) {
@@ -197,7 +197,7 @@ static int bperf_load_program(struct evlist *evlist)
 	 */
 	{
 		struct evsel *leader;
-		int num_events = evlist->core.nr_entries / nr_cgroups;
+		int num_events = evlist__nr_entries(evlist) / nr_cgroups;
 
 		evlist__for_each_entry(evlist, evsel) {
 			leader = evlist__find_evsel(evlist, evsel->core.idx % num_events);
@@ -258,7 +258,7 @@ static int bperf_cgrp__sync_counters(struct evlist *evlist)
 	unsigned int idx;
 	int prog_fd = bpf_program__fd(skel->progs.trigger_read);
 
-	perf_cpu_map__for_each_cpu(cpu, idx, evlist->core.all_cpus)
+	perf_cpu_map__for_each_cpu(cpu, idx, evlist__core(evlist)->all_cpus)
 		bperf_trigger_reading(prog_fd, cpu.cpu);
 
 	return 0;
@@ -336,7 +336,7 @@ static int bperf_cgrp__destroy(struct evsel *evsel)
 		return 0;
 
 	bperf_cgroup_bpf__destroy(skel);
-	evsel__delete(cgrp_switch);  // it'll destroy on_switch progs too
+	evsel__put(cgrp_switch);  // it'll destroy on_switch progs too
 
 	return 0;
 }
diff --git a/tools/perf/util/bpf_ftrace.c b/tools/perf/util/bpf_ftrace.c
index c456d24efa308..abeafd406e8e0 100644
--- a/tools/perf/util/bpf_ftrace.c
+++ b/tools/perf/util/bpf_ftrace.c
@@ -59,13 +59,13 @@ int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace)
 
 	/* don't need to set cpu filter for system-wide mode */
 	if (ftrace->target.cpu_list) {
-		ncpus = perf_cpu_map__nr(ftrace->evlist->core.user_requested_cpus);
+		ncpus = perf_cpu_map__nr(evlist__core(ftrace->evlist)->user_requested_cpus);
 		bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus);
 		skel->rodata->has_cpu = 1;
 	}
 
 	if (target__has_task(&ftrace->target) || target__none(&ftrace->target)) {
-		ntasks = perf_thread_map__nr(ftrace->evlist->core.threads);
+		ntasks = perf_thread_map__nr(evlist__core(ftrace->evlist)->threads);
 		bpf_map__set_max_entries(skel->maps.task_filter, ntasks);
 		skel->rodata->has_task = 1;
 	}
@@ -87,7 +87,8 @@ int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace)
 		fd = bpf_map__fd(skel->maps.cpu_filter);
 
 		for (i = 0; i < ncpus; i++) {
-			cpu = perf_cpu_map__cpu(ftrace->evlist->core.user_requested_cpus, i).cpu;
+			cpu = perf_cpu_map__cpu(
+				evlist__core(ftrace->evlist)->user_requested_cpus, i).cpu;
 			bpf_map_update_elem(fd, &cpu, &val, BPF_ANY);
 		}
 	}
@@ -99,7 +100,7 @@ int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace)
 		fd = bpf_map__fd(skel->maps.task_filter);
 
 		for (i = 0; i < ntasks; i++) {
-			pid = perf_thread_map__pid(ftrace->evlist->core.threads, i);
+			pid = perf_thread_map__pid(evlist__core(ftrace->evlist)->threads, i);
 			bpf_map_update_elem(fd, &pid, &val, BPF_ANY);
 		}
 	}
diff --git a/tools/perf/util/bpf_lock_contention.c b/tools/perf/util/bpf_lock_contention.c
index b1cfa63a488fa..c20bd075664e1 100644
--- a/tools/perf/util/bpf_lock_contention.c
+++ b/tools/perf/util/bpf_lock_contention.c
@@ -223,11 +223,11 @@ int lock_contention_prepare(struct lock_contention *con)
 
 	if (target__has_cpu(target)) {
 		skel->rodata->has_cpu = 1;
-		ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus);
+		ncpus = perf_cpu_map__nr(evlist__core(evlist)->user_requested_cpus);
 	}
 	if (target__has_task(target)) {
 		skel->rodata->has_task = 1;
-		ntasks = perf_thread_map__nr(evlist->core.threads);
+		ntasks = perf_thread_map__nr(evlist__core(evlist)->threads);
 	}
 	if (con->filters->nr_types) {
 		skel->rodata->has_type = 1;
@@ -334,7 +334,7 @@ int lock_contention_prepare(struct lock_contention *con)
 		fd = bpf_map__fd(skel->maps.cpu_filter);
 
 		for (i = 0; i < ncpus; i++) {
-			cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu;
+			cpu = perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, i).cpu;
 			bpf_map_update_elem(fd, &cpu, &val, BPF_ANY);
 		}
 	}
@@ -346,13 +346,13 @@ int lock_contention_prepare(struct lock_contention *con)
 		fd = bpf_map__fd(skel->maps.task_filter);
 
 		for (i = 0; i < ntasks; i++) {
-			pid = perf_thread_map__pid(evlist->core.threads, i);
+			pid = perf_thread_map__pid(evlist__core(evlist)->threads, i);
 			bpf_map_update_elem(fd, &pid, &val, BPF_ANY);
 		}
 	}
 
-	if (target__none(target) && evlist->workload.pid > 0) {
-		u32 pid = evlist->workload.pid;
+	if (target__none(target) && evlist__workload_pid(evlist) > 0) {
+		u32 pid = evlist__workload_pid(evlist);
 		u8 val = 1;
 
 		fd = bpf_map__fd(skel->maps.task_filter);
diff --git a/tools/perf/util/bpf_off_cpu.c b/tools/perf/util/bpf_off_cpu.c
index a3b699a5322f1..c4639f6a57766 100644
--- a/tools/perf/util/bpf_off_cpu.c
+++ b/tools/perf/util/bpf_off_cpu.c
@@ -1,23 +1,25 @@
 // SPDX-License-Identifier: GPL-2.0
-#include "util/bpf_counter.h"
-#include "util/debug.h"
-#include "util/evsel.h"
-#include "util/evlist.h"
-#include "util/off_cpu.h"
-#include "util/perf-hooks.h"
-#include "util/record.h"
-#include "util/session.h"
-#include "util/target.h"
-#include "util/cpumap.h"
-#include "util/thread_map.h"
-#include "util/cgroup.h"
-#include "util/strlist.h"
+#include <linux/time64.h>
+
 #include <bpf/bpf.h>
 #include <bpf/btf.h>
 #include <internal/xyarray.h>
-#include <linux/time64.h>
 
+#include "bpf_counter.h"
 #include "bpf_skel/off_cpu.skel.h"
+#include "cgroup.h"
+#include "cpumap.h"
+#include "debug.h"
+#include "evlist.h"
+#include "evsel.h"
+#include "off_cpu.h"
+#include "parse-events.h"
+#include "perf-hooks.h"
+#include "record.h"
+#include "session.h"
+#include "strlist.h"
+#include "target.h"
+#include "thread_map.h"
 
 #define MAX_STACKS  32
 #define MAX_PROC  4096
@@ -71,13 +73,13 @@ static void off_cpu_start(void *arg)
 
 	/* update task filter for the given workload */
 	if (skel->rodata->has_task && skel->rodata->uses_tgid &&
-	    perf_thread_map__pid(evlist->core.threads, 0) != -1) {
+	    perf_thread_map__pid(evlist__core(evlist)->threads, 0) != -1) {
 		int fd;
 		u32 pid;
 		u8 val = 1;
 
 		fd = bpf_map__fd(skel->maps.task_filter);
-		pid = perf_thread_map__pid(evlist->core.threads, 0);
+		pid = perf_thread_map__pid(evlist__core(evlist)->threads, 0);
 		bpf_map_update_elem(fd, &pid, &val, BPF_ANY);
 	}
 
@@ -166,7 +168,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target,
 
 	/* don't need to set cpu filter for system-wide mode */
 	if (target->cpu_list) {
-		ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus);
+		ncpus = perf_cpu_map__nr(evlist__core(evlist)->user_requested_cpus);
 		bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus);
 		skel->rodata->has_cpu = 1;
 	}
@@ -197,7 +199,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target,
 		skel->rodata->has_task = 1;
 		skel->rodata->uses_tgid = 1;
 	} else if (target__has_task(target)) {
-		ntasks = perf_thread_map__nr(evlist->core.threads);
+		ntasks = perf_thread_map__nr(evlist__core(evlist)->threads);
 		bpf_map__set_max_entries(skel->maps.task_filter, ntasks);
 		skel->rodata->has_task = 1;
 	} else if (target__none(target)) {
@@ -207,7 +209,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target,
 	}
 
 	if (evlist__first(evlist)->cgrp) {
-		ncgrps = evlist->core.nr_entries - 1; /* excluding a dummy */
+		ncgrps = evlist__nr_entries(evlist) - 1; /* excluding a dummy */
 		bpf_map__set_max_entries(skel->maps.cgroup_filter, ncgrps);
 
 		if (!cgroup_is_v2("perf_event"))
@@ -238,7 +240,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target,
 		fd = bpf_map__fd(skel->maps.cpu_filter);
 
 		for (i = 0; i < ncpus; i++) {
-			cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu;
+			cpu = perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, i).cpu;
 			bpf_map_update_elem(fd, &cpu, &val, BPF_ANY);
 		}
 	}
@@ -267,7 +269,7 @@ int off_cpu_prepare(struct evlist *evlist, struct target *target,
 		fd = bpf_map__fd(skel->maps.task_filter);
 
 		for (i = 0; i < ntasks; i++) {
-			pid = perf_thread_map__pid(evlist->core.threads, i);
+			pid = perf_thread_map__pid(evlist__core(evlist)->threads, i);
 			bpf_map_update_elem(fd, &pid, &val, BPF_ANY);
 		}
 	}
diff --git a/tools/perf/util/bpf_trace_augment.c b/tools/perf/util/bpf_trace_augment.c
index 9e706f0fa53d4..a9cf2a77ded17 100644
--- a/tools/perf/util/bpf_trace_augment.c
+++ b/tools/perf/util/bpf_trace_augment.c
@@ -1,11 +1,11 @@
 #include <bpf/libbpf.h>
 #include <internal/xyarray.h>
 
-#include "util/debug.h"
-#include "util/evlist.h"
-#include "util/trace_augment.h"
-
 #include "bpf_skel/augmented_raw_syscalls.skel.h"
+#include "debug.h"
+#include "evlist.h"
+#include "parse-events.h"
+#include "trace_augment.h"
 
 static struct augmented_raw_syscalls_bpf *skel;
 static struct evsel *bpf_output;
diff --git a/tools/perf/util/capstone.c b/tools/perf/util/capstone.c
index 5ad537fea4360..9bba78ee0c5a2 100644
--- a/tools/perf/util/capstone.c
+++ b/tools/perf/util/capstone.c
@@ -302,6 +302,7 @@ static void print_capstone_detail(struct cs_insn *insn, char *buf, size_t len,
 	for (i = 0; i < insn->detail->x86.op_count; i++) {
 		struct cs_x86_op *op = &insn->detail->x86.operands[i];
 		u64 orig_addr;
+		struct map *found_map = NULL;
 
 		if (op->type != X86_OP_MEM)
 			continue;
@@ -317,19 +318,22 @@ static void print_capstone_detail(struct cs_insn *insn, char *buf, size_t len,
 		if (dso__kernel(map__dso(map))) {
 			/*
 			 * The kernel maps can be split into sections, let's
-			 * find the map first and the search the symbol.
+			 * find the map first and then search the symbol.
 			 */
-			map = maps__find(map__kmaps(map), addr);
-			if (map == NULL)
+			found_map = maps__find(map__kmaps(map), addr);
+			if (found_map == NULL)
 				continue;
+			map = found_map;
 		}
 
 		/* convert it to map-relative address for search */
 		addr = map__map_ip(map, addr);
 
 		sym = map__find_symbol(map, addr);
-		if (sym == NULL)
+		if (sym == NULL) {
+			map__put(found_map);
 			continue;
+		}
 
 		if (addr == sym->start) {
 			scnprintf(buf, len, "\t# %"PRIx64" <%s>",
@@ -338,6 +342,7 @@ static void print_capstone_detail(struct cs_insn *insn, char *buf, size_t len,
 			scnprintf(buf, len, "\t# %"PRIx64" <%s+%#"PRIx64">",
 				  orig_addr, sym->name, addr - sym->start);
 		}
+		map__put(found_map);
 		break;
 	}
 }
diff --git a/tools/perf/util/cgroup.c b/tools/perf/util/cgroup.c
index 1b5664d1481f5..c7be16a7915e6 100644
--- a/tools/perf/util/cgroup.c
+++ b/tools/perf/util/cgroup.c
@@ -367,7 +367,7 @@ int parse_cgroups(const struct option *opt, const char *str,
 	char *s;
 	int ret, i;
 
-	if (list_empty(&evlist->core.entries)) {
+	if (list_empty(&evlist__core(evlist)->entries)) {
 		fprintf(stderr, "must define events before cgroups\n");
 		return -1;
 	}
@@ -423,7 +423,7 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro
 	int ret = -1;
 	int prefix_len;
 
-	if (evlist->core.nr_entries == 0) {
+	if (evlist__nr_entries(evlist) == 0) {
 		fprintf(stderr, "must define events before cgroups\n");
 		return -EINVAL;
 	}
@@ -436,11 +436,11 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro
 	}
 
 	/* save original events and init evlist */
-	evlist__splice_list_tail(orig_list, &evlist->core.entries);
-	evlist->core.nr_entries = 0;
+	evlist__splice_list_tail(orig_list, &evlist__core(evlist)->entries);
+	evlist__core(evlist)->nr_entries = 0;
 
-	orig_metric_events = evlist->metric_events;
-	metricgroup__rblist_init(&evlist->metric_events);
+	orig_metric_events = *evlist__metric_events(evlist);
+	metricgroup__rblist_init(evlist__metric_events(evlist));
 
 	if (has_pattern_string(str))
 		prefix_len = match_cgroups(str);
@@ -469,7 +469,7 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro
 
 		/* copy the list and set to the new cgroup. */
 		evlist__for_each_entry(orig_list, pos) {
-			struct evsel *evsel = evsel__clone(/*dest=*/NULL, pos);
+			struct evsel *evsel = evsel__clone(pos);
 
 			if (evsel == NULL)
 				goto out_err;
@@ -503,15 +503,15 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro
 		nr_cgroups++;
 
 		if (metricgroup__copy_metric_events(tmp_list, cgrp,
-						    &evlist->metric_events,
+						    evlist__metric_events(evlist),
 						    &orig_metric_events) < 0)
 			goto out_err;
 
-		evlist__splice_list_tail(evlist, &tmp_list->core.entries);
-		tmp_list->core.nr_entries = 0;
+		evlist__splice_list_tail(evlist, &evlist__core(tmp_list)->entries);
+		evlist__core(tmp_list)->nr_entries = 0;
 	}
 
-	if (list_empty(&evlist->core.entries)) {
+	if (list_empty(&evlist__core(evlist)->entries)) {
 		fprintf(stderr, "no cgroup matched: %s\n", str);
 		goto out_err;
 	}
@@ -520,8 +520,8 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro
 	cgrp_event_expanded = true;
 
 out_err:
-	evlist__delete(orig_list);
-	evlist__delete(tmp_list);
+	evlist__put(orig_list);
+	evlist__put(tmp_list);
 	metricgroup__rblist_exit(&orig_metric_events);
 	release_cgroup_list();
 
diff --git a/tools/perf/util/compress.h b/tools/perf/util/compress.h
index 6cfecfca16f24..ec6c38129e248 100644
--- a/tools/perf/util/compress.h
+++ b/tools/perf/util/compress.h
@@ -54,7 +54,8 @@ int zstd_fini(struct zstd_data *data);
 
 ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_t dst_size,
 				       void *src, size_t src_size, size_t max_record_size,
-				       size_t process_header(void *record, size_t increment));
+				       ssize_t process_header(void *record, size_t dst_size,
+							      size_t data_size));
 
 size_t zstd_decompress_stream(struct zstd_data *data, void *src, size_t src_size,
 			      void *dst, size_t dst_size);
@@ -75,7 +76,8 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data __maybe_unused,
 				       void *dst __maybe_unused, size_t dst_size __maybe_unused,
 				       void *src __maybe_unused, size_t src_size __maybe_unused,
 				       size_t max_record_size __maybe_unused,
-				       size_t process_header(void *record, size_t increment) __maybe_unused)
+				       ssize_t process_header(void *record, size_t dst_size,
+							      size_t data_size) __maybe_unused)
 {
 	return 0;
 }
diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c
index 5d0664ff73b79..95e3ec1171acc 100644
--- a/tools/perf/util/cs-etm.c
+++ b/tools/perf/util/cs-etm.c
@@ -18,6 +18,7 @@
 #include <stdlib.h>
 
 #include "auxtrace.h"
+#include "callchain.h"
 #include "color.h"
 #include "cs-etm.h"
 #include "cs-etm-decoder/cs-etm-decoder.h"
@@ -67,10 +68,13 @@ struct cs_etm_auxtrace {
 	bool snapshot_mode;
 	bool data_queued;
 	bool has_virtual_ts; /* Virtual/Kernel timestamps in the trace. */
+	bool use_thread_stack;
+	bool use_callchain;
 
 	int num_cpu;
 	u64 latest_kernel_timestamp;
 	u32 auxtrace_type;
+	u32 branches_filter;
 	u64 branches_sample_type;
 	u64 branches_id;
 	u64 instructions_sample_type;
@@ -84,10 +88,11 @@ struct cs_etm_auxtrace {
 struct cs_etm_traceid_queue {
 	u8 trace_chan_id;
 	u64 period_instructions;
-	size_t last_branch_pos;
+	u64 kernel_start;
 	union perf_event *event_buf;
+	unsigned int br_stack_sz;
 	struct branch_stack *last_branch;
-	struct branch_stack *last_branch_rb;
+	struct ip_callchain *callchain;
 	struct cs_etm_packet *prev_packet;
 	struct cs_etm_packet *packet;
 	struct cs_etm_packet_queue packet_queue;
@@ -645,6 +650,8 @@ static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
 					       queue->tid);
 	tidq->decode_thread = machine__findnew_thread(&etm->session->machines.host, -1,
 					       queue->tid);
+	if (!tidq->frontend_thread || !tidq->decode_thread)
+		goto out;
 
 	tidq->packet = zalloc(sizeof(struct cs_etm_packet));
 	if (!tidq->packet)
@@ -654,7 +661,7 @@ static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
 	if (!tidq->prev_packet)
 		goto out_free;
 
-	if (etm->synth_opts.last_branch) {
+	if (etm->use_thread_stack) {
 		size_t sz = sizeof(struct branch_stack);
 
 		sz += etm->synth_opts.last_branch_sz *
@@ -662,8 +669,16 @@ static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
 		tidq->last_branch = zalloc(sz);
 		if (!tidq->last_branch)
 			goto out_free;
-		tidq->last_branch_rb = zalloc(sz);
-		if (!tidq->last_branch_rb)
+
+		tidq->br_stack_sz = etm->synth_opts.last_branch_sz;
+	}
+
+	if (etm->synth_opts.callchain) {
+		/* Add 1 to callchain_sz for callchain context */
+		tidq->callchain =
+			zalloc(struct_size(tidq->callchain, ips,
+					   etm->synth_opts.callchain_sz + 1));
+		if (!tidq->callchain)
 			goto out_free;
 	}
 
@@ -674,11 +689,13 @@ static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
 	return 0;
 
 out_free:
-	zfree(&tidq->last_branch_rb);
+	zfree(&tidq->callchain);
 	zfree(&tidq->last_branch);
 	zfree(&tidq->prev_packet);
 	zfree(&tidq->packet);
 out:
+	thread__zput(tidq->frontend_thread);
+	thread__zput(tidq->decode_thread);
 	return rc;
 }
 
@@ -956,8 +973,8 @@ static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq)
 		thread__zput(tidq->frontend_thread);
 		thread__zput(tidq->decode_thread);
 		zfree(&tidq->event_buf);
+		zfree(&tidq->callchain);
 		zfree(&tidq->last_branch);
-		zfree(&tidq->last_branch_rb);
 		zfree(&tidq->prev_packet);
 		zfree(&tidq->packet);
 		zfree(&tidq);
@@ -1317,57 +1334,6 @@ static int cs_etm__queue_first_cs_timestamp(struct cs_etm_auxtrace *etm,
 	return ret;
 }
 
-static inline
-void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq,
-				 struct cs_etm_traceid_queue *tidq)
-{
-	struct branch_stack *bs_src = tidq->last_branch_rb;
-	struct branch_stack *bs_dst = tidq->last_branch;
-	size_t nr = 0;
-
-	/*
-	 * Set the number of records before early exit: ->nr is used to
-	 * determine how many branches to copy from ->entries.
-	 */
-	bs_dst->nr = bs_src->nr;
-
-	/*
-	 * Early exit when there is nothing to copy.
-	 */
-	if (!bs_src->nr)
-		return;
-
-	/*
-	 * As bs_src->entries is a circular buffer, we need to copy from it in
-	 * two steps.  First, copy the branches from the most recently inserted
-	 * branch ->last_branch_pos until the end of bs_src->entries buffer.
-	 */
-	nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;
-	memcpy(&bs_dst->entries[0],
-	       &bs_src->entries[tidq->last_branch_pos],
-	       sizeof(struct branch_entry) * nr);
-
-	/*
-	 * If we wrapped around at least once, the branches from the beginning
-	 * of the bs_src->entries buffer and until the ->last_branch_pos element
-	 * are older valid branches: copy them over.  The total number of
-	 * branches copied over will be equal to the number of branches asked by
-	 * the user in last_branch_sz.
-	 */
-	if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) {
-		memcpy(&bs_dst->entries[nr],
-		       &bs_src->entries[0],
-		       sizeof(struct branch_entry) * tidq->last_branch_pos);
-	}
-}
-
-static inline
-void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)
-{
-	tidq->last_branch_pos = 0;
-	tidq->last_branch_rb->nr = 0;
-}
-
 static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq,
 					 struct cs_etm_traceid_queue *tidq,
 					 struct cs_etm_packet *packet, u64 addr)
@@ -1384,6 +1350,18 @@ static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq,
 	return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2;
 }
 
+static inline int cs_etm__instr_size(struct cs_etm_queue *etmq,
+				     struct cs_etm_traceid_queue *tidq,
+				     struct cs_etm_packet *packet,
+				     u64 addr)
+{
+	if (packet->isa == CS_ETM_ISA_T32)
+		return cs_etm__t32_instr_size(etmq, tidq, packet, addr);
+
+	/* Otherwise, 4-byte instruction size for A32/A64 */
+	return 4;
+}
+
 static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet)
 {
 	/*
@@ -1412,51 +1390,17 @@ static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq,
 				     struct cs_etm_packet *packet,
 				     u64 offset)
 {
-	if (packet->isa == CS_ETM_ISA_T32) {
-		u64 addr = packet->start_addr;
-
-		while (offset) {
-			addr += cs_etm__t32_instr_size(etmq, tidq, packet,
-						       addr);
-			offset--;
-		}
-		return addr;
-	}
-
-	/* Assume a 4 byte instruction size (A32/A64) */
-	return packet->start_addr + offset * 4;
-}
-
-static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq,
-					  struct cs_etm_traceid_queue *tidq)
-{
-	struct branch_stack *bs = tidq->last_branch_rb;
-	struct branch_entry *be;
-
-	/*
-	 * The branches are recorded in a circular buffer in reverse
-	 * chronological order: we start recording from the last element of the
-	 * buffer down.  After writing the first element of the stack, move the
-	 * insert position back to the end of the buffer.
-	 */
-	if (!tidq->last_branch_pos)
-		tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;
+	u64 addr = packet->start_addr;
 
-	tidq->last_branch_pos -= 1;
+	/* 4-byte instruction size for A32/A64 */
+	if (packet->isa == CS_ETM_ISA_A64 || packet->isa == CS_ETM_ISA_A32)
+		return addr + offset * 4;
 
-	be       = &bs->entries[tidq->last_branch_pos];
-	be->from = cs_etm__last_executed_instr(tidq->prev_packet);
-	be->to	 = cs_etm__first_executed_instr(tidq->packet);
-	/* No support for mispredict */
-	be->flags.mispred = 0;
-	be->flags.predicted = 1;
-
-	/*
-	 * Increment bs->nr until reaching the number of last branches asked by
-	 * the user on the command line.
-	 */
-	if (bs->nr < etmq->etm->synth_opts.last_branch_sz)
-		bs->nr += 1;
+	while (offset) {
+		addr += cs_etm__instr_size(etmq, tidq, packet, addr);
+		offset--;
+	}
+	return addr;
 }
 
 static int cs_etm__inject_event(struct cs_etm_auxtrace *etm, union perf_event *event,
@@ -1594,16 +1538,7 @@ static void cs_etm__copy_insn(struct cs_etm_queue *etmq,
 		return;
 	}
 
-	/*
-	 * T32 instruction size might be 32-bit or 16-bit, decide by calling
-	 * cs_etm__t32_instr_size().
-	 */
-	if (packet->isa == CS_ETM_ISA_T32)
-		sample->insn_len = cs_etm__t32_instr_size(etmq, tidq, packet,
-							  sample->ip);
-	/* Otherwise, A64 and A32 instruction size are always 32-bit. */
-	else
-		sample->insn_len = 4;
+	sample->insn_len = cs_etm__instr_size(etmq, tidq, packet, sample->ip);
 
 	cs_etm__frontend_mem_access(etmq, tidq, packet, sample->ip,
 				    sample->insn_len, (void *)sample->insn);
@@ -1631,6 +1566,78 @@ static inline u64 cs_etm__resolve_sample_time(struct cs_etm_queue *etmq,
 		return etm->latest_kernel_timestamp;
 }
 
+static bool cs_etm__packet_has_taken_branch(struct cs_etm_packet *packet)
+{
+	if (packet->sample_type == CS_ETM_RANGE &&
+	    packet->last_instr_taken_branch)
+		return true;
+
+	return false;
+}
+
+static void cs_etm__add_stack_event(struct cs_etm_queue *etmq,
+				    struct cs_etm_traceid_queue *tidq)
+{
+	struct cs_etm_auxtrace *etm = etmq->etm;
+	u64 from, to;
+	int size;
+
+	if (!etm->synth_opts.branches && !etm->synth_opts.instructions)
+		return;
+
+	if (!cs_etm__packet_has_taken_branch(tidq->prev_packet))
+		return;
+
+	if (etmq->etm->use_thread_stack) {
+		from = cs_etm__last_executed_instr(tidq->prev_packet);
+		to = cs_etm__first_executed_instr(tidq->packet);
+
+		size = cs_etm__instr_size(etmq, tidq, tidq->prev_packet, from);
+
+		/* Enable callchain so thread stack entry can be allocated */
+		thread_stack__event(tidq->frontend_thread, tidq->prev_packet->cpu,
+				    tidq->prev_packet->flags, from, to, size,
+				    etmq->buffer->buffer_nr + 1,
+				    etmq->etm->use_callchain,
+				    tidq->br_stack_sz, 0);
+	} else {
+		thread_stack__set_trace_nr(tidq->frontend_thread,
+					   tidq->prev_packet->cpu,
+					   etmq->buffer->buffer_nr + 1);
+	}
+}
+
+static void cs_etm__sample_branch_stack(struct cs_etm_auxtrace *etm,
+					struct cs_etm_traceid_queue *tidq,
+					struct perf_sample *sample)
+{
+	if (etm->synth_opts.last_branch) {
+		thread_stack__br_sample(tidq->frontend_thread, tidq->packet->cpu,
+					tidq->last_branch, tidq->br_stack_sz);
+		sample->branch_stack = tidq->last_branch;
+	}
+
+	if (etm->synth_opts.callchain) {
+		if (tidq->kernel_start)
+			thread_stack__sample(tidq->frontend_thread,
+					     tidq->packet->cpu,
+					     tidq->callchain,
+					     etm->synth_opts.callchain_sz + 1,
+					     sample->ip, tidq->kernel_start);
+		else
+			/*
+			 * Clear the callchain when the kernel start address is
+			 * not available yet. The empty callchain can then be
+			 * consumed by cs_etm__inject_event().
+			 */
+			memset(tidq->callchain, 0,
+			       struct_size(tidq->callchain, ips,
+					   etm->synth_opts.callchain_sz + 1));
+
+		sample->callchain = tidq->callchain;
+	}
+}
+
 static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
 					    struct cs_etm_traceid_queue *tidq,
 					    struct cs_etm_packet *packet,
@@ -1660,9 +1667,7 @@ static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
 	sample.cpumode = event->sample.header.misc;
 
 	cs_etm__copy_insn(etmq, tidq, packet, &sample);
-
-	if (etm->synth_opts.last_branch)
-		sample.branch_stack = tidq->last_branch;
+	cs_etm__sample_branch_stack(etm, tidq, &sample);
 
 	if (etm->synth_opts.inject) {
 		ret = cs_etm__inject_event(etm, event, &sample,
@@ -1691,8 +1696,9 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
 {
 	int ret = 0;
 	struct cs_etm_auxtrace *etm = etmq->etm;
-	struct perf_sample sample = {.ip = 0,};
+	struct perf_sample sample;
 	union perf_event *event = tidq->event_buf;
+
 	struct dummy_branch_stack {
 		u64			nr;
 		u64			hw_idx;
@@ -1700,6 +1706,11 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
 	} dummy_bs;
 	u64 ip;
 
+	if (etm->branches_filter &&
+		!(etm->branches_filter & tidq->prev_packet->flags))
+		return 0;
+
+	perf_sample__init(&sample, /*all=*/true);
 	ip = cs_etm__last_executed_instr(tidq->prev_packet);
 
 	event->sample.header.type = PERF_RECORD_SAMPLE;
@@ -1752,6 +1763,7 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
 		"CS ETM Trace: failed to deliver instruction event, error %d\n",
 		ret);
 
+	perf_sample__exit(&sample);
 	return ret;
 }
 
@@ -1822,6 +1834,9 @@ static int cs_etm__synth_events(struct cs_etm_auxtrace *etm,
 		attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX;
 	}
 
+	if (etm->synth_opts.callchain)
+		attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
+
 	if (etm->synth_opts.instructions) {
 		attr.config = PERF_COUNT_HW_INSTRUCTIONS;
 		attr.sample_period = etm->synth_opts.period;
@@ -1849,14 +1864,7 @@ static int cs_etm__sample(struct cs_etm_queue *etmq,
 
 	tidq->period_instructions += tidq->packet->instr_count;
 
-	/*
-	 * Record a branch when the last instruction in
-	 * PREV_PACKET is a branch.
-	 */
-	if (etm->synth_opts.last_branch &&
-	    tidq->prev_packet->sample_type == CS_ETM_RANGE &&
-	    tidq->prev_packet->last_instr_taken_branch)
-		cs_etm__update_last_branch_rb(etmq, tidq);
+	cs_etm__add_stack_event(etmq, tidq);
 
 	if (etm->synth_opts.instructions &&
 	    tidq->period_instructions >= etm->instructions_sample_period) {
@@ -1915,10 +1923,6 @@ static int cs_etm__sample(struct cs_etm_queue *etmq,
 		u64 offset = etm->instructions_sample_period - instrs_prev;
 		u64 addr;
 
-		/* Prepare last branches for instruction sample */
-		if (etm->synth_opts.last_branch)
-			cs_etm__copy_last_branch_rb(etmq, tidq);
-
 		while (tidq->period_instructions >=
 				etm->instructions_sample_period) {
 			/*
@@ -1949,8 +1953,7 @@ static int cs_etm__sample(struct cs_etm_queue *etmq,
 			generate_sample = true;
 
 		/* Generate sample for branch taken packet */
-		if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&
-		    tidq->prev_packet->last_instr_taken_branch)
+		if (cs_etm__packet_has_taken_branch(tidq->prev_packet))
 			generate_sample = true;
 
 		if (generate_sample) {
@@ -1965,6 +1968,34 @@ static int cs_etm__sample(struct cs_etm_queue *etmq,
 	return 0;
 }
 
+static int cs_etm__context(struct cs_etm_queue *etmq,
+			   struct cs_etm_traceid_queue *tidq)
+{
+	ocsd_ex_level el = tidq->packet->el;
+	struct machine *machine;
+	int ret;
+
+	machine = cs_etm__get_machine(etmq, el);
+	if (!machine) {
+		ret = -EINVAL;
+		goto err;
+	}
+
+	tidq->kernel_start = machine__kernel_start(machine);
+
+	ret = cs_etm__etmq_update_thread(etmq, el, tidq->packet->tid,
+					 &tidq->frontend_thread);
+	if (ret)
+		goto err;
+
+	return 0;
+
+err:
+	thread__zput(tidq->frontend_thread);
+	tidq->kernel_start = 0;
+	return ret;
+}
+
 static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)
 {
 	/*
@@ -1998,10 +2029,6 @@ static int cs_etm__flush(struct cs_etm_queue *etmq,
 	    etmq->etm->synth_opts.instructions &&
 	    tidq->prev_packet->sample_type == CS_ETM_RANGE) {
 		u64 addr;
-
-		/* Prepare last branches for instruction sample */
-		cs_etm__copy_last_branch_rb(etmq, tidq);
-
 		/*
 		 * Generate a last branch event for the branches left in the
 		 * circular buffer at the end of the trace.
@@ -2032,8 +2059,8 @@ static int cs_etm__flush(struct cs_etm_queue *etmq,
 	cs_etm__packet_swap(etm, tidq);
 
 	/* Reset last branches after flush the trace */
-	if (etm->synth_opts.last_branch)
-		cs_etm__reset_last_branch_rb(tidq);
+	if (etm->use_thread_stack)
+		thread_stack__flush(tidq->frontend_thread);
 
 	return err;
 }
@@ -2057,9 +2084,6 @@ static int cs_etm__end_block(struct cs_etm_queue *etmq,
 	    tidq->prev_packet->sample_type == CS_ETM_RANGE) {
 		u64 addr;
 
-		/* Prepare last branches for instruction sample */
-		cs_etm__copy_last_branch_rb(etmq, tidq);
-
 		/*
 		 * Use the address of the end of the last reported execution
 		 * range.
@@ -2077,6 +2101,45 @@ static int cs_etm__end_block(struct cs_etm_queue *etmq,
 
 	return 0;
 }
+
+static int cs_etm__flush_stack_cb(struct thread *thread,
+				  void *data __maybe_unused)
+{
+	thread_stack__flush(thread);
+	return 0;
+}
+
+static void cs_etm__flush_machine_stack(struct cs_etm_queue *etmq, pid_t pid)
+{
+	struct machine *machine;
+
+	machine = machines__find(&etmq->etm->session->machines, pid);
+	if (machine)
+		machine__for_each_thread(machine, cs_etm__flush_stack_cb, NULL);
+}
+
+static void cs_etm__flush_all_stack(struct cs_etm_queue *etmq)
+{
+	enum cs_etm_pid_fmt pid_fmt = cs_etm__get_pid_fmt(etmq);
+
+	if (!etmq->etm->use_thread_stack)
+		return;
+
+	switch (pid_fmt) {
+	case CS_ETM_PIDFMT_CTXTID2:
+		/* Clear the guest stack if virtualization is supported */
+		cs_etm__flush_machine_stack(etmq, DEFAULT_GUEST_KERNEL_ID);
+		fallthrough;
+	case CS_ETM_PIDFMT_CTXTID:
+		cs_etm__flush_machine_stack(etmq, HOST_KERNEL_ID);
+		break;
+	case CS_ETM_PIDFMT_NONE:
+	default:
+		break;
+
+	}
+}
+
 /*
  * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue
  *			   if need be.
@@ -2099,6 +2162,12 @@ static int cs_etm__get_data_block(struct cs_etm_queue *etmq)
 		ret = cs_etm_decoder__reset(etmq->decoder);
 		if (ret)
 			return ret;
+
+		/*
+		 * Since the decoder is reset, this causes a global trace
+		 * discontinuity. Flush all thread stacks.
+		 */
+		cs_etm__flush_all_stack(etmq);
 	}
 
 	return etmq->buf_len;
@@ -2190,7 +2259,7 @@ static bool cs_etm__is_syscall(struct cs_etm_queue *etmq,
 	 * HVC cases; need to check if it's SVC instruction based on
 	 * packet address.
 	 */
-	if (magic == __perf_cs_etmv4_magic) {
+	if (magic == __perf_cs_etmv4_magic || magic == __perf_cs_ete_magic) {
 		if (packet->exception_number == CS_ETMV4_EXC_CALL &&
 		    cs_etm__is_svc_instr(etmq, tidq, prev_packet,
 					 prev_packet->end_addr))
@@ -2213,7 +2282,7 @@ static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,
 		    packet->exception_number == CS_ETMV3_EXC_FIQ)
 			return true;
 
-	if (magic == __perf_cs_etmv4_magic)
+	if (magic == __perf_cs_etmv4_magic || magic == __perf_cs_ete_magic)
 		if (packet->exception_number == CS_ETMV4_EXC_RESET ||
 		    packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT ||
 		    packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR ||
@@ -2243,7 +2312,7 @@ static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq,
 		    packet->exception_number == CS_ETMV3_EXC_GENERIC)
 			return true;
 
-	if (magic == __perf_cs_etmv4_magic) {
+	if (magic == __perf_cs_etmv4_magic || magic == __perf_cs_ete_magic) {
 		if (packet->exception_number == CS_ETMV4_EXC_TRAP ||
 		    packet->exception_number == CS_ETMV4_EXC_ALIGNMENT ||
 		    packet->exception_number == CS_ETMV4_EXC_INST_FAULT ||
@@ -2527,9 +2596,7 @@ static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq,
 			 * tracing the kernel the context packet will be emitted
 			 * between two ranges.
 			 */
-			ret = cs_etm__etmq_update_thread(etmq, tidq->packet->el,
-							 tidq->packet->tid,
-							 &tidq->frontend_thread);
+			ret = cs_etm__context(etmq, tidq);
 			if (ret)
 				goto out;
 			break;
@@ -3555,6 +3622,25 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event,
 		itrace_synth_opts__set_default(&etm->synth_opts,
 				session->itrace_synth_opts->default_no_sample);
 		etm->synth_opts.callchain = false;
+		etm->synth_opts.thread_stack = session->itrace_synth_opts->thread_stack;
+	}
+
+	if (etm->synth_opts.calls)
+		etm->branches_filter |= PERF_IP_FLAG_CALL |
+					PERF_IP_FLAG_TRACE_BEGIN |
+					PERF_IP_FLAG_TRACE_END;
+
+	if (etm->synth_opts.returns)
+		etm->branches_filter |= PERF_IP_FLAG_RETURN |
+					PERF_IP_FLAG_TRACE_BEGIN |
+					PERF_IP_FLAG_TRACE_END;
+
+	if (etm->synth_opts.callchain && !symbol_conf.use_callchain) {
+		symbol_conf.use_callchain = true;
+		if (callchain_register_param(&callchain_param) < 0) {
+			symbol_conf.use_callchain = false;
+			etm->synth_opts.callchain = false;
+		}
 	}
 
 	etm->session = session;
@@ -3606,6 +3692,14 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event,
 		etm->tc.cap_user_time_zero = tc->cap_user_time_zero;
 		etm->tc.cap_user_time_short = tc->cap_user_time_short;
 	}
+
+	etm->use_thread_stack = etm->synth_opts.thread_stack ||
+				etm->synth_opts.last_branch ||
+				etm->synth_opts.callchain;
+
+	etm->use_callchain = etm->synth_opts.thread_stack ||
+			     etm->synth_opts.callchain;
+
 	err = cs_etm__synth_events(etm, session);
 	if (err)
 		goto err_free_queues;
diff --git a/tools/perf/util/data-convert-bt.c b/tools/perf/util/data-convert-bt.c
index 5ff46bfcd0e19..e222371394c01 100644
--- a/tools/perf/util/data-convert-bt.c
+++ b/tools/perf/util/data-convert-bt.c
@@ -1362,7 +1362,7 @@ static void cleanup_events(struct perf_session *session)
 		zfree(&evsel->priv);
 	}
 
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	session->evlist = NULL;
 }
 
diff --git a/tools/perf/util/data-convert-json.c b/tools/perf/util/data-convert-json.c
index 40412c3dbdb25..40888b7c44671 100644
--- a/tools/perf/util/data-convert-json.c
+++ b/tools/perf/util/data-convert-json.c
@@ -258,6 +258,7 @@ static int process_sample_event(const struct perf_tool *tool,
 				trace_seq_init(&s);
 				tep_print_field(&s, sample->raw_data, fields[i]);
 				output_json_key_string(out, true, 3, fields[i]->name, s.buffer);
+				trace_seq_destroy(&s);
 
 				i++;
 			}
diff --git a/tools/perf/util/data.c b/tools/perf/util/data.c
index 94dc534a73866..33fd1b82019eb 100644
--- a/tools/perf/util/data.c
+++ b/tools/perf/util/data.c
@@ -345,9 +345,10 @@ static int open_dir(struct perf_data *data)
 	if (asprintf(&data->file.path, "%s/data", data->path) < 0)
 		return -1;
 
-	if (perf_data__is_write(data) &&
-	    mkdir(data->path, S_IRWXU) < 0)
+	if (perf_data__is_write(data) && mkdir(data->path, 0700) < 0) {
+		zfree(&data->file.path);
 		return -1;
+	}
 
 	ret = open_file(data);
 
@@ -360,9 +361,16 @@ static int open_dir(struct perf_data *data)
 
 int perf_data__open(struct perf_data *data)
 {
-	if (check_pipe(data))
+	int ret;
+
+	if (data->open)
 		return 0;
 
+	if (check_pipe(data)) {
+		data->open = true;
+		return 0;
+	}
+
 	/* currently it allows stdio for pipe only */
 	data->file.use_stdio = false;
 
@@ -375,16 +383,24 @@ int perf_data__open(struct perf_data *data)
 	if (perf_data__is_read(data))
 		data->is_dir = is_dir(data);
 
-	return perf_data__is_dir(data) ?
-	       open_dir(data) : open_file_dup(data);
+	ret = perf_data__is_dir(data) ? open_dir(data) : open_file_dup(data);
+
+	if (!ret)
+		data->open = true;
+
+	return ret;
 }
 
 void perf_data__close(struct perf_data *data)
 {
+	if (!data->open)
+		return;
+
 	if (perf_data__is_dir(data))
 		perf_data__close_dir(data);
 
 	perf_data_file__close(&data->file);
+	data->open = false;
 }
 
 static ssize_t perf_data_file__read(struct perf_data_file *file, void *buf, size_t size)
@@ -457,6 +473,7 @@ int perf_data__switch(struct perf_data *data,
 
 	if (!at_exit) {
 		perf_data_file__close(&data->file);
+		data->open = false;
 		ret = perf_data__open(data);
 		if (ret < 0)
 			goto out;
diff --git a/tools/perf/util/data.h b/tools/perf/util/data.h
index 8299fb5fa7dac..76f57f60361fb 100644
--- a/tools/perf/util/data.h
+++ b/tools/perf/util/data.h
@@ -50,6 +50,8 @@ struct perf_data {
 	const char		*path;
 	/** @file: Underlying file to be used. */
 	struct perf_data_file	 file;
+	/** @open: Has the file or directory been opened. */
+	bool			 open;
 	/** @is_pipe: Underlying file is a pipe. */
 	bool			 is_pipe;
 	/** @is_dir: Underlying file is a directory. */
@@ -59,7 +61,7 @@ struct perf_data {
 	/** @in_place_update: A file opened for reading but will be written to. */
 	bool			 in_place_update;
 	/** @mode: Read or write mode. */
-	enum perf_data_mode	 mode;
+	enum perf_data_mode	 mode:8;
 
 	struct {
 		/** @version: perf_dir_version. */
diff --git a/tools/perf/util/dso.h b/tools/perf/util/dso.h
index 2916b954a804b..55c4aaa53c382 100644
--- a/tools/perf/util/dso.h
+++ b/tools/perf/util/dso.h
@@ -9,6 +9,7 @@
 #include <stdbool.h>
 #include <stdio.h>
 #include <linux/bitops.h>
+#include <string.h>
 #include "build-id.h"
 #include "debuginfo.h"
 #include "mutex.h"
@@ -20,6 +21,40 @@ struct perf_env;
 
 #define DSO__NAME_KALLSYMS	"[kernel.kallsyms]"
 #define DSO__NAME_KCORE		"[kernel.kcore]"
+#define DSO__NAME_GUEST_KALLSYMS		"[guest.kernel.kallsyms]"
+#define DSO__NAME_GUEST_KALLSYMS_PID_PREFIX	"[guest.kernel.kallsyms."
+
+/*
+ * Validate names of the form "[guest.kernel.kallsyms.<pid>]", where
+ * <pid> is the PID of the guest VM and varies per guest, so it
+ * cannot be matched with strcmp() against a fixed string.
+ *
+ * Every character after the fixed prefix must be a decimal digit,
+ * with ']' immediately terminating the digit run and nothing
+ * following it. This rules out '/', "..", or any other character
+ * being smuggled into the name.
+ */
+static inline bool is_guest_kallsyms_pid_name(const char *name)
+{
+	const size_t prefix_len = sizeof(DSO__NAME_GUEST_KALLSYMS_PID_PREFIX) - 1;
+	size_t digits;
+
+	if (strncmp(name, DSO__NAME_GUEST_KALLSYMS_PID_PREFIX, prefix_len) != 0)
+		return false;
+
+	digits = strspn(name + prefix_len, "0123456789");
+	if (digits == 0)
+		return false;
+
+	/* ']' must terminate the digit run, with nothing trailing it */
+	if (name[prefix_len + digits] != ']')
+		return false;
+
+	if (name[prefix_len + digits + 1] != '\0')
+		return false;
+
+	return true;
+}
 
 /**
  * enum dso_binary_type - The kind of DSO generally associated with a memory
@@ -924,8 +959,28 @@ static inline bool dso__is_kcore(const struct dso *dso)
 static inline bool dso__is_kallsyms(const struct dso *dso)
 {
 	enum dso_binary_type bt = dso__binary_type(dso);
+	const char *name;
+
+	if (bt == DSO_BINARY_TYPE__KALLSYMS || bt == DSO_BINARY_TYPE__GUEST_KALLSYMS)
+		return true;
+
+	if (bt != DSO_BINARY_TYPE__NOT_FOUND)
+		return false;
+
+	if (!dso__kernel(dso))
+		return false;
+
+	name = dso__long_name(dso);
+	if (!name)
+		return false;
+
+	if (!strcmp(name, DSO__NAME_KALLSYMS))
+		return true;
+
+	if (!strcmp(name, DSO__NAME_GUEST_KALLSYMS))
+		return true;
 
-	return bt == DSO_BINARY_TYPE__KALLSYMS || bt == DSO_BINARY_TYPE__GUEST_KALLSYMS;
+	return is_guest_kallsyms_pid_name(name);
 }
 
 bool dso__is_object_file(const struct dso *dso);
diff --git a/tools/perf/util/evlist.c b/tools/perf/util/evlist.c
index 1a238b245b3a0..eb7c0d7be064e 100644
--- a/tools/perf/util/evlist.c
+++ b/tools/perf/util/evlist.c
@@ -5,67 +5,69 @@
  * Parts came from builtin-{top,stat,record}.c, see those files for further
  * copyright notes.
  */
-#include <api/fs/fs.h>
+#include "evlist.h"
+
 #include <errno.h>
 #include <inttypes.h>
-#include <poll.h>
-#include "cpumap.h"
-#include "util/mmap.h"
-#include "thread_map.h"
-#include "target.h"
-#include "dwarf-regs.h"
-#include "evlist.h"
-#include "evsel.h"
-#include "record.h"
-#include "debug.h"
-#include "units.h"
-#include "bpf_counter.h"
-#include <internal/lib.h> // page_size
-#include "affinity.h"
-#include "../perf.h"
-#include "asm/bug.h"
-#include "bpf-event.h"
-#include "util/event.h"
-#include "util/string2.h"
-#include "util/perf_api_probe.h"
-#include "util/evsel_fprintf.h"
-#include "util/pmu.h"
-#include "util/sample.h"
-#include "util/bpf-filter.h"
-#include "util/stat.h"
-#include "util/util.h"
-#include "util/env.h"
-#include "util/intel-tpebs.h"
-#include "util/metricgroup.h"
-#include "util/strbuf.h"
 #include <signal.h>
-#include <unistd.h>
-#include <sched.h>
 #include <stdlib.h>
 
-#include "parse-events.h"
-#include <subcmd/parse-options.h>
-
 #include <fcntl.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <sys/prctl.h>
-#include <sys/timerfd.h>
-#include <sys/wait.h>
-
 #include <linux/bitops.h>
+#include <linux/err.h>
 #include <linux/hash.h>
 #include <linux/log2.h>
-#include <linux/err.h>
 #include <linux/string.h>
 #include <linux/time64.h>
 #include <linux/zalloc.h>
+#include <poll.h>
+#include <sched.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/prctl.h>
+#include <sys/timerfd.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include <api/fs/fs.h>
+#include <internal/lib.h> // page_size
+#include <internal/rc_check.h>
+#include <internal/xyarray.h>
+#include <perf/cpumap.h>
 #include <perf/evlist.h>
 #include <perf/evsel.h>
-#include <perf/cpumap.h>
 #include <perf/mmap.h>
+#include <subcmd/parse-options.h>
 
-#include <internal/xyarray.h>
+#include "../perf.h"
+#include "affinity.h"
+#include "asm/bug.h"
+#include "bpf-event.h"
+#include "bpf-filter.h"
+#include "bpf_counter.h"
+#include "cpumap.h"
+#include "debug.h"
+#include "dwarf-regs.h"
+#include "env.h"
+#include "event.h"
+#include "evsel.h"
+#include "evsel_fprintf.h"
+#include "intel-tpebs.h"
+#include "metricgroup.h"
+#include "mmap.h"
+#include "parse-events.h"
+#include "perf_api_probe.h"
+#include "pmu.h"
+#include "pmus.h"
+#include "record.h"
+#include "sample.h"
+#include "stat.h"
+#include "strbuf.h"
+#include "string2.h"
+#include "target.h"
+#include "thread_map.h"
+#include "units.h"
+#include "util.h"
 
 #ifdef LACKS_SIGQUEUE_PROTOTYPE
 int sigqueue(pid_t pid, int sig, const union sigval value);
@@ -74,29 +76,31 @@ int sigqueue(pid_t pid, int sig, const union sigval value);
 #define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y))
 #define SID(e, x, y) xyarray__entry(e->core.sample_id, x, y)
 
-void evlist__init(struct evlist *evlist, struct perf_cpu_map *cpus,
-		  struct perf_thread_map *threads)
-{
-	perf_evlist__init(&evlist->core);
-	perf_evlist__set_maps(&evlist->core, cpus, threads);
-	evlist->workload.pid = -1;
-	evlist->bkw_mmap_state = BKW_MMAP_NOTREADY;
-	evlist->ctl_fd.fd = -1;
-	evlist->ctl_fd.ack = -1;
-	evlist->ctl_fd.pos = -1;
-	evlist->nr_br_cntr = -1;
-	metricgroup__rblist_init(&evlist->metric_events);
-	INIT_LIST_HEAD(&evlist->deferred_samples);
-}
+static void event_enable_timer__exit(struct event_enable_timer **ep);
 
 struct evlist *evlist__new(void)
 {
-	struct evlist *evlist = zalloc(sizeof(*evlist));
-
-	if (evlist != NULL)
-		evlist__init(evlist, NULL, NULL);
-
-	return evlist;
+	struct evlist *result;
+	RC_STRUCT(evlist) *evlist;
+
+	evlist = zalloc(sizeof(*evlist));
+	if (ADD_RC_CHK(result, evlist)) {
+		perf_evlist__init(evlist__core(result));
+		perf_evlist__set_maps(evlist__core(result), /*cpus=*/NULL, /*threads=*/NULL);
+		evlist__set_workload_pid(result, -1);
+		evlist__set_bkw_mmap_state(result, BKW_MMAP_NOTREADY);
+		evlist__set_ctl_fd_fd(result, -1);
+		evlist__set_ctl_fd_ack(result, -1);
+		evlist__set_ctl_fd_pos(result, -1);
+		evlist__set_nr_br_cntr(result, -1);
+		metricgroup__rblist_init(evlist__metric_events(result));
+		INIT_LIST_HEAD(&evlist->deferred_samples);
+		refcount_set(evlist__refcnt(result), 1);
+	} else {
+		free(evlist);
+		result = NULL;
+	}
+	return result;
 }
 
 struct evlist *evlist__new_default(const struct target *target, bool sample_callchains)
@@ -104,7 +108,6 @@ struct evlist *evlist__new_default(const struct target *target, bool sample_call
 	struct evlist *evlist = evlist__new();
 	bool can_profile_kernel;
 	struct perf_pmu *pmu = NULL;
-	struct evsel *evsel;
 	char buf[256];
 	int err;
 
@@ -131,14 +134,16 @@ struct evlist *evlist__new_default(const struct target *target, bool sample_call
 	}
 
 	/* If there is only 1 event a sample identifier isn't necessary. */
-	if (evlist->core.nr_entries > 1) {
+	if (evlist__nr_entries(evlist) > 1) {
+		struct evsel *evsel;
+
 		evlist__for_each_entry(evlist, evsel)
 			evsel__set_sample_id(evsel, /*can_sample_identifier=*/false);
 	}
 
 	return evlist;
 out_err:
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	return NULL;
 }
 
@@ -147,13 +152,23 @@ struct evlist *evlist__new_dummy(void)
 	struct evlist *evlist = evlist__new();
 
 	if (evlist && evlist__add_dummy(evlist)) {
-		evlist__delete(evlist);
+		evlist__put(evlist);
 		evlist = NULL;
 	}
 
 	return evlist;
 }
 
+struct evlist *evlist__get(struct evlist *evlist)
+{
+	struct evlist *result;
+
+	if (RC_CHK_GET(result, evlist))
+		refcount_inc(evlist__refcnt(evlist));
+
+	return result;
+}
+
 /**
  * evlist__set_id_pos - set the positions of event ids.
  * @evlist: selected event list
@@ -165,8 +180,8 @@ void evlist__set_id_pos(struct evlist *evlist)
 {
 	struct evsel *first = evlist__first(evlist);
 
-	evlist->id_pos = first->id_pos;
-	evlist->is_pos = first->is_pos;
+	RC_CHK_ACCESS(evlist)->id_pos =  first->id_pos;
+	RC_CHK_ACCESS(evlist)->is_pos =  first->is_pos;
 }
 
 static void evlist__update_id_pos(struct evlist *evlist)
@@ -185,49 +200,85 @@ static void evlist__purge(struct evlist *evlist)
 
 	evlist__for_each_entry_safe(evlist, n, pos) {
 		list_del_init(&pos->core.node);
+		if (pos->evlist) {
+			if (!RC_CHK_EQUAL(pos->evlist, evlist)) {
+				evlist__put(pos->evlist);
+			} else {
+				refcount_dec_and_test(evlist__refcnt(pos->evlist));
+				RC_CHK_PUT(pos->evlist);
+			}
+		}
 		pos->evlist = NULL;
-		evsel__delete(pos);
+		evsel__put(pos);
 	}
 
-	evlist->core.nr_entries = 0;
+	evlist__core(evlist)->nr_entries = 0;
 }
 
-void evlist__exit(struct evlist *evlist)
+static void evlist__exit(struct evlist *evlist)
 {
-	metricgroup__rblist_exit(&evlist->metric_events);
-	event_enable_timer__exit(&evlist->eet);
-	zfree(&evlist->mmap);
-	zfree(&evlist->overwrite_mmap);
-	perf_evlist__exit(&evlist->core);
+	metricgroup__rblist_exit(evlist__metric_events(evlist));
+	event_enable_timer__exit(&RC_CHK_ACCESS(evlist)->eet);
+	free(evlist__mmap(evlist));
+	free(evlist__overwrite_mmap(evlist));
+	perf_evlist__exit(evlist__core(evlist));
 }
 
-void evlist__delete(struct evlist *evlist)
+void evlist__put(struct evlist *evlist)
 {
+	struct evsel *evsel;
+	unsigned int count, ref_cnt;
+
 	if (evlist == NULL)
 		return;
 
+	if (refcount_dec_and_test(evlist__refcnt(evlist)))
+		goto out_delete;
+
+retry:
+	count = refcount_read(evlist__refcnt(evlist));
+	ref_cnt = count;
+	evlist__for_each_entry(evlist, evsel) {
+		if (RC_CHK_EQUAL(evsel->evlist, evlist) && count &&
+		    refcount_read(&evsel->refcnt) == 1)
+			count--;
+	}
+	if (refcount_read(evlist__refcnt(evlist)) != ref_cnt)
+		goto retry;
+
+	if (count != 0) {
+		/*
+		 * Not the last reference except for back references from
+		 * evsels.
+		 */
+		RC_CHK_PUT(evlist);
+		return;
+	}
+out_delete:
 	evlist__free_stats(evlist);
-	evlist__munmap(evlist);
+	evlist__do_munmap(evlist);
 	evlist__close(evlist);
 	evlist__purge(evlist);
 	evlist__exit(evlist);
-	free(evlist);
+	RC_CHK_FREE(evlist);
 }
 
 void evlist__add(struct evlist *evlist, struct evsel *entry)
 {
-	perf_evlist__add(&evlist->core, &entry->core);
-	entry->evlist = evlist;
+	perf_evlist__add(evlist__core(evlist), &entry->core);
+	evlist__put(entry->evlist);
+	entry->evlist = evlist__get(evlist);
 	entry->tracking = !entry->core.idx;
 
-	if (evlist->core.nr_entries == 1)
+	if (evlist__nr_entries(evlist) == 1)
 		evlist__set_id_pos(evlist);
 }
 
 void evlist__remove(struct evlist *evlist, struct evsel *evsel)
 {
+	perf_evlist__remove(evlist__core(evlist), &evsel->core);
+	evlist__put(evsel->evlist);
 	evsel->evlist = NULL;
-	perf_evlist__remove(&evlist->core, &evsel->core);
 }
 
 void evlist__splice_list_tail(struct evlist *evlist, struct list_head *list)
@@ -276,7 +327,7 @@ int __evlist__set_tracepoints_handlers(struct evlist *evlist,
 
 static void evlist__set_leader(struct evlist *evlist)
 {
-	perf_evlist__set_leader(&evlist->core);
+	perf_evlist__set_leader(evlist__core(evlist));
 }
 
 static struct evsel *evlist__dummy_event(struct evlist *evlist)
@@ -290,7 +341,7 @@ static struct evsel *evlist__dummy_event(struct evlist *evlist)
 		.sample_period = 1,
 	};
 
-	return evsel__new_idx(&attr, evlist->core.nr_entries);
+	return evsel__new_idx(&attr, evlist__nr_entries(evlist));
 }
 
 int evlist__add_dummy(struct evlist *evlist)
@@ -379,8 +430,8 @@ static bool evlist__use_affinity(struct evlist *evlist)
 	struct perf_cpu_map *used_cpus = NULL;
 	bool ret = false;
 
-	if (evlist->no_affinity || !evlist->core.user_requested_cpus ||
-	    cpu_map__is_dummy(evlist->core.user_requested_cpus))
+	if (evlist__no_affinity(evlist) || !evlist__core(evlist)->user_requested_cpus ||
+	    cpu_map__is_dummy(evlist__core(evlist)->user_requested_cpus))
 		return false;
 
 	evlist__for_each_entry(evlist, pos) {
@@ -435,7 +486,7 @@ void evlist_cpu_iterator__init(struct evlist_cpu_iterator *itr, struct evlist *e
 		.evsel = NULL,
 		.cpu_map_idx = 0,
 		.evlist_cpu_map_idx = 0,
-		.evlist_cpu_map_nr = perf_cpu_map__nr(evlist->core.all_cpus),
+		.evlist_cpu_map_nr = perf_cpu_map__nr(evlist__core(evlist)->all_cpus),
 		.cpu = (struct perf_cpu){ .cpu = -1},
 		.affinity = NULL,
 	};
@@ -451,7 +502,7 @@ void evlist_cpu_iterator__init(struct evlist_cpu_iterator *itr, struct evlist *e
 			itr->affinity = &itr->saved_affinity;
 	}
 	itr->evsel = evlist__first(evlist);
-	itr->cpu = perf_cpu_map__cpu(evlist->core.all_cpus, 0);
+	itr->cpu = perf_cpu_map__cpu(evlist__core(evlist)->all_cpus, 0);
 	if (itr->affinity)
 		affinity__set(itr->affinity, itr->cpu.cpu);
 	itr->cpu_map_idx = perf_cpu_map__idx(itr->evsel->core.cpus, itr->cpu);
@@ -486,7 +537,7 @@ void evlist_cpu_iterator__next(struct evlist_cpu_iterator *evlist_cpu_itr)
 	if (evlist_cpu_itr->evlist_cpu_map_idx < evlist_cpu_itr->evlist_cpu_map_nr) {
 		evlist_cpu_itr->evsel = evlist__first(evlist_cpu_itr->container);
 		evlist_cpu_itr->cpu =
-			perf_cpu_map__cpu(evlist_cpu_itr->container->core.all_cpus,
+			perf_cpu_map__cpu(evlist__core(evlist_cpu_itr->container)->all_cpus,
 					  evlist_cpu_itr->evlist_cpu_map_idx);
 		if (evlist_cpu_itr->affinity)
 			affinity__set(evlist_cpu_itr->affinity, evlist_cpu_itr->cpu.cpu);
@@ -513,7 +564,7 @@ static int evsel__strcmp(struct evsel *pos, char *evsel_name)
 	return !evsel__name_is(pos, evsel_name);
 }
 
-static int evlist__is_enabled(struct evlist *evlist)
+static bool evlist__is_enabled(struct evlist *evlist)
 {
 	struct evsel *pos;
 
@@ -570,10 +621,7 @@ static void __evlist__disable(struct evlist *evlist, char *evsel_name, bool excl
 	 * If we disabled only single event, we need to check
 	 * the enabled state of the evlist manually.
 	 */
-	if (evsel_name)
-		evlist->enabled = evlist__is_enabled(evlist);
-	else
-		evlist->enabled = false;
+	evlist__set_enabled(evlist, evsel_name ? evlist__is_enabled(evlist) : false);
 }
 
 void evlist__disable(struct evlist *evlist)
@@ -624,7 +672,7 @@ static void __evlist__enable(struct evlist *evlist, char *evsel_name, bool excl_
 	 * so the toggle can work properly and toggle to
 	 * 'disabled' state.
 	 */
-	evlist->enabled = true;
+	evlist__set_enabled(evlist, true);
 }
 
 void evlist__enable(struct evlist *evlist)
@@ -644,23 +692,24 @@ void evlist__enable_evsel(struct evlist *evlist, char *evsel_name)
 
 void evlist__toggle_enable(struct evlist *evlist)
 {
-	(evlist->enabled ? evlist__disable : evlist__enable)(evlist);
+	(evlist__enabled(evlist) ? evlist__disable : evlist__enable)(evlist);
 }
 
 int evlist__add_pollfd(struct evlist *evlist, int fd)
 {
-	return perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN, fdarray_flag__default);
+	return perf_evlist__add_pollfd(evlist__core(evlist), fd, NULL, POLLIN,
+				       fdarray_flag__default);
 }
 
 int evlist__filter_pollfd(struct evlist *evlist, short revents_and_mask)
 {
-	return perf_evlist__filter_pollfd(&evlist->core, revents_and_mask);
+	return perf_evlist__filter_pollfd(evlist__core(evlist), revents_and_mask);
 }
 
 #ifdef HAVE_EVENTFD_SUPPORT
 int evlist__add_wakeup_eventfd(struct evlist *evlist, int fd)
 {
-	return perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN,
+	return perf_evlist__add_pollfd(evlist__core(evlist), fd, NULL, POLLIN,
 				       fdarray_flag__nonfilterable |
 				       fdarray_flag__non_perf_event);
 }
@@ -668,7 +717,7 @@ int evlist__add_wakeup_eventfd(struct evlist *evlist, int fd)
 
 int evlist__poll(struct evlist *evlist, int timeout)
 {
-	return perf_evlist__poll(&evlist->core, timeout);
+	return perf_evlist__poll(evlist__core(evlist), timeout);
 }
 
 struct perf_sample_id *evlist__id2sid(struct evlist *evlist, u64 id)
@@ -678,7 +727,7 @@ struct perf_sample_id *evlist__id2sid(struct evlist *evlist, u64 id)
 	int hash;
 
 	hash = hash_64(id, PERF_EVLIST__HLIST_BITS);
-	head = &evlist->core.heads[hash];
+	head = &evlist__core(evlist)->heads[hash];
 
 	hlist_for_each_entry(sid, head, node)
 		if (sid->id == id)
@@ -691,7 +740,7 @@ struct evsel *evlist__id2evsel(struct evlist *evlist, u64 id)
 {
 	struct perf_sample_id *sid;
 
-	if (evlist->core.nr_entries == 1 || !id)
+	if (evlist__nr_entries(evlist) == 1 || !id)
 		return evlist__first(evlist);
 
 	sid = evlist__id2sid(evlist, id);
@@ -726,13 +775,13 @@ static int evlist__event2id(struct evlist *evlist, union perf_event *event, u64
 	n = (event->header.size - sizeof(event->header)) >> 3;
 
 	if (event->header.type == PERF_RECORD_SAMPLE) {
-		if (evlist->id_pos >= n)
+		if (evlist__id_pos(evlist) >= n)
 			return -1;
-		*id = array[evlist->id_pos];
+		*id = array[evlist__id_pos(evlist)];
 	} else {
-		if (evlist->is_pos > n)
+		if (evlist__is_pos(evlist) > n)
 			return -1;
-		n -= evlist->is_pos;
+		n -= evlist__is_pos(evlist);
 		*id = array[n];
 	}
 	return 0;
@@ -746,7 +795,7 @@ struct evsel *evlist__event2evsel(struct evlist *evlist, union perf_event *event
 	int hash;
 	u64 id;
 
-	if (evlist->core.nr_entries == 1)
+	if (evlist__nr_entries(evlist) == 1)
 		return first;
 
 	if (!first->core.attr.sample_id_all &&
@@ -761,7 +810,7 @@ struct evsel *evlist__event2evsel(struct evlist *evlist, union perf_event *event
 		return first;
 
 	hash = hash_64(id, PERF_EVLIST__HLIST_BITS);
-	head = &evlist->core.heads[hash];
+	head = &evlist__core(evlist)->heads[hash];
 
 	hlist_for_each_entry(sid, head, node) {
 		if (sid->id == id)
@@ -774,11 +823,11 @@ static int evlist__set_paused(struct evlist *evlist, bool value)
 {
 	int i;
 
-	if (!evlist->overwrite_mmap)
+	if (!evlist__overwrite_mmap(evlist))
 		return 0;
 
-	for (i = 0; i < evlist->core.nr_mmaps; i++) {
-		int fd = evlist->overwrite_mmap[i].core.fd;
+	for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) {
+		int fd = evlist__overwrite_mmap(evlist)[i].core.fd;
 		int err;
 
 		if (fd < 0)
@@ -804,20 +853,20 @@ static void evlist__munmap_nofree(struct evlist *evlist)
 {
 	int i;
 
-	if (evlist->mmap)
-		for (i = 0; i < evlist->core.nr_mmaps; i++)
-			perf_mmap__munmap(&evlist->mmap[i].core);
+	if (evlist__mmap(evlist))
+		for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++)
+			perf_mmap__munmap(&evlist__mmap(evlist)[i].core);
 
-	if (evlist->overwrite_mmap)
-		for (i = 0; i < evlist->core.nr_mmaps; i++)
-			perf_mmap__munmap(&evlist->overwrite_mmap[i].core);
+	if (evlist__overwrite_mmap(evlist))
+		for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++)
+			perf_mmap__munmap(&evlist__overwrite_mmap(evlist)[i].core);
 }
 
-void evlist__munmap(struct evlist *evlist)
+void evlist__do_munmap(struct evlist *evlist)
 {
 	evlist__munmap_nofree(evlist);
-	zfree(&evlist->mmap);
-	zfree(&evlist->overwrite_mmap);
+	zfree(&RC_CHK_ACCESS(evlist)->mmap);
+	zfree(&RC_CHK_ACCESS(evlist)->overwrite_mmap);
 }
 
 static void perf_mmap__unmap_cb(struct perf_mmap *map)
@@ -831,12 +880,12 @@ static struct mmap *evlist__alloc_mmap(struct evlist *evlist,
 				       bool overwrite)
 {
 	int i;
-	struct mmap *map = calloc(evlist->core.nr_mmaps, sizeof(struct mmap));
+	struct mmap *map = calloc(evlist__core(evlist)->nr_mmaps, sizeof(struct mmap));
 
 	if (!map)
 		return NULL;
 
-	for (i = 0; i < evlist->core.nr_mmaps; i++) {
+	for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) {
 		struct perf_mmap *prev = i ? &map[i - 1].core : NULL;
 
 		/*
@@ -854,41 +903,73 @@ static struct mmap *evlist__alloc_mmap(struct evlist *evlist,
 	return map;
 }
 
+static struct evlist *from_list_start(struct perf_evlist *core)
+{
+#ifdef REFCNT_CHECKING
+	RC_STRUCT(evlist) *core_evlist = container_of(core, RC_STRUCT(evlist), core);
+	struct evlist *evlist;
+
+	if (ADD_RC_CHK(evlist, core_evlist))
+		refcount_inc(evlist__refcnt(evlist));
+
+	return evlist;
+#else
+	return container_of(core, struct evlist, core);
+#endif
+}
+
+static void from_list_end(struct evlist *evlist __maybe_unused)
+{
+#ifdef REFCNT_CHECKING
+	evlist__put(evlist);
+#endif
+}
+
 static void
 perf_evlist__mmap_cb_idx(struct perf_evlist *_evlist,
 			 struct perf_evsel *_evsel,
 			 struct perf_mmap_param *_mp,
 			 int idx)
 {
-	struct evlist *evlist = container_of(_evlist, struct evlist, core);
+	struct evlist *evlist = from_list_start(_evlist);
 	struct mmap_params *mp = container_of(_mp, struct mmap_params, core);
 	struct evsel *evsel = container_of(_evsel, struct evsel, core);
 
+	if (!evlist)
+		return;
+
 	auxtrace_mmap_params__set_idx(&mp->auxtrace_mp, evlist, evsel, idx);
+
+	from_list_end(evlist);
 }
 
 static struct perf_mmap*
 perf_evlist__mmap_cb_get(struct perf_evlist *_evlist, bool overwrite, int idx)
 {
-	struct evlist *evlist = container_of(_evlist, struct evlist, core);
+	struct evlist *evlist = from_list_start(_evlist);
 	struct mmap *maps;
 
-	maps = overwrite ? evlist->overwrite_mmap : evlist->mmap;
+	if (!evlist)
+		return NULL;
+
+	maps = overwrite ? evlist__overwrite_mmap(evlist) : evlist__mmap(evlist);
 
 	if (!maps) {
 		maps = evlist__alloc_mmap(evlist, overwrite);
-		if (!maps)
+		if (!maps) {
+			from_list_end(evlist);
 			return NULL;
+		}
 
 		if (overwrite) {
-			evlist->overwrite_mmap = maps;
-			if (evlist->bkw_mmap_state == BKW_MMAP_NOTREADY)
+			RC_CHK_ACCESS(evlist)->overwrite_mmap = maps;
+			if (evlist__bkw_mmap_state(evlist) == BKW_MMAP_NOTREADY)
 				evlist__toggle_bkw_mmap(evlist, BKW_MMAP_RUNNING);
 		} else {
-			evlist->mmap = maps;
+			RC_CHK_ACCESS(evlist)->mmap = maps;
 		}
 	}
-
+	from_list_end(evlist);
 	return &maps[idx].core;
 }
 
@@ -1045,16 +1126,16 @@ int evlist__mmap_ex(struct evlist *evlist, unsigned int pages,
 		.mmap = perf_evlist__mmap_cb_mmap,
 	};
 
-	evlist->core.mmap_len = evlist__mmap_size(pages);
-	pr_debug("mmap size %zuB\n", evlist->core.mmap_len);
+	evlist__core(evlist)->mmap_len = evlist__mmap_size(pages);
+	pr_debug("mmap size %zuB\n", evlist__core(evlist)->mmap_len);
 
-	auxtrace_mmap_params__init(&mp.auxtrace_mp, evlist->core.mmap_len,
+	auxtrace_mmap_params__init(&mp.auxtrace_mp, evlist__core(evlist)->mmap_len,
 				   auxtrace_pages, auxtrace_overwrite);
 
-	return perf_evlist__mmap_ops(&evlist->core, &ops, &mp.core);
+	return perf_evlist__mmap_ops(evlist__core(evlist), &ops, &mp.core);
 }
 
-int evlist__mmap(struct evlist *evlist, unsigned int pages)
+int evlist__do_mmap(struct evlist *evlist, unsigned int pages)
 {
 	return evlist__mmap_ex(evlist, pages, 0, false, 0, PERF_AFFINITY_SYS, 1, 0);
 }
@@ -1096,9 +1177,9 @@ int evlist__create_maps(struct evlist *evlist, struct target *target)
 	if (!cpus)
 		goto out_delete_threads;
 
-	evlist->core.has_user_cpus = !!target->cpu_list;
+	evlist__core(evlist)->has_user_cpus = !!target->cpu_list;
 
-	perf_evlist__set_maps(&evlist->core, cpus, threads);
+	perf_evlist__set_maps(evlist__core(evlist), cpus, threads);
 
 	/* as evlist now has references, put count here */
 	perf_cpu_map__put(cpus);
@@ -1238,15 +1319,15 @@ bool evlist__valid_sample_type(struct evlist *evlist)
 {
 	struct evsel *pos;
 
-	if (evlist->core.nr_entries == 1)
+	if (evlist__nr_entries(evlist) == 1)
 		return true;
 
-	if (evlist->id_pos < 0 || evlist->is_pos < 0)
+	if (evlist__id_pos(evlist) < 0 || evlist__is_pos(evlist) < 0)
 		return false;
 
 	evlist__for_each_entry(evlist, pos) {
-		if (pos->id_pos != evlist->id_pos ||
-		    pos->is_pos != evlist->is_pos)
+		if (pos->id_pos != evlist__id_pos(evlist) ||
+		    pos->is_pos != evlist__is_pos(evlist))
 			return false;
 	}
 
@@ -1257,18 +1338,18 @@ u64 __evlist__combined_sample_type(struct evlist *evlist)
 {
 	struct evsel *evsel;
 
-	if (evlist->combined_sample_type)
-		return evlist->combined_sample_type;
+	if (RC_CHK_ACCESS(evlist)->combined_sample_type)
+		return RC_CHK_ACCESS(evlist)->combined_sample_type;
 
 	evlist__for_each_entry(evlist, evsel)
-		evlist->combined_sample_type |= evsel->core.attr.sample_type;
+		RC_CHK_ACCESS(evlist)->combined_sample_type |= evsel->core.attr.sample_type;
 
-	return evlist->combined_sample_type;
+	return RC_CHK_ACCESS(evlist)->combined_sample_type;
 }
 
 u64 evlist__combined_sample_type(struct evlist *evlist)
 {
-	evlist->combined_sample_type = 0;
+	RC_CHK_ACCESS(evlist)->combined_sample_type = 0;
 	return __evlist__combined_sample_type(evlist);
 }
 
@@ -1345,7 +1426,7 @@ void evlist__update_br_cntr(struct evlist *evlist)
 				evlist__new_abbr_name(evsel->abbr_name);
 		}
 	}
-	evlist->nr_br_cntr = i;
+	evlist__set_nr_br_cntr(evlist, i);
 }
 
 bool evlist__valid_read_format(struct evlist *evlist)
@@ -1395,11 +1476,6 @@ bool evlist__sample_id_all(struct evlist *evlist)
 	return first->core.attr.sample_id_all;
 }
 
-void evlist__set_selected(struct evlist *evlist, struct evsel *evsel)
-{
-	evlist->selected = evsel;
-}
-
 void evlist__close(struct evlist *evlist)
 {
 	struct evsel *evsel;
@@ -1416,7 +1492,7 @@ void evlist__close(struct evlist *evlist)
 		perf_evsel__free_fd(&evsel->core);
 		perf_evsel__free_id(&evsel->core);
 	}
-	perf_evlist__reset_id_hash(&evlist->core);
+	perf_evlist__reset_id_hash(evlist__core(evlist));
 }
 
 static int evlist__create_syswide_maps(struct evlist *evlist)
@@ -1443,7 +1519,7 @@ static int evlist__create_syswide_maps(struct evlist *evlist)
 		return -ENOMEM;
 	}
 
-	perf_evlist__set_maps(&evlist->core, cpus, threads);
+	perf_evlist__set_maps(evlist__core(evlist), cpus, threads);
 	perf_thread_map__put(threads);
 	perf_cpu_map__put(cpus);
 	return 0;
@@ -1458,7 +1534,8 @@ int evlist__open(struct evlist *evlist)
 	 * Default: one fd per CPU, all threads, aka systemwide
 	 * as sys_perf_event_open(cpu = -1, thread = -1) is EINVAL
 	 */
-	if (evlist->core.threads == NULL && evlist->core.user_requested_cpus == NULL) {
+	if (evlist__core(evlist)->threads == NULL &&
+	    evlist__core(evlist)->user_requested_cpus == NULL) {
 		err = evlist__create_syswide_maps(evlist);
 		if (err < 0)
 			goto out_err;
@@ -1485,7 +1562,7 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const
 	int child_ready_pipe[2], go_pipe[2];
 	char bf;
 
-	evlist->workload.cork_fd = -1;
+	evlist__set_workload_cork_fd(evlist, -1);
 
 	if (pipe(child_ready_pipe) < 0) {
 		perror("failed to create 'ready' pipe");
@@ -1497,13 +1574,13 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const
 		goto out_close_ready_pipe;
 	}
 
-	evlist->workload.pid = fork();
-	if (evlist->workload.pid < 0) {
+	evlist__set_workload_pid(evlist, fork());
+	if (evlist__workload_pid(evlist) < 0) {
 		perror("failed to fork");
 		goto out_close_pipes;
 	}
 
-	if (!evlist->workload.pid) {
+	if (!evlist__workload_pid(evlist)) {
 		int ret;
 
 		if (pipe_output)
@@ -1569,12 +1646,13 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const
 	}
 
 	if (target__none(target)) {
-		if (evlist->core.threads == NULL) {
+		if (evlist__core(evlist)->threads == NULL) {
 			fprintf(stderr, "FATAL: evlist->threads need to be set at this point (%s:%d).\n",
 				__func__, __LINE__);
 			goto out_close_pipes;
 		}
-		perf_thread_map__set_pid(evlist->core.threads, 0, evlist->workload.pid);
+		perf_thread_map__set_pid(evlist__core(evlist)->threads, 0,
+					 evlist__workload_pid(evlist));
 	}
 
 	close(child_ready_pipe[1]);
@@ -1588,7 +1666,7 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const
 	}
 
 	fcntl(go_pipe[1], F_SETFD, FD_CLOEXEC);
-	evlist->workload.cork_fd = go_pipe[1];
+	evlist__set_workload_cork_fd(evlist, go_pipe[1]);
 	close(child_ready_pipe[0]);
 	return 0;
 
@@ -1603,18 +1681,18 @@ int evlist__prepare_workload(struct evlist *evlist, struct target *target, const
 
 int evlist__start_workload(struct evlist *evlist)
 {
-	if (evlist->workload.cork_fd >= 0) {
+	if (evlist__workload_cork_fd(evlist) >= 0) {
 		char bf = 0;
 		int ret;
 		/*
 		 * Remove the cork, let it rip!
 		 */
-		ret = write(evlist->workload.cork_fd, &bf, 1);
+		ret = write(evlist__workload_cork_fd(evlist), &bf, 1);
 		if (ret < 0)
 			perror("unable to write to pipe");
 
-		close(evlist->workload.cork_fd);
-		evlist->workload.cork_fd = -1;
+		close(evlist__workload_cork_fd(evlist));
+		evlist__set_workload_cork_fd(evlist, -1);
 		return ret;
 	}
 
@@ -1625,10 +1703,10 @@ void evlist__cancel_workload(struct evlist *evlist)
 {
 	int status;
 
-	if (evlist->workload.cork_fd >= 0) {
-		close(evlist->workload.cork_fd);
-		evlist->workload.cork_fd = -1;
-		waitpid(evlist->workload.pid, &status, WNOHANG);
+	if (evlist__workload_cork_fd(evlist) >= 0) {
+		close(evlist__workload_cork_fd(evlist));
+		evlist__set_workload_cork_fd(evlist, -1);
+		waitpid(evlist__workload_pid(evlist), &status, WNOHANG);
 	}
 }
 
@@ -1722,7 +1800,8 @@ int evlist__strerror_open(struct evlist *evlist, int err, char *buf, size_t size
 
 int evlist__strerror_mmap(struct evlist *evlist, int err, char *buf, size_t size)
 {
-	int pages_attempted = evlist->core.mmap_len / 1024, pages_max_per_user, printed = 0;
+	int pages_attempted = evlist__core(evlist)->mmap_len / 1024;
+	int pages_max_per_user, printed = 0;
 
 	switch (err) {
 	case EPERM:
@@ -1765,7 +1844,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel)
 			list_move_tail(&evsel->core.node, &move);
 	}
 
-	list_splice(&move, &evlist->core.entries);
+	list_splice(&move, &evlist__core(evlist)->entries);
 }
 
 struct evsel *evlist__get_tracking_event(struct evlist *evlist)
@@ -1807,7 +1886,7 @@ struct evsel *evlist__findnew_tracking_event(struct evlist *evlist, bool system_
 
 		evlist__set_tracking_event(evlist, evsel);
 	} else if (system_wide) {
-		perf_evlist__go_system_wide(&evlist->core, &evsel->core);
+		perf_evlist__go_system_wide(evlist__core(evlist), &evsel->core);
 	}
 
 	return evsel;
@@ -1829,14 +1908,14 @@ struct evsel *evlist__find_evsel_by_str(struct evlist *evlist, const char *str)
 
 void evlist__toggle_bkw_mmap(struct evlist *evlist, enum bkw_mmap_state state)
 {
-	enum bkw_mmap_state old_state = evlist->bkw_mmap_state;
+	enum bkw_mmap_state old_state = evlist__bkw_mmap_state(evlist);
 	enum action {
 		NONE,
 		PAUSE,
 		RESUME,
 	} action = NONE;
 
-	if (!evlist->overwrite_mmap)
+	if (!evlist__overwrite_mmap(evlist))
 		return;
 
 	switch (old_state) {
@@ -1866,7 +1945,7 @@ void evlist__toggle_bkw_mmap(struct evlist *evlist, enum bkw_mmap_state state)
 		WARN_ONCE(1, "Shouldn't get there\n");
 	}
 
-	evlist->bkw_mmap_state = state;
+	evlist__set_bkw_mmap_state(evlist, state);
 
 	switch (action) {
 	case PAUSE:
@@ -2044,40 +2123,41 @@ int evlist__initialize_ctlfd(struct evlist *evlist, int fd, int ack)
 		return 0;
 	}
 
-	evlist->ctl_fd.pos = perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN,
-						     fdarray_flag__nonfilterable |
-						     fdarray_flag__non_perf_event);
-	if (evlist->ctl_fd.pos < 0) {
-		evlist->ctl_fd.pos = -1;
+	evlist__set_ctl_fd_pos(evlist,
+			       perf_evlist__add_pollfd(evlist__core(evlist), fd, NULL, POLLIN,
+						       fdarray_flag__nonfilterable |
+						       fdarray_flag__non_perf_event));
+	if (evlist__ctl_fd_pos(evlist) < 0) {
+		evlist__set_ctl_fd_pos(evlist, -1);
 		pr_err("Failed to add ctl fd entry: %m\n");
 		return -1;
 	}
 
-	evlist->ctl_fd.fd = fd;
-	evlist->ctl_fd.ack = ack;
+	evlist__set_ctl_fd_fd(evlist, fd);
+	evlist__set_ctl_fd_ack(evlist, ack);
 
 	return 0;
 }
 
 bool evlist__ctlfd_initialized(struct evlist *evlist)
 {
-	return evlist->ctl_fd.pos >= 0;
+	return evlist__ctl_fd_pos(evlist) >= 0;
 }
 
 int evlist__finalize_ctlfd(struct evlist *evlist)
 {
-	struct pollfd *entries = evlist->core.pollfd.entries;
+	struct pollfd *entries = evlist__core(evlist)->pollfd.entries;
 
 	if (!evlist__ctlfd_initialized(evlist))
 		return 0;
 
-	entries[evlist->ctl_fd.pos].fd = -1;
-	entries[evlist->ctl_fd.pos].events = 0;
-	entries[evlist->ctl_fd.pos].revents = 0;
+	entries[evlist__ctl_fd_pos(evlist)].fd = -1;
+	entries[evlist__ctl_fd_pos(evlist)].events = 0;
+	entries[evlist__ctl_fd_pos(evlist)].revents = 0;
 
-	evlist->ctl_fd.pos = -1;
-	evlist->ctl_fd.ack = -1;
-	evlist->ctl_fd.fd = -1;
+	evlist__set_ctl_fd_pos(evlist, -1);
+	evlist__set_ctl_fd_ack(evlist, -1);
+	evlist__set_ctl_fd_fd(evlist, -1);
 
 	return 0;
 }
@@ -2094,7 +2174,7 @@ static int evlist__ctlfd_recv(struct evlist *evlist, enum evlist_ctl_cmd *cmd,
 	data_size--;
 
 	do {
-		err = read(evlist->ctl_fd.fd, &c, 1);
+		err = read(evlist__ctl_fd_fd(evlist), &c, 1);
 		if (err > 0) {
 			if (c == '\n' || c == '\0')
 				break;
@@ -2108,7 +2188,8 @@ static int evlist__ctlfd_recv(struct evlist *evlist, enum evlist_ctl_cmd *cmd,
 			if (errno == EAGAIN || errno == EWOULDBLOCK)
 				err = 0;
 			else
-				pr_err("Failed to read from ctlfd %d: %m\n", evlist->ctl_fd.fd);
+				pr_err("Failed to read from ctlfd %d: %m\n",
+				       evlist__ctl_fd_fd(evlist));
 		}
 		break;
 	} while (1);
@@ -2146,13 +2227,13 @@ int evlist__ctlfd_ack(struct evlist *evlist)
 {
 	int err;
 
-	if (evlist->ctl_fd.ack == -1)
+	if (evlist__ctl_fd_ack(evlist) == -1)
 		return 0;
 
-	err = write(evlist->ctl_fd.ack, EVLIST_CTL_CMD_ACK_TAG,
+	err = write(evlist__ctl_fd_ack(evlist), EVLIST_CTL_CMD_ACK_TAG,
 		    sizeof(EVLIST_CTL_CMD_ACK_TAG));
 	if (err == -1)
-		pr_err("failed to write to ctl_ack_fd %d: %m\n", evlist->ctl_fd.ack);
+		pr_err("failed to write to ctl_ack_fd %d: %m\n", evlist__ctl_fd_ack(evlist));
 
 	return err;
 }
@@ -2253,8 +2334,8 @@ int evlist__ctlfd_process(struct evlist *evlist, enum evlist_ctl_cmd *cmd)
 {
 	int err = 0;
 	char cmd_data[EVLIST_CTL_CMD_MAX_LEN];
-	int ctlfd_pos = evlist->ctl_fd.pos;
-	struct pollfd *entries = evlist->core.pollfd.entries;
+	int ctlfd_pos = evlist__ctl_fd_pos(evlist);
+	struct pollfd *entries = evlist__core(evlist)->pollfd.entries;
 
 	if (!evlist__ctlfd_initialized(evlist) || !entries[ctlfd_pos].revents)
 		return 0;
@@ -2425,14 +2506,15 @@ int evlist__parse_event_enable_time(struct evlist *evlist, struct record_opts *o
 		goto free_eet_times;
 	}
 
-	eet->pollfd_pos = perf_evlist__add_pollfd(&evlist->core, eet->timerfd, NULL, POLLIN, flags);
+	eet->pollfd_pos = perf_evlist__add_pollfd(evlist__core(evlist), eet->timerfd,
+						  NULL, POLLIN, flags);
 	if (eet->pollfd_pos < 0) {
 		err = eet->pollfd_pos;
 		goto close_timerfd;
 	}
 
 	eet->evlist = evlist;
-	evlist->eet = eet;
+	RC_CHK_ACCESS(evlist)->eet = eet;
 	opts->target.initial_delay = eet->times[0].start;
 
 	return 0;
@@ -2482,7 +2564,7 @@ int event_enable_timer__process(struct event_enable_timer *eet)
 	if (!eet)
 		return 0;
 
-	entries = eet->evlist->core.pollfd.entries;
+	entries = evlist__core(eet->evlist)->pollfd.entries;
 	revents = entries[eet->pollfd_pos].revents;
 	entries[eet->pollfd_pos].revents = 0;
 
@@ -2518,7 +2600,7 @@ int event_enable_timer__process(struct event_enable_timer *eet)
 	return 0;
 }
 
-void event_enable_timer__exit(struct event_enable_timer **ep)
+static void event_enable_timer__exit(struct event_enable_timer **ep)
 {
 	if (!ep || !*ep)
 		return;
@@ -2622,7 +2704,7 @@ void evlist__warn_user_requested_cpus(struct evlist *evlist, const char *cpu_lis
 }
 
 /* Should uniquify be disabled for the evlist? */
-static bool evlist__disable_uniquify(const struct evlist *evlist)
+static bool evlist__disable_uniquify(struct evlist *evlist)
 {
 	struct evsel *counter;
 	struct perf_pmu *last_pmu = NULL;
diff --git a/tools/perf/util/evlist.h b/tools/perf/util/evlist.h
index e507f5f20ef61..838e263b76f39 100644
--- a/tools/perf/util/evlist.h
+++ b/tools/perf/util/evlist.h
@@ -2,29 +2,33 @@
 #ifndef __PERF_EVLIST_H
 #define __PERF_EVLIST_H 1
 
+#include <signal.h>
+
 #include <linux/compiler.h>
 #include <linux/kernel.h>
-#include <linux/refcount.h>
 #include <linux/list.h>
+#include <linux/refcount.h>
+#include <pthread.h>
+#include <unistd.h>
+
 #include <api/fd/array.h>
 #include <internal/evlist.h>
 #include <internal/evsel.h>
+#include <internal/rc_check.h>
 #include <perf/evlist.h>
+
 #include "affinity.h"
 #include "events_stats.h"
 #include "evsel.h"
 #include "rblist.h"
-#include <pthread.h>
-#include <signal.h>
-#include <unistd.h>
 
-struct pollfd;
-struct thread_map;
 struct perf_cpu_map;
 struct perf_stat_config;
+struct pollfd;
 struct record_opts;
 struct strbuf;
 struct target;
+struct thread_map;
 
 /*
  * State machine of bkw_mmap_state:
@@ -56,8 +60,9 @@ enum bkw_mmap_state {
 
 struct event_enable_timer;
 
-struct evlist {
+DECLARE_RC_STRUCT(evlist) {
 	struct perf_evlist core;
+	refcount_t	 refcnt;
 	bool		 enabled;
 	bool		 no_affinity;
 	int		 id_pos;
@@ -82,7 +87,7 @@ struct evlist {
 	struct {
 		pthread_t		th;
 		volatile int		done;
-	} thread;
+	} sb_thread;
 	struct {
 		int	fd;	/* control file descriptor */
 		int	ack;	/* ack file descriptor for control commands */
@@ -103,13 +108,232 @@ struct evsel_str_handler {
 	void	   *handler;
 };
 
+static inline struct perf_evlist *evlist__core(struct evlist *evlist)
+{
+	return &RC_CHK_ACCESS(evlist)->core;
+}
+
+static inline const struct perf_evlist *evlist__const_core(const struct evlist *evlist)
+{
+	return &RC_CHK_ACCESS(evlist)->core;
+}
+
+static inline int evlist__nr_entries(const struct evlist *evlist)
+{
+	return evlist__const_core(evlist)->nr_entries;
+}
+
+static inline bool evlist__enabled(const struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->enabled;
+}
+
+static inline void evlist__set_enabled(struct evlist *evlist, bool enabled)
+{
+	RC_CHK_ACCESS(evlist)->enabled = enabled;
+}
+
+static inline bool evlist__no_affinity(const struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->no_affinity;
+}
+
+static inline void evlist__set_no_affinity(struct evlist *evlist, bool no_affinity)
+{
+	RC_CHK_ACCESS(evlist)->no_affinity = no_affinity;
+}
+
+static inline int evlist__sb_thread_done(const struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->sb_thread.done;
+}
+
+static inline void evlist__set_sb_thread_done(struct evlist *evlist, int done)
+{
+	RC_CHK_ACCESS(evlist)->sb_thread.done = done;
+}
+
+static inline pthread_t *evlist__sb_thread_th(struct evlist *evlist)
+{
+	return &RC_CHK_ACCESS(evlist)->sb_thread.th;
+}
+
+static inline int evlist__id_pos(const struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->id_pos;
+}
+
+static inline int evlist__is_pos(const struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->is_pos;
+}
+
+static inline struct event_enable_timer *evlist__event_enable_timer(struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->eet;
+}
+
+static inline enum bkw_mmap_state evlist__bkw_mmap_state(const struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->bkw_mmap_state;
+}
+
+static inline void evlist__set_bkw_mmap_state(struct evlist *evlist, enum bkw_mmap_state state)
+{
+	RC_CHK_ACCESS(evlist)->bkw_mmap_state = state;
+}
+
+static inline struct mmap *evlist__mmap(struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->mmap;
+}
+
+static inline struct mmap *evlist__overwrite_mmap(struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->overwrite_mmap;
+}
+
+static inline struct events_stats *evlist__stats(struct evlist *evlist)
+{
+	return &RC_CHK_ACCESS(evlist)->stats;
+}
+
+static inline u64 evlist__first_sample_time(const struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->first_sample_time;
+}
+
+static inline void evlist__set_first_sample_time(struct evlist *evlist, u64 first)
+{
+	RC_CHK_ACCESS(evlist)->first_sample_time = first;
+}
+
+static inline u64 evlist__last_sample_time(const struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->last_sample_time;
+}
+
+static inline void evlist__set_last_sample_time(struct evlist *evlist, u64 last)
+{
+	RC_CHK_ACCESS(evlist)->last_sample_time = last;
+}
+
+static inline int evlist__nr_br_cntr(const struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->nr_br_cntr;
+}
+
+static inline void evlist__set_nr_br_cntr(struct evlist *evlist, int nr)
+{
+	RC_CHK_ACCESS(evlist)->nr_br_cntr = nr;
+}
+
+static inline struct perf_session *evlist__session(struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->session;
+}
+
+static inline void evlist__set_session(struct evlist *evlist, struct perf_session *session)
+{
+	RC_CHK_ACCESS(evlist)->session = session;
+}
+
+static inline void (*evlist__trace_event_sample_raw(struct evlist *evlist))
+			(struct evlist *evlist,
+			 union perf_event *event,
+			 struct perf_sample *sample)
+{
+	return RC_CHK_ACCESS(evlist)->trace_event_sample_raw;
+}
+
+static inline void evlist__set_trace_event_sample_raw(struct evlist *evlist,
+						void (*fun)(struct evlist *evlist,
+							union perf_event *event,
+							struct perf_sample *sample))
+{
+	RC_CHK_ACCESS(evlist)->trace_event_sample_raw = fun;
+}
+
+static inline pid_t evlist__workload_pid(const struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->workload.pid;
+}
+
+static inline void evlist__set_workload_pid(struct evlist *evlist, pid_t pid)
+{
+	RC_CHK_ACCESS(evlist)->workload.pid = pid;
+}
+
+static inline int evlist__workload_cork_fd(const struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->workload.cork_fd;
+}
+
+static inline void evlist__set_workload_cork_fd(struct evlist *evlist, int cork_fd)
+{
+	RC_CHK_ACCESS(evlist)->workload.cork_fd = cork_fd;
+}
+
+static inline int evlist__ctl_fd_fd(const struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->ctl_fd.fd;
+}
+
+static inline void evlist__set_ctl_fd_fd(struct evlist *evlist, int fd)
+{
+	RC_CHK_ACCESS(evlist)->ctl_fd.fd = fd;
+}
+
+static inline int evlist__ctl_fd_ack(const struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->ctl_fd.ack;
+}
+
+static inline void evlist__set_ctl_fd_ack(struct evlist *evlist, int ack)
+{
+	RC_CHK_ACCESS(evlist)->ctl_fd.ack = ack;
+}
+
+static inline int evlist__ctl_fd_pos(const struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->ctl_fd.pos;
+}
+
+static inline void evlist__set_ctl_fd_pos(struct evlist *evlist, int pos)
+{
+	RC_CHK_ACCESS(evlist)->ctl_fd.pos = pos;
+}
+
+static inline refcount_t *evlist__refcnt(struct evlist *evlist)
+{
+	return &RC_CHK_ACCESS(evlist)->refcnt;
+}
+
+static inline struct rblist *evlist__metric_events(struct evlist *evlist)
+{
+	return &RC_CHK_ACCESS(evlist)->metric_events;
+}
+
+static inline struct list_head *evlist__deferred_samples(struct evlist *evlist)
+{
+	return &RC_CHK_ACCESS(evlist)->deferred_samples;
+}
+
+static inline struct evsel *evlist__selected(struct evlist *evlist)
+{
+	return RC_CHK_ACCESS(evlist)->selected;
+}
+
+static inline void evlist__set_selected(struct evlist *evlist, struct evsel *evsel)
+{
+	RC_CHK_ACCESS(evlist)->selected = evsel;
+}
+
 struct evlist *evlist__new(void);
 struct evlist *evlist__new_default(const struct target *target, bool sample_callchains);
 struct evlist *evlist__new_dummy(void);
-void evlist__init(struct evlist *evlist, struct perf_cpu_map *cpus,
-		  struct perf_thread_map *threads);
-void evlist__exit(struct evlist *evlist);
-void evlist__delete(struct evlist *evlist);
+struct evlist *evlist__get(struct evlist *evlist);
+void evlist__put(struct evlist *evlist);
 
 void evlist__add(struct evlist *evlist, struct evsel *entry);
 void evlist__remove(struct evlist *evlist, struct evsel *evsel);
@@ -198,8 +422,8 @@ int evlist__mmap_ex(struct evlist *evlist, unsigned int pages,
 			 unsigned int auxtrace_pages,
 			 bool auxtrace_overwrite, int nr_cblocks,
 			 int affinity, int flush, int comp_level);
-int evlist__mmap(struct evlist *evlist, unsigned int pages);
-void evlist__munmap(struct evlist *evlist);
+int evlist__do_mmap(struct evlist *evlist, unsigned int pages);
+void evlist__do_munmap(struct evlist *evlist);
 
 size_t evlist__mmap_size(unsigned long pages);
 
@@ -211,8 +435,6 @@ void evlist__enable_evsel(struct evlist *evlist, char *evsel_name);
 void evlist__disable_non_dummy(struct evlist *evlist);
 void evlist__enable_non_dummy(struct evlist *evlist);
 
-void evlist__set_selected(struct evlist *evlist, struct evsel *evsel);
-
 int evlist__create_maps(struct evlist *evlist, struct target *target);
 int evlist__apply_filters(struct evlist *evlist, struct evsel **err_evsel,
 			  struct target *target);
@@ -235,26 +457,26 @@ void evlist__splice_list_tail(struct evlist *evlist, struct list_head *list);
 
 static inline bool evlist__empty(struct evlist *evlist)
 {
-	return list_empty(&evlist->core.entries);
+	return list_empty(&evlist__core(evlist)->entries);
 }
 
 static inline struct evsel *evlist__first(struct evlist *evlist)
 {
-	struct perf_evsel *evsel = perf_evlist__first(&evlist->core);
+	struct perf_evsel *evsel = perf_evlist__first(evlist__core(evlist));
 
 	return container_of(evsel, struct evsel, core);
 }
 
 static inline struct evsel *evlist__last(struct evlist *evlist)
 {
-	struct perf_evsel *evsel = perf_evlist__last(&evlist->core);
+	struct perf_evsel *evsel = perf_evlist__last(evlist__core(evlist));
 
 	return container_of(evsel, struct evsel, core);
 }
 
 static inline int evlist__nr_groups(struct evlist *evlist)
 {
-	return perf_evlist__nr_groups(&evlist->core);
+	return perf_evlist__nr_groups(evlist__core(evlist));
 }
 
 int evlist__strerror_open(struct evlist *evlist, int err, char *buf, size_t size);
@@ -277,7 +499,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel);
  * @evsel: struct evsel iterator
  */
 #define evlist__for_each_entry(evlist, evsel) \
-	__evlist__for_each_entry(&(evlist)->core.entries, evsel)
+	__evlist__for_each_entry(&evlist__core(evlist)->entries, evsel)
 
 /**
  * __evlist__for_each_entry_continue - continue iteration thru all the evsels
@@ -293,7 +515,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel);
  * @evsel: struct evsel iterator
  */
 #define evlist__for_each_entry_continue(evlist, evsel) \
-	__evlist__for_each_entry_continue(&(evlist)->core.entries, evsel)
+	__evlist__for_each_entry_continue(&evlist__core(evlist)->entries, evsel)
 
 /**
  * __evlist__for_each_entry_from - continue iteration from @evsel (included)
@@ -309,7 +531,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel);
  * @evsel: struct evsel iterator
  */
 #define evlist__for_each_entry_from(evlist, evsel) \
-	__evlist__for_each_entry_from(&(evlist)->core.entries, evsel)
+	__evlist__for_each_entry_from(&evlist__core(evlist)->entries, evsel)
 
 /**
  * __evlist__for_each_entry_reverse - iterate thru all the evsels in reverse order
@@ -325,7 +547,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel);
  * @evsel: struct evsel iterator
  */
 #define evlist__for_each_entry_reverse(evlist, evsel) \
-	__evlist__for_each_entry_reverse(&(evlist)->core.entries, evsel)
+	__evlist__for_each_entry_reverse(&evlist__core(evlist)->entries, evsel)
 
 /**
  * __evlist__for_each_entry_safe - safely iterate thru all the evsels
@@ -343,7 +565,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel);
  * @tmp: struct evsel temp iterator
  */
 #define evlist__for_each_entry_safe(evlist, tmp, evsel) \
-	__evlist__for_each_entry_safe(&(evlist)->core.entries, tmp, evsel)
+	__evlist__for_each_entry_safe(&evlist__core(evlist)->entries, tmp, evsel)
 
 /** Iterator state for evlist__for_each_cpu */
 struct evlist_cpu_iterator {
@@ -449,7 +671,6 @@ int evlist__ctlfd_ack(struct evlist *evlist);
 int evlist__parse_event_enable_time(struct evlist *evlist, struct record_opts *opts,
 				    const char *str, int unset);
 int event_enable_timer__start(struct event_enable_timer *eet);
-void event_enable_timer__exit(struct event_enable_timer **ep);
 int event_enable_timer__process(struct event_enable_timer *eet);
 
 struct evsel *evlist__find_evsel(struct evlist *evlist, int idx);
diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c
index ea9fa04429f08..3404c40dee34e 100644
--- a/tools/perf/util/evsel.c
+++ b/tools/perf/util/evsel.c
@@ -422,6 +422,11 @@ void evsel__set_sample_id(struct evsel *evsel,
 	evsel->core.attr.read_format |= PERF_FORMAT_ID;
 }
 
+bool evsel__is_non_perf_event_open_pmu(const struct evsel *evsel)
+{
+	return evsel->pmu && evsel->pmu->type > PERF_PMU_TYPE_PE_END;
+}
+
 /**
  * evsel__is_function_event - Return whether given evsel is a function
  * trace event
@@ -440,10 +445,11 @@ bool evsel__is_function_event(struct evsel *evsel)
 #undef FUNCTION_EVENT
 }
 
-void evsel__init(struct evsel *evsel,
+static void evsel__init(struct evsel *evsel,
 		 struct perf_event_attr *attr, int idx)
 {
 	perf_evsel__init(&evsel->core, attr, idx);
+	refcount_set(&evsel->refcnt, 1);
 	evsel->tracking	   = !idx;
 	evsel->unit	   = strdup("");
 	evsel->scale	   = 1.0;
@@ -525,7 +531,7 @@ static int evsel__copy_config_terms(struct evsel *dst, struct evsel *src)
  * The assumption is that @orig is not configured nor opened yet.
  * So we only care about the attributes that can be set while it's parsed.
  */
-struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig)
+struct evsel *evsel__clone(struct evsel *orig)
 {
 	struct evsel *evsel;
 
@@ -538,11 +544,7 @@ struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig)
 	if (orig->bpf_obj)
 		return NULL;
 
-	if (dest)
-		evsel = dest;
-	else
-		evsel = evsel__new(&orig->core.attr);
-
+	evsel = evsel__new(&orig->core.attr);
 	if (evsel == NULL)
 		return NULL;
 
@@ -627,7 +629,7 @@ struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig)
 	return evsel;
 
 out_err:
-	evsel__delete(evsel);
+	evsel__put(evsel);
 	return NULL;
 }
 
@@ -686,6 +688,12 @@ struct evsel *evsel__newtp_idx(const char *sys, const char *name, int idx, bool
 	return ERR_PTR(err);
 }
 
+struct evsel *evsel__get(struct evsel *evsel)
+{
+	refcount_inc(&evsel->refcnt);
+	return evsel;
+}
+
 #ifdef HAVE_LIBTRACEEVENT
 struct tep_event *evsel__tp_format(struct evsel *evsel)
 {
@@ -703,7 +711,7 @@ struct tep_event *evsel__tp_format(struct evsel *evsel)
 		tp_format = trace_event__tp_format(evsel->tp_sys, evsel->tp_name);
 
 	if (IS_ERR(tp_format)) {
-		int err = -PTR_ERR(evsel->tp_format);
+		int err = -PTR_ERR(tp_format);
 
 		errno = err;
 		pr_err("Error getting tracepoint format '%s': %m\n",
@@ -2024,7 +2032,7 @@ void evsel__set_priv_destructor(void (*destructor)(void *priv))
 	evsel__priv_destructor = destructor;
 }
 
-void evsel__exit(struct evsel *evsel)
+static void evsel__exit(struct evsel *evsel)
 {
 	assert(list_empty(&evsel->core.node));
 	assert(evsel->evlist == NULL);
@@ -2061,11 +2069,14 @@ void evsel__exit(struct evsel *evsel)
 	}
 }
 
-void evsel__delete(struct evsel *evsel)
+void evsel__put(struct evsel *evsel)
 {
 	if (!evsel)
 		return;
 
+	if (!refcount_dec_and_test(&evsel->refcnt))
+		return;
+
 	evsel__exit(evsel);
 	free(evsel);
 }
@@ -3350,7 +3361,7 @@ static inline bool evsel__has_branch_counters(const struct evsel *evsel)
 	if (!leader || !evsel->evlist)
 		return false;
 
-	if (evsel->evlist->nr_br_cntr < 0)
+	if (evlist__nr_br_cntr(evsel->evlist) < 0)
 		evlist__update_br_cntr(evsel->evlist);
 
 	if (leader->br_cntr_nr > 0)
@@ -3407,7 +3418,7 @@ int __evsel__parse_sample(struct evsel *evsel, union perf_event *event,
 	union u64_swap u;
 
 	perf_sample__init(data, /*all=*/true);
-	data->evsel = evsel;
+	data->evsel = evsel__get(evsel);
 	data->cpu = data->pid = data->tid = -1;
 	data->stream_id = data->id = data->time = -1ULL;
 	data->period = evsel->core.attr.sample_period;
@@ -4382,7 +4393,7 @@ int evsel__open_strerror(struct evsel *evsel, struct target *target,
 
 struct perf_session *evsel__session(struct evsel *evsel)
 {
-	return evsel && evsel->evlist ? evsel->evlist->session : NULL;
+	return evsel && evsel->evlist ? evlist__session(evsel->evlist) : NULL;
 }
 
 struct perf_env *evsel__env(struct evsel *evsel)
@@ -4407,7 +4418,7 @@ static int store_evsel_ids(struct evsel *evsel, struct evlist *evlist)
 		     thread++) {
 			int fd = FD(evsel, cpu_map_idx, thread);
 
-			if (perf_evlist__id_add_fd(&evlist->core, &evsel->core,
+			if (perf_evlist__id_add_fd(evlist__core(evlist), &evsel->core,
 						   cpu_map_idx, thread, fd) < 0)
 				return -1;
 		}
diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h
index 163fc2b6a7eac..6b0c08958616b 100644
--- a/tools/perf/util/evsel.h
+++ b/tools/perf/util/evsel.h
@@ -2,28 +2,31 @@
 #ifndef __PERF_EVSEL_H
 #define __PERF_EVSEL_H 1
 
-#include <linux/list.h>
 #include <stdbool.h>
-#include <sys/types.h>
+
+#include <linux/list.h>
 #include <linux/perf_event.h>
+#include <linux/refcount.h>
 #include <linux/types.h>
+#include <sys/types.h>
+
 #include <internal/evsel.h>
 #include <perf/evsel.h>
+
 #include "symbol_conf.h"
-#include "pmus.h"
-#include "pmu.h"
 
+struct bperf_follower_bpf;
+struct bperf_leader_bpf;
+struct bpf_counter_ops;
 struct bpf_object;
 struct cgroup;
+struct hashmap;
 struct perf_counts;
+struct perf_pmu;
 struct perf_stat_config;
 struct perf_stat_evsel;
-union perf_event;
-struct bpf_counter_ops;
 struct target;
-struct hashmap;
-struct bperf_leader_bpf;
-struct bperf_follower_bpf;
+union perf_event;
 
 typedef int (evsel__sb_cb_t)(union perf_event *event, void *data);
 
@@ -45,6 +48,7 @@ typedef int (evsel__sb_cb_t)(union perf_event *event, void *data);
 struct evsel {
 	struct perf_evsel	core;
 	struct evlist		*evlist;
+	refcount_t		refcnt;
 	off_t			id_offset;
 	int			id_pos;
 	int			is_pos;
@@ -271,7 +275,7 @@ static inline struct evsel *evsel__new(struct perf_event_attr *attr)
 	return evsel__new_idx(attr, 0);
 }
 
-struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig);
+struct evsel *evsel__clone(struct evsel *orig);
 
 int copy_config_terms(struct list_head *dst, struct list_head *src);
 void free_config_terms(struct list_head *config_terms);
@@ -286,14 +290,13 @@ static inline struct evsel *evsel__newtp(const char *sys, const char *name)
 	return evsel__newtp_idx(sys, name, 0, true);
 }
 
+struct evsel *evsel__get(struct evsel *evsel);
+void evsel__put(struct evsel *evsel);
+
 #ifdef HAVE_LIBTRACEEVENT
 struct tep_event *evsel__tp_format(struct evsel *evsel);
 #endif
 
-void evsel__init(struct evsel *evsel, struct perf_event_attr *attr, int idx);
-void evsel__exit(struct evsel *evsel);
-void evsel__delete(struct evsel *evsel);
-
 void evsel__set_priv_destructor(void (*destructor)(void *priv));
 
 struct callchain_param;
@@ -361,10 +364,6 @@ void arch_evsel__apply_ratio_to_prev(struct evsel *evsel, struct perf_event_attr
 int evsel__set_filter(struct evsel *evsel, const char *filter);
 int evsel__append_tp_filter(struct evsel *evsel, const char *filter);
 int evsel__append_addr_filter(struct evsel *evsel, const char *filter);
-static inline bool evsel__is_non_perf_event_open_pmu(const struct evsel *evsel)
-{
-	return evsel->pmu && evsel->pmu->type > PERF_PMU_TYPE_PE_END;
-}
 
 int evsel__enable_cpu(struct evsel *evsel, int cpu_map_idx);
 int evsel__enable(struct evsel *evsel);
@@ -503,6 +502,7 @@ static inline bool evsel__is_group_event(struct evsel *evsel)
 	return evsel__is_group_leader(evsel) && evsel->core.nr_members > 1;
 }
 
+bool evsel__is_non_perf_event_open_pmu(const struct evsel *evsel);
 bool evsel__is_function_event(struct evsel *evsel);
 
 static inline bool evsel__is_bpf_output(struct evsel *evsel)
@@ -534,7 +534,7 @@ for ((_evsel) = list_entry((_leader)->core.node.next, struct evsel, core.node);
 	(_evsel) = list_entry((_evsel)->core.node.next, struct evsel, core.node))
 
 #define for_each_group_member(_evsel, _leader)				\
-	for_each_group_member_head(_evsel, _leader, &(_leader)->evlist->core.entries)
+	for_each_group_member_head(_evsel, _leader, &evlist__core((_leader)->evlist)->entries)
 
 /* Iterates group WITH the leader. */
 #define for_each_group_evsel_head(_evsel, _leader, _head)				\
@@ -544,7 +544,7 @@ for ((_evsel) = _leader;								\
 	(_evsel) = list_entry((_evsel)->core.node.next, struct evsel, core.node))
 
 #define for_each_group_evsel(_evsel, _leader)				\
-	for_each_group_evsel_head(_evsel, _leader, &(_leader)->evlist->core.entries)
+	for_each_group_evsel_head(_evsel, _leader, &evlist__core((_leader)->evlist)->entries)
 
 static inline bool evsel__has_branch_callstack(const struct evsel *evsel)
 {
diff --git a/tools/perf/util/expr.c b/tools/perf/util/expr.c
index 232998fef72ba..8aef3c7418f7c 100644
--- a/tools/perf/util/expr.c
+++ b/tools/perf/util/expr.c
@@ -468,7 +468,7 @@ double expr__has_event(const struct expr_parse_ctx *ctx, bool compute_ids, const
 		ret = parse_event(tmp, id) ? 0 : 1;
 	}
 out:
-	evlist__delete(tmp);
+	evlist__put(tmp);
 	return ret;
 }
 
diff --git a/tools/perf/util/header.c b/tools/perf/util/header.c
index 091d8f7f6bd2c..e90e541f546b4 100644
--- a/tools/perf/util/header.c
+++ b/tools/perf/util/header.c
@@ -385,7 +385,7 @@ static int write_tracing_data(struct feat_fd *ff,
 		return -1;
 
 #ifdef HAVE_LIBTRACEEVENT
-	return read_tracing_data(ff->fd, &evlist->core.entries);
+	return read_tracing_data(ff->fd, &evlist__core(evlist)->entries);
 #else
 	pr_err("ERROR: Trying to write tracing data without libtraceevent support.\n");
 	return -1;
@@ -434,8 +434,8 @@ static int write_osrelease(struct feat_fd *ff,
 	struct utsname uts;
 	const char *release = NULL;
 
-	if (evlist->session)
-		release = perf_env__os_release(perf_session__env(evlist->session));
+	if (evlist__session(evlist))
+		release = perf_env__os_release(perf_session__env(evlist__session(evlist)));
 
 	if (!release) {
 		int ret = uname(&uts);
@@ -452,8 +452,8 @@ static int write_arch(struct feat_fd *ff, struct evlist *evlist)
 	struct utsname uts;
 	const char *arch = NULL;
 
-	if (evlist->session)
-		arch = perf_env__arch(perf_session__env(evlist->session));
+	if (evlist__session(evlist))
+		arch = perf_env__arch(perf_session__env(evlist__session(evlist)));
 
 	if (!arch) {
 		int ret = uname(&uts);
@@ -469,7 +469,7 @@ static int write_e_machine(struct feat_fd *ff, struct evlist *evlist)
 {
 	/* e_machine expanded from 16 to 32-bits for alignment. */
 	uint32_t e_flags;
-	uint32_t e_machine = perf_session__e_machine(evlist->session, &e_flags);
+	uint32_t e_machine = perf_session__e_machine(evlist__session(evlist), &e_flags);
 	int ret;
 
 	ret = do_write(ff, &e_machine, sizeof(e_machine));
@@ -605,7 +605,7 @@ static int write_event_desc(struct feat_fd *ff,
 	u32 nre, nri, sz;
 	int ret;
 
-	nre = evlist->core.nr_entries;
+	nre = evlist__nr_entries(evlist);
 
 	/*
 	 * write number of events
@@ -987,7 +987,7 @@ int __weak get_cpuid(char *buffer __maybe_unused, size_t sz __maybe_unused,
 
 static int write_cpuid(struct feat_fd *ff, struct evlist *evlist)
 {
-	struct perf_cpu cpu = perf_cpu_map__min(evlist->core.all_cpus);
+	struct perf_cpu cpu = perf_cpu_map__min(evlist__core(evlist)->all_cpus);
 	char buffer[64];
 	int ret;
 
@@ -1420,14 +1420,14 @@ static int write_sample_time(struct feat_fd *ff,
 			     struct evlist *evlist)
 {
 	int ret;
+	u64 data = evlist__first_sample_time(evlist);
 
-	ret = do_write(ff, &evlist->first_sample_time,
-		       sizeof(evlist->first_sample_time));
+	ret = do_write(ff, &data, sizeof(data));
 	if (ret < 0)
 		return ret;
 
-	return do_write(ff, &evlist->last_sample_time,
-			sizeof(evlist->last_sample_time));
+	data = evlist__last_sample_time(evlist);
+	return do_write(ff, &data, sizeof(data));
 }
 
 
@@ -2551,16 +2551,16 @@ static void print_sample_time(struct feat_fd *ff, FILE *fp)
 
 	session = container_of(ff->ph, struct perf_session, header);
 
-	timestamp__scnprintf_usec(session->evlist->first_sample_time,
+	timestamp__scnprintf_usec(evlist__first_sample_time(session->evlist),
 				  time_buf, sizeof(time_buf));
 	fprintf(fp, "# time of first sample : %s\n", time_buf);
 
-	timestamp__scnprintf_usec(session->evlist->last_sample_time,
+	timestamp__scnprintf_usec(evlist__last_sample_time(session->evlist),
 				  time_buf, sizeof(time_buf));
 	fprintf(fp, "# time of last sample : %s\n", time_buf);
 
-	d = (double)(session->evlist->last_sample_time -
-		session->evlist->first_sample_time) / NSEC_PER_MSEC;
+	d = (double)(evlist__last_sample_time(session->evlist) -
+		evlist__first_sample_time(session->evlist)) / NSEC_PER_MSEC;
 
 	fprintf(fp, "# sample duration : %10.3f ms\n", d);
 }
@@ -3519,8 +3519,8 @@ static int process_sample_time(struct feat_fd *ff, void *data __maybe_unused)
 	if (ret)
 		return -1;
 
-	session->evlist->first_sample_time = first_sample_time;
-	session->evlist->last_sample_time = last_sample_time;
+	evlist__set_first_sample_time(session->evlist, first_sample_time);
+	evlist__set_last_sample_time(session->evlist, last_sample_time);
 	return 0;
 }
 
@@ -4610,7 +4610,7 @@ int perf_session__write_header(struct perf_session *session,
 					     /*write_attrs_after_data=*/false);
 }
 
-size_t perf_session__data_offset(const struct evlist *evlist)
+size_t perf_session__data_offset(struct evlist *evlist)
 {
 	struct evsel *evsel;
 	size_t data_offset;
@@ -4619,7 +4619,7 @@ size_t perf_session__data_offset(const struct evlist *evlist)
 	evlist__for_each_entry(evlist, evsel) {
 		data_offset += evsel->core.ids * sizeof(u64);
 	}
-	data_offset += evlist->core.nr_entries * sizeof(struct perf_file_attr);
+	data_offset += evlist__nr_entries(evlist) * sizeof(struct perf_file_attr);
 
 	return data_offset;
 }
@@ -5110,7 +5110,7 @@ int perf_session__read_header(struct perf_session *session)
 	if (session->evlist == NULL)
 		return -ENOMEM;
 
-	session->evlist->session = session;
+	evlist__set_session(session->evlist, session);
 	session->machines.host.env = &header->env;
 
 	/*
@@ -5186,7 +5186,7 @@ int perf_session__read_header(struct perf_session *session)
 			pr_err("Invalid ids section size %" PRIu64 " for attr %d, not aligned to u64\n",
 			       f_attr.ids.size, i);
 			err = -EINVAL;
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 
 		/*
@@ -5199,7 +5199,7 @@ int perf_session__read_header(struct perf_session *session)
 			pr_err("Invalid ids section size %" PRIu64 " for attr %d, too many IDs\n",
 			       f_attr.ids.size, i);
 			err = -EINVAL;
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 
 		/*
@@ -5212,19 +5212,19 @@ int perf_session__read_header(struct perf_session *session)
 			pr_err("Invalid ids section for attr %d: offset=%" PRIu64 " size=%" PRIu64 " exceeds file size %" PRIu64 "\n",
 			       i, f_attr.ids.offset, f_attr.ids.size, (u64)input_stat.st_size);
 			err = -EINVAL;
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 
 		tmp = lseek(fd, 0, SEEK_CUR);
 		evsel = evsel__new(&f_attr.attr);
 
 		if (evsel == NULL)
-			goto out_delete_evlist;
+			goto out_put_evlist;
 
 		evsel->needs_swap = header->needs_swap;
 		/*
 		 * Do it before so that if perf_evsel__alloc_id fails, this
-		 * entry gets purged too at evlist__delete().
+		 * entry gets purged too at evlist__put().
 		 */
 		evlist__add(session->evlist, evsel);
 
@@ -5235,7 +5235,7 @@ int perf_session__read_header(struct perf_session *session)
 		 * hattr->ids threads.
 		 */
 		if (perf_evsel__alloc_id(&evsel->core, 1, nr_ids))
-			goto out_delete_evlist;
+			goto out_put_evlist;
 
 		lseek(fd, f_attr.ids.offset, SEEK_SET);
 
@@ -5243,7 +5243,8 @@ int perf_session__read_header(struct perf_session *session)
 			if (perf_header__getbuffer64(header, fd, &f_id, sizeof(f_id)))
 				goto out_errno;
 
-			perf_evlist__id_add(&session->evlist->core, &evsel->core, 0, j, f_id);
+			perf_evlist__id_add(evlist__core(session->evlist),
+					    &evsel->core, 0, j, f_id);
 		}
 
 		lseek(fd, tmp, SEEK_SET);
@@ -5265,18 +5266,18 @@ int perf_session__read_header(struct perf_session *session)
 		err = perf_header__process_sections(header, fd, &session->tevent,
 						    perf_file_section__process);
 		if (err < 0)
-			goto out_delete_evlist;
+			goto out_put_evlist;
 
 		if (evlist__prepare_tracepoint_events(session->evlist,
 						      session->tevent.pevent)) {
 			err = -ENOMEM;
-			goto out_delete_evlist;
+			goto out_put_evlist;
 		}
 #else
 		err = perf_header__process_sections(header, fd, NULL,
 						    perf_file_section__process);
 		if (err < 0)
-			goto out_delete_evlist;
+			goto out_put_evlist;
 #endif
 	}
 
@@ -5302,8 +5303,8 @@ int perf_session__read_header(struct perf_session *session)
 out_errno:
 	return -errno;
 
-out_delete_evlist:
-	evlist__delete(session->evlist);
+out_put_evlist:
+	evlist__put(session->evlist);
 	session->evlist = NULL;
 	return err;
 }
@@ -5607,7 +5608,7 @@ int perf_event__process_attr(const struct perf_tool *tool __maybe_unused,
 	 */
 	ids = (void *)&event->attr.attr + attr_size;
 	for (i = 0; i < n_ids; i++) {
-		perf_evlist__id_add(&evlist->core, &evsel->core, 0, i, ids[i]);
+		perf_evlist__id_add(evlist__core(evlist), &evsel->core, 0, i, ids[i]);
 	}
 
 	return 0;
diff --git a/tools/perf/util/header.h b/tools/perf/util/header.h
index 86b1a72026d3f..5e03f884b7cc0 100644
--- a/tools/perf/util/header.h
+++ b/tools/perf/util/header.h
@@ -158,7 +158,7 @@ int perf_session__inject_header(struct perf_session *session,
 				struct feat_copier *fc,
 				bool write_attrs_after_data);
 
-size_t perf_session__data_offset(const struct evlist *evlist);
+size_t perf_session__data_offset(struct evlist *evlist);
 
 void perf_header__set_feat(struct perf_header *header, int feat);
 void perf_header__clear_feat(struct perf_header *header, int feat);
diff --git a/tools/perf/util/intel-pt.c b/tools/perf/util/intel-pt.c
index 56a9e439f5f8f..d83439c8c7b0a 100644
--- a/tools/perf/util/intel-pt.c
+++ b/tools/perf/util/intel-pt.c
@@ -1830,9 +1830,9 @@ static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
 		ptq->last_br_cyc_cnt = ptq->ipc_cyc_cnt;
 	}
 
-	perf_sample__exit(&sample);
 	ret = intel_pt_deliver_synth_event(pt, event, &sample,
 					    pt->branches_sample_type);
+	perf_sample__exit(&sample);
 	return ret;
 }
 
@@ -1967,6 +1967,7 @@ static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq)
 	union perf_event *event = ptq->event_buf;
 	struct perf_sample sample = { .ip = 0, };
 	struct perf_synth_intel_ptwrite raw;
+	int ret;
 
 	if (intel_pt_skip_event(pt))
 		return 0;
@@ -1983,8 +1984,9 @@ static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq)
 	sample.raw_size = perf_synth__raw_size(raw);
 	sample.raw_data = perf_synth__raw_data(&raw);
 
-	return intel_pt_deliver_synth_event(pt, event, &sample,
-					    pt->ptwrites_sample_type);
+	ret = intel_pt_deliver_synth_event(pt, event, &sample, pt->ptwrites_sample_type);
+	perf_sample__exit(&sample);
+	return ret;
 }
 
 static int intel_pt_synth_cbr_sample(struct intel_pt_queue *ptq)
diff --git a/tools/perf/util/intel-tpebs.c b/tools/perf/util/intel-tpebs.c
index bc3b79bfa01a7..b41171b5df77d 100644
--- a/tools/perf/util/intel-tpebs.c
+++ b/tools/perf/util/intel-tpebs.c
@@ -98,8 +98,9 @@ static int evsel__tpebs_start_perf_record(struct evsel *evsel)
 	record_argv[i++] = "-o";
 	record_argv[i++] = PERF_DATA;
 
-	if (!perf_cpu_map__is_any_cpu_or_is_empty(evsel->evlist->core.user_requested_cpus)) {
-		cpu_map__snprint(evsel->evlist->core.user_requested_cpus, cpumap_buf,
+	if (!perf_cpu_map__is_any_cpu_or_is_empty(
+			evlist__core(evsel->evlist)->user_requested_cpus)) {
+		cpu_map__snprint(evlist__core(evsel->evlist)->user_requested_cpus, cpumap_buf,
 				 sizeof(cpumap_buf));
 		record_argv[i++] = "-C";
 		record_argv[i++] = cpumap_buf;
@@ -176,7 +177,7 @@ static bool should_ignore_sample(const struct perf_sample *sample, const struct
 	if (t->evsel->evlist == NULL)
 		return true;
 
-	workload_pid = t->evsel->evlist->workload.pid;
+	workload_pid = evlist__workload_pid(t->evsel->evlist);
 	if (workload_pid < 0 || workload_pid == sample_pid)
 		return false;
 
diff --git a/tools/perf/util/iostat.c b/tools/perf/util/iostat.c
index b770bd473af71..c9d5028a47f39 100644
--- a/tools/perf/util/iostat.c
+++ b/tools/perf/util/iostat.c
@@ -4,7 +4,7 @@
 
 enum iostat_mode_t iostat_mode = IOSTAT_NONE;
 
-__weak int iostat_prepare(struct evlist *evlist __maybe_unused,
+__weak int iostat_prepare(struct evlist **evlist __maybe_unused,
 			  struct perf_stat_config *config __maybe_unused)
 {
 	return -1;
diff --git a/tools/perf/util/iostat.h b/tools/perf/util/iostat.h
index a4e7299c5c2fb..df8a241fbc32a 100644
--- a/tools/perf/util/iostat.h
+++ b/tools/perf/util/iostat.h
@@ -30,7 +30,7 @@ extern enum iostat_mode_t iostat_mode;
 
 typedef void (*iostat_print_counter_t)(struct perf_stat_config *, struct evsel *, void *);
 
-int iostat_prepare(struct evlist *evlist, struct perf_stat_config *config);
+int iostat_prepare(struct evlist **evlist, struct perf_stat_config *config);
 int iostat_parse(const struct option *opt, const char *str,
 		 int unset __maybe_unused);
 void iostat_list(struct evlist *evlist, struct perf_stat_config *config);
diff --git a/tools/perf/util/kvm-stat-arch/book3s_hv_exits.h b/tools/perf/util/kvm-stat-arch/book3s_hv_exits.h
index 2011376c7ab58..2688ca7d03990 100644
--- a/tools/perf/util/kvm-stat-arch/book3s_hv_exits.h
+++ b/tools/perf/util/kvm-stat-arch/book3s_hv_exits.h
@@ -26,8 +26,10 @@
 	{0xe00, "H_DATA_STORAGE"}, \
 	{0xe20, "H_INST_STORAGE"}, \
 	{0xe40, "H_EMUL_ASSIST"}, \
+	{0xea0, "H_VIRT"}, \
 	{0xf00, "PERFMON"}, \
 	{0xf20, "ALTIVEC"}, \
-	{0xf40, "VSX"}
+	{0xf40, "VSX"}, \
+	{0xf80, "H_FAC_UNAVAIL"}
 
 #endif
diff --git a/tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c b/tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c
index 96d9c4ae02094..315b24f9a5929 100644
--- a/tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c
+++ b/tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c
@@ -9,7 +9,6 @@
 
 #include "book3s_hv_exits.h"
 #include "book3s_hcalls.h"
-#include <subcmd/parse-options.h>
 
 #define NR_TPS 4
 
@@ -177,35 +176,15 @@ int __cpu_isa_init_powerpc(struct perf_kvm_stat *kvm)
  */
 int __kvm_add_default_arch_event_powerpc(int *argc, const char **argv)
 {
-	const char **tmp;
-	bool event = false;
-	int i, j = *argc;
+	int j = *argc;
 
-	const struct option event_options[] = {
-		OPT_BOOLEAN('e', "event", &event, NULL),
-		OPT_END()
-	};
-
-	tmp = calloc(j + 1, sizeof(char *));
-	if (!tmp)
+	if (!perf_pmus__have_event("trace_imc", "trace_cycles"))
 		return -EINVAL;
 
-	for (i = 0; i < j; i++)
-		tmp[i] = argv[i];
-
-	parse_options(j, tmp, event_options, NULL, PARSE_OPT_KEEP_UNKNOWN);
-	if (!event) {
-		if (perf_pmus__have_event("trace_imc", "trace_cycles")) {
-			argv[j++] = strdup("-e");
-			argv[j++] = strdup("trace_imc/trace_cycles/");
-			*argc += 2;
-		} else {
-			free(tmp);
-			return -EINVAL;
-		}
-	}
+	argv[j++] = "-e";
+	argv[j++] = "trace_imc/trace_cycles/";
+	*argc += 2;
 
-	free(tmp);
 	return 0;
 }
 
diff --git a/tools/perf/util/kvm-stat-arch/kvm-stat-x86.c b/tools/perf/util/kvm-stat-arch/kvm-stat-x86.c
index 788d216f08521..97041f5100fc8 100644
--- a/tools/perf/util/kvm-stat-arch/kvm-stat-x86.c
+++ b/tools/perf/util/kvm-stat-arch/kvm-stat-x86.c
@@ -7,7 +7,6 @@
 #include "../../../arch/x86/include/uapi/asm/svm.h"
 #include "../../../arch/x86/include/uapi/asm/vmx.h"
 #include "../../../arch/x86/include/uapi/asm/kvm.h"
-#include <subcmd/parse-options.h>
 
 define_exit_reasons_table(vmx_exit_reasons, VMX_EXIT_REASONS);
 define_exit_reasons_table(svm_exit_reasons, SVM_EXIT_REASONS);
@@ -211,39 +210,13 @@ int __cpu_isa_init_x86(struct perf_kvm_stat *kvm, const char *cpuid)
  */
 int __kvm_add_default_arch_event_x86(int *argc, const char **argv)
 {
-	const char **tmp;
-	bool event = false;
-	int ret = 0, i, j = *argc;
-
-	const struct option event_options[] = {
-		OPT_BOOLEAN('e', "event", &event, NULL),
-		OPT_BOOLEAN(0, "pfm-events", &event, NULL),
-		OPT_END()
-	};
-
-	if (!x86__is_intel_cpu())
-		return 0;
-
-	tmp = calloc(j + 1, sizeof(char *));
-	if (!tmp)
-		return -ENOMEM;
-
-	for (i = 0; i < j; i++)
-		tmp[i] = argv[i];
-
-	parse_options(j, tmp, event_options, NULL, PARSE_OPT_KEEP_UNKNOWN);
-	if (!event) {
-		argv[j++] = STRDUP_FAIL_EXIT("-e");
-		argv[j++] = STRDUP_FAIL_EXIT("cycles");
-		*argc += 2;
-	}
+	int j = *argc;
 
-	free(tmp);
-	return 0;
+	argv[j++] = "-e";
+	argv[j++] = "cycles";
+	*argc += 2;
 
-EXIT:
-	free(tmp);
-	return ret;
+	return 0;
 }
 
 const char * const *__kvm_events_tp_x86(void)
diff --git a/tools/perf/util/kvm-stat.c b/tools/perf/util/kvm-stat.c
index 755ab659a05c3..c571d2ed711a8 100644
--- a/tools/perf/util/kvm-stat.c
+++ b/tools/perf/util/kvm-stat.c
@@ -1,8 +1,10 @@
 // SPDX-License-Identifier: GPL-2.0
 #include "debug.h"
+#include "env.h"
 #include "evsel.h"
 #include "kvm-stat.h"
 #include <dwarf-regs.h>
+#include <subcmd/parse-options.h>
 
 bool kvm_exit_event(struct evsel *evsel)
 {
@@ -268,3 +270,42 @@ int kvm_add_default_arch_event(uint16_t e_machine, int *argc, const char **argv)
 		return 0;
 	}
 }
+
+bool kvm_need_default_arch_event(uint16_t e_machine, int argc, const char **argv)
+{
+	const char **tmp_argv;
+	bool event = false;
+	int i;
+
+	const struct option event_options[] = {
+		OPT_BOOLEAN('e', "event", &event, NULL),
+		OPT_BOOLEAN(0, "pfm-events", &event, NULL),
+		OPT_END()
+	};
+
+	switch (e_machine) {
+	case EM_PPC:
+	case EM_PPC64:
+		break;
+	case EM_X86_64:
+	case EM_386:
+		if (!x86__is_intel_cpu())
+			return false;
+		break;
+	default:
+		return false;
+	}
+
+	/* parse_options() may change the argv, let's make a copy */
+	tmp_argv = calloc(argc + 1, sizeof(char *));
+	if (!tmp_argv)
+		return false;
+
+	for (i = 0; i < argc; i++)
+		tmp_argv[i] = argv[i];
+
+	parse_options(argc, tmp_argv, event_options, NULL, PARSE_OPT_KEEP_UNKNOWN);
+	free(tmp_argv);
+
+	return !event;
+}
diff --git a/tools/perf/util/kvm-stat.h b/tools/perf/util/kvm-stat.h
index cdbd921a555f4..1db31c0751875 100644
--- a/tools/perf/util/kvm-stat.h
+++ b/tools/perf/util/kvm-stat.h
@@ -174,12 +174,20 @@ const char * const *__kvm_skip_events_riscv(void);
 const char * const *__kvm_skip_events_s390(void);
 const char * const *__kvm_skip_events_x86(void);
 
+bool kvm_need_default_arch_event(uint16_t e_machine, int argc, const char **argv);
 int kvm_add_default_arch_event(uint16_t e_machine, int *argc, const char **argv);
 int __kvm_add_default_arch_event_powerpc(int *argc, const char **argv);
 int __kvm_add_default_arch_event_x86(int *argc, const char **argv);
 
 #else /* !HAVE_LIBTRACEEVENT */
 
+static inline bool kvm_need_default_arch_event(uint16_t e_machine __maybe_unused,
+					       int argc __maybe_unused,
+					       const char **argv __maybe_unused)
+{
+	return false;
+}
+
 static inline int kvm_add_default_arch_event(uint16_t e_machine __maybe_unused,
 					     int *argc __maybe_unused,
 					     const char **argv __maybe_unused)
@@ -221,14 +229,4 @@ static inline struct kvm_info *kvm_info__new(void)
 	return ki;
 }
 
-#define STRDUP_FAIL_EXIT(s)		\
-	({	char *_p;		\
-		_p = strdup(s);		\
-		if (!_p) {		\
-			ret = -ENOMEM;	\
-			goto EXIT;	\
-		}			\
-		_p;			\
-	})
-
 #endif /* __PERF_KVM_STAT_H */
diff --git a/tools/perf/util/map.h b/tools/perf/util/map.h
index 979b3e11b9bcf..fb0279810ae99 100644
--- a/tools/perf/util/map.h
+++ b/tools/perf/util/map.h
@@ -2,14 +2,13 @@
 #ifndef __PERF_MAP_H
 #define __PERF_MAP_H
 
-#include <linux/refcount.h>
-#include <linux/compiler.h>
-#include <linux/list.h>
-#include <linux/rbtree.h>
+#include <stdbool.h>
 #include <stdio.h>
 #include <string.h>
-#include <stdbool.h>
+
+#include <linux/refcount.h>
 #include <linux/types.h>
+
 #include <internal/rc_check.h>
 
 struct dso;
diff --git a/tools/perf/util/metricgroup.c b/tools/perf/util/metricgroup.c
index c2ce3e53aaee7..5a60cb95e31c7 100644
--- a/tools/perf/util/metricgroup.c
+++ b/tools/perf/util/metricgroup.c
@@ -214,7 +214,7 @@ static void metric__free(struct metric *m)
 	zfree(&m->metric_refs);
 	expr__ctx_free(m->pctx);
 	zfree(&m->modifier);
-	evlist__delete(m->evlist);
+	evlist__put(m->evlist);
 	free(m);
 }
 
@@ -1262,7 +1262,8 @@ static int parse_ids(bool metric_no_merge, bool fake_pmu,
 		     struct expr_parse_ctx *ids, const char *modifier,
 		     bool group_events, const bool tool_events[TOOL_PMU__EVENT_MAX],
 		     struct evlist **out_evlist,
-		     const char *filter_pmu)
+		     const char *filter_pmu,
+		     bool cputype_filter)
 {
 	struct parse_events_error parse_error;
 	struct evlist *parsed_evlist;
@@ -1317,7 +1318,9 @@ static int parse_ids(bool metric_no_merge, bool fake_pmu,
 	pr_debug("Parsing metric events '%s'\n", events.buf);
 	parse_events_error__init(&parse_error);
 	ret = __parse_events(parsed_evlist, events.buf, filter_pmu,
-			     &parse_error, fake_pmu, /*warn_if_reordered=*/false,
+			     cputype_filter,
+			     &parse_error, fake_pmu,
+			     /*warn_if_reordered=*/false,
 			     /*fake_tp=*/false);
 	if (ret) {
 		parse_events_error__print(&parse_error, events.buf);
@@ -1331,7 +1334,7 @@ static int parse_ids(bool metric_no_merge, bool fake_pmu,
 	parsed_evlist = NULL;
 err_out:
 	parse_events_error__exit(&parse_error);
-	evlist__delete(parsed_evlist);
+	evlist__put(parsed_evlist);
 	strbuf_release(&events);
 	return ret;
 }
@@ -1382,7 +1385,7 @@ static struct evsel *pick_display_evsel(struct list_head *metric_list,
 }
 
 static int parse_groups(struct evlist *perf_evlist,
-			const char *pmu, const char *str,
+			const char *pmu, bool cputype_filter, const char *str,
 			bool metric_no_group,
 			bool metric_no_merge,
 			bool metric_no_threshold,
@@ -1420,7 +1423,8 @@ static int parse_groups(struct evlist *perf_evlist,
 					/*group_events=*/false,
 					tool_events,
 					&combined_evlist,
-					(pmu && strcmp(pmu, "all") == 0) ? NULL : pmu);
+					(pmu && strcmp(pmu, "all") == 0) ? NULL : pmu,
+					cputype_filter);
 		}
 		if (combined)
 			expr__ctx_free(combined);
@@ -1476,7 +1480,8 @@ static int parse_groups(struct evlist *perf_evlist,
 		if (!metric_evlist) {
 			ret = parse_ids(metric_no_merge, fake_pmu, m->pctx, m->modifier,
 					m->group_events, tool_events, &m->evlist,
-					(pmu && strcmp(pmu, "all") == 0) ? NULL : pmu);
+					(pmu && strcmp(pmu, "all") == 0) ? NULL : pmu,
+					cputype_filter);
 			if (ret)
 				goto out;
 
@@ -1490,7 +1495,7 @@ static int parse_groups(struct evlist *perf_evlist,
 			goto out;
 		}
 
-		me = metricgroup__lookup(&perf_evlist->metric_events,
+		me = metricgroup__lookup(evlist__metric_events(perf_evlist),
 					 pick_display_evsel(&metric_list, metric_events),
 					 /*create=*/true);
 
@@ -1541,22 +1546,26 @@ static int parse_groups(struct evlist *perf_evlist,
 
 
 	if (combined_evlist) {
-		evlist__splice_list_tail(perf_evlist, &combined_evlist->core.entries);
-		evlist__delete(combined_evlist);
+		evlist__splice_list_tail(perf_evlist, &evlist__core(combined_evlist)->entries);
+		evlist__put(combined_evlist);
+		combined_evlist = NULL;
 	}
 
 	list_for_each_entry(m, &metric_list, nd) {
 		if (m->evlist)
-			evlist__splice_list_tail(perf_evlist, &m->evlist->core.entries);
+			evlist__splice_list_tail(perf_evlist, &evlist__core(m->evlist)->entries);
 	}
 
 out:
+	if (combined_evlist)
+		evlist__put(combined_evlist);
 	metricgroup__free_metrics(&metric_list);
 	return ret;
 }
 
 int metricgroup__parse_groups(struct evlist *perf_evlist,
 			      const char *pmu,
+			      bool cputype_filter,
 			      const char *str,
 			      bool metric_no_group,
 			      bool metric_no_merge,
@@ -1570,16 +1579,17 @@ int metricgroup__parse_groups(struct evlist *perf_evlist,
 	if (hardware_aware_grouping)
 		pr_debug("Use hardware aware grouping instead of traditional metric grouping method\n");
 
-	return parse_groups(perf_evlist, pmu, str, metric_no_group, metric_no_merge,
+	return parse_groups(perf_evlist, pmu, cputype_filter, str, metric_no_group, metric_no_merge,
 			    metric_no_threshold, user_requested_cpu_list, system_wide,
 			    /*fake_pmu=*/false, table);
 }
 
 int metricgroup__parse_groups_test(struct evlist *evlist,
 				   const struct pmu_metrics_table *table,
-				   const char *str)
+				   const char *str,
+				   bool cputype_filter)
 {
-	return parse_groups(evlist, "all", str,
+	return parse_groups(evlist, "all", cputype_filter, str,
 			    /*metric_no_group=*/false,
 			    /*metric_no_merge=*/false,
 			    /*metric_no_threshold=*/false,
@@ -1683,8 +1693,10 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp,
 			new_expr->metric_expr = old_expr->metric_expr;
 			new_expr->metric_threshold = old_expr->metric_threshold;
 			new_expr->metric_name = strdup(old_expr->metric_name);
-			if (!new_expr->metric_name)
+			if (!new_expr->metric_name) {
+				free(new_expr);
 				return -ENOMEM;
+			}
 
 			new_expr->metric_unit = old_expr->metric_unit;
 			new_expr->runtime = old_expr->runtime;
@@ -1697,6 +1709,7 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp,
 				alloc_size = sizeof(*new_expr->metric_refs);
 				new_expr->metric_refs = calloc(nr + 1, alloc_size);
 				if (!new_expr->metric_refs) {
+					zfree(&new_expr->metric_name);
 					free(new_expr);
 					return -ENOMEM;
 				}
@@ -1713,6 +1726,7 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp,
 			alloc_size = sizeof(*new_expr->metric_events);
 			new_expr->metric_events = calloc(nr + 1, alloc_size);
 			if (!new_expr->metric_events) {
+				zfree(&new_expr->metric_name);
 				zfree(&new_expr->metric_refs);
 				free(new_expr);
 				return -ENOMEM;
@@ -1723,6 +1737,7 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp,
 				evsel = old_expr->metric_events[idx];
 				evsel = evlist__find_evsel(evlist, evsel->core.idx);
 				if (evsel == NULL) {
+					zfree(&new_expr->metric_name);
 					zfree(&new_expr->metric_events);
 					zfree(&new_expr->metric_refs);
 					free(new_expr);
diff --git a/tools/perf/util/metricgroup.h b/tools/perf/util/metricgroup.h
index 4be6bfc13c467..6a66f14dd01bd 100644
--- a/tools/perf/util/metricgroup.h
+++ b/tools/perf/util/metricgroup.h
@@ -71,6 +71,7 @@ struct metric_event *metricgroup__lookup(struct rblist *metric_events,
 					 bool create);
 int metricgroup__parse_groups(struct evlist *perf_evlist,
 			      const char *pmu,
+			      bool cputype_filter,
 			      const char *str,
 			      bool metric_no_group,
 			      bool metric_no_merge,
@@ -80,7 +81,8 @@ int metricgroup__parse_groups(struct evlist *perf_evlist,
 			      bool hardware_aware_grouping);
 int metricgroup__parse_groups_test(struct evlist *evlist,
 				   const struct pmu_metrics_table *table,
-				   const char *str);
+				   const char *str,
+				   bool cputype_filter);
 
 int metricgroup__for_each_metric(const struct pmu_metrics_table *table, pmu_metric_iter_fn fn,
 				 void *data);
diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c
index 943569e82b82f..cc7ad331a49f3 100644
--- a/tools/perf/util/parse-events.c
+++ b/tools/perf/util/parse-events.c
@@ -429,6 +429,9 @@ bool parse_events__filter_pmu(const struct parse_events_state *parse_state,
 	if (parse_state->pmu_filter == NULL)
 		return false;
 
+	if (parse_state->cputype_filter && !pmu->is_core)
+		return false;
+
 	return perf_pmu__wildcard_match(pmu, parse_state->pmu_filter) == 0;
 }
 
@@ -2288,18 +2291,20 @@ static int parse_events__sort_events_and_fix_groups(struct list_head *list)
 	return (idx_changed || num_leaders != orig_num_leaders) ? 1 : 0;
 }
 
-int __parse_events(struct evlist *evlist, const char *str, const char *pmu_filter,
+int __parse_events(struct evlist *evlist, const char *str,
+		   const char *pmu_filter, bool cputype_filter,
 		   struct parse_events_error *err, bool fake_pmu,
 		   bool warn_if_reordered, bool fake_tp)
 {
 	struct parse_events_state parse_state = {
-		.list	  = LIST_HEAD_INIT(parse_state.list),
-		.idx	  = evlist->core.nr_entries,
-		.error	  = err,
-		.stoken	  = PE_START_EVENTS,
-		.fake_pmu = fake_pmu,
-		.fake_tp  = fake_tp,
-		.pmu_filter = pmu_filter,
+		.list			= LIST_HEAD_INIT(parse_state.list),
+		.idx			= evlist__nr_entries(evlist),
+		.error			= err,
+		.stoken			= PE_START_EVENTS,
+		.fake_pmu		= fake_pmu,
+		.fake_tp		= fake_tp,
+		.pmu_filter		= pmu_filter,
+		.cputype_filter		= cputype_filter,
 		.match_legacy_cache_terms = true,
 	};
 	int ret, ret2;
@@ -2312,15 +2317,15 @@ int __parse_events(struct evlist *evlist, const char *str, const char *pmu_filte
 	}
 
 	ret2 = parse_events__sort_events_and_fix_groups(&parse_state.list);
-	if (ret2 < 0)
-		return ret;
+	if (ret2 < 0 && !ret)
+		ret = ret2;
 
 	/*
 	 * Add list to the evlist even with errors to allow callers to clean up.
 	 */
 	evlist__splice_list_tail(evlist, &parse_state.list);
 
-	if (ret2 && warn_if_reordered && !parse_state.wild_card_pmus) {
+	if (ret2 > 0 && warn_if_reordered && !parse_state.wild_card_pmus) {
 		evlist__uniquify_evsel_names(evlist, &stat_config);
 		pr_warning("WARNING: events were regrouped to match PMUs\n");
 
@@ -2343,7 +2348,7 @@ int __parse_events(struct evlist *evlist, const char *str, const char *pmu_filte
 
 	/*
 	 * There are 2 users - builtin-record and builtin-test objects.
-	 * Both call evlist__delete in case of error, so we dont
+	 * Both call evlist__put in case of error, so we dont
 	 * need to bother.
 	 */
 	return ret;
@@ -2518,8 +2523,9 @@ int parse_events_option(const struct option *opt, const char *str,
 	int ret;
 
 	parse_events_error__init(&err);
-	ret = __parse_events(*args->evlistp, str, args->pmu_filter, &err,
-			     /*fake_pmu=*/false, /*warn_if_reordered=*/true,
+	ret = __parse_events(*args->evlistp, str, args->pmu_filter,
+			     args->cputype_filter, &err, /*fake_pmu=*/false,
+			     /*warn_if_reordered=*/true,
 			     /*fake_tp=*/false);
 
 	if (ret) {
@@ -2546,7 +2552,7 @@ int parse_events_option_new_evlist(const struct option *opt, const char *str, in
 	}
 	ret = parse_events_option(opt, str, unset);
 	if (ret) {
-		evlist__delete(*args->evlistp);
+		evlist__put(*args->evlistp);
 		*args->evlistp = NULL;
 	}
 
@@ -2568,7 +2574,7 @@ foreach_evsel_in_last_glob(struct evlist *evlist,
 	 *
 	 * So no need to WARN here, let *func do this.
 	 */
-	if (evlist->core.nr_entries > 0)
+	if (evlist__nr_entries(evlist) > 0)
 		last = evlist__last(evlist);
 
 	do {
@@ -2578,7 +2584,7 @@ foreach_evsel_in_last_glob(struct evlist *evlist,
 		if (!last)
 			return 0;
 
-		if (last->core.node.prev == &evlist->core.entries)
+		if (last->core.node.prev == &evlist__core(evlist)->entries)
 			return 0;
 		last = list_entry(last->core.node.prev, struct evsel, core.node);
 	} while (!last->cmdline_group_boundary);
diff --git a/tools/perf/util/parse-events.h b/tools/perf/util/parse-events.h
index 3577ab2137304..b14c832b03a1e 100644
--- a/tools/perf/util/parse-events.h
+++ b/tools/perf/util/parse-events.h
@@ -26,20 +26,23 @@ const char *event_type(size_t type);
 struct parse_events_option_args {
 	struct evlist **evlistp;
 	const char *pmu_filter;
+	bool cputype_filter;
 };
 int parse_events_option(const struct option *opt, const char *str, int unset);
 int parse_events_option_new_evlist(const struct option *opt, const char *str, int unset);
-__attribute__((nonnull(1, 2, 4)))
-int __parse_events(struct evlist *evlist, const char *str, const char *pmu_filter,
-		   struct parse_events_error *error, bool fake_pmu,
-		   bool warn_if_reordered, bool fake_tp);
+__attribute__((nonnull(1, 2, 5))) int
+__parse_events(struct evlist *evlist, const char *str, const char *pmu_filter,
+	       bool cputype_filter, struct parse_events_error *error,
+	       bool fake_pmu, bool warn_if_reordered, bool fake_tp);
 
 __attribute__((nonnull(1, 2, 3)))
 static inline int parse_events(struct evlist *evlist, const char *str,
 			       struct parse_events_error *err)
 {
-	return __parse_events(evlist, str, /*pmu_filter=*/NULL, err, /*fake_pmu=*/false,
-			      /*warn_if_reordered=*/true, /*fake_tp=*/false);
+	return __parse_events(evlist, str, /*pmu_filter=*/NULL,
+			      /*cputype_filter=*/false, err, /*fake_pmu=*/false,
+			      /*warn_if_reordered=*/true,
+			      /*fake_tp=*/false);
 }
 
 int parse_event(struct evlist *evlist, const char *str);
@@ -161,6 +164,8 @@ struct parse_events_state {
 	bool			   fake_tp;
 	/* If non-null, when wildcard matching only match the given PMU. */
 	const char		  *pmu_filter;
+	/* If true, the pmu_filter was set by --cputype option. */
+	bool cputype_filter;
 	/* Should PE_LEGACY_NAME tokens be generated for config terms? */
 	bool			   match_legacy_cache_terms;
 	/* Were multiple PMUs scanned to find events? */
diff --git a/tools/perf/util/parse-events.y b/tools/perf/util/parse-events.y
index c194de5ec1ec7..b531b1f0ceb33 100644
--- a/tools/perf/util/parse-events.y
+++ b/tools/perf/util/parse-events.y
@@ -47,7 +47,7 @@ static void free_list_evsel(struct list_head* list_evsel)
 
 	list_for_each_entry_safe(evsel, tmp, list_evsel, core.node) {
 		list_del_init(&evsel->core.node);
-		evsel__delete(evsel);
+		evsel__put(evsel);
 	}
 	free(list_evsel);
 }
diff --git a/tools/perf/util/perf_api_probe.c b/tools/perf/util/perf_api_probe.c
index 6ecf38314f01c..f61c4ec52827b 100644
--- a/tools/perf/util/perf_api_probe.c
+++ b/tools/perf/util/perf_api_probe.c
@@ -1,14 +1,18 @@
 /* SPDX-License-Identifier: GPL-2.0 */
+#include "perf_api_probe.h"
 
-#include "perf-sys.h"
-#include "util/cloexec.h"
-#include "util/evlist.h"
-#include "util/evsel.h"
-#include "util/parse-events.h"
-#include "util/perf_api_probe.h"
-#include <perf/cpumap.h>
 #include <errno.h>
 
+#include <perf/cpumap.h>
+
+#include "cloexec.h"
+#include "evlist.h"
+#include "evsel.h"
+#include "parse-events.h"
+#include "perf-sys.h"
+#include "pmu.h"
+#include "pmus.h"
+
 typedef void (*setup_probe_fn_t)(struct evsel *evsel);
 
 static int perf_do_probe_api(setup_probe_fn_t fn, struct perf_cpu cpu, const char *str)
@@ -53,7 +57,7 @@ static int perf_do_probe_api(setup_probe_fn_t fn, struct perf_cpu cpu, const cha
 	err = 0;
 
 out_delete:
-	evlist__delete(evlist);
+	evlist__put(evlist);
 	return err;
 }
 
diff --git a/tools/perf/util/pfm.c b/tools/perf/util/pfm.c
index d9043f4afbe7b..f80d6b0df47ae 100644
--- a/tools/perf/util/pfm.c
+++ b/tools/perf/util/pfm.c
@@ -85,7 +85,7 @@ int parse_libpfm_events_option(const struct option *opt, const char *str,
 		}
 
 		pmu = perf_pmus__find_by_type((unsigned int)attr.type);
-		evsel = parse_events__add_event(evlist->core.nr_entries,
+		evsel = parse_events__add_event(evlist__nr_entries(evlist),
 						&attr, q, /*metric_id=*/NULL,
 						pmu);
 		if (evsel == NULL)
@@ -159,7 +159,7 @@ static bool is_libpfm_event_supported(const char *name, struct perf_cpu_map *cpu
 		result = false;
 
 	evsel__close(evsel);
-	evsel__delete(evsel);
+	evsel__put(evsel);
 
 	return result;
 }
diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c
index a550f030b85df..836e3b5615cd8 100644
--- a/tools/perf/util/pmu.c
+++ b/tools/perf/util/pmu.c
@@ -2660,8 +2660,12 @@ bool perf_pmu__wildcard_match(const struct perf_pmu *pmu, const char *wildcard_t
 		pmu->name,
 		pmu->alias_name,
 	};
-	bool need_fnmatch = strisglob(wildcard_to_match);
+	bool need_fnmatch;
 
+	if (pmu->is_core && !strcmp(wildcard_to_match, "default_core"))
+		return true;
+
+	need_fnmatch = strisglob(wildcard_to_match);
 	if (!strncmp(wildcard_to_match, "uncore_", 7))
 		wildcard_to_match += 7;
 
diff --git a/tools/perf/util/pmus.c b/tools/perf/util/pmus.c
index 5e3f571450fe7..e0a4cb2428ca4 100644
--- a/tools/perf/util/pmus.c
+++ b/tools/perf/util/pmus.c
@@ -150,6 +150,8 @@ struct perf_pmu *perf_pmus__find(const char *name)
 	bool core_pmu;
 	unsigned int to_read_pmus = 0;
 
+	if (!strcmp(name, "default_core"))
+		return perf_pmus__find_core_pmu();
 	/*
 	 * Once PMU is loaded it stays in the list,
 	 * so we keep us from multiple reading/parsing
diff --git a/tools/perf/util/print-events.c b/tools/perf/util/print-events.c
index cb27e2898aa05..0242243681b6b 100644
--- a/tools/perf/util/print-events.c
+++ b/tools/perf/util/print-events.c
@@ -174,7 +174,7 @@ bool is_event_supported(u8 type, u64 config)
 		}
 
 		evsel__close(evsel);
-		evsel__delete(evsel);
+		evsel__put(evsel);
 	}
 
 	perf_thread_map__put(tmap);
diff --git a/tools/perf/util/probe-file.c b/tools/perf/util/probe-file.c
index 4032572cbf55d..4d12693a83b3a 100644
--- a/tools/perf/util/probe-file.c
+++ b/tools/perf/util/probe-file.c
@@ -197,6 +197,8 @@ struct strlist *probe_file__get_rawlist(int fd)
 		idx = strlen(p) - 1;
 		if (p[idx] == '\n')
 			p[idx] = '\0';
+		if (buf[0] == '#')
+			continue;
 		ret = strlist__add(sl, buf);
 		if (ret < 0) {
 			pr_debug("strlist__add failed (%d)\n", ret);
diff --git a/tools/perf/util/python.c b/tools/perf/util/python.c
index cc1019d29a5d0..af85c0771c448 100644
--- a/tools/perf/util/python.c
+++ b/tools/perf/util/python.c
@@ -1,33 +1,62 @@
 // SPDX-License-Identifier: GPL-2.0
+#define PY_SSIZE_T_CLEAN
 #include <Python.h>
-#include <structmember.h>
+
 #include <inttypes.h>
-#include <poll.h>
+
 #include <linux/err.h>
+#include <poll.h>
+#include <unistd.h>
+
+#include <internal/lib.h>
 #include <perf/cpumap.h>
-#ifdef HAVE_LIBTRACEEVENT
-#include <event-parse.h>
-#endif
 #include <perf/mmap.h>
+#include <structmember.h>
+
+#include "addr_location.h"
+#include "build-id.h"
 #include "callchain.h"
+#include "comm.h"
+#include "config.h"
 #include "counts.h"
+#include "data.h"
+#include "debug.h"
+#include "dso.h"
+#include "dwarf-regs.h"
+#include "event.h"
+#include "branch.h"
 #include "evlist.h"
 #include "evsel.h"
-#include "event.h"
 #include "expr.h"
+#include "map.h"
+#include "metricgroup.h"
+#include "mmap.h"
+#include "pmus.h"
 #include "print_binary.h"
 #include "record.h"
+#include "sample.h"
+#include "session.h"
+#include "srccode.h"
+#include "srcline.h"
 #include "strbuf.h"
+#include "symbol.h"
+#include "stat.h"
+#include "header.h"
+#include "trace/beauty/syscalltbl.h"
+#include "thread.h"
 #include "thread_map.h"
+#include "tool.h"
 #include "tp_pmu.h"
 #include "trace-event.h"
-#include "metricgroup.h"
-#include "mmap.h"
-#include "util/sample.h"
-#include <internal/lib.h>
+
+#ifdef HAVE_LIBTRACEEVENT
+#include <event-parse.h>
+#endif
 
 PyMODINIT_FUNC PyInit_perf(void);
 
+static PyObject *pyrf_evsel__from_evsel(struct evsel *evsel);
+
 #define member_def(type, member, ptype, help) \
 	{ #member, ptype, \
 	  offsetof(struct pyrf_event, event) + offsetof(struct type, member), \
@@ -38,30 +67,83 @@ PyMODINIT_FUNC PyInit_perf(void);
 	  offsetof(struct pyrf_event, sample) + offsetof(struct perf_sample, member), \
 	  0, help }
 
+#define CHECK_INITIALIZED(ptr, msg) \
+	do { \
+		if (!(ptr)) { \
+			PyErr_SetString(PyExc_ValueError, msg " not initialized"); \
+			return NULL; \
+		} \
+	} while (0)
+
+#define CHECK_INITIALIZED_INT(ptr, msg) \
+	do { \
+		if (!(ptr)) { \
+			PyErr_SetString(PyExc_ValueError, msg " not initialized"); \
+			return -1; \
+		} \
+	} while (0)
+
 struct pyrf_event {
 	PyObject_HEAD
-	struct evsel *evsel;
+	/** @sample: The parsed sample from the event. */
 	struct perf_sample sample;
-	union perf_event   event;
+	/** @al: The address location from machine__resolve, lazily computed. */
+	struct addr_location al;
+	/** @al_resolved: True when machine__resolve been called. */
+	bool al_resolved;
+	/** @callchain: Resolved callchain, eagerly computed if requested. */
+	PyObject *callchain;
+	/** @brstack: Resolved branch stack, eagerly computed if requested. */
+	PyObject *brstack;
+	/** @event: The underlying perf_event that may be in a file or ring buffer. */
+	union perf_event event;
 };
 
 #define sample_members \
-	sample_member_def(sample_ip, ip, T_ULONGLONG, "event ip"),			 \
 	sample_member_def(sample_pid, pid, T_INT, "event pid"),			 \
 	sample_member_def(sample_tid, tid, T_INT, "event tid"),			 \
 	sample_member_def(sample_time, time, T_ULONGLONG, "event timestamp"),		 \
-	sample_member_def(sample_addr, addr, T_ULONGLONG, "event addr"),		 \
 	sample_member_def(sample_id, id, T_ULONGLONG, "event id"),			 \
 	sample_member_def(sample_stream_id, stream_id, T_ULONGLONG, "event stream id"), \
 	sample_member_def(sample_period, period, T_ULONGLONG, "event period"),		 \
 	sample_member_def(sample_cpu, cpu, T_UINT, "event cpu"),
 
+static PyObject *pyrf_event__get_evsel(PyObject *self, void *closure __maybe_unused)
+{
+	struct pyrf_event *pevent = (void *)self;
+
+	if (!pevent->sample.evsel)
+		Py_RETURN_NONE;
+
+	return pyrf_evsel__from_evsel(pevent->sample.evsel);
+}
+
+static PyGetSetDef pyrf_event__getset[] = {
+	{
+		.name = "evsel",
+		.get = pyrf_event__get_evsel,
+		.set = NULL,
+		.doc = "tracking event.",
+	},
+	{ .name = NULL, },
+};
+
+static void pyrf_event__delete(struct pyrf_event *pevent)
+{
+	if (pevent->al_resolved)
+		addr_location__exit(&pevent->al);
+	Py_XDECREF(pevent->callchain);
+	Py_XDECREF(pevent->brstack);
+	perf_sample__exit(&pevent->sample);
+	Py_TYPE(pevent)->tp_free((PyObject *)pevent);
+}
+
 static const char pyrf_mmap_event__doc[] = PyDoc_STR("perf mmap event object.");
 
 static PyMemberDef pyrf_mmap_event__members[] = {
 	sample_members
 	member_def(perf_event_header, type, T_UINT, "event type"),
-	member_def(perf_event_header, misc, T_UINT, "event misc"),
+	member_def(perf_event_header, misc, T_USHORT, "event misc"),
 	member_def(perf_record_mmap, pid, T_UINT, "event pid"),
 	member_def(perf_record_mmap, tid, T_UINT, "event tid"),
 	member_def(perf_record_mmap, start, T_ULONGLONG, "start of the map"),
@@ -94,12 +176,155 @@ static PyTypeObject pyrf_mmap_event__type = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	.tp_name	= "perf.mmap_event",
 	.tp_basicsize	= sizeof(struct pyrf_event),
+	.tp_dealloc	= (destructor)pyrf_event__delete,
 	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
 	.tp_doc		= pyrf_mmap_event__doc,
 	.tp_members	= pyrf_mmap_event__members,
+	.tp_getset	= pyrf_event__getset,
 	.tp_repr	= (reprfunc)pyrf_mmap_event__repr,
 };
 
+static const char pyrf_mmap2_event__doc[] = PyDoc_STR("perf mmap2 event object.");
+
+static PyObject *pyrf_mmap2_event__get_maj(PyObject *self, void *closure __maybe_unused)
+{
+	struct pyrf_event *pevent = (void *)self;
+
+	if (pevent->event.header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID)
+		Py_RETURN_NONE;
+
+	return PyLong_FromUnsignedLong(pevent->event.mmap2.maj);
+}
+
+static PyObject *pyrf_mmap2_event__get_min(PyObject *self, void *closure __maybe_unused)
+{
+	struct pyrf_event *pevent = (void *)self;
+
+	if (pevent->event.header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID)
+		Py_RETURN_NONE;
+
+	return PyLong_FromUnsignedLong(pevent->event.mmap2.min);
+}
+
+static PyObject *pyrf_mmap2_event__get_ino(PyObject *self, void *closure __maybe_unused)
+{
+	struct pyrf_event *pevent = (void *)self;
+
+	if (pevent->event.header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID)
+		Py_RETURN_NONE;
+
+	return PyLong_FromUnsignedLongLong(pevent->event.mmap2.ino);
+}
+
+static PyObject *pyrf_mmap2_event__get_ino_generation(PyObject *self, void *closure __maybe_unused)
+{
+	struct pyrf_event *pevent = (void *)self;
+
+	if (pevent->event.header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID)
+		Py_RETURN_NONE;
+
+	return PyLong_FromUnsignedLongLong(pevent->event.mmap2.ino_generation);
+}
+
+static PyObject *pyrf_mmap2_event__get_build_id(PyObject *self, void *closure __maybe_unused)
+{
+	struct pyrf_event *pevent = (void *)self;
+
+	if (!(pevent->event.header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID))
+		Py_RETURN_NONE;
+
+	int size = pevent->event.mmap2.build_id_size;
+
+	if (size > 20)
+		size = 20;
+
+	return PyBytes_FromStringAndSize((const char *)pevent->event.mmap2.build_id, size);
+}
+
+static PyGetSetDef pyrf_mmap2_event__getset[] = {
+	{
+		.name = "evsel",
+		.get = pyrf_event__get_evsel,
+		.set = NULL,
+		.doc = "tracking event.",
+	},
+	{
+		.name = "maj",
+		.get = pyrf_mmap2_event__get_maj,
+		.set = NULL,
+		.doc = "major number.",
+	},
+	{
+		.name = "min",
+		.get = pyrf_mmap2_event__get_min,
+		.set = NULL,
+		.doc = "minor number.",
+	},
+	{
+		.name = "ino",
+		.get = pyrf_mmap2_event__get_ino,
+		.set = NULL,
+		.doc = "inode number.",
+	},
+	{
+		.name = "ino_generation",
+		.get = pyrf_mmap2_event__get_ino_generation,
+		.set = NULL,
+		.doc = "inode generation.",
+	},
+	{
+		.name = "build_id",
+		.get = pyrf_mmap2_event__get_build_id,
+		.set = NULL,
+		.doc = "binary build ID.",
+	},
+	{ .name = NULL, },
+};
+
+static PyMemberDef pyrf_mmap2_event__members[] = {
+	sample_members
+	member_def(perf_event_header, type, T_UINT, "event type"),
+	member_def(perf_event_header, misc, T_USHORT, "event misc"),
+	member_def(perf_record_mmap2, pid, T_UINT, "event pid"),
+	member_def(perf_record_mmap2, tid, T_UINT, "event tid"),
+	member_def(perf_record_mmap2, start, T_ULONGLONG, "start of the map"),
+	member_def(perf_record_mmap2, len, T_ULONGLONG, "map length"),
+	member_def(perf_record_mmap2, pgoff, T_ULONGLONG, "page offset"),
+	member_def(perf_record_mmap2, prot, T_UINT, "protection"),
+	member_def(perf_record_mmap2, flags, T_UINT, "flags"),
+	member_def(perf_record_mmap2, filename, T_STRING_INPLACE, "backing store"),
+	{ .name = NULL, },
+};
+
+static PyObject *pyrf_mmap2_event__repr(const struct pyrf_event *pevent)
+{
+	PyObject *ret;
+	char *s;
+
+	if (asprintf(&s, "{ type: mmap2, pid: %u, tid: %u, start: %#" PRI_lx64 ", length: %#" PRI_lx64 ", offset: %#" PRI_lx64 ", flags: %#x, prot: %#x, filename: %s }",
+		     pevent->event.mmap2.pid, pevent->event.mmap2.tid,
+		     pevent->event.mmap2.start, pevent->event.mmap2.len,
+		     pevent->event.mmap2.pgoff, pevent->event.mmap2.flags,
+		     pevent->event.mmap2.prot, pevent->event.mmap2.filename) < 0)
+		return PyErr_NoMemory();
+
+	ret = PyUnicode_FromString(s);
+	free(s);
+	return ret;
+}
+
+static PyTypeObject pyrf_mmap2_event__type = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_name	= "perf.mmap2_event",
+	.tp_basicsize	= sizeof(struct pyrf_event),
+	.tp_dealloc	= (destructor)pyrf_event__delete,
+	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
+	.tp_doc		= pyrf_mmap2_event__doc,
+	.tp_members	= pyrf_mmap2_event__members,
+	.tp_getset	= pyrf_mmap2_event__getset,
+	.tp_repr	= (reprfunc)pyrf_mmap2_event__repr,
+};
+
 static const char pyrf_task_event__doc[] = PyDoc_STR("perf task (fork/exit) event object.");
 
 static PyMemberDef pyrf_task_event__members[] = {
@@ -129,9 +354,11 @@ static PyTypeObject pyrf_task_event__type = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	.tp_name	= "perf.task_event",
 	.tp_basicsize	= sizeof(struct pyrf_event),
+	.tp_dealloc	= (destructor)pyrf_event__delete,
 	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
 	.tp_doc		= pyrf_task_event__doc,
 	.tp_members	= pyrf_task_event__members,
+	.tp_getset	= pyrf_event__getset,
 	.tp_repr	= (reprfunc)pyrf_task_event__repr,
 };
 
@@ -158,9 +385,11 @@ static PyTypeObject pyrf_comm_event__type = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	.tp_name	= "perf.comm_event",
 	.tp_basicsize	= sizeof(struct pyrf_event),
+	.tp_dealloc	= (destructor)pyrf_event__delete,
 	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
 	.tp_doc		= pyrf_comm_event__doc,
 	.tp_members	= pyrf_comm_event__members,
+	.tp_getset	= pyrf_event__getset,
 	.tp_repr	= (reprfunc)pyrf_comm_event__repr,
 };
 
@@ -190,9 +419,11 @@ static PyTypeObject pyrf_throttle_event__type = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	.tp_name	= "perf.throttle_event",
 	.tp_basicsize	= sizeof(struct pyrf_event),
+	.tp_dealloc	= (destructor)pyrf_event__delete,
 	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
 	.tp_doc		= pyrf_throttle_event__doc,
 	.tp_members	= pyrf_throttle_event__members,
+	.tp_getset	= pyrf_event__getset,
 	.tp_repr	= (reprfunc)pyrf_throttle_event__repr,
 };
 
@@ -200,6 +431,7 @@ static const char pyrf_lost_event__doc[] = PyDoc_STR("perf lost event object.");
 
 static PyMemberDef pyrf_lost_event__members[] = {
 	sample_members
+	member_def(perf_event_header, type, T_UINT, "event type"),
 	member_def(perf_record_lost, id, T_ULONGLONG, "event id"),
 	member_def(perf_record_lost, lost, T_ULONGLONG, "number of lost events"),
 	{ .name = NULL, },
@@ -225,16 +457,90 @@ static PyTypeObject pyrf_lost_event__type = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	.tp_name	= "perf.lost_event",
 	.tp_basicsize	= sizeof(struct pyrf_event),
+	.tp_dealloc	= (destructor)pyrf_event__delete,
 	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
 	.tp_doc		= pyrf_lost_event__doc,
 	.tp_members	= pyrf_lost_event__members,
+	.tp_getset	= pyrf_event__getset,
 	.tp_repr	= (reprfunc)pyrf_lost_event__repr,
 };
 
+static const char pyrf_stat_event__doc[] = PyDoc_STR("perf stat event object.");
+
+static PyMemberDef pyrf_stat_event__members[] = {
+	sample_members
+	member_def(perf_event_header, type, T_UINT, "event type"),
+	member_def(perf_record_stat, id, T_ULONGLONG, "event id"),
+	member_def(perf_record_stat, cpu, T_UINT, "event cpu"),
+	member_def(perf_record_stat, thread, T_UINT, "event thread"),
+	member_def(perf_record_stat, val, T_ULONGLONG, "counter value"),
+	member_def(perf_record_stat, ena, T_ULONGLONG, "enabled time"),
+	member_def(perf_record_stat, run, T_ULONGLONG, "running time"),
+	{ .name = NULL, },
+};
+
+static PyObject *pyrf_stat_event__repr(const struct pyrf_event *pevent)
+{
+	return PyUnicode_FromFormat(
+		"{ type: stat, id: %llu, cpu: %u, thread: %u, val: %llu, ena: %llu, run: %llu }",
+		pevent->event.stat.id,
+		pevent->event.stat.cpu,
+		pevent->event.stat.thread,
+		pevent->event.stat.val,
+		pevent->event.stat.ena,
+		pevent->event.stat.run);
+}
+
+static PyTypeObject pyrf_stat_event__type = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_name	= "perf.stat_event",
+	.tp_basicsize	= sizeof(struct pyrf_event),
+	.tp_new		= PyType_GenericNew,
+	.tp_dealloc	= (destructor)pyrf_event__delete,
+	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
+	.tp_doc		= pyrf_stat_event__doc,
+	.tp_members	= pyrf_stat_event__members,
+	.tp_getset	= pyrf_event__getset,
+	.tp_repr	= (reprfunc)pyrf_stat_event__repr,
+};
+
+static const char pyrf_stat_round_event__doc[] = PyDoc_STR("perf stat round event object.");
+
+static PyMemberDef pyrf_stat_round_event__members[] = {
+	sample_members
+	member_def(perf_event_header, type, T_UINT, "event type"),
+	{ .name = "stat_round_type", .type = T_ULONGLONG,
+	  .offset = offsetof(struct pyrf_event, event) + offsetof(struct perf_record_stat_round, type),
+	  .doc = "round type" },
+	member_def(perf_record_stat_round, time, T_ULONGLONG, "round time"),
+	{ .name = NULL, },
+};
+
+static PyObject *pyrf_stat_round_event__repr(const struct pyrf_event *pevent)
+{
+	return PyUnicode_FromFormat("{ type: stat_round, type: %llu, time: %llu }",
+				   pevent->event.stat_round.type,
+				   pevent->event.stat_round.time);
+}
+
+static PyTypeObject pyrf_stat_round_event__type = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_name	= "perf.stat_round_event",
+	.tp_basicsize	= sizeof(struct pyrf_event),
+	.tp_new		= PyType_GenericNew,
+	.tp_dealloc	= (destructor)pyrf_event__delete,
+	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
+	.tp_doc		= pyrf_stat_round_event__doc,
+	.tp_members	= pyrf_stat_round_event__members,
+	.tp_getset	= pyrf_event__getset,
+	.tp_repr	= (reprfunc)pyrf_stat_round_event__repr,
+};
+
 static const char pyrf_read_event__doc[] = PyDoc_STR("perf read event object.");
 
 static PyMemberDef pyrf_read_event__members[] = {
 	sample_members
+	member_def(perf_event_header, type, T_UINT, "event type"),
 	member_def(perf_record_read, pid, T_UINT, "event pid"),
 	member_def(perf_record_read, tid, T_UINT, "event tid"),
 	{ .name = NULL, },
@@ -255,9 +561,11 @@ static PyTypeObject pyrf_read_event__type = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	.tp_name	= "perf.read_event",
 	.tp_basicsize	= sizeof(struct pyrf_event),
+	.tp_dealloc	= (destructor)pyrf_event__delete,
 	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
 	.tp_doc		= pyrf_read_event__doc,
 	.tp_members	= pyrf_read_event__members,
+	.tp_getset	= pyrf_event__getset,
 	.tp_repr	= (reprfunc)pyrf_read_event__repr,
 };
 
@@ -265,16 +573,17 @@ static const char pyrf_sample_event__doc[] = PyDoc_STR("perf sample event object
 
 static PyMemberDef pyrf_sample_event__members[] = {
 	sample_members
+	sample_member_def(sample_ip, ip, T_ULONGLONG, "event ip"),
+	sample_member_def(sample_addr, addr, T_ULONGLONG, "event addr"),
+	sample_member_def(sample_phys_addr, phys_addr, T_ULONGLONG, "event physical addr"),
+	sample_member_def(sample_weight, weight, T_ULONGLONG, "event weight"),
+	sample_member_def(sample_data_src, data_src, T_ULONGLONG, "event data source"),
+	sample_member_def(sample_insn_count, insn_cnt, T_ULONGLONG, "event instruction count"),
+	sample_member_def(sample_cyc_count, cyc_cnt, T_ULONGLONG, "event cycle count"),
 	member_def(perf_event_header, type, T_UINT, "event type"),
 	{ .name = NULL, },
 };
 
-static void pyrf_sample_event__delete(struct pyrf_event *pevent)
-{
-	perf_sample__exit(&pevent->sample);
-	Py_TYPE(pevent)->tp_free((PyObject*)pevent);
-}
-
 static PyObject *pyrf_sample_event__repr(const struct pyrf_event *pevent)
 {
 	PyObject *ret;
@@ -292,7 +601,9 @@ static PyObject *pyrf_sample_event__repr(const struct pyrf_event *pevent)
 #ifdef HAVE_LIBTRACEEVENT
 static bool is_tracepoint(const struct pyrf_event *pevent)
 {
-	return pevent->evsel->core.attr.type == PERF_TYPE_TRACEPOINT;
+	if (!pevent->sample.evsel)
+		return false;
+	return pevent->sample.evsel->core.attr.type == PERF_TYPE_TRACEPOINT;
 }
 
 static PyObject*
@@ -339,7 +650,7 @@ tracepoint_field(const struct pyrf_event *pe, struct tep_format_field *field)
 static PyObject*
 get_tracepoint_field(struct pyrf_event *pevent, PyObject *attr_name)
 {
-	struct evsel *evsel = pevent->evsel;
+	struct evsel *evsel = pevent->sample.evsel;
 	struct tep_event *tp_format = evsel__tp_format(evsel);
 	struct tep_format_field *field;
 
@@ -362,163 +673,861 @@ get_tracepoint_field(struct pyrf_event *pevent, PyObject *attr_name)
 }
 #endif /* HAVE_LIBTRACEEVENT */
 
-static PyObject*
-pyrf_sample_event__getattro(struct pyrf_event *pevent, PyObject *attr_name)
+static int pyrf_sample_event__resolve_al(struct pyrf_event *pevent)
 {
-	PyObject *obj = NULL;
-
-#ifdef HAVE_LIBTRACEEVENT
-	if (is_tracepoint(pevent))
-		obj = get_tracepoint_field(pevent, attr_name);
-#endif
+	struct evsel *evsel = pevent->sample.evsel;
+	struct evlist *evlist = evsel ? evsel->evlist : NULL;
+	struct perf_session *session = evlist ? evlist__session(evlist) : NULL;
 
-	return obj ?: PyObject_GenericGetAttr((PyObject *) pevent, attr_name);
-}
+	if (pevent->al_resolved)
+		return 0;
 
-static PyTypeObject pyrf_sample_event__type = {
-	PyVarObject_HEAD_INIT(NULL, 0)
-	.tp_name	= "perf.sample_event",
-	.tp_basicsize	= sizeof(struct pyrf_event),
-	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
-	.tp_doc		= pyrf_sample_event__doc,
-	.tp_members	= pyrf_sample_event__members,
-	.tp_repr	= (reprfunc)pyrf_sample_event__repr,
-	.tp_getattro	= (getattrofunc) pyrf_sample_event__getattro,
-};
+	if (!session)
+		return -1;
 
-static const char pyrf_context_switch_event__doc[] = PyDoc_STR("perf context_switch event object.");
+	addr_location__init(&pevent->al);
+	if (machine__resolve(&session->machines.host, &pevent->al, &pevent->sample) < 0) {
+		addr_location__exit(&pevent->al);
+		return -1;
+	}
 
-static PyMemberDef pyrf_context_switch_event__members[] = {
-	sample_members
-	member_def(perf_event_header, type, T_UINT, "event type"),
-	member_def(perf_record_switch, next_prev_pid, T_UINT, "next/prev pid"),
-	member_def(perf_record_switch, next_prev_tid, T_UINT, "next/prev tid"),
-	{ .name = NULL, },
-};
+	pevent->al_resolved = true;
+	return 0;
+}
 
-static PyObject *pyrf_context_switch_event__repr(const struct pyrf_event *pevent)
+static PyObject *pyrf_sample_event__get_dso(struct pyrf_event *pevent,
+					    void *closure __maybe_unused)
 {
-	PyObject *ret;
-	char *s;
+	if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.map)
+		Py_RETURN_NONE;
 
-	if (asprintf(&s, "{ type: context_switch, next_prev_pid: %u, next_prev_tid: %u, switch_out: %u }",
-		     pevent->event.context_switch.next_prev_pid,
-		     pevent->event.context_switch.next_prev_tid,
-		     !!(pevent->event.header.misc & PERF_RECORD_MISC_SWITCH_OUT)) < 0) {
-		ret = PyErr_NoMemory();
-	} else {
-		ret = PyUnicode_FromString(s);
-		free(s);
-	}
-	return ret;
+	return PyUnicode_FromString(dso__name(map__dso(pevent->al.map)));
 }
 
-static PyTypeObject pyrf_context_switch_event__type = {
-	PyVarObject_HEAD_INIT(NULL, 0)
-	.tp_name	= "perf.context_switch_event",
-	.tp_basicsize	= sizeof(struct pyrf_event),
-	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
-	.tp_doc		= pyrf_context_switch_event__doc,
-	.tp_members	= pyrf_context_switch_event__members,
-	.tp_repr	= (reprfunc)pyrf_context_switch_event__repr,
-};
+static PyObject *pyrf_sample_event__get_dso_long_name(struct pyrf_event *pevent,
+						      void *closure __maybe_unused)
+{
+	if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.map)
+		Py_RETURN_NONE;
 
-static int pyrf_event__setup_types(void)
+	return PyUnicode_FromString(dso__long_name(map__dso(pevent->al.map)));
+}
+
+static PyObject *pyrf_sample_event__get_dso_bid(struct pyrf_event *pevent,
+						void *closure __maybe_unused)
 {
-	int err;
-	pyrf_mmap_event__type.tp_new =
-	pyrf_task_event__type.tp_new =
-	pyrf_comm_event__type.tp_new =
-	pyrf_lost_event__type.tp_new =
-	pyrf_read_event__type.tp_new =
-	pyrf_sample_event__type.tp_new =
-	pyrf_context_switch_event__type.tp_new =
-	pyrf_throttle_event__type.tp_new = PyType_GenericNew;
+	char sbuild_id[SBUILD_ID_SIZE];
 
-	pyrf_sample_event__type.tp_dealloc = (destructor)pyrf_sample_event__delete,
+	if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.map)
+		Py_RETURN_NONE;
 
-	err = PyType_Ready(&pyrf_mmap_event__type);
-	if (err < 0)
-		goto out;
-	err = PyType_Ready(&pyrf_lost_event__type);
-	if (err < 0)
-		goto out;
-	err = PyType_Ready(&pyrf_task_event__type);
-	if (err < 0)
-		goto out;
-	err = PyType_Ready(&pyrf_comm_event__type);
-	if (err < 0)
-		goto out;
-	err = PyType_Ready(&pyrf_throttle_event__type);
-	if (err < 0)
-		goto out;
-	err = PyType_Ready(&pyrf_read_event__type);
-	if (err < 0)
-		goto out;
-	err = PyType_Ready(&pyrf_sample_event__type);
-	if (err < 0)
-		goto out;
-	err = PyType_Ready(&pyrf_context_switch_event__type);
-	if (err < 0)
-		goto out;
-out:
-	return err;
+	build_id__snprintf(dso__bid(map__dso(pevent->al.map)), sbuild_id, sizeof(sbuild_id));
+	return PyUnicode_FromString(sbuild_id);
 }
 
-static PyTypeObject *pyrf_event__type[] = {
-	[PERF_RECORD_MMAP]	 = &pyrf_mmap_event__type,
-	[PERF_RECORD_LOST]	 = &pyrf_lost_event__type,
-	[PERF_RECORD_COMM]	 = &pyrf_comm_event__type,
-	[PERF_RECORD_EXIT]	 = &pyrf_task_event__type,
-	[PERF_RECORD_THROTTLE]	 = &pyrf_throttle_event__type,
-	[PERF_RECORD_UNTHROTTLE] = &pyrf_throttle_event__type,
-	[PERF_RECORD_FORK]	 = &pyrf_task_event__type,
-	[PERF_RECORD_READ]	 = &pyrf_read_event__type,
-	[PERF_RECORD_SAMPLE]	 = &pyrf_sample_event__type,
-	[PERF_RECORD_SWITCH]	 = &pyrf_context_switch_event__type,
-	[PERF_RECORD_SWITCH_CPU_WIDE]  = &pyrf_context_switch_event__type,
-};
-
-static PyObject *pyrf_event__new(const union perf_event *event)
+static PyObject *pyrf_sample_event__get_map_start(struct pyrf_event *pevent,
+						  void *closure __maybe_unused)
 {
-	struct pyrf_event *pevent;
-	PyTypeObject *ptype;
+	if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.map)
+		Py_RETURN_NONE;
 
-	if ((event->header.type < PERF_RECORD_MMAP ||
-	     event->header.type > PERF_RECORD_SAMPLE) &&
-	    !(event->header.type == PERF_RECORD_SWITCH ||
-	      event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)) {
-		PyErr_Format(PyExc_TypeError, "Unexpected header type %u",
-			     event->header.type);
-		return NULL;
-	}
+	return PyLong_FromUnsignedLong(map__start(pevent->al.map));
+}
 
-	// FIXME this better be dynamic or we need to parse everything
-	// before calling perf_mmap__consume(), including tracepoint fields.
-	if (sizeof(pevent->event) < event->header.size) {
-		PyErr_Format(PyExc_TypeError, "Unexpected event size: %zd < %u",
-			     sizeof(pevent->event), event->header.size);
-		return NULL;
-	}
+static PyObject *pyrf_sample_event__get_map_end(struct pyrf_event *pevent,
+						void *closure __maybe_unused)
+{
+	if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.map)
+		Py_RETURN_NONE;
 
-	ptype = pyrf_event__type[event->header.type];
-	pevent = PyObject_New(struct pyrf_event, ptype);
-	if (pevent != NULL)
-		memcpy(&pevent->event, event, event->header.size);
-	return (PyObject *)pevent;
+	return PyLong_FromUnsignedLong(map__end(pevent->al.map));
 }
 
-struct pyrf_cpu_map {
-	PyObject_HEAD
+static PyObject *pyrf_sample_event__get_map_pgoff(struct pyrf_event *pevent,
+						  void *closure __maybe_unused)
+{
+	if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.map)
+		Py_RETURN_NONE;
 
-	struct perf_cpu_map *cpus;
-};
+	return PyLong_FromUnsignedLongLong(map__pgoff(pevent->al.map));
+}
 
-static int pyrf_cpu_map__init(struct pyrf_cpu_map *pcpus,
-			      PyObject *args, PyObject *kwargs)
+static PyObject *pyrf_sample_event__get_symbol(struct pyrf_event *pevent,
+					       void *closure __maybe_unused)
 {
-	static char *kwlist[] = { "cpustr", NULL };
-	char *cpustr = NULL;
+	if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.sym)
+		Py_RETURN_NONE;
+
+	return PyUnicode_FromString(pevent->al.sym->name);
+}
+
+static PyObject *pyrf_sample_event__get_sym_start(struct pyrf_event *pevent,
+						  void *closure __maybe_unused)
+{
+	if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.sym)
+		Py_RETURN_NONE;
+
+	return PyLong_FromUnsignedLongLong(pevent->al.sym->start);
+}
+
+static PyObject *pyrf_sample_event__get_sym_end(struct pyrf_event *pevent,
+						void *closure __maybe_unused)
+{
+	if (pyrf_sample_event__resolve_al(pevent) < 0 || !pevent->al.sym)
+		Py_RETURN_NONE;
+
+	return PyLong_FromUnsignedLongLong(pevent->al.sym->end);
+}
+
+static PyObject *pyrf_sample_event__get_raw_buf(struct pyrf_event *pevent,
+						void *closure __maybe_unused)
+{
+	if (pevent->event.header.type != PERF_RECORD_SAMPLE)
+		Py_RETURN_NONE;
+
+	return PyBytes_FromStringAndSize((const char *)pevent->sample.raw_data,
+					 pevent->sample.raw_size);
+}
+
+static PyObject *pyrf_sample_event__srccode(PyObject *self, PyObject *args)
+{
+	struct pyrf_event *pevent = (void *)self;
+	u64 addr = pevent->sample.ip;
+	char *srcfile = NULL;
+	char *srccode = NULL;
+	unsigned int line = 0;
+	int len = 0;
+	PyObject *result;
+	struct addr_location al;
+
+	if (!PyArg_ParseTuple(args, "|K", &addr))
+		return NULL;
+
+	if (pyrf_sample_event__resolve_al(pevent) < 0)
+		Py_RETURN_NONE;
+
+	if (addr != pevent->sample.ip) {
+		addr_location__init(&al);
+		thread__find_symbol_fb(pevent->al.thread, pevent->sample.cpumode, addr, &al);
+	} else {
+		addr_location__init(&al);
+		al.thread = thread__get(pevent->al.thread);
+		al.map = map__get(pevent->al.map);
+		al.sym = pevent->al.sym;
+		al.addr = pevent->al.addr;
+	}
+
+	if (al.map) {
+		struct dso *dso = map__dso(al.map);
+
+		if (dso) {
+			srcfile = get_srcline_split(dso, map__rip_2objdump(al.map, addr),
+						    &line);
+		}
+	}
+	addr_location__exit(&al);
+
+	if (srcfile) {
+		srccode = find_sourceline(srcfile, line, &len);
+		result = Py_BuildValue("(sIs#)", srcfile, line, srccode, (Py_ssize_t)len);
+		free(srcfile);
+	} else {
+		result = Py_BuildValue("(sIs#)", NULL, 0, NULL, (Py_ssize_t)0);
+	}
+
+	return result;
+}
+
+static PyObject *pyrf_sample_event__insn(PyObject *self, PyObject *args __maybe_unused)
+{
+	struct pyrf_event *pevent = (void *)self;
+	struct thread *thread;
+	struct machine *machine;
+
+	if (pyrf_sample_event__resolve_al(pevent) < 0)
+		Py_RETURN_NONE;
+
+	thread = pevent->al.thread;
+
+	if (!thread || !thread__maps(thread))
+		Py_RETURN_NONE;
+
+	machine = maps__machine(thread__maps(thread));
+	if (!machine)
+		Py_RETURN_NONE;
+
+	if (pevent->sample.ip && !pevent->sample.insn_len)
+		perf_sample__fetch_insn(&pevent->sample, thread, machine);
+
+	if (!pevent->sample.insn_len)
+		Py_RETURN_NONE;
+
+	return PyBytes_FromStringAndSize((const char *)pevent->sample.insn,
+					 pevent->sample.insn_len);
+}
+
+struct pyrf_callchain_node {
+	PyObject_HEAD
+	u64 ip;
+	struct map *map;
+	struct symbol *sym;
+};
+
+static void pyrf_callchain_node__delete(struct pyrf_callchain_node *pnode)
+{
+	map__put(pnode->map);
+	Py_TYPE(pnode)->tp_free((PyObject *)pnode);
+}
+
+static PyObject *pyrf_callchain_node__get_ip(struct pyrf_callchain_node *pnode,
+					     void *closure __maybe_unused)
+{
+	return PyLong_FromUnsignedLongLong(pnode->ip);
+}
+
+static PyObject *pyrf_callchain_node__get_symbol(struct pyrf_callchain_node *pnode,
+						 void *closure __maybe_unused)
+{
+	if (pnode->sym)
+		return PyUnicode_FromString(pnode->sym->name);
+	return PyUnicode_FromString("[unknown]");
+}
+
+static PyObject *pyrf_callchain_node__get_dso(struct pyrf_callchain_node *pnode,
+					      void *closure __maybe_unused)
+{
+	const char *dsoname = "[unknown]";
+
+	if (pnode->map) {
+		struct dso *dso = map__dso(pnode->map);
+
+		if (dso) {
+			if (symbol_conf.show_kernel_path && dso__long_name(dso))
+				dsoname = dso__long_name(dso);
+			else
+				dsoname = dso__name(dso);
+		}
+	}
+	return PyUnicode_FromString(dsoname);
+}
+
+static PyGetSetDef pyrf_callchain_node__getset[] = {
+	{ .name = "ip",     .get = (getter)pyrf_callchain_node__get_ip, },
+	{ .name = "symbol", .get = (getter)pyrf_callchain_node__get_symbol, },
+	{ .name = "dso",    .get = (getter)pyrf_callchain_node__get_dso, },
+	{ .name = NULL, },
+};
+
+static PyTypeObject pyrf_callchain_node__type = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_name	= "perf.callchain_node",
+	.tp_basicsize	= sizeof(struct pyrf_callchain_node),
+	.tp_dealloc	= (destructor)pyrf_callchain_node__delete,
+	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
+	.tp_doc		= "perf callchain node object.",
+	.tp_getset	= pyrf_callchain_node__getset,
+};
+
+struct pyrf_callchain_frame {
+	u64 ip;
+	struct map *map;
+	struct symbol *sym;
+};
+
+struct pyrf_callchain {
+	PyObject_HEAD
+	struct pyrf_callchain_frame *frames;
+	u64 nr_frames;
+};
+
+static void pyrf_callchain__delete(struct pyrf_callchain *pchain)
+{
+	if (pchain->frames) {
+		for (u64 i = 0; i < pchain->nr_frames; i++)
+			map__put(pchain->frames[i].map);
+		free(pchain->frames);
+	}
+	Py_TYPE(pchain)->tp_free((PyObject *)pchain);
+}
+
+static Py_ssize_t pyrf_callchain__length(PyObject *obj)
+{
+	struct pyrf_callchain *pchain = (void *)obj;
+
+	return pchain->nr_frames;
+}
+
+static PyObject *pyrf_callchain__item(PyObject *obj, Py_ssize_t i)
+{
+	struct pyrf_callchain *pchain = (void *)obj;
+	struct pyrf_callchain_node *pnode;
+
+	if (i < 0 || i >= (Py_ssize_t)pchain->nr_frames) {
+		PyErr_SetString(PyExc_IndexError, "Index out of range");
+		return NULL;
+	}
+
+	pnode = PyObject_New(struct pyrf_callchain_node, &pyrf_callchain_node__type);
+	if (!pnode)
+		return NULL;
+
+	pnode->ip = pchain->frames[i].ip;
+	pnode->map = map__get(pchain->frames[i].map);
+	pnode->sym = pchain->frames[i].sym;
+
+	return (PyObject *)pnode;
+}
+
+static PySequenceMethods pyrf_callchain__sequence_methods = {
+	.sq_length = pyrf_callchain__length,
+	.sq_item   = pyrf_callchain__item,
+};
+
+static PyTypeObject pyrf_callchain__type = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_name	= "perf.callchain",
+	.tp_basicsize	= sizeof(struct pyrf_callchain),
+	.tp_dealloc	= (destructor)pyrf_callchain__delete,
+	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
+	.tp_doc		= "perf callchain object.",
+	.tp_as_sequence	= &pyrf_callchain__sequence_methods,
+};
+
+static PyObject *pyrf_sample_event__get_callchain(PyObject *self, void *closure __maybe_unused)
+{
+	struct pyrf_event *pevent = (void *)self;
+
+	if (!pevent->callchain)
+		Py_RETURN_NONE;
+
+	Py_INCREF(pevent->callchain);
+	return pevent->callchain;
+}
+
+struct pyrf_branch_entry {
+	PyObject_HEAD
+	u64 from;
+	u64 to;
+	struct branch_flags flags;
+};
+
+static void pyrf_branch_entry__delete(struct pyrf_branch_entry *pentry)
+{
+	Py_TYPE(pentry)->tp_free((PyObject *)pentry);
+}
+
+static PyObject *pyrf_branch_entry__get_from(struct pyrf_branch_entry *pentry,
+					     void *closure __maybe_unused)
+{
+	return PyLong_FromUnsignedLongLong(pentry->from);
+}
+
+static PyObject *pyrf_branch_entry__get_to(struct pyrf_branch_entry *pentry,
+					   void *closure __maybe_unused)
+{
+	return PyLong_FromUnsignedLongLong(pentry->to);
+}
+
+static PyObject *pyrf_branch_entry__get_mispred(struct pyrf_branch_entry *pentry,
+						void *closure __maybe_unused)
+{
+	return PyBool_FromLong(pentry->flags.mispred);
+}
+
+static PyObject *pyrf_branch_entry__get_predicted(struct pyrf_branch_entry *pentry,
+						  void *closure __maybe_unused)
+{
+	return PyBool_FromLong(pentry->flags.predicted);
+}
+
+static PyObject *pyrf_branch_entry__get_in_tx(struct pyrf_branch_entry *pentry,
+					      void *closure __maybe_unused)
+{
+	return PyBool_FromLong(pentry->flags.in_tx);
+}
+
+static PyObject *pyrf_branch_entry__get_abort(struct pyrf_branch_entry *pentry,
+					      void *closure __maybe_unused)
+{
+	return PyBool_FromLong(pentry->flags.abort);
+}
+
+static PyObject *pyrf_branch_entry__get_cycles(struct pyrf_branch_entry *pentry,
+					       void *closure __maybe_unused)
+{
+	return PyLong_FromUnsignedLongLong(pentry->flags.cycles);
+}
+
+static PyObject *pyrf_branch_entry__get_type(struct pyrf_branch_entry *pentry,
+					     void *closure __maybe_unused)
+{
+	return PyLong_FromUnsignedLongLong((unsigned long long)pentry->flags.type);
+}
+
+static PyGetSetDef pyrf_branch_entry__getset[] = {
+	{ .name = "from_ip",      .get = (getter)pyrf_branch_entry__get_from, },
+	{ .name = "to_ip",        .get = (getter)pyrf_branch_entry__get_to, },
+	{ .name = "mispred",   .get = (getter)pyrf_branch_entry__get_mispred, },
+	{ .name = "predicted", .get = (getter)pyrf_branch_entry__get_predicted, },
+	{ .name = "in_tx",     .get = (getter)pyrf_branch_entry__get_in_tx, },
+	{ .name = "abort",     .get = (getter)pyrf_branch_entry__get_abort, },
+	{ .name = "cycles",    .get = (getter)pyrf_branch_entry__get_cycles, },
+	{ .name = "type",      .get = (getter)pyrf_branch_entry__get_type, },
+	{ .name = NULL, },
+};
+
+static PyTypeObject pyrf_branch_entry__type = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_name	= "perf.branch_entry",
+	.tp_basicsize	= sizeof(struct pyrf_branch_entry),
+	.tp_dealloc	= (destructor)pyrf_branch_entry__delete,
+	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
+	.tp_doc		= "perf branch entry object.",
+	.tp_getset	= pyrf_branch_entry__getset,
+};
+
+struct pyrf_branch_stack {
+	PyObject_HEAD
+	struct branch_entry *entries;
+	u64 nr;
+};
+
+static void pyrf_branch_stack__delete(struct pyrf_branch_stack *pstack)
+{
+	free(pstack->entries);
+	Py_TYPE(pstack)->tp_free((PyObject *)pstack);
+}
+
+static Py_ssize_t pyrf_branch_stack__length(PyObject *obj)
+{
+	struct pyrf_branch_stack *pstack = (void *)obj;
+
+	return pstack->nr;
+}
+
+static PyObject *pyrf_branch_stack__item(PyObject *obj, Py_ssize_t i)
+{
+	struct pyrf_branch_stack *pstack = (void *)obj;
+	struct pyrf_branch_entry *pentry;
+
+	if (i < 0 || i >= (Py_ssize_t)pstack->nr) {
+		PyErr_SetString(PyExc_IndexError, "Index out of range");
+		return NULL;
+	}
+
+	pentry = PyObject_New(struct pyrf_branch_entry, &pyrf_branch_entry__type);
+	if (!pentry)
+		return NULL;
+
+	pentry->from = pstack->entries[i].from;
+	pentry->to = pstack->entries[i].to;
+	pentry->flags = pstack->entries[i].flags;
+
+	return (PyObject *)pentry;
+}
+
+static PySequenceMethods pyrf_branch_stack__sequence_methods = {
+	.sq_length = pyrf_branch_stack__length,
+	.sq_item   = pyrf_branch_stack__item,
+};
+
+static PyTypeObject pyrf_branch_stack__type = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_name	= "perf.branch_stack",
+	.tp_basicsize	= sizeof(struct pyrf_branch_stack),
+	.tp_dealloc	= (destructor)pyrf_branch_stack__delete,
+	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
+	.tp_doc		= "perf branch stack object.",
+	.tp_as_sequence	= &pyrf_branch_stack__sequence_methods,
+};
+
+static PyObject *pyrf_sample_event__get_brstack(PyObject *self, void *closure __maybe_unused)
+{
+	struct pyrf_event *pevent = (void *)self;
+
+	if (!pevent->brstack)
+		Py_RETURN_NONE;
+
+	Py_INCREF(pevent->brstack);
+	return pevent->brstack;
+}
+
+static PyObject*
+pyrf_sample_event__getattro(struct pyrf_event *pevent, PyObject *attr_name)
+{
+	PyObject *obj = NULL;
+
+#ifdef HAVE_LIBTRACEEVENT
+	if (is_tracepoint(pevent))
+		obj = get_tracepoint_field(pevent, attr_name);
+#endif
+
+	return obj ?: PyObject_GenericGetAttr((PyObject *) pevent, attr_name);
+}
+
+static PyGetSetDef pyrf_sample_event__getset[] = {
+	{
+		.name = "callchain",
+		.get = pyrf_sample_event__get_callchain,
+		.set = NULL,
+		.doc = "event callchain.",
+	},
+	{
+		.name = "brstack",
+		.get = pyrf_sample_event__get_brstack,
+		.set = NULL,
+		.doc = "event branch stack.",
+	},
+	{
+		.name = "raw_buf",
+		.get = (getter)pyrf_sample_event__get_raw_buf,
+		.set = NULL,
+		.doc = "event raw buffer.",
+	},
+	{
+		.name = "evsel",
+		.get = pyrf_event__get_evsel,
+		.set = NULL,
+		.doc = "tracking event.",
+	},
+	{
+		.name = "dso",
+		.get = (getter)pyrf_sample_event__get_dso,
+		.set = NULL,
+		.doc = "event dso short name.",
+	},
+	{
+		.name = "dso_long_name",
+		.get = (getter)pyrf_sample_event__get_dso_long_name,
+		.set = NULL,
+		.doc = "event dso long name.",
+	},
+	{
+		.name = "dso_bid",
+		.get = (getter)pyrf_sample_event__get_dso_bid,
+		.set = NULL,
+		.doc = "event dso build id.",
+	},
+	{
+		.name = "map_start",
+		.get = (getter)pyrf_sample_event__get_map_start,
+		.set = NULL,
+		.doc = "event map start address.",
+	},
+	{
+		.name = "map_end",
+		.get = (getter)pyrf_sample_event__get_map_end,
+		.set = NULL,
+		.doc = "event map end address.",
+	},
+	{
+		.name = "map_pgoff",
+		.get = (getter)pyrf_sample_event__get_map_pgoff,
+		.set = NULL,
+		.doc = "event map page offset.",
+	},
+	{
+		.name = "symbol",
+		.get = (getter)pyrf_sample_event__get_symbol,
+		.set = NULL,
+		.doc = "event symbol name.",
+	},
+	{
+		.name = "sym_start",
+		.get = (getter)pyrf_sample_event__get_sym_start,
+		.set = NULL,
+		.doc = "event symbol start address.",
+	},
+	{
+		.name = "sym_end",
+		.get = (getter)pyrf_sample_event__get_sym_end,
+		.set = NULL,
+		.doc = "event symbol end address.",
+	},
+	{ .name = NULL, },
+};
+
+static PyMethodDef pyrf_sample_event__methods[] = {
+	{
+		.ml_name  = "srccode",
+		.ml_meth  = (PyCFunction)pyrf_sample_event__srccode,
+		.ml_flags = METH_VARARGS,
+		.ml_doc	  = PyDoc_STR("Get source code for an address.")
+	},
+	{
+		.ml_name  = "insn",
+		.ml_meth  = (PyCFunction)pyrf_sample_event__insn,
+		.ml_flags = METH_NOARGS,
+		.ml_doc	  = PyDoc_STR("Get instruction bytes for a sample.")
+	},
+	{ .ml_name = NULL, }
+};
+
+static PyTypeObject pyrf_sample_event__type = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_name	= "perf.sample_event",
+	.tp_basicsize	= sizeof(struct pyrf_event),
+	.tp_dealloc	= (destructor)pyrf_event__delete,
+	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
+	.tp_doc		= pyrf_sample_event__doc,
+	.tp_members	= pyrf_sample_event__members,
+	.tp_getset	= pyrf_sample_event__getset,
+	.tp_methods	= pyrf_sample_event__methods,
+	.tp_repr	= (reprfunc)pyrf_sample_event__repr,
+	.tp_getattro	= (getattrofunc) pyrf_sample_event__getattro,
+};
+
+static const char pyrf_context_switch_event__doc[] = PyDoc_STR("perf context_switch event object.");
+
+static PyMemberDef pyrf_context_switch_event__members[] = {
+	sample_members
+	member_def(perf_event_header, type, T_UINT, "event type"),
+	member_def(perf_record_switch, next_prev_pid, T_UINT, "next/prev pid"),
+	member_def(perf_record_switch, next_prev_tid, T_UINT, "next/prev tid"),
+	{ .name = NULL, },
+};
+
+static PyObject *pyrf_context_switch_event__repr(const struct pyrf_event *pevent)
+{
+	PyObject *ret;
+	char *s;
+
+	if (asprintf(&s, "{ type: context_switch, next_prev_pid: %u, next_prev_tid: %u, switch_out: %u }",
+		     pevent->event.context_switch.next_prev_pid,
+		     pevent->event.context_switch.next_prev_tid,
+		     !!(pevent->event.header.misc & PERF_RECORD_MISC_SWITCH_OUT)) < 0) {
+		ret = PyErr_NoMemory();
+	} else {
+		ret = PyUnicode_FromString(s);
+		free(s);
+	}
+	return ret;
+}
+
+static PyTypeObject pyrf_context_switch_event__type = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_name	= "perf.context_switch_event",
+	.tp_basicsize	= sizeof(struct pyrf_event),
+	.tp_dealloc	= (destructor)pyrf_event__delete,
+	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
+	.tp_doc		= pyrf_context_switch_event__doc,
+	.tp_members	= pyrf_context_switch_event__members,
+	.tp_getset	= pyrf_event__getset,
+	.tp_repr	= (reprfunc)pyrf_context_switch_event__repr,
+};
+
+static int pyrf_event__setup_types(void)
+{
+	int err;
+
+	err = PyType_Ready(&pyrf_mmap_event__type);
+	if (err < 0)
+		goto out;
+	err = PyType_Ready(&pyrf_mmap2_event__type);
+	if (err < 0)
+		goto out;
+	err = PyType_Ready(&pyrf_lost_event__type);
+	if (err < 0)
+		goto out;
+	err = PyType_Ready(&pyrf_task_event__type);
+	if (err < 0)
+		goto out;
+	err = PyType_Ready(&pyrf_comm_event__type);
+	if (err < 0)
+		goto out;
+	err = PyType_Ready(&pyrf_throttle_event__type);
+	if (err < 0)
+		goto out;
+	err = PyType_Ready(&pyrf_read_event__type);
+	if (err < 0)
+		goto out;
+	err = PyType_Ready(&pyrf_sample_event__type);
+	if (err < 0)
+		goto out;
+	err = PyType_Ready(&pyrf_context_switch_event__type);
+	if (err < 0)
+		goto out;
+	err = PyType_Ready(&pyrf_stat_event__type);
+	if (err < 0)
+		goto out;
+	err = PyType_Ready(&pyrf_stat_round_event__type);
+	if (err < 0)
+		goto out;
+	err = PyType_Ready(&pyrf_callchain_node__type);
+	if (err < 0)
+		goto out;
+	err = PyType_Ready(&pyrf_callchain__type);
+	if (err < 0)
+		goto out;
+	err = PyType_Ready(&pyrf_branch_entry__type);
+	if (err < 0)
+		goto out;
+	err = PyType_Ready(&pyrf_branch_stack__type);
+	if (err < 0)
+		goto out;
+out:
+	return err;
+}
+
+static PyTypeObject *pyrf_event__type[] = {
+	[PERF_RECORD_MMAP]	 = &pyrf_mmap_event__type,
+	[PERF_RECORD_MMAP2]	 = &pyrf_mmap2_event__type,
+	[PERF_RECORD_LOST]	 = &pyrf_lost_event__type,
+	[PERF_RECORD_COMM]	 = &pyrf_comm_event__type,
+	[PERF_RECORD_EXIT]	 = &pyrf_task_event__type,
+	[PERF_RECORD_THROTTLE]	 = &pyrf_throttle_event__type,
+	[PERF_RECORD_UNTHROTTLE] = &pyrf_throttle_event__type,
+	[PERF_RECORD_FORK]	 = &pyrf_task_event__type,
+	[PERF_RECORD_READ]	 = &pyrf_read_event__type,
+	[PERF_RECORD_SAMPLE]	 = &pyrf_sample_event__type,
+	[PERF_RECORD_SWITCH]	 = &pyrf_context_switch_event__type,
+	[PERF_RECORD_SWITCH_CPU_WIDE]  = &pyrf_context_switch_event__type,
+	[PERF_RECORD_STAT]	 = &pyrf_stat_event__type,
+	[PERF_RECORD_STAT_ROUND] = &pyrf_stat_round_event__type,
+};
+
+static PyObject *pyrf_event__new(const union perf_event *event, struct evsel *evsel,
+				 struct perf_session *session,
+				 struct machine *machine)
+{
+	struct pyrf_event *pevent;
+	struct perf_sample *sample;
+	int err;
+	u32 min_size;
+	bool needs_swap;
+
+	if (!machine)
+		machine = session ? &session->machines.host : NULL;
+
+	if (event->header.type >= ARRAY_SIZE(pyrf_event__type) ||
+	    pyrf_event__type[event->header.type] == NULL) {
+		return PyErr_Format(PyExc_TypeError, "Unexpected header type %u",
+			     event->header.type);
+	}
+
+	if (perf_event__too_small(event, &min_size)) {
+		return PyErr_Format(PyExc_ValueError, "Event size %u too small for type %u",
+				    event->header.size, event->header.type);
+	}
+
+	size_t copy_size = event->header.size;
+
+	if (copy_size > sizeof(pevent->event)) {
+		return PyErr_Format(PyExc_TypeError, "Unexpected event size: %zd < %zu",
+				    sizeof(pevent->event), copy_size);
+	}
+
+	pevent = PyObject_New(struct pyrf_event, pyrf_event__type[event->header.type]);
+	if (pevent == NULL)
+		return PyErr_NoMemory();
+
+	/* Copy the event for memory safety and initialize variables. */
+	memcpy(&pevent->event, event, copy_size);
+	if (copy_size < sizeof(pevent->event))
+		memset((char *)&pevent->event + copy_size, 0, sizeof(pevent->event) - copy_size);
+
+	if (event->header.type == PERF_RECORD_MMAP2)
+		pevent->event.mmap2.filename[sizeof(pevent->event.mmap2.filename) - 1] = '\0';
+
+	perf_sample__init(&pevent->sample, /*all=*/true);
+	pevent->callchain = NULL;
+	pevent->brstack = NULL;
+	pevent->al_resolved = false;
+	addr_location__init(&pevent->al);
+
+	if (!evsel)
+		return (PyObject *)pevent;
+
+	/* Parse the sample again so that pointers are within the copied event. */
+	needs_swap = evsel->needs_swap;
+
+	evsel->needs_swap = false;
+	err = evsel__parse_sample(evsel, &pevent->event, &pevent->sample);
+	evsel->needs_swap = needs_swap;
+	if (err < 0) {
+		Py_DECREF(pevent);
+		return PyErr_Format(PyExc_OSError,
+				    "perf: can't parse sample, err=%d", err);
+	}
+	sample = &pevent->sample;
+	if (machine && sample->callchain) {
+		struct addr_location al;
+		struct callchain_cursor *cursor;
+		u64 i;
+		struct pyrf_callchain *pchain;
+
+		addr_location__init(&al);
+		if (machine__resolve(machine, &al, sample) >= 0) {
+			cursor = get_tls_callchain_cursor();
+			if (thread__resolve_callchain(al.thread, cursor, sample,
+						      NULL, NULL, PERF_MAX_STACK_DEPTH) == 0) {
+				callchain_cursor_commit(cursor);
+
+				pchain = PyObject_New(struct pyrf_callchain, &pyrf_callchain__type);
+				if (!pchain) {
+					addr_location__exit(&al);
+					Py_DECREF(pevent);
+					return NULL;
+				}
+				pchain->nr_frames = cursor->nr;
+				pchain->frames = calloc(pchain->nr_frames,
+							sizeof(*pchain->frames));
+				if (!pchain->frames) {
+					Py_DECREF(pchain);
+					addr_location__exit(&al);
+					Py_DECREF(pevent);
+					return PyErr_NoMemory();
+				}
+				struct callchain_cursor_node *node;
+
+				for (i = 0; i < pchain->nr_frames; i++) {
+					node = callchain_cursor_current(cursor);
+					pchain->frames[i].ip = node->ip;
+					pchain->frames[i].map =
+						map__get(node->ms.map);
+					pchain->frames[i].sym = node->ms.sym;
+					callchain_cursor_advance(cursor);
+				}
+				pevent->callchain = (PyObject *)pchain;
+			}
+			addr_location__exit(&al);
+		}
+	}
+	if (sample->branch_stack) {
+		struct branch_stack *bs = sample->branch_stack;
+		struct branch_entry *entries = perf_sample__branch_entries(sample);
+		struct pyrf_branch_stack *pstack;
+
+		pstack = PyObject_New(struct pyrf_branch_stack, &pyrf_branch_stack__type);
+		if (!pstack) {
+			Py_DECREF(pevent);
+			return NULL;
+		}
+		pstack->nr = bs->nr;
+		pstack->entries = calloc(bs->nr, sizeof(struct branch_entry));
+		if (!pstack->entries) {
+			Py_DECREF(pstack);
+			Py_DECREF(pevent);
+			return PyErr_NoMemory();
+		}
+		memcpy(pstack->entries, entries,
+		       bs->nr * sizeof(struct branch_entry));
+		pevent->brstack = (PyObject *)pstack;
+	}
+	return (PyObject *)pevent;
+}
+
+struct pyrf_cpu_map {
+	PyObject_HEAD
+
+	struct perf_cpu_map *cpus;
+};
+
+static int pyrf_cpu_map__init(struct pyrf_cpu_map *pcpus,
+			      PyObject *args, PyObject *kwargs)
+{
+	static char *kwlist[] = { "cpustr", NULL };
+	char *cpustr = NULL;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|s",
 					 kwlist, &cpustr))
@@ -942,7 +1951,7 @@ static int pyrf_counts_values__setup_types(void)
 struct pyrf_evsel {
 	PyObject_HEAD
 
-	struct evsel evsel;
+	struct evsel *evsel;
 };
 
 static int pyrf_evsel__init(struct pyrf_evsel *pevsel,
@@ -983,6 +1992,7 @@ static int pyrf_evsel__init(struct pyrf_evsel *pevsel,
 		"bp_type",
 		"bp_addr",
 		"bp_len",
+		"idx",
 		 NULL
 	};
 	u64 sample_period = 0;
@@ -1004,11 +2014,11 @@ static int pyrf_evsel__init(struct pyrf_evsel *pevsel,
 	    watermark = 0,
 	    precise_ip = 0,
 	    mmap_data = 0,
-	    sample_id_all = 1;
-	int idx = 0;
+	    sample_id_all = 1,
+	    idx = 0;
 
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs,
-					 "|iKiKKiiiiiiiiiiiiiiiiiiiiiiKK", kwlist,
+					 "|iKiKKiiiiiiiiiiiiiiiiiiiiiiKKi", kwlist,
 					 &attr.type, &attr.config, &attr.sample_freq,
 					 &sample_period, &attr.sample_type,
 					 &attr.read_format, &disabled, &inherit,
@@ -1050,34 +2060,41 @@ static int pyrf_evsel__init(struct pyrf_evsel *pevsel,
 	attr.sample_id_all  = sample_id_all;
 	attr.size	    = sizeof(attr);
 
-	evsel__init(&pevsel->evsel, &attr, idx);
+	evsel__put(pevsel->evsel);
+	pevsel->evsel = evsel__new(&attr);
+	if (!pevsel->evsel) {
+		PyErr_NoMemory();
+		return -1;
+	}
 	return 0;
 }
 
 static void pyrf_evsel__delete(struct pyrf_evsel *pevsel)
 {
-	evsel__exit(&pevsel->evsel);
+	evsel__put(pevsel->evsel);
 	Py_TYPE(pevsel)->tp_free((PyObject*)pevsel);
 }
 
 static PyObject *pyrf_evsel__open(struct pyrf_evsel *pevsel,
 				  PyObject *args, PyObject *kwargs)
 {
-	struct evsel *evsel = &pevsel->evsel;
+	struct evsel *evsel = pevsel->evsel;
 	struct perf_cpu_map *cpus = NULL;
 	struct perf_thread_map *threads = NULL;
 	PyObject *pcpus = NULL, *pthreads = NULL;
 	int group = 0, inherit = 0;
 	static char *kwlist[] = { "cpus", "threads", "group", "inherit", NULL };
 
+	CHECK_INITIALIZED(evsel, "evsel");
+
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|OOii", kwlist,
 					 &pcpus, &pthreads, &group, &inherit))
 		return NULL;
 
-	if (pthreads != NULL)
+	if (pthreads != NULL && pthreads != Py_None)
 		threads = ((struct pyrf_thread_map *)pthreads)->threads;
 
-	if (pcpus != NULL)
+	if (pcpus != NULL && pcpus != Py_None)
 		cpus = ((struct pyrf_cpu_map *)pcpus)->cpus;
 
 	evsel->core.attr.inherit = inherit;
@@ -1090,161 +2107,407 @@ static PyObject *pyrf_evsel__open(struct pyrf_evsel *pevsel,
 		return NULL;
 	}
 
-	Py_INCREF(Py_None);
-	return Py_None;
+	Py_INCREF(Py_None);
+	return Py_None;
+}
+
+static PyObject *pyrf_evsel__cpus(struct pyrf_evsel *pevsel)
+{
+	struct pyrf_cpu_map *pcpu_map;
+
+	CHECK_INITIALIZED(pevsel->evsel, "evsel");
+
+	pcpu_map = PyObject_New(struct pyrf_cpu_map, &pyrf_cpu_map__type);
+	if (pcpu_map)
+		pcpu_map->cpus = perf_cpu_map__get(pevsel->evsel->core.cpus);
+
+	return (PyObject *)pcpu_map;
+}
+
+static PyObject *pyrf_evsel__threads(struct pyrf_evsel *pevsel)
+{
+	struct pyrf_thread_map *pthread_map;
+
+	CHECK_INITIALIZED(pevsel->evsel, "evsel");
+
+	pthread_map = PyObject_New(struct pyrf_thread_map, &pyrf_thread_map__type);
+	if (pthread_map)
+		pthread_map->threads = perf_thread_map__get(pevsel->evsel->core.threads);
+
+	return (PyObject *)pthread_map;
+}
+
+/*
+ * Ensure evsel's counts and prev_raw_counts are allocated, the latter
+ * used by tool PMUs to compute the cumulative count as expected by
+ * stat's process_counter_values.
+ */
+static int evsel__ensure_counts(struct evsel *evsel)
+{
+	int nthreads, ncpus;
+
+	if (evsel->counts != NULL)
+		return 0;
+
+	nthreads = perf_thread_map__nr(evsel->core.threads);
+	ncpus = perf_cpu_map__nr(evsel->core.cpus);
+
+	evsel->counts = perf_counts__new(ncpus, nthreads);
+	if (evsel->counts == NULL)
+		return -ENOMEM;
+
+	evsel->prev_raw_counts = perf_counts__new(ncpus, nthreads);
+	if (evsel->prev_raw_counts == NULL)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static PyObject *pyrf_evsel__read(struct pyrf_evsel *pevsel,
+				  PyObject *args, PyObject *kwargs)
+{
+	struct evsel *evsel = pevsel->evsel;
+	int cpu = 0, cpu_idx, thread = 0, thread_idx;
+	struct perf_counts_values *old_count, *new_count;
+	struct pyrf_counts_values *count_values;
+
+	CHECK_INITIALIZED(evsel, "evsel");
+
+	count_values = PyObject_New(struct pyrf_counts_values,
+							       &pyrf_counts_values__type);
+	if (!count_values)
+		return NULL;
+
+	if (!PyArg_ParseTuple(args, "ii", &cpu, &thread))
+		return NULL;
+
+	cpu_idx = perf_cpu_map__idx(evsel->core.cpus, (struct perf_cpu){.cpu = cpu});
+	if (cpu_idx < 0) {
+		PyErr_Format(PyExc_TypeError, "CPU %d is not part of evsel's CPUs", cpu);
+		return NULL;
+	}
+	thread_idx = perf_thread_map__idx(evsel->core.threads, thread);
+	if (thread_idx < 0) {
+		PyErr_Format(PyExc_TypeError, "Thread %d is not part of evsel's threads",
+			     thread);
+		return NULL;
+	}
+
+	if (evsel__ensure_counts(evsel))
+		return PyErr_NoMemory();
+
+	/* Set up pointers to the old and newly read counter values. */
+	old_count = perf_counts(evsel->prev_raw_counts, cpu_idx, thread_idx);
+	new_count = perf_counts(evsel->counts, cpu_idx, thread_idx);
+	/* Update the value in evsel->counts. */
+	evsel__read_counter(evsel, cpu_idx, thread_idx);
+	/* Copy the value and turn it into the delta from old_count. */
+	count_values->values = *new_count;
+	count_values->values.val -= old_count->val;
+	count_values->values.ena -= old_count->ena;
+	count_values->values.run -= old_count->run;
+	/* Save the new count over the old_count for the next read. */
+	*old_count = *new_count;
+	return (PyObject *)count_values;
+}
+
+static PyObject *pyrf_evsel__str(PyObject *self)
+{
+	struct pyrf_evsel *pevsel = (void *)self;
+	struct evsel *evsel = pevsel->evsel;
+
+	if (!evsel)
+		return PyUnicode_FromString("evsel(uninitialized)");
+
+	return PyUnicode_FromFormat("evsel(%s)", evsel__name(evsel));
+}
+
+static PyMethodDef pyrf_evsel__methods[] = {
+	{
+		.ml_name  = "open",
+		.ml_meth  = (PyCFunction)pyrf_evsel__open,
+		.ml_flags = METH_VARARGS | METH_KEYWORDS,
+		.ml_doc	  = PyDoc_STR("open the event selector file descriptor table.")
+	},
+	{
+		.ml_name  = "cpus",
+		.ml_meth  = (PyCFunction)pyrf_evsel__cpus,
+		.ml_flags = METH_NOARGS,
+		.ml_doc	  = PyDoc_STR("CPUs the event is to be used with.")
+	},
+	{
+		.ml_name  = "threads",
+		.ml_meth  = (PyCFunction)pyrf_evsel__threads,
+		.ml_flags = METH_NOARGS,
+		.ml_doc	  = PyDoc_STR("threads the event is to be used with.")
+	},
+	{
+		.ml_name  = "read",
+		.ml_meth  = (PyCFunction)pyrf_evsel__read,
+		.ml_flags = METH_VARARGS | METH_KEYWORDS,
+		.ml_doc	  = PyDoc_STR("read counters")
+	},
+	{ .ml_name = NULL, }
+};
+
+static PyObject *pyrf_evsel__get_tracking(PyObject *self, void *closure __maybe_unused)
+{
+	struct pyrf_evsel *pevsel = (void *)self;
+
+	CHECK_INITIALIZED(pevsel->evsel, "evsel");
+
+	if (pevsel->evsel->tracking)
+		Py_RETURN_TRUE;
+	else
+		Py_RETURN_FALSE;
+}
+
+static int pyrf_evsel__set_tracking(PyObject *self, PyObject *val, void *closure __maybe_unused)
+{
+	struct pyrf_evsel *pevsel = (void *)self;
+	int is_true;
+
+	CHECK_INITIALIZED_INT(pevsel->evsel, "evsel");
+
+	is_true = PyObject_IsTrue(val);
+	if (is_true < 0)
+		return -1;
+
+	pevsel->evsel->tracking = is_true;
+	return 0;
+}
+
+static int pyrf_evsel__set_attr_config(PyObject *self, PyObject *val, void *closure __maybe_unused)
+{
+	struct pyrf_evsel *pevsel = (void *)self;
+
+	CHECK_INITIALIZED_INT(pevsel->evsel, "evsel");
+
+	pevsel->evsel->core.attr.config = PyLong_AsUnsignedLongLong(val);
+	return PyErr_Occurred() ? -1 : 0;
+}
+
+static PyObject *pyrf_evsel__get_attr_config(PyObject *self, void *closure __maybe_unused)
+{
+	struct pyrf_evsel *pevsel = (void *)self;
+
+	CHECK_INITIALIZED(pevsel->evsel, "evsel");
+
+	return PyLong_FromUnsignedLongLong(pevsel->evsel->core.attr.config);
+}
+
+static int pyrf_evsel__set_attr_read_format(PyObject *self, PyObject *val, void *closure __maybe_unused)
+{
+	struct pyrf_evsel *pevsel = (void *)self;
+
+	CHECK_INITIALIZED_INT(pevsel->evsel, "evsel");
+
+	pevsel->evsel->core.attr.read_format = PyLong_AsUnsignedLongLong(val);
+	return PyErr_Occurred() ? -1 : 0;
+}
+
+static PyObject *pyrf_evsel__get_attr_read_format(PyObject *self, void *closure __maybe_unused)
+{
+	struct pyrf_evsel *pevsel = (void *)self;
+
+	CHECK_INITIALIZED(pevsel->evsel, "evsel");
+
+	return PyLong_FromUnsignedLongLong(pevsel->evsel->core.attr.read_format);
+}
+
+static int pyrf_evsel__set_attr_sample_period(PyObject *self, PyObject *val, void *closure __maybe_unused)
+{
+	struct pyrf_evsel *pevsel = (void *)self;
+
+	CHECK_INITIALIZED_INT(pevsel->evsel, "evsel");
+
+	pevsel->evsel->core.attr.sample_period = PyLong_AsUnsignedLongLong(val);
+	return PyErr_Occurred() ? -1 : 0;
+}
+
+static PyObject *pyrf_evsel__get_attr_sample_period(PyObject *self, void *closure __maybe_unused)
+{
+	struct pyrf_evsel *pevsel = (void *)self;
+
+	CHECK_INITIALIZED(pevsel->evsel, "evsel");
+
+	return PyLong_FromUnsignedLongLong(pevsel->evsel->core.attr.sample_period);
 }
 
-static PyObject *pyrf_evsel__cpus(struct pyrf_evsel *pevsel)
+static int pyrf_evsel__set_attr_sample_type(PyObject *self, PyObject *val, void *closure __maybe_unused)
 {
-	struct pyrf_cpu_map *pcpu_map = PyObject_New(struct pyrf_cpu_map, &pyrf_cpu_map__type);
+	struct pyrf_evsel *pevsel = (void *)self;
 
-	if (pcpu_map)
-		pcpu_map->cpus = perf_cpu_map__get(pevsel->evsel.core.cpus);
+	CHECK_INITIALIZED_INT(pevsel->evsel, "evsel");
 
-	return (PyObject *)pcpu_map;
+	pevsel->evsel->core.attr.sample_type = PyLong_AsUnsignedLongLong(val);
+	return PyErr_Occurred() ? -1 : 0;
 }
 
-static PyObject *pyrf_evsel__threads(struct pyrf_evsel *pevsel)
+static PyObject *pyrf_evsel__get_attr_sample_type(PyObject *self, void *closure __maybe_unused)
 {
-	struct pyrf_thread_map *pthread_map =
-		PyObject_New(struct pyrf_thread_map, &pyrf_thread_map__type);
+	struct pyrf_evsel *pevsel = (void *)self;
 
-	if (pthread_map)
-		pthread_map->threads = perf_thread_map__get(pevsel->evsel.core.threads);
+	CHECK_INITIALIZED(pevsel->evsel, "evsel");
 
-	return (PyObject *)pthread_map;
+	return PyLong_FromUnsignedLongLong(pevsel->evsel->core.attr.sample_type);
 }
 
-/*
- * Ensure evsel's counts and prev_raw_counts are allocated, the latter
- * used by tool PMUs to compute the cumulative count as expected by
- * stat's process_counter_values.
- */
-static int evsel__ensure_counts(struct evsel *evsel)
+static PyObject *pyrf_evsel__get_attr_size(PyObject *self, void *closure __maybe_unused)
 {
-	int nthreads, ncpus;
+	struct pyrf_evsel *pevsel = (void *)self;
 
-	if (evsel->counts != NULL)
-		return 0;
+	CHECK_INITIALIZED(pevsel->evsel, "evsel");
 
-	nthreads = perf_thread_map__nr(evsel->core.threads);
-	ncpus = perf_cpu_map__nr(evsel->core.cpus);
+	return PyLong_FromUnsignedLong(pevsel->evsel->core.attr.size);
+}
 
-	evsel->counts = perf_counts__new(ncpus, nthreads);
-	if (evsel->counts == NULL)
-		return -ENOMEM;
+static int pyrf_evsel__set_attr_type(PyObject *self, PyObject *val, void *closure __maybe_unused)
+{
+	struct pyrf_evsel *pevsel = (void *)self;
 
-	evsel->prev_raw_counts = perf_counts__new(ncpus, nthreads);
-	if (evsel->prev_raw_counts == NULL)
-		return -ENOMEM;
+	CHECK_INITIALIZED_INT(pevsel->evsel, "evsel");
 
-	return 0;
+	pevsel->evsel->core.attr.type = PyLong_AsUnsignedLong(val);
+	return PyErr_Occurred() ? -1 : 0;
 }
 
-static PyObject *pyrf_evsel__read(struct pyrf_evsel *pevsel,
-				  PyObject *args, PyObject *kwargs)
+static PyObject *pyrf_evsel__get_attr_type(PyObject *self, void *closure __maybe_unused)
 {
-	struct evsel *evsel = &pevsel->evsel;
-	int cpu = 0, cpu_idx, thread = 0, thread_idx;
-	struct perf_counts_values *old_count, *new_count;
-	struct pyrf_counts_values *count_values = PyObject_New(struct pyrf_counts_values,
-							       &pyrf_counts_values__type);
+	struct pyrf_evsel *pevsel = (void *)self;
 
-	if (!count_values)
-		return NULL;
+	CHECK_INITIALIZED(pevsel->evsel, "evsel");
 
-	if (!PyArg_ParseTuple(args, "ii", &cpu, &thread))
-		return NULL;
+	return PyLong_FromUnsignedLong(pevsel->evsel->core.attr.type);
+}
 
-	cpu_idx = perf_cpu_map__idx(evsel->core.cpus, (struct perf_cpu){.cpu = cpu});
-	if (cpu_idx < 0) {
-		PyErr_Format(PyExc_TypeError, "CPU %d is not part of evsel's CPUs", cpu);
-		return NULL;
-	}
-	thread_idx = perf_thread_map__idx(evsel->core.threads, thread);
-	if (thread_idx < 0) {
-		PyErr_Format(PyExc_TypeError, "Thread %d is not part of evsel's threads",
-			     thread);
-		return NULL;
-	}
+static int pyrf_evsel__set_attr_wakeup_events(PyObject *self, PyObject *val, void *closure __maybe_unused)
+{
+	struct pyrf_evsel *pevsel = (void *)self;
 
-	if (evsel__ensure_counts(evsel))
-		return PyErr_NoMemory();
+	CHECK_INITIALIZED_INT(pevsel->evsel, "evsel");
 
-	/* Set up pointers to the old and newly read counter values. */
-	old_count = perf_counts(evsel->prev_raw_counts, cpu_idx, thread_idx);
-	new_count = perf_counts(evsel->counts, cpu_idx, thread_idx);
-	/* Update the value in evsel->counts. */
-	evsel__read_counter(evsel, cpu_idx, thread_idx);
-	/* Copy the value and turn it into the delta from old_count. */
-	count_values->values = *new_count;
-	count_values->values.val -= old_count->val;
-	count_values->values.ena -= old_count->ena;
-	count_values->values.run -= old_count->run;
-	/* Save the new count over the old_count for the next read. */
-	*old_count = *new_count;
-	return (PyObject *)count_values;
+	pevsel->evsel->core.attr.wakeup_events = PyLong_AsUnsignedLong(val);
+	return PyErr_Occurred() ? -1 : 0;
 }
 
-static PyObject *pyrf_evsel__str(PyObject *self)
+static PyObject *pyrf_evsel__get_attr_wakeup_events(PyObject *self, void *closure __maybe_unused)
 {
 	struct pyrf_evsel *pevsel = (void *)self;
-	struct evsel *evsel = &pevsel->evsel;
 
-	return PyUnicode_FromFormat("evsel(%s/%s/)", evsel__pmu_name(evsel), evsel__name(evsel));
+	CHECK_INITIALIZED(pevsel->evsel, "evsel");
+
+	return PyLong_FromUnsignedLong(pevsel->evsel->core.attr.wakeup_events);
 }
 
-static PyMethodDef pyrf_evsel__methods[] = {
+static PyObject *pyrf_evsel__get_ids(struct pyrf_evsel *pevsel, void *closure __maybe_unused)
+{
+	struct evsel *evsel;
+	PyObject *list;
+
+	CHECK_INITIALIZED(pevsel->evsel, "evsel");
+
+	evsel = pevsel->evsel;
+	list = PyList_New(0);
+
+	if (!list)
+		return NULL;
+
+	for (u32 i = 0; i < evsel->core.ids; i++) {
+		PyObject *id = PyLong_FromUnsignedLongLong(evsel->core.id[i]);
+		int ret;
+
+		if (!id) {
+			Py_DECREF(list);
+			return NULL;
+		}
+		ret = PyList_Append(list, id);
+		Py_DECREF(id);
+		if (ret < 0) {
+			Py_DECREF(list);
+			return NULL;
+		}
+	}
+
+	return list;
+}
+
+static PyGetSetDef pyrf_evsel__getset[] = {
 	{
-		.ml_name  = "open",
-		.ml_meth  = (PyCFunction)pyrf_evsel__open,
-		.ml_flags = METH_VARARGS | METH_KEYWORDS,
-		.ml_doc	  = PyDoc_STR("open the event selector file descriptor table.")
+		.name = "ids",
+		.get = (getter)pyrf_evsel__get_ids,
+		.set = NULL,
+		.doc = "event IDs.",
 	},
 	{
-		.ml_name  = "cpus",
-		.ml_meth  = (PyCFunction)pyrf_evsel__cpus,
-		.ml_flags = METH_NOARGS,
-		.ml_doc	  = PyDoc_STR("CPUs the event is to be used with.")
+		.name = "tracking",
+		.get = pyrf_evsel__get_tracking,
+		.set = pyrf_evsel__set_tracking,
+		.doc = "tracking event.",
 	},
 	{
-		.ml_name  = "threads",
-		.ml_meth  = (PyCFunction)pyrf_evsel__threads,
-		.ml_flags = METH_NOARGS,
-		.ml_doc	  = PyDoc_STR("threads the event is to be used with.")
+		.name = "config",
+		.get = pyrf_evsel__get_attr_config,
+		.set = pyrf_evsel__set_attr_config,
+		.doc = "attribute config.",
 	},
 	{
-		.ml_name  = "read",
-		.ml_meth  = (PyCFunction)pyrf_evsel__read,
-		.ml_flags = METH_VARARGS | METH_KEYWORDS,
-		.ml_doc	  = PyDoc_STR("read counters")
+		.name = "read_format",
+		.get = pyrf_evsel__get_attr_read_format,
+		.set = pyrf_evsel__set_attr_read_format,
+		.doc = "attribute read_format.",
 	},
-	{ .ml_name = NULL, }
+	{
+		.name = "sample_period",
+		.get = pyrf_evsel__get_attr_sample_period,
+		.set = pyrf_evsel__set_attr_sample_period,
+		.doc = "attribute sample_period.",
+	},
+	{
+		.name = "sample_type",
+		.get = pyrf_evsel__get_attr_sample_type,
+		.set = pyrf_evsel__set_attr_sample_type,
+		.doc = "attribute sample_type.",
+	},
+	{
+		.name = "size",
+		.get = pyrf_evsel__get_attr_size,
+		.doc = "attribute size.",
+	},
+	{
+		.name = "type",
+		.get = pyrf_evsel__get_attr_type,
+		.set = pyrf_evsel__set_attr_type,
+		.doc = "attribute type.",
+	},
+	{
+		.name = "wakeup_events",
+		.get = pyrf_evsel__get_attr_wakeup_events,
+		.set = pyrf_evsel__set_attr_wakeup_events,
+		.doc = "attribute wakeup_events.",
+	},
+	{ .name = NULL},
 };
 
-#define evsel_member_def(member, ptype, help) \
-	{ #member, ptype, \
-	  offsetof(struct pyrf_evsel, evsel.member), \
-	  0, help }
-
-#define evsel_attr_member_def(member, ptype, help) \
-	{ #member, ptype, \
-	  offsetof(struct pyrf_evsel, evsel.core.attr.member), \
-	  0, help }
+static const char pyrf_evsel__doc[] = PyDoc_STR("perf event selector list object.");
 
-static PyMemberDef pyrf_evsel__members[] = {
-	evsel_member_def(tracking, T_BOOL, "tracking event."),
-	evsel_attr_member_def(type, T_UINT, "attribute type."),
-	evsel_attr_member_def(size, T_UINT, "attribute size."),
-	evsel_attr_member_def(config, T_ULONGLONG, "attribute config."),
-	evsel_attr_member_def(sample_period, T_ULONGLONG, "attribute sample_period."),
-	evsel_attr_member_def(sample_type, T_ULONGLONG, "attribute sample_type."),
-	evsel_attr_member_def(read_format, T_ULONGLONG, "attribute read_format."),
-	evsel_attr_member_def(wakeup_events, T_UINT, "attribute wakeup_events."),
-	{ .name = NULL, },
-};
+static PyObject *pyrf_evsel__getattro(struct pyrf_evsel *pevsel, PyObject *attr_name)
+{
+	if (!pevsel->evsel) {
+		PyErr_SetString(PyExc_ValueError, "evsel not initialized");
+		return NULL;
+	}
+	return PyObject_GenericGetAttr((PyObject *) pevsel, attr_name);
+}
 
-static const char pyrf_evsel__doc[] = PyDoc_STR("perf event selector list object.");
+static int pyrf_evsel__setattro(struct pyrf_evsel *pevsel, PyObject *attr_name, PyObject *value)
+{
+	if (!pevsel->evsel) {
+		PyErr_SetString(PyExc_ValueError, "evsel not initialized");
+		return -1;
+	}
+	return PyObject_GenericSetAttr((PyObject *) pevsel, attr_name, value);
+}
 
 static PyTypeObject pyrf_evsel__type = {
 	PyVarObject_HEAD_INIT(NULL, 0)
@@ -1253,23 +2516,35 @@ static PyTypeObject pyrf_evsel__type = {
 	.tp_dealloc	= (destructor)pyrf_evsel__delete,
 	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
 	.tp_doc		= pyrf_evsel__doc,
-	.tp_members	= pyrf_evsel__members,
+	.tp_getset	= pyrf_evsel__getset,
 	.tp_methods	= pyrf_evsel__methods,
 	.tp_init	= (initproc)pyrf_evsel__init,
 	.tp_str         = pyrf_evsel__str,
 	.tp_repr        = pyrf_evsel__str,
+	.tp_getattro	= (getattrofunc) pyrf_evsel__getattro,
+	.tp_setattro	= (setattrofunc) pyrf_evsel__setattro,
 };
 
+static PyObject *pyrf_evsel__new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
+{
+	struct pyrf_evsel *pevsel;
+
+	pevsel = (struct pyrf_evsel *)PyType_GenericNew(type, args, kwargs);
+	if (pevsel)
+		pevsel->evsel = NULL;
+	return (PyObject *)pevsel;
+}
+
 static int pyrf_evsel__setup_types(void)
 {
-	pyrf_evsel__type.tp_new = PyType_GenericNew;
+	pyrf_evsel__type.tp_new = pyrf_evsel__new;
 	return PyType_Ready(&pyrf_evsel__type);
 }
 
 struct pyrf_evlist {
 	PyObject_HEAD
 
-	struct evlist evlist;
+	struct evlist *evlist;
 };
 
 static int pyrf_evlist__init(struct pyrf_evlist *pevlist,
@@ -1279,40 +2554,55 @@ static int pyrf_evlist__init(struct pyrf_evlist *pevlist,
 	struct perf_cpu_map *cpus;
 	struct perf_thread_map *threads;
 
-	if (!PyArg_ParseTuple(args, "OO", &pcpus, &pthreads))
+	if (!PyArg_ParseTuple(args, "O!O!",
+			      &pyrf_cpu_map__type, &pcpus,
+			      &pyrf_thread_map__type, &pthreads))
 		return -1;
 
+	evlist__put(pevlist->evlist);
+	pevlist->evlist = evlist__new();
+	if (!pevlist->evlist) {
+		PyErr_NoMemory();
+		return -1;
+	}
 	threads = ((struct pyrf_thread_map *)pthreads)->threads;
 	cpus = ((struct pyrf_cpu_map *)pcpus)->cpus;
-	evlist__init(&pevlist->evlist, cpus, threads);
+	perf_evlist__set_maps(evlist__core(pevlist->evlist), cpus, threads);
+
 	return 0;
 }
 
 static void pyrf_evlist__delete(struct pyrf_evlist *pevlist)
 {
-	evlist__exit(&pevlist->evlist);
+	evlist__put(pevlist->evlist);
 	Py_TYPE(pevlist)->tp_free((PyObject*)pevlist);
 }
 
 static PyObject *pyrf_evlist__all_cpus(struct pyrf_evlist *pevlist)
 {
-	struct pyrf_cpu_map *pcpu_map = PyObject_New(struct pyrf_cpu_map, &pyrf_cpu_map__type);
+	struct pyrf_cpu_map *pcpu_map;
 
+	CHECK_INITIALIZED(pevlist->evlist, "evlist");
+
+	pcpu_map = PyObject_New(struct pyrf_cpu_map, &pyrf_cpu_map__type);
 	if (pcpu_map)
-		pcpu_map->cpus = perf_cpu_map__get(pevlist->evlist.core.all_cpus);
+		pcpu_map->cpus = perf_cpu_map__get(evlist__core(pevlist->evlist)->all_cpus);
 
 	return (PyObject *)pcpu_map;
 }
 
 static PyObject *pyrf_evlist__metrics(struct pyrf_evlist *pevlist)
 {
-	PyObject *list = PyList_New(/*len=*/0);
+	PyObject *list;
 	struct rb_node *node;
 
+	CHECK_INITIALIZED(pevlist->evlist, "evlist");
+
+	list = PyList_New(/*len=*/0);
 	if (!list)
 		return NULL;
 
-	for (node = rb_first_cached(&pevlist->evlist.metric_events.entries); node;
+	for (node = rb_first_cached(&evlist__metric_events(pevlist->evlist)->entries); node;
 	     node = rb_next(node)) {
 		struct metric_event *me = container_of(node, struct metric_event, nd);
 		struct list_head *pos;
@@ -1415,10 +2705,12 @@ static PyObject *pyrf_evlist__compute_metric(struct pyrf_evlist *pevlist,
 	double result = 0;
 	struct evsel *metric_evsel = NULL;
 
+	CHECK_INITIALIZED(pevlist->evlist, "evlist");
+
 	if (!PyArg_ParseTuple(args, "sii", &metric, &cpu, &thread))
 		return NULL;
 
-	for (node = rb_first_cached(&pevlist->evlist.metric_events.entries);
+	for (node = rb_first_cached(&evlist__metric_events(pevlist->evlist)->entries);
 	     mexp == NULL && node;
 	     node = rb_next(node)) {
 		struct metric_event *me = container_of(node, struct metric_event, nd);
@@ -1434,7 +2726,7 @@ static PyObject *pyrf_evlist__compute_metric(struct pyrf_evlist *pevlist,
 			if (e->metric_events[0] == NULL)
 				continue;
 
-			evlist__for_each_entry(&pevlist->evlist, pos2) {
+			evlist__for_each_entry(pevlist->evlist, pos2) {
 				if (pos2->metric_leader != e->metric_events[0])
 					continue;
 				cpu_idx = perf_cpu_map__idx(pos2->core.cpus,
@@ -1479,15 +2771,18 @@ static PyObject *pyrf_evlist__compute_metric(struct pyrf_evlist *pevlist,
 static PyObject *pyrf_evlist__mmap(struct pyrf_evlist *pevlist,
 				   PyObject *args, PyObject *kwargs)
 {
-	struct evlist *evlist = &pevlist->evlist;
+	struct evlist *evlist;
 	static char *kwlist[] = { "pages", "overwrite", NULL };
 	int pages = 128, overwrite = false;
 
+	CHECK_INITIALIZED(pevlist->evlist, "evlist");
+
+	evlist = pevlist->evlist;
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|ii", kwlist,
 					 &pages, &overwrite))
 		return NULL;
 
-	if (evlist__mmap(evlist, pages) < 0) {
+	if (evlist__do_mmap(evlist, pages) < 0) {
 		PyErr_SetFromErrno(PyExc_OSError);
 		return NULL;
 	}
@@ -1499,10 +2794,13 @@ static PyObject *pyrf_evlist__mmap(struct pyrf_evlist *pevlist,
 static PyObject *pyrf_evlist__poll(struct pyrf_evlist *pevlist,
 				   PyObject *args, PyObject *kwargs)
 {
-	struct evlist *evlist = &pevlist->evlist;
+	struct evlist *evlist;
 	static char *kwlist[] = { "timeout", NULL };
 	int timeout = -1, n;
 
+	CHECK_INITIALIZED(pevlist->evlist, "evlist");
+
+	evlist = pevlist->evlist;
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|i", kwlist, &timeout))
 		return NULL;
 
@@ -1519,13 +2817,18 @@ static PyObject *pyrf_evlist__get_pollfd(struct pyrf_evlist *pevlist,
 					 PyObject *args __maybe_unused,
 					 PyObject *kwargs __maybe_unused)
 {
-	struct evlist *evlist = &pevlist->evlist;
-        PyObject *list = PyList_New(0);
+	struct evlist *evlist;
+	PyObject *list;
 	int i;
 
-	for (i = 0; i < evlist->core.pollfd.nr; ++i) {
+	CHECK_INITIALIZED(pevlist->evlist, "evlist");
+
+	evlist = pevlist->evlist;
+	list = PyList_New(0);
+
+	for (i = 0; i < evlist__core(evlist)->pollfd.nr; ++i) {
 		PyObject *file;
-		file = PyFile_FromFd(evlist->core.pollfd.entries[i].fd, "perf", "r", -1,
+		file = PyFile_FromFd(evlist__core(evlist)->pollfd.entries[i].fd, "perf", "r", -1,
 				     NULL, NULL, NULL, 0);
 		if (file == NULL)
 			goto free_list;
@@ -1548,27 +2851,33 @@ static PyObject *pyrf_evlist__add(struct pyrf_evlist *pevlist,
 				  PyObject *args,
 				  PyObject *kwargs __maybe_unused)
 {
-	struct evlist *evlist = &pevlist->evlist;
+	struct evlist *evlist;
 	PyObject *pevsel;
 	struct evsel *evsel;
 
-	if (!PyArg_ParseTuple(args, "O", &pevsel))
+	CHECK_INITIALIZED(pevlist->evlist, "evlist");
+
+	evlist = pevlist->evlist;
+	if (!PyArg_ParseTuple(args, "O!", &pyrf_evsel__type, &pevsel))
 		return NULL;
 
-	Py_INCREF(pevsel);
-	evsel = &((struct pyrf_evsel *)pevsel)->evsel;
-	evsel->core.idx = evlist->core.nr_entries;
-	evlist__add(evlist, evsel);
+	CHECK_INITIALIZED(((struct pyrf_evsel *)pevsel)->evsel, "evsel");
+
+	evsel = ((struct pyrf_evsel *)pevsel)->evsel;
+	CHECK_INITIALIZED(evsel, "evsel");
 
-	return Py_BuildValue("i", evlist->core.nr_entries);
+	evsel->core.idx = evlist__nr_entries(evlist);
+	evlist__add(evlist, evsel__get(evsel));
+
+	return Py_BuildValue("i", evlist__nr_entries(evlist));
 }
 
 static struct mmap *get_md(struct evlist *evlist, int cpu)
 {
 	int i;
 
-	for (i = 0; i < evlist->core.nr_mmaps; i++) {
-		struct mmap *md = &evlist->mmap[i];
+	for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) {
+		struct mmap *md = &evlist__mmap(evlist)[i];
 
 		if (md->core.cpu.cpu == cpu)
 			return md;
@@ -1580,65 +2889,60 @@ static struct mmap *get_md(struct evlist *evlist, int cpu)
 static PyObject *pyrf_evlist__read_on_cpu(struct pyrf_evlist *pevlist,
 					  PyObject *args, PyObject *kwargs)
 {
-	struct evlist *evlist = &pevlist->evlist;
+	struct evlist *evlist;
 	union perf_event *event;
+	struct evsel *evsel;
 	int sample_id_all = 1, cpu;
 	static char *kwlist[] = { "cpu", "sample_id_all", NULL };
 	struct mmap *md;
+	PyObject *pyevent;
 	int err;
 
+	CHECK_INITIALIZED(pevlist->evlist, "evlist");
+
+	evlist = pevlist->evlist;
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "i|i", kwlist,
 					 &cpu, &sample_id_all))
 		return NULL;
 
 	md = get_md(evlist, cpu);
-	if (!md) {
-		PyErr_Format(PyExc_TypeError, "Unknown CPU '%d'", cpu);
-		return NULL;
+	if (!md)
+		return PyErr_Format(PyExc_TypeError, "Unknown CPU '%d'", cpu);
+
+	err = perf_mmap__read_init(&md->core);
+	if (err < 0) {
+		if (err == -EAGAIN)
+			Py_RETURN_NONE;
+		return PyErr_Format(PyExc_OSError,
+				    "perf: error mmap read init, err=%d", err);
 	}
 
-	if (perf_mmap__read_init(&md->core) < 0)
-		goto end;
-
 	event = perf_mmap__read_event(&md->core);
-	if (event != NULL) {
-		PyObject *pyevent = pyrf_event__new(event);
-		struct pyrf_event *pevent = (struct pyrf_event *)pyevent;
-		struct evsel *evsel;
-
-		if (pyevent == NULL)
-			return PyErr_NoMemory();
-
-		evsel = evlist__event2evsel(evlist, event);
-		if (!evsel) {
-			Py_DECREF(pyevent);
-			Py_INCREF(Py_None);
-			return Py_None;
-		}
-
-		pevent->evsel = evsel;
+	if (event == NULL)
+		Py_RETURN_NONE;
 
+	evsel = evlist__event2evsel(evlist, event);
+	if (!evsel) {
+		/* Unknown evsel. */
 		perf_mmap__consume(&md->core);
-
-		err = evsel__parse_sample(evsel, &pevent->event, &pevent->sample);
-		if (err) {
-			Py_DECREF(pyevent);
-			return PyErr_Format(PyExc_OSError,
-					    "perf: can't parse sample, err=%d", err);
-		}
-
-		return pyevent;
+		Py_RETURN_NONE;
 	}
-end:
-	Py_INCREF(Py_None);
-	return Py_None;
+	pyevent = pyrf_event__new(event, evsel, evlist__session(evlist), /*machine=*/NULL);
+	perf_mmap__consume(&md->core);
+	if (pyevent == NULL)
+		return PyErr_Occurred() ? NULL : PyErr_NoMemory();
+
+	return pyevent;
 }
 
 static PyObject *pyrf_evlist__open(struct pyrf_evlist *pevlist,
 				   PyObject *args, PyObject *kwargs)
 {
-	struct evlist *evlist = &pevlist->evlist;
+	struct evlist *evlist;
 
+	CHECK_INITIALIZED(pevlist->evlist, "evlist");
+
+	evlist = pevlist->evlist;
 	if (evlist__open(evlist) < 0) {
 		PyErr_SetFromErrno(PyExc_OSError);
 		return NULL;
@@ -1650,8 +2954,11 @@ static PyObject *pyrf_evlist__open(struct pyrf_evlist *pevlist,
 
 static PyObject *pyrf_evlist__close(struct pyrf_evlist *pevlist)
 {
-	struct evlist *evlist = &pevlist->evlist;
+	struct evlist *evlist;
+
+	CHECK_INITIALIZED(pevlist->evlist, "evlist");
 
+	evlist = pevlist->evlist;
 	evlist__close(evlist);
 
 	Py_INCREF(Py_None);
@@ -1676,8 +2983,11 @@ static PyObject *pyrf_evlist__config(struct pyrf_evlist *pevlist)
 		.no_buffering        = true,
 		.no_inherit          = true,
 	};
-	struct evlist *evlist = &pevlist->evlist;
+	struct evlist *evlist;
 
+	CHECK_INITIALIZED(pevlist->evlist, "evlist");
+
+	evlist = pevlist->evlist;
 	evlist__config(evlist, &opts, &callchain_param);
 	Py_INCREF(Py_None);
 	return Py_None;
@@ -1685,14 +2995,16 @@ static PyObject *pyrf_evlist__config(struct pyrf_evlist *pevlist)
 
 static PyObject *pyrf_evlist__disable(struct pyrf_evlist *pevlist)
 {
-	evlist__disable(&pevlist->evlist);
+	CHECK_INITIALIZED(pevlist->evlist, "evlist");
+	evlist__disable(pevlist->evlist);
 	Py_INCREF(Py_None);
 	return Py_None;
 }
 
 static PyObject *pyrf_evlist__enable(struct pyrf_evlist *pevlist)
 {
-	evlist__enable(&pevlist->evlist);
+	CHECK_INITIALIZED(pevlist->evlist, "evlist");
+	evlist__enable(pevlist->evlist);
 	Py_INCREF(Py_None);
 	return Py_None;
 }
@@ -1783,7 +3095,21 @@ static Py_ssize_t pyrf_evlist__length(PyObject *obj)
 {
 	struct pyrf_evlist *pevlist = (void *)obj;
 
-	return pevlist->evlist.core.nr_entries;
+	if (!pevlist->evlist)
+		return 0;
+
+	return evlist__nr_entries(pevlist->evlist);
+}
+
+static PyObject *pyrf_evsel__from_evsel(struct evsel *evsel)
+{
+	struct pyrf_evsel *pevsel = PyObject_New(struct pyrf_evsel, &pyrf_evsel__type);
+
+	if (!pevsel)
+		return NULL;
+
+	pevsel->evsel = evsel__get(evsel);
+	return (PyObject *)pevsel;
 }
 
 static PyObject *pyrf_evlist__item(PyObject *obj, Py_ssize_t i)
@@ -1791,17 +3117,16 @@ static PyObject *pyrf_evlist__item(PyObject *obj, Py_ssize_t i)
 	struct pyrf_evlist *pevlist = (void *)obj;
 	struct evsel *pos;
 
-	if (i >= pevlist->evlist.core.nr_entries) {
+	if (!pevlist->evlist || i >= evlist__nr_entries(pevlist->evlist)) {
 		PyErr_SetString(PyExc_IndexError, "Index out of range");
 		return NULL;
 	}
 
-	evlist__for_each_entry(&pevlist->evlist, pos) {
+	evlist__for_each_entry(pevlist->evlist, pos) {
 		if (i-- == 0)
 			break;
 	}
-
-	return Py_BuildValue("O", container_of(pos, struct pyrf_evsel, evsel));
+	return pyrf_evsel__from_evsel(pos);
 }
 
 static PyObject *pyrf_evlist__str(PyObject *self)
@@ -1812,14 +3137,14 @@ static PyObject *pyrf_evlist__str(PyObject *self)
 	bool first = true;
 	PyObject *result;
 
+	if (!pevlist->evlist)
+		return PyUnicode_FromString("evlist(uninitialized)");
+
 	strbuf_addstr(&sb, "evlist([");
-	evlist__for_each_entry(&pevlist->evlist, pos) {
+	evlist__for_each_entry(pevlist->evlist, pos) {
 		if (!first)
 			strbuf_addch(&sb, ',');
-		if (!pos->pmu)
-			strbuf_addstr(&sb, evsel__name(pos));
-		else
-			strbuf_addf(&sb, "%s/%s/", pos->pmu->name, evsel__name(pos));
+		strbuf_addstr(&sb, evsel__name(pos));
 		first = false;
 	}
 	strbuf_addstr(&sb, "])");
@@ -1835,6 +3160,24 @@ static PySequenceMethods pyrf_evlist__sequence_methods = {
 
 static const char pyrf_evlist__doc[] = PyDoc_STR("perf event selector list object.");
 
+static PyObject *pyrf_evlist__getattro(struct pyrf_evlist *pevlist, PyObject *attr_name)
+{
+	if (!pevlist->evlist) {
+		PyErr_SetString(PyExc_ValueError, "evlist not initialized");
+		return NULL;
+	}
+	return PyObject_GenericGetAttr((PyObject *) pevlist, attr_name);
+}
+
+static int pyrf_evlist__setattro(struct pyrf_evlist *pevlist, PyObject *attr_name, PyObject *value)
+{
+	if (!pevlist->evlist) {
+		PyErr_SetString(PyExc_ValueError, "evlist not initialized");
+		return -1;
+	}
+	return PyObject_GenericSetAttr((PyObject *) pevlist, attr_name, value);
+}
+
 static PyTypeObject pyrf_evlist__type = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	.tp_name	= "perf.evlist",
@@ -1847,11 +3190,23 @@ static PyTypeObject pyrf_evlist__type = {
 	.tp_init	= (initproc)pyrf_evlist__init,
 	.tp_repr        = pyrf_evlist__str,
 	.tp_str         = pyrf_evlist__str,
+	.tp_getattro	= (getattrofunc) pyrf_evlist__getattro,
+	.tp_setattro	= (setattrofunc) pyrf_evlist__setattro,
 };
 
+static PyObject *pyrf_evlist__new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
+{
+	struct pyrf_evlist *pevlist;
+
+	pevlist = (struct pyrf_evlist *)PyType_GenericNew(type, args, kwargs);
+	if (pevlist)
+		pevlist->evlist = NULL;
+	return (PyObject *)pevlist;
+}
+
 static int pyrf_evlist__setup_types(void)
 {
-	pyrf_evlist__type.tp_new = PyType_GenericNew;
+	pyrf_evlist__type.tp_new = pyrf_evlist__new;
 	return PyType_Ready(&pyrf_evlist__type);
 }
 
@@ -1871,6 +3226,7 @@ static const struct perf_constant perf__constants[] = {
 	PERF_CONST(TYPE_BREAKPOINT),
 
 	PERF_CONST(COUNT_HW_CPU_CYCLES),
+	PERF_CONST(COUNT_HW_REF_CPU_CYCLES),
 	PERF_CONST(COUNT_HW_INSTRUCTIONS),
 	PERF_CONST(COUNT_HW_CACHE_REFERENCES),
 	PERF_CONST(COUNT_HW_CACHE_MISSES),
@@ -1935,6 +3291,8 @@ static const struct perf_constant perf__constants[] = {
 	PERF_CONST(RECORD_LOST_SAMPLES),
 	PERF_CONST(RECORD_SWITCH),
 	PERF_CONST(RECORD_SWITCH_CPU_WIDE),
+	PERF_CONST(RECORD_STAT),
+	PERF_CONST(RECORD_STAT_ROUND),
 
 	PERF_CONST(RECORD_MISC_SWITCH_OUT),
 	{ .name = NULL, },
@@ -1954,157 +3312,107 @@ static PyObject *pyrf__tracepoint(struct pyrf_evsel *pevsel,
 	return PyLong_FromLong(tp_pmu__id(sys, name));
 }
 
-static PyObject *pyrf_evsel__from_evsel(struct evsel *evsel)
-{
-	struct pyrf_evsel *pevsel = PyObject_New(struct pyrf_evsel, &pyrf_evsel__type);
-
-	if (!pevsel)
-		return NULL;
-
-	memset(&pevsel->evsel, 0, sizeof(pevsel->evsel));
-	evsel__init(&pevsel->evsel, &evsel->core.attr, evsel->core.idx);
-
-	evsel__clone(&pevsel->evsel, evsel);
-	if (evsel__is_group_leader(evsel))
-		evsel__set_leader(&pevsel->evsel, &pevsel->evsel);
-	return (PyObject *)pevsel;
-}
-
-static int evlist__pos(struct evlist *evlist, struct evsel *evsel)
-{
-	struct evsel *pos;
-	int idx = 0;
-
-	evlist__for_each_entry(evlist, pos) {
-		if (evsel == pos)
-			return idx;
-		idx++;
-	}
-	return -1;
-}
-
-static struct evsel *evlist__at(struct evlist *evlist, int idx)
-{
-	struct evsel *pos;
-	int idx2 = 0;
-
-	evlist__for_each_entry(evlist, pos) {
-		if (idx == idx2)
-			return pos;
-		idx2++;
-	}
-	return NULL;
-}
-
 static PyObject *pyrf_evlist__from_evlist(struct evlist *evlist)
 {
 	struct pyrf_evlist *pevlist = PyObject_New(struct pyrf_evlist, &pyrf_evlist__type);
-	struct evsel *pos;
-	struct rb_node *node;
 
 	if (!pevlist)
 		return NULL;
 
-	memset(&pevlist->evlist, 0, sizeof(pevlist->evlist));
-	evlist__init(&pevlist->evlist, evlist->core.all_cpus, evlist->core.threads);
-	evlist__for_each_entry(evlist, pos) {
-		struct pyrf_evsel *pevsel = (void *)pyrf_evsel__from_evsel(pos);
-
-		evlist__add(&pevlist->evlist, &pevsel->evsel);
-	}
-	evlist__for_each_entry(&pevlist->evlist, pos) {
-		struct evsel *leader = evsel__leader(pos);
-
-		if (pos != leader) {
-			int idx = evlist__pos(evlist, leader);
-
-			if (idx >= 0)
-				evsel__set_leader(pos, evlist__at(&pevlist->evlist, idx));
-			else if (leader == NULL)
-				evsel__set_leader(pos, pos);
-		}
-
-		leader = pos->metric_leader;
-
-		if (pos != leader) {
-			int idx = evlist__pos(evlist, leader);
-
-			if (idx >= 0)
-				pos->metric_leader = evlist__at(&pevlist->evlist, idx);
-			else if (leader == NULL)
-				pos->metric_leader = pos;
-		}
-	}
-	metricgroup__copy_metric_events(&pevlist->evlist, /*cgrp=*/NULL,
-					&pevlist->evlist.metric_events,
-					&evlist->metric_events);
-	for (node = rb_first_cached(&pevlist->evlist.metric_events.entries); node;
-	     node = rb_next(node)) {
-		struct metric_event *me = container_of(node, struct metric_event, nd);
-		struct list_head *mpos;
-		int idx = evlist__pos(evlist, me->evsel);
-
-		if (idx >= 0)
-			me->evsel = evlist__at(&pevlist->evlist, idx);
-		list_for_each(mpos, &me->head) {
-			struct metric_expr *e = container_of(mpos, struct metric_expr, nd);
-
-			for (int j = 0; e->metric_events[j]; j++) {
-				idx = evlist__pos(evlist, e->metric_events[j]);
-				if (idx >= 0)
-					e->metric_events[j] = evlist__at(&pevlist->evlist, idx);
-			}
-		}
-	}
+	pevlist->evlist = evlist__get(evlist);
 	return (PyObject *)pevlist;
 }
 
 static PyObject *pyrf__parse_events(PyObject *self, PyObject *args)
 {
 	const char *input;
-	struct evlist evlist = {};
+	struct evlist *evlist = evlist__new();
 	struct parse_events_error err;
 	PyObject *result;
 	PyObject *pcpus = NULL, *pthreads = NULL;
 	struct perf_cpu_map *cpus;
 	struct perf_thread_map *threads;
 
-	if (!PyArg_ParseTuple(args, "s|OO", &input, &pcpus, &pthreads))
+	if (!evlist)
+		return PyErr_NoMemory();
+
+	if (!PyArg_ParseTuple(args, "s|OO", &input, &pcpus, &pthreads)) {
+		evlist__put(evlist);
+		return NULL;
+	}
+
+	if (pthreads && pthreads != Py_None &&
+	    !PyObject_TypeCheck(pthreads, &pyrf_thread_map__type)) {
+		PyErr_SetString(PyExc_TypeError, "threads must be a perf.thread_map or None");
+		evlist__put(evlist);
+		return NULL;
+	}
+
+	if (pcpus && pcpus != Py_None &&
+	    !PyObject_TypeCheck(pcpus, &pyrf_cpu_map__type)) {
+		PyErr_SetString(PyExc_TypeError, "cpus must be a perf.cpu_map or None");
+		evlist__put(evlist);
 		return NULL;
+	}
 
-	threads = pthreads ? ((struct pyrf_thread_map *)pthreads)->threads : NULL;
-	cpus = pcpus ? ((struct pyrf_cpu_map *)pcpus)->cpus : NULL;
+	threads = (pthreads && pthreads != Py_None) ?
+			((struct pyrf_thread_map *)pthreads)->threads : NULL;
+	cpus = (pcpus && pcpus != Py_None) ?
+			((struct pyrf_cpu_map *)pcpus)->cpus : NULL;
 
 	parse_events_error__init(&err);
-	evlist__init(&evlist, cpus, threads);
-	if (parse_events(&evlist, input, &err)) {
+	perf_evlist__set_maps(evlist__core(evlist), cpus, threads);
+	if (parse_events(evlist, input, &err)) {
 		parse_events_error__print(&err, input);
 		PyErr_SetFromErrno(PyExc_OSError);
+		evlist__put(evlist);
 		return NULL;
 	}
-	result = pyrf_evlist__from_evlist(&evlist);
-	evlist__exit(&evlist);
+	result = pyrf_evlist__from_evlist(evlist);
+	evlist__put(evlist);
 	return result;
 }
 
 static PyObject *pyrf__parse_metrics(PyObject *self, PyObject *args)
 {
 	const char *input, *pmu = NULL;
-	struct evlist evlist = {};
+	struct evlist *evlist = evlist__new();
 	PyObject *result;
 	PyObject *pcpus = NULL, *pthreads = NULL;
 	struct perf_cpu_map *cpus;
 	struct perf_thread_map *threads;
 	int ret;
 
-	if (!PyArg_ParseTuple(args, "s|sOO", &input, &pmu, &pcpus, &pthreads))
+	if (!evlist)
+		return PyErr_NoMemory();
+
+	if (!PyArg_ParseTuple(args, "s|sOO", &input, &pmu, &pcpus, &pthreads)) {
+		evlist__put(evlist);
+		return NULL;
+	}
+
+	if (pthreads && pthreads != Py_None &&
+	    !PyObject_TypeCheck(pthreads, &pyrf_thread_map__type)) {
+		PyErr_SetString(PyExc_TypeError, "threads must be a perf.thread_map or None");
+		evlist__put(evlist);
 		return NULL;
+	}
+
+	if (pcpus && pcpus != Py_None &&
+	    !PyObject_TypeCheck(pcpus, &pyrf_cpu_map__type)) {
+		PyErr_SetString(PyExc_TypeError, "cpus must be a perf.cpu_map or None");
+		evlist__put(evlist);
+		return NULL;
+	}
 
-	threads = pthreads ? ((struct pyrf_thread_map *)pthreads)->threads : NULL;
-	cpus = pcpus ? ((struct pyrf_cpu_map *)pcpus)->cpus : NULL;
+	threads = (pthreads && pthreads != Py_None) ?
+			((struct pyrf_thread_map *)pthreads)->threads : NULL;
+	cpus = (pcpus && pcpus != Py_None) ?
+			((struct pyrf_cpu_map *)pcpus)->cpus : NULL;
 
-	evlist__init(&evlist, cpus, threads);
-	ret = metricgroup__parse_groups(&evlist, pmu ?: "all", input,
+	perf_evlist__set_maps(evlist__core(evlist), cpus, threads);
+	ret = metricgroup__parse_groups(evlist, pmu ?: "all",
+					/*cputype_filter=*/false, input,
 					/*metric_no_group=*/ false,
 					/*metric_no_merge=*/ false,
 					/*metric_no_threshold=*/ true,
@@ -2112,12 +3420,13 @@ static PyObject *pyrf__parse_metrics(PyObject *self, PyObject *args)
 					/*system_wide=*/true,
 					/*hardware_aware_grouping=*/ false);
 	if (ret) {
+		evlist__put(evlist);
 		errno = -ret;
 		PyErr_SetFromErrno(PyExc_OSError);
 		return NULL;
 	}
-	result = pyrf_evlist__from_evlist(&evlist);
-	evlist__exit(&evlist);
+	result = pyrf_evlist__from_evlist(evlist);
+	evlist__put(evlist);
 	return result;
 }
 
@@ -2200,7 +3509,544 @@ static PyObject *pyrf__metrics(PyObject *self, PyObject *args)
 	return list;
 }
 
+struct pyrf_data {
+	PyObject_HEAD
+
+	struct perf_data data;
+};
+
+static int pyrf_data__init(struct pyrf_data *pdata, PyObject *args, PyObject *kwargs)
+{
+	static char *kwlist[] = { "path", "fd", NULL };
+	char *path = NULL;
+	int fd = -1;
+
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|si", kwlist, &path, &fd))
+		return -1;
+
+	if (pdata->data.open)
+		perf_data__close(&pdata->data);
+	free((char *)pdata->data.path);
+	pdata->data.path = NULL;
+
+	if (fd != -1) {
+		struct stat st;
+
+		if (fstat(fd, &st) < 0 || !S_ISFIFO(st.st_mode)) {
+			PyErr_SetString(PyExc_ValueError,
+					"fd argument is only supported for pipes");
+			return -1;
+		}
+		if (!path)
+			path = "-";
+		else if (strcmp(path, "-") != 0) {
+			PyErr_SetString(PyExc_ValueError,
+					"path must be '-' when fd is provided");
+			return -1;
+		}
+		fd = dup(fd);
+		if (fd < 0) {
+			PyErr_SetFromErrno(PyExc_OSError);
+			return -1;
+		}
+	} else if (path && strcmp(path, "-") == 0) {
+		fd = dup(0);
+		if (fd < 0) {
+			PyErr_SetFromErrno(PyExc_OSError);
+			return -1;
+		}
+	}
+
+	if (!path)
+		path = "perf.data";
+
+	pdata->data.path = strdup(path);
+	if (!pdata->data.path) {
+		if (fd != -1)
+			close(fd);
+		PyErr_NoMemory();
+		return -1;
+	}
+
+	pdata->data.mode = PERF_DATA_MODE_READ;
+	pdata->data.file.fd = fd;
+	if (perf_data__open(&pdata->data) < 0) {
+		PyErr_Format(PyExc_IOError, "Failed to open perf data: %s",
+			     pdata->data.path ? pdata->data.path : "perf.data");
+		return -1;
+	}
+	return 0;
+}
+
+static void pyrf_data__delete(struct pyrf_data *pdata)
+{
+	perf_data__close(&pdata->data);
+	free((char *)pdata->data.path);
+	Py_TYPE(pdata)->tp_free((PyObject *)pdata);
+}
+
+static PyObject *pyrf_data__str(PyObject *self)
+{
+	const struct pyrf_data *pdata = (const struct pyrf_data *)self;
+
+	if (!pdata->data.path)
+		return PyUnicode_FromString("[uninitialized]");
+	return PyUnicode_FromString(pdata->data.path);
+}
+
+static PyObject *pyrf_data__new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
+{
+	struct pyrf_data *pdata;
+
+	pdata = (struct pyrf_data *)PyType_GenericNew(type, args, kwargs);
+	if (pdata)
+		memset(&pdata->data, 0, sizeof(pdata->data));
+	return (PyObject *)pdata;
+}
+
+static const char pyrf_data__doc[] = PyDoc_STR("perf data file object.");
+
+static PyTypeObject pyrf_data__type = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_name	= "perf.data",
+	.tp_basicsize	= sizeof(struct pyrf_data),
+	.tp_dealloc	= (destructor)pyrf_data__delete,
+	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
+	.tp_doc		= pyrf_data__doc,
+	.tp_init	= (initproc)pyrf_data__init,
+	.tp_repr	= pyrf_data__str,
+	.tp_str		= pyrf_data__str,
+};
+
+static int pyrf_data__setup_types(void)
+{
+	pyrf_data__type.tp_new = pyrf_data__new;
+	return PyType_Ready(&pyrf_data__type);
+}
+
+struct pyrf_thread {
+	PyObject_HEAD
+
+	struct thread *thread;
+};
+
+static void pyrf_thread__delete(struct pyrf_thread *pthread)
+{
+	thread__put(pthread->thread);
+	Py_TYPE(pthread)->tp_free((PyObject *)pthread);
+}
+
+static PyObject *pyrf_thread__comm(PyObject *obj)
+{
+	struct pyrf_thread *pthread = (void *)obj;
+	const char *str;
+
+	CHECK_INITIALIZED(pthread->thread, "perf.thread");
+
+	str = thread__comm_str(pthread->thread);
+
+	if (!str)
+		Py_RETURN_NONE;
+
+	return PyUnicode_FromString(str);
+}
+
+static PyMethodDef pyrf_thread__methods[] = {
+	{
+		.ml_name  = "comm",
+		.ml_meth  = (PyCFunction)pyrf_thread__comm,
+		.ml_flags = METH_NOARGS,
+		.ml_doc	  = PyDoc_STR("Comm(and) associated with this thread.")
+	},
+	{ .ml_name = NULL, }
+};
+
+static PyObject *pyrf_thread__get_pid(struct pyrf_thread *pthread, void *closure __maybe_unused)
+{
+	return PyLong_FromLong(thread__pid(pthread->thread));
+}
+
+static PyObject *pyrf_thread__get_tid(struct pyrf_thread *pthread, void *closure __maybe_unused)
+{
+	return PyLong_FromLong(thread__tid(pthread->thread));
+}
+
+static PyObject *pyrf_thread__get_ppid(struct pyrf_thread *pthread, void *closure __maybe_unused)
+{
+	return PyLong_FromLong(thread__ppid(pthread->thread));
+}
+
+static PyObject *pyrf_thread__get_cpu(struct pyrf_thread *pthread, void *closure __maybe_unused)
+{
+	return PyLong_FromLong(thread__cpu(pthread->thread));
+}
+
+static PyGetSetDef pyrf_thread__getset[] = {
+	{ .name = "pid", .get = (getter)pyrf_thread__get_pid, .doc = "process ID" },
+	{ .name = "tid", .get = (getter)pyrf_thread__get_tid, .doc = "thread ID" },
+	{ .name = "ppid", .get = (getter)pyrf_thread__get_ppid, .doc = "parent process ID" },
+	{ .name = "cpu", .get = (getter)pyrf_thread__get_cpu, .doc = "cpu number" },
+	{ .name = NULL }
+};
+
+static const char pyrf_thread__doc[] = PyDoc_STR("perf thread object.");
+
+static PyTypeObject pyrf_thread__type = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_name	= "perf.thread",
+	.tp_basicsize	= sizeof(struct pyrf_thread),
+	.tp_dealloc	= (destructor)pyrf_thread__delete,
+	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
+	.tp_methods	= pyrf_thread__methods,
+	.tp_getset	= pyrf_thread__getset,
+	.tp_doc		= pyrf_thread__doc,
+};
+
+static int pyrf_thread__setup_types(void)
+{
+	return PyType_Ready(&pyrf_thread__type);
+}
+
+static PyObject *pyrf_thread__from_thread(struct thread *thread)
+{
+	struct pyrf_thread *pthread = PyObject_New(struct pyrf_thread, &pyrf_thread__type);
+
+	if (!pthread)
+		return NULL;
+
+	pthread->thread = thread__get(thread);
+	return (PyObject *)pthread;
+}
+
+struct pyrf_session {
+	PyObject_HEAD
+
+	struct perf_session *session;
+	struct perf_tool tool;
+	struct pyrf_data *pdata;
+	PyObject *sample;
+	PyObject *stat;
+};
+
+static int pyrf_session_tool__sample(const struct perf_tool *tool,
+				     union perf_event *event,
+				     struct perf_sample *sample,
+				     struct machine *machine)
+{
+	struct pyrf_session *psession = container_of(tool, struct pyrf_session, tool);
+	PyObject *pyevent = pyrf_event__new(event, sample->evsel, psession->session, machine);
+	PyObject *ret;
+
+	if (pyevent == NULL)
+		return -ENOMEM;
+
+	ret = PyObject_CallFunction(psession->sample, "O", pyevent);
+	if (!ret) {
+		Py_DECREF(pyevent);
+		return -1;
+	}
+	Py_DECREF(ret);
+	Py_DECREF(pyevent);
+	return 0;
+}
+
+static int pyrf_session_tool__stat(const struct perf_tool *tool,
+				   struct perf_session *session,
+				   union perf_event *event)
+{
+	struct pyrf_session *psession = container_of(tool, struct pyrf_session, tool);
+	struct evsel *evsel = evlist__id2evsel(session->evlist, event->stat.id);
+	PyObject *pyevent = pyrf_event__new(event, /*evsel=*/NULL, psession->session,
+					    /*machine=*/NULL);
+	const char *name = evsel ? evsel__name(evsel) : "unknown";
+	PyObject *ret;
+
+	if (pyevent == NULL)
+		return -ENOMEM;
+
+	ret = PyObject_CallFunction(psession->stat, "Oz", pyevent, name);
+	if (!ret) {
+		Py_DECREF(pyevent);
+		return -1;
+	}
+	Py_DECREF(ret);
+	Py_DECREF(pyevent);
+	return 0;
+}
+
+static int pyrf_session_tool__stat_round(const struct perf_tool *tool,
+					 struct perf_session *session __maybe_unused,
+					 union perf_event *event)
+{
+	struct pyrf_session *psession = container_of(tool, struct pyrf_session, tool);
+	PyObject *pyevent = pyrf_event__new(event, /*evsel=*/NULL, psession->session,
+					    /*machine=*/NULL);
+	PyObject *ret;
+
+	if (pyevent == NULL)
+		return -ENOMEM;
+
+	ret = PyObject_CallFunction(psession->stat, "Oz", pyevent, NULL);
+	if (!ret) {
+		Py_DECREF(pyevent);
+		return -1;
+	}
+	Py_DECREF(ret);
+	Py_DECREF(pyevent);
+	return 0;
+}
+
+static PyObject *pyrf_session__find_thread(struct pyrf_session *psession, PyObject *args)
+{
+	struct machine *machine;
+	struct thread *thread = NULL;
+	PyObject *result;
+	int pid;
+
+	CHECK_INITIALIZED(psession->session, "session");
+
+	if (!PyArg_ParseTuple(args, "i", &pid))
+		return NULL;
+
+	machine = &psession->session->machines.host;
+	thread = machine__find_thread(machine, pid, pid);
+
+	if (!thread) {
+		machine = perf_session__find_machine(psession->session, pid);
+		if (machine)
+			thread = machine__find_thread(machine, pid, pid);
+	}
+
+	if (!thread) {
+		PyErr_Format(PyExc_TypeError, "Failed to find thread %d", pid);
+		return NULL;
+	}
+	result = pyrf_thread__from_thread(thread);
+	thread__put(thread);
+	return result;
+}
+
+static PyObject *pyrf_session__new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
+{
+	struct pyrf_data *pdata;
+	PyObject *sample = NULL, *stat = NULL;
+	static char *kwlist[] = { "data", "sample", "stat", NULL };
+	struct pyrf_session *psession;
+	struct perf_session *session;
+
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O!|OO", kwlist, &pyrf_data__type, &pdata,
+					 &sample, &stat))
+		return NULL;
+
+	psession = PyObject_New(struct pyrf_session, type);
+	if (!psession)
+		return NULL;
+
+	psession->session = NULL;
+	psession->sample = NULL;
+	psession->stat = NULL;
+	psession->pdata = NULL;
+
+	Py_INCREF(pdata);
+	psession->pdata = pdata;
+
+	perf_tool__init(&psession->tool, /*ordered_events=*/true);
+	psession->tool.ordering_requires_timestamps = true;
+
+	#define ADD_TOOL(name)						\
+	do {								\
+		if (name) {						\
+			if (!PyCallable_Check(name)) {			\
+				PyErr_SetString(PyExc_TypeError, #name " must be callable"); \
+				goto err_out;				\
+			}						\
+			psession->tool.name = pyrf_session_tool__##name; \
+			Py_INCREF(name);				\
+			psession->name = name;				\
+		}							\
+	} while (0)
+
+	ADD_TOOL(sample);
+	ADD_TOOL(stat);
+	#undef ADD_TOOL
+
+	if (stat)
+		psession->tool.stat_round = pyrf_session_tool__stat_round;
+
+
+	psession->tool.comm		= perf_event__process_comm;
+	psession->tool.mmap		= perf_event__process_mmap;
+	psession->tool.mmap2            = perf_event__process_mmap2;
+	psession->tool.namespaces       = perf_event__process_namespaces;
+	psession->tool.cgroup           = perf_event__process_cgroup;
+	psession->tool.exit             = perf_event__process_exit;
+	psession->tool.fork             = perf_event__process_fork;
+	psession->tool.ksymbol          = perf_event__process_ksymbol;
+	psession->tool.text_poke        = perf_event__process_text_poke;
+	psession->tool.build_id         = perf_event__process_build_id;
+	psession->tool.attr		= perf_event__process_attr;
+	psession->tool.feature		= perf_event__process_feature;
+
+	session = perf_session__new(&pdata->data, &psession->tool);
+	if (IS_ERR(session)) {
+		PyErr_Format(PyExc_IOError, "failed to create session: %ld", PTR_ERR(session));
+		goto err_out;
+	}
+	psession->session = session;
+
+	symbol_conf.use_callchain = true;
+	symbol_conf.show_kernel_path = true;
+	symbol_conf.inline_name = false;
+	if (symbol__init(perf_session__env(session)) < 0) {
+		PyErr_SetString(PyExc_OSError, "perf: symbol__init failed");
+		goto err_out;
+	}
+
+
+
+	return (PyObject *)psession;
+err_out:
+	Py_DECREF(psession);
+	return NULL;
+}
+
+static void pyrf_session__delete(struct pyrf_session *psession)
+{
+	perf_session__delete(psession->session);
+	Py_XDECREF(psession->pdata);
+	Py_XDECREF(psession->sample);
+	Py_XDECREF(psession->stat);
+	Py_TYPE(psession)->tp_free((PyObject *)psession);
+}
+
+static PyObject *pyrf_session__find_thread_events(struct pyrf_session *psession)
+{
+	int err;
+
+	CHECK_INITIALIZED(psession->session, "session");
+
+	err = perf_session__process_events(psession->session);
+
+	if (PyErr_Occurred())
+		return NULL;
+
+	if (err < 0) {
+		PyErr_Format(PyExc_OSError, "Process events failed: %d", err);
+		return NULL;
+	}
+
+	Py_RETURN_NONE;
+}
+
+static PyMethodDef pyrf_session__methods[] = {
+	{
+		.ml_name  = "process_events",
+		.ml_meth  = (PyCFunction)pyrf_session__find_thread_events,
+		.ml_flags = METH_NOARGS,
+		.ml_doc	  = PyDoc_STR("Iterate and process events.")
+	},
+	{
+		.ml_name  = "find_thread",
+		.ml_meth  = (PyCFunction)pyrf_session__find_thread,
+		.ml_flags = METH_VARARGS,
+		.ml_doc	  = PyDoc_STR("Returns the thread associated with a pid.")
+	},
+	{ .ml_name = NULL, }
+};
+
+static const char pyrf_session__doc[] = PyDoc_STR("perf session object.");
+
+static PyObject *pyrf_session__getattro(struct pyrf_session *psession, PyObject *attr_name)
+{
+	if (!psession->session) {
+		PyErr_SetString(PyExc_ValueError, "session not initialized");
+		return NULL;
+	}
+	return PyObject_GenericGetAttr((PyObject *) psession, attr_name);
+}
+
+static int pyrf_session__setattro(struct pyrf_session *psession, PyObject *attr_name,
+				  PyObject *value)
+{
+	if (!psession->session) {
+		PyErr_SetString(PyExc_ValueError, "session not initialized");
+		return -1;
+	}
+	return PyObject_GenericSetAttr((PyObject *) psession, attr_name, value);
+}
+
+static PyTypeObject pyrf_session__type = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_name	= "perf.session",
+	.tp_basicsize	= sizeof(struct pyrf_session),
+	.tp_dealloc	= (destructor)pyrf_session__delete,
+	.tp_flags	= Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,
+	.tp_methods	= pyrf_session__methods,
+	.tp_doc		= pyrf_session__doc,
+	.tp_new		= pyrf_session__new,
+	.tp_getattro	= (getattrofunc) pyrf_session__getattro,
+	.tp_setattro	= (setattrofunc) pyrf_session__setattro,
+};
+
+static int pyrf_session__setup_types(void)
+{
+	return PyType_Ready(&pyrf_session__type);
+}
+
+static PyObject *pyrf__syscall_name(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+	const char *name;
+	int id;
+	int elf_machine = EM_HOST;
+	static char *kwlist[] = { "id", "elf_machine", NULL };
+
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "i|$i", kwlist, &id, &elf_machine))
+		return NULL;
+
+	name = syscalltbl__name(elf_machine, id);
+	if (!name)
+		Py_RETURN_NONE;
+	return PyUnicode_FromString(name);
+}
+
+static PyObject *pyrf__syscall_id(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+	const char *name;
+	int id;
+	int elf_machine = EM_HOST;
+	static char *kwlist[] = { "name", "elf_machine", NULL };
+
+	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|$i", kwlist, &name, &elf_machine))
+		return NULL;
+
+	id = syscalltbl__id(elf_machine, name);
+	if (id < 0) {
+		PyErr_Format(PyExc_ValueError, "Failed to find syscall %s", name);
+		return NULL;
+	}
+	return PyLong_FromLong(id);
+}
+
+static PyObject *pyrf__config_get(PyObject *self, PyObject *args)
+{
+	const char *config_name, *val;
+
+	if (!PyArg_ParseTuple(args, "s", &config_name))
+		return NULL;
+
+	val = perf_config_get(config_name);
+	if (!val)
+		Py_RETURN_NONE;
+	return PyUnicode_FromString(val);
+}
+
 static PyMethodDef perf__methods[] = {
+	{
+		.ml_name  = "config_get",
+		.ml_meth  = (PyCFunction) pyrf__config_get,
+		.ml_flags = METH_VARARGS,
+		.ml_doc	  = PyDoc_STR("Get a perf config value.")
+	},
 	{
 		.ml_name  = "metrics",
 		.ml_meth  = (PyCFunction) pyrf__metrics,
@@ -2233,6 +4079,18 @@ static PyMethodDef perf__methods[] = {
 		.ml_flags = METH_NOARGS,
 		.ml_doc	  = PyDoc_STR("Returns a sequence of pmus.")
 	},
+	{
+		.ml_name  = "syscall_name",
+		.ml_meth  = (PyCFunction) pyrf__syscall_name,
+		.ml_flags = METH_VARARGS | METH_KEYWORDS,
+		.ml_doc	  = PyDoc_STR("Turns a syscall number to a string.")
+	},
+	{
+		.ml_name  = "syscall_id",
+		.ml_meth  = (PyCFunction) pyrf__syscall_id,
+		.ml_flags = METH_VARARGS | METH_KEYWORDS,
+		.ml_doc	  = PyDoc_STR("Turns a syscall name to a number.")
+	},
 	{ .ml_name = NULL, }
 };
 
@@ -2254,29 +4112,48 @@ PyMODINIT_FUNC PyInit_perf(void)
 	};
 	PyObject *module = PyModule_Create(&moduledef);
 
-	if (module == NULL ||
-	    pyrf_event__setup_types() < 0 ||
+	if (module == NULL)
+		return NULL;
+
+	if (pyrf_event__setup_types() < 0 ||
 	    pyrf_evlist__setup_types() < 0 ||
 	    pyrf_evsel__setup_types() < 0 ||
 	    pyrf_thread_map__setup_types() < 0 ||
 	    pyrf_cpu_map__setup_types() < 0 ||
 	    pyrf_pmu_iterator__setup_types() < 0 ||
 	    pyrf_pmu__setup_types() < 0 ||
-	    pyrf_counts_values__setup_types() < 0)
-		return module;
+	    pyrf_counts_values__setup_types() < 0 ||
+	    pyrf_data__setup_types() < 0 ||
+	    pyrf_session__setup_types() < 0 ||
+	    pyrf_thread__setup_types() < 0) {
+		Py_DECREF(module);
+		return NULL;
+	}
 
 	/* The page_size is placed in util object. */
 	page_size = sysconf(_SC_PAGE_SIZE);
 
 	Py_INCREF(&pyrf_evlist__type);
-	PyModule_AddObject(module, "evlist", (PyObject*)&pyrf_evlist__type);
+	PyModule_AddObject(module, "evlist", (PyObject *)&pyrf_evlist__type);
 
 	Py_INCREF(&pyrf_evsel__type);
-	PyModule_AddObject(module, "evsel", (PyObject*)&pyrf_evsel__type);
+	PyModule_AddObject(module, "evsel", (PyObject *)&pyrf_evsel__type);
+
+	Py_INCREF(&pyrf_thread__type);
+	PyModule_AddObject(module, "thread", (PyObject *)&pyrf_thread__type);
+
+	Py_INCREF(&pyrf_callchain__type);
+	PyModule_AddObject(module, "callchain", (PyObject *)&pyrf_callchain__type);
+
+	Py_INCREF(&pyrf_callchain_node__type);
+	PyModule_AddObject(module, "callchain_node", (PyObject *)&pyrf_callchain_node__type);
 
 	Py_INCREF(&pyrf_mmap_event__type);
 	PyModule_AddObject(module, "mmap_event", (PyObject *)&pyrf_mmap_event__type);
 
+	Py_INCREF(&pyrf_mmap2_event__type);
+	PyModule_AddObject(module, "mmap2_event", (PyObject *)&pyrf_mmap2_event__type);
+
 	Py_INCREF(&pyrf_lost_event__type);
 	PyModule_AddObject(module, "lost_event", (PyObject *)&pyrf_lost_event__type);
 
@@ -2301,6 +4178,12 @@ PyMODINIT_FUNC PyInit_perf(void)
 	Py_INCREF(&pyrf_context_switch_event__type);
 	PyModule_AddObject(module, "switch_event", (PyObject *)&pyrf_context_switch_event__type);
 
+	Py_INCREF(&pyrf_stat_event__type);
+	PyModule_AddObject(module, "stat_event", (PyObject *)&pyrf_stat_event__type);
+
+	Py_INCREF(&pyrf_stat_round_event__type);
+	PyModule_AddObject(module, "stat_round_event", (PyObject *)&pyrf_stat_round_event__type);
+
 	Py_INCREF(&pyrf_thread_map__type);
 	PyModule_AddObject(module, "thread_map", (PyObject*)&pyrf_thread_map__type);
 
@@ -2310,6 +4193,24 @@ PyMODINIT_FUNC PyInit_perf(void)
 	Py_INCREF(&pyrf_counts_values__type);
 	PyModule_AddObject(module, "counts_values", (PyObject *)&pyrf_counts_values__type);
 
+	Py_INCREF(&pyrf_data__type);
+	PyModule_AddObject(module, "data", (PyObject *)&pyrf_data__type);
+
+	Py_INCREF(&pyrf_session__type);
+	PyModule_AddObject(module, "session", (PyObject *)&pyrf_session__type);
+
+	Py_INCREF(&pyrf_branch_entry__type);
+	if (PyModule_AddObject(module, "branch_entry", (PyObject *)&pyrf_branch_entry__type) < 0) {
+		Py_DECREF(&pyrf_branch_entry__type);
+		goto error;
+	}
+
+	Py_INCREF(&pyrf_branch_stack__type);
+	if (PyModule_AddObject(module, "branch_stack", (PyObject *)&pyrf_branch_stack__type) < 0) {
+		Py_DECREF(&pyrf_branch_stack__type);
+		goto error;
+	}
+
 	dict = PyModule_GetDict(module);
 	if (dict == NULL)
 		goto error;
@@ -2323,7 +4224,9 @@ PyMODINIT_FUNC PyInit_perf(void)
 	}
 
 error:
-	if (PyErr_Occurred())
-		PyErr_SetString(PyExc_ImportError, "perf: Init failed!");
+	if (PyErr_Occurred()) {
+		Py_XDECREF(module);
+		return NULL;
+	}
 	return module;
 }
diff --git a/tools/perf/util/record.c b/tools/perf/util/record.c
index e867de8ddaaa4..38e8aee3106b7 100644
--- a/tools/perf/util/record.c
+++ b/tools/perf/util/record.c
@@ -99,7 +99,7 @@ void evlist__config(struct evlist *evlist, struct record_opts *opts, struct call
 	bool use_comm_exec;
 	bool sample_id = opts->sample_id;
 
-	if (perf_cpu_map__cpu(evlist->core.user_requested_cpus, 0).cpu < 0)
+	if (perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, 0).cpu < 0)
 		opts->no_inherit = true;
 
 	use_comm_exec = perf_can_comm_exec();
@@ -122,7 +122,7 @@ void evlist__config(struct evlist *evlist, struct record_opts *opts, struct call
 		 */
 		use_sample_identifier = perf_can_sample_identifier();
 		sample_id = true;
-	} else if (evlist->core.nr_entries > 1) {
+	} else if (evlist__nr_entries(evlist) > 1) {
 		struct evsel *first = evlist__first(evlist);
 
 		evlist__for_each_entry(evlist, evsel) {
@@ -237,7 +237,8 @@ bool evlist__can_select_event(struct evlist *evlist, const char *str)
 
 	evsel = evlist__last(temp_evlist);
 
-	if (!evlist || perf_cpu_map__is_any_cpu_or_is_empty(evlist->core.user_requested_cpus)) {
+	if (!evlist ||
+	    perf_cpu_map__is_any_cpu_or_is_empty(evlist__core(evlist)->user_requested_cpus)) {
 		struct perf_cpu_map *cpus = perf_cpu_map__new_online_cpus();
 
 		if (cpus)
@@ -245,7 +246,7 @@ bool evlist__can_select_event(struct evlist *evlist, const char *str)
 
 		perf_cpu_map__put(cpus);
 	} else {
-		cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, 0);
+		cpu = perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, 0);
 	}
 
 	while (1) {
@@ -264,7 +265,7 @@ bool evlist__can_select_event(struct evlist *evlist, const char *str)
 	ret = true;
 
 out_delete:
-	evlist__delete(temp_evlist);
+	evlist__put(temp_evlist);
 	return ret;
 }
 
diff --git a/tools/perf/util/s390-sample-raw.c b/tools/perf/util/s390-sample-raw.c
index 52bbca5c56c8e..aaf8838b2cb23 100644
--- a/tools/perf/util/s390-sample-raw.c
+++ b/tools/perf/util/s390-sample-raw.c
@@ -12,25 +12,26 @@
  * sample was taken from.
  */
 
-#include <unistd.h>
+#include <inttypes.h>
 #include <stdio.h>
 #include <string.h>
-#include <inttypes.h>
 
-#include <sys/stat.h>
+#include <asm/byteorder.h>
 #include <linux/compiler.h>
 #include <linux/err.h>
-#include <asm/byteorder.h>
+#include <sys/stat.h>
+#include <unistd.h>
 
+#include "color.h"
 #include "debug.h"
-#include "session.h"
 #include "evlist.h"
-#include "color.h"
 #include "hashmap.h"
-#include "sample-raw.h"
+#include "pmu.h"
+#include "pmus.h"
 #include "s390-cpumcf-kernel.h"
-#include "util/pmu.h"
-#include "util/sample.h"
+#include "sample-raw.h"
+#include "sample.h"
+#include "session.h"
 
 static size_t ctrset_size(struct cf_ctrset_entry *set)
 {
@@ -342,6 +343,7 @@ void evlist__s390_sample_raw(struct evlist *evlist, union perf_event *event,
 		sample->evsel = evlist__event2evsel(evlist, event);
 		if (!sample->evsel)
 			return;
+		evsel__get(sample->evsel);
 	}
 
 	/* Check for raw data in sample */
diff --git a/tools/perf/util/sample-raw.c b/tools/perf/util/sample-raw.c
index e20b73c0c5bd7..f5ae9f4689834 100644
--- a/tools/perf/util/sample-raw.c
+++ b/tools/perf/util/sample-raw.c
@@ -18,11 +18,11 @@ void evlist__init_trace_event_sample_raw(struct evlist *evlist, struct perf_env
 	uint16_t e_machine = perf_env__e_machine(env, /*e_flags=*/NULL);
 
 	if (e_machine == EM_S390) {
-		evlist->trace_event_sample_raw = evlist__s390_sample_raw;
+		evlist__set_trace_event_sample_raw(evlist, evlist__s390_sample_raw);
 	} else if (e_machine == EM_X86_64 || e_machine == EM_386) {
 		const char *cpuid = perf_env__cpuid(env);
 
 		if (cpuid && strstarts(cpuid, "AuthenticAMD") && evlist__has_amd_ibs(evlist))
-			evlist->trace_event_sample_raw = evlist__amd_sample_raw;
+			evlist__set_trace_event_sample_raw(evlist, evlist__amd_sample_raw);
 	}
 }
diff --git a/tools/perf/util/sample.c b/tools/perf/util/sample.c
index cf73329326d72..bccc19e2aaf25 100644
--- a/tools/perf/util/sample.c
+++ b/tools/perf/util/sample.c
@@ -1,18 +1,23 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #include "sample.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <elf.h>
+#include <linux/zalloc.h>
+
+#include "../../arch/x86/include/asm/insn.h"
 #include "debug.h"
+#include "evsel.h"
 #include "thread.h"
-#include <elf.h>
+
 #ifndef EM_CSKY
 #define EM_CSKY		252
 #endif
 #ifndef EM_LOONGARCH
 #define EM_LOONGARCH	258
 #endif
-#include <linux/zalloc.h>
-#include <stdlib.h>
-#include <string.h>
-#include "../../arch/x86/include/asm/insn.h"
 
 void perf_sample__init(struct perf_sample *sample, bool all)
 {
@@ -29,6 +34,8 @@ void perf_sample__init(struct perf_sample *sample, bool all)
 
 void perf_sample__exit(struct perf_sample *sample)
 {
+	evsel__put(sample->evsel);
+	sample->evsel = NULL;
 	zfree(&sample->user_regs);
 	zfree(&sample->intr_regs);
 	if (sample->merged_callchain) {
diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c
index f391a822480db..3237870a1a34b 100644
--- a/tools/perf/util/session.c
+++ b/tools/perf/util/session.c
@@ -205,7 +205,7 @@ struct perf_session *__perf_session__new(struct perf_data *data,
 		session->machines.host.env = host_env;
 	}
 	if (session->evlist)
-		session->evlist->session = session;
+		evlist__set_session(session->evlist, session);
 
 	session->machines.host.single_address_space =
 		perf_env__single_address_space(session->machines.host.env);
@@ -265,7 +265,7 @@ void perf_session__delete(struct perf_session *session)
 	machines__exit(&session->machines);
 	if (session->data) {
 		if (perf_data__is_read(session->data))
-			evlist__delete(session->evlist);
+			evlist__put(session->evlist);
 		perf_data__close(session->data);
 	}
 #ifdef HAVE_LIBTRACEEVENT
@@ -1549,8 +1549,8 @@ static void dump_event(struct evlist *evlist, union perf_event *event,
 	       file_offset, file_path, event->header.size, event->header.type);
 
 	trace_event(event);
-	if (event->header.type == PERF_RECORD_SAMPLE && evlist->trace_event_sample_raw)
-		evlist->trace_event_sample_raw(evlist, event, sample);
+	if (event->header.type == PERF_RECORD_SAMPLE && evlist__trace_event_sample_raw(evlist))
+		evlist__trace_event_sample_raw(evlist)(evlist, event, sample);
 
 	if (sample)
 		evlist__print_tstamp(evlist, event, sample);
@@ -1751,7 +1751,7 @@ static int deliver_sample_value(struct evlist *evlist,
 	}
 
 	if (!storage || sid->evsel == NULL) {
-		++evlist->stats.nr_unknown_id;
+		++evlist__stats(evlist)->nr_unknown_id;
 		return 0;
 	}
 
@@ -1845,13 +1845,17 @@ static int evlist__deliver_deferred_callchain(struct evlist *evlist,
 		struct evsel *saved_evsel = sample->evsel;
 
 		sample->evsel = evlist__id2evsel(evlist, sample->id);
+		if (sample->evsel)
+			sample->evsel = evsel__get(sample->evsel);
 		ret = tool->callchain_deferred(tool, event, sample, machine);
+		evsel__put(sample->evsel);
 		sample->evsel = saved_evsel;
 		return ret;
 	}
 
-	list_for_each_entry_safe(de, tmp, &evlist->deferred_samples, list) {
+	list_for_each_entry_safe(de, tmp, evlist__deferred_samples(evlist), list) {
 		struct perf_sample orig_sample;
+		struct evsel *new_evsel;
 
 		perf_sample__init(&orig_sample, /*all=*/false);
 		ret = evlist__parse_sample(evlist, de->event, &orig_sample);
@@ -1872,7 +1876,11 @@ static int evlist__deliver_deferred_callchain(struct evlist *evlist,
 		else
 			orig_sample.deferred_callchain = false;
 
-		orig_sample.evsel = evlist__id2evsel(evlist, orig_sample.id);
+		new_evsel = evlist__id2evsel(evlist, orig_sample.id);
+		if (new_evsel != orig_sample.evsel) {
+			evsel__put(orig_sample.evsel);
+			orig_sample.evsel = evsel__get(new_evsel);
+		}
 		ret = evlist__deliver_sample(evlist, tool, de->event,
 					     &orig_sample, machine);
 
@@ -1899,8 +1907,9 @@ static int session__flush_deferred_samples(struct perf_session *session,
 	struct deferred_event *de, *tmp;
 	int ret = 0;
 
-	list_for_each_entry_safe(de, tmp, &evlist->deferred_samples, list) {
+	list_for_each_entry_safe(de, tmp, evlist__deferred_samples(evlist), list) {
 		struct perf_sample sample;
+		struct evsel *new_evsel;
 
 		perf_sample__init(&sample, /*all=*/false);
 		ret = evlist__parse_sample(evlist, de->event, &sample);
@@ -1911,7 +1920,11 @@ static int session__flush_deferred_samples(struct perf_session *session,
 		}
 		sample.file_offset = de->file_offset;
 
-		sample.evsel = evlist__id2evsel(evlist, sample.id);
+		new_evsel = evlist__id2evsel(evlist, sample.id);
+		if (new_evsel != sample.evsel) {
+			evsel__put(sample.evsel);
+			sample.evsel = evsel__get(new_evsel);
+		}
 		ret = evlist__deliver_sample(evlist, tool, de->event,
 					     &sample, machine);
 
@@ -1957,21 +1970,23 @@ static int machines__deliver_event(struct machines *machines,
 
 	dump_event(evlist, event, file_offset, sample, file_path);
 
-	if (!sample->evsel)
+	if (!sample->evsel) {
 		sample->evsel = evlist__id2evsel(evlist, sample->id);
+		if (sample->evsel)
+			sample->evsel = evsel__get(sample->evsel);
+	}
 	else
 		assert(sample->evsel == evlist__id2evsel(evlist, sample->id));
-
 	machine = machines__find_for_cpumode(machines, event, sample);
 
 	switch (event->header.type) {
 	case PERF_RECORD_SAMPLE:
 		if (sample->evsel == NULL) {
-			++evlist->stats.nr_unknown_id;
+			++evlist__stats(evlist)->nr_unknown_id;
 			return 0;
 		}
 		if (machine == NULL) {
-			++evlist->stats.nr_unprocessable_samples;
+			++evlist__stats(evlist)->nr_unprocessable_samples;
 			dump_sample(machine, event, sample);
 			return 0;
 		}
@@ -1990,7 +2005,7 @@ static int machines__deliver_event(struct machines *machines,
 			}
 			memcpy(de->event, event, sz);
 			de->file_offset = sample->file_offset;
-			list_add_tail(&de->list, &evlist->deferred_samples);
+			list_add_tail(&de->list, evlist__deferred_samples(evlist));
 			return 0;
 		}
 		return evlist__deliver_sample(evlist, tool, event, sample, machine);
@@ -2002,7 +2017,7 @@ static int machines__deliver_event(struct machines *machines,
 		return tool->mmap(tool, event, sample, machine);
 	case PERF_RECORD_MMAP2:
 		if (event->header.misc & PERF_RECORD_MISC_PROC_MAP_PARSE_TIMEOUT)
-			++evlist->stats.nr_proc_map_timeout;
+			++evlist__stats(evlist)->nr_proc_map_timeout;
 		if (!perf_event__check_nul(event->mmap2.filename,
 					   (void *)event + event->header.size,
 					   "MMAP2", file_offset))
@@ -2047,13 +2062,13 @@ static int machines__deliver_event(struct machines *machines,
 		return tool->exit(tool, event, sample, machine);
 	case PERF_RECORD_LOST:
 		if (tool->lost == perf_event__process_lost)
-			evlist->stats.total_lost += event->lost.lost;
+			evlist__stats(evlist)->total_lost += event->lost.lost;
 		return tool->lost(tool, event, sample, machine);
 	case PERF_RECORD_LOST_SAMPLES:
 		if (event->header.misc & PERF_RECORD_MISC_LOST_SAMPLES_BPF)
-			evlist->stats.total_dropped_samples += event->lost_samples.lost;
+			evlist__stats(evlist)->total_dropped_samples += event->lost_samples.lost;
 		else if (tool->lost_samples == perf_event__process_lost_samples)
-			evlist->stats.total_lost_samples += event->lost_samples.lost;
+			evlist__stats(evlist)->total_lost_samples += event->lost_samples.lost;
 		return tool->lost_samples(tool, event, sample, machine);
 	case PERF_RECORD_READ:
 		dump_read(sample->evsel, event);
@@ -2065,11 +2080,11 @@ static int machines__deliver_event(struct machines *machines,
 	case PERF_RECORD_AUX:
 		if (tool->aux == perf_event__process_aux) {
 			if (event->aux.flags & PERF_AUX_FLAG_TRUNCATED)
-				evlist->stats.total_aux_lost += 1;
+				evlist__stats(evlist)->total_aux_lost += 1;
 			if (event->aux.flags & PERF_AUX_FLAG_PARTIAL)
-				evlist->stats.total_aux_partial += 1;
+				evlist__stats(evlist)->total_aux_partial += 1;
 			if (event->aux.flags & PERF_AUX_FLAG_COLLISION)
-				evlist->stats.total_aux_collision += 1;
+				evlist__stats(evlist)->total_aux_collision += 1;
 		}
 		return tool->aux(tool, event, sample, machine);
 	case PERF_RECORD_ITRACE_START:
@@ -2105,7 +2120,7 @@ static int machines__deliver_event(struct machines *machines,
 		return evlist__deliver_deferred_callchain(evlist, tool, event,
 							  sample, machine);
 	default:
-		++evlist->stats.nr_unknown_events;
+		++evlist__stats(evlist)->nr_unknown_events;
 		return -1;
 	}
 }
@@ -2517,7 +2532,7 @@ int perf_session__deliver_synth_event(struct perf_session *session,
 	struct evlist *evlist = session->evlist;
 	const struct perf_tool *tool = session->tool;
 
-	events_stats__inc(&evlist->stats, event->header.type);
+	events_stats__inc(evlist__stats(evlist), event->header.type);
 
 	if (event->header.type >= PERF_RECORD_USER_TYPE_START)
 		return perf_session__process_user_event(session, event, 0, NULL);
@@ -2658,7 +2673,7 @@ static const u32 perf_event__min_size[PERF_RECORD_HEADER_MAX] = {
  * Caller must ensure event->header.type < PERF_RECORD_HEADER_MAX.
  * If min is non-NULL, stores the required minimum on failure.
  */
-static bool perf_event__too_small(const union perf_event *event, u32 *min)
+bool perf_event__too_small(const union perf_event *event, u32 *min)
 {
 	u32 min_sz = perf_event__min_size[event->header.type];
 
@@ -2927,7 +2942,7 @@ static s64 perf_session__process_event(struct perf_session *session,
 		return 0;
 	}
 
-	events_stats__inc(&evlist->stats, event->header.type);
+	events_stats__inc(evlist__stats(evlist), event->header.type);
 
 	if (event->header.type >= PERF_RECORD_USER_TYPE_START)
 		return perf_session__process_user_event(session, event, file_offset, file_path);
@@ -2988,7 +3003,7 @@ perf_session__warn_order(const struct perf_session *session)
 
 static void perf_session__warn_about_errors(const struct perf_session *session)
 {
-	const struct events_stats *stats = &session->evlist->stats;
+	const struct events_stats *stats = evlist__stats(session->evlist);
 
 	if (session->tool->lost == perf_event__process_lost &&
 	    stats->nr_events[PERF_RECORD_LOST] != 0) {
@@ -3821,7 +3836,7 @@ size_t perf_session__fprintf_nr_events(struct perf_session *session, FILE *fp)
 
 	ret = fprintf(fp, "\nAggregated stats:%s\n", msg);
 
-	ret += events_stats__fprintf(&session->evlist->stats, fp);
+	ret += events_stats__fprintf(evlist__stats(session->evlist), fp);
 	return ret;
 }
 
diff --git a/tools/perf/util/session.h b/tools/perf/util/session.h
index d554e2a1a50ed..ac5803d5fb4e1 100644
--- a/tools/perf/util/session.h
+++ b/tools/perf/util/session.h
@@ -122,6 +122,8 @@ void perf_session__delete(struct perf_session *session);
 
 void perf_event_header__bswap(struct perf_event_header *hdr);
 
+bool perf_event__too_small(const union perf_event *event, u32 *min);
+
 int perf_session__peek_event(struct perf_session *session, off_t file_offset,
 			     void *buf, size_t buf_sz,
 			     union perf_event **event_ptr,
diff --git a/tools/perf/util/setup.py b/tools/perf/util/setup.py
index a0ce76624a230..cb14554454a21 100644
--- a/tools/perf/util/setup.py
+++ b/tools/perf/util/setup.py
@@ -2,6 +2,7 @@ from os import getenv, path
 from subprocess import Popen, PIPE
 from re import sub
 import shlex
+import shutil
 
 cc = getenv("CC")
 assert cc, "Environment variable CC not set"
@@ -73,6 +74,10 @@ class install_lib(_install_lib):
         _install_lib.finalize_options(self)
         self.build_dir = build_lib
 
+    def run(self):
+        _install_lib.run(self)
+        shutil.copy2(f'{src_perf}/python/perf.pyi', self.install_dir)
+
 
 # switch off several checks (need to be at the end of cflags list)
 extra_cflags = ['-fno-strict-aliasing', '-Wno-write-strings', '-Wno-unused-parameter', '-Wno-redundant-decls' ]
diff --git a/tools/perf/util/sideband_evlist.c b/tools/perf/util/sideband_evlist.c
index 388846f17bc13..ba043db6cedcb 100644
--- a/tools/perf/util/sideband_evlist.c
+++ b/tools/perf/util/sideband_evlist.c
@@ -8,6 +8,7 @@
 #include <perf/mmap.h>
 #include <linux/perf_event.h>
 #include <limits.h>
+#include <poll.h>
 #include <pthread.h>
 #include <sched.h>
 #include <stdbool.h>
@@ -22,7 +23,7 @@ int evlist__add_sb_event(struct evlist *evlist, struct perf_event_attr *attr,
 		attr->sample_id_all = 1;
 	}
 
-	evsel = evsel__new_idx(attr, evlist->core.nr_entries);
+	evsel = evsel__new_idx(attr, evlist__nr_entries(evlist));
 	if (!evsel)
 		return -1;
 
@@ -49,14 +50,27 @@ static void *perf_evlist__poll_thread(void *arg)
 	while (!done) {
 		bool got_data = false;
 
-		if (evlist->thread.done)
+		if (evlist__sb_thread_done(evlist))
 			draining = true;
 
 		if (!draining)
 			evlist__poll(evlist, 1000);
 
-		for (i = 0; i < evlist->core.nr_mmaps; i++) {
-			struct mmap *map = &evlist->mmap[i];
+		/*
+		 * When a thread of the monitored target exits, its per-cpu
+		 * ring-buffer fd is closed and starts returning POLLHUP. Such
+		 * dead fds are never requested for POLLIN, but poll() reports
+		 * POLLHUP/POLLERR unconditionally, so leaving them in the
+		 * pollfd array makes the following evlist__poll() return
+		 * immediately forever, spinning this thread at 100% CPU.
+		 *
+		 * Filter them out here, mirroring what the 'perf record' main
+		 * loop does after fdarray__poll().
+		 */
+		evlist__filter_pollfd(evlist, POLLERR | POLLHUP);
+
+		for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) {
+			struct mmap *map = &evlist__mmap(evlist)[i];
 			union perf_event *event;
 
 			if (perf_mmap__read_init(&map->core))
@@ -102,9 +116,9 @@ int evlist__start_sb_thread(struct evlist *evlist, struct target *target)
 		return 0;
 
 	if (evlist__create_maps(evlist, target))
-		goto out_delete_evlist;
+		goto out_put_evlist;
 
-	if (evlist->core.nr_entries > 1) {
+	if (evlist__nr_entries(evlist) > 1) {
 		bool can_sample_identifier = perf_can_sample_identifier();
 
 		evlist__for_each_entry(evlist, counter)
@@ -114,27 +128,27 @@ int evlist__start_sb_thread(struct evlist *evlist, struct target *target)
 	}
 
 	evlist__for_each_entry(evlist, counter) {
-		if (evsel__open(counter, evlist->core.user_requested_cpus,
-				evlist->core.threads) < 0)
-			goto out_delete_evlist;
+		if (evsel__open(counter, evlist__core(evlist)->user_requested_cpus,
+				evlist__core(evlist)->threads) < 0)
+			goto out_put_evlist;
 	}
 
-	if (evlist__mmap(evlist, UINT_MAX))
-		goto out_delete_evlist;
+	if (evlist__do_mmap(evlist, UINT_MAX))
+		goto out_put_evlist;
 
 	evlist__for_each_entry(evlist, counter) {
 		if (evsel__enable(counter))
-			goto out_delete_evlist;
+			goto out_put_evlist;
 	}
 
-	evlist->thread.done = 0;
-	if (pthread_create(&evlist->thread.th, NULL, perf_evlist__poll_thread, evlist))
-		goto out_delete_evlist;
+	evlist__set_sb_thread_done(evlist, 0);
+	if (pthread_create(evlist__sb_thread_th(evlist), NULL, perf_evlist__poll_thread, evlist))
+		goto out_put_evlist;
 
 	return 0;
 
-out_delete_evlist:
-	evlist__delete(evlist);
+out_put_evlist:
+	evlist__put(evlist);
 	evlist = NULL;
 	return -1;
 }
@@ -143,7 +157,7 @@ void evlist__stop_sb_thread(struct evlist *evlist)
 {
 	if (!evlist)
 		return;
-	evlist->thread.done = 1;
-	pthread_join(evlist->thread.th, NULL);
-	evlist__delete(evlist);
+	evlist__set_sb_thread_done(evlist, 1);
+	pthread_join(*evlist__sb_thread_th(evlist), NULL);
+	evlist__put(evlist);
 }
diff --git a/tools/perf/util/sort.c b/tools/perf/util/sort.c
index 005e7d85dc4a5..dcf9189786f8a 100644
--- a/tools/perf/util/sort.c
+++ b/tools/perf/util/sort.c
@@ -3487,7 +3487,7 @@ static struct evsel *find_evsel(struct evlist *evlist, char *event_name)
 	if (event_name[0] == '%') {
 		int nr = strtol(event_name+1, NULL, 0);
 
-		if (nr > evlist->core.nr_entries)
+		if (nr > evlist__nr_entries(evlist))
 			return NULL;
 
 		evsel = evlist__first(evlist);
diff --git a/tools/perf/util/stat-display.c b/tools/perf/util/stat-display.c
index 0a5750bb59fa3..f94f1324d24ac 100644
--- a/tools/perf/util/stat-display.c
+++ b/tools/perf/util/stat-display.c
@@ -667,7 +667,7 @@ static void print_metric_header(struct perf_stat_config *config,
 
 	/* In case of iostat, print metric header for first root port only */
 	if (config->iostat_run &&
-	    os->evsel->priv != os->evsel->evlist->selected->priv)
+		os->evsel->priv != evlist__selected(os->evsel->evlist)->priv)
 		return;
 
 	if (os->evsel->cgrp != os->cgrp)
@@ -1126,7 +1126,7 @@ static void print_no_aggr_metric(struct perf_stat_config *config,
 	unsigned int all_idx;
 	struct perf_cpu cpu;
 
-	perf_cpu_map__for_each_cpu(cpu, all_idx, evlist->core.user_requested_cpus) {
+	perf_cpu_map__for_each_cpu(cpu, all_idx, evlist__core(evlist)->user_requested_cpus) {
 		struct evsel *counter;
 		bool first = true;
 
@@ -1543,7 +1543,7 @@ void evlist__print_counters(struct evlist *evlist, struct perf_stat_config *conf
 	evlist__uniquify_evsel_names(evlist, config);
 
 	if (config->iostat_run)
-		evlist->selected = evlist__first(evlist);
+		evlist__set_selected(evlist, evlist__first(evlist));
 
 	if (config->interval)
 		prepare_timestamp(config, &os, ts);
diff --git a/tools/perf/util/stat-shadow.c b/tools/perf/util/stat-shadow.c
index c17373bb0e1e7..525a3fe4a46ec 100644
--- a/tools/perf/util/stat-shadow.c
+++ b/tools/perf/util/stat-shadow.c
@@ -2,20 +2,24 @@
 #include <errno.h>
 #include <math.h>
 #include <stdio.h>
-#include "evsel.h"
-#include "stat.h"
+
+#include <linux/zalloc.h>
+
+#include "cgroup.h"
 #include "color.h"
 #include "debug.h"
-#include "pmu.h"
-#include "rblist.h"
 #include "evlist.h"
+#include "evsel.h"
 #include "expr.h"
-#include "metricgroup.h"
-#include "cgroup.h"
-#include "units.h"
+#include "hashmap.h"
 #include "iostat.h"
-#include "util/hashmap.h"
+#include "metricgroup.h"
+#include "pmu.h"
+#include "pmus.h"
+#include "rblist.h"
+#include "stat.h"
 #include "tool_pmu.h"
+#include "units.h"
 
 static bool tool_pmu__is_time_event(const struct perf_stat_config *config,
 				   const struct evsel *evsel, int *tool_aggr_idx)
@@ -283,7 +287,7 @@ void *perf_stat__print_shadow_stats_metricgroup(struct perf_stat_config *config,
 	void *ctxp = out->ctx;
 	bool header_printed = false;
 	const char *name = NULL;
-	struct rblist *metric_events = &evsel->evlist->metric_events;
+	struct rblist *metric_events = evlist__metric_events(evsel->evlist);
 
 	me = metricgroup__lookup(metric_events, evsel, false);
 	if (me == NULL)
@@ -351,5 +355,5 @@ bool perf_stat__skip_metric_event(struct evsel *evsel)
 	if (!evsel->default_metricgroup)
 		return false;
 
-	return !metricgroup__lookup(&evsel->evlist->metric_events, evsel, false);
+	return !metricgroup__lookup(evlist__metric_events(evsel->evlist), evsel, false);
 }
diff --git a/tools/perf/util/stat.c b/tools/perf/util/stat.c
index 14d169e22e8f5..25f31a1743682 100644
--- a/tools/perf/util/stat.c
+++ b/tools/perf/util/stat.c
@@ -1,21 +1,25 @@
 // SPDX-License-Identifier: GPL-2.0
+#include "stat.h"
+
 #include <errno.h>
-#include <linux/err.h>
 #include <inttypes.h>
 #include <math.h>
 #include <string.h>
+
+#include <linux/err.h>
+#include <linux/zalloc.h>
+
 #include "counts.h"
 #include "cpumap.h"
 #include "debug.h"
+#include "evlist.h"
+#include "evsel.h"
+#include "hashmap.h"
 #include "header.h"
-#include "stat.h"
+#include "pmu.h"
 #include "session.h"
 #include "target.h"
-#include "evlist.h"
-#include "evsel.h"
 #include "thread_map.h"
-#include "util/hashmap.h"
-#include <linux/zalloc.h>
 
 void update_stats(struct stats *stats, u64 val)
 {
@@ -543,8 +547,8 @@ static void evsel__merge_aliases(struct evsel *evsel)
 	struct evlist *evlist = evsel->evlist;
 	struct evsel *alias;
 
-	alias = list_prepare_entry(evsel, &(evlist->core.entries), core.node);
-	list_for_each_entry_continue(alias, &evlist->core.entries, core.node) {
+	alias = list_prepare_entry(evsel, &(evlist__core(evlist)->entries), core.node);
+	list_for_each_entry_continue(alias, &evlist__core(evlist)->entries, core.node) {
 		if (alias->first_wildcard_match == evsel) {
 			/* Merge the same events on different PMUs. */
 			evsel__merge_aggr_counters(evsel, alias);
diff --git a/tools/perf/util/stream.c b/tools/perf/util/stream.c
index 3de4a61308539..7bccd23783444 100644
--- a/tools/perf/util/stream.c
+++ b/tools/perf/util/stream.c
@@ -131,7 +131,7 @@ static int evlist__init_callchain_streams(struct evlist *evlist,
 	struct evsel *pos;
 	int i = 0;
 
-	BUG_ON(els->nr_evsel < evlist->core.nr_entries);
+	BUG_ON(els->nr_evsel < evlist__nr_entries(evlist));
 
 	evlist__for_each_entry(evlist, pos) {
 		struct hists *hists = evsel__hists(pos);
@@ -148,7 +148,7 @@ static int evlist__init_callchain_streams(struct evlist *evlist,
 struct evlist_streams *evlist__create_streams(struct evlist *evlist,
 					      int nr_streams_max)
 {
-	int nr_evsel = evlist->core.nr_entries, ret = -1;
+	int nr_evsel = evlist__nr_entries(evlist), ret = -1;
 	struct evlist_streams *els = evlist_streams__new(nr_evsel,
 							 nr_streams_max);
 
diff --git a/tools/perf/util/symbol-elf.c b/tools/perf/util/symbol-elf.c
index 39562bdec8b90..f6f6b54da1311 100644
--- a/tools/perf/util/symbol-elf.c
+++ b/tools/perf/util/symbol-elf.c
@@ -2216,6 +2216,10 @@ static int kcore_copy__process_kallsyms(void *arg, const char *name, char type,
 	if (!kallsyms__is_function(type))
 		return 0;
 
+	/* Ignore livepatch symbols */
+	if (is_livepatch_symbol(name))
+		return 0;
+
 	if (strchr(name, '[')) {
 		if (!kci->first_module_symbol || start < kci->first_module_symbol)
 			kci->first_module_symbol = start;
diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c
index cd379ced19e5b..a562702b4841d 100644
--- a/tools/perf/util/symbol.c
+++ b/tools/perf/util/symbol.c
@@ -880,8 +880,8 @@ static int map__process_kallsym_symbol(void *arg, const char *name,
 	if (!symbol_type__filter(type))
 		return 0;
 
-	/* Ignore mapping symbols in kallsyms */
-	if (is_ignored_kernel_symbol(name))
+	/* Ignore mapping and livepatch symbols in kallsyms */
+	if (is_ignored_kernel_symbol(name) || is_livepatch_symbol(name))
 		return 0;
 
 	/*
diff --git a/tools/perf/util/symbol.h b/tools/perf/util/symbol.h
index a715253357037..d0bac824c79c8 100644
--- a/tools/perf/util/symbol.h
+++ b/tools/perf/util/symbol.h
@@ -8,7 +8,9 @@
 #include <stdint.h>
 #include <stdatomic.h>
 #include <linux/list.h>
+#include <linux/livepatch_external.h>
 #include <linux/rbtree.h>
+#include <linux/string.h>
 #include <stdio.h>
 #include <errno.h>
 #include "addr_location.h"
@@ -45,6 +47,16 @@ static inline bool is_ignored_kernel_symbol(const char *str)
 	return str[0] == '$';
 }
 
+/*
+ * Livepatch symbols (.klp.sym.*) are relocation placeholders whose resolved
+ * addresses alias existing kernel symbols.  They carry a [module] tag which
+ * confuses module boundary tracking and symbol table lookups.
+ */
+static inline bool is_livepatch_symbol(const char *str)
+{
+	return strstarts(str, KLP_SYM_PREFIX);
+}
+
 /*
  * libelf 0.8.x and earlier do not support ELF_C_READ_MMAP;
  * for newer versions we can use mmap to reduce memory usage:
diff --git a/tools/perf/util/synthetic-events.c b/tools/perf/util/synthetic-events.c
index 5307d707711d8..b75f9dcf4dbfe 100644
--- a/tools/perf/util/synthetic-events.c
+++ b/tools/perf/util/synthetic-events.c
@@ -2247,7 +2247,7 @@ int perf_event__synthesize_tracing_data(const struct perf_tool *tool, int fd, st
 	 * - write the tracing data from the temp file
 	 *   to the pipe
 	 */
-	tdata = tracing_data_get(&evlist->core.entries, fd, true);
+	tdata = tracing_data_get(&evlist__core(evlist)->entries, fd, true);
 	if (!tdata)
 		return -1;
 
@@ -2404,13 +2404,16 @@ int perf_event__synthesize_stat_events(struct perf_stat_config *config, const st
 	}
 
 	err = perf_event__synthesize_extra_attr(tool, evlist, process, attrs);
-	err = perf_event__synthesize_thread_map2(tool, evlist->core.threads, process, NULL);
+	err = perf_event__synthesize_thread_map2(tool, evlist__core(evlist)->threads,
+						process, /*machine=*/NULL);
 	if (err < 0) {
 		pr_err("Couldn't synthesize thread map.\n");
 		return err;
 	}
 
-	err = perf_event__synthesize_cpu_map(tool, evlist->core.user_requested_cpus, process, NULL);
+	err = perf_event__synthesize_cpu_map(tool,
+					     evlist__core(evlist)->user_requested_cpus,
+					     process, /*machine=*/NULL);
 	if (err < 0) {
 		pr_err("Couldn't synthesize thread map.\n");
 		return err;
@@ -2518,7 +2521,7 @@ int perf_event__synthesize_for_pipe(const struct perf_tool *tool,
 	ret += err;
 
 #ifdef HAVE_LIBTRACEEVENT
-	if (have_tracepoints(&evlist->core.entries)) {
+	if (have_tracepoints(&evlist__core(evlist)->entries)) {
 		int fd = perf_data__fd(data);
 
 		/*
diff --git a/tools/perf/util/time-utils.c b/tools/perf/util/time-utils.c
index d43c4577d7ebc..5558a5a0fea4a 100644
--- a/tools/perf/util/time-utils.c
+++ b/tools/perf/util/time-utils.c
@@ -473,8 +473,8 @@ int perf_time__parse_for_ranges_reltime(const char *time_str,
 		return -ENOMEM;
 
 	if (has_percent || reltime) {
-		if (session->evlist->first_sample_time == 0 &&
-		    session->evlist->last_sample_time == 0) {
+		if (evlist__first_sample_time(session->evlist) == 0 &&
+		    evlist__last_sample_time(session->evlist) == 0) {
 			pr_err("HINT: no first/last sample time found in perf data.\n"
 			       "Please use latest perf binary to execute 'perf record'\n"
 			       "(if '--buildid-all' is enabled, please set '--timestamp-boundary').\n");
@@ -486,8 +486,8 @@ int perf_time__parse_for_ranges_reltime(const char *time_str,
 		num = perf_time__percent_parse_str(
 				ptime_range, size,
 				time_str,
-				session->evlist->first_sample_time,
-				session->evlist->last_sample_time);
+				evlist__first_sample_time(session->evlist),
+				evlist__last_sample_time(session->evlist));
 	} else {
 		num = perf_time__parse_strs(ptime_range, time_str, size);
 	}
@@ -499,8 +499,8 @@ int perf_time__parse_for_ranges_reltime(const char *time_str,
 		int i;
 
 		for (i = 0; i < num; i++) {
-			ptime_range[i].start += session->evlist->first_sample_time;
-			ptime_range[i].end += session->evlist->first_sample_time;
+			ptime_range[i].start += evlist__first_sample_time(session->evlist);
+			ptime_range[i].end += evlist__first_sample_time(session->evlist);
 		}
 	}
 
diff --git a/tools/perf/util/top.c b/tools/perf/util/top.c
index b06e10a116bb3..851a26be69315 100644
--- a/tools/perf/util/top.c
+++ b/tools/perf/util/top.c
@@ -71,7 +71,7 @@ size_t perf_top__header_snprintf(struct perf_top *top, char *bf, size_t size)
 			       esamples_percent);
 	}
 
-	if (top->evlist->core.nr_entries == 1) {
+	if (evlist__nr_entries(top->evlist) == 1) {
 		struct evsel *first = evlist__first(top->evlist);
 		ret += SNPRINTF(bf + ret, size - ret, "%" PRIu64 "%s ",
 				(uint64_t)first->core.attr.sample_period,
@@ -94,7 +94,7 @@ size_t perf_top__header_snprintf(struct perf_top *top, char *bf, size_t size)
 	else
 		ret += SNPRINTF(bf + ret, size - ret, " (all");
 
-	nr_cpus = perf_cpu_map__nr(top->evlist->core.user_requested_cpus);
+	nr_cpus = perf_cpu_map__nr(evlist__core(top->evlist)->user_requested_cpus);
 	if (target->cpu_list)
 		ret += SNPRINTF(bf + ret, size - ret, ", CPU%s: %s)",
 				nr_cpus > 1 ? "s" : "",
diff --git a/tools/perf/util/zstd.c b/tools/perf/util/zstd.c
index 21a0eb58597c2..d98014902f012 100644
--- a/tools/perf/util/zstd.c
+++ b/tools/perf/util/zstd.c
@@ -31,9 +31,11 @@ int zstd_fini(struct zstd_data *data)
 
 ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_t dst_size,
 				       void *src, size_t src_size, size_t max_record_size,
-				       size_t process_header(void *record, size_t increment))
+				       ssize_t process_header(void *record, size_t dst_size,
+							      size_t data_size))
 {
-	size_t ret, size, compressed = 0;
+	size_t ret, compressed = 0;
+	ssize_t size;
 	ZSTD_inBuffer input = { src, src_size, 0 };
 	ZSTD_outBuffer output;
 	void *record;
@@ -55,12 +57,9 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_
 
 	while (input.pos < input.size) {
 		record = dst;
-		/* process_header writes the event header into record */
-		if (dst_size < sizeof(struct perf_event_header))
-			goto reset;
-		size = process_header(record, 0);
+		size = process_header(record, dst_size, 0);
 		/* Output buffer full — cannot fit even the record header */
-		if (size > dst_size)
+		if (size < 0)
 			goto reset;
 		compressed += size;
 		dst += size;
@@ -74,17 +73,21 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_
 				(long)src_size, ZSTD_getErrorName(ret));
 			goto reset;
 		}
-		size = output.pos;
+		compressed += output.pos;
+		dst += output.pos;
+		dst_size -= output.pos;
 		/*
 		 * No progress: ZSTD couldn't emit any bytes into the
 		 * remaining output buffer.  Calling process_header
-		 * with size=0 would re-trigger header initialization,
+		 * with output.pos=0 would re-trigger header initialization,
 		 * double-subtracting the header size from dst_size and
 		 * underflowing the unsigned counter.
 		 */
-		if (size == 0)
+		if (output.pos == 0)
+			goto reset;
+		size = process_header(record, dst_size, output.pos);
+		if (size < 0)
 			goto reset;
-		size = process_header(record, size);
 		compressed += size;
 		dst += size;
 		dst_size -= size;
diff --git a/tools/platform/x86/amd/Makefile b/tools/platform/x86/amd/Makefile
new file mode 100644
index 0000000000000..5d820df5871f3
--- /dev/null
+++ b/tools/platform/x86/amd/Makefile
@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: GPL-2.0
+ifeq ($(srctree),)
+srctree := $(patsubst %/,%,$(dir $(CURDIR)))
+srctree := $(patsubst %/,%,$(dir $(srctree)))
+srctree := $(patsubst %/,%,$(dir $(srctree)))
+srctree := $(patsubst %/,%,$(dir $(srctree)))
+endif
+
+# Include common tools build infrastructure
+include $(srctree)/tools/scripts/Makefile.include
+
+CC		= $(CROSS_COMPILE)gcc
+BUILD_OUTPUT	:= $(CURDIR)
+PREFIX		?= /usr
+DESTDIR		?=
+
+ifeq ("$(origin O)", "command line")
+	BUILD_OUTPUT := $(O)
+endif
+
+# Include paths: tools/include has linux/kernel.h with ARRAY_SIZE
+INCLUDES	= -I$(srctree)/tools/include
+INCLUDES	+= -I$(srctree)/tools/include/uapi
+INCLUDES	+= -I$(srctree)/include/uapi
+INCLUDES	+= -I$(srctree)/include
+
+override CFLAGS += -O2 -Wall -Wextra -D_GNU_SOURCE $(INCLUDES)
+override CFLAGS += -D_FILE_OFFSET_BITS=64
+override CFLAGS += -D__EXPORTED_HEADERS__
+
+TARGETS		= test_amd_pmf
+
+all: $(TARGETS)
+
+test_amd_pmf: test-pmf.c
+	$(QUIET_CC)$(CC) $(CFLAGS) $< -o $(BUILD_OUTPUT)/$@ $(LDFLAGS)
+
+.PHONY: clean
+clean:
+	$(call QUIET_CLEAN, test_amd_pmf)$(RM) $(BUILD_OUTPUT)/test_amd_pmf
+
+.PHONY: install
+install: $(TARGETS)
+	$(INSTALL) -d $(DESTDIR)$(PREFIX)/bin
+	$(INSTALL) $(BUILD_OUTPUT)/test_amd_pmf $(DESTDIR)$(PREFIX)/bin/test_amd_pmf
+
+.PHONY: help
+help:
+	@echo "AMD Platform Tools Makefile"
+	@echo ""
+	@echo "Targets:"
+	@echo "  all           - Build all tools (default)"
+	@echo "  test_amd_pmf  - Build the PMF test tool"
+	@echo "  clean         - Remove built files"
+	@echo "  install       - Install tools to $(PREFIX)/bin"
+	@echo ""
+	@echo "Variables:"
+	@echo "  O=<dir>        - Build output directory"
+	@echo "  PREFIX         - Installation prefix (default: /usr)"
+	@echo "  DESTDIR        - Destination directory for install"
diff --git a/tools/platform/x86/amd/test-pmf.c b/tools/platform/x86/amd/test-pmf.c
new file mode 100644
index 0000000000000..82978afa6e1b7
--- /dev/null
+++ b/tools/platform/x86/amd/test-pmf.c
@@ -0,0 +1,142 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * AMD Platform Management Framework Test Tool
+ *
+ * Copyright (c) 2026, Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Authors: Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
+ *          Sanket Goswami <Sanket.Goswami@amd.com>
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <inttypes.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <unistd.h>
+
+#include <linux/amd-pmf.h>
+#include <linux/kernel.h>
+
+#define DEVICE_NODE	"/dev/amdpmf_interface"
+
+/* Feature flag names */
+static const char * const feature_names[] = {
+	"Auto Mode",
+	"Static Power Slider",
+	"Policy Builder (Smart PC)",
+	"Dynamic Power Slider AC",
+	"Dynamic Power Slider DC",
+};
+
+static const char *banner =
+	"====================================================\n"
+	"      AMD PMF Metrics info and Feature Status\n"
+	"====================================================\n\n";
+
+/* Print feature flags */
+static void pmf_print_features(uint32_t flags)
+{
+	size_t i;
+
+	for (i = 0; i < ARRAY_SIZE(feature_names); i++)
+		printf("  [%c] %s\n", (flags & (1U << i)) ? 'x' : ' ', feature_names[i]);
+}
+
+/* Print BIOS parameters */
+static void pmf_print_bios_params(const char *type, const __u32 *params)
+{
+	int i;
+
+	for (i = 0; i < AMD_PMF_BIOS_PARAMS_MAX; i++)
+		printf("  Custom BIOS %s%d: %u\n", type, i + 1, params[i]);
+}
+
+/* Open the PMF device */
+static int pmf_open_device(void)
+{
+	int fd;
+
+	fd = open(DEVICE_NODE, O_RDONLY);
+	if (fd < 0)
+		fprintf(stderr, "Error: Cannot open %s: %s\n", DEVICE_NODE, strerror(errno));
+
+	return fd;
+}
+
+/* Query PMF info using the single IOCTL */
+static int pmf_get_info(int fd, struct amd_pmf_info *info)
+{
+	int ret;
+
+	/* Zero-initialize and set size for versioning */
+	memset(info, 0, sizeof(*info));
+	info->size = sizeof(*info);
+
+	ret = ioctl(fd, IOCTL_AMD_PMF_POPULATE_DATA, info);
+	if (ret < 0) {
+		fprintf(stderr, "Error: IOCTL_AMD_PMF_POPULATE_DATA failed: %s\n", strerror(errno));
+		return ret;
+	}
+
+	return 0;
+}
+
+static void pmf_print_info(const struct amd_pmf_info *info)
+{
+	printf("%s", banner);
+
+	/* Feature status */
+	printf("Feature Status:\n");
+	pmf_print_features(info->features_supported);
+
+	/* Device states */
+	printf("\nDevice States:\n");
+	printf("  Platform Type:     %s\n", amd_pmf_get_platform_type(info->platform_type));
+	printf("  Laptop Placement:  %s\n", amd_pmf_get_laptop_placement(info->laptop_placement));
+	printf("  Lid State:         %s\n", info->lid_state ? "Closed" : "Open");
+	printf("  User Presence:     %s\n", info->user_presence ? "Present" : "Away");
+	printf("  Slider Position:   %s\n", amd_pmf_get_slider_position(info->slider_position));
+
+	/* Thermal and power metrics */
+	printf("\nThermal/Power Metrics:\n");
+	printf("  Skin Temperature:  %d\n", info->skin_temp / 100);
+	printf("  GFX Busy:          %u\n", info->gfx_busy);
+	printf("  Ambient Light:     %d\n", info->ambient_light);
+	printf("  Avg C0 Residency:  %u\n", info->avg_c0_residency);
+	printf("  Max C0 Residency:  %u\n", info->max_c0_residency);
+	printf("  Socket Power:      %u\n", info->socket_power);
+
+	/* BIOS parameters */
+	printf("\nCustom BIOS Input Parameters:\n");
+	pmf_print_bios_params("Input", info->bios_input);
+	printf("\nCustom BIOS Output Parameters:\n");
+	pmf_print_bios_params("Output", info->bios_output);
+
+	printf("\n=================================================\n");
+}
+
+int main(void)
+{
+	struct amd_pmf_info info;
+	int fd, ret;
+
+	fd = pmf_open_device();
+	if (fd < 0)
+		return -1;
+
+	/* Query all info with single IOCTL */
+	ret = pmf_get_info(fd, &info);
+	close(fd);
+
+	if (ret < 0)
+		return -1;
+
+	pmf_print_info(&info);
+
+	return 0;
+}
diff --git a/tools/power/cpupower/Makefile b/tools/power/cpupower/Makefile
index 969716dfe8def..cd8b7315fe74b 100644
--- a/tools/power/cpupower/Makefile
+++ b/tools/power/cpupower/Makefile
@@ -131,7 +131,7 @@ override CFLAGS += -DVERSION=\"$(VERSION)\" -DPACKAGE=\"$(PACKAGE)\" \
 
 UTIL_OBJS =  utils/helpers/amd.o utils/helpers/msr.o \
 	utils/helpers/sysfs.o utils/helpers/misc.o utils/helpers/cpuid.o \
-	utils/helpers/pci.o utils/helpers/bitmask.o \
+	utils/helpers/pci.o utils/helpers/bitmask.o utils/helpers/cppc.o \
 	utils/idle_monitor/nhm_idle.o utils/idle_monitor/snb_idle.o \
 	utils/idle_monitor/hsw_ext_idle.o \
 	utils/idle_monitor/amd_fam14h_idle.o utils/idle_monitor/cpuidle_sysfs.o \
diff --git a/tools/power/cpupower/utils/cpufreq-info.c b/tools/power/cpupower/utils/cpufreq-info.c
index 5a242b491a9d0..11629ae49f984 100644
--- a/tools/power/cpupower/utils/cpufreq-info.c
+++ b/tools/power/cpupower/utils/cpufreq-info.c
@@ -270,10 +270,10 @@ static int get_freq_hardware(unsigned int cpu, unsigned int human)
 {
 	unsigned long freq;
 
-	if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_APERF))
+	freq = cpufreq_get_freq_hardware(cpu);
+	if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_APERF) && !freq)
 		return -EINVAL;
 
-	freq = cpufreq_get_freq_hardware(cpu);
 	printf(_("  current CPU frequency: "));
 	if (!freq) {
 		printf("Unable to call hardware\n");
@@ -477,12 +477,13 @@ static int get_latency(unsigned int cpu, unsigned int human)
 }
 
 /* --performance / -c */
-
 static int get_perf_cap(unsigned int cpu)
 {
 	if (cpupower_cpu_info.vendor == X86_VENDOR_AMD &&
 	    cpupower_cpu_info.caps & CPUPOWER_CAP_AMD_PSTATE)
 		amd_pstate_show_perf_and_freq(cpu, no_rounding);
+	else
+		cppc_show_perf_and_freq(cpu, no_rounding);
 
 	return 0;
 }
@@ -513,8 +514,8 @@ static void debug_output_one(unsigned int cpu)
 
 	get_available_governors(cpu);
 	get_policy(cpu);
-	if (get_freq_hardware(cpu, 1) < 0)
-		get_freq_kernel(cpu, 1);
+	get_freq_hardware(cpu, 1);
+	get_freq_kernel(cpu, 1);
 	get_boost_mode(cpu);
 	get_perf_cap(cpu);
 }
diff --git a/tools/power/cpupower/utils/helpers/cppc.c b/tools/power/cpupower/utils/helpers/cppc.c
new file mode 100644
index 0000000000000..3493ce8551ea7
--- /dev/null
+++ b/tools/power/cpupower/utils/helpers/cppc.c
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <stdio.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+
+#include "helpers/helpers.h"
+#include "cpufreq.h"
+#include "acpi_cppc.h"
+
+#define cppc_to_frequency(perf) (roundf(slope * (perf) + intercept))
+
+void cppc_show_perf_and_freq(unsigned int cpu, int no_rounding)
+{
+	int64_t nominal = acpi_cppc_get_data(cpu, NOMINAL_PERF);
+	int64_t nominal_freq = acpi_cppc_get_data(cpu, NOMINAL_FREQ) * 1000;
+	int64_t lowest = acpi_cppc_get_data(cpu, LOWEST_PERF);
+	int64_t lowest_freq = acpi_cppc_get_data(cpu, LOWEST_FREQ) * 1000;
+	unsigned long non_linear = acpi_cppc_get_data(cpu, LOWEST_NONLINEAR_PERF);
+	unsigned long highest = acpi_cppc_get_data(cpu, HIGHEST_PERF);
+	float slope, intercept;
+
+	/* do the optional freq fields look invalid? */
+	if (!nominal_freq || !lowest_freq || nominal == lowest)
+		return;
+
+	slope = (float)(nominal_freq - lowest_freq) / (nominal - lowest);
+	intercept = lowest_freq - slope * lowest;
+
+	printf(_("  CPPC limits:\n"));
+	printf(_("    Highest Performance: %lu. Maximum Frequency: "),
+	       highest);
+	/*
+	 * If boost isn't active, the cpuinfo_max doesn't indicate real max
+	 * frequency.
+	 */
+	print_speed(cppc_to_frequency(highest), no_rounding);
+	printf(".\n");
+
+	printf(_("    Nominal Performance: %lu. Nominal Frequency: "),
+	       acpi_cppc_get_data(cpu, NOMINAL_PERF));
+	print_speed(nominal_freq,  no_rounding);
+	printf(".\n");
+
+	printf(_("    Lowest Non-linear Performance: %lu. Lowest Non-linear Frequency: "),
+	       non_linear);
+	print_speed(cppc_to_frequency(non_linear), no_rounding);
+	printf(".\n");
+
+	printf(_("    Lowest Performance: %lu. Lowest Frequency: "),
+	       acpi_cppc_get_data(cpu, LOWEST_PERF));
+	print_speed(lowest_freq, no_rounding);
+	printf(".\n");
+}
diff --git a/tools/power/cpupower/utils/helpers/helpers.h b/tools/power/cpupower/utils/helpers/helpers.h
index a3ad80b9c2c2b..9c5126b639663 100644
--- a/tools/power/cpupower/utils/helpers/helpers.h
+++ b/tools/power/cpupower/utils/helpers/helpers.h
@@ -221,4 +221,6 @@ void print_online_cpus(void);
 void print_offline_cpus(void);
 void print_speed(unsigned long speed, int no_rounding);
 
+void cppc_show_perf_and_freq(unsigned int cpu, int no_rounding);
+
 #endif /* __CPUPOWERUTILS_HELPERS__ */
diff --git a/tools/power/x86/turbostat/turbostat.8 b/tools/power/x86/turbostat/turbostat.8
index 344ede2f8546c..c905198d05017 100644
--- a/tools/power/x86/turbostat/turbostat.8
+++ b/tools/power/x86/turbostat/turbostat.8
@@ -92,7 +92,7 @@ name as necessary to disambiguate it from others is necessary.  Note that option
 		'%x' hex identifier of the PMT MMIO region.
 .fi
 .PP
-\fB--cpu cpu-set\fP limit output to system summary plus the specified cpu-set.  If cpu-set is the string "core", then the system summary plus the first CPU in each core are printed -- eg. subsequent HT siblings are not printed.  Or if cpu-set is the string "package", then the system summary plus the first CPU in each package is printed.  Otherwise, the system summary plus the specified set of CPUs are printed.  The cpu-set is ordered from low to high, comma delimited with ".." and "-" permitted to denote a range. eg. 1,2,8,14..17,21-44
+\fB--cpu cpu-set\fP limit output to system summary plus the specified cpu-set.  If cpu-set is the string "core", then the system summary plus the first CPU in each core are printed -- eg. subsequent HT siblings are not printed.  Or if cpu-set is the string "package", then the system summary plus the first CPU in each package is printed.  Otherwise, the system summary plus the specified set of CPUs are printed.  The cpu-set is ordered from low to high, comma delimited with ".." and "-" permitted to denote a range. eg. 1,2,8,14..17,21-44  Multiple invocations of --cpu are OR'd together.
 .PP
 \fB--hide column\fP do not show the specified built-in columns.  May be invoked multiple times, or with a comma-separated list of column names.
 .PP
@@ -269,7 +269,7 @@ Periodic output goes to stdout, by default, unless --out is used to specify an o
 The 5-second interval can be changed with the "-i sec" option.
 .nf
 sudo turbostat --quiet --show CPU,frequency
-	Core	CPU	Avg_MHz	Busy%	Bzy_MHz	TSC_MHz	CPU%c7	UncMhz
+	Core	CPU	Avg_MHz	Busy%	Bzy_MHz	TSC_MHz	CPU%c7	UncMHz
 	-	-	524	12.48	4198	3096	74.53	3800
 	0	0	4	0.09	4081	3096	98.88	3800
 	0	4	1	0.02	4063	3096
@@ -279,7 +279,7 @@ sudo turbostat --quiet --show CPU,frequency
 	2	6	3	0.08	4159	3096
 	3	3	1	0.04	4046	3096	99.66
 	3	7	0	0.01	3989	3096
-	Core	CPU	Avg_MHz	Busy%	Bzy_MHz	TSC_MHz	CPU%c7	UncMhz
+	Core	CPU	Avg_MHz	Busy%	Bzy_MHz	TSC_MHz	CPU%c7	UncMHz
 	-	-	525	12.52	4198	3096	74.54	3800
 	0	0	4	0.10	4051	3096	99.49	3800
 	0	4	2	0.04	3993	3096
diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c
index 920694c3c1ec9..4ad7cb1df5c5b 100644
--- a/tools/power/x86/turbostat/turbostat.c
+++ b/tools/power/x86/turbostat/turbostat.c
@@ -98,6 +98,8 @@
 
 #define ROUND_UP_TO_PAGE_SIZE(n) (((n) + 0x1000UL-1UL) & ~(0x1000UL-1UL))
 
+FILE *outf;
+
 enum counter_scope { SCOPE_CPU, SCOPE_CORE, SCOPE_PACKAGE };
 enum counter_type { COUNTER_ITEMS, COUNTER_CYCLES, COUNTER_SECONDS, COUNTER_USEC, COUNTER_K2M };
 enum counter_format { FORMAT_RAW, FORMAT_DELTA, FORMAT_PERCENT, FORMAT_AVERAGE };
@@ -138,8 +140,6 @@ struct msr_counter {
 	enum counter_format format;
 	struct msr_counter *next;
 	unsigned int flags;
-#define	FLAGS_HIDE	(1 << 0)
-#define	FLAGS_SHOW	(1 << 1)
 #define	SYSFS_PERCPU	(1 << 1)
 };
 static int use_android_msr_path;
@@ -292,33 +292,53 @@ enum bic_names {
 	MAX_BIC
 };
 
-void print_bic_set(char *s, cpu_set_t *set)
+#define bic_set_t cpu_set_t	/* implement bic_set_t using cpu_set_t */
+
+/*
+ * debug helper
+ */
+static void print_bic_set(const char *s, const bic_set_t *set)
 {
 	int i;
 
 	assert(MAX_BIC < CPU_SETSIZE);
 
-	printf("%s:", s);
+	fprintf(outf, "%s:", s);
 
 	for (i = 0; i < MAX_BIC; ++i) {
 
 		if (CPU_ISSET(i, set))
-			printf(" %s", bic[i].name);
+			fprintf(outf, " %s", bic[i].name);
 	}
-	putchar('\n');
+	fputc('\n', outf);
 }
 
-static cpu_set_t bic_group_topology;
-static cpu_set_t bic_group_thermal_pwr;
-static cpu_set_t bic_group_frequency;
-static cpu_set_t bic_group_hw_idle;
-static cpu_set_t bic_group_sw_idle;
-static cpu_set_t bic_group_idle;
-static cpu_set_t bic_group_cache;
-static cpu_set_t bic_group_other;
-static cpu_set_t bic_group_disabled_by_default;
-static cpu_set_t bic_enabled;
-static cpu_set_t bic_present;
+static bic_set_t bic_group_topology;
+static bic_set_t bic_group_thermal_pwr;
+static bic_set_t bic_group_frequency;
+static bic_set_t bic_group_hw_idle;
+static bic_set_t bic_group_sw_idle;
+static bic_set_t bic_group_idle;
+static bic_set_t bic_group_cache;
+static bic_set_t bic_group_other;
+static bic_set_t bic_group_disabled_by_default;
+static bic_set_t bic_enabled;
+static bic_set_t bic_present;
+
+static void dump_bic_sets(void)
+{
+	print_bic_set("bic_enabled", &bic_enabled);
+	print_bic_set("bic_present", &bic_present);
+	print_bic_set("bic_group_topology", &bic_group_topology);
+	print_bic_set("bic_group_thermal_pwr", &bic_group_thermal_pwr);
+	print_bic_set("bic_group_frequency", &bic_group_frequency);
+	print_bic_set("bic_group_hw_idle", &bic_group_hw_idle);
+	print_bic_set("bic_group_sw_idle", &bic_group_sw_idle);
+	print_bic_set("bic_group_idle", &bic_group_idle);
+	print_bic_set("bic_group_cache", &bic_group_cache);
+	print_bic_set("bic_group_other", &bic_group_other);
+	print_bic_set("bic_group_disabled_by_default", &bic_group_disabled_by_default);
+}
 
 /* modify */
 #define BIC_INIT(set) CPU_ZERO(set)
@@ -334,7 +354,7 @@ static cpu_set_t bic_present;
 #define DO_BIC_READ(COUNTER_NUMBER) CPU_ISSET(COUNTER_NUMBER, &bic_present)
 #define DO_BIC(COUNTER_NUMBER) (CPU_ISSET(COUNTER_NUMBER, &bic_enabled) && CPU_ISSET(COUNTER_NUMBER, &bic_present))
 
-static void bic_set_all(cpu_set_t *set)
+static void bic_set_all(bic_set_t *set)
 {
 	int i;
 
@@ -348,7 +368,7 @@ static void bic_set_all(cpu_set_t *set)
  * bic_clear_bits()
  * clear all the bits from "clr" in "dst"
  */
-static void bic_clear_bits(cpu_set_t *dst, cpu_set_t *clr)
+static void bic_clear_bits(bic_set_t *dst, bic_set_t *clr)
 {
 	int i;
 
@@ -485,7 +505,6 @@ static void bic_groups_init(void)
 #define PCLUNL 15		/* Unlimited */
 
 char *proc_stat = "/proc/stat";
-FILE *outf;
 int *fd_percpu;
 int *fd_instr_count_percpu;
 int *fd_llc_percpu;
@@ -504,7 +523,7 @@ unsigned int valid_rapl_msrs;
 unsigned int summary_only;
 unsigned int list_header_only;
 unsigned int dump_only;
-unsigned int force_load;
+bool force_load;
 unsigned int cpuid_has_aperf_mperf;
 unsigned int cpuid_has_hv;
 unsigned int has_aperf_access;
@@ -570,9 +589,8 @@ struct gfx_sysfs_info {
 
 static struct gfx_sysfs_info gfx_info[GFX_MAX];
 
-int get_msr(int cpu, off_t offset, unsigned long long *msr);
-int add_counter(unsigned int msr_num, char *path, char *name,
-		unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int package_num);
+void get_msr(int cpu, off_t offset, unsigned long long *msr);
+int add_counter(unsigned int, const char *, const char *, unsigned int, enum counter_scope, enum counter_type, enum counter_format, int, int);
 
 /* Model specific support Start */
 
@@ -629,8 +647,8 @@ double slm_bclk(void)
 	unsigned int i;
 	double freq;
 
-	if (get_msr(master_cpu, MSR_FSB_FREQ, &msr))
-		fprintf(outf, "SLM BCLK: unknown\n");
+	get_msr(master_cpu, MSR_FSB_FREQ, &msr);
+	fprintf(outf, "SLM BCLK: unknown\n");
 
 	i = msr & 0xf;
 	if (i >= SLM_BCLK_FREQS) {
@@ -640,7 +658,7 @@ double slm_bclk(void)
 	freq = slm_freq_table[i];
 
 	if (!quiet)
-		fprintf(outf, "SLM BCLK: %.1f Mhz\n", freq);
+		fprintf(outf, "SLM BCLK: %.1f MHz\n", freq);
 
 	return freq;
 }
@@ -1312,7 +1330,6 @@ static struct perf_model_support turbostat_perf_model_support[] = {
 	{ INTEL_ATOM_CRESTMONT, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {}, {} },	/* GRR */
 	{ INTEL_ATOM_DARKMONT_X, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {}, {} },	/* CWF */
 
-	{ INTEL_ALDERLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
 	{ INTEL_ALDERLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
 	{ INTEL_ALDERLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
 	{ INTEL_RAPTORLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
@@ -1343,7 +1360,7 @@ void probe_platform_features(unsigned int family, unsigned int model)
 
 	if (authentic_amd || hygon_genuine) {
 		/* fallback to default features on unsupported models */
-		force_load++;
+		force_load = true;
 		if (max_extended_level >= 0x80000007) {
 			unsigned int eax, ebx, ecx, edx;
 
@@ -1366,7 +1383,7 @@ void probe_platform_features(unsigned int family, unsigned int model)
 	}
 
 end:
-	if (force_load && !platform) {
+	if ((force_load == true) && !platform) {
 		fprintf(outf, "Forced to run on unsupported platform!\n");
 		platform = &default_features;
 	}
@@ -1407,10 +1424,9 @@ void init_perf_model_support(unsigned int family, unsigned int model)
 int backwards_count;
 char *progname;
 
-#define CPU_SUBSET_MAXCPUS	8192	/* need to use before probe... */
-cpu_set_t *cpu_present_set, *cpu_possible_set, *cpu_effective_set, *cpu_allowed_set, *cpu_affinity_set, *cpu_subset;
+cpu_set_t *cpu_present_set, *cpu_possible_set, *cpu_effective_set, *cpu_allowed_set, *cpu_affinity_set, *cpuset_cmdline;
 cpu_set_t *perf_pcore_set, *perf_ecore_set, *perf_lcore_set;
-size_t cpu_present_setsize, cpu_possible_setsize, cpu_effective_setsize, cpu_allowed_setsize, cpu_affinity_setsize, cpu_subset_size;
+size_t cpu_setsize;
 #define MAX_ADDED_THREAD_COUNTERS 24
 #define MAX_ADDED_CORE_COUNTERS 8
 #define MAX_ADDED_PACKAGE_COUNTERS 16
@@ -1995,15 +2011,17 @@ int pmt_telemdir_filter(const struct dirent *e)
 {
 	unsigned int dummy;
 
-	return sscanf(e->d_name, "telem%u", &dummy);
+	return (sscanf(e->d_name, "telem%u", &dummy) == 1);
 }
 
 int pmt_telemdir_sort(const struct dirent **a, const struct dirent **b)
 {
 	unsigned int aidx = 0, bidx = 0;
 
-	sscanf((*a)->d_name, "telem%u", &aidx);
-	sscanf((*b)->d_name, "telem%u", &bidx);
+	if (sscanf((*a)->d_name, "telem%u", &aidx) != 1)
+		aidx = 0;
+	if (sscanf((*b)->d_name, "telem%u", &bidx) != 1)
+		bidx = 0;
 
 	return (aidx > bidx) ? 1 : (aidx < bidx) ? -1 : 0;
 }
@@ -2034,8 +2052,11 @@ const struct dirent *pmt_diriter_begin(struct pmt_diriter_t *iter, const char *p
 			return NULL;
 
 		num_names = scandir(pmt_root_path, &iter->namelist, pmt_telemdir_filter, pmt_telemdir_sort);
-		if (num_names == -1)
+		if (num_names == -1) {
+			closedir(iter->dir);
+			iter->dir = NULL;
 			return NULL;
+		}
 	}
 
 	iter->current_name_idx = 0;
@@ -2063,8 +2084,10 @@ void pmt_diriter_remove(struct pmt_diriter_t *iter)
 	iter->num_names = 0;
 	iter->current_name_idx = 0;
 
-	closedir(iter->dir);
-	iter->dir = NULL;
+	if (iter->dir) {
+		closedir(iter->dir);
+		iter->dir = NULL;
+	}
 }
 
 unsigned int pmt_counter_get_width(const struct pmt_counter *p)
@@ -2436,7 +2459,7 @@ int cpu_is_not_present(int cpu)
 	if (cpu < 0)
 		return 1;
 
-	return !CPU_ISSET_S(cpu, cpu_present_setsize, cpu_present_set);
+	return !CPU_ISSET_S(cpu, cpu_setsize, cpu_present_set);
 }
 
 int cpu_is_not_allowed(int cpu)
@@ -2444,7 +2467,7 @@ int cpu_is_not_allowed(int cpu)
 	if (cpu < 0)
 		return 1;
 
-	return !CPU_ISSET_S(cpu, cpu_allowed_setsize, cpu_allowed_set);
+	return !CPU_ISSET_S(cpu, cpu_setsize, cpu_allowed_set);
 }
 
 #define GLOBAL_CORE_ID(core_id, pkg_id)	(core_id + pkg_id * (topo.max_core_id + 1))
@@ -2537,9 +2560,9 @@ int is_cpu_first_thread_in_package(struct thread_data *t, struct core_data *c, s
 
 int cpu_migrate(int cpu)
 {
-	CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set);
-	CPU_SET_S(cpu, cpu_affinity_setsize, cpu_affinity_set);
-	if (sched_setaffinity(0, cpu_affinity_setsize, cpu_affinity_set) == -1)
+	CPU_ZERO_S(cpu_setsize, cpu_affinity_set);
+	CPU_SET_S(cpu, cpu_setsize, cpu_affinity_set);
+	if (sched_setaffinity(0, cpu_setsize, cpu_affinity_set) == -1)
 		return -1;
 	else
 		return 0;
@@ -2625,18 +2648,12 @@ int get_instr_count_fd(int cpu)
 	return fd_instr_count_percpu[cpu];
 }
 
-int get_msr(int cpu, off_t offset, unsigned long long *msr)
+void get_msr(int cpu, off_t offset, unsigned long long *msr)
 {
-	ssize_t retval;
-
 	assert(!no_msr);
 
-	retval = pread(get_msr_fd(cpu), msr, sizeof(*msr), offset);
-
-	if (retval != sizeof *msr)
+	if (pread(get_msr_fd(cpu), msr, sizeof(*msr), offset) != sizeof *msr)
 		err(-1, "cpu%d: msr offset 0x%llx read failed", cpu, (unsigned long long)offset);
-
-	return 0;
 }
 
 int add_msr_counter(int cpu, off_t offset)
@@ -2780,7 +2797,7 @@ void help(void)
  * for all the strings in comma separate name_list,
  * set the approprate bit in return value.
  */
-void bic_lookup(cpu_set_t *ret_set, char *name_list, enum show_hide_mode mode)
+void bic_lookup(bic_set_t *ret_set, char *name_list, enum show_hide_mode mode)
 {
 	unsigned int i;
 
@@ -3280,8 +3297,6 @@ int dump_counters(PER_THREAD_PARAMS)
 
 double rapl_counter_get_value(const struct rapl_counter *c, enum rapl_unit desired_unit, double interval)
 {
-	assert(desired_unit != RAPL_UNIT_INVALID);
-
 	/*
 	 * For now we don't expect anything other than joules,
 	 * so just simplify the logic.
@@ -3367,7 +3382,7 @@ int format_counters(PER_THREAD_PARAMS)
 		return 0;
 
 	/*if not summary line and --cpu is used */
-	if ((t != average.threads) && (cpu_subset && !CPU_ISSET_S(t->cpu_id, cpu_subset_size, cpu_subset)))
+	if ((t != average.threads) && (cpuset_cmdline && !CPU_ISSET_S(t->cpu_id, cpu_setsize, cpuset_cmdline)))
 		return 0;
 
 	if (DO_BIC(BIC_USEC)) {
@@ -4532,8 +4547,7 @@ int get_mp(int cpu, struct msr_counter *mp, unsigned long long *counterp, char *
 {
 	if (mp->msr_num != 0) {
 		assert(!no_msr);
-		if (get_msr(cpu, mp->msr_num, counterp))
-			return -1;
+		get_msr(cpu, mp->msr_num, counterp);
 	} else {
 		char path[128 + PATH_BYTES];
 
@@ -4897,13 +4911,10 @@ int get_rapl_counters(int cpu, unsigned int domain, struct core_data *c, struct
 				fprintf(stderr, "Reading rapl counter via msr at %u\n", i);
 
 			assert(!no_msr);
-			if (rci->flags[i] & RAPL_COUNTER_FLAG_USE_MSR_SUM) {
-				if (get_msr_sum(cpu, rci->msr[i], &rci->data[i]))
-					return -13 - i;
-			} else {
-				if (get_msr(cpu, rci->msr[i], &rci->data[i]))
-					return -13 - i;
-			}
+			if (rci->flags[i] & RAPL_COUNTER_FLAG_USE_MSR_SUM)
+				get_msr_sum(cpu, rci->msr[i], &rci->data[i]);
+			else
+				get_msr(cpu, rci->msr[i], &rci->data[i]);
 
 			rci->data[i] &= rci->msr_mask[i];
 			if (rci->msr_shift[i] >= 0)
@@ -5032,8 +5043,7 @@ int get_cstate_counters(unsigned int cpu, PER_THREAD_PARAMS)
 
 		case COUNTER_SOURCE_MSR:
 			assert(!no_msr);
-			if (get_msr(cpu, cci->msr[i], &cci->data[i]))
-				return -13 - i;
+			get_msr(cpu, cci->msr[i], &cci->data[i]);
 
 			if (debug >= 2)
 				fprintf(stderr, "cstate via %s0x%llx %u: %llu\n", "msr", cci->msr[i], i, cci->data[i]);
@@ -5131,8 +5141,7 @@ int get_smi_aperf_mperf(unsigned int cpu, struct thread_data *t)
 		case COUNTER_SOURCE_MSR:
 			assert(!no_msr);
 
-			if (get_msr(cpu, mci->msr[i], &mci->data[i]))
-				return -2 - i;
+			get_msr(cpu, mci->msr[i], &mci->data[i]);
 
 			mci->data[i] &= mci->msr_mask[i];
 
@@ -5297,12 +5306,10 @@ int get_counters(PER_THREAD_PARAMS)
 	}
 
 	if (DO_BIC(BIC_Mod_c6))
-		if (get_msr(cpu, MSR_MODULE_C6_RES_MS, &c->mc6_us))
-			return -8;
+		get_msr(cpu, MSR_MODULE_C6_RES_MS, &c->mc6_us);
 
 	if (DO_BIC(BIC_CoreTmp)) {
-		if (get_msr(cpu, MSR_IA32_THERM_STATUS, &msr))
-			return -9;
+		get_msr(cpu, MSR_IA32_THERM_STATUS, &msr);
 		c->core_temp_c = tj_max - ((msr >> 16) & 0x7F);
 	}
 
@@ -5324,22 +5331,14 @@ int get_counters(PER_THREAD_PARAMS)
 	if (!is_cpu_first_core_in_package(t, p))
 		goto done;
 
-	if (DO_BIC(BIC_Totl_c0)) {
-		if (get_msr(cpu, MSR_PKG_WEIGHTED_CORE_C0_RES, &p->pkg_wtd_core_c0))
-			return -10;
-	}
-	if (DO_BIC(BIC_Any_c0)) {
-		if (get_msr(cpu, MSR_PKG_ANY_CORE_C0_RES, &p->pkg_any_core_c0))
-			return -11;
-	}
-	if (DO_BIC(BIC_GFX_c0)) {
-		if (get_msr(cpu, MSR_PKG_ANY_GFXE_C0_RES, &p->pkg_any_gfxe_c0))
-			return -12;
-	}
-	if (DO_BIC(BIC_CPUGFX)) {
-		if (get_msr(cpu, MSR_PKG_BOTH_CORE_GFXE_C0_RES, &p->pkg_both_core_gfxe_c0))
-			return -13;
-	}
+	if (DO_BIC(BIC_Totl_c0))
+		get_msr(cpu, MSR_PKG_WEIGHTED_CORE_C0_RES, &p->pkg_wtd_core_c0);
+	if (DO_BIC(BIC_Any_c0))
+		get_msr(cpu, MSR_PKG_ANY_CORE_C0_RES, &p->pkg_any_core_c0);
+	if (DO_BIC(BIC_GFX_c0))
+		get_msr(cpu, MSR_PKG_ANY_GFXE_C0_RES, &p->pkg_any_gfxe_c0);
+	if (DO_BIC(BIC_CPUGFX))
+		get_msr(cpu, MSR_PKG_BOTH_CORE_GFXE_C0_RES, &p->pkg_both_core_gfxe_c0);
 
 	if (DO_BIC(BIC_CPU_LPI))
 		p->cpu_lpi = cpuidle_cur_cpu_lpi_us;
@@ -5353,8 +5352,7 @@ int get_counters(PER_THREAD_PARAMS)
 	}
 
 	if (DO_BIC(BIC_PkgTmp)) {
-		if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr))
-			return -17;
+		get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr);
 		p->pkg_temp_c = tj_max - ((msr >> 16) & 0x7F);
 	}
 
@@ -5833,7 +5831,7 @@ void free_fd_percpu(void)
 	if (!fd_percpu)
 		return;
 
-	for (i = 0; i < topo.max_cpu_num + 1; ++i) {
+	for (i = 0; i <= topo.max_cpu_num; ++i) {
 		if (fd_percpu[i] != 0)
 			close(fd_percpu[i]);
 	}
@@ -5847,7 +5845,7 @@ void free_fd_instr_count_percpu(void)
 	if (!fd_instr_count_percpu)
 		return;
 
-	for (int i = 0; i < topo.max_cpu_num + 1; ++i) {
+	for (int i = 0; i <= topo.max_cpu_num; ++i) {
 		if (fd_instr_count_percpu[i] != 0)
 			close(fd_instr_count_percpu[i]);
 	}
@@ -5861,7 +5859,7 @@ void free_fd_llc_percpu(void)
 	if (!fd_llc_percpu)
 		return;
 
-	for (int i = 0; i < topo.max_cpu_num + 1; ++i) {
+	for (int i = 0; i <= topo.max_cpu_num; ++i) {
 		if (fd_llc_percpu[i] != 0)
 			close(fd_llc_percpu[i]);
 	}
@@ -5878,7 +5876,7 @@ void free_fd_l2_percpu(void)
 	if (!fd_l2_percpu)
 		return;
 
-	for (int i = 0; i < topo.max_cpu_num + 1; ++i) {
+	for (int i = 0; i <= topo.max_cpu_num; ++i) {
 		if (fd_l2_percpu[i] != 0)
 			close(fd_l2_percpu[i]);
 	}
@@ -5915,7 +5913,7 @@ void free_fd_msr(void)
 	if (!msr_counter_info)
 		return;
 
-	for (int cpu = 0; cpu < topo.max_cpu_num; ++cpu) {
+	for (int cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
 		if (msr_counter_info[cpu].fd_perf != -1)
 			close(msr_counter_info[cpu].fd_perf);
 	}
@@ -5978,19 +5976,15 @@ void free_all_buffers(void)
 
 	CPU_FREE(cpu_present_set);
 	cpu_present_set = NULL;
-	cpu_present_setsize = 0;
 
 	CPU_FREE(cpu_effective_set);
 	cpu_effective_set = NULL;
-	cpu_effective_setsize = 0;
 
 	CPU_FREE(cpu_allowed_set);
 	cpu_allowed_set = NULL;
-	cpu_allowed_setsize = 0;
 
 	CPU_FREE(cpu_affinity_set);
 	cpu_affinity_set = NULL;
-	cpu_affinity_setsize = 0;
 
 	if (perf_pcore_set) {
 		CPU_FREE(perf_pcore_set);
@@ -6165,7 +6159,7 @@ int get_physical_node_id(struct cpu_topology *thiscpu)
 
 static int parse_cpu_str(char *cpu_str, cpu_set_t *cpu_set, int cpu_set_size)
 {
-	unsigned int start, end;
+	int start, end;
 	char *next = cpu_str;
 
 	while (next && *next) {
@@ -6178,7 +6172,7 @@ static int parse_cpu_str(char *cpu_str, cpu_set_t *cpu_set, int cpu_set_size)
 
 		start = strtoul(next, &next, 10);
 
-		if (start >= CPU_SUBSET_MAXCPUS)
+		if (start > topo.max_cpu_num)
 			return 1;
 		CPU_SET_S(start, cpu_set_size, cpu_set);
 
@@ -6205,7 +6199,7 @@ static int parse_cpu_str(char *cpu_str, cpu_set_t *cpu_set, int cpu_set_size)
 			return 1;
 
 		while (++start <= end) {
-			if (start >= CPU_SUBSET_MAXCPUS)
+			if (start > topo.max_cpu_num)
 				return 1;
 			CPU_SET_S(start, cpu_set_size, cpu_set);
 		}
@@ -6345,7 +6339,7 @@ static void update_effective_set(bool startup)
 {
 	update_effective_str(startup);
 
-	if (parse_cpu_str(cpu_effective_str, cpu_effective_set, cpu_effective_setsize))
+	if (parse_cpu_str(cpu_effective_str, cpu_effective_set, cpu_setsize))
 		err(1, "%s: cpu str malformat %s", PATH_EFFECTIVE_CPUS, cpu_effective_str);
 }
 
@@ -6373,26 +6367,6 @@ void re_initialize(void)
 	fprintf(outf, "turbostat: re-initialized with num_cpus %d, allowed_cpus %d\n", topo.num_cpus, topo.allowed_cpus);
 }
 
-void set_max_cpu_num(void)
-{
-	FILE *filep;
-	int current_cpu;
-	unsigned long dummy;
-	char pathname[64];
-
-	current_cpu = sched_getcpu();
-	if (current_cpu < 0)
-		err(1, "cannot find calling cpu ID");
-	sprintf(pathname, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", current_cpu);
-
-	filep = fopen_or_die(pathname, "r");
-	topo.max_cpu_num = 0;
-	while (fscanf(filep, "%lx,", &dummy) == 1)
-		topo.max_cpu_num += BITMASK_SIZE;
-	fclose(filep);
-	topo.max_cpu_num--;	/* 0 based */
-}
-
 /*
  * count_cpus()
  * remember the last one seen, it will be the max
@@ -6407,7 +6381,7 @@ int count_cpus(int cpu)
 
 int mark_cpu_present(int cpu)
 {
-	CPU_SET_S(cpu, cpu_present_setsize, cpu_present_set);
+	CPU_SET_S(cpu, cpu_setsize, cpu_present_set);
 	return 0;
 }
 
@@ -6710,7 +6684,7 @@ void do_sleep(void)
 
 int get_msr_sum(int cpu, off_t offset, unsigned long long *msr)
 {
-	int ret, idx;
+	int idx;
 	unsigned long long msr_cur, msr_last;
 
 	assert(!no_msr);
@@ -6722,9 +6696,7 @@ int get_msr_sum(int cpu, off_t offset, unsigned long long *msr)
 	if (idx < 0)
 		return idx;
 	/* get_msr_sum() = sum + (get_msr() - last) */
-	ret = get_msr(cpu, offset, &msr_cur);
-	if (ret)
-		return ret;
+	get_msr(cpu, offset, &msr_cur);
 	msr_last = per_cpu_msr_sum[cpu].entries[idx].last;
 	DELTA_WRAP32(msr_cur, msr_last);
 	*msr = msr_last + per_cpu_msr_sum[cpu].entries[idx].sum;
@@ -6737,7 +6709,7 @@ timer_t timerid;
 /* Timer callback, update the sum of MSRs periodically. */
 static int update_msr_sum(PER_THREAD_PARAMS)
 {
-	int i, ret;
+	int i;
 	int cpu = t->cpu_id;
 
 	UNUSED(c);
@@ -6754,11 +6726,7 @@ static int update_msr_sum(PER_THREAD_PARAMS)
 		offset = idx_to_offset(i);
 		if (offset < 0)
 			continue;
-		ret = get_msr(cpu, offset, &msr_cur);
-		if (ret) {
-			fprintf(outf, "Can not update msr(0x%llx)\n", (unsigned long long)offset);
-			continue;
-		}
+		get_msr(cpu, offset, &msr_cur);
 
 		msr_last = per_cpu_msr_sum[cpu].entries[i].last;
 		per_cpu_msr_sum[cpu].entries[i].last = msr_cur & 0xffffffff;
@@ -7568,27 +7536,21 @@ int print_hwp(PER_THREAD_PARAMS)
 		return -1;
 	}
 
-	if (get_msr(cpu, MSR_PM_ENABLE, &msr))
-		return 0;
-
+	get_msr(cpu, MSR_PM_ENABLE, &msr);
 	fprintf(outf, "cpu%d: MSR_PM_ENABLE: 0x%08llx (%sHWP)\n", cpu, msr, (msr & (1 << 0)) ? "" : "No-");
 
 	/* MSR_PM_ENABLE[1] == 1 if HWP is enabled and MSRs visible */
 	if ((msr & (1 << 0)) == 0)
 		return 0;
 
-	if (get_msr(cpu, MSR_HWP_CAPABILITIES, &msr))
-		return 0;
-
+	get_msr(cpu, MSR_HWP_CAPABILITIES, &msr);
 	fprintf(outf, "cpu%d: MSR_HWP_CAPABILITIES: 0x%08llx "
 		"(high %d guar %d eff %d low %d)\n",
 		cpu, msr,
 		(unsigned int)HWP_HIGHEST_PERF(msr),
 		(unsigned int)HWP_GUARANTEED_PERF(msr), (unsigned int)HWP_MOSTEFFICIENT_PERF(msr), (unsigned int)HWP_LOWEST_PERF(msr));
 
-	if (get_msr(cpu, MSR_HWP_REQUEST, &msr))
-		return 0;
-
+	get_msr(cpu, MSR_HWP_REQUEST, &msr);
 	fprintf(outf, "cpu%d: MSR_HWP_REQUEST: 0x%08llx "
 		"(min %d max %d des %d epp 0x%x window 0x%x pkg 0x%x)\n",
 		cpu, msr,
@@ -7598,9 +7560,7 @@ int print_hwp(PER_THREAD_PARAMS)
 		(unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3), (unsigned int)(((msr) >> 42) & 0x1));
 
 	if (has_hwp_pkg) {
-		if (get_msr(cpu, MSR_HWP_REQUEST_PKG, &msr))
-			return 0;
-
+		get_msr(cpu, MSR_HWP_REQUEST_PKG, &msr);
 		fprintf(outf, "cpu%d: MSR_HWP_REQUEST_PKG: 0x%08llx "
 			"(min %d max %d des %d epp 0x%x window 0x%x)\n",
 			cpu, msr,
@@ -7609,15 +7569,11 @@ int print_hwp(PER_THREAD_PARAMS)
 			(unsigned int)(((msr) >> 16) & 0xff), (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3));
 	}
 	if (has_hwp_notify) {
-		if (get_msr(cpu, MSR_HWP_INTERRUPT, &msr))
-			return 0;
-
+		get_msr(cpu, MSR_HWP_INTERRUPT, &msr);
 		fprintf(outf, "cpu%d: MSR_HWP_INTERRUPT: 0x%08llx "
 			"(%s_Guaranteed_Perf_Change, %s_Excursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "EN" : "Dis", ((msr) & 0x2) ? "EN" : "Dis");
 	}
-	if (get_msr(cpu, MSR_HWP_STATUS, &msr))
-		return 0;
-
+	get_msr(cpu, MSR_HWP_STATUS, &msr);
 	fprintf(outf, "cpu%d: MSR_HWP_STATUS: 0x%08llx "
 		"(%sGuaranteed_Perf_Change, %sExcursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "" : "No-", ((msr) & 0x4) ? "" : "No-");
 
@@ -7730,9 +7686,10 @@ double get_tdp_intel(void)
 {
 	unsigned long long msr;
 
-	if (valid_rapl_msrs & RAPL_PKG_POWER_INFO)
-		if (!get_msr(master_cpu, MSR_PKG_POWER_INFO, &msr))
-			return ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units;
+	if (valid_rapl_msrs & RAPL_PKG_POWER_INFO) {
+		get_msr(master_cpu, MSR_PKG_POWER_INFO, &msr);
+		return ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units;
+	}
 	return get_quirk_tdp();
 }
 
@@ -7770,8 +7727,7 @@ void rapl_probe_intel(void)
 		CLR_BIC(BIC_RAM__, &bic_enabled);
 
 	/* units on package 0, verify later other packages match */
-	if (get_msr(master_cpu, MSR_RAPL_POWER_UNIT, &msr))
-		return;
+	get_msr(master_cpu, MSR_RAPL_POWER_UNIT, &msr);
 
 	rapl_power_units = 1.0 / (1 << (msr & 0xF));
 	if (platform->has_rapl_divisor)
@@ -7818,9 +7774,7 @@ void rapl_probe_amd(void)
 	if (!valid_rapl_msrs || no_msr)
 		return;
 
-	if (get_msr(master_cpu, MSR_RAPL_PWR_UNIT, &msr))
-		return;
-
+	get_msr(master_cpu, MSR_RAPL_PWR_UNIT, &msr);
 	rapl_time_units = ldexp(1.0, -(msr >> 16 & 0xf));
 	rapl_energy_units = ldexp(1.0, -(msr >> 8 & 0x1f));
 	rapl_power_units = ldexp(1.0, -(msr & 0xf));
@@ -7976,7 +7930,7 @@ static int print_rapl_sysfs(void)
 
 		snprintf(path, PATH_MAX, "%s/%s", PATH_RAPL_SYSFS, entry->d_name);
 		if ((cdir = opendir(path)) == NULL) {
-			perror("opendir() error");
+			warn("%s", path);
 			return 1;
 		}
 
@@ -8019,21 +7973,17 @@ int print_rapl(PER_THREAD_PARAMS)
 
 	if (valid_rapl_msrs & RAPL_AMD_F17H) {
 		msr_name = "MSR_RAPL_PWR_UNIT";
-		if (get_msr(cpu, MSR_RAPL_PWR_UNIT, &msr))
-			return -1;
+		get_msr(cpu, MSR_RAPL_PWR_UNIT, &msr);
 	} else {
 		msr_name = "MSR_RAPL_POWER_UNIT";
-		if (get_msr(cpu, MSR_RAPL_POWER_UNIT, &msr))
-			return -1;
+		get_msr(cpu, MSR_RAPL_POWER_UNIT, &msr);
 	}
 
 	fprintf(outf, "cpu%d: %s: 0x%08llx (%f Watts, %f Joules, %f sec.)\n", cpu, msr_name, msr, rapl_power_units, rapl_energy_units, rapl_time_units);
 
 	if (valid_rapl_msrs & RAPL_PKG_POWER_INFO) {
 
-		if (get_msr(cpu, MSR_PKG_POWER_INFO, &msr))
-			return -5;
-
+		get_msr(cpu, MSR_PKG_POWER_INFO, &msr);
 		fprintf(outf, "cpu%d: MSR_PKG_POWER_INFO: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n",
 			cpu, msr,
 			((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units,
@@ -8043,9 +7993,7 @@ int print_rapl(PER_THREAD_PARAMS)
 	}
 	if (valid_rapl_msrs & RAPL_PKG) {
 
-		if (get_msr(cpu, MSR_PKG_POWER_LIMIT, &msr))
-			return -9;
-
+		get_msr(cpu, MSR_PKG_POWER_LIMIT, &msr);
 		fprintf(outf, "cpu%d: MSR_PKG_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 63) & 1 ? "" : "UN");
 
 		print_power_limit_msr(cpu, msr, "PKG Limit #1");
@@ -8055,17 +8003,13 @@ int print_rapl(PER_THREAD_PARAMS)
 			((msr >> 32) & 0x7FFF) * rapl_power_units,
 			(1.0 + (((msr >> 54) & 0x3) / 4.0)) * (1 << ((msr >> 49) & 0x1F)) * rapl_time_units, ((msr >> 48) & 1) ? "EN" : "DIS");
 
-		if (get_msr(cpu, MSR_VR_CURRENT_CONFIG, &msr))
-			return -9;
-
+		get_msr(cpu, MSR_VR_CURRENT_CONFIG, &msr);
 		fprintf(outf, "cpu%d: MSR_VR_CURRENT_CONFIG: 0x%08llx\n", cpu, msr);
 		fprintf(outf, "cpu%d: PKG Limit #4: %f Watts (%slocked)\n", cpu, ((msr >> 0) & 0x1FFF) * rapl_power_units, (msr >> 31) & 1 ? "" : "UN");
 	}
 
 	if (valid_rapl_msrs & RAPL_DRAM_POWER_INFO) {
-		if (get_msr(cpu, MSR_DRAM_POWER_INFO, &msr))
-			return -6;
-
+		get_msr(cpu, MSR_DRAM_POWER_INFO, &msr);
 		fprintf(outf, "cpu%d: MSR_DRAM_POWER_INFO,: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n",
 			cpu, msr,
 			((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units,
@@ -8073,32 +8017,25 @@ int print_rapl(PER_THREAD_PARAMS)
 			((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units);
 	}
 	if (valid_rapl_msrs & RAPL_DRAM) {
-		if (get_msr(cpu, MSR_DRAM_POWER_LIMIT, &msr))
-			return -9;
+		get_msr(cpu, MSR_DRAM_POWER_LIMIT, &msr);
 		fprintf(outf, "cpu%d: MSR_DRAM_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN");
 
 		print_power_limit_msr(cpu, msr, "DRAM Limit");
 	}
 	if (valid_rapl_msrs & RAPL_CORE_POLICY) {
-		if (get_msr(cpu, MSR_PP0_POLICY, &msr))
-			return -7;
-
+		get_msr(cpu, MSR_PP0_POLICY, &msr);
 		fprintf(outf, "cpu%d: MSR_PP0_POLICY: %lld\n", cpu, msr & 0xF);
 	}
 	if (valid_rapl_msrs & RAPL_CORE_POWER_LIMIT) {
-		if (get_msr(cpu, MSR_PP0_POWER_LIMIT, &msr))
-			return -9;
+		get_msr(cpu, MSR_PP0_POWER_LIMIT, &msr);
 		fprintf(outf, "cpu%d: MSR_PP0_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN");
 		print_power_limit_msr(cpu, msr, "Cores Limit");
 	}
 	if (valid_rapl_msrs & RAPL_GFX) {
-		if (get_msr(cpu, MSR_PP1_POLICY, &msr))
-			return -8;
-
+		get_msr(cpu, MSR_PP1_POLICY, &msr);
 		fprintf(outf, "cpu%d: MSR_PP1_POLICY: %lld\n", cpu, msr & 0xF);
 
-		if (get_msr(cpu, MSR_PP1_POWER_LIMIT, &msr))
-			return -9;
+		get_msr(cpu, MSR_PP1_POWER_LIMIT, &msr);
 		fprintf(outf, "cpu%d: MSR_PP1_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN");
 		print_power_limit_msr(cpu, msr, "GFX Limit");
 	}
@@ -8113,7 +8050,6 @@ int print_rapl(PER_THREAD_PARAMS)
  */
 void probe_rapl_msrs(void)
 {
-	int ret;
 	off_t offset;
 	unsigned long long msr_value;
 
@@ -8127,12 +8063,7 @@ void probe_rapl_msrs(void)
 	if (offset < 0)
 		return;
 
-	ret = get_msr(master_cpu, offset, &msr_value);
-	if (ret) {
-		if (debug)
-			fprintf(outf, "Can not read RAPL_PKG_ENERGY MSR(0x%llx)\n", (unsigned long long)offset);
-		return;
-	}
+	get_msr(master_cpu, offset, &msr_value);
 	if (msr_value == 0) {
 		if (debug)
 			fprintf(outf, "RAPL_PKG_ENERGY MSR(0x%llx) == ZERO: disabling all RAPL MSRs\n", (unsigned long long)offset);
@@ -8212,17 +8143,17 @@ int set_temperature_target(PER_THREAD_PARAMS)
 	if (!platform->has_nhm_msrs || no_msr)
 		goto guess;
 
-	if (get_msr(master_cpu, MSR_IA32_TEMPERATURE_TARGET, &msr))
-		goto guess;
-
+	get_msr(master_cpu, MSR_IA32_TEMPERATURE_TARGET, &msr);
 	tcc_default = (msr >> 16) & 0xFF;
 
 	if (!quiet) {
 		int bits = platform->tcc_offset_bits;
 		unsigned long long enabled = 0;
 
-		if (bits && !get_msr(master_cpu, MSR_PLATFORM_INFO, &enabled))
+		if (bits) {
+			get_msr(master_cpu, MSR_PLATFORM_INFO, &enabled);
 			enabled = (enabled >> 30) & 1;
+		}
 
 		if (bits && enabled) {
 			tcc_offset = (msr >> 24) & GENMASK(bits - 1, 0);
@@ -8274,15 +8205,11 @@ int print_thermal(PER_THREAD_PARAMS)
 	}
 
 	if (do_ptm && is_cpu_first_core_in_package(t, p)) {
-		if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr))
-			return 0;
-
+		get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr);
 		dts = (msr >> 16) & 0x7F;
 		fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_STATUS: 0x%08llx (%d C)\n", cpu, msr, tj_max - dts);
 
-		if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &msr))
-			return 0;
-
+		get_msr(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &msr);
 		dts = (msr >> 16) & 0x7F;
 		dts2 = (msr >> 8) & 0x7F;
 		fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2);
@@ -8291,16 +8218,12 @@ int print_thermal(PER_THREAD_PARAMS)
 	if (do_dts && debug) {
 		unsigned int resolution;
 
-		if (get_msr(cpu, MSR_IA32_THERM_STATUS, &msr))
-			return 0;
-
+		get_msr(cpu, MSR_IA32_THERM_STATUS, &msr);
 		dts = (msr >> 16) & 0x7F;
 		resolution = (msr >> 27) & 0xF;
 		fprintf(outf, "cpu%d: MSR_IA32_THERM_STATUS: 0x%08llx (%d C +/- %d)\n", cpu, msr, tj_max - dts, resolution);
 
-		if (get_msr(cpu, MSR_IA32_THERM_INTERRUPT, &msr))
-			return 0;
-
+		get_msr(cpu, MSR_IA32_THERM_INTERRUPT, &msr);
 		dts = (msr >> 16) & 0x7F;
 		dts2 = (msr >> 8) & 0x7F;
 		fprintf(outf, "cpu%d: MSR_IA32_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2);
@@ -8359,9 +8282,9 @@ void decode_feature_control_msr(void)
 	if (quiet)
 		return;
 
-	if (!get_msr(master_cpu, MSR_IA32_FEAT_CTL, &msr))
-		fprintf(outf, "cpu%d: MSR_IA32_FEATURE_CONTROL: 0x%08llx (%sLocked %s)\n",
-			master_cpu, msr, msr & FEAT_CTL_LOCKED ? "" : "UN-", msr & (1 << 18) ? "SGX" : "");
+	get_msr(master_cpu, MSR_IA32_FEAT_CTL, &msr);
+	fprintf(outf, "cpu%d: MSR_IA32_FEATURE_CONTROL: 0x%08llx (%sLocked %s)\n",
+		master_cpu, msr, msr & FEAT_CTL_LOCKED ? "" : "UN-", msr & (1 << 18) ? "SGX" : "");
 }
 
 void decode_misc_enable_msr(void)
@@ -8374,13 +8297,13 @@ void decode_misc_enable_msr(void)
 	if (!genuine_intel)
 		return;
 
-	if (!get_msr(master_cpu, MSR_IA32_MISC_ENABLE, &msr))
-		fprintf(outf, "cpu%d: MSR_IA32_MISC_ENABLE: 0x%08llx (%sTCC %sEIST %sMWAIT %sPREFETCH %sTURBO)\n",
-			master_cpu, msr,
-			msr & MSR_IA32_MISC_ENABLE_TM1 ? "" : "No-",
-			msr & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP ? "" : "No-",
-			msr & MSR_IA32_MISC_ENABLE_MWAIT ? "" : "No-",
-			msr & MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE ? "No-" : "", msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ? "No-" : "");
+	get_msr(master_cpu, MSR_IA32_MISC_ENABLE, &msr);
+	fprintf(outf, "cpu%d: MSR_IA32_MISC_ENABLE: 0x%08llx (%sTCC %sEIST %sMWAIT %sPREFETCH %sTURBO)\n",
+		master_cpu, msr,
+		msr & MSR_IA32_MISC_ENABLE_TM1 ? "" : "No-",
+		msr & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP ? "" : "No-",
+		msr & MSR_IA32_MISC_ENABLE_MWAIT ? "" : "No-",
+		msr & MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE ? "No-" : "", msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ? "No-" : "");
 }
 
 void decode_misc_feature_control(void)
@@ -8393,10 +8316,10 @@ void decode_misc_feature_control(void)
 	if (!platform->has_msr_misc_feature_control)
 		return;
 
-	if (!get_msr(master_cpu, MSR_MISC_FEATURE_CONTROL, &msr))
-		fprintf(outf,
-			"cpu%d: MSR_MISC_FEATURE_CONTROL: 0x%08llx (%sL2-Prefetch %sL2-Prefetch-pair %sL1-Prefetch %sL1-IP-Prefetch)\n",
-			master_cpu, msr, msr & (0 << 0) ? "No-" : "", msr & (1 << 0) ? "No-" : "", msr & (2 << 0) ? "No-" : "", msr & (3 << 0) ? "No-" : "");
+	get_msr(master_cpu, MSR_MISC_FEATURE_CONTROL, &msr);
+	fprintf(outf,
+		"cpu%d: MSR_MISC_FEATURE_CONTROL: 0x%08llx (%sL2-Prefetch %sL2-Prefetch-pair %sL1-Prefetch %sL1-IP-Prefetch)\n",
+		master_cpu, msr, msr & (0 << 0) ? "No-" : "", msr & (1 << 0) ? "No-" : "", msr & (2 << 0) ? "No-" : "", msr & (3 << 0) ? "No-" : "");
 }
 
 /*
@@ -8416,9 +8339,9 @@ void decode_misc_pwr_mgmt_msr(void)
 	if (!platform->has_msr_misc_pwr_mgmt)
 		return;
 
-	if (!get_msr(master_cpu, MSR_MISC_PWR_MGMT, &msr))
-		fprintf(outf, "cpu%d: MSR_MISC_PWR_MGMT: 0x%08llx (%sable-EIST_Coordination %sable-EPB %sable-OOB)\n",
-			master_cpu, msr, msr & (1 << 0) ? "DIS" : "EN", msr & (1 << 1) ? "EN" : "DIS", msr & (1 << 8) ? "EN" : "DIS");
+	get_msr(master_cpu, MSR_MISC_PWR_MGMT, &msr);
+	fprintf(outf, "cpu%d: MSR_MISC_PWR_MGMT: 0x%08llx (%sable-EIST_Coordination %sable-EPB %sable-OOB)\n",
+		master_cpu, msr, msr & (1 << 0) ? "DIS" : "EN", msr & (1 << 1) ? "EN" : "DIS", msr & (1 << 8) ? "EN" : "DIS");
 }
 
 /*
@@ -8437,11 +8360,11 @@ void decode_c6_demotion_policy_msr(void)
 	if (!platform->has_msr_c6_demotion_policy_config)
 		return;
 
-	if (!get_msr(master_cpu, MSR_CC6_DEMOTION_POLICY_CONFIG, &msr))
-		fprintf(outf, "cpu%d: MSR_CC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-CC6-Demotion)\n", master_cpu, msr, msr & (1 << 0) ? "EN" : "DIS");
+	get_msr(master_cpu, MSR_CC6_DEMOTION_POLICY_CONFIG, &msr);
+	fprintf(outf, "cpu%d: MSR_CC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-CC6-Demotion)\n", master_cpu, msr, msr & (1 << 0) ? "EN" : "DIS");
 
-	if (!get_msr(master_cpu, MSR_MC6_DEMOTION_POLICY_CONFIG, &msr))
-		fprintf(outf, "cpu%d: MSR_MC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-MC6-Demotion)\n", master_cpu, msr, msr & (1 << 0) ? "EN" : "DIS");
+	get_msr(master_cpu, MSR_MC6_DEMOTION_POLICY_CONFIG, &msr);
+	fprintf(outf, "cpu%d: MSR_MC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-MC6-Demotion)\n", master_cpu, msr, msr & (1 << 0) ? "EN" : "DIS");
 }
 
 void print_dev_latency(void)
@@ -8547,7 +8470,7 @@ int initialize_cpu_set_from_sysfs(cpu_set_t *cpu_set, char *sysfs_path, char *sy
 		warn("read %s", sysfs_path);
 		goto err;
 	}
-	if (parse_cpu_str(cpuset_buf, cpu_set, cpu_possible_setsize)) {
+	if (parse_cpu_str(cpuset_buf, cpu_set, cpu_setsize)) {
 		warnx("%s: cpu str malformat %s\n", sysfs_path, cpu_effective_str);
 		goto err;
 	}
@@ -8583,7 +8506,7 @@ void linux_perf_init_hybrid_cpus(void)
 		perf_pcore_set = CPU_ALLOC((topo.max_cpu_num + 1));
 		if (perf_pcore_set == NULL)
 			err(3, "CPU_ALLOC");
-		CPU_ZERO_S(cpu_possible_setsize, perf_pcore_set);
+		CPU_ZERO_S(cpu_setsize, perf_pcore_set);
 		initialize_cpu_set_from_sysfs(perf_pcore_set, perf_cpu_pcore_path, "cpus");
 		if (debug)
 			print_cpu_set("perf pcores", perf_pcore_set);
@@ -8595,7 +8518,7 @@ void linux_perf_init_hybrid_cpus(void)
 		perf_ecore_set = CPU_ALLOC((topo.max_cpu_num + 1));
 		if (perf_ecore_set == NULL)
 			err(3, "CPU_ALLOC");
-		CPU_ZERO_S(cpu_possible_setsize, perf_ecore_set);
+		CPU_ZERO_S(cpu_setsize, perf_ecore_set);
 		initialize_cpu_set_from_sysfs(perf_ecore_set, perf_cpu_ecore_path, "cpus");
 		if (debug)
 			print_cpu_set("perf ecores", perf_ecore_set);
@@ -8607,7 +8530,7 @@ void linux_perf_init_hybrid_cpus(void)
 		perf_lcore_set = CPU_ALLOC((topo.max_cpu_num + 1));
 		if (perf_lcore_set == NULL)
 			err(3, "CPU_ALLOC");
-		CPU_ZERO_S(cpu_possible_setsize, perf_lcore_set);
+		CPU_ZERO_S(cpu_setsize, perf_lcore_set);
 		initialize_cpu_set_from_sysfs(perf_lcore_set, perf_cpu_lcore_path, "cpus");
 		if (debug)
 			print_cpu_set("perf lcores", perf_lcore_set);
@@ -8691,7 +8614,7 @@ void rapl_perf_init(void)
 
 		memset(domain_visited, 0, num_domains * sizeof(*domain_visited));
 
-		for (int cpu = 0; cpu < topo.max_cpu_num + 1; ++cpu) {
+		for (int cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
 
 			if (cpu_is_not_allowed(cpu))
 				continue;
@@ -9136,13 +9059,10 @@ void process_cpuid()
 	cpuid_has_hv = ecx_flags & (1 << 31);
 
 	if (!no_msr) {
-		if (get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch)) {
-			warnx("get_msr(UCODE)");
-		} else {
-			ucode_patch_valid = true;
-			if (!authentic_amd && !hygon_genuine)
-				ucode_patch >>= 32;
-		}
+		get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch);
+		ucode_patch_valid = true;
+		if (!authentic_amd && !hygon_genuine)
+			ucode_patch >>= 32;
 	}
 
 	/*
@@ -9432,7 +9352,7 @@ void perf_l2_init(void)
 			}
 			continue;
 		}
-		if (perf_pcore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_pcore_set)) {
+		if (perf_pcore_set && CPU_ISSET_S(cpu, cpu_setsize, perf_pcore_set)) {
 			fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.pcore, perf_model_support->first.refs, -1, PERF_FORMAT_GROUP);
 			if (fd_l2_percpu[cpu] == -1) {
 				warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.pcore, perf_model_support->first.refs);
@@ -9445,7 +9365,7 @@ void perf_l2_init(void)
 				free_fd_l2_percpu();
 				return;
 			}
-		} else if (perf_ecore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_ecore_set)) {
+		} else if (perf_ecore_set && CPU_ISSET_S(cpu, cpu_setsize, perf_ecore_set)) {
 			fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.ecore, perf_model_support->second.refs, -1, PERF_FORMAT_GROUP);
 			if (fd_l2_percpu[cpu] == -1) {
 				warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.ecore, perf_model_support->second.refs);
@@ -9458,7 +9378,7 @@ void perf_l2_init(void)
 				free_fd_l2_percpu();
 				return;
 			}
-		} else if (perf_lcore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_lcore_set)) {
+		} else if (perf_lcore_set && CPU_ISSET_S(cpu, cpu_setsize, perf_lcore_set)) {
 			fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.lcore, perf_model_support->third.refs, -1, PERF_FORMAT_GROUP);
 			if (fd_l2_percpu[cpu] == -1) {
 				warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.lcore, perf_model_support->third.refs);
@@ -9521,6 +9441,35 @@ int set_thread_siblings(struct cpu_topology *thiscpu)
 	return (ht_id - 1);
 }
 
+/*
+ * get_max_cpu_num(path)
+ * return the last number of the cpu list in this file
+ */
+static int get_max_cpu_num(char *path)
+{
+	FILE *filep;
+	int retval, num;
+	char c;
+
+	filep = fopen(path, "r");
+	if (filep == NULL) {
+		warn("%s", path);
+		return 255;
+	}
+
+	while ((retval = fscanf(filep, "%d%c", &num, &c)) == 2) {
+		if (c == '\n')
+			break;
+		if (c != ',' && c != '-')
+			errx(1, "Bad Format '%s'", path);
+	}
+	if (retval != 2)
+		errx(1, "Bad format '%s'", path);
+
+	fclose(filep);
+	return num;
+}
+
 void topology_probe(bool startup)
 {
 	int i;
@@ -9528,8 +9477,7 @@ void topology_probe(bool startup)
 	int max_package_id = 0;
 	int max_siblings = 0;
 
-	/* Initialize num_cpus, max_cpu_num */
-	set_max_cpu_num();
+	/* Initialize num_cpus */
 	topo.num_cpus = 0;
 	for_all_proc_cpus(count_cpus);
 	if (!summary_only)
@@ -9548,8 +9496,7 @@ void topology_probe(bool startup)
 	cpu_present_set = CPU_ALLOC((topo.max_cpu_num + 1));
 	if (cpu_present_set == NULL)
 		err(3, "CPU_ALLOC");
-	cpu_present_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
-	CPU_ZERO_S(cpu_present_setsize, cpu_present_set);
+	CPU_ZERO_S(cpu_setsize, cpu_present_set);
 	for_all_proc_cpus(mark_cpu_present);
 	if (debug)
 		print_cpu_set("present set", cpu_present_set);
@@ -9560,8 +9507,7 @@ void topology_probe(bool startup)
 	cpu_possible_set = CPU_ALLOC((topo.max_cpu_num + 1));
 	if (cpu_possible_set == NULL)
 		err(3, "CPU_ALLOC");
-	cpu_possible_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
-	CPU_ZERO_S(cpu_possible_setsize, cpu_possible_set);
+	CPU_ZERO_S(cpu_setsize, cpu_possible_set);
 	initialize_cpu_set_from_sysfs(cpu_possible_set, "/sys/devices/system/cpu", "possible");
 	if (debug)
 		print_cpu_set("possible set", cpu_possible_set);
@@ -9572,8 +9518,7 @@ void topology_probe(bool startup)
 	cpu_effective_set = CPU_ALLOC((topo.max_cpu_num + 1));
 	if (cpu_effective_set == NULL)
 		err(3, "CPU_ALLOC");
-	cpu_effective_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
-	CPU_ZERO_S(cpu_effective_setsize, cpu_effective_set);
+	CPU_ZERO_S(cpu_setsize, cpu_effective_set);
 	update_effective_set(startup);
 	if (debug)
 		print_cpu_set("effective set", cpu_effective_set);
@@ -9584,48 +9529,47 @@ void topology_probe(bool startup)
 	cpu_allowed_set = CPU_ALLOC((topo.max_cpu_num + 1));
 	if (cpu_allowed_set == NULL)
 		err(3, "CPU_ALLOC");
-	cpu_allowed_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
-	CPU_ZERO_S(cpu_allowed_setsize, cpu_allowed_set);
+	CPU_ZERO_S(cpu_setsize, cpu_allowed_set);
 
 	/*
 	 * Validate and update cpu_allowed_set.
 	 *
-	 * Make sure all cpus in cpu_subset are also in cpu_present_set during startup.
-	 * Give a warning when cpus in cpu_subset become unavailable at runtime.
+	 * Make sure all cpus in cpuset_cmdline are also in cpu_present_set during startup.
+	 * Give a warning when cpus in cpuset_cmdline become unavailable at runtime.
 	 * Give a warning when cpus are not effective because of cgroup setting.
 	 *
-	 * cpu_allowed_set is the intersection of cpu_present_set/cpu_effective_set/cpu_subset.
+	 * cpu_allowed_set is the intersection of cpu_present_set/cpu_effective_set/cpuset_cmdline.
 	 */
-	for (i = 0; i < CPU_SUBSET_MAXCPUS; ++i) {
-		if (cpu_subset && !CPU_ISSET_S(i, cpu_subset_size, cpu_subset))
+	for (i = 0; i <= topo.max_cpu_num; ++i) {
+		if (cpuset_cmdline && !CPU_ISSET_S(i, cpu_setsize, cpuset_cmdline))
 			continue;
 
-		if (!CPU_ISSET_S(i, cpu_present_setsize, cpu_present_set)) {
-			if (cpu_subset) {
-				/* cpus in cpu_subset must be in cpu_present_set during startup */
+		if (!CPU_ISSET_S(i, cpu_setsize, cpu_present_set)) {
+			if (cpuset_cmdline) {
+				/* cpus in cpuset_cmdline must be in cpu_present_set during startup */
 				if (startup)
-					err(1, "cpu%d not present", i);
+					errx(1, "cpu%d not present", i);
 				else
 					fprintf(stderr, "cpu%d not present\n", i);
 			}
 			continue;
 		}
 
-		if (CPU_COUNT_S(cpu_effective_setsize, cpu_effective_set)) {
-			if (!CPU_ISSET_S(i, cpu_effective_setsize, cpu_effective_set)) {
+		if (CPU_COUNT_S(cpu_setsize, cpu_effective_set)) {
+			if (!CPU_ISSET_S(i, cpu_setsize, cpu_effective_set)) {
 				fprintf(stderr, "cpu%d not effective\n", i);
 				continue;
 			}
 		}
 
-		CPU_SET_S(i, cpu_allowed_setsize, cpu_allowed_set);
+		CPU_SET_S(i, cpu_setsize, cpu_allowed_set);
 	}
 	if (debug)
 		print_cpu_set("allowed set", cpu_allowed_set);
 
-	if (!CPU_COUNT_S(cpu_allowed_setsize, cpu_allowed_set))
-		err(-ENODEV, "No valid cpus found");
-	sched_setaffinity(0, cpu_allowed_setsize, cpu_allowed_set);
+	if (!CPU_COUNT_S(cpu_setsize, cpu_allowed_set))
+		errx(-ENODEV, "No valid cpus found");
+	sched_setaffinity(0, cpu_setsize, cpu_allowed_set);
 
 	/*
 	 * Allocate and initialize cpu_affinity_set
@@ -9633,8 +9577,7 @@ void topology_probe(bool startup)
 	cpu_affinity_set = CPU_ALLOC((topo.max_cpu_num + 1));
 	if (cpu_affinity_set == NULL)
 		err(3, "CPU_ALLOC");
-	cpu_affinity_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
-	CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set);
+	CPU_ZERO_S(cpu_setsize, cpu_affinity_set);
 
 	for_all_proc_cpus(clear_ht_id);
 
@@ -9782,7 +9725,7 @@ void allocate_counters(struct counters *counters)
 	if (counters->threads == NULL)
 		goto error;
 
-	for (i = 0; i < topo.max_cpu_num + 1; i++)
+	for (i = 0; i <= topo.max_cpu_num; i++)
 		(counters->threads)[i].cpu_id = -1;
 
 	counters->cores = calloc(num_cores, sizeof(struct core_data));
@@ -9911,7 +9854,7 @@ void set_master_cpu(void)
 {
 	int i;
 
-	for (i = 0; i < topo.max_cpu_num + 1; ++i) {
+	for (i = 0; i <= topo.max_cpu_num; ++i) {
 		if (cpu_is_not_allowed(i))
 			continue;
 		master_cpu = i;
@@ -10028,7 +9971,7 @@ int added_perf_counters_init_(struct perf_counter_info *pinfo)
 
 		memset(domain_visited, 0, max_num_domains * sizeof(*domain_visited));
 
-		for (int cpu = 0; cpu < topo.max_cpu_num + 1; ++cpu) {
+		for (int cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
 
 			next_domain = cpu_to_domain(pinfo, cpu);
 
@@ -10439,7 +10382,7 @@ void pmt_init(void)
 		mod_num = 0;	/* Relative module number for current PMT file. */
 
 		/* Open the counter for each CPU. */
-		for (cpu_num = 0; cpu_num < topo.max_cpu_num;) {
+		for (cpu_num = 0; cpu_num <= topo.max_cpu_num;) {
 
 			if (cpu_is_not_allowed(cpu_num))
 				goto next_loop_iter;
@@ -10525,11 +10468,11 @@ void turbostat_init()
 void affinitize_child(void)
 {
 	/* Prefer cpu_possible_set, if available */
-	if (sched_setaffinity(0, cpu_possible_setsize, cpu_possible_set)) {
+	if (sched_setaffinity(0, cpu_setsize, cpu_possible_set)) {
 		warn("sched_setaffinity cpu_possible_set");
 
 		/* Otherwise, allow child to run on same cpu set as turbostat */
-		if (sched_setaffinity(0, cpu_allowed_setsize, cpu_allowed_set))
+		if (sched_setaffinity(0, cpu_setsize, cpu_allowed_set))
 			warn("sched_setaffinity cpu_allowed_set");
 	}
 }
@@ -10634,7 +10577,7 @@ void print_bootcmd(void)
 	fclose(fp);
 }
 
-struct msr_counter *find_msrp_by_name(struct msr_counter *head, char *name)
+struct msr_counter *find_msrp_by_name(struct msr_counter *head, const char *name)
 {
 	struct msr_counter *mp;
 
@@ -10647,7 +10590,7 @@ struct msr_counter *find_msrp_by_name(struct msr_counter *head, char *name)
 	return NULL;
 }
 
-int add_counter(unsigned int msr_num, char *path, char *name,
+int add_counter(unsigned int msr_num, const char *path, const char *name,
 		unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int id)
 {
 	struct msr_counter *msrp;
@@ -10668,10 +10611,11 @@ int add_counter(unsigned int msr_num, char *path, char *name,
 				fprintf(stderr, "%s: %s FOUND\n", __func__, name);
 			break;
 		}
-		if (sys.added_thread_counters++ >= MAX_ADDED_THREAD_COUNTERS) {
+		if (sys.added_thread_counters >= MAX_ADDED_THREAD_COUNTERS) {
 			warnx("ignoring thread counter %s", name);
 			return -1;
 		}
+		sys.added_thread_counters++;
 		break;
 	case SCOPE_CORE:
 		msrp = find_msrp_by_name(sys.cp, name);
@@ -10680,10 +10624,11 @@ int add_counter(unsigned int msr_num, char *path, char *name,
 				fprintf(stderr, "%s: %s FOUND\n", __func__, name);
 			break;
 		}
-		if (sys.added_core_counters++ >= MAX_ADDED_CORE_COUNTERS) {
+		if (sys.added_core_counters >= MAX_ADDED_CORE_COUNTERS) {
 			warnx("ignoring core counter %s", name);
 			return -1;
 		}
+		sys.added_core_counters++;
 		break;
 	case SCOPE_PACKAGE:
 		msrp = find_msrp_by_name(sys.pp, name);
@@ -10692,10 +10637,11 @@ int add_counter(unsigned int msr_num, char *path, char *name,
 				fprintf(stderr, "%s: %s FOUND\n", __func__, name);
 			break;
 		}
-		if (sys.added_package_counters++ >= MAX_ADDED_PACKAGE_COUNTERS) {
+		if (sys.added_package_counters >= MAX_ADDED_PACKAGE_COUNTERS) {
 			warnx("ignoring package counter %s", name);
 			return -1;
 		}
+		sys.added_package_counters++;
 		break;
 	default:
 		warnx("ignoring counter %s with unknown scope", name);
@@ -10734,10 +10680,8 @@ int add_counter(unsigned int msr_num, char *path, char *name,
 		struct sysfs_path *sp;
 
 		sp = calloc(1, sizeof(struct sysfs_path));
-		if (sp == NULL) {
-			perror("calloc");
-			exit(1);
-		}
+		if (sp == NULL)
+			err(-1, "%s", path);
 		strncpy(sp->path, path, PATH_BYTES - 1);
 		sp->id = id;
 		sp->next = msrp->sp;
@@ -11437,18 +11381,18 @@ void probe_cpuidle_counts(void)
 
 /*
  * parse cpuset with following syntax
- * 1,2,4..6,8-10 and set bits in cpu_subset
+ * 1,2,4..6,8-10 and set bits in cpuset_cmdline
  */
 void parse_cpu_command(char *optarg)
 {
 	if (!strcmp(optarg, "core")) {
-		if (cpu_subset)
+		if (cpuset_cmdline)
 			goto error;
 		show_core_only++;
 		return;
 	}
 	if (!strcmp(optarg, "package")) {
-		if (cpu_subset)
+		if (cpuset_cmdline)
 			goto error;
 		show_pkg_only++;
 		return;
@@ -11456,15 +11400,23 @@ void parse_cpu_command(char *optarg)
 	if (show_core_only || show_pkg_only)
 		goto error;
 
-	cpu_subset = CPU_ALLOC(CPU_SUBSET_MAXCPUS);
-	if (cpu_subset == NULL)
-		err(3, "CPU_ALLOC");
-	cpu_subset_size = CPU_ALLOC_SIZE(CPU_SUBSET_MAXCPUS);
+	/*
+	 * cpuset_cmdline is allocated on the first invocation of --cpu
+	 * It is re-used for subsequent invocations.
+	 * It is never freed.
+	 */
+	if (!cpuset_cmdline) {
+		cpuset_cmdline = CPU_ALLOC(topo.max_cpu_num + 1);
+		if (cpuset_cmdline == NULL)
+			err(3, "CPU_ALLOC");
 
-	CPU_ZERO_S(cpu_subset_size, cpu_subset);
+		CPU_ZERO_S(cpu_setsize, cpuset_cmdline);
+	}
 
-	if (parse_cpu_str(optarg, cpu_subset, cpu_subset_size))
+	if (parse_cpu_str(optarg, cpuset_cmdline, cpu_setsize))
 		goto error;
+	if (debug)
+		print_cpu_set("--cpu", cpuset_cmdline);
 
 	return;
 
@@ -11546,7 +11498,7 @@ void cmdline(int argc, char **argv)
 			bic_lookup(&bic_enabled, optarg, SHOW_LIST);
 			break;
 		case 'f':
-			force_load++;
+			force_load = true;
 			break;
 		case 'd':
 			debug++;
@@ -11558,7 +11510,7 @@ void cmdline(int argc, char **argv)
 			 *  multiple invocations simply clear more bits in enabled mask
 			 */
 			{
-				cpu_set_t bic_group_hide;
+				bic_set_t bic_group_hide;
 
 				BIC_INIT(&bic_group_hide);
 
@@ -11671,11 +11623,16 @@ int main(int argc, char **argv)
 
 	ret = write(fd, "0\n", 2);
 	if (ret == -1)
-		perror("Can't update cgroup\n");
+		warn("/sys/fs/cgroup/cgroup.procs");
 
 	close(fd);
 
 skip_cgroup_setting:
+
+	/* probe max_cpu_num and set cpu_setsize early for use by cmdline() */
+	topo.max_cpu_num = get_max_cpu_num("/sys/devices/system/cpu/possible");
+	cpu_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
+
 	outf = stderr;
 	cmdline(argc, argv);
 
@@ -11698,8 +11655,10 @@ int main(int argc, char **argv)
 		msr_sum_record();
 
 	/* dump counters and exit */
-	if (dump_only)
+	if (dump_only) {
+		dump_bic_sets();
 		return get_and_dump_counters();
+	}
 
 	/* list header and exit */
 	if (list_header_only) {
diff --git a/tools/sched_ext/include/scx/cid.bpf.h b/tools/sched_ext/include/scx/cid.bpf.h
index db247e42fb45f..69fb4e97bc771 100644
--- a/tools/sched_ext/include/scx/cid.bpf.h
+++ b/tools/sched_ext/include/scx/cid.bpf.h
@@ -391,7 +391,9 @@ static __always_inline bool cmask_equal(const struct scx_cmask __arena *a,
 
 	if (a->base != b->base || a->nr_cids != b->nr_cids)
 		return false;
-	nr_words = CMASK_NR_WORDS(a->nr_cids);
+	if (a->nr_cids == 0)
+		return true;
+	nr_words = (a->base + a->nr_cids - 1) / 64 - a->base / 64 + 1;
 
 	bpf_for(i, 0, CMASK_MAX_WORDS) {
 		if (i >= nr_words)
@@ -402,36 +404,6 @@ static __always_inline bool cmask_equal(const struct scx_cmask __arena *a,
 	return true;
 }
 
-/*
- * True iff every bit set in @a is also set in @b over the intersection of
- * their ranges. Bits of @a outside @b's range fail the test.
- */
-static __always_inline bool cmask_subset(const struct scx_cmask __arena *a,
-					 const struct scx_cmask __arena *b)
-{
-	u32 a_end = a->base + a->nr_cids;
-	u32 b_end = b->base + b->nr_cids;
-	u32 a_wbase = a->base / 64;
-	u32 b_wbase = b->base / 64;
-	u32 nr_words, i;
-
-	/* any bit of @a outside @b's range is a subset violation */
-	if (a->base < b->base || a_end > b_end)
-		return false;
-
-	nr_words = CMASK_NR_WORDS(a->nr_cids);
-	bpf_for(i, 0, CMASK_MAX_WORDS) {
-		u32 wi_b;
-
-		if (i >= nr_words)
-			break;
-		wi_b = a_wbase + i - b_wbase;
-		if (a->bits[i] & ~b->bits[wi_b])
-			return false;
-	}
-	return true;
-}
-
 /**
  * cmask_next_set - find the first set bit at or after @cid
  * @m: cmask to search
@@ -488,16 +460,66 @@ static __always_inline u32 cmask_first_set(const struct scx_cmask __arena *m)
 	     (cid) < (m)->base + (m)->nr_cids;					\
 	     (cid) = cmask_next_set((m), (cid) + 1))
 
+/*
+ * True iff every bit set in @a is also set in @b. Matches the kernel-side
+ * scx_cmask_subset(): ranges don't need to nest, and set bits of @a outside
+ * @b's range fail the test.
+ */
+static __always_inline bool cmask_subset(const struct scx_cmask __arena *a,
+					 const struct scx_cmask __arena *b)
+{
+	u32 a_end = a->base + a->nr_cids;
+	u32 b_end = b->base + b->nr_cids;
+	u32 a_wbase = a->base / 64;
+	u32 b_wbase = b->base / 64;
+	u32 lo = a->base > b->base ? a->base : b->base;
+	u32 hi = a_end < b_end ? a_end : b_end;
+	u32 lo_word, hi_word, i;
+
+	/* set bits of @a outside @b's range can't be in @b */
+	if (a->base < b->base &&
+	    cmask_next_set(a, a->base) < (b->base < a_end ? b->base : a_end))
+		return false;
+	if (a_end > b_end &&
+	    cmask_next_set(a, a->base > b_end ? a->base : b_end) < a_end)
+		return false;
+
+	if (lo >= hi)
+		return true;
+
+	/*
+	 * Walk the words the range intersection spans. Plain word tests
+	 * suffice: the scans above guarantee @a has no set bit outside @b's
+	 * range and padding bits are kept clear by all cmask helpers.
+	 */
+	lo_word = lo / 64;
+	hi_word = (hi - 1) / 64;
+
+	bpf_for(i, 0, CMASK_MAX_WORDS) {
+		u32 w = lo_word + i;
+
+		if (w > hi_word)
+			break;
+		if (a->bits[w - a_wbase] & ~b->bits[w - b_wbase])
+			return false;
+	}
+	return true;
+}
+
 /*
  * Population count over [base, base + nr_cids). Padding bits in the head/tail
  * words are guaranteed zero by the mutating helpers, so a flat popcount over
- * all words is correct.
+ * the words the range spans is correct.
  */
 static __always_inline u32 cmask_weight(const struct scx_cmask __arena *m)
 {
-	u32 nr_words = CMASK_NR_WORDS(m->nr_cids), i;
+	u32 nr_words, i;
 	u32 count = 0;
 
+	if (!m->nr_cids)
+		return 0;
+	nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1;
+
 	bpf_for(i, 0, CMASK_MAX_WORDS) {
 		if (i >= nr_words)
 			break;
@@ -646,6 +668,83 @@ static __always_inline u32 cmask_next_and_set_wrap(const struct scx_cmask __aren
 	return found < start ? found : a_end;
 }
 
+/*
+ * Like cmask_next_and_set() but over the intersection of THREE masks. Return
+ * a->base + a->nr_cids if no cid is set in all three at or after @start.
+ */
+static __always_inline u32 cmask_next_and2_set(const struct scx_cmask __arena *a,
+					       const struct scx_cmask __arena *b,
+					       const struct scx_cmask __arena *c,
+					       u32 start)
+{
+	u32 a_end = a->base + a->nr_cids;
+	u32 b_end = b->base + b->nr_cids;
+	u32 c_end = c->base + c->nr_cids;
+	u32 a_wbase = a->base / 64;
+	u32 b_wbase = b->base / 64;
+	u32 c_wbase = c->base / 64;
+	u32 lo = a->base > b->base ? a->base : b->base;
+	u32 hi = a_end < b_end ? a_end : b_end;
+	u32 last_wi, start_wi, start_bit, i;
+
+	lo = lo > c->base ? lo : c->base;
+	hi = hi < c_end ? hi : c_end;
+
+	if (lo >= hi)
+		return a_end;
+	if (start < lo)
+		start = lo;
+	if (start >= hi)
+		return a_end;
+
+	last_wi = (hi - 1) / 64;
+	start_wi = start / 64;
+	start_bit = start & 63;
+
+	bpf_for(i, 0, CMASK_MAX_WORDS) {
+		u32 abs_wi = start_wi + i;
+		u64 word;
+		u32 found;
+
+		if (abs_wi > last_wi)
+			break;
+
+		word = a->bits[abs_wi - a_wbase] & b->bits[abs_wi - b_wbase] &
+		       c->bits[abs_wi - c_wbase];
+		if (i == 0)
+			word &= GENMASK_U64(63, start_bit);
+		if (!word)
+			continue;
+
+		found = abs_wi * 64 + ctzll(word);
+		if (found >= hi)
+			return a_end;
+		return found;
+	}
+	return a_end;
+}
+
+/*
+ * Round-robin variant of cmask_next_and2_set(): wrap to @a->base if the
+ * three-way intersection has no cid in the forward half. Return a->base +
+ * a->nr_cids if empty.
+ */
+static __always_inline u32 cmask_next_and2_set_wrap(const struct scx_cmask __arena *a,
+						    const struct scx_cmask __arena *b,
+						    const struct scx_cmask __arena *c,
+						    u32 start)
+{
+	u32 a_end = a->base + a->nr_cids;
+	u32 found;
+
+	found = cmask_next_and2_set(a, b, c, start);
+	if (found < a_end || start <= a->base)
+		return found;
+
+	found = cmask_next_and2_set(a, b, c, a->base);
+	return found < start ? found : a_end;
+}
+
 /**
  * cmask_from_cpumask - translate a kernel cpumask to a cid-space cmask
  * @m: cmask to fill. Zeroed first; only bits within [@m->base, @m->base +
diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h
index 9591a6e778ce8..acc2b131ea8f0 100644
--- a/tools/sched_ext/include/scx/common.bpf.h
+++ b/tools/sched_ext/include/scx/common.bpf.h
@@ -96,7 +96,6 @@ s32 scx_bpf_pick_any_cpu_node(const cpumask_t *cpus_allowed, int node, u64 flags
 s32 scx_bpf_pick_any_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym;
 bool scx_bpf_task_running(const struct task_struct *p) __ksym;
 s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym;
-struct rq *scx_bpf_cpu_rq(s32 cpu) __ksym;
 struct rq *scx_bpf_locked_rq(void) __ksym;
 struct task_struct *scx_bpf_cpu_curr(s32 cpu) __ksym __weak;
 struct task_struct *scx_bpf_tid_to_task(u64 tid) __ksym __weak;
@@ -105,7 +104,7 @@ void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __
 s32 scx_bpf_cpu_to_cid(s32 cpu) __ksym __weak;
 s32 scx_bpf_cid_to_cpu(s32 cid) __ksym __weak;
 void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out) __ksym __weak;
-s32 scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak;
+void scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak;
 s32 scx_bpf_task_cid(const struct task_struct *p) __ksym __weak;
 s32 scx_bpf_this_cid(void) __ksym __weak;
 struct task_struct *scx_bpf_cid_curr(s32 cid) __ksym __weak;
@@ -115,6 +114,14 @@ u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak;
 u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak;
 void scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak;
 
+/* sub-scheduler cap control, scx_bpf_sub_caps() cgroup_id 0 == self */
+s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask *cmask,
+		      struct scx_cmask *denied) __ksym __weak;
+void scx_bpf_sub_revoke(u64 cgroup_id, u64 caps, const struct scx_cmask *cmask) __ksym __weak;
+s32 scx_bpf_sub_caps(u64 cgroup_id, u64 caps, struct scx_cmask *out) __ksym __weak;
+s32 scx_bpf_sub_kill_bstr(u64 cgroup_id, char *fmt,
+			  unsigned long long *data, u32 data__sz) __ksym __weak;
+
 /*
  * Use the following as @it__iter when calling scx_bpf_dsq_move[_vtime]() from
  * within bpf_for_each() loops.
@@ -160,6 +167,22 @@ void ___scx_bpf_bstr_format_checker(const char *fmt, ...) {}
 	___scx_bpf_bstr_format_checker(fmt, ##args);				\
 })
 
+/*
+ * scx_bpf_sub_kill() wraps the scx_bpf_sub_kill_bstr() kfunc with variadic
+ * arguments instead of an array of u64. It kills the direct child sub-scheduler
+ * @cgid, passing the formatted reason to its user space, and evaluates to the
+ * kfunc's return value. On a kernel without sub-scheduler support the kfunc is
+ * absent and it returns -EOPNOTSUPP.
+ */
+#define scx_bpf_sub_kill(cgid, fmt, args...)					\
+({										\
+	scx_bpf_bstr_preamble(fmt, args)					\
+	___scx_bpf_bstr_format_checker(fmt, ##args);				\
+	bpf_ksym_exists(scx_bpf_sub_kill_bstr) ?				\
+		scx_bpf_sub_kill_bstr((cgid), ___fmt, ___param,			\
+				      sizeof(___param)) : -EOPNOTSUPP;		\
+})
+
 /*
  * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments
  * instead of an array of u64. Invoking this macro will cause the scheduler to
@@ -983,8 +1006,8 @@ extern struct irqtime___local cpu_irqtime __ksym __weak;
 static inline struct rq___local *get_current_rq(u32 cpu)
 {
 	/*
-	 * This is a workaround to get an rq pointer since we decided to
-	 * deprecate scx_bpf_cpu_rq().
+	 * This is a workaround to get an rq pointer now that
+	 * scx_bpf_cpu_rq() has been removed.
 	 *
 	 * WARNING: The caller must hold the rq lock for @cpu. This is
 	 * guaranteed when called from scheduling callbacks (ops.running,
diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h
index 87f15f2962348..1b45655a64a3e 100644
--- a/tools/sched_ext/include/scx/compat.bpf.h
+++ b/tools/sched_ext/include/scx/compat.bpf.h
@@ -84,7 +84,7 @@ bool scx_bpf_dispatch_vtime_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, st
  *
  * Compat macro will be dropped on v6.19 release.
  */
-int bpf_cpumask_populate(struct cpumask *dst, void *src, size_t src__sz) __ksym __weak;
+int bpf_cpumask_populate(struct bpf_cpumask *dst, void *src, size_t src__sz) __ksym __weak;
 
 #define __COMPAT_bpf_cpumask_populate(cpumask, src, size__sz)		\
 	(bpf_ksym_exists(bpf_cpumask_populate) ?			\
@@ -122,15 +122,18 @@ static inline bool scx_bpf_sub_dispatch(u64 cgroup_id)
 }
 
 /*
- * v7.2: scx_bpf_cid_override() for explicit cpu->cid mapping. Ignore if
+ * v7.3: scx_bpf_cid_override() for explicit cid and shard mapping. Ignore if
  * missing.
  */
-void scx_bpf_cid_override___compat(const s32 *cpu_to_cid, u32 cpu_to_cid__sz) __ksym __weak;
+void scx_bpf_cid_override___compat(const s32 *cpu_to_cid, u32 cpu_to_cid__sz,
+				    const s32 *shard_start, u32 shard_start__sz) __ksym __weak;
 
-static inline void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz)
+static inline void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz,
+					 const s32 *shard_start, u32 shard_start__sz)
 {
 	if (bpf_ksym_exists(scx_bpf_cid_override___compat))
-		return scx_bpf_cid_override___compat(cpu_to_cid, cpu_to_cid__sz);
+		scx_bpf_cid_override___compat(cpu_to_cid, cpu_to_cid__sz,
+					      shard_start, shard_start__sz);
 }
 
 /**
@@ -233,23 +236,6 @@ static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags)
 	 scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) :			\
 	 scx_bpf_pick_any_cpu(cpus_allowed, flags))
 
-/*
- * v6.18: Add a helper to retrieve the current task running on a CPU.
- *
- * Keep this helper available until v6.20 for compatibility.
- */
-static inline struct task_struct *__COMPAT_scx_bpf_cpu_curr(int cpu)
-{
-	struct rq *rq;
-
-	if (bpf_ksym_exists(scx_bpf_cpu_curr))
-		return scx_bpf_cpu_curr(cpu);
-
-	rq = scx_bpf_cpu_rq(cpu);
-
-	return rq ? rq->curr : NULL;
-}
-
 /*
  * v6.19: To work around BPF maximum parameter limit, the following kfuncs are
  * replaced with variants that pack scalar arguments in a struct. Wrappers are
diff --git a/tools/sched_ext/include/scx/compat.h b/tools/sched_ext/include/scx/compat.h
index 602f07061ee39..23d9ef3e4c9d2 100644
--- a/tools/sched_ext/include/scx/compat.h
+++ b/tools/sched_ext/include/scx/compat.h
@@ -28,7 +28,7 @@ static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v
 	const struct btf_type *t;
 	const char *n;
 	s32 tid;
-	int i;
+	__u32 i;
 
 	__COMPAT_load_vmlinux_btf();
 
@@ -42,7 +42,7 @@ static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v
 	if (btf_is_enum(t)) {
 		struct btf_enum *e = btf_enum(t);
 
-		for (i = 0; i < BTF_INFO_VLEN(t->info); i++) {
+		for (i = 0; i < btf_vlen(t); i++) {
 			n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off);
 			SCX_BUG_ON(!n, "btf__name_by_offset()");
 			if (!strcmp(n, name)) {
@@ -53,7 +53,7 @@ static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v
 	} else if (btf_is_enum64(t)) {
 		struct btf_enum64 *e = btf_enum64(t);
 
-		for (i = 0; i < BTF_INFO_VLEN(t->info); i++) {
+		for (i = 0; i < btf_vlen(t); i++) {
 			n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off);
 			SCX_BUG_ON(!n, "btf__name_by_offset()");
 			if (!strcmp(n, name)) {
@@ -85,7 +85,7 @@ static inline bool __COMPAT_struct_has_field(const char *type, const char *field
 	const struct btf_member *m;
 	const char *n;
 	s32 tid;
-	int i;
+	__u32 i;
 
 	__COMPAT_load_vmlinux_btf();
 	tid = btf__find_by_name_kind(__COMPAT_vmlinux_btf, type, BTF_KIND_STRUCT);
@@ -97,7 +97,7 @@ static inline bool __COMPAT_struct_has_field(const char *type, const char *field
 
 	m = btf_members(t);
 
-	for (i = 0; i < BTF_INFO_VLEN(t->info); i++) {
+	for (i = 0; i < btf_vlen(t); i++) {
 		n = btf__name_by_offset(__COMPAT_vmlinux_btf, m[i].name_off);
 		SCX_BUG_ON(!n, "btf__name_by_offset()");
 			if (!strcmp(n, field))
diff --git a/tools/sched_ext/scx_cpu0.c b/tools/sched_ext/scx_cpu0.c
index 4966e3d4c7249..84a47aee2f95e 100644
--- a/tools/sched_ext/scx_cpu0.c
+++ b/tools/sched_ext/scx_cpu0.c
@@ -62,7 +62,7 @@ int main(int argc, char **argv)
 {
 	struct scx_cpu0 *skel;
 	struct bpf_link *link;
-	__u32 opt;
+	__s32 opt;
 	__u64 ecode;
 
 	libbpf_set_print(libbpf_print_fn);
diff --git a/tools/sched_ext/scx_flatcg.bpf.c b/tools/sched_ext/scx_flatcg.bpf.c
index fec3595818269..0fd214cc61dae 100644
--- a/tools/sched_ext/scx_flatcg.bpf.c
+++ b/tools/sched_ext/scx_flatcg.bpf.c
@@ -256,7 +256,7 @@ static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc)
 	 * and thus can't be updated and repositioned. Instead, we collect the
 	 * vtime deltas separately and apply it asynchronously here.
 	 */
-	delta = __sync_fetch_and_sub(&cgc->cvtime_delta, cgc->cvtime_delta);
+	delta = __sync_fetch_and_and(&cgc->cvtime_delta, 0);
 	cvtime = cgv_node->cvtime + delta;
 
 	/*
@@ -570,7 +570,8 @@ void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable)
 	cgc = find_cgrp_ctx(cgrp);
 	if (cgc) {
 		__sync_fetch_and_add(&cgc->cvtime_delta,
-				     p->se.sum_exec_runtime - taskc->bypassed_at);
+				     (p->se.sum_exec_runtime - taskc->bypassed_at) *
+				     FCG_HWEIGHT_ONE / (cgc->hweight ?: 1));
 		taskc->bypassed_at = 0;
 	}
 	bpf_cgroup_release(cgrp);
diff --git a/tools/sched_ext/scx_flatcg.c b/tools/sched_ext/scx_flatcg.c
index de2bef86d64d6..7799782b76d18 100644
--- a/tools/sched_ext/scx_flatcg.c
+++ b/tools/sched_ext/scx_flatcg.c
@@ -105,12 +105,12 @@ static void fcg_read_stats(struct scx_flatcg *skel, __u64 *stats)
 	__u64 *cnts;
 	__u32 idx;
 
+	memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS);
+
 	cnts = calloc(skel->rodata->nr_cpus, sizeof(__u64));
 	if (!cnts)
 		return;
 
-	memset(stats, 0, sizeof(stats[0]) * FCG_NR_STATS);
-
 	for (idx = 0; idx < FCG_NR_STATS; idx++) {
 		int ret, cpu;
 
diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c
index fd9a82a676278..09aee49120c28 100644
--- a/tools/sched_ext/scx_qmap.bpf.c
+++ b/tools/sched_ext/scx_qmap.bpf.c
@@ -1,21 +1,39 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
- * A simple five-level FIFO queue scheduler.
+ * scx_qmap: a demonstration and testing scheduler for sched_ext features.
  *
- * There are five FIFOs implemented as arena-backed doubly-linked lists
- * threaded through per-task context. A task gets assigned to one depending on
- * its compound weight. Each CPU round robins through the FIFOs and dispatches
- * more from FIFOs with higher indices - 1 from queue0, 2 from queue1, 4 from
- * queue2 and so on.
- *
- * This scheduler demonstrates:
+ * A simple scheduler that exercises a broad set of sched_ext features. Unlikely
+ * to be useful for real workloads. It demonstrates:
  *
  * - BPF-side queueing using TIDs.
  * - BPF arena for scheduler state.
  * - Core-sched support.
+ * - Hierarchical sub-scheduling: delegating cpus to child cgroup schedulers.
+ *
+ * Base design: Five FIFOs (arena-backed doubly-linked lists through per-task
+ * context). A task is assigned to a FIFO by its compound weight. Each cpu
+ * round-robins the FIFOs, dispatching more from higher ones.
+ *
+ * Sub-scheduling: Any qmap sched can delegate cpus to its own child cgroup
+ * schedulers and keep the rest for its tasks. Terminology:
+ *
+ *   excl   - A cpu the delegatee owns wholly (ENQ_IMMED|ENQ|PREEMPT).
+ *   shared - A cpu delegated as ENQ_IMMED only. Time-shared.
+ *   held_excl / held_shared - What this node was handed by its parent.
+ *            held-excl cpus are re-delegatable. A held-shared cpu is a
+ *            time-share that stays self-local.
+ *   self   - The excl cpus the node kept for itself, plus all of held_shared.
+ *   owner  - Who holds a cid - a child slot, CID_SELF, or CID_NONE.
+ *
+ * The scheduler splits its held-excl cpus among self and the children in
+ * proportion to each node's cpu.weight, handing each the floor of its share as
+ * excl cpus. The leftover from rounding forms a shared pool the round-robin
+ * timer hands around. With no excl cpu to delegate, the node evicts its
+ * children.
  *
- * This scheduler is primarily for demonstration and testing of sched_ext
- * features and unlikely to be useful for actual workloads.
+ * This policy is a demonstration only, not a practical one. The split
+ * considers only direct children and is not work-conserving. It only exists to
+ * drive sub-sched primitives with as simple logic as possible.
  *
  * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
  * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
@@ -52,24 +70,25 @@ const volatile bool always_enq_immed;
 const volatile u32 immed_stress_nth;
 const volatile u32 max_tasks;
 
+/* sub-sched: period for handing the round-robin cid pool to the next child */
+const volatile u64 round_robin_ns;
+
 /*
  * Optional cid-override test harness. When cid_override_mode is non-zero,
- * qmap_init() calls scx_bpf_cid_override() with the caller-supplied
- * cpu_to_cid array to exercise the kfunc's acceptance and error paths.
- *
- *   0 = disabled
- *   1 = valid reverse mapping
- *   2 = invalid: duplicate cid assignment
- *   3 = invalid: out-of-range cid
+ * qmap_init_cids() calls scx_bpf_cid_override() with the caller-supplied arrays
+ * to exercise the kfunc's acceptance and error paths. See enum
+ * qmap_cid_override for the modes.
  */
 const volatile u32 cid_override_mode;
+const volatile u32 cid_override_nr_shards;
 /*
- * Array lives in bss (writable) because scx_bpf_cid_override()'s BPF
- * verifier signature treats its len-paired pointer as read/write - rodata
+ * Arrays live in bss (writable) because scx_bpf_cid_override()'s BPF
+ * verifier signature treats its len-paired pointers as read/write - rodata
  * fails verification with "write into map forbidden". Userspace populates
- * it before SCX_OPS_LOAD, same as rodata, and nothing writes it after.
+ * them before SCX_OPS_LOAD, same as rodata, and nothing writes them after.
  */
 s32 cid_override_cpu_to_cid[SCX_QMAP_MAX_CPUS];
+s32 cid_override_shard_start[SCX_QMAP_MAX_CPUS];
 
 UEI_DEFINE(uei);
 
@@ -91,12 +110,12 @@ struct {
 
 struct qmap_arena __arena_global qa;
 
-/*
- * Global idle-cid tracking, maintained via update_idle / cpu_offline and
- * scanned by the direct-dispatch path. Allocated in qmap_init() from one
- * arena page, sized to the full cid space.
- */
-struct scx_cmask __arena *qa_idle_cids;
+/* ensure that BPF and userspace are seeing the same size for qmap_cmask */
+_Static_assert(QMAP_CMASK_WORDS == CMASK_NR_WORDS(SCX_QMAP_MAX_CPUS),
+	       "QMAP_CMASK_WORDS must equal CMASK_NR_WORDS(SCX_QMAP_MAX_CPUS)");
+_Static_assert(sizeof(struct qmap_cmask) ==
+	       struct_size_t(struct scx_cmask, bits, QMAP_CMASK_WORDS),
+	       "qmap_cmask must be exactly sized to back a full scx_cmask");
 
 /* Per-queue locks. Each in its own .data section as bpf_res_spin_lock requires. */
 __hidden struct bpf_res_spin_lock qa_q_lock0 SEC(".data.qa_q_lock0");
@@ -198,7 +217,7 @@ static int qmap_spin_lock(struct bpf_res_spin_lock *lock)
 }
 
 /*
- * Try prev_cid, then scan taskc->cpus_allowed AND qa_idle_cids round-robin
+ * Try prev_cid, then scan cpus_allowed AND idle_cids AND self_cids round-robin
  * from prev_cid + 1. Atomic claim retries on race; bounded by
  * IDLE_PICK_RETRIES to keep the verifier's insn budget in check.
  */
@@ -214,17 +233,19 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid,
 	if (!always_enq_immed && p->nr_cpus_allowed == 1)
 		return prev_cid;
 
-	if (cmask_test_and_clear(prev_cid, qa_idle_cids))
+	if (cmask_test(prev_cid, &qa.self_cids.mask) &&
+	    cmask_test_and_clear(prev_cid, &qa.idle_cids.mask))
 		return prev_cid;
 
 	cid = prev_cid;
 	bpf_for(i, 0, IDLE_PICK_RETRIES) {
-		cid = cmask_next_and_set_wrap(&taskc->cpus_allowed,
-					      qa_idle_cids, cid + 1);
+		cid = cmask_next_and2_set_wrap(&taskc->cpus_allowed,
+					       &qa.idle_cids.mask,
+					       &qa.self_cids.mask, cid + 1);
 		barrier_var(cid);
 		if (cid >= nr_cids)
 			return -1;
-		if (cmask_test_and_clear(cid, qa_idle_cids))
+		if (cmask_test_and_clear(cid, &qa.idle_cids.mask))
 			return cid;
 	}
 	return -1;
@@ -348,6 +369,33 @@ s32 BPF_STRUCT_OPS(qmap_select_cid, struct task_struct *p,
 	}
 }
 
+/*
+ * A received time-shared cid is held ENQ_IMMED-only, so inserts must set
+ * SCX_ENQ_IMMED.
+ */
+static u64 needs_immed(s32 cid)
+{
+	return qa.cid_shared[cid] ? SCX_ENQ_IMMED : 0;
+}
+
+/* first cid this node does NOT hold for fault injection, -1 if none */
+static s32 first_unavail_cid(void)
+{
+	s32 nr_cids = qa.nr_cids, c;
+
+	if (nr_cids > SCX_QMAP_MAX_CPUS) {
+		scx_bpf_error("-ERANGE");
+		return -1;
+	}
+
+	bpf_for(c, 0, nr_cids) {
+		if (!cmask_test(c, &qa.held_excl.mask) &&
+		    !cmask_test(c, &qa.held_shared.mask))
+			return c;
+	}
+	return -1;
+}
+
 static int weight_to_idx(u32 weight)
 {
 	/* Coarsely map the compound weight to a FIFO. */
@@ -371,9 +419,16 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
 	s32 cid;
 
 	if (enq_flags & SCX_ENQ_REENQ) {
+		u64 reason = p->scx.flags & SCX_TASK_REENQ_REASON_MASK;
+
 		__sync_fetch_and_add(&qa.nr_reenqueued, 1);
 		if (scx_bpf_task_cid(p) == 0)
 			__sync_fetch_and_add(&qa.nr_reenqueued_cid0, 1);
+		/* cap-loss and IMMED-handback bounces, relocated below */
+		if (reason == SCX_TASK_REENQ_CAP)
+			__sync_fetch_and_add(&qa.nr_reenq_cap, 1);
+		else if (reason == SCX_TASK_REENQ_IMMED)
+			__sync_fetch_and_add(&qa.nr_reenq_immed, 1);
 	}
 
 	if (p->flags & PF_KTHREAD) {
@@ -396,6 +451,49 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
 	 */
 	taskc->core_sched_seq = qa.core_sched_tail_seqs[idx]++;
 
+	/*
+	 * A node with children delegates most cids. A task of ours that can run
+	 * on none of our self cids (e.g. a per-NUMA kthread pinned to delegated
+	 * cids) would starve in SHARED/FIFO since we never pull those on a
+	 * delegated cid. Force it onto its first allowed cid's local DSQ with
+	 * needs_immed(): if we hold access there it runs, else the kernel
+	 * rejects and bounces it back via REENQ_CAP. Best-effort
+	 * anti-starvation nudge.
+	 */
+	if (qa.nr_sub_scheds && !(enq_flags & SCX_ENQ_REENQ) &&
+	    !cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) {
+		s32 c = cmask_next_set_wrap(&taskc->cpus_allowed, 0);
+
+		if (c >= 0 && c < scx_bpf_nr_cids()) {
+			taskc->force_local = false;
+			scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | c, slice_ns,
+					   enq_flags | needs_immed(c));
+			return;
+		}
+	}
+
+	/*
+	 * Fault injection: deliberately dispatch one of our own tasks to a cid
+	 * we don't hold. The kernel cap check must reject it and re-enqueue
+	 * with SCX_TASK_REENQ_CAP, so nr_inject_attempts tracks nr_reenq_cap
+	 * and proves delivery-time enforcement. Throttled.
+	 */
+	if (qa.inject_mode == QMAP_INJ_WRONG_CID && p->nr_cpus_allowed > 1 &&
+	    !(enq_flags & SCX_ENQ_REENQ)) {
+		static u32 inj_cnt;
+
+		if (!(++inj_cnt % 64)) {
+			s32 bad = first_unavail_cid();
+
+			if (bad >= 0 && cmask_test(bad, &taskc->cpus_allowed)) {
+				__sync_fetch_and_add(&qa.nr_inject_attempts, 1);
+				scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | bad,
+						   slice_ns, enq_flags);
+				return;
+			}
+		}
+	}
+
 	/*
 	 * IMMED stress testing: Every immed_stress_nth'th enqueue, dispatch
 	 * directly to prev_cpu's local DSQ even when busy to force dsq->nr > 1
@@ -418,7 +516,8 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
 	 */
 	if (taskc->force_local) {
 		taskc->force_local = false;
-		scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, slice_ns, enq_flags);
+		scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, slice_ns,
+				   enq_flags | needs_immed(scx_bpf_task_cid(p)));
 		return;
 	}
 
@@ -433,7 +532,8 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
 	if (!__COMPAT_is_enq_cpu_selected(enq_flags) &&
 	    (cid = pick_direct_dispatch_cid(p, scx_bpf_task_cid(p), taskc)) >= 0) {
 		__sync_fetch_and_add(&qa.nr_ddsp_from_enq, 1);
-		scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | cid, slice_ns, enq_flags);
+		scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | cid, slice_ns,
+				   enq_flags | needs_immed(cid));
 		return;
 	}
 
@@ -447,8 +547,9 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
 		s32 cid;
 
 		scx_bpf_dsq_insert(p, SHARED_DSQ, 0, enq_flags);
-		cid = cmask_next_and_set_wrap(&taskc->cpus_allowed,
-					      qa_idle_cids, 0);
+		cid = cmask_next_and2_set_wrap(&taskc->cpus_allowed,
+					       &qa.idle_cids.mask,
+					       &qa.self_cids.mask, 0);
 		if (cid < scx_bpf_nr_cids())
 			scx_bpf_kick_cid(cid, SCX_KICK_IDLE);
 		return;
@@ -495,7 +596,7 @@ static void update_core_sched_head_seq(struct task_struct *p)
  * moving them to HIGHPRI_DSQ and then consuming them first. This makes minor
  * difference only when dsp_batch is larger than 1.
  *
- * scx_bpf_dispatch[_vtime]_from_dsq() are allowed both from ops.dispatch() and
+ * scx_bpf_dsq_move[_vtime]() are allowed both from ops.dispatch() and
  * non-rq-lock holding BPF programs. As demonstration, this function is called
  * from qmap_dispatch() and monitor_timerfn().
  */
@@ -534,16 +635,19 @@ static bool dispatch_highpri(bool from_timer)
 		if (!(taskc = lookup_task_ctx(p)))
 			return false;
 
-		if (cmask_test(this_cid, &taskc->cpus_allowed))
+		/* only run highpri tasks on cids this node holds, not delegated ones */
+		if (cmask_test(this_cid, &taskc->cpus_allowed) &&
+		    cmask_test(this_cid, &qa.self_cids.mask))
 			cid = this_cid;
 		else
-			cid = cmask_next_set_wrap(&taskc->cpus_allowed,
-						  this_cid + 1);
+			cid = cmask_next_and_set_wrap(&taskc->cpus_allowed,
+						      &qa.self_cids.mask,
+						      this_cid + 1);
 		if (cid >= nr_cids)
 			continue;
 
 		if (scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | cid,
-				     SCX_ENQ_PREEMPT)) {
+				     SCX_ENQ_PREEMPT | needs_immed(cid))) {
 			if (cid == this_cid) {
 				dispatched = true;
 				__sync_fetch_and_add(&qa.nr_expedited_local, 1);
@@ -569,12 +673,38 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev)
 	struct cpu_ctx __arena *cpuc;
 	task_ctx_t *taskc;
 	u32 batch = dsp_batch ?: 1;
+	s32 owner;
 	s32 i;
 
 	if (dispatch_highpri(false))
 		return;
 
-	if (!qa.nr_highpri_queued && scx_bpf_dsq_move_to_local(SHARED_DSQ, 0))
+	/*
+	 * Sub-sched routing: a child-owned cid goes to its owner. Never run
+	 * this node's own tasks on a delegated cid. Read without the guard.
+	 */
+	owner = qa.part.cid_owner[cid];
+	if (owner == CID_SHARED) {
+		/* route to the live rr holder (0 = self, runs below) */
+		s32 pos = qa.part.rr_pos;
+		u64 holder_cgid = (pos >= 0 && pos < MAX_PARTS) ?
+				  qa.part.rr_slots[pos] : 0;
+
+		if (holder_cgid) {
+			scx_bpf_sub_dispatch(holder_cgid);
+			return;
+		}
+	} else if (owner >= 0 && owner < MAX_SUB_SCHEDS) {
+		u64 cgid = qa.sub_sched_ctxs[owner].cgroup_id;
+
+		if (cgid) {
+			if (scx_bpf_sub_dispatch(cgid))
+				__sync_fetch_and_add(&qa.sub_sched_ctxs[owner].nr_dsps, 1);
+			return;
+		}
+	}
+
+	if (!qa.nr_highpri_queued && scx_bpf_dsq_move_to_local(SHARED_DSQ, needs_immed(cid)))
 		return;
 
 	if (dsp_inf_loop_after && qa.nr_dispatched > dsp_inf_loop_after) {
@@ -664,7 +794,7 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev)
 			if (!batch || !scx_bpf_dispatch_nr_slots()) {
 				if (dispatch_highpri(false))
 					return;
-				scx_bpf_dsq_move_to_local(SHARED_DSQ, 0);
+				scx_bpf_dsq_move_to_local(SHARED_DSQ, needs_immed(cid));
 				return;
 			}
 			if (!cpuc->dsp_cnt)
@@ -674,12 +804,6 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev)
 		cpuc->dsp_cnt = 0;
 	}
 
-	for (i = 0; i < MAX_SUB_SCHEDS; i++) {
-		if (qa.sub_sched_cgroup_ids[i] &&
-		    scx_bpf_sub_dispatch(qa.sub_sched_cgroup_ids[i]))
-			return;
-	}
-
 	/*
 	 * No other tasks. @prev will keep running. Update its core_sched_seq as
 	 * if the task were enqueued and dispatched immediately.
@@ -912,10 +1036,18 @@ void BPF_STRUCT_OPS(qmap_cgroup_set_bandwidth, struct cgroup *cgrp,
 void BPF_STRUCT_OPS(qmap_update_idle, s32 cid, bool idle)
 {
 	QMAP_TOUCH_ARENA();
+
+	/*
+	 * The kernel delivers update_idle() for every cid this node holds
+	 * SCX_CAP_BASE on. Track every cid's idle state regardless of
+	 * delegation: the direct-dispatch pick masks idle_cids with self_cids
+	 * at selection, so a cid already idle when it returns to self needs no
+	 * reseed here.
+	 */
 	if (idle)
-		cmask_set(cid, qa_idle_cids);
+		cmask_set(cid, &qa.idle_cids.mask);
 	else
-		cmask_clear(cid, qa_idle_cids);
+		cmask_clear(cid, &qa.idle_cids.mask);
 }
 
 void BPF_STRUCT_OPS(qmap_set_cmask, struct task_struct *p,
@@ -1067,6 +1199,511 @@ static int lowpri_timerfn(void *map, int *key, struct bpf_timer *timer)
 	return 0;
 }
 
+struct round_robin_timer {
+	struct bpf_timer timer;
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARRAY);
+	__uint(max_entries, 1);
+	__type(key, u32);
+	__type(value, struct round_robin_timer);
+} round_robin_timer SEC(".maps");
+
+/*
+ * Partition update synchronization. qa.part can be written from concurrent
+ * contexts. This single-runner guard admits one writer at a time without
+ * holding a lock across the grant/revoke kfuncs. part_pending coalesces
+ * repartition requests that arrive while it is held.
+ *
+ * They live in .bss, not the arena: rr_advance() runs from a bpf_timer
+ * callback, where the verifier rejects atomic ops on arena memory.
+ */
+static u64 part_busy;
+static u64 part_pending;
+
+static bool part_try_start(void)
+{
+	/* set busy, report whether it was previously clear (we acquired it) */
+	return !__sync_fetch_and_or(&part_busy, 1);
+}
+
+static void part_end(void)
+{
+	__sync_fetch_and_and(&part_busy, 0);
+}
+
+/*
+ * compute_partition() scratch.
+ *
+ * The excl-held cids are handed out in cid order: position 0..nr_excl-1 over
+ * the held cids is split into contiguous ranges, one per participant that gets
+ * at least one excl cid. Range k is owned by cp_range_owner[k] and ends at the
+ * cumulative position cp_range_end[k].
+ */
+static s32 cp_range_owner[MAX_PARTS];	/* exclusive range k: its owner id ... */
+static s32 cp_range_end[MAX_PARTS];	/* ... and the cumulative position it ends at */
+
+/* a participant in the partition: self or an attached child */
+struct participant {
+	s32 slot;	/* child slot, or CID_SELF */
+	u32 weight;	/* cpu.weight */
+};
+
+/**
+ * place_one - assign one excl-held cid to its owner
+ * @cid: the excl-held cid to place
+ * @n: its position among the excl-held cids, in [0, nr_excl)
+ * @total_excl:	how many positions are owned exclusively (the rest are shared)
+ *
+ * Position @n below @total_excl is owned exclusively. It falls in the range
+ * whose cumulative end it is under, owned by cp_range_owner[]. A position at or
+ * above @total_excl is the rounding leftover which joins the shared pool.
+ *
+ * A separate __noinline function to help verification.
+ */
+__noinline int place_one(s32 cid, s32 n, s32 total_excl)
+{
+	s32 owner = CID_SELF, i, s;
+
+	if (cid < 0 || cid >= SCX_QMAP_MAX_CPUS || n < 0 || n >= SCX_QMAP_MAX_CPUS ||
+	    total_excl < 0) {
+		scx_bpf_error("-ERANGE");
+		return 0;
+	}
+
+	if (n < total_excl) {
+		for (i = 0; i < MAX_PARTS; i++) {
+			if (n < cp_range_end[i]) {
+				owner = cp_range_owner[i];
+				break;
+			}
+		}
+		qa.part.cid_owner[cid] = owner;
+	} else {
+		s = n - total_excl;
+		if (s < 0 || s >= MAX_PARTS) {
+			scx_bpf_error("-ERANGE");
+			return 0;
+		}
+		qa.part.shared_cids[s] = cid;
+		/* time-shared: dispatch resolves the live holder via rr_pos */
+		qa.part.cid_owner[cid] = CID_SHARED;
+	}
+	return 0;
+}
+
+/**
+ * compute_partition - build the cid partition from this node's held caps
+ *
+ * Decide each cid's owner, the shared pool and the rr rotation. __noinline to
+ * help verification. See the comment at the top of the file.
+ */
+__noinline void compute_partition(void)
+{
+	s32 nr_cids = qa.nr_cids;
+	s32 nr_excl, total_excl = 0, nr_rr = 0;
+	s32 sum_w, i, cid, n = 0, share, self_w;
+	u64 cgid_snap[MAX_SUB_SCHEDS];
+	s32 w_snap[MAX_SUB_SCHEDS];
+
+	if (nr_cids > SCX_QMAP_MAX_CPUS) {
+		scx_bpf_error("-ERANGE");
+		return;
+	}
+
+	/* find out the cids we hold */
+	scx_bpf_sub_caps(0, SCX_CAP_ENQ, (void *)(long)&qa.held_excl.mask);
+	scx_bpf_sub_caps(0, SCX_CAP_ENQ_IMMED, (void *)(long)&qa.held_shared.mask);
+	cmask_andnot(&qa.held_shared.mask, &qa.held_excl.mask);	/* held only as ENQ_IMMED */
+
+	qa.part.nr_shared = 0;
+	qa.part.nr_rr = 0;
+	qa.part.rr_pos = 0;
+
+	nr_excl = cmask_weight(&qa.held_excl.mask);
+	qa.part.nr_excl = nr_excl;
+
+	/* no excl cid: held_shared stays self-local, the rest unheld */
+	if (!nr_excl) {
+		bpf_for(cid, 0, nr_cids) {
+			if (cmask_test(cid, &qa.held_shared.mask))
+				qa.part.cid_owner[cid] = CID_SELF;
+			else
+				qa.part.cid_owner[cid] = CID_NONE;
+		}
+		return;
+	}
+
+	/*
+	 * Snapshot membership and weights so the sum_w and share loops agree.
+	 * A mid-compute change would otherwise wrap nr_shared negative.
+	 */
+	self_w = qa.self_weight ?: 100;
+	bpf_for(i, 0, MAX_SUB_SCHEDS) {
+		cgid_snap[i] = qa.sub_sched_ctxs[i].cgroup_id;
+		w_snap[i] = cgid_snap[i] ? (qa.sub_sched_ctxs[i].weight ?: 100) : 0;
+	}
+
+	/*
+	 * Participants are self plus each child. Give each a fixed range/rr
+	 * slot: self at slot 0, child i at slot i+1.
+	 *
+	 * sum_w totals every participant's weight.
+	 */
+	sum_w = self_w;
+	bpf_for(i, 0, MAX_SUB_SCHEDS) {
+		barrier_var(sum_w);
+		sum_w += w_snap[i];
+	}
+
+	/*
+	 * Split [0, nr_excl) into one contiguous range per participant, each
+	 * the floor of its weight share. cp_range_owner[]/cp_range_end[] record
+	 * each range's owner and cumulative end, total_excl counts the
+	 * exclusive slots, and the rest (nr_excl - total_excl) are shared.
+	 * rr_slots[] lists every participant for the round-robin.
+	 */
+	share = (u64)nr_excl * self_w / sum_w;
+	total_excl += share;
+	cp_range_owner[0] = CID_SELF;
+	cp_range_end[0] = total_excl;
+	qa.part.rr_slots[nr_rr++] = 0;		/* self holds slot 0 (cgid 0 = no grant) */
+
+	bpf_for(i, 0, MAX_SUB_SCHEDS) {
+		u64 cgid = cgid_snap[i];
+		s32 w = w_snap[i];
+
+		barrier_var(total_excl);
+		share = (u64)nr_excl * w / sum_w;
+		total_excl += share;
+		cp_range_owner[i + 1] = cgid ? i : CID_NONE;
+		cp_range_end[i + 1] = total_excl;
+
+		if (cgid) {
+			barrier_var(nr_rr);
+			if (nr_rr < 0 || nr_rr >= MAX_PARTS) {
+				scx_bpf_error("-ERANGE");
+				return;
+			}
+			qa.part.rr_slots[nr_rr++] = cgid;
+		}
+	}
+
+	/* assign each cid: held-excl by position, the rest self/none */
+	bpf_for(cid, 0, nr_cids) {
+		if (cmask_test(cid, &qa.held_excl.mask)) {
+			place_one(cid, n, total_excl);
+			n++;
+			barrier_var(n);
+		} else if (cmask_test(cid, &qa.held_shared.mask)) {
+			qa.part.cid_owner[cid] = CID_SELF;	/* time-share, self-local */
+		} else {
+			qa.part.cid_owner[cid] = CID_NONE;	/* not held */
+		}
+	}
+
+	qa.part.nr_shared = nr_excl - total_excl;
+	qa.part.nr_rr = nr_rr;
+}
+
+/*
+ * Charge elapsed wall time to each cid's current owner. Runs under the
+ * partition guard before every ownership change and from the stats flush, so
+ * alloc_ns[] reflects the layout that was in effect. Shared-pool time is
+ * charged to the live round-robin holder.
+ */
+static __noinline void account_alloc(void)
+{
+	u64 now = bpf_ktime_get_ns();
+	s32 rr_owner = CID_SELF;
+	s32 nr_cids = qa.nr_cids;
+	u64 delta;
+	s32 cid, i;
+
+	if (nr_cids < 0 || nr_cids > SCX_QMAP_MAX_CPUS) {
+		scx_bpf_error("-ERANGE");
+		return;
+	}
+
+	/* first call starts the clock */
+	if (!qa.alloc_ts) {
+		qa.alloc_ts = now;
+		return;
+	}
+	delta = now - qa.alloc_ts;
+	qa.alloc_ts = now;
+	qa.alloc_window_ns += delta;
+
+	/* resolve the live shared-pool holder to an owner id */
+	if (qa.part.nr_shared && qa.part.nr_rr) {
+		u32 pos = qa.part.rr_pos;
+		u64 cgid = pos < MAX_PARTS ? qa.part.rr_slots[pos] : 0;
+
+		if (cgid) {
+			rr_owner = CID_NONE;
+			bpf_for(i, 0, MAX_SUB_SCHEDS)
+				if (qa.sub_sched_ctxs[i].cgroup_id == cgid)
+					rr_owner = i;
+		}
+	}
+
+	bpf_for(cid, 0, nr_cids) {
+		s32 owner = qa.part.cid_owner[cid];
+
+		if (owner == CID_SHARED)
+			owner = rr_owner;
+		if (owner >= 0 && owner < MAX_SUB_SCHEDS)
+			qa.alloc_ns[owner] += delta;
+		else if (owner == CID_SELF)
+			qa.self_alloc_ns += delta;
+	}
+}
+
+/*
+ * apply_partition - execute the plan compute_partition() built
+ *
+ * Turn the owner map into the per-child, shared and self cmasks and issue the
+ * grant/revoke kfuncs as a delta against each child's previous grant. If no
+ * excl cid, evict every child.
+ */
+__noinline void apply_partition(void)
+{
+	s32 nr_cids = qa.nr_cids;
+	s32 nr_shared = qa.part.nr_shared;
+	s32 i, cid;
+
+	if (nr_cids < 0 || nr_cids > SCX_QMAP_MAX_CPUS ||
+	    nr_shared < 0 || nr_shared > MAX_PARTS) {
+		scx_bpf_error("-ERANGE");
+		return;
+	}
+
+	/* no excl cpu: run own tasks on the held shares, evict children */
+	if (!qa.part.nr_excl) {
+		cmask_copy(&qa.self_cids.mask, &qa.held_shared.mask);
+		bpf_for(i, 0, MAX_SUB_SCHEDS)
+			if (qa.sub_sched_ctxs[i].cgroup_id)
+				scx_bpf_sub_kill(qa.sub_sched_ctxs[i].cgroup_id,
+						 "parent holds no excl cpu to distribute");
+		return;
+	}
+
+	/*
+	 * Snapshot the old pool. The per-child revoke below clears ENQ_IMMED on
+	 * the previously-granted pool, so a cid that left the pool (now a
+	 * sibling's excl) doesn't keep a stale ENQ_IMMED on its last holder.
+	 */
+	cmask_copy(&qa.prev_rr_cids.mask, &qa.rr_cids.mask);
+
+	/* turn the owner map into the rr pool, per-child excl, and self sets */
+	cmask_init(&qa.rr_cids.mask, 0, nr_cids);
+	cmask_init(&qa.self_cids.mask, 0, nr_cids);
+
+	/* snapshot each child's grant, then rebuild the new sets below */
+	bpf_for(i, 0, MAX_SUB_SCHEDS) {
+		cmask_copy(&qa.sub_sched_ctxs[i].prev_granted.mask,
+			   &qa.sub_sched_ctxs[i].granted_cids.mask);
+		cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, nr_cids);
+	}
+
+	bpf_for(i, 0, nr_shared)
+		cmask_set(qa.part.shared_cids[i], &qa.rr_cids.mask);
+	bpf_for(cid, 0, nr_cids) {
+		s32 o = qa.part.cid_owner[cid];
+
+		if (cmask_test(cid, &qa.rr_cids.mask))
+			continue;
+		if (o >= 0 && o < MAX_SUB_SCHEDS)
+			cmask_set(cid, &qa.sub_sched_ctxs[o].granted_cids.mask);
+		else if (o == CID_SELF)
+			cmask_set(cid, &qa.self_cids.mask);
+	}
+
+	/*
+	 * Apply each child's exclusive cids as a delta against its previous
+	 * grant. Separately clear the previous shared grant (ENQ_IMMED on the
+	 * old pool), covering cids still pooled and cids that left for a
+	 * sibling's excl. The current holder is granted the new pool below.
+	 */
+	bpf_for(i, 0, MAX_SUB_SCHEDS) {
+		struct sub_sched_ctx __arena *ssc = &qa.sub_sched_ctxs[i];
+		u64 cgid = ssc->cgroup_id;
+
+		if (!cgid)
+			continue;
+
+		cmask_copy(&qa.to_revoke_cids.mask, &ssc->prev_granted.mask);
+		cmask_andnot(&qa.to_revoke_cids.mask, &ssc->granted_cids.mask);
+		cmask_copy(&qa.to_grant_cids.mask, &ssc->granted_cids.mask);
+		cmask_andnot(&qa.to_grant_cids.mask, &ssc->prev_granted.mask);
+
+		scx_bpf_sub_revoke(cgid, SCX_CAP_ENQ_IMMED,
+				   (void *)(long)&qa.prev_rr_cids.mask);
+		scx_bpf_sub_revoke(cgid, SCX_CAP_ENQ | SCX_CAP_PREEMPT | SCX_CAP_ENQ_IMMED,
+				   (void *)(long)&qa.to_revoke_cids.mask);
+		scx_bpf_sub_grant(cgid, SCX_CAP_ENQ | SCX_CAP_PREEMPT | SCX_CAP_ENQ_IMMED,
+				  (void *)(long)&qa.to_grant_cids.mask, NULL);
+	}
+
+	/* the current holder of the shared pool gets ENQ_IMMED on all of it */
+	if (nr_shared) {
+		s32 pos = qa.part.rr_pos;
+		u64 holder_cgid;
+
+		if (pos < 0 || pos >= MAX_PARTS) {
+			scx_bpf_error("-ERANGE");
+			return;
+		}
+
+		holder_cgid = qa.part.rr_slots[pos];	/* 0 = self, nothing to grant */
+		if (holder_cgid)
+			scx_bpf_sub_grant(holder_cgid, SCX_CAP_ENQ_IMMED,
+					  (void *)(long)&qa.rr_cids.mask, NULL);
+	}
+}
+
+/*
+ * Recompute the split off the node's held caps and apply it. The contexts this
+ * runs from (the sub-sched callbacks, the userspace poke, the rr timer) are not
+ * serialized by the kernel, so a single runner does the work. A caller that
+ * finds the guard held leaves part_pending set; the holder drains it before
+ * releasing, with the rr timer as a backstop.
+ */
+static void redistribute(void)
+{
+	s32 i;
+
+	__sync_fetch_and_or(&part_pending, 1);
+
+	if (!part_try_start())
+		return;
+
+	bpf_for(i, 0, 1024) {
+		__sync_fetch_and_and(&part_pending, 0);
+		/* charge elapsed time to the current partition before rebuilding it */
+		account_alloc();
+		compute_partition();
+		apply_partition();
+		if (!__sync_fetch_and_or(&part_pending, 0))
+			break;
+	}
+
+	part_end();
+}
+
+/* userspace pokes this (PROG_RUN) to resplit after a cpu.weight change */
+SEC("syscall")
+int repartition(void *ctx)
+{
+	redistribute();
+	return 0;
+}
+
+/*
+ * Userspace pokes this (PROG_RUN) to bring alloc_ns[] current before reading
+ * it for the stats display. Skipping when the partition guard is held is
+ * fine - alloc_ts is untouched, so the elapsed time is charged next time.
+ */
+SEC("syscall")
+int flush_alloc(void *ctx)
+{
+	if (part_try_start()) {
+		account_alloc();
+		part_end();
+	}
+	return 0;
+}
+
+/*
+ * Hand the shared pool to the next participant in the rotation. Self's turn
+ * just revokes the pool back to this sched. A child's turn grants it ENQ_IMMED
+ * on the entire pool. As only excl-held cids are time-shared, a wall-clock
+ * rotation works. Driven by the round-robin timer.
+ */
+static void rr_advance(void)
+{
+	s32 nr_shared, old_pos, new_pos;
+	u64 old_cgid, new_cgid;
+	u32 nr_rr;		/* unsigned for % */
+
+	/* a redistribute holds the partition and rebuilds the pool, so skip */
+	if (!part_try_start())
+		return;
+
+	nr_rr = qa.part.nr_rr;
+	nr_shared = qa.part.nr_shared;
+
+	if (nr_shared < 0 || nr_shared > MAX_PARTS) {
+		scx_bpf_error("-ERANGE");
+		return;
+	}
+
+	if (nr_shared && nr_rr >= 2) {
+		/* close out the outgoing holder's pool time */
+		account_alloc();
+
+		old_pos = qa.part.rr_pos;
+		new_pos = (old_pos + 1) % nr_rr;
+		old_cgid = qa.part.rr_slots[old_pos];
+		new_cgid = qa.part.rr_slots[new_pos];
+		qa.part.rr_pos = new_pos;
+
+		/*
+		 * Move the ENQ_IMMED cap to the next participant. The shared
+		 * cids stay marked CID_SHARED. qmap_dispatch() resolves the
+		 * live holder via rr_pos without the guard, so a dispatch
+		 * racing this handoff may reenqueue a task once. Harmless for a
+		 * time-share.
+		 */
+		if (old_cgid)
+			scx_bpf_sub_revoke(old_cgid, SCX_CAP_ENQ_IMMED,
+					   (void *)(long)&qa.rr_cids.mask);
+		if (new_cgid)
+			scx_bpf_sub_grant(new_cgid, SCX_CAP_ENQ_IMMED,
+					  (void *)(long)&qa.rr_cids.mask, NULL);
+	}
+
+	part_end();
+
+	/* a resplit queued while we held the guard supersedes this rotation */
+	if (__sync_fetch_and_or(&part_pending, 0))
+		redistribute();
+}
+
+/* advance the time-shared cid pool every round_robin_ns */
+static int round_robin_timerfn(void *map, int *key, struct bpf_timer *timer)
+{
+	rr_advance();
+	bpf_timer_start(timer, round_robin_ns, 0);
+	return 0;
+}
+
+/*
+ * Custom cid layout for the cid-override test. On invalid input the kfunc
+ * scx_error()s and aborts the scheduler.
+ */
+s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init_cids)
+{
+	u32 nr_cpu_ids = scx_bpf_nr_cpu_ids();
+
+	if (!cid_override_mode)
+		return 0;
+
+	/* bound the count so the verifier accepts cpu_to_cid's mem/len pair */
+	if (nr_cpu_ids > SCX_QMAP_MAX_CPUS) {
+		scx_bpf_error("nr_cpu_ids=%u exceeds SCX_QMAP_MAX_CPUS=%d",
+			      nr_cpu_ids, SCX_QMAP_MAX_CPUS);
+		return -EINVAL;
+	}
+
+	scx_bpf_cid_override((const s32 *)cid_override_cpu_to_cid,
+			     nr_cpu_ids * sizeof(s32),
+			     (const s32 *)cid_override_shard_start,
+			     cid_override_nr_shards * sizeof(s32));
+	return 0;
+}
+
 s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init)
 {
 	u8 __arena *slab;
@@ -1089,16 +1726,6 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init)
 		return -EINVAL;
 	}
 
-	/*
-	 * cid-override test hook. Must run before anything that reads the
-	 * cid space (scx_bpf_nr_cids, cmask_init, etc.). On invalid input,
-	 * the kfunc calls scx_error() which aborts the scheduler.
-	 */
-	if (cid_override_mode) {
-		scx_bpf_cid_override((const s32 *)cid_override_cpu_to_cid,
-				     nr_cpu_ids * sizeof(s32));
-	}
-
 	/*
 	 * Allocate the task_ctx slab in arena and thread the entire slab onto
 	 * the free list. max_tasks is set by userspace before load. Each entry
@@ -1129,16 +1756,43 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init)
 	}
 	qa.task_free_head = (task_ctx_t *)slab;
 
+	/* cache the cid count, trusted to be <= SCX_QMAP_MAX_CPUS hereafter */
+	qa.nr_cids = nr_cids;
+
+	/* cmasks are embedded in qa, so they only need initializing */
+	cmask_init(&qa.idle_cids.mask, 0, nr_cids);
+	cmask_init(&qa.rr_cids.mask, 0, nr_cids);
+	cmask_init(&qa.prev_rr_cids.mask, 0, nr_cids);
+	cmask_init(&qa.self_cids.mask, 0, nr_cids);
+	cmask_init(&qa.to_revoke_cids.mask, 0, nr_cids);
+	cmask_init(&qa.to_grant_cids.mask, 0, nr_cids);
+	cmask_init(&qa.held_excl.mask, 0, nr_cids);
+	cmask_init(&qa.held_shared.mask, 0, nr_cids);
+
+	scx_bpf_sub_caps(0, SCX_CAP_ENQ, (void *)(long)&qa.held_excl.mask);
+	scx_bpf_sub_caps(0, SCX_CAP_ENQ_IMMED, (void *)(long)&qa.held_shared.mask);
+	cmask_andnot(&qa.held_shared.mask, &qa.held_excl.mask);
+
+	bpf_for(i, 0, MAX_SUB_SCHEDS) {
+		cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, nr_cids);
+		cmask_init(&qa.sub_sched_ctxs[i].prev_granted.mask, 0, nr_cids);
+	}
+
 	/*
-	 * Allocate and initialize the idle cmask. Starts empty - update_idle
-	 * fills it as cpus enter idle.
+	 * The root starts holding every cid. qmap_sub_ecaps_updated() maintains
+	 * per-cid shared state as effective caps settle, and redistribute()
+	 * rebuilds owner and self from held caps. A non-root node starts with
+	 * nothing.
 	 */
-	qa_idle_cids = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0);
-	if (!qa_idle_cids) {
-		scx_bpf_error("failed to allocate idle cmask");
-		return -ENOMEM;
+	bpf_for(i, 0, nr_cids) {
+		if (!sub_cgroup_id) {
+			cmask_set(i, &qa.self_cids.mask);
+			qa.part.cid_owner[i] = CID_SELF;
+		} else {
+			qa.part.cid_owner[i] = CID_NONE;
+		}
 	}
-	cmask_init(qa_idle_cids, 0, nr_cids);
+	qa.part.nr_shared = 0;
 
 	ret = scx_bpf_create_dsq(SHARED_DSQ, -1);
 	if (ret) {
@@ -1177,6 +1831,16 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init)
 			return ret;
 	}
 
+	/* sub-sched: drive the boundary-cid round-robin from a bpf timer */
+	timer = bpf_map_lookup_elem(&round_robin_timer, &key);
+	if (!timer)
+		return -ESRCH;
+	bpf_timer_init(timer, &round_robin_timer, CLOCK_MONOTONIC);
+	bpf_timer_set_callback(timer, round_robin_timerfn);
+	ret = bpf_timer_start(timer, round_robin_ns, 0);
+	if (ret)
+		return ret;
+
 	return 0;
 }
 
@@ -1189,13 +1853,20 @@ s32 BPF_STRUCT_OPS(qmap_sub_attach, struct scx_sub_attach_args *args)
 {
 	s32 i;
 
+	/* as long as there is at least one excl cpu, children can attach */
+	if (!cmask_weight(&qa.held_excl.mask))
+		return -ENOSPC;
+
 	for (i = 0; i < MAX_SUB_SCHEDS; i++) {
-		if (!qa.sub_sched_cgroup_ids[i]) {
-			qa.sub_sched_cgroup_ids[i] = args->ops->sub_cgroup_id;
-			bpf_printk("attaching sub-sched[%d] on %s",
-				   i, args->cgroup_path);
-			return 0;
-		}
+		if (qa.sub_sched_ctxs[i].cgroup_id)
+			continue;
+
+		qa.sub_sched_ctxs[i].cgroup_id = args->ops->sub_cgroup_id;
+		qa.sub_sched_ctxs[i].weight = 100;	/* until userspace feeds it */
+		qa.nr_sub_scheds++;
+		bpf_printk("attaching sub-sched[%d] on %s", i, args->cgroup_path);
+		redistribute();
+		return 0;
 	}
 
 	return -ENOSPC;
@@ -1206,15 +1877,37 @@ void BPF_STRUCT_OPS(qmap_sub_detach, struct scx_sub_detach_args *args)
 	s32 i;
 
 	for (i = 0; i < MAX_SUB_SCHEDS; i++) {
-		if (qa.sub_sched_cgroup_ids[i] == args->ops->sub_cgroup_id) {
-			qa.sub_sched_cgroup_ids[i] = 0;
-			bpf_printk("detaching sub-sched[%d] on %s",
-				   i, args->cgroup_path);
-			break;
-		}
+		if (qa.sub_sched_ctxs[i].cgroup_id != args->ops->sub_cgroup_id)
+			continue;
+
+		qa.sub_sched_ctxs[i].cgroup_id = 0;
+		qa.sub_sched_ctxs[i].weight = 100;
+		cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, qa.nr_cids);
+		qa.nr_sub_scheds--;
+		bpf_printk("detaching sub-sched[%d] on %s", i, args->cgroup_path);
+		redistribute();
+		break;
 	}
 }
 
+void BPF_STRUCT_OPS(qmap_sub_caps_updated, const struct scx_cmask *cmask, u64 caps)
+{
+	/* our held caps changed, redistribute */
+	redistribute();
+}
+
+void BPF_STRUCT_OPS(qmap_sub_ecaps_updated, s32 cid, u64 before, u64 after)
+{
+	/*
+	 * Effective caps updated. Track which cids hold shared caps so a self
+	 * task placed there enqueues IMMED.
+	 */
+	if (after & SCX_CAP_ENQ_IMMED)
+		qa.cid_shared[cid] = (after & SCX_CAP_ENQ) ? 0 : 1;
+	else
+		qa.cid_shared[cid] = 0;
+}
+
 SCX_OPS_CID_DEFINE(qmap_ops,
 	       .flags			= SCX_OPS_ENQ_EXITING | SCX_OPS_TID_TO_TASK,
 	       .select_cid		= (void *)qmap_select_cid,
@@ -1235,6 +1928,9 @@ SCX_OPS_CID_DEFINE(qmap_ops,
 	       .cgroup_set_bandwidth	= (void *)qmap_cgroup_set_bandwidth,
 	       .sub_attach		= (void *)qmap_sub_attach,
 	       .sub_detach		= (void *)qmap_sub_detach,
+	       .sub_caps_updated	= (void *)qmap_sub_caps_updated,
+	       .sub_ecaps_updated	= (void *)qmap_sub_ecaps_updated,
+	       .init_cids		= (void *)qmap_init_cids,
 	       .init			= (void *)qmap_init,
 	       .exit			= (void *)qmap_exit,
 	       .timeout_ms		= 5000U,
diff --git a/tools/sched_ext/scx_qmap.c b/tools/sched_ext/scx_qmap.c
index 67ddd483a4c75..dda3ddf5b7494 100644
--- a/tools/sched_ext/scx_qmap.c
+++ b/tools/sched_ext/scx_qmap.c
@@ -4,6 +4,9 @@
  * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
  * Copyright (c) 2022 David Vernet <dvernet@meta.com>
  */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -12,19 +15,44 @@
 #include <libgen.h>
 #include <sys/mman.h>
 #include <sys/stat.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <string.h>
+#include <time.h>
 #include <bpf/bpf.h>
 #include <scx/common.h>
 #include "scx_qmap.h"
 #include "scx_qmap.bpf.skel.h"
 
+/* kernfs file-handle type for open_by_handle_at(), from linux/exportfs.h */
+#ifndef FILEID_KERNFS
+#define FILEID_KERNFS	0xfe
+#endif
+
 const char help_fmt[] =
 "A simple five-level FIFO queue sched_ext scheduler.\n"
 "\n"
-"See the top-level comment in .bpf.c for more details.\n"
+"It also demonstrates hierarchical sub-scheduling: a scheduler can hand some\n"
+"of its cpus to a child cgroup that runs its own scheduler. Run one qmap as\n"
+"the parent, then run another qmap on a child cgroup with -c to attach it\n"
+"beneath the parent.\n"
+"\n"
+"The policy below is deliberately simplistic and the resulting behavior can\n"
+"look odd. qmap is a demo: it exists to exercise every sub-scheduling\n"
+"primitive the kernel offers with as little code as possible, not to schedule\n"
+"well.\n"
+"\n"
+"A parent divides the full cpus it holds among itself and its children in\n"
+"proportion to cpu.weight. The cpus left over by rounding are time-shared,\n"
+"handed to each participant in turn every -R ms. A cpu a scheduler only\n"
+"holds a time-share of is never handed further down, and a parent left with\n"
+"no full cpu of its own shuts its children down.\n"
+"\n"
+"See the top-of-file comment in .bpf.c for the design.\n"
 "\n"
 "Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-b COUNT]\n"
 "       [-N COUNT] [-P] [-M] [-H] [-c CG_PATH] [-d PID] [-D LEN] [-S] [-p] [-I]\n"
-"       [-F COUNT] [-v]\n"
+"       [-F COUNT] [-i SEC] [-R MS] [-J MODE] [-v]\n"
 "\n"
 "  -s SLICE_US   Override slice duration\n"
 "  -e COUNT      Trigger scx_bpf_error() after COUNT enqueues\n"
@@ -43,7 +71,10 @@ const char help_fmt[] =
 "  -p            Switch only tasks on SCHED_EXT policy instead of all\n"
 "  -I            Turn on SCX_OPS_ALWAYS_ENQ_IMMED\n"
 "  -F COUNT      IMMED stress: force every COUNT'th enqueue to a busy local DSQ (use with -I)\n"
-"  -C MODE       cid-override test (shuffle|bad-dup|bad-range)\n"
+"  -C MODE       cid-override test (shuffle|bad-dup|bad-range|bad-mono)\n"
+"  -i SEC        Stats and weight-refresh interval, seconds (default 5)\n"
+"  -R MS         Round-robin period for time-shared cpus, ms (default 200)\n"
+"  -J MODE       Fault injection (wrong-cid: dispatch to a cid not held)\n"
 "  -v            Print libbpf debug messages\n"
 "  -h            Display this help and exit\n";
 
@@ -62,14 +93,230 @@ static void sigint_handler(int dummy)
 	exit_req = 1;
 }
 
+/*
+ * Open a cgroup directory directly from its id. In cgroup2 the cgroup id is the
+ * kernfs node id, so a FILEID_KERNFS handle built from the id resolves to the
+ * directory via open_by_handle_at() against the cgroup mount.
+ */
+static int open_cgroup_by_id(u64 cgid)
+{
+	static int mnt_fd = -1;
+	struct {
+		struct file_handle fh;
+		u64 id;
+	} h;
+
+	if (mnt_fd < 0) {
+		mnt_fd = open("/sys/fs/cgroup", O_RDONLY | O_DIRECTORY);
+		if (mnt_fd < 0)
+			return -1;
+	}
+	h.fh.handle_bytes = sizeof(h.id);
+	h.fh.handle_type = FILEID_KERNFS;
+	h.id = cgid;
+	return open_by_handle_at(mnt_fd, &h.fh, O_RDONLY | O_DIRECTORY);
+}
+
+/* read a cgroup's cpu.weight (1-10000) by id, 0 if unavailable */
+static u32 read_cgroup_weight(u64 cgid)
+{
+	char buf[32];
+	int dfd, wfd;
+	u32 w = 0;
+	ssize_t n;
+
+	dfd = open_cgroup_by_id(cgid);
+	if (dfd < 0)
+		return 0;
+	wfd = openat(dfd, "cpu.weight", O_RDONLY);
+	close(dfd);
+	if (wfd < 0)
+		return 0;
+	n = read(wfd, buf, sizeof(buf) - 1);
+	close(wfd);
+	if (n > 0) {
+		buf[n] = '\0';
+		w = strtoul(buf, NULL, 10);
+	}
+	return w;
+}
+
+/* read each direct child's cpu.weight into the arena, true if any changed */
+static bool feed_weights(struct qmap_arena *qa)
+{
+	bool changed = false;
+	int i;
+
+	for (i = 0; i < MAX_SUB_SCHEDS; i++) {
+		u64 cgid = qa->sub_sched_ctxs[i].cgroup_id;
+		u32 w;
+
+		if (!cgid)
+			continue;
+		/* racy against slot reuse but weight is advisory and self-corrects */
+		w = read_cgroup_weight(cgid);
+		if (w && w != qa->sub_sched_ctxs[i].weight) {
+			qa->sub_sched_ctxs[i].weight = w;
+			changed = true;
+		}
+	}
+	return changed;
+}
+
+static void invoke_repartition(struct scx_qmap *skel)
+{
+	LIBBPF_OPTS(bpf_test_run_opts, opts);
+
+	bpf_prog_test_run_opts(bpf_program__fd(skel->progs.repartition), &opts);
+}
+
+static void invoke_flush_alloc(struct scx_qmap *skel)
+{
+	LIBBPF_OPTS(bpf_test_run_opts, opts);
+
+	bpf_prog_test_run_opts(bpf_program__fd(skel->progs.flush_alloc), &opts);
+}
+
+/* previous counter snapshots for the per-interval hier stats */
+struct hier_prev {
+	u64 alloc_ns[MAX_SUB_SCHEDS];
+	u64 self_alloc_ns;
+	u64 alloc_window_ns;
+	u64 nr_dsps[MAX_SUB_SCHEDS];
+	u64 nr_reenq_cap;
+	u64 nr_reenq_immed;
+	u64 nr_inject_attempts;
+};
+
+/* current wall-clock time as "HH:MM:SS" for the startup and interval headers */
+static const char *tstamp(char *buf, size_t sz)
+{
+	time_t now = time(NULL);
+
+	strftime(buf, sz, "%H:%M:%S", localtime(&now));
+	return buf;
+}
+
+/* format the cids whose cid_owner[] matches @owner as "0-3,8", "-" if none */
+static void format_cid_ranges(struct qmap_arena *qa, s32 owner, char *buf, size_t sz)
+{
+	u32 nr = qa->nr_cids, cid;
+	size_t off = 0;
+	s32 start = -1;
+
+	buf[0] = '\0';
+	for (cid = 0; cid <= nr; cid++) {
+		bool match = cid < nr && qa->part.cid_owner[cid] == owner;
+		int n;
+
+		if (match) {
+			if (start < 0)
+				start = cid;
+			continue;
+		}
+		if (start < 0)
+			continue;
+
+		if (start == (s32)cid - 1)
+			n = snprintf(buf + off, sz - off, "%s%d",
+				     off ? "," : "", start);
+		else
+			n = snprintf(buf + off, sz - off, "%s%d-%d",
+				     off ? "," : "", start, cid - 1);
+		if (n < 0 || (size_t)n >= sz - off) {
+			strcpy(&buf[sz - 4], "...");
+			return;
+		}
+		off += n;
+		start = -1;
+	}
+	if (!off)
+		strcpy(buf, "-");
+}
+
+/* partition summary + one row per sched: weight, cpus, dispatch rate, cids */
+static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cgid)
+{
+	char ranges[128], who[16];
+	const char *rr = "-";
+	double secs;
+	u32 i;
+
+	/*
+	 * account_alloc() bumps alloc_window_ns together with the per-owner
+	 * counters, so dividing by the same window yields exact cid counts.
+	 */
+	secs = (qa->alloc_window_ns - prev->alloc_window_ns) / 1e9;
+	prev->alloc_window_ns = qa->alloc_window_ns;
+
+	/* resolve the live shared-pool holder */
+	if (qa->part.nr_shared && qa->part.nr_rr) {
+		u64 cgid = qa->part.rr_slots[qa->part.rr_pos];
+
+		rr = "self";
+		if (cgid) {
+			rr = "?";
+			for (i = 0; i < MAX_SUB_SCHEDS; i++) {
+				if (qa->sub_sched_ctxs[i].cgroup_id == cgid) {
+					snprintf(who, sizeof(who), "sub%u", i);
+					rr = who;
+					break;
+				}
+			}
+		}
+	}
+
+	format_cid_ranges(qa, CID_SHARED, ranges, sizeof(ranges));
+	printf("hier   : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu inj=+%llu\n",
+	       (unsigned long long)qa->nr_sub_scheds, qa->part.nr_excl, ranges, rr,
+	       (unsigned long long)(qa->nr_reenq_cap - prev->nr_reenq_cap),
+	       (unsigned long long)(qa->nr_reenq_immed - prev->nr_reenq_immed),
+	       (unsigned long long)(qa->nr_inject_attempts - prev->nr_inject_attempts));
+	prev->nr_reenq_cap = qa->nr_reenq_cap;
+	prev->nr_reenq_immed = qa->nr_reenq_immed;
+	prev->nr_inject_attempts = qa->nr_inject_attempts;
+
+	printf("hier   : %-4s %10s %4s %6s %8s  %s\n",
+	       "", "cgroup", "w", "alloc", "disp/s", "cids");
+
+	format_cid_ranges(qa, CID_SELF, ranges, sizeof(ranges));
+	printf("hier   : %-4s %10llu %4u %6.2f %8s  %s\n", "self",
+	       (unsigned long long)own_cgid, qa->self_weight,
+	       secs > 0 ? (qa->self_alloc_ns - prev->self_alloc_ns) / (secs * 1e9) : 0.0,
+	       "-", ranges);
+	prev->self_alloc_ns = qa->self_alloc_ns;
+
+	for (i = 0; i < MAX_SUB_SCHEDS; i++) {
+		struct sub_sched_ctx *sc = &qa->sub_sched_ctxs[i];
+
+		if (!sc->cgroup_id)
+			continue;
+
+		snprintf(who, sizeof(who), "sub%u", i);
+		format_cid_ranges(qa, i, ranges, sizeof(ranges));
+		printf("hier   : %-4s %10llu %4u %6.2f %8.1f  %s\n", who,
+		       (unsigned long long)sc->cgroup_id, sc->weight,
+		       secs > 0 ? (qa->alloc_ns[i] - prev->alloc_ns[i]) / (secs * 1e9) : 0.0,
+		       secs > 0 ? (sc->nr_dsps - prev->nr_dsps[i]) / secs : 0.0,
+		       ranges);
+		prev->alloc_ns[i] = qa->alloc_ns[i];
+		prev->nr_dsps[i] = sc->nr_dsps;
+	}
+}
+
 int main(int argc, char **argv)
 {
 	struct scx_qmap *skel;
 	struct bpf_link *link;
 	struct qmap_arena *qa;
-	__u32 test_error_cnt = 0;
-	__u64 ecode;
-	int opt;
+	u32 test_error_cnt = 0;
+	u64 ecode;
+	int opt, stats_intv = 5, i, round_robin_ms = 200;
+	struct hier_prev hprev = {};
+	const char *sub_cg_path = NULL;
+	char tbuf[32];
+	u32 inject_mode = 0;
+	u64 own_cgid = 0;
 
 	libbpf_set_print(libbpf_print_fn);
 	signal(SIGINT, sigint_handler);
@@ -89,7 +336,7 @@ int main(int argc, char **argv)
 	skel->rodata->slice_ns = __COMPAT_ENUM_OR_ZERO("scx_public_consts", "SCX_SLICE_DFL");
 	skel->rodata->max_tasks = 16384;
 
-	while ((opt = getopt(argc, argv, "s:e:t:T:l:b:N:PMHc:d:D:SpIF:C:vh")) != -1) {
+	while ((opt = getopt(argc, argv, "s:e:t:T:l:b:N:PMHc:d:D:SpIF:C:i:R:J:vh")) != -1) {
 		switch (opt) {
 		case 's':
 			skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000;
@@ -129,6 +376,8 @@ int main(int argc, char **argv)
 			}
 			skel->struct_ops.qmap_ops->sub_cgroup_id = st.st_ino;
 			skel->rodata->sub_cgroup_id = st.st_ino;
+			own_cgid = st.st_ino;
+			sub_cg_path = optarg;
 			break;
 		}
 		case 'd':
@@ -155,32 +404,78 @@ int main(int argc, char **argv)
 		case 'C': {
 			u32 nr_cpus = libbpf_num_possible_cpus();
 			u32 mode, i;
+			s32 shard_sz = 4;
 
 			if (!strcmp(optarg, "shuffle"))
-				mode = 1;
+				mode = QMAP_CID_OVR_SHUFFLE;
 			else if (!strcmp(optarg, "bad-dup"))
-				mode = 2;
+				mode = QMAP_CID_OVR_BAD_DUP;
 			else if (!strcmp(optarg, "bad-range"))
-				mode = 3;
+				mode = QMAP_CID_OVR_BAD_RANGE;
+			else if (!strcmp(optarg, "bad-mono"))
+				mode = QMAP_CID_OVR_BAD_MONO;
 			else {
 				fprintf(stderr, "unknown cid-override mode '%s'\n", optarg);
 				return 1;
 			}
 			skel->rodata->cid_override_mode = mode;
 
-			/* shuffle: reversed cpu_to_cid, bad-dup: dup cid 0, bad-range: identity */
+			/* shuffle: reversed cpu_to_cid; others: identity */
 			for (i = 0; i < nr_cpus; i++) {
-				if (mode == 1)
+				if (mode == QMAP_CID_OVR_SHUFFLE)
 					skel->bss->cid_override_cpu_to_cid[i] = nr_cpus - 1 - i;
 				else
 					skel->bss->cid_override_cpu_to_cid[i] = i;
 			}
-			if (mode == 2 && nr_cpus >= 2)
+			if (mode == QMAP_CID_OVR_BAD_DUP && nr_cpus >= 2)
 				skel->bss->cid_override_cpu_to_cid[1] = 0;
-			if (mode == 3)
+			if (mode == QMAP_CID_OVR_BAD_RANGE)
 				skel->bss->cid_override_cpu_to_cid[0] = (s32)nr_cpus;
+
+			/*
+			 * bad-mono needs >= 3 shards to build a 0-based but
+			 * non-monotonic shard_start. Shrink the shard size so
+			 * the test runs on any machine with >= 3 cpus.
+			 */
+			if (mode == QMAP_CID_OVR_BAD_MONO) {
+				if (nr_cpus < 3) {
+					fprintf(stderr, "bad-mono needs >= 3 cpus (have %u)\n",
+						nr_cpus);
+					return 1;
+				}
+				shard_sz = nr_cpus / 3;
+			}
+
+			/* shards of shard_sz each */
+			skel->rodata->cid_override_nr_shards = (nr_cpus + shard_sz - 1) / shard_sz;
+			for (i = 0; i < skel->rodata->cid_override_nr_shards; i++)
+				skel->bss->cid_override_shard_start[i] = i * shard_sz;
+
+			if (mode == QMAP_CID_OVR_BAD_MONO) {
+				/* swap [1] and [2] to break monotonicity */
+				s32 tmp = skel->bss->cid_override_shard_start[1];
+				skel->bss->cid_override_shard_start[1] =
+					skel->bss->cid_override_shard_start[2];
+				skel->bss->cid_override_shard_start[2] = tmp;
+			}
 			break;
 		}
+		case 'i':
+			stats_intv = atoi(optarg);
+			if (stats_intv < 1)
+				stats_intv = 1;
+			break;
+		case 'R':
+			round_robin_ms = atoi(optarg);
+			if (round_robin_ms < 10)
+				round_robin_ms = 10;
+			break;
+		case 'J':
+			if (!strcmp(optarg, "wrong-cid"))
+				inject_mode = QMAP_INJ_WRONG_CID;
+			else
+				inject_mode = strtoul(optarg, NULL, 0);
+			break;
 		case 'v':
 			verbose = true;
 			break;
@@ -190,27 +485,43 @@ int main(int argc, char **argv)
 		}
 	}
 
+	skel->rodata->round_robin_ns = (u64)round_robin_ms * 1000000;
+
 	SCX_OPS_LOAD(skel, qmap_ops, scx_qmap, uei);
 	link = SCX_OPS_ATTACH(skel, qmap_ops, scx_qmap);
 
 	qa = &skel->arena->qa;
 	qa->test_error_cnt = test_error_cnt;
+	qa->inject_mode = inject_mode;
+
+	if (sub_cg_path)
+		printf("%s scx_qmap started: sub-scheduler on %s, stats every %ds\n",
+		       tstamp(tbuf, sizeof(tbuf)), sub_cg_path, stats_intv);
+	else
+		printf("%s scx_qmap started: root scheduler, stats every %ds\n",
+		       tstamp(tbuf, sizeof(tbuf)), stats_intv);
+	fflush(stdout);
 
 	while (!exit_req && !UEI_EXITED(skel, uei)) {
 		long nr_enqueued = qa->nr_enqueued;
 		long nr_dispatched = qa->nr_dispatched;
+		u32 self_weight;
+		bool repart;
 
+		printf("---- %s ----\n",
+		       tstamp(tbuf, sizeof(tbuf)));
 		printf("stats  : enq=%lu dsp=%lu delta=%ld reenq/cid0=%llu/%llu deq=%llu core=%llu enq_ddsp=%llu\n",
 		       nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched,
-		       qa->nr_reenqueued, qa->nr_reenqueued_cid0,
-		       qa->nr_dequeued,
-		       qa->nr_core_sched_execed,
-		       qa->nr_ddsp_from_enq);
+		       (unsigned long long)qa->nr_reenqueued,
+		       (unsigned long long)qa->nr_reenqueued_cid0,
+		       (unsigned long long)qa->nr_dequeued,
+		       (unsigned long long)qa->nr_core_sched_execed,
+		       (unsigned long long)qa->nr_ddsp_from_enq);
 		printf("         exp_local=%llu exp_remote=%llu exp_timer=%llu exp_lost=%llu\n",
-		       qa->nr_expedited_local,
-		       qa->nr_expedited_remote,
-		       qa->nr_expedited_from_timer,
-		       qa->nr_expedited_lost);
+		       (unsigned long long)qa->nr_expedited_local,
+		       (unsigned long long)qa->nr_expedited_remote,
+		       (unsigned long long)qa->nr_expedited_from_timer,
+		       (unsigned long long)qa->nr_expedited_lost);
 		if (__COMPAT_has_ksym("scx_bpf_cidperf_cur"))
 			printf("cpuperf: cur min/avg/max=%u/%u/%u target min/avg/max=%u/%u/%u\n",
 			       qa->cpuperf_min,
@@ -219,8 +530,27 @@ int main(int argc, char **argv)
 			       qa->cpuperf_target_min,
 			       qa->cpuperf_target_avg,
 			       qa->cpuperf_target_max);
+
+		self_weight = own_cgid ? read_cgroup_weight(own_cgid) : 100;
+		if (!self_weight)
+			self_weight = 100;
+
+		repart = feed_weights(qa);
+
+		if (self_weight != qa->self_weight) {
+			qa->self_weight = self_weight;
+			repart = true;
+		}
+
+		if (repart)
+			invoke_repartition(skel);
+
+		invoke_flush_alloc(skel);
+		print_hier(qa, &hprev, own_cgid);
 		fflush(stdout);
-		sleep(1);
+
+		for (i = 0; i < stats_intv && !exit_req && !UEI_EXITED(skel, uei); i++)
+			sleep(1);
 	}
 
 	bpf_link__destroy(link);
diff --git a/tools/sched_ext/scx_qmap.h b/tools/sched_ext/scx_qmap.h
index d15a705d5ac52..87642d21fce38 100644
--- a/tools/sched_ext/scx_qmap.h
+++ b/tools/sched_ext/scx_qmap.h
@@ -20,6 +20,7 @@
 #endif
 
 #define MAX_SUB_SCHEDS		8
+#define MAX_PARTS		(MAX_SUB_SCHEDS + 1)	/* participants: children + self */
 
 /*
  * cpu_ctxs[] is sized to a fixed cap so the layout is shared between BPF and
@@ -27,38 +28,104 @@
  */
 #define SCX_QMAP_MAX_CPUS	1024
 
-struct cpu_ctx {
-	__u64 dsp_idx;		/* dispatch index */
-	__u64 dsp_cnt;		/* remaining count */
-	__u32 avg_weight;
-	__u32 cpuperf_target;
+/*
+ * An owner id identifies who holds a cid: a child slot in [0, MAX_SUB_SCHEDS),
+ * CID_SELF for this node, CID_NONE for a cid not currently held, or CID_SHARED
+ * for a cid in the round-robin pool (its live holder is rr_slots[rr_pos]). Used
+ * by the partition's cid_owner[].
+ */
+#define CID_SELF	(-1)
+#define CID_NONE	(-2)
+#define CID_SHARED	(-3)
+
+/* -C cid-override test modes. Selects cid_override_mode in scx_qmap.bpf.c. */
+enum qmap_cid_override {
+	QMAP_CID_OVR_OFF	= 0,	/* disabled */
+	QMAP_CID_OVR_SHUFFLE	= 1,	/* valid reversed cpu->cid mapping */
+	QMAP_CID_OVR_BAD_DUP	= 2,	/* invalid: duplicate cid assignment */
+	QMAP_CID_OVR_BAD_RANGE	= 3,	/* invalid: out-of-range cid */
+	QMAP_CID_OVR_BAD_MONO	= 4,	/* invalid: non-monotonic shard_start */
 };
 
-/* Opaque to userspace; defined in scx_qmap.bpf.c. */
-struct task_ctx;
+struct cpu_ctx {
+	u64 dsp_idx;		/* dispatch index */
+	u64 dsp_cnt;		/* remaining count */
+	u32 avg_weight;
+	u32 cpuperf_target;
+};
 
 struct qmap_fifo {
 	struct task_ctx __arena *head;
 	struct task_ctx __arena *tail;
-	__s32 idx;
+	s32 idx;
+};
+
+/* -J fault-injection modes. Selects inject_mode in struct qmap_arena. */
+enum qmap_inject {
+	QMAP_INJ_OFF		= 0,
+	QMAP_INJ_WRONG_CID	= 1,	/* dispatch to a cid we don't hold */
+};
+
+/*
+ * scx_cmask's are embedded in struct qmap_arena with inline backing storage.
+ * The bpf side uses &field.mask with the normal cmask_* helpers. Userspace
+ * doesn't have access to the type definition and sees same-sized opaque words.
+ * _Static_assert()'s in .bpf.c ensure that they are in sync.
+ */
+#define QMAP_CMASK_WORDS	(((SCX_QMAP_MAX_CPUS) + 63) / 64 + 1)
+struct qmap_cmask {
+#ifdef __BPF__
+	union {
+		struct scx_cmask mask;
+		u64 words[QMAP_CMASK_WORDS + 2];
+	};
+#else
+	u64 words[QMAP_CMASK_WORDS + 2];
+#endif
+};
+
+/* Opaque to userspace; defined in scx_qmap.bpf.c. */
+struct task_ctx;
+
+/* per-direct-child state for the sub-scheduler */
+struct sub_sched_ctx {
+	u64 cgroup_id;
+	u32 weight;			/* cpu.weight, fed by userspace */
+	u64 nr_dsps;
+	struct qmap_cmask granted_cids;	/* cids granted excl to this child */
+	struct qmap_cmask prev_granted;	/* last grant, for delta calculation */
+};
+
+/*
+ * compute_partition() builds the following from this node's held caps, and
+ * apply_partition()/rr_advance() execute it. Userspace only reads for the
+ * hierarchy display.
+ */
+struct qmap_partition {
+	u32 nr_excl;			/* number of excl-held (delegatable) cids */
+	s32 cid_owner[SCX_QMAP_MAX_CPUS]; /* per cid: owner id, or CID_NONE */
+	s32 shared_cids[MAX_PARTS];	/* the round-robin cid pool */
+	u32 nr_shared;			/* number of shared_cids entries */
+	u64 rr_slots[MAX_PARTS];	/* rotation order: holder cgroup_id, 0 = self */
+	u32 nr_rr;			/* number of rr_slots entries */
+	u32 rr_pos;			/* current rotation index */
 };
 
 struct qmap_arena {
 	/* userspace-visible stats */
-	__u64 nr_enqueued, nr_dispatched, nr_reenqueued, nr_reenqueued_cid0;
-	__u64 nr_dequeued, nr_ddsp_from_enq;
-	__u64 nr_core_sched_execed;
-	__u64 nr_expedited_local, nr_expedited_remote;
-	__u64 nr_expedited_lost, nr_expedited_from_timer;
-	__u64 nr_highpri_queued;
-	__u32 test_error_cnt;
-	__u32 cpuperf_min, cpuperf_avg, cpuperf_max;
-	__u32 cpuperf_target_min, cpuperf_target_avg, cpuperf_target_max;
+	u64 nr_enqueued, nr_dispatched, nr_reenqueued, nr_reenqueued_cid0;
+	u64 nr_dequeued, nr_ddsp_from_enq;
+	u64 nr_core_sched_execed;
+	u64 nr_expedited_local, nr_expedited_remote;
+	u64 nr_expedited_lost, nr_expedited_from_timer;
+	u64 nr_highpri_queued;
+	u32 test_error_cnt;
+	u32 cpuperf_min, cpuperf_avg, cpuperf_max;
+	u32 cpuperf_target_min, cpuperf_target_avg, cpuperf_target_max;
 
 	/* kernel-side runtime state */
-	__u64 sub_sched_cgroup_ids[MAX_SUB_SCHEDS];
-	__u64 core_sched_head_seqs[5];
-	__u64 core_sched_tail_seqs[5];
+	u64 core_sched_head_seqs[5];
+	u64 core_sched_tail_seqs[5];
 
 	struct cpu_ctx cpu_ctxs[SCX_QMAP_MAX_CPUS];
 
@@ -68,6 +135,46 @@ struct qmap_arena {
 
 	/* five priority FIFOs, each a doubly-linked list through task_ctx */
 	struct qmap_fifo fifos[5];
+
+	/*
+	 * Hierarchical sub-scheduling state. See the design comment at the top
+	 * of scx_qmap.bpf.c.
+	 */
+	u32 nr_cids;			/* cid count, cached at init */
+
+	/* bpf-owned partition: read by userspace for display */
+	struct qmap_partition part;
+
+	struct sub_sched_ctx sub_sched_ctxs[MAX_SUB_SCHEDS]; /* per-child context */
+	u64 nr_sub_scheds;		/* number of attached children */
+	u32 self_weight;		/* this node's cpu.weight, fed by userspace */
+
+	/* bpf-internal per-cid state */
+	u8 cid_shared[SCX_QMAP_MAX_CPUS]; /* per cid: 1 if held shared (ENQ_IMMED-only) */
+
+	/* allocated cid-time, charged per owner by account_alloc() */
+	u64 alloc_ns[MAX_SUB_SCHEDS];	/* per child slot */
+	u64 self_alloc_ns;
+	u64 alloc_ts;			/* last accounting timestamp */
+	u64 alloc_window_ns;		/* total accounted time, the alloc denominator */
+
+	/* bpf-internal cmasks (embedded, see struct qmap_cmask) */
+	struct qmap_cmask self_cids;	/* cids this node runs its own tasks on */
+	struct qmap_cmask idle_cids;	/* idle state of all cids regardless of delegation */
+	struct qmap_cmask rr_cids;	/* the shared pool, as a mask for grant/revoke */
+
+	/* scratch cmasks */
+	struct qmap_cmask to_revoke_cids; /* delta cids to revoke */
+	struct qmap_cmask to_grant_cids; /* delta cids to grant */
+	struct qmap_cmask prev_rr_cids; /* previous shared pool, to clear stale grants */
+	struct qmap_cmask held_excl;	/* cids held excl (ENQ): delegatable */
+	struct qmap_cmask held_shared;	/* cids held shared (ENQ_IMMED only): self-local */
+
+	/* bpf -> userspace: stats */
+	u64 nr_reenq_cap;		/* SCX_TASK_REENQ_CAP bounces */
+	u64 nr_reenq_immed;		/* SCX_TASK_REENQ_IMMED bounces */
+	u64 nr_inject_attempts;		/* fault-injection: dispatches to an unheld cid */
+	u32 inject_mode;		/* fault-injection mode (QMAP_INJ_*) */
 };
 
 #endif /* __SCX_QMAP_H */
diff --git a/tools/sched_ext/scx_sdt.c b/tools/sched_ext/scx_sdt.c
index bf664b2d37859..ef197b266a362 100644
--- a/tools/sched_ext/scx_sdt.c
+++ b/tools/sched_ext/scx_sdt.c
@@ -44,7 +44,7 @@ int main(int argc, char **argv)
 {
 	struct scx_sdt *skel;
 	struct bpf_link *link;
-	__u32 opt;
+	__s32 opt;
 	__u64 ecode;
 
 	libbpf_set_print(libbpf_print_fn);
diff --git a/tools/sched_ext/scx_simple.c b/tools/sched_ext/scx_simple.c
index c3b48611712ba..34f9785335b7d 100644
--- a/tools/sched_ext/scx_simple.c
+++ b/tools/sched_ext/scx_simple.c
@@ -64,7 +64,7 @@ int main(int argc, char **argv)
 {
 	struct scx_simple *skel;
 	struct bpf_link *link;
-	__u32 opt;
+	__s32 opt;
 	__u64 ecode;
 
 	libbpf_set_print(libbpf_print_fn);
diff --git a/tools/sched_ext/scx_userland.c b/tools/sched_ext/scx_userland.c
index 616043c165e66..192b79c7e4aac 100644
--- a/tools/sched_ext/scx_userland.c
+++ b/tools/sched_ext/scx_userland.c
@@ -326,7 +326,7 @@ static int spawn_stats_thread(void)
 static void pre_bootstrap(int argc, char **argv)
 {
 	int err;
-	__u32 opt;
+	__s32 opt;
 	struct sched_param sched_param = {
 		.sched_priority = sched_get_priority_max(SCHED_EXT),
 	};
diff --git a/tools/testing/cxl/test/Kbuild b/tools/testing/cxl/test/Kbuild
index c168e3c998a7d..9a24ddc284886 100644
--- a/tools/testing/cxl/test/Kbuild
+++ b/tools/testing/cxl/test/Kbuild
@@ -5,10 +5,12 @@ obj-m += cxl_test.o
 obj-m += cxl_mock.o
 obj-m += cxl_mock_mem.o
 obj-m += cxl_translate.o
+obj-m += cxl_mock_accel.o
 
 cxl_test-y := cxl.o
 cxl_test-y += hmem_test.o
 cxl_mock-y := mock.o
 cxl_mock_mem-y := mem.o
+cxl_mock_accel-y := accel.o
 
 KBUILD_CFLAGS := $(filter-out -Wmissing-prototypes -Wmissing-declarations, $(KBUILD_CFLAGS))
diff --git a/tools/testing/cxl/test/accel.c b/tools/testing/cxl/test/accel.c
new file mode 100644
index 0000000000000..8e6f4687ca02a
--- /dev/null
+++ b/tools/testing/cxl/test/accel.c
@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: GPL-2.0-only
+// Copyright(c) 2026 Intel Corporation. All rights reserved.
+
+#include <linux/platform_device.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/sizes.h>
+#include <cxl/mailbox.h>
+#include <cxlmem.h>
+
+struct mock_cxl_accel {
+	struct cxl_dev_state cxlds;
+	struct cxl_memdev *cxlmd;
+};
+
+static int cxl_mock_accel_probe(struct platform_device *pdev)
+{
+	struct mock_cxl_accel *cxl_accel;
+	struct device *dev = &pdev->dev;
+	struct cxl_dev_state *cxlds;
+	struct cxl_memdev *cxlmd;
+	struct range mock_range;
+	int rc;
+
+	cxl_accel = devm_cxl_dev_state_create(&pdev->dev, CXL_DEVTYPE_DEVMEM,
+					      pdev->id + 1, 0,
+					      struct mock_cxl_accel, cxlds,
+					      false);
+	if (!cxl_accel)
+		return -ENOMEM;
+
+	cxlds = &cxl_accel->cxlds;
+	cxlds->media_ready = true;
+	rc = cxl_set_capacity(cxlds, SZ_512M);
+	if (rc)
+		return rc;
+
+	cxlmd = devm_cxl_probe_mem(cxlds, &mock_range);
+	if (IS_ERR(cxlmd))
+		return PTR_ERR(cxlmd);
+	cxl_accel->cxlmd = cxlmd;
+
+	dev_dbg(dev, "Probed mock accelerator with range %pra\n", &mock_range);
+
+	return 0;
+}
+
+static const struct platform_device_id cxl_mock_accel_ids[] = {
+	{ .name = "cxl_type2_accel" },
+	{ }
+};
+MODULE_DEVICE_TABLE(platform, cxl_mock_accel_ids);
+
+static struct platform_driver cxl_mock_accel_driver = {
+	.probe = cxl_mock_accel_probe,
+	.id_table = cxl_mock_accel_ids,
+	.driver = {
+		.name = KBUILD_MODNAME,
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+	},
+};
+
+module_platform_driver(cxl_mock_accel_driver);
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("cxl_test: accelerator device mock module");
+MODULE_IMPORT_NS("CXL");
diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c
index ef92dd35e030c..62bd92b3be45f 100644
--- a/tools/testing/cxl/test/cxl.c
+++ b/tools/testing/cxl/test/cxl.c
@@ -17,6 +17,7 @@
 static int interleave_arithmetic;
 static bool extended_linear_cache;
 static bool fail_autoassemble;
+static bool type2_test;
 
 #define FAKE_QTG_ID	42
 
@@ -27,6 +28,7 @@ static bool fail_autoassemble;
 #define NR_CXL_SWITCH_PORTS 2
 #define NR_CXL_PORT_DECODERS 8
 #define NR_BRIDGES (NR_CXL_HOST_BRIDGES + NR_CXL_SINGLE_HOST + NR_CXL_RCH)
+#define NR_CXL_TYPE2_ACCEL 1
 
 #define MOCK_AUTO_REGION_SIZE_DEFAULT SZ_512M
 static int mock_auto_region_size = MOCK_AUTO_REGION_SIZE_DEFAULT;
@@ -384,6 +386,19 @@ static struct {
 	},
 };
 
+static struct acpi_cedt_cfmws type2_cfmws0 = {
+	.header = {
+		.type = ACPI_CEDT_TYPE_CFMWS,
+		.length = sizeof(mock_cedt.cfmws0),
+	},
+	.interleave_ways = 0,
+	.granularity = 4,
+	.restrictions = ACPI_CEDT_CFMWS_RESTRICT_DEVMEM |
+			ACPI_CEDT_CFMWS_RESTRICT_VOLATILE,
+	.qtg_id = FAKE_QTG_ID,
+	.window_size = SZ_256M * 4,
+};
+
 struct acpi_cedt_cfmws *mock_cfmws[] = {
 	[0] = &mock_cedt.cfmws0.cfmws,
 	[1] = &mock_cedt.cfmws1.cfmws,
@@ -476,6 +491,11 @@ static void cfmws_elc_update(struct acpi_cedt_cfmws *window, int index)
 	window->window_size = mock_auto_region_size * 2;
 }
 
+static void update_type2_cfmws(void)
+{
+	memcpy(&mock_cedt.cfmws0.cfmws, &type2_cfmws0, sizeof(type2_cfmws0));
+}
+
 static int populate_cedt(void)
 {
 	struct cxl_mock_res *res;
@@ -497,11 +517,15 @@ static int populate_cedt(void)
 		chbs->length = size;
 	}
 
+	if (type2_test)
+		update_type2_cfmws();
+
 	for (i = cfmws_start; i <= cfmws_end; i++) {
 		struct acpi_cedt_cfmws *window = mock_cfmws[i];
 		int align = SZ_256M;
 
-		cfmws_elc_update(window, i);
+		if (i == 0 && !type2_test)
+			cfmws_elc_update(window, i);
 		if (window->restrictions & ACPI_CEDT_CFMWS_RESTRICT_VOLATILE)
 			align = max_t(int, SZ_256M, PMD_SIZE);
 		res = alloc_mock_res(window->window_size, align);
@@ -1060,74 +1084,26 @@ static int first_decoder(struct device *dev, const void *data)
 	return 0;
 }
 
-/*
- * Initialize a decoder during HDM enumeration.
- *
- * If a saved registry entry exists:
- *   - enabled decoders are restored from the saved programming
- *   - disabled decoders are initialized in a clean disabled state
- *
- * If no registry entry exists the decoder follows the normal mock
- * initialization path, including the special auto-region setup for
- * the first endpoints under host-bridge0.
- *
- * Returns true if decoder state was restored from the registry. In
- * that case the saved decode configuration (including target mapping)
- * has already been applied and the map_targets() is skipped.
- */
-static bool mock_init_hdm_decoder(struct cxl_decoder *cxld)
-{
-	struct acpi_cedt_cfmws *window = mock_cfmws[0];
-	struct platform_device *pdev = NULL;
-	struct cxl_endpoint_decoder *cxled;
-	struct cxl_switch_decoder *cxlsd;
-	struct cxl_port *port, *iter;
-	struct cxl_test_decoder *td;
-	struct cxl_memdev *cxlmd;
-	struct cxl_dport *dport;
-	struct device *dev;
-	bool hb0 = false;
-	u64 base;
-	int i;
+enum cxld_init_type {
+	MOCK_DECODER_INIT_DEFAULT,
+	MOCK_DECODER_INIT_SAVED,
+	MOCK_DECODER_INIT_TYPE3_AUTO,
+	MOCK_DECODER_INIT_TYPE2_AUTO,
+};
 
-	if (is_endpoint_decoder(&cxld->dev)) {
-		cxled = to_cxl_endpoint_decoder(&cxld->dev);
-		cxlmd = cxled_to_memdev(cxled);
-		WARN_ON(!dev_is_platform(cxlmd->dev.parent));
-		pdev = to_platform_device(cxlmd->dev.parent);
+static enum cxld_init_type get_decoder_init_type(struct cxl_decoder *cxld,
+						 struct platform_device *pdev,
+						 bool hb0,
+						 struct cxl_test_decoder **td)
+{
+	struct cxl_test_decoder *found_td = cxld_registry_find(cxld);
 
-		/* check is endpoint is attach to host-bridge0 */
-		port = cxled_to_port(cxled);
-		do {
-			if (port->uport_dev == &cxl_host_bridge[0]->dev) {
-				hb0 = true;
-				break;
-			}
-			if (is_cxl_port(port->dev.parent))
-				port = to_cxl_port(port->dev.parent);
-			else
-				port = NULL;
-		} while (port);
-		port = cxled_to_port(cxled);
-	} else {
-		port = to_cxl_port(cxld->dev.parent);
+	if (found_td) {
+		*td = found_td;
+		return MOCK_DECODER_INIT_SAVED;
 	}
 
-	td = cxld_registry_find(cxld);
-	if (td) {
-		bool enabled;
-
-		if (is_switch_decoder(&cxld->dev))
-			enabled = td->cxlsd.cxld.flags & CXL_DECODER_F_ENABLE;
-		else
-			enabled = td->cxled.cxld.flags & CXL_DECODER_F_ENABLE;
-
-		if (enabled)
-			return !cxld_registry_restore(cxld, td);
-
-		init_disabled_mock_decoder(cxld);
-		return false;
-	}
+	*td = NULL;
 
 	/*
 	 * The first decoder on the first 2 devices on the first switch
@@ -1138,15 +1114,119 @@ static bool mock_init_hdm_decoder(struct cxl_decoder *cxld)
 	 * See 'cxl list -BMPu -m cxl_mem.0,cxl_mem.4'
 	 */
 	if (!is_endpoint_decoder(&cxld->dev) || !hb0 || pdev->id % 4 ||
-	    pdev->id > 4 || cxld->id > 0) {
-		default_mock_decoder(cxld);
-		return false;
-	}
+	    pdev->id > 4 || cxld->id > 0)
+		return MOCK_DECODER_INIT_DEFAULT;
+
+	return type2_test ? MOCK_DECODER_INIT_TYPE2_AUTO :
+			    MOCK_DECODER_INIT_TYPE3_AUTO;
+}
+
+static bool mock_decoder_handle_saved(struct cxl_decoder *cxld, struct cxl_test_decoder *td)
+{
+	bool enabled;
+
+	if (is_switch_decoder(&cxld->dev))
+		enabled = td->cxlsd.cxld.flags & CXL_DECODER_F_ENABLE;
+	else
+		enabled = td->cxled.cxld.flags & CXL_DECODER_F_ENABLE;
+
+	if (enabled)
+		return !cxld_registry_restore(cxld, td);
+
+	init_disabled_mock_decoder(cxld);
+	return false;
+}
+
+static void mock_init_hdm_type2_cxled(struct cxl_endpoint_decoder *cxled,
+				      struct cxl_port *port)
+{
+	struct acpi_cedt_cfmws *window = mock_cfmws[0];
+	struct cxl_decoder *cxld = &cxled->cxld;
+	struct cxl_switch_decoder *cxlsd;
+	struct cxl_dport *dport;
+	struct cxl_port *root_port;
+	struct device *dev;
+	u64 base;
+
+	base = window->base_hpa;
+	cxld->hpa_range = (struct range) {
+		.start = base,
+		.end = base + mock_auto_region_size - 1,
+	};
+
+	cxld->interleave_ways = 1;
+	eig_to_granularity(window->granularity, &cxld->interleave_granularity);
+	cxld->target_type = CXL_DECODER_DEVMEM;
+	cxld->flags = CXL_DECODER_F_ENABLE;
+	cxled->state = CXL_DECODER_STATE_AUTO;
+	port->commit_end = cxld->id;
+	devm_cxl_dpa_reserve(cxled, 0,
+			     mock_auto_region_size / cxld->interleave_ways, 0);
+	cxld->commit = mock_decoder_commit;
+	cxld->reset = mock_decoder_reset;
+
+	WARN_ON_ONCE(!cxld_registry_new(cxld));
+	/*
+	 * Now that endpoint decoder is set up, walk up the hierarchy
+	 * and setup the root port decoder targeting @cxlmd.
+	 */
+	dport = port->parent_dport;
+	root_port = dport->port;
+	dev = device_find_child(&root_port->dev, NULL, first_decoder);
+	/*
+	 * Ancestor ports are guaranteed to be enumerated before
+	 * @port, and all ports have at least one decoder.
+	 */
+	if (WARN_ON(!dev))
+		return;
+
+	cxlsd = to_cxl_switch_decoder(dev);
+	cxld = &cxlsd->cxld;
+	cxld->target_type = CXL_DECODER_DEVMEM;
+	cxld->flags = CXL_DECODER_F_ENABLE;
+	root_port->commit_end = 0;
+	cxld->interleave_ways = 1;
+	cxld->interleave_granularity = 4096;
+	cxld->target_map[0] = dport->port_id;
+	cxld->hpa_range = (struct range) {
+		.start = base,
+		.end = base + mock_auto_region_size - 1,
+	};
+	cxld->commit = mock_decoder_commit;
+	cxld->reset = mock_decoder_reset;
+
+	/*
+	 * Only target_map[] is programmed above, mimicking
+	 * firmware. On real hardware target[] is populated as
+	 * dports enumerate, via update_decoder_targets(). The
+	 * mock's dports are already bound by now, so fire that
+	 * resolution explicitly here rather than stamping
+	 * target[] directly.
+	 */
+	cxl_port_update_decoder_targets(root_port, dport);
+
+	cxld_registry_update(cxld);
+	put_device(dev);
+}
+
+static void mock_init_hdm_type3_cxled(struct cxl_endpoint_decoder *cxled,
+				      struct cxl_port *port,
+				      struct platform_device *pdev,
+				      bool hb0)
+{
+	struct acpi_cedt_cfmws *window = mock_cfmws[0];
+	struct cxl_decoder *cxld = &cxled->cxld;
+	struct cxl_switch_decoder *cxlsd;
+	struct cxl_dport *dport;
+	struct cxl_port *iter;
+	struct device *dev;
+	u64 base;
+	int i;
 
 	/* Simulate missing cxl_mem.4 configuration */
 	if (hb0 && pdev->id == 4 && cxld->id == 0 && fail_autoassemble) {
 		default_mock_decoder(cxld);
-		return false;
+		return;
 	}
 
 	base = window->base_hpa;
@@ -1228,7 +1308,76 @@ static bool mock_init_hdm_decoder(struct cxl_decoder *cxld)
 		cxld_registry_update(cxld);
 		put_device(dev);
 	}
+}
+
+/*
+ * Initialize a decoder during HDM enumeration.
+ *
+ * If a saved registry entry exists:
+ *   - enabled decoders are restored from the saved programming
+ *   - disabled decoders are initialized in a clean disabled state
+ *
+ * If no registry entry exists the decoder follows the normal mock
+ * initialization path, including the special auto-region setup for
+ * the first endpoints under host-bridge0.
+ *
+ * Returns true if decoder state was restored from the registry. In
+ * that case the saved decode configuration (including target mapping)
+ * has already been applied and the map_targets() is skipped.
+ */
+static bool mock_init_hdm_decoder(struct cxl_decoder *cxld)
+{
+	struct cxl_endpoint_decoder *cxled = NULL;
+	struct platform_device *pdev = NULL;
+	struct cxl_test_decoder *td;
+	struct cxl_memdev *cxlmd;
+	struct cxl_port *port;
+	bool hb0 = false;
+
+	if (is_endpoint_decoder(&cxld->dev)) {
+		cxled = to_cxl_endpoint_decoder(&cxld->dev);
+		cxlmd = cxled_to_memdev(cxled);
+		WARN_ON(!dev_is_platform(cxlmd->dev.parent));
+		pdev = to_platform_device(cxlmd->dev.parent);
+
+		/* check is endpoint is attach to host-bridge0 */
+		port = cxled_to_port(cxled);
+		do {
+			if (port->uport_dev == &cxl_host_bridge[0]->dev) {
+				hb0 = true;
+				break;
+			}
+			if (is_cxl_port(port->dev.parent))
+				port = to_cxl_port(port->dev.parent);
+			else
+				port = NULL;
+		} while (port);
+		port = cxled_to_port(cxled);
+	} else {
+		port = to_cxl_port(cxld->dev.parent);
+	}
 
+	switch (get_decoder_init_type(cxld, pdev, hb0, &td)) {
+	case MOCK_DECODER_INIT_SAVED:
+		if (WARN_ON(!td))
+			return false;
+		return mock_decoder_handle_saved(cxld, td);
+	case MOCK_DECODER_INIT_DEFAULT:
+		/*
+		 * The default path picks up all the decoders that are not
+		 * endpoint.
+		 */
+		default_mock_decoder(cxld);
+		return false;
+	case MOCK_DECODER_INIT_TYPE3_AUTO:
+		mock_init_hdm_type3_cxled(cxled, port, pdev, hb0);
+		return false;
+	case MOCK_DECODER_INIT_TYPE2_AUTO:
+		mock_init_hdm_type2_cxled(cxled, port);
+		return false;
+	default:
+		return false;
+	}
 	return false;
 }
 
@@ -1563,8 +1712,10 @@ static __init int cxl_rch_topo_init(void)
 		struct platform_device *pdev;
 
 		pdev = platform_device_alloc("cxl_host_bridge", idx);
-		if (!pdev)
+		if (!pdev) {
+			rc = -ENOMEM;
 			goto err_bridge;
+		}
 
 		mock_companion(adev, &pdev->dev);
 		rc = cxl_mock_platform_device_add(pdev, &cxl_rch[i]);
@@ -1618,8 +1769,10 @@ static __init int cxl_single_topo_init(void)
 
 		pdev = platform_device_alloc("cxl_host_bridge",
 					     NR_CXL_HOST_BRIDGES + i);
-		if (!pdev)
+		if (!pdev) {
+			rc = -ENOMEM;
 			goto err_bridge;
+		}
 
 		mock_companion(adev, &pdev->dev);
 		rc = cxl_mock_platform_device_add(pdev, &cxl_hb_single[i]);
@@ -1640,8 +1793,10 @@ static __init int cxl_single_topo_init(void)
 
 		pdev = platform_device_alloc("cxl_root_port",
 					     NR_MULTI_ROOT + i);
-		if (!pdev)
+		if (!pdev) {
+			rc = -ENOMEM;
 			goto err_port;
+		}
 		pdev->dev.parent = &bridge->dev;
 
 		rc = cxl_mock_platform_device_add(pdev, &cxl_root_single[i]);
@@ -1655,8 +1810,10 @@ static __init int cxl_single_topo_init(void)
 
 		pdev = platform_device_alloc("cxl_switch_uport",
 					     NR_MULTI_ROOT + i);
-		if (!pdev)
+		if (!pdev) {
+			rc = -ENOMEM;
 			goto err_uport;
+		}
 		pdev->dev.parent = &root_port->dev;
 
 		rc = cxl_mock_platform_device_add(pdev, &cxl_swu_single[i]);
@@ -1671,8 +1828,10 @@ static __init int cxl_single_topo_init(void)
 
 		pdev = platform_device_alloc("cxl_switch_dport",
 					     i + NR_MEM_MULTI);
-		if (!pdev)
+		if (!pdev) {
+			rc = -ENOMEM;
 			goto err_dport;
+		}
 		pdev->dev.parent = &uport->dev;
 
 		rc = cxl_mock_platform_device_add(pdev, &cxl_swd_single[i]);
@@ -1724,19 +1883,84 @@ static void cxl_single_topo_exit(void)
 	}
 }
 
-static void cxl_mem_exit(void)
+static void cxl_type3_mem_exit(void)
 {
+	struct platform_device *pdev;
 	int i;
 
-	for (i = ARRAY_SIZE(cxl_rcd) - 1; i >= 0; i--)
+	for (i = ARRAY_SIZE(cxl_rcd) - 1; i >= 0; i--) {
+		pdev = cxl_rcd[i];
+		if (!pdev)
+			continue;
 		platform_device_unregister(cxl_rcd[i]);
-	for (i = ARRAY_SIZE(cxl_mem_single) - 1; i >= 0; i--)
+	}
+
+	for (i = ARRAY_SIZE(cxl_mem_single) - 1; i >= 0; i--) {
+		pdev = cxl_mem_single[i];
+		if (!pdev)
+			continue;
 		platform_device_unregister(cxl_mem_single[i]);
-	for (i = ARRAY_SIZE(cxl_mem) - 1; i >= 0; i--)
+	}
+
+	for (i = ARRAY_SIZE(cxl_mem) - 1; i >= 0; i--) {
+		pdev = cxl_mem[i];
+		if (!pdev)
+			continue;
+		platform_device_unregister(pdev);
+	}
+}
+
+static void cxl_type2_mem_exit(void)
+{
+	for (int i = NR_CXL_TYPE2_ACCEL - 1; i >= 0; i--) {
+		struct platform_device *pdev = cxl_mem[i];
+
+		if (!pdev)
+			continue;
+		platform_device_unregister(pdev);
+	}
+}
+
+static void cxl_mem_exit(void)
+{
+	if (type2_test) {
+		cxl_type2_mem_exit();
+		return;
+	}
+
+	cxl_type3_mem_exit();
+}
+
+static int cxl_type2_mem_init(void)
+{
+	int i, rc;
+
+	for (i = 0; i < NR_CXL_TYPE2_ACCEL; i++) {
+		struct platform_device *dport = cxl_root_port[i];
+		struct platform_device *pdev;
+
+		pdev = platform_device_alloc("cxl_type2_accel", i);
+		if (!pdev) {
+			rc = -ENOMEM;
+			goto err_mem;
+		}
+		pdev->dev.parent = &dport->dev;
+		set_dev_node(&pdev->dev, i % 2);
+
+		rc = cxl_mock_platform_device_add(pdev, &cxl_mem[i]);
+		if (rc)
+			goto err_mem;
+	}
+
+	return 0;
+
+err_mem:
+	for (i = NR_CXL_TYPE2_ACCEL - 1; i >= 0; i--)
 		platform_device_unregister(cxl_mem[i]);
+	return rc;
 }
 
-static int cxl_mem_init(void)
+static int cxl_type3_mem_init(void)
 {
 	int i, rc;
 
@@ -1745,8 +1969,10 @@ static int cxl_mem_init(void)
 		struct platform_device *pdev;
 
 		pdev = platform_device_alloc("cxl_mem", i);
-		if (!pdev)
+		if (!pdev) {
+			rc = -ENOMEM;
 			goto err_mem;
+		}
 		pdev->dev.parent = &dport->dev;
 		set_dev_node(&pdev->dev, i % 2);
 
@@ -1760,8 +1986,10 @@ static int cxl_mem_init(void)
 		struct platform_device *pdev;
 
 		pdev = platform_device_alloc("cxl_mem", NR_MEM_MULTI + i);
-		if (!pdev)
+		if (!pdev) {
+			rc = -ENOMEM;
 			goto err_single;
+		}
 		pdev->dev.parent = &dport->dev;
 		set_dev_node(&pdev->dev, i % 2);
 
@@ -1776,8 +2004,10 @@ static int cxl_mem_init(void)
 		struct platform_device *pdev;
 
 		pdev = platform_device_alloc("cxl_rcd", idx);
-		if (!pdev)
+		if (!pdev) {
+			rc = -ENOMEM;
 			goto err_rcd;
+		}
 		pdev->dev.parent = &rch->dev;
 		set_dev_node(&pdev->dev, i % 2);
 
@@ -1800,6 +2030,13 @@ static int cxl_mem_init(void)
 	return rc;
 }
 
+static int cxl_mem_init(void)
+{
+	if (type2_test)
+		return cxl_type2_mem_init();
+	return cxl_type3_mem_init();
+}
+
 static ssize_t
 decoder_reset_preserve_registry_show(struct device *dev,
 				     struct device_attribute *attr, char *buf)
@@ -1827,58 +2064,50 @@ static struct attribute *cxl_acpi_attrs[] = {
 };
 ATTRIBUTE_GROUPS(cxl_acpi);
 
-static __init int cxl_test_init(void)
+static bool __init have_multiple_modparms(void)
 {
-	int rc, i;
-	struct range mappable;
+	int count = 0;
 
-	if (!IS_ALIGNED(mock_auto_region_size, PMD_SIZE)) {
-		pr_err_once("mock_auto_region_size %d must be PMD-aligned\n",
-			    mock_auto_region_size);
-		return -EINVAL;
-	}
+	if (interleave_arithmetic)
+		count++;
+	if (extended_linear_cache)
+		count++;
+	if (hmem_test)
+		count++;
+	if (type2_test)
+		count++;
 
-	cxl_acpi_test();
-	cxl_core_test();
-	cxl_mem_test();
-	cxl_pmem_test();
-	cxl_port_test();
+	return count > 1;
+}
 
-	register_cxl_mock_ops(&cxl_mock_ops);
+static void host_bridges_remove(void)
+{
+	int i;
 
-	cxl_mock_pool = gen_pool_create(ilog2(SZ_2M), NUMA_NO_NODE);
-	if (!cxl_mock_pool) {
-		rc = -ENOMEM;
-		goto err_gen_pool_create;
-	}
-	mappable = mhp_get_pluggable_range(true);
+	for (i = ARRAY_SIZE(cxl_host_bridge) - 1; i >= 0; i--) {
+		struct platform_device *pdev = cxl_host_bridge[i];
 
-	rc = gen_pool_add(cxl_mock_pool,
-			  min(iomem_resource.end + 1 - SZ_64G,
-			      mappable.end + 1 - SZ_64G),
-			  SZ_64G, NUMA_NO_NODE);
-	if (rc)
-		goto err_gen_pool_add;
+		if (!pdev)
+			continue;
 
-	if (interleave_arithmetic == 1) {
-		cfmws_start = CFMWS_XOR_ARRAY_START;
-		cfmws_end = CFMWS_XOR_ARRAY_END;
-	} else {
-		cfmws_start = CFMWS_MOD_ARRAY_START;
-		cfmws_end = CFMWS_MOD_ARRAY_END;
+		sysfs_remove_link(&pdev->dev.kobj, "physical_node");
+		platform_device_unregister(cxl_host_bridge[i]);
 	}
+}
 
-	rc = populate_cedt();
-	if (rc)
-		goto err_populate;
+static int host_bridges_populate(void)
+{
+	int rc = 0;
 
-	for (i = 0; i < ARRAY_SIZE(cxl_host_bridge); i++) {
+	for (int i = 0; i < ARRAY_SIZE(cxl_host_bridge); i++) {
 		struct acpi_device *adev = &host_bridge[i];
 		struct platform_device *pdev;
 
 		pdev = platform_device_alloc("cxl_host_bridge", i);
-		if (!pdev)
+		if (!pdev) {
+			rc = -ENOMEM;
 			goto err_bridge;
+		}
 
 		mock_companion(adev, &pdev->dev);
 		rc = cxl_mock_platform_device_add(pdev, &cxl_host_bridge[i]);
@@ -1892,14 +2121,40 @@ static __init int cxl_test_init(void)
 			goto err_bridge;
 	}
 
-	for (i = 0; i < ARRAY_SIZE(cxl_root_port); i++) {
+	return 0;
+
+err_bridge:
+	host_bridges_remove();
+	return rc;
+}
+
+static void cxl_rootports_remove(void)
+{
+	for (int i = ARRAY_SIZE(cxl_root_port) - 1; i >= 0; i--) {
+		struct platform_device *pdev = cxl_root_port[i];
+
+		if (!pdev)
+			continue;
+
+		platform_device_unregister(pdev);
+	}
+}
+
+static int cxl_rootports_populate(void)
+{
+	int rc = 0;
+
+	for (int i = 0; i < ARRAY_SIZE(cxl_root_port); i++) {
 		struct platform_device *bridge =
 			cxl_host_bridge[i % ARRAY_SIZE(cxl_host_bridge)];
 		struct platform_device *pdev;
 
 		pdev = platform_device_alloc("cxl_root_port", i);
-		if (!pdev)
+		if (!pdev) {
+			rc = -ENOMEM;
 			goto err_port;
+		}
+
 		pdev->dev.parent = &bridge->dev;
 
 		rc = cxl_mock_platform_device_add(pdev, &cxl_root_port[i]);
@@ -1907,14 +2162,39 @@ static __init int cxl_test_init(void)
 			goto err_port;
 	}
 
-	BUILD_BUG_ON(ARRAY_SIZE(cxl_switch_uport) != ARRAY_SIZE(cxl_root_port));
-	for (i = 0; i < ARRAY_SIZE(cxl_switch_uport); i++) {
+	return 0;
+
+err_port:
+	cxl_rootports_remove();
+	return rc;
+}
+
+static void cxl_usps_remove(void)
+{
+	for (int i = ARRAY_SIZE(cxl_switch_uport) - 1; i >= 0; i--) {
+		struct platform_device *pdev = cxl_switch_uport[i];
+
+		if (!pdev)
+			continue;
+
+		platform_device_unregister(cxl_switch_uport[i]);
+	}
+}
+
+static int cxl_usps_populate(void)
+{
+	int rc = 0;
+
+	for (int i = 0; i < ARRAY_SIZE(cxl_switch_uport); i++) {
 		struct platform_device *root_port = cxl_root_port[i];
 		struct platform_device *pdev;
 
 		pdev = platform_device_alloc("cxl_switch_uport", i);
-		if (!pdev)
+		if (!pdev) {
+			rc = -ENOMEM;
 			goto err_uport;
+		}
+
 		pdev->dev.parent = &root_port->dev;
 
 		rc = cxl_mock_platform_device_add(pdev, &cxl_switch_uport[i]);
@@ -1922,14 +2202,40 @@ static __init int cxl_test_init(void)
 			goto err_uport;
 	}
 
-	for (i = 0; i < ARRAY_SIZE(cxl_switch_dport); i++) {
+	return 0;
+
+err_uport:
+	cxl_usps_remove();
+	return rc;
+}
+
+static void cxl_dsps_remove(void)
+{
+	for (int i = ARRAY_SIZE(cxl_switch_dport) - 1; i >= 0; i--) {
+		struct platform_device *pdev = cxl_switch_dport[i];
+
+		if (!pdev)
+			continue;
+
+		platform_device_unregister(cxl_switch_dport[i]);
+	}
+}
+
+
+static int cxl_dsps_populate(void)
+{
+	int rc = 0;
+
+	for (int i = 0; i < ARRAY_SIZE(cxl_switch_dport); i++) {
 		struct platform_device *uport =
 			cxl_switch_uport[i % ARRAY_SIZE(cxl_switch_uport)];
 		struct platform_device *pdev;
 
 		pdev = platform_device_alloc("cxl_switch_dport", i);
-		if (!pdev)
+		if (!pdev) {
+			rc = -ENOMEM;
 			goto err_dport;
+		}
 		pdev->dev.parent = &uport->dev;
 
 		rc = cxl_mock_platform_device_add(pdev, &cxl_switch_dport[i]);
@@ -1937,17 +2243,181 @@ static __init int cxl_test_init(void)
 			goto err_dport;
 	}
 
+	return 0;
+
+err_dport:
+	cxl_dsps_remove();
+	return rc;
+}
+
+static void cxl_switches_remove(void)
+{
+	cxl_dsps_remove();
+	cxl_usps_remove();
+}
+
+static int cxl_switches_populate(void)
+{
+	int rc;
+
+	BUILD_BUG_ON(ARRAY_SIZE(cxl_switch_uport) != ARRAY_SIZE(cxl_root_port));
+	rc = cxl_usps_populate();
+	if (rc)
+		return rc;
+
+	rc = cxl_dsps_populate();
+	if (rc) {
+		cxl_usps_remove();
+		return rc;
+	}
+
+	return 0;
+}
+
+static void cxl_type2_topo_exit(void)
+{
+	cxl_rootports_remove();
+	host_bridges_remove();
+}
+
+static int cxl_type2_topo_init(void)
+{
+	int rc;
+
+	rc = host_bridges_populate();
+	if (rc)
+		return rc;
+
+	rc = cxl_rootports_populate();
+	if (rc) {
+		host_bridges_remove();
+		return rc;
+	}
+
+	return 0;
+}
+
+static void cxl_type3_topo_exit(void)
+{
+	cxl_rch_topo_exit();
+	cxl_single_topo_exit();
+	cxl_switches_remove();
+	cxl_rootports_remove();
+	host_bridges_remove();
+}
+
+static int cxl_type3_topo_init(void)
+{
+	int rc;
+
+	rc = host_bridges_populate();
+	if (rc)
+		return rc;
+
+	rc = cxl_rootports_populate();
+	if (rc)
+		goto err_host_bridges;
+
+	rc = cxl_switches_populate();
+	if (rc)
+		goto err_root_ports;
+
 	rc = cxl_single_topo_init();
 	if (rc)
-		goto err_dport;
+		goto err_switches;
 
 	rc = cxl_rch_topo_init();
 	if (rc)
 		goto err_single;
 
+	return 0;
+
+err_single:
+	cxl_single_topo_exit();
+err_switches:
+	cxl_switches_remove();
+err_root_ports:
+	cxl_rootports_remove();
+err_host_bridges:
+	host_bridges_remove();
+	return rc;
+}
+
+static void cxl_topo_exit(void)
+{
+	if (type2_test) {
+		cxl_type2_topo_exit();
+		return;
+	}
+
+	cxl_type3_topo_exit();
+}
+
+static int cxl_topo_init(void)
+{
+	if (type2_test)
+		return cxl_type2_topo_init();
+	return cxl_type3_topo_init();
+}
+
+static __init int cxl_test_init(void)
+{
+	struct range mappable;
+	int rc;
+
+	/* Enforce a single module param active at a time */
+	if (have_multiple_modparms())
+		return -EINVAL;
+
+	if (!IS_ALIGNED(mock_auto_region_size, PMD_SIZE)) {
+		pr_err_once("mock_auto_region_size %d must be PMD-aligned\n",
+			    mock_auto_region_size);
+		return -EINVAL;
+	}
+
+	cxl_acpi_test();
+	cxl_core_test();
+	cxl_mem_test();
+	cxl_pmem_test();
+	cxl_port_test();
+
+	register_cxl_mock_ops(&cxl_mock_ops);
+
+	cxl_mock_pool = gen_pool_create(ilog2(SZ_2M), NUMA_NO_NODE);
+	if (!cxl_mock_pool) {
+		rc = -ENOMEM;
+		goto err_gen_pool_create;
+	}
+	mappable = mhp_get_pluggable_range(true);
+
+	rc = gen_pool_add(cxl_mock_pool,
+			  min(iomem_resource.end + 1 - SZ_64G,
+			      mappable.end + 1 - SZ_64G),
+			  SZ_64G, NUMA_NO_NODE);
+	if (rc)
+		goto err_gen_pool_add;
+
+	if (interleave_arithmetic == 1) {
+		cfmws_start = CFMWS_XOR_ARRAY_START;
+		cfmws_end = CFMWS_XOR_ARRAY_END;
+	} else {
+		cfmws_start = CFMWS_MOD_ARRAY_START;
+		cfmws_end = CFMWS_MOD_ARRAY_END;
+	}
+
+	rc = populate_cedt();
+	if (rc)
+		goto err_populate;
+
+	rc = cxl_topo_init();
+	if (rc)
+		goto err_populate;
+
 	cxl_acpi = platform_device_alloc("cxl_acpi", 0);
-	if (!cxl_acpi)
-		goto err_rch;
+	if (!cxl_acpi) {
+		rc = -ENOMEM;
+		goto err_topo;
+	}
 
 	mock_companion(&acpi0017_mock, &cxl_acpi->dev);
 	acpi0017_mock.dev.bus = &platform_bus_type;
@@ -1955,7 +2425,7 @@ static __init int cxl_test_init(void)
 
 	rc = cxl_mock_platform_device_add(cxl_acpi, NULL);
 	if (rc)
-		goto err_rch;
+		goto err_topo;
 
 	rc = cxl_mem_init();
 	if (rc)
@@ -1971,28 +2441,8 @@ static __init int cxl_test_init(void)
 	cxl_mem_exit();
 err_root:
 	platform_device_unregister(cxl_acpi);
-err_rch:
-	cxl_rch_topo_exit();
-err_single:
-	cxl_single_topo_exit();
-err_dport:
-	for (i = ARRAY_SIZE(cxl_switch_dport) - 1; i >= 0; i--)
-		platform_device_unregister(cxl_switch_dport[i]);
-err_uport:
-	for (i = ARRAY_SIZE(cxl_switch_uport) - 1; i >= 0; i--)
-		platform_device_unregister(cxl_switch_uport[i]);
-err_port:
-	for (i = ARRAY_SIZE(cxl_root_port) - 1; i >= 0; i--)
-		platform_device_unregister(cxl_root_port[i]);
-err_bridge:
-	for (i = ARRAY_SIZE(cxl_host_bridge) - 1; i >= 0; i--) {
-		struct platform_device *pdev = cxl_host_bridge[i];
-
-		if (!pdev)
-			continue;
-		sysfs_remove_link(&pdev->dev.kobj, "physical_node");
-		platform_device_unregister(cxl_host_bridge[i]);
-	}
+err_topo:
+	cxl_topo_exit();
 err_populate:
 	depopulate_all_mock_resources();
 err_gen_pool_add:
@@ -2015,27 +2465,10 @@ static void free_decoder_registry(void)
 
 static __exit void cxl_test_exit(void)
 {
-	int i;
-
 	hmem_test_exit();
 	cxl_mem_exit();
 	platform_device_unregister(cxl_acpi);
-	cxl_rch_topo_exit();
-	cxl_single_topo_exit();
-	for (i = ARRAY_SIZE(cxl_switch_dport) - 1; i >= 0; i--)
-		platform_device_unregister(cxl_switch_dport[i]);
-	for (i = ARRAY_SIZE(cxl_switch_uport) - 1; i >= 0; i--)
-		platform_device_unregister(cxl_switch_uport[i]);
-	for (i = ARRAY_SIZE(cxl_root_port) - 1; i >= 0; i--)
-		platform_device_unregister(cxl_root_port[i]);
-	for (i = ARRAY_SIZE(cxl_host_bridge) - 1; i >= 0; i--) {
-		struct platform_device *pdev = cxl_host_bridge[i];
-
-		if (!pdev)
-			continue;
-		sysfs_remove_link(&pdev->dev.kobj, "physical_node");
-		platform_device_unregister(cxl_host_bridge[i]);
-	}
+	cxl_topo_exit();
 	depopulate_all_mock_resources();
 	gen_pool_destroy(cxl_mock_pool);
 	unregister_cxl_mock_ops(&cxl_mock_ops);
@@ -2049,6 +2482,8 @@ module_param(extended_linear_cache, bool, 0444);
 MODULE_PARM_DESC(extended_linear_cache, "Enable extended linear cache support");
 module_param(fail_autoassemble, bool, 0444);
 MODULE_PARM_DESC(fail_autoassemble, "Simulate missing member of an auto-region");
+module_param(type2_test, bool, 0444);
+MODULE_PARM_DESC(type2_test, "Enable type 2 support testing");
 module_init(cxl_test_init);
 module_exit(cxl_test_exit);
 MODULE_LICENSE("GPL v2");
diff --git a/tools/testing/cxl/test/hmem_test.c b/tools/testing/cxl/test/hmem_test.c
index 3a1a089e1721b..0fa00f7e16db5 100644
--- a/tools/testing/cxl/test/hmem_test.c
+++ b/tools/testing/cxl/test/hmem_test.c
@@ -3,8 +3,9 @@
 #include <linux/moduleparam.h>
 #include <linux/workqueue.h>
 #include "../../../drivers/dax/bus.h"
+#include "mock.h"
 
-static bool hmem_test;
+bool hmem_test;
 
 static void hmem_test_work(struct work_struct *work)
 {
diff --git a/tools/testing/cxl/test/mock.h b/tools/testing/cxl/test/mock.h
index 4f57dc80ae7d5..846d7c5d6eaa9 100644
--- a/tools/testing/cxl/test/mock.h
+++ b/tools/testing/cxl/test/mock.h
@@ -5,6 +5,8 @@
 #include <linux/dax.h>
 #include <cxl.h>
 
+extern bool hmem_test;
+
 struct cxl_mock_ops {
 	struct list_head list;
 	bool (*is_mock_adev)(struct acpi_device *dev);
diff --git a/tools/testing/kunit/configs/all_tests.config b/tools/testing/kunit/configs/all_tests.config
index bccc2c77196d5..6825c2e855a55 100644
--- a/tools/testing/kunit/configs/all_tests.config
+++ b/tools/testing/kunit/configs/all_tests.config
@@ -21,6 +21,7 @@ CONFIG_VFAT_FS=y
 CONFIG_PCI=y
 CONFIG_USB4=y
 CONFIG_I2C=y
+CONFIG_GPIOLIB=y
 
 CONFIG_NET=y
 CONFIG_MCTP=y
diff --git a/tools/testing/memblock/internal.h b/tools/testing/memblock/internal.h
index b6b1d147fd750..4168e646af098 100644
--- a/tools/testing/memblock/internal.h
+++ b/tools/testing/memblock/internal.h
@@ -28,22 +28,6 @@ static inline struct page *virt_to_page(void *virt)
 	return virt;
 }
 
-void memblock_free_pages(unsigned long pfn, unsigned int order)
-{
-}
-
-static inline void accept_memory(phys_addr_t start, unsigned long size)
-{
-}
-
-unsigned long free_reserved_area(void *start, void *end, int poison, const char *s);
-void free_reserved_page(struct page *page);
-
-static inline bool deferred_pages_enabled(void)
-{
-	return false;
-}
-
 #define for_each_valid_pfn(pfn, start_pfn, end_pfn)			 \
 	for ((pfn) = (start_pfn); (pfn) < (end_pfn); (pfn)++)
 
@@ -60,10 +44,6 @@ static inline bool __is_kernel(unsigned long addr)
 #define for_each_valid_pfn(pfn, start_pfn, end_pfn)                     \
        for ((pfn) = (start_pfn); (pfn) < (end_pfn); (pfn)++)
 
-static inline void init_deferred_page(unsigned long pfn, int nid)
-{
-}
-
 #define __SetPageReserved(p)	((void)(p))
 
 #endif
diff --git a/tools/testing/memblock/mm_init.h b/tools/testing/memblock/mm_init.h
new file mode 100644
index 0000000000000..95bc5f2e8ed82
--- /dev/null
+++ b/tools/testing/memblock/mm_init.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef __MM_MM_INIT_H
+#define __MM_MM_INIT_H
+
+void memblock_free_pages(unsigned long pfn, unsigned int order)
+{
+}
+
+static inline void accept_memory(phys_addr_t start, unsigned long size)
+{
+}
+
+unsigned long free_reserved_area(void *start, void *end, int poison, const char *s);
+void free_reserved_page(struct page *page);
+
+static inline bool deferred_pages_enabled(void)
+{
+	return false;
+}
+
+static inline void init_deferred_page(unsigned long pfn, int nid)
+{
+}
+#endif /* __MM_MM_INIT_H */
diff --git a/tools/testing/nvdimm/test/ndtest.c b/tools/testing/nvdimm/test/ndtest.c
index 8e3b6be53839b..2051ad5d4882d 100644
--- a/tools/testing/nvdimm/test/ndtest.c
+++ b/tools/testing/nvdimm/test/ndtest.c
@@ -376,8 +376,7 @@ static void *ndtest_alloc_resource(struct ndtest_priv *p, size_t size,
 buf_err:
 	if (__dma && size >= DIMM_SIZE)
 		gen_pool_free(ndtest_pool, __dma, size);
-	if (buf)
-		vfree(buf);
+	vfree(buf);
 	kfree(res);
 
 	return NULL;
diff --git a/tools/testing/nvdimm/test/nfit.c b/tools/testing/nvdimm/test/nfit.c
index f87e9f251d131..009fe107b0d74 100644
--- a/tools/testing/nvdimm/test/nfit.c
+++ b/tools/testing/nvdimm/test/nfit.c
@@ -1644,8 +1644,7 @@ static void *__test_alloc(struct nfit_test *t, size_t size, dma_addr_t *dma,
  err:
 	if (*dma && size >= DIMM_SIZE)
 		gen_pool_free(nfit_pool, *dma, size);
-	if (buf)
-		vfree(buf);
+	vfree(buf);
 	kfree(nfit_res);
 	return NULL;
 }
diff --git a/tools/testing/radix-tree/maple.c b/tools/testing/radix-tree/maple.c
index 0607913a3022d..d967e76a3c065 100644
--- a/tools/testing/radix-tree/maple.c
+++ b/tools/testing/radix-tree/maple.c
@@ -35234,7 +35234,7 @@ static noinline void __init check_prealloc(struct maple_tree *mt)
 	mt_set_non_kernel(1);
 	/* Spanning store */
 	mas_set_range(&mas, 1, 100);
-	MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0);
+	MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_NOWAIT) == 0);
 	allocated = mas_allocated(&mas);
 	height = mas_mt_height(&mas);
 	MT_BUG_ON(mt, allocated != 0);
@@ -35257,7 +35257,7 @@ static noinline void __init check_prealloc(struct maple_tree *mt)
 	MT_BUG_ON(mt, mas_allocated(&mas) != 0);
 	mas_set_range(&mas, 0, 200);
 	mt_set_non_kernel(1);
-	MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0);
+	MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_NOWAIT) == 0);
 	allocated = mas_allocated(&mas);
 	height = mas_mt_height(&mas);
 	MT_BUG_ON(mt, allocated != 0);
diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
index 8d4db2241cc2a..74183c27f6e3b 100644
--- a/tools/testing/selftests/Makefile
+++ b/tools/testing/selftests/Makefile
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 TARGETS += acct
+TARGETS += alloc_tag
 TARGETS += alsa
 TARGETS += amd-pstate
 TARGETS += arm64
@@ -14,6 +15,7 @@ TARGETS += core
 TARGETS += cpufreq
 TARGETS += cpu-hotplug
 TARGETS += damon
+TARGETS += dax
 TARGETS += devices/error_logs
 TARGETS += devices/probe
 TARGETS += dmabuf-heaps
@@ -32,6 +34,7 @@ TARGETS += exec
 TARGETS += fchmodat2
 TARGETS += filesystems
 TARGETS += filesystems/binderfs
+TARGETS += filesystems/ceph
 TARGETS += filesystems/epoll
 TARGETS += filesystems/fat
 TARGETS += filesystems/overlayfs
diff --git a/tools/testing/selftests/acct/.gitignore b/tools/testing/selftests/acct/.gitignore
index 9e9c61c5bfd6f..fe0896f54e152 100644
--- a/tools/testing/selftests/acct/.gitignore
+++ b/tools/testing/selftests/acct/.gitignore
@@ -1,4 +1,5 @@
 acct_syscall
 taskstats_fill_stats_tgid
+cgroupstats
 config
 process_log
diff --git a/tools/testing/selftests/acct/Makefile b/tools/testing/selftests/acct/Makefile
index 083cab5ddb72c..93a11a28a6361 100644
--- a/tools/testing/selftests/acct/Makefile
+++ b/tools/testing/selftests/acct/Makefile
@@ -1,8 +1,19 @@
 # SPDX-License-Identifier: GPL-2.0
 TEST_GEN_PROGS := acct_syscall
 TEST_GEN_PROGS += taskstats_fill_stats_tgid
+TEST_GEN_PROGS += cgroupstats
+
+NETLINK_HELPER_PROGS := cgroupstats taskstats_fill_stats_tgid
 
 CFLAGS += -Wall
 LDLIBS += -lpthread
 
 include ../lib.mk
+
+$(NETLINK_HELPER_PROGS): %: %.c netlink_helper.c netlink_helper.h
+	$(call msg,CC,,$@)
+	$(Q)$(LINK.c) $< netlink_helper.c $(LDLIBS) -o $@
+
+$(addprefix $(OUTPUT)/,$(NETLINK_HELPER_PROGS)): $(OUTPUT)/%: %.c netlink_helper.c netlink_helper.h
+	$(call msg,CC,,$@)
+	$(Q)$(LINK.c) $< netlink_helper.c $(LDLIBS) -o $@
diff --git a/tools/testing/selftests/acct/cgroupstats.c b/tools/testing/selftests/acct/cgroupstats.c
new file mode 100644
index 0000000000000..0b421a4ca72b3
--- /dev/null
+++ b/tools/testing/selftests/acct/cgroupstats.c
@@ -0,0 +1,231 @@
+// SPDX-License-Identifier: GPL-2.0
+#define _GNU_SOURCE
+
+#include <errno.h>
+#include <fcntl.h>
+#include <linux/cgroupstats.h>
+#include <linux/genetlink.h>
+#include <linux/netlink.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mount.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "netlink_helper.h"
+#include "kselftest.h"
+
+static int send_cgroupstats_cmd(int fd, int family_id, uint32_t cgroup_fd,
+				int flags)
+{
+	struct {
+		struct nlmsghdr nlh;
+		struct genlmsghdr genl;
+		char buf[256];
+	} req = { 0 };
+	struct nlattr *na;
+
+	req.nlh.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN);
+	req.nlh.nlmsg_type = family_id;
+	req.nlh.nlmsg_flags = NLM_F_REQUEST | flags;
+	req.nlh.nlmsg_seq = 2;
+	req.nlh.nlmsg_pid = getpid();
+
+	req.genl.cmd = CGROUPSTATS_CMD_GET;
+	req.genl.version = 1;
+
+	na = (struct nlattr *)((char *)&req + NLMSG_ALIGN(req.nlh.nlmsg_len));
+	na->nla_type = CGROUPSTATS_CMD_ATTR_FD;
+	na->nla_len = NLA_HDRLEN + sizeof(cgroup_fd);
+	memcpy(nla_data(na), &cgroup_fd, sizeof(cgroup_fd));
+	req.nlh.nlmsg_len = NLMSG_ALIGN(req.nlh.nlmsg_len) + NLA_ALIGN(na->nla_len);
+
+	return send_request(fd, &req, req.nlh.nlmsg_len);
+}
+
+/*
+ * Receive and decode a cgroupstats response.
+ *
+ * Returns:
+ *   0             — success, stats filled from CGROUPSTATS_CMD_NEW reply
+ *   <0            — NLMSG_ERROR errno (e.g. -EBADF, -EINVAL)
+ */
+static int recv_cgroupstats_response(int fd, struct cgroupstats *stats)
+{
+	char resp[8192];
+	struct nlmsghdr *nlh;
+	struct genlmsghdr *genl;
+	struct nlattr *na;
+	int len;
+	int rem;
+
+	memset(stats, 0, sizeof(*stats));
+
+	len = recv(fd, resp, sizeof(resp), 0);
+	if (len < 0)
+		return -errno;
+
+	for (nlh = (struct nlmsghdr *)resp; NLMSG_OK(nlh, len);
+	     nlh = NLMSG_NEXT(nlh, len)) {
+		if (nlh->nlmsg_type == NLMSG_ERROR) {
+			struct nlmsgerr *err = NLMSG_DATA(nlh);
+
+			return err->error;
+		}
+
+		genl = (struct genlmsghdr *)NLMSG_DATA(nlh);
+		if (genl->cmd != CGROUPSTATS_CMD_NEW)
+			continue;
+
+		rem = nlh->nlmsg_len - NLMSG_HDRLEN - GENL_HDRLEN;
+		na = (struct nlattr *)((char *)genl + GENL_HDRLEN);
+		while (nla_ok(na, rem)) {
+			if (na->nla_type == CGROUPSTATS_TYPE_CGROUP_STATS) {
+				memcpy(stats, nla_data(na), sizeof(*stats));
+				return 0;
+			}
+			na = nla_next(na, &rem);
+		}
+	}
+
+	return -EIO;
+}
+
+/* mkdtemp() modifies the template in place, so this cannot be const. */
+static char cg_mountpoint[32];
+static bool cg_mounted;
+
+static int setup_cgroup_v1(void)
+{
+	strcpy(cg_mountpoint, "/tmp/cgstats_test_XXXXXX");
+
+	if (!mkdtemp(cg_mountpoint))
+		return -errno;
+
+	if (mount("cgstats_test", cg_mountpoint, "cgroup", 0,
+		  "none,name=cgstats_test") < 0) {
+		int ret = -errno;
+
+		rmdir(cg_mountpoint);
+		return ret;
+	}
+
+	cg_mounted = true;
+	return 0;
+}
+
+static void cleanup_cgroup_v1(void)
+{
+	if (!cg_mounted)
+		return;
+	umount2(cg_mountpoint, MNT_DETACH);
+	rmdir(cg_mountpoint);
+	cg_mounted = false;
+}
+
+int main(void)
+{
+	struct cgroupstats stats;
+	uint64_t total_tasks;
+	int family_id;
+	int nl_fd;
+	int cg_fd;
+	int ret;
+
+	ksft_print_header();
+
+	nl_fd = netlink_open();
+	if (nl_fd < 0)
+		ksft_exit_skip("failed to open generic netlink socket: %s\n",
+			       strerror(-nl_fd));
+
+	family_id = get_family_id(nl_fd, TASKSTATS_GENL_NAME);
+	if (family_id < 0)
+		ksft_exit_skip("taskstats generic netlink family unavailable: %s\n",
+			       strerror(-family_id));
+
+	ksft_set_plan(3);
+
+	/*
+	 * Test 1: mount a private cgroup v1 hierarchy, query it, and
+	 * verify the response contains sane task counts. If the test
+	 * environment cannot create a private cgroup v1 mount, skip this
+	 * case and continue with the unprivileged regression checks below.
+	 */
+	ret = setup_cgroup_v1();
+	if (ret) {
+		ksft_test_result_skip("cgroupstats query: cannot mount cgroup v1: %s\n",
+				      strerror(-ret));
+	} else {
+		cg_fd = open(cg_mountpoint, O_RDONLY | O_DIRECTORY);
+		if (cg_fd < 0) {
+			ksft_test_result_fail("cgroupstats query: open mountpoint: %s\n",
+					      strerror(errno));
+		} else {
+			ret = send_cgroupstats_cmd(nl_fd, family_id,
+						   (uint32_t)cg_fd, 0);
+			if (ret) {
+				ksft_test_result_fail("cgroupstats query: send: %s\n",
+						      strerror(-ret));
+			} else {
+				ret = recv_cgroupstats_response(nl_fd, &stats);
+				if (ret < 0) {
+					ksft_test_result_fail("cgroupstats query: %s\n",
+							      strerror(-ret));
+				} else {
+					total_tasks = (uint64_t)stats.nr_sleeping +
+						      (uint64_t)stats.nr_running +
+						      (uint64_t)stats.nr_stopped +
+						      (uint64_t)stats.nr_uninterruptible +
+						      (uint64_t)stats.nr_io_wait;
+
+					ksft_print_msg("cgroupstats query: total_tasks=%llu\n",
+						       (unsigned long long)total_tasks);
+
+					ksft_test_result(total_tasks > 0,
+							 "cgroupstats query returns valid stats\n");
+				}
+			}
+			close(cg_fd);
+		}
+	}
+	cleanup_cgroup_v1();
+
+	/*
+	 * Test 2: invalid fd without NLM_F_ACK.  The kernel should
+	 * return -EBADF via NLMSG_ERROR regardless of whether the
+	 * client requested an explicit ACK.
+	 */
+	ret = send_cgroupstats_cmd(nl_fd, family_id, 0xFFFFFFFF, 0);
+	if (ret)
+		ksft_exit_fail_msg("send test 2 failed: %s\n", strerror(-ret));
+
+	ret = recv_cgroupstats_response(nl_fd, &stats);
+	ksft_print_msg("bad fd (no ACK): response=%d (%s)\n",
+		       ret, ret < 0 ? strerror(-ret) : "unexpected success");
+	ksft_test_result(ret == -EBADF,
+			 "cgroupstats rejects bad fd without NLM_F_ACK\n");
+
+	/*
+	 * Test 3: invalid fd with NLM_F_ACK.  Same expectation as
+	 * test 2, but exercised through a different netlink flag
+	 * path in the kernel's ack/error handling.
+	 */
+	ret = send_cgroupstats_cmd(nl_fd, family_id, 0xFFFFFFFF, NLM_F_ACK);
+	if (ret)
+		ksft_exit_fail_msg("send test 3 failed: %s\n", strerror(-ret));
+
+	ret = recv_cgroupstats_response(nl_fd, &stats);
+	ksft_print_msg("bad fd (with ACK): response=%d (%s)\n",
+		       ret, ret < 0 ? strerror(-ret) : "unexpected success");
+	ksft_test_result(ret == -EBADF,
+			 "cgroupstats rejects bad fd with NLM_F_ACK\n");
+
+	close(nl_fd);
+	ksft_finished();
+	return ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS;
+}
diff --git a/tools/testing/selftests/acct/netlink_helper.c b/tools/testing/selftests/acct/netlink_helper.c
new file mode 100644
index 0000000000000..3ed834f0e7707
--- /dev/null
+++ b/tools/testing/selftests/acct/netlink_helper.c
@@ -0,0 +1,116 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <errno.h>
+#include <stdint.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <unistd.h>
+#include <linux/genetlink.h>
+
+#include "netlink_helper.h"
+
+int netlink_open(void)
+{
+	struct timeval tv = { .tv_sec = ACCT_RCV_TIMEOUT_SEC };
+	struct sockaddr_nl addr = {
+		.nl_family = AF_NETLINK,
+		.nl_pid = getpid(),
+	};
+	int fd;
+
+	fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_GENERIC);
+	if (fd < 0)
+		return -errno;
+
+	if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) < 0) {
+		int err = -errno;
+
+		close(fd);
+		return err;
+	}
+
+	if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
+		int err = -errno;
+
+		close(fd);
+		return err;
+	}
+
+	return fd;
+}
+
+int send_request(int fd, void *buf, size_t len)
+{
+	struct sockaddr_nl addr = {
+		.nl_family = AF_NETLINK,
+	};
+
+	if (sendto(fd, buf, len, 0, (struct sockaddr *)&addr, sizeof(addr)) < 0)
+		return -errno;
+
+	return 0;
+}
+
+/*
+ * Resolve the generic netlink family ID for @name.
+ * Returns the family ID (>= 0) on success, negative errno on failure.
+ */
+int get_family_id(int fd, const char *name)
+{
+	struct {
+		struct nlmsghdr nlh;
+		struct genlmsghdr genl;
+		char buf[256];
+	} req = { 0 };
+	char resp[8192];
+	struct nlmsghdr *nlh;
+	struct genlmsghdr *genl;
+	struct nlattr *na;
+	int len;
+	int rem;
+	int ret;
+
+	req.nlh.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN);
+	req.nlh.nlmsg_type = GENL_ID_CTRL;
+	req.nlh.nlmsg_flags = NLM_F_REQUEST;
+	req.nlh.nlmsg_seq = 1;
+	req.nlh.nlmsg_pid = getpid();
+
+	req.genl.cmd = CTRL_CMD_GETFAMILY;
+	req.genl.version = 1;
+
+	na = (struct nlattr *)((char *)&req + NLMSG_ALIGN(req.nlh.nlmsg_len));
+	na->nla_type = CTRL_ATTR_FAMILY_NAME;
+	na->nla_len = NLA_HDRLEN + strlen(name) + 1;
+	memcpy(nla_data(na), name, strlen(name) + 1);
+	req.nlh.nlmsg_len = NLMSG_ALIGN(req.nlh.nlmsg_len) + NLA_ALIGN(na->nla_len);
+
+	ret = send_request(fd, &req, req.nlh.nlmsg_len);
+	if (ret)
+		return ret;
+
+	len = recv(fd, resp, sizeof(resp), 0);
+	if (len < 0)
+		return -errno;
+
+	for (nlh = (struct nlmsghdr *)resp; NLMSG_OK(nlh, len);
+	     nlh = NLMSG_NEXT(nlh, len)) {
+		if (nlh->nlmsg_type == NLMSG_ERROR) {
+			struct nlmsgerr *err = NLMSG_DATA(nlh);
+
+			return err->error ? err->error : -ENOENT;
+		}
+
+		genl = (struct genlmsghdr *)NLMSG_DATA(nlh);
+		rem = nlh->nlmsg_len - NLMSG_HDRLEN - GENL_HDRLEN;
+		na = (struct nlattr *)((char *)genl + GENL_HDRLEN);
+		while (nla_ok(na, rem)) {
+			if (na->nla_type == CTRL_ATTR_FAMILY_ID)
+				return *(uint16_t *)nla_data(na);
+			na = nla_next(na, &rem);
+		}
+	}
+
+	return -ENOENT;
+}
diff --git a/tools/testing/selftests/acct/netlink_helper.h b/tools/testing/selftests/acct/netlink_helper.h
new file mode 100644
index 0000000000000..0320729c4c06f
--- /dev/null
+++ b/tools/testing/selftests/acct/netlink_helper.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Shared generic netlink helpers for the acct selftests.
+ */
+#ifndef ACSELFTESTS_ACCT_NETLINK_HELPER_H
+#define ACSELFTESTS_ACCT_NETLINK_HELPER_H
+
+#include <stdbool.h>
+#include <linux/netlink.h>
+
+#ifndef NLA_ALIGNTO
+#define NLA_ALIGNTO 4
+#define NLA_ALIGN(len) (((len) + NLA_ALIGNTO - 1) & ~(NLA_ALIGNTO - 1))
+#define NLA_HDRLEN ((int)NLA_ALIGN(sizeof(struct nlattr)))
+#endif
+
+/* Fail an individual test case instead of hanging the whole binary. */
+#define ACCT_RCV_TIMEOUT_SEC 2
+
+static inline void *nla_data(const struct nlattr *na)
+{
+	return (void *)((char *)na + NLA_HDRLEN);
+}
+
+static inline bool nla_ok(const struct nlattr *na, int remaining)
+{
+	return remaining >= (int)sizeof(*na) &&
+	       na->nla_len >= sizeof(*na) &&
+	       na->nla_len <= remaining;
+}
+
+static inline struct nlattr *nla_next(const struct nlattr *na, int *remaining)
+{
+	int aligned_len = NLA_ALIGN(na->nla_len);
+
+	*remaining -= aligned_len;
+	return (struct nlattr *)((char *)na + aligned_len);
+}
+
+int netlink_open(void);
+int send_request(int fd, void *buf, size_t len);
+int get_family_id(int fd, const char *name);
+
+#endif /* ACSELFTESTS_ACCT_NETLINK_HELPER_H */
diff --git a/tools/testing/selftests/acct/taskstats_fill_stats_tgid.c b/tools/testing/selftests/acct/taskstats_fill_stats_tgid.c
index d6cab4ae26f28..9a4c1554dee36 100644
--- a/tools/testing/selftests/acct/taskstats_fill_stats_tgid.c
+++ b/tools/testing/selftests/acct/taskstats_fill_stats_tgid.c
@@ -16,14 +16,9 @@
 #include <time.h>
 #include <unistd.h>
 
+#include "netlink_helper.h"
 #include "kselftest.h"
 
-#ifndef NLA_ALIGN
-#define NLA_ALIGNTO 4
-#define NLA_ALIGN(len) (((len) + NLA_ALIGNTO - 1) & ~(NLA_ALIGNTO - 1))
-#define NLA_HDRLEN ((int)NLA_ALIGN(sizeof(struct nlattr)))
-#endif
-
 #define BUSY_NS (200ULL * 1000 * 1000)
 
 struct worker_ctx {
@@ -35,26 +30,6 @@ struct worker_ctx {
 
 static unsigned long busy_sink;
 
-static void *taskstats_nla_data(const struct nlattr *na)
-{
-	return (void *)((char *)na + NLA_HDRLEN);
-}
-
-static bool taskstats_nla_ok(const struct nlattr *na, int remaining)
-{
-	return remaining >= (int)sizeof(*na) &&
-	       na->nla_len >= sizeof(*na) &&
-	       na->nla_len <= remaining;
-}
-
-static struct nlattr *taskstats_nla_next(const struct nlattr *na, int *remaining)
-{
-	int aligned_len = NLA_ALIGN(na->nla_len);
-
-	*remaining -= aligned_len;
-	return (struct nlattr *)((char *)na + aligned_len);
-}
-
 static uint64_t timespec_diff_ns(const struct timespec *start,
 				 const struct timespec *end)
 {
@@ -84,99 +59,6 @@ static void burn_cpu_for_ns(uint64_t runtime_ns)
 	busy_sink = acc;
 }
 
-static int netlink_open(void)
-{
-	struct sockaddr_nl addr = {
-		.nl_family = AF_NETLINK,
-		.nl_pid = getpid(),
-	};
-	int fd;
-
-	fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_GENERIC);
-	if (fd < 0)
-		return -errno;
-
-	if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
-		int err = -errno;
-
-		close(fd);
-		return err;
-	}
-
-	return fd;
-}
-
-static int send_request(int fd, void *buf, size_t len)
-{
-	struct sockaddr_nl addr = {
-		.nl_family = AF_NETLINK,
-	};
-
-	if (sendto(fd, buf, len, 0, (struct sockaddr *)&addr, sizeof(addr)) < 0)
-		return -errno;
-
-	return 0;
-}
-
-static int get_family_id(int fd, const char *name)
-{
-	struct {
-		struct nlmsghdr nlh;
-		struct genlmsghdr genl;
-		char buf[256];
-	} req = { 0 };
-	char resp[8192];
-	struct nlmsghdr *nlh;
-	struct genlmsghdr *genl;
-	struct nlattr *na;
-	int len;
-	int rem;
-	int ret;
-
-	req.nlh.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN);
-	req.nlh.nlmsg_type = GENL_ID_CTRL;
-	req.nlh.nlmsg_flags = NLM_F_REQUEST;
-	req.nlh.nlmsg_seq = 1;
-	req.nlh.nlmsg_pid = getpid();
-
-	req.genl.cmd = CTRL_CMD_GETFAMILY;
-	req.genl.version = 1;
-
-	na = (struct nlattr *)((char *)&req + NLMSG_ALIGN(req.nlh.nlmsg_len));
-	na->nla_type = CTRL_ATTR_FAMILY_NAME;
-	na->nla_len = NLA_HDRLEN + strlen(name) + 1;
-	memcpy(taskstats_nla_data(na), name, strlen(name) + 1);
-	req.nlh.nlmsg_len = NLMSG_ALIGN(req.nlh.nlmsg_len) + NLA_ALIGN(na->nla_len);
-
-	ret = send_request(fd, &req, req.nlh.nlmsg_len);
-	if (ret)
-		return ret;
-
-	len = recv(fd, resp, sizeof(resp), 0);
-	if (len < 0)
-		return -errno;
-
-	for (nlh = (struct nlmsghdr *)resp; NLMSG_OK(nlh, len);
-	     nlh = NLMSG_NEXT(nlh, len)) {
-		if (nlh->nlmsg_type == NLMSG_ERROR) {
-			struct nlmsgerr *err = NLMSG_DATA(nlh);
-
-			return err->error ? err->error : -ENOENT;
-		}
-
-		genl = (struct genlmsghdr *)NLMSG_DATA(nlh);
-		rem = nlh->nlmsg_len - NLMSG_HDRLEN - GENL_HDRLEN;
-		na = (struct nlattr *)((char *)genl + GENL_HDRLEN);
-		while (taskstats_nla_ok(na, rem)) {
-			if (na->nla_type == CTRL_ATTR_FAMILY_ID)
-				return *(uint16_t *)taskstats_nla_data(na);
-			na = taskstats_nla_next(na, &rem);
-		}
-	}
-
-	return -ENOENT;
-}
-
 static int get_taskstats(int fd, int family_id, uint16_t attr_type, uint32_t id,
 			 struct taskstats *stats)
 {
@@ -209,7 +91,7 @@ static int get_taskstats(int fd, int family_id, uint16_t attr_type, uint32_t id,
 	na = (struct nlattr *)((char *)&req + NLMSG_ALIGN(req.nlh.nlmsg_len));
 	na->nla_type = attr_type;
 	na->nla_len = NLA_HDRLEN + sizeof(id);
-	memcpy(taskstats_nla_data(na), &id, sizeof(id));
+	memcpy(nla_data(na), &id, sizeof(id));
 	req.nlh.nlmsg_len = NLMSG_ALIGN(req.nlh.nlmsg_len) + NLA_ALIGN(na->nla_len);
 
 	ret = send_request(fd, &req, req.nlh.nlmsg_len);
@@ -231,21 +113,21 @@ static int get_taskstats(int fd, int family_id, uint16_t attr_type, uint32_t id,
 		genl = (struct genlmsghdr *)NLMSG_DATA(nlh);
 		rem = nlh->nlmsg_len - NLMSG_HDRLEN - GENL_HDRLEN;
 		na = (struct nlattr *)((char *)genl + GENL_HDRLEN);
-		while (taskstats_nla_ok(na, rem)) {
+		while (nla_ok(na, rem)) {
 			if (na->nla_type == TASKSTATS_TYPE_AGGR_PID ||
 			    na->nla_type == TASKSTATS_TYPE_AGGR_TGID) {
-				nested = (struct nlattr *)taskstats_nla_data(na);
+				nested = (struct nlattr *)nla_data(na);
 				nrem = na->nla_len - NLA_HDRLEN;
-				while (taskstats_nla_ok(nested, nrem)) {
+				while (nla_ok(nested, nrem)) {
 					if (nested->nla_type == TASKSTATS_TYPE_STATS) {
-						memcpy(stats, taskstats_nla_data(nested),
+						memcpy(stats, nla_data(nested),
 						       sizeof(*stats));
 						return 0;
 					}
-					nested = taskstats_nla_next(nested, &nrem);
+					nested = nla_next(nested, &nrem);
 				}
 			}
-			na = taskstats_nla_next(na, &rem);
+			na = nla_next(na, &rem);
 		}
 	}
 
diff --git a/tools/testing/selftests/alloc_tag/Makefile b/tools/testing/selftests/alloc_tag/Makefile
new file mode 100644
index 0000000000000..f41d92d06b184
--- /dev/null
+++ b/tools/testing/selftests/alloc_tag/Makefile
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0
+
+TEST_GEN_PROGS := allocinfo_ioctl_test
+
+CFLAGS += -Wall
+CFLAGS += $(KHDR_INCLUDES)
+
+include ../lib.mk
+
diff --git a/tools/testing/selftests/alloc_tag/allocinfo_ioctl_test.c b/tools/testing/selftests/alloc_tag/allocinfo_ioctl_test.c
new file mode 100644
index 0000000000000..7d94abb785a14
--- /dev/null
+++ b/tools/testing/selftests/alloc_tag/allocinfo_ioctl_test.c
@@ -0,0 +1,548 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+/* kselftest for allocinfo ioctl
+ * allocinfo ioctl retrives allocinfo data through ioctl
+ * Copyright (C) 2026 Google, Inc.
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdbool.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <linux/types.h>
+#include <linux/alloc_tag.h>
+#include "../kselftest.h"
+
+#define MAX_LINE_LEN		512
+#define ALLOCINFO_PROC		"/proc/allocinfo"
+
+enum ioctl_ret {
+	IOCTL_SUCCESS = 0,
+	IOCTL_FAILURE = 1,
+	IOCTL_INVALID_DATA = 2,
+};
+
+#define VEC_MAX_ENTRIES 32
+
+struct allocinfo_tag_data_vec {
+	struct allocinfo_tag_data tag[VEC_MAX_ENTRIES];
+	__u64 count;
+};
+
+static inline int __allocinfo_get_content_id(int dev_fd, struct allocinfo_content_id *params)
+{
+	return ioctl(dev_fd, ALLOCINFO_IOC_CONTENT_ID, params);
+}
+
+static inline int __allocinfo_get_at(int dev_fd, struct allocinfo_get_at *params)
+{
+	return ioctl(dev_fd, ALLOCINFO_IOC_GET_AT, params);
+}
+
+static inline int __allocinfo_get_next(int dev_fd, struct allocinfo_tag_data *params)
+{
+	return ioctl(dev_fd, ALLOCINFO_IOC_GET_NEXT, params);
+}
+
+static bool match_entry(const struct allocinfo_tag_data *procfs_entry,
+			const struct allocinfo_tag_data *tag_data,
+			bool match_bytes, bool match_calls, bool match_lineno,
+			bool match_function, bool match_filename)
+{
+	if (match_bytes && tag_data->counter.bytes != procfs_entry->counter.bytes) {
+		ksft_print_msg("size retrieved through ioctl does not match procfs\n");
+		return false;
+	}
+
+	if (match_calls && tag_data->counter.calls != procfs_entry->counter.calls) {
+		ksft_print_msg("call count retrieved through ioctl does not match procfs\n");
+		return false;
+	}
+
+	if (match_lineno && tag_data->tag.lineno != procfs_entry->tag.lineno) {
+		ksft_print_msg("lineno retrieved through ioctl does not match procfs\n");
+		return false;
+	}
+
+	if (match_function &&
+	    strncmp(tag_data->tag.function, procfs_entry->tag.function, ALLOCINFO_STR_SIZE)) {
+		ksft_print_msg("function retrieved through ioctl does not match procfs\n");
+		return false;
+	}
+
+	if (match_filename &&
+	    strncmp(tag_data->tag.filename, procfs_entry->tag.filename, ALLOCINFO_STR_SIZE)) {
+		ksft_print_msg("filename retrieved through ioctl does not match procfs\n");
+		return false;
+	}
+	return true;
+}
+
+static bool match_entries(const struct allocinfo_tag_data_vec *procfs_entries,
+			  const struct allocinfo_tag_data_vec *tags,
+			  bool match_bytes, bool match_calls, bool match_lineno,
+			  bool match_function, bool match_filename)
+{
+	__u64 i;
+
+	if (procfs_entries->count != tags->count) {
+		ksft_print_msg("Entry count mismatch. ioctl entries: %llu, proc entries: %llu\n",
+			       tags->count, procfs_entries->count);
+		return false;
+	}
+	for (i = 0; i < procfs_entries->count; i++) {
+		if (!match_entry(&procfs_entries->tag[i], &tags->tag[i],
+				 match_bytes, match_calls, match_lineno,
+				 match_function, match_filename)) {
+			ksft_print_msg("%lluth entry does not match.\n", i);
+			return false;
+		}
+	}
+	return true;
+}
+
+static const char *allocinfo_str(const char *str)
+{
+	size_t len = strlen(str);
+
+	if (len >= ALLOCINFO_STR_SIZE)
+		str += (len - ALLOCINFO_STR_SIZE) + 1;
+	return str;
+}
+
+static void allocinfo_copy_str(char *dest, const char *src)
+{
+	strncpy(dest, allocinfo_str(src), ALLOCINFO_STR_SIZE - 1);
+	dest[ALLOCINFO_STR_SIZE - 1] = '\0';
+}
+
+static int get_filtered_procfs_entries(struct allocinfo_tag_data_vec *procfs_entries,
+				       const struct allocinfo_filter *filter)
+{
+	FILE *fp = fopen(ALLOCINFO_PROC, "r");
+	char line[MAX_LINE_LEN];
+	int matches;
+	struct allocinfo_tag_data procfs_entry;
+
+	if (!fp) {
+		ksft_print_msg("Failed to open " ALLOCINFO_PROC " for reading\n");
+		return 1;
+	}
+	memset(procfs_entries, 0, sizeof(*procfs_entries));
+	while (fgets(line, sizeof(line), fp) && procfs_entries->count < VEC_MAX_ENTRIES) {
+		char filename[MAX_LINE_LEN];
+		char function[MAX_LINE_LEN];
+
+		memset(&procfs_entry, 0, sizeof(procfs_entry));
+		matches = sscanf(line, "%llu %llu %[^:]:%llu func:%s",
+				 &procfs_entry.counter.bytes,
+				 &procfs_entry.counter.calls,
+				 filename,
+				 &procfs_entry.tag.lineno,
+				 function);
+
+		if (matches != 5)
+			continue;
+
+		allocinfo_copy_str(procfs_entry.tag.filename, filename);
+		allocinfo_copy_str(procfs_entry.tag.function, function);
+
+		if (filter->mask & ALLOCINFO_FILTER_MASK_FILENAME) {
+			if (strncmp(procfs_entry.tag.filename,
+				    filter->fields.filename, ALLOCINFO_STR_SIZE))
+				continue;
+		}
+		if (filter->mask & ALLOCINFO_FILTER_MASK_FUNCTION) {
+			if (strncmp(procfs_entry.tag.function,
+				    filter->fields.function, ALLOCINFO_STR_SIZE))
+				continue;
+		}
+		if (filter->mask & ALLOCINFO_FILTER_MASK_LINENO) {
+			if (procfs_entry.tag.lineno != filter->fields.lineno)
+				continue;
+		}
+		if (filter->mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) {
+			if (procfs_entry.counter.bytes < filter->min_size)
+				continue;
+		}
+		if (filter->mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) {
+			if (procfs_entry.counter.bytes > filter->max_size)
+				continue;
+		}
+
+		memcpy(&procfs_entries->tag[procfs_entries->count++], &procfs_entry,
+		       sizeof(procfs_entry));
+	}
+	fclose(fp);
+	return 0;
+}
+
+static enum ioctl_ret get_filtered_ioctl_entries(struct allocinfo_tag_data_vec *tags,
+						 const struct allocinfo_filter *filter,
+						 __u64 start_pos)
+{
+	int fd = open(ALLOCINFO_PROC, O_RDONLY);
+
+	if (fd < 0) {
+		ksft_print_msg("Failed to open " ALLOCINFO_PROC " for IOCTL\n");
+		return IOCTL_FAILURE;
+	}
+
+	struct allocinfo_content_id start_cont_id, end_cont_id;
+	struct allocinfo_get_at get_at_params;
+	const int max_retries = 10;
+	int retry_count = 0;
+	int status;
+
+	/*
+	 * __allocinfo_get_content_id may return different values if a kernel module was loaded
+	 * between the two calls. If that happens, the data gathered cannot be considered consistent
+	 * and hence needs to be fetched again to avoid flakiness.
+	 */
+	do {
+		if (__allocinfo_get_content_id(fd, &start_cont_id)) {
+			ksft_print_msg("allocinfo_get_content_id failed\n");
+			status = IOCTL_FAILURE;
+			break;
+		}
+
+		memset(tags, 0, sizeof(*tags));
+		memset(&get_at_params, 0, sizeof(get_at_params));
+		memcpy(&get_at_params.filter, filter, sizeof(*filter));
+		get_at_params.pos = start_pos;
+		if (__allocinfo_get_at(fd, &get_at_params)) {
+			ksft_print_msg("allocinfo_get_at failed\n");
+			status = IOCTL_FAILURE;
+			break;
+		}
+		memcpy(&tags->tag[tags->count++], &get_at_params.data, sizeof(get_at_params.data));
+
+		while (tags->count < VEC_MAX_ENTRIES &&
+		       __allocinfo_get_next(fd, &tags->tag[tags->count]) == 0)
+			tags->count++;
+
+		if (__allocinfo_get_content_id(fd, &end_cont_id)) {
+			ksft_print_msg("allocinfo_get_content_id failed\n");
+			status = IOCTL_FAILURE;
+			break;
+		}
+
+		if (start_cont_id.id == end_cont_id.id) {
+			status = IOCTL_SUCCESS;
+		} else {
+			ksft_print_msg("allocinfo_get_content_id mismatch, retrying...\n");
+			status = IOCTL_INVALID_DATA;
+		}
+	} while (status == IOCTL_INVALID_DATA && retry_count++ < max_retries);
+
+	close(fd);
+	return status;
+}
+
+static int run_filter_test(const struct allocinfo_filter *filter)
+{
+	struct allocinfo_tag_data_vec *tags = malloc(sizeof(*tags));
+	struct allocinfo_tag_data_vec *procfs_entries = malloc(sizeof(*procfs_entries));
+	int ioctl_status;
+	int ret = KSFT_PASS;
+
+	if (!tags || !procfs_entries) {
+		ksft_print_msg("Memory allocation failed.\n");
+		ret = KSFT_FAIL;
+		goto exit;
+	}
+
+	if (get_filtered_procfs_entries(procfs_entries, filter)) {
+		ksft_print_msg("Error retrieving entries from " ALLOCINFO_PROC "\n");
+		ret = KSFT_SKIP;
+		goto exit;
+	}
+
+	if (procfs_entries->count == 0) {
+		ksft_print_msg("No entries found in " ALLOCINFO_PROC ", skipping test\n");
+		ret = KSFT_SKIP;
+		goto exit;
+	}
+
+	ioctl_status = get_filtered_ioctl_entries(tags, filter, 0);
+	if (ioctl_status == IOCTL_INVALID_DATA) {
+		ksft_print_msg("Trouble retrieving valid IOCTL entries, skipping.\n");
+		ret = KSFT_SKIP;
+		goto exit;
+	}
+	if (ioctl_status == IOCTL_FAILURE) {
+		ksft_print_msg("Error retrieving IOCTL entries.\n");
+		ret = KSFT_FAIL;
+		goto exit;
+	}
+
+	if (!match_entries(procfs_entries, tags, false, false, true, true, true))
+		ret = KSFT_FAIL;
+
+exit:
+	free(tags);
+	free(procfs_entries);
+	return ret;
+}
+
+static int test_filename_filter(void)
+{
+	struct allocinfo_filter filter;
+	const char *target_filename = "mm/memory.c";
+
+	memset(&filter, 0, sizeof(filter));
+	filter.mask |= ALLOCINFO_FILTER_MASK_FILENAME;
+	strncpy(filter.fields.filename, target_filename, ALLOCINFO_STR_SIZE);
+
+	return run_filter_test(&filter);
+}
+
+static int test_function_filter(void)
+{
+	struct allocinfo_filter filter;
+	const char *target_function = "dup_mm";
+
+	memset(&filter, 0, sizeof(filter));
+	filter.mask |= ALLOCINFO_FILTER_MASK_FUNCTION;
+	strncpy(filter.fields.function, target_function, ALLOCINFO_STR_SIZE);
+
+	return run_filter_test(&filter);
+}
+
+static int test_size_filter(void)
+{
+	int fd;
+	struct allocinfo_tag_data_vec *tags = malloc(sizeof(*tags));
+	struct allocinfo_tag_data_vec *procfs_entries = malloc(sizeof(*procfs_entries));
+	struct allocinfo_filter filter;
+	int ret = KSFT_PASS;
+	__u64 target_size, i, pos;
+	struct allocinfo_tag_data *found_tag = NULL;
+	const char *target_function = "do_init_module";
+	struct allocinfo_content_id start_cont_id, end_cont_id;
+	int retry = 0;
+	const int max_retries = 10;
+
+	if (!tags || !procfs_entries) {
+		ksft_print_msg("Memory allocation failed.\n");
+		ret = KSFT_FAIL;
+		goto freemem;
+	}
+
+	fd = open(ALLOCINFO_PROC, O_RDONLY);
+	if (fd < 0) {
+		ksft_print_msg("Failed to open " ALLOCINFO_PROC ": %s\n", strerror(errno));
+		ret = KSFT_SKIP;
+		goto freemem;
+	}
+
+	do {
+		found_tag = NULL;
+		pos = 0;
+
+		if (__allocinfo_get_content_id(fd, &start_cont_id)) {
+			ksft_print_msg("allocinfo_get_content_id failed\n");
+			ret = KSFT_FAIL;
+			goto exit;
+		}
+
+		memset(&filter, 0, sizeof(filter));
+		filter.mask |= ALLOCINFO_FILTER_MASK_FUNCTION;
+		strncpy(filter.fields.function, target_function, ALLOCINFO_STR_SIZE);
+
+		if (get_filtered_procfs_entries(procfs_entries, &filter)) {
+			ksft_print_msg("Error retrieving entries from " ALLOCINFO_PROC "\n");
+			ret = KSFT_SKIP;
+			goto exit;
+		}
+
+		if (procfs_entries->count == 0) {
+			ksft_print_msg("Function %s not found in procfs\n", target_function);
+			ret = KSFT_SKIP;
+			goto exit;
+		}
+
+		target_size = procfs_entries->tag[0].counter.bytes;
+
+		memset(&filter, 0, sizeof(filter));
+		filter.mask |= ALLOCINFO_FILTER_MASK_MIN_SIZE | ALLOCINFO_FILTER_MASK_MAX_SIZE;
+		filter.min_size = target_size;
+		filter.max_size = target_size;
+
+		while (1) {
+			struct allocinfo_get_at get_at_params;
+
+			memset(&get_at_params, 0, sizeof(get_at_params));
+			memcpy(&get_at_params.filter, &filter, sizeof(filter));
+			get_at_params.pos = pos;
+
+			if (__allocinfo_get_at(fd, &get_at_params))
+				break;
+
+			tags->count = 0;
+			memcpy(&tags->tag[tags->count++], &get_at_params.data,
+			       sizeof(get_at_params.data));
+
+			while (tags->count < VEC_MAX_ENTRIES &&
+			       __allocinfo_get_next(fd, &tags->tag[tags->count]) == 0)
+				tags->count++;
+
+			for (i = 0; i < tags->count; i++) {
+				if (strcmp(tags->tag[i].tag.function, target_function) == 0) {
+					found_tag = &tags->tag[i];
+					break;
+				}
+			}
+
+			if (found_tag || tags->count < VEC_MAX_ENTRIES)
+				break;
+
+			pos += tags->count;
+		}
+
+		if (__allocinfo_get_content_id(fd, &end_cont_id)) {
+			ksft_print_msg("allocinfo_get_content_id failed\n");
+			ret = KSFT_FAIL;
+			goto exit;
+		}
+
+		if (start_cont_id.id == end_cont_id.id)
+			break;
+
+		ksft_print_msg("Module load detected during size verification, retrying...\n");
+	} while (retry++ < max_retries);
+
+	if (start_cont_id.id == end_cont_id.id && !found_tag) {
+		ksft_print_msg("Entry with function %s not found in IOCTL results\n",
+			       target_function);
+		ret = KSFT_FAIL;
+	} else if (start_cont_id.id != end_cont_id.id) {
+		ksft_print_msg("Failed to match content_ids for procfs and IOCTL, skipping...\n");
+		ret = KSFT_SKIP;
+	} else if (found_tag && found_tag->counter.bytes != target_size) {
+		ksft_print_msg("IOCTL entry size %llu does not match target size %llu\n",
+			       found_tag->counter.bytes, target_size);
+		ret = KSFT_FAIL;
+	}
+
+exit:
+	close(fd);
+freemem:
+	free(tags);
+	free(procfs_entries);
+	return ret;
+}
+
+static int test_lineno_filter(void)
+{
+	struct allocinfo_tag_data_vec *tags = malloc(sizeof(*tags));
+	struct allocinfo_tag_data_vec *procfs_entries = malloc(sizeof(*procfs_entries));
+	struct allocinfo_filter filter;
+	enum ioctl_ret ioctl_status;
+	int ret = KSFT_PASS;
+	__u64 target_lineno, i;
+	struct allocinfo_tag_data *target_tag;
+	bool found = false;
+
+	if (!tags || !procfs_entries) {
+		ksft_print_msg("Memory allocation failed.\n");
+		ret = KSFT_FAIL;
+		goto exit;
+	}
+
+	memset(&filter, 0, sizeof(filter));
+
+	if (get_filtered_procfs_entries(procfs_entries, &filter)) {
+		ksft_print_msg("Error retrieving entries from " ALLOCINFO_PROC "\n");
+		ret = KSFT_SKIP;
+		goto exit;
+	}
+	if (procfs_entries->count == 0) {
+		ksft_print_msg("Could not retrieve procfs entries\n");
+		ret = KSFT_SKIP;
+		goto exit;
+	}
+	/*
+	 * We depend on the procfs results to determine the line number for the filter before
+	 * making the ioctl query. Hence, we cannot reuse run_filter_test here.
+	 */
+	target_tag = &procfs_entries->tag[0];
+	target_lineno = target_tag->tag.lineno;
+
+	filter.mask |= ALLOCINFO_FILTER_MASK_LINENO;
+	filter.fields.lineno = target_lineno;
+
+	ioctl_status = get_filtered_ioctl_entries(tags, &filter, 0);
+	if (ioctl_status == IOCTL_INVALID_DATA) {
+		ksft_print_msg("Trouble retrieving valid IOCTL entries, skipping.\n");
+		ret = KSFT_SKIP;
+		goto exit;
+	}
+	if (ioctl_status == IOCTL_FAILURE) {
+		ksft_print_msg("Error retrieving IOCTL entries.\n");
+		ret = KSFT_FAIL;
+		goto exit;
+	}
+
+	for (i = 0; i < tags->count; i++) {
+		if (tags->tag[i].tag.lineno != target_lineno) {
+			ksft_print_msg("IOCTL entry %llu has incorrect lineno %llu.\n",
+				       i, tags->tag[i].tag.lineno);
+			ret = KSFT_FAIL;
+			goto exit;
+		}
+
+		if (strncmp(tags->tag[i].tag.function, target_tag->tag.function,
+			    ALLOCINFO_STR_SIZE) == 0 &&
+		    strncmp(tags->tag[i].tag.filename, target_tag->tag.filename,
+			    ALLOCINFO_STR_SIZE) == 0)
+			found = true;
+	}
+
+	if (!found) {
+		ksft_print_msg("Original procfs entry not found in IOCTL lineno filter results.\n");
+		ret = KSFT_FAIL;
+	}
+
+exit:
+	free(tags);
+	free(procfs_entries);
+	return ret;
+}
+
+int main(int argc, char *argv[])
+{
+	int ret;
+
+	ksft_set_plan(4);
+
+	ret = test_filename_filter();
+	if (ret == KSFT_SKIP)
+		ksft_test_result_skip("Skipping test_filename_filter\n");
+	else
+		ksft_test_result(ret == KSFT_PASS, "test_filename_filter\n");
+
+	ret = test_function_filter();
+	if (ret == KSFT_SKIP)
+		ksft_test_result_skip("Skipping test_function_filter\n");
+	else
+		ksft_test_result(ret == KSFT_PASS, "test_function_filter\n");
+
+	ret = test_size_filter();
+	if (ret == KSFT_SKIP)
+		ksft_test_result_skip("Skipping test_size_filter\n");
+	else
+		ksft_test_result(ret == KSFT_PASS, "test_size_filter\n");
+
+	ret = test_lineno_filter();
+	if (ret == KSFT_SKIP)
+		ksft_test_result_skip("Skipping test_lineno_filter\n");
+	else
+		ksft_test_result(ret == KSFT_PASS, "test_lineno_filter\n");
+
+	ksft_finished();
+}
diff --git a/tools/testing/selftests/alsa/mixer-test.c b/tools/testing/selftests/alsa/mixer-test.c
index d4f845c328045..a329f901c5edf 100644
--- a/tools/testing/selftests/alsa/mixer-test.c
+++ b/tools/testing/selftests/alsa/mixer-test.c
@@ -84,6 +84,7 @@ static void find_controls(void)
 		if (err < 0) {
 			ksft_print_msg("Failed to get hctl for card %d: %s\n",
 				       card, snd_strerror(err));
+			free(card_data);
 			goto next_card;
 		}
 
@@ -339,9 +340,9 @@ static bool ctl_value_index_valid(struct ctl_data *ctl,
 		}
 
 		if (int64_val > snd_ctl_elem_info_get_max64(ctl->info)) {
-			ksft_print_msg("%s.%d value %lld more than maximum %ld\n",
+			ksft_print_msg("%s.%d value %lld more than maximum %lld\n",
 				       ctl->name, index, int64_val,
-				       snd_ctl_elem_info_get_max(ctl->info));
+				       snd_ctl_elem_info_get_max64(ctl->info));
 			return false;
 		}
 
diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c
index 3d9d2cd7764bd..c4a3a6b3eb83d 100644
--- a/tools/testing/selftests/bpf/bench.c
+++ b/tools/testing/selftests/bpf/bench.c
@@ -539,12 +539,12 @@ extern const struct bench bench_trig_uretprobe_multi_push;
 extern const struct bench bench_trig_uprobe_multi_ret;
 extern const struct bench bench_trig_uretprobe_multi_ret;
 #ifdef __x86_64__
-extern const struct bench bench_trig_uprobe_nop5;
-extern const struct bench bench_trig_uretprobe_nop5;
-extern const struct bench bench_trig_uprobe_multi_nop5;
-extern const struct bench bench_trig_uretprobe_multi_nop5;
+extern const struct bench bench_trig_uprobe_nop10;
+extern const struct bench bench_trig_uretprobe_nop10;
+extern const struct bench bench_trig_uprobe_multi_nop10;
+extern const struct bench bench_trig_uretprobe_multi_nop10;
 extern const struct bench bench_trig_usdt_nop;
-extern const struct bench bench_trig_usdt_nop5;
+extern const struct bench bench_trig_usdt_nop10;
 #endif
 
 extern const struct bench bench_rb_libbpf;
@@ -622,12 +622,12 @@ static const struct bench *benchs[] = {
 	&bench_trig_uprobe_multi_ret,
 	&bench_trig_uretprobe_multi_ret,
 #ifdef __x86_64__
-	&bench_trig_uprobe_nop5,
-	&bench_trig_uretprobe_nop5,
-	&bench_trig_uprobe_multi_nop5,
-	&bench_trig_uretprobe_multi_nop5,
+	&bench_trig_uprobe_nop10,
+	&bench_trig_uretprobe_nop10,
+	&bench_trig_uprobe_multi_nop10,
+	&bench_trig_uretprobe_multi_nop10,
 	&bench_trig_usdt_nop,
-	&bench_trig_usdt_nop5,
+	&bench_trig_usdt_nop10,
 #endif
 	/* ringbuf/perfbuf benchmarks */
 	&bench_rb_libbpf,
diff --git a/tools/testing/selftests/bpf/benchs/bench_trigger.c b/tools/testing/selftests/bpf/benchs/bench_trigger.c
index 2f22ec61667b9..61513efc167a3 100644
--- a/tools/testing/selftests/bpf/benchs/bench_trigger.c
+++ b/tools/testing/selftests/bpf/benchs/bench_trigger.c
@@ -396,15 +396,15 @@ static void *uprobe_producer_ret(void *input)
 }
 
 #ifdef __x86_64__
-__nocf_check __weak void uprobe_target_nop5(void)
+__nocf_check __weak void uprobe_target_nop10(void)
 {
-	asm volatile (".byte 0x0f, 0x1f, 0x44, 0x00, 0x00");
+	asm volatile (".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00");
 }
 
-static void *uprobe_producer_nop5(void *input)
+static void *uprobe_producer_nop10(void *input)
 {
 	while (true)
-		uprobe_target_nop5();
+		uprobe_target_nop10();
 	return NULL;
 }
 
@@ -418,7 +418,7 @@ static void *uprobe_producer_usdt_nop(void *input)
 	return NULL;
 }
 
-static void *uprobe_producer_usdt_nop5(void *input)
+static void *uprobe_producer_usdt_nop10(void *input)
 {
 	while (true)
 		usdt_2();
@@ -542,24 +542,24 @@ static void uretprobe_multi_ret_setup(void)
 }
 
 #ifdef __x86_64__
-static void uprobe_nop5_setup(void)
+static void uprobe_nop10_setup(void)
 {
-	usetup(false, false /* !use_multi */, &uprobe_target_nop5);
+	usetup(false, false /* !use_multi */, &uprobe_target_nop10);
 }
 
-static void uretprobe_nop5_setup(void)
+static void uretprobe_nop10_setup(void)
 {
-	usetup(true, false /* !use_multi */, &uprobe_target_nop5);
+	usetup(true, false /* !use_multi */, &uprobe_target_nop10);
 }
 
-static void uprobe_multi_nop5_setup(void)
+static void uprobe_multi_nop10_setup(void)
 {
-	usetup(false, true /* use_multi */, &uprobe_target_nop5);
+	usetup(false, true /* use_multi */, &uprobe_target_nop10);
 }
 
-static void uretprobe_multi_nop5_setup(void)
+static void uretprobe_multi_nop10_setup(void)
 {
-	usetup(true, true /* use_multi */, &uprobe_target_nop5);
+	usetup(true, true /* use_multi */, &uprobe_target_nop10);
 }
 
 static void usdt_setup(const char *name)
@@ -598,7 +598,7 @@ static void usdt_nop_setup(void)
 	usdt_setup("usdt_1");
 }
 
-static void usdt_nop5_setup(void)
+static void usdt_nop10_setup(void)
 {
 	usdt_setup("usdt_2");
 }
@@ -665,10 +665,10 @@ BENCH_TRIG_USERMODE(uretprobe_multi_nop, nop, "uretprobe-multi-nop");
 BENCH_TRIG_USERMODE(uretprobe_multi_push, push, "uretprobe-multi-push");
 BENCH_TRIG_USERMODE(uretprobe_multi_ret, ret, "uretprobe-multi-ret");
 #ifdef __x86_64__
-BENCH_TRIG_USERMODE(uprobe_nop5, nop5, "uprobe-nop5");
-BENCH_TRIG_USERMODE(uretprobe_nop5, nop5, "uretprobe-nop5");
-BENCH_TRIG_USERMODE(uprobe_multi_nop5, nop5, "uprobe-multi-nop5");
-BENCH_TRIG_USERMODE(uretprobe_multi_nop5, nop5, "uretprobe-multi-nop5");
+BENCH_TRIG_USERMODE(uprobe_nop10, nop10, "uprobe-nop10");
+BENCH_TRIG_USERMODE(uretprobe_nop10, nop10, "uretprobe-nop10");
+BENCH_TRIG_USERMODE(uprobe_multi_nop10, nop10, "uprobe-multi-nop10");
+BENCH_TRIG_USERMODE(uretprobe_multi_nop10, nop10, "uretprobe-multi-nop10");
 BENCH_TRIG_USERMODE(usdt_nop, usdt_nop, "usdt-nop");
-BENCH_TRIG_USERMODE(usdt_nop5, usdt_nop5, "usdt-nop5");
+BENCH_TRIG_USERMODE(usdt_nop10, usdt_nop10, "usdt-nop10");
 #endif
diff --git a/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh b/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh
index 9ec59423b9492..e490b337e9600 100755
--- a/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh
+++ b/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh
@@ -2,7 +2,7 @@
 
 set -eufo pipefail
 
-for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret,nop5} usdt-nop usdt-nop5
+for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret,nop10} usdt-nop usdt-nop10
 do
 	summary=$(sudo ./bench -w2 -d5 -a trig-$i | tail -n1 | cut -d'(' -f1 | cut -d' ' -f3-)
 	printf "%-15s: %s\n" $i "$summary"
diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing/selftests/bpf/bpf_experimental.h
index 67ff7882299ec..63e4e472fe36e 100644
--- a/tools/testing/selftests/bpf/bpf_experimental.h
+++ b/tools/testing/selftests/bpf/bpf_experimental.h
@@ -364,6 +364,9 @@ extern void bpf_iter_dmabuf_destroy(struct bpf_iter_dmabuf *it) __weak __ksym;
 extern int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__str,
 				 struct bpf_dynptr *value_p) __weak __ksym;
 
+extern int bpf_sock_read_xattr(struct socket *sock, const char *name__str,
+			       struct bpf_dynptr *value_p) __weak __ksym;
+
 #define PREEMPT_BITS	8
 #define SOFTIRQ_BITS	8
 #define HARDIRQ_BITS	4
diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h
new file mode 100644
index 0000000000000..e2431ea6fdd68
--- /dev/null
+++ b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h
@@ -0,0 +1,34 @@
+#pragma once
+
+#define BITS_PER_BYTE		8
+#define BYTES_TO_BITS(nb)	((nb) * BITS_PER_BYTE)
+
+#define BITS_PER_LONG_LONG	(sizeof(long long) * BITS_PER_BYTE)
+#define BITS_TO_LONG_LONGS(nr)	(((nr) + BITS_PER_LONG_LONG - 1) / BITS_PER_LONG_LONG)
+#define BIT_MASK(nr)		(1ULL << ((nr) % BITS_PER_LONG_LONG))
+#define BIT_WORD(nr)		((nr) / BITS_PER_LONG_LONG)
+
+struct arena_bitmap {
+	u64 bits[0];
+};
+
+struct arena_bitmap __arena *bmp_alloc(size_t bits);
+void bmp_free(struct arena_bitmap __arena *bmp);
+
+void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp);
+void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp);
+void bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp);
+void bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp);
+bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp);
+bool bmp_test_and_clear_bit(u32 bit, struct arena_bitmap __arena *bmp);
+bool bmp_test_and_set_bit(u32 bit, struct arena_bitmap __arena *bmp);
+
+void bmp_clear(size_t bits, struct arena_bitmap __arena *bmp);
+void bmp_and(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src1, struct arena_bitmap __arena *src2);
+void bmp_or(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src1, struct arena_bitmap __arena *src2);
+bool bmp_empty(size_t bits, struct arena_bitmap __arena *bmp);
+void bmp_copy(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src);
+
+bool bmp_intersects(size_t bits, struct arena_bitmap __arena *arg1, struct arena_bitmap __arena *arg2);
+bool bmp_subset(size_t bits, struct arena_bitmap __arena *big, struct arena_bitmap __arena *small);
+void bmp_print(size_t bits, struct arena_bitmap __arena *bmp);
diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/common.h b/tools/testing/selftests/bpf/libarena/include/libarena/common.h
index a3eb1641ac36d..931ace9a49e2b 100644
--- a/tools/testing/selftests/bpf/libarena/include/libarena/common.h
+++ b/tools/testing/selftests/bpf/libarena/include/libarena/common.h
@@ -43,7 +43,7 @@ struct {
  * imprecise. To force the variable to be imprecise, initialize it with
  * the opaque volatile variable 0 instead of the constant 0.
  */
-extern const volatile u32 zero;
+volatile u32 zero __weak;
 extern volatile u64 asan_violated;
 
 int arena_fls(__u64 word);
diff --git a/tools/testing/selftests/bpf/libarena/selftests/st_asan_buddy.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/st_asan_buddy.bpf.c
deleted file mode 100644
index 686caba2c643b..0000000000000
--- a/tools/testing/selftests/bpf/libarena/selftests/st_asan_buddy.bpf.c
+++ /dev/null
@@ -1,258 +0,0 @@
-// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
-/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
-
-#include <libarena/common.h>
-#include <libarena/asan.h>
-#include <libarena/buddy.h>
-
-/* Required for parsing the ASAN call stacks. */
-#include "test_progs_compat.h"
-
-extern struct buddy __arena buddy;
-
-#ifdef BPF_ARENA_ASAN
-
-#include "st_asan_common.h"
-
-static __always_inline int asan_test_buddy_oob_single(size_t alloc_size)
-{
-	u8 __arena *mem;
-	int ret, i;
-
-	ret = asan_validate();
-	if (ret < 0)
-		return ret;
-
-	mem = buddy_alloc(&buddy, alloc_size);
-	if (!mem) {
-		arena_stdout("buddy_alloc failed for size %lu", alloc_size);
-		return -ENOMEM;
-	}
-
-	ret = asan_validate();
-	if (ret < 0)
-		return ret;
-
-	for (i = zero; i < alloc_size && can_loop; i++) {
-		mem[i] = 0xba;
-		ret = asan_validate_addr(false, &mem[i]);
-		if (ret < 0)
-			return ret;
-	}
-
-	mem[alloc_size] = 0xba;
-	ret = asan_validate_addr(true, &mem[alloc_size]);
-	if (ret < 0)
-		return ret;
-
-	buddy_free(&buddy, mem);
-
-	return 0;
-}
-
-/*
- * Factored out because asan_validate_addr is complex enough to cause
- * verification failures if verified with the rest of asan_test_buddy_uaf_single.
- */
-__weak int asan_test_buddy_byte(u8 __arena *mem, int i, bool freed)
-{
-	int ret;
-
-	/* The header in freed blocks doesn't get poisoned. */
-	if (freed && BUDDY_HEADER_OFF <= i &&
-		i < BUDDY_HEADER_OFF + sizeof(struct buddy_header))
-		return 0;
-
-	mem[i] = 0xba;
-	ret = asan_validate_addr(freed, &mem[i]);
-	if (ret < 0)
-		return ret;
-
-	return 0;
-}
-
-__weak int asan_test_buddy_uaf_single(size_t alloc_size)
-{
-	u8 __arena *mem;
-	int ret;
-	int i;
-
-	mem = buddy_alloc(&buddy, alloc_size);
-	if (!mem) {
-		arena_stdout("buddy_alloc failed for size %lu", alloc_size);
-		return -ENOMEM;
-	}
-
-	ret = asan_validate();
-	if (ret < 0)
-		return ret;
-
-	for (i = zero; i < alloc_size && can_loop; i++) {
-		ret = asan_test_buddy_byte(mem, i, false);
-		if (ret)
-			return ret;
-	}
-
-	ret = asan_validate();
-	if (ret < 0)
-		return ret;
-
-	buddy_free(&buddy, mem);
-
-	for (i = zero; i < alloc_size && can_loop; i++) {
-		ret = asan_test_buddy_byte(mem, i, true);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-struct buddy_blob {
-	volatile u8 mem[48];
-	u8 oob;
-};
-
-static __always_inline int asan_test_buddy_blob_single(void)
-{
-	volatile struct buddy_blob __arena *blob;
-	const size_t alloc_size = sizeof(struct buddy_blob) - 1;
-	int ret;
-
-	blob = buddy_alloc(&buddy, alloc_size);
-	if (!blob)
-		return -ENOMEM;
-
-	blob->mem[0] = 0xba;
-	ret = asan_validate_addr(false, &blob->mem[0]);
-	if (ret < 0)
-		return ret;
-
-	blob->mem[47] = 0xba;
-	ret = asan_validate_addr(false, &blob->mem[47]);
-	if (ret < 0)
-		return ret;
-
-	blob->oob = 0;
-	ret = asan_validate_addr(true, &blob->oob);
-	if (ret < 0)
-		return ret;
-
-	buddy_free(&buddy, (void __arena *)blob);
-
-	return 0;
-}
-
-SEC("syscall")
-__stderr("Memory violation for address {{.*}} for write of size 1")
-__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}")
-__stderr("Call trace:\n"
-"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n"
-"|[ \t]+[^\n]+\n)*}}")
-__weak int asan_test_buddy_oob(void)
-{
-	size_t sizes[] = {
-		7, 8, 17, 18, 64, 256, 317, 512, 1024,
-	};
-	int ret, i;
-
-	ret = buddy_init(&buddy);
-	if (ret) {
-		arena_stdout("buddy_init failed with %d", ret);
-		return ret;
-	}
-
-	for (i = zero; i < sizeof(sizes) / sizeof(sizes[0]) && can_loop; i++) {
-		ret = asan_test_buddy_oob_single(sizes[i]);
-		if (ret) {
-			arena_stdout("%s:%d Failed for size %lu", __func__,
-				   __LINE__, sizes[i]);
-			buddy_destroy(&buddy);
-			return ret;
-		}
-	}
-
-	buddy_destroy(&buddy);
-
-	ret = asan_validate();
-	if (ret < 0)
-		return ret;
-
-	return 0;
-}
-
-SEC("syscall")
-__stderr("Memory violation for address {{.*}} for write of size 1")
-__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}")
-__stderr("Call trace:\n"
-"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n"
-"|[ \t]+[^\n]+\n)*}}")
-__weak int asan_test_buddy_uaf(void)
-{
-	size_t sizes[] = { 16, 32, 64, 128, 256, 512, 1024, 16384 };
-	int ret, i;
-
-	ret = buddy_init(&buddy);
-	if (ret) {
-		arena_stdout("buddy_init failed with %d", ret);
-		return ret;
-	}
-
-	for (i = zero; i < sizeof(sizes) / sizeof(sizes[0]) && can_loop; i++) {
-		ret = asan_test_buddy_uaf_single(sizes[i]);
-		if (ret) {
-			arena_stdout("%s:%d Failed for size %lu", __func__,
-				   __LINE__, sizes[i]);
-			buddy_destroy(&buddy);
-			return ret;
-		}
-	}
-
-	buddy_destroy(&buddy);
-
-	ret = asan_validate();
-	if (ret < 0)
-		return ret;
-
-	return 0;
-}
-
-SEC("syscall")
-__stderr("Memory violation for address {{.*}} for write of size 1")
-__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}")
-__stderr("Call trace:\n"
-"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n"
-"|[ \t]+[^\n]+\n)*}}")
-__weak int asan_test_buddy_blob(void)
-{
-	const int iters = 10;
-	int ret, i;
-
-	ret = buddy_init(&buddy);
-	if (ret) {
-		arena_stdout("buddy_init failed with %d", ret);
-		return ret;
-	}
-
-	for (i = zero; i < iters && can_loop; i++) {
-		ret = asan_test_buddy_blob_single();
-		if (ret) {
-			arena_stdout("%s:%d Failed on iteration %d", __func__,
-				   __LINE__, i);
-			buddy_destroy(&buddy);
-			return ret;
-		}
-	}
-
-	buddy_destroy(&buddy);
-
-	ret = asan_validate();
-	if (ret < 0)
-		return ret;
-
-	return 0;
-}
-
-#endif
-
-__weak char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/libarena/selftests/st_asan_common.h b/tools/testing/selftests/bpf/libarena/selftests/st_asan_common.h
deleted file mode 100644
index 34a7918cb4cf6..0000000000000
--- a/tools/testing/selftests/bpf/libarena/selftests/st_asan_common.h
+++ /dev/null
@@ -1,52 +0,0 @@
-// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
-/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
-
-#pragma once
-
-#define ST_PAGES 64
-
-static inline void print_asan_map_state(void __arena *addr)
-{
-	arena_stdout("%s:%d ASAN %p -> (val: %x gran: %x set: [%s])",
-			__func__, __LINE__, addr,
-			*(s8 __arena *)(addr), ASAN_GRANULE(addr),
-			asan_shadow_set(addr) ? "yes" : "no");
-}
-
-/*
- * Emit an error and force the current function to exit if the ASAN
- * violation state is unexpected. Reset the violation state after.
- */
-static inline int asan_validate_addr(bool cond, void __arena *addr)
-{
-	if ((asan_violated != 0) == cond) {
-		asan_violated = 0;
-		return 0;
-	}
-
-	arena_stdout("%s:%d ASAN asan_violated %lx", __func__, __LINE__,
-			(u64)asan_violated);
-	print_asan_map_state(addr);
-
-	asan_violated = 0;
-
-	return -EINVAL;
-}
-
-static inline int asan_validate(void)
-{
-	if (!asan_violated)
-		return 0;
-
-	arena_stdout("%s:%d Found ASAN violation at %lx", __func__, __LINE__,
-			asan_violated);
-
-	asan_violated = 0;
-
-	return -EINVAL;
-}
-
-struct blob {
-	volatile u8 mem[59];
-	u8 oob;
-};
diff --git a/tools/testing/selftests/bpf/libarena/selftests/st_buddy.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/st_buddy.bpf.c
deleted file mode 100644
index b45a306816c0e..0000000000000
--- a/tools/testing/selftests/bpf/libarena/selftests/st_buddy.bpf.c
+++ /dev/null
@@ -1,209 +0,0 @@
-// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
-/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
-
-#include <libarena/common.h>
-
-#include <libarena/asan.h>
-#include <libarena/buddy.h>
-
-extern struct buddy __arena buddy;
-
-struct segarr_entry {
-	u8 __arena *block;
-	size_t sz;
-	u8 poison;
-};
-
-#define SEGARRLEN (512)
-static struct segarr_entry __arena segarr[SEGARRLEN];
-static void __arena *ptrs[17];
-size_t __arena alloc_sizes[] = { 3, 17, 1025, 129, 16350, 333, 9, 517 };
-size_t __arena alloc_multiple_sizes[] = { 3, 17, 1025, 129, 16350, 333, 9, 517, 2099 };
-size_t __arena alloc_free_sizes[] = { 3, 17, 64, 129, 256, 333, 512, 517 };
-size_t __arena alignment_sizes[] = { 1, 3, 7, 8, 9, 15, 16, 17, 31,
-				     32, 64, 100, 128, 255, 256, 512, 1000 };
-
-SEC("syscall")
-__weak int test_buddy_create(void)
-{
-	const int iters = 10;
-	int ret, i;
-
-	for (i = zero; i < iters && can_loop; i++) {
-		ret = buddy_init(&buddy);
-		if (ret)
-			return ret;
-
-		ret = buddy_destroy(&buddy);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-SEC("syscall")
-__weak int test_buddy_alloc(void)
-{
-	void __arena *mem;
-	int ret, i;
-
-	for (i = zero; i < 8 && can_loop; i++) {
-		ret = buddy_init(&buddy);
-		if (ret)
-			return ret;
-
-		mem = buddy_alloc(&buddy, alloc_sizes[i]);
-		if (!mem) {
-			buddy_destroy(&buddy);
-			return -ENOMEM;
-		}
-
-		buddy_destroy(&buddy);
-	}
-
-	return 0;
-}
-
-SEC("syscall")
-__weak int test_buddy_alloc_free(void)
-{
-	const int iters = 800;
-	void __arena *mem;
-	int ret, i;
-
-	ret = buddy_init(&buddy);
-	if (ret)
-		return ret;
-
-	for (i = zero; i < iters && can_loop; i++) {
-		mem = buddy_alloc(&buddy, alloc_free_sizes[(i * 5) % 8]);
-		if (!mem) {
-			buddy_destroy(&buddy);
-			return -ENOMEM;
-		}
-
-		buddy_free(&buddy, mem);
-	}
-
-	buddy_destroy(&buddy);
-
-	return 0;
-}
-
-SEC("syscall")
-__weak int test_buddy_alloc_multiple(void)
-{
-	int ret, j;
-	u32 i, idx;
-	u8 __arena *mem;
-	size_t sz;
-	u8 poison;
-
-	ret = buddy_init(&buddy);
-	if (ret)
-		return ret;
-
-	/*
-	 * Cycle through each size, allocating an entry in the
-	 * segarr. Continue for SEGARRLEN iterations. For every
-	 * allocation write down the size, use the current index
-	 * as a poison value, and log it with the pointer in the
-	 * segarr entry. Use the poison value to poison the entire
-	 * allocated memory according to the size given.
-	 */
-	for (i = zero; i < SEGARRLEN && can_loop; i++) {
-		sz = alloc_multiple_sizes[i % 9];
-		poison = (u8)i;
-
-		mem = buddy_alloc(&buddy, sz);
-		if (!mem) {
-			buddy_destroy(&buddy);
-			arena_stdout("%s:%d", __func__, __LINE__);
-			return -ENOMEM;
-		}
-
-		segarr[i].block = mem;
-		segarr[i].sz = sz;
-		segarr[i].poison = poison;
-
-		for (j = zero; j < sz && can_loop; j++) {
-			mem[j] = poison;
-			if (mem[j] != poison) {
-				buddy_destroy(&buddy);
-				return -EINVAL;
-			}
-		}
-	}
-
-	/*
-	 * Go to (i * 17) % SEGARRLEN, and free the block pointed to.
-	 * Before freeing, check all bytes have the poisoned value
-	 * corresponding to the element. If any values are unexpected,
-	 * return an error. Skip some elements to test destroying the
-	 * buddy allocator while data is still allocated.
-	 */
-	for (i = 10; i < SEGARRLEN && can_loop; i++) {
-		idx = (i * 17) % SEGARRLEN;
-
-		mem = segarr[idx].block;
-		sz = segarr[idx].sz;
-		poison = segarr[idx].poison;
-
-		for (j = zero; j < sz && can_loop; j++) {
-			if (mem[j] != poison) {
-				buddy_destroy(&buddy);
-				arena_stdout("%s:%d %lx %u vs %u", __func__,
-					   __LINE__, (uintptr_t)&mem[j],
-					   mem[j], poison);
-				return -EINVAL;
-			}
-		}
-
-		buddy_free(&buddy, mem);
-	}
-
-	buddy_destroy(&buddy);
-
-	return 0;
-}
-
-SEC("syscall")
-__weak int test_buddy_alignment(void)
-{
-	int ret, i;
-
-	ret = buddy_init(&buddy);
-	if (ret)
-		return ret;
-
-	/* Allocate various sizes and check alignment */
-	for (i = zero; i < 17 && can_loop; i++) {
-		ptrs[i] = buddy_alloc(&buddy, alignment_sizes[i]);
-		if (!ptrs[i]) {
-			arena_stdout("alignment test: alloc failed for size %lu",
-				   alignment_sizes[i]);
-			buddy_destroy(&buddy);
-			return -ENOMEM;
-		}
-
-		/* Check 8-byte alignment */
-		if ((u64)ptrs[i] & 0x7) {
-			arena_stdout(
-				"alignment test: ptr %llx not 8-byte aligned (size %lu)",
-				(u64)ptrs[i], alignment_sizes[i]);
-			buddy_destroy(&buddy);
-			return -EINVAL;
-		}
-	}
-
-	/* Free all allocations */
-	for (i = zero; i < 17 && can_loop; i++)
-		buddy_free(&buddy, ptrs[i]);
-
-	buddy_destroy(&buddy);
-
-	return 0;
-}
-
-__weak char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_asan_buddy.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_asan_buddy.bpf.c
new file mode 100644
index 0000000000000..3266a28f53d76
--- /dev/null
+++ b/tools/testing/selftests/bpf/libarena/selftests/test_asan_buddy.bpf.c
@@ -0,0 +1,262 @@
+// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
+
+#include <libarena/common.h>
+#include <libarena/asan.h>
+#include <libarena/buddy.h>
+
+/* Required for parsing the ASAN call stacks. */
+#include "test_progs_compat.h"
+
+extern struct buddy __arena buddy;
+
+#ifdef BPF_ARENA_ASAN
+
+#include "test_asan_common.h"
+
+static __always_inline int asan_test_buddy_oob_single(size_t alloc_size)
+{
+	u8 __arena *mem;
+	int ret, i;
+
+	ret = asan_validate();
+	if (ret < 0)
+		return ret;
+
+	mem = buddy_alloc(&buddy, alloc_size);
+	if (!mem) {
+		arena_stdout("buddy_alloc failed for size %lu", alloc_size);
+		return -ENOMEM;
+	}
+
+	ret = asan_validate();
+	if (ret < 0)
+		return ret;
+
+	for (i = zero; i < alloc_size && can_loop; i++) {
+		mem[i] = 0xba;
+		ret = asan_validate_addr(false, &mem[i]);
+		if (ret < 0)
+			return ret;
+	}
+
+	mem[alloc_size] = 0xba;
+	ret = asan_validate_addr(true, &mem[alloc_size]);
+	if (ret < 0)
+		return ret;
+
+	buddy_free(&buddy, mem);
+
+	return 0;
+}
+
+/*
+ * Factored out because asan_validate_addr is complex enough to cause
+ * verification failures if verified with the rest of asan_test_buddy_uaf_single.
+ */
+__weak int asan_test_buddy_byte(u8 __arena *mem, int i, bool freed)
+{
+	int ret;
+
+	/* The header in freed blocks doesn't get poisoned. */
+	if (freed && BUDDY_HEADER_OFF <= i &&
+		i < BUDDY_HEADER_OFF + sizeof(struct buddy_header))
+		return 0;
+
+	mem[i] = 0xba;
+	ret = asan_validate_addr(freed, &mem[i]);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+__weak int asan_test_buddy_uaf_single(size_t alloc_size)
+{
+	u8 __arena *mem;
+	int ret;
+	int i;
+
+	mem = buddy_alloc(&buddy, alloc_size);
+	if (!mem) {
+		arena_stdout("buddy_alloc failed for size %lu", alloc_size);
+		return -ENOMEM;
+	}
+
+	ret = asan_validate();
+	if (ret < 0)
+		return ret;
+
+	for (i = zero; i < alloc_size && can_loop; i++) {
+		ret = asan_test_buddy_byte(mem, i, false);
+		if (ret)
+			return ret;
+	}
+
+	ret = asan_validate();
+	if (ret < 0)
+		return ret;
+
+	buddy_free(&buddy, mem);
+
+	for (i = zero; i < alloc_size && can_loop; i++) {
+		ret = asan_test_buddy_byte(mem, i, true);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+struct buddy_blob {
+	volatile u8 mem[48];
+	u8 oob;
+};
+
+static __always_inline int asan_test_buddy_blob_single(void)
+{
+	volatile struct buddy_blob __arena *blob;
+	const size_t alloc_size = sizeof(struct buddy_blob) - 1;
+	int ret;
+
+	blob = buddy_alloc(&buddy, alloc_size);
+	if (!blob)
+		return -ENOMEM;
+
+	blob->mem[0] = 0xba;
+	ret = asan_validate_addr(false, &blob->mem[0]);
+	if (ret < 0)
+		return ret;
+
+	blob->mem[47] = 0xba;
+	ret = asan_validate_addr(false, &blob->mem[47]);
+	if (ret < 0)
+		return ret;
+
+	blob->oob = 0;
+	ret = asan_validate_addr(true, &blob->oob);
+	if (ret < 0)
+		return ret;
+
+	buddy_free(&buddy, (void __arena *)blob);
+
+	return 0;
+}
+
+SEC("syscall")
+__stderr("Memory violation for address {{.*}} for write of size 1")
+__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}")
+__stderr("Call trace:\n"
+"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n"
+"|[ \t]+[^\n]+\n)*}}")
+__weak int asan_test_buddy_oob(void)
+{
+	size_t sizes[] = {
+		7, 8, 17, 18, 64, 256, 317, 512, 1024,
+	};
+	int ret;
+	u32 i;
+
+	ret = buddy_init(&buddy);
+	if (ret) {
+		arena_stdout("buddy_init failed with %d", ret);
+		return ret;
+	}
+
+	for (i = zero; i < sizeof(sizes) / sizeof(sizes[0]) && can_loop; i++) {
+		barrier_var(i);
+		ret = asan_test_buddy_oob_single(sizes[i]);
+		if (ret) {
+			arena_stdout("%s:%d Failed for size %lu", __func__,
+				   __LINE__, sizes[i]);
+			buddy_destroy(&buddy);
+			return ret;
+		}
+	}
+
+	buddy_destroy(&buddy);
+
+	ret = asan_validate();
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+SEC("syscall")
+__stderr("Memory violation for address {{.*}} for write of size 1")
+__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}")
+__stderr("Call trace:\n"
+"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n"
+"|[ \t]+[^\n]+\n)*}}")
+__weak int asan_test_buddy_uaf(void)
+{
+	size_t sizes[] = { 16, 32, 64, 128, 256, 512, 1024, 16384 };
+	int ret;
+	u32 i;
+
+	ret = buddy_init(&buddy);
+	if (ret) {
+		arena_stdout("buddy_init failed with %d", ret);
+		return ret;
+	}
+
+	for (i = zero; i < sizeof(sizes) / sizeof(sizes[0]) && can_loop; i++) {
+		barrier_var(i);
+		ret = asan_test_buddy_uaf_single(sizes[i]);
+		if (ret) {
+			arena_stdout("%s:%d Failed for size %lu", __func__,
+				   __LINE__, sizes[i]);
+			buddy_destroy(&buddy);
+			return ret;
+		}
+	}
+
+	buddy_destroy(&buddy);
+
+	ret = asan_validate();
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+SEC("syscall")
+__stderr("Memory violation for address {{.*}} for write of size 1")
+__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}")
+__stderr("Call trace:\n"
+"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n"
+"|[ \t]+[^\n]+\n)*}}")
+__weak int asan_test_buddy_blob(void)
+{
+	const int iters = 10;
+	int ret, i;
+
+	ret = buddy_init(&buddy);
+	if (ret) {
+		arena_stdout("buddy_init failed with %d", ret);
+		return ret;
+	}
+
+	for (i = zero; i < iters && can_loop; i++) {
+		ret = asan_test_buddy_blob_single();
+		if (ret) {
+			arena_stdout("%s:%d Failed on iteration %d", __func__,
+				   __LINE__, i);
+			buddy_destroy(&buddy);
+			return ret;
+		}
+	}
+
+	buddy_destroy(&buddy);
+
+	ret = asan_validate();
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+#endif
+
+__weak char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_asan_common.h b/tools/testing/selftests/bpf/libarena/selftests/test_asan_common.h
new file mode 100644
index 0000000000000..34a7918cb4cf6
--- /dev/null
+++ b/tools/testing/selftests/bpf/libarena/selftests/test_asan_common.h
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
+
+#pragma once
+
+#define ST_PAGES 64
+
+static inline void print_asan_map_state(void __arena *addr)
+{
+	arena_stdout("%s:%d ASAN %p -> (val: %x gran: %x set: [%s])",
+			__func__, __LINE__, addr,
+			*(s8 __arena *)(addr), ASAN_GRANULE(addr),
+			asan_shadow_set(addr) ? "yes" : "no");
+}
+
+/*
+ * Emit an error and force the current function to exit if the ASAN
+ * violation state is unexpected. Reset the violation state after.
+ */
+static inline int asan_validate_addr(bool cond, void __arena *addr)
+{
+	if ((asan_violated != 0) == cond) {
+		asan_violated = 0;
+		return 0;
+	}
+
+	arena_stdout("%s:%d ASAN asan_violated %lx", __func__, __LINE__,
+			(u64)asan_violated);
+	print_asan_map_state(addr);
+
+	asan_violated = 0;
+
+	return -EINVAL;
+}
+
+static inline int asan_validate(void)
+{
+	if (!asan_violated)
+		return 0;
+
+	arena_stdout("%s:%d Found ASAN violation at %lx", __func__, __LINE__,
+			asan_violated);
+
+	asan_violated = 0;
+
+	return -EINVAL;
+}
+
+struct blob {
+	volatile u8 mem[59];
+	u8 oob;
+};
diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c
new file mode 100644
index 0000000000000..76319a529f02b
--- /dev/null
+++ b/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c
@@ -0,0 +1,394 @@
+#include <libarena/common.h>
+
+#include <libarena/asan.h>
+#include <libarena/bitmap.h>
+
+#define TEST_BITS	(2 * BITS_PER_LONG_LONG)
+#define TEST_WORDS	BITS_TO_LONG_LONGS(TEST_BITS)
+#define MID_BIT		(BITS_PER_LONG_LONG + 1)
+#define LAST_BIT	(TEST_BITS - 1)
+
+static void test_bmp_setall(struct arena_bitmap __arena *bmp)
+{
+	volatile u32 i;
+
+	for (i = zero; i < TEST_WORDS && can_loop; i++)
+		bmp->bits[i] = ~0ULL;
+}
+
+SEC("syscall")
+__weak int test_bitmap_alloc_free(void)
+{
+	struct arena_bitmap __arena *bmp;
+
+	bmp = bmp_alloc(TEST_BITS);
+	if (!bmp)
+		return -ENOMEM;
+
+	if (!bmp_empty(TEST_BITS, bmp))
+		goto err;
+
+	__bmp_set_bit(LAST_BIT, bmp);
+	if (!bmp_test_bit(LAST_BIT, bmp))
+		goto err;
+
+	__bmp_clear_bit(LAST_BIT, bmp);
+	if (bmp_test_bit(LAST_BIT, bmp))
+		goto err;
+
+	bmp_free(bmp);
+	return 0;
+
+err:
+	bmp_free(bmp);
+	return -EINVAL;
+}
+
+SEC("syscall")
+__weak int test_bitmap_bit_ops(void)
+{
+	struct arena_bitmap __arena *bmp;
+
+	bmp = bmp_alloc(TEST_BITS);
+	if (!bmp)
+		return -ENOMEM;
+
+	__bmp_set_bit(0, bmp);
+	if (!bmp_test_bit(0, bmp))
+		goto err;
+
+	__bmp_set_bit(MID_BIT, bmp);
+	if (!bmp_test_bit(MID_BIT, bmp))
+		goto err;
+
+	__bmp_set_bit(LAST_BIT, bmp);
+	if (!bmp_test_bit(LAST_BIT, bmp))
+		goto err;
+
+	if (bmp_test_bit(MID_BIT - 1, bmp))
+		goto err;
+
+	__bmp_clear_bit(MID_BIT, bmp);
+	if (bmp_test_bit(MID_BIT, bmp))
+		goto err;
+
+	if (!bmp_test_bit(0, bmp))
+		goto err;
+
+	if (!bmp_test_bit(LAST_BIT, bmp))
+		goto err;
+
+	__bmp_clear_bit(0, bmp);
+	__bmp_clear_bit(LAST_BIT, bmp);
+	if (!bmp_empty(TEST_BITS, bmp))
+		goto err;
+
+	if (bmp->bits[0])
+		goto err;
+
+	if (bmp->bits[1])
+		goto err;
+
+	bmp_free(bmp);
+	return 0;
+
+err:
+	bmp_free(bmp);
+	return -EINVAL;
+}
+
+static bool test_bitmap_test_and_clear_single(struct arena_bitmap __arena *bmp, size_t ind)
+{
+	if (bmp_test_and_clear_bit(ind, bmp))
+		return false;
+
+	__bmp_set_bit(ind, bmp);
+
+	if (!bmp_test_and_clear_bit(ind, bmp))
+		return false;
+
+	if (bmp_test_bit(ind, bmp))
+		return false;
+
+	if (bmp_test_and_clear_bit(ind, bmp))
+		return false;
+
+	return true;
+}
+
+static bool test_bitmap_test_and_set_single(struct arena_bitmap __arena *bmp, size_t ind)
+{
+	if (bmp_test_and_set_bit(ind, bmp))
+		return false;
+
+	if (!bmp_test_and_set_bit(ind, bmp))
+		return false;
+
+	if (!bmp_test_bit(ind, bmp))
+		return false;
+
+	__bmp_clear_bit(ind, bmp);
+
+	if (bmp_test_and_set_bit(ind, bmp))
+		return false;
+
+	return true;
+}
+
+SEC("syscall")
+__weak int test_bitmap_test_and_clear_bit(void)
+{
+	struct arena_bitmap __arena *bmp;
+
+	bmp = bmp_alloc(TEST_BITS);
+	if (!bmp)
+		return -ENOMEM;
+
+	if (!test_bitmap_test_and_clear_single(bmp, 0))
+		goto err;
+
+	if (!test_bitmap_test_and_clear_single(bmp, MID_BIT))
+		goto err;
+
+	if (!test_bitmap_test_and_clear_single(bmp, LAST_BIT))
+		goto err;
+
+	if (!bmp_empty(TEST_BITS, bmp))
+		goto err;
+
+	bmp_free(bmp);
+	return 0;
+
+err:
+	bmp_free(bmp);
+	return -EINVAL;
+}
+
+SEC("syscall")
+__weak int test_bitmap_test_and_set_bit(void)
+{
+	struct arena_bitmap __arena *bmp;
+
+	bmp = bmp_alloc(TEST_BITS);
+	if (!bmp)
+		return -ENOMEM;
+
+	if (!test_bitmap_test_and_set_single(bmp, 0))
+		goto err;
+
+	if (!test_bitmap_test_and_set_single(bmp, MID_BIT))
+		goto err;
+
+	if (!test_bitmap_test_and_set_single(bmp, LAST_BIT))
+		goto err;
+
+	bmp_free(bmp);
+	return 0;
+
+err:
+	bmp_free(bmp);
+	return -EINVAL;
+}
+
+
+SEC("syscall")
+__weak int test_bitmap_and(void)
+{
+	struct arena_bitmap __arena *src1 = NULL, *src2 = NULL, *dst = NULL;
+
+	src1 = bmp_alloc(TEST_BITS);
+	src2 = bmp_alloc(TEST_BITS);
+	dst = bmp_alloc(TEST_BITS);
+	if (!src1 || !src2 || !dst)
+		goto err;
+
+	test_bmp_setall(dst);
+
+	__bmp_set_bit(0, src1);
+	__bmp_set_bit(MID_BIT, src1);
+	__bmp_set_bit(LAST_BIT, src1);
+
+	__bmp_set_bit(MID_BIT, src2);
+	__bmp_set_bit(LAST_BIT, src2);
+
+	bmp_and(TEST_BITS, dst, src1, src2);
+
+	if (bmp_test_bit(0, dst))
+		goto err;
+	if (!bmp_test_bit(MID_BIT, dst))
+		goto err;
+	if (!bmp_test_bit(LAST_BIT, dst))
+		goto err;
+
+	if (dst->bits[0])
+		goto err;
+	if (dst->bits[1] != (BIT_MASK(MID_BIT) | BIT_MASK(LAST_BIT)))
+		goto err;
+
+	bmp_free(src1);
+	bmp_free(src2);
+	bmp_free(dst);
+	return 0;
+
+err:
+	bmp_free(src1);
+	bmp_free(src2);
+	bmp_free(dst);
+	return -EINVAL;
+}
+
+SEC("syscall")
+__weak int test_bitmap_or(void)
+{
+	struct arena_bitmap __arena *src1 = NULL, *src2 = NULL, *dst = NULL;
+
+	src1 = bmp_alloc(TEST_BITS);
+	src2 = bmp_alloc(TEST_BITS);
+	dst = bmp_alloc(TEST_BITS);
+	if (!src1 || !src2 || !dst)
+		goto err;
+
+	test_bmp_setall(dst);
+
+	__bmp_set_bit(0, src1);
+	__bmp_set_bit(LAST_BIT, src1);
+
+	__bmp_set_bit(MID_BIT, src2);
+	__bmp_set_bit(LAST_BIT, src2);
+
+	bmp_or(TEST_BITS, dst, src1, src2);
+
+	if (!bmp_test_bit(0, dst))
+		goto err;
+	if (!bmp_test_bit(MID_BIT, dst))
+		goto err;
+	if (!bmp_test_bit(LAST_BIT, dst))
+		goto err;
+
+	if (dst->bits[0] != BIT_MASK(0))
+		goto err;
+	if (dst->bits[1] != (BIT_MASK(MID_BIT) | BIT_MASK(LAST_BIT)))
+		goto err;
+
+	bmp_free(src1);
+	bmp_free(src2);
+	bmp_free(dst);
+	return 0;
+
+err:
+	bmp_free(src1);
+	bmp_free(src2);
+	bmp_free(dst);
+	return -EINVAL;
+}
+
+SEC("syscall")
+__weak int test_bitmap_subset(void)
+{
+	struct arena_bitmap __arena *big = NULL, *small = NULL;
+
+	big = bmp_alloc(TEST_BITS);
+	small = bmp_alloc(TEST_BITS);
+	if (!big || !small)
+		goto err;
+
+	if (!bmp_subset(TEST_BITS, big, small))
+		goto err;
+
+	__bmp_set_bit(0, small);
+	if (bmp_subset(TEST_BITS, big, small))
+		goto err;
+
+	__bmp_set_bit(0, big);
+	if (!bmp_subset(TEST_BITS, big, small))
+		goto err;
+
+	__bmp_set_bit(LAST_BIT, small);
+	if (bmp_subset(TEST_BITS, big, small))
+		goto err;
+
+	__bmp_set_bit(LAST_BIT, big);
+	__bmp_set_bit(MID_BIT, big);
+	if (!bmp_subset(TEST_BITS, big, small))
+		goto err;
+
+	if (bmp_subset(TEST_BITS, small, big))
+		goto err;
+
+	bmp_free(big);
+	bmp_free(small);
+	return 0;
+
+err:
+	bmp_free(big);
+	bmp_free(small);
+	return -EINVAL;
+
+}
+
+SEC("syscall")
+__weak int test_bitmap_intersects(void)
+{
+	struct arena_bitmap __arena *arg1 = NULL, *arg2 = NULL;
+
+	arg1 = bmp_alloc(TEST_BITS);
+	arg2 = bmp_alloc(TEST_BITS);
+	if (!arg1 || !arg2)
+		goto err;
+
+	if (bmp_intersects(TEST_BITS, arg1, arg2))
+		goto err;
+
+	__bmp_set_bit(0, arg1);
+	__bmp_set_bit(MID_BIT, arg2);
+	if (bmp_intersects(TEST_BITS, arg1, arg2))
+		goto err;
+
+	__bmp_set_bit(LAST_BIT, arg1);
+	__bmp_set_bit(LAST_BIT, arg2);
+	if (!bmp_intersects(TEST_BITS, arg1, arg2))
+		goto err;
+
+	bmp_free(arg1);
+	bmp_free(arg2);
+	return 0;
+
+err:
+	bmp_free(arg1);
+	bmp_free(arg2);
+	return -EINVAL;
+}
+
+SEC("syscall")
+__weak int test_bitmap_copy(void)
+{
+	struct arena_bitmap __arena *arg1 = NULL, *arg2 = NULL;
+
+	arg1 = bmp_alloc(TEST_BITS);
+	arg2 = bmp_alloc(TEST_BITS);
+	if (!arg1 || !arg2)
+		goto err;
+
+	__bmp_set_bit(0, arg1);
+	__bmp_set_bit(MID_BIT, arg1);
+
+	/* Make sure those get overwritten. */
+	__bmp_set_bit(1, arg2);
+	__bmp_set_bit(MID_BIT + 2, arg2);
+
+	bmp_copy(TEST_BITS, arg2, arg1);
+
+	/* Bitmaps are equal if a subset of each other. */
+	if (!bmp_subset(TEST_BITS, arg1, arg2) ||
+	    !bmp_subset(TEST_BITS, arg2, arg1))
+		goto err;
+
+	bmp_free(arg1);
+	bmp_free(arg2);
+	return 0;
+
+err:
+	bmp_free(arg1);
+	bmp_free(arg2);
+	return -EINVAL;
+}
diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_buddy.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_buddy.bpf.c
new file mode 100644
index 0000000000000..5628f09870121
--- /dev/null
+++ b/tools/testing/selftests/bpf/libarena/selftests/test_buddy.bpf.c
@@ -0,0 +1,213 @@
+// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
+
+#include <libarena/common.h>
+
+#include <libarena/asan.h>
+#include <libarena/buddy.h>
+
+extern struct buddy __arena buddy;
+
+struct segarr_entry {
+	u8 __arena *block;
+	size_t sz;
+	u8 poison;
+};
+
+#define SEGARRLEN (512)
+static struct segarr_entry __arena segarr[SEGARRLEN];
+static void __arena *ptrs[17];
+size_t __arena alloc_sizes[] = { 3, 17, 1025, 129, 16350, 333, 9, 517 };
+size_t __arena alloc_multiple_sizes[] = { 3, 17, 1025, 129, 16350, 333, 9, 517, 2099 };
+size_t __arena alloc_free_sizes[] = { 3, 17, 64, 129, 256, 333, 512, 517 };
+size_t __arena alignment_sizes[] = { 1, 3, 7, 8, 9, 15, 16, 17, 31,
+				     32, 64, 100, 128, 255, 256, 512, 1000 };
+
+SEC("syscall")
+__weak int test_buddy_create(void)
+{
+	const int iters = 10;
+	int ret, i;
+
+	for (i = zero; i < iters && can_loop; i++) {
+		ret = buddy_init(&buddy);
+		if (ret)
+			return ret;
+
+		ret = buddy_destroy(&buddy);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+SEC("syscall")
+__weak int test_buddy_alloc(void)
+{
+	void __arena *mem;
+	int ret, i;
+
+	for (i = zero; i < 8 && can_loop; i++) {
+		ret = buddy_init(&buddy);
+		if (ret)
+			return ret;
+
+		mem = buddy_alloc(&buddy, alloc_sizes[i]);
+		if (!mem) {
+			buddy_destroy(&buddy);
+			return -ENOMEM;
+		}
+
+		buddy_destroy(&buddy);
+	}
+
+	return 0;
+}
+
+SEC("syscall")
+__weak int test_buddy_alloc_free(void)
+{
+	const int iters = 800;
+	void __arena *mem;
+	int ret, i;
+
+	ret = buddy_init(&buddy);
+	if (ret)
+		return ret;
+
+	for (i = zero; i < iters && can_loop; i++) {
+		mem = buddy_alloc(&buddy, alloc_free_sizes[(i * 5) % 8]);
+		if (!mem) {
+			buddy_destroy(&buddy);
+			return -ENOMEM;
+		}
+
+		buddy_free(&buddy, mem);
+	}
+
+	buddy_destroy(&buddy);
+
+	return 0;
+}
+
+SEC("syscall")
+__weak int test_buddy_alloc_multiple(void)
+{
+	int ret, j;
+	u32 i, idx;
+	u8 __arena *mem;
+	size_t sz;
+	u8 poison;
+
+	ret = buddy_init(&buddy);
+	if (ret)
+		return ret;
+
+	/*
+	 * Cycle through each size, allocating an entry in the
+	 * segarr. Continue for SEGARRLEN iterations. For every
+	 * allocation write down the size, use the current index
+	 * as a poison value, and log it with the pointer in the
+	 * segarr entry. Use the poison value to poison the entire
+	 * allocated memory according to the size given.
+	 */
+	for (i = zero; i < SEGARRLEN && can_loop; i++) {
+		sz = alloc_multiple_sizes[i % 9];
+		poison = (u8)i;
+
+		mem = buddy_alloc(&buddy, sz);
+		if (!mem) {
+			buddy_destroy(&buddy);
+			arena_stdout("%s:%d", __func__, __LINE__);
+			return -ENOMEM;
+		}
+
+		segarr[i].block = mem;
+		segarr[i].sz = sz;
+		segarr[i].poison = poison;
+
+		for (j = zero; j < sz && can_loop; j++) {
+			mem[j] = poison;
+			if (mem[j] != poison) {
+				buddy_destroy(&buddy);
+				return -EINVAL;
+			}
+		}
+	}
+
+	/*
+	 * Go to (i * 17) % SEGARRLEN, and free the block pointed to.
+	 * Before freeing, check all bytes have the poisoned value
+	 * corresponding to the element. If any values are unexpected,
+	 * return an error. Skip some elements to test destroying the
+	 * buddy allocator while data is still allocated.
+	 */
+	for (i = 10; i < SEGARRLEN && can_loop; i++) {
+		idx = (i * 17) % SEGARRLEN;
+
+		mem = segarr[idx].block;
+		sz = segarr[idx].sz;
+		poison = segarr[idx].poison;
+
+		for (j = zero; j < sz && can_loop; j++) {
+			if (mem[j] != poison) {
+				buddy_destroy(&buddy);
+				arena_stdout("%s:%d %lx %u vs %u", __func__,
+					   __LINE__, (uintptr_t)&mem[j],
+					   mem[j], poison);
+				return -EINVAL;
+			}
+		}
+
+		buddy_free(&buddy, mem);
+	}
+
+	buddy_destroy(&buddy);
+
+	return 0;
+}
+
+SEC("syscall")
+__weak int test_buddy_alignment(void)
+{
+	int ret;
+	u32 i;
+
+	ret = buddy_init(&buddy);
+	if (ret)
+		return ret;
+
+	/* Allocate various sizes and check alignment */
+	for (i = zero; i < 17 && can_loop; i++) {
+		barrier_var(i);
+		ptrs[i] = buddy_alloc(&buddy, alignment_sizes[i]);
+		if (!ptrs[i]) {
+			arena_stdout("alignment test: alloc failed for size %lu",
+				   alignment_sizes[i]);
+			buddy_destroy(&buddy);
+			return -ENOMEM;
+		}
+
+		/* Check 8-byte alignment */
+		if ((u64)ptrs[i] & 0x7) {
+			arena_stdout(
+				"alignment test: ptr %llx not 8-byte aligned (size %lu)",
+				(u64)ptrs[i], alignment_sizes[i]);
+			buddy_destroy(&buddy);
+			return -EINVAL;
+		}
+	}
+
+	/* Free all allocations */
+	for (i = zero; i < 17 && can_loop; i++) {
+		barrier_var(i);
+		buddy_free(&buddy, ptrs[i]);
+	}
+
+	buddy_destroy(&buddy);
+
+	return 0;
+}
+
+__weak char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c
new file mode 100644
index 0000000000000..ea1fac95b461e
--- /dev/null
+++ b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c
@@ -0,0 +1,190 @@
+// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
+
+#include <bpf_atomic.h>
+
+#include <libarena/common.h>
+
+#include <libarena/asan.h>
+#include <libarena/bitmap.h>
+
+#define TEST_BITMAP_THREADS	2
+#define TEST_BITMAP_BITS	(2 * BITS_PER_LONG_LONG)
+#define TEST_BITMAP_SYNC_SPINS	BPF_MAX_LOOPS
+#define TEST_BITMAP_ITERS	10 * 1000 * 1000
+
+static struct arena_bitmap __arena *bitmap;
+static volatile u64 started;
+static volatile bool test_abort;
+
+/*
+ * The test needs cmpxchg atomics on arena memory.
+ */
+#if defined(ENABLE_ATOMICS_TESTS) &&				  \
+	(defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
+	 defined(__TARGET_ARCH_s390) || \
+	 defined(__TARGET_ARCH_powerpc) ||				  \
+	 (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64))
+static bool bitmap_tests_enabled(void)
+{
+	return true;
+}
+#else
+static bool bitmap_tests_enabled(void)
+{
+	return false;
+}
+#endif
+
+__weak
+int bitmap_wait_for_start(void)
+{
+	u64 i;
+
+	__sync_fetch_and_add(&started, 1);
+
+	for (i = zero; i < TEST_BITMAP_SYNC_SPINS && can_loop; i++) {
+		if (test_abort)
+			return -EINTR;
+		if (smp_load_acquire(&started) >= TEST_BITMAP_THREADS)
+			return 0;
+	}
+
+	test_abort = true;
+	return -ETIMEDOUT;
+}
+
+/*
+ * The test makes sure writes don't clobber each other by overwriting
+ * the same word. One thread always writes on even bits, the other on
+ * odds. Both should be able to operate on the bitmap oblivious of the
+ * other's operations.
+ */
+__weak
+int bitmap_test_bit_sequence(u32 bit)
+{
+	if (bmp_test_and_clear_bit(bit, bitmap))
+		return -EINVAL;
+
+	if (bmp_test_and_set_bit(bit, bitmap))
+		return -EINVAL;
+	if (!bmp_test_bit(bit, bitmap))
+		return -EINVAL;
+
+	if (!bmp_test_and_set_bit(bit, bitmap))
+		return -EINVAL;
+	if (!bmp_test_bit(bit, bitmap))
+		return -EINVAL;
+
+	if (!bmp_test_and_clear_bit(bit, bitmap))
+		return -EINVAL;
+	if (bmp_test_bit(bit, bitmap))
+		return -EINVAL;
+
+	if (bmp_test_and_clear_bit(bit, bitmap))
+		return -EINVAL;
+
+	bmp_set_bit(bit, bitmap);
+	if (!bmp_test_bit(bit, bitmap))
+		return -EINVAL;
+
+	bmp_clear_bit(bit, bitmap);
+	if (bmp_test_bit(bit, bitmap))
+		return -EINVAL;
+
+	bmp_set_bit(bit, bitmap);
+	if (!bmp_test_bit(bit, bitmap))
+		return -EINVAL;
+
+	return 0;
+
+}
+
+static void bitmap_test_reset_single(int parity)
+{
+	u32 bit;
+
+	for (bit = parity; bit < TEST_BITMAP_BITS && can_loop; bit += 2)
+		bmp_clear_bit(bit, bitmap);
+
+}
+
+static int bitmap_test_common_single(int parity)
+{
+	u32 bit;
+	int ret;
+
+	for (bit = parity; bit < TEST_BITMAP_BITS && can_loop; bit += 2) {
+		if (test_abort)
+			return -EINTR;
+
+		ret = bitmap_test_bit_sequence(bit);
+		if (ret) {
+			test_abort = true;
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int bitmap_test_common(int parity)
+{
+	int ret;
+	u32 i;
+
+	arena_subprog_init();
+
+	ret = bitmap_wait_for_start();
+	if (ret)
+		return ret;
+
+	for (i = zero; i < TEST_BITMAP_ITERS && can_loop; i++) {
+		ret = bitmap_test_common_single(parity);
+		if (ret)
+			return ret;
+
+		if (test_abort)
+			break;
+
+		bitmap_test_reset_single(parity);
+	}
+
+	return 0;
+}
+
+SEC("syscall") int parallel_test_bitmap__enabled(void)
+{
+	return bitmap_tests_enabled() ? 0 : -EOPNOTSUPP;
+}
+
+SEC("syscall") int parallel_test_bitmap__init(void)
+{
+	bitmap = bmp_alloc(TEST_BITMAP_BITS);
+	if (!bitmap)
+		return -ENOMEM;
+
+	return 0;
+}
+
+SEC("syscall") int parallel_test_bitmap__fini(void)
+{
+	int ret = 0;
+
+	if (!bitmap)
+		return -EINVAL;
+
+	bmp_free(bitmap);
+	bitmap = NULL;
+
+	return ret;
+}
+
+SEC("syscall") int parallel_test_bitmap__0(void)
+{
+	return bitmap_test_common(0);
+}
+
+SEC("syscall") int parallel_test_bitmap__1(void)
+{
+	return bitmap_test_common(1);
+}
diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_parallel_spmc.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_spmc.bpf.c
index f08f2a92e1946..5fa96eb740951 100644
--- a/tools/testing/selftests/bpf/libarena/selftests/test_parallel_spmc.bpf.c
+++ b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_spmc.bpf.c
@@ -155,7 +155,7 @@ int spmc_quiesce_on_owner(u64 epoch)
 {
 	u64 i;
 
-	bpf_for(i, 0, TEST_SPMC_SYNC_SPINS) {
+	for (i = zero; i < TEST_SPMC_SYNC_SPINS && can_loop; i++) {
 		if (test_abort)
 			return -EINTR;
 		if (smp_load_acquire(&owner_epoch) >= epoch)
@@ -175,8 +175,7 @@ int spmc_quiesce_on_stealer(u64 epoch)
 	int err = -ETIMEDOUT;
 
 	target = STEALER_EPOCH(epoch);
-	bpf_for(i, 0, TEST_SPMC_SYNC_SPINS) {
-
+	for (i = zero; i < TEST_SPMC_SYNC_SPINS && can_loop; i++) {
 		if (test_abort) {
 			err = -EINTR;
 			break;
@@ -391,7 +390,7 @@ int spmc_wait_for_stealers_to_start(u64 target)
 {
 	u64 i;
 
-	bpf_for(i, 0, TEST_SPMC_SYNC_SPINS) {
+	for (i = zero; i < TEST_SPMC_SYNC_SPINS && can_loop; i++) {
 		if (test_abort)
 			return -EINTR;
 		if (READ_ONCE(stealers_started) >= target)
@@ -537,7 +536,7 @@ static int spmc_wait_for_round_steals(u64 target)
 
 	arena_subprog_init();
 
-	bpf_for(i, 0, TEST_SPMC_SYNC_SPINS) {
+	for (i = zero; i < TEST_SPMC_SYNC_SPINS && can_loop; i++) {
 		if (test_abort)
 			return -EINTR;
 		if (round_steals >= target)
diff --git a/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c
new file mode 100644
index 0000000000000..5ff8e688ddc76
--- /dev/null
+++ b/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c
@@ -0,0 +1,245 @@
+// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
+/*
+ * Copyright (c) 2025-2026 Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2025-2026 Emil Tsalapatis <emil@etsalapatis.com>
+ */
+
+#include <libarena/common.h>
+
+#include <libarena/asan.h>
+#include <libarena/bitmap.h>
+
+__weak
+struct arena_bitmap __arena *bmp_alloc(size_t bits)
+{
+	struct arena_bitmap __arena *bmp;
+	size_t size = BITS_TO_LONG_LONGS(bits) * sizeof(bmp->bits[0]);
+
+	/* Assume long-aligned masks. */
+	if (bits % BITS_PER_LONG_LONG)
+		return NULL;
+
+	bmp = (struct arena_bitmap __arena *)arena_malloc(size);
+	if (!bmp)
+		return NULL;
+
+	bmp_clear(bits, bmp);
+
+	return bmp;
+}
+
+__weak
+void bmp_free(struct arena_bitmap __arena *bmp)
+{
+	arena_free(bmp);
+}
+
+__weak
+void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp)
+{
+	bmp->bits[BIT_WORD(bit)] |= BIT_MASK(bit);
+}
+
+__weak
+void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp)
+{
+	bmp->bits[BIT_WORD(bit)] &= ~BIT_MASK(bit);
+}
+
+__weak
+bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp)
+{
+	return bmp->bits[BIT_WORD(bit)] & BIT_MASK(bit);
+}
+
+__weak
+bool bmp_test_and_clear_bit(u32 bit, struct arena_bitmap __arena *bmp)
+{
+	u64 val = BIT_MASK(bit);
+	u32 idx = BIT_WORD(bit);
+	u64 old, new, actual;
+
+	do {
+		old = bmp->bits[idx];
+
+		if (!(old & val))
+			return false;
+
+		new = old & ~val;
+		actual = cmpxchg(&bmp->bits[idx], old, new);
+
+		if (actual == old)
+			return true;
+
+	} while (can_loop);
+
+	return false;
+}
+
+__weak
+bool bmp_test_and_set_bit(u32 bit, struct arena_bitmap __arena *bmp)
+{
+	u64 val = BIT_MASK(bit);
+	u32 idx = BIT_WORD(bit);
+	u64 old, new, actual;
+
+	do {
+		old = bmp->bits[idx];
+
+		if ((old & val))
+			return true;
+
+		new = old | val;
+		actual = cmpxchg(&bmp->bits[idx], old, new);
+
+		if (actual == old)
+			return false;
+
+	} while (can_loop);
+
+	return false;
+}
+
+__weak
+void bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp)
+{
+	u64 val = BIT_MASK(bit);
+	u32 idx = BIT_WORD(bit);
+	u64 old, new, actual;
+
+	do {
+		old = bmp->bits[idx];
+		new = old & ~val;
+		actual = cmpxchg(&bmp->bits[idx], old, new);
+
+	} while (actual != old && can_loop);
+}
+
+__weak
+void bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp)
+{
+	u64 val = BIT_MASK(bit);
+	u32 idx = BIT_WORD(bit);
+	u64 old, new, actual;
+
+	do {
+		old = bmp->bits[idx];
+		new = old | val;
+		actual = cmpxchg(&bmp->bits[idx], old, new);
+
+	} while (actual != old && can_loop);
+}
+
+__weak
+void bmp_clear(size_t bits, struct arena_bitmap __arena *bmp)
+{
+	size_t nwords = BITS_TO_LONG_LONGS(bits);
+	volatile u32 i;
+
+	for (i = zero; i < nwords && can_loop; i++)
+		bmp->bits[i] = 0;
+}
+
+static __always_inline u64 bmp_last_word_mask(size_t bits)
+{
+	u32 rem = bits % BITS_PER_LONG_LONG;
+
+	return rem ? (1ULL << rem) - 1 : ~0ULL;
+}
+
+__weak
+void bmp_and(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src1, struct arena_bitmap __arena *src2)
+{
+	size_t nwords = BITS_TO_LONG_LONGS(bits);
+	volatile u32 i;
+
+	for (i = zero; i < nwords && can_loop; i++)
+		dst->bits[i] = src1->bits[i] & src2->bits[i];
+
+	if (nwords && bits % BITS_PER_LONG_LONG)
+		dst->bits[nwords - 1] &= bmp_last_word_mask(bits);
+}
+
+__weak
+void bmp_or(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src1, struct arena_bitmap __arena *src2)
+{
+	size_t nwords = BITS_TO_LONG_LONGS(bits);
+	volatile u32 i;
+
+	for (i = zero; i < nwords && can_loop; i++)
+		dst->bits[i] = src1->bits[i] | src2->bits[i];
+
+	if (nwords && bits % BITS_PER_LONG_LONG)
+		dst->bits[nwords - 1] &= bmp_last_word_mask(bits);
+}
+
+__weak
+bool bmp_empty(size_t bits, struct arena_bitmap __arena *bmp)
+{
+	size_t nwords = BITS_TO_LONG_LONGS(bits);
+	volatile u32 i;
+
+	for (i = zero; i < nwords && can_loop; i++) {
+		u64 mask = (i == nwords - 1) ? bmp_last_word_mask(bits) : ~0ULL;
+
+		if (bmp->bits[i] & mask)
+			return false;
+	}
+
+	return true;
+}
+
+__weak
+void bmp_copy(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src)
+{
+	size_t nwords = BITS_TO_LONG_LONGS(bits);
+	volatile u32 i;
+
+	for (i = zero; i < nwords && can_loop; i++)
+		dst->bits[i] = src->bits[i];
+
+	if (nwords && bits % BITS_PER_LONG_LONG)
+		dst->bits[nwords - 1] &= bmp_last_word_mask(bits);
+}
+
+__weak
+bool bmp_subset(size_t bits, struct arena_bitmap __arena *big, struct arena_bitmap __arena *small)
+{
+	size_t nwords = BITS_TO_LONG_LONGS(bits);
+	volatile u32 i;
+
+	for (i = zero; i < nwords && can_loop; i++) {
+		u64 mask = (i == nwords - 1) ? bmp_last_word_mask(bits) : ~0ULL;
+
+		if (~big->bits[i] & small->bits[i] & mask)
+			return false;
+	}
+
+	return true;
+}
+
+__weak
+bool bmp_intersects(size_t bits, struct arena_bitmap __arena *arg1, struct arena_bitmap __arena *arg2)
+{
+	size_t nwords = BITS_TO_LONG_LONGS(bits);
+	volatile u32 i;
+
+	for (i = zero; i < nwords && can_loop; i++) {
+		u64 mask = (i == nwords - 1) ? bmp_last_word_mask(bits) : ~0ULL;
+
+		if (arg1->bits[i] & arg2->bits[i] & mask)
+			return true;
+	}
+
+	return false;
+}
+
+__weak
+void bmp_print(size_t bits, struct arena_bitmap __arena *bmp)
+{
+	size_t nwords = BITS_TO_LONG_LONGS(bits);
+	volatile u32 i;
+
+	for (i = zero; i < nwords && can_loop; i++)
+		arena_stderr("%016llx ", bmp->bits[i]);
+}
diff --git a/tools/testing/selftests/bpf/libarena/src/common.bpf.c b/tools/testing/selftests/bpf/libarena/src/common.bpf.c
index 50be57213dfb8..569f0f64d518b 100644
--- a/tools/testing/selftests/bpf/libarena/src/common.bpf.c
+++ b/tools/testing/selftests/bpf/libarena/src/common.bpf.c
@@ -4,9 +4,8 @@
 #include <libarena/asan.h>
 #include <libarena/buddy.h>
 
-const volatile u32 zero = 0;
-
 struct buddy __arena buddy;
+volatile u32 zero = 0;
 
 int arena_fls(__u64 word)
 {
@@ -38,6 +37,12 @@ __weak int arena_buddy_reset(void)
 	return buddy_init(&buddy);
 }
 
+SEC("syscall")
+__weak int arena_buddy_destroy(void)
+{
+	return buddy_destroy(&buddy);
+}
+
 __weak void __arena *arena_malloc(size_t size)
 {
 	return buddy_alloc(&buddy, size);
diff --git a/tools/testing/selftests/bpf/network_helpers.c b/tools/testing/selftests/bpf/network_helpers.c
index b82f572641b7d..db935a9d9fc1a 100644
--- a/tools/testing/selftests/bpf/network_helpers.c
+++ b/tools/testing/selftests/bpf/network_helpers.c
@@ -111,7 +111,7 @@ int start_server_addr(int type, const struct sockaddr_storage *addr, socklen_t a
 	if (settimeo(fd, opts->timeout_ms))
 		goto error_close;
 
-	if (type == SOCK_STREAM &&
+	if ((type & SOCK_TYPE_MASK) == SOCK_STREAM &&
 	    setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on))) {
 		log_err("Failed to enable SO_REUSEADDR");
 		goto error_close;
@@ -128,7 +128,7 @@ int start_server_addr(int type, const struct sockaddr_storage *addr, socklen_t a
 		goto error_close;
 	}
 
-	if (type == SOCK_STREAM) {
+	if ((type & SOCK_TYPE_MASK) == SOCK_STREAM) {
 		if (listen(fd, opts->backlog ? MAX(opts->backlog, 0) : 1) < 0) {
 			log_err("Failed to listed on socket");
 			goto error_close;
diff --git a/tools/testing/selftests/bpf/network_helpers.h b/tools/testing/selftests/bpf/network_helpers.h
index 79a010c88e11c..75133119c04a3 100644
--- a/tools/testing/selftests/bpf/network_helpers.h
+++ b/tools/testing/selftests/bpf/network_helpers.h
@@ -25,6 +25,11 @@ typedef __u16 __sum16;
 #define VIP_NUM 5
 #define MAGIC_BYTES 123
 
+/* include/linux/net.h */
+#ifndef SOCK_TYPE_MASK
+#define SOCK_TYPE_MASK 0xf
+#endif
+
 struct network_helper_opts {
 	int timeout_ms;
 	int proto;
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c
index fe30181e63367..eb05fc82f81ba 100644
--- a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c
+++ b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c
@@ -14,6 +14,7 @@
 #include "tcp_ca_incompl_cong_ops.skel.h"
 #include "tcp_ca_unsupp_cong_op.skel.h"
 #include "tcp_ca_kfunc.skel.h"
+#include "tcp_ca_untrusted_btf_write.skel.h"
 #include "bpf_cc_cubic.skel.h"
 
 static const unsigned int total_bytes = 10 * 1024 * 1024;
@@ -579,6 +580,15 @@ static void test_tcp_ca_kfunc(void)
 	tcp_ca_kfunc__destroy(skel);
 }
 
+static void test_untrusted_btf_write(void)
+{
+	struct tcp_ca_untrusted_btf_write *skel;
+
+	skel = tcp_ca_untrusted_btf_write__open_and_load();
+	ASSERT_ERR_PTR(skel, "tcp_ca_untrusted_btf_write__open_and_load");
+	tcp_ca_untrusted_btf_write__destroy(skel);
+}
+
 static void test_cc_cubic(void)
 {
 	struct cb_opts cb_opts = {
@@ -637,6 +647,8 @@ void test_bpf_tcp_ca(void)
 		test_link_replace();
 	if (test__start_subtest("tcp_ca_kfunc"))
 		test_tcp_ca_kfunc();
+	if (test__start_subtest("untrusted_btf_write"))
+		test_untrusted_btf_write();
 	if (test__start_subtest("cc_cubic"))
 		test_cc_cubic();
 	if (test__start_subtest("dctcp_autoattach_map"))
diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c
index 92c20803ea765..4a87d7163c8c7 100644
--- a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c
+++ b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c
@@ -335,18 +335,6 @@ static void test_fmod_ret_freplace(void)
 	bpf_object__close(pkt_obj);
 }
 
-
-static void test_func_sockmap_update(void)
-{
-	const char *prog_name[] = {
-		"freplace/cls_redirect",
-	};
-	test_fexit_bpf2bpf_common("./freplace_cls_redirect.bpf.o",
-				  "./test_cls_redirect.bpf.o",
-				  ARRAY_SIZE(prog_name),
-				  prog_name, false, NULL);
-}
-
 static void test_func_replace_void(void)
 {
 	const char *prog_name[] = {
@@ -599,8 +587,6 @@ void serial_test_fexit_bpf2bpf(void)
 		test_func_replace();
 	if (test__start_subtest("func_replace_verify"))
 		test_func_replace_verify();
-	if (test__start_subtest("func_sockmap_update"))
-		test_func_sockmap_update();
 	if (test__start_subtest("func_replace_return_code"))
 		test_func_replace_return_code();
 	if (test__start_subtest("func_map_prog_compatibility"))
diff --git a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c
index f589eefbf9fbd..0918321c8e63d 100644
--- a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c
+++ b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c
@@ -4,6 +4,7 @@
 #include <string.h>
 #include <linux/bpf.h>
 #include <linux/limits.h>
+#include <bpf/btf.h>
 #include <test_progs.h>
 #include "trace_helpers.h"
 #include "test_fill_link_info.skel.h"
@@ -24,6 +25,22 @@ static __u64 kmulti_cookies[] = { 3, 1, 2 };
 #define KPROBE_FUNC "bpf_fentry_test1"
 static __u64 kprobe_addr;
 
+static const char * const tmulti_syms[] = {
+	"bpf_fentry_test2",
+	"bpf_fentry_test1",
+	"bpf_fentry_test3",
+};
+
+static __u64 tmulti_cookies[] = { 30, 10, 20 };
+#define TRACING_MULTI_CNT ARRAY_SIZE(tmulti_syms)
+
+struct tmulti_target {
+	const char *name;
+	__u64 addr;
+	__u64 cookie;
+	__u32 id;
+};
+
 #define UPROBE_FILE "/proc/self/exe"
 static ssize_t uprobe_offset;
 /* uprobe attach point */
@@ -396,6 +413,224 @@ static void test_kprobe_multi_fill_link_info(struct test_fill_link_info *skel,
 	bpf_link__destroy(link);
 }
 
+static int tmulti_target_cmp(const void *a, const void *b)
+{
+	const struct tmulti_target *ta = a;
+	const struct tmulti_target *tb = b;
+
+	return (ta->id > tb->id) - (ta->id < tb->id);
+}
+
+static int setup_tmulti_targets(const struct bpf_program *prog,
+				struct tmulti_target *targets,
+				__u32 *btf_obj_id)
+{
+	struct bpf_prog_info prog_info;
+	__u32 len = sizeof(prog_info);
+	struct btf *btf;
+	int err, i;
+	__s32 id;
+
+	btf = btf__load_vmlinux_btf();
+	if (!ASSERT_OK_PTR(btf, "btf__load_vmlinux_btf"))
+		return -1;
+
+	for (i = 0; i < TRACING_MULTI_CNT; i++) {
+		id = btf__find_by_name_kind(btf, tmulti_syms[i], BTF_KIND_FUNC);
+		if (!ASSERT_GT(id, 0, "btf__find_by_name_kind"))
+			goto error;
+
+		targets[i].name = tmulti_syms[i];
+		targets[i].addr = ksym_get_addr(tmulti_syms[i]);
+		targets[i].cookie = tmulti_cookies[i];
+		targets[i].id = id;
+	}
+
+	memset(&prog_info, 0, len);
+	err = bpf_prog_get_info_by_fd(bpf_program__fd(prog), &prog_info, &len);
+	if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd"))
+		goto error;
+	if (!ASSERT_GT(prog_info.attach_btf_obj_id, 0, "attach_btf_obj_id"))
+		goto error;
+	*btf_obj_id = prog_info.attach_btf_obj_id;
+
+	/*
+	 * The kernel tracing multi attach sorts ids. We sort as well,
+	 * so we can easily compare ids and cookies later.
+	 */
+	qsort(targets, TRACING_MULTI_CNT, sizeof(targets[0]), tmulti_target_cmp);
+	btf__free(btf);
+	return 0;
+
+error:
+	btf__free(btf);
+	return -1;
+}
+
+static int verify_tracing_multi_link_info(int fd, const struct bpf_program *prog,
+					  const struct tmulti_target *targets,
+					  __u32 btf_obj_id, bool has_cookies)
+{
+	enum bpf_attach_type attach_type = bpf_program__expected_attach_type(prog);
+	__u64 addrs[TRACING_MULTI_CNT], cookies[TRACING_MULTI_CNT];
+	__u32 ids[TRACING_MULTI_CNT];
+	struct bpf_link_info info;
+	__u32 len = sizeof(info);
+	int err, i;
+
+	memset(&info, 0, sizeof(info));
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	if (!ASSERT_OK(err, "bpf_link_get_info_by_fd"))
+		return -1;
+
+	if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type"))
+		return -1;
+
+	ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type");
+	ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count");
+
+	memset(ids, 0, sizeof(ids));
+	memset(cookies, 0, sizeof(cookies));
+	memset(addrs, 0, sizeof(addrs));
+
+	info.tracing_multi.ids = ptr_to_u64(ids);
+	info.tracing_multi.addrs = ptr_to_u64(addrs);
+	info.tracing_multi.cookies = has_cookies ? ptr_to_u64(cookies) : 0;
+	info.tracing_multi.count = TRACING_MULTI_CNT;
+
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	if (!ASSERT_OK(err, "bpf_link_get_info_by_fd"))
+		return -1;
+
+	if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type"))
+		return -1;
+
+	ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type");
+	ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count");
+	ASSERT_EQ(info.tracing_multi.btf_obj_id, btf_obj_id, "tracing_multi.btf_obj_id");
+
+	for (i = 0; i < TRACING_MULTI_CNT; i++) {
+		ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids");
+		ASSERT_EQ(cookies[i], has_cookies ? targets[i].cookie : 0, "tracing_multi.cookies");
+
+		if (targets[i].addr) {
+			struct ksym *ksym;
+
+			if (!ASSERT_NEQ(addrs[i], 0, "tracing_multi.addrs"))
+				return -1;
+			ksym = ksym_search(addrs[i]);
+			if (!ASSERT_OK_PTR(ksym, "ksym_search"))
+				return -1;
+			ASSERT_STREQ(ksym->name, targets[i].name, "tracing_multi.addr_name");
+		} else {
+			ASSERT_EQ(addrs[i], 0, "tracing_multi.addrs");
+		}
+	}
+
+	return 0;
+}
+
+static void verify_tracing_multi_invalid_user_buffer(int fd, const struct tmulti_target *targets)
+{
+	__u32 ids[TRACING_MULTI_CNT] = {};
+	struct bpf_link_info info;
+	__u32 len = sizeof(info);
+	int err, i;
+
+	/* Wrong info setup (ids != NULL and cnt == 0) -> EINVAL */
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.ids = ptr_to_u64(ids);
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_EQ(err, -EINVAL, "tracing_multi.invalid_count");
+
+	/* Smaller than actual count provided -> ENOSPC */
+	memset(ids, 0, sizeof(ids));
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.ids = ptr_to_u64(ids);
+	info.tracing_multi.count = TRACING_MULTI_CNT - 1;
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_EQ(err, -ENOSPC, "tracing_multi.small_count");
+	for (i = 0; i < TRACING_MULTI_CNT - 1; i++)
+		ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.partial_ids");
+	/* check that the last entry is not populated */
+	ASSERT_EQ(ids[i], 0, "tracing_multi.partial_ids");
+
+	/* Bigger than actual count provided -> OK */
+	memset(ids, 0, sizeof(ids));
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.ids = ptr_to_u64(ids);
+	info.tracing_multi.count = TRACING_MULTI_CNT + 1;
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_OK(err, "tracing_multi.big_count");
+	for (i = 0; i < TRACING_MULTI_CNT; i++)
+		ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids");
+
+	/* Invalid ids pointer -> EFAULT */
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.ids = 0x1;
+	info.tracing_multi.count = TRACING_MULTI_CNT;
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_btf_ids");
+
+	/* Invalid cookies pointer -> EFAULT */
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.cookies = 0x1;
+	info.tracing_multi.count = TRACING_MULTI_CNT;
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_cookies");
+
+	/* Invalid addrs pointer -> EFAULT */
+	memset(&info, 0, sizeof(info));
+	info.tracing_multi.addrs = 0x1;
+	info.tracing_multi.count = TRACING_MULTI_CNT;
+	err = bpf_link_get_info_by_fd(fd, &info, &len);
+	ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_addrs");
+}
+
+static void test_tracing_multi_fill_link_info(struct test_fill_link_info *skel,
+					      bool has_cookies, bool invalid)
+{
+	LIBBPF_OPTS(bpf_tracing_multi_opts, opts);
+	struct tmulti_target targets[TRACING_MULTI_CNT];
+	__u32 ids[TRACING_MULTI_CNT], btf_obj_id;
+	__u64 cookies[TRACING_MULTI_CNT];
+	struct bpf_link *link;
+	int link_fd, err, i;
+
+#ifndef __x86_64__
+	test__skip();
+	return;
+#endif
+
+	if (setup_tmulti_targets(skel->progs.tmulti_run, targets, &btf_obj_id))
+		return;
+
+	for (i = 0; i < TRACING_MULTI_CNT; i++) {
+		ids[i] = targets[i].id;
+		cookies[i] = targets[i].cookie;
+	}
+
+	opts.ids = ids;
+	opts.cnt = TRACING_MULTI_CNT;
+	if (has_cookies)
+		opts.cookies = cookies;
+
+	link = bpf_program__attach_tracing_multi(skel->progs.tmulti_run, NULL, &opts);
+	if (!ASSERT_OK_PTR(link, "bpf_program__attach_tracing_multi"))
+		return;
+
+	link_fd = bpf_link__fd(link);
+	if (invalid) {
+		verify_tracing_multi_invalid_user_buffer(link_fd, targets);
+	} else {
+		err = verify_tracing_multi_link_info(link_fd, skel->progs.tmulti_run,
+						     targets, btf_obj_id, has_cookies);
+		ASSERT_OK(err, "verify_tracing_multi_link_info");
+	}
+
+	bpf_link__destroy(link);
+}
+
 #define SEC(name) __attribute__((section(name), used))
 
 static short uprobe_link_info_sema_1 SEC(".probes");
@@ -640,6 +875,13 @@ void test_fill_link_info(void)
 	if (test__start_subtest("kprobe_multi_invalid_ubuff"))
 		test_kprobe_multi_fill_link_info(skel, true, true, true);
 
+	if (test__start_subtest("tracing_multi_link_info")) {
+		test_tracing_multi_fill_link_info(skel, false, false);
+		test_tracing_multi_fill_link_info(skel, true, false);
+	}
+	if (test__start_subtest("tracing_multi_invalid_ubuff"))
+		test_tracing_multi_fill_link_info(skel, true, true);
+
 	if (test__start_subtest("uprobe_multi_link_info"))
 		test_uprobe_multi_fill_link_info(skel, false, false);
 	if (test__start_subtest("uretprobe_multi_link_info"))
diff --git a/tools/testing/selftests/bpf/prog_tests/get_smp_processor_id.c b/tools/testing/selftests/bpf/prog_tests/get_smp_processor_id.c
new file mode 100644
index 0000000000000..1b5c738ab81fb
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/get_smp_processor_id.c
@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <test_progs.h>
+#include "bpf/libbpf_internal.h"
+#include "get_smp_processor_id.skel.h"
+
+void test_get_smp_processor_id(void)
+{
+	LIBBPF_OPTS(bpf_test_run_opts, opts,
+		    .flags = BPF_F_TEST_RUN_ON_CPU,
+		    .cpu = 0,
+	);
+	struct get_smp_processor_id *skel;
+	int prog_fd, err, online_cpu_nr, i;
+	bool *online = NULL;
+
+	err = parse_cpu_mask_file("/sys/devices/system/cpu/online",
+				  &online, &online_cpu_nr);
+	if (!ASSERT_OK(err, "parse_cpu_mask_file"))
+		return;
+
+	skel = get_smp_processor_id__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "get_smp_processor_id__open_and_load"))
+		goto cleanup;
+
+	prog_fd = bpf_program__fd(skel->progs.call_bpf_get_smp_processor_id);
+
+	for (i = 0; i < online_cpu_nr; i++) {
+		if (!online[i])
+			continue;
+
+		opts.cpu = i;
+		skel->bss->cpu_nr_result = -1;
+
+		err = bpf_prog_test_run_opts(prog_fd, &opts);
+		if (!ASSERT_OK(err, "bpf_prog_test_run_opts"))
+			goto cleanup;
+
+		ASSERT_EQ(skel->bss->cpu_nr_result, opts.cpu, "cpu_nr_result");
+	}
+
+cleanup:
+	free(online);
+	get_smp_processor_id__destroy(skel);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/icmp_send_kfunc.c b/tools/testing/selftests/bpf/prog_tests/icmp_send_kfunc.c
new file mode 100644
index 0000000000000..9318d4bc7ce8b
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/icmp_send_kfunc.c
@@ -0,0 +1,292 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <test_progs.h>
+#include <network_helpers.h>
+#include <cgroup_helpers.h>
+#include <linux/errqueue.h>
+#include <poll.h>
+#include <unistd.h>
+#include "icmp_send.skel.h"
+
+#define TIMEOUT_MS 1000
+
+#define ICMP_DEST_UNREACH 3
+#define ICMPV6_DEST_UNREACH 1
+
+#define ICMP_HOST_UNREACH 1
+#define ICMP_FRAG_NEEDED 4
+#define NR_ICMP_UNREACH 15
+#define ICMPV6_REJECT_ROUTE 6
+
+#define KFUNC_RET_UNSET -1
+
+static int connect_to_fd_nonblock(int server_fd)
+{
+	struct sockaddr_storage addr;
+	socklen_t len = sizeof(addr);
+	int fd, err, on = 1;
+
+	if (getsockname(server_fd, (struct sockaddr *)&addr, &len))
+		return -1;
+
+	fd = socket(addr.ss_family, SOCK_STREAM | SOCK_NONBLOCK, 0);
+	if (fd < 0)
+		return -1;
+
+	if (addr.ss_family == AF_INET6 &&
+	    setsockopt(fd, IPPROTO_IPV6, IPV6_RECVERR, &on, sizeof(on)) < 0) {
+		close(fd);
+		return -1;
+	}
+
+	err = connect(fd, (struct sockaddr *)&addr, len);
+	if (err < 0 && errno != EINPROGRESS) {
+		close(fd);
+		return -1;
+	}
+
+	return fd;
+}
+
+static void read_icmp_errqueue(int sockfd, int expected_code, int af)
+{
+	int expected_ee_type = (af == AF_INET) ? ICMP_DEST_UNREACH :
+						 ICMPV6_DEST_UNREACH;
+	int expected_origin = (af == AF_INET) ? SO_EE_ORIGIN_ICMP :
+						SO_EE_ORIGIN_ICMP6;
+	int expected_level = (af == AF_INET) ? IPPROTO_IP : IPPROTO_IPV6;
+	int expected_type = (af == AF_INET) ? IP_RECVERR : IPV6_RECVERR;
+	struct sock_extended_err *sock_err;
+	char ctrl_buf[512];
+	struct msghdr msg = {
+		.msg_control = ctrl_buf,
+		.msg_controllen = sizeof(ctrl_buf),
+	};
+	struct pollfd pfd = {
+		.fd = sockfd,
+		.events = POLLERR,
+	};
+	struct cmsghdr *cm;
+	ssize_t n;
+
+	if (!ASSERT_GE(poll(&pfd, 1, TIMEOUT_MS), 1, "poll_errqueue"))
+		return;
+
+	n = recvmsg(sockfd, &msg, MSG_ERRQUEUE);
+	if (!ASSERT_GE(n, 0, "recvmsg_errqueue"))
+		return;
+
+	cm = CMSG_FIRSTHDR(&msg);
+	if (!ASSERT_NEQ(cm, NULL, "cm_firsthdr_null"))
+		return;
+
+	for (; cm; cm = CMSG_NXTHDR(&msg, cm)) {
+		if (cm->cmsg_level != expected_level ||
+		    cm->cmsg_type != expected_type)
+			continue;
+
+		sock_err = (struct sock_extended_err *)CMSG_DATA(cm);
+
+		if (!ASSERT_EQ(sock_err->ee_origin, expected_origin,
+			       "sock_err_origin"))
+			return;
+		if (!ASSERT_EQ(sock_err->ee_type, expected_ee_type,
+			       "sock_err_type_dest_unreach"))
+			return;
+		ASSERT_EQ(sock_err->ee_code, expected_code, "sock_err_code");
+		return;
+	}
+
+	ASSERT_FAIL("no IP_RECVERR/IPV6_RECVERR control message found");
+}
+
+static bool valid_unreach_code(int code, int af)
+{
+	if (code < 0)
+		return false;
+
+	if (af == AF_INET)
+		return code <= NR_ICMP_UNREACH && code != ICMP_FRAG_NEEDED;
+
+	return code <= ICMPV6_REJECT_ROUTE;
+}
+
+static void trigger_prog_read_icmp_errqueue(struct icmp_send *skel, int code,
+					    int af, const char *ip)
+{
+	int srv_fd = -1, client_fd = -1;
+	int port;
+
+	srv_fd = start_server(af, SOCK_STREAM, ip, 0, TIMEOUT_MS);
+	if (!ASSERT_OK_FD(srv_fd, "start_server"))
+		return;
+
+	port = get_socket_local_port(srv_fd);
+	if (!ASSERT_GE(port, 0, "get_socket_local_port")) {
+		close(srv_fd);
+		return;
+	}
+
+	skel->bss->server_port = ntohs(port);
+	skel->bss->unreach_type = (af == AF_INET) ? ICMP_DEST_UNREACH :
+						    ICMPV6_DEST_UNREACH;
+	skel->bss->unreach_code = code;
+	skel->data->kfunc_ret = KFUNC_RET_UNSET;
+
+	client_fd = connect_to_fd_nonblock(srv_fd);
+	if (!ASSERT_OK_FD(client_fd, "client_connect_nonblock")) {
+		close(srv_fd);
+		return;
+	}
+
+	if (valid_unreach_code(code, af))
+		read_icmp_errqueue(client_fd, code, af);
+
+	close(client_fd);
+	close(srv_fd);
+}
+
+static void run_icmp_test(struct icmp_send *skel, int af, const char *ip,
+			  int max_code)
+{
+	for (int code = 0; code <= max_code; code++) {
+		if (af == AF_INET && code == ICMP_FRAG_NEEDED)
+			continue;
+
+		trigger_prog_read_icmp_errqueue(skel, code, af, ip);
+		ASSERT_EQ(skel->data->kfunc_ret, 0, "kfunc_ret");
+	}
+
+	/* Test invalid codes */
+	trigger_prog_read_icmp_errqueue(skel, -1, af, ip);
+	ASSERT_EQ(skel->data->kfunc_ret, -EINVAL, "kfunc_ret");
+
+	trigger_prog_read_icmp_errqueue(skel, max_code + 1, af, ip);
+	ASSERT_EQ(skel->data->kfunc_ret, -EINVAL, "kfunc_ret");
+
+	if (af == AF_INET) {
+		trigger_prog_read_icmp_errqueue(skel, ICMP_FRAG_NEEDED, af, ip);
+		ASSERT_EQ(skel->data->kfunc_ret, -EINVAL, "kfunc_ret");
+	}
+}
+
+static void run_icmp_no_route_test(struct icmp_send *skel, int af)
+{
+	union {
+		struct ipv4_packet v4;
+		struct ipv6_packet v6;
+	} pkt;
+	DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts,
+		.data_in = &pkt,
+	);
+	int err;
+
+	switch (af) {
+	case AF_INET:
+		pkt.v4 = pkt_v4;
+		pkt.v4.iph.version = 4;
+		pkt.v4.iph.daddr = htonl(INADDR_LOOPBACK);
+		pkt.v4.tcp.dest = htons(80);
+		opts.data_size_in = sizeof(pkt.v4);
+		skel->bss->unreach_type = ICMP_DEST_UNREACH;
+		break;
+	case AF_INET6:
+		pkt.v6 = pkt_v6;
+		pkt.v6.iph.version = 6;
+		pkt.v6.iph.daddr = in6addr_loopback;
+		pkt.v6.tcp.dest = htons(80);
+		opts.data_size_in = sizeof(pkt.v6);
+		skel->bss->unreach_type = ICMPV6_DEST_UNREACH;
+		break;
+	default:
+		ASSERT_FAIL("af_not_supported");
+		return;
+	}
+
+	skel->bss->server_port = 80;
+	skel->data->kfunc_ret = KFUNC_RET_UNSET;
+
+	err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.egress), &opts);
+	if (!ASSERT_OK(err, "test_run"))
+		return;
+
+	ASSERT_EQ(skel->data->kfunc_ret, -ENETUNREACH, "kfunc_ret_no_route");
+}
+
+void test_icmp_send_unreach_cgroup(void)
+{
+	struct icmp_send *skel;
+	int cgroup_fd = -1;
+
+	skel = icmp_send__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "skel_open"))
+		goto cleanup;
+
+	cgroup_fd = test__join_cgroup("/icmp_send_unreach_cgroup");
+	if (!ASSERT_OK_FD(cgroup_fd, "join_cgroup"))
+		goto cleanup;
+
+	skel->links.egress =
+		bpf_program__attach_cgroup(skel->progs.egress, cgroup_fd);
+	if (!ASSERT_OK_PTR(skel->links.egress, "prog_attach_cgroup"))
+		goto cleanup;
+
+	if (test__start_subtest("ipv4"))
+		run_icmp_test(skel, AF_INET, "127.0.0.1", NR_ICMP_UNREACH);
+
+	if (test__start_subtest("ipv6"))
+		run_icmp_test(skel, AF_INET6, "::1", ICMPV6_REJECT_ROUTE);
+
+	if (test__start_subtest("no_route_ipv4"))
+		run_icmp_no_route_test(skel, AF_INET);
+
+	if (test__start_subtest("no_route_ipv6"))
+		run_icmp_no_route_test(skel, AF_INET6);
+
+cleanup:
+	icmp_send__destroy(skel);
+	if (cgroup_fd >= 0)
+		close(cgroup_fd);
+}
+
+void test_icmp_send_unreach_recursion(void)
+{
+	struct icmp_send *skel;
+	int cgroup_fd = -1;
+	int err;
+
+	err = setup_cgroup_environment();
+	if (!ASSERT_OK(err, "setup_cgroup_environment"))
+		return;
+
+	skel = icmp_send__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "skel_open"))
+		goto cleanup;
+
+	cgroup_fd = get_root_cgroup();
+	if (!ASSERT_OK_FD(cgroup_fd, "get_root_cgroup"))
+		goto cleanup;
+
+	skel->data->target_pid = getpid();
+	skel->links.recursion =
+		bpf_program__attach_cgroup(skel->progs.recursion, cgroup_fd);
+	if (!ASSERT_OK_PTR(skel->links.recursion, "prog_attach_cgroup"))
+		goto cleanup;
+
+	trigger_prog_read_icmp_errqueue(skel, ICMP_HOST_UNREACH, AF_INET,
+					"127.0.0.1");
+
+	/*
+	 * Because there's recursion involved, the first call will return at
+	 * index 1 since it will return the second, and the second call will
+	 * return at index 0 since it will return the first.
+	 */
+	ASSERT_EQ(skel->bss->rec_count, 2, "rec_count");
+	ASSERT_EQ(skel->data->rec_kfunc_rets[0], -EBUSY, "kfunc_rets[0]");
+	ASSERT_EQ(skel->data->rec_kfunc_rets[1], 0, "kfunc_rets[1]");
+
+cleanup:
+	icmp_send__destroy(skel);
+	if (cgroup_fd >= 0)
+		close(cgroup_fd);
+	cleanup_cgroup_environment();
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c
index 3df07680f9e06..67a30bf695090 100644
--- a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c
+++ b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c
@@ -66,6 +66,7 @@ static struct kfunc_test_params kfunc_tests[] = {
 	TC_FAIL(kfunc_call_test_get_mem_fail_rdonly, 0, "R0 cannot write into rdonly_mem"),
 	TC_FAIL(kfunc_call_test_get_mem_fail_use_after_free, 0, "invalid mem access 'scalar'"),
 	TC_FAIL(kfunc_call_test_get_mem_fail_oob, 0, "min value is outside of the allowed memory range"),
+	TC_FAIL(kfunc_call_test_get_mem_fail_oversized, 0, "rdonly/rdwr_buf_size exceeds u32 max"),
 	TC_FAIL(kfunc_call_test_get_mem_fail_not_const, 0, "is not a const"),
 	TC_FAIL(kfunc_call_test_mem_acquire_fail, 0, "acquire kernel function does not return PTR_TO_BTF_ID"),
 	TC_FAIL(kfunc_call_test_pointer_arg_type_mismatch, 0, "R1 expected pointer to ctx, but got scalar"),
diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_implicit_args_tracing.c b/tools/testing/selftests/bpf/prog_tests/kfunc_implicit_args_tracing.c
new file mode 100644
index 0000000000000..61cc5aaba0254
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/kfunc_implicit_args_tracing.c
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
+
+#include <test_progs.h>
+#include "kfunc_implicit_args_tracing.skel.h"
+
+void test_kfunc_implicit_args_tracing(void)
+{
+	struct kfunc_implicit_args_tracing *skel;
+	LIBBPF_OPTS(bpf_test_run_opts, topts);
+	int err, fd;
+
+	skel = kfunc_implicit_args_tracing__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "open_and_load"))
+		return;
+
+	err = kfunc_implicit_args_tracing__attach(skel);
+	if (!ASSERT_OK(err, "attach"))
+		goto cleanup;
+
+	fd = bpf_program__fd(skel->progs.trigger_implicit_arg);
+	err = bpf_prog_test_run_opts(fd, &topts);
+	if (!ASSERT_OK(err, "test_run"))
+		goto cleanup;
+
+	ASSERT_EQ(topts.retval, 5, "kfunc_retval");
+	ASSERT_EQ(skel->bss->fentry_arg_cnt, 2, "fentry_arg_cnt");
+	ASSERT_NEQ(skel->bss->fentry_aux_arg, 0, "fentry_aux_arg");
+	ASSERT_EQ(skel->bss->fentry_result, 1, "fentry_result");
+	ASSERT_EQ(skel->bss->fexit_arg_cnt, 2, "fexit_arg_cnt");
+	ASSERT_NEQ(skel->bss->fexit_aux_arg, 0, "fexit_aux_arg");
+	ASSERT_EQ(skel->bss->fexit_result, 1, "fexit_result");
+
+cleanup:
+	kfunc_implicit_args_tracing__destroy(skel);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/libarena.c b/tools/testing/selftests/bpf/prog_tests/libarena.c
index 61ea68dce4105..ba5a5a50f7c07 100644
--- a/tools/testing/selftests/bpf/prog_tests/libarena.c
+++ b/tools/testing/selftests/bpf/prog_tests/libarena.c
@@ -15,7 +15,12 @@ static void run_libarena_test(struct libarena *skel, struct bpf_program *prog,
 {
 	int ret;
 
-	if (!strstr(name, "test_buddy")) {
+	if (strstr(name, "test_buddy")) {
+		/* Buddy tests initialize the allocator directly. */
+		ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_destroy));
+		if (!ASSERT_OK(ret, "arena_buddy_destroy"))
+			return;
+	} else {
 		ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_reset));
 		if (!ASSERT_OK(ret, "arena_buddy_reset"))
 			return;
@@ -24,7 +29,6 @@ static void run_libarena_test(struct libarena *skel, struct bpf_program *prog,
 	ret = libarena_run_prog(bpf_program__fd(prog));
 
 	ASSERT_OK(ret, name);
-
 }
 
 static void *run_libarena_parallel_prog(void *arg)
diff --git a/tools/testing/selftests/bpf/prog_tests/libarena_asan.c b/tools/testing/selftests/bpf/prog_tests/libarena_asan.c
index d59d9dd12ef2b..f897405f701dd 100644
--- a/tools/testing/selftests/bpf/prog_tests/libarena_asan.c
+++ b/tools/testing/selftests/bpf/prog_tests/libarena_asan.c
@@ -17,7 +17,12 @@ static void run_libarena_asan_test(struct libarena_asan *skel,
 {
 	int ret;
 
-	if (!strstr(name, "test_buddy")) {
+	if (strstr(name, "test_buddy")) {
+		/* Buddy tests initialize the allocator directly. */
+		ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_destroy));
+		if (!ASSERT_OK(ret, "arena_buddy_destroy"))
+			return;
+	} else {
 		ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_reset));
 		if (!ASSERT_OK(ret, "arena_buddy_reset"))
 			return;
@@ -90,4 +95,3 @@ void test_libarena_asan(void)
 
 	return;
 }
-
diff --git a/tools/testing/selftests/bpf/prog_tests/linked_list.c b/tools/testing/selftests/bpf/prog_tests/linked_list.c
index 8defea0253ed9..c3d133c6a00d4 100644
--- a/tools/testing/selftests/bpf/prog_tests/linked_list.c
+++ b/tools/testing/selftests/bpf/prog_tests/linked_list.c
@@ -68,6 +68,7 @@ static struct {
 	{ "obj_type_id_oor", "local type ID argument must be in range [0, U32_MAX]" },
 	{ "obj_new_no_composite", "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct" },
 	{ "obj_new_no_struct", "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct" },
+	{ "obj_new_flex_array", "access beyond struct obj_new_flex" },
 	{ "obj_drop_non_zero_off", "R1 must have zero offset when passed to release func" },
 	{ "new_null_ret", "R0 invalid mem access 'ptr_or_null_'" },
 	{ "obj_new_acq", "Unreleased reference id=" },
diff --git a/tools/testing/selftests/bpf/prog_tests/mptcp.c b/tools/testing/selftests/bpf/prog_tests/mptcp.c
index 8fade8bdc4516..32dfc1c511af6 100644
--- a/tools/testing/selftests/bpf/prog_tests/mptcp.c
+++ b/tools/testing/selftests/bpf/prog_tests/mptcp.c
@@ -264,7 +264,7 @@ static int verify_mptcpify(int server_fd, int client_fd)
 	return err;
 }
 
-static int run_mptcpify(int cgroup_fd)
+static int run_mptcpify(int cgroup_fd, int type)
 {
 	int server_fd, client_fd, err = 0;
 	struct mptcpify *mptcpify_skel;
@@ -280,7 +280,7 @@ static int run_mptcpify(int cgroup_fd)
 		goto out;
 
 	/* without MPTCP */
-	server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0);
+	server_fd = start_server(AF_INET, type, NULL, 0, 0);
 	if (!ASSERT_GE(server_fd, 0, "start_server")) {
 		err = -EIO;
 		goto out;
@@ -317,7 +317,14 @@ static void test_mptcpify(void)
 	if (!ASSERT_OK_PTR(netns, "netns_new"))
 		goto fail;
 
-	ASSERT_OK(run_mptcpify(cgroup_fd), "run_mptcpify");
+	ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM), "run_mptcpify");
+	/* userspace sets flags such as SOCK_CLOEXEC together with the type;
+	 * the BPF prog must still upgrade the socket to MPTCP. See
+	 * update_socket_protocol() in net/socket.c, which runs before the
+	 * type is masked with SOCK_TYPE_MASK.
+	 */
+	ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM | SOCK_CLOEXEC),
+		  "run_mptcpify_cloexec");
 
 fail:
 	netns_free(netns);
diff --git a/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_bad_access.c b/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_bad_access.c
new file mode 100644
index 0000000000000..b8538fc4fc3f6
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_bad_access.c
@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <test_progs.h>
+#include "test_kmods/bpf_testmod.h"
+#include "bpf_util.h"
+
+static void check_attach_reject(const struct bpf_insn *program, size_t prog_len)
+{
+	LIBBPF_OPTS(bpf_prog_load_opts, opts);
+	char error[4096];
+	int bpf_fd, tp_fd;
+
+	opts.log_level = 2;
+	opts.log_buf = error;
+	opts.log_size = sizeof(error);
+
+	bpf_fd = bpf_prog_load(BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, NULL, "GPL v2",
+			       program, prog_len, &opts);
+	if (!ASSERT_GE(bpf_fd, 0, "prog_load"))
+		return;
+
+	tp_fd = bpf_raw_tracepoint_open("bpf_testmod_test_writable_bare_tp", bpf_fd);
+	ASSERT_EQ(tp_fd, -EINVAL, "bpf_raw_tracepoint_open");
+	if (tp_fd >= 0)
+		close(tp_fd);
+
+	close(bpf_fd);
+}
+
+void test_raw_tp_writable_reject_bad_access(void)
+{
+	const struct bpf_insn program[] = {
+		/* r6 is our tp buffer */
+		BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, 0),
+		/* one byte beyond the end of the writable context */
+		BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_6,
+			    sizeof(struct bpf_testmod_test_writable_ctx)),
+		BPF_EXIT_INSN(),
+	};
+
+	const struct bpf_insn negative_var_off_program[] = {
+		BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, 0),
+		/* make var_off negative, but keep the effective access offset non-negative */
+		BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, -8),
+		/* one byte beyond the end of the writable context */
+		BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_6,
+			    sizeof(struct bpf_testmod_test_writable_ctx) + 8),
+		BPF_EXIT_INSN(),
+	};
+
+	if (test__start_subtest("past_end"))
+		check_attach_reject(program, ARRAY_SIZE(program));
+
+	if (test__start_subtest("negative_var_off_past_end"))
+		check_attach_reject(negative_var_off_program,
+				    ARRAY_SIZE(negative_var_off_program));
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_nbd_invalid.c b/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_nbd_invalid.c
deleted file mode 100644
index 216b0dfac0fe6..0000000000000
--- a/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_nbd_invalid.c
+++ /dev/null
@@ -1,43 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-#include <test_progs.h>
-#include <linux/nbd.h>
-#include "bpf_util.h"
-
-void test_raw_tp_writable_reject_nbd_invalid(void)
-{
-	__u32 duration = 0;
-	char error[4096];
-	int bpf_fd = -1, tp_fd = -1;
-
-	const struct bpf_insn program[] = {
-		/* r6 is our tp buffer */
-		BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, 0),
-		/* one byte beyond the end of the nbd_request struct */
-		BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_6,
-			    sizeof(struct nbd_request)),
-		BPF_EXIT_INSN(),
-	};
-
-	LIBBPF_OPTS(bpf_prog_load_opts, opts,
-		.log_level = 2,
-		.log_buf = error,
-		.log_size = sizeof(error),
-	);
-
-	bpf_fd = bpf_prog_load(BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, NULL, "GPL v2",
-			       program, ARRAY_SIZE(program),
-			       &opts);
-	if (CHECK(bpf_fd < 0, "bpf_raw_tracepoint_writable load",
-		  "failed: %d errno %d\n", bpf_fd, errno))
-		return;
-
-	tp_fd = bpf_raw_tracepoint_open("nbd_send_request", bpf_fd);
-	if (CHECK(tp_fd >= 0, "bpf_raw_tracepoint_writable open",
-		  "erroneously succeeded\n"))
-		goto out_bpffd;
-
-	close(tp_fd);
-out_bpffd:
-	close(bpf_fd);
-}
diff --git a/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c b/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c
index f0a8c828f8f10..7541f4966abc5 100644
--- a/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c
+++ b/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c
@@ -3,6 +3,8 @@
 #include <test_progs.h>
 #include <network_helpers.h>
 #include <sys/sysinfo.h>
+#include <sys/syscall.h>
+#include <linux/perf_event.h>
 
 #include "res_spin_lock.skel.h"
 #include "res_spin_lock_fail.skel.h"
@@ -102,11 +104,29 @@ void test_res_spin_lock_success(void)
 
 void serial_test_res_spin_lock_stress(void)
 {
+	struct perf_event_attr attr = {
+		.size = sizeof(attr),
+		.type = PERF_TYPE_HARDWARE,
+		.config = PERF_COUNT_HW_CPU_CYCLES,
+	};
+	int pmu_fd;
+
 	if (libbpf_num_possible_cpus() < 3) {
 		test__skip();
 		return;
 	}
 
+	pmu_fd = syscall(__NR_perf_event_open, &attr, 0, -1, -1, 0);
+	if (pmu_fd < 0) {
+		if (errno == ENOENT || errno == EOPNOTSUPP) {
+			test__skip();
+			return;
+		}
+		ASSERT_OK(-errno, "perf_event_open pmu probe");
+		return;
+	}
+	close(pmu_fd);
+
 	ASSERT_OK(load_module("bpf_test_rqspinlock.ko", false), "load module AA");
 	sleep(5);
 	unload_module("bpf_test_rqspinlock", false);
diff --git a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c
index 41dfaaabb73f4..ac51fd4548214 100644
--- a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c
+++ b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c
@@ -10,7 +10,11 @@
 #include <linux/btf_ids.h>
 #include "test_progs.h"
 
-static int duration;
+#define BTF_DATA_FILE "resolve_btfids.test.o.BTF"
+
+#ifndef KF_FASTCALL
+#define KF_FASTCALL (1 << 12)
+#endif
 
 struct symbol {
 	const char	*name;
@@ -28,12 +32,41 @@ struct symbol test_symbols[] = {
 	{ "func",    BTF_KIND_FUNC,    -1 },
 };
 
+struct kfunc_symbol {
+	const char	*name;
+	s32		 id;
+	u32		 flags;
+};
+
+static struct kfunc_symbol kfunc_symbols[] = {
+	{ "kfunc_a", -1, 0 },
+	{ "kfunc_b", -1, KF_FASTCALL },
+};
+
 /* Align the .BTF_ids section to 4 bytes */
 asm (
 ".pushsection " BTF_IDS_SECTION " ,\"a\"; \n"
 ".balign 4, 0;                            \n"
 ".popsection;                             \n");
 
+/*
+ * test_list_local, test_set and test_kfunc_set are .local symbols placed
+ * in .BTF_ids by inline asm, and are read here directly by C name. To the
+ * compiler they are plain, default-visibility extern objects.
+ *
+ * When test_progs is linked as a position-independent executable (PIE),
+ * taking the address of such an extern is routed through the GOT. The
+ * GNU assembler on aarch64 unconditionally converts references to .local
+ * symbols into section + addend form (".BTF_ids + <offset>"), but a GOT
+ * slot cannot carry an addend (the AArch64 ELF spec mandates zero), so
+ * the linker resolves it to the .BTF_ids base.
+ *
+ * Mark them hidden so the compiler treats them as non-interposable and
+ * emits a direct, addend-preserving PC-relative access instead of a GOT
+ * load, in both PIE and non-PIE builds. test_list_global is .globl and
+ * not affected, so it is left at default visibility.
+ */
+#pragma GCC visibility push(hidden)
 BTF_ID_LIST(test_list_local)
 BTF_ID_UNUSED
 BTF_ID(typedef, S)
@@ -43,24 +76,39 @@ BTF_ID(struct,  S)
 BTF_ID(union,   U)
 BTF_ID(func,    func)
 
-extern __u32 test_list_global[];
-BTF_ID_LIST_GLOBAL(test_list_global, 1)
-BTF_ID_UNUSED
+BTF_SET_START(test_set)
 BTF_ID(typedef, S)
 BTF_ID(typedef, T)
 BTF_ID(typedef, U)
 BTF_ID(struct,  S)
 BTF_ID(union,   U)
 BTF_ID(func,    func)
+BTF_SET_END(test_set)
 
-BTF_SET_START(test_set)
+BTF_KFUNCS_START(test_kfunc_set)
+BTF_ID_FLAGS(func, kfunc_a)
+BTF_ID_FLAGS(func, kfunc_b, KF_FASTCALL)
+BTF_KFUNCS_END(test_kfunc_set)
+
+/*
+ * Same kfuncs in reverse declaration order, so resolve_btfids has to
+ * actually sort at least one of the two sets.
+ */
+BTF_KFUNCS_START(test_kfunc_set_rev)
+BTF_ID_FLAGS(func, kfunc_b, KF_FASTCALL)
+BTF_ID_FLAGS(func, kfunc_a)
+BTF_KFUNCS_END(test_kfunc_set_rev)
+#pragma GCC visibility pop
+
+extern __u32 test_list_global[];
+BTF_ID_LIST_GLOBAL(test_list_global, 1)
+BTF_ID_UNUSED
 BTF_ID(typedef, S)
 BTF_ID(typedef, T)
 BTF_ID(typedef, U)
 BTF_ID(struct,  S)
 BTF_ID(union,   U)
 BTF_ID(func,    func)
-BTF_SET_END(test_set)
 
 static int
 __resolve_symbol(struct btf *btf, int type_id)
@@ -70,10 +118,10 @@ __resolve_symbol(struct btf *btf, int type_id)
 	unsigned int i;
 
 	type = btf__type_by_id(btf, type_id);
-	if (!type) {
-		PRINT_FAIL("Failed to get type for ID %d\n", type_id);
+	if (!ASSERT_OK_PTR(type, "btf__type_by_id"))
 		return -1;
-	}
+
+	str = btf__name_by_offset(btf, type->name_off);
 
 	for (i = 0; i < ARRAY_SIZE(test_symbols); i++) {
 		if (test_symbols[i].id >= 0)
@@ -82,64 +130,80 @@ __resolve_symbol(struct btf *btf, int type_id)
 		if (BTF_INFO_KIND(type->info) != test_symbols[i].type)
 			continue;
 
-		str = btf__name_by_offset(btf, type->name_off);
-		if (!str) {
-			PRINT_FAIL("Failed to get name for BTF ID %d\n", type_id);
-			return -1;
-		}
-
 		if (!strcmp(str, test_symbols[i].name))
 			test_symbols[i].id = type_id;
 	}
 
+	if (!btf_is_func(type))
+		return 0;
+
+	for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) {
+		if (kfunc_symbols[i].id >= 0)
+			continue;
+		if (!strcmp(str, kfunc_symbols[i].name))
+			kfunc_symbols[i].id = type_id;
+	}
+
 	return 0;
 }
 
-static int resolve_symbols(void)
+static int resolve_symbols(struct btf *btf)
 {
-	struct btf *btf;
+	__u32 nr = btf__type_cnt(btf);
 	int type_id;
-	__u32 nr;
-
-	btf = btf__parse_raw("resolve_btfids.test.o.BTF");
-	if (CHECK(libbpf_get_error(btf), "resolve",
-		  "Failed to load BTF from resolve_btfids.test.o.BTF\n"))
-		return -1;
-
-	nr = btf__type_cnt(btf);
 
 	for (type_id = 1; type_id < nr; type_id++) {
 		if (__resolve_symbol(btf, type_id))
-			break;
+			return -1;
 	}
-
-	btf__free(btf);
 	return 0;
 }
 
+static void check_kfunc_set(struct btf_id_set8 *set)
+{
+	unsigned int i, j;
+
+	ASSERT_EQ(set->flags, BTF_SET8_KFUNCS, "kfunc_set_flags");
+	ASSERT_EQ(set->cnt, ARRAY_SIZE(kfunc_symbols), "kfunc_set_cnt");
+
+	for (i = 0; i < set->cnt; i++) {
+		for (j = 0; j < ARRAY_SIZE(kfunc_symbols); j++) {
+			if (kfunc_symbols[j].id == (s32)set->pairs[i].id) {
+				ASSERT_EQ(set->pairs[i].flags,
+					  kfunc_symbols[j].flags, "kfunc_flags_check");
+				break;
+			}
+		}
+
+		ASSERT_TRUE(j < ARRAY_SIZE(kfunc_symbols), "kfunc_id_found");
+
+		if (i > 0) {
+			ASSERT_LE(set->pairs[i - 1].id,
+				  set->pairs[i].id, "kfunc_sort_check");
+		}
+	}
+}
+
 void test_resolve_btfids(void)
 {
 	__u32 *test_list, *test_lists[] = { test_list_local, test_list_global };
 	unsigned int i, j;
-	int ret = 0;
+	struct btf *btf;
 
-	if (resolve_symbols())
+	btf = btf__parse_raw(BTF_DATA_FILE);
+	if (!ASSERT_OK_PTR(btf, "btf_parse"))
 		return;
 
+	if (resolve_symbols(btf))
+		goto out;
+
 	/* Check BTF_ID_LIST(test_list_local) and
 	 * BTF_ID_LIST_GLOBAL(test_list_global) IDs
 	 */
 	for (j = 0; j < ARRAY_SIZE(test_lists); j++) {
 		test_list = test_lists[j];
-		for (i = 0; i < ARRAY_SIZE(test_symbols); i++) {
-			ret = CHECK(test_list[i] != test_symbols[i].id,
-				    "id_check",
-				    "wrong ID for %s (%d != %d)\n",
-				    test_symbols[i].name,
-				    test_list[i], test_symbols[i].id);
-			if (ret)
-				return;
-		}
+		for (i = 0; i < ARRAY_SIZE(test_symbols); i++)
+			ASSERT_EQ(test_list[i], test_symbols[i].id, test_symbols[i].name);
 	}
 
 	/* Check BTF_SET_START(test_set) IDs */
@@ -153,15 +217,16 @@ void test_resolve_btfids(void)
 			break;
 		}
 
-		ret = CHECK(!found, "id_check",
-			    "ID %d not found in test_symbols\n",
-			    test_set.ids[i]);
-		if (ret)
+		if (!ASSERT_TRUE(found, "id_in_test_symbols"))
 			break;
 
-		if (i > 0) {
-			if (!ASSERT_LE(test_set.ids[i - 1], test_set.ids[i], "sort_check"))
-				return;
-		}
+		if (i > 0)
+			ASSERT_LE(test_set.ids[i - 1], test_set.ids[i], "sort_check");
 	}
+
+	check_kfunc_set(&test_kfunc_set);
+	check_kfunc_set(&test_kfunc_set_rev);
+
+out:
+	btf__free(btf);
 }
diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c
index 5fc417e31fc61..77381d345435c 100644
--- a/tools/testing/selftests/bpf/prog_tests/signed_loader.c
+++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c
@@ -11,6 +11,8 @@
 #include <linux/keyctl.h>
 #include <linux/bpf.h>
 
+#include <bpf/btf.h>
+
 #include "bpf/libbpf_internal.h" /* for libbpf_sha256() */
 #include "bpf/skel_internal.h"	 /* for loader ctx layout (bpf_loader_ctx etc) */
 
@@ -19,8 +21,6 @@
 #include "test_signed_loader_data.skel.h"
 #include "test_signed_loader_lsm.skel.h"
 
-#define SIG_MATCH_INSNS 33 /* excl (5) + 4 * sha-dword (7) */
-
 enum {
 	BPF_SIG_UNSIGNED = 0,
 	BPF_SIG_VERIFIED,
@@ -35,7 +35,8 @@ enum {
 };
 
 static int load_loader(const void *insns, __u32 insns_sz, int map_fd,
-		       const void *sig, __u32 sig_sz, __s32 keyring_id)
+		       const void *sig, __u32 sig_sz, __s32 keyring_id,
+		       __u32 fd_array_cnt)
 {
 	union bpf_attr attr;
 	int fd;
@@ -52,6 +53,7 @@ static int load_loader(const void *insns, __u32 insns_sz, int map_fd,
 		attr.signature_size = sig_sz;
 		attr.keyring_id = keyring_id;
 	}
+	attr.fd_array_cnt = fd_array_cnt;
 	memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog"));
 	fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr,
 		     offsetofend(union bpf_attr, keyring_id));
@@ -62,14 +64,12 @@ static int run_gen_loader(const void *insns, __u32 insns_sz,
 			  const void *data, __u32 data_sz,
 			  const void *excl, __u32 excl_sz,
 			  const void *sig, __u32 sig_sz,
-			  bool get_hash, void *ctx, __u32 ctx_sz, bool *loader_ran)
+			  void *ctx, __u32 ctx_sz, bool *loader_ran)
 {
 	LIBBPF_OPTS(bpf_map_create_opts, mopts,
 		    .excl_prog_hash = excl,
 		    .excl_prog_hash_size = excl_sz);
-	__u8 hbuf[SHA256_DIGEST_LENGTH];
-	struct bpf_map_info info;
-	__u32 ilen = sizeof(info), key = 0;
+	__u32 key = 0;
 	union bpf_attr attr;
 	int map_fd, prog_fd, ret;
 
@@ -87,15 +87,6 @@ static int run_gen_loader(const void *insns, __u32 insns_sz,
 		ret = -errno;
 		goto out_map;
 	}
-	if (get_hash) {
-		memset(&info, 0, sizeof(info));
-		info.hash = ptr_to_u64(hbuf);
-		info.hash_size = sizeof(hbuf);
-		if (bpf_map_get_info_by_fd(map_fd, &info, &ilen)) {
-			ret = -errno;
-			goto out_map;
-		}
-	}
 
 	memset(&attr, 0, sizeof(attr));
 	attr.prog_type = BPF_PROG_TYPE_SYSCALL;
@@ -108,6 +99,7 @@ static int run_gen_loader(const void *insns, __u32 insns_sz,
 		attr.signature = ptr_to_u64(sig);
 		attr.signature_size = sig_sz;
 		attr.keyring_id = KEY_SPEC_SESSION_KEYRING;
+		attr.fd_array_cnt = 1;
 	}
 	memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog"));
 	prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr,
@@ -236,79 +228,6 @@ static int sign_buf(const char *dir, const void *buf, __u32 len,
 	return ret;
 }
 
-static void check_sig_match_shape(const struct bpf_insn *in, int n)
-{
-	int a = -1, cleanup = -1, i, base, t, br[5], nb = 0;
-
-	/* BPF_PSEUDO_MAP_IDX (the struct bpf_map * form) is used only here. */
-	for (i = 0; i + 1 < n; i++) {
-		if (in[i].code == (BPF_LD | BPF_IMM | BPF_DW) &&
-		    in[i].src_reg == BPF_PSEUDO_MAP_IDX) {
-			a = i;
-			break;
-		}
-	}
-	if (!ASSERT_GE(a, 0, "emit_signature_match present"))
-		return;
-	if (!ASSERT_LE(a + SIG_MATCH_INSNS, n, "block fits in program"))
-		return;
-
-	/* excl check: r2 = *(u32 *)(map + 32); if r2 != 1 goto cleanup */
-	ASSERT_EQ(in[a + 2].code, (BPF_LDX | BPF_MEM | BPF_W), "excl load width");
-	ASSERT_EQ(in[a + 2].off, SHA256_DIGEST_LENGTH, "excl field offset");
-	ASSERT_EQ(in[a + 4].code, (BPF_JMP | BPF_JNE | BPF_K), "excl branch op");
-	ASSERT_EQ(in[a + 4].imm, 1, "excl compared to 1");
-	br[nb++] = a + 4;
-
-	/* 4 sha-dword checks: r2 = *(u64 *)(map + i*8); if r2 != r3 goto cleanup */
-	for (i = 0; i < 4; i++) {
-		base = a + 5 + i * 7;
-		ASSERT_EQ(in[base + 2].code, (BPF_LDX | BPF_MEM | BPF_DW), "sha load width");
-		ASSERT_EQ(in[base + 2].off, i * 8, "sha dword offset");
-		ASSERT_EQ(in[base + 3].code, (BPF_LD | BPF_IMM | BPF_DW), "sha imm64 (H_meta)");
-		ASSERT_EQ(in[base + 6].code, (BPF_JMP | BPF_JNE | BPF_X), "sha branch op");
-		br[nb++] = base + 6;
-	}
-
-	/*
-	 * Locate the real cleanup label so we can pin the exact jump target,
-	 * not just "some backward label". bpf_gen__init() emits the cleanup
-	 * block as a prog-fd close loop whose first instruction is the label
-	 * every error branch jumps to.
-	 */
-	for (i = 0; i + 2 < a; i++) {
-		if (in[i].code == (BPF_LDX | BPF_MEM | BPF_W) &&
-		    in[i].dst_reg == BPF_REG_1 && in[i].src_reg == BPF_REG_10 &&
-		    in[i + 1].code == (BPF_JMP | BPF_JSLE | BPF_K) &&
-		    in[i + 1].dst_reg == BPF_REG_1 && in[i + 1].imm == 0 &&
-		    in[i + 1].off == 1 &&
-		    in[i + 2].code == (BPF_JMP | BPF_CALL) &&
-		    in[i + 2].imm == BPF_FUNC_sys_close) {
-			cleanup = i;
-			break;
-		}
-	}
-	if (!ASSERT_GE(cleanup, 0, "cleanup label located"))
-		return;
-	for (i = 0; i < nb; i++) {
-		t = br[i] + 1 + in[br[i]].off;
-		ASSERT_EQ(t, cleanup, "sig-match lands on cleanup");
-	}
-	/*
-	 * Same invariant for every other cleanup-bound jump in the program:
-	 * emit_check_err() is the only source of "if (r7 < 0) goto cleanup",
-	 * so each of those must also resolve exactly to cleanup.
-	 */
-	for (i = 0, t = 0; i < n; i++) {
-		if (in[i].code != (BPF_JMP | BPF_JSLT | BPF_K) ||
-		    in[i].dst_reg != BPF_REG_7 || in[i].imm != 0 || in[i].off >= 0)
-			continue;
-		ASSERT_EQ(i + 1 + in[i].off, cleanup, "err-check lands on cleanup");
-		t++;
-	}
-	ASSERT_GT(t, 0, "found emit_check_err jumps");
-}
-
 struct gen_loader_fixture {
 	struct test_signed_loader *skel;
 	struct gen_loader_opts gopts;
@@ -372,16 +291,6 @@ static void gen_loader_fixture_fini(struct gen_loader_fixture *f)
 	test_signed_loader__destroy(f->skel);
 }
 
-static void metadata_check_shape(void)
-{
-	struct gen_loader_fixture f;
-
-	if (gen_loader_fixture_init(&f) == 0)
-		check_sig_match_shape((const struct bpf_insn *)f.gopts.insns,
-				      f.gopts.insns_sz / sizeof(struct bpf_insn));
-	gen_loader_fixture_fini(&f);
-}
-
 static void metadata_match(void)
 {
 	struct gen_loader_fixture f;
@@ -391,94 +300,265 @@ static void metadata_match(void)
 	if (gen_loader_fixture_init(&f) == 0) {
 		r = run_gen_loader(f.gopts.insns, f.gopts.insns_sz, f.blob,
 				   f.data_sz, f.excl, sizeof(f.excl), NULL, 0,
-				   true, f.ctx, f.ctx_sz, &ran);
+				   f.ctx, f.ctx_sz, &ran);
 		ASSERT_TRUE(ran, "loader ran");
 		ASSERT_EQ(r, 0, "honest loader retval");
 	}
 	gen_loader_fixture_fini(&f);
 }
 
-static void metadata_sha_mismatch(void)
+static void signature_enforced(void)
 {
+	static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, };
 	struct gen_loader_fixture f;
-	bool ran;
-	int r;
+	int fd;
 
 	if (gen_loader_fixture_init(&f) == 0) {
 		/*
-		 * blob[0] lives in the loader's fd_array scratch (first add_data in
-		 * bpf_gen__init); a 0-map program never reads it, so flipping it
-		 * changes only map->sha. The metadata check is the only thing that
-		 * can notice -> isolates emit_signature_match.
+		 * A present-but-invalid signature (the cert bytes are not a
+		 * PKCS#7 signature) must be rejected at load: the signature
+		 * path is honored, not ignored. (The valid path is covered by
+		 * the signed lskels.) Pin -EBADMSG, the PKCS#7 parse failure:
+		 * a looser fd < 0 check could also be satisfied by the sparse
+		 * fd_array rejection (-EACCES) that the loader's map reference
+		 * would trip even if the signature were silently ignored.
 		 */
-		f.blob[0] ^= 0xff;
-		r = run_gen_loader(f.gopts.insns, f.gopts.insns_sz, f.blob,
-				   f.data_sz, f.excl, sizeof(f.excl), NULL, 0,
-				   true, f.ctx, f.ctx_sz, &ran);
-		ASSERT_TRUE(ran, "loader ran");
-		ASSERT_EQ(r, -EINVAL, "tampered blob rejected by emit_signature_match");
+		fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk,
+				 sizeof(junk), KEY_SPEC_SESSION_KEYRING, 0);
+		ASSERT_EQ(fd, -EBADMSG, "invalid signature rejected at load");
+		if (fd >= 0)
+			close(fd);
 	}
 	gen_loader_fixture_fini(&f);
 }
 
-static void metadata_not_exclusive(void)
+static void signed_nonexcl_fd_array_rejected(void)
 {
+	static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, };
 	struct gen_loader_fixture f;
-	bool ran;
-	int r;
+	int map_fd, fd;
 
 	if (gen_loader_fixture_init(&f) == 0) {
 		/*
-		 * Correct blob but a non-exclusive metadata map: the verifier does
-		 * not reject (excl_prog_sha unset), so the runtime map->excl == 1
-		 * check in the loader must.
+		 * A signed program may only bind exclusive maps through fd_array
+		 * (their contents are folded into the signature). Binding a
+		 * non-exclusive map is rejected, before the signature is even
+		 * examined.
 		 */
-		r = run_gen_loader(f.gopts.insns, f.gopts.insns_sz, f.blob,
-				   f.data_sz, NULL, 0, NULL, 0, true, f.ctx,
-				   f.ctx_sz, &ran);
-		ASSERT_TRUE(ran, "loader ran");
-		ASSERT_EQ(r, -EINVAL, "non-exclusive metadata map rejected");
+		map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "nonexcl", 4,
+					f.data_sz, 1, NULL);
+		if (ASSERT_OK_FD(map_fd, "nonexcl_map")) {
+			if (ASSERT_OK(bpf_map_freeze(map_fd), "freeze")) {
+				fd = load_loader(f.gopts.insns, f.gopts.insns_sz,
+						 map_fd, junk, sizeof(junk),
+						 KEY_SPEC_SESSION_KEYRING, 1);
+				ASSERT_EQ(fd, -EPERM,
+					  "non-exclusive map in signed fd_array rejected");
+				if (fd >= 0)
+					close(fd);
+			}
+			close(map_fd);
+		}
 	}
 	gen_loader_fixture_fini(&f);
 }
 
-static void metadata_hash_not_computed(void)
+static void signed_unfrozen_fd_array_rejected(void)
 {
+	static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, };
+	LIBBPF_OPTS(bpf_map_create_opts, mopts);
 	struct gen_loader_fixture f;
-	bool ran;
-	int r;
+	__u32 key = 0;
+	int map_fd, fd;
 
 	if (gen_loader_fixture_init(&f) == 0) {
 		/*
-		 * Correct, exclusive, frozen map, but its hash was never computed
-		 * (no OBJ_GET_INFO_BY_FD), so map->sha stays zero. The loader must
-		 * fail closed rather than treat an unset hash as a match.
+		 * The metadata map must be frozen before a signed load so the
+		 * folded bytes cannot change afterwards. Bind an exclusive map
+		 * with matching contents but skip the freeze: the load must be
+		 * rejected by the frozen check with -EPERM. The exclusivity
+		 * check right after it would pass, so the errno uniquely pins
+		 * the freeze requirement.
 		 */
-		r = run_gen_loader(f.gopts.insns, f.gopts.insns_sz, f.blob,
-				   f.data_sz, f.excl, sizeof(f.excl), NULL, 0,
-				   false, f.ctx, f.ctx_sz, &ran);
-		ASSERT_TRUE(ran, "loader ran");
-		ASSERT_EQ(r, -EINVAL, "uncomputed metadata hash rejected");
+		mopts.excl_prog_hash = f.excl;
+		mopts.excl_prog_hash_size = sizeof(f.excl);
+		map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "unfrozen", 4,
+					f.data_sz, 1, &mopts);
+		if (ASSERT_OK_FD(map_fd, "unfrozen_map")) {
+			if (ASSERT_OK(bpf_map_update_elem(map_fd, &key, f.blob, 0),
+				      "update")) {
+				fd = load_loader(f.gopts.insns, f.gopts.insns_sz,
+						 map_fd, junk, sizeof(junk),
+						 KEY_SPEC_SESSION_KEYRING, 1);
+				ASSERT_EQ(fd, -EPERM,
+					  "unfrozen map in signed fd_array rejected");
+				if (fd >= 0)
+					close(fd);
+			}
+			close(map_fd);
+		}
 	}
 	gen_loader_fixture_fini(&f);
 }
 
-static void signature_enforced(void)
+static void signed_nonarray_fd_array_rejected(void)
+{
+	static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, };
+	LIBBPF_OPTS(bpf_map_create_opts, mopts);
+	struct gen_loader_fixture f;
+	int map_fd, fd;
+
+	if (gen_loader_fixture_init(&f) == 0) {
+		/*
+		 * Only a plain BPF_MAP_TYPE_ARRAY may be folded into the
+		 * signature. An exclusive map of any other type is rejected
+		 * (-EINVAL) rather than folded - this is the type gate that
+		 * keeps arena maps (map_direct_value_addr() returns a user
+		 * address) and insn-array maps (buffer smaller than value_size)
+		 * out of the hashed region, where the old code would have
+		 * memcpy()'d from them. A hash map stands in here: it is
+		 * exclusive (bound to the loader digest) but not an array.
+		 */
+		mopts.excl_prog_hash = f.excl;
+		mopts.excl_prog_hash_size = sizeof(f.excl);
+		map_fd = bpf_map_create(BPF_MAP_TYPE_HASH, "excl_hash", 4, 4, 1,
+					&mopts);
+		if (ASSERT_OK_FD(map_fd, "excl_hash_map")) {
+			fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd,
+					 junk, sizeof(junk),
+					 KEY_SPEC_SESSION_KEYRING, 1);
+			ASSERT_EQ(fd, -EINVAL,
+				  "non-array map in signed fd_array rejected");
+			if (fd >= 0)
+				close(fd);
+			close(map_fd);
+		}
+	}
+	gen_loader_fixture_fini(&f);
+}
+
+static int setup_meta_map(const struct gen_loader_fixture *f);
+
+static void signed_btf_fd_array_rejected(void)
+{
+	char dir_tmpl[] = "/tmp/signed_loader_btfXXXXXX", *dir = NULL;
+	__u32 sig_sz = 8192;
+	int map_fd = -1, prog_fd = -1;
+	unsigned char *buf = NULL;
+	struct gen_loader_fixture f;
+	bool have_fixture = false;
+	struct btf *btf = NULL;
+	union bpf_attr attr;
+	int fds[2];
+	__u8 sig[8192];
+
+	syscall(__NR_request_key, "keyring", "_uid.0", NULL,
+		KEY_SPEC_SESSION_KEYRING);
+	dir = mkdtemp(dir_tmpl);
+	if (!ASSERT_OK_PTR(dir, "mkdtemp"))
+		return;
+	if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) {
+		rmdir(dir);
+		return;
+	}
+	have_fixture = true;
+	if (gen_loader_fixture_init(&f) != 0)
+		goto out;
+
+	/*
+	 * fd_array binds maps and BTFs alike, but only exclusive array maps are
+	 * folded into the signature. Build an otherwise genuinely signed load -
+	 * insns || metadata, exclusive frozen map at fd_array[0] - then smuggle
+	 * an extra BTF into fd_array[1]. A signed program may not bind any BTF,
+	 * so resolving the fd_array entries rejects the BTF with -EACCES (in
+	 * __add_used_btf(), before the signature is even verified).
+	 */
+	buf = malloc((size_t)f.gopts.insns_sz + f.data_sz);
+	if (!ASSERT_OK_PTR(buf, "signbuf"))
+		goto out;
+	memcpy(buf, f.gopts.insns, f.gopts.insns_sz);
+	memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz);
+	if (!ASSERT_OK(sign_buf(dir, buf, f.gopts.insns_sz + f.data_sz, sig,
+			       &sig_sz), "sign insns||metadata"))
+		goto out;
+
+	map_fd = setup_meta_map(&f);
+	if (!ASSERT_OK_FD(map_fd, "meta_map"))
+		goto out;
+	btf = btf__new_empty();
+	if (!ASSERT_OK_PTR(btf, "btf_new_empty"))
+		goto out;
+	btf__add_int(btf, "int", 4, BTF_INT_SIGNED);
+	if (!ASSERT_OK(btf__load_into_kernel(btf), "btf_load"))
+		goto out;
+
+	fds[0] = map_fd;
+	fds[1] = btf__fd(btf);
+	memset(&attr, 0, sizeof(attr));
+	attr.prog_type = BPF_PROG_TYPE_SYSCALL;
+	attr.insns = ptr_to_u64(f.gopts.insns);
+	attr.insn_cnt = f.gopts.insns_sz / sizeof(struct bpf_insn);
+	attr.license = ptr_to_u64("Dual BSD/GPL");
+	attr.prog_flags = BPF_F_SLEEPABLE;
+	attr.fd_array = ptr_to_u64(fds);
+	attr.fd_array_cnt = 2;
+	attr.signature = ptr_to_u64(sig);
+	attr.signature_size = sig_sz;
+	attr.keyring_id = KEY_SPEC_SESSION_KEYRING;
+	memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog"));
+	prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr,
+			  offsetofend(union bpf_attr, keyring_id));
+	ASSERT_EQ(prog_fd < 0 ? -errno : prog_fd, -EACCES,
+		  "BTF in signed fd_array rejected");
+	if (prog_fd >= 0)
+		close(prog_fd);
+out:
+	if (btf)
+		btf__free(btf);
+	if (map_fd >= 0)
+		close(map_fd);
+	if (have_fixture)
+		gen_loader_fixture_fini(&f);
+	if (dir)
+		run_setup("cleanup", dir);
+	free(buf);
+}
+
+static void signature_failure_logs(void)
 {
 	static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, };
+	char log_buf[1024] = {};
 	struct gen_loader_fixture f;
+	union bpf_attr attr;
 	int fd;
 
 	if (gen_loader_fixture_init(&f) == 0) {
 		/*
-		 * A present-but-invalid signature (the cert bytes are not a
-		 * PKCS#7 signature) must be rejected at load: the signature
-		 * path is honored, not ignored. (The valid path is covered by
-		 * the signed lskels.)
+		 * Signature verification now runs inside bpf_check(), so a
+		 * failure is reported through the verifier log. A present-but-
+		 * invalid signature is rejected and the log says why.
 		 */
-		fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk,
-				 sizeof(junk), KEY_SPEC_SESSION_KEYRING);
+		memset(&attr, 0, sizeof(attr));
+		attr.prog_type = BPF_PROG_TYPE_SYSCALL;
+		attr.insns = ptr_to_u64(f.gopts.insns);
+		attr.insn_cnt = f.gopts.insns_sz / sizeof(struct bpf_insn);
+		attr.license = ptr_to_u64("Dual BSD/GPL");
+		attr.prog_flags = BPF_F_SLEEPABLE;
+		attr.signature = ptr_to_u64(junk);
+		attr.signature_size = sizeof(junk);
+		attr.keyring_id = KEY_SPEC_SESSION_KEYRING;
+		attr.log_level = 1;
+		attr.log_buf = ptr_to_u64(log_buf);
+		attr.log_size = sizeof(log_buf);
+		memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog"));
+
+		fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr,
+			     offsetofend(union bpf_attr, keyring_id));
 		ASSERT_LT(fd, 0, "invalid signature rejected at load");
+		if (fd >= 0)
+			close(fd);
+		ASSERT_HAS_SUBSTR(log_buf, "signature verification failed",
+				  "verifier logs signature failure");
 	}
 	gen_loader_fixture_fini(&f);
 }
@@ -495,8 +575,31 @@ static void signature_too_large(void)
 		 * is rejected before the buffer is read.
 		 */
 		fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk,
-				 64 << 20, KEY_SPEC_SESSION_KEYRING);
+				 64 << 20, KEY_SPEC_SESSION_KEYRING, 0);
 		ASSERT_EQ(fd, -EINVAL, "oversized signature rejected");
+		if (fd >= 0)
+			close(fd);
+	}
+	gen_loader_fixture_fini(&f);
+}
+
+static void signature_zero_size(void)
+{
+	static const __u8 junk[64] = {};
+	struct gen_loader_fixture f;
+	int fd;
+
+	if (gen_loader_fixture_init(&f) == 0) {
+		/*
+		 * A present signature with signature_size == 0 is rejected
+		 * up front, before the keyring is resolved or the signature
+		 * buffer is read.
+		 */
+		fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk,
+				 0, KEY_SPEC_SESSION_KEYRING, 0);
+		ASSERT_EQ(fd, -EINVAL, "zero-size signature rejected");
+		if (fd >= 0)
+			close(fd);
 	}
 	gen_loader_fixture_fini(&f);
 }
@@ -515,8 +618,10 @@ static void signature_bad_keyring(void)
 		 * large positive serial takes the user-keyring path and won't exist.
 		 */
 		fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk,
-				 sizeof(junk), INT_MAX);
+				 sizeof(junk), INT_MAX, 0);
 		ASSERT_EQ(fd, -EINVAL, "signature with bad keyring_id rejected");
+		if (fd >= 0)
+			close(fd);
 	}
 	gen_loader_fixture_fini(&f);
 }
@@ -575,7 +680,7 @@ static void metadata_ctx_max_entries_ignored(void)
 	memcpy(blob, gopts.data, data_sz);
 
 	r = run_gen_loader(gopts.insns, gopts.insns_sz, blob, data_sz,
-			   excl, sizeof(excl), NULL, 0, true, ctx, ctx_sz, &ran);
+			   excl, sizeof(excl), NULL, 0, ctx, ctx_sz, &ran);
 	if (!ASSERT_TRUE(ran, "loader ran") ||
 	    !ASSERT_EQ(r, 0, "loader retval"))
 		goto free_blob;
@@ -661,7 +766,7 @@ static void metadata_ctx_initial_value_ignored(void)
 	memcpy(blob, gopts.data, data_sz);
 
 	r = run_gen_loader(gopts.insns, gopts.insns_sz, blob, data_sz,
-			   excl, sizeof(excl), NULL, 0, true, ctx, ctx_sz, &ran);
+			   excl, sizeof(excl), NULL, 0, ctx, ctx_sz, &ran);
 	if (!ASSERT_TRUE(ran, "loader ran") ||
 	    !ASSERT_EQ(r, 0, "loader retval"))
 		goto free_blob;
@@ -714,6 +819,7 @@ static void signature_authenticates_insns(void)
 	__u8 excl[SHA256_DIGEST_LENGTH], sig[8192];
 	__u32 sig_sz = sizeof(sig), insns_sz, data_sz, ctx_sz;
 	unsigned char *insns = NULL, *tampered = NULL, *blob = NULL;
+	unsigned char *signbuf = NULL;
 	int nr_maps = 0, nr_progs = 0, r;
 	struct bpf_program *p;
 	struct bpf_map *m;
@@ -760,29 +866,141 @@ static void signature_authenticates_insns(void)
 	memcpy(blob, gopts.data, data_sz);
 	libbpf_sha256(insns, insns_sz, excl);
 
-	if (!ASSERT_OK(sign_buf(dir, insns, insns_sz, sig, &sig_sz), "sign-file"))
+	signbuf = malloc((size_t)insns_sz + data_sz);
+	if (!ASSERT_OK_PTR(signbuf, "signbuf"))
+		goto cleanup;
+	memcpy(signbuf, insns, insns_sz);
+	memcpy(signbuf + insns_sz, blob, data_sz);
+	if (!ASSERT_OK(sign_buf(dir, signbuf, insns_sz + data_sz, sig, &sig_sz),
+		       "sign-file"))
 		goto cleanup;
 
 	memset(ctx, 0, ctx_sz);
 	((struct bpf_loader_ctx *)ctx)->sz = ctx_sz;
 	r = run_gen_loader(insns, insns_sz, blob, data_sz, excl, sizeof(excl),
-			   sig, sig_sz, true, ctx, ctx_sz, &ran);
+			   sig, sig_sz, ctx, ctx_sz, &ran);
 	ASSERT_TRUE(ran, "valid signature: loader loaded and ran");
 	ASSERT_EQ(r, 0, "valid signature accepted");
 	close_loader_ctx_fds(ctx, nr_maps, nr_progs);
 
 	memcpy(tampered, insns, insns_sz);
 	tampered[insns_sz / 2] ^= 0xff;
+	/*
+	 * Bind the metadata map to the tampered loader's own digest, so the
+	 * verifier's exclusive-map check (excl_prog_sha == prog->digest) passes
+	 * and the signature - verified after the maps are resolved - is what
+	 * rejects the load. This is the attacker's best case: even after
+	 * re-binding the exclusive map to their tampered loader, the signature
+	 * over the original insns || metadata still fails. (Leaving the map
+	 * bound to the original digest would instead trip the excl check first.)
+	 */
+	libbpf_sha256(tampered, insns_sz, excl);
 	memset(ctx, 0, ctx_sz);
 	((struct bpf_loader_ctx *)ctx)->sz = ctx_sz;
 	r = run_gen_loader(tampered, insns_sz, blob, data_sz, excl, sizeof(excl),
-			   sig, sig_sz, true, ctx, ctx_sz, &ran);
+			   sig, sig_sz, ctx, ctx_sz, &ran);
 	ASSERT_FALSE(ran, "tampered loader rejected before run");
 	ASSERT_EQ(r, -EKEYREJECTED, "signature is bound to the instructions");
 cleanup:
 	free(insns);
 	free(tampered);
 	free(blob);
+	free(signbuf);
+	free(ctx);
+	test_signed_loader__destroy(skel);
+	run_setup("cleanup", dir);
+}
+
+static void signature_authenticates_metadata(void)
+{
+	LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true);
+	char dir_tmpl[] = "/tmp/signed_loaderXXXXXX", *dir;
+	struct test_signed_loader *skel = NULL;
+	__u8 excl[SHA256_DIGEST_LENGTH], sig[8192];
+	__u32 sig_sz = sizeof(sig), insns_sz, data_sz, ctx_sz;
+	unsigned char *insns = NULL, *blob = NULL;
+	unsigned char *signbuf = NULL;
+	int nr_maps = 0, nr_progs = 0, r;
+	struct bpf_program *p;
+	struct bpf_map *m;
+	void *ctx = NULL;
+	bool ran;
+
+	syscall(__NR_request_key, "keyring", "_uid.0", NULL,
+		KEY_SPEC_SESSION_KEYRING);
+	dir = mkdtemp(dir_tmpl);
+	if (!ASSERT_OK_PTR(dir, "mkdtemp"))
+		return;
+	if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) {
+		rmdir(dir);
+		return;
+	}
+
+	skel = test_signed_loader__open();
+	if (!ASSERT_OK_PTR(skel, "skel_open"))
+		goto cleanup;
+	if (!ASSERT_OK(bpf_object__gen_loader(skel->obj, &gopts), "gen_loader"))
+		goto cleanup;
+	if (!ASSERT_OK(bpf_object__load(skel->obj), "gen_load"))
+		goto cleanup;
+
+	bpf_object__for_each_program(p, skel->obj)
+		nr_progs++;
+	bpf_object__for_each_map(m, skel->obj)
+		nr_maps++;
+	ctx_sz = sizeof(struct bpf_loader_ctx) +
+		 nr_maps * sizeof(struct bpf_map_desc) +
+		 nr_progs * sizeof(struct bpf_prog_desc);
+	insns_sz = gopts.insns_sz;
+	data_sz = gopts.data_sz;
+	ctx = calloc(1, ctx_sz);
+	insns = malloc(insns_sz);
+	blob = malloc(data_sz);
+	if (!ASSERT_OK_PTR(ctx, "ctx") ||
+	    !ASSERT_OK_PTR(insns, "insns") ||
+	    !ASSERT_OK_PTR(blob, "blob"))
+		goto cleanup;
+	memcpy(insns, gopts.insns, insns_sz);
+	memcpy(blob, gopts.data, data_sz);
+	libbpf_sha256(insns, insns_sz, excl);
+
+	signbuf = malloc((size_t)insns_sz + data_sz);
+	if (!ASSERT_OK_PTR(signbuf, "signbuf"))
+		goto cleanup;
+	memcpy(signbuf, insns, insns_sz);
+	memcpy(signbuf + insns_sz, blob, data_sz);
+	if (!ASSERT_OK(sign_buf(dir, signbuf, insns_sz + data_sz, sig, &sig_sz),
+		       "sign-file"))
+		goto cleanup;
+
+	memset(ctx, 0, ctx_sz);
+	((struct bpf_loader_ctx *)ctx)->sz = ctx_sz;
+	r = run_gen_loader(insns, insns_sz, blob, data_sz, excl, sizeof(excl),
+			   sig, sig_sz, ctx, ctx_sz, &ran);
+	ASSERT_TRUE(ran, "valid signature: loader loaded and ran");
+	ASSERT_EQ(r, 0, "valid signature accepted");
+	close_loader_ctx_fds(ctx, nr_maps, nr_progs);
+
+	/*
+	 * Tamper the metadata after signing while leaving the instructions
+	 * and thus the exclusive hash binding untouched: the map freezes
+	 * fine and excl_prog_sha still matches the loader's digest, so the
+	 * load reaches signature verification, which folds the live frozen
+	 * map bytes into the checked payload and must reject the modified
+	 * blob. A kernel folding anything but the map contents themselves
+	 * would wrongly accept this load.
+	 */
+	blob[data_sz / 2] ^= 0xff;
+	memset(ctx, 0, ctx_sz);
+	((struct bpf_loader_ctx *)ctx)->sz = ctx_sz;
+	r = run_gen_loader(insns, insns_sz, blob, data_sz, excl, sizeof(excl),
+			   sig, sig_sz, ctx, ctx_sz, &ran);
+	ASSERT_FALSE(ran, "tampered metadata rejected before run");
+	ASSERT_EQ(r, -EKEYREJECTED, "signature is bound to the metadata");
+cleanup:
+	free(insns);
+	free(blob);
+	free(signbuf);
 	free(ctx);
 	test_signed_loader__destroy(skel);
 	run_setup("cleanup", dir);
@@ -1007,10 +1225,11 @@ static void lsm_signature_verdict(void)
 {
 	char dir_tmpl[] = "/tmp/signed_loader_lsmXXXXXX", *dir = NULL;
 	struct test_signed_loader_lsm *lsm = NULL;
+	__u32 sig_sz = 8192, msig_sz = 8192;
 	int map_fd = -1, prog_fd = -1;
 	bool have_fixture = false;
 	struct gen_loader_fixture f;
-	__u32 sig_sz = 8192;
+	unsigned char *buf;
 	__s32 ses_serial;
 	__u8 sig[8192];
 
@@ -1029,7 +1248,7 @@ static void lsm_signature_verdict(void)
 	if (!ASSERT_OK_FD(map_fd, "meta_map_unsigned"))
 		goto out;
 	lsm->bss->seen = 0;
-	prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, NULL, 0, 0);
+	prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, NULL, 0, 0, 0);
 	close(map_fd);
 	map_fd = -1;
 	if (!ASSERT_OK_FD(prog_fd, "unsigned loader load"))
@@ -1062,22 +1281,51 @@ static void lsm_signature_verdict(void)
 		goto out;
 	lsm->bss->seen = 0;
 	prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, sig,
-			      sig_sz, KEY_SPEC_SESSION_KEYRING);
+			      sig_sz, KEY_SPEC_SESSION_KEYRING, 0);
 	close(map_fd);
 	map_fd = -1;
-	if (!ASSERT_OK_FD(prog_fd, "signed loader load"))
-		goto out;
-	close(prog_fd);
+	ASSERT_EQ(prog_fd, -EACCES, "unfolded metadata rejected");
+	if (prog_fd >= 0)
+		close(prog_fd);
 	prog_fd = -1;
 
 	ses_serial = syscall(__NR_keyctl, KEYCTL_GET_KEYRING_ID,
 			     KEY_SPEC_SESSION_KEYRING, 0);
 	ASSERT_EQ(lsm->bss->seen, 1, "signed: one observed load");
-	ASSERT_EQ(lsm->bss->sig_verdict, BPF_SIG_VERIFIED, "signed verdict");
+	ASSERT_EQ(lsm->bss->sig_verdict, BPF_SIG_VERIFIED,
+		  "admission saw a valid signature");
 	ASSERT_EQ(lsm->bss->sig_keyring_type, BPF_SIG_KEYRING_USER, "signed keyring type");
 	ASSERT_GT(ses_serial, 0, "session keyring serial resolved");
 	ASSERT_EQ(lsm->bss->sig_keyring_serial, ses_serial,
 		  "signed: validated against session keyring");
+
+	buf = malloc((size_t)f.gopts.insns_sz + f.data_sz);
+	if (!ASSERT_OK_PTR(buf, "meta_signbuf"))
+		goto out;
+	memcpy(buf, f.gopts.insns, f.gopts.insns_sz);
+	memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz);
+	if (!ASSERT_OK(sign_buf(dir, buf, f.gopts.insns_sz + f.data_sz,
+				sig, &msig_sz), "sign insns||metadata")) {
+		free(buf);
+		goto out;
+	}
+	free(buf);
+
+	map_fd = setup_meta_map(&f);
+	if (!ASSERT_OK_FD(map_fd, "meta_map_bound"))
+		goto out;
+	lsm->bss->seen = 0;
+	prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, sig,
+			      msig_sz, KEY_SPEC_SESSION_KEYRING, 1);
+	close(map_fd);
+	map_fd = -1;
+	if (!ASSERT_OK_FD(prog_fd, "metadata-bound loader load"))
+		goto out;
+	close(prog_fd);
+	prog_fd = -1;
+	ASSERT_EQ(lsm->bss->seen, 1, "metadata: one observed load");
+	ASSERT_EQ(lsm->bss->sig_verdict, BPF_SIG_VERIFIED,
+		  "metadata-bound verdict");
 out:
 	if (map_fd >= 0)
 		close(map_fd);
@@ -1090,22 +1338,471 @@ static void lsm_signature_verdict(void)
 	test_signed_loader_lsm__destroy(lsm);
 }
 
+/*
+ * Load-time metadata verification: the kernel folds the frozen metadata map
+ * into the signature (insns || metadata) and checks it at BPF_PROG_LOAD via
+ * fd_array_cnt, rather than the loader checking from within BPF. Sign that
+ * concatenation, hand the kernel the map, and confirm the signed loader loads,
+ * runs, and installs its target.
+ */
+static int loadtime_drive(const char *dir, const void *insns, __u32 insns_sz,
+			  const void *data, __u32 data_sz, const __u8 *excl,
+			  void *ctx, __u32 ctx_sz, int *load_ret, bool *ran)
+{
+	LIBBPF_OPTS(bpf_map_create_opts, mopts,
+		    .excl_prog_hash = excl,
+		    .excl_prog_hash_size = SHA256_DIGEST_LENGTH);
+	__u32 sig_sz = 8192, key = 0;
+	unsigned char *buf = NULL;
+	int map_fd, prog_fd, ret = 0;
+	union bpf_attr attr;
+	__u8 sig[8192];
+
+	*ran = false;
+	*load_ret = 0;
+
+	/*
+	 * Metadata map, bound to the loader digest and frozen, exactly as
+	 * skel_internal.h's bpf_load_and_run() sets it up.
+	 */
+	map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4,
+				data_sz, 1, &mopts);
+	if (map_fd < 0) {
+		ret = -errno;
+		goto out_load;
+	}
+	if (bpf_map_update_elem(map_fd, &key, data, 0) || bpf_map_freeze(map_fd)) {
+		ret = -errno;
+		goto out_load;
+	}
+
+	/* Sign insns || metadata, the same bytes the kernel reconstructs. */
+	buf = malloc((size_t)insns_sz + data_sz);
+	if (!buf) {
+		ret = -ENOMEM;
+		goto out_load;
+	}
+	memcpy(buf, insns, insns_sz);
+	memcpy(buf + insns_sz, data, data_sz);
+	ret = sign_buf(dir, buf, insns_sz + data_sz, sig, &sig_sz);
+	if (ret)
+		goto out_load;
+
+	memset(&attr, 0, sizeof(attr));
+	attr.prog_type = BPF_PROG_TYPE_SYSCALL;
+	attr.insns = ptr_to_u64(insns);
+	attr.insn_cnt = insns_sz / sizeof(struct bpf_insn);
+	attr.license = ptr_to_u64("Dual BSD/GPL");
+	attr.prog_flags = BPF_F_SLEEPABLE;
+	attr.fd_array = ptr_to_u64(&map_fd);
+	attr.signature = ptr_to_u64(sig);
+	attr.signature_size = sig_sz;
+	attr.keyring_id = KEY_SPEC_SESSION_KEYRING;
+	attr.fd_array_cnt = 1;
+	memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog"));
+	prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr,
+			  offsetofend(union bpf_attr, keyring_id));
+	if (prog_fd < 0) {
+		ret = -errno;
+		goto out_load;
+	}
+
+	memset(&attr, 0, sizeof(attr));
+	attr.test.prog_fd = prog_fd;
+	attr.test.ctx_in = ptr_to_u64(ctx);
+	attr.test.ctx_size_in = ctx_sz;
+	if (syscall(__NR_bpf, BPF_PROG_RUN, &attr,
+		    offsetofend(union bpf_attr, test)) < 0) {
+		ret = -errno;
+		goto out_prog;
+	}
+	*ran = true;
+	ret = (int)attr.test.retval;
+out_prog:
+	close(prog_fd);
+	goto out_map;
+out_load:
+	*load_ret = ret;
+out_map:
+	free(buf);
+	if (map_fd >= 0)
+		close(map_fd);
+	return ret;
+}
+
+static void loadtime_verify(struct bpf_object *obj, int expect_maps)
+{
+	LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true);
+	char dir_tmpl[] = "/tmp/signed_loader_ltXXXXXX", *dir = NULL;
+	int nr_maps = 0, nr_progs = 0, load_ret = 0, r;
+	__u8 excl[SHA256_DIGEST_LENGTH];
+	struct bpf_prog_desc *pd;
+	struct bpf_map_desc *md;
+	unsigned char *blob = NULL;
+	struct bpf_program *p;
+	struct bpf_map *m;
+	__u32 ctx_sz, data_sz;
+	void *ctx = NULL;
+	bool ran = false;
+
+	syscall(__NR_request_key, "keyring", "_uid.0", NULL,
+		KEY_SPEC_SESSION_KEYRING);
+	dir = mkdtemp(dir_tmpl);
+	if (!ASSERT_OK_PTR(dir, "mkdtemp"))
+		return;
+	if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) {
+		rmdir(dir);
+		return;
+	}
+
+	if (!ASSERT_OK(bpf_object__gen_loader(obj, &gopts), "gen_loader"))
+		goto out;
+	if (!ASSERT_OK(bpf_object__load(obj), "gen_load"))
+		goto out;
+
+	bpf_object__for_each_program(p, obj)
+		nr_progs++;
+	bpf_object__for_each_map(m, obj)
+		nr_maps++;
+	if (!ASSERT_EQ(nr_maps, expect_maps, "fixture map count"))
+		goto out;
+
+	ctx_sz = sizeof(struct bpf_loader_ctx) +
+		 nr_maps * sizeof(struct bpf_map_desc) +
+		 nr_progs * sizeof(struct bpf_prog_desc);
+	ctx = calloc(1, ctx_sz);
+	if (!ASSERT_OK_PTR(ctx, "ctx_alloc"))
+		goto out;
+	((struct bpf_loader_ctx *)ctx)->sz = ctx_sz;
+
+	data_sz = gopts.data_sz;
+	blob = malloc(data_sz);
+	if (!ASSERT_OK_PTR(blob, "blob_alloc"))
+		goto out;
+	memcpy(blob, gopts.data, data_sz);
+
+	/* excl_prog_hash = SHA256(loader insns) == the loader's prog->digest. */
+	libbpf_sha256(gopts.insns, gopts.insns_sz, excl);
+
+	r = loadtime_drive(dir, gopts.insns, gopts.insns_sz, blob, data_sz,
+			   excl, ctx, ctx_sz, &load_ret, &ran);
+	ASSERT_OK(load_ret, "signed loader loaded (insns || metadata)");
+	ASSERT_TRUE(ran, "loader ran");
+	ASSERT_EQ(r, 0, "loader installed its target");
+
+	md = (struct bpf_map_desc *)((char *)ctx + sizeof(struct bpf_loader_ctx));
+	pd = (struct bpf_prog_desc *)(md + nr_maps);
+	ASSERT_GT(pd[0].prog_fd, 0, "target program installed");
+	if (nr_maps)
+		ASSERT_GT(md[0].map_fd, 0, "target map installed");
+
+	close_loader_ctx_fds(ctx, nr_maps, nr_progs);
+out:
+	free(blob);
+	free(ctx);
+	if (dir)
+		run_setup("cleanup", dir);
+}
+
+static void loadtime_no_map(void)
+{
+	struct test_signed_loader *skel = test_signed_loader__open();
+
+	if (!ASSERT_OK_PTR(skel, "skel_open"))
+		return;
+	loadtime_verify(skel->obj, 0);
+	test_signed_loader__destroy(skel);
+}
+
+static void loadtime_with_map(void)
+{
+	struct test_signed_loader_map *skel = test_signed_loader_map__open();
+
+	if (!ASSERT_OK_PTR(skel, "skel_open"))
+		return;
+	loadtime_verify(skel->obj, 1);
+	test_signed_loader_map__destroy(skel);
+}
+
+/*
+ * A signed program need not bind any map. A plain BPF_PROG_TYPE_SYSCALL
+ * program with no fd_array is signed over its instructions alone: the kernel
+ * verifies the signature, folds no metadata, and the program loads. Exercise
+ * the fd_array == NULL / fd_array_cnt == 0 path, and confirm the signature
+ * still authenticates the instructions (a tampered copy is rejected).
+ */
+static void signed_no_fd_array(void)
+{
+	struct bpf_insn insns[] = {
+		BPF_MOV64_IMM(BPF_REG_0, 0),
+		BPF_EXIT_INSN(),
+	};
+	char dir_tmpl[] = "/tmp/signed_loaderXXXXXX", *dir;
+	__u32 sig_sz = 8192;
+	union bpf_attr attr;
+	__u8 sig[8192];
+	int prog_fd, err;
+
+	syscall(__NR_request_key, "keyring", "_uid.0", NULL,
+		KEY_SPEC_SESSION_KEYRING);
+	dir = mkdtemp(dir_tmpl);
+	if (!ASSERT_OK_PTR(dir, "mkdtemp"))
+		return;
+	if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) {
+		rmdir(dir);
+		return;
+	}
+
+	/* No metadata map: the signed payload is the instructions alone. */
+	if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz),
+		       "sign-file"))
+		goto cleanup;
+
+	memset(&attr, 0, sizeof(attr));
+	attr.prog_type = BPF_PROG_TYPE_SYSCALL;
+	attr.insns = ptr_to_u64(insns);
+	attr.insn_cnt = ARRAY_SIZE(insns);
+	attr.license = ptr_to_u64("Dual BSD/GPL");
+	attr.prog_flags = BPF_F_SLEEPABLE;
+	attr.signature = ptr_to_u64(sig);
+	attr.signature_size = sig_sz;
+	attr.keyring_id = KEY_SPEC_SESSION_KEYRING;
+	/* fd_array and fd_array_cnt deliberately left NULL/0. */
+	memcpy(attr.prog_name, "signed_nomap", sizeof("signed_nomap"));
+
+	prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr,
+			  offsetofend(union bpf_attr, keyring_id));
+	if (!ASSERT_GE(prog_fd, 0, "map-less signed program loaded")) {
+		if (prog_fd >= 0)
+			close(prog_fd);
+		goto cleanup;
+	}
+	close(prog_fd);
+
+	/* The signature covers the instructions, so tampering must be rejected. */
+	insns[0].imm = 1;
+	prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr,
+			  offsetofend(union bpf_attr, keyring_id));
+	err = prog_fd < 0 ? -errno : prog_fd;
+	ASSERT_EQ(err, -EKEYREJECTED, "tampered map-less program rejected");
+	if (prog_fd >= 0)
+		close(prog_fd);
+cleanup:
+	run_setup("cleanup", dir);
+}
+
+/*
+ * A signed program may reach maps only through fd_array indices, so the kernel
+ * folds (and thus attests) them. A direct BPF_PSEUDO_MAP_FD reference - a raw,
+ * unfolded fd baked into the signed instructions - is rejected by the verifier.
+ */
+static void signed_map_by_fd_rejected(void)
+{
+	struct bpf_insn insns[] = {
+		BPF_LD_MAP_FD(BPF_REG_1, 0),
+		BPF_MOV64_IMM(BPF_REG_0, 0),
+		BPF_EXIT_INSN(),
+	};
+	char dir_tmpl[] = "/tmp/signed_loaderXXXXXX", *dir;
+	__u32 sig_sz = 8192;
+	union bpf_attr attr;
+	__u8 sig[8192];
+	int map_fd, prog_fd, err;
+
+	map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "sig_mapfd", 4, 4, 1, NULL);
+	if (!ASSERT_GE(map_fd, 0, "map_create"))
+		return;
+	insns[0].imm = map_fd;	/* bake the raw map fd into the ld_imm64 */
+
+	syscall(__NR_request_key, "keyring", "_uid.0", NULL,
+		KEY_SPEC_SESSION_KEYRING);
+	dir = mkdtemp(dir_tmpl);
+	if (!ASSERT_OK_PTR(dir, "mkdtemp"))
+		goto out_map;
+	if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) {
+		rmdir(dir);
+		goto out_map;
+	}
+
+	/* Sign the instructions, raw map fd and all. */
+	if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz),
+		       "sign-file"))
+		goto cleanup;
+
+	memset(&attr, 0, sizeof(attr));
+	attr.prog_type = BPF_PROG_TYPE_SYSCALL;
+	attr.insns = ptr_to_u64(insns);
+	attr.insn_cnt = ARRAY_SIZE(insns);
+	attr.license = ptr_to_u64("Dual BSD/GPL");
+	attr.prog_flags = BPF_F_SLEEPABLE;
+	attr.signature = ptr_to_u64(sig);
+	attr.signature_size = sig_sz;
+	attr.keyring_id = KEY_SPEC_SESSION_KEYRING;
+	/* No fd_array: the map is reached by a raw fd in the instructions. */
+	memcpy(attr.prog_name, "signed_mapfd", sizeof("signed_mapfd"));
+
+	prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr,
+			  offsetofend(union bpf_attr, keyring_id));
+	err = prog_fd < 0 ? -errno : prog_fd;
+	ASSERT_EQ(err, -EINVAL, "signed program referencing a map by fd rejected");
+	if (prog_fd >= 0)
+		close(prog_fd);
+cleanup:
+	run_setup("cleanup", dir);
+out_map:
+	close(map_fd);
+}
+
+/*
+ * A signed program may reach maps only through the continuous fd_array, so the
+ * kernel folds (and thus attests) them. Referencing a map by fd_array *index*
+ * while leaving fd_array_cnt at 0 selects the sparse path, which resolves a map
+ * the signature never covered; the verifier rejects it up front with -EACCES.
+ */
+static void signed_sparse_fd_array_rejected(void)
+{
+	struct bpf_insn insns[] = {
+		BPF_LD_IMM64_RAW(BPF_REG_1, BPF_PSEUDO_MAP_IDX, 0),
+		BPF_MOV64_IMM(BPF_REG_0, 0),
+		BPF_EXIT_INSN(),
+	};
+	char dir_tmpl[] = "/tmp/signed_loader_spXXXXXX", *dir;
+	__u32 sig_sz = 8192;
+	union bpf_attr attr;
+	__u8 sig[8192];
+	int map_fd, prog_fd, err;
+
+	map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "sig_sparse", 4, 4, 1, NULL);
+	if (!ASSERT_GE(map_fd, 0, "map_create"))
+		return;
+
+	syscall(__NR_request_key, "keyring", "_uid.0", NULL,
+		KEY_SPEC_SESSION_KEYRING);
+	dir = mkdtemp(dir_tmpl);
+	if (!ASSERT_OK_PTR(dir, "mkdtemp"))
+		goto out_map;
+	if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) {
+		rmdir(dir);
+		goto out_map;
+	}
+
+	/* Sign the instructions alone; the sparse map is not folded. */
+	if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz),
+		       "sign-file"))
+		goto cleanup;
+
+	memset(&attr, 0, sizeof(attr));
+	attr.prog_type = BPF_PROG_TYPE_SYSCALL;
+	attr.insns = ptr_to_u64(insns);
+	attr.insn_cnt = ARRAY_SIZE(insns);
+	attr.license = ptr_to_u64("Dual BSD/GPL");
+	attr.prog_flags = BPF_F_SLEEPABLE;
+	attr.fd_array = ptr_to_u64(&map_fd);
+	attr.fd_array_cnt = 0; /* sparse: force lazy map resolution */
+	attr.signature = ptr_to_u64(sig);
+	attr.signature_size = sig_sz;
+	attr.keyring_id = KEY_SPEC_SESSION_KEYRING;
+	memcpy(attr.prog_name, "signed_sparse", sizeof("signed_sparse"));
+
+	prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr,
+			  offsetofend(union bpf_attr, keyring_id));
+	err = prog_fd < 0 ? -errno : prog_fd;
+	ASSERT_EQ(err, -EACCES, "signed program binding a sparse fd_array map rejected");
+	if (prog_fd >= 0)
+		close(prog_fd);
+cleanup:
+	run_setup("cleanup", dir);
+out_map:
+	close(map_fd);
+}
+
+static void signed_module_kfunc_rejected(void)
+{
+	struct bpf_insn insns[] = {
+		BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 1, 1),
+		BPF_MOV64_IMM(BPF_REG_0, 0),
+		BPF_EXIT_INSN(),
+	};
+	char dir_tmpl[] = "/tmp/signed_loader_kfnXXXXXX", *dir;
+	int prog_fd, err, fds[2];
+	struct btf *btf = NULL;
+	__u32 sig_sz = 8192;
+	union bpf_attr attr;
+	__u8 sig[8192];
+
+	syscall(__NR_request_key, "keyring", "_uid.0", NULL,
+		KEY_SPEC_SESSION_KEYRING);
+	dir = mkdtemp(dir_tmpl);
+	if (!ASSERT_OK_PTR(dir, "mkdtemp"))
+		return;
+	if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) {
+		rmdir(dir);
+		return;
+	}
+	if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz),
+		       "sign-file"))
+		goto cleanup;
+	btf = btf__new_empty();
+	if (!ASSERT_OK_PTR(btf, "btf_new_empty"))
+		goto cleanup;
+	btf__add_int(btf, "int", 4, BTF_INT_SIGNED);
+	if (!ASSERT_OK(btf__load_into_kernel(btf), "btf_load"))
+		goto cleanup;
+	fds[0] = -1;
+	fds[1] = btf__fd(btf);
+
+	memset(&attr, 0, sizeof(attr));
+	attr.prog_type = BPF_PROG_TYPE_SYSCALL;
+	attr.insns = ptr_to_u64(insns);
+	attr.insn_cnt = ARRAY_SIZE(insns);
+	attr.license = ptr_to_u64("Dual BSD/GPL");
+	attr.prog_flags = BPF_F_SLEEPABLE;
+	attr.fd_array = ptr_to_u64(fds);
+	attr.fd_array_cnt = 0; /* sparse: force lazy kfunc BTF resolution */
+	attr.signature = ptr_to_u64(sig);
+	attr.signature_size = sig_sz;
+	attr.keyring_id = KEY_SPEC_SESSION_KEYRING;
+	memcpy(attr.prog_name, "signed_kfunc", sizeof("signed_kfunc"));
+
+	prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr,
+			  offsetofend(union bpf_attr, keyring_id));
+	err = prog_fd < 0 ? -errno : prog_fd;
+	if (prog_fd >= 0)
+		close(prog_fd);
+
+	ASSERT_EQ(err, -EACCES, "module kfunc BTF in signed program rejected");
+cleanup:
+	if (btf)
+		btf__free(btf);
+	run_setup("cleanup", dir);
+}
+
 void test_signed_loader(void)
 {
-	if (test__start_subtest("metadata_check_shape"))
-		metadata_check_shape();
+	if (test__start_subtest("loadtime_no_map"))
+		loadtime_no_map();
+	if (test__start_subtest("loadtime_with_map"))
+		loadtime_with_map();
 	if (test__start_subtest("metadata_match"))
 		metadata_match();
-	if (test__start_subtest("metadata_sha_mismatch"))
-		metadata_sha_mismatch();
-	if (test__start_subtest("metadata_not_exclusive"))
-		metadata_not_exclusive();
-	if (test__start_subtest("metadata_hash_not_computed"))
-		metadata_hash_not_computed();
 	if (test__start_subtest("signature_enforced"))
 		signature_enforced();
+	if (test__start_subtest("signed_nonexcl_fd_array_rejected"))
+		signed_nonexcl_fd_array_rejected();
+	if (test__start_subtest("signed_unfrozen_fd_array_rejected"))
+		signed_unfrozen_fd_array_rejected();
+	if (test__start_subtest("signed_nonarray_fd_array_rejected"))
+		signed_nonarray_fd_array_rejected();
+	if (test__start_subtest("signed_btf_fd_array_rejected"))
+		signed_btf_fd_array_rejected();
+	if (test__start_subtest("signed_module_kfunc_rejected"))
+		signed_module_kfunc_rejected();
+	if (test__start_subtest("signature_failure_logs"))
+		signature_failure_logs();
 	if (test__start_subtest("signature_too_large"))
 		signature_too_large();
+	if (test__start_subtest("signature_zero_size"))
+		signature_zero_size();
 	if (test__start_subtest("signature_bad_keyring"))
 		signature_bad_keyring();
 	if (test__start_subtest("metadata_ctx_max_entries_ignored"))
@@ -1114,6 +1811,8 @@ void test_signed_loader(void)
 		metadata_ctx_initial_value_ignored();
 	if (test__start_subtest("signature_authenticates_insns"))
 		signature_authenticates_insns();
+	if (test__start_subtest("signature_authenticates_metadata"))
+		signature_authenticates_metadata();
 	if (test__start_subtest("hash_requires_frozen"))
 		hash_requires_frozen();
 	if (test__start_subtest("no_update_after_freeze"))
@@ -1132,4 +1831,10 @@ void test_signed_loader(void)
 		map_hash_unsupported_type();
 	if (test__start_subtest("lsm_signature_verdict"))
 		lsm_signature_verdict();
+	if (test__start_subtest("signed_no_fd_array"))
+		signed_no_fd_array();
+	if (test__start_subtest("signed_map_by_fd_rejected"))
+		signed_map_by_fd_rejected();
+	if (test__start_subtest("signed_sparse_fd_array_rejected"))
+		signed_sparse_fd_array_rejected();
 }
diff --git a/tools/testing/selftests/bpf/prog_tests/sock_xattr.c b/tools/testing/selftests/bpf/prog_tests/sock_xattr.c
new file mode 100644
index 0000000000000..b5816e90f01a7
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/sock_xattr.c
@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2026 Christian Brauner */
+
+#include <errno.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/xattr.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <test_progs.h>
+
+#include "sock_read_xattr.skel.h"
+
+static const char xattr_value[] = "bpf_sock_value";
+static const char xattr_name[] = "user.bpf_test";
+
+static void test_read_sock_xattr(void)
+{
+	struct sockaddr_in addr = {};
+	struct sock_read_xattr *skel = NULL;
+	struct bpf_link *link = NULL;
+	int sock_fd = -1, err;
+
+	sock_fd = socket(AF_INET, SOCK_STREAM, 0);
+	if (!ASSERT_OK_FD(sock_fd, "socket"))
+		return;
+
+	err = fsetxattr(sock_fd, xattr_name, xattr_value, sizeof(xattr_value), 0);
+	if (!ASSERT_OK(err, "fsetxattr"))
+		goto out;
+
+	skel = sock_read_xattr__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "sock_read_xattr__open_and_load"))
+		goto out;
+
+	skel->bss->monitored_pid = sys_gettid();
+
+	/* Only attach the functional program; the verifier-only programs
+	 * above are not pid-gated and would clobber the shared globals.
+	 */
+	link = bpf_program__attach(skel->progs.read_sock_xattr);
+	if (!ASSERT_OK_PTR(link, "attach read_sock_xattr"))
+		goto out;
+
+	addr.sin_family = AF_INET;
+	addr.sin_port = htons(1234);
+	addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+	/* Only the lsm/socket_connect hook matters; the connect may fail. */
+	connect(sock_fd, (struct sockaddr *)&addr, sizeof(addr));
+
+	ASSERT_EQ(skel->data->read_ret, sizeof(xattr_value), "read_ret");
+	ASSERT_STREQ(skel->bss->value, xattr_value, "value");
+
+out:
+	bpf_link__destroy(link);
+	if (sock_fd >= 0)
+		close(sock_fd);
+	sock_read_xattr__destroy(skel);
+}
+
+void test_sock_xattr(void)
+{
+	RUN_TESTS(sock_read_xattr);
+
+	if (test__start_subtest("read_sock_xattr"))
+		test_read_sock_xattr();
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c
index cb3229711f93a..1fef6ec2ba7ae 100644
--- a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c
+++ b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c
@@ -7,7 +7,6 @@
 
 #include "test_progs.h"
 #include "test_skmsg_load_helpers.skel.h"
-#include "test_sockmap_update.skel.h"
 #include "test_sockmap_invalid_update.skel.h"
 #include "test_sockmap_skb_verdict_attach.skel.h"
 #include "test_sockmap_progs_query.skel.h"
@@ -235,53 +234,6 @@ static void test_skmsg_helpers_with_link(enum bpf_map_type map_type)
 	test_skmsg_load_helpers__destroy(skel);
 }
 
-static void test_sockmap_update(enum bpf_map_type map_type)
-{
-	int err, prog, src;
-	struct test_sockmap_update *skel;
-	struct bpf_map *dst_map;
-	const __u32 zero = 0;
-	char dummy[14] = {0};
-	LIBBPF_OPTS(bpf_test_run_opts, topts,
-		.data_in = dummy,
-		.data_size_in = sizeof(dummy),
-		.repeat = 1,
-	);
-	__s64 sk;
-
-	sk = connected_socket_v4();
-	if (!ASSERT_NEQ(sk, -1, "connected_socket_v4"))
-		return;
-
-	skel = test_sockmap_update__open_and_load();
-	if (!ASSERT_OK_PTR(skel, "open_and_load"))
-		goto close_sk;
-
-	prog = bpf_program__fd(skel->progs.copy_sock_map);
-	src = bpf_map__fd(skel->maps.src);
-	if (map_type == BPF_MAP_TYPE_SOCKMAP)
-		dst_map = skel->maps.dst_sock_map;
-	else
-		dst_map = skel->maps.dst_sock_hash;
-
-	err = bpf_map_update_elem(src, &zero, &sk, BPF_NOEXIST);
-	if (!ASSERT_OK(err, "update_elem(src)"))
-		goto out;
-
-	err = bpf_prog_test_run_opts(prog, &topts);
-	if (!ASSERT_OK(err, "test_run"))
-		goto out;
-	if (!ASSERT_NEQ(topts.retval, 0, "test_run retval"))
-		goto out;
-
-	compare_cookies(skel->maps.src, dst_map);
-
-out:
-	test_sockmap_update__destroy(skel);
-close_sk:
-	close(sk);
-}
-
 static void test_sockmap_invalid_update(void)
 {
 	struct test_sockmap_invalid_update *skel;
@@ -853,7 +805,7 @@ static void test_sockmap_many_socket(void)
 		return;
 	}
 
-	udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0);
+	udp = socket_loopback(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK);
 	if (udp < 0) {
 		close(dgram);
 		close(tcp);
@@ -922,7 +874,7 @@ static void test_sockmap_many_maps(void)
 		return;
 	}
 
-	udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0);
+	udp = socket_loopback(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK);
 	if (udp < 0) {
 		close(dgram);
 		close(tcp);
@@ -993,7 +945,7 @@ static void test_sockmap_same_sock(void)
 		return;
 	}
 
-	udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0);
+	udp = socket_loopback(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK);
 	if (udp < 0) {
 		close(dgram);
 		close(tcp);
@@ -1373,6 +1325,43 @@ static void test_sockmap_multi_channels(int sotype)
 	test_sockmap_pass_prog__destroy(skel);
 }
 
+/* A socket in a sockmap without a verdict program keeps its ingress data
+ * in sk_receive_queue: FIONREAD must account for it.
+ */
+static void test_sockmap_no_verdict_fionread(void)
+{
+	int err, map, zero = 0, sent, avail;
+	int c0 = -1, c1 = -1, p0 = -1, p1 = -1;
+	struct test_sockmap_pass_prog *skel;
+	char buf[256] = "0123456789";
+
+	skel = test_sockmap_pass_prog__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "open_and_load"))
+		return;
+	map = bpf_map__fd(skel->maps.sock_map_rx);
+
+	err = create_socket_pairs(AF_INET, SOCK_STREAM, &c0, &c1, &p0, &p1);
+	if (!ASSERT_OK(err, "create_socket_pairs()"))
+		goto out;
+
+	err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST);
+	if (!ASSERT_OK(err, "bpf_map_update_elem(c1)"))
+		goto out_close;
+
+	sent = xsend(p1, &buf, sizeof(buf), 0);
+	ASSERT_EQ(sent, sizeof(buf), "xsend(p1)");
+	avail = wait_for_fionread(c1, sizeof(buf), IO_TIMEOUT_SEC);
+	ASSERT_EQ(avail, sizeof(buf), "ioctl(FIONREAD)");
+
+out_close:
+	close(c0);
+	close(p0);
+	close(c1);
+	close(p1);
+out:
+	test_sockmap_pass_prog__destroy(skel);
+}
+
 void test_sockmap_basic(void)
 {
 	if (test__start_subtest("sockmap create_update_free"))
@@ -1385,10 +1374,6 @@ void test_sockmap_basic(void)
 		test_skmsg_helpers(BPF_MAP_TYPE_SOCKMAP);
 	if (test__start_subtest("sockhash sk_msg load helpers"))
 		test_skmsg_helpers(BPF_MAP_TYPE_SOCKHASH);
-	if (test__start_subtest("sockmap update"))
-		test_sockmap_update(BPF_MAP_TYPE_SOCKMAP);
-	if (test__start_subtest("sockhash update"))
-		test_sockmap_update(BPF_MAP_TYPE_SOCKHASH);
 	if (test__start_subtest("sockmap update in unsafe context"))
 		test_sockmap_invalid_update();
 	if (test__start_subtest("sockmap copy"))
@@ -1415,6 +1400,8 @@ void test_sockmap_basic(void)
 		test_sockmap_skb_verdict_shutdown();
 	if (test__start_subtest("sockmap skb_verdict fionread"))
 		test_sockmap_skb_verdict_fionread(true);
+	if (test__start_subtest("sockmap no_verdict fionread"))
+		test_sockmap_no_verdict_fionread();
 	if (test__start_subtest("sockmap skb_verdict fionread on drop"))
 		test_sockmap_skb_verdict_fionread(false);
 	if (test__start_subtest("sockmap skb_verdict change tail"))
diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c
index cc0c68bab9079..1c96a3cf4b979 100644
--- a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c
+++ b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c
@@ -53,8 +53,8 @@ static void test_insert_opened(struct test_sockmap_listen *skel __always_unused,
 			       int family, int sotype, int mapfd)
 {
 	u32 key = 0;
-	u64 value;
 	int err, s;
+	u64 value;
 
 	s = xsocket(family, sotype, 0);
 	if (s == -1)
@@ -63,11 +63,8 @@ static void test_insert_opened(struct test_sockmap_listen *skel __always_unused,
 	errno = 0;
 	value = s;
 	err = bpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST);
-	if (sotype == SOCK_STREAM) {
-		if (!err || errno != EOPNOTSUPP)
-			FAIL_ERRNO("map_update: expected EOPNOTSUPP");
-	} else if (err)
-		FAIL_ERRNO("map_update: expected success");
+	ASSERT_ERR(err, "map_update");
+	ASSERT_EQ(errno, EOPNOTSUPP, "errno");
 	xclose(s);
 }
 
@@ -77,8 +74,8 @@ static void test_insert_bound(struct test_sockmap_listen *skel __always_unused,
 	struct sockaddr_storage addr;
 	socklen_t len = 0;
 	u32 key = 0;
-	u64 value;
 	int err, s;
+	u64 value;
 
 	init_addr_loopback(family, &addr, &len);
 
@@ -93,8 +90,12 @@ static void test_insert_bound(struct test_sockmap_listen *skel __always_unused,
 	errno = 0;
 	value = s;
 	err = bpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST);
-	if (!err || errno != EOPNOTSUPP)
-		FAIL_ERRNO("map_update: expected EOPNOTSUPP");
+	if (sotype == SOCK_STREAM) {
+		ASSERT_ERR(err, "map_update");
+		ASSERT_EQ(errno, EOPNOTSUPP, "errno");
+	} else {
+		ASSERT_OK(err, "map_update");
+	}
 close:
 	xclose(s);
 }
@@ -1289,7 +1290,7 @@ static void test_ops(struct test_sockmap_listen *skel, struct bpf_map *map,
 		/* insert */
 		TEST(test_insert_invalid),
 		TEST(test_insert_opened),
-		TEST(test_insert_bound, SOCK_STREAM),
+		TEST(test_insert_bound),
 		TEST(test_insert),
 		/* delete */
 		TEST(test_delete_after_insert),
diff --git a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c
index 64fbda0823092..af8968b89ad7c 100644
--- a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c
+++ b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c
@@ -192,6 +192,8 @@ static int create_netkit(int mode, char *prim, char *peer)
 	req.n.nlmsg_len += sizeof(struct ifinfomsg);
 	addattr_l(&req.n, sizeof(req), IFLA_IFNAME, peer, strlen(peer));
 	addattr_nest_end(&req.n, peer_info);
+	addattr32(&req.n, sizeof(req), IFLA_NETKIT_SCRUB,
+		  NETKIT_SCRUB_NONE);
 	addattr_nest_end(&req.n, data);
 	addattr_nest_end(&req.n, linkinfo);
 
@@ -405,6 +407,24 @@ static int netns_load_bpf(const struct bpf_program *src_prog,
 	return -1;
 }
 
+static struct bpf_link *netns_attach_nk(const char *ns, int ifindex,
+					struct bpf_program *prog)
+{
+	LIBBPF_OPTS(bpf_netkit_opts, optl);
+	struct nstoken *nstoken = NULL;
+	struct bpf_link *link = NULL;
+
+	nstoken = open_netns(ns);
+	if (!ASSERT_OK_PTR(nstoken, "setns"))
+		goto cleanup;
+
+	link = bpf_program__attach_netkit(prog, ifindex, &optl);
+cleanup:
+	if (nstoken)
+		close_netns(nstoken);
+	return link;
+}
+
 static void test_tcp(int family, const char *addr, __u16 port)
 {
 	int listen_fd = -1, accept_fd = -1, client_fd = -1;
@@ -1082,6 +1102,53 @@ static void test_tc_redirect_peer(struct netns_setup_result *setup_result)
 	close_netns(nstoken);
 }
 
+static void test_tc_redirect_peer_ing(struct netns_setup_result *setup_result)
+{
+	struct test_tc_peer *skel;
+	struct nstoken *nstoken;
+	int err;
+
+	nstoken = open_netns(NS_FWD);
+	if (!ASSERT_OK_PTR(nstoken, "setns fwd"))
+		return;
+
+	skel = test_tc_peer__open();
+	if (!ASSERT_OK_PTR(skel, "test_tc_peer__open"))
+		goto done;
+
+	skel->rodata->IFINDEX_SRC = setup_result->ifindex_src_fwd;
+	skel->rodata->IFINDEX_DST = setup_result->ifindex_dst_fwd;
+	ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc_src_ing,
+		  BPF_NETKIT_PRIMARY), 0, "src_prog_attach_type");
+	ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc_dst_ing,
+		  BPF_NETKIT_PRIMARY), 0, "dst_prog_attach_type");
+
+	err = test_tc_peer__load(skel);
+	if (!ASSERT_OK(err, "test_tc_peer__load"))
+		goto done;
+
+	skel->links.tc_src_ing = netns_attach_nk(NS_SRC,
+						 setup_result->ifindex_src,
+						 skel->progs.tc_src_ing);
+	if (!ASSERT_OK_PTR(skel->links.tc_src_ing, "attach_src"))
+		goto done;
+	skel->links.tc_dst_ing = netns_attach_nk(NS_DST,
+						 setup_result->ifindex_dst,
+						 skel->progs.tc_dst_ing);
+	if (!ASSERT_OK_PTR(skel->links.tc_dst_ing, "attach_dst"))
+		goto done;
+
+	if (!ASSERT_OK(set_forwarding(false), "disable forwarding"))
+		goto done;
+
+	test_connectivity();
+
+done:
+	if (skel)
+		test_tc_peer__destroy(skel);
+	close_netns(nstoken);
+}
+
 static int tun_open(char *name)
 {
 	struct ifreq ifr;
@@ -1280,6 +1347,7 @@ static void *test_tc_redirect_run_tests(void *arg)
 
 	RUN_TEST(tc_redirect_peer, MODE_VETH);
 	RUN_TEST(tc_redirect_peer, MODE_NETKIT);
+	RUN_TEST(tc_redirect_peer_ing, MODE_NETKIT);
 	RUN_TEST(tc_redirect_peer_l3, MODE_VETH);
 	RUN_TEST(tc_redirect_peer_l3, MODE_NETKIT);
 	RUN_TEST(tc_redirect_neigh, MODE_VETH);
diff --git a/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c b/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c
new file mode 100644
index 0000000000000..d0ba2ca587b03
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <test_progs.h>
+
+#include "map_kptr.skel.h"
+
+void test_map_uninit_mem_exposure(void)
+{
+	size_t value_sz, slot_sz, lookup_sz, tail_sz;
+	int err, key, nr_cpus, cpu, map_fd;
+	__u8 *value = NULL, *zero = NULL;
+	struct bpf_program *prog;
+	struct map_kptr *skel;
+
+	nr_cpus = libbpf_num_possible_cpus();
+	if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus"))
+		return;
+
+	skel = map_kptr__open();
+	if (!ASSERT_OK_PTR(skel, "map_kptr__open"))
+		return;
+
+	bpf_object__for_each_program(prog, skel->obj) {
+		err = bpf_program__set_autoload(prog, false);
+		if (!ASSERT_OK(err, "bpf_program__set_autoload"))
+			goto out;
+	}
+
+	err = map_kptr__load(skel);
+	if (!ASSERT_OK(err, "map_kptr__load"))
+		goto out;
+
+	value_sz = bpf_map__value_size((skel)->maps.pcpu_array);
+	slot_sz = roundup(value_sz, 8);
+	tail_sz = slot_sz - value_sz;
+	if (!ASSERT_NEQ(tail_sz, 0, "tail_sz"))
+		goto out;
+
+	lookup_sz = slot_sz * nr_cpus;
+	map_fd = bpf_map__fd(skel->maps.pcpu_array);
+
+	value = malloc(lookup_sz);
+	zero = calloc(1, tail_sz);
+	if (!ASSERT_OK_PTR(value, "malloc value") || !ASSERT_OK_PTR(zero, "calloc zero"))
+		goto out;
+
+	key = 0;
+	memset(value, 0x2B, lookup_sz);
+	err = bpf_map_update_elem(map_fd, &key, value, BPF_ANY);
+	if (!ASSERT_OK(err, "bpf_map_update_elem"))
+		goto out;
+
+	memset(value, 0xFF, lookup_sz);
+	err = bpf_map_lookup_elem(map_fd, &key, value);
+	if (!ASSERT_OK(err, "bpf_map_lookup_elem"))
+		goto out;
+
+	for (cpu = 0; cpu < nr_cpus; cpu++) {
+		__u8 *tail = value + cpu * slot_sz + value_sz;
+
+		if (!ASSERT_MEMEQ(tail, zero, tail_sz, "zeroed tail bytes"))
+			goto out;
+	}
+
+out:
+	free(zero);
+	free(value);
+	map_kptr__destroy(skel);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/test_xsk.c b/tools/testing/selftests/bpf/prog_tests/test_xsk.c
index 6eb9096d084c0..477aedbb01bad 100644
--- a/tools/testing/selftests/bpf/prog_tests/test_xsk.c
+++ b/tools/testing/selftests/bpf/prog_tests/test_xsk.c
@@ -1164,8 +1164,8 @@ static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk,
 		       bool test_timeout)
 {
 	u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len;
+	struct xsk_umem_info *umem = ifobject->xsk_arr[0].umem_real;
 	struct pkt_stream *pkt_stream = xsk->pkt_stream;
-	struct xsk_umem_info *umem = xsk->umem;
 	bool use_poll = ifobject->use_poll;
 	struct pollfd fds = { };
 	int ret;
@@ -1514,7 +1514,7 @@ static int thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobjec
 	umem_tx->base_addr = 0;
 	umem_tx->next_buffer = 0;
 
-	ret = xsk_configure(test, ifobject, umem_tx, true);
+	ret = xsk_configure(test, ifobject, umem_rx, true);
 	if (ret)
 		return ret;
 	ifobject->xsk = &ifobject->xsk_arr[0];
diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c
index f02ffc7f41d78..0aa9532a05cfe 100644
--- a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c
+++ b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c
@@ -460,6 +460,73 @@ static void test_intersect(void)
 	tracing_multi_intersect__destroy(skel);
 }
 
+static void test_fentry_after_multi(void)
+{
+	static const char * const funcs[] = {
+		"bpf_fentry_test1",
+	};
+	struct bpf_link *fentry_link = NULL, *multi_link = NULL;
+	struct tracing_multi_intersect *skel = NULL;
+	LIBBPF_OPTS(bpf_tracing_multi_opts, opts);
+	LIBBPF_OPTS(bpf_test_run_opts, topts);
+	__u32 *ids = NULL;
+	int err;
+
+	skel = tracing_multi_intersect__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "tracing_multi_intersect__open_and_load"))
+		return;
+
+	skel->bss->pid = getpid();
+
+	ids = get_ids(funcs, ARRAY_SIZE(funcs), NULL);
+	if (!ASSERT_OK_PTR(ids, "get_ids"))
+		goto cleanup;
+
+	opts.ids = ids;
+	opts.cnt = ARRAY_SIZE(funcs);
+	multi_link = bpf_program__attach_tracing_multi(skel->progs.fentry_1, NULL, &opts);
+	if (!ASSERT_OK_PTR(multi_link, "attach_multi"))
+		goto cleanup;
+
+	fentry_link = bpf_program__attach(skel->progs.fentry);
+	if (!ASSERT_OK_PTR(fentry_link, "attach_fentry"))
+		goto cleanup;
+
+	err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.fentry_1), &topts);
+	if (!ASSERT_OK(err, "test_run"))
+		goto cleanup;
+	ASSERT_EQ(skel->bss->test_result_fentry_1, 1, "multi_fentry");
+	ASSERT_EQ(skel->bss->test_result_fentry, 1, "fentry");
+
+	err = bpf_link__destroy(fentry_link);
+	fentry_link = NULL;
+	if (!ASSERT_OK(err, "destroy_fentry"))
+		goto cleanup;
+
+	err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.fentry_1), &topts);
+	if (!ASSERT_OK(err, "test_run_multi"))
+		goto cleanup;
+	ASSERT_EQ(skel->bss->test_result_fentry_1, 2, "multi_fentry_only");
+	ASSERT_EQ(skel->bss->test_result_fentry, 1, "fentry_detached");
+
+	err = bpf_link__destroy(multi_link);
+	multi_link = NULL;
+	if (!ASSERT_OK(err, "destroy_multi"))
+		goto cleanup;
+
+	err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.fentry_1), &topts);
+	if (!ASSERT_OK(err, "test_run_detached"))
+		goto cleanup;
+	ASSERT_EQ(skel->bss->test_result_fentry_1, 2, "multi_fentry_detached");
+	ASSERT_EQ(skel->bss->test_result_fentry, 1, "fentry_still_detached");
+
+cleanup:
+	bpf_link__destroy(fentry_link);
+	bpf_link__destroy(multi_link);
+	free(ids);
+	tracing_multi_intersect__destroy(skel);
+}
+
 static void test_session(void)
 {
 	LIBBPF_OPTS(bpf_test_run_opts, topts);
@@ -957,4 +1024,6 @@ void test_tracing_multi_test(void)
 	if (test__start_subtest("attach_api_fails"))
 		test_attach_api_fails();
 	RUN_TESTS(tracing_multi_verifier);
+	if (test__start_subtest("fentry_after_multi"))
+		test_fentry_after_multi();
 }
diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c b/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c
index 955a37751b52d..e193206fc5d20 100644
--- a/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c
+++ b/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c
@@ -4,6 +4,8 @@
 
 #ifdef __x86_64__
 
+#define _GNU_SOURCE
+#include <sched.h>
 #include <unistd.h>
 #include <asm/ptrace.h>
 #include <linux/compiler.h>
@@ -13,11 +15,12 @@
 #include <sys/syscall.h>
 #include <sys/prctl.h>
 #include <asm/prctl.h>
+#include <stdnoreturn.h>
 #include "uprobe_syscall.skel.h"
 #include "uprobe_syscall_executed.skel.h"
 #include "bpf/libbpf_internal.h"
 
-#define USDT_NOP .byte 0x0f, 0x1f, 0x44, 0x00, 0x00
+#define USDT_NOP .byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00
 #include "usdt.h"
 
 #pragma GCC diagnostic ignored "-Wattributes"
@@ -26,7 +29,7 @@ __attribute__((aligned(16)))
 __nocf_check __weak __naked unsigned long uprobe_regs_trigger(void)
 {
 	asm volatile (
-		".byte 0x0f, 0x1f, 0x44, 0x00, 0x00\n" /* nop5 */
+		".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */
 		"movq $0xdeadbeef, %rax\n"
 		"ret\n"
 	);
@@ -345,9 +348,9 @@ static void test_uretprobe_syscall_call(void)
 __attribute__((aligned(16)))
 __nocf_check __weak __naked void uprobe_test(void)
 {
-	asm volatile ("					\n"
-		".byte 0x0f, 0x1f, 0x44, 0x00, 0x00	\n"
-		"ret					\n"
+	asm volatile (
+		".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */
+		"ret\n"
 	);
 }
 
@@ -357,6 +360,50 @@ __nocf_check __weak void usdt_test(void)
 	USDT(optimized_uprobe, usdt);
 }
 
+/*
+ * Assembly-level red zone clobbering test. Stores known values in the
+ * red zone (below RSP), executes a nop10 (uprobe site), and checks that
+ * the values survived. Returns 0 if intact, 1 if clobbered.
+ *
+ * The nop5 optimization used CALL (which pushes a return address to
+ * [rsp-8]), the value at -8(%rsp) was overwritten. The nop10 optimization
+ * should escape that by moving stackpointer below the redzone before
+ * doing the CALL.
+ *
+ * Align the code at 64 bytes, to make sure nop10 is not on page boundary.
+ */
+__attribute__((aligned(64)))
+__nocf_check __weak __naked unsigned long uprobe_red_zone_test(void)
+{
+	asm volatile (
+		"movabs $0x1111111111111111, %%rax\n"
+		"movq   %%rax, -8(%%rsp)\n"
+		"movabs $0x2222222222222222, %%rax\n"
+		"movq   %%rax, -16(%%rsp)\n"
+		"movabs $0x3333333333333333, %%rax\n"
+		"movq   %%rax, -24(%%rsp)\n"
+
+		".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10: uprobe site */
+
+		"movabs $0x1111111111111111, %%rax\n"
+		"cmpq   %%rax, -8(%%rsp)\n"
+		"jne    1f\n"
+		"movabs $0x2222222222222222, %%rax\n"
+		"cmpq   %%rax, -16(%%rsp)\n"
+		"jne    1f\n"
+		"movabs $0x3333333333333333, %%rax\n"
+		"cmpq   %%rax, -24(%%rsp)\n"
+		"jne    1f\n"
+
+		"xorl   %%eax, %%eax\n"
+		"retq\n"
+		"1:\n"
+		"movl   $1, %%eax\n"
+		"retq\n"
+		::: "rax", "memory"
+	);
+}
+
 static int find_uprobes_trampoline(void *tramp_addr)
 {
 	void *start, *end;
@@ -388,14 +435,15 @@ static int find_uprobes_trampoline(void *tramp_addr)
 	return ret;
 }
 
-static unsigned char nop5[5] = { 0x0f, 0x1f, 0x44, 0x00, 0x00 };
+static unsigned char nop10[10]  = { 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 };
+static unsigned char lea_rsp[5] = { 0x48, 0x8d, 0x64, 0x24, 0x80 };
 
-static void *find_nop5(void *fn)
+static void *find_nop10(void *fn)
 {
 	int i;
 
-	for (i = 0; i < 10; i++) {
-		if (!memcmp(nop5, fn + i, 5))
+	for (i = 0; i < 128; i++) {
+		if (!memcmp(nop10, fn + i, 10))
 			return fn + i;
 	}
 	return NULL;
@@ -403,6 +451,16 @@ static void *find_nop5(void *fn)
 
 typedef void (__attribute__((nocf_check)) *trigger_t)(void);
 
+static void check_attach_notrigger(struct uprobe_syscall_executed *skel,
+				   void *addr, int executed)
+{
+	unsigned char *op = addr;
+
+	/* Make sure bpf program was not executed. */
+	ASSERT_EQ(skel->bss->executed, executed, "executed");
+	ASSERT_EQ(*op, 0xcc, "int3");
+}
+
 static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigger,
 			  void *addr, int executed)
 {
@@ -420,7 +478,8 @@ static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigge
 	ASSERT_EQ(skel->bss->executed, executed, "executed");
 
 	/* .. and check the trampoline is as expected. */
-	call = (struct __arch_relative_insn *) addr;
+	ASSERT_OK(memcmp(addr, lea_rsp, 5), "lea_rsp");
+	call = (struct __arch_relative_insn *)(addr + 5);
 	tramp = (void *) (call + 1) + call->raddr;
 	ASSERT_EQ(call->op, 0xe8, "call");
 	ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline");
@@ -428,21 +487,26 @@ static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigge
 	return tramp;
 }
 
-static void check_detach(void *addr, void *tramp)
+static bool check_detach(void *addr, void *tramp)
 {
+	static const unsigned char nop10_prefix[] = { 0x66, 0x2e, 0x0f, 0x1f, 0x84 };
+	bool ok = true;
+
 	/* [uprobes_trampoline] stays after detach */
-	ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline");
-	ASSERT_OK(memcmp(addr, nop5, 5), "nop5");
+	ok &= ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline");
+	ok &= ASSERT_OK(memcmp(addr, nop10_prefix, 5), "nop10_prefix");
+	return ok;
 }
 
-static void check(struct uprobe_syscall_executed *skel, struct bpf_link *link,
-		  trigger_t trigger, void *addr, int executed)
+static void *check(struct uprobe_syscall_executed *skel, struct bpf_link *link,
+		   trigger_t trigger, void *addr, int executed)
 {
 	void *tramp;
 
 	tramp = check_attach(skel, trigger, addr, executed);
 	bpf_link__destroy(link);
 	check_detach(addr, tramp);
+	return tramp;
 }
 
 static void test_uprobe_legacy(void)
@@ -453,6 +517,7 @@ static void test_uprobe_legacy(void)
 	);
 	struct bpf_link *link;
 	unsigned long offset;
+	void *tramp;
 
 	offset = get_uprobe_offset(&uprobe_test);
 	if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
@@ -470,7 +535,30 @@ static void test_uprobe_legacy(void)
 	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts"))
 		goto cleanup;
 
-	check(skel, link, uprobe_test, uprobe_test, 2);
+	tramp = check(skel, link, uprobe_test, uprobe_test, 2);
+
+	/* reattach and detach without triggering optimization */
+	link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe,
+					       0, "/proc/self/exe", offset, NULL);
+	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts"))
+		goto cleanup;
+
+	check_attach_notrigger(skel, uprobe_test, 2);
+
+	bpf_link__destroy(link);
+	if (!check_detach(uprobe_test, tramp))
+		goto cleanup;
+
+	uprobe_test();
+	ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe");
+
+	/* reattach with triggering optimization */
+	link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe,
+				0, "/proc/self/exe", offset, NULL);
+	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts"))
+		goto cleanup;
+
+	check(skel, link, uprobe_test, uprobe_test, 4);
 
 	/* uretprobe */
 	skel->bss->executed = 0;
@@ -492,6 +580,7 @@ static void test_uprobe_multi(void)
 	LIBBPF_OPTS(bpf_uprobe_multi_opts, opts);
 	struct bpf_link *link;
 	unsigned long offset;
+	void *tramp;
 
 	offset = get_uprobe_offset(&uprobe_test);
 	if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
@@ -512,7 +601,30 @@ static void test_uprobe_multi(void)
 	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
 		goto cleanup;
 
-	check(skel, link, uprobe_test, uprobe_test, 2);
+	tramp = check(skel, link, uprobe_test, uprobe_test, 2);
+
+	/* reattach and detach without triggering optimization */
+	link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi,
+				0, "/proc/self/exe", NULL, &opts);
+	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
+		goto cleanup;
+
+	check_attach_notrigger(skel, uprobe_test, 2);
+
+	bpf_link__destroy(link);
+	if (!check_detach(uprobe_test, tramp))
+		goto cleanup;
+
+	uprobe_test();
+	ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe");
+
+	/* reattach with triggering optimization */
+	link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi,
+				0, "/proc/self/exe", NULL, &opts);
+	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
+		goto cleanup;
+
+	check(skel, link, uprobe_test, uprobe_test, 4);
 
 	/* uretprobe.multi */
 	skel->bss->executed = 0;
@@ -536,6 +648,7 @@ static void test_uprobe_session(void)
 	);
 	struct bpf_link *link;
 	unsigned long offset;
+	void *tramp;
 
 	offset = get_uprobe_offset(&uprobe_test);
 	if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
@@ -555,7 +668,30 @@ static void test_uprobe_session(void)
 	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
 		goto cleanup;
 
-	check(skel, link, uprobe_test, uprobe_test, 4);
+	tramp = check(skel, link, uprobe_test, uprobe_test, 4);
+
+	/* reattach and detach without triggering optimization */
+	link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session,
+				0, "/proc/self/exe", NULL, &opts);
+	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
+		goto cleanup;
+
+	check_attach_notrigger(skel, uprobe_test, 4);
+
+	bpf_link__destroy(link);
+	if (!check_detach(uprobe_test, tramp))
+		goto cleanup;
+
+	uprobe_test();
+	ASSERT_EQ(skel->bss->executed, 4, "executed_no_probe");
+
+	/* reattach with triggering optimization */
+	link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session,
+				0, "/proc/self/exe", NULL, &opts);
+	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
+		goto cleanup;
+
+	check(skel, link, uprobe_test, uprobe_test, 8);
 
 cleanup:
 	uprobe_syscall_executed__destroy(skel);
@@ -565,11 +701,11 @@ static void test_uprobe_usdt(void)
 {
 	struct uprobe_syscall_executed *skel;
 	struct bpf_link *link;
-	void *addr;
+	void *addr, *tramp;
 
 	errno = 0;
-	addr = find_nop5(usdt_test);
-	if (!ASSERT_OK_PTR(addr, "find_nop5"))
+	addr = find_nop10(usdt_test);
+	if (!ASSERT_OK_PTR(addr, "find_nop10"))
 		return;
 
 	skel = uprobe_syscall_executed__open_and_load();
@@ -584,7 +720,32 @@ static void test_uprobe_usdt(void)
 	if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt"))
 		goto cleanup;
 
-	check(skel, link, usdt_test, addr, 2);
+	tramp = check(skel, link, usdt_test, addr, 2);
+
+	/* reattach and detach without triggering optimization */
+	link = bpf_program__attach_usdt(skel->progs.test_usdt,
+				-1 /* all PIDs */, "/proc/self/exe",
+				"optimized_uprobe", "usdt", NULL);
+	if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt"))
+		goto cleanup;
+
+	check_attach_notrigger(skel, addr, 2);
+
+	bpf_link__destroy(link);
+	if (!check_detach(addr, tramp))
+		goto cleanup;
+
+	usdt_test();
+	ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe");
+
+	/* reattach with triggering optimization */
+	link = bpf_program__attach_usdt(skel->progs.test_usdt,
+				-1 /* all PIDs */, "/proc/self/exe",
+				"optimized_uprobe", "usdt", NULL);
+	if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt"))
+		goto cleanup;
+
+	check(skel, link, usdt_test, addr, 4);
 
 cleanup:
 	uprobe_syscall_executed__destroy(skel);
@@ -757,12 +918,124 @@ static void test_uprobe_race(void)
 #define __NR_uprobe 336
 #endif
 
+static void test_uprobe_red_zone(void)
+{
+	struct uprobe_syscall_executed *skel;
+	struct bpf_link *link;
+	void *nop10_addr;
+	size_t offset;
+	int i;
+
+	nop10_addr = find_nop10(uprobe_red_zone_test);
+	if (!ASSERT_NEQ(nop10_addr, NULL, "find_nop10"))
+		return;
+
+	skel = uprobe_syscall_executed__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "open_and_load"))
+		return;
+
+	offset = get_uprobe_offset(nop10_addr);
+	link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe,
+			0, "/proc/self/exe", offset, NULL);
+	if (!ASSERT_OK_PTR(link, "attach_uprobe"))
+		goto cleanup;
+
+	for (i = 0; i < 10; i++)
+		ASSERT_EQ(uprobe_red_zone_test(), 0, "red_zone_intact");
+
+	bpf_link__destroy(link);
+
+cleanup:
+	uprobe_syscall_executed__destroy(skel);
+}
+
 static void test_uprobe_error(void)
 {
 	long err = syscall(__NR_uprobe);
 
 	ASSERT_EQ(err, -1, "error");
-	ASSERT_EQ(errno, ENXIO, "errno");
+	ASSERT_EQ(errno, EPROTO, "errno");
+}
+
+__attribute__((aligned(16)))
+__nocf_check __weak __naked void uprobe_fork_test(void)
+{
+	asm volatile (
+		".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */
+		"ret\n"
+	);
+}
+
+static noreturn int child_func(void *arg)
+{
+	struct uprobe_syscall_executed *skel = arg;
+
+	/* Make sure the child's probe is still there and optimized.. */
+	if (memcmp(uprobe_fork_test, lea_rsp, sizeof(lea_rsp)))
+		_exit(1);
+
+	skel->bss->pid = getpid();
+
+	/* .. and it executes properly. */
+	uprobe_fork_test();
+
+	if (skel->bss->executed != 3)
+		_exit(2);
+
+	_exit(0);
+}
+
+static void test_uprobe_fork_optimized(bool clone_vm)
+{
+	struct uprobe_syscall_executed *skel = NULL;
+	unsigned long offset;
+	int pid, status, err;
+	char stack[65535];
+
+	offset = get_uprobe_offset(&uprobe_fork_test);
+	if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
+		return;
+
+	skel = uprobe_syscall_executed__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "open_and_load"))
+		goto cleanup;
+
+	skel->links.test_uprobe = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe,
+					-1, "/proc/self/exe", offset, NULL);
+	if (!ASSERT_OK_PTR(skel->links.test_uprobe, "attach_uprobe"))
+		goto cleanup;
+
+	skel->bss->pid = getpid();
+
+	/* Trigger optimization of uprobe in uprobe_fork_test.  */
+	uprobe_fork_test();
+	uprobe_fork_test();
+
+	/* Make sure it got optimied. */
+	if (!ASSERT_OK(memcmp(uprobe_fork_test, lea_rsp, sizeof(lea_rsp)), "optimized"))
+		goto cleanup;
+
+	if (clone_vm) {
+		pid = clone(child_func, stack + sizeof(stack), CLONE_VM|SIGCHLD, skel);
+		if (!ASSERT_GT(pid, 0, "clone"))
+			goto cleanup;
+	} else {
+		pid = fork();
+		if (!ASSERT_GE(pid, 0, "fork"))
+			goto cleanup;
+		if (pid == 0)
+			child_func(skel);
+	}
+
+	/* Wait for the child and verify it exited properly with 0. */
+	err = waitpid(pid, &status, 0);
+	if (ASSERT_EQ(err, pid, "waitpid")) {
+		ASSERT_EQ(WIFEXITED(status), 1, "child_exited");
+		ASSERT_EQ(WEXITSTATUS(status), 0, "child_exit_code");
+	}
+
+cleanup:
+	uprobe_syscall_executed__destroy(skel);
 }
 
 static void __test_uprobe_syscall(void)
@@ -783,6 +1056,12 @@ static void __test_uprobe_syscall(void)
 		test_uprobe_usdt();
 	if (test__start_subtest("uprobe_race"))
 		test_uprobe_race();
+	if (test__start_subtest("uprobe_red_zone"))
+		test_uprobe_red_zone();
+	if (test__start_subtest("uprobe_optimized_fork"))
+		test_uprobe_fork_optimized(false);
+	if (test__start_subtest("uprobe_optimized_clone_vm"))
+		test_uprobe_fork_optimized(true);
 	if (test__start_subtest("uprobe_error"))
 		test_uprobe_error();
 	if (test__start_subtest("uprobe_regs_equal"))
diff --git a/tools/testing/selftests/bpf/prog_tests/usdt.c b/tools/testing/selftests/bpf/prog_tests/usdt.c
index 69759b27794d7..8004c9568ffa5 100644
--- a/tools/testing/selftests/bpf/prog_tests/usdt.c
+++ b/tools/testing/selftests/bpf/prog_tests/usdt.c
@@ -250,9 +250,10 @@ static void subtest_basic_usdt(bool optimized)
 #ifdef __x86_64__
 extern void usdt_1(void);
 extern void usdt_2(void);
+extern void usdt_red_zone_trigger(void);
 
 static unsigned char nop1[1] = { 0x90 };
-static unsigned char nop1_nop5_combo[6] = { 0x90, 0x0f, 0x1f, 0x44, 0x00, 0x00 };
+static unsigned char nop1_nop10_combo[11] = { 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 };
 
 static void *find_instr(void *fn, unsigned char *instr, size_t cnt)
 {
@@ -271,17 +272,17 @@ static void subtest_optimized_attach(void)
 	__u8 *addr_1, *addr_2;
 
 	/* usdt_1 USDT probe has single nop instruction */
-	addr_1 = find_instr(usdt_1, nop1_nop5_combo, 6);
-	if (!ASSERT_NULL(addr_1, "usdt_1_find_nop1_nop5_combo"))
+	addr_1 = find_instr(usdt_1, nop1_nop10_combo, 11);
+	if (!ASSERT_NULL(addr_1, "usdt_1_find_nop1_nop10_combo"))
 		return;
 
 	addr_1 = find_instr(usdt_1, nop1, 1);
 	if (!ASSERT_OK_PTR(addr_1, "usdt_1_find_nop1"))
 		return;
 
-	/* usdt_2 USDT probe has nop,nop5 instructions combo */
-	addr_2 = find_instr(usdt_2, nop1_nop5_combo, 6);
-	if (!ASSERT_OK_PTR(addr_2, "usdt_2_find_nop1_nop5_combo"))
+	/* usdt_2 USDT probe has nop,nop10 instructions combo */
+	addr_2 = find_instr(usdt_2, nop1_nop10_combo, 11);
+	if (!ASSERT_OK_PTR(addr_2, "usdt_2_find_nop1_nop10_combo"))
 		return;
 
 	skel = test_usdt__open_and_load();
@@ -309,12 +310,12 @@ static void subtest_optimized_attach(void)
 
 	bpf_link__destroy(skel->links.usdt_executed);
 
-	/* we expect the nop5 ip */
+	/* we expect the nop10 ip */
 	skel->bss->expected_ip = (unsigned long) addr_2 + 1;
 
 	/*
 	 * Attach program on top of usdt_2 which is probe defined on top
-	 * of nop1,nop5 combo, so the probe gets optimized on top of nop5.
+	 * of nop1,nop10 combo, so the probe gets optimized on top of nop10.
 	 */
 	skel->links.usdt_executed = bpf_program__attach_usdt(skel->progs.usdt_executed,
 						     0 /*self*/, "/proc/self/exe",
@@ -328,13 +329,64 @@ static void subtest_optimized_attach(void)
 	/* nop stays on addr_2 address */
 	ASSERT_EQ(*addr_2, 0x90, "nop");
 
-	/* call is on addr_2 + 1 address */
-	ASSERT_EQ(*(addr_2 + 1), 0xe8, "call");
+	/*
+	 * lea -0x80(%rsp), %rsp
+	 * call ...
+	 */
+	static unsigned char expected[] = { 0x48, 0x8d, 0x64, 0x24, 0x80, 0xe8 };
+
+	ASSERT_MEMEQ(addr_2 + 1, expected, sizeof(expected), "lea_and_call");
 	ASSERT_EQ(skel->bss->executed, 4, "executed");
 
 cleanup:
 	test_usdt__destroy(skel);
 }
+
+/*
+ * Test that USDT arguments survive nop10 optimization in a function where
+ * the compiler places operands in the red zone.
+ *
+ * Signal handlers are prone to having the compiler place USDT argument
+ * operands in the red zone (below rsp).
+ *
+ * The nop5 optimization used CALL (which pushes a return address to
+ * [rsp-8]), the value at -8(%rsp) was overwritten. The nop10 optimization
+ * should escape that by moving stackpointer below the redzone before
+ * doing the CALL.
+ */
+static void subtest_optimized_red_zone(void)
+{
+	struct test_usdt *skel;
+	int i;
+
+	skel = test_usdt__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "open_and_load"))
+		return;
+
+	skel->bss->expected_arg[0] = 0xDEADBEEF;
+	skel->bss->expected_arg[1] = 0xCAFEBABE;
+	skel->bss->expected_arg[2] = 0xFEEDFACE;
+	skel->bss->expected_pid = getpid();
+
+	skel->links.usdt_check_arg = bpf_program__attach_usdt(
+		skel->progs.usdt_check_arg, 0, "/proc/self/exe",
+		"optimized_attach", "usdt_red_zone", NULL);
+	if (!ASSERT_OK_PTR(skel->links.usdt_check_arg, "attach_usdt_red_zone"))
+		goto cleanup;
+
+	for (i = 0; i < 10; i++)
+		usdt_red_zone_trigger();
+
+	ASSERT_EQ(skel->bss->arg_total, 10, "arg_total");
+	ASSERT_EQ(skel->bss->arg_bad, 0, "arg_bad");
+	ASSERT_EQ(skel->bss->arg_last[0], 0xDEADBEEF, "arg_last_1");
+	ASSERT_EQ(skel->bss->arg_last[1], 0xCAFEBABE, "arg_last_2");
+	ASSERT_EQ(skel->bss->arg_last[2], 0xFEEDFACE, "arg_last_3");
+
+cleanup:
+	test_usdt__destroy(skel);
+}
+
 #endif
 
 unsigned short test_usdt_100_semaphore SEC(".probes");
@@ -608,6 +660,8 @@ void test_usdt(void)
 		subtest_basic_usdt(true);
 	if (test__start_subtest("optimized_attach"))
 		subtest_optimized_attach();
+	if (test__start_subtest("optimized_red_zone"))
+		subtest_optimized_red_zone();
 #endif
 	if (test__start_subtest("multispec"))
 		subtest_multispec_usdt();
diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c
index 8a3d69e2453c3..be97f6887f0e7 100644
--- a/tools/testing/selftests/bpf/prog_tests/verifier.c
+++ b/tools/testing/selftests/bpf/prog_tests/verifier.c
@@ -78,6 +78,7 @@
 #include "verifier_precision.skel.h"
 #include "verifier_prevent_map_lookup.skel.h"
 #include "verifier_private_stack.skel.h"
+#include "verifier_ptr_to_buf.skel.h"
 #include "verifier_raw_stack.skel.h"
 #include "verifier_raw_tp_writable.skel.h"
 #include "verifier_reg_equal.skel.h"
@@ -230,6 +231,7 @@ void test_verifier_or_jmp32_k(void)           { RUN(verifier_or_jmp32_k); }
 void test_verifier_precision(void)            { RUN(verifier_precision); }
 void test_verifier_prevent_map_lookup(void)   { RUN(verifier_prevent_map_lookup); }
 void test_verifier_private_stack(void)        { RUN(verifier_private_stack); }
+void test_verifier_ptr_to_buf(void)           { RUN(verifier_ptr_to_buf); }
 void test_verifier_raw_stack(void)            { RUN(verifier_raw_stack); }
 void test_verifier_raw_tp_writable(void)      { RUN(verifier_raw_tp_writable); }
 void test_verifier_reg_equal(void)            { RUN(verifier_reg_equal); }
diff --git a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h
index d8dacef37c163..c4b4388545650 100644
--- a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h
+++ b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h
@@ -8,6 +8,9 @@
 #define AF_INET			2
 #define AF_INET6		10
 
+/* include/linux/net.h */
+#define SOCK_TYPE_MASK		0xf
+
 #define SOL_SOCKET		1
 #define SO_REUSEADDR		2
 #define SO_SNDBUF		7
diff --git a/tools/testing/selftests/bpf/progs/btf_data.c b/tools/testing/selftests/bpf/progs/btf_data.c
index baa525275bde3..8587658012c3e 100644
--- a/tools/testing/selftests/bpf/progs/btf_data.c
+++ b/tools/testing/selftests/bpf/progs/btf_data.c
@@ -48,3 +48,13 @@ int func(struct root_struct *root)
 {
 	return 0;
 }
+
+int kfunc_a(struct root_struct *root)
+{
+	return 0;
+}
+
+int kfunc_b(struct root_struct *root)
+{
+	return 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/cpumask_common.h b/tools/testing/selftests/bpf/progs/cpumask_common.h
index 86085b79f5cac..8fe01308d2107 100644
--- a/tools/testing/selftests/bpf/progs/cpumask_common.h
+++ b/tools/testing/selftests/bpf/progs/cpumask_common.h
@@ -61,7 +61,7 @@ u32 bpf_cpumask_any_distribute(const struct cpumask *src) __ksym __weak;
 u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1,
 				   const struct cpumask *src2) __ksym __weak;
 u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym __weak;
-int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t src__sz) __ksym __weak;
+int bpf_cpumask_populate(struct bpf_cpumask *cpumask, void *src, size_t src__sz) __ksym __weak;
 
 void bpf_rcu_read_lock(void) __ksym __weak;
 void bpf_rcu_read_unlock(void) __ksym __weak;
diff --git a/tools/testing/selftests/bpf/progs/cpumask_failure.c b/tools/testing/selftests/bpf/progs/cpumask_failure.c
index 4c45346fe6f7a..74b4cd4bcdbbb 100644
--- a/tools/testing/selftests/bpf/progs/cpumask_failure.c
+++ b/tools/testing/selftests/bpf/progs/cpumask_failure.c
@@ -231,7 +231,7 @@ int BPF_PROG(test_populate_invalid_destination, struct task_struct *task, u64 cl
 	u64 bits;
 	int ret;
 
-	ret = bpf_cpumask_populate((struct cpumask *)invalid, &bits, sizeof(bits));
+	ret = bpf_cpumask_populate(invalid, &bits, sizeof(bits));
 	if (!ret)
 		err = 2;
 
@@ -252,7 +252,7 @@ int BPF_PROG(test_populate_invalid_source, struct task_struct *task, u64 clone_f
 		return 0;
 	}
 
-	ret = bpf_cpumask_populate((struct cpumask *)local, garbage, 8);
+	ret = bpf_cpumask_populate(local, garbage, 8);
 	if (!ret)
 		err = 2;
 
@@ -260,3 +260,22 @@ int BPF_PROG(test_populate_invalid_source, struct task_struct *task, u64 clone_f
 
 	return 0;
 }
+
+SEC("tp_btf/task_newtask")
+__failure __msg("expected pointer to STRUCT bpf_cpumask but R1 has a pointer to STRUCT cpumask")
+int BPF_PROG(test_populate_borrowed_destination, struct task_struct *task, u64 clone_flags)
+{
+	u64 bits;
+	int ret;
+
+	/*
+	 * task->cpus_ptr is a borrowed, read-only struct cpumask *, not an
+	 * owned struct bpf_cpumask *. The verifier must reject it as a
+	 * writable destination for bpf_cpumask_populate().
+	 */
+	ret = bpf_cpumask_populate((struct bpf_cpumask *)task->cpus_ptr, &bits, sizeof(bits));
+	if (!ret)
+		err = 2;
+
+	return 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/cpumask_success.c b/tools/testing/selftests/bpf/progs/cpumask_success.c
index 774706e7b058b..36f77b9732d46 100644
--- a/tools/testing/selftests/bpf/progs/cpumask_success.c
+++ b/tools/testing/selftests/bpf/progs/cpumask_success.c
@@ -785,7 +785,7 @@ int BPF_PROG(test_populate_reject_small_mask, struct task_struct *task, u64 clon
 		return 0;
 
 	/* The kfunc should prevent this operation */
-	ret = bpf_cpumask_populate((struct cpumask *)local, &toofewbits, sizeof(toofewbits));
+	ret = bpf_cpumask_populate(local, &toofewbits, sizeof(toofewbits));
 	if (ret != -EACCES)
 		err = 2;
 
@@ -824,7 +824,7 @@ int BPF_PROG(test_populate_reject_unaligned, struct task_struct *task, u64 clone
 	/* Misalign the source array by a byte. */
 	src = &((char *)bits)[1];
 
-	ret = bpf_cpumask_populate((struct cpumask *)mask, src, CPUMASK_TEST_MASKLEN);
+	ret = bpf_cpumask_populate(mask, src, CPUMASK_TEST_MASKLEN);
 	if (ret != -EINVAL)
 		err = 2;
 
@@ -855,7 +855,7 @@ int BPF_PROG(test_populate, struct task_struct *task, u64 clone_flags)
 	}
 
 	/* Pass the entire bits array, the kfunc will only copy the valid bits. */
-	ret = bpf_cpumask_populate((struct cpumask *)mask, bits, CPUMASK_TEST_MASKLEN);
+	ret = bpf_cpumask_populate(mask, bits, CPUMASK_TEST_MASKLEN);
 	if (ret) {
 		err = 2;
 		goto out;
diff --git a/tools/testing/selftests/bpf/progs/freplace_cls_redirect.c b/tools/testing/selftests/bpf/progs/freplace_cls_redirect.c
deleted file mode 100644
index 7e94412d47a5c..0000000000000
--- a/tools/testing/selftests/bpf/progs/freplace_cls_redirect.c
+++ /dev/null
@@ -1,34 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-// Copyright (c) 2020 Facebook
-
-#include <linux/stddef.h>
-#include <linux/bpf.h>
-#include <linux/pkt_cls.h>
-#include <bpf/bpf_endian.h>
-#include <bpf/bpf_helpers.h>
-
-struct {
-	__uint(type, BPF_MAP_TYPE_SOCKMAP);
-	__type(key, int);
-	__type(value, int);
-	__uint(max_entries, 2);
-} sock_map SEC(".maps");
-
-SEC("freplace/cls_redirect")
-int freplace_cls_redirect_test(struct __sk_buff *skb)
-{
-	int ret = 0;
-	const int zero = 0;
-	struct bpf_sock *sk;
-
-	sk = bpf_map_lookup_elem(&sock_map, &zero);
-	if (!sk)
-		return TC_ACT_SHOT;
-
-	ret = bpf_map_update_elem(&sock_map, &zero, sk, 0);
-	bpf_sk_release(sk);
-
-	return ret == 0 ? TC_ACT_OK : TC_ACT_SHOT;
-}
-
-char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/get_smp_processor_id.c b/tools/testing/selftests/bpf/progs/get_smp_processor_id.c
new file mode 100644
index 0000000000000..cf4791a5cf076
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/get_smp_processor_id.c
@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include "bpf_misc.h"
+
+__u64 cpu_nr_result;
+
+SEC("raw_tp")
+void call_bpf_get_smp_processor_id(void)
+{
+	register __u64 r0 asm("r0") = -1;
+	asm volatile ("call %[bpf_get_smp_processor_id];"
+		      : "+r"(r0)
+		      : __imm(bpf_get_smp_processor_id)
+		      : "r1", "r2", "r3", "r4", "r5", "memory");
+	cpu_nr_result = r0;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/icmp_send.c b/tools/testing/selftests/bpf/progs/icmp_send.c
new file mode 100644
index 0000000000000..c642ccdf9fd54
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/icmp_send.c
@@ -0,0 +1,123 @@
+// SPDX-License-Identifier: GPL-2.0
+#include "vmlinux.h"
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_endian.h>
+
+/* 127.0.0.1 in host byte order */
+#define SERVER_IP 0x7F000001
+/* ::1 in host byte order (last 32-bit word) */
+#define SERVER_IP6_LO 0x00000001
+
+__u16 server_port = 0;
+int unreach_type = 0;
+int unreach_code = 0;
+int kfunc_ret = -1;
+int target_pid = -1;
+
+unsigned int rec_count = 0;
+int rec_kfunc_rets[] = { -1, -1 };
+
+SEC("cgroup_skb/egress")
+int egress(struct __sk_buff *skb)
+{
+	void *data = (void *)(long)skb->data;
+	void *data_end = (void *)(long)skb->data_end;
+	struct iphdr *iph;
+	struct ipv6hdr *ip6h;
+	struct tcphdr *tcph;
+	__u8 version;
+
+	if (data + 1 > data_end)
+		return SK_PASS;
+
+	version = (*((__u8 *)data)) >> 4;
+
+	if (version == 4) {
+		iph = data;
+		if ((void *)(iph + 1) > data_end ||
+		    iph->protocol != IPPROTO_TCP ||
+		    iph->daddr != bpf_htonl(SERVER_IP))
+			return SK_PASS;
+
+		tcph = (void *)iph + iph->ihl * 4;
+		if ((void *)(tcph + 1) > data_end ||
+		    tcph->dest != bpf_htons(server_port))
+			return SK_PASS;
+
+	} else if (version == 6) {
+		ip6h = data;
+		if ((void *)(ip6h + 1) > data_end ||
+		    ip6h->nexthdr != IPPROTO_TCP)
+			return SK_PASS;
+
+		if (ip6h->daddr.in6_u.u6_addr32[0] != 0 ||
+		    ip6h->daddr.in6_u.u6_addr32[1] != 0 ||
+		    ip6h->daddr.in6_u.u6_addr32[2] != 0 ||
+		    ip6h->daddr.in6_u.u6_addr32[3] != bpf_htonl(SERVER_IP6_LO))
+			return SK_PASS;
+
+		tcph = (void *)(ip6h + 1);
+		if ((void *)(tcph + 1) > data_end ||
+		    tcph->dest != bpf_htons(server_port))
+			return SK_PASS;
+	} else {
+		return SK_PASS;
+	}
+
+	kfunc_ret = bpf_icmp_send(skb, unreach_type, unreach_code);
+
+	return SK_DROP;
+}
+
+SEC("cgroup_skb/egress")
+int recursion(struct __sk_buff *skb)
+{
+	void *data = (void *)(long)skb->data;
+	void *data_end = (void *)(long)skb->data_end;
+	struct icmphdr *icmph;
+	struct tcphdr *tcph;
+	struct iphdr *iph;
+	int ret;
+
+	if ((bpf_get_current_pid_tgid() >> 32) != target_pid)
+		return SK_PASS;
+
+	iph = data;
+	if ((void *)(iph + 1) > data_end || iph->version != 4)
+		return SK_PASS;
+
+	if (iph->daddr != bpf_htonl(SERVER_IP))
+		return SK_PASS;
+
+	if (iph->protocol == IPPROTO_TCP) {
+		tcph = (void *)iph + iph->ihl * 4;
+		if ((void *)(tcph + 1) > data_end ||
+		    tcph->dest != bpf_htons(server_port))
+			return SK_PASS;
+	} else if (iph->protocol == IPPROTO_ICMP) {
+		icmph = (void *)iph + iph->ihl * 4;
+		if ((void *)(icmph + 1) > data_end ||
+		    icmph->type != unreach_type || icmph->code != unreach_code)
+			return SK_PASS;
+	} else {
+		return SK_PASS;
+	}
+
+	/*
+	 * This call will provoke a recursion: the ICMP packet generated by the
+	 * kfunc will re-trigger this program since we are in the root cgroup in
+	 * which the kernel ICMP socket belongs. However when re-entering the
+	 * kfunc, it should return EBUSY.
+	 */
+	ret = bpf_icmp_send(skb, unreach_type, unreach_code);
+	rec_kfunc_rets[rec_count & 1] = ret;
+	__sync_fetch_and_add(&rec_count, 1);
+
+	/* Let the first ICMP error message pass */
+	if (iph->protocol == IPPROTO_ICMP)
+		return SK_PASS;
+
+	return SK_DROP;
+}
+
+char LICENSE[] SEC("license") = "Dual BSD/GPL";
diff --git a/tools/testing/selftests/bpf/progs/kfunc_call_fail.c b/tools/testing/selftests/bpf/progs/kfunc_call_fail.c
index a1963497f0bff..6144ce3ff0b24 100644
--- a/tools/testing/selftests/bpf/progs/kfunc_call_fail.c
+++ b/tools/testing/selftests/bpf/progs/kfunc_call_fail.c
@@ -103,6 +103,39 @@ int kfunc_call_test_get_mem_fail_oob(struct __sk_buff *skb)
 	return ret;
 }
 
+SEC("?tc")
+int kfunc_call_test_get_mem_fail_oversized(struct __sk_buff *skb)
+{
+	struct prog_test_ref_kfunc *pt;
+	unsigned long s = 0;
+	int *p = NULL;
+	int ret = 0;
+
+	pt = bpf_kfunc_call_test_acquire(&s);
+	if (pt) {
+		/*
+		 * rdwr_buf_size is a const int, so a C literal is narrowed to
+		 * 32 bits before the call. Force the full 64-bit value 2^64 - 192
+		 * (0xffffffffffffff40, > U32_MAX) into the argument register with
+		 * a 64-bit immediate load. The verifier records r0_size from the
+		 * full register value and must reject it before that value is
+		 * truncated into R0's u32 mem_size.
+		 */
+		asm volatile (
+			"r1 = %[pt];"
+			"r2 = %[oversized] ll;"
+			"call %[get_rdwr_mem];"
+			"%[p] = r0;"
+			: [p] "=r"(p)
+			: [pt] "r"(pt),
+			  [oversized] "i"(0xffffffffffffff40LL),
+			  [get_rdwr_mem] "i"(bpf_kfunc_call_test_get_rdwr_mem)
+			: "r0", "r1", "r2", "r3", "r4", "r5");
+		bpf_kfunc_call_test_release(pt);
+	}
+	return ret;
+}
+
 int not_const_size = 2 * sizeof(int);
 
 SEC("?tc")
diff --git a/tools/testing/selftests/bpf/progs/kfunc_implicit_args_tracing.c b/tools/testing/selftests/bpf/progs/kfunc_implicit_args_tracing.c
new file mode 100644
index 0000000000000..995f8b8b5b9e8
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/kfunc_implicit_args_tracing.c
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
+
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include <errno.h>
+
+extern int bpf_kfunc_implicit_arg(int a) __weak __ksym;
+
+char _license[] SEC("license") = "GPL";
+
+/* Shared arg checks; reports arg count and aux, returns 1 on success. */
+static __always_inline __u64
+check_implicit_args(void *ctx, __u64 *arg_cnt, __u64 *aux_arg)
+{
+	__u64 a = 0, aux = 0, z = 0;
+	__u64 result;
+	__s64 err;
+
+	*arg_cnt = bpf_get_func_arg_cnt(ctx);
+	result = *arg_cnt == 2;
+
+	err = bpf_get_func_arg(ctx, 0, &a);
+	result &= err == 0 && (int)a == 5;
+
+	err = bpf_get_func_arg(ctx, 1, &aux);
+	*aux_arg = aux;
+	result &= err == 0 && aux != 0;
+
+	err = bpf_get_func_arg(ctx, 2, &z);
+	result &= err == -EINVAL;
+
+	return result;
+}
+
+__u64 fentry_result;
+__u64 fentry_arg_cnt;
+__u64 fentry_aux_arg;
+
+SEC("fentry/bpf_kfunc_implicit_arg")
+int BPF_PROG(trace_implicit_arg_fentry)
+{
+	__u64 ret = 0;
+	__s64 err;
+
+	fentry_result = check_implicit_args(ctx, &fentry_arg_cnt, &fentry_aux_arg);
+
+	err = bpf_get_func_ret(ctx, &ret);
+	fentry_result &= err == -EOPNOTSUPP;
+
+	return 0;
+}
+
+__u64 fexit_result;
+__u64 fexit_arg_cnt;
+__u64 fexit_aux_arg;
+
+SEC("fexit/bpf_kfunc_implicit_arg")
+int BPF_PROG(trace_implicit_arg_fexit)
+{
+	__u64 ret = 0;
+	__s64 err;
+
+	fexit_result = check_implicit_args(ctx, &fexit_arg_cnt, &fexit_aux_arg);
+
+	err = bpf_get_func_ret(ctx, &ret);
+	fexit_result &= err == 0 && ret == 5;
+
+	return 0;
+}
+
+SEC("syscall")
+int trigger_implicit_arg(void *ctx)
+{
+	return bpf_kfunc_implicit_arg(5);
+}
diff --git a/tools/testing/selftests/bpf/progs/linked_list_fail.c b/tools/testing/selftests/bpf/progs/linked_list_fail.c
index ddd26d1a083f7..031e77a288eec 100644
--- a/tools/testing/selftests/bpf/progs/linked_list_fail.c
+++ b/tools/testing/selftests/bpf/progs/linked_list_fail.c
@@ -167,6 +167,16 @@ CHECK_OP(push_back);
 #undef CHECK_OP
 #undef INIT
 
+struct obj_new_flex_elem {
+	int lo;
+	int hi;
+};
+
+struct obj_new_flex {
+	int hdr;
+	struct obj_new_flex_elem cells[];
+};
+
 SEC("?kprobe/xyz")
 int map_compat_kprobe(void *ctx)
 {
@@ -230,6 +240,19 @@ int obj_new_no_struct(void *ctx)
 	return 0;
 }
 
+SEC("?tc")
+int obj_new_flex_array(void *ctx)
+{
+	struct obj_new_flex *p;
+
+	p = bpf_obj_new_impl(bpf_core_type_id_local(struct obj_new_flex), NULL);
+	if (!p)
+		return 0;
+	p->cells[0].hi = 42;
+	bpf_obj_drop_impl(p, NULL);
+	return 0;
+}
+
 SEC("?tc")
 int obj_drop_non_zero_off(void *ctx)
 {
diff --git a/tools/testing/selftests/bpf/progs/map_kptr.c b/tools/testing/selftests/bpf/progs/map_kptr.c
index 3fbefc568e0a6..0d87c97dac991 100644
--- a/tools/testing/selftests/bpf/progs/map_kptr.c
+++ b/tools/testing/selftests/bpf/progs/map_kptr.c
@@ -4,6 +4,18 @@
 #include <bpf/bpf_helpers.h>
 #include "../test_kmods/bpf_testmod_kfunc.h"
 
+struct map_uninit_value {
+	struct prog_test_ref_kfunc __kptr_untrusted *unref_ptr;
+	__u32 data;
+} __attribute__((packed));
+
+struct {
+	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+	__type(key, int);
+	__type(value, struct map_uninit_value);
+	__uint(max_entries, 1);
+} pcpu_array SEC(".maps");
+
 struct map_value {
 	struct prog_test_ref_kfunc __kptr_untrusted *unref_ptr;
 	struct prog_test_ref_kfunc __kptr *ref_ptr;
diff --git a/tools/testing/selftests/bpf/progs/mptcpify.c b/tools/testing/selftests/bpf/progs/mptcpify.c
index cbdc730c3a471..e3f8cb54dbe97 100644
--- a/tools/testing/selftests/bpf/progs/mptcpify.c
+++ b/tools/testing/selftests/bpf/progs/mptcpify.c
@@ -15,7 +15,7 @@ int BPF_PROG(mptcpify, int family, int type, int protocol)
 		return protocol;
 
 	if ((family == AF_INET || family == AF_INET6) &&
-	    type == SOCK_STREAM &&
+	    (type & SOCK_TYPE_MASK) == SOCK_STREAM &&
 	    (!protocol || protocol == IPPROTO_TCP)) {
 		return IPPROTO_MPTCP;
 	}
diff --git a/tools/testing/selftests/bpf/progs/sock_read_xattr.c b/tools/testing/selftests/bpf/progs/sock_read_xattr.c
new file mode 100644
index 0000000000000..c4a8eae8cc3c9
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/sock_read_xattr.c
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2026 Christian Brauner */
+
+#include <vmlinux.h>
+#include <bpf/bpf_tracing.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_core_read.h>
+#include "bpf_experimental.h"
+#include "bpf_misc.h"
+
+char _license[] SEC("license") = "GPL";
+
+char value[16];
+int read_ret = -1;
+__u32 monitored_pid = 0;
+
+static __always_inline void read_xattr(struct socket *sock)
+{
+	struct bpf_dynptr value_ptr;
+
+	bpf_dynptr_from_mem(value, sizeof(value), 0, &value_ptr);
+	bpf_sock_read_xattr(sock, "user.bpf_test", &value_ptr);
+}
+
+SEC("lsm.s/socket_connect")
+__success
+int BPF_PROG(trusted_sock_ptr_sleepable, struct socket *sock)
+{
+	read_xattr(sock);
+	return 0;
+}
+
+SEC("lsm/socket_connect")
+__success
+int BPF_PROG(trusted_sock_ptr_non_sleepable, struct socket *sock)
+{
+	read_xattr(sock);
+	return 0;
+}
+
+SEC("lsm.s/socket_connect")
+__success
+int BPF_PROG(read_sock_xattr, struct socket *sock)
+{
+	struct bpf_dynptr value_ptr;
+	__u32 pid = bpf_get_current_pid_tgid() >> 32;
+
+	if (pid != monitored_pid)
+		return 0;
+
+	bpf_dynptr_from_mem(value, sizeof(value), 0, &value_ptr);
+	read_ret = bpf_sock_read_xattr(sock, "user.bpf_test", &value_ptr);
+	return 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c b/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c
index 0660f29dca955..3177bc5b733ac 100644
--- a/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c
+++ b/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c
@@ -44,8 +44,18 @@ int bpf_prog2(struct __sk_buff *skb)
 	__sink(lport);
 	__sink(rport);
 
-	if (data + 8 > data_end)
-		return SK_DROP;
+	if (data + 8 > data_end) {
+		if (bpf_skb_pull_data(skb, 8))
+			return SK_DROP;
+
+		data = (void *)(long)skb->data;
+		data_end = (void *)(long)skb->data_end;
+
+		if (data + 8 > data_end)
+			return SK_DROP;
+
+		d = data;
+	}
 
 	map = d[0];
 	sk = d[1];
diff --git a/tools/testing/selftests/bpf/progs/tcp_ca_untrusted_btf_write.c b/tools/testing/selftests/bpf/progs/tcp_ca_untrusted_btf_write.c
new file mode 100644
index 0000000000000..eda4697aac80b
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/tcp_ca_untrusted_btf_write.c
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include "bpf_tracing_net.h"
+#include <bpf/bpf_core_read.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+char _license[] SEC("license") = "GPL";
+
+SEC("struct_ops")
+void BPF_PROG(untrusted_btf_write_init, struct sock *sk)
+{
+	struct tcp_sock *tp;
+	int v = 1;
+	void *p;
+
+	p = bpf_rdonly_cast(&v, 0);
+	tp = bpf_rdonly_cast(p, bpf_core_type_id_kernel(struct tcp_sock));
+	tp->snd_cwnd = 1;
+}
+
+SEC(".struct_ops")
+struct tcp_congestion_ops untrusted_btf_write = {
+	.init = (void *)untrusted_btf_write_init,
+	.name = "bpf_ro_btf",
+};
diff --git a/tools/testing/selftests/bpf/progs/test_fill_link_info.c b/tools/testing/selftests/bpf/progs/test_fill_link_info.c
index 137bd6292163a..c85081538e93b 100644
--- a/tools/testing/selftests/bpf/progs/test_fill_link_info.c
+++ b/tools/testing/selftests/bpf/progs/test_fill_link_info.c
@@ -58,4 +58,10 @@ int BPF_PROG(umulti_run)
 	return 0;
 }
 
+SEC("fentry.multi")
+int BPF_PROG(tmulti_run)
+{
+	return 0;
+}
+
 char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/test_signed_loader.c b/tools/testing/selftests/bpf/progs/test_signed_loader.c
index d9a4b85f9391f..50451a69b99a2 100644
--- a/tools/testing/selftests/bpf/progs/test_signed_loader.c
+++ b/tools/testing/selftests/bpf/progs/test_signed_loader.c
@@ -4,10 +4,11 @@
 
 /*
  * Minimal, map-less program. Driven through libbpf's gen_loader (gen_hash)
- * by prog_tests/signed_loader.c so the generated light-skeleton loader (with
- * the emit_signature_match metadata check) can be exercised against good
- * and tampered metadata. A socket filter needs no load-time attach resolution,
- * and having no maps keeps the generated loader's ctx trivial (0 maps, 1 prog).
+ * by prog_tests/signed_loader.c so the generated light-skeleton loader can be
+ * exercised against good and tampered metadata, which the kernel now verifies
+ * at load time via the insns||metadata signature. A socket filter needs no
+ * load-time attach resolution, and having no maps keeps the generated loader's
+ * ctx trivial (0 maps, 1 prog).
  */
 SEC("socket")
 int probe(void *ctx)
diff --git a/tools/testing/selftests/bpf/progs/test_sockmap_update.c b/tools/testing/selftests/bpf/progs/test_sockmap_update.c
deleted file mode 100644
index 6d64ea536e3d8..0000000000000
--- a/tools/testing/selftests/bpf/progs/test_sockmap_update.c
+++ /dev/null
@@ -1,48 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-// Copyright (c) 2020 Cloudflare
-#include "vmlinux.h"
-#include <bpf/bpf_helpers.h>
-
-struct {
-	__uint(type, BPF_MAP_TYPE_SOCKMAP);
-	__uint(max_entries, 1);
-	__type(key, __u32);
-	__type(value, __u64);
-} src SEC(".maps");
-
-struct {
-	__uint(type, BPF_MAP_TYPE_SOCKMAP);
-	__uint(max_entries, 1);
-	__type(key, __u32);
-	__type(value, __u64);
-} dst_sock_map SEC(".maps");
-
-struct {
-	__uint(type, BPF_MAP_TYPE_SOCKHASH);
-	__uint(max_entries, 1);
-	__type(key, __u32);
-	__type(value, __u64);
-} dst_sock_hash SEC(".maps");
-
-SEC("tc")
-int copy_sock_map(void *ctx)
-{
-	struct bpf_sock *sk;
-	bool failed = false;
-	__u32 key = 0;
-
-	sk = bpf_map_lookup_elem(&src, &key);
-	if (!sk)
-		return SK_DROP;
-
-	if (bpf_map_update_elem(&dst_sock_map, &key, sk, 0))
-		failed = true;
-
-	if (bpf_map_update_elem(&dst_sock_hash, &key, sk, 0))
-		failed = true;
-
-	bpf_sk_release(sk);
-	return failed ? SK_DROP : SK_PASS;
-}
-
-char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/test_tc_peer.c b/tools/testing/selftests/bpf/progs/test_tc_peer.c
index 365eacb5dc341..cfb9ef7f467c5 100644
--- a/tools/testing/selftests/bpf/progs/test_tc_peer.c
+++ b/tools/testing/selftests/bpf/progs/test_tc_peer.c
@@ -34,6 +34,28 @@ int tc_src(struct __sk_buff *skb)
 	return bpf_redirect_peer(IFINDEX_DST, 0);
 }
 
+SEC("tc")
+int tc_dst_ing(struct __sk_buff *skb)
+{
+	if (!skb->mark) {
+		skb->mark = 0x1;
+		return bpf_redirect_peer(IFINDEX_SRC, BPF_F_EGRESS);
+	}
+
+	return bpf_redirect(IFINDEX_DST, 0);
+}
+
+SEC("tc")
+int tc_src_ing(struct __sk_buff *skb)
+{
+	if (!skb->mark) {
+		skb->mark = 0x1;
+		return bpf_redirect_peer(IFINDEX_DST, BPF_F_EGRESS);
+	}
+
+	return bpf_redirect(IFINDEX_SRC, 0);
+}
+
 SEC("tc")
 int tc_dst_l3(struct __sk_buff *skb)
 {
diff --git a/tools/testing/selftests/bpf/progs/test_usdt.c b/tools/testing/selftests/bpf/progs/test_usdt.c
index f00cb52874e0b..0ee78fb050a10 100644
--- a/tools/testing/selftests/bpf/progs/test_usdt.c
+++ b/tools/testing/selftests/bpf/progs/test_usdt.c
@@ -149,5 +149,30 @@ int usdt_executed(struct pt_regs *ctx)
 		executed++;
 	return 0;
 }
+
+int arg_total;
+int arg_bad;
+long arg_last[3];
+long expected_arg[3];
+int expected_pid;
+
+SEC("usdt")
+int BPF_USDT(usdt_check_arg, long arg1, long arg2, long arg3)
+{
+	if (expected_pid != (bpf_get_current_pid_tgid() >> 32))
+		return 0;
+
+	__sync_fetch_and_add(&arg_total, 1);
+	arg_last[0] = arg1;
+	arg_last[1] = arg2;
+	arg_last[2] = arg3;
+
+	if (arg1 != expected_arg[0] ||
+	    arg2 != expected_arg[1] ||
+	    arg3 != expected_arg[2])
+		__sync_fetch_and_add(&arg_bad, 1);
+
+	return 0;
+}
 #endif
 char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/tracing_multi_intersect_attach.c b/tools/testing/selftests/bpf/progs/tracing_multi_intersect_attach.c
index cd5be0bb6ffdd..5b0af8f4c62ff 100644
--- a/tools/testing/selftests/bpf/progs/tracing_multi_intersect_attach.c
+++ b/tools/testing/selftests/bpf/progs/tracing_multi_intersect_attach.c
@@ -11,6 +11,14 @@ __u64 test_result_fentry_1 = 0;
 __u64 test_result_fentry_2 = 0;
 __u64 test_result_fexit_1 = 0;
 __u64 test_result_fexit_2 = 0;
+__u64 test_result_fentry = 0;
+
+SEC("fentry/bpf_fentry_test1")
+int BPF_PROG(fentry)
+{
+	tracing_multi_arg_check(ctx, &test_result_fentry, false);
+	return 0;
+}
 
 SEC("fentry.multi")
 int BPF_PROG(fentry_1)
diff --git a/tools/testing/selftests/bpf/progs/verifier_gotox.c b/tools/testing/selftests/bpf/progs/verifier_gotox.c
index f88aa4cdb2792..5b18c9a27717b 100644
--- a/tools/testing/selftests/bpf/progs/verifier_gotox.c
+++ b/tools/testing/selftests/bpf/progs/verifier_gotox.c
@@ -384,6 +384,31 @@ jt0_%=:								\
 	: __clobber_all);
 }
 
+/* check valid spill/fill, ptr to insn */
+SEC("socket")
+__success
+__naked void spill_fill_ptr_to_insn(void)
+{
+	asm volatile (
+	".pushsection .jumptables,\"\",@progbits;"
+	"jt0_%=:"
+	".quad ret0_%= - socket;"
+	".size jt0_%=, 8;"
+	".global jt0_%=;"
+	".popsection;"
+	"r0 = jt0_%= ll;"
+	"r0 = *(u64 *)(r0 + 0);"
+	"*(u64 *)(r10 - 8) = r0;"
+	"r0 = *(u64 *)(r10 - 8);"
+	".8byte %[gotox_r0];"
+	"ret0_%=:"
+	"r0 = 0;"
+	"exit;"
+	:
+	: __imm_insn(gotox_r0, BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_0, 0, 0, 0))
+	: __clobber_all);
+}
+
 #endif /* __TARGET_ARCH_x86 || __TARGET_ARCH_arm64 || __TARGET_ARCH_powerpc*/
 
 char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_map_ptr.c b/tools/testing/selftests/bpf/progs/verifier_map_ptr.c
index 1661936598703..e0a65835c861b 100644
--- a/tools/testing/selftests/bpf/progs/verifier_map_ptr.c
+++ b/tools/testing/selftests/bpf/progs/verifier_map_ptr.c
@@ -72,14 +72,15 @@ __naked void bpf_map_ptr_write_rejected(void)
 
 /*
  * struct bpf_map starts with the SHA256 hash sha[32] at offset 0 (a readable
- * byte array), the u32 excl field at offset 32, and the ops pointer at offset
- * 40. Reading a u32 at offset 41 reaches into the middle of the ops pointer,
- * i.e. a partial pointer access, which is rejected.
+ * byte array), followed by the ops pointer at offset 32 and the inner_map_meta
+ * pointer at offset 40. Reading a u32 at offset 41 reaches into the middle of
+ * the inner_map_meta pointer, i.e. a partial pointer access, which is
+ * rejected.
  */
 SEC("socket")
 __description("bpf_map_ptr: read non-existent field rejected")
 __failure
-__msg("cannot access ptr member ops with moff 40 in struct bpf_map with off 41 size 4")
+__msg("cannot access ptr member inner_map_meta with moff 40 in struct bpf_map with off 41 size 4")
 __failure_unpriv
 __msg_unpriv("access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN")
 __flag(BPF_F_ANY_ALIGNMENT)
@@ -97,23 +98,23 @@ __naked void read_non_existent_field_rejected(void)
 }
 
 /*
- * The u32 excl field spans offsets 32..35 (mend 36). Reading a u32 at offset
- * 33 starts inside excl but extends past its end, which the verifier rejects
+ * The sha byte array spans offsets 0..31 (mend 32). Reading a u32 at offset
+ * 30 starts inside sha but extends past its end, which the verifier rejects
  * as an out-of-bounds scalar access.
  */
 SEC("socket")
-__description("bpf_map_ptr: read beyond excl field rejected")
+__description("bpf_map_ptr: read beyond sha field rejected")
 __failure
-__msg("access beyond the end of member excl (mend:36) in struct bpf_map with off 33 size 4")
+__msg("access beyond the end of member sha (mend:32) in struct bpf_map with off 30 size 4")
 __failure_unpriv
 __msg_unpriv("access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN")
 __flag(BPF_F_ANY_ALIGNMENT)
-__naked void read_beyond_excl_field_rejected(void)
+__naked void read_beyond_sha_field_rejected(void)
 {
 	asm volatile ("					\
 	r6 = 0;						\
 	r1 = %[map_array_48b] ll;			\
-	r6 = *(u32*)(r1 + 33);				\
+	r6 = *(u32*)(r1 + 30);				\
 	r0 = 1;						\
 	exit;						\
 "	:
@@ -131,7 +132,7 @@ __naked void ptr_read_ops_field_accepted(void)
 	asm volatile ("					\
 	r6 = 0;						\
 	r1 = %[map_array_48b] ll;			\
-	r6 = *(u64*)(r1 + 40);				\
+	r6 = *(u64*)(r1 + 32);				\
 	r0 = 1;						\
 	exit;						\
 "	:
diff --git a/tools/testing/selftests/bpf/progs/verifier_ptr_to_buf.c b/tools/testing/selftests/bpf/progs/verifier_ptr_to_buf.c
new file mode 100644
index 0000000000000..12cf24db46a0a
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/verifier_ptr_to_buf.c
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include "bpf_misc.h"
+
+SEC("iter/bpf_map_elem")
+__description("PTR_TO_BUF: reject negative const offset")
+__failure
+__msg("invalid negative rdwr buffer offset")
+__naked void ptr_to_buf_reject_negative_const_offset(void)
+{
+	asm volatile ("r0 = 0;					\
+	 r2 = *(u64 *)(r1 + %[value_off]);			\
+	 if r2 == 0 goto l0_%=;					\
+	 r2 += -8;						\
+	 r0 = *(u64 *)(r2 + 0);					\
+l0_%=:								\
+	 exit;							\
+	"
+	:
+	: __imm_const(value_off,
+		      offsetof(struct bpf_iter__bpf_map_elem, value))
+	: __clobber_all);
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_raw_tp_writable.c b/tools/testing/selftests/bpf/progs/verifier_raw_tp_writable.c
index 14a0172e2141e..4055a6443bc20 100644
--- a/tools/testing/selftests/bpf/progs/verifier_raw_tp_writable.c
+++ b/tools/testing/selftests/bpf/progs/verifier_raw_tp_writable.c
@@ -47,4 +47,20 @@ l0_%=:	/* shift the buffer pointer to a variable location */\
 	: __clobber_all);
 }
 
+SEC("raw_tracepoint.w")
+__description("raw_tracepoint_writable: reject negative const offset")
+__failure
+__msg("invalid negative tracepoint buffer offset")
+__naked void tracepoint_writable_reject_negative_const_offset(void)
+{
+	asm volatile ("					\
+	r6 = *(u64 *)(r1 + 0);				\
+	r6 += -8;					\
+	r0 = *(u64 *)(r6 + 0);				\
+	exit;						\
+"	:
+	:
+	: __clobber_all);
+}
+
 char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c
index e38f102da45f2..663d15fc5fd26 100644
--- a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c
+++ b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c
@@ -4,6 +4,13 @@
 #include <bpf/bpf_helpers.h>
 #include "bpf_misc.h"
 
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(max_entries, 1);
+	__type(key, long long);
+	__type(value, long long);
+} map_hash_8b SEC(".maps");
+
 /* Check that precision marks propagate through scalar IDs.
  * Registers r{0,1,2} have the same scalar ID.
  * Range information is propagated for scalars sharing same ID.
@@ -915,4 +922,53 @@ __naked void linked_regs_and_subreg_def(void)
 	: __clobber_all);
 }
 
+/*
+ * A scalar is spilled to the stack and then filled twice: once via a
+ * sign-extending load (BPF_MEMSX) into r4 and once via a zero-extending
+ * load (BPF_MEM) into r5. coerce_reg_to_size_sx() gives r4 a different
+ * value than the spilled/zero-extended siblings, so r4 must not keep the
+ * shared scalar id. Otherwise the later 'if r5 == 0x80000000' refines r4
+ * through sync_linked_regs() to a known 0x80000000, while at runtime r4
+ * is the sign-extended 0xffffffff80000000. The test turns that discrepancy
+ * into an out-of-bounds map value access (r4 >> 63 is believed 0 but is 1
+ * at runtime), which must be rejected.
+ */
+SEC("socket")
+__failure __msg("R0 max value is outside of the allowed memory range")
+__naked void ldsx_fill_scalar_id_not_shared(void)
+{
+	asm volatile ("					\
+	r1 = 0;						\
+	*(u64*)(r10 - 8) = r1;				\
+	r2 = r10;					\
+	r2 += -8;					\
+	r1 = %[map_hash_8b] ll;				\
+	call %[bpf_map_lookup_elem];			\
+	if r0 == 0 goto l0_%=;				\
+	/* r7 = unknown u32, keep only bit 31 */	\
+	r7 = *(u32*)(r0 + 0);				\
+	r2 = 0x80000000 ll;				\
+	r7 &= r2;					\
+	/* link r6 and r7 via a fresh scalar id */	\
+	r6 = r7;					\
+	/* spill r7 (u32) to the stack */		\
+	*(u32*)(r10 - 8) = r7;				\
+	/* sign-extending fill: must drop the id */	\
+	r4 = *(s32*)(r10 - 8);				\
+	/* zero-extending fill: keeps the id */		\
+	r5 = *(u32*)(r10 - 8);				\
+	/* r5 becomes known 0x80000000 on fall-through */\
+	if r5 != r2 goto l0_%=;				\
+	/* verifier believes r4 == 0 here, runtime is 1 */\
+	r4 >>= 63;					\
+	r0 += r4;					\
+	r0 = *(u8*)(r0 + 7);				\
+l0_%=:	r0 = 0;						\
+	exit;						\
+"	:
+	: __imm(bpf_map_lookup_elem),
+	  __imm_addr(map_hash_8b)
+	: __clobber_all);
+}
+
 char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_sockmap_mutate.c b/tools/testing/selftests/bpf/progs/verifier_sockmap_mutate.c
index fe4b123187b86..20332a731d4e4 100644
--- a/tools/testing/selftests/bpf/progs/verifier_sockmap_mutate.c
+++ b/tools/testing/selftests/bpf/progs/verifier_sockmap_mutate.c
@@ -74,7 +74,7 @@ static __always_inline void test_sockmap_lookup_and_mutate(void)
 }
 
 SEC("action")
-__success
+__failure __msg("cannot update sockmap in this context")
 int test_sched_act(struct __sk_buff *skb)
 {
 	test_sockmap_mutate(skb->sk);
@@ -82,7 +82,7 @@ int test_sched_act(struct __sk_buff *skb)
 }
 
 SEC("classifier")
-__success
+__failure __msg("cannot update sockmap in this context")
 int test_sched_cls(struct __sk_buff *skb)
 {
 	test_sockmap_mutate(skb->sk);
@@ -90,7 +90,7 @@ int test_sched_cls(struct __sk_buff *skb)
 }
 
 SEC("flow_dissector")
-__success
+__failure __msg("cannot update sockmap in this context")
 int test_flow_dissector_delete(struct __sk_buff *skb __always_unused)
 {
 	test_sockmap_delete();
@@ -98,7 +98,7 @@ int test_flow_dissector_delete(struct __sk_buff *skb __always_unused)
 }
 
 SEC("flow_dissector")
-__failure __msg("program of this type cannot use helper bpf_sk_release")
+__failure __msg("cannot update sockmap in this context")
 int test_flow_dissector_update(struct __sk_buff *skb __always_unused)
 {
 	test_sockmap_lookup_and_update(); /* no access to skb->sk */
@@ -146,7 +146,7 @@ int test_sk_reuseport(struct sk_reuseport_md *ctx)
 }
 
 SEC("socket")
-__success
+__failure __msg("cannot update sockmap in this context")
 int test_socket_filter(struct __sk_buff *skb)
 {
 	test_sockmap_mutate(skb->sk);
@@ -179,7 +179,7 @@ int test_sockops_update_dedicated(struct bpf_sock_ops *ctx)
 }
 
 SEC("xdp")
-__success
+__failure __msg("cannot update sockmap in this context")
 int test_xdp(struct xdp_md *ctx __always_unused)
 {
 	test_sockmap_lookup_and_mutate();
diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c
index 0174887e28f5b..8b166c42c4e0e 100644
--- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c
+++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c
@@ -634,6 +634,32 @@ __naked void partial_stack_load_preserves_partial_zeros(void)
 	: __clobber_common);
 }
 
+SEC("raw_tp")
+__log_level(2)
+__success
+__msg("mark_precise: frame0: regs= stack=-8")
+__msg("R2=0")
+__naked void stack_load_preserves_mixed_zero_and_zero_spill(void)
+{
+	asm volatile (
+		/* fp-8 has scalar const-zero spill bytes and STACK_ZERO bytes. */
+		".8byte %[fp4_st_zero];" /* LLVM-18+: *(u32 *)(r10 -4) = 0; */
+		"r0 = 0;"
+		"*(u32 *)(r10 -8) = r0;"
+
+		"r1 = %[single_byte_buf];"
+		"r2 = *(u64 *)(r10 -8);"
+		"r1 += r2;"
+		"*(u8 *)(r1 + 0) = r2;" /* this should be fine */
+
+		"r0 = 0;"
+		"exit;"
+	:
+	: __imm_ptr(single_byte_buf),
+	  __imm_insn(fp4_st_zero, BPF_ST_MEM(BPF_W, BPF_REG_FP, -4, 0))
+	: __clobber_common);
+}
+
 char two_byte_buf[2] SEC(".data.two_byte_buf");
 
 SEC("raw_tp")
@@ -1377,4 +1403,46 @@ __naked void partial_fill_from_cleaned_pointer_spill(void)
 		      ::: __clobber_all);
 }
 
+/* check valid spill/fill, ptr to tp buffer */
+SEC("raw_tracepoint.w")
+__success
+__naked void spill_fill_ptr_to_tp_buffer(void)
+{
+	asm volatile (
+	"r6 = *(u64*)(r1 + 0);"	/* r6 is the writable tracepoint buffer */
+	"*(u64*)(r10 - 8) = r6;"
+	"r7 = *(u64*)(r10 - 8);"
+	"r0 = 0;"
+	"*(u64*)(r7 + 0) = r0;"	/* should be able to write through the buffer */
+	"r0 = 0;"
+	"exit;"
+	::: __clobber_all);
+}
+
+__noinline int spill_fill_dynptr_subprog(struct bpf_dynptr *dptr)
+{
+	long *p;
+
+	asm volatile ("*(u64 *)(r10 - 8) = %[dptr];" /* spill the CONST_PTR_TO_DYNPTR argument */
+		      "%[dptr] = *(u64 *)(r10 - 8);"
+		      : [dptr] "+r"(dptr) :: "memory");
+	p = bpf_dynptr_data(dptr, 0, sizeof(*p));
+	if (!p)
+		return 0;
+	return 0;
+}
+
+static char dptr_mem_buf[16];
+
+/* check valid spill/fill, const ptr to dynptr */
+SEC("socket")
+__success
+int spill_fill_const_ptr_to_dynptr(void)
+{
+	struct bpf_dynptr ptr;
+
+	bpf_dynptr_from_mem(dptr_mem_buf, sizeof(dptr_mem_buf), 0, &ptr);
+	return spill_fill_dynptr_subprog(&ptr);
+}
+
 char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_var_off.c b/tools/testing/selftests/bpf/progs/verifier_var_off.c
index f345466bca686..24cd0a7636730 100644
--- a/tools/testing/selftests/bpf/progs/verifier_var_off.c
+++ b/tools/testing/selftests/bpf/progs/verifier_var_off.c
@@ -59,6 +59,116 @@ __naked void stack_read_priv_vs_unpriv(void)
 "	::: __clobber_all);
 }
 
+SEC("cgroup/skb")
+__description("variable-offset stack read preserves spilled zero")
+__success
+__log_level(2)
+__msg("mark_precise: frame0: regs= stack=-8")
+__msg("R3=0")
+__retval(0)
+__naked void stack_read_var_off_preserves_spilled_zero(void)
+{
+	asm volatile ("					\
+	r0 = 0;						\
+	*(u64*)(r10 - 8) = r0;			\
+	r2 = *(u32*)(r1 + 0);				\
+	r2 &= 7;					\
+	r2 -= 8;					\
+	r2 += r10;					\
+	r3 = *(u8*)(r2 + 0);				\
+	r1 = r10;					\
+	r1 += -1;					\
+	r1 += r3;					\
+	*(u8*)(r1 + 0) = r3;			\
+	r0 = 0;						\
+	exit;						\
+"	::: __clobber_all);
+}
+
+SEC("cgroup/skb")
+__description("variable-offset stack read preserves spilled zero across slots")
+__success
+__log_level(2)
+__msg("mark_precise: frame0: regs= stack=-8,-16")
+__msg("R3=0")
+__retval(0)
+__naked void stack_read_var_off_preserves_spilled_zero_across_slots(void)
+{
+	asm volatile ("					\
+	r0 = 0;						\
+	*(u64*)(r10 - 8) = r0;			\
+	*(u64*)(r10 - 16) = r0;			\
+	r2 = *(u32*)(r1 + 0);				\
+	r2 &= 15;					\
+	r2 -= 16;					\
+	r2 += r10;					\
+	r3 = *(u8*)(r2 + 0);				\
+	r1 = r10;					\
+	r1 += -1;					\
+	r1 += r3;					\
+	*(u8*)(r1 + 0) = r3;			\
+	r0 = 0;						\
+	exit;						\
+"	::: __clobber_all);
+}
+
+SEC("cgroup/skb")
+__description("variable-offset stack read preserves partial spilled zero")
+__success
+__log_level(2)
+__msg("mark_precise: frame0: regs= stack=-8")
+__msg("R3=0")
+__retval(0)
+__naked void stack_read_var_off_preserves_partial_spilled_zero(void)
+{
+	asm volatile ("					\
+	r0 = 0;						\
+	*(u8*)(r10 - 9) = r0;			\
+	*(u8*)(r10 - 10) = r0;			\
+	*(u8*)(r10 - 11) = r0;			\
+	*(u8*)(r10 - 12) = r0;			\
+	*(u8*)(r10 - 13) = r0;			\
+	*(u8*)(r10 - 14) = r0;			\
+	*(u8*)(r10 - 15) = r0;			\
+	*(u32*)(r10 - 8) = r0;			\
+	r2 = *(u32*)(r1 + 0);				\
+	r2 &= 15;					\
+	if r2 > 10 goto l0_%=;			\
+	r2 -= 15;					\
+	r2 += r10;					\
+	r3 = *(u8*)(r2 + 0);				\
+	r1 = r10;					\
+	r1 += -1;					\
+	r1 += r3;					\
+	*(u8*)(r1 + 0) = r3;			\
+l0_%=: r0 = 0;					\
+	exit;						\
+"	::: __clobber_all);
+}
+
+SEC("cgroup/skb")
+__description("variable-offset stack read partial spill with misc data")
+__failure
+__msg("invalid variable-offset write to stack R1")
+__naked void stack_read_var_off_partial_spill_with_misc_data(void)
+{
+	asm volatile ("					\
+	r0 = 0;						\
+	*(u32*)(r10 - 8) = r0;			\
+	r2 = *(u32*)(r1 + 0);				\
+	r2 &= 7;					\
+	r2 -= 8;					\
+	r2 += r10;					\
+	r3 = *(u8*)(r2 + 0);				\
+	r1 = r10;					\
+	r1 += -1;					\
+	r1 += r3;					\
+	*(u8*)(r1 + 0) = 0;			\
+	r0 = 0;						\
+	exit;						\
+"	::: __clobber_all);
+}
+
 SEC("cgroup/skb")
 __description("variable-offset stack read, uninitialized")
 __success
diff --git a/tools/testing/selftests/bpf/test_maps.c b/tools/testing/selftests/bpf/test_maps.c
index c32da7bd8be27..6a2641ee78975 100644
--- a/tools/testing/selftests/bpf/test_maps.c
+++ b/tools/testing/selftests/bpf/test_maps.c
@@ -759,16 +759,15 @@ static void test_sockmap(unsigned int tasks, void *data)
 		goto out_sockmap;
 	}
 
-	/* Test update with unsupported UDP socket */
+	/* Test update with unsupported unbound UDP socket */
 	udp = socket(AF_INET, SOCK_DGRAM, 0);
-	i = 0;
-	err = bpf_map_update_elem(fd, &i, &udp, BPF_ANY);
-	if (err) {
-		printf("Failed socket update SOCK_DGRAM '%i:%i'\n",
-		       i, udp);
+	CHECK(udp < 0, "socket(AF_INET, SOCK_DGRAM)", "errno:%d\n", errno);
+	err = bpf_map_update_elem(fd, &(int){0}, &udp, BPF_ANY);
+	close(udp);
+	if (!err) {
+		printf("Unexpectedly succeeded unbound UDP update '0:%i'\n", udp);
 		goto out_sockmap;
 	}
-	close(udp);
 
 	/* Test update without programs */
 	for (i = 0; i < 6; i++) {
diff --git a/tools/testing/selftests/bpf/test_sockmap.c b/tools/testing/selftests/bpf/test_sockmap.c
index ac814eb63edb6..aaf2050e88450 100644
--- a/tools/testing/selftests/bpf/test_sockmap.c
+++ b/tools/testing/selftests/bpf/test_sockmap.c
@@ -435,7 +435,8 @@ static int msg_alloc_iov(struct msghdr *msg,
 	return 0;
 unwind_iov:
 	for (i--; i >= 0 ; i--)
-		free(msg->msg_iov[i].iov_base);
+		free(iov[i].iov_base);
+	free(iov);
 	return -ENOMEM;
 }
 
diff --git a/tools/testing/selftests/bpf/usdt.h b/tools/testing/selftests/bpf/usdt.h
index c71e21df38b32..75687f50f4e21 100644
--- a/tools/testing/selftests/bpf/usdt.h
+++ b/tools/testing/selftests/bpf/usdt.h
@@ -313,7 +313,7 @@ struct usdt_sema { volatile unsigned short active; };
 #if defined(__ia64__) || defined(__s390__) || defined(__s390x__)
 #define USDT_NOP			nop 0
 #elif defined(__x86_64__)
-#define USDT_NOP                       .byte 0x90, 0x0f, 0x1f, 0x44, 0x00, 0x0 /* nop, nop5 */
+#define USDT_NOP                       .byte 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 /* nop, nop10 */
 #else
 #define USDT_NOP			nop
 #endif
diff --git a/tools/testing/selftests/bpf/usdt_2.c b/tools/testing/selftests/bpf/usdt_2.c
index 789883aaca4c3..5e38f8605b02b 100644
--- a/tools/testing/selftests/bpf/usdt_2.c
+++ b/tools/testing/selftests/bpf/usdt_2.c
@@ -3,7 +3,7 @@
 #if defined(__x86_64__)
 
 /*
- * Include usdt.h with default nop,nop5 instructions combo.
+ * Include usdt.h with default nop,nop10 instructions combo.
  */
 #include "usdt.h"
 
@@ -13,4 +13,17 @@ void usdt_2(void)
 	USDT(optimized_attach, usdt_2);
 }
 
+static volatile unsigned long usdt_red_zone_arg1 = 0xDEADBEEF;
+static volatile unsigned long usdt_red_zone_arg2 = 0xCAFEBABE;
+static volatile unsigned long usdt_red_zone_arg3 = 0xFEEDFACE;
+
+void __attribute__((noinline)) usdt_red_zone_trigger(void)
+{
+	unsigned long a1 = usdt_red_zone_arg1;
+	unsigned long a2 = usdt_red_zone_arg2;
+	unsigned long a3 = usdt_red_zone_arg3;
+
+	USDT(optimized_attach, usdt_red_zone, a1, a2, a3);
+}
+
 #endif
diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c
index a7db6f04f7e14..c9c257784ee3d 100644
--- a/tools/testing/selftests/bpf/veristat.c
+++ b/tools/testing/selftests/bpf/veristat.c
@@ -1248,6 +1248,29 @@ static void fixup_obj_maps(struct bpf_object *obj)
 
 		/* fix up map size, if necessary */
 		switch (bpf_map__type(map)) {
+		/*
+		 * if the verifier doesn't use max_entries
+		 * then set to 1 to avoid -ENOMEM
+		 */
+		case BPF_MAP_TYPE_HASH:
+		case BPF_MAP_TYPE_PERCPU_HASH:
+		case BPF_MAP_TYPE_LRU_HASH:
+		case BPF_MAP_TYPE_LRU_PERCPU_HASH:
+		case BPF_MAP_TYPE_SOCKHASH:
+		case BPF_MAP_TYPE_DEVMAP_HASH:
+		case BPF_MAP_TYPE_QUEUE:
+		case BPF_MAP_TYPE_STACK:
+		case BPF_MAP_TYPE_BLOOM_FILTER:
+		case BPF_MAP_TYPE_STACK_TRACE:
+			bpf_map__set_max_entries(map, 1);
+			break;
+
+		/* ringbufs must be page-aligned */
+		case BPF_MAP_TYPE_RINGBUF:
+		case BPF_MAP_TYPE_USER_RINGBUF:
+			bpf_map__set_max_entries(map, sysconf(_SC_PAGESIZE));
+			break;
+
 		case BPF_MAP_TYPE_SK_STORAGE:
 		case BPF_MAP_TYPE_TASK_STORAGE:
 		case BPF_MAP_TYPE_INODE_STORAGE:
diff --git a/tools/testing/selftests/cgroup/lib/include/cgroup_util.h b/tools/testing/selftests/cgroup/lib/include/cgroup_util.h
index febc1723d0903..8ebb2b4d4ec07 100644
--- a/tools/testing/selftests/cgroup/lib/include/cgroup_util.h
+++ b/tools/testing/selftests/cgroup/lib/include/cgroup_util.h
@@ -8,6 +8,7 @@
 
 #define MB(x) (x << 20)
 
+#define NSEC_PER_USEC	1000L
 #define USEC_PER_SEC	1000000L
 #define NSEC_PER_SEC	1000000000L
 
diff --git a/tools/testing/selftests/cgroup/test_cpu.c b/tools/testing/selftests/cgroup/test_cpu.c
index 7a40d76b95487..a5eccfcabef5d 100644
--- a/tools/testing/selftests/cgroup/test_cpu.c
+++ b/tools/testing/selftests/cgroup/test_cpu.c
@@ -639,6 +639,31 @@ test_cpucg_nested_weight_underprovisioned(const char *root)
 	return run_cpucg_nested_weight_test(root, false);
 }
 
+/*
+ * Best effort attempt to get the kernel's HZ value from the config.
+ * Return the HZ value if found otherwise return 1000 (the default) to
+ * indicate failure.
+ */
+static long
+get_config_hz(void)
+{
+	long hz = 1000;
+	FILE *f;
+	char cmd[256] = "zcat /proc/config.gz 2>/dev/null | grep '^CONFIG_HZ='";
+
+	f = popen(cmd, "r");
+
+	if (!f)
+		return hz;
+
+	if (fscanf(f, "CONFIG_HZ=%ld", &hz) == EOF)
+		goto out;
+
+out:
+	pclose(f);
+	return hz;
+}
+
 /*
  * This test creates a cgroup with some maximum value within a period, and
  * verifies that a process in the cgroup is not overscheduled.
@@ -646,15 +671,18 @@ test_cpucg_nested_weight_underprovisioned(const char *root)
 static int test_cpucg_max(const char *root)
 {
 	int ret = KSFT_FAIL;
+	long hz = get_config_hz();
 	long quota_usec = 1000;
 	long default_period_usec = 100000; /* cpu.max's default period */
 	long duration_seconds = 1;
 
-	long duration_usec = duration_seconds * USEC_PER_SEC;
+	long duration_usec;
 	long usage_usec, n_periods, remainder_usec, expected_usage_usec;
 	char *cpucg;
 	char quota_buf[32];
 
+	duration_usec = duration_seconds * USEC_PER_SEC * 1000 / hz;
+
 	snprintf(quota_buf, sizeof(quota_buf), "%ld", quota_usec);
 
 	cpucg = cg_name(root, "cpucg_test");
@@ -670,8 +698,8 @@ static int test_cpucg_max(const char *root)
 	struct cpu_hog_func_param param = {
 		.nprocs = 1,
 		.ts = {
-			.tv_sec = duration_seconds,
-			.tv_nsec = 0,
+			.tv_sec = duration_usec / USEC_PER_SEC,
+			.tv_nsec = duration_usec % USEC_PER_SEC * NSEC_PER_USEC,
 		},
 		.clock_type = CPU_HOG_CLOCK_WALL,
 	};
@@ -710,15 +738,18 @@ static int test_cpucg_max(const char *root)
 static int test_cpucg_max_nested(const char *root)
 {
 	int ret = KSFT_FAIL;
+	long hz = get_config_hz();
 	long quota_usec = 1000;
 	long default_period_usec = 100000; /* cpu.max's default period */
 	long duration_seconds = 1;
 
-	long duration_usec = duration_seconds * USEC_PER_SEC;
+	long duration_usec;
 	long usage_usec, n_periods, remainder_usec, expected_usage_usec;
 	char *parent, *child;
 	char quota_buf[32];
 
+	duration_usec = duration_seconds * USEC_PER_SEC * 1000 / hz;
+
 	snprintf(quota_buf, sizeof(quota_buf), "%ld", quota_usec);
 
 	parent = cg_name(root, "cpucg_parent");
@@ -741,8 +772,8 @@ static int test_cpucg_max_nested(const char *root)
 	struct cpu_hog_func_param param = {
 		.nprocs = 1,
 		.ts = {
-			.tv_sec = duration_seconds,
-			.tv_nsec = 0,
+			.tv_sec = duration_usec / USEC_PER_SEC,
+			.tv_nsec = duration_usec % USEC_PER_SEC * NSEC_PER_USEC,
 		},
 		.clock_type = CPU_HOG_CLOCK_WALL,
 	};
diff --git a/tools/testing/selftests/cgroup/test_cpuset.c b/tools/testing/selftests/cgroup/test_cpuset.c
index c5cf8b56ceb8f..8b4c4a9dd78b9 100644
--- a/tools/testing/selftests/cgroup/test_cpuset.c
+++ b/tools/testing/selftests/cgroup/test_cpuset.c
@@ -1,7 +1,13 @@
 // SPDX-License-Identifier: GPL-2.0
 
+#define _GNU_SOURCE
+#include <assert.h>
 #include <linux/limits.h>
+#include <pthread.h>
+#include <sched.h>
 #include <signal.h>
+#include <sys/syscall.h>
+#include <unistd.h>
 
 #include "kselftest.h"
 #include "cgroup_util.h"
@@ -232,6 +238,242 @@ static int test_cpuset_perms_subtree(const char *root)
 	return ret;
 }
 
+static int get_cpu_affinity(cpu_set_t *mask)
+{
+	CPU_ZERO(mask);
+	return sched_getaffinity(0, sizeof(*mask), mask);
+}
+
+static int cpu_set_equal(cpu_set_t *dst, unsigned long mask)
+{
+	cpu_set_t expected;
+
+	CPU_ZERO(&expected);
+	assert(sizeof(mask) < CPU_SETSIZE);
+
+	for (int cpu = 0; cpu < sizeof(mask); ++cpu)
+		if ((1UL << cpu) & mask)
+			CPU_SET(cpu, &expected);
+
+	return CPU_EQUAL(&expected, dst);
+}
+
+enum test_phase {
+	AFFINITY_SETUP,
+	AFFINITY_THREAD_A_READY,
+	AFFINITY_THREADS_READY,
+	AFFINITY_CONTROLLER_DISABLED,
+	AFFINITY_COMPLETE,
+	AFFINITY_ERROR
+};
+
+struct thread_args {
+	const char *cgroup;
+	cpu_set_t *affinity_before;
+	cpu_set_t *affinity_after;
+	enum test_phase ready_phase;
+};
+
+static pthread_mutex_t test_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t test_cond = PTHREAD_COND_INITIALIZER;
+static enum test_phase test_phase;
+
+static void *affinity_thread_fn(void *arg)
+{
+	struct thread_args *args = (struct thread_args *)arg;
+
+	if (cg_enter_current_thread(args->cgroup))
+		goto fail;
+
+	if (get_cpu_affinity(args->affinity_before) != 0)
+		goto fail;
+
+	pthread_mutex_lock(&test_mutex);
+	if (test_phase < args->ready_phase)
+		test_phase = args->ready_phase;
+	pthread_cond_broadcast(&test_cond);
+
+	while (test_phase < AFFINITY_CONTROLLER_DISABLED)
+		pthread_cond_wait(&test_cond, &test_mutex);
+	pthread_mutex_unlock(&test_mutex);
+
+	if (get_cpu_affinity(args->affinity_after) != 0)
+		goto fail;
+
+
+	return NULL;
+
+fail:
+	pthread_mutex_lock(&test_mutex);
+	test_phase = AFFINITY_ERROR;
+	pthread_cond_broadcast(&test_cond);
+	pthread_mutex_unlock(&test_mutex);
+	return NULL;
+}
+
+/*
+ * Test that disabling cpuset controller properly updates thread affinity.
+ *
+ * This test exposes a bug in cpuset_attach() where threads in child cgroups
+ * don't get their affinity updated when the cpuset controller is disabled.
+ *
+ * Setup:
+ * - Create parent cgroup with cpuset.cpus=0-1
+ * - Create child A with cpuset.cpus=0-1
+ * - Create child B with cpuset.cpus=1
+ * - Place multithreaded process: group leader + thread_a in A, thread_b in B
+ * - Disable cpuset controller on parent
+ *
+ * Expected: thread_b's affinity should expand from {1} to {0-1}
+ * Buggy: thread_b's affinity remains {1}
+ */
+static int test_cpuset_affinity_on_controller_disable(const char *root)
+{
+	char *parent = NULL, *child_a = NULL, *child_b = NULL;
+	pthread_t thread_a, thread_b;
+	int thread_a_created = 0, thread_b_created = 0;
+	cpu_set_t affinity_a_before, affinity_a_after;
+	cpu_set_t affinity_b_before, affinity_b_after;
+	int ret = KSFT_FAIL;
+
+	parent = cg_name(root, "cpuset_affinity_test");
+	if (!parent)
+		goto cleanup;
+	if (cg_create(parent))
+		goto cleanup;
+	if (cg_write(parent, "cgroup.type", "threaded"))
+		goto cleanup;
+
+	child_a = cg_name(parent, "A");
+	if (!child_a)
+		goto cleanup;
+	if (cg_create(child_a))
+		goto cleanup;
+	if (cg_write(child_a, "cgroup.type", "threaded"))
+		goto cleanup;
+
+	child_b = cg_name(parent, "B");
+	if (!child_b)
+		goto cleanup;
+	if (cg_create(child_b))
+		goto cleanup;
+	if (cg_write(child_b, "cgroup.type", "threaded"))
+		goto cleanup;
+
+	/* Now enable cpuset controller in parent */
+	if (cg_write(parent, "cgroup.subtree_control", "+cpuset")) {
+		ret = KSFT_SKIP;
+		goto cleanup;
+	}
+
+	/* Set CPU affinity constraints */
+	if (cg_write(parent, "cpuset.cpus", "0-1"))
+		goto cleanup;
+	if (cg_write(child_a, "cpuset.cpus", "0-1"))
+		goto cleanup;
+	if (cg_write(child_b, "cpuset.cpus", "1"))
+		goto cleanup;
+
+	/* Move group leader (main thread) to child A */
+	if (cg_enter_current(child_a))
+		goto cleanup;
+
+	/* Create threads - they will move themselves to their respective cgroups */
+	test_phase = AFFINITY_SETUP;
+
+	struct thread_args args_a = {
+		.cgroup = child_a,
+		.affinity_before = &affinity_a_before,
+		.affinity_after = &affinity_a_after,
+		.ready_phase = AFFINITY_THREAD_A_READY,
+	};
+	if (pthread_create(&thread_a, NULL, affinity_thread_fn, &args_a))
+		goto cleanup;
+	thread_a_created = 1;
+
+	struct thread_args args_b = {
+		.cgroup = child_b,
+		.affinity_before = &affinity_b_before,
+		.affinity_after = &affinity_b_after,
+		.ready_phase = AFFINITY_THREADS_READY,
+	};
+	if (pthread_create(&thread_b, NULL, affinity_thread_fn, &args_b))
+		goto cleanup_threads;
+	thread_b_created = 1;
+
+	pthread_mutex_lock(&test_mutex);
+	while (test_phase < AFFINITY_THREADS_READY)
+		pthread_cond_wait(&test_cond, &test_mutex);
+
+	/* If a thread failed during setup, bail out */
+	if (test_phase == AFFINITY_ERROR) {
+		pthread_mutex_unlock(&test_mutex);
+		goto cleanup_threads;
+	}
+	pthread_mutex_unlock(&test_mutex);
+
+	if (!cpu_set_equal(&affinity_a_before, 0x3)) {
+		ksft_print_msg("FAIL: thread_a initial affinity incorrect\n");
+		goto cleanup_threads;
+	}
+
+	if (!cpu_set_equal(&affinity_b_before, 0x2)) {
+		ksft_print_msg("FAIL: thread_b initial affinity incorrect\n");
+		goto cleanup_threads;
+	}
+
+	/* Disable cpuset controller - this should trigger affinity update */
+	if (cg_write(parent, "cgroup.subtree_control", "-cpuset"))
+		goto cleanup_threads;
+
+	/* Signal threads to save their final affinity and exit */
+	pthread_mutex_lock(&test_mutex);
+	test_phase = AFFINITY_CONTROLLER_DISABLED;
+	pthread_cond_broadcast(&test_cond);
+	pthread_mutex_unlock(&test_mutex);
+
+	pthread_join(thread_a, NULL);
+	pthread_join(thread_b, NULL);
+
+	/* Verify thread affinities AFTER disabling controller */
+	if (!cpu_set_equal(&affinity_a_after, 0x3)) {
+		ksft_print_msg("FAIL: thread_a final affinity incorrect\n");
+		goto cleanup;
+	}
+
+	if (!cpu_set_equal(&affinity_b_after, 0x3)) {
+		ksft_print_msg("FAIL: thread_b affinity did not expand to {0-1}\n");
+		goto cleanup;
+	}
+
+	ret = KSFT_PASS;
+	goto cleanup;
+
+cleanup_threads:
+	pthread_mutex_lock(&test_mutex);
+	test_phase = AFFINITY_COMPLETE;
+	pthread_cond_broadcast(&test_cond);
+	pthread_mutex_unlock(&test_mutex);
+
+	if (thread_a_created)
+		pthread_join(thread_a, NULL);
+	if (thread_b_created)
+		pthread_join(thread_b, NULL);
+
+cleanup:
+	/* Move back to root before cleanup */
+	cg_enter_current(root);
+
+	cg_destroy(child_b);
+	free(child_b);
+	cg_destroy(child_a);
+	free(child_a);
+	cg_destroy(parent);
+	free(parent);
+
+	return ret;
+}
+
 
 #define T(x) { x, #x }
 struct cpuset_test {
@@ -241,6 +483,7 @@ struct cpuset_test {
 	T(test_cpuset_perms_object_allow),
 	T(test_cpuset_perms_object_deny),
 	T(test_cpuset_perms_subtree),
+	T(test_cpuset_affinity_on_controller_disable),
 };
 #undef T
 
diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh
index 0d41aa0d343d9..ca9bc38fdb95d 100755
--- a/tools/testing/selftests/cgroup/test_cpuset_prs.sh
+++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh
@@ -495,13 +495,20 @@ REMOTE_TEST_MATRIX=(
 	# Narrowing cpuset.cpus to previously sibling-excluded CPUs should
 	# not return CPUs that were never actually owned.
 	"  C1-4:P1   .   C1-2:P1  C1-3:P2  .       .  \
-	      .      .     .         C3    .       .     p1:4|c11:1-2|c12:3 \
+	      .      .     .       C3      .       .     p1:4|c11:1-2|c12:3 \
 							 p1:P1|c11:P1|c12:P2 3"
 	# Expanding cpuset.cpus to include a previously sibling-excluded CPU
 	# after the sibling has become a member should correctly request it.
 	"  C1-4:P1   .   C1-2:P1  C1-3:P2  .       .  \
-	      .      .      P0     C2-3    .       .     p1:1,4|c11:1|c12:2-3 \
+	      .      .     P0      C2-3    .       .     p1:1,4|c11:1|c12:2-3 \
 							 p1:P1|c11:P0|c12:P2 2-3"
+	# Cpusets with empty cpuset.cpus should inherit parent's effective_cpus
+	"  C1-4:P1 C5-6   C1-2     .       C5      .  \
+	      .      P1    P1      .       .       .     p1:3-4|p2:5-6|c11:1-2|c12:3-4|c21:5|c22:5-6 \
+							 p1:P1|p2:P1|c11:P1"
+	"  C1-4:P1 C5-6   C1-2     .       C5      .  \
+	      .      P1    P1      .      O5=0     .     p1:3-4|p2:6|c11:1-2|c12:3-4|c21:6|c22:6 \
+							 p1:P1|p2:P1|c11:P1"
 )
 
 #
diff --git a/tools/testing/selftests/cgroup/test_hugetlb_memcg.c b/tools/testing/selftests/cgroup/test_hugetlb_memcg.c
index b627d84358b1b..8c5aced813b62 100644
--- a/tools/testing/selftests/cgroup/test_hugetlb_memcg.c
+++ b/tools/testing/selftests/cgroup/test_hugetlb_memcg.c
@@ -199,7 +199,10 @@ static int test_hugetlb_memcg(char *root)
 int main(int argc, char **argv)
 {
 	char root[PATH_MAX];
-	int ret = EXIT_SUCCESS, has_memory_hugetlb_acc;
+	int has_memory_hugetlb_acc;
+
+	ksft_print_header();
+	ksft_set_plan(1);
 
 	has_memory_hugetlb_acc = proc_mount_contains("memory_hugetlb_accounting");
 	if (has_memory_hugetlb_acc < 0)
@@ -211,7 +214,7 @@ int main(int argc, char **argv)
 	if (get_hugepage_size() != 2048) {
 		ksft_print_msg("test_hugetlb_memcg requires 2MB hugepages\n");
 		ksft_test_result_skip("test_hugetlb_memcg\n");
-		return ret;
+		ksft_finished();
 	}
 
 	if (cg_find_unified_root(root, sizeof(root), NULL))
@@ -233,10 +236,9 @@ int main(int argc, char **argv)
 		ksft_test_result_skip("test_hugetlb_memcg\n");
 		break;
 	default:
-		ret = EXIT_FAILURE;
 		ksft_test_result_fail("test_hugetlb_memcg\n");
 		break;
 	}
 
-	return ret;
+	ksft_finished();
 }
diff --git a/tools/testing/selftests/damon/Makefile b/tools/testing/selftests/damon/Makefile
index 2180c328a8252..ece244e5c5b95 100644
--- a/tools/testing/selftests/damon/Makefile
+++ b/tools/testing/selftests/damon/Makefile
@@ -13,6 +13,7 @@ TEST_PROGS += sysfs.py
 TEST_PROGS += sysfs_update_schemes_tried_regions_wss_estimation.py
 TEST_PROGS += damos_quota.py damos_quota_goal.py damos_apply_interval.py
 TEST_PROGS += damos_tried_regions.py damon_nr_regions.py
+TEST_PROGS += sysfs_refresh.py
 TEST_PROGS += reclaim.sh lru_sort.sh
 
 # regression tests (reproducers of previously found bugs)
diff --git a/tools/testing/selftests/damon/_damon_sysfs.py b/tools/testing/selftests/damon/_damon_sysfs.py
index 8b12cc0484405..e6a2265d721e8 100644
--- a/tools/testing/selftests/damon/_damon_sysfs.py
+++ b/tools/testing/selftests/damon/_damon_sysfs.py
@@ -271,7 +271,7 @@ class DamosFilter:
         self.type_ = type_
         self.matching = matching
         self.allow = allow
-        self.memcg_path = memcg_path,
+        self.memcg_path = memcg_path
         self.addr_start = addr_start
         self.addr_end = addr_end
         self.target_idx = target_idx
@@ -624,17 +624,23 @@ class DamonCtx:
     pause = None
     idx = None
 
-    def __init__(self, ops='paddr', monitoring_attrs=DamonAttrs(), targets=[],
-            schemes=[], pause=False):
+    def __init__(self, ops='paddr', monitoring_attrs=None, targets=None,
+            schemes=None, pause=False):
         self.ops = ops
+        if monitoring_attrs is None:
+            monitoring_attrs = DamonAttrs()
         self.monitoring_attrs = monitoring_attrs
         self.monitoring_attrs.context = self
 
+        if targets is None:
+            targets = []
         self.targets = targets
         for idx, target in enumerate(self.targets):
             target.idx = idx
             target.context = self
 
+        if schemes is None:
+            schemes = []
         self.schemes = schemes
         for idx, scheme in enumerate(self.schemes):
             scheme.idx = idx
@@ -692,12 +698,14 @@ class DamonCtx:
 class Kdamond:
     state = None
     pid = None
+    refresh_ms = None
     contexts = None
     idx = None      # index of this kdamond between siblings
     kdamonds = None # parent
 
-    def __init__(self, contexts=[]):
+    def __init__(self, contexts=[], refresh_ms=None):
         self.contexts = contexts
+        self.refresh_ms = refresh_ms
         for idx, context in enumerate(self.contexts):
             context.idx = idx
             context.kdamond = self
@@ -720,6 +728,11 @@ class Kdamond:
             err = context.stage()
             if err is not None:
                 return err
+        if self.refresh_ms is not None:
+            err = write_file(os.path.join(self.sysfs_dir(), 'refresh_ms'),
+                             '%d' % self.refresh_ms)
+            if err is not None:
+                return err
         err = write_file(os.path.join(self.sysfs_dir(), 'state'), 'on')
         if err is not None:
             return err
@@ -831,7 +844,7 @@ class Kdamond:
                 for goal in scheme.quota.goals:
                     err = goal.stage()
                     if err is not None:
-                        print('commit_schemes_quota_goals failed stagign: %s'%
+                        print('commit_schemes_quota_goals failed staging: %s'%
                               err)
                         exit(1)
         return write_file(os.path.join(self.sysfs_dir(), 'state'),
diff --git a/tools/testing/selftests/damon/damos_apply_interval.py b/tools/testing/selftests/damon/damos_apply_interval.py
index f04d43702481c..0f2f36584e48c 100755
--- a/tools/testing/selftests/damon/damos_apply_interval.py
+++ b/tools/testing/selftests/damon/damos_apply_interval.py
@@ -56,7 +56,7 @@ def main():
     # Because the second scheme was having the apply interval that is ten times
     # lower than that of the first scheme, the second scheme should be tried
     # about ten times more frequently than the first scheme.  For possible
-    # timing errors, check if it was at least nine times more freuqnetly tried.
+    # timing errors, check if it was at least nine times more frequently tried.
     ratio = nr_tried_stats[1] / nr_tried_stats[0]
     if ratio < 9:
         print('%d / %d = %f (< 9)' %
diff --git a/tools/testing/selftests/damon/damos_quota_goal.py b/tools/testing/selftests/damon/damos_quota_goal.py
index f76e0412b564c..661e4ba4765ae 100755
--- a/tools/testing/selftests/damon/damos_quota_goal.py
+++ b/tools/testing/selftests/damon/damos_quota_goal.py
@@ -66,7 +66,7 @@ def main():
             # effective quota was already minimum that cannot be more reduced
             if expect_increase is False and last_effective_bytes == 1:
                 continue
-            print('efective bytes not changed: %d' % goal.effective_bytes)
+            print('effective bytes not changed: %d' % goal.effective_bytes)
             exit(1)
 
         increased = last_effective_bytes < goal.effective_bytes
diff --git a/tools/testing/selftests/damon/damos_tried_regions.py b/tools/testing/selftests/damon/damos_tried_regions.py
index 3b347eb28bd28..d6472e6a6e082 100755
--- a/tools/testing/selftests/damon/damos_tried_regions.py
+++ b/tools/testing/selftests/damon/damos_tried_regions.py
@@ -55,10 +55,10 @@ def main():
     collected_nr_regions.sort()
     sample = collected_nr_regions[4]
     print('50-th percentile nr_regions: %d' % sample)
-    print('expectation (>= 14) is %s' % 'met' if sample >= 14 else 'not met')
+    print('expectation (>= 14) is %s' % ('met' if sample >= 14 else 'not met'))
     if collected_nr_regions[4] < 14:
         print('full nr_regions:')
-        print('\n'.join(collected_nr_regions))
+        print('\n'.join(['%d' % x for x in collected_nr_regions]))
         exit(1)
 
 if __name__ == '__main__':
diff --git a/tools/testing/selftests/damon/drgn_dump_damon_status.py b/tools/testing/selftests/damon/drgn_dump_damon_status.py
index 972948e6215f1..09552e91bc782 100755
--- a/tools/testing/selftests/damon/drgn_dump_damon_status.py
+++ b/tools/testing/selftests/damon/drgn_dump_damon_status.py
@@ -59,7 +59,6 @@ def region_to_dict(region):
         ['ar', addr_range_to_dict],
         ['sampling_addr', int],
         ['nr_accesses', int],
-        ['nr_accesses_bp', int],
         ['age', int],
         ])
 
@@ -163,7 +162,7 @@ def damos_filter_to_dict(damos_filter):
                                int(damos_filter.addr_range.end)]
     elif type_ == 'target':
         dict_['target_idx'] = int(damos_filter.target_idx)
-    elif type_ == 'hugeapge_size':
+    elif type_ == 'hugepage_size':
         dict_['sz_range'] = [int(damos_filter.sz_range.min),
                              int(damos_filter.sz_range.max)]
     return dict_
diff --git a/tools/testing/selftests/damon/sysfs.py b/tools/testing/selftests/damon/sysfs.py
index aa03a1187489f..3ffa054b63867 100755
--- a/tools/testing/selftests/damon/sysfs.py
+++ b/tools/testing/selftests/damon/sysfs.py
@@ -119,7 +119,7 @@ def assert_access_pattern_committed(pattern, dump):
                 'max_nr_accesses', dump)
     assert_true(dump['min_age_region'] == pattern.age[0], 'min_age_region',
                 dump)
-    assert_true(dump['max_age_region'] == pattern.age[1], 'miaxage_region',
+    assert_true(dump['max_age_region'] == pattern.age[1], 'max_age_region',
                 dump)
 
 def assert_scheme_committed(scheme, dump):
@@ -129,7 +129,7 @@ def assert_scheme_committed(scheme, dump):
             'cold': 1,
             'pageout': 2,
             'hugepage': 3,
-            'nohugeapge': 4,
+            'nohugepage': 4,
             'collapse': 5,
             'lru_prio': 6,
             'lru_deprio': 7,
@@ -250,6 +250,35 @@ def assert_ctxs_committed(kdamonds):
             if ctx in ctxs_paused_for_dump:
                 ctx.pause = False
 
+def test_memcg_filter_memcg_path_staging():
+    global kdamonds
+    memcg_filter = _damon_sysfs.DamosFilter(
+            type_='memcg', matching=True, allow=True, memcg_path='/')
+    kdamonds = _damon_sysfs.Kdamonds(
+            [_damon_sysfs.Kdamond(
+                contexts=[_damon_sysfs.DamonCtx(
+                    targets=[_damon_sysfs.DamonTarget(pid=-1)],
+                    schemes=[_damon_sysfs.Damos(
+                        ops_filters=[memcg_filter])],
+                    )])])
+    kdamonds.start()
+
+    shown, rd_err = _damon_sysfs.read_file(
+            os.path.join(memcg_filter.sysfs_dir(), 'memcg_path'))
+    if rd_err is not None:
+        print('memcg_path staging: sysfs read (%s)' % rd_err)
+        kdamonds.stop()
+        exit(1)
+    if shown.rstrip('\n') != memcg_filter.memcg_path:
+        print('memcg_path staging: memcg_path readback '
+              '(shown=%s, expected=%s)' %
+              (shown.rstrip('\n'), memcg_filter.memcg_path))
+        kdamonds.stop()
+        exit(1)
+
+    kdamonds.stop()
+    kdamonds = None
+
 def main():
     global kdamonds
     kdamonds = _damon_sysfs.Kdamonds(
@@ -356,5 +385,7 @@ def main():
     assert_ctxs_committed(kdamonds)
     kdamonds.stop()
 
+    test_memcg_filter_memcg_path_staging()
+
 if __name__ == '__main__':
     main()
diff --git a/tools/testing/selftests/damon/sysfs.sh b/tools/testing/selftests/damon/sysfs.sh
index 78f4badb5bebb..f7fb94b84e716 100755
--- a/tools/testing/selftests/damon/sysfs.sh
+++ b/tools/testing/selftests/damon/sysfs.sh
@@ -3,7 +3,7 @@
 
 source _common.sh
 
-# Kselftest frmework requirement - SKIP code is 4.
+# Kselftest framework requirement - SKIP code is 4.
 ksft_skip=4
 
 ensure_write_succ()
@@ -28,7 +28,7 @@ ensure_write_fail()
 
 	if (echo "$content" > "$file") 2> /dev/null
 	then
-		echo "writing $content to $file succeed ($fail_reason)"
+		echo "writing $content to $file succeeded ($reason)"
 		echo "expected failure because $reason"
 		exit 1
 	fi
@@ -67,7 +67,7 @@ ensure_file()
 			echo "$file permission: expected $permission but $perm"
 			exit 1
 		fi
-	elif [ "$to_ensure" = "not_exist" ] && [ -f "$dir" ]
+	elif [ "$to_ensure" = "not_exist" ] && [ -f "$file" ]
 	then
 		echo "$file is not expected but found"
 		exit 1
@@ -99,14 +99,55 @@ test_stats()
 	done
 }
 
+test_dest()
+{
+	dest_dir=$1
+	ensure_file "$dest_dir/id" "exist" "600"
+	ensure_file "$dest_dir/weight" "exist" "600"
+}
+
+test_dests()
+{
+	dests_dir=$1
+	ensure_file "$dests_dir/nr_dests" "exist" "600"
+	ensure_write_succ "$dests_dir/nr_dests" "1" "valid input"
+	test_dest "$dests_dir/0"
+
+	ensure_write_succ "$dests_dir/nr_dests" "2" "valid input"
+	test_dest "$dests_dir/0"
+	test_dest "$dests_dir/1"
+
+	ensure_write_succ "$dests_dir/nr_dests" "0" "valid input"
+	ensure_dir "$dests_dir/0" "not_exist"
+	ensure_dir "$dests_dir/1" "not_exist"
+}
+
 test_filter()
 {
 	filter_dir=$1
 	ensure_file "$filter_dir/type" "exist" "600"
-	ensure_write_succ "$filter_dir/type" "anon" "valid input"
-	ensure_write_succ "$filter_dir/type" "memcg" "valid input"
-	ensure_write_succ "$filter_dir/type" "addr" "valid input"
-	ensure_write_succ "$filter_dir/type" "target" "valid input"
+
+	local dir_name=$(basename "$(dirname "$filter_dir")")
+	if  [ "$dir_name" = "filters" ] || [ "$dir_name" = "ops_filters" ]
+	then
+		ensure_write_succ "$filter_dir/type" "anon" "valid input"
+		ensure_write_succ "$filter_dir/type" "memcg" "valid input"
+	fi
+	if  [ "$dir_name" = "filters" ] || [ "$dir_name" = "core_filters" ]
+	then
+		ensure_write_succ "$filter_dir/type" "addr" "valid input"
+		ensure_write_succ "$filter_dir/type" "target" "valid input"
+	fi
+	if [ "$dir_name" = "core_filters" ]
+	then
+		ensure_write_fail "$filter_dir/type" "anon" "ops type"
+		ensure_write_fail "$filter_dir/type" "memcg" "ops type"
+	fi
+	if [ "$dir_name"  = "ops_filters" ]
+	then
+		ensure_write_fail "$filter_dir/type" "addr" "core type"
+		ensure_write_fail "$filter_dir/type" "target" "core type"
+	fi
 	ensure_write_fail "$filter_dir/type" "foo" "invalid input"
 	ensure_file "$filter_dir/matching" "exist" "600"
 	ensure_file "$filter_dir/memcg_path" "exist" "600"
@@ -158,6 +199,20 @@ test_goal()
 	ensure_dir "$goal_dir" "exist"
 	ensure_file "$goal_dir/target_value" "exist" "600"
 	ensure_file "$goal_dir/current_value" "exist" "600"
+	ensure_file "$goal_dir/target_metric" "exist" "600"
+	local fpath="$goal_dir/target_metric"
+	ensure_write_succ "$fpath" "user_input" "valid input"
+	ensure_write_succ "$fpath" "some_mem_psi_us" "valid input"
+	ensure_write_succ "$fpath" "node_mem_used_bp" "valid input"
+	ensure_write_succ "$fpath" "node_mem_free_bp" "valid input"
+	ensure_write_succ "$fpath" "node_memcg_used_bp" "valid input"
+	ensure_write_succ "$fpath" "node_memcg_free_bp" "valid input"
+	ensure_write_succ "$fpath" "active_mem_bp" "valid input"
+	ensure_write_succ "$fpath" "inactive_mem_bp" "valid input"
+	ensure_write_succ "$fpath" "node_eligible_mem_bp" "valid input"
+	ensure_write_fail "$fpath" "foo" "invalid input"
+	ensure_file "$goal_dir/nid" "exist" "600"
+	ensure_file "$goal_dir/path" "exist" "600"
 }
 
 test_goals()
@@ -207,7 +262,10 @@ test_scheme()
 	ensure_file "$scheme_dir/apply_interval_us" "exist" "600"
 	test_quotas "$scheme_dir/quotas"
 	test_watermarks "$scheme_dir/watermarks"
+	test_dests "$scheme_dir/dests"
 	test_filters "$scheme_dir/filters"
+	test_filters "$scheme_dir/core_filters"
+	test_filters "$scheme_dir/ops_filters"
 	test_stats "$scheme_dir/stats"
 	test_tried_regions "$scheme_dir/tried_regions"
 }
@@ -346,8 +404,13 @@ test_probes()
 	ensure_write_succ "$probes_dir/nr_probes" "1" "valid input"
 	test_probe "$probes_dir/0"
 
+	ensure_write_succ "$probes_dir/nr_probes" "2" "valid input"
+	test_probe "$probes_dir/0"
+	test_probe "$probes_dir/1"
+
 	ensure_write_succ "$probes_dir/nr_probes" "0" "valid input"
 	ensure_dir "$probes_dir/0" "not_exist"
+	ensure_dir "$probes_dir/1" "not_exist"
 }
 
 test_monitoring_attrs()
@@ -363,7 +426,7 @@ test_context()
 {
 	context_dir=$1
 	ensure_dir "$context_dir" "exist"
-	ensure_file "$context_dir/avail_operations" "exit" 400
+	ensure_file "$context_dir/avail_operations" "exist" 400
 	ensure_file "$context_dir/operations" "exist" 600
 	ensure_file "$context_dir/addr_unit" "exist" 600
 	ensure_file "$context_dir/pause" "exist" 600
diff --git a/tools/testing/selftests/damon/sysfs_refresh.py b/tools/testing/selftests/damon/sysfs_refresh.py
new file mode 100755
index 0000000000000..012b7e8f509f5
--- /dev/null
+++ b/tools/testing/selftests/damon/sysfs_refresh.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+
+import os
+import subprocess
+import time
+
+import _damon_sysfs
+
+def main():
+    # Continuously access a memory region for far longer than the test needs,
+    # so the kdamond always has a live target to monitor while we poll.
+    sz_region = 10 * 1024 * 1024
+    proc = subprocess.Popen(
+            ['./access_memory', '1', '%d' % sz_region, '60000', 'repeat'])
+
+    # A 'stat' scheme with the default (maximally wide) access pattern matches
+    # every monitored region, so its 'nr_tried' stat increases as the kdamond
+    # runs.  refresh_ms should make DAMON update the schemes' stats files under
+    # sysfs on its own, without a manual 'update_schemes_stats' request.
+    kdamond = _damon_sysfs.Kdamond(
+            refresh_ms=100,
+            contexts=[_damon_sysfs.DamonCtx(
+                ops='vaddr',
+                targets=[_damon_sysfs.DamonTarget(pid=proc.pid)],
+                schemes=[_damon_sysfs.Damos(action='stat')],
+                )])
+    kdamonds = _damon_sysfs.Kdamonds([kdamond])
+
+    err = kdamonds.start()
+    if err is not None:
+        # Kernels older than the refresh_ms feature have no such file; treat
+        # that as unsupported rather than a failure.
+        if not os.path.exists(os.path.join(kdamond.sysfs_dir(), 'refresh_ms')):
+            proc.terminate()
+            proc.wait()
+            print('kdamond has no refresh_ms file; skipping')
+            exit(_damon_sysfs.ksft_skip)
+        proc.terminate()
+        proc.wait()
+        print('kdamond start failed: %s' % err)
+        exit(1)
+
+    scheme = kdamond.contexts[0].schemes[0]
+    nr_tried_path = os.path.join(scheme.sysfs_dir(), 'stats', 'nr_tried')
+
+    try:
+        # Poll the stat file directly.  We never request an update (e.g.
+        # 'update_schemes_stats'), so 'nr_tried' can become non-zero only
+        # through the periodic refresh that refresh_ms enables.
+        nr_tried = 0
+        deadline = time.monotonic() + 10
+        while time.monotonic() < deadline:
+            if proc.poll() is not None:
+                print('the access_memory target exited unexpectedly')
+                exit(1)
+            content, err = _damon_sysfs.read_file(nr_tried_path)
+            if err is not None:
+                print('reading %s failed: %s' % (nr_tried_path, err))
+                exit(1)
+            nr_tried = int(content)
+            if nr_tried > 0:
+                break
+            time.sleep(0.1)
+    finally:
+        kdamonds.stop()
+        proc.terminate()
+        proc.wait()
+
+    if nr_tried == 0:
+        print('refresh_ms did not auto-update the schemes stats')
+        exit(1)
+
+if __name__ == '__main__':
+    main()
diff --git a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py
index 35c724a63f6cf..16fdc6e7fc566 100755
--- a/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py
+++ b/tools/testing/selftests/damon/sysfs_update_schemes_tried_regions_wss_estimation.py
@@ -7,7 +7,7 @@ import time
 import _damon_sysfs
 
 def pass_wss_estimation(sz_region):
-    # access two regions of given size, 2 seocnds per each region
+    # access two regions of given size, 2 seconds per each region
     proc = subprocess.Popen(
             ['./access_memory', '2', '%d' % sz_region, '2000', 'repeat'])
     kdamonds = _damon_sysfs.Kdamonds([_damon_sysfs.Kdamond(
diff --git a/tools/testing/selftests/dax/Makefile b/tools/testing/selftests/dax/Makefile
new file mode 100644
index 0000000000000..25a4f3d73a5ba
--- /dev/null
+++ b/tools/testing/selftests/dax/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0
+all:
+
+TEST_PROGS := dax-kmem-hotplug.sh
+
+include ../lib.mk
diff --git a/tools/testing/selftests/dax/config b/tools/testing/selftests/dax/config
new file mode 100644
index 0000000000000..4c9aaeb6ceb41
--- /dev/null
+++ b/tools/testing/selftests/dax/config
@@ -0,0 +1,4 @@
+CONFIG_DEV_DAX=m
+CONFIG_DEV_DAX_KMEM=m
+CONFIG_MEMORY_HOTPLUG=y
+CONFIG_MEMORY_HOTREMOVE=y
diff --git a/tools/testing/selftests/dax/dax-kmem-hotplug.sh b/tools/testing/selftests/dax/dax-kmem-hotplug.sh
new file mode 100755
index 0000000000000..9299120ee509c
--- /dev/null
+++ b/tools/testing/selftests/dax/dax-kmem-hotplug.sh
@@ -0,0 +1,317 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Exercise the dax/kmem "state" sysfs attribute:
+#   /sys/bus/dax/devices/daxX.Y/state  ->  unplugged | online | online_kernel | online_movable
+#
+# The test needs a dax device already bound to the kmem driver.
+#
+# This test mutates a device's memory: online/offline cycles migrate any
+# in-use pages, and the optional unbind subtest wedges the device until
+# reboot. The tester must identify the target device and opt into the
+# destructive unbind tests.
+#
+#   DAX_KMEM_TEST_DEV=daxX.Y   test this specific device
+#   DAX_KMEM_TEST_DEV=auto     auto-discover the first kmem-bound dax device
+#                              (best-effort: it may be a device in use!)
+#   DAX_KMEM_TEST_UNBIND=1     also run the destructive unbind-while-online test
+#
+# If DAX_KMEM_TEST_DEV is unset the whole test SKIPs.
+#
+# A dax device can be provisioned with the memmap= boot param, e.g.:
+#   memmap=2G!4G
+#
+# then, in the booted system:
+#
+#   ndctl create-namespace -m devdax -e namespace0.0 -f
+#   daxctl reconfigure-device -N -m system-ram dax0.0   # bind kmem
+#   DAX_KMEM_TEST_DEV=auto ./dax-kmem-hotplug.sh
+
+# shellcheck disable=SC1091
+DIR="$(dirname "$(readlink -f "$0")")"
+. "$DIR"/../kselftest/ktap_helpers.sh
+
+DAX_BASE=/sys/bus/dax/devices
+MEM_BASE=/sys/devices/system/memory
+
+memtotal_kb() { awk '/^MemTotal:/ {print $2}' /proc/meminfo; }
+get_state() { cat "$HP" 2>/dev/null; }
+# set_state STATE -- write a state to the state attribute; returns the
+# write's exit status (0 = accepted by the kernel)
+set_state() { echo "$1" > "$HP" 2>/dev/null; }
+
+is_kmem_dax() {
+	local drv
+	[ -e "$DAX_BASE/$1/state" ] || return 1
+	drv=$(readlink "$DAX_BASE/$1/driver" 2>/dev/null)
+	[ "$(basename "${drv:-}")" = kmem ]
+}
+
+find_kmem_dax() {
+	local d
+	for d in "$DAX_BASE"/dax*; do
+		is_kmem_dax "$(basename "$d")" || continue
+		basename "$d"
+		return 0
+	done
+	return 1
+}
+
+# find_device_blocks -- print every memoryN block backing this dax device.
+# The blocks are derived from the device's own range(s) in /proc/iomem (the
+# reserved resource is named after the device), so we act on *its* blocks
+# rather than guessing by NUMA node - the target node may also hold unrelated
+# (and non-offlineable) memory.
+find_device_blocks() {
+	local bs
+	bs=$(cat "$MEM_BASE/block_size_bytes" 2>/dev/null)	# hex, no leading 0x
+	[ -n "$bs" ] || return 1
+	grep -E " : ${DAX}\$" /proc/iomem | while read -r line; do
+		local range s e i
+		range=${line%% :*}; range=${range// /}
+		s=${range%-*}; e=${range#*-}
+		for (( i = 0x$s / 0x$bs; i <= 0x$e / 0x$bs; i++ )); do
+			echo "memory$i"
+		done
+	done
+}
+
+# find_device_block -- print the first online block backing this dax device.
+find_device_block() {
+	local b
+	for b in $(find_device_blocks); do
+		[ -f "$MEM_BASE/$b/state" ] || continue
+		[ "$(cat "$MEM_BASE/$b/state")" = online ] || continue
+		echo "$b"
+		return 0
+	done
+	return 1
+}
+
+ktap_print_header
+
+if [ "$UID" != 0 ]; then
+	ktap_skip_all "must be run as root"
+	exit "$KSFT_SKIP"
+fi
+
+# Device selection is opt-in - see the header for why.
+DEV_SEL=${DAX_KMEM_TEST_DEV:-}
+if [ -z "$DEV_SEL" ]; then
+	ktap_skip_all "set DAX_KMEM_TEST_DEV=<daxX.Y|auto> to opt in (mutates device memory)"
+	exit "$KSFT_SKIP"
+fi
+if [ "$DEV_SEL" = auto ]; then
+	DAX=$(find_kmem_dax)
+else
+	DAX=$DEV_SEL
+fi
+if [ -z "$DAX" ] || ! is_kmem_dax "$DAX"; then
+	ktap_skip_all "no kmem-bound dax device with a state attribute (${DEV_SEL})"
+	exit "$KSFT_SKIP"
+fi
+HP=$DAX_BASE/$DAX/state
+ORIG=$(get_state)
+
+# A failure to reach the baseline is environmental (memory in use), not an
+# interface failure, so skip rather than fail.
+set_state unplugged; rc=$?
+if [ "$rc" != 0 ] || [ "$(get_state)" != unplugged ]; then
+	ktap_skip_all "$DAX: cannot reach 'unplugged' baseline (memory in use?)"
+	[ -n "$ORIG" ] && set_state "$ORIG"
+	exit "$KSFT_SKIP"
+fi
+mt_unplugged=$(memtotal_kb)
+
+DRV=/sys/bus/dax/drivers/kmem
+AOB=$MEM_BASE/auto_online_blocks
+
+ktap_print_msg "using $DAX (initial state was: $ORIG)"
+ktap_set_plan 10
+
+# A public (N_MEMORY) kmem node onlined into a kernel zone (online/online_kernel)
+# collects unmovable allocations and can then never be offlined, which would
+# wedge the device for the rest of this test.  So this test only ever
+# successfully onlines online_movable, the one mode that is reliably unpluggable.
+
+set_state online_movable; rc=$?
+mt_online=$(memtotal_kb)
+if [ "$rc" = 0 ] && [ "$(get_state)" = online_movable ] && [ "$mt_online" -gt "$mt_unplugged" ]; then
+	ktap_test_pass "online_movable: state=online_movable, MemTotal $mt_unplugged -> $mt_online kB"
+else
+	ktap_test_fail "online_movable: rc=$rc state=$(get_state) MemTotal $mt_unplugged -> $mt_online"
+fi
+
+set_state online_movable; rc=$?
+if [ "$rc" = 0 ] && [ "$(get_state)" = online_movable ]; then
+	ktap_test_pass "online_movable idempotent"
+else
+	ktap_test_fail "online_movable idempotent: rc=$rc state=$(get_state)"
+fi
+
+# A different online type is rejected without an intervening unplug.  The write
+# is refused before any hotplug, so this never actually onlines a kernel zone.
+set_state online_kernel; rc=$?
+if [ "$rc" != 0 ] && [ "$(get_state)" = online_movable ]; then
+	ktap_test_pass "reject online_kernel without intervening unplug (no kernel-zone online)"
+else
+	ktap_test_fail "online_movable->online_kernel not rejected: rc=$rc state=$(get_state)"
+fi
+
+set_state unplugged; rc=$?
+mt=$(memtotal_kb)
+if [ "$rc" = 0 ] && [ "$(get_state)" = unplugged ] && [ "$mt" -lt "$mt_online" ]; then
+	ktap_test_pass "unplug from online_movable: MemTotal $mt_online -> $mt kB"
+else
+	ktap_test_fail "unplug from online_movable: rc=$rc state=$(get_state) MemTotal $mt_online -> $mt"
+fi
+
+before=$(get_state)
+set_state bogus_state; rc=$?
+if [ "$rc" != 0 ] && [ "$(get_state)" = "$before" ]; then
+	ktap_test_pass "reject invalid state string"
+else
+	ktap_test_fail "invalid state not rejected: rc=$rc state=$(get_state)"
+fi
+
+# An online_movable -> unplug cycle must re-acquire the per-range resources on
+# each online and release them on each unplug.  Assert every iteration grows
+# MemTotal past the baseline and returns exactly to it; memory left online after
+# unplug (off > baseline) is a partial-free failure.
+set_state unplugged
+cycle_ok=1; fail_i=0; on=0; off=0
+for i in 1 2 3; do
+	if ! set_state online_movable; then cycle_ok=0; fail_i=$i; break; fi
+	on=$(memtotal_kb)
+	if ! set_state unplugged; then cycle_ok=0; fail_i=$i; break; fi
+	off=$(memtotal_kb)
+	# online must grow past baseline, and unplug must return to it - a
+	# partial free (memory left online) is a failure, not just off == on.
+	if [ "$on" -le "$mt_unplugged" ] || [ "$off" -gt "$mt_unplugged" ]; then
+		cycle_ok=0; fail_i=$i; break
+	fi
+done
+if [ "$cycle_ok" = 1 ]; then
+	ktap_test_pass "online_movable/unplug cycle re-acquires resources (3x: added and freed each time)"
+else
+	ktap_test_fail "online_movable/unplug cycle regressed at iteration $fail_i (on=$on off=$off baseline=$mt_unplugged)"
+fi
+
+# Desync: toggle a block through the legacy per-block memoryN/state interface
+# behind the driver's back, then unplug the whole device via daxX.Y/state.
+#
+# The driver only updates daxX.Y/state on its own writes, so it still reports
+# online_movable while a block underneath is already offline.
+#
+# Whole-device unplug must still succeed (within reason, an actor changing a
+# device from online_movable to online_kernel can no longer guarantee unplug).
+# At the very least, an already-offline block should not produce an error.
+set_state unplugged
+set_state online_movable
+blk=$(find_device_block)
+if [ -n "$blk" ] && echo offline > "$MEM_BASE/$blk/state" 2>/dev/null; then
+	# daxX.Y/state is now stale (still online_movable); unplug the device.
+	set_state unplugged; rc=$?
+	mt=$(memtotal_kb)
+	if [ "$rc" = 0 ] && [ "$(get_state)" = unplugged ] && [ "$mt" -le "$mt_unplugged" ]; then
+		ktap_test_pass "unplug tolerates a block pre-offlined via memoryN/state ($blk)"
+	else
+		ktap_test_fail "desync unplug: rc=$rc state=$(get_state) MemTotal=$mt baseline=$mt_unplugged"
+	fi
+else
+	set_state unplugged 2>/dev/null
+	ktap_test_skip "could not locate a device block to offline for desync test"
+fi
+
+# change system default online policy while the device is unbound, and show
+# the new system default policy is utilized across bindings.
+set_state unplugged
+if [ -w "$AOB" ] && [ -w "$DRV/unbind" ] && [ -w "$DRV/bind" ]; then
+	orig_aob=$(cat "$AOB")
+	echo "$DAX" > "$DRV/unbind" 2>/dev/null
+	echo offline > "$AOB" 2>/dev/null
+	echo "$DAX" > "$DRV/bind" 2>/dev/null
+	sleep 1
+	st=$(get_state)
+	echo "$orig_aob" > "$AOB" 2>/dev/null		# restore system policy
+	if [ "$st" = offline ]; then
+		ktap_test_pass "online policy resolved at bind: auto_online_blocks=offline -> state=offline"
+	else
+		ktap_test_fail "bind-time policy not honored: state=$st (expected offline)"
+	fi
+	set_state unplugged 2>/dev/null
+else
+	ktap_test_skip "auto_online_blocks or driver bind/unbind not writable"
+fi
+
+# Blocks offlined out-of-band (via memoryN/state) leave daxX.Y/state stale
+# (still online_movable) while every block is actually offline.  A driver unbind
+# must still hot-remove the offline memory and free its resources rather than
+# trust the stale state and leak until reboot.  Unbind uses remove_memory(),
+# which never offlines, so removing already-offline blocks is non-destructive and
+# the device rebinds cleanly afterwards.
+if [ -w "$DRV/unbind" ] && [ -w "$DRV/bind" ]; then
+	set_state unplugged
+	set_state online_movable
+	offl_ok=1
+	for b in $(find_device_blocks); do
+		[ -f "$MEM_BASE/$b/state" ] || continue
+		[ "$(cat "$MEM_BASE/$b/state")" = online ] || continue
+		echo offline > "$MEM_BASE/$b/state" 2>/dev/null || offl_ok=0
+	done
+	# daxX.Y/state is now stale (still online_movable) while all blocks are
+	# offline; the unbind must hot-remove them anyway.
+	if [ "$offl_ok" = 1 ] && [ "$(get_state)" = online_movable ]; then
+		echo "$DAX" > "$DRV/unbind" 2>/dev/null
+		mt_after=$(memtotal_kb)
+		leaked=$(grep -cE " : ${DAX}\$" /proc/iomem)	# before rebind
+		echo "$DAX" > "$DRV/bind" 2>/dev/null		# restore for later steps
+		sleep 1
+		if [ "$mt_after" -le "$mt_unplugged" ] && [ "$leaked" = 0 ]; then
+			ktap_test_pass "unbind with stale online state hot-removes offlined blocks (no leak)"
+		else
+			ktap_test_fail "desync unbind leaked: MemTotal=$mt_after baseline=$mt_unplugged iomem_left=$leaked"
+		fi
+		set_state unplugged 2>/dev/null
+	else
+		ktap_test_skip "could not offline all device blocks for desync-unbind test"
+	fi
+else
+	ktap_test_skip "driver bind/unbind not writable for desync-unbind test"
+fi
+
+[ -n "$ORIG" ] && set_state "$ORIG"
+
+# DESTRUCTIVE and opt-in only (DAX_KMEM_TEST_UNBIND=1):
+#
+# unbinding the driver while memory is online causes the resources to leak - but
+# the unbind should not deadlock.  Instead the driver leaks it with a warning.
+
+# This leaves the memory online and the device unbound until reboot, so it runs
+# last and only when explicitly requested.  online_movable only: this test
+# never onlines a public node into a kernel zone.
+
+if [ "${DAX_KMEM_TEST_UNBIND:-}" = 1 ] && [ -w "$DRV/unbind" ]; then
+	set_state unplugged; set_state online_movable
+fi
+if [ "${DAX_KMEM_TEST_UNBIND:-}" = 1 ] && [ "$(get_state)" = online_movable ] &&
+   [ -w "$DRV/unbind" ]; then
+	mt_on=$(memtotal_kb)
+	dmesg -C 2>/dev/null
+	echo "$DAX" > "$DRV/unbind" 2>/dev/null
+	mt_after=$(memtotal_kb)
+	# The leaked "System RAM (kmem)" regions stay in the iomem tree; reading
+	# their names dereferences res_name, which a buggy unbind already freed.
+	# Walk /proc/iomem to provoke that use-after-free (caught by KASAN).
+	cat /proc/iomem > /dev/null 2>&1
+	splat=$(dmesg 2>/dev/null | grep -ciE "KASAN|BUG:|use-after-free|general protection|Oops|refcount_t")
+	if [ "$splat" = 0 ] && [ "$mt_after" -ge "$mt_on" ]; then
+		ktap_test_pass "unbind while online: memory left online, no UAF/oops (MemTotal $mt_on -> $mt_after kB)"
+	else
+		ktap_test_fail "unbind while online regressed: splat=$splat MemTotal $mt_on -> $mt_after kB"
+	fi
+else
+	ktap_test_skip "destructive unbind-while-online test (set DAX_KMEM_TEST_UNBIND=1)"
+fi
+
+ktap_finished
diff --git a/tools/testing/selftests/dax/settings b/tools/testing/selftests/dax/settings
new file mode 100644
index 0000000000000..ba4d85f74cd6b
--- /dev/null
+++ b/tools/testing/selftests/dax/settings
@@ -0,0 +1 @@
+timeout=90
diff --git a/tools/testing/selftests/drivers/net/hw/Makefile b/tools/testing/selftests/drivers/net/hw/Makefile
index fd0535a96d845..234db5c2c90cc 100644
--- a/tools/testing/selftests/drivers/net/hw/Makefile
+++ b/tools/testing/selftests/drivers/net/hw/Makefile
@@ -20,6 +20,7 @@ TEST_GEN_FILES := \
 TEST_PROGS = \
 	csum.py \
 	devlink_port_split.py \
+	devlink_rate_cross_esw.py \
 	devlink_rate_tc_bw.py \
 	devmem.py \
 	ethtool.sh \
diff --git a/tools/testing/selftests/drivers/net/hw/devlink_rate_cross_esw.py b/tools/testing/selftests/drivers/net/hw/devlink_rate_cross_esw.py
new file mode 100755
index 0000000000000..4416f024cb761
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/hw/devlink_rate_cross_esw.py
@@ -0,0 +1,296 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+
+"""
+Devlink Rate Cross-eswitch Scheduling Test Suite
+==================================================
+
+Control-plane tests for cross-eswitch TX scheduling via devlink-rate.
+Validates that VFs from different PFs on the same chip can share
+rate groups using the cross-device parent-dev attribute.
+
+Preconditions:
+- NETIF points to a bond device with exactly two interfaces.
+- the interfaces must be two PFs from different devices sharing the same chip.
+- (for mlx5): the two interfaces are in switchdev mode and configured in a LAG:
+  - devlink dev eswitch set $DEV1 mode switchdev
+  - devlink dev eswitch set $DEV2 mode switchdev
+  - devlink dev param set $DEV1 name esw_multiport value 1 cmode runtime
+  - devlink dev param set $DEV2 name esw_multiport value 1 cmode runtime
+- test cases will be skipped if:
+  - the number of interfaces in the bond device is != 2.
+  - the kernel doesn't support devlink rates.
+  - the devlink API doesn't support cross-device parents (ENODEV).
+  - cross-esw rate scheduling returns EOPNOTSUPP.
+"""
+
+import errno
+import glob
+import os
+import time
+
+from lib.py import ksft_pr, ksft_eq, ksft_run, ksft_exit
+from lib.py import KsftSkipEx, KsftFailEx
+from lib.py import NetDrvEnv, DevlinkFamily
+from lib.py import NlError
+from lib.py import cmd, defer, ip, tool
+
+
+# --- Discovery and setup ---
+
+
+def get_bond_slaves(bond_ifname):
+    """Returns sorted list of slave netdev names for a bond."""
+    pattern = f"/sys/class/net/{bond_ifname}/lower_*"
+    lowers = glob.glob(pattern)
+    if not lowers:
+        raise KsftSkipEx(f"No bond slaves for {bond_ifname}")
+    slaves = []
+    for path in sorted(lowers):
+        name = os.path.basename(path)
+        if name.startswith("lower_"):
+            name = name[len("lower_"):]
+        slaves.append(name)
+    return slaves
+
+
+def discover_pfs(cfg):
+    """Discovers both PFs from bond slaves."""
+    slaves = get_bond_slaves(cfg.ifname)
+    if len(slaves) != 2:
+        raise KsftSkipEx(f"Need 2 bond slaves, found {len(slaves)}")
+
+    pf0, pf1 = slaves[0], slaves[1]
+    ksft_pr(f"PF0: {pf0} PF1: {pf1}")
+    return pf0, pf1
+
+
+def get_pci_addr(ifname):
+    """Resolves PCI address for a network interface."""
+    return os.path.basename(os.path.realpath(f"/sys/class/net/{ifname}/device"))
+
+
+def get_vf_port_index(pf_pci):
+    """Finds devlink port-index for vf0 under pf_pci."""
+    ports = tool("devlink", "port show", json=True)["port"]
+    for port_name, props in ports.items():
+        if port_name.startswith(f"pci/{pf_pci}/") and props.get("vfnum") == 0:
+            return int(port_name.split("/")[-1])
+    raise KsftSkipEx(f"VF port not found for {pf_pci}")
+
+
+def cleanup_esw(pf):
+    """Removes VFs if created by tests."""
+    cmd(f"echo 0 > /sys/class/net/{pf}/device/sriov_numvfs", shell=True, fail=False)
+
+
+def setup_esw(pf):
+    """Creates 1 VF on 'pf'."""
+    path = f"/sys/class/net/{pf}/device/sriov_numvfs"
+    cmd(f"echo 0 > {path}", shell=True)
+    cmd(f"echo 1 > {path}", shell=True)
+    defer(cleanup_esw, pf)
+    time.sleep(2)
+
+    vf_dir = f"/sys/class/net/{pf}/device/virtfn0/net"
+    entries = os.listdir(vf_dir) if os.path.isdir(vf_dir) else []
+    if not entries:
+        raise KsftSkipEx(f"VF not found for {pf}")
+    ip(f"link set dev {entries[0]} up")
+
+    pf_pci = get_pci_addr(pf)
+    vf_idx = get_vf_port_index(pf_pci)
+    ksft_pr(f"Created VF {vf_idx} on PF {pf} ({pf_pci})")
+    return pf_pci, vf_idx
+
+
+# --- Rate operation helpers ---
+
+
+def rate_new(devnl, dev_pci, node_name, **kwargs):
+    """Creates rate node."""
+    params = {
+        "bus-name": "pci",
+        "dev-name": dev_pci,
+        "rate-node-name": node_name,
+    }
+    params.update(kwargs)
+    try:
+        devnl.rate_new(params)
+    except NlError as e:
+        if e.error == errno.EOPNOTSUPP:
+            raise KsftSkipEx("rate_new not supported") from e
+        raise KsftFailEx("rate_new failed") from e
+
+
+def rate_get(devnl, dev_pci, node_name):
+    """Gets rate node."""
+    params = {
+        "bus-name": "pci",
+        "dev-name": dev_pci,
+        "rate-node-name": node_name,
+    }
+    return devnl.rate_get(params)
+
+
+def rate_get_leaf(devnl, dev_pci, port_index):
+    """Gets rate leaf (VF)."""
+    params = {
+        "bus-name": "pci",
+        "dev-name": dev_pci,
+        "port-index": port_index,
+    }
+    return devnl.rate_get(params)
+
+
+def rate_del(devnl, dev_pci, node_name):
+    """Deletes rate node."""
+    devnl.rate_del({
+        "bus-name": "pci",
+        "dev-name": dev_pci,
+        "rate-node-name": node_name,
+    })
+
+
+def rate_set_leaf(devnl, dev_pci, port_index, **kwargs):
+    """Sets rate attributes on a leaf (VF)."""
+    params = {
+        "bus-name": "pci",
+        "dev-name": dev_pci,
+        "port-index": port_index,
+    }
+    params.update(kwargs)
+    try:
+        devnl.rate_set(params)
+    except NlError as e:
+        if e.error == errno.EOPNOTSUPP:
+            raise KsftSkipEx("rate_set not supported") from e
+        raise KsftFailEx("rate_set failed") from e
+
+
+def rate_set_leaf_parent(devnl, dev_pci, port_index,
+                         parent_name, parent_dev_pci=None):
+    """Sets a leaf's parent, optionally cross-esw."""
+    params = {
+        "bus-name": "pci",
+        "dev-name": dev_pci,
+        "port-index": port_index,
+        "rate-parent-node-name": parent_name,
+    }
+    if parent_dev_pci:
+        params["parent-dev"] = {
+            "bus-name": "pci",
+            "dev-name": parent_dev_pci,
+        }
+    try:
+        devnl.rate_set(params)
+    except NlError as e:
+        if e.error == errno.EOPNOTSUPP:
+            raise KsftSkipEx("rate_set not supported") from e
+        if parent_dev_pci and e.error == errno.ENODEV:
+            raise KsftSkipEx("Cross-esw scheduling not supported") from e
+        raise KsftFailEx("rate_set failed") from e
+
+
+def rate_clear_leaf_parent(devnl, dev_pci, port_index):
+    """Clears a leaf's parent."""
+    rate_set_leaf_parent(devnl, dev_pci, port_index, "")
+
+
+def rate_set_node(devnl, dev_pci, node_name, **kwargs):
+    """Sets rate attributes on a node."""
+    params = {
+        "bus-name": "pci",
+        "dev-name": dev_pci,
+        "rate-node-name": node_name,
+    }
+    params.update(kwargs)
+    devnl.rate_set(params)
+
+
+# --- Test cases ---
+
+
+def test_same_esw_parent(cfg):
+    """Assigns PF0's VF to PF0's group (same esw baseline)."""
+    pf0, _ = discover_pfs(cfg)
+    pf0_pci, vf0_idx = setup_esw(pf0)
+
+    rate_new(cfg.devnl, pf0_pci, "group0")
+    defer(rate_del, cfg.devnl, pf0_pci, "group0")
+    ksft_pr("rate-new succeeded")
+
+    rate_set_leaf_parent(cfg.devnl, pf0_pci, vf0_idx, "group0")
+    defer(rate_clear_leaf_parent, cfg.devnl, pf0_pci, vf0_idx)
+
+    ksft_pr("Same-esw parent assignment succeeded")
+
+
+def test_cross_esw_parent(cfg):
+    """Sets cross-esw parent, then clear it."""
+    pf0, pf1 = discover_pfs(cfg)
+    pf0_pci, _ = setup_esw(pf0)
+    pf1_pci, vf1_idx = setup_esw(pf1)
+
+    rate_new(cfg.devnl, pf0_pci, "group1")
+    defer(rate_del, cfg.devnl, pf0_pci, "group1")
+    ksft_pr("rate-new succeeded")
+
+    rate_set_leaf_parent(cfg.devnl, pf1_pci, vf1_idx,
+                         "group1", parent_dev_pci=pf0_pci)
+    defer(rate_clear_leaf_parent, cfg.devnl, pf1_pci, vf1_idx)
+
+    ksft_pr("Cross-esw parent set and clear succeeded")
+
+
+def test_tx_rates_on_cross_esw(cfg):
+    """Sets tx_max on group and tx_share on leaves in a cross-esw setup."""
+    pf0, pf1 = discover_pfs(cfg)
+    pf0_pci, vf0_idx = setup_esw(pf0)
+    pf1_pci, vf1_idx = setup_esw(pf1)
+
+    rate_new(cfg.devnl, pf0_pci, "group2", **{"rate-tx-max": 10000000})
+    defer(rate_del, cfg.devnl, pf0_pci, "group2")
+    ksft_pr("rate-new succeeded")
+
+    rate_set_leaf_parent(cfg.devnl, pf1_pci, vf1_idx,
+                         "group2", parent_dev_pci=pf0_pci)
+    defer(rate_clear_leaf_parent, cfg.devnl, pf1_pci, vf1_idx)
+    ksft_pr("set parent cross-esw succeeded")
+
+    rate_set_leaf_parent(cfg.devnl, pf0_pci, vf0_idx, "group2")
+    defer(rate_clear_leaf_parent, cfg.devnl, pf0_pci, vf0_idx)
+    ksft_pr("set parent same esw succeeded")
+
+    rate_set_leaf(cfg.devnl, pf0_pci, vf0_idx, **{"rate-tx-share": 1000000})
+    rate = rate_get_leaf(cfg.devnl, pf0_pci, vf0_idx)
+    ksft_eq(rate["rate-tx-share"], 1000000)
+    rate_set_leaf(cfg.devnl, pf1_pci, vf1_idx, **{"rate-tx-share": 2000000})
+    rate = rate_get_leaf(cfg.devnl, pf1_pci, vf1_idx)
+    ksft_eq(rate["rate-tx-share"], 2000000)
+    rate_set_node(cfg.devnl, pf0_pci, "group2", **{"rate-tx-max": 250000000})
+    rate = rate_get(cfg.devnl, pf0_pci, "group2")
+    ksft_eq(rate["rate-tx-max"], 250000000)
+
+    ksft_pr("tx_max and tx_share set on cross-esw group")
+
+
+def main() -> None:
+    """Main function."""
+
+    with NetDrvEnv(__file__, nsim_test=False) as cfg:
+        cfg.devnl = DevlinkFamily()
+
+        ksft_run(
+            cases=[
+                test_same_esw_parent,
+                test_cross_esw_parent,
+                test_tx_rates_on_cross_esw,
+            ],
+            args=(cfg,),
+        )
+    ksft_exit()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/testing/selftests/drivers/net/hw/rss_ctx.py b/tools/testing/selftests/drivers/net/hw/rss_ctx.py
index f36f76d6ca592..5b25fa89c629d 100755
--- a/tools/testing/selftests/drivers/net/hw/rss_ctx.py
+++ b/tools/testing/selftests/drivers/net/hw/rss_ctx.py
@@ -651,9 +651,14 @@ def test_rss_context_overlap(cfg, other_ctx=0):
         ntuple = defer(ethtool, f"-N {cfg.ifname} delete {ntuple_id}")
 
     # Test the main context
-    cnts = _get_rx_cnts(cfg)
-    GenerateTraffic(cfg, port=port).wait_pkts_and_stop(20000)
-    cnts = _get_rx_cnts(cfg, prev=cnts)
+    attempts = 3
+    for attempt in range(attempts):
+        cnts = _get_rx_cnts(cfg)
+        GenerateTraffic(cfg, port=port).wait_pkts_and_stop(20000)
+        cnts = _get_rx_cnts(cfg, prev=cnts)
+        if sum(cnts[:2]) >= 7000 and sum(cnts[2:4]) >= 7000:
+            break
+        ksft_pr(f"Skewed queue distribution, attempt {attempt + 1}/{attempts}: " + str(cnts))
 
     ksft_ge(sum(cnts[ :4]), 20000, "traffic on main context: " + str(cnts))
     ksft_ge(sum(cnts[ :2]),  7000, "traffic on main context (1/2): " + str(cnts))
diff --git a/tools/testing/selftests/drivers/net/hw/toeplitz.py b/tools/testing/selftests/drivers/net/hw/toeplitz.py
index cd7e080e6f84a..571732198b936 100755
--- a/tools/testing/selftests/drivers/net/hw/toeplitz.py
+++ b/tools/testing/selftests/drivers/net/hw/toeplitz.py
@@ -21,6 +21,8 @@ from lib.py import ksft_variants, KsftNamedVariant, KsftSkipEx, KsftFailEx
 ETH_RSS_HASH_TOP = 1
 # Must match RPS_MAX_CPUS in toeplitz.c
 RPS_MAX_CPUS = 16
+# Cap Rx queues so IRQ pinning leaves free CPUs in the RPS_MAX_CPUS range
+QUEUE_CAP = 8
 
 
 def _check_rps_and_rfs_not_configured(cfg):
@@ -48,6 +50,25 @@ def _get_cpu_for_irq(irq):
         return int(data)
 
 
+def _cap_queue_count(cfg):
+    ehdr = {"header": {"dev-index": cfg.ifindex}}
+    chans = cfg.ethnl.channels_get(ehdr)
+
+    config = {}
+    restore = {}
+    for key in ("combined-count", "rx-count"):
+        cur = chans.get(key, 0)
+        if cur > QUEUE_CAP:
+            config[key] = QUEUE_CAP
+            restore[key] = cur
+
+    if not config:
+        return
+
+    cfg.ethnl.channels_set(ehdr | config)
+    defer(cfg.ethnl.channels_set, ehdr | restore)
+
+
 def _get_irq_cpus(cfg):
     """
     Read the list of IRQs for the device Rx queues.
@@ -177,6 +198,7 @@ def test(cfg, proto_flag, ipver, grp):
     ]
 
     if grp:
+        _cap_queue_count(cfg)
         _check_rps_and_rfs_not_configured(cfg)
     if grp == "rss":
         irq_cpus = ",".join([str(x) for x in _get_irq_cpus(cfg)])
diff --git a/tools/testing/selftests/drivers/net/lib/py/env.py b/tools/testing/selftests/drivers/net/lib/py/env.py
index e4ab99b905b16..e4acf3d8333fd 100644
--- a/tools/testing/selftests/drivers/net/lib/py/env.py
+++ b/tools/testing/selftests/drivers/net/lib/py/env.py
@@ -159,13 +159,7 @@ class NetDrvEpEnv(NetDrvEnvBase):
 
         self.remote = Remote(kind, args, src_path)
 
-        self.addr_ipver = "6" if self.addr_v["6"] else "4"
-        self.addr = self.addr_v[self.addr_ipver]
-        self.remote_addr = self.remote_addr_v[self.addr_ipver]
-
-        # Bracketed addresses, some commands need IPv6 to be inside []
-        self.baddr = f"[{self.addr_v['6']}]" if self.addr_v["6"] else self.addr_v["4"]
-        self.remote_baddr = f"[{self.remote_addr_v['6']}]" if self.remote_addr_v["6"] else self.remote_addr_v["4"]
+        self.set_ipver("6" if self.addr_v["6"] else "4")
 
         self.ifname = self.dev['ifname']
         self.ifindex = self.dev['ifindex']
@@ -252,6 +246,25 @@ class NetDrvEpEnv(NetDrvEnvBase):
         if not self.addr_v[ipver] or not self.remote_addr_v[ipver]:
             raise KsftSkipEx(f"Test requires IPv{ipver} connectivity")
 
+    def set_ipver(self, ipver):
+        """
+        Modify the IP version used by the generic address fields.
+        """
+        if ipver == getattr(self, "addr_ipver", None):
+            return
+
+        self.require_ipver(ipver)
+
+        self.addr_ipver = ipver
+        self.addr = self.addr_v[ipver]
+        self.remote_addr = self.remote_addr_v[ipver]
+
+        # Bracketed addresses, some commands need IPv6 to be inside []
+        self.baddr = (f"[{self.addr_v['6']}]" if ipver == "6"
+                      else self.addr_v["4"])
+        self.remote_baddr = (f"[{self.remote_addr_v['6']}]" if ipver == "6"
+                             else self.remote_addr_v["4"])
+
     def require_nsim(self, nsim_test=True):
         """Require or exclude netdevsim for this test"""
         if nsim_test and self._ns is None:
diff --git a/tools/testing/selftests/drivers/net/netconsole/netcons_resume.sh b/tools/testing/selftests/drivers/net/netconsole/netcons_resume.sh
index cb59cf436dd0a..d9111f2102bcd 100755
--- a/tools/testing/selftests/drivers/net/netconsole/netcons_resume.sh
+++ b/tools/testing/selftests/drivers/net/netconsole/netcons_resume.sh
@@ -44,7 +44,8 @@ function trigger_reactivation() {
 	# Restore MACs
 	ip netns exec "${NAMESPACE}" ip link set "${DSTIF}" \
 		address "${SAVED_DSTMAC}"
-	if [ "${BINDMODE}" == "mac" ]; then
+	if [ "${BINDMODE}" == "mac" ] &&
+		[ "$(mac_get "${SRCIF}")" != "${SAVED_SRCMAC}" ]; then
 		ip link set dev "${SRCIF}" down
 		ip link set dev "${SRCIF}" address "${SAVED_SRCMAC}"
 		# Rename device in order to trigger target resume, as initial
diff --git a/tools/testing/selftests/drivers/net/xdp.py b/tools/testing/selftests/drivers/net/xdp.py
index 2ad5932299e82..0369929f3c51c 100755
--- a/tools/testing/selftests/drivers/net/xdp.py
+++ b/tools/testing/selftests/drivers/net/xdp.py
@@ -172,25 +172,45 @@ def _test_pass(cfg, bpf_info, msg_sz):
     ksft_eq(stats[XDPStats.RX.value], stats[XDPStats.PASS.value], "RX and PASS stats mismatch")
 
 
-def test_xdp_native_pass_sb(cfg):
+_ipvers = [
+    KsftNamedVariant("ipv4", "4"),
+    KsftNamedVariant("ipv6", "6"),
+]
+
+
+def _set_ipver_defer_restore(cfg, ipver):
+    old_ipver = cfg.addr_ipver
+    cfg.set_ipver(ipver)
+    defer(cfg.set_ipver, old_ipver)
+
+
+@ksft_variants(_ipvers)
+def test_xdp_native_pass_sb(cfg, ipver):
     """
     Tests the XDP_PASS action for single buffer case.
 
     Args:
         cfg: Configuration object containing network settings.
+        ipver: IP version to use ("4" or "6").
     """
+    _set_ipver_defer_restore(cfg, ipver)
+
     bpf_info = BPFProgInfo("xdp_prog", "xdp_native.bpf.o", "xdp", 1500)
 
     _test_pass(cfg, bpf_info, 256)
 
 
-def test_xdp_native_pass_mb(cfg):
+@ksft_variants(_ipvers)
+def test_xdp_native_pass_mb(cfg, ipver):
     """
     Tests the XDP_PASS action for a multi-buff size.
 
     Args:
         cfg: Configuration object containing network settings.
+        ipver: IP version to use ("4" or "6").
     """
+    _set_ipver_defer_restore(cfg, ipver)
+
     bpf_info = BPFProgInfo("xdp_prog_frags", "xdp_native.bpf.o", "xdp.frags", 9000)
 
     _test_pass(cfg, bpf_info, 8000)
@@ -219,25 +239,33 @@ def _test_drop(cfg, bpf_info, msg_sz):
     ksft_eq(stats[XDPStats.RX.value], stats[XDPStats.DROP.value], "RX and DROP stats mismatch")
 
 
-def test_xdp_native_drop_sb(cfg):
+@ksft_variants(_ipvers)
+def test_xdp_native_drop_sb(cfg, ipver):
     """
     Tests the XDP_DROP action for a signle-buff case.
 
     Args:
         cfg: Configuration object containing network settings.
+        ipver: IP version to use ("4" or "6").
     """
+    _set_ipver_defer_restore(cfg, ipver)
+
     bpf_info = BPFProgInfo("xdp_prog", "xdp_native.bpf.o", "xdp", 1500)
 
     _test_drop(cfg, bpf_info, 256)
 
 
-def test_xdp_native_drop_mb(cfg):
+@ksft_variants(_ipvers)
+def test_xdp_native_drop_mb(cfg, ipver):
     """
     Tests the XDP_DROP action for a multi-buff case.
 
     Args:
         cfg: Configuration object containing network settings.
+        ipver: IP version to use ("4" or "6").
     """
+    _set_ipver_defer_restore(cfg, ipver)
+
     bpf_info = BPFProgInfo("xdp_prog_frags", "xdp_native.bpf.o", "xdp.frags", 9000)
 
     _test_drop(cfg, bpf_info, 8000)
@@ -287,13 +315,17 @@ def _test_xdp_native_tx(cfg, bpf_info, payload_lens):
         ksft_eq(stats[XDPStats.TX.value], expected_pkts, "TX stats mismatch")
 
 
-def test_xdp_native_tx_sb(cfg):
+@ksft_variants(_ipvers)
+def test_xdp_native_tx_sb(cfg, ipver):
     """
     Tests the XDP_TX action for a single-buff case.
 
     Args:
         cfg: Configuration object containing network settings.
+        ipver: IP version to use ("4" or "6").
     """
+    _set_ipver_defer_restore(cfg, ipver)
+
     bpf_info = BPFProgInfo("xdp_prog", "xdp_native.bpf.o", "xdp", 1500)
 
     # Ensure there's enough room for an ETH / IP / UDP header
@@ -302,13 +334,17 @@ def test_xdp_native_tx_sb(cfg):
     _test_xdp_native_tx(cfg, bpf_info, [0, 1500 // 2, 1500 - pkt_hdr_len])
 
 
-def test_xdp_native_tx_mb(cfg):
+@ksft_variants(_ipvers)
+def test_xdp_native_tx_mb(cfg, ipver):
     """
     Tests the XDP_TX action for a multi-buff case.
 
     Args:
         cfg: Configuration object containing network settings.
+        ipver: IP version to use ("4" or "6").
     """
+    _set_ipver_defer_restore(cfg, ipver)
+
     bpf_info = BPFProgInfo("xdp_prog_frags", "xdp_native.bpf.o",
                            "xdp.frags", 9000)
     # The first packet ensures we exercise the fragmented code path.
@@ -447,13 +483,17 @@ def _test_xdp_native_tail_adjst(cfg, pkt_sz_lst, offset_lst):
     return {"status": "pass"}
 
 
-def test_xdp_native_adjst_tail_grow_data(cfg):
+@ksft_variants(_ipvers)
+def test_xdp_native_adjst_tail_grow_data(cfg, ipver):
     """
     Tests the XDP tail adjustment by growing packet data.
 
     Args:
         cfg: Configuration object containing network settings.
+        ipver: IP version to use ("4" or "6").
     """
+    _set_ipver_defer_restore(cfg, ipver)
+
     pkt_sz_lst = [512, 1024, 2048]
     offset_lst = [1, 16, 32, 64, 128, 256]
     res = _test_xdp_native_tail_adjst(
@@ -465,13 +505,17 @@ def test_xdp_native_adjst_tail_grow_data(cfg):
     _validate_res(res, offset_lst, pkt_sz_lst)
 
 
-def test_xdp_native_adjst_tail_shrnk_data(cfg):
+@ksft_variants(_ipvers)
+def test_xdp_native_adjst_tail_shrnk_data(cfg, ipver):
     """
     Tests the XDP tail adjustment by shrinking packet data.
 
     Args:
         cfg: Configuration object containing network settings.
+        ipver: IP version to use ("4" or "6").
     """
+    _set_ipver_defer_restore(cfg, ipver)
+
     pkt_sz_lst = [512, 1024, 2048]
     offset_lst = [-16, -32, -64, -128, -256]
     res = _test_xdp_native_tail_adjst(
@@ -535,7 +579,7 @@ def _test_xdp_native_head_adjst(cfg, prog, pkt_sz_lst, offset_lst):
             # after we eat into it. We send large-enough packets, but if HDS
             # is enabled head will only contain headers. Don't try to eat
             # more than 28 bytes (UDPv4 + eth hdr left: (14 + 20 + 8) - 14)
-            l2_cut_off = 28 if cfg.addr_ipver == 4 else 48
+            l2_cut_off = 28 if cfg.addr_ipver == "4" else 48
             if pkt_sz > hds_thresh and offset > l2_cut_off:
                 ksft_pr(
                 f"Failed run: pkt_sz ({pkt_sz}) > HDS threshold ({hds_thresh}) and "
@@ -579,18 +623,22 @@ def _test_xdp_native_head_adjst(cfg, prog, pkt_sz_lst, offset_lst):
     return {"status": "pass"}
 
 
-def test_xdp_native_adjst_head_grow_data(cfg):
+@ksft_variants(_ipvers)
+def test_xdp_native_adjst_head_grow_data(cfg, ipver):
     """
     Tests the XDP headroom growth support.
 
     Args:
         cfg: Configuration object containing network settings.
+        ipver: IP version to use ("4" or "6").
 
     This function sets up the packet size and offset lists, then calls the
     _test_xdp_native_head_adjst_mb function to perform the actual test. The
     test is passed if the headroom is successfully extended for given packet
     sizes and offsets.
     """
+    _set_ipver_defer_restore(cfg, ipver)
+
     pkt_sz_lst = [512, 1024, 2048]
 
     # Negative values result in headroom shrinking, resulting in growing of payload
@@ -600,18 +648,22 @@ def test_xdp_native_adjst_head_grow_data(cfg):
     _validate_res(res, offset_lst, pkt_sz_lst)
 
 
-def test_xdp_native_adjst_head_shrnk_data(cfg):
+@ksft_variants(_ipvers)
+def test_xdp_native_adjst_head_shrnk_data(cfg, ipver):
     """
     Tests the XDP headroom shrinking support.
 
     Args:
         cfg: Configuration object containing network settings.
+        ipver: IP version to use ("4" or "6").
 
     This function sets up the packet size and offset lists, then calls the
     _test_xdp_native_head_adjst_mb function to perform the actual test. The
     test is passed if the headroom is successfully shrunk for given packet
     sizes and offsets.
     """
+    _set_ipver_defer_restore(cfg, ipver)
+
     pkt_sz_lst = [512, 1024, 2048]
 
     # Positive values result in headroom growing, resulting in shrinking of payload
@@ -621,12 +673,19 @@ def test_xdp_native_adjst_head_shrnk_data(cfg):
     _validate_res(res, offset_lst, pkt_sz_lst)
 
 
-@ksft_variants([
-    KsftNamedVariant("pass", XDPAction.PASS),
-    KsftNamedVariant("drop", XDPAction.DROP),
-    KsftNamedVariant("tx", XDPAction.TX),
-])
-def test_xdp_native_qstats(cfg, act):
+def _qstats_variants():
+    actions = [
+        ("pass", XDPAction.PASS),
+        ("drop", XDPAction.DROP),
+        ("tx", XDPAction.TX),
+    ]
+    for ipver in ["4", "6"]:
+        for name, act in actions:
+            yield KsftNamedVariant(f"{name}_ipv{ipver}", act, ipver)
+
+
+@ksft_variants(_qstats_variants())
+def test_xdp_native_qstats(cfg, act, ipver):
     """
     Send 1000 messages. Expect XDP action specified in @act.
     Make sure the packets were counted to interface level qstats
@@ -634,6 +693,7 @@ def test_xdp_native_qstats(cfg, act):
     """
 
     cfg.require_cmd("socat")
+    _set_ipver_defer_restore(cfg, ipver)
 
     bpf_info = BPFProgInfo("xdp_prog", "xdp_native.bpf.o", "xdp", 1500)
     prog_info = _load_xdp_prog(cfg, bpf_info)
diff --git a/tools/testing/selftests/filesystems/.gitignore b/tools/testing/selftests/filesystems/.gitignore
index a78f894157de3..9eb185fb2f9dd 100644
--- a/tools/testing/selftests/filesystems/.gitignore
+++ b/tools/testing/selftests/filesystems/.gitignore
@@ -6,3 +6,4 @@ file_stressor
 anon_inode_test
 kernfs_test
 idmapped_tmpfile
+ustat_test
diff --git a/tools/testing/selftests/filesystems/Makefile b/tools/testing/selftests/filesystems/Makefile
index a7ec2ba2dd833..03be337c1f351 100644
--- a/tools/testing/selftests/filesystems/Makefile
+++ b/tools/testing/selftests/filesystems/Makefile
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 
 CFLAGS += $(KHDR_INCLUDES)
-TEST_GEN_PROGS := devpts_pts file_stressor anon_inode_test kernfs_test fclog
+TEST_GEN_PROGS := devpts_pts file_stressor anon_inode_test kernfs_test fclog ustat_test
 TEST_GEN_PROGS += idmapped_tmpfile
 TEST_GEN_PROGS_EXTENDED := dnotify_test
 
diff --git a/tools/testing/selftests/filesystems/ceph/Makefile b/tools/testing/selftests/filesystems/ceph/Makefile
new file mode 100644
index 0000000000000..4ad3e8d40d907
--- /dev/null
+++ b/tools/testing/selftests/filesystems/ceph/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0
+
+TEST_PROGS := run_validation.sh
+TEST_FILES := reset_stress.sh reset_corner_cases.sh \
+	      validate_consistency.py README settings
+
+include ../../lib.mk
diff --git a/tools/testing/selftests/filesystems/ceph/README b/tools/testing/selftests/filesystems/ceph/README
new file mode 100644
index 0000000000000..eb0092b38f80b
--- /dev/null
+++ b/tools/testing/selftests/filesystems/ceph/README
@@ -0,0 +1,84 @@
+# CephFS Client Reset Test Suite
+
+Test suite for the CephFS kernel client manual session reset feature.
+This trimmed set contains the single-client stress test, the targeted
+corner-case test, and the one-shot validation harness used during
+feature bring-up.
+
+## Prerequisites
+
+- Linux kernel with the CephFS client reset feature (this branch)
+- A running Ceph cluster with at least one MDS
+- Root access (debugfs requires it)
+- Python 3 (for validators)
+- flock utility (for lock tests, usually in util-linux)
+
+## Test inventory
+
+| Test | Script(s) | What it covers |
+|------|-----------|----------------|
+| Single-client stress | `reset_stress.sh` | I/O + resets + data integrity on one mount |
+| Corner cases | `reset_corner_cases.sh` | EBUSY, dirty caps, flock reclaim, unmount-during-reset |
+| Validation harness | `run_validation.sh` | baseline + corner cases + moderate/aggressive stress + final status check |
+
+## Quick start
+
+Stress run:
+
+    sudo ./reset_stress.sh --mount-point /mnt/cephfs --profile moderate
+
+Corner cases:
+
+    sudo ./reset_corner_cases.sh --mount-point /mnt/cephfs
+
+End-to-end validation:
+
+    sudo ./run_validation.sh --mount-point /mnt/cephfs
+
+## Stress profiles
+
+    baseline   - no resets, 1 IO + 1 rename, 600s
+    moderate   - reset every 5-15s, 2 IO + 1 rename, 900s
+    aggressive - reset every 1-5s, 4 IO + 2 rename, 900s
+    soak       - reset every 5-15s, 2 IO + 1 rename, 3600s
+
+## Key options (all scripts)
+
+    --mount-point PATH   CephFS mount point (required)
+    --client-id ID       Debugfs client id (auto-detected if one)
+
+reset_stress.sh additionally accepts:
+
+    --profile NAME       baseline|moderate|aggressive|soak
+    --duration-sec N     Override profile runtime
+    --no-reset           Disable reset injection
+    --out-dir PATH       Artifact directory
+
+## Corner case tests
+
+    [1/4] ebusy_rejection       Second reset rejected while first in-flight
+    [2/4] dirty_caps_at_reset   Reset with unflushed dirty caps
+    [3/4] flock_after_reset     Stale lock EIO + fresh lock after holder exit
+    [4/4] unmount_during_reset  umount during active reset (destroy-path wakeup)
+
+Test 4 requires creating a second CephFS mount instance and SKIPs if
+the host cannot do so.  See `--help` output for details.
+
+## Troubleshooting
+
+**No writable Ceph reset interface found:**
+Kernel lacks the reset feature, debugfs not mounted, or not root.
+Check: `ls /sys/kernel/debug/ceph/*/reset/`
+
+**Multiple Ceph clients found:**
+Use `--client-id` to select one.
+List: `ls /sys/kernel/debug/ceph/`
+
+## Files
+
+| File | Role |
+|------|------|
+| `reset_stress.sh` | Single-client stress test runner |
+| `validate_consistency.py` | Single-client post-run validator |
+| `reset_corner_cases.sh` | Corner case harness (4 sequential tests) |
+| `run_validation.sh` | One-shot validation harness |
diff --git a/tools/testing/selftests/filesystems/ceph/reset_corner_cases.sh b/tools/testing/selftests/filesystems/ceph/reset_corner_cases.sh
new file mode 100755
index 0000000000000..a6dae84a616dd
--- /dev/null
+++ b/tools/testing/selftests/filesystems/ceph/reset_corner_cases.sh
@@ -0,0 +1,646 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# CephFS client reset corner case tests.
+# Runs a checklist of targeted tests that exercise specific reset
+# code paths not covered by the stress tests.
+#
+# Requires: mounted CephFS, debugfs access (root), flock(1) utility.
+
+set -uo pipefail
+
+KSFT_SKIP=4
+
+# kselftest auto-detect: when invoked with no arguments (e.g. by
+# "make run_tests"), find a CephFS mount automatically or skip.
+if [[ $# -eq 0 ]]; then
+	MOUNT_POINT="$(findmnt -t ceph -n -o TARGET 2>/dev/null | head -1)"
+	if [[ -z "$MOUNT_POINT" ]]; then
+		echo "SKIP: No CephFS mount found and --mount-point not specified"
+		exit "$KSFT_SKIP"
+	fi
+	exec "$0" --mount-point "$MOUNT_POINT"
+fi
+
+MOUNT_POINT=""
+DEBUGFS_ROOT="/sys/kernel/debug/ceph"
+DEBUGFS_CLIENT=""
+TRIGGER_PATH=""
+STATUS_PATH=""
+TEMP_MNT=""
+
+PASS_COUNT=0
+FAIL_COUNT=0
+SKIP_COUNT=0
+TOTAL=4
+
+log()
+{
+	printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$1"
+}
+
+result()
+{
+	local num="$1"
+	local name="$2"
+	local status="$3"
+	local detail="${4:-}"
+
+	case "$status" in
+	PASS) PASS_COUNT=$((PASS_COUNT + 1)) ;;
+	FAIL) FAIL_COUNT=$((FAIL_COUNT + 1)) ;;
+	SKIP) SKIP_COUNT=$((SKIP_COUNT + 1)) ;;
+	esac
+
+	if [[ -n "$detail" ]]; then
+		printf '[%d/%d] %-30s %s  (%s)\n' "$num" "$TOTAL" "$name" "$status" "$detail"
+	else
+		printf '[%d/%d] %-30s %s\n' "$num" "$TOTAL" "$name" "$status"
+	fi
+}
+
+read_status_field()
+{
+	local field="$1"
+	awk -F': ' -v key="$field" '$1 == key {print $2}' "$STATUS_PATH" 2>/dev/null
+}
+
+wait_reset_done()
+{
+	local timeout="${1:-30}"
+	local elapsed=0
+
+	while [[ "$(read_status_field "phase")" != "idle" ]]; do
+		sleep 1
+		elapsed=$((elapsed + 1))
+		if [[ "$elapsed" -ge "$timeout" ]]; then
+			return 1
+		fi
+	done
+	return 0
+}
+
+list_reset_clients()
+{
+	local entry
+
+	for entry in "$DEBUGFS_ROOT"/*/; do
+		entry="$(basename "$entry")"
+		[[ -d "$DEBUGFS_ROOT/$entry/reset" ]] || continue
+		[[ -w "$DEBUGFS_ROOT/$entry/reset/trigger" ]] || continue
+		printf '%s\n' "$entry"
+	done
+}
+
+wait_status_nonidle()
+{
+	local status_path="$1"
+	local timeout="${2:-10}"
+	local polls=$((timeout * 10))
+	local phase
+
+	while [[ "$polls" -gt 0 ]]; do
+		phase="$(awk -F': ' '$1 == "phase" {print $2}' "$status_path" 2>/dev/null)"
+		if [[ -n "$phase" && "$phase" != "idle" ]]; then
+			return 0
+		fi
+		sleep 0.1
+		polls=$((polls - 1))
+	done
+
+	return 1
+}
+
+discover_debugfs()
+{
+	local candidates=()
+	local entry
+
+	if [[ -n "$DEBUGFS_CLIENT" ]]; then
+		if [[ ! -d "$DEBUGFS_ROOT/$DEBUGFS_CLIENT/reset" ]]; then
+			echo "SKIP: reset debugfs not found for $DEBUGFS_CLIENT" >&2
+			exit "$KSFT_SKIP"
+		fi
+		return 0
+	fi
+
+	for entry in "$DEBUGFS_ROOT"/*/; do
+		entry="$(basename "$entry")"
+		[[ -d "$DEBUGFS_ROOT/$entry/reset" ]] || continue
+		[[ -w "$DEBUGFS_ROOT/$entry/reset/trigger" ]] || continue
+		candidates+=("$entry")
+	done
+
+	if [[ ${#candidates[@]} -eq 0 ]]; then
+		echo "SKIP: No writable Ceph reset interface found under $DEBUGFS_ROOT" >&2
+		exit "$KSFT_SKIP"
+	fi
+
+	if [[ ${#candidates[@]} -gt 1 ]]; then
+		echo "SKIP: Multiple Ceph clients found: ${candidates[*]}. Use --client-id." >&2
+		exit "$KSFT_SKIP"
+	fi
+
+	DEBUGFS_CLIENT="${candidates[0]}"
+}
+
+# --- Test 1: ebusy_rejection ------------------------------------------------
+#
+# Trigger a reset while another is guaranteed in-flight.  Creates
+# dirty state so the first reset enters DRAINING (which takes
+# measurable time), then polls until phase != idle and issues the
+# second trigger.  The second trigger must fail (the kernel returns
+# -EBUSY), and only one reset must be counted in the accounting.
+
+test_ebusy_rejection()
+{
+	local num=1
+	local name="ebusy_rejection"
+	local testfile="$MOUNT_POINT/.reset_corner_ebusy_$$"
+	local tc_before tc_after sc_before sc_after second_rc phase elapsed
+
+	tc_before="$(read_status_field "trigger_count")"
+	sc_before="$(read_status_field "success_count")"
+
+	# Create dirty state so the first reset enters DRAINING
+	echo "ebusy_dirty_data" > "$testfile"
+	sync "$testfile"
+
+	python3 -c "
+import os, sys
+fd = os.open('$testfile', os.O_WRONLY | os.O_APPEND)
+os.write(fd, b'dirty_for_ebusy_test\n')
+sys.stdout.write('written')
+" 2>/dev/null || {
+		result "$num" "$name" FAIL "dirty write failed"
+		rm -f "$testfile"
+		return
+	}
+
+	# Trigger the first reset -- it will drain dirty state
+	echo "ebusy_first" > "$TRIGGER_PATH" 2>/dev/null || {
+		result "$num" "$name" FAIL "first trigger failed"
+		rm -f "$testfile"
+		return
+	}
+
+	# Poll until phase is non-idle (quiescing or draining)
+	elapsed=0
+	while true; do
+		phase="$(read_status_field "phase")"
+		if [[ "$phase" != "idle" ]]; then
+			break
+		fi
+		sleep 0.1
+		elapsed=$((elapsed + 1))
+		if [[ "$elapsed" -ge 50 ]]; then
+			result "$num" "$name" SKIP \
+				"first reset completed before overlap could be tested"
+			rm -f "$testfile" 2>/dev/null
+			return
+		fi
+	done
+
+	# Issue the second trigger -- should be rejected with EBUSY
+	second_rc=0
+	echo "ebusy_second" > "$TRIGGER_PATH" 2>/dev/null && second_rc=0 || second_rc=$?
+
+	if ! wait_reset_done 30; then
+		result "$num" "$name" FAIL "first reset never completed"
+		rm -f "$testfile"
+		return
+	fi
+
+	tc_after="$(read_status_field "trigger_count")"
+	sc_after="$(read_status_field "success_count")"
+
+	if [[ "$((tc_after - tc_before))" -ne 1 ]]; then
+		result "$num" "$name" FAIL "trigger_count +$((tc_after - tc_before)), expected +1"
+		rm -f "$testfile"
+		return
+	fi
+
+	if [[ "$((sc_after - sc_before))" -ne 1 ]]; then
+		result "$num" "$name" FAIL "success_count +$((sc_after - sc_before)), expected +1"
+		rm -f "$testfile"
+		return
+	fi
+
+	if [[ "$second_rc" -eq 0 ]]; then
+		result "$num" "$name" FAIL "second trigger did not return error"
+		rm -f "$testfile"
+		return
+	fi
+
+	rm -f "$testfile" 2>/dev/null
+	result "$num" "$name" PASS
+}
+
+# --- Test 2: dirty_caps_at_reset --------------------------------------------
+#
+# Write to a file without fsync (dirty caps), trigger reset, then
+# verify the file is not corrupt.  Manual reset drains dirty caps
+# before teardown (best-effort, 5s timeout).  For a non-stuck cap
+# the dirty write should be flushed during drain and persist.
+# If the drain window is too short, only the synced first line
+# persists -- that is acceptable (data loss is documented for
+# unflushed writes).
+
+test_dirty_caps_at_reset()
+{
+	local num=2
+	local name="dirty_caps_at_reset"
+	local testfile="$MOUNT_POINT/.reset_corner_dirty_caps_$$"
+	local content_after line_count sc_before sc_after le
+
+	sc_before="$(read_status_field "success_count")"
+
+	echo "line_1_before_dirty_write" > "$testfile"
+	sync "$testfile"
+
+	python3 -c "
+import os, sys
+fd = os.open('$testfile', os.O_WRONLY | os.O_APPEND)
+os.write(fd, b'line_2_dirty_no_fsync\n')
+# deliberately no fsync -- leave caps dirty
+sys.stdout.write('written')
+" 2>/dev/null || {
+		result "$num" "$name" FAIL "dirty write failed"
+		rm -f "$testfile"
+		return
+	}
+
+	echo "dirty_caps_test" > "$TRIGGER_PATH" 2>/dev/null || {
+		result "$num" "$name" FAIL "reset trigger failed"
+		rm -f "$testfile"
+		return
+	}
+
+	if ! wait_reset_done 30; then
+		result "$num" "$name" FAIL "reset did not complete"
+		rm -f "$testfile"
+		return
+	fi
+
+	sc_after="$(read_status_field "success_count")"
+	if [[ "$sc_after" -le "$sc_before" ]]; then
+		result "$num" "$name" FAIL "success_count did not increment (reset not exercised)"
+		rm -f "$testfile"
+		return
+	fi
+
+	sync "$testfile" 2>/dev/null || true
+	content_after="$(cat "$testfile" 2>/dev/null)" || {
+		result "$num" "$name" FAIL "cannot read file after reset"
+		rm -f "$testfile"
+		return
+	}
+
+	if [[ -z "$content_after" ]]; then
+		result "$num" "$name" FAIL "file is empty after reset"
+		rm -f "$testfile"
+		return
+	fi
+
+	line_count="$(echo "$content_after" | wc -l)"
+	if [[ "$line_count" -lt 1 ]]; then
+		result "$num" "$name" FAIL "file has $line_count lines, expected >= 1"
+		rm -f "$testfile"
+		return
+	fi
+
+	echo "$content_after" | head -1 | grep -q "line_1_before_dirty_write" || {
+		result "$num" "$name" FAIL "first line corrupted"
+		rm -f "$testfile"
+		return
+	}
+
+	le="$(read_status_field "last_errno")"
+	if [[ "$le" != "0" ]]; then
+		result "$num" "$name" FAIL "last_errno=$le, expected 0"
+		rm -f "$testfile"
+		return
+	fi
+
+	rm -f "$testfile"
+	result "$num" "$name" PASS "file intact ($line_count lines)"
+}
+
+# --- Test 3: flock_after_reset ----------------------------------------------
+#
+# Take an exclusive flock, trigger reset, verify stale lock state is
+# marked with CEPH_I_ERROR_FILELOCK (same-client flock attempt returns
+# EIO).  After the original holder exits (releasing the local lock
+# reference and clearing the error flag), a fresh lock can be acquired.
+#
+# The lock holder uses the fd-based flock form with exec, so killing
+# $lock_pid closes the lock fd immediately (no orphaned child with an
+# inherited fd copy that would prevent the VFS flock release).
+
+test_flock_after_reset()
+{
+	local num=3
+	local name="flock_after_reset"
+	local testfile="$MOUNT_POINT/.reset_corner_flock_$$"
+	local lock_pid probe_rc sc_before sc_after
+
+	sc_before="$(read_status_field "success_count")"
+
+	echo "flock_test_content" > "$testfile"
+	sync "$testfile"
+
+	# Hold lock via fd in a subshell; exec ensures killing $lock_pid
+	# closes the lock fd directly (no fork/child fd inheritance).
+	(
+		exec 9<"$testfile"
+		flock --exclusive --nonblock 9 || exit 1
+		exec sleep 300
+	) &
+	lock_pid=$!
+	sleep 0.5
+
+	if ! kill -0 "$lock_pid" 2>/dev/null; then
+		result "$num" "$name" FAIL "flock holder died immediately"
+		rm -f "$testfile"
+		return
+	fi
+
+	echo "flock_after_reset_test" > "$TRIGGER_PATH" 2>/dev/null || {
+		kill "$lock_pid" 2>/dev/null; wait "$lock_pid" 2>/dev/null
+		result "$num" "$name" FAIL "reset trigger failed"
+		rm -f "$testfile"
+		return
+	}
+
+	if ! wait_reset_done 30; then
+		kill "$lock_pid" 2>/dev/null; wait "$lock_pid" 2>/dev/null
+		result "$num" "$name" FAIL "reset did not complete"
+		rm -f "$testfile"
+		return
+	fi
+
+	sc_after="$(read_status_field "success_count")"
+	if [[ "$sc_after" -le "$sc_before" ]]; then
+		kill "$lock_pid" 2>/dev/null; wait "$lock_pid" 2>/dev/null
+		result "$num" "$name" FAIL "success_count did not increment"
+		rm -f "$testfile"
+		return
+	fi
+
+	# After teardown, CEPH_I_ERROR_FILELOCK is set on the inode.
+	# A same-client lock attempt should fail (EIO), NOT succeed.
+	probe_rc=0
+	flock --exclusive --nonblock "$testfile" true 2>/dev/null && probe_rc=0 || probe_rc=$?
+	if [[ "$probe_rc" -eq 0 ]]; then
+		kill "$lock_pid" 2>/dev/null; wait "$lock_pid" 2>/dev/null
+		result "$num" "$name" FAIL \
+			"same-client probe succeeded, expected EIO from stale lock state"
+		rm -f "$testfile"
+		return
+	fi
+
+	# Kill the holder -- the exec'd sleep IS $lock_pid, so killing it
+	# closes fd 9 directly.  VFS flock release fires ceph_fl_release_lock(),
+	# which decrements i_filelock_ref to 0 and clears CEPH_I_ERROR_FILELOCK.
+	kill "$lock_pid" 2>/dev/null
+	wait "$lock_pid" 2>/dev/null
+
+	# After the holder exits, a fresh lock should be acquirable.
+	# The reset teardown sends SESSION_REQUEST_CLOSE so the MDS
+	# releases locks promptly, but retry briefly in case the
+	# message races with the connection close.
+	local attempt
+	probe_rc=1
+	for attempt in 1 2 3 4 5; do
+		probe_rc=0
+		flock --exclusive --nonblock "$testfile" true 2>/dev/null \
+			&& probe_rc=0 || probe_rc=$?
+		[[ "$probe_rc" -eq 0 ]] && break
+		sleep 1
+	done
+	if [[ "$probe_rc" -ne 0 ]]; then
+		result "$num" "$name" FAIL \
+			"cannot acquire fresh lock after holder exit (rc=$probe_rc, ${attempt} attempts)"
+		rm -f "$testfile"
+		return
+	fi
+
+	# Verify file content survived
+	grep -q "flock_test_content" "$testfile" 2>/dev/null || {
+		result "$num" "$name" FAIL "file content corrupted after reset"
+		rm -f "$testfile"
+		return
+	}
+
+	rm -f "$testfile"
+	result "$num" "$name" PASS "stale lock detected, fresh lock acquired after holder exit"
+}
+
+# --- Test 4: unmount_during_reset -------------------------------------------
+#
+# Mount a fresh CephFS, trigger reset, immediately unmount. The
+# ceph_mdsc_destroy() path must wake blocked waiters with -ESHUTDOWN
+# and not hang.
+
+test_unmount_during_reset()
+{
+	local num=4
+	local name="unmount_during_reset"
+	local temp_mnt="/tmp/ceph_corner_mnt_$$"
+	local mount_opts=""
+	local mount_src=""
+	local temp_trigger=""
+	local temp_status=""
+	local temp_client=""
+	local temp_file="$temp_mnt/.reset_corner_umount_$$"
+	local phase=""
+	local trigger_ok=0
+	local attempt
+	local -a new_clients=()
+	declare -A existing_clients=()
+
+	mount_src="$(awk -v mp="$MOUNT_POINT" '$2 == mp && $3 == "ceph" {print $1; exit}' /proc/mounts 2>/dev/null)"
+	mount_opts="$(awk -v mp="$MOUNT_POINT" '$2 == mp && $3 == "ceph" {print $4; exit}' /proc/mounts 2>/dev/null)"
+
+	if [[ -z "$mount_src" ]]; then
+		result "$num" "$name" SKIP "cannot determine mount source from /proc/mounts"
+		return
+	fi
+
+	while IFS= read -r existing; do
+		[[ -n "$existing" ]] || continue
+		existing_clients["$existing"]=1
+	done < <(list_reset_clients)
+
+	mkdir -p "$temp_mnt"
+
+	if ! mount -t ceph "$mount_src" "$temp_mnt" -o "$mount_opts" 2>/dev/null; then
+		result "$num" "$name" SKIP "cannot mount additional CephFS instance"
+		rmdir "$temp_mnt" 2>/dev/null
+		return
+	fi
+
+	ls "$temp_mnt" > /dev/null 2>&1
+	sync
+	sleep 1
+
+	for attempt in $(seq 1 50); do
+		new_clients=()
+		while IFS= read -r entry; do
+			[[ -n "$entry" ]] || continue
+			if [[ -n "${existing_clients[$entry]+x}" ]]; then
+				continue
+			fi
+			new_clients+=("$entry")
+		done < <(list_reset_clients)
+
+		if [[ "${#new_clients[@]}" -eq 1 ]]; then
+			temp_client="${new_clients[0]}"
+			break
+		fi
+
+		if [[ "${#new_clients[@]}" -gt 1 ]]; then
+			break
+		fi
+
+		sleep 0.1
+	done
+
+	if [[ -z "$temp_client" ]]; then
+		umount "$temp_mnt" 2>/dev/null || umount -l "$temp_mnt" 2>/dev/null
+		rmdir "$temp_mnt" 2>/dev/null
+		result "$num" "$name" SKIP "cannot identify debugfs client for temp mount"
+		return
+	fi
+
+	if [[ "${#new_clients[@]}" -gt 1 ]]; then
+		umount "$temp_mnt" 2>/dev/null || umount -l "$temp_mnt" 2>/dev/null
+		rmdir "$temp_mnt" 2>/dev/null
+		result "$num" "$name" SKIP "multiple new debugfs clients appeared"
+		return
+	fi
+
+	temp_trigger="$DEBUGFS_ROOT/$temp_client/reset/trigger"
+	temp_status="$DEBUGFS_ROOT/$temp_client/reset/status"
+
+	echo "umount_dirty_seed" > "$temp_file" 2>/dev/null || {
+		umount "$temp_mnt" 2>/dev/null || umount -l "$temp_mnt" 2>/dev/null
+		rmdir "$temp_mnt" 2>/dev/null
+		result "$num" "$name" FAIL "cannot create dirty state on temp mount"
+		return
+	}
+	sync "$temp_file"
+	python3 -c "
+import os, sys
+fd = os.open('$temp_file', os.O_WRONLY | os.O_APPEND)
+os.write(fd, b'dirty_for_umount_test\\n')
+os.close(fd)
+" 2>/dev/null || {
+		umount "$temp_mnt" 2>/dev/null || umount -l "$temp_mnt" 2>/dev/null
+		rmdir "$temp_mnt" 2>/dev/null
+		result "$num" "$name" FAIL "cannot dirty temp mount for reset overlap"
+		return
+	}
+
+	echo "unmount_test" > "$temp_trigger" 2>/dev/null && trigger_ok=1 || trigger_ok=0
+	if [[ "$trigger_ok" -ne 1 ]]; then
+		umount "$temp_mnt" 2>/dev/null || umount -l "$temp_mnt" 2>/dev/null
+		rmdir "$temp_mnt" 2>/dev/null
+		result "$num" "$name" FAIL "cannot trigger reset on temp mount"
+		return
+	fi
+
+	if ! wait_status_nonidle "$temp_status" 10; then
+		phase="$(awk -F': ' '$1 == "phase" {print $2}' "$temp_status" 2>/dev/null)"
+		umount "$temp_mnt" 2>/dev/null || umount -l "$temp_mnt" 2>/dev/null
+		rmdir "$temp_mnt" 2>/dev/null
+		result "$num" "$name" FAIL \
+			"reset never became active before umount (phase=${phase:-unknown})"
+		return
+	fi
+
+	local umount_ok=0
+	timeout 30 umount "$temp_mnt" 2>/dev/null && umount_ok=1
+
+	if [[ "$umount_ok" -ne 1 ]]; then
+		umount -l "$temp_mnt" 2>/dev/null || true
+		rmdir "$temp_mnt" 2>/dev/null
+		result "$num" "$name" FAIL "umount hung for >30s"
+		return
+	fi
+
+	rmdir "$temp_mnt" 2>/dev/null
+
+	ls "$MOUNT_POINT" > /dev/null 2>&1 || {
+		result "$num" "$name" FAIL "original mount unhealthy after test"
+		return
+	}
+
+	result "$num" "$name" PASS
+}
+
+# --- Main --------------------------------------------------------------------
+
+usage()
+{
+	cat <<EOF
+Usage: $0 --mount-point <path> [--client-id <id>] [--debugfs-root <path>]
+
+Runs targeted corner-case tests for the CephFS client reset feature.
+Requires root (debugfs access) and a mounted CephFS filesystem.
+
+Options:
+  --mount-point PATH     CephFS mount point (required)
+  --client-id ID         Ceph debugfs client id (auto-detect if one client)
+  --debugfs-root PATH    Debugfs ceph root (default: /sys/kernel/debug/ceph)
+  --help                 Show this message
+EOF
+}
+
+main()
+{
+	while [[ $# -gt 0 ]]; do
+		case "$1" in
+		--mount-point)   MOUNT_POINT="$2"; shift 2 ;;
+		--client-id)     DEBUGFS_CLIENT="$2"; shift 2 ;;
+		--debugfs-root)  DEBUGFS_ROOT="$2"; shift 2 ;;
+		--help|-h)       usage; exit 0 ;;
+		*)               echo "Unknown option: $1" >&2; usage; exit 2 ;;
+		esac
+	done
+
+	if [[ -z "$MOUNT_POINT" ]]; then
+		echo "--mount-point is required" >&2
+		usage
+		exit 2
+	fi
+
+	if [[ ! -d "$MOUNT_POINT" ]]; then
+		echo "SKIP: Mount point does not exist: $MOUNT_POINT" >&2
+		exit "$KSFT_SKIP"
+	fi
+
+	discover_debugfs
+	TRIGGER_PATH="$DEBUGFS_ROOT/$DEBUGFS_CLIENT/reset/trigger"
+	STATUS_PATH="$DEBUGFS_ROOT/$DEBUGFS_CLIENT/reset/status"
+
+	log "CephFS client reset corner case tests"
+	log "Mount: $MOUNT_POINT"
+	log "Client: $DEBUGFS_CLIENT"
+	echo ""
+
+	test_ebusy_rejection
+	test_dirty_caps_at_reset
+	test_flock_after_reset
+	test_unmount_during_reset
+
+	echo ""
+	echo "Results: $PASS_COUNT passed, $FAIL_COUNT failed, $SKIP_COUNT skipped (of $TOTAL)"
+
+	if [[ "$FAIL_COUNT" -gt 0 ]]; then
+		exit 1
+	fi
+	exit 0
+}
+
+main "$@"
diff --git a/tools/testing/selftests/filesystems/ceph/reset_stress.sh b/tools/testing/selftests/filesystems/ceph/reset_stress.sh
new file mode 100755
index 0000000000000..c503c75a5f7a9
--- /dev/null
+++ b/tools/testing/selftests/filesystems/ceph/reset_stress.sh
@@ -0,0 +1,694 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# CephFS reset stress test:
+# - Runs concurrent I/O and rename workloads
+# - Triggers random client resets through debugfs
+# - Validates consistency and recovery behavior
+
+set -euo pipefail
+
+KSFT_SKIP=4
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# kselftest auto-detect: when invoked with no arguments (e.g. by
+# "make run_tests"), find a CephFS mount automatically or skip.
+if [[ $# -eq 0 ]]; then
+	MOUNT_POINT="$(findmnt -t ceph -n -o TARGET 2>/dev/null | head -1)"
+	if [[ -z "$MOUNT_POINT" ]]; then
+		echo "SKIP: No CephFS mount found and --mount-point not specified"
+		exit "$KSFT_SKIP"
+	fi
+	exec "$0" --mount-point "$MOUNT_POINT"
+fi
+
+PROFILE="moderate"
+DURATION_SEC=""
+COOLDOWN_SEC=20
+FILE_COUNT=64
+IO_WORKERS=""
+RENAME_WORKERS=""
+MOUNT_POINT=""
+OUT_DIR=""
+CLIENT_ID=""
+DEBUGFS_ROOT="/sys/kernel/debug/ceph"
+SLO_SECONDS=30
+EXPECT_RESET=1
+DMESG_CMD=""
+SUDO=""
+
+RESET_MIN_SEC=5
+RESET_MAX_SEC=15
+
+RUN_ID="$(date +%Y%m%d-%H%M%S)"
+WORKLOAD_FLAG=""
+RESET_FLAG=""
+DATA_DIR=""
+
+IO_LOG=""
+RENAME_LOG=""
+RESET_LOG=""
+STATUS_LOG=""
+STATUS_BEFORE=""
+STATUS_FINAL=""
+DMESG_LOG=""
+SUMMARY_LOG=""
+REPORT_JSON=""
+
+RESET_PID=0
+STATUS_PID=0
+declare -a IO_WORKER_PIDS=()
+declare -a RENAME_WORKER_PIDS=()
+
+usage()
+{
+	cat <<EOF
+Usage: $0 --mount-point <cephfs_mount> [options]
+
+Required:
+  --mount-point PATH       CephFS mount point to test under
+
+Options:
+  --profile NAME           baseline|moderate|aggressive|soak (default: moderate)
+  --duration-sec N         Override profile runtime in seconds
+  --cooldown-sec N         Workload drain time after injector stop (default: 20)
+  --file-count N           Number of logical files (default: 64)
+  --io-workers N           Number of concurrent I/O workers (profile default)
+  --rename-workers N       Number of concurrent rename workers (profile default)
+  --out-dir PATH           Artifact directory (default: /tmp/ceph_reset_stress_<ts>)
+  --client-id ID           Ceph debugfs client id; auto-detect if one client exists
+  --debugfs-root PATH      Debugfs Ceph root (default: /sys/kernel/debug/ceph)
+  --slo-seconds N          Max allowed post-reset stall window (default: 30)
+  --no-reset               Disable reset injector (baseline mode helper)
+  --help                   Show this message
+
+Examples:
+  $0 --mount-point /mnt/cephfs --profile moderate
+  $0 --mount-point /mnt/cephfs --profile aggressive --duration-sec 300
+  $0 --mount-point /mnt/cephfs --profile baseline --no-reset
+EOF
+}
+
+now_ms()
+{
+	date +%s%3N
+}
+
+set_profile_defaults()
+{
+	case "$PROFILE" in
+	baseline)
+		RESET_MIN_SEC=0
+		RESET_MAX_SEC=0
+		EXPECT_RESET=0
+		: "${DURATION_SEC:=600}"
+		: "${IO_WORKERS:=1}"
+		: "${RENAME_WORKERS:=1}"
+		;;
+	moderate)
+		RESET_MIN_SEC=5
+		RESET_MAX_SEC=15
+		: "${DURATION_SEC:=900}"
+		: "${IO_WORKERS:=2}"
+		: "${RENAME_WORKERS:=1}"
+		;;
+	aggressive)
+		RESET_MIN_SEC=1
+		RESET_MAX_SEC=5
+		: "${DURATION_SEC:=900}"
+		: "${IO_WORKERS:=4}"
+		: "${RENAME_WORKERS:=2}"
+		;;
+	soak)
+		RESET_MIN_SEC=5
+		RESET_MAX_SEC=15
+		: "${DURATION_SEC:=3600}"
+		: "${IO_WORKERS:=2}"
+		: "${RENAME_WORKERS:=1}"
+		;;
+	*)
+		echo "Unknown profile: $PROFILE" >&2
+		exit 2
+		;;
+	esac
+}
+
+log_summary()
+{
+	local msg="$1"
+	printf '[%s] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$msg" | tee -a "$SUMMARY_LOG"
+}
+
+discover_client_id()
+{
+	local candidates=()
+	local entry
+
+	if [[ -n "$CLIENT_ID" ]]; then
+		if ! $SUDO test -d "$DEBUGFS_ROOT/$CLIENT_ID/reset"; then
+			echo "SKIP: reset debugfs not found for client-id=$CLIENT_ID" >&2
+			exit "$KSFT_SKIP"
+		fi
+		return 0
+	fi
+
+	if ! $SUDO test -d "$DEBUGFS_ROOT"; then
+		echo "SKIP: Debugfs root not found: $DEBUGFS_ROOT" >&2
+		exit "$KSFT_SKIP"
+	fi
+
+	while IFS= read -r entry; do
+		$SUDO test -d "$DEBUGFS_ROOT/$entry/reset" || continue
+		$SUDO test -w "$DEBUGFS_ROOT/$entry/reset/trigger" || continue
+		candidates+=("$entry")
+	done < <($SUDO ls -1 "$DEBUGFS_ROOT" 2>/dev/null || true)
+
+	if [[ ${#candidates[@]} -eq 1 ]]; then
+		CLIENT_ID="${candidates[0]}"
+		return 0
+	fi
+
+	if [[ ${#candidates[@]} -eq 0 ]]; then
+		echo "SKIP: No writable Ceph reset interface found under $DEBUGFS_ROOT" >&2
+		exit "$KSFT_SKIP"
+	fi
+
+	echo "SKIP: Multiple Ceph clients found (${candidates[*]}). Use --client-id." >&2
+	exit "$KSFT_SKIP"
+}
+
+init_dataset()
+{
+	local i
+	mkdir -p "$DATA_DIR/A" "$DATA_DIR/B"
+
+	for ((i = 0; i < FILE_COUNT; i++)); do
+		printf 'seed logical_id=%05d ts_ms=%s\n' "$i" "$(now_ms)" > "$DATA_DIR/A/file_$(printf '%05d' "$i")"
+	done
+}
+
+io_worker()
+{
+	set +e
+	local worker_id="$1"
+	local seq=0
+	local id
+	local relpath
+	local abspath
+	local payload
+	local hash
+	local ts
+
+	while [[ -f "$WORKLOAD_FLAG" ]]; do
+		id="$(printf '%05d' $((RANDOM % FILE_COUNT)))"
+		if [[ -f "$DATA_DIR/A/file_$id" ]]; then
+			relpath="A/file_$id"
+		elif [[ -f "$DATA_DIR/B/file_$id" ]]; then
+			relpath="B/file_$id"
+		else
+			sleep 0.02
+			continue
+		fi
+
+		abspath="$DATA_DIR/$relpath"
+		alt_relpath=""
+		if [[ "$relpath" == A/* ]]; then
+			alt_relpath="B/file_$id"
+		else
+			alt_relpath="A/file_$id"
+		fi
+		alt_abspath="$DATA_DIR/$alt_relpath"
+		payload="worker=${worker_id} io_seq=${seq} id=${id} ts_ms=$(now_ms)"
+		result="$(
+			python3 - "$abspath" "$alt_abspath" "$payload" <<'PY'
+import hashlib
+import os
+import sys
+
+path = sys.argv[1]
+alt_path = sys.argv[2]
+payload = sys.argv[3]
+
+try:
+    fd = os.open(path, os.O_RDWR | os.O_APPEND)
+    actual = path
+except FileNotFoundError:
+    try:
+        fd = os.open(alt_path, os.O_RDWR | os.O_APPEND)
+        actual = alt_path
+    except FileNotFoundError:
+        sys.exit(1)
+
+try:
+    os.write(fd, (payload + "\n").encode())
+    os.fsync(fd)
+    os.lseek(fd, 0, os.SEEK_SET)
+    digest = hashlib.sha256()
+    while True:
+        chunk = os.read(fd, 1 << 20)
+        if not chunk:
+            break
+        digest.update(chunk)
+    print(actual + " " + digest.hexdigest())
+finally:
+    os.close(fd)
+PY
+		)" || {
+			sleep 0.02
+			continue
+		}
+
+		actual_abspath="${result%% *}"
+		hash="${result#* }"
+		if [[ "$actual_abspath" == "$alt_abspath" ]]; then
+			relpath="$alt_relpath"
+		fi
+
+		ts="$(now_ms)"
+		printf '%s,%s,%s,%s,%s\n' "$ts" "$seq" "$id" "$relpath" "$hash" >> "$IO_LOG"
+		seq=$((seq + 1))
+		sleep 0.02
+	done
+}
+
+rename_worker()
+{
+	set +e
+	local worker_id="$1"
+	local seq=0
+	local id
+	local src_rel
+	local dst_rel
+	local rc
+	local ts
+
+	while [[ -f "$WORKLOAD_FLAG" ]]; do
+		id="$(printf '%05d' $((RANDOM % FILE_COUNT)))"
+
+		if [[ -f "$DATA_DIR/A/file_$id" ]]; then
+			src_rel="A/file_$id"
+			dst_rel="B/file_$id"
+		elif [[ -f "$DATA_DIR/B/file_$id" ]]; then
+			src_rel="B/file_$id"
+			dst_rel="A/file_$id"
+		else
+			sleep 0.02
+			continue
+		fi
+
+		ts="$(now_ms)"
+		if mv -T "$DATA_DIR/$src_rel" "$DATA_DIR/$dst_rel" 2>/dev/null; then
+			rc=0
+		else
+			rc=$?
+		fi
+		printf '%s,%s,%s,%s,%s,%s,%s\n' "$ts" "$worker_id" "$seq" "$id" "$src_rel" "$dst_rel" "$rc" >> "$RENAME_LOG"
+		seq=$((seq + 1))
+		sleep 0.02
+	done
+}
+
+random_sleep_seconds()
+{
+	local min_sec="$1"
+	local max_sec="$2"
+	local wait_sec
+	local span
+
+	span=$((max_sec - min_sec + 1))
+	wait_sec=$((min_sec + RANDOM % span))
+	sleep "$wait_sec"
+}
+
+reset_injector()
+{
+	set +e
+	local trigger_path="$1"
+	local seq=0
+	local ts
+	local reason
+	local rc
+
+	while [[ -f "$RESET_FLAG" ]]; do
+		random_sleep_seconds "$RESET_MIN_SEC" "$RESET_MAX_SEC"
+		[[ -f "$RESET_FLAG" ]] || break
+
+		ts="$(now_ms)"
+		reason="stress_${seq}_${ts}"
+		if echo "$reason" | $SUDO tee "$trigger_path" > /dev/null 2>&1; then
+			rc=0
+		else
+			rc=$?
+		fi
+		printf '%s,%s,%s,%s\n' "$ts" "$seq" "$reason" "$rc" >> "$RESET_LOG"
+		seq=$((seq + 1))
+	done
+}
+
+status_sampler()
+{
+	set +e
+	local status_path="$1"
+	local ts
+	local kv_line
+
+	while [[ -f "$WORKLOAD_FLAG" || -f "$RESET_FLAG" ]]; do
+		ts="$(now_ms)"
+		if $SUDO test -r "$status_path"; then
+			kv_line="$($SUDO awk -F': ' 'NF>=2 {gsub(/ /, "", $1); gsub(/ /, "", $2); printf "%s=%s;", $1, $2}' "$status_path")"
+			printf '%s,%s\n' "$ts" "$kv_line" >> "$STATUS_LOG"
+		fi
+		sleep 1
+	done
+}
+
+stop_pid_with_timeout()
+{
+	local pid="$1"
+	local name="$2"
+	local timeout="$3"
+	local waited=0
+
+	if [[ "$pid" -le 0 ]]; then
+		return 0
+	fi
+
+	while kill -0 "$pid" 2>/dev/null; do
+		if (( waited >= timeout )); then
+			log_summary "Timeout waiting for $name (pid=$pid), sending SIGTERM/SIGKILL"
+			kill -TERM "$pid" 2>/dev/null || true
+			sleep 1
+			kill -KILL "$pid" 2>/dev/null || true
+			wait "$pid" 2>/dev/null || true
+			return 1
+		fi
+		sleep 1
+		waited=$((waited + 1))
+	done
+
+	wait "$pid" 2>/dev/null || true
+	return 0
+}
+
+detect_privileges()
+{
+	if [[ -r "$DEBUGFS_ROOT" ]]; then
+		SUDO=""
+	elif sudo -n true 2>/dev/null; then
+		SUDO="sudo"
+	else
+		echo "WARNING: $DEBUGFS_ROOT is not readable and passwordless sudo is not available" >&2
+		echo "WARNING: reset injection, debugfs status checks, and dmesg capture will not work" >&2
+	fi
+
+	if $SUDO dmesg > /dev/null 2>&1; then
+		DMESG_CMD="$SUDO dmesg"
+	else
+		DMESG_CMD=""
+		echo "WARNING: dmesg is not accessible; kernel errors (hung tasks) will not be detected" >&2
+	fi
+}
+
+check_dmesg()
+{
+	local start_epoch="$1"
+
+	if [[ -z "$DMESG_CMD" ]]; then
+		return 0
+	fi
+
+	if ! $DMESG_CMD --since "@$start_epoch" > "$DMESG_LOG" 2>/dev/null; then
+		if ! $DMESG_CMD > "$DMESG_LOG" 2>/dev/null; then
+			log_summary "WARNING: dmesg capture failed unexpectedly"
+			return 0
+		fi
+		log_summary "dmesg --since unsupported; captured full dmesg"
+	fi
+
+	if grep -qi "hung task" "$DMESG_LOG" 2>/dev/null; then
+		log_summary "ERROR: kernel log contains 'hung task' during test window"
+		return 1
+	fi
+
+	return 0
+}
+
+cleanup()
+{
+	rm -f "$WORKLOAD_FLAG" "$RESET_FLAG"
+	local pid
+	for pid in "${IO_WORKER_PIDS[@]}" "${RENAME_WORKER_PIDS[@]}" "$RESET_PID" "$STATUS_PID"; do
+		[[ "$pid" -gt 0 ]] 2>/dev/null && kill "$pid" 2>/dev/null || true
+	done
+	wait 2>/dev/null || true
+}
+
+parse_args()
+{
+	while [[ $# -gt 0 ]]; do
+		case "$1" in
+		--mount-point)
+			MOUNT_POINT="$2"
+			shift 2
+			;;
+		--profile)
+			PROFILE="$2"
+			shift 2
+			;;
+		--duration-sec)
+			DURATION_SEC="$2"
+			shift 2
+			;;
+		--cooldown-sec)
+			COOLDOWN_SEC="$2"
+			shift 2
+			;;
+		--file-count)
+			FILE_COUNT="$2"
+			shift 2
+			;;
+		--io-workers)
+			IO_WORKERS="$2"
+			shift 2
+			;;
+		--rename-workers)
+			RENAME_WORKERS="$2"
+			shift 2
+			;;
+		--out-dir)
+			OUT_DIR="$2"
+			shift 2
+			;;
+		--client-id)
+			CLIENT_ID="$2"
+			shift 2
+			;;
+		--debugfs-root)
+			DEBUGFS_ROOT="$2"
+			shift 2
+			;;
+		--slo-seconds)
+			SLO_SECONDS="$2"
+			shift 2
+			;;
+		--no-reset)
+			EXPECT_RESET=0
+			shift
+			;;
+		--help|-h)
+			usage
+			exit 0
+			;;
+		*)
+			echo "Unknown option: $1" >&2
+			usage
+			exit 2
+			;;
+		esac
+	done
+}
+
+main()
+{
+	local start_epoch
+	local trigger_path=""
+	local status_path=""
+	local final_rc=0
+	local reset_enabled=0
+	local i
+
+	parse_args "$@"
+
+	if [[ -z "$MOUNT_POINT" ]]; then
+		echo "--mount-point is required" >&2
+		usage
+		exit 2
+	fi
+
+	if [[ ! -d "$MOUNT_POINT" ]]; then
+		echo "SKIP: Mount point does not exist: $MOUNT_POINT" >&2
+		exit "$KSFT_SKIP"
+	fi
+
+	if ! touch "$MOUNT_POINT/.ceph_reset_test_probe" 2>/dev/null; then
+		echo "SKIP: Mount point is not writable: $MOUNT_POINT" >&2
+		exit "$KSFT_SKIP"
+	fi
+	rm -f "$MOUNT_POINT/.ceph_reset_test_probe"
+
+	if ! command -v python3 > /dev/null 2>&1; then
+		echo "SKIP: python3 is required but not found in PATH" >&2
+		exit "$KSFT_SKIP"
+	fi
+
+	if ! stat -f -c '%T' "$MOUNT_POINT" 2>/dev/null | grep -qi ceph; then
+		echo "WARNING: $MOUNT_POINT does not appear to be a CephFS mount" >&2
+	fi
+
+	detect_privileges
+
+	set_profile_defaults
+	if [[ "$EXPECT_RESET" -eq 0 ]]; then
+		PROFILE="baseline"
+		RESET_MIN_SEC=0
+		RESET_MAX_SEC=0
+	fi
+
+	if ! [[ "$IO_WORKERS" =~ ^[0-9]+$ && "$RENAME_WORKERS" =~ ^[0-9]+$ ]]; then
+		echo "io-workers and rename-workers must be integers" >&2
+		exit 2
+	fi
+
+	if [[ "$IO_WORKERS" -le 0 || "$RENAME_WORKERS" -le 0 ]]; then
+		echo "io-workers and rename-workers must be > 0" >&2
+		exit 2
+	fi
+
+	if [[ -z "$OUT_DIR" ]]; then
+		OUT_DIR="/tmp/ceph_reset_stress_${RUN_ID}"
+	fi
+	mkdir -p "$OUT_DIR"
+
+	WORKLOAD_FLAG="$OUT_DIR/workload.running"
+	RESET_FLAG="$OUT_DIR/reset.running"
+
+	DATA_DIR="$MOUNT_POINT/ceph_reset_stress_${RUN_ID}"
+	mkdir -p "$DATA_DIR"
+
+	IO_LOG="$OUT_DIR/io.log"
+	RENAME_LOG="$OUT_DIR/rename.log"
+	RESET_LOG="$OUT_DIR/reset.log"
+	STATUS_LOG="$OUT_DIR/status.log"
+	STATUS_BEFORE="$OUT_DIR/reset_status.before"
+	STATUS_FINAL="$OUT_DIR/reset_status.final"
+	DMESG_LOG="$OUT_DIR/dmesg.log"
+	SUMMARY_LOG="$OUT_DIR/summary.log"
+	REPORT_JSON="$OUT_DIR/validator_report.json"
+
+	: > "$IO_LOG"
+	: > "$RENAME_LOG"
+	: > "$RESET_LOG"
+	: > "$STATUS_LOG"
+	: > "$SUMMARY_LOG"
+
+	start_epoch="$(date +%s)"
+
+	log_summary "Starting Ceph reset stress test"
+	log_summary "Profile=$PROFILE duration=${DURATION_SEC}s cooldown=${COOLDOWN_SEC}s file_count=${FILE_COUNT} io_workers=${IO_WORKERS} rename_workers=${RENAME_WORKERS}"
+	[[ -n "$SUDO" ]] && log_summary "Using sudo for privileged operations"
+	[[ -z "$DMESG_CMD" ]] && log_summary "WARNING: dmesg not available; hung task detection disabled"
+	log_summary "Artifacts=$OUT_DIR"
+	log_summary "Data dir=$DATA_DIR"
+
+	init_dataset
+
+	if [[ "$EXPECT_RESET" -eq 1 ]]; then
+		discover_client_id
+		trigger_path="$DEBUGFS_ROOT/$CLIENT_ID/reset/trigger"
+		status_path="$DEBUGFS_ROOT/$CLIENT_ID/reset/status"
+		if ! $SUDO test -w "$trigger_path"; then
+			echo "SKIP: Reset trigger is not writable: $trigger_path" >&2
+			exit "$KSFT_SKIP"
+		fi
+		if ! $SUDO test -r "$status_path"; then
+			echo "SKIP: Reset status is not readable: $status_path" >&2
+			exit "$KSFT_SKIP"
+		fi
+		$SUDO cat "$status_path" > "$STATUS_BEFORE" || true
+		reset_enabled=1
+		log_summary "Using ceph client id: $CLIENT_ID"
+	fi
+
+	trap cleanup EXIT INT TERM
+
+	touch "$WORKLOAD_FLAG"
+	for ((i = 0; i < IO_WORKERS; i++)); do
+		io_worker "$i" &
+		IO_WORKER_PIDS+=("$!")
+	done
+
+	for ((i = 0; i < RENAME_WORKERS; i++)); do
+		rename_worker "$i" &
+		RENAME_WORKER_PIDS+=("$!")
+	done
+
+	if [[ "$reset_enabled" -eq 1 ]]; then
+		touch "$RESET_FLAG"
+		reset_injector "$trigger_path" &
+		RESET_PID=$!
+
+		status_sampler "$status_path" &
+		STATUS_PID=$!
+	fi
+
+	sleep "$DURATION_SEC"
+
+	if [[ "$reset_enabled" -eq 1 ]]; then
+		rm -f "$RESET_FLAG"
+		stop_pid_with_timeout "$RESET_PID" "reset_injector" 20 || final_rc=1
+		log_summary "Injector stopped; entering cooldown=${COOLDOWN_SEC}s"
+	fi
+
+	sleep "$COOLDOWN_SEC"
+
+	rm -f "$WORKLOAD_FLAG"
+	for i in "${!IO_WORKER_PIDS[@]}"; do
+		stop_pid_with_timeout "${IO_WORKER_PIDS[$i]}" "io_worker[$i]" 20 || final_rc=1
+	done
+	for i in "${!RENAME_WORKER_PIDS[@]}"; do
+		stop_pid_with_timeout "${RENAME_WORKER_PIDS[$i]}" "rename_worker[$i]" 20 || final_rc=1
+	done
+
+	if [[ "$reset_enabled" -eq 1 ]]; then
+		stop_pid_with_timeout "$STATUS_PID" "status_sampler" 10 || final_rc=1
+		$SUDO cat "$status_path" > "$STATUS_FINAL" || true
+	fi
+
+	if ! check_dmesg "$start_epoch"; then
+		final_rc=1
+	fi
+
+	if ! python3 "$SCRIPT_DIR/validate_consistency.py" \
+		--data-dir "$DATA_DIR" \
+		--file-count "$FILE_COUNT" \
+		--io-log "$IO_LOG" \
+		--rename-log "$RENAME_LOG" \
+		--reset-log "$RESET_LOG" \
+		--status-final "$STATUS_FINAL" \
+		--slo-seconds "$SLO_SECONDS" \
+		--report-json "$REPORT_JSON" \
+		$( [[ "$reset_enabled" -eq 1 ]] && echo "--expect-reset" ); then
+		final_rc=1
+	fi
+
+	if [[ "$final_rc" -eq 0 ]]; then
+		log_summary "PASS: stress run completed successfully"
+	else
+		log_summary "FAIL: stress run detected one or more failures"
+	fi
+
+	log_summary "Artifacts available in: $OUT_DIR"
+	exit "$final_rc"
+}
+
+main "$@"
diff --git a/tools/testing/selftests/filesystems/ceph/run_validation.sh b/tools/testing/selftests/filesystems/ceph/run_validation.sh
new file mode 100755
index 0000000000000..5d521e4f9e9b2
--- /dev/null
+++ b/tools/testing/selftests/filesystems/ceph/run_validation.sh
@@ -0,0 +1,350 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# CephFS client reset - single-command validation.
+# Runs all test stages in sequence with per-stage timeouts.
+# If any stage hangs (filesystem stuck, process blocked), the
+# timeout kills it and reports failure.
+#
+# Usage:
+#   sudo ./run_validation.sh --mount-point /mnt/mycephfs
+#
+# Expected output on success:
+#
+#   === CephFS Client Reset Validation ===
+#   [stage 1/5] baseline         PASS  (60s, no resets)
+#   [stage 2/5] corner_cases     PASS  (4/4 passed)
+#   [stage 3/5] moderate         PASS  (120s, resets every 5-15s)
+#   [stage 4/5] aggressive       PASS  (120s, resets every 1-5s)
+#   [stage 5/5] status_check     PASS  (phase=idle, last_errno=0)
+#
+#   RESULT: 5/5 stages passed
+#   Artifacts: /tmp/ceph_reset_validation_<timestamp>
+
+set -uo pipefail
+
+KSFT_SKIP=4
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# kselftest auto-detect: when invoked with no arguments (e.g. by
+# "make run_tests"), find a CephFS mount automatically or skip.
+if [[ $# -eq 0 ]]; then
+	MOUNT_POINT="$(findmnt -t ceph -n -o TARGET 2>/dev/null | head -1)"
+	if [[ -z "$MOUNT_POINT" ]]; then
+		echo "SKIP: No CephFS mount found and --mount-point not specified"
+		exit "$KSFT_SKIP"
+	fi
+	exec "$0" --mount-point "$MOUNT_POINT"
+fi
+
+MOUNT_POINT=""
+CLIENT_ID=""
+declare -a CLIENT_ARGS=()
+declare -a DEBUGFS_ARGS=()
+RUN_ID="$(date +%Y%m%d-%H%M%S)"
+OUT_DIR="/tmp/ceph_reset_validation_${RUN_ID}"
+DEBUGFS_ROOT="/sys/kernel/debug/ceph"
+
+# Timeout margins: stage runtime + cooldown + validation + safety buffer
+STAGE1_TIMEOUT=120    # 60s run + 20s cooldown + 40s buffer
+STAGE2_TIMEOUT=300    # 4 corner cases, 30s each worst case + buffer
+STAGE3_TIMEOUT=240    # 120s run + 20s cooldown + 100s buffer
+STAGE4_TIMEOUT=240    # 120s run + 20s cooldown + 100s buffer
+STAGE5_TIMEOUT=10     # just reading debugfs
+
+PASS=0
+FAIL=0
+TOTAL=5
+
+usage()
+{
+	cat <<EOF
+Usage: $0 --mount-point <cephfs_mount> [options]
+
+Required:
+  --mount-point PATH    CephFS mount point
+
+Options:
+  --out-dir PATH        Artifact directory (default: /tmp/ceph_reset_validation_<ts>)
+  --client-id ID        Ceph debugfs client id (optional)
+  --debugfs-root PATH   Debugfs Ceph root (default: /sys/kernel/debug/ceph)
+  --help                Show this message
+EOF
+}
+
+stage_result()
+{
+	local num="$1"
+	local name="$2"
+	local status="$3"
+	local detail="$4"
+
+	if [[ "$status" == "PASS" ]]; then
+		PASS=$((PASS + 1))
+	else
+		FAIL=$((FAIL + 1))
+	fi
+	printf '[stage %d/%d] %-16s %s  (%s)\n' "$num" "$TOTAL" "$name" "$status" "$detail"
+}
+
+# Run a command with a timeout. Returns 0 on success, 1 on failure/timeout.
+# Sets RUN_TIMED_OUT=1 if killed by timeout.
+#
+# The stage command runs in its own session/process group (via setsid).
+# On timeout the entire process group is killed, not just the top-level
+# script PID.  This is required because stage scripts (reset_stress.sh,
+# reset_corner_cases.sh) spawn child processes - I/O workers, rename
+# workers, reset injectors, samplers - that would otherwise survive the
+# timeout and bleed into later stages, invalidating results.
+RUN_TIMED_OUT=0
+
+run_with_timeout()
+{
+	local timeout_sec="$1"
+	local logfile="$2"
+	shift 2
+
+	RUN_TIMED_OUT=0
+
+	# Start the stage in its own session via setsid so all descendant
+	# processes share a process group that we can kill atomically.
+	# In a non-interactive script, background children are not process
+	# group leaders, so setsid(1) calls setsid(2) directly (no extra
+	# fork) and the PID we capture IS the group leader.
+	setsid "$@" > "$logfile" 2>&1 &
+	local pid=$!
+
+	# Watchdog: on timeout, kill the entire process group
+	(
+		sleep "$timeout_sec"
+		if kill -0 "$pid" 2>/dev/null; then
+			echo "TIMEOUT: stage exceeded ${timeout_sec}s, killing process group $pid" >> "$logfile"
+			kill -TERM -- -"$pid" 2>/dev/null
+			sleep 2
+			kill -KILL -- -"$pid" 2>/dev/null
+		fi
+	) &
+	local watchdog_pid=$!
+
+	# Wait for the stage command
+	wait "$pid" 2>/dev/null
+	local rc=$?
+
+	# Kill the watchdog if it's still running
+	kill "$watchdog_pid" 2>/dev/null
+	wait "$watchdog_pid" 2>/dev/null
+
+	# Check if it was killed by timeout
+	if grep -q "^TIMEOUT:" "$logfile" 2>/dev/null; then
+		RUN_TIMED_OUT=1
+		return 1
+	fi
+
+	return "$rc"
+}
+
+find_status_path()
+{
+	local entry
+
+	if [[ -n "$CLIENT_ID" ]]; then
+		if [[ -r "$DEBUGFS_ROOT/$CLIENT_ID/reset/status" ]]; then
+			echo "$DEBUGFS_ROOT/$CLIENT_ID/reset/status"
+			return 0
+		fi
+		return 1
+	fi
+
+	for entry in "$DEBUGFS_ROOT"/*/; do
+		if [[ -r "${entry}reset/status" ]]; then
+			echo "${entry}reset/status"
+			return 0
+		fi
+	done
+	return 1
+}
+
+read_status_field()
+{
+	local status_path="$1"
+	local field="$2"
+	awk -F': ' -v key="$field" '$1 == key {print $2}' "$status_path" 2>/dev/null
+}
+
+# --- Parse arguments -------------------------------------------------------
+
+while [[ $# -gt 0 ]]; do
+	case "$1" in
+	--mount-point)  MOUNT_POINT="$2"; shift 2 ;;
+	--out-dir)      OUT_DIR="$2"; shift 2 ;;
+	--client-id)    CLIENT_ID="$2"; shift 2 ;;
+	--debugfs-root) DEBUGFS_ROOT="$2"; shift 2 ;;
+	--help|-h)      usage; exit 0 ;;
+	*)              echo "Unknown option: $1" >&2; usage; exit 2 ;;
+	esac
+done
+
+if [[ -z "$MOUNT_POINT" ]]; then
+	echo "SKIP: --mount-point is required" >&2
+	usage
+	exit "$KSFT_SKIP"
+fi
+
+if [[ ! -d "$MOUNT_POINT" ]]; then
+	echo "SKIP: Mount point does not exist: $MOUNT_POINT" >&2
+	exit "$KSFT_SKIP"
+fi
+
+# Auto-detect client id when not specified, so all stages (including
+# stage 5 status check) use the same client consistently.
+if [[ -z "$CLIENT_ID" ]]; then
+	candidates=()
+	for entry in "$DEBUGFS_ROOT"/*/; do
+		name="$(basename "$entry")"
+		if [[ -r "${entry}reset/status" ]]; then
+			candidates+=("$name")
+		fi
+	done
+	if [[ ${#candidates[@]} -eq 1 ]]; then
+		CLIENT_ID="${candidates[0]}"
+	elif [[ ${#candidates[@]} -gt 1 ]]; then
+		echo "SKIP: Multiple Ceph clients found (${candidates[*]}). Use --client-id." >&2
+		exit "$KSFT_SKIP"
+	fi
+fi
+
+if [[ -n "$CLIENT_ID" ]]; then
+	CLIENT_ARGS=(--client-id "$CLIENT_ID")
+fi
+DEBUGFS_ARGS=(--debugfs-root "$DEBUGFS_ROOT")
+
+# Quick sanity: can we write to the mount?
+if ! touch "$MOUNT_POINT/.validation_probe_$$" 2>/dev/null; then
+	echo "SKIP: Mount point is not writable: $MOUNT_POINT" >&2
+	exit "$KSFT_SKIP"
+fi
+rm -f "$MOUNT_POINT/.validation_probe_$$"
+
+mkdir -p "$OUT_DIR"
+
+echo ""
+echo "=== CephFS Client Reset Validation ==="
+echo ""
+
+# --- Stage 1: Baseline (no resets) -----------------------------------------
+
+stage1_out="$OUT_DIR/stage1_baseline"
+if run_with_timeout "$STAGE1_TIMEOUT" "$stage1_out.log" \
+	"$SCRIPT_DIR/reset_stress.sh" \
+	--mount-point "$MOUNT_POINT" \
+	--profile baseline \
+	--no-reset \
+	--duration-sec 60 \
+	"${CLIENT_ARGS[@]}" \
+	"${DEBUGFS_ARGS[@]}" \
+	--out-dir "$stage1_out"; then
+	stage_result 1 "baseline" "PASS" "60s, no resets"
+elif [[ "$RUN_TIMED_OUT" -eq 1 ]]; then
+	stage_result 1 "baseline" "FAIL" "HUNG: killed after ${STAGE1_TIMEOUT}s"
+else
+	stage_result 1 "baseline" "FAIL" "see $stage1_out.log"
+fi
+
+# --- Stage 2: Corner cases -------------------------------------------------
+
+stage2_out="$OUT_DIR/stage2_corner_cases"
+mkdir -p "$stage2_out"
+if run_with_timeout "$STAGE2_TIMEOUT" "$stage2_out/output.log" \
+	"$SCRIPT_DIR/reset_corner_cases.sh" \
+	"${CLIENT_ARGS[@]}" \
+	"${DEBUGFS_ARGS[@]}" \
+	--mount-point "$MOUNT_POINT"; then
+	pass_line=$(grep -Eo '[0-9]+ passed, [0-9]+ failed, [0-9]+ skipped' "$stage2_out/output.log" | tail -1)
+	stage_result 2 "corner_cases" "PASS" "${pass_line:-all tests passed}"
+elif [[ "$RUN_TIMED_OUT" -eq 1 ]]; then
+	stage_result 2 "corner_cases" "FAIL" "HUNG: killed after ${STAGE2_TIMEOUT}s"
+else
+	fail_line=$(grep -c 'FAIL' "$stage2_out/output.log" 2>/dev/null || echo "?")
+	stage_result 2 "corner_cases" "FAIL" "${fail_line} failures, see $stage2_out/output.log"
+fi
+
+# --- Stage 3: Moderate resets -----------------------------------------------
+
+stage3_out="$OUT_DIR/stage3_moderate"
+if run_with_timeout "$STAGE3_TIMEOUT" "$stage3_out.log" \
+	"$SCRIPT_DIR/reset_stress.sh" \
+	--mount-point "$MOUNT_POINT" \
+	--profile moderate \
+	--duration-sec 120 \
+	"${CLIENT_ARGS[@]}" \
+	"${DEBUGFS_ARGS[@]}" \
+	--out-dir "$stage3_out"; then
+	stage_result 3 "moderate" "PASS" "120s, resets every 5-15s"
+elif [[ "$RUN_TIMED_OUT" -eq 1 ]]; then
+	stage_result 3 "moderate" "FAIL" "HUNG: killed after ${STAGE3_TIMEOUT}s"
+else
+	stage_result 3 "moderate" "FAIL" "see $stage3_out.log"
+fi
+
+# --- Stage 4: Aggressive resets ---------------------------------------------
+
+stage4_out="$OUT_DIR/stage4_aggressive"
+if run_with_timeout "$STAGE4_TIMEOUT" "$stage4_out.log" \
+	"$SCRIPT_DIR/reset_stress.sh" \
+	--mount-point "$MOUNT_POINT" \
+	--profile aggressive \
+	--duration-sec 120 \
+	"${CLIENT_ARGS[@]}" \
+	"${DEBUGFS_ARGS[@]}" \
+	--out-dir "$stage4_out"; then
+	stage_result 4 "aggressive" "PASS" "120s, resets every 1-5s"
+elif [[ "$RUN_TIMED_OUT" -eq 1 ]]; then
+	stage_result 4 "aggressive" "FAIL" "HUNG: killed after ${STAGE4_TIMEOUT}s"
+else
+	stage_result 4 "aggressive" "FAIL" "see $stage4_out.log"
+fi
+
+# --- Stage 5: Post-run status check ----------------------------------------
+
+status_path=""
+if status_path=$(find_status_path); then
+	phase=$(read_status_field "$status_path" "phase")
+	last_errno=$(read_status_field "$status_path" "last_errno")
+	failure_count=$(read_status_field "$status_path" "failure_count")
+	drain_timed_out=$(read_status_field "$status_path" "drain_timed_out")
+	sessions_reset=$(read_status_field "$status_path" "sessions_reset")
+	blocked=$(read_status_field "$status_path" "blocked_requests")
+
+	# Save full status
+	cat "$status_path" > "$OUT_DIR/final_status.txt" 2>/dev/null
+
+	errors=""
+	[[ "$phase" != "idle" ]] && errors="${errors}phase=$phase "
+	[[ "$last_errno" != "0" ]] && errors="${errors}last_errno=$last_errno "
+	[[ "$failure_count" != "0" && -n "$failure_count" ]] && errors="${errors}failure_count=$failure_count "
+	[[ "$blocked" != "0" ]] && errors="${errors}blocked_requests=$blocked "
+
+	if [[ -z "$errors" ]]; then
+		detail="phase=$phase, last_errno=$last_errno, failure_count=${failure_count:-0}"
+		[[ "$drain_timed_out" == "yes" ]] && detail="$detail, drain_timed_out=yes"
+		[[ -n "$sessions_reset" ]] && detail="$detail, sessions_reset=$sessions_reset"
+		stage_result 5 "status_check" "PASS" "$detail"
+	else
+		stage_result 5 "status_check" "FAIL" "$errors"
+	fi
+else
+	stage_result 5 "status_check" "FAIL" "cannot read reset/status"
+fi
+
+# --- Summary ----------------------------------------------------------------
+
+echo ""
+if [[ "$FAIL" -eq 0 ]]; then
+	echo "RESULT: $PASS/$TOTAL stages passed"
+else
+	echo "RESULT: $PASS/$TOTAL stages passed, $FAIL FAILED"
+fi
+echo "Artifacts: $OUT_DIR"
+echo ""
+
+exit "$FAIL"
diff --git a/tools/testing/selftests/filesystems/ceph/settings b/tools/testing/selftests/filesystems/ceph/settings
new file mode 100644
index 0000000000000..79b65bdf05db6
--- /dev/null
+++ b/tools/testing/selftests/filesystems/ceph/settings
@@ -0,0 +1 @@
+timeout=1200
diff --git a/tools/testing/selftests/filesystems/ceph/validate_consistency.py b/tools/testing/selftests/filesystems/ceph/validate_consistency.py
new file mode 100755
index 0000000000000..c230a59bdb3a4
--- /dev/null
+++ b/tools/testing/selftests/filesystems/ceph/validate_consistency.py
@@ -0,0 +1,297 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+
+import argparse
+import bisect
+import hashlib
+import json
+import os
+from pathlib import Path
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        while True:
+            chunk = handle.read(1 << 20)
+            if not chunk:
+                break
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def parse_io_log(path: Path):
+    records = []
+    if not path.exists():
+        return records
+    with path.open("r", encoding="utf-8") as handle:
+        for line_no, line in enumerate(handle, 1):
+            line = line.strip()
+            if not line:
+                continue
+            parts = line.split(",")
+            if len(parts) != 5:
+                raise ValueError(f"io log line {line_no}: expected 5 columns, got {len(parts)}")
+            ts_ms, seq, logical_id, relpath, digest = parts
+            records.append(
+                {
+                    "ts_ms": int(ts_ms),
+                    "seq": int(seq),
+                    "logical_id": int(logical_id),
+                    "relpath": relpath,
+                    "digest": digest,
+                }
+            )
+    return records
+
+
+def parse_rename_log(path: Path):
+    records = []
+    if not path.exists():
+        return records
+    with path.open("r", encoding="utf-8") as handle:
+        for line_no, line in enumerate(handle, 1):
+            line = line.strip()
+            if not line:
+                continue
+            parts = line.split(",")
+            if len(parts) == 6:
+                ts_ms, seq, logical_id, src_rel, dst_rel, rc = parts
+            elif len(parts) == 7:
+                ts_ms, worker_id, seq, logical_id, src_rel, dst_rel, rc = parts
+                _ = worker_id  # worker id is informational only
+            else:
+                raise ValueError(
+                    f"rename log line {line_no}: expected 6 or 7 columns, got {len(parts)}"
+                )
+            records.append(
+                {
+                    "ts_ms": int(ts_ms),
+                    "seq": int(seq),
+                    "logical_id": int(logical_id),
+                    "src_rel": src_rel,
+                    "dst_rel": dst_rel,
+                    "rc": int(rc),
+                }
+            )
+    return records
+
+
+def parse_reset_log(path: Path):
+    records = []
+    if not path.exists():
+        return records
+    with path.open("r", encoding="utf-8") as handle:
+        for line_no, line in enumerate(handle, 1):
+            line = line.strip()
+            if not line:
+                continue
+            parts = line.split(",")
+            if len(parts) != 4:
+                raise ValueError(f"reset log line {line_no}: expected 4 columns, got {len(parts)}")
+            ts_ms, seq, reason, rc = parts
+            records.append(
+                {
+                    "ts_ms": int(ts_ms),
+                    "seq": int(seq),
+                    "reason": reason,
+                    "rc": int(rc),
+                }
+            )
+    return records
+
+
+def parse_status_file(path: Path):
+    status = {}
+    if not path.exists():
+        return status
+    with path.open("r", encoding="utf-8") as handle:
+        for line in handle:
+            line = line.strip()
+            if not line or ":" not in line:
+                continue
+            key, value = line.split(":", 1)
+            status[key.strip()] = value.strip()
+    return status
+
+
+def to_int(value: str, default: int = 0):
+    try:
+        return int(value)
+    except Exception:
+        return default
+
+
+def validate_namespace(data_dir: Path, file_count: int, issues):
+    actual_locations = {}
+    actual_paths = {}
+    for logical_id in range(file_count):
+        name = f"file_{logical_id:05d}"
+        found = []
+        for subdir in ("A", "B"):
+            candidate = data_dir / subdir / name
+            if candidate.exists():
+                found.append((subdir, candidate))
+        if len(found) != 1:
+            issues.append(
+                f"namespace invariant failed for logical_id={logical_id:05d}: expected exactly one file in A/B, found {len(found)}"
+            )
+            continue
+        actual_locations[logical_id] = found[0][0]
+        actual_paths[logical_id] = found[0][1]
+    return actual_locations, actual_paths
+
+
+def validate_rename_invariant(rename_records, actual_locations, issues):
+    expected_locations = {}
+    for rec in rename_records:
+        if rec["rc"] != 0:
+            continue
+        dst = rec["dst_rel"]
+        if "/" not in dst:
+            continue
+        expected_locations[rec["logical_id"]] = dst.split("/", 1)[0]
+
+    for logical_id, expected in expected_locations.items():
+        actual = actual_locations.get(logical_id)
+        if actual is None:
+            continue
+        if actual != expected:
+            issues.append(
+                f"rename invariant failed for logical_id={logical_id:05d}: expected location={expected}, actual={actual}"
+            )
+
+
+def validate_data_invariant(io_records, actual_paths, issues):
+    expected_hash = {}
+    for rec in io_records:
+        digest = rec["digest"]
+        if not digest:
+            continue
+        expected_hash[rec["logical_id"]] = digest
+
+    for logical_id, digest in expected_hash.items():
+        path = actual_paths.get(logical_id)
+        if path is None:
+            continue
+        actual_digest = sha256_file(path)
+        if digest != actual_digest:
+            issues.append(
+                f"data invariant failed for logical_id={logical_id:05d}: expected digest={digest}, actual digest={actual_digest}"
+            )
+
+
+def validate_reset_and_slo(args, reset_records, io_records, rename_records, status, issues):
+    if not args.expect_reset:
+        return
+
+    successful_reset_times = [rec["ts_ms"] for rec in reset_records if rec["rc"] == 0]
+    if not successful_reset_times:
+        issues.append("expected reset activity but no successful reset trigger was observed")
+
+    phase = status.get("phase")
+    blocked_requests = to_int(status.get("blocked_requests", "0"), default=-1)
+    last_errno = to_int(status.get("last_errno", "0"), default=1)
+    failure_count = to_int(status.get("failure_count", "0"), default=-1)
+
+    if phase is None:
+        issues.append("missing final reset status file or phase field")
+    elif phase.lower() != "idle":
+        issues.append(f"recovery invariant failed: phase={phase}, expected idle")
+
+    if blocked_requests != 0:
+        issues.append(f"recovery invariant failed: blocked_requests={blocked_requests}, expected 0")
+    if last_errno != 0:
+        issues.append(f"recovery invariant failed: last_errno={last_errno}, expected 0")
+    if failure_count > 0:
+        issues.append(
+            f"recovery invariant failed: failure_count={failure_count}, "
+            "one or more resets failed during the run"
+        )
+
+    op_times = [rec["ts_ms"] for rec in io_records]
+    op_times.extend(rec["ts_ms"] for rec in rename_records if rec["rc"] == 0)
+    op_times.sort()
+
+    if successful_reset_times and not op_times:
+        issues.append("recovery SLO failed: no workload completion events were recorded")
+        return
+
+    slo_ms = args.slo_seconds * 1000
+    for ts in successful_reset_times:
+        idx = bisect.bisect_left(op_times, ts)
+        if idx >= len(op_times):
+            issues.append(f"recovery SLO failed: no operation completion observed after reset at ts_ms={ts}")
+            continue
+        delta = op_times[idx] - ts
+        if delta > slo_ms:
+            issues.append(
+                f"recovery SLO failed: first post-reset completion at {delta}ms exceeds threshold {slo_ms}ms (reset ts_ms={ts})"
+            )
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Validate Ceph reset stress artifacts")
+    parser.add_argument("--data-dir", required=True)
+    parser.add_argument("--file-count", required=True, type=int)
+    parser.add_argument("--io-log", required=True)
+    parser.add_argument("--rename-log", required=True)
+    parser.add_argument("--reset-log", required=True)
+    parser.add_argument("--status-final", required=False, default="")
+    parser.add_argument("--slo-seconds", required=False, type=int, default=30)
+    parser.add_argument("--expect-reset", action="store_true")
+    parser.add_argument("--report-json", required=False, default="")
+    args = parser.parse_args()
+
+    data_dir = Path(args.data_dir)
+    io_log = Path(args.io_log)
+    rename_log = Path(args.rename_log)
+    reset_log = Path(args.reset_log)
+    status_final = Path(args.status_final) if args.status_final else Path("__missing_status__")
+
+    issues = []
+
+    if not data_dir.exists():
+        issues.append(f"data directory is missing: {data_dir}")
+
+    try:
+        io_records = parse_io_log(io_log)
+        rename_records = parse_rename_log(rename_log)
+        reset_records = parse_reset_log(reset_log)
+    except Exception as exc:
+        issues.append(f"log parsing failed: {exc}")
+        io_records = []
+        rename_records = []
+        reset_records = []
+
+    status = parse_status_file(status_final)
+
+    actual_locations, actual_paths = validate_namespace(data_dir, args.file_count, issues)
+    validate_rename_invariant(rename_records, actual_locations, issues)
+    validate_data_invariant(io_records, actual_paths, issues)
+    validate_reset_and_slo(args, reset_records, io_records, rename_records, status, issues)
+
+    report = {
+        "file_count": args.file_count,
+        "io_records": len(io_records),
+        "rename_records": len(rename_records),
+        "reset_records": len(reset_records),
+        "expect_reset": args.expect_reset,
+        "issues": issues,
+    }
+
+    if args.report_json:
+        report_path = Path(args.report_json)
+        report_path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
+
+    if issues:
+        print("FAIL: consistency validation found issues")
+        for issue in issues:
+            print(f"  - {issue}")
+        raise SystemExit(1)
+
+    print("PASS: consistency validation succeeded")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/testing/selftests/filesystems/fclog.c b/tools/testing/selftests/filesystems/fclog.c
index 551c4a0f395a1..593a5136e9919 100644
--- a/tools/testing/selftests/filesystems/fclog.c
+++ b/tools/testing/selftests/filesystems/fclog.c
@@ -6,10 +6,8 @@
 
 #include <assert.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <sched.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
 #include <unistd.h>
 #include <sys/mount.h>
 
diff --git a/tools/testing/selftests/filesystems/overlayfs/.gitignore b/tools/testing/selftests/filesystems/overlayfs/.gitignore
index e23a18c8b37f2..077f7a1281687 100644
--- a/tools/testing/selftests/filesystems/overlayfs/.gitignore
+++ b/tools/testing/selftests/filesystems/overlayfs/.gitignore
@@ -1,3 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0-only
 dev_in_maps
 set_layers_via_fds
+idmapped_mounts
diff --git a/tools/testing/selftests/filesystems/overlayfs/Makefile b/tools/testing/selftests/filesystems/overlayfs/Makefile
index d3ad4a77db9ba..b3185f684add6 100644
--- a/tools/testing/selftests/filesystems/overlayfs/Makefile
+++ b/tools/testing/selftests/filesystems/overlayfs/Makefile
@@ -8,7 +8,9 @@ LOCAL_HDRS += ../wrappers.h log.h
 
 TEST_GEN_PROGS := dev_in_maps
 TEST_GEN_PROGS += set_layers_via_fds
+TEST_GEN_PROGS += idmapped_mounts
 
 include ../../lib.mk
 
 $(OUTPUT)/set_layers_via_fds: ../utils.c
+$(OUTPUT)/idmapped_mounts: ../utils.c
diff --git a/tools/testing/selftests/filesystems/overlayfs/idmapped_mounts.c b/tools/testing/selftests/filesystems/overlayfs/idmapped_mounts.c
new file mode 100644
index 0000000000000..44a75839f4ed4
--- /dev/null
+++ b/tools/testing/selftests/filesystems/overlayfs/idmapped_mounts.c
@@ -0,0 +1,501 @@
+// SPDX-License-Identifier: GPL-2.0
+#define _GNU_SOURCE
+
+#include <fcntl.h>
+#include <limits.h>
+#include <sched.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+
+#include <linux/mount.h>
+#include <linux/types.h>
+
+#include "kselftest_harness.h"
+#include "../wrappers.h"
+#include "../utils.h"
+
+/*
+ * An idmapping that maps the mount-visible id range [0, ID_RANGE) onto the
+ * host/overlay-final id range [ID_HOST, ID_HOST + ID_RANGE).  Through such an
+ * idmapped overlay mount, an overlay-final id of ID_HOST + n is reported as n,
+ * and an id of n requested through the mount is stored as ID_HOST + n.
+ */
+#define ID_NS	 0
+#define ID_HOST	 10000
+#define ID_RANGE 10000
+
+/*
+ * For the composition test the lower layer's on-disk ids live in a
+ * separate range and are mapped by an idmapped lower layer onto the
+ * overlay-final range [ID_HOST, ID_HOST + ID_RANGE).
+ */
+#define LAYER_HOST 20000
+
+#ifndef MOUNT_ATTR_IDMAP
+#define MOUNT_ATTR_IDMAP 0x00100000
+#endif
+
+#ifndef __NR_mount_setattr
+#define __NR_mount_setattr 442
+#endif
+
+static inline int sys_mount_setattr(int dfd, const char *path,
+				    unsigned int flags,
+				    struct mount_attr *attr, size_t size)
+{
+	return syscall(__NR_mount_setattr, dfd, path, flags, attr, size);
+}
+
+static bool ovl_supported(void)
+{
+	int fd = sys_fsopen("overlay", 0);
+
+	if (fd < 0)
+		return false;
+	close(fd);
+	return true;
+}
+
+/* base/{l,u,w} owned by ID_HOST so they map to ID_NS through the idmap. */
+static int setup_layers(const char *base)
+{
+	static const char *sub[] = { "", "/l", "/u", "/w" };
+	char path[PATH_MAX];
+
+	for (size_t i = 0; i < ARRAY_SIZE(sub); i++) {
+		snprintf(path, sizeof(path), "%s%s", base, sub[i]);
+		if (mkdir(path, 0755) && errno != EEXIST)
+			return -1;
+		if (i && chown(path, ID_HOST, ID_HOST))
+			return -1;
+	}
+	return 0;
+}
+
+static int ovl_mount(const char *base, bool nfs_export)
+{
+	char lower[PATH_MAX], upper[PATH_MAX], work[PATH_MAX];
+	int fsfd, ovl;
+
+	snprintf(lower, sizeof(lower), "%s/l", base);
+	snprintf(upper, sizeof(upper), "%s/u", base);
+	snprintf(work, sizeof(work), "%s/w", base);
+
+	fsfd = sys_fsopen("overlay", 0);
+	if (fsfd < 0)
+		return -1;
+
+	if (sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "source", "test", 0) ||
+	    sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "lowerdir", lower, 0) ||
+	    sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "upperdir", upper, 0) ||
+	    sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "workdir", work, 0))
+		goto err;
+	if (nfs_export &&
+	    (sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "index", "on", 0) ||
+	     sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "nfs_export", "on", 0)))
+		goto err;
+	if (sys_fsconfig(fsfd, FSCONFIG_CMD_CREATE, NULL, NULL, 0))
+		goto err;
+
+	ovl = sys_fsmount(fsfd, 0, 0);
+	close(fsfd);
+	return ovl;
+err:
+	close(fsfd);
+	return -1;
+}
+
+/* Idmap the (still detached, not yet visible) overlay mount @mfd. */
+static int ovl_idmap(int mfd)
+{
+	struct mount_attr attr = {
+		.attr_set = MOUNT_ATTR_IDMAP,
+	};
+	int ret, userns_fd;
+
+	/*
+	 * get_userns_fd(fs_id, mount_id, range): a file whose filesystem id
+	 * is fs_id + n is shown through the idmapped mount as mount_id + n.
+	 * Here the overlay-final (fs side) range is [ID_HOST, ..) and the
+	 * caller-visible (mount side) range is [ID_NS, ..).
+	 */
+	userns_fd = get_userns_fd(ID_HOST, ID_NS, ID_RANGE);
+	if (userns_fd < 0)
+		return -1;
+
+	attr.userns_fd = userns_fd;
+	ret = sys_mount_setattr(mfd, "", AT_EMPTY_PATH, &attr, sizeof(attr));
+	close(userns_fd);
+	return ret;
+}
+
+/* Clone @path into a detached, idmapped mount usable as an overlay layer. */
+static int idmapped_layer_fd(const char *path, int nsid, int hostid, int range)
+{
+	struct mount_attr attr = {
+		.attr_set = MOUNT_ATTR_IDMAP,
+	};
+	int fd_tree, userns_fd;
+
+	fd_tree = sys_open_tree(AT_FDCWD, path,
+			       OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
+	if (fd_tree < 0)
+		return -1;
+	userns_fd = get_userns_fd(nsid, hostid, range);
+	if (userns_fd < 0) {
+		close(fd_tree);
+		return -1;
+	}
+	attr.userns_fd = userns_fd;
+	if (sys_mount_setattr(fd_tree, "", AT_EMPTY_PATH, &attr,
+			      sizeof(attr))) {
+		close(userns_fd);
+		close(fd_tree);
+		return -1;
+	}
+	close(userns_fd);
+	return fd_tree;
+}
+
+/* Overlay with a layer passed by fd (idmapped) plus a plain upper/work. */
+static int ovl_mount_lower_fd(const char *upper, const char *work, int fd_lower)
+{
+	int fsfd, ovl;
+
+	fsfd = sys_fsopen("overlay", 0);
+	if (fsfd < 0)
+		return -1;
+
+	if (sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "source", "test", 0) ||
+	    sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "upperdir", upper, 0) ||
+	    sys_fsconfig(fsfd, FSCONFIG_SET_STRING, "workdir", work, 0) ||
+	    sys_fsconfig(fsfd, FSCONFIG_SET_FD, "lowerdir+", NULL, fd_lower) ||
+	    sys_fsconfig(fsfd, FSCONFIG_CMD_CREATE, NULL, NULL, 0))
+		goto err;
+
+	ovl = sys_fsmount(fsfd, 0, 0);
+	close(fsfd);
+	return ovl;
+err:
+	close(fsfd);
+	return -1;
+}
+
+/*
+ * Mount an overlay inside user namespace @u1 (so the overlay sb's s_user_ns is
+ * not the initial namespace) and idmap that overlay mount with @u2.  Runs in a
+ * child that joins @u1; returns 0 on success.
+ */
+static int userns_overlay_child(int u1)
+{
+	struct mount_attr attr = {
+		.attr_set = MOUNT_ATTR_IDMAP,
+	};
+	struct stat st;
+	int ovl, u2;
+
+	/* Become root in the overlay sb's user namespace u1. */
+	if (!switch_userns(u1, 0, 0, false))
+		return fprintf(stderr, "userns: switch_userns: %m\n"), -1;
+	if (unshare(CLONE_NEWNS) ||
+	    sys_mount(NULL, "/", NULL, MS_SLAVE | MS_REC, NULL))
+		return fprintf(stderr, "userns: unshare/slave: %m\n"), -1;
+	if (sys_mount("tmpfs", "/tmp", "tmpfs", 0, NULL))
+		return fprintf(stderr, "userns: mount tmpfs: %m\n"), -1;
+	if (setup_layers("/tmp/ovl"))
+		return fprintf(stderr, "userns: setup_layers: %m\n"), -1;
+	if (mknod("/tmp/ovl/l/file", S_IFREG | 0644, 0) ||
+	    chown("/tmp/ovl/l/file", ID_HOST + 5, ID_HOST + 5))
+		return fprintf(stderr, "userns: lower file: %m\n"), -1;
+
+	ovl = ovl_mount("/tmp/ovl", false);
+	if (ovl < 0)
+		return fprintf(stderr, "userns: ovl_mount: %m\n"), -1;
+
+	/*
+	 * mount_setattr() requires CAP_SYS_ADMIN over the idmap user
+	 * namespace, so it must be a child of u1.  Create it now, from
+	 * inside u1.
+	 */
+	u2 = get_userns_fd(ID_HOST, ID_NS, ID_RANGE);
+	if (u2 < 0)
+		return fprintf(stderr, "userns: get_userns_fd: %m\n"), -1;
+	attr.userns_fd = u2;
+	if (sys_mount_setattr(ovl, "", AT_EMPTY_PATH, &attr, sizeof(attr)))
+		return fprintf(stderr, "userns: mount_setattr: %m\n"), -1;
+	close(u2);
+
+	if (fstatat(ovl, "file", &st, 0))
+		return fprintf(stderr, "userns: fstatat: %m\n"), -1;
+	if (st.st_uid != ID_NS + 5 || st.st_gid != ID_NS + 5) {
+		fprintf(stderr, "userns: got %u:%u expected %u:%u\n",
+			st.st_uid, st.st_gid, ID_NS + 5, ID_NS + 5);
+		return -1;
+	}
+	return 0;
+}
+
+FIXTURE(idmapped_overlay) {
+	char base[64];
+};
+
+FIXTURE_SETUP(idmapped_overlay)
+{
+	/* Private mount namespace so test mounts need no cleanup. */
+	ASSERT_EQ(unshare(CLONE_NEWNS), 0);
+	ASSERT_EQ(sys_mount(NULL, "/", NULL, MS_SLAVE | MS_REC, NULL), 0);
+
+	/* tmpfs for the layers so we can chown them to arbitrary ids. */
+	ASSERT_EQ(sys_mount("tmpfs", "/tmp", "tmpfs", 0, NULL), 0);
+
+	snprintf(self->base, sizeof(self->base), "/tmp/ovl");
+	ASSERT_EQ(setup_layers(self->base), 0);
+}
+
+FIXTURE_TEARDOWN(idmapped_overlay)
+{
+}
+
+/* A file owned by ID_HOST + 5 is reported as ID_NS + 5 through the idmap. */
+TEST_F(idmapped_overlay, getattr)
+{
+	char path[PATH_MAX];
+	struct stat st;
+	int ovl;
+
+	if (!ovl_supported())
+		SKIP(return, "overlayfs not supported");
+
+	snprintf(path, sizeof(path), "%s/l/file", self->base);
+	ASSERT_EQ(mknod(path, S_IFREG | 0644, 0), 0);
+	ASSERT_EQ(chown(path, ID_HOST + 5, ID_HOST + 5), 0);
+
+	ovl = ovl_mount(self->base, false);
+	ASSERT_GE(ovl, 0);
+	ASSERT_EQ(ovl_idmap(ovl), 0);
+
+	ASSERT_EQ(fstatat(ovl, "file", &st, 0), 0);
+	EXPECT_EQ(st.st_uid, ID_NS + 5);
+	EXPECT_EQ(st.st_gid, ID_NS + 5);
+
+	EXPECT_EQ(close(ovl), 0);
+}
+
+/*
+ * Every creation path initializes the new owner through the mount idmap:
+ * created as caller id ID_NS, stored on the upper layer as overlay-final
+ * ID_HOST.  Covers ovl_create() (regular file), ovl_mkdir(), ovl_mknod()
+ * and ovl_symlink() (which share ovl_create_object()), plus the separate
+ * ovl_tmpfile() path.
+ */
+TEST_F(idmapped_overlay, create)
+{
+	static const char *names[] = { "reg", "dir", "fifo", "lnk" };
+	char path[PATH_MAX];
+	struct stat st;
+	int ovl, fd;
+
+	if (!ovl_supported())
+		SKIP(return, "overlayfs not supported");
+
+	ovl = ovl_mount(self->base, false);
+	ASSERT_GE(ovl, 0);
+	ASSERT_EQ(ovl_idmap(ovl), 0);
+
+	/* One object per creation operation, all as caller id ID_NS. */
+	fd = openat(ovl, "reg", O_CREAT | O_WRONLY | O_EXCL, 0644);
+	ASSERT_GE(fd, 0);
+	EXPECT_EQ(close(fd), 0);
+	ASSERT_EQ(mkdirat(ovl, "dir", 0755), 0);
+	ASSERT_EQ(mknodat(ovl, "fifo", S_IFIFO | 0644, 0), 0);
+	ASSERT_EQ(symlinkat("target", ovl, "lnk"), 0);
+
+	for (size_t i = 0; i < ARRAY_SIZE(names); i++) {
+		/* Reported as ID_NS through the idmapped mount ... */
+		ASSERT_EQ(fstatat(ovl, names[i], &st, AT_SYMLINK_NOFOLLOW), 0);
+		EXPECT_EQ(st.st_uid, ID_NS);
+		EXPECT_EQ(st.st_gid, ID_NS);
+		/* ... and stored as ID_HOST on the upper layer. */
+		snprintf(path, sizeof(path), "%s/u/%s", self->base, names[i]);
+		ASSERT_EQ(lstat(path, &st), 0);
+		EXPECT_EQ(st.st_uid, ID_HOST);
+		EXPECT_EQ(st.st_gid, ID_HOST);
+	}
+
+	/* O_TMPFILE goes through the separate ovl_tmpfile() path. */
+	fd = openat(ovl, ".", O_TMPFILE | O_WRONLY, 0644);
+	ASSERT_GE(fd, 0);
+	/* Inside the mount: caller id ID_NS. */
+	ASSERT_EQ(fstat(fd, &st), 0);
+	EXPECT_EQ(st.st_uid, ID_NS);
+	EXPECT_EQ(st.st_gid, ID_NS);
+	/* Link it in so the upper backing file can be inspected too. */
+	ASSERT_EQ(linkat(fd, "", ovl, "tmp", AT_EMPTY_PATH), 0);
+	EXPECT_EQ(close(fd), 0);
+	snprintf(path, sizeof(path), "%s/u/tmp", self->base);
+	ASSERT_EQ(lstat(path, &st), 0);
+	EXPECT_EQ(st.st_uid, ID_HOST);
+	EXPECT_EQ(st.st_gid, ID_HOST);
+
+	EXPECT_EQ(close(ovl), 0);
+}
+
+/* chown through the idmapped mount round-trips: ID_NS + 5 <-> ID_HOST + 5. */
+TEST_F(idmapped_overlay, chown)
+{
+	char path[PATH_MAX];
+	struct stat st;
+	int ovl, fd;
+
+	if (!ovl_supported())
+		SKIP(return, "overlayfs not supported");
+
+	ovl = ovl_mount(self->base, false);
+	ASSERT_GE(ovl, 0);
+	ASSERT_EQ(ovl_idmap(ovl), 0);
+
+	fd = openat(ovl, "f", O_CREAT | O_WRONLY | O_EXCL, 0644);
+	ASSERT_GE(fd, 0);
+	EXPECT_EQ(close(fd), 0);
+
+	ASSERT_EQ(fchownat(ovl, "f", ID_NS + 5, ID_NS + 5, 0), 0);
+
+	ASSERT_EQ(fstatat(ovl, "f", &st, 0), 0);
+	EXPECT_EQ(st.st_uid, ID_NS + 5);
+	EXPECT_EQ(st.st_gid, ID_NS + 5);
+
+	snprintf(path, sizeof(path), "%s/u/f", self->base);
+	ASSERT_EQ(stat(path, &st), 0);
+	EXPECT_EQ(st.st_uid, ID_HOST + 5);
+	EXPECT_EQ(st.st_gid, ID_HOST + 5);
+
+	EXPECT_EQ(close(ovl), 0);
+}
+
+/*
+ * Composition: an idmapped lower layer underneath an idmapped overlay mount.
+ * An on-disk id is mapped by the layer idmap into the overlay-final range and
+ * then by the mount idmap into the caller's range:
+ *
+ *   on-disk LAYER_HOST+7  --layer-->  ID_HOST+7  --mount-->  ID_NS+7
+ */
+TEST_F(idmapped_overlay, composition)
+{
+	char lower[PATH_MAX], upper[PATH_MAX], work[PATH_MAX], path[PATH_MAX];
+	struct stat st;
+	int ovl, fd_lower;
+
+	if (!ovl_supported())
+		SKIP(return, "overlayfs not supported");
+
+	snprintf(lower, sizeof(lower), "%s/l", self->base);
+	snprintf(upper, sizeof(upper), "%s/u", self->base);
+	snprintf(work, sizeof(work), "%s/w", self->base);
+
+	/* Put the lower layer's ids in the on-disk [LAYER_HOST, ..) range. */
+	ASSERT_EQ(chown(lower, LAYER_HOST, LAYER_HOST), 0);
+	snprintf(path, sizeof(path), "%s/l/file", self->base);
+	ASSERT_EQ(mknod(path, S_IFREG | 0644, 0), 0);
+	ASSERT_EQ(chown(path, LAYER_HOST + 7, LAYER_HOST + 7), 0);
+
+	/* Idmapped lower: on-disk LAYER_HOST <-> overlay-final ID_HOST. */
+	fd_lower = idmapped_layer_fd(lower, LAYER_HOST, ID_HOST, ID_RANGE);
+	ASSERT_GE(fd_lower, 0);
+
+	ovl = ovl_mount_lower_fd(upper, work, fd_lower);
+	ASSERT_GE(ovl, 0);
+	EXPECT_EQ(close(fd_lower), 0);
+
+	/* Idmap the overlay mount: overlay-final ID_HOST <-> caller ID_NS. */
+	ASSERT_EQ(ovl_idmap(ovl), 0);
+
+	ASSERT_EQ(fstatat(ovl, "file", &st, 0), 0);
+	EXPECT_EQ(st.st_uid, ID_NS + 7);
+	EXPECT_EQ(st.st_gid, ID_NS + 7);
+
+	EXPECT_EQ(close(ovl), 0);
+}
+
+/* An idmapped overlay mount whose sb lives inside a user namespace. */
+TEST_F(idmapped_overlay, userns)
+{
+	int u1;
+	pid_t pid;
+
+	if (!ovl_supported())
+		SKIP(return, "overlayfs not supported");
+
+	/* u1 backs the overlay sb: identity-mapped, but not the init ns. */
+	u1 = get_userns_fd(0, 0, 65536);
+	if (u1 < 0)
+		SKIP(return, "user namespaces not available");
+
+	pid = fork();
+	ASSERT_GE(pid, 0);
+	if (pid == 0) {
+		int ret = userns_overlay_child(u1);
+
+		_exit(ret ? EXIT_FAILURE : EXIT_SUCCESS);
+	}
+	EXPECT_EQ(wait_for_pid(pid), 0);
+
+	EXPECT_EQ(close(u1), 0);
+}
+
+/*
+ * An nfs_export overlay can be idmapped, and decodable file handles round-trip
+ * through the idmapped mount with correctly mapped ownership.  Overlay file
+ * handles encode object identity, not ownership, so the mount idmap does not
+ * affect them; it only maps the owner reported once a handle is reopened.
+ */
+TEST_F(idmapped_overlay, nfs_export_handles)
+{
+	char path[PATH_MAX], mnt[128];
+	union {
+		struct file_handle fh;
+		char buf[sizeof(struct file_handle) + MAX_HANDLE_SZ];
+	} fhu;
+	struct file_handle *fh = &fhu.fh;
+	struct stat st;
+	int ovl, mfd, fd, mount_id;
+
+	if (!ovl_supported())
+		SKIP(return, "overlayfs not supported");
+
+	snprintf(path, sizeof(path), "%s/l/file", self->base);
+	ASSERT_EQ(mknod(path, S_IFREG | 0644, 0), 0);
+	ASSERT_EQ(chown(path, ID_HOST + 7, ID_HOST + 7), 0);
+
+	/* nfs_export=on gives decodable overlay file handles. */
+	ovl = ovl_mount(self->base, true);
+	if (ovl < 0)
+		SKIP(return, "overlayfs nfs_export not supported");
+	ASSERT_EQ(ovl_idmap(ovl), 0);
+
+	/* Attach the idmapped mount so handles can be resolved against it. */
+	snprintf(mnt, sizeof(mnt), "%s/mnt", self->base);
+	ASSERT_EQ(mkdir(mnt, 0755), 0);
+	ASSERT_EQ(sys_move_mount(ovl, "", AT_FDCWD, mnt,
+				 MOVE_MOUNT_F_EMPTY_PATH), 0);
+
+	snprintf(path, sizeof(path), "%s/file", mnt);
+	fh->handle_bytes = MAX_HANDLE_SZ;
+	ASSERT_EQ(name_to_handle_at(AT_FDCWD, path, fh, &mount_id, 0), 0);
+
+	mfd = open(mnt, O_RDONLY | O_DIRECTORY);
+	ASSERT_GE(mfd, 0);
+	fd = open_by_handle_at(mfd, fh, O_RDONLY);
+	EXPECT_EQ(close(mfd), 0);
+	ASSERT_GE(fd, 0);
+
+	ASSERT_EQ(fstat(fd, &st), 0);
+	EXPECT_EQ(st.st_uid, ID_NS + 7);
+	EXPECT_EQ(st.st_gid, ID_NS + 7);
+
+	EXPECT_EQ(close(fd), 0);
+	EXPECT_EQ(close(ovl), 0);
+}
+
+TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/filesystems/overlayfs/set_layers_via_fds.c b/tools/testing/selftests/filesystems/overlayfs/set_layers_via_fds.c
index 3c0b931833489..7a293544233d7 100644
--- a/tools/testing/selftests/filesystems/overlayfs/set_layers_via_fds.c
+++ b/tools/testing/selftests/filesystems/overlayfs/set_layers_via_fds.c
@@ -624,7 +624,7 @@ TEST_F(set_layers_via_fds, set_layers_via_detached_mount_fds)
 
 	ASSERT_EQ(sys_move_mount(fd_tmpfs, "", -EBADF, "/set_layers_via_fds_tmpfs", MOVE_MOUNT_F_EMPTY_PATH), 0);
 
-	fd_tmp = open_tree(fd_tmpfs, "u", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
+	fd_tmp = sys_open_tree(fd_tmpfs, "u", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
 	ASSERT_GE(fd_tmp, 0);
 
 	layer_fds[0] = openat(fd_tmp, "upper", O_CLOEXEC | O_DIRECTORY | O_PATH);
@@ -633,25 +633,25 @@ TEST_F(set_layers_via_fds, set_layers_via_detached_mount_fds)
 	layer_fds[1] = openat(fd_tmp, "work", O_CLOEXEC | O_DIRECTORY | O_PATH);
 	ASSERT_GE(layer_fds[1], 0);
 
-	layer_fds[2] = open_tree(fd_tmpfs, "l1", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
+	layer_fds[2] = sys_open_tree(fd_tmpfs, "l1", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
 	ASSERT_GE(layer_fds[2], 0);
 
-	layer_fds[3] = open_tree(fd_tmpfs, "l2", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
+	layer_fds[3] = sys_open_tree(fd_tmpfs, "l2", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
 	ASSERT_GE(layer_fds[3], 0);
 
-	layer_fds[4] = open_tree(fd_tmpfs, "l3", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
+	layer_fds[4] = sys_open_tree(fd_tmpfs, "l3", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
 	ASSERT_GE(layer_fds[4], 0);
 
-	layer_fds[5] = open_tree(fd_tmpfs, "l4", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
+	layer_fds[5] = sys_open_tree(fd_tmpfs, "l4", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
 	ASSERT_GE(layer_fds[5], 0);
 
-	layer_fds[6] = open_tree(fd_tmpfs, "d1", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
+	layer_fds[6] = sys_open_tree(fd_tmpfs, "d1", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
 	ASSERT_GE(layer_fds[6], 0);
 
-	layer_fds[7] = open_tree(fd_tmpfs, "d2", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
+	layer_fds[7] = sys_open_tree(fd_tmpfs, "d2", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
 	ASSERT_GE(layer_fds[7], 0);
 
-	layer_fds[8] = open_tree(fd_tmpfs, "d3", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
+	layer_fds[8] = sys_open_tree(fd_tmpfs, "d3", OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
 	ASSERT_GE(layer_fds[8], 0);
 
 	ASSERT_EQ(close(fd_tmpfs), 0);
diff --git a/tools/testing/selftests/filesystems/statmount/statmount_test.c b/tools/testing/selftests/filesystems/statmount/statmount_test.c
index 8dc018d47a935..60c2c544db6a4 100644
--- a/tools/testing/selftests/filesystems/statmount/statmount_test.c
+++ b/tools/testing/selftests/filesystems/statmount/statmount_test.c
@@ -82,6 +82,9 @@ static void cleanup_namespace(void)
 {
 	int ret;
 
+	if (f_mountinfo)
+		fclose(f_mountinfo);
+
 	ret = fchdir(orig_root);
 	if (ret == -1)
 		ksft_perror("fchdir to original root");
@@ -515,7 +518,7 @@ static void test_statmount_mnt_opts(void)
 		return;
 	}
 
-	ksft_test_result_fail("didnt't find mount entry\n");
+	ksft_test_result_fail("didn't find mount entry\n");
 	free(sm);
 	free(line);
 }
diff --git a/tools/testing/selftests/filesystems/ustat_test.c b/tools/testing/selftests/filesystems/ustat_test.c
new file mode 100644
index 0000000000000..d429fd18d7795
--- /dev/null
+++ b/tools/testing/selftests/filesystems/ustat_test.c
@@ -0,0 +1,135 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Test ustat(2): looking up superblocks by device number.
+ *
+ * ustat() resolves a device number to a mounted superblock via
+ * user_get_super(). Check that the device number of a mounted tmpfs (an
+ * anonymous device) resolves, that it stops resolving once the filesystem
+ * is unmounted and that bogus device numbers report EINVAL.
+ */
+#define _GNU_SOURCE
+#include <errno.h>
+#include <fcntl.h>
+#include <sched.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+#include "../kselftest_harness.h"
+
+/* struct ustat is not exported through UAPI, mirror include/linux/types.h. */
+struct ustat_buf {
+	int		f_tfree;
+	unsigned long	f_tinode;
+	char		f_fname[6];
+	char		f_fpack[6];
+	/* slack in case an architecture lays the struct out differently */
+	char		pad[64];
+};
+
+#ifdef __NR_ustat
+
+/*
+ * The kernel decodes @dev with new_decode_dev(), which matches the low 32
+ * bits of the st_dev encoding stat(2) returns for any major below 4096.
+ */
+static int sys_ustat(unsigned int dev, struct ustat_buf *buf)
+{
+	return syscall(__NR_ustat, dev, buf);
+}
+
+static int write_string(const char *path, const char *string)
+{
+	ssize_t len = strlen(string);
+	int fd;
+
+	fd = open(path, O_WRONLY);
+	if (fd < 0)
+		return -1;
+	if (write(fd, string, len) != len) {
+		close(fd);
+		return -1;
+	}
+	return close(fd);
+}
+
+/* Enter namespaces in which mounting a tmpfs instance is allowed. */
+static int setup_namespaces(void)
+{
+	uid_t uid = getuid();
+	gid_t gid = getgid();
+	char map[64];
+
+	if (unshare(CLONE_NEWNS | (uid ? CLONE_NEWUSER : 0)))
+		return -1;
+
+	if (uid) {
+		if (write_string("/proc/self/setgroups", "deny"))
+			return -1;
+		snprintf(map, sizeof(map), "0 %d 1", uid);
+		if (write_string("/proc/self/uid_map", map))
+			return -1;
+		snprintf(map, sizeof(map), "0 %d 1", gid);
+		if (write_string("/proc/self/gid_map", map))
+			return -1;
+	}
+
+	return mount(NULL, "/", NULL, MS_REC | MS_PRIVATE, NULL);
+}
+
+TEST(resolves_mounted_superblock)
+{
+	char dir[] = "/tmp/ustat_test.XXXXXX";
+	struct ustat_buf ub;
+	struct stat st;
+
+	ASSERT_NE(NULL, mkdtemp(dir));
+
+	if (setup_namespaces()) {
+		rmdir(dir);
+		SKIP(return, "cannot set up namespaces: %s", strerror(errno));
+	}
+
+	ASSERT_EQ(0, mount("ustat_test", dir, "tmpfs", 0, NULL));
+	ASSERT_EQ(0, stat(dir, &st));
+
+	memset(&ub, 0xff, sizeof(ub));
+	ASSERT_EQ(0, sys_ustat(st.st_dev, &ub))
+		TH_LOG("ustat(%u): %s", (unsigned int)st.st_dev,
+		       strerror(errno));
+
+	ASSERT_EQ(0, umount(dir));
+
+	/* The unmount removed the superblock, the device is gone. */
+	ASSERT_EQ(-1, sys_ustat(st.st_dev, &ub));
+	ASSERT_EQ(EINVAL, errno);
+
+	rmdir(dir);
+}
+
+TEST(bogus_device_numbers)
+{
+	struct ustat_buf ub;
+
+	ASSERT_EQ(-1, sys_ustat(0, &ub));
+	ASSERT_EQ(EINVAL, errno);
+
+	/* major 4095, minor 1048575: nothing plausible lives there */
+	ASSERT_EQ(-1, sys_ustat((0xfffu << 8) | 0xffu | (0xfff00u << 12), &ub));
+	ASSERT_EQ(EINVAL, errno);
+}
+
+#else /* !__NR_ustat */
+
+TEST(unsupported)
+{
+	SKIP(return, "ustat(2) is not available on this architecture");
+}
+
+#endif /* __NR_ustat */
+
+TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/ftrace/poll.c b/tools/testing/selftests/ftrace/poll.c
index 53258f7515e77..4da86a20dc85d 100644
--- a/tools/testing/selftests/ftrace/poll.c
+++ b/tools/testing/selftests/ftrace/poll.c
@@ -66,7 +66,7 @@ int main(int argc, char *argv[])
 	}
 	close(pfd.fd);
 
-	/* If timeout happned (ret == 0), exit code is 1 */
+	/* If timeout happened (ret == 0), exit code is 1 */
 	if (ret == 0)
 		return 1;
 
diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_fprobe_module.tc b/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_fprobe_module.tc
index 2915206777b6c..89660a9adf445 100644
--- a/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_fprobe_module.tc
+++ b/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_fprobe_module.tc
@@ -16,23 +16,32 @@ echo > dynamic_events
 FUNC1='foo_bar*'
 FUNC2='vfs_read'
 
+:;: "Save enabled functions count" ;:
+ecount=`cat enabled_functions | wc -l`
+
+count_enabled_functions() {
+    count=`cat enabled_functions | wc -l`
+    count=$(($count-$ecount))
+    echo $count
+}
+
 :;: "Add an event on the test module" ;:
 echo "f:test1 $FUNC1" >> dynamic_events
 echo 1 > events/fprobes/test1/enable
 
 :;: "Ensure it is enabled" ;:
-funcs=`cat enabled_functions | wc -l`
+funcs=`count_enabled_functions`
 test $funcs -ne 0
 
 :;: "Check the enabled_functions is cleared on unloading" ;:
 rmmod trace-events-sample
-funcs=`cat enabled_functions | wc -l`
+funcs=`count_enabled_functions`
 test $funcs -eq 0
 
 :;: "Check it is kept clean" ;:
 modprobe trace-events-sample
 echo 1 > events/fprobes/test1/enable || echo "OK"
-funcs=`cat enabled_functions | wc -l`
+funcs=`count_enabled_functions`
 test $funcs -eq 0
 
 :;: "Add another event not on the test module" ;:
@@ -40,19 +49,19 @@ echo "f:test2 $FUNC2" >> dynamic_events
 echo 1 > events/fprobes/test2/enable
 
 :;: "Ensure it is enabled" ;:
-ofuncs=`cat enabled_functions | wc -l`
+ofuncs=`count_enabled_functions`
 test $ofuncs -ne 0
 
 :;: "Disable and remove the first event"
 echo 0 > events/fprobes/test1/enable
 echo "-:fprobes/test1" >> dynamic_events
-funcs=`cat enabled_functions | wc -l`
+funcs=`count_enabled_functions`
 test $ofuncs -eq $funcs
 
 :;: "Disable and remove other events" ;:
 echo 0 > events/fprobes/enable
 echo > dynamic_events
-funcs=`cat enabled_functions | wc -l`
+funcs=`count_enabled_functions`
 test $funcs -eq 0
 
 rmmod trace-events-sample
@@ -63,12 +72,12 @@ echo "f:test1 $FUNC1" >> dynamic_events
 echo 1 > events/fprobes/test1/enable
 echo "f:test2 $FUNC2" >> dynamic_events
 echo 1 > events/fprobes/test2/enable
-ofuncs=`cat enabled_functions | wc -l`
+ofuncs=`count_enabled_functions`
 test $ofuncs -ne 0
 
 :;: "Unload module (ftrace entry should be removed)" ;:
 rmmod trace-events-sample
-funcs=`cat enabled_functions | wc -l`
+funcs=`count_enabled_functions`
 test $funcs -ne 0
 test $ofuncs -ne $funcs
 
@@ -77,7 +86,7 @@ echo 0 > events/fprobes/test2/enable
 echo "-:fprobes/test2" >> dynamic_events
 
 :;: "Ensure ftrace is disabled." ;:
-funcs=`cat enabled_functions | wc -l`
+funcs=`count_enabled_functions`
 test $funcs -eq 0
 
 echo 0 > events/fprobes/enable
diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/btf_probe_event.tc b/tools/testing/selftests/ftrace/test.d/dynevent/btf_probe_event.tc
new file mode 100644
index 0000000000000..bf71368c31a4a
--- /dev/null
+++ b/tools/testing/selftests/ftrace/test.d/dynevent/btf_probe_event.tc
@@ -0,0 +1,51 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0
+# description: BTF event with typecast and percpu access
+# requires: dynamic_events "this_cpu_read(<fetcharg>)":README "[(structname[,field])]<argname>[->field[->field|.field...]]":README
+
+# Check if the sample module is loaded
+if ! lsmod | grep -q trace_events_sample; then
+  modprobe trace-events-sample || exit_unresolved
+fi
+
+echo 0 > events/enable
+echo > dynamic_events
+
+# The sample_timer_cb(struct timer_list *t) is called.
+# We want to check (STRUCT,FIELD)VAR typecast and this_cpu_read() access.
+# (foo_timer_data,timer)t converts t to struct foo_timer_data * using container_of.
+# data->counter is a per-cpu pointer to int.
+# this_cpu_read(data->counter) should give the value of the counter.
+
+echo 'f:mysample/myevent sample_timer_cb name=(foo_timer_data,timer)t->name:string count=this_cpu_read((foo_timer_data,timer)t->counter)' >> dynamic_events
+
+echo 1 > events/mysample/myevent/enable
+echo 1 > events/sample-trace/foo_timer_fn/enable
+
+sleep 2
+
+echo 0 > events/mysample/myevent/enable
+echo 0 > events/sample-trace/foo_timer_fn/enable
+
+# Compare the values.
+MATCH=0
+while read line; do
+  if echo $line | grep -q "foo_timer_fn:"; then
+    NAME=`echo $line | sed 's/.*name=\([^ ]*\) .*/\1/'`
+    COUNT=`echo $line | sed 's/.*count=\([^ ]*\).*/\1/'`
+    if grep -q "myevent:.*name=\"${NAME}\" count=$COUNT" trace; then
+       MATCH=$((MATCH+1))
+    fi
+  fi
+done < trace
+
+if [ $MATCH -eq 0 ]; then
+  echo "No matching events found"
+  exit_fail
+fi
+
+# Clean up
+echo 0 > events/mysample/myevent/enable
+echo 0 > events/sample-trace/foo_timer_fn/enable
+echo > dynamic_events
+clear_trace
diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/btf_typecast_accepted.tc b/tools/testing/selftests/ftrace/test.d/dynevent/btf_typecast_accepted.tc
new file mode 100644
index 0000000000000..dd5552727054c
--- /dev/null
+++ b/tools/testing/selftests/ftrace/test.d/dynevent/btf_typecast_accepted.tc
@@ -0,0 +1,103 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0
+# description: BTF typecast and percpu access syntax validation
+# requires: dynamic_events "this_cpu_read(<fetcharg>)":README "[(structname[,field])]<argname>[->field[->field|.field...]]":README
+
+KPROBES=
+FPROBES=
+
+if grep -qF "p[:[<group>/][<event>]] <place> [<args>]" README ; then
+  KPROBES=yes
+fi
+if grep -qF "f[:[<group>/][<event>]] <func-name>[%return] [<args>]" README ; then
+  FPROBES=yes
+fi
+
+if [ -z "$KPROBES" -a -z "$FPROBES" ] ; then
+  exit_unsupported
+fi
+
+echo 0 > events/enable
+echo > dynamic_events
+
+# Load trace-events-sample module if available to have per-CPU counter structure defined
+if ! lsmod | grep -q trace_events_sample; then
+  modprobe trace-events-sample || exit_unresolved
+fi
+
+if [ "$FPROBES" ] ; then
+  # 1. Test basic typecast on fprobe
+  echo 'f:fpevent1 vfs_read name=(file)file->f_path.dentry->d_name.name:string' >> dynamic_events
+  # 2. Test parenthesized typecast target on fprobe
+  echo 'f:fpevent2 vfs_read name=(file)(file)->f_path.dentry->d_name.name:string' >> dynamic_events
+  # 3. Test nested typecasts on fprobe
+  echo 'f:fpevent3 vfs_read name=(dentry)((file)file->f_path.dentry)->d_name.name:string' >> dynamic_events
+  # 4. Test container_of-style typecast with field option on fprobe
+  echo 'f:fpevent4 vfs_read name=(file,f_path)file->f_mode' >> dynamic_events
+  # 5. Test typecast on return value on fprobe
+  echo 'f:fpevent5 vfs_read%return name=(file)$retval->f_path.dentry->d_name.name:string' >> dynamic_events
+  # 6. Test $current variable support on fprobe
+  echo 'f:fpevent6 vfs_read pid=$current->pid' >> dynamic_events
+  echo 'f:fpevent7 vfs_read pid=(task_struct)$current->pid' >> dynamic_events
+  echo 'f:fpevent8 vfs_read pid=(task_struct,group_leader)$current->pid' >> dynamic_events
+
+  # Test this_cpu_read and this_cpu_ptr on fprobe
+  echo 'f:fpevent9 sample_timer_cb name=(foo_timer_data,timer)t->name:string count=this_cpu_read((foo_timer_data,timer)t->counter)' >> dynamic_events
+  echo 'f:fpevent10 sample_timer_cb ptr=this_cpu_ptr((foo_timer_data,timer)t->counter)' >> dynamic_events
+fi
+
+if [ "$KPROBES" ] ; then
+  # 7. Test basic typecast on kprobe
+  echo 'p:kpevent1 vfs_read name=(file)file->f_path.dentry->d_name.name:string' >> dynamic_events
+  # 8. Test parenthesized typecast target on kprobe
+  echo 'p:kpevent2 vfs_read name=(file)(file)->f_path.dentry->d_name.name:string' >> dynamic_events
+  # 9. Test nested typecasts on kprobe
+  echo 'p:kpevent3 vfs_read name=(dentry)((file)file->f_path.dentry)->d_name.name:string' >> dynamic_events
+  # 10. Test container_of-style typecast with field option on kprobe
+  echo 'p:kpevent4 vfs_read name=(file,f_path)file->f_mode' >> dynamic_events
+  # 11. Test typecast on return value on kretprobe
+  echo 'r:kpevent5 vfs_read name=(file)$retval->f_path.dentry->d_name.name:string' >> dynamic_events
+  # 12. Test $current variable support on kprobe
+  echo 'p:kpevent6 vfs_read pid=$current->pid' >> dynamic_events
+  echo 'p:kpevent7 vfs_read pid=(task_struct)$current->pid' >> dynamic_events
+  echo 'p:kpevent8 vfs_read pid=(task_struct,group_leader)$current->pid' >> dynamic_events
+
+  # Test this_cpu_read and this_cpu_ptr on kprobe
+  echo 'p:kpevent9 sample_timer_cb name=(foo_timer_data,timer)t->name:string count=this_cpu_read((foo_timer_data,timer)t->counter)' >> dynamic_events
+  echo 'p:kpevent10 sample_timer_cb ptr=this_cpu_ptr((foo_timer_data,timer)t->counter)' >> dynamic_events
+fi
+
+# Verify the events exist in dynamic_events
+if [ "$FPROBES" ] ; then
+  grep -q "fpevent1 " dynamic_events
+  grep -q "fpevent2 " dynamic_events
+  grep -q "fpevent3 " dynamic_events
+  grep -q "fpevent4 " dynamic_events
+  grep -q "fpevent5 " dynamic_events
+  grep -q "fpevent6 " dynamic_events
+  grep -q "fpevent7 " dynamic_events
+  grep -q "fpevent8 " dynamic_events
+  if lsmod | grep -q trace_events_sample; then
+    grep -q "fpevent9 " dynamic_events
+    grep -q "fpevent10 " dynamic_events
+  fi
+fi
+
+if [ "$KPROBES" ] ; then
+  grep -q "kpevent1 " dynamic_events
+  grep -q "kpevent2 " dynamic_events
+  grep -q "kpevent3 " dynamic_events
+  grep -q "kpevent4 " dynamic_events
+  grep -q "kpevent5 " dynamic_events
+  grep -q "kpevent6 " dynamic_events
+  grep -q "kpevent7 " dynamic_events
+  grep -q "kpevent8 " dynamic_events
+  if lsmod | grep -q trace_events_sample; then
+    grep -q "kpevent9 " dynamic_events
+    grep -q "kpevent10 " dynamic_events
+  fi
+fi
+
+# Clean up
+echo > dynamic_events
+clear_trace
diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/eprobes_syntax_errors.tc b/tools/testing/selftests/ftrace/test.d/dynevent/eprobes_syntax_errors.tc
index 2a680c086047f..c2e3f9d19f130 100644
--- a/tools/testing/selftests/ftrace/test.d/dynevent/eprobes_syntax_errors.tc
+++ b/tools/testing/selftests/ftrace/test.d/dynevent/eprobes_syntax_errors.tc
@@ -10,7 +10,7 @@ check_error() { # command-with-error-pos-by-^
 check_error 'e ^a.'			# NO_EVENT_INFO
 check_error 'e ^.b'			# NO_EVENT_INFO
 check_error 'e ^a.b'			# BAD_ATTACH_EVENT
-check_error 'e syscalls/sys_enter_openat ^foo'	# BAD_ATTACH_ARG
+check_error 'e syscalls/sys_enter_openat ^foo'	# NO_EVENT_FIELD
 check_error 'e:^/bar syscalls/sys_enter_openat'	# NO_GROUP_NAME
 check_error 'e:^12345678901234567890123456789012345678901234567890123456789012345/bar syscalls/sys_enter_openat'	# GROUP_TOO_LONG
 
@@ -19,11 +19,19 @@ check_error 'e:^ syscalls/sys_enter_openat'		# NO_EVENT_NAME
 check_error 'e:foo/^12345678901234567890123456789012345678901234567890123456789012345 syscalls/sys_enter_openat'	# EVENT_TOO_LONG
 check_error 'e:foo/^bar.1 syscalls/sys_enter_openat'	# BAD_EVENT_NAME
 
-check_error 'e:foo/bar syscalls/sys_enter_openat arg=^dfd'	# BAD_FETCH_ARG
 check_error 'e:foo/bar syscalls/sys_enter_openat arg=^$foo'	# BAD_ATTACH_ARG
 
+check_error 'e:foo/bar syscalls/sys_enter_openat arg=^COMM'	# NO_EVENT_FIELD
+if grep -q "\$current.*" README; then
+  check_error 'e:foo/bar syscalls/sys_enter_openat arg=^current'	# NO_EVENT_FIELD
+fi
+
 if grep -q '<attached-group>\.<attached-event>.*\[if <filter>\]' README; then
   check_error 'e:foo/bar syscalls/sys_enter_openat if ^'	# NO_EP_FILTER
 fi
 
+if grep -q 'this_cpu_read(<fetcharg>)' README; then
+  check_error 'e:foo/bar syscalls/sys_enter_openat arg=^this_cpu_read(file)'	# NOSUP_PERCPU
+fi
+
 exit 0
diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_syntax_errors.tc b/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_syntax_errors.tc
index fee479295e2fa..e9d7e6919c7f3 100644
--- a/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_syntax_errors.tc
+++ b/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_syntax_errors.tc
@@ -112,6 +112,18 @@ check_error 'f vfs_read%return $retval->^foo'	# NO_PTR_STRCT
 check_error 'f vfs_read file->^foo'		# NO_BTF_FIELD
 check_error 'f vfs_read file^-.foo'		# BAD_HYPHEN
 check_error 'f vfs_read ^file:string'		# BAD_TYPE4STR
+if grep -qF "[(structname" README ; then
+check_error 'f vfs_read arg1=(task_struct)file^'		# TYPECAST_REQ_FIELD
+check_error 'f vfs_read arg1=(a)((b)((c)(^(d)file->d)->c)->b)->a'	# TOO_MANY_NESTED
+check_error 'f vfs_read arg1=(task_struct,^in_execve)file->comm'	# TYPECAST_NOT_ALIGNED
+check_error 'f vfs_read arg1=(task_struct,^foo_bar)file->pid'	# NO_BTF_FIELD
+check_error 'f vfs_read arg1=(^task_struct1234)file->pid'	# NO_PTR_STRCT
+check_error 'f vfs_read arg1=(task_struct,se^->group_node)file->comm'	# TYPECAST_BAD_ARROW
+check_error 'f vfs_read arg1=(task_struct,^->pid)file->comm'	# NO_BTF_FIELD
+check_error 'f vfs_read arg1=(task_struct,^.pid)file->comm'	# NO_BTF_FIELD
+check_error 'f vfs_read arg1=(task_struct,^.)file->comm'	# NO_BTF_FIELD
+check_error 'f vfs_read arg1=(task_struct)^@symbol+10->comm'	# TYPECAST_SYM_OFFSET
+fi
 fi
 
 else
diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_syntax_errors.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_syntax_errors.tc
index 8f1c58f0c2397..21ce8414459f3 100644
--- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_syntax_errors.tc
+++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_syntax_errors.tc
@@ -115,6 +115,18 @@ check_error 'p vfs_read+20 ^$arg*'		# NOFENTRY_ARGS
 check_error 'p vfs_read ^hoge'			# NO_BTFARG
 check_error 'p kfree ^$arg10'			# NO_BTFARG (exceed the number of parameters)
 check_error 'r kfree ^$retval'			# NO_RETVAL
+if grep -qF "[(structname" README ; then
+check_error 'p vfs_read arg1=(task_struct)file^'		# TYPECAST_REQ_FIELD
+check_error 'p vfs_read arg1=(a)((b)((c)(^(d)file->d)->c)->b)->a'	# TOO_MANY_NESTED
+check_error 'p vfs_read arg1=(task_struct,^in_execve)file->comm'	# TYPECAST_NOT_ALIGNED
+check_error 'p vfs_read arg1=(task_struct,^foo_bar)file->pid'	# NO_BTF_FIELD
+check_error 'p vfs_read arg1=(^task_struct1234)file->pid'		# NO_PTR_STRCT
+check_error 'p vfs_read arg1=(task_struct,se^->group_node)file->comm'	# TYPECAST_BAD_ARROW
+check_error 'p vfs_read arg1=(task_struct,^->pid)file->comm'	# NO_BTF_FIELD
+check_error 'p vfs_read arg1=(task_struct,^.pid)file->comm'	# NO_BTF_FIELD
+check_error 'p vfs_read arg1=(task_struct,^.)file->comm'	# NO_BTF_FIELD
+check_error 'p vfs_read arg1=(task_struct)^@symbol+10->comm'	# TYPECAST_SYM_OFFSET
+fi
 else
 check_error 'p vfs_read ^$arg*'			# NOSUP_BTFARG
 fi
diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc b/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc
index c817158b99db6..e12dc967ec761 100644
--- a/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc
+++ b/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc
@@ -28,4 +28,9 @@ if grep -q ".*symstr.*" README; then
 check_error 'p /bin/sh:10 $stack0:^symstr'	# BAD_TYPE
 fi
 
+# $current is not supported by uprobe
+if grep -q "\$current.*" README; then
+check_error 'p /bin/sh:10 ^$current:u8'	# BAD_VAR
+fi
+
 exit 0
diff --git a/tools/testing/selftests/ftrace/test.d/trigger/trigger-hist-poll.tc b/tools/testing/selftests/ftrace/test.d/trigger/trigger-hist-poll.tc
index 8d275e3238d9f..04eb8546fc07f 100644
--- a/tools/testing/selftests/ftrace/test.d/trigger/trigger-hist-poll.tc
+++ b/tools/testing/selftests/ftrace/test.d/trigger/trigger-hist-poll.tc
@@ -1,7 +1,7 @@
 #!/bin/sh
 # SPDX-License-Identifier: GPL-2.0
 # description: event trigger - test poll wait on histogram
-# requires: set_event events/sched/sched_process_free/trigger events/sched/sched_process_free/hist
+# requires: set_event events/sched/sched_process_exit/trigger events/sched/sched_process_exit/hist
 # flags: instance
 
 POLL=${FTRACETEST_ROOT}/poll
@@ -11,7 +11,7 @@ if [ ! -x ${POLL} ]; then
   exit_unresolved
 fi
 
-EVENT=events/sched/sched_process_free/
+EVENT=events/sched/sched_process_exit/
 
 # Check poll ops is supported. Before implementing poll on hist file, it
 # returns soon with POLLIN | POLLOUT, but not POLLPRI.
diff --git a/tools/testing/selftests/futex/functional/.gitignore b/tools/testing/selftests/futex/functional/.gitignore
index 23b9fea8d190a..7c39d10b38e4d 100644
--- a/tools/testing/selftests/futex/functional/.gitignore
+++ b/tools/testing/selftests/futex/functional/.gitignore
@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0-only
+futex_lock_pi_exiting
 futex_numa_mpol
 futex_priv_hash
 futex_requeue
diff --git a/tools/testing/selftests/futex/functional/Makefile b/tools/testing/selftests/futex/functional/Makefile
index 5c1c824f97400..a03bd5acba503 100644
--- a/tools/testing/selftests/futex/functional/Makefile
+++ b/tools/testing/selftests/futex/functional/Makefile
@@ -11,7 +11,8 @@ endif
 
 LOCAL_HDRS := \
 	../include/futextest.h \
-	../include/atomic.h
+	../include/atomic.h \
+	../include/futex_thread.h
 TEST_GEN_PROGS := \
 	futex_wait_timeout \
 	futex_wait_wouldblock \
@@ -26,7 +27,8 @@ TEST_GEN_PROGS := \
 	futex_numa_mpol \
 	futex_waitv \
 	futex_numa \
-	robust_list
+	robust_list \
+	futex_lock_pi_exiting
 
 TEST_PROGS := run.sh
 
diff --git a/tools/testing/selftests/futex/functional/futex_lock_pi_exiting.c b/tools/testing/selftests/futex/functional/futex_lock_pi_exiting.c
new file mode 100644
index 0000000000000..623c5f3a18367
--- /dev/null
+++ b/tools/testing/selftests/futex/functional/futex_lock_pi_exiting.c
@@ -0,0 +1,263 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/******************************************************************************
+ *
+ * futex_lock_pi_exiting.c
+ *
+ * Coverage for the FUTEX_LOCK_PI owner-exiting path.  futex_wait_timeout.c
+ * already covers FUTEX_LOCK_PI timeout semantics and robust_list.c covers
+ * owner death via the robust list, but nothing exercises FUTEX_LOCK_PI when a
+ * non-robust PI owner exits while holding the lock, nor the basic ownership /
+ * EDEADLK / unlock word semantics.
+ *
+ * DESCRIPTION
+ *      Three tests:
+ *
+ *      1. lock_unlock_basic - uncontended FUTEX_LOCK_PI semantics: the futex
+ *         word carries the owner TID, a recursive lock by the owner returns
+ *         EDEADLK, and FUTEX_UNLOCK_PI clears the word.
+ *
+ *      2. owner_dies_with_blocked_waiter - a thread acquires a PI futex and
+ *         exits while holding it.  do_exit() runs futex_cleanup_begin() (which
+ *         flips the task's futex state to FUTEX_STATE_EXITING) and
+ *         exit_pi_state_list() (which hands off / tears down the pi_state).  A
+ *         contending FUTEX_LOCK_PI waiter must end up in one of:
+ *
+ *           0          - ownership was transferred to / acquired by the waiter
+ *           EOWNERDEAD - previous owner died holding the lock; the caller is
+ *                        now the owner and must acknowledge by unlocking
+ *           ESRCH      - the owner encoded in the futex word is already gone
+ *
+ *         and on the first two it must actually own the lock afterwards.
+ *
+ *      3. stress_owner_exits - hammer that same exiting-owner path.  This is
+ *         where the following bug lived: the 'exiting' task pointer was not
+ *         reset at the retry label, so after wait_for_owner_exiting() dropped
+ *         its reference a subsequent retry that returned a non-EBUSY error fed
+ *         the stale pointer back in and tripped WARN_ON_ONCE(exiting).  That
+ *         warning is invisible to user space, so this test cannot observe it
+ *         through a syscall return value; it only becomes a visible failure
+ *         (crash) on a kernel booted with panic_on_warn=1 (or built with
+ *         CONFIG_BUG_ON_DATA_CORRUPTION).  The loop drives the path so that
+ *         such a kernel trips on it - the canonical way fuzz/CI catch these.
+ *
+ *        Fix:    210d36d892de ("futex: Clear stale exiting pointer in
+ *                              futex_lock_pi() retry path")
+ *        Fixes:  3ef240eaff36 ("futex: Prevent exit livelock")
+ *
+ * AUTHOR
+ *      Based on futex test boilerplate by Darren Hart <dvhart@linux.intel.com>
+ *
+ *****************************************************************************/
+
+#define _GNU_SOURCE
+
+#include <errno.h>
+#include <pthread.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/syscall.h>
+
+#include "futextest.h"
+#include "kselftest_harness.h"
+
+/*
+ * Iterations for the stress variant.  Enough to repeatedly land in the narrow
+ * EXITING window while keeping the test fast.
+ */
+#define STRESS_ITERS 1000
+
+static futex_t pi_futex;
+static pthread_barrier_t locked_barrier;
+static pthread_barrier_t release_barrier;
+
+static pid_t sys_gettid(void)
+{
+	return syscall(SYS_gettid);
+}
+
+/*
+ * Owner thread: acquire the PI futex and exit while still holding it.  Two
+ * modes:
+ *   park == 0: signal that we hold the lock, then exit immediately (racy; the
+ *              waiter races against our exit path).
+ *   park == 1: signal that we hold the lock and keep holding until released
+ *              via release_barrier, so a waiter has time to contend as a real
+ *              PI waiter before we die.
+ */
+static void *owner_thread(void *arg)
+{
+	long park = (long)arg;
+
+	if (futex_lock_pi(&pi_futex, NULL, 0, FUTEX_PRIVATE_FLAG) != 0)
+		return (void *)(intptr_t)-errno;
+
+	pthread_barrier_wait(&locked_barrier);
+
+	if (park)
+		pthread_barrier_wait(&release_barrier);
+
+	/* Die while still holding the lock. */
+	pthread_exit((void *)0);
+}
+
+/*
+ * Block on the PI futex as a waiter.  Returns 0 on acquisition, otherwise the
+ * positive errno.
+ */
+static int waiter_lock_pi(void)
+{
+	int ret = futex_lock_pi(&pi_futex, NULL, 0, FUTEX_PRIVATE_FLAG);
+
+	return ret == 0 ? 0 : errno;
+}
+
+static int outcome_ok(int outcome)
+{
+	return outcome == 0 || outcome == EOWNERDEAD || outcome == ESRCH;
+}
+
+/* Results published by waiter_thread() for the owning thread to assert on. */
+static int waiter_outcome;
+static int waiter_owns;
+
+/*
+ * Waiter thread for the blocked-waiter test.  Contends for the lock and, when
+ * it acquires, records whether the futex word actually carries its TID and
+ * releases the lock itself (FUTEX_UNLOCK_PI must run in the owning thread).
+ */
+static void *waiter_thread(void *arg)
+{
+	pid_t tid = sys_gettid();
+
+	waiter_outcome = waiter_lock_pi();
+	if (waiter_outcome == 0 || waiter_outcome == EOWNERDEAD) {
+		waiter_owns = (pi_futex & FUTEX_TID_MASK) == (futex_t)tid;
+		futex_unlock_pi(&pi_futex, FUTEX_PRIVATE_FLAG);
+	}
+	return NULL;
+}
+
+FIXTURE(lock_pi_exiting) {
+};
+
+FIXTURE_SETUP(lock_pi_exiting) {
+}
+
+FIXTURE_TEARDOWN(lock_pi_exiting) {
+}
+
+/*
+ * Uncontended FUTEX_LOCK_PI semantics, fully deterministic.
+ */
+TEST_F(lock_pi_exiting, lock_unlock_basic)
+{
+	pid_t tid = sys_gettid();
+	int ret;
+
+	pi_futex = FUTEX_INITIALIZER;
+
+	/* Acquire: we become the owner, our TID lands in the futex word. */
+	ret = futex_lock_pi(&pi_futex, NULL, 0, FUTEX_PRIVATE_FLAG);
+	ASSERT_EQ(ret, 0)
+		TH_LOG("lock failed: errno=%d (%s)", errno, strerror(errno));
+	ASSERT_EQ(pi_futex & FUTEX_TID_MASK, (futex_t)tid)
+		TH_LOG("owner TID not in futex word: 0x%08x", pi_futex);
+
+	/* A recursive lock by the owner must be refused, not deadlock. */
+	errno = 0;
+	ret = futex_lock_pi(&pi_futex, NULL, 0, FUTEX_PRIVATE_FLAG);
+	ASSERT_EQ(ret, -1);
+	ASSERT_EQ(errno, EDEADLK)
+		TH_LOG("recursive lock: expected EDEADLK, got errno=%d", errno);
+
+	/* Release: the futex word is handed back clean. */
+	ret = futex_unlock_pi(&pi_futex, FUTEX_PRIVATE_FLAG);
+	ASSERT_EQ(ret, 0)
+		TH_LOG("unlock failed: errno=%d", errno);
+	ASSERT_EQ(pi_futex, (futex_t)0)
+		TH_LOG("futex word not cleared after unlock: 0x%08x", pi_futex);
+}
+
+/*
+ * A PI waiter inherits the lock when the owner dies holding it.
+ *
+ * The owner parks while holding the lock, this thread contends for it, then
+ * the owner exits.  The waiter must come out cleanly (no hang, no unexpected
+ * error) and, when it acquires, must actually own the lock.
+ */
+TEST_F(lock_pi_exiting, owner_dies_with_blocked_waiter)
+{
+	pthread_t owner, waiter;
+
+	pthread_barrier_init(&locked_barrier, NULL, 2);
+	pthread_barrier_init(&release_barrier, NULL, 2);
+	pi_futex = FUTEX_INITIALIZER;
+	waiter_outcome = -1;
+	waiter_owns = 0;
+
+	ASSERT_EQ(pthread_create(&owner, NULL, owner_thread, (void *)1), 0);
+
+	/* Wait until the owner actually holds the lock. */
+	pthread_barrier_wait(&locked_barrier);
+
+	/* Start the waiter and give it time to block as a real PI waiter. */
+	ASSERT_EQ(pthread_create(&waiter, NULL, waiter_thread, NULL), 0);
+	usleep(1000);
+
+	/* Release the owner so it dies while the waiter is queued on it. */
+	pthread_barrier_wait(&release_barrier);
+
+	pthread_join(waiter, NULL);
+	pthread_join(owner, NULL);
+
+	ASSERT_TRUE(outcome_ok(waiter_outcome)) {
+		TH_LOG("unexpected FUTEX_LOCK_PI outcome: %d (%s)",
+		       waiter_outcome, strerror(waiter_outcome));
+	}
+	if (waiter_outcome == 0 || waiter_outcome == EOWNERDEAD) {
+		ASSERT_TRUE(waiter_owns)
+			TH_LOG("waiter acquired but futex word lacks its TID");
+	}
+
+	pthread_barrier_destroy(&locked_barrier);
+	pthread_barrier_destroy(&release_barrier);
+}
+
+/*
+ * Stress: repeatedly let an owner exit while a waiter contends for the lock.
+ *
+ * Each iteration drives the FUTEX_STATE_EXITING -> -EBUSY -> retry path that
+ * the stale-'exiting'-pointer bug lived on (210d36d892de).  The warning it
+ * fixed is invisible to user space, so on a normally-configured kernel both
+ * the buggy and fixed kernels pass here; the point is to make a kernel booted
+ * with panic_on_warn=1 trip during one of these iterations.
+ */
+TEST_F(lock_pi_exiting, stress_owner_exits)
+{
+	for (int i = 0; i < STRESS_ITERS; i++) {
+		pthread_t owner;
+		int outcome;
+
+		pthread_barrier_init(&locked_barrier, NULL, 2);
+		pi_futex = FUTEX_INITIALIZER;
+
+		ASSERT_EQ(pthread_create(&owner, NULL, owner_thread, (void *)0), 0);
+
+		/* Owner holds the lock; race FUTEX_LOCK_PI against its exit. */
+		pthread_barrier_wait(&locked_barrier);
+
+		outcome = waiter_lock_pi();
+		ASSERT_TRUE(outcome_ok(outcome)) {
+			TH_LOG("iter %d: unexpected outcome %d (%s)",
+			       i, outcome, strerror(outcome));
+		}
+		if (outcome == 0 || outcome == EOWNERDEAD)
+			futex_unlock_pi(&pi_futex, FUTEX_PRIVATE_FLAG);
+
+		pthread_join(owner, NULL);
+		pthread_barrier_destroy(&locked_barrier);
+	}
+}
+
+TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/futex/functional/futex_numa_mpol.c b/tools/testing/selftests/futex/functional/futex_numa_mpol.c
index 78c0f7a59e170..4ffcf41efe1ff 100644
--- a/tools/testing/selftests/futex/functional/futex_numa_mpol.c
+++ b/tools/testing/selftests/futex/functional/futex_numa_mpol.c
@@ -9,6 +9,7 @@
 #include <pthread.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <unistd.h>
 #ifdef LIBNUMA_VER_SUFFICIENT
 #include <numa.h>
@@ -28,9 +29,9 @@ static pthread_barrier_t barrier_main;
 static pthread_t threads[MAX_THREADS];
 
 struct thread_args {
-	void *futex_ptr;
-	unsigned int flags;
-	int result;
+	void		*futex_ptr;
+	unsigned int	flags;
+	int		result;
 };
 
 static struct thread_args thread_args[MAX_THREADS];
@@ -54,7 +55,7 @@ static void *thread_lock_fn(void *arg)
 	return NULL;
 }
 
-static void create_max_threads(void *futex_ptr)
+static void create_max_threads(struct __test_metadata *_metadata, void *futex_ptr)
 {
 	int i, ret;
 
@@ -63,28 +64,29 @@ static void create_max_threads(void *futex_ptr)
 		thread_args[i].flags = FUTEX2_SIZE_U32 | FUTEX_PRIVATE_FLAG | FUTEX2_NUMA;
 		thread_args[i].result = 0;
 		ret = pthread_create(&threads[i], NULL, thread_lock_fn, &thread_args[i]);
-		if (ret)
-			ksft_exit_fail_msg("pthread_create failed\n");
+		ASSERT_EQ(ret, 0)
+			TH_LOG("pthread_create failed");
 	}
 }
 
-static void join_max_threads(void)
+static void join_max_threads(struct __test_metadata *_metadata)
 {
 	int i, ret;
 
 	for (i = 0; i < MAX_THREADS; i++) {
 		ret = pthread_join(threads[i], NULL);
-		if (ret)
-			ksft_exit_fail_msg("pthread_join failed for thread %d\n", i);
+		ASSERT_EQ(ret, 0)
+			TH_LOG("pthread_join failed for thread %d", i);
 	}
 }
 
-static void __test_futex(void *futex_ptr, int err_value, unsigned int futex_flags)
+static void __test_futex(struct __test_metadata *_metadata, void *futex_ptr, int err_value,
+			 unsigned int futex_flags)
 {
-	int to_wake, ret, i, need_exit = 0;
+	int to_wake, ret, i;
 
 	pthread_barrier_init(&barrier_main, NULL, MAX_THREADS + 1);
-	create_max_threads(futex_ptr);
+	create_max_threads(_metadata, futex_ptr);
 	pthread_barrier_wait(&barrier_main);
 	to_wake = MAX_THREADS;
 
@@ -92,45 +94,50 @@ static void __test_futex(void *futex_ptr, int err_value, unsigned int futex_flag
 		ret = futex2_wake(futex_ptr, to_wake, futex_flags);
 
 		if (err_value) {
-			if (ret >= 0)
-				ksft_exit_fail_msg("futex2_wake(%d, 0x%x) should fail, but didn't\n",
-						   to_wake, futex_flags);
+			EXPECT_LT(ret, 0) {
+				TH_LOG("futex2_wake(%d, 0x%x) should fail, but didn't",
+				       to_wake, futex_flags);
+			}
 
-			if (errno != err_value)
-				ksft_exit_fail_msg("futex2_wake(%d, 0x%x) expected error was %d, but returned %d (%s)\n",
-						   to_wake, futex_flags, err_value, errno, strerror(errno));
+			EXPECT_EQ(errno, err_value) {
+				TH_LOG("futex2_wake(%d, 0x%x) expected error was %d, but returned %d (%s)",
+				       to_wake, futex_flags, err_value, errno, strerror(errno));
+			}
 
 			break;
 		}
 		if (ret < 0) {
-			ksft_exit_fail_msg("Failed futex2_wake(%d, 0x%x): %m\n",
-					   to_wake, futex_flags);
+			if (errno == ENOSYS || (errno == EINVAL && (futex_flags & FUTEX2_NUMA)))
+				SKIP(return, "futex2 or FUTEX2_NUMA not supported by kernel");
+
+			ASSERT_GE(ret, 0) {
+				TH_LOG("Failed futex2_wake(%d, 0x%x): %s",
+				       to_wake, futex_flags, strerror(errno));
+			}
 		}
 		if (!ret)
 			usleep(50);
 		to_wake -= ret;
 
 	} while (to_wake);
-	join_max_threads();
+	join_max_threads(_metadata);
 
 	for (i = 0; i < MAX_THREADS; i++) {
-		if (err_value && thread_args[i].result != -1) {
-			ksft_print_msg("Thread %d should fail but succeeded (%d)\n",
+		if (err_value) {
+			EXPECT_EQ(thread_args[i].result, -1) {
+				TH_LOG("Thread %d should fail but succeeded (%d)",
 				       i, thread_args[i].result);
-			need_exit = 1;
-		}
-		if (!err_value && thread_args[i].result != 0) {
-			ksft_print_msg("Thread %d failed (%d)\n", i, thread_args[i].result);
-			need_exit = 1;
+			}
+		} else {
+			EXPECT_EQ(thread_args[i].result, 0)
+				TH_LOG("Thread %d failed (%d)", i, thread_args[i].result);
 		}
 	}
-	if (need_exit)
-		ksft_exit_fail_msg("Aborting due to earlier errors.\n");
 }
 
-static void test_futex(void *futex_ptr, int err_value)
+static void test_futex(struct __test_metadata *_metadata, void *futex_ptr, int err_value)
 {
-	__test_futex(futex_ptr, err_value, FUTEX2_SIZE_U32 | FUTEX_PRIVATE_FLAG | FUTEX2_NUMA);
+	__test_futex(_metadata, futex_ptr, err_value, FUTEX2_SIZE_U32 | FUTEX_PRIVATE_FLAG | FUTEX2_NUMA);
 }
 
 TEST(futex_numa_mpol)
@@ -141,43 +148,41 @@ TEST(futex_numa_mpol)
 
 	mem_size = sysconf(_SC_PAGE_SIZE);
 	futex_ptr = mmap(NULL, mem_size * 2, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
-	if (futex_ptr == MAP_FAILED)
-		ksft_exit_fail_msg("mmap() for %d bytes failed\n", mem_size);
+	ASSERT_NE(futex_ptr, MAP_FAILED)
+		TH_LOG("mmap() for %d bytes failed: %s", mem_size, strerror(errno));
 
 	/* Create an invalid memory region for the "Memory out of range" test */
 	mprotect(futex_ptr + mem_size, mem_size, PROT_NONE);
 
 	futex_numa = futex_ptr;
 
-	ksft_print_msg("Regular test\n");
+	TH_LOG("Regular test");
 	futex_numa->futex = 0;
 	futex_numa->numa = FUTEX_NO_NODE;
-	test_futex(futex_ptr, 0);
+	test_futex(_metadata, futex_ptr, 0);
 
-	if (futex_numa->numa == FUTEX_NO_NODE)
-		ksft_exit_fail_msg("NUMA node is left uninitialized\n");
+	EXPECT_NE(futex_numa->numa, FUTEX_NO_NODE)
+		TH_LOG("NUMA node is left uninitialized");
 
 	/* FUTEX2_NUMA futex must be 8-byte aligned */
-	ksft_print_msg("Mis-aligned futex\n");
-	test_futex(futex_ptr + mem_size - 4, EINVAL);
+	TH_LOG("Mis-aligned futex");
+	test_futex(_metadata, futex_ptr + mem_size - 4, EINVAL);
 
-	ksft_print_msg("Memory out of range\n");
-	test_futex(futex_ptr + mem_size, EFAULT);
+	TH_LOG("Memory out of range");
+	test_futex(_metadata, futex_ptr + mem_size, EFAULT);
 
 	futex_numa->numa = FUTEX_NO_NODE;
 	mprotect(futex_ptr, mem_size, PROT_READ);
-	ksft_print_msg("Memory, RO\n");
-	test_futex(futex_ptr, EFAULT);
+	TH_LOG("Memory, RO");
+	test_futex(_metadata, futex_ptr, EFAULT);
 
 	mprotect(futex_ptr, mem_size, PROT_NONE);
-	ksft_print_msg("Memory, no access\n");
-	test_futex(futex_ptr, EFAULT);
+	TH_LOG("Memory, no access");
+	test_futex(_metadata, futex_ptr, EFAULT);
 
 	mprotect(futex_ptr, mem_size, PROT_READ | PROT_WRITE);
-	ksft_print_msg("Memory back to RW\n");
-	test_futex(futex_ptr, 0);
-
-	ksft_test_result_pass("futex2 memory boundary tests passed\n");
+	TH_LOG("Memory back to RW");
+	test_futex(_metadata, futex_ptr, 0);
 
 	/* MPOL test. Does not work as expected */
 #ifdef LIBNUMA_VER_SUFFICIENT
@@ -190,25 +195,23 @@ TEST(futex_numa_mpol)
 			    sizeof(nodemask) * 8, 0);
 		if (ret == 0) {
 			ret = numa_set_mempolicy_home_node(futex_ptr, mem_size, i, 0);
-			if (ret != 0)
-				ksft_exit_fail_msg("Failed to set home node: %m, %d\n", errno);
+			ASSERT_EQ(ret, 0)
+				TH_LOG("Failed to set home node: %s, %d", strerror(errno), errno);
 
-			ksft_print_msg("Node %d test\n", i);
+			TH_LOG("Node %d test", i);
 			futex_numa->futex = 0;
 			futex_numa->numa = FUTEX_NO_NODE;
 
-			ret = futex2_wake(futex_ptr, 0, FUTEX2_SIZE_U32 | FUTEX_PRIVATE_FLAG | FUTEX2_NUMA | FUTEX2_MPOL);
-			if (ret < 0)
-				ksft_test_result_fail("Failed to wake 0 with MPOL: %m\n");
-			if (futex_numa->numa != i) {
-				ksft_exit_fail_msg("Returned NUMA node is %d expected %d\n",
-						   futex_numa->numa, i);
-			}
+			ret = futex2_wake(futex_ptr, 0, FUTEX2_SIZE_U32 | FUTEX_PRIVATE_FLAG |
+					  FUTEX2_NUMA | FUTEX2_MPOL);
+			EXPECT_GE(ret, 0)
+				TH_LOG("Failed to wake 0 with MPOL: %s", strerror(errno));
+			EXPECT_EQ(futex_numa->numa, i)
+				TH_LOG("Returned NUMA node is %d expected %d", futex_numa->numa, i);
 		}
 	}
-	ksft_test_result_pass("futex2 MPOL hints test passed\n");
 #else
-	ksft_test_result_skip("futex2 MPOL hints test requires libnuma 2.0.18+\n");
+	SKIP(return, "futex2 MPOL hints test requires libnuma 2.0.18+");
 #endif
 	munmap(futex_ptr, mem_size * 2);
 }
diff --git a/tools/testing/selftests/futex/functional/futex_priv_hash.c b/tools/testing/selftests/futex/functional/futex_priv_hash.c
index e8079d7c65e81..a8742e2045402 100644
--- a/tools/testing/selftests/futex/functional/futex_priv_hash.c
+++ b/tools/testing/selftests/futex/functional/futex_priv_hash.c
@@ -9,6 +9,7 @@
 #include <pthread.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <unistd.h>
 
 #include <linux/prctl.h>
@@ -39,31 +40,28 @@ static int futex_hash_slots_get(void)
 	return prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_GET_SLOTS);
 }
 
-static void futex_hash_slots_set_verify(int slots)
+static void futex_hash_slots_set_verify(struct __test_metadata *_metadata, int slots)
 {
 	int ret;
 
 	ret = futex_hash_slots_set(slots);
-	if (ret != 0) {
-		ksft_test_result_fail("Failed to set slots to %d: %m\n", slots);
-		ksft_finished();
-	}
+	ASSERT_EQ(ret, 0)
+		TH_LOG("Failed to set slots to %d: %s", slots, strerror(errno));
+
 	ret = futex_hash_slots_get();
-	if (ret != slots) {
-		ksft_test_result_fail("Set %d slots but PR_FUTEX_HASH_GET_SLOTS returns: %d, %m\n",
-		       slots, ret);
-		ksft_finished();
+	ASSERT_EQ(ret, slots) {
+		TH_LOG("Set %d slots but PR_FUTEX_HASH_GET_SLOTS returns: %d, %s",
+		       slots, ret, strerror(errno));
 	}
-	ksft_test_result_pass("SET and GET slots %d passed\n", slots);
 }
 
-static void futex_hash_slots_set_must_fail(int slots)
+static void futex_hash_slots_set_must_fail(struct __test_metadata *_metadata, int slots)
 {
 	int ret;
 
 	ret = futex_hash_slots_set(slots);
-	ksft_test_result(ret < 0, "futex_hash_slots_set(%d)\n",
-			 slots);
+	EXPECT_LT(ret, 0)
+		TH_LOG("futex_hash_slots_set(%d) should fail but succeeded", slots);
 }
 
 static void *thread_return_fn(void *arg)
@@ -82,32 +80,32 @@ static void *thread_lock_fn(void *arg)
 	return NULL;
 }
 
-static void create_max_threads(void *(*thread_fn)(void *))
+static void create_max_threads(struct __test_metadata *_metadata, void *(*thread_fn)(void *))
 {
 	int i, ret;
 
 	for (i = 0; i < MAX_THREADS; i++) {
 		ret = pthread_create(&threads[i], NULL, thread_fn, NULL);
-		if (ret)
-			ksft_exit_fail_msg("pthread_create failed: %m\n");
+		ASSERT_EQ(ret, 0)
+			TH_LOG("pthread_create failed: %s", strerror(errno));
 	}
 }
 
-static void join_max_threads(void)
+static void join_max_threads(struct __test_metadata *_metadata)
 {
 	int i, ret;
 
 	for (i = 0; i < MAX_THREADS; i++) {
 		ret = pthread_join(threads[i], NULL);
-		if (ret)
-			ksft_exit_fail_msg("pthread_join failed for thread %d\n", i);
+		ASSERT_EQ(ret, 0)
+			TH_LOG("pthread_join failed for thread %d: %s", i, strerror(errno));
 	}
 }
 
 #define SEC_IN_NSEC	1000000000
 #define MSEC_IN_NSEC	1000000
 
-static void futex_dummy_op(void)
+static void futex_dummy_op(struct __test_metadata *_metadata)
 {
 	pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
 	struct timespec timeout;
@@ -121,11 +119,11 @@ static void futex_dummy_op(void)
 		timeout.tv_sec++;
 	}
 	ret = pthread_mutex_timedlock(&lock, &timeout);
-	if (ret == 0)
-		ksft_exit_fail_msg("Successfully locked an already locked mutex.\n");
+	ASSERT_NE(ret, 0)
+		TH_LOG("Successfully locked an already locked mutex");
 
-	if (ret != ETIMEDOUT)
-		ksft_exit_fail_msg("pthread_mutex_timedlock() did not timeout: %d.\n", ret);
+	ASSERT_EQ(ret, ETIMEDOUT)
+		TH_LOG("pthread_mutex_timedlock() did not timeout: %d", ret);
 }
 
 static const char *test_msg_auto_create = "Automatic hash bucket init on thread creation.\n";
@@ -140,50 +138,48 @@ TEST(priv_hash)
 	ret = pthread_mutexattr_init(&mutex_attr_pi);
 	ret |= pthread_mutexattr_setprotocol(&mutex_attr_pi, PTHREAD_PRIO_INHERIT);
 	ret |= pthread_mutex_init(&global_lock, &mutex_attr_pi);
-	if (ret != 0) {
-		ksft_exit_fail_msg("Failed to initialize pthread mutex.\n");
-	}
+	ASSERT_EQ(ret, 0)
+		TH_LOG("Failed to initialize pthread mutex");
+
 	/* First thread, expect to be 0, not yet initialized */
 	ret = futex_hash_slots_get();
-	if (ret != 0)
-		ksft_exit_fail_msg("futex_hash_slots_get() failed: %d, %m\n", ret);
+	if (ret < 0 && errno == EINVAL)
+		SKIP(return, "PR_FUTEX_HASH not supported by kernel");
+
+	ASSERT_EQ(ret, 0)
+		TH_LOG("futex_hash_slots_get() failed: %d, %s", ret, strerror(errno));
 
-	ksft_test_result_pass("Basic get slots and immutable status.\n");
 	ret = pthread_create(&threads[0], NULL, thread_return_fn, NULL);
-	if (ret != 0)
-		ksft_exit_fail_msg("pthread_create() failed: %d, %m\n", ret);
+	ASSERT_EQ(ret, 0)
+		TH_LOG("pthread_create() failed: %d, %s", ret, strerror(errno));
 
 	ret = pthread_join(threads[0], NULL);
-	if (ret != 0)
-		ksft_exit_fail_msg("pthread_join() failed: %d, %m\n", ret);
+	ASSERT_EQ(ret, 0)
+		TH_LOG("pthread_join() failed: %d, %s", ret, strerror(errno));
 
 	/* First thread, has to initialize private hash */
 	futex_slots1 = futex_hash_slots_get();
-	if (futex_slots1 <= 0) {
-		ksft_print_msg("Current hash buckets: %d\n", futex_slots1);
-		ksft_exit_fail_msg("%s", test_msg_auto_create);
-	}
-
-	ksft_test_result_pass("%s", test_msg_auto_create);
+	EXPECT_GT(futex_slots1, 0)
+		TH_LOG("Current hash buckets: %d. %s", futex_slots1, test_msg_auto_create);
 
 	online_cpus = sysconf(_SC_NPROCESSORS_ONLN);
 	ret = pthread_barrier_init(&barrier_main, NULL, MAX_THREADS + 1);
-	if (ret != 0)
-		ksft_exit_fail_msg("pthread_barrier_init failed: %m.\n");
+	ASSERT_EQ(ret, 0)
+		TH_LOG("pthread_barrier_init failed: %s", strerror(errno));
 
 	ret = pthread_mutex_lock(&global_lock);
-	if (ret != 0)
-		ksft_exit_fail_msg("pthread_mutex_lock failed: %m.\n");
+	ASSERT_EQ(ret, 0)
+		TH_LOG("pthread_mutex_lock failed: %s", strerror(errno));
 
 	counter = 0;
-	create_max_threads(thread_lock_fn);
+	create_max_threads(_metadata, thread_lock_fn);
 	pthread_barrier_wait(&barrier_main);
 
 	/*
 	 * The current default size of hash buckets is 16. The auto increase
 	 * works only if more than 16 CPUs are available.
 	 */
-	ksft_print_msg("Online CPUs: %d\n", online_cpus);
+	TH_LOG("Online CPUs: %d", online_cpus);
 	if (online_cpus > 16) {
 retry_getslots:
 		futex_slotsn = futex_hash_slots_get();
@@ -200,71 +196,76 @@ TEST(priv_hash)
 			 * sleep for 100ms and issue a futex operation.
 			 */
 			if (retry > 0) {
-				futex_dummy_op();
+				futex_dummy_op(_metadata);
 				goto retry_getslots;
 			}
-			ksft_print_msg("Expected increase of hash buckets but got: %d -> %d\n",
-				       futex_slots1, futex_slotsn);
-			ksft_exit_fail_msg("%s", test_msg_auto_inc);
+			EXPECT_NE(futex_slots1, futex_slotsn) {
+				TH_LOG("Expected increase of hash buckets but got: %d -> %d. %s",
+				       futex_slots1, futex_slotsn, test_msg_auto_inc);
+			}
 		}
-		ksft_test_result_pass("%s", test_msg_auto_inc);
 	} else {
-		ksft_test_result_skip("%s", test_msg_auto_inc);
+		SKIP(return, "Automatic increase with more than 16 CPUs (only %d online)", online_cpus);
 	}
 	ret = pthread_mutex_unlock(&global_lock);
 
 	/* Once the user changes it, it has to be what is set */
-	futex_hash_slots_set_verify(2);
-	futex_hash_slots_set_verify(4);
-	futex_hash_slots_set_verify(8);
-	futex_hash_slots_set_verify(32);
-	futex_hash_slots_set_verify(16);
+	futex_hash_slots_set_verify(_metadata, 2);
+	futex_hash_slots_set_verify(_metadata, 4);
+	futex_hash_slots_set_verify(_metadata, 8);
+	futex_hash_slots_set_verify(_metadata, 32);
+	futex_hash_slots_set_verify(_metadata, 16);
 
 	ret = futex_hash_slots_set(15);
-	ksft_test_result(ret < 0, "Use 15 slots\n");
+	EXPECT_LT(ret, 0)
+		TH_LOG("Use 15 slots should fail but succeeded");
+
+	futex_hash_slots_set_verify(_metadata, 2);
+	join_max_threads(_metadata);
+
+	EXPECT_EQ(counter, MAX_THREADS)
+		TH_LOG("Created and waited for %d of %d threads", counter, MAX_THREADS);
 
-	futex_hash_slots_set_verify(2);
-	join_max_threads();
-	ksft_test_result(counter == MAX_THREADS, "Created and waited for %d of %d threads\n",
-			 counter, MAX_THREADS);
 	counter = 0;
 	/* Once the user set something, auto resize must be disabled */
 	ret = pthread_barrier_init(&barrier_main, NULL, MAX_THREADS);
+	ASSERT_EQ(ret, 0)
+		TH_LOG("pthread_barrier_init failed: %s", strerror(errno));
 
-	create_max_threads(thread_lock_fn);
-	join_max_threads();
+	create_max_threads(_metadata, thread_lock_fn);
+	join_max_threads(_metadata);
 
 	ret = futex_hash_slots_get();
-	ksft_test_result(ret == 2, "No more auto-resize after manual setting, got %d\n",
-			 ret);
+	EXPECT_EQ(ret, 2)
+		TH_LOG("No more auto-resize after manual setting, got %d", ret);
 
-	futex_hash_slots_set_must_fail(1 << 29);
-	futex_hash_slots_set_verify(4);
+	futex_hash_slots_set_must_fail(_metadata, 1 << 29);
+	futex_hash_slots_set_verify(_metadata, 4);
 
 	/*
 	 * Once the global hash has been requested, then this requested can not
 	 * be undone.
 	 */
 	ret = futex_hash_slots_set(0);
-	ksft_test_result(ret == 0, "Global hash request\n");
-	if (ret != 0)
-		return;
+	ASSERT_EQ(ret, 0)
+		TH_LOG("Global hash request failed: %s", strerror(errno));
 
-	futex_hash_slots_set_must_fail(4);
-	futex_hash_slots_set_must_fail(8);
-	futex_hash_slots_set_must_fail(8);
-	futex_hash_slots_set_must_fail(0);
-	futex_hash_slots_set_must_fail(6);
+	futex_hash_slots_set_must_fail(_metadata, 4);
+	futex_hash_slots_set_must_fail(_metadata, 8);
+	futex_hash_slots_set_must_fail(_metadata, 8);
+	futex_hash_slots_set_must_fail(_metadata, 0);
+	futex_hash_slots_set_must_fail(_metadata, 6);
 
 	ret = pthread_barrier_init(&barrier_main, NULL, MAX_THREADS);
-	if (ret != 0)
-		ksft_exit_fail_msg("pthread_barrier_init failed: %m\n");
+	ASSERT_EQ(ret, 0)
+		TH_LOG("pthread_barrier_init failed: %s", strerror(errno));
 
-	create_max_threads(thread_lock_fn);
-	join_max_threads();
+	create_max_threads(_metadata, thread_lock_fn);
+	join_max_threads(_metadata);
 
 	ret = futex_hash_slots_get();
-	ksft_test_result(ret == 0, "Continue to use global hash\n");
+	EXPECT_EQ(ret, 0)
+		TH_LOG("Continue to use global hash failed");
 }
 
 TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/futex/functional/futex_requeue.c b/tools/testing/selftests/futex/functional/futex_requeue.c
index dcf0d5f2f3122..22e704610a859 100644
--- a/tools/testing/selftests/futex/functional/futex_requeue.c
+++ b/tools/testing/selftests/futex/functional/futex_requeue.c
@@ -5,55 +5,61 @@
  * futex cmp requeue test by André Almeida <andrealmeid@collabora.com>
  */
 
-#include <pthread.h>
 #include <limits.h>
+#include <pthread.h>
+#include <string.h>
 
 #include "futextest.h"
+#include "futex_thread.h"
 #include "kselftest_harness.h"
 
-#define timeout_ns  30000000
-#define WAKE_WAIT_US 10000
+#define FUTEX_WAIT_TIMEOUT_SECS			2
 
 volatile futex_t *f1;
 
-void *waiterfn(void *arg)
+static int waiterfn(void *arg)
 {
-	struct timespec to;
-
-	to.tv_sec = 0;
-	to.tv_nsec = timeout_ns;
+	struct __test_metadata *_metadata = (struct __test_metadata *)arg;
+	struct timespec to = { .tv_sec = FUTEX_WAIT_TIMEOUT_SECS };
+	int res;
 
-	if (futex_wait(f1, *f1, &to, 0))
-		printf("waiter failed errno %d\n", errno);
+	res = futex_wait(f1, *f1, &to, 0);
+	if (res) {
+		EXPECT_EQ(res, 0)
+			TH_LOG("waiter failed errno %d: %s", errno, strerror(errno));
+	}
 
-	return NULL;
+	return 0;
 }
 
 TEST(requeue_single)
 {
+	struct futex_thread waiter;
 	volatile futex_t _f1 = 0;
 	volatile futex_t f2 = 0;
-	pthread_t waiter[10];
 
 	f1 = &_f1;
 
 	/*
 	 * Requeue a waiter from f1 to f2, and wake f2.
 	 */
-	ASSERT_EQ(0, pthread_create(&waiter[0], NULL, waiterfn, NULL));
+	ASSERT_EQ(futex_thread_create(&waiter, waiterfn, _metadata), 0)
+		TH_LOG("pthread_create failed");
 
-	usleep(WAKE_WAIT_US);
+	ASSERT_EQ(futex_wait_for_thread(&waiter, _metadata), 0)
+		TH_LOG("Wait for thread failed");
 
-	EXPECT_EQ(1, futex_cmp_requeue(f1, 0, &f2, 0, 1, 0));
-	EXPECT_EQ(1, futex_wake(&f2, 1, 0));
+	EXPECT_EQ(futex_cmp_requeue(f1, 0, &f2, 0, 1, 0), 1);
+	EXPECT_EQ(futex_wake(&f2, 1, 0), 1);
+
+	EXPECT_EQ(futex_thread_destroy(&waiter), 0);
 }
 
 TEST(requeue_multiple)
 {
+	struct futex_thread waiter[10];
 	volatile futex_t _f1 = 0;
 	volatile futex_t f2 = 0;
-	pthread_t waiter[10];
-	int i;
 
 	f1 = &_f1;
 
@@ -61,13 +67,21 @@ TEST(requeue_multiple)
 	 * Create 10 waiters at f1. At futex_requeue, wake 3 and requeue 7.
 	 * At futex_wake, wake INT_MAX (should be exactly 7).
 	 */
-	for (i = 0; i < 10; i++)
-		ASSERT_EQ(0, pthread_create(&waiter[i], NULL, waiterfn, NULL));
+	for (int i = 0; i < 10; i++) {
+		ASSERT_EQ(futex_thread_create(&waiter[i], waiterfn, _metadata), 0)
+			TH_LOG("pthread_create failed for waiter %d", i);
+	}
+
+	for (int i = 0; i < 10; i++) {
+		ASSERT_EQ(futex_wait_for_thread(&waiter[i], _metadata), 0)
+			TH_LOG("Wait for waiter thread %d failed", i);
+	}
 
-	usleep(WAKE_WAIT_US);
+	EXPECT_EQ(futex_cmp_requeue(f1, 0, &f2, 3, 7, 0), 10);
+	EXPECT_EQ(futex_wake(&f2, INT_MAX, 0), 7);
 
-	EXPECT_EQ(10, futex_cmp_requeue(f1, 0, &f2, 3, 7, 0));
-	EXPECT_EQ(7, futex_wake(&f2, INT_MAX, 0));
+	for (int i = 0; i < 10; i++)
+		EXPECT_EQ(futex_thread_destroy(&waiter[i]), 0);
 }
 
 TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/futex/functional/futex_requeue_pi.c b/tools/testing/selftests/futex/functional/futex_requeue_pi.c
index 46d2858e15a86..65326cbef3a44 100644
--- a/tools/testing/selftests/futex/functional/futex_requeue_pi.c
+++ b/tools/testing/selftests/futex/functional/futex_requeue_pi.c
@@ -43,12 +43,13 @@ futex_t f2 = FUTEX_INITIALIZER;
 futex_t wake_complete = FUTEX_INITIALIZER;
 
 struct thread_arg {
-	long id;
-	struct timespec *timeout;
-	int lock;
-	int ret;
+	struct __test_metadata	*_metadata;
+	long			id;
+	struct timespec		*timeout;
+	int			lock;
+	int			ret;
 };
-#define THREAD_ARG_INITIALIZER { 0, NULL, 0, 0 }
+#define THREAD_ARG_INITIALIZER { NULL, 0, NULL, 0, 0 }
 
 FIXTURE(args)
 {
@@ -118,7 +119,7 @@ FIXTURE_VARIANT_ADD_TIMEOUT(5000);
 FIXTURE_VARIANT_ADD_TIMEOUT(500000);
 FIXTURE_VARIANT_ADD_TIMEOUT(2000000000);
 
-int create_rt_thread(pthread_t *pth, void*(*func)(void *), void *arg,
+int create_rt_thread(struct __test_metadata *_metadata, pthread_t *pth, void*(*func)(void *), void *arg,
 		     int policy, int prio)
 {
 	int ret;
@@ -129,29 +130,22 @@ int create_rt_thread(pthread_t *pth, void*(*func)(void *), void *arg,
 	memset(&schedp, 0, sizeof(schedp));
 
 	ret = pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED);
-	if (ret) {
-		ksft_exit_fail_msg("pthread_attr_setinheritsched\n");
-		return -1;
-	}
+	ASSERT_EQ(ret, 0)
+		TH_LOG("pthread_attr_setinheritsched failed");
 
 	ret = pthread_attr_setschedpolicy(&attr, policy);
-	if (ret) {
-		ksft_exit_fail_msg("pthread_attr_setschedpolicy\n");
-		return -1;
-	}
+	ASSERT_EQ(ret, 0)
+		TH_LOG("pthread_attr_setschedpolicy failed");
 
 	schedp.sched_priority = prio;
 	ret = pthread_attr_setschedparam(&attr, &schedp);
-	if (ret) {
-		ksft_exit_fail_msg("pthread_attr_setschedparam\n");
-		return -1;
-	}
+	ASSERT_EQ(ret, 0)
+		TH_LOG("pthread_attr_setschedparam failed");
 
 	ret = pthread_create(pth, &attr, func, arg);
-	if (ret) {
-		ksft_exit_fail_msg("pthread_create\n");
-		return -1;
-	}
+	ASSERT_EQ(ret, 0)
+		TH_LOG("pthread_create failed");
+
 	return 0;
 }
 
@@ -159,70 +153,76 @@ int create_rt_thread(pthread_t *pth, void*(*func)(void *), void *arg,
 void *waiterfn(void *arg)
 {
 	struct thread_arg *args = (struct thread_arg *)arg;
+	struct __test_metadata *_metadata = args->_metadata;
 	futex_t old_val;
 
-	ksft_print_dbg_msg("Waiter %ld: running\n", args->id);
+	TH_LOG("Waiter %ld: running", args->id);
 	/* Each thread sleeps for a different amount of time
 	 * This is to avoid races, because we don't lock the
-	 * external mutex here */
+	 * external mutex here
+	 */
 	usleep(1000 * (long)args->id);
 
 	old_val = f1;
 	atomic_inc(&waiters_blocked);
-	ksft_print_dbg_msg("Calling futex_wait_requeue_pi: %p (%u) -> %p\n",
-	     &f1, f1, &f2);
+	TH_LOG("Calling futex_wait_requeue_pi: %p (%u) -> %p", &f1, f1, &f2);
 	args->ret = futex_wait_requeue_pi(&f1, old_val, &f2, args->timeout,
 					  FUTEX_PRIVATE_FLAG);
 
-	ksft_print_dbg_msg("waiter %ld woke with %d %s\n", args->id, args->ret,
-	     args->ret < 0 ? strerror(errno) : "");
+	TH_LOG("waiter %ld woke with %d %s", args->id, args->ret,
+	       args->ret < 0 ? strerror(errno) : "");
 	atomic_inc(&waiters_woken);
 	if (args->ret < 0) {
-		if (args->timeout && errno == ETIMEDOUT)
+		if (args->timeout && errno == ETIMEDOUT) {
 			args->ret = 0;
-		else {
-			ksft_exit_fail_msg("futex_wait_requeue_pi\n");
+		} else {
+			ASSERT_EQ(args->ret, 0)
+				TH_LOG("futex_wait_requeue_pi failed: %s", strerror(errno));
 		}
 		futex_lock_pi(&f2, NULL, 0, FUTEX_PRIVATE_FLAG);
 	}
 	futex_unlock_pi(&f2, FUTEX_PRIVATE_FLAG);
 
-	ksft_print_dbg_msg("Waiter %ld: exiting with %d\n", args->id, args->ret);
+	TH_LOG("Waiter %ld: exiting with %d", args->id, args->ret);
 	pthread_exit((void *)&args->ret);
 }
 
 void *broadcast_wakerfn(void *arg)
 {
 	struct thread_arg *args = (struct thread_arg *)arg;
+	struct __test_metadata *_metadata = args->_metadata;
 	int nr_requeue = INT_MAX;
 	int task_count = 0;
 	futex_t old_val;
 	int nr_wake = 1;
 	int i = 0;
 
-	ksft_print_dbg_msg("Waker: waiting for waiters to block\n");
+	TH_LOG("Waker: waiting for waiters to block");
 	while (waiters_blocked.val < THREAD_MAX)
 		usleep(1000);
 	usleep(1000);
 
-	ksft_print_dbg_msg("Waker: Calling broadcast\n");
+	TH_LOG("Waker: Calling broadcast");
 	if (args->lock) {
-		ksft_print_dbg_msg("Calling FUTEX_LOCK_PI on mutex=%x @ %p\n", f2, &f2);
+		TH_LOG("Calling FUTEX_LOCK_PI on mutex=%x @ %p", f2, &f2);
 		futex_lock_pi(&f2, NULL, 0, FUTEX_PRIVATE_FLAG);
 	}
  continue_requeue:
 	old_val = f1;
 	args->ret = futex_cmp_requeue_pi(&f1, old_val, &f2, nr_wake, nr_requeue,
-				   FUTEX_PRIVATE_FLAG);
+					 FUTEX_PRIVATE_FLAG);
 	if (args->ret < 0) {
-		ksft_exit_fail_msg("FUTEX_CMP_REQUEUE_PI failed\n");
+		ASSERT_GE(args->ret, 0)
+			TH_LOG("FUTEX_CMP_REQUEUE_PI failed: %s", strerror(errno));
 	} else if (++i < MAX_WAKE_ITERS) {
 		task_count += args->ret;
 		if (task_count < THREAD_MAX - waiters_woken.val)
 			goto continue_requeue;
 	} else {
-		ksft_exit_fail_msg("max broadcast iterations (%d) reached with %d/%d tasks woken or requeued\n",
-		       MAX_WAKE_ITERS, task_count, THREAD_MAX);
+		ASSERT_TRUE(0) {
+			TH_LOG("max broadcast iterations (%d) reached with %d/%d tasks woken or requeued",
+			       MAX_WAKE_ITERS, task_count, THREAD_MAX);
+		}
 	}
 
 	futex_wake(&wake_complete, 1, FUTEX_PRIVATE_FLAG);
@@ -233,33 +233,33 @@ void *broadcast_wakerfn(void *arg)
 	if (args->ret > 0)
 		args->ret = task_count;
 
-	ksft_print_dbg_msg("Waker: exiting with %d\n", args->ret);
+	TH_LOG("Waker: exiting with %d", args->ret);
 	pthread_exit((void *)&args->ret);
 }
 
 void *signal_wakerfn(void *arg)
 {
 	struct thread_arg *args = (struct thread_arg *)arg;
+	struct __test_metadata *_metadata = args->_metadata;
 	unsigned int old_val;
 	int nr_requeue = 0;
 	int task_count = 0;
 	int nr_wake = 1;
 	int i = 0;
 
-	ksft_print_dbg_msg("Waker: waiting for waiters to block\n");
+	TH_LOG("Waker: waiting for waiters to block");
 	while (waiters_blocked.val < THREAD_MAX)
 		usleep(1000);
 	usleep(1000);
 
 	while (task_count < THREAD_MAX && waiters_woken.val < THREAD_MAX) {
-		ksft_print_dbg_msg("task_count: %d, waiters_woken: %d\n",
+		TH_LOG("task_count: %d, waiters_woken: %d",
 		     task_count, waiters_woken.val);
 		if (args->lock) {
-			ksft_print_dbg_msg("Calling FUTEX_LOCK_PI on mutex=%x @ %p\n",
-			    f2, &f2);
+			TH_LOG("Calling FUTEX_LOCK_PI on mutex=%x @ %p", f2, &f2);
 			futex_lock_pi(&f2, NULL, 0, FUTEX_PRIVATE_FLAG);
 		}
-		ksft_print_dbg_msg("Waker: Calling signal\n");
+		TH_LOG("Waker: Calling signal");
 		/* cond_signal */
 		old_val = f1;
 		args->ret = futex_cmp_requeue_pi(&f1, old_val, &f2,
@@ -267,23 +267,27 @@ void *signal_wakerfn(void *arg)
 						 FUTEX_PRIVATE_FLAG);
 		if (args->ret < 0)
 			args->ret = -errno;
-		ksft_print_dbg_msg("futex: %x\n", f2);
+		TH_LOG("futex: %x", f2);
 		if (args->lock) {
-			ksft_print_dbg_msg("Calling FUTEX_UNLOCK_PI on mutex=%x @ %p\n",
+			TH_LOG("Calling FUTEX_UNLOCK_PI on mutex=%x @ %p",
 			    f2, &f2);
 			futex_unlock_pi(&f2, FUTEX_PRIVATE_FLAG);
 		}
-		ksft_print_dbg_msg("futex: %x\n", f2);
-		if (args->ret < 0)
-			ksft_exit_fail_msg("FUTEX_CMP_REQUEUE_PI failed\n");
+		TH_LOG("futex: %x", f2);
+		if (args->ret < 0) {
+			ASSERT_GE(args->ret, 0)
+				TH_LOG("FUTEX_CMP_REQUEUE_PI failed: %s", strerror(-args->ret));
+		}
 
 		task_count += args->ret;
 		usleep(SIGNAL_PERIOD_US);
 		i++;
 		/* we have to loop at least THREAD_MAX times */
 		if (i > MAX_WAKE_ITERS + THREAD_MAX) {
-			ksft_exit_fail_msg("max signaling iterations (%d) reached, giving up on pending waiters.\n",
-			      MAX_WAKE_ITERS + THREAD_MAX);
+			ASSERT_TRUE(0) {
+				TH_LOG("max signaling iterations (%d) reached, giving up on pending waiters.",
+				       MAX_WAKE_ITERS + THREAD_MAX);
+			}
 		}
 	}
 
@@ -292,14 +296,15 @@ void *signal_wakerfn(void *arg)
 	if (args->ret >= 0)
 		args->ret = task_count;
 
-	ksft_print_dbg_msg("Waker: exiting with %d\n", args->ret);
-	ksft_print_dbg_msg("Waker: waiters_woken: %d\n", waiters_woken.val);
+	TH_LOG("Waker: exiting with %d", args->ret);
+	TH_LOG("Waker: waiters_woken: %d", waiters_woken.val);
 	pthread_exit((void *)&args->ret);
 }
 
 void *third_party_blocker(void *arg)
 {
 	struct thread_arg *args = (struct thread_arg *)arg;
+	struct __test_metadata *_metadata = args->_metadata;
 	int ret2 = 0;
 
 	args->ret = futex_lock_pi(&f2, NULL, 0, FUTEX_PRIVATE_FLAG);
@@ -310,8 +315,10 @@ void *third_party_blocker(void *arg)
 	ret2 = futex_unlock_pi(&f2, FUTEX_PRIVATE_FLAG);
 
  out:
-	if (args->ret || ret2)
-		ksft_exit_fail_msg("third_party_blocker() futex error");
+	if (args->ret || ret2) {
+		ASSERT_TRUE(0)
+			TH_LOG("%s() futex error", __func__);
+	}
 
 	pthread_exit((void *)&args->ret);
 }
@@ -330,20 +337,19 @@ TEST_F(args, futex_requeue_pi)
 	bool lock = variant->locked;
 	int *waiter_ret, i, ret = 0;
 
-	ksft_print_msg(
-		"\tArguments: broadcast=%d locked=%d owner=%d timeout=%ldns\n",
+	TH_LOG("Arguments: broadcast=%d locked=%d owner=%d timeout=%ldns",
 		broadcast, lock, third_party_owner, timeout_ns);
 
 	if (timeout_ns) {
 		time_t secs;
 
-		ksft_print_dbg_msg("timeout_ns = %ld\n", timeout_ns);
+		TH_LOG("timeout_ns = %ld", timeout_ns);
 		ret = clock_gettime(CLOCK_MONOTONIC, &ts);
 		secs = (ts.tv_nsec + timeout_ns) / 1000000000;
 		ts.tv_nsec = ((int64_t)ts.tv_nsec + timeout_ns) % 1000000000;
 		ts.tv_sec += secs;
-		ksft_print_dbg_msg("ts.tv_sec  = %ld\n", ts.tv_sec);
-		ksft_print_dbg_msg("ts.tv_nsec = %ld\n", ts.tv_nsec);
+		TH_LOG("ts.tv_sec  = %ld", ts.tv_sec);
+		TH_LOG("ts.tv_nsec = %ld", ts.tv_nsec);
 		tsp = &ts;
 	}
 
@@ -351,34 +357,29 @@ TEST_F(args, futex_requeue_pi)
 		wakerfn = broadcast_wakerfn;
 
 	if (third_party_owner) {
-		if (create_rt_thread(&blocker, third_party_blocker,
-				     (void *)&blocker_arg, SCHED_FIFO, 1)) {
-			ksft_exit_fail_msg("Creating third party blocker thread failed\n");
-		}
+		blocker_arg._metadata = _metadata;
+		create_rt_thread(_metadata, &blocker, third_party_blocker,
+				 (void *)&blocker_arg, SCHED_FIFO, 1);
 	}
 
 	atomic_set(&waiters_woken, 0);
 	for (i = 0; i < THREAD_MAX; i++) {
+		args[i]._metadata = _metadata;
 		args[i].id = i;
 		args[i].timeout = tsp;
-		ksft_print_dbg_msg("Starting thread %d\n", i);
-		if (create_rt_thread(&waiter[i], waiterfn, (void *)&args[i],
-				     SCHED_FIFO, 1)) {
-			ksft_exit_fail_msg("Creating waiting thread failed\n");
-		}
+		TH_LOG("Starting thread %d", i);
+		create_rt_thread(_metadata, &waiter[i], waiterfn, (void *)&args[i],
+				 SCHED_FIFO, 1);
 	}
+	waker_arg._metadata = _metadata;
 	waker_arg.lock = lock;
-	if (create_rt_thread(&waker, wakerfn, (void *)&waker_arg,
-			     SCHED_FIFO, 1)) {
-		ksft_exit_fail_msg("Creating waker thread failed\n");
-	}
+	create_rt_thread(_metadata, &waker, wakerfn, (void *)&waker_arg, SCHED_FIFO, 1);
 
 	/* Wait for threads to finish */
 	/* Store the first error or failure encountered in waiter_ret */
 	waiter_ret = &args[0].ret;
 	for (i = 0; i < THREAD_MAX; i++)
-		pthread_join(waiter[i],
-			     *waiter_ret ? NULL : (void **)&waiter_ret);
+		pthread_join(waiter[i], *waiter_ret ? NULL : (void **)&waiter_ret);
 
 	if (third_party_owner)
 		pthread_join(blocker, NULL);
@@ -393,8 +394,8 @@ TEST_F(args, futex_requeue_pi)
 			ret = blocker_arg.ret;
 	}
 
-	if (ret)
-		ksft_test_result_fail("fail");
+	EXPECT_EQ(ret, 0)
+		TH_LOG("Test failed with error code: %d", ret);
 }
 
 TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/futex/functional/futex_requeue_pi_mismatched_ops.c b/tools/testing/selftests/futex/functional/futex_requeue_pi_mismatched_ops.c
index f686e605359ca..35bb8a807bb9c 100644
--- a/tools/testing/selftests/futex/functional/futex_requeue_pi_mismatched_ops.c
+++ b/tools/testing/selftests/futex/functional/futex_requeue_pi_mismatched_ops.c
@@ -29,14 +29,17 @@
 
 futex_t f1 = FUTEX_INITIALIZER;
 futex_t f2 = FUTEX_INITIALIZER;
-int child_ret = 0;
+int child_ret;
 
 void *blocking_child(void *arg)
 {
+	struct __test_metadata *_metadata = (struct __test_metadata *)arg;
+
 	child_ret = futex_wait(&f1, f1, NULL, FUTEX_PRIVATE_FLAG);
 	if (child_ret < 0) {
 		child_ret = -errno;
-		ksft_exit_fail_msg("futex_wait\n");
+		ASSERT_EQ(child_ret, 0)
+			TH_LOG("futex_wait failed: %s", strerror(errno));
 	}
 	return (void *)&child_ret;
 }
@@ -46,8 +49,8 @@ TEST(requeue_pi_mismatched_ops)
 	pthread_t child;
 	int ret;
 
-	if (pthread_create(&child, NULL, blocking_child, NULL))
-		ksft_exit_fail_msg("pthread_create\n");
+	ASSERT_EQ(pthread_create(&child, NULL, blocking_child, _metadata), 0)
+		TH_LOG("pthread_create failed");
 
 	/* Allow the child to block in the kernel. */
 	sleep(1);
@@ -67,27 +70,33 @@ TEST(requeue_pi_mismatched_ops)
 			 * FUTEX_WAKE.
 			 */
 			ret = futex_wake(&f1, 1, FUTEX_PRIVATE_FLAG);
-			if (ret == 1)
+			if (ret == 1) {
 				ret = 0;
-			else if (ret < 0)
-				ksft_exit_fail_msg("futex_wake\n");
-			else
-				ksft_exit_fail_msg("futex_wake did not wake the child\n");
+			} else if (ret < 0) {
+				ASSERT_GE(ret, 0)
+					TH_LOG("futex_wake failed: %s", strerror(errno));
+			} else {
+				ASSERT_TRUE(0)
+					TH_LOG("futex_wake did not wake the child");
+			}
 		} else {
-			ksft_exit_fail_msg("futex_cmp_requeue_pi\n");
+			ASSERT_TRUE(0)
+				TH_LOG("futex_cmp_requeue_pi failed with unexpected errno: %s", strerror(errno));
 		}
 	} else if (ret > 0) {
-		ksft_test_result_fail("futex_cmp_requeue_pi failed to detect the mismatch\n");
+		EXPECT_EQ(ret, 0)
+			TH_LOG("futex_cmp_requeue_pi failed to detect the mismatch");
 	} else {
-		ksft_exit_fail_msg("futex_cmp_requeue_pi found no waiters\n");
+		ASSERT_TRUE(0)
+			TH_LOG("futex_cmp_requeue_pi found no waiters");
 	}
 
 	pthread_join(child, NULL);
 
-	if (!ret && !child_ret)
-		ksft_test_result_pass("futex_requeue_pi_mismatched_ops passed\n");
-	else
-		ksft_test_result_pass("futex_requeue_pi_mismatched_ops failed\n");
+	EXPECT_EQ(ret, 0)
+		TH_LOG("Test failed: ret=%d", ret);
+	EXPECT_EQ(child_ret, 0)
+		TH_LOG("Child failed: child_ret=%d", child_ret);
 }
 
 TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c b/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c
index a18ccae73eb15..4933612a7b55c 100644
--- a/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c
+++ b/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c
@@ -35,10 +35,10 @@ futex_t f1 = FUTEX_INITIALIZER;
 futex_t f2 = FUTEX_INITIALIZER;
 atomic_t requeued = ATOMIC_INITIALIZER;
 
-int waiter_ret = 0;
+int waiter_ret;
 
-int create_rt_thread(pthread_t *pth, void*(*func)(void *), void *arg,
-		     int policy, int prio)
+int create_rt_thread(struct __test_metadata *_metadata, pthread_t *pth, void*(*func)(void *),
+		     void *arg, int policy, int prio)
 {
 	struct sched_param schedp;
 	pthread_attr_t attr;
@@ -48,45 +48,47 @@ int create_rt_thread(pthread_t *pth, void*(*func)(void *), void *arg,
 	memset(&schedp, 0, sizeof(schedp));
 
 	ret = pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED);
-	if (ret)
-		ksft_exit_fail_msg("pthread_attr_setinheritsched\n");
+	ASSERT_EQ(ret, 0)
+		TH_LOG("pthread_attr_setinheritsched failed");
 
 	ret = pthread_attr_setschedpolicy(&attr, policy);
-	if (ret)
-		ksft_exit_fail_msg("pthread_attr_setschedpolicy\n");
+	ASSERT_EQ(ret, 0)
+		TH_LOG("pthread_attr_setschedpolicy failed");
 
 	schedp.sched_priority = prio;
 	ret = pthread_attr_setschedparam(&attr, &schedp);
-	if (ret)
-		ksft_exit_fail_msg("pthread_attr_setschedparam\n");
+	ASSERT_EQ(ret, 0)
+		TH_LOG("pthread_attr_setschedparam failed");
 
 	ret = pthread_create(pth, &attr, func, arg);
-	if (ret)
-		ksft_exit_fail_msg("pthread_create\n");
+	ASSERT_EQ(ret, 0)
+		TH_LOG("pthread_create failed");
 
 	return 0;
 }
 
 void handle_signal(int signo)
 {
-	ksft_print_dbg_msg("signal received %s requeue\n",
-	     requeued.val ? "after" : "prior to");
+	printf("INFO: signal received %s requeue\n", requeued.val ? "after" : "prior to");
 }
 
 void *waiterfn(void *arg)
 {
+	struct __test_metadata *_metadata = (struct __test_metadata *)arg;
 	unsigned int old_val;
 	int res;
 
-	ksft_print_dbg_msg("Waiter running\n");
-	ksft_print_dbg_msg("Calling FUTEX_LOCK_PI on f2=%x @ %p\n", f2, &f2);
+	TH_LOG("Waiter running");
+	TH_LOG("Calling FUTEX_LOCK_PI on f2=%x @ %p", f2, &f2);
 	old_val = f1;
 	res = futex_wait_requeue_pi(&f1, old_val, &(f2), NULL,
 				    FUTEX_PRIVATE_FLAG);
 	if (!requeued.val || errno != EWOULDBLOCK) {
-		ksft_test_result_fail("unexpected return from futex_wait_requeue_pi: %d (%s)\n",
-		     res, strerror(errno));
-		ksft_print_dbg_msg("w2:futex: %x\n", f2);
+		EXPECT_TRUE(0) {
+			TH_LOG("unexpected return from futex_wait_requeue_pi: %d (%s)",
+			       res, strerror(errno));
+		}
+		TH_LOG("w2:futex: %x", f2);
 		if (!res)
 			futex_unlock_pi(&f2, FUTEX_PRIVATE_FLAG);
 	}
@@ -94,7 +96,6 @@ void *waiterfn(void *arg)
 	pthread_exit(NULL);
 }
 
-
 TEST(futex_requeue_pi_signal_restart)
 {
 	unsigned int old_val;
@@ -105,19 +106,17 @@ TEST(futex_requeue_pi_signal_restart)
 	sa.sa_handler = handle_signal;
 	sigemptyset(&sa.sa_mask);
 	sa.sa_flags = 0;
-	if (sigaction(SIGUSR1, &sa, NULL))
-		ksft_exit_fail_msg("sigaction\n");
+	ASSERT_EQ(sigaction(SIGUSR1, &sa, NULL), 0)
+		TH_LOG("sigaction failed");
 
-	ksft_print_dbg_msg("m1:f2: %x\n", f2);
-	ksft_print_dbg_msg("Creating waiter\n");
-	res = create_rt_thread(&waiter, waiterfn, NULL, SCHED_FIFO, 1);
-	if (res)
-		ksft_exit_fail_msg("Creating waiting thread failed");
+	TH_LOG("m1:f2: %x", f2);
+	TH_LOG("Creating waiter");
+	create_rt_thread(_metadata, &waiter, waiterfn, _metadata, SCHED_FIFO, 1);
 
-	ksft_print_dbg_msg("Calling FUTEX_LOCK_PI on f2=%x @ %p\n", f2, &f2);
-	ksft_print_dbg_msg("m2:f2: %x\n", f2);
+	TH_LOG("Calling FUTEX_LOCK_PI on f2=%x @ %p", f2, &f2);
+	TH_LOG("m2:f2: %x", f2);
 	futex_lock_pi(&f2, 0, 0, FUTEX_PRIVATE_FLAG);
-	ksft_print_dbg_msg("m3:f2: %x\n", f2);
+	TH_LOG("m3:f2: %x", f2);
 
 	while (1) {
 		/*
@@ -125,11 +124,11 @@ TEST(futex_requeue_pi_signal_restart)
 		 * restart futex_wait_requeue_pi() in the kernel. Wait for the
 		 * waiter to block on f1 again.
 		 */
-		ksft_print_dbg_msg("Issuing SIGUSR1 to waiter\n");
+		TH_LOG("Issuing SIGUSR1 to waiter");
 		pthread_kill(waiter, SIGUSR1);
 		usleep(DELAY_US);
 
-		ksft_print_dbg_msg("Requeueing waiter via FUTEX_CMP_REQUEUE_PI\n");
+		TH_LOG("Requeueing waiter via FUTEX_CMP_REQUEUE_PI");
 		old_val = f1;
 		res = futex_cmp_requeue_pi(&f1, old_val, &(f2), 1, 0,
 					   FUTEX_PRIVATE_FLAG);
@@ -143,10 +142,11 @@ TEST(futex_requeue_pi_signal_restart)
 			atomic_set(&requeued, 1);
 			break;
 		} else if (res < 0) {
-			ksft_exit_fail_msg("FUTEX_CMP_REQUEUE_PI failed\n");
+			ASSERT_GE(res, 0)
+				TH_LOG("FUTEX_CMP_REQUEUE_PI failed: %s", strerror(errno));
 		}
 	}
-	ksft_print_dbg_msg("m4:f2: %x\n", f2);
+	TH_LOG("m4:f2: %x", f2);
 
 	/*
 	 * Signal the waiter after requeue, waiter should return from
@@ -154,14 +154,14 @@ TEST(futex_requeue_pi_signal_restart)
 	 * futex_unlock_pi() can't happen before the signal wakeup is detected
 	 * in the kernel.
 	 */
-	ksft_print_dbg_msg("Issuing SIGUSR1 to waiter\n");
+	TH_LOG("Issuing SIGUSR1 to waiter");
 	pthread_kill(waiter, SIGUSR1);
-	ksft_print_dbg_msg("Waiting for waiter to return\n");
+	TH_LOG("Waiting for waiter to return");
 	pthread_join(waiter, NULL);
 
-	ksft_print_dbg_msg("Calling FUTEX_UNLOCK_PI on mutex=%x @ %p\n", f2, &f2);
+	TH_LOG("Calling FUTEX_UNLOCK_PI on mutex=%x @ %p", f2, &f2);
 	futex_unlock_pi(&f2, FUTEX_PRIVATE_FLAG);
-	ksft_print_dbg_msg("m5:f2: %x\n", f2);
+	TH_LOG("m5:f2: %x", f2);
 }
 
 TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/futex/functional/futex_wait.c b/tools/testing/selftests/futex/functional/futex_wait.c
index 7b88794090071..5292f5d2c3b58 100644
--- a/tools/testing/selftests/futex/functional/futex_wait.c
+++ b/tools/testing/selftests/futex/functional/futex_wait.c
@@ -5,10 +5,11 @@
  * futex cmp requeue test by André Almeida <andrealmeid@collabora.com>
  */
 
+#include <fcntl.h>
 #include <pthread.h>
+#include <stdlib.h>
 #include <sys/shm.h>
 #include <sys/mman.h>
-#include <fcntl.h>
 
 #include "futextest.h"
 #include "kselftest_harness.h"
@@ -19,125 +20,157 @@
 
 void *futex;
 
+struct waiter_args {
+	struct __test_metadata	*_metadata;
+	unsigned int		flags;
+};
+
 static void *waiterfn(void *arg)
 {
+	struct waiter_args *args = (struct waiter_args *)arg;
+	struct __test_metadata *_metadata = args->_metadata;
 	struct timespec to;
-	unsigned int flags = 0;
-
-	if (arg)
-		flags = *((unsigned int *) arg);
+	int res;
 
 	to.tv_sec = 0;
 	to.tv_nsec = timeout_ns;
 
-	if (futex_wait(futex, 0, &to, flags))
-		printf("waiter failed errno %d\n", errno);
+	res = futex_wait(futex, 0, &to, args->flags);
+	if (res) {
+		EXPECT_EQ(res, 0)
+			TH_LOG("waiter failed errno %d: %s", errno, strerror(errno));
+	}
 
+	free(args);
 	return NULL;
 }
 
 TEST(private_futex)
 {
-	unsigned int flags = FUTEX_PRIVATE_FLAG;
+	struct waiter_args *args = malloc(sizeof(*args));
 	u_int32_t f_private = 0;
 	pthread_t waiter;
 	int res;
 
+	args->_metadata = _metadata;
+	args->flags = FUTEX_PRIVATE_FLAG;
 	futex = &f_private;
 
 	/* Testing a private futex */
-	ksft_print_dbg_msg("Calling private futex_wait on futex: %p\n", futex);
-	if (pthread_create(&waiter, NULL, waiterfn, (void *) &flags))
-		ksft_exit_fail_msg("pthread_create failed\n");
+	TH_LOG("Calling private futex_wait on futex: %p", futex);
+	ASSERT_EQ(pthread_create(&waiter, NULL, waiterfn, args), 0)
+		TH_LOG("pthread_create failed");
 
 	usleep(WAKE_WAIT_US);
 
-	ksft_print_dbg_msg("Calling private futex_wake on futex: %p\n", futex);
+	TH_LOG("Calling private futex_wake on futex: %p", futex);
 	res = futex_wake(futex, 1, FUTEX_PRIVATE_FLAG);
-	if (res != 1) {
-		ksft_test_result_fail("futex_wake private returned: %d %s\n",
-				      errno, strerror(errno));
-	} else {
-		ksft_test_result_pass("futex_wake private succeeds\n");
-	}
+	EXPECT_EQ(res, 1)
+		TH_LOG("futex_wake private returned: %d %s", res, res < 0 ? strerror(errno) : "");
+
+	pthread_join(waiter, NULL);
 }
 
 TEST(anon_page)
 {
+	struct waiter_args *args = malloc(sizeof(*args));
 	u_int32_t *shared_data;
 	pthread_t waiter;
 	int res, shm_id;
 
+	args->_metadata = _metadata;
+	args->flags = 0;
+
 	/* Testing an anon page shared memory */
 	shm_id = shmget(IPC_PRIVATE, 4096, IPC_CREAT | 0666);
 	if (shm_id < 0) {
-		if (errno == ENOSYS)
-			ksft_exit_skip("shmget syscall not supported\n");
-		perror("shmget");
-		exit(1);
+		if (errno == ENOSYS) {
+			free(args);
+			SKIP(return, "shmget syscall not supported");
+		}
+		ASSERT_GE(shm_id, 0)
+			TH_LOG("shmget failed: %s", strerror(errno));
 	}
 
 	shared_data = shmat(shm_id, NULL, 0);
+	if (shared_data == (void *)-1) {
+		free(args);
+		ASSERT_NE(shared_data, (void *)-1)
+			TH_LOG("shmat failed: %s", strerror(errno));
+	}
 
 	*shared_data = 0;
 	futex = shared_data;
 
-	ksft_print_dbg_msg("Calling shared (page anon) futex_wait on futex: %p\n", futex);
-	if (pthread_create(&waiter, NULL, waiterfn, NULL))
-		ksft_exit_fail_msg("pthread_create failed\n");
+	TH_LOG("Calling shared (page anon) futex_wait on futex: %p", futex);
+	ASSERT_EQ(pthread_create(&waiter, NULL, waiterfn, args), 0)
+		TH_LOG("pthread_create failed");
 
 	usleep(WAKE_WAIT_US);
 
-	ksft_print_dbg_msg("Calling shared (page anon) futex_wake on futex: %p\n", futex);
+	TH_LOG("Calling shared (page anon) futex_wake on futex: %p", futex);
 	res = futex_wake(futex, 1, 0);
-	if (res != 1) {
-		ksft_test_result_fail("futex_wake shared (page anon) returned: %d %s\n",
-				      errno, strerror(errno));
-	} else {
-		ksft_test_result_pass("futex_wake shared (page anon) succeeds\n");
+	EXPECT_EQ(res, 1) {
+		TH_LOG("futex_wake shared (page anon) returned: %d %s",
+		       res, res < 0 ? strerror(errno) : "");
 	}
 
+	pthread_join(waiter, NULL);
 	shmdt(shared_data);
 }
 
 TEST(file_backed)
 {
+	struct waiter_args *args = malloc(sizeof(*args));
 	u_int32_t f_private = 0;
 	pthread_t waiter;
 	int res, fd;
 	void *shm;
 
+	args->_metadata = _metadata;
+	args->flags = 0;
+
 	/* Testing a file backed shared memory */
-	fd = open(SHM_PATH, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
-	if (fd < 0)
-		ksft_exit_fail_msg("open\n");
+	fd = open(SHM_PATH, O_RDWR | O_CREAT, 0600);
+	if (fd < 0) {
+		free(args);
+		ASSERT_GE(fd, 0)
+			TH_LOG("open failed: %s", strerror(errno));
+	}
 
-	if (ftruncate(fd, sizeof(f_private)))
-		ksft_exit_fail_msg("ftruncate\n");
+	if (ftruncate(fd, sizeof(f_private))) {
+		free(args);
+		close(fd);
+		ASSERT_TRUE(0)
+			TH_LOG("ftruncate failed: %s", strerror(errno));
+	}
 
 	shm = mmap(NULL, sizeof(f_private), PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
-	if (shm == MAP_FAILED)
-		ksft_exit_fail_msg("mmap\n");
+	if (shm == MAP_FAILED) {
+		free(args);
+		close(fd);
+		ASSERT_NE(shm, MAP_FAILED)
+			TH_LOG("mmap failed: %s", strerror(errno));
+	}
 
 	memcpy(shm, &f_private, sizeof(f_private));
 
 	futex = shm;
 
-	ksft_print_dbg_msg("Calling shared (file backed) futex_wait on futex: %p\n", futex);
-	if (pthread_create(&waiter, NULL, waiterfn, NULL))
-		ksft_exit_fail_msg("pthread_create failed\n");
+	TH_LOG("Calling shared (file backed) futex_wait on futex: %p", futex);
+	ASSERT_EQ(pthread_create(&waiter, NULL, waiterfn, args), 0)
+		TH_LOG("pthread_create failed");
 
 	usleep(WAKE_WAIT_US);
 
-	ksft_print_dbg_msg("Calling shared (file backed) futex_wake on futex: %p\n", futex);
+	TH_LOG("Calling shared (file backed) futex_wake on futex: %p", futex);
 	res = futex_wake(shm, 1, 0);
-	if (res != 1) {
-		ksft_test_result_fail("futex_wake shared (file backed) returned: %d %s\n",
-				      errno, strerror(errno));
-	} else {
-		ksft_test_result_pass("futex_wake shared (file backed) succeeds\n");
+	EXPECT_EQ(res, 1) {
+		TH_LOG("futex_wake shared (file backed) returned: %d %s",
+		       res, res < 0 ? strerror(errno) : "");
 	}
 
+	pthread_join(waiter, NULL);
 	munmap(shm, sizeof(f_private));
 	remove(SHM_PATH);
 	close(fd);
diff --git a/tools/testing/selftests/futex/functional/futex_wait_private_mapped_file.c b/tools/testing/selftests/futex/functional/futex_wait_private_mapped_file.c
index 2a749f9b14ebb..a9f7a02e3a0b7 100644
--- a/tools/testing/selftests/futex/functional/futex_wait_private_mapped_file.c
+++ b/tools/testing/selftests/futex/functional/futex_wait_private_mapped_file.c
@@ -26,6 +26,7 @@
 #include <pthread.h>
 #include <libgen.h>
 #include <signal.h>
+#include <string.h>
 
 #include "futextest.h"
 #include "kselftest_harness.h"
@@ -41,17 +42,22 @@ struct timespec wait_timeout = { .tv_sec = 5, .tv_nsec = 0};
 
 void *thr_futex_wait(void *arg)
 {
+	struct __test_metadata *_metadata = (struct __test_metadata *)arg;
 	int ret;
 
-	ksft_print_dbg_msg("futex wait\n");
+	TH_LOG("futex wait");
 	ret = futex_wait(&val, 1, &wait_timeout, 0);
-	if (ret && errno != EWOULDBLOCK && errno != ETIMEDOUT)
-		ksft_exit_fail_msg("futex error.\n");
+	if (ret && errno != EWOULDBLOCK && errno != ETIMEDOUT) {
+		ASSERT_TRUE(0)
+			TH_LOG("futex error: %s", strerror(errno));
+	}
 
-	if (ret && errno == ETIMEDOUT)
-		ksft_exit_fail_msg("waiter timedout\n");
+	if (ret && errno == ETIMEDOUT) {
+		ASSERT_TRUE(0)
+			TH_LOG("waiter timedout");
+	}
 
-	ksft_print_dbg_msg("futex_wait: ret = %d, errno = %d\n", ret, errno);
+	TH_LOG("futex_wait: ret = %d, errno = %d", ret, errno);
 
 	return NULL;
 }
@@ -61,22 +67,20 @@ TEST(wait_private_mapped_file)
 	pthread_t thr;
 	int res;
 
-	res = pthread_create(&thr, NULL, thr_futex_wait, NULL);
-	if (res < 0)
-		ksft_exit_fail_msg("pthread_create error\n");
+	res = pthread_create(&thr, NULL, thr_futex_wait, _metadata);
+	ASSERT_EQ(res, 0)
+		TH_LOG("pthread_create error");
 
-	ksft_print_dbg_msg("wait a while\n");
+	TH_LOG("wait a while");
 	usleep(WAKE_WAIT_US);
 	val = 2;
 	res = futex_wake(&val, 1, 0);
-	ksft_print_dbg_msg("futex_wake %d\n", res);
-	if (res != 1)
-		ksft_exit_fail_msg("FUTEX_WAKE didn't find the waiting thread.\n");
+	TH_LOG("futex_wake %d", res);
+	EXPECT_EQ(res, 1)
+		TH_LOG("FUTEX_WAKE didn't find the waiting thread");
 
-	ksft_print_dbg_msg("join\n");
+	TH_LOG("join");
 	pthread_join(thr, NULL);
-
-	ksft_test_result_pass("wait_private_mapped_file");
 }
 
 TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/futex/functional/futex_wait_timeout.c b/tools/testing/selftests/futex/functional/futex_wait_timeout.c
index 674dd13af4210..6e6e770fe96ad 100644
--- a/tools/testing/selftests/futex/functional/futex_wait_timeout.c
+++ b/tools/testing/selftests/futex/functional/futex_wait_timeout.c
@@ -31,53 +31,47 @@ static pthread_barrier_t barrier;
  */
 void *get_pi_lock(void *arg)
 {
+	struct __test_metadata *_metadata = (struct __test_metadata *)arg;
 	int ret;
 	volatile futex_t lock = 0;
 
 	ret = futex_lock_pi(&futex_pi, NULL, 0, 0);
-	if (ret != 0)
-		ksft_exit_fail_msg("futex_lock_pi failed\n");
+	ASSERT_EQ(ret, 0)
+		TH_LOG("futex_lock_pi failed");
 
 	pthread_barrier_wait(&barrier);
 
 	/* Blocks forever */
 	ret = futex_wait(&lock, 0, NULL, 0);
-	ksft_exit_fail_msg("futex_wait failed\n");
+	ASSERT_TRUE(0)
+		TH_LOG("futex_wait returned unexpectedly: %d", ret);
 
 	return NULL;
 }
 
-/*
- * Check if the function returned the expected error
- */
-static void test_timeout(int res, char *test_name, int err)
-{
-	if (!res || errno != err) {
-		ksft_test_result_fail("%s returned %d\n", test_name,
-				      res < 0 ? errno : res);
-	} else {
-		ksft_test_result_pass("%s succeeds\n", test_name);
-	}
-}
-
-/*
- * Calculate absolute timeout and correct overflow
- */
-static int futex_get_abs_timeout(clockid_t clockid, struct timespec *to,
-				 long timeout_ns)
-{
-	if (clock_gettime(clockid, to))
-		ksft_exit_fail_msg("clock_gettime failed\n");
-
-	to->tv_nsec += timeout_ns;
-
-	if (to->tv_nsec >= 1000000000) {
-		to->tv_sec++;
-		to->tv_nsec -= 1000000000;
-	}
-
-	return 0;
-}
+#define TEST_TIMEOUT(_res, _test_name, _err) do {				\
+	if ((_res) < 0 && errno == ENOSYS && (_err) != ENOSYS) {		\
+		SKIP(return, "%s is not supported (ENOSYS)", _test_name);	\
+	}									\
+	EXPECT_EQ((_res), -1)							\
+		TH_LOG("%s returned unexpected result: %d", _test_name, (_res));\
+	if ((_res) == -1) {							\
+		EXPECT_EQ(errno, (_err)) {					\
+			TH_LOG("%s returned unexpected errno: %d (expected %d)",\
+			       _test_name, errno, (_err));			\
+		}								\
+	}									\
+} while (0)
+
+#define GET_ABS_TIMEOUT(_clockid, _to, _timeout_ns) do {		\
+	ASSERT_EQ(clock_gettime((_clockid), (_to)), 0)			\
+		TH_LOG("clock_gettime failed");				\
+	(_to)->tv_nsec += (_timeout_ns);				\
+	if ((_to)->tv_nsec >= 1000000000) {				\
+		(_to)->tv_sec++;					\
+		(_to)->tv_nsec -= 1000000000;				\
+	}								\
+} while (0)
 
 TEST(wait_bitset)
 {
@@ -90,19 +84,17 @@ TEST(wait_bitset)
 	to.tv_nsec = timeout_ns;
 
 	res = futex_wait(&f1, f1, &to, 0);
-	test_timeout(res, "futex_wait relative", ETIMEDOUT);
+	TEST_TIMEOUT(res, "futex_wait relative", ETIMEDOUT);
 
 	/* FUTEX_WAIT_BITSET with CLOCK_REALTIME */
-	if (futex_get_abs_timeout(CLOCK_REALTIME, &to, timeout_ns))
-		ksft_test_result_error("get_time error");
+	GET_ABS_TIMEOUT(CLOCK_REALTIME, &to, timeout_ns);
 	res = futex_wait_bitset(&f1, f1, &to, 1, FUTEX_CLOCK_REALTIME);
-	test_timeout(res, "futex_wait_bitset realtime", ETIMEDOUT);
+	TEST_TIMEOUT(res, "futex_wait_bitset realtime", ETIMEDOUT);
 
 	/* FUTEX_WAIT_BITSET with CLOCK_MONOTONIC */
-	if (futex_get_abs_timeout(CLOCK_MONOTONIC, &to, timeout_ns))
-		ksft_test_result_error("get_time error");
+	GET_ABS_TIMEOUT(CLOCK_MONOTONIC, &to, timeout_ns);
 	res = futex_wait_bitset(&f1, f1, &to, 1, 0);
-	test_timeout(res, "futex_wait_bitset monotonic", ETIMEDOUT);
+	TEST_TIMEOUT(res, "futex_wait_bitset monotonic", ETIMEDOUT);
 }
 
 TEST(requeue_pi)
@@ -112,17 +104,14 @@ TEST(requeue_pi)
 	int res;
 
 	/* FUTEX_WAIT_REQUEUE_PI with CLOCK_REALTIME */
-	if (futex_get_abs_timeout(CLOCK_REALTIME, &to, timeout_ns))
-		ksft_test_result_error("get_time error");
+	GET_ABS_TIMEOUT(CLOCK_REALTIME, &to, timeout_ns);
 	res = futex_wait_requeue_pi(&f1, f1, &futex_pi, &to, FUTEX_CLOCK_REALTIME);
-	test_timeout(res, "futex_wait_requeue_pi realtime", ETIMEDOUT);
+	TEST_TIMEOUT(res, "futex_wait_requeue_pi realtime", ETIMEDOUT);
 
 	/* FUTEX_WAIT_REQUEUE_PI with CLOCK_MONOTONIC */
-	if (futex_get_abs_timeout(CLOCK_MONOTONIC, &to, timeout_ns))
-		ksft_test_result_error("get_time error");
+	GET_ABS_TIMEOUT(CLOCK_MONOTONIC, &to, timeout_ns);
 	res = futex_wait_requeue_pi(&f1, f1, &futex_pi, &to, 0);
-	test_timeout(res, "futex_wait_requeue_pi monotonic", ETIMEDOUT);
-
+	TEST_TIMEOUT(res, "futex_wait_requeue_pi monotonic", ETIMEDOUT);
 }
 
 TEST(lock_pi)
@@ -133,7 +122,8 @@ TEST(lock_pi)
 
 	/* Create a thread that will lock forever so any waiter will timeout */
 	pthread_barrier_init(&barrier, NULL, 2);
-	pthread_create(&thread, NULL, get_pi_lock, NULL);
+	ASSERT_EQ(pthread_create(&thread, NULL, get_pi_lock, _metadata), 0)
+		TH_LOG("pthread_create failed");
 
 	/* Wait until the other thread calls futex_lock_pi() */
 	pthread_barrier_wait(&barrier);
@@ -149,14 +139,13 @@ TEST(lock_pi)
 	 * time or your time machine) the monotonic clock value is always
 	 * smaller than realtime and the syscall will timeout immediately.
 	 */
-	if (futex_get_abs_timeout(CLOCK_REALTIME, &to, timeout_ns))
-		ksft_test_result_error("get_time error");
+	GET_ABS_TIMEOUT(CLOCK_REALTIME, &to, timeout_ns);
 	res = futex_lock_pi(&futex_pi, &to, 0, 0);
-	test_timeout(res, "futex_lock_pi realtime", ETIMEDOUT);
+	TEST_TIMEOUT(res, "futex_lock_pi realtime", ETIMEDOUT);
 
 	/* Test operations that don't support FUTEX_CLOCK_REALTIME */
 	res = futex_lock_pi(&futex_pi, NULL, 0, FUTEX_CLOCK_REALTIME);
-	test_timeout(res, "futex_lock_pi invalid timeout flag", ENOSYS);
+	TEST_TIMEOUT(res, "futex_lock_pi invalid timeout flag", ENOSYS);
 }
 
 TEST(waitv)
@@ -171,17 +160,18 @@ TEST(waitv)
 	struct timespec to;
 	int res;
 
+	if (!is_futex_waitv_supported())
+		SKIP(return, "futex_waitv syscall not supported");
+
 	/* futex_waitv with CLOCK_MONOTONIC */
-	if (futex_get_abs_timeout(CLOCK_MONOTONIC, &to, timeout_ns))
-		ksft_test_result_error("get_time error");
+	GET_ABS_TIMEOUT(CLOCK_MONOTONIC, &to, timeout_ns);
 	res = futex_waitv(&waitv, 1, 0, &to, CLOCK_MONOTONIC);
-	test_timeout(res, "futex_waitv monotonic", ETIMEDOUT);
+	TEST_TIMEOUT(res, "futex_waitv monotonic", ETIMEDOUT);
 
 	/* futex_waitv with CLOCK_REALTIME */
-	if (futex_get_abs_timeout(CLOCK_REALTIME, &to, timeout_ns))
-		ksft_test_result_error("get_time error");
+	GET_ABS_TIMEOUT(CLOCK_REALTIME, &to, timeout_ns);
 	res = futex_waitv(&waitv, 1, 0, &to, CLOCK_REALTIME);
-	test_timeout(res, "futex_waitv realtime", ETIMEDOUT);
+	TEST_TIMEOUT(res, "futex_waitv realtime", ETIMEDOUT);
 }
 
 TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/futex/functional/futex_wait_uninitialized_heap.c b/tools/testing/selftests/futex/functional/futex_wait_uninitialized_heap.c
index b07d68a67f31c..bbffc23e0006a 100644
--- a/tools/testing/selftests/futex/functional/futex_wait_uninitialized_heap.c
+++ b/tools/testing/selftests/futex/functional/futex_wait_uninitialized_heap.c
@@ -17,17 +17,18 @@
  *
  *****************************************************************************/
 
+#include <errno.h>
+#include <libgen.h>
 #include <pthread.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <linux/futex.h>
 #include <sys/mman.h>
 #include <syscall.h>
 #include <sys/types.h>
 #include <sys/stat.h>
-#include <unistd.h>
-#include <errno.h>
-#include <linux/futex.h>
-#include <libgen.h>
 
 #include "futextest.h"
 #include "kselftest_harness.h"
@@ -40,6 +41,7 @@ void *buf;
 
 void *wait_thread(void *arg)
 {
+	struct __test_metadata *_metadata = (struct __test_metadata *)arg;
 	int res;
 
 	child_ret = true;
@@ -47,7 +49,8 @@ void *wait_thread(void *arg)
 	child_blocked = 0;
 
 	if (res != 0 && errno != EWOULDBLOCK) {
-		ksft_exit_fail_msg("futex failure\n");
+		EXPECT_EQ(res, 0)
+			TH_LOG("futex failure: %s", strerror(errno));
 		child_ret = false;
 	}
 	pthread_exit(NULL);
@@ -63,21 +66,23 @@ TEST(futex_wait_uninitialized_heap)
 
 	buf = mmap(NULL, page_size, PROT_READ|PROT_WRITE,
 		   MAP_PRIVATE|MAP_ANONYMOUS, 0, 0);
-	if (buf == (void *)-1)
-		ksft_exit_fail_msg("mmap\n");
+	ASSERT_NE(buf, MAP_FAILED)
+		TH_LOG("mmap failed: %s", strerror(errno));
 
-	ret = pthread_create(&thr, NULL, wait_thread, NULL);
-	if (ret)
-		ksft_exit_fail_msg("pthread_create\n");
+	ret = pthread_create(&thr, NULL, wait_thread, _metadata);
+	ASSERT_EQ(ret, 0)
+		TH_LOG("pthread_create failed");
 
-	ksft_print_dbg_msg("waiting %dus for child to return\n", WAIT_US);
+	TH_LOG("waiting %dus for child to return", WAIT_US);
 	usleep(WAIT_US);
 
-	if (child_blocked)
-		ksft_test_result_fail("child blocked in kernel\n");
+	EXPECT_EQ(child_blocked, 0)
+		TH_LOG("child blocked in kernel");
+	EXPECT_TRUE(child_ret)
+		TH_LOG("child error");
 
-	if (!child_ret)
-		ksft_test_result_fail("child error\n");
+	pthread_join(thr, NULL);
+	munmap(buf, page_size);
 }
 
 TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c b/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c
index 9ff936ecf1640..ab039dda3e516 100644
--- a/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c
+++ b/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c
@@ -28,20 +28,20 @@
 
 #define timeout_ns 100000
 
+
 TEST(futex_wait_wouldblock)
 {
 	struct timespec to = {.tv_sec = 0, .tv_nsec = timeout_ns};
 	futex_t f1 = FUTEX_INITIALIZER;
 	int res;
 
-	ksft_print_dbg_msg("Calling futex_wait on f1: %u @ %p with val=%u\n", f1, &f1, f1+1);
+	TH_LOG("Calling futex_wait on f1: %u @ %p with val=%u", f1, &f1, f1+1);
 	res = futex_wait(&f1, f1+1, &to, FUTEX_PRIVATE_FLAG);
-	if (!res || errno != EWOULDBLOCK) {
-		ksft_test_result_fail("futex_wait returned: %d %s\n",
-				      res ? errno : res,
-				      res ? strerror(errno) : "");
-	} else {
-		ksft_test_result_pass("futex_wait\n");
+	EXPECT_EQ(res, -1)
+		TH_LOG("futex_wait returned unexpected result: %d", res);
+	if (res == -1) {
+		EXPECT_EQ(errno, EWOULDBLOCK)
+			TH_LOG("futex_wait returned unexpected errno: %d", errno);
 	}
 }
 
@@ -57,8 +57,11 @@ TEST(futex_waitv_wouldblock)
 	};
 	int res;
 
-	if (clock_gettime(CLOCK_MONOTONIC, &to))
-		ksft_exit_fail_msg("clock_gettime failed %d\n", errno);
+	if (!is_futex_waitv_supported())
+		SKIP(return, "futex_waitv syscall not supported");
+
+	ASSERT_EQ(clock_gettime(CLOCK_MONOTONIC, &to), 0)
+		TH_LOG("clock_gettime failed");
 
 	to.tv_nsec += timeout_ns;
 
@@ -67,14 +70,13 @@ TEST(futex_waitv_wouldblock)
 		to.tv_nsec -= 1000000000;
 	}
 
-	ksft_print_dbg_msg("Calling futex_waitv on f1: %u @ %p with val=%u\n", f1, &f1, f1+1);
+	TH_LOG("Calling futex_waitv on f1: %u @ %p with val=%u", f1, &f1, f1+1);
 	res = futex_waitv(&waitv, 1, 0, &to, CLOCK_MONOTONIC);
-	if (!res || errno != EWOULDBLOCK) {
-		ksft_test_result_fail("futex_waitv returned: %d %s\n",
-				      res ? errno : res,
-				      res ? strerror(errno) : "");
-	} else {
-		ksft_test_result_pass("futex_waitv\n");
+	EXPECT_EQ(res, -1)
+		TH_LOG("futex_waitv returned unexpected result: %d", res);
+	if (res == -1) {
+		EXPECT_EQ(errno, EWOULDBLOCK)
+			TH_LOG("futex_waitv returned unexpected errno: %d", errno);
 	}
 }
 
diff --git a/tools/testing/selftests/futex/functional/futex_waitv.c b/tools/testing/selftests/futex/functional/futex_waitv.c
index b5ada9fdb26f0..4858d5faeecfc 100644
--- a/tools/testing/selftests/futex/functional/futex_waitv.c
+++ b/tools/testing/selftests/futex/functional/futex_waitv.c
@@ -25,24 +25,26 @@
 static struct futex_waitv waitv[NR_FUTEXES];
 u_int32_t futexes[NR_FUTEXES] = {0};
 
+
 void *waiterfn(void *arg)
 {
+	struct __test_metadata *_metadata = (struct __test_metadata *)arg;
 	struct timespec to;
 	int res;
 
 	/* setting absolute timeout for futex2 */
-	if (clock_gettime(CLOCK_MONOTONIC, &to))
-		ksft_exit_fail_msg("gettime64 failed\n");
+	ASSERT_EQ(clock_gettime(CLOCK_MONOTONIC, &to), 0)
+		TH_LOG("gettime64 failed");
 
 	to.tv_sec++;
 
 	res = futex_waitv(waitv, NR_FUTEXES, 0, &to, CLOCK_MONOTONIC);
 	if (res < 0) {
-		ksft_test_result_fail("futex_waitv returned: %d %s\n",
-				      errno, strerror(errno));
-	} else if (res != NR_FUTEXES - 1) {
-		ksft_test_result_fail("futex_waitv returned: %d, expecting %d\n",
-				      res, NR_FUTEXES - 1);
+		EXPECT_EQ(res, NR_FUTEXES - 1)
+			TH_LOG("futex_waitv failed: %s", strerror(errno));
+	} else {
+		EXPECT_EQ(res, NR_FUTEXES - 1)
+			TH_LOG("futex_waitv returned %d, expected %d", res, NR_FUTEXES - 1);
 	}
 
 	return NULL;
@@ -53,6 +55,9 @@ TEST(private_waitv)
 	pthread_t waiter;
 	int res, i;
 
+	if (!is_futex_waitv_supported())
+		SKIP(return, "futex_waitv syscall not supported");
+
 	for (i = 0; i < NR_FUTEXES; i++) {
 		waitv[i].uaddr = (uintptr_t)&futexes[i];
 		waitv[i].flags = FUTEX_32 | FUTEX_PRIVATE_FLAG;
@@ -61,19 +66,14 @@ TEST(private_waitv)
 	}
 
 	/* Private waitv */
-	if (pthread_create(&waiter, NULL, waiterfn, NULL))
-		ksft_exit_fail_msg("pthread_create failed\n");
+	ASSERT_EQ(pthread_create(&waiter, NULL, waiterfn, _metadata), 0)
+		TH_LOG("pthread_create failed");
 
 	usleep(WAKE_WAIT_US);
 
 	res = futex_wake(u64_to_ptr(waitv[NR_FUTEXES - 1].uaddr), 1, FUTEX_PRIVATE_FLAG);
-	if (res != 1) {
-		ksft_test_result_fail("futex_wake private returned: %d %s\n",
-				      res ? errno : res,
-				      res ? strerror(errno) : "");
-	} else {
-		ksft_test_result_pass("futex_waitv private\n");
-	}
+	EXPECT_EQ(res, 1)
+		TH_LOG("futex_wake private returned: %d %s", res, res < 0 ? strerror(errno) : "");
 }
 
 TEST(shared_waitv)
@@ -81,15 +81,18 @@ TEST(shared_waitv)
 	pthread_t waiter;
 	int res, i;
 
+	if (!is_futex_waitv_supported())
+		SKIP(return, "futex_waitv syscall not supported");
+
 	/* Shared waitv */
 	for (i = 0; i < NR_FUTEXES; i++) {
 		int shm_id = shmget(IPC_PRIVATE, 4096, IPC_CREAT | 0666);
 
 		if (shm_id < 0) {
 			if (errno == ENOSYS)
-				ksft_exit_skip("shmget syscall not supported\n");
-			perror("shmget");
-			exit(1);
+				SKIP(return, "shmget syscall not supported");
+			ASSERT_GE(shm_id, 0)
+				TH_LOG("shmget failed");
 		}
 
 		unsigned int *shared_data = shmat(shm_id, NULL, 0);
@@ -101,19 +104,14 @@ TEST(shared_waitv)
 		waitv[i].__reserved = 0;
 	}
 
-	if (pthread_create(&waiter, NULL, waiterfn, NULL))
-		ksft_exit_fail_msg("pthread_create failed\n");
+	ASSERT_EQ(pthread_create(&waiter, NULL, waiterfn, _metadata), 0)
+		TH_LOG("pthread_create failed");
 
 	usleep(WAKE_WAIT_US);
 
 	res = futex_wake(u64_to_ptr(waitv[NR_FUTEXES - 1].uaddr), 1, 0);
-	if (res != 1) {
-		ksft_test_result_fail("futex_wake shared returned: %d %s\n",
-				      res ? errno : res,
-				      res ? strerror(errno) : "");
-	} else {
-		ksft_test_result_pass("futex_waitv shared\n");
-	}
+	EXPECT_EQ(res, 1)
+		TH_LOG("futex_wake shared returned: %d %s", res, res < 0 ? strerror(errno) : "");
 
 	for (i = 0; i < NR_FUTEXES; i++)
 		shmdt(u64_to_ptr(waitv[i].uaddr));
@@ -124,21 +122,24 @@ TEST(invalid_flag)
 	struct timespec to;
 	int res;
 
+	if (!is_futex_waitv_supported())
+		SKIP(return, "futex_waitv syscall not supported");
+
 	/* Testing a waiter without FUTEX_32 flag */
 	waitv[0].flags = FUTEX_PRIVATE_FLAG;
 
-	if (clock_gettime(CLOCK_MONOTONIC, &to))
-		ksft_exit_fail_msg("gettime64 failed\n");
+	ASSERT_EQ(clock_gettime(CLOCK_MONOTONIC, &to), 0)
+		TH_LOG("gettime64 failed");
 
 	to.tv_sec++;
 
 	res = futex_waitv(waitv, NR_FUTEXES, 0, &to, CLOCK_MONOTONIC);
-	if (res == EINVAL) {
-		ksft_test_result_fail("futex_waitv private returned: %d %s\n",
-				      res ? errno : res,
-				      res ? strerror(errno) : "");
-	} else {
-		ksft_test_result_pass("futex_waitv without FUTEX_32\n");
+
+	EXPECT_EQ(res, -1)
+		TH_LOG("futex_waitv returned unexpected result: %d", res);
+	if (res == -1) {
+		EXPECT_EQ(errno, EINVAL)
+			TH_LOG("futex_waitv returned unexpected errno: %d", errno);
 	}
 }
 
@@ -147,22 +148,25 @@ TEST(unaligned_address)
 	struct timespec to;
 	int res;
 
+	if (!is_futex_waitv_supported())
+		SKIP(return, "futex_waitv syscall not supported");
+
 	/* Testing a waiter with an unaligned address */
 	waitv[0].flags = FUTEX_PRIVATE_FLAG | FUTEX_32;
 	waitv[0].uaddr = 1;
 
-	if (clock_gettime(CLOCK_MONOTONIC, &to))
-		ksft_exit_fail_msg("gettime64 failed\n");
+	ASSERT_EQ(clock_gettime(CLOCK_MONOTONIC, &to), 0)
+		TH_LOG("gettime64 failed");
 
 	to.tv_sec++;
 
 	res = futex_waitv(waitv, NR_FUTEXES, 0, &to, CLOCK_MONOTONIC);
-	if (res == EINVAL) {
-		ksft_test_result_fail("futex_wake private returned: %d %s\n",
-				      res ? errno : res,
-				      res ? strerror(errno) : "");
-	} else {
-		ksft_test_result_pass("futex_waitv with an unaligned address\n");
+
+	EXPECT_EQ(res, -1)
+		TH_LOG("futex_waitv returned unexpected result: %d", res);
+	if (res == -1) {
+		EXPECT_EQ(errno, EINVAL)
+			TH_LOG("futex_waitv returned unexpected errno: %d", errno);
 	}
 }
 
@@ -171,36 +175,39 @@ TEST(null_address)
 	struct timespec to;
 	int res;
 
+	if (!is_futex_waitv_supported())
+		SKIP(return, "futex_waitv syscall not supported");
+
 	/* Testing a NULL address for waiters.uaddr */
 	waitv[0].uaddr = 0x00000000;
 
-	if (clock_gettime(CLOCK_MONOTONIC, &to))
-		ksft_exit_fail_msg("gettime64 failed\n");
+	ASSERT_EQ(clock_gettime(CLOCK_MONOTONIC, &to), 0)
+		TH_LOG("gettime64 failed");
 
 	to.tv_sec++;
 
 	res = futex_waitv(waitv, NR_FUTEXES, 0, &to, CLOCK_MONOTONIC);
-	if (res == EINVAL) {
-		ksft_test_result_fail("futex_waitv private returned: %d %s\n",
-				      res ? errno : res,
-				      res ? strerror(errno) : "");
-	} else {
-		ksft_test_result_pass("futex_waitv NULL address in waitv.uaddr\n");
+
+	EXPECT_EQ(res, -1)
+		TH_LOG("futex_waitv returned unexpected result: %d", res);
+	if (res == -1) {
+		EXPECT_EQ(errno, EINVAL)
+			TH_LOG("futex_waitv returned unexpected errno: %d", errno);
 	}
 
 	/* Testing a NULL address for *waiters */
-	if (clock_gettime(CLOCK_MONOTONIC, &to))
-		ksft_exit_fail_msg("gettime64 failed\n");
+	ASSERT_EQ(clock_gettime(CLOCK_MONOTONIC, &to), 0)
+		TH_LOG("gettime64 failed");
 
 	to.tv_sec++;
 
 	res = futex_waitv(NULL, NR_FUTEXES, 0, &to, CLOCK_MONOTONIC);
-	if (res == EINVAL) {
-		ksft_test_result_fail("futex_waitv private returned: %d %s\n",
-				      res ? errno : res,
-				      res ? strerror(errno) : "");
-	} else {
-		ksft_test_result_pass("futex_waitv NULL address in *waiters\n");
+
+	EXPECT_EQ(res, -1)
+		TH_LOG("futex_waitv returned unexpected result: %d", res);
+	if (res == -1) {
+		EXPECT_EQ(errno, EINVAL)
+			TH_LOG("futex_waitv returned unexpected errno: %d", errno);
 	}
 }
 
@@ -209,19 +216,22 @@ TEST(invalid_clockid)
 	struct timespec to;
 	int res;
 
+	if (!is_futex_waitv_supported())
+		SKIP(return, "futex_waitv syscall not supported");
+
 	/* Testing an invalid clockid */
-	if (clock_gettime(CLOCK_MONOTONIC, &to))
-		ksft_exit_fail_msg("gettime64 failed\n");
+	ASSERT_EQ(clock_gettime(CLOCK_MONOTONIC, &to), 0)
+		TH_LOG("gettime64 failed");
 
 	to.tv_sec++;
 
 	res = futex_waitv(NULL, NR_FUTEXES, 0, &to, CLOCK_TAI);
-	if (res == EINVAL) {
-		ksft_test_result_fail("futex_waitv private returned: %d %s\n",
-				      res ? errno : res,
-				      res ? strerror(errno) : "");
-	} else {
-		ksft_test_result_pass("futex_waitv invalid clockid\n");
+
+	EXPECT_EQ(res, -1)
+		TH_LOG("futex_waitv returned unexpected result: %d", res);
+	if (res == -1) {
+		EXPECT_EQ(errno, EINVAL)
+			TH_LOG("futex_waitv returned unexpected errno: %d", errno);
 	}
 }
 
diff --git a/tools/testing/selftests/futex/functional/robust_list.c b/tools/testing/selftests/futex/functional/robust_list.c
index b3fab60181d5e..3f713bd31f4cb 100644
--- a/tools/testing/selftests/futex/functional/robust_list.c
+++ b/tools/testing/selftests/futex/functional/robust_list.c
@@ -25,24 +25,24 @@
 #define _GNU_SOURCE
 
 #include "futextest.h"
-#include "../../kselftest_harness.h"
+#include "kselftest_harness.h"
 
 #include <dlfcn.h>
 #include <errno.h>
 #include <pthread.h>
 #include <signal.h>
-#include <stdint.h>
 #include <stdatomic.h>
 #include <stdbool.h>
 #include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
 #include <sys/auxv.h>
 #include <sys/mman.h>
 #include <sys/wait.h>
 
 #define STACK_SIZE (1024 * 1024)
-
 #define FUTEX_TIMEOUT 3
-
 #define SLEEP_US 100
 
 #if __SIZEOF_LONG__ == 8
@@ -71,30 +71,46 @@ static int sys_futex_robust_unlock(_Atomic(uint32_t) *uaddr, unsigned int op, in
  * Basic lock struct, contains just the futex word and the robust list element
  * Real implementations have also a *prev to easily walk in the list
  */
+typedef _Atomic(unsigned int) atomic_futex_t;
+
 struct lock_struct {
-	_Atomic(unsigned int)	futex;
+	atomic_futex_t		futex;
 	struct robust_list	list;
 };
 
+struct child_args {
+	struct __test_metadata	*_metadata;
+	void			*arg;
+};
+
 /*
  * Helper function to spawn a child thread. Returns -1 on error, pid on success
  */
-static int create_child(int (*fn)(void *arg), void *arg)
+static int create_child(struct __test_metadata *_metadata, int (*fn)(void *arg), void *arg)
 {
+	struct child_args *cargs = malloc(sizeof(*cargs));
 	char *stack;
 	pid_t pid;
 
+	if (!cargs)
+		return -1;
+	cargs->_metadata = _metadata;
+	cargs->arg = arg;
+
 	stack = mmap(NULL, STACK_SIZE, PROT_READ | PROT_WRITE,
 		     MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
-	if (stack == MAP_FAILED)
+	if (stack == MAP_FAILED) {
+		free(cargs);
 		return -1;
+	}
 
 	stack += STACK_SIZE;
 
-	pid = clone(fn, stack, CLONE_VM | SIGCHLD, arg);
-
-	if (pid == -1)
+	pid = clone(fn, stack, CLONE_VM | SIGCHLD, cargs);
+	if (pid == -1) {
+		free(cargs);
 		return -1;
+	}
 
 	return pid;
 }
@@ -123,7 +139,7 @@ static int set_list(struct robust_list_head *head)
  */
 static int mutex_lock(struct lock_struct *lock, struct robust_list_head *head, bool error_inject)
 {
-	_Atomic(unsigned int) *futex = &lock->futex;
+	atomic_futex_t *futex = &lock->futex;
 	unsigned int zero = 0;
 	pid_t tid = gettid();
 	int ret = -1;
@@ -183,21 +199,21 @@ static int mutex_lock(struct lock_struct *lock, struct robust_list_head *head, b
  */
 static int child_fn_lock(void *arg)
 {
-	struct lock_struct *lock = arg;
+	struct child_args *cargs = arg;
+	struct __test_metadata *_metadata = cargs->_metadata;
+	struct lock_struct *lock = cargs->arg;
 	struct robust_list_head head;
 	int ret;
 
+	free(cargs);
+
 	ret = set_list(&head);
-	if (ret) {
-		ksft_test_result_fail("set_robust_list error\n");
-		return ret;
-	}
+	ASSERT_EQ(ret, 0)
+		TH_LOG("set_robust_list error");
 
 	ret = mutex_lock(lock, &head, false);
-	if (ret) {
-		ksft_test_result_fail("mutex_lock error\n");
-		return ret;
-	}
+	ASSERT_EQ(ret, 0)
+		TH_LOG("mutex_lock error");
 
 	pthread_barrier_wait(&barrier);
 
@@ -220,7 +236,7 @@ static int child_fn_lock(void *arg)
 TEST(test_robustness)
 {
 	struct lock_struct lock = { .futex = 0 };
-	_Atomic(unsigned int) *futex = &lock.futex;
+	atomic_futex_t *futex = &lock.futex;
 	struct robust_list_head head;
 	int ret, pid, wstatus;
 
@@ -234,7 +250,7 @@ TEST(test_robustness)
 	ret = pthread_barrier_init(&barrier, NULL, 2);
 	ASSERT_EQ(ret, 0);
 
-	pid = create_child(&child_fn_lock, &lock);
+	pid = create_child(_metadata, &child_fn_lock, &lock);
 	ASSERT_NE(pid, -1);
 
 	pthread_barrier_wait(&barrier);
@@ -251,9 +267,8 @@ TEST(test_robustness)
 	wait(&wstatus);
 	pthread_barrier_destroy(&barrier);
 
-	/* Pass only if the child hasn't return error */
-	if (!WEXITSTATUS(wstatus))
-		ksft_test_result_pass("%s\n", __func__);
+	EXPECT_EQ(WEXITSTATUS(wstatus), 0)
+		TH_LOG("child failed");
 }
 
 /*
@@ -279,8 +294,6 @@ TEST(test_set_robust_list_invalid_size)
 	ret = set_robust_list(&head, 0);
 	ASSERT_EQ(ret, -1);
 	ASSERT_EQ(errno, EINVAL);
-
-	ksft_test_result_pass("%s\n", __func__);
 }
 
 /*
@@ -307,20 +320,20 @@ TEST(test_get_robust_list_self)
 	ASSERT_EQ(ret, 0);
 	ASSERT_EQ(get_head, &head2);
 	ASSERT_EQ(head_size, len_ptr);
-
-	ksft_test_result_pass("%s\n", __func__);
 }
 
 static int child_list(void *arg)
 {
-	struct robust_list_head *head = arg;
+	struct child_args *cargs = arg;
+	struct __test_metadata *_metadata = cargs->_metadata;
+	struct robust_list_head *head = cargs->arg;
 	int ret;
 
+	free(cargs);
+
 	ret = set_robust_list(head, sizeof(*head));
-	if (ret) {
-		ksft_test_result_fail("set_robust_list error\n");
-		return -1;
-	}
+	ASSERT_EQ(ret, 0)
+		TH_LOG("set_robust_list error");
 
 	/*
 	 * After setting the list head, wait until the main thread can call
@@ -350,7 +363,7 @@ TEST(test_get_robust_list_child)
 	ret = pthread_barrier_init(&barrier2, NULL, 2);
 	ASSERT_EQ(ret, 0);
 
-	tid = create_child(&child_list, &head);
+	tid = create_child(_metadata, &child_list, &head);
 	ASSERT_NE(tid, -1);
 
 	pthread_barrier_wait(&barrier);
@@ -365,28 +378,27 @@ TEST(test_get_robust_list_child)
 	pthread_barrier_destroy(&barrier);
 	pthread_barrier_destroy(&barrier2);
 
-	/* Pass only if the child hasn't return error */
-	if (!WEXITSTATUS(wstatus))
-		ksft_test_result_pass("%s\n", __func__);
+	EXPECT_EQ(WEXITSTATUS(wstatus), 0)
+		TH_LOG("child failed");
 }
 
 static int child_fn_lock_with_error(void *arg)
 {
-	struct lock_struct *lock = arg;
+	struct child_args *cargs = arg;
+	struct __test_metadata *_metadata = cargs->_metadata;
+	struct lock_struct *lock = cargs->arg;
 	struct robust_list_head head;
 	int ret;
 
+	free(cargs);
+
 	ret = set_list(&head);
-	if (ret) {
-		ksft_test_result_fail("set_robust_list error\n");
-		return -1;
-	}
+	ASSERT_EQ(ret, 0)
+		TH_LOG("set_robust_list error");
 
 	ret = mutex_lock(lock, &head, true);
-	if (ret) {
-		ksft_test_result_fail("mutex_lock error\n");
-		return -1;
-	}
+	ASSERT_EQ(ret, 0)
+		TH_LOG("mutex_lock error");
 
 	pthread_barrier_wait(&barrier);
 
@@ -404,7 +416,7 @@ static int child_fn_lock_with_error(void *arg)
 TEST(test_set_list_op_pending)
 {
 	struct lock_struct lock = { .futex = 0 };
-	_Atomic(unsigned int) *futex = &lock.futex;
+	atomic_futex_t *futex = &lock.futex;
 	struct robust_list_head head;
 	int ret, wstatus;
 
@@ -414,7 +426,7 @@ TEST(test_set_list_op_pending)
 	ret = pthread_barrier_init(&barrier, NULL, 2);
 	ASSERT_EQ(ret, 0);
 
-	ret = create_child(&child_fn_lock_with_error, &lock);
+	ret = create_child(_metadata, &child_fn_lock_with_error, &lock);
 	ASSERT_NE(ret, -1);
 
 	pthread_barrier_wait(&barrier);
@@ -427,21 +439,21 @@ TEST(test_set_list_op_pending)
 	wait(&wstatus);
 	pthread_barrier_destroy(&barrier);
 
-	/* Pass only if the child hasn't return error */
-	if (!WEXITSTATUS(wstatus))
-		ksft_test_result_pass("%s\n", __func__);
-	else
-		ksft_test_result_fail("%s\n", __func__);
+	EXPECT_EQ(WEXITSTATUS(wstatus), 0)
+		TH_LOG("child failed");
 }
 
 #define CHILD_NR 10
 
 static int child_lock_holder(void *arg)
 {
-	struct lock_struct *locks = arg;
+	struct child_args *cargs = arg;
+	struct lock_struct *locks = cargs->arg;
 	struct robust_list_head head;
 	int i;
 
+	free(cargs);
+
 	set_list(&head);
 
 	for (i = 0; i < CHILD_NR; i++) {
@@ -460,22 +472,21 @@ static int child_lock_holder(void *arg)
 
 static int child_wait_lock(void *arg)
 {
-	struct lock_struct *lock = arg;
+	struct child_args *cargs = arg;
+	struct __test_metadata *_metadata = cargs->_metadata;
+	struct lock_struct *lock = cargs->arg;
 	struct robust_list_head head;
 	int ret;
 
+	free(cargs);
+
 	pthread_barrier_wait(&barrier2);
 	ret = mutex_lock(lock, &head, false);
+	ASSERT_EQ(ret, 0)
+		TH_LOG("mutex_lock error");
 
-	if (ret) {
-		ksft_test_result_fail("mutex_lock error\n");
-		return -1;
-	}
-
-	if (!(lock->futex & FUTEX_OWNER_DIED)) {
-		ksft_test_result_fail("futex not marked with FUTEX_OWNER_DIED\n");
-		return -1;
-	}
+	ASSERT_TRUE(lock->futex & FUTEX_OWNER_DIED)
+		TH_LOG("futex not marked with FUTEX_OWNER_DIED");
 
 	return 0;
 }
@@ -495,18 +506,20 @@ TEST(test_robust_list_multiple_elements)
 	ret = pthread_barrier_init(&barrier2, NULL, CHILD_NR + 1);
 	ASSERT_EQ(ret, 0);
 
-	pids[0] = create_child(&child_lock_holder, &locks);
+	pids[0] = create_child(_metadata, &child_lock_holder, &locks);
+	ASSERT_NE(pids[0], -1);
 
 	/* Wait until the locker thread takes the look */
 	pthread_barrier_wait(&barrier);
 
-	for (i = 0; i < CHILD_NR; i++)
-		pids[i+1] = create_child(&child_wait_lock, &locks[i]);
+	for (i = 0; i < CHILD_NR; i++) {
+		pids[i+1] = create_child(_metadata, &child_wait_lock, &locks[i]);
+		ASSERT_NE(pids[i+1], -1);
+	}
 
-	/* Wait for all children to return */
+	/* Wait for all children to return (holder + all waiters) */
 	ret = 0;
-
-	for (i = 0; i < CHILD_NR; i++) {
+	for (i = 0; i < CHILD_NR + 1; i++) {
 		waitpid(pids[i], &wstatus, 0);
 		if (WEXITSTATUS(wstatus))
 			ret = -1;
@@ -515,22 +528,23 @@ TEST(test_robust_list_multiple_elements)
 	pthread_barrier_destroy(&barrier);
 	pthread_barrier_destroy(&barrier2);
 
-	/* Pass only if the child hasn't return error */
-	if (!ret)
-		ksft_test_result_pass("%s\n", __func__);
+	EXPECT_EQ(ret, 0)
+		TH_LOG("One or more children failed");
 }
 
 static int child_circular_list(void *arg)
 {
-	static struct robust_list_head head;
+	struct child_args *cargs = arg;
+	struct __test_metadata *_metadata = cargs->_metadata;
+	struct robust_list_head head;
 	struct lock_struct a, b, c;
 	int ret;
 
+	free(cargs);
+
 	ret = set_list(&head);
-	if (ret) {
-		ksft_test_result_fail("set_list error\n");
-		return -1;
-	}
+	ASSERT_EQ(ret, 0)
+		TH_LOG("set_list error");
 
 	head.list.next = &a.list;
 
@@ -552,14 +566,15 @@ static int child_circular_list(void *arg)
 TEST(test_circular_list)
 {
 	int wstatus;
+	pid_t pid;
 
-	create_child(child_circular_list, NULL);
+	pid = create_child(_metadata, child_circular_list, NULL);
+	ASSERT_NE(pid, -1);
 
 	wait(&wstatus);
 
-	/* Pass only if the child hasn't return error */
-	if (!WEXITSTATUS(wstatus))
-		ksft_test_result_pass("%s\n", __func__);
+	EXPECT_EQ(WEXITSTATUS(wstatus), 0)
+		TH_LOG("child failed");
 }
 
 /*
diff --git a/tools/testing/selftests/futex/functional/run.sh b/tools/testing/selftests/futex/functional/run.sh
index e88545c06d57a..d1a681b798bd3 100755
--- a/tools/testing/selftests/futex/functional/run.sh
+++ b/tools/testing/selftests/futex/functional/run.sh
@@ -51,3 +51,6 @@ echo
 
 echo
 ./futex_numa_mpol
+
+echo
+./futex_lock_pi_exiting
diff --git a/tools/testing/selftests/futex/include/futex2test.h b/tools/testing/selftests/futex/include/futex2test.h
index 1f625b39948a9..53e88b60ac6d2 100644
--- a/tools/testing/selftests/futex/include/futex2test.h
+++ b/tools/testing/selftests/futex/include/futex2test.h
@@ -5,7 +5,9 @@
  * Copyright 2021 Collabora Ltd.
  */
 #include <linux/time_types.h>
+#include <errno.h>
 #include <stdint.h>
+#include <stdbool.h>
 
 #define u64_to_ptr(x) ((void *)(uintptr_t)(x))
 
@@ -96,3 +98,11 @@ static inline int futex2_wake(void *uaddr, int nr, unsigned int flags)
 {
 	return syscall(__NR_futex_wake, uaddr, ~0U, nr, flags);
 }
+
+static inline bool is_futex_waitv_supported(void)
+{
+	struct timespec ts = {0, 0};
+	int res = futex_waitv(NULL, 0, 0, &ts, CLOCK_MONOTONIC);
+
+	return !(res < 0 && errno == ENOSYS);
+}
diff --git a/tools/testing/selftests/futex/include/futex_thread.h b/tools/testing/selftests/futex/include/futex_thread.h
new file mode 100644
index 0000000000000..f71b5b604d4d1
--- /dev/null
+++ b/tools/testing/selftests/futex/include/futex_thread.h
@@ -0,0 +1,115 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#ifndef _FUTEX_THREAD_H
+#define _FUTEX_THREAD_H
+#include <errno.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "kselftest_harness.h"
+
+#define USEC_PER_SEC		1000000L
+#define WAIT_FOR_THREAD_SECS	2
+#define WAIT_FOR_THREAD_USECS	(WAIT_FOR_THREAD_SECS * USEC_PER_SEC)
+#define WAIT_THREAD_RETRIES	100
+
+struct futex_thread {
+	pthread_t		thread;
+	pthread_barrier_t	barrier;
+	pid_t			tid;
+	int			(*threadfn)(void *arg);
+	void			*arg;
+	int			retval;
+};
+
+static inline int __wait_for_thread(FILE *fp)
+{
+	unsigned int sleep_time_us = WAIT_FOR_THREAD_USECS / WAIT_THREAD_RETRIES;
+	char buf[80] = "";
+
+	for (int i = 0; i < WAIT_THREAD_RETRIES; i++) {
+		if (!fgets(buf, sizeof(buf), fp))
+			return EIO;
+		if (!strncmp(buf, "futex", 5))
+			return 0;
+		usleep(sleep_time_us);
+		rewind(fp);
+	}
+	return ETIMEDOUT;
+}
+
+static void *__futex_thread_fn(void *arg)
+{
+	struct futex_thread *t = arg;
+
+	t->tid = gettid();
+	pthread_barrier_wait(&t->barrier);
+	t->retval = t->threadfn(t->arg);
+	return NULL;
+}
+
+/**
+ * futex_wait_for_thread - Wait for the child thread to sleep in the futex context
+ * @t:          Thread handle.
+ * @_metadata:	Test metadata for TH_LOG() context
+ */
+static inline int futex_wait_for_thread(struct futex_thread *t, struct __test_metadata *_metadata)
+{
+	char fname[80];
+	FILE *fp;
+	int res;
+
+	snprintf(fname, sizeof(fname), "/proc/%d/wchan", t->tid);
+	fp = fopen(fname, "r");
+	if (!fp) {
+		/* If /proc/... is not available, sleep */
+		if (errno != ENOENT)
+			return errno;
+		TH_LOG("/proc/$PID/wchan not accessible, continue with sleep()");
+		sleep(WAIT_FOR_THREAD_SECS);
+		return 0;
+	}
+
+	res = __wait_for_thread(fp);
+	fclose(fp);
+	return res;
+}
+
+/**
+ * futex_thread_create - Create a new thread for testing.
+ * @t:        The handle of the newly created thread.
+ * @threadfn: The new thread starts execution by invoking threadfn
+ * @arg:      The parameters passed to threadfn.
+ */
+static inline int futex_thread_create(struct futex_thread *t, int (*threadfn)(void *), void *arg)
+{
+	pthread_barrier_init(&t->barrier, NULL, 2);
+
+	t->tid = 0;
+	t->threadfn = threadfn;
+	t->arg = arg;
+
+	if (pthread_create(&t->thread, NULL, __futex_thread_fn, t) < 0) {
+		int ret = errno;
+		pthread_barrier_destroy(&t->barrier);
+		return ret;
+	}
+
+	pthread_barrier_wait(&t->barrier);
+	return 0;
+}
+
+/**
+ * futex_thread_destroy - Wait for and reclaim the resources of the thread.
+ * @t:      Thread handle.
+ */
+static inline int futex_thread_destroy(struct futex_thread *t)
+{
+	pthread_join(t->thread, NULL);
+	pthread_barrier_destroy(&t->barrier);
+	return t->retval;
+}
+
+#endif
diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm
index d28a057fa6c2d..eb8af7b8a29a7 100644
--- a/tools/testing/selftests/kvm/Makefile.kvm
+++ b/tools/testing/selftests/kvm/Makefile.kvm
@@ -11,6 +11,7 @@ LIBKVM += lib/kvm_util.c
 LIBKVM += lib/lru_gen_util.c
 LIBKVM += lib/memstress.c
 LIBKVM += lib/guest_sprintf.c
+LIBKVM += lib/proc_util.c
 LIBKVM += lib/rbtree.c
 LIBKVM += lib/sparsebit.c
 LIBKVM += lib/test_util.c
@@ -120,6 +121,7 @@ TEST_GEN_PROGS_x86 += x86/svm_nested_soft_inject_test
 TEST_GEN_PROGS_x86 += x86/svm_nested_vmcb12_gpa
 TEST_GEN_PROGS_x86 += x86/svm_nested_pat_test
 TEST_GEN_PROGS_x86 += x86/svm_lbr_nested_state
+TEST_GEN_PROGS_x86 += x86/svm_pmu_host_guest_test
 TEST_GEN_PROGS_x86 += x86/tsc_scaling_sync
 TEST_GEN_PROGS_x86 += x86/sync_regs_test
 TEST_GEN_PROGS_x86 += x86/ucna_injection_test
@@ -156,6 +158,7 @@ TEST_GEN_PROGS_x86 += coalesced_io_test
 TEST_GEN_PROGS_x86 += dirty_log_perf_test
 TEST_GEN_PROGS_x86 += guest_memfd_test
 TEST_GEN_PROGS_x86 += hardware_disable_test
+TEST_GEN_PROGS_x86 += irq_test
 TEST_GEN_PROGS_x86 += mmu_stress_test
 TEST_GEN_PROGS_x86 += rseq_test
 TEST_GEN_PROGS_x86 += steal_time
@@ -174,6 +177,7 @@ TEST_GEN_PROGS_arm64 += arm64/hello_el2
 TEST_GEN_PROGS_arm64 += arm64/host_sve
 TEST_GEN_PROGS_arm64 += arm64/hypercalls
 TEST_GEN_PROGS_arm64 += arm64/external_aborts
+TEST_GEN_PROGS_arm64 += arm64/mmio_sign_ext
 TEST_GEN_PROGS_arm64 += arm64/page_fault_test
 TEST_GEN_PROGS_arm64 += arm64/psci_test
 TEST_GEN_PROGS_arm64 += arm64/sea_to_user
@@ -258,6 +262,7 @@ OVERRIDE_TARGETS = 1
 # which causes the environment variable to override the makefile).
 include ../lib.mk
 include ../cgroup/lib/libcgroup.mk
+include ../vfio/lib/libvfio.mk
 
 INSTALL_HDR_PATH = $(top_srcdir)/usr
 LINUX_HDR_PATH = $(INSTALL_HDR_PATH)/include/
@@ -312,7 +317,9 @@ LIBKVM_S := $(filter %.S,$(LIBKVM))
 LIBKVM_C_OBJ := $(patsubst %.c, $(OUTPUT)/%.o, $(LIBKVM_C))
 LIBKVM_S_OBJ := $(patsubst %.S, $(OUTPUT)/%.o, $(LIBKVM_S))
 LIBKVM_STRING_OBJ := $(patsubst %.c, $(OUTPUT)/%.o, $(LIBKVM_STRING))
-LIBKVM_OBJS = $(LIBKVM_C_OBJ) $(LIBKVM_S_OBJ) $(LIBKVM_STRING_OBJ) $(LIBCGROUP_O)
+LIBKVM_OBJS = $(LIBKVM_C_OBJ) $(LIBKVM_S_OBJ) $(LIBKVM_STRING_OBJ)
+LIBKVM_OBJS += $(LIBCGROUP_O)
+LIBKVM_OBJS += $(LIBVFIO_O)
 SPLIT_TEST_GEN_PROGS := $(patsubst %, $(OUTPUT)/%, $(SPLIT_TESTS))
 SPLIT_TEST_GEN_OBJ := $(patsubst %, $(OUTPUT)/$(ARCH)/%.o, $(SPLIT_TESTS))
 
diff --git a/tools/testing/selftests/kvm/arch_timer.c b/tools/testing/selftests/kvm/arch_timer.c
index 90c475a61b22b..f8b02597897bd 100644
--- a/tools/testing/selftests/kvm/arch_timer.c
+++ b/tools/testing/selftests/kvm/arch_timer.c
@@ -85,7 +85,7 @@ static u32 test_get_pcpu(void)
 	cpu_set_t online_cpuset;
 
 	nproc_conf = get_nprocs_conf();
-	sched_getaffinity(0, sizeof(cpu_set_t), &online_cpuset);
+	kvm_sched_getaffinity(0, sizeof(cpu_set_t), &online_cpuset);
 
 	/* Randomly find an available pCPU to place a vCPU on */
 	do {
diff --git a/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c b/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c
index f7625eb711d6b..d9c9377a63256 100644
--- a/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c
+++ b/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c
@@ -1039,7 +1039,7 @@ int main(int argc, char *argv[])
 	if (!parse_args(argc, argv))
 		exit(KSFT_SKIP);
 
-	sched_getaffinity(0, sizeof(default_cpuset), &default_cpuset);
+	kvm_sched_getaffinity(0, sizeof(default_cpuset), &default_cpuset);
 	set_counter_defaults();
 
 	if (test_args.test_virtual) {
diff --git a/tools/testing/selftests/kvm/arm64/mmio_sign_ext.c b/tools/testing/selftests/kvm/arm64/mmio_sign_ext.c
new file mode 100644
index 0000000000000..25196f1e6322d
--- /dev/null
+++ b/tools/testing/selftests/kvm/arm64/mmio_sign_ext.c
@@ -0,0 +1,255 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * mmio_sign_ext - Test sign-extending MMIO load emulation (LDRSB/LDRSH/LDRSW)
+ *
+ * Copyright (c) 2026 Google LLC
+ * Author: Fuad Tabba <fuad.tabba@linux.dev>
+ */
+
+#include <asm/ptrace.h>
+
+#include "processor.h"
+#include "test_util.h"
+
+#define MMIO_ADDR	0x8000000ULL
+
+/* AP[1]: allow unprivileged (EL0) access to a mapping. */
+#define PTE_USER	BIT(6)
+
+/* SPSR for ERET to EL0t with DAIF masked. */
+#define SPSR_EL0	(PSR_MODE_EL0t | PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)
+
+struct mmio_test {
+	const char *name;
+	uint64_t data;		/* access-width value, host byte order */
+	uint8_t len;
+	uint64_t expected;	/* sign-extended result; same for LE and BE */
+};
+
+/* Paired 1:1, in order, with the loads in guest_loads_le() and el0_be_loads. */
+static const struct mmio_test tests[] = {
+	/* LDRSB Xt: byte sign-extended to 64 bits */
+	{ "LDRSB Xt 0xFF",	0xFF,		1, 0xFFFFFFFFFFFFFFFFULL },
+	{ "LDRSB Xt 0x7F",	0x7F,		1, 0x7FULL },
+
+	/* LDRSB Wt: byte sign-extended to 32 bits, upper 32 bits zeroed */
+	{ "LDRSB Wt 0xFF",	0xFF,		1, 0xFFFFFFFFULL },
+	{ "LDRSB Wt 0x7F",	0x7F,		1, 0x7FULL },
+
+	/* LDRSH Xt: halfword sign-extended to 64 bits */
+	{ "LDRSH Xt 0x8001",	0x8001,		2, 0xFFFFFFFFFFFF8001ULL },
+	{ "LDRSH Xt 0x7FFF",	0x7FFF,		2, 0x7FFFULL },
+
+	/* LDRSH Wt: halfword sign-extended to 32 bits, upper 32 bits zeroed */
+	{ "LDRSH Wt 0x8001",	0x8001,		2, 0xFFFF8001ULL },
+	{ "LDRSH Wt 0x7FFF",	0x7FFF,		2, 0x7FFFULL },
+
+	/* LDRSW Xt: word sign-extended to 64 bits (no Wt form) */
+	{ "LDRSW Xt 0x80000001", 0x80000001,	4, 0xFFFFFFFF80000001ULL },
+	{ "LDRSW Xt 0x7FFFFFFF", 0x7FFFFFFF,	4, 0x7FFFFFFFULL },
+};
+
+/* Issue one sign-extending load from MMIO and report the result. */
+#define GUEST_LDRS(load) do {						\
+	uint64_t val;							\
+									\
+	asm volatile(load : "=r"(val) : "r"(MMIO_ADDR) : "memory");	\
+	GUEST_SYNC(val);						\
+} while (0)
+
+/* Little-endian pass: loads issued at EL1. */
+static void guest_loads_le(void)
+{
+	GUEST_LDRS("ldrsb %0, [%1]");
+	GUEST_LDRS("ldrsb %0, [%1]");
+	GUEST_LDRS("ldrsb %w0, [%1]");
+	GUEST_LDRS("ldrsb %w0, [%1]");
+	GUEST_LDRS("ldrsh %0, [%1]");
+	GUEST_LDRS("ldrsh %0, [%1]");
+	GUEST_LDRS("ldrsh %w0, [%1]");
+	GUEST_LDRS("ldrsh %w0, [%1]");
+	GUEST_LDRS("ldrsw %0, [%1]");
+	GUEST_LDRS("ldrsw %0, [%1]");
+}
+
+/*
+ * Run the big-endian loads at EL0, where SCTLR_EL1.E0E flips only the data
+ * endianness; at EL1, SCTLR_EL1.EE would also flip the page-table walk and
+ * fault on the little-endian tables. x0 holds MMIO_ADDR; results return in
+ * x19..x28 (tests[] order) via a single SVC.
+ */
+extern char el0_be_loads[];
+asm(
+"	.pushsection .text, \"ax\"\n"
+"	.global el0_be_loads\n"
+"el0_be_loads:\n"
+"	ldrsb	x19, [x0]\n"
+"	ldrsb	x20, [x0]\n"
+"	ldrsb	w21, [x0]\n"
+"	ldrsb	w22, [x0]\n"
+"	ldrsh	x23, [x0]\n"
+"	ldrsh	x24, [x0]\n"
+"	ldrsh	w25, [x0]\n"
+"	ldrsh	w26, [x0]\n"
+"	ldrsw	x27, [x0]\n"
+"	ldrsw	x28, [x0]\n"
+"	svc	#0\n"
+"	.popsection\n"
+);
+
+/* EL1 handler for the EL0 SVC: report the results, then finish. */
+static void el0_svc_handler(struct ex_regs *regs)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(tests); i++)
+		GUEST_SYNC(regs->regs[19 + i]);
+
+	GUEST_DONE();
+}
+
+static bool guest_mixed_endian_el0(void)
+{
+	uint64_t mmfr0 = read_sysreg(id_aa64mmfr0_el1);
+
+	return SYS_FIELD_GET(ID_AA64MMFR0_EL1, BIGEND, mmfr0) ||
+	       SYS_FIELD_GET(ID_AA64MMFR0_EL1, BIGENDEL0, mmfr0);
+}
+
+static void guest_code(void)
+{
+	guest_loads_le();
+
+	if (guest_mixed_endian_el0()) {
+		write_sysreg(read_sysreg(sctlr_el1) | SCTLR_EL1_E0E, sctlr_el1);
+		isb();
+
+		asm volatile(
+		"	msr	elr_el1, %[pc]\n"
+		"	msr	spsr_el1, %[spsr]\n"
+		"	mov	x0, %[mmio]\n"
+		"	isb\n"
+		"	eret\n"
+		:
+		: [pc] "r"(el0_be_loads),
+		  [spsr] "r"((uint64_t)SPSR_EL0),
+		  [mmio] "r"(MMIO_ADDR)
+		: "x0", "memory");
+		__builtin_unreachable();	/* el0_svc_handler ends the test */
+	}
+
+	GUEST_DONE();
+}
+
+static void handle_mmio(struct kvm_run *run, const struct mmio_test *t, bool be)
+{
+	int i;
+
+	TEST_ASSERT_EQ(run->mmio.phys_addr, MMIO_ADDR);
+	TEST_ASSERT(!run->mmio.is_write, "Expected MMIO read for %s", t->name);
+	TEST_ASSERT_EQ(run->mmio.len, t->len);
+
+	memset(run->mmio.data, 0, sizeof(run->mmio.data));
+	if (be) {
+		/* The guest reads the device bytes most-significant first. */
+		for (i = 0; i < t->len; i++)
+			run->mmio.data[i] = t->data >> (8 * (t->len - 1 - i));
+	} else {
+		/* Works because arm64 KVM hosts are always little-endian. */
+		memcpy(run->mmio.data, &t->data, t->len);
+	}
+}
+
+static void expect_sync(struct kvm_vcpu *vcpu, struct ucall *uc,
+			const struct mmio_test *t)
+{
+	switch (get_ucall(vcpu, uc)) {
+	case UCALL_SYNC:
+		TEST_ASSERT(uc->args[1] == t->expected,
+			    "%s: got %#lx, want %#lx", t->name,
+			    (unsigned long)uc->args[1], (unsigned long)t->expected);
+		break;
+	case UCALL_ABORT:
+		REPORT_GUEST_ASSERT(*uc);
+		break;
+	default:
+		TEST_FAIL("Unexpected ucall for %s", t->name);
+	}
+}
+
+/* OR PTE_USER into the leaf descriptors covering [gva, gva + len). */
+static void make_el0_accessible(struct kvm_vm *vm, uint64_t gva, uint64_t len)
+{
+	uint64_t addr;
+
+	for (addr = gva & ~((uint64_t)vm->page_size - 1); addr < gva + len;
+	     addr += vm->page_size)
+		*virt_get_pte_hva(vm, addr) |= PTE_USER;
+}
+
+static bool vcpu_mixed_endian_el0(struct kvm_vcpu *vcpu)
+{
+	uint64_t mmfr0 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64MMFR0_EL1));
+
+	return SYS_FIELD_GET(ID_AA64MMFR0_EL1, BIGEND, mmfr0) ||
+	       SYS_FIELD_GET(ID_AA64MMFR0_EL1, BIGENDEL0, mmfr0);
+}
+
+int main(void)
+{
+	struct kvm_vcpu *vcpu;
+	struct kvm_vm *vm;
+	struct ucall uc;
+	unsigned int i;
+	bool be;
+
+	vm = vm_create_with_one_vcpu(&vcpu, guest_code);
+	virt_map(vm, MMIO_ADDR, MMIO_ADDR, 1);
+
+	vm_init_descriptor_tables(vm);
+	vcpu_init_descriptor_tables(vcpu);
+	vm_install_sync_handler(vm, VECTOR_SYNC_LOWER_64, ESR_ELx_EC_SVC64,
+				el0_svc_handler);
+
+	be = vcpu_mixed_endian_el0(vcpu);
+	if (be)
+		make_el0_accessible(vm, MMIO_ADDR, vm->page_size);
+
+	ksft_print_header();
+	ksft_set_plan(ARRAY_SIZE(tests) * (be ? 2 : 1));
+
+	/* Little-endian pass: one load and one result per iteration. */
+	for (i = 0; i < ARRAY_SIZE(tests); i++) {
+		const struct mmio_test *t = &tests[i];
+
+		vcpu_run(vcpu);
+		TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_MMIO);
+		handle_mmio(vcpu->run, t, false);
+
+		vcpu_run(vcpu);
+		expect_sync(vcpu, &uc, t);
+
+		ksft_test_result_pass("%s\n", t->name);
+	}
+
+	if (be) {
+		/* The EL0 stub issues all the loads, then reports the results. */
+		for (i = 0; i < ARRAY_SIZE(tests); i++) {
+			vcpu_run(vcpu);
+			TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_MMIO);
+			handle_mmio(vcpu->run, &tests[i], true);
+		}
+		for (i = 0; i < ARRAY_SIZE(tests); i++) {
+			vcpu_run(vcpu);
+			expect_sync(vcpu, &uc, &tests[i]);
+			ksft_test_result_pass("BE %s\n", tests[i].name);
+		}
+	}
+
+	vcpu_run(vcpu);
+	TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_DONE, "Expected UCALL_DONE");
+
+	kvm_vm_free(vm);
+
+	ksft_finished();
+}
diff --git a/tools/testing/selftests/kvm/demand_paging_test.c b/tools/testing/selftests/kvm/demand_paging_test.c
index 302c4923d0930..f8b3d0b68830d 100644
--- a/tools/testing/selftests/kvm/demand_paging_test.c
+++ b/tools/testing/selftests/kvm/demand_paging_test.c
@@ -57,7 +57,7 @@ static void vcpu_worker(struct memstress_vcpu_args *vcpu_args)
 static int handle_uffd_page_request(int uffd_mode, int uffd,
 		struct uffd_msg *msg)
 {
-	pid_t tid = syscall(__NR_gettid);
+	pid_t tid = kvm_gettid();
 	u64 addr = msg->arg.pagefault.address;
 	struct timespec start;
 	struct timespec ts_diff;
diff --git a/tools/testing/selftests/kvm/dirty_log_perf_test.c b/tools/testing/selftests/kvm/dirty_log_perf_test.c
index ef779fa91827c..7c5abe1ae9e01 100644
--- a/tools/testing/selftests/kvm/dirty_log_perf_test.c
+++ b/tools/testing/selftests/kvm/dirty_log_perf_test.c
@@ -311,7 +311,7 @@ int main(int argc, char *argv[])
 	int opt;
 
 	/* Override the seed to be deterministic by default. */
-	guest_random_seed = 1;
+	kvm_random_seed = 1;
 
 	dirty_log_manual_caps =
 		kvm_check_cap(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2);
@@ -357,7 +357,7 @@ int main(int argc, char *argv[])
 			p.phys_offset = strtoull(optarg, NULL, 0);
 			break;
 		case 'r':
-			guest_random_seed = atoi_positive("Random seed", optarg);
+			kvm_random_seed = atoi_positive("Random seed", optarg);
 			break;
 		case 's':
 			p.backing_src = parse_backing_src_type(optarg);
diff --git a/tools/testing/selftests/kvm/dirty_log_test.c b/tools/testing/selftests/kvm/dirty_log_test.c
index 74ca096bf976b..e8419d7da1eab 100644
--- a/tools/testing/selftests/kvm/dirty_log_test.c
+++ b/tools/testing/selftests/kvm/dirty_log_test.c
@@ -121,7 +121,7 @@ static void guest_code(void)
 	while (true) {
 		while (!READ_ONCE(vcpu_stop)) {
 			addr = guest_test_virt_mem;
-			addr += (guest_random_u64(&guest_rng) % guest_num_pages)
+			addr += (kvm_random_u64(&kvm_rng) % guest_num_pages)
 				* guest_page_size;
 			addr = align_down(addr, host_page_size);
 
@@ -708,8 +708,7 @@ static void run_test(enum vm_guest_mode mode, void *arg)
 
 		sync_global_to_guest(vm, iteration);
 
-		WRITE_ONCE(nr_writes, 0);
-		sync_global_to_guest(vm, nr_writes);
+		WRITE_AND_SYNC_TO_GUEST(vm, nr_writes, 0);
 
 		dirty_ring_prev_iteration_last_page = dirty_ring_last_page;
 		WRITE_ONCE(dirty_ring_vcpu_ring_full, false);
@@ -775,16 +774,14 @@ static void run_test(enum vm_guest_mode mode, void *arg)
 		 * writing memory during verification, pages that this thread
 		 * sees as clean may be written with this iteration's value.
 		 */
-		WRITE_ONCE(vcpu_stop, true);
-		sync_global_to_guest(vm, vcpu_stop);
+		WRITE_AND_SYNC_TO_GUEST(vm, vcpu_stop, true);
 		sem_wait(&sem_vcpu_stop);
 
 		/*
 		 * Clear vcpu_stop after the vCPU thread has acknowledge the
 		 * stop request and is waiting, i.e. is definitely not running!
 		 */
-		WRITE_ONCE(vcpu_stop, false);
-		sync_global_to_guest(vm, vcpu_stop);
+		WRITE_AND_SYNC_TO_GUEST(vm, vcpu_stop, false);
 
 		/*
 		 * Sync the number of writes performed before verification, the
diff --git a/tools/testing/selftests/kvm/include/kvm_syscalls.h b/tools/testing/selftests/kvm/include/kvm_syscalls.h
index 6cb3bed29b81a..5dae6143ddb04 100644
--- a/tools/testing/selftests/kvm/include/kvm_syscalls.h
+++ b/tools/testing/selftests/kvm/include/kvm_syscalls.h
@@ -12,6 +12,7 @@
 #include <sys/mman.h>
 #include <sys/syscall.h>
 
+#include <sched.h>
 #include <test_util.h>
 
 #define MAP_ARGS0(m,...)
@@ -83,11 +84,17 @@ static inline int kvm_dup(int fd)
 	return new_fd;
 }
 
+static inline pid_t kvm_gettid(void)
+{
+	return syscall(__NR_gettid);
+}
+
 __KVM_SYSCALL_DEFINE(munmap, 2, void *, mem, size_t, size);
 __KVM_SYSCALL_DEFINE(close, 1, int, fd);
 __KVM_SYSCALL_DEFINE(fallocate, 4, int, fd, int, mode, loff_t, offset, loff_t, len);
 __KVM_SYSCALL_DEFINE(ftruncate, 2, unsigned int, fd, off_t, length);
 __KVM_SYSCALL_DEFINE(madvise, 3, void *, addr, size_t, length, int, advice);
+__KVM_SYSCALL_DEFINE(sched_getaffinity, 3, pid_t, pid, size_t, cpusetsize, cpu_set_t *, mask);
 
 #define kvm_free_fd(fd)		\
 do {				\
diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h
index 04a910164a296..b39e713c30a46 100644
--- a/tools/testing/selftests/kvm/include/kvm_util.h
+++ b/tools/testing/selftests/kvm/include/kvm_util.h
@@ -1094,6 +1094,8 @@ static inline void pin_task_to_cpu(pthread_t task, int cpu)
 	TEST_ASSERT(!r, "Failed to set thread affinity to pCPU '%u'", cpu);
 }
 
+void pin_task_to_random_cpu(pthread_t task, cpu_set_t *possible_cpus);
+
 static inline int pin_task_to_any_cpu(pthread_t task)
 {
 	int cpu = sched_getcpu();
@@ -1138,6 +1140,16 @@ vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages)
 	memcpy(&(g), _p, sizeof(g));				\
 })
 
+#define SYNC_FROM_GUEST_AND_READ(_vm, _variable) ({		\
+	sync_global_from_guest(_vm, _variable);			\
+	READ_ONCE(_variable);					\
+})
+
+#define WRITE_AND_SYNC_TO_GUEST(_vm, _variable, _value) do {	\
+	WRITE_ONCE(_variable, _value);				\
+	sync_global_to_guest(_vm, _variable);			\
+} while (0)
+
 /*
  * Write a global value, but only in the VM's (guest's) domain.  Primarily used
  * for "globals" that hold per-VM values (VMs always duplicate code and global
diff --git a/tools/testing/selftests/kvm/include/proc_util.h b/tools/testing/selftests/kvm/include/proc_util.h
new file mode 100644
index 0000000000000..d1ddc967d11dc
--- /dev/null
+++ b/tools/testing/selftests/kvm/include/proc_util.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef SELFTEST_KVM_PROC_UTIL_H
+#define SELFTEST_KVM_PROC_UTIL_H
+
+#include <stdint.h>
+
+unsigned int vfio_msix_to_host_irq(const char *vfio_device_bdf, int msix);
+
+void proc_irq_set_smp_affinity(unsigned int irq, int cpu);
+
+#endif /* SELFTEST_KVM_PROC_UTIL_H */
diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h
index a56271c237ae9..d64c8a228207a 100644
--- a/tools/testing/selftests/kvm/include/test_util.h
+++ b/tools/testing/selftests/kvm/include/test_util.h
@@ -108,32 +108,35 @@ struct timespec timespec_sub(struct timespec ts1, struct timespec ts2);
 struct timespec timespec_elapsed(struct timespec start);
 struct timespec timespec_div(struct timespec ts, int divisor);
 
-struct guest_random_state {
+struct kvm_random_state {
 	u32 seed;
 };
 
-extern u32 guest_random_seed;
-extern struct guest_random_state guest_rng;
+extern u32 kvm_random_seed;
+extern struct kvm_random_state kvm_rng;
 
-struct guest_random_state new_guest_random_state(u32 seed);
-u32 guest_random_u32(struct guest_random_state *state);
+struct kvm_random_state new_kvm_random_state(u32 seed);
+u32 kvm_random_u32(struct kvm_random_state *state);
 
-static inline bool __guest_random_bool(struct guest_random_state *state,
+static inline bool __kvm_random_bool(struct kvm_random_state *state,
 				       u8 percent)
 {
-	return (guest_random_u32(state) % 100) < percent;
+	return (kvm_random_u32(state) % 100) < percent;
 }
 
-static inline bool guest_random_bool(struct guest_random_state *state)
+static inline bool kvm_random_bool(struct kvm_random_state *state)
 {
-	return __guest_random_bool(state, 50);
+	return __kvm_random_bool(state, 50);
 }
 
-static inline u64 guest_random_u64(struct guest_random_state *state)
+static inline u64 kvm_random_u64(struct kvm_random_state *state)
 {
-	return ((u64)guest_random_u32(state) << 32) | guest_random_u32(state);
+	return ((u64)kvm_random_u32(state) << 32) | kvm_random_u32(state);
 }
 
+u64 kvm_random_u64_in_range(struct kvm_random_state *state, u64 min,
+			    u64 max);
+
 enum vm_mem_backing_src_type {
 	VM_MEM_SRC_ANONYMOUS,
 	VM_MEM_SRC_ANONYMOUS_THP,
diff --git a/tools/testing/selftests/kvm/include/x86/kvm_util_arch.h b/tools/testing/selftests/kvm/include/x86/kvm_util_arch.h
index c33ab6e041717..6904dbda79f92 100644
--- a/tools/testing/selftests/kvm/include/x86/kvm_util_arch.h
+++ b/tools/testing/selftests/kvm/include/x86/kvm_util_arch.h
@@ -55,9 +55,9 @@ static inline bool __vm_arch_has_protected_memory(struct kvm_vm_arch *arch)
 do {											\
 	const typeof(mem) val = (__val);						\
 											\
-	if (!is_forced_emulation_enabled || guest_random_bool(&guest_rng)) {		\
+	if (!is_forced_emulation_enabled || kvm_random_bool(&kvm_rng)) {		\
 		(mem) = val;								\
-	} else if (guest_random_bool(&guest_rng)) {					\
+	} else if (kvm_random_bool(&kvm_rng)) {					\
 		__asm__ __volatile__(KVM_FEP "mov %1, %0"				\
 				     : "+m" (mem)					\
 				     : "r" (val) : "memory");				\
diff --git a/tools/testing/selftests/kvm/include/x86/pmu.h b/tools/testing/selftests/kvm/include/x86/pmu.h
index 98537cc8840d1..608ed83d7c6a6 100644
--- a/tools/testing/selftests/kvm/include/x86/pmu.h
+++ b/tools/testing/selftests/kvm/include/x86/pmu.h
@@ -38,6 +38,12 @@
 #define ARCH_PERFMON_EVENTSEL_INV		BIT_ULL(23)
 #define ARCH_PERFMON_EVENTSEL_CMASK		GENMASK_ULL(31, 24)
 
+/*
+ * These are AMD-specific bits.
+ */
+#define AMD64_EVENTSEL_GUESTONLY		BIT_ULL(40)
+#define AMD64_EVENTSEL_HOSTONLY			BIT_ULL(41)
+
 /* RDPMC control flags, Intel only. */
 #define INTEL_RDPMC_METRICS			BIT_ULL(29)
 #define INTEL_RDPMC_FIXED			BIT_ULL(30)
diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h
index 513e4a1075fac..7d3a27bc0d842 100644
--- a/tools/testing/selftests/kvm/include/x86/processor.h
+++ b/tools/testing/selftests/kvm/include/x86/processor.h
@@ -1226,6 +1226,8 @@ struct idt_entry {
 void vm_install_exception_handler(struct kvm_vm *vm, int vector,
 			void (*handler)(struct ex_regs *));
 
+gva_t vm_alloc_stack(struct kvm_vm *vm, int nr_pages);
+
 /*
  * Exception fixup morphs #DE to an arbitrary magic vector so that '0' can be
  * used to signal "no expcetion".
@@ -1392,6 +1394,14 @@ static inline bool kvm_is_pmu_enabled(void)
 	return get_kvm_param_bool("enable_pmu");
 }
 
+static inline bool kvm_is_mediated_pmu_enabled(void)
+{
+	if (host_cpu_is_intel)
+		return get_kvm_intel_param_bool("enable_mediated_pmu");
+
+	return get_kvm_amd_param_bool("enable_mediated_pmu");
+}
+
 static inline bool kvm_is_forced_emulation_enabled(void)
 {
 	return !!get_kvm_param_integer("force_emulation_prefix");
diff --git a/tools/testing/selftests/kvm/include/x86/svm_util.h b/tools/testing/selftests/kvm/include/x86/svm_util.h
index 6c013eb838beb..c201c30485e72 100644
--- a/tools/testing/selftests/kvm/include/x86/svm_util.h
+++ b/tools/testing/selftests/kvm/include/x86/svm_util.h
@@ -28,6 +28,9 @@ struct svm_test_data {
 	void *msr_hva;
 	u64 msr_gpa;
 
+	/* Stack */
+	void *stack; /* gva */
+
 	/* NPT */
 	u64 ncr3_gpa;
 };
@@ -57,7 +60,7 @@ static inline void vmmcall(void)
 		)
 
 struct svm_test_data *vcpu_alloc_svm(struct kvm_vm *vm, gva_t *p_svm_gva);
-void generic_svm_setup(struct svm_test_data *svm, void *guest_rip, void *guest_rsp);
+void generic_svm_setup(struct svm_test_data *svm, void *guest_rip);
 void run_guest(struct vmcb *vmcb, u64 vmcb_gpa);
 
 static inline bool kvm_cpu_has_npt(void)
diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h
index 90fffaf915958..4bcfd60e3aecb 100644
--- a/tools/testing/selftests/kvm/include/x86/vmx.h
+++ b/tools/testing/selftests/kvm/include/x86/vmx.h
@@ -524,6 +524,8 @@ struct vmx_pages {
 	u64 apic_access_gpa;
 	void *apic_access;
 
+	void *stack;
+
 	u64 eptp_gpa;
 };
 
@@ -552,7 +554,7 @@ union vmx_ctrl_msr {
 
 struct vmx_pages *vcpu_alloc_vmx(struct kvm_vm *vm, gva_t *p_vmx_gva);
 bool prepare_for_vmx_operation(struct vmx_pages *vmx);
-void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip, void *guest_rsp);
+void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip);
 bool load_vmcs(struct vmx_pages *vmx);
 
 bool ept_1g_pages_supported(void);
diff --git a/tools/testing/selftests/kvm/irq_test.c b/tools/testing/selftests/kvm/irq_test.c
new file mode 100644
index 0000000000000..240f6f0fdbe43
--- /dev/null
+++ b/tools/testing/selftests/kvm/irq_test.c
@@ -0,0 +1,362 @@
+// SPDX-License-Identifier: GPL-2.0
+#include "kvm_util.h"
+#include "test_util.h"
+#include "apic.h"
+#include "processor.h"
+#include "proc_util.h"
+
+#include <libvfio.h>
+#include <linux/sizes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <sys/eventfd.h>
+#include <sys/sysinfo.h>
+
+static u64 timeout_ns = 2ULL * 1000 * 1000 * 1000;
+static bool guest_ready_for_irqs[KVM_MAX_VCPUS];
+static bool guest_received_irq[KVM_MAX_VCPUS];
+static bool guest_received_nmi[KVM_MAX_VCPUS];
+static bool x2apic = true;
+static bool irq_affinity;
+static bool done;
+
+#define GUEST_RECEIVED_IRQ(__vcpu)	\
+	SYNC_FROM_GUEST_AND_READ((__vcpu)->vm, guest_received_irq[(__vcpu)->id])
+#define GUEST_RECEIVED_NMI(__vcpu)	\
+	SYNC_FROM_GUEST_AND_READ((__vcpu)->vm, guest_received_nmi[(__vcpu)->id])
+
+#define GUEST_RECEIVED_INTERRUPT(__vcpu, __nmi)	\
+	((__nmi) ? GUEST_RECEIVED_NMI(__vcpu) : GUEST_RECEIVED_IRQ(__vcpu))
+
+static u32 guest_get_vcpu_id(void)
+{
+	if (x2apic)
+		return x2apic_read_reg(APIC_ID);
+	else
+		return xapic_read_reg(APIC_ID) >> 24;
+}
+
+static void guest_irq_handler(struct ex_regs *regs)
+{
+	WRITE_ONCE(guest_received_irq[guest_get_vcpu_id()], true);
+
+	if (x2apic)
+		x2apic_write_reg(APIC_EOI, 0);
+	else
+		xapic_write_reg(APIC_EOI, 0);
+}
+
+static void guest_nmi_handler(struct ex_regs *regs)
+{
+	WRITE_ONCE(guest_received_nmi[guest_get_vcpu_id()], true);
+}
+
+static void guest_code(void)
+{
+	if (x2apic)
+		x2apic_enable();
+	else
+		xapic_enable();
+
+	sti_nop();
+
+	WRITE_ONCE(guest_ready_for_irqs[guest_get_vcpu_id()], true);
+
+	while (!READ_ONCE(done))
+		cpu_relax();
+
+	GUEST_DONE();
+}
+
+static void *vcpu_thread_main(void *arg)
+{
+	struct kvm_vcpu *vcpu = arg;
+	struct ucall uc;
+
+	vcpu_run(vcpu);
+	TEST_ASSERT_EQ(UCALL_DONE, get_ucall(vcpu, &uc));
+
+	return NULL;
+}
+
+static int vfio_setup_msi(struct vfio_pci_device *device)
+{
+	const int flags = MAP_SHARED | MAP_ANONYMOUS;
+	const int prot = PROT_READ | PROT_WRITE;
+	struct iova_allocator *allocator;
+	struct dma_region *region;
+
+	/* Sanity check that the device+driver can actually send MSIs. */
+	TEST_REQUIRE(device->driver.ops);
+	TEST_REQUIRE(device->driver.ops->send_msi);
+
+	/*
+	 * Set up a DMA-able region for the driver to use.   Very few devices
+	 * provide a way to arbitrarily send interrupts (MSIs), e.g. by writing
+	 * an MMIO register.  Instead, most devices send MSIs when an action is
+	 * completed, and practically all actions involve DMA of some form.
+	 */
+	allocator = iova_allocator_init(device->iommu);
+
+	region = &device->driver.region;
+	region->size = SZ_2M;
+	region->iova = iova_allocator_alloc(allocator, region->size);
+	region->vaddr = kvm_mmap(region->size, prot, flags, -1);
+	TEST_ASSERT(region->vaddr != MAP_FAILED, "mmap() failed\n");
+	iommu_map(device->iommu, region);
+
+	iova_allocator_cleanup(allocator);
+
+	vfio_pci_driver_init(device);
+
+	return device->driver.msi;
+}
+
+static void trigger_interrupt(struct vfio_pci_device *device, int eventfd)
+{
+	if (device)
+		vfio_pci_driver_send_msi(device);
+	else
+		eventfd_write(eventfd, 1);
+}
+
+
+static void kvm_route_msi(struct kvm_vm *vm, u32 gsi, struct kvm_vcpu *vcpu,
+			  u8 vector, bool use_nmi)
+{
+	struct {
+		struct kvm_irq_routing header;
+		struct kvm_irq_routing_entry entry;
+	} routing = {
+		.header.nr = 1,
+		.entry = {
+			.gsi = gsi,
+			.type = KVM_IRQ_ROUTING_MSI,
+			.u.msi.address_lo = 0xFEE00000 | (vcpu->id & GENMASK(7, 0)) << 12,
+			.u.msi.address_hi = vcpu->id & GENMASK(31, 8),
+			.u.msi.data = use_nmi ? NMI_VECTOR | (4 << 8) : vector,
+		},
+	};
+
+	vm_ioctl(vm, KVM_SET_GSI_ROUTING, &routing.header);
+}
+
+static void kvm_set_empty_gsi_routing(struct kvm_vm *vm)
+{
+	struct kvm_irq_routing routing = {};
+
+	vm_ioctl(vm, KVM_SET_GSI_ROUTING, &routing);
+}
+
+static const char *probe_iommu_type(void)
+{
+	int io_fd;
+
+	io_fd = open("/dev/iommu", O_RDONLY);
+	if (io_fd >= 0) {
+		close(io_fd);
+		return MODE_IOMMUFD;
+	}
+
+	io_fd = __open_path_or_exit("/dev/vfio/vfio", O_RDONLY,
+				    "Is VFIO (or IOMMUFD) loaded and enabled?");
+	close(io_fd);
+	return MODE_VFIO_TYPE1_IOMMU;
+}
+
+static void help(const char *name)
+{
+	printf("Usage: %s [-a] [-d <segment:bus:device.function>] [-e] [-h] [-i nr_irqs] [-m] [-n] [-t iommu_type] [-v nr_vcpus] [-x]\n", name);
+	printf("\n");
+	printf("Tests KVM interrupt routing and delivery via irqfd.\n");
+	printf("-a	Affine the device's host IRQ to a random physical CPU\n");
+	printf("-d	Use a VFIO device to send MSI-X interrupts instead of manually signaling the eventfd\n");
+	printf("-e	Set empty GSI routing in-between some interrupts\n");
+	printf("-i	The number of IRQs to generate during the test\n");
+	printf("-m	Pin target vCPU to random physical CPU before triggering interrupt\n");
+	printf("-n	Deliver 50 percent of IRQs as non-maskable interrupts\n");
+	printf("-t	Override the IOMMU type to use (vfio_type1_iommu or iommufd)\n");
+	printf("-v	Number of vCPUS to run\n");
+	printf("-x	Use xAPIC mode instead of x2APIC mode in the guest\n");
+	printf("\n");
+	exit(KSFT_FAIL);
+}
+
+int main(int argc, char **argv)
+{
+	/*
+	 * Pick a random vector and a random GSI to use for device IRQ.
+	 *
+	 * Pick an IRQ vector in range [32, UINT8_MAX]. Min value is 32 because
+	 * Linux/x86 reserves vectors 0-31 for exceptions and architecture
+	 * defined NMIs and interrupts.
+	 *
+	 * Pick a GSI in range [24, KVM_MAX_IRQ_ROUTES - 1]. The min value is 24
+	 * because KVM reserves GSIs 0-15 for legacy ISA IRQs and 16-23 only go
+	 * to the IOAPIC. The max is KVM_MAX_IRQ_ROUTES - 1, because
+	 * KVM_MAX_IRQ_ROUTES is exclusive.
+	 */
+	u32 gsi = kvm_random_u64_in_range(&kvm_rng, 24, KVM_MAX_IRQ_ROUTES - 1);
+	u8 vector = kvm_random_u64_in_range(&kvm_rng, 32, UINT8_MAX);
+
+	pthread_t vcpu_threads[KVM_MAX_VCPUS];
+	struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
+	struct vfio_pci_device *device = NULL;
+	int nr_irqs = 1000, nr_vcpus = 1;
+	bool set_empty_routing = false;
+	const char *device_bdf = NULL;
+	const char *iommu_type = NULL;
+	int i, j, c, msix, eventfd;
+	bool migrate_vcpus = false;
+	cpu_set_t available_cpus;
+	bool use_nmi = false;
+	struct iommu *iommu;
+	struct kvm_vm *vm;
+	int irq, irq_cpu;
+
+	while ((c = getopt(argc, argv, "ad:ehi:mnt:v:x")) != -1) {
+		switch (c) {
+		case 'a':
+			irq_affinity = true;
+			break;
+		case 'd':
+			device_bdf = optarg;
+			break;
+		case 'e':
+			set_empty_routing = true;
+			break;
+		case 'i':
+			nr_irqs = atoi_positive("Number of IRQs", optarg);
+			break;
+		case 'm':
+			migrate_vcpus = true;
+			break;
+		case 'n':
+			use_nmi = true;
+			break;
+		case 't':
+			iommu_type = optarg;
+			break;
+		case 'v':
+			nr_vcpus = atoi_positive("Number of vCPUS", optarg);
+			TEST_ASSERT(nr_vcpus <= KVM_MAX_VCPUS,
+				    "KVM selftests support at most %u vCPUs", KVM_MAX_VCPUS);
+			break;
+		case 'x':
+			x2apic = false;
+			break;
+		case 'h':
+		default:
+			help(argv[0]);
+		}
+	}
+
+	TEST_REQUIRE(kvm_arch_has_default_irqchip());
+
+	vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus);
+	vm_enable_cap(vm, KVM_CAP_X2APIC_API, KVM_X2APIC_API_USE_32BIT_IDS |
+					      KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK);
+
+	vm_install_exception_handler(vm, vector, guest_irq_handler);
+	vm_install_exception_handler(vm, NMI_VECTOR, guest_nmi_handler);
+
+	if (!x2apic) {
+		TEST_ASSERT(nr_vcpus < 256, "xAPIC can only target IDs [0-254] (255 vCPUs)");
+		virt_pg_map(vm, APIC_DEFAULT_GPA, APIC_DEFAULT_GPA);
+	}
+
+	if (device_bdf) {
+		if (!iommu_type)
+			iommu_type = probe_iommu_type();
+		iommu = iommu_init(iommu_type);
+		device = vfio_pci_device_init(device_bdf, iommu);
+		msix = vfio_setup_msi(device);
+		irq = vfio_msix_to_host_irq(device_bdf, msix);
+		eventfd = device->msi_eventfds[msix];
+		printf("Using device %s MSI-X[%d] (IRQ-%u)\n", device_bdf, msix,
+		       irq);
+	} else {
+		TEST_ASSERT(!irq_affinity,
+			    "Setting IRQ affinity (-a) requires a backing device (-d)");
+
+		eventfd = kvm_new_eventfd();
+		irq = -1;
+	}
+
+	pr_info("Injecting interrupts for GSI %d (guest vector 0x%x) %d times\n",
+		gsi, vector, nr_irqs);
+
+	kvm_assign_irqfd(vm, gsi, eventfd);
+
+	sync_global_to_guest(vm, x2apic);
+
+	if (migrate_vcpus)
+		kvm_sched_getaffinity(0, sizeof(available_cpus), &available_cpus);
+
+	for (i = 0; i < nr_vcpus; i++)
+		pthread_create(&vcpu_threads[i], NULL, vcpu_thread_main, vcpus[i]);
+
+	for (i = 0; i < nr_vcpus; i++) {
+		struct kvm_vcpu *vcpu = vcpus[i];
+
+		while (!SYNC_FROM_GUEST_AND_READ(vm, guest_ready_for_irqs[vcpu->id]))
+			continue;
+	}
+
+	irq_cpu = -1;
+
+	for (i = 0; i < nr_irqs; i++) {
+		const bool do_set_empty_routing = set_empty_routing && (i & BIT(3));
+		const bool do_use_nmi = use_nmi && (i & BIT(2));
+		struct kvm_vcpu *vcpu = vcpus[i % nr_vcpus];
+		struct timespec start;
+
+		if (do_set_empty_routing)
+			kvm_set_empty_gsi_routing(vm);
+
+		kvm_route_msi(vm, gsi, vcpu, vector, do_use_nmi);
+
+		if (irq_affinity) {
+			irq_cpu = kvm_random_u64(&kvm_rng) % get_nprocs();
+			proc_irq_set_smp_affinity(irq, irq_cpu);
+		}
+
+		if (migrate_vcpus)
+			pin_task_to_random_cpu(vcpu_threads[i % nr_vcpus], &available_cpus);
+
+		for (j = 0; j < nr_vcpus; j++) {
+			TEST_ASSERT(!GUEST_RECEIVED_IRQ(vcpus[j]),
+				    "IRQ flag for vCPU %d not clear prior to test",
+				    vcpus[j]->id);
+			TEST_ASSERT(!GUEST_RECEIVED_NMI(vcpus[j]),
+				    "NMI flag for vCPU %d not clear prior to test",
+				    vcpus[j]->id);
+		}
+
+		trigger_interrupt(device, eventfd);
+
+		clock_gettime(CLOCK_MONOTONIC, &start);
+		while (!GUEST_RECEIVED_INTERRUPT(vcpu, do_use_nmi) &&
+		       timespec_to_ns(timespec_elapsed(start)) <= timeout_ns)
+			cpu_relax();
+
+		TEST_ASSERT(GUEST_RECEIVED_INTERRUPT(vcpu, do_use_nmi),
+			    "vCPU %d timed out waiting for %s (vector 0x%x) from GSI %d (via CPU %d)\n",
+			    vcpu->id, do_use_nmi ? "NMI" : "IRQ",
+			    do_use_nmi ? NMI_VECTOR : vector, gsi, irq_cpu);
+
+		if (do_use_nmi)
+			WRITE_AND_SYNC_TO_GUEST(vm, guest_received_nmi[vcpu->id], false);
+		else
+			WRITE_AND_SYNC_TO_GUEST(vm, guest_received_irq[vcpu->id], false);
+	}
+
+	WRITE_AND_SYNC_TO_GUEST(vm, done, true);
+
+	for (i = 0; i < nr_vcpus; i++)
+		pthread_join(vcpu_threads[i], NULL);
+
+	return 0;
+}
diff --git a/tools/testing/selftests/kvm/lib/assert.c b/tools/testing/selftests/kvm/lib/assert.c
index 8be0d09ecf0f9..1d72dcdfce3b6 100644
--- a/tools/testing/selftests/kvm/lib/assert.c
+++ b/tools/testing/selftests/kvm/lib/assert.c
@@ -10,6 +10,7 @@
 #include <sys/syscall.h>
 
 #include "kselftest.h"
+#include "kvm_syscalls.h"
 
 #ifdef __GLIBC__
 #include <execinfo.h>
@@ -64,11 +65,6 @@ static void test_dump_stack(void)
 static void test_dump_stack(void) {}
 #endif
 
-static pid_t _gettid(void)
-{
-	return syscall(SYS_gettid);
-}
-
 void __attribute__((noinline))
 test_assert(bool exp, const char *exp_str,
 	const char *file, unsigned int line, const char *fmt, ...)
@@ -81,7 +77,7 @@ test_assert(bool exp, const char *exp_str,
 		fprintf(stderr, "==== Test Assertion Failure ====\n"
 			"  %s:%u: %s\n"
 			"  pid=%d tid=%d errno=%d - %s\n",
-			file, line, exp_str, getpid(), _gettid(),
+			file, line, exp_str, getpid(), kvm_gettid(),
 			errno, strerror(errno));
 		test_dump_stack();
 		if (fmt) {
diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c
index 195f3fdae1e39..3794575d2ca0c 100644
--- a/tools/testing/selftests/kvm/lib/kvm_util.c
+++ b/tools/testing/selftests/kvm/lib/kvm_util.c
@@ -15,14 +15,22 @@
 #include <sys/resource.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <time.h>
 #include <unistd.h>
 #include <linux/kernel.h>
 
 #define KVM_UTIL_MIN_PFN	2
 
-u32 guest_random_seed;
-struct guest_random_state guest_rng;
-static u32 last_guest_seed;
+u32 kvm_random_seed;
+struct kvm_random_state kvm_rng;
+static u32 last_kvm_seed;
+
+static void kvm_seed_rng(u32 seed)
+{
+	kvm_random_seed = last_kvm_seed = seed;
+	pr_info("Random seed: 0x%x\n", kvm_random_seed);
+	kvm_rng = new_kvm_random_state(kvm_random_seed);
+}
 
 static size_t vcpu_mmap_sz(void);
 
@@ -515,12 +523,10 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus,
 	slot0 = memslot2region(vm, 0);
 	ucall_init(vm, slot0->region.guest_phys_addr + slot0->region.memory_size);
 
-	if (guest_random_seed != last_guest_seed) {
-		pr_info("Random seed: 0x%x\n", guest_random_seed);
-		last_guest_seed = guest_random_seed;
-	}
-	guest_rng = new_guest_random_state(guest_random_seed);
-	sync_global_to_guest(vm, guest_rng);
+	if (kvm_random_seed != last_kvm_seed)
+		kvm_seed_rng(kvm_random_seed);
+
+	sync_global_to_guest(vm, kvm_rng);
 
 	kvm_arch_vm_post_create(vm, nr_runnable_vcpus);
 
@@ -662,19 +668,39 @@ void kvm_print_vcpu_pinning_help(void)
 	       "     (default: no pinning)\n", name, name);
 }
 
+void pin_task_to_random_cpu(pthread_t task, cpu_set_t *possible_cpus)
+{
+	int target_idx;
+	int nr_cpus;
+	int cpu;
+
+	nr_cpus = CPU_COUNT(possible_cpus);
+	TEST_ASSERT(nr_cpus > 0, "No CPUs available in possible_cpus");
+
+	target_idx = kvm_random_u64(&kvm_rng) % nr_cpus;
+
+	for (cpu = 0; cpu < CPU_SETSIZE; cpu++) {
+		if (CPU_ISSET(cpu, possible_cpus) && target_idx-- == 0) {
+			pin_task_to_cpu(task, cpu);
+			return;
+		}
+	}
+
+	TEST_FAIL("Failed to find random CPU in possible_cpus");
+}
+
 void kvm_parse_vcpu_pinning(const char *pcpus_string, u32 vcpu_to_pcpu[],
 			    int nr_vcpus)
 {
 	cpu_set_t allowed_mask;
 	char *cpu, *cpu_list;
 	char delim[2] = ",";
-	int i, r;
+	int i;
 
 	cpu_list = strdup(pcpus_string);
 	TEST_ASSERT(cpu_list, "strdup() allocation failed.");
 
-	r = sched_getaffinity(0, sizeof(allowed_mask), &allowed_mask);
-	TEST_ASSERT(!r, "sched_getaffinity() failed");
+	kvm_sched_getaffinity(0, sizeof(allowed_mask), &allowed_mask);
 
 	cpu = strtok(cpu_list, delim);
 
@@ -2279,8 +2305,8 @@ void __attribute((constructor)) kvm_selftest_init(void)
 	sigaction(SIGILL, &sig_sa, NULL);
 	sigaction(SIGFPE, &sig_sa, NULL);
 
-	guest_random_seed = last_guest_seed = random();
-	pr_info("Random seed: 0x%x\n", guest_random_seed);
+	srandom(time(0));
+	kvm_seed_rng(random());
 
 	kvm_selftest_arch_init();
 }
diff --git a/tools/testing/selftests/kvm/lib/memstress.c b/tools/testing/selftests/kvm/lib/memstress.c
index 6dcd15910a060..3599b75d97c91 100644
--- a/tools/testing/selftests/kvm/lib/memstress.c
+++ b/tools/testing/selftests/kvm/lib/memstress.c
@@ -48,14 +48,14 @@ void memstress_guest_code(u32 vcpu_idx)
 {
 	struct memstress_args *args = &memstress_args;
 	struct memstress_vcpu_args *vcpu_args = &args->vcpu_args[vcpu_idx];
-	struct guest_random_state rand_state;
+	struct kvm_random_state rand_state;
 	gva_t gva;
 	u64 pages;
 	u64 addr;
 	u64 page;
 	int i;
 
-	rand_state = new_guest_random_state(guest_random_seed + vcpu_idx);
+	rand_state = new_kvm_random_state(kvm_random_seed + vcpu_idx);
 
 	gva = vcpu_args->gva;
 	pages = vcpu_args->pages;
@@ -69,13 +69,13 @@ void memstress_guest_code(u32 vcpu_idx)
 
 		for (i = 0; i < pages; i++) {
 			if (args->random_access)
-				page = guest_random_u32(&rand_state) % pages;
+				page = kvm_random_u32(&rand_state) % pages;
 			else
 				page = i;
 
 			addr = gva + (page * args->guest_page_size);
 
-			if (__guest_random_bool(&rand_state, args->write_percent))
+			if (__kvm_random_bool(&rand_state, args->write_percent))
 				*(u64 *)addr = 0x0123456789ABCDEF;
 			else
 				READ_ONCE(*(u64 *)addr);
diff --git a/tools/testing/selftests/kvm/lib/proc_util.c b/tools/testing/selftests/kvm/lib/proc_util.c
new file mode 100644
index 0000000000000..3960b3841d637
--- /dev/null
+++ b/tools/testing/selftests/kvm/lib/proc_util.c
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: GPL-2.0
+#include "kvm_util.h"
+#include "test_util.h"
+#include "proc_util.h"
+
+static FILE *open_proc_interrupts(void)
+{
+	FILE *fp;
+
+	fp = fopen("/proc/interrupts", "r");
+	TEST_ASSERT(fp, "fopen(/proc/interrupts) failed");
+
+	return fp;
+}
+
+unsigned int vfio_msix_to_host_irq(const char *device_bdf, int msix)
+{
+	char search_string[64];
+	char line[4096];
+	int irq = -1;
+	FILE *fp;
+
+	fp = open_proc_interrupts();
+
+	snprintf(search_string, sizeof(search_string), "vfio-msix[%d]", msix);
+
+	while (fgets(line, sizeof(line), fp)) {
+		if (strstr(line, device_bdf) && strstr(line, search_string)) {
+			TEST_ASSERT_EQ(1, sscanf(line, "%d:", &irq));
+			break;
+		}
+	}
+
+	fclose(fp);
+
+	TEST_ASSERT(irq != -1, "Failed to locate IRQ for %s %s", device_bdf,
+		    search_string);
+	return (unsigned int)irq;
+}
+
+void proc_irq_set_smp_affinity(unsigned int irq, int cpu)
+{
+	char path[PATH_MAX];
+	int r, fd;
+
+	snprintf(path, sizeof(path), "/proc/irq/%u/smp_affinity_list", irq);
+	fd = open(path, O_RDWR);
+	TEST_ASSERT(fd >= 0, "Failed to open %s", path);
+
+	r = dprintf(fd, "%d\n", cpu);
+	TEST_ASSERT(r > 0, "Failed to affinitize IRQ-%u to CPU %d", irq, cpu);
+
+	kvm_close(fd);
+}
diff --git a/tools/testing/selftests/kvm/lib/test_util.c b/tools/testing/selftests/kvm/lib/test_util.c
index bab1bd2b775b6..6b00ab11f3c02 100644
--- a/tools/testing/selftests/kvm/lib/test_util.c
+++ b/tools/testing/selftests/kvm/lib/test_util.c
@@ -17,6 +17,7 @@
 #include "linux/kernel.h"
 
 #include "test_util.h"
+#include "kvm_syscalls.h"
 
 sigjmp_buf expect_sigbus_jmpbuf;
 
@@ -30,18 +31,36 @@ void __attribute__((used)) expect_sigbus_handler(int signum)
  * Park-Miller LCG using standard constants.
  */
 
-struct guest_random_state new_guest_random_state(u32 seed)
+struct kvm_random_state new_kvm_random_state(u32 seed)
 {
-	struct guest_random_state s = {.seed = seed};
+	struct kvm_random_state s = {.seed = seed};
 	return s;
 }
 
-u32 guest_random_u32(struct guest_random_state *state)
+u32 kvm_random_u32(struct kvm_random_state *state)
 {
 	state->seed = (u64)state->seed * 48271 % ((u32)(1 << 31) - 1);
 	return state->seed;
 }
 
+/* Returns a random u64 in the inclusive range [min, max] */
+u64 kvm_random_u64_in_range(struct kvm_random_state *state, u64 min,
+			    u64 max)
+{
+	u64 value;
+	u64 range;
+
+	TEST_ASSERT(min <= max, "PEBKAC, min = 0x%lx, max = 0x%lx", min, max);
+
+	value = kvm_random_u64(state);
+
+	range = max - min;
+	if (range == ULLONG_MAX)
+		return value;
+
+	return min + (value % (range + 1));
+}
+
 /*
  * Parses "[0-9]+[kmgt]?".
  */
@@ -377,7 +396,7 @@ long get_run_delay(void)
 	long val[2];
 	FILE *fp;
 
-	sprintf(path, "/proc/%ld/schedstat", syscall(SYS_gettid));
+	sprintf(path, "/proc/%ld/schedstat", (long)kvm_gettid());
 	fp = fopen(path, "r");
 	/* Return MIN_RUN_DELAY_NS upon failure just to be safe */
 	if (fscanf(fp, "%ld %ld ", &val[0], &val[1]) < 2)
diff --git a/tools/testing/selftests/kvm/lib/x86/memstress.c b/tools/testing/selftests/kvm/lib/x86/memstress.c
index 61cf952cd2dc2..e19e8b5a09c5a 100644
--- a/tools/testing/selftests/kvm/lib/x86/memstress.c
+++ b/tools/testing/selftests/kvm/lib/x86/memstress.c
@@ -30,21 +30,15 @@ __asm__(
 "	ud2;"
 );
 
-#define L2_GUEST_STACK_SIZE 64
-
 static void l1_vmx_code(struct vmx_pages *vmx, u64 vcpu_id)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
-	unsigned long *rsp;
-
 	GUEST_ASSERT(vmx->vmcs_gpa);
 	GUEST_ASSERT(prepare_for_vmx_operation(vmx));
 	GUEST_ASSERT(load_vmcs(vmx));
 	GUEST_ASSERT(ept_1g_pages_supported());
 
-	rsp = &l2_guest_stack[L2_GUEST_STACK_SIZE - 1];
-	*rsp = vcpu_id;
-	prepare_vmcs(vmx, memstress_l2_guest_entry, rsp);
+	*(u64 *)vmx->stack = vcpu_id;
+	prepare_vmcs(vmx, memstress_l2_guest_entry);
 
 	GUEST_ASSERT(!vmlaunch());
 	GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_VMCALL);
@@ -53,13 +47,8 @@ static void l1_vmx_code(struct vmx_pages *vmx, u64 vcpu_id)
 
 static void l1_svm_code(struct svm_test_data *svm, u64 vcpu_id)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
-	unsigned long *rsp;
-
-
-	rsp = &l2_guest_stack[L2_GUEST_STACK_SIZE - 1];
-	*rsp = vcpu_id;
-	generic_svm_setup(svm, memstress_l2_guest_entry, rsp);
+	*(u64 *)svm->stack = vcpu_id;
+	generic_svm_setup(svm, memstress_l2_guest_entry);
 
 	run_guest(svm->vmcb, svm->vmcb_gpa);
 	GUEST_ASSERT_EQ(svm->vmcb->control.exit_code, SVM_EXIT_VMMCALL);
diff --git a/tools/testing/selftests/kvm/lib/x86/processor.c b/tools/testing/selftests/kvm/lib/x86/processor.c
index 4ca48de7a926d..ef56dcefe0119 100644
--- a/tools/testing/selftests/kvm/lib/x86/processor.c
+++ b/tools/testing/selftests/kvm/lib/x86/processor.c
@@ -778,6 +778,30 @@ void assert_on_unhandled_exception(struct kvm_vcpu *vcpu)
 		REPORT_GUEST_ASSERT(uc);
 }
 
+gva_t vm_alloc_stack(struct kvm_vm *vm, int nr_pages)
+{
+	int size = nr_pages * getpagesize();
+	gva_t stack_gva;
+
+	stack_gva = __vm_alloc(vm, size, DEFAULT_GUEST_STACK_VADDR_MIN, MEM_REGION_DATA);
+	stack_gva += size;
+
+	/*
+	 * Align stack to match calling sequence requirements in section "The
+	 * Stack Frame" of the System V ABI AMD64 Architecture Processor
+	 * Supplement, which requires the value (%rsp + 8) to be a multiple of
+	 * 16 when control is transferred to the function entry point.
+	 *
+	 * If this code is ever used to launch a vCPU with 32-bit entry point it
+	 * may need to subtract 4 bytes instead of 8 bytes.
+	 */
+	TEST_ASSERT(IS_ALIGNED(stack_gva, PAGE_SIZE),
+		    "__vm_alloc() did not provide a page-aligned address");
+	stack_gva -= 8;
+
+	return stack_gva;
+}
+
 void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus)
 {
 	int r;
@@ -820,27 +844,8 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id)
 {
 	struct kvm_mp_state mp_state;
 	struct kvm_regs regs;
-	gva_t stack_gva;
 	struct kvm_vcpu *vcpu;
 
-	stack_gva = __vm_alloc(vm, DEFAULT_STACK_PGS * getpagesize(),
-			       DEFAULT_GUEST_STACK_VADDR_MIN, MEM_REGION_DATA);
-
-	stack_gva += DEFAULT_STACK_PGS * getpagesize();
-
-	/*
-	 * Align stack to match calling sequence requirements in section "The
-	 * Stack Frame" of the System V ABI AMD64 Architecture Processor
-	 * Supplement, which requires the value (%rsp + 8) to be a multiple of
-	 * 16 when control is transferred to the function entry point.
-	 *
-	 * If this code is ever used to launch a vCPU with 32-bit entry point it
-	 * may need to subtract 4 bytes instead of 8 bytes.
-	 */
-	TEST_ASSERT(IS_ALIGNED(stack_gva, PAGE_SIZE),
-		    "__vm_alloc() did not provide a page-aligned address");
-	stack_gva -= 8;
-
 	vcpu = __vm_vcpu_add(vm, vcpu_id);
 	vcpu_init_cpuid(vcpu, kvm_get_supported_cpuid());
 	vcpu_init_sregs(vm, vcpu);
@@ -849,7 +854,7 @@ struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id)
 	/* Setup guest general purpose registers */
 	vcpu_regs_get(vcpu, &regs);
 	regs.rflags = regs.rflags | X86_EFLAGS_FIXED;
-	regs.rsp = stack_gva;
+	regs.rsp = vm_alloc_stack(vm, DEFAULT_STACK_PGS);
 	vcpu_regs_set(vcpu, &regs);
 
 	/* Setup the MP state */
diff --git a/tools/testing/selftests/kvm/lib/x86/svm.c b/tools/testing/selftests/kvm/lib/x86/svm.c
index 3b01605ab016c..1445b890986fd 100644
--- a/tools/testing/selftests/kvm/lib/x86/svm.c
+++ b/tools/testing/selftests/kvm/lib/x86/svm.c
@@ -46,6 +46,8 @@ vcpu_alloc_svm(struct kvm_vm *vm, gva_t *p_svm_gva)
 	svm->msr_gpa = addr_gva2gpa(vm, (uintptr_t)svm->msr);
 	memset(svm->msr_hva, 0, getpagesize());
 
+	svm->stack = (void *)vm_alloc_stack(vm, 1);
+
 	if (vm->stage2_mmu.pgd_created)
 		svm->ncr3_gpa = vm->stage2_mmu.pgd;
 
@@ -81,7 +83,7 @@ void vm_enable_npt(struct kvm_vm *vm)
 	tdp_mmu_init(vm, vm->mmu.pgtable_levels, &pte_masks);
 }
 
-void generic_svm_setup(struct svm_test_data *svm, void *guest_rip, void *guest_rsp)
+void generic_svm_setup(struct svm_test_data *svm, void *guest_rip)
 {
 	struct vmcb *vmcb = svm->vmcb;
 	u64 vmcb_gpa = svm->vmcb_gpa;
@@ -122,7 +124,7 @@ void generic_svm_setup(struct svm_test_data *svm, void *guest_rip, void *guest_r
 	ctrl->msrpm_base_pa = svm->msr_gpa;
 
 	vmcb->save.rip = (u64)guest_rip;
-	vmcb->save.rsp = (u64)guest_rsp;
+	vmcb->save.rsp = (u64)svm->stack;
 	guest_regs.rdi = (u64)svm;
 
 	if (svm->ncr3_gpa) {
diff --git a/tools/testing/selftests/kvm/lib/x86/vmx.c b/tools/testing/selftests/kvm/lib/x86/vmx.c
index 7c10ba6e6fb40..cd09c9de4485a 100644
--- a/tools/testing/selftests/kvm/lib/x86/vmx.c
+++ b/tools/testing/selftests/kvm/lib/x86/vmx.c
@@ -116,6 +116,8 @@ vcpu_alloc_vmx(struct kvm_vm *vm, gva_t *p_vmx_gva)
 	vmx->vmwrite_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->vmwrite);
 	memset(vmx->vmwrite_hva, 0, getpagesize());
 
+	vmx->stack = (void *)vm_alloc_stack(vm, 1);
+
 	if (vm->stage2_mmu.pgd_created)
 		vmx->eptp_gpa = vm->stage2_mmu.pgd;
 
@@ -366,11 +368,11 @@ static inline void init_vmcs_guest_state(void *rip, void *rsp)
 	vmwrite(GUEST_SYSENTER_EIP, vmreadz(HOST_IA32_SYSENTER_EIP));
 }
 
-void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip, void *guest_rsp)
+void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip)
 {
 	init_vmcs_control_fields(vmx);
 	init_vmcs_host_state();
-	init_vmcs_guest_state(guest_rip, guest_rsp);
+	init_vmcs_guest_state(guest_rip, vmx->stack);
 }
 
 bool kvm_cpu_has_ept(void)
diff --git a/tools/testing/selftests/kvm/mmu_stress_test.c b/tools/testing/selftests/kvm/mmu_stress_test.c
index 54d281419d310..3d5f33a63b2b4 100644
--- a/tools/testing/selftests/kvm/mmu_stress_test.c
+++ b/tools/testing/selftests/kvm/mmu_stress_test.c
@@ -155,10 +155,8 @@ static void *vcpu_worker(void *data)
 		    "Expected EFAULT on write to RO memory, got r = %d, errno = %d", r, errno);
 
 	atomic_inc(&nr_ro_faults);
-	if (atomic_read(&nr_ro_faults) == nr_vcpus) {
-		WRITE_ONCE(all_vcpus_hit_ro_fault, true);
-		sync_global_to_guest(vm, all_vcpus_hit_ro_fault);
-	}
+	if (atomic_read(&nr_ro_faults) == nr_vcpus)
+		WRITE_AND_SYNC_TO_GUEST(vm, all_vcpus_hit_ro_fault, true);
 
 #if defined(__x86_64__) || defined(__aarch64__)
 	/*
@@ -257,11 +255,7 @@ static void rendezvous_with_vcpus(struct timespec *time, const char *name)
 static void calc_default_nr_vcpus(void)
 {
 	cpu_set_t possible_mask;
-	int r;
-
-	r = sched_getaffinity(0, sizeof(possible_mask), &possible_mask);
-	TEST_ASSERT(!r, "sched_getaffinity failed, errno = %d (%s)",
-		    errno, strerror(errno));
+	kvm_sched_getaffinity(0, sizeof(possible_mask), &possible_mask);
 
 	nr_vcpus = CPU_COUNT(&possible_mask);
 	TEST_ASSERT(nr_vcpus > 0, "Uh, no CPUs?");
@@ -383,8 +377,7 @@ int main(int argc, char *argv[])
 	rendezvous_with_vcpus(&time_run2, "run 2");
 
 	mprotect(mem, slot_size, PROT_READ);
-	mprotect_ro_done = true;
-	sync_global_to_guest(vm, mprotect_ro_done);
+	WRITE_AND_SYNC_TO_GUEST(vm, mprotect_ro_done, true);
 
 	rendezvous_with_vcpus(&time_ro, "mprotect RO");
 	mprotect(mem, slot_size, PROT_READ | PROT_WRITE);
diff --git a/tools/testing/selftests/kvm/rseq_test.c b/tools/testing/selftests/kvm/rseq_test.c
index f80ad6b47d16b..557e393c223b9 100644
--- a/tools/testing/selftests/kvm/rseq_test.c
+++ b/tools/testing/selftests/kvm/rseq_test.c
@@ -226,9 +226,7 @@ int main(int argc, char *argv[])
 		}
 	}
 
-	r = sched_getaffinity(0, sizeof(possible_mask), &possible_mask);
-	TEST_ASSERT(!r, "sched_getaffinity failed, errno = %d (%s)", errno,
-		    strerror(errno));
+	kvm_sched_getaffinity(0, sizeof(possible_mask), &possible_mask);
 
 	calc_min_max_cpu();
 
@@ -244,7 +242,7 @@ int main(int argc, char *argv[])
 	vm = vm_create_with_one_vcpu(&vcpu, guest_code);
 
 	pthread_create(&migration_thread, NULL, migration_worker,
-		       (void *)(unsigned long)syscall(SYS_gettid));
+		       (void *)(unsigned long)kvm_gettid());
 
 	if (latency >= 0) {
 		/*
diff --git a/tools/testing/selftests/kvm/steal_time.c b/tools/testing/selftests/kvm/steal_time.c
index 76fcdd1fd3cb4..0b88c743ee7d5 100644
--- a/tools/testing/selftests/kvm/steal_time.c
+++ b/tools/testing/selftests/kvm/steal_time.c
@@ -70,8 +70,8 @@ static bool is_steal_time_supported(struct kvm_vcpu *vcpu)
 static void steal_time_init(struct kvm_vcpu *vcpu, u32 i)
 {
 	/* ST_GPA_BASE is identity mapped */
-	st_gva[i] = (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE);
-	sync_global_to_guest(vcpu->vm, st_gva[i]);
+	WRITE_AND_SYNC_TO_GUEST(vcpu->vm, st_gva[i],
+				(void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE));
 
 	vcpu_set_msr(vcpu, MSR_KVM_STEAL_TIME, (ulong)st_gva[i] | KVM_MSR_ENABLED);
 }
@@ -187,8 +187,7 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i)
 	};
 
 	/* ST_GPA_BASE is identity mapped */
-	st_gva[i] = (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE);
-	sync_global_to_guest(vm, st_gva[i]);
+	WRITE_AND_SYNC_TO_GUEST(vm, st_gva[i], (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE));
 
 	st_ipa = (ulong)st_gva[i];
 	vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev);
@@ -310,10 +309,8 @@ static bool is_steal_time_supported(struct kvm_vcpu *vcpu)
 static void steal_time_init(struct kvm_vcpu *vcpu, u32 i)
 {
 	/* ST_GPA_BASE is identity mapped */
-	st_gva[i] = (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE);
-	st_gpa[i] = addr_gva2gpa(vcpu->vm, (gva_t)st_gva[i]);
-	sync_global_to_guest(vcpu->vm, st_gva[i]);
-	sync_global_to_guest(vcpu->vm, st_gpa[i]);
+	WRITE_AND_SYNC_TO_GUEST(vcpu->vm, st_gva[i], (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE));
+	WRITE_AND_SYNC_TO_GUEST(vcpu->vm, st_gpa[i], addr_gva2gpa(vcpu->vm, (gva_t)st_gva[i]));
 }
 
 static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx)
@@ -442,8 +439,7 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i)
 	};
 
 	/* ST_GPA_BASE is identity mapped */
-	st_gva[i] = (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE);
-	sync_global_to_guest(vm, st_gva[i]);
+	WRITE_AND_SYNC_TO_GUEST(vm, st_gva[i], (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE));
 
 	err = __vcpu_ioctl(vcpu, KVM_HAS_DEVICE_ATTR, &attr);
 	TEST_ASSERT(err == 0, "No PV stealtime Feature");
@@ -508,23 +504,18 @@ int main(int ac, char **av)
 {
 	struct kvm_vcpu *vcpus[NR_VCPUS];
 	struct kvm_vm *vm;
-	pthread_attr_t attr;
 	pthread_t thread;
 	cpu_set_t cpuset;
 	unsigned int gpages;
 	long stolen_time;
 	long run_delay;
 	bool verbose;
-	int i;
+	int i, cpu;
 
 	verbose = ac > 1 && (!strncmp(av[1], "-v", 3) || !strncmp(av[1], "--verbose", 10));
 
 	/* Set CPU affinity so we can force preemption of the VCPU */
-	CPU_ZERO(&cpuset);
-	CPU_SET(0, &cpuset);
-	pthread_attr_init(&attr);
-	pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset);
-	pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
+	cpu = pin_self_to_any_cpu();
 
 	/* Create a VM and an identity mapped memslot for the steal time structure */
 	vm = vm_create_with_vcpus(NR_VCPUS, guest_code, vcpus);
@@ -549,8 +540,7 @@ int main(int ac, char **av)
 
 		/* Second VCPU run, expect guest stolen time to be <= run_delay */
 		run_vcpu(vcpus[i]);
-		sync_global_from_guest(vm, guest_stolen_time[i]);
-		stolen_time = guest_stolen_time[i];
+		stolen_time = SYNC_FROM_GUEST_AND_READ(vm, guest_stolen_time[i]);
 		run_delay = get_run_delay();
 		TEST_ASSERT(stolen_time <= run_delay,
 			    "Expected stolen time <= %ld, got %ld",
@@ -558,7 +548,11 @@ int main(int ac, char **av)
 
 		/* Steal time from the VCPU. The steal time thread has the same CPU affinity as the VCPUs. */
 		run_delay = get_run_delay();
-		pthread_create(&thread, &attr, do_steal_time, NULL);
+		pthread_create(&thread, NULL, do_steal_time, NULL);
+		pthread_getaffinity_np(thread, sizeof(cpuset), &cpuset);
+		TEST_ASSERT(CPU_COUNT(&cpuset) == 1 && CPU_ISSET(cpu, &cpuset),
+			    "Worker failed to inherit parent's CPU affinity");
+
 		do
 			sched_yield();
 		while (get_run_delay() - run_delay < MIN_RUN_DELAY_NS);
@@ -570,8 +564,7 @@ int main(int ac, char **av)
 
 		/* Run VCPU again to confirm stolen time is consistent with run_delay */
 		run_vcpu(vcpus[i]);
-		sync_global_from_guest(vm, guest_stolen_time[i]);
-		stolen_time = guest_stolen_time[i] - stolen_time;
+		stolen_time = SYNC_FROM_GUEST_AND_READ(vm, guest_stolen_time[i]) - stolen_time;
 		TEST_ASSERT(stolen_time >= run_delay,
 			    "Expected stolen time >= %ld, got %ld",
 			    run_delay, stolen_time);
diff --git a/tools/testing/selftests/kvm/x86/aperfmperf_test.c b/tools/testing/selftests/kvm/x86/aperfmperf_test.c
index c91660103137b..845cb685f1743 100644
--- a/tools/testing/selftests/kvm/x86/aperfmperf_test.c
+++ b/tools/testing/selftests/kvm/x86/aperfmperf_test.c
@@ -54,8 +54,6 @@ static void guest_read_aperf_mperf(void)
 		GUEST_SYNC2(rdmsr(MSR_IA32_APERF), rdmsr(MSR_IA32_MPERF));
 }
 
-#define L2_GUEST_STACK_SIZE	64
-
 static void l2_guest_code(void)
 {
 	guest_read_aperf_mperf();
@@ -64,21 +62,18 @@ static void l2_guest_code(void)
 
 static void l1_svm_code(struct svm_test_data *svm)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	struct vmcb *vmcb = svm->vmcb;
 
-	generic_svm_setup(svm, l2_guest_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_guest_code);
 	run_guest(vmcb, svm->vmcb_gpa);
 }
 
 static void l1_vmx_code(struct vmx_pages *vmx)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
-
 	GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true);
 	GUEST_ASSERT_EQ(load_vmcs(vmx), true);
 
-	prepare_vmcs(vmx, NULL, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	prepare_vmcs(vmx, NULL);
 
 	/*
 	 * Enable MSR bitmaps (the bitmap itself is allocated, zeroed, and set
diff --git a/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c b/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c
index 5b3aef109cfc5..77ce87c41a868 100644
--- a/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c
+++ b/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c
@@ -52,8 +52,6 @@ static void l2_guest_code(void)
 static void guest_code(struct vmx_pages *vmx_pages,
 		       struct hyperv_test_pages *hv_pages)
 {
-#define L2_GUEST_STACK_SIZE 64
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 
 	/* Set up Hyper-V enlightenments and eVMCS */
 	wrmsr(HV_X64_MSR_GUEST_OS_ID, HYPERV_LINUX_OS_ID);
@@ -62,8 +60,7 @@ static void guest_code(struct vmx_pages *vmx_pages,
 
 	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
 	GUEST_ASSERT(load_evmcs(hv_pages));
-	prepare_vmcs(vmx_pages, l2_guest_code,
-		     &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	prepare_vmcs(vmx_pages, l2_guest_code);
 
 	GUEST_ASSERT(!vmlaunch());
 
diff --git a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c
index c7fa114aee20f..1bda2cd3f7396 100644
--- a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c
+++ b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c
@@ -78,9 +78,6 @@ void l2_guest_code(void)
 void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages,
 		gpa_t hv_hcall_page_gpa)
 {
-#define L2_GUEST_STACK_SIZE 64
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
-
 	wrmsr(HV_X64_MSR_GUEST_OS_ID, HYPERV_LINUX_OS_ID);
 	wrmsr(HV_X64_MSR_HYPERCALL, hv_hcall_page_gpa);
 
@@ -100,8 +97,7 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages,
 	GUEST_SYNC(4);
 	GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa);
 
-	prepare_vmcs(vmx_pages, l2_guest_code,
-		     &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	prepare_vmcs(vmx_pages, l2_guest_code);
 
 	GUEST_SYNC(5);
 	GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa);
diff --git a/tools/testing/selftests/kvm/x86/hyperv_svm_test.c b/tools/testing/selftests/kvm/x86/hyperv_svm_test.c
index 7a62f6a9d606d..1f74b0fa9b835 100644
--- a/tools/testing/selftests/kvm/x86/hyperv_svm_test.c
+++ b/tools/testing/selftests/kvm/x86/hyperv_svm_test.c
@@ -18,8 +18,6 @@
 #include "svm_util.h"
 #include "hyperv.h"
 
-#define L2_GUEST_STACK_SIZE 256
-
 /* Exit to L1 from L2 with RDMSR instruction */
 static inline void rdmsr_from_l2(u32 msr)
 {
@@ -69,7 +67,6 @@ static void __attribute__((__flatten__)) guest_code(struct svm_test_data *svm,
 						    struct hyperv_test_pages *hv_pages,
 						    gpa_t pgs_gpa)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	struct vmcb *vmcb = svm->vmcb;
 	struct hv_vmcb_enlightenments *hve = &vmcb->control.hv_enlightenments;
 
@@ -81,8 +78,7 @@ static void __attribute__((__flatten__)) guest_code(struct svm_test_data *svm,
 
 	GUEST_ASSERT(svm->vmcb_gpa);
 	/* Prepare for L2 execution. */
-	generic_svm_setup(svm, l2_guest_code,
-			  &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_guest_code);
 
 	/* L2 TLB flush setup */
 	hve->partition_assist_page = hv_pages->partition_assist_gpa;
diff --git a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c
index 52014a3210c88..25a182be00a97 100644
--- a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c
+++ b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c
@@ -26,8 +26,6 @@ static void guest_generate_buslocks(void)
 		atomic_inc(val);
 }
 
-#define L2_GUEST_STACK_SIZE	64
-
 static void l2_guest_code(void)
 {
 	guest_generate_buslocks();
@@ -36,21 +34,18 @@ static void l2_guest_code(void)
 
 static void l1_svm_code(struct svm_test_data *svm)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	struct vmcb *vmcb = svm->vmcb;
 
-	generic_svm_setup(svm, l2_guest_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_guest_code);
 	run_guest(vmcb, svm->vmcb_gpa);
 }
 
 static void l1_vmx_code(struct vmx_pages *vmx)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
-
 	GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true);
 	GUEST_ASSERT_EQ(load_vmcs(vmx), true);
 
-	prepare_vmcs(vmx, NULL, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	prepare_vmcs(vmx, NULL);
 
 	GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code));
 	GUEST_ASSERT(!vmlaunch());
diff --git a/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c b/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c
index 761fec2934080..b974cfb347d6e 100644
--- a/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c
@@ -21,8 +21,6 @@ enum {
 	PORT_L0_EXIT = 0x2000,
 };
 
-#define L2_GUEST_STACK_SIZE 64
-
 static void l2_guest_code(void)
 {
 	/* Exit to L0 */
@@ -32,14 +30,11 @@ static void l2_guest_code(void)
 
 static void l1_vmx_code(struct vmx_pages *vmx_pages)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
-
 	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
 	GUEST_ASSERT(load_vmcs(vmx_pages));
 
 	/* Prepare the VMCS for L2 execution. */
-	prepare_vmcs(vmx_pages, l2_guest_code,
-		     &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	prepare_vmcs(vmx_pages, l2_guest_code);
 
 	GUEST_ASSERT(!vmlaunch());
 	GUEST_ASSERT(0);
@@ -47,11 +42,8 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages)
 
 static void l1_svm_code(struct svm_test_data *svm)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
-
 	/* Prepare the VMCB for L2 execution. */
-	generic_svm_setup(svm, l2_guest_code,
-			  &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_guest_code);
 
 	run_guest(svm->vmcb, svm->vmcb_gpa);
 	GUEST_ASSERT(0);
diff --git a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c
index 0e67cce835701..26b474bf13535 100644
--- a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c
@@ -40,8 +40,6 @@
 
 #define TEST_HVA(vm, idx)		addr_gpa2hva(vm, TEST_GPA(idx))
 
-#define L2_GUEST_STACK_SIZE 64
-
 /* Use the page offset bits to communicate the access+fault type. */
 #define TEST_SYNC_READ_FAULT		BIT(0)
 #define TEST_SYNC_WRITE_FAULT		BIT(1)
@@ -92,7 +90,6 @@ static void l2_guest_code_tdp_disabled(void)
 
 void l1_vmx_code(struct vmx_pages *vmx)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	void *l2_rip;
 
 	GUEST_ASSERT(vmx->vmcs_gpa);
@@ -104,7 +101,7 @@ void l1_vmx_code(struct vmx_pages *vmx)
 	else
 		l2_rip = l2_guest_code_tdp_disabled;
 
-	prepare_vmcs(vmx, l2_rip, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	prepare_vmcs(vmx, l2_rip);
 
 	GUEST_SYNC(TEST_SYNC_NO_FAULT);
 	GUEST_ASSERT(!vmlaunch());
@@ -115,7 +112,6 @@ void l1_vmx_code(struct vmx_pages *vmx)
 
 static void l1_svm_code(struct svm_test_data *svm)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	void *l2_rip;
 
 	if (svm->ncr3_gpa)
@@ -123,7 +119,7 @@ static void l1_svm_code(struct svm_test_data *svm)
 	else
 		l2_rip = l2_guest_code_tdp_disabled;
 
-	generic_svm_setup(svm, l2_rip, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_rip);
 
 	GUEST_SYNC(TEST_SYNC_NO_FAULT);
 	run_guest(svm->vmcb, svm->vmcb_gpa);
diff --git a/tools/testing/selftests/kvm/x86/nested_emulation_test.c b/tools/testing/selftests/kvm/x86/nested_emulation_test.c
index fb7dcbe53ac73..e08c6b0697e50 100644
--- a/tools/testing/selftests/kvm/x86/nested_emulation_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_emulation_test.c
@@ -57,7 +57,7 @@ static void guest_code(void *test_data)
 		struct svm_test_data *svm = test_data;
 		struct vmcb *vmcb = svm->vmcb;
 
-		generic_svm_setup(svm, NULL, NULL);
+		generic_svm_setup(svm, NULL);
 		vmcb->save.idtr.limit = 0;
 		vmcb->save.rip = (u64)l2_guest_code;
 
@@ -69,7 +69,7 @@ static void guest_code(void *test_data)
 		GUEST_ASSERT(prepare_for_vmx_operation(test_data));
 		GUEST_ASSERT(load_vmcs(test_data));
 
-		prepare_vmcs(test_data, NULL, NULL);
+		prepare_vmcs(test_data, NULL);
 		GUEST_ASSERT(!vmwrite(GUEST_IDTR_LIMIT, 0));
 		GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code));
 		GUEST_ASSERT(!vmwrite(EXCEPTION_BITMAP, 0));
diff --git a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c
index 186e980aa8eee..aeec3121c8e83 100644
--- a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c
@@ -5,8 +5,6 @@
 #include "vmx.h"
 #include "svm_util.h"
 
-#define L2_GUEST_STACK_SIZE 256
-
 /*
  * Arbitrary, never shoved into KVM/hardware, just need to avoid conflict with
  * the "real" exceptions used, #SS/#GP/#DF (12/13/8).
@@ -91,9 +89,8 @@ static void svm_run_l2(struct svm_test_data *svm, void *l2_code, int vector,
 static void l1_svm_code(struct svm_test_data *svm)
 {
 	struct vmcb_control_area *ctrl = &svm->vmcb->control;
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 
-	generic_svm_setup(svm, NULL, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, NULL);
 	svm->vmcb->save.idtr.limit = 0;
 	ctrl->intercept |= BIT_ULL(INTERCEPT_SHUTDOWN);
 
@@ -128,13 +125,11 @@ static void vmx_run_l2(void *l2_code, int vector, u32 error_code)
 
 static void l1_vmx_code(struct vmx_pages *vmx)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
-
 	GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true);
 
 	GUEST_ASSERT_EQ(load_vmcs(vmx), true);
 
-	prepare_vmcs(vmx, NULL, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	prepare_vmcs(vmx, NULL);
 	GUEST_ASSERT_EQ(vmwrite(GUEST_IDTR_LIMIT, 0), 0);
 
 	/*
diff --git a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c
index 11fd2467d8233..8c2ba9674558e 100644
--- a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c
@@ -11,8 +11,6 @@
 #include "kselftest.h"
 
 
-#define L2_GUEST_STACK_SIZE 64
-
 static void l2_guest_code(void)
 {
 	vmcall();
@@ -20,11 +18,9 @@ static void l2_guest_code(void)
 
 static void l1_svm_code(struct svm_test_data *svm)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	uintptr_t save_cr3;
 
-	generic_svm_setup(svm, l2_guest_code,
-			  &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_guest_code);
 
 	/* Try to run L2 with invalid CR3 and make sure it fails */
 	save_cr3 = svm->vmcb->save.cr3;
@@ -42,14 +38,12 @@ static void l1_svm_code(struct svm_test_data *svm)
 
 static void l1_vmx_code(struct vmx_pages *vmx_pages)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	uintptr_t save_cr3;
 
 	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
 	GUEST_ASSERT(load_vmcs(vmx_pages));
 
-	prepare_vmcs(vmx_pages, l2_guest_code,
-		     &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	prepare_vmcs(vmx_pages, l2_guest_code);
 
 	/* Try to run L2 with invalid CR3 and make sure it fails */
 	save_cr3 = vmreadz(GUEST_CR3);
diff --git a/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c
index fa95568f55ffc..2e04563790ff2 100644
--- a/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c
@@ -9,8 +9,6 @@
 #include "svm_util.h"
 #include "vmx.h"
 
-#define L2_GUEST_STACK_SIZE 64
-
 enum test_type {
 	TEST_FINAL_PAGE_UNMAPPED,	    /* Final data page not present */
 	TEST_PT_PAGE_UNMAPPED,		    /* Page table page not present */
@@ -54,14 +52,13 @@ static void l2_guest_code_ins(void)
 static void l1_vmx_code(struct vmx_pages *vmx, u64 expected_fault_gpa,
 			u64 test_type)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	u64 exit_qual;
 
 	GUEST_ASSERT(vmx->vmcs_gpa);
 	GUEST_ASSERT(prepare_for_vmx_operation(vmx));
 	GUEST_ASSERT(load_vmcs(vmx));
 
-	prepare_vmcs(vmx, l2_entry, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	prepare_vmcs(vmx, l2_entry);
 
 	GUEST_ASSERT(!vmlaunch());
 
@@ -120,12 +117,10 @@ static void l1_vmx_code(struct vmx_pages *vmx, u64 expected_fault_gpa,
 static void l1_svm_code(struct svm_test_data *svm, u64 expected_fault_gpa,
 			 u64 test_type)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	struct vmcb *vmcb = svm->vmcb;
 	u64 exit_info_1;
 
-	generic_svm_setup(svm, l2_entry,
-			  &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_entry);
 
 	run_guest(vmcb, svm->vmcb_gpa);
 
diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c
index f0e4adac47510..cb79d7b9619c2 100644
--- a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c
@@ -34,8 +34,6 @@
 #define TSC_ADJUST_VALUE (1ll << 32)
 #define TSC_OFFSET_VALUE -(1ll << 48)
 
-#define L2_GUEST_STACK_SIZE 64
-
 enum {
 	PORT_ABORT = 0x1000,
 	PORT_REPORT,
@@ -75,8 +73,6 @@ static void l2_guest_code(void)
 
 static void l1_guest_code(void *data)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
-
 	/* Set TSC from L1 and make sure TSC_ADJUST is updated correctly */
 	GUEST_ASSERT(rdtsc() < TSC_ADJUST_VALUE);
 	wrmsr(MSR_IA32_TSC, rdtsc() - TSC_ADJUST_VALUE);
@@ -93,8 +89,7 @@ static void l1_guest_code(void *data)
 		GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
 		GUEST_ASSERT(load_vmcs(vmx_pages));
 
-		prepare_vmcs(vmx_pages, l2_guest_code,
-			     &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+		prepare_vmcs(vmx_pages, l2_guest_code);
 		control = vmreadz(CPU_BASED_VM_EXEC_CONTROL);
 		control |= CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_USE_TSC_OFFSETTING;
 		vmwrite(CPU_BASED_VM_EXEC_CONTROL, control);
@@ -105,8 +100,7 @@ static void l1_guest_code(void *data)
 	} else {
 		struct svm_test_data *svm = data;
 
-		generic_svm_setup(svm, l2_guest_code,
-				  &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+		generic_svm_setup(svm, l2_guest_code);
 
 		svm->vmcb->control.tsc_offset = TSC_OFFSET_VALUE;
 		run_guest(svm->vmcb, svm->vmcb_gpa);
diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c
index 190e93af20a14..18f765835bf4c 100644
--- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c
@@ -22,8 +22,6 @@
 #define TSC_OFFSET_L2 ((u64)-33125236320908)
 #define TSC_MULTIPLIER_L2 (L2_SCALE_FACTOR << 48)
 
-#define L2_GUEST_STACK_SIZE 64
-
 enum { USLEEP, UCHECK_L1, UCHECK_L2 };
 #define GUEST_SLEEP(sec)         ucall(UCALL_SYNC, 2, USLEEP, sec)
 #define GUEST_CHECK(level, freq) ucall(UCALL_SYNC, 2, level, freq)
@@ -82,13 +80,10 @@ static void l2_guest_code(void)
 
 static void l1_svm_code(struct svm_test_data *svm)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
-
 	/* check that L1's frequency looks alright before launching L2 */
 	check_tsc_freq(UCHECK_L1);
 
-	generic_svm_setup(svm, l2_guest_code,
-			  &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_guest_code);
 
 	/* enable TSC scaling for L2 */
 	wrmsr(MSR_AMD64_TSC_RATIO, L2_SCALE_FACTOR << 32);
@@ -105,7 +100,6 @@ static void l1_svm_code(struct svm_test_data *svm)
 
 static void l1_vmx_code(struct vmx_pages *vmx_pages)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	u32 control;
 
 	/* check that L1's frequency looks alright before launching L2 */
@@ -115,7 +109,7 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages)
 	GUEST_ASSERT(load_vmcs(vmx_pages));
 
 	/* prepare the VMCS for L2 execution */
-	prepare_vmcs(vmx_pages, l2_guest_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	prepare_vmcs(vmx_pages, l2_guest_code);
 
 	/* enable TSC offsetting and TSC scaling for L2 */
 	control = vmreadz(CPU_BASED_VM_EXEC_CONTROL);
diff --git a/tools/testing/selftests/kvm/x86/nested_vmsave_vmload_test.c b/tools/testing/selftests/kvm/x86/nested_vmsave_vmload_test.c
index 85d3f4cc76f39..a130759f39a19 100644
--- a/tools/testing/selftests/kvm/x86/nested_vmsave_vmload_test.c
+++ b/tools/testing/selftests/kvm/x86/nested_vmsave_vmload_test.c
@@ -28,8 +28,6 @@
 
 #define TEST_VMCB_L2_GPA		TEST_VMCB_L1_GPA(0)
 
-#define L2_GUEST_STACK_SIZE		64
-
 static void l2_guest_code_vmsave(void)
 {
 	asm volatile("vmsave %0" : : "a"(TEST_VMCB_L2_GPA) : "memory");
@@ -70,10 +68,8 @@ static void l2_guest_code_vmcb1(void)
 
 static void l1_guest_code(struct svm_test_data *svm)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
-
 	/* Each test case initializes the guest RIP below */
-	generic_svm_setup(svm, NULL, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, NULL);
 
 	/* Set VMSAVE/VMLOAD intercepts and make sure they work with.. */
 	svm->vmcb->control.intercept |= (BIT_ULL(INTERCEPT_VMSAVE) |
diff --git a/tools/testing/selftests/kvm/x86/sev_dbg_test.c b/tools/testing/selftests/kvm/x86/sev_dbg_test.c
index a9d8e4c059f96..eaa8201b937d3 100644
--- a/tools/testing/selftests/kvm/x86/sev_dbg_test.c
+++ b/tools/testing/selftests/kvm/x86/sev_dbg_test.c
@@ -34,7 +34,7 @@ static void validate_buffers(void)
 
 static void ____test_sev_dbg(struct kvm_vm *vm, int i, int j, int nr_bytes)
 {
-	u8 pattern = guest_random_u32(&guest_rng);
+	u8 pattern = kvm_random_u32(&kvm_rng);
 
 	if (i + nr_bytes > BUFFER_SIZE || j + nr_bytes > BUFFER_SIZE)
 		return;
diff --git a/tools/testing/selftests/kvm/x86/sev_migrate_tests.c b/tools/testing/selftests/kvm/x86/sev_migrate_tests.c
index 42bc023d51937..d59abb198d86f 100644
--- a/tools/testing/selftests/kvm/x86/sev_migrate_tests.c
+++ b/tools/testing/selftests/kvm/x86/sev_migrate_tests.c
@@ -313,6 +313,49 @@ static void test_sev_mirror_parameters(void)
 	kvm_vm_free(vm_no_vcpu);
 }
 
+static void test_sev_snp_migrate_reject(void)
+{
+	struct kvm_vm *src_vm, *dst_vm;
+	int ret;
+
+	src_vm = vm_create_barebones_type(KVM_X86_SNP_VM);
+	snp_vm_init(src_vm);
+	__vm_vcpu_add(src_vm, 0);
+	vm_sev_launch(src_vm, snp_default_policy(), NULL);
+
+	dst_vm = vm_create_barebones_type(KVM_X86_SNP_VM);
+	__vm_vcpu_add(dst_vm, 0);
+
+	ret = __sev_migrate_from(dst_vm, src_vm);
+	TEST_ASSERT(ret == -1 && errno == EINVAL,
+		    "SNP VM migration should be rejected. ret: %d, errno: %d",
+		    ret, errno);
+
+	kvm_vm_free(src_vm);
+	kvm_vm_free(dst_vm);
+}
+
+static void test_sev_snp_mirror_reject(void)
+{
+	struct kvm_vm *src_vm, *dst_vm;
+	int ret;
+
+	src_vm = vm_create_barebones_type(KVM_X86_SNP_VM);
+	snp_vm_init(src_vm);
+	__vm_vcpu_add(src_vm, 0);
+	vm_sev_launch(src_vm, snp_default_policy(), NULL);
+
+	dst_vm = aux_vm_create(false);
+
+	ret = __sev_mirror_create(dst_vm, src_vm);
+	TEST_ASSERT(ret == -1 && errno == EINVAL,
+		    "SNP VM mirroring should be rejected. ret: %d, errno: %d",
+		    ret, errno);
+
+	kvm_vm_free(src_vm);
+	kvm_vm_free(dst_vm);
+}
+
 static void test_sev_move_copy(void)
 {
 	struct kvm_vm *dst_vm, *dst2_vm, *dst3_vm, *sev_vm, *mirror_vm,
@@ -384,12 +427,16 @@ int main(int argc, char *argv[])
 		test_sev_migrate_parameters();
 		if (kvm_has_cap(KVM_CAP_VM_COPY_ENC_CONTEXT_FROM))
 			test_sev_move_copy();
+		if (kvm_cpu_has(X86_FEATURE_SEV_SNP))
+			test_sev_snp_migrate_reject();
 	}
 	if (kvm_has_cap(KVM_CAP_VM_COPY_ENC_CONTEXT_FROM)) {
 		test_sev_mirror(/* es= */ false);
 		if (have_sev_es)
 			test_sev_mirror(/* es= */ true);
 		test_sev_mirror_parameters();
+		if (kvm_cpu_has(X86_FEATURE_SEV_SNP))
+			test_sev_snp_mirror_reject();
 	}
 	return 0;
 }
diff --git a/tools/testing/selftests/kvm/x86/smm_test.c b/tools/testing/selftests/kvm/x86/smm_test.c
index 740051167dbd4..e2542f4ced605 100644
--- a/tools/testing/selftests/kvm/x86/smm_test.c
+++ b/tools/testing/selftests/kvm/x86/smm_test.c
@@ -63,8 +63,6 @@ static void l2_guest_code(void)
 
 static void guest_code(void *arg)
 {
-	#define L2_GUEST_STACK_SIZE 64
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	u64 apicbase = rdmsr(MSR_IA32_APICBASE);
 	struct svm_test_data *svm = arg;
 	struct vmx_pages *vmx_pages = arg;
@@ -81,13 +79,11 @@ static void guest_code(void *arg)
 
 	if (arg) {
 		if (this_cpu_has(X86_FEATURE_SVM)) {
-			generic_svm_setup(svm, l2_guest_code,
-					  &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+			generic_svm_setup(svm, l2_guest_code);
 		} else {
 			GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
 			GUEST_ASSERT(load_vmcs(vmx_pages));
-			prepare_vmcs(vmx_pages, l2_guest_code,
-				     &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+			prepare_vmcs(vmx_pages, l2_guest_code);
 		}
 
 		sync_with_host(5);
diff --git a/tools/testing/selftests/kvm/x86/state_test.c b/tools/testing/selftests/kvm/x86/state_test.c
index 409c6cc9f9214..4a1056a6cb8dc 100644
--- a/tools/testing/selftests/kvm/x86/state_test.c
+++ b/tools/testing/selftests/kvm/x86/state_test.c
@@ -19,8 +19,6 @@
 #include "vmx.h"
 #include "svm_util.h"
 
-#define L2_GUEST_STACK_SIZE 256
-
 void svm_l2_guest_code(void)
 {
 	GUEST_SYNC(4);
@@ -35,13 +33,11 @@ void svm_l2_guest_code(void)
 
 static void svm_l1_guest_code(struct svm_test_data *svm)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	struct vmcb *vmcb = svm->vmcb;
 
 	GUEST_ASSERT(svm->vmcb_gpa);
 	/* Prepare for L2 execution. */
-	generic_svm_setup(svm, svm_l2_guest_code,
-			  &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, svm_l2_guest_code);
 
 	vmcb->control.int_ctl |= (V_GIF_ENABLE_MASK | V_GIF_MASK);
 
@@ -78,8 +74,6 @@ void vmx_l2_guest_code(void)
 
 static void vmx_l1_guest_code(struct vmx_pages *vmx_pages)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
-
 	GUEST_ASSERT(vmx_pages->vmcs_gpa);
 	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
 	GUEST_SYNC(3);
@@ -89,8 +83,7 @@ static void vmx_l1_guest_code(struct vmx_pages *vmx_pages)
 	GUEST_SYNC(4);
 	GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa);
 
-	prepare_vmcs(vmx_pages, vmx_l2_guest_code,
-		     &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	prepare_vmcs(vmx_pages, vmx_l2_guest_code);
 
 	GUEST_SYNC(5);
 	GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa);
diff --git a/tools/testing/selftests/kvm/x86/svm_int_ctl_test.c b/tools/testing/selftests/kvm/x86/svm_int_ctl_test.c
index d3cc5e4f78831..7b1f4a4818bdd 100644
--- a/tools/testing/selftests/kvm/x86/svm_int_ctl_test.c
+++ b/tools/testing/selftests/kvm/x86/svm_int_ctl_test.c
@@ -54,15 +54,12 @@ static void l2_guest_code(struct svm_test_data *svm)
 
 static void l1_guest_code(struct svm_test_data *svm)
 {
-	#define L2_GUEST_STACK_SIZE 64
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	struct vmcb *vmcb = svm->vmcb;
 
 	x2apic_enable();
 
 	/* Prepare for L2 execution. */
-	generic_svm_setup(svm, l2_guest_code,
-			  &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_guest_code);
 
 	/* No virtual interrupt masking */
 	vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
diff --git a/tools/testing/selftests/kvm/x86/svm_lbr_nested_state.c b/tools/testing/selftests/kvm/x86/svm_lbr_nested_state.c
index 7fbfaa054c952..77c6ce9f45078 100644
--- a/tools/testing/selftests/kvm/x86/svm_lbr_nested_state.c
+++ b/tools/testing/selftests/kvm/x86/svm_lbr_nested_state.c
@@ -9,8 +9,6 @@
 #include "svm_util.h"
 
 
-#define L2_GUEST_STACK_SIZE 64
-
 #define DO_BRANCH() do { asm volatile("jmp 1f\n 1: nop"); } while (0)
 
 struct lbr_branch {
@@ -55,7 +53,6 @@ static void l2_guest_code(struct svm_test_data *svm)
 
 static void l1_guest_code(struct svm_test_data *svm, bool nested_lbrv)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	struct vmcb *vmcb = svm->vmcb;
 	struct lbr_branch l1_branch;
 
@@ -65,8 +62,7 @@ static void l1_guest_code(struct svm_test_data *svm, bool nested_lbrv)
 	CHECK_BRANCH_MSRS(&l1_branch);
 
 	/* Run L2, which will also do the same */
-	generic_svm_setup(svm, l2_guest_code,
-			  &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_guest_code);
 
 	if (nested_lbrv)
 		vmcb->control.misc_ctl2 = SVM_MISC2_ENABLE_V_LBR;
diff --git a/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c b/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c
index 6a89eaffc6578..6bc301207cbcb 100644
--- a/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c
+++ b/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c
@@ -8,8 +8,6 @@
 #include "kselftest.h"
 
 
-#define L2_GUEST_STACK_SIZE 64
-
 static void l2_guest_code(void)
 {
 	unsigned long efer = rdmsr(MSR_EFER);
@@ -24,10 +22,7 @@ static void l2_guest_code(void)
 
 static void l1_guest_code(struct svm_test_data *svm)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
-
-	generic_svm_setup(svm, l2_guest_code,
-			  &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_guest_code);
 	run_guest(svm->vmcb, svm->vmcb_gpa);
 
 	/* Unreachable, L1 should be shutdown */
diff --git a/tools/testing/selftests/kvm/x86/svm_nested_pat_test.c b/tools/testing/selftests/kvm/x86/svm_nested_pat_test.c
index 92da8ff34da19..14ec9d6ad1950 100644
--- a/tools/testing/selftests/kvm/x86/svm_nested_pat_test.c
+++ b/tools/testing/selftests/kvm/x86/svm_nested_pat_test.c
@@ -25,8 +25,6 @@
 #include "processor.h"
 #include "svm_util.h"
 
-#define L2_GUEST_STACK_SIZE 256
-
 #define PAT_DEFAULT		0x0007040600070406ULL
 #define L1_PAT_VALUE		0x0007040600070404ULL  /* Change PA0 to WT */
 #define L2_VMCB12_PAT		0x0606060606060606ULL  /* All WB */
@@ -59,14 +57,13 @@ static void l2_guest_code(void)
 
 static void l1_guest_code(struct svm_test_data *svm)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	struct vmcb *vmcb = svm->vmcb;
 	int i;
 
 	wrmsr(MSR_IA32_CR_PAT, L1_PAT_VALUE);
 	GUEST_ASSERT_EQ(rdmsr(MSR_IA32_CR_PAT), L1_PAT_VALUE);
 
-	generic_svm_setup(svm, l2_guest_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_guest_code);
 
 	vmcb->save.g_pat = L2_VMCB12_PAT;
 	vmcb->control.intercept &= ~(1ULL << INTERCEPT_MSR_PROT);
@@ -94,11 +91,10 @@ static void l1_guest_code(struct svm_test_data *svm)
 
 static void l1_guest_code_invalid_gpat(struct svm_test_data *svm)
 {
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	struct vmcb *vmcb = svm->vmcb;
 
 	/* VMRUN should fail without running L2 */
-	generic_svm_setup(svm, NULL, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, NULL);
 
 	vmcb->save.g_pat = INVALID_PAT_VALUE;
 	run_guest(vmcb, svm->vmcb_gpa);
diff --git a/tools/testing/selftests/kvm/x86/svm_nested_shutdown_test.c b/tools/testing/selftests/kvm/x86/svm_nested_shutdown_test.c
index c6ea3d609a629..2a4a216954bb3 100644
--- a/tools/testing/selftests/kvm/x86/svm_nested_shutdown_test.c
+++ b/tools/testing/selftests/kvm/x86/svm_nested_shutdown_test.c
@@ -19,12 +19,9 @@ static void l2_guest_code(struct svm_test_data *svm)
 
 static void l1_guest_code(struct svm_test_data *svm, struct idt_entry *idt)
 {
-	#define L2_GUEST_STACK_SIZE 64
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	struct vmcb *vmcb = svm->vmcb;
 
-	generic_svm_setup(svm, l2_guest_code,
-			  &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_guest_code);
 
 	vmcb->control.intercept &= ~(BIT(INTERCEPT_SHUTDOWN));
 
diff --git a/tools/testing/selftests/kvm/x86/svm_nested_soft_inject_test.c b/tools/testing/selftests/kvm/x86/svm_nested_soft_inject_test.c
index f72f11d4c4f83..0b640d09d1943 100644
--- a/tools/testing/selftests/kvm/x86/svm_nested_soft_inject_test.c
+++ b/tools/testing/selftests/kvm/x86/svm_nested_soft_inject_test.c
@@ -78,17 +78,13 @@ static void l2_guest_code_nmi(void)
 
 static void l1_guest_code(struct svm_test_data *svm, u64 is_nmi, u64 idt_alt)
 {
-	#define L2_GUEST_STACK_SIZE 64
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	struct vmcb *vmcb = svm->vmcb;
 
 	if (is_nmi)
 		x2apic_enable();
 
 	/* Prepare for L2 execution. */
-	generic_svm_setup(svm,
-			  is_nmi ? l2_guest_code_nmi : l2_guest_code_int,
-			  &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, is_nmi ? l2_guest_code_nmi : l2_guest_code_int);
 
 	vmcb->control.intercept_exceptions |= BIT(PF_VECTOR) | BIT(UD_VECTOR);
 	vmcb->control.intercept |= BIT(INTERCEPT_NMI) | BIT(INTERCEPT_HLT);
diff --git a/tools/testing/selftests/kvm/x86/svm_nested_vmcb12_gpa.c b/tools/testing/selftests/kvm/x86/svm_nested_vmcb12_gpa.c
index a4935ce2fb998..b3f45035745ff 100644
--- a/tools/testing/selftests/kvm/x86/svm_nested_vmcb12_gpa.c
+++ b/tools/testing/selftests/kvm/x86/svm_nested_vmcb12_gpa.c
@@ -9,14 +9,9 @@
 #include "kvm_test_harness.h"
 #include "test_util.h"
 
-
-#define L2_GUEST_STACK_SIZE 64
-
 #define SYNC_GP 101
 #define SYNC_L2_STARTED 102
 
-static unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
-
 static void guest_gp_handler(struct ex_regs *regs)
 {
 	GUEST_SYNC(SYNC_GP);
@@ -30,28 +25,28 @@ static void l2_code(void)
 
 static void l1_vmrun(struct svm_test_data *svm, gpa_t gpa)
 {
-	generic_svm_setup(svm, l2_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_code);
 
 	asm volatile ("vmrun %[gpa]" : : [gpa] "a" (gpa) : "memory");
 }
 
 static void l1_vmload(struct svm_test_data *svm, gpa_t gpa)
 {
-	generic_svm_setup(svm, l2_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_code);
 
 	asm volatile ("vmload %[gpa]" : : [gpa] "a" (gpa) : "memory");
 }
 
 static void l1_vmsave(struct svm_test_data *svm, gpa_t gpa)
 {
-	generic_svm_setup(svm, l2_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_code);
 
 	asm volatile ("vmsave %[gpa]" : : [gpa] "a" (gpa) : "memory");
 }
 
 static void l1_vmexit(struct svm_test_data *svm, gpa_t gpa)
 {
-	generic_svm_setup(svm, l2_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_code);
 
 	run_guest(svm->vmcb, svm->vmcb_gpa);
 	GUEST_ASSERT(svm->vmcb->control.exit_code == SVM_EXIT_VMMCALL);
diff --git a/tools/testing/selftests/kvm/x86/svm_pmu_host_guest_test.c b/tools/testing/selftests/kvm/x86/svm_pmu_host_guest_test.c
new file mode 100644
index 0000000000000..c5b5cd788d93f
--- /dev/null
+++ b/tools/testing/selftests/kvm/x86/svm_pmu_host_guest_test.c
@@ -0,0 +1,215 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * KVM nested SVM PMU Host-Only/Guest-Only test
+ *
+ * Copyright (C) 2026, Google LLC.
+ *
+ * Test that KVM correctly virtualizes the AMD PMU Host-Only (bit 41) and
+ * Guest-Only (bit 40) event selector bits across all SVM state
+ * transitions.
+ *
+ * Programs 4 PMCs simultaneously with all combinations of Host-Only and
+ * Guest-Only bits, then verifies correct counting behavior with different
+ * combinations of EFER.SVME and host/guest mode -- as well as event filtering.
+ */
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "test_util.h"
+#include "kvm_util.h"
+#include "processor.h"
+#include "svm_util.h"
+#include "pmu.h"
+
+#define EVENTSEL_RETIRED_INSNS	(ARCH_PERFMON_EVENTSEL_OS |	\
+				 ARCH_PERFMON_EVENTSEL_USR |	\
+				 ARCH_PERFMON_EVENTSEL_ENABLE |	\
+				 AMD_ZEN_INSTRUCTIONS_RETIRED)
+
+/* PMC configurations: index corresponds to Host-Only | Guest-Only bits */
+#define PMC_NONE	0  /* Neither bit set */
+#define PMC_G		1  /* Guest-Only bit set */
+#define PMC_H		2  /* Host-Only bit set */
+#define PMC_HG		3  /* Both bits set */
+#define NR_PMCS		4
+
+#define LOOP_INSNS	1000
+
+static __always_inline void run_instruction_loop(void)
+{
+	unsigned int i;
+
+	for (i = 0; i < LOOP_INSNS; i++)
+		__asm__ __volatile__("nop");
+}
+
+static __always_inline void read_counters(uint64_t *counts)
+{
+	int i;
+
+	for (i = 0; i < NR_PMCS; i++)
+		counts[i] = rdmsr(MSR_F15H_PERF_CTR + 2 * i);
+}
+
+static __always_inline void run_and_measure(uint64_t *deltas)
+{
+	uint64_t before[NR_PMCS], after[NR_PMCS];
+	int i;
+
+	read_counters(before);
+	run_instruction_loop();
+	read_counters(after);
+
+	for (i = 0; i < NR_PMCS; i++)
+		deltas[i] = after[i] - before[i];
+}
+
+static void assert_pmc_counts(uint64_t *deltas, unsigned int expected_counting)
+{
+	int i;
+
+	for (i = 0; i < NR_PMCS; i++) {
+		if (expected_counting & BIT(i))
+			GUEST_ASSERT_NE(deltas[i], 0);
+		else
+			GUEST_ASSERT_EQ(deltas[i], 0);
+	}
+}
+
+static uint64_t l2_deltas[NR_PMCS];
+
+static void l2_guest_code(void)
+{
+	run_and_measure(l2_deltas);
+	vmmcall();
+}
+
+static void l1_guest_code(struct svm_test_data *svm)
+{
+	struct vmcb *vmcb = svm->vmcb;
+	uint64_t deltas[NR_PMCS];
+	uint64_t eventsel;
+	int i;
+
+	/* Program 4 PMCs with all combinations of Host-Only/Guest-Only bits */
+	for (i = 0; i < NR_PMCS; i++) {
+		eventsel = EVENTSEL_RETIRED_INSNS;
+		if (i & PMC_G)
+			eventsel |= AMD64_EVENTSEL_GUESTONLY;
+		if (i & PMC_H)
+			eventsel |= AMD64_EVENTSEL_HOSTONLY;
+		wrmsr(MSR_F15H_PERF_CTL + 2 * i, eventsel);
+		wrmsr(MSR_F15H_PERF_CTR + 2 * i, 0);
+	}
+
+	/* Step 1: SVME=0 - Only the counter with neither bits set counts */
+	wrmsr(MSR_EFER, rdmsr(MSR_EFER) & ~EFER_SVME);
+	run_and_measure(deltas);
+	assert_pmc_counts(deltas, BIT(PMC_NONE));
+
+	/* Step 2: Set SVME=1 - In L1 "host mode"; Guest-Only stops */
+	wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_SVME);
+	run_and_measure(deltas);
+	assert_pmc_counts(deltas, BIT(PMC_NONE) | BIT(PMC_H) | BIT(PMC_HG));
+
+	/* Step 3: VMRUN to L2 - In "guest mode"; Host-Only stops */
+	generic_svm_setup(svm, l2_guest_code);
+	vmcb->control.intercept &= ~(1ULL << INTERCEPT_MSR_PROT);
+
+	run_guest(vmcb, svm->vmcb_gpa);
+
+	GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_VMMCALL);
+	assert_pmc_counts(l2_deltas, BIT(PMC_NONE) | BIT(PMC_G) | BIT(PMC_HG));
+
+	/* Step 4: After VMEXIT to L1 - Back in "host mode"; Guest-Only stops */
+	run_and_measure(deltas);
+	assert_pmc_counts(deltas, BIT(PMC_NONE) | BIT(PMC_H) | BIT(PMC_HG));
+
+	/* Step 5: Set KVM_PMU_EVENT_DENY - all counters stop */
+	GUEST_SYNC(KVM_PMU_EVENT_DENY);
+	run_and_measure(deltas);
+	assert_pmc_counts(deltas, 0);
+
+	/* Step 6: Set KVM_PMU_EVENT_ALLOW - back to all except Guest-only */
+	GUEST_SYNC(KVM_PMU_EVENT_ALLOW);
+	run_and_measure(deltas);
+	assert_pmc_counts(deltas, BIT(PMC_NONE) | BIT(PMC_H) | BIT(PMC_HG));
+
+	/* Step 7: Clear Host-Only for PMC_HG - counter stops in "host mode" */
+	eventsel = rdmsr(MSR_F15H_PERF_CTL + 2 * PMC_HG);
+	wrmsr(MSR_F15H_PERF_CTL + 2 * PMC_HG, eventsel & ~AMD64_EVENTSEL_HOSTONLY);
+	run_and_measure(deltas);
+	assert_pmc_counts(deltas, BIT(PMC_NONE) | BIT(PMC_H));
+
+	/* Step 8: Restore Host-Only for PMC_HG - counter counts again */
+	wrmsr(MSR_F15H_PERF_CTL + 2 * PMC_HG, eventsel);
+	run_and_measure(deltas);
+	assert_pmc_counts(deltas, BIT(PMC_NONE) | BIT(PMC_H) | BIT(PMC_HG));
+
+	/* Step 9: Clear SVME - Only the counter with neither bits set counts */
+	wrmsr(MSR_EFER, rdmsr(MSR_EFER) & ~EFER_SVME);
+	run_and_measure(deltas);
+	assert_pmc_counts(deltas, BIT(PMC_NONE));
+
+	GUEST_DONE();
+}
+
+static struct kvm_pmu_event_filter *alloc_event_filter(u64 event)
+{
+	struct kvm_pmu_event_filter *filter;
+
+	filter = malloc(sizeof(*filter) + sizeof(event));
+	TEST_ASSERT(filter != NULL, "Filter allocation failed");
+
+	memset(filter, 0, sizeof(*filter));
+	memcpy(filter->events, &event, sizeof(event));
+	filter->nevents = 1;
+	filter->action = KVM_PMU_EVENT_ALLOW;
+
+	return filter;
+}
+
+int main(int argc, char *argv[])
+{
+	struct kvm_pmu_event_filter *filter;
+	struct kvm_vcpu *vcpu;
+	struct kvm_vm *vm;
+	struct ucall uc;
+	gva_t svm_gva;
+
+	TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM));
+	TEST_REQUIRE(kvm_is_pmu_enabled());
+	TEST_REQUIRE(kvm_is_mediated_pmu_enabled());
+
+	vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code);
+
+	vcpu_alloc_svm(vm, &svm_gva);
+	vcpu_args_set(vcpu, 1, svm_gva);
+
+	filter = alloc_event_filter(AMD_ZEN_INSTRUCTIONS_RETIRED);
+
+	for (;;) {
+		vcpu_run(vcpu);
+		TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
+
+		switch (get_ucall(vcpu, &uc)) {
+		case UCALL_ABORT:
+			REPORT_GUEST_ASSERT(uc);
+			goto done;
+		case UCALL_DONE:
+			goto done;
+		case UCALL_SYNC:
+			filter->action = uc.args[1];
+			vm_ioctl(vm, KVM_SET_PMU_EVENT_FILTER, filter);
+			break;
+		default:
+			TEST_FAIL("Unknown ucall %lu", uc.cmd);
+			goto done;
+		}
+	}
+done:
+	kvm_vm_free(vm);
+	return 0;
+}
diff --git a/tools/testing/selftests/kvm/x86/svm_vmcall_test.c b/tools/testing/selftests/kvm/x86/svm_vmcall_test.c
index b1887242f3b8e..7c57fb7e64221 100644
--- a/tools/testing/selftests/kvm/x86/svm_vmcall_test.c
+++ b/tools/testing/selftests/kvm/x86/svm_vmcall_test.c
@@ -19,13 +19,10 @@ static void l2_guest_code(struct svm_test_data *svm)
 
 static void l1_guest_code(struct svm_test_data *svm)
 {
-	#define L2_GUEST_STACK_SIZE 64
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	struct vmcb *vmcb = svm->vmcb;
 
 	/* Prepare for L2 execution. */
-	generic_svm_setup(svm, l2_guest_code,
-			  &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_guest_code);
 
 	run_guest(vmcb, svm->vmcb_gpa);
 
diff --git a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c
index f1c488e0d4975..0d83516f4bd08 100644
--- a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c
+++ b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c
@@ -21,9 +21,6 @@ static void l2_guest_code(void)
 		     : : [port] "d" (ARBITRARY_IO_PORT) : "rax");
 }
 
-#define L2_GUEST_STACK_SIZE 64
-unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
-
 void l1_guest_code_vmx(struct vmx_pages *vmx)
 {
 
@@ -31,8 +28,7 @@ void l1_guest_code_vmx(struct vmx_pages *vmx)
 	GUEST_ASSERT(prepare_for_vmx_operation(vmx));
 	GUEST_ASSERT(load_vmcs(vmx));
 
-	prepare_vmcs(vmx, l2_guest_code,
-		     &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	prepare_vmcs(vmx, l2_guest_code);
 
 	GUEST_ASSERT(!vmlaunch());
 	/* L2 should triple fault after a triple fault event injected. */
@@ -44,8 +40,7 @@ void l1_guest_code_svm(struct svm_test_data *svm)
 {
 	struct vmcb *vmcb = svm->vmcb;
 
-	generic_svm_setup(svm, l2_guest_code,
-			&l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	generic_svm_setup(svm, l2_guest_code);
 
 	/* don't intercept shutdown to test the case of SVM allowing to do so */
 	vmcb->control.intercept &= ~(BIT(INTERCEPT_SHUTDOWN));
diff --git a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c
index 1720113eae799..463f73aa9159a 100644
--- a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c
+++ b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c
@@ -36,16 +36,13 @@ static void l2_guest_code(void)
 
 static void l1_guest_code(struct vmx_pages *vmx_pages, unsigned long high_gpa)
 {
-#define L2_GUEST_STACK_SIZE 64
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	u32 control;
 
 	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
 	GUEST_ASSERT(load_vmcs(vmx_pages));
 
 	/* Prepare the VMCS for L2 execution. */
-	prepare_vmcs(vmx_pages, l2_guest_code,
-		     &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	prepare_vmcs(vmx_pages, l2_guest_code);
 	control = vmreadz(CPU_BASED_VM_EXEC_CONTROL);
 	control |= CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
 	vmwrite(CPU_BASED_VM_EXEC_CONTROL, control);
diff --git a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c
index 80a4fd1e5bbbe..f9b88a6f6113d 100644
--- a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c
+++ b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c
@@ -31,15 +31,13 @@ static void l2_guest_code(void)
 
 static void l1_guest_code(struct vmx_pages *vmx_pages)
 {
-#define L2_GUEST_STACK_SIZE 64
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	u32 control;
 
 	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
 	GUEST_ASSERT(load_vmcs(vmx_pages));
 
 	/* Prepare the VMCS for L2 execution. */
-	prepare_vmcs(vmx_pages, l2_guest_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	prepare_vmcs(vmx_pages, l2_guest_code);
 	control = vmreadz(CPU_BASED_VM_EXEC_CONTROL);
 	control |= CPU_BASED_USE_MSR_BITMAPS;
 	vmwrite(CPU_BASED_VM_EXEC_CONTROL, control);
diff --git a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c
index a2eaceed9ad52..6d88c54f69faa 100644
--- a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c
+++ b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c
@@ -25,15 +25,11 @@ static void l2_guest_code(void)
 
 static void l1_guest_code(struct vmx_pages *vmx_pages)
 {
-#define L2_GUEST_STACK_SIZE 64
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
-
 	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
 	GUEST_ASSERT(load_vmcs(vmx_pages));
 
 	/* Prepare the VMCS for L2 execution. */
-	prepare_vmcs(vmx_pages, l2_guest_code,
-		     &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	prepare_vmcs(vmx_pages, l2_guest_code);
 
 	/*
 	 * L2 must be run without unrestricted guest, verify that the selftests
diff --git a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c
index f13dee3173837..75073efa926da 100644
--- a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c
+++ b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c
@@ -27,8 +27,6 @@ static void l2_guest_code(void)
 
 static void l1_guest_code(struct vmx_pages *vmx_pages)
 {
-#define L2_GUEST_STACK_SIZE 64
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	u64 guest_cr4;
 	gpa_t pml5_pa, pml4_pa;
 	u64 *pml5;
@@ -42,8 +40,7 @@ static void l1_guest_code(struct vmx_pages *vmx_pages)
 	GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages));
 	GUEST_ASSERT(load_vmcs(vmx_pages));
 
-	prepare_vmcs(vmx_pages, l2_guest_code,
-		     &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	prepare_vmcs(vmx_pages, l2_guest_code);
 
 	/*
 	 * Set up L2 with a 4-level page table by pointing its CR3 to
diff --git a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c
index 1b7b6ba23de76..eb8021c33cd43 100644
--- a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c
+++ b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c
@@ -66,8 +66,6 @@ void l2_guest_code(void)
 
 void l1_guest_code(struct vmx_pages *vmx_pages)
 {
-#define L2_GUEST_STACK_SIZE 64
-	unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
 	u64 l1_vmx_pt_start;
 	u64 l1_vmx_pt_finish;
 	u64 l1_tsc_deadline, l2_tsc_deadline;
@@ -77,8 +75,7 @@ void l1_guest_code(struct vmx_pages *vmx_pages)
 	GUEST_ASSERT(load_vmcs(vmx_pages));
 	GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa);
 
-	prepare_vmcs(vmx_pages, l2_guest_code,
-		     &l2_guest_stack[L2_GUEST_STACK_SIZE]);
+	prepare_vmcs(vmx_pages, l2_guest_code);
 
 	/*
 	 * Check for Preemption timer support
diff --git a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c
index 39ce9a9369f5b..3a326c5e74cab 100644
--- a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c
+++ b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c
@@ -17,7 +17,7 @@
  * amongst the available numa nodes on the machine.
  *
  * Migration is a command line option. When used on non-numa machines will 
- * exit with error. Test is still usefull on non-numa for testing IPIs.
+ * exit with error. Test is still useful on non-numa for testing IPIs.
  */
 #include <getopt.h>
 #include <pthread.h>
diff --git a/tools/testing/selftests/landlock/fs_test.c b/tools/testing/selftests/landlock/fs_test.c
index 86e08aa6e0a7e..e672089e93293 100644
--- a/tools/testing/selftests/landlock/fs_test.c
+++ b/tools/testing/selftests/landlock/fs_test.c
@@ -6927,7 +6927,7 @@ TEST_F_FORK(layout2_overlay, same_content_different_file)
 		ASSERT_EQ(0, test_open(path_entry, O_RDWR));
 	}
 
-	/* Only allowes access to the merge hierarchy. */
+	/* Only allows access to the merge hierarchy. */
 	enforce_fs(_metadata, ACCESS_RW, layer5_merge_only);
 
 	/* Checks new accesses on lower layer. */
diff --git a/tools/testing/selftests/landlock/net_test.c b/tools/testing/selftests/landlock/net_test.c
index 2ed1f76b7a8b1..be2eb88092fb3 100644
--- a/tools/testing/selftests/landlock/net_test.c
+++ b/tools/testing/selftests/landlock/net_test.c
@@ -1281,6 +1281,103 @@ TEST_F(protocol, connect_unspec)
 	EXPECT_EQ(0, close(bind_fd));
 }
 
+TEST_F(protocol, tcp_fastopen)
+{
+	const bool restricted = variant->sandbox == TCP_SANDBOX &&
+				variant->prot.type == SOCK_STREAM &&
+				(variant->prot.protocol == IPPROTO_TCP ||
+				 variant->prot.protocol == IPPROTO_IP) &&
+				(variant->prot.domain == AF_INET ||
+				 variant->prot.domain == AF_INET6);
+	const struct landlock_ruleset_attr ruleset_attr = {
+		.handled_access_net = LANDLOCK_ACCESS_NET_CONNECT_TCP,
+	};
+	int bind_fd, client_fd, status;
+	char buf;
+	pid_t child;
+
+	bind_fd = socket_variant(&self->srv0);
+	ASSERT_LE(0, bind_fd);
+	EXPECT_EQ(0, bind_variant(bind_fd, &self->srv0));
+	if (self->srv0.protocol.type == SOCK_STREAM)
+		EXPECT_EQ(0, listen(bind_fd, backlog));
+
+	child = fork();
+	ASSERT_LE(0, child);
+	if (child == 0) {
+		int connect_fd, ret;
+
+		/* Closes listening socket for the child. */
+		EXPECT_EQ(0, close(bind_fd));
+
+		connect_fd = socket_variant(&self->srv0);
+		ASSERT_LE(0, connect_fd);
+
+		if (variant->sandbox == TCP_SANDBOX) {
+			const int ruleset_fd = landlock_create_ruleset(
+				&ruleset_attr, sizeof(ruleset_attr), 0);
+			ASSERT_LE(0, ruleset_fd);
+
+			enforce_ruleset(_metadata, ruleset_fd);
+			EXPECT_EQ(0, close(ruleset_fd));
+		}
+
+		/* Fast Open with no address. */
+		ret = sendto_variant(connect_fd, NULL, NULL, 0, MSG_FASTOPEN);
+		if (self->srv0.protocol.domain == AF_UNIX) {
+			EXPECT_EQ(-ENOTCONN, ret);
+		} else if (self->srv0.protocol.type == SOCK_DGRAM) {
+			EXPECT_EQ(-EDESTADDRREQ, ret);
+		} else {
+			EXPECT_EQ(-EINVAL, ret);
+		}
+
+		/* Fast Open to a denied address. */
+		ret = sendto_variant(connect_fd, &self->srv0, "A", 1,
+				     MSG_FASTOPEN);
+		if (restricted) {
+			EXPECT_EQ(-EACCES, ret);
+		} else if (self->srv0.protocol.domain == AF_UNIX &&
+			   self->srv0.protocol.type == SOCK_STREAM) {
+			EXPECT_EQ(-EOPNOTSUPP, ret);
+		} else {
+			EXPECT_EQ(0, ret);
+		}
+
+		EXPECT_EQ(0, close(connect_fd));
+		_exit(_metadata->exit_code);
+		return;
+	}
+
+	client_fd = bind_fd;
+	if (!restricted && self->srv0.protocol.type == SOCK_STREAM &&
+	    self->srv0.protocol.domain != AF_UNIX) {
+		client_fd = accept(bind_fd, NULL, 0);
+		ASSERT_LE(0, client_fd);
+	}
+
+	if (restricted) {
+		EXPECT_EQ(-1, read(client_fd, &buf, 1));
+		EXPECT_EQ(ENOTCONN, errno);
+	} else if (self->srv0.protocol.domain == AF_UNIX &&
+		   self->srv0.protocol.type == SOCK_STREAM) {
+		EXPECT_EQ(-1, read(client_fd, &buf, 1));
+		EXPECT_EQ(EINVAL, errno);
+	} else {
+		EXPECT_EQ(1, read(client_fd, &buf, 1));
+		EXPECT_EQ('A', buf);
+	}
+
+	EXPECT_EQ(child, waitpid(child, &status, 0));
+	EXPECT_EQ(1, WIFEXITED(status));
+	EXPECT_EQ(EXIT_SUCCESS, WEXITSTATUS(status));
+
+	if (client_fd != bind_fd)
+		EXPECT_LE(0, close(client_fd));
+
+	EXPECT_EQ(0, close(bind_fd));
+}
+
 TEST_F(protocol, sendmsg_stream)
 {
 	int srv0_fd, tmp_fd, client_fd, res;
diff --git a/tools/testing/selftests/landlock/scoped_signal_test.c b/tools/testing/selftests/landlock/scoped_signal_test.c
index f24f2c28f62e5..2d37d0c06c069 100644
--- a/tools/testing/selftests/landlock/scoped_signal_test.c
+++ b/tools/testing/selftests/landlock/scoped_signal_test.c
@@ -249,12 +249,12 @@ TEST_F(scoped_domains, check_access_signal)
 		_metadata->exit_code = KSFT_FAIL;
 }
 
-enum thread_return {
-	THREAD_INVALID = 0,
-	THREAD_SUCCESS = 1,
-	THREAD_ERROR = 2,
-	THREAD_TEST_FAILED = 3,
-};
+/* clang-format off */
+#define THREAD_INVALID		((void *)0)
+#define THREAD_SUCCESS		((void *)1)
+#define THREAD_ERROR		((void *)2)
+#define THREAD_TEST_FAILED	((void *)3)
+/* clang-format on */
 
 static void *thread_sync(void *arg)
 {
@@ -262,15 +262,15 @@ static void *thread_sync(void *arg)
 	char buf;
 
 	if (read(pipe_read, &buf, 1) != 1)
-		return (void *)THREAD_ERROR;
+		return THREAD_ERROR;
 
-	return (void *)THREAD_SUCCESS;
+	return THREAD_SUCCESS;
 }
 
 TEST(signal_scoping_thread_before)
 {
 	pthread_t no_sandbox_thread;
-	enum thread_return ret = THREAD_INVALID;
+	void *ret = THREAD_INVALID;
 	int thread_pipe[2];
 
 	drop_caps(_metadata);
@@ -285,7 +285,7 @@ TEST(signal_scoping_thread_before)
 	EXPECT_EQ(0, pthread_kill(no_sandbox_thread, 0));
 	EXPECT_EQ(1, write(thread_pipe[1], ".", 1));
 
-	EXPECT_EQ(0, pthread_join(no_sandbox_thread, (void **)&ret));
+	EXPECT_EQ(0, pthread_join(no_sandbox_thread, &ret));
 	EXPECT_EQ(THREAD_SUCCESS, ret);
 
 	EXPECT_EQ(0, close(thread_pipe[0]));
@@ -295,7 +295,7 @@ TEST(signal_scoping_thread_before)
 TEST(signal_scoping_thread_after)
 {
 	pthread_t scoped_thread;
-	enum thread_return ret = THREAD_INVALID;
+	void *ret = THREAD_INVALID;
 	int thread_pipe[2];
 
 	drop_caps(_metadata);
@@ -310,7 +310,7 @@ TEST(signal_scoping_thread_after)
 	EXPECT_EQ(0, pthread_kill(scoped_thread, 0));
 	EXPECT_EQ(1, write(thread_pipe[1], ".", 1));
 
-	EXPECT_EQ(0, pthread_join(scoped_thread, (void **)&ret));
+	EXPECT_EQ(0, pthread_join(scoped_thread, &ret));
 	EXPECT_EQ(THREAD_SUCCESS, ret);
 
 	EXPECT_EQ(0, close(thread_pipe[0]));
@@ -327,20 +327,20 @@ void *thread_setuid(void *ptr)
 	char buf;
 
 	if (read(arg->pipe_read, &buf, 1) != 1)
-		return (void *)THREAD_ERROR;
+		return THREAD_ERROR;
 
 	/* libc's setuid() should update all thread's credentials. */
 	if (getuid() != arg->new_uid)
-		return (void *)THREAD_TEST_FAILED;
+		return THREAD_TEST_FAILED;
 
-	return (void *)THREAD_SUCCESS;
+	return THREAD_SUCCESS;
 }
 
 TEST(signal_scoping_thread_setuid)
 {
 	struct thread_setuid_args arg;
 	pthread_t no_sandbox_thread;
-	enum thread_return ret = THREAD_INVALID;
+	void *ret = THREAD_INVALID;
 	int pipe_parent[2];
 	int prev_uid;
 
@@ -367,7 +367,7 @@ TEST(signal_scoping_thread_setuid)
 	EXPECT_EQ(arg.new_uid, getuid());
 	EXPECT_EQ(1, write(pipe_parent[1], ".", 1));
 
-	EXPECT_EQ(0, pthread_join(no_sandbox_thread, (void **)&ret));
+	EXPECT_EQ(0, pthread_join(no_sandbox_thread, &ret));
 	EXPECT_EQ(THREAD_SUCCESS, ret);
 
 	clear_cap(_metadata, CAP_SETUID);
@@ -400,6 +400,24 @@ static int setup_signal_handler(int signal)
 	return sigaction(SIGURG, &sa, NULL);
 }
 
+/*
+ * MSG_OOB might be disabled in the kernel via the CONFIG_AF_UNIX_OOB
+ * switch, so this function can be used for probing for its availability.
+ */
+static bool has_af_unix_oob(void)
+{
+	bool available = false;
+	int sp[2];
+
+	if (socketpair(AF_UNIX, SOCK_STREAM, 0, sp) == 0) {
+		available = (send(sp[0], ".", 1, MSG_OOB) == 1);
+		close(sp[0]);
+		close(sp[1]);
+	}
+
+	return available;
+}
+
 /* clang-format off */
 FIXTURE(fown) {};
 /* clang-format on */
@@ -462,6 +480,9 @@ TEST_F(fown, sigurg_socket)
 	int pipe_parent[2], pipe_child[2];
 	pid_t child;
 
+	if (!has_af_unix_oob())
+		SKIP(return, "CONFIG_AF_UNIX_OOB / MSG_OOB not available");
+
 	memset(&server_address, 0, sizeof(server_address));
 	set_unix_address(&server_address, 0);
 
@@ -667,20 +688,20 @@ static void *thread_setown_scoped(void *arg)
 	ruleset_fd =
 		landlock_create_ruleset(&ruleset_attr, sizeof(ruleset_attr), 0);
 	if (ruleset_fd < 0)
-		return (void *)THREAD_ERROR;
+		return THREAD_ERROR;
 	if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) ||
 	    landlock_restrict_self(ruleset_fd, 0)) {
 		close(ruleset_fd);
-		return (void *)THREAD_ERROR;
+		return THREAD_ERROR;
 	}
 	close(ruleset_fd);
 
 	/* Makes this process group own the SIGIO source. */
 	if (fcntl(fd, F_SETSIG, SIGURG) || fcntl(fd, F_SETOWN, -getpgrp()) ||
 	    fcntl(fd, F_SETFL, O_ASYNC))
-		return (void *)THREAD_ERROR;
+		return THREAD_ERROR;
 
-	return (void *)THREAD_SUCCESS;
+	return THREAD_SUCCESS;
 }
 
 /*
@@ -702,7 +723,7 @@ TEST(sigio_to_pgid_self)
 {
 	int trigger[2];
 	pthread_t thread;
-	enum thread_return ret = THREAD_INVALID;
+	void *ret = THREAD_INVALID;
 	int i;
 
 	drop_caps(_metadata);
@@ -722,7 +743,7 @@ TEST(sigio_to_pgid_self)
 	 */
 	ASSERT_EQ(0, pthread_create(&thread, NULL, thread_setown_scoped,
 				    &trigger[0]));
-	ASSERT_EQ(0, pthread_join(thread, (void **)&ret));
+	ASSERT_EQ(0, pthread_join(thread, &ret));
 	ASSERT_EQ(THREAD_SUCCESS, ret);
 
 	/* Fans SIGURG out to the process group. */
diff --git a/tools/testing/selftests/liveupdate/.gitignore b/tools/testing/selftests/liveupdate/.gitignore
index 661827083ab60..cb08ddb0dfee0 100644
--- a/tools/testing/selftests/liveupdate/.gitignore
+++ b/tools/testing/selftests/liveupdate/.gitignore
@@ -6,4 +6,5 @@
 !*.sh
 !.gitignore
 !config
+!config.*
 !Makefile
diff --git a/tools/testing/selftests/liveupdate/config b/tools/testing/selftests/liveupdate/config
index 91d03f9a6a395..016d009dba134 100644
--- a/tools/testing/selftests/liveupdate/config
+++ b/tools/testing/selftests/liveupdate/config
@@ -1,4 +1,5 @@
 CONFIG_BLK_DEV_INITRD=y
+CONFIG_DEVTMPFS=y
 CONFIG_KEXEC_FILE=y
 CONFIG_KEXEC_HANDOVER=y
 CONFIG_KEXEC_HANDOVER_ENABLE_DEFAULT=y
diff --git a/tools/testing/selftests/liveupdate/config.aarch64 b/tools/testing/selftests/liveupdate/config.aarch64
new file mode 100644
index 0000000000000..445716403925f
--- /dev/null
+++ b/tools/testing/selftests/liveupdate/config.aarch64
@@ -0,0 +1,2 @@
+CONFIG_SERIAL_AMBA_PL011=y
+CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
diff --git a/tools/testing/selftests/liveupdate/config.x86_64 b/tools/testing/selftests/liveupdate/config.x86_64
new file mode 100644
index 0000000000000..810d9c9d213e8
--- /dev/null
+++ b/tools/testing/selftests/liveupdate/config.x86_64
@@ -0,0 +1,2 @@
+CONFIG_SERIAL_8250=y
+CONFIG_SERIAL_8250_CONSOLE=y
diff --git a/tools/testing/selftests/liveupdate/init.c b/tools/testing/selftests/liveupdate/init.c
new file mode 100644
index 0000000000000..fb08bd58b9b96
--- /dev/null
+++ b/tools/testing/selftests/liveupdate/init.c
@@ -0,0 +1,179 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (c) 2025, Google LLC.
+ * Pasha Tatashin <pasha.tatashin@soleen.com>
+ */
+#include <fcntl.h>
+#include <linux/kexec.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mount.h>
+#include <sys/reboot.h>
+#include <sys/syscall.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#define COMMAND_LINE_SIZE 2048
+#define KERNEL_IMAGE "/kernel"
+#define INITRD_IMAGE "/initrd.img"
+#define TEST_BINARY "/test_binary"
+
+static int mount_filesystems(void)
+{
+	if (mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) < 0) {
+		fprintf(stderr, "INIT: Warning: Failed to mount devtmpfs\n");
+		return -1;
+	}
+
+	if (mount("debugfs", "/debugfs", "debugfs", 0, NULL) < 0) {
+		fprintf(stderr, "INIT: Failed to mount debugfs\n");
+		return -1;
+	}
+
+	if (mount("proc", "/proc", "proc", 0, NULL) < 0) {
+		fprintf(stderr, "INIT: Failed to mount proc\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+static long kexec_file_load(int kernel_fd, int initrd_fd,
+			    unsigned long cmdline_len, const char *cmdline,
+			    unsigned long flags)
+{
+	return syscall(__NR_kexec_file_load, kernel_fd, initrd_fd, cmdline_len,
+		       cmdline, flags);
+}
+
+static int kexec_load(void)
+{
+	char cmdline[COMMAND_LINE_SIZE];
+	int kernel_fd, initrd_fd, err;
+	ssize_t len;
+	int fd;
+
+	fd = open("/proc/cmdline", O_RDONLY);
+	if (fd < 0) {
+		fprintf(stderr, "INIT: Failed to read /proc/cmdline\n");
+
+		return -1;
+	}
+
+	len = read(fd, cmdline, sizeof(cmdline) - 1);
+	close(fd);
+	if (len < 0)
+		return -1;
+
+	cmdline[len] = 0;
+	if (len > 0 && cmdline[len - 1] == '\n')
+		cmdline[len - 1] = 0;
+
+	strncat(cmdline, " luo_stage=2", sizeof(cmdline) - strlen(cmdline) - 1);
+
+	kernel_fd = open(KERNEL_IMAGE, O_RDONLY);
+	if (kernel_fd < 0) {
+		fprintf(stderr, "INIT: Failed to open kernel image\n");
+		return -1;
+	}
+
+	initrd_fd = open(INITRD_IMAGE, O_RDONLY);
+	if (initrd_fd < 0) {
+		fprintf(stderr, "INIT: Failed to open initrd image\n");
+		close(kernel_fd);
+		return -1;
+	}
+
+	err = kexec_file_load(kernel_fd, initrd_fd, strlen(cmdline) + 1,
+			      cmdline, 0);
+
+	close(initrd_fd);
+	close(kernel_fd);
+
+	return err;
+}
+
+static int run_test(int stage)
+{
+	char stage_arg[32];
+	int status;
+	pid_t pid;
+
+	snprintf(stage_arg, sizeof(stage_arg), "%d", stage);
+
+	pid = fork();
+	if (pid < 0)
+		return -1;
+
+	if (!pid) {
+		char *const argv[] = {TEST_BINARY, "-s", stage_arg, NULL};
+
+		execve(TEST_BINARY, argv, NULL);
+		fprintf(stderr, "INIT: execve failed\n");
+		_exit(1);
+	}
+
+	waitpid(pid, &status, 0);
+
+	return (WIFEXITED(status) && WEXITSTATUS(status) == 0) ? 0 : -1;
+}
+
+static int get_current_stage(void)
+{
+	char cmdline[COMMAND_LINE_SIZE];
+	ssize_t len;
+	int fd;
+
+	fd = open("/proc/cmdline", O_RDONLY);
+	if (fd < 0)
+		return -1;
+
+	len = read(fd, cmdline, sizeof(cmdline) - 1);
+	close(fd);
+
+	if (len < 0)
+		return -1;
+
+	cmdline[len] = 0;
+
+	return strstr(cmdline, "luo_stage=2") ? 2 : 1;
+}
+
+int main(int argc, char *argv[])
+{
+	int current_stage;
+	int err;
+
+	if (mount_filesystems())
+		goto err_reboot;
+
+	current_stage = get_current_stage();
+	if (current_stage < 0) {
+		fprintf(stderr, "INIT: Failed to read cmdline");
+		goto err_reboot;
+	}
+
+	printf("INIT: Starting Stage %d\n", current_stage);
+
+	if (current_stage == 1 && kexec_load()) {
+		fprintf(stderr, "INIT: Failed to load kexec kernel\n");
+		goto err_reboot;
+	}
+
+	if (run_test(current_stage)) {
+		fprintf(stderr, "INIT: Test binary returned failure\n");
+		goto err_reboot;
+	}
+
+	printf("INIT: Stage %d completed successfully.\n", current_stage);
+	reboot(current_stage == 1 ? RB_KEXEC : RB_AUTOBOOT);
+
+	return 0;
+
+err_reboot:
+	reboot(RB_AUTOBOOT);
+
+	return -1;
+}
diff --git a/tools/testing/selftests/liveupdate/luo_test_utils.c b/tools/testing/selftests/liveupdate/luo_test_utils.c
index 333a3530051b4..1439b2b48ed0b 100644
--- a/tools/testing/selftests/liveupdate/luo_test_utils.c
+++ b/tools/testing/selftests/liveupdate/luo_test_utils.c
@@ -21,6 +21,7 @@
 #include <sys/stat.h>
 #include <errno.h>
 #include <stdarg.h>
+#include <linux/unistd.h>
 
 #include "luo_test_utils.h"
 
@@ -81,7 +82,7 @@ int luo_retrieve_session(int luo_fd, const char *name)
 int create_and_preserve_memfd(int session_fd, int token, const char *data)
 {
 	struct liveupdate_session_preserve_fd arg = { .size = sizeof(arg) };
-	long page_size = sysconf(_SC_PAGE_SIZE);
+	long page_size = getpagesize();
 	void *map = MAP_FAILED;
 	int mfd = -1, ret = -1;
 
@@ -117,7 +118,7 @@ int restore_and_verify_memfd(int session_fd, int token,
 			     const char *expected_data)
 {
 	struct liveupdate_session_retrieve_fd arg = { .size = sizeof(arg) };
-	long page_size = sysconf(_SC_PAGE_SIZE);
+	long page_size = getpagesize();
 	void *map = MAP_FAILED;
 	int mfd = -1, ret = -1;
 
@@ -228,16 +229,11 @@ void daemonize_and_wait(void)
 
 static int parse_stage_args(int argc, char *argv[])
 {
-	static struct option long_options[] = {
-		{"stage", required_argument, 0, 's'},
-		{0, 0, 0, 0}
-	};
-	int option_index = 0;
 	int stage = 1;
 	int opt;
 
 	optind = 1;
-	while ((opt = getopt_long(argc, argv, "s:", long_options, &option_index)) != -1) {
+	while ((opt = getopt(argc, argv, "s:")) != -1) {
 		switch (opt) {
 		case 's':
 			stage = atoi(optarg);
@@ -248,6 +244,7 @@ static int parse_stage_args(int argc, char *argv[])
 			fail_exit("Unknown argument");
 		}
 	}
+
 	return stage;
 }
 
@@ -275,7 +272,7 @@ int luo_test(int argc, char *argv[],
 		fail_exit("Failed to check for state session");
 
 	if (target_stage != detected_stage) {
-		ksft_exit_fail_msg("Stage mismatch Requested --stage %d, but system is in stage %d.\n"
+		ksft_exit_fail_msg("Stage mismatch Requested stage %d, but system is in stage %d.\n"
 				   "(State session %s: %s)\n",
 				   target_stage, detected_stage, state_session_name,
 				   (detected_stage == 2) ? "EXISTS" : "MISSING");
diff --git a/tools/testing/selftests/liveupdate/run-vmtests.sh b/tools/testing/selftests/liveupdate/run-vmtests.sh
new file mode 100755
index 0000000000000..d656ce58c5a7d
--- /dev/null
+++ b/tools/testing/selftests/liveupdate/run-vmtests.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+SCRIPT_DIR=$(dirname "$(realpath "$0")")
+TEST_RUNNER="$SCRIPT_DIR/vmtest.sh"
+
+TARGETS=("x86_64" "aarch64")
+
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+passed=0
+failed=0
+skipped=0
+
+TEST_NAMES=(
+	"luo_kexec_simple"
+	"luo_multi_session"
+	"luo_stress_files"
+	"luo_stress_sessions"
+)
+
+function usage() {
+	cat <<EOF
+$0 [-k] [-o output_dir] [-h]
+Options:
+	-k)	keep logs
+	-o)	specify output directory
+	-h)	display this help
+EOF
+}
+
+function pass() {
+	echo -e "${GREEN}PASS${NC}"
+	((passed++))
+}
+
+function skip() {
+	echo -e "${YELLOW}SKIP${NC}"
+	((skipped++))
+}
+
+function fail() {
+	echo -e "${RED}FAIL${NC}"
+	((failed++))
+}
+
+function main() {
+	while getopts 'hko:' opt; do
+		case $opt in
+		k) keep_logs=1 ;;
+		o) output_dir=$OPTARG ;;
+		h) usage; exit 0 ;;
+		*) echo Unknown argument "$opt"
+		   usage; exit 1 ;;
+		esac
+	done
+
+	if [ -n "$keep_logs" ]; then
+		if [ -z "$output_dir" ]; then
+			output_dir="$SCRIPT_DIR/results_$(date +%Y%m%d_%H%M%S)"
+		fi;
+		mkdir -p "$output_dir"
+	else
+		output_dir=$(mktemp -d /tmp/luo.XXXXXXXX)
+		trap 'rm -fr "$output_dir"' EXIT
+	fi
+
+	for arch in "${TARGETS[@]}"; do
+		for test_name in "${TEST_NAMES[@]}"; do
+			log="$output_dir/${arch}_${test_name}.log"
+
+			printf "  -> %-8s %-24s ... " "$arch" "$test_name"
+
+			"$TEST_RUNNER" -t "$arch" -T "$test_name" &> "$log"
+			exit_code=$?
+
+			case $exit_code in
+			0) pass;;
+			4) skip;;
+			*) fail;;
+			esac
+		done
+		echo ""
+	done
+
+	echo "SUMMARY: PASS=$passed SKIP=$skipped FAIL=$failed"
+	if [ -n "$keep_logs" ]; then
+		echo "Logs: $output_dir"
+	fi
+
+	exit $((failed != 0))
+}
+
+main "$@"
diff --git a/tools/testing/selftests/liveupdate/vmtest.sh b/tools/testing/selftests/liveupdate/vmtest.sh
new file mode 100755
index 0000000000000..b0000fae14618
--- /dev/null
+++ b/tools/testing/selftests/liveupdate/vmtest.sh
@@ -0,0 +1,263 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+set -ue
+
+CROSS_COMPILE="${CROSS_COMPILE:-""}"
+
+test_dir=$(realpath "$(dirname "$0")")
+kernel_dir=$(realpath "$test_dir/../../../..")
+
+workspace_dir=""
+headers_dir=""
+initrd=""
+KEEP_WORKSPACE=0
+
+source "$test_dir/../kselftest/ktap_helpers.sh"
+
+function get_arch_conf() {
+	local arch=$1
+	if [[ "$arch" == "arm64" ]]; then
+		QEMU_CMD="qemu-system-aarch64 -M virt -cpu max"
+		KERNEL_IMAGE="Image"
+		KERNEL_CMDLINE="console=ttyAMA0"
+	elif [[ "$arch" == "x86" ]]; then
+		QEMU_CMD="qemu-system-x86_64"
+		KERNEL_IMAGE="bzImage"
+		KERNEL_CMDLINE="console=ttyS0"
+	else
+		echo "Unsupported architecture: $arch"
+		exit 1
+	fi
+}
+
+function usage() {
+	cat <<EOF
+$0 [-d build_dir] [-j jobs] [-t target_arch] [-T test_name] [-w workspace_dir] [-k] [-h]
+Options:
+	-d)	path to the kernel build directory (default: .luo_test_build.<arch>)
+	-j)	number of jobs for compilation
+	-t)	run test for target_arch (aarch64, x86_64)
+	-T)	test name to run (default: luo_kexec_simple)
+	-w)	custom workspace directory (default: creates temp dir)
+	-k)	keep workspace directory after successful test
+	-h)	display this help
+EOF
+}
+
+function cleanup() {
+	if [ "$KEEP_WORKSPACE" -eq 1 ]; then
+		echo "# Workspace preserved at: $workspace_dir"
+	else
+		rm -fr "$workspace_dir"
+	fi
+
+	ktap_finished
+}
+
+function skip() {
+	local msg=${1:-""}
+	ktap_test_skip "$msg"
+	exit "$KSFT_SKIP"
+}
+
+function fail() {
+	local msg=${1:-""}
+	ktap_test_fail "$msg"
+	exit "$KSFT_FAIL"
+}
+
+function detect_cross_compile() {
+	local target=$1
+	local host=$(uname -m)
+
+	[[ "$host" == "arm64" ]] && host="aarch64"
+	[[ "$target" == "arm64" ]] && target="aarch64"
+
+	if [[ "$host" == "$target" ]]; then
+		CROSS_COMPILE=""
+		return
+	fi
+
+	if [[ -n "$CROSS_COMPILE" ]]; then
+		return
+	fi
+
+	local candidate=""
+	case "$target" in
+		aarch64) candidate="aarch64-linux-gnu-" ;;
+		x86_64)  candidate="x86_64-linux-gnu-" ;;
+		*)       skip "Auto-detection for target '$target' not supported. Please set CROSS_COMPILE manually." ;;
+	esac
+
+	if command -v "${candidate}gcc" &> /dev/null; then
+		CROSS_COMPILE="$candidate"
+	else
+		skip "Compiler '${candidate}gcc' not found. Please install it (e.g., 'apt install gcc-aarch64-linux-gnu') or set CROSS_COMPILE."
+	fi
+}
+
+function build_kernel() {
+	local build_dir=$1
+	local make_cmd=$2
+	local kimage=$3
+	local target_arch=$4
+
+	local luo_config="$build_dir/luo.config"
+	local kconfig="$build_dir/.config"
+	local common_conf="$test_dir/config"
+	local arch_conf="$test_dir/config.$target_arch"
+
+	echo "# Building kernel in: $build_dir"
+
+	cat "$arch_conf" "$common_conf" | tee "$kconfig" > "$luo_config"
+	$make_cmd olddefconfig
+
+	# verify that kernel confiration has all necessary options
+	while read -r opt ; do
+		grep "$opt" "$kconfig" &>/dev/null || skip "$opt is missing"
+	done < "$luo_config"
+
+	$make_cmd "$kimage"
+	$make_cmd headers_install INSTALL_HDR_PATH="$headers_dir"
+}
+
+function mkinitrd() {
+	local build_dir=$1
+	local kernel_path=$2
+	local test_name=$3
+
+	# Compile the test binary and the init process
+	"$CROSS_COMPILE"gcc -static -O2 -nostdinc -nostdlib \
+		-I "$headers_dir/include" \
+		-I "$kernel_dir/tools/include/nolibc" \
+		-I "$test_dir" \
+		-o "$workspace_dir/test_binary" \
+		"$test_dir/$test_name.c" "$test_dir/luo_test_utils.c"
+
+	"$CROSS_COMPILE"gcc -s -static -Os -nostdinc -nostdlib		\
+			-fno-asynchronous-unwind-tables -fno-ident	\
+			-fno-stack-protector				\
+			-I "$headers_dir/include"			\
+			-I "$kernel_dir/tools/include/nolibc"		\
+			-o "$workspace_dir/init" "$test_dir/init.c"
+
+	cat > "$workspace_dir/cpio_list_inner" <<EOF
+dir /dev 0755 0 0
+dir /proc 0755 0 0
+dir /debugfs 0755 0 0
+nod /dev/console 0600 0 0 c 5 1
+file /init $workspace_dir/init 0755 0 0
+file /test_binary $workspace_dir/test_binary 0755 0 0
+EOF
+
+	# Generate inner_initrd.cpio
+	"$build_dir/usr/gen_init_cpio" "$workspace_dir/cpio_list_inner" > "$workspace_dir/inner_initrd.cpio"
+
+	cat > "$workspace_dir/cpio_list" <<EOF
+dir /dev 0755 0 0
+dir /proc 0755 0 0
+dir /debugfs 0755 0 0
+nod /dev/console 0600 0 0 c 5 1
+file /init $workspace_dir/init 0755 0 0
+file /kernel $kernel_path 0644 0 0
+file /test_binary $workspace_dir/test_binary 0755 0 0
+file /initrd.img $workspace_dir/inner_initrd.cpio 0644 0 0
+EOF
+
+	# Generate the final initrd
+	"$build_dir/usr/gen_init_cpio" "$workspace_dir/cpio_list" > "$initrd"
+}
+
+function run_qemu() {
+	local qemu_cmd=$1
+	local cmdline=$2
+	local kernel_path=$3
+	local serial="$workspace_dir/qemu.serial"
+
+	cmdline="$cmdline liveupdate=on panic=-1"
+
+	echo "# Serial Log: $serial"
+	timeout 30s \
+	$qemu_cmd -m 1G -smp 2 -no-reboot -nographic -nodefaults \
+		  -accel tcg -accel hvf -accel kvm \
+		  -serial file:"$serial" \
+		  -append "$cmdline" \
+		  -kernel "$kernel_path" \
+		  -initrd "$initrd"
+
+	grep "TEST PASSED" "$serial" &> /dev/null || fail "Liveupdate failed"
+}
+
+function target_to_arch() {
+	local target=$1
+	case $target in
+	     aarch64) echo "arm64" ;;
+	     x86_64) echo "x86" ;;
+	     *) skip "architecture $target is not supported"
+	esac
+}
+
+function main() {
+	local build_dir=""
+	local jobs=$(nproc)
+	local target="$(uname -m)"
+	local test_name="luo_kexec_simple"
+	local workspace_arg=""
+
+	set -o errtrace
+	trap fail ERR
+
+	while getopts 'hd:j:t:T:w:k' opt; do
+		case $opt in
+		d) build_dir="$OPTARG" ;;
+		j) jobs="$OPTARG" ;;
+		t) target="$OPTARG" ;;
+		T) test_name="$OPTARG" ;;
+		w) workspace_arg="$OPTARG" ;;
+		k) KEEP_WORKSPACE=1 ;;
+		h) usage; exit 0 ;;
+		*) echo "Unknown argument $opt"; usage; exit 1 ;;
+		esac
+	done
+
+	ktap_print_header
+	ktap_set_plan 1
+	trap cleanup EXIT
+
+	if [ -n "$workspace_arg" ]; then
+		workspace_dir="$(realpath -m "$workspace_arg")"
+		mkdir -p "$workspace_dir"
+	else
+		workspace_dir=$(mktemp -d /tmp/luo-test.XXXXXXXX)
+	fi
+
+	echo "# Workspace created at: $workspace_dir"
+	headers_dir="$workspace_dir/usr"
+	initrd="$workspace_dir/initrd.cpio"
+
+	detect_cross_compile "$target"
+
+	local arch=$(target_to_arch "$target")
+
+	if [ -z "$build_dir" ]; then
+		build_dir="$kernel_dir/.luo_test_build.$arch"
+	fi
+
+	mkdir -p "$build_dir"
+	build_dir=$(realpath "$build_dir")
+	get_arch_conf "$arch"
+
+	local make_cmd="make -s ARCH=$arch CROSS_COMPILE=$CROSS_COMPILE -j$jobs"
+	local make_cmd_build="$make_cmd -C $kernel_dir O=$build_dir"
+
+	build_kernel "$build_dir" "$make_cmd_build" "$KERNEL_IMAGE" "$target"
+
+	local final_kernel="$build_dir/arch/$arch/boot/$KERNEL_IMAGE"
+	mkinitrd "$build_dir" "$final_kernel" "$test_name"
+
+	run_qemu "$QEMU_CMD" "$KERNEL_CMDLINE" "$final_kernel"
+	ktap_test_pass "$test_name succeeded"
+}
+
+main "$@"
diff --git a/tools/testing/selftests/lkdtm/tests.txt b/tools/testing/selftests/lkdtm/tests.txt
index d8180bbe31e8b..bec57a02913ad 100644
--- a/tools/testing/selftests/lkdtm/tests.txt
+++ b/tools/testing/selftests/lkdtm/tests.txt
@@ -78,7 +78,7 @@ USERCOPY_STACK_FRAME_TO
 USERCOPY_STACK_FRAME_FROM
 USERCOPY_STACK_BEYOND
 USERCOPY_KERNEL
-STACKLEAK_ERASING OK: the rest of the thread stack is properly erased
+KSTACK_ERASE OK: the rest of the thread stack is properly erased
 CFI_FORWARD_PROTO
 CFI_BACKWARD call trace:|ok: control flow unchanged
 FORTIFY_STRSCPY detected buffer overflow
diff --git a/tools/testing/selftests/lsm/common.c b/tools/testing/selftests/lsm/common.c
index 9ad258912646c..927dce4f04cb2 100644
--- a/tools/testing/selftests/lsm/common.c
+++ b/tools/testing/selftests/lsm/common.c
@@ -76,7 +76,7 @@ int attr_lsm_count(void)
 		return 0;
 
 	if (read_sysfs_lsms(names, sysconf(_SC_PAGESIZE)))
-		return 0;
+		goto out;
 
 	if (strstr(names, "selinux"))
 		count++;
@@ -85,5 +85,7 @@ int attr_lsm_count(void)
 	if (strstr(names, "apparmor"))
 		count++;
 
+out:
+	free(names);
 	return count;
 }
diff --git a/tools/testing/selftests/mincore/mincore_selftest.c b/tools/testing/selftests/mincore/mincore_selftest.c
index cdd022c1c4974..18dfbf318b1f4 100644
--- a/tools/testing/selftests/mincore/mincore_selftest.c
+++ b/tools/testing/selftests/mincore/mincore_selftest.c
@@ -242,8 +242,10 @@ TEST(check_file_mmap)
 	}
 
 	/*
-	 * Touch a page in the middle of the mapping. We expect the next
-	 * few pages (the readahead window) to be populated too.
+	 * Touch a page in the middle of the mapping. We expect some
+	 * surrounding pages (the readahead window) to be populated too.
+	 * Depending on the page size and readahead setting, the pages may
+	 * land before the faulted page rather than after it.
 	 */
 	addr[FILE_SIZE / 2] = 1;
 	retval = mincore(addr, FILE_SIZE, vec);
@@ -252,6 +254,12 @@ TEST(check_file_mmap)
 		TH_LOG("Page not found in memory after use");
 	}
 
+	i = FILE_SIZE / 2 / page_size - 1;
+	while (i >= 0 && vec[i]) {
+		ra_pages++;
+		i--;
+	}
+
 	i = FILE_SIZE / 2 / page_size + 1;
 	while (i < vec_size && vec[i]) {
 		ra_pages++;
diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile
index e6df968f0971c..eae504bd94c83 100644
--- a/tools/testing/selftests/mm/Makefile
+++ b/tools/testing/selftests/mm/Makefile
@@ -66,7 +66,6 @@ TEST_GEN_FILES += hugetlb-mremap
 TEST_GEN_FILES += hugetlb-read-hwpoison
 TEST_GEN_FILES += hugetlb-shm
 TEST_GEN_FILES += hugetlb-soft-offline
-TEST_GEN_FILES += hugetlb-vmemmap
 TEST_GEN_FILES += khugepaged
 TEST_GEN_FILES += madv_populate
 TEST_GEN_FILES += map_fixed_noreplace
@@ -150,6 +149,7 @@ TEST_PROGS += ksft_gup_test.sh
 TEST_PROGS += ksft_hmm.sh
 TEST_PROGS += ksft_hugetlb.sh
 TEST_PROGS += ksft_hugevm.sh
+TEST_PROGS += ksft_kmemleak_confirm.sh
 TEST_PROGS += ksft_kmemleak_dedup.sh
 TEST_PROGS += ksft_ksm.sh
 TEST_PROGS += ksft_ksm_numa.sh
@@ -174,6 +174,10 @@ TEST_PROGS += ksft_userfaultfd.sh
 TEST_PROGS += ksft_vma_merge.sh
 TEST_PROGS += ksft_vmalloc.sh
 
+# Destructive: every successful run panics the kernel.  Installed and
+# kept executable, but not run from a default kselftest invocation.
+TEST_PROGS_EXTENDED += hwpoison-panic.sh
+
 TEST_FILES := test_vmalloc.sh
 TEST_FILES += test_hmm.sh
 TEST_FILES += va_high_addr_switch.sh
diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c
index 2f2b9879d1005..e2642eca0d02b 100644
--- a/tools/testing/selftests/mm/hmm-tests.c
+++ b/tools/testing/selftests/mm/hmm-tests.c
@@ -29,6 +29,10 @@
 #include <sys/mman.h>
 #include <sys/ioctl.h>
 #include <sys/time.h>
+#include <sys/syscall.h>
+#include <sys/eventfd.h>
+#include <linux/userfaultfd.h>
+#include <poll.h>
 
 /*
  * This is a private UAPI to the kernel test module so it isn't exported
@@ -224,6 +228,53 @@ static void hmm_buffer_free(struct hmm_buffer *buffer)
 	free(buffer);
 }
 
+/*
+ * Allocate a buffer structure with memory mapping and mirror.
+ *
+ * @mmap_size:   total size of the mmap region (may differ from @mirror_size
+ *               for alignment padding in THP tests).
+ * @mirror_size: size of the mirror data buffer (the actual working set).
+ * @prot:        protection flags for the mmap (e.g. PROT_READ | PROT_WRITE).
+ * @flags:       flags for the mmap (e.g. MAP_PRIVATE, MAP_SHARED,
+ *               MAP_ANONYMOUS, MAP_HUGETLB).
+ * @fd:          file descriptor for the mmap; pass -1 for MAP_ANONYMOUS.
+ *
+ * All internal allocations are checked; returns NULL and cleans up on any
+ * failure. Caller must ASSERT_NE or otherwise check the return value.
+ */
+static struct hmm_buffer *hmm_buffer_alloc(unsigned long mmap_size,
+						unsigned long mirror_size,
+						int prot, int flags,
+						int fd)
+{
+	struct hmm_buffer *buffer;
+
+	buffer = malloc(sizeof(*buffer));
+	if (!buffer) {
+		perror("malloc buffer");
+		return NULL;
+	}
+
+	buffer->fd = fd;
+	buffer->size = mmap_size;
+	buffer->mirror = malloc(mirror_size);
+	if (!buffer->mirror) {
+		perror("malloc mirror");
+		free(buffer);
+		return NULL;
+	}
+
+	buffer->ptr = mmap(NULL, mmap_size, prot, flags, fd, 0);
+	if (buffer->ptr == MAP_FAILED) {
+		perror("mmap");
+		free(buffer->mirror);
+		free(buffer);
+		return NULL;
+	}
+
+	return buffer;
+}
+
 /*
  * Create a temporary file that will be deleted on close.
  */
@@ -314,20 +365,11 @@ TEST_F(hmm, anon_read)
 	ASSERT_NE(npages, 0);
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/*
 	 * Initialize buffer in system memory but leave the first two pages
 	 * zero (pte_none and pfn_zero).
@@ -377,20 +419,11 @@ TEST_F(hmm, anon_read_prot)
 	ASSERT_NE(npages, 0);
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Initialize buffer in system memory. */
 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 		ptr[i] = i;
@@ -436,20 +469,11 @@ TEST_F(hmm, anon_write)
 	ASSERT_NE(npages, 0);
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Initialize data that the device will write to buffer->ptr. */
 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 		ptr[i] = i;
@@ -484,20 +508,11 @@ TEST_F(hmm, anon_write_prot)
 	ASSERT_NE(npages, 0);
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Simulate a device reading a zero page of memory. */
 	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1);
 	ASSERT_EQ(ret, 0);
@@ -559,20 +574,10 @@ TEST_F(hmm, anon_write_child)
 			ASSERT_NE(npages, 0);
 			size = npages << self->page_shift;
 
-			buffer = malloc(sizeof(*buffer));
+			buffer = hmm_buffer_alloc(size * 2, size,
+						  PROT_READ | PROT_WRITE,
+						  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 			ASSERT_NE(buffer, NULL);
-
-			buffer->fd = -1;
-			buffer->size = size * 2;
-			buffer->mirror = malloc(size);
-			ASSERT_NE(buffer->mirror, NULL);
-
-			buffer->ptr = mmap(NULL, size * 2,
-					   PROT_READ | PROT_WRITE,
-					   MAP_PRIVATE | MAP_ANONYMOUS,
-					   buffer->fd, 0);
-			ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 			old_ptr = buffer->ptr;
 			if (use_thp) {
 				map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
@@ -661,20 +666,11 @@ TEST_F(hmm, anon_write_child_shared)
 	ASSERT_NE(npages, 0);
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_SHARED | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_SHARED | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Initialize buffer->ptr so we can tell if it is written. */
 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 		ptr[i] = i;
@@ -736,20 +732,10 @@ TEST_F(hmm, anon_write_huge)
 
 	size = 2 * read_pmd_pagesize();
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
-
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	size /= 2;
 	npages = size >> self->page_shift;
 	map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
@@ -795,22 +781,11 @@ TEST_F(hmm, anon_write_hugetlbfs)
 	size = ALIGN(TWOMEG, default_hsize);
 	npages = size >> self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
-	ASSERT_NE(buffer, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-				   PROT_READ | PROT_WRITE,
-				   MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
-				   -1, 0);
-	if (buffer->ptr == MAP_FAILED) {
-		free(buffer);
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1);
+	if (!buffer)
 		SKIP(return, "Huge page could not be allocated");
-	}
-
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
 
 	/* Initialize data that the device will write to buffer->ptr. */
 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
@@ -852,14 +827,11 @@ TEST_F(hmm, file_read)
 	fd = hmm_create_file(size);
 	ASSERT_GE(fd, 0);
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ,
+				  MAP_SHARED, fd);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = fd;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
 	/* Write initial contents of the file. */
 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 		ptr[i] = i;
@@ -867,12 +839,6 @@ TEST_F(hmm, file_read)
 	ASSERT_EQ(len, size);
 	memset(buffer->mirror, 0, size);
 
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ,
-			   MAP_SHARED,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Simulate a device reading system memory. */
 	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);
 	ASSERT_EQ(ret, 0);
@@ -907,20 +873,11 @@ TEST_F(hmm, file_write)
 	fd = hmm_create_file(size);
 	ASSERT_GE(fd, 0);
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_SHARED, fd);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = fd;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_SHARED,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Initialize data that the device will write to buffer->ptr. */
 	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 		ptr[i] = i;
@@ -960,20 +917,11 @@ TEST_F(hmm, migrate)
 	ASSERT_NE(npages, 0);
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Initialize buffer in system memory. */
 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 		ptr[i] = i;
@@ -1010,20 +958,11 @@ TEST_F(hmm, migrate_file_private)
 	fd = hmm_create_file(size);
 	ASSERT_GE(fd, 0);
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE, fd);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = fd;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_PRIVATE,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Initialize buffer in system memory. */
 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 		ptr[i] = i;
@@ -1058,20 +997,11 @@ TEST_F(hmm, migrate_fault)
 	ASSERT_NE(npages, 0);
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Initialize buffer in system memory. */
 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 		ptr[i] = i;
@@ -1114,18 +1044,11 @@ TEST_F(hmm, migrate_release)
 	ASSERT_NE(npages, 0);
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,
-			   MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Initialize buffer in system memory. */
 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 		ptr[i] = i;
@@ -1164,20 +1087,11 @@ TEST_F(hmm, migrate_shared)
 	ASSERT_NE(npages, 0);
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_SHARED | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_SHARED | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Migrate memory to device. */
 	ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
 	ASSERT_EQ(ret, -ENOENT);
@@ -1201,20 +1115,10 @@ TEST_F(hmm2, migrate_mixed)
 	npages = 6;
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_NONE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
-
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	/* Reserve a range of addresses. */
-	buffer->ptr = mmap(NULL, size,
-			   PROT_NONE,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
 	p = buffer->ptr;
 
 	/* Migrating a protected area should be an error. */
@@ -1295,20 +1199,11 @@ TEST_F(hmm, migrate_multiple)
 	size = npages << self->page_shift;
 
 	for (c = 0; c < NTIMES; c++) {
-		buffer = malloc(sizeof(*buffer));
+		buffer = hmm_buffer_alloc(size, size,
+					  PROT_READ | PROT_WRITE,
+					  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 		ASSERT_NE(buffer, NULL);
 
-		buffer->fd = -1;
-		buffer->size = size;
-		buffer->mirror = malloc(size);
-		ASSERT_NE(buffer->mirror, NULL);
-
-		buffer->ptr = mmap(NULL, size,
-				   PROT_READ | PROT_WRITE,
-				   MAP_PRIVATE | MAP_ANONYMOUS,
-				   buffer->fd, 0);
-		ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 		/* Initialize buffer in system memory. */
 		for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 			ptr[i] = i;
@@ -1354,20 +1249,11 @@ TEST_F(hmm, anon_read_multiple)
 	size = npages << self->page_shift;
 
 	for (c = 0; c < NTIMES; c++) {
-		buffer = malloc(sizeof(*buffer));
+		buffer = hmm_buffer_alloc(size, size,
+					  PROT_READ | PROT_WRITE,
+					  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 		ASSERT_NE(buffer, NULL);
 
-		buffer->fd = -1;
-		buffer->size = size;
-		buffer->mirror = malloc(size);
-		ASSERT_NE(buffer->mirror, NULL);
-
-		buffer->ptr = mmap(NULL, size,
-				   PROT_READ | PROT_WRITE,
-				   MAP_PRIVATE | MAP_ANONYMOUS,
-				   buffer->fd, 0);
-		ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 		/* Initialize buffer in system memory. */
 		for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 			ptr[i] = i + c;
@@ -1420,20 +1306,11 @@ TEST_F(hmm, anon_teardown)
 		int *ptr;
 		int rc;
 
-		buffer = malloc(sizeof(*buffer));
+		buffer = hmm_buffer_alloc(size, size,
+					  PROT_READ | PROT_WRITE,
+					  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 		ASSERT_NE(buffer, NULL);
 
-		buffer->fd = -1;
-		buffer->size = size;
-		buffer->mirror = malloc(size);
-		ASSERT_NE(buffer->mirror, NULL);
-
-		buffer->ptr = mmap(NULL, size,
-				   PROT_READ | PROT_WRITE,
-				   MAP_PRIVATE | MAP_ANONYMOUS,
-				   buffer->fd, 0);
-		ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 		/* Initialize buffer in system memory. */
 		for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 			ptr[i] = i + c;
@@ -1474,22 +1351,11 @@ TEST_F(hmm, mixedmap)
 	npages = 1;
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, npages,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE, self->fd);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(npages);
-	ASSERT_NE(buffer->mirror, NULL);
-
-
-	/* Reserve a range of addresses. */
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_PRIVATE,
-			   self->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Simulate a device snapshotting CPU pagetables. */
 	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
 	ASSERT_EQ(ret, 0);
@@ -1519,20 +1385,10 @@ TEST_F(hmm2, snapshot)
 	npages = 7;
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, npages,
+				  PROT_NONE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
-
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(npages);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	/* Reserve a range of addresses. */
-	buffer->ptr = mmap(NULL, size,
-			   PROT_NONE,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
 	p = buffer->ptr;
 
 	/* Punch a hole after the first page address. */
@@ -1626,21 +1482,11 @@ TEST_F(hmm, compound)
 	size = ALIGN(TWOMEG, default_hsize);
 	npages = size >> self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
-	ASSERT_NE(buffer, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-				   PROT_READ | PROT_WRITE,
-				   MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
-				   -1, 0);
-	if (buffer->ptr == MAP_FAILED) {
-		free(buffer);
-		return;
-	}
-
-	buffer->size = size;
-	buffer->mirror = malloc(npages);
-	ASSERT_NE(buffer->mirror, NULL);
+	buffer = hmm_buffer_alloc(size, npages,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1);
+	if (!buffer)
+		SKIP(return, "Huge page could not be allocated");
 
 	/* Initialize the pages the device will snapshot in buffer->ptr. */
 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
@@ -1707,21 +1553,11 @@ TEST_F(hmm2, double_map)
 	npages = 6;
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	/* Reserve a range of addresses. */
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Initialize buffer in system memory. */
 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 		ptr[i] = i;
@@ -1783,20 +1619,11 @@ TEST_F(hmm, exclusive)
 	ASSERT_NE(npages, 0);
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Initialize buffer in system memory. */
 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 		ptr[i] = i;
@@ -1837,20 +1664,11 @@ TEST_F(hmm, exclusive_mprotect)
 	ASSERT_NE(npages, 0);
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Initialize buffer in system memory. */
 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 		ptr[i] = i;
@@ -1892,20 +1710,11 @@ TEST_F(hmm, exclusive_cow)
 	ASSERT_NE(npages, 0);
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Initialize buffer in system memory. */
 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 		ptr[i] = i;
@@ -1993,20 +1802,11 @@ TEST_F(hmm, hmm_gup_test)
 	npages = 4;
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Initialize buffer in system memory. */
 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 		ptr[i] = i;
@@ -2079,20 +1879,11 @@ TEST_F(hmm, hmm_cow_in_device)
 	npages = 4;
 	size = npages << self->page_shift;
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
 
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
-
-	buffer->ptr = mmap(NULL, size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	/* Initialize buffer in system memory. */
 	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 		ptr[i] = i;
@@ -2152,21 +1943,12 @@ TEST_F(hmm, migrate_anon_huge_empty)
 
 	size = read_pmd_pagesize();
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(2 * size, size,
+				  PROT_READ,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
-
-	buffer->fd = -1;
-	buffer->size = 2 * size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
 	memset(buffer->mirror, 0xFF, size);
 
-	buffer->ptr = mmap(NULL, 2 * size,
-			   PROT_READ,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	npages = size >> self->page_shift;
 	map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
 	ret = madvise(map, size, MADV_HUGEPAGE);
@@ -2204,21 +1986,12 @@ TEST_F(hmm, migrate_anon_huge_zero)
 
 	size = read_pmd_pagesize();
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(2 * size, size,
+				  PROT_READ,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
-
-	buffer->fd = -1;
-	buffer->size = 2 * size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
 	memset(buffer->mirror, 0xFF, size);
 
-	buffer->ptr = mmap(NULL, 2 * size,
-			   PROT_READ,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	npages = size >> self->page_shift;
 	map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
 	ret = madvise(map, size, MADV_HUGEPAGE);
@@ -2267,21 +2040,12 @@ TEST_F(hmm, migrate_anon_huge_free)
 
 	size = read_pmd_pagesize();
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(2 * size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
-
-	buffer->fd = -1;
-	buffer->size = 2 * size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
 	memset(buffer->mirror, 0xFF, size);
 
-	buffer->ptr = mmap(NULL, 2 * size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	npages = size >> self->page_shift;
 	map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
 	ret = madvise(map, size, MADV_HUGEPAGE);
@@ -2329,21 +2093,12 @@ TEST_F(hmm, migrate_anon_huge_fault)
 
 	size = read_pmd_pagesize();
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(2 * size, size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
-
-	buffer->fd = -1;
-	buffer->size = 2 * size;
-	buffer->mirror = malloc(size);
-	ASSERT_NE(buffer->mirror, NULL);
 	memset(buffer->mirror, 0xFF, size);
 
-	buffer->ptr = mmap(NULL, 2 * size,
-			   PROT_READ | PROT_WRITE,
-			   MAP_PRIVATE | MAP_ANONYMOUS,
-			   buffer->fd, 0);
-	ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 	npages = size >> self->page_shift;
 	map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
 	old_ptr = buffer->ptr;
@@ -2425,21 +2180,11 @@ TEST_F(hmm, migrate_partial_unmap_fault)
 
 	for (use_thp = 0; use_thp < 2; ++use_thp) {
 		for (j = 0; j < ARRAY_SIZE(offsets); ++j) {
-			buffer = malloc(sizeof(*buffer));
+			buffer = hmm_buffer_alloc(2 * size, size,
+						  PROT_READ | PROT_WRITE,
+						  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 			ASSERT_NE(buffer, NULL);
-
-			buffer->fd = -1;
-			buffer->size = 2 * size;
-			buffer->mirror = malloc(size);
-			ASSERT_NE(buffer->mirror, NULL);
 			memset(buffer->mirror, 0xFF, size);
-
-			buffer->ptr = mmap(NULL, 2 * size,
-					   PROT_READ | PROT_WRITE,
-					   MAP_PRIVATE | MAP_ANONYMOUS,
-					   buffer->fd, 0);
-			ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 			npages = size >> self->page_shift;
 			map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
 			if (use_thp)
@@ -2505,21 +2250,11 @@ TEST_F(hmm, migrate_remap_fault)
 					if (dont_unmap)
 						flags |= MREMAP_DONTUNMAP;
 
-					buffer = malloc(sizeof(*buffer));
+					buffer = hmm_buffer_alloc(8 * size, size,
+								  PROT_READ | PROT_WRITE,
+								  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 					ASSERT_NE(buffer, NULL);
-
-					buffer->fd = -1;
-					buffer->size = 8 * size;
-					buffer->mirror = malloc(size);
-					ASSERT_NE(buffer->mirror, NULL);
 					memset(buffer->mirror, 0xFF, size);
-
-					buffer->ptr = mmap(NULL, buffer->size,
-							   PROT_READ | PROT_WRITE,
-							   MAP_PRIVATE | MAP_ANONYMOUS,
-							   buffer->fd, 0);
-					ASSERT_NE(buffer->ptr, MAP_FAILED);
-
 					npages = size >> self->page_shift;
 					map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
 					if (use_thp)
@@ -2590,18 +2325,13 @@ TEST_F(hmm, migrate_anon_huge_err)
 
 	size = read_pmd_pagesize();
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(2 * size, 2 * size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
-
-	buffer->fd = -1;
-	buffer->size = 2 * size;
-	buffer->mirror = malloc(2 * size);
-	ASSERT_NE(buffer->mirror, NULL);
 	memset(buffer->mirror, 0xFF, 2 * size);
 
-	old_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE,
-			MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0);
-	ASSERT_NE(old_ptr, MAP_FAILED);
+	old_ptr = buffer->ptr;
 
 	npages = size >> self->page_shift;
 	map = (void *)ALIGN((uintptr_t)old_ptr, size);
@@ -2685,18 +2415,13 @@ TEST_F(hmm, migrate_anon_huge_zero_err)
 
 	size = read_pmd_pagesize();
 
-	buffer = malloc(sizeof(*buffer));
+	buffer = hmm_buffer_alloc(2 * size, 2 * size,
+				  PROT_READ,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
 	ASSERT_NE(buffer, NULL);
-
-	buffer->fd = -1;
-	buffer->size = 2 * size;
-	buffer->mirror = malloc(2 * size);
-	ASSERT_NE(buffer->mirror, NULL);
 	memset(buffer->mirror, 0xFF, 2 * size);
 
-	old_ptr = mmap(NULL, 2 * size, PROT_READ,
-			MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0);
-	ASSERT_NE(old_ptr, MAP_FAILED);
+	old_ptr = buffer->ptr;
 
 	npages = size >> self->page_shift;
 	map = (void *)ALIGN((uintptr_t)old_ptr, size);
@@ -2766,19 +2491,6 @@ static double get_time_ms(void)
 	return (tv.tv_sec * 1000.0) + (tv.tv_usec / 1000.0);
 }
 
-static inline struct hmm_buffer *hmm_buffer_alloc(unsigned long size)
-{
-	struct hmm_buffer *buffer;
-
-	buffer = malloc(sizeof(*buffer));
-
-	buffer->fd = -1;
-	buffer->size = size;
-	buffer->mirror = malloc(size);
-	memset(buffer->mirror, 0xFF, size);
-	return buffer;
-}
-
 static void print_benchmark_results(const char *test_name, size_t buffer_size,
 				     struct benchmark_results *thp,
 				     struct benchmark_results *regular)
@@ -2823,14 +2535,14 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz
 	int ret, i;
 	int *ptr;
 
-	buffer = hmm_buffer_alloc(buffer_size);
-
-	/* Map memory */
-	buffer->ptr = mmap(NULL, buffer_size, PROT_READ | PROT_WRITE,
-			  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-
-	if (buffer->ptr == MAP_FAILED)
-		return -1;
+	buffer = hmm_buffer_alloc(buffer_size, buffer_size,
+				  PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE | MAP_ANONYMOUS, -1);
+	if (!buffer) {
+		ret = -1;
+		goto cleanup;
+	}
+	memset(buffer->mirror, 0xFF, buffer_size);
 
 	/* Apply THP hint if requested */
 	if (use_thp)
@@ -2839,7 +2551,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz
 		ret = madvise(buffer->ptr, buffer_size, MADV_NOHUGEPAGE);
 
 	if (ret)
-		return ret;
+		goto cleanup;
 
 	/* Initialize memory to make sure pages are allocated */
 	ptr = (int *)buffer->ptr;
@@ -2849,11 +2561,11 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz
 	/* Warmup iteration */
 	ret = hmm_migrate_sys_to_dev(fd, buffer, npages);
 	if (ret)
-		return ret;
+		goto cleanup;
 
 	ret = hmm_migrate_dev_to_sys(fd, buffer, npages);
 	if (ret)
-		return ret;
+		goto cleanup;
 
 	/* Benchmark iterations */
 	for (i = 0; i < iterations; i++) {
@@ -2862,7 +2574,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz
 
 		ret = hmm_migrate_sys_to_dev(fd, buffer, npages);
 		if (ret)
-			return ret;
+			goto cleanup;
 
 		end = get_time_ms();
 		s2d_total += (end - start);
@@ -2872,7 +2584,7 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz
 
 		ret = hmm_migrate_dev_to_sys(fd, buffer, npages);
 		if (ret)
-			return ret;
+			goto cleanup;
 
 		end = get_time_ms();
 		d2s_total += (end - start);
@@ -2886,9 +2598,9 @@ static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_siz
 	results->throughput_d2s = (buffer_size / (1024.0 * 1024.0 * 1024.0)) /
 				 (results->dev_to_sys_time / 1000.0);
 
-	/* Cleanup */
+cleanup:
 	hmm_buffer_free(buffer);
-	return 0;
+	return ret;
 }
 
 /*
@@ -2949,4 +2661,150 @@ TEST_F_TIMEOUT(hmm, benchmark_thp_migration, 120)
 					&thp_results, &regular_results);
 	}
 }
+/*
+ * Test that HMM can fault in pages backed by userfaultfd using the
+ * hmm_range_fault_unlocked_timeout() path with no timeout. This exercises
+ * the lock-drop retry logic in the HMM framework.
+ */
+struct uffd_thread_args {
+	int uffd;
+	int stop_fd;
+	void *page_buffer;
+	unsigned long page_size;
+};
+
+static void *uffd_handler_thread(void *arg)
+{
+	struct uffd_thread_args *args = arg;
+	struct uffd_msg msg;
+	struct uffdio_copy copy;
+	struct pollfd pollfd[2];
+	int ret;
+
+	pollfd[0].fd = args->uffd;
+	pollfd[0].events = POLLIN;
+	pollfd[1].fd = args->stop_fd;
+	pollfd[1].events = POLLIN;
+
+	while (1) {
+		ret = poll(pollfd, 2, -1);
+		if (ret <= 0)
+			break;
+		if (pollfd[1].revents)
+			break;
+		if (!(pollfd[0].revents & POLLIN))
+			break;
+
+		ret = read(args->uffd, &msg, sizeof(msg));
+		if (ret != sizeof(msg))
+			break;
+
+		if (msg.event != UFFD_EVENT_PAGEFAULT)
+			break;
+
+		/* Fill the page with a known pattern */
+		memset(args->page_buffer, 0xAB, args->page_size);
+
+		copy.dst = msg.arg.pagefault.address & ~(args->page_size - 1);
+		copy.src = (unsigned long)args->page_buffer;
+		copy.len = args->page_size;
+		copy.mode = 0;
+		copy.copy = 0;
+
+		ret = ioctl(args->uffd, UFFDIO_COPY, &copy);
+		if (ret < 0)
+			break;
+	}
+
+	return NULL;
+}
+
+TEST_F(hmm, userfaultfd_read)
+{
+	struct hmm_buffer *buffer;
+	struct uffd_thread_args uffd_args;
+	unsigned long npages;
+	unsigned long size;
+	unsigned long i;
+	unsigned char *ptr;
+	pthread_t thread;
+	int uffd;
+	int stop_fd;
+	int ret;
+	struct uffdio_api api;
+	struct uffdio_register reg;
+	uint64_t stop = 1;
+	ssize_t nwrite;
+
+	npages = 4;
+	size = npages << self->page_shift;
+
+	/* Create userfaultfd */
+	uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
+	if (uffd < 0)
+		SKIP(return, "userfaultfd not available");
+
+	api.api = UFFD_API;
+	api.features = 0;
+	ret = ioctl(uffd, UFFDIO_API, &api);
+	ASSERT_EQ(ret, 0);
+
+	buffer = malloc(sizeof(*buffer));
+	ASSERT_NE(buffer, NULL);
+
+	buffer->fd = -1;
+	buffer->size = size;
+	buffer->mirror = malloc(size);
+	ASSERT_NE(buffer->mirror, NULL);
+
+	/* Create anonymous mapping */
+	buffer->ptr = mmap(NULL, size,
+			   PROT_READ | PROT_WRITE,
+			   MAP_PRIVATE | MAP_ANONYMOUS,
+			   -1, 0);
+	ASSERT_NE(buffer->ptr, MAP_FAILED);
+
+	/* Register the region with userfaultfd */
+	reg.range.start = (unsigned long)buffer->ptr;
+	reg.range.len = size;
+	reg.mode = UFFDIO_REGISTER_MODE_MISSING;
+	ret = ioctl(uffd, UFFDIO_REGISTER, &reg);
+	ASSERT_EQ(ret, 0);
+
+	/* Set up the handler thread */
+	uffd_args.uffd = uffd;
+	stop_fd = eventfd(0, EFD_CLOEXEC);
+	ASSERT_GE(stop_fd, 0);
+	uffd_args.stop_fd = stop_fd;
+	uffd_args.page_buffer = malloc(self->page_size);
+	ASSERT_NE(uffd_args.page_buffer, NULL);
+	uffd_args.page_size = self->page_size;
+
+	ret = pthread_create(&thread, NULL, uffd_handler_thread, &uffd_args);
+	ASSERT_EQ(ret, 0);
+
+	/*
+	 * Use the unlocked read path which allows the mmap lock to be
+	 * dropped during the fault, enabling userfaultfd resolution.
+	 */
+	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ_UNLOCKED,
+			      buffer, npages);
+	ASSERT_EQ(ret, 0);
+	ASSERT_EQ(buffer->cpages, npages);
+
+	/* Verify the device read the data filled by the uffd handler */
+	ptr = buffer->mirror;
+	for (i = 0; i < size; ++i)
+		ASSERT_EQ(ptr[i], (unsigned char)0xAB);
+
+	nwrite = write(stop_fd, &stop, sizeof(stop));
+	ASSERT_EQ(nwrite, sizeof(stop));
+	pthread_join(thread, NULL);
+	close(stop_fd);
+	free(uffd_args.page_buffer);
+	close(uffd);
+	hmm_buffer_free(buffer);
+}
+
+
 TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/mm/hugepage_settings.c b/tools/testing/selftests/mm/hugepage_settings.c
index 2eab2110ac6a4..d7917dce3abac 100644
--- a/tools/testing/selftests/mm/hugepage_settings.c
+++ b/tools/testing/selftests/mm/hugepage_settings.c
@@ -437,7 +437,7 @@ void hugetlb_set_nr_pages(unsigned long size, unsigned long nr)
 
 	hugetlb_sysfs_path(path, sizeof(path), size, "nr_hugepages");
 
-	write_num(path, nr);
+	write_num_ignore_einval(path, nr);
 }
 
 unsigned long hugetlb_free_pages(unsigned long size)
diff --git a/tools/testing/selftests/mm/hugetlb-vmemmap.c b/tools/testing/selftests/mm/hugetlb-vmemmap.c
deleted file mode 100644
index 507df78a158d8..0000000000000
--- a/tools/testing/selftests/mm/hugetlb-vmemmap.c
+++ /dev/null
@@ -1,132 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * A test case of using hugepage memory in a user application using the
- * mmap system call with MAP_HUGETLB flag.  Before running this program
- * make sure the administrator has allocated enough default sized huge
- * pages to cover the 2 MB allocation.
- */
-#include <stdlib.h>
-#include <stdio.h>
-#include <unistd.h>
-#include <sys/mman.h>
-#include <fcntl.h>
-#include "vm_util.h"
-#include "hugepage_settings.h"
-
-#define PAGE_COMPOUND_HEAD	(1UL << 15)
-#define PAGE_COMPOUND_TAIL	(1UL << 16)
-#define PAGE_HUGE		(1UL << 17)
-
-#define HEAD_PAGE_FLAGS		(PAGE_COMPOUND_HEAD | PAGE_HUGE)
-#define TAIL_PAGE_FLAGS		(PAGE_COMPOUND_TAIL | PAGE_HUGE)
-
-#define PM_PFRAME_BITS		55
-#define PM_PFRAME_MASK		~((1UL << PM_PFRAME_BITS) - 1)
-
-static size_t pagesize;
-static size_t maplength;
-
-static void write_bytes(char *addr, size_t length)
-{
-	unsigned long i;
-
-	for (i = 0; i < length; i++)
-		*(addr + i) = (char)i;
-}
-
-static unsigned long virt_to_pfn(void *addr)
-{
-	int fd;
-	unsigned long pagemap;
-
-	fd = open("/proc/self/pagemap", O_RDONLY);
-	if (fd < 0)
-		return -1UL;
-
-	lseek(fd, (unsigned long)addr / pagesize * sizeof(pagemap), SEEK_SET);
-	read(fd, &pagemap, sizeof(pagemap));
-	close(fd);
-
-	return pagemap & ~PM_PFRAME_MASK;
-}
-
-static int check_page_flags(unsigned long pfn)
-{
-	int fd, i;
-	unsigned long pageflags;
-
-	fd = open("/proc/kpageflags", O_RDONLY);
-	if (fd < 0)
-		return -1;
-
-	lseek(fd, pfn * sizeof(pageflags), SEEK_SET);
-
-	read(fd, &pageflags, sizeof(pageflags));
-	if ((pageflags & HEAD_PAGE_FLAGS) != HEAD_PAGE_FLAGS) {
-		close(fd);
-		ksft_print_msg("Head page flags (%lx) is invalid\n", pageflags);
-		return -1;
-	}
-
-	/*
-	 * pages other than the first page must be tail and shouldn't be head;
-	 * this also verifies kernel has correctly set the fake page_head to tail
-	 * while hugetlb_free_vmemmap is enabled.
-	 */
-	for (i = 1; i < maplength / pagesize; i++) {
-		read(fd, &pageflags, sizeof(pageflags));
-		if ((pageflags & TAIL_PAGE_FLAGS) != TAIL_PAGE_FLAGS ||
-		    (pageflags & HEAD_PAGE_FLAGS) == HEAD_PAGE_FLAGS) {
-			close(fd);
-			ksft_print_msg("Tail page flags (%lx) is invalid\n", pageflags);
-			return -1;
-		}
-	}
-
-	close(fd);
-
-	return 0;
-}
-
-int main(int argc, char **argv)
-{
-	void *addr;
-	unsigned long pfn;
-	int ret;
-
-	ksft_print_header();
-	ksft_set_plan(1);
-
-	if (!hugetlb_setup_default(1))
-		ksft_exit_skip("Not enough free huge pages\n");
-
-	pagesize  = psize();
-	maplength = default_huge_page_size();
-	if (!maplength)
-		ksft_exit_skip("Unable to determine huge page size\n");
-
-	addr = mmap(NULL, maplength, PROT_READ | PROT_WRITE,
-			MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
-	if (addr == MAP_FAILED)
-		ksft_exit_fail_perror("mmap");
-
-	/* Trigger allocation of HugeTLB page. */
-	write_bytes(addr, maplength);
-
-	pfn = virt_to_pfn(addr);
-	if (pfn == -1UL) {
-		ksft_perror("virt_to_pfn");
-		munmap(addr, maplength);
-		ksft_exit_fail();
-	}
-
-	ksft_print_msg("Returned address is %p whose pfn is %lx\n", addr, pfn);
-
-	ret = check_page_flags(pfn);
-
-	if (munmap(addr, maplength))
-		ksft_exit_fail_perror("munmap");
-
-	ksft_test_result(!ret, "HugeTLB vmemmap page flags\n");
-	ksft_finished();
-}
diff --git a/tools/testing/selftests/mm/hwpoison-panic.sh b/tools/testing/selftests/mm/hwpoison-panic.sh
new file mode 100755
index 0000000000000..d953d13673324
--- /dev/null
+++ b/tools/testing/selftests/mm/hwpoison-panic.sh
@@ -0,0 +1,255 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Verify vm.panic_on_unrecoverable_memory_failure by injecting a hwpoison
+# error on a kernel-owned page and confirming the kernel panics.
+#
+# Three "kinds" of kernel-owned page can be targeted, selectable via the
+# first positional argument (default: rodata):
+#
+#   rodata  - a PG_reserved page in the kernel rodata range
+#             (sourced from /proc/iomem "Kernel rodata").  Exercises
+#             memory_failure() -> get_any_page() on a PageReserved page.
+#
+#   slab    - a slab page found via /proc/kpageflags (KPF_SLAB).
+#             Exercises memory_failure() -> get_any_page() on a non
+#             PG_reserved kernel-owned page.  This path is what catches
+#             regressions where get_any_page() collapses kernel-owned
+#             pages into a transient -EIO instead of -ENOTRECOVERABLE.
+#
+#   pgtable - a page-table page found via /proc/kpageflags (KPF_PGTABLE).
+#             Same path as slab, different page type.
+#
+# This test is DESTRUCTIVE: a successful run crashes the kernel.  It is
+# meant to be executed inside a disposable VM (e.g. virtme-ng) with a
+# serial console captured by the harness.  It is skipped unless the
+# caller opts in via RUN_DESTRUCTIVE=1.
+#
+# Test passes externally: the kernel must panic with
+#   "Memory failure: <pfn>: unrecoverable page"
+# A return from the inject means no panic fired: that is a failure,
+# unless the target PFN raced to a different page type before injection,
+# in which case the run is inconclusive and is skipped.
+#
+# Author: Breno Leitao <leitao@debian.org>
+
+set -u
+
+# KTAP output helpers (ktap_print_msg, ktap_skip_all, ktap_exit_fail_msg, ...).
+DIR="$(dirname "$(readlink -f "$0")")"
+# shellcheck source=../kselftest/ktap_helpers.sh
+source "${DIR}"/../kselftest/ktap_helpers.sh
+
+sysctl_path=/proc/sys/vm/panic_on_unrecoverable_memory_failure
+inject_path=/sys/devices/system/memory/hard_offline_page
+kpageflags_path=/proc/kpageflags
+unpoison_path=/sys/kernel/debug/hwpoison/unpoison-pfn
+
+# /proc/kpageflags bit positions (see include/uapi/linux/kernel-page-flags.h)
+KPF_SLAB=7
+KPF_COMPOUND_TAIL=16
+KPF_HWPOISON=19
+KPF_NOPAGE=20
+KPF_PGTABLE=26
+KPF_RESERVED=32
+
+pagesize=$(getconf PAGE_SIZE)
+
+kind=${1:-rodata}
+
+if [ "$(id -u)" -ne 0 ]; then
+	ktap_skip_all "must run as root"
+	exit "$KSFT_SKIP"
+fi
+
+if [ ! -w "$sysctl_path" ]; then
+	ktap_skip_all "$sysctl_path not present (kernel without the sysctl?)"
+	exit "$KSFT_SKIP"
+fi
+
+if [ ! -w "$inject_path" ]; then
+	ktap_skip_all "$inject_path not present (no MEMORY_HOTPLUG?)"
+	exit "$KSFT_SKIP"
+fi
+
+if [ "${RUN_DESTRUCTIVE:-0}" != "1" ]; then
+	ktap_skip_all "destructive test; re-run with RUN_DESTRUCTIVE=1 inside a disposable VM"
+	exit "$KSFT_SKIP"
+fi
+
+# Pick a PFN inside the kernel image rodata region of /proc/iomem.
+# This is preferred over a top-level "Reserved" entry because top-level
+# Reserved ranges are often firmware holes that have no backing struct
+# page; pfn_to_online_page() returns NULL on those and memory_failure()
+# bails out with -ENXIO before reaching the panic path.
+#
+# "Kernel rodata" is reported as a sub-resource of "System RAM" on every
+# major architecture, which guarantees:
+#   - the PFN is backed by struct page (within an online memory range);
+#   - PG_reserved is set on the page (kernel image area);
+#   - the memory is read-only, so setting PG_hwpoison on it does not
+#     corrupt writable kernel state if the panic somehow does not fire.
+#
+# /proc/iomem entries look like (indented for sub-resources):
+#     "  02500000-02ffffff : Kernel rodata"
+pick_rodata_phys_addr() {
+	awk -v pagesize="$(getconf PAGE_SIZE)" '
+	# Convert a hex string to a number without relying on the gawk-only
+	# strtonum().  mawk lacks it and would otherwise spuriously skip
+	# this test on distros that ship mawk as /usr/bin/awk.
+	function hex2num(s,   n, i, c, v) {
+		n = 0
+		for (i = 1; i <= length(s); i++) {
+			c = tolower(substr(s, i, 1))
+			v = index("0123456789abcdef", c) - 1
+			if (v < 0)
+				return -1
+			n = n * 16 + v
+		}
+		return n
+	}
+	/: Kernel rodata[[:space:]]*$/ {
+		sub(/^[[:space:]]+/, "")
+		n = split($0, a, /[- ]/)
+		start = hex2num(a[1])
+		end   = hex2num(a[2])
+		if (end <= start)
+			next
+		# Page-align upward and emit the first byte of that page.
+		pfn = int((start + pagesize - 1) / pagesize)
+		printf "0x%x\n", pfn * pagesize
+		exit 0
+	}
+	' /proc/iomem
+}
+
+# Walk /proc/kpageflags and return the phys addr of the first PFN that
+# has bit $1 set, with KPF_HWPOISON, KPF_NOPAGE and KPF_COMPOUND_TAIL
+# all clear (so we attack a real, non-tail, not-already-poisoned page).
+#
+# We skip the first 16 MiB of PFNs to step past low-memory special
+# ranges (BIOS/EFI/ACPI/etc.) that often are PG_reserved and would not
+# exhibit the slab/pgtable type we are looking for.
+pick_kpageflags_phys_addr() {
+	local want_bit=$1
+	local pagesize skip_pfn
+
+	[ -r "$kpageflags_path" ] || return
+
+	pagesize=$(getconf PAGE_SIZE)
+	skip_pfn=$(((16 * 1024 * 1024) / pagesize))
+
+	od -An -tx8 -v -w8 -j "$((skip_pfn * 8))" "$kpageflags_path" 2>/dev/null | \
+	awk -v want_bit="$want_bit" \
+	    -v hwp_bit="$KPF_HWPOISON" \
+	    -v nopage_bit="$KPF_NOPAGE" \
+	    -v tail_bit="$KPF_COMPOUND_TAIL" \
+	    -v base_pfn="$skip_pfn" \
+	    -v pagesize="$pagesize" '
+	# Test whether bit "b" is set in the 16-hex-digit value "hex".
+	# Done with substring + per-digit lookup so we never rely on awk
+	# bitwise operators (mawk lacks them), 64-bit FP precision or the
+	# gawk-only strtonum().
+	function bit_set(hex, b,    di, bi, c, v) {
+		di = int(b / 4)
+		bi = b - di * 4
+		c = substr(hex, length(hex) - di, 1)
+		v = index("0123456789abcdef", tolower(c)) - 1
+		if (bi == 0) return (v % 2) == 1
+		if (bi == 1) return int(v / 2) % 2 == 1
+		if (bi == 2) return int(v / 4) % 2 == 1
+		return int(v / 8) % 2 == 1
+	}
+	{
+		gsub(/^[[:space:]]+/, "")
+		h = $1
+		if (bit_set(h, want_bit) &&
+		    !bit_set(h, hwp_bit) &&
+		    !bit_set(h, nopage_bit) &&
+		    !bit_set(h, tail_bit)) {
+			pfn = base_pfn + NR - 1
+			printf "0x%x\n", pfn * pagesize
+			exit 0
+		}
+	}
+	'
+}
+
+# Return 0 if /proc/kpageflags bit $2 is set for PFN $1, 1 if it is
+# clear, or 2 if the word cannot be read.  Used to re-confirm the target
+# page type after a non-panicking inject.
+kpageflags_bit_set() {
+	local word
+
+	word=$(od -An -tx8 -v -j "$(($1 * 8))" -N 8 "$kpageflags_path" 2>/dev/null | tr -d '[:space:]')
+	[ -n "$word" ] || return 2
+	(( (16#$word >> $2) & 1 ))
+}
+
+# Best-effort: drop the PG_hwpoison marker set by the inject so a failed
+# run does not leave a poisoned page behind.  hard_offline_page() injects
+# with MF_SW_SIMULATED, so the page stays unpoisonable through the
+# hwpoison debugfs interface (needs CONFIG_HWPOISON_INJECT + debugfs).
+try_unpoison() {
+	[ -w "$unpoison_path" ] || return 0
+	echo "$1" > "$unpoison_path" 2>/dev/null || true
+}
+
+case "$kind" in
+rodata)
+	phys_addr=$(pick_rodata_phys_addr)
+	recheck_bit=$KPF_RESERVED
+	missing_msg='no "Kernel rodata" entry in /proc/iomem'
+	;;
+slab)
+	phys_addr=$(pick_kpageflags_phys_addr "$KPF_SLAB")
+	recheck_bit=$KPF_SLAB
+	missing_msg="no usable slab PFN found in $kpageflags_path"
+	;;
+pgtable)
+	phys_addr=$(pick_kpageflags_phys_addr "$KPF_PGTABLE")
+	recheck_bit=$KPF_PGTABLE
+	missing_msg="no usable page-table PFN found in $kpageflags_path"
+	;;
+*)
+	ktap_exit_fail_msg "unknown kind '$kind' (expected: rodata|slab|pgtable)"
+	;;
+esac
+
+if [ -z "$phys_addr" ]; then
+	ktap_skip_all "$missing_msg"
+	exit "$KSFT_SKIP"
+fi
+
+ktap_print_msg "enabling $sysctl_path"
+prior=$(cat "$sysctl_path")
+echo 1 > "$sysctl_path" || ktap_exit_fail_msg "failed to enable sysctl"
+
+pfn=$((phys_addr / pagesize))
+ktap_print_msg "injecting hwpoison at phys 0x$(printf '%x' "$phys_addr") (pfn 0x$(printf '%x' "$pfn"), kind=$kind)"
+ktap_print_msg "expecting kernel panic: 'Memory failure: <pfn>: unrecoverable page'"
+
+# A successful run never returns from the inject -- it panics the kernel.
+# Reaching the code below therefore means no panic fired.  Note whether
+# the write itself succeeded, then put the machine back: restore the
+# sysctl and best-effort unpoison the page we just marked.
+if echo "$phys_addr" > "$inject_path"; then
+	verdict="inject returned without panic; sysctl ineffective"
+else
+	verdict="inject failed before reaching the panic path"
+fi
+
+echo "$prior" > "$sysctl_path"
+try_unpoison "$pfn"
+
+# The page type can change between selection and injection (e.g. a slab
+# or page-table page is freed and reused).  Only treat a missing panic as
+# a failure if the target PFN is still the kernel-owned type we aimed at;
+# if it raced to another type the run is inconclusive, so skip instead.
+kpageflags_bit_set "$pfn" "$recheck_bit"
+case $? in
+0)	ktap_exit_fail_msg "$verdict (page still $kind)" ;;
+1)	ktap_skip_all "target PFN no longer $kind; raced before inject, inconclusive"
+	exit "$KSFT_SKIP" ;;
+*)	ktap_exit_fail_msg "$verdict (could not reconfirm page type via $kpageflags_path)" ;;
+esac
diff --git a/tools/testing/selftests/mm/ksft_kmemleak_confirm.sh b/tools/testing/selftests/mm/ksft_kmemleak_confirm.sh
new file mode 100755
index 0000000000000..3a8576e835c8d
--- /dev/null
+++ b/tools/testing/selftests/mm/ksft_kmemleak_confirm.sh
@@ -0,0 +1,132 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Functional test for kmemleak's N-consecutive-scan leak confirmation
+# (the min_unref_scans module parameter).
+#
+# kmemleak only reports an object once it has stayed unreferenced for
+# min_unref_scans consecutive scans. The default of 1 reports on the first
+# scan (historical behaviour); higher values filter transient false
+# positives where a live object's only reference is briefly invisible to a
+# single scan (e.g. an RCU tree update in flight while the scan runs). The
+# test loads samples/kmemleak's helper module to create orphan allocations
+# and, counting only those orphans (matched by their [kmemleak_test]
+# backtrace so unrelated leaks already present on the system are ignored),
+# checks that:
+#   - a freshly allocated object is greyed on its first scan (its checksum
+#     settles then), so nothing can be reported before that priming scan;
+#     each case below primes once first,
+#   - with the default threshold (min_unref_scans=1) one scan after priming
+#     reports the orphans,
+#   - raising the threshold to 2 needs two scans after priming: one is not
+#     enough, the second reports,
+#   - the parameter reads back what was written.
+#
+# The "one post-prime scan is not enough at min_unref_scans=2" check is the
+# core regression test: raising min_unref_scans must push the report
+# strictly later. Like ksft_kmemleak_dedup.sh, if the module yields no
+# detectable orphan at all in the running environment the test skips rather
+# than failing.
+#
+# Author: Breno Leitao <leitao@debian.org>
+
+# KTAP output helpers (ktap_skip_all, ktap_exit_fail_msg, ktap_test_pass, ...).
+DIR="$(dirname "$(readlink -f "$0")")"
+# shellcheck source=../kselftest/ktap_helpers.sh
+source "${DIR}"/../kselftest/ktap_helpers.sh
+
+KMEMLEAK=/sys/kernel/debug/kmemleak
+PARAM=/sys/module/kmemleak/parameters/min_unref_scans
+MODULE=kmemleak-test
+AGE=6		# seconds; must exceed kmemleak's 5s minimum object age
+
+ktap_print_header
+
+[ "$(id -u)" -eq 0 ] || { ktap_skip_all "must run as root"; exit "$KSFT_SKIP"; }
+[ -r "$KMEMLEAK" ] ||
+	{ ktap_skip_all "no kmemleak debugfs (CONFIG_DEBUG_KMEMLEAK)"; exit "$KSFT_SKIP"; }
+[ -w "$PARAM" ] ||
+	{ ktap_skip_all "min_unref_scans module parameter not present"; exit "$KSFT_SKIP"; }
+modinfo "$MODULE" >/dev/null 2>&1 ||
+	{ ktap_skip_all "$MODULE not built (CONFIG_SAMPLE_KMEMLEAK)"; exit "$KSFT_SKIP"; }
+
+# kmemleak can be present but disabled at runtime (kmemleak=off boot arg,
+# or it self-disabled after an internal error); a "scan" then returns
+# EPERM. Probe once and skip if so.
+echo scan > "$KMEMLEAK" 2>/dev/null ||
+	{ ktap_skip_all "kmemleak is disabled (check dmesg or kmemleak= boot arg)"; exit "$KSFT_SKIP"; }
+
+prev=$(cat "$PARAM")
+# shellcheck disable=SC2317  # invoked indirectly via trap
+cleanup() {
+	echo "$prev" > "$PARAM" 2>/dev/null		# restore the parameter
+	echo scan=on > "$KMEMLEAK" 2>/dev/null		# re-enable auto scan
+	rmmod "$MODULE" 2>/dev/null
+	echo clear > "$KMEMLEAK" 2>/dev/null
+}
+trap cleanup EXIT
+
+# Stop the automatic scan thread: only our manual scans should advance an
+# object's consecutive-unreferenced run. An auto scan landing between two
+# manual scans would change the result and make the test flaky.
+echo scan=off > "$KMEMLEAK" 2>/dev/null
+
+# Create a fresh, aged set of orphan objects from the helper module's init
+# path (its kmalloc/vmalloc/percpu allocations are dropped right away).
+# Pre-existing reported leaks are greyed first ("clear") so only our
+# orphans are counted. The module is left loaded on purpose: once it is
+# unloaded its symbols are gone, so the orphan backtraces no longer resolve
+# to [kmemleak_test] and could not be matched below.
+gen_orphans() {
+	rmmod "$MODULE" 2>/dev/null
+	echo clear > "$KMEMLEAK"
+	modprobe "$MODULE" ||
+		{ ktap_skip_all "failed to load $MODULE"; exit "$KSFT_SKIP"; }
+	sleep "$AGE"
+}
+
+scan() { echo scan > "$KMEMLEAK"; }
+
+# Number of helper-module orphans currently reported by kmemleak. Matching
+# the module's own backtrace ([kmemleak_test]) keeps the count immune to
+# unrelated leaks on the running system. kmemleak only lists an object here
+# once it has been reported, so this reflects the confirmation gating.
+count_orphans() {
+	c=$(grep -c '\[kmemleak_test\]' "$KMEMLEAK" 2>/dev/null)
+	echo "${c:-0}"
+}
+
+# 0) the parameter reads back what was written.
+echo 3 > "$PARAM"
+[ "$(cat "$PARAM")" = "3" ] || ktap_exit_fail_msg "min_unref_scans did not read back as 3"
+
+# Priming scan: kmemleak greys a freshly allocated object on its first scan
+# (its checksum settles then), so nothing can be reported until a second
+# scan. Every case below runs this priming scan before counting.
+prime() { scan; }
+
+# 1) min_unref_scans=1 (default): one scan after priming reports the
+#    orphans. This also establishes that the helper produces detectable
+#    orphans here.
+echo 1 > "$PARAM"
+gen_orphans
+prime
+scan
+first=$(count_orphans)
+[ "$first" -gt 0 ] ||
+	{ ktap_skip_all "$MODULE produced no detectable orphans (cannot test min_unref_scans)"; exit "$KSFT_SKIP"; }
+
+# 2) min_unref_scans=2: after priming, one scan is not enough (still
+#    gated), the second reports. The gated-scan-zero check is the core
+#    regression.
+echo 2 > "$PARAM"
+gen_orphans
+prime
+scan; s1=$(count_orphans)
+scan; s2=$(count_orphans)
+[ "$s1" -eq 0 ] || ktap_exit_fail_msg "min_unref_scans=2: $s1 orphan(s) after 1 post-prime scan (must be 0)"
+[ "$s2" -gt 0 ] || ktap_exit_fail_msg "min_unref_scans=2: no report after 2 post-prime scans (false negative)"
+
+ktap_set_plan 1
+ktap_test_pass "min_unref_scans=1 reported $first orphan(s) one scan after priming; =2 held them one scan longer ($s1 after one scan, $s2 after two); param read-back ok"
+ktap_finished
diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c
index a050f4840cfa3..5fd7792a0d479 100644
--- a/tools/testing/selftests/mm/ksm_tests.c
+++ b/tools/testing/selftests/mm/ksm_tests.c
@@ -288,8 +288,8 @@ static bool assert_ksm_pages_count(long dupl_page_count)
 static int ksm_save_def(struct ksm_sysfs *ksm_sysfs)
 {
 	if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) ||
-	    numa_available() ? 0 :
-		ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) ||
+	    (numa_available() ? 0 :
+		ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes)) ||
 	    ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) ||
 	    ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) ||
 	    ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) ||
@@ -304,8 +304,8 @@ static int ksm_save_def(struct ksm_sysfs *ksm_sysfs)
 static int ksm_restore(struct ksm_sysfs *ksm_sysfs)
 {
 	if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) ||
-	    numa_available() ? 0 :
-		ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) ||
+	    (numa_available() ? 0 :
+		ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes)) ||
 	    ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) ||
 	    ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) ||
 	    ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) ||
@@ -440,9 +440,9 @@ static int get_next_mem_node(int node)
 		mem_node = i % (max_node + 1);
 		node_size = numa_node_size(mem_node, NULL);
 		if (node_size > 0)
-			break;
+			return mem_node;
 	}
-	return mem_node;
+	return -ENODEV;
 }
 
 static int get_first_mem_node(void)
@@ -455,8 +455,8 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo
 {
 	void *numa1_map_ptr, *numa2_map_ptr;
 	struct timespec start_time;
+	int first_node, second_node;
 	int page_count = 2;
-	int first_node;
 
 	if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) {
 		ksft_perror("clock_gettime");
@@ -467,17 +467,19 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo
 		ksft_print_msg("NUMA support not enabled\n");
 		return KSFT_SKIP;
 	}
-	if (numa_num_configured_nodes() <= 1) {
-		ksft_print_msg("At least 2 NUMA nodes must be available\n");
+	first_node = get_first_mem_node();
+	second_node = get_next_mem_node(first_node);
+
+	if (second_node < 0) {
+		ksft_print_msg("At least 2 NUMA nodes with memory must be available\n");
 		return KSFT_SKIP;
 	}
 	if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes))
 		return KSFT_FAIL;
 
 	/* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */
-	first_node = get_first_mem_node();
 	numa1_map_ptr = numa_alloc_onnode(page_size, first_node);
-	numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node));
+	numa2_map_ptr = numa_alloc_onnode(page_size, second_node);
 	if (!numa1_map_ptr || !numa2_map_ptr) {
 		ksft_perror("numa_alloc_onnode");
 		return KSFT_FAIL;
@@ -844,8 +846,8 @@ int main(int argc, char *argv[])
 
 	if (ksm_write_sysfs(KSM_FP("run"), 2) ||
 	    ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) ||
-	    numa_available() ? 0 :
-		ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) ||
+	    (numa_available() ? 0 :
+		ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1)) ||
 	    ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count))
 		ksft_exit_fail_msg("Cannot set up KSM tunables\n");
 
diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c
index 6f8971d5b3ce4..f9bcff8e78fab 100644
--- a/tools/testing/selftests/mm/pagemap_ioctl.c
+++ b/tools/testing/selftests/mm/pagemap_ioctl.c
@@ -1051,6 +1051,57 @@ static void test_simple(void)
 	ksft_test_result(i == TEST_ITERATIONS, "Test %s\n", __func__);
 }
 
+/*
+ * A range that was populated and then MADV_DONTNEED'd is genuine pte_none
+ * with no uffd-wp marker. Such a pte must read the same regardless of which
+ * PAGEMAP_SCAN path serves the request: both the PAGE_IS_WRITTEN fast path and
+ * the generic path (reached e.g. via category_anyof_mask) must report every
+ * page written.
+ */
+static void unpopulated_scan_test(void)
+{
+	int npages = 16, i;
+	long mem_size = npages * page_size;
+	struct page_region regions[16];
+	long fast = 0, slow = 0, ret;
+	char *mem;
+
+	mem = mmap(NULL, mem_size, PROT_READ | PROT_WRITE,
+		   MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+	if (mem == MAP_FAILED)
+		ksft_exit_fail_msg("%s mmap failed\n", __func__);
+
+	wp_init(mem, mem_size);
+
+	/* Populate, then drop: the ptes become pte_none without a marker. */
+	memset(mem, 1, mem_size);
+	if (madvise(mem, mem_size, MADV_DONTNEED))
+		ksft_exit_fail_msg("%s MADV_DONTNEED failed\n", __func__);
+
+	/* Fast path: category_mask == return_mask == PAGE_IS_WRITTEN. */
+	ret = pagemap_ioctl(mem, mem_size, regions, npages, 0, 0,
+			    PAGE_IS_WRITTEN, 0, 0, PAGE_IS_WRITTEN);
+	if (ret < 0)
+		ksft_exit_fail_msg("%s fast scan failed\n", __func__);
+	for (i = 0; i < ret; i++)
+		fast += LEN(regions[i]);
+
+	/* Generic path: same query expressed via category_anyof_mask. */
+	ret = pagemap_ioctl(mem, mem_size, regions, npages, 0, 0,
+			    0, PAGE_IS_WRITTEN, 0, PAGE_IS_WRITTEN);
+	if (ret < 0)
+		ksft_exit_fail_msg("%s generic scan failed\n", __func__);
+	for (i = 0; i < ret; i++)
+		slow += LEN(regions[i]);
+
+	ksft_test_result(fast == npages && slow == npages,
+			 "%s unpopulated ptes reported written by both paths (%ld, %ld of %d)\n",
+			 __func__, fast, slow, npages);
+
+	wp_free(mem, mem_size);
+	munmap(mem, mem_size);
+}
+
 int sanity_tests(void)
 {
 	unsigned long long mem_size, vec_size;
@@ -1559,7 +1610,7 @@ int main(int __attribute__((unused)) argc, char *argv[])
 	if (!hugetlb_setup_default(4))
 		ksft_print_msg("HugeTLB test will be skipped\n");
 
-	ksft_set_plan(117);
+	ksft_set_plan(118);
 
 	page_size = getpagesize();
 	hpage_size = read_pmd_pagesize();
@@ -1737,6 +1788,9 @@ int main(int __attribute__((unused)) argc, char *argv[])
 	/* 17. ZEROPFN tests */
 	zeropfn_tests();
 
+	/* 18. Unpopulated pte scan-path consistency */
+	unpopulated_scan_test();
+
 	close(pagemap_fd);
 	ksft_finished();
 }
diff --git a/tools/testing/selftests/mm/pkey-helpers.h b/tools/testing/selftests/mm/pkey-helpers.h
index 2c377f4e9df1d..46a8a1878dc1f 100644
--- a/tools/testing/selftests/mm/pkey-helpers.h
+++ b/tools/testing/selftests/mm/pkey-helpers.h
@@ -68,7 +68,9 @@ static inline void sigsafe_printf(const char *format, ...)
 #define dprintf3(args...) dprintf_level(3, args)
 #define dprintf4(args...) dprintf_level(4, args)
 
-extern void abort_hooks(void);
+void tracing_on(void);
+void tracing_off(void);
+void abort_hooks(void);
 #define pkey_assert(condition) do {		\
 	if (!(condition)) {			\
 		dprintf0("# assert() at %s::%d test_nr: %d iteration: %d\n", \
diff --git a/tools/testing/selftests/mm/pkey-powerpc.h b/tools/testing/selftests/mm/pkey-powerpc.h
index 17bf2d1b0192e..2ce85580b4048 100644
--- a/tools/testing/selftests/mm/pkey-powerpc.h
+++ b/tools/testing/selftests/mm/pkey-powerpc.h
@@ -126,7 +126,7 @@ static inline void *malloc_pkey_with_mprotect_subpage(long size, int prot, u16 p
 			size, prot, pkey);
 	pkey_assert(pkey < NR_PKEYS);
 	ptr = mmap(NULL, size, prot, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
-	pkey_assert(ptr != (void *)-1);
+	pkey_assert(ptr != MAP_FAILED);
 
 	ret = syscall(__NR_subpage_prot, ptr, size, NULL);
 	if (ret) {
diff --git a/tools/testing/selftests/mm/pkey_sighandler_tests.c b/tools/testing/selftests/mm/pkey_sighandler_tests.c
index 302fef54049c8..c218d0510a2a4 100644
--- a/tools/testing/selftests/mm/pkey_sighandler_tests.c
+++ b/tools/testing/selftests/mm/pkey_sighandler_tests.c
@@ -19,7 +19,6 @@
 #include <stdint.h>
 #include <stdbool.h>
 #include <signal.h>
-#include <assert.h>
 #include <stdlib.h>
 #include <sys/mman.h>
 #include <sys/types.h>
@@ -207,22 +206,25 @@ static void test_sigsegv_handler_with_pkey0_disabled(void)
 	struct sigaction sa;
 	pthread_attr_t attr;
 	pthread_t thr;
+	int ret;
 
 	sa.sa_flags = SA_SIGINFO;
 
 	sa.sa_sigaction = sigsegv_handler;
 	sigemptyset(&sa.sa_mask);
-	if (sigaction(SIGSEGV, &sa, NULL) == -1) {
-		perror("sigaction");
-		exit(EXIT_FAILURE);
-	}
+	ret = sigaction(SIGSEGV, &sa, NULL);
+	pkey_assert(ret == 0);
 
 	memset(&siginfo, 0, sizeof(siginfo));
 
 	pthread_attr_init(&attr);
 	pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
 
-	pthread_create(&thr, &attr, thread_segv_with_pkey0_disabled, NULL);
+	ret = pthread_create(&thr, &attr, thread_segv_with_pkey0_disabled, NULL);
+	if (ret) {
+		errno = ret;
+		pkey_assert(0);
+	}
 
 	pthread_mutex_lock(&mutex);
 	while (siginfo.si_signo == 0)
@@ -247,22 +249,25 @@ static void test_sigsegv_handler_cannot_access_stack(void)
 	struct sigaction sa;
 	pthread_attr_t attr;
 	pthread_t thr;
+	int ret;
 
 	sa.sa_flags = SA_SIGINFO;
 
 	sa.sa_sigaction = sigsegv_handler;
 	sigemptyset(&sa.sa_mask);
-	if (sigaction(SIGSEGV, &sa, NULL) == -1) {
-		perror("sigaction");
-		exit(EXIT_FAILURE);
-	}
+	ret = sigaction(SIGSEGV, &sa, NULL);
+	pkey_assert(ret == 0);
 
 	memset(&siginfo, 0, sizeof(siginfo));
 
 	pthread_attr_init(&attr);
 	pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
 
-	pthread_create(&thr, &attr, thread_segv_pkuerr_stack, NULL);
+	ret = pthread_create(&thr, &attr, thread_segv_pkuerr_stack, NULL);
+	if (ret) {
+		errno = ret;
+		pkey_assert(0);
+	}
 
 	pthread_mutex_lock(&mutex);
 	while (siginfo.si_signo == 0)
@@ -285,24 +290,22 @@ static void test_sigsegv_handler_with_different_pkey_for_stack(void)
 	static stack_t sigstack;
 	void *stack;
 	int pkey;
-	int parent_pid = 0;
 	int child_pid = 0;
 	u64 pkey_reg;
+	long ret;
 
 	sa.sa_flags = SA_SIGINFO | SA_ONSTACK;
 
 	sa.sa_sigaction = sigsegv_handler;
 
 	sigemptyset(&sa.sa_mask);
-	if (sigaction(SIGSEGV, &sa, NULL) == -1) {
-		perror("sigaction");
-		exit(EXIT_FAILURE);
-	}
+	ret = sigaction(SIGSEGV, &sa, NULL);
+	pkey_assert(ret == 0);
 
 	stack = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE,
 		     MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 
-	assert(stack != MAP_FAILED);
+	pkey_assert(stack != MAP_FAILED);
 
 	/* Allow access to MPK 0 and MPK 1 */
 	pkey_reg = pkey_reg_restrictive_default();
@@ -317,33 +320,41 @@ static void test_sigsegv_handler_with_different_pkey_for_stack(void)
 	/* Set up alternate signal stack that will use the default MPK */
 	sigstack.ss_sp = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE,
 			      MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+	pkey_assert(sigstack.ss_sp != MAP_FAILED);
 	sigstack.ss_flags = 0;
 	sigstack.ss_size = STACK_SIZE;
 
 	memset(&siginfo, 0, sizeof(siginfo));
 
 	/* Use clone to avoid newer glibcs using rseq on new threads */
-	long ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES |
-			     CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM |
-			     CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID |
-			     CLONE_DETACHED,
-			     stack + STACK_SIZE,
-			     &parent_pid,
-			     &child_pid);
+	ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES |
+			CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM |
+			CLONE_DETACHED,
+			stack + STACK_SIZE,
+			NULL,
+			NULL);
 
 	if (ret < 0) {
 		errno = -ret;
-		perror("clone");
+		pkey_assert(0);
 	} else if (ret == 0) {
 		thread_segv_maperr_ptr(&sigstack);
 		syscall_raw(SYS_exit, 0, 0, 0, 0, 0, 0);
 	}
 
+	child_pid = ret;
+
 	pthread_mutex_lock(&mutex);
 	while (siginfo.si_signo == 0)
 		pthread_cond_wait(&cond, &mutex);
 	pthread_mutex_unlock(&mutex);
 
+	/* Wait for child to exit before returning */
+	do {
+		sched_yield();
+		ret = syscall_raw(SYS_tkill, child_pid, 0, 0, 0, 0, 0);
+	} while (ret != -ESRCH && ret != -EINVAL);
+
 	ksft_test_result(siginfo.si_signo == SIGSEGV &&
 			 siginfo.si_code == SEGV_MAPERR &&
 			 siginfo.si_addr == NULL,
@@ -358,6 +369,7 @@ static void test_pkru_preserved_after_sigusr1(void)
 {
 	struct sigaction sa;
 	u64 pkey_reg;
+	int ret;
 
 	/* Allow access to MPK 0 and an arbitrary set of keys */
 	pkey_reg = pkey_reg_restrictive_default();
@@ -369,10 +381,8 @@ static void test_pkru_preserved_after_sigusr1(void)
 
 	sa.sa_sigaction = sigusr1_handler;
 	sigemptyset(&sa.sa_mask);
-	if (sigaction(SIGUSR1, &sa, NULL) == -1) {
-		perror("sigaction");
-		exit(EXIT_FAILURE);
-	}
+	ret = sigaction(SIGUSR1, &sa, NULL);
+	pkey_assert(ret == 0);
 
 	memset(&siginfo, 0, sizeof(siginfo));
 
@@ -441,9 +451,15 @@ static void test_pkru_sigreturn(void)
 	static stack_t sigstack;
 	void *stack;
 	int pkey;
-	int parent_pid = 0;
 	int child_pid = 0;
 	u64 pkey_reg;
+	long ret;
+
+	/*
+	 * SIGSEGV handler is reset to SIG_DFL below; turn tracing off first
+	 * so a crash does not leave ftrace enabled.
+	 */
+	tracing_off();
 
 	sa.sa_handler = SIG_DFL;
 	sa.sa_flags = 0;
@@ -453,24 +469,20 @@ static void test_pkru_sigreturn(void)
 	 * For this testcase, we do not want to handle SIGSEGV. Reset handler
 	 * to default so that the application can crash if it receives SIGSEGV.
 	 */
-	if (sigaction(SIGSEGV, &sa, NULL) == -1) {
-		perror("sigaction");
-		exit(EXIT_FAILURE);
-	}
+	ret = sigaction(SIGSEGV, &sa, NULL);
+	pkey_assert(ret == 0);
 
 	sa.sa_flags = SA_SIGINFO | SA_ONSTACK;
 	sa.sa_sigaction = sigusr2_handler;
 	sigemptyset(&sa.sa_mask);
 
-	if (sigaction(SIGUSR2, &sa, NULL) == -1) {
-		perror("sigaction");
-		exit(EXIT_FAILURE);
-	}
+	ret = sigaction(SIGUSR2, &sa, NULL);
+	pkey_assert(ret == 0);
 
 	stack = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE,
 		     MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 
-	assert(stack != MAP_FAILED);
+	pkey_assert(stack != MAP_FAILED);
 
 	/*
 	 * Allow access to MPK 0 and MPK 2. The child thread (to be created
@@ -490,21 +502,21 @@ static void test_pkru_sigreturn(void)
 	/* Set up alternate signal stack that will use the default MPK */
 	sigstack.ss_sp = mmap(0, STACK_SIZE, PROT_READ | PROT_WRITE,
 			      MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+	pkey_assert(sigstack.ss_sp != MAP_FAILED);
 	sigstack.ss_flags = 0;
 	sigstack.ss_size = STACK_SIZE;
 
 	/* Use clone to avoid newer glibcs using rseq on new threads */
-	long ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES |
-			     CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM |
-			     CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID |
-			     CLONE_DETACHED,
-			     stack + STACK_SIZE,
-			     &parent_pid,
-			     &child_pid);
+	ret = clone_raw(CLONE_VM | CLONE_FS | CLONE_FILES |
+			CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM |
+			CLONE_DETACHED,
+			stack + STACK_SIZE,
+			NULL,
+			NULL);
 
 	if (ret < 0) {
 		errno = -ret;
-		perror("clone");
+		pkey_assert(0);
 	}  else if (ret == 0) {
 		thread_sigusr2_self(&sigstack);
 		syscall_raw(SYS_exit, 0, 0, 0, 0, 0, 0);
@@ -530,16 +542,17 @@ static void (*pkey_tests[])(void) = {
 
 int main(int argc, char *argv[])
 {
-	int i;
-
 	ksft_print_header();
 	ksft_set_plan(ARRAY_SIZE(pkey_tests));
 
 	if (!is_pkeys_supported())
 		ksft_exit_skip("pkeys not supported\n");
 
-	for (i = 0; i < ARRAY_SIZE(pkey_tests); i++)
-		(*pkey_tests[i])();
+	for (test_nr = 0; test_nr < ARRAY_SIZE(pkey_tests); test_nr++) {
+		tracing_on();
+		(*pkey_tests[test_nr])();
+		tracing_off();
+	}
 
 	ksft_finished();
 	return 0;
diff --git a/tools/testing/selftests/mm/pkey_util.c b/tools/testing/selftests/mm/pkey_util.c
index 255b332f7a08b..fbef3cd45447b 100644
--- a/tools/testing/selftests/mm/pkey_util.c
+++ b/tools/testing/selftests/mm/pkey_util.c
@@ -2,9 +2,99 @@
 #define __SANE_USERSPACE_TYPES__
 #include <sys/syscall.h>
 #include <unistd.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <string.h>
 
 #include "pkey-helpers.h"
 
+int iteration_nr = 1;
+int test_nr;
+int dprint_in_signal;
+
+#if CONTROL_TRACING > 0
+static void cat_into_file(char *str, char *file)
+{
+	int fd = open(file, O_RDWR);
+	int ret;
+
+	dprintf2("%s(): writing '%s' to '%s'\n", __func__, str, file);
+	/*
+	 * these need to be raw because they are called under
+	 * pkey_assert()
+	 */
+	if (fd < 0) {
+		fprintf(stderr, "error opening '%s'\n", file);
+		perror("error: ");
+		exit(__LINE__);
+	}
+
+	ret = write(fd, str, strlen(str));
+	if (ret != strlen(str)) {
+		perror("write to file failed");
+		fprintf(stderr, "filename: '%s' str: '%s'\n", file, str);
+		exit(__LINE__);
+	}
+	close(fd);
+}
+
+static int warned_tracing;
+static int tracing_root_ok(void)
+{
+	if (geteuid() != 0) {
+		if (!warned_tracing)
+			fprintf(stderr, "WARNING: not run as root, "
+					"can not do tracing control\n");
+		warned_tracing = 1;
+		return 0;
+	}
+	return 1;
+}
+#endif
+
+void tracing_on(void)
+{
+#if CONTROL_TRACING > 0
+#define TRACEDIR "/sys/kernel/tracing"
+	char pidstr[32];
+
+	if (!tracing_root_ok())
+		return;
+
+	sprintf(pidstr, "%d", getpid());
+	cat_into_file("0", TRACEDIR "/tracing_on");
+	cat_into_file("\n", TRACEDIR "/trace");
+	if (1) {
+		cat_into_file("function_graph", TRACEDIR "/current_tracer");
+		cat_into_file("1", TRACEDIR "/options/funcgraph-proc");
+	} else {
+		cat_into_file("nop", TRACEDIR "/current_tracer");
+	}
+	cat_into_file(pidstr, TRACEDIR "/set_ftrace_pid");
+	cat_into_file("1", TRACEDIR "/tracing_on");
+	dprintf1("enabled tracing\n");
+#endif
+}
+
+void tracing_off(void)
+{
+#if CONTROL_TRACING > 0
+	if (!tracing_root_ok())
+		return;
+	cat_into_file("0", "/sys/kernel/tracing/tracing_on");
+#endif
+}
+
+void abort_hooks(void)
+{
+	fflush(stdout);
+	fprintf(stderr, "running %s()...\n", __func__);
+	tracing_off();
+#ifdef SLEEP_ON_ABORT
+	sleep(SLEEP_ON_ABORT);
+#endif
+}
+
 int sys_pkey_alloc(unsigned long flags, unsigned long init_val)
 {
 	int ret = syscall(SYS_pkey_alloc, flags, init_val);
diff --git a/tools/testing/selftests/mm/protection_keys.c b/tools/testing/selftests/mm/protection_keys.c
index 9a6d954ee3712..ae6e1530b3548 100644
--- a/tools/testing/selftests/mm/protection_keys.c
+++ b/tools/testing/selftests/mm/protection_keys.c
@@ -49,11 +49,7 @@
 #include "hugepage_settings.h"
 #include "pkey-helpers.h"
 
-int iteration_nr = 1;
-int test_nr;
-
 u64 shadow_pkey_reg;
-int dprint_in_signal;
 
 noinline int read_ptr(int *ptr)
 {
@@ -62,89 +58,6 @@ noinline int read_ptr(int *ptr)
 	return *ptr;
 }
 
-#if CONTROL_TRACING > 0
-static void cat_into_file(char *str, char *file)
-{
-	int fd = open(file, O_RDWR);
-	int ret;
-
-	dprintf2("%s(): writing '%s' to '%s'\n", __func__, str, file);
-	/*
-	 * these need to be raw because they are called under
-	 * pkey_assert()
-	 */
-	if (fd < 0) {
-		fprintf(stderr, "error opening '%s'\n", str);
-		perror("error: ");
-		exit(__LINE__);
-	}
-
-	ret = write(fd, str, strlen(str));
-	if (ret != strlen(str)) {
-		perror("write to file failed");
-		fprintf(stderr, "filename: '%s' str: '%s'\n", file, str);
-		exit(__LINE__);
-	}
-	close(fd);
-}
-
-static int warned_tracing;
-static int tracing_root_ok(void)
-{
-	if (geteuid() != 0) {
-		if (!warned_tracing)
-			fprintf(stderr, "WARNING: not run as root, "
-					"can not do tracing control\n");
-		warned_tracing = 1;
-		return 0;
-	}
-	return 1;
-}
-#endif
-
-static void tracing_on(void)
-{
-#if CONTROL_TRACING > 0
-#define TRACEDIR "/sys/kernel/tracing"
-	char pidstr[32];
-
-	if (!tracing_root_ok())
-		return;
-
-	sprintf(pidstr, "%d", getpid());
-	cat_into_file("0", TRACEDIR "/tracing_on");
-	cat_into_file("\n", TRACEDIR "/trace");
-	if (1) {
-		cat_into_file("function_graph", TRACEDIR "/current_tracer");
-		cat_into_file("1", TRACEDIR "/options/funcgraph-proc");
-	} else {
-		cat_into_file("nop", TRACEDIR "/current_tracer");
-	}
-	cat_into_file(pidstr, TRACEDIR "/set_ftrace_pid");
-	cat_into_file("1", TRACEDIR "/tracing_on");
-	dprintf1("enabled tracing\n");
-#endif
-}
-
-static void tracing_off(void)
-{
-#if CONTROL_TRACING > 0
-	if (!tracing_root_ok())
-		return;
-	cat_into_file("0", "/sys/kernel/tracing/tracing_on");
-#endif
-}
-
-void abort_hooks(void)
-{
-	fflush(stdout);
-	fprintf(stderr, "running %s()...\n", __func__);
-	tracing_off();
-#ifdef SLEEP_ON_ABORT
-	sleep(SLEEP_ON_ABORT);
-#endif
-}
-
 /*
  * This attempts to have roughly a page of instructions followed by a few
  * instructions that do a write, and another page of instructions.  That
@@ -669,7 +582,7 @@ static void *malloc_pkey_with_mprotect(long size, int prot, u16 pkey)
 			size, prot, pkey);
 	pkey_assert(pkey < NR_PKEYS);
 	ptr = mmap(NULL, size, prot, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
-	pkey_assert(ptr != (void *)-1);
+	pkey_assert(ptr != MAP_FAILED);
 	ret = mprotect_pkey((void *)ptr, PAGE_SIZE, prot, pkey);
 	pkey_assert(!ret);
 	record_pkey_malloc(ptr, size, prot);
@@ -692,7 +605,7 @@ static void *malloc_pkey_anon_huge(long size, int prot, u16 pkey)
 	 */
 	size = ALIGN_UP(size, HPAGE_SIZE * 2);
 	ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
-	pkey_assert(ptr != (void *)-1);
+	pkey_assert(ptr != MAP_FAILED);
 	record_pkey_malloc(ptr, size, prot);
 	mprotect_pkey(ptr, size, prot, pkey);
 
@@ -750,7 +663,7 @@ static void *malloc_pkey_hugetlb(long size, int prot, u16 pkey)
 	size = ALIGN_UP(size, HPAGE_SIZE * 2);
 	pkey_assert(pkey < NR_PKEYS);
 	ptr = mmap(NULL, size, PROT_NONE, flags, -1, 0);
-	pkey_assert(ptr != (void *)-1);
+	pkey_assert(ptr != MAP_FAILED);
 	mprotect_pkey(ptr, size, prot, pkey);
 
 	record_pkey_malloc(ptr, size, prot);
@@ -779,7 +692,7 @@ static void *malloc_pkey(long size, int prot, u16 pkey)
 		pkey_assert(malloc_type < nr_malloc_types);
 
 		ret = pkey_malloc[malloc_type](size, prot, pkey);
-		pkey_assert(ret != (void *)-1);
+		pkey_assert(ret != MAP_FAILED);
 
 		malloc_type++;
 		if (malloc_type >= nr_malloc_types)
@@ -1197,6 +1110,7 @@ static void arch_force_pkey_reg_init(void)
 	 * doing the XSAVE size enumeration dance.
 	 */
 	buf = mmap(NULL, 1*MB, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
+	pkey_assert(buf != MAP_FAILED);
 
 	/* These __builtins require compiling with -mxsave */
 
@@ -1763,7 +1677,8 @@ int main(void)
 		ksft_print_msg("running PKEY tests for unsupported CPU/OS\n");
 
 		ptr  = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
-		assert(ptr != (void *)-1);
+		if (ptr == MAP_FAILED)
+			ksft_exit_fail_perror("mmap");
 		test_mprotect_pkey_on_unsupported_cpu(ptr, 1);
 		ksft_test_result_pass("pkey on unsupported CPU/OS\n");
 		ksft_finished();
diff --git a/tools/testing/selftests/mm/rmap.c b/tools/testing/selftests/mm/rmap.c
index 53f2058b0ef2b..1c293ad3f8b83 100644
--- a/tools/testing/selftests/mm/rmap.c
+++ b/tools/testing/selftests/mm/rmap.c
@@ -430,4 +430,85 @@ TEST_F(migrate, ksm)
 	propagate_children(_metadata, data);
 }
 
+static bool range_maps_the_same_pfn(int pagemap_fd, void *region, int nr_pages)
+{
+	int i;
+	int retries = 0;
+	unsigned long first_pfn;
+
+retry:
+	if (retries > 10)
+		return false;
+
+	first_pfn = pagemap_get_pfn(pagemap_fd, region);
+	for (i = 0; i < nr_pages; i++) {
+		if (pagemap_get_pfn(pagemap_fd, region + i * getpagesize()) != first_pfn) {
+			/*
+			 * Retry up to 10 times at most in case of the low chance of page
+			 * compaction migrating the page while we check for pfn.
+			 */
+			retries++;
+			goto retry;
+		}
+	}
+
+	return true;
+}
+
+TEST_F(migrate, ksm_and_mremap)
+{
+	unsigned long old_pfn, new_pfn;
+	void *region, *mremap_region;
+	const int nr_pages = 16;
+	size_t mmap_size;
+	int pagemap_fd;
+
+	/* Skip if KSM is not available */
+	if (ksm_stop() < 0)
+		SKIP(return, "accessing \"/sys/kernel/mm/ksm/run\" failed");
+	if (ksm_get_full_scans() < 0)
+		SKIP(return, "accessing \"/sys/kernel/mm/ksm/full_scan\" failed");
+
+	pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
+	if (pagemap_fd < 0)
+		SKIP(return, "opening pagemap failed");
+
+	/* Allocate and populate twice the anon pages initially. */
+	mmap_size = 2 * nr_pages * getpagesize();
+	region = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE,
+		      MAP_PRIVATE | MAP_ANON, -1, 0);
+	ASSERT_NE(region, MAP_FAILED);
+	memset(region, 0x77, mmap_size);
+
+	/* mremap the second half over the first half, to stress rmap handling */
+	mmap_size /= 2;
+	mremap_region = mremap(region + mmap_size, mmap_size, mmap_size,
+			       MREMAP_MAYMOVE | MREMAP_FIXED, region);
+	ASSERT_EQ(mremap_region, region);
+
+	/* Merge all pages into a single KSM page. */
+	madvise(region, mmap_size, MADV_MERGEABLE);
+	ASSERT_EQ(ksm_start(), 0);
+
+	/* The whole range should map the same KSM page. */
+	old_pfn = pagemap_get_pfn(pagemap_fd, region);
+	if (old_pfn == -1ul)
+		SKIP(return, "Obtaining PFN failed");
+	ksm_start();
+	ASSERT_TRUE(range_maps_the_same_pfn(pagemap_fd, region, nr_pages));
+
+	/*
+	 * Migrate the KSM page; the whole range should map the new (migrated)
+	 * KSM page.
+	 */
+	ASSERT_EQ(try_to_move_page(region), 0);
+
+	new_pfn = pagemap_get_pfn(pagemap_fd, region);
+	if (new_pfn == -1ul)
+		SKIP(return, "Obtaining PFN failed");
+	ASSERT_NE(new_pfn, old_pfn);
+	ASSERT_TRUE(range_maps_the_same_pfn(pagemap_fd, region, nr_pages));
+}
+
+
 TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh
index 8c296dedf0474..687d115e3bd80 100755
--- a/tools/testing/selftests/mm/run_vmtests.sh
+++ b/tools/testing/selftests/mm/run_vmtests.sh
@@ -262,7 +262,6 @@ echo "TAP version 13" | tap_output
 CATEGORY="hugetlb" run_test ./hugetlb-mmap
 CATEGORY="hugetlb" run_test ./hugetlb-shm
 CATEGORY="hugetlb" run_test ./hugetlb-mremap
-CATEGORY="hugetlb" run_test ./hugetlb-vmemmap
 CATEGORY="hugetlb" run_test ./hugetlb-madvise
 CATEGORY="hugetlb" run_test ./hugetlb_dio
 CATEGORY="hugetlb" run_test ./hugetlb_fault_after_madv
@@ -303,7 +302,7 @@ CATEGORY="compaction" run_test ./compaction_test
 
 if command -v sudo &> /dev/null && sudo -u nobody ls ./on-fault-limit >/dev/null;
 then
-	CATEGORY="mlock" run_test sudo -u nobody ./on-fault-limit
+	CATEGORY="mlock" run_test sudo -u nobody sh -c ./on-fault-limit
 else
 	echo "# SKIP ./on-fault-limit"
 fi
diff --git a/tools/testing/selftests/mm/uffd-common.c b/tools/testing/selftests/mm/uffd-common.c
index edd02328f77ba..f48f5d4594ab0 100644
--- a/tools/testing/selftests/mm/uffd-common.c
+++ b/tools/testing/selftests/mm/uffd-common.c
@@ -639,8 +639,13 @@ int __copy_page(uffd_global_test_opts_t *gopts, unsigned long offset, bool retry
 		uffdio_copy.mode = 0;
 	uffdio_copy.copy = 0;
 	if (ioctl(gopts->uffd, UFFDIO_COPY, &uffdio_copy)) {
-		/* real retval in ufdio_copy.copy */
-		if (uffdio_copy.copy != -EEXIST)
+		/*
+		 * real retval in uffdio_copy.copy
+		 *
+		 * -EEXIST: the page was faulted in concurrently
+		 * -ENOENT: the destination range was concurrently removed
+		 */
+		if (uffdio_copy.copy != -EEXIST && uffdio_copy.copy != -ENOENT)
 			err("UFFDIO_COPY error: %"PRId64,
 			    (int64_t)uffdio_copy.copy);
 		wake_range(gopts->uffd, uffdio_copy.dst, gopts->page_size);
diff --git a/tools/testing/selftests/mm/uffd-unit-tests.c b/tools/testing/selftests/mm/uffd-unit-tests.c
index a6c14109e8188..ef9b3956bdcfd 100644
--- a/tools/testing/selftests/mm/uffd-unit-tests.c
+++ b/tools/testing/selftests/mm/uffd-unit-tests.c
@@ -7,6 +7,8 @@
 
 #include "uffd-common.h"
 
+#include <linux/fs.h>
+#include <sys/uio.h>
 #include "../../../../mm/gup_test.h"
 
 #ifdef __NR_userfaultfd
@@ -109,6 +111,10 @@ static void uffd_test_skip(const char *message)
 
 static void test_uffd_api(bool use_dev)
 {
+	const uint64_t expected_ioctls =
+		BIT_ULL(_UFFDIO_REGISTER) |
+		BIT_ULL(_UFFDIO_UNREGISTER) |
+		BIT_ULL(_UFFDIO_API);
 	struct uffdio_api uffdio_api;
 	int uffd;
 
@@ -148,6 +154,15 @@ static void test_uffd_api(bool use_dev)
 		goto out;
 	}
 
+	/* Verify returned fd-level ioctls bitmask */
+	if ((uffdio_api.ioctls & expected_ioctls) != expected_ioctls) {
+		uffd_test_fail("UFFDIO_API missing expected ioctls: "
+			       "got=0x%"PRIx64", expected=0x%"PRIx64,
+			       (uint64_t)uffdio_api.ioctls,
+			       expected_ioctls);
+		goto out;
+	}
+
 	/* Test double requests of UFFDIO_API with a random feature set */
 	uffdio_api.features = BIT_ULL(0);
 	if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
@@ -602,6 +617,814 @@ void uffd_minor_collapse_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *
 	uffd_minor_test_common(gopts, true, false);
 }
 
+static int uffd_register_rwp(int uffd, void *addr, uint64_t len)
+{
+	struct uffdio_register reg = {
+		.range = { .start = (unsigned long)addr, .len = len },
+		.mode = UFFDIO_REGISTER_MODE_RWP,
+	};
+
+	if (ioctl(uffd, UFFDIO_REGISTER, &reg) == -1)
+		return -errno;
+	return 0;
+}
+
+static void rwprotect_range(int uffd, __u64 start, __u64 len, bool protect)
+{
+	struct uffdio_rwprotect rwp = {
+		.range = { .start = start, .len = len },
+		.mode = protect ? UFFDIO_RWPROTECT_MODE_RWP : 0,
+	};
+
+	if (ioctl(uffd, UFFDIO_RWPROTECT, &rwp))
+		err("UFFDIO_RWPROTECT failed");
+}
+
+static void set_async_mode(int uffd, bool enable)
+{
+	struct uffdio_set_mode mode = { };
+
+	if (enable)
+		mode.enable = UFFD_FEATURE_RWP_ASYNC;
+	else
+		mode.disable = UFFD_FEATURE_RWP_ASYNC;
+
+	if (ioctl(uffd, UFFDIO_SET_MODE, &mode))
+		err("UFFDIO_SET_MODE failed");
+}
+
+/*
+ * Test async RWP faults on anonymous memory.
+ * Populate pages, register MODE_RWP with RWP_ASYNC,
+ * RW-protect, re-access, verify content preserved and no faults delivered.
+ */
+static void uffd_rwp_async_test(uffd_global_test_opts_t *gopts,
+				       uffd_test_args_t *args)
+{
+	unsigned long nr_pages = gopts->nr_pages;
+	unsigned long page_size = gopts->page_size;
+	unsigned long p;
+
+	/* Populate all pages with known content */
+	for (p = 0; p < nr_pages; p++)
+		memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size);
+
+	/* Register MODE_RWP */
+	if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
+			  nr_pages * page_size))
+		err("register failure");
+
+	/* RW-protect all pages (sets protnone) */
+	rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
+			 nr_pages * page_size, true);
+
+	/* Access all pages — should auto-resolve, no faults */
+	for (p = 0; p < nr_pages; p++) {
+		unsigned char *page = (unsigned char *)gopts->area_dst +
+				      p * page_size;
+		unsigned char expected = p % 255 + 1;
+
+		if (page[0] != expected) {
+			uffd_test_fail("page %lu content mismatch: %u != %u",
+				       p, page[0], expected);
+			return;
+		}
+	}
+
+	uffd_test_pass();
+}
+
+/*
+ * Fault handler for RWP — unprotect the page via UFFDIO_RWPROTECT.
+ */
+static void uffd_handle_rwp_fault(uffd_global_test_opts_t *gopts,
+				  struct uffd_msg *msg,
+				  struct uffd_args *uargs)
+{
+	if (!(msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_RWP))
+		err("expected RWP fault, got 0x%llx",
+		    msg->arg.pagefault.flags);
+
+	rwprotect_range(gopts->uffd, msg->arg.pagefault.address,
+			gopts->page_size, false);
+	uargs->minor_faults++;
+}
+
+/*
+ * Test sync RWP faults on anonymous memory.
+ * Populate pages, register MODE_RWP (sync), RW-protect,
+ * access from worker thread, verify fault delivered, UFFDIO_RWPROTECT resolves.
+ */
+static void uffd_rwp_sync_test(uffd_global_test_opts_t *gopts,
+				      uffd_test_args_t *args)
+{
+	unsigned long nr_pages = gopts->nr_pages;
+	unsigned long page_size = gopts->page_size;
+	pthread_t uffd_mon;
+	struct uffd_args uargs = { };
+	bool failed = false;
+	char c = '\0';
+	unsigned long p;
+
+	uargs.gopts = gopts;
+	uargs.handle_fault = uffd_handle_rwp_fault;
+
+	/* Populate all pages */
+	for (p = 0; p < nr_pages; p++)
+		memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size);
+
+	/* Register MODE_RWP */
+	if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
+			  nr_pages * page_size))
+		err("register failure");
+
+	/* RW-protect all pages */
+	rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
+			 nr_pages * page_size, true);
+
+	/* Start fault handler thread */
+	if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs))
+		err("uffd_poll_thread create");
+
+	/* Access all pages — triggers sync RWP faults, handler unprotects */
+	for (p = 0; p < nr_pages; p++) {
+		unsigned char *page = (unsigned char *)gopts->area_dst +
+				      p * page_size;
+
+		if (page[0] != (p % 255 + 1)) {
+			uffd_test_fail("page %lu content mismatch", p);
+			failed = true;
+			goto out;
+		}
+	}
+
+out:
+	/*
+	 * Stop the handler before reading minor_faults: the last fault
+	 * resolution rwprotect_range()s before incrementing the counter,
+	 * so the main thread can race ahead of the increment.
+	 */
+	if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
+		err("pipe write");
+	if (pthread_join(uffd_mon, NULL))
+		err("join() failed");
+
+	if (failed)
+		return;
+	if (uargs.minor_faults == 0)
+		uffd_test_fail("expected RWP faults, got 0");
+	else
+		uffd_test_pass();
+}
+
+/*
+ * Test PAGEMAP_SCAN working-set discovery via the "hot" (accessed) scan.
+ *
+ * The working-set primitive is to find pages that were accessed: scan for
+ * PAGE_IS_ACCESSED, which is set once an access clears the protnone+uffd
+ * marker.  A VMM treats every access fault as "hot" (RWP here; MINOR/MISSING
+ * for non-resident pages) and reclaims the rest from the backing file.
+ *
+ * We deliberately do NOT use an inverted "cold" scan: that only sees
+ * VMA-resident ptes, so for a file mapping it misses cached-but-unmapped (and
+ * never-faulted, pre-populated) pages, which are pte_none and thus invisible.
+ * Hot tracking + file-level reclaim covers them; a cold pte scan cannot.
+ */
+static void uffd_rwp_pagemap_test(uffd_global_test_opts_t *gopts,
+					  uffd_test_args_t *args)
+{
+	unsigned long nr_pages = gopts->nr_pages;
+	unsigned long page_size = gopts->page_size;
+	unsigned long p;
+	struct page_region regions[16];
+	struct pm_scan_arg pm_arg;
+	int pagemap_fd;
+	long ret;
+
+	/* Need at least 4 pages */
+	if (nr_pages < 4) {
+		uffd_test_skip("need at least 4 pages");
+		return;
+	}
+
+	/* Populate all pages */
+	for (p = 0; p < nr_pages; p++)
+		memset(gopts->area_dst + p * page_size, 0xab, page_size);
+
+	/* Register and RW-protect */
+	if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
+			  nr_pages * page_size))
+		err("register failure");
+
+	rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
+			 nr_pages * page_size, true);
+
+	/* Touch first half of pages to re-activate them (async auto-resolve) */
+	for (p = 0; p < nr_pages / 2; p++) {
+		volatile char *page = gopts->area_dst + p * page_size;
+		(void)*page;
+	}
+
+	uint64_t start = (uint64_t)gopts->area_dst;
+	uint64_t boundary = start + (nr_pages / 2) * page_size;
+	uint64_t end = start + nr_pages * page_size;
+
+	pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
+	if (pagemap_fd < 0)
+		err("open pagemap");
+
+	/*
+	 * Hot scan: report the pages that were accessed.  PAGE_IS_ACCESSED is
+	 * set once the protnone+uffd marker is cleared (by the access, async
+	 * auto-resolve here).  The touched first half must come back as exactly
+	 * one hot region [start, boundary); the untouched second half must not
+	 * appear.
+	 */
+	memset(&pm_arg, 0, sizeof(pm_arg));
+	pm_arg.size = sizeof(pm_arg);
+	pm_arg.start = start;
+	pm_arg.end = end;
+	pm_arg.vec = (uint64_t)regions;
+	pm_arg.vec_len = ARRAY_SIZE(regions);
+	pm_arg.category_mask = PAGE_IS_ACCESSED;
+	pm_arg.return_mask = PAGE_IS_ACCESSED;
+
+	ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &pm_arg);
+	close(pagemap_fd);
+
+	if (ret < 0) {
+		uffd_test_fail("PAGEMAP_SCAN failed: %s", strerror(errno));
+		return;
+	}
+
+	if (ret != 1 || regions[0].start != start ||
+	    regions[0].end != boundary) {
+		uffd_test_fail("hot set wrong: got %ld regions [0x%lx,0x%lx), expected 1 [0x%lx,0x%lx)",
+			       ret, (unsigned long)regions[0].start,
+			       (unsigned long)regions[0].end,
+			       (unsigned long)start, (unsigned long)boundary);
+		return;
+	}
+
+	uffd_test_pass();
+}
+
+/*
+ * Test that RWP protection survives a mprotect(PROT_NONE) ->
+ * mprotect(PROT_READ|PROT_WRITE) round-trip. The uffd-wp bit on a
+ * VM_UFFD_RWP VMA must continue to carry PROT_NONE semantics after
+ * mprotect() changes the base protection; otherwise accesses would
+ * silently succeed and the pagemap bit would stick without a fault
+ * ever clearing it.
+ */
+static void uffd_rwp_mprotect_test(uffd_global_test_opts_t *gopts,
+				   uffd_test_args_t *args)
+{
+	unsigned long nr_pages = gopts->nr_pages;
+	unsigned long page_size = gopts->page_size;
+	unsigned long p;
+	struct page_region regions[16];
+	struct pm_scan_arg pm_arg;
+	int pagemap_fd;
+	uint64_t value;
+	long ret;
+
+	/* Populate all pages */
+	for (p = 0; p < nr_pages; p++)
+		memset(gopts->area_dst + p * page_size, 0xab, page_size);
+
+	/* Register and RW-protect the whole range */
+	if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
+			      nr_pages * page_size))
+		err("register failure");
+	rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
+			nr_pages * page_size, true);
+
+	/* Round-trip mprotect(): PROT_NONE -> PROT_READ|PROT_WRITE */
+	if (mprotect(gopts->area_dst, nr_pages * page_size, PROT_NONE))
+		err("mprotect() PROT_NONE");
+	if (mprotect(gopts->area_dst, nr_pages * page_size,
+		     PROT_READ | PROT_WRITE))
+		err("mprotect() PROT_READ|PROT_WRITE");
+
+	/*
+	 * The marker must survive the round-trip; if mprotect() dropped it,
+	 * the touches below would not fault and the scan would pass
+	 * vacuously.
+	 */
+	pagemap_fd = pagemap_open();
+	value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
+	close(pagemap_fd);
+	if (!(value & PM_UFFD_WP)) {
+		uffd_test_fail("RWP marker lost across mprotect()");
+		return;
+	}
+
+	/* Touch every page. Async RWP must auto-resolve each fault. */
+	for (p = 0; p < nr_pages; p++) {
+		volatile char *page = gopts->area_dst + p * page_size;
+		(void)*page;
+	}
+
+	/*
+	 * After touching, no page should remain RW-protected. A stuck
+	 * uffd-wp bit would mean mprotect() silently dropped PROT_NONE and
+	 * the access never faulted.
+	 */
+	pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
+	if (pagemap_fd < 0)
+		err("open pagemap");
+
+	memset(&pm_arg, 0, sizeof(pm_arg));
+	pm_arg.size = sizeof(pm_arg);
+	pm_arg.start = (uint64_t)gopts->area_dst;
+	pm_arg.end = (uint64_t)gopts->area_dst + nr_pages * page_size;
+	pm_arg.vec = (uint64_t)regions;
+	pm_arg.vec_len = ARRAY_SIZE(regions);
+	pm_arg.category_mask = PAGE_IS_ACCESSED;
+	pm_arg.category_inverted = PAGE_IS_ACCESSED;
+	pm_arg.return_mask = PAGE_IS_ACCESSED;
+
+	ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &pm_arg);
+	close(pagemap_fd);
+
+	if (ret < 0) {
+		uffd_test_fail("PAGEMAP_SCAN failed: %s", strerror(errno));
+		return;
+	}
+	if (ret != 0) {
+		uffd_test_fail("expected no cold pages after mprotect()+touch, got %ld regions",
+			       ret);
+		return;
+	}
+
+	uffd_test_pass();
+}
+
+/*
+ * Test that GUP resolves through protnone PTEs (async mode).
+ * vmsplice() into a pipe pins user pages via get_user_pages_fast() --
+ * unlike write(), which goes through copy_from_user() and ordinary
+ * hardware page faults -- so it exercises gup_can_follow_protnone() on
+ * the RW-protected PTE. In async mode the kernel auto-restores
+ * permissions and GUP returns the page.
+ */
+static void uffd_rwp_gup_test(uffd_global_test_opts_t *gopts,
+				     uffd_test_args_t *args)
+{
+	struct iovec iov;
+	char buf;
+	int pipefd[2];
+
+	/* Populate first page with known content */
+	memset(gopts->area_dst, 0xCD, gopts->page_size);
+
+	if (uffd_register_rwp(gopts->uffd, gopts->area_dst, gopts->page_size))
+		err("register failure");
+
+	rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
+			gopts->page_size, true);
+
+	if (pipe(pipefd))
+		err("pipe");
+
+	/*
+	 * One byte's worth of iov is enough to GUP the containing page and
+	 * keeps the pipe transfer well under any pipe-capacity limit even on
+	 * hugetlb-backed runs.
+	 */
+	iov.iov_base = gopts->area_dst;
+	iov.iov_len = 1;
+	if (vmsplice(pipefd[1], &iov, 1, 0) != 1) {
+		uffd_test_fail("vmsplice from RW-protected page failed: %s",
+			       strerror(errno));
+		goto out;
+	}
+
+	if (read(pipefd[0], &buf, 1) != 1) {
+		uffd_test_fail("read from pipe failed");
+		goto out;
+	}
+
+	if (buf != (char)0xCD) {
+		uffd_test_fail("content mismatch: got 0x%02x, expected 0xCD",
+			       (unsigned char)buf);
+		goto out;
+	}
+
+	uffd_test_pass();
+out:
+	close(pipefd[0]);
+	close(pipefd[1]);
+}
+
+/*
+ * Test runtime toggle between async and sync modes.
+ * Start in async mode (detection), flip to sync (eviction), verify faults
+ * block, resolve them, flip back to async.
+ */
+static void uffd_rwp_async_toggle_test(uffd_global_test_opts_t *gopts,
+					      uffd_test_args_t *args)
+{
+	unsigned long nr_pages = gopts->nr_pages;
+	unsigned long page_size = gopts->page_size;
+	struct uffd_args uargs = { };
+	pthread_t uffd_mon;
+	char c = '\0';
+	unsigned long p;
+
+	uargs.gopts = gopts;
+	uargs.handle_fault = uffd_handle_rwp_fault;
+
+	/* Populate */
+	for (p = 0; p < nr_pages; p++)
+		memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size);
+
+	if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
+			  nr_pages * page_size))
+		err("register failure");
+
+	/* Phase 1: async detection — RW-protect, access first half */
+	rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
+			 nr_pages * page_size, true);
+
+	for (p = 0; p < nr_pages / 2; p++) {
+		volatile char *page = gopts->area_dst + p * page_size;
+		(void)*page;  /* auto-resolves in async mode */
+	}
+
+	/* Phase 2: flip to sync for eviction */
+	set_async_mode(gopts->uffd, false);
+
+	/* Start handler — will receive faults for cold pages */
+	if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs))
+		err("uffd_poll_thread create");
+
+	/* Access second half (cold pages) — should trigger sync faults */
+	for (p = nr_pages / 2; p < nr_pages; p++) {
+		unsigned char *page = (unsigned char *)gopts->area_dst +
+				      p * page_size;
+		if (page[0] != (p % 255 + 1)) {
+			uffd_test_fail("page %lu content mismatch", p);
+			goto out;
+		}
+	}
+
+	/*
+	 * Stop the handler before reading minor_faults: the last fault
+	 * resolution rwprotect_range()s before incrementing the counter,
+	 * so the main thread can race ahead of the increment. Stopping
+	 * here also makes Phase 3 a clean async-only test -- with the
+	 * handler still running it would silently resolve any sync fault
+	 * the kernel erroneously delivers, masking a regression.
+	 */
+	if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
+		err("pipe write");
+	if (pthread_join(uffd_mon, NULL))
+		err("join() failed");
+
+	if (uargs.minor_faults == 0) {
+		uffd_test_fail("expected sync faults, got 0");
+		return;
+	}
+
+	/* Phase 3: flip back to async */
+	set_async_mode(gopts->uffd, true);
+
+	/* RW-protect and access again — should auto-resolve */
+	rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
+			 nr_pages * page_size, true);
+
+	for (p = 0; p < nr_pages; p++) {
+		volatile char *page = gopts->area_dst + p * page_size;
+		(void)*page;
+	}
+
+	uffd_test_pass();
+	return;
+out:
+	if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
+		err("pipe write");
+	if (pthread_join(uffd_mon, NULL))
+		err("join() failed");
+}
+
+/*
+ * Test that RW-protected pages become accessible after closing uffd.
+ */
+static void uffd_rwp_close_test(uffd_global_test_opts_t *gopts,
+				       uffd_test_args_t *args)
+{
+	unsigned long nr_pages = gopts->nr_pages;
+	unsigned long page_size = gopts->page_size;
+	unsigned long p;
+
+	/* Populate */
+	for (p = 0; p < nr_pages; p++)
+		memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size);
+
+	if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
+			  nr_pages * page_size))
+		err("register failure");
+
+	rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
+			 nr_pages * page_size, true);
+
+	/* Close uffd — should restore protnone PTEs */
+	close(gopts->uffd);
+	gopts->uffd = -1;
+
+	/* All pages should be accessible with original content */
+	for (p = 0; p < nr_pages; p++) {
+		unsigned char *page = (unsigned char *)gopts->area_dst +
+				      p * page_size;
+		unsigned char expected = p % 255 + 1;
+
+		if (page[0] != expected) {
+			uffd_test_fail("page %lu not accessible after close", p);
+			return;
+		}
+	}
+
+	uffd_test_pass();
+}
+
+/*
+ * Test that RWP protection is preserved across fork() when
+ * UFFD_FEATURE_EVENT_FORK is enabled. Without preservation, the child's
+ * PTEs would lose the uffd-wp marker and RWP-protected accesses would
+ * silently fall through to do_numa_page().
+ */
+static void uffd_rwp_fork_test(uffd_global_test_opts_t *gopts,
+			       uffd_test_args_t *args)
+{
+	unsigned long nr_pages = gopts->nr_pages;
+	unsigned long page_size = gopts->page_size;
+	int pagemap_fd;
+	uint64_t value;
+
+	if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
+			      nr_pages * page_size))
+		err("register failed");
+
+	/* Populate + RWP-protect */
+	*gopts->area_dst = 1;
+	rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
+			page_size, true);
+
+	/* Parent: verify uffd-wp bit is set before fork */
+	pagemap_fd = pagemap_open();
+	value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
+	pagemap_check_wp(value, true);
+
+	/*
+	 * Fork with EVENT_FORK: child inherits VM_UFFD_RWP. Child reads
+	 * its own pagemap and must still see the uffd-wp bit set.
+	 */
+	if (pagemap_test_fork(gopts, true, false)) {
+		uffd_test_fail("RWP marker lost in child after fork");
+		goto out;
+	}
+
+	uffd_test_pass();
+out:
+	close(pagemap_fd);
+}
+
+/*
+ * Test that RWP protection on a pinned anon page is preserved across fork().
+ * Pinning forces copy_present_page() in the child path, which must restore
+ * PAGE_NONE on top of the uffd bit. Using async mode, a read in the child
+ * auto-resolves if — and only if — the PTE was actually protnone+uffd; the
+ * cleared uffd bit afterward proves the fault path ran.
+ */
+static void uffd_rwp_fork_pin_test(uffd_global_test_opts_t *gopts,
+				   uffd_test_args_t *args)
+{
+	unsigned long page_size = gopts->page_size;
+	fork_event_args fevent_args = { .gopts = gopts, .child_uffd = -1 };
+	pin_args pin_args = {};
+	int pagemap_fd, status;
+	pthread_t fevent_thread;
+	uint64_t value;
+	pid_t child;
+
+	if (uffd_register_rwp(gopts->uffd, gopts->area_dst, page_size))
+		err("register failed");
+
+	/* Populate. */
+	*gopts->area_dst = 1;
+
+	/* RO-longterm pin so fork() takes copy_present_page() for this PTE. */
+	if (pin_pages(&pin_args, gopts->area_dst, page_size)) {
+		uffd_test_skip("Possibly CONFIG_GUP_TEST missing or unprivileged");
+		uffd_unregister(gopts->uffd, gopts->area_dst, page_size);
+		return;
+	}
+
+	/* RWP-protect: PTE is now PAGE_NONE + uffd bit. */
+	rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, page_size, true);
+
+	pagemap_fd = pagemap_open();
+	value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
+	pagemap_check_wp(value, true);
+
+	/*
+	 * UFFD_FEATURE_EVENT_FORK is required so the child inherits
+	 * VM_UFFD_RWP and the marker; without it dup_userfaultfd() resets
+	 * the child VMA and the test would pass for the wrong reason.
+	 * dup_userfaultfd() blocks until the EVENT_FORK message is consumed,
+	 * so spawn a reader before the fork().
+	 */
+	gopts->ready_for_fork = false;
+	if (pthread_create(&fevent_thread, NULL, fork_event_consumer,
+			   &fevent_args))
+		err("pthread_create() for fork event consumer");
+	while (!gopts->ready_for_fork)
+		; /* Wait for consumer to start polling. */
+
+	child = fork();
+	if (child < 0)
+		err("fork");
+	if (child == 0) {
+		volatile char c;
+		int cfd;
+
+		/*
+		 * Precondition: the child must have inherited the marker.
+		 * If copy_present_page() dropped it together with PAGE_NONE,
+		 * the read below would succeed without the fault path and
+		 * the after-read check would pass for the wrong reason.
+		 */
+		cfd = pagemap_open();
+		value = pagemap_get_entry(cfd, gopts->area_dst);
+		if (!(value & PM_UFFD_WP)) {
+			close(cfd);
+			_exit(2);
+		}
+
+		/*
+		 * Read the pinned page. Only reaches the fault path if the
+		 * child PTE is protnone + uffd; async mode auto-resolves and
+		 * clears the uffd bit. If copy_present_page() dropped
+		 * PAGE_NONE, the read would silently succeed and the bit
+		 * would still be set.
+		 */
+		c = *(volatile char *)gopts->area_dst;
+		(void)c;
+
+		value = pagemap_get_entry(cfd, gopts->area_dst);
+		close(cfd);
+		_exit((value & PM_UFFD_WP) ? 1 : 0);
+	}
+	if (waitpid(child, &status, 0) < 0)
+		err("waitpid");
+	if (pthread_join(fevent_thread, NULL))
+		err("pthread_join() for fork event consumer");
+	if (fevent_args.child_uffd >= 0)
+		close(fevent_args.child_uffd);
+
+	unpin_pages(&pin_args);
+	close(pagemap_fd);
+	if (uffd_unregister(gopts->uffd, gopts->area_dst, page_size))
+		err("unregister failed");
+
+	if (WIFEXITED(status) && WEXITSTATUS(status) == 2) {
+		uffd_test_fail("RWP marker not inherited by child");
+		return;
+	}
+	if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
+		uffd_test_fail("RWP not enforced in child after pinned fork");
+		return;
+	}
+
+	uffd_test_pass();
+}
+
+/*
+ * A non-exclusive (forked, COW-shared) anon page that is RWP-protected and
+ * then swapped out must keep tracking across swap-in. On the write that
+ * swaps it back in, do_swap_page() restores PAGE_NONE and the access retries
+ * through the RWP fault path, instead of being COWed straight to a fresh
+ * accessible page -- which would silently drop the marker for a non-exclusive
+ * folio. Sync mode lets us observe the fault directly: with the bug, the
+ * write COWs without delivering any RWP fault.
+ *
+ * Needs a swap device; skipped if MADV_PAGEOUT cannot evict the page.
+ */
+static void uffd_rwp_swap_cow_test(uffd_global_test_opts_t *gopts,
+				   uffd_test_args_t *args)
+{
+	unsigned long page_size = gopts->page_size;
+	struct uffd_args uargs = { };
+	int pagemap_fd, go[2], i;
+	pthread_t uffd_mon;
+	char c = '\0';
+	pid_t child;
+
+	uargs.gopts = gopts;
+	uargs.handle_fault = uffd_handle_rwp_fault;
+
+	if (uffd_register_rwp(gopts->uffd, gopts->area_dst, page_size))
+		err("register failed");
+
+	/* Populate one page (exclusive at this point). */
+	*gopts->area_dst = 0x11;
+
+	/* RWP-protect: PTE becomes PAGE_NONE + uffd bit (still exclusive). */
+	rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, page_size, true);
+
+	/*
+	 * Swap the page out while it is still exclusive: a shared (forked)
+	 * folio does not get reclaimed by MADV_PAGEOUT. Retry, since a hot
+	 * page may just be rotated on the first reclaim pass.
+	 */
+	pagemap_fd = pagemap_open();
+	for (i = 0; i < 100; i++) {
+		if (madvise(gopts->area_dst, page_size, MADV_PAGEOUT))
+			err("MADV_PAGEOUT");
+		if (pagemap_is_swapped(pagemap_fd, gopts->area_dst))
+			break;
+		usleep(10000);
+	}
+	if (!pagemap_is_swapped(pagemap_fd, gopts->area_dst)) {
+		uffd_test_skip("MADV_PAGEOUT did not swap the page; is swap enabled?");
+		close(pagemap_fd);
+		uffd_unregister(gopts->uffd, gopts->area_dst, page_size);
+		return;
+	}
+
+	/*
+	 * fork() now: the child duplicates the swap entry, so the slot becomes
+	 * non-exclusive. The child parks (keeping the reference) until the
+	 * parent has faulted the page back in.
+	 */
+	if (pipe(go))
+		err("pipe");
+	child = fork();
+	if (child < 0)
+		err("fork");
+	if (child == 0) {
+		close(go[1]);
+		read(go[0], &c, 1);
+		_exit(0);
+	}
+	close(go[0]);
+
+	if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs))
+		err("uffd_poll_thread create");
+
+	/*
+	 * Write the page: swaps it back in (do_swap_page) on a non-exclusive
+	 * folio with FAULT_FLAG_WRITE. The marker must survive and deliver an
+	 * RWP fault rather than COW silently.
+	 */
+	*gopts->area_dst = 0x22;
+
+	if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
+		err("pipe write");
+	if (pthread_join(uffd_mon, NULL))
+		err("join failed");
+
+	if (uargs.minor_faults == 0)
+		uffd_test_fail("no RWP fault on swapped-in non-exclusive page");
+	else
+		uffd_test_pass();
+
+	close(pagemap_fd);
+	if (write(go[1], &c, 1) != 1)
+		err("child release");
+	close(go[1]);
+	waitpid(child, NULL, 0);
+}
+
+/*
+ * WP and RWP share the uffd-wp PTE bit and cannot coexist in the same VMA.
+ * Registration requesting both modes must be rejected.
+ */
+static void uffd_rwp_wp_exclusive_test(uffd_global_test_opts_t *gopts,
+				       uffd_test_args_t *args)
+{
+	unsigned long nr_pages = gopts->nr_pages;
+	unsigned long page_size = gopts->page_size;
+	struct uffdio_register reg = { };
+
+	reg.range.start = (unsigned long)gopts->area_dst;
+	reg.range.len = nr_pages * page_size;
+	reg.mode = UFFDIO_REGISTER_MODE_WP | UFFDIO_REGISTER_MODE_RWP;
+
+	if (ioctl(gopts->uffd, UFFDIO_REGISTER, &reg) == 0) {
+		uffd_test_fail("register with WP|RWP unexpectedly succeeded");
+		return;
+	}
+	if (errno != EINVAL) {
+		uffd_test_fail("register with WP|RWP: expected EINVAL, got %d",
+			       errno);
+		return;
+	}
+	uffd_test_pass();
+}
+
 static sigjmp_buf jbuf, *sigbuf;
 
 static void sighndl(int sig, siginfo_t *siginfo, void *ptr)
@@ -1604,6 +2427,83 @@ uffd_test_case_t uffd_tests[] = {
 		/* We can't test MADV_COLLAPSE, so try our luck */
 		.uffd_feature_required = UFFD_FEATURE_MINOR_SHMEM,
 	},
+	{
+		.name = "rwp-async",
+		.uffd_fn = uffd_rwp_async_test,
+		.mem_targets = MEM_ALL,
+		.uffd_feature_required =
+		UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC,
+	},
+	{
+		.name = "rwp-sync",
+		.uffd_fn = uffd_rwp_sync_test,
+		.mem_targets = MEM_ALL,
+		.uffd_feature_required = UFFD_FEATURE_RWP,
+	},
+	{
+		.name = "rwp-pagemap",
+		.uffd_fn = uffd_rwp_pagemap_test,
+		.mem_targets = MEM_ALL,
+		.uffd_feature_required =
+		UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC,
+	},
+	{
+		.name = "rwp-mprotect",
+		.uffd_fn = uffd_rwp_mprotect_test,
+		.mem_targets = MEM_ALL,
+		.uffd_feature_required =
+		UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC,
+	},
+	{
+		.name = "rwp-gup",
+		.uffd_fn = uffd_rwp_gup_test,
+		.mem_targets = MEM_ALL,
+		.uffd_feature_required =
+		UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC,
+	},
+	{
+		.name = "rwp-async-toggle",
+		.uffd_fn = uffd_rwp_async_toggle_test,
+		.mem_targets = MEM_ALL,
+		.uffd_feature_required =
+		UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC,
+	},
+	{
+		.name = "rwp-close",
+		.uffd_fn = uffd_rwp_close_test,
+		.mem_targets = MEM_ALL,
+		.uffd_feature_required = UFFD_FEATURE_RWP,
+	},
+	{
+		.name = "rwp-fork",
+		.uffd_fn = uffd_rwp_fork_test,
+		.mem_targets = MEM_ALL,
+		.uffd_feature_required =
+		UFFD_FEATURE_RWP | UFFD_FEATURE_EVENT_FORK,
+	},
+	{
+		.name = "rwp-fork-pin",
+		.uffd_fn = uffd_rwp_fork_pin_test,
+		.mem_targets = MEM_ANON,
+		.uffd_feature_required =
+		UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC |
+		UFFD_FEATURE_EVENT_FORK,
+	},
+	{
+		.name = "rwp-swap-cow",
+		.uffd_fn = uffd_rwp_swap_cow_test,
+		.mem_targets = MEM_ANON,
+		.uffd_feature_required = UFFD_FEATURE_RWP,
+	},
+	{
+		.name = "rwp-wp-exclusive",
+		.uffd_fn = uffd_rwp_wp_exclusive_test,
+		.mem_targets = MEM_ALL,
+		.uffd_feature_required =
+		UFFD_FEATURE_RWP |
+		UFFD_FEATURE_PAGEFAULT_FLAG_WP |
+		UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
+	},
 	{
 		.name = "sigbus",
 		.uffd_fn = uffd_sigbus_test,
@@ -1814,6 +2714,25 @@ int main(int argc, char *argv[])
 				uffd_test_skip(errmsg);
 				continue;
 			}
+			/*
+			 * RWP tracks protection on ptes; a THP-backed shmem/anon
+			 * range (e.g. shmem_enabled=always) would split on
+			 * rwprotect and change behaviour under the test. Keep
+			 * such ranges off THP. hugetlb is huge by definition and
+			 * rejects MADV_NOHUGEPAGE, so skip it.
+			 */
+			if ((test->uffd_feature_required & UFFD_FEATURE_RWP) &&
+			    !(mem_type->mem_flag & (MEM_HUGETLB | MEM_HUGETLB_PRIVATE))) {
+				unsigned long len = gopts.nr_pages * gopts.page_size;
+
+				/*
+				 * EINVAL means CONFIG_TRANSPARENT_HUGEPAGE=n:
+				 * nothing to opt out of.
+				 */
+				if (madvise(gopts.area_dst, len, MADV_NOHUGEPAGE) &&
+				    errno != EINVAL)
+					err("madvise(MADV_NOHUGEPAGE)");
+			}
 			test->uffd_fn(&gopts, &args);
 			uffd_test_ctx_clear(&gopts);
 		}
diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c
index 311fc5b4513eb..ef1ea11981a74 100644
--- a/tools/testing/selftests/mm/vm_util.c
+++ b/tools/testing/selftests/mm/vm_util.c
@@ -719,7 +719,7 @@ int read_file(const char *path, char *buf, size_t buflen)
 	return (unsigned int) numread;
 }
 
-void write_file(const char *path, const char *buf, size_t buflen)
+static void __write_file(const char *path, const char *buf, size_t buflen, bool ignore_einval)
 {
 	int fd, saved_errno;
 	ssize_t numwritten;
@@ -735,14 +735,22 @@ void write_file(const char *path, const char *buf, size_t buflen)
 	saved_errno = errno;
 	close(fd);
 	errno = saved_errno;
-	if (numwritten < 0)
+	if (numwritten < 0) {
+		if (ignore_einval && errno == EINVAL)
+			return;
 		ksft_exit_fail_msg("%s write(%.*s) failed: %s\n", path, (int)(buflen - 1),
 				buf, strerror(errno));
+	}
 	if (numwritten != buflen - 1)
 		ksft_exit_fail_msg("%s write(%.*s) is truncated, expected %zu bytes, got %zd bytes\n",
 				path, (int)(buflen - 1), buf, buflen - 1, numwritten);
 }
 
+void write_file(const char *path, const char *buf, size_t buflen)
+{
+	__write_file(path, buf, buflen, /* ignore_einval = */ false);
+}
+
 unsigned long read_num(const char *path)
 {
 	char buf[21];
@@ -753,12 +761,22 @@ unsigned long read_num(const char *path)
 	return strtoul(buf, NULL, 10);
 }
 
-void write_num(const char *path, unsigned long num)
+static void __write_num(const char *path, unsigned long num, bool ignore_einval)
 {
 	char buf[21];
 
 	sprintf(buf, "%lu", num);
-	write_file(path, buf, strlen(buf) + 1);
+	__write_file(path, buf, strlen(buf) + 1, ignore_einval);
+}
+
+void write_num(const char *path, unsigned long num)
+{
+	return __write_num(path, num, /* ignore_einval = */ false);
+}
+
+void write_num_ignore_einval(const char *path, unsigned long num)
+{
+	return __write_num(path, num, /* ignore_einval = */ true);
 }
 
 static unsigned long shmall, shmmax;
diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h
index ea8fc8fdf0eb0..7799154b67eed 100644
--- a/tools/testing/selftests/mm/vm_util.h
+++ b/tools/testing/selftests/mm/vm_util.h
@@ -168,6 +168,7 @@ void write_file(const char *path, const char *buf, size_t buflen);
 int read_file(const char *path, char *buf, size_t buflen);
 unsigned long read_num(const char *path);
 void write_num(const char *path, unsigned long num);
+void write_num_ignore_einval(const char *path, unsigned long num);
 
 void shm_limits_prepare(unsigned long length);
 void __shm_limits_restore(void);
diff --git a/tools/testing/selftests/net/getsockopt_iter.c b/tools/testing/selftests/net/getsockopt_iter.c
index 209569354d0e3..fe5a5268bc34e 100644
--- a/tools/testing/selftests/net/getsockopt_iter.c
+++ b/tools/testing/selftests/net/getsockopt_iter.c
@@ -11,6 +11,8 @@
  *   that always reports the required buffer length back via optlen,
  *   even when the user buffer is too small to receive any group bits.
  * - vsock:   SO_VM_SOCKETS_BUFFER_SIZE covers the u64 path.
+ * - raw:     ICMP_FILTER covers a fixed-size struct payload that clamps
+ *            the length down on a short buffer instead of failing.
  *
  * Author: Breno Leitao <leitao@debian.org>
  */
@@ -24,12 +26,20 @@
 #include <linux/rtnetlink.h>
 #include <linux/time_types.h>
 #include <linux/vm_sockets.h>
+#include <linux/icmp.h>
+#include <netinet/in.h>
 #include <sys/socket.h>
 #include "kselftest_harness.h"
 
 #ifndef AF_VSOCK
 #define AF_VSOCK 40
 #endif
+#ifndef SOL_RAW
+#define SOL_RAW 255
+#endif
+#ifndef ICMP_FILTER
+#define ICMP_FILTER 1
+#endif
 
 /* ---------- netlink ---------- */
 
@@ -297,4 +307,91 @@ TEST_F(vsock, connect_timeout_old_exact)
 	ASSERT_EQ(sizeof(tv), optlen);
 }
 
+/* ---------- raw (ipv4) ---------- */
+
+FIXTURE(raw)
+{
+	int fd;
+};
+
+FIXTURE_SETUP(raw)
+{
+	struct icmp_filter filt = { .data = 0xdeadbeef };
+
+	self->fd = socket(AF_INET, SOCK_RAW, IPPROTO_ICMP);
+	if (self->fd < 0)
+		SKIP(return, "SOCK_RAW/ICMP socket: %s", strerror(errno));
+
+	if (setsockopt(self->fd, SOL_RAW, ICMP_FILTER, &filt, sizeof(filt)) < 0)
+		SKIP(return, "set ICMP_FILTER: %s", strerror(errno));
+}
+
+FIXTURE_TEARDOWN(raw)
+{
+	if (self->fd >= 0)
+		close(self->fd);
+}
+
+TEST_F(raw, icmpfilter_exact)
+{
+	struct icmp_filter filt = {};
+	socklen_t optlen = sizeof(filt);
+
+	ASSERT_EQ(0, getsockopt(self->fd, SOL_RAW, ICMP_FILTER,
+				&filt, &optlen));
+	ASSERT_EQ(sizeof(filt), optlen);
+	ASSERT_EQ(0xdeadbeef, filt.data);
+}
+
+TEST_F(raw, icmpfilter_oversize_clamped)
+{
+	char buf[16] = {};
+	socklen_t optlen = sizeof(buf);
+
+	ASSERT_EQ(0, getsockopt(self->fd, SOL_RAW, ICMP_FILTER,
+				buf, &optlen));
+	ASSERT_EQ(sizeof(struct icmp_filter), optlen);
+}
+
+/* Unlike the int/u64 options above, ICMP_FILTER clamps the length down
+ * to the user buffer instead of returning EINVAL: a short buffer
+ * succeeds and reports the truncated length back via optlen.
+ */
+TEST_F(raw, icmpfilter_undersize_clamped)
+{
+	char buf[2] = {};
+	socklen_t optlen = sizeof(buf);
+
+	ASSERT_EQ(0, getsockopt(self->fd, SOL_RAW, ICMP_FILTER,
+				buf, &optlen));
+	ASSERT_EQ(sizeof(buf), optlen);
+}
+
+TEST_F(raw, icmpfilter_wrong_proto)
+{
+	struct icmp_filter filt;
+	socklen_t optlen = sizeof(filt);
+	int fd;
+
+	fd = socket(AF_INET, SOCK_RAW, IPPROTO_UDP);
+	if (fd < 0)
+		SKIP(return, "SOCK_RAW/UDP socket: %s", strerror(errno));
+
+	ASSERT_EQ(-1, getsockopt(fd, SOL_RAW, ICMP_FILTER, &filt, &optlen));
+	ASSERT_EQ(EOPNOTSUPP, errno);
+	close(fd);
+}
+
+TEST_F(raw, bad_optname)
+{
+	socklen_t optlen;
+	int val;
+
+	optlen = sizeof(val);
+
+	ASSERT_EQ(-1, getsockopt(self->fd, SOL_RAW, 0x7fff, &val, &optlen));
+	ASSERT_EQ(ENOPROTOOPT, errno);
+	ASSERT_EQ(sizeof(val), optlen);
+}
+
 TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/net/netfilter/nft_flowtable.sh b/tools/testing/selftests/net/netfilter/nft_flowtable.sh
index 08ad07500e8a7..fb1c59d45567a 100755
--- a/tools/testing/selftests/net/netfilter/nft_flowtable.sh
+++ b/tools/testing/selftests/net/netfilter/nft_flowtable.sh
@@ -736,6 +736,61 @@ if ! test_tcp_forwarding_nat "$ns1" "$ns2" 1 "on bridge"; then
 	ret=1
 fi
 
+if ip -net "$nsr1" link show tun0 > /dev/null 2>&1 &&
+   ip -net "$nsr2" link show tun0 > /dev/null 2>&1; then
+	ip -net "$nsr1" route change default via 192.168.100.2
+	ip -net "$nsr2" route change default via 192.168.100.1
+	ip -6 -net "$nsr1" route delete default
+	ip -6 -net "$nsr1" route add default via fee1:3::2
+	ip -6 -net "$nsr2" route delete default
+	ip -6 -net "$nsr2" route add default via fee1:3::1
+	ip -net "$ns2" route add default via 10.0.2.1
+	ip -6 -net "$ns2" route add default via dead:2::1
+
+	ip netns exec "$nsr1" nft -a insert rule inet filter forward \
+		'meta oif "tun0" tcp dport 12345 ct mark set 1 flow add @f1 counter name routed_orig accept'
+	ip netns exec "$nsr1" nft -a insert rule inet filter forward \
+		'meta oif "tun6" tcp dport 12345 ct mark set 1 flow add @f1 counter name routed_orig accept'
+	ip netns exec "$nsr1" nft -a insert rule inet filter forward \
+		'meta oif "veth0" tcp sport 12345 ct mark set 1 flow add @f1 counter name routed_repl accept'
+	ip netns exec "$nsr1" nft -a insert rule inet filter forward \
+		'meta oif "br0" tcp sport 12345 ct mark set 1 flow add @f1 counter name routed_repl accept'
+	ip netns exec "$nsr1" nft -a insert rule inet filter forward \
+		'meta oif "tun0" accept'
+	ip netns exec "$nsr1" nft -a insert rule inet filter forward \
+		'meta oif "tun6" accept'
+
+	ip netns exec "$nsr1" nft reset counters table inet filter >/dev/null
+
+	if test_tcp_forwarding "$ns1" "$ns2" 1 4 10.0.2.99 12345; then
+		check_counters "bridge + IPIP tunnel"
+	else
+		echo "FAIL: flow offload for ns1/ns2 with bridge + IPIP tunnel" 1>&2
+		ip netns exec "$nsr1" nft list ruleset
+		ret=1
+	fi
+
+	if test_tcp_forwarding "$ns1" "$ns2" 1 6 "[dead:2::99]" 12345; then
+		check_counters "bridge + IP6IP6 tunnel"
+	else
+		echo "FAIL: flow offload for ns1/ns2 with bridge + IP6IP6 tunnel" 1>&2
+		ip netns exec "$nsr1" nft list ruleset
+		ret=1
+	fi
+
+	ip -net "$nsr1" route change default via 192.168.10.2
+	ip -net "$nsr2" route change default via 192.168.10.1
+	ip -net "$ns2" route del default via 10.0.2.1
+	ip -6 -net "$nsr1" route delete default
+	ip -6 -net "$nsr1" route add default via fee1:2::2
+	ip -6 -net "$nsr2" route delete default
+	ip -6 -net "$nsr2" route add default via fee1:2::1
+	ip -6 -net "$ns2" route del default via dead:2::1
+else
+	echo "SKIP: bridge + tunnel flowtable regression (tun0 missing)"
+	[ "$ret" -eq 0 ] && ret=$ksft_skip
+fi
+
 
 # Another test:
 # Add bridge interface br0 to Router1, with NAT and VLAN.
diff --git a/tools/testing/selftests/net/openvswitch/openvswitch.sh b/tools/testing/selftests/net/openvswitch/openvswitch.sh
index 2954245129a2f..f75ee723415a9 100755
--- a/tools/testing/selftests/net/openvswitch/openvswitch.sh
+++ b/tools/testing/selftests/net/openvswitch/openvswitch.sh
@@ -32,6 +32,7 @@ tests="
 	dec_ttl					ttl: dec_ttl decrements IP TTL
 	flow_set				flow-set: Flow modify
 	action_set				set: SET action rewrites fields
+	trunc					trunc: output truncation
 	psample					psample: Sampling packets with psample"
 
 info() {
@@ -443,6 +444,92 @@ test_action_set() {
 	return 0
 }
 
+# trunc test
+# - trunc(14): truncate to ETH_HLEN, strips IP payload, ping fails
+# - trunc(1) and trunc(13): kernel rejects below ETH_HLEN (EINVAL)
+# - restore normal forwarding and verify recovery
+test_trunc() {
+	sbx_add "test_trunc" || return $?
+	ovs_add_dp "test_trunc" trunctest || return 1
+
+	info "create namespaces"
+	for ns in client server; do
+		ovs_add_netns_and_veths "test_trunc" "trunctest" \
+		    "$ns" "${ns:0:1}0" "${ns:0:1}1" || return 1
+	done
+
+	ip netns exec client ip addr add 10.0.0.1/24 dev c1
+	ip netns exec client ip link set c1 up
+	ip netns exec server ip addr add 10.0.0.2/24 dev s1
+	ip netns exec server ip link set s1 up
+
+	ovs_add_flow "test_trunc" trunctest \
+	    'in_port(1),eth(),eth_type(0x0806),arp()' '2' || return 1
+	ovs_add_flow "test_trunc" trunctest \
+	    'in_port(2),eth(),eth_type(0x0806),arp()' '1' || return 1
+	ovs_add_flow "test_trunc" trunctest \
+	    'in_port(1),eth(),eth_type(0x0800),ipv4()' \
+	    '2' || return 1
+	ovs_add_flow "test_trunc" trunctest \
+	    'in_port(2),eth(),eth_type(0x0800),ipv4()' \
+	    '1' || return 1
+
+	info "verify connectivity without truncation"
+	ovs_sbx "test_trunc" ip netns exec client \
+	    ping -c 1 -W 2 10.0.0.2 || return 1
+
+	# trunc below ETH_HLEN must be rejected by the kernel
+	info "verify trunc(1) is rejected"
+	ovs_add_flow "test_trunc" trunctest \
+	    'in_port(1),eth(),eth_type(0x0800),ipv4()' \
+	    'trunc(1),2' &> /dev/null \
+	    && { info "trunc(1) should be rejected"; return 1; }
+
+	info "verify trunc(13) is rejected"
+	ovs_add_flow "test_trunc" trunctest \
+	    'in_port(1),eth(),eth_type(0x0800),ipv4()' \
+	    'trunc(13),2' &> /dev/null \
+	    && { info "trunc(13) should be rejected"; return 1; }
+
+	ovs_del_flows "test_trunc" trunctest
+	ovs_add_flow "test_trunc" trunctest \
+	    'in_port(1),eth(),eth_type(0x0806),arp()' '2' || return 1
+	ovs_add_flow "test_trunc" trunctest \
+	    'in_port(2),eth(),eth_type(0x0806),arp()' '1' || return 1
+
+	info "add trunc(14) forwarding flow"
+	ovs_add_flow "test_trunc" trunctest \
+	    'in_port(1),eth(),eth_type(0x0800),ipv4()' \
+	    'trunc(14),2' || return 1
+	ovs_add_flow "test_trunc" trunctest \
+	    'in_port(2),eth(),eth_type(0x0800),ipv4()' \
+	    '1' || return 1
+
+	info "verify ping fails with trunc(14)"
+	ovs_sbx "test_trunc" ip netns exec client \
+	    ping -c 1 -W 2 10.0.0.2 >/dev/null 2>&1 \
+	    && { info "ping should fail with trunc(14)"
+	         return 1; }
+
+	ovs_del_flows "test_trunc" trunctest
+	ovs_add_flow "test_trunc" trunctest \
+	    'in_port(1),eth(),eth_type(0x0806),arp()' '2' || return 1
+	ovs_add_flow "test_trunc" trunctest \
+	    'in_port(2),eth(),eth_type(0x0806),arp()' '1' || return 1
+	ovs_add_flow "test_trunc" trunctest \
+	    'in_port(1),eth(),eth_type(0x0800),ipv4()' \
+	    '2' || return 1
+	ovs_add_flow "test_trunc" trunctest \
+	    'in_port(2),eth(),eth_type(0x0800),ipv4()' \
+	    '1' || return 1
+
+	info "verify connectivity restored"
+	ovs_sbx "test_trunc" ip netns exec client \
+	    ping -c 1 -W 2 10.0.0.2 || return 1
+
+	return 0
+}
+
 # psample test
 # - use psample to observe packets
 test_psample() {
diff --git a/tools/testing/selftests/net/rtnetlink.py b/tools/testing/selftests/net/rtnetlink.py
index 3622413d793da..0c67c7c00d841 100755
--- a/tools/testing/selftests/net/rtnetlink.py
+++ b/tools/testing/selftests/net/rtnetlink.py
@@ -2,27 +2,106 @@
 # SPDX-License-Identifier: GPL-2.0
 
 import socket
+import struct
 import time
-from lib.py import bkg, ip, ksft_exit, ksft_run, ksft_ge, ksft_true, KsftSkipEx
+from lib.py import bkg, ip, ksft_exit, ksft_run, ksft_eq, ksft_ge, ksft_true, KsftSkipEx
 from lib.py import CmdExitFailure, NetNS, NetNSEnter, RtnlAddrFamily
 
 IPV4_ALL_HOSTS_MULTICAST = b'\xe0\x00\x00\x01'
+IPV4_TEST_MULTICAST = b'\xef\x01\x01\x01'
+IPV6_TEST_MULTICAST = bytes.fromhex('ff020000000000000000000000000123')
+
+
+def _users_for(rtnl: RtnlAddrFamily, family: int, grp: bytes, ifindex: int):
+    """Return mc-users for grp on ifindex, or 0 if absent."""
+
+    addrs = rtnl.getmulticast({"ifa-family": family}, dump=True)
+    matches = [addr for addr in addrs
+               if addr['multicast'] == grp and addr['ifa-index'] == ifindex]
+    if not matches:
+        return 0
+    if 'mc-users' not in matches[0]:
+        return None
+
+    return matches[0]['mc-users']
+
 
 def dump_mcaddr_check() -> None:
     """
-    Verify that at least one interface has the IPv4 all-hosts multicast address.
-    At least the loopback interface should have this address.
+    Verify IPv4 multicast addresses and their user counts in RTM_GETMULTICAST.
+    """
+
+    with NetNS() as ns:
+        with NetNSEnter(str(ns)):
+            ip("link set lo up")
+            rtnl = RtnlAddrFamily()
+            lo_idx = socket.if_nametoindex('lo')
+            addresses = rtnl.getmulticast({"ifa-family": socket.AF_INET}, dump=True)
+
+            all_host_multicasts = [
+                addr for addr in addresses
+                if addr['multicast'] == IPV4_ALL_HOSTS_MULTICAST
+            ]
+
+            ksft_ge(len(all_host_multicasts), 1,
+                    "No interface found with the IPv4 all-hosts multicast address")
+
+            mreq = IPV4_TEST_MULTICAST + socket.inet_aton('127.0.0.1')
+            before = _users_for(rtnl, socket.AF_INET, IPV4_TEST_MULTICAST, lo_idx)
+            if before is None:
+                raise KsftSkipEx("kernel does not expose IFA_MC_USERS")
+
+            s1 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+            s2 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+            try:
+                s1.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
+                s2.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
+
+                after_join = _users_for(rtnl, socket.AF_INET,
+                                        IPV4_TEST_MULTICAST, lo_idx)
+                if after_join is None:
+                    raise KsftSkipEx("kernel does not expose IFA_MC_USERS")
+                ksft_eq(after_join - before, 2,
+                        f"users delta != 2 after two joins "
+                        f"(before={before}, after={after_join})")
+            finally:
+                s1.close()
+                s2.close()
+
+
+def dump_mcaddr6_check() -> None:
+    """
+    Verify IPv6 multicast addresses and their user counts in RTM_GETMULTICAST.
     """
 
-    rtnl = RtnlAddrFamily()
-    addresses = rtnl.getmulticast({"ifa-family": socket.AF_INET}, dump=True)
+    with NetNS() as ns:
+        with NetNSEnter(str(ns)):
+            ip("link set lo up")
+            rtnl = RtnlAddrFamily()
+            lo_idx = socket.if_nametoindex('lo')
+            before = _users_for(rtnl, socket.AF_INET6,
+                                IPV6_TEST_MULTICAST, lo_idx)
+            if before is None:
+                raise KsftSkipEx("kernel does not expose IFA_MC_USERS for IPv6")
+
+            mreq = IPV6_TEST_MULTICAST + struct.pack('=I', lo_idx)
+            s1 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+            s2 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+            try:
+                s1.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq)
+                s2.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq)
 
-    all_host_multicasts = [
-        addr for addr in addresses if addr['multicast'] == IPV4_ALL_HOSTS_MULTICAST
-    ]
+                after_join = _users_for(rtnl, socket.AF_INET6,
+                                        IPV6_TEST_MULTICAST, lo_idx)
+                if after_join is None:
+                    raise KsftSkipEx("kernel does not expose IFA_MC_USERS for IPv6")
+                ksft_eq(after_join - before, 2,
+                        f"IPv6 users delta != 2 after two joins "
+                        f"(before={before}, after={after_join})")
+            finally:
+                s1.close()
+                s2.close()
 
-    ksft_ge(len(all_host_multicasts), 1,
-            "No interface found with the IPv4 all-hosts multicast address")
 
 def ipv4_devconf_notify() -> None:
     """
@@ -56,7 +135,7 @@ def ipv4_devconf_notify() -> None:
               f"No 'forwarding on' notificiation found for interface {ifname}")
 
 def main() -> None:
-    ksft_run([dump_mcaddr_check, ipv4_devconf_notify])
+    ksft_run([dump_mcaddr_check, dump_mcaddr6_check, ipv4_devconf_notify])
     ksft_exit()
 
 if __name__ == "__main__":
diff --git a/tools/testing/selftests/nolibc/Makefile.include b/tools/testing/selftests/nolibc/Makefile.include
index c30ca3a9ef142..ea520eac64a67 100644
--- a/tools/testing/selftests/nolibc/Makefile.include
+++ b/tools/testing/selftests/nolibc/Makefile.include
@@ -5,7 +5,7 @@ _CFLAGS_STACKPROTECTOR ?= $(call try-run, \
 	echo 'void foo(void) {}' | $(CC) -x c - -o - -S $(CLANG_CROSS_FLAGS) $(__CFLAGS_STACKPROTECTOR) | grep -q __stack_chk_guard, \
 	$(__CFLAGS_STACKPROTECTOR))
 _CFLAGS_SANITIZER ?= $(call cc-option,-fsanitize=undefined -fsanitize-trap=all)
-CFLAGS_NOLIBC_TEST  ?= -Os -fno-ident -fno-asynchronous-unwind-tables -std=c89 \
+CFLAGS_NOLIBC_TEST  ?= -Os -fno-ident -fno-asynchronous-unwind-tables -std=c89 -ggdb \
 		-W -Wall -Wextra -Wundef -Wwrite-strings \
 		$(call cc-option,-fno-stack-protector) $(call cc-option,-Wmissing-prototypes) \
 		$(_CFLAGS_STACKPROTECTOR) $(_CFLAGS_SANITIZER)
diff --git a/tools/testing/selftests/nolibc/nolibc-test.c b/tools/testing/selftests/nolibc/nolibc-test.c
index c1c1ce43a0473..996e8d13508e2 100644
--- a/tools/testing/selftests/nolibc/nolibc-test.c
+++ b/tools/testing/selftests/nolibc/nolibc-test.c
@@ -854,6 +854,58 @@ static int test_dirent(void)
 	return 0;
 }
 
+int test_getcwd(void)
+{
+	char cwd_syscall[PATH_MAX];
+	char cwd_proc[PATH_MAX];
+	ssize_t len;
+
+	/* Read where the link /proc/self/cwd points */
+	len = readlink("/proc/self/cwd", cwd_proc, sizeof(cwd_proc) - 1);
+	if (len <= 0)
+		return __LINE__;
+
+	/* Terminate the string from readlink() */
+	cwd_proc[len] = '\0';
+
+	/* Get the cwd via syscall */
+	if (getcwd(cwd_syscall, sizeof(cwd_syscall)) == NULL)
+		return __LINE__;
+
+	/* Fail if they aren't the same */
+	if (strcmp(cwd_proc, cwd_syscall) != 0)
+		return __LINE__;
+
+	/* Try getcwd() with NULL for the buffer,
+	 * should return NULL and an error in errno.
+	 * Other libc's allow this by allocating a buffer
+	 * internally.
+	 */
+	if (is_nolibc) {
+		errno = 0;
+		if (getcwd(NULL, 0) != NULL || !errno)
+			return __LINE__;
+	}
+
+	/* Try getcwd() with a buffer but make the size 0,
+	 * should return NULL and an error in errno.
+	 */
+	errno = 0;
+	if (getcwd(cwd_syscall, 0) != NULL || !errno)
+		return __LINE__;
+
+	/* Try getcwd() with a buffer but make the size 1,
+	 * should return NULL and an error in errno because
+	 * the string written to the buffer is terminated
+	 * so you need at least 2 bytes even for "/".
+	 */
+	errno = 0;
+	if (getcwd(cwd_syscall, 1) != NULL || !errno)
+		return __LINE__;
+
+	return 0;
+}
+
 int test_getrandom(void)
 {
 	uint64_t rng = 0;
@@ -1555,6 +1607,7 @@ int run_syscall(int min, int max)
 		CASE_TEST(clock_getres);      EXPECT_SYSZR(1, clock_getres(CLOCK_MONOTONIC, &ts)); break;
 		CASE_TEST(clock_gettime);     EXPECT_SYSZR(1, clock_gettime(CLOCK_MONOTONIC, &ts)); break;
 		CASE_TEST(clock_settime);     EXPECT_SYSER(1, clock_settime(CLOCK_MONOTONIC, &ts), -1, EINVAL); break;
+		CASE_TEST(getcwd);            EXPECT_SYSZR(proc, test_getcwd()); break;
 		CASE_TEST(getpid);            EXPECT_SYSNE(1, getpid(), -1); break;
 		CASE_TEST(getppid);           EXPECT_SYSNE(1, getppid(), -1); break;
 		CASE_TEST(gettid);            EXPECT_SYSNE(has_gettid, gettid(), -1); break;
diff --git a/tools/testing/selftests/proc/proc-pidns.c b/tools/testing/selftests/proc/proc-pidns.c
index 25b9a2933c456..6f7c10fe97b30 100644
--- a/tools/testing/selftests/proc/proc-pidns.c
+++ b/tools/testing/selftests/proc/proc-pidns.c
@@ -6,6 +6,7 @@
 
 #include <assert.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <sched.h>
 #include <stdbool.h>
 #include <stdlib.h>
diff --git a/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h
index d3133587d9968..5a29d6bec51f4 100644
--- a/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h
+++ b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h
@@ -8,13 +8,11 @@
 #ifndef _RSEQ_X86_THREAD_POINTER
 #define _RSEQ_X86_THREAD_POINTER
 
-#include <features.h>
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#if __GNUC_PREREQ (11, 1)
+#if __GNUC__ > 11 || (__GNUC__ == 11 && __GNUC_MINOR__ >= 1)
 static inline void *rseq_thread_pointer(void)
 {
 	return __builtin_thread_pointer();
diff --git a/tools/testing/selftests/rseq/rseq.c b/tools/testing/selftests/rseq/rseq.c
index be0d0a97031ef..1cd633923b1d6 100644
--- a/tools/testing/selftests/rseq/rseq.c
+++ b/tools/testing/selftests/rseq/rseq.c
@@ -73,7 +73,7 @@ static int rseq_ownership;
 #define ORIG_RSEQ_ALLOC_SIZE		32
 
 /*
- * Use a union to ensure we allocate a TLS area of 1024 bytes to accomodate an
+ * Use a union to ensure we allocate a TLS area of 1024 bytes to accommodate an
  * rseq registration that is larger than the current rseq ABI.
  */
 union rseq_tls {
diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile
index 5d2dffca0e918..3cfe90e0f34fa 100644
--- a/tools/testing/selftests/sched_ext/Makefile
+++ b/tools/testing/selftests/sched_ext/Makefile
@@ -176,6 +176,7 @@ auto-test-targets :=			\
 	maybe_null			\
 	minimal				\
 	non_scx_kfunc_deny		\
+	nohz_tick			\
 	numa				\
 	allowed_cpus			\
 	peek_dsq			\
diff --git a/tools/testing/selftests/sched_ext/nohz_tick.bpf.c b/tools/testing/selftests/sched_ext/nohz_tick.bpf.c
new file mode 100644
index 0000000000000..6998c5dd6bcb9
--- /dev/null
+++ b/tools/testing/selftests/sched_ext/nohz_tick.bpf.c
@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES
+ *
+ * Exercise tick dependency transitions between infinite and finite slices.
+ */
+#include <scx/common.bpf.h>
+
+char _license[] SEC("license") = "GPL";
+
+const volatile s32 test_cpu;
+bool finite_phase;
+u64 nr_inf_running;
+u64 nr_finite_running;
+u64 nr_finite_ticks;
+
+UEI_DEFINE(uei);
+
+s32 BPF_STRUCT_OPS(nohz_tick_select_cpu, struct task_struct *p, s32 prev_cpu,
+		   u64 wake_flags)
+{
+	return prev_cpu;
+}
+
+void BPF_STRUCT_OPS(nohz_tick_enqueue, struct task_struct *p, u64 enq_flags)
+{
+	u64 slice = finite_phase ? 1000000ULL : SCX_SLICE_INF;
+
+	scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, slice, enq_flags);
+	if (enq_flags & SCX_ENQ_LAST)
+		scx_bpf_kick_cpu(test_cpu, SCX_KICK_IDLE);
+}
+
+void BPF_STRUCT_OPS(nohz_tick_running, struct task_struct *p)
+{
+	if (bpf_get_smp_processor_id() != test_cpu)
+		return;
+
+	if (finite_phase)
+		__sync_fetch_and_add(&nr_finite_running, 1);
+	else
+		__sync_fetch_and_add(&nr_inf_running, 1);
+}
+
+void BPF_STRUCT_OPS(nohz_tick_tick, struct task_struct *p)
+{
+	if (bpf_get_smp_processor_id() == test_cpu && finite_phase)
+		__sync_fetch_and_add(&nr_finite_ticks, 1);
+}
+
+void BPF_STRUCT_OPS(nohz_tick_exit, struct scx_exit_info *ei)
+{
+	UEI_RECORD(uei, ei);
+}
+
+SEC(".struct_ops.link")
+struct sched_ext_ops nohz_tick_ops = {
+	.select_cpu		= (void *)nohz_tick_select_cpu,
+	.enqueue		= (void *)nohz_tick_enqueue,
+	.running		= (void *)nohz_tick_running,
+	.tick			= (void *)nohz_tick_tick,
+	.exit			= (void *)nohz_tick_exit,
+	.name			= "nohz_tick",
+	.timeout_ms		= 1000U,
+};
diff --git a/tools/testing/selftests/sched_ext/nohz_tick.c b/tools/testing/selftests/sched_ext/nohz_tick.c
new file mode 100644
index 0000000000000..028f54391c2ca
--- /dev/null
+++ b/tools/testing/selftests/sched_ext/nohz_tick.c
@@ -0,0 +1,347 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES
+ *
+ * Validate that a finite-slice EXT task restarts the scheduler tick when it
+ * follows an infinite-slice EXT task and an idle interval on a NOHZ_FULL CPU.
+ */
+#define _GNU_SOURCE
+
+#include <bpf/bpf.h>
+#include <errno.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/prctl.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include <scx/common.h>
+
+#include "nohz_tick.bpf.skel.h"
+#include "scx_test.h"
+
+#ifndef SCHED_EXT
+#define SCHED_EXT 7
+#endif
+
+#define MIN_FINITE_TICKS 3
+#define PHASE_TIMEOUT_MS 1000
+
+struct nohz_tick_ctx {
+	struct nohz_tick *skel;
+	cpu_set_t original_mask;
+	int test_cpu;
+};
+
+static int first_allowed_cpu(const cpu_set_t *mask, int first, int last)
+{
+	int cpu;
+
+	for (cpu = first; cpu <= last && cpu < CPU_SETSIZE; cpu++)
+		if (CPU_ISSET(cpu, mask))
+			return cpu;
+
+	return -1;
+}
+
+static int find_nohz_full_cpu(const cpu_set_t *allowed)
+{
+	char buf[4096], *cur, *end;
+	FILE *file;
+
+	file = fopen("/sys/devices/system/cpu/nohz_full", "r");
+	if (!file)
+		return -1;
+	if (!fgets(buf, sizeof(buf), file)) {
+		fclose(file);
+		return -1;
+	}
+	fclose(file);
+
+	cur = buf;
+	while (*cur) {
+		long first, last;
+		int cpu;
+
+		while (*cur == ' ' || *cur == '\t' || *cur == ',')
+			cur++;
+		if (*cur < '0' || *cur > '9')
+			break;
+
+		errno = 0;
+		first = strtol(cur, &end, 10);
+		if (errno || end == cur || first < 0 || first >= CPU_SETSIZE)
+			return -1;
+		cur = end;
+		last = first;
+		if (*cur == '-') {
+			cur++;
+			errno = 0;
+			last = strtol(cur, &end, 10);
+			if (errno || end == cur || last < first)
+				return -1;
+			cur = end;
+		}
+
+		cpu = first_allowed_cpu(allowed, first, last);
+		if (cpu >= 0)
+			return cpu;
+	}
+
+	return -1;
+}
+
+static pid_t start_worker(int cpu)
+{
+	struct sched_param param = {};
+	cpu_set_t mask;
+	pid_t parent;
+	pid_t pid;
+
+	parent = getpid();
+	pid = fork();
+	if (pid != 0)
+		return pid;
+	if (prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() != parent)
+		_exit(1);
+
+	/*
+	 * Become EXT before touching the target so it stays idle until wakeup.
+	 */
+	if (sched_setscheduler(0, SCHED_EXT, &param))
+		_exit(1);
+
+	CPU_ZERO(&mask);
+	CPU_SET(cpu, &mask);
+	if (sched_setaffinity(0, sizeof(mask), &mask))
+		_exit(1);
+
+	for (;;)
+		asm volatile("" ::: "memory");
+}
+
+static void stop_worker(pid_t pid)
+{
+	if (pid <= 0)
+		return;
+
+	kill(pid, SIGKILL);
+	waitpid(pid, NULL, 0);
+}
+
+static int pause_worker(pid_t pid)
+{
+	int status;
+
+	if (kill(pid, SIGSTOP))
+		return -errno;
+	if (waitpid(pid, &status, WUNTRACED) != pid)
+		return -errno;
+	if (!WIFSTOPPED(status))
+		return -ECHILD;
+
+	return 0;
+}
+
+static bool wait_for_counter(const u64 *counter, u64 value, int timeout_ms)
+{
+	int elapsed;
+
+	for (elapsed = 0; elapsed < timeout_ms; elapsed++) {
+		if (__atomic_load_n(counter, __ATOMIC_RELAXED) >= value)
+			return true;
+		usleep(1000);
+	}
+
+	return false;
+}
+
+static enum scx_test_status setup(void **ctx_ptr)
+{
+	struct nohz_tick_ctx *ctx;
+	cpu_set_t controller_mask;
+	int cpu;
+
+	ctx = calloc(1, sizeof(*ctx));
+	SCX_FAIL_IF(!ctx, "Failed to allocate context");
+	if (sched_getaffinity(0, sizeof(ctx->original_mask),
+			      &ctx->original_mask)) {
+		free(ctx);
+		SCX_FAIL("Failed to get affinity (%d)", errno);
+	}
+
+	cpu = find_nohz_full_cpu(&ctx->original_mask);
+	if (cpu < 0) {
+		fprintf(stderr, "SKIP: no allowed NOHZ_FULL CPU\n");
+		free(ctx);
+		return SCX_TEST_SKIP;
+	}
+
+	controller_mask = ctx->original_mask;
+	CPU_CLR(cpu, &controller_mask);
+	if (CPU_COUNT(&controller_mask) == 0) {
+		fprintf(stderr, "SKIP: no housekeeping CPU available\n");
+		free(ctx);
+		return SCX_TEST_SKIP;
+	}
+
+	ctx->test_cpu = cpu;
+	ctx->skel = nohz_tick__open();
+	if (!ctx->skel) {
+		free(ctx);
+		SCX_FAIL("Failed to open skeleton");
+	}
+
+	SCX_ENUM_INIT(ctx->skel);
+	ctx->skel->rodata->test_cpu = cpu;
+	ctx->skel->struct_ops.nohz_tick_ops->flags |= SCX_OPS_SWITCH_PARTIAL |
+							   SCX_OPS_ENQ_LAST;
+	if (nohz_tick__load(ctx->skel)) {
+		nohz_tick__destroy(ctx->skel);
+		free(ctx);
+		SCX_FAIL("Failed to load skeleton");
+	}
+
+	if (sched_setaffinity(0, sizeof(controller_mask), &controller_mask)) {
+		nohz_tick__destroy(ctx->skel);
+		free(ctx);
+		SCX_FAIL("Failed to move controller off CPU %d (%d)", cpu, errno);
+	}
+
+	*ctx_ptr = ctx;
+	return SCX_TEST_PASS;
+}
+
+static enum scx_test_status run(void *ctx_ptr)
+{
+	struct nohz_tick_ctx *ctx = ctx_ptr;
+	struct nohz_tick *skel = ctx->skel;
+	struct bpf_link *link = NULL;
+	enum scx_test_status status = SCX_TEST_FAIL;
+	pid_t finite_worker = -1;
+	pid_t inf_worker = -1;
+	u64 finite_running;
+	u64 finite_ticks;
+	int ret;
+
+	link = bpf_map__attach_struct_ops(skel->maps.nohz_tick_ops);
+	if (!link) {
+		SCX_ERR("Failed to attach scheduler");
+		goto out;
+	}
+
+	/*
+	 * Establish SCX_RQ_CAN_STOP_TICK with an infinite-slice task.
+	 */
+	inf_worker = start_worker(ctx->test_cpu);
+	if (inf_worker < 0) {
+		SCX_ERR("Failed to start infinite-slice worker (%d)", errno);
+		goto out;
+	}
+	if (!wait_for_counter(&skel->bss->nr_inf_running, 1,
+			      PHASE_TIMEOUT_MS)) {
+		SCX_ERR("Infinite-slice worker was not scheduled");
+		goto out;
+	}
+
+	/* Block without exiting so the rq retains the infinite-slice state. */
+	ret = pause_worker(inf_worker);
+	if (ret) {
+		SCX_ERR("Failed to stop infinite-slice worker (%d)", ret);
+		goto out;
+	}
+
+	/* Let the target enter idle with its tick stopped. */
+	usleep(100000);
+
+	/*
+	 * The next EXT task receives a finite slice and must restart the tick.
+	 */
+	__atomic_store_n(&skel->bss->finite_phase, true, __ATOMIC_RELEASE);
+	finite_worker = start_worker(ctx->test_cpu);
+	if (finite_worker < 0) {
+		SCX_ERR("Failed to start finite-slice worker (%d)", errno);
+		goto out;
+	}
+	if (!wait_for_counter(&skel->bss->nr_finite_running, 1,
+			      PHASE_TIMEOUT_MS)) {
+		SCX_ERR("Finite-slice worker was not scheduled");
+		goto out;
+	}
+	if (!wait_for_counter(&skel->bss->nr_finite_ticks, MIN_FINITE_TICKS,
+			      PHASE_TIMEOUT_MS)) {
+		SCX_ERR("Finite-slice worker received only %llu scheduler ticks",
+			(unsigned long long)skel->bss->nr_finite_ticks);
+		goto out;
+	}
+	stop_worker(finite_worker);
+	finite_worker = -1;
+
+	/*
+	 * Leave the CPU idle after a finite-slice task. The next finite-slice
+	 * task must restart the tick even though the slice type is unchanged.
+	 */
+	usleep(100000);
+	finite_running = __atomic_load_n(&skel->bss->nr_finite_running,
+					 __ATOMIC_RELAXED);
+	finite_ticks = __atomic_load_n(&skel->bss->nr_finite_ticks,
+				       __ATOMIC_RELAXED);
+
+	finite_worker = start_worker(ctx->test_cpu);
+	if (finite_worker < 0) {
+		SCX_ERR("Failed to start second finite-slice worker (%d)", errno);
+		goto out;
+	}
+	if (!wait_for_counter(&skel->bss->nr_finite_running,
+			      finite_running + 1, PHASE_TIMEOUT_MS)) {
+		SCX_ERR("Second finite-slice worker was not scheduled");
+		goto out;
+	}
+	if (!wait_for_counter(&skel->bss->nr_finite_ticks,
+			      finite_ticks + MIN_FINITE_TICKS,
+			      PHASE_TIMEOUT_MS)) {
+		SCX_ERR("Second finite-slice worker received only %llu scheduler ticks",
+			(unsigned long long)(skel->bss->nr_finite_ticks -
+					     finite_ticks));
+		goto out;
+	}
+
+	if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) {
+		SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)",
+			(unsigned long long)skel->data->uei.kind,
+			(long long)skel->data->uei.exit_code);
+		goto out;
+	}
+
+	fprintf(stderr, "CPU %d received %llu finite-slice ticks\n",
+		ctx->test_cpu,
+		(unsigned long long)skel->bss->nr_finite_ticks);
+	status = SCX_TEST_PASS;
+out:
+	stop_worker(finite_worker);
+	stop_worker(inf_worker);
+	if (link)
+		bpf_link__destroy(link);
+	return status;
+}
+
+static void cleanup(void *ctx_ptr)
+{
+	struct nohz_tick_ctx *ctx = ctx_ptr;
+
+	sched_setaffinity(0, sizeof(ctx->original_mask), &ctx->original_mask);
+	nohz_tick__destroy(ctx->skel);
+	free(ctx);
+}
+
+struct scx_test nohz_tick = {
+	.name = "nohz_tick",
+	.description = "Verify finite EXT slices restart the NOHZ_FULL tick",
+	.setup = setup,
+	.run = run,
+	.cleanup = cleanup,
+};
+REGISTER_SCX_TEST(&nohz_tick)
diff --git a/tools/testing/selftests/sched_ext/prog_run.c b/tools/testing/selftests/sched_ext/prog_run.c
index 05974820ca69d..1129ec2aaddc7 100644
--- a/tools/testing/selftests/sched_ext/prog_run.c
+++ b/tools/testing/selftests/sched_ext/prog_run.c
@@ -28,7 +28,8 @@ static enum scx_test_status setup(void **ctx)
 static enum scx_test_status run(void *ctx)
 {
 	struct prog_run *skel = ctx;
-	struct bpf_link *link;
+	struct bpf_link *link = NULL;
+	enum scx_test_status status = SCX_TEST_PASS;
 	int prog_fd, err = 0;
 
 	prog_fd = bpf_program__fd(skel->progs.prog_run_syscall);
@@ -42,23 +43,40 @@ static enum scx_test_status run(void *ctx)
 	link = bpf_map__attach_struct_ops(skel->maps.prog_run_ops);
 	if (!link) {
 		SCX_ERR("Failed to attach scheduler");
-		close(prog_fd);
-		return SCX_TEST_FAIL;
+		status = SCX_TEST_FAIL;
+		goto out;
 	}
 
 	err = bpf_prog_test_run_opts(prog_fd, &topts);
-	SCX_EQ(err, 0);
+	if (err) {
+		SCX_ERR("BPF_PROG_RUN failed (%d)", err);
+		status = SCX_TEST_FAIL;
+		goto out;
+	}
 
 	/* Assumes uei.kind is written last */
 	while (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_NONE))
 		sched_yield();
 
-	SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG_BPF));
-	SCX_EQ(skel->data->uei.exit_code, 0xdeadbeef);
+	if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_UNREG_BPF)) {
+		SCX_ERR("Unexpected exit kind: %llu",
+			(unsigned long long)skel->data->uei.kind);
+		status = SCX_TEST_FAIL;
+		goto out;
+	}
+	if (skel->data->uei.exit_code != 0xdeadbeef) {
+		SCX_ERR("Unexpected exit code: %lld",
+			(long long)skel->data->uei.exit_code);
+		status = SCX_TEST_FAIL;
+		goto out;
+	}
+
+out:
 	close(prog_fd);
-	bpf_link__destroy(link);
+	if (link)
+		bpf_link__destroy(link);
 
-	return SCX_TEST_PASS;
+	return status;
 }
 
 static void cleanup(void *ctx)
diff --git a/tools/testing/selftests/seccomp/seccomp_bpf.c b/tools/testing/selftests/seccomp/seccomp_bpf.c
index 358b6c65e120e..0622bc2acad40 100644
--- a/tools/testing/selftests/seccomp/seccomp_bpf.c
+++ b/tools/testing/selftests/seccomp/seccomp_bpf.c
@@ -5178,7 +5178,8 @@ FIXTURE_SETUP(UPROBE)
 		ASSERT_GE(bit, 0);
 	}
 
-	offset = get_uprobe_offset(variant->uretprobe ? probed_uretprobe : probed_uprobe);
+	offset = get_uprobe_offset(variant->uretprobe ? (void *)probed_uretprobe
+						      : (void *)probed_uprobe);
 	ASSERT_GE(offset, 0);
 
 	if (variant->uretprobe)
diff --git a/tools/testing/selftests/syscall_user_dispatch/config b/tools/testing/selftests/syscall_user_dispatch/config
index 039e303e59d7d..22c4dfe167ca9 100644
--- a/tools/testing/selftests/syscall_user_dispatch/config
+++ b/tools/testing/selftests/syscall_user_dispatch/config
@@ -1 +1 @@
-CONFIG_GENERIC_ENTRY=y
+CONFIG_SYSCALL_USER_DISPATCH=y
diff --git a/tools/testing/selftests/tc-testing/tc-tests/infra/qdiscs.json b/tools/testing/selftests/tc-testing/tc-tests/infra/qdiscs.json
index a1f97a4b606e8..0cf12c50fb74b 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/infra/qdiscs.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/infra/qdiscs.json
@@ -1540,5 +1540,169 @@
             "$TC qdisc del dev $DUMMY root",
             "$IP addr del 10.10.10.10/24 dev $DUMMY || true"
         ]
+    },
+    {
+        "id": "fb6c",
+        "name": "Force multiq to dequeue from its child's gso_skb with qfq leaf",
+        "category": [
+            "qdisc",
+            "tbf",
+            "multiq",
+            "qfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1 4\" > /sys/bus/netdevsim/new_device",
+            "$IP link set dev $ETH up || true",
+            "$IP l set addr 01:02:03:04:05:06 dev $ETH || true",
+            "$IP n add dev $ETH 10.10.11.1 lladdr 01:02:03:04:05:06 dev $ETH || true",
+            "$IP addr add 10.10.11.10/24 dev $ETH || true",
+            "$TC qdisc add dev $ETH root handle 1: tbf rate 88bit burst 1661b peakrate 2257333 minburst 1024 limit 7b",
+            "$TC qdisc add dev $ETH parent 1: handle 2: multiq",
+            "$TC qdisc add dev $ETH parent 2:1 handle 3: qfq",
+            "$TC class add dev $ETH classid 3:1 parent 3: qfq maxpkt 512 weight 1",
+            "$TC filter add dev $ETH parent 2: protocol all prio 1 matchall action skbedit queue_mapping 0",
+            "$TC filter add dev $ETH parent 3: protocol all prio 1 matchall classid 3:1 action ok"
+        ],
+        "cmdUnderTest": "ping -c 1 10.10.11.1 -W0.01 -I$ETH || true",
+        "expExitCode": "0",
+        "verifyCmd": "$TC -s -j qdisc ls dev $ETH parent 1:",
+        "matchJSON": [
+            {
+                "kind": "multiq",
+                "handle": "2:",
+                "bytes": 98,
+                "packets": 1,
+                "backlog": 0,
+                "qlen": 0
+            }
+        ],
+        "teardown": [
+            "$TC qdisc del dev $ETH handle 1: root",
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    },
+    {
+        "id": "1922",
+        "name": "Force multiq to dequeue from its child's gso_skb with dualpi2 leaf",
+        "category": [
+            "qdisc",
+            "tbf",
+            "multiq",
+            "dualpi2"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1 4\" > /sys/bus/netdevsim/new_device",
+            "$IP link set dev $ETH up || true",
+            "$IP l set addr 01:02:03:04:05:06 dev $ETH || true",
+            "$IP n add dev $ETH 10.10.11.1 lladdr 01:02:03:04:05:06 dev $ETH || true",
+            "$IP addr add 10.10.11.10/24 dev $ETH || true",
+            "$TC qdisc add dev $ETH root handle 1: tbf rate 88bit burst 1661b peakrate 2257333 minburst 1024 limit 7b",
+            "$TC qdisc add dev $ETH parent 1: handle 2: multiq",
+            "$TC qdisc add dev $ETH parent 2:1 handle 3: dualpi2",
+            "$TC filter add dev $ETH parent 2: protocol ip prio 1 u32 match ip dst 10.10.11.1 action skbedit queue_mapping 0",
+            "$TC filter add dev $ETH parent 3: protocol ip prio 1 u32 match ip dst 10.10.11.1 classid 3:1 action ok"
+        ],
+        "cmdUnderTest": "ping -c 1 10.10.11.1 -W0.01 -I$ETH || true",
+        "expExitCode": "0",
+        "verifyCmd": "$TC -j -s qdisc ls dev $ETH handle 3:",
+        "matchJSON": [
+            {
+                "kind": "dualpi2",
+                "handle": "3:",
+                "bytes": 98,
+                "packets": 1,
+                "backlog": 0,
+                "qlen": 0
+            }
+        ],
+        "teardown": [
+            "$TC qdisc del dev $ETH handle 1: root",
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    },
+    {
+        "id": "476f",
+        "name": "Force taprio to dequeue from its child's gso_skb with qfq leaf",
+        "category": [
+            "qdisc",
+            "tbf",
+            "multiq",
+            "qfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1 4\" > /sys/bus/netdevsim/new_device",
+            "$IP link set dev $ETH up || true",
+            "$IP l set addr 01:02:03:04:05:06 dev $ETH || true",
+            "$IP n add dev $ETH 10.10.11.1 lladdr 01:02:03:04:05:06 dev $ETH || true",
+            "$TC qdisc add dev $ETH root handle 1: taprio num_tc 2 map 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 queues 1@0 1@1 base-time 9000000000000000000 sched-entry S 03 200000 flags 0x0 clockid CLOCK_TAI",
+            "$TC qdisc add dev $ETH parent 1:1 handle 3: qfq",
+            "$TC class add dev $ETH classid 3:1 parent 3: qfq maxpkt 512 weight 1",
+            "$TC filter add dev $ETH parent 3: protocol all prio 1 matchall classid 3:1 action ok"
+        ],
+        "cmdUnderTest": "ping -c 1 10.10.11.1 -W0.01 -I$ETH || true",
+        "expExitCode": "0",
+        "verifyCmd": "$TC -s -j qdisc ls dev $ETH",
+        "matchJSON": [
+            {
+                "kind": "taprio",
+                "handle": "1:",
+                "bytes": 98,
+                "packets": 1,
+                "backlog": 0,
+                "qlen": 0
+            }
+        ],
+        "teardown": [
+            "$TC qdisc del dev $ETH handle 1: root",
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    },
+    {
+        "id": "0235",
+        "name": "Force taprio to dequeue from its child's gso_skb with dualpi2 leaf",
+        "category": [
+            "qdisc",
+            "tbf",
+            "taprio",
+            "dualpi2"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1 4\" > /sys/bus/netdevsim/new_device",
+            "$IP link set dev $ETH up || true",
+            "$IP l set addr 01:02:03:04:05:06 dev $ETH || true",
+            "$IP n add dev $ETH 10.10.11.1 lladdr 01:02:03:04:05:06 dev $ETH || true",
+            "$TC qdisc add dev $ETH root handle 1: taprio num_tc 2 map 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 queues 1@0 1@1 base-time 9000000000000000000 sched-entry S 03 200000 flags 0x0 clockid CLOCK_TAI",
+            "$TC qdisc replace dev $ETH parent 1:1 handle 3: dualpi2",
+            "$TC filter add dev $ETH parent 3: protocol ip prio 1 u32 match ip dst 10.10.11.1 classid 3:1 action ok"
+        ],
+        "cmdUnderTest": "ping -c 1 10.10.11.1 -W0.01 -I$ETH || true",
+        "expExitCode": "0",
+        "verifyCmd": "$TC -j -s qdisc ls dev $ETH handle 3:",
+        "matchJSON": [
+            {
+                "kind": "dualpi2",
+                "handle": "3:",
+                "bytes": 98,
+                "packets": 1,
+                "backlog": 0,
+                "qlen": 0
+            }
+        ],
+        "teardown": [
+            "$TC qdisc del dev $ETH handle 1: root",
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
     }
 ]
diff --git a/tools/testing/selftests/timers/Makefile b/tools/testing/selftests/timers/Makefile
index 32203593c62e1..0e73a16874c4c 100644
--- a/tools/testing/selftests/timers/Makefile
+++ b/tools/testing/selftests/timers/Makefile
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-CFLAGS += -O3 -Wl,-no-as-needed -Wall -I $(top_srcdir)
+CFLAGS += -O3 -Wl,-no-as-needed -Wall
 LDLIBS += -lrt -lpthread -lm
 
 # these are all "safe" tests that don't modify
diff --git a/tools/testing/selftests/timers/adjtick.c b/tools/testing/selftests/timers/adjtick.c
index 5b3ef708d6e9f..22d274d5520ff 100644
--- a/tools/testing/selftests/timers/adjtick.c
+++ b/tools/testing/selftests/timers/adjtick.c
@@ -22,10 +22,11 @@
 #include <sys/time.h>
 #include <sys/timex.h>
 #include <time.h>
-#include <include/vdso/time64.h>
-
 #include "kselftest.h"
 
+#define NSEC_PER_SEC		1000000000LL
+#define USEC_PER_SEC		1000000LL
+
 #define MILLION			1000000
 
 long systick;
diff --git a/tools/testing/selftests/timers/alarmtimer-suspend.c b/tools/testing/selftests/timers/alarmtimer-suspend.c
index aa66c805f6a42..d55d5b0377c67 100644
--- a/tools/testing/selftests/timers/alarmtimer-suspend.c
+++ b/tools/testing/selftests/timers/alarmtimer-suspend.c
@@ -28,10 +28,11 @@
 #include <signal.h>
 #include <stdlib.h>
 #include <pthread.h>
-#include <include/vdso/time64.h>
 #include <errno.h>
 #include "kselftest.h"
 
+#define NSEC_PER_SEC 1000000000LL
+
 #define UNREASONABLE_LAT (NSEC_PER_SEC * 5) /* hopefully we resume in 5 secs */
 
 #define SUSPEND_SECS 15
diff --git a/tools/testing/selftests/timers/inconsistency-check.c b/tools/testing/selftests/timers/inconsistency-check.c
index e53e63e186835..9ab7066b4e324 100644
--- a/tools/testing/selftests/timers/inconsistency-check.c
+++ b/tools/testing/selftests/timers/inconsistency-check.c
@@ -28,9 +28,10 @@
 #include <sys/timex.h>
 #include <string.h>
 #include <signal.h>
-#include <include/vdso/time64.h>
 #include "kselftest.h"
 
+#define NSEC_PER_SEC 1000000000LL
+
 /* CLOCK_HWSPECIFIC == CLOCK_SGI_CYCLE (Deprecated) */
 #define CLOCK_HWSPECIFIC		10
 
diff --git a/tools/testing/selftests/timers/leap-a-day.c b/tools/testing/selftests/timers/leap-a-day.c
index 3568cfb3e8157..71b49734e08cd 100644
--- a/tools/testing/selftests/timers/leap-a-day.c
+++ b/tools/testing/selftests/timers/leap-a-day.c
@@ -9,16 +9,19 @@
  *  kernel's leap-second behavior, as well as how well applications
  *  handle the leap-second discontinuity.
  *
- *  Usage: leap-a-day [-s] [-i <num>]
+ *  Usage: leap-a-day [-w] [-i <num>] [-t]
  *
  *  Options:
- *	-s:	Each iteration, set the date to 10 seconds before midnight GMT.
- *		This speeds up the number of leapsecond transitions tested,
- *		but because it calls settimeofday frequently, advancing the
- *		time by 24 hours every ~16 seconds, it may cause application
- *		disruption.
+ *	-w:	Only set the leap-second flag and wait for the leap second
+ *		each iteration, instead of advancing the time. By default the
+ *		date is set to 10 seconds before midnight GMT, which speeds up
+ *		the number of leapsecond transitions tested, but because it
+ *		calls settimeofday frequently, advancing the time by 24 hours
+ *		every ~16 seconds, it may cause application disruption.
  *
- *	-i:	Number of iterations to run (default: infinite)
+ *	-i:	Number of iterations to run (-1 = infinite, default: 10)
+ *
+ *	-t:	Print TAI time.
  *
  *  Other notes: Disabling NTP prior to running this is advised, as the two
  *		 may conflict in their commands to the kernel.
@@ -48,9 +51,10 @@
 #include <string.h>
 #include <signal.h>
 #include <unistd.h>
-#include <include/vdso/time64.h>
 #include "kselftest.h"
 
+#define NSEC_PER_SEC 1000000000LL
+
 #define CLOCK_TAI 11
 
 time_t next_leap;
@@ -186,7 +190,7 @@ int main(int argc, char **argv)
 	int opt;
 
 	/* Process arguments */
-	while ((opt = getopt(argc, argv, "sti:")) != -1) {
+	while ((opt = getopt(argc, argv, "wti:")) != -1) {
 		switch (opt) {
 		case 'w':
 			printf("Only setting leap-flag, not changing time. It could take up to a day for leap to trigger.\n");
diff --git a/tools/testing/selftests/timers/mqueue-lat.c b/tools/testing/selftests/timers/mqueue-lat.c
index c0d9368e4fcae..ce8700c95cac5 100644
--- a/tools/testing/selftests/timers/mqueue-lat.c
+++ b/tools/testing/selftests/timers/mqueue-lat.c
@@ -29,9 +29,10 @@
 #include <signal.h>
 #include <errno.h>
 #include <mqueue.h>
-#include <include/vdso/time64.h>
 #include "kselftest.h"
 
+#define NSEC_PER_SEC 1000000000LL
+
 
 #define TARGET_TIMEOUT		100000000	/* 100ms in nanoseconds */
 #define UNRESONABLE_LATENCY	40000000	/* 40ms in nanosecs */
diff --git a/tools/testing/selftests/timers/nanosleep.c b/tools/testing/selftests/timers/nanosleep.c
index a054680b3372e..ceff18149a482 100644
--- a/tools/testing/selftests/timers/nanosleep.c
+++ b/tools/testing/selftests/timers/nanosleep.c
@@ -27,9 +27,10 @@
 #include <sys/timex.h>
 #include <string.h>
 #include <signal.h>
-#include <include/vdso/time64.h>
 #include "kselftest.h"
 
+#define NSEC_PER_SEC 1000000000LL
+
 /* CLOCK_HWSPECIFIC == CLOCK_SGI_CYCLE (Deprecated) */
 #define CLOCK_HWSPECIFIC		10
 
diff --git a/tools/testing/selftests/timers/nsleep-lat.c b/tools/testing/selftests/timers/nsleep-lat.c
index a7ba1eb1e21b6..15f6493ca7b56 100644
--- a/tools/testing/selftests/timers/nsleep-lat.c
+++ b/tools/testing/selftests/timers/nsleep-lat.c
@@ -24,9 +24,10 @@
 #include <sys/timex.h>
 #include <string.h>
 #include <signal.h>
-#include <include/vdso/time64.h>
 #include "kselftest.h"
 
+#define NSEC_PER_SEC 1000000000LL
+
 #define UNRESONABLE_LATENCY 40000000 /* 40ms in nanosecs */
 
 /* CLOCK_HWSPECIFIC == CLOCK_SGI_CYCLE (Deprecated) */
diff --git a/tools/testing/selftests/timers/posix_timers.c b/tools/testing/selftests/timers/posix_timers.c
index 2f3bac9fc6e87..fa168d802f9e1 100644
--- a/tools/testing/selftests/timers/posix_timers.c
+++ b/tools/testing/selftests/timers/posix_timers.c
@@ -16,12 +16,14 @@
 #include <string.h>
 #include <unistd.h>
 #include <time.h>
-#include <include/vdso/time64.h>
 #include <pthread.h>
 #include <stdbool.h>
 
 #include "kselftest.h"
 
+#define NSEC_PER_SEC 1000000000LL
+#define USEC_PER_SEC 1000000LL
+
 #define DELAY 2
 
 static void __fatal_error(const char *test, const char *name, const char *what)
diff --git a/tools/testing/selftests/timers/raw_skew.c b/tools/testing/selftests/timers/raw_skew.c
index a7bae7d809161..2dd16cb4cdd0e 100644
--- a/tools/testing/selftests/timers/raw_skew.c
+++ b/tools/testing/selftests/timers/raw_skew.c
@@ -25,9 +25,10 @@
 #include <sys/time.h>
 #include <sys/timex.h>
 #include <time.h>
-#include <include/vdso/time64.h>
 #include "kselftest.h"
 
+#define NSEC_PER_SEC 1000000000LL
+
 #define shift_right(x, s) ({		\
 	__typeof__(x) __x = (x);	\
 	__typeof__(s) __s = (s);	\
diff --git a/tools/testing/selftests/timers/set-2038.c b/tools/testing/selftests/timers/set-2038.c
index ecc171de47281..c1235638406d1 100644
--- a/tools/testing/selftests/timers/set-2038.c
+++ b/tools/testing/selftests/timers/set-2038.c
@@ -27,9 +27,10 @@
 #include <unistd.h>
 #include <time.h>
 #include <sys/time.h>
-#include <include/vdso/time64.h>
 #include "kselftest.h"
 
+#define NSEC_PER_SEC 1000000000LL
+
 #define KTIME_MAX	((long long)~((unsigned long long)1 << 63))
 #define KTIME_SEC_MAX	(KTIME_MAX / NSEC_PER_SEC)
 
diff --git a/tools/testing/selftests/timers/set-timer-lat.c b/tools/testing/selftests/timers/set-timer-lat.c
index 44d2e3614fa56..e092c18befff5 100644
--- a/tools/testing/selftests/timers/set-timer-lat.c
+++ b/tools/testing/selftests/timers/set-timer-lat.c
@@ -28,9 +28,10 @@
 #include <signal.h>
 #include <stdlib.h>
 #include <pthread.h>
-#include <include/vdso/time64.h>
 #include "kselftest.h"
 
+#define NSEC_PER_SEC 1000000000LL
+
 /* CLOCK_HWSPECIFIC == CLOCK_SGI_CYCLE (Deprecated) */
 #define CLOCK_HWSPECIFIC		10
 
diff --git a/tools/testing/selftests/timers/valid-adjtimex.c b/tools/testing/selftests/timers/valid-adjtimex.c
index e1e56d3097d6b..dc2559eb11a51 100644
--- a/tools/testing/selftests/timers/valid-adjtimex.c
+++ b/tools/testing/selftests/timers/valid-adjtimex.c
@@ -29,9 +29,11 @@
 #include <string.h>
 #include <signal.h>
 #include <unistd.h>
-#include <include/vdso/time64.h>
 #include "kselftest.h"
 
+#define NSEC_PER_SEC 1000000000LL
+#define USEC_PER_SEC 1000000LL
+
 #define ADJ_SETOFFSET 0x0100
 
 #include <sys/syscall.h>
diff --git a/tools/testing/shared/linux/kmemleak.h b/tools/testing/shared/linux/kmemleak.h
index 155f112786c4f..65a383090fb51 100644
--- a/tools/testing/shared/linux/kmemleak.h
+++ b/tools/testing/shared/linux/kmemleak.h
@@ -1 +1,2 @@
 static inline void kmemleak_update_trace(const void *ptr) { }
+static inline void kmemleak_transient_leak(const void *ptr) { }
diff --git a/tools/testing/vma/Makefile b/tools/testing/vma/Makefile
index e72b45dedda54..ef6cc558afe10 100644
--- a/tools/testing/vma/Makefile
+++ b/tools/testing/vma/Makefile
@@ -10,7 +10,7 @@ OFILES = $(SHARED_OFILES) main.o shared.o maple-shim.o
 TARGETS = vma
 
 # These can be varied to test different sizes.
-CFLAGS += -DNUM_VMA_FLAG_BITS=128 -DNUM_MM_FLAG_BITS=128
+CFLAGS += -DNUM_VMA_FLAG_BITS=64 -DNUM_MM_FLAG_BITS=64
 
 main.o: main.c shared.c shared.h vma_internal.h tests/merge.c tests/mmap.c tests/vma.c ../../../mm/vma.c ../../../mm/vma_init.c ../../../mm/vma_exec.c ../../../mm/vma.h include/custom.h include/dup.h include/stubs.h
 
diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h
index bf26b3f48d3ab..cdeb53bbdd1b1 100644
--- a/tools/testing/vma/include/dup.h
+++ b/tools/testing/vma/include/dup.h
@@ -245,8 +245,10 @@ enum {
 #define VM_STACK	INIT_VM_FLAG(STACK)
 #ifdef CONFIG_STACK_GROWS_UP
 #define VM_STACK_EARLY	INIT_VM_FLAG(STACK_EARLY)
+#define VMA_STACK_EARLY mk_vma_flags(VMA_STACK_EARLY_BIT)
 #else
 #define VM_STACK_EARLY	VM_NONE
+#define VMA_STACK_EARLY EMPTY_VMA_FLAGS
 #endif
 #ifdef CONFIG_ARCH_HAS_PKEYS
 #define VM_PKEY_SHIFT ((__force int)VMA_HIGH_ARCH_0_BIT)
@@ -315,6 +317,8 @@ enum {
 
 /* Bits set in the VMA until the stack is in its final location */
 #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY)
+#define VMA_STACK_INCOMPLETE_SETUP append_vma_flags(		\
+	VMA_STACK_EARLY, VMA_RAND_READ_BIT, VMA_SEQ_READ_BIT)
 
 #define TASK_EXEC_BIT ((current->personality & READ_IMPLIES_EXEC) ? \
 		       VM_EXEC_BIT : VM_READ_BIT)
@@ -1163,6 +1167,11 @@ static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi)
 	return mas_find(&vmi->mas, ULONG_MAX);
 }
 
+static inline bool vma_is_attached(struct vm_area_struct *vma)
+{
+	return refcount_read(&vma->vm_refcnt);
+}
+
 /*
  * WARNING: to avoid racing with vma_mark_attached()/vma_mark_detached(), these
  * assertions should be made either under mmap_write_lock or when the object
@@ -1170,12 +1179,12 @@ static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi)
  */
 static inline void vma_assert_attached(struct vm_area_struct *vma)
 {
-	WARN_ON_ONCE(!refcount_read(&vma->vm_refcnt));
+	WARN_ON_ONCE(!vma_is_attached(vma));
 }
 
 static inline void vma_assert_detached(struct vm_area_struct *vma)
 {
-	WARN_ON_ONCE(refcount_read(&vma->vm_refcnt));
+	WARN_ON_ONCE(vma_is_attached(vma));
 }
 
 static inline void vma_assert_write_locked(struct vm_area_struct *);
@@ -1301,6 +1310,16 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma)
 	return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
 }
 
+static inline pgoff_t vma_start_pgoff(const struct vm_area_struct *vma)
+{
+	return vma->vm_pgoff;
+}
+
+static inline pgoff_t vma_end_pgoff(const struct vm_area_struct *vma)
+{
+	return vma_start_pgoff(vma) + vma_pages(vma);
+}
+
 static inline int vfs_mmap_prepare(struct file *file, struct vm_area_desc *desc)
 {
 	return file->f_op->mmap_prepare(desc);
@@ -1532,9 +1551,36 @@ static inline int get_sysctl_max_map_count(void)
 #define pgtable_supports_soft_dirty()	IS_ENABLED(CONFIG_MEM_SOFT_DIRTY)
 #endif
 
-static inline pgprot_t vma_get_page_prot(vma_flags_t vma_flags)
+static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags)
 {
 	const vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags);
 
 	return vm_get_page_prot(vm_flags);
 }
+
+static inline pgoff_t linear_page_delta(const struct vm_area_struct *vma,
+					const unsigned long address)
+{
+	return (address - vma->vm_start) >> PAGE_SHIFT;
+}
+
+static inline pgoff_t linear_page_index(const struct vm_area_struct *vma,
+					const unsigned long address)
+{
+	pgoff_t pgoff;
+
+	pgoff = linear_page_delta(vma, address);
+	pgoff += vma_start_pgoff(vma);
+	return pgoff;
+}
+
+static inline void vma_assert_can_modify(struct vm_area_struct *vma)
+{
+	if (vma_is_attached(vma))
+		vma_assert_write_locked(vma);
+}
+
+static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma)
+{
+	return vma_flags_to_page_prot(vma->flags);
+}
diff --git a/tools/testing/vma/include/stubs.h b/tools/testing/vma/include/stubs.h
index 64164e25658fa..d6136e19a8af3 100644
--- a/tools/testing/vma/include/stubs.h
+++ b/tools/testing/vma/include/stubs.h
@@ -257,13 +257,13 @@ static inline void vm_acct_memory(long pages)
 {
 }
 
-static inline void vma_interval_tree_insert(struct vm_area_struct *vma,
-					    struct rb_root_cached *rb)
+static inline void mapping_rmap_tree_insert(struct vm_area_struct *vma,
+					    struct address_space *mapping)
 {
 }
 
-static inline void vma_interval_tree_remove(struct vm_area_struct *vma,
-					    struct rb_root_cached *rb)
+static inline void mapping_rmap_tree_remove(struct vm_area_struct *vma,
+					    struct address_space *mapping)
 {
 }
 
@@ -271,13 +271,13 @@ static inline void flush_dcache_mmap_unlock(struct address_space *mapping)
 {
 }
 
-static inline void anon_vma_interval_tree_insert(struct anon_vma_chain *avc,
-						 struct rb_root_cached *rb)
+static inline void anon_rmap_tree_insert(struct anon_vma_chain *avc,
+					 struct anon_vma *anon_vma)
 {
 }
 
-static inline void anon_vma_interval_tree_remove(struct anon_vma_chain *avc,
-						 struct rb_root_cached *rb)
+static inline void anon_rmap_tree_remove(struct anon_vma_chain *avc,
+					 struct anon_vma *anon_vma)
 {
 }
 
diff --git a/tools/testing/vma/shared.c b/tools/testing/vma/shared.c
index 2565a5aecb80c..bea9ea6db02a4 100644
--- a/tools/testing/vma/shared.c
+++ b/tools/testing/vma/shared.c
@@ -120,12 +120,3 @@ unsigned long rlimit(unsigned int limit)
 {
 	return (unsigned long)-1;
 }
-
-void vma_set_range(struct vm_area_struct *vma,
-		   unsigned long start, unsigned long end,
-		   pgoff_t pgoff)
-{
-	vma->vm_start = start;
-	vma->vm_end = end;
-	vma->vm_pgoff = pgoff;
-}
diff --git a/tools/testing/vma/shared.h b/tools/testing/vma/shared.h
index 8b9e3b11c3cbf..97cd7a679dc19 100644
--- a/tools/testing/vma/shared.h
+++ b/tools/testing/vma/shared.h
@@ -21,19 +21,35 @@
 		}							\
 	} while (0)
 
-#define ASSERT_TRUE(_expr)						\
-	do {								\
-		if (!(_expr)) {						\
-			fprintf(stderr,					\
-				"Assert FAILED at %s:%d:%s(): %s is FALSE.\n", \
-				__FILE__, __LINE__, __FUNCTION__, #_expr); \
-			return false;					\
-		}							\
+#define __ASSERT_TRUE(_expr, _fmt, ...)					   \
+	do {								   \
+		if (!(_expr)) {						   \
+			fprintf(stderr,					   \
+				"Assert FAILED at %s:%d:%s(): %s is FALSE" \
+				_fmt ".\n",				   \
+				__FILE__, __LINE__, __FUNCTION__, #_expr   \
+				__VA_OPT__(,) __VA_ARGS__);		   \
+			return false;					   \
+		}							   \
 	} while (0)
 
+#define __TO_SCALAR(x)	((unsigned long long)(uintptr_t)(x))
+
+#define ASSERT_TRUE(_expr) __ASSERT_TRUE(_expr, "")
 #define ASSERT_FALSE(_expr) ASSERT_TRUE(!(_expr))
-#define ASSERT_EQ(_val1, _val2) ASSERT_TRUE((_val1) == (_val2))
-#define ASSERT_NE(_val1, _val2) ASSERT_TRUE((_val1) != (_val2))
+#define ASSERT_EQ(_val1, _val2) do {				 \
+	__typeof__(_val1) __val1 = (_val1);			 \
+	__typeof__(_val2) __val2 = (_val2);			 \
+	__ASSERT_TRUE(__val1 == __val2, " (0x%llx != 0x%llx)",	 \
+		      __TO_SCALAR(__val1), __TO_SCALAR(__val2)); \
+	} while (0)
+
+#define ASSERT_NE(_val1, _val2) do {				 \
+	__typeof__(_val1) __val1 = (_val1);			 \
+	__typeof__(_val2) __val2 = (_val2);			 \
+	__ASSERT_TRUE(__val1 != __val2, " (0x%llx == 0x%llx)",	 \
+		      __TO_SCALAR(__val1), __TO_SCALAR(__val2)); \
+	} while (0)
 
 #define ASSERT_FLAGS_SAME_MASK(_flags, _flags_other) \
 	ASSERT_TRUE(vma_flags_same_mask((_flags), (_flags_other)))
@@ -53,8 +69,6 @@
 #define ASSERT_FLAGS_NONEMPTY(_flags) \
 	ASSERT_FALSE(vma_flags_empty(_flags))
 
-#define IS_SET(_val, _flags) ((_val & _flags) == _flags)
-
 extern bool fail_prealloc;
 
 /* Override vma_iter_prealloc() so we can choose to fail it. */
@@ -125,8 +139,3 @@ void __vma_set_dummy_anon_vma(struct vm_area_struct *vma,
 /* Provide a simple dummy VMA/anon_vma dummy setup for testing. */
 void vma_set_dummy_anon_vma(struct vm_area_struct *vma,
 			    struct anon_vma_chain *avc);
-
-/* Helper function to specify a VMA's range. */
-void vma_set_range(struct vm_area_struct *vma,
-		   unsigned long start, unsigned long end,
-		   pgoff_t pgoff);
diff --git a/tools/testing/vma/tests/merge.c b/tools/testing/vma/tests/merge.c
index 03b6f9820e0ac..e357accc84990 100644
--- a/tools/testing/vma/tests/merge.c
+++ b/tools/testing/vma/tests/merge.c
@@ -118,7 +118,7 @@ static bool test_simple_merge(void)
 
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0x3000);
-	ASSERT_EQ(vma->vm_pgoff, 0);
+	ASSERT_EQ(vma_start_pgoff(vma), 0);
 	ASSERT_FLAGS_SAME_MASK(&vma->flags, vma_flags);
 
 	detach_free_vma(vma);
@@ -150,7 +150,7 @@ static bool test_simple_modify(void)
 
 	ASSERT_EQ(vma->vm_start, 0x1000);
 	ASSERT_EQ(vma->vm_end, 0x2000);
-	ASSERT_EQ(vma->vm_pgoff, 1);
+	ASSERT_EQ(vma_start_pgoff(vma), 1);
 
 	/*
 	 * Now walk through the three split VMAs and make sure they are as
@@ -162,7 +162,7 @@ static bool test_simple_modify(void)
 
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0x1000);
-	ASSERT_EQ(vma->vm_pgoff, 0);
+	ASSERT_EQ(vma_start_pgoff(vma), 0);
 
 	detach_free_vma(vma);
 	vma_iter_clear(&vmi);
@@ -171,7 +171,7 @@ static bool test_simple_modify(void)
 
 	ASSERT_EQ(vma->vm_start, 0x1000);
 	ASSERT_EQ(vma->vm_end, 0x2000);
-	ASSERT_EQ(vma->vm_pgoff, 1);
+	ASSERT_EQ(vma_start_pgoff(vma), 1);
 
 	detach_free_vma(vma);
 	vma_iter_clear(&vmi);
@@ -180,7 +180,7 @@ static bool test_simple_modify(void)
 
 	ASSERT_EQ(vma->vm_start, 0x2000);
 	ASSERT_EQ(vma->vm_end, 0x3000);
-	ASSERT_EQ(vma->vm_pgoff, 2);
+	ASSERT_EQ(vma_start_pgoff(vma), 2);
 
 	detach_free_vma(vma);
 	mtree_destroy(&mm.mm_mt);
@@ -209,7 +209,7 @@ static bool test_simple_expand(void)
 
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0x3000);
-	ASSERT_EQ(vma->vm_pgoff, 0);
+	ASSERT_EQ(vma_start_pgoff(vma), 0);
 
 	detach_free_vma(vma);
 	mtree_destroy(&mm.mm_mt);
@@ -227,11 +227,11 @@ static bool test_simple_shrink(void)
 
 	ASSERT_FALSE(attach_vma(&mm, vma));
 
-	ASSERT_FALSE(vma_shrink(&vmi, vma, 0, 0x1000, 0));
+	ASSERT_FALSE(vma_shrink(&vmi, vma, 0x1000));
 
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0x1000);
-	ASSERT_EQ(vma->vm_pgoff, 0);
+	ASSERT_EQ(vma_start_pgoff(vma), 0);
 
 	detach_free_vma(vma);
 	mtree_destroy(&mm.mm_mt);
@@ -324,7 +324,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky,
 	ASSERT_TRUE(merged);
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0x4000);
-	ASSERT_EQ(vma->vm_pgoff, 0);
+	ASSERT_EQ(vma_start_pgoff(vma), 0);
 	ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(mm.map_count, 3);
@@ -343,7 +343,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky,
 	ASSERT_TRUE(merged);
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0x5000);
-	ASSERT_EQ(vma->vm_pgoff, 0);
+	ASSERT_EQ(vma_start_pgoff(vma), 0);
 	ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(mm.map_count, 3);
@@ -364,7 +364,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky,
 	ASSERT_TRUE(merged);
 	ASSERT_EQ(vma->vm_start, 0x6000);
 	ASSERT_EQ(vma->vm_end, 0x9000);
-	ASSERT_EQ(vma->vm_pgoff, 6);
+	ASSERT_EQ(vma_start_pgoff(vma), 6);
 	ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(mm.map_count, 3);
@@ -384,7 +384,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky,
 	ASSERT_TRUE(merged);
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0x9000);
-	ASSERT_EQ(vma->vm_pgoff, 0);
+	ASSERT_EQ(vma_start_pgoff(vma), 0);
 	ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(mm.map_count, 2);
@@ -404,7 +404,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky,
 	ASSERT_TRUE(merged);
 	ASSERT_EQ(vma->vm_start, 0xa000);
 	ASSERT_EQ(vma->vm_end, 0xc000);
-	ASSERT_EQ(vma->vm_pgoff, 0xa);
+	ASSERT_EQ(vma_start_pgoff(vma), 0xa);
 	ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(mm.map_count, 2);
@@ -423,7 +423,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky,
 	ASSERT_TRUE(merged);
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0xc000);
-	ASSERT_EQ(vma->vm_pgoff, 0);
+	ASSERT_EQ(vma_start_pgoff(vma), 0);
 	ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(mm.map_count, 1);
@@ -443,7 +443,7 @@ static bool __test_merge_new(bool is_sticky, bool a_is_sticky, bool b_is_sticky,
 		ASSERT_NE(vma, NULL);
 		ASSERT_EQ(vma->vm_start, 0);
 		ASSERT_EQ(vma->vm_end, 0xc000);
-		ASSERT_EQ(vma->vm_pgoff, 0);
+		ASSERT_EQ(vma_start_pgoff(vma), 0);
 		ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
 
 		detach_free_vma(vma);
@@ -805,7 +805,7 @@ static bool test_vma_merge_new_with_close(void)
 	ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS);
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0x5000);
-	ASSERT_EQ(vma->vm_pgoff, 0);
+	ASSERT_EQ(vma_start_pgoff(vma), 0);
 	ASSERT_EQ(vma->vm_ops, &vm_ops);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(mm.map_count, 2);
@@ -865,7 +865,7 @@ static bool __test_merge_existing(bool prev_is_sticky, bool middle_is_sticky, bo
 	ASSERT_EQ(vma_next->anon_vma, &dummy_anon_vma);
 	ASSERT_EQ(vma->vm_start, 0x2000);
 	ASSERT_EQ(vma->vm_end, 0x3000);
-	ASSERT_EQ(vma->vm_pgoff, 2);
+	ASSERT_EQ(vma_start_pgoff(vma), 2);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_TRUE(vma_write_started(vma_next));
 	ASSERT_EQ(mm.map_count, 2);
@@ -931,7 +931,7 @@ static bool __test_merge_existing(bool prev_is_sticky, bool middle_is_sticky, bo
 	ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma);
 	ASSERT_EQ(vma->vm_start, 0x6000);
 	ASSERT_EQ(vma->vm_end, 0x7000);
-	ASSERT_EQ(vma->vm_pgoff, 6);
+	ASSERT_EQ(vma_start_pgoff(vma), 6);
 	ASSERT_TRUE(vma_write_started(vma_prev));
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(mm.map_count, 2);
@@ -1416,7 +1416,7 @@ static bool test_merge_extend(void)
 	ASSERT_EQ(vma_merge_extend(&vmi, vma, 0x2000), vma);
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0x4000);
-	ASSERT_EQ(vma->vm_pgoff, 0);
+	ASSERT_EQ(vma_start_pgoff(vma), 0);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(mm.map_count, 1);
 
@@ -1456,7 +1456,7 @@ static bool test_expand_only_mode(void)
 	ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS);
 	ASSERT_EQ(vma->vm_start, 0x3000);
 	ASSERT_EQ(vma->vm_end, 0x9000);
-	ASSERT_EQ(vma->vm_pgoff, 3);
+	ASSERT_EQ(vma_start_pgoff(vma), 3);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(vma_iter_addr(&vmi), 0x3000);
 	vma_assert_attached(vma);
diff --git a/tools/testing/vma/vma_internal.h b/tools/testing/vma/vma_internal.h
index e12ab2c80f950..4f6c5666ac07d 100644
--- a/tools/testing/vma/vma_internal.h
+++ b/tools/testing/vma/vma_internal.h
@@ -14,8 +14,8 @@
 
 #include <stdlib.h>
 
-#define CONFIG_MMU
-#define CONFIG_PER_VMA_LOCK
+#define CONFIG_MMU		1
+#define CONFIG_PER_VMA_LOCK	1
 
 #ifdef __CONCAT
 #undef __CONCAT
diff --git a/tools/tracing/rtla/Makefile b/tools/tracing/rtla/Makefile
index 60a1025389881..387bc6cc18f02 100644
--- a/tools/tracing/rtla/Makefile
+++ b/tools/tracing/rtla/Makefile
@@ -45,6 +45,9 @@ else
   LIB_OUTPUT = $(CURDIR)/lib
 endif
 
+LIB_CTYPE = $(LIB_OUTPUT)/ctype.o
+LIB_CTYPE_SRC = $(srctree)/tools/lib/ctype.c
+
 LIB_STRING = $(LIB_OUTPUT)/string.o
 LIB_STRING_SRC = $(srctree)/tools/lib/string.c
 
@@ -117,12 +120,12 @@ tests/bpf/bpf_action_map.o: tests/bpf/bpf_action_map.c
 	$(Q)echo "BPF skeleton support is disabled, skipping tests/bpf/bpf_action_map.o"
 endif
 
-$(RTLA): $(RTLA_IN) $(LIBSUBCMD) $(LIB_STRING) $(LIB_STR_ERROR_R)
-	$(QUIET_LINK)$(CC) $(LDFLAGS) -o $(RTLA) $(RTLA_IN) $(LIBSUBCMD) $(LIB_STRING) $(LIB_STR_ERROR_R) $(EXTLIBS)
+$(RTLA): $(RTLA_IN) $(LIBSUBCMD) $(LIB_CTYPE) $(LIB_STRING) $(LIB_STR_ERROR_R)
+	$(QUIET_LINK)$(CC) $(LDFLAGS) -o $(RTLA) $(RTLA_IN) $(LIBSUBCMD) $(LIB_CTYPE) $(LIB_STRING) $(LIB_STR_ERROR_R) $(EXTLIBS)
 
-static: $(RTLA_IN) $(LIBSUBCMD) $(LIB_STRING) $(LIB_STR_ERROR_R)
+static: $(RTLA_IN) $(LIBSUBCMD) $(LIB_CTYPE) $(LIB_STRING) $(LIB_STR_ERROR_R)
 	$(eval LDFLAGS += -static)
-	$(QUIET_LINK)$(CC) -static $(LDFLAGS) -o $(RTLA)-static $(RTLA_IN) $(LIBSUBCMD) $(LIB_STRING) $(LIB_STR_ERROR_R) $(EXTLIBS)
+	$(QUIET_LINK)$(CC) -static $(LDFLAGS) -o $(RTLA)-static $(RTLA_IN) $(LIBSUBCMD) $(LIB_CTYPE) $(LIB_STRING) $(LIB_STR_ERROR_R) $(EXTLIBS)
 
 rtla.%: fixdep FORCE
 	make -f $(srctree)/tools/build/Makefile.build dir=. $@
@@ -150,6 +153,9 @@ $(LIB_STR_ERROR_R): $(LIB_STR_ERROR_R_SRC) | $(LIB_OUTPUT)
 $(LIB_STRING): $(LIB_STRING_SRC) | $(LIB_OUTPUT)
 	$(QUIET_CC)$(CC) $(CFLAGS) -c -o $@ $<
 
+$(LIB_CTYPE): $(LIB_CTYPE_SRC) | $(LIB_OUTPUT)
+	$(QUIET_CC)$(CC) $(CFLAGS) -c -o $@ $<
+
 libsubcmd-clean:
 	$(call QUIET_CLEAN, libsubcmd)
 	$(Q)$(RM) -r -- $(LIBSUBCMD_OUTPUT)
diff --git a/tools/tracing/rtla/src/common.c b/tools/tracing/rtla/src/common.c
index d0a8a6edbf0cb..8c7f5e75b2ec8 100644
--- a/tools/tracing/rtla/src/common.c
+++ b/tools/tracing/rtla/src/common.c
@@ -5,6 +5,7 @@
 #include <signal.h>
 #include <stdlib.h>
 #include <string.h>
+#include <unistd.h>
 #include <sys/sysinfo.h>
 
 #include "common.h"
diff --git a/tools/workqueue/wq_dump.py b/tools/workqueue/wq_dump.py
index ce4161f52f2f1..a0c72237531f9 100644
--- a/tools/workqueue/wq_dump.py
+++ b/tools/workqueue/wq_dump.py
@@ -46,6 +46,11 @@ each workqueue:
 
 import sys
 
+import argparse
+parser = argparse.ArgumentParser(description=desc,
+                                 formatter_class=argparse.RawTextHelpFormatter)
+args = parser.parse_args()
+
 import drgn
 from drgn.helpers.linux.list import list_for_each_entry,list_empty
 from drgn.helpers.linux.percpu import per_cpu_ptr
@@ -53,11 +58,6 @@ from drgn.helpers.linux.cpumask import for_each_cpu,for_each_possible_cpu
 from drgn.helpers.linux.nodemask import for_each_node
 from drgn.helpers.linux.idr import idr_for_each
 
-import argparse
-parser = argparse.ArgumentParser(description=desc,
-                                 formatter_class=argparse.RawTextHelpFormatter)
-args = parser.parse_args()
-
 def err(s):
     print(s, file=sys.stderr, flush=True)
     sys.exit(1)
diff --git a/tools/workqueue/wq_monitor.py b/tools/workqueue/wq_monitor.py
index 9e964c5be40c9..7f47fa398e3c8 100644
--- a/tools/workqueue/wq_monitor.py
+++ b/tools/workqueue/wq_monitor.py
@@ -37,9 +37,6 @@ import re
 import time
 import json
 
-import drgn
-from drgn.helpers.linux.list import list_for_each_entry
-
 import argparse
 parser = argparse.ArgumentParser(description=desc,
                                  formatter_class=argparse.RawTextHelpFormatter)
@@ -51,6 +48,9 @@ parser.add_argument('-j', '--json', action='store_true',
                     help='Output in json')
 args = parser.parse_args()
 
+import drgn
+from drgn.helpers.linux.list import list_for_each_entry
+
 workqueues              = prog['workqueues']
 
 WQ_UNBOUND              = prog['WQ_UNBOUND']
diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c
index db57c5766ab60..c4dd27be1851c 100644
--- a/virt/kvm/guest_memfd.c
+++ b/virt/kvm/guest_memfd.c
@@ -185,6 +185,10 @@ static void __kvm_gmem_invalidate_start(struct gmem_file *f, pgoff_t start,
 		}
 
 		flush |= kvm_mmu_unmap_gfn_range(kvm, &gfn_range);
+
+#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
+		kvm_arch_gmem_invalidate_range(kvm, &gfn_range);
+#endif
 	}
 
 	if (flush)
@@ -440,7 +444,7 @@ static int kvm_gmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpo
 static struct mempolicy *kvm_gmem_get_policy(struct vm_area_struct *vma,
 					     unsigned long addr, pgoff_t *ilx)
 {
-	pgoff_t pgoff = vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT);
+	pgoff_t pgoff = linear_page_index(vma, addr);
 	struct inode *inode = file_inode(vma->vm_file);
 
 	*ilx = inode->i_ino;
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index e44c20c049610..c40a2aca23034 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -4188,6 +4188,8 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id)
 		goto vcpu_decrement;
 	}
 
+	vcpu->vcpu_idx = -1;
+
 	BUILD_BUG_ON(sizeof(struct kvm_run) > PAGE_SIZE);
 	page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
 	if (!page) {
@@ -4216,6 +4218,11 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id)
 		goto unlock_vcpu_destroy;
 	}
 
+	/*
+	 * Set the vCPU's index *before* the vCPU is reachable by other tasks.
+	 * Unwind the index back to -1 on failure so that KVM can use the index
+	 * to detect that the vCPU is unreachable, e.g. for lockdep asserts.
+	 */
 	vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus);
 	r = xa_insert(&kvm->vcpu_array, vcpu->vcpu_idx, vcpu, GFP_KERNEL_ACCOUNT);
 	WARN_ON_ONCE(r == -EBUSY);
@@ -4254,6 +4261,7 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id)
 	kvm_put_kvm_no_destroy(kvm);
 	xa_erase(&kvm->vcpu_array, vcpu->vcpu_idx);
 unlock_vcpu_destroy:
+	vcpu->vcpu_idx = -1;
 	mutex_unlock(&kvm->lock);
 	kvm_dirty_ring_free(&vcpu->dirty_ring);
 arch_vcpu_destroy:
@@ -6069,25 +6077,19 @@ struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
 					 gpa_t addr)
 {
 	struct kvm_io_bus *bus;
-	int dev_idx, srcu_idx;
-	struct kvm_io_device *iodev = NULL;
+	int dev_idx;
 
-	srcu_idx = srcu_read_lock(&kvm->srcu);
+	lockdep_assert_held(&kvm->srcu);
 
 	bus = kvm_get_bus_srcu(kvm, bus_idx);
 	if (!bus)
-		goto out_unlock;
+		return NULL;
 
 	dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1);
 	if (dev_idx < 0)
-		goto out_unlock;
-
-	iodev = bus->range[dev_idx].dev;
-
-out_unlock:
-	srcu_read_unlock(&kvm->srcu, srcu_idx);
+		return NULL;
 
-	return iodev;
+	return bus->range[dev_idx].dev;
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_io_bus_get_dev);
 
@@ -6559,6 +6561,7 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module)
 err_gmem:
 	kvm_vfio_ops_exit();
 err_vfio:
+	debugfs_remove_recursive(kvm_debugfs_dir);
 	kvm_async_pf_deinit();
 err_async_pf:
 	kvm_irqfd_exit();
